2023-06-03T19:11:04.660 INFO:root:teuthology version: 0.0.1.dev116+g62814ab 2023-06-03T19:11:04.662 DEBUG:teuthology.run:Teuthology command: teuthology --verbose --owner scheduled_teuthology@teuthology --name teuthology-2023-06-03_14:23:09-upgrade:pacific-x-reef-distro-default-smithi --archive /home/teuthworker/archive/teuthology-2023-06-03_14:23:09-upgrade:pacific-x-reef-distro-default-smithi/7295631 --description upgrade:pacific-x/stress-split/{0-distro/ubuntu_20.04 0-roles 1-start 2-first-half-tasks/radosbench 3-stress-tasks/{radosbench rbd-cls rbd-import-export rbd_api readwrite snaps-few-objects} 4-second-half-tasks/rbd-import-export mon_election/classic} -- /home/teuthworker/archive/teuthology-2023-06-03_14:23:09-upgrade:pacific-x-reef-distro-default-smithi/7295631/orig.config.yaml 2023-06-03T19:11:04.735 DEBUG:teuthology.report:Pushing job info to https://paddles.front.sepia.ceph.com/ 2023-06-03T19:11:04.829 INFO:teuthology.run:Config: archive_path: /home/teuthworker/archive/teuthology-2023-06-03_14:23:09-upgrade:pacific-x-reef-distro-default-smithi/7295631 branch: reef description: upgrade:pacific-x/stress-split/{0-distro/ubuntu_20.04 0-roles 1-start 2-first-half-tasks/radosbench 3-stress-tasks/{radosbench rbd-cls rbd-import-export rbd_api readwrite snaps-few-objects} 4-second-half-tasks/rbd-import-export mon_election/classic} email: ceph-qa@ceph.io first-half-sequence: - cephadm.shell: env: - sha1 mon.a: - ceph config set mgr mgr/cephadm/daemon_cache_timeout 60 - ceph config set global log_to_journald false --force - ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 - ceph orch ps - echo wait for minority of mons to upgrade - while ! ceph mon versions | grep $sha1 ; do sleep 2 ; done - ceph orch ps - ceph orch upgrade pause - sleep 60 - ceph orch upgrade resume - echo wait for majority of mons to upgrade - 'while ! ceph mon versions | grep $sha1 | egrep '': [23]'' ; do sleep 2 ; done' - ceph orch ps - ceph orch upgrade pause - sleep 60 - ceph orch upgrade resume - echo wait for all mons to upgrade - 'while ! ceph mon versions | grep $sha1 | grep '': 3'' ; do sleep 2 ; done' - ceph orch ps - ceph orch upgrade pause - sleep 60 - ceph orch upgrade resume - echo wait for half of osds to upgrade - 'while ! ceph osd versions | grep $sha1 | egrep '': [45678]''; do sleep 2 ; done' - ceph orch upgrade pause - ceph orch ps - ceph orch ps - ceph versions first-half-tasks: - full_sequential: - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - print: '**** done end radosbench.yaml' first_in_suite: false job_id: '7295631' kernel: kdb: true sha1: distro last_in_suite: false machine_type: smithi meta: - desc: 'run randomized correctness test for rados operations generate write load with rados bench ' - desc: 'run randomized correctness test for rados operations generate write load with rados bench ' - desc: 'run basic cls tests for rbd ' - desc: 'run basic import/export cli tests for rbd ' - desc: 'librbd C and C++ api tests ' - desc: 'randomized correctness test for rados operations on a replicated pool, using only reads, writes, and deletes ' - desc: 'randomized correctness test for rados operations on a replicated pool with snapshot operations ' - desc: 'run basic import/export cli tests for rbd ' name: teuthology-2023-06-03_14:23:09-upgrade:pacific-x-reef-distro-default-smithi no_nested_subset: false nuke-on-error: true openstack: - volumes: count: 4 size: 10 os_type: ubuntu os_version: '20.04' overrides: admin_socket: branch: reef ceph: conf: global: mon election default strategy: 1 mgr: debug mgr: 20 debug ms: 1 mon: debug mon: 20 debug ms: 1 debug paxos: 20 osd: debug ms: 1 debug osd: 20 osd shutdown pgref assert: true create_rbd_pool: true flavor: default log-ignorelist: - \(MDS_ALL_DOWN\) - \(MDS_UP_LESS_THAN_MAX\) sha1: 8e24bcf462a1d94c2e12aa4e26419634883a1e88 ceph-deploy: conf: client: log file: /var/log/ceph/ceph-$name.$pid.log mon: osd default pool size: 2 install: ceph: flavor: default sha1: 8e24bcf462a1d94c2e12aa4e26419634883a1e88 kernel: hwe: true workunit: branch: reef sha1: dbb22f29210f43ee95e67f3b727f470d3faecd40 owner: scheduled_teuthology@teuthology priority: 100 repo: https://github.com/ceph/ceph.git roles: - - mon.a - mon.c - mgr.y - osd.0 - osd.1 - osd.2 - osd.3 - client.0 - node-exporter.a - alertmanager.a - - mon.b - mgr.x - osd.4 - osd.5 - osd.6 - osd.7 - client.1 - prometheus.a - grafana.a - node-exporter.b second-half-sequence: sequential: - cephadm.shell: env: - sha1 mon.a: - ceph orch upgrade resume - sleep 60 - echo wait for upgrade to complete - while ceph orch upgrade status | jq '.in_progress' | grep true ; do ceph orch ps ; ceph versions ; sleep 30 ; done - echo upgrade complete - ceph orch ps - ceph versions - ceph versions | jq -e '.overall | length == 1' - ceph versions | jq -e '.overall | keys' | grep $sha1 second-half-tasks: - workunit: branch: pacific clients: client.0: - rbd/import_export.sh env: RBD_CREATE_ARGS: --new-format - print: '**** done rbd/import_export.sh 5-workload' seed: 6510 sha1: 8e24bcf462a1d94c2e12aa4e26419634883a1e88 sleep_before_teardown: 0 stress-tasks: - thrashosds: aggressive_pg_num_changes: false chance_force_recovery: 0 chance_pgnum_grow: 1 chance_pgpnum_fix: 1 chance_thrash_cluster_full: 0 chance_thrash_pg_upmap: 0 chance_thrash_pg_upmap_items: 0 disable_objectstore_tool_tests: true timeout: 1200 - full_sequential: - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - print: '**** done end radosbench.yaml' - workunit: branch: pacific clients: client.0: - cls/test_cls_rbd.sh - print: '**** done cls/test_cls_rbd.sh 5-workload' - workunit: branch: pacific clients: client.0: - rbd/import_export.sh env: RBD_CREATE_ARGS: --new-format - print: '**** done rbd/import_export.sh 5-workload' - workunit: branch: pacific clients: client.0: - rbd/test_librbd.sh - print: '**** done rbd/test_librbd.sh 7-workload' - full_sequential: - rados: clients: - client.0 objects: 500 op_weights: delete: 10 read: 45 write: 45 ops: 4000 write_append_excl: false - print: '**** done rados/readwrite 5-workload' - full_sequential: - rados: clients: - client.0 objects: 50 op_weights: delete: 50 read: 100 rollback: 50 snap_create: 50 snap_remove: 50 write: 100 ops: 4000 write_append_excl: false - print: '**** done rados/snaps-few-objects 5-workload' suite: upgrade:pacific-x suite_branch: reef suite_path: /home/teuthworker/src/github.com_ceph_ceph_dbb22f29210f43ee95e67f3b727f470d3faecd40/qa suite_relpath: qa suite_repo: https://github.com/ceph/ceph.git suite_sha1: dbb22f29210f43ee95e67f3b727f470d3faecd40 targets: smithi089.front.sepia.ceph.com: ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBFyT/r7xuKfmg50w0OdQebr4A/ptJo1Zk6DyZNZNZM/q5rW+sznh+wvHqeT3e4YtCdZLaXyvMms8o8u8H5nXOYM= smithi164.front.sepia.ceph.com: ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIGETOy3xONfbx7kUFCNiUtRLNfkcrcEsWzPdq6Np/6Bw tasks: - install: branch: pacific exclude_packages: - ceph-volume - cephadm: cephadm_branch: pacific cephadm_git_url: https://github.com/ceph/ceph conf: osd: osd_class_default_list: '*' osd_class_load_list: '*' image: quay.ceph.io/ceph-ci/ceph:pacific - cephadm.shell: mon.a: - ceph fs volume create foo - ceph config set mon mon_warn_on_insecure_global_id_reclaim false --force - ceph config set mon mon_warn_on_insecure_global_id_reclaim_allowed false --force - ceph.healthy: null - print: '**** upgrading first half of cluster, with stress ****' - parallel: - first-half-tasks - first-half-sequence - print: '**** done upgrading first half of cluster ****' - ceph.healthy: null - print: '**** applying stress + thrashing to mixed-version cluster ****' - parallel: - stress-tasks - ceph.healthy: null - print: '**** finishing upgrade ****' - parallel: - second-half-tasks - second-half-sequence - ceph.healthy: null teuthology: fragments_dropped: [] meta: {} postmerge: [] teuthology_branch: main teuthology_sha1: 62814aba234085f597115b9ba9bdc56af117de9e timestamp: 2023-06-03_14:23:09 tube: smithi user: teuthology verbose: true worker_log: /home/teuthworker/archive/worker_logs/dispatcher.smithi.23459 2023-06-03T19:11:04.829 INFO:teuthology.run:suite_path is set to /home/teuthworker/src/github.com_ceph_ceph_dbb22f29210f43ee95e67f3b727f470d3faecd40/qa; will attempt to use it 2023-06-03T19:11:04.830 INFO:teuthology.run:Found tasks at /home/teuthworker/src/github.com_ceph_ceph_dbb22f29210f43ee95e67f3b727f470d3faecd40/qa/tasks 2023-06-03T19:11:04.830 INFO:teuthology.run_tasks:Running task internal.check_packages... 2023-06-03T19:11:04.833 INFO:teuthology.task.internal:Checking packages... 2023-06-03T19:11:04.863 INFO:teuthology.task.internal:Checking packages for os_type 'ubuntu', flavor 'default' and ceph hash '8e24bcf462a1d94c2e12aa4e26419634883a1e88' 2023-06-03T19:11:04.864 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2023-06-03T19:11:04.864 INFO:teuthology.packaging:ref: None 2023-06-03T19:11:04.864 INFO:teuthology.packaging:tag: None 2023-06-03T19:11:04.865 INFO:teuthology.packaging:branch: reef 2023-06-03T19:11:04.865 INFO:teuthology.packaging:sha1: 8e24bcf462a1d94c2e12aa4e26419634883a1e88 2023-06-03T19:11:04.865 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=ubuntu%2F20.04%2Fx86_64&ref=reef 2023-06-03T19:11:05.078 INFO:teuthology.task.internal:Found packages for ceph version 18.0.0-3536-g6eba6641-1focal 2023-06-03T19:11:05.079 INFO:teuthology.run_tasks:Running task internal.buildpackages_prep... 2023-06-03T19:11:05.085 INFO:teuthology.task.internal:no buildpackages task found 2023-06-03T19:11:05.086 INFO:teuthology.run_tasks:Running task internal.save_config... 2023-06-03T19:11:05.132 INFO:teuthology.task.internal:Saving configuration 2023-06-03T19:11:05.155 INFO:teuthology.run_tasks:Running task internal.check_lock... 2023-06-03T19:11:05.163 INFO:teuthology.task.internal.check_lock:Checking locks... 2023-06-03T19:11:05.190 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi089.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/teuthology-2023-06-03_14:23:09-upgrade:pacific-x-reef-distro-default-smithi/7295631', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'ubuntu', 'os_version': '20.04', 'arch': 'x86_64', 'locked': True, 'locked_since': '2023-06-03 19:04:51.482033', 'locked_by': 'scheduled_teuthology@teuthology', 'mac_address': None, 'ssh_pub_key': 'ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIGETOy3xONfbx7kUFCNiUtRLNfkcrcEsWzPdq6Np/6Bw'} 2023-06-03T19:11:05.211 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi164.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/teuthology-2023-06-03_14:23:09-upgrade:pacific-x-reef-distro-default-smithi/7295631', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'ubuntu', 'os_version': '20.04', 'arch': 'x86_64', 'locked': True, 'locked_since': '2023-06-03 19:04:51.483517', 'locked_by': 'scheduled_teuthology@teuthology', 'mac_address': None, 'ssh_pub_key': 'ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIGETOy3xONfbx7kUFCNiUtRLNfkcrcEsWzPdq6Np/6Bw'} 2023-06-03T19:11:05.211 INFO:teuthology.run_tasks:Running task internal.add_remotes... 2023-06-03T19:11:05.218 INFO:teuthology.task.internal:roles: ubuntu@smithi089.front.sepia.ceph.com - ['mon.a', 'mon.c', 'mgr.y', 'osd.0', 'osd.1', 'osd.2', 'osd.3', 'client.0', 'node-exporter.a', 'alertmanager.a'] 2023-06-03T19:11:05.218 INFO:teuthology.task.internal:roles: ubuntu@smithi164.front.sepia.ceph.com - ['mon.b', 'mgr.x', 'osd.4', 'osd.5', 'osd.6', 'osd.7', 'client.1', 'prometheus.a', 'grafana.a', 'node-exporter.b'] 2023-06-03T19:11:05.218 INFO:teuthology.run_tasks:Running task console_log... 2023-06-03T19:11:05.299 DEBUG:teuthology.exit:Installing handler: Handler(exiter=, func=.kill_console_loggers at 0x7f6bad4d9280>, signals=[15]) 2023-06-03T19:11:05.300 INFO:teuthology.run_tasks:Running task internal.connect... 2023-06-03T19:11:05.306 INFO:teuthology.task.internal:Opening connections... 2023-06-03T19:11:05.307 DEBUG:teuthology.task.internal:connecting to ubuntu@smithi089.front.sepia.ceph.com 2023-06-03T19:11:05.309 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi089.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2023-06-03T19:11:05.363 DEBUG:teuthology.task.internal:connecting to ubuntu@smithi164.front.sepia.ceph.com 2023-06-03T19:11:05.364 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi164.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2023-06-03T19:11:05.420 INFO:teuthology.run_tasks:Running task internal.push_inventory... 2023-06-03T19:11:05.426 DEBUG:teuthology.orchestra.run.smithi089:> uname -m 2023-06-03T19:11:05.440 INFO:teuthology.orchestra.run.smithi089.stdout:x86_64 2023-06-03T19:11:05.441 DEBUG:teuthology.orchestra.run.smithi089:> cat /etc/os-release 2023-06-03T19:11:05.445 INFO:teuthology.orchestra.run.smithi089.stdout:NAME="Ubuntu" 2023-06-03T19:11:05.445 INFO:teuthology.orchestra.run.smithi089.stdout:VERSION="20.04.4 LTS (Focal Fossa)" 2023-06-03T19:11:05.446 INFO:teuthology.orchestra.run.smithi089.stdout:ID=ubuntu 2023-06-03T19:11:05.446 INFO:teuthology.orchestra.run.smithi089.stdout:ID_LIKE=debian 2023-06-03T19:11:05.446 INFO:teuthology.orchestra.run.smithi089.stdout:PRETTY_NAME="Ubuntu 20.04.4 LTS" 2023-06-03T19:11:05.446 INFO:teuthology.orchestra.run.smithi089.stdout:VERSION_ID="20.04" 2023-06-03T19:11:05.446 INFO:teuthology.orchestra.run.smithi089.stdout:HOME_URL="https://www.ubuntu.com/" 2023-06-03T19:11:05.446 INFO:teuthology.orchestra.run.smithi089.stdout:SUPPORT_URL="https://help.ubuntu.com/" 2023-06-03T19:11:05.446 INFO:teuthology.orchestra.run.smithi089.stdout:BUG_REPORT_URL="https://bugs.launchpad.net/ubuntu/" 2023-06-03T19:11:05.447 INFO:teuthology.orchestra.run.smithi089.stdout:PRIVACY_POLICY_URL="https://www.ubuntu.com/legal/terms-and-policies/privacy-policy" 2023-06-03T19:11:05.447 INFO:teuthology.orchestra.run.smithi089.stdout:VERSION_CODENAME=focal 2023-06-03T19:11:05.447 INFO:teuthology.orchestra.run.smithi089.stdout:UBUNTU_CODENAME=focal 2023-06-03T19:11:05.447 INFO:teuthology.lock.ops:Updating smithi089.front.sepia.ceph.com on lock server 2023-06-03T19:11:05.471 DEBUG:teuthology.orchestra.run.smithi164:> uname -m 2023-06-03T19:11:05.502 INFO:teuthology.orchestra.run.smithi164.stdout:x86_64 2023-06-03T19:11:05.503 DEBUG:teuthology.orchestra.run.smithi164:> cat /etc/os-release 2023-06-03T19:11:05.548 INFO:teuthology.orchestra.run.smithi164.stdout:NAME="Ubuntu" 2023-06-03T19:11:05.549 INFO:teuthology.orchestra.run.smithi164.stdout:VERSION="20.04.4 LTS (Focal Fossa)" 2023-06-03T19:11:05.549 INFO:teuthology.orchestra.run.smithi164.stdout:ID=ubuntu 2023-06-03T19:11:05.549 INFO:teuthology.orchestra.run.smithi164.stdout:ID_LIKE=debian 2023-06-03T19:11:05.549 INFO:teuthology.orchestra.run.smithi164.stdout:PRETTY_NAME="Ubuntu 20.04.4 LTS" 2023-06-03T19:11:05.549 INFO:teuthology.orchestra.run.smithi164.stdout:VERSION_ID="20.04" 2023-06-03T19:11:05.549 INFO:teuthology.orchestra.run.smithi164.stdout:HOME_URL="https://www.ubuntu.com/" 2023-06-03T19:11:05.550 INFO:teuthology.orchestra.run.smithi164.stdout:SUPPORT_URL="https://help.ubuntu.com/" 2023-06-03T19:11:05.550 INFO:teuthology.orchestra.run.smithi164.stdout:BUG_REPORT_URL="https://bugs.launchpad.net/ubuntu/" 2023-06-03T19:11:05.550 INFO:teuthology.orchestra.run.smithi164.stdout:PRIVACY_POLICY_URL="https://www.ubuntu.com/legal/terms-and-policies/privacy-policy" 2023-06-03T19:11:05.550 INFO:teuthology.orchestra.run.smithi164.stdout:VERSION_CODENAME=focal 2023-06-03T19:11:05.550 INFO:teuthology.orchestra.run.smithi164.stdout:UBUNTU_CODENAME=focal 2023-06-03T19:11:05.550 INFO:teuthology.lock.ops:Updating smithi164.front.sepia.ceph.com on lock server 2023-06-03T19:11:05.571 INFO:teuthology.run_tasks:Running task internal.serialize_remote_roles... 2023-06-03T19:11:05.585 INFO:teuthology.run_tasks:Running task internal.check_conflict... 2023-06-03T19:11:05.605 INFO:teuthology.task.internal:Checking for old test directory... 2023-06-03T19:11:05.606 DEBUG:teuthology.orchestra.run.smithi089:> test '!' -e /home/ubuntu/cephtest 2023-06-03T19:11:05.608 DEBUG:teuthology.orchestra.run.smithi164:> test '!' -e /home/ubuntu/cephtest 2023-06-03T19:11:05.614 INFO:teuthology.run_tasks:Running task internal.check_ceph_data... 2023-06-03T19:11:05.637 INFO:teuthology.task.internal:Checking for non-empty /var/lib/ceph... 2023-06-03T19:11:05.638 DEBUG:teuthology.orchestra.run.smithi089:> test -z $(ls -A /var/lib/ceph) 2023-06-03T19:11:05.640 DEBUG:teuthology.orchestra.run.smithi164:> test -z $(ls -A /var/lib/ceph) 2023-06-03T19:11:05.673 INFO:teuthology.run_tasks:Running task internal.vm_setup... 2023-06-03T19:11:05.727 INFO:teuthology.run_tasks:Running task kernel... 2023-06-03T19:11:05.744 INFO:teuthology.task.kernel:normalize config orig: {'kdb': True, 'sha1': 'distro'} 2023-06-03T19:11:05.745 INFO:teuthology.task.kernel:normalize config orig: {'hwe': True} 2023-06-03T19:11:05.745 DEBUG:teuthology.task.kernel:normalized overrides {'mon.a': {'hwe': True}, 'mon.c': {'hwe': True}, 'mgr.y': {'hwe': True}, 'osd.0': {'hwe': True}, 'osd.1': {'hwe': True}, 'osd.2': {'hwe': True}, 'osd.3': {'hwe': True}, 'client.0': {'hwe': True}, 'node-exporter.a': {'hwe': True}, 'alertmanager.a': {'hwe': True}, 'mon.b': {'hwe': True}, 'mgr.x': {'hwe': True}, 'osd.4': {'hwe': True}, 'osd.5': {'hwe': True}, 'osd.6': {'hwe': True}, 'osd.7': {'hwe': True}, 'client.1': {'hwe': True}, 'prometheus.a': {'hwe': True}, 'grafana.a': {'hwe': True}, 'node-exporter.b': {'hwe': True}} 2023-06-03T19:11:05.746 INFO:teuthology.task.kernel:config {'mon.a': {'kdb': True, 'sha1': 'distro', 'hwe': True}, 'mon.b': {'kdb': True, 'sha1': 'distro', 'hwe': True}}, timeout 300 2023-06-03T19:11:05.746 DEBUG:teuthology.orchestra.run.smithi089:> test -f /run/.containerenv -o -f /.dockerenv 2023-06-03T19:11:05.747 DEBUG:teuthology.orchestra.run.smithi164:> test -f /run/.containerenv -o -f /.dockerenv 2023-06-03T19:11:05.752 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T19:11:05.752 DEBUG:teuthology.orchestra.run.smithi089:> uname -r 2023-06-03T19:11:05.753 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T19:11:05.754 DEBUG:teuthology.orchestra.run.smithi164:> uname -r 2023-06-03T19:11:05.760 INFO:teuthology.orchestra.run.smithi164.stdout:5.4.0-124-generic 2023-06-03T19:11:05.761 INFO:teuthology.task.kernel:Running kernel on smithi164: 5.4.0-124-generic 2023-06-03T19:11:05.761 DEBUG:teuthology.orchestra.run.smithi164:> sudo apt-get clean 2023-06-03T19:11:05.799 INFO:teuthology.orchestra.run.smithi089.stdout:5.4.0-124-generic 2023-06-03T19:11:05.800 INFO:teuthology.task.kernel:Running kernel on smithi089: 5.4.0-124-generic 2023-06-03T19:11:05.800 DEBUG:teuthology.orchestra.run.smithi089:> sudo apt-get clean 2023-06-03T19:11:05.904 DEBUG:teuthology.orchestra.run.smithi089:> sudo apt-get update 2023-06-03T19:11:05.916 DEBUG:teuthology.orchestra.run.smithi164:> sudo apt-get update 2023-06-03T19:11:06.087 INFO:teuthology.orchestra.run.smithi089.stdout:Hit:1 http://archive.ubuntu.com/ubuntu focal InRelease 2023-06-03T19:11:06.091 INFO:teuthology.orchestra.run.smithi089.stdout:Get:2 http://archive.ubuntu.com/ubuntu focal-updates InRelease [114 kB] 2023-06-03T19:11:06.099 INFO:teuthology.orchestra.run.smithi164.stdout:Hit:1 http://archive.ubuntu.com/ubuntu focal InRelease 2023-06-03T19:11:06.100 INFO:teuthology.orchestra.run.smithi164.stdout:Get:2 http://archive.ubuntu.com/ubuntu focal-updates InRelease [114 kB] 2023-06-03T19:11:06.100 INFO:teuthology.orchestra.run.smithi164.stdout:Get:3 http://security.ubuntu.com/ubuntu focal-security InRelease [114 kB] 2023-06-03T19:11:06.149 INFO:teuthology.orchestra.run.smithi089.stdout:Get:3 http://archive.ubuntu.com/ubuntu focal-backports InRelease [108 kB] 2023-06-03T19:11:06.150 INFO:teuthology.orchestra.run.smithi164.stdout:Get:4 http://archive.ubuntu.com/ubuntu focal-backports InRelease [108 kB] 2023-06-03T19:11:06.191 INFO:teuthology.orchestra.run.smithi089.stdout:Get:4 http://security.ubuntu.com/ubuntu focal-security InRelease [114 kB] 2023-06-03T19:11:06.484 INFO:teuthology.orchestra.run.smithi089.stdout:Get:5 http://archive.ubuntu.com/ubuntu focal-updates/main i386 Packages [836 kB] 2023-06-03T19:11:06.492 INFO:teuthology.orchestra.run.smithi164.stdout:Get:5 http://security.ubuntu.com/ubuntu focal-security/main i386 Packages [606 kB] 2023-06-03T19:11:06.568 INFO:teuthology.orchestra.run.smithi164.stdout:Get:6 http://security.ubuntu.com/ubuntu focal-security/main amd64 Packages [2,229 kB] 2023-06-03T19:11:06.587 INFO:teuthology.orchestra.run.smithi089.stdout:Get:6 http://archive.ubuntu.com/ubuntu focal-updates/main amd64 Packages [2,611 kB] 2023-06-03T19:11:06.612 INFO:teuthology.orchestra.run.smithi164.stdout:Get:7 http://security.ubuntu.com/ubuntu focal-security/main Translation-en [358 kB] 2023-06-03T19:11:06.612 INFO:teuthology.orchestra.run.smithi164.stdout:Get:8 http://security.ubuntu.com/ubuntu focal-security/main amd64 c-n-f Metadata [13.0 kB] 2023-06-03T19:11:06.613 INFO:teuthology.orchestra.run.smithi164.stdout:Get:9 http://security.ubuntu.com/ubuntu focal-security/restricted amd64 Packages [1,833 kB] 2023-06-03T19:11:06.622 INFO:teuthology.orchestra.run.smithi164.stdout:Get:10 http://security.ubuntu.com/ubuntu focal-security/restricted i386 Packages [30.3 kB] 2023-06-03T19:11:06.622 INFO:teuthology.orchestra.run.smithi164.stdout:Get:11 http://security.ubuntu.com/ubuntu focal-security/restricted Translation-en [256 kB] 2023-06-03T19:11:06.623 INFO:teuthology.orchestra.run.smithi089.stdout:Get:7 http://archive.ubuntu.com/ubuntu focal-updates/main Translation-en [439 kB] 2023-06-03T19:11:06.624 INFO:teuthology.orchestra.run.smithi164.stdout:Get:12 http://security.ubuntu.com/ubuntu focal-security/restricted amd64 c-n-f Metadata [640 B] 2023-06-03T19:11:06.624 INFO:teuthology.orchestra.run.smithi164.stdout:Get:13 http://security.ubuntu.com/ubuntu focal-security/universe i386 Packages [600 kB] 2023-06-03T19:11:06.629 INFO:teuthology.orchestra.run.smithi164.stdout:Get:14 http://archive.ubuntu.com/ubuntu focal-updates/main i386 Packages [836 kB] 2023-06-03T19:11:06.630 INFO:teuthology.orchestra.run.smithi164.stdout:Get:15 http://security.ubuntu.com/ubuntu focal-security/universe amd64 Packages [842 kB] 2023-06-03T19:11:06.633 INFO:teuthology.orchestra.run.smithi089.stdout:Get:8 http://archive.ubuntu.com/ubuntu focal-updates/main amd64 c-n-f Metadata [16.8 kB] 2023-06-03T19:11:06.633 INFO:teuthology.orchestra.run.smithi089.stdout:Get:9 http://archive.ubuntu.com/ubuntu focal-updates/restricted i386 Packages [31.6 kB] 2023-06-03T19:11:06.633 INFO:teuthology.orchestra.run.smithi089.stdout:Get:10 http://archive.ubuntu.com/ubuntu focal-updates/restricted amd64 Packages [1,937 kB] 2023-06-03T19:11:06.651 INFO:teuthology.orchestra.run.smithi089.stdout:Get:11 http://archive.ubuntu.com/ubuntu focal-updates/restricted Translation-en [273 kB] 2023-06-03T19:11:06.654 INFO:teuthology.orchestra.run.smithi089.stdout:Get:12 http://archive.ubuntu.com/ubuntu focal-updates/restricted amd64 c-n-f Metadata [636 B] 2023-06-03T19:11:06.670 INFO:teuthology.orchestra.run.smithi164.stdout:Get:16 http://security.ubuntu.com/ubuntu focal-security/universe Translation-en [173 kB] 2023-06-03T19:11:06.689 INFO:teuthology.orchestra.run.smithi089.stdout:Get:13 http://archive.ubuntu.com/ubuntu focal-updates/universe amd64 Packages [1,067 kB] 2023-06-03T19:11:06.697 INFO:teuthology.orchestra.run.smithi164.stdout:Get:17 http://archive.ubuntu.com/ubuntu focal-updates/main amd64 Packages [2,611 kB] 2023-06-03T19:11:06.733 INFO:teuthology.orchestra.run.smithi164.stdout:Get:18 http://archive.ubuntu.com/ubuntu focal-updates/main Translation-en [439 kB] 2023-06-03T19:11:06.736 INFO:teuthology.orchestra.run.smithi164.stdout:Get:19 http://archive.ubuntu.com/ubuntu focal-updates/main amd64 c-n-f Metadata [16.8 kB] 2023-06-03T19:11:06.737 INFO:teuthology.orchestra.run.smithi164.stdout:Get:20 http://archive.ubuntu.com/ubuntu focal-updates/restricted i386 Packages [31.6 kB] 2023-06-03T19:11:06.737 INFO:teuthology.orchestra.run.smithi164.stdout:Get:21 http://archive.ubuntu.com/ubuntu focal-updates/restricted amd64 Packages [1,937 kB] 2023-06-03T19:11:06.749 INFO:teuthology.orchestra.run.smithi164.stdout:Get:22 http://archive.ubuntu.com/ubuntu focal-updates/restricted Translation-en [273 kB] 2023-06-03T19:11:06.751 INFO:teuthology.orchestra.run.smithi164.stdout:Get:23 http://archive.ubuntu.com/ubuntu focal-updates/restricted amd64 c-n-f Metadata [636 B] 2023-06-03T19:11:06.752 INFO:teuthology.orchestra.run.smithi164.stdout:Get:24 http://security.ubuntu.com/ubuntu focal-security/universe amd64 c-n-f Metadata [18.4 kB] 2023-06-03T19:11:06.753 INFO:teuthology.orchestra.run.smithi164.stdout:Get:25 http://security.ubuntu.com/ubuntu focal-security/multiverse i386 Packages [7,204 B] 2023-06-03T19:11:06.754 INFO:teuthology.orchestra.run.smithi164.stdout:Get:26 http://security.ubuntu.com/ubuntu focal-security/multiverse amd64 Packages [22.9 kB] 2023-06-03T19:11:06.755 INFO:teuthology.orchestra.run.smithi164.stdout:Get:27 http://security.ubuntu.com/ubuntu focal-security/multiverse Translation-en [5,488 B] 2023-06-03T19:11:06.756 INFO:teuthology.orchestra.run.smithi164.stdout:Get:28 http://security.ubuntu.com/ubuntu focal-security/multiverse amd64 c-n-f Metadata [540 B] 2023-06-03T19:11:06.786 INFO:teuthology.orchestra.run.smithi164.stdout:Get:29 http://archive.ubuntu.com/ubuntu focal-updates/universe amd64 Packages [1,067 kB] 2023-06-03T19:11:06.808 INFO:teuthology.orchestra.run.smithi089.stdout:Get:14 http://security.ubuntu.com/ubuntu focal-security/main amd64 Packages [2,229 kB] 2023-06-03T19:11:06.820 INFO:teuthology.orchestra.run.smithi089.stdout:Get:15 http://archive.ubuntu.com/ubuntu focal-updates/universe i386 Packages [730 kB] 2023-06-03T19:11:06.828 INFO:teuthology.orchestra.run.smithi089.stdout:Get:16 http://archive.ubuntu.com/ubuntu focal-updates/universe Translation-en [254 kB] 2023-06-03T19:11:06.831 INFO:teuthology.orchestra.run.smithi089.stdout:Get:17 http://archive.ubuntu.com/ubuntu focal-updates/universe amd64 c-n-f Metadata [24.9 kB] 2023-06-03T19:11:06.831 INFO:teuthology.orchestra.run.smithi089.stdout:Get:18 http://archive.ubuntu.com/ubuntu focal-updates/multiverse amd64 Packages [25.2 kB] 2023-06-03T19:11:06.832 INFO:teuthology.orchestra.run.smithi089.stdout:Get:19 http://archive.ubuntu.com/ubuntu focal-updates/multiverse i386 Packages [8,436 B] 2023-06-03T19:11:06.840 INFO:teuthology.orchestra.run.smithi089.stdout:Get:20 http://archive.ubuntu.com/ubuntu focal-updates/multiverse Translation-en [7,408 B] 2023-06-03T19:11:06.840 INFO:teuthology.orchestra.run.smithi089.stdout:Get:21 http://archive.ubuntu.com/ubuntu focal-updates/multiverse amd64 c-n-f Metadata [612 B] 2023-06-03T19:11:06.841 INFO:teuthology.orchestra.run.smithi089.stdout:Get:22 http://archive.ubuntu.com/ubuntu focal-backports/main amd64 Packages [45.7 kB] 2023-06-03T19:11:06.841 INFO:teuthology.orchestra.run.smithi089.stdout:Get:23 http://archive.ubuntu.com/ubuntu focal-backports/main i386 Packages [36.1 kB] 2023-06-03T19:11:06.841 INFO:teuthology.orchestra.run.smithi089.stdout:Get:24 http://archive.ubuntu.com/ubuntu focal-backports/main amd64 c-n-f Metadata [1,420 B] 2023-06-03T19:11:06.873 INFO:teuthology.orchestra.run.smithi089.stdout:Get:25 http://archive.ubuntu.com/ubuntu focal-backports/universe amd64 Packages [25.0 kB] 2023-06-03T19:11:06.911 INFO:teuthology.orchestra.run.smithi089.stdout:Get:26 http://archive.ubuntu.com/ubuntu focal-backports/universe i386 Packages [13.8 kB] 2023-06-03T19:11:06.916 INFO:teuthology.orchestra.run.smithi089.stdout:Get:27 http://archive.ubuntu.com/ubuntu focal-backports/universe amd64 c-n-f Metadata [880 B] 2023-06-03T19:11:06.917 INFO:teuthology.orchestra.run.smithi164.stdout:Get:30 http://archive.ubuntu.com/ubuntu focal-updates/universe i386 Packages [730 kB] 2023-06-03T19:11:06.926 INFO:teuthology.orchestra.run.smithi164.stdout:Get:31 http://archive.ubuntu.com/ubuntu focal-updates/universe Translation-en [254 kB] 2023-06-03T19:11:06.928 INFO:teuthology.orchestra.run.smithi164.stdout:Get:32 http://archive.ubuntu.com/ubuntu focal-updates/universe amd64 c-n-f Metadata [24.9 kB] 2023-06-03T19:11:06.928 INFO:teuthology.orchestra.run.smithi164.stdout:Get:33 http://archive.ubuntu.com/ubuntu focal-updates/multiverse i386 Packages [8,436 B] 2023-06-03T19:11:06.928 INFO:teuthology.orchestra.run.smithi164.stdout:Get:34 http://archive.ubuntu.com/ubuntu focal-updates/multiverse amd64 Packages [25.2 kB] 2023-06-03T19:11:06.929 INFO:teuthology.orchestra.run.smithi164.stdout:Get:35 http://archive.ubuntu.com/ubuntu focal-updates/multiverse Translation-en [7,408 B] 2023-06-03T19:11:06.929 INFO:teuthology.orchestra.run.smithi164.stdout:Get:36 http://archive.ubuntu.com/ubuntu focal-updates/multiverse amd64 c-n-f Metadata [612 B] 2023-06-03T19:11:06.929 INFO:teuthology.orchestra.run.smithi164.stdout:Get:37 http://archive.ubuntu.com/ubuntu focal-backports/main amd64 Packages [45.7 kB] 2023-06-03T19:11:06.933 INFO:teuthology.orchestra.run.smithi164.stdout:Get:38 http://archive.ubuntu.com/ubuntu focal-backports/main i386 Packages [36.1 kB] 2023-06-03T19:11:06.934 INFO:teuthology.orchestra.run.smithi164.stdout:Get:39 http://archive.ubuntu.com/ubuntu focal-backports/main amd64 c-n-f Metadata [1,420 B] 2023-06-03T19:11:06.968 INFO:teuthology.orchestra.run.smithi164.stdout:Get:40 http://archive.ubuntu.com/ubuntu focal-backports/universe amd64 Packages [25.0 kB] 2023-06-03T19:11:07.006 INFO:teuthology.orchestra.run.smithi164.stdout:Get:41 http://archive.ubuntu.com/ubuntu focal-backports/universe i386 Packages [13.8 kB] 2023-06-03T19:11:07.012 INFO:teuthology.orchestra.run.smithi164.stdout:Get:42 http://archive.ubuntu.com/ubuntu focal-backports/universe amd64 c-n-f Metadata [880 B] 2023-06-03T19:11:07.270 INFO:teuthology.orchestra.run.smithi089.stdout:Get:28 http://security.ubuntu.com/ubuntu focal-security/main i386 Packages [606 kB] 2023-06-03T19:11:07.285 INFO:teuthology.orchestra.run.smithi089.stdout:Get:29 http://security.ubuntu.com/ubuntu focal-security/main Translation-en [358 kB] 2023-06-03T19:11:07.293 INFO:teuthology.orchestra.run.smithi089.stdout:Get:30 http://security.ubuntu.com/ubuntu focal-security/main amd64 c-n-f Metadata [13.0 kB] 2023-06-03T19:11:07.294 INFO:teuthology.orchestra.run.smithi089.stdout:Get:31 http://security.ubuntu.com/ubuntu focal-security/restricted amd64 Packages [1,833 kB] 2023-06-03T19:11:07.364 INFO:teuthology.orchestra.run.smithi089.stdout:Get:32 http://security.ubuntu.com/ubuntu focal-security/restricted i386 Packages [30.3 kB] 2023-06-03T19:11:07.364 INFO:teuthology.orchestra.run.smithi089.stdout:Get:33 http://security.ubuntu.com/ubuntu focal-security/restricted Translation-en [256 kB] 2023-06-03T19:11:07.371 INFO:teuthology.orchestra.run.smithi089.stdout:Get:34 http://security.ubuntu.com/ubuntu focal-security/restricted amd64 c-n-f Metadata [640 B] 2023-06-03T19:11:07.371 INFO:teuthology.orchestra.run.smithi089.stdout:Get:35 http://security.ubuntu.com/ubuntu focal-security/universe i386 Packages [600 kB] 2023-06-03T19:11:07.386 INFO:teuthology.orchestra.run.smithi089.stdout:Get:36 http://security.ubuntu.com/ubuntu focal-security/universe amd64 Packages [842 kB] 2023-06-03T19:11:07.570 INFO:teuthology.orchestra.run.smithi089.stdout:Get:37 http://security.ubuntu.com/ubuntu focal-security/universe Translation-en [173 kB] 2023-06-03T19:11:07.976 INFO:teuthology.orchestra.run.smithi089.stdout:Get:38 http://security.ubuntu.com/ubuntu focal-security/universe amd64 c-n-f Metadata [18.4 kB] 2023-06-03T19:11:07.982 INFO:teuthology.orchestra.run.smithi089.stdout:Get:39 http://security.ubuntu.com/ubuntu focal-security/multiverse amd64 Packages [22.9 kB] 2023-06-03T19:11:07.991 INFO:teuthology.orchestra.run.smithi089.stdout:Get:40 http://security.ubuntu.com/ubuntu focal-security/multiverse i386 Packages [7,204 B] 2023-06-03T19:11:07.998 INFO:teuthology.orchestra.run.smithi089.stdout:Get:41 http://security.ubuntu.com/ubuntu focal-security/multiverse Translation-en [5,488 B] 2023-06-03T19:11:07.998 INFO:teuthology.orchestra.run.smithi089.stdout:Get:42 http://security.ubuntu.com/ubuntu focal-security/multiverse amd64 c-n-f Metadata [540 B] 2023-06-03T19:11:10.172 INFO:teuthology.orchestra.run.smithi164.stdout:Fetched 15.7 MB in 2s (6,843 kB/s) 2023-06-03T19:11:10.192 INFO:teuthology.orchestra.run.smithi089.stdout:Fetched 15.7 MB in 2s (6,647 kB/s) 2023-06-03T19:11:11.547 INFO:teuthology.orchestra.run.smithi164.stdout:Reading package lists... 2023-06-03T19:11:11.559 INFO:teuthology.orchestra.run.smithi089.stdout:Reading package lists... 2023-06-03T19:11:11.572 DEBUG:teuthology.orchestra.run.smithi164:> sudo DEBIAN_FRONTEND=noninteractive apt-get -y install linux-image-generic-hwe-20.04 2023-06-03T19:11:11.586 DEBUG:teuthology.orchestra.run.smithi089:> sudo DEBIAN_FRONTEND=noninteractive apt-get -y install linux-image-generic-hwe-20.04 2023-06-03T19:11:11.645 INFO:teuthology.orchestra.run.smithi164.stdout:Reading package lists... 2023-06-03T19:11:11.667 INFO:teuthology.orchestra.run.smithi089.stdout:Reading package lists... 2023-06-03T19:11:11.838 INFO:teuthology.orchestra.run.smithi164.stdout:Building dependency tree... 2023-06-03T19:11:11.839 INFO:teuthology.orchestra.run.smithi164.stdout:Reading state information... 2023-06-03T19:11:11.871 INFO:teuthology.orchestra.run.smithi089.stdout:Building dependency tree... 2023-06-03T19:11:11.871 INFO:teuthology.orchestra.run.smithi089.stdout:Reading state information... 2023-06-03T19:11:12.005 INFO:teuthology.orchestra.run.smithi164.stdout:The following additional packages will be installed: 2023-06-03T19:11:12.005 INFO:teuthology.orchestra.run.smithi164.stdout: libdbus-glib-1-2 libevdev2 libimobiledevice6 libplist3 libupower-glib3 2023-06-03T19:11:12.005 INFO:teuthology.orchestra.run.smithi164.stdout: libusbmuxd6 linux-image-5.15.0-73-generic linux-modules-5.15.0-73-generic 2023-06-03T19:11:12.006 INFO:teuthology.orchestra.run.smithi164.stdout: linux-modules-extra-5.15.0-73-generic thermald upower usbmuxd 2023-06-03T19:11:12.006 INFO:teuthology.orchestra.run.smithi164.stdout:Suggested packages: 2023-06-03T19:11:12.007 INFO:teuthology.orchestra.run.smithi164.stdout: libusbmuxd-tools fdutils linux-doc | linux-hwe-5.15-source-5.15.0 2023-06-03T19:11:12.007 INFO:teuthology.orchestra.run.smithi164.stdout: linux-hwe-5.15-tools linux-headers-5.15.0-73-generic 2023-06-03T19:11:12.045 INFO:teuthology.orchestra.run.smithi089.stdout:The following additional packages will be installed: 2023-06-03T19:11:12.045 INFO:teuthology.orchestra.run.smithi089.stdout: libdbus-glib-1-2 libevdev2 libimobiledevice6 libplist3 libupower-glib3 2023-06-03T19:11:12.046 INFO:teuthology.orchestra.run.smithi089.stdout: libusbmuxd6 linux-image-5.15.0-73-generic linux-modules-5.15.0-73-generic 2023-06-03T19:11:12.046 INFO:teuthology.orchestra.run.smithi089.stdout: linux-modules-extra-5.15.0-73-generic thermald upower usbmuxd 2023-06-03T19:11:12.046 INFO:teuthology.orchestra.run.smithi089.stdout:Suggested packages: 2023-06-03T19:11:12.046 INFO:teuthology.orchestra.run.smithi089.stdout: libusbmuxd-tools fdutils linux-doc | linux-hwe-5.15-source-5.15.0 2023-06-03T19:11:12.047 INFO:teuthology.orchestra.run.smithi089.stdout: linux-hwe-5.15-tools linux-headers-5.15.0-73-generic 2023-06-03T19:11:12.069 INFO:teuthology.orchestra.run.smithi164.stdout:The following NEW packages will be installed: 2023-06-03T19:11:12.069 INFO:teuthology.orchestra.run.smithi164.stdout: libdbus-glib-1-2 libevdev2 libimobiledevice6 libplist3 libupower-glib3 2023-06-03T19:11:12.070 INFO:teuthology.orchestra.run.smithi164.stdout: libusbmuxd6 linux-image-5.15.0-73-generic linux-image-generic-hwe-20.04 2023-06-03T19:11:12.070 INFO:teuthology.orchestra.run.smithi164.stdout: linux-modules-5.15.0-73-generic linux-modules-extra-5.15.0-73-generic 2023-06-03T19:11:12.070 INFO:teuthology.orchestra.run.smithi164.stdout: thermald upower usbmuxd 2023-06-03T19:11:12.110 INFO:teuthology.orchestra.run.smithi089.stdout:The following NEW packages will be installed: 2023-06-03T19:11:12.110 INFO:teuthology.orchestra.run.smithi089.stdout: libdbus-glib-1-2 libevdev2 libimobiledevice6 libplist3 libupower-glib3 2023-06-03T19:11:12.111 INFO:teuthology.orchestra.run.smithi089.stdout: libusbmuxd6 linux-image-5.15.0-73-generic linux-image-generic-hwe-20.04 2023-06-03T19:11:12.111 INFO:teuthology.orchestra.run.smithi089.stdout: linux-modules-5.15.0-73-generic linux-modules-extra-5.15.0-73-generic 2023-06-03T19:11:12.112 INFO:teuthology.orchestra.run.smithi089.stdout: thermald upower usbmuxd 2023-06-03T19:11:12.163 INFO:teuthology.orchestra.run.smithi089.stdout:0 upgraded, 13 newly installed, 0 to remove and 201 not upgraded. 2023-06-03T19:11:12.163 INFO:teuthology.orchestra.run.smithi089.stdout:Need to get 90.8 MB of archives. 2023-06-03T19:11:12.163 INFO:teuthology.orchestra.run.smithi089.stdout:After this operation, 497 MB of additional disk space will be used. 2023-06-03T19:11:12.164 INFO:teuthology.orchestra.run.smithi089.stdout:Get:1 http://archive.ubuntu.com/ubuntu focal/main amd64 libdbus-glib-1-2 amd64 0.110-5fakssync1 [59.1 kB] 2023-06-03T19:11:12.219 INFO:teuthology.orchestra.run.smithi089.stdout:Get:2 http://archive.ubuntu.com/ubuntu focal/main amd64 libplist3 amd64 2.1.0-4build2 [31.6 kB] 2023-06-03T19:11:12.224 INFO:teuthology.orchestra.run.smithi089.stdout:Get:3 http://archive.ubuntu.com/ubuntu focal/main amd64 libusbmuxd6 amd64 2.0.1-2 [19.1 kB] 2023-06-03T19:11:12.228 INFO:teuthology.orchestra.run.smithi089.stdout:Get:4 http://archive.ubuntu.com/ubuntu focal/main amd64 libimobiledevice6 amd64 1.2.1~git20191129.9f79242-1build1 [65.2 kB] 2023-06-03T19:11:12.243 INFO:teuthology.orchestra.run.smithi089.stdout:Get:5 http://archive.ubuntu.com/ubuntu focal/main amd64 libupower-glib3 amd64 0.99.11-1build2 [43.2 kB] 2023-06-03T19:11:12.250 INFO:teuthology.orchestra.run.smithi089.stdout:Get:6 http://archive.ubuntu.com/ubuntu focal-updates/main amd64 linux-modules-5.15.0-73-generic amd64 5.15.0-73.80~20.04.1 [20.9 MB] 2023-06-03T19:11:12.252 INFO:teuthology.orchestra.run.smithi164.stdout:0 upgraded, 13 newly installed, 0 to remove and 201 not upgraded. 2023-06-03T19:11:12.252 INFO:teuthology.orchestra.run.smithi164.stdout:Need to get 90.8 MB of archives. 2023-06-03T19:11:12.253 INFO:teuthology.orchestra.run.smithi164.stdout:After this operation, 497 MB of additional disk space will be used. 2023-06-03T19:11:12.253 INFO:teuthology.orchestra.run.smithi164.stdout:Get:1 http://archive.ubuntu.com/ubuntu focal/main amd64 libdbus-glib-1-2 amd64 0.110-5fakssync1 [59.1 kB] 2023-06-03T19:11:12.524 INFO:teuthology.orchestra.run.smithi164.stdout:Get:2 http://archive.ubuntu.com/ubuntu focal/main amd64 libplist3 amd64 2.1.0-4build2 [31.6 kB] 2023-06-03T19:11:12.541 INFO:teuthology.orchestra.run.smithi089.stdout:Get:7 http://archive.ubuntu.com/ubuntu focal-updates/main amd64 linux-image-5.15.0-73-generic amd64 5.15.0-73.80~20.04.1 [11.3 MB] 2023-06-03T19:11:12.553 INFO:teuthology.orchestra.run.smithi164.stdout:Get:3 http://archive.ubuntu.com/ubuntu focal/main amd64 libusbmuxd6 amd64 2.0.1-2 [19.1 kB] 2023-06-03T19:11:12.571 INFO:teuthology.orchestra.run.smithi164.stdout:Get:4 http://archive.ubuntu.com/ubuntu focal/main amd64 libimobiledevice6 amd64 1.2.1~git20191129.9f79242-1build1 [65.2 kB] 2023-06-03T19:11:12.645 INFO:teuthology.orchestra.run.smithi164.stdout:Get:5 http://archive.ubuntu.com/ubuntu focal/main amd64 libupower-glib3 amd64 0.99.11-1build2 [43.2 kB] 2023-06-03T19:11:12.681 INFO:teuthology.orchestra.run.smithi164.stdout:Get:6 http://archive.ubuntu.com/ubuntu focal-updates/main amd64 linux-modules-5.15.0-73-generic amd64 5.15.0-73.80~20.04.1 [20.9 MB] 2023-06-03T19:11:12.693 INFO:teuthology.orchestra.run.smithi089.stdout:Get:8 http://archive.ubuntu.com/ubuntu focal-updates/main amd64 linux-modules-extra-5.15.0-73-generic amd64 5.15.0-73.80~20.04.1 [58.0 MB] 2023-06-03T19:11:13.375 INFO:teuthology.orchestra.run.smithi089.stdout:Get:9 http://archive.ubuntu.com/ubuntu focal-updates/main amd64 linux-image-generic-hwe-20.04 amd64 5.15.0.73.80~20.04.34 [2,708 B] 2023-06-03T19:11:13.454 INFO:teuthology.orchestra.run.smithi089.stdout:Get:10 http://archive.ubuntu.com/ubuntu focal-updates/main amd64 libevdev2 amd64 1.9.0+dfsg-1ubuntu0.2 [31.6 kB] 2023-06-03T19:11:13.455 INFO:teuthology.orchestra.run.smithi089.stdout:Get:11 http://archive.ubuntu.com/ubuntu focal-updates/main amd64 thermald amd64 1.9.1-1ubuntu0.6 [233 kB] 2023-06-03T19:11:13.495 INFO:teuthology.orchestra.run.smithi089.stdout:Get:12 http://archive.ubuntu.com/ubuntu focal/main amd64 upower amd64 0.99.11-1build2 [104 kB] 2023-06-03T19:11:13.566 INFO:teuthology.orchestra.run.smithi089.stdout:Get:13 http://archive.ubuntu.com/ubuntu focal/main amd64 usbmuxd amd64 1.1.1~git20191130.9af2b12-1 [38.4 kB] 2023-06-03T19:11:13.620 INFO:teuthology.orchestra.run.smithi164.stdout:Get:7 http://archive.ubuntu.com/ubuntu focal-updates/main amd64 linux-image-5.15.0-73-generic amd64 5.15.0-73.80~20.04.1 [11.3 MB] 2023-06-03T19:11:13.943 INFO:teuthology.orchestra.run.smithi164.stdout:Get:8 http://archive.ubuntu.com/ubuntu focal-updates/main amd64 linux-modules-extra-5.15.0-73-generic amd64 5.15.0-73.80~20.04.1 [58.0 MB] 2023-06-03T19:11:14.232 INFO:teuthology.orchestra.run.smithi089.stdout:Fetched 90.8 MB in 1s (62.6 MB/s) 2023-06-03T19:11:14.442 INFO:teuthology.orchestra.run.smithi089.stdout:Selecting previously unselected package libdbus-glib-1-2:amd64. 2023-06-03T19:11:15.842 INFO:teuthology.orchestra.run.smithi164.stdout:Get:9 http://archive.ubuntu.com/ubuntu focal-updates/main amd64 linux-image-generic-hwe-20.04 amd64 5.15.0.73.80~20.04.34 [2,708 B] 2023-06-03T19:11:15.842 INFO:teuthology.orchestra.run.smithi164.stdout:Get:10 http://archive.ubuntu.com/ubuntu focal-updates/main amd64 libevdev2 amd64 1.9.0+dfsg-1ubuntu0.2 [31.6 kB] 2023-06-03T19:11:15.843 INFO:teuthology.orchestra.run.smithi164.stdout:Get:11 http://archive.ubuntu.com/ubuntu focal-updates/main amd64 thermald amd64 1.9.1-1ubuntu0.6 [233 kB] 2023-06-03T19:11:16.013 INFO:teuthology.orchestra.run.smithi164.stdout:Get:12 http://archive.ubuntu.com/ubuntu focal/main amd64 upower amd64 0.99.11-1build2 [104 kB] 2023-06-03T19:11:16.162 INFO:teuthology.orchestra.run.smithi089.stdout:(Reading database ... (Reading database ... 5% (Reading database ... 10% (Reading database ... 15% (Reading database ... 20% (Reading database ... 25% (Reading database ... 30% (Reading database ... 35% (Reading database ... 40% (Reading database ... 45% (Reading database ... 50% (Reading database ... 55% (Reading database ... 60% (Reading database ... 65% (Reading database ... 70% (Reading database ... 75% (Reading database ... 80% (Reading database ... 85% (Reading database ... 90% (Reading database ... 95% (Reading database ... 100% (Reading database ... 104373 files and directories currently installed.) 2023-06-03T19:11:16.167 INFO:teuthology.orchestra.run.smithi089.stdout:Preparing to unpack .../00-libdbus-glib-1-2_0.110-5fakssync1_amd64.deb ... 2023-06-03T19:11:16.307 INFO:teuthology.orchestra.run.smithi089.stdout:Unpacking libdbus-glib-1-2:amd64 (0.110-5fakssync1) ... 2023-06-03T19:11:16.358 INFO:teuthology.orchestra.run.smithi164.stdout:Get:13 http://archive.ubuntu.com/ubuntu focal/main amd64 usbmuxd amd64 1.1.1~git20191130.9af2b12-1 [38.4 kB] 2023-06-03T19:11:16.618 INFO:teuthology.orchestra.run.smithi089.stdout:Selecting previously unselected package libplist3:amd64. 2023-06-03T19:11:16.631 INFO:teuthology.orchestra.run.smithi089.stdout:Preparing to unpack .../01-libplist3_2.1.0-4build2_amd64.deb ... 2023-06-03T19:11:16.660 INFO:teuthology.orchestra.run.smithi089.stdout:Unpacking libplist3:amd64 (2.1.0-4build2) ... 2023-06-03T19:11:16.979 INFO:teuthology.orchestra.run.smithi089.stdout:Selecting previously unselected package libusbmuxd6:amd64. 2023-06-03T19:11:16.992 INFO:teuthology.orchestra.run.smithi089.stdout:Preparing to unpack .../02-libusbmuxd6_2.0.1-2_amd64.deb ... 2023-06-03T19:11:17.029 INFO:teuthology.orchestra.run.smithi089.stdout:Unpacking libusbmuxd6:amd64 (2.0.1-2) ... 2023-06-03T19:11:17.108 INFO:teuthology.orchestra.run.smithi164.stdout:Fetched 90.8 MB in 4s (21.1 MB/s) 2023-06-03T19:11:17.342 INFO:teuthology.orchestra.run.smithi164.stdout:Selecting previously unselected package libdbus-glib-1-2:amd64. 2023-06-03T19:11:17.382 INFO:teuthology.orchestra.run.smithi089.stdout:Selecting previously unselected package libimobiledevice6:amd64. 2023-06-03T19:11:17.395 INFO:teuthology.orchestra.run.smithi089.stdout:Preparing to unpack .../03-libimobiledevice6_1.2.1~git20191129.9f79242-1build1_amd64.deb ... 2023-06-03T19:11:17.432 INFO:teuthology.orchestra.run.smithi089.stdout:Unpacking libimobiledevice6:amd64 (1.2.1~git20191129.9f79242-1build1) ... 2023-06-03T19:11:17.743 INFO:teuthology.orchestra.run.smithi089.stdout:Selecting previously unselected package libupower-glib3:amd64. 2023-06-03T19:11:17.758 INFO:teuthology.orchestra.run.smithi089.stdout:Preparing to unpack .../04-libupower-glib3_0.99.11-1build2_amd64.deb ... 2023-06-03T19:11:17.793 INFO:teuthology.orchestra.run.smithi089.stdout:Unpacking libupower-glib3:amd64 (0.99.11-1build2) ... 2023-06-03T19:11:18.080 INFO:teuthology.orchestra.run.smithi089.stdout:Selecting previously unselected package linux-modules-5.15.0-73-generic. 2023-06-03T19:11:18.094 INFO:teuthology.orchestra.run.smithi089.stdout:Preparing to unpack .../05-linux-modules-5.15.0-73-generic_5.15.0-73.80~20.04.1_amd64.deb ... 2023-06-03T19:11:18.138 INFO:teuthology.orchestra.run.smithi089.stdout:Unpacking linux-modules-5.15.0-73-generic (5.15.0-73.80~20.04.1) ... 2023-06-03T19:11:19.252 INFO:teuthology.orchestra.run.smithi164.stdout:(Reading database ... (Reading database ... 5% (Reading database ... 10% (Reading database ... 15% (Reading database ... 20% (Reading database ... 25% (Reading database ... 30% (Reading database ... 35% (Reading database ... 40% (Reading database ... 45% (Reading database ... 50% (Reading database ... 55% (Reading database ... 60% (Reading database ... 65% (Reading database ... 70% (Reading database ... 75% (Reading database ... 80% (Reading database ... 85% (Reading database ... 90% (Reading database ... 95% (Reading database ... 100% (Reading database ... 104373 files and directories currently installed.) 2023-06-03T19:11:19.256 INFO:teuthology.orchestra.run.smithi164.stdout:Preparing to unpack .../00-libdbus-glib-1-2_0.110-5fakssync1_amd64.deb ... 2023-06-03T19:11:19.481 INFO:teuthology.orchestra.run.smithi164.stdout:Unpacking libdbus-glib-1-2:amd64 (0.110-5fakssync1) ... 2023-06-03T19:11:19.834 INFO:teuthology.orchestra.run.smithi164.stdout:Selecting previously unselected package libplist3:amd64. 2023-06-03T19:11:19.848 INFO:teuthology.orchestra.run.smithi164.stdout:Preparing to unpack .../01-libplist3_2.1.0-4build2_amd64.deb ... 2023-06-03T19:11:19.917 INFO:teuthology.orchestra.run.smithi164.stdout:Unpacking libplist3:amd64 (2.1.0-4build2) ... 2023-06-03T19:11:20.278 INFO:teuthology.orchestra.run.smithi164.stdout:Selecting previously unselected package libusbmuxd6:amd64. 2023-06-03T19:11:20.292 INFO:teuthology.orchestra.run.smithi164.stdout:Preparing to unpack .../02-libusbmuxd6_2.0.1-2_amd64.deb ... 2023-06-03T19:11:20.361 INFO:teuthology.orchestra.run.smithi164.stdout:Unpacking libusbmuxd6:amd64 (2.0.1-2) ... 2023-06-03T19:11:20.656 INFO:teuthology.orchestra.run.smithi089.stdout:Selecting previously unselected package linux-image-5.15.0-73-generic. 2023-06-03T19:11:20.670 INFO:teuthology.orchestra.run.smithi089.stdout:Preparing to unpack .../06-linux-image-5.15.0-73-generic_5.15.0-73.80~20.04.1_amd64.deb ... 2023-06-03T19:11:20.706 INFO:teuthology.orchestra.run.smithi164.stdout:Selecting previously unselected package libimobiledevice6:amd64. 2023-06-03T19:11:20.719 INFO:teuthology.orchestra.run.smithi164.stdout:Preparing to unpack .../03-libimobiledevice6_1.2.1~git20191129.9f79242-1build1_amd64.deb ... 2023-06-03T19:11:20.760 INFO:teuthology.orchestra.run.smithi089.stdout:Unpacking linux-image-5.15.0-73-generic (5.15.0-73.80~20.04.1) ... 2023-06-03T19:11:20.764 INFO:teuthology.orchestra.run.smithi164.stdout:Unpacking libimobiledevice6:amd64 (1.2.1~git20191129.9f79242-1build1) ... 2023-06-03T19:11:21.093 INFO:teuthology.orchestra.run.smithi164.stdout:Selecting previously unselected package libupower-glib3:amd64. 2023-06-03T19:11:21.107 INFO:teuthology.orchestra.run.smithi164.stdout:Preparing to unpack .../04-libupower-glib3_0.99.11-1build2_amd64.deb ... 2023-06-03T19:11:21.142 INFO:teuthology.orchestra.run.smithi164.stdout:Unpacking libupower-glib3:amd64 (0.99.11-1build2) ... 2023-06-03T19:11:21.292 INFO:teuthology.orchestra.run.smithi089.stdout:Selecting previously unselected package linux-modules-extra-5.15.0-73-generic. 2023-06-03T19:11:21.305 INFO:teuthology.orchestra.run.smithi089.stdout:Preparing to unpack .../07-linux-modules-extra-5.15.0-73-generic_5.15.0-73.80~20.04.1_amd64.deb ... 2023-06-03T19:11:21.366 INFO:teuthology.orchestra.run.smithi089.stdout:Unpacking linux-modules-extra-5.15.0-73-generic (5.15.0-73.80~20.04.1) ... 2023-06-03T19:11:21.504 INFO:teuthology.orchestra.run.smithi164.stdout:Selecting previously unselected package linux-modules-5.15.0-73-generic. 2023-06-03T19:11:21.517 INFO:teuthology.orchestra.run.smithi164.stdout:Preparing to unpack .../05-linux-modules-5.15.0-73-generic_5.15.0-73.80~20.04.1_amd64.deb ... 2023-06-03T19:11:21.562 INFO:teuthology.orchestra.run.smithi164.stdout:Unpacking linux-modules-5.15.0-73-generic (5.15.0-73.80~20.04.1) ... 2023-06-03T19:11:24.246 INFO:teuthology.orchestra.run.smithi164.stdout:Selecting previously unselected package linux-image-5.15.0-73-generic. 2023-06-03T19:11:24.262 INFO:teuthology.orchestra.run.smithi164.stdout:Preparing to unpack .../06-linux-image-5.15.0-73-generic_5.15.0-73.80~20.04.1_amd64.deb ... 2023-06-03T19:11:24.372 INFO:teuthology.orchestra.run.smithi164.stdout:Unpacking linux-image-5.15.0-73-generic (5.15.0-73.80~20.04.1) ... 2023-06-03T19:11:24.941 INFO:teuthology.orchestra.run.smithi164.stdout:Selecting previously unselected package linux-modules-extra-5.15.0-73-generic. 2023-06-03T19:11:24.957 INFO:teuthology.orchestra.run.smithi164.stdout:Preparing to unpack .../07-linux-modules-extra-5.15.0-73-generic_5.15.0-73.80~20.04.1_amd64.deb ... 2023-06-03T19:11:24.999 INFO:teuthology.orchestra.run.smithi164.stdout:Unpacking linux-modules-extra-5.15.0-73-generic (5.15.0-73.80~20.04.1) ... 2023-06-03T19:11:28.086 INFO:teuthology.orchestra.run.smithi089.stdout:Selecting previously unselected package linux-image-generic-hwe-20.04. 2023-06-03T19:11:28.101 INFO:teuthology.orchestra.run.smithi089.stdout:Preparing to unpack .../08-linux-image-generic-hwe-20.04_5.15.0.73.80~20.04.34_amd64.deb ... 2023-06-03T19:11:28.144 INFO:teuthology.orchestra.run.smithi089.stdout:Unpacking linux-image-generic-hwe-20.04 (5.15.0.73.80~20.04.34) ... 2023-06-03T19:11:28.463 INFO:teuthology.orchestra.run.smithi089.stdout:Selecting previously unselected package libevdev2:amd64. 2023-06-03T19:11:28.476 INFO:teuthology.orchestra.run.smithi089.stdout:Preparing to unpack .../09-libevdev2_1.9.0+dfsg-1ubuntu0.2_amd64.deb ... 2023-06-03T19:11:28.513 INFO:teuthology.orchestra.run.smithi089.stdout:Unpacking libevdev2:amd64 (1.9.0+dfsg-1ubuntu0.2) ... 2023-06-03T19:11:28.882 INFO:teuthology.orchestra.run.smithi089.stdout:Selecting previously unselected package thermald. 2023-06-03T19:11:28.896 INFO:teuthology.orchestra.run.smithi089.stdout:Preparing to unpack .../10-thermald_1.9.1-1ubuntu0.6_amd64.deb ... 2023-06-03T19:11:28.941 INFO:teuthology.orchestra.run.smithi089.stdout:Unpacking thermald (1.9.1-1ubuntu0.6) ... 2023-06-03T19:11:29.604 INFO:teuthology.orchestra.run.smithi089.stdout:Selecting previously unselected package upower. 2023-06-03T19:11:29.620 INFO:teuthology.orchestra.run.smithi089.stdout:Preparing to unpack .../11-upower_0.99.11-1build2_amd64.deb ... 2023-06-03T19:11:29.681 INFO:teuthology.orchestra.run.smithi089.stdout:Unpacking upower (0.99.11-1build2) ... 2023-06-03T19:11:30.125 INFO:teuthology.orchestra.run.smithi089.stdout:Selecting previously unselected package usbmuxd. 2023-06-03T19:11:30.140 INFO:teuthology.orchestra.run.smithi089.stdout:Preparing to unpack .../12-usbmuxd_1.1.1~git20191130.9af2b12-1_amd64.deb ... 2023-06-03T19:11:30.183 INFO:teuthology.orchestra.run.smithi089.stdout:Unpacking usbmuxd (1.1.1~git20191130.9af2b12-1) ... 2023-06-03T19:11:30.500 INFO:teuthology.orchestra.run.smithi089.stdout:Setting up libplist3:amd64 (2.1.0-4build2) ... 2023-06-03T19:11:30.621 INFO:teuthology.orchestra.run.smithi089.stdout:Setting up libdbus-glib-1-2:amd64 (0.110-5fakssync1) ... 2023-06-03T19:11:30.748 INFO:teuthology.orchestra.run.smithi089.stdout:Setting up libusbmuxd6:amd64 (2.0.1-2) ... 2023-06-03T19:11:30.874 INFO:teuthology.orchestra.run.smithi089.stdout:Setting up libupower-glib3:amd64 (0.99.11-1build2) ... 2023-06-03T19:11:31.000 INFO:teuthology.orchestra.run.smithi089.stdout:Setting up libimobiledevice6:amd64 (1.2.1~git20191129.9f79242-1build1) ... 2023-06-03T19:11:31.143 INFO:teuthology.orchestra.run.smithi089.stdout:Setting up libevdev2:amd64 (1.9.0+dfsg-1ubuntu0.2) ... 2023-06-03T19:11:31.269 INFO:teuthology.orchestra.run.smithi089.stdout:Setting up upower (0.99.11-1build2) ... 2023-06-03T19:11:31.795 INFO:teuthology.orchestra.run.smithi089.stdout:upower.service is a disabled or a static unit, not starting it. 2023-06-03T19:11:31.847 INFO:teuthology.orchestra.run.smithi089.stdout:Setting up usbmuxd (1.1.1~git20191130.9af2b12-1) ... 2023-06-03T19:11:32.026 INFO:teuthology.orchestra.run.smithi089.stdout:Warning: The home dir /var/lib/usbmux you specified can't be accessed: No such file or directory 2023-06-03T19:11:32.038 INFO:teuthology.orchestra.run.smithi089.stdout:Adding system user `usbmux' (UID 114) ... 2023-06-03T19:11:32.038 INFO:teuthology.orchestra.run.smithi089.stdout:Adding new user `usbmux' (UID 114) with group `plugdev' ... 2023-06-03T19:11:32.307 INFO:teuthology.orchestra.run.smithi164.stdout:Selecting previously unselected package linux-image-generic-hwe-20.04. 2023-06-03T19:11:32.325 INFO:teuthology.orchestra.run.smithi164.stdout:Preparing to unpack .../08-linux-image-generic-hwe-20.04_5.15.0.73.80~20.04.34_amd64.deb ... 2023-06-03T19:11:32.366 INFO:teuthology.orchestra.run.smithi164.stdout:Unpacking linux-image-generic-hwe-20.04 (5.15.0.73.80~20.04.34) ... 2023-06-03T19:11:32.548 INFO:teuthology.orchestra.run.smithi089.stdout:Not creating home directory `/var/lib/usbmux'. 2023-06-03T19:11:32.701 INFO:teuthology.orchestra.run.smithi164.stdout:Selecting previously unselected package libevdev2:amd64. 2023-06-03T19:11:32.719 INFO:teuthology.orchestra.run.smithi164.stdout:Preparing to unpack .../09-libevdev2_1.9.0+dfsg-1ubuntu0.2_amd64.deb ... 2023-06-03T19:11:32.765 INFO:teuthology.orchestra.run.smithi089.stdout:usbmuxd.service is a disabled or a static unit, not starting it. 2023-06-03T19:11:32.776 INFO:teuthology.orchestra.run.smithi164.stdout:Unpacking libevdev2:amd64 (1.9.0+dfsg-1ubuntu0.2) ... 2023-06-03T19:11:32.811 INFO:teuthology.orchestra.run.smithi089.stdout:Setting up thermald (1.9.1-1ubuntu0.6) ... 2023-06-03T19:11:33.119 INFO:teuthology.orchestra.run.smithi089.stdout:Created symlink /etc/systemd/system/dbus-org.freedesktop.thermald.service -> /lib/systemd/system/thermald.service. 2023-06-03T19:11:33.119 INFO:teuthology.orchestra.run.smithi089.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/thermald.service -> /lib/systemd/system/thermald.service. 2023-06-03T19:11:33.170 INFO:teuthology.orchestra.run.smithi164.stdout:Selecting previously unselected package thermald. 2023-06-03T19:11:33.184 INFO:teuthology.orchestra.run.smithi164.stdout:Preparing to unpack .../10-thermald_1.9.1-1ubuntu0.6_amd64.deb ... 2023-06-03T19:11:33.237 INFO:teuthology.orchestra.run.smithi164.stdout:Unpacking thermald (1.9.1-1ubuntu0.6) ... 2023-06-03T19:11:33.590 INFO:teuthology.orchestra.run.smithi089.stdout:Setting up linux-image-5.15.0-73-generic (5.15.0-73.80~20.04.1) ... 2023-06-03T19:11:34.200 INFO:teuthology.orchestra.run.smithi164.stdout:Selecting previously unselected package upower. 2023-06-03T19:11:34.216 INFO:teuthology.orchestra.run.smithi164.stdout:Preparing to unpack .../11-upower_0.99.11-1build2_amd64.deb ... 2023-06-03T19:11:34.281 INFO:teuthology.orchestra.run.smithi164.stdout:Unpacking upower (0.99.11-1build2) ... 2023-06-03T19:11:34.722 INFO:teuthology.orchestra.run.smithi164.stdout:Selecting previously unselected package usbmuxd. 2023-06-03T19:11:34.735 INFO:teuthology.orchestra.run.smithi164.stdout:Preparing to unpack .../12-usbmuxd_1.1.1~git20191130.9af2b12-1_amd64.deb ... 2023-06-03T19:11:34.780 INFO:teuthology.orchestra.run.smithi164.stdout:Unpacking usbmuxd (1.1.1~git20191130.9af2b12-1) ... 2023-06-03T19:11:34.961 INFO:teuthology.orchestra.run.smithi089.stdout:I: /vmlinuz is now a symlink to boot/vmlinuz-5.15.0-73-generic 2023-06-03T19:11:34.961 INFO:teuthology.orchestra.run.smithi089.stdout:I: /initrd.img is now a symlink to boot/initrd.img-5.15.0-73-generic 2023-06-03T19:11:35.121 INFO:teuthology.orchestra.run.smithi164.stdout:Setting up libplist3:amd64 (2.1.0-4build2) ... 2023-06-03T19:11:35.243 INFO:teuthology.orchestra.run.smithi089.stdout:Setting up linux-modules-5.15.0-73-generic (5.15.0-73.80~20.04.1) ... 2023-06-03T19:11:35.268 INFO:teuthology.orchestra.run.smithi164.stdout:Setting up libdbus-glib-1-2:amd64 (0.110-5fakssync1) ... 2023-06-03T19:11:35.419 INFO:teuthology.orchestra.run.smithi164.stdout:Setting up libusbmuxd6:amd64 (2.0.1-2) ... 2023-06-03T19:11:35.545 INFO:teuthology.orchestra.run.smithi164.stdout:Setting up libupower-glib3:amd64 (0.99.11-1build2) ... 2023-06-03T19:11:35.687 INFO:teuthology.orchestra.run.smithi164.stdout:Setting up libimobiledevice6:amd64 (1.2.1~git20191129.9f79242-1build1) ... 2023-06-03T19:11:35.838 INFO:teuthology.orchestra.run.smithi164.stdout:Setting up libevdev2:amd64 (1.9.0+dfsg-1ubuntu0.2) ... 2023-06-03T19:11:35.965 INFO:teuthology.orchestra.run.smithi164.stdout:Setting up upower (0.99.11-1build2) ... 2023-06-03T19:11:36.544 INFO:teuthology.orchestra.run.smithi164.stdout:upower.service is a disabled or a static unit, not starting it. 2023-06-03T19:11:36.602 INFO:teuthology.orchestra.run.smithi164.stdout:Setting up usbmuxd (1.1.1~git20191130.9af2b12-1) ... 2023-06-03T19:11:36.635 INFO:teuthology.orchestra.run.smithi089.stdout:Setting up linux-modules-extra-5.15.0-73-generic (5.15.0-73.80~20.04.1) ... 2023-06-03T19:11:36.783 INFO:teuthology.orchestra.run.smithi164.stdout:Warning: The home dir /var/lib/usbmux you specified can't be accessed: No such file or directory 2023-06-03T19:11:36.789 INFO:teuthology.orchestra.run.smithi164.stdout:Adding system user `usbmux' (UID 114) ... 2023-06-03T19:11:36.789 INFO:teuthology.orchestra.run.smithi164.stdout:Adding new user `usbmux' (UID 114) with group `plugdev' ... 2023-06-03T19:11:37.344 INFO:teuthology.orchestra.run.smithi164.stdout:Not creating home directory `/var/lib/usbmux'. 2023-06-03T19:11:37.569 INFO:teuthology.orchestra.run.smithi164.stdout:usbmuxd.service is a disabled or a static unit, not starting it. 2023-06-03T19:11:37.641 INFO:teuthology.orchestra.run.smithi164.stdout:Setting up thermald (1.9.1-1ubuntu0.6) ... 2023-06-03T19:11:37.953 INFO:teuthology.orchestra.run.smithi164.stdout:Created symlink /etc/systemd/system/dbus-org.freedesktop.thermald.service -> /lib/systemd/system/thermald.service. 2023-06-03T19:11:37.953 INFO:teuthology.orchestra.run.smithi164.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/thermald.service -> /lib/systemd/system/thermald.service. 2023-06-03T19:11:38.004 INFO:teuthology.orchestra.run.smithi089.stdout:Setting up linux-image-generic-hwe-20.04 (5.15.0.73.80~20.04.34) ... 2023-06-03T19:11:38.121 INFO:teuthology.orchestra.run.smithi089.stdout:Processing triggers for libc-bin (2.31-0ubuntu9.9) ... 2023-06-03T19:11:38.419 INFO:teuthology.orchestra.run.smithi164.stdout:Setting up linux-image-5.15.0-73-generic (5.15.0-73.80~20.04.1) ... 2023-06-03T19:11:38.452 INFO:teuthology.orchestra.run.smithi089.stdout:Processing triggers for man-db (2.9.1-1) ... 2023-06-03T19:11:39.213 INFO:teuthology.orchestra.run.smithi089.stdout:Processing triggers for dbus (1.12.16-2ubuntu2.2) ... 2023-06-03T19:11:39.330 INFO:teuthology.orchestra.run.smithi089.stdout:Processing triggers for linux-image-5.15.0-73-generic (5.15.0-73.80~20.04.1) ... 2023-06-03T19:11:39.400 INFO:teuthology.orchestra.run.smithi089.stdout:/etc/kernel/postinst.d/initramfs-tools: 2023-06-03T19:11:39.401 INFO:teuthology.orchestra.run.smithi089.stdout:update-initramfs: Generating /boot/initrd.img-5.15.0-73-generic 2023-06-03T19:11:39.778 INFO:teuthology.orchestra.run.smithi164.stdout:I: /vmlinuz is now a symlink to boot/vmlinuz-5.15.0-73-generic 2023-06-03T19:11:39.778 INFO:teuthology.orchestra.run.smithi164.stdout:I: /initrd.img is now a symlink to boot/initrd.img-5.15.0-73-generic 2023-06-03T19:11:39.939 INFO:teuthology.orchestra.run.smithi164.stdout:Setting up linux-modules-5.15.0-73-generic (5.15.0-73.80~20.04.1) ... 2023-06-03T19:11:41.390 INFO:teuthology.orchestra.run.smithi164.stdout:Setting up linux-modules-extra-5.15.0-73-generic (5.15.0-73.80~20.04.1) ... 2023-06-03T19:11:42.800 INFO:teuthology.orchestra.run.smithi164.stdout:Setting up linux-image-generic-hwe-20.04 (5.15.0.73.80~20.04.34) ... 2023-06-03T19:11:42.909 INFO:teuthology.orchestra.run.smithi164.stdout:Processing triggers for libc-bin (2.31-0ubuntu9.9) ... 2023-06-03T19:11:43.181 INFO:teuthology.orchestra.run.smithi164.stdout:Processing triggers for man-db (2.9.1-1) ... 2023-06-03T19:11:43.925 INFO:teuthology.orchestra.run.smithi164.stdout:Processing triggers for dbus (1.12.16-2ubuntu2.2) ... 2023-06-03T19:11:44.017 INFO:teuthology.orchestra.run.smithi164.stdout:Processing triggers for linux-image-5.15.0-73-generic (5.15.0-73.80~20.04.1) ... 2023-06-03T19:11:44.117 INFO:teuthology.orchestra.run.smithi164.stdout:/etc/kernel/postinst.d/initramfs-tools: 2023-06-03T19:11:44.117 INFO:teuthology.orchestra.run.smithi164.stdout:update-initramfs: Generating /boot/initrd.img-5.15.0-73-generic 2023-06-03T19:12:05.416 INFO:teuthology.orchestra.run.smithi089.stdout:/etc/kernel/postinst.d/zz-update-grub: 2023-06-03T19:12:05.416 INFO:teuthology.orchestra.run.smithi089.stdout:Sourcing file `/etc/default/grub' 2023-06-03T19:12:05.424 INFO:teuthology.orchestra.run.smithi089.stdout:Sourcing file `/etc/default/grub.d/init-select.cfg' 2023-06-03T19:12:05.445 INFO:teuthology.orchestra.run.smithi089.stdout:Generating grub configuration file ... 2023-06-03T19:12:05.753 INFO:teuthology.orchestra.run.smithi089.stdout:Found linux image: /boot/vmlinuz-5.15.0-73-generic 2023-06-03T19:12:05.794 INFO:teuthology.orchestra.run.smithi089.stdout:Found initrd image: /boot/initrd.img-5.15.0-73-generic 2023-06-03T19:12:06.193 INFO:teuthology.orchestra.run.smithi089.stdout:Found linux image: /boot/vmlinuz-5.4.0-124-generic 2023-06-03T19:12:06.205 INFO:teuthology.orchestra.run.smithi089.stdout:Found initrd image: /boot/initrd.img-5.4.0-124-generic 2023-06-03T19:12:08.331 INFO:teuthology.orchestra.run.smithi089.stdout:done 2023-06-03T19:12:08.749 DEBUG:teuthology.orchestra.run.smithi089:> dpkg -s linux-image-generic-hwe-20.04 2023-06-03T19:12:08.795 INFO:teuthology.orchestra.run.smithi089.stdout:Package: linux-image-generic-hwe-20.04 2023-06-03T19:12:08.795 INFO:teuthology.orchestra.run.smithi089.stdout:Status: install ok installed 2023-06-03T19:12:08.795 INFO:teuthology.orchestra.run.smithi089.stdout:Priority: optional 2023-06-03T19:12:08.795 INFO:teuthology.orchestra.run.smithi089.stdout:Section: kernel 2023-06-03T19:12:08.796 INFO:teuthology.orchestra.run.smithi089.stdout:Installed-Size: 20 2023-06-03T19:12:08.796 INFO:teuthology.orchestra.run.smithi089.stdout:Maintainer: Ubuntu Kernel Team 2023-06-03T19:12:08.796 INFO:teuthology.orchestra.run.smithi089.stdout:Architecture: amd64 2023-06-03T19:12:08.796 INFO:teuthology.orchestra.run.smithi089.stdout:Source: linux-meta-hwe-5.15 2023-06-03T19:12:08.796 INFO:teuthology.orchestra.run.smithi089.stdout:Version: 5.15.0.73.80~20.04.34 2023-06-03T19:12:08.796 INFO:teuthology.orchestra.run.smithi089.stdout:Provides: spl-modules (= 2.1.5-1ubuntu6~22.04.1), v4l2loopback-modules (= 0.12.7-2ubuntu2~22.04.1), virtualbox-guest-modules (= 5.15.0-73), wireguard-modules (= 1.0.0), zfs-modules (= 2.1.5-1ubuntu6~22.04.1) 2023-06-03T19:12:08.796 INFO:teuthology.orchestra.run.smithi089.stdout:Depends: linux-image-5.15.0-73-generic, linux-modules-extra-5.15.0-73-generic, linux-firmware, intel-microcode, amd64-microcode 2023-06-03T19:12:08.797 INFO:teuthology.orchestra.run.smithi089.stdout:Recommends: thermald 2023-06-03T19:12:08.797 INFO:teuthology.orchestra.run.smithi089.stdout:Description: Generic Linux kernel image 2023-06-03T19:12:08.797 INFO:teuthology.orchestra.run.smithi089.stdout: This package will always depend on the latest generic kernel image 2023-06-03T19:12:08.797 INFO:teuthology.orchestra.run.smithi089.stdout: available. 2023-06-03T19:12:08.797 INFO:teuthology.task.kernel:Not newest distro kernel. Current: 5.4.0-124-generic Expected: 5.15.0-73-generic 2023-06-03T19:12:08.798 INFO:teuthology.task.kernel:Skipping firmware on distro kernel 2023-06-03T19:12:08.798 DEBUG:teuthology.task.kernel:src is distro, skipping download 2023-06-03T19:12:08.798 INFO:teuthology.task.kernel:Installing distro kernel on mon.a... 2023-06-03T19:12:08.798 DEBUG:teuthology.task.kernel:install_kernel(remote=ubuntu@smithi089.front.sepia.ceph.com, path=None, version=distro) 2023-06-03T19:12:08.799 DEBUG:teuthology.orchestra.run.smithi089:> sudo apt-get clean 2023-06-03T19:12:08.880 DEBUG:teuthology.orchestra.run.smithi089:> sudo apt-get update 2023-06-03T19:12:09.028 INFO:teuthology.orchestra.run.smithi089.stdout:Hit:1 http://archive.ubuntu.com/ubuntu focal InRelease 2023-06-03T19:12:09.035 INFO:teuthology.orchestra.run.smithi089.stdout:Hit:2 http://archive.ubuntu.com/ubuntu focal-updates InRelease 2023-06-03T19:12:09.052 INFO:teuthology.orchestra.run.smithi089.stdout:Hit:3 http://archive.ubuntu.com/ubuntu focal-backports InRelease 2023-06-03T19:12:09.148 INFO:teuthology.orchestra.run.smithi089.stdout:Hit:4 http://security.ubuntu.com/ubuntu focal-security InRelease 2023-06-03T19:12:09.704 INFO:teuthology.orchestra.run.smithi164.stdout:/etc/kernel/postinst.d/zz-update-grub: 2023-06-03T19:12:09.705 INFO:teuthology.orchestra.run.smithi164.stdout:Sourcing file `/etc/default/grub' 2023-06-03T19:12:09.720 INFO:teuthology.orchestra.run.smithi164.stdout:Sourcing file `/etc/default/grub.d/init-select.cfg' 2023-06-03T19:12:09.746 INFO:teuthology.orchestra.run.smithi164.stdout:Generating grub configuration file ... 2023-06-03T19:12:10.156 INFO:teuthology.orchestra.run.smithi164.stdout:Found linux image: /boot/vmlinuz-5.15.0-73-generic 2023-06-03T19:12:10.177 INFO:teuthology.orchestra.run.smithi164.stdout:Found initrd image: /boot/initrd.img-5.15.0-73-generic 2023-06-03T19:12:10.673 INFO:teuthology.orchestra.run.smithi164.stdout:Found linux image: /boot/vmlinuz-5.4.0-124-generic 2023-06-03T19:12:10.684 INFO:teuthology.orchestra.run.smithi164.stdout:Found initrd image: /boot/initrd.img-5.4.0-124-generic 2023-06-03T19:12:10.729 INFO:teuthology.orchestra.run.smithi089.stdout:Reading package lists... 2023-06-03T19:12:10.754 DEBUG:teuthology.orchestra.run.smithi089:> sudo DEBIAN_FRONTEND=noninteractive apt-get -y install linux-image-generic-hwe-20.04 2023-06-03T19:12:10.821 INFO:teuthology.orchestra.run.smithi089.stdout:Reading package lists... 2023-06-03T19:12:11.015 INFO:teuthology.orchestra.run.smithi089.stdout:Building dependency tree... 2023-06-03T19:12:11.015 INFO:teuthology.orchestra.run.smithi089.stdout:Reading state information... 2023-06-03T19:12:11.220 INFO:teuthology.orchestra.run.smithi089.stdout:linux-image-generic-hwe-20.04 is already the newest version (5.15.0.73.80~20.04.34). 2023-06-03T19:12:11.221 INFO:teuthology.orchestra.run.smithi089.stdout:0 upgraded, 0 newly installed, 0 to remove and 201 not upgraded. 2023-06-03T19:12:11.222 DEBUG:teuthology.orchestra.run.smithi089:> dpkg -s linux-image-generic-hwe-20.04 2023-06-03T19:12:11.239 INFO:teuthology.orchestra.run.smithi089.stdout:Package: linux-image-generic-hwe-20.04 2023-06-03T19:12:11.239 INFO:teuthology.orchestra.run.smithi089.stdout:Status: install ok installed 2023-06-03T19:12:11.239 INFO:teuthology.orchestra.run.smithi089.stdout:Priority: optional 2023-06-03T19:12:11.240 INFO:teuthology.orchestra.run.smithi089.stdout:Section: kernel 2023-06-03T19:12:11.240 INFO:teuthology.orchestra.run.smithi089.stdout:Installed-Size: 20 2023-06-03T19:12:11.240 INFO:teuthology.orchestra.run.smithi089.stdout:Maintainer: Ubuntu Kernel Team 2023-06-03T19:12:11.240 INFO:teuthology.orchestra.run.smithi089.stdout:Architecture: amd64 2023-06-03T19:12:11.240 INFO:teuthology.orchestra.run.smithi089.stdout:Source: linux-meta-hwe-5.15 2023-06-03T19:12:11.240 INFO:teuthology.orchestra.run.smithi089.stdout:Version: 5.15.0.73.80~20.04.34 2023-06-03T19:12:11.241 INFO:teuthology.orchestra.run.smithi089.stdout:Provides: spl-modules (= 2.1.5-1ubuntu6~22.04.1), v4l2loopback-modules (= 0.12.7-2ubuntu2~22.04.1), virtualbox-guest-modules (= 5.15.0-73), wireguard-modules (= 1.0.0), zfs-modules (= 2.1.5-1ubuntu6~22.04.1) 2023-06-03T19:12:11.241 INFO:teuthology.orchestra.run.smithi089.stdout:Depends: linux-image-5.15.0-73-generic, linux-modules-extra-5.15.0-73-generic, linux-firmware, intel-microcode, amd64-microcode 2023-06-03T19:12:11.241 INFO:teuthology.orchestra.run.smithi089.stdout:Recommends: thermald 2023-06-03T19:12:11.241 INFO:teuthology.orchestra.run.smithi089.stdout:Description: Generic Linux kernel image 2023-06-03T19:12:11.241 INFO:teuthology.orchestra.run.smithi089.stdout: This package will always depend on the latest generic kernel image 2023-06-03T19:12:11.241 INFO:teuthology.orchestra.run.smithi089.stdout: available. 2023-06-03T19:12:11.242 DEBUG:teuthology.orchestra.run.smithi089:> mktemp 2023-06-03T19:12:11.249 INFO:teuthology.orchestra.run.smithi089.stdout:/tmp/tmp.h7EZLLEgMj 2023-06-03T19:12:11.250 DEBUG:teuthology.orchestra.run.smithi089:> sudo cp /boot/grub/grub.cfg /tmp/tmp.h7EZLLEgMj 2023-06-03T19:12:11.305 DEBUG:teuthology.orchestra.run.smithi089:> sudo chmod 0666 /tmp/tmp.h7EZLLEgMj 2023-06-03T19:12:11.352 DEBUG:teuthology.orchestra.remote:smithi089:/tmp/tmp.h7EZLLEgMj is 9KB 2023-06-03T19:12:11.366 DEBUG:teuthology.orchestra.run.smithi089:> rm -fr /tmp/tmp.h7EZLLEgMj 2023-06-03T19:12:11.374 DEBUG:teuthology.orchestra.run.smithi089:> sudo rm -f -- /etc/grub.d/01_ceph_kernel 2023-06-03T19:12:11.431 DEBUG:teuthology.orchestra.run.smithi089:> set -ex 2023-06-03T19:12:11.431 DEBUG:teuthology.orchestra.run.smithi089:> sudo dd of=/etc/grub.d/01_ceph_kernel 2023-06-03T19:12:11.431 DEBUG:teuthology.orchestra.run.smithi089:> sudo chmod 755 /etc/grub.d/01_ceph_kernel 2023-06-03T19:12:11.462 INFO:teuthology.task.kernel:Distro Kernel Version: 5.15.0-73-generic 2023-06-03T19:12:11.462 DEBUG:teuthology.orchestra.run.smithi089:> sudo update-grub 2023-06-03T19:12:12.670 INFO:teuthology.orchestra.run.smithi089.stderr:Sourcing file `/etc/default/grub' 2023-06-03T19:12:12.675 INFO:teuthology.orchestra.run.smithi089.stderr:Sourcing file `/etc/default/grub.d/init-select.cfg' 2023-06-03T19:12:12.687 INFO:teuthology.orchestra.run.smithi089.stderr:Generating grub configuration file ... 2023-06-03T19:12:12.747 INFO:teuthology.orchestra.run.smithi164.stdout:done 2023-06-03T19:12:12.927 INFO:teuthology.orchestra.run.smithi089.stderr:Found linux image: /boot/vmlinuz-5.15.0-73-generic 2023-06-03T19:12:12.938 INFO:teuthology.orchestra.run.smithi089.stderr:Found initrd image: /boot/initrd.img-5.15.0-73-generic 2023-06-03T19:12:13.306 DEBUG:teuthology.orchestra.run.smithi164:> dpkg -s linux-image-generic-hwe-20.04 2023-06-03T19:12:13.319 INFO:teuthology.orchestra.run.smithi089.stderr:Found linux image: /boot/vmlinuz-5.4.0-124-generic 2023-06-03T19:12:13.330 INFO:teuthology.orchestra.run.smithi089.stderr:Found initrd image: /boot/initrd.img-5.4.0-124-generic 2023-06-03T19:12:13.345 INFO:teuthology.orchestra.run.smithi164.stdout:Package: linux-image-generic-hwe-20.04 2023-06-03T19:12:13.345 INFO:teuthology.orchestra.run.smithi164.stdout:Status: install ok installed 2023-06-03T19:12:13.345 INFO:teuthology.orchestra.run.smithi164.stdout:Priority: optional 2023-06-03T19:12:13.346 INFO:teuthology.orchestra.run.smithi164.stdout:Section: kernel 2023-06-03T19:12:13.346 INFO:teuthology.orchestra.run.smithi164.stdout:Installed-Size: 20 2023-06-03T19:12:13.346 INFO:teuthology.orchestra.run.smithi164.stdout:Maintainer: Ubuntu Kernel Team 2023-06-03T19:12:13.346 INFO:teuthology.orchestra.run.smithi164.stdout:Architecture: amd64 2023-06-03T19:12:13.347 INFO:teuthology.orchestra.run.smithi164.stdout:Source: linux-meta-hwe-5.15 2023-06-03T19:12:13.347 INFO:teuthology.orchestra.run.smithi164.stdout:Version: 5.15.0.73.80~20.04.34 2023-06-03T19:12:13.347 INFO:teuthology.orchestra.run.smithi164.stdout:Provides: spl-modules (= 2.1.5-1ubuntu6~22.04.1), v4l2loopback-modules (= 0.12.7-2ubuntu2~22.04.1), virtualbox-guest-modules (= 5.15.0-73), wireguard-modules (= 1.0.0), zfs-modules (= 2.1.5-1ubuntu6~22.04.1) 2023-06-03T19:12:13.347 INFO:teuthology.orchestra.run.smithi164.stdout:Depends: linux-image-5.15.0-73-generic, linux-modules-extra-5.15.0-73-generic, linux-firmware, intel-microcode, amd64-microcode 2023-06-03T19:12:13.347 INFO:teuthology.orchestra.run.smithi164.stdout:Recommends: thermald 2023-06-03T19:12:13.348 INFO:teuthology.orchestra.run.smithi164.stdout:Description: Generic Linux kernel image 2023-06-03T19:12:13.348 INFO:teuthology.orchestra.run.smithi164.stdout: This package will always depend on the latest generic kernel image 2023-06-03T19:12:13.348 INFO:teuthology.orchestra.run.smithi164.stdout: available. 2023-06-03T19:12:13.348 INFO:teuthology.task.kernel:Not newest distro kernel. Current: 5.4.0-124-generic Expected: 5.15.0-73-generic 2023-06-03T19:12:13.348 INFO:teuthology.task.kernel:Skipping firmware on distro kernel 2023-06-03T19:12:13.349 DEBUG:teuthology.task.kernel:src is distro, skipping download 2023-06-03T19:12:13.349 INFO:teuthology.task.kernel:Installing distro kernel on mon.b... 2023-06-03T19:12:13.349 DEBUG:teuthology.task.kernel:install_kernel(remote=ubuntu@smithi164.front.sepia.ceph.com, path=None, version=distro) 2023-06-03T19:12:13.349 DEBUG:teuthology.orchestra.run.smithi164:> sudo apt-get clean 2023-06-03T19:12:13.471 DEBUG:teuthology.orchestra.run.smithi164:> sudo apt-get update 2023-06-03T19:12:13.601 INFO:teuthology.orchestra.run.smithi164.stdout:Hit:1 http://security.ubuntu.com/ubuntu focal-security InRelease 2023-06-03T19:12:13.719 INFO:teuthology.orchestra.run.smithi164.stdout:Hit:2 http://archive.ubuntu.com/ubuntu focal InRelease 2023-06-03T19:12:13.801 INFO:teuthology.orchestra.run.smithi164.stdout:Hit:3 http://archive.ubuntu.com/ubuntu focal-updates InRelease 2023-06-03T19:12:13.883 INFO:teuthology.orchestra.run.smithi164.stdout:Hit:4 http://archive.ubuntu.com/ubuntu focal-backports InRelease 2023-06-03T19:12:14.357 INFO:teuthology.orchestra.run.smithi089.stderr:done 2023-06-03T19:12:14.358 DEBUG:teuthology.orchestra.run.smithi089:> sudo shutdown -r now 2023-06-03T19:12:15.304 INFO:teuthology.orchestra.run.smithi164.stdout:Reading package lists... 2023-06-03T19:12:15.327 DEBUG:teuthology.orchestra.run.smithi164:> sudo DEBIAN_FRONTEND=noninteractive apt-get -y install linux-image-generic-hwe-20.04 2023-06-03T19:12:15.399 INFO:teuthology.orchestra.run.smithi164.stdout:Reading package lists... 2023-06-03T19:12:15.597 INFO:teuthology.orchestra.run.smithi164.stdout:Building dependency tree... 2023-06-03T19:12:15.598 INFO:teuthology.orchestra.run.smithi164.stdout:Reading state information... 2023-06-03T19:12:15.795 INFO:teuthology.orchestra.run.smithi164.stdout:linux-image-generic-hwe-20.04 is already the newest version (5.15.0.73.80~20.04.34). 2023-06-03T19:12:15.795 INFO:teuthology.orchestra.run.smithi164.stdout:0 upgraded, 0 newly installed, 0 to remove and 201 not upgraded. 2023-06-03T19:12:15.796 DEBUG:teuthology.orchestra.run.smithi164:> dpkg -s linux-image-generic-hwe-20.04 2023-06-03T19:12:15.816 INFO:teuthology.orchestra.run.smithi164.stdout:Package: linux-image-generic-hwe-20.04 2023-06-03T19:12:15.816 INFO:teuthology.orchestra.run.smithi164.stdout:Status: install ok installed 2023-06-03T19:12:15.816 INFO:teuthology.orchestra.run.smithi164.stdout:Priority: optional 2023-06-03T19:12:15.817 INFO:teuthology.orchestra.run.smithi164.stdout:Section: kernel 2023-06-03T19:12:15.817 INFO:teuthology.orchestra.run.smithi164.stdout:Installed-Size: 20 2023-06-03T19:12:15.817 INFO:teuthology.orchestra.run.smithi164.stdout:Maintainer: Ubuntu Kernel Team 2023-06-03T19:12:15.817 INFO:teuthology.orchestra.run.smithi164.stdout:Architecture: amd64 2023-06-03T19:12:15.817 INFO:teuthology.orchestra.run.smithi164.stdout:Source: linux-meta-hwe-5.15 2023-06-03T19:12:15.817 INFO:teuthology.orchestra.run.smithi164.stdout:Version: 5.15.0.73.80~20.04.34 2023-06-03T19:12:15.817 INFO:teuthology.orchestra.run.smithi164.stdout:Provides: spl-modules (= 2.1.5-1ubuntu6~22.04.1), v4l2loopback-modules (= 0.12.7-2ubuntu2~22.04.1), virtualbox-guest-modules (= 5.15.0-73), wireguard-modules (= 1.0.0), zfs-modules (= 2.1.5-1ubuntu6~22.04.1) 2023-06-03T19:12:15.818 INFO:teuthology.orchestra.run.smithi164.stdout:Depends: linux-image-5.15.0-73-generic, linux-modules-extra-5.15.0-73-generic, linux-firmware, intel-microcode, amd64-microcode 2023-06-03T19:12:15.818 INFO:teuthology.orchestra.run.smithi164.stdout:Recommends: thermald 2023-06-03T19:12:15.818 INFO:teuthology.orchestra.run.smithi164.stdout:Description: Generic Linux kernel image 2023-06-03T19:12:15.818 INFO:teuthology.orchestra.run.smithi164.stdout: This package will always depend on the latest generic kernel image 2023-06-03T19:12:15.818 INFO:teuthology.orchestra.run.smithi164.stdout: available. 2023-06-03T19:12:15.818 DEBUG:teuthology.orchestra.run.smithi164:> mktemp 2023-06-03T19:12:15.824 INFO:teuthology.orchestra.run.smithi164.stdout:/tmp/tmp.LCcsFttQt9 2023-06-03T19:12:15.825 DEBUG:teuthology.orchestra.run.smithi164:> sudo cp /boot/grub/grub.cfg /tmp/tmp.LCcsFttQt9 2023-06-03T19:12:15.838 DEBUG:teuthology.orchestra.run.smithi164:> sudo chmod 0666 /tmp/tmp.LCcsFttQt9 2023-06-03T19:12:15.925 DEBUG:teuthology.orchestra.remote:smithi164:/tmp/tmp.LCcsFttQt9 is 9KB 2023-06-03T19:12:15.938 DEBUG:teuthology.orchestra.run.smithi164:> rm -fr /tmp/tmp.LCcsFttQt9 2023-06-03T19:12:15.945 DEBUG:teuthology.orchestra.run.smithi164:> sudo rm -f -- /etc/grub.d/01_ceph_kernel 2023-06-03T19:12:16.007 DEBUG:teuthology.orchestra.run.smithi164:> set -ex 2023-06-03T19:12:16.007 DEBUG:teuthology.orchestra.run.smithi164:> sudo dd of=/etc/grub.d/01_ceph_kernel 2023-06-03T19:12:16.007 DEBUG:teuthology.orchestra.run.smithi164:> sudo chmod 755 /etc/grub.d/01_ceph_kernel 2023-06-03T19:12:16.047 INFO:teuthology.task.kernel:Distro Kernel Version: 5.15.0-73-generic 2023-06-03T19:12:16.047 DEBUG:teuthology.orchestra.run.smithi164:> sudo update-grub 2023-06-03T19:12:17.461 INFO:teuthology.orchestra.run.smithi164.stderr:Sourcing file `/etc/default/grub' 2023-06-03T19:12:17.468 INFO:teuthology.orchestra.run.smithi164.stderr:Sourcing file `/etc/default/grub.d/init-select.cfg' 2023-06-03T19:12:17.479 INFO:teuthology.orchestra.run.smithi164.stderr:Generating grub configuration file ... 2023-06-03T19:12:17.801 INFO:teuthology.orchestra.run.smithi164.stderr:Found linux image: /boot/vmlinuz-5.15.0-73-generic 2023-06-03T19:12:17.814 INFO:teuthology.orchestra.run.smithi164.stderr:Found initrd image: /boot/initrd.img-5.15.0-73-generic 2023-06-03T19:12:18.253 INFO:teuthology.orchestra.run.smithi164.stderr:Found linux image: /boot/vmlinuz-5.4.0-124-generic 2023-06-03T19:12:18.264 INFO:teuthology.orchestra.run.smithi164.stderr:Found initrd image: /boot/initrd.img-5.4.0-124-generic 2023-06-03T19:12:19.276 INFO:teuthology.orchestra.run.smithi164.stderr:done 2023-06-03T19:12:19.277 DEBUG:teuthology.orchestra.run.smithi164:> sudo shutdown -r now 2023-06-03T19:12:44.384 INFO:teuthology.task.kernel:Checking client mon.a for new kernel version... 2023-06-03T19:12:44.385 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi089.front.sepia.ceph.com' 2023-06-03T19:12:44.385 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi089.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2023-06-03T19:12:49.286 INFO:teuthology.task.kernel:Checking client mon.b for new kernel version... 2023-06-03T19:12:49.287 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi164.front.sepia.ceph.com' 2023-06-03T19:12:49.288 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi164.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2023-06-03T19:13:18.960 DEBUG:teuthology.orchestra.remote:[Errno None] Unable to connect to port 22 on 172.21.15.89 2023-06-03T19:13:23.824 DEBUG:teuthology.orchestra.remote:[Errno None] Unable to connect to port 22 on 172.21.15.164 2023-06-03T19:13:48.984 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi089.front.sepia.ceph.com' 2023-06-03T19:13:48.985 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi089.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2023-06-03T19:13:52.016 DEBUG:teuthology.orchestra.remote:[Errno None] Unable to connect to port 22 on 172.21.15.89 2023-06-03T19:13:53.825 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi164.front.sepia.ceph.com' 2023-06-03T19:13:53.826 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi164.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2023-06-03T19:14:09.140 DEBUG:teuthology.orchestra.remote:[Errno None] Unable to connect to port 22 on 172.21.15.164 2023-06-03T19:14:22.028 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi089.front.sepia.ceph.com' 2023-06-03T19:14:22.029 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi089.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2023-06-03T19:14:22.335 DEBUG:teuthology.orchestra.run.smithi089:> true 2023-06-03T19:14:22.875 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi089.front.sepia.ceph.com' 2023-06-03T19:14:22.876 INFO:teuthology.task.kernel:Checking kernel version of mon.a, want "5.15.0-73-generic"... 2023-06-03T19:14:22.876 DEBUG:teuthology.orchestra.run.smithi089:> uname -r 2023-06-03T19:14:22.932 INFO:teuthology.orchestra.run.smithi089.stdout:5.15.0-73-generic 2023-06-03T19:14:22.933 DEBUG:teuthology.task.kernel:current kernel version is 5.15.0-73-generic vs 5.15.0-73-generic 2023-06-03T19:14:22.933 DEBUG:teuthology.task.kernel:utsrelease strings match, do not need to install 2023-06-03T19:14:22.933 DEBUG:teuthology.task.kernel:Distro of this test job: ubuntu 2023-06-03T19:14:23.934 INFO:teuthology.task.kernel:Enabling kdb on mon.a... 2023-06-03T19:14:23.934 DEBUG:teuthology.orchestra.run.smithi089:> echo ttyS1 | sudo tee /sys/module/kgdboc/parameters/kgdboc 2023-06-03T19:14:24.034 INFO:teuthology.orchestra.run.smithi089.stdout:ttyS1 2023-06-03T19:14:24.067 DEBUG:teuthology.parallel:result is None 2023-06-03T19:14:39.141 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi164.front.sepia.ceph.com' 2023-06-03T19:14:39.142 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi164.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2023-06-03T19:14:39.374 DEBUG:teuthology.orchestra.run.smithi164:> true 2023-06-03T19:14:39.866 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi164.front.sepia.ceph.com' 2023-06-03T19:14:39.867 INFO:teuthology.task.kernel:Checking kernel version of mon.b, want "5.15.0-73-generic"... 2023-06-03T19:14:39.867 DEBUG:teuthology.orchestra.run.smithi164:> uname -r 2023-06-03T19:14:39.914 INFO:teuthology.orchestra.run.smithi164.stdout:5.15.0-73-generic 2023-06-03T19:14:39.915 DEBUG:teuthology.task.kernel:current kernel version is 5.15.0-73-generic vs 5.15.0-73-generic 2023-06-03T19:14:39.915 DEBUG:teuthology.task.kernel:utsrelease strings match, do not need to install 2023-06-03T19:14:39.916 DEBUG:teuthology.task.kernel:Distro of this test job: ubuntu 2023-06-03T19:14:40.917 INFO:teuthology.task.kernel:Enabling kdb on mon.b... 2023-06-03T19:14:40.917 DEBUG:teuthology.orchestra.run.smithi164:> echo ttyS1 | sudo tee /sys/module/kgdboc/parameters/kgdboc 2023-06-03T19:14:41.258 INFO:teuthology.orchestra.run.smithi164.stdout:ttyS1 2023-06-03T19:14:41.287 DEBUG:teuthology.parallel:result is None 2023-06-03T19:14:41.287 INFO:teuthology.run_tasks:Running task internal.base... 2023-06-03T19:14:41.294 INFO:teuthology.task.internal:Creating test directory... 2023-06-03T19:14:41.294 DEBUG:teuthology.orchestra.run.smithi089:> mkdir -p -m0755 -- /home/ubuntu/cephtest 2023-06-03T19:14:41.297 DEBUG:teuthology.orchestra.run.smithi164:> mkdir -p -m0755 -- /home/ubuntu/cephtest 2023-06-03T19:14:41.303 INFO:teuthology.run_tasks:Running task internal.archive_upload... 2023-06-03T19:14:41.312 INFO:teuthology.run_tasks:Running task internal.archive... 2023-06-03T19:14:41.326 INFO:teuthology.task.internal:Creating archive directory... 2023-06-03T19:14:41.327 DEBUG:teuthology.orchestra.run.smithi089:> install -d -m0755 -- /home/ubuntu/cephtest/archive 2023-06-03T19:14:41.346 DEBUG:teuthology.orchestra.run.smithi164:> install -d -m0755 -- /home/ubuntu/cephtest/archive 2023-06-03T19:14:41.516 INFO:teuthology.run_tasks:Running task internal.coredump... 2023-06-03T19:14:41.532 INFO:teuthology.task.internal:Enabling coredump saving... 2023-06-03T19:14:41.533 DEBUG:teuthology.orchestra.run.smithi089:> install -d -m0755 -- /home/ubuntu/cephtest/archive/coredump && sudo sysctl -w kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core && echo kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core | sudo tee -a /etc/sysctl.conf 2023-06-03T19:14:41.538 DEBUG:teuthology.orchestra.run.smithi164:> install -d -m0755 -- /home/ubuntu/cephtest/archive/coredump && sudo sysctl -w kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core && echo kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core | sudo tee -a /etc/sysctl.conf 2023-06-03T19:14:41.554 INFO:teuthology.orchestra.run.smithi089.stdout:kernel.core_pattern = /home/ubuntu/cephtest/archive/coredump/%t.%p.core 2023-06-03T19:14:41.558 INFO:teuthology.orchestra.run.smithi164.stdout:kernel.core_pattern = /home/ubuntu/cephtest/archive/coredump/%t.%p.core 2023-06-03T19:14:41.560 INFO:teuthology.orchestra.run.smithi089.stdout:kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core 2023-06-03T19:14:41.564 INFO:teuthology.orchestra.run.smithi164.stdout:kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core 2023-06-03T19:14:41.566 INFO:teuthology.run_tasks:Running task internal.sudo... 2023-06-03T19:14:41.580 INFO:teuthology.task.internal:Configuring sudo... 2023-06-03T19:14:41.581 DEBUG:teuthology.orchestra.run.smithi089:> sudo sed -i.orig.teuthology -e 's/^\([^#]*\) \(requiretty\)/\1 !\2/g' -e 's/^\([^#]*\) !\(visiblepw\)/\1 \2/g' /etc/sudoers 2023-06-03T19:14:41.607 DEBUG:teuthology.orchestra.run.smithi164:> sudo sed -i.orig.teuthology -e 's/^\([^#]*\) \(requiretty\)/\1 !\2/g' -e 's/^\([^#]*\) !\(visiblepw\)/\1 \2/g' /etc/sudoers 2023-06-03T19:14:41.625 INFO:teuthology.run_tasks:Running task internal.syslog... 2023-06-03T19:14:41.647 INFO:teuthology.task.internal.syslog:Starting syslog monitoring... 2023-06-03T19:14:41.647 DEBUG:teuthology.orchestra.run.smithi089:> mkdir -p -m0755 -- /home/ubuntu/cephtest/archive/syslog 2023-06-03T19:14:41.666 DEBUG:teuthology.orchestra.run.smithi164:> mkdir -p -m0755 -- /home/ubuntu/cephtest/archive/syslog 2023-06-03T19:14:41.674 DEBUG:teuthology.orchestra.run.smithi089:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/kern.log 2023-06-03T19:14:41.719 DEBUG:teuthology.orchestra.run.smithi089:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/misc.log 2023-06-03T19:14:41.768 DEBUG:teuthology.orchestra.run.smithi089:> set -ex 2023-06-03T19:14:41.768 DEBUG:teuthology.orchestra.run.smithi089:> sudo dd of=/etc/rsyslog.d/80-cephtest.conf 2023-06-03T19:14:41.837 DEBUG:teuthology.orchestra.run.smithi164:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/kern.log 2023-06-03T19:14:41.845 DEBUG:teuthology.orchestra.run.smithi164:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/misc.log 2023-06-03T19:14:41.895 DEBUG:teuthology.orchestra.run.smithi164:> set -ex 2023-06-03T19:14:41.895 DEBUG:teuthology.orchestra.run.smithi164:> sudo dd of=/etc/rsyslog.d/80-cephtest.conf 2023-06-03T19:14:41.957 DEBUG:teuthology.orchestra.run.smithi089:> sudo service rsyslog restart 2023-06-03T19:14:41.959 DEBUG:teuthology.orchestra.run.smithi164:> sudo service rsyslog restart 2023-06-03T19:14:42.047 INFO:teuthology.run_tasks:Running task internal.timer... 2023-06-03T19:14:42.056 INFO:teuthology.task.internal:Starting timer... 2023-06-03T19:14:42.056 INFO:teuthology.run_tasks:Running task pcp... 2023-06-03T19:14:42.112 INFO:teuthology.run_tasks:Running task selinux... 2023-06-03T19:14:42.204 DEBUG:teuthology.task.selinux:Excluding smithi089: OS 'ubuntu' does not support SELinux 2023-06-03T19:14:42.204 DEBUG:teuthology.task.selinux:Excluding smithi164: OS 'ubuntu' does not support SELinux 2023-06-03T19:14:42.204 DEBUG:teuthology.task.selinux:Getting current SELinux state 2023-06-03T19:14:42.205 DEBUG:teuthology.task.selinux:Existing SELinux modes: {} 2023-06-03T19:14:42.205 INFO:teuthology.task.selinux:Putting SELinux into permissive mode 2023-06-03T19:14:42.205 INFO:teuthology.run_tasks:Running task ansible.cephlab... 2023-06-03T19:14:42.266 DEBUG:teuthology.repo_utils:Setting repo remote to https://git.ceph.com/ceph-cm-ansible.git 2023-06-03T19:14:42.279 INFO:teuthology.repo_utils:Fetching main from origin 2023-06-03T19:14:42.489 INFO:teuthology.repo_utils:Resetting repo at /home/teuthworker/src/git.ceph.com_ceph-cm-ansible_main to origin/main 2023-06-03T19:14:42.744 INFO:teuthology.task.ansible:Playbook: [{'import_playbook': 'ansible_managed.yml'}, {'import_playbook': 'teuthology.yml'}, {'hosts': 'testnodes', 'tasks': [{'set_fact': {'ran_from_cephlab_playbook': True}}]}, {'import_playbook': 'testnodes.yml'}, {'import_playbook': 'container-host.yml'}, {'import_playbook': 'cobbler.yml'}, {'import_playbook': 'paddles.yml'}, {'import_playbook': 'pulpito.yml'}, {'hosts': 'testnodes', 'become': True, 'tasks': [{'name': 'Touch /ceph-qa-ready', 'file': {'path': '/ceph-qa-ready', 'state': 'touch'}, 'when': 'ran_from_cephlab_playbook|bool'}]}] 2023-06-03T19:14:42.745 DEBUG:teuthology.task.ansible:Running ansible-playbook -v --extra-vars '{"ansible_ssh_user": "ubuntu"}' -i /etc/ansible/hosts --limit smithi089.front.sepia.ceph.com,smithi164.front.sepia.ceph.com /home/teuthworker/src/git.ceph.com_ceph-cm-ansible_main/cephlab.yml 2023-06-03T19:20:16.892 DEBUG:teuthology.task.ansible:Reconnecting to [Remote(name='ubuntu@smithi089.front.sepia.ceph.com'), Remote(name='ubuntu@smithi164.front.sepia.ceph.com')] 2023-06-03T19:20:16.894 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi089.front.sepia.ceph.com' 2023-06-03T19:20:16.895 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi089.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2023-06-03T19:20:16.972 DEBUG:teuthology.orchestra.run.smithi089:> true 2023-06-03T19:20:17.094 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi164.front.sepia.ceph.com' 2023-06-03T19:20:17.095 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi164.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2023-06-03T19:20:17.174 DEBUG:teuthology.orchestra.run.smithi164:> true 2023-06-03T19:20:17.290 INFO:teuthology.run_tasks:Running task clock... 2023-06-03T19:20:17.309 INFO:teuthology.task.clock:Syncing clocks and checking initial clock skew... 2023-06-03T19:20:17.309 INFO:teuthology.orchestra.run:Running command with timeout 360 2023-06-03T19:20:17.310 DEBUG:teuthology.orchestra.run.smithi089:> sudo systemctl stop ntp.service || sudo systemctl stop ntpd.service || sudo systemctl stop chronyd.service ; sudo ntpd -gq || sudo chronyc makestep ; sudo systemctl start ntp.service || sudo systemctl start ntpd.service || sudo systemctl start chronyd.service ; PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2023-06-03T19:20:17.312 INFO:teuthology.orchestra.run:Running command with timeout 360 2023-06-03T19:20:17.313 DEBUG:teuthology.orchestra.run.smithi164:> sudo systemctl stop ntp.service || sudo systemctl stop ntpd.service || sudo systemctl stop chronyd.service ; sudo ntpd -gq || sudo chronyc makestep ; sudo systemctl start ntp.service || sudo systemctl start ntpd.service || sudo systemctl start chronyd.service ; PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2023-06-03T19:20:17.343 INFO:teuthology.orchestra.run.smithi089.stdout: 3 Jun 19:20:17 ntpd[21671]: ntpd 4.2.8p12@1.3728-o (1): Starting 2023-06-03T19:20:17.343 INFO:teuthology.orchestra.run.smithi089.stdout: 3 Jun 19:20:17 ntpd[21671]: Command line: ntpd -gq 2023-06-03T19:20:17.343 INFO:teuthology.orchestra.run.smithi089.stdout: 3 Jun 19:20:17 ntpd[21671]: proto: precision = 0.083 usec (-23) 2023-06-03T19:20:17.344 INFO:teuthology.orchestra.run.smithi089.stderr:restrict 0.0.0.0: KOD does nothing without LIMITED. 2023-06-03T19:20:17.344 INFO:teuthology.orchestra.run.smithi089.stdout: 3 Jun 19:20:17 ntpd[21671]: restrict 0.0.0.0: KOD does nothing without LIMITED. 2023-06-03T19:20:17.345 INFO:teuthology.orchestra.run.smithi089.stderr:restrict ::: KOD does nothing without LIMITED. 2023-06-03T19:20:17.345 INFO:teuthology.orchestra.run.smithi089.stdout: 3 Jun 19:20:17 ntpd[21671]: restrict ::: KOD does nothing without LIMITED. 2023-06-03T19:20:17.346 INFO:teuthology.orchestra.run.smithi089.stdout: 3 Jun 19:20:17 ntpd[21671]: Listen and drop on 0 v6wildcard [::]:123 2023-06-03T19:20:17.346 INFO:teuthology.orchestra.run.smithi089.stdout: 3 Jun 19:20:17 ntpd[21671]: Listen and drop on 1 v4wildcard 0.0.0.0:123 2023-06-03T19:20:17.346 INFO:teuthology.orchestra.run.smithi089.stdout: 3 Jun 19:20:17 ntpd[21671]: Listen normally on 2 lo 127.0.0.1:123 2023-06-03T19:20:17.346 INFO:teuthology.orchestra.run.smithi089.stdout: 3 Jun 19:20:17 ntpd[21671]: Listen normally on 3 enp3s0f1 172.21.15.89:123 2023-06-03T19:20:17.346 INFO:teuthology.orchestra.run.smithi089.stdout: 3 Jun 19:20:17 ntpd[21671]: Listen normally on 4 lo [::1]:123 2023-06-03T19:20:17.346 INFO:teuthology.orchestra.run.smithi089.stdout: 3 Jun 19:20:17 ntpd[21671]: Listen normally on 5 enp3s0f1 [fe80::ec4:7aff:fe8f:d501%5]:123 2023-06-03T19:20:17.347 INFO:teuthology.orchestra.run.smithi089.stdout: 3 Jun 19:20:17 ntpd[21671]: Listening on routing socket on fd #22 for interface updates 2023-06-03T19:20:17.363 INFO:teuthology.orchestra.run.smithi164.stdout: 3 Jun 19:20:17 ntpd[21698]: ntpd 4.2.8p12@1.3728-o (1): Starting 2023-06-03T19:20:17.363 INFO:teuthology.orchestra.run.smithi164.stdout: 3 Jun 19:20:17 ntpd[21698]: Command line: ntpd -gq 2023-06-03T19:20:17.363 INFO:teuthology.orchestra.run.smithi164.stdout: 3 Jun 19:20:17 ntpd[21698]: proto: precision = 0.100 usec (-23) 2023-06-03T19:20:17.364 INFO:teuthology.orchestra.run.smithi164.stderr:restrict 0.0.0.0: KOD does nothing without LIMITED. 2023-06-03T19:20:17.364 INFO:teuthology.orchestra.run.smithi164.stdout: 3 Jun 19:20:17 ntpd[21698]: restrict 0.0.0.0: KOD does nothing without LIMITED. 2023-06-03T19:20:17.365 INFO:teuthology.orchestra.run.smithi164.stderr:restrict ::: KOD does nothing without LIMITED. 2023-06-03T19:20:17.365 INFO:teuthology.orchestra.run.smithi164.stdout: 3 Jun 19:20:17 ntpd[21698]: restrict ::: KOD does nothing without LIMITED. 2023-06-03T19:20:17.366 INFO:teuthology.orchestra.run.smithi164.stdout: 3 Jun 19:20:17 ntpd[21698]: Listen and drop on 0 v6wildcard [::]:123 2023-06-03T19:20:17.366 INFO:teuthology.orchestra.run.smithi164.stdout: 3 Jun 19:20:17 ntpd[21698]: Listen and drop on 1 v4wildcard 0.0.0.0:123 2023-06-03T19:20:17.366 INFO:teuthology.orchestra.run.smithi164.stdout: 3 Jun 19:20:17 ntpd[21698]: Listen normally on 2 lo 127.0.0.1:123 2023-06-03T19:20:17.366 INFO:teuthology.orchestra.run.smithi164.stdout: 3 Jun 19:20:17 ntpd[21698]: Listen normally on 3 enp3s0f1 172.21.15.164:123 2023-06-03T19:20:17.366 INFO:teuthology.orchestra.run.smithi164.stdout: 3 Jun 19:20:17 ntpd[21698]: Listen normally on 4 lo [::1]:123 2023-06-03T19:20:17.366 INFO:teuthology.orchestra.run.smithi164.stdout: 3 Jun 19:20:17 ntpd[21698]: Listen normally on 5 enp3s0f1 [fe80::ec4:7aff:fe88:73f3%5]:123 2023-06-03T19:20:17.366 INFO:teuthology.orchestra.run.smithi164.stdout: 3 Jun 19:20:17 ntpd[21698]: Listening on routing socket on fd #22 for interface updates 2023-06-03T19:20:18.344 INFO:teuthology.orchestra.run.smithi089.stderr: 3 Jun 19:20:18 ntpd[21671]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2023-06-03T19:20:18.344 INFO:teuthology.orchestra.run.smithi089.stderr: 3 Jun 19:20:18 ntpd[21671]: can't open /var/log/ntpstats/rawstats.20230603: Permission denied 2023-06-03T19:20:18.345 INFO:teuthology.orchestra.run.smithi089.stderr: 3 Jun 19:20:18 ntpd[21671]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2023-06-03T19:20:18.345 INFO:teuthology.orchestra.run.smithi089.stderr: 3 Jun 19:20:18 ntpd[21671]: can't open /var/log/ntpstats/rawstats.20230603: Permission denied 2023-06-03T19:20:18.345 INFO:teuthology.orchestra.run.smithi089.stderr: 3 Jun 19:20:18 ntpd[21671]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2023-06-03T19:20:18.345 INFO:teuthology.orchestra.run.smithi089.stderr: 3 Jun 19:20:18 ntpd[21671]: can't open /var/log/ntpstats/rawstats.20230603: Permission denied 2023-06-03T19:20:18.345 INFO:teuthology.orchestra.run.smithi089.stderr: 3 Jun 19:20:18 ntpd[21671]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2023-06-03T19:20:18.345 INFO:teuthology.orchestra.run.smithi089.stderr: 3 Jun 19:20:18 ntpd[21671]: can't open /var/log/ntpstats/rawstats.20230603: Permission denied 2023-06-03T19:20:18.345 INFO:teuthology.orchestra.run.smithi089.stderr: 3 Jun 19:20:18 ntpd[21671]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2023-06-03T19:20:18.346 INFO:teuthology.orchestra.run.smithi089.stderr: 3 Jun 19:20:18 ntpd[21671]: can't open /var/log/ntpstats/peerstats.20230603: Permission denied 2023-06-03T19:20:18.346 INFO:teuthology.orchestra.run.smithi089.stderr: 3 Jun 19:20:18 ntpd[21671]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2023-06-03T19:20:18.346 INFO:teuthology.orchestra.run.smithi089.stderr: 3 Jun 19:20:18 ntpd[21671]: can't open /var/log/ntpstats/rawstats.20230603: Permission denied 2023-06-03T19:20:18.346 INFO:teuthology.orchestra.run.smithi089.stderr: 3 Jun 19:20:18 ntpd[21671]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2023-06-03T19:20:18.346 INFO:teuthology.orchestra.run.smithi089.stderr: 3 Jun 19:20:18 ntpd[21671]: can't open /var/log/ntpstats/peerstats.20230603: Permission denied 2023-06-03T19:20:18.363 INFO:teuthology.orchestra.run.smithi164.stderr: 3 Jun 19:20:18 ntpd[21698]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2023-06-03T19:20:18.363 INFO:teuthology.orchestra.run.smithi164.stderr: 3 Jun 19:20:18 ntpd[21698]: can't open /var/log/ntpstats/rawstats.20230603: Permission denied 2023-06-03T19:20:18.364 INFO:teuthology.orchestra.run.smithi164.stderr: 3 Jun 19:20:18 ntpd[21698]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2023-06-03T19:20:18.364 INFO:teuthology.orchestra.run.smithi164.stderr: 3 Jun 19:20:18 ntpd[21698]: can't open /var/log/ntpstats/rawstats.20230603: Permission denied 2023-06-03T19:20:18.364 INFO:teuthology.orchestra.run.smithi164.stderr: 3 Jun 19:20:18 ntpd[21698]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2023-06-03T19:20:18.364 INFO:teuthology.orchestra.run.smithi164.stderr: 3 Jun 19:20:18 ntpd[21698]: can't open /var/log/ntpstats/peerstats.20230603: Permission denied 2023-06-03T19:20:20.344 INFO:teuthology.orchestra.run.smithi089.stderr: 3 Jun 19:20:20 ntpd[21671]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2023-06-03T19:20:20.345 INFO:teuthology.orchestra.run.smithi089.stderr: 3 Jun 19:20:20 ntpd[21671]: can't open /var/log/ntpstats/rawstats.20230603: Permission denied 2023-06-03T19:20:20.345 INFO:teuthology.orchestra.run.smithi089.stderr: 3 Jun 19:20:20 ntpd[21671]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2023-06-03T19:20:20.345 INFO:teuthology.orchestra.run.smithi089.stderr: 3 Jun 19:20:20 ntpd[21671]: can't open /var/log/ntpstats/rawstats.20230603: Permission denied 2023-06-03T19:20:20.345 INFO:teuthology.orchestra.run.smithi089.stderr: 3 Jun 19:20:20 ntpd[21671]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2023-06-03T19:20:20.345 INFO:teuthology.orchestra.run.smithi089.stderr: 3 Jun 19:20:20 ntpd[21671]: can't open /var/log/ntpstats/rawstats.20230603: Permission denied 2023-06-03T19:20:20.346 INFO:teuthology.orchestra.run.smithi089.stderr: 3 Jun 19:20:20 ntpd[21671]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2023-06-03T19:20:20.346 INFO:teuthology.orchestra.run.smithi089.stderr: 3 Jun 19:20:20 ntpd[21671]: can't open /var/log/ntpstats/rawstats.20230603: Permission denied 2023-06-03T19:20:20.346 INFO:teuthology.orchestra.run.smithi089.stderr: 3 Jun 19:20:20 ntpd[21671]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2023-06-03T19:20:20.346 INFO:teuthology.orchestra.run.smithi089.stderr: 3 Jun 19:20:20 ntpd[21671]: can't open /var/log/ntpstats/peerstats.20230603: Permission denied 2023-06-03T19:20:20.346 INFO:teuthology.orchestra.run.smithi089.stderr: 3 Jun 19:20:20 ntpd[21671]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2023-06-03T19:20:20.346 INFO:teuthology.orchestra.run.smithi089.stderr: 3 Jun 19:20:20 ntpd[21671]: can't open /var/log/ntpstats/rawstats.20230603: Permission denied 2023-06-03T19:20:20.346 INFO:teuthology.orchestra.run.smithi089.stderr: 3 Jun 19:20:20 ntpd[21671]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2023-06-03T19:20:20.347 INFO:teuthology.orchestra.run.smithi089.stderr: 3 Jun 19:20:20 ntpd[21671]: can't open /var/log/ntpstats/peerstats.20230603: Permission denied 2023-06-03T19:20:20.347 INFO:teuthology.orchestra.run.smithi089.stderr: 3 Jun 19:20:20 ntpd[21671]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2023-06-03T19:20:20.347 INFO:teuthology.orchestra.run.smithi089.stderr: 3 Jun 19:20:20 ntpd[21671]: can't open /var/log/ntpstats/rawstats.20230603: Permission denied 2023-06-03T19:20:20.347 INFO:teuthology.orchestra.run.smithi089.stderr: 3 Jun 19:20:20 ntpd[21671]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2023-06-03T19:20:20.348 INFO:teuthology.orchestra.run.smithi089.stderr: 3 Jun 19:20:20 ntpd[21671]: can't open /var/log/ntpstats/peerstats.20230603: Permission denied 2023-06-03T19:20:20.363 INFO:teuthology.orchestra.run.smithi164.stderr: 3 Jun 19:20:20 ntpd[21698]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2023-06-03T19:20:20.364 INFO:teuthology.orchestra.run.smithi164.stderr: 3 Jun 19:20:20 ntpd[21698]: can't open /var/log/ntpstats/rawstats.20230603: Permission denied 2023-06-03T19:20:20.364 INFO:teuthology.orchestra.run.smithi164.stderr: 3 Jun 19:20:20 ntpd[21698]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2023-06-03T19:20:20.364 INFO:teuthology.orchestra.run.smithi164.stderr: 3 Jun 19:20:20 ntpd[21698]: can't open /var/log/ntpstats/rawstats.20230603: Permission denied 2023-06-03T19:20:20.364 INFO:teuthology.orchestra.run.smithi164.stderr: 3 Jun 19:20:20 ntpd[21698]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2023-06-03T19:20:20.364 INFO:teuthology.orchestra.run.smithi164.stderr: 3 Jun 19:20:20 ntpd[21698]: can't open /var/log/ntpstats/rawstats.20230603: Permission denied 2023-06-03T19:20:20.364 INFO:teuthology.orchestra.run.smithi164.stderr: 3 Jun 19:20:20 ntpd[21698]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2023-06-03T19:20:20.365 INFO:teuthology.orchestra.run.smithi164.stderr: 3 Jun 19:20:20 ntpd[21698]: can't open /var/log/ntpstats/peerstats.20230603: Permission denied 2023-06-03T19:20:22.344 INFO:teuthology.orchestra.run.smithi089.stderr: 3 Jun 19:20:22 ntpd[21671]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2023-06-03T19:20:22.345 INFO:teuthology.orchestra.run.smithi089.stderr: 3 Jun 19:20:22 ntpd[21671]: can't open /var/log/ntpstats/rawstats.20230603: Permission denied 2023-06-03T19:20:22.345 INFO:teuthology.orchestra.run.smithi089.stderr: 3 Jun 19:20:22 ntpd[21671]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2023-06-03T19:20:22.345 INFO:teuthology.orchestra.run.smithi089.stderr: 3 Jun 19:20:22 ntpd[21671]: can't open /var/log/ntpstats/rawstats.20230603: Permission denied 2023-06-03T19:20:22.345 INFO:teuthology.orchestra.run.smithi089.stderr: 3 Jun 19:20:22 ntpd[21671]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2023-06-03T19:20:22.345 INFO:teuthology.orchestra.run.smithi089.stderr: 3 Jun 19:20:22 ntpd[21671]: can't open /var/log/ntpstats/rawstats.20230603: Permission denied 2023-06-03T19:20:22.345 INFO:teuthology.orchestra.run.smithi089.stderr: 3 Jun 19:20:22 ntpd[21671]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2023-06-03T19:20:22.345 INFO:teuthology.orchestra.run.smithi089.stderr: 3 Jun 19:20:22 ntpd[21671]: can't open /var/log/ntpstats/rawstats.20230603: Permission denied 2023-06-03T19:20:22.346 INFO:teuthology.orchestra.run.smithi089.stderr: 3 Jun 19:20:22 ntpd[21671]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2023-06-03T19:20:22.346 INFO:teuthology.orchestra.run.smithi089.stderr: 3 Jun 19:20:22 ntpd[21671]: can't open /var/log/ntpstats/peerstats.20230603: Permission denied 2023-06-03T19:20:22.346 INFO:teuthology.orchestra.run.smithi089.stderr: 3 Jun 19:20:22 ntpd[21671]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2023-06-03T19:20:22.346 INFO:teuthology.orchestra.run.smithi089.stderr: 3 Jun 19:20:22 ntpd[21671]: can't open /var/log/ntpstats/rawstats.20230603: Permission denied 2023-06-03T19:20:22.346 INFO:teuthology.orchestra.run.smithi089.stderr: 3 Jun 19:20:22 ntpd[21671]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2023-06-03T19:20:22.346 INFO:teuthology.orchestra.run.smithi089.stderr: 3 Jun 19:20:22 ntpd[21671]: can't open /var/log/ntpstats/peerstats.20230603: Permission denied 2023-06-03T19:20:22.346 INFO:teuthology.orchestra.run.smithi089.stderr: 3 Jun 19:20:22 ntpd[21671]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2023-06-03T19:20:22.347 INFO:teuthology.orchestra.run.smithi089.stderr: 3 Jun 19:20:22 ntpd[21671]: can't open /var/log/ntpstats/rawstats.20230603: Permission denied 2023-06-03T19:20:22.347 INFO:teuthology.orchestra.run.smithi089.stderr: 3 Jun 19:20:22 ntpd[21671]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2023-06-03T19:20:22.347 INFO:teuthology.orchestra.run.smithi089.stderr: 3 Jun 19:20:22 ntpd[21671]: can't open /var/log/ntpstats/peerstats.20230603: Permission denied 2023-06-03T19:20:22.365 INFO:teuthology.orchestra.run.smithi164.stderr: 3 Jun 19:20:22 ntpd[21698]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2023-06-03T19:20:22.366 INFO:teuthology.orchestra.run.smithi164.stderr: 3 Jun 19:20:22 ntpd[21698]: can't open /var/log/ntpstats/rawstats.20230603: Permission denied 2023-06-03T19:20:22.366 INFO:teuthology.orchestra.run.smithi164.stderr: 3 Jun 19:20:22 ntpd[21698]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2023-06-03T19:20:22.367 INFO:teuthology.orchestra.run.smithi164.stderr: 3 Jun 19:20:22 ntpd[21698]: can't open /var/log/ntpstats/rawstats.20230603: Permission denied 2023-06-03T19:20:22.367 INFO:teuthology.orchestra.run.smithi164.stderr: 3 Jun 19:20:22 ntpd[21698]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2023-06-03T19:20:22.367 INFO:teuthology.orchestra.run.smithi164.stderr: 3 Jun 19:20:22 ntpd[21698]: can't open /var/log/ntpstats/rawstats.20230603: Permission denied 2023-06-03T19:20:22.367 INFO:teuthology.orchestra.run.smithi164.stderr: 3 Jun 19:20:22 ntpd[21698]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2023-06-03T19:20:22.368 INFO:teuthology.orchestra.run.smithi164.stderr: 3 Jun 19:20:22 ntpd[21698]: can't open /var/log/ntpstats/rawstats.20230603: Permission denied 2023-06-03T19:20:22.368 INFO:teuthology.orchestra.run.smithi164.stderr: 3 Jun 19:20:22 ntpd[21698]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2023-06-03T19:20:22.368 INFO:teuthology.orchestra.run.smithi164.stderr: 3 Jun 19:20:22 ntpd[21698]: can't open /var/log/ntpstats/peerstats.20230603: Permission denied 2023-06-03T19:20:22.369 INFO:teuthology.orchestra.run.smithi164.stderr: 3 Jun 19:20:22 ntpd[21698]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2023-06-03T19:20:22.369 INFO:teuthology.orchestra.run.smithi164.stderr: 3 Jun 19:20:22 ntpd[21698]: can't open /var/log/ntpstats/rawstats.20230603: Permission denied 2023-06-03T19:20:22.369 INFO:teuthology.orchestra.run.smithi164.stderr: 3 Jun 19:20:22 ntpd[21698]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2023-06-03T19:20:22.370 INFO:teuthology.orchestra.run.smithi164.stderr: 3 Jun 19:20:22 ntpd[21698]: can't open /var/log/ntpstats/peerstats.20230603: Permission denied 2023-06-03T19:20:22.370 INFO:teuthology.orchestra.run.smithi164.stderr: 3 Jun 19:20:22 ntpd[21698]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2023-06-03T19:20:22.370 INFO:teuthology.orchestra.run.smithi164.stderr: 3 Jun 19:20:22 ntpd[21698]: can't open /var/log/ntpstats/rawstats.20230603: Permission denied 2023-06-03T19:20:22.371 INFO:teuthology.orchestra.run.smithi164.stderr: 3 Jun 19:20:22 ntpd[21698]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2023-06-03T19:20:22.371 INFO:teuthology.orchestra.run.smithi164.stderr: 3 Jun 19:20:22 ntpd[21698]: can't open /var/log/ntpstats/peerstats.20230603: Permission denied 2023-06-03T19:20:24.344 INFO:teuthology.orchestra.run.smithi089.stderr: 3 Jun 19:20:24 ntpd[21671]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2023-06-03T19:20:24.344 INFO:teuthology.orchestra.run.smithi089.stderr: 3 Jun 19:20:24 ntpd[21671]: can't open /var/log/ntpstats/rawstats.20230603: Permission denied 2023-06-03T19:20:24.344 INFO:teuthology.orchestra.run.smithi089.stderr: 3 Jun 19:20:24 ntpd[21671]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2023-06-03T19:20:24.344 INFO:teuthology.orchestra.run.smithi089.stderr: 3 Jun 19:20:24 ntpd[21671]: can't open /var/log/ntpstats/rawstats.20230603: Permission denied 2023-06-03T19:20:24.344 INFO:teuthology.orchestra.run.smithi089.stderr: 3 Jun 19:20:24 ntpd[21671]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2023-06-03T19:20:24.345 INFO:teuthology.orchestra.run.smithi089.stderr: 3 Jun 19:20:24 ntpd[21671]: can't open /var/log/ntpstats/rawstats.20230603: Permission denied 2023-06-03T19:20:24.345 INFO:teuthology.orchestra.run.smithi089.stderr: 3 Jun 19:20:24 ntpd[21671]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2023-06-03T19:20:24.345 INFO:teuthology.orchestra.run.smithi089.stderr: 3 Jun 19:20:24 ntpd[21671]: can't open /var/log/ntpstats/rawstats.20230603: Permission denied 2023-06-03T19:20:24.345 INFO:teuthology.orchestra.run.smithi089.stderr: 3 Jun 19:20:24 ntpd[21671]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2023-06-03T19:20:24.345 INFO:teuthology.orchestra.run.smithi089.stderr: 3 Jun 19:20:24 ntpd[21671]: can't open /var/log/ntpstats/peerstats.20230603: Permission denied 2023-06-03T19:20:24.346 INFO:teuthology.orchestra.run.smithi089.stdout: 3 Jun 19:20:24 ntpd[21671]: ntpd: time slew -0.004371 s 2023-06-03T19:20:24.346 INFO:teuthology.orchestra.run.smithi089.stdout:ntpd: time slew -0.004371s 2023-06-03T19:20:24.346 INFO:teuthology.orchestra.run.smithi089.stderr: 3 Jun 19:20:24 ntpd[21671]: couldn't unlink /var/log/ntpstats/loopstats: Permission denied 2023-06-03T19:20:24.346 INFO:teuthology.orchestra.run.smithi089.stderr: 3 Jun 19:20:24 ntpd[21671]: can't open /var/log/ntpstats/loopstats.20230603: Permission denied 2023-06-03T19:20:24.364 INFO:teuthology.orchestra.run.smithi164.stderr: 3 Jun 19:20:24 ntpd[21698]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2023-06-03T19:20:24.364 INFO:teuthology.orchestra.run.smithi164.stderr: 3 Jun 19:20:24 ntpd[21698]: can't open /var/log/ntpstats/rawstats.20230603: Permission denied 2023-06-03T19:20:24.365 INFO:teuthology.orchestra.run.smithi164.stderr: 3 Jun 19:20:24 ntpd[21698]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2023-06-03T19:20:24.365 INFO:teuthology.orchestra.run.smithi164.stderr: 3 Jun 19:20:24 ntpd[21698]: can't open /var/log/ntpstats/rawstats.20230603: Permission denied 2023-06-03T19:20:24.365 INFO:teuthology.orchestra.run.smithi164.stderr: 3 Jun 19:20:24 ntpd[21698]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2023-06-03T19:20:24.365 INFO:teuthology.orchestra.run.smithi164.stderr: 3 Jun 19:20:24 ntpd[21698]: can't open /var/log/ntpstats/rawstats.20230603: Permission denied 2023-06-03T19:20:24.365 INFO:teuthology.orchestra.run.smithi164.stderr: 3 Jun 19:20:24 ntpd[21698]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2023-06-03T19:20:24.365 INFO:teuthology.orchestra.run.smithi164.stderr: 3 Jun 19:20:24 ntpd[21698]: can't open /var/log/ntpstats/rawstats.20230603: Permission denied 2023-06-03T19:20:24.365 INFO:teuthology.orchestra.run.smithi164.stderr: 3 Jun 19:20:24 ntpd[21698]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2023-06-03T19:20:24.365 INFO:teuthology.orchestra.run.smithi164.stderr: 3 Jun 19:20:24 ntpd[21698]: can't open /var/log/ntpstats/peerstats.20230603: Permission denied 2023-06-03T19:20:24.366 INFO:teuthology.orchestra.run.smithi164.stderr: 3 Jun 19:20:24 ntpd[21698]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2023-06-03T19:20:24.366 INFO:teuthology.orchestra.run.smithi164.stderr: 3 Jun 19:20:24 ntpd[21698]: can't open /var/log/ntpstats/rawstats.20230603: Permission denied 2023-06-03T19:20:24.366 INFO:teuthology.orchestra.run.smithi164.stderr: 3 Jun 19:20:24 ntpd[21698]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2023-06-03T19:20:24.366 INFO:teuthology.orchestra.run.smithi164.stderr: 3 Jun 19:20:24 ntpd[21698]: can't open /var/log/ntpstats/peerstats.20230603: Permission denied 2023-06-03T19:20:24.366 INFO:teuthology.orchestra.run.smithi164.stderr: 3 Jun 19:20:24 ntpd[21698]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2023-06-03T19:20:24.366 INFO:teuthology.orchestra.run.smithi164.stderr: 3 Jun 19:20:24 ntpd[21698]: can't open /var/log/ntpstats/rawstats.20230603: Permission denied 2023-06-03T19:20:24.366 INFO:teuthology.orchestra.run.smithi164.stderr: 3 Jun 19:20:24 ntpd[21698]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2023-06-03T19:20:24.367 INFO:teuthology.orchestra.run.smithi164.stderr: 3 Jun 19:20:24 ntpd[21698]: can't open /var/log/ntpstats/peerstats.20230603: Permission denied 2023-06-03T19:20:24.367 INFO:teuthology.orchestra.run.smithi164.stdout: 3 Jun 19:20:24 ntpd[21698]: ntpd: time slew +0.005268 s 2023-06-03T19:20:24.367 INFO:teuthology.orchestra.run.smithi164.stdout:ntpd: time slew +0.005268s 2023-06-03T19:20:24.367 INFO:teuthology.orchestra.run.smithi164.stderr: 3 Jun 19:20:24 ntpd[21698]: couldn't unlink /var/log/ntpstats/loopstats: Permission denied 2023-06-03T19:20:24.368 INFO:teuthology.orchestra.run.smithi164.stderr: 3 Jun 19:20:24 ntpd[21698]: can't open /var/log/ntpstats/loopstats.20230603: Permission denied 2023-06-03T19:20:24.398 INFO:teuthology.orchestra.run.smithi089.stdout: remote refid st t when poll reach delay offset jitter 2023-06-03T19:20:24.399 INFO:teuthology.orchestra.run.smithi089.stdout:============================================================================== 2023-06-03T19:20:24.399 INFO:teuthology.orchestra.run.smithi089.stdout: hv01.front.sepi .INIT. 16 u - 64 0 0.000 0.000 0.000 2023-06-03T19:20:24.399 INFO:teuthology.orchestra.run.smithi089.stdout: hv02.front.sepi .INIT. 16 u - 64 0 0.000 0.000 0.000 2023-06-03T19:20:24.399 INFO:teuthology.orchestra.run.smithi089.stdout: hv03.front.sepi .INIT. 16 u - 64 0 0.000 0.000 0.000 2023-06-03T19:20:24.399 INFO:teuthology.orchestra.run.smithi089.stdout: hv04.front.sepi .INIT. 16 u - 64 0 0.000 0.000 0.000 2023-06-03T19:20:24.487 INFO:teuthology.orchestra.run.smithi164.stdout: remote refid st t when poll reach delay offset jitter 2023-06-03T19:20:24.487 INFO:teuthology.orchestra.run.smithi164.stdout:============================================================================== 2023-06-03T19:20:24.488 INFO:teuthology.orchestra.run.smithi164.stdout: hv01.front.sepi .INIT. 16 u - 64 0 0.000 0.000 0.000 2023-06-03T19:20:24.488 INFO:teuthology.orchestra.run.smithi164.stdout: hv02.front.sepi .INIT. 16 u - 64 0 0.000 0.000 0.000 2023-06-03T19:20:24.488 INFO:teuthology.orchestra.run.smithi164.stdout: hv03.front.sepi .INIT. 16 u - 64 0 0.000 0.000 0.000 2023-06-03T19:20:24.488 INFO:teuthology.orchestra.run.smithi164.stdout: hv04.front.sepi .INIT. 16 u - 64 0 0.000 0.000 0.000 2023-06-03T19:20:24.489 INFO:teuthology.run_tasks:Running task install... 2023-06-03T19:20:24.498 DEBUG:teuthology.task.install:project ceph 2023-06-03T19:20:24.499 DEBUG:teuthology.task.install:INSTALL overrides: {'ceph': {'flavor': 'default', 'sha1': '8e24bcf462a1d94c2e12aa4e26419634883a1e88'}} 2023-06-03T19:20:24.499 DEBUG:teuthology.task.install:config {'branch': 'pacific', 'exclude_packages': ['ceph-volume'], 'flavor': 'default', 'sha1': '8e24bcf462a1d94c2e12aa4e26419634883a1e88'} 2023-06-03T19:20:24.500 INFO:teuthology.task.install:Using flavor: default 2023-06-03T19:20:24.507 DEBUG:teuthology.task.install:Package list is: {'deb': ['ceph', 'cephadm', 'ceph-mds', 'ceph-mgr', 'ceph-common', 'ceph-fuse', 'ceph-test', 'radosgw', 'python3-rados', 'python3-rgw', 'python3-cephfs', 'python3-rbd', 'libcephfs2', 'libcephfs-dev', 'librados2', 'librbd1', 'rbd-fuse'], 'rpm': ['ceph-radosgw', 'ceph-test', 'ceph', 'ceph-base', 'cephadm', 'ceph-immutable-object-cache', 'ceph-mgr', 'ceph-mgr-dashboard', 'ceph-mgr-diskprediction-local', 'ceph-mgr-rook', 'ceph-mgr-cephadm', 'ceph-fuse', 'librados-devel', 'libcephfs2', 'libcephfs-devel', 'librados2', 'librbd1', 'python3-rados', 'python3-rgw', 'python3-cephfs', 'python3-rbd', 'rbd-fuse', 'rbd-mirror', 'rbd-nbd']} 2023-06-03T19:20:24.507 INFO:teuthology.task.install:extra packages: [] 2023-06-03T19:20:24.508 DEBUG:teuthology.orchestra.run.smithi089:> sudo apt-key list | grep Ceph 2023-06-03T19:20:24.509 DEBUG:teuthology.orchestra.run.smithi164:> sudo apt-key list | grep Ceph 2023-06-03T19:20:24.619 INFO:teuthology.orchestra.run.smithi089.stderr:Warning: apt-key output should not be parsed (stdout is not a terminal) 2023-06-03T19:20:24.626 INFO:teuthology.orchestra.run.smithi164.stderr:Warning: apt-key output should not be parsed (stdout is not a terminal) 2023-06-03T19:20:24.677 INFO:teuthology.orchestra.run.smithi089.stdout:uid [ unknown] Ceph automated package build (Ceph automated package build) 2023-06-03T19:20:24.677 INFO:teuthology.orchestra.run.smithi089.stdout:uid [ unknown] Ceph.com (release key) 2023-06-03T19:20:24.678 INFO:teuthology.task.install.deb:Installing packages: ceph, cephadm, ceph-mds, ceph-mgr, ceph-common, ceph-fuse, ceph-test, radosgw, python3-rados, python3-rgw, python3-cephfs, python3-rbd, libcephfs2, libcephfs-dev, librados2, librbd1, rbd-fuse on remote deb x86_64 2023-06-03T19:20:24.678 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2023-06-03T19:20:24.679 INFO:teuthology.packaging:ref: None 2023-06-03T19:20:24.679 INFO:teuthology.packaging:tag: None 2023-06-03T19:20:24.679 INFO:teuthology.packaging:branch: pacific 2023-06-03T19:20:24.679 INFO:teuthology.packaging:sha1: 8e24bcf462a1d94c2e12aa4e26419634883a1e88 2023-06-03T19:20:24.680 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=ubuntu%2F20.04%2Fx86_64&ref=pacific 2023-06-03T19:20:24.684 INFO:teuthology.orchestra.run.smithi164.stdout:uid [ unknown] Ceph automated package build (Ceph automated package build) 2023-06-03T19:20:24.685 INFO:teuthology.orchestra.run.smithi164.stdout:uid [ unknown] Ceph.com (release key) 2023-06-03T19:20:24.685 INFO:teuthology.task.install.deb:Installing packages: ceph, cephadm, ceph-mds, ceph-mgr, ceph-common, ceph-fuse, ceph-test, radosgw, python3-rados, python3-rgw, python3-cephfs, python3-rbd, libcephfs2, libcephfs-dev, librados2, librbd1, rbd-fuse on remote deb x86_64 2023-06-03T19:20:24.686 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2023-06-03T19:20:24.686 INFO:teuthology.packaging:ref: None 2023-06-03T19:20:24.686 INFO:teuthology.packaging:tag: None 2023-06-03T19:20:24.686 INFO:teuthology.packaging:branch: pacific 2023-06-03T19:20:24.686 INFO:teuthology.packaging:sha1: 8e24bcf462a1d94c2e12aa4e26419634883a1e88 2023-06-03T19:20:24.687 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=ubuntu%2F20.04%2Fx86_64&ref=pacific 2023-06-03T19:20:25.011 INFO:teuthology.task.install.deb:Pulling from https://4.chacra.ceph.com/r/ceph/pacific/d8c5d34975dce1c5eb0aa3a7979a4d9b9a99d1ec/ubuntu/focal/flavors/default/ 2023-06-03T19:20:25.011 INFO:teuthology.task.install.deb:Package version is 16.2.13-257-gd8c5d349-1focal 2023-06-03T19:20:25.026 INFO:teuthology.task.install.deb:Pulling from https://4.chacra.ceph.com/r/ceph/pacific/d8c5d34975dce1c5eb0aa3a7979a4d9b9a99d1ec/ubuntu/focal/flavors/default/ 2023-06-03T19:20:25.027 INFO:teuthology.task.install.deb:Package version is 16.2.13-257-gd8c5d349-1focal 2023-06-03T19:20:25.127 DEBUG:teuthology.orchestra.run.smithi089:> set -ex 2023-06-03T19:20:25.127 DEBUG:teuthology.orchestra.run.smithi089:> sudo dd of=/etc/apt/sources.list.d/ceph.list 2023-06-03T19:20:25.141 DEBUG:teuthology.orchestra.run.smithi164:> set -ex 2023-06-03T19:20:25.141 DEBUG:teuthology.orchestra.run.smithi164:> sudo dd of=/etc/apt/sources.list.d/ceph.list 2023-06-03T19:20:25.144 DEBUG:teuthology.orchestra.run.smithi089:> sudo apt-get update 2023-06-03T19:20:25.158 DEBUG:teuthology.orchestra.run.smithi164:> sudo apt-get update 2023-06-03T19:20:25.342 INFO:teuthology.orchestra.run.smithi164.stdout:Hit:1 http://archive.ubuntu.com/ubuntu focal InRelease 2023-06-03T19:20:25.343 INFO:teuthology.orchestra.run.smithi164.stdout:Hit:2 http://security.ubuntu.com/ubuntu focal-security InRelease 2023-06-03T19:20:25.350 INFO:teuthology.orchestra.run.smithi164.stdout:Hit:3 http://archive.ubuntu.com/ubuntu focal-updates InRelease 2023-06-03T19:20:25.351 INFO:teuthology.orchestra.run.smithi089.stdout:Hit:1 http://security.ubuntu.com/ubuntu focal-security InRelease 2023-06-03T19:20:25.367 INFO:teuthology.orchestra.run.smithi164.stdout:Hit:4 http://archive.ubuntu.com/ubuntu focal-backports InRelease 2023-06-03T19:20:25.386 INFO:teuthology.orchestra.run.smithi089.stdout:Ign:2 https://4.chacra.ceph.com/r/ceph/pacific/d8c5d34975dce1c5eb0aa3a7979a4d9b9a99d1ec/ubuntu/focal/flavors/default focal InRelease 2023-06-03T19:20:25.399 INFO:teuthology.orchestra.run.smithi164.stdout:Ign:5 https://4.chacra.ceph.com/r/ceph/pacific/d8c5d34975dce1c5eb0aa3a7979a4d9b9a99d1ec/ubuntu/focal/flavors/default focal InRelease 2023-06-03T19:20:25.408 INFO:teuthology.orchestra.run.smithi089.stdout:Get:3 https://4.chacra.ceph.com/r/ceph/pacific/d8c5d34975dce1c5eb0aa3a7979a4d9b9a99d1ec/ubuntu/focal/flavors/default focal Release [7,680 B] 2023-06-03T19:20:25.424 INFO:teuthology.orchestra.run.smithi164.stdout:Get:6 https://4.chacra.ceph.com/r/ceph/pacific/d8c5d34975dce1c5eb0aa3a7979a4d9b9a99d1ec/ubuntu/focal/flavors/default focal Release [7,680 B] 2023-06-03T19:20:25.431 INFO:teuthology.orchestra.run.smithi089.stdout:Ign:4 https://4.chacra.ceph.com/r/ceph/pacific/d8c5d34975dce1c5eb0aa3a7979a4d9b9a99d1ec/ubuntu/focal/flavors/default focal Release.gpg 2023-06-03T19:20:25.443 INFO:teuthology.orchestra.run.smithi164.stdout:Ign:7 https://4.chacra.ceph.com/r/ceph/pacific/d8c5d34975dce1c5eb0aa3a7979a4d9b9a99d1ec/ubuntu/focal/flavors/default focal Release.gpg 2023-06-03T19:20:25.449 INFO:teuthology.orchestra.run.smithi089.stdout:Hit:5 http://archive.ubuntu.com/ubuntu focal InRelease 2023-06-03T19:20:25.451 INFO:teuthology.orchestra.run.smithi089.stdout:Get:6 https://4.chacra.ceph.com/r/ceph/pacific/d8c5d34975dce1c5eb0aa3a7979a4d9b9a99d1ec/ubuntu/focal/flavors/default focal/main amd64 Packages [17.2 kB] 2023-06-03T19:20:25.464 INFO:teuthology.orchestra.run.smithi164.stdout:Get:8 https://4.chacra.ceph.com/r/ceph/pacific/d8c5d34975dce1c5eb0aa3a7979a4d9b9a99d1ec/ubuntu/focal/flavors/default focal/main amd64 Packages [17.2 kB] 2023-06-03T19:20:25.485 INFO:teuthology.orchestra.run.smithi089.stdout:Get:7 https://4.chacra.ceph.com/r/ceph/pacific/d8c5d34975dce1c5eb0aa3a7979a4d9b9a99d1ec/ubuntu/focal/flavors/default focal/main i386 Packages [3,102 B] 2023-06-03T19:20:25.504 INFO:teuthology.orchestra.run.smithi164.stdout:Get:9 https://4.chacra.ceph.com/r/ceph/pacific/d8c5d34975dce1c5eb0aa3a7979a4d9b9a99d1ec/ubuntu/focal/flavors/default focal/main i386 Packages [3,102 B] 2023-06-03T19:20:25.531 INFO:teuthology.orchestra.run.smithi089.stdout:Hit:8 http://archive.ubuntu.com/ubuntu focal-updates InRelease 2023-06-03T19:20:25.614 INFO:teuthology.orchestra.run.smithi089.stdout:Hit:9 http://archive.ubuntu.com/ubuntu focal-backports InRelease 2023-06-03T19:20:25.952 INFO:teuthology.orchestra.run.smithi164.stdout:Fetched 28.0 kB in 1s (47.4 kB/s) 2023-06-03T19:20:25.986 INFO:teuthology.orchestra.run.smithi089.stdout:Fetched 28.0 kB in 1s (44.2 kB/s) 2023-06-03T19:20:26.950 INFO:teuthology.orchestra.run.smithi164.stdout:Reading package lists... 2023-06-03T19:20:26.978 DEBUG:teuthology.orchestra.run.smithi164:> sudo DEBIAN_FRONTEND=noninteractive apt-get -y --force-yes -o Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confold" install ceph=16.2.13-257-gd8c5d349-1focal cephadm=16.2.13-257-gd8c5d349-1focal ceph-mds=16.2.13-257-gd8c5d349-1focal ceph-mgr=16.2.13-257-gd8c5d349-1focal ceph-common=16.2.13-257-gd8c5d349-1focal ceph-fuse=16.2.13-257-gd8c5d349-1focal ceph-test=16.2.13-257-gd8c5d349-1focal radosgw=16.2.13-257-gd8c5d349-1focal python3-rados=16.2.13-257-gd8c5d349-1focal python3-rgw=16.2.13-257-gd8c5d349-1focal python3-cephfs=16.2.13-257-gd8c5d349-1focal python3-rbd=16.2.13-257-gd8c5d349-1focal libcephfs2=16.2.13-257-gd8c5d349-1focal libcephfs-dev=16.2.13-257-gd8c5d349-1focal librados2=16.2.13-257-gd8c5d349-1focal librbd1=16.2.13-257-gd8c5d349-1focal rbd-fuse=16.2.13-257-gd8c5d349-1focal 2023-06-03T19:20:26.980 INFO:teuthology.orchestra.run.smithi089.stdout:Reading package lists... 2023-06-03T19:20:27.007 DEBUG:teuthology.orchestra.run.smithi089:> sudo DEBIAN_FRONTEND=noninteractive apt-get -y --force-yes -o Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confold" install ceph=16.2.13-257-gd8c5d349-1focal cephadm=16.2.13-257-gd8c5d349-1focal ceph-mds=16.2.13-257-gd8c5d349-1focal ceph-mgr=16.2.13-257-gd8c5d349-1focal ceph-common=16.2.13-257-gd8c5d349-1focal ceph-fuse=16.2.13-257-gd8c5d349-1focal ceph-test=16.2.13-257-gd8c5d349-1focal radosgw=16.2.13-257-gd8c5d349-1focal python3-rados=16.2.13-257-gd8c5d349-1focal python3-rgw=16.2.13-257-gd8c5d349-1focal python3-cephfs=16.2.13-257-gd8c5d349-1focal python3-rbd=16.2.13-257-gd8c5d349-1focal libcephfs2=16.2.13-257-gd8c5d349-1focal libcephfs-dev=16.2.13-257-gd8c5d349-1focal librados2=16.2.13-257-gd8c5d349-1focal librbd1=16.2.13-257-gd8c5d349-1focal rbd-fuse=16.2.13-257-gd8c5d349-1focal 2023-06-03T19:20:27.050 INFO:teuthology.orchestra.run.smithi164.stdout:Reading package lists... 2023-06-03T19:20:27.080 INFO:teuthology.orchestra.run.smithi089.stdout:Reading package lists... 2023-06-03T19:20:27.244 INFO:teuthology.orchestra.run.smithi164.stdout:Building dependency tree... 2023-06-03T19:20:27.245 INFO:teuthology.orchestra.run.smithi164.stdout:Reading state information... 2023-06-03T19:20:27.250 INFO:teuthology.orchestra.run.smithi089.stdout:Building dependency tree... 2023-06-03T19:20:27.251 INFO:teuthology.orchestra.run.smithi089.stdout:Reading state information... 2023-06-03T19:20:27.409 INFO:teuthology.orchestra.run.smithi089.stdout:The following packages were automatically installed and are no longer required: 2023-06-03T19:20:27.409 INFO:teuthology.orchestra.run.smithi089.stdout: libboost-iostreams1.71.0 libboost-thread1.71.0 2023-06-03T19:20:27.410 INFO:teuthology.orchestra.run.smithi089.stdout:Use 'sudo apt autoremove' to remove them. 2023-06-03T19:20:27.410 INFO:teuthology.orchestra.run.smithi089.stdout:The following additional packages will be installed: 2023-06-03T19:20:27.410 INFO:teuthology.orchestra.run.smithi089.stdout: ceph-base ceph-mgr-cephadm ceph-mgr-dashboard ceph-mgr-diskprediction-local 2023-06-03T19:20:27.411 INFO:teuthology.orchestra.run.smithi089.stdout: ceph-mgr-k8sevents ceph-mgr-modules-core ceph-mon ceph-osd jq libjaeger 2023-06-03T19:20:27.411 INFO:teuthology.orchestra.run.smithi089.stdout: libjq1 libjs-sphinxdoc libjs-underscore liblbfgsb0 libleveldb1d 2023-06-03T19:20:27.411 INFO:teuthology.orchestra.run.smithi089.stdout: liblttng-ust0 liblua5.3-0 liboath0 libonig5 librabbitmq4 libradosstriper1 2023-06-03T19:20:27.411 INFO:teuthology.orchestra.run.smithi089.stdout: librdkafka1 librgw2 libsqlite3-mod-ceph nvme-cli python-pastedeploy-tpl 2023-06-03T19:20:27.411 INFO:teuthology.orchestra.run.smithi089.stdout: python3-atomicwrites python3-attr python3-bcrypt python3-blinker python3-bs4 2023-06-03T19:20:27.411 INFO:teuthology.orchestra.run.smithi089.stdout: python3-cachetools python3-ceph-argparse python3-ceph-common 2023-06-03T19:20:27.412 INFO:teuthology.orchestra.run.smithi089.stdout: python3-cherrypy3 python3-cryptography python3-decorator python3-google-auth 2023-06-03T19:20:27.412 INFO:teuthology.orchestra.run.smithi089.stdout: python3-html5lib python3-jinja2 python3-joblib python3-jwt 2023-06-03T19:20:27.412 INFO:teuthology.orchestra.run.smithi089.stdout: python3-kubernetes python3-logutils python3-lxml python3-mako 2023-06-03T19:20:27.412 INFO:teuthology.orchestra.run.smithi089.stdout: python3-markupsafe python3-oauthlib python3-openssl python3-packaging 2023-06-03T19:20:27.413 INFO:teuthology.orchestra.run.smithi089.stdout: python3-paste python3-pastedeploy python3-pastescript python3-pecan 2023-06-03T19:20:27.413 INFO:teuthology.orchestra.run.smithi089.stdout: python3-pluggy python3-prettytable python3-psutil python3-py python3-pyasn1 2023-06-03T19:20:27.413 INFO:teuthology.orchestra.run.smithi089.stdout: python3-pyasn1-modules python3-pyinotify python3-pytest python3-repoze.lru 2023-06-03T19:20:27.413 INFO:teuthology.orchestra.run.smithi089.stdout: python3-requests-oauthlib python3-routes python3-rsa python3-scipy 2023-06-03T19:20:27.414 INFO:teuthology.orchestra.run.smithi089.stdout: python3-simplegeneric python3-simplejson python3-singledispatch 2023-06-03T19:20:27.414 INFO:teuthology.orchestra.run.smithi089.stdout: python3-sklearn python3-sklearn-lib python3-soupsieve python3-tempita 2023-06-03T19:20:27.414 INFO:teuthology.orchestra.run.smithi089.stdout: python3-waitress python3-wcwidth python3-webencodings python3-webob 2023-06-03T19:20:27.414 INFO:teuthology.orchestra.run.smithi089.stdout: python3-websocket python3-webtest python3-werkzeug socat xmlstarlet 2023-06-03T19:20:27.415 INFO:teuthology.orchestra.run.smithi164.stdout:The following packages were automatically installed and are no longer required: 2023-06-03T19:20:27.415 INFO:teuthology.orchestra.run.smithi089.stdout:Suggested packages: 2023-06-03T19:20:27.416 INFO:teuthology.orchestra.run.smithi089.stdout: python3-influxdb python-attr-doc python-blinker-doc python-cryptography-doc 2023-06-03T19:20:27.416 INFO:teuthology.orchestra.run.smithi089.stdout: python3-cryptography-vectors python3-genshi python-jinja2-doc python3-crypto 2023-06-03T19:20:27.416 INFO:teuthology.orchestra.run.smithi089.stdout: python3-lxml-dbg python-lxml-doc python3-beaker python-mako-doc 2023-06-03T19:20:27.416 INFO:teuthology.orchestra.run.smithi089.stdout: python-openssl-doc python3-openssl-dbg httpd-wsgi libapache2-mod-python 2023-06-03T19:20:27.417 INFO:teuthology.orchestra.run.smithi089.stdout: libapache2-mod-scgi libjs-mochikit python-pecan-doc python-psutil-doc 2023-06-03T19:20:27.417 INFO:teuthology.orchestra.run.smithi089.stdout: subversion python-pyinotify-doc python-scipy-doc python3-dap 2023-06-03T19:20:27.417 INFO:teuthology.orchestra.run.smithi089.stdout: python-sklearn-doc ipython3 python-waitress-doc python-webob-doc 2023-06-03T19:20:27.417 INFO:teuthology.orchestra.run.smithi089.stdout: python-webtest-doc python-werkzeug-doc python3-termcolor python3-watchdog 2023-06-03T19:20:27.417 INFO:teuthology.orchestra.run.smithi089.stdout:Recommended packages: 2023-06-03T19:20:27.418 INFO:teuthology.orchestra.run.smithi089.stdout: btrfs-tools 2023-06-03T19:20:27.419 INFO:teuthology.orchestra.run.smithi164.stdout: libboost-iostreams1.71.0 libboost-thread1.71.0 2023-06-03T19:20:27.420 INFO:teuthology.orchestra.run.smithi164.stdout:Use 'sudo apt autoremove' to remove them. 2023-06-03T19:20:27.420 INFO:teuthology.orchestra.run.smithi164.stdout:The following additional packages will be installed: 2023-06-03T19:20:27.420 INFO:teuthology.orchestra.run.smithi164.stdout: ceph-base ceph-mgr-cephadm ceph-mgr-dashboard ceph-mgr-diskprediction-local 2023-06-03T19:20:27.420 INFO:teuthology.orchestra.run.smithi164.stdout: ceph-mgr-k8sevents ceph-mgr-modules-core ceph-mon ceph-osd jq libjaeger 2023-06-03T19:20:27.420 INFO:teuthology.orchestra.run.smithi164.stdout: libjq1 libjs-sphinxdoc libjs-underscore liblbfgsb0 libleveldb1d 2023-06-03T19:20:27.421 INFO:teuthology.orchestra.run.smithi164.stdout: liblttng-ust0 liblua5.3-0 liboath0 libonig5 librabbitmq4 libradosstriper1 2023-06-03T19:20:27.421 INFO:teuthology.orchestra.run.smithi164.stdout: librdkafka1 librgw2 libsqlite3-mod-ceph nvme-cli python-pastedeploy-tpl 2023-06-03T19:20:27.421 INFO:teuthology.orchestra.run.smithi164.stdout: python3-atomicwrites python3-attr python3-bcrypt python3-blinker python3-bs4 2023-06-03T19:20:27.421 INFO:teuthology.orchestra.run.smithi164.stdout: python3-cachetools python3-ceph-argparse python3-ceph-common 2023-06-03T19:20:27.422 INFO:teuthology.orchestra.run.smithi164.stdout: python3-cherrypy3 python3-cryptography python3-decorator python3-google-auth 2023-06-03T19:20:27.422 INFO:teuthology.orchestra.run.smithi164.stdout: python3-html5lib python3-jinja2 python3-joblib python3-jwt 2023-06-03T19:20:27.422 INFO:teuthology.orchestra.run.smithi164.stdout: python3-kubernetes python3-logutils python3-lxml python3-mako 2023-06-03T19:20:27.422 INFO:teuthology.orchestra.run.smithi164.stdout: python3-markupsafe python3-oauthlib python3-openssl python3-packaging 2023-06-03T19:20:27.422 INFO:teuthology.orchestra.run.smithi164.stdout: python3-paste python3-pastedeploy python3-pastescript python3-pecan 2023-06-03T19:20:27.423 INFO:teuthology.orchestra.run.smithi164.stdout: python3-pluggy python3-prettytable python3-psutil python3-py python3-pyasn1 2023-06-03T19:20:27.423 INFO:teuthology.orchestra.run.smithi164.stdout: python3-pyasn1-modules python3-pyinotify python3-pytest python3-repoze.lru 2023-06-03T19:20:27.423 INFO:teuthology.orchestra.run.smithi164.stdout: python3-requests-oauthlib python3-routes python3-rsa python3-scipy 2023-06-03T19:20:27.423 INFO:teuthology.orchestra.run.smithi164.stdout: python3-simplegeneric python3-simplejson python3-singledispatch 2023-06-03T19:20:27.424 INFO:teuthology.orchestra.run.smithi164.stdout: python3-sklearn python3-sklearn-lib python3-soupsieve python3-tempita 2023-06-03T19:20:27.424 INFO:teuthology.orchestra.run.smithi164.stdout: python3-waitress python3-wcwidth python3-webencodings python3-webob 2023-06-03T19:20:27.424 INFO:teuthology.orchestra.run.smithi164.stdout: python3-websocket python3-webtest python3-werkzeug socat xmlstarlet 2023-06-03T19:20:27.424 INFO:teuthology.orchestra.run.smithi164.stdout:Suggested packages: 2023-06-03T19:20:27.424 INFO:teuthology.orchestra.run.smithi164.stdout: python3-influxdb python-attr-doc python-blinker-doc python-cryptography-doc 2023-06-03T19:20:27.425 INFO:teuthology.orchestra.run.smithi164.stdout: python3-cryptography-vectors python3-genshi python-jinja2-doc python3-crypto 2023-06-03T19:20:27.425 INFO:teuthology.orchestra.run.smithi164.stdout: python3-lxml-dbg python-lxml-doc python3-beaker python-mako-doc 2023-06-03T19:20:27.425 INFO:teuthology.orchestra.run.smithi164.stdout: python-openssl-doc python3-openssl-dbg httpd-wsgi libapache2-mod-python 2023-06-03T19:20:27.425 INFO:teuthology.orchestra.run.smithi164.stdout: libapache2-mod-scgi libjs-mochikit python-pecan-doc python-psutil-doc 2023-06-03T19:20:27.426 INFO:teuthology.orchestra.run.smithi164.stdout: subversion python-pyinotify-doc python-scipy-doc python3-dap 2023-06-03T19:20:27.426 INFO:teuthology.orchestra.run.smithi164.stdout: python-sklearn-doc ipython3 python-waitress-doc python-webob-doc 2023-06-03T19:20:27.426 INFO:teuthology.orchestra.run.smithi164.stdout: python-webtest-doc python-werkzeug-doc python3-termcolor python3-watchdog 2023-06-03T19:20:27.426 INFO:teuthology.orchestra.run.smithi164.stdout:Recommended packages: 2023-06-03T19:20:27.427 INFO:teuthology.orchestra.run.smithi164.stdout: btrfs-tools 2023-06-03T19:20:27.524 INFO:teuthology.orchestra.run.smithi089.stdout:The following NEW packages will be installed: 2023-06-03T19:20:27.525 INFO:teuthology.orchestra.run.smithi089.stdout: ceph ceph-base ceph-common ceph-fuse ceph-mds ceph-mgr ceph-mgr-cephadm 2023-06-03T19:20:27.525 INFO:teuthology.orchestra.run.smithi089.stdout: ceph-mgr-dashboard ceph-mgr-diskprediction-local ceph-mgr-k8sevents 2023-06-03T19:20:27.525 INFO:teuthology.orchestra.run.smithi089.stdout: ceph-mgr-modules-core ceph-mon ceph-osd ceph-test cephadm jq libcephfs-dev 2023-06-03T19:20:27.526 INFO:teuthology.orchestra.run.smithi089.stdout: libcephfs2 libjaeger libjq1 libjs-sphinxdoc libjs-underscore liblbfgsb0 2023-06-03T19:20:27.526 INFO:teuthology.orchestra.run.smithi089.stdout: libleveldb1d liblttng-ust0 liblua5.3-0 liboath0 libonig5 librabbitmq4 2023-06-03T19:20:27.526 INFO:teuthology.orchestra.run.smithi089.stdout: libradosstriper1 librdkafka1 librgw2 libsqlite3-mod-ceph nvme-cli 2023-06-03T19:20:27.526 INFO:teuthology.orchestra.run.smithi089.stdout: python-pastedeploy-tpl python3-atomicwrites python3-attr python3-bcrypt 2023-06-03T19:20:27.526 INFO:teuthology.orchestra.run.smithi089.stdout: python3-blinker python3-bs4 python3-cachetools python3-ceph-argparse 2023-06-03T19:20:27.527 INFO:teuthology.orchestra.run.smithi089.stdout: python3-ceph-common python3-cephfs python3-cherrypy3 python3-cryptography 2023-06-03T19:20:27.527 INFO:teuthology.orchestra.run.smithi089.stdout: python3-decorator python3-google-auth python3-html5lib python3-jinja2 2023-06-03T19:20:27.527 INFO:teuthology.orchestra.run.smithi089.stdout: python3-joblib python3-jwt python3-kubernetes python3-logutils python3-lxml 2023-06-03T19:20:27.527 INFO:teuthology.orchestra.run.smithi089.stdout: python3-mako python3-markupsafe python3-oauthlib python3-openssl 2023-06-03T19:20:27.527 INFO:teuthology.orchestra.run.smithi089.stdout: python3-packaging python3-paste python3-pastedeploy python3-pastescript 2023-06-03T19:20:27.527 INFO:teuthology.orchestra.run.smithi089.stdout: python3-pecan python3-pluggy python3-prettytable python3-psutil python3-py 2023-06-03T19:20:27.527 INFO:teuthology.orchestra.run.smithi089.stdout: python3-pyasn1 python3-pyasn1-modules python3-pyinotify python3-pytest 2023-06-03T19:20:27.527 INFO:teuthology.orchestra.run.smithi089.stdout: python3-rados python3-rbd python3-repoze.lru python3-requests-oauthlib 2023-06-03T19:20:27.528 INFO:teuthology.orchestra.run.smithi089.stdout: python3-rgw python3-routes python3-rsa python3-scipy python3-simplegeneric 2023-06-03T19:20:27.528 INFO:teuthology.orchestra.run.smithi089.stdout: python3-simplejson python3-singledispatch python3-sklearn 2023-06-03T19:20:27.528 INFO:teuthology.orchestra.run.smithi089.stdout: python3-sklearn-lib python3-soupsieve python3-tempita python3-waitress 2023-06-03T19:20:27.528 INFO:teuthology.orchestra.run.smithi089.stdout: python3-wcwidth python3-webencodings python3-webob python3-websocket 2023-06-03T19:20:27.528 INFO:teuthology.orchestra.run.smithi089.stdout: python3-webtest python3-werkzeug radosgw rbd-fuse socat xmlstarlet 2023-06-03T19:20:27.529 INFO:teuthology.orchestra.run.smithi164.stdout:The following NEW packages will be installed: 2023-06-03T19:20:27.529 INFO:teuthology.orchestra.run.smithi164.stdout: ceph ceph-base ceph-common ceph-fuse ceph-mds ceph-mgr ceph-mgr-cephadm 2023-06-03T19:20:27.529 INFO:teuthology.orchestra.run.smithi164.stdout: ceph-mgr-dashboard ceph-mgr-diskprediction-local ceph-mgr-k8sevents 2023-06-03T19:20:27.529 INFO:teuthology.orchestra.run.smithi164.stdout: ceph-mgr-modules-core ceph-mon ceph-osd ceph-test cephadm jq libcephfs-dev 2023-06-03T19:20:27.529 INFO:teuthology.orchestra.run.smithi089.stdout:The following packages will be upgraded: 2023-06-03T19:20:27.530 INFO:teuthology.orchestra.run.smithi089.stdout: librados2 librbd1 2023-06-03T19:20:27.530 INFO:teuthology.orchestra.run.smithi164.stdout: libcephfs2 libjaeger libjq1 libjs-sphinxdoc libjs-underscore liblbfgsb0 2023-06-03T19:20:27.531 INFO:teuthology.orchestra.run.smithi164.stdout: libleveldb1d liblttng-ust0 liblua5.3-0 liboath0 libonig5 librabbitmq4 2023-06-03T19:20:27.531 INFO:teuthology.orchestra.run.smithi164.stdout: libradosstriper1 librdkafka1 librgw2 libsqlite3-mod-ceph nvme-cli 2023-06-03T19:20:27.531 INFO:teuthology.orchestra.run.smithi164.stdout: python-pastedeploy-tpl python3-atomicwrites python3-attr python3-bcrypt 2023-06-03T19:20:27.531 INFO:teuthology.orchestra.run.smithi164.stdout: python3-blinker python3-bs4 python3-cachetools python3-ceph-argparse 2023-06-03T19:20:27.531 INFO:teuthology.orchestra.run.smithi164.stdout: python3-ceph-common python3-cephfs python3-cherrypy3 python3-cryptography 2023-06-03T19:20:27.531 INFO:teuthology.orchestra.run.smithi164.stdout: python3-decorator python3-google-auth python3-html5lib python3-jinja2 2023-06-03T19:20:27.532 INFO:teuthology.orchestra.run.smithi164.stdout: python3-joblib python3-jwt python3-kubernetes python3-logutils python3-lxml 2023-06-03T19:20:27.532 INFO:teuthology.orchestra.run.smithi164.stdout: python3-mako python3-markupsafe python3-oauthlib python3-openssl 2023-06-03T19:20:27.532 INFO:teuthology.orchestra.run.smithi164.stdout: python3-packaging python3-paste python3-pastedeploy python3-pastescript 2023-06-03T19:20:27.532 INFO:teuthology.orchestra.run.smithi164.stdout: python3-pecan python3-pluggy python3-prettytable python3-psutil python3-py 2023-06-03T19:20:27.532 INFO:teuthology.orchestra.run.smithi164.stdout: python3-pyasn1 python3-pyasn1-modules python3-pyinotify python3-pytest 2023-06-03T19:20:27.532 INFO:teuthology.orchestra.run.smithi164.stdout: python3-rados python3-rbd python3-repoze.lru python3-requests-oauthlib 2023-06-03T19:20:27.532 INFO:teuthology.orchestra.run.smithi164.stdout: python3-rgw python3-routes python3-rsa python3-scipy python3-simplegeneric 2023-06-03T19:20:27.532 INFO:teuthology.orchestra.run.smithi164.stdout: python3-simplejson python3-singledispatch python3-sklearn 2023-06-03T19:20:27.533 INFO:teuthology.orchestra.run.smithi164.stdout: python3-sklearn-lib python3-soupsieve python3-tempita python3-waitress 2023-06-03T19:20:27.533 INFO:teuthology.orchestra.run.smithi164.stdout: python3-wcwidth python3-webencodings python3-webob python3-websocket 2023-06-03T19:20:27.533 INFO:teuthology.orchestra.run.smithi164.stdout: python3-webtest python3-werkzeug radosgw rbd-fuse socat xmlstarlet 2023-06-03T19:20:27.533 INFO:teuthology.orchestra.run.smithi164.stdout:The following packages will be upgraded: 2023-06-03T19:20:27.533 INFO:teuthology.orchestra.run.smithi164.stdout: librados2 librbd1 2023-06-03T19:20:27.719 INFO:teuthology.orchestra.run.smithi089.stdout:2 upgraded, 98 newly installed, 0 to remove and 183 not upgraded. 2023-06-03T19:20:27.720 INFO:teuthology.orchestra.run.smithi089.stdout:Need to get 173 MB of archives. 2023-06-03T19:20:27.720 INFO:teuthology.orchestra.run.smithi089.stdout:After this operation, 804 MB of additional disk space will be used. 2023-06-03T19:20:27.720 INFO:teuthology.orchestra.run.smithi089.stdout:Get:1 http://archive.ubuntu.com/ubuntu focal/main amd64 liblttng-ust0 amd64 2.11.0-1 [160 kB] 2023-06-03T19:20:27.720 INFO:teuthology.orchestra.run.smithi089.stdout:Get:2 https://4.chacra.ceph.com/r/ceph/pacific/d8c5d34975dce1c5eb0aa3a7979a4d9b9a99d1ec/ubuntu/focal/flavors/default focal/main amd64 libjaeger amd64 16.2.13-257-gd8c5d349-1focal [3,956 B] 2023-06-03T19:20:27.724 INFO:teuthology.orchestra.run.smithi164.stdout:2 upgraded, 98 newly installed, 0 to remove and 183 not upgraded. 2023-06-03T19:20:27.724 INFO:teuthology.orchestra.run.smithi164.stdout:Need to get 173 MB of archives. 2023-06-03T19:20:27.724 INFO:teuthology.orchestra.run.smithi164.stdout:After this operation, 804 MB of additional disk space will be used. 2023-06-03T19:20:27.724 INFO:teuthology.orchestra.run.smithi164.stdout:Get:1 https://4.chacra.ceph.com/r/ceph/pacific/d8c5d34975dce1c5eb0aa3a7979a4d9b9a99d1ec/ubuntu/focal/flavors/default focal/main amd64 libjaeger amd64 16.2.13-257-gd8c5d349-1focal [3,956 B] 2023-06-03T19:20:27.725 INFO:teuthology.orchestra.run.smithi164.stdout:Get:2 http://archive.ubuntu.com/ubuntu focal/main amd64 liblttng-ust0 amd64 2.11.0-1 [160 kB] 2023-06-03T19:20:27.798 INFO:teuthology.orchestra.run.smithi089.stdout:Get:3 https://4.chacra.ceph.com/r/ceph/pacific/d8c5d34975dce1c5eb0aa3a7979a4d9b9a99d1ec/ubuntu/focal/flavors/default focal/main amd64 librbd1 amd64 16.2.13-257-gd8c5d349-1focal [3,394 kB] 2023-06-03T19:20:27.801 INFO:teuthology.orchestra.run.smithi164.stdout:Get:3 https://4.chacra.ceph.com/r/ceph/pacific/d8c5d34975dce1c5eb0aa3a7979a4d9b9a99d1ec/ubuntu/focal/flavors/default focal/main amd64 librbd1 amd64 16.2.13-257-gd8c5d349-1focal [3,394 kB] 2023-06-03T19:20:28.114 INFO:teuthology.orchestra.run.smithi089.stdout:Get:4 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-prettytable all 0.7.2-5 [20.1 kB] 2023-06-03T19:20:28.122 INFO:teuthology.orchestra.run.smithi089.stdout:Get:5 http://archive.ubuntu.com/ubuntu focal/main amd64 liblua5.3-0 amd64 5.3.3-1.1ubuntu2 [116 kB] 2023-06-03T19:20:28.123 INFO:teuthology.orchestra.run.smithi164.stdout:Get:4 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-prettytable all 0.7.2-5 [20.1 kB] 2023-06-03T19:20:28.130 INFO:teuthology.orchestra.run.smithi164.stdout:Get:5 http://archive.ubuntu.com/ubuntu focal/main amd64 liblua5.3-0 amd64 5.3.3-1.1ubuntu2 [116 kB] 2023-06-03T19:20:28.145 INFO:teuthology.orchestra.run.smithi089.stdout:Get:6 https://4.chacra.ceph.com/r/ceph/pacific/d8c5d34975dce1c5eb0aa3a7979a4d9b9a99d1ec/ubuntu/focal/flavors/default focal/main amd64 librados2 amd64 16.2.13-257-gd8c5d349-1focal [3,387 kB] 2023-06-03T19:20:28.165 INFO:teuthology.orchestra.run.smithi089.stdout:Get:7 http://archive.ubuntu.com/ubuntu focal/main amd64 librabbitmq4 amd64 0.10.0-1 [35.8 kB] 2023-06-03T19:20:28.175 INFO:teuthology.orchestra.run.smithi164.stdout:Get:6 http://archive.ubuntu.com/ubuntu focal/main amd64 librabbitmq4 amd64 0.10.0-1 [35.8 kB] 2023-06-03T19:20:28.179 INFO:teuthology.orchestra.run.smithi089.stdout:Get:8 http://archive.ubuntu.com/ubuntu focal/universe amd64 librdkafka1 amd64 1.2.1-1ubuntu1 [410 kB] 2023-06-03T19:20:28.188 INFO:teuthology.orchestra.run.smithi164.stdout:Get:7 http://archive.ubuntu.com/ubuntu focal/universe amd64 librdkafka1 amd64 1.2.1-1ubuntu1 [410 kB] 2023-06-03T19:20:28.211 INFO:teuthology.orchestra.run.smithi164.stdout:Get:8 https://4.chacra.ceph.com/r/ceph/pacific/d8c5d34975dce1c5eb0aa3a7979a4d9b9a99d1ec/ubuntu/focal/flavors/default focal/main amd64 librados2 amd64 16.2.13-257-gd8c5d349-1focal [3,387 kB] 2023-06-03T19:20:28.305 INFO:teuthology.orchestra.run.smithi089.stdout:Get:9 http://archive.ubuntu.com/ubuntu focal/main amd64 libleveldb1d amd64 1.22-3ubuntu2 [139 kB] 2023-06-03T19:20:28.315 INFO:teuthology.orchestra.run.smithi164.stdout:Get:9 http://archive.ubuntu.com/ubuntu focal/main amd64 libleveldb1d amd64 1.22-3ubuntu2 [139 kB] 2023-06-03T19:20:28.327 INFO:teuthology.orchestra.run.smithi089.stdout:Get:10 https://4.chacra.ceph.com/r/ceph/pacific/d8c5d34975dce1c5eb0aa3a7979a4d9b9a99d1ec/ubuntu/focal/flavors/default focal/main amd64 libcephfs2 amd64 16.2.13-257-gd8c5d349-1focal [728 kB] 2023-06-03T19:20:28.328 INFO:teuthology.orchestra.run.smithi089.stdout:Get:11 http://archive.ubuntu.com/ubuntu focal/main amd64 liboath0 amd64 2.6.1-1.3 [44.7 kB] 2023-06-03T19:20:28.334 INFO:teuthology.orchestra.run.smithi089.stdout:Get:12 http://archive.ubuntu.com/ubuntu focal-updates/main amd64 python3-cryptography amd64 2.8-3ubuntu0.1 [211 kB] 2023-06-03T19:20:28.338 INFO:teuthology.orchestra.run.smithi164.stdout:Get:10 http://archive.ubuntu.com/ubuntu focal/main amd64 liboath0 amd64 2.6.1-1.3 [44.7 kB] 2023-06-03T19:20:28.344 INFO:teuthology.orchestra.run.smithi164.stdout:Get:11 http://archive.ubuntu.com/ubuntu focal-updates/main amd64 python3-cryptography amd64 2.8-3ubuntu0.1 [211 kB] 2023-06-03T19:20:28.365 INFO:teuthology.orchestra.run.smithi089.stdout:Get:13 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-openssl all 19.0.0-1build1 [43.3 kB] 2023-06-03T19:20:28.370 INFO:teuthology.orchestra.run.smithi089.stdout:Get:14 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-bcrypt amd64 3.1.7-2ubuntu1 [30.4 kB] 2023-06-03T19:20:28.372 INFO:teuthology.orchestra.run.smithi089.stdout:Get:15 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-cherrypy3 all 8.9.1-6 [160 kB] 2023-06-03T19:20:28.374 INFO:teuthology.orchestra.run.smithi089.stdout:Get:16 https://4.chacra.ceph.com/r/ceph/pacific/d8c5d34975dce1c5eb0aa3a7979a4d9b9a99d1ec/ubuntu/focal/flavors/default focal/main amd64 python3-rados amd64 16.2.13-257-gd8c5d349-1focal [412 kB] 2023-06-03T19:20:28.376 INFO:teuthology.orchestra.run.smithi164.stdout:Get:12 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-openssl all 19.0.0-1build1 [43.3 kB] 2023-06-03T19:20:28.380 INFO:teuthology.orchestra.run.smithi164.stdout:Get:13 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-bcrypt amd64 3.1.7-2ubuntu1 [30.4 kB] 2023-06-03T19:20:28.383 INFO:teuthology.orchestra.run.smithi164.stdout:Get:14 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-cherrypy3 all 8.9.1-6 [160 kB] 2023-06-03T19:20:28.387 INFO:teuthology.orchestra.run.smithi089.stdout:Get:17 https://4.chacra.ceph.com/r/ceph/pacific/d8c5d34975dce1c5eb0aa3a7979a4d9b9a99d1ec/ubuntu/focal/flavors/default focal/main amd64 python3-ceph-argparse all 16.2.13-257-gd8c5d349-1focal [22.1 kB] 2023-06-03T19:20:28.388 INFO:teuthology.orchestra.run.smithi089.stdout:Get:18 https://4.chacra.ceph.com/r/ceph/pacific/d8c5d34975dce1c5eb0aa3a7979a4d9b9a99d1ec/ubuntu/focal/flavors/default focal/main amd64 python3-cephfs amd64 16.2.13-257-gd8c5d349-1focal [212 kB] 2023-06-03T19:20:28.395 INFO:teuthology.orchestra.run.smithi089.stdout:Get:19 https://4.chacra.ceph.com/r/ceph/pacific/d8c5d34975dce1c5eb0aa3a7979a4d9b9a99d1ec/ubuntu/focal/flavors/default focal/main amd64 python3-ceph-common all 16.2.13-257-gd8c5d349-1focal [38.4 kB] 2023-06-03T19:20:28.396 INFO:teuthology.orchestra.run.smithi089.stdout:Get:20 https://4.chacra.ceph.com/r/ceph/pacific/d8c5d34975dce1c5eb0aa3a7979a4d9b9a99d1ec/ubuntu/focal/flavors/default focal/main amd64 python3-rbd amd64 16.2.13-257-gd8c5d349-1focal [385 kB] 2023-06-03T19:20:28.432 INFO:teuthology.orchestra.run.smithi089.stdout:Get:21 https://4.chacra.ceph.com/r/ceph/pacific/d8c5d34975dce1c5eb0aa3a7979a4d9b9a99d1ec/ubuntu/focal/flavors/default focal/main amd64 librgw2 amd64 16.2.13-257-gd8c5d349-1focal [3,687 kB] 2023-06-03T19:20:28.459 INFO:teuthology.orchestra.run.smithi164.stdout:Get:15 https://4.chacra.ceph.com/r/ceph/pacific/d8c5d34975dce1c5eb0aa3a7979a4d9b9a99d1ec/ubuntu/focal/flavors/default focal/main amd64 libcephfs2 amd64 16.2.13-257-gd8c5d349-1focal [728 kB] 2023-06-03T19:20:28.516 INFO:teuthology.orchestra.run.smithi164.stdout:Get:16 https://4.chacra.ceph.com/r/ceph/pacific/d8c5d34975dce1c5eb0aa3a7979a4d9b9a99d1ec/ubuntu/focal/flavors/default focal/main amd64 python3-rados amd64 16.2.13-257-gd8c5d349-1focal [412 kB] 2023-06-03T19:20:28.529 INFO:teuthology.orchestra.run.smithi164.stdout:Get:17 https://4.chacra.ceph.com/r/ceph/pacific/d8c5d34975dce1c5eb0aa3a7979a4d9b9a99d1ec/ubuntu/focal/flavors/default focal/main amd64 python3-ceph-argparse all 16.2.13-257-gd8c5d349-1focal [22.1 kB] 2023-06-03T19:20:28.530 INFO:teuthology.orchestra.run.smithi164.stdout:Get:18 https://4.chacra.ceph.com/r/ceph/pacific/d8c5d34975dce1c5eb0aa3a7979a4d9b9a99d1ec/ubuntu/focal/flavors/default focal/main amd64 python3-cephfs amd64 16.2.13-257-gd8c5d349-1focal [212 kB] 2023-06-03T19:20:28.551 INFO:teuthology.orchestra.run.smithi089.stdout:Get:22 http://archive.ubuntu.com/ubuntu focal-updates/main amd64 python3-jwt all 1.7.1-2ubuntu2.1 [18.0 kB] 2023-06-03T19:20:28.563 INFO:teuthology.orchestra.run.smithi164.stdout:Get:19 http://archive.ubuntu.com/ubuntu focal-updates/main amd64 python3-jwt all 1.7.1-2ubuntu2.1 [18.0 kB] 2023-06-03T19:20:28.572 INFO:teuthology.orchestra.run.smithi164.stdout:Get:20 https://4.chacra.ceph.com/r/ceph/pacific/d8c5d34975dce1c5eb0aa3a7979a4d9b9a99d1ec/ubuntu/focal/flavors/default focal/main amd64 python3-ceph-common all 16.2.13-257-gd8c5d349-1focal [38.4 kB] 2023-06-03T19:20:28.573 INFO:teuthology.orchestra.run.smithi164.stdout:Get:21 https://4.chacra.ceph.com/r/ceph/pacific/d8c5d34975dce1c5eb0aa3a7979a4d9b9a99d1ec/ubuntu/focal/flavors/default focal/main amd64 python3-rbd amd64 16.2.13-257-gd8c5d349-1focal [385 kB] 2023-06-03T19:20:28.585 INFO:teuthology.orchestra.run.smithi164.stdout:Get:22 https://4.chacra.ceph.com/r/ceph/pacific/d8c5d34975dce1c5eb0aa3a7979a4d9b9a99d1ec/ubuntu/focal/flavors/default focal/main amd64 librgw2 amd64 16.2.13-257-gd8c5d349-1focal [3,687 kB] 2023-06-03T19:20:28.631 INFO:teuthology.orchestra.run.smithi089.stdout:Get:23 https://4.chacra.ceph.com/r/ceph/pacific/d8c5d34975dce1c5eb0aa3a7979a4d9b9a99d1ec/ubuntu/focal/flavors/default focal/main amd64 python3-rgw amd64 16.2.13-257-gd8c5d349-1focal [116 kB] 2023-06-03T19:20:28.634 INFO:teuthology.orchestra.run.smithi089.stdout:Get:24 https://4.chacra.ceph.com/r/ceph/pacific/d8c5d34975dce1c5eb0aa3a7979a4d9b9a99d1ec/ubuntu/focal/flavors/default focal/main amd64 libradosstriper1 amd64 16.2.13-257-gd8c5d349-1focal [448 kB] 2023-06-03T19:20:28.678 INFO:teuthology.orchestra.run.smithi089.stdout:Get:25 https://4.chacra.ceph.com/r/ceph/pacific/d8c5d34975dce1c5eb0aa3a7979a4d9b9a99d1ec/ubuntu/focal/flavors/default focal/main amd64 ceph-common amd64 16.2.13-257-gd8c5d349-1focal [22.6 MB] 2023-06-03T19:20:28.719 INFO:teuthology.orchestra.run.smithi089.stdout:Get:26 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-logutils all 0.3.3-7 [17.3 kB] 2023-06-03T19:20:28.730 INFO:teuthology.orchestra.run.smithi164.stdout:Get:23 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-logutils all 0.3.3-7 [17.3 kB] 2023-06-03T19:20:28.748 INFO:teuthology.orchestra.run.smithi089.stdout:Get:27 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-markupsafe amd64 1.1.0-1build2 [13.9 kB] 2023-06-03T19:20:28.760 INFO:teuthology.orchestra.run.smithi164.stdout:Get:24 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-markupsafe amd64 1.1.0-1build2 [13.9 kB] 2023-06-03T19:20:28.773 INFO:teuthology.orchestra.run.smithi089.stdout:Get:28 http://archive.ubuntu.com/ubuntu focal-updates/main amd64 python3-mako all 1.1.0+ds1-1ubuntu2.1 [59.2 kB] 2023-06-03T19:20:28.784 INFO:teuthology.orchestra.run.smithi164.stdout:Get:25 http://archive.ubuntu.com/ubuntu focal-updates/main amd64 python3-mako all 1.1.0+ds1-1ubuntu2.1 [59.2 kB] 2023-06-03T19:20:28.865 INFO:teuthology.orchestra.run.smithi164.stdout:Get:26 https://4.chacra.ceph.com/r/ceph/pacific/d8c5d34975dce1c5eb0aa3a7979a4d9b9a99d1ec/ubuntu/focal/flavors/default focal/main amd64 python3-rgw amd64 16.2.13-257-gd8c5d349-1focal [116 kB] 2023-06-03T19:20:28.869 INFO:teuthology.orchestra.run.smithi164.stdout:Get:27 https://4.chacra.ceph.com/r/ceph/pacific/d8c5d34975dce1c5eb0aa3a7979a4d9b9a99d1ec/ubuntu/focal/flavors/default focal/main amd64 libradosstriper1 amd64 16.2.13-257-gd8c5d349-1focal [448 kB] 2023-06-03T19:20:28.888 INFO:teuthology.orchestra.run.smithi089.stdout:Get:29 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-simplegeneric all 0.8.1-3 [11.3 kB] 2023-06-03T19:20:28.900 INFO:teuthology.orchestra.run.smithi164.stdout:Get:28 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-simplegeneric all 0.8.1-3 [11.3 kB] 2023-06-03T19:20:28.901 INFO:teuthology.orchestra.run.smithi089.stdout:Get:30 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-singledispatch all 3.4.0.3-2build1 [7,128 B] 2023-06-03T19:20:28.905 INFO:teuthology.orchestra.run.smithi089.stdout:Get:31 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-webob all 1:1.8.5-2 [84.4 kB] 2023-06-03T19:20:28.913 INFO:teuthology.orchestra.run.smithi164.stdout:Get:29 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-singledispatch all 3.4.0.3-2build1 [7,128 B] 2023-06-03T19:20:28.917 INFO:teuthology.orchestra.run.smithi164.stdout:Get:30 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-webob all 1:1.8.5-2 [84.4 kB] 2023-06-03T19:20:28.928 INFO:teuthology.orchestra.run.smithi164.stdout:Get:31 https://4.chacra.ceph.com/r/ceph/pacific/d8c5d34975dce1c5eb0aa3a7979a4d9b9a99d1ec/ubuntu/focal/flavors/default focal/main amd64 ceph-common amd64 16.2.13-257-gd8c5d349-1focal [22.6 MB] 2023-06-03T19:20:28.972 INFO:teuthology.orchestra.run.smithi089.stdout:Get:32 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-soupsieve all 1.9.5+dfsg-1 [29.1 kB] 2023-06-03T19:20:28.984 INFO:teuthology.orchestra.run.smithi164.stdout:Get:32 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-soupsieve all 1.9.5+dfsg-1 [29.1 kB] 2023-06-03T19:20:28.988 INFO:teuthology.orchestra.run.smithi089.stdout:Get:33 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-bs4 all 4.8.2-1 [83.0 kB] 2023-06-03T19:20:29.000 INFO:teuthology.orchestra.run.smithi164.stdout:Get:33 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-bs4 all 4.8.2-1 [83.0 kB] 2023-06-03T19:20:29.023 INFO:teuthology.orchestra.run.smithi089.stdout:Get:34 http://archive.ubuntu.com/ubuntu focal-updates/main amd64 python3-waitress all 1.4.1-1ubuntu0.1 [81.4 kB] 2023-06-03T19:20:29.035 INFO:teuthology.orchestra.run.smithi164.stdout:Get:34 http://archive.ubuntu.com/ubuntu focal-updates/main amd64 python3-waitress all 1.4.1-1ubuntu0.1 [81.4 kB] 2023-06-03T19:20:29.050 INFO:teuthology.orchestra.run.smithi089.stdout:Get:35 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-tempita all 0.5.2-6 [14.8 kB] 2023-06-03T19:20:29.062 INFO:teuthology.orchestra.run.smithi164.stdout:Get:35 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-tempita all 0.5.2-6 [14.8 kB] 2023-06-03T19:20:29.218 INFO:teuthology.orchestra.run.smithi089.stdout:Get:36 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-paste all 3.4.0+dfsg1-1 [456 kB] 2023-06-03T19:20:29.230 INFO:teuthology.orchestra.run.smithi164.stdout:Get:36 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-paste all 3.4.0+dfsg1-1 [456 kB] 2023-06-03T19:20:29.744 INFO:teuthology.orchestra.run.smithi089.stdout:Get:37 http://archive.ubuntu.com/ubuntu focal/main amd64 python-pastedeploy-tpl all 2.1.0-1 [5,016 B] 2023-06-03T19:20:29.745 INFO:teuthology.orchestra.run.smithi089.stdout:Get:38 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-pastedeploy all 2.1.0-1 [26.5 kB] 2023-06-03T19:20:29.747 INFO:teuthology.orchestra.run.smithi089.stdout:Get:39 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-webtest all 2.0.32-2 [28.2 kB] 2023-06-03T19:20:29.751 INFO:teuthology.orchestra.run.smithi089.stdout:Get:40 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-pecan all 1.3.3-3ubuntu1 [85.8 kB] 2023-06-03T19:20:29.755 INFO:teuthology.orchestra.run.smithi164.stdout:Get:37 http://archive.ubuntu.com/ubuntu focal/main amd64 python-pastedeploy-tpl all 2.1.0-1 [5,016 B] 2023-06-03T19:20:29.755 INFO:teuthology.orchestra.run.smithi164.stdout:Get:38 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-pastedeploy all 2.1.0-1 [26.5 kB] 2023-06-03T19:20:29.758 INFO:teuthology.orchestra.run.smithi164.stdout:Get:39 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-webtest all 2.0.32-2 [28.2 kB] 2023-06-03T19:20:29.762 INFO:teuthology.orchestra.run.smithi164.stdout:Get:40 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-pecan all 1.3.3-3ubuntu1 [85.8 kB] 2023-06-03T19:20:29.763 INFO:teuthology.orchestra.run.smithi089.stdout:Get:41 http://archive.ubuntu.com/ubuntu focal-updates/main amd64 python3-werkzeug all 0.16.1+dfsg1-2ubuntu0.1 [183 kB] 2023-06-03T19:20:29.773 INFO:teuthology.orchestra.run.smithi164.stdout:Get:41 http://archive.ubuntu.com/ubuntu focal-updates/main amd64 python3-werkzeug all 0.16.1+dfsg1-2ubuntu0.1 [183 kB] 2023-06-03T19:20:29.786 INFO:teuthology.orchestra.run.smithi089.stdout:Get:42 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-jinja2 all 2.10.1-2 [95.5 kB] 2023-06-03T19:20:29.796 INFO:teuthology.orchestra.run.smithi164.stdout:Get:42 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-jinja2 all 2.10.1-2 [95.5 kB] 2023-06-03T19:20:29.798 INFO:teuthology.orchestra.run.smithi089.stdout:Get:43 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-repoze.lru all 0.7-2 [12.1 kB] 2023-06-03T19:20:29.800 INFO:teuthology.orchestra.run.smithi089.stdout:Get:44 http://archive.ubuntu.com/ubuntu focal-updates/main amd64 libjs-underscore all 1.9.1~dfsg-1ubuntu0.20.04.1 [99.5 kB] 2023-06-03T19:20:29.809 INFO:teuthology.orchestra.run.smithi164.stdout:Get:43 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-repoze.lru all 0.7-2 [12.1 kB] 2023-06-03T19:20:29.810 INFO:teuthology.orchestra.run.smithi164.stdout:Get:44 http://archive.ubuntu.com/ubuntu focal-updates/main amd64 libjs-underscore all 1.9.1~dfsg-1ubuntu0.20.04.1 [99.5 kB] 2023-06-03T19:20:29.834 INFO:teuthology.orchestra.run.smithi089.stdout:Get:45 http://archive.ubuntu.com/ubuntu focal/main amd64 libjs-sphinxdoc all 1.8.5-7ubuntu3 [97.1 kB] 2023-06-03T19:20:29.844 INFO:teuthology.orchestra.run.smithi164.stdout:Get:45 http://archive.ubuntu.com/ubuntu focal/main amd64 libjs-sphinxdoc all 1.8.5-7ubuntu3 [97.1 kB] 2023-06-03T19:20:29.847 INFO:teuthology.orchestra.run.smithi089.stdout:Get:46 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-routes all 2.4.1-2 [88.1 kB] 2023-06-03T19:20:29.857 INFO:teuthology.orchestra.run.smithi164.stdout:Get:46 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-routes all 2.4.1-2 [88.1 kB] 2023-06-03T19:20:30.023 INFO:teuthology.orchestra.run.smithi089.stdout:Get:47 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-decorator all 4.4.2-0ubuntu1 [10.3 kB] 2023-06-03T19:20:30.031 INFO:teuthology.orchestra.run.smithi164.stdout:Get:47 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-decorator all 4.4.2-0ubuntu1 [10.3 kB] 2023-06-03T19:20:30.114 INFO:teuthology.orchestra.run.smithi089.stdout:Get:48 https://4.chacra.ceph.com/r/ceph/pacific/d8c5d34975dce1c5eb0aa3a7979a4d9b9a99d1ec/ubuntu/focal/flavors/default focal/main amd64 ceph-base amd64 16.2.13-257-gd8c5d349-1focal [5,911 kB] 2023-06-03T19:20:30.114 INFO:teuthology.orchestra.run.smithi164.stdout:Get:48 http://archive.ubuntu.com/ubuntu focal/universe amd64 liblbfgsb0 amd64 3.0+dfsg.3-7build1 [29.5 kB] 2023-06-03T19:20:30.157 INFO:teuthology.orchestra.run.smithi089.stdout:Get:49 http://archive.ubuntu.com/ubuntu focal/universe amd64 liblbfgsb0 amd64 3.0+dfsg.3-7build1 [29.5 kB] 2023-06-03T19:20:30.198 INFO:teuthology.orchestra.run.smithi164.stdout:Get:49 http://archive.ubuntu.com/ubuntu focal/universe amd64 python3-scipy amd64 1.3.3-3build1 [11.1 MB] 2023-06-03T19:20:30.242 INFO:teuthology.orchestra.run.smithi089.stdout:Get:50 http://archive.ubuntu.com/ubuntu focal/universe amd64 python3-scipy amd64 1.3.3-3build1 [11.1 MB] 2023-06-03T19:20:30.353 INFO:teuthology.orchestra.run.smithi089.stdout:Get:51 https://4.chacra.ceph.com/r/ceph/pacific/d8c5d34975dce1c5eb0aa3a7979a4d9b9a99d1ec/ubuntu/focal/flavors/default focal/main amd64 ceph-mgr-modules-core all 16.2.13-257-gd8c5d349-1focal [203 kB] 2023-06-03T19:20:30.391 INFO:teuthology.orchestra.run.smithi089.stdout:Get:52 https://4.chacra.ceph.com/r/ceph/pacific/d8c5d34975dce1c5eb0aa3a7979a4d9b9a99d1ec/ubuntu/focal/flavors/default focal/main amd64 libsqlite3-mod-ceph amd64 16.2.13-257-gd8c5d349-1focal [126 kB] 2023-06-03T19:20:30.395 INFO:teuthology.orchestra.run.smithi089.stdout:Get:53 https://4.chacra.ceph.com/r/ceph/pacific/d8c5d34975dce1c5eb0aa3a7979a4d9b9a99d1ec/ubuntu/focal/flavors/default focal/main amd64 ceph-mgr amd64 16.2.13-257-gd8c5d349-1focal [1,532 kB] 2023-06-03T19:20:30.477 INFO:teuthology.orchestra.run.smithi089.stdout:Get:54 https://4.chacra.ceph.com/r/ceph/pacific/d8c5d34975dce1c5eb0aa3a7979a4d9b9a99d1ec/ubuntu/focal/flavors/default focal/main amd64 ceph-mon amd64 16.2.13-257-gd8c5d349-1focal [6,986 kB] 2023-06-03T19:20:30.700 INFO:teuthology.orchestra.run.smithi164.stdout:Get:50 https://4.chacra.ceph.com/r/ceph/pacific/d8c5d34975dce1c5eb0aa3a7979a4d9b9a99d1ec/ubuntu/focal/flavors/default focal/main amd64 ceph-base amd64 16.2.13-257-gd8c5d349-1focal [5,911 kB] 2023-06-03T19:20:30.882 INFO:teuthology.orchestra.run.smithi089.stdout:Get:55 https://4.chacra.ceph.com/r/ceph/pacific/d8c5d34975dce1c5eb0aa3a7979a4d9b9a99d1ec/ubuntu/focal/flavors/default focal/main amd64 ceph-osd amd64 16.2.13-257-gd8c5d349-1focal [26.2 MB] 2023-06-03T19:20:30.971 INFO:teuthology.orchestra.run.smithi164.stdout:Get:51 http://archive.ubuntu.com/ubuntu focal/universe amd64 python3-sklearn-lib amd64 0.22.2.post1+dfsg-5 [2,358 kB] 2023-06-03T19:20:31.013 INFO:teuthology.orchestra.run.smithi164.stdout:Get:52 http://archive.ubuntu.com/ubuntu focal/universe amd64 python3-joblib all 0.14.0-3 [192 kB] 2023-06-03T19:20:31.014 INFO:teuthology.orchestra.run.smithi164.stdout:Get:53 http://archive.ubuntu.com/ubuntu focal/universe amd64 python3-sklearn all 0.22.2.post1+dfsg-5 [1,773 kB] 2023-06-03T19:20:31.036 INFO:teuthology.orchestra.run.smithi089.stdout:Get:56 http://archive.ubuntu.com/ubuntu focal/universe amd64 python3-sklearn-lib amd64 0.22.2.post1+dfsg-5 [2,358 kB] 2023-06-03T19:20:31.038 INFO:teuthology.orchestra.run.smithi164.stdout:Get:54 https://4.chacra.ceph.com/r/ceph/pacific/d8c5d34975dce1c5eb0aa3a7979a4d9b9a99d1ec/ubuntu/focal/flavors/default focal/main amd64 ceph-mgr-modules-core all 16.2.13-257-gd8c5d349-1focal [203 kB] 2023-06-03T19:20:31.044 INFO:teuthology.orchestra.run.smithi164.stdout:Get:55 https://4.chacra.ceph.com/r/ceph/pacific/d8c5d34975dce1c5eb0aa3a7979a4d9b9a99d1ec/ubuntu/focal/flavors/default focal/main amd64 libsqlite3-mod-ceph amd64 16.2.13-257-gd8c5d349-1focal [126 kB] 2023-06-03T19:20:31.048 INFO:teuthology.orchestra.run.smithi164.stdout:Get:56 https://4.chacra.ceph.com/r/ceph/pacific/d8c5d34975dce1c5eb0aa3a7979a4d9b9a99d1ec/ubuntu/focal/flavors/default focal/main amd64 ceph-mgr amd64 16.2.13-257-gd8c5d349-1focal [1,532 kB] 2023-06-03T19:20:31.054 INFO:teuthology.orchestra.run.smithi089.stdout:Get:57 http://archive.ubuntu.com/ubuntu focal/universe amd64 python3-joblib all 0.14.0-3 [192 kB] 2023-06-03T19:20:31.056 INFO:teuthology.orchestra.run.smithi089.stdout:Get:58 http://archive.ubuntu.com/ubuntu focal/universe amd64 python3-sklearn all 0.22.2.post1+dfsg-5 [1,773 kB] 2023-06-03T19:20:31.091 INFO:teuthology.orchestra.run.smithi164.stdout:Get:57 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-cachetools all 4.0.0-1 [9,820 B] 2023-06-03T19:20:31.091 INFO:teuthology.orchestra.run.smithi164.stdout:Get:58 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-pyasn1 all 0.4.2-3build1 [46.7 kB] 2023-06-03T19:20:31.092 INFO:teuthology.orchestra.run.smithi164.stdout:Get:59 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-pyasn1-modules all 0.2.1-0.2build1 [32.9 kB] 2023-06-03T19:20:31.092 INFO:teuthology.orchestra.run.smithi164.stdout:Get:60 http://archive.ubuntu.com/ubuntu focal/universe amd64 python3-rsa all 4.0-3 [24.7 kB] 2023-06-03T19:20:31.092 INFO:teuthology.orchestra.run.smithi164.stdout:Get:61 http://archive.ubuntu.com/ubuntu focal/universe amd64 python3-google-auth all 1.5.1-2 [35.6 kB] 2023-06-03T19:20:31.140 INFO:teuthology.orchestra.run.smithi089.stdout:Get:59 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-cachetools all 4.0.0-1 [9,820 B] 2023-06-03T19:20:31.140 INFO:teuthology.orchestra.run.smithi089.stdout:Get:60 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-pyasn1 all 0.4.2-3build1 [46.7 kB] 2023-06-03T19:20:31.141 INFO:teuthology.orchestra.run.smithi089.stdout:Get:61 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-pyasn1-modules all 0.2.1-0.2build1 [32.9 kB] 2023-06-03T19:20:31.141 INFO:teuthology.orchestra.run.smithi089.stdout:Get:62 http://archive.ubuntu.com/ubuntu focal/universe amd64 python3-rsa all 4.0-3 [24.7 kB] 2023-06-03T19:20:31.141 INFO:teuthology.orchestra.run.smithi089.stdout:Get:63 http://archive.ubuntu.com/ubuntu focal/universe amd64 python3-google-auth all 1.5.1-2 [35.6 kB] 2023-06-03T19:20:31.154 INFO:teuthology.orchestra.run.smithi164.stdout:Get:62 https://4.chacra.ceph.com/r/ceph/pacific/d8c5d34975dce1c5eb0aa3a7979a4d9b9a99d1ec/ubuntu/focal/flavors/default focal/main amd64 ceph-mon amd64 16.2.13-257-gd8c5d349-1focal [6,986 kB] 2023-06-03T19:20:31.258 INFO:teuthology.orchestra.run.smithi164.stdout:Get:63 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-blinker all 1.4+dfsg1-0.3ubuntu1 [13.2 kB] 2023-06-03T19:20:31.305 INFO:teuthology.orchestra.run.smithi089.stdout:Get:64 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-blinker all 1.4+dfsg1-0.3ubuntu1 [13.2 kB] 2023-06-03T19:20:31.341 INFO:teuthology.orchestra.run.smithi164.stdout:Get:64 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-oauthlib all 3.1.0-1ubuntu2 [84.8 kB] 2023-06-03T19:20:31.389 INFO:teuthology.orchestra.run.smithi089.stdout:Get:65 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-oauthlib all 3.1.0-1ubuntu2 [84.8 kB] 2023-06-03T19:20:31.522 INFO:teuthology.orchestra.run.smithi164.stdout:Get:65 http://archive.ubuntu.com/ubuntu focal/universe amd64 python3-requests-oauthlib all 1.0.0-1.1build1 [16.8 kB] 2023-06-03T19:20:31.538 INFO:teuthology.orchestra.run.smithi164.stdout:Get:66 http://archive.ubuntu.com/ubuntu focal/universe amd64 python3-websocket all 0.53.0-2ubuntu1 [32.3 kB] 2023-06-03T19:20:31.569 INFO:teuthology.orchestra.run.smithi089.stdout:Get:66 http://archive.ubuntu.com/ubuntu focal/universe amd64 python3-requests-oauthlib all 1.0.0-1.1build1 [16.8 kB] 2023-06-03T19:20:31.570 INFO:teuthology.orchestra.run.smithi164.stdout:Get:67 http://archive.ubuntu.com/ubuntu focal/universe amd64 python3-kubernetes all 7.0.0~a1-2 [271 kB] 2023-06-03T19:20:31.584 INFO:teuthology.orchestra.run.smithi089.stdout:Get:67 http://archive.ubuntu.com/ubuntu focal/universe amd64 python3-websocket all 0.53.0-2ubuntu1 [32.3 kB] 2023-06-03T19:20:31.617 INFO:teuthology.orchestra.run.smithi089.stdout:Get:68 http://archive.ubuntu.com/ubuntu focal/universe amd64 python3-kubernetes all 7.0.0~a1-2 [271 kB] 2023-06-03T19:20:31.701 INFO:teuthology.orchestra.run.smithi164.stdout:Get:68 http://archive.ubuntu.com/ubuntu focal/universe amd64 libonig5 amd64 6.9.4-1 [142 kB] 2023-06-03T19:20:31.712 INFO:teuthology.orchestra.run.smithi164.stdout:Get:69 https://4.chacra.ceph.com/r/ceph/pacific/d8c5d34975dce1c5eb0aa3a7979a4d9b9a99d1ec/ubuntu/focal/flavors/default focal/main amd64 ceph-osd amd64 16.2.13-257-gd8c5d349-1focal [26.2 MB] 2023-06-03T19:20:31.738 INFO:teuthology.orchestra.run.smithi164.stdout:Get:70 http://archive.ubuntu.com/ubuntu focal-updates/universe amd64 libjq1 amd64 1.6-1ubuntu0.20.04.1 [121 kB] 2023-06-03T19:20:31.747 INFO:teuthology.orchestra.run.smithi089.stdout:Get:69 http://archive.ubuntu.com/ubuntu focal/universe amd64 libonig5 amd64 6.9.4-1 [142 kB] 2023-06-03T19:20:31.761 INFO:teuthology.orchestra.run.smithi164.stdout:Get:71 http://archive.ubuntu.com/ubuntu focal-updates/universe amd64 jq amd64 1.6-1ubuntu0.20.04.1 [50.2 kB] 2023-06-03T19:20:31.770 INFO:teuthology.orchestra.run.smithi164.stdout:Get:72 http://archive.ubuntu.com/ubuntu focal/main amd64 socat amd64 1.7.3.3-2 [323 kB] 2023-06-03T19:20:31.783 INFO:teuthology.orchestra.run.smithi089.stdout:Get:70 http://archive.ubuntu.com/ubuntu focal-updates/universe amd64 libjq1 amd64 1.6-1ubuntu0.20.04.1 [121 kB] 2023-06-03T19:20:31.807 INFO:teuthology.orchestra.run.smithi089.stdout:Get:71 http://archive.ubuntu.com/ubuntu focal-updates/universe amd64 jq amd64 1.6-1ubuntu0.20.04.1 [50.2 kB] 2023-06-03T19:20:31.812 INFO:teuthology.orchestra.run.smithi164.stdout:Get:73 http://archive.ubuntu.com/ubuntu focal/universe amd64 xmlstarlet amd64 1.6.1-2build1 [267 kB] 2023-06-03T19:20:31.815 INFO:teuthology.orchestra.run.smithi089.stdout:Get:72 http://archive.ubuntu.com/ubuntu focal/main amd64 socat amd64 1.7.3.3-2 [323 kB] 2023-06-03T19:20:31.839 INFO:teuthology.orchestra.run.smithi164.stdout:Get:74 http://archive.ubuntu.com/ubuntu focal-updates/main amd64 nvme-cli amd64 1.9-1ubuntu0.1 [283 kB] 2023-06-03T19:20:31.858 INFO:teuthology.orchestra.run.smithi089.stdout:Get:73 http://archive.ubuntu.com/ubuntu focal/universe amd64 xmlstarlet amd64 1.6.1-2build1 [267 kB] 2023-06-03T19:20:31.884 INFO:teuthology.orchestra.run.smithi089.stdout:Get:74 http://archive.ubuntu.com/ubuntu focal-updates/main amd64 nvme-cli amd64 1.9-1ubuntu0.1 [283 kB] 2023-06-03T19:20:32.027 INFO:teuthology.orchestra.run.smithi164.stdout:Get:75 http://archive.ubuntu.com/ubuntu focal/universe amd64 python3-atomicwrites all 1.1.5-2build1 [7,016 B] 2023-06-03T19:20:32.071 INFO:teuthology.orchestra.run.smithi089.stdout:Get:75 http://archive.ubuntu.com/ubuntu focal/universe amd64 python3-atomicwrites all 1.1.5-2build1 [7,016 B] 2023-06-03T19:20:32.110 INFO:teuthology.orchestra.run.smithi164.stdout:Get:76 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-attr all 19.3.0-2 [33.9 kB] 2023-06-03T19:20:32.155 INFO:teuthology.orchestra.run.smithi089.stdout:Get:76 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-attr all 19.3.0-2 [33.9 kB] 2023-06-03T19:20:32.212 INFO:teuthology.orchestra.run.smithi164.stdout:Get:77 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-webencodings all 0.5.1-1ubuntu1 [11.0 kB] 2023-06-03T19:20:32.235 INFO:teuthology.orchestra.run.smithi164.stdout:Get:78 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-html5lib all 1.0.1-2 [84.3 kB] 2023-06-03T19:20:32.256 INFO:teuthology.orchestra.run.smithi089.stdout:Get:77 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-webencodings all 0.5.1-1ubuntu1 [11.0 kB] 2023-06-03T19:20:32.279 INFO:teuthology.orchestra.run.smithi089.stdout:Get:78 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-html5lib all 1.0.1-2 [84.3 kB] 2023-06-03T19:20:32.351 INFO:teuthology.orchestra.run.smithi164.stdout:Get:79 http://archive.ubuntu.com/ubuntu focal-updates/main amd64 python3-lxml amd64 4.5.0-1ubuntu0.5 [1,384 kB] 2023-06-03T19:20:32.459 INFO:teuthology.orchestra.run.smithi089.stdout:Get:79 http://archive.ubuntu.com/ubuntu focal-updates/main amd64 python3-lxml amd64 4.5.0-1ubuntu0.5 [1,384 kB] 2023-06-03T19:20:32.523 INFO:teuthology.orchestra.run.smithi089.stdout:Get:80 https://4.chacra.ceph.com/r/ceph/pacific/d8c5d34975dce1c5eb0aa3a7979a4d9b9a99d1ec/ubuntu/focal/flavors/default focal/main amd64 ceph amd64 16.2.13-257-gd8c5d349-1focal [4,048 B] 2023-06-03T19:20:32.523 INFO:teuthology.orchestra.run.smithi089.stdout:Get:81 https://4.chacra.ceph.com/r/ceph/pacific/d8c5d34975dce1c5eb0aa3a7979a4d9b9a99d1ec/ubuntu/focal/flavors/default focal/main amd64 ceph-fuse amd64 16.2.13-257-gd8c5d349-1focal [834 kB] 2023-06-03T19:20:32.529 INFO:teuthology.orchestra.run.smithi089.stdout:Get:82 https://4.chacra.ceph.com/r/ceph/pacific/d8c5d34975dce1c5eb0aa3a7979a4d9b9a99d1ec/ubuntu/focal/flavors/default focal/main amd64 ceph-mds amd64 16.2.13-257-gd8c5d349-1focal [2,357 kB] 2023-06-03T19:20:32.546 INFO:teuthology.orchestra.run.smithi089.stdout:Get:83 https://4.chacra.ceph.com/r/ceph/pacific/d8c5d34975dce1c5eb0aa3a7979a4d9b9a99d1ec/ubuntu/focal/flavors/default focal/main amd64 cephadm amd64 16.2.13-257-gd8c5d349-1focal [76.4 kB] 2023-06-03T19:20:32.547 INFO:teuthology.orchestra.run.smithi089.stdout:Get:84 https://4.chacra.ceph.com/r/ceph/pacific/d8c5d34975dce1c5eb0aa3a7979a4d9b9a99d1ec/ubuntu/focal/flavors/default focal/main amd64 ceph-mgr-cephadm all 16.2.13-257-gd8c5d349-1focal [100 kB] 2023-06-03T19:20:32.547 INFO:teuthology.orchestra.run.smithi089.stdout:Get:85 https://4.chacra.ceph.com/r/ceph/pacific/d8c5d34975dce1c5eb0aa3a7979a4d9b9a99d1ec/ubuntu/focal/flavors/default focal/main amd64 ceph-mgr-dashboard all 16.2.13-257-gd8c5d349-1focal [2,823 kB] 2023-06-03T19:20:32.638 INFO:teuthology.orchestra.run.smithi164.stdout:Get:80 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-packaging all 20.3-1 [26.8 kB] 2023-06-03T19:20:32.639 INFO:teuthology.orchestra.run.smithi164.stdout:Get:81 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-pastescript all 2.0.2-4 [54.6 kB] 2023-06-03T19:20:32.644 INFO:teuthology.orchestra.run.smithi164.stdout:Get:82 http://archive.ubuntu.com/ubuntu focal/universe amd64 python3-pluggy all 0.13.0-2 [18.4 kB] 2023-06-03T19:20:32.645 INFO:teuthology.orchestra.run.smithi164.stdout:Get:83 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-psutil amd64 5.5.1-1ubuntu4 [141 kB] 2023-06-03T19:20:32.655 INFO:teuthology.orchestra.run.smithi164.stdout:Get:84 http://archive.ubuntu.com/ubuntu focal-updates/universe amd64 python3-py all 1.8.1-1ubuntu0.1 [65.6 kB] 2023-06-03T19:20:32.660 INFO:teuthology.orchestra.run.smithi164.stdout:Get:85 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-pyinotify all 0.9.6-1.2ubuntu1 [24.8 kB] 2023-06-03T19:20:32.689 INFO:teuthology.orchestra.run.smithi089.stdout:Get:86 https://4.chacra.ceph.com/r/ceph/pacific/d8c5d34975dce1c5eb0aa3a7979a4d9b9a99d1ec/ubuntu/focal/flavors/default focal/main amd64 ceph-mgr-diskprediction-local all 16.2.13-257-gd8c5d349-1focal [7,884 kB] 2023-06-03T19:20:32.710 INFO:teuthology.orchestra.run.smithi089.stdout:Get:87 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-packaging all 20.3-1 [26.8 kB] 2023-06-03T19:20:32.711 INFO:teuthology.orchestra.run.smithi089.stdout:Get:88 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-pastescript all 2.0.2-4 [54.6 kB] 2023-06-03T19:20:32.716 INFO:teuthology.orchestra.run.smithi089.stdout:Get:89 http://archive.ubuntu.com/ubuntu focal/universe amd64 python3-pluggy all 0.13.0-2 [18.4 kB] 2023-06-03T19:20:32.716 INFO:teuthology.orchestra.run.smithi089.stdout:Get:90 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-psutil amd64 5.5.1-1ubuntu4 [141 kB] 2023-06-03T19:20:32.727 INFO:teuthology.orchestra.run.smithi089.stdout:Get:91 http://archive.ubuntu.com/ubuntu focal-updates/universe amd64 python3-py all 1.8.1-1ubuntu0.1 [65.6 kB] 2023-06-03T19:20:32.732 INFO:teuthology.orchestra.run.smithi089.stdout:Get:92 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-pyinotify all 0.9.6-1.2ubuntu1 [24.8 kB] 2023-06-03T19:20:32.825 INFO:teuthology.orchestra.run.smithi164.stdout:Get:86 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-wcwidth all 0.1.8+dfsg1-3 [17.4 kB] 2023-06-03T19:20:32.896 INFO:teuthology.orchestra.run.smithi089.stdout:Get:93 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-wcwidth all 0.1.8+dfsg1-3 [17.4 kB] 2023-06-03T19:20:32.991 INFO:teuthology.orchestra.run.smithi164.stdout:Get:87 http://archive.ubuntu.com/ubuntu focal/universe amd64 python3-pytest all 4.6.9-1 [178 kB] 2023-06-03T19:20:33.062 INFO:teuthology.orchestra.run.smithi089.stdout:Get:94 http://archive.ubuntu.com/ubuntu focal/universe amd64 python3-pytest all 4.6.9-1 [178 kB] 2023-06-03T19:20:33.139 INFO:teuthology.orchestra.run.smithi089.stdout:Get:95 https://4.chacra.ceph.com/r/ceph/pacific/d8c5d34975dce1c5eb0aa3a7979a4d9b9a99d1ec/ubuntu/focal/flavors/default focal/main amd64 ceph-mgr-k8sevents all 16.2.13-257-gd8c5d349-1focal [4,164 B] 2023-06-03T19:20:33.139 INFO:teuthology.orchestra.run.smithi089.stdout:Get:96 https://4.chacra.ceph.com/r/ceph/pacific/d8c5d34975dce1c5eb0aa3a7979a4d9b9a99d1ec/ubuntu/focal/flavors/default focal/main amd64 ceph-test amd64 16.2.13-257-gd8c5d349-1focal [49.3 MB] 2023-06-03T19:20:33.244 INFO:teuthology.orchestra.run.smithi164.stdout:Get:88 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-simplejson amd64 3.16.0-2ubuntu2 [50.8 kB] 2023-06-03T19:20:33.313 INFO:teuthology.orchestra.run.smithi089.stdout:Get:97 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-simplejson amd64 3.16.0-2ubuntu2 [50.8 kB] 2023-06-03T19:20:34.003 INFO:teuthology.orchestra.run.smithi164.stdout:Get:89 https://4.chacra.ceph.com/r/ceph/pacific/d8c5d34975dce1c5eb0aa3a7979a4d9b9a99d1ec/ubuntu/focal/flavors/default focal/main amd64 ceph amd64 16.2.13-257-gd8c5d349-1focal [4,048 B] 2023-06-03T19:20:34.004 INFO:teuthology.orchestra.run.smithi164.stdout:Get:90 https://4.chacra.ceph.com/r/ceph/pacific/d8c5d34975dce1c5eb0aa3a7979a4d9b9a99d1ec/ubuntu/focal/flavors/default focal/main amd64 ceph-fuse amd64 16.2.13-257-gd8c5d349-1focal [834 kB] 2023-06-03T19:20:34.009 INFO:teuthology.orchestra.run.smithi164.stdout:Get:91 https://4.chacra.ceph.com/r/ceph/pacific/d8c5d34975dce1c5eb0aa3a7979a4d9b9a99d1ec/ubuntu/focal/flavors/default focal/main amd64 ceph-mds amd64 16.2.13-257-gd8c5d349-1focal [2,357 kB] 2023-06-03T19:20:34.187 INFO:teuthology.orchestra.run.smithi164.stdout:Get:92 https://4.chacra.ceph.com/r/ceph/pacific/d8c5d34975dce1c5eb0aa3a7979a4d9b9a99d1ec/ubuntu/focal/flavors/default focal/main amd64 cephadm amd64 16.2.13-257-gd8c5d349-1focal [76.4 kB] 2023-06-03T19:20:34.189 INFO:teuthology.orchestra.run.smithi164.stdout:Get:93 https://4.chacra.ceph.com/r/ceph/pacific/d8c5d34975dce1c5eb0aa3a7979a4d9b9a99d1ec/ubuntu/focal/flavors/default focal/main amd64 ceph-mgr-cephadm all 16.2.13-257-gd8c5d349-1focal [100 kB] 2023-06-03T19:20:34.191 INFO:teuthology.orchestra.run.smithi164.stdout:Get:94 https://4.chacra.ceph.com/r/ceph/pacific/d8c5d34975dce1c5eb0aa3a7979a4d9b9a99d1ec/ubuntu/focal/flavors/default focal/main amd64 ceph-mgr-dashboard all 16.2.13-257-gd8c5d349-1focal [2,823 kB] 2023-06-03T19:20:34.531 INFO:teuthology.orchestra.run.smithi164.stdout:Get:95 https://4.chacra.ceph.com/r/ceph/pacific/d8c5d34975dce1c5eb0aa3a7979a4d9b9a99d1ec/ubuntu/focal/flavors/default focal/main amd64 ceph-mgr-diskprediction-local all 16.2.13-257-gd8c5d349-1focal [7,884 kB] 2023-06-03T19:20:35.131 INFO:teuthology.orchestra.run.smithi164.stdout:Get:96 https://4.chacra.ceph.com/r/ceph/pacific/d8c5d34975dce1c5eb0aa3a7979a4d9b9a99d1ec/ubuntu/focal/flavors/default focal/main amd64 ceph-mgr-k8sevents all 16.2.13-257-gd8c5d349-1focal [4,164 B] 2023-06-03T19:20:35.132 INFO:teuthology.orchestra.run.smithi164.stdout:Get:97 https://4.chacra.ceph.com/r/ceph/pacific/d8c5d34975dce1c5eb0aa3a7979a4d9b9a99d1ec/ubuntu/focal/flavors/default focal/main amd64 ceph-test amd64 16.2.13-257-gd8c5d349-1focal [49.3 MB] 2023-06-03T19:20:35.984 INFO:teuthology.orchestra.run.smithi089.stdout:Get:98 https://4.chacra.ceph.com/r/ceph/pacific/d8c5d34975dce1c5eb0aa3a7979a4d9b9a99d1ec/ubuntu/focal/flavors/default focal/main amd64 libcephfs-dev amd64 16.2.13-257-gd8c5d349-1focal [21.5 kB] 2023-06-03T19:20:35.984 INFO:teuthology.orchestra.run.smithi089.stdout:Get:99 https://4.chacra.ceph.com/r/ceph/pacific/d8c5d34975dce1c5eb0aa3a7979a4d9b9a99d1ec/ubuntu/focal/flavors/default focal/main amd64 radosgw amd64 16.2.13-257-gd8c5d349-1focal [11.3 MB] 2023-06-03T19:20:36.365 INFO:teuthology.orchestra.run.smithi089.stdout:Get:100 https://4.chacra.ceph.com/r/ceph/pacific/d8c5d34975dce1c5eb0aa3a7979a4d9b9a99d1ec/ubuntu/focal/flavors/default focal/main amd64 rbd-fuse amd64 16.2.13-257-gd8c5d349-1focal [78.5 kB] 2023-06-03T19:20:37.017 INFO:teuthology.orchestra.run.smithi089.stdout:Fetched 173 MB in 9s (19.6 MB/s) 2023-06-03T19:20:37.112 INFO:teuthology.orchestra.run.smithi089.stdout:Selecting previously unselected package libjaeger. 2023-06-03T19:20:37.159 INFO:teuthology.orchestra.run.smithi089.stdout:(Reading database ... (Reading database ... 5% (Reading database ... 10% (Reading database ... 15% (Reading database ... 20% (Reading database ... 25% (Reading database ... 30% (Reading database ... 35% (Reading database ... 40% (Reading database ... 45% (Reading database ... 50% (Reading database ... 55% (Reading database ... 60% (Reading database ... 65% (Reading database ... 70% (Reading database ... 75% (Reading database ... 80% (Reading database ... 85% (Reading database ... 90% (Reading database ... 95% (Reading database ... 100% (Reading database ... 112643 files and directories currently installed.) 2023-06-03T19:20:37.162 INFO:teuthology.orchestra.run.smithi089.stdout:Preparing to unpack .../00-libjaeger_16.2.13-257-gd8c5d349-1focal_amd64.deb ... 2023-06-03T19:20:37.203 INFO:teuthology.orchestra.run.smithi089.stdout:Unpacking libjaeger (16.2.13-257-gd8c5d349-1focal) ... 2023-06-03T19:20:37.522 INFO:teuthology.orchestra.run.smithi089.stdout:Selecting previously unselected package liblttng-ust0:amd64. 2023-06-03T19:20:37.537 INFO:teuthology.orchestra.run.smithi089.stdout:Preparing to unpack .../01-liblttng-ust0_2.11.0-1_amd64.deb ... 2023-06-03T19:20:37.622 INFO:teuthology.orchestra.run.smithi089.stdout:Unpacking liblttng-ust0:amd64 (2.11.0-1) ... 2023-06-03T19:20:37.974 INFO:teuthology.orchestra.run.smithi089.stdout:Preparing to unpack .../02-librbd1_16.2.13-257-gd8c5d349-1focal_amd64.deb ... 2023-06-03T19:20:38.109 INFO:teuthology.orchestra.run.smithi089.stdout:Unpacking librbd1 (16.2.13-257-gd8c5d349-1focal) over (15.2.17-0ubuntu0.20.04.4) ... 2023-06-03T19:20:38.761 INFO:teuthology.orchestra.run.smithi089.stdout:Preparing to unpack .../03-librados2_16.2.13-257-gd8c5d349-1focal_amd64.deb ... 2023-06-03T19:20:38.889 INFO:teuthology.orchestra.run.smithi089.stdout:Unpacking librados2 (16.2.13-257-gd8c5d349-1focal) over (15.2.17-0ubuntu0.20.04.4) ... 2023-06-03T19:20:39.087 INFO:teuthology.orchestra.run.smithi164.stdout:Get:98 https://4.chacra.ceph.com/r/ceph/pacific/d8c5d34975dce1c5eb0aa3a7979a4d9b9a99d1ec/ubuntu/focal/flavors/default focal/main amd64 libcephfs-dev amd64 16.2.13-257-gd8c5d349-1focal [21.5 kB] 2023-06-03T19:20:39.087 INFO:teuthology.orchestra.run.smithi164.stdout:Get:99 https://4.chacra.ceph.com/r/ceph/pacific/d8c5d34975dce1c5eb0aa3a7979a4d9b9a99d1ec/ubuntu/focal/flavors/default focal/main amd64 radosgw amd64 16.2.13-257-gd8c5d349-1focal [11.3 MB] 2023-06-03T19:20:39.559 INFO:teuthology.orchestra.run.smithi089.stdout:Selecting previously unselected package libcephfs2. 2023-06-03T19:20:39.575 INFO:teuthology.orchestra.run.smithi089.stdout:Preparing to unpack .../04-libcephfs2_16.2.13-257-gd8c5d349-1focal_amd64.deb ... 2023-06-03T19:20:39.618 INFO:teuthology.orchestra.run.smithi089.stdout:Unpacking libcephfs2 (16.2.13-257-gd8c5d349-1focal) ... 2023-06-03T19:20:39.650 INFO:teuthology.orchestra.run.smithi164.stdout:Get:100 https://4.chacra.ceph.com/r/ceph/pacific/d8c5d34975dce1c5eb0aa3a7979a4d9b9a99d1ec/ubuntu/focal/flavors/default focal/main amd64 rbd-fuse amd64 16.2.13-257-gd8c5d349-1focal [78.5 kB] 2023-06-03T19:20:39.929 INFO:teuthology.orchestra.run.smithi089.stdout:Selecting previously unselected package python3-rados. 2023-06-03T19:20:39.942 INFO:teuthology.orchestra.run.smithi089.stdout:Preparing to unpack .../05-python3-rados_16.2.13-257-gd8c5d349-1focal_amd64.deb ... 2023-06-03T19:20:39.987 INFO:teuthology.orchestra.run.smithi089.stdout:Unpacking python3-rados (16.2.13-257-gd8c5d349-1focal) ... 2023-06-03T19:20:40.296 INFO:teuthology.orchestra.run.smithi164.stdout:Fetched 173 MB in 12s (14.3 MB/s) 2023-06-03T19:20:40.380 INFO:teuthology.orchestra.run.smithi164.stdout:Selecting previously unselected package libjaeger. 2023-06-03T19:20:40.426 INFO:teuthology.orchestra.run.smithi164.stdout:(Reading database ... (Reading database ... 5% (Reading database ... 10% (Reading database ... 15% (Reading database ... 20% (Reading database ... 25% (Reading database ... 30% (Reading database ... 35% (Reading database ... 40% (Reading database ... 45% (Reading database ... 50% (Reading database ... 55% (Reading database ... 60% (Reading database ... 65% (Reading database ... 70% (Reading database ... 75% (Reading database ... 80% (Reading database ... 85% (Reading database ... 90% (Reading database ... 95% (Reading database ... 100% (Reading database ... 112643 files and directories currently installed.) 2023-06-03T19:20:40.429 INFO:teuthology.orchestra.run.smithi164.stdout:Preparing to unpack .../00-libjaeger_16.2.13-257-gd8c5d349-1focal_amd64.deb ... 2023-06-03T19:20:40.448 INFO:teuthology.orchestra.run.smithi089.stdout:Selecting previously unselected package python3-ceph-argparse. 2023-06-03T19:20:40.462 INFO:teuthology.orchestra.run.smithi164.stdout:Unpacking libjaeger (16.2.13-257-gd8c5d349-1focal) ... 2023-06-03T19:20:40.462 INFO:teuthology.orchestra.run.smithi089.stdout:Preparing to unpack .../06-python3-ceph-argparse_16.2.13-257-gd8c5d349-1focal_all.deb ... 2023-06-03T19:20:40.514 INFO:teuthology.orchestra.run.smithi089.stdout:Unpacking python3-ceph-argparse (16.2.13-257-gd8c5d349-1focal) ... 2023-06-03T19:20:40.756 INFO:teuthology.orchestra.run.smithi164.stdout:Selecting previously unselected package liblttng-ust0:amd64. 2023-06-03T19:20:40.769 INFO:teuthology.orchestra.run.smithi164.stdout:Preparing to unpack .../01-liblttng-ust0_2.11.0-1_amd64.deb ... 2023-06-03T19:20:40.826 INFO:teuthology.orchestra.run.smithi089.stdout:Selecting previously unselected package python3-cephfs. 2023-06-03T19:20:40.841 INFO:teuthology.orchestra.run.smithi089.stdout:Preparing to unpack .../07-python3-cephfs_16.2.13-257-gd8c5d349-1focal_amd64.deb ... 2023-06-03T19:20:40.848 INFO:teuthology.orchestra.run.smithi164.stdout:Unpacking liblttng-ust0:amd64 (2.11.0-1) ... 2023-06-03T19:20:40.884 INFO:teuthology.orchestra.run.smithi089.stdout:Unpacking python3-cephfs (16.2.13-257-gd8c5d349-1focal) ... 2023-06-03T19:20:41.191 INFO:teuthology.orchestra.run.smithi164.stdout:Preparing to unpack .../02-librbd1_16.2.13-257-gd8c5d349-1focal_amd64.deb ... 2023-06-03T19:20:41.220 INFO:teuthology.orchestra.run.smithi089.stdout:Selecting previously unselected package python3-ceph-common. 2023-06-03T19:20:41.234 INFO:teuthology.orchestra.run.smithi089.stdout:Preparing to unpack .../08-python3-ceph-common_16.2.13-257-gd8c5d349-1focal_all.deb ... 2023-06-03T19:20:41.278 INFO:teuthology.orchestra.run.smithi089.stdout:Unpacking python3-ceph-common (16.2.13-257-gd8c5d349-1focal) ... 2023-06-03T19:20:41.318 INFO:teuthology.orchestra.run.smithi164.stdout:Unpacking librbd1 (16.2.13-257-gd8c5d349-1focal) over (15.2.17-0ubuntu0.20.04.4) ... 2023-06-03T19:20:41.595 INFO:teuthology.orchestra.run.smithi089.stdout:Selecting previously unselected package python3-prettytable. 2023-06-03T19:20:41.602 INFO:teuthology.orchestra.run.smithi089.stdout:Preparing to unpack .../09-python3-prettytable_0.7.2-5_all.deb ... 2023-06-03T19:20:41.645 INFO:teuthology.orchestra.run.smithi089.stdout:Unpacking python3-prettytable (0.7.2-5) ... 2023-06-03T19:20:41.898 INFO:teuthology.orchestra.run.smithi089.stdout:Selecting previously unselected package python3-rbd. 2023-06-03T19:20:41.913 INFO:teuthology.orchestra.run.smithi089.stdout:Preparing to unpack .../10-python3-rbd_16.2.13-257-gd8c5d349-1focal_amd64.deb ... 2023-06-03T19:20:41.956 INFO:teuthology.orchestra.run.smithi089.stdout:Unpacking python3-rbd (16.2.13-257-gd8c5d349-1focal) ... 2023-06-03T19:20:41.971 INFO:teuthology.orchestra.run.smithi164.stdout:Preparing to unpack .../03-librados2_16.2.13-257-gd8c5d349-1focal_amd64.deb ... 2023-06-03T19:20:42.097 INFO:teuthology.orchestra.run.smithi164.stdout:Unpacking librados2 (16.2.13-257-gd8c5d349-1focal) over (15.2.17-0ubuntu0.20.04.4) ... 2023-06-03T19:20:42.309 INFO:teuthology.orchestra.run.smithi089.stdout:Selecting previously unselected package liblua5.3-0:amd64. 2023-06-03T19:20:42.325 INFO:teuthology.orchestra.run.smithi089.stdout:Preparing to unpack .../11-liblua5.3-0_5.3.3-1.1ubuntu2_amd64.deb ... 2023-06-03T19:20:42.368 INFO:teuthology.orchestra.run.smithi089.stdout:Unpacking liblua5.3-0:amd64 (5.3.3-1.1ubuntu2) ... 2023-06-03T19:20:42.704 INFO:teuthology.orchestra.run.smithi089.stdout:Selecting previously unselected package librabbitmq4:amd64. 2023-06-03T19:20:42.717 INFO:teuthology.orchestra.run.smithi089.stdout:Preparing to unpack .../12-librabbitmq4_0.10.0-1_amd64.deb ... 2023-06-03T19:20:42.754 INFO:teuthology.orchestra.run.smithi089.stdout:Unpacking librabbitmq4:amd64 (0.10.0-1) ... 2023-06-03T19:20:42.759 INFO:teuthology.orchestra.run.smithi164.stdout:Selecting previously unselected package libcephfs2. 2023-06-03T19:20:42.774 INFO:teuthology.orchestra.run.smithi164.stdout:Preparing to unpack .../04-libcephfs2_16.2.13-257-gd8c5d349-1focal_amd64.deb ... 2023-06-03T19:20:42.818 INFO:teuthology.orchestra.run.smithi164.stdout:Unpacking libcephfs2 (16.2.13-257-gd8c5d349-1focal) ... 2023-06-03T19:20:43.092 INFO:teuthology.orchestra.run.smithi089.stdout:Selecting previously unselected package librdkafka1:amd64. 2023-06-03T19:20:43.105 INFO:teuthology.orchestra.run.smithi089.stdout:Preparing to unpack .../13-librdkafka1_1.2.1-1ubuntu1_amd64.deb ... 2023-06-03T19:20:43.150 INFO:teuthology.orchestra.run.smithi089.stdout:Unpacking librdkafka1:amd64 (1.2.1-1ubuntu1) ... 2023-06-03T19:20:43.154 INFO:teuthology.orchestra.run.smithi164.stdout:Selecting previously unselected package python3-rados. 2023-06-03T19:20:43.167 INFO:teuthology.orchestra.run.smithi164.stdout:Preparing to unpack .../05-python3-rados_16.2.13-257-gd8c5d349-1focal_amd64.deb ... 2023-06-03T19:20:43.203 INFO:teuthology.orchestra.run.smithi164.stdout:Unpacking python3-rados (16.2.13-257-gd8c5d349-1focal) ... 2023-06-03T19:20:43.495 INFO:teuthology.orchestra.run.smithi089.stdout:Selecting previously unselected package librgw2. 2023-06-03T19:20:43.507 INFO:teuthology.orchestra.run.smithi164.stdout:Selecting previously unselected package python3-ceph-argparse. 2023-06-03T19:20:43.508 INFO:teuthology.orchestra.run.smithi089.stdout:Preparing to unpack .../14-librgw2_16.2.13-257-gd8c5d349-1focal_amd64.deb ... 2023-06-03T19:20:43.520 INFO:teuthology.orchestra.run.smithi164.stdout:Preparing to unpack .../06-python3-ceph-argparse_16.2.13-257-gd8c5d349-1focal_all.deb ... 2023-06-03T19:20:43.553 INFO:teuthology.orchestra.run.smithi089.stdout:Unpacking librgw2 (16.2.13-257-gd8c5d349-1focal) ... 2023-06-03T19:20:43.565 INFO:teuthology.orchestra.run.smithi164.stdout:Unpacking python3-ceph-argparse (16.2.13-257-gd8c5d349-1focal) ... 2023-06-03T19:20:43.856 INFO:teuthology.orchestra.run.smithi164.stdout:Selecting previously unselected package python3-cephfs. 2023-06-03T19:20:43.869 INFO:teuthology.orchestra.run.smithi164.stdout:Preparing to unpack .../07-python3-cephfs_16.2.13-257-gd8c5d349-1focal_amd64.deb ... 2023-06-03T19:20:43.897 INFO:teuthology.orchestra.run.smithi164.stdout:Unpacking python3-cephfs (16.2.13-257-gd8c5d349-1focal) ... 2023-06-03T19:20:44.147 INFO:teuthology.orchestra.run.smithi089.stdout:Selecting previously unselected package python3-rgw. 2023-06-03T19:20:44.162 INFO:teuthology.orchestra.run.smithi089.stdout:Preparing to unpack .../15-python3-rgw_16.2.13-257-gd8c5d349-1focal_amd64.deb ... 2023-06-03T19:20:44.206 INFO:teuthology.orchestra.run.smithi089.stdout:Unpacking python3-rgw (16.2.13-257-gd8c5d349-1focal) ... 2023-06-03T19:20:44.217 INFO:teuthology.orchestra.run.smithi164.stdout:Selecting previously unselected package python3-ceph-common. 2023-06-03T19:20:44.231 INFO:teuthology.orchestra.run.smithi164.stdout:Preparing to unpack .../08-python3-ceph-common_16.2.13-257-gd8c5d349-1focal_all.deb ... 2023-06-03T19:20:44.276 INFO:teuthology.orchestra.run.smithi164.stdout:Unpacking python3-ceph-common (16.2.13-257-gd8c5d349-1focal) ... 2023-06-03T19:20:44.587 INFO:teuthology.orchestra.run.smithi164.stdout:Selecting previously unselected package python3-prettytable. 2023-06-03T19:20:44.603 INFO:teuthology.orchestra.run.smithi164.stdout:Preparing to unpack .../09-python3-prettytable_0.7.2-5_all.deb ... 2023-06-03T19:20:44.645 INFO:teuthology.orchestra.run.smithi164.stdout:Unpacking python3-prettytable (0.7.2-5) ... 2023-06-03T19:20:44.769 INFO:teuthology.orchestra.run.smithi089.stdout:Selecting previously unselected package libleveldb1d:amd64. 2023-06-03T19:20:44.783 INFO:teuthology.orchestra.run.smithi089.stdout:Preparing to unpack .../16-libleveldb1d_1.22-3ubuntu2_amd64.deb ... 2023-06-03T19:20:44.827 INFO:teuthology.orchestra.run.smithi089.stdout:Unpacking libleveldb1d:amd64 (1.22-3ubuntu2) ... 2023-06-03T19:20:44.898 INFO:teuthology.orchestra.run.smithi164.stdout:Selecting previously unselected package python3-rbd. 2023-06-03T19:20:44.910 INFO:teuthology.orchestra.run.smithi164.stdout:Preparing to unpack .../10-python3-rbd_16.2.13-257-gd8c5d349-1focal_amd64.deb ... 2023-06-03T19:20:44.947 INFO:teuthology.orchestra.run.smithi164.stdout:Unpacking python3-rbd (16.2.13-257-gd8c5d349-1focal) ... 2023-06-03T19:20:45.156 INFO:teuthology.orchestra.run.smithi089.stdout:Selecting previously unselected package liboath0. 2023-06-03T19:20:45.170 INFO:teuthology.orchestra.run.smithi089.stdout:Preparing to unpack .../17-liboath0_2.6.1-1.3_amd64.deb ... 2023-06-03T19:20:45.214 INFO:teuthology.orchestra.run.smithi089.stdout:Unpacking liboath0 (2.6.1-1.3) ... 2023-06-03T19:20:45.510 INFO:teuthology.orchestra.run.smithi164.stdout:Selecting previously unselected package liblua5.3-0:amd64. 2023-06-03T19:20:45.525 INFO:teuthology.orchestra.run.smithi089.stdout:Selecting previously unselected package libradosstriper1. 2023-06-03T19:20:45.526 INFO:teuthology.orchestra.run.smithi164.stdout:Preparing to unpack .../11-liblua5.3-0_5.3.3-1.1ubuntu2_amd64.deb ... 2023-06-03T19:20:45.539 INFO:teuthology.orchestra.run.smithi089.stdout:Preparing to unpack .../18-libradosstriper1_16.2.13-257-gd8c5d349-1focal_amd64.deb ... 2023-06-03T19:20:45.577 INFO:teuthology.orchestra.run.smithi164.stdout:Unpacking liblua5.3-0:amd64 (5.3.3-1.1ubuntu2) ... 2023-06-03T19:20:45.583 INFO:teuthology.orchestra.run.smithi089.stdout:Unpacking libradosstriper1 (16.2.13-257-gd8c5d349-1focal) ... 2023-06-03T19:20:45.963 INFO:teuthology.orchestra.run.smithi164.stdout:Selecting previously unselected package librabbitmq4:amd64. 2023-06-03T19:20:45.976 INFO:teuthology.orchestra.run.smithi164.stdout:Preparing to unpack .../12-librabbitmq4_0.10.0-1_amd64.deb ... 2023-06-03T19:20:46.021 INFO:teuthology.orchestra.run.smithi164.stdout:Unpacking librabbitmq4:amd64 (0.10.0-1) ... 2023-06-03T19:20:46.037 INFO:teuthology.orchestra.run.smithi089.stdout:Selecting previously unselected package ceph-common. 2023-06-03T19:20:46.050 INFO:teuthology.orchestra.run.smithi089.stdout:Preparing to unpack .../19-ceph-common_16.2.13-257-gd8c5d349-1focal_amd64.deb ... 2023-06-03T19:20:46.095 INFO:teuthology.orchestra.run.smithi089.stdout:Unpacking ceph-common (16.2.13-257-gd8c5d349-1focal) ... 2023-06-03T19:20:46.350 INFO:teuthology.orchestra.run.smithi164.stdout:Selecting previously unselected package librdkafka1:amd64. 2023-06-03T19:20:46.366 INFO:teuthology.orchestra.run.smithi164.stdout:Preparing to unpack .../13-librdkafka1_1.2.1-1ubuntu1_amd64.deb ... 2023-06-03T19:20:46.409 INFO:teuthology.orchestra.run.smithi164.stdout:Unpacking librdkafka1:amd64 (1.2.1-1ubuntu1) ... 2023-06-03T19:20:46.754 INFO:teuthology.orchestra.run.smithi164.stdout:Selecting previously unselected package librgw2. 2023-06-03T19:20:46.768 INFO:teuthology.orchestra.run.smithi164.stdout:Preparing to unpack .../14-librgw2_16.2.13-257-gd8c5d349-1focal_amd64.deb ... 2023-06-03T19:20:46.812 INFO:teuthology.orchestra.run.smithi164.stdout:Unpacking librgw2 (16.2.13-257-gd8c5d349-1focal) ... 2023-06-03T19:20:47.390 INFO:teuthology.orchestra.run.smithi164.stdout:Selecting previously unselected package python3-rgw. 2023-06-03T19:20:47.406 INFO:teuthology.orchestra.run.smithi164.stdout:Preparing to unpack .../15-python3-rgw_16.2.13-257-gd8c5d349-1focal_amd64.deb ... 2023-06-03T19:20:47.449 INFO:teuthology.orchestra.run.smithi164.stdout:Unpacking python3-rgw (16.2.13-257-gd8c5d349-1focal) ... 2023-06-03T19:20:47.760 INFO:teuthology.orchestra.run.smithi164.stdout:Selecting previously unselected package libleveldb1d:amd64. 2023-06-03T19:20:47.773 INFO:teuthology.orchestra.run.smithi164.stdout:Preparing to unpack .../16-libleveldb1d_1.22-3ubuntu2_amd64.deb ... 2023-06-03T19:20:47.818 INFO:teuthology.orchestra.run.smithi164.stdout:Unpacking libleveldb1d:amd64 (1.22-3ubuntu2) ... 2023-06-03T19:20:48.170 INFO:teuthology.orchestra.run.smithi164.stdout:Selecting previously unselected package liboath0. 2023-06-03T19:20:48.186 INFO:teuthology.orchestra.run.smithi164.stdout:Preparing to unpack .../17-liboath0_2.6.1-1.3_amd64.deb ... 2023-06-03T19:20:48.229 INFO:teuthology.orchestra.run.smithi164.stdout:Unpacking liboath0 (2.6.1-1.3) ... 2023-06-03T19:20:48.325 INFO:teuthology.orchestra.run.smithi089.stdout:Selecting previously unselected package ceph-base. 2023-06-03T19:20:48.338 INFO:teuthology.orchestra.run.smithi089.stdout:Preparing to unpack .../20-ceph-base_16.2.13-257-gd8c5d349-1focal_amd64.deb ... 2023-06-03T19:20:48.393 INFO:teuthology.orchestra.run.smithi089.stdout:Unpacking ceph-base (16.2.13-257-gd8c5d349-1focal) ... 2023-06-03T19:20:48.540 INFO:teuthology.orchestra.run.smithi164.stdout:Selecting previously unselected package libradosstriper1. 2023-06-03T19:20:48.553 INFO:teuthology.orchestra.run.smithi164.stdout:Preparing to unpack .../18-libradosstriper1_16.2.13-257-gd8c5d349-1focal_amd64.deb ... 2023-06-03T19:20:48.590 INFO:teuthology.orchestra.run.smithi164.stdout:Unpacking libradosstriper1 (16.2.13-257-gd8c5d349-1focal) ... 2023-06-03T19:20:49.085 INFO:teuthology.orchestra.run.smithi164.stdout:Selecting previously unselected package ceph-common. 2023-06-03T19:20:49.099 INFO:teuthology.orchestra.run.smithi164.stdout:Preparing to unpack .../19-ceph-common_16.2.13-257-gd8c5d349-1focal_amd64.deb ... 2023-06-03T19:20:49.143 INFO:teuthology.orchestra.run.smithi164.stdout:Unpacking ceph-common (16.2.13-257-gd8c5d349-1focal) ... 2023-06-03T19:20:49.397 INFO:teuthology.orchestra.run.smithi089.stdout:Selecting previously unselected package python3-cryptography. 2023-06-03T19:20:49.410 INFO:teuthology.orchestra.run.smithi089.stdout:Preparing to unpack .../21-python3-cryptography_2.8-3ubuntu0.1_amd64.deb ... 2023-06-03T19:20:49.457 INFO:teuthology.orchestra.run.smithi089.stdout:Unpacking python3-cryptography (2.8-3ubuntu0.1) ... 2023-06-03T19:20:49.825 INFO:teuthology.orchestra.run.smithi089.stdout:Selecting previously unselected package python3-openssl. 2023-06-03T19:20:49.839 INFO:teuthology.orchestra.run.smithi089.stdout:Preparing to unpack .../22-python3-openssl_19.0.0-1build1_all.deb ... 2023-06-03T19:20:49.883 INFO:teuthology.orchestra.run.smithi089.stdout:Unpacking python3-openssl (19.0.0-1build1) ... 2023-06-03T19:20:50.170 INFO:teuthology.orchestra.run.smithi089.stdout:Selecting previously unselected package ceph-mgr-modules-core. 2023-06-03T19:20:50.185 INFO:teuthology.orchestra.run.smithi089.stdout:Preparing to unpack .../23-ceph-mgr-modules-core_16.2.13-257-gd8c5d349-1focal_all.deb ... 2023-06-03T19:20:50.228 INFO:teuthology.orchestra.run.smithi089.stdout:Unpacking ceph-mgr-modules-core (16.2.13-257-gd8c5d349-1focal) ... 2023-06-03T19:20:50.591 INFO:teuthology.orchestra.run.smithi089.stdout:Selecting previously unselected package python3-bcrypt. 2023-06-03T19:20:50.606 INFO:teuthology.orchestra.run.smithi089.stdout:Preparing to unpack .../24-python3-bcrypt_3.1.7-2ubuntu1_amd64.deb ... 2023-06-03T19:20:50.649 INFO:teuthology.orchestra.run.smithi089.stdout:Unpacking python3-bcrypt (3.1.7-2ubuntu1) ... 2023-06-03T19:20:50.985 INFO:teuthology.orchestra.run.smithi089.stdout:Selecting previously unselected package python3-cherrypy3. 2023-06-03T19:20:50.999 INFO:teuthology.orchestra.run.smithi089.stdout:Preparing to unpack .../25-python3-cherrypy3_8.9.1-6_all.deb ... 2023-06-03T19:20:51.043 INFO:teuthology.orchestra.run.smithi089.stdout:Unpacking python3-cherrypy3 (8.9.1-6) ... 2023-06-03T19:20:51.388 INFO:teuthology.orchestra.run.smithi089.stdout:Selecting previously unselected package python3-jwt. 2023-06-03T19:20:51.403 INFO:teuthology.orchestra.run.smithi089.stdout:Preparing to unpack .../26-python3-jwt_1.7.1-2ubuntu2.1_all.deb ... 2023-06-03T19:20:51.406 INFO:teuthology.orchestra.run.smithi164.stdout:Selecting previously unselected package ceph-base. 2023-06-03T19:20:51.422 INFO:teuthology.orchestra.run.smithi164.stdout:Preparing to unpack .../20-ceph-base_16.2.13-257-gd8c5d349-1focal_amd64.deb ... 2023-06-03T19:20:51.447 INFO:teuthology.orchestra.run.smithi089.stdout:Unpacking python3-jwt (1.7.1-2ubuntu2.1) ... 2023-06-03T19:20:51.474 INFO:teuthology.orchestra.run.smithi164.stdout:Unpacking ceph-base (16.2.13-257-gd8c5d349-1focal) ... 2023-06-03T19:20:51.760 INFO:teuthology.orchestra.run.smithi089.stdout:Selecting previously unselected package python3-logutils. 2023-06-03T19:20:51.774 INFO:teuthology.orchestra.run.smithi089.stdout:Preparing to unpack .../27-python3-logutils_0.3.3-7_all.deb ... 2023-06-03T19:20:51.818 INFO:teuthology.orchestra.run.smithi089.stdout:Unpacking python3-logutils (0.3.3-7) ... 2023-06-03T19:20:52.130 INFO:teuthology.orchestra.run.smithi089.stdout:Selecting previously unselected package python3-markupsafe. 2023-06-03T19:20:52.146 INFO:teuthology.orchestra.run.smithi089.stdout:Preparing to unpack .../28-python3-markupsafe_1.1.0-1build2_amd64.deb ... 2023-06-03T19:20:52.188 INFO:teuthology.orchestra.run.smithi089.stdout:Unpacking python3-markupsafe (1.1.0-1build2) ... 2023-06-03T19:20:52.461 INFO:teuthology.orchestra.run.smithi164.stdout:Selecting previously unselected package python3-cryptography. 2023-06-03T19:20:52.469 INFO:teuthology.orchestra.run.smithi164.stdout:Preparing to unpack .../21-python3-cryptography_2.8-3ubuntu0.1_amd64.deb ... 2023-06-03T19:20:52.491 INFO:teuthology.orchestra.run.smithi089.stdout:Selecting previously unselected package python3-mako. 2023-06-03T19:20:52.506 INFO:teuthology.orchestra.run.smithi089.stdout:Preparing to unpack .../29-python3-mako_1.1.0+ds1-1ubuntu2.1_all.deb ... 2023-06-03T19:20:52.511 INFO:teuthology.orchestra.run.smithi164.stdout:Unpacking python3-cryptography (2.8-3ubuntu0.1) ... 2023-06-03T19:20:52.550 INFO:teuthology.orchestra.run.smithi089.stdout:Unpacking python3-mako (1.1.0+ds1-1ubuntu2.1) ... 2023-06-03T19:20:52.866 INFO:teuthology.orchestra.run.smithi089.stdout:Selecting previously unselected package python3-simplegeneric. 2023-06-03T19:20:52.875 INFO:teuthology.orchestra.run.smithi089.stdout:Preparing to unpack .../30-python3-simplegeneric_0.8.1-3_all.deb ... 2023-06-03T19:20:52.881 INFO:teuthology.orchestra.run.smithi164.stdout:Selecting previously unselected package python3-openssl. 2023-06-03T19:20:52.889 INFO:teuthology.orchestra.run.smithi164.stdout:Preparing to unpack .../22-python3-openssl_19.0.0-1build1_all.deb ... 2023-06-03T19:20:52.916 INFO:teuthology.orchestra.run.smithi089.stdout:Unpacking python3-simplegeneric (0.8.1-3) ... 2023-06-03T19:20:52.931 INFO:teuthology.orchestra.run.smithi164.stdout:Unpacking python3-openssl (19.0.0-1build1) ... 2023-06-03T19:20:53.208 INFO:teuthology.orchestra.run.smithi164.stdout:Selecting previously unselected package ceph-mgr-modules-core. 2023-06-03T19:20:53.217 INFO:teuthology.orchestra.run.smithi164.stdout:Preparing to unpack .../23-ceph-mgr-modules-core_16.2.13-257-gd8c5d349-1focal_all.deb ... 2023-06-03T19:20:53.228 INFO:teuthology.orchestra.run.smithi089.stdout:Selecting previously unselected package python3-singledispatch. 2023-06-03T19:20:53.236 INFO:teuthology.orchestra.run.smithi089.stdout:Preparing to unpack .../31-python3-singledispatch_3.4.0.3-2build1_all.deb ... 2023-06-03T19:20:53.258 INFO:teuthology.orchestra.run.smithi164.stdout:Unpacking ceph-mgr-modules-core (16.2.13-257-gd8c5d349-1focal) ... 2023-06-03T19:20:53.277 INFO:teuthology.orchestra.run.smithi089.stdout:Unpacking python3-singledispatch (3.4.0.3-2build1) ... 2023-06-03T19:20:53.573 INFO:teuthology.orchestra.run.smithi089.stdout:Selecting previously unselected package python3-webob. 2023-06-03T19:20:53.586 INFO:teuthology.orchestra.run.smithi089.stdout:Preparing to unpack .../32-python3-webob_1%3a1.8.5-2_all.deb ... 2023-06-03T19:20:53.630 INFO:teuthology.orchestra.run.smithi164.stdout:Selecting previously unselected package python3-bcrypt. 2023-06-03T19:20:53.630 INFO:teuthology.orchestra.run.smithi089.stdout:Unpacking python3-webob (1:1.8.5-2) ... 2023-06-03T19:20:53.643 INFO:teuthology.orchestra.run.smithi164.stdout:Preparing to unpack .../24-python3-bcrypt_3.1.7-2ubuntu1_amd64.deb ... 2023-06-03T19:20:53.688 INFO:teuthology.orchestra.run.smithi164.stdout:Unpacking python3-bcrypt (3.1.7-2ubuntu1) ... 2023-06-03T19:20:53.942 INFO:teuthology.orchestra.run.smithi089.stdout:Selecting previously unselected package python3-soupsieve. 2023-06-03T19:20:53.956 INFO:teuthology.orchestra.run.smithi089.stdout:Preparing to unpack .../33-python3-soupsieve_1.9.5+dfsg-1_all.deb ... 2023-06-03T19:20:54.000 INFO:teuthology.orchestra.run.smithi089.stdout:Unpacking python3-soupsieve (1.9.5+dfsg-1) ... 2023-06-03T19:20:54.001 INFO:teuthology.orchestra.run.smithi164.stdout:Selecting previously unselected package python3-cherrypy3. 2023-06-03T19:20:54.013 INFO:teuthology.orchestra.run.smithi164.stdout:Preparing to unpack .../25-python3-cherrypy3_8.9.1-6_all.deb ... 2023-06-03T19:20:54.050 INFO:teuthology.orchestra.run.smithi164.stdout:Unpacking python3-cherrypy3 (8.9.1-6) ... 2023-06-03T19:20:54.295 INFO:teuthology.orchestra.run.smithi089.stdout:Selecting previously unselected package python3-bs4. 2023-06-03T19:20:54.310 INFO:teuthology.orchestra.run.smithi089.stdout:Preparing to unpack .../34-python3-bs4_4.8.2-1_all.deb ... 2023-06-03T19:20:54.353 INFO:teuthology.orchestra.run.smithi089.stdout:Unpacking python3-bs4 (4.8.2-1) ... 2023-06-03T19:20:54.404 INFO:teuthology.orchestra.run.smithi164.stdout:Selecting previously unselected package python3-jwt. 2023-06-03T19:20:54.417 INFO:teuthology.orchestra.run.smithi164.stdout:Preparing to unpack .../26-python3-jwt_1.7.1-2ubuntu2.1_all.deb ... 2023-06-03T19:20:54.454 INFO:teuthology.orchestra.run.smithi164.stdout:Unpacking python3-jwt (1.7.1-2ubuntu2.1) ... 2023-06-03T19:20:54.731 INFO:teuthology.orchestra.run.smithi089.stdout:Selecting previously unselected package python3-waitress. 2023-06-03T19:20:54.737 INFO:teuthology.orchestra.run.smithi164.stdout:Selecting previously unselected package python3-logutils. 2023-06-03T19:20:54.740 INFO:teuthology.orchestra.run.smithi089.stdout:Preparing to unpack .../35-python3-waitress_1.4.1-1ubuntu0.1_all.deb ... 2023-06-03T19:20:54.750 INFO:teuthology.orchestra.run.smithi164.stdout:Preparing to unpack .../27-python3-logutils_0.3.3-7_all.deb ... 2023-06-03T19:20:54.796 INFO:teuthology.orchestra.run.smithi164.stdout:Unpacking python3-logutils (0.3.3-7) ... 2023-06-03T19:20:54.817 INFO:teuthology.orchestra.run.smithi089.stdout:Unpacking python3-waitress (1.4.1-1ubuntu0.1) ... 2023-06-03T19:20:55.107 INFO:teuthology.orchestra.run.smithi164.stdout:Selecting previously unselected package python3-markupsafe. 2023-06-03T19:20:55.122 INFO:teuthology.orchestra.run.smithi164.stdout:Preparing to unpack .../28-python3-markupsafe_1.1.0-1build2_amd64.deb ... 2023-06-03T19:20:55.143 INFO:teuthology.orchestra.run.smithi089.stdout:Selecting previously unselected package python3-tempita. 2023-06-03T19:20:55.157 INFO:teuthology.orchestra.run.smithi089.stdout:Preparing to unpack .../36-python3-tempita_0.5.2-6_all.deb ... 2023-06-03T19:20:55.165 INFO:teuthology.orchestra.run.smithi164.stdout:Unpacking python3-markupsafe (1.1.0-1build2) ... 2023-06-03T19:20:55.201 INFO:teuthology.orchestra.run.smithi089.stdout:Unpacking python3-tempita (0.5.2-6) ... 2023-06-03T19:20:55.460 INFO:teuthology.orchestra.run.smithi164.stdout:Selecting previously unselected package python3-mako. 2023-06-03T19:20:55.473 INFO:teuthology.orchestra.run.smithi164.stdout:Preparing to unpack .../29-python3-mako_1.1.0+ds1-1ubuntu2.1_all.deb ... 2023-06-03T19:20:55.488 INFO:teuthology.orchestra.run.smithi089.stdout:Selecting previously unselected package python3-paste. 2023-06-03T19:20:55.502 INFO:teuthology.orchestra.run.smithi089.stdout:Preparing to unpack .../37-python3-paste_3.4.0+dfsg1-1_all.deb ... 2023-06-03T19:20:55.518 INFO:teuthology.orchestra.run.smithi164.stdout:Unpacking python3-mako (1.1.0+ds1-1ubuntu2.1) ... 2023-06-03T19:20:55.546 INFO:teuthology.orchestra.run.smithi089.stdout:Unpacking python3-paste (3.4.0+dfsg1-1) ... 2023-06-03T19:20:55.858 INFO:teuthology.orchestra.run.smithi089.stdout:Selecting previously unselected package python-pastedeploy-tpl. 2023-06-03T19:20:55.866 INFO:teuthology.orchestra.run.smithi089.stdout:Preparing to unpack .../38-python-pastedeploy-tpl_2.1.0-1_all.deb ... 2023-06-03T19:20:55.897 INFO:teuthology.orchestra.run.smithi164.stdout:Selecting previously unselected package python3-simplegeneric. 2023-06-03T19:20:55.907 INFO:teuthology.orchestra.run.smithi089.stdout:Unpacking python-pastedeploy-tpl (2.1.0-1) ... 2023-06-03T19:20:55.910 INFO:teuthology.orchestra.run.smithi164.stdout:Preparing to unpack .../30-python3-simplegeneric_0.8.1-3_all.deb ... 2023-06-03T19:20:55.946 INFO:teuthology.orchestra.run.smithi164.stdout:Unpacking python3-simplegeneric (0.8.1-3) ... 2023-06-03T19:20:56.193 INFO:teuthology.orchestra.run.smithi089.stdout:Selecting previously unselected package python3-pastedeploy. 2023-06-03T19:20:56.210 INFO:teuthology.orchestra.run.smithi089.stdout:Preparing to unpack .../39-python3-pastedeploy_2.1.0-1_all.deb ... 2023-06-03T19:20:56.224 INFO:teuthology.orchestra.run.smithi164.stdout:Selecting previously unselected package python3-singledispatch. 2023-06-03T19:20:56.239 INFO:teuthology.orchestra.run.smithi164.stdout:Preparing to unpack .../31-python3-singledispatch_3.4.0.3-2build1_all.deb ... 2023-06-03T19:20:56.261 INFO:teuthology.orchestra.run.smithi089.stdout:Unpacking python3-pastedeploy (2.1.0-1) ... 2023-06-03T19:20:56.283 INFO:teuthology.orchestra.run.smithi164.stdout:Unpacking python3-singledispatch (3.4.0.3-2build1) ... 2023-06-03T19:20:56.547 INFO:teuthology.orchestra.run.smithi089.stdout:Selecting previously unselected package python3-webtest. 2023-06-03T19:20:56.560 INFO:teuthology.orchestra.run.smithi164.stdout:Selecting previously unselected package python3-webob. 2023-06-03T19:20:56.562 INFO:teuthology.orchestra.run.smithi089.stdout:Preparing to unpack .../40-python3-webtest_2.0.32-2_all.deb ... 2023-06-03T19:20:56.573 INFO:teuthology.orchestra.run.smithi164.stdout:Preparing to unpack .../32-python3-webob_1%3a1.8.5-2_all.deb ... 2023-06-03T19:20:56.605 INFO:teuthology.orchestra.run.smithi089.stdout:Unpacking python3-webtest (2.0.32-2) ... 2023-06-03T19:20:56.610 INFO:teuthology.orchestra.run.smithi164.stdout:Unpacking python3-webob (1:1.8.5-2) ... 2023-06-03T19:20:56.899 INFO:teuthology.orchestra.run.smithi089.stdout:Selecting previously unselected package python3-pecan. 2023-06-03T19:20:56.914 INFO:teuthology.orchestra.run.smithi089.stdout:Preparing to unpack .../41-python3-pecan_1.3.3-3ubuntu1_all.deb ... 2023-06-03T19:20:56.939 INFO:teuthology.orchestra.run.smithi164.stdout:Selecting previously unselected package python3-soupsieve. 2023-06-03T19:20:56.954 INFO:teuthology.orchestra.run.smithi164.stdout:Preparing to unpack .../33-python3-soupsieve_1.9.5+dfsg-1_all.deb ... 2023-06-03T19:20:56.957 INFO:teuthology.orchestra.run.smithi089.stdout:Unpacking python3-pecan (1.3.3-3ubuntu1) ... 2023-06-03T19:20:56.997 INFO:teuthology.orchestra.run.smithi164.stdout:Unpacking python3-soupsieve (1.9.5+dfsg-1) ... 2023-06-03T19:20:57.299 INFO:teuthology.orchestra.run.smithi164.stdout:Selecting previously unselected package python3-bs4. 2023-06-03T19:20:57.313 INFO:teuthology.orchestra.run.smithi164.stdout:Preparing to unpack .../34-python3-bs4_4.8.2-1_all.deb ... 2023-06-03T19:20:57.314 INFO:teuthology.orchestra.run.smithi089.stdout:Selecting previously unselected package python3-werkzeug. 2023-06-03T19:20:57.329 INFO:teuthology.orchestra.run.smithi089.stdout:Preparing to unpack .../42-python3-werkzeug_0.16.1+dfsg1-2ubuntu0.1_all.deb ... 2023-06-03T19:20:57.357 INFO:teuthology.orchestra.run.smithi164.stdout:Unpacking python3-bs4 (4.8.2-1) ... 2023-06-03T19:20:57.372 INFO:teuthology.orchestra.run.smithi089.stdout:Unpacking python3-werkzeug (0.16.1+dfsg1-2ubuntu0.1) ... 2023-06-03T19:20:57.686 INFO:teuthology.orchestra.run.smithi164.stdout:Selecting previously unselected package python3-waitress. 2023-06-03T19:20:57.694 INFO:teuthology.orchestra.run.smithi164.stdout:Preparing to unpack .../35-python3-waitress_1.4.1-1ubuntu0.1_all.deb ... 2023-06-03T19:20:57.725 INFO:teuthology.orchestra.run.smithi089.stdout:Selecting previously unselected package libsqlite3-mod-ceph. 2023-06-03T19:20:57.738 INFO:teuthology.orchestra.run.smithi164.stdout:Unpacking python3-waitress (1.4.1-1ubuntu0.1) ... 2023-06-03T19:20:57.739 INFO:teuthology.orchestra.run.smithi089.stdout:Preparing to unpack .../43-libsqlite3-mod-ceph_16.2.13-257-gd8c5d349-1focal_amd64.deb ... 2023-06-03T19:20:57.783 INFO:teuthology.orchestra.run.smithi089.stdout:Unpacking libsqlite3-mod-ceph (16.2.13-257-gd8c5d349-1focal) ... 2023-06-03T19:20:58.055 INFO:teuthology.orchestra.run.smithi164.stdout:Selecting previously unselected package python3-tempita. 2023-06-03T19:20:58.064 INFO:teuthology.orchestra.run.smithi164.stdout:Preparing to unpack .../36-python3-tempita_0.5.2-6_all.deb ... 2023-06-03T19:20:58.105 INFO:teuthology.orchestra.run.smithi164.stdout:Unpacking python3-tempita (0.5.2-6) ... 2023-06-03T19:20:58.120 INFO:teuthology.orchestra.run.smithi089.stdout:Selecting previously unselected package ceph-mgr. 2023-06-03T19:20:58.134 INFO:teuthology.orchestra.run.smithi089.stdout:Preparing to unpack .../44-ceph-mgr_16.2.13-257-gd8c5d349-1focal_amd64.deb ... 2023-06-03T19:20:58.170 INFO:teuthology.orchestra.run.smithi089.stdout:Unpacking ceph-mgr (16.2.13-257-gd8c5d349-1focal) ... 2023-06-03T19:20:58.408 INFO:teuthology.orchestra.run.smithi164.stdout:Selecting previously unselected package python3-paste. 2023-06-03T19:20:58.417 INFO:teuthology.orchestra.run.smithi164.stdout:Preparing to unpack .../37-python3-paste_3.4.0+dfsg1-1_all.deb ... 2023-06-03T19:20:58.458 INFO:teuthology.orchestra.run.smithi164.stdout:Unpacking python3-paste (3.4.0+dfsg1-1) ... 2023-06-03T19:20:58.656 INFO:teuthology.orchestra.run.smithi089.stdout:Selecting previously unselected package ceph-mon. 2023-06-03T19:20:58.670 INFO:teuthology.orchestra.run.smithi089.stdout:Preparing to unpack .../45-ceph-mon_16.2.13-257-gd8c5d349-1focal_amd64.deb ... 2023-06-03T19:20:58.714 INFO:teuthology.orchestra.run.smithi089.stdout:Unpacking ceph-mon (16.2.13-257-gd8c5d349-1focal) ... 2023-06-03T19:20:58.786 INFO:teuthology.orchestra.run.smithi164.stdout:Selecting previously unselected package python-pastedeploy-tpl. 2023-06-03T19:20:58.799 INFO:teuthology.orchestra.run.smithi164.stdout:Preparing to unpack .../38-python-pastedeploy-tpl_2.1.0-1_all.deb ... 2023-06-03T19:20:58.836 INFO:teuthology.orchestra.run.smithi164.stdout:Unpacking python-pastedeploy-tpl (2.1.0-1) ... 2023-06-03T19:20:59.164 INFO:teuthology.orchestra.run.smithi164.stdout:Selecting previously unselected package python3-pastedeploy. 2023-06-03T19:20:59.177 INFO:teuthology.orchestra.run.smithi164.stdout:Preparing to unpack .../39-python3-pastedeploy_2.1.0-1_all.deb ... 2023-06-03T19:20:59.222 INFO:teuthology.orchestra.run.smithi164.stdout:Unpacking python3-pastedeploy (2.1.0-1) ... 2023-06-03T19:20:59.526 INFO:teuthology.orchestra.run.smithi164.stdout:Selecting previously unselected package python3-webtest. 2023-06-03T19:20:59.534 INFO:teuthology.orchestra.run.smithi164.stdout:Preparing to unpack .../40-python3-webtest_2.0.32-2_all.deb ... 2023-06-03T19:20:59.593 INFO:teuthology.orchestra.run.smithi164.stdout:Unpacking python3-webtest (2.0.32-2) ... 2023-06-03T19:20:59.776 INFO:teuthology.orchestra.run.smithi089.stdout:Selecting previously unselected package ceph-osd. 2023-06-03T19:20:59.790 INFO:teuthology.orchestra.run.smithi089.stdout:Preparing to unpack .../46-ceph-osd_16.2.13-257-gd8c5d349-1focal_amd64.deb ... 2023-06-03T19:20:59.834 INFO:teuthology.orchestra.run.smithi089.stdout:Unpacking ceph-osd (16.2.13-257-gd8c5d349-1focal) ... 2023-06-03T19:20:59.904 INFO:teuthology.orchestra.run.smithi164.stdout:Selecting previously unselected package python3-pecan. 2023-06-03T19:20:59.912 INFO:teuthology.orchestra.run.smithi164.stdout:Preparing to unpack .../41-python3-pecan_1.3.3-3ubuntu1_all.deb ... 2023-06-03T19:20:59.954 INFO:teuthology.orchestra.run.smithi164.stdout:Unpacking python3-pecan (1.3.3-3ubuntu1) ... 2023-06-03T19:21:00.308 INFO:teuthology.orchestra.run.smithi164.stdout:Selecting previously unselected package python3-werkzeug. 2023-06-03T19:21:00.323 INFO:teuthology.orchestra.run.smithi164.stdout:Preparing to unpack .../42-python3-werkzeug_0.16.1+dfsg1-2ubuntu0.1_all.deb ... 2023-06-03T19:21:00.366 INFO:teuthology.orchestra.run.smithi164.stdout:Unpacking python3-werkzeug (0.16.1+dfsg1-2ubuntu0.1) ... 2023-06-03T19:21:00.753 INFO:teuthology.orchestra.run.smithi164.stdout:Selecting previously unselected package libsqlite3-mod-ceph. 2023-06-03T19:21:00.768 INFO:teuthology.orchestra.run.smithi164.stdout:Preparing to unpack .../43-libsqlite3-mod-ceph_16.2.13-257-gd8c5d349-1focal_amd64.deb ... 2023-06-03T19:21:01.203 INFO:teuthology.orchestra.run.smithi164.stdout:Unpacking libsqlite3-mod-ceph (16.2.13-257-gd8c5d349-1focal) ... 2023-06-03T19:21:01.848 INFO:teuthology.orchestra.run.smithi164.stdout:Selecting previously unselected package ceph-mgr. 2023-06-03T19:21:01.863 INFO:teuthology.orchestra.run.smithi164.stdout:Preparing to unpack .../44-ceph-mgr_16.2.13-257-gd8c5d349-1focal_amd64.deb ... 2023-06-03T19:21:01.906 INFO:teuthology.orchestra.run.smithi164.stdout:Unpacking ceph-mgr (16.2.13-257-gd8c5d349-1focal) ... 2023-06-03T19:21:02.418 INFO:teuthology.orchestra.run.smithi164.stdout:Selecting previously unselected package ceph-mon. 2023-06-03T19:21:02.431 INFO:teuthology.orchestra.run.smithi164.stdout:Preparing to unpack .../45-ceph-mon_16.2.13-257-gd8c5d349-1focal_amd64.deb ... 2023-06-03T19:21:02.477 INFO:teuthology.orchestra.run.smithi164.stdout:Unpacking ceph-mon (16.2.13-257-gd8c5d349-1focal) ... 2023-06-03T19:21:02.832 INFO:teuthology.orchestra.run.smithi089.stdout:Selecting previously unselected package ceph. 2023-06-03T19:21:02.840 INFO:teuthology.orchestra.run.smithi089.stdout:Preparing to unpack .../47-ceph_16.2.13-257-gd8c5d349-1focal_amd64.deb ... 2023-06-03T19:21:02.882 INFO:teuthology.orchestra.run.smithi089.stdout:Unpacking ceph (16.2.13-257-gd8c5d349-1focal) ... 2023-06-03T19:21:03.201 INFO:teuthology.orchestra.run.smithi089.stdout:Selecting previously unselected package ceph-fuse. 2023-06-03T19:21:03.210 INFO:teuthology.orchestra.run.smithi089.stdout:Preparing to unpack .../48-ceph-fuse_16.2.13-257-gd8c5d349-1focal_amd64.deb ... 2023-06-03T19:21:03.251 INFO:teuthology.orchestra.run.smithi089.stdout:Unpacking ceph-fuse (16.2.13-257-gd8c5d349-1focal) ... 2023-06-03T19:21:03.513 INFO:teuthology.orchestra.run.smithi164.stdout:Selecting previously unselected package ceph-osd. 2023-06-03T19:21:03.528 INFO:teuthology.orchestra.run.smithi164.stdout:Preparing to unpack .../46-ceph-osd_16.2.13-257-gd8c5d349-1focal_amd64.deb ... 2023-06-03T19:21:03.571 INFO:teuthology.orchestra.run.smithi164.stdout:Unpacking ceph-osd (16.2.13-257-gd8c5d349-1focal) ... 2023-06-03T19:21:03.672 INFO:teuthology.orchestra.run.smithi089.stdout:Selecting previously unselected package ceph-mds. 2023-06-03T19:21:03.685 INFO:teuthology.orchestra.run.smithi089.stdout:Preparing to unpack .../49-ceph-mds_16.2.13-257-gd8c5d349-1focal_amd64.deb ... 2023-06-03T19:21:03.730 INFO:teuthology.orchestra.run.smithi089.stdout:Unpacking ceph-mds (16.2.13-257-gd8c5d349-1focal) ... 2023-06-03T19:21:04.258 INFO:teuthology.orchestra.run.smithi089.stdout:Selecting previously unselected package cephadm. 2023-06-03T19:21:04.266 INFO:teuthology.orchestra.run.smithi089.stdout:Preparing to unpack .../50-cephadm_16.2.13-257-gd8c5d349-1focal_amd64.deb ... 2023-06-03T19:21:04.308 INFO:teuthology.orchestra.run.smithi089.stdout:Unpacking cephadm (16.2.13-257-gd8c5d349-1focal) ... 2023-06-03T19:21:04.619 INFO:teuthology.orchestra.run.smithi089.stdout:Selecting previously unselected package python3-jinja2. 2023-06-03T19:21:04.627 INFO:teuthology.orchestra.run.smithi089.stdout:Preparing to unpack .../51-python3-jinja2_2.10.1-2_all.deb ... 2023-06-03T19:21:04.660 INFO:teuthology.orchestra.run.smithi089.stdout:Unpacking python3-jinja2 (2.10.1-2) ... 2023-06-03T19:21:04.938 INFO:teuthology.orchestra.run.smithi089.stdout:Selecting previously unselected package ceph-mgr-cephadm. 2023-06-03T19:21:04.947 INFO:teuthology.orchestra.run.smithi089.stdout:Preparing to unpack .../52-ceph-mgr-cephadm_16.2.13-257-gd8c5d349-1focal_all.deb ... 2023-06-03T19:21:04.980 INFO:teuthology.orchestra.run.smithi089.stdout:Unpacking ceph-mgr-cephadm (16.2.13-257-gd8c5d349-1focal) ... 2023-06-03T19:21:05.308 INFO:teuthology.orchestra.run.smithi089.stdout:Selecting previously unselected package python3-repoze.lru. 2023-06-03T19:21:05.322 INFO:teuthology.orchestra.run.smithi089.stdout:Preparing to unpack .../53-python3-repoze.lru_0.7-2_all.deb ... 2023-06-03T19:21:05.367 INFO:teuthology.orchestra.run.smithi089.stdout:Unpacking python3-repoze.lru (0.7-2) ... 2023-06-03T19:21:05.636 INFO:teuthology.orchestra.run.smithi089.stdout:Selecting previously unselected package libjs-underscore. 2023-06-03T19:21:05.650 INFO:teuthology.orchestra.run.smithi089.stdout:Preparing to unpack .../54-libjs-underscore_1.9.1~dfsg-1ubuntu0.20.04.1_all.deb ... 2023-06-03T19:21:05.694 INFO:teuthology.orchestra.run.smithi089.stdout:Unpacking libjs-underscore (1.9.1~dfsg-1ubuntu0.20.04.1) ... 2023-06-03T19:21:05.972 INFO:teuthology.orchestra.run.smithi089.stdout:Selecting previously unselected package libjs-sphinxdoc. 2023-06-03T19:21:05.985 INFO:teuthology.orchestra.run.smithi089.stdout:Preparing to unpack .../55-libjs-sphinxdoc_1.8.5-7ubuntu3_all.deb ... 2023-06-03T19:21:06.013 INFO:teuthology.orchestra.run.smithi089.stdout:Unpacking libjs-sphinxdoc (1.8.5-7ubuntu3) ... 2023-06-03T19:21:06.235 INFO:teuthology.orchestra.run.smithi164.stdout:Selecting previously unselected package ceph. 2023-06-03T19:21:06.248 INFO:teuthology.orchestra.run.smithi164.stdout:Preparing to unpack .../47-ceph_16.2.13-257-gd8c5d349-1focal_amd64.deb ... 2023-06-03T19:21:06.291 INFO:teuthology.orchestra.run.smithi089.stdout:Selecting previously unselected package python3-routes. 2023-06-03T19:21:06.293 INFO:teuthology.orchestra.run.smithi164.stdout:Unpacking ceph (16.2.13-257-gd8c5d349-1focal) ... 2023-06-03T19:21:06.304 INFO:teuthology.orchestra.run.smithi089.stdout:Preparing to unpack .../56-python3-routes_2.4.1-2_all.deb ... 2023-06-03T19:21:06.349 INFO:teuthology.orchestra.run.smithi089.stdout:Unpacking python3-routes (2.4.1-2) ... 2023-06-03T19:21:06.612 INFO:teuthology.orchestra.run.smithi164.stdout:Selecting previously unselected package ceph-fuse. 2023-06-03T19:21:06.627 INFO:teuthology.orchestra.run.smithi164.stdout:Preparing to unpack .../48-ceph-fuse_16.2.13-257-gd8c5d349-1focal_amd64.deb ... 2023-06-03T19:21:06.671 INFO:teuthology.orchestra.run.smithi164.stdout:Unpacking ceph-fuse (16.2.13-257-gd8c5d349-1focal) ... 2023-06-03T19:21:06.713 INFO:teuthology.orchestra.run.smithi089.stdout:Selecting previously unselected package ceph-mgr-dashboard. 2023-06-03T19:21:06.727 INFO:teuthology.orchestra.run.smithi089.stdout:Preparing to unpack .../57-ceph-mgr-dashboard_16.2.13-257-gd8c5d349-1focal_all.deb ... 2023-06-03T19:21:06.771 INFO:teuthology.orchestra.run.smithi089.stdout:Unpacking ceph-mgr-dashboard (16.2.13-257-gd8c5d349-1focal) ... 2023-06-03T19:21:07.090 INFO:teuthology.orchestra.run.smithi164.stdout:Selecting previously unselected package ceph-mds. 2023-06-03T19:21:07.106 INFO:teuthology.orchestra.run.smithi164.stdout:Preparing to unpack .../49-ceph-mds_16.2.13-257-gd8c5d349-1focal_amd64.deb ... 2023-06-03T19:21:07.148 INFO:teuthology.orchestra.run.smithi164.stdout:Unpacking ceph-mds (16.2.13-257-gd8c5d349-1focal) ... 2023-06-03T19:21:07.668 INFO:teuthology.orchestra.run.smithi164.stdout:Selecting previously unselected package cephadm. 2023-06-03T19:21:07.682 INFO:teuthology.orchestra.run.smithi164.stdout:Preparing to unpack .../50-cephadm_16.2.13-257-gd8c5d349-1focal_amd64.deb ... 2023-06-03T19:21:07.718 INFO:teuthology.orchestra.run.smithi164.stdout:Unpacking cephadm (16.2.13-257-gd8c5d349-1focal) ... 2023-06-03T19:21:07.764 INFO:teuthology.orchestra.run.smithi089.stdout:Selecting previously unselected package python3-decorator. 2023-06-03T19:21:07.779 INFO:teuthology.orchestra.run.smithi089.stdout:Preparing to unpack .../58-python3-decorator_4.4.2-0ubuntu1_all.deb ... 2023-06-03T19:21:07.821 INFO:teuthology.orchestra.run.smithi089.stdout:Unpacking python3-decorator (4.4.2-0ubuntu1) ... 2023-06-03T19:21:08.038 INFO:teuthology.orchestra.run.smithi164.stdout:Selecting previously unselected package python3-jinja2. 2023-06-03T19:21:08.052 INFO:teuthology.orchestra.run.smithi164.stdout:Preparing to unpack .../51-python3-jinja2_2.10.1-2_all.deb ... 2023-06-03T19:21:08.096 INFO:teuthology.orchestra.run.smithi164.stdout:Unpacking python3-jinja2 (2.10.1-2) ... 2023-06-03T19:21:08.124 INFO:teuthology.orchestra.run.smithi089.stdout:Selecting previously unselected package liblbfgsb0:amd64. 2023-06-03T19:21:08.139 INFO:teuthology.orchestra.run.smithi089.stdout:Preparing to unpack .../59-liblbfgsb0_3.0+dfsg.3-7build1_amd64.deb ... 2023-06-03T19:21:08.182 INFO:teuthology.orchestra.run.smithi089.stdout:Unpacking liblbfgsb0:amd64 (3.0+dfsg.3-7build1) ... 2023-06-03T19:21:08.407 INFO:teuthology.orchestra.run.smithi164.stdout:Selecting previously unselected package ceph-mgr-cephadm. 2023-06-03T19:21:08.422 INFO:teuthology.orchestra.run.smithi164.stdout:Preparing to unpack .../52-ceph-mgr-cephadm_16.2.13-257-gd8c5d349-1focal_all.deb ... 2023-06-03T19:21:08.466 INFO:teuthology.orchestra.run.smithi164.stdout:Unpacking ceph-mgr-cephadm (16.2.13-257-gd8c5d349-1focal) ... 2023-06-03T19:21:08.485 INFO:teuthology.orchestra.run.smithi089.stdout:Selecting previously unselected package python3-scipy. 2023-06-03T19:21:08.499 INFO:teuthology.orchestra.run.smithi089.stdout:Preparing to unpack .../60-python3-scipy_1.3.3-3build1_amd64.deb ... 2023-06-03T19:21:08.543 INFO:teuthology.orchestra.run.smithi089.stdout:Unpacking python3-scipy (1.3.3-3build1) ... 2023-06-03T19:21:08.794 INFO:teuthology.orchestra.run.smithi164.stdout:Selecting previously unselected package python3-repoze.lru. 2023-06-03T19:21:08.807 INFO:teuthology.orchestra.run.smithi164.stdout:Preparing to unpack .../53-python3-repoze.lru_0.7-2_all.deb ... 2023-06-03T19:21:08.843 INFO:teuthology.orchestra.run.smithi164.stdout:Unpacking python3-repoze.lru (0.7-2) ... 2023-06-03T19:21:09.121 INFO:teuthology.orchestra.run.smithi164.stdout:Selecting previously unselected package libjs-underscore. 2023-06-03T19:21:09.135 INFO:teuthology.orchestra.run.smithi164.stdout:Preparing to unpack .../54-libjs-underscore_1.9.1~dfsg-1ubuntu0.20.04.1_all.deb ... 2023-06-03T19:21:09.179 INFO:teuthology.orchestra.run.smithi164.stdout:Unpacking libjs-underscore (1.9.1~dfsg-1ubuntu0.20.04.1) ... 2023-06-03T19:21:09.465 INFO:teuthology.orchestra.run.smithi164.stdout:Selecting previously unselected package libjs-sphinxdoc. 2023-06-03T19:21:09.479 INFO:teuthology.orchestra.run.smithi164.stdout:Preparing to unpack .../55-libjs-sphinxdoc_1.8.5-7ubuntu3_all.deb ... 2023-06-03T19:21:09.523 INFO:teuthology.orchestra.run.smithi164.stdout:Unpacking libjs-sphinxdoc (1.8.5-7ubuntu3) ... 2023-06-03T19:21:09.817 INFO:teuthology.orchestra.run.smithi164.stdout:Selecting previously unselected package python3-routes. 2023-06-03T19:21:09.832 INFO:teuthology.orchestra.run.smithi164.stdout:Preparing to unpack .../56-python3-routes_2.4.1-2_all.deb ... 2023-06-03T19:21:09.870 INFO:teuthology.orchestra.run.smithi089.stdout:Selecting previously unselected package python3-sklearn-lib. 2023-06-03T19:21:09.875 INFO:teuthology.orchestra.run.smithi164.stdout:Unpacking python3-routes (2.4.1-2) ... 2023-06-03T19:21:09.884 INFO:teuthology.orchestra.run.smithi089.stdout:Preparing to unpack .../61-python3-sklearn-lib_0.22.2.post1+dfsg-5_amd64.deb ... 2023-06-03T19:21:09.928 INFO:teuthology.orchestra.run.smithi089.stdout:Unpacking python3-sklearn-lib (0.22.2.post1+dfsg-5) ... 2023-06-03T19:21:10.187 INFO:teuthology.orchestra.run.smithi164.stdout:Selecting previously unselected package ceph-mgr-dashboard. 2023-06-03T19:21:10.201 INFO:teuthology.orchestra.run.smithi164.stdout:Preparing to unpack .../57-ceph-mgr-dashboard_16.2.13-257-gd8c5d349-1focal_all.deb ... 2023-06-03T19:21:10.237 INFO:teuthology.orchestra.run.smithi164.stdout:Unpacking ceph-mgr-dashboard (16.2.13-257-gd8c5d349-1focal) ... 2023-06-03T19:21:10.473 INFO:teuthology.orchestra.run.smithi089.stdout:Selecting previously unselected package python3-joblib. 2023-06-03T19:21:10.488 INFO:teuthology.orchestra.run.smithi089.stdout:Preparing to unpack .../62-python3-joblib_0.14.0-3_all.deb ... 2023-06-03T19:21:10.531 INFO:teuthology.orchestra.run.smithi089.stdout:Unpacking python3-joblib (0.14.0-3) ... 2023-06-03T19:21:10.952 INFO:teuthology.orchestra.run.smithi089.stdout:Selecting previously unselected package python3-sklearn. 2023-06-03T19:21:10.967 INFO:teuthology.orchestra.run.smithi089.stdout:Preparing to unpack .../63-python3-sklearn_0.22.2.post1+dfsg-5_all.deb ... 2023-06-03T19:21:11.010 INFO:teuthology.orchestra.run.smithi089.stdout:Unpacking python3-sklearn (0.22.2.post1+dfsg-5) ... 2023-06-03T19:21:11.227 INFO:teuthology.orchestra.run.smithi164.stdout:Selecting previously unselected package python3-decorator. 2023-06-03T19:21:11.237 INFO:teuthology.orchestra.run.smithi164.stdout:Preparing to unpack .../58-python3-decorator_4.4.2-0ubuntu1_all.deb ... 2023-06-03T19:21:11.294 INFO:teuthology.orchestra.run.smithi164.stdout:Unpacking python3-decorator (4.4.2-0ubuntu1) ... 2023-06-03T19:21:11.596 INFO:teuthology.orchestra.run.smithi164.stdout:Selecting previously unselected package liblbfgsb0:amd64. 2023-06-03T19:21:11.605 INFO:teuthology.orchestra.run.smithi164.stdout:Preparing to unpack .../59-liblbfgsb0_3.0+dfsg.3-7build1_amd64.deb ... 2023-06-03T19:21:11.646 INFO:teuthology.orchestra.run.smithi164.stdout:Unpacking liblbfgsb0:amd64 (3.0+dfsg.3-7build1) ... 2023-06-03T19:21:11.686 INFO:teuthology.orchestra.run.smithi089.stdout:Selecting previously unselected package ceph-mgr-diskprediction-local. 2023-06-03T19:21:11.702 INFO:teuthology.orchestra.run.smithi089.stdout:Preparing to unpack .../64-ceph-mgr-diskprediction-local_16.2.13-257-gd8c5d349-1focal_all.deb ... 2023-06-03T19:21:11.744 INFO:teuthology.orchestra.run.smithi089.stdout:Unpacking ceph-mgr-diskprediction-local (16.2.13-257-gd8c5d349-1focal) ... 2023-06-03T19:21:12.027 INFO:teuthology.orchestra.run.smithi164.stdout:Selecting previously unselected package python3-scipy. 2023-06-03T19:21:12.038 INFO:teuthology.orchestra.run.smithi164.stdout:Preparing to unpack .../60-python3-scipy_1.3.3-3build1_amd64.deb ... 2023-06-03T19:21:12.082 INFO:teuthology.orchestra.run.smithi164.stdout:Unpacking python3-scipy (1.3.3-3build1) ... 2023-06-03T19:21:12.872 INFO:teuthology.orchestra.run.smithi089.stdout:Selecting previously unselected package python3-cachetools. 2023-06-03T19:21:12.887 INFO:teuthology.orchestra.run.smithi089.stdout:Preparing to unpack .../65-python3-cachetools_4.0.0-1_all.deb ... 2023-06-03T19:21:12.930 INFO:teuthology.orchestra.run.smithi089.stdout:Unpacking python3-cachetools (4.0.0-1) ... 2023-06-03T19:21:13.225 INFO:teuthology.orchestra.run.smithi089.stdout:Selecting previously unselected package python3-pyasn1. 2023-06-03T19:21:13.238 INFO:teuthology.orchestra.run.smithi089.stdout:Preparing to unpack .../66-python3-pyasn1_0.4.2-3build1_all.deb ... 2023-06-03T19:21:13.283 INFO:teuthology.orchestra.run.smithi089.stdout:Unpacking python3-pyasn1 (0.4.2-3build1) ... 2023-06-03T19:21:13.450 INFO:teuthology.orchestra.run.smithi164.stdout:Selecting previously unselected package python3-sklearn-lib. 2023-06-03T19:21:13.464 INFO:teuthology.orchestra.run.smithi164.stdout:Preparing to unpack .../61-python3-sklearn-lib_0.22.2.post1+dfsg-5_amd64.deb ... 2023-06-03T19:21:13.508 INFO:teuthology.orchestra.run.smithi164.stdout:Unpacking python3-sklearn-lib (0.22.2.post1+dfsg-5) ... 2023-06-03T19:21:13.603 INFO:teuthology.orchestra.run.smithi089.stdout:Selecting previously unselected package python3-pyasn1-modules. 2023-06-03T19:21:13.617 INFO:teuthology.orchestra.run.smithi089.stdout:Preparing to unpack .../67-python3-pyasn1-modules_0.2.1-0.2build1_all.deb ... 2023-06-03T19:21:13.661 INFO:teuthology.orchestra.run.smithi089.stdout:Unpacking python3-pyasn1-modules (0.2.1-0.2build1) ... 2023-06-03T19:21:13.981 INFO:teuthology.orchestra.run.smithi089.stdout:Selecting previously unselected package python3-rsa. 2023-06-03T19:21:13.995 INFO:teuthology.orchestra.run.smithi089.stdout:Preparing to unpack .../68-python3-rsa_4.0-3_all.deb ... 2023-06-03T19:21:14.028 INFO:teuthology.orchestra.run.smithi164.stdout:Selecting previously unselected package python3-joblib. 2023-06-03T19:21:14.039 INFO:teuthology.orchestra.run.smithi089.stdout:Unpacking python3-rsa (4.0-3) ... 2023-06-03T19:21:14.043 INFO:teuthology.orchestra.run.smithi164.stdout:Preparing to unpack .../62-python3-joblib_0.14.0-3_all.deb ... 2023-06-03T19:21:14.086 INFO:teuthology.orchestra.run.smithi164.stdout:Unpacking python3-joblib (0.14.0-3) ... 2023-06-03T19:21:14.342 INFO:teuthology.orchestra.run.smithi089.stdout:Selecting previously unselected package python3-google-auth. 2023-06-03T19:21:14.358 INFO:teuthology.orchestra.run.smithi089.stdout:Preparing to unpack .../69-python3-google-auth_1.5.1-2_all.deb ... 2023-06-03T19:21:14.400 INFO:teuthology.orchestra.run.smithi089.stdout:Unpacking python3-google-auth (1.5.1-2) ... 2023-06-03T19:21:14.464 INFO:teuthology.orchestra.run.smithi164.stdout:Selecting previously unselected package python3-sklearn. 2023-06-03T19:21:14.478 INFO:teuthology.orchestra.run.smithi164.stdout:Preparing to unpack .../63-python3-sklearn_0.22.2.post1+dfsg-5_all.deb ... 2023-06-03T19:21:14.522 INFO:teuthology.orchestra.run.smithi164.stdout:Unpacking python3-sklearn (0.22.2.post1+dfsg-5) ... 2023-06-03T19:21:14.720 INFO:teuthology.orchestra.run.smithi089.stdout:Selecting previously unselected package python3-blinker. 2023-06-03T19:21:14.735 INFO:teuthology.orchestra.run.smithi089.stdout:Preparing to unpack .../70-python3-blinker_1.4+dfsg1-0.3ubuntu1_all.deb ... 2023-06-03T19:21:14.779 INFO:teuthology.orchestra.run.smithi089.stdout:Unpacking python3-blinker (1.4+dfsg1-0.3ubuntu1) ... 2023-06-03T19:21:15.098 INFO:teuthology.orchestra.run.smithi089.stdout:Selecting previously unselected package python3-oauthlib. 2023-06-03T19:21:15.113 INFO:teuthology.orchestra.run.smithi089.stdout:Preparing to unpack .../71-python3-oauthlib_3.1.0-1ubuntu2_all.deb ... 2023-06-03T19:21:15.157 INFO:teuthology.orchestra.run.smithi089.stdout:Unpacking python3-oauthlib (3.1.0-1ubuntu2) ... 2023-06-03T19:21:15.247 INFO:teuthology.orchestra.run.smithi164.stdout:Selecting previously unselected package ceph-mgr-diskprediction-local. 2023-06-03T19:21:15.261 INFO:teuthology.orchestra.run.smithi164.stdout:Preparing to unpack .../64-ceph-mgr-diskprediction-local_16.2.13-257-gd8c5d349-1focal_all.deb ... 2023-06-03T19:21:15.305 INFO:teuthology.orchestra.run.smithi164.stdout:Unpacking ceph-mgr-diskprediction-local (16.2.13-257-gd8c5d349-1focal) ... 2023-06-03T19:21:15.485 INFO:teuthology.orchestra.run.smithi089.stdout:Selecting previously unselected package python3-requests-oauthlib. 2023-06-03T19:21:15.500 INFO:teuthology.orchestra.run.smithi089.stdout:Preparing to unpack .../72-python3-requests-oauthlib_1.0.0-1.1build1_all.deb ... 2023-06-03T19:21:15.543 INFO:teuthology.orchestra.run.smithi089.stdout:Unpacking python3-requests-oauthlib (1.0.0-1.1build1) ... 2023-06-03T19:21:15.855 INFO:teuthology.orchestra.run.smithi089.stdout:Selecting previously unselected package python3-websocket. 2023-06-03T19:21:15.870 INFO:teuthology.orchestra.run.smithi089.stdout:Preparing to unpack .../73-python3-websocket_0.53.0-2ubuntu1_all.deb ... 2023-06-03T19:21:15.913 INFO:teuthology.orchestra.run.smithi089.stdout:Unpacking python3-websocket (0.53.0-2ubuntu1) ... 2023-06-03T19:21:16.417 INFO:teuthology.orchestra.run.smithi164.stdout:Selecting previously unselected package python3-cachetools. 2023-06-03T19:21:16.432 INFO:teuthology.orchestra.run.smithi164.stdout:Preparing to unpack .../65-python3-cachetools_4.0.0-1_all.deb ... 2023-06-03T19:21:16.475 INFO:teuthology.orchestra.run.smithi164.stdout:Unpacking python3-cachetools (4.0.0-1) ... 2023-06-03T19:21:16.667 INFO:teuthology.orchestra.run.smithi089.stdout:Selecting previously unselected package python3-kubernetes. 2023-06-03T19:21:16.681 INFO:teuthology.orchestra.run.smithi089.stdout:Preparing to unpack .../74-python3-kubernetes_7.0.0~a1-2_all.deb ... 2023-06-03T19:21:16.725 INFO:teuthology.orchestra.run.smithi089.stdout:Unpacking python3-kubernetes (7.0.0~a1-2) ... 2023-06-03T19:21:16.786 INFO:teuthology.orchestra.run.smithi164.stdout:Selecting previously unselected package python3-pyasn1. 2023-06-03T19:21:16.800 INFO:teuthology.orchestra.run.smithi164.stdout:Preparing to unpack .../66-python3-pyasn1_0.4.2-3build1_all.deb ... 2023-06-03T19:21:16.844 INFO:teuthology.orchestra.run.smithi164.stdout:Unpacking python3-pyasn1 (0.4.2-3build1) ... 2023-06-03T19:21:17.272 INFO:teuthology.orchestra.run.smithi089.stdout:Selecting previously unselected package ceph-mgr-k8sevents. 2023-06-03T19:21:17.287 INFO:teuthology.orchestra.run.smithi089.stdout:Preparing to unpack .../75-ceph-mgr-k8sevents_16.2.13-257-gd8c5d349-1focal_all.deb ... 2023-06-03T19:21:17.329 INFO:teuthology.orchestra.run.smithi089.stdout:Unpacking ceph-mgr-k8sevents (16.2.13-257-gd8c5d349-1focal) ... 2023-06-03T19:21:17.497 INFO:teuthology.orchestra.run.smithi164.stdout:Selecting previously unselected package python3-pyasn1-modules. 2023-06-03T19:21:17.511 INFO:teuthology.orchestra.run.smithi164.stdout:Preparing to unpack .../67-python3-pyasn1-modules_0.2.1-0.2build1_all.deb ... 2023-06-03T19:21:17.555 INFO:teuthology.orchestra.run.smithi164.stdout:Unpacking python3-pyasn1-modules (0.2.1-0.2build1) ... 2023-06-03T19:21:17.641 INFO:teuthology.orchestra.run.smithi089.stdout:Selecting previously unselected package libonig5:amd64. 2023-06-03T19:21:17.655 INFO:teuthology.orchestra.run.smithi089.stdout:Preparing to unpack .../76-libonig5_6.9.4-1_amd64.deb ... 2023-06-03T19:21:17.699 INFO:teuthology.orchestra.run.smithi089.stdout:Unpacking libonig5:amd64 (6.9.4-1) ... 2023-06-03T19:21:17.866 INFO:teuthology.orchestra.run.smithi164.stdout:Selecting previously unselected package python3-rsa. 2023-06-03T19:21:17.880 INFO:teuthology.orchestra.run.smithi164.stdout:Preparing to unpack .../68-python3-rsa_4.0-3_all.deb ... 2023-06-03T19:21:17.916 INFO:teuthology.orchestra.run.smithi164.stdout:Unpacking python3-rsa (4.0-3) ... 2023-06-03T19:21:18.052 INFO:teuthology.orchestra.run.smithi089.stdout:Selecting previously unselected package libjq1:amd64. 2023-06-03T19:21:18.067 INFO:teuthology.orchestra.run.smithi089.stdout:Preparing to unpack .../77-libjq1_1.6-1ubuntu0.20.04.1_amd64.deb ... 2023-06-03T19:21:18.110 INFO:teuthology.orchestra.run.smithi089.stdout:Unpacking libjq1:amd64 (1.6-1ubuntu0.20.04.1) ... 2023-06-03T19:21:18.213 INFO:teuthology.orchestra.run.smithi164.stdout:Selecting previously unselected package python3-google-auth. 2023-06-03T19:21:18.227 INFO:teuthology.orchestra.run.smithi164.stdout:Preparing to unpack .../69-python3-google-auth_1.5.1-2_all.deb ... 2023-06-03T19:21:18.272 INFO:teuthology.orchestra.run.smithi164.stdout:Unpacking python3-google-auth (1.5.1-2) ... 2023-06-03T19:21:18.388 INFO:teuthology.orchestra.run.smithi089.stdout:Selecting previously unselected package jq. 2023-06-03T19:21:18.403 INFO:teuthology.orchestra.run.smithi089.stdout:Preparing to unpack .../78-jq_1.6-1ubuntu0.20.04.1_amd64.deb ... 2023-06-03T19:21:18.446 INFO:teuthology.orchestra.run.smithi089.stdout:Unpacking jq (1.6-1ubuntu0.20.04.1) ... 2023-06-03T19:21:18.591 INFO:teuthology.orchestra.run.smithi164.stdout:Selecting previously unselected package python3-blinker. 2023-06-03T19:21:18.606 INFO:teuthology.orchestra.run.smithi164.stdout:Preparing to unpack .../70-python3-blinker_1.4+dfsg1-0.3ubuntu1_all.deb ... 2023-06-03T19:21:18.649 INFO:teuthology.orchestra.run.smithi164.stdout:Unpacking python3-blinker (1.4+dfsg1-0.3ubuntu1) ... 2023-06-03T19:21:18.682 INFO:teuthology.orchestra.run.smithi089.stdout:Selecting previously unselected package socat. 2023-06-03T19:21:18.691 INFO:teuthology.orchestra.run.smithi089.stdout:Preparing to unpack .../79-socat_1.7.3.3-2_amd64.deb ... 2023-06-03T19:21:18.732 INFO:teuthology.orchestra.run.smithi089.stdout:Unpacking socat (1.7.3.3-2) ... 2023-06-03T19:21:18.977 INFO:teuthology.orchestra.run.smithi164.stdout:Selecting previously unselected package python3-oauthlib. 2023-06-03T19:21:18.990 INFO:teuthology.orchestra.run.smithi164.stdout:Preparing to unpack .../71-python3-oauthlib_3.1.0-1ubuntu2_all.deb ... 2023-06-03T19:21:19.035 INFO:teuthology.orchestra.run.smithi164.stdout:Unpacking python3-oauthlib (3.1.0-1ubuntu2) ... 2023-06-03T19:21:19.068 INFO:teuthology.orchestra.run.smithi089.stdout:Selecting previously unselected package xmlstarlet. 2023-06-03T19:21:19.077 INFO:teuthology.orchestra.run.smithi089.stdout:Preparing to unpack .../80-xmlstarlet_1.6.1-2build1_amd64.deb ... 2023-06-03T19:21:19.118 INFO:teuthology.orchestra.run.smithi089.stdout:Unpacking xmlstarlet (1.6.1-2build1) ... 2023-06-03T19:21:19.364 INFO:teuthology.orchestra.run.smithi164.stdout:Selecting previously unselected package python3-requests-oauthlib. 2023-06-03T19:21:19.377 INFO:teuthology.orchestra.run.smithi164.stdout:Preparing to unpack .../72-python3-requests-oauthlib_1.0.0-1.1build1_all.deb ... 2023-06-03T19:21:19.421 INFO:teuthology.orchestra.run.smithi164.stdout:Unpacking python3-requests-oauthlib (1.0.0-1.1build1) ... 2023-06-03T19:21:19.525 INFO:teuthology.orchestra.run.smithi089.stdout:Selecting previously unselected package ceph-test. 2023-06-03T19:21:19.534 INFO:teuthology.orchestra.run.smithi089.stdout:Preparing to unpack .../81-ceph-test_16.2.13-257-gd8c5d349-1focal_amd64.deb ... 2023-06-03T19:21:19.593 INFO:teuthology.orchestra.run.smithi089.stdout:Unpacking ceph-test (16.2.13-257-gd8c5d349-1focal) ... 2023-06-03T19:21:19.733 INFO:teuthology.orchestra.run.smithi164.stdout:Selecting previously unselected package python3-websocket. 2023-06-03T19:21:19.742 INFO:teuthology.orchestra.run.smithi164.stdout:Preparing to unpack .../73-python3-websocket_0.53.0-2ubuntu1_all.deb ... 2023-06-03T19:21:19.783 INFO:teuthology.orchestra.run.smithi164.stdout:Unpacking python3-websocket (0.53.0-2ubuntu1) ... 2023-06-03T19:21:20.102 INFO:teuthology.orchestra.run.smithi164.stdout:Selecting previously unselected package python3-kubernetes. 2023-06-03T19:21:20.111 INFO:teuthology.orchestra.run.smithi164.stdout:Preparing to unpack .../74-python3-kubernetes_7.0.0~a1-2_all.deb ... 2023-06-03T19:21:20.152 INFO:teuthology.orchestra.run.smithi164.stdout:Unpacking python3-kubernetes (7.0.0~a1-2) ... 2023-06-03T19:21:20.741 INFO:teuthology.orchestra.run.smithi164.stdout:Selecting previously unselected package ceph-mgr-k8sevents. 2023-06-03T19:21:20.755 INFO:teuthology.orchestra.run.smithi164.stdout:Preparing to unpack .../75-ceph-mgr-k8sevents_16.2.13-257-gd8c5d349-1focal_all.deb ... 2023-06-03T19:21:20.799 INFO:teuthology.orchestra.run.smithi164.stdout:Unpacking ceph-mgr-k8sevents (16.2.13-257-gd8c5d349-1focal) ... 2023-06-03T19:21:21.101 INFO:teuthology.orchestra.run.smithi164.stdout:Selecting previously unselected package libonig5:amd64. 2023-06-03T19:21:21.115 INFO:teuthology.orchestra.run.smithi164.stdout:Preparing to unpack .../76-libonig5_6.9.4-1_amd64.deb ... 2023-06-03T19:21:21.159 INFO:teuthology.orchestra.run.smithi164.stdout:Unpacking libonig5:amd64 (6.9.4-1) ... 2023-06-03T19:21:21.529 INFO:teuthology.orchestra.run.smithi164.stdout:Selecting previously unselected package libjq1:amd64. 2023-06-03T19:21:21.544 INFO:teuthology.orchestra.run.smithi164.stdout:Preparing to unpack .../77-libjq1_1.6-1ubuntu0.20.04.1_amd64.deb ... 2023-06-03T19:21:21.587 INFO:teuthology.orchestra.run.smithi164.stdout:Unpacking libjq1:amd64 (1.6-1ubuntu0.20.04.1) ... 2023-06-03T19:21:21.848 INFO:teuthology.orchestra.run.smithi164.stdout:Selecting previously unselected package jq. 2023-06-03T19:21:21.863 INFO:teuthology.orchestra.run.smithi164.stdout:Preparing to unpack .../78-jq_1.6-1ubuntu0.20.04.1_amd64.deb ... 2023-06-03T19:21:21.908 INFO:teuthology.orchestra.run.smithi164.stdout:Unpacking jq (1.6-1ubuntu0.20.04.1) ... 2023-06-03T19:21:22.176 INFO:teuthology.orchestra.run.smithi164.stdout:Selecting previously unselected package socat. 2023-06-03T19:21:22.191 INFO:teuthology.orchestra.run.smithi164.stdout:Preparing to unpack .../79-socat_1.7.3.3-2_amd64.deb ... 2023-06-03T19:21:22.259 INFO:teuthology.orchestra.run.smithi164.stdout:Unpacking socat (1.7.3.3-2) ... 2023-06-03T19:21:22.588 INFO:teuthology.orchestra.run.smithi164.stdout:Selecting previously unselected package xmlstarlet. 2023-06-03T19:21:22.603 INFO:teuthology.orchestra.run.smithi164.stdout:Preparing to unpack .../80-xmlstarlet_1.6.1-2build1_amd64.deb ... 2023-06-03T19:21:22.645 INFO:teuthology.orchestra.run.smithi164.stdout:Unpacking xmlstarlet (1.6.1-2build1) ... 2023-06-03T19:21:23.077 INFO:teuthology.orchestra.run.smithi164.stdout:Selecting previously unselected package ceph-test. 2023-06-03T19:21:23.092 INFO:teuthology.orchestra.run.smithi164.stdout:Preparing to unpack .../81-ceph-test_16.2.13-257-gd8c5d349-1focal_amd64.deb ... 2023-06-03T19:21:23.135 INFO:teuthology.orchestra.run.smithi164.stdout:Unpacking ceph-test (16.2.13-257-gd8c5d349-1focal) ... 2023-06-03T19:21:23.988 INFO:teuthology.orchestra.run.smithi089.stdout:Selecting previously unselected package libcephfs-dev. 2023-06-03T19:21:24.003 INFO:teuthology.orchestra.run.smithi089.stdout:Preparing to unpack .../82-libcephfs-dev_16.2.13-257-gd8c5d349-1focal_amd64.deb ... 2023-06-03T19:21:24.045 INFO:teuthology.orchestra.run.smithi089.stdout:Unpacking libcephfs-dev (16.2.13-257-gd8c5d349-1focal) ... 2023-06-03T19:21:24.449 INFO:teuthology.orchestra.run.smithi089.stdout:Selecting previously unselected package nvme-cli. 2023-06-03T19:21:24.466 INFO:teuthology.orchestra.run.smithi089.stdout:Preparing to unpack .../83-nvme-cli_1.9-1ubuntu0.1_amd64.deb ... 2023-06-03T19:21:24.522 INFO:teuthology.orchestra.run.smithi089.stdout:Unpacking nvme-cli (1.9-1ubuntu0.1) ... 2023-06-03T19:21:24.890 INFO:teuthology.orchestra.run.smithi089.stdout:Selecting previously unselected package python3-atomicwrites. 2023-06-03T19:21:24.906 INFO:teuthology.orchestra.run.smithi089.stdout:Preparing to unpack .../84-python3-atomicwrites_1.1.5-2build1_all.deb ... 2023-06-03T19:21:24.948 INFO:teuthology.orchestra.run.smithi089.stdout:Unpacking python3-atomicwrites (1.1.5-2build1) ... 2023-06-03T19:21:25.251 INFO:teuthology.orchestra.run.smithi089.stdout:Selecting previously unselected package python3-attr. 2023-06-03T19:21:25.266 INFO:teuthology.orchestra.run.smithi089.stdout:Preparing to unpack .../85-python3-attr_19.3.0-2_all.deb ... 2023-06-03T19:21:25.309 INFO:teuthology.orchestra.run.smithi089.stdout:Unpacking python3-attr (19.3.0-2) ... 2023-06-03T19:21:25.637 INFO:teuthology.orchestra.run.smithi089.stdout:Selecting previously unselected package python3-webencodings. 2023-06-03T19:21:25.651 INFO:teuthology.orchestra.run.smithi089.stdout:Preparing to unpack .../86-python3-webencodings_0.5.1-1ubuntu1_all.deb ... 2023-06-03T19:21:25.695 INFO:teuthology.orchestra.run.smithi089.stdout:Unpacking python3-webencodings (0.5.1-1ubuntu1) ... 2023-06-03T19:21:26.023 INFO:teuthology.orchestra.run.smithi089.stdout:Selecting previously unselected package python3-html5lib. 2023-06-03T19:21:26.039 INFO:teuthology.orchestra.run.smithi089.stdout:Preparing to unpack .../87-python3-html5lib_1.0.1-2_all.deb ... 2023-06-03T19:21:26.081 INFO:teuthology.orchestra.run.smithi089.stdout:Unpacking python3-html5lib (1.0.1-2) ... 2023-06-03T19:21:26.443 INFO:teuthology.orchestra.run.smithi089.stdout:Selecting previously unselected package python3-lxml:amd64. 2023-06-03T19:21:26.459 INFO:teuthology.orchestra.run.smithi089.stdout:Preparing to unpack .../88-python3-lxml_4.5.0-1ubuntu0.5_amd64.deb ... 2023-06-03T19:21:26.501 INFO:teuthology.orchestra.run.smithi089.stdout:Unpacking python3-lxml:amd64 (4.5.0-1ubuntu0.5) ... 2023-06-03T19:21:26.955 INFO:teuthology.orchestra.run.smithi089.stdout:Selecting previously unselected package python3-packaging. 2023-06-03T19:21:26.971 INFO:teuthology.orchestra.run.smithi089.stdout:Preparing to unpack .../89-python3-packaging_20.3-1_all.deb ... 2023-06-03T19:21:27.012 INFO:teuthology.orchestra.run.smithi089.stdout:Unpacking python3-packaging (20.3-1) ... 2023-06-03T19:21:27.324 INFO:teuthology.orchestra.run.smithi089.stdout:Selecting previously unselected package python3-pastescript. 2023-06-03T19:21:27.338 INFO:teuthology.orchestra.run.smithi089.stdout:Preparing to unpack .../90-python3-pastescript_2.0.2-4_all.deb ... 2023-06-03T19:21:27.382 INFO:teuthology.orchestra.run.smithi089.stdout:Unpacking python3-pastescript (2.0.2-4) ... 2023-06-03T19:21:27.406 INFO:teuthology.orchestra.run.smithi164.stdout:Selecting previously unselected package libcephfs-dev. 2023-06-03T19:21:27.420 INFO:teuthology.orchestra.run.smithi164.stdout:Preparing to unpack .../82-libcephfs-dev_16.2.13-257-gd8c5d349-1focal_amd64.deb ... 2023-06-03T19:21:27.455 INFO:teuthology.orchestra.run.smithi164.stdout:Unpacking libcephfs-dev (16.2.13-257-gd8c5d349-1focal) ... 2023-06-03T19:21:27.702 INFO:teuthology.orchestra.run.smithi089.stdout:Selecting previously unselected package python3-pluggy. 2023-06-03T19:21:27.719 INFO:teuthology.orchestra.run.smithi089.stdout:Preparing to unpack .../91-python3-pluggy_0.13.0-2_all.deb ... 2023-06-03T19:21:27.760 INFO:teuthology.orchestra.run.smithi089.stdout:Unpacking python3-pluggy (0.13.0-2) ... 2023-06-03T19:21:27.851 INFO:teuthology.orchestra.run.smithi164.stdout:Selecting previously unselected package nvme-cli. 2023-06-03T19:21:27.867 INFO:teuthology.orchestra.run.smithi164.stdout:Preparing to unpack .../83-nvme-cli_1.9-1ubuntu0.1_amd64.deb ... 2023-06-03T19:21:27.925 INFO:teuthology.orchestra.run.smithi164.stdout:Unpacking nvme-cli (1.9-1ubuntu0.1) ... 2023-06-03T19:21:28.072 INFO:teuthology.orchestra.run.smithi089.stdout:Selecting previously unselected package python3-psutil. 2023-06-03T19:21:28.086 INFO:teuthology.orchestra.run.smithi089.stdout:Preparing to unpack .../92-python3-psutil_5.5.1-1ubuntu4_amd64.deb ... 2023-06-03T19:21:28.139 INFO:teuthology.orchestra.run.smithi089.stdout:Unpacking python3-psutil (5.5.1-1ubuntu4) ... 2023-06-03T19:21:28.299 INFO:teuthology.orchestra.run.smithi164.stdout:Selecting previously unselected package python3-atomicwrites. 2023-06-03T19:21:28.315 INFO:teuthology.orchestra.run.smithi164.stdout:Preparing to unpack .../84-python3-atomicwrites_1.1.5-2build1_all.deb ... 2023-06-03T19:21:28.357 INFO:teuthology.orchestra.run.smithi164.stdout:Unpacking python3-atomicwrites (1.1.5-2build1) ... 2023-06-03T19:21:28.467 INFO:teuthology.orchestra.run.smithi089.stdout:Selecting previously unselected package python3-py. 2023-06-03T19:21:28.481 INFO:teuthology.orchestra.run.smithi089.stdout:Preparing to unpack .../93-python3-py_1.8.1-1ubuntu0.1_all.deb ... 2023-06-03T19:21:28.525 INFO:teuthology.orchestra.run.smithi089.stdout:Unpacking python3-py (1.8.1-1ubuntu0.1) ... 2023-06-03T19:21:28.660 INFO:teuthology.orchestra.run.smithi164.stdout:Selecting previously unselected package python3-attr. 2023-06-03T19:21:28.674 INFO:teuthology.orchestra.run.smithi164.stdout:Preparing to unpack .../85-python3-attr_19.3.0-2_all.deb ... 2023-06-03T19:21:28.718 INFO:teuthology.orchestra.run.smithi164.stdout:Unpacking python3-attr (19.3.0-2) ... 2023-06-03T19:21:28.828 INFO:teuthology.orchestra.run.smithi089.stdout:Selecting previously unselected package python3-pyinotify. 2023-06-03T19:21:28.843 INFO:teuthology.orchestra.run.smithi089.stdout:Preparing to unpack .../94-python3-pyinotify_0.9.6-1.2ubuntu1_all.deb ... 2023-06-03T19:21:28.886 INFO:teuthology.orchestra.run.smithi089.stdout:Unpacking python3-pyinotify (0.9.6-1.2ubuntu1) ... 2023-06-03T19:21:29.104 INFO:teuthology.orchestra.run.smithi164.stdout:Selecting previously unselected package python3-webencodings. 2023-06-03T19:21:29.118 INFO:teuthology.orchestra.run.smithi164.stdout:Preparing to unpack .../86-python3-webencodings_0.5.1-1ubuntu1_all.deb ... 2023-06-03T19:21:29.162 INFO:teuthology.orchestra.run.smithi164.stdout:Unpacking python3-webencodings (0.5.1-1ubuntu1) ... 2023-06-03T19:21:29.206 INFO:teuthology.orchestra.run.smithi089.stdout:Selecting previously unselected package python3-wcwidth. 2023-06-03T19:21:29.220 INFO:teuthology.orchestra.run.smithi089.stdout:Preparing to unpack .../95-python3-wcwidth_0.1.8+dfsg1-3_all.deb ... 2023-06-03T19:21:29.264 INFO:teuthology.orchestra.run.smithi089.stdout:Unpacking python3-wcwidth (0.1.8+dfsg1-3) ... 2023-06-03T19:21:29.473 INFO:teuthology.orchestra.run.smithi164.stdout:Selecting previously unselected package python3-html5lib. 2023-06-03T19:21:29.488 INFO:teuthology.orchestra.run.smithi164.stdout:Preparing to unpack .../87-python3-html5lib_1.0.1-2_all.deb ... 2023-06-03T19:21:29.532 INFO:teuthology.orchestra.run.smithi164.stdout:Unpacking python3-html5lib (1.0.1-2) ... 2023-06-03T19:21:29.567 INFO:teuthology.orchestra.run.smithi089.stdout:Selecting previously unselected package python3-pytest. 2023-06-03T19:21:29.582 INFO:teuthology.orchestra.run.smithi089.stdout:Preparing to unpack .../96-python3-pytest_4.6.9-1_all.deb ... 2023-06-03T19:21:29.625 INFO:teuthology.orchestra.run.smithi089.stdout:Unpacking python3-pytest (4.6.9-1) ... 2023-06-03T19:21:29.876 INFO:teuthology.orchestra.run.smithi164.stdout:Selecting previously unselected package python3-lxml:amd64. 2023-06-03T19:21:29.891 INFO:teuthology.orchestra.run.smithi164.stdout:Preparing to unpack .../88-python3-lxml_4.5.0-1ubuntu0.5_amd64.deb ... 2023-06-03T19:21:29.934 INFO:teuthology.orchestra.run.smithi164.stdout:Unpacking python3-lxml:amd64 (4.5.0-1ubuntu0.5) ... 2023-06-03T19:21:29.962 INFO:teuthology.orchestra.run.smithi089.stdout:Selecting previously unselected package python3-simplejson. 2023-06-03T19:21:29.977 INFO:teuthology.orchestra.run.smithi089.stdout:Preparing to unpack .../97-python3-simplejson_3.16.0-2ubuntu2_amd64.deb ... 2023-06-03T19:21:30.020 INFO:teuthology.orchestra.run.smithi089.stdout:Unpacking python3-simplejson (3.16.0-2ubuntu2) ... 2023-06-03T19:21:30.405 INFO:teuthology.orchestra.run.smithi164.stdout:Selecting previously unselected package python3-packaging. 2023-06-03T19:21:30.419 INFO:teuthology.orchestra.run.smithi164.stdout:Preparing to unpack .../89-python3-packaging_20.3-1_all.deb ... 2023-06-03T19:21:30.462 INFO:teuthology.orchestra.run.smithi164.stdout:Unpacking python3-packaging (20.3-1) ... 2023-06-03T19:21:30.466 INFO:teuthology.orchestra.run.smithi089.stdout:Selecting previously unselected package radosgw. 2023-06-03T19:21:30.482 INFO:teuthology.orchestra.run.smithi089.stdout:Preparing to unpack .../98-radosgw_16.2.13-257-gd8c5d349-1focal_amd64.deb ... 2023-06-03T19:21:30.524 INFO:teuthology.orchestra.run.smithi089.stdout:Unpacking radosgw (16.2.13-257-gd8c5d349-1focal) ... 2023-06-03T19:21:30.765 INFO:teuthology.orchestra.run.smithi164.stdout:Selecting previously unselected package python3-pastescript. 2023-06-03T19:21:30.779 INFO:teuthology.orchestra.run.smithi164.stdout:Preparing to unpack .../90-python3-pastescript_2.0.2-4_all.deb ... 2023-06-03T19:21:30.823 INFO:teuthology.orchestra.run.smithi164.stdout:Unpacking python3-pastescript (2.0.2-4) ... 2023-06-03T19:21:31.135 INFO:teuthology.orchestra.run.smithi164.stdout:Selecting previously unselected package python3-pluggy. 2023-06-03T19:21:31.151 INFO:teuthology.orchestra.run.smithi164.stdout:Preparing to unpack .../91-python3-pluggy_0.13.0-2_all.deb ... 2023-06-03T19:21:31.193 INFO:teuthology.orchestra.run.smithi164.stdout:Unpacking python3-pluggy (0.13.0-2) ... 2023-06-03T19:21:31.521 INFO:teuthology.orchestra.run.smithi164.stdout:Selecting previously unselected package python3-psutil. 2023-06-03T19:21:31.536 INFO:teuthology.orchestra.run.smithi164.stdout:Preparing to unpack .../92-python3-psutil_5.5.1-1ubuntu4_amd64.deb ... 2023-06-03T19:21:31.571 INFO:teuthology.orchestra.run.smithi164.stdout:Unpacking python3-psutil (5.5.1-1ubuntu4) ... 2023-06-03T19:21:31.719 INFO:teuthology.orchestra.run.smithi089.stdout:Selecting previously unselected package rbd-fuse. 2023-06-03T19:21:31.733 INFO:teuthology.orchestra.run.smithi089.stdout:Preparing to unpack .../99-rbd-fuse_16.2.13-257-gd8c5d349-1focal_amd64.deb ... 2023-06-03T19:21:31.777 INFO:teuthology.orchestra.run.smithi089.stdout:Unpacking rbd-fuse (16.2.13-257-gd8c5d349-1focal) ... 2023-06-03T19:21:31.908 INFO:teuthology.orchestra.run.smithi164.stdout:Selecting previously unselected package python3-py. 2023-06-03T19:21:31.921 INFO:teuthology.orchestra.run.smithi164.stdout:Preparing to unpack .../93-python3-py_1.8.1-1ubuntu0.1_all.deb ... 2023-06-03T19:21:31.965 INFO:teuthology.orchestra.run.smithi164.stdout:Unpacking python3-py (1.8.1-1ubuntu0.1) ... 2023-06-03T19:21:32.077 INFO:teuthology.orchestra.run.smithi089.stdout:Setting up python3-attr (19.3.0-2) ... 2023-06-03T19:21:32.294 INFO:teuthology.orchestra.run.smithi164.stdout:Selecting previously unselected package python3-pyinotify. 2023-06-03T19:21:32.307 INFO:teuthology.orchestra.run.smithi164.stdout:Preparing to unpack .../94-python3-pyinotify_0.9.6-1.2ubuntu1_all.deb ... 2023-06-03T19:21:32.352 INFO:teuthology.orchestra.run.smithi164.stdout:Unpacking python3-pyinotify (0.9.6-1.2ubuntu1) ... 2023-06-03T19:21:32.374 INFO:teuthology.orchestra.run.smithi089.stdout:Setting up libleveldb1d:amd64 (1.22-3ubuntu2) ... 2023-06-03T19:21:32.501 INFO:teuthology.orchestra.run.smithi089.stdout:Setting up librabbitmq4:amd64 (0.10.0-1) ... 2023-06-03T19:21:32.619 INFO:teuthology.orchestra.run.smithi089.stdout:Setting up nvme-cli (1.9-1ubuntu0.1) ... 2023-06-03T19:21:32.663 INFO:teuthology.orchestra.run.smithi164.stdout:Selecting previously unselected package python3-wcwidth. 2023-06-03T19:21:32.679 INFO:teuthology.orchestra.run.smithi164.stdout:Preparing to unpack .../95-python3-wcwidth_0.1.8+dfsg1-3_all.deb ... 2023-06-03T19:21:32.721 INFO:teuthology.orchestra.run.smithi164.stdout:Unpacking python3-wcwidth (0.1.8+dfsg1-3) ... 2023-06-03T19:21:32.803 INFO:teuthology.orchestra.run.smithi089.stdout:Setting up cephadm (16.2.13-257-gd8c5d349-1focal) ... 2023-06-03T19:21:33.024 INFO:teuthology.orchestra.run.smithi164.stdout:Selecting previously unselected package python3-pytest. 2023-06-03T19:21:33.038 INFO:teuthology.orchestra.run.smithi164.stdout:Preparing to unpack .../96-python3-pytest_4.6.9-1_all.deb ... 2023-06-03T19:21:33.082 INFO:teuthology.orchestra.run.smithi164.stdout:Unpacking python3-pytest (4.6.9-1) ... 2023-06-03T19:21:33.390 INFO:teuthology.orchestra.run.smithi089.stdout:Adding system user cephadm....done 2023-06-03T19:21:33.444 INFO:teuthology.orchestra.run.smithi164.stdout:Selecting previously unselected package python3-simplejson. 2023-06-03T19:21:33.458 INFO:teuthology.orchestra.run.smithi164.stdout:Preparing to unpack .../97-python3-simplejson_3.16.0-2ubuntu2_amd64.deb ... 2023-06-03T19:21:33.502 INFO:teuthology.orchestra.run.smithi164.stdout:Unpacking python3-simplejson (3.16.0-2ubuntu2) ... 2023-06-03T19:21:33.543 INFO:teuthology.orchestra.run.smithi089.stdout:Setting up python3-waitress (1.4.1-1ubuntu0.1) ... 2023-06-03T19:21:33.861 INFO:teuthology.orchestra.run.smithi089.stdout:Setting up python3-repoze.lru (0.7-2) ... 2023-06-03T19:21:33.923 INFO:teuthology.orchestra.run.smithi164.stdout:Selecting previously unselected package radosgw. 2023-06-03T19:21:33.937 INFO:teuthology.orchestra.run.smithi164.stdout:Preparing to unpack .../98-radosgw_16.2.13-257-gd8c5d349-1focal_amd64.deb ... 2023-06-03T19:21:33.981 INFO:teuthology.orchestra.run.smithi164.stdout:Unpacking radosgw (16.2.13-257-gd8c5d349-1focal) ... 2023-06-03T19:21:34.171 INFO:teuthology.orchestra.run.smithi089.stdout:Setting up liboath0 (2.6.1-1.3) ... 2023-06-03T19:21:34.297 INFO:teuthology.orchestra.run.smithi089.stdout:Setting up python3-py (1.8.1-1ubuntu0.1) ... 2023-06-03T19:21:34.615 INFO:teuthology.orchestra.run.smithi089.stdout:Setting up python3-joblib (0.14.0-3) ... 2023-06-03T19:21:35.001 INFO:teuthology.orchestra.run.smithi089.stdout:Setting up python3-jwt (1.7.1-2ubuntu2.1) ... 2023-06-03T19:21:35.149 INFO:teuthology.orchestra.run.smithi164.stdout:Selecting previously unselected package rbd-fuse. 2023-06-03T19:21:35.163 INFO:teuthology.orchestra.run.smithi164.stdout:Preparing to unpack .../99-rbd-fuse_16.2.13-257-gd8c5d349-1focal_amd64.deb ... 2023-06-03T19:21:35.207 INFO:teuthology.orchestra.run.smithi164.stdout:Unpacking rbd-fuse (16.2.13-257-gd8c5d349-1focal) ... 2023-06-03T19:21:35.294 INFO:teuthology.orchestra.run.smithi089.stdout:Setting up python3-cachetools (4.0.0-1) ... 2023-06-03T19:21:35.506 INFO:teuthology.orchestra.run.smithi164.stdout:Setting up python3-attr (19.3.0-2) ... 2023-06-03T19:21:35.578 INFO:teuthology.orchestra.run.smithi089.stdout:Setting up python3-bcrypt (3.1.7-2ubuntu1) ... 2023-06-03T19:21:35.828 INFO:teuthology.orchestra.run.smithi164.stdout:Setting up libleveldb1d:amd64 (1.22-3ubuntu2) ... 2023-06-03T19:21:35.863 INFO:teuthology.orchestra.run.smithi089.stdout:Setting up python3-pyinotify (0.9.6-1.2ubuntu1) ... 2023-06-03T19:21:35.954 INFO:teuthology.orchestra.run.smithi164.stdout:Setting up librabbitmq4:amd64 (0.10.0-1) ... 2023-06-03T19:21:36.080 INFO:teuthology.orchestra.run.smithi164.stdout:Setting up nvme-cli (1.9-1ubuntu0.1) ... 2023-06-03T19:21:36.172 INFO:teuthology.orchestra.run.smithi089.stdout:Setting up python3-ceph-argparse (16.2.13-257-gd8c5d349-1focal) ... 2023-06-03T19:21:36.282 INFO:teuthology.orchestra.run.smithi164.stdout:Setting up cephadm (16.2.13-257-gd8c5d349-1focal) ... 2023-06-03T19:21:36.474 INFO:teuthology.orchestra.run.smithi089.stdout:Setting up python3-sklearn-lib (0.22.2.post1+dfsg-5) ... 2023-06-03T19:21:36.610 INFO:teuthology.orchestra.run.smithi089.stdout:Setting up python3-markupsafe (1.1.0-1build2) ... 2023-06-03T19:21:36.884 INFO:teuthology.orchestra.run.smithi164.stdout:Adding system user cephadm....done 2023-06-03T19:21:36.913 INFO:teuthology.orchestra.run.smithi089.stdout:Setting up python3-webencodings (0.5.1-1ubuntu1) ... 2023-06-03T19:21:37.038 INFO:teuthology.orchestra.run.smithi164.stdout:Setting up python3-waitress (1.4.1-1ubuntu0.1) ... 2023-06-03T19:21:37.206 INFO:teuthology.orchestra.run.smithi089.stdout:Setting up python3-psutil (5.5.1-1ubuntu4) ... 2023-06-03T19:21:37.373 INFO:teuthology.orchestra.run.smithi164.stdout:Setting up python3-repoze.lru (0.7-2) ... 2023-06-03T19:21:37.591 INFO:teuthology.orchestra.run.smithi089.stdout:Setting up python3-atomicwrites (1.1.5-2build1) ... 2023-06-03T19:21:37.640 INFO:teuthology.orchestra.run.smithi164.stdout:Setting up liboath0 (2.6.1-1.3) ... 2023-06-03T19:21:37.766 INFO:teuthology.orchestra.run.smithi164.stdout:Setting up python3-py (1.8.1-1ubuntu0.1) ... 2023-06-03T19:21:37.875 INFO:teuthology.orchestra.run.smithi089.stdout:Setting up python3-simplejson (3.16.0-2ubuntu2) ... 2023-06-03T19:21:38.093 INFO:teuthology.orchestra.run.smithi164.stdout:Setting up python3-joblib (0.14.0-3) ... 2023-06-03T19:21:38.226 INFO:teuthology.orchestra.run.smithi089.stdout:Setting up python3-decorator (4.4.2-0ubuntu1) ... 2023-06-03T19:21:38.499 INFO:teuthology.orchestra.run.smithi164.stdout:Setting up python3-jwt (1.7.1-2ubuntu2.1) ... 2023-06-03T19:21:38.527 INFO:teuthology.orchestra.run.smithi089.stdout:Setting up libjaeger (16.2.13-257-gd8c5d349-1focal) ... 2023-06-03T19:21:38.654 INFO:teuthology.orchestra.run.smithi089.stdout:Setting up python3-jinja2 (2.10.1-2) ... 2023-06-03T19:21:38.783 INFO:teuthology.orchestra.run.smithi164.stdout:Setting up python3-cachetools (4.0.0-1) ... 2023-06-03T19:21:38.988 INFO:teuthology.orchestra.run.smithi089.stdout:Setting up python3-tempita (0.5.2-6) ... 2023-06-03T19:21:39.085 INFO:teuthology.orchestra.run.smithi164.stdout:Setting up python3-bcrypt (3.1.7-2ubuntu1) ... 2023-06-03T19:21:39.281 INFO:teuthology.orchestra.run.smithi089.stdout:Setting up python3-packaging (20.3-1) ... 2023-06-03T19:21:39.369 INFO:teuthology.orchestra.run.smithi164.stdout:Setting up python3-pyinotify (0.9.6-1.2ubuntu1) ... 2023-06-03T19:21:39.590 INFO:teuthology.orchestra.run.smithi089.stdout:Setting up python-pastedeploy-tpl (2.1.0-1) ... 2023-06-03T19:21:39.662 INFO:teuthology.orchestra.run.smithi164.stdout:Setting up python3-ceph-argparse (16.2.13-257-gd8c5d349-1focal) ... 2023-06-03T19:21:39.716 INFO:teuthology.orchestra.run.smithi089.stdout:Setting up python3-wcwidth (0.1.8+dfsg1-3) ... 2023-06-03T19:21:39.947 INFO:teuthology.orchestra.run.smithi164.stdout:Setting up python3-sklearn-lib (0.22.2.post1+dfsg-5) ... 2023-06-03T19:21:40.009 INFO:teuthology.orchestra.run.smithi089.stdout:Setting up python3-paste (3.4.0+dfsg1-1) ... 2023-06-03T19:21:40.088 INFO:teuthology.orchestra.run.smithi164.stdout:Setting up python3-markupsafe (1.1.0-1build2) ... 2023-06-03T19:21:40.373 INFO:teuthology.orchestra.run.smithi164.stdout:Setting up python3-webencodings (0.5.1-1ubuntu1) ... 2023-06-03T19:21:40.402 INFO:teuthology.orchestra.run.smithi089.stdout:Setting up python3-werkzeug (0.16.1+dfsg1-2ubuntu0.1) ... 2023-06-03T19:21:40.649 INFO:teuthology.orchestra.run.smithi164.stdout:Setting up python3-psutil (5.5.1-1ubuntu4) ... 2023-06-03T19:21:40.771 INFO:teuthology.orchestra.run.smithi089.stdout:Setting up python3-cryptography (2.8-3ubuntu0.1) ... 2023-06-03T19:21:41.017 INFO:teuthology.orchestra.run.smithi164.stdout:Setting up python3-atomicwrites (1.1.5-2build1) ... 2023-06-03T19:21:41.148 INFO:teuthology.orchestra.run.smithi089.stdout:Setting up socat (1.7.3.3-2) ... 2023-06-03T19:21:41.274 INFO:teuthology.orchestra.run.smithi089.stdout:Setting up python3-ceph-common (16.2.13-257-gd8c5d349-1focal) ... 2023-06-03T19:21:41.292 INFO:teuthology.orchestra.run.smithi164.stdout:Setting up python3-simplejson (3.16.0-2ubuntu2) ... 2023-06-03T19:21:41.584 INFO:teuthology.orchestra.run.smithi089.stdout:Setting up python3-html5lib (1.0.1-2) ... 2023-06-03T19:21:41.627 INFO:teuthology.orchestra.run.smithi164.stdout:Setting up python3-decorator (4.4.2-0ubuntu1) ... 2023-06-03T19:21:41.920 INFO:teuthology.orchestra.run.smithi164.stdout:Setting up libjaeger (16.2.13-257-gd8c5d349-1focal) ... 2023-06-03T19:21:41.927 INFO:teuthology.orchestra.run.smithi089.stdout:Setting up librdkafka1:amd64 (1.2.1-1ubuntu1) ... 2023-06-03T19:21:42.038 INFO:teuthology.orchestra.run.smithi164.stdout:Setting up python3-jinja2 (2.10.1-2) ... 2023-06-03T19:21:42.053 INFO:teuthology.orchestra.run.smithi089.stdout:Setting up xmlstarlet (1.6.1-2build1) ... 2023-06-03T19:21:42.180 INFO:teuthology.orchestra.run.smithi089.stdout:Setting up python3-pluggy (0.13.0-2) ... 2023-06-03T19:21:42.364 INFO:teuthology.orchestra.run.smithi164.stdout:Setting up python3-tempita (0.5.2-6) ... 2023-06-03T19:21:42.472 INFO:teuthology.orchestra.run.smithi089.stdout:Setting up python3-lxml:amd64 (4.5.0-1ubuntu0.5) ... 2023-06-03T19:21:42.674 INFO:teuthology.orchestra.run.smithi164.stdout:Setting up python3-packaging (20.3-1) ... 2023-06-03T19:21:42.782 INFO:teuthology.orchestra.run.smithi089.stdout:Setting up liblua5.3-0:amd64 (5.3.3-1.1ubuntu2) ... 2023-06-03T19:21:42.908 INFO:teuthology.orchestra.run.smithi089.stdout:Setting up python3-pyasn1 (0.4.2-3build1) ... 2023-06-03T19:21:42.975 INFO:teuthology.orchestra.run.smithi164.stdout:Setting up python-pastedeploy-tpl (2.1.0-1) ... 2023-06-03T19:21:43.118 INFO:teuthology.orchestra.run.smithi164.stdout:Setting up python3-wcwidth (0.1.8+dfsg1-3) ... 2023-06-03T19:21:43.226 INFO:teuthology.orchestra.run.smithi089.stdout:Setting up python3-singledispatch (3.4.0.3-2build1) ... 2023-06-03T19:21:43.402 INFO:teuthology.orchestra.run.smithi164.stdout:Setting up python3-paste (3.4.0+dfsg1-1) ... 2023-06-03T19:21:43.503 INFO:teuthology.orchestra.run.smithi089.stdout:Setting up python3-logutils (0.3.3-7) ... 2023-06-03T19:21:43.787 INFO:teuthology.orchestra.run.smithi164.stdout:Setting up python3-werkzeug (0.16.1+dfsg1-2ubuntu0.1) ... 2023-06-03T19:21:43.795 INFO:teuthology.orchestra.run.smithi089.stdout:Setting up liblttng-ust0:amd64 (2.11.0-1) ... 2023-06-03T19:21:43.921 INFO:teuthology.orchestra.run.smithi089.stdout:Setting up python3-simplegeneric (0.8.1-3) ... 2023-06-03T19:21:44.130 INFO:teuthology.orchestra.run.smithi164.stdout:Setting up python3-cryptography (2.8-3ubuntu0.1) ... 2023-06-03T19:21:44.214 INFO:teuthology.orchestra.run.smithi089.stdout:Setting up python3-prettytable (0.7.2-5) ... 2023-06-03T19:21:44.490 INFO:teuthology.orchestra.run.smithi164.stdout:Setting up socat (1.7.3.3-2) ... 2023-06-03T19:21:44.501 INFO:teuthology.orchestra.run.smithi089.stdout:Setting up python3-soupsieve (1.9.5+dfsg-1) ... 2023-06-03T19:21:44.616 INFO:teuthology.orchestra.run.smithi164.stdout:Setting up python3-ceph-common (16.2.13-257-gd8c5d349-1focal) ... 2023-06-03T19:21:44.794 INFO:teuthology.orchestra.run.smithi089.stdout:Setting up python3-websocket (0.53.0-2ubuntu1) ... 2023-06-03T19:21:44.926 INFO:teuthology.orchestra.run.smithi164.stdout:Setting up python3-html5lib (1.0.1-2) ... 2023-06-03T19:21:45.186 INFO:teuthology.orchestra.run.smithi089.stdout:update-alternatives: using /usr/bin/python3-wsdump to provide /usr/bin/wsdump (wsdump) in auto mode 2023-06-03T19:21:45.286 INFO:teuthology.orchestra.run.smithi164.stdout:Setting up librdkafka1:amd64 (1.2.1-1ubuntu1) ... 2023-06-03T19:21:45.403 INFO:teuthology.orchestra.run.smithi164.stdout:Setting up xmlstarlet (1.6.1-2build1) ... 2023-06-03T19:21:45.438 INFO:teuthology.orchestra.run.smithi089.stdout:Setting up libonig5:amd64 (6.9.4-1) ... 2023-06-03T19:21:45.513 INFO:teuthology.orchestra.run.smithi164.stdout:Setting up python3-pluggy (0.13.0-2) ... 2023-06-03T19:21:45.564 INFO:teuthology.orchestra.run.smithi089.stdout:Setting up python3-mako (1.1.0+ds1-1ubuntu2.1) ... 2023-06-03T19:21:45.831 INFO:teuthology.orchestra.run.smithi164.stdout:Setting up python3-lxml:amd64 (4.5.0-1ubuntu0.5) ... 2023-06-03T19:21:45.882 INFO:teuthology.orchestra.run.smithi089.stdout:Setting up python3-cherrypy3 (8.9.1-6) ... 2023-06-03T19:21:46.140 INFO:teuthology.orchestra.run.smithi164.stdout:Setting up liblua5.3-0:amd64 (5.3.3-1.1ubuntu2) ... 2023-06-03T19:21:46.234 INFO:teuthology.orchestra.run.smithi089.stdout:Setting up libjs-underscore (1.9.1~dfsg-1ubuntu0.20.04.1) ... 2023-06-03T19:21:46.267 INFO:teuthology.orchestra.run.smithi164.stdout:Setting up python3-pyasn1 (0.4.2-3build1) ... 2023-06-03T19:21:46.360 INFO:teuthology.orchestra.run.smithi089.stdout:Setting up liblbfgsb0:amd64 (3.0+dfsg.3-7build1) ... 2023-06-03T19:21:46.486 INFO:teuthology.orchestra.run.smithi089.stdout:Setting up python3-webob (1:1.8.5-2) ... 2023-06-03T19:21:46.593 INFO:teuthology.orchestra.run.smithi164.stdout:Setting up python3-singledispatch (3.4.0.3-2build1) ... 2023-06-03T19:21:46.812 INFO:teuthology.orchestra.run.smithi089.stdout:Setting up python3-blinker (1.4+dfsg1-0.3ubuntu1) ... 2023-06-03T19:21:46.877 INFO:teuthology.orchestra.run.smithi164.stdout:Setting up python3-logutils (0.3.3-7) ... 2023-06-03T19:21:47.154 INFO:teuthology.orchestra.run.smithi164.stdout:Setting up liblttng-ust0:amd64 (2.11.0-1) ... 2023-06-03T19:21:47.280 INFO:teuthology.orchestra.run.smithi164.stdout:Setting up python3-simplegeneric (0.8.1-3) ... 2023-06-03T19:21:47.418 INFO:teuthology.orchestra.run.smithi089.stdout:Setting up librados2 (16.2.13-257-gd8c5d349-1focal) ... 2023-06-03T19:21:47.556 INFO:teuthology.orchestra.run.smithi164.stdout:Setting up python3-prettytable (0.7.2-5) ... 2023-06-03T19:21:47.597 INFO:teuthology.orchestra.run.smithi089.stdout:Setting up python3-scipy (1.3.3-3build1) ... 2023-06-03T19:21:48.132 INFO:teuthology.orchestra.run.smithi164.stdout:Setting up python3-soupsieve (1.9.5+dfsg-1) ... 2023-06-03T19:21:48.433 INFO:teuthology.orchestra.run.smithi164.stdout:Setting up python3-websocket (0.53.0-2ubuntu1) ... 2023-06-03T19:21:48.822 INFO:teuthology.orchestra.run.smithi164.stdout:update-alternatives: using /usr/bin/python3-wsdump to provide /usr/bin/wsdump (wsdump) in auto mode 2023-06-03T19:21:49.072 INFO:teuthology.orchestra.run.smithi089.stdout:Setting up librgw2 (16.2.13-257-gd8c5d349-1focal) ... 2023-06-03T19:21:49.086 INFO:teuthology.orchestra.run.smithi164.stdout:Setting up libonig5:amd64 (6.9.4-1) ... 2023-06-03T19:21:49.198 INFO:teuthology.orchestra.run.smithi089.stdout:Setting up libsqlite3-mod-ceph (16.2.13-257-gd8c5d349-1focal) ... 2023-06-03T19:21:49.212 INFO:teuthology.orchestra.run.smithi164.stdout:Setting up python3-mako (1.1.0+ds1-1ubuntu2.1) ... 2023-06-03T19:21:49.315 INFO:teuthology.orchestra.run.smithi089.stdout:Setting up libjq1:amd64 (1.6-1ubuntu0.20.04.1) ... 2023-06-03T19:21:49.442 INFO:teuthology.orchestra.run.smithi089.stdout:Setting up python3-pytest (4.6.9-1) ... 2023-06-03T19:21:49.521 INFO:teuthology.orchestra.run.smithi164.stdout:Setting up python3-cherrypy3 (8.9.1-6) ... 2023-06-03T19:21:49.827 INFO:teuthology.orchestra.run.smithi089.stdout:Setting up python3-openssl (19.0.0-1build1) ... 2023-06-03T19:21:49.873 INFO:teuthology.orchestra.run.smithi164.stdout:Setting up libjs-underscore (1.9.1~dfsg-1ubuntu0.20.04.1) ... 2023-06-03T19:21:49.999 INFO:teuthology.orchestra.run.smithi164.stdout:Setting up liblbfgsb0:amd64 (3.0+dfsg.3-7build1) ... 2023-06-03T19:21:50.119 INFO:teuthology.orchestra.run.smithi089.stdout:Setting up python3-pastedeploy (2.1.0-1) ... 2023-06-03T19:21:50.126 INFO:teuthology.orchestra.run.smithi164.stdout:Setting up python3-webob (1:1.8.5-2) ... 2023-06-03T19:21:50.404 INFO:teuthology.orchestra.run.smithi089.stdout:Setting up python3-bs4 (4.8.2-1) ... 2023-06-03T19:21:50.460 INFO:teuthology.orchestra.run.smithi164.stdout:Setting up python3-blinker (1.4+dfsg1-0.3ubuntu1) ... 2023-06-03T19:21:50.747 INFO:teuthology.orchestra.run.smithi089.stdout:Setting up libcephfs2 (16.2.13-257-gd8c5d349-1focal) ... 2023-06-03T19:21:50.753 INFO:teuthology.orchestra.run.smithi164.stdout:Setting up librados2 (16.2.13-257-gd8c5d349-1focal) ... 2023-06-03T19:21:50.863 INFO:teuthology.orchestra.run.smithi164.stdout:Setting up python3-scipy (1.3.3-3build1) ... 2023-06-03T19:21:50.864 INFO:teuthology.orchestra.run.smithi089.stdout:Setting up python3-oauthlib (3.1.0-1ubuntu2) ... 2023-06-03T19:21:51.191 INFO:teuthology.orchestra.run.smithi089.stdout:Setting up libradosstriper1 (16.2.13-257-gd8c5d349-1focal) ... 2023-06-03T19:21:51.318 INFO:teuthology.orchestra.run.smithi089.stdout:Setting up python3-pyasn1-modules (0.2.1-0.2build1) ... 2023-06-03T19:21:51.636 INFO:teuthology.orchestra.run.smithi089.stdout:Setting up python3-sklearn (0.22.2.post1+dfsg-5) ... 2023-06-03T19:21:52.327 INFO:teuthology.orchestra.run.smithi164.stdout:Setting up librgw2 (16.2.13-257-gd8c5d349-1focal) ... 2023-06-03T19:21:52.453 INFO:teuthology.orchestra.run.smithi164.stdout:Setting up libsqlite3-mod-ceph (16.2.13-257-gd8c5d349-1focal) ... 2023-06-03T19:21:52.578 INFO:teuthology.orchestra.run.smithi164.stdout:Setting up libjq1:amd64 (1.6-1ubuntu0.20.04.1) ... 2023-06-03T19:21:52.691 INFO:teuthology.orchestra.run.smithi089.stdout:Setting up librbd1 (16.2.13-257-gd8c5d349-1focal) ... 2023-06-03T19:21:52.713 INFO:teuthology.orchestra.run.smithi164.stdout:Setting up python3-pytest (4.6.9-1) ... 2023-06-03T19:21:52.826 INFO:teuthology.orchestra.run.smithi089.stdout:Setting up libjs-sphinxdoc (1.8.5-7ubuntu3) ... 2023-06-03T19:21:52.952 INFO:teuthology.orchestra.run.smithi089.stdout:Setting up jq (1.6-1ubuntu0.20.04.1) ... 2023-06-03T19:21:53.078 INFO:teuthology.orchestra.run.smithi089.stdout:Setting up python3-rsa (4.0-3) ... 2023-06-03T19:21:53.090 INFO:teuthology.orchestra.run.smithi164.stdout:Setting up python3-openssl (19.0.0-1build1) ... 2023-06-03T19:21:53.379 INFO:teuthology.orchestra.run.smithi089.stdout:Setting up python3-webtest (2.0.32-2) ... 2023-06-03T19:21:53.399 INFO:teuthology.orchestra.run.smithi164.stdout:Setting up python3-pastedeploy (2.1.0-1) ... 2023-06-03T19:21:53.680 INFO:teuthology.orchestra.run.smithi089.stdout:Setting up ceph-mgr-modules-core (16.2.13-257-gd8c5d349-1focal) ... 2023-06-03T19:21:53.684 INFO:teuthology.orchestra.run.smithi164.stdout:Setting up python3-bs4 (4.8.2-1) ... 2023-06-03T19:21:53.806 INFO:teuthology.orchestra.run.smithi089.stdout:Setting up ceph-fuse (16.2.13-257-gd8c5d349-1focal) ... 2023-06-03T19:21:54.016 INFO:teuthology.orchestra.run.smithi089.stdout:Created symlink /etc/systemd/system/remote-fs.target.wants/ceph-fuse.target -> /lib/systemd/system/ceph-fuse.target. 2023-06-03T19:21:54.016 INFO:teuthology.orchestra.run.smithi089.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-fuse.target -> /lib/systemd/system/ceph-fuse.target. 2023-06-03T19:21:54.018 INFO:teuthology.orchestra.run.smithi164.stdout:Setting up libcephfs2 (16.2.13-257-gd8c5d349-1focal) ... 2023-06-03T19:21:54.144 INFO:teuthology.orchestra.run.smithi164.stdout:Setting up python3-oauthlib (3.1.0-1ubuntu2) ... 2023-06-03T19:21:54.480 INFO:teuthology.orchestra.run.smithi164.stdout:Setting up libradosstriper1 (16.2.13-257-gd8c5d349-1focal) ... 2023-06-03T19:21:54.508 INFO:teuthology.orchestra.run.smithi089.stdout:Setting up libcephfs-dev (16.2.13-257-gd8c5d349-1focal) ... 2023-06-03T19:21:54.606 INFO:teuthology.orchestra.run.smithi164.stdout:Setting up python3-pyasn1-modules (0.2.1-0.2build1) ... 2023-06-03T19:21:54.643 INFO:teuthology.orchestra.run.smithi089.stdout:Setting up python3-requests-oauthlib (1.0.0-1.1build1) ... 2023-06-03T19:21:54.932 INFO:teuthology.orchestra.run.smithi164.stdout:Setting up python3-sklearn (0.22.2.post1+dfsg-5) ... 2023-06-03T19:21:55.145 INFO:teuthology.orchestra.run.smithi089.stdout:Setting up python3-pastescript (2.0.2-4) ... 2023-06-03T19:21:55.488 INFO:teuthology.orchestra.run.smithi089.stdout:Setting up python3-pecan (1.3.3-3ubuntu1) ... 2023-06-03T19:21:55.848 INFO:teuthology.orchestra.run.smithi089.stdout:Setting up python3-rados (16.2.13-257-gd8c5d349-1focal) ... 2023-06-03T19:21:55.974 INFO:teuthology.orchestra.run.smithi089.stdout:Setting up python3-rbd (16.2.13-257-gd8c5d349-1focal) ... 2023-06-03T19:21:56.020 INFO:teuthology.orchestra.run.smithi164.stdout:Setting up librbd1 (16.2.13-257-gd8c5d349-1focal) ... 2023-06-03T19:21:56.109 INFO:teuthology.orchestra.run.smithi089.stdout:Setting up rbd-fuse (16.2.13-257-gd8c5d349-1focal) ... 2023-06-03T19:21:56.146 INFO:teuthology.orchestra.run.smithi164.stdout:Setting up libjs-sphinxdoc (1.8.5-7ubuntu3) ... 2023-06-03T19:21:56.235 INFO:teuthology.orchestra.run.smithi089.stdout:Setting up python3-routes (2.4.1-2) ... 2023-06-03T19:21:56.272 INFO:teuthology.orchestra.run.smithi164.stdout:Setting up jq (1.6-1ubuntu0.20.04.1) ... 2023-06-03T19:21:56.399 INFO:teuthology.orchestra.run.smithi164.stdout:Setting up python3-rsa (4.0-3) ... 2023-06-03T19:21:56.536 INFO:teuthology.orchestra.run.smithi089.stdout:Setting up python3-google-auth (1.5.1-2) ... 2023-06-03T19:21:56.691 INFO:teuthology.orchestra.run.smithi164.stdout:Setting up python3-webtest (2.0.32-2) ... 2023-06-03T19:21:56.931 INFO:teuthology.orchestra.run.smithi089.stdout:Setting up python3-rgw (16.2.13-257-gd8c5d349-1focal) ... 2023-06-03T19:21:56.984 INFO:teuthology.orchestra.run.smithi164.stdout:Setting up ceph-mgr-modules-core (16.2.13-257-gd8c5d349-1focal) ... 2023-06-03T19:21:57.067 INFO:teuthology.orchestra.run.smithi089.stdout:Setting up python3-cephfs (16.2.13-257-gd8c5d349-1focal) ... 2023-06-03T19:21:57.111 INFO:teuthology.orchestra.run.smithi164.stdout:Setting up ceph-fuse (16.2.13-257-gd8c5d349-1focal) ... 2023-06-03T19:21:57.334 INFO:teuthology.orchestra.run.smithi164.stdout:Created symlink /etc/systemd/system/remote-fs.target.wants/ceph-fuse.target -> /lib/systemd/system/ceph-fuse.target. 2023-06-03T19:21:57.334 INFO:teuthology.orchestra.run.smithi164.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-fuse.target -> /lib/systemd/system/ceph-fuse.target. 2023-06-03T19:21:57.352 INFO:teuthology.orchestra.run.smithi089.stdout:Setting up python3-kubernetes (7.0.0~a1-2) ... 2023-06-03T19:21:57.786 INFO:teuthology.orchestra.run.smithi164.stdout:Setting up libcephfs-dev (16.2.13-257-gd8c5d349-1focal) ... 2023-06-03T19:21:57.914 INFO:teuthology.orchestra.run.smithi164.stdout:Setting up python3-requests-oauthlib (1.0.0-1.1build1) ... 2023-06-03T19:21:58.207 INFO:teuthology.orchestra.run.smithi164.stdout:Setting up python3-pastescript (2.0.2-4) ... 2023-06-03T19:21:58.297 INFO:teuthology.orchestra.run.smithi089.stdout:Setting up ceph-common (16.2.13-257-gd8c5d349-1focal) ... 2023-06-03T19:21:58.525 INFO:teuthology.orchestra.run.smithi164.stdout:Setting up python3-pecan (1.3.3-3ubuntu1) ... 2023-06-03T19:21:58.826 INFO:teuthology.orchestra.run.smithi089.stdout:Adding group ceph....done 2023-06-03T19:21:58.902 INFO:teuthology.orchestra.run.smithi164.stdout:Setting up python3-rados (16.2.13-257-gd8c5d349-1focal) ... 2023-06-03T19:21:59.029 INFO:teuthology.orchestra.run.smithi164.stdout:Setting up python3-rbd (16.2.13-257-gd8c5d349-1focal) ... 2023-06-03T19:21:59.180 INFO:teuthology.orchestra.run.smithi164.stdout:Setting up rbd-fuse (16.2.13-257-gd8c5d349-1focal) ... 2023-06-03T19:21:59.228 INFO:teuthology.orchestra.run.smithi089.stdout:Adding system user ceph....done 2023-06-03T19:21:59.306 INFO:teuthology.orchestra.run.smithi164.stdout:Setting up python3-routes (2.4.1-2) ... 2023-06-03T19:21:59.404 INFO:teuthology.orchestra.run.smithi089.stdout:Setting system user ceph properties....done 2023-06-03T19:21:59.424 INFO:teuthology.orchestra.run.smithi089.stdout:chown: cannot access '/var/log/ceph/*.log*': No such file or directory 2023-06-03T19:21:59.563 INFO:teuthology.orchestra.run.smithi089.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph.target -> /lib/systemd/system/ceph.target. 2023-06-03T19:21:59.599 INFO:teuthology.orchestra.run.smithi164.stdout:Setting up python3-google-auth (1.5.1-2) ... 2023-06-03T19:21:59.883 INFO:teuthology.orchestra.run.smithi089.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/rbdmap.service -> /lib/systemd/system/rbdmap.service. 2023-06-03T19:21:59.977 INFO:teuthology.orchestra.run.smithi164.stdout:Setting up python3-rgw (16.2.13-257-gd8c5d349-1focal) ... 2023-06-03T19:22:00.104 INFO:teuthology.orchestra.run.smithi164.stdout:Setting up python3-cephfs (16.2.13-257-gd8c5d349-1focal) ... 2023-06-03T19:22:00.384 INFO:teuthology.orchestra.run.smithi089.stdout:Setting up ceph-test (16.2.13-257-gd8c5d349-1focal) ... 2023-06-03T19:22:00.405 INFO:teuthology.orchestra.run.smithi164.stdout:Setting up python3-kubernetes (7.0.0~a1-2) ... 2023-06-03T19:22:00.510 INFO:teuthology.orchestra.run.smithi089.stdout:Setting up radosgw (16.2.13-257-gd8c5d349-1focal) ... 2023-06-03T19:22:00.777 INFO:teuthology.orchestra.run.smithi089.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-radosgw.target -> /lib/systemd/system/ceph-radosgw.target. 2023-06-03T19:22:00.777 INFO:teuthology.orchestra.run.smithi089.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-radosgw.target -> /lib/systemd/system/ceph-radosgw.target. 2023-06-03T19:22:01.351 INFO:teuthology.orchestra.run.smithi164.stdout:Setting up ceph-common (16.2.13-257-gd8c5d349-1focal) ... 2023-06-03T19:22:01.445 INFO:teuthology.orchestra.run.smithi089.stdout:Setting up ceph-base (16.2.13-257-gd8c5d349-1focal) ... 2023-06-03T19:22:01.858 INFO:teuthology.orchestra.run.smithi089.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-crash.service -> /lib/systemd/system/ceph-crash.service. 2023-06-03T19:22:01.886 INFO:teuthology.orchestra.run.smithi164.stdout:Adding group ceph....done 2023-06-03T19:22:02.291 INFO:teuthology.orchestra.run.smithi164.stdout:Adding system user ceph....done 2023-06-03T19:22:02.327 INFO:teuthology.orchestra.run.smithi089.stdout:Setting up ceph-mds (16.2.13-257-gd8c5d349-1focal) ... 2023-06-03T19:22:02.477 INFO:teuthology.orchestra.run.smithi164.stdout:Setting system user ceph properties....done 2023-06-03T19:22:02.501 INFO:teuthology.orchestra.run.smithi164.stdout:chown: cannot access '/var/log/ceph/*.log*': No such file or directory 2023-06-03T19:22:02.548 INFO:teuthology.orchestra.run.smithi089.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mds.target -> /lib/systemd/system/ceph-mds.target. 2023-06-03T19:22:02.548 INFO:teuthology.orchestra.run.smithi089.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mds.target -> /lib/systemd/system/ceph-mds.target. 2023-06-03T19:22:02.647 INFO:teuthology.orchestra.run.smithi164.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph.target -> /lib/systemd/system/ceph.target. 2023-06-03T19:22:02.971 INFO:teuthology.orchestra.run.smithi164.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/rbdmap.service -> /lib/systemd/system/rbdmap.service. 2023-06-03T19:22:02.995 INFO:teuthology.orchestra.run.smithi089.stdout:Setting up ceph-mgr (16.2.13-257-gd8c5d349-1focal) ... 2023-06-03T19:22:03.226 INFO:teuthology.orchestra.run.smithi089.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mgr.target -> /lib/systemd/system/ceph-mgr.target. 2023-06-03T19:22:03.227 INFO:teuthology.orchestra.run.smithi089.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mgr.target -> /lib/systemd/system/ceph-mgr.target. 2023-06-03T19:22:03.431 INFO:teuthology.orchestra.run.smithi164.stdout:Setting up ceph-test (16.2.13-257-gd8c5d349-1focal) ... 2023-06-03T19:22:03.557 INFO:teuthology.orchestra.run.smithi164.stdout:Setting up radosgw (16.2.13-257-gd8c5d349-1focal) ... 2023-06-03T19:22:03.696 INFO:teuthology.orchestra.run.smithi089.stdout:Setting up ceph-osd (16.2.13-257-gd8c5d349-1focal) ... 2023-06-03T19:22:03.827 INFO:teuthology.orchestra.run.smithi164.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-radosgw.target -> /lib/systemd/system/ceph-radosgw.target. 2023-06-03T19:22:03.827 INFO:teuthology.orchestra.run.smithi164.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-radosgw.target -> /lib/systemd/system/ceph-radosgw.target. 2023-06-03T19:22:04.196 INFO:teuthology.orchestra.run.smithi089.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-osd.target -> /lib/systemd/system/ceph-osd.target. 2023-06-03T19:22:04.196 INFO:teuthology.orchestra.run.smithi089.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-osd.target -> /lib/systemd/system/ceph-osd.target. 2023-06-03T19:22:04.493 INFO:teuthology.orchestra.run.smithi164.stdout:Setting up ceph-base (16.2.13-257-gd8c5d349-1focal) ... 2023-06-03T19:22:04.639 INFO:teuthology.orchestra.run.smithi089.stdout:Setting up ceph-mgr-k8sevents (16.2.13-257-gd8c5d349-1focal) ... 2023-06-03T19:22:04.765 INFO:teuthology.orchestra.run.smithi089.stdout:Setting up ceph-mgr-diskprediction-local (16.2.13-257-gd8c5d349-1focal) ... 2023-06-03T19:22:04.862 INFO:teuthology.orchestra.run.smithi164.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-crash.service -> /lib/systemd/system/ceph-crash.service. 2023-06-03T19:22:04.908 INFO:teuthology.orchestra.run.smithi089.stdout:Setting up ceph-mon (16.2.13-257-gd8c5d349-1focal) ... 2023-06-03T19:22:05.141 INFO:teuthology.orchestra.run.smithi089.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mon.target -> /lib/systemd/system/ceph-mon.target. 2023-06-03T19:22:05.141 INFO:teuthology.orchestra.run.smithi089.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mon.target -> /lib/systemd/system/ceph-mon.target. 2023-06-03T19:22:05.337 INFO:teuthology.orchestra.run.smithi164.stdout:Setting up ceph-mds (16.2.13-257-gd8c5d349-1focal) ... 2023-06-03T19:22:05.558 INFO:teuthology.orchestra.run.smithi164.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mds.target -> /lib/systemd/system/ceph-mds.target. 2023-06-03T19:22:05.559 INFO:teuthology.orchestra.run.smithi164.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mds.target -> /lib/systemd/system/ceph-mds.target. 2023-06-03T19:22:05.610 INFO:teuthology.orchestra.run.smithi089.stdout:Setting up ceph-mgr-cephadm (16.2.13-257-gd8c5d349-1focal) ... 2023-06-03T19:22:05.753 INFO:teuthology.orchestra.run.smithi089.stdout:Setting up ceph (16.2.13-257-gd8c5d349-1focal) ... 2023-06-03T19:22:05.879 INFO:teuthology.orchestra.run.smithi089.stdout:Setting up ceph-mgr-dashboard (16.2.13-257-gd8c5d349-1focal) ... 2023-06-03T19:22:06.013 INFO:teuthology.orchestra.run.smithi164.stdout:Setting up ceph-mgr (16.2.13-257-gd8c5d349-1focal) ... 2023-06-03T19:22:06.022 INFO:teuthology.orchestra.run.smithi089.stdout:Processing triggers for systemd (245.4-4ubuntu3.17) ... 2023-06-03T19:22:06.238 INFO:teuthology.orchestra.run.smithi164.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mgr.target -> /lib/systemd/system/ceph-mgr.target. 2023-06-03T19:22:06.238 INFO:teuthology.orchestra.run.smithi164.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mgr.target -> /lib/systemd/system/ceph-mgr.target. 2023-06-03T19:22:06.306 INFO:teuthology.orchestra.run.smithi089.stdout:Processing triggers for man-db (2.9.1-1) ... 2023-06-03T19:22:06.689 INFO:teuthology.orchestra.run.smithi164.stdout:Setting up ceph-osd (16.2.13-257-gd8c5d349-1focal) ... 2023-06-03T19:22:07.198 INFO:teuthology.orchestra.run.smithi164.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-osd.target -> /lib/systemd/system/ceph-osd.target. 2023-06-03T19:22:07.198 INFO:teuthology.orchestra.run.smithi164.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-osd.target -> /lib/systemd/system/ceph-osd.target. 2023-06-03T19:22:07.658 INFO:teuthology.orchestra.run.smithi164.stdout:Setting up ceph-mgr-k8sevents (16.2.13-257-gd8c5d349-1focal) ... 2023-06-03T19:22:07.784 INFO:teuthology.orchestra.run.smithi164.stdout:Setting up ceph-mgr-diskprediction-local (16.2.13-257-gd8c5d349-1focal) ... 2023-06-03T19:22:07.927 INFO:teuthology.orchestra.run.smithi164.stdout:Setting up ceph-mon (16.2.13-257-gd8c5d349-1focal) ... 2023-06-03T19:22:08.153 INFO:teuthology.orchestra.run.smithi164.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mon.target -> /lib/systemd/system/ceph-mon.target. 2023-06-03T19:22:08.153 INFO:teuthology.orchestra.run.smithi164.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mon.target -> /lib/systemd/system/ceph-mon.target. 2023-06-03T19:22:08.620 INFO:teuthology.orchestra.run.smithi164.stdout:Setting up ceph-mgr-cephadm (16.2.13-257-gd8c5d349-1focal) ... 2023-06-03T19:22:08.762 INFO:teuthology.orchestra.run.smithi164.stdout:Setting up ceph (16.2.13-257-gd8c5d349-1focal) ... 2023-06-03T19:22:08.914 INFO:teuthology.orchestra.run.smithi164.stdout:Setting up ceph-mgr-dashboard (16.2.13-257-gd8c5d349-1focal) ... 2023-06-03T19:22:09.053 INFO:teuthology.orchestra.run.smithi164.stdout:Processing triggers for systemd (245.4-4ubuntu3.17) ... 2023-06-03T19:22:09.253 INFO:teuthology.orchestra.run.smithi089.stdout:Processing triggers for libc-bin (2.31-0ubuntu9.9) ... 2023-06-03T19:22:09.354 INFO:teuthology.orchestra.run.smithi164.stdout:Processing triggers for man-db (2.9.1-1) ... 2023-06-03T19:22:09.701 INFO:teuthology.orchestra.run.smithi089.stderr:W: --force-yes is deprecated, use one of the options starting with --allow instead. 2023-06-03T19:22:09.703 DEBUG:teuthology.parallel:result is None 2023-06-03T19:22:12.266 INFO:teuthology.orchestra.run.smithi164.stdout:Processing triggers for libc-bin (2.31-0ubuntu9.9) ... 2023-06-03T19:22:12.726 INFO:teuthology.orchestra.run.smithi164.stderr:W: --force-yes is deprecated, use one of the options starting with --allow instead. 2023-06-03T19:22:12.729 DEBUG:teuthology.parallel:result is None 2023-06-03T19:22:12.729 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2023-06-03T19:22:12.730 INFO:teuthology.packaging:ref: None 2023-06-03T19:22:12.730 INFO:teuthology.packaging:tag: None 2023-06-03T19:22:12.730 INFO:teuthology.packaging:branch: pacific 2023-06-03T19:22:12.730 INFO:teuthology.packaging:sha1: 8e24bcf462a1d94c2e12aa4e26419634883a1e88 2023-06-03T19:22:12.731 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=ubuntu%2F20.04%2Fx86_64&ref=pacific 2023-06-03T19:22:13.035 DEBUG:teuthology.orchestra.run.smithi089:> dpkg-query -W -f '${Version}' ceph 2023-06-03T19:22:13.061 INFO:teuthology.orchestra.run.smithi089.stdout:16.2.13-257-gd8c5d349-1focal 2023-06-03T19:22:13.061 INFO:teuthology.packaging:The installed version of ceph is 16.2.13-257-gd8c5d349-1focal 2023-06-03T19:22:13.062 INFO:teuthology.task.install:The correct ceph version 16.2.13-257-gd8c5d349-1focal is installed. 2023-06-03T19:22:13.062 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2023-06-03T19:22:13.062 INFO:teuthology.packaging:ref: None 2023-06-03T19:22:13.063 INFO:teuthology.packaging:tag: None 2023-06-03T19:22:13.063 INFO:teuthology.packaging:branch: pacific 2023-06-03T19:22:13.063 INFO:teuthology.packaging:sha1: 8e24bcf462a1d94c2e12aa4e26419634883a1e88 2023-06-03T19:22:13.063 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=ubuntu%2F20.04%2Fx86_64&ref=pacific 2023-06-03T19:22:13.381 DEBUG:teuthology.orchestra.run.smithi164:> dpkg-query -W -f '${Version}' ceph 2023-06-03T19:22:13.405 INFO:teuthology.orchestra.run.smithi164.stdout:16.2.13-257-gd8c5d349-1focal 2023-06-03T19:22:13.405 INFO:teuthology.packaging:The installed version of ceph is 16.2.13-257-gd8c5d349-1focal 2023-06-03T19:22:13.405 INFO:teuthology.task.install:The correct ceph version 16.2.13-257-gd8c5d349-1focal is installed. 2023-06-03T19:22:13.406 INFO:teuthology.task.install.util:Shipping valgrind.supp... 2023-06-03T19:22:13.407 DEBUG:teuthology.orchestra.run.smithi089:> set -ex 2023-06-03T19:22:13.407 DEBUG:teuthology.orchestra.run.smithi089:> sudo dd of=/home/ubuntu/cephtest/valgrind.supp 2023-06-03T19:22:13.422 DEBUG:teuthology.orchestra.run.smithi164:> set -ex 2023-06-03T19:22:13.423 DEBUG:teuthology.orchestra.run.smithi164:> sudo dd of=/home/ubuntu/cephtest/valgrind.supp 2023-06-03T19:22:13.462 INFO:teuthology.task.install.util:Shipping 'daemon-helper'... 2023-06-03T19:22:13.462 DEBUG:teuthology.orchestra.run.smithi089:> set -ex 2023-06-03T19:22:13.463 DEBUG:teuthology.orchestra.run.smithi089:> sudo dd of=/usr/bin/daemon-helper 2023-06-03T19:22:13.480 DEBUG:teuthology.orchestra.run.smithi089:> sudo chmod a=rx -- /usr/bin/daemon-helper 2023-06-03T19:22:13.536 DEBUG:teuthology.orchestra.run.smithi164:> set -ex 2023-06-03T19:22:13.536 DEBUG:teuthology.orchestra.run.smithi164:> sudo dd of=/usr/bin/daemon-helper 2023-06-03T19:22:13.553 DEBUG:teuthology.orchestra.run.smithi164:> sudo chmod a=rx -- /usr/bin/daemon-helper 2023-06-03T19:22:13.610 INFO:teuthology.task.install.util:Shipping 'adjust-ulimits'... 2023-06-03T19:22:13.611 DEBUG:teuthology.orchestra.run.smithi089:> set -ex 2023-06-03T19:22:13.613 DEBUG:teuthology.orchestra.run.smithi089:> sudo dd of=/usr/bin/adjust-ulimits 2023-06-03T19:22:13.632 DEBUG:teuthology.orchestra.run.smithi089:> sudo chmod a=rx -- /usr/bin/adjust-ulimits 2023-06-03T19:22:13.689 DEBUG:teuthology.orchestra.run.smithi164:> set -ex 2023-06-03T19:22:13.689 DEBUG:teuthology.orchestra.run.smithi164:> sudo dd of=/usr/bin/adjust-ulimits 2023-06-03T19:22:13.704 DEBUG:teuthology.orchestra.run.smithi164:> sudo chmod a=rx -- /usr/bin/adjust-ulimits 2023-06-03T19:22:13.761 INFO:teuthology.run_tasks:Running task cephadm... 2023-06-03T19:22:13.868 INFO:tasks.cephadm:Config: {'cephadm_branch': 'pacific', 'cephadm_git_url': 'https://github.com/ceph/ceph', 'conf': {'osd': {'osd_class_default_list': '*', 'osd_class_load_list': '*', 'debug ms': 1, 'debug osd': 20, 'osd shutdown pgref assert': True}, 'global': {'mon election default strategy': 1}, 'mgr': {'debug mgr': 20, 'debug ms': 1}, 'mon': {'debug mon': 20, 'debug ms': 1, 'debug paxos': 20}}, 'image': 'quay.ceph.io/ceph-ci/ceph:pacific', 'create_rbd_pool': True, 'flavor': 'default', 'log-ignorelist': ['\\(MDS_ALL_DOWN\\)', '\\(MDS_UP_LESS_THAN_MAX\\)'], 'sha1': '8e24bcf462a1d94c2e12aa4e26419634883a1e88'} 2023-06-03T19:22:13.869 INFO:tasks.cephadm:Cluster image is quay.ceph.io/ceph-ci/ceph:pacific 2023-06-03T19:22:13.880 INFO:tasks.cephadm:Cluster fsid is f1db3478-0243-11ee-9b1d-001a4aab830c 2023-06-03T19:22:13.881 INFO:tasks.cephadm:Choosing monitor IPs and ports... 2023-06-03T19:22:13.881 INFO:tasks.cephadm:Monitor IPs: {'mon.a': '172.21.15.89', 'mon.c': '[v2:172.21.15.89:3301,v1:172.21.15.89:6790]', 'mon.b': '172.21.15.164'} 2023-06-03T19:22:13.882 INFO:tasks.cephadm:First mon is mon.a on smithi089 2023-06-03T19:22:13.882 INFO:tasks.cephadm:First mgr is y 2023-06-03T19:22:13.882 INFO:tasks.cephadm:Normalizing hostnames... 2023-06-03T19:22:13.882 DEBUG:teuthology.orchestra.run.smithi089:> sudo hostname $(hostname -s) 2023-06-03T19:22:13.900 DEBUG:teuthology.orchestra.run.smithi164:> sudo hostname $(hostname -s) 2023-06-03T19:22:13.915 INFO:tasks.cephadm:Downloading cephadm (repo https://github.com/ceph/ceph ref pacific)... 2023-06-03T19:22:13.916 DEBUG:teuthology.orchestra.run.smithi089:> curl --silent https://raw.githubusercontent.com/ceph/ceph/pacific/src/cephadm/cephadm > /home/ubuntu/cephtest/cephadm && ls -l /home/ubuntu/cephtest/cephadm 2023-06-03T19:22:14.213 INFO:teuthology.orchestra.run.smithi089.stdout:-rw-rw-r-- 1 ubuntu ubuntu 350873 Jun 3 19:22 /home/ubuntu/cephtest/cephadm 2023-06-03T19:22:14.213 DEBUG:teuthology.orchestra.run.smithi164:> curl --silent https://raw.githubusercontent.com/ceph/ceph/pacific/src/cephadm/cephadm > /home/ubuntu/cephtest/cephadm && ls -l /home/ubuntu/cephtest/cephadm 2023-06-03T19:22:14.478 INFO:teuthology.orchestra.run.smithi164.stdout:-rw-rw-r-- 1 ubuntu ubuntu 350873 Jun 3 19:22 /home/ubuntu/cephtest/cephadm 2023-06-03T19:22:14.478 DEBUG:teuthology.orchestra.run.smithi089:> test -s /home/ubuntu/cephtest/cephadm && test $(stat -c%s /home/ubuntu/cephtest/cephadm) -gt 1000 && chmod +x /home/ubuntu/cephtest/cephadm 2023-06-03T19:22:14.489 DEBUG:teuthology.orchestra.run.smithi164:> test -s /home/ubuntu/cephtest/cephadm && test $(stat -c%s /home/ubuntu/cephtest/cephadm) -gt 1000 && chmod +x /home/ubuntu/cephtest/cephadm 2023-06-03T19:22:14.532 INFO:tasks.cephadm:Pulling image quay.ceph.io/ceph-ci/ceph:pacific on all hosts... 2023-06-03T19:22:14.533 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific pull 2023-06-03T19:22:14.535 DEBUG:teuthology.orchestra.run.smithi164:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific pull 2023-06-03T19:22:14.808 INFO:teuthology.orchestra.run.smithi164.stderr:Pulling container image quay.ceph.io/ceph-ci/ceph:pacific... 2023-06-03T19:22:14.814 INFO:teuthology.orchestra.run.smithi089.stderr:Pulling container image quay.ceph.io/ceph-ci/ceph:pacific... 2023-06-03T19:22:38.484 INFO:teuthology.orchestra.run.smithi164.stdout:{ 2023-06-03T19:22:38.485 INFO:teuthology.orchestra.run.smithi164.stdout: "ceph_version": "ceph version 16.2.13-257-gd8c5d349 (d8c5d34975dce1c5eb0aa3a7979a4d9b9a99d1ec) pacific (stable)", 2023-06-03T19:22:38.485 INFO:teuthology.orchestra.run.smithi164.stdout: "image_id": "b644dd95899d36b95338c4867afb8766daf2db3e405d810db12f57a4acfe053c", 2023-06-03T19:22:38.485 INFO:teuthology.orchestra.run.smithi164.stdout: "repo_digests": [ 2023-06-03T19:22:38.485 INFO:teuthology.orchestra.run.smithi164.stdout: "quay.ceph.io/ceph-ci/ceph@sha256:b41db2dcff2a7a7fc551b7b1b28c0ae195aaddc42f719f7bcd571090f6080399" 2023-06-03T19:22:38.485 INFO:teuthology.orchestra.run.smithi164.stdout: ] 2023-06-03T19:22:38.485 INFO:teuthology.orchestra.run.smithi164.stdout:} 2023-06-03T19:22:39.803 INFO:teuthology.orchestra.run.smithi089.stdout:{ 2023-06-03T19:22:39.803 INFO:teuthology.orchestra.run.smithi089.stdout: "ceph_version": "ceph version 16.2.13-257-gd8c5d349 (d8c5d34975dce1c5eb0aa3a7979a4d9b9a99d1ec) pacific (stable)", 2023-06-03T19:22:39.803 INFO:teuthology.orchestra.run.smithi089.stdout: "image_id": "b644dd95899d36b95338c4867afb8766daf2db3e405d810db12f57a4acfe053c", 2023-06-03T19:22:39.804 INFO:teuthology.orchestra.run.smithi089.stdout: "repo_digests": [ 2023-06-03T19:22:39.804 INFO:teuthology.orchestra.run.smithi089.stdout: "quay.ceph.io/ceph-ci/ceph@sha256:b41db2dcff2a7a7fc551b7b1b28c0ae195aaddc42f719f7bcd571090f6080399" 2023-06-03T19:22:39.804 INFO:teuthology.orchestra.run.smithi089.stdout: ] 2023-06-03T19:22:39.804 INFO:teuthology.orchestra.run.smithi089.stdout:} 2023-06-03T19:22:39.826 DEBUG:teuthology.orchestra.run.smithi089:> sudo mkdir -p /etc/ceph 2023-06-03T19:22:39.843 DEBUG:teuthology.orchestra.run.smithi164:> sudo mkdir -p /etc/ceph 2023-06-03T19:22:39.857 DEBUG:teuthology.orchestra.run.smithi089:> sudo chmod 777 /etc/ceph 2023-06-03T19:22:39.900 DEBUG:teuthology.orchestra.run.smithi164:> sudo chmod 777 /etc/ceph 2023-06-03T19:22:39.915 INFO:tasks.cephadm:Writing seed config... 2023-06-03T19:22:39.917 INFO:tasks.cephadm: override: [osd] osd_class_default_list = * 2023-06-03T19:22:39.917 INFO:tasks.cephadm: override: [osd] osd_class_load_list = * 2023-06-03T19:22:39.917 INFO:tasks.cephadm: override: [osd] debug ms = 1 2023-06-03T19:22:39.917 INFO:tasks.cephadm: override: [osd] debug osd = 20 2023-06-03T19:22:39.918 INFO:tasks.cephadm: override: [osd] osd shutdown pgref assert = True 2023-06-03T19:22:39.918 INFO:tasks.cephadm: override: [global] mon election default strategy = 1 2023-06-03T19:22:39.918 INFO:tasks.cephadm: override: [mgr] debug mgr = 20 2023-06-03T19:22:39.918 INFO:tasks.cephadm: override: [mgr] debug ms = 1 2023-06-03T19:22:39.918 INFO:tasks.cephadm: override: [mon] debug mon = 20 2023-06-03T19:22:39.918 INFO:tasks.cephadm: override: [mon] debug ms = 1 2023-06-03T19:22:39.919 INFO:tasks.cephadm: override: [mon] debug paxos = 20 2023-06-03T19:22:39.919 DEBUG:teuthology.orchestra.run.smithi089:> set -ex 2023-06-03T19:22:39.920 DEBUG:teuthology.orchestra.run.smithi089:> dd of=/home/ubuntu/cephtest/seed.ceph.conf 2023-06-03T19:22:39.951 DEBUG:tasks.cephadm:Final config: [global] # make logging friendly to teuthology log_to_file = true log_to_stderr = false log to journald = false mon cluster log file level = debug mon clock drift allowed = 1.000 # replicate across OSDs, not hosts osd crush chooseleaf type = 0 #osd pool default size = 2 osd pool default erasure code profile = plugin=jerasure technique=reed_sol_van k=2 m=1 crush-failure-domain=osd # enable some debugging auth debug = true ms die on old message = true ms die on bug = true debug asserts on shutdown = true # adjust warnings mon max pg per osd = 10000# >= luminous mon pg warn max object skew = 0 mon osd allow primary affinity = true mon osd allow pg remap = true mon warn on legacy crush tunables = false mon warn on crush straw calc version zero = false mon warn on no sortbitwise = false mon warn on osd down out interval zero = false mon warn on too few osds = false mon_warn_on_pool_pg_num_not_power_of_two = false # disable pg_autoscaler by default for new pools osd_pool_default_pg_autoscale_mode = off # tests delete pools mon allow pool delete = true fsid = f1db3478-0243-11ee-9b1d-001a4aab830c mon election default strategy = 1 [osd] osd scrub load threshold = 5.0 osd scrub max interval = 600 osd mclock profile = high_recovery_ops osd recover clone overlap = true osd recovery max chunk = 1048576 osd deep scrub update digest min age = 30 osd map max advance = 10 osd memory target autotune = true # debugging osd debug shutdown = true osd debug op order = true osd debug verify stray on activate = true osd debug pg log writeout = true osd debug verify cached snaps = true osd debug verify missing on start = true osd debug misdirected ops = true osd op queue = debug_random osd op queue cut off = debug_random osd shutdown pgref assert = True bdev debug aio = true osd sloppy crc = true osd_class_default_list = * osd_class_load_list = * debug ms = 1 debug osd = 20 [mgr] mon reweight min pgs per osd = 4 mon reweight min bytes per osd = 10 mgr/telemetry/nag = false debug mgr = 20 debug ms = 1 [mon] mon data avail warn = 5 mon mgr mkfs grace = 240 mon reweight min pgs per osd = 4 mon osd reporter subtree level = osd mon osd prime pg temp = true mon reweight min bytes per osd = 10 # rotate auth tickets quickly to exercise renewal paths auth mon ticket ttl = 660# 11m auth service ticket ttl = 240# 4m # don't complain about global id reclaim mon_warn_on_insecure_global_id_reclaim = false mon_warn_on_insecure_global_id_reclaim_allowed = false debug mon = 20 debug ms = 1 debug paxos = 20 [client.rgw] rgw cache enabled = true rgw enable ops log = true rgw enable usage log = true 2023-06-03T19:22:39.951 DEBUG:teuthology.orchestra.run.smithi089:mon.a> sudo journalctl -f -n 0 -u ceph-f1db3478-0243-11ee-9b1d-001a4aab830c@mon.a.service 2023-06-03T19:22:39.994 DEBUG:teuthology.orchestra.run.smithi089:mgr.y> sudo journalctl -f -n 0 -u ceph-f1db3478-0243-11ee-9b1d-001a4aab830c@mgr.y.service 2023-06-03T19:22:40.042 INFO:tasks.cephadm:Bootstrapping... 2023-06-03T19:22:40.042 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific -v bootstrap --fsid f1db3478-0243-11ee-9b1d-001a4aab830c --config /home/ubuntu/cephtest/seed.ceph.conf --output-config /etc/ceph/ceph.conf --output-keyring /etc/ceph/ceph.client.admin.keyring --output-pub-ssh-key /home/ubuntu/cephtest/ceph.pub --mon-id a --mgr-id y --orphan-initial-daemons --skip-monitoring-stack --mon-ip 172.21.15.89 --skip-admin-label && sudo chmod +r /etc/ceph/ceph.client.admin.keyring 2023-06-03T19:22:40.251 INFO:teuthology.orchestra.run.smithi089.stderr:-------------------------------------------------------------------------------- 2023-06-03T19:22:40.251 INFO:teuthology.orchestra.run.smithi089.stderr:cephadm ['--image', 'quay.ceph.io/ceph-ci/ceph:pacific', '-v', 'bootstrap', '--fsid', 'f1db3478-0243-11ee-9b1d-001a4aab830c', '--config', '/home/ubuntu/cephtest/seed.ceph.conf', '--output-config', '/etc/ceph/ceph.conf', '--output-keyring', '/etc/ceph/ceph.client.admin.keyring', '--output-pub-ssh-key', '/home/ubuntu/cephtest/ceph.pub', '--mon-id', 'a', '--mgr-id', 'y', '--orphan-initial-daemons', '--skip-monitoring-stack', '--mon-ip', '172.21.15.89', '--skip-admin-label'] 2023-06-03T19:22:40.251 INFO:teuthology.orchestra.run.smithi089.stderr:Verifying podman|docker is present... 2023-06-03T19:22:40.251 INFO:teuthology.orchestra.run.smithi089.stderr:Verifying lvm2 is present... 2023-06-03T19:22:40.251 INFO:teuthology.orchestra.run.smithi089.stderr:Verifying time synchronization is in place... 2023-06-03T19:22:40.258 INFO:teuthology.orchestra.run.smithi089.stderr:systemctl: Failed to get unit file state for chrony.service: No such file or directory 2023-06-03T19:22:40.258 INFO:journalctl@ceph.mgr.y.smithi089.stdout:-- Logs begin at Sun 2022-08-14 12:20:24 UTC. -- 2023-06-03T19:22:40.264 INFO:teuthology.orchestra.run.smithi089.stderr:systemctl: inactive 2023-06-03T19:22:40.265 INFO:journalctl@ceph.mon.a.smithi089.stdout:-- Logs begin at Sun 2022-08-14 12:20:24 UTC. -- 2023-06-03T19:22:40.270 INFO:teuthology.orchestra.run.smithi089.stderr:systemctl: Failed to get unit file state for chronyd.service: No such file or directory 2023-06-03T19:22:40.277 INFO:teuthology.orchestra.run.smithi089.stderr:systemctl: inactive 2023-06-03T19:22:40.284 INFO:teuthology.orchestra.run.smithi089.stderr:systemctl: masked 2023-06-03T19:22:40.291 INFO:teuthology.orchestra.run.smithi089.stderr:systemctl: inactive 2023-06-03T19:22:40.297 INFO:teuthology.orchestra.run.smithi089.stderr:systemctl: Failed to get unit file state for ntpd.service: No such file or directory 2023-06-03T19:22:40.303 INFO:teuthology.orchestra.run.smithi089.stderr:systemctl: inactive 2023-06-03T19:22:40.312 INFO:teuthology.orchestra.run.smithi089.stderr:systemctl: enabled 2023-06-03T19:22:40.318 INFO:teuthology.orchestra.run.smithi089.stderr:systemctl: active 2023-06-03T19:22:40.319 INFO:teuthology.orchestra.run.smithi089.stderr:Unit ntp.service is enabled and running 2023-06-03T19:22:40.319 INFO:teuthology.orchestra.run.smithi089.stderr:Repeating the final host check... 2023-06-03T19:22:40.319 INFO:teuthology.orchestra.run.smithi089.stderr:docker (/usr/bin/docker) is present 2023-06-03T19:22:40.319 INFO:teuthology.orchestra.run.smithi089.stderr:systemctl is present 2023-06-03T19:22:40.320 INFO:teuthology.orchestra.run.smithi089.stderr:lvcreate is present 2023-06-03T19:22:40.325 INFO:teuthology.orchestra.run.smithi089.stderr:systemctl: Failed to get unit file state for chrony.service: No such file or directory 2023-06-03T19:22:40.333 INFO:teuthology.orchestra.run.smithi089.stderr:systemctl: inactive 2023-06-03T19:22:40.340 INFO:teuthology.orchestra.run.smithi089.stderr:systemctl: Failed to get unit file state for chronyd.service: No such file or directory 2023-06-03T19:22:40.347 INFO:teuthology.orchestra.run.smithi089.stderr:systemctl: inactive 2023-06-03T19:22:40.353 INFO:teuthology.orchestra.run.smithi089.stderr:systemctl: masked 2023-06-03T19:22:40.359 INFO:teuthology.orchestra.run.smithi089.stderr:systemctl: inactive 2023-06-03T19:22:40.366 INFO:teuthology.orchestra.run.smithi089.stderr:systemctl: Failed to get unit file state for ntpd.service: No such file or directory 2023-06-03T19:22:40.373 INFO:teuthology.orchestra.run.smithi089.stderr:systemctl: inactive 2023-06-03T19:22:40.381 INFO:teuthology.orchestra.run.smithi089.stderr:systemctl: enabled 2023-06-03T19:22:40.388 INFO:teuthology.orchestra.run.smithi089.stderr:systemctl: active 2023-06-03T19:22:40.390 INFO:teuthology.orchestra.run.smithi089.stderr:Unit ntp.service is enabled and running 2023-06-03T19:22:40.390 INFO:teuthology.orchestra.run.smithi089.stderr:Host looks OK 2023-06-03T19:22:40.390 INFO:teuthology.orchestra.run.smithi089.stderr:Cluster fsid: f1db3478-0243-11ee-9b1d-001a4aab830c 2023-06-03T19:22:40.390 INFO:teuthology.orchestra.run.smithi089.stderr:Acquiring lock 140433042571216 on /run/cephadm/f1db3478-0243-11ee-9b1d-001a4aab830c.lock 2023-06-03T19:22:40.391 INFO:teuthology.orchestra.run.smithi089.stderr:Lock 140433042571216 acquired on /run/cephadm/f1db3478-0243-11ee-9b1d-001a4aab830c.lock 2023-06-03T19:22:40.391 INFO:teuthology.orchestra.run.smithi089.stderr:Verifying IP 172.21.15.89 port 3300 ... 2023-06-03T19:22:40.391 INFO:teuthology.orchestra.run.smithi089.stderr:Verifying IP 172.21.15.89 port 6789 ... 2023-06-03T19:22:40.391 INFO:teuthology.orchestra.run.smithi089.stderr:Base mon IP(s) is [172.21.15.89:3300, 172.21.15.89:6789], mon addrv is [v2:172.21.15.89:3300,v1:172.21.15.89:6789] 2023-06-03T19:22:40.396 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/sbin/ip: default via 172.21.15.254 dev enp3s0f1 2023-06-03T19:22:40.396 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/sbin/ip: default via 172.21.15.254 dev enp3s0f1 proto dhcp src 172.21.15.89 metric 100 2023-06-03T19:22:40.397 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/sbin/ip: 172.17.0.0/16 dev docker0 proto kernel scope link src 172.17.0.1 linkdown 2023-06-03T19:22:40.397 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/sbin/ip: 172.21.0.0/20 dev enp3s0f1 proto kernel scope link src 172.21.15.89 2023-06-03T19:22:40.397 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/sbin/ip: 172.21.15.254 dev enp3s0f1 proto dhcp scope link src 172.21.15.89 metric 100 2023-06-03T19:22:40.401 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/sbin/ip: ::1 dev lo proto kernel metric 256 pref medium 2023-06-03T19:22:40.401 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/sbin/ip: fe80::/64 dev enp3s0f1 proto kernel metric 256 pref medium 2023-06-03T19:22:40.402 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/sbin/ip: default via fe80::327c:5e00:6487:71e0 dev enp3s0f1 proto ra metric 100 expires 1758sec pref medium 2023-06-03T19:22:40.406 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/sbin/ip: 1: lo: mtu 65536 state UNKNOWN qlen 1000 2023-06-03T19:22:40.406 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/sbin/ip: inet6 ::1/128 scope host 2023-06-03T19:22:40.406 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/sbin/ip: valid_lft forever preferred_lft forever 2023-06-03T19:22:40.406 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/sbin/ip: 5: enp3s0f1: mtu 1500 state UP qlen 1000 2023-06-03T19:22:40.406 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/sbin/ip: inet6 fe80::ec4:7aff:fe8f:d501/64 scope link 2023-06-03T19:22:40.407 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/sbin/ip: valid_lft forever preferred_lft forever 2023-06-03T19:22:40.408 INFO:teuthology.orchestra.run.smithi089.stderr:Mon IP `172.21.15.89` is in CIDR network `172.21.0.0/20` 2023-06-03T19:22:40.408 INFO:teuthology.orchestra.run.smithi089.stderr:Mon IP `172.21.15.89` is in CIDR network `172.21.0.0/20` 2023-06-03T19:22:40.408 INFO:teuthology.orchestra.run.smithi089.stderr:Mon IP `172.21.15.89` is in CIDR network `172.21.15.254/32` 2023-06-03T19:22:40.408 INFO:teuthology.orchestra.run.smithi089.stderr:Mon IP `172.21.15.89` is in CIDR network `172.21.15.254/32` 2023-06-03T19:22:40.408 INFO:teuthology.orchestra.run.smithi089.stderr:Inferred mon public CIDR from local network configuration ['172.21.0.0/20', '172.21.0.0/20', '172.21.15.254/32', '172.21.15.254/32'] 2023-06-03T19:22:40.409 INFO:teuthology.orchestra.run.smithi089.stderr:Internal network (--cluster-network) has not been provided, OSD replication will default to the public_network 2023-06-03T19:22:40.409 INFO:teuthology.orchestra.run.smithi089.stderr:Pulling container image quay.ceph.io/ceph-ci/ceph:pacific... 2023-06-03T19:22:40.619 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/docker: pacific: Pulling from ceph-ci/ceph 2023-06-03T19:22:40.658 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/docker: Digest: sha256:b41db2dcff2a7a7fc551b7b1b28c0ae195aaddc42f719f7bcd571090f6080399 2023-06-03T19:22:40.658 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/docker: Status: Image is up to date for quay.ceph.io/ceph-ci/ceph:pacific 2023-06-03T19:22:40.691 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/docker: quay.ceph.io/ceph-ci/ceph:pacific 2023-06-03T19:22:41.838 INFO:teuthology.orchestra.run.smithi089.stderr:ceph: ceph version 16.2.13-257-gd8c5d349 (d8c5d34975dce1c5eb0aa3a7979a4d9b9a99d1ec) pacific (stable) 2023-06-03T19:22:42.442 INFO:teuthology.orchestra.run.smithi089.stderr:Ceph version: ceph version 16.2.13-257-gd8c5d349 (d8c5d34975dce1c5eb0aa3a7979a4d9b9a99d1ec) pacific (stable) 2023-06-03T19:22:42.442 INFO:teuthology.orchestra.run.smithi089.stderr:Extracting ceph user uid/gid from container image... 2023-06-03T19:22:43.312 INFO:teuthology.orchestra.run.smithi089.stderr:stat: 167 167 2023-06-03T19:22:43.979 INFO:teuthology.orchestra.run.smithi089.stderr:Creating initial keys... 2023-06-03T19:22:44.877 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph-authtool: AQAEk3tkjuLGMxAAsPNy0lvnm7K0nDTSI4chGg== 2023-06-03T19:22:46.490 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph-authtool: AQAGk3tkQlKuHBAA5ntlkfT+98vnZ+J+TYUCrQ== 2023-06-03T19:22:47.996 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph-authtool: AQAHk3tkdzjUOhAA0XAzghYRqtnG8DZO8xBjhg== 2023-06-03T19:22:48.654 INFO:teuthology.orchestra.run.smithi089.stderr:Creating initial monmap... 2023-06-03T19:22:49.601 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/monmaptool: /usr/bin/monmaptool: monmap file /tmp/monmap 2023-06-03T19:22:49.601 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/monmaptool: /usr/bin/monmaptool: set fsid to f1db3478-0243-11ee-9b1d-001a4aab830c 2023-06-03T19:22:49.602 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/monmaptool: /usr/bin/monmaptool: writing epoch 0 to /tmp/monmap (1 monitors) 2023-06-03T19:22:50.292 INFO:teuthology.orchestra.run.smithi089.stderr:monmaptool for a [v2:172.21.15.89:3300,v1:172.21.15.89:6789] on /usr/bin/monmaptool: monmap file /tmp/monmap 2023-06-03T19:22:50.292 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/monmaptool: set fsid to f1db3478-0243-11ee-9b1d-001a4aab830c 2023-06-03T19:22:50.292 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/monmaptool: writing epoch 0 to /tmp/monmap (1 monitors) 2023-06-03T19:22:50.293 INFO:teuthology.orchestra.run.smithi089.stderr: 2023-06-03T19:22:50.293 INFO:teuthology.orchestra.run.smithi089.stderr:Creating mon... 2023-06-03T19:22:51.245 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph-mon: debug 2023-06-03T19:22:51.230+0000 7fe36b84e880 0 set uid:gid to 167:167 (ceph:ceph) 2023-06-03T19:22:51.245 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph-mon: debug 2023-06-03T19:22:51.230+0000 7fe36b84e880 1 imported monmap: 2023-06-03T19:22:51.245 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph-mon: epoch 0 2023-06-03T19:22:51.245 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph-mon: fsid f1db3478-0243-11ee-9b1d-001a4aab830c 2023-06-03T19:22:51.245 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph-mon: last_changed 2023-06-03T19:22:49.591927+0000 2023-06-03T19:22:51.245 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph-mon: created 2023-06-03T19:22:49.591927+0000 2023-06-03T19:22:51.246 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph-mon: min_mon_release 0 (unknown) 2023-06-03T19:22:51.246 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph-mon: election_strategy: 1 2023-06-03T19:22:51.246 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph-mon: 0: [v2:172.21.15.89:3300/0,v1:172.21.15.89:6789/0] mon.a 2023-06-03T19:22:51.246 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph-mon: 2023-06-03T19:22:51.246 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph-mon: debug 2023-06-03T19:22:51.230+0000 7fe36b84e880 0 /usr/bin/ceph-mon: set fsid to f1db3478-0243-11ee-9b1d-001a4aab830c 2023-06-03T19:22:51.248 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph-mon: debug 2023-06-03T19:22:51.230+0000 7fe36b84e880 4 rocksdb: RocksDB version: 6.8.1 2023-06-03T19:22:51.248 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph-mon: 2023-06-03T19:22:51.248 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph-mon: debug 2023-06-03T19:22:51.230+0000 7fe36b84e880 4 rocksdb: Git sha rocksdb_build_git_sha:@0@ 2023-06-03T19:22:51.248 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph-mon: debug 2023-06-03T19:22:51.234+0000 7fe36b84e880 4 rocksdb: Compile date May 28 2023 2023-06-03T19:22:51.248 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph-mon: debug 2023-06-03T19:22:51.234+0000 7fe36b84e880 4 rocksdb: DB SUMMARY 2023-06-03T19:22:51.248 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph-mon: 2023-06-03T19:22:51.249 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph-mon: debug 2023-06-03T19:22:51.234+0000 7fe36b84e880 4 rocksdb: SST files in /var/lib/ceph/mon/ceph-a/store.db dir, Total Num: 0, files: 2023-06-03T19:22:51.249 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph-mon: 2023-06-03T19:22:51.249 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph-mon: debug 2023-06-03T19:22:51.234+0000 7fe36b84e880 4 rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-a/store.db: 2023-06-03T19:22:51.249 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph-mon: 2023-06-03T19:22:51.249 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph-mon: debug 2023-06-03T19:22:51.234+0000 7fe36b84e880 4 rocksdb: Options.error_if_exists: 0 2023-06-03T19:22:51.249 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph-mon: debug 2023-06-03T19:22:51.234+0000 7fe36b84e880 4 rocksdb: Options.create_if_missing: 1 2023-06-03T19:22:51.249 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph-mon: debug 2023-06-03T19:22:51.234+0000 7fe36b84e880 4 rocksdb: Options.paranoid_checks: 1 2023-06-03T19:22:51.249 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph-mon: debug 2023-06-03T19:22:51.234+0000 7fe36b84e880 4 rocksdb: Options.env: 0x5561f19bd080 2023-06-03T19:22:51.250 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph-mon: debug 2023-06-03T19:22:51.234+0000 7fe36b84e880 4 rocksdb: Options.fs: Posix File System 2023-06-03T19:22:51.250 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph-mon: debug 2023-06-03T19:22:51.234+0000 7fe36b84e880 4 rocksdb: Options.info_log: 0x5561f2a53a80 2023-06-03T19:22:51.250 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph-mon: debug 2023-06-03T19:22:51.234+0000 7fe36b84e880 4 rocksdb: Options.max_file_opening_threads: 16 2023-06-03T19:22:51.250 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph-mon: debug 2023-06-03T19:22:51.234+0000 7fe36b84e880 4 rocksdb: Options.statistics: (nil) 2023-06-03T19:22:51.264 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph-mon: debug 2023-06-03T19:22:51.234+0000 7fe36b84e880 4 rocksdb: Options.use_fsync: 0 2023-06-03T19:22:51.264 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph-mon: debug 2023-06-03T19:22:51.234+0000 7fe36b84e880 4 rocksdb: Options.max_log_file_size: 0 2023-06-03T19:22:51.264 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph-mon: debug 2023-06-03T19:22:51.234+0000 7fe36b84e880 4 rocksdb: Options.max_manifest_file_size: 1073741824 2023-06-03T19:22:51.264 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph-mon: debug 2023-06-03T19:22:51.234+0000 7fe36b84e880 4 rocksdb: Options.log_file_time_to_roll: 0 2023-06-03T19:22:51.265 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph-mon: debug 2023-06-03T19:22:51.234+0000 7fe36b84e880 4 rocksdb: Options.keep_log_file_num: 1000 2023-06-03T19:22:51.265 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph-mon: debug 2023-06-03T19:22:51.234+0000 7fe36b84e880 4 rocksdb: Options.recycle_log_file_num: 0 2023-06-03T19:22:51.266 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph-mon: debug 2023-06-03T19:22:51.234+0000 7fe36b84e880 4 rocksdb: Options.allow_fallocate: 1 2023-06-03T19:22:51.266 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph-mon: debug 2023-06-03T19:22:51.234+0000 7fe36b84e880 4 rocksdb: Options.allow_mmap_reads: 0 2023-06-03T19:22:51.266 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph-mon: debug 2023-06-03T19:22:51.234+0000 7fe36b84e880 4 rocksdb: Options.allow_mmap_writes: 0 2023-06-03T19:22:51.266 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph-mon: debug 2023-06-03T19:22:51.234+0000 7fe36b84e880 4 rocksdb: Options.use_direct_reads: 0 2023-06-03T19:22:51.266 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph-mon: debug 2023-06-03T19:22:51.234+0000 7fe36b84e880 4 rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2023-06-03T19:22:51.266 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph-mon: debug 2023-06-03T19:22:51.234+0000 7fe36b84e880 4 rocksdb: Options.create_missing_column_families: 0 2023-06-03T19:22:51.267 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph-mon: debug 2023-06-03T19:22:51.234+0000 7fe36b84e880 4 rocksdb: Options.db_log_dir: 2023-06-03T19:22:51.267 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph-mon: debug 2023-06-03T19:22:51.234+0000 7fe36b84e880 4 rocksdb: Options.wal_dir: /var/lib/ceph/mon/ceph-a/store.db 2023-06-03T19:22:51.267 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph-mon: debug 2023-06-03T19:22:51.234+0000 7fe36b84e880 4 rocksdb: Options.table_cache_numshardbits: 6 2023-06-03T19:22:51.267 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph-mon: debug 2023-06-03T19:22:51.234+0000 7fe36b84e880 4 rocksdb: Options.max_subcompactions: 1 2023-06-03T19:22:51.267 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph-mon: debug 2023-06-03T19:22:51.234+0000 7fe36b84e880 4 rocksdb: Options.max_background_flushes: -1 2023-06-03T19:22:51.267 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph-mon: debug 2023-06-03T19:22:51.234+0000 7fe36b84e880 4 rocksdb: Options.WAL_ttl_seconds: 0 2023-06-03T19:22:51.267 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph-mon: debug 2023-06-03T19:22:51.234+0000 7fe36b84e880 4 rocksdb: Options.WAL_size_limit_MB: 0 2023-06-03T19:22:51.267 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph-mon: debug 2023-06-03T19:22:51.234+0000 7fe36b84e880 4 rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2023-06-03T19:22:51.268 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph-mon: debug 2023-06-03T19:22:51.234+0000 7fe36b84e880 4 rocksdb: Options.manifest_preallocation_size: 4194304 2023-06-03T19:22:51.268 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph-mon: debug 2023-06-03T19:22:51.234+0000 7fe36b84e880 4 rocksdb: Options.is_fd_close_on_exec: 1 2023-06-03T19:22:51.268 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph-mon: debug 2023-06-03T19:22:51.234+0000 7fe36b84e880 4 rocksdb: Options.advise_random_on_open: 1 2023-06-03T19:22:51.268 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph-mon: debug 2023-06-03T19:22:51.234+0000 7fe36b84e880 4 rocksdb: Options.db_write_buffer_size: 0 2023-06-03T19:22:51.268 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph-mon: debug 2023-06-03T19:22:51.234+0000 7fe36b84e880 4 rocksdb: Options.write_buffer_manager: 0x5561f2a5b770 2023-06-03T19:22:51.268 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph-mon: debug 2023-06-03T19:22:51.234+0000 7fe36b84e880 4 rocksdb: Options.access_hint_on_compaction_start: 1 2023-06-03T19:22:51.268 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph-mon: debug 2023-06-03T19:22:51.234+0000 7fe36b84e880 4 rocksdb: Options.new_table_reader_for_compaction_inputs: 0 2023-06-03T19:22:51.269 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph-mon: debug 2023-06-03T19:22:51.234+0000 7fe36b84e880 4 rocksdb: Options.random_access_max_buffer_size: 1048576 2023-06-03T19:22:51.269 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph-mon: debug 2023-06-03T19:22:51.234+0000 7fe36b84e880 4 rocksdb: Options.use_adaptive_mutex: 0 2023-06-03T19:22:51.269 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph-mon: debug 2023-06-03T19:22:51.234+0000 7fe36b84e880 4 rocksdb: Options.rate_limiter: (nil) 2023-06-03T19:22:51.269 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph-mon: debug 2023-06-03T19:22:51.234+0000 7fe36b84e880 4 rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2023-06-03T19:22:51.269 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph-mon: debug 2023-06-03T19:22:51.234+0000 7fe36b84e880 4 rocksdb: Options.wal_recovery_mode: 2 2023-06-03T19:22:51.269 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph-mon: debug 2023-06-03T19:22:51.234+0000 7fe36b84e880 4 rocksdb: Options.enable_thread_tracking: 0 2023-06-03T19:22:51.269 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph-mon: debug 2023-06-03T19:22:51.234+0000 7fe36b84e880 4 rocksdb: Options.enable_pipelined_write: 0 2023-06-03T19:22:51.269 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph-mon: debug 2023-06-03T19:22:51.234+0000 7fe36b84e880 4 rocksdb: Options.unordered_write: 0 2023-06-03T19:22:51.270 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph-mon: debug 2023-06-03T19:22:51.234+0000 7fe36b84e880 4 rocksdb: Options.allow_concurrent_memtable_write: 1 2023-06-03T19:22:51.270 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph-mon: debug 2023-06-03T19:22:51.234+0000 7fe36b84e880 4 rocksdb: Options.enable_write_thread_adaptive_yield: 1 2023-06-03T19:22:51.270 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph-mon: debug 2023-06-03T19:22:51.234+0000 7fe36b84e880 4 rocksdb: Options.write_thread_max_yield_usec: 100 2023-06-03T19:22:51.270 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph-mon: debug 2023-06-03T19:22:51.234+0000 7fe36b84e880 4 rocksdb: Options.write_thread_slow_yield_usec: 3 2023-06-03T19:22:51.270 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph-mon: debug 2023-06-03T19:22:51.234+0000 7fe36b84e880 4 rocksdb: Options.row_cache: None 2023-06-03T19:22:51.270 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph-mon: debug 2023-06-03T19:22:51.234+0000 7fe36b84e880 4 rocksdb: Options.wal_filter: None 2023-06-03T19:22:51.270 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph-mon: debug 2023-06-03T19:22:51.234+0000 7fe36b84e880 4 rocksdb: Options.avoid_flush_during_recovery: 0 2023-06-03T19:22:51.270 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph-mon: debug 2023-06-03T19:22:51.234+0000 7fe36b84e880 4 rocksdb: Options.allow_ingest_behind: 0 2023-06-03T19:22:51.271 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph-mon: debug 2023-06-03T19:22:51.234+0000 7fe36b84e880 4 rocksdb: Options.preserve_deletes: 0 2023-06-03T19:22:51.271 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph-mon: debug 2023-06-03T19:22:51.234+0000 7fe36b84e880 4 rocksdb: Options.two_write_queues: 0 2023-06-03T19:22:51.271 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph-mon: debug 2023-06-03T19:22:51.234+0000 7fe36b84e880 4 rocksdb: Options.manual_wal_flush: 0 2023-06-03T19:22:51.271 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph-mon: debug 2023-06-03T19:22:51.234+0000 7fe36b84e880 4 rocksdb: Options.atomic_flush: 0 2023-06-03T19:22:51.271 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph-mon: debug 2023-06-03T19:22:51.234+0000 7fe36b84e880 4 rocksdb: Options.avoid_unnecessary_blocking_io: 0 2023-06-03T19:22:51.271 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph-mon: debug 2023-06-03T19:22:51.234+0000 7fe36b84e880 4 rocksdb: Options.persist_stats_to_disk: 0 2023-06-03T19:22:51.271 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph-mon: debug 2023-06-03T19:22:51.234+0000 7fe36b84e880 4 rocksdb: Options.write_dbid_to_manifest: 0 2023-06-03T19:22:51.271 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph-mon: debug 2023-06-03T19:22:51.234+0000 7fe36b84e880 4 rocksdb: Options.log_readahead_size: 0 2023-06-03T19:22:51.272 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph-mon: debug 2023-06-03T19:22:51.234+0000 7fe36b84e880 4 rocksdb: Options.sst_file_checksum_func: Unknown 2023-06-03T19:22:51.272 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph-mon: debug 2023-06-03T19:22:51.234+0000 7fe36b84e880 4 rocksdb: Options.max_background_jobs: 2 2023-06-03T19:22:51.272 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph-mon: debug 2023-06-03T19:22:51.234+0000 7fe36b84e880 4 rocksdb: Options.max_background_compactions: -1 2023-06-03T19:22:51.272 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph-mon: debug 2023-06-03T19:22:51.234+0000 7fe36b84e880 4 rocksdb: Options.avoid_flush_during_shutdown: 0 2023-06-03T19:22:51.272 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph-mon: debug 2023-06-03T19:22:51.234+0000 7fe36b84e880 4 rocksdb: Options.writable_file_max_buffer_size: 1048576 2023-06-03T19:22:51.272 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph-mon: debug 2023-06-03T19:22:51.234+0000 7fe36b84e880 4 rocksdb: Options.delayed_write_rate : 16777216 2023-06-03T19:22:51.272 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph-mon: debug 2023-06-03T19:22:51.234+0000 7fe36b84e880 4 rocksdb: Options.max_total_wal_size: 0 2023-06-03T19:22:51.273 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph-mon: debug 2023-06-03T19:22:51.234+0000 7fe36b84e880 4 rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2023-06-03T19:22:51.273 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph-mon: debug 2023-06-03T19:22:51.234+0000 7fe36b84e880 4 rocksdb: Options.stats_dump_period_sec: 600 2023-06-03T19:22:51.273 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph-mon: debug 2023-06-03T19:22:51.234+0000 7fe36b84e880 4 rocksdb: Options.stats_persist_period_sec: 600 2023-06-03T19:22:51.273 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph-mon: debug 2023-06-03T19:22:51.234+0000 7fe36b84e880 4 rocksdb: Options.stats_history_buffer_size: 1048576 2023-06-03T19:22:51.273 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph-mon: debug 2023-06-03T19:22:51.234+0000 7fe36b84e880 4 rocksdb: Options.max_open_files: -1 2023-06-03T19:22:51.273 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph-mon: debug 2023-06-03T19:22:51.234+0000 7fe36b84e880 4 rocksdb: Options.bytes_per_sync: 0 2023-06-03T19:22:51.273 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph-mon: debug 2023-06-03T19:22:51.234+0000 7fe36b84e880 4 rocksdb: Options.wal_bytes_per_sync: 0 2023-06-03T19:22:51.273 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph-mon: debug 2023-06-03T19:22:51.234+0000 7fe36b84e880 4 rocksdb: Options.strict_bytes_per_sync: 0 2023-06-03T19:22:51.274 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph-mon: debug 2023-06-03T19:22:51.234+0000 7fe36b84e880 4 rocksdb: Options.compaction_readahead_size: 0 2023-06-03T19:22:51.274 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph-mon: debug 2023-06-03T19:22:51.234+0000 7fe36b84e880 4 rocksdb: Compression algorithms supported: 2023-06-03T19:22:51.274 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph-mon: debug 2023-06-03T19:22:51.234+0000 7fe36b84e880 4 rocksdb: kZSTDNotFinalCompression supported: 0 2023-06-03T19:22:51.274 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph-mon: debug 2023-06-03T19:22:51.234+0000 7fe36b84e880 4 rocksdb: kZSTD supported: 0 2023-06-03T19:22:51.274 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph-mon: debug 2023-06-03T19:22:51.234+0000 7fe36b84e880 4 rocksdb: kXpressCompression supported: 0 2023-06-03T19:22:51.274 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph-mon: debug 2023-06-03T19:22:51.234+0000 7fe36b84e880 4 rocksdb: kLZ4HCCompression supported: 1 2023-06-03T19:22:51.274 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph-mon: debug 2023-06-03T19:22:51.234+0000 7fe36b84e880 4 rocksdb: kLZ4Compression supported: 1 2023-06-03T19:22:51.275 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph-mon: debug 2023-06-03T19:22:51.234+0000 7fe36b84e880 4 rocksdb: kBZip2Compression supported: 0 2023-06-03T19:22:51.275 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph-mon: debug 2023-06-03T19:22:51.234+0000 7fe36b84e880 4 rocksdb: kZlibCompression supported: 1 2023-06-03T19:22:51.275 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph-mon: debug 2023-06-03T19:22:51.234+0000 7fe36b84e880 4 rocksdb: kSnappyCompression supported: 1 2023-06-03T19:22:51.275 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph-mon: debug 2023-06-03T19:22:51.234+0000 7fe36b84e880 4 rocksdb: Fast CRC32 supported: Supported on x86 2023-06-03T19:22:51.275 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph-mon: debug 2023-06-03T19:22:51.234+0000 7fe36b84e880 4 rocksdb: [db_impl/db_impl_open.cc:273] Creating manifest 1 2023-06-03T19:22:51.275 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph-mon: 2023-06-03T19:22:51.275 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph-mon: debug 2023-06-03T19:22:51.234+0000 7fe36b84e880 4 rocksdb: [version_set.cc:4413] Recovering from manifest file: /var/lib/ceph/mon/ceph-a/store.db/MANIFEST-000001 2023-06-03T19:22:51.275 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph-mon: 2023-06-03T19:22:51.276 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph-mon: debug 2023-06-03T19:22:51.234+0000 7fe36b84e880 4 rocksdb: [column_family.cc:552] --------------- Options for column family [default]: 2023-06-03T19:22:51.276 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph-mon: 2023-06-03T19:22:51.276 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph-mon: debug 2023-06-03T19:22:51.234+0000 7fe36b84e880 4 rocksdb: Options.comparator: leveldb.BytewiseComparator 2023-06-03T19:22:51.276 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph-mon: debug 2023-06-03T19:22:51.234+0000 7fe36b84e880 4 rocksdb: Options.merge_operator: 2023-06-03T19:22:51.276 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph-mon: debug 2023-06-03T19:22:51.234+0000 7fe36b84e880 4 rocksdb: Options.compaction_filter: None 2023-06-03T19:22:51.276 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph-mon: debug 2023-06-03T19:22:51.234+0000 7fe36b84e880 4 rocksdb: Options.compaction_filter_factory: None 2023-06-03T19:22:51.276 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph-mon: debug 2023-06-03T19:22:51.234+0000 7fe36b84e880 4 rocksdb: Options.memtable_factory: SkipListFactory 2023-06-03T19:22:51.277 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph-mon: debug 2023-06-03T19:22:51.234+0000 7fe36b84e880 4 rocksdb: Options.table_factory: BlockBasedTable 2023-06-03T19:22:51.277 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph-mon: debug 2023-06-03T19:22:51.234+0000 7fe36b84e880 4 rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x5561f2969d20) 2023-06-03T19:22:51.277 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph-mon: cache_index_and_filter_blocks: 1 2023-06-03T19:22:51.277 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph-mon: cache_index_and_filter_blocks_with_high_priority: 0 2023-06-03T19:22:51.277 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph-mon: pin_l0_filter_and_index_blocks_in_cache: 0 2023-06-03T19:22:51.277 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph-mon: pin_top_level_index_and_filter: 1 2023-06-03T19:22:51.277 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph-mon: index_type: 0 2023-06-03T19:22:51.277 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph-mon: data_block_index_type: 0 2023-06-03T19:22:51.278 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph-mon: index_shortening: 1 2023-06-03T19:22:51.278 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph-mon: data_block_hash_table_util_ratio: 0.750000 2023-06-03T19:22:51.278 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph-mon: hash_index_allow_collision: 1 2023-06-03T19:22:51.278 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph-mon: checksum: 1 2023-06-03T19:22:51.278 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph-mon: no_block_cache: 0 2023-06-03T19:22:51.278 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph-mon: block_cache: 0x5561f29a0d10 2023-06-03T19:22:51.278 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph-mon: block_cache_name: BinnedLRUCache 2023-06-03T19:22:51.278 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph-mon: block_cache_options: 2023-06-03T19:22:51.279 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph-mon: capacity : 536870912 2023-06-03T19:22:51.279 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph-mon: num_shard_bits : 4 2023-06-03T19:22:51.279 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph-mon: strict_capacity_limit : 0 2023-06-03T19:22:51.279 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph-mon: high_pri_pool_ratio: 0.000 2023-06-03T19:22:51.279 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph-mon: block_cache_compressed: (nil) 2023-06-03T19:22:51.279 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph-mon: persistent_cache: (nil) 2023-06-03T19:22:51.279 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph-mon: block_size: 4096 2023-06-03T19:22:51.279 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph-mon: block_size_deviation: 10 2023-06-03T19:22:51.280 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph-mon: block_restart_interval: 16 2023-06-03T19:22:51.280 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph-mon: index_block_restart_interval: 1 2023-06-03T19:22:51.280 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph-mon: metadata_block_size: 4096 2023-06-03T19:22:51.280 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph-mon: partition_filters: 0 2023-06-03T19:22:51.280 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph-mon: use_delta_encoding: 1 2023-06-03T19:22:51.280 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph-mon: filter_policy: rocksdb.BuiltinBloomFilter 2023-06-03T19:22:51.280 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph-mon: whole_key_filtering: 1 2023-06-03T19:22:51.281 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph-mon: verify_compression: 0 2023-06-03T19:22:51.281 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph-mon: read_amp_bytes_per_bit: 0 2023-06-03T19:22:51.281 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph-mon: format_version: 2 2023-06-03T19:22:51.281 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph-mon: enable_index_compression: 1 2023-06-03T19:22:51.281 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph-mon: block_align: 0 2023-06-03T19:22:51.281 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph-mon: 2023-06-03T19:22:51.281 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph-mon: debug 2023-06-03T19:22:51.234+0000 7fe36b84e880 4 rocksdb: Options.write_buffer_size: 33554432 2023-06-03T19:22:51.281 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph-mon: debug 2023-06-03T19:22:51.234+0000 7fe36b84e880 4 rocksdb: Options.max_write_buffer_number: 2 2023-06-03T19:22:51.282 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph-mon: debug 2023-06-03T19:22:51.234+0000 7fe36b84e880 4 rocksdb: Options.compression: NoCompression 2023-06-03T19:22:51.282 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph-mon: debug 2023-06-03T19:22:51.234+0000 7fe36b84e880 4 rocksdb: Options.bottommost_compression: Disabled 2023-06-03T19:22:51.282 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph-mon: debug 2023-06-03T19:22:51.234+0000 7fe36b84e880 4 rocksdb: Options.prefix_extractor: nullptr 2023-06-03T19:22:51.282 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph-mon: debug 2023-06-03T19:22:51.234+0000 7fe36b84e880 4 rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2023-06-03T19:22:51.282 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph-mon: debug 2023-06-03T19:22:51.234+0000 7fe36b84e880 4 rocksdb: Options.num_levels: 7 2023-06-03T19:22:51.282 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph-mon: debug 2023-06-03T19:22:51.234+0000 7fe36b84e880 4 rocksdb: Options.min_write_buffer_number_to_merge: 1 2023-06-03T19:22:51.282 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph-mon: debug 2023-06-03T19:22:51.234+0000 7fe36b84e880 4 rocksdb: Options.max_write_buffer_number_to_maintain: 0 2023-06-03T19:22:51.283 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph-mon: debug 2023-06-03T19:22:51.234+0000 7fe36b84e880 4 rocksdb: Options.max_write_buffer_size_to_maintain: 0 2023-06-03T19:22:51.283 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph-mon: debug 2023-06-03T19:22:51.234+0000 7fe36b84e880 4 rocksdb: Options.bottommost_compression_opts.window_bits: -14 2023-06-03T19:22:51.283 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph-mon: debug 2023-06-03T19:22:51.234+0000 7fe36b84e880 4 rocksdb: Options.bottommost_compression_opts.level: 32767 2023-06-03T19:22:51.283 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph-mon: debug 2023-06-03T19:22:51.234+0000 7fe36b84e880 4 rocksdb: Options.bottommost_compression_opts.strategy: 0 2023-06-03T19:22:51.283 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph-mon: debug 2023-06-03T19:22:51.234+0000 7fe36b84e880 4 rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2023-06-03T19:22:51.283 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph-mon: debug 2023-06-03T19:22:51.234+0000 7fe36b84e880 4 rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2023-06-03T19:22:51.283 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph-mon: debug 2023-06-03T19:22:51.234+0000 7fe36b84e880 4 rocksdb: Options.bottommost_compression_opts.enabled: false 2023-06-03T19:22:51.283 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph-mon: debug 2023-06-03T19:22:51.234+0000 7fe36b84e880 4 rocksdb: Options.compression_opts.window_bits: -14 2023-06-03T19:22:51.284 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph-mon: debug 2023-06-03T19:22:51.234+0000 7fe36b84e880 4 rocksdb: Options.compression_opts.level: 32767 2023-06-03T19:22:51.284 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph-mon: debug 2023-06-03T19:22:51.234+0000 7fe36b84e880 4 rocksdb: Options.compression_opts.strategy: 0 2023-06-03T19:22:51.284 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph-mon: debug 2023-06-03T19:22:51.234+0000 7fe36b84e880 4 rocksdb: Options.compression_opts.max_dict_bytes: 0 2023-06-03T19:22:51.284 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph-mon: debug 2023-06-03T19:22:51.234+0000 7fe36b84e880 4 rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2023-06-03T19:22:51.284 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph-mon: debug 2023-06-03T19:22:51.234+0000 7fe36b84e880 4 rocksdb: Options.compression_opts.enabled: false 2023-06-03T19:22:51.284 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph-mon: debug 2023-06-03T19:22:51.234+0000 7fe36b84e880 4 rocksdb: Options.level0_file_num_compaction_trigger: 4 2023-06-03T19:22:51.284 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph-mon: debug 2023-06-03T19:22:51.234+0000 7fe36b84e880 4 rocksdb: Options.level0_slowdown_writes_trigger: 20 2023-06-03T19:22:51.284 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph-mon: debug 2023-06-03T19:22:51.234+0000 7fe36b84e880 4 rocksdb: Options.level0_stop_writes_trigger: 36 2023-06-03T19:22:51.285 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph-mon: debug 2023-06-03T19:22:51.234+0000 7fe36b84e880 4 rocksdb: Options.target_file_size_base: 67108864 2023-06-03T19:22:51.285 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph-mon: debug 2023-06-03T19:22:51.234+0000 7fe36b84e880 4 rocksdb: Options.target_file_size_multiplier: 1 2023-06-03T19:22:51.285 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph-mon: debug 2023-06-03T19:22:51.234+0000 7fe36b84e880 4 rocksdb: Options.max_bytes_for_level_base: 268435456 2023-06-03T19:22:51.285 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph-mon: debug 2023-06-03T19:22:51.234+0000 7fe36b84e880 4 rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2023-06-03T19:22:51.285 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph-mon: debug 2023-06-03T19:22:51.234+0000 7fe36b84e880 4 rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2023-06-03T19:22:51.285 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph-mon: debug 2023-06-03T19:22:51.234+0000 7fe36b84e880 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2023-06-03T19:22:51.285 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph-mon: debug 2023-06-03T19:22:51.234+0000 7fe36b84e880 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2023-06-03T19:22:51.286 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph-mon: debug 2023-06-03T19:22:51.234+0000 7fe36b84e880 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2023-06-03T19:22:51.286 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph-mon: debug 2023-06-03T19:22:51.234+0000 7fe36b84e880 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2023-06-03T19:22:51.286 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph-mon: debug 2023-06-03T19:22:51.234+0000 7fe36b84e880 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2023-06-03T19:22:51.286 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph-mon: debug 2023-06-03T19:22:51.234+0000 7fe36b84e880 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2023-06-03T19:22:51.286 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph-mon: debug 2023-06-03T19:22:51.234+0000 7fe36b84e880 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2023-06-03T19:22:51.286 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph-mon: debug 2023-06-03T19:22:51.234+0000 7fe36b84e880 4 rocksdb: Options.max_sequential_skip_in_iterations: 8 2023-06-03T19:22:51.286 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph-mon: debug 2023-06-03T19:22:51.234+0000 7fe36b84e880 4 rocksdb: Options.max_compaction_bytes: 1677721600 2023-06-03T19:22:51.286 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph-mon: debug 2023-06-03T19:22:51.234+0000 7fe36b84e880 4 rocksdb: Options.arena_block_size: 4194304 2023-06-03T19:22:51.287 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph-mon: debug 2023-06-03T19:22:51.234+0000 7fe36b84e880 4 rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2023-06-03T19:22:51.287 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph-mon: debug 2023-06-03T19:22:51.234+0000 7fe36b84e880 4 rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2023-06-03T19:22:51.287 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph-mon: debug 2023-06-03T19:22:51.234+0000 7fe36b84e880 4 rocksdb: Options.rate_limit_delay_max_milliseconds: 100 2023-06-03T19:22:51.287 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph-mon: debug 2023-06-03T19:22:51.234+0000 7fe36b84e880 4 rocksdb: Options.disable_auto_compactions: 0 2023-06-03T19:22:51.287 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph-mon: debug 2023-06-03T19:22:51.234+0000 7fe36b84e880 4 rocksdb: Options.compaction_style: kCompactionStyleLevel 2023-06-03T19:22:51.287 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph-mon: debug 2023-06-03T19:22:51.234+0000 7fe36b84e880 4 rocksdb: Options.compaction_pri: kMinOverlappingRatio 2023-06-03T19:22:51.287 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph-mon: debug 2023-06-03T19:22:51.234+0000 7fe36b84e880 4 rocksdb: Options.compaction_options_universal.size_ratio: 1 2023-06-03T19:22:51.287 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph-mon: debug 2023-06-03T19:22:51.234+0000 7fe36b84e880 4 rocksdb: Options.compaction_options_universal.min_merge_width: 2 2023-06-03T19:22:51.288 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph-mon: debug 2023-06-03T19:22:51.234+0000 7fe36b84e880 4 rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2023-06-03T19:22:51.288 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph-mon: debug 2023-06-03T19:22:51.234+0000 7fe36b84e880 4 rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2023-06-03T19:22:51.288 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph-mon: debug 2023-06-03T19:22:51.234+0000 7fe36b84e880 4 rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2023-06-03T19:22:51.288 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph-mon: debug 2023-06-03T19:22:51.234+0000 7fe36b84e880 4 rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2023-06-03T19:22:51.288 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph-mon: debug 2023-06-03T19:22:51.234+0000 7fe36b84e880 4 rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2023-06-03T19:22:51.288 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph-mon: debug 2023-06-03T19:22:51.234+0000 7fe36b84e880 4 rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2023-06-03T19:22:51.288 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph-mon: debug 2023-06-03T19:22:51.234+0000 7fe36b84e880 4 rocksdb: Options.table_properties_collectors: 2023-06-03T19:22:51.288 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph-mon: debug 2023-06-03T19:22:51.234+0000 7fe36b84e880 4 rocksdb: Options.inplace_update_support: 0 2023-06-03T19:22:51.289 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph-mon: debug 2023-06-03T19:22:51.234+0000 7fe36b84e880 4 rocksdb: Options.inplace_update_num_locks: 10000 2023-06-03T19:22:51.289 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph-mon: debug 2023-06-03T19:22:51.234+0000 7fe36b84e880 4 rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2023-06-03T19:22:51.289 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph-mon: debug 2023-06-03T19:22:51.234+0000 7fe36b84e880 4 rocksdb: Options.memtable_whole_key_filtering: 0 2023-06-03T19:22:51.289 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph-mon: debug 2023-06-03T19:22:51.234+0000 7fe36b84e880 4 rocksdb: Options.memtable_huge_page_size: 0 2023-06-03T19:22:51.289 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph-mon: debug 2023-06-03T19:22:51.234+0000 7fe36b84e880 4 rocksdb: Options.bloom_locality: 0 2023-06-03T19:22:51.289 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph-mon: debug 2023-06-03T19:22:51.234+0000 7fe36b84e880 4 rocksdb: Options.max_successive_merges: 0 2023-06-03T19:22:51.289 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph-mon: debug 2023-06-03T19:22:51.234+0000 7fe36b84e880 4 rocksdb: Options.optimize_filters_for_hits: 0 2023-06-03T19:22:51.290 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph-mon: debug 2023-06-03T19:22:51.234+0000 7fe36b84e880 4 rocksdb: Options.paranoid_file_checks: 0 2023-06-03T19:22:51.290 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph-mon: debug 2023-06-03T19:22:51.234+0000 7fe36b84e880 4 rocksdb: Options.force_consistency_checks: 0 2023-06-03T19:22:51.290 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph-mon: debug 2023-06-03T19:22:51.234+0000 7fe36b84e880 4 rocksdb: Options.report_bg_io_stats: 0 2023-06-03T19:22:51.290 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph-mon: debug 2023-06-03T19:22:51.234+0000 7fe36b84e880 4 rocksdb: Options.ttl: 2592000 2023-06-03T19:22:51.290 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph-mon: debug 2023-06-03T19:22:51.234+0000 7fe36b84e880 4 rocksdb: Options.periodic_compaction_seconds: 0 2023-06-03T19:22:51.290 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph-mon: debug 2023-06-03T19:22:51.234+0000 7fe36b84e880 4 rocksdb: [version_set.cc:4568] Recovered from manifest file:/var/lib/ceph/mon/ceph-a/store.db/MANIFEST-000001 succeeded,manifest_file_number is 1, next_file_number is 3, last_sequence is 0, log_number is 0,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2023-06-03T19:22:51.290 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph-mon: 2023-06-03T19:22:51.290 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph-mon: debug 2023-06-03T19:22:51.234+0000 7fe36b84e880 4 rocksdb: [version_set.cc:4577] Column family [default] (ID 0), log number is 0 2023-06-03T19:22:51.291 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph-mon: 2023-06-03T19:22:51.291 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph-mon: debug 2023-06-03T19:22:51.238+0000 7fe36b84e880 4 rocksdb: DB pointer 0x5561f2a69800 2023-06-03T19:22:51.291 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph-mon: debug 2023-06-03T19:22:51.238+0000 7fe3546d9700 4 rocksdb: [db_impl/db_impl.cc:850] ------- DUMPING STATS ------- 2023-06-03T19:22:51.291 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph-mon: debug 2023-06-03T19:22:51.238+0000 7fe3546d9700 4 rocksdb: [db_impl/db_impl.cc:851] 2023-06-03T19:22:51.291 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph-mon: ** DB Stats ** 2023-06-03T19:22:51.291 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph-mon: Uptime(secs): 0.0 total, 0.0 interval 2023-06-03T19:22:51.291 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph-mon: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2023-06-03T19:22:51.291 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph-mon: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2023-06-03T19:22:51.292 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph-mon: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2023-06-03T19:22:51.292 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph-mon: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2023-06-03T19:22:51.292 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph-mon: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 MB, 0.00 MB/s 2023-06-03T19:22:51.292 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph-mon: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2023-06-03T19:22:51.292 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph-mon: 2023-06-03T19:22:51.292 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph-mon: ** Compaction Stats [default] ** 2023-06-03T19:22:51.292 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph-mon: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2023-06-03T19:22:51.292 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph-mon: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2023-06-03T19:22:51.293 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph-mon: Sum 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 2023-06-03T19:22:51.293 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph-mon: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 2023-06-03T19:22:51.293 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph-mon: 2023-06-03T19:22:51.293 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph-mon: ** Compaction Stats [default] ** 2023-06-03T19:22:51.293 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph-mon: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2023-06-03T19:22:51.293 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph-mon: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2023-06-03T19:22:51.293 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph-mon: Uptime(secs): 0.0 total, 0.0 interval 2023-06-03T19:22:51.294 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph-mon: Flush(GB): cumulative 0.000, interval 0.000 2023-06-03T19:22:51.294 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph-mon: AddFile(GB): cumulative 0.000, interval 0.000 2023-06-03T19:22:51.294 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph-mon: AddFile(Total Files): cumulative 0, interval 0 2023-06-03T19:22:51.294 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph-mon: AddFile(L0 Files): cumulative 0, interval 0 2023-06-03T19:22:51.294 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph-mon: AddFile(Keys): cumulative 0, interval 0 2023-06-03T19:22:51.294 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph-mon: Cumulative compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-06-03T19:22:51.294 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph-mon: Interval compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-06-03T19:22:51.294 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph-mon: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2023-06-03T19:22:51.295 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph-mon: 2023-06-03T19:22:51.295 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph-mon: ** File Read Latency Histogram By Level [default] ** 2023-06-03T19:22:51.295 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph-mon: 2023-06-03T19:22:51.295 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph-mon: ** Compaction Stats [default] ** 2023-06-03T19:22:51.295 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph-mon: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2023-06-03T19:22:51.296 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph-mon: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2023-06-03T19:22:51.296 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph-mon: Sum 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 2023-06-03T19:22:51.297 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph-mon: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 2023-06-03T19:22:51.298 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph-mon: 2023-06-03T19:22:51.298 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph-mon: ** Compaction Stats [default] ** 2023-06-03T19:22:51.298 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph-mon: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2023-06-03T19:22:51.298 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph-mon: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2023-06-03T19:22:51.298 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph-mon: Uptime(secs): 0.0 total, 0.0 interval 2023-06-03T19:22:51.298 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph-mon: Flush(GB): cumulative 0.000, interval 0.000 2023-06-03T19:22:51.298 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph-mon: AddFile(GB): cumulative 0.000, interval 0.000 2023-06-03T19:22:51.299 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph-mon: AddFile(Total Files): cumulative 0, interval 0 2023-06-03T19:22:51.299 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph-mon: AddFile(L0 Files): cumulative 0, interval 0 2023-06-03T19:22:51.299 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph-mon: AddFile(Keys): cumulative 0, interval 0 2023-06-03T19:22:51.299 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph-mon: Cumulative compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-06-03T19:22:51.299 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph-mon: Interval compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-06-03T19:22:51.299 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph-mon: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2023-06-03T19:22:51.299 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph-mon: 2023-06-03T19:22:51.299 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph-mon: ** File Read Latency Histogram By Level [default] ** 2023-06-03T19:22:51.300 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph-mon: 2023-06-03T19:22:51.300 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph-mon: debug 2023-06-03T19:22:51.238+0000 7fe36b84e880 4 rocksdb: [db_impl/db_impl.cc:397] Shutdown: canceling all background work 2023-06-03T19:22:51.300 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph-mon: debug 2023-06-03T19:22:51.238+0000 7fe36b84e880 4 rocksdb: [db_impl/db_impl.cc:573] Shutdown complete 2023-06-03T19:22:51.300 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph-mon: debug 2023-06-03T19:22:51.238+0000 7fe36b84e880 0 /usr/bin/ceph-mon: created monfs at /var/lib/ceph/mon/ceph-a for mon.a 2023-06-03T19:22:51.916 INFO:teuthology.orchestra.run.smithi089.stderr:create mon.a on 2023-06-03T19:22:51.943 INFO:teuthology.orchestra.run.smithi089.stderr:systemctl: Removed /etc/systemd/system/multi-user.target.wants/ceph.target. 2023-06-03T19:22:52.146 INFO:teuthology.orchestra.run.smithi089.stderr:systemctl: Created symlink /etc/systemd/system/multi-user.target.wants/ceph.target -> /etc/systemd/system/ceph.target. 2023-06-03T19:22:52.352 INFO:teuthology.orchestra.run.smithi089.stderr:systemctl: Created symlink /etc/systemd/system/multi-user.target.wants/ceph-f1db3478-0243-11ee-9b1d-001a4aab830c.target -> /etc/systemd/system/ceph-f1db3478-0243-11ee-9b1d-001a4aab830c.target. 2023-06-03T19:22:52.353 INFO:teuthology.orchestra.run.smithi089.stderr:systemctl: Created symlink /etc/systemd/system/ceph.target.wants/ceph-f1db3478-0243-11ee-9b1d-001a4aab830c.target -> /etc/systemd/system/ceph-f1db3478-0243-11ee-9b1d-001a4aab830c.target. 2023-06-03T19:22:52.783 INFO:teuthology.orchestra.run.smithi089.stderr:systemctl: Failed to reset failed state of unit ceph-f1db3478-0243-11ee-9b1d-001a4aab830c@mon.a.service: Unit ceph-f1db3478-0243-11ee-9b1d-001a4aab830c@mon.a.service not loaded. 2023-06-03T19:22:52.793 INFO:teuthology.orchestra.run.smithi089.stderr:systemctl: Created symlink /etc/systemd/system/ceph-f1db3478-0243-11ee-9b1d-001a4aab830c.target.wants/ceph-f1db3478-0243-11ee-9b1d-001a4aab830c@mon.a.service -> /etc/systemd/system/ceph-f1db3478-0243-11ee-9b1d-001a4aab830c@.service. 2023-06-03T19:22:53.002 INFO:teuthology.orchestra.run.smithi089.stderr:firewalld does not appear to be present 2023-06-03T19:22:53.002 INFO:teuthology.orchestra.run.smithi089.stderr:Not possible to enable service . firewalld.service is not available 2023-06-03T19:22:53.003 INFO:teuthology.orchestra.run.smithi089.stderr:Waiting for mon to start... 2023-06-03T19:22:53.003 INFO:teuthology.orchestra.run.smithi089.stderr:Waiting for mon... 2023-06-03T19:22:54.533 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:22:54 smithi089 bash[26026]: cluster 2023-06-03T19:22:54.441120+0000 mon.a (mon.0) 1 : cluster [INF] mon.a is new leader, mons a in quorum (ranks 0) 2023-06-03T19:22:54.660 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: cluster: 2023-06-03T19:22:54.661 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: id: f1db3478-0243-11ee-9b1d-001a4aab830c 2023-06-03T19:22:54.661 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: health: HEALTH_OK 2023-06-03T19:22:54.662 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: 2023-06-03T19:22:54.662 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: services: 2023-06-03T19:22:54.662 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: mon: 1 daemons, quorum a (age 0.204594s) 2023-06-03T19:22:54.662 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: mgr: no daemons active 2023-06-03T19:22:54.663 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: osd: 0 osds: 0 up, 0 in 2023-06-03T19:22:54.663 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: 2023-06-03T19:22:54.663 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: data: 2023-06-03T19:22:54.663 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: pools: 0 pools, 0 pgs 2023-06-03T19:22:54.663 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: objects: 0 objects, 0 B 2023-06-03T19:22:54.663 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: usage: 0 B used, 0 B / 0 B avail 2023-06-03T19:22:54.663 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: pgs: 2023-06-03T19:22:54.664 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: 2023-06-03T19:22:55.300 INFO:teuthology.orchestra.run.smithi089.stderr:mon is available 2023-06-03T19:22:55.300 INFO:teuthology.orchestra.run.smithi089.stderr:Assimilating anything we can from ceph.conf... 2023-06-03T19:22:55.783 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:22:55 smithi089 bash[26026]: cluster 2023-06-03T19:22:54.445972+0000 mon.a (mon.0) 2 : cluster [INF] mon.a is new leader, mons a in quorum (ranks 0) 2023-06-03T19:22:55.783 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:22:55 smithi089 bash[26026]: cluster 2023-06-03T19:22:54.446316+0000 mon.a (mon.0) 3 : cluster [DBG] monmap e1: 1 mons at {a=[v2:172.21.15.89:3300/0,v1:172.21.15.89:6789/0]} removed_ranks: {} 2023-06-03T19:22:55.783 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:22:55 smithi089 bash[26026]: cluster 2023-06-03T19:22:54.449421+0000 mon.a (mon.0) 4 : cluster [DBG] fsmap 2023-06-03T19:22:55.783 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:22:55 smithi089 bash[26026]: cluster 2023-06-03T19:22:54.453313+0000 mon.a (mon.0) 5 : cluster [DBG] osdmap e1: 0 total, 0 up, 0 in 2023-06-03T19:22:55.784 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:22:55 smithi089 bash[26026]: cluster 2023-06-03T19:22:54.453781+0000 mon.a (mon.0) 6 : cluster [DBG] mgrmap e1: no daemons active 2023-06-03T19:22:55.784 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:22:55 smithi089 bash[26026]: audit 2023-06-03T19:22:54.650460+0000 mon.a (mon.0) 7 : audit [DBG] from='client.? 172.21.15.89:0/3264094090' entity='client.admin' cmd=[{"prefix": "status"}]: dispatch 2023-06-03T19:22:56.509 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: 2023-06-03T19:22:56.510 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: [global] 2023-06-03T19:22:56.510 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: fsid = f1db3478-0243-11ee-9b1d-001a4aab830c 2023-06-03T19:22:56.510 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: log_to_journald = false 2023-06-03T19:22:56.511 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: mon_host = [v2:172.21.15.89:3300,v1:172.21.15.89:6789] 2023-06-03T19:22:56.511 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: mon_osd_allow_pg_remap = true 2023-06-03T19:22:56.511 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: mon_osd_allow_primary_affinity = true 2023-06-03T19:22:56.511 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: mon_warn_on_no_sortbitwise = false 2023-06-03T19:22:56.512 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: osd_crush_chooseleaf_type = 0 2023-06-03T19:22:56.512 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: 2023-06-03T19:22:56.512 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: [mgr] 2023-06-03T19:22:56.512 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: mgr/telemetry/nag = false 2023-06-03T19:22:56.512 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: 2023-06-03T19:22:56.512 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: [osd] 2023-06-03T19:22:56.512 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: osd_map_max_advance = 10 2023-06-03T19:22:56.513 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: osd_sloppy_crc = true 2023-06-03T19:22:57.033 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:22:56 smithi089 bash[26026]: audit 2023-06-03T19:22:56.494373+0000 mon.a (mon.0) 8 : audit [INF] from='client.? 172.21.15.89:0/2329394193' entity='client.admin' cmd=[{"prefix": "config assimilate-conf"}]: dispatch 2023-06-03T19:22:57.033 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:22:56 smithi089 bash[26026]: audit 2023-06-03T19:22:56.500087+0000 mon.a (mon.0) 9 : audit [INF] from='client.? 172.21.15.89:0/2329394193' entity='client.admin' cmd='[{"prefix": "config assimilate-conf"}]': finished 2023-06-03T19:22:57.155 INFO:teuthology.orchestra.run.smithi089.stderr:Generating new minimal ceph.conf... 2023-06-03T19:22:58.783 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:22:58 smithi089 bash[26026]: audit 2023-06-03T19:22:58.255260+0000 mon.a (mon.0) 10 : audit [DBG] from='client.? 172.21.15.89:0/2196192529' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-03T19:22:58.944 INFO:teuthology.orchestra.run.smithi089.stderr:Restarting the monitor... 2023-06-03T19:22:59.229 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:22:58 smithi089 systemd[1]: Stopping Ceph mon.a for f1db3478-0243-11ee-9b1d-001a4aab830c... 2023-06-03T19:22:59.229 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:22:58 smithi089 bash[26422]: Error response from daemon: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-mon.a 2023-06-03T19:22:59.229 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:22:59 smithi089 bash[26026]: debug 2023-06-03T19:22:59.114+0000 7f012a7c6700 -1 received signal: Terminated from /sbin/docker-init -- /usr/bin/ceph-mon -n mon.a -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-stderr=true (PID: 1) UID: 0 2023-06-03T19:22:59.229 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:22:59 smithi089 bash[26026]: debug 2023-06-03T19:22:59.114+0000 7f012a7c6700 -1 mon.a@0(leader) e1 *** Got Signal Terminated *** 2023-06-03T19:22:59.877 INFO:teuthology.orchestra.run.smithi089.stderr:Setting mon public_network to 172.21.15.254/32,172.21.0.0/20 2023-06-03T19:23:00.283 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:22:59 smithi089 bash[26462]: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-mon-a 2023-06-03T19:23:00.283 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:22:59 smithi089 systemd[1]: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c@mon.a.service: Succeeded. 2023-06-03T19:23:00.283 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:22:59 smithi089 systemd[1]: Stopped Ceph mon.a for f1db3478-0243-11ee-9b1d-001a4aab830c. 2023-06-03T19:23:00.284 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:22:59 smithi089 systemd[1]: Started Ceph mon.a for f1db3478-0243-11ee-9b1d-001a4aab830c. 2023-06-03T19:23:01.533 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:23:01 smithi089 bash[26572]: debug 2023-06-03T19:23:01.318+0000 7fc1e1245880 0 set uid:gid to 167:167 (ceph:ceph) 2023-06-03T19:23:01.534 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:23:01 smithi089 bash[26572]: debug 2023-06-03T19:23:01.318+0000 7fc1e1245880 0 ceph version 16.2.13-257-gd8c5d349 (d8c5d34975dce1c5eb0aa3a7979a4d9b9a99d1ec) pacific (stable), process ceph-mon, pid 7 2023-06-03T19:23:01.534 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:23:01 smithi089 bash[26572]: debug 2023-06-03T19:23:01.318+0000 7fc1e1245880 0 pidfile_write: ignore empty --pid-file 2023-06-03T19:23:01.534 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:23:01 smithi089 bash[26572]: debug 2023-06-03T19:23:01.322+0000 7fc1e1245880 0 load: jerasure load: lrc load: isa 2023-06-03T19:23:01.534 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:23:01 smithi089 bash[26572]: debug 2023-06-03T19:23:01.322+0000 7fc1e1245880 4 rocksdb: RocksDB version: 6.8.1 2023-06-03T19:23:01.534 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:23:01 smithi089 bash[26572]: debug 2023-06-03T19:23:01.322+0000 7fc1e1245880 4 rocksdb: Git sha rocksdb_build_git_sha:@0@ 2023-06-03T19:23:01.535 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:23:01 smithi089 bash[26572]: debug 2023-06-03T19:23:01.322+0000 7fc1e1245880 4 rocksdb: Compile date May 28 2023 2023-06-03T19:23:01.535 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:23:01 smithi089 bash[26572]: debug 2023-06-03T19:23:01.322+0000 7fc1e1245880 4 rocksdb: DB SUMMARY 2023-06-03T19:23:01.535 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:23:01 smithi089 bash[26572]: debug 2023-06-03T19:23:01.322+0000 7fc1e1245880 4 rocksdb: CURRENT file: CURRENT 2023-06-03T19:23:01.535 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:23:01 smithi089 bash[26572]: debug 2023-06-03T19:23:01.322+0000 7fc1e1245880 4 rocksdb: IDENTITY file: IDENTITY 2023-06-03T19:23:01.535 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:23:01 smithi089 bash[26572]: debug 2023-06-03T19:23:01.322+0000 7fc1e1245880 4 rocksdb: MANIFEST file: MANIFEST-000005 size: 131 Bytes 2023-06-03T19:23:01.536 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:23:01 smithi089 bash[26572]: debug 2023-06-03T19:23:01.322+0000 7fc1e1245880 4 rocksdb: SST files in /var/lib/ceph/mon/ceph-a/store.db dir, Total Num: 1, files: 000004.sst 2023-06-03T19:23:01.536 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:23:01 smithi089 bash[26572]: debug 2023-06-03T19:23:01.322+0000 7fc1e1245880 4 rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-a/store.db: 000006.log size: 86703 ; 2023-06-03T19:23:01.536 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:23:01 smithi089 bash[26572]: debug 2023-06-03T19:23:01.322+0000 7fc1e1245880 4 rocksdb: Options.error_if_exists: 0 2023-06-03T19:23:01.536 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:23:01 smithi089 bash[26572]: debug 2023-06-03T19:23:01.322+0000 7fc1e1245880 4 rocksdb: Options.create_if_missing: 0 2023-06-03T19:23:01.536 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:23:01 smithi089 bash[26572]: debug 2023-06-03T19:23:01.322+0000 7fc1e1245880 4 rocksdb: Options.paranoid_checks: 1 2023-06-03T19:23:01.536 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:23:01 smithi089 bash[26572]: debug 2023-06-03T19:23:01.322+0000 7fc1e1245880 4 rocksdb: Options.env: 0x55a0d7fbd080 2023-06-03T19:23:01.536 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:23:01 smithi089 bash[26572]: debug 2023-06-03T19:23:01.322+0000 7fc1e1245880 4 rocksdb: Options.fs: Posix File System 2023-06-03T19:23:01.537 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:23:01 smithi089 bash[26572]: debug 2023-06-03T19:23:01.322+0000 7fc1e1245880 4 rocksdb: Options.info_log: 0x55a0d88721a0 2023-06-03T19:23:01.537 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:23:01 smithi089 bash[26572]: debug 2023-06-03T19:23:01.322+0000 7fc1e1245880 4 rocksdb: Options.max_file_opening_threads: 16 2023-06-03T19:23:01.537 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:23:01 smithi089 bash[26572]: debug 2023-06-03T19:23:01.322+0000 7fc1e1245880 4 rocksdb: Options.statistics: (nil) 2023-06-03T19:23:01.537 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:23:01 smithi089 bash[26572]: debug 2023-06-03T19:23:01.322+0000 7fc1e1245880 4 rocksdb: Options.use_fsync: 0 2023-06-03T19:23:01.537 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:23:01 smithi089 bash[26572]: debug 2023-06-03T19:23:01.322+0000 7fc1e1245880 4 rocksdb: Options.max_log_file_size: 0 2023-06-03T19:23:01.537 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:23:01 smithi089 bash[26572]: debug 2023-06-03T19:23:01.322+0000 7fc1e1245880 4 rocksdb: Options.max_manifest_file_size: 1073741824 2023-06-03T19:23:01.538 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:23:01 smithi089 bash[26572]: debug 2023-06-03T19:23:01.322+0000 7fc1e1245880 4 rocksdb: Options.log_file_time_to_roll: 0 2023-06-03T19:23:01.538 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:23:01 smithi089 bash[26572]: debug 2023-06-03T19:23:01.322+0000 7fc1e1245880 4 rocksdb: Options.keep_log_file_num: 1000 2023-06-03T19:23:01.538 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:23:01 smithi089 bash[26572]: debug 2023-06-03T19:23:01.322+0000 7fc1e1245880 4 rocksdb: Options.recycle_log_file_num: 0 2023-06-03T19:23:01.539 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:23:01 smithi089 bash[26572]: debug 2023-06-03T19:23:01.322+0000 7fc1e1245880 4 rocksdb: Options.allow_fallocate: 1 2023-06-03T19:23:01.540 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:23:01 smithi089 bash[26572]: debug 2023-06-03T19:23:01.322+0000 7fc1e1245880 4 rocksdb: Options.allow_mmap_reads: 0 2023-06-03T19:23:01.540 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:23:01 smithi089 bash[26572]: debug 2023-06-03T19:23:01.322+0000 7fc1e1245880 4 rocksdb: Options.allow_mmap_writes: 0 2023-06-03T19:23:01.540 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:23:01 smithi089 bash[26572]: debug 2023-06-03T19:23:01.322+0000 7fc1e1245880 4 rocksdb: Options.use_direct_reads: 0 2023-06-03T19:23:01.540 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:23:01 smithi089 bash[26572]: debug 2023-06-03T19:23:01.322+0000 7fc1e1245880 4 rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2023-06-03T19:23:01.540 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:23:01 smithi089 bash[26572]: debug 2023-06-03T19:23:01.322+0000 7fc1e1245880 4 rocksdb: Options.create_missing_column_families: 0 2023-06-03T19:23:01.540 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:23:01 smithi089 bash[26572]: debug 2023-06-03T19:23:01.322+0000 7fc1e1245880 4 rocksdb: Options.db_log_dir: 2023-06-03T19:23:01.541 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:23:01 smithi089 bash[26572]: debug 2023-06-03T19:23:01.322+0000 7fc1e1245880 4 rocksdb: Options.wal_dir: /var/lib/ceph/mon/ceph-a/store.db 2023-06-03T19:23:01.541 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:23:01 smithi089 bash[26572]: debug 2023-06-03T19:23:01.322+0000 7fc1e1245880 4 rocksdb: Options.table_cache_numshardbits: 6 2023-06-03T19:23:01.541 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:23:01 smithi089 bash[26572]: debug 2023-06-03T19:23:01.322+0000 7fc1e1245880 4 rocksdb: Options.max_subcompactions: 1 2023-06-03T19:23:01.541 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:23:01 smithi089 bash[26572]: debug 2023-06-03T19:23:01.322+0000 7fc1e1245880 4 rocksdb: Options.max_background_flushes: -1 2023-06-03T19:23:01.542 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:23:01 smithi089 bash[26572]: debug 2023-06-03T19:23:01.322+0000 7fc1e1245880 4 rocksdb: Options.WAL_ttl_seconds: 0 2023-06-03T19:23:01.542 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:23:01 smithi089 bash[26572]: debug 2023-06-03T19:23:01.322+0000 7fc1e1245880 4 rocksdb: Options.WAL_size_limit_MB: 0 2023-06-03T19:23:01.542 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:23:01 smithi089 bash[26572]: debug 2023-06-03T19:23:01.322+0000 7fc1e1245880 4 rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2023-06-03T19:23:01.542 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:23:01 smithi089 bash[26572]: debug 2023-06-03T19:23:01.322+0000 7fc1e1245880 4 rocksdb: Options.manifest_preallocation_size: 4194304 2023-06-03T19:23:01.542 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:23:01 smithi089 bash[26572]: debug 2023-06-03T19:23:01.322+0000 7fc1e1245880 4 rocksdb: Options.is_fd_close_on_exec: 1 2023-06-03T19:23:01.542 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:23:01 smithi089 bash[26572]: debug 2023-06-03T19:23:01.322+0000 7fc1e1245880 4 rocksdb: Options.advise_random_on_open: 1 2023-06-03T19:23:01.543 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:23:01 smithi089 bash[26572]: debug 2023-06-03T19:23:01.322+0000 7fc1e1245880 4 rocksdb: Options.db_write_buffer_size: 0 2023-06-03T19:23:01.543 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:23:01 smithi089 bash[26572]: debug 2023-06-03T19:23:01.322+0000 7fc1e1245880 4 rocksdb: Options.write_buffer_manager: 0x55a0d9564210 2023-06-03T19:23:01.543 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:23:01 smithi089 bash[26572]: debug 2023-06-03T19:23:01.322+0000 7fc1e1245880 4 rocksdb: Options.access_hint_on_compaction_start: 1 2023-06-03T19:23:01.543 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:23:01 smithi089 bash[26572]: debug 2023-06-03T19:23:01.322+0000 7fc1e1245880 4 rocksdb: Options.new_table_reader_for_compaction_inputs: 0 2023-06-03T19:23:01.543 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:23:01 smithi089 bash[26572]: debug 2023-06-03T19:23:01.322+0000 7fc1e1245880 4 rocksdb: Options.random_access_max_buffer_size: 1048576 2023-06-03T19:23:01.543 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:23:01 smithi089 bash[26572]: debug 2023-06-03T19:23:01.322+0000 7fc1e1245880 4 rocksdb: Options.use_adaptive_mutex: 0 2023-06-03T19:23:01.543 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:23:01 smithi089 bash[26572]: debug 2023-06-03T19:23:01.322+0000 7fc1e1245880 4 rocksdb: Options.rate_limiter: (nil) 2023-06-03T19:23:01.544 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:23:01 smithi089 bash[26572]: debug 2023-06-03T19:23:01.322+0000 7fc1e1245880 4 rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2023-06-03T19:23:01.544 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:23:01 smithi089 bash[26572]: debug 2023-06-03T19:23:01.322+0000 7fc1e1245880 4 rocksdb: Options.wal_recovery_mode: 2 2023-06-03T19:23:01.544 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:23:01 smithi089 bash[26572]: debug 2023-06-03T19:23:01.322+0000 7fc1e1245880 4 rocksdb: Options.enable_thread_tracking: 0 2023-06-03T19:23:01.544 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:23:01 smithi089 bash[26572]: debug 2023-06-03T19:23:01.322+0000 7fc1e1245880 4 rocksdb: Options.enable_pipelined_write: 0 2023-06-03T19:23:01.544 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:23:01 smithi089 bash[26572]: debug 2023-06-03T19:23:01.322+0000 7fc1e1245880 4 rocksdb: Options.unordered_write: 0 2023-06-03T19:23:01.544 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:23:01 smithi089 bash[26572]: debug 2023-06-03T19:23:01.322+0000 7fc1e1245880 4 rocksdb: Options.allow_concurrent_memtable_write: 1 2023-06-03T19:23:01.544 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:23:01 smithi089 bash[26572]: debug 2023-06-03T19:23:01.322+0000 7fc1e1245880 4 rocksdb: Options.enable_write_thread_adaptive_yield: 1 2023-06-03T19:23:01.545 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:23:01 smithi089 bash[26572]: debug 2023-06-03T19:23:01.322+0000 7fc1e1245880 4 rocksdb: Options.write_thread_max_yield_usec: 100 2023-06-03T19:23:01.545 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:23:01 smithi089 bash[26572]: debug 2023-06-03T19:23:01.322+0000 7fc1e1245880 4 rocksdb: Options.write_thread_slow_yield_usec: 3 2023-06-03T19:23:01.545 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:23:01 smithi089 bash[26572]: debug 2023-06-03T19:23:01.322+0000 7fc1e1245880 4 rocksdb: Options.row_cache: None 2023-06-03T19:23:01.545 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:23:01 smithi089 bash[26572]: debug 2023-06-03T19:23:01.322+0000 7fc1e1245880 4 rocksdb: Options.wal_filter: None 2023-06-03T19:23:01.545 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:23:01 smithi089 bash[26572]: debug 2023-06-03T19:23:01.322+0000 7fc1e1245880 4 rocksdb: Options.avoid_flush_during_recovery: 0 2023-06-03T19:23:01.545 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:23:01 smithi089 bash[26572]: debug 2023-06-03T19:23:01.322+0000 7fc1e1245880 4 rocksdb: Options.allow_ingest_behind: 0 2023-06-03T19:23:01.546 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:23:01 smithi089 bash[26572]: debug 2023-06-03T19:23:01.322+0000 7fc1e1245880 4 rocksdb: Options.preserve_deletes: 0 2023-06-03T19:23:01.546 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:23:01 smithi089 bash[26572]: debug 2023-06-03T19:23:01.322+0000 7fc1e1245880 4 rocksdb: Options.two_write_queues: 0 2023-06-03T19:23:01.546 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:23:01 smithi089 bash[26572]: debug 2023-06-03T19:23:01.322+0000 7fc1e1245880 4 rocksdb: Options.manual_wal_flush: 0 2023-06-03T19:23:01.546 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:23:01 smithi089 bash[26572]: debug 2023-06-03T19:23:01.322+0000 7fc1e1245880 4 rocksdb: Options.atomic_flush: 0 2023-06-03T19:23:01.546 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:23:01 smithi089 bash[26572]: debug 2023-06-03T19:23:01.322+0000 7fc1e1245880 4 rocksdb: Options.avoid_unnecessary_blocking_io: 0 2023-06-03T19:23:01.546 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:23:01 smithi089 bash[26572]: debug 2023-06-03T19:23:01.322+0000 7fc1e1245880 4 rocksdb: Options.persist_stats_to_disk: 0 2023-06-03T19:23:01.546 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:23:01 smithi089 bash[26572]: debug 2023-06-03T19:23:01.322+0000 7fc1e1245880 4 rocksdb: Options.write_dbid_to_manifest: 0 2023-06-03T19:23:01.547 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:23:01 smithi089 bash[26572]: debug 2023-06-03T19:23:01.322+0000 7fc1e1245880 4 rocksdb: Options.log_readahead_size: 0 2023-06-03T19:23:01.547 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:23:01 smithi089 bash[26572]: debug 2023-06-03T19:23:01.322+0000 7fc1e1245880 4 rocksdb: Options.sst_file_checksum_func: Unknown 2023-06-03T19:23:01.547 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:23:01 smithi089 bash[26572]: debug 2023-06-03T19:23:01.322+0000 7fc1e1245880 4 rocksdb: Options.max_background_jobs: 2 2023-06-03T19:23:01.547 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:23:01 smithi089 bash[26572]: debug 2023-06-03T19:23:01.322+0000 7fc1e1245880 4 rocksdb: Options.max_background_compactions: -1 2023-06-03T19:23:01.547 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:23:01 smithi089 bash[26572]: debug 2023-06-03T19:23:01.322+0000 7fc1e1245880 4 rocksdb: Options.avoid_flush_during_shutdown: 0 2023-06-03T19:23:01.547 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:23:01 smithi089 bash[26572]: debug 2023-06-03T19:23:01.322+0000 7fc1e1245880 4 rocksdb: Options.writable_file_max_buffer_size: 1048576 2023-06-03T19:23:01.548 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:23:01 smithi089 bash[26572]: debug 2023-06-03T19:23:01.322+0000 7fc1e1245880 4 rocksdb: Options.delayed_write_rate : 16777216 2023-06-03T19:23:01.548 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:23:01 smithi089 bash[26572]: debug 2023-06-03T19:23:01.322+0000 7fc1e1245880 4 rocksdb: Options.max_total_wal_size: 0 2023-06-03T19:23:01.548 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:23:01 smithi089 bash[26572]: debug 2023-06-03T19:23:01.322+0000 7fc1e1245880 4 rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2023-06-03T19:23:01.548 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:23:01 smithi089 bash[26572]: debug 2023-06-03T19:23:01.322+0000 7fc1e1245880 4 rocksdb: Options.stats_dump_period_sec: 600 2023-06-03T19:23:01.548 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:23:01 smithi089 bash[26572]: debug 2023-06-03T19:23:01.322+0000 7fc1e1245880 4 rocksdb: Options.stats_persist_period_sec: 600 2023-06-03T19:23:01.548 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:23:01 smithi089 bash[26572]: debug 2023-06-03T19:23:01.322+0000 7fc1e1245880 4 rocksdb: Options.stats_history_buffer_size: 1048576 2023-06-03T19:23:01.548 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:23:01 smithi089 bash[26572]: debug 2023-06-03T19:23:01.322+0000 7fc1e1245880 4 rocksdb: Options.max_open_files: -1 2023-06-03T19:23:01.549 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:23:01 smithi089 bash[26572]: debug 2023-06-03T19:23:01.322+0000 7fc1e1245880 4 rocksdb: Options.bytes_per_sync: 0 2023-06-03T19:23:01.549 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:23:01 smithi089 bash[26572]: debug 2023-06-03T19:23:01.322+0000 7fc1e1245880 4 rocksdb: Options.wal_bytes_per_sync: 0 2023-06-03T19:23:01.549 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:23:01 smithi089 bash[26572]: debug 2023-06-03T19:23:01.322+0000 7fc1e1245880 4 rocksdb: Options.strict_bytes_per_sync: 0 2023-06-03T19:23:01.549 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:23:01 smithi089 bash[26572]: debug 2023-06-03T19:23:01.322+0000 7fc1e1245880 4 rocksdb: Options.compaction_readahead_size: 0 2023-06-03T19:23:01.549 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:23:01 smithi089 bash[26572]: debug 2023-06-03T19:23:01.322+0000 7fc1e1245880 4 rocksdb: Compression algorithms supported: 2023-06-03T19:23:01.549 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:23:01 smithi089 bash[26572]: debug 2023-06-03T19:23:01.322+0000 7fc1e1245880 4 rocksdb: kZSTDNotFinalCompression supported: 0 2023-06-03T19:23:01.549 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:23:01 smithi089 bash[26572]: debug 2023-06-03T19:23:01.322+0000 7fc1e1245880 4 rocksdb: kZSTD supported: 0 2023-06-03T19:23:01.549 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:23:01 smithi089 bash[26572]: debug 2023-06-03T19:23:01.322+0000 7fc1e1245880 4 rocksdb: kXpressCompression supported: 0 2023-06-03T19:23:01.550 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:23:01 smithi089 bash[26572]: debug 2023-06-03T19:23:01.322+0000 7fc1e1245880 4 rocksdb: kLZ4HCCompression supported: 1 2023-06-03T19:23:01.550 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:23:01 smithi089 bash[26572]: debug 2023-06-03T19:23:01.322+0000 7fc1e1245880 4 rocksdb: kLZ4Compression supported: 1 2023-06-03T19:23:01.550 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:23:01 smithi089 bash[26572]: debug 2023-06-03T19:23:01.322+0000 7fc1e1245880 4 rocksdb: kBZip2Compression supported: 0 2023-06-03T19:23:01.550 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:23:01 smithi089 bash[26572]: debug 2023-06-03T19:23:01.322+0000 7fc1e1245880 4 rocksdb: kZlibCompression supported: 1 2023-06-03T19:23:01.550 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:23:01 smithi089 bash[26572]: debug 2023-06-03T19:23:01.322+0000 7fc1e1245880 4 rocksdb: kSnappyCompression supported: 1 2023-06-03T19:23:01.550 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:23:01 smithi089 bash[26572]: debug 2023-06-03T19:23:01.322+0000 7fc1e1245880 4 rocksdb: Fast CRC32 supported: Supported on x86 2023-06-03T19:23:01.550 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:23:01 smithi089 bash[26572]: debug 2023-06-03T19:23:01.322+0000 7fc1e1245880 4 rocksdb: [version_set.cc:4413] Recovering from manifest file: /var/lib/ceph/mon/ceph-a/store.db/MANIFEST-000005 2023-06-03T19:23:01.550 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:23:01 smithi089 bash[26572]: debug 2023-06-03T19:23:01.322+0000 7fc1e1245880 4 rocksdb: [column_family.cc:552] --------------- Options for column family [default]: 2023-06-03T19:23:01.551 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:23:01 smithi089 bash[26572]: debug 2023-06-03T19:23:01.322+0000 7fc1e1245880 4 rocksdb: Options.comparator: leveldb.BytewiseComparator 2023-06-03T19:23:01.551 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:23:01 smithi089 bash[26572]: debug 2023-06-03T19:23:01.322+0000 7fc1e1245880 4 rocksdb: Options.merge_operator: 2023-06-03T19:23:01.551 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:23:01 smithi089 bash[26572]: debug 2023-06-03T19:23:01.322+0000 7fc1e1245880 4 rocksdb: Options.compaction_filter: None 2023-06-03T19:23:01.551 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:23:01 smithi089 bash[26572]: debug 2023-06-03T19:23:01.322+0000 7fc1e1245880 4 rocksdb: Options.compaction_filter_factory: None 2023-06-03T19:23:01.551 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:23:01 smithi089 bash[26572]: debug 2023-06-03T19:23:01.322+0000 7fc1e1245880 4 rocksdb: Options.memtable_factory: SkipListFactory 2023-06-03T19:23:01.552 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:23:01 smithi089 bash[26572]: debug 2023-06-03T19:23:01.322+0000 7fc1e1245880 4 rocksdb: Options.table_factory: BlockBasedTable 2023-06-03T19:23:01.552 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:23:01 smithi089 bash[26572]: debug 2023-06-03T19:23:01.322+0000 7fc1e1245880 4 rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x55a0d8815d18) 2023-06-03T19:23:01.552 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:23:01 smithi089 bash[26572]: cache_index_and_filter_blocks: 1 2023-06-03T19:23:01.552 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:23:01 smithi089 bash[26572]: cache_index_and_filter_blocks_with_high_priority: 0 2023-06-03T19:23:01.552 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:23:01 smithi089 bash[26572]: pin_l0_filter_and_index_blocks_in_cache: 0 2023-06-03T19:23:01.552 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:23:01 smithi089 bash[26572]: pin_top_level_index_and_filter: 1 2023-06-03T19:23:01.553 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:23:01 smithi089 bash[26572]: index_type: 0 2023-06-03T19:23:01.553 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:23:01 smithi089 bash[26572]: data_block_index_type: 0 2023-06-03T19:23:01.553 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:23:01 smithi089 bash[26572]: index_shortening: 1 2023-06-03T19:23:01.553 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:23:01 smithi089 bash[26572]: data_block_hash_table_util_ratio: 0.750000 2023-06-03T19:23:01.553 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:23:01 smithi089 bash[26572]: hash_index_allow_collision: 1 2023-06-03T19:23:01.553 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:23:01 smithi089 bash[26572]: checksum: 1 2023-06-03T19:23:01.553 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:23:01 smithi089 bash[26572]: no_block_cache: 0 2023-06-03T19:23:01.553 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:23:01 smithi089 bash[26572]: block_cache: 0x55a0d884cf10 2023-06-03T19:23:01.554 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:23:01 smithi089 bash[26572]: block_cache_name: BinnedLRUCache 2023-06-03T19:23:01.554 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:23:01 smithi089 bash[26572]: block_cache_options: 2023-06-03T19:23:01.554 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:23:01 smithi089 bash[26572]: capacity : 536870912 2023-06-03T19:23:01.554 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:23:01 smithi089 bash[26572]: num_shard_bits : 4 2023-06-03T19:23:01.554 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:23:01 smithi089 bash[26572]: strict_capacity_limit : 0 2023-06-03T19:23:01.554 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:23:01 smithi089 bash[26572]: high_pri_pool_ratio: 0.000 2023-06-03T19:23:01.555 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:23:01 smithi089 bash[26572]: block_cache_compressed: (nil) 2023-06-03T19:23:01.555 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:23:01 smithi089 bash[26572]: persistent_cache: (nil) 2023-06-03T19:23:01.555 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:23:01 smithi089 bash[26572]: block_size: 4096 2023-06-03T19:23:01.555 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:23:01 smithi089 bash[26572]: block_size_deviation: 10 2023-06-03T19:23:01.555 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:23:01 smithi089 bash[26572]: block_restart_interval: 16 2023-06-03T19:23:01.556 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:23:01 smithi089 bash[26572]: index_block_restart_interval: 1 2023-06-03T19:23:01.556 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:23:01 smithi089 bash[26572]: metadata_block_size: 4096 2023-06-03T19:23:01.556 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:23:01 smithi089 bash[26572]: partition_filters: 0 2023-06-03T19:23:01.556 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:23:01 smithi089 bash[26572]: use_delta_encoding: 1 2023-06-03T19:23:01.556 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:23:01 smithi089 bash[26572]: filter_policy: rocksdb.BuiltinBloomFilter 2023-06-03T19:23:01.556 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:23:01 smithi089 bash[26572]: whole_key_filtering: 1 2023-06-03T19:23:01.556 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:23:01 smithi089 bash[26572]: verify_compression: 0 2023-06-03T19:23:01.557 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:23:01 smithi089 bash[26572]: read_amp_bytes_per_bit: 0 2023-06-03T19:23:01.557 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:23:01 smithi089 bash[26572]: format_version: 2 2023-06-03T19:23:01.557 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:23:01 smithi089 bash[26572]: enable_index_compression: 1 2023-06-03T19:23:01.557 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:23:01 smithi089 bash[26572]: block_align: 0 2023-06-03T19:23:01.557 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:23:01 smithi089 bash[26572]: debug 2023-06-03T19:23:01.322+0000 7fc1e1245880 4 rocksdb: Options.write_buffer_size: 33554432 2023-06-03T19:23:01.557 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:23:01 smithi089 bash[26572]: debug 2023-06-03T19:23:01.322+0000 7fc1e1245880 4 rocksdb: Options.max_write_buffer_number: 2 2023-06-03T19:23:01.557 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:23:01 smithi089 bash[26572]: debug 2023-06-03T19:23:01.322+0000 7fc1e1245880 4 rocksdb: Options.compression: NoCompression 2023-06-03T19:23:01.558 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:23:01 smithi089 bash[26572]: debug 2023-06-03T19:23:01.322+0000 7fc1e1245880 4 rocksdb: Options.bottommost_compression: Disabled 2023-06-03T19:23:01.558 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:23:01 smithi089 bash[26572]: debug 2023-06-03T19:23:01.322+0000 7fc1e1245880 4 rocksdb: Options.prefix_extractor: nullptr 2023-06-03T19:23:01.558 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:23:01 smithi089 bash[26572]: debug 2023-06-03T19:23:01.322+0000 7fc1e1245880 4 rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2023-06-03T19:23:01.558 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:23:01 smithi089 bash[26572]: debug 2023-06-03T19:23:01.322+0000 7fc1e1245880 4 rocksdb: Options.num_levels: 7 2023-06-03T19:23:01.558 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:23:01 smithi089 bash[26572]: debug 2023-06-03T19:23:01.322+0000 7fc1e1245880 4 rocksdb: Options.min_write_buffer_number_to_merge: 1 2023-06-03T19:23:01.558 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:23:01 smithi089 bash[26572]: debug 2023-06-03T19:23:01.322+0000 7fc1e1245880 4 rocksdb: Options.max_write_buffer_number_to_maintain: 0 2023-06-03T19:23:01.559 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:23:01 smithi089 bash[26572]: debug 2023-06-03T19:23:01.322+0000 7fc1e1245880 4 rocksdb: Options.max_write_buffer_size_to_maintain: 0 2023-06-03T19:23:01.559 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:23:01 smithi089 bash[26572]: debug 2023-06-03T19:23:01.322+0000 7fc1e1245880 4 rocksdb: Options.bottommost_compression_opts.window_bits: -14 2023-06-03T19:23:01.559 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:23:01 smithi089 bash[26572]: debug 2023-06-03T19:23:01.322+0000 7fc1e1245880 4 rocksdb: Options.bottommost_compression_opts.level: 32767 2023-06-03T19:23:01.559 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:23:01 smithi089 bash[26572]: debug 2023-06-03T19:23:01.322+0000 7fc1e1245880 4 rocksdb: Options.bottommost_compression_opts.strategy: 0 2023-06-03T19:23:01.559 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:23:01 smithi089 bash[26572]: debug 2023-06-03T19:23:01.322+0000 7fc1e1245880 4 rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2023-06-03T19:23:01.559 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:23:01 smithi089 bash[26572]: debug 2023-06-03T19:23:01.322+0000 7fc1e1245880 4 rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2023-06-03T19:23:01.559 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:23:01 smithi089 bash[26572]: debug 2023-06-03T19:23:01.322+0000 7fc1e1245880 4 rocksdb: Options.bottommost_compression_opts.enabled: false 2023-06-03T19:23:01.560 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:23:01 smithi089 bash[26572]: debug 2023-06-03T19:23:01.322+0000 7fc1e1245880 4 rocksdb: Options.compression_opts.window_bits: -14 2023-06-03T19:23:01.560 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:23:01 smithi089 bash[26572]: debug 2023-06-03T19:23:01.322+0000 7fc1e1245880 4 rocksdb: Options.compression_opts.level: 32767 2023-06-03T19:23:01.560 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:23:01 smithi089 bash[26572]: debug 2023-06-03T19:23:01.322+0000 7fc1e1245880 4 rocksdb: Options.compression_opts.strategy: 0 2023-06-03T19:23:01.560 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:23:01 smithi089 bash[26572]: debug 2023-06-03T19:23:01.322+0000 7fc1e1245880 4 rocksdb: Options.compression_opts.max_dict_bytes: 0 2023-06-03T19:23:01.560 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:23:01 smithi089 bash[26572]: debug 2023-06-03T19:23:01.322+0000 7fc1e1245880 4 rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2023-06-03T19:23:01.560 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:23:01 smithi089 bash[26572]: debug 2023-06-03T19:23:01.322+0000 7fc1e1245880 4 rocksdb: Options.compression_opts.enabled: false 2023-06-03T19:23:01.561 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:23:01 smithi089 bash[26572]: debug 2023-06-03T19:23:01.322+0000 7fc1e1245880 4 rocksdb: Options.level0_file_num_compaction_trigger: 4 2023-06-03T19:23:01.561 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:23:01 smithi089 bash[26572]: debug 2023-06-03T19:23:01.322+0000 7fc1e1245880 4 rocksdb: Options.level0_slowdown_writes_trigger: 20 2023-06-03T19:23:01.561 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:23:01 smithi089 bash[26572]: debug 2023-06-03T19:23:01.322+0000 7fc1e1245880 4 rocksdb: Options.level0_stop_writes_trigger: 36 2023-06-03T19:23:01.561 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:23:01 smithi089 bash[26572]: debug 2023-06-03T19:23:01.322+0000 7fc1e1245880 4 rocksdb: Options.target_file_size_base: 67108864 2023-06-03T19:23:01.561 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:23:01 smithi089 bash[26572]: debug 2023-06-03T19:23:01.322+0000 7fc1e1245880 4 rocksdb: Options.target_file_size_multiplier: 1 2023-06-03T19:23:01.561 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:23:01 smithi089 bash[26572]: debug 2023-06-03T19:23:01.322+0000 7fc1e1245880 4 rocksdb: Options.max_bytes_for_level_base: 268435456 2023-06-03T19:23:01.561 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:23:01 smithi089 bash[26572]: debug 2023-06-03T19:23:01.322+0000 7fc1e1245880 4 rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2023-06-03T19:23:01.561 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:23:01 smithi089 bash[26572]: debug 2023-06-03T19:23:01.322+0000 7fc1e1245880 4 rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2023-06-03T19:23:01.562 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:23:01 smithi089 bash[26572]: debug 2023-06-03T19:23:01.322+0000 7fc1e1245880 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2023-06-03T19:23:01.562 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:23:01 smithi089 bash[26572]: debug 2023-06-03T19:23:01.322+0000 7fc1e1245880 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2023-06-03T19:23:01.562 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:23:01 smithi089 bash[26572]: debug 2023-06-03T19:23:01.322+0000 7fc1e1245880 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2023-06-03T19:23:01.562 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:23:01 smithi089 bash[26572]: debug 2023-06-03T19:23:01.322+0000 7fc1e1245880 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2023-06-03T19:23:01.562 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:23:01 smithi089 bash[26572]: debug 2023-06-03T19:23:01.322+0000 7fc1e1245880 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2023-06-03T19:23:01.562 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:23:01 smithi089 bash[26572]: debug 2023-06-03T19:23:01.322+0000 7fc1e1245880 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2023-06-03T19:23:01.562 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:23:01 smithi089 bash[26572]: debug 2023-06-03T19:23:01.322+0000 7fc1e1245880 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2023-06-03T19:23:01.562 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:23:01 smithi089 bash[26572]: debug 2023-06-03T19:23:01.322+0000 7fc1e1245880 4 rocksdb: Options.max_sequential_skip_in_iterations: 8 2023-06-03T19:23:01.563 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:23:01 smithi089 bash[26572]: debug 2023-06-03T19:23:01.322+0000 7fc1e1245880 4 rocksdb: Options.max_compaction_bytes: 1677721600 2023-06-03T19:23:01.563 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:23:01 smithi089 bash[26572]: debug 2023-06-03T19:23:01.322+0000 7fc1e1245880 4 rocksdb: Options.arena_block_size: 4194304 2023-06-03T19:23:01.563 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:23:01 smithi089 bash[26572]: debug 2023-06-03T19:23:01.322+0000 7fc1e1245880 4 rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2023-06-03T19:23:01.563 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:23:01 smithi089 bash[26572]: debug 2023-06-03T19:23:01.322+0000 7fc1e1245880 4 rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2023-06-03T19:23:01.563 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:23:01 smithi089 bash[26572]: debug 2023-06-03T19:23:01.322+0000 7fc1e1245880 4 rocksdb: Options.rate_limit_delay_max_milliseconds: 100 2023-06-03T19:23:01.563 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:23:01 smithi089 bash[26572]: debug 2023-06-03T19:23:01.322+0000 7fc1e1245880 4 rocksdb: Options.disable_auto_compactions: 0 2023-06-03T19:23:01.563 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:23:01 smithi089 bash[26572]: debug 2023-06-03T19:23:01.322+0000 7fc1e1245880 4 rocksdb: Options.compaction_style: kCompactionStyleLevel 2023-06-03T19:23:01.563 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:23:01 smithi089 bash[26572]: debug 2023-06-03T19:23:01.322+0000 7fc1e1245880 4 rocksdb: Options.compaction_pri: kMinOverlappingRatio 2023-06-03T19:23:01.564 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:23:01 smithi089 bash[26572]: debug 2023-06-03T19:23:01.322+0000 7fc1e1245880 4 rocksdb: Options.compaction_options_universal.size_ratio: 1 2023-06-03T19:23:01.564 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:23:01 smithi089 bash[26572]: debug 2023-06-03T19:23:01.322+0000 7fc1e1245880 4 rocksdb: Options.compaction_options_universal.min_merge_width: 2 2023-06-03T19:23:01.564 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:23:01 smithi089 bash[26572]: debug 2023-06-03T19:23:01.322+0000 7fc1e1245880 4 rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2023-06-03T19:23:01.564 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:23:01 smithi089 bash[26572]: debug 2023-06-03T19:23:01.322+0000 7fc1e1245880 4 rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2023-06-03T19:23:01.564 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:23:01 smithi089 bash[26572]: debug 2023-06-03T19:23:01.322+0000 7fc1e1245880 4 rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2023-06-03T19:23:01.564 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:23:01 smithi089 bash[26572]: debug 2023-06-03T19:23:01.322+0000 7fc1e1245880 4 rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2023-06-03T19:23:01.565 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:23:01 smithi089 bash[26572]: debug 2023-06-03T19:23:01.322+0000 7fc1e1245880 4 rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2023-06-03T19:23:01.565 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:23:01 smithi089 bash[26572]: debug 2023-06-03T19:23:01.322+0000 7fc1e1245880 4 rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2023-06-03T19:23:01.565 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:23:01 smithi089 bash[26572]: debug 2023-06-03T19:23:01.322+0000 7fc1e1245880 4 rocksdb: Options.table_properties_collectors: 2023-06-03T19:23:01.565 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:23:01 smithi089 bash[26572]: debug 2023-06-03T19:23:01.322+0000 7fc1e1245880 4 rocksdb: Options.inplace_update_support: 0 2023-06-03T19:23:01.565 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:23:01 smithi089 bash[26572]: debug 2023-06-03T19:23:01.322+0000 7fc1e1245880 4 rocksdb: Options.inplace_update_num_locks: 10000 2023-06-03T19:23:01.565 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:23:01 smithi089 bash[26572]: debug 2023-06-03T19:23:01.322+0000 7fc1e1245880 4 rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2023-06-03T19:23:01.565 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:23:01 smithi089 bash[26572]: debug 2023-06-03T19:23:01.322+0000 7fc1e1245880 4 rocksdb: Options.memtable_whole_key_filtering: 0 2023-06-03T19:23:01.565 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:23:01 smithi089 bash[26572]: debug 2023-06-03T19:23:01.322+0000 7fc1e1245880 4 rocksdb: Options.memtable_huge_page_size: 0 2023-06-03T19:23:01.566 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:23:01 smithi089 bash[26572]: debug 2023-06-03T19:23:01.322+0000 7fc1e1245880 4 rocksdb: Options.bloom_locality: 0 2023-06-03T19:23:01.566 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:23:01 smithi089 bash[26572]: debug 2023-06-03T19:23:01.322+0000 7fc1e1245880 4 rocksdb: Options.max_successive_merges: 0 2023-06-03T19:23:01.566 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:23:01 smithi089 bash[26572]: debug 2023-06-03T19:23:01.322+0000 7fc1e1245880 4 rocksdb: Options.optimize_filters_for_hits: 0 2023-06-03T19:23:01.566 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:23:01 smithi089 bash[26572]: debug 2023-06-03T19:23:01.322+0000 7fc1e1245880 4 rocksdb: Options.paranoid_file_checks: 0 2023-06-03T19:23:01.566 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:23:01 smithi089 bash[26572]: debug 2023-06-03T19:23:01.322+0000 7fc1e1245880 4 rocksdb: Options.force_consistency_checks: 0 2023-06-03T19:23:01.567 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:23:01 smithi089 bash[26572]: debug 2023-06-03T19:23:01.322+0000 7fc1e1245880 4 rocksdb: Options.report_bg_io_stats: 0 2023-06-03T19:23:01.567 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:23:01 smithi089 bash[26572]: debug 2023-06-03T19:23:01.322+0000 7fc1e1245880 4 rocksdb: Options.ttl: 2592000 2023-06-03T19:23:01.567 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:23:01 smithi089 bash[26572]: debug 2023-06-03T19:23:01.322+0000 7fc1e1245880 4 rocksdb: Options.periodic_compaction_seconds: 0 2023-06-03T19:23:01.567 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:23:01 smithi089 bash[26572]: debug 2023-06-03T19:23:01.326+0000 7fc1e1245880 4 rocksdb: [version_set.cc:4568] Recovered from manifest file:/var/lib/ceph/mon/ceph-a/store.db/MANIFEST-000005 succeeded,manifest_file_number is 5, next_file_number is 7, last_sequence is 5, log_number is 4,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2023-06-03T19:23:01.567 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:23:01 smithi089 bash[26572]: debug 2023-06-03T19:23:01.326+0000 7fc1e1245880 4 rocksdb: [version_set.cc:4577] Column family [default] (ID 0), log number is 4 2023-06-03T19:23:01.567 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:23:01 smithi089 bash[26572]: debug 2023-06-03T19:23:01.326+0000 7fc1e1245880 4 rocksdb: EVENT_LOG_v1 {"time_micros": 1685820181329033, "job": 1, "event": "recovery_started", "log_files": [6]} 2023-06-03T19:23:01.567 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:23:01 smithi089 bash[26572]: debug 2023-06-03T19:23:01.326+0000 7fc1e1245880 4 rocksdb: [db_impl/db_impl_open.cc:760] Recovering log #6 mode 2 2023-06-03T19:23:01.568 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:23:01 smithi089 bash[26572]: debug 2023-06-03T19:23:01.326+0000 7fc1e1245880 3 rocksdb: [le/block_based/filter_policy.cc:584] Using legacy Bloom filter with high (20) bits/key. Dramatic filter space and/or accuracy improvement is available with format_version>=5. 2023-06-03T19:23:01.568 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:23:01 smithi089 bash[26572]: debug 2023-06-03T19:23:01.326+0000 7fc1e1245880 4 rocksdb: EVENT_LOG_v1 {"time_micros": 1685820181331043, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 7, "file_size": 83303, "table_properties": {"data_size": 81576, "index_size": 332, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 0, "index_value_is_delta_encoded": 0, "filter_size": 581, "raw_key_size": 9829, "raw_average_key_size": 47, "raw_value_size": 75897, "raw_average_value_size": 368, "num_data_blocks": 11, "num_entries": 206, "num_deletions": 3, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "rocksdb.BuiltinBloomFilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; ", "creation_time": 1685820181, "oldest_key_time": 3, "file_creation_time": 0}} 2023-06-03T19:23:01.568 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:23:01 smithi089 bash[26572]: debug 2023-06-03T19:23:01.326+0000 7fc1e1245880 4 rocksdb: [version_set.cc:3826] Creating manifest 8 2023-06-03T19:23:01.568 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:23:01 smithi089 bash[26572]: debug 2023-06-03T19:23:01.326+0000 7fc1e1245880 4 rocksdb: EVENT_LOG_v1 {"time_micros": 1685820181331755, "job": 1, "event": "recovery_finished"} 2023-06-03T19:23:01.568 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:23:01 smithi089 bash[26572]: debug 2023-06-03T19:23:01.330+0000 7fc1e1245880 4 rocksdb: DB pointer 0x55a0d8915800 2023-06-03T19:23:01.568 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:23:01 smithi089 bash[26572]: debug 2023-06-03T19:23:01.330+0000 7fc1c8050700 4 rocksdb: [db_impl/db_impl.cc:850] ------- DUMPING STATS ------- 2023-06-03T19:23:01.568 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:23:01 smithi089 bash[26572]: debug 2023-06-03T19:23:01.330+0000 7fc1c8050700 4 rocksdb: [db_impl/db_impl.cc:851] 2023-06-03T19:23:01.569 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:23:01 smithi089 bash[26572]: ** DB Stats ** 2023-06-03T19:23:01.569 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:23:01 smithi089 bash[26572]: Uptime(secs): 0.0 total, 0.0 interval 2023-06-03T19:23:01.569 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:23:01 smithi089 bash[26572]: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2023-06-03T19:23:01.569 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:23:01 smithi089 bash[26572]: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2023-06-03T19:23:01.569 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:23:01 smithi089 bash[26572]: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2023-06-03T19:23:01.569 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:23:01 smithi089 bash[26572]: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2023-06-03T19:23:01.569 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:23:01 smithi089 bash[26572]: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 MB, 0.00 MB/s 2023-06-03T19:23:01.569 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:23:01 smithi089 bash[26572]: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2023-06-03T19:23:01.570 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:23:01 smithi089 bash[26572]: ** Compaction Stats [default] ** 2023-06-03T19:23:01.570 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:23:01 smithi089 bash[26572]: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2023-06-03T19:23:01.570 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:23:01 smithi089 bash[26572]: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2023-06-03T19:23:01.570 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:23:01 smithi089 bash[26572]: L0 2/0 83.00 KB 0.5 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 59.7 0.00 0.00 1 0.001 0 0 2023-06-03T19:23:01.570 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:23:01 smithi089 bash[26572]: Sum 2/0 83.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 59.7 0.00 0.00 1 0.001 0 0 2023-06-03T19:23:01.570 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:23:01 smithi089 bash[26572]: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 59.7 0.00 0.00 1 0.001 0 0 2023-06-03T19:23:01.570 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:23:01 smithi089 bash[26572]: ** Compaction Stats [default] ** 2023-06-03T19:23:01.570 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:23:01 smithi089 bash[26572]: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2023-06-03T19:23:01.571 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:23:01 smithi089 bash[26572]: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2023-06-03T19:23:01.571 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:23:01 smithi089 bash[26572]: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 59.7 0.00 0.00 1 0.001 0 0 2023-06-03T19:23:01.571 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:23:01 smithi089 bash[26572]: Uptime(secs): 0.0 total, 0.0 interval 2023-06-03T19:23:01.571 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:23:01 smithi089 bash[26572]: Flush(GB): cumulative 0.000, interval 0.000 2023-06-03T19:23:01.571 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:23:01 smithi089 bash[26572]: AddFile(GB): cumulative 0.000, interval 0.000 2023-06-03T19:23:01.571 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:23:01 smithi089 bash[26572]: AddFile(Total Files): cumulative 0, interval 0 2023-06-03T19:23:01.571 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:23:01 smithi089 bash[26572]: AddFile(L0 Files): cumulative 0, interval 0 2023-06-03T19:23:01.571 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:23:01 smithi089 bash[26572]: AddFile(Keys): cumulative 0, interval 0 2023-06-03T19:23:01.572 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:23:01 smithi089 bash[26572]: Cumulative compaction: 0.00 GB write, 12.02 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-06-03T19:23:01.572 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:23:01 smithi089 bash[26572]: Interval compaction: 0.00 GB write, 12.02 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-06-03T19:23:01.572 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:23:01 smithi089 bash[26572]: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2023-06-03T19:23:01.572 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:23:01 smithi089 bash[26572]: ** File Read Latency Histogram By Level [default] ** 2023-06-03T19:23:01.573 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:23:01 smithi089 bash[26572]: ** Compaction Stats [default] ** 2023-06-03T19:23:01.573 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:23:01 smithi089 bash[26572]: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2023-06-03T19:23:01.573 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:23:01 smithi089 bash[26572]: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2023-06-03T19:23:01.573 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:23:01 smithi089 bash[26572]: L0 2/0 83.00 KB 0.5 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 59.7 0.00 0.00 1 0.001 0 0 2023-06-03T19:23:01.574 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:23:01 smithi089 bash[26572]: Sum 2/0 83.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 59.7 0.00 0.00 1 0.001 0 0 2023-06-03T19:23:01.574 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:23:01 smithi089 bash[26572]: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 2023-06-03T19:23:01.574 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:23:01 smithi089 bash[26572]: ** Compaction Stats [default] ** 2023-06-03T19:23:01.574 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:23:01 smithi089 bash[26572]: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2023-06-03T19:23:01.574 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:23:01 smithi089 bash[26572]: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2023-06-03T19:23:01.574 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:23:01 smithi089 bash[26572]: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 59.7 0.00 0.00 1 0.001 0 0 2023-06-03T19:23:01.575 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:23:01 smithi089 bash[26572]: Uptime(secs): 0.0 total, 0.0 interval 2023-06-03T19:23:01.575 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:23:01 smithi089 bash[26572]: Flush(GB): cumulative 0.000, interval 0.000 2023-06-03T19:23:01.575 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:23:01 smithi089 bash[26572]: AddFile(GB): cumulative 0.000, interval 0.000 2023-06-03T19:23:01.575 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:23:01 smithi089 bash[26572]: AddFile(Total Files): cumulative 0, interval 0 2023-06-03T19:23:01.576 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:23:01 smithi089 bash[26572]: AddFile(L0 Files): cumulative 0, interval 0 2023-06-03T19:23:01.576 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:23:01 smithi089 bash[26572]: AddFile(Keys): cumulative 0, interval 0 2023-06-03T19:23:01.576 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:23:01 smithi089 bash[26572]: Cumulative compaction: 0.00 GB write, 11.96 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-06-03T19:23:01.576 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:23:01 smithi089 bash[26572]: Interval compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-06-03T19:23:01.576 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:23:01 smithi089 bash[26572]: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2023-06-03T19:23:01.577 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:23:01 smithi089 bash[26572]: ** File Read Latency Histogram By Level [default] ** 2023-06-03T19:23:01.577 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:23:01 smithi089 bash[26572]: debug 2023-06-03T19:23:01.330+0000 7fc1e1245880 0 starting mon.a rank 0 at public addrs [v2:172.21.15.89:3300/0,v1:172.21.15.89:6789/0] at bind addrs [v2:172.21.15.89:3300/0,v1:172.21.15.89:6789/0] mon_data /var/lib/ceph/mon/ceph-a fsid f1db3478-0243-11ee-9b1d-001a4aab830c 2023-06-03T19:23:01.577 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:23:01 smithi089 bash[26572]: debug 2023-06-03T19:23:01.330+0000 7fc1e1245880 1 mon.a@-1(???) e1 preinit fsid f1db3478-0243-11ee-9b1d-001a4aab830c 2023-06-03T19:23:01.577 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:23:01 smithi089 bash[26572]: debug 2023-06-03T19:23:01.330+0000 7fc1e1245880 0 mon.a@-1(???).mds e1 new map 2023-06-03T19:23:01.577 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:23:01 smithi089 bash[26572]: debug 2023-06-03T19:23:01.330+0000 7fc1e1245880 0 mon.a@-1(???).mds e1 print_map 2023-06-03T19:23:01.577 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:23:01 smithi089 bash[26572]: e1 2023-06-03T19:23:01.577 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:23:01 smithi089 bash[26572]: enable_multiple, ever_enabled_multiple: 1,1 2023-06-03T19:23:01.578 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:23:01 smithi089 bash[26572]: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2023-06-03T19:23:01.578 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:23:01 smithi089 bash[26572]: legacy client fscid: -1 2023-06-03T19:23:01.578 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:23:01 smithi089 bash[26572]: 2023-06-03T19:23:01.578 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:23:01 smithi089 bash[26572]: No filesystems configured 2023-06-03T19:23:01.578 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:23:01 smithi089 bash[26572]: debug 2023-06-03T19:23:01.330+0000 7fc1e1245880 0 mon.a@-1(???).osd e1 crush map has features 3314932999778484224, adjusting msgr requires 2023-06-03T19:23:01.579 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:23:01 smithi089 bash[26572]: debug 2023-06-03T19:23:01.330+0000 7fc1e1245880 0 mon.a@-1(???).osd e1 crush map has features 288514050185494528, adjusting msgr requires 2023-06-03T19:23:01.579 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:23:01 smithi089 bash[26572]: debug 2023-06-03T19:23:01.330+0000 7fc1e1245880 0 mon.a@-1(???).osd e1 crush map has features 288514050185494528, adjusting msgr requires 2023-06-03T19:23:01.579 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:23:01 smithi089 bash[26572]: debug 2023-06-03T19:23:01.330+0000 7fc1e1245880 0 mon.a@-1(???).osd e1 crush map has features 288514050185494528, adjusting msgr requires 2023-06-03T19:23:01.579 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:23:01 smithi089 bash[26572]: audit 2023-06-03T19:22:58.255260+0000 mon.a (mon.0) 10 : audit [DBG] from='client.? 172.21.15.89:0/2196192529' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-03T19:23:01.579 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:23:01 smithi089 bash[26572]: debug 2023-06-03T19:23:01.330+0000 7fc1e1245880 1 mon.a@-1(???).paxosservice(auth 1..2) refresh upgraded, format 0 -> 3 2023-06-03T19:23:01.579 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:23:01 smithi089 bash[26572]: cluster 2023-06-03T19:23:01.357705+0000 mon.a (mon.0) 1 : cluster [INF] mon.a is new leader, mons a in quorum (ranks 0) 2023-06-03T19:23:01.580 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:23:01 smithi089 bash[26572]: cluster 2023-06-03T19:23:01.357861+0000 mon.a (mon.0) 2 : cluster [DBG] monmap e1: 1 mons at {a=[v2:172.21.15.89:3300/0,v1:172.21.15.89:6789/0]} removed_ranks: {} 2023-06-03T19:23:01.580 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:23:01 smithi089 bash[26572]: cluster 2023-06-03T19:23:01.358891+0000 mon.a (mon.0) 3 : cluster [DBG] fsmap 2023-06-03T19:23:01.580 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:23:01 smithi089 bash[26572]: cluster 2023-06-03T19:23:01.358971+0000 mon.a (mon.0) 4 : cluster [DBG] osdmap e1: 0 total, 0 up, 0 in 2023-06-03T19:23:01.580 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:23:01 smithi089 bash[26572]: cluster 2023-06-03T19:23:01.359750+0000 mon.a (mon.0) 5 : cluster [DBG] mgrmap e1: no daemons active 2023-06-03T19:23:02.314 INFO:teuthology.orchestra.run.smithi089.stderr:Wrote config to /etc/ceph/ceph.conf 2023-06-03T19:23:02.314 INFO:teuthology.orchestra.run.smithi089.stderr:Wrote keyring to /etc/ceph/ceph.client.admin.keyring 2023-06-03T19:23:02.314 INFO:teuthology.orchestra.run.smithi089.stderr:Creating mgr... 2023-06-03T19:23:02.314 INFO:teuthology.orchestra.run.smithi089.stderr:Verifying port 9283 ... 2023-06-03T19:23:02.536 INFO:teuthology.orchestra.run.smithi089.stderr:systemctl: Failed to reset failed state of unit ceph-f1db3478-0243-11ee-9b1d-001a4aab830c@mgr.y.service: Unit ceph-f1db3478-0243-11ee-9b1d-001a4aab830c@mgr.y.service not loaded. 2023-06-03T19:23:02.544 INFO:teuthology.orchestra.run.smithi089.stderr:systemctl: Created symlink /etc/systemd/system/ceph-f1db3478-0243-11ee-9b1d-001a4aab830c.target.wants/ceph-f1db3478-0243-11ee-9b1d-001a4aab830c@mgr.y.service -> /etc/systemd/system/ceph-f1db3478-0243-11ee-9b1d-001a4aab830c@.service. 2023-06-03T19:23:02.773 INFO:teuthology.orchestra.run.smithi089.stderr:firewalld does not appear to be present 2023-06-03T19:23:02.774 INFO:teuthology.orchestra.run.smithi089.stderr:Not possible to enable service . firewalld.service is not available 2023-06-03T19:23:02.774 INFO:teuthology.orchestra.run.smithi089.stderr:firewalld does not appear to be present 2023-06-03T19:23:02.774 INFO:teuthology.orchestra.run.smithi089.stderr:Not possible to open ports <[9283]>. firewalld.service is not available 2023-06-03T19:23:02.774 INFO:teuthology.orchestra.run.smithi089.stderr:Waiting for mgr to start... 2023-06-03T19:23:02.775 INFO:teuthology.orchestra.run.smithi089.stderr:Waiting for mgr... 2023-06-03T19:23:03.033 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:23:02 smithi089 bash[26572]: audit 2023-06-03T19:23:01.580393+0000 mon.a (mon.0) 6 : audit [INF] from='client.? 172.21.15.89:0/2840325856' entity='client.admin' 2023-06-03T19:23:04.445 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: 2023-06-03T19:23:04.445 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: { 2023-06-03T19:23:04.445 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: "fsid": "f1db3478-0243-11ee-9b1d-001a4aab830c", 2023-06-03T19:23:04.446 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: "health": { 2023-06-03T19:23:04.446 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: "status": "HEALTH_OK", 2023-06-03T19:23:04.446 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: "checks": {}, 2023-06-03T19:23:04.446 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: "mutes": [] 2023-06-03T19:23:04.446 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: }, 2023-06-03T19:23:04.446 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: "election_epoch": 5, 2023-06-03T19:23:04.446 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: "quorum": [ 2023-06-03T19:23:04.446 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: 0 2023-06-03T19:23:04.447 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: ], 2023-06-03T19:23:04.447 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: "quorum_names": [ 2023-06-03T19:23:04.447 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: "a" 2023-06-03T19:23:04.447 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: ], 2023-06-03T19:23:04.447 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: "quorum_age": 3, 2023-06-03T19:23:04.447 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: "monmap": { 2023-06-03T19:23:04.447 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: "epoch": 1, 2023-06-03T19:23:04.447 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: "min_mon_release_name": "pacific", 2023-06-03T19:23:04.448 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: "num_mons": 1 2023-06-03T19:23:04.448 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: }, 2023-06-03T19:23:04.448 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: "osdmap": { 2023-06-03T19:23:04.448 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: "epoch": 1, 2023-06-03T19:23:04.448 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: "num_osds": 0, 2023-06-03T19:23:04.448 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: "num_up_osds": 0, 2023-06-03T19:23:04.448 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: "osd_up_since": 0, 2023-06-03T19:23:04.449 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: "num_in_osds": 0, 2023-06-03T19:23:04.451 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: "osd_in_since": 0, 2023-06-03T19:23:04.451 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: "num_remapped_pgs": 0 2023-06-03T19:23:04.451 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: }, 2023-06-03T19:23:04.452 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: "pgmap": { 2023-06-03T19:23:04.452 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: "pgs_by_state": [], 2023-06-03T19:23:04.452 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: "num_pgs": 0, 2023-06-03T19:23:04.452 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: "num_pools": 0, 2023-06-03T19:23:04.452 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: "num_objects": 0, 2023-06-03T19:23:04.452 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: "data_bytes": 0, 2023-06-03T19:23:04.452 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: "bytes_used": 0, 2023-06-03T19:23:04.453 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: "bytes_avail": 0, 2023-06-03T19:23:04.453 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: "bytes_total": 0 2023-06-03T19:23:04.453 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: }, 2023-06-03T19:23:04.453 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: "fsmap": { 2023-06-03T19:23:04.453 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: "epoch": 1, 2023-06-03T19:23:04.453 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: "by_rank": [], 2023-06-03T19:23:04.453 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: "up:standby": 0 2023-06-03T19:23:04.453 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: }, 2023-06-03T19:23:04.454 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: "mgrmap": { 2023-06-03T19:23:04.454 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: "available": false, 2023-06-03T19:23:04.454 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: "num_standbys": 0, 2023-06-03T19:23:04.454 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: "modules": [ 2023-06-03T19:23:04.454 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: "iostat", 2023-06-03T19:23:04.454 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: "nfs", 2023-06-03T19:23:04.454 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: "restful" 2023-06-03T19:23:04.454 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: ], 2023-06-03T19:23:04.455 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: "services": {} 2023-06-03T19:23:04.455 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: }, 2023-06-03T19:23:04.455 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: "servicemap": { 2023-06-03T19:23:04.455 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: "epoch": 1, 2023-06-03T19:23:04.455 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: "modified": "2023-06-03T19:22:54.447760+0000", 2023-06-03T19:23:04.455 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: "services": {} 2023-06-03T19:23:04.455 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: }, 2023-06-03T19:23:04.456 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: "progress_events": {} 2023-06-03T19:23:04.456 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: } 2023-06-03T19:23:04.783 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:23:04 smithi089 bash[26572]: audit 2023-06-03T19:23:04.433835+0000 mon.a (mon.0) 7 : audit [DBG] from='client.? 172.21.15.89:0/2505748734' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2023-06-03T19:23:04.783 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 19:23:04 smithi089 bash[26845]: debug 2023-06-03T19:23:04.474+0000 7f5cf3c43680 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2023-06-03T19:23:05.056 INFO:teuthology.orchestra.run.smithi089.stderr:mgr not available, waiting (1/15)... 2023-06-03T19:23:05.283 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 19:23:04 smithi089 bash[26845]: debug 2023-06-03T19:23:04.954+0000 7f5cf3c43680 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2023-06-03T19:23:06.265 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 19:23:05 smithi089 bash[26845]: debug 2023-06-03T19:23:05.906+0000 7f5cf3c43680 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2023-06-03T19:23:06.265 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 19:23:06 smithi089 bash[26845]: debug 2023-06-03T19:23:06.002+0000 7f5cf3c43680 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2023-06-03T19:23:06.533 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 19:23:06 smithi089 bash[26845]: debug 2023-06-03T19:23:06.250+0000 7f5cf3c43680 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2023-06-03T19:23:07.645 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 19:23:07 smithi089 bash[26845]: debug 2023-06-03T19:23:07.378+0000 7f5cf3c43680 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2023-06-03T19:23:07.645 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 19:23:07 smithi089 bash[26845]: debug 2023-06-03T19:23:07.538+0000 7f5cf3c43680 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2023-06-03T19:23:07.981 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 19:23:07 smithi089 bash[26845]: debug 2023-06-03T19:23:07.630+0000 7f5cf3c43680 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2023-06-03T19:23:07.982 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 19:23:07 smithi089 bash[26845]: debug 2023-06-03T19:23:07.746+0000 7f5cf3c43680 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2023-06-03T19:23:08.283 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 19:23:07 smithi089 bash[26845]: debug 2023-06-03T19:23:07.970+0000 7f5cf3c43680 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2023-06-03T19:23:08.460 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: 2023-06-03T19:23:08.461 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: { 2023-06-03T19:23:08.461 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: "fsid": "f1db3478-0243-11ee-9b1d-001a4aab830c", 2023-06-03T19:23:08.462 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: "health": { 2023-06-03T19:23:08.463 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: "status": "HEALTH_OK", 2023-06-03T19:23:08.463 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: "checks": {}, 2023-06-03T19:23:08.463 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: "mutes": [] 2023-06-03T19:23:08.463 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: }, 2023-06-03T19:23:08.463 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: "election_epoch": 5, 2023-06-03T19:23:08.463 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: "quorum": [ 2023-06-03T19:23:08.464 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: 0 2023-06-03T19:23:08.464 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: ], 2023-06-03T19:23:08.464 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: "quorum_names": [ 2023-06-03T19:23:08.464 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: "a" 2023-06-03T19:23:08.464 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: ], 2023-06-03T19:23:08.464 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: "quorum_age": 7, 2023-06-03T19:23:08.464 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: "monmap": { 2023-06-03T19:23:08.465 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: "epoch": 1, 2023-06-03T19:23:08.465 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: "min_mon_release_name": "pacific", 2023-06-03T19:23:08.465 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: "num_mons": 1 2023-06-03T19:23:08.465 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: }, 2023-06-03T19:23:08.468 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: "osdmap": { 2023-06-03T19:23:08.468 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: "epoch": 1, 2023-06-03T19:23:08.468 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: "num_osds": 0, 2023-06-03T19:23:08.468 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: "num_up_osds": 0, 2023-06-03T19:23:08.468 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: "osd_up_since": 0, 2023-06-03T19:23:08.468 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: "num_in_osds": 0, 2023-06-03T19:23:08.469 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: "osd_in_since": 0, 2023-06-03T19:23:08.469 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: "num_remapped_pgs": 0 2023-06-03T19:23:08.469 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: }, 2023-06-03T19:23:08.469 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: "pgmap": { 2023-06-03T19:23:08.469 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: "pgs_by_state": [], 2023-06-03T19:23:08.469 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: "num_pgs": 0, 2023-06-03T19:23:08.469 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: "num_pools": 0, 2023-06-03T19:23:08.470 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: "num_objects": 0, 2023-06-03T19:23:08.470 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: "data_bytes": 0, 2023-06-03T19:23:08.470 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: "bytes_used": 0, 2023-06-03T19:23:08.470 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: "bytes_avail": 0, 2023-06-03T19:23:08.470 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: "bytes_total": 0 2023-06-03T19:23:08.470 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: }, 2023-06-03T19:23:08.470 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: "fsmap": { 2023-06-03T19:23:08.470 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: "epoch": 1, 2023-06-03T19:23:08.471 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: "by_rank": [], 2023-06-03T19:23:08.471 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: "up:standby": 0 2023-06-03T19:23:08.471 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: }, 2023-06-03T19:23:08.471 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: "mgrmap": { 2023-06-03T19:23:08.471 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: "available": false, 2023-06-03T19:23:08.471 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: "num_standbys": 0, 2023-06-03T19:23:08.471 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: "modules": [ 2023-06-03T19:23:08.471 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: "iostat", 2023-06-03T19:23:08.472 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: "nfs", 2023-06-03T19:23:08.472 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: "restful" 2023-06-03T19:23:08.472 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: ], 2023-06-03T19:23:08.472 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: "services": {} 2023-06-03T19:23:08.472 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: }, 2023-06-03T19:23:08.472 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: "servicemap": { 2023-06-03T19:23:08.472 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: "epoch": 1, 2023-06-03T19:23:08.472 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: "modified": "2023-06-03T19:22:54.447760+0000", 2023-06-03T19:23:08.473 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: "services": {} 2023-06-03T19:23:08.473 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: }, 2023-06-03T19:23:08.473 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: "progress_events": {} 2023-06-03T19:23:08.473 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: } 2023-06-03T19:23:08.633 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:23:08 smithi089 bash[26572]: audit 2023-06-03T19:23:08.450778+0000 mon.a (mon.0) 8 : audit [DBG] from='client.? 172.21.15.89:0/690298392' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2023-06-03T19:23:08.634 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 19:23:08 smithi089 bash[26845]: debug 2023-06-03T19:23:08.334+0000 7f5cf3c43680 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2023-06-03T19:23:08.634 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 19:23:08 smithi089 bash[26845]: debug 2023-06-03T19:23:08.458+0000 7f5cf3c43680 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2023-06-03T19:23:09.055 INFO:teuthology.orchestra.run.smithi089.stderr:mgr not available, waiting (2/15)... 2023-06-03T19:23:09.783 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 19:23:09 smithi089 bash[26845]: debug 2023-06-03T19:23:09.338+0000 7f5cf3c43680 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2023-06-03T19:23:09.783 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 19:23:09 smithi089 bash[26845]: debug 2023-06-03T19:23:09.438+0000 7f5cf3c43680 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2023-06-03T19:23:09.783 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 19:23:09 smithi089 bash[26845]: debug 2023-06-03T19:23:09.554+0000 7f5cf3c43680 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2023-06-03T19:23:10.082 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 19:23:09 smithi089 bash[26845]: debug 2023-06-03T19:23:09.778+0000 7f5cf3c43680 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2023-06-03T19:23:10.413 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 19:23:10 smithi089 bash[26845]: debug 2023-06-03T19:23:10.070+0000 7f5cf3c43680 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2023-06-03T19:23:10.924 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 19:23:10 smithi089 bash[26845]: debug 2023-06-03T19:23:10.234+0000 7f5cf3c43680 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2023-06-03T19:23:10.925 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 19:23:10 smithi089 bash[26845]: debug 2023-06-03T19:23:10.398+0000 7f5cf3c43680 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2023-06-03T19:23:10.925 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 19:23:10 smithi089 bash[26845]: debug 2023-06-03T19:23:10.610+0000 7f5cf3c43680 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2023-06-03T19:23:11.033 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 19:23:10 smithi089 bash[26845]: debug 2023-06-03T19:23:10.718+0000 7f5cf3c43680 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2023-06-03T19:23:11.034 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:23:10 smithi089 bash[26572]: cluster 2023-06-03T19:23:10.743190+0000 mon.a (mon.0) 9 : cluster [INF] Activating manager daemon y 2023-06-03T19:23:11.034 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:23:10 smithi089 bash[26572]: cluster 2023-06-03T19:23:10.748978+0000 mon.a (mon.0) 10 : cluster [DBG] mgrmap e2: y(active, starting, since 0.00598247s) 2023-06-03T19:23:11.035 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:23:10 smithi089 bash[26572]: audit 2023-06-03T19:23:10.750738+0000 mon.a (mon.0) 11 : audit [DBG] from='mgr.14100 172.21.15.89:0/1747588390' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2023-06-03T19:23:11.035 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:23:10 smithi089 bash[26572]: audit 2023-06-03T19:23:10.751003+0000 mon.a (mon.0) 12 : audit [DBG] from='mgr.14100 172.21.15.89:0/1747588390' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2023-06-03T19:23:11.035 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:23:10 smithi089 bash[26572]: audit 2023-06-03T19:23:10.751145+0000 mon.a (mon.0) 13 : audit [DBG] from='mgr.14100 172.21.15.89:0/1747588390' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2023-06-03T19:23:11.035 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:23:10 smithi089 bash[26572]: audit 2023-06-03T19:23:10.751424+0000 mon.a (mon.0) 14 : audit [DBG] from='mgr.14100 172.21.15.89:0/1747588390' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-06-03T19:23:11.035 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:23:10 smithi089 bash[26572]: audit 2023-06-03T19:23:10.752008+0000 mon.a (mon.0) 15 : audit [DBG] from='mgr.14100 172.21.15.89:0/1747588390' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2023-06-03T19:23:11.036 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:23:10 smithi089 bash[26572]: cluster 2023-06-03T19:23:10.776418+0000 mon.a (mon.0) 16 : cluster [INF] Manager daemon y is now available 2023-06-03T19:23:11.036 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:23:10 smithi089 bash[26572]: audit 2023-06-03T19:23:10.787862+0000 mon.a (mon.0) 17 : audit [INF] from='mgr.14100 172.21.15.89:0/1747588390' entity='mgr.y' 2023-06-03T19:23:12.283 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:23:11 smithi089 bash[26572]: audit 2023-06-03T19:23:10.800166+0000 mon.a (mon.0) 18 : audit [INF] from='mgr.14100 172.21.15.89:0/1747588390' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2023-06-03T19:23:12.283 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:23:11 smithi089 bash[26572]: audit 2023-06-03T19:23:10.802057+0000 mon.a (mon.0) 19 : audit [INF] from='mgr.14100 172.21.15.89:0/1747588390' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2023-06-03T19:23:12.283 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:23:11 smithi089 bash[26572]: audit 2023-06-03T19:23:10.810390+0000 mon.a (mon.0) 20 : audit [INF] from='mgr.14100 172.21.15.89:0/1747588390' entity='mgr.y' 2023-06-03T19:23:12.284 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:23:11 smithi089 bash[26572]: audit 2023-06-03T19:23:10.812935+0000 mon.a (mon.0) 21 : audit [INF] from='mgr.14100 172.21.15.89:0/1747588390' entity='mgr.y' 2023-06-03T19:23:12.284 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:23:11 smithi089 bash[26572]: cluster 2023-06-03T19:23:11.752981+0000 mon.a (mon.0) 22 : cluster [DBG] mgrmap e3: y(active, since 1.00999s) 2023-06-03T19:23:12.640 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: 2023-06-03T19:23:12.640 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: { 2023-06-03T19:23:12.641 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: "fsid": "f1db3478-0243-11ee-9b1d-001a4aab830c", 2023-06-03T19:23:12.641 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: "health": { 2023-06-03T19:23:12.641 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: "status": "HEALTH_OK", 2023-06-03T19:23:12.641 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: "checks": {}, 2023-06-03T19:23:12.641 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: "mutes": [] 2023-06-03T19:23:12.641 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: }, 2023-06-03T19:23:12.641 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: "election_epoch": 5, 2023-06-03T19:23:12.642 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: "quorum": [ 2023-06-03T19:23:12.642 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: 0 2023-06-03T19:23:12.642 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: ], 2023-06-03T19:23:12.642 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: "quorum_names": [ 2023-06-03T19:23:12.642 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: "a" 2023-06-03T19:23:12.642 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: ], 2023-06-03T19:23:12.642 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: "quorum_age": 11, 2023-06-03T19:23:12.643 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: "monmap": { 2023-06-03T19:23:12.644 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: "epoch": 1, 2023-06-03T19:23:12.644 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: "min_mon_release_name": "pacific", 2023-06-03T19:23:12.644 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: "num_mons": 1 2023-06-03T19:23:12.644 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: }, 2023-06-03T19:23:12.644 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: "osdmap": { 2023-06-03T19:23:12.644 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: "epoch": 1, 2023-06-03T19:23:12.644 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: "num_osds": 0, 2023-06-03T19:23:12.644 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: "num_up_osds": 0, 2023-06-03T19:23:12.645 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: "osd_up_since": 0, 2023-06-03T19:23:12.645 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: "num_in_osds": 0, 2023-06-03T19:23:12.647 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: "osd_in_since": 0, 2023-06-03T19:23:12.647 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: "num_remapped_pgs": 0 2023-06-03T19:23:12.647 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: }, 2023-06-03T19:23:12.648 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: "pgmap": { 2023-06-03T19:23:12.648 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: "pgs_by_state": [], 2023-06-03T19:23:12.648 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: "num_pgs": 0, 2023-06-03T19:23:12.648 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: "num_pools": 0, 2023-06-03T19:23:12.648 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: "num_objects": 0, 2023-06-03T19:23:12.648 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: "data_bytes": 0, 2023-06-03T19:23:12.648 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: "bytes_used": 0, 2023-06-03T19:23:12.649 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: "bytes_avail": 0, 2023-06-03T19:23:12.649 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: "bytes_total": 0 2023-06-03T19:23:12.649 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: }, 2023-06-03T19:23:12.649 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: "fsmap": { 2023-06-03T19:23:12.649 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: "epoch": 1, 2023-06-03T19:23:12.649 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: "by_rank": [], 2023-06-03T19:23:12.649 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: "up:standby": 0 2023-06-03T19:23:12.649 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: }, 2023-06-03T19:23:12.650 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: "mgrmap": { 2023-06-03T19:23:12.650 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: "available": true, 2023-06-03T19:23:12.650 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: "num_standbys": 0, 2023-06-03T19:23:12.650 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: "modules": [ 2023-06-03T19:23:12.650 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: "iostat", 2023-06-03T19:23:12.650 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: "nfs", 2023-06-03T19:23:12.650 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: "restful" 2023-06-03T19:23:12.650 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: ], 2023-06-03T19:23:12.651 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: "services": {} 2023-06-03T19:23:12.651 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: }, 2023-06-03T19:23:12.651 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: "servicemap": { 2023-06-03T19:23:12.651 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: "epoch": 1, 2023-06-03T19:23:12.651 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: "modified": "2023-06-03T19:22:54.447760+0000", 2023-06-03T19:23:12.651 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: "services": {} 2023-06-03T19:23:12.651 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: }, 2023-06-03T19:23:12.651 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: "progress_events": {} 2023-06-03T19:23:12.652 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: } 2023-06-03T19:23:13.033 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:23:12 smithi089 bash[26572]: audit 2023-06-03T19:23:12.630226+0000 mon.a (mon.0) 23 : audit [DBG] from='client.? 172.21.15.89:0/1189482447' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2023-06-03T19:23:13.310 INFO:teuthology.orchestra.run.smithi089.stderr:mgr is available 2023-06-03T19:23:14.080 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:23:13 smithi089 bash[26572]: cluster 2023-06-03T19:23:12.805203+0000 mon.a (mon.0) 24 : cluster [DBG] mgrmap e4: y(active, since 2s) 2023-06-03T19:23:14.675 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: 2023-06-03T19:23:14.675 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: [global] 2023-06-03T19:23:14.676 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: fsid = f1db3478-0243-11ee-9b1d-001a4aab830c 2023-06-03T19:23:14.676 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: log_to_journald = false 2023-06-03T19:23:14.676 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: mon_osd_allow_pg_remap = true 2023-06-03T19:23:14.676 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: mon_osd_allow_primary_affinity = true 2023-06-03T19:23:14.676 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: mon_warn_on_no_sortbitwise = false 2023-06-03T19:23:14.676 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: osd_crush_chooseleaf_type = 0 2023-06-03T19:23:14.677 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: 2023-06-03T19:23:14.677 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: [mgr] 2023-06-03T19:23:14.677 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: mgr/telemetry/nag = false 2023-06-03T19:23:14.677 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: 2023-06-03T19:23:14.677 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: [osd] 2023-06-03T19:23:14.677 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: osd_map_max_advance = 10 2023-06-03T19:23:14.677 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: osd_sloppy_crc = true 2023-06-03T19:23:15.033 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:23:14 smithi089 bash[26572]: audit 2023-06-03T19:23:14.663272+0000 mon.a (mon.0) 25 : audit [INF] from='client.? 172.21.15.89:0/3124731704' entity='client.admin' cmd=[{"prefix": "config assimilate-conf"}]: dispatch 2023-06-03T19:23:15.333 INFO:teuthology.orchestra.run.smithi089.stderr:Enabling cephadm module... 2023-06-03T19:23:17.675 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:23:17 smithi089 bash[26572]: audit 2023-06-03T19:23:16.768493+0000 mon.a (mon.0) 26 : audit [INF] from='client.? 172.21.15.89:0/3075501863' entity='client.admin' cmd=[{"prefix": "mgr module enable", "module": "cephadm"}]: dispatch 2023-06-03T19:23:17.675 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 19:23:17 smithi089 bash[26845]: ignoring --setuser ceph since I am not root 2023-06-03T19:23:17.675 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 19:23:17 smithi089 bash[26845]: ignoring --setgroup ceph since I am not root 2023-06-03T19:23:17.676 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 19:23:17 smithi089 bash[26845]: debug 2023-06-03T19:23:17.554+0000 7f5226547680 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2023-06-03T19:23:18.360 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 19:23:17 smithi089 bash[26845]: debug 2023-06-03T19:23:17.662+0000 7f5226547680 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2023-06-03T19:23:18.361 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 19:23:18 smithi089 bash[26845]: debug 2023-06-03T19:23:18.122+0000 7f5226547680 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2023-06-03T19:23:18.783 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:23:18 smithi089 bash[26572]: audit 2023-06-03T19:23:17.355128+0000 mon.a (mon.0) 27 : audit [INF] from='client.? 172.21.15.89:0/3075501863' entity='client.admin' cmd='[{"prefix": "mgr module enable", "module": "cephadm"}]': finished 2023-06-03T19:23:18.784 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:23:18 smithi089 bash[26572]: cluster 2023-06-03T19:23:17.355213+0000 mon.a (mon.0) 28 : cluster [DBG] mgrmap e5: y(active, since 6s) 2023-06-03T19:23:19.283 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 19:23:19 smithi089 bash[26845]: debug 2023-06-03T19:23:19.033+0000 7f5226547680 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2023-06-03T19:23:19.283 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 19:23:19 smithi089 bash[26845]: debug 2023-06-03T19:23:19.129+0000 7f5226547680 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2023-06-03T19:23:19.660 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: { 2023-06-03T19:23:19.660 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: "epoch": 5, 2023-06-03T19:23:19.661 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: "available": true, 2023-06-03T19:23:19.661 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: "active_name": "y", 2023-06-03T19:23:19.661 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: "num_standby": 0 2023-06-03T19:23:19.661 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: } 2023-06-03T19:23:19.690 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 19:23:19 smithi089 bash[26845]: debug 2023-06-03T19:23:19.389+0000 7f5226547680 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2023-06-03T19:23:20.033 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:23:19 smithi089 bash[26572]: audit 2023-06-03T19:23:19.650358+0000 mon.a (mon.0) 29 : audit [DBG] from='client.? 172.21.15.89:0/2124689660' entity='client.admin' cmd=[{"prefix": "mgr stat"}]: dispatch 2023-06-03T19:23:20.309 INFO:teuthology.orchestra.run.smithi089.stderr:Waiting for the mgr to restart... 2023-06-03T19:23:20.309 INFO:teuthology.orchestra.run.smithi089.stderr:Waiting for mgr epoch 5... 2023-06-03T19:23:20.707 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 19:23:20 smithi089 bash[26845]: debug 2023-06-03T19:23:20.529+0000 7f5226547680 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2023-06-03T19:23:21.033 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 19:23:20 smithi089 bash[26845]: debug 2023-06-03T19:23:20.693+0000 7f5226547680 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2023-06-03T19:23:21.033 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 19:23:20 smithi089 bash[26845]: debug 2023-06-03T19:23:20.789+0000 7f5226547680 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2023-06-03T19:23:21.033 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 19:23:20 smithi089 bash[26845]: debug 2023-06-03T19:23:20.901+0000 7f5226547680 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2023-06-03T19:23:21.515 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 19:23:21 smithi089 bash[26845]: debug 2023-06-03T19:23:21.129+0000 7f5226547680 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2023-06-03T19:23:21.783 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 19:23:21 smithi089 bash[26845]: debug 2023-06-03T19:23:21.501+0000 7f5226547680 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2023-06-03T19:23:21.783 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 19:23:21 smithi089 bash[26845]: debug 2023-06-03T19:23:21.621+0000 7f5226547680 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2023-06-03T19:23:22.783 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 19:23:22 smithi089 bash[26845]: debug 2023-06-03T19:23:22.481+0000 7f5226547680 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2023-06-03T19:23:22.783 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 19:23:22 smithi089 bash[26845]: debug 2023-06-03T19:23:22.589+0000 7f5226547680 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2023-06-03T19:23:22.783 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 19:23:22 smithi089 bash[26845]: debug 2023-06-03T19:23:22.701+0000 7f5226547680 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2023-06-03T19:23:23.230 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 19:23:22 smithi089 bash[26845]: debug 2023-06-03T19:23:22.925+0000 7f5226547680 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2023-06-03T19:23:23.533 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 19:23:23 smithi089 bash[26845]: debug 2023-06-03T19:23:23.217+0000 7f5226547680 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2023-06-03T19:23:23.533 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 19:23:23 smithi089 bash[26845]: debug 2023-06-03T19:23:23.381+0000 7f5226547680 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2023-06-03T19:23:23.890 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 19:23:23 smithi089 bash[26845]: debug 2023-06-03T19:23:23.553+0000 7f5226547680 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2023-06-03T19:23:23.890 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 19:23:23 smithi089 bash[26845]: debug 2023-06-03T19:23:23.769+0000 7f5226547680 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2023-06-03T19:23:24.283 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 19:23:23 smithi089 bash[26845]: debug 2023-06-03T19:23:23.877+0000 7f5226547680 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2023-06-03T19:23:24.987 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: { 2023-06-03T19:23:24.988 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: "mgrmap_epoch": 7, 2023-06-03T19:23:24.988 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: "initialized": true 2023-06-03T19:23:24.988 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: } 2023-06-03T19:23:25.182 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:23:24 smithi089 bash[26572]: cluster 2023-06-03T19:23:23.903255+0000 mon.a (mon.0) 30 : cluster [INF] Active manager daemon y restarted 2023-06-03T19:23:25.182 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:23:24 smithi089 bash[26572]: cluster 2023-06-03T19:23:23.904157+0000 mon.a (mon.0) 31 : cluster [INF] Activating manager daemon y 2023-06-03T19:23:25.182 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:23:24 smithi089 bash[26572]: cluster 2023-06-03T19:23:23.907387+0000 mon.a (mon.0) 32 : cluster [DBG] osdmap e2: 0 total, 0 up, 0 in 2023-06-03T19:23:25.182 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:23:24 smithi089 bash[26572]: cluster 2023-06-03T19:23:23.958629+0000 mon.a (mon.0) 33 : cluster [DBG] mgrmap e6: y(active, starting, since 0.0546363s) 2023-06-03T19:23:25.182 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:23:24 smithi089 bash[26572]: audit 2023-06-03T19:23:23.960580+0000 mon.a (mon.0) 34 : audit [DBG] from='mgr.14116 172.21.15.89:0/1027675781' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-06-03T19:23:25.183 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:23:24 smithi089 bash[26572]: audit 2023-06-03T19:23:23.962420+0000 mon.a (mon.0) 35 : audit [DBG] from='mgr.14116 172.21.15.89:0/1027675781' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2023-06-03T19:23:25.183 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:23:24 smithi089 bash[26572]: audit 2023-06-03T19:23:23.963743+0000 mon.a (mon.0) 36 : audit [DBG] from='mgr.14116 172.21.15.89:0/1027675781' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2023-06-03T19:23:25.183 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:23:24 smithi089 bash[26572]: audit 2023-06-03T19:23:23.964101+0000 mon.a (mon.0) 37 : audit [DBG] from='mgr.14116 172.21.15.89:0/1027675781' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2023-06-03T19:23:25.183 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:23:24 smithi089 bash[26572]: audit 2023-06-03T19:23:23.964379+0000 mon.a (mon.0) 38 : audit [DBG] from='mgr.14116 172.21.15.89:0/1027675781' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2023-06-03T19:23:25.183 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:23:24 smithi089 bash[26572]: cluster 2023-06-03T19:23:23.988567+0000 mon.a (mon.0) 39 : cluster [INF] Manager daemon y is now available 2023-06-03T19:23:25.183 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:23:24 smithi089 bash[26572]: cephadm 2023-06-03T19:23:24.005454+0000 mgr.y (mgr.14116) 1 : cephadm [INF] Found migration_current of "None". Setting to last migration. 2023-06-03T19:23:25.183 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:23:24 smithi089 bash[26572]: audit 2023-06-03T19:23:24.008560+0000 mon.a (mon.0) 40 : audit [INF] from='mgr.14116 172.21.15.89:0/1027675781' entity='mgr.y' 2023-06-03T19:23:25.184 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:23:24 smithi089 bash[26572]: audit 2023-06-03T19:23:24.128064+0000 mon.a (mon.0) 41 : audit [INF] from='mgr.14116 172.21.15.89:0/1027675781' entity='mgr.y' 2023-06-03T19:23:25.184 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:23:24 smithi089 bash[26572]: audit 2023-06-03T19:23:24.130068+0000 mon.a (mon.0) 42 : audit [DBG] from='mgr.14116 172.21.15.89:0/1027675781' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-03T19:23:25.184 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:23:24 smithi089 bash[26572]: audit 2023-06-03T19:23:24.133639+0000 mon.a (mon.0) 43 : audit [DBG] from='mgr.14116 172.21.15.89:0/1027675781' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-03T19:23:25.184 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:23:24 smithi089 bash[26572]: audit 2023-06-03T19:23:24.137506+0000 mon.a (mon.0) 44 : audit [DBG] from='mgr.14116 172.21.15.89:0/1027675781' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-03T19:23:25.184 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:23:24 smithi089 bash[26572]: audit 2023-06-03T19:23:24.158399+0000 mon.a (mon.0) 45 : audit [INF] from='mgr.14116 172.21.15.89:0/1027675781' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2023-06-03T19:23:25.184 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:23:24 smithi089 bash[26572]: audit 2023-06-03T19:23:24.160365+0000 mon.a (mon.0) 46 : audit [INF] from='mgr.14116 172.21.15.89:0/1027675781' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2023-06-03T19:23:25.726 INFO:teuthology.orchestra.run.smithi089.stderr:mgr epoch 5 is available 2023-06-03T19:23:25.727 INFO:teuthology.orchestra.run.smithi089.stderr:Setting orchestrator backend to cephadm... 2023-06-03T19:23:26.034 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:23:25 smithi089 bash[26572]: audit 2023-06-03T19:23:24.963821+0000 mgr.y (mgr.14116) 2 : audit [DBG] from='client.14120 -' entity='client.admin' cmd=[{"prefix": "get_command_descriptions"}]: dispatch 2023-06-03T19:23:26.034 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:23:25 smithi089 bash[26572]: cluster 2023-06-03T19:23:24.965400+0000 mon.a (mon.0) 47 : cluster [DBG] mgrmap e7: y(active, since 1.06141s) 2023-06-03T19:23:26.034 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:23:25 smithi089 bash[26572]: audit 2023-06-03T19:23:24.978036+0000 mgr.y (mgr.14116) 3 : audit [DBG] from='client.14120 -' entity='client.admin' cmd=[{"prefix": "mgr_status"}]: dispatch 2023-06-03T19:23:27.243 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:23:26 smithi089 bash[26572]: cluster 2023-06-03T19:23:25.965553+0000 mon.a (mon.0) 48 : cluster [DBG] mgrmap e8: y(active, since 2s) 2023-06-03T19:23:28.533 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:23:28 smithi089 bash[26572]: audit 2023-06-03T19:23:27.192235+0000 mgr.y (mgr.14116) 4 : audit [DBG] from='client.14126 -' entity='client.admin' cmd=[{"prefix": "orch set backend", "module_name": "cephadm", "target": ["mon-mgr", ""]}]: dispatch 2023-06-03T19:23:28.533 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:23:28 smithi089 bash[26572]: audit 2023-06-03T19:23:27.204212+0000 mon.a (mon.0) 49 : audit [INF] from='mgr.14116 172.21.15.89:0/1027675781' entity='mgr.y' 2023-06-03T19:23:28.534 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:23:28 smithi089 bash[26572]: audit 2023-06-03T19:23:27.215943+0000 mon.a (mon.0) 50 : audit [DBG] from='mgr.14116 172.21.15.89:0/1027675781' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-03T19:23:29.278 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: value unchanged 2023-06-03T19:23:29.783 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:23:29 smithi089 bash[26572]: audit 2023-06-03T19:23:29.269056+0000 mgr.y (mgr.14116) 5 : audit [DBG] from='client.14128 -' entity='client.admin' cmd=[{"prefix": "cephadm set-user", "user": "root", "target": ["mon-mgr", ""]}]: dispatch 2023-06-03T19:23:29.930 INFO:teuthology.orchestra.run.smithi089.stderr:Generating ssh key... 2023-06-03T19:23:32.033 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 19:23:31 smithi089 bash[26845]: Generating public/private rsa key pair. 2023-06-03T19:23:32.033 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 19:23:31 smithi089 bash[26845]: Your identification has been saved in /tmp/tmpjm1_fbjk/key. 2023-06-03T19:23:32.034 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 19:23:31 smithi089 bash[26845]: Your public key has been saved in /tmp/tmpjm1_fbjk/key.pub. 2023-06-03T19:23:32.034 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 19:23:31 smithi089 bash[26845]: The key fingerprint is: 2023-06-03T19:23:32.034 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 19:23:31 smithi089 bash[26845]: SHA256:z47TfWW33zuUqnpFo5HmqfLK33dZ5FbjY8Bb0E58h18 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c 2023-06-03T19:23:32.034 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 19:23:31 smithi089 bash[26845]: The key's randomart image is: 2023-06-03T19:23:32.034 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 19:23:31 smithi089 bash[26845]: +---[RSA 3072]----+ 2023-06-03T19:23:32.034 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 19:23:31 smithi089 bash[26845]: | o. | 2023-06-03T19:23:32.034 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 19:23:31 smithi089 bash[26845]: | ..+E| 2023-06-03T19:23:32.034 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 19:23:31 smithi089 bash[26845]: | o +.+| 2023-06-03T19:23:32.035 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 19:23:31 smithi089 bash[26845]: | + = ++| 2023-06-03T19:23:32.035 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 19:23:31 smithi089 bash[26845]: | S o = *o+| 2023-06-03T19:23:32.035 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 19:23:31 smithi089 bash[26845]: | o + o *B| 2023-06-03T19:23:32.035 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 19:23:31 smithi089 bash[26845]: | .+.. ++*| 2023-06-03T19:23:32.035 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 19:23:31 smithi089 bash[26845]: | ...+o..o.=.| 2023-06-03T19:23:32.035 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 19:23:31 smithi089 bash[26845]: | o*=+oo...*| 2023-06-03T19:23:32.035 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 19:23:31 smithi089 bash[26845]: +----[SHA256]-----+ 2023-06-03T19:23:32.783 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:23:32 smithi089 bash[26572]: audit 2023-06-03T19:23:31.359697+0000 mgr.y (mgr.14116) 6 : audit [DBG] from='client.14130 -' entity='client.admin' cmd=[{"prefix": "cephadm generate-key", "target": ["mon-mgr", ""]}]: dispatch 2023-06-03T19:23:32.783 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:23:32 smithi089 bash[26572]: cephadm 2023-06-03T19:23:31.360063+0000 mgr.y (mgr.14116) 7 : cephadm [INF] Generating ssh key... 2023-06-03T19:23:32.784 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:23:32 smithi089 bash[26572]: audit 2023-06-03T19:23:31.752001+0000 mon.a (mon.0) 51 : audit [INF] from='mgr.14116 172.21.15.89:0/1027675781' entity='mgr.y' 2023-06-03T19:23:32.784 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:23:32 smithi089 bash[26572]: audit 2023-06-03T19:23:31.755656+0000 mon.a (mon.0) 52 : audit [INF] from='mgr.14116 172.21.15.89:0/1027675781' entity='mgr.y' 2023-06-03T19:23:33.848 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDdz5/mCuTK/5PbgBKiI6mvA4cssdhUdBgTu6U6fEiC5fPMhOB31KRbMfHhx4yhwEHMKHA48ozeD+WP3JzM6VncdEIhaYYzffvo/BsN6HeHOqKewC/iXQpNyrRXZ3Rbm9A1VPemvkHzmFDxiw6zZjgY2J5vwcy5/ZrOS9EnsWRJlNh+lbPsnXylrtH79fplUEM/NJU4L54tvU0INnlog4mgFxFoqx7emsDfp5Pdxc0fyuEsS73fxmvrBRNmJNBw6nDone8xWbV1xxeiLjfeJRYRRaAT5Oe0k48wyoIWR7Osw91MshQYQLa5Ctmbc0PjvCmpipsYQyn2muREOzfRa7yb/CaSEsklvFNQcI5YQtY7G0ld5avbqLAbhxfLLTyH2pcTlHwrKd26mC2CVc7oFbkV36Lcr971KmF5RwzaWkosWhFWFFS6SzuhpCTvyaoFDe9coAhBrjhgTKXn6+lW4VqmCnQi3PtkBGp5p69us+MaLWCs/P8taimxeNvv4z3DOoU= ceph-f1db3478-0243-11ee-9b1d-001a4aab830c 2023-06-03T19:23:34.450 INFO:teuthology.orchestra.run.smithi089.stderr:Wrote public SSH key to /home/ubuntu/cephtest/ceph.pub 2023-06-03T19:23:34.451 INFO:teuthology.orchestra.run.smithi089.stderr:Adding key to root@localhost authorized_keys... 2023-06-03T19:23:34.451 INFO:teuthology.orchestra.run.smithi089.stderr:Adding host smithi089... 2023-06-03T19:23:34.783 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:23:34 smithi089 bash[26572]: audit 2023-06-03T19:23:33.839149+0000 mgr.y (mgr.14116) 8 : audit [DBG] from='client.14132 -' entity='client.admin' cmd=[{"prefix": "cephadm get-pub-key", "target": ["mon-mgr", ""]}]: dispatch 2023-06-03T19:23:36.283 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 19:23:35 smithi089 bash[26845]: Warning: Permanently added '172.21.15.89' (ECDSA) to the list of known hosts. 2023-06-03T19:23:36.695 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: Added host 'smithi089' with addr '172.21.15.89' 2023-06-03T19:23:37.679 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:23:37 smithi089 bash[26572]: audit 2023-06-03T19:23:35.876651+0000 mgr.y (mgr.14116) 9 : audit [DBG] from='client.14134 -' entity='client.admin' cmd=[{"prefix": "orch host add", "hostname": "smithi089", "addr": "172.21.15.89", "target": ["mon-mgr", ""]}]: dispatch 2023-06-03T19:23:37.680 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:23:37 smithi089 bash[26572]: audit 2023-06-03T19:23:36.684978+0000 mon.a (mon.0) 53 : audit [INF] from='mgr.14116 172.21.15.89:0/1027675781' entity='mgr.y' 2023-06-03T19:23:37.680 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:23:37 smithi089 bash[26572]: audit 2023-06-03T19:23:36.714268+0000 mon.a (mon.0) 54 : audit [DBG] from='mgr.14116 172.21.15.89:0/1027675781' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-03T19:23:37.681 INFO:teuthology.orchestra.run.smithi089.stderr:Deploying unmanaged mon service... 2023-06-03T19:23:38.783 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:23:38 smithi089 bash[26572]: cephadm 2023-06-03T19:23:36.685682+0000 mgr.y (mgr.14116) 10 : cephadm [INF] Added host smithi089 2023-06-03T19:23:39.925 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: Scheduled mon update... 2023-06-03T19:23:40.700 INFO:teuthology.orchestra.run.smithi089.stderr:Deploying unmanaged mgr service... 2023-06-03T19:23:41.036 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:23:40 smithi089 bash[26572]: audit 2023-06-03T19:23:39.910890+0000 mgr.y (mgr.14116) 11 : audit [DBG] from='client.14136 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mon", "unmanaged": true, "target": ["mon-mgr", ""]}]: dispatch 2023-06-03T19:23:41.037 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:23:40 smithi089 bash[26572]: cephadm 2023-06-03T19:23:39.912367+0000 mgr.y (mgr.14116) 12 : cephadm [INF] Saving service mon spec with placement count:5 2023-06-03T19:23:41.037 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:23:40 smithi089 bash[26572]: audit 2023-06-03T19:23:39.915932+0000 mon.a (mon.0) 55 : audit [INF] from='mgr.14116 172.21.15.89:0/1027675781' entity='mgr.y' 2023-06-03T19:23:41.037 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:23:40 smithi089 bash[26572]: audit 2023-06-03T19:23:40.286009+0000 mon.a (mon.0) 56 : audit [INF] from='mgr.14116 172.21.15.89:0/1027675781' entity='mgr.y' 2023-06-03T19:23:41.037 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:23:40 smithi089 bash[26572]: audit 2023-06-03T19:23:40.532419+0000 mon.a (mon.0) 57 : audit [INF] from='mgr.14116 172.21.15.89:0/1027675781' entity='mgr.y' 2023-06-03T19:23:42.232 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: Scheduled mgr update... 2023-06-03T19:23:43.533 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:23:43 smithi089 bash[26572]: audit 2023-06-03T19:23:42.215228+0000 mgr.y (mgr.14116) 13 : audit [DBG] from='client.14138 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mgr", "unmanaged": true, "target": ["mon-mgr", ""]}]: dispatch 2023-06-03T19:23:43.533 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:23:43 smithi089 bash[26572]: cephadm 2023-06-03T19:23:42.217563+0000 mgr.y (mgr.14116) 14 : cephadm [INF] Saving service mgr spec with placement count:2 2023-06-03T19:23:43.533 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:23:43 smithi089 bash[26572]: audit 2023-06-03T19:23:42.221807+0000 mon.a (mon.0) 58 : audit [INF] from='mgr.14116 172.21.15.89:0/1027675781' entity='mgr.y' 2023-06-03T19:23:45.533 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:23:45 smithi089 bash[26572]: cluster 2023-06-03T19:23:43.965470+0000 mgr.y (mgr.14116) 15 : cluster [DBG] pgmap v3: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-06-03T19:23:45.533 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:23:45 smithi089 bash[26572]: audit 2023-06-03T19:23:44.320806+0000 mon.a (mon.0) 59 : audit [INF] from='client.? 172.21.15.89:0/752768553' entity='client.admin' 2023-06-03T19:23:47.032 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:23:46 smithi089 bash[26572]: audit 2023-06-03T19:23:45.877456+0000 mon.a (mon.0) 60 : audit [INF] from='mgr.14116 172.21.15.89:0/1027675781' entity='mgr.y' 2023-06-03T19:23:47.033 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:23:46 smithi089 bash[26572]: cluster 2023-06-03T19:23:45.965713+0000 mgr.y (mgr.14116) 16 : cluster [DBG] pgmap v4: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-06-03T19:23:47.033 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:23:46 smithi089 bash[26572]: audit 2023-06-03T19:23:46.550494+0000 mon.a (mon.0) 61 : audit [INF] from='client.? 172.21.15.89:0/1673164099' entity='client.admin' 2023-06-03T19:23:47.190 INFO:teuthology.orchestra.run.smithi089.stderr:Enabling the dashboard module... 2023-06-03T19:23:49.343 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:23:49 smithi089 bash[26572]: cluster 2023-06-03T19:23:47.966123+0000 mgr.y (mgr.14116) 17 : cluster [DBG] pgmap v5: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-06-03T19:23:49.343 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:23:49 smithi089 bash[26572]: audit 2023-06-03T19:23:48.747137+0000 mon.a (mon.0) 62 : audit [INF] from='client.? 172.21.15.89:0/1423663939' entity='client.admin' cmd=[{"prefix": "mgr module enable", "module": "dashboard"}]: dispatch 2023-06-03T19:23:49.344 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 19:23:49 smithi089 bash[26845]: ignoring --setuser ceph since I am not root 2023-06-03T19:23:49.344 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 19:23:49 smithi089 bash[26845]: ignoring --setgroup ceph since I am not root 2023-06-03T19:23:49.344 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 19:23:49 smithi089 bash[26845]: debug 2023-06-03T19:23:49.225+0000 7f3c393de680 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2023-06-03T19:23:49.532 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 19:23:49 smithi089 bash[26845]: debug 2023-06-03T19:23:49.329+0000 7f3c393de680 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2023-06-03T19:23:50.282 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:23:50 smithi089 bash[26572]: audit 2023-06-03T19:23:49.024953+0000 mon.a (mon.0) 63 : audit [INF] from='client.? 172.21.15.89:0/1423663939' entity='client.admin' cmd='[{"prefix": "mgr module enable", "module": "dashboard"}]': finished 2023-06-03T19:23:50.282 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:23:50 smithi089 bash[26572]: cluster 2023-06-03T19:23:49.025115+0000 mon.a (mon.0) 64 : cluster [DBG] mgrmap e9: y(active, since 25s) 2023-06-03T19:23:50.283 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 19:23:49 smithi089 bash[26845]: debug 2023-06-03T19:23:49.802+0000 7f3c393de680 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2023-06-03T19:23:51.032 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 19:23:50 smithi089 bash[26845]: debug 2023-06-03T19:23:50.714+0000 7f3c393de680 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2023-06-03T19:23:51.032 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 19:23:50 smithi089 bash[26845]: debug 2023-06-03T19:23:50.814+0000 7f3c393de680 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2023-06-03T19:23:51.452 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: { 2023-06-03T19:23:51.452 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: "epoch": 9, 2023-06-03T19:23:51.452 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: "available": true, 2023-06-03T19:23:51.452 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: "active_name": "y", 2023-06-03T19:23:51.453 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: "num_standby": 0 2023-06-03T19:23:51.453 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: } 2023-06-03T19:23:51.499 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 19:23:51 smithi089 bash[26845]: debug 2023-06-03T19:23:51.070+0000 7f3c393de680 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2023-06-03T19:23:52.483 INFO:teuthology.orchestra.run.smithi089.stderr:Waiting for the mgr to restart... 2023-06-03T19:23:52.484 INFO:teuthology.orchestra.run.smithi089.stderr:Waiting for mgr epoch 9... 2023-06-03T19:23:52.485 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:23:52 smithi089 bash[26572]: audit 2023-06-03T19:23:51.443548+0000 mon.a (mon.0) 65 : audit [DBG] from='client.? 172.21.15.89:0/3402859013' entity='client.admin' cmd=[{"prefix": "mgr stat"}]: dispatch 2023-06-03T19:23:52.485 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 19:23:52 smithi089 bash[26845]: debug 2023-06-03T19:23:52.210+0000 7f3c393de680 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2023-06-03T19:23:52.485 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 19:23:52 smithi089 bash[26845]: debug 2023-06-03T19:23:52.374+0000 7f3c393de680 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2023-06-03T19:23:52.486 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 19:23:52 smithi089 bash[26845]: debug 2023-06-03T19:23:52.474+0000 7f3c393de680 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2023-06-03T19:23:52.782 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 19:23:52 smithi089 bash[26845]: debug 2023-06-03T19:23:52.594+0000 7f3c393de680 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2023-06-03T19:23:53.103 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 19:23:52 smithi089 bash[26845]: debug 2023-06-03T19:23:52.830+0000 7f3c393de680 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2023-06-03T19:23:53.424 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 19:23:53 smithi089 bash[26845]: debug 2023-06-03T19:23:53.210+0000 7f3c393de680 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2023-06-03T19:23:53.425 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 19:23:53 smithi089 bash[26845]: debug 2023-06-03T19:23:53.326+0000 7f3c393de680 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2023-06-03T19:23:54.362 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 19:23:54 smithi089 bash[26845]: debug 2023-06-03T19:23:54.182+0000 7f3c393de680 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2023-06-03T19:23:54.362 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 19:23:54 smithi089 bash[26845]: debug 2023-06-03T19:23:54.282+0000 7f3c393de680 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2023-06-03T19:23:54.640 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 19:23:54 smithi089 bash[26845]: debug 2023-06-03T19:23:54.402+0000 7f3c393de680 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2023-06-03T19:23:54.954 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 19:23:54 smithi089 bash[26845]: debug 2023-06-03T19:23:54.630+0000 7f3c393de680 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2023-06-03T19:23:55.282 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 19:23:54 smithi089 bash[26845]: debug 2023-06-03T19:23:54.942+0000 7f3c393de680 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2023-06-03T19:23:55.282 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 19:23:55 smithi089 bash[26845]: debug 2023-06-03T19:23:55.106+0000 7f3c393de680 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2023-06-03T19:23:55.613 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 19:23:55 smithi089 bash[26845]: debug 2023-06-03T19:23:55.278+0000 7f3c393de680 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2023-06-03T19:23:55.613 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 19:23:55 smithi089 bash[26845]: debug 2023-06-03T19:23:55.494+0000 7f3c393de680 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2023-06-03T19:23:55.868 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 19:23:55 smithi089 bash[26845]: debug 2023-06-03T19:23:55.602+0000 7f3c393de680 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2023-06-03T19:23:56.281 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 19:23:55 smithi089 bash[26845]: Warning: Permanently added '172.21.15.89' (ECDSA) to the list of known hosts. 2023-06-03T19:23:56.711 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: { 2023-06-03T19:23:56.711 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: "mgrmap_epoch": 11, 2023-06-03T19:23:56.712 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: "initialized": true 2023-06-03T19:23:56.712 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: } 2023-06-03T19:23:56.919 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:23:56 smithi089 bash[26572]: cluster 2023-06-03T19:23:55.626825+0000 mon.a (mon.0) 66 : cluster [INF] Active manager daemon y restarted 2023-06-03T19:23:56.919 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:23:56 smithi089 bash[26572]: cluster 2023-06-03T19:23:55.627711+0000 mon.a (mon.0) 67 : cluster [INF] Activating manager daemon y 2023-06-03T19:23:56.920 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:23:56 smithi089 bash[26572]: cluster 2023-06-03T19:23:55.630853+0000 mon.a (mon.0) 68 : cluster [DBG] osdmap e3: 0 total, 0 up, 0 in 2023-06-03T19:23:56.920 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:23:56 smithi089 bash[26572]: cluster 2023-06-03T19:23:55.683156+0000 mon.a (mon.0) 69 : cluster [DBG] mgrmap e10: y(active, starting, since 0.0556412s) 2023-06-03T19:23:56.920 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:23:56 smithi089 bash[26572]: audit 2023-06-03T19:23:55.685048+0000 mon.a (mon.0) 70 : audit [DBG] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-06-03T19:23:56.920 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:23:56 smithi089 bash[26572]: audit 2023-06-03T19:23:55.686564+0000 mon.a (mon.0) 71 : audit [DBG] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2023-06-03T19:23:56.920 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:23:56 smithi089 bash[26572]: audit 2023-06-03T19:23:55.687523+0000 mon.a (mon.0) 72 : audit [DBG] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2023-06-03T19:23:56.920 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:23:56 smithi089 bash[26572]: audit 2023-06-03T19:23:55.687775+0000 mon.a (mon.0) 73 : audit [DBG] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2023-06-03T19:23:56.921 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:23:56 smithi089 bash[26572]: audit 2023-06-03T19:23:55.687980+0000 mon.a (mon.0) 74 : audit [DBG] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2023-06-03T19:23:56.921 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:23:56 smithi089 bash[26572]: cluster 2023-06-03T19:23:55.691203+0000 mon.a (mon.0) 75 : cluster [INF] Manager daemon y is now available 2023-06-03T19:23:56.921 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:23:56 smithi089 bash[26572]: audit 2023-06-03T19:23:55.814700+0000 mon.a (mon.0) 76 : audit [DBG] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-03T19:23:56.921 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:23:56 smithi089 bash[26572]: audit 2023-06-03T19:23:55.818959+0000 mon.a (mon.0) 77 : audit [DBG] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-03T19:23:56.921 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:23:56 smithi089 bash[26572]: audit 2023-06-03T19:23:55.855138+0000 mon.a (mon.0) 78 : audit [INF] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2023-06-03T19:23:56.921 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:23:56 smithi089 bash[26572]: audit 2023-06-03T19:23:55.857447+0000 mon.a (mon.0) 79 : audit [INF] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2023-06-03T19:23:57.419 INFO:teuthology.orchestra.run.smithi089.stderr:mgr epoch 9 is available 2023-06-03T19:23:57.419 INFO:teuthology.orchestra.run.smithi089.stderr:Generating a dashboard self-signed certificate... 2023-06-03T19:23:57.781 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:23:57 smithi089 bash[26572]: audit 2023-06-03T19:23:56.688456+0000 mgr.y (mgr.14146) 1 : audit [DBG] from='client.14150 -' entity='client.admin' cmd=[{"prefix": "get_command_descriptions"}]: dispatch 2023-06-03T19:23:57.782 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:23:57 smithi089 bash[26572]: cluster 2023-06-03T19:23:56.689600+0000 mon.a (mon.0) 80 : cluster [DBG] mgrmap e11: y(active, since 1.06209s) 2023-06-03T19:23:57.782 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:23:57 smithi089 bash[26572]: audit 2023-06-03T19:23:56.702802+0000 mgr.y (mgr.14146) 2 : audit [DBG] from='client.14150 -' entity='client.admin' cmd=[{"prefix": "mgr_status"}]: dispatch 2023-06-03T19:23:59.031 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:23:58 smithi089 bash[26572]: cluster 2023-06-03T19:23:57.693445+0000 mon.a (mon.0) 81 : cluster [DBG] mgrmap e12: y(active, since 2s) 2023-06-03T19:23:59.061 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: Self-signed certificate created 2023-06-03T19:23:59.799 INFO:teuthology.orchestra.run.smithi089.stderr:Creating initial admin user... 2023-06-03T19:24:00.281 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:24:00 smithi089 bash[26572]: audit 2023-06-03T19:23:59.005928+0000 mgr.y (mgr.14146) 3 : audit [DBG] from='client.14156 -' entity='client.admin' cmd=[{"prefix": "dashboard create-self-signed-cert", "target": ["mon-mgr", ""]}]: dispatch 2023-06-03T19:24:00.282 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:24:00 smithi089 bash[26572]: audit 2023-06-03T19:23:59.049652+0000 mon.a (mon.0) 82 : audit [INF] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' 2023-06-03T19:24:00.282 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:24:00 smithi089 bash[26572]: audit 2023-06-03T19:23:59.053288+0000 mon.a (mon.0) 83 : audit [INF] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' 2023-06-03T19:24:01.489 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: {"username": "admin", "password": "$2b$12$GFtTiJKPj5O/F66b.E.FpuytRDM6bFV8jEvs/STOBuegYpcD9kS2W", "roles": ["administrator"], "name": null, "email": null, "lastUpdate": 1685820241, "enabled": true, "pwdExpirationDate": null, "pwdUpdateRequired": true} 2023-06-03T19:24:02.181 INFO:teuthology.orchestra.run.smithi089.stderr:Fetching dashboard port number... 2023-06-03T19:24:02.781 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:24:02 smithi089 bash[26572]: audit 2023-06-03T19:24:01.252992+0000 mgr.y (mgr.14146) 4 : audit [DBG] from='client.14158 -' entity='client.admin' cmd=[{"prefix": "dashboard ac-user-create", "username": "admin", "rolename": "administrator", "force_password": true, "pwd_update_required": true, "target": ["mon-mgr", ""]}]: dispatch 2023-06-03T19:24:02.781 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:24:02 smithi089 bash[26572]: audit 2023-06-03T19:24:01.481516+0000 mon.a (mon.0) 84 : audit [INF] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' 2023-06-03T19:24:02.781 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:24:02 smithi089 bash[26572]: audit 2023-06-03T19:24:01.583559+0000 mon.a (mon.0) 85 : audit [INF] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' 2023-06-03T19:24:03.662 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: 8443 2023-06-03T19:24:03.842 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:24:03 smithi089 bash[26572]: cluster 2023-06-03T19:24:02.586948+0000 mon.a (mon.0) 86 : cluster [DBG] mgrmap e13: y(active, since 6s) 2023-06-03T19:24:04.345 INFO:teuthology.orchestra.run.smithi089.stderr:firewalld does not appear to be present 2023-06-03T19:24:04.345 INFO:teuthology.orchestra.run.smithi089.stderr:Not possible to open ports <[8443]>. firewalld.service is not available 2023-06-03T19:24:04.347 INFO:teuthology.orchestra.run.smithi089.stderr:Ceph Dashboard is now available at: 2023-06-03T19:24:04.348 INFO:teuthology.orchestra.run.smithi089.stderr: 2023-06-03T19:24:04.348 INFO:teuthology.orchestra.run.smithi089.stderr: URL: https://smithi089.front.sepia.ceph.com:8443/ 2023-06-03T19:24:04.348 INFO:teuthology.orchestra.run.smithi089.stderr: User: admin 2023-06-03T19:24:04.348 INFO:teuthology.orchestra.run.smithi089.stderr: Password: vobfg0d2nb 2023-06-03T19:24:04.348 INFO:teuthology.orchestra.run.smithi089.stderr: 2023-06-03T19:24:04.348 INFO:teuthology.orchestra.run.smithi089.stderr:Enabling autotune for osd_memory_target 2023-06-03T19:24:04.781 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:24:04 smithi089 bash[26572]: audit 2023-06-03T19:24:03.654386+0000 mon.a (mon.0) 87 : audit [DBG] from='client.? 172.21.15.89:0/1400546800' entity='client.admin' cmd=[{"prefix": "config get", "who": "mgr", "key": "mgr/dashboard/ssl_server_port"}]: dispatch 2023-06-03T19:24:08.592 INFO:teuthology.orchestra.run.smithi089.stderr:/usr/bin/ceph: set mgr/dashboard/cluster/status 2023-06-03T19:24:09.446 INFO:teuthology.orchestra.run.smithi089.stderr:You can access the Ceph CLI as following in case of multi-cluster or non-default config: 2023-06-03T19:24:09.446 INFO:teuthology.orchestra.run.smithi089.stderr: 2023-06-03T19:24:09.446 INFO:teuthology.orchestra.run.smithi089.stderr: sudo /home/ubuntu/cephtest/cephadm shell --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring 2023-06-03T19:24:09.447 INFO:teuthology.orchestra.run.smithi089.stderr: 2023-06-03T19:24:09.447 INFO:teuthology.orchestra.run.smithi089.stderr:Or, if you are only running a single cluster on this host: 2023-06-03T19:24:09.447 INFO:teuthology.orchestra.run.smithi089.stderr: 2023-06-03T19:24:09.447 INFO:teuthology.orchestra.run.smithi089.stderr: sudo /home/ubuntu/cephtest/cephadm shell 2023-06-03T19:24:09.447 INFO:teuthology.orchestra.run.smithi089.stderr: 2023-06-03T19:24:09.447 INFO:teuthology.orchestra.run.smithi089.stderr:Please consider enabling telemetry to help improve Ceph: 2023-06-03T19:24:09.447 INFO:teuthology.orchestra.run.smithi089.stderr: 2023-06-03T19:24:09.447 INFO:teuthology.orchestra.run.smithi089.stderr: ceph telemetry on 2023-06-03T19:24:09.448 INFO:teuthology.orchestra.run.smithi089.stderr: 2023-06-03T19:24:09.448 INFO:teuthology.orchestra.run.smithi089.stderr:For more information see: 2023-06-03T19:24:09.448 INFO:teuthology.orchestra.run.smithi089.stderr: 2023-06-03T19:24:09.448 INFO:teuthology.orchestra.run.smithi089.stderr: https://docs.ceph.com/en/pacific/mgr/telemetry/ 2023-06-03T19:24:09.448 INFO:teuthology.orchestra.run.smithi089.stderr: 2023-06-03T19:24:09.449 INFO:teuthology.orchestra.run.smithi089.stderr:Bootstrap complete. 2023-06-03T19:24:09.479 INFO:tasks.cephadm:Fetching config... 2023-06-03T19:24:09.480 DEBUG:teuthology.orchestra.run.smithi089:> set -ex 2023-06-03T19:24:09.480 DEBUG:teuthology.orchestra.run.smithi089:> dd if=/etc/ceph/ceph.conf of=/dev/stdout 2023-06-03T19:24:09.487 INFO:tasks.cephadm:Fetching client.admin keyring... 2023-06-03T19:24:09.488 DEBUG:teuthology.orchestra.run.smithi089:> set -ex 2023-06-03T19:24:09.488 DEBUG:teuthology.orchestra.run.smithi089:> dd if=/etc/ceph/ceph.client.admin.keyring of=/dev/stdout 2023-06-03T19:24:09.536 INFO:tasks.cephadm:Fetching mon keyring... 2023-06-03T19:24:09.537 DEBUG:teuthology.orchestra.run.smithi089:> set -ex 2023-06-03T19:24:09.537 DEBUG:teuthology.orchestra.run.smithi089:> sudo dd if=/var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/mon.a/keyring of=/dev/stdout 2023-06-03T19:24:09.594 INFO:tasks.cephadm:Fetching pub ssh key... 2023-06-03T19:24:09.594 DEBUG:teuthology.orchestra.run.smithi089:> set -ex 2023-06-03T19:24:09.594 DEBUG:teuthology.orchestra.run.smithi089:> dd if=/home/ubuntu/cephtest/ceph.pub of=/dev/stdout 2023-06-03T19:24:09.645 INFO:tasks.cephadm:Installing pub ssh key for root users... 2023-06-03T19:24:09.646 DEBUG:teuthology.orchestra.run.smithi089:> sudo install -d -m 0700 /root/.ssh && echo 'ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDdz5/mCuTK/5PbgBKiI6mvA4cssdhUdBgTu6U6fEiC5fPMhOB31KRbMfHhx4yhwEHMKHA48ozeD+WP3JzM6VncdEIhaYYzffvo/BsN6HeHOqKewC/iXQpNyrRXZ3Rbm9A1VPemvkHzmFDxiw6zZjgY2J5vwcy5/ZrOS9EnsWRJlNh+lbPsnXylrtH79fplUEM/NJU4L54tvU0INnlog4mgFxFoqx7emsDfp5Pdxc0fyuEsS73fxmvrBRNmJNBw6nDone8xWbV1xxeiLjfeJRYRRaAT5Oe0k48wyoIWR7Osw91MshQYQLa5Ctmbc0PjvCmpipsYQyn2muREOzfRa7yb/CaSEsklvFNQcI5YQtY7G0ld5avbqLAbhxfLLTyH2pcTlHwrKd26mC2CVc7oFbkV36Lcr971KmF5RwzaWkosWhFWFFS6SzuhpCTvyaoFDe9coAhBrjhgTKXn6+lW4VqmCnQi3PtkBGp5p69us+MaLWCs/P8taimxeNvv4z3DOoU= ceph-f1db3478-0243-11ee-9b1d-001a4aab830c' | sudo tee -a /root/.ssh/authorized_keys && sudo chmod 0600 /root/.ssh/authorized_keys 2023-06-03T19:24:09.707 INFO:teuthology.orchestra.run.smithi089.stdout:ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDdz5/mCuTK/5PbgBKiI6mvA4cssdhUdBgTu6U6fEiC5fPMhOB31KRbMfHhx4yhwEHMKHA48ozeD+WP3JzM6VncdEIhaYYzffvo/BsN6HeHOqKewC/iXQpNyrRXZ3Rbm9A1VPemvkHzmFDxiw6zZjgY2J5vwcy5/ZrOS9EnsWRJlNh+lbPsnXylrtH79fplUEM/NJU4L54tvU0INnlog4mgFxFoqx7emsDfp5Pdxc0fyuEsS73fxmvrBRNmJNBw6nDone8xWbV1xxeiLjfeJRYRRaAT5Oe0k48wyoIWR7Osw91MshQYQLa5Ctmbc0PjvCmpipsYQyn2muREOzfRa7yb/CaSEsklvFNQcI5YQtY7G0ld5avbqLAbhxfLLTyH2pcTlHwrKd26mC2CVc7oFbkV36Lcr971KmF5RwzaWkosWhFWFFS6SzuhpCTvyaoFDe9coAhBrjhgTKXn6+lW4VqmCnQi3PtkBGp5p69us+MaLWCs/P8taimxeNvv4z3DOoU= ceph-f1db3478-0243-11ee-9b1d-001a4aab830c 2023-06-03T19:24:09.716 DEBUG:teuthology.orchestra.run.smithi164:> sudo install -d -m 0700 /root/.ssh && echo 'ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDdz5/mCuTK/5PbgBKiI6mvA4cssdhUdBgTu6U6fEiC5fPMhOB31KRbMfHhx4yhwEHMKHA48ozeD+WP3JzM6VncdEIhaYYzffvo/BsN6HeHOqKewC/iXQpNyrRXZ3Rbm9A1VPemvkHzmFDxiw6zZjgY2J5vwcy5/ZrOS9EnsWRJlNh+lbPsnXylrtH79fplUEM/NJU4L54tvU0INnlog4mgFxFoqx7emsDfp5Pdxc0fyuEsS73fxmvrBRNmJNBw6nDone8xWbV1xxeiLjfeJRYRRaAT5Oe0k48wyoIWR7Osw91MshQYQLa5Ctmbc0PjvCmpipsYQyn2muREOzfRa7yb/CaSEsklvFNQcI5YQtY7G0ld5avbqLAbhxfLLTyH2pcTlHwrKd26mC2CVc7oFbkV36Lcr971KmF5RwzaWkosWhFWFFS6SzuhpCTvyaoFDe9coAhBrjhgTKXn6+lW4VqmCnQi3PtkBGp5p69us+MaLWCs/P8taimxeNvv4z3DOoU= ceph-f1db3478-0243-11ee-9b1d-001a4aab830c' | sudo tee -a /root/.ssh/authorized_keys && sudo chmod 0600 /root/.ssh/authorized_keys 2023-06-03T19:24:09.738 INFO:teuthology.orchestra.run.smithi164.stdout:ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDdz5/mCuTK/5PbgBKiI6mvA4cssdhUdBgTu6U6fEiC5fPMhOB31KRbMfHhx4yhwEHMKHA48ozeD+WP3JzM6VncdEIhaYYzffvo/BsN6HeHOqKewC/iXQpNyrRXZ3Rbm9A1VPemvkHzmFDxiw6zZjgY2J5vwcy5/ZrOS9EnsWRJlNh+lbPsnXylrtH79fplUEM/NJU4L54tvU0INnlog4mgFxFoqx7emsDfp5Pdxc0fyuEsS73fxmvrBRNmJNBw6nDone8xWbV1xxeiLjfeJRYRRaAT5Oe0k48wyoIWR7Osw91MshQYQLa5Ctmbc0PjvCmpipsYQyn2muREOzfRa7yb/CaSEsklvFNQcI5YQtY7G0ld5avbqLAbhxfLLTyH2pcTlHwrKd26mC2CVc7oFbkV36Lcr971KmF5RwzaWkosWhFWFFS6SzuhpCTvyaoFDe9coAhBrjhgTKXn6+lW4VqmCnQi3PtkBGp5p69us+MaLWCs/P8taimxeNvv4z3DOoU= ceph-f1db3478-0243-11ee-9b1d-001a4aab830c 2023-06-03T19:24:09.750 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph config set mgr mgr/cephadm/allow_ptrace true 2023-06-03T19:24:09.772 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:24:09 smithi089 bash[26572]: audit 2023-06-03T19:24:08.582323+0000 mon.a (mon.0) 88 : audit [INF] from='client.? 172.21.15.89:0/3246983186' entity='client.admin' 2023-06-03T19:24:15.161 INFO:tasks.cephadm:Distributing conf and client.admin keyring to all hosts + 0755 2023-06-03T19:24:15.162 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph orch client-keyring set client.admin '*' --mode 0755 2023-06-03T19:24:15.780 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:24:15 smithi089 bash[26572]: audit 2023-06-03T19:24:14.427744+0000 mon.a (mon.0) 89 : audit [INF] from='client.? 172.21.15.89:0/3781871243' entity='client.admin' 2023-06-03T19:24:16.683 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:24:16 smithi089 bash[26572]: cluster 2023-06-03T19:24:15.689068+0000 mgr.y (mgr.14146) 5 : cluster [DBG] pgmap v3: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-06-03T19:24:17.780 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:24:17 smithi089 bash[26572]: audit 2023-06-03T19:24:16.450578+0000 mon.a (mon.0) 90 : audit [INF] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' 2023-06-03T19:24:17.780 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:24:17 smithi089 bash[26572]: audit 2023-06-03T19:24:16.755697+0000 mon.a (mon.0) 91 : audit [INF] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd/host:smithi089", "name": "osd_memory_target"}]: dispatch 2023-06-03T19:24:17.781 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:24:17 smithi089 bash[26572]: audit 2023-06-03T19:24:16.761022+0000 mon.a (mon.0) 92 : audit [INF] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' 2023-06-03T19:24:17.781 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:24:17 smithi089 bash[26572]: audit 2023-06-03T19:24:16.763804+0000 mon.a (mon.0) 93 : audit [DBG] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-03T19:24:17.781 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:24:17 smithi089 bash[26572]: audit 2023-06-03T19:24:16.770010+0000 mon.a (mon.0) 94 : audit [INF] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' 2023-06-03T19:24:18.780 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:24:18 smithi089 bash[26572]: cluster 2023-06-03T19:24:17.689451+0000 mgr.y (mgr.14146) 6 : cluster [DBG] pgmap v4: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-06-03T19:24:19.507 INFO:tasks.cephadm:Writing (initial) conf and keyring to smithi164 2023-06-03T19:24:19.508 DEBUG:teuthology.orchestra.run.smithi164:> set -ex 2023-06-03T19:24:19.508 DEBUG:teuthology.orchestra.run.smithi164:> dd of=/etc/ceph/ceph.conf 2023-06-03T19:24:19.515 DEBUG:teuthology.orchestra.run.smithi164:> set -ex 2023-06-03T19:24:19.516 DEBUG:teuthology.orchestra.run.smithi164:> dd of=/etc/ceph/ceph.client.admin.keyring 2023-06-03T19:24:19.565 INFO:tasks.cephadm:Adding host smithi164 to orchestrator... 2023-06-03T19:24:19.566 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph orch host add smithi164 2023-06-03T19:24:20.030 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:24:19 smithi089 bash[26572]: audit 2023-06-03T19:24:18.788161+0000 mon.a (mon.0) 95 : audit [INF] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' 2023-06-03T19:24:20.031 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:24:19 smithi089 bash[26572]: audit 2023-06-03T19:24:18.790684+0000 mon.a (mon.0) 96 : audit [DBG] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-03T19:24:20.031 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:24:19 smithi089 bash[26572]: audit 2023-06-03T19:24:18.793226+0000 mon.a (mon.0) 97 : audit [DBG] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-03T19:24:20.031 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:24:19 smithi089 bash[26572]: audit 2023-06-03T19:24:18.794445+0000 mon.a (mon.0) 98 : audit [INF] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-03T19:24:20.031 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:24:19 smithi089 bash[26572]: cephadm 2023-06-03T19:24:18.796477+0000 mgr.y (mgr.14146) 8 : cephadm [INF] Updating smithi089:/etc/ceph/ceph.conf 2023-06-03T19:24:20.031 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:24:19 smithi089 bash[26572]: cephadm 2023-06-03T19:24:18.815835+0000 mgr.y (mgr.14146) 9 : cephadm [INF] Updating smithi089:/etc/ceph/ceph.client.admin.keyring 2023-06-03T19:24:20.031 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:24:19 smithi089 bash[26572]: audit 2023-06-03T19:24:18.845497+0000 mon.a (mon.0) 99 : audit [INF] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' 2023-06-03T19:24:20.031 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:24:19 smithi089 bash[26572]: audit 2023-06-03T19:24:18.849100+0000 mon.a (mon.0) 100 : audit [INF] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' 2023-06-03T19:24:21.164 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:24:20 smithi089 bash[26572]: cluster 2023-06-03T19:24:19.689880+0000 mgr.y (mgr.14146) 10 : cluster [DBG] pgmap v5: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-06-03T19:24:22.663 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:24:22 smithi089 bash[26572]: cluster 2023-06-03T19:24:21.690306+0000 mgr.y (mgr.14146) 11 : cluster [DBG] pgmap v6: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-06-03T19:24:23.364 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 19:24:23 smithi089 bash[26845]: Warning: Permanently added 'smithi164,172.21.15.164' (ECDSA) to the list of known hosts. 2023-06-03T19:24:23.780 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:24:23 smithi089 bash[26572]: audit 2023-06-03T19:24:23.026040+0000 mgr.y (mgr.14146) 12 : audit [DBG] from='client.14170 -' entity='client.admin' cmd=[{"prefix": "orch host add", "hostname": "smithi164", "target": ["mon-mgr", ""]}]: dispatch 2023-06-03T19:24:23.879 INFO:teuthology.orchestra.run.smithi089.stdout:Added host 'smithi164' with addr '172.21.15.164' 2023-06-03T19:24:24.280 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 19:24:23 smithi089 bash[26845]: Warning: Permanently added '172.21.15.164' (ECDSA) to the list of known hosts. 2023-06-03T19:24:24.603 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph orch host ls --format=json 2023-06-03T19:24:25.280 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:24:24 smithi089 bash[26572]: cluster 2023-06-03T19:24:23.690865+0000 mgr.y (mgr.14146) 13 : cluster [DBG] pgmap v7: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-06-03T19:24:25.280 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:24:24 smithi089 bash[26572]: audit 2023-06-03T19:24:23.872681+0000 mon.a (mon.0) 101 : audit [INF] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' 2023-06-03T19:24:25.280 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:24:24 smithi089 bash[26572]: cephadm 2023-06-03T19:24:23.873502+0000 mgr.y (mgr.14146) 14 : cephadm [INF] Added host smithi164 2023-06-03T19:24:25.281 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:24:24 smithi089 bash[26572]: audit 2023-06-03T19:24:23.881254+0000 mon.a (mon.0) 102 : audit [DBG] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-03T19:24:25.281 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:24:24 smithi089 bash[26572]: audit 2023-06-03T19:24:24.437274+0000 mon.a (mon.0) 103 : audit [INF] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' 2023-06-03T19:24:27.030 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:24:26 smithi089 bash[26572]: cluster 2023-06-03T19:24:25.691379+0000 mgr.y (mgr.14146) 15 : cluster [DBG] pgmap v8: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-06-03T19:24:27.030 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:24:26 smithi089 bash[26572]: audit 2023-06-03T19:24:25.706910+0000 mon.a (mon.0) 104 : audit [INF] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' 2023-06-03T19:24:27.920 INFO:teuthology.orchestra.run.smithi089.stdout: 2023-06-03T19:24:27.920 INFO:teuthology.orchestra.run.smithi089.stdout:[{"addr": "172.21.15.89", "hostname": "smithi089", "labels": [], "status": ""}, {"addr": "172.21.15.164", "hostname": "smithi164", "labels": [], "status": ""}] 2023-06-03T19:24:28.634 INFO:tasks.cephadm:Setting crush tunables to default 2023-06-03T19:24:28.635 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph osd crush tunables default 2023-06-03T19:24:29.030 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:24:28 smithi089 bash[26572]: cluster 2023-06-03T19:24:27.691693+0000 mgr.y (mgr.14146) 16 : cluster [DBG] pgmap v9: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-06-03T19:24:29.030 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:24:28 smithi089 bash[26572]: audit 2023-06-03T19:24:27.913099+0000 mgr.y (mgr.14146) 17 : audit [DBG] from='client.14172 -' entity='client.admin' cmd=[{"prefix": "orch host ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-06-03T19:24:31.029 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:24:30 smithi089 bash[26572]: cluster 2023-06-03T19:24:29.692106+0000 mgr.y (mgr.14146) 18 : cluster [DBG] pgmap v10: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-06-03T19:24:32.668 INFO:teuthology.orchestra.run.smithi089.stderr:adjusted tunables profile to default 2023-06-03T19:24:32.721 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:24:32 smithi089 bash[26572]: audit 2023-06-03T19:24:31.285360+0000 mon.a (mon.0) 105 : audit [INF] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' 2023-06-03T19:24:32.721 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:24:32 smithi089 bash[26572]: audit 2023-06-03T19:24:31.593120+0000 mon.a (mon.0) 106 : audit [INF] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' 2023-06-03T19:24:32.721 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:24:32 smithi089 bash[26572]: audit 2023-06-03T19:24:31.594287+0000 mon.a (mon.0) 107 : audit [INF] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd/host:smithi164", "name": "osd_memory_target"}]: dispatch 2023-06-03T19:24:32.721 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:24:32 smithi089 bash[26572]: audit 2023-06-03T19:24:31.595925+0000 mon.a (mon.0) 108 : audit [DBG] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-03T19:24:32.721 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:24:32 smithi089 bash[26572]: audit 2023-06-03T19:24:31.597163+0000 mon.a (mon.0) 109 : audit [INF] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-03T19:24:32.722 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:24:32 smithi089 bash[26572]: cephadm 2023-06-03T19:24:31.598948+0000 mgr.y (mgr.14146) 19 : cephadm [INF] Updating smithi164:/etc/ceph/ceph.conf 2023-06-03T19:24:32.722 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:24:32 smithi089 bash[26572]: cephadm 2023-06-03T19:24:31.618804+0000 mgr.y (mgr.14146) 20 : cephadm [INF] Updating smithi164:/etc/ceph/ceph.client.admin.keyring 2023-06-03T19:24:32.722 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:24:32 smithi089 bash[26572]: audit 2023-06-03T19:24:31.649032+0000 mon.a (mon.0) 110 : audit [INF] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' 2023-06-03T19:24:32.722 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:24:32 smithi089 bash[26572]: audit 2023-06-03T19:24:31.653644+0000 mon.a (mon.0) 111 : audit [INF] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' 2023-06-03T19:24:32.722 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:24:32 smithi089 bash[26572]: cluster 2023-06-03T19:24:31.692481+0000 mgr.y (mgr.14146) 21 : cluster [DBG] pgmap v11: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-06-03T19:24:32.722 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:24:32 smithi089 bash[26572]: audit 2023-06-03T19:24:31.929749+0000 mon.a (mon.0) 112 : audit [INF] from='client.? 172.21.15.89:0/2070461420' entity='client.admin' cmd=[{"prefix": "osd crush tunables", "profile": "default"}]: dispatch 2023-06-03T19:24:33.429 INFO:tasks.cephadm:Adding mon.a on smithi089 2023-06-03T19:24:33.429 INFO:tasks.cephadm:Adding mon.c on smithi089 2023-06-03T19:24:33.430 INFO:tasks.cephadm:Adding mon.b on smithi164 2023-06-03T19:24:33.430 DEBUG:teuthology.orchestra.run.smithi164:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph orch apply mon '3;smithi089:172.21.15.89=a;smithi089:[v2:172.21.15.89:3301,v1:172.21.15.89:6790]=c;smithi164:172.21.15.164=b' 2023-06-03T19:24:33.780 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:24:33 smithi089 bash[26572]: audit 2023-06-03T19:24:32.655510+0000 mon.a (mon.0) 113 : audit [INF] from='client.? 172.21.15.89:0/2070461420' entity='client.admin' cmd='[{"prefix": "osd crush tunables", "profile": "default"}]': finished 2023-06-03T19:24:33.780 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:24:33 smithi089 bash[26572]: cluster 2023-06-03T19:24:32.655632+0000 mon.a (mon.0) 114 : cluster [DBG] osdmap e4: 0 total, 0 up, 0 in 2023-06-03T19:24:35.029 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:24:34 smithi089 bash[26572]: cluster 2023-06-03T19:24:33.692977+0000 mgr.y (mgr.14146) 22 : cluster [DBG] pgmap v13: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-06-03T19:24:36.570 INFO:teuthology.orchestra.run.smithi164.stdout:Scheduled mon update... 2023-06-03T19:24:37.029 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:24:36 smithi089 bash[26572]: cluster 2023-06-03T19:24:35.693382+0000 mgr.y (mgr.14146) 23 : cluster [DBG] pgmap v14: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-06-03T19:24:37.030 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:24:36 smithi089 bash[26572]: audit 2023-06-03T19:24:36.563565+0000 mon.a (mon.0) 115 : audit [INF] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' 2023-06-03T19:24:37.030 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:24:36 smithi089 bash[26572]: audit 2023-06-03T19:24:36.581195+0000 mon.a (mon.0) 116 : audit [DBG] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-03T19:24:37.030 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:24:36 smithi089 bash[26572]: audit 2023-06-03T19:24:36.585082+0000 mon.a (mon.0) 117 : audit [DBG] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-03T19:24:37.030 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:24:36 smithi089 bash[26572]: audit 2023-06-03T19:24:36.586844+0000 mon.a (mon.0) 118 : audit [INF] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-03T19:24:37.030 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:24:36 smithi089 bash[26572]: audit 2023-06-03T19:24:36.595633+0000 mon.a (mon.0) 119 : audit [INF] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' 2023-06-03T19:24:37.031 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:24:36 smithi089 bash[26572]: audit 2023-06-03T19:24:36.599167+0000 mon.a (mon.0) 120 : audit [INF] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-06-03T19:24:37.031 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:24:36 smithi089 bash[26572]: audit 2023-06-03T19:24:36.600619+0000 mon.a (mon.0) 121 : audit [DBG] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-03T19:24:37.564 DEBUG:teuthology.orchestra.run.smithi089:mon.c> sudo journalctl -f -n 0 -u ceph-f1db3478-0243-11ee-9b1d-001a4aab830c@mon.c.service 2023-06-03T19:24:37.567 DEBUG:teuthology.orchestra.run.smithi164:mon.b> sudo journalctl -f -n 0 -u ceph-f1db3478-0243-11ee-9b1d-001a4aab830c@mon.b.service 2023-06-03T19:24:37.570 INFO:tasks.cephadm:Waiting for 3 mons in monmap... 2023-06-03T19:24:37.570 DEBUG:teuthology.orchestra.run.smithi164:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph mon dump -f json 2023-06-03T19:24:37.856 INFO:journalctl@ceph.mon.c.smithi089.stdout:-- Logs begin at Sun 2022-08-14 12:20:24 UTC. -- 2023-06-03T19:24:38.029 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:24:37 smithi089 bash[26572]: audit 2023-06-03T19:24:36.551750+0000 mgr.y (mgr.14146) 24 : audit [DBG] from='client.14176 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mon", "placement": "3;smithi089:172.21.15.89=a;smithi089:[v2:172.21.15.89:3301,v1:172.21.15.89:6790]=c;smithi164:172.21.15.164=b", "target": ["mon-mgr", ""]}]: dispatch 2023-06-03T19:24:38.030 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:24:37 smithi089 bash[26572]: cephadm 2023-06-03T19:24:36.557608+0000 mgr.y (mgr.14146) 25 : cephadm [INF] Saving service mon spec with placement smithi089:172.21.15.89=a;smithi089:[v2:172.21.15.89:3301,v1:172.21.15.89:6790]=c;smithi164:172.21.15.164=b;count:3 2023-06-03T19:24:38.030 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:24:37 smithi089 bash[26572]: cephadm 2023-06-03T19:24:36.601970+0000 mgr.y (mgr.14146) 26 : cephadm [INF] Deploying daemon mon.b on smithi164 2023-06-03T19:24:38.784 INFO:journalctl@ceph.mon.b.smithi164.stdout:-- Logs begin at Sun 2022-08-14 12:20:24 UTC. -- 2023-06-03T19:24:39.029 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:24:38 smithi089 bash[26572]: cluster 2023-06-03T19:24:37.693847+0000 mgr.y (mgr.14146) 27 : cluster [DBG] pgmap v15: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-06-03T19:24:41.029 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:24:40 smithi089 bash[26572]: cluster 2023-06-03T19:24:39.694294+0000 mgr.y (mgr.14146) 28 : cluster [DBG] pgmap v16: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-06-03T19:24:42.779 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:24:42 smithi089 bash[26572]: cluster 2023-06-03T19:24:41.694743+0000 mgr.y (mgr.14146) 29 : cluster [DBG] pgmap v17: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-06-03T19:24:42.926 INFO:teuthology.orchestra.run.smithi164.stdout: 2023-06-03T19:24:42.927 INFO:teuthology.orchestra.run.smithi164.stdout:{"epoch":1,"fsid":"f1db3478-0243-11ee-9b1d-001a4aab830c","modified":"2023-06-03T19:22:49.591927Z","created":"2023-06-03T19:22:49.591927Z","min_mon_release":16,"min_mon_release_name":"pacific","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","removed_ranks: ":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging"],"optional":[]},"mons":[{"rank":0,"name":"a","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:3300","nonce":0},{"type":"v1","addr":"172.21.15.89:6789","nonce":0}]},"addr":"172.21.15.89:6789/0","public_addr":"172.21.15.89:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2023-06-03T19:24:42.929 INFO:teuthology.orchestra.run.smithi164.stderr:dumped monmap epoch 1 2023-06-03T19:24:43.779 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:24:43 smithi089 bash[26572]: audit 2023-06-03T19:24:42.921065+0000 mon.a (mon.0) 122 : audit [DBG] from='client.? 172.21.15.164:0/3575828773' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-06-03T19:24:44.597 INFO:tasks.cephadm:Waiting for 3 mons in monmap... 2023-06-03T19:24:44.597 DEBUG:teuthology.orchestra.run.smithi164:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph mon dump -f json 2023-06-03T19:24:44.789 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:24:44 smithi164 bash[26855]: debug 2023-06-03T19:24:44.583+0000 7f532d3d6700 1 mon.b@-1(synchronizing).paxosservice(auth 1..3) refresh upgraded, format 0 -> 3 2023-06-03T19:24:44.835 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:24:44 smithi089 bash[26572]: audit 2023-06-03T19:24:43.572098+0000 mon.a (mon.0) 123 : audit [INF] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' 2023-06-03T19:24:44.835 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:24:44 smithi089 bash[26572]: audit 2023-06-03T19:24:43.573796+0000 mon.a (mon.0) 124 : audit [INF] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-06-03T19:24:44.835 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:24:44 smithi089 bash[26572]: audit 2023-06-03T19:24:43.575214+0000 mon.a (mon.0) 125 : audit [DBG] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-03T19:24:44.835 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:24:44 smithi089 bash[26572]: cephadm 2023-06-03T19:24:43.576518+0000 mgr.y (mgr.14146) 30 : cephadm [INF] Deploying daemon mon.c on smithi089 2023-06-03T19:24:44.836 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:24:44 smithi089 bash[26572]: cluster 2023-06-03T19:24:43.695176+0000 mgr.y (mgr.14146) 31 : cluster [DBG] pgmap v18: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-06-03T19:24:49.279 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:24:49 smithi089 bash[33173]: debug 2023-06-03T19:24:49.011+0000 7f68c98aa700 1 mon.c@-1(synchronizing).paxosservice(auth 1..3) refresh upgraded, format 0 -> 3 2023-06-03T19:24:50.029 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:24:49 smithi089 bash[26572]: audit 2023-06-03T19:24:44.598956+0000 mon.a (mon.0) 127 : audit [DBG] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-06-03T19:24:50.030 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:24:49 smithi089 bash[26572]: cluster 2023-06-03T19:24:44.599317+0000 mon.a (mon.0) 128 : cluster [INF] mon.a calling monitor election 2023-06-03T19:24:50.030 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:24:49 smithi089 bash[26572]: audit 2023-06-03T19:24:44.617991+0000 mon.a (mon.0) 129 : audit [DBG] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-06-03T19:24:50.030 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:24:49 smithi089 bash[26572]: audit 2023-06-03T19:24:45.596078+0000 mon.a (mon.0) 130 : audit [DBG] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-06-03T19:24:50.030 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:24:49 smithi089 bash[26572]: cluster 2023-06-03T19:24:45.695503+0000 mgr.y (mgr.14146) 32 : cluster [DBG] pgmap v19: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-06-03T19:24:50.030 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:24:49 smithi089 bash[26572]: audit 2023-06-03T19:24:46.595995+0000 mon.a (mon.0) 131 : audit [DBG] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-06-03T19:24:50.031 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:24:49 smithi089 bash[26572]: cluster 2023-06-03T19:24:46.596426+0000 mon.b (mon.1) 1 : cluster [INF] mon.b calling monitor election 2023-06-03T19:24:50.031 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:24:49 smithi089 bash[26572]: audit 2023-06-03T19:24:47.596138+0000 mon.a (mon.0) 132 : audit [DBG] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-06-03T19:24:50.031 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:24:49 smithi089 bash[26572]: cluster 2023-06-03T19:24:47.695903+0000 mgr.y (mgr.14146) 33 : cluster [DBG] pgmap v20: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-06-03T19:24:50.031 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:24:49 smithi089 bash[26572]: audit 2023-06-03T19:24:48.596532+0000 mon.a (mon.0) 133 : audit [DBG] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-06-03T19:24:50.031 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:24:49 smithi089 bash[26572]: audit 2023-06-03T19:24:49.018983+0000 mon.a (mon.0) 134 : audit [DBG] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-06-03T19:24:50.031 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:24:49 smithi089 bash[26572]: audit 2023-06-03T19:24:49.596925+0000 mon.a (mon.0) 135 : audit [DBG] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-06-03T19:24:50.031 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:24:49 smithi089 bash[26572]: cluster 2023-06-03T19:24:49.638124+0000 mon.a (mon.0) 136 : cluster [INF] mon.a is new leader, mons a,b in quorum (ranks 0,1) 2023-06-03T19:24:50.031 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:24:49 smithi089 bash[26572]: cluster 2023-06-03T19:24:49.645089+0000 mon.a (mon.0) 137 : cluster [DBG] monmap e2: 2 mons at {a=[v2:172.21.15.89:3300/0,v1:172.21.15.89:6789/0],b=[v2:172.21.15.164:3300/0,v1:172.21.15.164:6789/0]} removed_ranks: {} 2023-06-03T19:24:50.032 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:24:49 smithi089 bash[26572]: cluster 2023-06-03T19:24:49.646140+0000 mon.a (mon.0) 138 : cluster [DBG] fsmap 2023-06-03T19:24:50.032 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:24:49 smithi089 bash[26572]: cluster 2023-06-03T19:24:49.646201+0000 mon.a (mon.0) 139 : cluster [DBG] osdmap e4: 0 total, 0 up, 0 in 2023-06-03T19:24:50.032 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:24:49 smithi089 bash[26572]: cluster 2023-06-03T19:24:49.646597+0000 mon.a (mon.0) 140 : cluster [DBG] mgrmap e13: y(active, since 54s) 2023-06-03T19:24:50.032 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:24:49 smithi089 bash[26572]: cluster 2023-06-03T19:24:49.652395+0000 mon.a (mon.0) 141 : cluster [INF] overall HEALTH_OK 2023-06-03T19:24:50.032 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:24:49 smithi089 bash[26572]: audit 2023-06-03T19:24:49.656814+0000 mon.a (mon.0) 142 : audit [INF] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' 2023-06-03T19:24:50.032 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:24:49 smithi089 bash[26572]: audit 2023-06-03T19:24:49.658548+0000 mon.a (mon.0) 143 : audit [DBG] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-03T19:24:50.084 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:24:49 smithi164 bash[26855]: audit 2023-06-03T19:24:44.598956+0000 mon.a (mon.0) 127 : audit [DBG] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-06-03T19:24:50.085 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:24:49 smithi164 bash[26855]: cluster 2023-06-03T19:24:44.599317+0000 mon.a (mon.0) 128 : cluster [INF] mon.a calling monitor election 2023-06-03T19:24:50.085 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:24:49 smithi164 bash[26855]: audit 2023-06-03T19:24:44.617991+0000 mon.a (mon.0) 129 : audit [DBG] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-06-03T19:24:50.085 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:24:49 smithi164 bash[26855]: audit 2023-06-03T19:24:45.596078+0000 mon.a (mon.0) 130 : audit [DBG] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-06-03T19:24:50.085 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:24:49 smithi164 bash[26855]: cluster 2023-06-03T19:24:45.695503+0000 mgr.y (mgr.14146) 32 : cluster [DBG] pgmap v19: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-06-03T19:24:50.085 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:24:49 smithi164 bash[26855]: audit 2023-06-03T19:24:46.595995+0000 mon.a (mon.0) 131 : audit [DBG] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-06-03T19:24:50.085 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:24:49 smithi164 bash[26855]: cluster 2023-06-03T19:24:46.596426+0000 mon.b (mon.1) 1 : cluster [INF] mon.b calling monitor election 2023-06-03T19:24:50.086 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:24:49 smithi164 bash[26855]: audit 2023-06-03T19:24:47.596138+0000 mon.a (mon.0) 132 : audit [DBG] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-06-03T19:24:50.086 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:24:49 smithi164 bash[26855]: cluster 2023-06-03T19:24:47.695903+0000 mgr.y (mgr.14146) 33 : cluster [DBG] pgmap v20: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-06-03T19:24:50.086 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:24:49 smithi164 bash[26855]: audit 2023-06-03T19:24:48.596532+0000 mon.a (mon.0) 133 : audit [DBG] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-06-03T19:24:50.086 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:24:49 smithi164 bash[26855]: audit 2023-06-03T19:24:49.018983+0000 mon.a (mon.0) 134 : audit [DBG] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-06-03T19:24:50.086 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:24:49 smithi164 bash[26855]: audit 2023-06-03T19:24:49.596925+0000 mon.a (mon.0) 135 : audit [DBG] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-06-03T19:24:50.086 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:24:49 smithi164 bash[26855]: cluster 2023-06-03T19:24:49.638124+0000 mon.a (mon.0) 136 : cluster [INF] mon.a is new leader, mons a,b in quorum (ranks 0,1) 2023-06-03T19:24:50.086 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:24:49 smithi164 bash[26855]: cluster 2023-06-03T19:24:49.645089+0000 mon.a (mon.0) 137 : cluster [DBG] monmap e2: 2 mons at {a=[v2:172.21.15.89:3300/0,v1:172.21.15.89:6789/0],b=[v2:172.21.15.164:3300/0,v1:172.21.15.164:6789/0]} removed_ranks: {} 2023-06-03T19:24:50.086 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:24:49 smithi164 bash[26855]: cluster 2023-06-03T19:24:49.646140+0000 mon.a (mon.0) 138 : cluster [DBG] fsmap 2023-06-03T19:24:50.087 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:24:49 smithi164 bash[26855]: cluster 2023-06-03T19:24:49.646201+0000 mon.a (mon.0) 139 : cluster [DBG] osdmap e4: 0 total, 0 up, 0 in 2023-06-03T19:24:50.087 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:24:49 smithi164 bash[26855]: cluster 2023-06-03T19:24:49.646597+0000 mon.a (mon.0) 140 : cluster [DBG] mgrmap e13: y(active, since 54s) 2023-06-03T19:24:50.087 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:24:49 smithi164 bash[26855]: cluster 2023-06-03T19:24:49.652395+0000 mon.a (mon.0) 141 : cluster [INF] overall HEALTH_OK 2023-06-03T19:24:50.087 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:24:49 smithi164 bash[26855]: audit 2023-06-03T19:24:49.656814+0000 mon.a (mon.0) 142 : audit [INF] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' 2023-06-03T19:24:50.087 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:24:49 smithi164 bash[26855]: audit 2023-06-03T19:24:49.658548+0000 mon.a (mon.0) 143 : audit [DBG] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-03T19:24:50.947 INFO:teuthology.orchestra.run.smithi164.stdout: 2023-06-03T19:24:50.948 INFO:teuthology.orchestra.run.smithi164.stdout:{"epoch":2,"fsid":"f1db3478-0243-11ee-9b1d-001a4aab830c","modified":"2023-06-03T19:24:44.596257Z","created":"2023-06-03T19:22:49.591927Z","min_mon_release":16,"min_mon_release_name":"pacific","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","removed_ranks: ":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging"],"optional":[]},"mons":[{"rank":0,"name":"a","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:3300","nonce":0},{"type":"v1","addr":"172.21.15.89:6789","nonce":0}]},"addr":"172.21.15.89:6789/0","public_addr":"172.21.15.89:6789/0","priority":0,"weight":0,"crush_location":"{}"},{"rank":1,"name":"b","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.164:3300","nonce":0},{"type":"v1","addr":"172.21.15.164:6789","nonce":0}]},"addr":"172.21.15.164:6789/0","public_addr":"172.21.15.164:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0,1]} 2023-06-03T19:24:50.951 INFO:teuthology.orchestra.run.smithi164.stderr:dumped monmap epoch 2 2023-06-03T19:24:50.999 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:24:50 smithi164 bash[26855]: cluster 2023-06-03T19:24:49.696324+0000 mgr.y (mgr.14146) 34 : cluster [DBG] pgmap v21: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-06-03T19:24:50.999 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:24:50 smithi164 bash[26855]: audit 2023-06-03T19:24:50.019250+0000 mon.a (mon.0) 144 : audit [DBG] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-06-03T19:24:50.999 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:24:50 smithi164 bash[26855]: audit 2023-06-03T19:24:50.597052+0000 mon.a (mon.0) 145 : audit [DBG] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-06-03T19:24:51.029 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:24:50 smithi089 bash[26572]: cluster 2023-06-03T19:24:49.696324+0000 mgr.y (mgr.14146) 34 : cluster [DBG] pgmap v21: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-06-03T19:24:51.029 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:24:50 smithi089 bash[26572]: audit 2023-06-03T19:24:50.019250+0000 mon.a (mon.0) 144 : audit [DBG] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-06-03T19:24:51.029 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:24:50 smithi089 bash[26572]: audit 2023-06-03T19:24:50.597052+0000 mon.a (mon.0) 145 : audit [DBG] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-06-03T19:24:52.655 INFO:tasks.cephadm:Waiting for 3 mons in monmap... 2023-06-03T19:24:52.657 DEBUG:teuthology.orchestra.run.smithi164:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph mon dump -f json 2023-06-03T19:24:56.337 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:24:56 smithi089 bash[26572]: audit 2023-06-03T19:24:51.025366+0000 mon.a (mon.0) 149 : audit [DBG] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-06-03T19:24:56.338 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:24:56 smithi089 bash[26572]: audit 2023-06-03T19:24:51.025620+0000 mon.a (mon.0) 150 : audit [DBG] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-06-03T19:24:56.338 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:24:56 smithi089 bash[26572]: cluster 2023-06-03T19:24:51.025786+0000 mon.a (mon.0) 151 : cluster [INF] mon.a calling monitor election 2023-06-03T19:24:56.338 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:24:56 smithi089 bash[26572]: cluster 2023-06-03T19:24:51.029952+0000 mon.b (mon.1) 2 : cluster [INF] mon.b calling monitor election 2023-06-03T19:24:56.339 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:24:56 smithi089 bash[26572]: audit 2023-06-03T19:24:51.046128+0000 mon.a (mon.0) 152 : audit [DBG] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-06-03T19:24:56.339 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:24:56 smithi089 bash[26572]: cluster 2023-06-03T19:24:51.696763+0000 mgr.y (mgr.14146) 35 : cluster [DBG] pgmap v22: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-06-03T19:24:56.339 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:24:56 smithi089 bash[26572]: audit 2023-06-03T19:24:52.019688+0000 mon.a (mon.0) 153 : audit [DBG] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-06-03T19:24:56.339 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:24:56 smithi089 bash[26572]: audit 2023-06-03T19:24:53.019777+0000 mon.a (mon.0) 154 : audit [DBG] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-06-03T19:24:56.340 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:24:56 smithi089 bash[26572]: cluster 2023-06-03T19:24:53.021541+0000 mon.c (mon.2) 1 : cluster [INF] mon.c calling monitor election 2023-06-03T19:24:56.340 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:24:56 smithi089 bash[26572]: cluster 2023-06-03T19:24:53.697114+0000 mgr.y (mgr.14146) 36 : cluster [DBG] pgmap v23: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-06-03T19:24:56.340 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:24:56 smithi089 bash[26572]: audit 2023-06-03T19:24:54.019742+0000 mon.a (mon.0) 155 : audit [DBG] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-06-03T19:24:56.340 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:24:56 smithi089 bash[26572]: audit 2023-06-03T19:24:55.019954+0000 mon.a (mon.0) 156 : audit [DBG] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-06-03T19:24:56.340 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:24:56 smithi089 bash[26572]: cluster 2023-06-03T19:24:55.697544+0000 mgr.y (mgr.14146) 37 : cluster [DBG] pgmap v24: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-06-03T19:24:56.340 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:24:56 smithi089 bash[26572]: audit 2023-06-03T19:24:56.020064+0000 mon.a (mon.0) 157 : audit [DBG] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-06-03T19:24:56.340 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:24:56 smithi089 bash[26572]: cluster 2023-06-03T19:24:56.034690+0000 mon.a (mon.0) 158 : cluster [INF] mon.a calling monitor election 2023-06-03T19:24:56.341 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:24:56 smithi089 bash[26572]: cluster 2023-06-03T19:24:56.055573+0000 mon.a (mon.0) 159 : cluster [INF] mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2023-06-03T19:24:56.341 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:24:56 smithi089 bash[26572]: cluster 2023-06-03T19:24:56.062997+0000 mon.a (mon.0) 160 : cluster [DBG] monmap e3: 3 mons at {a=[v2:172.21.15.89:3300/0,v1:172.21.15.89:6789/0],b=[v2:172.21.15.164:3300/0,v1:172.21.15.164:6789/0],c=[v2:172.21.15.89:3301/0,v1:172.21.15.89:6790/0]} removed_ranks: {} 2023-06-03T19:24:56.341 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:24:56 smithi089 bash[26572]: cluster 2023-06-03T19:24:56.063867+0000 mon.a (mon.0) 161 : cluster [DBG] fsmap 2023-06-03T19:24:56.341 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:24:56 smithi089 bash[26572]: cluster 2023-06-03T19:24:56.063937+0000 mon.a (mon.0) 162 : cluster [DBG] osdmap e4: 0 total, 0 up, 0 in 2023-06-03T19:24:56.341 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:24:56 smithi089 bash[26572]: cluster 2023-06-03T19:24:56.064261+0000 mon.a (mon.0) 163 : cluster [DBG] mgrmap e13: y(active, since 60s) 2023-06-03T19:24:56.341 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:24:56 smithi089 bash[26572]: cluster 2023-06-03T19:24:56.069963+0000 mon.a (mon.0) 164 : cluster [INF] overall HEALTH_OK 2023-06-03T19:24:56.341 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:24:56 smithi089 bash[26572]: audit 2023-06-03T19:24:56.073619+0000 mon.a (mon.0) 165 : audit [INF] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' 2023-06-03T19:24:56.342 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:24:56 smithi089 bash[26572]: audit 2023-06-03T19:24:56.077804+0000 mon.a (mon.0) 166 : audit [INF] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' 2023-06-03T19:24:56.342 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:24:56 smithi089 bash[26572]: audit 2023-06-03T19:24:56.078918+0000 mon.a (mon.0) 167 : audit [DBG] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-03T19:24:56.342 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:24:56 smithi089 bash[26572]: audit 2023-06-03T19:24:56.080184+0000 mon.a (mon.0) 168 : audit [INF] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-03T19:24:56.342 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:24:56 smithi089 bash[26572]: cephadm 2023-06-03T19:24:56.081898+0000 mgr.y (mgr.14146) 38 : cephadm [INF] Updating smithi089:/etc/ceph/ceph.conf 2023-06-03T19:24:56.342 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:24:56 smithi089 bash[26572]: cephadm 2023-06-03T19:24:56.082297+0000 mgr.y (mgr.14146) 39 : cephadm [INF] Updating smithi164:/etc/ceph/ceph.conf 2023-06-03T19:24:56.342 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:24:56 smithi089 bash[26572]: audit 2023-06-03T19:24:56.108146+0000 mon.a (mon.0) 169 : audit [INF] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' 2023-06-03T19:24:56.342 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:24:56 smithi089 bash[26572]: audit 2023-06-03T19:24:56.113153+0000 mon.a (mon.0) 170 : audit [INF] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' 2023-06-03T19:24:56.343 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:24:56 smithi089 bash[33173]: cluster 2023-06-03T19:24:49.696324+0000 mgr.y (mgr.14146) 34 : cluster [DBG] pgmap v21: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-06-03T19:24:56.343 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:24:56 smithi089 bash[33173]: audit 2023-06-03T19:24:50.019250+0000 mon.a (mon.0) 144 : audit [DBG] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-06-03T19:24:56.343 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:24:56 smithi089 bash[33173]: audit 2023-06-03T19:24:50.597052+0000 mon.a (mon.0) 145 : audit [DBG] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-06-03T19:24:56.343 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:24:56 smithi089 bash[33173]: audit 2023-06-03T19:24:51.025366+0000 mon.a (mon.0) 149 : audit [DBG] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-06-03T19:24:56.344 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:24:56 smithi089 bash[33173]: audit 2023-06-03T19:24:51.025620+0000 mon.a (mon.0) 150 : audit [DBG] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-06-03T19:24:56.344 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:24:56 smithi089 bash[33173]: cluster 2023-06-03T19:24:51.025786+0000 mon.a (mon.0) 151 : cluster [INF] mon.a calling monitor election 2023-06-03T19:24:56.344 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:24:56 smithi089 bash[33173]: cluster 2023-06-03T19:24:51.029952+0000 mon.b (mon.1) 2 : cluster [INF] mon.b calling monitor election 2023-06-03T19:24:56.344 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:24:56 smithi089 bash[33173]: audit 2023-06-03T19:24:51.046128+0000 mon.a (mon.0) 152 : audit [DBG] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-06-03T19:24:56.344 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:24:56 smithi089 bash[33173]: cluster 2023-06-03T19:24:51.696763+0000 mgr.y (mgr.14146) 35 : cluster [DBG] pgmap v22: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-06-03T19:24:56.344 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:24:56 smithi089 bash[33173]: audit 2023-06-03T19:24:52.019688+0000 mon.a (mon.0) 153 : audit [DBG] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-06-03T19:24:56.344 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:24:56 smithi089 bash[33173]: audit 2023-06-03T19:24:53.019777+0000 mon.a (mon.0) 154 : audit [DBG] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-06-03T19:24:56.345 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:24:56 smithi089 bash[33173]: cluster 2023-06-03T19:24:53.021541+0000 mon.c (mon.2) 1 : cluster [INF] mon.c calling monitor election 2023-06-03T19:24:56.345 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:24:56 smithi089 bash[33173]: cluster 2023-06-03T19:24:53.697114+0000 mgr.y (mgr.14146) 36 : cluster [DBG] pgmap v23: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-06-03T19:24:56.345 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:24:56 smithi089 bash[33173]: audit 2023-06-03T19:24:54.019742+0000 mon.a (mon.0) 155 : audit [DBG] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-06-03T19:24:56.345 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:24:56 smithi089 bash[33173]: audit 2023-06-03T19:24:55.019954+0000 mon.a (mon.0) 156 : audit [DBG] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-06-03T19:24:56.345 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:24:56 smithi089 bash[33173]: cluster 2023-06-03T19:24:55.697544+0000 mgr.y (mgr.14146) 37 : cluster [DBG] pgmap v24: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-06-03T19:24:56.345 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:24:56 smithi089 bash[33173]: audit 2023-06-03T19:24:56.020064+0000 mon.a (mon.0) 157 : audit [DBG] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-06-03T19:24:56.345 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:24:56 smithi089 bash[33173]: cluster 2023-06-03T19:24:56.034690+0000 mon.a (mon.0) 158 : cluster [INF] mon.a calling monitor election 2023-06-03T19:24:56.345 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:24:56 smithi089 bash[33173]: cluster 2023-06-03T19:24:56.055573+0000 mon.a (mon.0) 159 : cluster [INF] mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2023-06-03T19:24:56.346 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:24:56 smithi089 bash[33173]: cluster 2023-06-03T19:24:56.062997+0000 mon.a (mon.0) 160 : cluster [DBG] monmap e3: 3 mons at {a=[v2:172.21.15.89:3300/0,v1:172.21.15.89:6789/0],b=[v2:172.21.15.164:3300/0,v1:172.21.15.164:6789/0],c=[v2:172.21.15.89:3301/0,v1:172.21.15.89:6790/0]} removed_ranks: {} 2023-06-03T19:24:56.346 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:24:56 smithi089 bash[33173]: cluster 2023-06-03T19:24:56.063867+0000 mon.a (mon.0) 161 : cluster [DBG] fsmap 2023-06-03T19:24:56.346 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:24:56 smithi089 bash[33173]: cluster 2023-06-03T19:24:56.063937+0000 mon.a (mon.0) 162 : cluster [DBG] osdmap e4: 0 total, 0 up, 0 in 2023-06-03T19:24:56.346 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:24:56 smithi089 bash[33173]: cluster 2023-06-03T19:24:56.064261+0000 mon.a (mon.0) 163 : cluster [DBG] mgrmap e13: y(active, since 60s) 2023-06-03T19:24:56.346 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:24:56 smithi089 bash[33173]: cluster 2023-06-03T19:24:56.069963+0000 mon.a (mon.0) 164 : cluster [INF] overall HEALTH_OK 2023-06-03T19:24:56.346 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:24:56 smithi089 bash[33173]: audit 2023-06-03T19:24:56.073619+0000 mon.a (mon.0) 165 : audit [INF] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' 2023-06-03T19:24:56.346 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:24:56 smithi089 bash[33173]: audit 2023-06-03T19:24:56.077804+0000 mon.a (mon.0) 166 : audit [INF] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' 2023-06-03T19:24:56.347 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:24:56 smithi089 bash[33173]: audit 2023-06-03T19:24:56.078918+0000 mon.a (mon.0) 167 : audit [DBG] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-03T19:24:56.347 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:24:56 smithi089 bash[33173]: audit 2023-06-03T19:24:56.080184+0000 mon.a (mon.0) 168 : audit [INF] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-03T19:24:56.347 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:24:56 smithi089 bash[33173]: cephadm 2023-06-03T19:24:56.081898+0000 mgr.y (mgr.14146) 38 : cephadm [INF] Updating smithi089:/etc/ceph/ceph.conf 2023-06-03T19:24:56.347 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:24:56 smithi089 bash[33173]: cephadm 2023-06-03T19:24:56.082297+0000 mgr.y (mgr.14146) 39 : cephadm [INF] Updating smithi164:/etc/ceph/ceph.conf 2023-06-03T19:24:56.347 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:24:56 smithi089 bash[33173]: audit 2023-06-03T19:24:56.108146+0000 mon.a (mon.0) 169 : audit [INF] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' 2023-06-03T19:24:56.347 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:24:56 smithi089 bash[33173]: audit 2023-06-03T19:24:56.113153+0000 mon.a (mon.0) 170 : audit [INF] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' 2023-06-03T19:24:56.585 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:24:56 smithi164 bash[26855]: audit 2023-06-03T19:24:51.025366+0000 mon.a (mon.0) 149 : audit [DBG] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-06-03T19:24:56.585 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:24:56 smithi164 bash[26855]: audit 2023-06-03T19:24:51.025620+0000 mon.a (mon.0) 150 : audit [DBG] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-06-03T19:24:56.585 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:24:56 smithi164 bash[26855]: cluster 2023-06-03T19:24:51.025786+0000 mon.a (mon.0) 151 : cluster [INF] mon.a calling monitor election 2023-06-03T19:24:56.585 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:24:56 smithi164 bash[26855]: cluster 2023-06-03T19:24:51.029952+0000 mon.b (mon.1) 2 : cluster [INF] mon.b calling monitor election 2023-06-03T19:24:56.585 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:24:56 smithi164 bash[26855]: audit 2023-06-03T19:24:51.046128+0000 mon.a (mon.0) 152 : audit [DBG] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-06-03T19:24:56.586 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:24:56 smithi164 bash[26855]: cluster 2023-06-03T19:24:51.696763+0000 mgr.y (mgr.14146) 35 : cluster [DBG] pgmap v22: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-06-03T19:24:56.586 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:24:56 smithi164 bash[26855]: audit 2023-06-03T19:24:52.019688+0000 mon.a (mon.0) 153 : audit [DBG] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-06-03T19:24:56.586 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:24:56 smithi164 bash[26855]: audit 2023-06-03T19:24:53.019777+0000 mon.a (mon.0) 154 : audit [DBG] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-06-03T19:24:56.586 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:24:56 smithi164 bash[26855]: cluster 2023-06-03T19:24:53.021541+0000 mon.c (mon.2) 1 : cluster [INF] mon.c calling monitor election 2023-06-03T19:24:56.586 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:24:56 smithi164 bash[26855]: cluster 2023-06-03T19:24:53.697114+0000 mgr.y (mgr.14146) 36 : cluster [DBG] pgmap v23: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-06-03T19:24:56.586 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:24:56 smithi164 bash[26855]: audit 2023-06-03T19:24:54.019742+0000 mon.a (mon.0) 155 : audit [DBG] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-06-03T19:24:56.586 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:24:56 smithi164 bash[26855]: audit 2023-06-03T19:24:55.019954+0000 mon.a (mon.0) 156 : audit [DBG] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-06-03T19:24:56.586 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:24:56 smithi164 bash[26855]: cluster 2023-06-03T19:24:55.697544+0000 mgr.y (mgr.14146) 37 : cluster [DBG] pgmap v24: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-06-03T19:24:56.587 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:24:56 smithi164 bash[26855]: audit 2023-06-03T19:24:56.020064+0000 mon.a (mon.0) 157 : audit [DBG] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-06-03T19:24:56.587 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:24:56 smithi164 bash[26855]: cluster 2023-06-03T19:24:56.034690+0000 mon.a (mon.0) 158 : cluster [INF] mon.a calling monitor election 2023-06-03T19:24:56.587 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:24:56 smithi164 bash[26855]: cluster 2023-06-03T19:24:56.055573+0000 mon.a (mon.0) 159 : cluster [INF] mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2023-06-03T19:24:56.587 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:24:56 smithi164 bash[26855]: cluster 2023-06-03T19:24:56.062997+0000 mon.a (mon.0) 160 : cluster [DBG] monmap e3: 3 mons at {a=[v2:172.21.15.89:3300/0,v1:172.21.15.89:6789/0],b=[v2:172.21.15.164:3300/0,v1:172.21.15.164:6789/0],c=[v2:172.21.15.89:3301/0,v1:172.21.15.89:6790/0]} removed_ranks: {} 2023-06-03T19:24:56.587 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:24:56 smithi164 bash[26855]: cluster 2023-06-03T19:24:56.063867+0000 mon.a (mon.0) 161 : cluster [DBG] fsmap 2023-06-03T19:24:56.587 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:24:56 smithi164 bash[26855]: cluster 2023-06-03T19:24:56.063937+0000 mon.a (mon.0) 162 : cluster [DBG] osdmap e4: 0 total, 0 up, 0 in 2023-06-03T19:24:56.587 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:24:56 smithi164 bash[26855]: cluster 2023-06-03T19:24:56.064261+0000 mon.a (mon.0) 163 : cluster [DBG] mgrmap e13: y(active, since 60s) 2023-06-03T19:24:56.588 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:24:56 smithi164 bash[26855]: cluster 2023-06-03T19:24:56.069963+0000 mon.a (mon.0) 164 : cluster [INF] overall HEALTH_OK 2023-06-03T19:24:56.588 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:24:56 smithi164 bash[26855]: audit 2023-06-03T19:24:56.073619+0000 mon.a (mon.0) 165 : audit [INF] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' 2023-06-03T19:24:56.588 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:24:56 smithi164 bash[26855]: audit 2023-06-03T19:24:56.077804+0000 mon.a (mon.0) 166 : audit [INF] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' 2023-06-03T19:24:56.588 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:24:56 smithi164 bash[26855]: audit 2023-06-03T19:24:56.078918+0000 mon.a (mon.0) 167 : audit [DBG] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-03T19:24:56.588 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:24:56 smithi164 bash[26855]: audit 2023-06-03T19:24:56.080184+0000 mon.a (mon.0) 168 : audit [INF] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-03T19:24:56.588 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:24:56 smithi164 bash[26855]: cephadm 2023-06-03T19:24:56.081898+0000 mgr.y (mgr.14146) 38 : cephadm [INF] Updating smithi089:/etc/ceph/ceph.conf 2023-06-03T19:24:56.588 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:24:56 smithi164 bash[26855]: cephadm 2023-06-03T19:24:56.082297+0000 mgr.y (mgr.14146) 39 : cephadm [INF] Updating smithi164:/etc/ceph/ceph.conf 2023-06-03T19:24:56.589 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:24:56 smithi164 bash[26855]: audit 2023-06-03T19:24:56.108146+0000 mon.a (mon.0) 169 : audit [INF] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' 2023-06-03T19:24:56.589 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:24:56 smithi164 bash[26855]: audit 2023-06-03T19:24:56.113153+0000 mon.a (mon.0) 170 : audit [INF] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' 2023-06-03T19:24:57.079 INFO:teuthology.orchestra.run.smithi164.stdout: 2023-06-03T19:24:57.080 INFO:teuthology.orchestra.run.smithi164.stdout:{"epoch":3,"fsid":"f1db3478-0243-11ee-9b1d-001a4aab830c","modified":"2023-06-03T19:24:51.019689Z","created":"2023-06-03T19:22:49.591927Z","min_mon_release":16,"min_mon_release_name":"pacific","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","removed_ranks: ":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging"],"optional":[]},"mons":[{"rank":0,"name":"a","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:3300","nonce":0},{"type":"v1","addr":"172.21.15.89:6789","nonce":0}]},"addr":"172.21.15.89:6789/0","public_addr":"172.21.15.89:6789/0","priority":0,"weight":0,"crush_location":"{}"},{"rank":1,"name":"b","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.164:3300","nonce":0},{"type":"v1","addr":"172.21.15.164:6789","nonce":0}]},"addr":"172.21.15.164:6789/0","public_addr":"172.21.15.164:6789/0","priority":0,"weight":0,"crush_location":"{}"},{"rank":2,"name":"c","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:3301","nonce":0},{"type":"v1","addr":"172.21.15.89:6790","nonce":0}]},"addr":"172.21.15.89:6790/0","public_addr":"172.21.15.89:6790/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0,1,2]} 2023-06-03T19:24:57.083 INFO:teuthology.orchestra.run.smithi164.stderr:dumped monmap epoch 3 2023-06-03T19:24:57.425 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:24:57 smithi089 bash[26572]: audit 2023-06-03T19:24:56.117548+0000 mon.a (mon.0) 171 : audit [INF] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' 2023-06-03T19:24:57.425 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:24:57 smithi089 bash[26572]: cephadm 2023-06-03T19:24:56.119702+0000 mgr.y (mgr.14146) 40 : cephadm [INF] Reconfiguring mon.a (unknown last config time)... 2023-06-03T19:24:57.425 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:24:57 smithi089 bash[26572]: audit 2023-06-03T19:24:56.125198+0000 mon.a (mon.0) 172 : audit [INF] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-06-03T19:24:57.425 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:24:57 smithi089 bash[26572]: audit 2023-06-03T19:24:56.126376+0000 mon.a (mon.0) 173 : audit [DBG] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-06-03T19:24:57.425 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:24:57 smithi089 bash[26572]: audit 2023-06-03T19:24:56.127263+0000 mon.a (mon.0) 174 : audit [DBG] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-03T19:24:57.425 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:24:57 smithi089 bash[26572]: cephadm 2023-06-03T19:24:56.128128+0000 mgr.y (mgr.14146) 41 : cephadm [INF] Reconfiguring daemon mon.a on smithi089 2023-06-03T19:24:57.426 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:24:57 smithi089 bash[26572]: audit 2023-06-03T19:24:57.020378+0000 mon.a (mon.0) 175 : audit [DBG] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-06-03T19:24:57.426 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:24:57 smithi089 bash[26572]: audit 2023-06-03T19:24:57.074129+0000 mon.a (mon.0) 176 : audit [DBG] from='client.? 172.21.15.164:0/2689213790' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-06-03T19:24:57.426 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:24:57 smithi089 bash[33173]: audit 2023-06-03T19:24:56.117548+0000 mon.a (mon.0) 171 : audit [INF] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' 2023-06-03T19:24:57.426 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:24:57 smithi089 bash[33173]: cephadm 2023-06-03T19:24:56.119702+0000 mgr.y (mgr.14146) 40 : cephadm [INF] Reconfiguring mon.a (unknown last config time)... 2023-06-03T19:24:57.426 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:24:57 smithi089 bash[33173]: audit 2023-06-03T19:24:56.125198+0000 mon.a (mon.0) 172 : audit [INF] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-06-03T19:24:57.426 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:24:57 smithi089 bash[33173]: audit 2023-06-03T19:24:56.126376+0000 mon.a (mon.0) 173 : audit [DBG] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-06-03T19:24:57.427 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:24:57 smithi089 bash[33173]: audit 2023-06-03T19:24:56.127263+0000 mon.a (mon.0) 174 : audit [DBG] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-03T19:24:57.427 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:24:57 smithi089 bash[33173]: cephadm 2023-06-03T19:24:56.128128+0000 mgr.y (mgr.14146) 41 : cephadm [INF] Reconfiguring daemon mon.a on smithi089 2023-06-03T19:24:57.427 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:24:57 smithi089 bash[33173]: audit 2023-06-03T19:24:57.020378+0000 mon.a (mon.0) 175 : audit [DBG] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-06-03T19:24:57.427 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:24:57 smithi089 bash[33173]: audit 2023-06-03T19:24:57.074129+0000 mon.a (mon.0) 176 : audit [DBG] from='client.? 172.21.15.164:0/2689213790' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-06-03T19:24:57.585 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:24:57 smithi164 bash[26855]: audit 2023-06-03T19:24:56.117548+0000 mon.a (mon.0) 171 : audit [INF] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' 2023-06-03T19:24:57.585 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:24:57 smithi164 bash[26855]: cephadm 2023-06-03T19:24:56.119702+0000 mgr.y (mgr.14146) 40 : cephadm [INF] Reconfiguring mon.a (unknown last config time)... 2023-06-03T19:24:57.585 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:24:57 smithi164 bash[26855]: audit 2023-06-03T19:24:56.125198+0000 mon.a (mon.0) 172 : audit [INF] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-06-03T19:24:57.585 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:24:57 smithi164 bash[26855]: audit 2023-06-03T19:24:56.126376+0000 mon.a (mon.0) 173 : audit [DBG] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-06-03T19:24:57.585 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:24:57 smithi164 bash[26855]: audit 2023-06-03T19:24:56.127263+0000 mon.a (mon.0) 174 : audit [DBG] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-03T19:24:57.586 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:24:57 smithi164 bash[26855]: cephadm 2023-06-03T19:24:56.128128+0000 mgr.y (mgr.14146) 41 : cephadm [INF] Reconfiguring daemon mon.a on smithi089 2023-06-03T19:24:57.586 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:24:57 smithi164 bash[26855]: audit 2023-06-03T19:24:57.020378+0000 mon.a (mon.0) 175 : audit [DBG] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-06-03T19:24:57.586 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:24:57 smithi164 bash[26855]: audit 2023-06-03T19:24:57.074129+0000 mon.a (mon.0) 176 : audit [DBG] from='client.? 172.21.15.164:0/2689213790' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-06-03T19:24:57.786 INFO:tasks.cephadm:Generating final ceph.conf file... 2023-06-03T19:24:57.786 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph config generate-minimal-conf 2023-06-03T19:24:58.279 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:24:58 smithi089 bash[26572]: cluster 2023-06-03T19:24:57.698084+0000 mgr.y (mgr.14146) 42 : cluster [DBG] pgmap v25: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-06-03T19:24:58.280 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:24:58 smithi089 bash[33173]: cluster 2023-06-03T19:24:57.698084+0000 mgr.y (mgr.14146) 42 : cluster [DBG] pgmap v25: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-06-03T19:24:58.584 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:24:58 smithi164 bash[26855]: cluster 2023-06-03T19:24:57.698084+0000 mgr.y (mgr.14146) 42 : cluster [DBG] pgmap v25: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-06-03T19:24:59.903 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:24:59 smithi089 bash[26572]: audit 2023-06-03T19:24:58.610238+0000 mon.a (mon.0) 177 : audit [INF] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' 2023-06-03T19:24:59.904 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:24:59 smithi089 bash[26572]: cephadm 2023-06-03T19:24:58.611077+0000 mgr.y (mgr.14146) 43 : cephadm [INF] Reconfiguring mon.c (monmap changed)... 2023-06-03T19:24:59.904 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:24:59 smithi089 bash[26572]: audit 2023-06-03T19:24:58.611468+0000 mon.a (mon.0) 178 : audit [INF] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-06-03T19:24:59.904 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:24:59 smithi089 bash[26572]: audit 2023-06-03T19:24:58.612591+0000 mon.a (mon.0) 179 : audit [DBG] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-06-03T19:24:59.904 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:24:59 smithi089 bash[26572]: audit 2023-06-03T19:24:58.613657+0000 mon.a (mon.0) 180 : audit [DBG] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-03T19:24:59.905 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:24:59 smithi089 bash[26572]: cephadm 2023-06-03T19:24:58.614715+0000 mgr.y (mgr.14146) 44 : cephadm [INF] Reconfiguring daemon mon.c on smithi089 2023-06-03T19:24:59.905 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:24:59 smithi089 bash[33173]: audit 2023-06-03T19:24:58.610238+0000 mon.a (mon.0) 177 : audit [INF] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' 2023-06-03T19:24:59.905 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:24:59 smithi089 bash[33173]: cephadm 2023-06-03T19:24:58.611077+0000 mgr.y (mgr.14146) 43 : cephadm [INF] Reconfiguring mon.c (monmap changed)... 2023-06-03T19:24:59.905 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:24:59 smithi089 bash[33173]: audit 2023-06-03T19:24:58.611468+0000 mon.a (mon.0) 178 : audit [INF] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-06-03T19:24:59.905 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:24:59 smithi089 bash[33173]: audit 2023-06-03T19:24:58.612591+0000 mon.a (mon.0) 179 : audit [DBG] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-06-03T19:24:59.905 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:24:59 smithi089 bash[33173]: audit 2023-06-03T19:24:58.613657+0000 mon.a (mon.0) 180 : audit [DBG] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-03T19:24:59.906 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:24:59 smithi089 bash[33173]: cephadm 2023-06-03T19:24:58.614715+0000 mgr.y (mgr.14146) 44 : cephadm [INF] Reconfiguring daemon mon.c on smithi089 2023-06-03T19:25:00.084 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:24:59 smithi164 bash[26855]: audit 2023-06-03T19:24:58.610238+0000 mon.a (mon.0) 177 : audit [INF] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' 2023-06-03T19:25:00.084 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:24:59 smithi164 bash[26855]: cephadm 2023-06-03T19:24:58.611077+0000 mgr.y (mgr.14146) 43 : cephadm [INF] Reconfiguring mon.c (monmap changed)... 2023-06-03T19:25:00.084 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:24:59 smithi164 bash[26855]: audit 2023-06-03T19:24:58.611468+0000 mon.a (mon.0) 178 : audit [INF] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-06-03T19:25:00.085 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:24:59 smithi164 bash[26855]: audit 2023-06-03T19:24:58.612591+0000 mon.a (mon.0) 179 : audit [DBG] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-06-03T19:25:00.085 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:24:59 smithi164 bash[26855]: audit 2023-06-03T19:24:58.613657+0000 mon.a (mon.0) 180 : audit [DBG] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-03T19:25:00.085 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:24:59 smithi164 bash[26855]: cephadm 2023-06-03T19:24:58.614715+0000 mgr.y (mgr.14146) 44 : cephadm [INF] Reconfiguring daemon mon.c on smithi089 2023-06-03T19:25:01.031 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:25:00 smithi089 bash[26572]: cluster 2023-06-03T19:24:59.698521+0000 mgr.y (mgr.14146) 45 : cluster [DBG] pgmap v26: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-06-03T19:25:01.032 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:25:00 smithi089 bash[33173]: cluster 2023-06-03T19:24:59.698521+0000 mgr.y (mgr.14146) 45 : cluster [DBG] pgmap v26: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-06-03T19:25:01.084 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:25:00 smithi164 bash[26855]: cluster 2023-06-03T19:24:59.698521+0000 mgr.y (mgr.14146) 45 : cluster [DBG] pgmap v26: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-06-03T19:25:02.571 INFO:teuthology.orchestra.run.smithi089.stdout:# minimal ceph.conf for f1db3478-0243-11ee-9b1d-001a4aab830c 2023-06-03T19:25:02.572 INFO:teuthology.orchestra.run.smithi089.stdout:[global] 2023-06-03T19:25:02.572 INFO:teuthology.orchestra.run.smithi089.stdout: fsid = f1db3478-0243-11ee-9b1d-001a4aab830c 2023-06-03T19:25:02.572 INFO:teuthology.orchestra.run.smithi089.stdout: mon_host = [v2:172.21.15.89:3300/0,v1:172.21.15.89:6789/0] [v2:172.21.15.164:3300/0,v1:172.21.15.164:6789/0] [v2:172.21.15.89:3301/0,v1:172.21.15.89:6790/0] 2023-06-03T19:25:02.779 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:25:02 smithi089 bash[26572]: audit 2023-06-03T19:25:01.496186+0000 mon.a (mon.0) 181 : audit [INF] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' 2023-06-03T19:25:02.779 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:25:02 smithi089 bash[26572]: cephadm 2023-06-03T19:25:01.497194+0000 mgr.y (mgr.14146) 46 : cephadm [INF] Reconfiguring mon.b (monmap changed)... 2023-06-03T19:25:02.779 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:25:02 smithi089 bash[26572]: audit 2023-06-03T19:25:01.497601+0000 mon.a (mon.0) 182 : audit [INF] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-06-03T19:25:02.779 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:25:02 smithi089 bash[26572]: audit 2023-06-03T19:25:01.498577+0000 mon.a (mon.0) 183 : audit [DBG] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-06-03T19:25:02.780 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:25:02 smithi089 bash[26572]: audit 2023-06-03T19:25:01.499466+0000 mon.a (mon.0) 184 : audit [DBG] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-03T19:25:02.780 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:25:02 smithi089 bash[26572]: cephadm 2023-06-03T19:25:01.500179+0000 mgr.y (mgr.14146) 47 : cephadm [INF] Reconfiguring daemon mon.b on smithi164 2023-06-03T19:25:02.780 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:25:02 smithi089 bash[26572]: cluster 2023-06-03T19:25:01.698932+0000 mgr.y (mgr.14146) 48 : cluster [DBG] pgmap v27: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-06-03T19:25:02.780 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:25:02 smithi089 bash[33173]: audit 2023-06-03T19:25:01.496186+0000 mon.a (mon.0) 181 : audit [INF] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' 2023-06-03T19:25:02.780 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:25:02 smithi089 bash[33173]: cephadm 2023-06-03T19:25:01.497194+0000 mgr.y (mgr.14146) 46 : cephadm [INF] Reconfiguring mon.b (monmap changed)... 2023-06-03T19:25:02.781 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:25:02 smithi089 bash[33173]: audit 2023-06-03T19:25:01.497601+0000 mon.a (mon.0) 182 : audit [INF] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-06-03T19:25:02.782 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:25:02 smithi089 bash[33173]: audit 2023-06-03T19:25:01.498577+0000 mon.a (mon.0) 183 : audit [DBG] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-06-03T19:25:02.782 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:25:02 smithi089 bash[33173]: audit 2023-06-03T19:25:01.499466+0000 mon.a (mon.0) 184 : audit [DBG] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-03T19:25:02.782 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:25:02 smithi089 bash[33173]: cephadm 2023-06-03T19:25:01.500179+0000 mgr.y (mgr.14146) 47 : cephadm [INF] Reconfiguring daemon mon.b on smithi164 2023-06-03T19:25:02.782 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:25:02 smithi089 bash[33173]: cluster 2023-06-03T19:25:01.698932+0000 mgr.y (mgr.14146) 48 : cluster [DBG] pgmap v27: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-06-03T19:25:02.787 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:25:02 smithi164 bash[26855]: audit 2023-06-03T19:25:01.496186+0000 mon.a (mon.0) 181 : audit [INF] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' 2023-06-03T19:25:02.787 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:25:02 smithi164 bash[26855]: cephadm 2023-06-03T19:25:01.497194+0000 mgr.y (mgr.14146) 46 : cephadm [INF] Reconfiguring mon.b (monmap changed)... 2023-06-03T19:25:02.788 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:25:02 smithi164 bash[26855]: audit 2023-06-03T19:25:01.497601+0000 mon.a (mon.0) 182 : audit [INF] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-06-03T19:25:02.788 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:25:02 smithi164 bash[26855]: audit 2023-06-03T19:25:01.498577+0000 mon.a (mon.0) 183 : audit [DBG] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-06-03T19:25:02.788 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:25:02 smithi164 bash[26855]: audit 2023-06-03T19:25:01.499466+0000 mon.a (mon.0) 184 : audit [DBG] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-03T19:25:02.789 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:25:02 smithi164 bash[26855]: cephadm 2023-06-03T19:25:01.500179+0000 mgr.y (mgr.14146) 47 : cephadm [INF] Reconfiguring daemon mon.b on smithi164 2023-06-03T19:25:02.789 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:25:02 smithi164 bash[26855]: cluster 2023-06-03T19:25:01.698932+0000 mgr.y (mgr.14146) 48 : cluster [DBG] pgmap v27: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-06-03T19:25:03.322 INFO:tasks.cephadm:Distributing (final) config and client.admin keyring... 2023-06-03T19:25:03.322 DEBUG:teuthology.orchestra.run.smithi089:> set -ex 2023-06-03T19:25:03.323 DEBUG:teuthology.orchestra.run.smithi089:> sudo dd of=/etc/ceph/ceph.conf 2023-06-03T19:25:03.338 DEBUG:teuthology.orchestra.run.smithi089:> set -ex 2023-06-03T19:25:03.338 DEBUG:teuthology.orchestra.run.smithi089:> sudo dd of=/etc/ceph/ceph.client.admin.keyring 2023-06-03T19:25:03.396 DEBUG:teuthology.orchestra.run.smithi164:> set -ex 2023-06-03T19:25:03.397 DEBUG:teuthology.orchestra.run.smithi164:> sudo dd of=/etc/ceph/ceph.conf 2023-06-03T19:25:03.414 DEBUG:teuthology.orchestra.run.smithi164:> set -ex 2023-06-03T19:25:03.414 DEBUG:teuthology.orchestra.run.smithi164:> sudo dd of=/etc/ceph/ceph.client.admin.keyring 2023-06-03T19:25:03.472 INFO:tasks.cephadm:Adding mgr.y on smithi089 2023-06-03T19:25:03.473 INFO:tasks.cephadm:Adding mgr.x on smithi164 2023-06-03T19:25:03.473 DEBUG:teuthology.orchestra.run.smithi164:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph orch apply mgr '2;smithi089=y;smithi164=x' 2023-06-03T19:25:03.721 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:25:03 smithi164 bash[26855]: audit 2023-06-03T19:25:02.566697+0000 mon.a (mon.0) 185 : audit [DBG] from='client.? 172.21.15.89:0/1799943580' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-03T19:25:03.721 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:25:03 smithi164 bash[26855]: audit 2023-06-03T19:25:03.418334+0000 mon.a (mon.0) 186 : audit [INF] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' 2023-06-03T19:25:03.721 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:25:03 smithi164 bash[26855]: audit 2023-06-03T19:25:03.420122+0000 mon.a (mon.0) 187 : audit [DBG] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-03T19:25:03.722 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:25:03 smithi164 bash[26855]: audit 2023-06-03T19:25:03.422727+0000 mon.a (mon.0) 188 : audit [DBG] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-03T19:25:03.722 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:25:03 smithi164 bash[26855]: audit 2023-06-03T19:25:03.423928+0000 mon.a (mon.0) 189 : audit [INF] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-03T19:25:03.722 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:25:03 smithi164 bash[26855]: audit 2023-06-03T19:25:03.430411+0000 mon.a (mon.0) 190 : audit [INF] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' 2023-06-03T19:25:03.779 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:25:03 smithi089 bash[26572]: audit 2023-06-03T19:25:02.566697+0000 mon.a (mon.0) 185 : audit [DBG] from='client.? 172.21.15.89:0/1799943580' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-03T19:25:03.780 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:25:03 smithi089 bash[26572]: audit 2023-06-03T19:25:03.418334+0000 mon.a (mon.0) 186 : audit [INF] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' 2023-06-03T19:25:03.780 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:25:03 smithi089 bash[26572]: audit 2023-06-03T19:25:03.420122+0000 mon.a (mon.0) 187 : audit [DBG] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-03T19:25:03.780 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:25:03 smithi089 bash[26572]: audit 2023-06-03T19:25:03.422727+0000 mon.a (mon.0) 188 : audit [DBG] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-03T19:25:03.780 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:25:03 smithi089 bash[26572]: audit 2023-06-03T19:25:03.423928+0000 mon.a (mon.0) 189 : audit [INF] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-03T19:25:03.781 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:25:03 smithi089 bash[26572]: audit 2023-06-03T19:25:03.430411+0000 mon.a (mon.0) 190 : audit [INF] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' 2023-06-03T19:25:03.781 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:25:03 smithi089 bash[33173]: audit 2023-06-03T19:25:02.566697+0000 mon.a (mon.0) 185 : audit [DBG] from='client.? 172.21.15.89:0/1799943580' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-03T19:25:03.782 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:25:03 smithi089 bash[33173]: audit 2023-06-03T19:25:03.418334+0000 mon.a (mon.0) 186 : audit [INF] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' 2023-06-03T19:25:03.782 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:25:03 smithi089 bash[33173]: audit 2023-06-03T19:25:03.420122+0000 mon.a (mon.0) 187 : audit [DBG] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-03T19:25:03.782 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:25:03 smithi089 bash[33173]: audit 2023-06-03T19:25:03.422727+0000 mon.a (mon.0) 188 : audit [DBG] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-03T19:25:03.782 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:25:03 smithi089 bash[33173]: audit 2023-06-03T19:25:03.423928+0000 mon.a (mon.0) 189 : audit [INF] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-03T19:25:03.782 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:25:03 smithi089 bash[33173]: audit 2023-06-03T19:25:03.430411+0000 mon.a (mon.0) 190 : audit [INF] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' 2023-06-03T19:25:04.781 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:25:04 smithi089 bash[26572]: cluster 2023-06-03T19:25:03.699437+0000 mgr.y (mgr.14146) 49 : cluster [DBG] pgmap v28: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-06-03T19:25:04.782 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:25:04 smithi089 bash[33173]: cluster 2023-06-03T19:25:03.699437+0000 mgr.y (mgr.14146) 49 : cluster [DBG] pgmap v28: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-06-03T19:25:04.799 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:25:04 smithi164 bash[26855]: cluster 2023-06-03T19:25:03.699437+0000 mgr.y (mgr.14146) 49 : cluster [DBG] pgmap v28: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-06-03T19:25:06.632 INFO:teuthology.orchestra.run.smithi164.stdout:Scheduled mgr update... 2023-06-03T19:25:07.029 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:25:06 smithi089 bash[26572]: cluster 2023-06-03T19:25:05.699851+0000 mgr.y (mgr.14146) 50 : cluster [DBG] pgmap v29: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-06-03T19:25:07.029 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:25:06 smithi089 bash[26572]: audit 2023-06-03T19:25:06.626536+0000 mon.a (mon.0) 191 : audit [INF] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' 2023-06-03T19:25:07.030 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:25:06 smithi089 bash[26572]: audit 2023-06-03T19:25:06.655817+0000 mon.a (mon.0) 192 : audit [DBG] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-03T19:25:07.030 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:25:06 smithi089 bash[26572]: audit 2023-06-03T19:25:06.659255+0000 mon.a (mon.0) 193 : audit [DBG] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-03T19:25:07.030 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:25:06 smithi089 bash[26572]: audit 2023-06-03T19:25:06.661118+0000 mon.a (mon.0) 194 : audit [INF] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-03T19:25:07.030 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:25:06 smithi089 bash[26572]: audit 2023-06-03T19:25:06.669044+0000 mon.a (mon.0) 195 : audit [INF] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' 2023-06-03T19:25:07.030 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:25:06 smithi089 bash[26572]: audit 2023-06-03T19:25:06.671226+0000 mon.a (mon.0) 196 : audit [INF] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-06-03T19:25:07.030 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:25:06 smithi089 bash[26572]: audit 2023-06-03T19:25:06.675364+0000 mon.a (mon.0) 197 : audit [INF] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]': finished 2023-06-03T19:25:07.030 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:25:06 smithi089 bash[26572]: audit 2023-06-03T19:25:06.676623+0000 mon.a (mon.0) 198 : audit [DBG] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-06-03T19:25:07.031 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:25:06 smithi089 bash[26572]: audit 2023-06-03T19:25:06.678053+0000 mon.a (mon.0) 199 : audit [DBG] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-03T19:25:07.031 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:25:06 smithi089 bash[33173]: cluster 2023-06-03T19:25:05.699851+0000 mgr.y (mgr.14146) 50 : cluster [DBG] pgmap v29: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-06-03T19:25:07.031 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:25:06 smithi089 bash[33173]: audit 2023-06-03T19:25:06.626536+0000 mon.a (mon.0) 191 : audit [INF] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' 2023-06-03T19:25:07.031 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:25:06 smithi089 bash[33173]: audit 2023-06-03T19:25:06.655817+0000 mon.a (mon.0) 192 : audit [DBG] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-03T19:25:07.031 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:25:06 smithi089 bash[33173]: audit 2023-06-03T19:25:06.659255+0000 mon.a (mon.0) 193 : audit [DBG] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-03T19:25:07.031 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:25:06 smithi089 bash[33173]: audit 2023-06-03T19:25:06.661118+0000 mon.a (mon.0) 194 : audit [INF] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-03T19:25:07.032 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:25:06 smithi089 bash[33173]: audit 2023-06-03T19:25:06.669044+0000 mon.a (mon.0) 195 : audit [INF] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' 2023-06-03T19:25:07.032 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:25:06 smithi089 bash[33173]: audit 2023-06-03T19:25:06.671226+0000 mon.a (mon.0) 196 : audit [INF] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-06-03T19:25:07.032 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:25:06 smithi089 bash[33173]: audit 2023-06-03T19:25:06.675364+0000 mon.a (mon.0) 197 : audit [INF] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]': finished 2023-06-03T19:25:07.032 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:25:06 smithi089 bash[33173]: audit 2023-06-03T19:25:06.676623+0000 mon.a (mon.0) 198 : audit [DBG] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-06-03T19:25:07.032 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:25:06 smithi089 bash[33173]: audit 2023-06-03T19:25:06.678053+0000 mon.a (mon.0) 199 : audit [DBG] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-03T19:25:07.084 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:25:06 smithi164 bash[26855]: cluster 2023-06-03T19:25:05.699851+0000 mgr.y (mgr.14146) 50 : cluster [DBG] pgmap v29: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-06-03T19:25:07.085 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:25:06 smithi164 bash[26855]: audit 2023-06-03T19:25:06.626536+0000 mon.a (mon.0) 191 : audit [INF] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' 2023-06-03T19:25:07.085 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:25:06 smithi164 bash[26855]: audit 2023-06-03T19:25:06.655817+0000 mon.a (mon.0) 192 : audit [DBG] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-03T19:25:07.085 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:25:06 smithi164 bash[26855]: audit 2023-06-03T19:25:06.659255+0000 mon.a (mon.0) 193 : audit [DBG] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-03T19:25:07.085 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:25:06 smithi164 bash[26855]: audit 2023-06-03T19:25:06.661118+0000 mon.a (mon.0) 194 : audit [INF] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-03T19:25:07.085 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:25:06 smithi164 bash[26855]: audit 2023-06-03T19:25:06.669044+0000 mon.a (mon.0) 195 : audit [INF] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' 2023-06-03T19:25:07.086 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:25:06 smithi164 bash[26855]: audit 2023-06-03T19:25:06.671226+0000 mon.a (mon.0) 196 : audit [INF] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-06-03T19:25:07.086 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:25:06 smithi164 bash[26855]: audit 2023-06-03T19:25:06.675364+0000 mon.a (mon.0) 197 : audit [INF] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]': finished 2023-06-03T19:25:07.086 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:25:06 smithi164 bash[26855]: audit 2023-06-03T19:25:06.676623+0000 mon.a (mon.0) 198 : audit [DBG] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-06-03T19:25:07.086 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:25:06 smithi164 bash[26855]: audit 2023-06-03T19:25:06.678053+0000 mon.a (mon.0) 199 : audit [DBG] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-03T19:25:07.661 DEBUG:teuthology.orchestra.run.smithi164:mgr.x> sudo journalctl -f -n 0 -u ceph-f1db3478-0243-11ee-9b1d-001a4aab830c@mgr.x.service 2023-06-03T19:25:07.663 INFO:tasks.cephadm:Deploying OSDs... 2023-06-03T19:25:07.664 DEBUG:teuthology.orchestra.run.smithi089:> set -ex 2023-06-03T19:25:07.664 DEBUG:teuthology.orchestra.run.smithi089:> dd if=/scratch_devs of=/dev/stdout 2023-06-03T19:25:07.671 DEBUG:teuthology.misc:devs=['/dev/vg_nvme/lv_1', '/dev/vg_nvme/lv_2', '/dev/vg_nvme/lv_3', '/dev/vg_nvme/lv_4'] 2023-06-03T19:25:07.672 DEBUG:teuthology.orchestra.run.smithi089:> stat /dev/vg_nvme/lv_1 2023-06-03T19:25:07.720 INFO:teuthology.orchestra.run.smithi089.stdout: File: /dev/vg_nvme/lv_1 -> ../dm-0 2023-06-03T19:25:07.720 INFO:teuthology.orchestra.run.smithi089.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2023-06-03T19:25:07.720 INFO:teuthology.orchestra.run.smithi089.stdout:Device: 5h/5d Inode: 494 Links: 1 2023-06-03T19:25:07.720 INFO:teuthology.orchestra.run.smithi089.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2023-06-03T19:25:07.721 INFO:teuthology.orchestra.run.smithi089.stdout:Access: 2023-06-03 19:18:44.140698828 +0000 2023-06-03T19:25:07.721 INFO:teuthology.orchestra.run.smithi089.stdout:Modify: 2023-06-03 19:18:43.800706669 +0000 2023-06-03T19:25:07.722 INFO:teuthology.orchestra.run.smithi089.stdout:Change: 2023-06-03 19:18:43.800706669 +0000 2023-06-03T19:25:07.722 INFO:teuthology.orchestra.run.smithi089.stdout: Birth: - 2023-06-03T19:25:07.722 DEBUG:teuthology.orchestra.run.smithi089:> sudo dd if=/dev/vg_nvme/lv_1 of=/dev/null count=1 2023-06-03T19:25:07.779 INFO:teuthology.orchestra.run.smithi089.stderr:1+0 records in 2023-06-03T19:25:07.779 INFO:teuthology.orchestra.run.smithi089.stderr:1+0 records out 2023-06-03T19:25:07.779 INFO:teuthology.orchestra.run.smithi089.stderr:512 bytes copied, 0.000334252 s, 1.5 MB/s 2023-06-03T19:25:07.780 DEBUG:teuthology.orchestra.run.smithi089:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_1 2023-06-03T19:25:07.829 DEBUG:teuthology.orchestra.run.smithi089:> stat /dev/vg_nvme/lv_2 2023-06-03T19:25:07.879 INFO:teuthology.orchestra.run.smithi089.stdout: File: /dev/vg_nvme/lv_2 -> ../dm-1 2023-06-03T19:25:07.880 INFO:teuthology.orchestra.run.smithi089.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2023-06-03T19:25:07.880 INFO:teuthology.orchestra.run.smithi089.stdout:Device: 5h/5d Inode: 499 Links: 1 2023-06-03T19:25:07.881 INFO:teuthology.orchestra.run.smithi089.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2023-06-03T19:25:07.881 INFO:teuthology.orchestra.run.smithi089.stdout:Access: 2023-06-03 19:18:44.696686006 +0000 2023-06-03T19:25:07.881 INFO:teuthology.orchestra.run.smithi089.stdout:Modify: 2023-06-03 19:18:44.336694308 +0000 2023-06-03T19:25:07.882 INFO:teuthology.orchestra.run.smithi089.stdout:Change: 2023-06-03 19:18:44.336694308 +0000 2023-06-03T19:25:07.882 INFO:teuthology.orchestra.run.smithi089.stdout: Birth: - 2023-06-03T19:25:07.882 DEBUG:teuthology.orchestra.run.smithi089:> sudo dd if=/dev/vg_nvme/lv_2 of=/dev/null count=1 2023-06-03T19:25:07.933 INFO:teuthology.orchestra.run.smithi089.stderr:1+0 records in 2023-06-03T19:25:07.934 INFO:teuthology.orchestra.run.smithi089.stderr:1+0 records out 2023-06-03T19:25:07.934 INFO:teuthology.orchestra.run.smithi089.stderr:512 bytes copied, 0.000410913 s, 1.2 MB/s 2023-06-03T19:25:07.937 DEBUG:teuthology.orchestra.run.smithi089:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_2 2023-06-03T19:25:07.993 DEBUG:teuthology.orchestra.run.smithi089:> stat /dev/vg_nvme/lv_3 2023-06-03T19:25:08.029 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:25:07 smithi089 bash[26572]: audit 2023-06-03T19:25:06.616031+0000 mgr.y (mgr.14146) 51 : audit [DBG] from='client.24110 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mgr", "placement": "2;smithi089=y;smithi164=x", "target": ["mon-mgr", ""]}]: dispatch 2023-06-03T19:25:08.029 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:25:07 smithi089 bash[26572]: cephadm 2023-06-03T19:25:06.619238+0000 mgr.y (mgr.14146) 52 : cephadm [INF] Saving service mgr spec with placement smithi089=y;smithi164=x;count:2 2023-06-03T19:25:08.029 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:25:07 smithi089 bash[26572]: cephadm 2023-06-03T19:25:06.679034+0000 mgr.y (mgr.14146) 53 : cephadm [INF] Deploying daemon mgr.x on smithi164 2023-06-03T19:25:08.030 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:25:07 smithi089 bash[33173]: audit 2023-06-03T19:25:06.616031+0000 mgr.y (mgr.14146) 51 : audit [DBG] from='client.24110 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mgr", "placement": "2;smithi089=y;smithi164=x", "target": ["mon-mgr", ""]}]: dispatch 2023-06-03T19:25:08.030 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:25:07 smithi089 bash[33173]: cephadm 2023-06-03T19:25:06.619238+0000 mgr.y (mgr.14146) 52 : cephadm [INF] Saving service mgr spec with placement smithi089=y;smithi164=x;count:2 2023-06-03T19:25:08.030 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:25:07 smithi089 bash[33173]: cephadm 2023-06-03T19:25:06.679034+0000 mgr.y (mgr.14146) 53 : cephadm [INF] Deploying daemon mgr.x on smithi164 2023-06-03T19:25:08.037 INFO:teuthology.orchestra.run.smithi089.stdout: File: /dev/vg_nvme/lv_3 -> ../dm-2 2023-06-03T19:25:08.037 INFO:teuthology.orchestra.run.smithi089.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2023-06-03T19:25:08.037 INFO:teuthology.orchestra.run.smithi089.stdout:Device: 5h/5d Inode: 505 Links: 1 2023-06-03T19:25:08.037 INFO:teuthology.orchestra.run.smithi089.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2023-06-03T19:25:08.037 INFO:teuthology.orchestra.run.smithi089.stdout:Access: 2023-06-03 19:18:44.892681487 +0000 2023-06-03T19:25:08.038 INFO:teuthology.orchestra.run.smithi089.stdout:Modify: 2023-06-03 19:18:44.888681578 +0000 2023-06-03T19:25:08.038 INFO:teuthology.orchestra.run.smithi089.stdout:Change: 2023-06-03 19:18:44.888681578 +0000 2023-06-03T19:25:08.038 INFO:teuthology.orchestra.run.smithi089.stdout: Birth: - 2023-06-03T19:25:08.038 DEBUG:teuthology.orchestra.run.smithi089:> sudo dd if=/dev/vg_nvme/lv_3 of=/dev/null count=1 2023-06-03T19:25:08.084 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:25:07 smithi164 bash[26855]: audit 2023-06-03T19:25:06.616031+0000 mgr.y (mgr.14146) 51 : audit [DBG] from='client.24110 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mgr", "placement": "2;smithi089=y;smithi164=x", "target": ["mon-mgr", ""]}]: dispatch 2023-06-03T19:25:08.084 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:25:07 smithi164 bash[26855]: cephadm 2023-06-03T19:25:06.619238+0000 mgr.y (mgr.14146) 52 : cephadm [INF] Saving service mgr spec with placement smithi089=y;smithi164=x;count:2 2023-06-03T19:25:08.084 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:25:07 smithi164 bash[26855]: cephadm 2023-06-03T19:25:06.679034+0000 mgr.y (mgr.14146) 53 : cephadm [INF] Deploying daemon mgr.x on smithi164 2023-06-03T19:25:08.095 INFO:teuthology.orchestra.run.smithi089.stderr:1+0 records in 2023-06-03T19:25:08.095 INFO:teuthology.orchestra.run.smithi089.stderr:1+0 records out 2023-06-03T19:25:08.095 INFO:teuthology.orchestra.run.smithi089.stderr:512 bytes copied, 0.000413942 s, 1.2 MB/s 2023-06-03T19:25:08.096 DEBUG:teuthology.orchestra.run.smithi089:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_3 2023-06-03T19:25:08.150 DEBUG:teuthology.orchestra.run.smithi089:> stat /dev/vg_nvme/lv_4 2023-06-03T19:25:08.196 INFO:journalctl@ceph.mgr.x.smithi164.stdout:-- Logs begin at Sun 2022-08-14 12:20:24 UTC. -- 2023-06-03T19:25:08.199 INFO:teuthology.orchestra.run.smithi089.stdout: File: /dev/vg_nvme/lv_4 -> ../dm-3 2023-06-03T19:25:08.200 INFO:teuthology.orchestra.run.smithi089.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2023-06-03T19:25:08.200 INFO:teuthology.orchestra.run.smithi089.stdout:Device: 5h/5d Inode: 513 Links: 1 2023-06-03T19:25:08.200 INFO:teuthology.orchestra.run.smithi089.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2023-06-03T19:25:08.200 INFO:teuthology.orchestra.run.smithi089.stdout:Access: 2023-06-03 19:18:45.764661377 +0000 2023-06-03T19:25:08.201 INFO:teuthology.orchestra.run.smithi089.stdout:Modify: 2023-06-03 19:18:45.448668664 +0000 2023-06-03T19:25:08.201 INFO:teuthology.orchestra.run.smithi089.stdout:Change: 2023-06-03 19:18:45.448668664 +0000 2023-06-03T19:25:08.201 INFO:teuthology.orchestra.run.smithi089.stdout: Birth: - 2023-06-03T19:25:08.202 DEBUG:teuthology.orchestra.run.smithi089:> sudo dd if=/dev/vg_nvme/lv_4 of=/dev/null count=1 2023-06-03T19:25:08.256 INFO:teuthology.orchestra.run.smithi089.stderr:1+0 records in 2023-06-03T19:25:08.257 INFO:teuthology.orchestra.run.smithi089.stderr:1+0 records out 2023-06-03T19:25:08.257 INFO:teuthology.orchestra.run.smithi089.stderr:512 bytes copied, 0.000388999 s, 1.3 MB/s 2023-06-03T19:25:08.257 DEBUG:teuthology.orchestra.run.smithi089:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_4 2023-06-03T19:25:08.310 DEBUG:teuthology.orchestra.run.smithi164:> set -ex 2023-06-03T19:25:08.310 DEBUG:teuthology.orchestra.run.smithi164:> dd if=/scratch_devs of=/dev/stdout 2023-06-03T19:25:08.320 DEBUG:teuthology.misc:devs=['/dev/vg_nvme/lv_1', '/dev/vg_nvme/lv_2', '/dev/vg_nvme/lv_3', '/dev/vg_nvme/lv_4'] 2023-06-03T19:25:08.321 DEBUG:teuthology.orchestra.run.smithi164:> stat /dev/vg_nvme/lv_1 2023-06-03T19:25:08.367 INFO:teuthology.orchestra.run.smithi164.stdout: File: /dev/vg_nvme/lv_1 -> ../dm-0 2023-06-03T19:25:08.368 INFO:teuthology.orchestra.run.smithi164.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2023-06-03T19:25:08.368 INFO:teuthology.orchestra.run.smithi164.stdout:Device: 5h/5d Inode: 495 Links: 1 2023-06-03T19:25:08.368 INFO:teuthology.orchestra.run.smithi164.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2023-06-03T19:25:08.368 INFO:teuthology.orchestra.run.smithi164.stdout:Access: 2023-06-03 19:18:44.111282983 +0000 2023-06-03T19:25:08.368 INFO:teuthology.orchestra.run.smithi164.stdout:Modify: 2023-06-03 19:18:43.779290635 +0000 2023-06-03T19:25:08.368 INFO:teuthology.orchestra.run.smithi164.stdout:Change: 2023-06-03 19:18:43.779290635 +0000 2023-06-03T19:25:08.369 INFO:teuthology.orchestra.run.smithi164.stdout: Birth: - 2023-06-03T19:25:08.369 DEBUG:teuthology.orchestra.run.smithi164:> sudo dd if=/dev/vg_nvme/lv_1 of=/dev/null count=1 2023-06-03T19:25:08.424 INFO:teuthology.orchestra.run.smithi164.stderr:1+0 records in 2023-06-03T19:25:08.424 INFO:teuthology.orchestra.run.smithi164.stderr:1+0 records out 2023-06-03T19:25:08.424 INFO:teuthology.orchestra.run.smithi164.stderr:512 bytes copied, 0.000251412 s, 2.0 MB/s 2023-06-03T19:25:08.425 DEBUG:teuthology.orchestra.run.smithi164:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_1 2023-06-03T19:25:08.473 DEBUG:teuthology.orchestra.run.smithi164:> stat /dev/vg_nvme/lv_2 2023-06-03T19:25:08.520 INFO:teuthology.orchestra.run.smithi164.stdout: File: /dev/vg_nvme/lv_2 -> ../dm-1 2023-06-03T19:25:08.520 INFO:teuthology.orchestra.run.smithi164.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2023-06-03T19:25:08.520 INFO:teuthology.orchestra.run.smithi164.stdout:Device: 5h/5d Inode: 499 Links: 1 2023-06-03T19:25:08.520 INFO:teuthology.orchestra.run.smithi164.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2023-06-03T19:25:08.521 INFO:teuthology.orchestra.run.smithi164.stdout:Access: 2023-06-03 19:18:44.671270075 +0000 2023-06-03T19:25:08.521 INFO:teuthology.orchestra.run.smithi164.stdout:Modify: 2023-06-03 19:18:44.339277727 +0000 2023-06-03T19:25:08.521 INFO:teuthology.orchestra.run.smithi164.stdout:Change: 2023-06-03 19:18:44.339277727 +0000 2023-06-03T19:25:08.521 INFO:teuthology.orchestra.run.smithi164.stdout: Birth: - 2023-06-03T19:25:08.521 DEBUG:teuthology.orchestra.run.smithi164:> sudo dd if=/dev/vg_nvme/lv_2 of=/dev/null count=1 2023-06-03T19:25:08.572 INFO:teuthology.orchestra.run.smithi164.stderr:1+0 records in 2023-06-03T19:25:08.572 INFO:teuthology.orchestra.run.smithi164.stderr:1+0 records out 2023-06-03T19:25:08.572 INFO:teuthology.orchestra.run.smithi164.stderr:512 bytes copied, 0.000271699 s, 1.9 MB/s 2023-06-03T19:25:08.573 DEBUG:teuthology.orchestra.run.smithi164:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_2 2023-06-03T19:25:08.623 DEBUG:teuthology.orchestra.run.smithi164:> stat /dev/vg_nvme/lv_3 2023-06-03T19:25:08.670 INFO:teuthology.orchestra.run.smithi164.stdout: File: /dev/vg_nvme/lv_3 -> ../dm-2 2023-06-03T19:25:08.671 INFO:teuthology.orchestra.run.smithi164.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2023-06-03T19:25:08.671 INFO:teuthology.orchestra.run.smithi164.stdout:Device: 5h/5d Inode: 507 Links: 1 2023-06-03T19:25:08.671 INFO:teuthology.orchestra.run.smithi164.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2023-06-03T19:25:08.671 INFO:teuthology.orchestra.run.smithi164.stdout:Access: 2023-06-03 19:18:45.227257259 +0000 2023-06-03T19:25:08.671 INFO:teuthology.orchestra.run.smithi164.stdout:Modify: 2023-06-03 19:18:44.883265189 +0000 2023-06-03T19:25:08.672 INFO:teuthology.orchestra.run.smithi164.stdout:Change: 2023-06-03 19:18:44.883265189 +0000 2023-06-03T19:25:08.672 INFO:teuthology.orchestra.run.smithi164.stdout: Birth: - 2023-06-03T19:25:08.672 DEBUG:teuthology.orchestra.run.smithi164:> sudo dd if=/dev/vg_nvme/lv_3 of=/dev/null count=1 2023-06-03T19:25:08.719 INFO:teuthology.orchestra.run.smithi164.stderr:1+0 records in 2023-06-03T19:25:08.720 INFO:teuthology.orchestra.run.smithi164.stderr:1+0 records out 2023-06-03T19:25:08.720 INFO:teuthology.orchestra.run.smithi164.stderr:512 bytes copied, 0.000269064 s, 1.9 MB/s 2023-06-03T19:25:08.721 DEBUG:teuthology.orchestra.run.smithi164:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_3 2023-06-03T19:25:08.771 DEBUG:teuthology.orchestra.run.smithi164:> stat /dev/vg_nvme/lv_4 2023-06-03T19:25:08.819 INFO:teuthology.orchestra.run.smithi164.stdout: File: /dev/vg_nvme/lv_4 -> ../dm-3 2023-06-03T19:25:08.820 INFO:teuthology.orchestra.run.smithi164.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2023-06-03T19:25:08.820 INFO:teuthology.orchestra.run.smithi164.stdout:Device: 5h/5d Inode: 511 Links: 1 2023-06-03T19:25:08.820 INFO:teuthology.orchestra.run.smithi164.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2023-06-03T19:25:08.821 INFO:teuthology.orchestra.run.smithi164.stdout:Access: 2023-06-03 19:18:45.451252097 +0000 2023-06-03T19:25:08.821 INFO:teuthology.orchestra.run.smithi164.stdout:Modify: 2023-06-03 19:18:45.447252188 +0000 2023-06-03T19:25:08.821 INFO:teuthology.orchestra.run.smithi164.stdout:Change: 2023-06-03 19:18:45.447252188 +0000 2023-06-03T19:25:08.821 INFO:teuthology.orchestra.run.smithi164.stdout: Birth: - 2023-06-03T19:25:08.822 DEBUG:teuthology.orchestra.run.smithi164:> sudo dd if=/dev/vg_nvme/lv_4 of=/dev/null count=1 2023-06-03T19:25:08.874 INFO:teuthology.orchestra.run.smithi164.stderr:1+0 records in 2023-06-03T19:25:08.875 INFO:teuthology.orchestra.run.smithi164.stderr:1+0 records out 2023-06-03T19:25:08.875 INFO:teuthology.orchestra.run.smithi164.stderr:512 bytes copied, 0.000362888 s, 1.4 MB/s 2023-06-03T19:25:08.877 DEBUG:teuthology.orchestra.run.smithi164:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_4 2023-06-03T19:25:08.927 INFO:tasks.cephadm:Deploying osd.0 on smithi089 with /dev/vg_nvme/lv_4... 2023-06-03T19:25:08.928 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph-volume lvm zap /dev/vg_nvme/lv_4 2023-06-03T19:25:08.991 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:25:08 smithi164 bash[26855]: cluster 2023-06-03T19:25:07.700282+0000 mgr.y (mgr.14146) 54 : cluster [DBG] pgmap v30: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-06-03T19:25:09.029 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:25:08 smithi089 bash[26572]: cluster 2023-06-03T19:25:07.700282+0000 mgr.y (mgr.14146) 54 : cluster [DBG] pgmap v30: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-06-03T19:25:09.029 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:25:08 smithi089 bash[33173]: cluster 2023-06-03T19:25:07.700282+0000 mgr.y (mgr.14146) 54 : cluster [DBG] pgmap v30: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-06-03T19:25:10.752 INFO:journalctl@ceph.mgr.x.smithi164.stdout:Jun 03 19:25:10 smithi164 bash[27934]: debug 2023-06-03T19:25:10.570+0000 7f9a9b554700 1 -- 172.21.15.164:0/3803749374 <== mon.1 v2:172.21.15.164:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x55f0c19d44e0 con 0x55f0c1ac9000 2023-06-03T19:25:10.752 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:25:10 smithi164 bash[26855]: audit 2023-06-03T19:25:09.455880+0000 mon.a (mon.0) 200 : audit [INF] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' 2023-06-03T19:25:10.752 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:25:10 smithi164 bash[26855]: audit 2023-06-03T19:25:09.458831+0000 mon.a (mon.0) 201 : audit [DBG] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-03T19:25:10.752 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:25:10 smithi164 bash[26855]: cluster 2023-06-03T19:25:09.700678+0000 mgr.y (mgr.14146) 55 : cluster [DBG] pgmap v31: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-06-03T19:25:10.779 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:25:10 smithi089 bash[26572]: audit 2023-06-03T19:25:09.455880+0000 mon.a (mon.0) 200 : audit [INF] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' 2023-06-03T19:25:10.779 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:25:10 smithi089 bash[26572]: audit 2023-06-03T19:25:09.458831+0000 mon.a (mon.0) 201 : audit [DBG] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-03T19:25:10.779 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:25:10 smithi089 bash[26572]: cluster 2023-06-03T19:25:09.700678+0000 mgr.y (mgr.14146) 55 : cluster [DBG] pgmap v31: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-06-03T19:25:10.779 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:25:10 smithi089 bash[33173]: audit 2023-06-03T19:25:09.455880+0000 mon.a (mon.0) 200 : audit [INF] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' 2023-06-03T19:25:10.780 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:25:10 smithi089 bash[33173]: audit 2023-06-03T19:25:09.458831+0000 mon.a (mon.0) 201 : audit [DBG] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-03T19:25:10.780 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:25:10 smithi089 bash[33173]: cluster 2023-06-03T19:25:09.700678+0000 mgr.y (mgr.14146) 55 : cluster [DBG] pgmap v31: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-06-03T19:25:11.084 INFO:journalctl@ceph.mgr.x.smithi164.stdout:Jun 03 19:25:10 smithi164 bash[27934]: debug 2023-06-03T19:25:10.738+0000 7f9aabb37680 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2023-06-03T19:25:11.084 INFO:journalctl@ceph.mgr.x.smithi164.stdout:Jun 03 19:25:10 smithi164 bash[27934]: debug 2023-06-03T19:25:10.842+0000 7f9aabb37680 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2023-06-03T19:25:11.584 INFO:journalctl@ceph.mgr.x.smithi164.stdout:Jun 03 19:25:11 smithi164 bash[27934]: debug 2023-06-03T19:25:11.302+0000 7f9aabb37680 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2023-06-03T19:25:12.235 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:25:11 smithi164 bash[26855]: audit 2023-06-03T19:25:10.875818+0000 mon.a (mon.0) 202 : audit [INF] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' 2023-06-03T19:25:12.279 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:25:11 smithi089 bash[26572]: audit 2023-06-03T19:25:10.875818+0000 mon.a (mon.0) 202 : audit [INF] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' 2023-06-03T19:25:12.280 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:25:11 smithi089 bash[33173]: audit 2023-06-03T19:25:10.875818+0000 mon.a (mon.0) 202 : audit [INF] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' 2023-06-03T19:25:12.579 INFO:journalctl@ceph.mgr.x.smithi164.stdout:Jun 03 19:25:12 smithi164 bash[27934]: debug 2023-06-03T19:25:12.222+0000 7f9aabb37680 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2023-06-03T19:25:12.579 INFO:journalctl@ceph.mgr.x.smithi164.stdout:Jun 03 19:25:12 smithi164 bash[27934]: debug 2023-06-03T19:25:12.318+0000 7f9aabb37680 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2023-06-03T19:25:12.651 INFO:teuthology.orchestra.run.smithi089.stderr:--> Zapping: /dev/vg_nvme/lv_4 2023-06-03T19:25:12.652 INFO:teuthology.orchestra.run.smithi089.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_4 bs=1M count=10 conv=fsync 2023-06-03T19:25:12.652 INFO:teuthology.orchestra.run.smithi089.stderr: stderr: 10+0 records in 2023-06-03T19:25:12.652 INFO:teuthology.orchestra.run.smithi089.stderr:10+0 records out 2023-06-03T19:25:12.652 INFO:teuthology.orchestra.run.smithi089.stderr: stderr: 10485760 bytes (10 MB, 10 MiB) copied, 0.0145742 s, 719 MB/s 2023-06-03T19:25:12.652 INFO:teuthology.orchestra.run.smithi089.stderr:--> Zapping successful for: 2023-06-03T19:25:13.028 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:25:12 smithi089 bash[33173]: cluster 2023-06-03T19:25:11.701062+0000 mgr.y (mgr.14146) 56 : cluster [DBG] pgmap v32: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-06-03T19:25:13.029 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:25:12 smithi089 bash[26572]: cluster 2023-06-03T19:25:11.701062+0000 mgr.y (mgr.14146) 56 : cluster [DBG] pgmap v32: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-06-03T19:25:13.334 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:25:12 smithi164 bash[26855]: cluster 2023-06-03T19:25:11.701062+0000 mgr.y (mgr.14146) 56 : cluster [DBG] pgmap v32: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-06-03T19:25:13.334 INFO:journalctl@ceph.mgr.x.smithi164.stdout:Jun 03 19:25:12 smithi164 bash[27934]: debug 2023-06-03T19:25:12.566+0000 7f9aabb37680 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2023-06-03T19:25:13.446 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph orch daemon add osd smithi089:vg_nvme/lv_4 2023-06-03T19:25:13.985 INFO:journalctl@ceph.mgr.x.smithi164.stdout:Jun 03 19:25:13 smithi164 bash[27934]: debug 2023-06-03T19:25:13.714+0000 7f9aabb37680 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2023-06-03T19:25:13.986 INFO:journalctl@ceph.mgr.x.smithi164.stdout:Jun 03 19:25:13 smithi164 bash[27934]: debug 2023-06-03T19:25:13.874+0000 7f9aabb37680 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2023-06-03T19:25:14.332 INFO:journalctl@ceph.mgr.x.smithi164.stdout:Jun 03 19:25:13 smithi164 bash[27934]: debug 2023-06-03T19:25:13.974+0000 7f9aabb37680 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2023-06-03T19:25:14.332 INFO:journalctl@ceph.mgr.x.smithi164.stdout:Jun 03 19:25:14 smithi164 bash[27934]: debug 2023-06-03T19:25:14.086+0000 7f9aabb37680 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2023-06-03T19:25:14.332 INFO:journalctl@ceph.mgr.x.smithi164.stdout:Jun 03 19:25:14 smithi164 bash[27934]: debug 2023-06-03T19:25:14.318+0000 7f9aabb37680 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2023-06-03T19:25:15.029 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:25:14 smithi089 bash[26572]: cluster 2023-06-03T19:25:13.701521+0000 mgr.y (mgr.14146) 57 : cluster [DBG] pgmap v33: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-06-03T19:25:15.029 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:25:14 smithi089 bash[26572]: audit 2023-06-03T19:25:14.524638+0000 mon.a (mon.0) 203 : audit [INF] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' 2023-06-03T19:25:15.029 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:25:14 smithi089 bash[26572]: audit 2023-06-03T19:25:14.525808+0000 mon.a (mon.0) 204 : audit [DBG] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-03T19:25:15.029 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:25:14 smithi089 bash[26572]: audit 2023-06-03T19:25:14.527024+0000 mon.a (mon.0) 205 : audit [INF] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-03T19:25:15.029 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:25:14 smithi089 bash[26572]: audit 2023-06-03T19:25:14.533187+0000 mon.a (mon.0) 206 : audit [INF] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' 2023-06-03T19:25:15.030 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:25:14 smithi089 bash[26572]: audit 2023-06-03T19:25:14.536273+0000 mon.a (mon.0) 207 : audit [INF] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-06-03T19:25:15.030 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:25:14 smithi089 bash[26572]: audit 2023-06-03T19:25:14.537378+0000 mon.a (mon.0) 208 : audit [DBG] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-06-03T19:25:15.030 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:25:14 smithi089 bash[26572]: audit 2023-06-03T19:25:14.538324+0000 mon.a (mon.0) 209 : audit [DBG] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-03T19:25:15.030 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:25:14 smithi089 bash[33173]: cluster 2023-06-03T19:25:13.701521+0000 mgr.y (mgr.14146) 57 : cluster [DBG] pgmap v33: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-06-03T19:25:15.030 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:25:14 smithi089 bash[33173]: audit 2023-06-03T19:25:14.524638+0000 mon.a (mon.0) 203 : audit [INF] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' 2023-06-03T19:25:15.030 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:25:14 smithi089 bash[33173]: audit 2023-06-03T19:25:14.525808+0000 mon.a (mon.0) 204 : audit [DBG] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-03T19:25:15.031 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:25:14 smithi089 bash[33173]: audit 2023-06-03T19:25:14.527024+0000 mon.a (mon.0) 205 : audit [INF] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-03T19:25:15.031 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:25:14 smithi089 bash[33173]: audit 2023-06-03T19:25:14.533187+0000 mon.a (mon.0) 206 : audit [INF] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' 2023-06-03T19:25:15.031 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:25:14 smithi089 bash[33173]: audit 2023-06-03T19:25:14.536273+0000 mon.a (mon.0) 207 : audit [INF] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-06-03T19:25:15.031 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:25:14 smithi089 bash[33173]: audit 2023-06-03T19:25:14.537378+0000 mon.a (mon.0) 208 : audit [DBG] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-06-03T19:25:15.031 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:25:14 smithi089 bash[33173]: audit 2023-06-03T19:25:14.538324+0000 mon.a (mon.0) 209 : audit [DBG] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-03T19:25:15.084 INFO:journalctl@ceph.mgr.x.smithi164.stdout:Jun 03 19:25:14 smithi164 bash[27934]: debug 2023-06-03T19:25:14.706+0000 7f9aabb37680 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2023-06-03T19:25:15.085 INFO:journalctl@ceph.mgr.x.smithi164.stdout:Jun 03 19:25:14 smithi164 bash[27934]: debug 2023-06-03T19:25:14.826+0000 7f9aabb37680 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2023-06-03T19:25:15.085 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:25:14 smithi164 bash[26855]: cluster 2023-06-03T19:25:13.701521+0000 mgr.y (mgr.14146) 57 : cluster [DBG] pgmap v33: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-06-03T19:25:15.085 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:25:14 smithi164 bash[26855]: audit 2023-06-03T19:25:14.524638+0000 mon.a (mon.0) 203 : audit [INF] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' 2023-06-03T19:25:15.086 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:25:14 smithi164 bash[26855]: audit 2023-06-03T19:25:14.525808+0000 mon.a (mon.0) 204 : audit [DBG] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-03T19:25:15.086 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:25:14 smithi164 bash[26855]: audit 2023-06-03T19:25:14.527024+0000 mon.a (mon.0) 205 : audit [INF] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-03T19:25:15.086 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:25:14 smithi164 bash[26855]: audit 2023-06-03T19:25:14.533187+0000 mon.a (mon.0) 206 : audit [INF] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' 2023-06-03T19:25:15.086 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:25:14 smithi164 bash[26855]: audit 2023-06-03T19:25:14.536273+0000 mon.a (mon.0) 207 : audit [INF] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-06-03T19:25:15.086 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:25:14 smithi164 bash[26855]: audit 2023-06-03T19:25:14.537378+0000 mon.a (mon.0) 208 : audit [DBG] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-06-03T19:25:15.087 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:25:14 smithi164 bash[26855]: audit 2023-06-03T19:25:14.538324+0000 mon.a (mon.0) 209 : audit [DBG] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-03T19:25:16.084 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:25:15 smithi164 bash[26855]: cephadm 2023-06-03T19:25:14.535902+0000 mgr.y (mgr.14146) 58 : cephadm [INF] Reconfiguring mgr.y (unknown last config time)... 2023-06-03T19:25:16.084 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:25:15 smithi164 bash[26855]: cephadm 2023-06-03T19:25:14.539271+0000 mgr.y (mgr.14146) 59 : cephadm [INF] Reconfiguring daemon mgr.y on smithi089 2023-06-03T19:25:16.085 INFO:journalctl@ceph.mgr.x.smithi164.stdout:Jun 03 19:25:15 smithi164 bash[27934]: debug 2023-06-03T19:25:15.698+0000 7f9aabb37680 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2023-06-03T19:25:16.085 INFO:journalctl@ceph.mgr.x.smithi164.stdout:Jun 03 19:25:15 smithi164 bash[27934]: debug 2023-06-03T19:25:15.798+0000 7f9aabb37680 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2023-06-03T19:25:16.085 INFO:journalctl@ceph.mgr.x.smithi164.stdout:Jun 03 19:25:15 smithi164 bash[27934]: debug 2023-06-03T19:25:15.914+0000 7f9aabb37680 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2023-06-03T19:25:16.284 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:25:15 smithi089 bash[33173]: cephadm 2023-06-03T19:25:14.535902+0000 mgr.y (mgr.14146) 58 : cephadm [INF] Reconfiguring mgr.y (unknown last config time)... 2023-06-03T19:25:16.284 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:25:15 smithi089 bash[33173]: cephadm 2023-06-03T19:25:14.539271+0000 mgr.y (mgr.14146) 59 : cephadm [INF] Reconfiguring daemon mgr.y on smithi089 2023-06-03T19:25:16.285 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:25:15 smithi089 bash[26572]: cephadm 2023-06-03T19:25:14.535902+0000 mgr.y (mgr.14146) 58 : cephadm [INF] Reconfiguring mgr.y (unknown last config time)... 2023-06-03T19:25:16.285 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:25:15 smithi089 bash[26572]: cephadm 2023-06-03T19:25:14.539271+0000 mgr.y (mgr.14146) 59 : cephadm [INF] Reconfiguring daemon mgr.y on smithi089 2023-06-03T19:25:16.446 INFO:journalctl@ceph.mgr.x.smithi164.stdout:Jun 03 19:25:16 smithi164 bash[27934]: debug 2023-06-03T19:25:16.134+0000 7f9aabb37680 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2023-06-03T19:25:16.785 INFO:journalctl@ceph.mgr.x.smithi164.stdout:Jun 03 19:25:16 smithi164 bash[27934]: debug 2023-06-03T19:25:16.434+0000 7f9aabb37680 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2023-06-03T19:25:16.785 INFO:journalctl@ceph.mgr.x.smithi164.stdout:Jun 03 19:25:16 smithi164 bash[27934]: debug 2023-06-03T19:25:16.598+0000 7f9aabb37680 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2023-06-03T19:25:17.084 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:25:16 smithi164 bash[26855]: cluster 2023-06-03T19:25:15.701908+0000 mgr.y (mgr.14146) 60 : cluster [DBG] pgmap v34: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-06-03T19:25:17.084 INFO:journalctl@ceph.mgr.x.smithi164.stdout:Jun 03 19:25:16 smithi164 bash[27934]: debug 2023-06-03T19:25:16.774+0000 7f9aabb37680 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2023-06-03T19:25:17.085 INFO:journalctl@ceph.mgr.x.smithi164.stdout:Jun 03 19:25:16 smithi164 bash[27934]: debug 2023-06-03T19:25:16.990+0000 7f9aabb37680 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2023-06-03T19:25:17.189 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:25:16 smithi089 bash[33173]: cluster 2023-06-03T19:25:15.701908+0000 mgr.y (mgr.14146) 60 : cluster [DBG] pgmap v34: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-06-03T19:25:17.190 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:25:16 smithi089 bash[26572]: cluster 2023-06-03T19:25:15.701908+0000 mgr.y (mgr.14146) 60 : cluster [DBG] pgmap v34: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-06-03T19:25:17.584 INFO:journalctl@ceph.mgr.x.smithi164.stdout:Jun 03 19:25:17 smithi164 bash[27934]: debug 2023-06-03T19:25:17.094+0000 7f9aabb37680 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2023-06-03T19:25:18.279 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:25:17 smithi089 bash[26572]: cluster 2023-06-03T19:25:17.123135+0000 mon.a (mon.0) 210 : cluster [DBG] Standby manager daemon x started 2023-06-03T19:25:18.279 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:25:17 smithi089 bash[26572]: audit 2023-06-03T19:25:17.127134+0000 mon.a (mon.0) 211 : audit [DBG] from='mgr.? 172.21.15.164:0/527386268' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-06-03T19:25:18.279 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:25:17 smithi089 bash[26572]: audit 2023-06-03T19:25:17.128082+0000 mon.a (mon.0) 212 : audit [DBG] from='mgr.? 172.21.15.164:0/527386268' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-06-03T19:25:18.279 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:25:17 smithi089 bash[26572]: audit 2023-06-03T19:25:17.129955+0000 mon.a (mon.0) 213 : audit [DBG] from='mgr.? 172.21.15.164:0/527386268' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-06-03T19:25:18.279 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:25:17 smithi089 bash[26572]: audit 2023-06-03T19:25:17.130955+0000 mon.a (mon.0) 214 : audit [DBG] from='mgr.? 172.21.15.164:0/527386268' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-06-03T19:25:18.280 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:25:17 smithi089 bash[33173]: cluster 2023-06-03T19:25:17.123135+0000 mon.a (mon.0) 210 : cluster [DBG] Standby manager daemon x started 2023-06-03T19:25:18.280 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:25:17 smithi089 bash[33173]: audit 2023-06-03T19:25:17.127134+0000 mon.a (mon.0) 211 : audit [DBG] from='mgr.? 172.21.15.164:0/527386268' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-06-03T19:25:18.280 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:25:17 smithi089 bash[33173]: audit 2023-06-03T19:25:17.128082+0000 mon.a (mon.0) 212 : audit [DBG] from='mgr.? 172.21.15.164:0/527386268' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-06-03T19:25:18.280 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:25:17 smithi089 bash[33173]: audit 2023-06-03T19:25:17.129955+0000 mon.a (mon.0) 213 : audit [DBG] from='mgr.? 172.21.15.164:0/527386268' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-06-03T19:25:18.280 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:25:17 smithi089 bash[33173]: audit 2023-06-03T19:25:17.130955+0000 mon.a (mon.0) 214 : audit [DBG] from='mgr.? 172.21.15.164:0/527386268' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-06-03T19:25:18.334 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:25:17 smithi164 bash[26855]: cluster 2023-06-03T19:25:17.123135+0000 mon.a (mon.0) 210 : cluster [DBG] Standby manager daemon x started 2023-06-03T19:25:18.334 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:25:17 smithi164 bash[26855]: audit 2023-06-03T19:25:17.127134+0000 mon.a (mon.0) 211 : audit [DBG] from='mgr.? 172.21.15.164:0/527386268' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-06-03T19:25:18.335 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:25:17 smithi164 bash[26855]: audit 2023-06-03T19:25:17.128082+0000 mon.a (mon.0) 212 : audit [DBG] from='mgr.? 172.21.15.164:0/527386268' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-06-03T19:25:18.335 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:25:17 smithi164 bash[26855]: audit 2023-06-03T19:25:17.129955+0000 mon.a (mon.0) 213 : audit [DBG] from='mgr.? 172.21.15.164:0/527386268' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-06-03T19:25:18.335 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:25:17 smithi164 bash[26855]: audit 2023-06-03T19:25:17.130955+0000 mon.a (mon.0) 214 : audit [DBG] from='mgr.? 172.21.15.164:0/527386268' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-06-03T19:25:19.279 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:25:18 smithi089 bash[26572]: cluster 2023-06-03T19:25:17.702350+0000 mgr.y (mgr.14146) 61 : cluster [DBG] pgmap v35: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-06-03T19:25:19.279 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:25:18 smithi089 bash[26572]: cluster 2023-06-03T19:25:17.905105+0000 mon.a (mon.0) 215 : cluster [DBG] mgrmap e14: y(active, since 82s), standbys: x 2023-06-03T19:25:19.280 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:25:18 smithi089 bash[26572]: audit 2023-06-03T19:25:17.905331+0000 mon.a (mon.0) 216 : audit [DBG] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2023-06-03T19:25:19.280 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:25:18 smithi089 bash[26572]: audit 2023-06-03T19:25:17.926655+0000 mon.a (mon.0) 217 : audit [INF] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' 2023-06-03T19:25:19.280 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:25:18 smithi089 bash[26572]: audit 2023-06-03T19:25:17.928792+0000 mon.a (mon.0) 218 : audit [DBG] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-03T19:25:19.280 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:25:18 smithi089 bash[26572]: audit 2023-06-03T19:25:18.205621+0000 mon.a (mon.0) 219 : audit [DBG] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-03T19:25:19.280 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:25:18 smithi089 bash[26572]: audit 2023-06-03T19:25:18.206942+0000 mon.a (mon.0) 220 : audit [INF] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-03T19:25:19.280 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:25:18 smithi089 bash[26572]: audit 2023-06-03T19:25:18.214752+0000 mon.a (mon.0) 221 : audit [INF] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' 2023-06-03T19:25:19.281 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:25:18 smithi089 bash[26572]: audit 2023-06-03T19:25:18.376348+0000 mon.a (mon.0) 222 : audit [DBG] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-06-03T19:25:19.281 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:25:18 smithi089 bash[26572]: audit 2023-06-03T19:25:18.379529+0000 mon.a (mon.0) 223 : audit [INF] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-06-03T19:25:19.281 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:25:18 smithi089 bash[26572]: audit 2023-06-03T19:25:18.380750+0000 mon.a (mon.0) 224 : audit [DBG] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-03T19:25:19.281 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:25:18 smithi089 bash[33173]: cluster 2023-06-03T19:25:17.702350+0000 mgr.y (mgr.14146) 61 : cluster [DBG] pgmap v35: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-06-03T19:25:19.282 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:25:18 smithi089 bash[33173]: cluster 2023-06-03T19:25:17.905105+0000 mon.a (mon.0) 215 : cluster [DBG] mgrmap e14: y(active, since 82s), standbys: x 2023-06-03T19:25:19.282 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:25:18 smithi089 bash[33173]: audit 2023-06-03T19:25:17.905331+0000 mon.a (mon.0) 216 : audit [DBG] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2023-06-03T19:25:19.282 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:25:18 smithi089 bash[33173]: audit 2023-06-03T19:25:17.926655+0000 mon.a (mon.0) 217 : audit [INF] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' 2023-06-03T19:25:19.282 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:25:18 smithi089 bash[33173]: audit 2023-06-03T19:25:17.928792+0000 mon.a (mon.0) 218 : audit [DBG] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-03T19:25:19.282 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:25:18 smithi089 bash[33173]: audit 2023-06-03T19:25:18.205621+0000 mon.a (mon.0) 219 : audit [DBG] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-03T19:25:19.282 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:25:18 smithi089 bash[33173]: audit 2023-06-03T19:25:18.206942+0000 mon.a (mon.0) 220 : audit [INF] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-03T19:25:19.282 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:25:18 smithi089 bash[33173]: audit 2023-06-03T19:25:18.214752+0000 mon.a (mon.0) 221 : audit [INF] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' 2023-06-03T19:25:19.283 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:25:18 smithi089 bash[33173]: audit 2023-06-03T19:25:18.376348+0000 mon.a (mon.0) 222 : audit [DBG] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-06-03T19:25:19.283 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:25:18 smithi089 bash[33173]: audit 2023-06-03T19:25:18.379529+0000 mon.a (mon.0) 223 : audit [INF] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-06-03T19:25:19.283 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:25:18 smithi089 bash[33173]: audit 2023-06-03T19:25:18.380750+0000 mon.a (mon.0) 224 : audit [DBG] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-03T19:25:19.335 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:25:18 smithi164 bash[26855]: cluster 2023-06-03T19:25:17.702350+0000 mgr.y (mgr.14146) 61 : cluster [DBG] pgmap v35: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-06-03T19:25:19.335 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:25:18 smithi164 bash[26855]: cluster 2023-06-03T19:25:17.905105+0000 mon.a (mon.0) 215 : cluster [DBG] mgrmap e14: y(active, since 82s), standbys: x 2023-06-03T19:25:19.336 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:25:18 smithi164 bash[26855]: audit 2023-06-03T19:25:17.905331+0000 mon.a (mon.0) 216 : audit [DBG] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2023-06-03T19:25:19.336 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:25:18 smithi164 bash[26855]: audit 2023-06-03T19:25:17.926655+0000 mon.a (mon.0) 217 : audit [INF] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' 2023-06-03T19:25:19.336 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:25:18 smithi164 bash[26855]: audit 2023-06-03T19:25:17.928792+0000 mon.a (mon.0) 218 : audit [DBG] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-03T19:25:19.336 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:25:18 smithi164 bash[26855]: audit 2023-06-03T19:25:18.205621+0000 mon.a (mon.0) 219 : audit [DBG] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-03T19:25:19.336 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:25:18 smithi164 bash[26855]: audit 2023-06-03T19:25:18.206942+0000 mon.a (mon.0) 220 : audit [INF] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-03T19:25:19.337 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:25:18 smithi164 bash[26855]: audit 2023-06-03T19:25:18.214752+0000 mon.a (mon.0) 221 : audit [INF] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' 2023-06-03T19:25:19.337 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:25:18 smithi164 bash[26855]: audit 2023-06-03T19:25:18.376348+0000 mon.a (mon.0) 222 : audit [DBG] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-06-03T19:25:19.337 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:25:18 smithi164 bash[26855]: audit 2023-06-03T19:25:18.379529+0000 mon.a (mon.0) 223 : audit [INF] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-06-03T19:25:19.337 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:25:18 smithi164 bash[26855]: audit 2023-06-03T19:25:18.380750+0000 mon.a (mon.0) 224 : audit [DBG] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-03T19:25:20.279 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:25:19 smithi089 bash[26572]: audit 2023-06-03T19:25:18.369296+0000 mgr.y (mgr.14146) 62 : audit [DBG] from='client.14208 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi089:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2023-06-03T19:25:20.279 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:25:19 smithi089 bash[33173]: audit 2023-06-03T19:25:18.369296+0000 mgr.y (mgr.14146) 62 : audit [DBG] from='client.14208 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi089:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2023-06-03T19:25:20.334 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:25:19 smithi164 bash[26855]: audit 2023-06-03T19:25:18.369296+0000 mgr.y (mgr.14146) 62 : audit [DBG] from='client.14208 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi089:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2023-06-03T19:25:21.278 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:25:20 smithi089 bash[26572]: cluster 2023-06-03T19:25:19.702795+0000 mgr.y (mgr.14146) 63 : cluster [DBG] pgmap v36: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-06-03T19:25:21.279 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:25:20 smithi089 bash[33173]: cluster 2023-06-03T19:25:19.702795+0000 mgr.y (mgr.14146) 63 : cluster [DBG] pgmap v36: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-06-03T19:25:21.334 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:25:20 smithi164 bash[26855]: cluster 2023-06-03T19:25:19.702795+0000 mgr.y (mgr.14146) 63 : cluster [DBG] pgmap v36: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-06-03T19:25:22.778 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:25:22 smithi089 bash[26572]: cluster 2023-06-03T19:25:21.703215+0000 mgr.y (mgr.14146) 64 : cluster [DBG] pgmap v37: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-06-03T19:25:22.779 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:25:22 smithi089 bash[33173]: cluster 2023-06-03T19:25:21.703215+0000 mgr.y (mgr.14146) 64 : cluster [DBG] pgmap v37: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-06-03T19:25:22.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:25:22 smithi164 bash[26855]: cluster 2023-06-03T19:25:21.703215+0000 mgr.y (mgr.14146) 64 : cluster [DBG] pgmap v37: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-06-03T19:25:25.029 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:25:24 smithi089 bash[26572]: cluster 2023-06-03T19:25:23.703729+0000 mgr.y (mgr.14146) 65 : cluster [DBG] pgmap v38: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-06-03T19:25:25.030 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:25:24 smithi089 bash[33173]: cluster 2023-06-03T19:25:23.703729+0000 mgr.y (mgr.14146) 65 : cluster [DBG] pgmap v38: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-06-03T19:25:25.084 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:25:24 smithi164 bash[26855]: cluster 2023-06-03T19:25:23.703729+0000 mgr.y (mgr.14146) 65 : cluster [DBG] pgmap v38: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-06-03T19:25:27.029 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:25:26 smithi089 bash[26572]: cluster 2023-06-03T19:25:25.704190+0000 mgr.y (mgr.14146) 66 : cluster [DBG] pgmap v39: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-06-03T19:25:27.030 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:25:26 smithi089 bash[33173]: cluster 2023-06-03T19:25:25.704190+0000 mgr.y (mgr.14146) 66 : cluster [DBG] pgmap v39: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-06-03T19:25:27.084 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:25:26 smithi164 bash[26855]: cluster 2023-06-03T19:25:25.704190+0000 mgr.y (mgr.14146) 66 : cluster [DBG] pgmap v39: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-06-03T19:25:28.029 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:25:27 smithi089 bash[26572]: audit 2023-06-03T19:25:26.841739+0000 mon.a (mon.0) 225 : audit [INF] from='client.? 172.21.15.89:0/2044031849' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "86063bfe-126e-46e7-93ed-c73813af165b"}]: dispatch 2023-06-03T19:25:28.029 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:25:27 smithi089 bash[26572]: audit 2023-06-03T19:25:26.846266+0000 mon.a (mon.0) 226 : audit [INF] from='client.? 172.21.15.89:0/2044031849' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "86063bfe-126e-46e7-93ed-c73813af165b"}]': finished 2023-06-03T19:25:28.030 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:25:27 smithi089 bash[26572]: cluster 2023-06-03T19:25:26.846313+0000 mon.a (mon.0) 227 : cluster [DBG] osdmap e5: 1 total, 0 up, 1 in 2023-06-03T19:25:28.030 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:25:27 smithi089 bash[26572]: audit 2023-06-03T19:25:26.846402+0000 mon.a (mon.0) 228 : audit [DBG] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-06-03T19:25:28.030 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:25:27 smithi089 bash[26572]: audit 2023-06-03T19:25:27.551643+0000 mon.c (mon.2) 2 : audit [DBG] from='client.? 172.21.15.89:0/2087884454' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-06-03T19:25:28.031 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:25:27 smithi089 bash[33173]: audit 2023-06-03T19:25:26.841739+0000 mon.a (mon.0) 225 : audit [INF] from='client.? 172.21.15.89:0/2044031849' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "86063bfe-126e-46e7-93ed-c73813af165b"}]: dispatch 2023-06-03T19:25:28.031 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:25:27 smithi089 bash[33173]: audit 2023-06-03T19:25:26.846266+0000 mon.a (mon.0) 226 : audit [INF] from='client.? 172.21.15.89:0/2044031849' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "86063bfe-126e-46e7-93ed-c73813af165b"}]': finished 2023-06-03T19:25:28.031 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:25:27 smithi089 bash[33173]: cluster 2023-06-03T19:25:26.846313+0000 mon.a (mon.0) 227 : cluster [DBG] osdmap e5: 1 total, 0 up, 1 in 2023-06-03T19:25:28.031 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:25:27 smithi089 bash[33173]: audit 2023-06-03T19:25:26.846402+0000 mon.a (mon.0) 228 : audit [DBG] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-06-03T19:25:28.031 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:25:27 smithi089 bash[33173]: audit 2023-06-03T19:25:27.551643+0000 mon.c (mon.2) 2 : audit [DBG] from='client.? 172.21.15.89:0/2087884454' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-06-03T19:25:28.084 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:25:27 smithi164 bash[26855]: audit 2023-06-03T19:25:26.841739+0000 mon.a (mon.0) 225 : audit [INF] from='client.? 172.21.15.89:0/2044031849' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "86063bfe-126e-46e7-93ed-c73813af165b"}]: dispatch 2023-06-03T19:25:28.085 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:25:27 smithi164 bash[26855]: audit 2023-06-03T19:25:26.846266+0000 mon.a (mon.0) 226 : audit [INF] from='client.? 172.21.15.89:0/2044031849' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "86063bfe-126e-46e7-93ed-c73813af165b"}]': finished 2023-06-03T19:25:28.085 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:25:27 smithi164 bash[26855]: cluster 2023-06-03T19:25:26.846313+0000 mon.a (mon.0) 227 : cluster [DBG] osdmap e5: 1 total, 0 up, 1 in 2023-06-03T19:25:28.085 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:25:27 smithi164 bash[26855]: audit 2023-06-03T19:25:26.846402+0000 mon.a (mon.0) 228 : audit [DBG] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-06-03T19:25:28.085 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:25:27 smithi164 bash[26855]: audit 2023-06-03T19:25:27.551643+0000 mon.c (mon.2) 2 : audit [DBG] from='client.? 172.21.15.89:0/2087884454' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-06-03T19:25:29.278 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:25:28 smithi089 bash[33173]: cluster 2023-06-03T19:25:27.704652+0000 mgr.y (mgr.14146) 67 : cluster [DBG] pgmap v41: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-06-03T19:25:29.279 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:25:28 smithi089 bash[26572]: cluster 2023-06-03T19:25:27.704652+0000 mgr.y (mgr.14146) 67 : cluster [DBG] pgmap v41: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-06-03T19:25:29.335 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:25:28 smithi164 bash[26855]: cluster 2023-06-03T19:25:27.704652+0000 mgr.y (mgr.14146) 67 : cluster [DBG] pgmap v41: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-06-03T19:25:31.280 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:25:30 smithi089 bash[26572]: cluster 2023-06-03T19:25:29.705096+0000 mgr.y (mgr.14146) 68 : cluster [DBG] pgmap v42: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-06-03T19:25:31.281 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:25:30 smithi089 bash[33173]: cluster 2023-06-03T19:25:29.705096+0000 mgr.y (mgr.14146) 68 : cluster [DBG] pgmap v42: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-06-03T19:25:31.335 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:25:30 smithi164 bash[26855]: cluster 2023-06-03T19:25:29.705096+0000 mgr.y (mgr.14146) 68 : cluster [DBG] pgmap v42: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-06-03T19:25:32.779 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:25:32 smithi089 bash[26572]: cluster 2023-06-03T19:25:31.705521+0000 mgr.y (mgr.14146) 69 : cluster [DBG] pgmap v43: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-06-03T19:25:32.779 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:25:32 smithi089 bash[33173]: cluster 2023-06-03T19:25:31.705521+0000 mgr.y (mgr.14146) 69 : cluster [DBG] pgmap v43: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-06-03T19:25:32.834 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:25:32 smithi164 bash[26855]: cluster 2023-06-03T19:25:31.705521+0000 mgr.y (mgr.14146) 69 : cluster [DBG] pgmap v43: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-06-03T19:25:35.035 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:25:34 smithi089 bash[26572]: cluster 2023-06-03T19:25:33.705948+0000 mgr.y (mgr.14146) 70 : cluster [DBG] pgmap v44: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-06-03T19:25:35.035 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:25:34 smithi089 bash[33173]: cluster 2023-06-03T19:25:33.705948+0000 mgr.y (mgr.14146) 70 : cluster [DBG] pgmap v44: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-06-03T19:25:35.084 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:25:34 smithi164 bash[26855]: cluster 2023-06-03T19:25:33.705948+0000 mgr.y (mgr.14146) 70 : cluster [DBG] pgmap v44: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-06-03T19:25:36.777 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:25:36 smithi089 bash[26572]: cluster 2023-06-03T19:25:35.706404+0000 mgr.y (mgr.14146) 71 : cluster [DBG] pgmap v45: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-06-03T19:25:37.030 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:25:36 smithi089 bash[33173]: cluster 2023-06-03T19:25:35.706404+0000 mgr.y (mgr.14146) 71 : cluster [DBG] pgmap v45: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-06-03T19:25:37.084 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:25:36 smithi164 bash[26855]: cluster 2023-06-03T19:25:35.706404+0000 mgr.y (mgr.14146) 71 : cluster [DBG] pgmap v45: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-06-03T19:25:39.029 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:25:38 smithi089 bash[33173]: cluster 2023-06-03T19:25:37.706759+0000 mgr.y (mgr.14146) 72 : cluster [DBG] pgmap v46: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-06-03T19:25:39.030 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:25:38 smithi089 bash[26572]: cluster 2023-06-03T19:25:37.706759+0000 mgr.y (mgr.14146) 72 : cluster [DBG] pgmap v46: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-06-03T19:25:39.084 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:25:38 smithi164 bash[26855]: cluster 2023-06-03T19:25:37.706759+0000 mgr.y (mgr.14146) 72 : cluster [DBG] pgmap v46: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-06-03T19:25:40.084 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:25:39 smithi164 bash[26855]: audit 2023-06-03T19:25:39.560089+0000 mon.a (mon.0) 229 : audit [INF] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2023-06-03T19:25:40.085 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:25:39 smithi164 bash[26855]: audit 2023-06-03T19:25:39.561813+0000 mon.a (mon.0) 230 : audit [DBG] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-03T19:25:40.278 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:25:39 smithi089 bash[33173]: audit 2023-06-03T19:25:39.560089+0000 mon.a (mon.0) 229 : audit [INF] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2023-06-03T19:25:40.279 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:25:39 smithi089 bash[33173]: audit 2023-06-03T19:25:39.561813+0000 mon.a (mon.0) 230 : audit [DBG] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-03T19:25:40.279 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:25:39 smithi089 bash[26572]: audit 2023-06-03T19:25:39.560089+0000 mon.a (mon.0) 229 : audit [INF] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2023-06-03T19:25:40.279 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:25:39 smithi089 bash[26572]: audit 2023-06-03T19:25:39.561813+0000 mon.a (mon.0) 230 : audit [DBG] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-03T19:25:41.028 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:25:40 smithi089 bash[26572]: cephadm 2023-06-03T19:25:39.562968+0000 mgr.y (mgr.14146) 73 : cephadm [INF] Deploying daemon osd.0 on smithi089 2023-06-03T19:25:41.029 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:25:40 smithi089 bash[26572]: cluster 2023-06-03T19:25:39.707189+0000 mgr.y (mgr.14146) 74 : cluster [DBG] pgmap v47: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-06-03T19:25:41.029 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:25:40 smithi089 bash[33173]: cephadm 2023-06-03T19:25:39.562968+0000 mgr.y (mgr.14146) 73 : cephadm [INF] Deploying daemon osd.0 on smithi089 2023-06-03T19:25:41.029 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:25:40 smithi089 bash[33173]: cluster 2023-06-03T19:25:39.707189+0000 mgr.y (mgr.14146) 74 : cluster [DBG] pgmap v47: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-06-03T19:25:41.084 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:25:40 smithi164 bash[26855]: cephadm 2023-06-03T19:25:39.562968+0000 mgr.y (mgr.14146) 73 : cephadm [INF] Deploying daemon osd.0 on smithi089 2023-06-03T19:25:41.085 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:25:40 smithi164 bash[26855]: cluster 2023-06-03T19:25:39.707189+0000 mgr.y (mgr.14146) 74 : cluster [DBG] pgmap v47: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-06-03T19:25:42.778 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:25:42 smithi089 bash[26572]: cluster 2023-06-03T19:25:41.707657+0000 mgr.y (mgr.14146) 75 : cluster [DBG] pgmap v48: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-06-03T19:25:42.779 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:25:42 smithi089 bash[33173]: cluster 2023-06-03T19:25:41.707657+0000 mgr.y (mgr.14146) 75 : cluster [DBG] pgmap v48: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-06-03T19:25:42.834 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:25:42 smithi164 bash[26855]: cluster 2023-06-03T19:25:41.707657+0000 mgr.y (mgr.14146) 75 : cluster [DBG] pgmap v48: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-06-03T19:25:45.528 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:25:45 smithi089 bash[26572]: cluster 2023-06-03T19:25:43.708076+0000 mgr.y (mgr.14146) 76 : cluster [DBG] pgmap v49: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-06-03T19:25:45.529 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:25:45 smithi089 bash[26572]: audit 2023-06-03T19:25:44.086886+0000 mon.a (mon.0) 231 : audit [INF] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' 2023-06-03T19:25:45.529 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:25:45 smithi089 bash[26572]: audit 2023-06-03T19:25:44.097543+0000 mon.a (mon.0) 232 : audit [DBG] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-03T19:25:45.529 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:25:45 smithi089 bash[33173]: cluster 2023-06-03T19:25:43.708076+0000 mgr.y (mgr.14146) 76 : cluster [DBG] pgmap v49: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-06-03T19:25:45.530 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:25:45 smithi089 bash[33173]: audit 2023-06-03T19:25:44.086886+0000 mon.a (mon.0) 231 : audit [INF] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' 2023-06-03T19:25:45.530 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:25:45 smithi089 bash[33173]: audit 2023-06-03T19:25:44.097543+0000 mon.a (mon.0) 232 : audit [DBG] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-03T19:25:45.584 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:25:45 smithi164 bash[26855]: cluster 2023-06-03T19:25:43.708076+0000 mgr.y (mgr.14146) 76 : cluster [DBG] pgmap v49: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-06-03T19:25:45.584 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:25:45 smithi164 bash[26855]: audit 2023-06-03T19:25:44.086886+0000 mon.a (mon.0) 231 : audit [INF] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' 2023-06-03T19:25:45.585 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:25:45 smithi164 bash[26855]: audit 2023-06-03T19:25:44.097543+0000 mon.a (mon.0) 232 : audit [DBG] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-03T19:25:47.451 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:25:47 smithi089 bash[26572]: cluster 2023-06-03T19:25:45.708595+0000 mgr.y (mgr.14146) 77 : cluster [DBG] pgmap v50: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-06-03T19:25:47.451 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:25:47 smithi089 bash[33173]: cluster 2023-06-03T19:25:45.708595+0000 mgr.y (mgr.14146) 77 : cluster [DBG] pgmap v50: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-06-03T19:25:47.584 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:25:47 smithi164 bash[26855]: cluster 2023-06-03T19:25:45.708595+0000 mgr.y (mgr.14146) 77 : cluster [DBG] pgmap v50: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-06-03T19:25:49.528 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:25:49 smithi089 bash[26572]: cluster 2023-06-03T19:25:47.709045+0000 mgr.y (mgr.14146) 78 : cluster [DBG] pgmap v51: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-06-03T19:25:49.529 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:25:49 smithi089 bash[33173]: cluster 2023-06-03T19:25:47.709045+0000 mgr.y (mgr.14146) 78 : cluster [DBG] pgmap v51: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-06-03T19:25:49.585 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:25:49 smithi164 bash[26855]: cluster 2023-06-03T19:25:47.709045+0000 mgr.y (mgr.14146) 78 : cluster [DBG] pgmap v51: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-06-03T19:25:51.384 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:25:51 smithi089 bash[26572]: cluster 2023-06-03T19:25:49.709500+0000 mgr.y (mgr.14146) 79 : cluster [DBG] pgmap v52: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-06-03T19:25:51.384 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:25:51 smithi089 bash[26572]: audit 2023-06-03T19:25:50.458654+0000 mon.a (mon.0) 233 : audit [INF] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' 2023-06-03T19:25:51.384 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:25:51 smithi089 bash[26572]: audit 2023-06-03T19:25:50.459878+0000 mon.a (mon.0) 234 : audit [DBG] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-03T19:25:51.385 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:25:51 smithi089 bash[26572]: audit 2023-06-03T19:25:50.461259+0000 mon.a (mon.0) 235 : audit [INF] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-03T19:25:51.385 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:25:51 smithi089 bash[26572]: audit 2023-06-03T19:25:50.469531+0000 mon.a (mon.0) 236 : audit [INF] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' 2023-06-03T19:25:51.385 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:25:51 smithi089 bash[33173]: cluster 2023-06-03T19:25:49.709500+0000 mgr.y (mgr.14146) 79 : cluster [DBG] pgmap v52: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-06-03T19:25:51.385 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:25:51 smithi089 bash[33173]: audit 2023-06-03T19:25:50.458654+0000 mon.a (mon.0) 233 : audit [INF] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' 2023-06-03T19:25:51.385 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:25:51 smithi089 bash[33173]: audit 2023-06-03T19:25:50.459878+0000 mon.a (mon.0) 234 : audit [DBG] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-03T19:25:51.385 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:25:51 smithi089 bash[33173]: audit 2023-06-03T19:25:50.461259+0000 mon.a (mon.0) 235 : audit [INF] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-03T19:25:51.386 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:25:51 smithi089 bash[33173]: audit 2023-06-03T19:25:50.469531+0000 mon.a (mon.0) 236 : audit [INF] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' 2023-06-03T19:25:51.584 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:25:51 smithi164 bash[26855]: cluster 2023-06-03T19:25:49.709500+0000 mgr.y (mgr.14146) 79 : cluster [DBG] pgmap v52: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-06-03T19:25:51.585 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:25:51 smithi164 bash[26855]: audit 2023-06-03T19:25:50.458654+0000 mon.a (mon.0) 233 : audit [INF] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' 2023-06-03T19:25:51.585 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:25:51 smithi164 bash[26855]: audit 2023-06-03T19:25:50.459878+0000 mon.a (mon.0) 234 : audit [DBG] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-03T19:25:51.585 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:25:51 smithi164 bash[26855]: audit 2023-06-03T19:25:50.461259+0000 mon.a (mon.0) 235 : audit [INF] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-03T19:25:51.585 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:25:51 smithi164 bash[26855]: audit 2023-06-03T19:25:50.469531+0000 mon.a (mon.0) 236 : audit [INF] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' 2023-06-03T19:25:52.278 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:25:52 smithi089 bash[26572]: audit 2023-06-03T19:25:51.124398+0000 mon.c (mon.2) 3 : audit [INF] from='osd.0 [v2:172.21.15.89:6802/1795715824,v1:172.21.15.89:6803/1795715824]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2023-06-03T19:25:52.279 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:25:52 smithi089 bash[26572]: audit 2023-06-03T19:25:51.124898+0000 mon.a (mon.0) 237 : audit [INF] from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2023-06-03T19:25:52.279 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:25:52 smithi089 bash[33173]: audit 2023-06-03T19:25:51.124398+0000 mon.c (mon.2) 3 : audit [INF] from='osd.0 [v2:172.21.15.89:6802/1795715824,v1:172.21.15.89:6803/1795715824]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2023-06-03T19:25:52.279 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:25:52 smithi089 bash[33173]: audit 2023-06-03T19:25:51.124898+0000 mon.a (mon.0) 237 : audit [INF] from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2023-06-03T19:25:52.584 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:25:52 smithi164 bash[26855]: audit 2023-06-03T19:25:51.124398+0000 mon.c (mon.2) 3 : audit [INF] from='osd.0 [v2:172.21.15.89:6802/1795715824,v1:172.21.15.89:6803/1795715824]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2023-06-03T19:25:52.585 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:25:52 smithi164 bash[26855]: audit 2023-06-03T19:25:51.124898+0000 mon.a (mon.0) 237 : audit [INF] from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2023-06-03T19:25:53.469 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:25:53 smithi089 bash[26572]: cluster 2023-06-03T19:25:51.709830+0000 mgr.y (mgr.14146) 80 : cluster [DBG] pgmap v53: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-06-03T19:25:53.469 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:25:53 smithi089 bash[26572]: audit 2023-06-03T19:25:52.110344+0000 mon.a (mon.0) 238 : audit [INF] from='osd.0 ' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2023-06-03T19:25:53.469 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:25:53 smithi089 bash[26572]: cluster 2023-06-03T19:25:52.110410+0000 mon.a (mon.0) 239 : cluster [DBG] osdmap e6: 1 total, 0 up, 1 in 2023-06-03T19:25:53.470 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:25:53 smithi089 bash[26572]: audit 2023-06-03T19:25:52.110563+0000 mon.a (mon.0) 240 : audit [DBG] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-06-03T19:25:53.470 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:25:53 smithi089 bash[26572]: audit 2023-06-03T19:25:52.111301+0000 mon.c (mon.2) 4 : audit [INF] from='osd.0 [v2:172.21.15.89:6802/1795715824,v1:172.21.15.89:6803/1795715824]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi089", "root=default"]}]: dispatch 2023-06-03T19:25:53.470 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:25:53 smithi089 bash[26572]: audit 2023-06-03T19:25:52.111858+0000 mon.a (mon.0) 241 : audit [INF] from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi089", "root=default"]}]: dispatch 2023-06-03T19:25:53.471 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:25:53 smithi089 bash[33173]: cluster 2023-06-03T19:25:51.709830+0000 mgr.y (mgr.14146) 80 : cluster [DBG] pgmap v53: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-06-03T19:25:53.471 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:25:53 smithi089 bash[33173]: audit 2023-06-03T19:25:52.110344+0000 mon.a (mon.0) 238 : audit [INF] from='osd.0 ' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2023-06-03T19:25:53.471 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:25:53 smithi089 bash[33173]: cluster 2023-06-03T19:25:52.110410+0000 mon.a (mon.0) 239 : cluster [DBG] osdmap e6: 1 total, 0 up, 1 in 2023-06-03T19:25:53.472 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:25:53 smithi089 bash[33173]: audit 2023-06-03T19:25:52.110563+0000 mon.a (mon.0) 240 : audit [DBG] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-06-03T19:25:53.472 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:25:53 smithi089 bash[33173]: audit 2023-06-03T19:25:52.111301+0000 mon.c (mon.2) 4 : audit [INF] from='osd.0 [v2:172.21.15.89:6802/1795715824,v1:172.21.15.89:6803/1795715824]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi089", "root=default"]}]: dispatch 2023-06-03T19:25:53.472 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:25:53 smithi089 bash[33173]: audit 2023-06-03T19:25:52.111858+0000 mon.a (mon.0) 241 : audit [INF] from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi089", "root=default"]}]: dispatch 2023-06-03T19:25:53.584 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:25:53 smithi164 bash[26855]: cluster 2023-06-03T19:25:51.709830+0000 mgr.y (mgr.14146) 80 : cluster [DBG] pgmap v53: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-06-03T19:25:53.585 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:25:53 smithi164 bash[26855]: audit 2023-06-03T19:25:52.110344+0000 mon.a (mon.0) 238 : audit [INF] from='osd.0 ' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2023-06-03T19:25:53.585 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:25:53 smithi164 bash[26855]: cluster 2023-06-03T19:25:52.110410+0000 mon.a (mon.0) 239 : cluster [DBG] osdmap e6: 1 total, 0 up, 1 in 2023-06-03T19:25:53.585 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:25:53 smithi164 bash[26855]: audit 2023-06-03T19:25:52.110563+0000 mon.a (mon.0) 240 : audit [DBG] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-06-03T19:25:53.586 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:25:53 smithi164 bash[26855]: audit 2023-06-03T19:25:52.111301+0000 mon.c (mon.2) 4 : audit [INF] from='osd.0 [v2:172.21.15.89:6802/1795715824,v1:172.21.15.89:6803/1795715824]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi089", "root=default"]}]: dispatch 2023-06-03T19:25:53.586 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:25:53 smithi164 bash[26855]: audit 2023-06-03T19:25:52.111858+0000 mon.a (mon.0) 241 : audit [INF] from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi089", "root=default"]}]: dispatch 2023-06-03T19:25:54.212 INFO:teuthology.orchestra.run.smithi089.stdout:Created osd(s) 0 on host 'smithi089' 2023-06-03T19:25:54.408 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:25:54 smithi089 bash[26572]: audit 2023-06-03T19:25:53.113235+0000 mon.a (mon.0) 242 : audit [INF] from='osd.0 ' entity='osd.0' cmd='[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi089", "root=default"]}]': finished 2023-06-03T19:25:54.409 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:25:54 smithi089 bash[26572]: cluster 2023-06-03T19:25:53.113421+0000 mon.a (mon.0) 243 : cluster [DBG] osdmap e7: 1 total, 0 up, 1 in 2023-06-03T19:25:54.409 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:25:54 smithi089 bash[26572]: audit 2023-06-03T19:25:53.113584+0000 mon.a (mon.0) 244 : audit [DBG] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-06-03T19:25:54.409 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:25:54 smithi089 bash[26572]: audit 2023-06-03T19:25:53.117780+0000 mon.a (mon.0) 245 : audit [DBG] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-06-03T19:25:54.409 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:25:54 smithi089 bash[26572]: cluster 2023-06-03T19:25:53.710239+0000 mgr.y (mgr.14146) 81 : cluster [DBG] pgmap v56: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-06-03T19:25:54.409 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:25:54 smithi089 bash[33173]: audit 2023-06-03T19:25:53.113235+0000 mon.a (mon.0) 242 : audit [INF] from='osd.0 ' entity='osd.0' cmd='[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi089", "root=default"]}]': finished 2023-06-03T19:25:54.410 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:25:54 smithi089 bash[33173]: cluster 2023-06-03T19:25:53.113421+0000 mon.a (mon.0) 243 : cluster [DBG] osdmap e7: 1 total, 0 up, 1 in 2023-06-03T19:25:54.410 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:25:54 smithi089 bash[33173]: audit 2023-06-03T19:25:53.113584+0000 mon.a (mon.0) 244 : audit [DBG] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-06-03T19:25:54.410 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:25:54 smithi089 bash[33173]: audit 2023-06-03T19:25:53.117780+0000 mon.a (mon.0) 245 : audit [DBG] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-06-03T19:25:54.410 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:25:54 smithi089 bash[33173]: cluster 2023-06-03T19:25:53.710239+0000 mgr.y (mgr.14146) 81 : cluster [DBG] pgmap v56: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-06-03T19:25:54.584 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:25:54 smithi164 bash[26855]: audit 2023-06-03T19:25:53.113235+0000 mon.a (mon.0) 242 : audit [INF] from='osd.0 ' entity='osd.0' cmd='[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi089", "root=default"]}]': finished 2023-06-03T19:25:54.585 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:25:54 smithi164 bash[26855]: cluster 2023-06-03T19:25:53.113421+0000 mon.a (mon.0) 243 : cluster [DBG] osdmap e7: 1 total, 0 up, 1 in 2023-06-03T19:25:54.585 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:25:54 smithi164 bash[26855]: audit 2023-06-03T19:25:53.113584+0000 mon.a (mon.0) 244 : audit [DBG] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-06-03T19:25:54.585 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:25:54 smithi164 bash[26855]: audit 2023-06-03T19:25:53.117780+0000 mon.a (mon.0) 245 : audit [DBG] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-06-03T19:25:54.585 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:25:54 smithi164 bash[26855]: cluster 2023-06-03T19:25:53.710239+0000 mgr.y (mgr.14146) 81 : cluster [DBG] pgmap v56: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-06-03T19:25:54.948 DEBUG:teuthology.orchestra.run.smithi089:osd.0> sudo journalctl -f -n 0 -u ceph-f1db3478-0243-11ee-9b1d-001a4aab830c@osd.0.service 2023-06-03T19:25:54.951 INFO:tasks.cephadm:Deploying osd.1 on smithi089 with /dev/vg_nvme/lv_3... 2023-06-03T19:25:54.951 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph-volume lvm zap /dev/vg_nvme/lv_3 2023-06-03T19:25:55.075 INFO:journalctl@ceph.osd.0.smithi089.stdout:-- Logs begin at Sun 2022-08-14 12:20:24 UTC. -- 2023-06-03T19:25:55.202 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:25:55 smithi089 bash[33173]: cluster 2023-06-03T19:25:52.155588+0000 osd.0 (osd.0) 1 : cluster [DBG] purged_snaps scrub starts 2023-06-03T19:25:55.202 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:25:55 smithi089 bash[33173]: cluster 2023-06-03T19:25:52.155706+0000 osd.0 (osd.0) 2 : cluster [DBG] purged_snaps scrub ok 2023-06-03T19:25:55.202 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:25:55 smithi089 bash[33173]: cluster 2023-06-03T19:25:54.122450+0000 mon.a (mon.0) 246 : cluster [INF] osd.0 [v2:172.21.15.89:6802/1795715824,v1:172.21.15.89:6803/1795715824] boot 2023-06-03T19:25:55.203 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:25:55 smithi089 bash[33173]: cluster 2023-06-03T19:25:54.122502+0000 mon.a (mon.0) 247 : cluster [DBG] osdmap e8: 1 total, 1 up, 1 in 2023-06-03T19:25:55.203 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:25:55 smithi089 bash[33173]: audit 2023-06-03T19:25:54.122879+0000 mon.a (mon.0) 248 : audit [DBG] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-06-03T19:25:55.203 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:25:55 smithi089 bash[33173]: audit 2023-06-03T19:25:54.207211+0000 mon.a (mon.0) 249 : audit [INF] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' 2023-06-03T19:25:55.203 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:25:55 smithi089 bash[33173]: audit 2023-06-03T19:25:54.247680+0000 mon.a (mon.0) 250 : audit [DBG] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-03T19:25:55.204 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:25:55 smithi089 bash[26572]: cluster 2023-06-03T19:25:52.155588+0000 osd.0 (osd.0) 1 : cluster [DBG] purged_snaps scrub starts 2023-06-03T19:25:55.204 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:25:55 smithi089 bash[26572]: cluster 2023-06-03T19:25:52.155706+0000 osd.0 (osd.0) 2 : cluster [DBG] purged_snaps scrub ok 2023-06-03T19:25:55.204 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:25:55 smithi089 bash[26572]: cluster 2023-06-03T19:25:54.122450+0000 mon.a (mon.0) 246 : cluster [INF] osd.0 [v2:172.21.15.89:6802/1795715824,v1:172.21.15.89:6803/1795715824] boot 2023-06-03T19:25:55.204 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:25:55 smithi089 bash[26572]: cluster 2023-06-03T19:25:54.122502+0000 mon.a (mon.0) 247 : cluster [DBG] osdmap e8: 1 total, 1 up, 1 in 2023-06-03T19:25:55.205 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:25:55 smithi089 bash[26572]: audit 2023-06-03T19:25:54.122879+0000 mon.a (mon.0) 248 : audit [DBG] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-06-03T19:25:55.205 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:25:55 smithi089 bash[26572]: audit 2023-06-03T19:25:54.207211+0000 mon.a (mon.0) 249 : audit [INF] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' 2023-06-03T19:25:55.205 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:25:55 smithi089 bash[26572]: audit 2023-06-03T19:25:54.247680+0000 mon.a (mon.0) 250 : audit [DBG] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-03T19:25:55.585 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:25:55 smithi164 bash[26855]: cluster 2023-06-03T19:25:52.155588+0000 osd.0 (osd.0) 1 : cluster [DBG] purged_snaps scrub starts 2023-06-03T19:25:55.585 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:25:55 smithi164 bash[26855]: cluster 2023-06-03T19:25:52.155706+0000 osd.0 (osd.0) 2 : cluster [DBG] purged_snaps scrub ok 2023-06-03T19:25:55.586 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:25:55 smithi164 bash[26855]: cluster 2023-06-03T19:25:54.122450+0000 mon.a (mon.0) 246 : cluster [INF] osd.0 [v2:172.21.15.89:6802/1795715824,v1:172.21.15.89:6803/1795715824] boot 2023-06-03T19:25:55.586 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:25:55 smithi164 bash[26855]: cluster 2023-06-03T19:25:54.122502+0000 mon.a (mon.0) 247 : cluster [DBG] osdmap e8: 1 total, 1 up, 1 in 2023-06-03T19:25:55.586 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:25:55 smithi164 bash[26855]: audit 2023-06-03T19:25:54.122879+0000 mon.a (mon.0) 248 : audit [DBG] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-06-03T19:25:55.586 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:25:55 smithi164 bash[26855]: audit 2023-06-03T19:25:54.207211+0000 mon.a (mon.0) 249 : audit [INF] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' 2023-06-03T19:25:55.586 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:25:55 smithi164 bash[26855]: audit 2023-06-03T19:25:54.247680+0000 mon.a (mon.0) 250 : audit [DBG] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-03T19:25:56.403 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:25:56 smithi089 bash[33173]: cluster 2023-06-03T19:25:55.127746+0000 mon.a (mon.0) 251 : cluster [DBG] osdmap e9: 1 total, 1 up, 1 in 2023-06-03T19:25:56.403 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:25:56 smithi089 bash[33173]: cluster 2023-06-03T19:25:55.710587+0000 mgr.y (mgr.14146) 82 : cluster [DBG] pgmap v59: 0 pgs: ; 0 B data, 4.7 MiB used, 89 GiB / 89 GiB avail 2023-06-03T19:25:56.404 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:25:56 smithi089 bash[26572]: cluster 2023-06-03T19:25:55.127746+0000 mon.a (mon.0) 251 : cluster [DBG] osdmap e9: 1 total, 1 up, 1 in 2023-06-03T19:25:56.404 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:25:56 smithi089 bash[26572]: cluster 2023-06-03T19:25:55.710587+0000 mgr.y (mgr.14146) 82 : cluster [DBG] pgmap v59: 0 pgs: ; 0 B data, 4.7 MiB used, 89 GiB / 89 GiB avail 2023-06-03T19:25:56.584 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:25:56 smithi164 bash[26855]: cluster 2023-06-03T19:25:55.127746+0000 mon.a (mon.0) 251 : cluster [DBG] osdmap e9: 1 total, 1 up, 1 in 2023-06-03T19:25:56.584 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:25:56 smithi164 bash[26855]: cluster 2023-06-03T19:25:55.710587+0000 mgr.y (mgr.14146) 82 : cluster [DBG] pgmap v59: 0 pgs: ; 0 B data, 4.7 MiB used, 89 GiB / 89 GiB avail 2023-06-03T19:25:58.602 INFO:teuthology.orchestra.run.smithi089.stderr:--> Zapping: /dev/vg_nvme/lv_3 2023-06-03T19:25:58.602 INFO:teuthology.orchestra.run.smithi089.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_3 bs=1M count=10 conv=fsync 2023-06-03T19:25:58.603 INFO:teuthology.orchestra.run.smithi089.stderr: stderr: 10+0 records in 2023-06-03T19:25:58.603 INFO:teuthology.orchestra.run.smithi089.stderr:10+0 records out 2023-06-03T19:25:58.603 INFO:teuthology.orchestra.run.smithi089.stderr: stderr: 10485760 bytes (10 MB, 10 MiB) copied, 0.0191078 s, 549 MB/s 2023-06-03T19:25:58.603 INFO:teuthology.orchestra.run.smithi089.stderr:--> Zapping successful for: 2023-06-03T19:25:59.029 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:25:58 smithi089 bash[26572]: cluster 2023-06-03T19:25:57.711152+0000 mgr.y (mgr.14146) 83 : cluster [DBG] pgmap v60: 0 pgs: ; 0 B data, 4.7 MiB used, 89 GiB / 89 GiB avail 2023-06-03T19:25:59.029 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:25:58 smithi089 bash[33173]: cluster 2023-06-03T19:25:57.711152+0000 mgr.y (mgr.14146) 83 : cluster [DBG] pgmap v60: 0 pgs: ; 0 B data, 4.7 MiB used, 89 GiB / 89 GiB avail 2023-06-03T19:25:59.085 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:25:58 smithi164 bash[26855]: cluster 2023-06-03T19:25:57.711152+0000 mgr.y (mgr.14146) 83 : cluster [DBG] pgmap v60: 0 pgs: ; 0 B data, 4.7 MiB used, 89 GiB / 89 GiB avail 2023-06-03T19:25:59.336 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph orch daemon add osd smithi089:vg_nvme/lv_3 2023-06-03T19:26:01.029 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:26:00 smithi089 bash[26572]: cluster 2023-06-03T19:25:59.711867+0000 mgr.y (mgr.14146) 84 : cluster [DBG] pgmap v61: 0 pgs: ; 0 B data, 4.7 MiB used, 89 GiB / 89 GiB avail 2023-06-03T19:26:01.029 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:26:00 smithi089 bash[33173]: cluster 2023-06-03T19:25:59.711867+0000 mgr.y (mgr.14146) 84 : cluster [DBG] pgmap v61: 0 pgs: ; 0 B data, 4.7 MiB used, 89 GiB / 89 GiB avail 2023-06-03T19:26:01.085 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:26:00 smithi164 bash[26855]: cluster 2023-06-03T19:25:59.711867+0000 mgr.y (mgr.14146) 84 : cluster [DBG] pgmap v61: 0 pgs: ; 0 B data, 4.7 MiB used, 89 GiB / 89 GiB avail 2023-06-03T19:26:02.529 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:26:02 smithi089 bash[26572]: cluster 2023-06-03T19:26:01.712461+0000 mgr.y (mgr.14146) 85 : cluster [DBG] pgmap v62: 0 pgs: ; 0 B data, 4.7 MiB used, 89 GiB / 89 GiB avail 2023-06-03T19:26:02.529 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:26:02 smithi089 bash[33173]: cluster 2023-06-03T19:26:01.712461+0000 mgr.y (mgr.14146) 85 : cluster [DBG] pgmap v62: 0 pgs: ; 0 B data, 4.7 MiB used, 89 GiB / 89 GiB avail 2023-06-03T19:26:02.834 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:26:02 smithi164 bash[26855]: cluster 2023-06-03T19:26:01.712461+0000 mgr.y (mgr.14146) 85 : cluster [DBG] pgmap v62: 0 pgs: ; 0 B data, 4.7 MiB used, 89 GiB / 89 GiB avail 2023-06-03T19:26:05.029 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:26:04 smithi089 bash[33173]: cluster 2023-06-03T19:26:03.713107+0000 mgr.y (mgr.14146) 86 : cluster [DBG] pgmap v63: 0 pgs: ; 0 B data, 4.7 MiB used, 89 GiB / 89 GiB avail 2023-06-03T19:26:05.029 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:26:04 smithi089 bash[33173]: audit 2023-06-03T19:26:04.425511+0000 mon.a (mon.0) 252 : audit [DBG] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-06-03T19:26:05.029 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:26:04 smithi089 bash[33173]: audit 2023-06-03T19:26:04.430264+0000 mon.a (mon.0) 253 : audit [INF] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-06-03T19:26:05.029 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:26:04 smithi089 bash[33173]: audit 2023-06-03T19:26:04.431390+0000 mon.a (mon.0) 254 : audit [DBG] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-03T19:26:05.030 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:26:04 smithi089 bash[26572]: cluster 2023-06-03T19:26:03.713107+0000 mgr.y (mgr.14146) 86 : cluster [DBG] pgmap v63: 0 pgs: ; 0 B data, 4.7 MiB used, 89 GiB / 89 GiB avail 2023-06-03T19:26:05.030 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:26:04 smithi089 bash[26572]: audit 2023-06-03T19:26:04.425511+0000 mon.a (mon.0) 252 : audit [DBG] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-06-03T19:26:05.030 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:26:04 smithi089 bash[26572]: audit 2023-06-03T19:26:04.430264+0000 mon.a (mon.0) 253 : audit [INF] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-06-03T19:26:05.030 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:26:04 smithi089 bash[26572]: audit 2023-06-03T19:26:04.431390+0000 mon.a (mon.0) 254 : audit [DBG] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-03T19:26:05.084 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:26:04 smithi164 bash[26855]: cluster 2023-06-03T19:26:03.713107+0000 mgr.y (mgr.14146) 86 : cluster [DBG] pgmap v63: 0 pgs: ; 0 B data, 4.7 MiB used, 89 GiB / 89 GiB avail 2023-06-03T19:26:05.085 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:26:04 smithi164 bash[26855]: audit 2023-06-03T19:26:04.425511+0000 mon.a (mon.0) 252 : audit [DBG] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-06-03T19:26:05.085 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:26:04 smithi164 bash[26855]: audit 2023-06-03T19:26:04.430264+0000 mon.a (mon.0) 253 : audit [INF] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-06-03T19:26:05.085 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:26:04 smithi164 bash[26855]: audit 2023-06-03T19:26:04.431390+0000 mon.a (mon.0) 254 : audit [DBG] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-03T19:26:06.084 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:26:05 smithi164 bash[26855]: audit 2023-06-03T19:26:04.421232+0000 mgr.y (mgr.14146) 87 : audit [DBG] from='client.14235 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi089:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2023-06-03T19:26:06.279 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:26:05 smithi089 bash[26572]: audit 2023-06-03T19:26:04.421232+0000 mgr.y (mgr.14146) 87 : audit [DBG] from='client.14235 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi089:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2023-06-03T19:26:06.279 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:26:05 smithi089 bash[33173]: audit 2023-06-03T19:26:04.421232+0000 mgr.y (mgr.14146) 87 : audit [DBG] from='client.14235 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi089:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2023-06-03T19:26:07.279 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:26:06 smithi089 bash[33173]: cluster 2023-06-03T19:26:05.713592+0000 mgr.y (mgr.14146) 88 : cluster [DBG] pgmap v64: 0 pgs: ; 0 B data, 4.7 MiB used, 89 GiB / 89 GiB avail 2023-06-03T19:26:07.279 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:26:06 smithi089 bash[33173]: cephadm 2023-06-03T19:26:05.923810+0000 mgr.y (mgr.14146) 89 : cephadm [INF] Detected new or changed devices on smithi089 2023-06-03T19:26:07.279 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:26:06 smithi089 bash[33173]: audit 2023-06-03T19:26:05.933579+0000 mon.a (mon.0) 255 : audit [INF] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' 2023-06-03T19:26:07.280 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:26:06 smithi089 bash[33173]: audit 2023-06-03T19:26:05.935166+0000 mon.a (mon.0) 256 : audit [INF] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-06-03T19:26:07.280 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:26:06 smithi089 bash[33173]: cephadm 2023-06-03T19:26:05.936161+0000 mgr.y (mgr.14146) 90 : cephadm [INF] Adjusting osd_memory_target on smithi089 to 16249M 2023-06-03T19:26:07.280 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:26:06 smithi089 bash[33173]: audit 2023-06-03T19:26:05.941516+0000 mon.a (mon.0) 257 : audit [INF] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' 2023-06-03T19:26:07.280 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:26:06 smithi089 bash[33173]: audit 2023-06-03T19:26:05.942980+0000 mon.a (mon.0) 258 : audit [DBG] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-03T19:26:07.280 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:26:06 smithi089 bash[33173]: audit 2023-06-03T19:26:05.944064+0000 mon.a (mon.0) 259 : audit [INF] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-03T19:26:07.280 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:26:06 smithi089 bash[33173]: audit 2023-06-03T19:26:05.950378+0000 mon.a (mon.0) 260 : audit [INF] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' 2023-06-03T19:26:07.281 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:26:06 smithi089 bash[26572]: cluster 2023-06-03T19:26:05.713592+0000 mgr.y (mgr.14146) 88 : cluster [DBG] pgmap v64: 0 pgs: ; 0 B data, 4.7 MiB used, 89 GiB / 89 GiB avail 2023-06-03T19:26:07.281 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:26:06 smithi089 bash[26572]: cephadm 2023-06-03T19:26:05.923810+0000 mgr.y (mgr.14146) 89 : cephadm [INF] Detected new or changed devices on smithi089 2023-06-03T19:26:07.281 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:26:06 smithi089 bash[26572]: audit 2023-06-03T19:26:05.933579+0000 mon.a (mon.0) 255 : audit [INF] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' 2023-06-03T19:26:07.281 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:26:06 smithi089 bash[26572]: audit 2023-06-03T19:26:05.935166+0000 mon.a (mon.0) 256 : audit [INF] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-06-03T19:26:07.281 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:26:06 smithi089 bash[26572]: cephadm 2023-06-03T19:26:05.936161+0000 mgr.y (mgr.14146) 90 : cephadm [INF] Adjusting osd_memory_target on smithi089 to 16249M 2023-06-03T19:26:07.281 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:26:06 smithi089 bash[26572]: audit 2023-06-03T19:26:05.941516+0000 mon.a (mon.0) 257 : audit [INF] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' 2023-06-03T19:26:07.282 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:26:06 smithi089 bash[26572]: audit 2023-06-03T19:26:05.942980+0000 mon.a (mon.0) 258 : audit [DBG] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-03T19:26:07.282 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:26:06 smithi089 bash[26572]: audit 2023-06-03T19:26:05.944064+0000 mon.a (mon.0) 259 : audit [INF] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-03T19:26:07.282 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:26:06 smithi089 bash[26572]: audit 2023-06-03T19:26:05.950378+0000 mon.a (mon.0) 260 : audit [INF] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' 2023-06-03T19:26:07.334 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:26:06 smithi164 bash[26855]: cluster 2023-06-03T19:26:05.713592+0000 mgr.y (mgr.14146) 88 : cluster [DBG] pgmap v64: 0 pgs: ; 0 B data, 4.7 MiB used, 89 GiB / 89 GiB avail 2023-06-03T19:26:07.335 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:26:06 smithi164 bash[26855]: cephadm 2023-06-03T19:26:05.923810+0000 mgr.y (mgr.14146) 89 : cephadm [INF] Detected new or changed devices on smithi089 2023-06-03T19:26:07.335 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:26:06 smithi164 bash[26855]: audit 2023-06-03T19:26:05.933579+0000 mon.a (mon.0) 255 : audit [INF] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' 2023-06-03T19:26:07.335 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:26:06 smithi164 bash[26855]: audit 2023-06-03T19:26:05.935166+0000 mon.a (mon.0) 256 : audit [INF] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-06-03T19:26:07.335 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:26:06 smithi164 bash[26855]: cephadm 2023-06-03T19:26:05.936161+0000 mgr.y (mgr.14146) 90 : cephadm [INF] Adjusting osd_memory_target on smithi089 to 16249M 2023-06-03T19:26:07.335 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:26:06 smithi164 bash[26855]: audit 2023-06-03T19:26:05.941516+0000 mon.a (mon.0) 257 : audit [INF] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' 2023-06-03T19:26:07.335 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:26:06 smithi164 bash[26855]: audit 2023-06-03T19:26:05.942980+0000 mon.a (mon.0) 258 : audit [DBG] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-03T19:26:07.335 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:26:06 smithi164 bash[26855]: audit 2023-06-03T19:26:05.944064+0000 mon.a (mon.0) 259 : audit [INF] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-03T19:26:07.336 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:26:06 smithi164 bash[26855]: audit 2023-06-03T19:26:05.950378+0000 mon.a (mon.0) 260 : audit [INF] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' 2023-06-03T19:26:09.230 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:26:08 smithi089 bash[26572]: cluster 2023-06-03T19:26:07.714058+0000 mgr.y (mgr.14146) 91 : cluster [DBG] pgmap v65: 0 pgs: ; 0 B data, 4.7 MiB used, 89 GiB / 89 GiB avail 2023-06-03T19:26:09.231 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:26:08 smithi089 bash[33173]: cluster 2023-06-03T19:26:07.714058+0000 mgr.y (mgr.14146) 91 : cluster [DBG] pgmap v65: 0 pgs: ; 0 B data, 4.7 MiB used, 89 GiB / 89 GiB avail 2023-06-03T19:26:09.334 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:26:08 smithi164 bash[26855]: cluster 2023-06-03T19:26:07.714058+0000 mgr.y (mgr.14146) 91 : cluster [DBG] pgmap v65: 0 pgs: ; 0 B data, 4.7 MiB used, 89 GiB / 89 GiB avail 2023-06-03T19:26:11.279 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:26:10 smithi089 bash[26572]: cluster 2023-06-03T19:26:09.714580+0000 mgr.y (mgr.14146) 92 : cluster [DBG] pgmap v66: 0 pgs: ; 0 B data, 4.7 MiB used, 89 GiB / 89 GiB avail 2023-06-03T19:26:11.279 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:26:10 smithi089 bash[33173]: cluster 2023-06-03T19:26:09.714580+0000 mgr.y (mgr.14146) 92 : cluster [DBG] pgmap v66: 0 pgs: ; 0 B data, 4.7 MiB used, 89 GiB / 89 GiB avail 2023-06-03T19:26:11.334 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:26:10 smithi164 bash[26855]: cluster 2023-06-03T19:26:09.714580+0000 mgr.y (mgr.14146) 92 : cluster [DBG] pgmap v66: 0 pgs: ; 0 B data, 4.7 MiB used, 89 GiB / 89 GiB avail 2023-06-03T19:26:12.779 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:26:12 smithi089 bash[26572]: cluster 2023-06-03T19:26:11.715161+0000 mgr.y (mgr.14146) 93 : cluster [DBG] pgmap v67: 0 pgs: ; 0 B data, 4.7 MiB used, 89 GiB / 89 GiB avail 2023-06-03T19:26:12.779 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:26:12 smithi089 bash[33173]: cluster 2023-06-03T19:26:11.715161+0000 mgr.y (mgr.14146) 93 : cluster [DBG] pgmap v67: 0 pgs: ; 0 B data, 4.7 MiB used, 89 GiB / 89 GiB avail 2023-06-03T19:26:12.834 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:26:12 smithi164 bash[26855]: cluster 2023-06-03T19:26:11.715161+0000 mgr.y (mgr.14146) 93 : cluster [DBG] pgmap v67: 0 pgs: ; 0 B data, 4.7 MiB used, 89 GiB / 89 GiB avail 2023-06-03T19:26:13.779 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:26:13 smithi089 bash[26572]: audit 2023-06-03T19:26:12.991661+0000 mon.a (mon.0) 261 : audit [INF] from='client.? 172.21.15.89:0/2892568601' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "04783a81-6b8c-4657-8446-00efc872b313"}]: dispatch 2023-06-03T19:26:13.779 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:26:13 smithi089 bash[26572]: audit 2023-06-03T19:26:13.000985+0000 mon.a (mon.0) 262 : audit [INF] from='client.? 172.21.15.89:0/2892568601' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "04783a81-6b8c-4657-8446-00efc872b313"}]': finished 2023-06-03T19:26:13.779 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:26:13 smithi089 bash[26572]: cluster 2023-06-03T19:26:13.001027+0000 mon.a (mon.0) 263 : cluster [DBG] osdmap e10: 2 total, 1 up, 2 in 2023-06-03T19:26:13.780 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:26:13 smithi089 bash[26572]: audit 2023-06-03T19:26:13.001124+0000 mon.a (mon.0) 264 : audit [DBG] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-06-03T19:26:13.780 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:26:13 smithi089 bash[33173]: audit 2023-06-03T19:26:12.991661+0000 mon.a (mon.0) 261 : audit [INF] from='client.? 172.21.15.89:0/2892568601' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "04783a81-6b8c-4657-8446-00efc872b313"}]: dispatch 2023-06-03T19:26:13.780 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:26:13 smithi089 bash[33173]: audit 2023-06-03T19:26:13.000985+0000 mon.a (mon.0) 262 : audit [INF] from='client.? 172.21.15.89:0/2892568601' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "04783a81-6b8c-4657-8446-00efc872b313"}]': finished 2023-06-03T19:26:13.780 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:26:13 smithi089 bash[33173]: cluster 2023-06-03T19:26:13.001027+0000 mon.a (mon.0) 263 : cluster [DBG] osdmap e10: 2 total, 1 up, 2 in 2023-06-03T19:26:13.780 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:26:13 smithi089 bash[33173]: audit 2023-06-03T19:26:13.001124+0000 mon.a (mon.0) 264 : audit [DBG] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-06-03T19:26:13.834 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:26:13 smithi164 bash[26855]: audit 2023-06-03T19:26:12.991661+0000 mon.a (mon.0) 261 : audit [INF] from='client.? 172.21.15.89:0/2892568601' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "04783a81-6b8c-4657-8446-00efc872b313"}]: dispatch 2023-06-03T19:26:13.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:26:13 smithi164 bash[26855]: audit 2023-06-03T19:26:13.000985+0000 mon.a (mon.0) 262 : audit [INF] from='client.? 172.21.15.89:0/2892568601' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "04783a81-6b8c-4657-8446-00efc872b313"}]': finished 2023-06-03T19:26:13.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:26:13 smithi164 bash[26855]: cluster 2023-06-03T19:26:13.001027+0000 mon.a (mon.0) 263 : cluster [DBG] osdmap e10: 2 total, 1 up, 2 in 2023-06-03T19:26:13.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:26:13 smithi164 bash[26855]: audit 2023-06-03T19:26:13.001124+0000 mon.a (mon.0) 264 : audit [DBG] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-06-03T19:26:14.779 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:26:14 smithi089 bash[26572]: cluster 2023-06-03T19:26:13.715535+0000 mgr.y (mgr.14146) 94 : cluster [DBG] pgmap v69: 0 pgs: ; 0 B data, 4.7 MiB used, 89 GiB / 89 GiB avail 2023-06-03T19:26:14.779 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:26:14 smithi089 bash[26572]: audit 2023-06-03T19:26:13.728864+0000 mon.c (mon.2) 5 : audit [DBG] from='client.? 172.21.15.89:0/1876090063' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-06-03T19:26:14.780 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:26:14 smithi089 bash[33173]: cluster 2023-06-03T19:26:13.715535+0000 mgr.y (mgr.14146) 94 : cluster [DBG] pgmap v69: 0 pgs: ; 0 B data, 4.7 MiB used, 89 GiB / 89 GiB avail 2023-06-03T19:26:14.780 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:26:14 smithi089 bash[33173]: audit 2023-06-03T19:26:13.728864+0000 mon.c (mon.2) 5 : audit [DBG] from='client.? 172.21.15.89:0/1876090063' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-06-03T19:26:14.834 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:26:14 smithi164 bash[26855]: cluster 2023-06-03T19:26:13.715535+0000 mgr.y (mgr.14146) 94 : cluster [DBG] pgmap v69: 0 pgs: ; 0 B data, 4.7 MiB used, 89 GiB / 89 GiB avail 2023-06-03T19:26:14.834 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:26:14 smithi164 bash[26855]: audit 2023-06-03T19:26:13.728864+0000 mon.c (mon.2) 5 : audit [DBG] from='client.? 172.21.15.89:0/1876090063' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-06-03T19:26:17.029 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:26:16 smithi089 bash[26572]: cluster 2023-06-03T19:26:15.715991+0000 mgr.y (mgr.14146) 95 : cluster [DBG] pgmap v70: 0 pgs: ; 0 B data, 4.7 MiB used, 89 GiB / 89 GiB avail 2023-06-03T19:26:17.030 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:26:16 smithi089 bash[33173]: cluster 2023-06-03T19:26:15.715991+0000 mgr.y (mgr.14146) 95 : cluster [DBG] pgmap v70: 0 pgs: ; 0 B data, 4.7 MiB used, 89 GiB / 89 GiB avail 2023-06-03T19:26:17.084 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:26:16 smithi164 bash[26855]: cluster 2023-06-03T19:26:15.715991+0000 mgr.y (mgr.14146) 95 : cluster [DBG] pgmap v70: 0 pgs: ; 0 B data, 4.7 MiB used, 89 GiB / 89 GiB avail 2023-06-03T19:26:19.084 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:26:18 smithi164 bash[26855]: cluster 2023-06-03T19:26:17.716479+0000 mgr.y (mgr.14146) 96 : cluster [DBG] pgmap v71: 0 pgs: ; 0 B data, 4.7 MiB used, 89 GiB / 89 GiB avail 2023-06-03T19:26:19.279 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:26:18 smithi089 bash[26572]: cluster 2023-06-03T19:26:17.716479+0000 mgr.y (mgr.14146) 96 : cluster [DBG] pgmap v71: 0 pgs: ; 0 B data, 4.7 MiB used, 89 GiB / 89 GiB avail 2023-06-03T19:26:19.280 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:26:18 smithi089 bash[33173]: cluster 2023-06-03T19:26:17.716479+0000 mgr.y (mgr.14146) 96 : cluster [DBG] pgmap v71: 0 pgs: ; 0 B data, 4.7 MiB used, 89 GiB / 89 GiB avail 2023-06-03T19:26:21.084 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:26:20 smithi164 bash[26855]: cluster 2023-06-03T19:26:19.717062+0000 mgr.y (mgr.14146) 97 : cluster [DBG] pgmap v72: 0 pgs: ; 0 B data, 4.7 MiB used, 89 GiB / 89 GiB avail 2023-06-03T19:26:21.279 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:26:20 smithi089 bash[26572]: cluster 2023-06-03T19:26:19.717062+0000 mgr.y (mgr.14146) 97 : cluster [DBG] pgmap v72: 0 pgs: ; 0 B data, 4.7 MiB used, 89 GiB / 89 GiB avail 2023-06-03T19:26:21.280 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:26:20 smithi089 bash[33173]: cluster 2023-06-03T19:26:19.717062+0000 mgr.y (mgr.14146) 97 : cluster [DBG] pgmap v72: 0 pgs: ; 0 B data, 4.7 MiB used, 89 GiB / 89 GiB avail 2023-06-03T19:26:22.743 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:26:22 smithi089 bash[33173]: cluster 2023-06-03T19:26:21.717470+0000 mgr.y (mgr.14146) 98 : cluster [DBG] pgmap v73: 0 pgs: ; 0 B data, 4.7 MiB used, 89 GiB / 89 GiB avail 2023-06-03T19:26:22.743 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:26:22 smithi089 bash[26572]: cluster 2023-06-03T19:26:21.717470+0000 mgr.y (mgr.14146) 98 : cluster [DBG] pgmap v73: 0 pgs: ; 0 B data, 4.7 MiB used, 89 GiB / 89 GiB avail 2023-06-03T19:26:22.834 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:26:22 smithi164 bash[26855]: cluster 2023-06-03T19:26:21.717470+0000 mgr.y (mgr.14146) 98 : cluster [DBG] pgmap v73: 0 pgs: ; 0 B data, 4.7 MiB used, 89 GiB / 89 GiB avail 2023-06-03T19:26:25.029 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:26:24 smithi089 bash[26572]: cluster 2023-06-03T19:26:23.717855+0000 mgr.y (mgr.14146) 99 : cluster [DBG] pgmap v74: 0 pgs: ; 0 B data, 4.7 MiB used, 89 GiB / 89 GiB avail 2023-06-03T19:26:25.030 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:26:24 smithi089 bash[33173]: cluster 2023-06-03T19:26:23.717855+0000 mgr.y (mgr.14146) 99 : cluster [DBG] pgmap v74: 0 pgs: ; 0 B data, 4.7 MiB used, 89 GiB / 89 GiB avail 2023-06-03T19:26:25.084 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:26:24 smithi164 bash[26855]: cluster 2023-06-03T19:26:23.717855+0000 mgr.y (mgr.14146) 99 : cluster [DBG] pgmap v74: 0 pgs: ; 0 B data, 4.7 MiB used, 89 GiB / 89 GiB avail 2023-06-03T19:26:27.084 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:26:26 smithi164 bash[26855]: cluster 2023-06-03T19:26:25.718310+0000 mgr.y (mgr.14146) 100 : cluster [DBG] pgmap v75: 0 pgs: ; 0 B data, 4.7 MiB used, 89 GiB / 89 GiB avail 2023-06-03T19:26:27.085 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:26:26 smithi164 bash[26855]: audit 2023-06-03T19:26:25.857182+0000 mon.a (mon.0) 265 : audit [INF] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2023-06-03T19:26:27.085 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:26:26 smithi164 bash[26855]: audit 2023-06-03T19:26:25.858552+0000 mon.a (mon.0) 266 : audit [DBG] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-03T19:26:27.086 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:26:26 smithi164 bash[26855]: cephadm 2023-06-03T19:26:25.859822+0000 mgr.y (mgr.14146) 101 : cephadm [INF] Deploying daemon osd.1 on smithi089 2023-06-03T19:26:27.112 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:26:26 smithi089 bash[26572]: cluster 2023-06-03T19:26:25.718310+0000 mgr.y (mgr.14146) 100 : cluster [DBG] pgmap v75: 0 pgs: ; 0 B data, 4.7 MiB used, 89 GiB / 89 GiB avail 2023-06-03T19:26:27.112 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:26:26 smithi089 bash[26572]: audit 2023-06-03T19:26:25.857182+0000 mon.a (mon.0) 265 : audit [INF] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2023-06-03T19:26:27.113 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:26:26 smithi089 bash[26572]: audit 2023-06-03T19:26:25.858552+0000 mon.a (mon.0) 266 : audit [DBG] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-03T19:26:27.113 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:26:26 smithi089 bash[26572]: cephadm 2023-06-03T19:26:25.859822+0000 mgr.y (mgr.14146) 101 : cephadm [INF] Deploying daemon osd.1 on smithi089 2023-06-03T19:26:27.114 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:26:26 smithi089 bash[33173]: cluster 2023-06-03T19:26:25.718310+0000 mgr.y (mgr.14146) 100 : cluster [DBG] pgmap v75: 0 pgs: ; 0 B data, 4.7 MiB used, 89 GiB / 89 GiB avail 2023-06-03T19:26:27.114 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:26:26 smithi089 bash[33173]: audit 2023-06-03T19:26:25.857182+0000 mon.a (mon.0) 265 : audit [INF] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2023-06-03T19:26:27.114 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:26:26 smithi089 bash[33173]: audit 2023-06-03T19:26:25.858552+0000 mon.a (mon.0) 266 : audit [DBG] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-03T19:26:27.114 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:26:26 smithi089 bash[33173]: cephadm 2023-06-03T19:26:25.859822+0000 mgr.y (mgr.14146) 101 : cephadm [INF] Deploying daemon osd.1 on smithi089 2023-06-03T19:26:29.030 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:26:28 smithi089 bash[26572]: cluster 2023-06-03T19:26:27.718803+0000 mgr.y (mgr.14146) 102 : cluster [DBG] pgmap v76: 0 pgs: ; 0 B data, 4.7 MiB used, 89 GiB / 89 GiB avail 2023-06-03T19:26:29.030 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:26:28 smithi089 bash[33173]: cluster 2023-06-03T19:26:27.718803+0000 mgr.y (mgr.14146) 102 : cluster [DBG] pgmap v76: 0 pgs: ; 0 B data, 4.7 MiB used, 89 GiB / 89 GiB avail 2023-06-03T19:26:29.084 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:26:28 smithi164 bash[26855]: cluster 2023-06-03T19:26:27.718803+0000 mgr.y (mgr.14146) 102 : cluster [DBG] pgmap v76: 0 pgs: ; 0 B data, 4.7 MiB used, 89 GiB / 89 GiB avail 2023-06-03T19:26:31.084 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:26:30 smithi164 bash[26855]: cluster 2023-06-03T19:26:29.719299+0000 mgr.y (mgr.14146) 103 : cluster [DBG] pgmap v77: 0 pgs: ; 0 B data, 4.7 MiB used, 89 GiB / 89 GiB avail 2023-06-03T19:26:31.085 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:26:30 smithi164 bash[26855]: audit 2023-06-03T19:26:30.359374+0000 mon.a (mon.0) 267 : audit [INF] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' 2023-06-03T19:26:31.085 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:26:30 smithi164 bash[26855]: audit 2023-06-03T19:26:30.379587+0000 mon.a (mon.0) 268 : audit [DBG] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-03T19:26:31.279 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:26:30 smithi089 bash[33173]: cluster 2023-06-03T19:26:29.719299+0000 mgr.y (mgr.14146) 103 : cluster [DBG] pgmap v77: 0 pgs: ; 0 B data, 4.7 MiB used, 89 GiB / 89 GiB avail 2023-06-03T19:26:31.280 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:26:30 smithi089 bash[33173]: audit 2023-06-03T19:26:30.359374+0000 mon.a (mon.0) 267 : audit [INF] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' 2023-06-03T19:26:31.280 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:26:30 smithi089 bash[33173]: audit 2023-06-03T19:26:30.379587+0000 mon.a (mon.0) 268 : audit [DBG] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-03T19:26:31.280 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:26:30 smithi089 bash[26572]: cluster 2023-06-03T19:26:29.719299+0000 mgr.y (mgr.14146) 103 : cluster [DBG] pgmap v77: 0 pgs: ; 0 B data, 4.7 MiB used, 89 GiB / 89 GiB avail 2023-06-03T19:26:31.280 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:26:30 smithi089 bash[26572]: audit 2023-06-03T19:26:30.359374+0000 mon.a (mon.0) 267 : audit [INF] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' 2023-06-03T19:26:31.280 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:26:30 smithi089 bash[26572]: audit 2023-06-03T19:26:30.379587+0000 mon.a (mon.0) 268 : audit [DBG] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-03T19:26:32.779 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:26:32 smithi089 bash[26572]: cluster 2023-06-03T19:26:31.719698+0000 mgr.y (mgr.14146) 104 : cluster [DBG] pgmap v78: 0 pgs: ; 0 B data, 4.7 MiB used, 89 GiB / 89 GiB avail 2023-06-03T19:26:32.780 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:26:32 smithi089 bash[33173]: cluster 2023-06-03T19:26:31.719698+0000 mgr.y (mgr.14146) 104 : cluster [DBG] pgmap v78: 0 pgs: ; 0 B data, 4.7 MiB used, 89 GiB / 89 GiB avail 2023-06-03T19:26:32.834 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:26:32 smithi164 bash[26855]: cluster 2023-06-03T19:26:31.719698+0000 mgr.y (mgr.14146) 104 : cluster [DBG] pgmap v78: 0 pgs: ; 0 B data, 4.7 MiB used, 89 GiB / 89 GiB avail 2023-06-03T19:26:35.029 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:26:34 smithi089 bash[26572]: cluster 2023-06-03T19:26:33.720097+0000 mgr.y (mgr.14146) 105 : cluster [DBG] pgmap v79: 0 pgs: ; 0 B data, 4.7 MiB used, 89 GiB / 89 GiB avail 2023-06-03T19:26:35.030 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:26:34 smithi089 bash[33173]: cluster 2023-06-03T19:26:33.720097+0000 mgr.y (mgr.14146) 105 : cluster [DBG] pgmap v79: 0 pgs: ; 0 B data, 4.7 MiB used, 89 GiB / 89 GiB avail 2023-06-03T19:26:35.084 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:26:34 smithi164 bash[26855]: cluster 2023-06-03T19:26:33.720097+0000 mgr.y (mgr.14146) 105 : cluster [DBG] pgmap v79: 0 pgs: ; 0 B data, 4.7 MiB used, 89 GiB / 89 GiB avail 2023-06-03T19:26:37.084 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:26:36 smithi164 bash[26855]: cluster 2023-06-03T19:26:35.720572+0000 mgr.y (mgr.14146) 106 : cluster [DBG] pgmap v80: 0 pgs: ; 0 B data, 4.7 MiB used, 89 GiB / 89 GiB avail 2023-06-03T19:26:37.279 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:26:36 smithi089 bash[26572]: cluster 2023-06-03T19:26:35.720572+0000 mgr.y (mgr.14146) 106 : cluster [DBG] pgmap v80: 0 pgs: ; 0 B data, 4.7 MiB used, 89 GiB / 89 GiB avail 2023-06-03T19:26:37.280 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:26:36 smithi089 bash[33173]: cluster 2023-06-03T19:26:35.720572+0000 mgr.y (mgr.14146) 106 : cluster [DBG] pgmap v80: 0 pgs: ; 0 B data, 4.7 MiB used, 89 GiB / 89 GiB avail 2023-06-03T19:26:38.084 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:26:37 smithi164 bash[26855]: audit 2023-06-03T19:26:36.791451+0000 mon.a (mon.0) 269 : audit [INF] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' 2023-06-03T19:26:38.085 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:26:37 smithi164 bash[26855]: audit 2023-06-03T19:26:37.074103+0000 mon.a (mon.0) 270 : audit [DBG] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-03T19:26:38.085 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:26:37 smithi164 bash[26855]: audit 2023-06-03T19:26:37.075267+0000 mon.a (mon.0) 271 : audit [INF] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-03T19:26:38.085 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:26:37 smithi164 bash[26855]: audit 2023-06-03T19:26:37.083737+0000 mon.a (mon.0) 272 : audit [INF] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' 2023-06-03T19:26:38.085 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:26:37 smithi164 bash[26855]: audit 2023-06-03T19:26:37.572700+0000 mon.c (mon.2) 6 : audit [INF] from='osd.1 [v2:172.21.15.89:6810/4020168282,v1:172.21.15.89:6811/4020168282]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2023-06-03T19:26:38.085 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:26:37 smithi164 bash[26855]: audit 2023-06-03T19:26:37.573327+0000 mon.a (mon.0) 273 : audit [INF] from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2023-06-03T19:26:38.280 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:26:37 smithi089 bash[26572]: audit 2023-06-03T19:26:36.791451+0000 mon.a (mon.0) 269 : audit [INF] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' 2023-06-03T19:26:38.280 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:26:37 smithi089 bash[26572]: audit 2023-06-03T19:26:37.074103+0000 mon.a (mon.0) 270 : audit [DBG] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-03T19:26:38.280 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:26:37 smithi089 bash[26572]: audit 2023-06-03T19:26:37.075267+0000 mon.a (mon.0) 271 : audit [INF] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-03T19:26:38.280 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:26:37 smithi089 bash[26572]: audit 2023-06-03T19:26:37.083737+0000 mon.a (mon.0) 272 : audit [INF] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' 2023-06-03T19:26:38.280 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:26:37 smithi089 bash[26572]: audit 2023-06-03T19:26:37.572700+0000 mon.c (mon.2) 6 : audit [INF] from='osd.1 [v2:172.21.15.89:6810/4020168282,v1:172.21.15.89:6811/4020168282]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2023-06-03T19:26:38.281 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:26:37 smithi089 bash[26572]: audit 2023-06-03T19:26:37.573327+0000 mon.a (mon.0) 273 : audit [INF] from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2023-06-03T19:26:38.281 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:26:37 smithi089 bash[33173]: audit 2023-06-03T19:26:36.791451+0000 mon.a (mon.0) 269 : audit [INF] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' 2023-06-03T19:26:38.281 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:26:37 smithi089 bash[33173]: audit 2023-06-03T19:26:37.074103+0000 mon.a (mon.0) 270 : audit [DBG] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-03T19:26:38.281 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:26:37 smithi089 bash[33173]: audit 2023-06-03T19:26:37.075267+0000 mon.a (mon.0) 271 : audit [INF] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-03T19:26:38.281 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:26:37 smithi089 bash[33173]: audit 2023-06-03T19:26:37.083737+0000 mon.a (mon.0) 272 : audit [INF] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' 2023-06-03T19:26:38.281 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:26:37 smithi089 bash[33173]: audit 2023-06-03T19:26:37.572700+0000 mon.c (mon.2) 6 : audit [INF] from='osd.1 [v2:172.21.15.89:6810/4020168282,v1:172.21.15.89:6811/4020168282]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2023-06-03T19:26:38.282 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:26:37 smithi089 bash[33173]: audit 2023-06-03T19:26:37.573327+0000 mon.a (mon.0) 273 : audit [INF] from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2023-06-03T19:26:39.367 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:26:39 smithi089 bash[26572]: cluster 2023-06-03T19:26:37.721104+0000 mgr.y (mgr.14146) 107 : cluster [DBG] pgmap v81: 0 pgs: ; 0 B data, 4.7 MiB used, 89 GiB / 89 GiB avail 2023-06-03T19:26:39.367 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:26:39 smithi089 bash[26572]: audit 2023-06-03T19:26:38.093793+0000 mon.a (mon.0) 274 : audit [INF] from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2023-06-03T19:26:39.368 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:26:39 smithi089 bash[26572]: cluster 2023-06-03T19:26:38.093916+0000 mon.a (mon.0) 275 : cluster [DBG] osdmap e11: 2 total, 1 up, 2 in 2023-06-03T19:26:39.368 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:26:39 smithi089 bash[26572]: audit 2023-06-03T19:26:38.094055+0000 mon.a (mon.0) 276 : audit [DBG] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-06-03T19:26:39.368 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:26:39 smithi089 bash[26572]: audit 2023-06-03T19:26:38.094872+0000 mon.c (mon.2) 7 : audit [INF] from='osd.1 [v2:172.21.15.89:6810/4020168282,v1:172.21.15.89:6811/4020168282]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi089", "root=default"]}]: dispatch 2023-06-03T19:26:39.368 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:26:39 smithi089 bash[26572]: audit 2023-06-03T19:26:38.095275+0000 mon.a (mon.0) 277 : audit [INF] from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi089", "root=default"]}]: dispatch 2023-06-03T19:26:39.368 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:26:39 smithi089 bash[33173]: cluster 2023-06-03T19:26:37.721104+0000 mgr.y (mgr.14146) 107 : cluster [DBG] pgmap v81: 0 pgs: ; 0 B data, 4.7 MiB used, 89 GiB / 89 GiB avail 2023-06-03T19:26:39.369 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:26:39 smithi089 bash[33173]: audit 2023-06-03T19:26:38.093793+0000 mon.a (mon.0) 274 : audit [INF] from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2023-06-03T19:26:39.369 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:26:39 smithi089 bash[33173]: cluster 2023-06-03T19:26:38.093916+0000 mon.a (mon.0) 275 : cluster [DBG] osdmap e11: 2 total, 1 up, 2 in 2023-06-03T19:26:39.369 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:26:39 smithi089 bash[33173]: audit 2023-06-03T19:26:38.094055+0000 mon.a (mon.0) 276 : audit [DBG] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-06-03T19:26:39.369 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:26:39 smithi089 bash[33173]: audit 2023-06-03T19:26:38.094872+0000 mon.c (mon.2) 7 : audit [INF] from='osd.1 [v2:172.21.15.89:6810/4020168282,v1:172.21.15.89:6811/4020168282]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi089", "root=default"]}]: dispatch 2023-06-03T19:26:39.369 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:26:39 smithi089 bash[33173]: audit 2023-06-03T19:26:38.095275+0000 mon.a (mon.0) 277 : audit [INF] from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi089", "root=default"]}]: dispatch 2023-06-03T19:26:39.585 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:26:39 smithi164 bash[26855]: cluster 2023-06-03T19:26:37.721104+0000 mgr.y (mgr.14146) 107 : cluster [DBG] pgmap v81: 0 pgs: ; 0 B data, 4.7 MiB used, 89 GiB / 89 GiB avail 2023-06-03T19:26:39.585 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:26:39 smithi164 bash[26855]: audit 2023-06-03T19:26:38.093793+0000 mon.a (mon.0) 274 : audit [INF] from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2023-06-03T19:26:39.585 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:26:39 smithi164 bash[26855]: cluster 2023-06-03T19:26:38.093916+0000 mon.a (mon.0) 275 : cluster [DBG] osdmap e11: 2 total, 1 up, 2 in 2023-06-03T19:26:39.585 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:26:39 smithi164 bash[26855]: audit 2023-06-03T19:26:38.094055+0000 mon.a (mon.0) 276 : audit [DBG] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-06-03T19:26:39.585 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:26:39 smithi164 bash[26855]: audit 2023-06-03T19:26:38.094872+0000 mon.c (mon.2) 7 : audit [INF] from='osd.1 [v2:172.21.15.89:6810/4020168282,v1:172.21.15.89:6811/4020168282]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi089", "root=default"]}]: dispatch 2023-06-03T19:26:39.586 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:26:39 smithi164 bash[26855]: audit 2023-06-03T19:26:38.095275+0000 mon.a (mon.0) 277 : audit [INF] from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi089", "root=default"]}]: dispatch 2023-06-03T19:26:40.279 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:26:40 smithi089 bash[33173]: audit 2023-06-03T19:26:39.093002+0000 mon.a (mon.0) 278 : audit [INF] from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi089", "root=default"]}]': finished 2023-06-03T19:26:40.280 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:26:40 smithi089 bash[33173]: cluster 2023-06-03T19:26:39.093126+0000 mon.a (mon.0) 279 : cluster [DBG] osdmap e12: 2 total, 1 up, 2 in 2023-06-03T19:26:40.280 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:26:40 smithi089 bash[33173]: audit 2023-06-03T19:26:39.093674+0000 mon.a (mon.0) 280 : audit [DBG] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-06-03T19:26:40.280 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:26:40 smithi089 bash[33173]: audit 2023-06-03T19:26:39.102549+0000 mon.a (mon.0) 281 : audit [DBG] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-06-03T19:26:40.281 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:26:40 smithi089 bash[26572]: audit 2023-06-03T19:26:39.093002+0000 mon.a (mon.0) 278 : audit [INF] from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi089", "root=default"]}]': finished 2023-06-03T19:26:40.281 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:26:40 smithi089 bash[26572]: cluster 2023-06-03T19:26:39.093126+0000 mon.a (mon.0) 279 : cluster [DBG] osdmap e12: 2 total, 1 up, 2 in 2023-06-03T19:26:40.281 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:26:40 smithi089 bash[26572]: audit 2023-06-03T19:26:39.093674+0000 mon.a (mon.0) 280 : audit [DBG] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-06-03T19:26:40.282 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:26:40 smithi089 bash[26572]: audit 2023-06-03T19:26:39.102549+0000 mon.a (mon.0) 281 : audit [DBG] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-06-03T19:26:40.584 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:26:40 smithi164 bash[26855]: audit 2023-06-03T19:26:39.093002+0000 mon.a (mon.0) 278 : audit [INF] from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi089", "root=default"]}]': finished 2023-06-03T19:26:40.585 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:26:40 smithi164 bash[26855]: cluster 2023-06-03T19:26:39.093126+0000 mon.a (mon.0) 279 : cluster [DBG] osdmap e12: 2 total, 1 up, 2 in 2023-06-03T19:26:40.585 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:26:40 smithi164 bash[26855]: audit 2023-06-03T19:26:39.093674+0000 mon.a (mon.0) 280 : audit [DBG] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-06-03T19:26:40.585 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:26:40 smithi164 bash[26855]: audit 2023-06-03T19:26:39.102549+0000 mon.a (mon.0) 281 : audit [DBG] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-06-03T19:26:40.709 INFO:teuthology.orchestra.run.smithi089.stdout:Created osd(s) 1 on host 'smithi089' 2023-06-03T19:26:41.280 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:26:41 smithi089 bash[26572]: cluster 2023-06-03T19:26:38.554392+0000 osd.1 (osd.1) 1 : cluster [DBG] purged_snaps scrub starts 2023-06-03T19:26:41.280 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:26:41 smithi089 bash[26572]: cluster 2023-06-03T19:26:38.554455+0000 osd.1 (osd.1) 2 : cluster [DBG] purged_snaps scrub ok 2023-06-03T19:26:41.280 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:26:41 smithi089 bash[26572]: cluster 2023-06-03T19:26:39.721474+0000 mgr.y (mgr.14146) 108 : cluster [DBG] pgmap v84: 0 pgs: ; 0 B data, 4.7 MiB used, 89 GiB / 89 GiB avail 2023-06-03T19:26:41.281 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:26:41 smithi089 bash[26572]: audit 2023-06-03T19:26:40.106391+0000 mon.a (mon.0) 282 : audit [DBG] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-06-03T19:26:41.282 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:26:41 smithi089 bash[26572]: cluster 2023-06-03T19:26:40.110313+0000 mon.a (mon.0) 283 : cluster [INF] osd.1 [v2:172.21.15.89:6810/4020168282,v1:172.21.15.89:6811/4020168282] boot 2023-06-03T19:26:41.282 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:26:41 smithi089 bash[26572]: cluster 2023-06-03T19:26:40.110383+0000 mon.a (mon.0) 284 : cluster [DBG] osdmap e13: 2 total, 2 up, 2 in 2023-06-03T19:26:41.282 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:26:41 smithi089 bash[26572]: audit 2023-06-03T19:26:40.110569+0000 mon.a (mon.0) 285 : audit [DBG] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-06-03T19:26:41.283 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:26:41 smithi089 bash[26572]: audit 2023-06-03T19:26:40.703243+0000 mon.a (mon.0) 286 : audit [INF] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' 2023-06-03T19:26:41.283 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:26:41 smithi089 bash[26572]: audit 2023-06-03T19:26:40.709031+0000 mon.a (mon.0) 287 : audit [DBG] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-03T19:26:41.284 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:26:41 smithi089 bash[33173]: cluster 2023-06-03T19:26:38.554392+0000 osd.1 (osd.1) 1 : cluster [DBG] purged_snaps scrub starts 2023-06-03T19:26:41.284 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:26:41 smithi089 bash[33173]: cluster 2023-06-03T19:26:38.554455+0000 osd.1 (osd.1) 2 : cluster [DBG] purged_snaps scrub ok 2023-06-03T19:26:41.285 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:26:41 smithi089 bash[33173]: cluster 2023-06-03T19:26:39.721474+0000 mgr.y (mgr.14146) 108 : cluster [DBG] pgmap v84: 0 pgs: ; 0 B data, 4.7 MiB used, 89 GiB / 89 GiB avail 2023-06-03T19:26:41.285 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:26:41 smithi089 bash[33173]: audit 2023-06-03T19:26:40.106391+0000 mon.a (mon.0) 282 : audit [DBG] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-06-03T19:26:41.285 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:26:41 smithi089 bash[33173]: cluster 2023-06-03T19:26:40.110313+0000 mon.a (mon.0) 283 : cluster [INF] osd.1 [v2:172.21.15.89:6810/4020168282,v1:172.21.15.89:6811/4020168282] boot 2023-06-03T19:26:41.285 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:26:41 smithi089 bash[33173]: cluster 2023-06-03T19:26:40.110383+0000 mon.a (mon.0) 284 : cluster [DBG] osdmap e13: 2 total, 2 up, 2 in 2023-06-03T19:26:41.286 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:26:41 smithi089 bash[33173]: audit 2023-06-03T19:26:40.110569+0000 mon.a (mon.0) 285 : audit [DBG] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-06-03T19:26:41.287 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:26:41 smithi089 bash[33173]: audit 2023-06-03T19:26:40.703243+0000 mon.a (mon.0) 286 : audit [INF] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' 2023-06-03T19:26:41.287 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:26:41 smithi089 bash[33173]: audit 2023-06-03T19:26:40.709031+0000 mon.a (mon.0) 287 : audit [DBG] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-03T19:26:41.433 DEBUG:teuthology.orchestra.run.smithi089:osd.1> sudo journalctl -f -n 0 -u ceph-f1db3478-0243-11ee-9b1d-001a4aab830c@osd.1.service 2023-06-03T19:26:41.479 INFO:tasks.cephadm:Deploying osd.2 on smithi089 with /dev/vg_nvme/lv_2... 2023-06-03T19:26:41.480 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph-volume lvm zap /dev/vg_nvme/lv_2 2023-06-03T19:26:41.585 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:26:41 smithi164 bash[26855]: cluster 2023-06-03T19:26:38.554392+0000 osd.1 (osd.1) 1 : cluster [DBG] purged_snaps scrub starts 2023-06-03T19:26:41.585 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:26:41 smithi164 bash[26855]: cluster 2023-06-03T19:26:38.554455+0000 osd.1 (osd.1) 2 : cluster [DBG] purged_snaps scrub ok 2023-06-03T19:26:41.585 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:26:41 smithi164 bash[26855]: cluster 2023-06-03T19:26:39.721474+0000 mgr.y (mgr.14146) 108 : cluster [DBG] pgmap v84: 0 pgs: ; 0 B data, 4.7 MiB used, 89 GiB / 89 GiB avail 2023-06-03T19:26:41.585 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:26:41 smithi164 bash[26855]: audit 2023-06-03T19:26:40.106391+0000 mon.a (mon.0) 282 : audit [DBG] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-06-03T19:26:41.586 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:26:41 smithi164 bash[26855]: cluster 2023-06-03T19:26:40.110313+0000 mon.a (mon.0) 283 : cluster [INF] osd.1 [v2:172.21.15.89:6810/4020168282,v1:172.21.15.89:6811/4020168282] boot 2023-06-03T19:26:41.586 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:26:41 smithi164 bash[26855]: cluster 2023-06-03T19:26:40.110383+0000 mon.a (mon.0) 284 : cluster [DBG] osdmap e13: 2 total, 2 up, 2 in 2023-06-03T19:26:41.586 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:26:41 smithi164 bash[26855]: audit 2023-06-03T19:26:40.110569+0000 mon.a (mon.0) 285 : audit [DBG] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-06-03T19:26:41.586 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:26:41 smithi164 bash[26855]: audit 2023-06-03T19:26:40.703243+0000 mon.a (mon.0) 286 : audit [INF] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' 2023-06-03T19:26:41.586 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:26:41 smithi164 bash[26855]: audit 2023-06-03T19:26:40.709031+0000 mon.a (mon.0) 287 : audit [DBG] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-03T19:26:41.679 INFO:journalctl@ceph.osd.1.smithi089.stdout:-- Logs begin at Sun 2022-08-14 12:20:24 UTC. -- 2023-06-03T19:26:42.529 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:26:42 smithi089 bash[26572]: cluster 2023-06-03T19:26:41.111593+0000 mon.a (mon.0) 288 : cluster [DBG] osdmap e14: 2 total, 2 up, 2 in 2023-06-03T19:26:42.530 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:26:42 smithi089 bash[26572]: cluster 2023-06-03T19:26:41.721873+0000 mgr.y (mgr.14146) 109 : cluster [DBG] pgmap v87: 0 pgs: ; 0 B data, 9.5 MiB used, 179 GiB / 179 GiB avail 2023-06-03T19:26:42.530 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:26:42 smithi089 bash[33173]: cluster 2023-06-03T19:26:41.111593+0000 mon.a (mon.0) 288 : cluster [DBG] osdmap e14: 2 total, 2 up, 2 in 2023-06-03T19:26:42.530 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:26:42 smithi089 bash[33173]: cluster 2023-06-03T19:26:41.721873+0000 mgr.y (mgr.14146) 109 : cluster [DBG] pgmap v87: 0 pgs: ; 0 B data, 9.5 MiB used, 179 GiB / 179 GiB avail 2023-06-03T19:26:42.585 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:26:42 smithi164 bash[26855]: cluster 2023-06-03T19:26:41.111593+0000 mon.a (mon.0) 288 : cluster [DBG] osdmap e14: 2 total, 2 up, 2 in 2023-06-03T19:26:42.585 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:26:42 smithi164 bash[26855]: cluster 2023-06-03T19:26:41.721873+0000 mgr.y (mgr.14146) 109 : cluster [DBG] pgmap v87: 0 pgs: ; 0 B data, 9.5 MiB used, 179 GiB / 179 GiB avail 2023-06-03T19:26:45.084 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:26:44 smithi164 bash[26855]: cluster 2023-06-03T19:26:43.722387+0000 mgr.y (mgr.14146) 110 : cluster [DBG] pgmap v88: 0 pgs: ; 0 B data, 9.5 MiB used, 179 GiB / 179 GiB avail 2023-06-03T19:26:45.151 INFO:teuthology.orchestra.run.smithi089.stderr:--> Zapping: /dev/vg_nvme/lv_2 2023-06-03T19:26:45.151 INFO:teuthology.orchestra.run.smithi089.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_2 bs=1M count=10 conv=fsync 2023-06-03T19:26:45.151 INFO:teuthology.orchestra.run.smithi089.stderr: stderr: 10+0 records in 2023-06-03T19:26:45.152 INFO:teuthology.orchestra.run.smithi089.stderr:10+0 records out 2023-06-03T19:26:45.152 INFO:teuthology.orchestra.run.smithi089.stderr:10485760 bytes (10 MB, 10 MiB) copied, 0.0145601 s, 720 MB/s 2023-06-03T19:26:45.152 INFO:teuthology.orchestra.run.smithi089.stderr:--> Zapping successful for: 2023-06-03T19:26:45.190 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:26:44 smithi089 bash[33173]: cluster 2023-06-03T19:26:43.722387+0000 mgr.y (mgr.14146) 110 : cluster [DBG] pgmap v88: 0 pgs: ; 0 B data, 9.5 MiB used, 179 GiB / 179 GiB avail 2023-06-03T19:26:45.190 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:26:44 smithi089 bash[26572]: cluster 2023-06-03T19:26:43.722387+0000 mgr.y (mgr.14146) 110 : cluster [DBG] pgmap v88: 0 pgs: ; 0 B data, 9.5 MiB used, 179 GiB / 179 GiB avail 2023-06-03T19:26:45.828 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph orch daemon add osd smithi089:vg_nvme/lv_2 2023-06-03T19:26:47.030 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:26:46 smithi089 bash[33173]: cluster 2023-06-03T19:26:45.722873+0000 mgr.y (mgr.14146) 111 : cluster [DBG] pgmap v89: 0 pgs: ; 0 B data, 9.6 MiB used, 179 GiB / 179 GiB avail 2023-06-03T19:26:47.030 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:26:46 smithi089 bash[26572]: cluster 2023-06-03T19:26:45.722873+0000 mgr.y (mgr.14146) 111 : cluster [DBG] pgmap v89: 0 pgs: ; 0 B data, 9.6 MiB used, 179 GiB / 179 GiB avail 2023-06-03T19:26:47.084 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:26:46 smithi164 bash[26855]: cluster 2023-06-03T19:26:45.722873+0000 mgr.y (mgr.14146) 111 : cluster [DBG] pgmap v89: 0 pgs: ; 0 B data, 9.6 MiB used, 179 GiB / 179 GiB avail 2023-06-03T19:26:49.084 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:26:48 smithi164 bash[26855]: cluster 2023-06-03T19:26:47.723501+0000 mgr.y (mgr.14146) 112 : cluster [DBG] pgmap v90: 0 pgs: ; 0 B data, 9.6 MiB used, 179 GiB / 179 GiB avail 2023-06-03T19:26:49.279 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:26:48 smithi089 bash[26572]: cluster 2023-06-03T19:26:47.723501+0000 mgr.y (mgr.14146) 112 : cluster [DBG] pgmap v90: 0 pgs: ; 0 B data, 9.6 MiB used, 179 GiB / 179 GiB avail 2023-06-03T19:26:49.280 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:26:48 smithi089 bash[33173]: cluster 2023-06-03T19:26:47.723501+0000 mgr.y (mgr.14146) 112 : cluster [DBG] pgmap v90: 0 pgs: ; 0 B data, 9.6 MiB used, 179 GiB / 179 GiB avail 2023-06-03T19:26:51.030 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:26:50 smithi089 bash[26572]: cluster 2023-06-03T19:26:49.724083+0000 mgr.y (mgr.14146) 113 : cluster [DBG] pgmap v91: 0 pgs: ; 0 B data, 9.6 MiB used, 179 GiB / 179 GiB avail 2023-06-03T19:26:51.030 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:26:50 smithi089 bash[33173]: cluster 2023-06-03T19:26:49.724083+0000 mgr.y (mgr.14146) 113 : cluster [DBG] pgmap v91: 0 pgs: ; 0 B data, 9.6 MiB used, 179 GiB / 179 GiB avail 2023-06-03T19:26:51.092 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:26:50 smithi164 bash[26855]: cluster 2023-06-03T19:26:49.724083+0000 mgr.y (mgr.14146) 113 : cluster [DBG] pgmap v91: 0 pgs: ; 0 B data, 9.6 MiB used, 179 GiB / 179 GiB avail 2023-06-03T19:26:52.030 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:26:51 smithi089 bash[26572]: audit 2023-06-03T19:26:51.192969+0000 mon.a (mon.0) 289 : audit [DBG] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-06-03T19:26:52.030 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:26:51 smithi089 bash[26572]: audit 2023-06-03T19:26:51.195807+0000 mon.a (mon.0) 290 : audit [INF] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-06-03T19:26:52.030 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:26:51 smithi089 bash[26572]: audit 2023-06-03T19:26:51.196828+0000 mon.a (mon.0) 291 : audit [DBG] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-03T19:26:52.031 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:26:51 smithi089 bash[33173]: audit 2023-06-03T19:26:51.192969+0000 mon.a (mon.0) 289 : audit [DBG] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-06-03T19:26:52.031 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:26:51 smithi089 bash[33173]: audit 2023-06-03T19:26:51.195807+0000 mon.a (mon.0) 290 : audit [INF] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-06-03T19:26:52.031 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:26:51 smithi089 bash[33173]: audit 2023-06-03T19:26:51.196828+0000 mon.a (mon.0) 291 : audit [DBG] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-03T19:26:52.084 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:26:51 smithi164 bash[26855]: audit 2023-06-03T19:26:51.192969+0000 mon.a (mon.0) 289 : audit [DBG] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-06-03T19:26:52.085 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:26:51 smithi164 bash[26855]: audit 2023-06-03T19:26:51.195807+0000 mon.a (mon.0) 290 : audit [INF] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-06-03T19:26:52.085 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:26:51 smithi164 bash[26855]: audit 2023-06-03T19:26:51.196828+0000 mon.a (mon.0) 291 : audit [DBG] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-03T19:26:53.085 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:26:52 smithi164 bash[26855]: audit 2023-06-03T19:26:51.187676+0000 mgr.y (mgr.14146) 114 : audit [DBG] from='client.14262 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi089:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2023-06-03T19:26:53.085 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:26:52 smithi164 bash[26855]: cluster 2023-06-03T19:26:51.724460+0000 mgr.y (mgr.14146) 115 : cluster [DBG] pgmap v92: 0 pgs: ; 0 B data, 9.6 MiB used, 179 GiB / 179 GiB avail 2023-06-03T19:26:53.085 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:26:52 smithi164 bash[26855]: audit 2023-06-03T19:26:52.501616+0000 mon.a (mon.0) 292 : audit [INF] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' 2023-06-03T19:26:53.086 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:26:52 smithi164 bash[26855]: audit 2023-06-03T19:26:52.503102+0000 mon.a (mon.0) 293 : audit [INF] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-06-03T19:26:53.086 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:26:52 smithi164 bash[26855]: audit 2023-06-03T19:26:52.504134+0000 mon.a (mon.0) 294 : audit [INF] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-06-03T19:26:53.086 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:26:52 smithi164 bash[26855]: audit 2023-06-03T19:26:52.508497+0000 mon.a (mon.0) 295 : audit [INF] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' 2023-06-03T19:26:53.086 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:26:52 smithi164 bash[26855]: audit 2023-06-03T19:26:52.509524+0000 mon.a (mon.0) 296 : audit [DBG] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-03T19:26:53.087 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:26:52 smithi164 bash[26855]: audit 2023-06-03T19:26:52.510487+0000 mon.a (mon.0) 297 : audit [INF] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-03T19:26:53.087 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:26:52 smithi164 bash[26855]: audit 2023-06-03T19:26:52.516538+0000 mon.a (mon.0) 298 : audit [INF] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' 2023-06-03T19:26:53.280 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:26:52 smithi089 bash[26572]: audit 2023-06-03T19:26:51.187676+0000 mgr.y (mgr.14146) 114 : audit [DBG] from='client.14262 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi089:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2023-06-03T19:26:53.280 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:26:52 smithi089 bash[26572]: cluster 2023-06-03T19:26:51.724460+0000 mgr.y (mgr.14146) 115 : cluster [DBG] pgmap v92: 0 pgs: ; 0 B data, 9.6 MiB used, 179 GiB / 179 GiB avail 2023-06-03T19:26:53.280 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:26:52 smithi089 bash[26572]: audit 2023-06-03T19:26:52.501616+0000 mon.a (mon.0) 292 : audit [INF] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' 2023-06-03T19:26:53.280 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:26:52 smithi089 bash[26572]: audit 2023-06-03T19:26:52.503102+0000 mon.a (mon.0) 293 : audit [INF] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-06-03T19:26:53.280 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:26:52 smithi089 bash[26572]: audit 2023-06-03T19:26:52.504134+0000 mon.a (mon.0) 294 : audit [INF] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-06-03T19:26:53.281 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:26:52 smithi089 bash[26572]: audit 2023-06-03T19:26:52.508497+0000 mon.a (mon.0) 295 : audit [INF] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' 2023-06-03T19:26:53.281 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:26:52 smithi089 bash[26572]: audit 2023-06-03T19:26:52.509524+0000 mon.a (mon.0) 296 : audit [DBG] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-03T19:26:53.281 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:26:52 smithi089 bash[26572]: audit 2023-06-03T19:26:52.510487+0000 mon.a (mon.0) 297 : audit [INF] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-03T19:26:53.281 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:26:52 smithi089 bash[26572]: audit 2023-06-03T19:26:52.516538+0000 mon.a (mon.0) 298 : audit [INF] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' 2023-06-03T19:26:53.281 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:26:52 smithi089 bash[33173]: audit 2023-06-03T19:26:51.187676+0000 mgr.y (mgr.14146) 114 : audit [DBG] from='client.14262 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi089:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2023-06-03T19:26:53.281 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:26:52 smithi089 bash[33173]: cluster 2023-06-03T19:26:51.724460+0000 mgr.y (mgr.14146) 115 : cluster [DBG] pgmap v92: 0 pgs: ; 0 B data, 9.6 MiB used, 179 GiB / 179 GiB avail 2023-06-03T19:26:53.282 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:26:52 smithi089 bash[33173]: audit 2023-06-03T19:26:52.501616+0000 mon.a (mon.0) 292 : audit [INF] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' 2023-06-03T19:26:53.282 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:26:52 smithi089 bash[33173]: audit 2023-06-03T19:26:52.503102+0000 mon.a (mon.0) 293 : audit [INF] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-06-03T19:26:53.282 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:26:52 smithi089 bash[33173]: audit 2023-06-03T19:26:52.504134+0000 mon.a (mon.0) 294 : audit [INF] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-06-03T19:26:53.282 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:26:52 smithi089 bash[33173]: audit 2023-06-03T19:26:52.508497+0000 mon.a (mon.0) 295 : audit [INF] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' 2023-06-03T19:26:53.282 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:26:52 smithi089 bash[33173]: audit 2023-06-03T19:26:52.509524+0000 mon.a (mon.0) 296 : audit [DBG] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-03T19:26:53.282 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:26:52 smithi089 bash[33173]: audit 2023-06-03T19:26:52.510487+0000 mon.a (mon.0) 297 : audit [INF] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-03T19:26:53.282 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:26:52 smithi089 bash[33173]: audit 2023-06-03T19:26:52.516538+0000 mon.a (mon.0) 298 : audit [INF] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' 2023-06-03T19:26:54.086 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:26:53 smithi164 bash[26855]: cephadm 2023-06-03T19:26:52.495261+0000 mgr.y (mgr.14146) 116 : cephadm [INF] Detected new or changed devices on smithi089 2023-06-03T19:26:54.087 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:26:53 smithi164 bash[26855]: cephadm 2023-06-03T19:26:52.504891+0000 mgr.y (mgr.14146) 117 : cephadm [INF] Adjusting osd_memory_target on smithi089 to 8124M 2023-06-03T19:26:54.280 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:26:53 smithi089 bash[26572]: cephadm 2023-06-03T19:26:52.495261+0000 mgr.y (mgr.14146) 116 : cephadm [INF] Detected new or changed devices on smithi089 2023-06-03T19:26:54.280 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:26:53 smithi089 bash[26572]: cephadm 2023-06-03T19:26:52.504891+0000 mgr.y (mgr.14146) 117 : cephadm [INF] Adjusting osd_memory_target on smithi089 to 8124M 2023-06-03T19:26:54.280 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:26:53 smithi089 bash[33173]: cephadm 2023-06-03T19:26:52.495261+0000 mgr.y (mgr.14146) 116 : cephadm [INF] Detected new or changed devices on smithi089 2023-06-03T19:26:54.280 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:26:53 smithi089 bash[33173]: cephadm 2023-06-03T19:26:52.504891+0000 mgr.y (mgr.14146) 117 : cephadm [INF] Adjusting osd_memory_target on smithi089 to 8124M 2023-06-03T19:26:55.085 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:26:54 smithi164 bash[26855]: cluster 2023-06-03T19:26:53.724895+0000 mgr.y (mgr.14146) 118 : cluster [DBG] pgmap v93: 0 pgs: ; 0 B data, 9.6 MiB used, 179 GiB / 179 GiB avail 2023-06-03T19:26:55.280 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:26:54 smithi089 bash[26572]: cluster 2023-06-03T19:26:53.724895+0000 mgr.y (mgr.14146) 118 : cluster [DBG] pgmap v93: 0 pgs: ; 0 B data, 9.6 MiB used, 179 GiB / 179 GiB avail 2023-06-03T19:26:55.280 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:26:54 smithi089 bash[33173]: cluster 2023-06-03T19:26:53.724895+0000 mgr.y (mgr.14146) 118 : cluster [DBG] pgmap v93: 0 pgs: ; 0 B data, 9.6 MiB used, 179 GiB / 179 GiB avail 2023-06-03T19:26:57.085 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:26:56 smithi164 bash[26855]: cluster 2023-06-03T19:26:55.725379+0000 mgr.y (mgr.14146) 119 : cluster [DBG] pgmap v94: 0 pgs: ; 0 B data, 9.6 MiB used, 179 GiB / 179 GiB avail 2023-06-03T19:26:57.163 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:26:56 smithi089 bash[33173]: cluster 2023-06-03T19:26:55.725379+0000 mgr.y (mgr.14146) 119 : cluster [DBG] pgmap v94: 0 pgs: ; 0 B data, 9.6 MiB used, 179 GiB / 179 GiB avail 2023-06-03T19:26:57.164 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:26:56 smithi089 bash[26572]: cluster 2023-06-03T19:26:55.725379+0000 mgr.y (mgr.14146) 119 : cluster [DBG] pgmap v94: 0 pgs: ; 0 B data, 9.6 MiB used, 179 GiB / 179 GiB avail 2023-06-03T19:26:59.029 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:26:58 smithi089 bash[33173]: cluster 2023-06-03T19:26:57.725894+0000 mgr.y (mgr.14146) 120 : cluster [DBG] pgmap v95: 0 pgs: ; 0 B data, 9.6 MiB used, 179 GiB / 179 GiB avail 2023-06-03T19:26:59.030 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:26:58 smithi089 bash[26572]: cluster 2023-06-03T19:26:57.725894+0000 mgr.y (mgr.14146) 120 : cluster [DBG] pgmap v95: 0 pgs: ; 0 B data, 9.6 MiB used, 179 GiB / 179 GiB avail 2023-06-03T19:26:59.085 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:26:58 smithi164 bash[26855]: cluster 2023-06-03T19:26:57.725894+0000 mgr.y (mgr.14146) 120 : cluster [DBG] pgmap v95: 0 pgs: ; 0 B data, 9.6 MiB used, 179 GiB / 179 GiB avail 2023-06-03T19:27:00.085 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:26:59 smithi164 bash[26855]: audit 2023-06-03T19:26:59.637736+0000 mon.a (mon.0) 299 : audit [INF] from='client.? 172.21.15.89:0/2428086503' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "d2d2c641-392a-444f-81bc-84eef6340f30"}]: dispatch 2023-06-03T19:27:00.085 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:26:59 smithi164 bash[26855]: audit 2023-06-03T19:26:59.644676+0000 mon.a (mon.0) 300 : audit [INF] from='client.? 172.21.15.89:0/2428086503' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "d2d2c641-392a-444f-81bc-84eef6340f30"}]': finished 2023-06-03T19:27:00.085 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:26:59 smithi164 bash[26855]: cluster 2023-06-03T19:26:59.644861+0000 mon.a (mon.0) 301 : cluster [DBG] osdmap e15: 3 total, 2 up, 3 in 2023-06-03T19:27:00.085 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:26:59 smithi164 bash[26855]: audit 2023-06-03T19:26:59.645066+0000 mon.a (mon.0) 302 : audit [DBG] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-06-03T19:27:00.280 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:26:59 smithi089 bash[26572]: audit 2023-06-03T19:26:59.637736+0000 mon.a (mon.0) 299 : audit [INF] from='client.? 172.21.15.89:0/2428086503' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "d2d2c641-392a-444f-81bc-84eef6340f30"}]: dispatch 2023-06-03T19:27:00.280 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:26:59 smithi089 bash[26572]: audit 2023-06-03T19:26:59.644676+0000 mon.a (mon.0) 300 : audit [INF] from='client.? 172.21.15.89:0/2428086503' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "d2d2c641-392a-444f-81bc-84eef6340f30"}]': finished 2023-06-03T19:27:00.280 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:26:59 smithi089 bash[26572]: cluster 2023-06-03T19:26:59.644861+0000 mon.a (mon.0) 301 : cluster [DBG] osdmap e15: 3 total, 2 up, 3 in 2023-06-03T19:27:00.280 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:26:59 smithi089 bash[26572]: audit 2023-06-03T19:26:59.645066+0000 mon.a (mon.0) 302 : audit [DBG] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-06-03T19:27:00.280 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:26:59 smithi089 bash[33173]: audit 2023-06-03T19:26:59.637736+0000 mon.a (mon.0) 299 : audit [INF] from='client.? 172.21.15.89:0/2428086503' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "d2d2c641-392a-444f-81bc-84eef6340f30"}]: dispatch 2023-06-03T19:27:00.281 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:26:59 smithi089 bash[33173]: audit 2023-06-03T19:26:59.644676+0000 mon.a (mon.0) 300 : audit [INF] from='client.? 172.21.15.89:0/2428086503' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "d2d2c641-392a-444f-81bc-84eef6340f30"}]': finished 2023-06-03T19:27:00.281 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:26:59 smithi089 bash[33173]: cluster 2023-06-03T19:26:59.644861+0000 mon.a (mon.0) 301 : cluster [DBG] osdmap e15: 3 total, 2 up, 3 in 2023-06-03T19:27:00.281 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:26:59 smithi089 bash[33173]: audit 2023-06-03T19:26:59.645066+0000 mon.a (mon.0) 302 : audit [DBG] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-06-03T19:27:01.085 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:27:00 smithi164 bash[26855]: cluster 2023-06-03T19:26:59.726393+0000 mgr.y (mgr.14146) 121 : cluster [DBG] pgmap v97: 0 pgs: ; 0 B data, 9.6 MiB used, 179 GiB / 179 GiB avail 2023-06-03T19:27:01.085 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:27:00 smithi164 bash[26855]: audit 2023-06-03T19:27:00.388625+0000 mon.a (mon.0) 303 : audit [DBG] from='client.? 172.21.15.89:0/3515478194' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-06-03T19:27:01.280 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:27:00 smithi089 bash[26572]: cluster 2023-06-03T19:26:59.726393+0000 mgr.y (mgr.14146) 121 : cluster [DBG] pgmap v97: 0 pgs: ; 0 B data, 9.6 MiB used, 179 GiB / 179 GiB avail 2023-06-03T19:27:01.280 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:27:00 smithi089 bash[26572]: audit 2023-06-03T19:27:00.388625+0000 mon.a (mon.0) 303 : audit [DBG] from='client.? 172.21.15.89:0/3515478194' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-06-03T19:27:01.281 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:27:00 smithi089 bash[33173]: cluster 2023-06-03T19:26:59.726393+0000 mgr.y (mgr.14146) 121 : cluster [DBG] pgmap v97: 0 pgs: ; 0 B data, 9.6 MiB used, 179 GiB / 179 GiB avail 2023-06-03T19:27:01.281 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:27:00 smithi089 bash[33173]: audit 2023-06-03T19:27:00.388625+0000 mon.a (mon.0) 303 : audit [DBG] from='client.? 172.21.15.89:0/3515478194' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-06-03T19:27:02.780 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:27:02 smithi089 bash[26572]: cluster 2023-06-03T19:27:01.726891+0000 mgr.y (mgr.14146) 122 : cluster [DBG] pgmap v98: 0 pgs: ; 0 B data, 9.6 MiB used, 179 GiB / 179 GiB avail 2023-06-03T19:27:02.780 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:27:02 smithi089 bash[33173]: cluster 2023-06-03T19:27:01.726891+0000 mgr.y (mgr.14146) 122 : cluster [DBG] pgmap v98: 0 pgs: ; 0 B data, 9.6 MiB used, 179 GiB / 179 GiB avail 2023-06-03T19:27:02.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:27:02 smithi164 bash[26855]: cluster 2023-06-03T19:27:01.726891+0000 mgr.y (mgr.14146) 122 : cluster [DBG] pgmap v98: 0 pgs: ; 0 B data, 9.6 MiB used, 179 GiB / 179 GiB avail 2023-06-03T19:27:05.085 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:27:04 smithi164 bash[26855]: cluster 2023-06-03T19:27:03.727281+0000 mgr.y (mgr.14146) 123 : cluster [DBG] pgmap v99: 0 pgs: ; 0 B data, 9.6 MiB used, 179 GiB / 179 GiB avail 2023-06-03T19:27:05.281 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:27:04 smithi089 bash[26572]: cluster 2023-06-03T19:27:03.727281+0000 mgr.y (mgr.14146) 123 : cluster [DBG] pgmap v99: 0 pgs: ; 0 B data, 9.6 MiB used, 179 GiB / 179 GiB avail 2023-06-03T19:27:05.281 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:27:04 smithi089 bash[33173]: cluster 2023-06-03T19:27:03.727281+0000 mgr.y (mgr.14146) 123 : cluster [DBG] pgmap v99: 0 pgs: ; 0 B data, 9.6 MiB used, 179 GiB / 179 GiB avail 2023-06-03T19:27:07.086 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:27:06 smithi164 bash[26855]: cluster 2023-06-03T19:27:05.727812+0000 mgr.y (mgr.14146) 124 : cluster [DBG] pgmap v100: 0 pgs: ; 0 B data, 9.6 MiB used, 179 GiB / 179 GiB avail 2023-06-03T19:27:07.280 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:27:06 smithi089 bash[26572]: cluster 2023-06-03T19:27:05.727812+0000 mgr.y (mgr.14146) 124 : cluster [DBG] pgmap v100: 0 pgs: ; 0 B data, 9.6 MiB used, 179 GiB / 179 GiB avail 2023-06-03T19:27:07.281 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:27:06 smithi089 bash[33173]: cluster 2023-06-03T19:27:05.727812+0000 mgr.y (mgr.14146) 124 : cluster [DBG] pgmap v100: 0 pgs: ; 0 B data, 9.6 MiB used, 179 GiB / 179 GiB avail 2023-06-03T19:27:09.085 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:27:08 smithi164 bash[26855]: cluster 2023-06-03T19:27:07.728313+0000 mgr.y (mgr.14146) 125 : cluster [DBG] pgmap v101: 0 pgs: ; 0 B data, 9.6 MiB used, 179 GiB / 179 GiB avail 2023-06-03T19:27:09.280 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:27:08 smithi089 bash[26572]: cluster 2023-06-03T19:27:07.728313+0000 mgr.y (mgr.14146) 125 : cluster [DBG] pgmap v101: 0 pgs: ; 0 B data, 9.6 MiB used, 179 GiB / 179 GiB avail 2023-06-03T19:27:09.280 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:27:08 smithi089 bash[33173]: cluster 2023-06-03T19:27:07.728313+0000 mgr.y (mgr.14146) 125 : cluster [DBG] pgmap v101: 0 pgs: ; 0 B data, 9.6 MiB used, 179 GiB / 179 GiB avail 2023-06-03T19:27:11.085 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:27:10 smithi164 bash[26855]: cluster 2023-06-03T19:27:09.728771+0000 mgr.y (mgr.14146) 126 : cluster [DBG] pgmap v102: 0 pgs: ; 0 B data, 9.6 MiB used, 179 GiB / 179 GiB avail 2023-06-03T19:27:11.280 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:27:10 smithi089 bash[33173]: cluster 2023-06-03T19:27:09.728771+0000 mgr.y (mgr.14146) 126 : cluster [DBG] pgmap v102: 0 pgs: ; 0 B data, 9.6 MiB used, 179 GiB / 179 GiB avail 2023-06-03T19:27:11.280 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:27:10 smithi089 bash[26572]: cluster 2023-06-03T19:27:09.728771+0000 mgr.y (mgr.14146) 126 : cluster [DBG] pgmap v102: 0 pgs: ; 0 B data, 9.6 MiB used, 179 GiB / 179 GiB avail 2023-06-03T19:27:12.780 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:27:12 smithi089 bash[33173]: cluster 2023-06-03T19:27:11.729180+0000 mgr.y (mgr.14146) 127 : cluster [DBG] pgmap v103: 0 pgs: ; 0 B data, 9.6 MiB used, 179 GiB / 179 GiB avail 2023-06-03T19:27:12.780 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:27:12 smithi089 bash[26572]: cluster 2023-06-03T19:27:11.729180+0000 mgr.y (mgr.14146) 127 : cluster [DBG] pgmap v103: 0 pgs: ; 0 B data, 9.6 MiB used, 179 GiB / 179 GiB avail 2023-06-03T19:27:12.834 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:27:12 smithi164 bash[26855]: cluster 2023-06-03T19:27:11.729180+0000 mgr.y (mgr.14146) 127 : cluster [DBG] pgmap v103: 0 pgs: ; 0 B data, 9.6 MiB used, 179 GiB / 179 GiB avail 2023-06-03T19:27:13.780 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:27:13 smithi089 bash[33173]: audit 2023-06-03T19:27:12.639003+0000 mon.a (mon.0) 304 : audit [INF] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2023-06-03T19:27:13.780 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:27:13 smithi089 bash[33173]: audit 2023-06-03T19:27:12.640182+0000 mon.a (mon.0) 305 : audit [DBG] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-03T19:27:13.780 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:27:13 smithi089 bash[33173]: cephadm 2023-06-03T19:27:12.641177+0000 mgr.y (mgr.14146) 128 : cephadm [INF] Deploying daemon osd.2 on smithi089 2023-06-03T19:27:13.781 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:27:13 smithi089 bash[26572]: audit 2023-06-03T19:27:12.639003+0000 mon.a (mon.0) 304 : audit [INF] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2023-06-03T19:27:13.781 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:27:13 smithi089 bash[26572]: audit 2023-06-03T19:27:12.640182+0000 mon.a (mon.0) 305 : audit [DBG] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-03T19:27:13.781 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:27:13 smithi089 bash[26572]: cephadm 2023-06-03T19:27:12.641177+0000 mgr.y (mgr.14146) 128 : cephadm [INF] Deploying daemon osd.2 on smithi089 2023-06-03T19:27:13.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:27:13 smithi164 bash[26855]: audit 2023-06-03T19:27:12.639003+0000 mon.a (mon.0) 304 : audit [INF] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2023-06-03T19:27:13.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:27:13 smithi164 bash[26855]: audit 2023-06-03T19:27:12.640182+0000 mon.a (mon.0) 305 : audit [DBG] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-03T19:27:13.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:27:13 smithi164 bash[26855]: cephadm 2023-06-03T19:27:12.641177+0000 mgr.y (mgr.14146) 128 : cephadm [INF] Deploying daemon osd.2 on smithi089 2023-06-03T19:27:14.780 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:27:14 smithi089 bash[33173]: cluster 2023-06-03T19:27:13.729695+0000 mgr.y (mgr.14146) 129 : cluster [DBG] pgmap v104: 0 pgs: ; 0 B data, 9.6 MiB used, 179 GiB / 179 GiB avail 2023-06-03T19:27:14.781 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:27:14 smithi089 bash[26572]: cluster 2023-06-03T19:27:13.729695+0000 mgr.y (mgr.14146) 129 : cluster [DBG] pgmap v104: 0 pgs: ; 0 B data, 9.6 MiB used, 179 GiB / 179 GiB avail 2023-06-03T19:27:14.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:27:14 smithi164 bash[26855]: cluster 2023-06-03T19:27:13.729695+0000 mgr.y (mgr.14146) 129 : cluster [DBG] pgmap v104: 0 pgs: ; 0 B data, 9.6 MiB used, 179 GiB / 179 GiB avail 2023-06-03T19:27:17.085 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:27:16 smithi164 bash[26855]: cluster 2023-06-03T19:27:15.730113+0000 mgr.y (mgr.14146) 130 : cluster [DBG] pgmap v105: 0 pgs: ; 0 B data, 9.6 MiB used, 179 GiB / 179 GiB avail 2023-06-03T19:27:17.086 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:27:16 smithi089 bash[26572]: cluster 2023-06-03T19:27:15.730113+0000 mgr.y (mgr.14146) 130 : cluster [DBG] pgmap v105: 0 pgs: ; 0 B data, 9.6 MiB used, 179 GiB / 179 GiB avail 2023-06-03T19:27:17.086 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:27:16 smithi089 bash[33173]: cluster 2023-06-03T19:27:15.730113+0000 mgr.y (mgr.14146) 130 : cluster [DBG] pgmap v105: 0 pgs: ; 0 B data, 9.6 MiB used, 179 GiB / 179 GiB avail 2023-06-03T19:27:18.780 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:27:18 smithi089 bash[33173]: audit 2023-06-03T19:27:17.337644+0000 mon.a (mon.0) 306 : audit [INF] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' 2023-06-03T19:27:18.780 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:27:18 smithi089 bash[33173]: audit 2023-06-03T19:27:17.348012+0000 mon.a (mon.0) 307 : audit [DBG] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-03T19:27:18.780 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:27:18 smithi089 bash[33173]: cluster 2023-06-03T19:27:17.730616+0000 mgr.y (mgr.14146) 131 : cluster [DBG] pgmap v106: 0 pgs: ; 0 B data, 9.6 MiB used, 179 GiB / 179 GiB avail 2023-06-03T19:27:18.781 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:27:18 smithi089 bash[26572]: audit 2023-06-03T19:27:17.337644+0000 mon.a (mon.0) 306 : audit [INF] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' 2023-06-03T19:27:18.781 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:27:18 smithi089 bash[26572]: audit 2023-06-03T19:27:17.348012+0000 mon.a (mon.0) 307 : audit [DBG] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-03T19:27:18.781 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:27:18 smithi089 bash[26572]: cluster 2023-06-03T19:27:17.730616+0000 mgr.y (mgr.14146) 131 : cluster [DBG] pgmap v106: 0 pgs: ; 0 B data, 9.6 MiB used, 179 GiB / 179 GiB avail 2023-06-03T19:27:18.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:27:18 smithi164 bash[26855]: audit 2023-06-03T19:27:17.337644+0000 mon.a (mon.0) 306 : audit [INF] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' 2023-06-03T19:27:18.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:27:18 smithi164 bash[26855]: audit 2023-06-03T19:27:17.348012+0000 mon.a (mon.0) 307 : audit [DBG] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-03T19:27:18.836 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:27:18 smithi164 bash[26855]: cluster 2023-06-03T19:27:17.730616+0000 mgr.y (mgr.14146) 131 : cluster [DBG] pgmap v106: 0 pgs: ; 0 B data, 9.6 MiB used, 179 GiB / 179 GiB avail 2023-06-03T19:27:21.030 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:27:20 smithi089 bash[33173]: cluster 2023-06-03T19:27:19.730989+0000 mgr.y (mgr.14146) 132 : cluster [DBG] pgmap v107: 0 pgs: ; 0 B data, 9.6 MiB used, 179 GiB / 179 GiB avail 2023-06-03T19:27:21.031 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:27:20 smithi089 bash[26572]: cluster 2023-06-03T19:27:19.730989+0000 mgr.y (mgr.14146) 132 : cluster [DBG] pgmap v107: 0 pgs: ; 0 B data, 9.6 MiB used, 179 GiB / 179 GiB avail 2023-06-03T19:27:21.085 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:27:20 smithi164 bash[26855]: cluster 2023-06-03T19:27:19.730989+0000 mgr.y (mgr.14146) 132 : cluster [DBG] pgmap v107: 0 pgs: ; 0 B data, 9.6 MiB used, 179 GiB / 179 GiB avail 2023-06-03T19:27:22.780 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:27:22 smithi089 bash[26572]: cluster 2023-06-03T19:27:21.731451+0000 mgr.y (mgr.14146) 133 : cluster [DBG] pgmap v108: 0 pgs: ; 0 B data, 9.6 MiB used, 179 GiB / 179 GiB avail 2023-06-03T19:27:22.780 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:27:22 smithi089 bash[33173]: cluster 2023-06-03T19:27:21.731451+0000 mgr.y (mgr.14146) 133 : cluster [DBG] pgmap v108: 0 pgs: ; 0 B data, 9.6 MiB used, 179 GiB / 179 GiB avail 2023-06-03T19:27:22.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:27:22 smithi164 bash[26855]: cluster 2023-06-03T19:27:21.731451+0000 mgr.y (mgr.14146) 133 : cluster [DBG] pgmap v108: 0 pgs: ; 0 B data, 9.6 MiB used, 179 GiB / 179 GiB avail 2023-06-03T19:27:25.030 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:27:24 smithi089 bash[26572]: audit 2023-06-03T19:27:23.730863+0000 mon.a (mon.0) 308 : audit [INF] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' 2023-06-03T19:27:25.031 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:27:24 smithi089 bash[26572]: cluster 2023-06-03T19:27:23.731800+0000 mgr.y (mgr.14146) 134 : cluster [DBG] pgmap v109: 0 pgs: ; 0 B data, 9.6 MiB used, 179 GiB / 179 GiB avail 2023-06-03T19:27:25.031 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:27:24 smithi089 bash[26572]: audit 2023-06-03T19:27:23.731931+0000 mon.a (mon.0) 309 : audit [DBG] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-03T19:27:25.031 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:27:24 smithi089 bash[26572]: audit 2023-06-03T19:27:23.732965+0000 mon.a (mon.0) 310 : audit [INF] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-03T19:27:25.031 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:27:24 smithi089 bash[26572]: audit 2023-06-03T19:27:23.739109+0000 mon.a (mon.0) 311 : audit [INF] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' 2023-06-03T19:27:25.031 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:27:24 smithi089 bash[26572]: audit 2023-06-03T19:27:24.395539+0000 mon.c (mon.2) 8 : audit [INF] from='osd.2 [v2:172.21.15.89:6818/2113694583,v1:172.21.15.89:6819/2113694583]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2023-06-03T19:27:25.032 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:27:24 smithi089 bash[26572]: audit 2023-06-03T19:27:24.396001+0000 mon.a (mon.0) 312 : audit [INF] from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2023-06-03T19:27:25.032 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:27:24 smithi089 bash[33173]: audit 2023-06-03T19:27:23.730863+0000 mon.a (mon.0) 308 : audit [INF] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' 2023-06-03T19:27:25.032 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:27:24 smithi089 bash[33173]: cluster 2023-06-03T19:27:23.731800+0000 mgr.y (mgr.14146) 134 : cluster [DBG] pgmap v109: 0 pgs: ; 0 B data, 9.6 MiB used, 179 GiB / 179 GiB avail 2023-06-03T19:27:25.032 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:27:24 smithi089 bash[33173]: audit 2023-06-03T19:27:23.731931+0000 mon.a (mon.0) 309 : audit [DBG] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-03T19:27:25.032 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:27:24 smithi089 bash[33173]: audit 2023-06-03T19:27:23.732965+0000 mon.a (mon.0) 310 : audit [INF] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-03T19:27:25.032 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:27:24 smithi089 bash[33173]: audit 2023-06-03T19:27:23.739109+0000 mon.a (mon.0) 311 : audit [INF] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' 2023-06-03T19:27:25.033 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:27:24 smithi089 bash[33173]: audit 2023-06-03T19:27:24.395539+0000 mon.c (mon.2) 8 : audit [INF] from='osd.2 [v2:172.21.15.89:6818/2113694583,v1:172.21.15.89:6819/2113694583]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2023-06-03T19:27:25.033 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:27:24 smithi089 bash[33173]: audit 2023-06-03T19:27:24.396001+0000 mon.a (mon.0) 312 : audit [INF] from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2023-06-03T19:27:25.085 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:27:24 smithi164 bash[26855]: audit 2023-06-03T19:27:23.730863+0000 mon.a (mon.0) 308 : audit [INF] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' 2023-06-03T19:27:25.085 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:27:24 smithi164 bash[26855]: cluster 2023-06-03T19:27:23.731800+0000 mgr.y (mgr.14146) 134 : cluster [DBG] pgmap v109: 0 pgs: ; 0 B data, 9.6 MiB used, 179 GiB / 179 GiB avail 2023-06-03T19:27:25.086 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:27:24 smithi164 bash[26855]: audit 2023-06-03T19:27:23.731931+0000 mon.a (mon.0) 309 : audit [DBG] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-03T19:27:25.086 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:27:24 smithi164 bash[26855]: audit 2023-06-03T19:27:23.732965+0000 mon.a (mon.0) 310 : audit [INF] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-03T19:27:25.086 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:27:24 smithi164 bash[26855]: audit 2023-06-03T19:27:23.739109+0000 mon.a (mon.0) 311 : audit [INF] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' 2023-06-03T19:27:25.086 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:27:24 smithi164 bash[26855]: audit 2023-06-03T19:27:24.395539+0000 mon.c (mon.2) 8 : audit [INF] from='osd.2 [v2:172.21.15.89:6818/2113694583,v1:172.21.15.89:6819/2113694583]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2023-06-03T19:27:25.086 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:27:24 smithi164 bash[26855]: audit 2023-06-03T19:27:24.396001+0000 mon.a (mon.0) 312 : audit [INF] from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2023-06-03T19:27:26.030 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:27:25 smithi089 bash[26572]: audit 2023-06-03T19:27:24.745584+0000 mon.a (mon.0) 313 : audit [INF] from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2023-06-03T19:27:26.031 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:27:25 smithi089 bash[26572]: cluster 2023-06-03T19:27:24.745668+0000 mon.a (mon.0) 314 : cluster [DBG] osdmap e16: 3 total, 2 up, 3 in 2023-06-03T19:27:26.031 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:27:25 smithi089 bash[26572]: audit 2023-06-03T19:27:24.745797+0000 mon.a (mon.0) 315 : audit [DBG] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-06-03T19:27:26.031 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:27:25 smithi089 bash[26572]: audit 2023-06-03T19:27:24.749158+0000 mon.c (mon.2) 9 : audit [INF] from='osd.2 [v2:172.21.15.89:6818/2113694583,v1:172.21.15.89:6819/2113694583]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi089", "root=default"]}]: dispatch 2023-06-03T19:27:26.031 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:27:25 smithi089 bash[26572]: audit 2023-06-03T19:27:24.749567+0000 mon.a (mon.0) 316 : audit [INF] from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi089", "root=default"]}]: dispatch 2023-06-03T19:27:26.031 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:27:25 smithi089 bash[33173]: audit 2023-06-03T19:27:24.745584+0000 mon.a (mon.0) 313 : audit [INF] from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2023-06-03T19:27:26.032 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:27:25 smithi089 bash[33173]: cluster 2023-06-03T19:27:24.745668+0000 mon.a (mon.0) 314 : cluster [DBG] osdmap e16: 3 total, 2 up, 3 in 2023-06-03T19:27:26.032 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:27:25 smithi089 bash[33173]: audit 2023-06-03T19:27:24.745797+0000 mon.a (mon.0) 315 : audit [DBG] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-06-03T19:27:26.032 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:27:25 smithi089 bash[33173]: audit 2023-06-03T19:27:24.749158+0000 mon.c (mon.2) 9 : audit [INF] from='osd.2 [v2:172.21.15.89:6818/2113694583,v1:172.21.15.89:6819/2113694583]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi089", "root=default"]}]: dispatch 2023-06-03T19:27:26.032 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:27:25 smithi089 bash[33173]: audit 2023-06-03T19:27:24.749567+0000 mon.a (mon.0) 316 : audit [INF] from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi089", "root=default"]}]: dispatch 2023-06-03T19:27:26.085 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:27:25 smithi164 bash[26855]: audit 2023-06-03T19:27:24.745584+0000 mon.a (mon.0) 313 : audit [INF] from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2023-06-03T19:27:26.085 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:27:25 smithi164 bash[26855]: cluster 2023-06-03T19:27:24.745668+0000 mon.a (mon.0) 314 : cluster [DBG] osdmap e16: 3 total, 2 up, 3 in 2023-06-03T19:27:26.085 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:27:25 smithi164 bash[26855]: audit 2023-06-03T19:27:24.745797+0000 mon.a (mon.0) 315 : audit [DBG] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-06-03T19:27:26.085 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:27:25 smithi164 bash[26855]: audit 2023-06-03T19:27:24.749158+0000 mon.c (mon.2) 9 : audit [INF] from='osd.2 [v2:172.21.15.89:6818/2113694583,v1:172.21.15.89:6819/2113694583]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi089", "root=default"]}]: dispatch 2023-06-03T19:27:26.085 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:27:25 smithi164 bash[26855]: audit 2023-06-03T19:27:24.749567+0000 mon.a (mon.0) 316 : audit [INF] from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi089", "root=default"]}]: dispatch 2023-06-03T19:27:27.030 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:27:26 smithi089 bash[26572]: cluster 2023-06-03T19:27:25.732227+0000 mgr.y (mgr.14146) 135 : cluster [DBG] pgmap v111: 0 pgs: ; 0 B data, 9.6 MiB used, 179 GiB / 179 GiB avail 2023-06-03T19:27:27.031 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:27:26 smithi089 bash[26572]: audit 2023-06-03T19:27:25.753411+0000 mon.a (mon.0) 317 : audit [INF] from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi089", "root=default"]}]': finished 2023-06-03T19:27:27.031 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:27:26 smithi089 bash[26572]: cluster 2023-06-03T19:27:25.753516+0000 mon.a (mon.0) 318 : cluster [DBG] osdmap e17: 3 total, 2 up, 3 in 2023-06-03T19:27:27.031 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:27:26 smithi089 bash[26572]: audit 2023-06-03T19:27:25.754106+0000 mon.a (mon.0) 319 : audit [DBG] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-06-03T19:27:27.031 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:27:26 smithi089 bash[26572]: audit 2023-06-03T19:27:25.761722+0000 mon.a (mon.0) 320 : audit [DBG] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-06-03T19:27:27.031 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:27:26 smithi089 bash[33173]: cluster 2023-06-03T19:27:25.732227+0000 mgr.y (mgr.14146) 135 : cluster [DBG] pgmap v111: 0 pgs: ; 0 B data, 9.6 MiB used, 179 GiB / 179 GiB avail 2023-06-03T19:27:27.032 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:27:26 smithi089 bash[33173]: audit 2023-06-03T19:27:25.753411+0000 mon.a (mon.0) 317 : audit [INF] from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi089", "root=default"]}]': finished 2023-06-03T19:27:27.032 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:27:26 smithi089 bash[33173]: cluster 2023-06-03T19:27:25.753516+0000 mon.a (mon.0) 318 : cluster [DBG] osdmap e17: 3 total, 2 up, 3 in 2023-06-03T19:27:27.032 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:27:26 smithi089 bash[33173]: audit 2023-06-03T19:27:25.754106+0000 mon.a (mon.0) 319 : audit [DBG] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-06-03T19:27:27.032 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:27:26 smithi089 bash[33173]: audit 2023-06-03T19:27:25.761722+0000 mon.a (mon.0) 320 : audit [DBG] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-06-03T19:27:27.086 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:27:26 smithi164 bash[26855]: cluster 2023-06-03T19:27:25.732227+0000 mgr.y (mgr.14146) 135 : cluster [DBG] pgmap v111: 0 pgs: ; 0 B data, 9.6 MiB used, 179 GiB / 179 GiB avail 2023-06-03T19:27:27.086 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:27:26 smithi164 bash[26855]: audit 2023-06-03T19:27:25.753411+0000 mon.a (mon.0) 317 : audit [INF] from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi089", "root=default"]}]': finished 2023-06-03T19:27:27.086 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:27:26 smithi164 bash[26855]: cluster 2023-06-03T19:27:25.753516+0000 mon.a (mon.0) 318 : cluster [DBG] osdmap e17: 3 total, 2 up, 3 in 2023-06-03T19:27:27.086 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:27:26 smithi164 bash[26855]: audit 2023-06-03T19:27:25.754106+0000 mon.a (mon.0) 319 : audit [DBG] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-06-03T19:27:27.086 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:27:26 smithi164 bash[26855]: audit 2023-06-03T19:27:25.761722+0000 mon.a (mon.0) 320 : audit [DBG] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-06-03T19:27:27.652 INFO:teuthology.orchestra.run.smithi089.stdout:Created osd(s) 2 on host 'smithi089' 2023-06-03T19:27:28.120 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:27:28 smithi089 bash[33173]: cluster 2023-06-03T19:27:25.383174+0000 osd.2 (osd.2) 1 : cluster [DBG] purged_snaps scrub starts 2023-06-03T19:27:28.120 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:27:28 smithi089 bash[33173]: cluster 2023-06-03T19:27:25.383234+0000 osd.2 (osd.2) 2 : cluster [DBG] purged_snaps scrub ok 2023-06-03T19:27:28.120 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:27:28 smithi089 bash[33173]: cluster 2023-06-03T19:27:26.765438+0000 mon.a (mon.0) 321 : cluster [INF] osd.2 [v2:172.21.15.89:6818/2113694583,v1:172.21.15.89:6819/2113694583] boot 2023-06-03T19:27:28.121 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:27:28 smithi089 bash[33173]: cluster 2023-06-03T19:27:26.765549+0000 mon.a (mon.0) 322 : cluster [DBG] osdmap e18: 3 total, 3 up, 3 in 2023-06-03T19:27:28.121 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:27:28 smithi089 bash[33173]: audit 2023-06-03T19:27:26.766189+0000 mon.a (mon.0) 323 : audit [DBG] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-06-03T19:27:28.121 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:27:28 smithi089 bash[33173]: audit 2023-06-03T19:27:27.643016+0000 mon.a (mon.0) 324 : audit [INF] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' 2023-06-03T19:27:28.121 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:27:28 smithi089 bash[26572]: cluster 2023-06-03T19:27:25.383174+0000 osd.2 (osd.2) 1 : cluster [DBG] purged_snaps scrub starts 2023-06-03T19:27:28.121 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:27:28 smithi089 bash[26572]: cluster 2023-06-03T19:27:25.383234+0000 osd.2 (osd.2) 2 : cluster [DBG] purged_snaps scrub ok 2023-06-03T19:27:28.121 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:27:28 smithi089 bash[26572]: cluster 2023-06-03T19:27:26.765438+0000 mon.a (mon.0) 321 : cluster [INF] osd.2 [v2:172.21.15.89:6818/2113694583,v1:172.21.15.89:6819/2113694583] boot 2023-06-03T19:27:28.122 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:27:28 smithi089 bash[26572]: cluster 2023-06-03T19:27:26.765549+0000 mon.a (mon.0) 322 : cluster [DBG] osdmap e18: 3 total, 3 up, 3 in 2023-06-03T19:27:28.122 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:27:28 smithi089 bash[26572]: audit 2023-06-03T19:27:26.766189+0000 mon.a (mon.0) 323 : audit [DBG] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-06-03T19:27:28.122 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:27:28 smithi089 bash[26572]: audit 2023-06-03T19:27:27.643016+0000 mon.a (mon.0) 324 : audit [INF] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' 2023-06-03T19:27:28.382 DEBUG:teuthology.orchestra.run.smithi089:osd.2> sudo journalctl -f -n 0 -u ceph-f1db3478-0243-11ee-9b1d-001a4aab830c@osd.2.service 2023-06-03T19:27:28.389 INFO:tasks.cephadm:Deploying osd.3 on smithi089 with /dev/vg_nvme/lv_1... 2023-06-03T19:27:28.389 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph-volume lvm zap /dev/vg_nvme/lv_1 2023-06-03T19:27:28.519 INFO:journalctl@ceph.osd.2.smithi089.stdout:-- Logs begin at Sun 2022-08-14 12:20:24 UTC. -- 2023-06-03T19:27:28.585 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:27:28 smithi164 bash[26855]: cluster 2023-06-03T19:27:25.383174+0000 osd.2 (osd.2) 1 : cluster [DBG] purged_snaps scrub starts 2023-06-03T19:27:28.585 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:27:28 smithi164 bash[26855]: cluster 2023-06-03T19:27:25.383234+0000 osd.2 (osd.2) 2 : cluster [DBG] purged_snaps scrub ok 2023-06-03T19:27:28.586 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:27:28 smithi164 bash[26855]: cluster 2023-06-03T19:27:26.765438+0000 mon.a (mon.0) 321 : cluster [INF] osd.2 [v2:172.21.15.89:6818/2113694583,v1:172.21.15.89:6819/2113694583] boot 2023-06-03T19:27:28.586 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:27:28 smithi164 bash[26855]: cluster 2023-06-03T19:27:26.765549+0000 mon.a (mon.0) 322 : cluster [DBG] osdmap e18: 3 total, 3 up, 3 in 2023-06-03T19:27:28.586 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:27:28 smithi164 bash[26855]: audit 2023-06-03T19:27:26.766189+0000 mon.a (mon.0) 323 : audit [DBG] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-06-03T19:27:28.586 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:27:28 smithi164 bash[26855]: audit 2023-06-03T19:27:27.643016+0000 mon.a (mon.0) 324 : audit [INF] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' 2023-06-03T19:27:29.530 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:27:29 smithi089 bash[33173]: audit 2023-06-03T19:27:27.646484+0000 mon.a (mon.0) 325 : audit [INF] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "osd pool create", "format": "json", "pool": "device_health_metrics", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]: dispatch 2023-06-03T19:27:29.531 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:27:29 smithi089 bash[33173]: cluster 2023-06-03T19:27:27.732748+0000 mgr.y (mgr.14146) 136 : cluster [DBG] pgmap v114: 0 pgs: ; 0 B data, 9.6 MiB used, 179 GiB / 179 GiB avail 2023-06-03T19:27:29.531 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:27:29 smithi089 bash[33173]: audit 2023-06-03T19:27:28.101883+0000 mon.a (mon.0) 326 : audit [DBG] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-03T19:27:29.532 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:27:29 smithi089 bash[33173]: audit 2023-06-03T19:27:28.108133+0000 mon.a (mon.0) 327 : audit [INF] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd='[{"prefix": "osd pool create", "format": "json", "pool": "device_health_metrics", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]': finished 2023-06-03T19:27:29.532 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:27:29 smithi089 bash[33173]: cluster 2023-06-03T19:27:28.108198+0000 mon.a (mon.0) 328 : cluster [DBG] osdmap e19: 3 total, 3 up, 3 in 2023-06-03T19:27:29.532 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:27:29 smithi089 bash[33173]: audit 2023-06-03T19:27:28.110088+0000 mon.a (mon.0) 329 : audit [INF] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": "device_health_metrics", "app": "mgr_devicehealth"}]: dispatch 2023-06-03T19:27:29.533 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:27:29 smithi089 bash[26572]: audit 2023-06-03T19:27:27.646484+0000 mon.a (mon.0) 325 : audit [INF] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "osd pool create", "format": "json", "pool": "device_health_metrics", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]: dispatch 2023-06-03T19:27:29.533 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:27:29 smithi089 bash[26572]: cluster 2023-06-03T19:27:27.732748+0000 mgr.y (mgr.14146) 136 : cluster [DBG] pgmap v114: 0 pgs: ; 0 B data, 9.6 MiB used, 179 GiB / 179 GiB avail 2023-06-03T19:27:29.533 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:27:29 smithi089 bash[26572]: audit 2023-06-03T19:27:28.101883+0000 mon.a (mon.0) 326 : audit [DBG] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-03T19:27:29.534 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:27:29 smithi089 bash[26572]: audit 2023-06-03T19:27:28.108133+0000 mon.a (mon.0) 327 : audit [INF] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd='[{"prefix": "osd pool create", "format": "json", "pool": "device_health_metrics", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]': finished 2023-06-03T19:27:29.534 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:27:29 smithi089 bash[26572]: cluster 2023-06-03T19:27:28.108198+0000 mon.a (mon.0) 328 : cluster [DBG] osdmap e19: 3 total, 3 up, 3 in 2023-06-03T19:27:29.534 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:27:29 smithi089 bash[26572]: audit 2023-06-03T19:27:28.110088+0000 mon.a (mon.0) 329 : audit [INF] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": "device_health_metrics", "app": "mgr_devicehealth"}]: dispatch 2023-06-03T19:27:29.585 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:27:29 smithi164 bash[26855]: audit 2023-06-03T19:27:27.646484+0000 mon.a (mon.0) 325 : audit [INF] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "osd pool create", "format": "json", "pool": "device_health_metrics", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]: dispatch 2023-06-03T19:27:29.585 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:27:29 smithi164 bash[26855]: cluster 2023-06-03T19:27:27.732748+0000 mgr.y (mgr.14146) 136 : cluster [DBG] pgmap v114: 0 pgs: ; 0 B data, 9.6 MiB used, 179 GiB / 179 GiB avail 2023-06-03T19:27:29.585 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:27:29 smithi164 bash[26855]: audit 2023-06-03T19:27:28.101883+0000 mon.a (mon.0) 326 : audit [DBG] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-03T19:27:29.585 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:27:29 smithi164 bash[26855]: audit 2023-06-03T19:27:28.108133+0000 mon.a (mon.0) 327 : audit [INF] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd='[{"prefix": "osd pool create", "format": "json", "pool": "device_health_metrics", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]': finished 2023-06-03T19:27:29.585 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:27:29 smithi164 bash[26855]: cluster 2023-06-03T19:27:28.108198+0000 mon.a (mon.0) 328 : cluster [DBG] osdmap e19: 3 total, 3 up, 3 in 2023-06-03T19:27:29.586 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:27:29 smithi164 bash[26855]: audit 2023-06-03T19:27:28.110088+0000 mon.a (mon.0) 329 : audit [INF] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": "device_health_metrics", "app": "mgr_devicehealth"}]: dispatch 2023-06-03T19:27:30.531 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:27:30 smithi089 bash[26572]: audit 2023-06-03T19:27:29.112316+0000 mon.a (mon.0) 330 : audit [INF] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": "device_health_metrics", "app": "mgr_devicehealth"}]': finished 2023-06-03T19:27:30.531 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:27:30 smithi089 bash[26572]: cluster 2023-06-03T19:27:29.112498+0000 mon.a (mon.0) 331 : cluster [DBG] osdmap e20: 3 total, 3 up, 3 in 2023-06-03T19:27:30.531 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:27:30 smithi089 bash[33173]: audit 2023-06-03T19:27:29.112316+0000 mon.a (mon.0) 330 : audit [INF] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": "device_health_metrics", "app": "mgr_devicehealth"}]': finished 2023-06-03T19:27:30.532 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:27:30 smithi089 bash[33173]: cluster 2023-06-03T19:27:29.112498+0000 mon.a (mon.0) 331 : cluster [DBG] osdmap e20: 3 total, 3 up, 3 in 2023-06-03T19:27:30.585 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:27:30 smithi164 bash[26855]: audit 2023-06-03T19:27:29.112316+0000 mon.a (mon.0) 330 : audit [INF] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": "device_health_metrics", "app": "mgr_devicehealth"}]': finished 2023-06-03T19:27:30.585 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:27:30 smithi164 bash[26855]: cluster 2023-06-03T19:27:29.112498+0000 mon.a (mon.0) 331 : cluster [DBG] osdmap e20: 3 total, 3 up, 3 in 2023-06-03T19:27:31.530 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:27:31 smithi089 bash[33173]: cluster 2023-06-03T19:27:29.733314+0000 mgr.y (mgr.14146) 137 : cluster [DBG] pgmap v117: 1 pgs: 1 unknown; 0 B data, 14 MiB used, 268 GiB / 268 GiB avail 2023-06-03T19:27:31.530 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:27:31 smithi089 bash[33173]: cluster 2023-06-03T19:27:30.129233+0000 mon.a (mon.0) 332 : cluster [DBG] osdmap e21: 3 total, 3 up, 3 in 2023-06-03T19:27:31.531 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:27:31 smithi089 bash[26572]: cluster 2023-06-03T19:27:29.733314+0000 mgr.y (mgr.14146) 137 : cluster [DBG] pgmap v117: 1 pgs: 1 unknown; 0 B data, 14 MiB used, 268 GiB / 268 GiB avail 2023-06-03T19:27:31.531 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:27:31 smithi089 bash[26572]: cluster 2023-06-03T19:27:30.129233+0000 mon.a (mon.0) 332 : cluster [DBG] osdmap e21: 3 total, 3 up, 3 in 2023-06-03T19:27:31.585 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:27:31 smithi164 bash[26855]: cluster 2023-06-03T19:27:29.733314+0000 mgr.y (mgr.14146) 137 : cluster [DBG] pgmap v117: 1 pgs: 1 unknown; 0 B data, 14 MiB used, 268 GiB / 268 GiB avail 2023-06-03T19:27:31.585 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:27:31 smithi164 bash[26855]: cluster 2023-06-03T19:27:30.129233+0000 mon.a (mon.0) 332 : cluster [DBG] osdmap e21: 3 total, 3 up, 3 in 2023-06-03T19:27:32.370 INFO:teuthology.orchestra.run.smithi089.stderr:--> Zapping: /dev/vg_nvme/lv_1 2023-06-03T19:27:32.371 INFO:teuthology.orchestra.run.smithi089.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_1 bs=1M count=10 conv=fsync 2023-06-03T19:27:32.371 INFO:teuthology.orchestra.run.smithi089.stderr: stderr: 10+0 records in 2023-06-03T19:27:32.371 INFO:teuthology.orchestra.run.smithi089.stderr:10+0 records out 2023-06-03T19:27:32.371 INFO:teuthology.orchestra.run.smithi089.stderr:10485760 bytes (10 MB, 10 MiB) copied, 0.016553 s, 633 MB/s 2023-06-03T19:27:32.371 INFO:teuthology.orchestra.run.smithi089.stderr:--> Zapping successful for: 2023-06-03T19:27:32.780 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:27:32 smithi089 bash[26572]: cluster 2023-06-03T19:27:31.733805+0000 mgr.y (mgr.14146) 138 : cluster [DBG] pgmap v119: 1 pgs: 1 active+clean; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2023-06-03T19:27:32.781 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:27:32 smithi089 bash[33173]: cluster 2023-06-03T19:27:31.733805+0000 mgr.y (mgr.14146) 138 : cluster [DBG] pgmap v119: 1 pgs: 1 active+clean; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2023-06-03T19:27:32.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:27:32 smithi164 bash[26855]: cluster 2023-06-03T19:27:31.733805+0000 mgr.y (mgr.14146) 138 : cluster [DBG] pgmap v119: 1 pgs: 1 active+clean; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2023-06-03T19:27:33.168 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph orch daemon add osd smithi089:vg_nvme/lv_1 2023-06-03T19:27:35.085 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:27:34 smithi164 bash[26855]: cluster 2023-06-03T19:27:33.734373+0000 mgr.y (mgr.14146) 139 : cluster [DBG] pgmap v120: 1 pgs: 1 active+clean; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2023-06-03T19:27:35.096 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:27:34 smithi089 bash[26572]: cluster 2023-06-03T19:27:33.734373+0000 mgr.y (mgr.14146) 139 : cluster [DBG] pgmap v120: 1 pgs: 1 active+clean; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2023-06-03T19:27:35.096 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:27:34 smithi089 bash[33173]: cluster 2023-06-03T19:27:33.734373+0000 mgr.y (mgr.14146) 139 : cluster [DBG] pgmap v120: 1 pgs: 1 active+clean; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2023-06-03T19:27:37.085 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:27:36 smithi164 bash[26855]: cluster 2023-06-03T19:27:35.734842+0000 mgr.y (mgr.14146) 140 : cluster [DBG] pgmap v121: 1 pgs: 1 active+clean; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2023-06-03T19:27:37.280 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:27:36 smithi089 bash[26572]: cluster 2023-06-03T19:27:35.734842+0000 mgr.y (mgr.14146) 140 : cluster [DBG] pgmap v121: 1 pgs: 1 active+clean; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2023-06-03T19:27:37.281 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:27:36 smithi089 bash[33173]: cluster 2023-06-03T19:27:35.734842+0000 mgr.y (mgr.14146) 140 : cluster [DBG] pgmap v121: 1 pgs: 1 active+clean; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2023-06-03T19:27:39.085 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:27:38 smithi164 bash[26855]: cluster 2023-06-03T19:27:37.735364+0000 mgr.y (mgr.14146) 141 : cluster [DBG] pgmap v122: 1 pgs: 1 active+clean; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2023-06-03T19:27:39.085 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:27:38 smithi164 bash[26855]: audit 2023-06-03T19:27:38.431606+0000 mon.a (mon.0) 333 : audit [DBG] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-06-03T19:27:39.085 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:27:38 smithi164 bash[26855]: audit 2023-06-03T19:27:38.436902+0000 mon.a (mon.0) 334 : audit [INF] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-06-03T19:27:39.085 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:27:38 smithi164 bash[26855]: audit 2023-06-03T19:27:38.438059+0000 mon.a (mon.0) 335 : audit [DBG] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-03T19:27:39.280 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:27:38 smithi089 bash[26572]: cluster 2023-06-03T19:27:37.735364+0000 mgr.y (mgr.14146) 141 : cluster [DBG] pgmap v122: 1 pgs: 1 active+clean; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2023-06-03T19:27:39.281 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:27:38 smithi089 bash[26572]: audit 2023-06-03T19:27:38.431606+0000 mon.a (mon.0) 333 : audit [DBG] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-06-03T19:27:39.281 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:27:38 smithi089 bash[26572]: audit 2023-06-03T19:27:38.436902+0000 mon.a (mon.0) 334 : audit [INF] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-06-03T19:27:39.281 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:27:38 smithi089 bash[26572]: audit 2023-06-03T19:27:38.438059+0000 mon.a (mon.0) 335 : audit [DBG] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-03T19:27:39.281 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:27:38 smithi089 bash[33173]: cluster 2023-06-03T19:27:37.735364+0000 mgr.y (mgr.14146) 141 : cluster [DBG] pgmap v122: 1 pgs: 1 active+clean; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2023-06-03T19:27:39.282 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:27:38 smithi089 bash[33173]: audit 2023-06-03T19:27:38.431606+0000 mon.a (mon.0) 333 : audit [DBG] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-06-03T19:27:39.282 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:27:38 smithi089 bash[33173]: audit 2023-06-03T19:27:38.436902+0000 mon.a (mon.0) 334 : audit [INF] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-06-03T19:27:39.282 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:27:38 smithi089 bash[33173]: audit 2023-06-03T19:27:38.438059+0000 mon.a (mon.0) 335 : audit [DBG] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-03T19:27:40.085 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:27:39 smithi164 bash[26855]: audit 2023-06-03T19:27:38.428101+0000 mgr.y (mgr.14146) 142 : audit [DBG] from='client.14289 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi089:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2023-06-03T19:27:40.280 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:27:39 smithi089 bash[26572]: audit 2023-06-03T19:27:38.428101+0000 mgr.y (mgr.14146) 142 : audit [DBG] from='client.14289 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi089:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2023-06-03T19:27:40.281 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:27:39 smithi089 bash[33173]: audit 2023-06-03T19:27:38.428101+0000 mgr.y (mgr.14146) 142 : audit [DBG] from='client.14289 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi089:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2023-06-03T19:27:41.281 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:27:40 smithi089 bash[26572]: cluster 2023-06-03T19:27:39.735870+0000 mgr.y (mgr.14146) 143 : cluster [DBG] pgmap v123: 1 pgs: 1 active+clean; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2023-06-03T19:27:41.281 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:27:40 smithi089 bash[26572]: cephadm 2023-06-03T19:27:39.839640+0000 mgr.y (mgr.14146) 144 : cephadm [INF] Detected new or changed devices on smithi089 2023-06-03T19:27:41.282 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:27:40 smithi089 bash[26572]: audit 2023-06-03T19:27:39.849574+0000 mon.a (mon.0) 336 : audit [INF] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' 2023-06-03T19:27:41.282 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:27:40 smithi089 bash[26572]: audit 2023-06-03T19:27:40.132494+0000 mon.a (mon.0) 337 : audit [INF] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-06-03T19:27:41.282 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:27:40 smithi089 bash[26572]: audit 2023-06-03T19:27:40.133923+0000 mon.a (mon.0) 338 : audit [INF] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-06-03T19:27:41.282 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:27:40 smithi089 bash[26572]: audit 2023-06-03T19:27:40.135264+0000 mon.a (mon.0) 339 : audit [INF] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2023-06-03T19:27:41.283 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:27:40 smithi089 bash[26572]: audit 2023-06-03T19:27:40.144224+0000 mon.a (mon.0) 340 : audit [INF] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' 2023-06-03T19:27:41.283 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:27:40 smithi089 bash[26572]: audit 2023-06-03T19:27:40.145632+0000 mon.a (mon.0) 341 : audit [DBG] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-03T19:27:41.283 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:27:40 smithi089 bash[26572]: audit 2023-06-03T19:27:40.146641+0000 mon.a (mon.0) 342 : audit [INF] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-03T19:27:41.284 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:27:40 smithi089 bash[26572]: audit 2023-06-03T19:27:40.152371+0000 mon.a (mon.0) 343 : audit [INF] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' 2023-06-03T19:27:41.284 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:27:40 smithi089 bash[33173]: cluster 2023-06-03T19:27:39.735870+0000 mgr.y (mgr.14146) 143 : cluster [DBG] pgmap v123: 1 pgs: 1 active+clean; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2023-06-03T19:27:41.284 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:27:40 smithi089 bash[33173]: cephadm 2023-06-03T19:27:39.839640+0000 mgr.y (mgr.14146) 144 : cephadm [INF] Detected new or changed devices on smithi089 2023-06-03T19:27:41.285 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:27:40 smithi089 bash[33173]: audit 2023-06-03T19:27:39.849574+0000 mon.a (mon.0) 336 : audit [INF] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' 2023-06-03T19:27:41.285 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:27:40 smithi089 bash[33173]: audit 2023-06-03T19:27:40.132494+0000 mon.a (mon.0) 337 : audit [INF] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-06-03T19:27:41.285 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:27:40 smithi089 bash[33173]: audit 2023-06-03T19:27:40.133923+0000 mon.a (mon.0) 338 : audit [INF] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-06-03T19:27:41.285 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:27:40 smithi089 bash[33173]: audit 2023-06-03T19:27:40.135264+0000 mon.a (mon.0) 339 : audit [INF] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2023-06-03T19:27:41.286 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:27:40 smithi089 bash[33173]: audit 2023-06-03T19:27:40.144224+0000 mon.a (mon.0) 340 : audit [INF] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' 2023-06-03T19:27:41.286 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:27:40 smithi089 bash[33173]: audit 2023-06-03T19:27:40.145632+0000 mon.a (mon.0) 341 : audit [DBG] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-03T19:27:41.286 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:27:40 smithi089 bash[33173]: audit 2023-06-03T19:27:40.146641+0000 mon.a (mon.0) 342 : audit [INF] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-03T19:27:41.286 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:27:40 smithi089 bash[33173]: audit 2023-06-03T19:27:40.152371+0000 mon.a (mon.0) 343 : audit [INF] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' 2023-06-03T19:27:41.335 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:27:40 smithi164 bash[26855]: cluster 2023-06-03T19:27:39.735870+0000 mgr.y (mgr.14146) 143 : cluster [DBG] pgmap v123: 1 pgs: 1 active+clean; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2023-06-03T19:27:41.335 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:27:40 smithi164 bash[26855]: cephadm 2023-06-03T19:27:39.839640+0000 mgr.y (mgr.14146) 144 : cephadm [INF] Detected new or changed devices on smithi089 2023-06-03T19:27:41.335 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:27:40 smithi164 bash[26855]: audit 2023-06-03T19:27:39.849574+0000 mon.a (mon.0) 336 : audit [INF] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' 2023-06-03T19:27:41.336 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:27:40 smithi164 bash[26855]: audit 2023-06-03T19:27:40.132494+0000 mon.a (mon.0) 337 : audit [INF] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-06-03T19:27:41.336 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:27:40 smithi164 bash[26855]: audit 2023-06-03T19:27:40.133923+0000 mon.a (mon.0) 338 : audit [INF] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-06-03T19:27:41.336 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:27:40 smithi164 bash[26855]: audit 2023-06-03T19:27:40.135264+0000 mon.a (mon.0) 339 : audit [INF] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2023-06-03T19:27:41.336 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:27:40 smithi164 bash[26855]: audit 2023-06-03T19:27:40.144224+0000 mon.a (mon.0) 340 : audit [INF] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' 2023-06-03T19:27:41.336 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:27:40 smithi164 bash[26855]: audit 2023-06-03T19:27:40.145632+0000 mon.a (mon.0) 341 : audit [DBG] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-03T19:27:41.336 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:27:40 smithi164 bash[26855]: audit 2023-06-03T19:27:40.146641+0000 mon.a (mon.0) 342 : audit [INF] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-03T19:27:41.337 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:27:40 smithi164 bash[26855]: audit 2023-06-03T19:27:40.152371+0000 mon.a (mon.0) 343 : audit [INF] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' 2023-06-03T19:27:42.280 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:27:41 smithi089 bash[26572]: cephadm 2023-06-03T19:27:40.136301+0000 mgr.y (mgr.14146) 145 : cephadm [INF] Adjusting osd_memory_target on smithi089 to 5416M 2023-06-03T19:27:42.281 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:27:41 smithi089 bash[33173]: cephadm 2023-06-03T19:27:40.136301+0000 mgr.y (mgr.14146) 145 : cephadm [INF] Adjusting osd_memory_target on smithi089 to 5416M 2023-06-03T19:27:42.335 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:27:41 smithi164 bash[26855]: cephadm 2023-06-03T19:27:40.136301+0000 mgr.y (mgr.14146) 145 : cephadm [INF] Adjusting osd_memory_target on smithi089 to 5416M 2023-06-03T19:27:43.261 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:27:42 smithi089 bash[33173]: cluster 2023-06-03T19:27:41.736414+0000 mgr.y (mgr.14146) 146 : cluster [DBG] pgmap v124: 1 pgs: 1 active+clean; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2023-06-03T19:27:43.262 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:27:42 smithi089 bash[26572]: cluster 2023-06-03T19:27:41.736414+0000 mgr.y (mgr.14146) 146 : cluster [DBG] pgmap v124: 1 pgs: 1 active+clean; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2023-06-03T19:27:43.335 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:27:42 smithi164 bash[26855]: cluster 2023-06-03T19:27:41.736414+0000 mgr.y (mgr.14146) 146 : cluster [DBG] pgmap v124: 1 pgs: 1 active+clean; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2023-06-03T19:27:45.030 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:27:44 smithi089 bash[33173]: cluster 2023-06-03T19:27:43.736956+0000 mgr.y (mgr.14146) 147 : cluster [DBG] pgmap v125: 1 pgs: 1 active+clean; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2023-06-03T19:27:45.031 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:27:44 smithi089 bash[26572]: cluster 2023-06-03T19:27:43.736956+0000 mgr.y (mgr.14146) 147 : cluster [DBG] pgmap v125: 1 pgs: 1 active+clean; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2023-06-03T19:27:45.335 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:27:44 smithi164 bash[26855]: cluster 2023-06-03T19:27:43.736956+0000 mgr.y (mgr.14146) 147 : cluster [DBG] pgmap v125: 1 pgs: 1 active+clean; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2023-06-03T19:27:47.281 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:27:46 smithi089 bash[33173]: cluster 2023-06-03T19:27:45.737434+0000 mgr.y (mgr.14146) 148 : cluster [DBG] pgmap v126: 1 pgs: 1 active+clean; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2023-06-03T19:27:47.281 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:27:46 smithi089 bash[26572]: cluster 2023-06-03T19:27:45.737434+0000 mgr.y (mgr.14146) 148 : cluster [DBG] pgmap v126: 1 pgs: 1 active+clean; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2023-06-03T19:27:47.335 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:27:46 smithi164 bash[26855]: cluster 2023-06-03T19:27:45.737434+0000 mgr.y (mgr.14146) 148 : cluster [DBG] pgmap v126: 1 pgs: 1 active+clean; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2023-06-03T19:27:48.281 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:27:47 smithi089 bash[26572]: audit 2023-06-03T19:27:47.165721+0000 mon.c (mon.2) 10 : audit [INF] from='client.? 172.21.15.89:0/4025741545' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "b38408eb-3acf-4a41-a314-72e4cd37b1ef"}]: dispatch 2023-06-03T19:27:48.281 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:27:47 smithi089 bash[26572]: audit 2023-06-03T19:27:47.166362+0000 mon.a (mon.0) 344 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "b38408eb-3acf-4a41-a314-72e4cd37b1ef"}]: dispatch 2023-06-03T19:27:48.281 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:27:47 smithi089 bash[26572]: audit 2023-06-03T19:27:47.173834+0000 mon.a (mon.0) 345 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "b38408eb-3acf-4a41-a314-72e4cd37b1ef"}]': finished 2023-06-03T19:27:48.282 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:27:47 smithi089 bash[26572]: cluster 2023-06-03T19:27:47.173956+0000 mon.a (mon.0) 346 : cluster [DBG] osdmap e22: 4 total, 3 up, 4 in 2023-06-03T19:27:48.282 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:27:47 smithi089 bash[26572]: audit 2023-06-03T19:27:47.174121+0000 mon.a (mon.0) 347 : audit [DBG] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-06-03T19:27:48.282 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:27:47 smithi089 bash[33173]: audit 2023-06-03T19:27:47.165721+0000 mon.c (mon.2) 10 : audit [INF] from='client.? 172.21.15.89:0/4025741545' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "b38408eb-3acf-4a41-a314-72e4cd37b1ef"}]: dispatch 2023-06-03T19:27:48.282 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:27:47 smithi089 bash[33173]: audit 2023-06-03T19:27:47.166362+0000 mon.a (mon.0) 344 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "b38408eb-3acf-4a41-a314-72e4cd37b1ef"}]: dispatch 2023-06-03T19:27:48.283 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:27:47 smithi089 bash[33173]: audit 2023-06-03T19:27:47.173834+0000 mon.a (mon.0) 345 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "b38408eb-3acf-4a41-a314-72e4cd37b1ef"}]': finished 2023-06-03T19:27:48.283 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:27:47 smithi089 bash[33173]: cluster 2023-06-03T19:27:47.173956+0000 mon.a (mon.0) 346 : cluster [DBG] osdmap e22: 4 total, 3 up, 4 in 2023-06-03T19:27:48.283 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:27:47 smithi089 bash[33173]: audit 2023-06-03T19:27:47.174121+0000 mon.a (mon.0) 347 : audit [DBG] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-06-03T19:27:48.335 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:27:47 smithi164 bash[26855]: audit 2023-06-03T19:27:47.165721+0000 mon.c (mon.2) 10 : audit [INF] from='client.? 172.21.15.89:0/4025741545' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "b38408eb-3acf-4a41-a314-72e4cd37b1ef"}]: dispatch 2023-06-03T19:27:48.336 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:27:47 smithi164 bash[26855]: audit 2023-06-03T19:27:47.166362+0000 mon.a (mon.0) 344 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "b38408eb-3acf-4a41-a314-72e4cd37b1ef"}]: dispatch 2023-06-03T19:27:48.336 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:27:47 smithi164 bash[26855]: audit 2023-06-03T19:27:47.173834+0000 mon.a (mon.0) 345 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "b38408eb-3acf-4a41-a314-72e4cd37b1ef"}]': finished 2023-06-03T19:27:48.336 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:27:47 smithi164 bash[26855]: cluster 2023-06-03T19:27:47.173956+0000 mon.a (mon.0) 346 : cluster [DBG] osdmap e22: 4 total, 3 up, 4 in 2023-06-03T19:27:48.337 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:27:47 smithi164 bash[26855]: audit 2023-06-03T19:27:47.174121+0000 mon.a (mon.0) 347 : audit [DBG] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-06-03T19:27:49.280 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:27:48 smithi089 bash[33173]: cluster 2023-06-03T19:27:47.737923+0000 mgr.y (mgr.14146) 149 : cluster [DBG] pgmap v128: 1 pgs: 1 active+clean; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2023-06-03T19:27:49.281 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:27:48 smithi089 bash[33173]: audit 2023-06-03T19:27:47.917471+0000 mon.c (mon.2) 11 : audit [DBG] from='client.? 172.21.15.89:0/959620252' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-06-03T19:27:49.281 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:27:48 smithi089 bash[26572]: cluster 2023-06-03T19:27:47.737923+0000 mgr.y (mgr.14146) 149 : cluster [DBG] pgmap v128: 1 pgs: 1 active+clean; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2023-06-03T19:27:49.281 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:27:48 smithi089 bash[26572]: audit 2023-06-03T19:27:47.917471+0000 mon.c (mon.2) 11 : audit [DBG] from='client.? 172.21.15.89:0/959620252' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-06-03T19:27:49.335 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:27:48 smithi164 bash[26855]: cluster 2023-06-03T19:27:47.737923+0000 mgr.y (mgr.14146) 149 : cluster [DBG] pgmap v128: 1 pgs: 1 active+clean; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2023-06-03T19:27:49.335 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:27:48 smithi164 bash[26855]: audit 2023-06-03T19:27:47.917471+0000 mon.c (mon.2) 11 : audit [DBG] from='client.? 172.21.15.89:0/959620252' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-06-03T19:27:51.030 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:27:50 smithi089 bash[33173]: cluster 2023-06-03T19:27:49.738469+0000 mgr.y (mgr.14146) 150 : cluster [DBG] pgmap v129: 1 pgs: 1 active+clean; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2023-06-03T19:27:51.031 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:27:50 smithi089 bash[26572]: cluster 2023-06-03T19:27:49.738469+0000 mgr.y (mgr.14146) 150 : cluster [DBG] pgmap v129: 1 pgs: 1 active+clean; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2023-06-03T19:27:51.335 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:27:50 smithi164 bash[26855]: cluster 2023-06-03T19:27:49.738469+0000 mgr.y (mgr.14146) 150 : cluster [DBG] pgmap v129: 1 pgs: 1 active+clean; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2023-06-03T19:27:52.781 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:27:52 smithi089 bash[26572]: cluster 2023-06-03T19:27:51.739070+0000 mgr.y (mgr.14146) 151 : cluster [DBG] pgmap v130: 1 pgs: 1 active+clean; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2023-06-03T19:27:52.781 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:27:52 smithi089 bash[33173]: cluster 2023-06-03T19:27:51.739070+0000 mgr.y (mgr.14146) 151 : cluster [DBG] pgmap v130: 1 pgs: 1 active+clean; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2023-06-03T19:27:52.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:27:52 smithi164 bash[26855]: cluster 2023-06-03T19:27:51.739070+0000 mgr.y (mgr.14146) 151 : cluster [DBG] pgmap v130: 1 pgs: 1 active+clean; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2023-06-03T19:27:55.085 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:27:54 smithi164 bash[26855]: cluster 2023-06-03T19:27:53.739634+0000 mgr.y (mgr.14146) 152 : cluster [DBG] pgmap v131: 1 pgs: 1 active+clean; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2023-06-03T19:27:55.280 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:27:54 smithi089 bash[26572]: cluster 2023-06-03T19:27:53.739634+0000 mgr.y (mgr.14146) 152 : cluster [DBG] pgmap v131: 1 pgs: 1 active+clean; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2023-06-03T19:27:55.281 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:27:54 smithi089 bash[33173]: cluster 2023-06-03T19:27:53.739634+0000 mgr.y (mgr.14146) 152 : cluster [DBG] pgmap v131: 1 pgs: 1 active+clean; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2023-06-03T19:27:57.085 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:27:56 smithi164 bash[26855]: cluster 2023-06-03T19:27:55.740172+0000 mgr.y (mgr.14146) 153 : cluster [DBG] pgmap v132: 1 pgs: 1 active+clean; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2023-06-03T19:27:57.137 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:27:56 smithi089 bash[26572]: cluster 2023-06-03T19:27:55.740172+0000 mgr.y (mgr.14146) 153 : cluster [DBG] pgmap v132: 1 pgs: 1 active+clean; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2023-06-03T19:27:57.138 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:27:56 smithi089 bash[33173]: cluster 2023-06-03T19:27:55.740172+0000 mgr.y (mgr.14146) 153 : cluster [DBG] pgmap v132: 1 pgs: 1 active+clean; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2023-06-03T19:27:59.085 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:27:58 smithi164 bash[26855]: cluster 2023-06-03T19:27:57.740727+0000 mgr.y (mgr.14146) 154 : cluster [DBG] pgmap v133: 1 pgs: 1 active+clean; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2023-06-03T19:27:59.099 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:27:58 smithi089 bash[26572]: cluster 2023-06-03T19:27:57.740727+0000 mgr.y (mgr.14146) 154 : cluster [DBG] pgmap v133: 1 pgs: 1 active+clean; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2023-06-03T19:27:59.099 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:27:58 smithi089 bash[33173]: cluster 2023-06-03T19:27:57.740727+0000 mgr.y (mgr.14146) 154 : cluster [DBG] pgmap v133: 1 pgs: 1 active+clean; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2023-06-03T19:28:01.031 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:28:00 smithi089 bash[26572]: cluster 2023-06-03T19:27:59.741170+0000 mgr.y (mgr.14146) 155 : cluster [DBG] pgmap v134: 1 pgs: 1 active+clean; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2023-06-03T19:28:01.031 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:28:00 smithi089 bash[26572]: audit 2023-06-03T19:28:00.365731+0000 mon.a (mon.0) 348 : audit [INF] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2023-06-03T19:28:01.031 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:28:00 smithi089 bash[26572]: audit 2023-06-03T19:28:00.367162+0000 mon.a (mon.0) 349 : audit [DBG] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-03T19:28:01.032 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:28:00 smithi089 bash[33173]: cluster 2023-06-03T19:27:59.741170+0000 mgr.y (mgr.14146) 155 : cluster [DBG] pgmap v134: 1 pgs: 1 active+clean; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2023-06-03T19:28:01.032 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:28:00 smithi089 bash[33173]: audit 2023-06-03T19:28:00.365731+0000 mon.a (mon.0) 348 : audit [INF] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2023-06-03T19:28:01.032 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:28:00 smithi089 bash[33173]: audit 2023-06-03T19:28:00.367162+0000 mon.a (mon.0) 349 : audit [DBG] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-03T19:28:01.085 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:28:00 smithi164 bash[26855]: cluster 2023-06-03T19:27:59.741170+0000 mgr.y (mgr.14146) 155 : cluster [DBG] pgmap v134: 1 pgs: 1 active+clean; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2023-06-03T19:28:01.085 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:28:00 smithi164 bash[26855]: audit 2023-06-03T19:28:00.365731+0000 mon.a (mon.0) 348 : audit [INF] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2023-06-03T19:28:01.085 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:28:00 smithi164 bash[26855]: audit 2023-06-03T19:28:00.367162+0000 mon.a (mon.0) 349 : audit [DBG] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-03T19:28:02.031 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:28:01 smithi089 bash[26572]: cephadm 2023-06-03T19:28:00.368189+0000 mgr.y (mgr.14146) 156 : cephadm [INF] Deploying daemon osd.3 on smithi089 2023-06-03T19:28:02.031 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:28:01 smithi089 bash[33173]: cephadm 2023-06-03T19:28:00.368189+0000 mgr.y (mgr.14146) 156 : cephadm [INF] Deploying daemon osd.3 on smithi089 2023-06-03T19:28:02.085 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:28:01 smithi164 bash[26855]: cephadm 2023-06-03T19:28:00.368189+0000 mgr.y (mgr.14146) 156 : cephadm [INF] Deploying daemon osd.3 on smithi089 2023-06-03T19:28:03.030 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:28:02 smithi089 bash[26572]: cluster 2023-06-03T19:28:01.741750+0000 mgr.y (mgr.14146) 157 : cluster [DBG] pgmap v135: 1 pgs: 1 active+clean; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2023-06-03T19:28:03.031 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:28:02 smithi089 bash[33173]: cluster 2023-06-03T19:28:01.741750+0000 mgr.y (mgr.14146) 157 : cluster [DBG] pgmap v135: 1 pgs: 1 active+clean; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2023-06-03T19:28:03.085 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:28:02 smithi164 bash[26855]: cluster 2023-06-03T19:28:01.741750+0000 mgr.y (mgr.14146) 157 : cluster [DBG] pgmap v135: 1 pgs: 1 active+clean; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2023-06-03T19:28:05.031 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:28:04 smithi089 bash[26572]: cluster 2023-06-03T19:28:03.742124+0000 mgr.y (mgr.14146) 158 : cluster [DBG] pgmap v136: 1 pgs: 1 active+clean; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2023-06-03T19:28:05.031 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:28:04 smithi089 bash[33173]: cluster 2023-06-03T19:28:03.742124+0000 mgr.y (mgr.14146) 158 : cluster [DBG] pgmap v136: 1 pgs: 1 active+clean; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2023-06-03T19:28:05.085 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:28:04 smithi164 bash[26855]: cluster 2023-06-03T19:28:03.742124+0000 mgr.y (mgr.14146) 158 : cluster [DBG] pgmap v136: 1 pgs: 1 active+clean; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2023-06-03T19:28:06.530 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:28:06 smithi089 bash[33173]: audit 2023-06-03T19:28:05.110930+0000 mon.a (mon.0) 350 : audit [INF] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' 2023-06-03T19:28:06.531 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:28:06 smithi089 bash[33173]: audit 2023-06-03T19:28:05.135633+0000 mon.a (mon.0) 351 : audit [DBG] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-03T19:28:06.531 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:28:06 smithi089 bash[26572]: audit 2023-06-03T19:28:05.110930+0000 mon.a (mon.0) 350 : audit [INF] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' 2023-06-03T19:28:06.531 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:28:06 smithi089 bash[26572]: audit 2023-06-03T19:28:05.135633+0000 mon.a (mon.0) 351 : audit [DBG] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-03T19:28:06.585 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:28:06 smithi164 bash[26855]: audit 2023-06-03T19:28:05.110930+0000 mon.a (mon.0) 350 : audit [INF] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' 2023-06-03T19:28:06.585 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:28:06 smithi164 bash[26855]: audit 2023-06-03T19:28:05.135633+0000 mon.a (mon.0) 351 : audit [DBG] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-03T19:28:07.281 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:28:07 smithi089 bash[26572]: cluster 2023-06-03T19:28:05.742391+0000 mgr.y (mgr.14146) 159 : cluster [DBG] pgmap v137: 1 pgs: 1 active+clean; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2023-06-03T19:28:07.281 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:28:07 smithi089 bash[33173]: cluster 2023-06-03T19:28:05.742391+0000 mgr.y (mgr.14146) 159 : cluster [DBG] pgmap v137: 1 pgs: 1 active+clean; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2023-06-03T19:28:07.585 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:28:07 smithi164 bash[26855]: cluster 2023-06-03T19:28:05.742391+0000 mgr.y (mgr.14146) 159 : cluster [DBG] pgmap v137: 1 pgs: 1 active+clean; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2023-06-03T19:28:09.530 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:28:09 smithi089 bash[26572]: cluster 2023-06-03T19:28:07.742889+0000 mgr.y (mgr.14146) 160 : cluster [DBG] pgmap v138: 1 pgs: 1 active+clean; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2023-06-03T19:28:09.531 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:28:09 smithi089 bash[33173]: cluster 2023-06-03T19:28:07.742889+0000 mgr.y (mgr.14146) 160 : cluster [DBG] pgmap v138: 1 pgs: 1 active+clean; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2023-06-03T19:28:09.585 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:28:09 smithi164 bash[26855]: cluster 2023-06-03T19:28:07.742889+0000 mgr.y (mgr.14146) 160 : cluster [DBG] pgmap v138: 1 pgs: 1 active+clean; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2023-06-03T19:28:11.530 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:28:11 smithi089 bash[26572]: cluster 2023-06-03T19:28:09.743327+0000 mgr.y (mgr.14146) 161 : cluster [DBG] pgmap v139: 1 pgs: 1 active+clean; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2023-06-03T19:28:11.531 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:28:11 smithi089 bash[33173]: cluster 2023-06-03T19:28:09.743327+0000 mgr.y (mgr.14146) 161 : cluster [DBG] pgmap v139: 1 pgs: 1 active+clean; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2023-06-03T19:28:11.585 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:28:11 smithi164 bash[26855]: cluster 2023-06-03T19:28:09.743327+0000 mgr.y (mgr.14146) 161 : cluster [DBG] pgmap v139: 1 pgs: 1 active+clean; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2023-06-03T19:28:12.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:28:12 smithi164 bash[26855]: audit 2023-06-03T19:28:11.560728+0000 mon.a (mon.0) 352 : audit [INF] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' 2023-06-03T19:28:12.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:28:12 smithi164 bash[26855]: audit 2023-06-03T19:28:11.561969+0000 mon.a (mon.0) 353 : audit [DBG] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-03T19:28:12.836 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:28:12 smithi164 bash[26855]: audit 2023-06-03T19:28:11.563052+0000 mon.a (mon.0) 354 : audit [INF] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-03T19:28:12.836 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:28:12 smithi164 bash[26855]: audit 2023-06-03T19:28:11.568587+0000 mon.a (mon.0) 355 : audit [INF] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' 2023-06-03T19:28:12.836 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:28:12 smithi164 bash[26855]: cluster 2023-06-03T19:28:11.743745+0000 mgr.y (mgr.14146) 162 : cluster [DBG] pgmap v140: 1 pgs: 1 active+clean; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2023-06-03T19:28:12.836 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:28:12 smithi164 bash[26855]: audit 2023-06-03T19:28:12.281583+0000 mon.a (mon.0) 356 : audit [INF] from='osd.3 [v2:172.21.15.89:6826/3649114317,v1:172.21.15.89:6827/3649114317]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2023-06-03T19:28:13.031 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:28:12 smithi089 bash[33173]: audit 2023-06-03T19:28:11.560728+0000 mon.a (mon.0) 352 : audit [INF] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' 2023-06-03T19:28:13.031 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:28:12 smithi089 bash[33173]: audit 2023-06-03T19:28:11.561969+0000 mon.a (mon.0) 353 : audit [DBG] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-03T19:28:13.031 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:28:12 smithi089 bash[33173]: audit 2023-06-03T19:28:11.563052+0000 mon.a (mon.0) 354 : audit [INF] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-03T19:28:13.031 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:28:12 smithi089 bash[33173]: audit 2023-06-03T19:28:11.568587+0000 mon.a (mon.0) 355 : audit [INF] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' 2023-06-03T19:28:13.031 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:28:12 smithi089 bash[33173]: cluster 2023-06-03T19:28:11.743745+0000 mgr.y (mgr.14146) 162 : cluster [DBG] pgmap v140: 1 pgs: 1 active+clean; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2023-06-03T19:28:13.032 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:28:12 smithi089 bash[33173]: audit 2023-06-03T19:28:12.281583+0000 mon.a (mon.0) 356 : audit [INF] from='osd.3 [v2:172.21.15.89:6826/3649114317,v1:172.21.15.89:6827/3649114317]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2023-06-03T19:28:13.032 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:28:12 smithi089 bash[26572]: audit 2023-06-03T19:28:11.560728+0000 mon.a (mon.0) 352 : audit [INF] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' 2023-06-03T19:28:13.032 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:28:12 smithi089 bash[26572]: audit 2023-06-03T19:28:11.561969+0000 mon.a (mon.0) 353 : audit [DBG] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-03T19:28:13.032 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:28:12 smithi089 bash[26572]: audit 2023-06-03T19:28:11.563052+0000 mon.a (mon.0) 354 : audit [INF] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-03T19:28:13.032 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:28:12 smithi089 bash[26572]: audit 2023-06-03T19:28:11.568587+0000 mon.a (mon.0) 355 : audit [INF] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' 2023-06-03T19:28:13.033 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:28:12 smithi089 bash[26572]: cluster 2023-06-03T19:28:11.743745+0000 mgr.y (mgr.14146) 162 : cluster [DBG] pgmap v140: 1 pgs: 1 active+clean; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2023-06-03T19:28:13.033 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:28:12 smithi089 bash[26572]: audit 2023-06-03T19:28:12.281583+0000 mon.a (mon.0) 356 : audit [INF] from='osd.3 [v2:172.21.15.89:6826/3649114317,v1:172.21.15.89:6827/3649114317]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2023-06-03T19:28:14.031 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:28:13 smithi089 bash[26572]: audit 2023-06-03T19:28:12.572850+0000 mon.a (mon.0) 357 : audit [INF] from='osd.3 [v2:172.21.15.89:6826/3649114317,v1:172.21.15.89:6827/3649114317]' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2023-06-03T19:28:14.031 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:28:13 smithi089 bash[26572]: cluster 2023-06-03T19:28:12.572926+0000 mon.a (mon.0) 358 : cluster [DBG] osdmap e23: 4 total, 3 up, 4 in 2023-06-03T19:28:14.031 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:28:13 smithi089 bash[26572]: audit 2023-06-03T19:28:12.573103+0000 mon.a (mon.0) 359 : audit [DBG] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-06-03T19:28:14.031 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:28:13 smithi089 bash[26572]: audit 2023-06-03T19:28:12.574153+0000 mon.a (mon.0) 360 : audit [INF] from='osd.3 [v2:172.21.15.89:6826/3649114317,v1:172.21.15.89:6827/3649114317]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi089", "root=default"]}]: dispatch 2023-06-03T19:28:14.032 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:28:13 smithi089 bash[33173]: audit 2023-06-03T19:28:12.572850+0000 mon.a (mon.0) 357 : audit [INF] from='osd.3 [v2:172.21.15.89:6826/3649114317,v1:172.21.15.89:6827/3649114317]' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2023-06-03T19:28:14.032 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:28:13 smithi089 bash[33173]: cluster 2023-06-03T19:28:12.572926+0000 mon.a (mon.0) 358 : cluster [DBG] osdmap e23: 4 total, 3 up, 4 in 2023-06-03T19:28:14.032 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:28:13 smithi089 bash[33173]: audit 2023-06-03T19:28:12.573103+0000 mon.a (mon.0) 359 : audit [DBG] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-06-03T19:28:14.032 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:28:13 smithi089 bash[33173]: audit 2023-06-03T19:28:12.574153+0000 mon.a (mon.0) 360 : audit [INF] from='osd.3 [v2:172.21.15.89:6826/3649114317,v1:172.21.15.89:6827/3649114317]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi089", "root=default"]}]: dispatch 2023-06-03T19:28:14.085 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:28:13 smithi164 bash[26855]: audit 2023-06-03T19:28:12.572850+0000 mon.a (mon.0) 357 : audit [INF] from='osd.3 [v2:172.21.15.89:6826/3649114317,v1:172.21.15.89:6827/3649114317]' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2023-06-03T19:28:14.085 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:28:13 smithi164 bash[26855]: cluster 2023-06-03T19:28:12.572926+0000 mon.a (mon.0) 358 : cluster [DBG] osdmap e23: 4 total, 3 up, 4 in 2023-06-03T19:28:14.085 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:28:13 smithi164 bash[26855]: audit 2023-06-03T19:28:12.573103+0000 mon.a (mon.0) 359 : audit [DBG] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-06-03T19:28:14.086 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:28:13 smithi164 bash[26855]: audit 2023-06-03T19:28:12.574153+0000 mon.a (mon.0) 360 : audit [INF] from='osd.3 [v2:172.21.15.89:6826/3649114317,v1:172.21.15.89:6827/3649114317]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi089", "root=default"]}]: dispatch 2023-06-03T19:28:14.906 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:28:14 smithi089 bash[26572]: audit 2023-06-03T19:28:13.580175+0000 mon.a (mon.0) 361 : audit [INF] from='osd.3 [v2:172.21.15.89:6826/3649114317,v1:172.21.15.89:6827/3649114317]' entity='osd.3' cmd='[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi089", "root=default"]}]': finished 2023-06-03T19:28:14.906 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:28:14 smithi089 bash[26572]: cluster 2023-06-03T19:28:13.580293+0000 mon.a (mon.0) 362 : cluster [DBG] osdmap e24: 4 total, 3 up, 4 in 2023-06-03T19:28:14.906 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:28:14 smithi089 bash[26572]: audit 2023-06-03T19:28:13.581075+0000 mon.a (mon.0) 363 : audit [DBG] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-06-03T19:28:14.906 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:28:14 smithi089 bash[26572]: audit 2023-06-03T19:28:13.587940+0000 mon.a (mon.0) 364 : audit [DBG] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-06-03T19:28:14.906 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:28:14 smithi089 bash[26572]: cluster 2023-06-03T19:28:13.744223+0000 mgr.y (mgr.14146) 163 : cluster [DBG] pgmap v143: 1 pgs: 1 active+clean; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2023-06-03T19:28:14.907 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:28:14 smithi089 bash[33173]: audit 2023-06-03T19:28:13.580175+0000 mon.a (mon.0) 361 : audit [INF] from='osd.3 [v2:172.21.15.89:6826/3649114317,v1:172.21.15.89:6827/3649114317]' entity='osd.3' cmd='[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi089", "root=default"]}]': finished 2023-06-03T19:28:14.907 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:28:14 smithi089 bash[33173]: cluster 2023-06-03T19:28:13.580293+0000 mon.a (mon.0) 362 : cluster [DBG] osdmap e24: 4 total, 3 up, 4 in 2023-06-03T19:28:14.907 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:28:14 smithi089 bash[33173]: audit 2023-06-03T19:28:13.581075+0000 mon.a (mon.0) 363 : audit [DBG] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-06-03T19:28:14.907 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:28:14 smithi089 bash[33173]: audit 2023-06-03T19:28:13.587940+0000 mon.a (mon.0) 364 : audit [DBG] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-06-03T19:28:14.907 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:28:14 smithi089 bash[33173]: cluster 2023-06-03T19:28:13.744223+0000 mgr.y (mgr.14146) 163 : cluster [DBG] pgmap v143: 1 pgs: 1 active+clean; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2023-06-03T19:28:15.085 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:28:14 smithi164 bash[26855]: audit 2023-06-03T19:28:13.580175+0000 mon.a (mon.0) 361 : audit [INF] from='osd.3 [v2:172.21.15.89:6826/3649114317,v1:172.21.15.89:6827/3649114317]' entity='osd.3' cmd='[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi089", "root=default"]}]': finished 2023-06-03T19:28:15.085 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:28:14 smithi164 bash[26855]: cluster 2023-06-03T19:28:13.580293+0000 mon.a (mon.0) 362 : cluster [DBG] osdmap e24: 4 total, 3 up, 4 in 2023-06-03T19:28:15.086 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:28:14 smithi164 bash[26855]: audit 2023-06-03T19:28:13.581075+0000 mon.a (mon.0) 363 : audit [DBG] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-06-03T19:28:15.086 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:28:14 smithi164 bash[26855]: audit 2023-06-03T19:28:13.587940+0000 mon.a (mon.0) 364 : audit [DBG] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-06-03T19:28:15.086 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:28:14 smithi164 bash[26855]: cluster 2023-06-03T19:28:13.744223+0000 mgr.y (mgr.14146) 163 : cluster [DBG] pgmap v143: 1 pgs: 1 active+clean; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2023-06-03T19:28:15.623 INFO:teuthology.orchestra.run.smithi089.stdout:Created osd(s) 3 on host 'smithi089' 2023-06-03T19:28:16.030 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:28:15 smithi089 bash[26572]: cluster 2023-06-03T19:28:13.272487+0000 osd.3 (osd.3) 1 : cluster [DBG] purged_snaps scrub starts 2023-06-03T19:28:16.031 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:28:15 smithi089 bash[26572]: cluster 2023-06-03T19:28:13.272579+0000 osd.3 (osd.3) 2 : cluster [DBG] purged_snaps scrub ok 2023-06-03T19:28:16.031 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:28:15 smithi089 bash[26572]: cluster 2023-06-03T19:28:14.591705+0000 mon.a (mon.0) 365 : cluster [INF] osd.3 [v2:172.21.15.89:6826/3649114317,v1:172.21.15.89:6827/3649114317] boot 2023-06-03T19:28:16.031 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:28:15 smithi089 bash[26572]: cluster 2023-06-03T19:28:14.591754+0000 mon.a (mon.0) 366 : cluster [DBG] osdmap e25: 4 total, 4 up, 4 in 2023-06-03T19:28:16.031 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:28:15 smithi089 bash[26572]: audit 2023-06-03T19:28:14.592942+0000 mon.a (mon.0) 367 : audit [DBG] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-06-03T19:28:16.031 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:28:15 smithi089 bash[33173]: cluster 2023-06-03T19:28:13.272487+0000 osd.3 (osd.3) 1 : cluster [DBG] purged_snaps scrub starts 2023-06-03T19:28:16.032 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:28:15 smithi089 bash[33173]: cluster 2023-06-03T19:28:13.272579+0000 osd.3 (osd.3) 2 : cluster [DBG] purged_snaps scrub ok 2023-06-03T19:28:16.032 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:28:15 smithi089 bash[33173]: cluster 2023-06-03T19:28:14.591705+0000 mon.a (mon.0) 365 : cluster [INF] osd.3 [v2:172.21.15.89:6826/3649114317,v1:172.21.15.89:6827/3649114317] boot 2023-06-03T19:28:16.032 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:28:15 smithi089 bash[33173]: cluster 2023-06-03T19:28:14.591754+0000 mon.a (mon.0) 366 : cluster [DBG] osdmap e25: 4 total, 4 up, 4 in 2023-06-03T19:28:16.032 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:28:15 smithi089 bash[33173]: audit 2023-06-03T19:28:14.592942+0000 mon.a (mon.0) 367 : audit [DBG] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-06-03T19:28:16.085 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:28:15 smithi164 bash[26855]: cluster 2023-06-03T19:28:13.272487+0000 osd.3 (osd.3) 1 : cluster [DBG] purged_snaps scrub starts 2023-06-03T19:28:16.085 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:28:15 smithi164 bash[26855]: cluster 2023-06-03T19:28:13.272579+0000 osd.3 (osd.3) 2 : cluster [DBG] purged_snaps scrub ok 2023-06-03T19:28:16.085 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:28:15 smithi164 bash[26855]: cluster 2023-06-03T19:28:14.591705+0000 mon.a (mon.0) 365 : cluster [INF] osd.3 [v2:172.21.15.89:6826/3649114317,v1:172.21.15.89:6827/3649114317] boot 2023-06-03T19:28:16.086 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:28:15 smithi164 bash[26855]: cluster 2023-06-03T19:28:14.591754+0000 mon.a (mon.0) 366 : cluster [DBG] osdmap e25: 4 total, 4 up, 4 in 2023-06-03T19:28:16.086 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:28:15 smithi164 bash[26855]: audit 2023-06-03T19:28:14.592942+0000 mon.a (mon.0) 367 : audit [DBG] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-06-03T19:28:16.410 DEBUG:teuthology.orchestra.run.smithi089:osd.3> sudo journalctl -f -n 0 -u ceph-f1db3478-0243-11ee-9b1d-001a4aab830c@osd.3.service 2023-06-03T19:28:16.412 INFO:tasks.cephadm:Deploying osd.4 on smithi164 with /dev/vg_nvme/lv_4... 2023-06-03T19:28:16.413 DEBUG:teuthology.orchestra.run.smithi164:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph-volume lvm zap /dev/vg_nvme/lv_4 2023-06-03T19:28:16.690 INFO:journalctl@ceph.osd.3.smithi089.stdout:-- Logs begin at Sun 2022-08-14 12:20:24 UTC. -- 2023-06-03T19:28:16.781 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:28:16 smithi089 bash[33173]: cluster 2023-06-03T19:28:15.609305+0000 mon.a (mon.0) 368 : cluster [DBG] osdmap e26: 4 total, 4 up, 4 in 2023-06-03T19:28:16.781 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:28:16 smithi089 bash[33173]: audit 2023-06-03T19:28:15.615937+0000 mon.a (mon.0) 369 : audit [INF] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' 2023-06-03T19:28:16.781 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:28:16 smithi089 bash[33173]: audit 2023-06-03T19:28:15.645595+0000 mon.a (mon.0) 370 : audit [DBG] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-03T19:28:16.782 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:28:16 smithi089 bash[33173]: cluster 2023-06-03T19:28:15.744610+0000 mgr.y (mgr.14146) 164 : cluster [DBG] pgmap v146: 1 pgs: 1 active+clean; 0 B data, 19 MiB used, 358 GiB / 358 GiB avail 2023-06-03T19:28:16.782 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:28:16 smithi089 bash[26572]: cluster 2023-06-03T19:28:15.609305+0000 mon.a (mon.0) 368 : cluster [DBG] osdmap e26: 4 total, 4 up, 4 in 2023-06-03T19:28:16.782 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:28:16 smithi089 bash[26572]: audit 2023-06-03T19:28:15.615937+0000 mon.a (mon.0) 369 : audit [INF] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' 2023-06-03T19:28:16.782 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:28:16 smithi089 bash[26572]: audit 2023-06-03T19:28:15.645595+0000 mon.a (mon.0) 370 : audit [DBG] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-03T19:28:16.783 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:28:16 smithi089 bash[26572]: cluster 2023-06-03T19:28:15.744610+0000 mgr.y (mgr.14146) 164 : cluster [DBG] pgmap v146: 1 pgs: 1 active+clean; 0 B data, 19 MiB used, 358 GiB / 358 GiB avail 2023-06-03T19:28:16.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:28:16 smithi164 bash[26855]: cluster 2023-06-03T19:28:15.609305+0000 mon.a (mon.0) 368 : cluster [DBG] osdmap e26: 4 total, 4 up, 4 in 2023-06-03T19:28:16.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:28:16 smithi164 bash[26855]: audit 2023-06-03T19:28:15.615937+0000 mon.a (mon.0) 369 : audit [INF] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' 2023-06-03T19:28:16.836 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:28:16 smithi164 bash[26855]: audit 2023-06-03T19:28:15.645595+0000 mon.a (mon.0) 370 : audit [DBG] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-03T19:28:16.836 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:28:16 smithi164 bash[26855]: cluster 2023-06-03T19:28:15.744610+0000 mgr.y (mgr.14146) 164 : cluster [DBG] pgmap v146: 1 pgs: 1 active+clean; 0 B data, 19 MiB used, 358 GiB / 358 GiB avail 2023-06-03T19:28:19.085 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:28:18 smithi164 bash[26855]: cluster 2023-06-03T19:28:17.745110+0000 mgr.y (mgr.14146) 165 : cluster [DBG] pgmap v147: 1 pgs: 1 active+clean; 0 B data, 20 MiB used, 358 GiB / 358 GiB avail 2023-06-03T19:28:19.281 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:28:18 smithi089 bash[26572]: cluster 2023-06-03T19:28:17.745110+0000 mgr.y (mgr.14146) 165 : cluster [DBG] pgmap v147: 1 pgs: 1 active+clean; 0 B data, 20 MiB used, 358 GiB / 358 GiB avail 2023-06-03T19:28:19.281 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:28:18 smithi089 bash[33173]: cluster 2023-06-03T19:28:17.745110+0000 mgr.y (mgr.14146) 165 : cluster [DBG] pgmap v147: 1 pgs: 1 active+clean; 0 B data, 20 MiB used, 358 GiB / 358 GiB avail 2023-06-03T19:28:19.804 INFO:teuthology.orchestra.run.smithi164.stderr:--> Zapping: /dev/vg_nvme/lv_4 2023-06-03T19:28:19.804 INFO:teuthology.orchestra.run.smithi164.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_4 bs=1M count=10 conv=fsync 2023-06-03T19:28:19.804 INFO:teuthology.orchestra.run.smithi164.stderr: stderr: 10+0 records in 2023-06-03T19:28:19.804 INFO:teuthology.orchestra.run.smithi164.stderr:10+0 records out 2023-06-03T19:28:19.805 INFO:teuthology.orchestra.run.smithi164.stderr: stderr: 10485760 bytes (10 MB, 10 MiB) copied, 0.0174411 s, 601 MB/s 2023-06-03T19:28:19.805 INFO:teuthology.orchestra.run.smithi164.stderr:--> Zapping successful for: 2023-06-03T19:28:20.475 DEBUG:teuthology.orchestra.run.smithi164:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph orch daemon add osd smithi164:vg_nvme/lv_4 2023-06-03T19:28:21.085 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:28:20 smithi164 bash[26855]: cluster 2023-06-03T19:28:19.745652+0000 mgr.y (mgr.14146) 166 : cluster [DBG] pgmap v148: 1 pgs: 1 active+clean; 0 B data, 20 MiB used, 358 GiB / 358 GiB avail 2023-06-03T19:28:21.280 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:28:20 smithi089 bash[26572]: cluster 2023-06-03T19:28:19.745652+0000 mgr.y (mgr.14146) 166 : cluster [DBG] pgmap v148: 1 pgs: 1 active+clean; 0 B data, 20 MiB used, 358 GiB / 358 GiB avail 2023-06-03T19:28:21.281 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:28:20 smithi089 bash[33173]: cluster 2023-06-03T19:28:19.745652+0000 mgr.y (mgr.14146) 166 : cluster [DBG] pgmap v148: 1 pgs: 1 active+clean; 0 B data, 20 MiB used, 358 GiB / 358 GiB avail 2023-06-03T19:28:22.781 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:28:22 smithi089 bash[26572]: cluster 2023-06-03T19:28:21.746112+0000 mgr.y (mgr.14146) 167 : cluster [DBG] pgmap v149: 1 pgs: 1 active+clean; 0 B data, 20 MiB used, 358 GiB / 358 GiB avail 2023-06-03T19:28:22.781 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:28:22 smithi089 bash[33173]: cluster 2023-06-03T19:28:21.746112+0000 mgr.y (mgr.14146) 167 : cluster [DBG] pgmap v149: 1 pgs: 1 active+clean; 0 B data, 20 MiB used, 358 GiB / 358 GiB avail 2023-06-03T19:28:22.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:28:22 smithi164 bash[26855]: cluster 2023-06-03T19:28:21.746112+0000 mgr.y (mgr.14146) 167 : cluster [DBG] pgmap v149: 1 pgs: 1 active+clean; 0 B data, 20 MiB used, 358 GiB / 358 GiB avail 2023-06-03T19:28:23.781 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:28:23 smithi089 bash[26572]: audit 2023-06-03T19:28:23.572181+0000 mon.a (mon.0) 371 : audit [DBG] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-06-03T19:28:23.781 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:28:23 smithi089 bash[26572]: audit 2023-06-03T19:28:23.577181+0000 mon.a (mon.0) 372 : audit [INF] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-06-03T19:28:23.781 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:28:23 smithi089 bash[26572]: audit 2023-06-03T19:28:23.578396+0000 mon.a (mon.0) 373 : audit [DBG] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-03T19:28:23.782 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:28:23 smithi089 bash[33173]: audit 2023-06-03T19:28:23.572181+0000 mon.a (mon.0) 371 : audit [DBG] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-06-03T19:28:23.782 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:28:23 smithi089 bash[33173]: audit 2023-06-03T19:28:23.577181+0000 mon.a (mon.0) 372 : audit [INF] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-06-03T19:28:23.782 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:28:23 smithi089 bash[33173]: audit 2023-06-03T19:28:23.578396+0000 mon.a (mon.0) 373 : audit [DBG] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-03T19:28:24.085 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:28:23 smithi164 bash[26855]: audit 2023-06-03T19:28:23.572181+0000 mon.a (mon.0) 371 : audit [DBG] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-06-03T19:28:24.085 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:28:23 smithi164 bash[26855]: audit 2023-06-03T19:28:23.577181+0000 mon.a (mon.0) 372 : audit [INF] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-06-03T19:28:24.086 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:28:23 smithi164 bash[26855]: audit 2023-06-03T19:28:23.578396+0000 mon.a (mon.0) 373 : audit [DBG] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-03T19:28:24.781 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:28:24 smithi089 bash[33173]: audit 2023-06-03T19:28:23.565881+0000 mgr.y (mgr.14146) 168 : audit [DBG] from='client.24223 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi164:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2023-06-03T19:28:24.781 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:28:24 smithi089 bash[33173]: cluster 2023-06-03T19:28:23.746643+0000 mgr.y (mgr.14146) 169 : cluster [DBG] pgmap v150: 1 pgs: 1 active+clean; 0 B data, 20 MiB used, 358 GiB / 358 GiB avail 2023-06-03T19:28:24.782 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:28:24 smithi089 bash[26572]: audit 2023-06-03T19:28:23.565881+0000 mgr.y (mgr.14146) 168 : audit [DBG] from='client.24223 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi164:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2023-06-03T19:28:24.782 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:28:24 smithi089 bash[26572]: cluster 2023-06-03T19:28:23.746643+0000 mgr.y (mgr.14146) 169 : cluster [DBG] pgmap v150: 1 pgs: 1 active+clean; 0 B data, 20 MiB used, 358 GiB / 358 GiB avail 2023-06-03T19:28:25.085 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:28:24 smithi164 bash[26855]: audit 2023-06-03T19:28:23.565881+0000 mgr.y (mgr.14146) 168 : audit [DBG] from='client.24223 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi164:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2023-06-03T19:28:25.085 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:28:24 smithi164 bash[26855]: cluster 2023-06-03T19:28:23.746643+0000 mgr.y (mgr.14146) 169 : cluster [DBG] pgmap v150: 1 pgs: 1 active+clean; 0 B data, 20 MiB used, 358 GiB / 358 GiB avail 2023-06-03T19:28:26.531 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:28:26 smithi089 bash[26572]: cephadm 2023-06-03T19:28:25.152338+0000 mgr.y (mgr.14146) 170 : cephadm [INF] Detected new or changed devices on smithi089 2023-06-03T19:28:26.532 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:28:26 smithi089 bash[26572]: audit 2023-06-03T19:28:25.163552+0000 mon.a (mon.0) 374 : audit [INF] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' 2023-06-03T19:28:26.532 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:28:26 smithi089 bash[26572]: audit 2023-06-03T19:28:25.165300+0000 mon.a (mon.0) 375 : audit [INF] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-06-03T19:28:26.532 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:28:26 smithi089 bash[26572]: audit 2023-06-03T19:28:25.166581+0000 mon.a (mon.0) 376 : audit [INF] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-06-03T19:28:26.532 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:28:26 smithi089 bash[26572]: audit 2023-06-03T19:28:25.167706+0000 mon.a (mon.0) 377 : audit [INF] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2023-06-03T19:28:26.532 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:28:26 smithi089 bash[26572]: audit 2023-06-03T19:28:25.168833+0000 mon.a (mon.0) 378 : audit [INF] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2023-06-03T19:28:26.532 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:28:26 smithi089 bash[26572]: cephadm 2023-06-03T19:28:25.169722+0000 mgr.y (mgr.14146) 171 : cephadm [INF] Adjusting osd_memory_target on smithi089 to 4062M 2023-06-03T19:28:26.533 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:28:26 smithi089 bash[26572]: audit 2023-06-03T19:28:25.176697+0000 mon.a (mon.0) 379 : audit [INF] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' 2023-06-03T19:28:26.533 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:28:26 smithi089 bash[26572]: audit 2023-06-03T19:28:25.178141+0000 mon.a (mon.0) 380 : audit [DBG] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-03T19:28:26.533 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:28:26 smithi089 bash[26572]: audit 2023-06-03T19:28:25.179265+0000 mon.a (mon.0) 381 : audit [INF] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-03T19:28:26.533 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:28:26 smithi089 bash[26572]: audit 2023-06-03T19:28:25.186979+0000 mon.a (mon.0) 382 : audit [INF] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' 2023-06-03T19:28:26.533 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:28:26 smithi089 bash[26572]: cluster 2023-06-03T19:28:25.747313+0000 mgr.y (mgr.14146) 172 : cluster [DBG] pgmap v151: 1 pgs: 1 active+clean; 0 B data, 20 MiB used, 358 GiB / 358 GiB avail 2023-06-03T19:28:26.533 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:28:26 smithi089 bash[33173]: cephadm 2023-06-03T19:28:25.152338+0000 mgr.y (mgr.14146) 170 : cephadm [INF] Detected new or changed devices on smithi089 2023-06-03T19:28:26.534 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:28:26 smithi089 bash[33173]: audit 2023-06-03T19:28:25.163552+0000 mon.a (mon.0) 374 : audit [INF] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' 2023-06-03T19:28:26.534 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:28:26 smithi089 bash[33173]: audit 2023-06-03T19:28:25.165300+0000 mon.a (mon.0) 375 : audit [INF] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-06-03T19:28:26.534 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:28:26 smithi089 bash[33173]: audit 2023-06-03T19:28:25.166581+0000 mon.a (mon.0) 376 : audit [INF] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-06-03T19:28:26.534 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:28:26 smithi089 bash[33173]: audit 2023-06-03T19:28:25.167706+0000 mon.a (mon.0) 377 : audit [INF] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2023-06-03T19:28:26.534 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:28:26 smithi089 bash[33173]: audit 2023-06-03T19:28:25.168833+0000 mon.a (mon.0) 378 : audit [INF] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2023-06-03T19:28:26.534 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:28:26 smithi089 bash[33173]: cephadm 2023-06-03T19:28:25.169722+0000 mgr.y (mgr.14146) 171 : cephadm [INF] Adjusting osd_memory_target on smithi089 to 4062M 2023-06-03T19:28:26.534 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:28:26 smithi089 bash[33173]: audit 2023-06-03T19:28:25.176697+0000 mon.a (mon.0) 379 : audit [INF] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' 2023-06-03T19:28:26.535 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:28:26 smithi089 bash[33173]: audit 2023-06-03T19:28:25.178141+0000 mon.a (mon.0) 380 : audit [DBG] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-03T19:28:26.535 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:28:26 smithi089 bash[33173]: audit 2023-06-03T19:28:25.179265+0000 mon.a (mon.0) 381 : audit [INF] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-03T19:28:26.535 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:28:26 smithi089 bash[33173]: audit 2023-06-03T19:28:25.186979+0000 mon.a (mon.0) 382 : audit [INF] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' 2023-06-03T19:28:26.535 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:28:26 smithi089 bash[33173]: cluster 2023-06-03T19:28:25.747313+0000 mgr.y (mgr.14146) 172 : cluster [DBG] pgmap v151: 1 pgs: 1 active+clean; 0 B data, 20 MiB used, 358 GiB / 358 GiB avail 2023-06-03T19:28:26.585 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:28:26 smithi164 bash[26855]: cephadm 2023-06-03T19:28:25.152338+0000 mgr.y (mgr.14146) 170 : cephadm [INF] Detected new or changed devices on smithi089 2023-06-03T19:28:26.586 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:28:26 smithi164 bash[26855]: audit 2023-06-03T19:28:25.163552+0000 mon.a (mon.0) 374 : audit [INF] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' 2023-06-03T19:28:26.586 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:28:26 smithi164 bash[26855]: audit 2023-06-03T19:28:25.165300+0000 mon.a (mon.0) 375 : audit [INF] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-06-03T19:28:26.586 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:28:26 smithi164 bash[26855]: audit 2023-06-03T19:28:25.166581+0000 mon.a (mon.0) 376 : audit [INF] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-06-03T19:28:26.586 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:28:26 smithi164 bash[26855]: audit 2023-06-03T19:28:25.167706+0000 mon.a (mon.0) 377 : audit [INF] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2023-06-03T19:28:26.586 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:28:26 smithi164 bash[26855]: audit 2023-06-03T19:28:25.168833+0000 mon.a (mon.0) 378 : audit [INF] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2023-06-03T19:28:26.586 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:28:26 smithi164 bash[26855]: cephadm 2023-06-03T19:28:25.169722+0000 mgr.y (mgr.14146) 171 : cephadm [INF] Adjusting osd_memory_target on smithi089 to 4062M 2023-06-03T19:28:26.587 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:28:26 smithi164 bash[26855]: audit 2023-06-03T19:28:25.176697+0000 mon.a (mon.0) 379 : audit [INF] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' 2023-06-03T19:28:26.587 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:28:26 smithi164 bash[26855]: audit 2023-06-03T19:28:25.178141+0000 mon.a (mon.0) 380 : audit [DBG] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-03T19:28:26.587 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:28:26 smithi164 bash[26855]: audit 2023-06-03T19:28:25.179265+0000 mon.a (mon.0) 381 : audit [INF] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-03T19:28:26.587 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:28:26 smithi164 bash[26855]: audit 2023-06-03T19:28:25.186979+0000 mon.a (mon.0) 382 : audit [INF] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' 2023-06-03T19:28:26.587 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:28:26 smithi164 bash[26855]: cluster 2023-06-03T19:28:25.747313+0000 mgr.y (mgr.14146) 172 : cluster [DBG] pgmap v151: 1 pgs: 1 active+clean; 0 B data, 20 MiB used, 358 GiB / 358 GiB avail 2023-06-03T19:28:29.085 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:28:28 smithi164 bash[26855]: cluster 2023-06-03T19:28:27.747926+0000 mgr.y (mgr.14146) 173 : cluster [DBG] pgmap v152: 1 pgs: 1 active+clean; 0 B data, 20 MiB used, 358 GiB / 358 GiB avail 2023-06-03T19:28:29.281 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:28:28 smithi089 bash[26572]: cluster 2023-06-03T19:28:27.747926+0000 mgr.y (mgr.14146) 173 : cluster [DBG] pgmap v152: 1 pgs: 1 active+clean; 0 B data, 20 MiB used, 358 GiB / 358 GiB avail 2023-06-03T19:28:29.281 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:28:28 smithi089 bash[33173]: cluster 2023-06-03T19:28:27.747926+0000 mgr.y (mgr.14146) 173 : cluster [DBG] pgmap v152: 1 pgs: 1 active+clean; 0 B data, 20 MiB used, 358 GiB / 358 GiB avail 2023-06-03T19:28:31.085 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:28:30 smithi164 bash[26855]: cluster 2023-06-03T19:28:29.748487+0000 mgr.y (mgr.14146) 174 : cluster [DBG] pgmap v153: 1 pgs: 1 active+clean; 0 B data, 20 MiB used, 358 GiB / 358 GiB avail 2023-06-03T19:28:31.281 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:28:30 smithi089 bash[26572]: cluster 2023-06-03T19:28:29.748487+0000 mgr.y (mgr.14146) 174 : cluster [DBG] pgmap v153: 1 pgs: 1 active+clean; 0 B data, 20 MiB used, 358 GiB / 358 GiB avail 2023-06-03T19:28:31.281 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:28:30 smithi089 bash[33173]: cluster 2023-06-03T19:28:29.748487+0000 mgr.y (mgr.14146) 174 : cluster [DBG] pgmap v153: 1 pgs: 1 active+clean; 0 B data, 20 MiB used, 358 GiB / 358 GiB avail 2023-06-03T19:28:32.781 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:28:32 smithi089 bash[33173]: cluster 2023-06-03T19:28:31.749080+0000 mgr.y (mgr.14146) 175 : cluster [DBG] pgmap v154: 1 pgs: 1 active+clean; 0 B data, 20 MiB used, 358 GiB / 358 GiB avail 2023-06-03T19:28:32.781 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:28:32 smithi089 bash[33173]: audit 2023-06-03T19:28:31.766360+0000 mon.b (mon.1) 3 : audit [INF] from='client.? 172.21.15.164:0/1488176482' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "a94be785-9f54-4ab8-962a-9cf974a5b5e3"}]: dispatch 2023-06-03T19:28:32.781 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:28:32 smithi089 bash[33173]: audit 2023-06-03T19:28:31.768812+0000 mon.a (mon.0) 383 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "a94be785-9f54-4ab8-962a-9cf974a5b5e3"}]: dispatch 2023-06-03T19:28:32.781 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:28:32 smithi089 bash[33173]: audit 2023-06-03T19:28:31.777291+0000 mon.a (mon.0) 384 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "a94be785-9f54-4ab8-962a-9cf974a5b5e3"}]': finished 2023-06-03T19:28:32.781 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:28:32 smithi089 bash[33173]: cluster 2023-06-03T19:28:31.777351+0000 mon.a (mon.0) 385 : cluster [DBG] osdmap e27: 5 total, 4 up, 5 in 2023-06-03T19:28:32.782 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:28:32 smithi089 bash[33173]: audit 2023-06-03T19:28:31.777552+0000 mon.a (mon.0) 386 : audit [DBG] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-06-03T19:28:32.782 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:28:32 smithi089 bash[26572]: cluster 2023-06-03T19:28:31.749080+0000 mgr.y (mgr.14146) 175 : cluster [DBG] pgmap v154: 1 pgs: 1 active+clean; 0 B data, 20 MiB used, 358 GiB / 358 GiB avail 2023-06-03T19:28:32.782 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:28:32 smithi089 bash[26572]: audit 2023-06-03T19:28:31.766360+0000 mon.b (mon.1) 3 : audit [INF] from='client.? 172.21.15.164:0/1488176482' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "a94be785-9f54-4ab8-962a-9cf974a5b5e3"}]: dispatch 2023-06-03T19:28:32.782 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:28:32 smithi089 bash[26572]: audit 2023-06-03T19:28:31.768812+0000 mon.a (mon.0) 383 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "a94be785-9f54-4ab8-962a-9cf974a5b5e3"}]: dispatch 2023-06-03T19:28:32.782 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:28:32 smithi089 bash[26572]: audit 2023-06-03T19:28:31.777291+0000 mon.a (mon.0) 384 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "a94be785-9f54-4ab8-962a-9cf974a5b5e3"}]': finished 2023-06-03T19:28:32.783 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:28:32 smithi089 bash[26572]: cluster 2023-06-03T19:28:31.777351+0000 mon.a (mon.0) 385 : cluster [DBG] osdmap e27: 5 total, 4 up, 5 in 2023-06-03T19:28:32.783 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:28:32 smithi089 bash[26572]: audit 2023-06-03T19:28:31.777552+0000 mon.a (mon.0) 386 : audit [DBG] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-06-03T19:28:32.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:28:32 smithi164 bash[26855]: cluster 2023-06-03T19:28:31.749080+0000 mgr.y (mgr.14146) 175 : cluster [DBG] pgmap v154: 1 pgs: 1 active+clean; 0 B data, 20 MiB used, 358 GiB / 358 GiB avail 2023-06-03T19:28:32.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:28:32 smithi164 bash[26855]: audit 2023-06-03T19:28:31.766360+0000 mon.b (mon.1) 3 : audit [INF] from='client.? 172.21.15.164:0/1488176482' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "a94be785-9f54-4ab8-962a-9cf974a5b5e3"}]: dispatch 2023-06-03T19:28:32.836 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:28:32 smithi164 bash[26855]: audit 2023-06-03T19:28:31.768812+0000 mon.a (mon.0) 383 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "a94be785-9f54-4ab8-962a-9cf974a5b5e3"}]: dispatch 2023-06-03T19:28:32.836 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:28:32 smithi164 bash[26855]: audit 2023-06-03T19:28:31.777291+0000 mon.a (mon.0) 384 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "a94be785-9f54-4ab8-962a-9cf974a5b5e3"}]': finished 2023-06-03T19:28:32.836 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:28:32 smithi164 bash[26855]: cluster 2023-06-03T19:28:31.777351+0000 mon.a (mon.0) 385 : cluster [DBG] osdmap e27: 5 total, 4 up, 5 in 2023-06-03T19:28:32.836 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:28:32 smithi164 bash[26855]: audit 2023-06-03T19:28:31.777552+0000 mon.a (mon.0) 386 : audit [DBG] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-06-03T19:28:33.781 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:28:33 smithi089 bash[26572]: audit 2023-06-03T19:28:32.441755+0000 mon.b (mon.1) 4 : audit [DBG] from='client.? 172.21.15.164:0/4028194818' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-06-03T19:28:33.781 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:28:33 smithi089 bash[33173]: audit 2023-06-03T19:28:32.441755+0000 mon.b (mon.1) 4 : audit [DBG] from='client.? 172.21.15.164:0/4028194818' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-06-03T19:28:33.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:28:33 smithi164 bash[26855]: audit 2023-06-03T19:28:32.441755+0000 mon.b (mon.1) 4 : audit [DBG] from='client.? 172.21.15.164:0/4028194818' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-06-03T19:28:34.781 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:28:34 smithi089 bash[26572]: cluster 2023-06-03T19:28:33.749615+0000 mgr.y (mgr.14146) 176 : cluster [DBG] pgmap v156: 1 pgs: 1 active+clean; 0 B data, 20 MiB used, 358 GiB / 358 GiB avail 2023-06-03T19:28:34.781 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:28:34 smithi089 bash[33173]: cluster 2023-06-03T19:28:33.749615+0000 mgr.y (mgr.14146) 176 : cluster [DBG] pgmap v156: 1 pgs: 1 active+clean; 0 B data, 20 MiB used, 358 GiB / 358 GiB avail 2023-06-03T19:28:34.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:28:34 smithi164 bash[26855]: cluster 2023-06-03T19:28:33.749615+0000 mgr.y (mgr.14146) 176 : cluster [DBG] pgmap v156: 1 pgs: 1 active+clean; 0 B data, 20 MiB used, 358 GiB / 358 GiB avail 2023-06-03T19:28:37.085 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:28:36 smithi164 bash[26855]: cluster 2023-06-03T19:28:35.750139+0000 mgr.y (mgr.14146) 177 : cluster [DBG] pgmap v157: 1 pgs: 1 active+clean; 0 B data, 20 MiB used, 358 GiB / 358 GiB avail 2023-06-03T19:28:37.281 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:28:36 smithi089 bash[33173]: cluster 2023-06-03T19:28:35.750139+0000 mgr.y (mgr.14146) 177 : cluster [DBG] pgmap v157: 1 pgs: 1 active+clean; 0 B data, 20 MiB used, 358 GiB / 358 GiB avail 2023-06-03T19:28:37.281 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:28:36 smithi089 bash[26572]: cluster 2023-06-03T19:28:35.750139+0000 mgr.y (mgr.14146) 177 : cluster [DBG] pgmap v157: 1 pgs: 1 active+clean; 0 B data, 20 MiB used, 358 GiB / 358 GiB avail 2023-06-03T19:28:39.085 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:28:38 smithi164 bash[26855]: cluster 2023-06-03T19:28:37.750678+0000 mgr.y (mgr.14146) 178 : cluster [DBG] pgmap v158: 1 pgs: 1 active+clean; 0 B data, 20 MiB used, 358 GiB / 358 GiB avail 2023-06-03T19:28:39.281 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:28:38 smithi089 bash[33173]: cluster 2023-06-03T19:28:37.750678+0000 mgr.y (mgr.14146) 178 : cluster [DBG] pgmap v158: 1 pgs: 1 active+clean; 0 B data, 20 MiB used, 358 GiB / 358 GiB avail 2023-06-03T19:28:39.281 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:28:38 smithi089 bash[26572]: cluster 2023-06-03T19:28:37.750678+0000 mgr.y (mgr.14146) 178 : cluster [DBG] pgmap v158: 1 pgs: 1 active+clean; 0 B data, 20 MiB used, 358 GiB / 358 GiB avail 2023-06-03T19:28:41.085 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:28:40 smithi164 bash[26855]: cluster 2023-06-03T19:28:39.751203+0000 mgr.y (mgr.14146) 179 : cluster [DBG] pgmap v159: 1 pgs: 1 active+clean; 0 B data, 20 MiB used, 358 GiB / 358 GiB avail 2023-06-03T19:28:41.281 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:28:40 smithi089 bash[26572]: cluster 2023-06-03T19:28:39.751203+0000 mgr.y (mgr.14146) 179 : cluster [DBG] pgmap v159: 1 pgs: 1 active+clean; 0 B data, 20 MiB used, 358 GiB / 358 GiB avail 2023-06-03T19:28:41.281 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:28:40 smithi089 bash[33173]: cluster 2023-06-03T19:28:39.751203+0000 mgr.y (mgr.14146) 179 : cluster [DBG] pgmap v159: 1 pgs: 1 active+clean; 0 B data, 20 MiB used, 358 GiB / 358 GiB avail 2023-06-03T19:28:42.585 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:28:42 smithi164 bash[26855]: cluster 2023-06-03T19:28:41.751763+0000 mgr.y (mgr.14146) 180 : cluster [DBG] pgmap v160: 1 pgs: 1 active+clean; 0 B data, 20 MiB used, 358 GiB / 358 GiB avail 2023-06-03T19:28:42.780 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:28:42 smithi089 bash[26572]: cluster 2023-06-03T19:28:41.751763+0000 mgr.y (mgr.14146) 180 : cluster [DBG] pgmap v160: 1 pgs: 1 active+clean; 0 B data, 20 MiB used, 358 GiB / 358 GiB avail 2023-06-03T19:28:42.781 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:28:42 smithi089 bash[33173]: cluster 2023-06-03T19:28:41.751763+0000 mgr.y (mgr.14146) 180 : cluster [DBG] pgmap v160: 1 pgs: 1 active+clean; 0 B data, 20 MiB used, 358 GiB / 358 GiB avail 2023-06-03T19:28:45.085 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:28:44 smithi164 bash[26855]: cluster 2023-06-03T19:28:43.752307+0000 mgr.y (mgr.14146) 181 : cluster [DBG] pgmap v161: 1 pgs: 1 active+clean; 0 B data, 20 MiB used, 358 GiB / 358 GiB avail 2023-06-03T19:28:45.085 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:28:44 smithi164 bash[26855]: audit 2023-06-03T19:28:44.228497+0000 mon.a (mon.0) 387 : audit [INF] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2023-06-03T19:28:45.086 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:28:44 smithi164 bash[26855]: audit 2023-06-03T19:28:44.230326+0000 mon.a (mon.0) 388 : audit [DBG] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-03T19:28:45.281 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:28:44 smithi089 bash[33173]: cluster 2023-06-03T19:28:43.752307+0000 mgr.y (mgr.14146) 181 : cluster [DBG] pgmap v161: 1 pgs: 1 active+clean; 0 B data, 20 MiB used, 358 GiB / 358 GiB avail 2023-06-03T19:28:45.281 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:28:44 smithi089 bash[33173]: audit 2023-06-03T19:28:44.228497+0000 mon.a (mon.0) 387 : audit [INF] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2023-06-03T19:28:45.281 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:28:44 smithi089 bash[33173]: audit 2023-06-03T19:28:44.230326+0000 mon.a (mon.0) 388 : audit [DBG] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-03T19:28:45.281 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:28:44 smithi089 bash[26572]: cluster 2023-06-03T19:28:43.752307+0000 mgr.y (mgr.14146) 181 : cluster [DBG] pgmap v161: 1 pgs: 1 active+clean; 0 B data, 20 MiB used, 358 GiB / 358 GiB avail 2023-06-03T19:28:45.282 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:28:44 smithi089 bash[26572]: audit 2023-06-03T19:28:44.228497+0000 mon.a (mon.0) 387 : audit [INF] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2023-06-03T19:28:45.282 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:28:44 smithi089 bash[26572]: audit 2023-06-03T19:28:44.230326+0000 mon.a (mon.0) 388 : audit [DBG] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-03T19:28:46.085 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:28:45 smithi164 bash[26855]: cephadm 2023-06-03T19:28:44.231767+0000 mgr.y (mgr.14146) 182 : cephadm [INF] Deploying daemon osd.4 on smithi164 2023-06-03T19:28:46.281 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:28:45 smithi089 bash[26572]: cephadm 2023-06-03T19:28:44.231767+0000 mgr.y (mgr.14146) 182 : cephadm [INF] Deploying daemon osd.4 on smithi164 2023-06-03T19:28:46.281 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:28:45 smithi089 bash[33173]: cephadm 2023-06-03T19:28:44.231767+0000 mgr.y (mgr.14146) 182 : cephadm [INF] Deploying daemon osd.4 on smithi164 2023-06-03T19:28:47.085 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:28:46 smithi164 bash[26855]: cluster 2023-06-03T19:28:45.752873+0000 mgr.y (mgr.14146) 183 : cluster [DBG] pgmap v162: 1 pgs: 1 active+clean; 0 B data, 20 MiB used, 358 GiB / 358 GiB avail 2023-06-03T19:28:47.104 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:28:46 smithi089 bash[26572]: cluster 2023-06-03T19:28:45.752873+0000 mgr.y (mgr.14146) 183 : cluster [DBG] pgmap v162: 1 pgs: 1 active+clean; 0 B data, 20 MiB used, 358 GiB / 358 GiB avail 2023-06-03T19:28:47.104 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:28:46 smithi089 bash[33173]: cluster 2023-06-03T19:28:45.752873+0000 mgr.y (mgr.14146) 183 : cluster [DBG] pgmap v162: 1 pgs: 1 active+clean; 0 B data, 20 MiB used, 358 GiB / 358 GiB avail 2023-06-03T19:28:49.085 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:28:48 smithi164 bash[26855]: cluster 2023-06-03T19:28:47.753389+0000 mgr.y (mgr.14146) 184 : cluster [DBG] pgmap v163: 1 pgs: 1 active+clean; 0 B data, 20 MiB used, 358 GiB / 358 GiB avail 2023-06-03T19:28:49.085 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:28:48 smithi164 bash[26855]: audit 2023-06-03T19:28:48.652854+0000 mon.a (mon.0) 389 : audit [INF] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' 2023-06-03T19:28:49.086 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:28:48 smithi164 bash[26855]: audit 2023-06-03T19:28:48.669219+0000 mon.a (mon.0) 390 : audit [DBG] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-03T19:28:49.281 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:28:48 smithi089 bash[26572]: cluster 2023-06-03T19:28:47.753389+0000 mgr.y (mgr.14146) 184 : cluster [DBG] pgmap v163: 1 pgs: 1 active+clean; 0 B data, 20 MiB used, 358 GiB / 358 GiB avail 2023-06-03T19:28:49.281 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:28:48 smithi089 bash[26572]: audit 2023-06-03T19:28:48.652854+0000 mon.a (mon.0) 389 : audit [INF] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' 2023-06-03T19:28:49.281 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:28:48 smithi089 bash[26572]: audit 2023-06-03T19:28:48.669219+0000 mon.a (mon.0) 390 : audit [DBG] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-03T19:28:49.281 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:28:48 smithi089 bash[33173]: cluster 2023-06-03T19:28:47.753389+0000 mgr.y (mgr.14146) 184 : cluster [DBG] pgmap v163: 1 pgs: 1 active+clean; 0 B data, 20 MiB used, 358 GiB / 358 GiB avail 2023-06-03T19:28:49.282 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:28:48 smithi089 bash[33173]: audit 2023-06-03T19:28:48.652854+0000 mon.a (mon.0) 389 : audit [INF] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' 2023-06-03T19:28:49.282 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:28:48 smithi089 bash[33173]: audit 2023-06-03T19:28:48.669219+0000 mon.a (mon.0) 390 : audit [DBG] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-03T19:28:51.111 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:28:50 smithi164 bash[26855]: cluster 2023-06-03T19:28:49.753971+0000 mgr.y (mgr.14146) 185 : cluster [DBG] pgmap v164: 1 pgs: 1 active+clean; 0 B data, 20 MiB used, 358 GiB / 358 GiB avail 2023-06-03T19:28:51.281 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:28:50 smithi089 bash[26572]: cluster 2023-06-03T19:28:49.753971+0000 mgr.y (mgr.14146) 185 : cluster [DBG] pgmap v164: 1 pgs: 1 active+clean; 0 B data, 20 MiB used, 358 GiB / 358 GiB avail 2023-06-03T19:28:51.281 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:28:50 smithi089 bash[33173]: cluster 2023-06-03T19:28:49.753971+0000 mgr.y (mgr.14146) 185 : cluster [DBG] pgmap v164: 1 pgs: 1 active+clean; 0 B data, 20 MiB used, 358 GiB / 358 GiB avail 2023-06-03T19:28:52.781 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:28:52 smithi089 bash[26572]: cluster 2023-06-03T19:28:51.754547+0000 mgr.y (mgr.14146) 186 : cluster [DBG] pgmap v165: 1 pgs: 1 active+clean; 0 B data, 20 MiB used, 358 GiB / 358 GiB avail 2023-06-03T19:28:52.781 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:28:52 smithi089 bash[33173]: cluster 2023-06-03T19:28:51.754547+0000 mgr.y (mgr.14146) 186 : cluster [DBG] pgmap v165: 1 pgs: 1 active+clean; 0 B data, 20 MiB used, 358 GiB / 358 GiB avail 2023-06-03T19:28:52.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:28:52 smithi164 bash[26855]: cluster 2023-06-03T19:28:51.754547+0000 mgr.y (mgr.14146) 186 : cluster [DBG] pgmap v165: 1 pgs: 1 active+clean; 0 B data, 20 MiB used, 358 GiB / 358 GiB avail 2023-06-03T19:28:55.085 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:28:54 smithi164 bash[26855]: cluster 2023-06-03T19:28:53.755097+0000 mgr.y (mgr.14146) 187 : cluster [DBG] pgmap v166: 1 pgs: 1 active+clean; 0 B data, 20 MiB used, 358 GiB / 358 GiB avail 2023-06-03T19:28:55.280 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:28:54 smithi089 bash[26572]: cluster 2023-06-03T19:28:53.755097+0000 mgr.y (mgr.14146) 187 : cluster [DBG] pgmap v166: 1 pgs: 1 active+clean; 0 B data, 20 MiB used, 358 GiB / 358 GiB avail 2023-06-03T19:28:55.281 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:28:54 smithi089 bash[33173]: cluster 2023-06-03T19:28:53.755097+0000 mgr.y (mgr.14146) 187 : cluster [DBG] pgmap v166: 1 pgs: 1 active+clean; 0 B data, 20 MiB used, 358 GiB / 358 GiB avail 2023-06-03T19:28:56.085 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:28:55 smithi164 bash[26855]: audit 2023-06-03T19:28:54.851006+0000 mon.a (mon.0) 391 : audit [INF] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' 2023-06-03T19:28:56.086 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:28:55 smithi164 bash[26855]: audit 2023-06-03T19:28:55.137481+0000 mon.a (mon.0) 392 : audit [DBG] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-03T19:28:56.086 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:28:55 smithi164 bash[26855]: audit 2023-06-03T19:28:55.139170+0000 mon.a (mon.0) 393 : audit [INF] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-03T19:28:56.086 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:28:55 smithi164 bash[26855]: audit 2023-06-03T19:28:55.149130+0000 mon.a (mon.0) 394 : audit [INF] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' 2023-06-03T19:28:56.086 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:28:55 smithi164 bash[26855]: audit 2023-06-03T19:28:55.643630+0000 mon.b (mon.1) 5 : audit [INF] from='osd.4 [v2:172.21.15.164:6800/538382489,v1:172.21.15.164:6801/538382489]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2023-06-03T19:28:56.086 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:28:55 smithi164 bash[26855]: audit 2023-06-03T19:28:55.646114+0000 mon.a (mon.0) 395 : audit [INF] from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2023-06-03T19:28:56.281 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:28:55 smithi089 bash[26572]: audit 2023-06-03T19:28:54.851006+0000 mon.a (mon.0) 391 : audit [INF] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' 2023-06-03T19:28:56.281 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:28:55 smithi089 bash[26572]: audit 2023-06-03T19:28:55.137481+0000 mon.a (mon.0) 392 : audit [DBG] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-03T19:28:56.281 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:28:55 smithi089 bash[26572]: audit 2023-06-03T19:28:55.139170+0000 mon.a (mon.0) 393 : audit [INF] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-03T19:28:56.281 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:28:55 smithi089 bash[26572]: audit 2023-06-03T19:28:55.149130+0000 mon.a (mon.0) 394 : audit [INF] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' 2023-06-03T19:28:56.281 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:28:55 smithi089 bash[26572]: audit 2023-06-03T19:28:55.643630+0000 mon.b (mon.1) 5 : audit [INF] from='osd.4 [v2:172.21.15.164:6800/538382489,v1:172.21.15.164:6801/538382489]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2023-06-03T19:28:56.282 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:28:55 smithi089 bash[26572]: audit 2023-06-03T19:28:55.646114+0000 mon.a (mon.0) 395 : audit [INF] from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2023-06-03T19:28:56.282 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:28:55 smithi089 bash[33173]: audit 2023-06-03T19:28:54.851006+0000 mon.a (mon.0) 391 : audit [INF] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' 2023-06-03T19:28:56.282 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:28:55 smithi089 bash[33173]: audit 2023-06-03T19:28:55.137481+0000 mon.a (mon.0) 392 : audit [DBG] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-03T19:28:56.282 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:28:55 smithi089 bash[33173]: audit 2023-06-03T19:28:55.139170+0000 mon.a (mon.0) 393 : audit [INF] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-03T19:28:56.282 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:28:55 smithi089 bash[33173]: audit 2023-06-03T19:28:55.149130+0000 mon.a (mon.0) 394 : audit [INF] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' 2023-06-03T19:28:56.283 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:28:55 smithi089 bash[33173]: audit 2023-06-03T19:28:55.643630+0000 mon.b (mon.1) 5 : audit [INF] from='osd.4 [v2:172.21.15.164:6800/538382489,v1:172.21.15.164:6801/538382489]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2023-06-03T19:28:56.283 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:28:55 smithi089 bash[33173]: audit 2023-06-03T19:28:55.646114+0000 mon.a (mon.0) 395 : audit [INF] from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2023-06-03T19:28:57.175 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:28:56 smithi164 bash[26855]: cluster 2023-06-03T19:28:55.755650+0000 mgr.y (mgr.14146) 188 : cluster [DBG] pgmap v167: 1 pgs: 1 active+clean; 0 B data, 20 MiB used, 358 GiB / 358 GiB avail 2023-06-03T19:28:57.175 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:28:56 smithi164 bash[26855]: audit 2023-06-03T19:28:56.158017+0000 mon.b (mon.1) 6 : audit [INF] from='osd.4 [v2:172.21.15.164:6800/538382489,v1:172.21.15.164:6801/538382489]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi164", "root=default"]}]: dispatch 2023-06-03T19:28:57.176 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:28:56 smithi164 bash[26855]: audit 2023-06-03T19:28:56.158468+0000 mon.a (mon.0) 396 : audit [INF] from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2023-06-03T19:28:57.176 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:28:56 smithi164 bash[26855]: cluster 2023-06-03T19:28:56.158574+0000 mon.a (mon.0) 397 : cluster [DBG] osdmap e28: 5 total, 4 up, 5 in 2023-06-03T19:28:57.176 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:28:56 smithi164 bash[26855]: audit 2023-06-03T19:28:56.158757+0000 mon.a (mon.0) 398 : audit [DBG] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-06-03T19:28:57.177 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:28:56 smithi164 bash[26855]: audit 2023-06-03T19:28:56.160215+0000 mon.a (mon.0) 399 : audit [INF] from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi164", "root=default"]}]: dispatch 2023-06-03T19:28:57.281 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:28:56 smithi089 bash[26572]: cluster 2023-06-03T19:28:55.755650+0000 mgr.y (mgr.14146) 188 : cluster [DBG] pgmap v167: 1 pgs: 1 active+clean; 0 B data, 20 MiB used, 358 GiB / 358 GiB avail 2023-06-03T19:28:57.282 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:28:56 smithi089 bash[26572]: audit 2023-06-03T19:28:56.158017+0000 mon.b (mon.1) 6 : audit [INF] from='osd.4 [v2:172.21.15.164:6800/538382489,v1:172.21.15.164:6801/538382489]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi164", "root=default"]}]: dispatch 2023-06-03T19:28:57.283 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:28:56 smithi089 bash[26572]: audit 2023-06-03T19:28:56.158468+0000 mon.a (mon.0) 396 : audit [INF] from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2023-06-03T19:28:57.283 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:28:56 smithi089 bash[26572]: cluster 2023-06-03T19:28:56.158574+0000 mon.a (mon.0) 397 : cluster [DBG] osdmap e28: 5 total, 4 up, 5 in 2023-06-03T19:28:57.283 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:28:56 smithi089 bash[26572]: audit 2023-06-03T19:28:56.158757+0000 mon.a (mon.0) 398 : audit [DBG] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-06-03T19:28:57.284 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:28:56 smithi089 bash[26572]: audit 2023-06-03T19:28:56.160215+0000 mon.a (mon.0) 399 : audit [INF] from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi164", "root=default"]}]: dispatch 2023-06-03T19:28:57.285 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:28:56 smithi089 bash[33173]: cluster 2023-06-03T19:28:55.755650+0000 mgr.y (mgr.14146) 188 : cluster [DBG] pgmap v167: 1 pgs: 1 active+clean; 0 B data, 20 MiB used, 358 GiB / 358 GiB avail 2023-06-03T19:28:57.285 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:28:56 smithi089 bash[33173]: audit 2023-06-03T19:28:56.158017+0000 mon.b (mon.1) 6 : audit [INF] from='osd.4 [v2:172.21.15.164:6800/538382489,v1:172.21.15.164:6801/538382489]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi164", "root=default"]}]: dispatch 2023-06-03T19:28:57.285 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:28:56 smithi089 bash[33173]: audit 2023-06-03T19:28:56.158468+0000 mon.a (mon.0) 396 : audit [INF] from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2023-06-03T19:28:57.286 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:28:56 smithi089 bash[33173]: cluster 2023-06-03T19:28:56.158574+0000 mon.a (mon.0) 397 : cluster [DBG] osdmap e28: 5 total, 4 up, 5 in 2023-06-03T19:28:57.286 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:28:56 smithi089 bash[33173]: audit 2023-06-03T19:28:56.158757+0000 mon.a (mon.0) 398 : audit [DBG] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-06-03T19:28:57.286 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:28:56 smithi089 bash[33173]: audit 2023-06-03T19:28:56.160215+0000 mon.a (mon.0) 399 : audit [INF] from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi164", "root=default"]}]: dispatch 2023-06-03T19:28:58.335 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:28:58 smithi164 bash[26855]: audit 2023-06-03T19:28:57.161844+0000 mon.a (mon.0) 400 : audit [INF] from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi164", "root=default"]}]': finished 2023-06-03T19:28:58.336 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:28:58 smithi164 bash[26855]: cluster 2023-06-03T19:28:57.162004+0000 mon.a (mon.0) 401 : cluster [DBG] osdmap e29: 5 total, 4 up, 5 in 2023-06-03T19:28:58.336 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:28:58 smithi164 bash[26855]: audit 2023-06-03T19:28:57.162332+0000 mon.a (mon.0) 402 : audit [DBG] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-06-03T19:28:58.336 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:28:58 smithi164 bash[26855]: audit 2023-06-03T19:28:57.168464+0000 mon.a (mon.0) 403 : audit [DBG] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-06-03T19:28:58.337 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:28:58 smithi164 bash[26855]: cluster 2023-06-03T19:28:57.756178+0000 mgr.y (mgr.14146) 189 : cluster [DBG] pgmap v170: 1 pgs: 1 active+clean; 0 B data, 20 MiB used, 358 GiB / 358 GiB avail 2023-06-03T19:28:58.485 INFO:teuthology.orchestra.run.smithi164.stdout:Created osd(s) 4 on host 'smithi164' 2023-06-03T19:28:58.531 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:28:58 smithi089 bash[26572]: audit 2023-06-03T19:28:57.161844+0000 mon.a (mon.0) 400 : audit [INF] from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi164", "root=default"]}]': finished 2023-06-03T19:28:58.531 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:28:58 smithi089 bash[26572]: cluster 2023-06-03T19:28:57.162004+0000 mon.a (mon.0) 401 : cluster [DBG] osdmap e29: 5 total, 4 up, 5 in 2023-06-03T19:28:58.532 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:28:58 smithi089 bash[26572]: audit 2023-06-03T19:28:57.162332+0000 mon.a (mon.0) 402 : audit [DBG] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-06-03T19:28:58.532 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:28:58 smithi089 bash[26572]: audit 2023-06-03T19:28:57.168464+0000 mon.a (mon.0) 403 : audit [DBG] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-06-03T19:28:58.532 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:28:58 smithi089 bash[26572]: cluster 2023-06-03T19:28:57.756178+0000 mgr.y (mgr.14146) 189 : cluster [DBG] pgmap v170: 1 pgs: 1 active+clean; 0 B data, 20 MiB used, 358 GiB / 358 GiB avail 2023-06-03T19:28:58.533 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:28:58 smithi089 bash[33173]: audit 2023-06-03T19:28:57.161844+0000 mon.a (mon.0) 400 : audit [INF] from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi164", "root=default"]}]': finished 2023-06-03T19:28:58.533 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:28:58 smithi089 bash[33173]: cluster 2023-06-03T19:28:57.162004+0000 mon.a (mon.0) 401 : cluster [DBG] osdmap e29: 5 total, 4 up, 5 in 2023-06-03T19:28:58.533 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:28:58 smithi089 bash[33173]: audit 2023-06-03T19:28:57.162332+0000 mon.a (mon.0) 402 : audit [DBG] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-06-03T19:28:58.533 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:28:58 smithi089 bash[33173]: audit 2023-06-03T19:28:57.168464+0000 mon.a (mon.0) 403 : audit [DBG] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-06-03T19:28:58.533 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:28:58 smithi089 bash[33173]: cluster 2023-06-03T19:28:57.756178+0000 mgr.y (mgr.14146) 189 : cluster [DBG] pgmap v170: 1 pgs: 1 active+clean; 0 B data, 20 MiB used, 358 GiB / 358 GiB avail 2023-06-03T19:28:59.152 DEBUG:teuthology.orchestra.run.smithi164:osd.4> sudo journalctl -f -n 0 -u ceph-f1db3478-0243-11ee-9b1d-001a4aab830c@osd.4.service 2023-06-03T19:28:59.155 INFO:tasks.cephadm:Deploying osd.5 on smithi164 with /dev/vg_nvme/lv_3... 2023-06-03T19:28:59.156 DEBUG:teuthology.orchestra.run.smithi164:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph-volume lvm zap /dev/vg_nvme/lv_3 2023-06-03T19:28:59.381 INFO:journalctl@ceph.osd.4.smithi164.stdout:-- Logs begin at Sun 2022-08-14 12:20:24 UTC. -- 2023-06-03T19:28:59.531 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:28:59 smithi089 bash[26572]: cluster 2023-06-03T19:28:56.685599+0000 osd.4 (osd.4) 1 : cluster [DBG] purged_snaps scrub starts 2023-06-03T19:28:59.531 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:28:59 smithi089 bash[26572]: cluster 2023-06-03T19:28:56.685705+0000 osd.4 (osd.4) 2 : cluster [DBG] purged_snaps scrub ok 2023-06-03T19:28:59.532 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:28:59 smithi089 bash[26572]: cluster 2023-06-03T19:28:58.167828+0000 mon.a (mon.0) 404 : cluster [INF] osd.4 [v2:172.21.15.164:6800/538382489,v1:172.21.15.164:6801/538382489] boot 2023-06-03T19:28:59.532 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:28:59 smithi089 bash[26572]: cluster 2023-06-03T19:28:58.167930+0000 mon.a (mon.0) 405 : cluster [DBG] osdmap e30: 5 total, 5 up, 5 in 2023-06-03T19:28:59.532 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:28:59 smithi089 bash[26572]: audit 2023-06-03T19:28:58.168376+0000 mon.a (mon.0) 406 : audit [DBG] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-06-03T19:28:59.532 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:28:59 smithi089 bash[26572]: audit 2023-06-03T19:28:58.169606+0000 mon.a (mon.0) 407 : audit [DBG] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-06-03T19:28:59.532 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:28:59 smithi089 bash[26572]: audit 2023-06-03T19:28:58.477529+0000 mon.a (mon.0) 408 : audit [INF] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' 2023-06-03T19:28:59.532 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:28:59 smithi089 bash[26572]: audit 2023-06-03T19:28:58.477815+0000 mon.a (mon.0) 409 : audit [DBG] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-03T19:28:59.533 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:28:59 smithi089 bash[33173]: cluster 2023-06-03T19:28:56.685599+0000 osd.4 (osd.4) 1 : cluster [DBG] purged_snaps scrub starts 2023-06-03T19:28:59.533 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:28:59 smithi089 bash[33173]: cluster 2023-06-03T19:28:56.685705+0000 osd.4 (osd.4) 2 : cluster [DBG] purged_snaps scrub ok 2023-06-03T19:28:59.533 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:28:59 smithi089 bash[33173]: cluster 2023-06-03T19:28:58.167828+0000 mon.a (mon.0) 404 : cluster [INF] osd.4 [v2:172.21.15.164:6800/538382489,v1:172.21.15.164:6801/538382489] boot 2023-06-03T19:28:59.533 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:28:59 smithi089 bash[33173]: cluster 2023-06-03T19:28:58.167930+0000 mon.a (mon.0) 405 : cluster [DBG] osdmap e30: 5 total, 5 up, 5 in 2023-06-03T19:28:59.533 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:28:59 smithi089 bash[33173]: audit 2023-06-03T19:28:58.168376+0000 mon.a (mon.0) 406 : audit [DBG] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-06-03T19:28:59.533 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:28:59 smithi089 bash[33173]: audit 2023-06-03T19:28:58.169606+0000 mon.a (mon.0) 407 : audit [DBG] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-06-03T19:28:59.533 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:28:59 smithi089 bash[33173]: audit 2023-06-03T19:28:58.477529+0000 mon.a (mon.0) 408 : audit [INF] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' 2023-06-03T19:28:59.534 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:28:59 smithi089 bash[33173]: audit 2023-06-03T19:28:58.477815+0000 mon.a (mon.0) 409 : audit [DBG] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-03T19:28:59.585 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:28:59 smithi164 bash[26855]: cluster 2023-06-03T19:28:56.685599+0000 osd.4 (osd.4) 1 : cluster [DBG] purged_snaps scrub starts 2023-06-03T19:28:59.586 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:28:59 smithi164 bash[26855]: cluster 2023-06-03T19:28:56.685705+0000 osd.4 (osd.4) 2 : cluster [DBG] purged_snaps scrub ok 2023-06-03T19:28:59.586 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:28:59 smithi164 bash[26855]: cluster 2023-06-03T19:28:58.167828+0000 mon.a (mon.0) 404 : cluster [INF] osd.4 [v2:172.21.15.164:6800/538382489,v1:172.21.15.164:6801/538382489] boot 2023-06-03T19:28:59.586 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:28:59 smithi164 bash[26855]: cluster 2023-06-03T19:28:58.167930+0000 mon.a (mon.0) 405 : cluster [DBG] osdmap e30: 5 total, 5 up, 5 in 2023-06-03T19:28:59.586 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:28:59 smithi164 bash[26855]: audit 2023-06-03T19:28:58.168376+0000 mon.a (mon.0) 406 : audit [DBG] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-06-03T19:28:59.586 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:28:59 smithi164 bash[26855]: audit 2023-06-03T19:28:58.169606+0000 mon.a (mon.0) 407 : audit [DBG] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-06-03T19:28:59.586 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:28:59 smithi164 bash[26855]: audit 2023-06-03T19:28:58.477529+0000 mon.a (mon.0) 408 : audit [INF] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' 2023-06-03T19:28:59.586 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:28:59 smithi164 bash[26855]: audit 2023-06-03T19:28:58.477815+0000 mon.a (mon.0) 409 : audit [DBG] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-03T19:29:00.444 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:29:00 smithi164 bash[26855]: cluster 2023-06-03T19:28:59.171212+0000 mon.a (mon.0) 410 : cluster [DBG] osdmap e31: 5 total, 5 up, 5 in 2023-06-03T19:29:00.445 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:29:00 smithi164 bash[26855]: cluster 2023-06-03T19:28:59.756688+0000 mgr.y (mgr.14146) 190 : cluster [DBG] pgmap v173: 1 pgs: 1 active+clean; 0 B data, 25 MiB used, 447 GiB / 447 GiB avail 2023-06-03T19:29:00.531 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:29:00 smithi089 bash[26572]: cluster 2023-06-03T19:28:59.171212+0000 mon.a (mon.0) 410 : cluster [DBG] osdmap e31: 5 total, 5 up, 5 in 2023-06-03T19:29:00.531 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:29:00 smithi089 bash[26572]: cluster 2023-06-03T19:28:59.756688+0000 mgr.y (mgr.14146) 190 : cluster [DBG] pgmap v173: 1 pgs: 1 active+clean; 0 B data, 25 MiB used, 447 GiB / 447 GiB avail 2023-06-03T19:29:00.531 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:29:00 smithi089 bash[33173]: cluster 2023-06-03T19:28:59.171212+0000 mon.a (mon.0) 410 : cluster [DBG] osdmap e31: 5 total, 5 up, 5 in 2023-06-03T19:29:00.531 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:29:00 smithi089 bash[33173]: cluster 2023-06-03T19:28:59.756688+0000 mgr.y (mgr.14146) 190 : cluster [DBG] pgmap v173: 1 pgs: 1 active+clean; 0 B data, 25 MiB used, 447 GiB / 447 GiB avail 2023-06-03T19:29:01.531 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:29:01 smithi089 bash[26572]: cluster 2023-06-03T19:29:00.186062+0000 mon.a (mon.0) 411 : cluster [DBG] osdmap e32: 5 total, 5 up, 5 in 2023-06-03T19:29:01.532 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:29:01 smithi089 bash[33173]: cluster 2023-06-03T19:29:00.186062+0000 mon.a (mon.0) 411 : cluster [DBG] osdmap e32: 5 total, 5 up, 5 in 2023-06-03T19:29:01.585 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:29:01 smithi164 bash[26855]: cluster 2023-06-03T19:29:00.186062+0000 mon.a (mon.0) 411 : cluster [DBG] osdmap e32: 5 total, 5 up, 5 in 2023-06-03T19:29:02.639 INFO:teuthology.orchestra.run.smithi164.stderr:--> Zapping: /dev/vg_nvme/lv_3 2023-06-03T19:29:02.640 INFO:teuthology.orchestra.run.smithi164.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_3 bs=1M count=10 conv=fsync 2023-06-03T19:29:02.640 INFO:teuthology.orchestra.run.smithi164.stderr: stderr: 10+0 records in 2023-06-03T19:29:02.640 INFO:teuthology.orchestra.run.smithi164.stderr:10+0 records out 2023-06-03T19:29:02.640 INFO:teuthology.orchestra.run.smithi164.stderr: stderr: 10485760 bytes (10 MB, 10 MiB) copied, 0.0146291 s, 717 MB/s 2023-06-03T19:29:02.640 INFO:teuthology.orchestra.run.smithi164.stderr:--> Zapping successful for: 2023-06-03T19:29:02.704 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:29:02 smithi164 bash[26855]: cluster 2023-06-03T19:29:01.757309+0000 mgr.y (mgr.14146) 191 : cluster [DBG] pgmap v175: 1 pgs: 1 active+clean; 0 B data, 25 MiB used, 447 GiB / 447 GiB avail 2023-06-03T19:29:02.781 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:29:02 smithi089 bash[26572]: cluster 2023-06-03T19:29:01.757309+0000 mgr.y (mgr.14146) 191 : cluster [DBG] pgmap v175: 1 pgs: 1 active+clean; 0 B data, 25 MiB used, 447 GiB / 447 GiB avail 2023-06-03T19:29:02.781 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:29:02 smithi089 bash[33173]: cluster 2023-06-03T19:29:01.757309+0000 mgr.y (mgr.14146) 191 : cluster [DBG] pgmap v175: 1 pgs: 1 active+clean; 0 B data, 25 MiB used, 447 GiB / 447 GiB avail 2023-06-03T19:29:03.312 DEBUG:teuthology.orchestra.run.smithi164:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph orch daemon add osd smithi164:vg_nvme/lv_3 2023-06-03T19:29:05.085 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:29:04 smithi164 bash[26855]: cluster 2023-06-03T19:29:03.757836+0000 mgr.y (mgr.14146) 192 : cluster [DBG] pgmap v176: 1 pgs: 1 active+clean; 0 B data, 25 MiB used, 447 GiB / 447 GiB avail 2023-06-03T19:29:05.281 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:29:04 smithi089 bash[26572]: cluster 2023-06-03T19:29:03.757836+0000 mgr.y (mgr.14146) 192 : cluster [DBG] pgmap v176: 1 pgs: 1 active+clean; 0 B data, 25 MiB used, 447 GiB / 447 GiB avail 2023-06-03T19:29:05.281 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:29:04 smithi089 bash[33173]: cluster 2023-06-03T19:29:03.757836+0000 mgr.y (mgr.14146) 192 : cluster [DBG] pgmap v176: 1 pgs: 1 active+clean; 0 B data, 25 MiB used, 447 GiB / 447 GiB avail 2023-06-03T19:29:07.085 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:29:06 smithi164 bash[26855]: cluster 2023-06-03T19:29:05.758394+0000 mgr.y (mgr.14146) 193 : cluster [DBG] pgmap v177: 1 pgs: 1 active+clean; 0 B data, 25 MiB used, 447 GiB / 447 GiB avail 2023-06-03T19:29:07.281 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:29:06 smithi089 bash[26572]: cluster 2023-06-03T19:29:05.758394+0000 mgr.y (mgr.14146) 193 : cluster [DBG] pgmap v177: 1 pgs: 1 active+clean; 0 B data, 25 MiB used, 447 GiB / 447 GiB avail 2023-06-03T19:29:07.281 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:29:06 smithi089 bash[33173]: cluster 2023-06-03T19:29:05.758394+0000 mgr.y (mgr.14146) 193 : cluster [DBG] pgmap v177: 1 pgs: 1 active+clean; 0 B data, 25 MiB used, 447 GiB / 447 GiB avail 2023-06-03T19:29:09.085 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:29:08 smithi164 bash[26855]: cluster 2023-06-03T19:29:07.758956+0000 mgr.y (mgr.14146) 194 : cluster [DBG] pgmap v178: 1 pgs: 1 active+clean; 0 B data, 25 MiB used, 447 GiB / 447 GiB avail 2023-06-03T19:29:09.086 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:29:08 smithi164 bash[26855]: audit 2023-06-03T19:29:08.306416+0000 mon.a (mon.0) 412 : audit [DBG] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-06-03T19:29:09.086 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:29:08 smithi164 bash[26855]: audit 2023-06-03T19:29:08.311892+0000 mon.a (mon.0) 413 : audit [INF] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-06-03T19:29:09.086 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:29:08 smithi164 bash[26855]: audit 2023-06-03T19:29:08.313280+0000 mon.a (mon.0) 414 : audit [DBG] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-03T19:29:09.281 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:29:08 smithi089 bash[26572]: cluster 2023-06-03T19:29:07.758956+0000 mgr.y (mgr.14146) 194 : cluster [DBG] pgmap v178: 1 pgs: 1 active+clean; 0 B data, 25 MiB used, 447 GiB / 447 GiB avail 2023-06-03T19:29:09.281 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:29:08 smithi089 bash[26572]: audit 2023-06-03T19:29:08.306416+0000 mon.a (mon.0) 412 : audit [DBG] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-06-03T19:29:09.281 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:29:08 smithi089 bash[26572]: audit 2023-06-03T19:29:08.311892+0000 mon.a (mon.0) 413 : audit [INF] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-06-03T19:29:09.281 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:29:08 smithi089 bash[26572]: audit 2023-06-03T19:29:08.313280+0000 mon.a (mon.0) 414 : audit [DBG] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-03T19:29:09.282 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:29:08 smithi089 bash[33173]: cluster 2023-06-03T19:29:07.758956+0000 mgr.y (mgr.14146) 194 : cluster [DBG] pgmap v178: 1 pgs: 1 active+clean; 0 B data, 25 MiB used, 447 GiB / 447 GiB avail 2023-06-03T19:29:09.282 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:29:08 smithi089 bash[33173]: audit 2023-06-03T19:29:08.306416+0000 mon.a (mon.0) 412 : audit [DBG] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-06-03T19:29:09.282 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:29:08 smithi089 bash[33173]: audit 2023-06-03T19:29:08.311892+0000 mon.a (mon.0) 413 : audit [INF] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-06-03T19:29:09.282 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:29:08 smithi089 bash[33173]: audit 2023-06-03T19:29:08.313280+0000 mon.a (mon.0) 414 : audit [DBG] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-03T19:29:10.281 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:29:09 smithi089 bash[26572]: audit 2023-06-03T19:29:08.299481+0000 mgr.y (mgr.14146) 195 : audit [DBG] from='client.24250 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi164:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2023-06-03T19:29:10.281 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:29:09 smithi089 bash[26572]: audit 2023-06-03T19:29:09.614352+0000 mon.a (mon.0) 415 : audit [INF] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' 2023-06-03T19:29:10.281 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:29:09 smithi089 bash[26572]: audit 2023-06-03T19:29:09.616171+0000 mon.a (mon.0) 416 : audit [INF] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-06-03T19:29:10.281 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:29:09 smithi089 bash[26572]: audit 2023-06-03T19:29:09.622107+0000 mon.a (mon.0) 417 : audit [INF] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' 2023-06-03T19:29:10.282 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:29:09 smithi089 bash[26572]: audit 2023-06-03T19:29:09.623546+0000 mon.a (mon.0) 418 : audit [DBG] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-03T19:29:10.282 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:29:09 smithi089 bash[26572]: audit 2023-06-03T19:29:09.624564+0000 mon.a (mon.0) 419 : audit [INF] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-03T19:29:10.282 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:29:09 smithi089 bash[26572]: audit 2023-06-03T19:29:09.630048+0000 mon.a (mon.0) 420 : audit [INF] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' 2023-06-03T19:29:10.282 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:29:09 smithi089 bash[33173]: audit 2023-06-03T19:29:08.299481+0000 mgr.y (mgr.14146) 195 : audit [DBG] from='client.24250 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi164:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2023-06-03T19:29:10.282 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:29:09 smithi089 bash[33173]: audit 2023-06-03T19:29:09.614352+0000 mon.a (mon.0) 415 : audit [INF] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' 2023-06-03T19:29:10.282 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:29:09 smithi089 bash[33173]: audit 2023-06-03T19:29:09.616171+0000 mon.a (mon.0) 416 : audit [INF] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-06-03T19:29:10.283 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:29:09 smithi089 bash[33173]: audit 2023-06-03T19:29:09.622107+0000 mon.a (mon.0) 417 : audit [INF] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' 2023-06-03T19:29:10.283 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:29:09 smithi089 bash[33173]: audit 2023-06-03T19:29:09.623546+0000 mon.a (mon.0) 418 : audit [DBG] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-03T19:29:10.283 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:29:09 smithi089 bash[33173]: audit 2023-06-03T19:29:09.624564+0000 mon.a (mon.0) 419 : audit [INF] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-03T19:29:10.283 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:29:09 smithi089 bash[33173]: audit 2023-06-03T19:29:09.630048+0000 mon.a (mon.0) 420 : audit [INF] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' 2023-06-03T19:29:10.335 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:29:09 smithi164 bash[26855]: audit 2023-06-03T19:29:08.299481+0000 mgr.y (mgr.14146) 195 : audit [DBG] from='client.24250 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi164:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2023-06-03T19:29:10.336 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:29:09 smithi164 bash[26855]: audit 2023-06-03T19:29:09.614352+0000 mon.a (mon.0) 415 : audit [INF] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' 2023-06-03T19:29:10.336 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:29:09 smithi164 bash[26855]: audit 2023-06-03T19:29:09.616171+0000 mon.a (mon.0) 416 : audit [INF] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-06-03T19:29:10.336 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:29:09 smithi164 bash[26855]: audit 2023-06-03T19:29:09.622107+0000 mon.a (mon.0) 417 : audit [INF] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' 2023-06-03T19:29:10.336 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:29:09 smithi164 bash[26855]: audit 2023-06-03T19:29:09.623546+0000 mon.a (mon.0) 418 : audit [DBG] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-03T19:29:10.336 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:29:09 smithi164 bash[26855]: audit 2023-06-03T19:29:09.624564+0000 mon.a (mon.0) 419 : audit [INF] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-03T19:29:10.336 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:29:09 smithi164 bash[26855]: audit 2023-06-03T19:29:09.630048+0000 mon.a (mon.0) 420 : audit [INF] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' 2023-06-03T19:29:11.281 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:29:10 smithi089 bash[33173]: cephadm 2023-06-03T19:29:09.605578+0000 mgr.y (mgr.14146) 196 : cephadm [INF] Detected new or changed devices on smithi164 2023-06-03T19:29:11.281 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:29:10 smithi089 bash[33173]: cephadm 2023-06-03T19:29:09.617406+0000 mgr.y (mgr.14146) 197 : cephadm [INF] Adjusting osd_memory_target on smithi164 to 17274M 2023-06-03T19:29:11.281 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:29:10 smithi089 bash[33173]: cluster 2023-06-03T19:29:09.759514+0000 mgr.y (mgr.14146) 198 : cluster [DBG] pgmap v179: 1 pgs: 1 active+clean; 0 B data, 25 MiB used, 447 GiB / 447 GiB avail 2023-06-03T19:29:11.281 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:29:10 smithi089 bash[26572]: cephadm 2023-06-03T19:29:09.605578+0000 mgr.y (mgr.14146) 196 : cephadm [INF] Detected new or changed devices on smithi164 2023-06-03T19:29:11.282 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:29:10 smithi089 bash[26572]: cephadm 2023-06-03T19:29:09.617406+0000 mgr.y (mgr.14146) 197 : cephadm [INF] Adjusting osd_memory_target on smithi164 to 17274M 2023-06-03T19:29:11.282 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:29:10 smithi089 bash[26572]: cluster 2023-06-03T19:29:09.759514+0000 mgr.y (mgr.14146) 198 : cluster [DBG] pgmap v179: 1 pgs: 1 active+clean; 0 B data, 25 MiB used, 447 GiB / 447 GiB avail 2023-06-03T19:29:11.335 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:29:10 smithi164 bash[26855]: cephadm 2023-06-03T19:29:09.605578+0000 mgr.y (mgr.14146) 196 : cephadm [INF] Detected new or changed devices on smithi164 2023-06-03T19:29:11.336 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:29:10 smithi164 bash[26855]: cephadm 2023-06-03T19:29:09.617406+0000 mgr.y (mgr.14146) 197 : cephadm [INF] Adjusting osd_memory_target on smithi164 to 17274M 2023-06-03T19:29:11.336 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:29:10 smithi164 bash[26855]: cluster 2023-06-03T19:29:09.759514+0000 mgr.y (mgr.14146) 198 : cluster [DBG] pgmap v179: 1 pgs: 1 active+clean; 0 B data, 25 MiB used, 447 GiB / 447 GiB avail 2023-06-03T19:29:12.781 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:29:12 smithi089 bash[26572]: cluster 2023-06-03T19:29:11.760081+0000 mgr.y (mgr.14146) 199 : cluster [DBG] pgmap v180: 1 pgs: 1 active+clean; 0 B data, 25 MiB used, 447 GiB / 447 GiB avail 2023-06-03T19:29:12.781 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:29:12 smithi089 bash[33173]: cluster 2023-06-03T19:29:11.760081+0000 mgr.y (mgr.14146) 199 : cluster [DBG] pgmap v180: 1 pgs: 1 active+clean; 0 B data, 25 MiB used, 447 GiB / 447 GiB avail 2023-06-03T19:29:12.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:29:12 smithi164 bash[26855]: cluster 2023-06-03T19:29:11.760081+0000 mgr.y (mgr.14146) 199 : cluster [DBG] pgmap v180: 1 pgs: 1 active+clean; 0 B data, 25 MiB used, 447 GiB / 447 GiB avail 2023-06-03T19:29:15.085 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:29:14 smithi164 bash[26855]: cluster 2023-06-03T19:29:13.760628+0000 mgr.y (mgr.14146) 200 : cluster [DBG] pgmap v181: 1 pgs: 1 active+clean; 0 B data, 25 MiB used, 447 GiB / 447 GiB avail 2023-06-03T19:29:15.281 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:29:14 smithi089 bash[26572]: cluster 2023-06-03T19:29:13.760628+0000 mgr.y (mgr.14146) 200 : cluster [DBG] pgmap v181: 1 pgs: 1 active+clean; 0 B data, 25 MiB used, 447 GiB / 447 GiB avail 2023-06-03T19:29:15.281 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:29:14 smithi089 bash[33173]: cluster 2023-06-03T19:29:13.760628+0000 mgr.y (mgr.14146) 200 : cluster [DBG] pgmap v181: 1 pgs: 1 active+clean; 0 B data, 25 MiB used, 447 GiB / 447 GiB avail 2023-06-03T19:29:17.086 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:29:16 smithi164 bash[26855]: cluster 2023-06-03T19:29:15.761198+0000 mgr.y (mgr.14146) 201 : cluster [DBG] pgmap v182: 1 pgs: 1 active+clean; 0 B data, 25 MiB used, 447 GiB / 447 GiB avail 2023-06-03T19:29:17.086 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:29:16 smithi164 bash[26855]: audit 2023-06-03T19:29:16.698105+0000 mon.b (mon.1) 7 : audit [INF] from='client.? 172.21.15.164:0/2252969429' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "b8a5e8a1-23cc-442b-90aa-19bd0036dd3c"}]: dispatch 2023-06-03T19:29:17.086 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:29:16 smithi164 bash[26855]: audit 2023-06-03T19:29:16.700537+0000 mon.a (mon.0) 421 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "b8a5e8a1-23cc-442b-90aa-19bd0036dd3c"}]: dispatch 2023-06-03T19:29:17.086 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:29:16 smithi164 bash[26855]: audit 2023-06-03T19:29:16.708348+0000 mon.a (mon.0) 422 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "b8a5e8a1-23cc-442b-90aa-19bd0036dd3c"}]': finished 2023-06-03T19:29:17.086 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:29:16 smithi164 bash[26855]: cluster 2023-06-03T19:29:16.708488+0000 mon.a (mon.0) 423 : cluster [DBG] osdmap e33: 6 total, 5 up, 6 in 2023-06-03T19:29:17.086 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:29:16 smithi164 bash[26855]: audit 2023-06-03T19:29:16.708758+0000 mon.a (mon.0) 424 : audit [DBG] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-06-03T19:29:17.281 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:29:16 smithi089 bash[26572]: cluster 2023-06-03T19:29:15.761198+0000 mgr.y (mgr.14146) 201 : cluster [DBG] pgmap v182: 1 pgs: 1 active+clean; 0 B data, 25 MiB used, 447 GiB / 447 GiB avail 2023-06-03T19:29:17.281 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:29:16 smithi089 bash[26572]: audit 2023-06-03T19:29:16.698105+0000 mon.b (mon.1) 7 : audit [INF] from='client.? 172.21.15.164:0/2252969429' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "b8a5e8a1-23cc-442b-90aa-19bd0036dd3c"}]: dispatch 2023-06-03T19:29:17.281 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:29:16 smithi089 bash[26572]: audit 2023-06-03T19:29:16.700537+0000 mon.a (mon.0) 421 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "b8a5e8a1-23cc-442b-90aa-19bd0036dd3c"}]: dispatch 2023-06-03T19:29:17.281 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:29:16 smithi089 bash[26572]: audit 2023-06-03T19:29:16.708348+0000 mon.a (mon.0) 422 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "b8a5e8a1-23cc-442b-90aa-19bd0036dd3c"}]': finished 2023-06-03T19:29:17.281 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:29:16 smithi089 bash[26572]: cluster 2023-06-03T19:29:16.708488+0000 mon.a (mon.0) 423 : cluster [DBG] osdmap e33: 6 total, 5 up, 6 in 2023-06-03T19:29:17.282 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:29:16 smithi089 bash[26572]: audit 2023-06-03T19:29:16.708758+0000 mon.a (mon.0) 424 : audit [DBG] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-06-03T19:29:17.282 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:29:16 smithi089 bash[33173]: cluster 2023-06-03T19:29:15.761198+0000 mgr.y (mgr.14146) 201 : cluster [DBG] pgmap v182: 1 pgs: 1 active+clean; 0 B data, 25 MiB used, 447 GiB / 447 GiB avail 2023-06-03T19:29:17.282 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:29:16 smithi089 bash[33173]: audit 2023-06-03T19:29:16.698105+0000 mon.b (mon.1) 7 : audit [INF] from='client.? 172.21.15.164:0/2252969429' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "b8a5e8a1-23cc-442b-90aa-19bd0036dd3c"}]: dispatch 2023-06-03T19:29:17.282 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:29:16 smithi089 bash[33173]: audit 2023-06-03T19:29:16.700537+0000 mon.a (mon.0) 421 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "b8a5e8a1-23cc-442b-90aa-19bd0036dd3c"}]: dispatch 2023-06-03T19:29:17.282 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:29:16 smithi089 bash[33173]: audit 2023-06-03T19:29:16.708348+0000 mon.a (mon.0) 422 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "b8a5e8a1-23cc-442b-90aa-19bd0036dd3c"}]': finished 2023-06-03T19:29:17.283 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:29:16 smithi089 bash[33173]: cluster 2023-06-03T19:29:16.708488+0000 mon.a (mon.0) 423 : cluster [DBG] osdmap e33: 6 total, 5 up, 6 in 2023-06-03T19:29:17.283 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:29:16 smithi089 bash[33173]: audit 2023-06-03T19:29:16.708758+0000 mon.a (mon.0) 424 : audit [DBG] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-06-03T19:29:18.281 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:29:17 smithi089 bash[26572]: audit 2023-06-03T19:29:17.463897+0000 mon.b (mon.1) 8 : audit [DBG] from='client.? 172.21.15.164:0/3965790469' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-06-03T19:29:18.281 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:29:17 smithi089 bash[33173]: audit 2023-06-03T19:29:17.463897+0000 mon.b (mon.1) 8 : audit [DBG] from='client.? 172.21.15.164:0/3965790469' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-06-03T19:29:18.335 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:29:17 smithi164 bash[26855]: audit 2023-06-03T19:29:17.463897+0000 mon.b (mon.1) 8 : audit [DBG] from='client.? 172.21.15.164:0/3965790469' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-06-03T19:29:19.281 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:29:18 smithi089 bash[26572]: cluster 2023-06-03T19:29:17.761751+0000 mgr.y (mgr.14146) 202 : cluster [DBG] pgmap v184: 1 pgs: 1 active+clean; 0 B data, 25 MiB used, 447 GiB / 447 GiB avail 2023-06-03T19:29:19.281 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:29:18 smithi089 bash[33173]: cluster 2023-06-03T19:29:17.761751+0000 mgr.y (mgr.14146) 202 : cluster [DBG] pgmap v184: 1 pgs: 1 active+clean; 0 B data, 25 MiB used, 447 GiB / 447 GiB avail 2023-06-03T19:29:19.335 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:29:18 smithi164 bash[26855]: cluster 2023-06-03T19:29:17.761751+0000 mgr.y (mgr.14146) 202 : cluster [DBG] pgmap v184: 1 pgs: 1 active+clean; 0 B data, 25 MiB used, 447 GiB / 447 GiB avail 2023-06-03T19:29:21.281 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:29:20 smithi089 bash[33173]: cluster 2023-06-03T19:29:19.762318+0000 mgr.y (mgr.14146) 203 : cluster [DBG] pgmap v185: 1 pgs: 1 active+clean; 0 B data, 25 MiB used, 447 GiB / 447 GiB avail 2023-06-03T19:29:21.281 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:29:20 smithi089 bash[26572]: cluster 2023-06-03T19:29:19.762318+0000 mgr.y (mgr.14146) 203 : cluster [DBG] pgmap v185: 1 pgs: 1 active+clean; 0 B data, 25 MiB used, 447 GiB / 447 GiB avail 2023-06-03T19:29:21.336 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:29:20 smithi164 bash[26855]: cluster 2023-06-03T19:29:19.762318+0000 mgr.y (mgr.14146) 203 : cluster [DBG] pgmap v185: 1 pgs: 1 active+clean; 0 B data, 25 MiB used, 447 GiB / 447 GiB avail 2023-06-03T19:29:22.781 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:29:22 smithi089 bash[26572]: cluster 2023-06-03T19:29:21.762873+0000 mgr.y (mgr.14146) 204 : cluster [DBG] pgmap v186: 1 pgs: 1 active+clean; 0 B data, 25 MiB used, 447 GiB / 447 GiB avail 2023-06-03T19:29:22.781 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:29:22 smithi089 bash[33173]: cluster 2023-06-03T19:29:21.762873+0000 mgr.y (mgr.14146) 204 : cluster [DBG] pgmap v186: 1 pgs: 1 active+clean; 0 B data, 25 MiB used, 447 GiB / 447 GiB avail 2023-06-03T19:29:22.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:29:22 smithi164 bash[26855]: cluster 2023-06-03T19:29:21.762873+0000 mgr.y (mgr.14146) 204 : cluster [DBG] pgmap v186: 1 pgs: 1 active+clean; 0 B data, 25 MiB used, 447 GiB / 447 GiB avail 2023-06-03T19:29:25.085 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:29:24 smithi164 bash[26855]: cluster 2023-06-03T19:29:23.763396+0000 mgr.y (mgr.14146) 205 : cluster [DBG] pgmap v187: 1 pgs: 1 active+clean; 0 B data, 25 MiB used, 447 GiB / 447 GiB avail 2023-06-03T19:29:25.281 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:29:24 smithi089 bash[26572]: cluster 2023-06-03T19:29:23.763396+0000 mgr.y (mgr.14146) 205 : cluster [DBG] pgmap v187: 1 pgs: 1 active+clean; 0 B data, 25 MiB used, 447 GiB / 447 GiB avail 2023-06-03T19:29:25.281 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:29:24 smithi089 bash[33173]: cluster 2023-06-03T19:29:23.763396+0000 mgr.y (mgr.14146) 205 : cluster [DBG] pgmap v187: 1 pgs: 1 active+clean; 0 B data, 25 MiB used, 447 GiB / 447 GiB avail 2023-06-03T19:29:27.085 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:29:26 smithi164 bash[26855]: cluster 2023-06-03T19:29:25.763941+0000 mgr.y (mgr.14146) 206 : cluster [DBG] pgmap v188: 1 pgs: 1 active+clean; 0 B data, 25 MiB used, 447 GiB / 447 GiB avail 2023-06-03T19:29:27.281 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:29:26 smithi089 bash[26572]: cluster 2023-06-03T19:29:25.763941+0000 mgr.y (mgr.14146) 206 : cluster [DBG] pgmap v188: 1 pgs: 1 active+clean; 0 B data, 25 MiB used, 447 GiB / 447 GiB avail 2023-06-03T19:29:27.281 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:29:26 smithi089 bash[33173]: cluster 2023-06-03T19:29:25.763941+0000 mgr.y (mgr.14146) 206 : cluster [DBG] pgmap v188: 1 pgs: 1 active+clean; 0 B data, 25 MiB used, 447 GiB / 447 GiB avail 2023-06-03T19:29:29.085 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:29:28 smithi164 bash[26855]: cluster 2023-06-03T19:29:27.764495+0000 mgr.y (mgr.14146) 207 : cluster [DBG] pgmap v189: 1 pgs: 1 active+clean; 0 B data, 25 MiB used, 447 GiB / 447 GiB avail 2023-06-03T19:29:29.281 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:29:28 smithi089 bash[26572]: cluster 2023-06-03T19:29:27.764495+0000 mgr.y (mgr.14146) 207 : cluster [DBG] pgmap v189: 1 pgs: 1 active+clean; 0 B data, 25 MiB used, 447 GiB / 447 GiB avail 2023-06-03T19:29:29.281 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:29:28 smithi089 bash[33173]: cluster 2023-06-03T19:29:27.764495+0000 mgr.y (mgr.14146) 207 : cluster [DBG] pgmap v189: 1 pgs: 1 active+clean; 0 B data, 25 MiB used, 447 GiB / 447 GiB avail 2023-06-03T19:29:30.086 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:29:29 smithi164 bash[26855]: audit 2023-06-03T19:29:29.459970+0000 mon.a (mon.0) 425 : audit [INF] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2023-06-03T19:29:30.086 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:29:29 smithi164 bash[26855]: audit 2023-06-03T19:29:29.461655+0000 mon.a (mon.0) 426 : audit [DBG] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-03T19:29:30.281 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:29:29 smithi089 bash[26572]: audit 2023-06-03T19:29:29.459970+0000 mon.a (mon.0) 425 : audit [INF] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2023-06-03T19:29:30.281 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:29:29 smithi089 bash[26572]: audit 2023-06-03T19:29:29.461655+0000 mon.a (mon.0) 426 : audit [DBG] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-03T19:29:30.281 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:29:29 smithi089 bash[33173]: audit 2023-06-03T19:29:29.459970+0000 mon.a (mon.0) 425 : audit [INF] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2023-06-03T19:29:30.281 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:29:29 smithi089 bash[33173]: audit 2023-06-03T19:29:29.461655+0000 mon.a (mon.0) 426 : audit [DBG] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-03T19:29:30.960 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:29:30 smithi164 bash[26855]: cephadm 2023-06-03T19:29:29.463142+0000 mgr.y (mgr.14146) 208 : cephadm [INF] Deploying daemon osd.5 on smithi164 2023-06-03T19:29:30.961 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:29:30 smithi164 bash[26855]: cluster 2023-06-03T19:29:29.765045+0000 mgr.y (mgr.14146) 209 : cluster [DBG] pgmap v190: 1 pgs: 1 active+clean; 0 B data, 25 MiB used, 447 GiB / 447 GiB avail 2023-06-03T19:29:31.281 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:29:30 smithi089 bash[26572]: cephadm 2023-06-03T19:29:29.463142+0000 mgr.y (mgr.14146) 208 : cephadm [INF] Deploying daemon osd.5 on smithi164 2023-06-03T19:29:31.281 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:29:30 smithi089 bash[26572]: cluster 2023-06-03T19:29:29.765045+0000 mgr.y (mgr.14146) 209 : cluster [DBG] pgmap v190: 1 pgs: 1 active+clean; 0 B data, 25 MiB used, 447 GiB / 447 GiB avail 2023-06-03T19:29:31.281 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:29:30 smithi089 bash[33173]: cephadm 2023-06-03T19:29:29.463142+0000 mgr.y (mgr.14146) 208 : cephadm [INF] Deploying daemon osd.5 on smithi164 2023-06-03T19:29:31.281 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:29:30 smithi089 bash[33173]: cluster 2023-06-03T19:29:29.765045+0000 mgr.y (mgr.14146) 209 : cluster [DBG] pgmap v190: 1 pgs: 1 active+clean; 0 B data, 25 MiB used, 447 GiB / 447 GiB avail 2023-06-03T19:29:32.585 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:29:32 smithi164 bash[26855]: cluster 2023-06-03T19:29:31.765594+0000 mgr.y (mgr.14146) 210 : cluster [DBG] pgmap v191: 1 pgs: 1 active+clean; 0 B data, 25 MiB used, 447 GiB / 447 GiB avail 2023-06-03T19:29:32.785 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:29:32 smithi089 bash[26572]: cluster 2023-06-03T19:29:31.765594+0000 mgr.y (mgr.14146) 210 : cluster [DBG] pgmap v191: 1 pgs: 1 active+clean; 0 B data, 25 MiB used, 447 GiB / 447 GiB avail 2023-06-03T19:29:32.787 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:29:32 smithi089 bash[33173]: cluster 2023-06-03T19:29:31.765594+0000 mgr.y (mgr.14146) 210 : cluster [DBG] pgmap v191: 1 pgs: 1 active+clean; 0 B data, 25 MiB used, 447 GiB / 447 GiB avail 2023-06-03T19:29:35.085 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:29:34 smithi164 bash[26855]: cluster 2023-06-03T19:29:33.766119+0000 mgr.y (mgr.14146) 211 : cluster [DBG] pgmap v192: 1 pgs: 1 active+clean; 0 B data, 25 MiB used, 447 GiB / 447 GiB avail 2023-06-03T19:29:35.085 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:29:34 smithi164 bash[26855]: audit 2023-06-03T19:29:33.868997+0000 mon.a (mon.0) 427 : audit [DBG] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-03T19:29:35.086 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:29:34 smithi164 bash[26855]: audit 2023-06-03T19:29:33.873382+0000 mon.a (mon.0) 428 : audit [INF] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' 2023-06-03T19:29:35.281 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:29:34 smithi089 bash[26572]: cluster 2023-06-03T19:29:33.766119+0000 mgr.y (mgr.14146) 211 : cluster [DBG] pgmap v192: 1 pgs: 1 active+clean; 0 B data, 25 MiB used, 447 GiB / 447 GiB avail 2023-06-03T19:29:35.281 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:29:34 smithi089 bash[26572]: audit 2023-06-03T19:29:33.868997+0000 mon.a (mon.0) 427 : audit [DBG] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-03T19:29:35.281 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:29:34 smithi089 bash[26572]: audit 2023-06-03T19:29:33.873382+0000 mon.a (mon.0) 428 : audit [INF] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' 2023-06-03T19:29:35.281 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:29:34 smithi089 bash[33173]: cluster 2023-06-03T19:29:33.766119+0000 mgr.y (mgr.14146) 211 : cluster [DBG] pgmap v192: 1 pgs: 1 active+clean; 0 B data, 25 MiB used, 447 GiB / 447 GiB avail 2023-06-03T19:29:35.282 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:29:34 smithi089 bash[33173]: audit 2023-06-03T19:29:33.868997+0000 mon.a (mon.0) 427 : audit [DBG] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-03T19:29:35.282 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:29:34 smithi089 bash[33173]: audit 2023-06-03T19:29:33.873382+0000 mon.a (mon.0) 428 : audit [INF] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' 2023-06-03T19:29:37.126 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:29:36 smithi164 bash[26855]: cluster 2023-06-03T19:29:35.766632+0000 mgr.y (mgr.14146) 212 : cluster [DBG] pgmap v193: 1 pgs: 1 active+clean; 0 B data, 25 MiB used, 447 GiB / 447 GiB avail 2023-06-03T19:29:37.281 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:29:36 smithi089 bash[26572]: cluster 2023-06-03T19:29:35.766632+0000 mgr.y (mgr.14146) 212 : cluster [DBG] pgmap v193: 1 pgs: 1 active+clean; 0 B data, 25 MiB used, 447 GiB / 447 GiB avail 2023-06-03T19:29:37.281 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:29:36 smithi089 bash[33173]: cluster 2023-06-03T19:29:35.766632+0000 mgr.y (mgr.14146) 212 : cluster [DBG] pgmap v193: 1 pgs: 1 active+clean; 0 B data, 25 MiB used, 447 GiB / 447 GiB avail 2023-06-03T19:29:39.281 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:29:38 smithi089 bash[33173]: cluster 2023-06-03T19:29:37.767208+0000 mgr.y (mgr.14146) 213 : cluster [DBG] pgmap v194: 1 pgs: 1 active+clean; 0 B data, 25 MiB used, 447 GiB / 447 GiB avail 2023-06-03T19:29:39.281 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:29:38 smithi089 bash[26572]: cluster 2023-06-03T19:29:37.767208+0000 mgr.y (mgr.14146) 213 : cluster [DBG] pgmap v194: 1 pgs: 1 active+clean; 0 B data, 25 MiB used, 447 GiB / 447 GiB avail 2023-06-03T19:29:39.335 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:29:38 smithi164 bash[26855]: cluster 2023-06-03T19:29:37.767208+0000 mgr.y (mgr.14146) 213 : cluster [DBG] pgmap v194: 1 pgs: 1 active+clean; 0 B data, 25 MiB used, 447 GiB / 447 GiB avail 2023-06-03T19:29:41.335 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:29:41 smithi164 bash[26855]: cluster 2023-06-03T19:29:39.767730+0000 mgr.y (mgr.14146) 214 : cluster [DBG] pgmap v195: 1 pgs: 1 active+clean; 0 B data, 25 MiB used, 447 GiB / 447 GiB avail 2023-06-03T19:29:41.336 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:29:41 smithi164 bash[26855]: audit 2023-06-03T19:29:40.072825+0000 mon.a (mon.0) 429 : audit [INF] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' 2023-06-03T19:29:41.336 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:29:41 smithi164 bash[26855]: audit 2023-06-03T19:29:40.074192+0000 mon.a (mon.0) 430 : audit [DBG] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-03T19:29:41.336 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:29:41 smithi164 bash[26855]: audit 2023-06-03T19:29:40.075630+0000 mon.a (mon.0) 431 : audit [INF] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-03T19:29:41.336 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:29:41 smithi164 bash[26855]: audit 2023-06-03T19:29:40.083299+0000 mon.a (mon.0) 432 : audit [INF] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' 2023-06-03T19:29:41.336 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:29:41 smithi164 bash[26855]: audit 2023-06-03T19:29:40.613876+0000 mon.b (mon.1) 9 : audit [INF] from='osd.5 [v2:172.21.15.164:6808/3123006405,v1:172.21.15.164:6809/3123006405]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2023-06-03T19:29:41.337 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:29:41 smithi164 bash[26855]: audit 2023-06-03T19:29:40.616217+0000 mon.a (mon.0) 433 : audit [INF] from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2023-06-03T19:29:41.531 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:29:41 smithi089 bash[33173]: cluster 2023-06-03T19:29:39.767730+0000 mgr.y (mgr.14146) 214 : cluster [DBG] pgmap v195: 1 pgs: 1 active+clean; 0 B data, 25 MiB used, 447 GiB / 447 GiB avail 2023-06-03T19:29:41.531 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:29:41 smithi089 bash[33173]: audit 2023-06-03T19:29:40.072825+0000 mon.a (mon.0) 429 : audit [INF] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' 2023-06-03T19:29:41.531 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:29:41 smithi089 bash[33173]: audit 2023-06-03T19:29:40.074192+0000 mon.a (mon.0) 430 : audit [DBG] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-03T19:29:41.531 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:29:41 smithi089 bash[33173]: audit 2023-06-03T19:29:40.075630+0000 mon.a (mon.0) 431 : audit [INF] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-03T19:29:41.532 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:29:41 smithi089 bash[33173]: audit 2023-06-03T19:29:40.083299+0000 mon.a (mon.0) 432 : audit [INF] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' 2023-06-03T19:29:41.532 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:29:41 smithi089 bash[33173]: audit 2023-06-03T19:29:40.613876+0000 mon.b (mon.1) 9 : audit [INF] from='osd.5 [v2:172.21.15.164:6808/3123006405,v1:172.21.15.164:6809/3123006405]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2023-06-03T19:29:41.532 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:29:41 smithi089 bash[33173]: audit 2023-06-03T19:29:40.616217+0000 mon.a (mon.0) 433 : audit [INF] from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2023-06-03T19:29:41.532 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:29:41 smithi089 bash[26572]: cluster 2023-06-03T19:29:39.767730+0000 mgr.y (mgr.14146) 214 : cluster [DBG] pgmap v195: 1 pgs: 1 active+clean; 0 B data, 25 MiB used, 447 GiB / 447 GiB avail 2023-06-03T19:29:41.532 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:29:41 smithi089 bash[26572]: audit 2023-06-03T19:29:40.072825+0000 mon.a (mon.0) 429 : audit [INF] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' 2023-06-03T19:29:41.532 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:29:41 smithi089 bash[26572]: audit 2023-06-03T19:29:40.074192+0000 mon.a (mon.0) 430 : audit [DBG] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-03T19:29:41.533 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:29:41 smithi089 bash[26572]: audit 2023-06-03T19:29:40.075630+0000 mon.a (mon.0) 431 : audit [INF] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-03T19:29:41.533 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:29:41 smithi089 bash[26572]: audit 2023-06-03T19:29:40.083299+0000 mon.a (mon.0) 432 : audit [INF] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' 2023-06-03T19:29:41.533 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:29:41 smithi089 bash[26572]: audit 2023-06-03T19:29:40.613876+0000 mon.b (mon.1) 9 : audit [INF] from='osd.5 [v2:172.21.15.164:6808/3123006405,v1:172.21.15.164:6809/3123006405]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2023-06-03T19:29:41.533 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:29:41 smithi089 bash[26572]: audit 2023-06-03T19:29:40.616217+0000 mon.a (mon.0) 433 : audit [INF] from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2023-06-03T19:29:42.420 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:29:42 smithi164 bash[26855]: audit 2023-06-03T19:29:41.084584+0000 mon.b (mon.1) 10 : audit [INF] from='osd.5 [v2:172.21.15.164:6808/3123006405,v1:172.21.15.164:6809/3123006405]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi164", "root=default"]}]: dispatch 2023-06-03T19:29:42.421 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:29:42 smithi164 bash[26855]: audit 2023-06-03T19:29:41.085201+0000 mon.a (mon.0) 434 : audit [INF] from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2023-06-03T19:29:42.421 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:29:42 smithi164 bash[26855]: cluster 2023-06-03T19:29:41.085380+0000 mon.a (mon.0) 435 : cluster [DBG] osdmap e34: 6 total, 5 up, 6 in 2023-06-03T19:29:42.421 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:29:42 smithi164 bash[26855]: audit 2023-06-03T19:29:41.085609+0000 mon.a (mon.0) 436 : audit [DBG] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-06-03T19:29:42.421 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:29:42 smithi164 bash[26855]: audit 2023-06-03T19:29:41.086857+0000 mon.a (mon.0) 437 : audit [INF] from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi164", "root=default"]}]: dispatch 2023-06-03T19:29:42.531 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:29:42 smithi089 bash[33173]: audit 2023-06-03T19:29:41.084584+0000 mon.b (mon.1) 10 : audit [INF] from='osd.5 [v2:172.21.15.164:6808/3123006405,v1:172.21.15.164:6809/3123006405]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi164", "root=default"]}]: dispatch 2023-06-03T19:29:42.531 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:29:42 smithi089 bash[33173]: audit 2023-06-03T19:29:41.085201+0000 mon.a (mon.0) 434 : audit [INF] from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2023-06-03T19:29:42.531 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:29:42 smithi089 bash[33173]: cluster 2023-06-03T19:29:41.085380+0000 mon.a (mon.0) 435 : cluster [DBG] osdmap e34: 6 total, 5 up, 6 in 2023-06-03T19:29:42.531 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:29:42 smithi089 bash[33173]: audit 2023-06-03T19:29:41.085609+0000 mon.a (mon.0) 436 : audit [DBG] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-06-03T19:29:42.531 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:29:42 smithi089 bash[33173]: audit 2023-06-03T19:29:41.086857+0000 mon.a (mon.0) 437 : audit [INF] from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi164", "root=default"]}]: dispatch 2023-06-03T19:29:42.532 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:29:42 smithi089 bash[26572]: audit 2023-06-03T19:29:41.084584+0000 mon.b (mon.1) 10 : audit [INF] from='osd.5 [v2:172.21.15.164:6808/3123006405,v1:172.21.15.164:6809/3123006405]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi164", "root=default"]}]: dispatch 2023-06-03T19:29:42.532 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:29:42 smithi089 bash[26572]: audit 2023-06-03T19:29:41.085201+0000 mon.a (mon.0) 434 : audit [INF] from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2023-06-03T19:29:42.532 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:29:42 smithi089 bash[26572]: cluster 2023-06-03T19:29:41.085380+0000 mon.a (mon.0) 435 : cluster [DBG] osdmap e34: 6 total, 5 up, 6 in 2023-06-03T19:29:42.532 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:29:42 smithi089 bash[26572]: audit 2023-06-03T19:29:41.085609+0000 mon.a (mon.0) 436 : audit [DBG] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-06-03T19:29:42.532 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:29:42 smithi089 bash[26572]: audit 2023-06-03T19:29:41.086857+0000 mon.a (mon.0) 437 : audit [INF] from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi164", "root=default"]}]: dispatch 2023-06-03T19:29:43.335 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:29:43 smithi164 bash[26855]: cluster 2023-06-03T19:29:41.768286+0000 mgr.y (mgr.14146) 215 : cluster [DBG] pgmap v197: 1 pgs: 1 active+clean; 0 B data, 25 MiB used, 447 GiB / 447 GiB avail 2023-06-03T19:29:43.336 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:29:43 smithi164 bash[26855]: audit 2023-06-03T19:29:42.088902+0000 mon.a (mon.0) 438 : audit [INF] from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi164", "root=default"]}]': finished 2023-06-03T19:29:43.336 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:29:43 smithi164 bash[26855]: cluster 2023-06-03T19:29:42.089024+0000 mon.a (mon.0) 439 : cluster [DBG] osdmap e35: 6 total, 5 up, 6 in 2023-06-03T19:29:43.336 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:29:43 smithi164 bash[26855]: audit 2023-06-03T19:29:42.089738+0000 mon.a (mon.0) 440 : audit [DBG] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-06-03T19:29:43.336 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:29:43 smithi164 bash[26855]: audit 2023-06-03T19:29:42.096277+0000 mon.a (mon.0) 441 : audit [DBG] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-06-03T19:29:43.531 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:29:43 smithi089 bash[26572]: cluster 2023-06-03T19:29:41.768286+0000 mgr.y (mgr.14146) 215 : cluster [DBG] pgmap v197: 1 pgs: 1 active+clean; 0 B data, 25 MiB used, 447 GiB / 447 GiB avail 2023-06-03T19:29:43.531 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:29:43 smithi089 bash[26572]: audit 2023-06-03T19:29:42.088902+0000 mon.a (mon.0) 438 : audit [INF] from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi164", "root=default"]}]': finished 2023-06-03T19:29:43.531 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:29:43 smithi089 bash[26572]: cluster 2023-06-03T19:29:42.089024+0000 mon.a (mon.0) 439 : cluster [DBG] osdmap e35: 6 total, 5 up, 6 in 2023-06-03T19:29:43.531 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:29:43 smithi089 bash[26572]: audit 2023-06-03T19:29:42.089738+0000 mon.a (mon.0) 440 : audit [DBG] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-06-03T19:29:43.532 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:29:43 smithi089 bash[26572]: audit 2023-06-03T19:29:42.096277+0000 mon.a (mon.0) 441 : audit [DBG] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-06-03T19:29:43.532 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:29:43 smithi089 bash[33173]: cluster 2023-06-03T19:29:41.768286+0000 mgr.y (mgr.14146) 215 : cluster [DBG] pgmap v197: 1 pgs: 1 active+clean; 0 B data, 25 MiB used, 447 GiB / 447 GiB avail 2023-06-03T19:29:43.532 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:29:43 smithi089 bash[33173]: audit 2023-06-03T19:29:42.088902+0000 mon.a (mon.0) 438 : audit [INF] from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi164", "root=default"]}]': finished 2023-06-03T19:29:43.532 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:29:43 smithi089 bash[33173]: cluster 2023-06-03T19:29:42.089024+0000 mon.a (mon.0) 439 : cluster [DBG] osdmap e35: 6 total, 5 up, 6 in 2023-06-03T19:29:43.532 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:29:43 smithi089 bash[33173]: audit 2023-06-03T19:29:42.089738+0000 mon.a (mon.0) 440 : audit [DBG] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-06-03T19:29:43.532 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:29:43 smithi089 bash[33173]: audit 2023-06-03T19:29:42.096277+0000 mon.a (mon.0) 441 : audit [DBG] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-06-03T19:29:43.707 INFO:teuthology.orchestra.run.smithi164.stdout:Created osd(s) 5 on host 'smithi164' 2023-06-03T19:29:44.335 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:29:44 smithi164 bash[26855]: cluster 2023-06-03T19:29:41.572467+0000 osd.5 (osd.5) 1 : cluster [DBG] purged_snaps scrub starts 2023-06-03T19:29:44.335 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:29:44 smithi164 bash[26855]: cluster 2023-06-03T19:29:41.572529+0000 osd.5 (osd.5) 2 : cluster [DBG] purged_snaps scrub ok 2023-06-03T19:29:44.336 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:29:44 smithi164 bash[26855]: audit 2023-06-03T19:29:43.098953+0000 mon.a (mon.0) 442 : audit [DBG] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-06-03T19:29:44.336 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:29:44 smithi164 bash[26855]: cluster 2023-06-03T19:29:43.102435+0000 mon.a (mon.0) 443 : cluster [INF] osd.5 [v2:172.21.15.164:6808/3123006405,v1:172.21.15.164:6809/3123006405] boot 2023-06-03T19:29:44.336 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:29:44 smithi164 bash[26855]: cluster 2023-06-03T19:29:43.102469+0000 mon.a (mon.0) 444 : cluster [DBG] osdmap e36: 6 total, 6 up, 6 in 2023-06-03T19:29:44.336 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:29:44 smithi164 bash[26855]: audit 2023-06-03T19:29:43.102627+0000 mon.a (mon.0) 445 : audit [DBG] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-06-03T19:29:44.336 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:29:44 smithi164 bash[26855]: audit 2023-06-03T19:29:43.699194+0000 mon.a (mon.0) 446 : audit [INF] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' 2023-06-03T19:29:44.336 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:29:44 smithi164 bash[26855]: audit 2023-06-03T19:29:43.711049+0000 mon.a (mon.0) 447 : audit [DBG] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-03T19:29:44.366 DEBUG:teuthology.orchestra.run.smithi164:osd.5> sudo journalctl -f -n 0 -u ceph-f1db3478-0243-11ee-9b1d-001a4aab830c@osd.5.service 2023-06-03T19:29:44.370 INFO:tasks.cephadm:Deploying osd.6 on smithi164 with /dev/vg_nvme/lv_2... 2023-06-03T19:29:44.371 DEBUG:teuthology.orchestra.run.smithi164:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph-volume lvm zap /dev/vg_nvme/lv_2 2023-06-03T19:29:44.531 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:29:44 smithi089 bash[26572]: cluster 2023-06-03T19:29:41.572467+0000 osd.5 (osd.5) 1 : cluster [DBG] purged_snaps scrub starts 2023-06-03T19:29:44.531 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:29:44 smithi089 bash[26572]: cluster 2023-06-03T19:29:41.572529+0000 osd.5 (osd.5) 2 : cluster [DBG] purged_snaps scrub ok 2023-06-03T19:29:44.532 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:29:44 smithi089 bash[26572]: audit 2023-06-03T19:29:43.098953+0000 mon.a (mon.0) 442 : audit [DBG] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-06-03T19:29:44.532 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:29:44 smithi089 bash[26572]: cluster 2023-06-03T19:29:43.102435+0000 mon.a (mon.0) 443 : cluster [INF] osd.5 [v2:172.21.15.164:6808/3123006405,v1:172.21.15.164:6809/3123006405] boot 2023-06-03T19:29:44.532 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:29:44 smithi089 bash[26572]: cluster 2023-06-03T19:29:43.102469+0000 mon.a (mon.0) 444 : cluster [DBG] osdmap e36: 6 total, 6 up, 6 in 2023-06-03T19:29:44.532 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:29:44 smithi089 bash[26572]: audit 2023-06-03T19:29:43.102627+0000 mon.a (mon.0) 445 : audit [DBG] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-06-03T19:29:44.532 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:29:44 smithi089 bash[26572]: audit 2023-06-03T19:29:43.699194+0000 mon.a (mon.0) 446 : audit [INF] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' 2023-06-03T19:29:44.533 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:29:44 smithi089 bash[26572]: audit 2023-06-03T19:29:43.711049+0000 mon.a (mon.0) 447 : audit [DBG] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-03T19:29:44.533 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:29:44 smithi089 bash[33173]: cluster 2023-06-03T19:29:41.572467+0000 osd.5 (osd.5) 1 : cluster [DBG] purged_snaps scrub starts 2023-06-03T19:29:44.533 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:29:44 smithi089 bash[33173]: cluster 2023-06-03T19:29:41.572529+0000 osd.5 (osd.5) 2 : cluster [DBG] purged_snaps scrub ok 2023-06-03T19:29:44.533 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:29:44 smithi089 bash[33173]: audit 2023-06-03T19:29:43.098953+0000 mon.a (mon.0) 442 : audit [DBG] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-06-03T19:29:44.534 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:29:44 smithi089 bash[33173]: cluster 2023-06-03T19:29:43.102435+0000 mon.a (mon.0) 443 : cluster [INF] osd.5 [v2:172.21.15.164:6808/3123006405,v1:172.21.15.164:6809/3123006405] boot 2023-06-03T19:29:44.534 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:29:44 smithi089 bash[33173]: cluster 2023-06-03T19:29:43.102469+0000 mon.a (mon.0) 444 : cluster [DBG] osdmap e36: 6 total, 6 up, 6 in 2023-06-03T19:29:44.534 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:29:44 smithi089 bash[33173]: audit 2023-06-03T19:29:43.102627+0000 mon.a (mon.0) 445 : audit [DBG] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-06-03T19:29:44.534 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:29:44 smithi089 bash[33173]: audit 2023-06-03T19:29:43.699194+0000 mon.a (mon.0) 446 : audit [INF] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' 2023-06-03T19:29:44.534 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:29:44 smithi089 bash[33173]: audit 2023-06-03T19:29:43.711049+0000 mon.a (mon.0) 447 : audit [DBG] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-03T19:29:44.640 INFO:journalctl@ceph.osd.5.smithi164.stdout:-- Logs begin at Sun 2022-08-14 12:20:24 UTC. -- 2023-06-03T19:29:45.494 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:29:45 smithi164 bash[26855]: cluster 2023-06-03T19:29:43.768863+0000 mgr.y (mgr.14146) 216 : cluster [DBG] pgmap v200: 1 pgs: 1 active+clean; 0 B data, 25 MiB used, 447 GiB / 447 GiB avail 2023-06-03T19:29:45.494 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:29:45 smithi164 bash[26855]: cluster 2023-06-03T19:29:44.108102+0000 mon.a (mon.0) 448 : cluster [DBG] osdmap e37: 6 total, 6 up, 6 in 2023-06-03T19:29:45.531 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:29:45 smithi089 bash[26572]: cluster 2023-06-03T19:29:43.768863+0000 mgr.y (mgr.14146) 216 : cluster [DBG] pgmap v200: 1 pgs: 1 active+clean; 0 B data, 25 MiB used, 447 GiB / 447 GiB avail 2023-06-03T19:29:45.531 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:29:45 smithi089 bash[26572]: cluster 2023-06-03T19:29:44.108102+0000 mon.a (mon.0) 448 : cluster [DBG] osdmap e37: 6 total, 6 up, 6 in 2023-06-03T19:29:45.532 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:29:45 smithi089 bash[33173]: cluster 2023-06-03T19:29:43.768863+0000 mgr.y (mgr.14146) 216 : cluster [DBG] pgmap v200: 1 pgs: 1 active+clean; 0 B data, 25 MiB used, 447 GiB / 447 GiB avail 2023-06-03T19:29:45.532 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:29:45 smithi089 bash[33173]: cluster 2023-06-03T19:29:44.108102+0000 mon.a (mon.0) 448 : cluster [DBG] osdmap e37: 6 total, 6 up, 6 in 2023-06-03T19:29:46.531 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:29:46 smithi089 bash[33173]: cluster 2023-06-03T19:29:45.114165+0000 mon.a (mon.0) 449 : cluster [DBG] osdmap e38: 6 total, 6 up, 6 in 2023-06-03T19:29:46.531 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:29:46 smithi089 bash[26572]: cluster 2023-06-03T19:29:45.114165+0000 mon.a (mon.0) 449 : cluster [DBG] osdmap e38: 6 total, 6 up, 6 in 2023-06-03T19:29:46.585 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:29:46 smithi164 bash[26855]: cluster 2023-06-03T19:29:45.114165+0000 mon.a (mon.0) 449 : cluster [DBG] osdmap e38: 6 total, 6 up, 6 in 2023-06-03T19:29:47.531 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:29:47 smithi089 bash[26572]: cluster 2023-06-03T19:29:45.769406+0000 mgr.y (mgr.14146) 217 : cluster [DBG] pgmap v203: 1 pgs: 1 active+clean; 0 B data, 30 MiB used, 536 GiB / 536 GiB avail 2023-06-03T19:29:47.532 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:29:47 smithi089 bash[33173]: cluster 2023-06-03T19:29:45.769406+0000 mgr.y (mgr.14146) 217 : cluster [DBG] pgmap v203: 1 pgs: 1 active+clean; 0 B data, 30 MiB used, 536 GiB / 536 GiB avail 2023-06-03T19:29:47.585 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:29:47 smithi164 bash[26855]: cluster 2023-06-03T19:29:45.769406+0000 mgr.y (mgr.14146) 217 : cluster [DBG] pgmap v203: 1 pgs: 1 active+clean; 0 B data, 30 MiB used, 536 GiB / 536 GiB avail 2023-06-03T19:29:47.901 INFO:teuthology.orchestra.run.smithi164.stderr:--> Zapping: /dev/vg_nvme/lv_2 2023-06-03T19:29:47.901 INFO:teuthology.orchestra.run.smithi164.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_2 bs=1M count=10 conv=fsync 2023-06-03T19:29:47.902 INFO:teuthology.orchestra.run.smithi164.stderr: stderr: 10+0 records in 2023-06-03T19:29:47.902 INFO:teuthology.orchestra.run.smithi164.stderr:10+0 records out 2023-06-03T19:29:47.902 INFO:teuthology.orchestra.run.smithi164.stderr:10485760 bytes (10 MB, 10 MiB) copied, 0.017014 s, 616 MB/s 2023-06-03T19:29:47.902 INFO:teuthology.orchestra.run.smithi164.stderr:--> Zapping successful for: 2023-06-03T19:29:48.617 DEBUG:teuthology.orchestra.run.smithi164:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph orch daemon add osd smithi164:vg_nvme/lv_2 2023-06-03T19:29:49.335 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:29:49 smithi164 bash[26855]: cluster 2023-06-03T19:29:47.769924+0000 mgr.y (mgr.14146) 218 : cluster [DBG] pgmap v204: 1 pgs: 1 active+clean; 0 B data, 30 MiB used, 536 GiB / 536 GiB avail 2023-06-03T19:29:49.531 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:29:49 smithi089 bash[26572]: cluster 2023-06-03T19:29:47.769924+0000 mgr.y (mgr.14146) 218 : cluster [DBG] pgmap v204: 1 pgs: 1 active+clean; 0 B data, 30 MiB used, 536 GiB / 536 GiB avail 2023-06-03T19:29:49.531 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:29:49 smithi089 bash[33173]: cluster 2023-06-03T19:29:47.769924+0000 mgr.y (mgr.14146) 218 : cluster [DBG] pgmap v204: 1 pgs: 1 active+clean; 0 B data, 30 MiB used, 536 GiB / 536 GiB avail 2023-06-03T19:29:51.432 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:29:51 smithi164 bash[26855]: cluster 2023-06-03T19:29:49.770643+0000 mgr.y (mgr.14146) 219 : cluster [DBG] pgmap v205: 1 pgs: 1 active+clean; 0 B data, 30 MiB used, 536 GiB / 536 GiB avail 2023-06-03T19:29:51.531 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:29:51 smithi089 bash[26572]: cluster 2023-06-03T19:29:49.770643+0000 mgr.y (mgr.14146) 219 : cluster [DBG] pgmap v205: 1 pgs: 1 active+clean; 0 B data, 30 MiB used, 536 GiB / 536 GiB avail 2023-06-03T19:29:51.531 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:29:51 smithi089 bash[33173]: cluster 2023-06-03T19:29:49.770643+0000 mgr.y (mgr.14146) 219 : cluster [DBG] pgmap v205: 1 pgs: 1 active+clean; 0 B data, 30 MiB used, 536 GiB / 536 GiB avail 2023-06-03T19:29:52.781 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:29:52 smithi089 bash[33173]: cluster 2023-06-03T19:29:51.771321+0000 mgr.y (mgr.14146) 220 : cluster [DBG] pgmap v206: 1 pgs: 1 active+clean; 0 B data, 30 MiB used, 536 GiB / 536 GiB avail 2023-06-03T19:29:52.781 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:29:52 smithi089 bash[26572]: cluster 2023-06-03T19:29:51.771321+0000 mgr.y (mgr.14146) 220 : cluster [DBG] pgmap v206: 1 pgs: 1 active+clean; 0 B data, 30 MiB used, 536 GiB / 536 GiB avail 2023-06-03T19:29:52.803 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:29:52 smithi164 bash[26855]: cluster 2023-06-03T19:29:51.771321+0000 mgr.y (mgr.14146) 220 : cluster [DBG] pgmap v206: 1 pgs: 1 active+clean; 0 B data, 30 MiB used, 536 GiB / 536 GiB avail 2023-06-03T19:29:53.765 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:29:53 smithi164 bash[26855]: audit 2023-06-03T19:29:53.216468+0000 mon.a (mon.0) 450 : audit [DBG] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-06-03T19:29:53.766 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:29:53 smithi164 bash[26855]: audit 2023-06-03T19:29:53.220548+0000 mon.a (mon.0) 451 : audit [INF] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-06-03T19:29:53.766 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:29:53 smithi164 bash[26855]: audit 2023-06-03T19:29:53.221891+0000 mon.a (mon.0) 452 : audit [DBG] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-03T19:29:53.781 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:29:53 smithi089 bash[26572]: audit 2023-06-03T19:29:53.216468+0000 mon.a (mon.0) 450 : audit [DBG] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-06-03T19:29:53.781 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:29:53 smithi089 bash[26572]: audit 2023-06-03T19:29:53.220548+0000 mon.a (mon.0) 451 : audit [INF] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-06-03T19:29:53.781 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:29:53 smithi089 bash[26572]: audit 2023-06-03T19:29:53.221891+0000 mon.a (mon.0) 452 : audit [DBG] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-03T19:29:53.782 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:29:53 smithi089 bash[33173]: audit 2023-06-03T19:29:53.216468+0000 mon.a (mon.0) 450 : audit [DBG] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-06-03T19:29:53.782 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:29:53 smithi089 bash[33173]: audit 2023-06-03T19:29:53.220548+0000 mon.a (mon.0) 451 : audit [INF] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-06-03T19:29:53.782 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:29:53 smithi089 bash[33173]: audit 2023-06-03T19:29:53.221891+0000 mon.a (mon.0) 452 : audit [DBG] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-03T19:29:54.781 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:29:54 smithi089 bash[26572]: audit 2023-06-03T19:29:53.209572+0000 mgr.y (mgr.14146) 221 : audit [DBG] from='client.24277 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi164:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2023-06-03T19:29:54.781 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:29:54 smithi089 bash[26572]: cluster 2023-06-03T19:29:53.772006+0000 mgr.y (mgr.14146) 222 : cluster [DBG] pgmap v207: 1 pgs: 1 active+clean; 0 B data, 30 MiB used, 536 GiB / 536 GiB avail 2023-06-03T19:29:54.781 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:29:54 smithi089 bash[33173]: audit 2023-06-03T19:29:53.209572+0000 mgr.y (mgr.14146) 221 : audit [DBG] from='client.24277 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi164:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2023-06-03T19:29:54.781 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:29:54 smithi089 bash[33173]: cluster 2023-06-03T19:29:53.772006+0000 mgr.y (mgr.14146) 222 : cluster [DBG] pgmap v207: 1 pgs: 1 active+clean; 0 B data, 30 MiB used, 536 GiB / 536 GiB avail 2023-06-03T19:29:54.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:29:54 smithi164 bash[26855]: audit 2023-06-03T19:29:53.209572+0000 mgr.y (mgr.14146) 221 : audit [DBG] from='client.24277 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi164:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2023-06-03T19:29:54.836 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:29:54 smithi164 bash[26855]: cluster 2023-06-03T19:29:53.772006+0000 mgr.y (mgr.14146) 222 : cluster [DBG] pgmap v207: 1 pgs: 1 active+clean; 0 B data, 30 MiB used, 536 GiB / 536 GiB avail 2023-06-03T19:29:55.781 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:29:55 smithi089 bash[26572]: cephadm 2023-06-03T19:29:54.466607+0000 mgr.y (mgr.14146) 223 : cephadm [INF] Detected new or changed devices on smithi164 2023-06-03T19:29:55.781 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:29:55 smithi089 bash[26572]: audit 2023-06-03T19:29:54.472609+0000 mon.a (mon.0) 453 : audit [INF] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' 2023-06-03T19:29:55.782 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:29:55 smithi089 bash[26572]: audit 2023-06-03T19:29:54.474060+0000 mon.a (mon.0) 454 : audit [INF] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-06-03T19:29:55.782 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:29:55 smithi089 bash[26572]: audit 2023-06-03T19:29:54.475305+0000 mon.a (mon.0) 455 : audit [INF] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-06-03T19:29:55.782 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:29:55 smithi089 bash[26572]: cephadm 2023-06-03T19:29:54.476279+0000 mgr.y (mgr.14146) 224 : cephadm [INF] Adjusting osd_memory_target on smithi164 to 8637M 2023-06-03T19:29:55.782 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:29:55 smithi089 bash[26572]: audit 2023-06-03T19:29:54.481401+0000 mon.a (mon.0) 456 : audit [INF] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' 2023-06-03T19:29:55.782 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:29:55 smithi089 bash[26572]: audit 2023-06-03T19:29:54.482767+0000 mon.a (mon.0) 457 : audit [DBG] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-03T19:29:55.782 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:29:55 smithi089 bash[26572]: audit 2023-06-03T19:29:54.483770+0000 mon.a (mon.0) 458 : audit [INF] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-03T19:29:55.782 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:29:55 smithi089 bash[26572]: audit 2023-06-03T19:29:54.490138+0000 mon.a (mon.0) 459 : audit [INF] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' 2023-06-03T19:29:55.783 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:29:55 smithi089 bash[33173]: cephadm 2023-06-03T19:29:54.466607+0000 mgr.y (mgr.14146) 223 : cephadm [INF] Detected new or changed devices on smithi164 2023-06-03T19:29:55.783 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:29:55 smithi089 bash[33173]: audit 2023-06-03T19:29:54.472609+0000 mon.a (mon.0) 453 : audit [INF] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' 2023-06-03T19:29:55.783 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:29:55 smithi089 bash[33173]: audit 2023-06-03T19:29:54.474060+0000 mon.a (mon.0) 454 : audit [INF] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-06-03T19:29:55.783 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:29:55 smithi089 bash[33173]: audit 2023-06-03T19:29:54.475305+0000 mon.a (mon.0) 455 : audit [INF] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-06-03T19:29:55.783 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:29:55 smithi089 bash[33173]: cephadm 2023-06-03T19:29:54.476279+0000 mgr.y (mgr.14146) 224 : cephadm [INF] Adjusting osd_memory_target on smithi164 to 8637M 2023-06-03T19:29:55.783 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:29:55 smithi089 bash[33173]: audit 2023-06-03T19:29:54.481401+0000 mon.a (mon.0) 456 : audit [INF] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' 2023-06-03T19:29:55.784 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:29:55 smithi089 bash[33173]: audit 2023-06-03T19:29:54.482767+0000 mon.a (mon.0) 457 : audit [DBG] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-03T19:29:55.784 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:29:55 smithi089 bash[33173]: audit 2023-06-03T19:29:54.483770+0000 mon.a (mon.0) 458 : audit [INF] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-03T19:29:55.784 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:29:55 smithi089 bash[33173]: audit 2023-06-03T19:29:54.490138+0000 mon.a (mon.0) 459 : audit [INF] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' 2023-06-03T19:29:55.836 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:29:55 smithi164 bash[26855]: cephadm 2023-06-03T19:29:54.466607+0000 mgr.y (mgr.14146) 223 : cephadm [INF] Detected new or changed devices on smithi164 2023-06-03T19:29:55.837 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:29:55 smithi164 bash[26855]: audit 2023-06-03T19:29:54.472609+0000 mon.a (mon.0) 453 : audit [INF] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' 2023-06-03T19:29:55.837 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:29:55 smithi164 bash[26855]: audit 2023-06-03T19:29:54.474060+0000 mon.a (mon.0) 454 : audit [INF] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-06-03T19:29:55.837 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:29:55 smithi164 bash[26855]: audit 2023-06-03T19:29:54.475305+0000 mon.a (mon.0) 455 : audit [INF] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-06-03T19:29:55.837 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:29:55 smithi164 bash[26855]: cephadm 2023-06-03T19:29:54.476279+0000 mgr.y (mgr.14146) 224 : cephadm [INF] Adjusting osd_memory_target on smithi164 to 8637M 2023-06-03T19:29:55.837 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:29:55 smithi164 bash[26855]: audit 2023-06-03T19:29:54.481401+0000 mon.a (mon.0) 456 : audit [INF] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' 2023-06-03T19:29:55.837 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:29:55 smithi164 bash[26855]: audit 2023-06-03T19:29:54.482767+0000 mon.a (mon.0) 457 : audit [DBG] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-03T19:29:55.837 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:29:55 smithi164 bash[26855]: audit 2023-06-03T19:29:54.483770+0000 mon.a (mon.0) 458 : audit [INF] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-03T19:29:55.838 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:29:55 smithi164 bash[26855]: audit 2023-06-03T19:29:54.490138+0000 mon.a (mon.0) 459 : audit [INF] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' 2023-06-03T19:29:56.781 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:29:56 smithi089 bash[26572]: cluster 2023-06-03T19:29:55.772629+0000 mgr.y (mgr.14146) 225 : cluster [DBG] pgmap v208: 1 pgs: 1 active+clean; 0 B data, 30 MiB used, 536 GiB / 536 GiB avail 2023-06-03T19:29:56.781 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:29:56 smithi089 bash[33173]: cluster 2023-06-03T19:29:55.772629+0000 mgr.y (mgr.14146) 225 : cluster [DBG] pgmap v208: 1 pgs: 1 active+clean; 0 B data, 30 MiB used, 536 GiB / 536 GiB avail 2023-06-03T19:29:56.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:29:56 smithi164 bash[26855]: cluster 2023-06-03T19:29:55.772629+0000 mgr.y (mgr.14146) 225 : cluster [DBG] pgmap v208: 1 pgs: 1 active+clean; 0 B data, 30 MiB used, 536 GiB / 536 GiB avail 2023-06-03T19:29:59.149 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:29:58 smithi164 bash[26855]: cluster 2023-06-03T19:29:57.773129+0000 mgr.y (mgr.14146) 226 : cluster [DBG] pgmap v209: 1 pgs: 1 active+clean; 0 B data, 30 MiB used, 536 GiB / 536 GiB avail 2023-06-03T19:29:59.281 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:29:58 smithi089 bash[26572]: cluster 2023-06-03T19:29:57.773129+0000 mgr.y (mgr.14146) 226 : cluster [DBG] pgmap v209: 1 pgs: 1 active+clean; 0 B data, 30 MiB used, 536 GiB / 536 GiB avail 2023-06-03T19:29:59.281 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:29:58 smithi089 bash[33173]: cluster 2023-06-03T19:29:57.773129+0000 mgr.y (mgr.14146) 226 : cluster [DBG] pgmap v209: 1 pgs: 1 active+clean; 0 B data, 30 MiB used, 536 GiB / 536 GiB avail 2023-06-03T19:30:01.085 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:30:00 smithi164 bash[26855]: cluster 2023-06-03T19:29:59.773720+0000 mgr.y (mgr.14146) 227 : cluster [DBG] pgmap v210: 1 pgs: 1 active+clean; 0 B data, 30 MiB used, 536 GiB / 536 GiB avail 2023-06-03T19:30:01.086 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:30:00 smithi164 bash[26855]: cluster 2023-06-03T19:30:00.000165+0000 mon.a (mon.0) 460 : cluster [INF] overall HEALTH_OK 2023-06-03T19:30:01.281 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:30:00 smithi089 bash[26572]: cluster 2023-06-03T19:29:59.773720+0000 mgr.y (mgr.14146) 227 : cluster [DBG] pgmap v210: 1 pgs: 1 active+clean; 0 B data, 30 MiB used, 536 GiB / 536 GiB avail 2023-06-03T19:30:01.281 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:30:00 smithi089 bash[26572]: cluster 2023-06-03T19:30:00.000165+0000 mon.a (mon.0) 460 : cluster [INF] overall HEALTH_OK 2023-06-03T19:30:01.282 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:30:00 smithi089 bash[33173]: cluster 2023-06-03T19:29:59.773720+0000 mgr.y (mgr.14146) 227 : cluster [DBG] pgmap v210: 1 pgs: 1 active+clean; 0 B data, 30 MiB used, 536 GiB / 536 GiB avail 2023-06-03T19:30:01.282 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:30:00 smithi089 bash[33173]: cluster 2023-06-03T19:30:00.000165+0000 mon.a (mon.0) 460 : cluster [INF] overall HEALTH_OK 2023-06-03T19:30:02.781 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:30:02 smithi089 bash[26572]: audit 2023-06-03T19:30:01.624815+0000 mon.b (mon.1) 11 : audit [INF] from='client.? 172.21.15.164:0/3905771273' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "5c5edd3b-d92c-4f46-94ff-2d5828b2e732"}]: dispatch 2023-06-03T19:30:02.781 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:30:02 smithi089 bash[26572]: audit 2023-06-03T19:30:01.627558+0000 mon.a (mon.0) 461 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "5c5edd3b-d92c-4f46-94ff-2d5828b2e732"}]: dispatch 2023-06-03T19:30:02.782 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:30:02 smithi089 bash[26572]: audit 2023-06-03T19:30:01.635423+0000 mon.a (mon.0) 462 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "5c5edd3b-d92c-4f46-94ff-2d5828b2e732"}]': finished 2023-06-03T19:30:02.782 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:30:02 smithi089 bash[26572]: cluster 2023-06-03T19:30:01.635596+0000 mon.a (mon.0) 463 : cluster [DBG] osdmap e39: 7 total, 6 up, 7 in 2023-06-03T19:30:02.782 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:30:02 smithi089 bash[26572]: audit 2023-06-03T19:30:01.635816+0000 mon.a (mon.0) 464 : audit [DBG] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-06-03T19:30:02.782 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:30:02 smithi089 bash[26572]: cluster 2023-06-03T19:30:01.774233+0000 mgr.y (mgr.14146) 228 : cluster [DBG] pgmap v212: 1 pgs: 1 active+clean; 0 B data, 30 MiB used, 536 GiB / 536 GiB avail 2023-06-03T19:30:02.782 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:30:02 smithi089 bash[26572]: audit 2023-06-03T19:30:02.367683+0000 mon.b (mon.1) 12 : audit [DBG] from='client.? 172.21.15.164:0/3849062029' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-06-03T19:30:02.782 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:30:02 smithi089 bash[33173]: audit 2023-06-03T19:30:01.624815+0000 mon.b (mon.1) 11 : audit [INF] from='client.? 172.21.15.164:0/3905771273' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "5c5edd3b-d92c-4f46-94ff-2d5828b2e732"}]: dispatch 2023-06-03T19:30:02.783 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:30:02 smithi089 bash[33173]: audit 2023-06-03T19:30:01.627558+0000 mon.a (mon.0) 461 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "5c5edd3b-d92c-4f46-94ff-2d5828b2e732"}]: dispatch 2023-06-03T19:30:02.783 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:30:02 smithi089 bash[33173]: audit 2023-06-03T19:30:01.635423+0000 mon.a (mon.0) 462 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "5c5edd3b-d92c-4f46-94ff-2d5828b2e732"}]': finished 2023-06-03T19:30:02.783 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:30:02 smithi089 bash[33173]: cluster 2023-06-03T19:30:01.635596+0000 mon.a (mon.0) 463 : cluster [DBG] osdmap e39: 7 total, 6 up, 7 in 2023-06-03T19:30:02.783 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:30:02 smithi089 bash[33173]: audit 2023-06-03T19:30:01.635816+0000 mon.a (mon.0) 464 : audit [DBG] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-06-03T19:30:02.783 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:30:02 smithi089 bash[33173]: cluster 2023-06-03T19:30:01.774233+0000 mgr.y (mgr.14146) 228 : cluster [DBG] pgmap v212: 1 pgs: 1 active+clean; 0 B data, 30 MiB used, 536 GiB / 536 GiB avail 2023-06-03T19:30:02.783 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:30:02 smithi089 bash[33173]: audit 2023-06-03T19:30:02.367683+0000 mon.b (mon.1) 12 : audit [DBG] from='client.? 172.21.15.164:0/3849062029' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-06-03T19:30:02.836 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:30:02 smithi164 bash[26855]: audit 2023-06-03T19:30:01.624815+0000 mon.b (mon.1) 11 : audit [INF] from='client.? 172.21.15.164:0/3905771273' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "5c5edd3b-d92c-4f46-94ff-2d5828b2e732"}]: dispatch 2023-06-03T19:30:02.836 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:30:02 smithi164 bash[26855]: audit 2023-06-03T19:30:01.627558+0000 mon.a (mon.0) 461 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "5c5edd3b-d92c-4f46-94ff-2d5828b2e732"}]: dispatch 2023-06-03T19:30:02.836 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:30:02 smithi164 bash[26855]: audit 2023-06-03T19:30:01.635423+0000 mon.a (mon.0) 462 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "5c5edd3b-d92c-4f46-94ff-2d5828b2e732"}]': finished 2023-06-03T19:30:02.836 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:30:02 smithi164 bash[26855]: cluster 2023-06-03T19:30:01.635596+0000 mon.a (mon.0) 463 : cluster [DBG] osdmap e39: 7 total, 6 up, 7 in 2023-06-03T19:30:02.836 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:30:02 smithi164 bash[26855]: audit 2023-06-03T19:30:01.635816+0000 mon.a (mon.0) 464 : audit [DBG] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-06-03T19:30:02.837 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:30:02 smithi164 bash[26855]: cluster 2023-06-03T19:30:01.774233+0000 mgr.y (mgr.14146) 228 : cluster [DBG] pgmap v212: 1 pgs: 1 active+clean; 0 B data, 30 MiB used, 536 GiB / 536 GiB avail 2023-06-03T19:30:02.837 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:30:02 smithi164 bash[26855]: audit 2023-06-03T19:30:02.367683+0000 mon.b (mon.1) 12 : audit [DBG] from='client.? 172.21.15.164:0/3849062029' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-06-03T19:30:05.281 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:30:04 smithi089 bash[26572]: cluster 2023-06-03T19:30:03.774774+0000 mgr.y (mgr.14146) 229 : cluster [DBG] pgmap v213: 1 pgs: 1 active+clean; 0 B data, 30 MiB used, 536 GiB / 536 GiB avail 2023-06-03T19:30:05.281 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:30:04 smithi089 bash[33173]: cluster 2023-06-03T19:30:03.774774+0000 mgr.y (mgr.14146) 229 : cluster [DBG] pgmap v213: 1 pgs: 1 active+clean; 0 B data, 30 MiB used, 536 GiB / 536 GiB avail 2023-06-03T19:30:05.335 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:30:04 smithi164 bash[26855]: cluster 2023-06-03T19:30:03.774774+0000 mgr.y (mgr.14146) 229 : cluster [DBG] pgmap v213: 1 pgs: 1 active+clean; 0 B data, 30 MiB used, 536 GiB / 536 GiB avail 2023-06-03T19:30:07.281 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:30:06 smithi089 bash[26572]: cluster 2023-06-03T19:30:05.775328+0000 mgr.y (mgr.14146) 230 : cluster [DBG] pgmap v214: 1 pgs: 1 active+clean; 0 B data, 31 MiB used, 536 GiB / 536 GiB avail 2023-06-03T19:30:07.281 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:30:06 smithi089 bash[33173]: cluster 2023-06-03T19:30:05.775328+0000 mgr.y (mgr.14146) 230 : cluster [DBG] pgmap v214: 1 pgs: 1 active+clean; 0 B data, 31 MiB used, 536 GiB / 536 GiB avail 2023-06-03T19:30:07.335 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:30:06 smithi164 bash[26855]: cluster 2023-06-03T19:30:05.775328+0000 mgr.y (mgr.14146) 230 : cluster [DBG] pgmap v214: 1 pgs: 1 active+clean; 0 B data, 31 MiB used, 536 GiB / 536 GiB avail 2023-06-03T19:30:09.281 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:30:08 smithi089 bash[26572]: cluster 2023-06-03T19:30:07.775870+0000 mgr.y (mgr.14146) 231 : cluster [DBG] pgmap v215: 1 pgs: 1 active+clean; 0 B data, 31 MiB used, 536 GiB / 536 GiB avail 2023-06-03T19:30:09.281 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:30:08 smithi089 bash[33173]: cluster 2023-06-03T19:30:07.775870+0000 mgr.y (mgr.14146) 231 : cluster [DBG] pgmap v215: 1 pgs: 1 active+clean; 0 B data, 31 MiB used, 536 GiB / 536 GiB avail 2023-06-03T19:30:09.335 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:30:08 smithi164 bash[26855]: cluster 2023-06-03T19:30:07.775870+0000 mgr.y (mgr.14146) 231 : cluster [DBG] pgmap v215: 1 pgs: 1 active+clean; 0 B data, 31 MiB used, 536 GiB / 536 GiB avail 2023-06-03T19:30:11.241 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:30:10 smithi164 bash[26855]: cluster 2023-06-03T19:30:09.776408+0000 mgr.y (mgr.14146) 232 : cluster [DBG] pgmap v216: 1 pgs: 1 active+clean; 0 B data, 31 MiB used, 536 GiB / 536 GiB avail 2023-06-03T19:30:11.281 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:30:10 smithi089 bash[26572]: cluster 2023-06-03T19:30:09.776408+0000 mgr.y (mgr.14146) 232 : cluster [DBG] pgmap v216: 1 pgs: 1 active+clean; 0 B data, 31 MiB used, 536 GiB / 536 GiB avail 2023-06-03T19:30:11.281 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:30:10 smithi089 bash[33173]: cluster 2023-06-03T19:30:09.776408+0000 mgr.y (mgr.14146) 232 : cluster [DBG] pgmap v216: 1 pgs: 1 active+clean; 0 B data, 31 MiB used, 536 GiB / 536 GiB avail 2023-06-03T19:30:12.781 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:30:12 smithi089 bash[26572]: cluster 2023-06-03T19:30:11.777017+0000 mgr.y (mgr.14146) 233 : cluster [DBG] pgmap v217: 1 pgs: 1 active+clean; 0 B data, 31 MiB used, 536 GiB / 536 GiB avail 2023-06-03T19:30:12.781 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:30:12 smithi089 bash[33173]: cluster 2023-06-03T19:30:11.777017+0000 mgr.y (mgr.14146) 233 : cluster [DBG] pgmap v217: 1 pgs: 1 active+clean; 0 B data, 31 MiB used, 536 GiB / 536 GiB avail 2023-06-03T19:30:12.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:30:12 smithi164 bash[26855]: cluster 2023-06-03T19:30:11.777017+0000 mgr.y (mgr.14146) 233 : cluster [DBG] pgmap v217: 1 pgs: 1 active+clean; 0 B data, 31 MiB used, 536 GiB / 536 GiB avail 2023-06-03T19:30:15.281 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:30:14 smithi089 bash[33173]: cluster 2023-06-03T19:30:13.777554+0000 mgr.y (mgr.14146) 234 : cluster [DBG] pgmap v218: 1 pgs: 1 active+clean; 0 B data, 31 MiB used, 536 GiB / 536 GiB avail 2023-06-03T19:30:15.281 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:30:14 smithi089 bash[33173]: audit 2023-06-03T19:30:14.265545+0000 mon.a (mon.0) 465 : audit [INF] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2023-06-03T19:30:15.281 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:30:14 smithi089 bash[33173]: audit 2023-06-03T19:30:14.267528+0000 mon.a (mon.0) 466 : audit [DBG] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-03T19:30:15.282 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:30:14 smithi089 bash[26572]: cluster 2023-06-03T19:30:13.777554+0000 mgr.y (mgr.14146) 234 : cluster [DBG] pgmap v218: 1 pgs: 1 active+clean; 0 B data, 31 MiB used, 536 GiB / 536 GiB avail 2023-06-03T19:30:15.282 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:30:14 smithi089 bash[26572]: audit 2023-06-03T19:30:14.265545+0000 mon.a (mon.0) 465 : audit [INF] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2023-06-03T19:30:15.282 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:30:14 smithi089 bash[26572]: audit 2023-06-03T19:30:14.267528+0000 mon.a (mon.0) 466 : audit [DBG] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-03T19:30:15.335 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:30:14 smithi164 bash[26855]: cluster 2023-06-03T19:30:13.777554+0000 mgr.y (mgr.14146) 234 : cluster [DBG] pgmap v218: 1 pgs: 1 active+clean; 0 B data, 31 MiB used, 536 GiB / 536 GiB avail 2023-06-03T19:30:15.336 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:30:14 smithi164 bash[26855]: audit 2023-06-03T19:30:14.265545+0000 mon.a (mon.0) 465 : audit [INF] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2023-06-03T19:30:15.336 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:30:14 smithi164 bash[26855]: audit 2023-06-03T19:30:14.267528+0000 mon.a (mon.0) 466 : audit [DBG] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-03T19:30:16.085 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:30:15 smithi164 bash[26855]: cephadm 2023-06-03T19:30:14.269047+0000 mgr.y (mgr.14146) 235 : cephadm [INF] Deploying daemon osd.6 on smithi164 2023-06-03T19:30:16.281 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:30:15 smithi089 bash[26572]: cephadm 2023-06-03T19:30:14.269047+0000 mgr.y (mgr.14146) 235 : cephadm [INF] Deploying daemon osd.6 on smithi164 2023-06-03T19:30:16.281 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:30:15 smithi089 bash[33173]: cephadm 2023-06-03T19:30:14.269047+0000 mgr.y (mgr.14146) 235 : cephadm [INF] Deploying daemon osd.6 on smithi164 2023-06-03T19:30:17.141 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:30:16 smithi164 bash[26855]: cluster 2023-06-03T19:30:15.778102+0000 mgr.y (mgr.14146) 236 : cluster [DBG] pgmap v219: 1 pgs: 1 active+clean; 0 B data, 31 MiB used, 536 GiB / 536 GiB avail 2023-06-03T19:30:17.281 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:30:16 smithi089 bash[26572]: cluster 2023-06-03T19:30:15.778102+0000 mgr.y (mgr.14146) 236 : cluster [DBG] pgmap v219: 1 pgs: 1 active+clean; 0 B data, 31 MiB used, 536 GiB / 536 GiB avail 2023-06-03T19:30:17.281 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:30:16 smithi089 bash[33173]: cluster 2023-06-03T19:30:15.778102+0000 mgr.y (mgr.14146) 236 : cluster [DBG] pgmap v219: 1 pgs: 1 active+clean; 0 B data, 31 MiB used, 536 GiB / 536 GiB avail 2023-06-03T19:30:19.085 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:30:18 smithi164 bash[26855]: cluster 2023-06-03T19:30:17.778693+0000 mgr.y (mgr.14146) 237 : cluster [DBG] pgmap v220: 1 pgs: 1 active+clean; 0 B data, 31 MiB used, 536 GiB / 536 GiB avail 2023-06-03T19:30:19.086 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:30:18 smithi164 bash[26855]: audit 2023-06-03T19:30:18.743389+0000 mon.a (mon.0) 467 : audit [INF] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' 2023-06-03T19:30:19.086 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:30:18 smithi164 bash[26855]: audit 2023-06-03T19:30:18.775223+0000 mon.a (mon.0) 468 : audit [DBG] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-03T19:30:19.281 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:30:18 smithi089 bash[26572]: cluster 2023-06-03T19:30:17.778693+0000 mgr.y (mgr.14146) 237 : cluster [DBG] pgmap v220: 1 pgs: 1 active+clean; 0 B data, 31 MiB used, 536 GiB / 536 GiB avail 2023-06-03T19:30:19.281 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:30:18 smithi089 bash[26572]: audit 2023-06-03T19:30:18.743389+0000 mon.a (mon.0) 467 : audit [INF] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' 2023-06-03T19:30:19.281 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:30:18 smithi089 bash[26572]: audit 2023-06-03T19:30:18.775223+0000 mon.a (mon.0) 468 : audit [DBG] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-03T19:30:19.281 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:30:18 smithi089 bash[33173]: cluster 2023-06-03T19:30:17.778693+0000 mgr.y (mgr.14146) 237 : cluster [DBG] pgmap v220: 1 pgs: 1 active+clean; 0 B data, 31 MiB used, 536 GiB / 536 GiB avail 2023-06-03T19:30:19.282 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:30:18 smithi089 bash[33173]: audit 2023-06-03T19:30:18.743389+0000 mon.a (mon.0) 467 : audit [INF] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' 2023-06-03T19:30:19.282 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:30:18 smithi089 bash[33173]: audit 2023-06-03T19:30:18.775223+0000 mon.a (mon.0) 468 : audit [DBG] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-03T19:30:21.144 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:30:20 smithi164 bash[26855]: cluster 2023-06-03T19:30:19.779243+0000 mgr.y (mgr.14146) 238 : cluster [DBG] pgmap v221: 1 pgs: 1 active+clean; 0 B data, 31 MiB used, 536 GiB / 536 GiB avail 2023-06-03T19:30:21.281 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:30:20 smithi089 bash[26572]: cluster 2023-06-03T19:30:19.779243+0000 mgr.y (mgr.14146) 238 : cluster [DBG] pgmap v221: 1 pgs: 1 active+clean; 0 B data, 31 MiB used, 536 GiB / 536 GiB avail 2023-06-03T19:30:21.281 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:30:20 smithi089 bash[33173]: cluster 2023-06-03T19:30:19.779243+0000 mgr.y (mgr.14146) 238 : cluster [DBG] pgmap v221: 1 pgs: 1 active+clean; 0 B data, 31 MiB used, 536 GiB / 536 GiB avail 2023-06-03T19:30:22.781 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:30:22 smithi089 bash[26572]: cluster 2023-06-03T19:30:21.779796+0000 mgr.y (mgr.14146) 239 : cluster [DBG] pgmap v222: 1 pgs: 1 active+clean; 0 B data, 31 MiB used, 536 GiB / 536 GiB avail 2023-06-03T19:30:22.781 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:30:22 smithi089 bash[33173]: cluster 2023-06-03T19:30:21.779796+0000 mgr.y (mgr.14146) 239 : cluster [DBG] pgmap v222: 1 pgs: 1 active+clean; 0 B data, 31 MiB used, 536 GiB / 536 GiB avail 2023-06-03T19:30:22.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:30:22 smithi164 bash[26855]: cluster 2023-06-03T19:30:21.779796+0000 mgr.y (mgr.14146) 239 : cluster [DBG] pgmap v222: 1 pgs: 1 active+clean; 0 B data, 31 MiB used, 536 GiB / 536 GiB avail 2023-06-03T19:30:25.281 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:30:24 smithi089 bash[26572]: cluster 2023-06-03T19:30:23.780326+0000 mgr.y (mgr.14146) 240 : cluster [DBG] pgmap v223: 1 pgs: 1 active+clean; 0 B data, 31 MiB used, 536 GiB / 536 GiB avail 2023-06-03T19:30:25.281 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:30:24 smithi089 bash[33173]: cluster 2023-06-03T19:30:23.780326+0000 mgr.y (mgr.14146) 240 : cluster [DBG] pgmap v223: 1 pgs: 1 active+clean; 0 B data, 31 MiB used, 536 GiB / 536 GiB avail 2023-06-03T19:30:25.335 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:30:24 smithi164 bash[26855]: cluster 2023-06-03T19:30:23.780326+0000 mgr.y (mgr.14146) 240 : cluster [DBG] pgmap v223: 1 pgs: 1 active+clean; 0 B data, 31 MiB used, 536 GiB / 536 GiB avail 2023-06-03T19:30:26.281 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:30:26 smithi089 bash[33173]: audit 2023-06-03T19:30:25.023112+0000 mon.a (mon.0) 469 : audit [INF] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' 2023-06-03T19:30:26.281 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:30:26 smithi089 bash[33173]: audit 2023-06-03T19:30:25.312044+0000 mon.a (mon.0) 470 : audit [DBG] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-03T19:30:26.281 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:30:26 smithi089 bash[33173]: audit 2023-06-03T19:30:25.313920+0000 mon.a (mon.0) 471 : audit [INF] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-03T19:30:26.281 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:30:26 smithi089 bash[33173]: audit 2023-06-03T19:30:25.322870+0000 mon.a (mon.0) 472 : audit [INF] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' 2023-06-03T19:30:26.282 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:30:26 smithi089 bash[33173]: audit 2023-06-03T19:30:25.640826+0000 mon.b (mon.1) 13 : audit [INF] from='osd.6 [v2:172.21.15.164:6816/712403290,v1:172.21.15.164:6817/712403290]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2023-06-03T19:30:26.282 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:30:26 smithi089 bash[33173]: audit 2023-06-03T19:30:25.643310+0000 mon.a (mon.0) 473 : audit [INF] from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2023-06-03T19:30:26.282 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:30:26 smithi089 bash[26572]: audit 2023-06-03T19:30:25.023112+0000 mon.a (mon.0) 469 : audit [INF] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' 2023-06-03T19:30:26.282 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:30:26 smithi089 bash[26572]: audit 2023-06-03T19:30:25.312044+0000 mon.a (mon.0) 470 : audit [DBG] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-03T19:30:26.282 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:30:26 smithi089 bash[26572]: audit 2023-06-03T19:30:25.313920+0000 mon.a (mon.0) 471 : audit [INF] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-03T19:30:26.282 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:30:26 smithi089 bash[26572]: audit 2023-06-03T19:30:25.322870+0000 mon.a (mon.0) 472 : audit [INF] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' 2023-06-03T19:30:26.283 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:30:26 smithi089 bash[26572]: audit 2023-06-03T19:30:25.640826+0000 mon.b (mon.1) 13 : audit [INF] from='osd.6 [v2:172.21.15.164:6816/712403290,v1:172.21.15.164:6817/712403290]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2023-06-03T19:30:26.283 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:30:26 smithi089 bash[26572]: audit 2023-06-03T19:30:25.643310+0000 mon.a (mon.0) 473 : audit [INF] from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2023-06-03T19:30:26.335 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:30:26 smithi164 bash[26855]: audit 2023-06-03T19:30:25.023112+0000 mon.a (mon.0) 469 : audit [INF] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' 2023-06-03T19:30:26.336 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:30:26 smithi164 bash[26855]: audit 2023-06-03T19:30:25.312044+0000 mon.a (mon.0) 470 : audit [DBG] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-03T19:30:26.336 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:30:26 smithi164 bash[26855]: audit 2023-06-03T19:30:25.313920+0000 mon.a (mon.0) 471 : audit [INF] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-03T19:30:26.336 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:30:26 smithi164 bash[26855]: audit 2023-06-03T19:30:25.322870+0000 mon.a (mon.0) 472 : audit [INF] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' 2023-06-03T19:30:26.336 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:30:26 smithi164 bash[26855]: audit 2023-06-03T19:30:25.640826+0000 mon.b (mon.1) 13 : audit [INF] from='osd.6 [v2:172.21.15.164:6816/712403290,v1:172.21.15.164:6817/712403290]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2023-06-03T19:30:26.336 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:30:26 smithi164 bash[26855]: audit 2023-06-03T19:30:25.643310+0000 mon.a (mon.0) 473 : audit [INF] from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2023-06-03T19:30:27.332 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:30:27 smithi164 bash[26855]: cluster 2023-06-03T19:30:25.780934+0000 mgr.y (mgr.14146) 241 : cluster [DBG] pgmap v224: 1 pgs: 1 active+clean; 0 B data, 31 MiB used, 536 GiB / 536 GiB avail 2023-06-03T19:30:27.332 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:30:27 smithi164 bash[26855]: audit 2023-06-03T19:30:26.329810+0000 mon.a (mon.0) 474 : audit [INF] from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2023-06-03T19:30:27.333 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:30:27 smithi164 bash[26855]: cluster 2023-06-03T19:30:26.329909+0000 mon.a (mon.0) 475 : cluster [DBG] osdmap e40: 7 total, 6 up, 7 in 2023-06-03T19:30:27.333 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:30:27 smithi164 bash[26855]: audit 2023-06-03T19:30:26.330126+0000 mon.a (mon.0) 476 : audit [DBG] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-06-03T19:30:27.333 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:30:27 smithi164 bash[26855]: audit 2023-06-03T19:30:26.330312+0000 mon.b (mon.1) 14 : audit [INF] from='osd.6 [v2:172.21.15.164:6816/712403290,v1:172.21.15.164:6817/712403290]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi164", "root=default"]}]: dispatch 2023-06-03T19:30:27.333 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:30:27 smithi164 bash[26855]: audit 2023-06-03T19:30:26.332593+0000 mon.a (mon.0) 477 : audit [INF] from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi164", "root=default"]}]: dispatch 2023-06-03T19:30:27.531 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:30:27 smithi089 bash[33173]: cluster 2023-06-03T19:30:25.780934+0000 mgr.y (mgr.14146) 241 : cluster [DBG] pgmap v224: 1 pgs: 1 active+clean; 0 B data, 31 MiB used, 536 GiB / 536 GiB avail 2023-06-03T19:30:27.531 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:30:27 smithi089 bash[33173]: audit 2023-06-03T19:30:26.329810+0000 mon.a (mon.0) 474 : audit [INF] from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2023-06-03T19:30:27.531 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:30:27 smithi089 bash[33173]: cluster 2023-06-03T19:30:26.329909+0000 mon.a (mon.0) 475 : cluster [DBG] osdmap e40: 7 total, 6 up, 7 in 2023-06-03T19:30:27.532 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:30:27 smithi089 bash[33173]: audit 2023-06-03T19:30:26.330126+0000 mon.a (mon.0) 476 : audit [DBG] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-06-03T19:30:27.532 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:30:27 smithi089 bash[33173]: audit 2023-06-03T19:30:26.330312+0000 mon.b (mon.1) 14 : audit [INF] from='osd.6 [v2:172.21.15.164:6816/712403290,v1:172.21.15.164:6817/712403290]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi164", "root=default"]}]: dispatch 2023-06-03T19:30:27.532 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:30:27 smithi089 bash[33173]: audit 2023-06-03T19:30:26.332593+0000 mon.a (mon.0) 477 : audit [INF] from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi164", "root=default"]}]: dispatch 2023-06-03T19:30:27.532 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:30:27 smithi089 bash[26572]: cluster 2023-06-03T19:30:25.780934+0000 mgr.y (mgr.14146) 241 : cluster [DBG] pgmap v224: 1 pgs: 1 active+clean; 0 B data, 31 MiB used, 536 GiB / 536 GiB avail 2023-06-03T19:30:27.532 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:30:27 smithi089 bash[26572]: audit 2023-06-03T19:30:26.329810+0000 mon.a (mon.0) 474 : audit [INF] from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2023-06-03T19:30:27.533 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:30:27 smithi089 bash[26572]: cluster 2023-06-03T19:30:26.329909+0000 mon.a (mon.0) 475 : cluster [DBG] osdmap e40: 7 total, 6 up, 7 in 2023-06-03T19:30:27.533 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:30:27 smithi089 bash[26572]: audit 2023-06-03T19:30:26.330126+0000 mon.a (mon.0) 476 : audit [DBG] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-06-03T19:30:27.533 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:30:27 smithi089 bash[26572]: audit 2023-06-03T19:30:26.330312+0000 mon.b (mon.1) 14 : audit [INF] from='osd.6 [v2:172.21.15.164:6816/712403290,v1:172.21.15.164:6817/712403290]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi164", "root=default"]}]: dispatch 2023-06-03T19:30:27.534 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:30:27 smithi089 bash[26572]: audit 2023-06-03T19:30:26.332593+0000 mon.a (mon.0) 477 : audit [INF] from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi164", "root=default"]}]: dispatch 2023-06-03T19:30:28.585 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:30:28 smithi164 bash[26855]: audit 2023-06-03T19:30:27.334853+0000 mon.a (mon.0) 478 : audit [INF] from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi164", "root=default"]}]': finished 2023-06-03T19:30:28.586 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:30:28 smithi164 bash[26855]: cluster 2023-06-03T19:30:27.335149+0000 mon.a (mon.0) 479 : cluster [DBG] osdmap e41: 7 total, 6 up, 7 in 2023-06-03T19:30:28.586 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:30:28 smithi164 bash[26855]: audit 2023-06-03T19:30:27.335491+0000 mon.a (mon.0) 480 : audit [DBG] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-06-03T19:30:28.586 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:30:28 smithi164 bash[26855]: audit 2023-06-03T19:30:27.340490+0000 mon.a (mon.0) 481 : audit [DBG] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-06-03T19:30:28.586 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:30:28 smithi164 bash[26855]: cluster 2023-06-03T19:30:27.781418+0000 mgr.y (mgr.14146) 242 : cluster [DBG] pgmap v227: 1 pgs: 1 active+clean; 0 B data, 31 MiB used, 536 GiB / 536 GiB avail 2023-06-03T19:30:28.748 INFO:teuthology.orchestra.run.smithi164.stdout:Created osd(s) 6 on host 'smithi164' 2023-06-03T19:30:28.781 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:30:28 smithi089 bash[33173]: audit 2023-06-03T19:30:27.334853+0000 mon.a (mon.0) 478 : audit [INF] from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi164", "root=default"]}]': finished 2023-06-03T19:30:28.781 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:30:28 smithi089 bash[33173]: cluster 2023-06-03T19:30:27.335149+0000 mon.a (mon.0) 479 : cluster [DBG] osdmap e41: 7 total, 6 up, 7 in 2023-06-03T19:30:28.781 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:30:28 smithi089 bash[33173]: audit 2023-06-03T19:30:27.335491+0000 mon.a (mon.0) 480 : audit [DBG] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-06-03T19:30:28.781 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:30:28 smithi089 bash[33173]: audit 2023-06-03T19:30:27.340490+0000 mon.a (mon.0) 481 : audit [DBG] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-06-03T19:30:28.781 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:30:28 smithi089 bash[33173]: cluster 2023-06-03T19:30:27.781418+0000 mgr.y (mgr.14146) 242 : cluster [DBG] pgmap v227: 1 pgs: 1 active+clean; 0 B data, 31 MiB used, 536 GiB / 536 GiB avail 2023-06-03T19:30:28.782 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:30:28 smithi089 bash[26572]: audit 2023-06-03T19:30:27.334853+0000 mon.a (mon.0) 478 : audit [INF] from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi164", "root=default"]}]': finished 2023-06-03T19:30:28.782 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:30:28 smithi089 bash[26572]: cluster 2023-06-03T19:30:27.335149+0000 mon.a (mon.0) 479 : cluster [DBG] osdmap e41: 7 total, 6 up, 7 in 2023-06-03T19:30:28.782 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:30:28 smithi089 bash[26572]: audit 2023-06-03T19:30:27.335491+0000 mon.a (mon.0) 480 : audit [DBG] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-06-03T19:30:28.782 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:30:28 smithi089 bash[26572]: audit 2023-06-03T19:30:27.340490+0000 mon.a (mon.0) 481 : audit [DBG] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-06-03T19:30:28.782 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:30:28 smithi089 bash[26572]: cluster 2023-06-03T19:30:27.781418+0000 mgr.y (mgr.14146) 242 : cluster [DBG] pgmap v227: 1 pgs: 1 active+clean; 0 B data, 31 MiB used, 536 GiB / 536 GiB avail 2023-06-03T19:30:29.421 DEBUG:teuthology.orchestra.run.smithi164:osd.6> sudo journalctl -f -n 0 -u ceph-f1db3478-0243-11ee-9b1d-001a4aab830c@osd.6.service 2023-06-03T19:30:29.424 INFO:tasks.cephadm:Deploying osd.7 on smithi164 with /dev/vg_nvme/lv_1... 2023-06-03T19:30:29.424 DEBUG:teuthology.orchestra.run.smithi164:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph-volume lvm zap /dev/vg_nvme/lv_1 2023-06-03T19:30:29.639 INFO:journalctl@ceph.osd.6.smithi164.stdout:-- Logs begin at Sun 2022-08-14 12:20:24 UTC. -- 2023-06-03T19:30:29.669 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:30:29 smithi164 bash[26855]: cluster 2023-06-03T19:30:26.631875+0000 osd.6 (osd.6) 1 : cluster [DBG] purged_snaps scrub starts 2023-06-03T19:30:29.670 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:30:29 smithi164 bash[26855]: cluster 2023-06-03T19:30:26.631978+0000 osd.6 (osd.6) 2 : cluster [DBG] purged_snaps scrub ok 2023-06-03T19:30:29.670 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:30:29 smithi164 bash[26855]: cluster 2023-06-03T19:30:28.337855+0000 mon.a (mon.0) 482 : cluster [INF] osd.6 [v2:172.21.15.164:6816/712403290,v1:172.21.15.164:6817/712403290] boot 2023-06-03T19:30:29.670 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:30:29 smithi164 bash[26855]: cluster 2023-06-03T19:30:28.337933+0000 mon.a (mon.0) 483 : cluster [DBG] osdmap e42: 7 total, 7 up, 7 in 2023-06-03T19:30:29.670 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:30:29 smithi164 bash[26855]: audit 2023-06-03T19:30:28.338565+0000 mon.a (mon.0) 484 : audit [DBG] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-06-03T19:30:29.670 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:30:29 smithi164 bash[26855]: audit 2023-06-03T19:30:28.740638+0000 mon.a (mon.0) 485 : audit [INF] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' 2023-06-03T19:30:29.671 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:30:29 smithi164 bash[26855]: audit 2023-06-03T19:30:28.748934+0000 mon.a (mon.0) 486 : audit [DBG] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-03T19:30:29.781 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:30:29 smithi089 bash[26572]: cluster 2023-06-03T19:30:26.631875+0000 osd.6 (osd.6) 1 : cluster [DBG] purged_snaps scrub starts 2023-06-03T19:30:29.782 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:30:29 smithi089 bash[26572]: cluster 2023-06-03T19:30:26.631978+0000 osd.6 (osd.6) 2 : cluster [DBG] purged_snaps scrub ok 2023-06-03T19:30:29.782 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:30:29 smithi089 bash[26572]: cluster 2023-06-03T19:30:28.337855+0000 mon.a (mon.0) 482 : cluster [INF] osd.6 [v2:172.21.15.164:6816/712403290,v1:172.21.15.164:6817/712403290] boot 2023-06-03T19:30:29.783 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:30:29 smithi089 bash[26572]: cluster 2023-06-03T19:30:28.337933+0000 mon.a (mon.0) 483 : cluster [DBG] osdmap e42: 7 total, 7 up, 7 in 2023-06-03T19:30:29.783 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:30:29 smithi089 bash[26572]: audit 2023-06-03T19:30:28.338565+0000 mon.a (mon.0) 484 : audit [DBG] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-06-03T19:30:29.783 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:30:29 smithi089 bash[26572]: audit 2023-06-03T19:30:28.740638+0000 mon.a (mon.0) 485 : audit [INF] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' 2023-06-03T19:30:29.783 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:30:29 smithi089 bash[26572]: audit 2023-06-03T19:30:28.748934+0000 mon.a (mon.0) 486 : audit [DBG] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-03T19:30:29.783 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:30:29 smithi089 bash[33173]: cluster 2023-06-03T19:30:26.631875+0000 osd.6 (osd.6) 1 : cluster [DBG] purged_snaps scrub starts 2023-06-03T19:30:29.784 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:30:29 smithi089 bash[33173]: cluster 2023-06-03T19:30:26.631978+0000 osd.6 (osd.6) 2 : cluster [DBG] purged_snaps scrub ok 2023-06-03T19:30:29.784 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:30:29 smithi089 bash[33173]: cluster 2023-06-03T19:30:28.337855+0000 mon.a (mon.0) 482 : cluster [INF] osd.6 [v2:172.21.15.164:6816/712403290,v1:172.21.15.164:6817/712403290] boot 2023-06-03T19:30:29.784 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:30:29 smithi089 bash[33173]: cluster 2023-06-03T19:30:28.337933+0000 mon.a (mon.0) 483 : cluster [DBG] osdmap e42: 7 total, 7 up, 7 in 2023-06-03T19:30:29.784 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:30:29 smithi089 bash[33173]: audit 2023-06-03T19:30:28.338565+0000 mon.a (mon.0) 484 : audit [DBG] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-06-03T19:30:29.784 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:30:29 smithi089 bash[33173]: audit 2023-06-03T19:30:28.740638+0000 mon.a (mon.0) 485 : audit [INF] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' 2023-06-03T19:30:29.784 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:30:29 smithi089 bash[33173]: audit 2023-06-03T19:30:28.748934+0000 mon.a (mon.0) 486 : audit [DBG] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-03T19:30:30.637 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:30:30 smithi164 bash[26855]: cluster 2023-06-03T19:30:29.348859+0000 mon.a (mon.0) 487 : cluster [DBG] osdmap e43: 7 total, 7 up, 7 in 2023-06-03T19:30:30.638 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:30:30 smithi164 bash[26855]: cluster 2023-06-03T19:30:29.781970+0000 mgr.y (mgr.14146) 243 : cluster [DBG] pgmap v230: 1 pgs: 1 active+clean; 0 B data, 36 MiB used, 626 GiB / 626 GiB avail 2023-06-03T19:30:30.781 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:30:30 smithi089 bash[26572]: cluster 2023-06-03T19:30:29.348859+0000 mon.a (mon.0) 487 : cluster [DBG] osdmap e43: 7 total, 7 up, 7 in 2023-06-03T19:30:30.781 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:30:30 smithi089 bash[26572]: cluster 2023-06-03T19:30:29.781970+0000 mgr.y (mgr.14146) 243 : cluster [DBG] pgmap v230: 1 pgs: 1 active+clean; 0 B data, 36 MiB used, 626 GiB / 626 GiB avail 2023-06-03T19:30:30.781 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:30:30 smithi089 bash[33173]: cluster 2023-06-03T19:30:29.348859+0000 mon.a (mon.0) 487 : cluster [DBG] osdmap e43: 7 total, 7 up, 7 in 2023-06-03T19:30:30.782 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:30:30 smithi089 bash[33173]: cluster 2023-06-03T19:30:29.781970+0000 mgr.y (mgr.14146) 243 : cluster [DBG] pgmap v230: 1 pgs: 1 active+clean; 0 B data, 36 MiB used, 626 GiB / 626 GiB avail 2023-06-03T19:30:31.781 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:30:31 smithi089 bash[26572]: cluster 2023-06-03T19:30:30.367929+0000 mon.a (mon.0) 488 : cluster [DBG] osdmap e44: 7 total, 7 up, 7 in 2023-06-03T19:30:31.782 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:30:31 smithi089 bash[33173]: cluster 2023-06-03T19:30:30.367929+0000 mon.a (mon.0) 488 : cluster [DBG] osdmap e44: 7 total, 7 up, 7 in 2023-06-03T19:30:31.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:30:31 smithi164 bash[26855]: cluster 2023-06-03T19:30:30.367929+0000 mon.a (mon.0) 488 : cluster [DBG] osdmap e44: 7 total, 7 up, 7 in 2023-06-03T19:30:32.585 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:30:32 smithi164 bash[26855]: cluster 2023-06-03T19:30:31.782498+0000 mgr.y (mgr.14146) 244 : cluster [DBG] pgmap v232: 1 pgs: 1 active+clean; 0 B data, 36 MiB used, 626 GiB / 626 GiB avail 2023-06-03T19:30:32.781 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:30:32 smithi089 bash[26572]: cluster 2023-06-03T19:30:31.782498+0000 mgr.y (mgr.14146) 244 : cluster [DBG] pgmap v232: 1 pgs: 1 active+clean; 0 B data, 36 MiB used, 626 GiB / 626 GiB avail 2023-06-03T19:30:32.781 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:30:32 smithi089 bash[33173]: cluster 2023-06-03T19:30:31.782498+0000 mgr.y (mgr.14146) 244 : cluster [DBG] pgmap v232: 1 pgs: 1 active+clean; 0 B data, 36 MiB used, 626 GiB / 626 GiB avail 2023-06-03T19:30:32.981 INFO:teuthology.orchestra.run.smithi164.stderr:--> Zapping: /dev/vg_nvme/lv_1 2023-06-03T19:30:32.982 INFO:teuthology.orchestra.run.smithi164.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_1 bs=1M count=10 conv=fsync 2023-06-03T19:30:32.982 INFO:teuthology.orchestra.run.smithi164.stderr: stderr: 10+0 records in 2023-06-03T19:30:32.982 INFO:teuthology.orchestra.run.smithi164.stderr:10+0 records out 2023-06-03T19:30:32.982 INFO:teuthology.orchestra.run.smithi164.stderr: stderr: 10485760 bytes (10 MB, 10 MiB) copied, 0.0176194 s, 595 MB/s 2023-06-03T19:30:32.982 INFO:teuthology.orchestra.run.smithi164.stderr:--> Zapping successful for: 2023-06-03T19:30:33.816 DEBUG:teuthology.orchestra.run.smithi164:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph orch daemon add osd smithi164:vg_nvme/lv_1 2023-06-03T19:30:35.281 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:30:34 smithi089 bash[26572]: cluster 2023-06-03T19:30:33.783050+0000 mgr.y (mgr.14146) 245 : cluster [DBG] pgmap v233: 1 pgs: 1 active+clean; 0 B data, 36 MiB used, 626 GiB / 626 GiB avail 2023-06-03T19:30:35.281 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:30:34 smithi089 bash[33173]: cluster 2023-06-03T19:30:33.783050+0000 mgr.y (mgr.14146) 245 : cluster [DBG] pgmap v233: 1 pgs: 1 active+clean; 0 B data, 36 MiB used, 626 GiB / 626 GiB avail 2023-06-03T19:30:35.335 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:30:34 smithi164 bash[26855]: cluster 2023-06-03T19:30:33.783050+0000 mgr.y (mgr.14146) 245 : cluster [DBG] pgmap v233: 1 pgs: 1 active+clean; 0 B data, 36 MiB used, 626 GiB / 626 GiB avail 2023-06-03T19:30:37.085 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:30:36 smithi164 bash[26855]: cluster 2023-06-03T19:30:35.783637+0000 mgr.y (mgr.14146) 246 : cluster [DBG] pgmap v234: 1 pgs: 1 active+clean; 0 B data, 36 MiB used, 626 GiB / 626 GiB avail 2023-06-03T19:30:37.281 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:30:36 smithi089 bash[26572]: cluster 2023-06-03T19:30:35.783637+0000 mgr.y (mgr.14146) 246 : cluster [DBG] pgmap v234: 1 pgs: 1 active+clean; 0 B data, 36 MiB used, 626 GiB / 626 GiB avail 2023-06-03T19:30:37.281 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:30:36 smithi089 bash[33173]: cluster 2023-06-03T19:30:35.783637+0000 mgr.y (mgr.14146) 246 : cluster [DBG] pgmap v234: 1 pgs: 1 active+clean; 0 B data, 36 MiB used, 626 GiB / 626 GiB avail 2023-06-03T19:30:39.140 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:30:38 smithi164 bash[26855]: cluster 2023-06-03T19:30:37.784188+0000 mgr.y (mgr.14146) 247 : cluster [DBG] pgmap v235: 1 pgs: 1 active+clean; 0 B data, 36 MiB used, 626 GiB / 626 GiB avail 2023-06-03T19:30:39.141 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:30:38 smithi164 bash[26855]: audit 2023-06-03T19:30:38.586291+0000 mon.a (mon.0) 489 : audit [DBG] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-06-03T19:30:39.141 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:30:38 smithi164 bash[26855]: audit 2023-06-03T19:30:38.592358+0000 mon.a (mon.0) 490 : audit [INF] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-06-03T19:30:39.141 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:30:38 smithi164 bash[26855]: audit 2023-06-03T19:30:38.593873+0000 mon.a (mon.0) 491 : audit [DBG] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-03T19:30:39.281 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:30:38 smithi089 bash[26572]: cluster 2023-06-03T19:30:37.784188+0000 mgr.y (mgr.14146) 247 : cluster [DBG] pgmap v235: 1 pgs: 1 active+clean; 0 B data, 36 MiB used, 626 GiB / 626 GiB avail 2023-06-03T19:30:39.282 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:30:38 smithi089 bash[26572]: audit 2023-06-03T19:30:38.586291+0000 mon.a (mon.0) 489 : audit [DBG] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-06-03T19:30:39.282 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:30:38 smithi089 bash[26572]: audit 2023-06-03T19:30:38.592358+0000 mon.a (mon.0) 490 : audit [INF] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-06-03T19:30:39.282 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:30:38 smithi089 bash[26572]: audit 2023-06-03T19:30:38.593873+0000 mon.a (mon.0) 491 : audit [DBG] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-03T19:30:39.282 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:30:38 smithi089 bash[33173]: cluster 2023-06-03T19:30:37.784188+0000 mgr.y (mgr.14146) 247 : cluster [DBG] pgmap v235: 1 pgs: 1 active+clean; 0 B data, 36 MiB used, 626 GiB / 626 GiB avail 2023-06-03T19:30:39.282 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:30:38 smithi089 bash[33173]: audit 2023-06-03T19:30:38.586291+0000 mon.a (mon.0) 489 : audit [DBG] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-06-03T19:30:39.283 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:30:38 smithi089 bash[33173]: audit 2023-06-03T19:30:38.592358+0000 mon.a (mon.0) 490 : audit [INF] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-06-03T19:30:39.283 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:30:38 smithi089 bash[33173]: audit 2023-06-03T19:30:38.593873+0000 mon.a (mon.0) 491 : audit [DBG] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-03T19:30:40.281 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:30:39 smithi089 bash[26572]: audit 2023-06-03T19:30:38.579338+0000 mgr.y (mgr.14146) 248 : audit [DBG] from='client.24304 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi164:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2023-06-03T19:30:40.281 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:30:39 smithi089 bash[33173]: audit 2023-06-03T19:30:38.579338+0000 mgr.y (mgr.14146) 248 : audit [DBG] from='client.24304 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi164:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2023-06-03T19:30:40.335 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:30:39 smithi164 bash[26855]: audit 2023-06-03T19:30:38.579338+0000 mgr.y (mgr.14146) 248 : audit [DBG] from='client.24304 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi164:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2023-06-03T19:30:41.281 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:30:40 smithi089 bash[26572]: cluster 2023-06-03T19:30:39.784785+0000 mgr.y (mgr.14146) 249 : cluster [DBG] pgmap v236: 1 pgs: 1 active+clean; 0 B data, 36 MiB used, 626 GiB / 626 GiB avail 2023-06-03T19:30:41.281 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:30:40 smithi089 bash[26572]: cephadm 2023-06-03T19:30:39.871860+0000 mgr.y (mgr.14146) 250 : cephadm [INF] Detected new or changed devices on smithi164 2023-06-03T19:30:41.281 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:30:40 smithi089 bash[26572]: audit 2023-06-03T19:30:39.879028+0000 mon.a (mon.0) 492 : audit [INF] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' 2023-06-03T19:30:41.282 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:30:40 smithi089 bash[26572]: audit 2023-06-03T19:30:39.880654+0000 mon.a (mon.0) 493 : audit [INF] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-06-03T19:30:41.282 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:30:40 smithi089 bash[26572]: audit 2023-06-03T19:30:39.882097+0000 mon.a (mon.0) 494 : audit [INF] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-06-03T19:30:41.282 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:30:40 smithi089 bash[26572]: audit 2023-06-03T19:30:39.883158+0000 mon.a (mon.0) 495 : audit [INF] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2023-06-03T19:30:41.282 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:30:40 smithi089 bash[26572]: cephadm 2023-06-03T19:30:39.883929+0000 mgr.y (mgr.14146) 251 : cephadm [INF] Adjusting osd_memory_target on smithi164 to 5758M 2023-06-03T19:30:41.282 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:30:40 smithi089 bash[26572]: audit 2023-06-03T19:30:39.888580+0000 mon.a (mon.0) 496 : audit [INF] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' 2023-06-03T19:30:41.282 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:30:40 smithi089 bash[26572]: audit 2023-06-03T19:30:39.889889+0000 mon.a (mon.0) 497 : audit [DBG] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-03T19:30:41.282 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:30:40 smithi089 bash[26572]: audit 2023-06-03T19:30:39.890865+0000 mon.a (mon.0) 498 : audit [INF] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-03T19:30:41.283 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:30:40 smithi089 bash[26572]: audit 2023-06-03T19:30:39.896870+0000 mon.a (mon.0) 499 : audit [INF] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' 2023-06-03T19:30:41.283 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:30:40 smithi089 bash[33173]: cluster 2023-06-03T19:30:39.784785+0000 mgr.y (mgr.14146) 249 : cluster [DBG] pgmap v236: 1 pgs: 1 active+clean; 0 B data, 36 MiB used, 626 GiB / 626 GiB avail 2023-06-03T19:30:41.283 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:30:40 smithi089 bash[33173]: cephadm 2023-06-03T19:30:39.871860+0000 mgr.y (mgr.14146) 250 : cephadm [INF] Detected new or changed devices on smithi164 2023-06-03T19:30:41.283 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:30:40 smithi089 bash[33173]: audit 2023-06-03T19:30:39.879028+0000 mon.a (mon.0) 492 : audit [INF] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' 2023-06-03T19:30:41.283 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:30:40 smithi089 bash[33173]: audit 2023-06-03T19:30:39.880654+0000 mon.a (mon.0) 493 : audit [INF] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-06-03T19:30:41.283 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:30:40 smithi089 bash[33173]: audit 2023-06-03T19:30:39.882097+0000 mon.a (mon.0) 494 : audit [INF] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-06-03T19:30:41.284 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:30:40 smithi089 bash[33173]: audit 2023-06-03T19:30:39.883158+0000 mon.a (mon.0) 495 : audit [INF] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2023-06-03T19:30:41.284 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:30:40 smithi089 bash[33173]: cephadm 2023-06-03T19:30:39.883929+0000 mgr.y (mgr.14146) 251 : cephadm [INF] Adjusting osd_memory_target on smithi164 to 5758M 2023-06-03T19:30:41.284 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:30:40 smithi089 bash[33173]: audit 2023-06-03T19:30:39.888580+0000 mon.a (mon.0) 496 : audit [INF] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' 2023-06-03T19:30:41.284 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:30:40 smithi089 bash[33173]: audit 2023-06-03T19:30:39.889889+0000 mon.a (mon.0) 497 : audit [DBG] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-03T19:30:41.284 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:30:40 smithi089 bash[33173]: audit 2023-06-03T19:30:39.890865+0000 mon.a (mon.0) 498 : audit [INF] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-03T19:30:41.284 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:30:40 smithi089 bash[33173]: audit 2023-06-03T19:30:39.896870+0000 mon.a (mon.0) 499 : audit [INF] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' 2023-06-03T19:30:41.336 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:30:40 smithi164 bash[26855]: cluster 2023-06-03T19:30:39.784785+0000 mgr.y (mgr.14146) 249 : cluster [DBG] pgmap v236: 1 pgs: 1 active+clean; 0 B data, 36 MiB used, 626 GiB / 626 GiB avail 2023-06-03T19:30:41.336 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:30:40 smithi164 bash[26855]: cephadm 2023-06-03T19:30:39.871860+0000 mgr.y (mgr.14146) 250 : cephadm [INF] Detected new or changed devices on smithi164 2023-06-03T19:30:41.336 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:30:40 smithi164 bash[26855]: audit 2023-06-03T19:30:39.879028+0000 mon.a (mon.0) 492 : audit [INF] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' 2023-06-03T19:30:41.336 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:30:40 smithi164 bash[26855]: audit 2023-06-03T19:30:39.880654+0000 mon.a (mon.0) 493 : audit [INF] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-06-03T19:30:41.337 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:30:40 smithi164 bash[26855]: audit 2023-06-03T19:30:39.882097+0000 mon.a (mon.0) 494 : audit [INF] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-06-03T19:30:41.337 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:30:40 smithi164 bash[26855]: audit 2023-06-03T19:30:39.883158+0000 mon.a (mon.0) 495 : audit [INF] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2023-06-03T19:30:41.337 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:30:40 smithi164 bash[26855]: cephadm 2023-06-03T19:30:39.883929+0000 mgr.y (mgr.14146) 251 : cephadm [INF] Adjusting osd_memory_target on smithi164 to 5758M 2023-06-03T19:30:41.337 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:30:40 smithi164 bash[26855]: audit 2023-06-03T19:30:39.888580+0000 mon.a (mon.0) 496 : audit [INF] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' 2023-06-03T19:30:41.337 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:30:40 smithi164 bash[26855]: audit 2023-06-03T19:30:39.889889+0000 mon.a (mon.0) 497 : audit [DBG] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-03T19:30:41.337 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:30:40 smithi164 bash[26855]: audit 2023-06-03T19:30:39.890865+0000 mon.a (mon.0) 498 : audit [INF] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-03T19:30:41.337 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:30:40 smithi164 bash[26855]: audit 2023-06-03T19:30:39.896870+0000 mon.a (mon.0) 499 : audit [INF] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' 2023-06-03T19:30:42.781 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:30:42 smithi089 bash[26572]: cluster 2023-06-03T19:30:41.785305+0000 mgr.y (mgr.14146) 252 : cluster [DBG] pgmap v237: 1 pgs: 1 active+clean; 0 B data, 36 MiB used, 626 GiB / 626 GiB avail 2023-06-03T19:30:42.781 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:30:42 smithi089 bash[33173]: cluster 2023-06-03T19:30:41.785305+0000 mgr.y (mgr.14146) 252 : cluster [DBG] pgmap v237: 1 pgs: 1 active+clean; 0 B data, 36 MiB used, 626 GiB / 626 GiB avail 2023-06-03T19:30:42.836 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:30:42 smithi164 bash[26855]: cluster 2023-06-03T19:30:41.785305+0000 mgr.y (mgr.14146) 252 : cluster [DBG] pgmap v237: 1 pgs: 1 active+clean; 0 B data, 36 MiB used, 626 GiB / 626 GiB avail 2023-06-03T19:30:45.085 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:30:44 smithi164 bash[26855]: cluster 2023-06-03T19:30:43.785881+0000 mgr.y (mgr.14146) 253 : cluster [DBG] pgmap v238: 1 pgs: 1 active+clean; 0 B data, 36 MiB used, 626 GiB / 626 GiB avail 2023-06-03T19:30:45.281 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:30:44 smithi089 bash[26572]: cluster 2023-06-03T19:30:43.785881+0000 mgr.y (mgr.14146) 253 : cluster [DBG] pgmap v238: 1 pgs: 1 active+clean; 0 B data, 36 MiB used, 626 GiB / 626 GiB avail 2023-06-03T19:30:45.281 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:30:44 smithi089 bash[33173]: cluster 2023-06-03T19:30:43.785881+0000 mgr.y (mgr.14146) 253 : cluster [DBG] pgmap v238: 1 pgs: 1 active+clean; 0 B data, 36 MiB used, 626 GiB / 626 GiB avail 2023-06-03T19:30:47.281 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:30:46 smithi089 bash[26572]: cluster 2023-06-03T19:30:45.786465+0000 mgr.y (mgr.14146) 254 : cluster [DBG] pgmap v239: 1 pgs: 1 active+clean; 0 B data, 36 MiB used, 626 GiB / 626 GiB avail 2023-06-03T19:30:47.281 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:30:46 smithi089 bash[33173]: cluster 2023-06-03T19:30:45.786465+0000 mgr.y (mgr.14146) 254 : cluster [DBG] pgmap v239: 1 pgs: 1 active+clean; 0 B data, 36 MiB used, 626 GiB / 626 GiB avail 2023-06-03T19:30:47.335 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:30:46 smithi164 bash[26855]: cluster 2023-06-03T19:30:45.786465+0000 mgr.y (mgr.14146) 254 : cluster [DBG] pgmap v239: 1 pgs: 1 active+clean; 0 B data, 36 MiB used, 626 GiB / 626 GiB avail 2023-06-03T19:30:48.281 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:30:47 smithi089 bash[26572]: audit 2023-06-03T19:30:46.905669+0000 mon.b (mon.1) 15 : audit [INF] from='client.? 172.21.15.164:0/800234170' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "bdff9290-e6c4-407e-9113-eb39696c0e27"}]: dispatch 2023-06-03T19:30:48.281 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:30:47 smithi089 bash[26572]: audit 2023-06-03T19:30:46.908341+0000 mon.a (mon.0) 500 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "bdff9290-e6c4-407e-9113-eb39696c0e27"}]: dispatch 2023-06-03T19:30:48.281 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:30:47 smithi089 bash[26572]: audit 2023-06-03T19:30:46.916811+0000 mon.a (mon.0) 501 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "bdff9290-e6c4-407e-9113-eb39696c0e27"}]': finished 2023-06-03T19:30:48.282 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:30:47 smithi089 bash[26572]: cluster 2023-06-03T19:30:46.916977+0000 mon.a (mon.0) 502 : cluster [DBG] osdmap e45: 8 total, 7 up, 8 in 2023-06-03T19:30:48.282 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:30:47 smithi089 bash[26572]: audit 2023-06-03T19:30:46.917179+0000 mon.a (mon.0) 503 : audit [DBG] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-06-03T19:30:48.282 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:30:47 smithi089 bash[26572]: audit 2023-06-03T19:30:47.622612+0000 mon.b (mon.1) 16 : audit [DBG] from='client.? 172.21.15.164:0/3020916068' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-06-03T19:30:48.282 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:30:47 smithi089 bash[33173]: audit 2023-06-03T19:30:46.905669+0000 mon.b (mon.1) 15 : audit [INF] from='client.? 172.21.15.164:0/800234170' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "bdff9290-e6c4-407e-9113-eb39696c0e27"}]: dispatch 2023-06-03T19:30:48.282 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:30:47 smithi089 bash[33173]: audit 2023-06-03T19:30:46.908341+0000 mon.a (mon.0) 500 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "bdff9290-e6c4-407e-9113-eb39696c0e27"}]: dispatch 2023-06-03T19:30:48.282 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:30:47 smithi089 bash[33173]: audit 2023-06-03T19:30:46.916811+0000 mon.a (mon.0) 501 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "bdff9290-e6c4-407e-9113-eb39696c0e27"}]': finished 2023-06-03T19:30:48.283 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:30:47 smithi089 bash[33173]: cluster 2023-06-03T19:30:46.916977+0000 mon.a (mon.0) 502 : cluster [DBG] osdmap e45: 8 total, 7 up, 8 in 2023-06-03T19:30:48.283 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:30:47 smithi089 bash[33173]: audit 2023-06-03T19:30:46.917179+0000 mon.a (mon.0) 503 : audit [DBG] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-06-03T19:30:48.283 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:30:47 smithi089 bash[33173]: audit 2023-06-03T19:30:47.622612+0000 mon.b (mon.1) 16 : audit [DBG] from='client.? 172.21.15.164:0/3020916068' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-06-03T19:30:48.336 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:30:47 smithi164 bash[26855]: audit 2023-06-03T19:30:46.905669+0000 mon.b (mon.1) 15 : audit [INF] from='client.? 172.21.15.164:0/800234170' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "bdff9290-e6c4-407e-9113-eb39696c0e27"}]: dispatch 2023-06-03T19:30:48.336 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:30:47 smithi164 bash[26855]: audit 2023-06-03T19:30:46.908341+0000 mon.a (mon.0) 500 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "bdff9290-e6c4-407e-9113-eb39696c0e27"}]: dispatch 2023-06-03T19:30:48.336 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:30:47 smithi164 bash[26855]: audit 2023-06-03T19:30:46.916811+0000 mon.a (mon.0) 501 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "bdff9290-e6c4-407e-9113-eb39696c0e27"}]': finished 2023-06-03T19:30:48.336 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:30:47 smithi164 bash[26855]: cluster 2023-06-03T19:30:46.916977+0000 mon.a (mon.0) 502 : cluster [DBG] osdmap e45: 8 total, 7 up, 8 in 2023-06-03T19:30:48.336 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:30:47 smithi164 bash[26855]: audit 2023-06-03T19:30:46.917179+0000 mon.a (mon.0) 503 : audit [DBG] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-06-03T19:30:48.336 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:30:47 smithi164 bash[26855]: audit 2023-06-03T19:30:47.622612+0000 mon.b (mon.1) 16 : audit [DBG] from='client.? 172.21.15.164:0/3020916068' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-06-03T19:30:49.281 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:30:48 smithi089 bash[26572]: cluster 2023-06-03T19:30:47.786954+0000 mgr.y (mgr.14146) 255 : cluster [DBG] pgmap v241: 1 pgs: 1 active+clean; 0 B data, 36 MiB used, 626 GiB / 626 GiB avail 2023-06-03T19:30:49.281 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:30:48 smithi089 bash[33173]: cluster 2023-06-03T19:30:47.786954+0000 mgr.y (mgr.14146) 255 : cluster [DBG] pgmap v241: 1 pgs: 1 active+clean; 0 B data, 36 MiB used, 626 GiB / 626 GiB avail 2023-06-03T19:30:49.335 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:30:48 smithi164 bash[26855]: cluster 2023-06-03T19:30:47.786954+0000 mgr.y (mgr.14146) 255 : cluster [DBG] pgmap v241: 1 pgs: 1 active+clean; 0 B data, 36 MiB used, 626 GiB / 626 GiB avail 2023-06-03T19:30:51.281 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:30:50 smithi089 bash[26572]: cluster 2023-06-03T19:30:49.787524+0000 mgr.y (mgr.14146) 256 : cluster [DBG] pgmap v242: 1 pgs: 1 active+clean; 0 B data, 36 MiB used, 626 GiB / 626 GiB avail 2023-06-03T19:30:51.281 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:30:50 smithi089 bash[33173]: cluster 2023-06-03T19:30:49.787524+0000 mgr.y (mgr.14146) 256 : cluster [DBG] pgmap v242: 1 pgs: 1 active+clean; 0 B data, 36 MiB used, 626 GiB / 626 GiB avail 2023-06-03T19:30:51.335 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:30:50 smithi164 bash[26855]: cluster 2023-06-03T19:30:49.787524+0000 mgr.y (mgr.14146) 256 : cluster [DBG] pgmap v242: 1 pgs: 1 active+clean; 0 B data, 36 MiB used, 626 GiB / 626 GiB avail 2023-06-03T19:30:52.781 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:30:52 smithi089 bash[26572]: cluster 2023-06-03T19:30:51.788108+0000 mgr.y (mgr.14146) 257 : cluster [DBG] pgmap v243: 1 pgs: 1 active+clean; 0 B data, 36 MiB used, 626 GiB / 626 GiB avail 2023-06-03T19:30:52.781 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:30:52 smithi089 bash[33173]: cluster 2023-06-03T19:30:51.788108+0000 mgr.y (mgr.14146) 257 : cluster [DBG] pgmap v243: 1 pgs: 1 active+clean; 0 B data, 36 MiB used, 626 GiB / 626 GiB avail 2023-06-03T19:30:52.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:30:52 smithi164 bash[26855]: cluster 2023-06-03T19:30:51.788108+0000 mgr.y (mgr.14146) 257 : cluster [DBG] pgmap v243: 1 pgs: 1 active+clean; 0 B data, 36 MiB used, 626 GiB / 626 GiB avail 2023-06-03T19:30:55.281 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:30:54 smithi089 bash[26572]: cluster 2023-06-03T19:30:53.788676+0000 mgr.y (mgr.14146) 258 : cluster [DBG] pgmap v244: 1 pgs: 1 active+clean; 0 B data, 36 MiB used, 626 GiB / 626 GiB avail 2023-06-03T19:30:55.281 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:30:54 smithi089 bash[33173]: cluster 2023-06-03T19:30:53.788676+0000 mgr.y (mgr.14146) 258 : cluster [DBG] pgmap v244: 1 pgs: 1 active+clean; 0 B data, 36 MiB used, 626 GiB / 626 GiB avail 2023-06-03T19:30:55.335 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:30:54 smithi164 bash[26855]: cluster 2023-06-03T19:30:53.788676+0000 mgr.y (mgr.14146) 258 : cluster [DBG] pgmap v244: 1 pgs: 1 active+clean; 0 B data, 36 MiB used, 626 GiB / 626 GiB avail 2023-06-03T19:30:57.281 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:30:56 smithi089 bash[26572]: cluster 2023-06-03T19:30:55.789265+0000 mgr.y (mgr.14146) 259 : cluster [DBG] pgmap v245: 1 pgs: 1 active+clean; 0 B data, 36 MiB used, 626 GiB / 626 GiB avail 2023-06-03T19:30:57.281 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:30:56 smithi089 bash[33173]: cluster 2023-06-03T19:30:55.789265+0000 mgr.y (mgr.14146) 259 : cluster [DBG] pgmap v245: 1 pgs: 1 active+clean; 0 B data, 36 MiB used, 626 GiB / 626 GiB avail 2023-06-03T19:30:57.336 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:30:56 smithi164 bash[26855]: cluster 2023-06-03T19:30:55.789265+0000 mgr.y (mgr.14146) 259 : cluster [DBG] pgmap v245: 1 pgs: 1 active+clean; 0 B data, 36 MiB used, 626 GiB / 626 GiB avail 2023-06-03T19:30:59.281 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:30:58 smithi089 bash[26572]: cluster 2023-06-03T19:30:57.789801+0000 mgr.y (mgr.14146) 260 : cluster [DBG] pgmap v246: 1 pgs: 1 active+clean; 0 B data, 36 MiB used, 626 GiB / 626 GiB avail 2023-06-03T19:30:59.281 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:30:58 smithi089 bash[33173]: cluster 2023-06-03T19:30:57.789801+0000 mgr.y (mgr.14146) 260 : cluster [DBG] pgmap v246: 1 pgs: 1 active+clean; 0 B data, 36 MiB used, 626 GiB / 626 GiB avail 2023-06-03T19:30:59.340 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:30:58 smithi164 bash[26855]: cluster 2023-06-03T19:30:57.789801+0000 mgr.y (mgr.14146) 260 : cluster [DBG] pgmap v246: 1 pgs: 1 active+clean; 0 B data, 36 MiB used, 626 GiB / 626 GiB avail 2023-06-03T19:31:00.281 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:30:59 smithi089 bash[26572]: audit 2023-06-03T19:30:59.570716+0000 mon.a (mon.0) 504 : audit [INF] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2023-06-03T19:31:00.281 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:30:59 smithi089 bash[26572]: audit 2023-06-03T19:30:59.572353+0000 mon.a (mon.0) 505 : audit [DBG] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-03T19:31:00.281 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:30:59 smithi089 bash[33173]: audit 2023-06-03T19:30:59.570716+0000 mon.a (mon.0) 504 : audit [INF] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2023-06-03T19:31:00.282 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:30:59 smithi089 bash[33173]: audit 2023-06-03T19:30:59.572353+0000 mon.a (mon.0) 505 : audit [DBG] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-03T19:31:00.335 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:30:59 smithi164 bash[26855]: audit 2023-06-03T19:30:59.570716+0000 mon.a (mon.0) 504 : audit [INF] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2023-06-03T19:31:00.336 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:30:59 smithi164 bash[26855]: audit 2023-06-03T19:30:59.572353+0000 mon.a (mon.0) 505 : audit [DBG] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-03T19:31:01.054 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:31:00 smithi164 bash[26855]: cephadm 2023-06-03T19:30:59.573597+0000 mgr.y (mgr.14146) 261 : cephadm [INF] Deploying daemon osd.7 on smithi164 2023-06-03T19:31:01.055 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:31:00 smithi164 bash[26855]: cluster 2023-06-03T19:30:59.790370+0000 mgr.y (mgr.14146) 262 : cluster [DBG] pgmap v247: 1 pgs: 1 active+clean; 0 B data, 36 MiB used, 626 GiB / 626 GiB avail 2023-06-03T19:31:01.281 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:31:00 smithi089 bash[26572]: cephadm 2023-06-03T19:30:59.573597+0000 mgr.y (mgr.14146) 261 : cephadm [INF] Deploying daemon osd.7 on smithi164 2023-06-03T19:31:01.281 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:31:00 smithi089 bash[26572]: cluster 2023-06-03T19:30:59.790370+0000 mgr.y (mgr.14146) 262 : cluster [DBG] pgmap v247: 1 pgs: 1 active+clean; 0 B data, 36 MiB used, 626 GiB / 626 GiB avail 2023-06-03T19:31:01.281 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:31:00 smithi089 bash[33173]: cephadm 2023-06-03T19:30:59.573597+0000 mgr.y (mgr.14146) 261 : cephadm [INF] Deploying daemon osd.7 on smithi164 2023-06-03T19:31:01.281 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:31:00 smithi089 bash[33173]: cluster 2023-06-03T19:30:59.790370+0000 mgr.y (mgr.14146) 262 : cluster [DBG] pgmap v247: 1 pgs: 1 active+clean; 0 B data, 36 MiB used, 626 GiB / 626 GiB avail 2023-06-03T19:31:02.781 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:31:02 smithi089 bash[26572]: cluster 2023-06-03T19:31:01.790941+0000 mgr.y (mgr.14146) 263 : cluster [DBG] pgmap v248: 1 pgs: 1 active+clean; 0 B data, 36 MiB used, 626 GiB / 626 GiB avail 2023-06-03T19:31:02.781 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:31:02 smithi089 bash[33173]: cluster 2023-06-03T19:31:01.790941+0000 mgr.y (mgr.14146) 263 : cluster [DBG] pgmap v248: 1 pgs: 1 active+clean; 0 B data, 36 MiB used, 626 GiB / 626 GiB avail 2023-06-03T19:31:02.836 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:31:02 smithi164 bash[26855]: cluster 2023-06-03T19:31:01.790941+0000 mgr.y (mgr.14146) 263 : cluster [DBG] pgmap v248: 1 pgs: 1 active+clean; 0 B data, 36 MiB used, 626 GiB / 626 GiB avail 2023-06-03T19:31:05.335 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:31:05 smithi164 bash[26855]: cluster 2023-06-03T19:31:03.791522+0000 mgr.y (mgr.14146) 264 : cluster [DBG] pgmap v249: 1 pgs: 1 active+clean; 0 B data, 36 MiB used, 626 GiB / 626 GiB avail 2023-06-03T19:31:05.336 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:31:05 smithi164 bash[26855]: audit 2023-06-03T19:31:04.115607+0000 mon.a (mon.0) 506 : audit [INF] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' 2023-06-03T19:31:05.336 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:31:05 smithi164 bash[26855]: audit 2023-06-03T19:31:04.133129+0000 mon.a (mon.0) 507 : audit [DBG] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-03T19:31:05.531 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:31:05 smithi089 bash[26572]: cluster 2023-06-03T19:31:03.791522+0000 mgr.y (mgr.14146) 264 : cluster [DBG] pgmap v249: 1 pgs: 1 active+clean; 0 B data, 36 MiB used, 626 GiB / 626 GiB avail 2023-06-03T19:31:05.531 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:31:05 smithi089 bash[26572]: audit 2023-06-03T19:31:04.115607+0000 mon.a (mon.0) 506 : audit [INF] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' 2023-06-03T19:31:05.531 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:31:05 smithi089 bash[26572]: audit 2023-06-03T19:31:04.133129+0000 mon.a (mon.0) 507 : audit [DBG] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-03T19:31:05.532 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:31:05 smithi089 bash[33173]: cluster 2023-06-03T19:31:03.791522+0000 mgr.y (mgr.14146) 264 : cluster [DBG] pgmap v249: 1 pgs: 1 active+clean; 0 B data, 36 MiB used, 626 GiB / 626 GiB avail 2023-06-03T19:31:05.532 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:31:05 smithi089 bash[33173]: audit 2023-06-03T19:31:04.115607+0000 mon.a (mon.0) 506 : audit [INF] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' 2023-06-03T19:31:05.532 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:31:05 smithi089 bash[33173]: audit 2023-06-03T19:31:04.133129+0000 mon.a (mon.0) 507 : audit [DBG] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-03T19:31:07.401 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:31:07 smithi164 bash[26855]: cluster 2023-06-03T19:31:05.792057+0000 mgr.y (mgr.14146) 265 : cluster [DBG] pgmap v250: 1 pgs: 1 active+clean; 0 B data, 36 MiB used, 626 GiB / 626 GiB avail 2023-06-03T19:31:07.531 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:31:07 smithi089 bash[26572]: cluster 2023-06-03T19:31:05.792057+0000 mgr.y (mgr.14146) 265 : cluster [DBG] pgmap v250: 1 pgs: 1 active+clean; 0 B data, 36 MiB used, 626 GiB / 626 GiB avail 2023-06-03T19:31:07.531 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:31:07 smithi089 bash[33173]: cluster 2023-06-03T19:31:05.792057+0000 mgr.y (mgr.14146) 265 : cluster [DBG] pgmap v250: 1 pgs: 1 active+clean; 0 B data, 36 MiB used, 626 GiB / 626 GiB avail 2023-06-03T19:31:09.531 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:31:09 smithi089 bash[26572]: cluster 2023-06-03T19:31:07.792599+0000 mgr.y (mgr.14146) 266 : cluster [DBG] pgmap v251: 1 pgs: 1 active+clean; 0 B data, 36 MiB used, 626 GiB / 626 GiB avail 2023-06-03T19:31:09.531 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:31:09 smithi089 bash[33173]: cluster 2023-06-03T19:31:07.792599+0000 mgr.y (mgr.14146) 266 : cluster [DBG] pgmap v251: 1 pgs: 1 active+clean; 0 B data, 36 MiB used, 626 GiB / 626 GiB avail 2023-06-03T19:31:09.585 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:31:09 smithi164 bash[26855]: cluster 2023-06-03T19:31:07.792599+0000 mgr.y (mgr.14146) 266 : cluster [DBG] pgmap v251: 1 pgs: 1 active+clean; 0 B data, 36 MiB used, 626 GiB / 626 GiB avail 2023-06-03T19:31:11.311 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:31:11 smithi164 bash[26855]: cluster 2023-06-03T19:31:09.793240+0000 mgr.y (mgr.14146) 267 : cluster [DBG] pgmap v252: 1 pgs: 1 active+clean; 0 B data, 36 MiB used, 626 GiB / 626 GiB avail 2023-06-03T19:31:11.311 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:31:11 smithi164 bash[26855]: audit 2023-06-03T19:31:10.401721+0000 mon.a (mon.0) 508 : audit [INF] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' 2023-06-03T19:31:11.312 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:31:11 smithi164 bash[26855]: audit 2023-06-03T19:31:10.403355+0000 mon.a (mon.0) 509 : audit [DBG] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-03T19:31:11.312 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:31:11 smithi164 bash[26855]: audit 2023-06-03T19:31:10.404614+0000 mon.a (mon.0) 510 : audit [INF] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-03T19:31:11.312 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:31:11 smithi164 bash[26855]: audit 2023-06-03T19:31:10.410527+0000 mon.a (mon.0) 511 : audit [INF] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' 2023-06-03T19:31:11.312 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:31:11 smithi164 bash[26855]: audit 2023-06-03T19:31:11.055583+0000 mon.b (mon.1) 17 : audit [INF] from='osd.7 [v2:172.21.15.164:6824/711997991,v1:172.21.15.164:6825/711997991]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2023-06-03T19:31:11.312 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:31:11 smithi164 bash[26855]: audit 2023-06-03T19:31:11.058396+0000 mon.a (mon.0) 512 : audit [INF] from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2023-06-03T19:31:11.531 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:31:11 smithi089 bash[26572]: cluster 2023-06-03T19:31:09.793240+0000 mgr.y (mgr.14146) 267 : cluster [DBG] pgmap v252: 1 pgs: 1 active+clean; 0 B data, 36 MiB used, 626 GiB / 626 GiB avail 2023-06-03T19:31:11.532 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:31:11 smithi089 bash[26572]: audit 2023-06-03T19:31:10.401721+0000 mon.a (mon.0) 508 : audit [INF] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' 2023-06-03T19:31:11.532 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:31:11 smithi089 bash[26572]: audit 2023-06-03T19:31:10.403355+0000 mon.a (mon.0) 509 : audit [DBG] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-03T19:31:11.532 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:31:11 smithi089 bash[26572]: audit 2023-06-03T19:31:10.404614+0000 mon.a (mon.0) 510 : audit [INF] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-03T19:31:11.533 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:31:11 smithi089 bash[26572]: audit 2023-06-03T19:31:10.410527+0000 mon.a (mon.0) 511 : audit [INF] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' 2023-06-03T19:31:11.534 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:31:11 smithi089 bash[26572]: audit 2023-06-03T19:31:11.055583+0000 mon.b (mon.1) 17 : audit [INF] from='osd.7 [v2:172.21.15.164:6824/711997991,v1:172.21.15.164:6825/711997991]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2023-06-03T19:31:11.534 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:31:11 smithi089 bash[26572]: audit 2023-06-03T19:31:11.058396+0000 mon.a (mon.0) 512 : audit [INF] from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2023-06-03T19:31:11.535 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:31:11 smithi089 bash[33173]: cluster 2023-06-03T19:31:09.793240+0000 mgr.y (mgr.14146) 267 : cluster [DBG] pgmap v252: 1 pgs: 1 active+clean; 0 B data, 36 MiB used, 626 GiB / 626 GiB avail 2023-06-03T19:31:11.535 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:31:11 smithi089 bash[33173]: audit 2023-06-03T19:31:10.401721+0000 mon.a (mon.0) 508 : audit [INF] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' 2023-06-03T19:31:11.535 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:31:11 smithi089 bash[33173]: audit 2023-06-03T19:31:10.403355+0000 mon.a (mon.0) 509 : audit [DBG] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-03T19:31:11.535 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:31:11 smithi089 bash[33173]: audit 2023-06-03T19:31:10.404614+0000 mon.a (mon.0) 510 : audit [INF] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-03T19:31:11.535 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:31:11 smithi089 bash[33173]: audit 2023-06-03T19:31:10.410527+0000 mon.a (mon.0) 511 : audit [INF] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' 2023-06-03T19:31:11.535 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:31:11 smithi089 bash[33173]: audit 2023-06-03T19:31:11.055583+0000 mon.b (mon.1) 17 : audit [INF] from='osd.7 [v2:172.21.15.164:6824/711997991,v1:172.21.15.164:6825/711997991]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2023-06-03T19:31:11.536 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:31:11 smithi089 bash[33173]: audit 2023-06-03T19:31:11.058396+0000 mon.a (mon.0) 512 : audit [INF] from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2023-06-03T19:31:12.764 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:31:12 smithi164 bash[26855]: audit 2023-06-03T19:31:11.417379+0000 mon.b (mon.1) 18 : audit [INF] from='osd.7 [v2:172.21.15.164:6824/711997991,v1:172.21.15.164:6825/711997991]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi164", "root=default"]}]: dispatch 2023-06-03T19:31:12.765 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:31:12 smithi164 bash[26855]: audit 2023-06-03T19:31:11.418038+0000 mon.a (mon.0) 513 : audit [INF] from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2023-06-03T19:31:12.765 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:31:12 smithi164 bash[26855]: cluster 2023-06-03T19:31:11.418222+0000 mon.a (mon.0) 514 : cluster [DBG] osdmap e46: 8 total, 7 up, 8 in 2023-06-03T19:31:12.765 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:31:12 smithi164 bash[26855]: audit 2023-06-03T19:31:11.418444+0000 mon.a (mon.0) 515 : audit [DBG] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-06-03T19:31:12.765 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:31:12 smithi164 bash[26855]: audit 2023-06-03T19:31:11.419769+0000 mon.a (mon.0) 516 : audit [INF] from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi164", "root=default"]}]: dispatch 2023-06-03T19:31:12.765 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:31:12 smithi164 bash[26855]: cluster 2023-06-03T19:31:11.793905+0000 mgr.y (mgr.14146) 268 : cluster [DBG] pgmap v254: 1 pgs: 1 active+clean; 0 B data, 36 MiB used, 626 GiB / 626 GiB avail 2023-06-03T19:31:12.781 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:31:12 smithi089 bash[26572]: audit 2023-06-03T19:31:11.417379+0000 mon.b (mon.1) 18 : audit [INF] from='osd.7 [v2:172.21.15.164:6824/711997991,v1:172.21.15.164:6825/711997991]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi164", "root=default"]}]: dispatch 2023-06-03T19:31:12.781 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:31:12 smithi089 bash[26572]: audit 2023-06-03T19:31:11.418038+0000 mon.a (mon.0) 513 : audit [INF] from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2023-06-03T19:31:12.781 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:31:12 smithi089 bash[26572]: cluster 2023-06-03T19:31:11.418222+0000 mon.a (mon.0) 514 : cluster [DBG] osdmap e46: 8 total, 7 up, 8 in 2023-06-03T19:31:12.781 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:31:12 smithi089 bash[26572]: audit 2023-06-03T19:31:11.418444+0000 mon.a (mon.0) 515 : audit [DBG] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-06-03T19:31:12.782 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:31:12 smithi089 bash[26572]: audit 2023-06-03T19:31:11.419769+0000 mon.a (mon.0) 516 : audit [INF] from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi164", "root=default"]}]: dispatch 2023-06-03T19:31:12.782 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:31:12 smithi089 bash[26572]: cluster 2023-06-03T19:31:11.793905+0000 mgr.y (mgr.14146) 268 : cluster [DBG] pgmap v254: 1 pgs: 1 active+clean; 0 B data, 36 MiB used, 626 GiB / 626 GiB avail 2023-06-03T19:31:12.782 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:31:12 smithi089 bash[33173]: audit 2023-06-03T19:31:11.417379+0000 mon.b (mon.1) 18 : audit [INF] from='osd.7 [v2:172.21.15.164:6824/711997991,v1:172.21.15.164:6825/711997991]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi164", "root=default"]}]: dispatch 2023-06-03T19:31:12.782 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:31:12 smithi089 bash[33173]: audit 2023-06-03T19:31:11.418038+0000 mon.a (mon.0) 513 : audit [INF] from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2023-06-03T19:31:12.782 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:31:12 smithi089 bash[33173]: cluster 2023-06-03T19:31:11.418222+0000 mon.a (mon.0) 514 : cluster [DBG] osdmap e46: 8 total, 7 up, 8 in 2023-06-03T19:31:12.782 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:31:12 smithi089 bash[33173]: audit 2023-06-03T19:31:11.418444+0000 mon.a (mon.0) 515 : audit [DBG] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-06-03T19:31:12.783 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:31:12 smithi089 bash[33173]: audit 2023-06-03T19:31:11.419769+0000 mon.a (mon.0) 516 : audit [INF] from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi164", "root=default"]}]: dispatch 2023-06-03T19:31:12.783 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:31:12 smithi089 bash[33173]: cluster 2023-06-03T19:31:11.793905+0000 mgr.y (mgr.14146) 268 : cluster [DBG] pgmap v254: 1 pgs: 1 active+clean; 0 B data, 36 MiB used, 626 GiB / 626 GiB avail 2023-06-03T19:31:13.781 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:31:13 smithi089 bash[26572]: audit 2023-06-03T19:31:12.418891+0000 mon.a (mon.0) 517 : audit [INF] from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi164", "root=default"]}]': finished 2023-06-03T19:31:13.781 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:31:13 smithi089 bash[26572]: cluster 2023-06-03T19:31:12.419054+0000 mon.a (mon.0) 518 : cluster [DBG] osdmap e47: 8 total, 7 up, 8 in 2023-06-03T19:31:13.781 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:31:13 smithi089 bash[26572]: audit 2023-06-03T19:31:12.421476+0000 mon.a (mon.0) 519 : audit [DBG] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-06-03T19:31:13.782 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:31:13 smithi089 bash[26572]: audit 2023-06-03T19:31:12.429626+0000 mon.a (mon.0) 520 : audit [DBG] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-06-03T19:31:13.782 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:31:13 smithi089 bash[33173]: audit 2023-06-03T19:31:12.418891+0000 mon.a (mon.0) 517 : audit [INF] from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi164", "root=default"]}]': finished 2023-06-03T19:31:13.782 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:31:13 smithi089 bash[33173]: cluster 2023-06-03T19:31:12.419054+0000 mon.a (mon.0) 518 : cluster [DBG] osdmap e47: 8 total, 7 up, 8 in 2023-06-03T19:31:13.782 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:31:13 smithi089 bash[33173]: audit 2023-06-03T19:31:12.421476+0000 mon.a (mon.0) 519 : audit [DBG] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-06-03T19:31:13.782 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:31:13 smithi089 bash[33173]: audit 2023-06-03T19:31:12.429626+0000 mon.a (mon.0) 520 : audit [DBG] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-06-03T19:31:13.836 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:31:13 smithi164 bash[26855]: audit 2023-06-03T19:31:12.418891+0000 mon.a (mon.0) 517 : audit [INF] from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi164", "root=default"]}]': finished 2023-06-03T19:31:13.836 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:31:13 smithi164 bash[26855]: cluster 2023-06-03T19:31:12.419054+0000 mon.a (mon.0) 518 : cluster [DBG] osdmap e47: 8 total, 7 up, 8 in 2023-06-03T19:31:13.836 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:31:13 smithi164 bash[26855]: audit 2023-06-03T19:31:12.421476+0000 mon.a (mon.0) 519 : audit [DBG] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-06-03T19:31:13.836 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:31:13 smithi164 bash[26855]: audit 2023-06-03T19:31:12.429626+0000 mon.a (mon.0) 520 : audit [DBG] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-06-03T19:31:14.238 INFO:teuthology.orchestra.run.smithi164.stdout:Created osd(s) 7 on host 'smithi164' 2023-06-03T19:31:14.586 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:31:14 smithi164 bash[26855]: cluster 2023-06-03T19:31:12.018324+0000 osd.7 (osd.7) 1 : cluster [DBG] purged_snaps scrub starts 2023-06-03T19:31:14.586 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:31:14 smithi164 bash[26855]: cluster 2023-06-03T19:31:12.018410+0000 osd.7 (osd.7) 2 : cluster [DBG] purged_snaps scrub ok 2023-06-03T19:31:14.586 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:31:14 smithi164 bash[26855]: audit 2023-06-03T19:31:13.425714+0000 mon.a (mon.0) 521 : audit [DBG] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-06-03T19:31:14.586 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:31:14 smithi164 bash[26855]: cluster 2023-06-03T19:31:13.435432+0000 mon.a (mon.0) 522 : cluster [INF] osd.7 [v2:172.21.15.164:6824/711997991,v1:172.21.15.164:6825/711997991] boot 2023-06-03T19:31:14.586 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:31:14 smithi164 bash[26855]: cluster 2023-06-03T19:31:13.435495+0000 mon.a (mon.0) 523 : cluster [DBG] osdmap e48: 8 total, 8 up, 8 in 2023-06-03T19:31:14.587 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:31:14 smithi164 bash[26855]: audit 2023-06-03T19:31:13.436233+0000 mon.a (mon.0) 524 : audit [DBG] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-06-03T19:31:14.587 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:31:14 smithi164 bash[26855]: cluster 2023-06-03T19:31:13.794478+0000 mgr.y (mgr.14146) 269 : cluster [DBG] pgmap v257: 1 pgs: 1 active+clean; 0 B data, 36 MiB used, 626 GiB / 626 GiB avail 2023-06-03T19:31:14.587 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:31:14 smithi164 bash[26855]: audit 2023-06-03T19:31:14.229833+0000 mon.a (mon.0) 525 : audit [INF] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' 2023-06-03T19:31:14.587 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:31:14 smithi164 bash[26855]: audit 2023-06-03T19:31:14.239824+0000 mon.a (mon.0) 526 : audit [DBG] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-03T19:31:14.781 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:31:14 smithi089 bash[33173]: cluster 2023-06-03T19:31:12.018324+0000 osd.7 (osd.7) 1 : cluster [DBG] purged_snaps scrub starts 2023-06-03T19:31:14.781 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:31:14 smithi089 bash[33173]: cluster 2023-06-03T19:31:12.018410+0000 osd.7 (osd.7) 2 : cluster [DBG] purged_snaps scrub ok 2023-06-03T19:31:14.781 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:31:14 smithi089 bash[33173]: audit 2023-06-03T19:31:13.425714+0000 mon.a (mon.0) 521 : audit [DBG] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-06-03T19:31:14.782 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:31:14 smithi089 bash[33173]: cluster 2023-06-03T19:31:13.435432+0000 mon.a (mon.0) 522 : cluster [INF] osd.7 [v2:172.21.15.164:6824/711997991,v1:172.21.15.164:6825/711997991] boot 2023-06-03T19:31:14.782 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:31:14 smithi089 bash[33173]: cluster 2023-06-03T19:31:13.435495+0000 mon.a (mon.0) 523 : cluster [DBG] osdmap e48: 8 total, 8 up, 8 in 2023-06-03T19:31:14.782 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:31:14 smithi089 bash[33173]: audit 2023-06-03T19:31:13.436233+0000 mon.a (mon.0) 524 : audit [DBG] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-06-03T19:31:14.782 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:31:14 smithi089 bash[33173]: cluster 2023-06-03T19:31:13.794478+0000 mgr.y (mgr.14146) 269 : cluster [DBG] pgmap v257: 1 pgs: 1 active+clean; 0 B data, 36 MiB used, 626 GiB / 626 GiB avail 2023-06-03T19:31:14.782 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:31:14 smithi089 bash[33173]: audit 2023-06-03T19:31:14.229833+0000 mon.a (mon.0) 525 : audit [INF] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' 2023-06-03T19:31:14.782 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:31:14 smithi089 bash[33173]: audit 2023-06-03T19:31:14.239824+0000 mon.a (mon.0) 526 : audit [DBG] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-03T19:31:14.783 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:31:14 smithi089 bash[26572]: cluster 2023-06-03T19:31:12.018324+0000 osd.7 (osd.7) 1 : cluster [DBG] purged_snaps scrub starts 2023-06-03T19:31:14.783 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:31:14 smithi089 bash[26572]: cluster 2023-06-03T19:31:12.018410+0000 osd.7 (osd.7) 2 : cluster [DBG] purged_snaps scrub ok 2023-06-03T19:31:14.783 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:31:14 smithi089 bash[26572]: audit 2023-06-03T19:31:13.425714+0000 mon.a (mon.0) 521 : audit [DBG] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-06-03T19:31:14.783 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:31:14 smithi089 bash[26572]: cluster 2023-06-03T19:31:13.435432+0000 mon.a (mon.0) 522 : cluster [INF] osd.7 [v2:172.21.15.164:6824/711997991,v1:172.21.15.164:6825/711997991] boot 2023-06-03T19:31:14.783 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:31:14 smithi089 bash[26572]: cluster 2023-06-03T19:31:13.435495+0000 mon.a (mon.0) 523 : cluster [DBG] osdmap e48: 8 total, 8 up, 8 in 2023-06-03T19:31:14.783 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:31:14 smithi089 bash[26572]: audit 2023-06-03T19:31:13.436233+0000 mon.a (mon.0) 524 : audit [DBG] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-06-03T19:31:14.784 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:31:14 smithi089 bash[26572]: cluster 2023-06-03T19:31:13.794478+0000 mgr.y (mgr.14146) 269 : cluster [DBG] pgmap v257: 1 pgs: 1 active+clean; 0 B data, 36 MiB used, 626 GiB / 626 GiB avail 2023-06-03T19:31:14.784 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:31:14 smithi089 bash[26572]: audit 2023-06-03T19:31:14.229833+0000 mon.a (mon.0) 525 : audit [INF] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' 2023-06-03T19:31:14.784 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:31:14 smithi089 bash[26572]: audit 2023-06-03T19:31:14.239824+0000 mon.a (mon.0) 526 : audit [DBG] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-03T19:31:14.895 DEBUG:teuthology.orchestra.run.smithi164:osd.7> sudo journalctl -f -n 0 -u ceph-f1db3478-0243-11ee-9b1d-001a4aab830c@osd.7.service 2023-06-03T19:31:14.897 INFO:tasks.cephadm:Waiting for 8 OSDs to come up... 2023-06-03T19:31:14.898 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph osd stat -f json 2023-06-03T19:31:15.004 INFO:journalctl@ceph.osd.7.smithi164.stdout:-- Logs begin at Sun 2022-08-14 12:20:24 UTC. -- 2023-06-03T19:31:15.781 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:31:15 smithi089 bash[33173]: cluster 2023-06-03T19:31:14.465749+0000 mon.a (mon.0) 527 : cluster [DBG] osdmap e49: 8 total, 8 up, 8 in 2023-06-03T19:31:15.781 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:31:15 smithi089 bash[26572]: cluster 2023-06-03T19:31:14.465749+0000 mon.a (mon.0) 527 : cluster [DBG] osdmap e49: 8 total, 8 up, 8 in 2023-06-03T19:31:15.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:31:15 smithi164 bash[26855]: cluster 2023-06-03T19:31:14.465749+0000 mon.a (mon.0) 527 : cluster [DBG] osdmap e49: 8 total, 8 up, 8 in 2023-06-03T19:31:16.668 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:31:16 smithi089 bash[26572]: cluster 2023-06-03T19:31:15.464332+0000 mon.a (mon.0) 528 : cluster [DBG] osdmap e50: 8 total, 8 up, 8 in 2023-06-03T19:31:16.669 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:31:16 smithi089 bash[26572]: cluster 2023-06-03T19:31:15.795123+0000 mgr.y (mgr.14146) 270 : cluster [DBG] pgmap v260: 1 pgs: 1 active+clean; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:31:16.669 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:31:16 smithi089 bash[33173]: cluster 2023-06-03T19:31:15.464332+0000 mon.a (mon.0) 528 : cluster [DBG] osdmap e50: 8 total, 8 up, 8 in 2023-06-03T19:31:16.669 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:31:16 smithi089 bash[33173]: cluster 2023-06-03T19:31:15.795123+0000 mgr.y (mgr.14146) 270 : cluster [DBG] pgmap v260: 1 pgs: 1 active+clean; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:31:16.836 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:31:16 smithi164 bash[26855]: cluster 2023-06-03T19:31:15.464332+0000 mon.a (mon.0) 528 : cluster [DBG] osdmap e50: 8 total, 8 up, 8 in 2023-06-03T19:31:16.836 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:31:16 smithi164 bash[26855]: cluster 2023-06-03T19:31:15.795123+0000 mgr.y (mgr.14146) 270 : cluster [DBG] pgmap v260: 1 pgs: 1 active+clean; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:31:18.567 INFO:teuthology.orchestra.run.smithi089.stdout: 2023-06-03T19:31:19.031 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:31:18 smithi089 bash[26572]: cluster 2023-06-03T19:31:17.795777+0000 mgr.y (mgr.14146) 271 : cluster [DBG] pgmap v261: 1 pgs: 1 active+clean; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:31:19.031 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:31:18 smithi089 bash[26572]: audit 2023-06-03T19:31:18.559808+0000 mon.c (mon.2) 12 : audit [DBG] from='client.? 172.21.15.89:0/3712956050' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2023-06-03T19:31:19.032 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:31:18 smithi089 bash[33173]: cluster 2023-06-03T19:31:17.795777+0000 mgr.y (mgr.14146) 271 : cluster [DBG] pgmap v261: 1 pgs: 1 active+clean; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:31:19.032 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:31:18 smithi089 bash[33173]: audit 2023-06-03T19:31:18.559808+0000 mon.c (mon.2) 12 : audit [DBG] from='client.? 172.21.15.89:0/3712956050' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2023-06-03T19:31:19.335 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:31:18 smithi164 bash[26855]: cluster 2023-06-03T19:31:17.795777+0000 mgr.y (mgr.14146) 271 : cluster [DBG] pgmap v261: 1 pgs: 1 active+clean; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:31:19.336 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:31:18 smithi164 bash[26855]: audit 2023-06-03T19:31:18.559808+0000 mon.c (mon.2) 12 : audit [DBG] from='client.? 172.21.15.89:0/3712956050' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2023-06-03T19:31:19.349 INFO:teuthology.orchestra.run.smithi089.stdout:{"epoch":50,"num_osds":8,"num_up_osds":8,"osd_up_since":1685820673,"num_in_osds":8,"osd_in_since":1685820646,"num_remapped_pgs":0} 2023-06-03T19:31:19.349 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph osd dump --format=json 2023-06-03T19:31:21.281 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:31:20 smithi089 bash[26572]: cluster 2023-06-03T19:31:19.796453+0000 mgr.y (mgr.14146) 272 : cluster [DBG] pgmap v262: 1 pgs: 1 active+clean; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:31:21.282 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:31:20 smithi089 bash[33173]: cluster 2023-06-03T19:31:19.796453+0000 mgr.y (mgr.14146) 272 : cluster [DBG] pgmap v262: 1 pgs: 1 active+clean; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:31:21.336 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:31:20 smithi164 bash[26855]: cluster 2023-06-03T19:31:19.796453+0000 mgr.y (mgr.14146) 272 : cluster [DBG] pgmap v262: 1 pgs: 1 active+clean; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:31:22.781 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:31:22 smithi089 bash[26572]: cluster 2023-06-03T19:31:21.797038+0000 mgr.y (mgr.14146) 273 : cluster [DBG] pgmap v263: 1 pgs: 1 active+clean; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:31:22.781 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:31:22 smithi089 bash[33173]: cluster 2023-06-03T19:31:21.797038+0000 mgr.y (mgr.14146) 273 : cluster [DBG] pgmap v263: 1 pgs: 1 active+clean; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:31:22.836 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:31:22 smithi164 bash[26855]: cluster 2023-06-03T19:31:21.797038+0000 mgr.y (mgr.14146) 273 : cluster [DBG] pgmap v263: 1 pgs: 1 active+clean; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:31:24.144 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/mon.a/config 2023-06-03T19:31:25.031 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:31:24 smithi089 bash[26572]: cephadm 2023-06-03T19:31:23.649255+0000 mgr.y (mgr.14146) 274 : cephadm [INF] Detected new or changed devices on smithi164 2023-06-03T19:31:25.032 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:31:24 smithi089 bash[26572]: audit 2023-06-03T19:31:23.658161+0000 mon.a (mon.0) 529 : audit [INF] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' 2023-06-03T19:31:25.032 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:31:24 smithi089 bash[26572]: audit 2023-06-03T19:31:23.659978+0000 mon.a (mon.0) 530 : audit [INF] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-06-03T19:31:25.032 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:31:24 smithi089 bash[26572]: audit 2023-06-03T19:31:23.661291+0000 mon.a (mon.0) 531 : audit [INF] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-06-03T19:31:25.032 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:31:24 smithi089 bash[26572]: audit 2023-06-03T19:31:23.662397+0000 mon.a (mon.0) 532 : audit [INF] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2023-06-03T19:31:25.032 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:31:24 smithi089 bash[26572]: audit 2023-06-03T19:31:23.663494+0000 mon.a (mon.0) 533 : audit [INF] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2023-06-03T19:31:25.033 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:31:24 smithi089 bash[26572]: cephadm 2023-06-03T19:31:23.664345+0000 mgr.y (mgr.14146) 275 : cephadm [INF] Adjusting osd_memory_target on smithi164 to 4318M 2023-06-03T19:31:25.033 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:31:24 smithi089 bash[26572]: audit 2023-06-03T19:31:23.670820+0000 mon.a (mon.0) 534 : audit [INF] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' 2023-06-03T19:31:25.033 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:31:24 smithi089 bash[26572]: audit 2023-06-03T19:31:23.672269+0000 mon.a (mon.0) 535 : audit [DBG] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-03T19:31:25.033 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:31:24 smithi089 bash[26572]: audit 2023-06-03T19:31:23.673367+0000 mon.a (mon.0) 536 : audit [INF] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-03T19:31:25.033 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:31:24 smithi089 bash[26572]: audit 2023-06-03T19:31:23.679360+0000 mon.a (mon.0) 537 : audit [INF] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' 2023-06-03T19:31:25.033 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:31:24 smithi089 bash[26572]: cluster 2023-06-03T19:31:23.797546+0000 mgr.y (mgr.14146) 276 : cluster [DBG] pgmap v264: 1 pgs: 1 active+clean; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:31:25.033 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:31:24 smithi089 bash[33173]: cephadm 2023-06-03T19:31:23.649255+0000 mgr.y (mgr.14146) 274 : cephadm [INF] Detected new or changed devices on smithi164 2023-06-03T19:31:25.034 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:31:24 smithi089 bash[33173]: audit 2023-06-03T19:31:23.658161+0000 mon.a (mon.0) 529 : audit [INF] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' 2023-06-03T19:31:25.034 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:31:24 smithi089 bash[33173]: audit 2023-06-03T19:31:23.659978+0000 mon.a (mon.0) 530 : audit [INF] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-06-03T19:31:25.034 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:31:24 smithi089 bash[33173]: audit 2023-06-03T19:31:23.661291+0000 mon.a (mon.0) 531 : audit [INF] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-06-03T19:31:25.034 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:31:24 smithi089 bash[33173]: audit 2023-06-03T19:31:23.662397+0000 mon.a (mon.0) 532 : audit [INF] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2023-06-03T19:31:25.034 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:31:24 smithi089 bash[33173]: audit 2023-06-03T19:31:23.663494+0000 mon.a (mon.0) 533 : audit [INF] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2023-06-03T19:31:25.034 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:31:24 smithi089 bash[33173]: cephadm 2023-06-03T19:31:23.664345+0000 mgr.y (mgr.14146) 275 : cephadm [INF] Adjusting osd_memory_target on smithi164 to 4318M 2023-06-03T19:31:25.035 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:31:24 smithi089 bash[33173]: audit 2023-06-03T19:31:23.670820+0000 mon.a (mon.0) 534 : audit [INF] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' 2023-06-03T19:31:25.035 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:31:24 smithi089 bash[33173]: audit 2023-06-03T19:31:23.672269+0000 mon.a (mon.0) 535 : audit [DBG] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-03T19:31:25.035 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:31:24 smithi089 bash[33173]: audit 2023-06-03T19:31:23.673367+0000 mon.a (mon.0) 536 : audit [INF] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-03T19:31:25.035 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:31:24 smithi089 bash[33173]: audit 2023-06-03T19:31:23.679360+0000 mon.a (mon.0) 537 : audit [INF] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' 2023-06-03T19:31:25.035 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:31:24 smithi089 bash[33173]: cluster 2023-06-03T19:31:23.797546+0000 mgr.y (mgr.14146) 276 : cluster [DBG] pgmap v264: 1 pgs: 1 active+clean; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:31:25.086 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:31:24 smithi164 bash[26855]: cephadm 2023-06-03T19:31:23.649255+0000 mgr.y (mgr.14146) 274 : cephadm [INF] Detected new or changed devices on smithi164 2023-06-03T19:31:25.086 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:31:24 smithi164 bash[26855]: audit 2023-06-03T19:31:23.658161+0000 mon.a (mon.0) 529 : audit [INF] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' 2023-06-03T19:31:25.086 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:31:24 smithi164 bash[26855]: audit 2023-06-03T19:31:23.659978+0000 mon.a (mon.0) 530 : audit [INF] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-06-03T19:31:25.087 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:31:24 smithi164 bash[26855]: audit 2023-06-03T19:31:23.661291+0000 mon.a (mon.0) 531 : audit [INF] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-06-03T19:31:25.087 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:31:24 smithi164 bash[26855]: audit 2023-06-03T19:31:23.662397+0000 mon.a (mon.0) 532 : audit [INF] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2023-06-03T19:31:25.087 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:31:24 smithi164 bash[26855]: audit 2023-06-03T19:31:23.663494+0000 mon.a (mon.0) 533 : audit [INF] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2023-06-03T19:31:25.087 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:31:24 smithi164 bash[26855]: cephadm 2023-06-03T19:31:23.664345+0000 mgr.y (mgr.14146) 275 : cephadm [INF] Adjusting osd_memory_target on smithi164 to 4318M 2023-06-03T19:31:25.087 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:31:24 smithi164 bash[26855]: audit 2023-06-03T19:31:23.670820+0000 mon.a (mon.0) 534 : audit [INF] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' 2023-06-03T19:31:25.087 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:31:24 smithi164 bash[26855]: audit 2023-06-03T19:31:23.672269+0000 mon.a (mon.0) 535 : audit [DBG] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-03T19:31:25.087 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:31:24 smithi164 bash[26855]: audit 2023-06-03T19:31:23.673367+0000 mon.a (mon.0) 536 : audit [INF] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-03T19:31:25.087 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:31:24 smithi164 bash[26855]: audit 2023-06-03T19:31:23.679360+0000 mon.a (mon.0) 537 : audit [INF] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' 2023-06-03T19:31:25.088 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:31:24 smithi164 bash[26855]: cluster 2023-06-03T19:31:23.797546+0000 mgr.y (mgr.14146) 276 : cluster [DBG] pgmap v264: 1 pgs: 1 active+clean; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:31:27.281 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:31:26 smithi089 bash[26572]: cluster 2023-06-03T19:31:25.797977+0000 mgr.y (mgr.14146) 277 : cluster [DBG] pgmap v265: 1 pgs: 1 active+clean; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:31:27.281 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:31:26 smithi089 bash[33173]: cluster 2023-06-03T19:31:25.797977+0000 mgr.y (mgr.14146) 277 : cluster [DBG] pgmap v265: 1 pgs: 1 active+clean; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:31:27.335 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:31:26 smithi164 bash[26855]: cluster 2023-06-03T19:31:25.797977+0000 mgr.y (mgr.14146) 277 : cluster [DBG] pgmap v265: 1 pgs: 1 active+clean; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:31:27.385 INFO:teuthology.orchestra.run.smithi089.stdout: 2023-06-03T19:31:27.385 INFO:teuthology.orchestra.run.smithi089.stdout:{"epoch":50,"fsid":"f1db3478-0243-11ee-9b1d-001a4aab830c","created":"2023-06-03T19:22:54.447184+0000","modified":"2023-06-03T19:31:15.456815+0000","last_up_change":"2023-06-03T19:31:13.424226+0000","last_in_change":"2023-06-03T19:30:46.909676+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":1,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"pacific","pools":[{"pool":1,"pool_name":"device_health_metrics","create_time":"2023-06-03T19:27:28.100610+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"21","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr_devicehealth":{}}}],"osds":[{"osd":0,"uuid":"86063bfe-126e-46e7-93ed-c73813af165b","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":48,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:6802","nonce":1795715824},{"type":"v1","addr":"172.21.15.89:6803","nonce":1795715824}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:6804","nonce":1795715824},{"type":"v1","addr":"172.21.15.89:6805","nonce":1795715824}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:6808","nonce":1795715824},{"type":"v1","addr":"172.21.15.89:6809","nonce":1795715824}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:6806","nonce":1795715824},{"type":"v1","addr":"172.21.15.89:6807","nonce":1795715824}]},"public_addr":"172.21.15.89:6803/1795715824","cluster_addr":"172.21.15.89:6805/1795715824","heartbeat_back_addr":"172.21.15.89:6809/1795715824","heartbeat_front_addr":"172.21.15.89:6807/1795715824","state":["exists","up"]},{"osd":1,"uuid":"04783a81-6b8c-4657-8446-00efc872b313","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":31,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:6810","nonce":4020168282},{"type":"v1","addr":"172.21.15.89:6811","nonce":4020168282}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:6812","nonce":4020168282},{"type":"v1","addr":"172.21.15.89:6813","nonce":4020168282}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:6816","nonce":4020168282},{"type":"v1","addr":"172.21.15.89:6817","nonce":4020168282}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:6814","nonce":4020168282},{"type":"v1","addr":"172.21.15.89:6815","nonce":4020168282}]},"public_addr":"172.21.15.89:6811/4020168282","cluster_addr":"172.21.15.89:6813/4020168282","heartbeat_back_addr":"172.21.15.89:6817/4020168282","heartbeat_front_addr":"172.21.15.89:6815/4020168282","state":["exists","up"]},{"osd":2,"uuid":"d2d2c641-392a-444f-81bc-84eef6340f30","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:6818","nonce":2113694583},{"type":"v1","addr":"172.21.15.89:6819","nonce":2113694583}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:6820","nonce":2113694583},{"type":"v1","addr":"172.21.15.89:6821","nonce":2113694583}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:6824","nonce":2113694583},{"type":"v1","addr":"172.21.15.89:6825","nonce":2113694583}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:6822","nonce":2113694583},{"type":"v1","addr":"172.21.15.89:6823","nonce":2113694583}]},"public_addr":"172.21.15.89:6819/2113694583","cluster_addr":"172.21.15.89:6821/2113694583","heartbeat_back_addr":"172.21.15.89:6825/2113694583","heartbeat_front_addr":"172.21.15.89:6823/2113694583","state":["exists","up"]},{"osd":3,"uuid":"b38408eb-3acf-4a41-a314-72e4cd37b1ef","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":25,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:6826","nonce":3649114317},{"type":"v1","addr":"172.21.15.89:6827","nonce":3649114317}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:6828","nonce":3649114317},{"type":"v1","addr":"172.21.15.89:6829","nonce":3649114317}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:6832","nonce":3649114317},{"type":"v1","addr":"172.21.15.89:6833","nonce":3649114317}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:6830","nonce":3649114317},{"type":"v1","addr":"172.21.15.89:6831","nonce":3649114317}]},"public_addr":"172.21.15.89:6827/3649114317","cluster_addr":"172.21.15.89:6829/3649114317","heartbeat_back_addr":"172.21.15.89:6833/3649114317","heartbeat_front_addr":"172.21.15.89:6831/3649114317","state":["exists","up"]},{"osd":4,"uuid":"a94be785-9f54-4ab8-962a-9cf974a5b5e3","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":30,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.164:6800","nonce":538382489},{"type":"v1","addr":"172.21.15.164:6801","nonce":538382489}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.164:6802","nonce":538382489},{"type":"v1","addr":"172.21.15.164:6803","nonce":538382489}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.164:6806","nonce":538382489},{"type":"v1","addr":"172.21.15.164:6807","nonce":538382489}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.164:6804","nonce":538382489},{"type":"v1","addr":"172.21.15.164:6805","nonce":538382489}]},"public_addr":"172.21.15.164:6801/538382489","cluster_addr":"172.21.15.164:6803/538382489","heartbeat_back_addr":"172.21.15.164:6807/538382489","heartbeat_front_addr":"172.21.15.164:6805/538382489","state":["exists","up"]},{"osd":5,"uuid":"b8a5e8a1-23cc-442b-90aa-19bd0036dd3c","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":36,"up_thru":37,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.164:6808","nonce":3123006405},{"type":"v1","addr":"172.21.15.164:6809","nonce":3123006405}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.164:6810","nonce":3123006405},{"type":"v1","addr":"172.21.15.164:6811","nonce":3123006405}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.164:6814","nonce":3123006405},{"type":"v1","addr":"172.21.15.164:6815","nonce":3123006405}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.164:6812","nonce":3123006405},{"type":"v1","addr":"172.21.15.164:6813","nonce":3123006405}]},"public_addr":"172.21.15.164:6809/3123006405","cluster_addr":"172.21.15.164:6811/3123006405","heartbeat_back_addr":"172.21.15.164:6815/3123006405","heartbeat_front_addr":"172.21.15.164:6813/3123006405","state":["exists","up"]},{"osd":6,"uuid":"5c5edd3b-d92c-4f46-94ff-2d5828b2e732","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":42,"up_thru":43,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.164:6816","nonce":712403290},{"type":"v1","addr":"172.21.15.164:6817","nonce":712403290}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.164:6818","nonce":712403290},{"type":"v1","addr":"172.21.15.164:6819","nonce":712403290}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.164:6822","nonce":712403290},{"type":"v1","addr":"172.21.15.164:6823","nonce":712403290}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.164:6820","nonce":712403290},{"type":"v1","addr":"172.21.15.164:6821","nonce":712403290}]},"public_addr":"172.21.15.164:6817/712403290","cluster_addr":"172.21.15.164:6819/712403290","heartbeat_back_addr":"172.21.15.164:6823/712403290","heartbeat_front_addr":"172.21.15.164:6821/712403290","state":["exists","up"]},{"osd":7,"uuid":"bdff9290-e6c4-407e-9113-eb39696c0e27","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":48,"up_thru":49,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.164:6824","nonce":711997991},{"type":"v1","addr":"172.21.15.164:6825","nonce":711997991}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.164:6826","nonce":711997991},{"type":"v1","addr":"172.21.15.164:6827","nonce":711997991}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.164:6830","nonce":711997991},{"type":"v1","addr":"172.21.15.164:6831","nonce":711997991}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.164:6828","nonce":711997991},{"type":"v1","addr":"172.21.15.164:6829","nonce":711997991}]},"public_addr":"172.21.15.164:6825/711997991","cluster_addr":"172.21.15.164:6827/711997991","heartbeat_back_addr":"172.21.15.164:6831/711997991","heartbeat_front_addr":"172.21.15.164:6829/711997991","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-06-03T19:25:52.155714+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-06-03T19:26:38.554458+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-06-03T19:27:25.383237+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-06-03T19:28:13.272585+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-06-03T19:28:56.685710+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-06-03T19:29:41.572533+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-06-03T19:30:26.631983+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-06-03T19:31:12.018414+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.89:0/3944267183":"2023-06-04T19:23:55.626873+0000","172.21.15.89:6801/4116589411":"2023-06-04T19:23:55.626873+0000","172.21.15.89:6800/4116589411":"2023-06-04T19:23:55.626873+0000","172.21.15.89:0/274274920":"2023-06-04T19:23:55.626873+0000","172.21.15.89:0/1198702890":"2023-06-04T19:23:23.903333+0000","172.21.15.89:6800/1487566136":"2023-06-04T19:23:23.903333+0000","172.21.15.89:6801/1487566136":"2023-06-04T19:23:23.903333+0000","172.21.15.89:0/811967393":"2023-06-04T19:23:23.903333+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2023-06-03T19:31:28.155 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:31:27 smithi089 bash[26572]: audit 2023-06-03T19:31:27.377154+0000 mon.a (mon.0) 538 : audit [DBG] from='client.? 172.21.15.89:0/2119066087' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-06-03T19:31:28.155 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:31:27 smithi089 bash[33173]: audit 2023-06-03T19:31:27.377154+0000 mon.a (mon.0) 538 : audit [DBG] from='client.? 172.21.15.89:0/2119066087' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-06-03T19:31:28.156 INFO:tasks.cephadm.ceph_manager.ceph:[{'pool': 1, 'pool_name': 'device_health_metrics', 'create_time': '2023-06-03T19:27:28.100610+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 3, 'min_size': 2, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 1, 'pg_placement_num': 1, 'pg_placement_num_target': 1, 'pg_num_target': 1, 'pg_num_pending': 1, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '21', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_num_max': 32, 'pg_num_min': 1}, 'application_metadata': {'mgr_devicehealth': {}}}] 2023-06-03T19:31:28.157 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph osd pool get device_health_metrics pg_num 2023-06-03T19:31:28.335 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:31:27 smithi164 bash[26855]: audit 2023-06-03T19:31:27.377154+0000 mon.a (mon.0) 538 : audit [DBG] from='client.? 172.21.15.89:0/2119066087' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-06-03T19:31:29.281 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:31:28 smithi089 bash[26572]: cluster 2023-06-03T19:31:27.798469+0000 mgr.y (mgr.14146) 278 : cluster [DBG] pgmap v266: 1 pgs: 1 active+clean; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:31:29.281 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:31:28 smithi089 bash[33173]: cluster 2023-06-03T19:31:27.798469+0000 mgr.y (mgr.14146) 278 : cluster [DBG] pgmap v266: 1 pgs: 1 active+clean; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:31:29.336 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:31:28 smithi164 bash[26855]: cluster 2023-06-03T19:31:27.798469+0000 mgr.y (mgr.14146) 278 : cluster [DBG] pgmap v266: 1 pgs: 1 active+clean; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:31:31.281 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:31:30 smithi089 bash[26572]: cluster 2023-06-03T19:31:29.799091+0000 mgr.y (mgr.14146) 279 : cluster [DBG] pgmap v267: 1 pgs: 1 active+clean; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:31:31.281 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:31:30 smithi089 bash[33173]: cluster 2023-06-03T19:31:29.799091+0000 mgr.y (mgr.14146) 279 : cluster [DBG] pgmap v267: 1 pgs: 1 active+clean; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:31:31.335 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:31:30 smithi164 bash[26855]: cluster 2023-06-03T19:31:29.799091+0000 mgr.y (mgr.14146) 279 : cluster [DBG] pgmap v267: 1 pgs: 1 active+clean; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:31:32.781 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:31:32 smithi089 bash[33173]: cluster 2023-06-03T19:31:31.799643+0000 mgr.y (mgr.14146) 280 : cluster [DBG] pgmap v268: 1 pgs: 1 active+clean; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:31:32.781 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:31:32 smithi089 bash[26572]: cluster 2023-06-03T19:31:31.799643+0000 mgr.y (mgr.14146) 280 : cluster [DBG] pgmap v268: 1 pgs: 1 active+clean; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:31:32.836 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:31:32 smithi164 bash[26855]: cluster 2023-06-03T19:31:31.799643+0000 mgr.y (mgr.14146) 280 : cluster [DBG] pgmap v268: 1 pgs: 1 active+clean; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:31:32.943 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/mon.a/config 2023-06-03T19:31:35.281 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:31:34 smithi089 bash[33173]: cluster 2023-06-03T19:31:33.800000+0000 mgr.y (mgr.14146) 281 : cluster [DBG] pgmap v269: 1 pgs: 1 active+clean; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:31:35.281 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:31:34 smithi089 bash[26572]: cluster 2023-06-03T19:31:33.800000+0000 mgr.y (mgr.14146) 281 : cluster [DBG] pgmap v269: 1 pgs: 1 active+clean; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:31:35.336 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:31:34 smithi164 bash[26855]: cluster 2023-06-03T19:31:33.800000+0000 mgr.y (mgr.14146) 281 : cluster [DBG] pgmap v269: 1 pgs: 1 active+clean; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:31:36.243 INFO:teuthology.orchestra.run.smithi089.stdout:pg_num: 1 2023-06-03T19:31:37.053 INFO:tasks.cephadm:Adding prometheus.a on smithi164 2023-06-03T19:31:37.053 DEBUG:teuthology.orchestra.run.smithi164:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph orch apply prometheus '1;smithi164=a' 2023-06-03T19:31:37.256 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:31:36 smithi164 bash[26855]: cluster 2023-06-03T19:31:35.800594+0000 mgr.y (mgr.14146) 282 : cluster [DBG] pgmap v270: 1 pgs: 1 active+clean; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:31:37.257 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:31:36 smithi164 bash[26855]: audit 2023-06-03T19:31:36.236203+0000 mon.c (mon.2) 13 : audit [DBG] from='client.? 172.21.15.89:0/1259955112' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": "device_health_metrics", "var": "pg_num"}]: dispatch 2023-06-03T19:31:37.281 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:31:36 smithi089 bash[26572]: cluster 2023-06-03T19:31:35.800594+0000 mgr.y (mgr.14146) 282 : cluster [DBG] pgmap v270: 1 pgs: 1 active+clean; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:31:37.281 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:31:36 smithi089 bash[26572]: audit 2023-06-03T19:31:36.236203+0000 mon.c (mon.2) 13 : audit [DBG] from='client.? 172.21.15.89:0/1259955112' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": "device_health_metrics", "var": "pg_num"}]: dispatch 2023-06-03T19:31:37.281 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:31:36 smithi089 bash[33173]: cluster 2023-06-03T19:31:35.800594+0000 mgr.y (mgr.14146) 282 : cluster [DBG] pgmap v270: 1 pgs: 1 active+clean; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:31:37.282 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:31:36 smithi089 bash[33173]: audit 2023-06-03T19:31:36.236203+0000 mon.c (mon.2) 13 : audit [DBG] from='client.? 172.21.15.89:0/1259955112' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": "device_health_metrics", "var": "pg_num"}]: dispatch 2023-06-03T19:31:39.281 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:31:38 smithi089 bash[26572]: cluster 2023-06-03T19:31:37.801149+0000 mgr.y (mgr.14146) 283 : cluster [DBG] pgmap v271: 1 pgs: 1 active+clean; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:31:39.281 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:31:38 smithi089 bash[33173]: cluster 2023-06-03T19:31:37.801149+0000 mgr.y (mgr.14146) 283 : cluster [DBG] pgmap v271: 1 pgs: 1 active+clean; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:31:39.335 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:31:38 smithi164 bash[26855]: cluster 2023-06-03T19:31:37.801149+0000 mgr.y (mgr.14146) 283 : cluster [DBG] pgmap v271: 1 pgs: 1 active+clean; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:31:40.424 INFO:teuthology.orchestra.run.smithi164.stdout:Scheduled prometheus update... 2023-06-03T19:31:41.086 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:31:40 smithi164 bash[26855]: cluster 2023-06-03T19:31:39.801667+0000 mgr.y (mgr.14146) 284 : cluster [DBG] pgmap v272: 1 pgs: 1 active+clean; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:31:41.086 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:31:40 smithi164 bash[26855]: audit 2023-06-03T19:31:40.417049+0000 mon.a (mon.0) 539 : audit [INF] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' 2023-06-03T19:31:41.086 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:31:40 smithi164 bash[26855]: audit 2023-06-03T19:31:40.441380+0000 mon.a (mon.0) 540 : audit [DBG] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-03T19:31:41.086 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:31:40 smithi164 bash[26855]: audit 2023-06-03T19:31:40.715028+0000 mon.a (mon.0) 541 : audit [DBG] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-03T19:31:41.087 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:31:40 smithi164 bash[26855]: audit 2023-06-03T19:31:40.716642+0000 mon.a (mon.0) 542 : audit [INF] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-03T19:31:41.087 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:31:40 smithi164 bash[26855]: audit 2023-06-03T19:31:40.725155+0000 mon.a (mon.0) 543 : audit [INF] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' 2023-06-03T19:31:41.087 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:31:40 smithi164 bash[26855]: audit 2023-06-03T19:31:40.732919+0000 mon.a (mon.0) 544 : audit [INF] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "mgr module enable", "module": "prometheus"}]: dispatch 2023-06-03T19:31:41.272 DEBUG:teuthology.orchestra.run.smithi164:prometheus.a> sudo journalctl -f -n 0 -u ceph-f1db3478-0243-11ee-9b1d-001a4aab830c@prometheus.a.service 2023-06-03T19:31:41.275 INFO:tasks.cephadm:Adding node-exporter.a on smithi089 2023-06-03T19:31:41.275 INFO:tasks.cephadm:Adding node-exporter.b on smithi164 2023-06-03T19:31:41.276 DEBUG:teuthology.orchestra.run.smithi164:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph orch apply node-exporter '2;smithi089=a;smithi164=b' 2023-06-03T19:31:41.281 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:31:40 smithi089 bash[26572]: cluster 2023-06-03T19:31:39.801667+0000 mgr.y (mgr.14146) 284 : cluster [DBG] pgmap v272: 1 pgs: 1 active+clean; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:31:41.281 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:31:40 smithi089 bash[26572]: audit 2023-06-03T19:31:40.417049+0000 mon.a (mon.0) 539 : audit [INF] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' 2023-06-03T19:31:41.282 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:31:40 smithi089 bash[26572]: audit 2023-06-03T19:31:40.441380+0000 mon.a (mon.0) 540 : audit [DBG] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-03T19:31:41.282 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:31:40 smithi089 bash[26572]: audit 2023-06-03T19:31:40.715028+0000 mon.a (mon.0) 541 : audit [DBG] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-03T19:31:41.282 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:31:40 smithi089 bash[26572]: audit 2023-06-03T19:31:40.716642+0000 mon.a (mon.0) 542 : audit [INF] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-03T19:31:41.283 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:31:40 smithi089 bash[26572]: audit 2023-06-03T19:31:40.725155+0000 mon.a (mon.0) 543 : audit [INF] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' 2023-06-03T19:31:41.283 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:31:40 smithi089 bash[26572]: audit 2023-06-03T19:31:40.732919+0000 mon.a (mon.0) 544 : audit [INF] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "mgr module enable", "module": "prometheus"}]: dispatch 2023-06-03T19:31:41.283 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:31:40 smithi089 bash[33173]: cluster 2023-06-03T19:31:39.801667+0000 mgr.y (mgr.14146) 284 : cluster [DBG] pgmap v272: 1 pgs: 1 active+clean; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:31:41.284 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:31:40 smithi089 bash[33173]: audit 2023-06-03T19:31:40.417049+0000 mon.a (mon.0) 539 : audit [INF] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' 2023-06-03T19:31:41.284 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:31:40 smithi089 bash[33173]: audit 2023-06-03T19:31:40.441380+0000 mon.a (mon.0) 540 : audit [DBG] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-03T19:31:41.284 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:31:40 smithi089 bash[33173]: audit 2023-06-03T19:31:40.715028+0000 mon.a (mon.0) 541 : audit [DBG] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-03T19:31:41.284 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:31:40 smithi089 bash[33173]: audit 2023-06-03T19:31:40.716642+0000 mon.a (mon.0) 542 : audit [INF] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-03T19:31:41.285 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:31:40 smithi089 bash[33173]: audit 2023-06-03T19:31:40.725155+0000 mon.a (mon.0) 543 : audit [INF] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' 2023-06-03T19:31:41.285 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:31:40 smithi089 bash[33173]: audit 2023-06-03T19:31:40.732919+0000 mon.a (mon.0) 544 : audit [INF] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd=[{"prefix": "mgr module enable", "module": "prometheus"}]: dispatch 2023-06-03T19:31:41.475 INFO:journalctl@ceph.prometheus.a.smithi164.stdout:-- Logs begin at Sun 2022-08-14 12:20:24 UTC. -- 2023-06-03T19:31:42.031 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 19:31:41 smithi089 bash[26845]: ignoring --setuser ceph since I am not root 2023-06-03T19:31:42.031 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 19:31:41 smithi089 bash[26845]: ignoring --setgroup ceph since I am not root 2023-06-03T19:31:42.031 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 19:31:41 smithi089 bash[26845]: debug 2023-06-03T19:31:41.776+0000 7fc74b6e6700 1 -- 172.21.15.89:0/261772874 <== mon.0 v2:172.21.15.89:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x55954708c4e0 con 0x559547180800 2023-06-03T19:31:42.031 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 19:31:41 smithi089 bash[26845]: debug 2023-06-03T19:31:41.928+0000 7fc75b8ab680 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2023-06-03T19:31:42.032 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:31:41 smithi089 bash[33173]: audit 2023-06-03T19:31:40.406593+0000 mgr.y (mgr.14146) 285 : audit [DBG] from='client.24349 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "prometheus", "placement": "1;smithi164=a", "target": ["mon-mgr", ""]}]: dispatch 2023-06-03T19:31:42.032 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:31:41 smithi089 bash[33173]: cephadm 2023-06-03T19:31:40.409796+0000 mgr.y (mgr.14146) 286 : cephadm [INF] Saving service prometheus spec with placement smithi164=a;count:1 2023-06-03T19:31:42.032 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:31:41 smithi089 bash[33173]: audit 2023-06-03T19:31:41.734185+0000 mon.a (mon.0) 545 : audit [INF] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd='[{"prefix": "mgr module enable", "module": "prometheus"}]': finished 2023-06-03T19:31:42.032 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:31:41 smithi089 bash[33173]: cluster 2023-06-03T19:31:41.734255+0000 mon.a (mon.0) 546 : cluster [DBG] mgrmap e15: y(active, since 7m), standbys: x 2023-06-03T19:31:42.033 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:31:41 smithi089 bash[26572]: audit 2023-06-03T19:31:40.406593+0000 mgr.y (mgr.14146) 285 : audit [DBG] from='client.24349 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "prometheus", "placement": "1;smithi164=a", "target": ["mon-mgr", ""]}]: dispatch 2023-06-03T19:31:42.033 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:31:41 smithi089 bash[26572]: cephadm 2023-06-03T19:31:40.409796+0000 mgr.y (mgr.14146) 286 : cephadm [INF] Saving service prometheus spec with placement smithi164=a;count:1 2023-06-03T19:31:42.033 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:31:41 smithi089 bash[26572]: audit 2023-06-03T19:31:41.734185+0000 mon.a (mon.0) 545 : audit [INF] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd='[{"prefix": "mgr module enable", "module": "prometheus"}]': finished 2023-06-03T19:31:42.033 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:31:41 smithi089 bash[26572]: cluster 2023-06-03T19:31:41.734255+0000 mon.a (mon.0) 546 : cluster [DBG] mgrmap e15: y(active, since 7m), standbys: x 2023-06-03T19:31:42.051 INFO:journalctl@ceph.mgr.x.smithi164.stdout:Jun 03 19:31:41 smithi164 bash[27934]: ignoring --setuser ceph since I am not root 2023-06-03T19:31:42.051 INFO:journalctl@ceph.mgr.x.smithi164.stdout:Jun 03 19:31:41 smithi164 bash[27934]: ignoring --setgroup ceph since I am not root 2023-06-03T19:31:42.052 INFO:journalctl@ceph.mgr.x.smithi164.stdout:Jun 03 19:31:41 smithi164 bash[27934]: debug 2023-06-03T19:31:41.770+0000 7f46d1240700 1 -- 172.21.15.164:0/392498485 <== mon.1 v2:172.21.15.164:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x55fc4b32c4e0 con 0x55fc4b421000 2023-06-03T19:31:42.052 INFO:journalctl@ceph.mgr.x.smithi164.stdout:Jun 03 19:31:41 smithi164 bash[27934]: debug 2023-06-03T19:31:41.930+0000 7f46e1405680 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2023-06-03T19:31:42.052 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:31:41 smithi164 bash[26855]: audit 2023-06-03T19:31:40.406593+0000 mgr.y (mgr.14146) 285 : audit [DBG] from='client.24349 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "prometheus", "placement": "1;smithi164=a", "target": ["mon-mgr", ""]}]: dispatch 2023-06-03T19:31:42.052 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:31:41 smithi164 bash[26855]: cephadm 2023-06-03T19:31:40.409796+0000 mgr.y (mgr.14146) 286 : cephadm [INF] Saving service prometheus spec with placement smithi164=a;count:1 2023-06-03T19:31:42.052 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:31:41 smithi164 bash[26855]: audit 2023-06-03T19:31:41.734185+0000 mon.a (mon.0) 545 : audit [INF] from='mgr.14146 172.21.15.89:0/1164248838' entity='mgr.y' cmd='[{"prefix": "mgr module enable", "module": "prometheus"}]': finished 2023-06-03T19:31:42.053 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:31:41 smithi164 bash[26855]: cluster 2023-06-03T19:31:41.734255+0000 mon.a (mon.0) 546 : cluster [DBG] mgrmap e15: y(active, since 7m), standbys: x 2023-06-03T19:31:42.336 INFO:journalctl@ceph.mgr.x.smithi164.stdout:Jun 03 19:31:42 smithi164 bash[27934]: debug 2023-06-03T19:31:42.038+0000 7f46e1405680 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2023-06-03T19:31:42.507 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 19:31:42 smithi089 bash[26845]: debug 2023-06-03T19:31:42.032+0000 7fc75b8ab680 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2023-06-03T19:31:42.781 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 19:31:42 smithi089 bash[26845]: debug 2023-06-03T19:31:42.492+0000 7fc75b8ab680 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2023-06-03T19:31:42.833 INFO:journalctl@ceph.mgr.x.smithi164.stdout:Jun 03 19:31:42 smithi164 bash[27934]: debug 2023-06-03T19:31:42.518+0000 7f46e1405680 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2023-06-03T19:31:43.778 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 19:31:43 smithi089 bash[26845]: debug 2023-06-03T19:31:43.416+0000 7fc75b8ab680 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2023-06-03T19:31:43.778 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 19:31:43 smithi089 bash[26845]: debug 2023-06-03T19:31:43.512+0000 7fc75b8ab680 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2023-06-03T19:31:43.835 INFO:journalctl@ceph.mgr.x.smithi164.stdout:Jun 03 19:31:43 smithi164 bash[27934]: debug 2023-06-03T19:31:43.518+0000 7f46e1405680 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2023-06-03T19:31:43.836 INFO:journalctl@ceph.mgr.x.smithi164.stdout:Jun 03 19:31:43 smithi164 bash[27934]: debug 2023-06-03T19:31:43.622+0000 7f46e1405680 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2023-06-03T19:31:44.031 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 19:31:43 smithi089 bash[26845]: debug 2023-06-03T19:31:43.764+0000 7fc75b8ab680 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2023-06-03T19:31:44.335 INFO:journalctl@ceph.mgr.x.smithi164.stdout:Jun 03 19:31:43 smithi164 bash[27934]: debug 2023-06-03T19:31:43.874+0000 7f46e1405680 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2023-06-03T19:31:45.171 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 19:31:44 smithi089 bash[26845]: debug 2023-06-03T19:31:44.896+0000 7fc75b8ab680 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2023-06-03T19:31:45.172 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 19:31:45 smithi089 bash[26845]: debug 2023-06-03T19:31:45.060+0000 7fc75b8ab680 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2023-06-03T19:31:45.372 INFO:journalctl@ceph.mgr.x.smithi164.stdout:Jun 03 19:31:45 smithi164 bash[27934]: debug 2023-06-03T19:31:45.074+0000 7f46e1405680 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2023-06-03T19:31:45.373 INFO:journalctl@ceph.mgr.x.smithi164.stdout:Jun 03 19:31:45 smithi164 bash[27934]: debug 2023-06-03T19:31:45.250+0000 7f46e1405680 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2023-06-03T19:31:45.509 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 19:31:45 smithi089 bash[26845]: debug 2023-06-03T19:31:45.160+0000 7fc75b8ab680 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2023-06-03T19:31:45.509 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 19:31:45 smithi089 bash[26845]: debug 2023-06-03T19:31:45.272+0000 7fc75b8ab680 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2023-06-03T19:31:45.742 INFO:journalctl@ceph.mgr.x.smithi164.stdout:Jun 03 19:31:45 smithi164 bash[27934]: debug 2023-06-03T19:31:45.358+0000 7f46e1405680 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2023-06-03T19:31:45.743 INFO:journalctl@ceph.mgr.x.smithi164.stdout:Jun 03 19:31:45 smithi164 bash[27934]: debug 2023-06-03T19:31:45.482+0000 7f46e1405680 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2023-06-03T19:31:45.781 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 19:31:45 smithi089 bash[26845]: debug 2023-06-03T19:31:45.496+0000 7fc75b8ab680 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2023-06-03T19:31:46.085 INFO:journalctl@ceph.mgr.x.smithi164.stdout:Jun 03 19:31:45 smithi164 bash[27934]: debug 2023-06-03T19:31:45.730+0000 7f46e1405680 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2023-06-03T19:31:46.281 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 19:31:45 smithi089 bash[26845]: debug 2023-06-03T19:31:45.868+0000 7fc75b8ab680 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2023-06-03T19:31:46.281 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 19:31:45 smithi089 bash[26845]: debug 2023-06-03T19:31:45.992+0000 7fc75b8ab680 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2023-06-03T19:31:46.586 INFO:journalctl@ceph.mgr.x.smithi164.stdout:Jun 03 19:31:46 smithi164 bash[27934]: debug 2023-06-03T19:31:46.118+0000 7f46e1405680 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2023-06-03T19:31:46.586 INFO:journalctl@ceph.mgr.x.smithi164.stdout:Jun 03 19:31:46 smithi164 bash[27934]: debug 2023-06-03T19:31:46.238+0000 7f46e1405680 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2023-06-03T19:31:47.281 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 19:31:46 smithi089 bash[26845]: debug 2023-06-03T19:31:46.860+0000 7fc75b8ab680 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2023-06-03T19:31:47.281 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 19:31:46 smithi089 bash[26845]: debug 2023-06-03T19:31:46.964+0000 7fc75b8ab680 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2023-06-03T19:31:47.281 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 19:31:47 smithi089 bash[26845]: debug 2023-06-03T19:31:47.080+0000 7fc75b8ab680 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2023-06-03T19:31:47.567 INFO:journalctl@ceph.mgr.x.smithi164.stdout:Jun 03 19:31:47 smithi164 bash[27934]: debug 2023-06-03T19:31:47.106+0000 7f46e1405680 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2023-06-03T19:31:47.567 INFO:journalctl@ceph.mgr.x.smithi164.stdout:Jun 03 19:31:47 smithi164 bash[27934]: debug 2023-06-03T19:31:47.210+0000 7f46e1405680 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2023-06-03T19:31:47.568 INFO:journalctl@ceph.mgr.x.smithi164.stdout:Jun 03 19:31:47 smithi164 bash[27934]: debug 2023-06-03T19:31:47.326+0000 7f46e1405680 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2023-06-03T19:31:47.612 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 19:31:47 smithi089 bash[26845]: debug 2023-06-03T19:31:47.308+0000 7fc75b8ab680 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2023-06-03T19:31:47.836 INFO:journalctl@ceph.mgr.x.smithi164.stdout:Jun 03 19:31:47 smithi164 bash[27934]: debug 2023-06-03T19:31:47.554+0000 7f46e1405680 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2023-06-03T19:31:47.952 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 19:31:47 smithi089 bash[26845]: debug 2023-06-03T19:31:47.604+0000 7fc75b8ab680 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2023-06-03T19:31:47.953 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 19:31:47 smithi089 bash[26845]: debug 2023-06-03T19:31:47.772+0000 7fc75b8ab680 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2023-06-03T19:31:48.204 INFO:journalctl@ceph.mgr.x.smithi164.stdout:Jun 03 19:31:47 smithi164 bash[27934]: debug 2023-06-03T19:31:47.854+0000 7f46e1405680 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2023-06-03T19:31:48.204 INFO:journalctl@ceph.mgr.x.smithi164.stdout:Jun 03 19:31:48 smithi164 bash[27934]: debug 2023-06-03T19:31:48.022+0000 7f46e1405680 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2023-06-03T19:31:48.281 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 19:31:47 smithi089 bash[26845]: debug 2023-06-03T19:31:47.944+0000 7fc75b8ab680 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2023-06-03T19:31:48.281 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 19:31:48 smithi089 bash[26845]: debug 2023-06-03T19:31:48.164+0000 7fc75b8ab680 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2023-06-03T19:31:48.535 INFO:journalctl@ceph.mgr.x.smithi164.stdout:Jun 03 19:31:48 smithi164 bash[27934]: debug 2023-06-03T19:31:48.190+0000 7f46e1405680 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2023-06-03T19:31:48.535 INFO:journalctl@ceph.mgr.x.smithi164.stdout:Jun 03 19:31:48 smithi164 bash[27934]: debug 2023-06-03T19:31:48.406+0000 7f46e1405680 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2023-06-03T19:31:48.535 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:31:48 smithi164 bash[26855]: cluster 2023-06-03T19:31:48.303207+0000 mon.a (mon.0) 547 : cluster [INF] Active manager daemon y restarted 2023-06-03T19:31:48.535 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:31:48 smithi164 bash[26855]: cluster 2023-06-03T19:31:48.304020+0000 mon.a (mon.0) 548 : cluster [INF] Activating manager daemon y 2023-06-03T19:31:48.536 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:31:48 smithi164 bash[26855]: cluster 2023-06-03T19:31:48.310835+0000 mon.a (mon.0) 549 : cluster [DBG] osdmap e51: 8 total, 8 up, 8 in 2023-06-03T19:31:48.624 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 19:31:48 smithi089 bash[26845]: debug 2023-06-03T19:31:48.280+0000 7fc75b8ab680 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2023-06-03T19:31:48.626 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:31:48 smithi089 bash[26572]: cluster 2023-06-03T19:31:48.303207+0000 mon.a (mon.0) 547 : cluster [INF] Active manager daemon y restarted 2023-06-03T19:31:48.626 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:31:48 smithi089 bash[26572]: cluster 2023-06-03T19:31:48.304020+0000 mon.a (mon.0) 548 : cluster [INF] Activating manager daemon y 2023-06-03T19:31:48.626 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:31:48 smithi089 bash[26572]: cluster 2023-06-03T19:31:48.310835+0000 mon.a (mon.0) 549 : cluster [DBG] osdmap e51: 8 total, 8 up, 8 in 2023-06-03T19:31:48.627 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:31:48 smithi089 bash[33173]: cluster 2023-06-03T19:31:48.303207+0000 mon.a (mon.0) 547 : cluster [INF] Active manager daemon y restarted 2023-06-03T19:31:48.627 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:31:48 smithi089 bash[33173]: cluster 2023-06-03T19:31:48.304020+0000 mon.a (mon.0) 548 : cluster [INF] Activating manager daemon y 2023-06-03T19:31:48.627 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:31:48 smithi089 bash[33173]: cluster 2023-06-03T19:31:48.310835+0000 mon.a (mon.0) 549 : cluster [DBG] osdmap e51: 8 total, 8 up, 8 in 2023-06-03T19:31:48.836 INFO:journalctl@ceph.mgr.x.smithi164.stdout:Jun 03 19:31:48 smithi164 bash[27934]: debug 2023-06-03T19:31:48.518+0000 7f46e1405680 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2023-06-03T19:31:48.836 INFO:journalctl@ceph.mgr.x.smithi164.stdout:Jun 03 19:31:48 smithi164 bash[27934]: [03/Jun/2023:19:31:48] ENGINE Bus STARTING 2023-06-03T19:31:48.837 INFO:journalctl@ceph.mgr.x.smithi164.stdout:Jun 03 19:31:48 smithi164 bash[27934]: CherryPy Checker: 2023-06-03T19:31:48.837 INFO:journalctl@ceph.mgr.x.smithi164.stdout:Jun 03 19:31:48 smithi164 bash[27934]: The Application mounted at '' has an empty config. 2023-06-03T19:31:48.837 INFO:journalctl@ceph.mgr.x.smithi164.stdout:Jun 03 19:31:48 smithi164 bash[27934]: [03/Jun/2023:19:31:48] ENGINE Serving on http://:::9283 2023-06-03T19:31:48.837 INFO:journalctl@ceph.mgr.x.smithi164.stdout:Jun 03 19:31:48 smithi164 bash[27934]: [03/Jun/2023:19:31:48] ENGINE Bus STARTED 2023-06-03T19:31:49.031 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 19:31:48 smithi089 bash[26845]: [03/Jun/2023:19:31:48] ENGINE Bus STARTING 2023-06-03T19:31:49.031 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 19:31:48 smithi089 bash[26845]: Warning: Permanently added '172.21.15.89' (ECDSA) to the list of known hosts. 2023-06-03T19:31:49.031 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 19:31:48 smithi089 bash[26845]: Warning: Permanently added '172.21.15.164' (ECDSA) to the list of known hosts. 2023-06-03T19:31:49.031 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 19:31:48 smithi089 bash[26845]: CherryPy Checker: 2023-06-03T19:31:49.031 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 19:31:48 smithi089 bash[26845]: The Application mounted at '' has an empty config. 2023-06-03T19:31:49.032 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 19:31:48 smithi089 bash[26845]: [03/Jun/2023:19:31:48] ENGINE Serving on http://:::9283 2023-06-03T19:31:49.032 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 19:31:48 smithi089 bash[26845]: [03/Jun/2023:19:31:48] ENGINE Bus STARTED 2023-06-03T19:31:49.408 INFO:teuthology.orchestra.run.smithi164.stdout:Scheduled node-exporter update... 2023-06-03T19:31:49.782 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:31:49 smithi089 bash[26572]: cluster 2023-06-03T19:31:48.365899+0000 mon.a (mon.0) 550 : cluster [DBG] mgrmap e16: y(active, starting, since 0.0620316s), standbys: x 2023-06-03T19:31:49.782 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:31:49 smithi089 bash[26572]: audit 2023-06-03T19:31:48.371435+0000 mon.a (mon.0) 551 : audit [DBG] from='mgr.14445 172.21.15.89:0/573502486' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-06-03T19:31:49.782 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:31:49 smithi089 bash[26572]: audit 2023-06-03T19:31:48.371708+0000 mon.a (mon.0) 552 : audit [DBG] from='mgr.14445 172.21.15.89:0/573502486' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-06-03T19:31:49.782 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:31:49 smithi089 bash[26572]: audit 2023-06-03T19:31:48.371884+0000 mon.a (mon.0) 553 : audit [DBG] from='mgr.14445 172.21.15.89:0/573502486' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-06-03T19:31:49.783 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:31:49 smithi089 bash[26572]: audit 2023-06-03T19:31:48.372090+0000 mon.a (mon.0) 554 : audit [DBG] from='mgr.14445 172.21.15.89:0/573502486' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2023-06-03T19:31:49.783 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:31:49 smithi089 bash[26572]: audit 2023-06-03T19:31:48.372284+0000 mon.a (mon.0) 555 : audit [DBG] from='mgr.14445 172.21.15.89:0/573502486' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2023-06-03T19:31:49.783 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:31:49 smithi089 bash[26572]: audit 2023-06-03T19:31:48.372493+0000 mon.a (mon.0) 556 : audit [DBG] from='mgr.14445 172.21.15.89:0/573502486' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-06-03T19:31:49.783 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:31:49 smithi089 bash[26572]: audit 2023-06-03T19:31:48.372749+0000 mon.a (mon.0) 557 : audit [DBG] from='mgr.14445 172.21.15.89:0/573502486' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-06-03T19:31:49.783 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:31:49 smithi089 bash[26572]: audit 2023-06-03T19:31:48.373024+0000 mon.a (mon.0) 558 : audit [DBG] from='mgr.14445 172.21.15.89:0/573502486' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-06-03T19:31:49.783 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:31:49 smithi089 bash[26572]: audit 2023-06-03T19:31:48.373314+0000 mon.a (mon.0) 559 : audit [DBG] from='mgr.14445 172.21.15.89:0/573502486' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-06-03T19:31:49.783 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:31:49 smithi089 bash[26572]: audit 2023-06-03T19:31:48.373601+0000 mon.a (mon.0) 560 : audit [DBG] from='mgr.14445 172.21.15.89:0/573502486' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-06-03T19:31:49.783 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:31:49 smithi089 bash[26572]: audit 2023-06-03T19:31:48.373838+0000 mon.a (mon.0) 561 : audit [DBG] from='mgr.14445 172.21.15.89:0/573502486' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-06-03T19:31:49.784 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:31:49 smithi089 bash[26572]: audit 2023-06-03T19:31:48.374096+0000 mon.a (mon.0) 562 : audit [DBG] from='mgr.14445 172.21.15.89:0/573502486' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-06-03T19:31:49.784 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:31:49 smithi089 bash[26572]: audit 2023-06-03T19:31:48.374320+0000 mon.a (mon.0) 563 : audit [DBG] from='mgr.14445 172.21.15.89:0/573502486' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-06-03T19:31:49.784 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:31:49 smithi089 bash[26572]: audit 2023-06-03T19:31:48.374571+0000 mon.a (mon.0) 564 : audit [DBG] from='mgr.14445 172.21.15.89:0/573502486' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2023-06-03T19:31:49.784 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:31:49 smithi089 bash[26572]: audit 2023-06-03T19:31:48.374734+0000 mon.a (mon.0) 565 : audit [DBG] from='mgr.14445 172.21.15.89:0/573502486' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2023-06-03T19:31:49.784 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:31:49 smithi089 bash[26572]: audit 2023-06-03T19:31:48.375218+0000 mon.a (mon.0) 566 : audit [DBG] from='mgr.14445 172.21.15.89:0/573502486' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2023-06-03T19:31:49.784 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:31:49 smithi089 bash[26572]: cluster 2023-06-03T19:31:48.404625+0000 mon.a (mon.0) 567 : cluster [INF] Manager daemon y is now available 2023-06-03T19:31:49.784 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:31:49 smithi089 bash[26572]: cluster 2023-06-03T19:31:48.546997+0000 mon.a (mon.0) 568 : cluster [DBG] Standby manager daemon x restarted 2023-06-03T19:31:49.785 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:31:49 smithi089 bash[33173]: cluster 2023-06-03T19:31:48.365899+0000 mon.a (mon.0) 550 : cluster [DBG] mgrmap e16: y(active, starting, since 0.0620316s), standbys: x 2023-06-03T19:31:49.785 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:31:49 smithi089 bash[33173]: audit 2023-06-03T19:31:48.371435+0000 mon.a (mon.0) 551 : audit [DBG] from='mgr.14445 172.21.15.89:0/573502486' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-06-03T19:31:49.785 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:31:49 smithi089 bash[33173]: audit 2023-06-03T19:31:48.371708+0000 mon.a (mon.0) 552 : audit [DBG] from='mgr.14445 172.21.15.89:0/573502486' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-06-03T19:31:49.785 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:31:49 smithi089 bash[33173]: audit 2023-06-03T19:31:48.371884+0000 mon.a (mon.0) 553 : audit [DBG] from='mgr.14445 172.21.15.89:0/573502486' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-06-03T19:31:49.785 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:31:49 smithi089 bash[33173]: audit 2023-06-03T19:31:48.372090+0000 mon.a (mon.0) 554 : audit [DBG] from='mgr.14445 172.21.15.89:0/573502486' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2023-06-03T19:31:49.786 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:31:49 smithi089 bash[33173]: audit 2023-06-03T19:31:48.372284+0000 mon.a (mon.0) 555 : audit [DBG] from='mgr.14445 172.21.15.89:0/573502486' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2023-06-03T19:31:49.786 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:31:49 smithi089 bash[33173]: audit 2023-06-03T19:31:48.372493+0000 mon.a (mon.0) 556 : audit [DBG] from='mgr.14445 172.21.15.89:0/573502486' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-06-03T19:31:49.786 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:31:49 smithi089 bash[33173]: audit 2023-06-03T19:31:48.372749+0000 mon.a (mon.0) 557 : audit [DBG] from='mgr.14445 172.21.15.89:0/573502486' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-06-03T19:31:49.786 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:31:49 smithi089 bash[33173]: audit 2023-06-03T19:31:48.373024+0000 mon.a (mon.0) 558 : audit [DBG] from='mgr.14445 172.21.15.89:0/573502486' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-06-03T19:31:49.786 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:31:49 smithi089 bash[33173]: audit 2023-06-03T19:31:48.373314+0000 mon.a (mon.0) 559 : audit [DBG] from='mgr.14445 172.21.15.89:0/573502486' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-06-03T19:31:49.786 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:31:49 smithi089 bash[33173]: audit 2023-06-03T19:31:48.373601+0000 mon.a (mon.0) 560 : audit [DBG] from='mgr.14445 172.21.15.89:0/573502486' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-06-03T19:31:49.786 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:31:49 smithi089 bash[33173]: audit 2023-06-03T19:31:48.373838+0000 mon.a (mon.0) 561 : audit [DBG] from='mgr.14445 172.21.15.89:0/573502486' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-06-03T19:31:49.787 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:31:49 smithi089 bash[33173]: audit 2023-06-03T19:31:48.374096+0000 mon.a (mon.0) 562 : audit [DBG] from='mgr.14445 172.21.15.89:0/573502486' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-06-03T19:31:49.787 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:31:49 smithi089 bash[33173]: audit 2023-06-03T19:31:48.374320+0000 mon.a (mon.0) 563 : audit [DBG] from='mgr.14445 172.21.15.89:0/573502486' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-06-03T19:31:49.787 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:31:49 smithi089 bash[33173]: audit 2023-06-03T19:31:48.374571+0000 mon.a (mon.0) 564 : audit [DBG] from='mgr.14445 172.21.15.89:0/573502486' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2023-06-03T19:31:49.787 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:31:49 smithi089 bash[33173]: audit 2023-06-03T19:31:48.374734+0000 mon.a (mon.0) 565 : audit [DBG] from='mgr.14445 172.21.15.89:0/573502486' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2023-06-03T19:31:49.787 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:31:49 smithi089 bash[33173]: audit 2023-06-03T19:31:48.375218+0000 mon.a (mon.0) 566 : audit [DBG] from='mgr.14445 172.21.15.89:0/573502486' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2023-06-03T19:31:49.787 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:31:49 smithi089 bash[33173]: cluster 2023-06-03T19:31:48.404625+0000 mon.a (mon.0) 567 : cluster [INF] Manager daemon y is now available 2023-06-03T19:31:49.787 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:31:49 smithi089 bash[33173]: cluster 2023-06-03T19:31:48.546997+0000 mon.a (mon.0) 568 : cluster [DBG] Standby manager daemon x restarted 2023-06-03T19:31:49.788 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:31:49 smithi089 bash[26572]: cluster 2023-06-03T19:31:48.547206+0000 mon.a (mon.0) 569 : cluster [DBG] Standby manager daemon x started 2023-06-03T19:31:49.788 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:31:49 smithi089 bash[26572]: audit 2023-06-03T19:31:48.551005+0000 mon.b (mon.1) 19 : audit [DBG] from='mgr.? 172.21.15.164:0/3668496442' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-06-03T19:31:49.788 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:31:49 smithi089 bash[26572]: audit 2023-06-03T19:31:48.551022+0000 mon.a (mon.0) 570 : audit [DBG] from='mgr.14445 172.21.15.89:0/573502486' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-03T19:31:49.788 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:31:49 smithi089 bash[26572]: audit 2023-06-03T19:31:48.552894+0000 mon.b (mon.1) 20 : audit [DBG] from='mgr.? 172.21.15.164:0/3668496442' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-06-03T19:31:49.788 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:31:49 smithi089 bash[26572]: audit 2023-06-03T19:31:48.556081+0000 mon.a (mon.0) 571 : audit [DBG] from='mgr.14445 172.21.15.89:0/573502486' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-03T19:31:49.788 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:31:49 smithi089 bash[26572]: audit 2023-06-03T19:31:48.557381+0000 mon.b (mon.1) 21 : audit [DBG] from='mgr.? 172.21.15.164:0/3668496442' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-06-03T19:31:49.789 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:31:49 smithi089 bash[26572]: audit 2023-06-03T19:31:48.557962+0000 mon.b (mon.1) 22 : audit [DBG] from='mgr.? 172.21.15.164:0/3668496442' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-06-03T19:31:49.789 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:31:49 smithi089 bash[26572]: audit 2023-06-03T19:31:48.584974+0000 mon.a (mon.0) 572 : audit [INF] from='mgr.14445 172.21.15.89:0/573502486' entity='mgr.y' 2023-06-03T19:31:49.789 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:31:49 smithi089 bash[26572]: audit 2023-06-03T19:31:48.625019+0000 mon.a (mon.0) 573 : audit [INF] from='mgr.14445 172.21.15.89:0/573502486' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2023-06-03T19:31:49.789 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:31:49 smithi089 bash[26572]: audit 2023-06-03T19:31:48.632328+0000 mon.a (mon.0) 574 : audit [INF] from='mgr.14445 172.21.15.89:0/573502486' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2023-06-03T19:31:49.789 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:31:49 smithi089 bash[33173]: cluster 2023-06-03T19:31:48.547206+0000 mon.a (mon.0) 569 : cluster [DBG] Standby manager daemon x started 2023-06-03T19:31:49.789 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:31:49 smithi089 bash[33173]: audit 2023-06-03T19:31:48.551005+0000 mon.b (mon.1) 19 : audit [DBG] from='mgr.? 172.21.15.164:0/3668496442' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-06-03T19:31:49.789 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:31:49 smithi089 bash[33173]: audit 2023-06-03T19:31:48.551022+0000 mon.a (mon.0) 570 : audit [DBG] from='mgr.14445 172.21.15.89:0/573502486' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-03T19:31:49.790 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:31:49 smithi089 bash[33173]: audit 2023-06-03T19:31:48.552894+0000 mon.b (mon.1) 20 : audit [DBG] from='mgr.? 172.21.15.164:0/3668496442' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-06-03T19:31:49.790 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:31:49 smithi089 bash[33173]: audit 2023-06-03T19:31:48.556081+0000 mon.a (mon.0) 571 : audit [DBG] from='mgr.14445 172.21.15.89:0/573502486' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-03T19:31:49.790 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:31:49 smithi089 bash[33173]: audit 2023-06-03T19:31:48.557381+0000 mon.b (mon.1) 21 : audit [DBG] from='mgr.? 172.21.15.164:0/3668496442' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-06-03T19:31:49.790 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:31:49 smithi089 bash[33173]: audit 2023-06-03T19:31:48.557962+0000 mon.b (mon.1) 22 : audit [DBG] from='mgr.? 172.21.15.164:0/3668496442' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-06-03T19:31:49.790 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:31:49 smithi089 bash[33173]: audit 2023-06-03T19:31:48.584974+0000 mon.a (mon.0) 572 : audit [INF] from='mgr.14445 172.21.15.89:0/573502486' entity='mgr.y' 2023-06-03T19:31:49.790 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:31:49 smithi089 bash[33173]: audit 2023-06-03T19:31:48.625019+0000 mon.a (mon.0) 573 : audit [INF] from='mgr.14445 172.21.15.89:0/573502486' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2023-06-03T19:31:49.790 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:31:49 smithi089 bash[33173]: audit 2023-06-03T19:31:48.632328+0000 mon.a (mon.0) 574 : audit [INF] from='mgr.14445 172.21.15.89:0/573502486' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2023-06-03T19:31:49.836 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:31:49 smithi164 bash[26855]: cluster 2023-06-03T19:31:48.365899+0000 mon.a (mon.0) 550 : cluster [DBG] mgrmap e16: y(active, starting, since 0.0620316s), standbys: x 2023-06-03T19:31:49.836 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:31:49 smithi164 bash[26855]: audit 2023-06-03T19:31:48.371435+0000 mon.a (mon.0) 551 : audit [DBG] from='mgr.14445 172.21.15.89:0/573502486' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-06-03T19:31:49.837 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:31:49 smithi164 bash[26855]: audit 2023-06-03T19:31:48.371708+0000 mon.a (mon.0) 552 : audit [DBG] from='mgr.14445 172.21.15.89:0/573502486' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-06-03T19:31:49.837 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:31:49 smithi164 bash[26855]: audit 2023-06-03T19:31:48.371884+0000 mon.a (mon.0) 553 : audit [DBG] from='mgr.14445 172.21.15.89:0/573502486' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-06-03T19:31:49.837 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:31:49 smithi164 bash[26855]: audit 2023-06-03T19:31:48.372090+0000 mon.a (mon.0) 554 : audit [DBG] from='mgr.14445 172.21.15.89:0/573502486' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2023-06-03T19:31:49.837 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:31:49 smithi164 bash[26855]: audit 2023-06-03T19:31:48.372284+0000 mon.a (mon.0) 555 : audit [DBG] from='mgr.14445 172.21.15.89:0/573502486' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2023-06-03T19:31:49.837 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:31:49 smithi164 bash[26855]: audit 2023-06-03T19:31:48.372493+0000 mon.a (mon.0) 556 : audit [DBG] from='mgr.14445 172.21.15.89:0/573502486' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-06-03T19:31:49.838 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:31:49 smithi164 bash[26855]: audit 2023-06-03T19:31:48.372749+0000 mon.a (mon.0) 557 : audit [DBG] from='mgr.14445 172.21.15.89:0/573502486' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-06-03T19:31:49.838 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:31:49 smithi164 bash[26855]: audit 2023-06-03T19:31:48.373024+0000 mon.a (mon.0) 558 : audit [DBG] from='mgr.14445 172.21.15.89:0/573502486' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-06-03T19:31:49.838 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:31:49 smithi164 bash[26855]: audit 2023-06-03T19:31:48.373314+0000 mon.a (mon.0) 559 : audit [DBG] from='mgr.14445 172.21.15.89:0/573502486' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-06-03T19:31:49.838 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:31:49 smithi164 bash[26855]: audit 2023-06-03T19:31:48.373601+0000 mon.a (mon.0) 560 : audit [DBG] from='mgr.14445 172.21.15.89:0/573502486' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-06-03T19:31:49.838 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:31:49 smithi164 bash[26855]: audit 2023-06-03T19:31:48.373838+0000 mon.a (mon.0) 561 : audit [DBG] from='mgr.14445 172.21.15.89:0/573502486' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-06-03T19:31:49.839 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:31:49 smithi164 bash[26855]: audit 2023-06-03T19:31:48.374096+0000 mon.a (mon.0) 562 : audit [DBG] from='mgr.14445 172.21.15.89:0/573502486' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-06-03T19:31:49.839 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:31:49 smithi164 bash[26855]: audit 2023-06-03T19:31:48.374320+0000 mon.a (mon.0) 563 : audit [DBG] from='mgr.14445 172.21.15.89:0/573502486' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-06-03T19:31:49.839 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:31:49 smithi164 bash[26855]: audit 2023-06-03T19:31:48.374571+0000 mon.a (mon.0) 564 : audit [DBG] from='mgr.14445 172.21.15.89:0/573502486' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2023-06-03T19:31:49.839 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:31:49 smithi164 bash[26855]: audit 2023-06-03T19:31:48.374734+0000 mon.a (mon.0) 565 : audit [DBG] from='mgr.14445 172.21.15.89:0/573502486' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2023-06-03T19:31:49.839 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:31:49 smithi164 bash[26855]: audit 2023-06-03T19:31:48.375218+0000 mon.a (mon.0) 566 : audit [DBG] from='mgr.14445 172.21.15.89:0/573502486' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2023-06-03T19:31:49.839 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:31:49 smithi164 bash[26855]: cluster 2023-06-03T19:31:48.404625+0000 mon.a (mon.0) 567 : cluster [INF] Manager daemon y is now available 2023-06-03T19:31:49.839 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:31:49 smithi164 bash[26855]: cluster 2023-06-03T19:31:48.546997+0000 mon.a (mon.0) 568 : cluster [DBG] Standby manager daemon x restarted 2023-06-03T19:31:49.839 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:31:49 smithi164 bash[26855]: cluster 2023-06-03T19:31:48.547206+0000 mon.a (mon.0) 569 : cluster [DBG] Standby manager daemon x started 2023-06-03T19:31:49.840 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:31:49 smithi164 bash[26855]: audit 2023-06-03T19:31:48.551005+0000 mon.b (mon.1) 19 : audit [DBG] from='mgr.? 172.21.15.164:0/3668496442' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-06-03T19:31:49.840 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:31:49 smithi164 bash[26855]: audit 2023-06-03T19:31:48.551022+0000 mon.a (mon.0) 570 : audit [DBG] from='mgr.14445 172.21.15.89:0/573502486' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-03T19:31:49.840 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:31:49 smithi164 bash[26855]: audit 2023-06-03T19:31:48.552894+0000 mon.b (mon.1) 20 : audit [DBG] from='mgr.? 172.21.15.164:0/3668496442' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-06-03T19:31:49.840 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:31:49 smithi164 bash[26855]: audit 2023-06-03T19:31:48.556081+0000 mon.a (mon.0) 571 : audit [DBG] from='mgr.14445 172.21.15.89:0/573502486' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-03T19:31:49.840 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:31:49 smithi164 bash[26855]: audit 2023-06-03T19:31:48.557381+0000 mon.b (mon.1) 21 : audit [DBG] from='mgr.? 172.21.15.164:0/3668496442' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-06-03T19:31:49.840 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:31:49 smithi164 bash[26855]: audit 2023-06-03T19:31:48.557962+0000 mon.b (mon.1) 22 : audit [DBG] from='mgr.? 172.21.15.164:0/3668496442' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-06-03T19:31:49.840 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:31:49 smithi164 bash[26855]: audit 2023-06-03T19:31:48.584974+0000 mon.a (mon.0) 572 : audit [INF] from='mgr.14445 172.21.15.89:0/573502486' entity='mgr.y' 2023-06-03T19:31:49.841 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:31:49 smithi164 bash[26855]: audit 2023-06-03T19:31:48.625019+0000 mon.a (mon.0) 573 : audit [INF] from='mgr.14445 172.21.15.89:0/573502486' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2023-06-03T19:31:49.841 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:31:49 smithi164 bash[26855]: audit 2023-06-03T19:31:48.632328+0000 mon.a (mon.0) 574 : audit [INF] from='mgr.14445 172.21.15.89:0/573502486' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2023-06-03T19:31:50.203 DEBUG:teuthology.orchestra.run.smithi089:node-exporter.a> sudo journalctl -f -n 0 -u ceph-f1db3478-0243-11ee-9b1d-001a4aab830c@node-exporter.a.service 2023-06-03T19:31:50.206 DEBUG:teuthology.orchestra.run.smithi164:node-exporter.b> sudo journalctl -f -n 0 -u ceph-f1db3478-0243-11ee-9b1d-001a4aab830c@node-exporter.b.service 2023-06-03T19:31:50.209 INFO:tasks.cephadm:Adding alertmanager.a on smithi089 2023-06-03T19:31:50.209 DEBUG:teuthology.orchestra.run.smithi164:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph orch apply alertmanager '1;smithi089=a' 2023-06-03T19:31:50.264 INFO:journalctl@ceph.node-exporter.a.smithi089.stdout:-- Logs begin at Sun 2022-08-14 12:20:24 UTC. -- 2023-06-03T19:31:50.428 INFO:journalctl@ceph.node-exporter.b.smithi164.stdout:-- Logs begin at Sun 2022-08-14 12:20:24 UTC. -- 2023-06-03T19:31:50.531 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:31:50 smithi089 bash[26572]: cluster 2023-06-03T19:31:49.388147+0000 mon.a (mon.0) 575 : cluster [DBG] mgrmap e17: y(active, since 1.08428s), standbys: x 2023-06-03T19:31:50.532 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:31:50 smithi089 bash[26572]: cephadm 2023-06-03T19:31:49.397223+0000 mgr.y (mgr.14445) 3 : cephadm [INF] Saving service node-exporter spec with placement smithi089=a;smithi164=b;count:2 2023-06-03T19:31:50.532 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:31:50 smithi089 bash[26572]: audit 2023-06-03T19:31:49.401417+0000 mon.a (mon.0) 576 : audit [INF] from='mgr.14445 172.21.15.89:0/573502486' entity='mgr.y' 2023-06-03T19:31:50.532 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:31:50 smithi089 bash[33173]: cluster 2023-06-03T19:31:49.388147+0000 mon.a (mon.0) 575 : cluster [DBG] mgrmap e17: y(active, since 1.08428s), standbys: x 2023-06-03T19:31:50.533 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:31:50 smithi089 bash[33173]: cephadm 2023-06-03T19:31:49.397223+0000 mgr.y (mgr.14445) 3 : cephadm [INF] Saving service node-exporter spec with placement smithi089=a;smithi164=b;count:2 2023-06-03T19:31:50.533 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:31:50 smithi089 bash[33173]: audit 2023-06-03T19:31:49.401417+0000 mon.a (mon.0) 576 : audit [INF] from='mgr.14445 172.21.15.89:0/573502486' entity='mgr.y' 2023-06-03T19:31:50.586 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:31:50 smithi164 bash[26855]: cluster 2023-06-03T19:31:49.388147+0000 mon.a (mon.0) 575 : cluster [DBG] mgrmap e17: y(active, since 1.08428s), standbys: x 2023-06-03T19:31:50.586 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:31:50 smithi164 bash[26855]: cephadm 2023-06-03T19:31:49.397223+0000 mgr.y (mgr.14445) 3 : cephadm [INF] Saving service node-exporter spec with placement smithi089=a;smithi164=b;count:2 2023-06-03T19:31:50.586 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:31:50 smithi164 bash[26855]: audit 2023-06-03T19:31:49.401417+0000 mon.a (mon.0) 576 : audit [INF] from='mgr.14445 172.21.15.89:0/573502486' entity='mgr.y' 2023-06-03T19:31:51.705 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:31:51 smithi164 bash[26855]: cluster 2023-06-03T19:31:50.372093+0000 mgr.y (mgr.14445) 4 : cluster [DBG] pgmap v4: 1 pgs: 1 active+clean; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:31:51.706 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:31:51 smithi164 bash[26855]: cluster 2023-06-03T19:31:50.410653+0000 mon.a (mon.0) 577 : cluster [DBG] mgrmap e18: y(active, since 2s), standbys: x 2023-06-03T19:31:51.781 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:31:51 smithi089 bash[26572]: cluster 2023-06-03T19:31:50.372093+0000 mgr.y (mgr.14445) 4 : cluster [DBG] pgmap v4: 1 pgs: 1 active+clean; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:31:51.781 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:31:51 smithi089 bash[26572]: cluster 2023-06-03T19:31:50.410653+0000 mon.a (mon.0) 577 : cluster [DBG] mgrmap e18: y(active, since 2s), standbys: x 2023-06-03T19:31:51.781 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:31:51 smithi089 bash[33173]: cluster 2023-06-03T19:31:50.372093+0000 mgr.y (mgr.14445) 4 : cluster [DBG] pgmap v4: 1 pgs: 1 active+clean; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:31:51.782 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:31:51 smithi089 bash[33173]: cluster 2023-06-03T19:31:50.410653+0000 mon.a (mon.0) 577 : cluster [DBG] mgrmap e18: y(active, since 2s), standbys: x 2023-06-03T19:31:53.613 INFO:teuthology.orchestra.run.smithi164.stdout:Scheduled alertmanager update... 2023-06-03T19:31:53.781 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:31:53 smithi089 bash[26572]: cluster 2023-06-03T19:31:52.372673+0000 mgr.y (mgr.14445) 5 : cluster [DBG] pgmap v5: 1 pgs: 1 active+clean; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:31:53.781 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:31:53 smithi089 bash[33173]: cluster 2023-06-03T19:31:52.372673+0000 mgr.y (mgr.14445) 5 : cluster [DBG] pgmap v5: 1 pgs: 1 active+clean; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:31:53.836 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:31:53 smithi164 bash[26855]: cluster 2023-06-03T19:31:52.372673+0000 mgr.y (mgr.14445) 5 : cluster [DBG] pgmap v5: 1 pgs: 1 active+clean; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:31:54.413 DEBUG:teuthology.orchestra.run.smithi089:alertmanager.a> sudo journalctl -f -n 0 -u ceph-f1db3478-0243-11ee-9b1d-001a4aab830c@alertmanager.a.service 2023-06-03T19:31:54.416 INFO:tasks.cephadm:Adding grafana.a on smithi164 2023-06-03T19:31:54.416 DEBUG:teuthology.orchestra.run.smithi164:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph orch apply grafana '1;smithi164=a' 2023-06-03T19:31:54.660 INFO:journalctl@ceph.alertmanager.a.smithi089.stdout:-- Logs begin at Sun 2022-08-14 12:20:24 UTC. -- 2023-06-03T19:31:54.781 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:31:54 smithi089 bash[26572]: audit 2023-06-03T19:31:53.596153+0000 mgr.y (mgr.14445) 6 : audit [DBG] from='client.24385 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "alertmanager", "placement": "1;smithi089=a", "target": ["mon-mgr", ""]}]: dispatch 2023-06-03T19:31:54.781 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:31:54 smithi089 bash[26572]: cephadm 2023-06-03T19:31:53.599305+0000 mgr.y (mgr.14445) 7 : cephadm [INF] Saving service alertmanager spec with placement smithi089=a;count:1 2023-06-03T19:31:54.782 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:31:54 smithi089 bash[26572]: audit 2023-06-03T19:31:53.605641+0000 mon.a (mon.0) 578 : audit [INF] from='mgr.14445 172.21.15.89:0/573502486' entity='mgr.y' 2023-06-03T19:31:54.782 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:31:54 smithi089 bash[26572]: audit 2023-06-03T19:31:54.212998+0000 mon.a (mon.0) 579 : audit [INF] from='mgr.14445 172.21.15.89:0/573502486' entity='mgr.y' 2023-06-03T19:31:54.782 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:31:54 smithi089 bash[26572]: audit 2023-06-03T19:31:54.277773+0000 mon.a (mon.0) 580 : audit [INF] from='mgr.14445 172.21.15.89:0/573502486' entity='mgr.y' 2023-06-03T19:31:54.782 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:31:54 smithi089 bash[26572]: audit 2023-06-03T19:31:54.536949+0000 mon.a (mon.0) 581 : audit [DBG] from='mgr.14445 172.21.15.89:0/573502486' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-03T19:31:54.782 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:31:54 smithi089 bash[26572]: audit 2023-06-03T19:31:54.538346+0000 mon.a (mon.0) 582 : audit [INF] from='mgr.14445 172.21.15.89:0/573502486' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-03T19:31:54.782 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:31:54 smithi089 bash[26572]: audit 2023-06-03T19:31:54.597191+0000 mon.a (mon.0) 583 : audit [INF] from='mgr.14445 172.21.15.89:0/573502486' entity='mgr.y' 2023-06-03T19:31:54.782 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:31:54 smithi089 bash[26572]: audit 2023-06-03T19:31:54.601850+0000 mon.a (mon.0) 584 : audit [INF] from='mgr.14445 172.21.15.89:0/573502486' entity='mgr.y' 2023-06-03T19:31:54.783 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:31:54 smithi089 bash[33173]: audit 2023-06-03T19:31:53.596153+0000 mgr.y (mgr.14445) 6 : audit [DBG] from='client.24385 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "alertmanager", "placement": "1;smithi089=a", "target": ["mon-mgr", ""]}]: dispatch 2023-06-03T19:31:54.783 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:31:54 smithi089 bash[33173]: cephadm 2023-06-03T19:31:53.599305+0000 mgr.y (mgr.14445) 7 : cephadm [INF] Saving service alertmanager spec with placement smithi089=a;count:1 2023-06-03T19:31:54.783 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:31:54 smithi089 bash[33173]: audit 2023-06-03T19:31:53.605641+0000 mon.a (mon.0) 578 : audit [INF] from='mgr.14445 172.21.15.89:0/573502486' entity='mgr.y' 2023-06-03T19:31:54.783 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:31:54 smithi089 bash[33173]: audit 2023-06-03T19:31:54.212998+0000 mon.a (mon.0) 579 : audit [INF] from='mgr.14445 172.21.15.89:0/573502486' entity='mgr.y' 2023-06-03T19:31:54.783 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:31:54 smithi089 bash[33173]: audit 2023-06-03T19:31:54.277773+0000 mon.a (mon.0) 580 : audit [INF] from='mgr.14445 172.21.15.89:0/573502486' entity='mgr.y' 2023-06-03T19:31:54.783 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:31:54 smithi089 bash[33173]: audit 2023-06-03T19:31:54.536949+0000 mon.a (mon.0) 581 : audit [DBG] from='mgr.14445 172.21.15.89:0/573502486' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-03T19:31:54.783 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:31:54 smithi089 bash[33173]: audit 2023-06-03T19:31:54.538346+0000 mon.a (mon.0) 582 : audit [INF] from='mgr.14445 172.21.15.89:0/573502486' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-03T19:31:54.784 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:31:54 smithi089 bash[33173]: audit 2023-06-03T19:31:54.597191+0000 mon.a (mon.0) 583 : audit [INF] from='mgr.14445 172.21.15.89:0/573502486' entity='mgr.y' 2023-06-03T19:31:54.784 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:31:54 smithi089 bash[33173]: audit 2023-06-03T19:31:54.601850+0000 mon.a (mon.0) 584 : audit [INF] from='mgr.14445 172.21.15.89:0/573502486' entity='mgr.y' 2023-06-03T19:31:54.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:31:54 smithi164 bash[26855]: audit 2023-06-03T19:31:53.596153+0000 mgr.y (mgr.14445) 6 : audit [DBG] from='client.24385 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "alertmanager", "placement": "1;smithi089=a", "target": ["mon-mgr", ""]}]: dispatch 2023-06-03T19:31:54.836 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:31:54 smithi164 bash[26855]: cephadm 2023-06-03T19:31:53.599305+0000 mgr.y (mgr.14445) 7 : cephadm [INF] Saving service alertmanager spec with placement smithi089=a;count:1 2023-06-03T19:31:54.836 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:31:54 smithi164 bash[26855]: audit 2023-06-03T19:31:53.605641+0000 mon.a (mon.0) 578 : audit [INF] from='mgr.14445 172.21.15.89:0/573502486' entity='mgr.y' 2023-06-03T19:31:54.836 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:31:54 smithi164 bash[26855]: audit 2023-06-03T19:31:54.212998+0000 mon.a (mon.0) 579 : audit [INF] from='mgr.14445 172.21.15.89:0/573502486' entity='mgr.y' 2023-06-03T19:31:54.836 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:31:54 smithi164 bash[26855]: audit 2023-06-03T19:31:54.277773+0000 mon.a (mon.0) 580 : audit [INF] from='mgr.14445 172.21.15.89:0/573502486' entity='mgr.y' 2023-06-03T19:31:54.836 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:31:54 smithi164 bash[26855]: audit 2023-06-03T19:31:54.536949+0000 mon.a (mon.0) 581 : audit [DBG] from='mgr.14445 172.21.15.89:0/573502486' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-03T19:31:54.837 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:31:54 smithi164 bash[26855]: audit 2023-06-03T19:31:54.538346+0000 mon.a (mon.0) 582 : audit [INF] from='mgr.14445 172.21.15.89:0/573502486' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-03T19:31:54.837 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:31:54 smithi164 bash[26855]: audit 2023-06-03T19:31:54.597191+0000 mon.a (mon.0) 583 : audit [INF] from='mgr.14445 172.21.15.89:0/573502486' entity='mgr.y' 2023-06-03T19:31:54.837 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:31:54 smithi164 bash[26855]: audit 2023-06-03T19:31:54.601850+0000 mon.a (mon.0) 584 : audit [INF] from='mgr.14445 172.21.15.89:0/573502486' entity='mgr.y' 2023-06-03T19:31:55.890 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:31:55 smithi164 bash[26855]: cluster 2023-06-03T19:31:54.373193+0000 mgr.y (mgr.14445) 8 : cluster [DBG] pgmap v6: 1 pgs: 1 active+clean; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:31:55.890 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:31:55 smithi164 bash[26855]: cephadm 2023-06-03T19:31:54.540120+0000 mgr.y (mgr.14445) 9 : cephadm [INF] Updating smithi089:/etc/ceph/ceph.conf 2023-06-03T19:31:55.890 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:31:55 smithi164 bash[26855]: cephadm 2023-06-03T19:31:54.540835+0000 mgr.y (mgr.14445) 10 : cephadm [INF] Updating smithi164:/etc/ceph/ceph.conf 2023-06-03T19:31:55.890 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:31:55 smithi164 bash[26855]: cephadm 2023-06-03T19:31:54.560967+0000 mgr.y (mgr.14445) 11 : cephadm [INF] Updating smithi089:/etc/ceph/ceph.client.admin.keyring 2023-06-03T19:31:55.890 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:31:55 smithi164 bash[26855]: cephadm 2023-06-03T19:31:54.565207+0000 mgr.y (mgr.14445) 12 : cephadm [INF] Updating smithi164:/etc/ceph/ceph.client.admin.keyring 2023-06-03T19:31:55.890 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:31:55 smithi164 bash[26855]: audit 2023-06-03T19:31:54.613827+0000 mon.a (mon.0) 585 : audit [INF] from='mgr.14445 172.21.15.89:0/573502486' entity='mgr.y' 2023-06-03T19:31:55.891 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:31:55 smithi164 bash[26855]: cephadm 2023-06-03T19:31:54.635297+0000 mgr.y (mgr.14445) 13 : cephadm [INF] Deploying daemon prometheus.a on smithi164 2023-06-03T19:31:56.031 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:31:55 smithi089 bash[26572]: cluster 2023-06-03T19:31:54.373193+0000 mgr.y (mgr.14445) 8 : cluster [DBG] pgmap v6: 1 pgs: 1 active+clean; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:31:56.031 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:31:55 smithi089 bash[26572]: cephadm 2023-06-03T19:31:54.540120+0000 mgr.y (mgr.14445) 9 : cephadm [INF] Updating smithi089:/etc/ceph/ceph.conf 2023-06-03T19:31:56.031 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:31:55 smithi089 bash[26572]: cephadm 2023-06-03T19:31:54.540835+0000 mgr.y (mgr.14445) 10 : cephadm [INF] Updating smithi164:/etc/ceph/ceph.conf 2023-06-03T19:31:56.031 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:31:55 smithi089 bash[26572]: cephadm 2023-06-03T19:31:54.560967+0000 mgr.y (mgr.14445) 11 : cephadm [INF] Updating smithi089:/etc/ceph/ceph.client.admin.keyring 2023-06-03T19:31:56.032 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:31:55 smithi089 bash[26572]: cephadm 2023-06-03T19:31:54.565207+0000 mgr.y (mgr.14445) 12 : cephadm [INF] Updating smithi164:/etc/ceph/ceph.client.admin.keyring 2023-06-03T19:31:56.032 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:31:55 smithi089 bash[26572]: audit 2023-06-03T19:31:54.613827+0000 mon.a (mon.0) 585 : audit [INF] from='mgr.14445 172.21.15.89:0/573502486' entity='mgr.y' 2023-06-03T19:31:56.032 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:31:55 smithi089 bash[26572]: cephadm 2023-06-03T19:31:54.635297+0000 mgr.y (mgr.14445) 13 : cephadm [INF] Deploying daemon prometheus.a on smithi164 2023-06-03T19:31:56.032 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:31:55 smithi089 bash[33173]: cluster 2023-06-03T19:31:54.373193+0000 mgr.y (mgr.14445) 8 : cluster [DBG] pgmap v6: 1 pgs: 1 active+clean; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:31:56.032 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:31:55 smithi089 bash[33173]: cephadm 2023-06-03T19:31:54.540120+0000 mgr.y (mgr.14445) 9 : cephadm [INF] Updating smithi089:/etc/ceph/ceph.conf 2023-06-03T19:31:56.033 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:31:55 smithi089 bash[33173]: cephadm 2023-06-03T19:31:54.540835+0000 mgr.y (mgr.14445) 10 : cephadm [INF] Updating smithi164:/etc/ceph/ceph.conf 2023-06-03T19:31:56.033 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:31:55 smithi089 bash[33173]: cephadm 2023-06-03T19:31:54.560967+0000 mgr.y (mgr.14445) 11 : cephadm [INF] Updating smithi089:/etc/ceph/ceph.client.admin.keyring 2023-06-03T19:31:56.033 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:31:55 smithi089 bash[33173]: cephadm 2023-06-03T19:31:54.565207+0000 mgr.y (mgr.14445) 12 : cephadm [INF] Updating smithi164:/etc/ceph/ceph.client.admin.keyring 2023-06-03T19:31:56.033 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:31:55 smithi089 bash[33173]: audit 2023-06-03T19:31:54.613827+0000 mon.a (mon.0) 585 : audit [INF] from='mgr.14445 172.21.15.89:0/573502486' entity='mgr.y' 2023-06-03T19:31:56.033 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:31:55 smithi089 bash[33173]: cephadm 2023-06-03T19:31:54.635297+0000 mgr.y (mgr.14445) 13 : cephadm [INF] Deploying daemon prometheus.a on smithi164 2023-06-03T19:31:58.031 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:31:57 smithi089 bash[26572]: cluster 2023-06-03T19:31:56.373823+0000 mgr.y (mgr.14445) 14 : cluster [DBG] pgmap v7: 1 pgs: 1 active+clean; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:31:58.031 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:31:57 smithi089 bash[33173]: cluster 2023-06-03T19:31:56.373823+0000 mgr.y (mgr.14445) 14 : cluster [DBG] pgmap v7: 1 pgs: 1 active+clean; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:31:58.086 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:31:57 smithi164 bash[26855]: cluster 2023-06-03T19:31:56.373823+0000 mgr.y (mgr.14445) 14 : cluster [DBG] pgmap v7: 1 pgs: 1 active+clean; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:32:00.031 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:31:59 smithi089 bash[26572]: cluster 2023-06-03T19:31:58.374394+0000 mgr.y (mgr.14445) 15 : cluster [DBG] pgmap v8: 1 pgs: 1 active+clean; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:32:00.031 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:31:59 smithi089 bash[33173]: cluster 2023-06-03T19:31:58.374394+0000 mgr.y (mgr.14445) 15 : cluster [DBG] pgmap v8: 1 pgs: 1 active+clean; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:32:00.085 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:31:59 smithi164 bash[26855]: cluster 2023-06-03T19:31:58.374394+0000 mgr.y (mgr.14445) 15 : cluster [DBG] pgmap v8: 1 pgs: 1 active+clean; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:32:00.216 INFO:teuthology.orchestra.run.smithi164.stdout:Scheduled grafana update... 2023-06-03T19:32:01.531 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:32:01 smithi089 bash[26572]: audit 2023-06-03T19:32:00.199664+0000 mgr.y (mgr.14445) 16 : audit [DBG] from='client.24391 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "grafana", "placement": "1;smithi164=a", "target": ["mon-mgr", ""]}]: dispatch 2023-06-03T19:32:01.531 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:32:01 smithi089 bash[26572]: cephadm 2023-06-03T19:32:00.203027+0000 mgr.y (mgr.14445) 17 : cephadm [INF] Saving service grafana spec with placement smithi164=a;count:1 2023-06-03T19:32:01.532 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:32:01 smithi089 bash[26572]: audit 2023-06-03T19:32:00.209045+0000 mon.a (mon.0) 586 : audit [INF] from='mgr.14445 172.21.15.89:0/573502486' entity='mgr.y' 2023-06-03T19:32:01.532 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:32:01 smithi089 bash[26572]: cluster 2023-06-03T19:32:00.375052+0000 mgr.y (mgr.14445) 18 : cluster [DBG] pgmap v9: 1 pgs: 1 active+clean; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:32:01.532 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:32:01 smithi089 bash[33173]: audit 2023-06-03T19:32:00.199664+0000 mgr.y (mgr.14445) 16 : audit [DBG] from='client.24391 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "grafana", "placement": "1;smithi164=a", "target": ["mon-mgr", ""]}]: dispatch 2023-06-03T19:32:01.532 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:32:01 smithi089 bash[33173]: cephadm 2023-06-03T19:32:00.203027+0000 mgr.y (mgr.14445) 17 : cephadm [INF] Saving service grafana spec with placement smithi164=a;count:1 2023-06-03T19:32:01.532 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:32:01 smithi089 bash[33173]: audit 2023-06-03T19:32:00.209045+0000 mon.a (mon.0) 586 : audit [INF] from='mgr.14445 172.21.15.89:0/573502486' entity='mgr.y' 2023-06-03T19:32:01.533 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:32:01 smithi089 bash[33173]: cluster 2023-06-03T19:32:00.375052+0000 mgr.y (mgr.14445) 18 : cluster [DBG] pgmap v9: 1 pgs: 1 active+clean; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:32:01.587 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:32:01 smithi164 bash[26855]: audit 2023-06-03T19:32:00.199664+0000 mgr.y (mgr.14445) 16 : audit [DBG] from='client.24391 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "grafana", "placement": "1;smithi164=a", "target": ["mon-mgr", ""]}]: dispatch 2023-06-03T19:32:01.588 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:32:01 smithi164 bash[26855]: cephadm 2023-06-03T19:32:00.203027+0000 mgr.y (mgr.14445) 17 : cephadm [INF] Saving service grafana spec with placement smithi164=a;count:1 2023-06-03T19:32:01.588 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:32:01 smithi164 bash[26855]: audit 2023-06-03T19:32:00.209045+0000 mon.a (mon.0) 586 : audit [INF] from='mgr.14445 172.21.15.89:0/573502486' entity='mgr.y' 2023-06-03T19:32:01.588 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:32:01 smithi164 bash[26855]: cluster 2023-06-03T19:32:00.375052+0000 mgr.y (mgr.14445) 18 : cluster [DBG] pgmap v9: 1 pgs: 1 active+clean; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:32:02.090 DEBUG:teuthology.orchestra.run.smithi164:grafana.a> sudo journalctl -f -n 0 -u ceph-f1db3478-0243-11ee-9b1d-001a4aab830c@grafana.a.service 2023-06-03T19:32:02.092 INFO:tasks.cephadm:Setting up client nodes... 2023-06-03T19:32:02.093 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph auth get-or-create client.0 mon 'allow *' osd 'allow *' mds 'allow *' mgr 'allow *' 2023-06-03T19:32:02.145 INFO:journalctl@ceph.grafana.a.smithi164.stdout:-- Logs begin at Sun 2022-08-14 12:20:24 UTC. -- 2023-06-03T19:32:03.634 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:32:03 smithi089 bash[33173]: cluster 2023-06-03T19:32:02.375722+0000 mgr.y (mgr.14445) 19 : cluster [DBG] pgmap v10: 1 pgs: 1 active+clean; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:32:03.634 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:32:03 smithi089 bash[26572]: cluster 2023-06-03T19:32:02.375722+0000 mgr.y (mgr.14445) 19 : cluster [DBG] pgmap v10: 1 pgs: 1 active+clean; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:32:03.836 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:32:03 smithi164 bash[26855]: cluster 2023-06-03T19:32:02.375722+0000 mgr.y (mgr.14445) 19 : cluster [DBG] pgmap v10: 1 pgs: 1 active+clean; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:32:05.531 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:32:05 smithi089 bash[33173]: cluster 2023-06-03T19:32:04.376281+0000 mgr.y (mgr.14445) 20 : cluster [DBG] pgmap v11: 1 pgs: 1 active+clean; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:32:05.531 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:32:05 smithi089 bash[26572]: cluster 2023-06-03T19:32:04.376281+0000 mgr.y (mgr.14445) 20 : cluster [DBG] pgmap v11: 1 pgs: 1 active+clean; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:32:05.791 INFO:teuthology.orchestra.run.smithi089.stdout:[client.0] 2023-06-03T19:32:05.791 INFO:teuthology.orchestra.run.smithi089.stdout: key = AQA1lXtkaCI+LhAAn1Ug9SjPT9xY6HcSJgFf6Q== 2023-06-03T19:32:05.836 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:32:05 smithi164 bash[26855]: cluster 2023-06-03T19:32:04.376281+0000 mgr.y (mgr.14445) 20 : cluster [DBG] pgmap v11: 1 pgs: 1 active+clean; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:32:06.588 DEBUG:teuthology.orchestra.run.smithi089:> set -ex 2023-06-03T19:32:06.589 DEBUG:teuthology.orchestra.run.smithi089:> sudo dd of=/etc/ceph/ceph.client.0.keyring 2023-06-03T19:32:06.589 DEBUG:teuthology.orchestra.run.smithi089:> sudo chmod 0644 /etc/ceph/ceph.client.0.keyring 2023-06-03T19:32:06.612 DEBUG:teuthology.orchestra.run.smithi164:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph auth get-or-create client.1 mon 'allow *' osd 'allow *' mds 'allow *' mgr 'allow *' 2023-06-03T19:32:06.781 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:32:06 smithi089 bash[33173]: audit 2023-06-03T19:32:05.775495+0000 mon.a (mon.0) 587 : audit [INF] from='client.? 172.21.15.89:0/611704139' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2023-06-03T19:32:06.781 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:32:06 smithi089 bash[33173]: audit 2023-06-03T19:32:05.784396+0000 mon.a (mon.0) 588 : audit [INF] from='client.? 172.21.15.89:0/611704139' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2023-06-03T19:32:06.782 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:32:06 smithi089 bash[26572]: audit 2023-06-03T19:32:05.775495+0000 mon.a (mon.0) 587 : audit [INF] from='client.? 172.21.15.89:0/611704139' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2023-06-03T19:32:06.782 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:32:06 smithi089 bash[26572]: audit 2023-06-03T19:32:05.784396+0000 mon.a (mon.0) 588 : audit [INF] from='client.? 172.21.15.89:0/611704139' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2023-06-03T19:32:06.809 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:32:06 smithi164 bash[26855]: audit 2023-06-03T19:32:05.775495+0000 mon.a (mon.0) 587 : audit [INF] from='client.? 172.21.15.89:0/611704139' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2023-06-03T19:32:06.810 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:32:06 smithi164 bash[26855]: audit 2023-06-03T19:32:05.784396+0000 mon.a (mon.0) 588 : audit [INF] from='client.? 172.21.15.89:0/611704139' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2023-06-03T19:32:08.085 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:32:07 smithi164 bash[26855]: cluster 2023-06-03T19:32:06.376984+0000 mgr.y (mgr.14445) 21 : cluster [DBG] pgmap v12: 1 pgs: 1 active+clean; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:32:08.281 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:32:07 smithi089 bash[26572]: cluster 2023-06-03T19:32:06.376984+0000 mgr.y (mgr.14445) 21 : cluster [DBG] pgmap v12: 1 pgs: 1 active+clean; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:32:08.281 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:32:07 smithi089 bash[33173]: cluster 2023-06-03T19:32:06.376984+0000 mgr.y (mgr.14445) 21 : cluster [DBG] pgmap v12: 1 pgs: 1 active+clean; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:32:08.467 INFO:journalctl@ceph.prometheus.a.smithi164.stdout:Jun 03 19:32:08 smithi164 systemd[1]: Started Ceph prometheus.a for f1db3478-0243-11ee-9b1d-001a4aab830c. 2023-06-03T19:32:09.269 INFO:journalctl@ceph.node-exporter.a.smithi089.stdout:Jun 03 19:32:09 smithi089 systemd[1]: Started Ceph node-exporter.a for f1db3478-0243-11ee-9b1d-001a4aab830c. 2023-06-03T19:32:09.531 INFO:journalctl@ceph.node-exporter.a.smithi089.stdout:Jun 03 19:32:09 smithi089 bash[51960]: Unable to find image 'quay.io/prometheus/node-exporter:v1.3.1' locally 2023-06-03T19:32:09.531 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:32:09 smithi089 bash[26572]: cluster 2023-06-03T19:32:08.377603+0000 mgr.y (mgr.14445) 22 : cluster [DBG] pgmap v13: 1 pgs: 1 active+clean; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:32:09.531 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:32:09 smithi089 bash[26572]: audit 2023-06-03T19:32:08.455768+0000 mon.a (mon.0) 589 : audit [INF] from='mgr.14445 172.21.15.89:0/573502486' entity='mgr.y' 2023-06-03T19:32:09.532 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:32:09 smithi089 bash[26572]: cephadm 2023-06-03T19:32:08.458759+0000 mgr.y (mgr.14445) 23 : cephadm [INF] Deploying daemon node-exporter.a on smithi089 2023-06-03T19:32:09.532 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:32:09 smithi089 bash[26572]: audit 2023-06-03T19:32:08.599477+0000 mon.a (mon.0) 590 : audit [INF] from='mgr.14445 172.21.15.89:0/573502486' entity='mgr.y' 2023-06-03T19:32:09.532 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:32:09 smithi089 bash[26572]: audit 2023-06-03T19:32:09.223991+0000 mon.a (mon.0) 591 : audit [INF] from='mgr.14445 172.21.15.89:0/573502486' entity='mgr.y' 2023-06-03T19:32:09.532 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:32:09 smithi089 bash[33173]: cluster 2023-06-03T19:32:08.377603+0000 mgr.y (mgr.14445) 22 : cluster [DBG] pgmap v13: 1 pgs: 1 active+clean; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:32:09.532 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:32:09 smithi089 bash[33173]: audit 2023-06-03T19:32:08.455768+0000 mon.a (mon.0) 589 : audit [INF] from='mgr.14445 172.21.15.89:0/573502486' entity='mgr.y' 2023-06-03T19:32:09.532 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:32:09 smithi089 bash[33173]: cephadm 2023-06-03T19:32:08.458759+0000 mgr.y (mgr.14445) 23 : cephadm [INF] Deploying daemon node-exporter.a on smithi089 2023-06-03T19:32:09.533 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:32:09 smithi089 bash[33173]: audit 2023-06-03T19:32:08.599477+0000 mon.a (mon.0) 590 : audit [INF] from='mgr.14445 172.21.15.89:0/573502486' entity='mgr.y' 2023-06-03T19:32:09.533 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:32:09 smithi089 bash[33173]: audit 2023-06-03T19:32:09.223991+0000 mon.a (mon.0) 591 : audit [INF] from='mgr.14445 172.21.15.89:0/573502486' entity='mgr.y' 2023-06-03T19:32:09.727 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:32:09 smithi164 bash[26855]: cluster 2023-06-03T19:32:08.377603+0000 mgr.y (mgr.14445) 22 : cluster [DBG] pgmap v13: 1 pgs: 1 active+clean; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:32:09.727 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:32:09 smithi164 bash[26855]: audit 2023-06-03T19:32:08.455768+0000 mon.a (mon.0) 589 : audit [INF] from='mgr.14445 172.21.15.89:0/573502486' entity='mgr.y' 2023-06-03T19:32:09.728 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:32:09 smithi164 bash[26855]: cephadm 2023-06-03T19:32:08.458759+0000 mgr.y (mgr.14445) 23 : cephadm [INF] Deploying daemon node-exporter.a on smithi089 2023-06-03T19:32:09.728 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:32:09 smithi164 bash[26855]: audit 2023-06-03T19:32:08.599477+0000 mon.a (mon.0) 590 : audit [INF] from='mgr.14445 172.21.15.89:0/573502486' entity='mgr.y' 2023-06-03T19:32:09.728 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:32:09 smithi164 bash[26855]: audit 2023-06-03T19:32:09.223991+0000 mon.a (mon.0) 591 : audit [INF] from='mgr.14445 172.21.15.89:0/573502486' entity='mgr.y' 2023-06-03T19:32:10.024 INFO:journalctl@ceph.node-exporter.b.smithi164.stdout:Jun 03 19:32:09 smithi164 systemd[1]: Started Ceph node-exporter.b for f1db3478-0243-11ee-9b1d-001a4aab830c. 2023-06-03T19:32:10.031 INFO:journalctl@ceph.node-exporter.a.smithi089.stdout:Jun 03 19:32:09 smithi089 bash[51960]: v1.3.1: Pulling from prometheus/node-exporter 2023-06-03T19:32:10.031 INFO:journalctl@ceph.node-exporter.a.smithi089.stdout:Jun 03 19:32:09 smithi089 bash[51960]: aa2a8d90b84c: Pulling fs layer 2023-06-03T19:32:10.031 INFO:journalctl@ceph.node-exporter.a.smithi089.stdout:Jun 03 19:32:09 smithi089 bash[51960]: b45d31ee2d7f: Pulling fs layer 2023-06-03T19:32:10.031 INFO:journalctl@ceph.node-exporter.a.smithi089.stdout:Jun 03 19:32:09 smithi089 bash[51960]: b5db1e299295: Pulling fs layer 2023-06-03T19:32:10.336 INFO:journalctl@ceph.node-exporter.b.smithi164.stdout:Jun 03 19:32:10 smithi164 bash[46901]: Unable to find image 'quay.io/prometheus/node-exporter:v1.3.1' locally 2023-06-03T19:32:10.468 INFO:journalctl@ceph.node-exporter.a.smithi089.stdout:Jun 03 19:32:10 smithi089 bash[51960]: aa2a8d90b84c: Download complete 2023-06-03T19:32:10.468 INFO:journalctl@ceph.node-exporter.a.smithi089.stdout:Jun 03 19:32:10 smithi089 bash[51960]: b45d31ee2d7f: Verifying Checksum 2023-06-03T19:32:10.468 INFO:journalctl@ceph.node-exporter.a.smithi089.stdout:Jun 03 19:32:10 smithi089 bash[51960]: b45d31ee2d7f: Download complete 2023-06-03T19:32:10.469 INFO:journalctl@ceph.node-exporter.a.smithi089.stdout:Jun 03 19:32:10 smithi089 bash[51960]: b5db1e299295: Verifying Checksum 2023-06-03T19:32:10.469 INFO:journalctl@ceph.node-exporter.a.smithi089.stdout:Jun 03 19:32:10 smithi089 bash[51960]: b5db1e299295: Download complete 2023-06-03T19:32:10.469 INFO:journalctl@ceph.node-exporter.a.smithi089.stdout:Jun 03 19:32:10 smithi089 bash[51960]: aa2a8d90b84c: Pull complete 2023-06-03T19:32:10.751 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:32:10 smithi089 bash[26572]: cephadm 2023-06-03T19:32:09.225412+0000 mgr.y (mgr.14445) 24 : cephadm [INF] Deploying daemon node-exporter.b on smithi164 2023-06-03T19:32:10.751 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:32:10 smithi089 bash[26572]: audit 2023-06-03T19:32:09.974686+0000 mon.a (mon.0) 592 : audit [INF] from='mgr.14445 172.21.15.89:0/573502486' entity='mgr.y' 2023-06-03T19:32:10.751 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:32:10 smithi089 bash[33173]: cephadm 2023-06-03T19:32:09.225412+0000 mgr.y (mgr.14445) 24 : cephadm [INF] Deploying daemon node-exporter.b on smithi164 2023-06-03T19:32:10.752 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:32:10 smithi089 bash[33173]: audit 2023-06-03T19:32:09.974686+0000 mon.a (mon.0) 592 : audit [INF] from='mgr.14445 172.21.15.89:0/573502486' entity='mgr.y' 2023-06-03T19:32:10.836 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:32:10 smithi164 bash[26855]: cephadm 2023-06-03T19:32:09.225412+0000 mgr.y (mgr.14445) 24 : cephadm [INF] Deploying daemon node-exporter.b on smithi164 2023-06-03T19:32:10.836 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:32:10 smithi164 bash[26855]: audit 2023-06-03T19:32:09.974686+0000 mon.a (mon.0) 592 : audit [INF] from='mgr.14445 172.21.15.89:0/573502486' entity='mgr.y' 2023-06-03T19:32:11.031 INFO:journalctl@ceph.node-exporter.a.smithi089.stdout:Jun 03 19:32:10 smithi089 bash[51960]: b45d31ee2d7f: Pull complete 2023-06-03T19:32:11.336 INFO:journalctl@ceph.node-exporter.b.smithi164.stdout:Jun 03 19:32:11 smithi164 bash[46901]: v1.3.1: Pulling from prometheus/node-exporter 2023-06-03T19:32:11.336 INFO:journalctl@ceph.prometheus.a.smithi164.stdout:Jun 03 19:32:11 smithi164 bash[46758]: ts=2023-06-03T19:32:11.132Z caller=main.go:512 level=info msg="Starting Prometheus" version="(version=2.33.4, branch=HEAD, revision=83032011a5d3e6102624fe58241a374a7201fee8)" 2023-06-03T19:32:11.336 INFO:journalctl@ceph.prometheus.a.smithi164.stdout:Jun 03 19:32:11 smithi164 bash[46758]: ts=2023-06-03T19:32:11.132Z caller=main.go:517 level=info build_context="(go=go1.17.7, user=root@d13bf69e7be8, date=20220222-16:51:28)" 2023-06-03T19:32:11.336 INFO:journalctl@ceph.prometheus.a.smithi164.stdout:Jun 03 19:32:11 smithi164 bash[46758]: ts=2023-06-03T19:32:11.133Z caller=main.go:518 level=info host_details="(Linux 5.15.0-73-generic #80~20.04.1-Ubuntu SMP Wed May 17 14:58:14 UTC 2023 x86_64 smithi164 (none))" 2023-06-03T19:32:11.337 INFO:journalctl@ceph.prometheus.a.smithi164.stdout:Jun 03 19:32:11 smithi164 bash[46758]: ts=2023-06-03T19:32:11.133Z caller=main.go:519 level=info fd_limits="(soft=1048576, hard=1048576)" 2023-06-03T19:32:11.337 INFO:journalctl@ceph.prometheus.a.smithi164.stdout:Jun 03 19:32:11 smithi164 bash[46758]: ts=2023-06-03T19:32:11.133Z caller=main.go:520 level=info vm_limits="(soft=unlimited, hard=unlimited)" 2023-06-03T19:32:11.337 INFO:journalctl@ceph.prometheus.a.smithi164.stdout:Jun 03 19:32:11 smithi164 bash[46758]: ts=2023-06-03T19:32:11.134Z caller=web.go:570 level=info component=web msg="Start listening for connections" address=:9095 2023-06-03T19:32:11.337 INFO:journalctl@ceph.prometheus.a.smithi164.stdout:Jun 03 19:32:11 smithi164 bash[46758]: ts=2023-06-03T19:32:11.135Z caller=main.go:923 level=info msg="Starting TSDB ..." 2023-06-03T19:32:11.337 INFO:journalctl@ceph.prometheus.a.smithi164.stdout:Jun 03 19:32:11 smithi164 bash[46758]: ts=2023-06-03T19:32:11.135Z caller=tls_config.go:195 level=info component=web msg="TLS is disabled." http2=false 2023-06-03T19:32:11.337 INFO:journalctl@ceph.prometheus.a.smithi164.stdout:Jun 03 19:32:11 smithi164 bash[46758]: ts=2023-06-03T19:32:11.139Z caller=head.go:493 level=info component=tsdb msg="Replaying on-disk memory mappable chunks if any" 2023-06-03T19:32:11.338 INFO:journalctl@ceph.prometheus.a.smithi164.stdout:Jun 03 19:32:11 smithi164 bash[46758]: ts=2023-06-03T19:32:11.139Z caller=head.go:527 level=info component=tsdb msg="On-disk memory mappable chunks replay completed" duration=8.686µs 2023-06-03T19:32:11.338 INFO:journalctl@ceph.prometheus.a.smithi164.stdout:Jun 03 19:32:11 smithi164 bash[46758]: ts=2023-06-03T19:32:11.139Z caller=head.go:533 level=info component=tsdb msg="Replaying WAL, this may take a while" 2023-06-03T19:32:11.338 INFO:journalctl@ceph.prometheus.a.smithi164.stdout:Jun 03 19:32:11 smithi164 bash[46758]: ts=2023-06-03T19:32:11.139Z caller=head.go:604 level=info component=tsdb msg="WAL segment loaded" segment=0 maxSegment=0 2023-06-03T19:32:11.338 INFO:journalctl@ceph.prometheus.a.smithi164.stdout:Jun 03 19:32:11 smithi164 bash[46758]: ts=2023-06-03T19:32:11.139Z caller=head.go:610 level=info component=tsdb msg="WAL replay completed" checkpoint_replay_duration=50.283µs wal_replay_duration=323.045µs total_replay_duration=411.909µs 2023-06-03T19:32:11.338 INFO:journalctl@ceph.prometheus.a.smithi164.stdout:Jun 03 19:32:11 smithi164 bash[46758]: ts=2023-06-03T19:32:11.141Z caller=main.go:944 level=info fs_type=XFS_SUPER_MAGIC 2023-06-03T19:32:11.338 INFO:journalctl@ceph.prometheus.a.smithi164.stdout:Jun 03 19:32:11 smithi164 bash[46758]: ts=2023-06-03T19:32:11.141Z caller=main.go:947 level=info msg="TSDB started" 2023-06-03T19:32:11.339 INFO:journalctl@ceph.prometheus.a.smithi164.stdout:Jun 03 19:32:11 smithi164 bash[46758]: ts=2023-06-03T19:32:11.141Z caller=main.go:1128 level=info msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml 2023-06-03T19:32:11.339 INFO:journalctl@ceph.prometheus.a.smithi164.stdout:Jun 03 19:32:11 smithi164 bash[46758]: ts=2023-06-03T19:32:11.161Z caller=main.go:1165 level=info msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=20.679236ms db_storage=962ns remote_storage=1.63µs web_handler=363ns query_engine=822ns scrape=307.336µs scrape_sd=24.846µs notify=1.112µs notify_sd=1.575µs rules=20.100724ms 2023-06-03T19:32:11.339 INFO:journalctl@ceph.prometheus.a.smithi164.stdout:Jun 03 19:32:11 smithi164 bash[46758]: ts=2023-06-03T19:32:11.161Z caller=main.go:896 level=info msg="Server is ready to receive web requests." 2023-06-03T19:32:11.456 INFO:journalctl@ceph.node-exporter.a.smithi089.stdout:Jun 03 19:32:11 smithi089 bash[51960]: b5db1e299295: Pull complete 2023-06-03T19:32:11.457 INFO:journalctl@ceph.node-exporter.a.smithi089.stdout:Jun 03 19:32:11 smithi089 bash[51960]: Digest: sha256:f2269e73124dd0f60a7d19a2ce1264d33d08a985aed0ee6b0b89d0be470592cd 2023-06-03T19:32:11.457 INFO:journalctl@ceph.node-exporter.a.smithi089.stdout:Jun 03 19:32:11 smithi089 bash[51960]: Status: Downloaded newer image for quay.io/prometheus/node-exporter:v1.3.1 2023-06-03T19:32:11.781 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:32:11 smithi089 bash[33173]: cephadm 2023-06-03T19:32:09.986748+0000 mgr.y (mgr.14445) 25 : cephadm [INF] Deploying daemon alertmanager.a on smithi089 2023-06-03T19:32:11.781 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:32:11 smithi089 bash[33173]: cluster 2023-06-03T19:32:10.378233+0000 mgr.y (mgr.14445) 26 : cluster [DBG] pgmap v14: 1 pgs: 1 active+clean; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:32:11.781 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:32:11 smithi089 bash[26572]: cephadm 2023-06-03T19:32:09.986748+0000 mgr.y (mgr.14445) 25 : cephadm [INF] Deploying daemon alertmanager.a on smithi089 2023-06-03T19:32:11.781 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:32:11 smithi089 bash[26572]: cluster 2023-06-03T19:32:10.378233+0000 mgr.y (mgr.14445) 26 : cluster [DBG] pgmap v14: 1 pgs: 1 active+clean; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:32:11.836 INFO:journalctl@ceph.node-exporter.b.smithi164.stdout:Jun 03 19:32:11 smithi164 bash[46901]: aa2a8d90b84c: Pulling fs layer 2023-06-03T19:32:11.836 INFO:journalctl@ceph.node-exporter.b.smithi164.stdout:Jun 03 19:32:11 smithi164 bash[46901]: b45d31ee2d7f: Pulling fs layer 2023-06-03T19:32:11.837 INFO:journalctl@ceph.node-exporter.b.smithi164.stdout:Jun 03 19:32:11 smithi164 bash[46901]: b5db1e299295: Pulling fs layer 2023-06-03T19:32:11.837 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:32:11 smithi164 bash[26855]: cephadm 2023-06-03T19:32:09.986748+0000 mgr.y (mgr.14445) 25 : cephadm [INF] Deploying daemon alertmanager.a on smithi089 2023-06-03T19:32:11.837 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:32:11 smithi164 bash[26855]: cluster 2023-06-03T19:32:10.378233+0000 mgr.y (mgr.14445) 26 : cluster [DBG] pgmap v14: 1 pgs: 1 active+clean; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:32:12.043 INFO:teuthology.orchestra.run.smithi164.stdout:[client.1] 2023-06-03T19:32:12.044 INFO:teuthology.orchestra.run.smithi164.stdout: key = AQA8lXtkjDvBARAAL6ZxvSdQ6jDmTROJ/By+yg== 2023-06-03T19:32:12.124 INFO:journalctl@ceph.node-exporter.b.smithi164.stdout:Jun 03 19:32:11 smithi164 bash[46901]: aa2a8d90b84c: Download complete 2023-06-03T19:32:12.125 INFO:journalctl@ceph.node-exporter.b.smithi164.stdout:Jun 03 19:32:11 smithi164 bash[46901]: b45d31ee2d7f: Download complete 2023-06-03T19:32:12.125 INFO:journalctl@ceph.node-exporter.b.smithi164.stdout:Jun 03 19:32:11 smithi164 bash[46901]: b5db1e299295: Verifying Checksum 2023-06-03T19:32:12.125 INFO:journalctl@ceph.node-exporter.b.smithi164.stdout:Jun 03 19:32:11 smithi164 bash[46901]: b5db1e299295: Download complete 2023-06-03T19:32:12.125 INFO:journalctl@ceph.node-exporter.b.smithi164.stdout:Jun 03 19:32:12 smithi164 bash[46901]: aa2a8d90b84c: Pull complete 2023-06-03T19:32:12.586 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:32:12 smithi164 bash[26855]: audit 2023-06-03T19:32:12.026300+0000 mon.b (mon.1) 23 : audit [INF] from='client.? 172.21.15.164:0/4121799620' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2023-06-03T19:32:12.586 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:32:12 smithi164 bash[26855]: audit 2023-06-03T19:32:12.029227+0000 mon.a (mon.0) 593 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2023-06-03T19:32:12.586 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:32:12 smithi164 bash[26855]: audit 2023-06-03T19:32:12.035960+0000 mon.a (mon.0) 594 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2023-06-03T19:32:12.781 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:32:12 smithi089 bash[26572]: audit 2023-06-03T19:32:12.026300+0000 mon.b (mon.1) 23 : audit [INF] from='client.? 172.21.15.164:0/4121799620' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2023-06-03T19:32:12.781 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:32:12 smithi089 bash[26572]: audit 2023-06-03T19:32:12.029227+0000 mon.a (mon.0) 593 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2023-06-03T19:32:12.781 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:32:12 smithi089 bash[26572]: audit 2023-06-03T19:32:12.035960+0000 mon.a (mon.0) 594 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2023-06-03T19:32:12.782 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:32:12 smithi089 bash[33173]: audit 2023-06-03T19:32:12.026300+0000 mon.b (mon.1) 23 : audit [INF] from='client.? 172.21.15.164:0/4121799620' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2023-06-03T19:32:12.782 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:32:12 smithi089 bash[33173]: audit 2023-06-03T19:32:12.029227+0000 mon.a (mon.0) 593 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2023-06-03T19:32:12.782 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:32:12 smithi089 bash[33173]: audit 2023-06-03T19:32:12.035960+0000 mon.a (mon.0) 594 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2023-06-03T19:32:13.086 INFO:journalctl@ceph.node-exporter.b.smithi164.stdout:Jun 03 19:32:12 smithi164 bash[46901]: b45d31ee2d7f: Pull complete 2023-06-03T19:32:13.351 DEBUG:teuthology.orchestra.run.smithi164:> set -ex 2023-06-03T19:32:13.352 DEBUG:teuthology.orchestra.run.smithi164:> sudo dd of=/etc/ceph/ceph.client.1.keyring 2023-06-03T19:32:13.352 DEBUG:teuthology.orchestra.run.smithi164:> sudo chmod 0644 /etc/ceph/ceph.client.1.keyring 2023-06-03T19:32:13.371 INFO:tasks.cephadm:Waiting for OSDs to come up 2023-06-03T19:32:13.372 DEBUG:teuthology.orchestra.run.smithi089:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph --cluster ceph osd dump --format=json 2023-06-03T19:32:13.659 INFO:journalctl@ceph.node-exporter.b.smithi164.stdout:Jun 03 19:32:13 smithi164 bash[46901]: b5db1e299295: Pull complete 2023-06-03T19:32:13.660 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:32:13 smithi164 bash[26855]: cluster 2023-06-03T19:32:12.378793+0000 mgr.y (mgr.14445) 27 : cluster [DBG] pgmap v15: 1 pgs: 1 active+clean; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:32:13.707 INFO:teuthology.misc.health.smithi089.stdout: 2023-06-03T19:32:13.707 INFO:teuthology.misc.health.smithi089.stdout:{"epoch":51,"fsid":"f1db3478-0243-11ee-9b1d-001a4aab830c","created":"2023-06-03T19:22:54.447184+0000","modified":"2023-06-03T19:31:48.303305+0000","last_up_change":"2023-06-03T19:31:13.424226+0000","last_in_change":"2023-06-03T19:30:46.909676+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":1,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"pacific","pools":[{"pool":1,"pool_name":"device_health_metrics","create_time":"2023-06-03T19:27:28.100610+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"21","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr_devicehealth":{}}}],"osds":[{"osd":0,"uuid":"86063bfe-126e-46e7-93ed-c73813af165b","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":48,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:6802","nonce":1795715824},{"type":"v1","addr":"172.21.15.89:6803","nonce":1795715824}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:6804","nonce":1795715824},{"type":"v1","addr":"172.21.15.89:6805","nonce":1795715824}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:6808","nonce":1795715824},{"type":"v1","addr":"172.21.15.89:6809","nonce":1795715824}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:6806","nonce":1795715824},{"type":"v1","addr":"172.21.15.89:6807","nonce":1795715824}]},"public_addr":"172.21.15.89:6803/1795715824","cluster_addr":"172.21.15.89:6805/1795715824","heartbeat_back_addr":"172.21.15.89:6809/1795715824","heartbeat_front_addr":"172.21.15.89:6807/1795715824","state":["exists","up"]},{"osd":1,"uuid":"04783a81-6b8c-4657-8446-00efc872b313","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":31,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:6810","nonce":4020168282},{"type":"v1","addr":"172.21.15.89:6811","nonce":4020168282}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:6812","nonce":4020168282},{"type":"v1","addr":"172.21.15.89:6813","nonce":4020168282}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:6816","nonce":4020168282},{"type":"v1","addr":"172.21.15.89:6817","nonce":4020168282}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:6814","nonce":4020168282},{"type":"v1","addr":"172.21.15.89:6815","nonce":4020168282}]},"public_addr":"172.21.15.89:6811/4020168282","cluster_addr":"172.21.15.89:6813/4020168282","heartbeat_back_addr":"172.21.15.89:6817/4020168282","heartbeat_front_addr":"172.21.15.89:6815/4020168282","state":["exists","up"]},{"osd":2,"uuid":"d2d2c641-392a-444f-81bc-84eef6340f30","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:6818","nonce":2113694583},{"type":"v1","addr":"172.21.15.89:6819","nonce":2113694583}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:6820","nonce":2113694583},{"type":"v1","addr":"172.21.15.89:6821","nonce":2113694583}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:6824","nonce":2113694583},{"type":"v1","addr":"172.21.15.89:6825","nonce":2113694583}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:6822","nonce":2113694583},{"type":"v1","addr":"172.21.15.89:6823","nonce":2113694583}]},"public_addr":"172.21.15.89:6819/2113694583","cluster_addr":"172.21.15.89:6821/2113694583","heartbeat_back_addr":"172.21.15.89:6825/2113694583","heartbeat_front_addr":"172.21.15.89:6823/2113694583","state":["exists","up"]},{"osd":3,"uuid":"b38408eb-3acf-4a41-a314-72e4cd37b1ef","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":25,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:6826","nonce":3649114317},{"type":"v1","addr":"172.21.15.89:6827","nonce":3649114317}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:6828","nonce":3649114317},{"type":"v1","addr":"172.21.15.89:6829","nonce":3649114317}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:6832","nonce":3649114317},{"type":"v1","addr":"172.21.15.89:6833","nonce":3649114317}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:6830","nonce":3649114317},{"type":"v1","addr":"172.21.15.89:6831","nonce":3649114317}]},"public_addr":"172.21.15.89:6827/3649114317","cluster_addr":"172.21.15.89:6829/3649114317","heartbeat_back_addr":"172.21.15.89:6833/3649114317","heartbeat_front_addr":"172.21.15.89:6831/3649114317","state":["exists","up"]},{"osd":4,"uuid":"a94be785-9f54-4ab8-962a-9cf974a5b5e3","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":30,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.164:6800","nonce":538382489},{"type":"v1","addr":"172.21.15.164:6801","nonce":538382489}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.164:6802","nonce":538382489},{"type":"v1","addr":"172.21.15.164:6803","nonce":538382489}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.164:6806","nonce":538382489},{"type":"v1","addr":"172.21.15.164:6807","nonce":538382489}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.164:6804","nonce":538382489},{"type":"v1","addr":"172.21.15.164:6805","nonce":538382489}]},"public_addr":"172.21.15.164:6801/538382489","cluster_addr":"172.21.15.164:6803/538382489","heartbeat_back_addr":"172.21.15.164:6807/538382489","heartbeat_front_addr":"172.21.15.164:6805/538382489","state":["exists","up"]},{"osd":5,"uuid":"b8a5e8a1-23cc-442b-90aa-19bd0036dd3c","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":36,"up_thru":37,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.164:6808","nonce":3123006405},{"type":"v1","addr":"172.21.15.164:6809","nonce":3123006405}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.164:6810","nonce":3123006405},{"type":"v1","addr":"172.21.15.164:6811","nonce":3123006405}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.164:6814","nonce":3123006405},{"type":"v1","addr":"172.21.15.164:6815","nonce":3123006405}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.164:6812","nonce":3123006405},{"type":"v1","addr":"172.21.15.164:6813","nonce":3123006405}]},"public_addr":"172.21.15.164:6809/3123006405","cluster_addr":"172.21.15.164:6811/3123006405","heartbeat_back_addr":"172.21.15.164:6815/3123006405","heartbeat_front_addr":"172.21.15.164:6813/3123006405","state":["exists","up"]},{"osd":6,"uuid":"5c5edd3b-d92c-4f46-94ff-2d5828b2e732","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":42,"up_thru":43,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.164:6816","nonce":712403290},{"type":"v1","addr":"172.21.15.164:6817","nonce":712403290}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.164:6818","nonce":712403290},{"type":"v1","addr":"172.21.15.164:6819","nonce":712403290}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.164:6822","nonce":712403290},{"type":"v1","addr":"172.21.15.164:6823","nonce":712403290}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.164:6820","nonce":712403290},{"type":"v1","addr":"172.21.15.164:6821","nonce":712403290}]},"public_addr":"172.21.15.164:6817/712403290","cluster_addr":"172.21.15.164:6819/712403290","heartbeat_back_addr":"172.21.15.164:6823/712403290","heartbeat_front_addr":"172.21.15.164:6821/712403290","state":["exists","up"]},{"osd":7,"uuid":"bdff9290-e6c4-407e-9113-eb39696c0e27","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":48,"up_thru":49,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.164:6824","nonce":711997991},{"type":"v1","addr":"172.21.15.164:6825","nonce":711997991}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.164:6826","nonce":711997991},{"type":"v1","addr":"172.21.15.164:6827","nonce":711997991}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.164:6830","nonce":711997991},{"type":"v1","addr":"172.21.15.164:6831","nonce":711997991}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.164:6828","nonce":711997991},{"type":"v1","addr":"172.21.15.164:6829","nonce":711997991}]},"public_addr":"172.21.15.164:6825/711997991","cluster_addr":"172.21.15.164:6827/711997991","heartbeat_back_addr":"172.21.15.164:6831/711997991","heartbeat_front_addr":"172.21.15.164:6829/711997991","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-06-03T19:25:52.155714+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-06-03T19:26:38.554458+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-06-03T19:27:25.383237+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-06-03T19:28:13.272585+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-06-03T19:28:56.685710+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-06-03T19:29:41.572533+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-06-03T19:30:26.631983+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-06-03T19:31:12.018414+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.89:0/3944267183":"2023-06-04T19:23:55.626873+0000","172.21.15.89:0/470853232":"2023-06-04T19:31:48.303240+0000","172.21.15.89:6801/4116589411":"2023-06-04T19:23:55.626873+0000","172.21.15.89:6801/2256305550":"2023-06-04T19:31:48.303240+0000","172.21.15.89:6800/4116589411":"2023-06-04T19:23:55.626873+0000","172.21.15.89:0/274274920":"2023-06-04T19:23:55.626873+0000","172.21.15.89:0/1198702890":"2023-06-04T19:23:23.903333+0000","172.21.15.89:6800/1487566136":"2023-06-04T19:23:23.903333+0000","172.21.15.89:6800/2256305550":"2023-06-04T19:31:48.303240+0000","172.21.15.89:0/2392321634":"2023-06-04T19:31:48.303240+0000","172.21.15.89:6801/1487566136":"2023-06-04T19:23:23.903333+0000","172.21.15.89:0/811967393":"2023-06-04T19:23:23.903333+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2023-06-03T19:32:13.720 DEBUG:teuthology.misc:8 of 8 OSDs are up 2023-06-03T19:32:13.721 INFO:tasks.cephadm:Creating RBD pool 2023-06-03T19:32:13.721 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- sudo ceph --cluster ceph osd pool create rbd 8 2023-06-03T19:32:13.761 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:32:13 smithi089 bash[26572]: cluster 2023-06-03T19:32:12.378793+0000 mgr.y (mgr.14445) 27 : cluster [DBG] pgmap v15: 1 pgs: 1 active+clean; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:32:13.762 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:32:13 smithi089 bash[33173]: cluster 2023-06-03T19:32:12.378793+0000 mgr.y (mgr.14445) 27 : cluster [DBG] pgmap v15: 1 pgs: 1 active+clean; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:32:14.086 INFO:journalctl@ceph.node-exporter.b.smithi164.stdout:Jun 03 19:32:13 smithi164 bash[46901]: Digest: sha256:f2269e73124dd0f60a7d19a2ce1264d33d08a985aed0ee6b0b89d0be470592cd 2023-06-03T19:32:14.086 INFO:journalctl@ceph.node-exporter.b.smithi164.stdout:Jun 03 19:32:13 smithi164 bash[46901]: Status: Downloaded newer image for quay.io/prometheus/node-exporter:v1.3.1 2023-06-03T19:32:14.281 INFO:journalctl@ceph.node-exporter.a.smithi089.stdout:Jun 03 19:32:13 smithi089 bash[51960]: ts=2023-06-03T19:32:13.812Z caller=node_exporter.go:182 level=info msg="Starting node_exporter" version="(version=1.3.1, branch=HEAD, revision=a2321e7b940ddcff26873612bccdf7cd4c42b6b6)" 2023-06-03T19:32:14.281 INFO:journalctl@ceph.node-exporter.a.smithi089.stdout:Jun 03 19:32:13 smithi089 bash[51960]: ts=2023-06-03T19:32:13.812Z caller=node_exporter.go:183 level=info msg="Build context" build_context="(go=go1.17.3, user=root@243aafa5525c, date=20211205-11:09:49)" 2023-06-03T19:32:14.281 INFO:journalctl@ceph.node-exporter.a.smithi089.stdout:Jun 03 19:32:13 smithi089 bash[51960]: ts=2023-06-03T19:32:13.812Z caller=filesystem_common.go:111 level=info collector=filesystem msg="Parsed flag --collector.filesystem.mount-points-exclude" flag=^/(dev|proc|run/credentials/.+|sys|var/lib/docker/.+)($|/) 2023-06-03T19:32:14.282 INFO:journalctl@ceph.node-exporter.a.smithi089.stdout:Jun 03 19:32:13 smithi089 bash[51960]: ts=2023-06-03T19:32:13.812Z caller=filesystem_common.go:113 level=info collector=filesystem msg="Parsed flag --collector.filesystem.fs-types-exclude" flag=^(autofs|binfmt_misc|bpf|cgroup2?|configfs|debugfs|devpts|devtmpfs|fusectl|hugetlbfs|iso9660|mqueue|nsfs|overlay|proc|procfs|pstore|rpc_pipefs|securityfs|selinuxfs|squashfs|sysfs|tracefs)$ 2023-06-03T19:32:14.282 INFO:journalctl@ceph.node-exporter.a.smithi089.stdout:Jun 03 19:32:13 smithi089 bash[51960]: ts=2023-06-03T19:32:13.815Z caller=node_exporter.go:108 level=info msg="Enabled collectors" 2023-06-03T19:32:14.282 INFO:journalctl@ceph.node-exporter.a.smithi089.stdout:Jun 03 19:32:13 smithi089 bash[51960]: ts=2023-06-03T19:32:13.815Z caller=node_exporter.go:115 level=info collector=arp 2023-06-03T19:32:14.282 INFO:journalctl@ceph.node-exporter.a.smithi089.stdout:Jun 03 19:32:13 smithi089 bash[51960]: ts=2023-06-03T19:32:13.815Z caller=node_exporter.go:115 level=info collector=bcache 2023-06-03T19:32:14.285 INFO:journalctl@ceph.node-exporter.a.smithi089.stdout:Jun 03 19:32:13 smithi089 bash[51960]: ts=2023-06-03T19:32:13.815Z caller=node_exporter.go:115 level=info collector=bonding 2023-06-03T19:32:14.285 INFO:journalctl@ceph.node-exporter.a.smithi089.stdout:Jun 03 19:32:13 smithi089 bash[51960]: ts=2023-06-03T19:32:13.815Z caller=node_exporter.go:115 level=info collector=btrfs 2023-06-03T19:32:14.285 INFO:journalctl@ceph.node-exporter.a.smithi089.stdout:Jun 03 19:32:13 smithi089 bash[51960]: ts=2023-06-03T19:32:13.815Z caller=node_exporter.go:115 level=info collector=conntrack 2023-06-03T19:32:14.285 INFO:journalctl@ceph.node-exporter.a.smithi089.stdout:Jun 03 19:32:13 smithi089 bash[51960]: ts=2023-06-03T19:32:13.815Z caller=node_exporter.go:115 level=info collector=cpu 2023-06-03T19:32:14.286 INFO:journalctl@ceph.node-exporter.a.smithi089.stdout:Jun 03 19:32:13 smithi089 bash[51960]: ts=2023-06-03T19:32:13.815Z caller=node_exporter.go:115 level=info collector=cpufreq 2023-06-03T19:32:14.286 INFO:journalctl@ceph.node-exporter.a.smithi089.stdout:Jun 03 19:32:13 smithi089 bash[51960]: ts=2023-06-03T19:32:13.815Z caller=node_exporter.go:115 level=info collector=diskstats 2023-06-03T19:32:14.286 INFO:journalctl@ceph.node-exporter.a.smithi089.stdout:Jun 03 19:32:13 smithi089 bash[51960]: ts=2023-06-03T19:32:13.815Z caller=node_exporter.go:115 level=info collector=dmi 2023-06-03T19:32:14.287 INFO:journalctl@ceph.node-exporter.a.smithi089.stdout:Jun 03 19:32:13 smithi089 bash[51960]: ts=2023-06-03T19:32:13.815Z caller=node_exporter.go:115 level=info collector=edac 2023-06-03T19:32:14.287 INFO:journalctl@ceph.node-exporter.a.smithi089.stdout:Jun 03 19:32:13 smithi089 bash[51960]: ts=2023-06-03T19:32:13.815Z caller=node_exporter.go:115 level=info collector=entropy 2023-06-03T19:32:14.287 INFO:journalctl@ceph.node-exporter.a.smithi089.stdout:Jun 03 19:32:13 smithi089 bash[51960]: ts=2023-06-03T19:32:13.815Z caller=node_exporter.go:115 level=info collector=fibrechannel 2023-06-03T19:32:14.287 INFO:journalctl@ceph.node-exporter.a.smithi089.stdout:Jun 03 19:32:13 smithi089 bash[51960]: ts=2023-06-03T19:32:13.815Z caller=node_exporter.go:115 level=info collector=filefd 2023-06-03T19:32:14.288 INFO:journalctl@ceph.node-exporter.a.smithi089.stdout:Jun 03 19:32:13 smithi089 bash[51960]: ts=2023-06-03T19:32:13.815Z caller=node_exporter.go:115 level=info collector=filesystem 2023-06-03T19:32:14.288 INFO:journalctl@ceph.node-exporter.a.smithi089.stdout:Jun 03 19:32:13 smithi089 bash[51960]: ts=2023-06-03T19:32:13.815Z caller=node_exporter.go:115 level=info collector=hwmon 2023-06-03T19:32:14.288 INFO:journalctl@ceph.node-exporter.a.smithi089.stdout:Jun 03 19:32:13 smithi089 bash[51960]: ts=2023-06-03T19:32:13.815Z caller=node_exporter.go:115 level=info collector=infiniband 2023-06-03T19:32:14.289 INFO:journalctl@ceph.node-exporter.a.smithi089.stdout:Jun 03 19:32:13 smithi089 bash[51960]: ts=2023-06-03T19:32:13.815Z caller=node_exporter.go:115 level=info collector=ipvs 2023-06-03T19:32:14.289 INFO:journalctl@ceph.node-exporter.a.smithi089.stdout:Jun 03 19:32:13 smithi089 bash[51960]: ts=2023-06-03T19:32:13.815Z caller=node_exporter.go:115 level=info collector=loadavg 2023-06-03T19:32:14.289 INFO:journalctl@ceph.node-exporter.a.smithi089.stdout:Jun 03 19:32:13 smithi089 bash[51960]: ts=2023-06-03T19:32:13.815Z caller=node_exporter.go:115 level=info collector=mdadm 2023-06-03T19:32:14.289 INFO:journalctl@ceph.node-exporter.a.smithi089.stdout:Jun 03 19:32:13 smithi089 bash[51960]: ts=2023-06-03T19:32:13.815Z caller=node_exporter.go:115 level=info collector=meminfo 2023-06-03T19:32:14.290 INFO:journalctl@ceph.node-exporter.a.smithi089.stdout:Jun 03 19:32:13 smithi089 bash[51960]: ts=2023-06-03T19:32:13.815Z caller=node_exporter.go:115 level=info collector=netclass 2023-06-03T19:32:14.290 INFO:journalctl@ceph.node-exporter.a.smithi089.stdout:Jun 03 19:32:13 smithi089 bash[51960]: ts=2023-06-03T19:32:13.815Z caller=node_exporter.go:115 level=info collector=netdev 2023-06-03T19:32:14.290 INFO:journalctl@ceph.node-exporter.a.smithi089.stdout:Jun 03 19:32:13 smithi089 bash[51960]: ts=2023-06-03T19:32:13.815Z caller=node_exporter.go:115 level=info collector=netstat 2023-06-03T19:32:14.290 INFO:journalctl@ceph.node-exporter.a.smithi089.stdout:Jun 03 19:32:13 smithi089 bash[51960]: ts=2023-06-03T19:32:13.815Z caller=node_exporter.go:115 level=info collector=nfs 2023-06-03T19:32:14.290 INFO:journalctl@ceph.node-exporter.a.smithi089.stdout:Jun 03 19:32:13 smithi089 bash[51960]: ts=2023-06-03T19:32:13.815Z caller=node_exporter.go:115 level=info collector=nfsd 2023-06-03T19:32:14.291 INFO:journalctl@ceph.node-exporter.a.smithi089.stdout:Jun 03 19:32:13 smithi089 bash[51960]: ts=2023-06-03T19:32:13.815Z caller=node_exporter.go:115 level=info collector=nvme 2023-06-03T19:32:14.291 INFO:journalctl@ceph.node-exporter.a.smithi089.stdout:Jun 03 19:32:13 smithi089 bash[51960]: ts=2023-06-03T19:32:13.815Z caller=node_exporter.go:115 level=info collector=os 2023-06-03T19:32:14.291 INFO:journalctl@ceph.node-exporter.a.smithi089.stdout:Jun 03 19:32:13 smithi089 bash[51960]: ts=2023-06-03T19:32:13.815Z caller=node_exporter.go:115 level=info collector=powersupplyclass 2023-06-03T19:32:14.291 INFO:journalctl@ceph.node-exporter.a.smithi089.stdout:Jun 03 19:32:13 smithi089 bash[51960]: ts=2023-06-03T19:32:13.815Z caller=node_exporter.go:115 level=info collector=pressure 2023-06-03T19:32:14.292 INFO:journalctl@ceph.node-exporter.a.smithi089.stdout:Jun 03 19:32:13 smithi089 bash[51960]: ts=2023-06-03T19:32:13.815Z caller=node_exporter.go:115 level=info collector=rapl 2023-06-03T19:32:14.292 INFO:journalctl@ceph.node-exporter.a.smithi089.stdout:Jun 03 19:32:13 smithi089 bash[51960]: ts=2023-06-03T19:32:13.815Z caller=node_exporter.go:115 level=info collector=schedstat 2023-06-03T19:32:14.292 INFO:journalctl@ceph.node-exporter.a.smithi089.stdout:Jun 03 19:32:13 smithi089 bash[51960]: ts=2023-06-03T19:32:13.815Z caller=node_exporter.go:115 level=info collector=sockstat 2023-06-03T19:32:14.292 INFO:journalctl@ceph.node-exporter.a.smithi089.stdout:Jun 03 19:32:13 smithi089 bash[51960]: ts=2023-06-03T19:32:13.815Z caller=node_exporter.go:115 level=info collector=softnet 2023-06-03T19:32:14.293 INFO:journalctl@ceph.node-exporter.a.smithi089.stdout:Jun 03 19:32:13 smithi089 bash[51960]: ts=2023-06-03T19:32:13.815Z caller=node_exporter.go:115 level=info collector=stat 2023-06-03T19:32:14.293 INFO:journalctl@ceph.node-exporter.a.smithi089.stdout:Jun 03 19:32:13 smithi089 bash[51960]: ts=2023-06-03T19:32:13.815Z caller=node_exporter.go:115 level=info collector=tapestats 2023-06-03T19:32:14.293 INFO:journalctl@ceph.node-exporter.a.smithi089.stdout:Jun 03 19:32:13 smithi089 bash[51960]: ts=2023-06-03T19:32:13.815Z caller=node_exporter.go:115 level=info collector=textfile 2023-06-03T19:32:14.293 INFO:journalctl@ceph.node-exporter.a.smithi089.stdout:Jun 03 19:32:13 smithi089 bash[51960]: ts=2023-06-03T19:32:13.815Z caller=node_exporter.go:115 level=info collector=thermal_zone 2023-06-03T19:32:14.293 INFO:journalctl@ceph.node-exporter.a.smithi089.stdout:Jun 03 19:32:13 smithi089 bash[51960]: ts=2023-06-03T19:32:13.815Z caller=node_exporter.go:115 level=info collector=time 2023-06-03T19:32:14.293 INFO:journalctl@ceph.node-exporter.a.smithi089.stdout:Jun 03 19:32:13 smithi089 bash[51960]: ts=2023-06-03T19:32:13.816Z caller=node_exporter.go:115 level=info collector=udp_queues 2023-06-03T19:32:14.293 INFO:journalctl@ceph.node-exporter.a.smithi089.stdout:Jun 03 19:32:13 smithi089 bash[51960]: ts=2023-06-03T19:32:13.816Z caller=node_exporter.go:115 level=info collector=uname 2023-06-03T19:32:14.294 INFO:journalctl@ceph.node-exporter.a.smithi089.stdout:Jun 03 19:32:13 smithi089 bash[51960]: ts=2023-06-03T19:32:13.816Z caller=node_exporter.go:115 level=info collector=vmstat 2023-06-03T19:32:14.294 INFO:journalctl@ceph.node-exporter.a.smithi089.stdout:Jun 03 19:32:13 smithi089 bash[51960]: ts=2023-06-03T19:32:13.816Z caller=node_exporter.go:115 level=info collector=xfs 2023-06-03T19:32:14.294 INFO:journalctl@ceph.node-exporter.a.smithi089.stdout:Jun 03 19:32:13 smithi089 bash[51960]: ts=2023-06-03T19:32:13.816Z caller=node_exporter.go:115 level=info collector=zfs 2023-06-03T19:32:14.294 INFO:journalctl@ceph.node-exporter.a.smithi089.stdout:Jun 03 19:32:13 smithi089 bash[51960]: ts=2023-06-03T19:32:13.816Z caller=node_exporter.go:199 level=info msg="Listening on" address=:9100 2023-06-03T19:32:14.294 INFO:journalctl@ceph.node-exporter.a.smithi089.stdout:Jun 03 19:32:13 smithi089 bash[51960]: ts=2023-06-03T19:32:13.816Z caller=tls_config.go:195 level=info msg="TLS is disabled." http2=false 2023-06-03T19:32:14.888 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:32:14 smithi164 bash[26855]: audit 2023-06-03T19:32:13.608003+0000 mon.a (mon.0) 595 : audit [INF] from='mgr.14445 172.21.15.89:0/573502486' entity='mgr.y' 2023-06-03T19:32:14.888 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:32:14 smithi164 bash[26855]: audit 2023-06-03T19:32:13.699918+0000 mon.a (mon.0) 596 : audit [DBG] from='client.? 172.21.15.89:0/723832123' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-06-03T19:32:15.031 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:32:14 smithi089 bash[26572]: audit 2023-06-03T19:32:13.608003+0000 mon.a (mon.0) 595 : audit [INF] from='mgr.14445 172.21.15.89:0/573502486' entity='mgr.y' 2023-06-03T19:32:15.031 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:32:14 smithi089 bash[26572]: audit 2023-06-03T19:32:13.699918+0000 mon.a (mon.0) 596 : audit [DBG] from='client.? 172.21.15.89:0/723832123' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-06-03T19:32:15.031 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:32:14 smithi089 bash[33173]: audit 2023-06-03T19:32:13.608003+0000 mon.a (mon.0) 595 : audit [INF] from='mgr.14445 172.21.15.89:0/573502486' entity='mgr.y' 2023-06-03T19:32:15.032 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:32:14 smithi089 bash[33173]: audit 2023-06-03T19:32:13.699918+0000 mon.a (mon.0) 596 : audit [DBG] from='client.? 172.21.15.89:0/723832123' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-06-03T19:32:15.337 INFO:journalctl@ceph.node-exporter.b.smithi164.stdout:Jun 03 19:32:14 smithi164 bash[46901]: ts=2023-06-03T19:32:14.879Z caller=node_exporter.go:182 level=info msg="Starting node_exporter" version="(version=1.3.1, branch=HEAD, revision=a2321e7b940ddcff26873612bccdf7cd4c42b6b6)" 2023-06-03T19:32:15.337 INFO:journalctl@ceph.node-exporter.b.smithi164.stdout:Jun 03 19:32:14 smithi164 bash[46901]: ts=2023-06-03T19:32:14.879Z caller=node_exporter.go:183 level=info msg="Build context" build_context="(go=go1.17.3, user=root@243aafa5525c, date=20211205-11:09:49)" 2023-06-03T19:32:15.337 INFO:journalctl@ceph.node-exporter.b.smithi164.stdout:Jun 03 19:32:14 smithi164 bash[46901]: ts=2023-06-03T19:32:14.880Z caller=filesystem_common.go:111 level=info collector=filesystem msg="Parsed flag --collector.filesystem.mount-points-exclude" flag=^/(dev|proc|run/credentials/.+|sys|var/lib/docker/.+)($|/) 2023-06-03T19:32:15.337 INFO:journalctl@ceph.node-exporter.b.smithi164.stdout:Jun 03 19:32:14 smithi164 bash[46901]: ts=2023-06-03T19:32:14.880Z caller=filesystem_common.go:113 level=info collector=filesystem msg="Parsed flag --collector.filesystem.fs-types-exclude" flag=^(autofs|binfmt_misc|bpf|cgroup2?|configfs|debugfs|devpts|devtmpfs|fusectl|hugetlbfs|iso9660|mqueue|nsfs|overlay|proc|procfs|pstore|rpc_pipefs|securityfs|selinuxfs|squashfs|sysfs|tracefs)$ 2023-06-03T19:32:15.338 INFO:journalctl@ceph.node-exporter.b.smithi164.stdout:Jun 03 19:32:14 smithi164 bash[46901]: ts=2023-06-03T19:32:14.881Z caller=node_exporter.go:108 level=info msg="Enabled collectors" 2023-06-03T19:32:15.338 INFO:journalctl@ceph.node-exporter.b.smithi164.stdout:Jun 03 19:32:14 smithi164 bash[46901]: ts=2023-06-03T19:32:14.881Z caller=node_exporter.go:115 level=info collector=arp 2023-06-03T19:32:15.338 INFO:journalctl@ceph.node-exporter.b.smithi164.stdout:Jun 03 19:32:14 smithi164 bash[46901]: ts=2023-06-03T19:32:14.881Z caller=node_exporter.go:115 level=info collector=bcache 2023-06-03T19:32:15.338 INFO:journalctl@ceph.node-exporter.b.smithi164.stdout:Jun 03 19:32:14 smithi164 bash[46901]: ts=2023-06-03T19:32:14.881Z caller=node_exporter.go:115 level=info collector=bonding 2023-06-03T19:32:15.338 INFO:journalctl@ceph.node-exporter.b.smithi164.stdout:Jun 03 19:32:14 smithi164 bash[46901]: ts=2023-06-03T19:32:14.881Z caller=node_exporter.go:115 level=info collector=btrfs 2023-06-03T19:32:15.338 INFO:journalctl@ceph.node-exporter.b.smithi164.stdout:Jun 03 19:32:14 smithi164 bash[46901]: ts=2023-06-03T19:32:14.881Z caller=node_exporter.go:115 level=info collector=conntrack 2023-06-03T19:32:15.338 INFO:journalctl@ceph.node-exporter.b.smithi164.stdout:Jun 03 19:32:14 smithi164 bash[46901]: ts=2023-06-03T19:32:14.881Z caller=node_exporter.go:115 level=info collector=cpu 2023-06-03T19:32:15.339 INFO:journalctl@ceph.node-exporter.b.smithi164.stdout:Jun 03 19:32:14 smithi164 bash[46901]: ts=2023-06-03T19:32:14.881Z caller=node_exporter.go:115 level=info collector=cpufreq 2023-06-03T19:32:15.339 INFO:journalctl@ceph.node-exporter.b.smithi164.stdout:Jun 03 19:32:14 smithi164 bash[46901]: ts=2023-06-03T19:32:14.881Z caller=node_exporter.go:115 level=info collector=diskstats 2023-06-03T19:32:15.339 INFO:journalctl@ceph.node-exporter.b.smithi164.stdout:Jun 03 19:32:14 smithi164 bash[46901]: ts=2023-06-03T19:32:14.881Z caller=node_exporter.go:115 level=info collector=dmi 2023-06-03T19:32:15.339 INFO:journalctl@ceph.node-exporter.b.smithi164.stdout:Jun 03 19:32:14 smithi164 bash[46901]: ts=2023-06-03T19:32:14.881Z caller=node_exporter.go:115 level=info collector=edac 2023-06-03T19:32:15.339 INFO:journalctl@ceph.node-exporter.b.smithi164.stdout:Jun 03 19:32:14 smithi164 bash[46901]: ts=2023-06-03T19:32:14.881Z caller=node_exporter.go:115 level=info collector=entropy 2023-06-03T19:32:15.339 INFO:journalctl@ceph.node-exporter.b.smithi164.stdout:Jun 03 19:32:14 smithi164 bash[46901]: ts=2023-06-03T19:32:14.881Z caller=node_exporter.go:115 level=info collector=fibrechannel 2023-06-03T19:32:15.339 INFO:journalctl@ceph.node-exporter.b.smithi164.stdout:Jun 03 19:32:14 smithi164 bash[46901]: ts=2023-06-03T19:32:14.881Z caller=node_exporter.go:115 level=info collector=filefd 2023-06-03T19:32:15.339 INFO:journalctl@ceph.node-exporter.b.smithi164.stdout:Jun 03 19:32:14 smithi164 bash[46901]: ts=2023-06-03T19:32:14.881Z caller=node_exporter.go:115 level=info collector=filesystem 2023-06-03T19:32:15.340 INFO:journalctl@ceph.node-exporter.b.smithi164.stdout:Jun 03 19:32:14 smithi164 bash[46901]: ts=2023-06-03T19:32:14.881Z caller=node_exporter.go:115 level=info collector=hwmon 2023-06-03T19:32:15.340 INFO:journalctl@ceph.node-exporter.b.smithi164.stdout:Jun 03 19:32:14 smithi164 bash[46901]: ts=2023-06-03T19:32:14.881Z caller=node_exporter.go:115 level=info collector=infiniband 2023-06-03T19:32:15.340 INFO:journalctl@ceph.node-exporter.b.smithi164.stdout:Jun 03 19:32:14 smithi164 bash[46901]: ts=2023-06-03T19:32:14.881Z caller=node_exporter.go:115 level=info collector=ipvs 2023-06-03T19:32:15.340 INFO:journalctl@ceph.node-exporter.b.smithi164.stdout:Jun 03 19:32:14 smithi164 bash[46901]: ts=2023-06-03T19:32:14.881Z caller=node_exporter.go:115 level=info collector=loadavg 2023-06-03T19:32:15.340 INFO:journalctl@ceph.node-exporter.b.smithi164.stdout:Jun 03 19:32:14 smithi164 bash[46901]: ts=2023-06-03T19:32:14.881Z caller=node_exporter.go:115 level=info collector=mdadm 2023-06-03T19:32:15.340 INFO:journalctl@ceph.node-exporter.b.smithi164.stdout:Jun 03 19:32:14 smithi164 bash[46901]: ts=2023-06-03T19:32:14.881Z caller=node_exporter.go:115 level=info collector=meminfo 2023-06-03T19:32:15.340 INFO:journalctl@ceph.node-exporter.b.smithi164.stdout:Jun 03 19:32:14 smithi164 bash[46901]: ts=2023-06-03T19:32:14.881Z caller=node_exporter.go:115 level=info collector=netclass 2023-06-03T19:32:15.341 INFO:journalctl@ceph.node-exporter.b.smithi164.stdout:Jun 03 19:32:14 smithi164 bash[46901]: ts=2023-06-03T19:32:14.881Z caller=node_exporter.go:115 level=info collector=netdev 2023-06-03T19:32:15.341 INFO:journalctl@ceph.node-exporter.b.smithi164.stdout:Jun 03 19:32:14 smithi164 bash[46901]: ts=2023-06-03T19:32:14.881Z caller=node_exporter.go:115 level=info collector=netstat 2023-06-03T19:32:15.341 INFO:journalctl@ceph.node-exporter.b.smithi164.stdout:Jun 03 19:32:14 smithi164 bash[46901]: ts=2023-06-03T19:32:14.881Z caller=node_exporter.go:115 level=info collector=nfs 2023-06-03T19:32:15.341 INFO:journalctl@ceph.node-exporter.b.smithi164.stdout:Jun 03 19:32:14 smithi164 bash[46901]: ts=2023-06-03T19:32:14.881Z caller=node_exporter.go:115 level=info collector=nfsd 2023-06-03T19:32:15.341 INFO:journalctl@ceph.node-exporter.b.smithi164.stdout:Jun 03 19:32:14 smithi164 bash[46901]: ts=2023-06-03T19:32:14.881Z caller=node_exporter.go:115 level=info collector=nvme 2023-06-03T19:32:15.341 INFO:journalctl@ceph.node-exporter.b.smithi164.stdout:Jun 03 19:32:14 smithi164 bash[46901]: ts=2023-06-03T19:32:14.881Z caller=node_exporter.go:115 level=info collector=os 2023-06-03T19:32:15.341 INFO:journalctl@ceph.node-exporter.b.smithi164.stdout:Jun 03 19:32:14 smithi164 bash[46901]: ts=2023-06-03T19:32:14.881Z caller=node_exporter.go:115 level=info collector=powersupplyclass 2023-06-03T19:32:15.341 INFO:journalctl@ceph.node-exporter.b.smithi164.stdout:Jun 03 19:32:14 smithi164 bash[46901]: ts=2023-06-03T19:32:14.881Z caller=node_exporter.go:115 level=info collector=pressure 2023-06-03T19:32:15.342 INFO:journalctl@ceph.node-exporter.b.smithi164.stdout:Jun 03 19:32:14 smithi164 bash[46901]: ts=2023-06-03T19:32:14.881Z caller=node_exporter.go:115 level=info collector=rapl 2023-06-03T19:32:15.342 INFO:journalctl@ceph.node-exporter.b.smithi164.stdout:Jun 03 19:32:14 smithi164 bash[46901]: ts=2023-06-03T19:32:14.881Z caller=node_exporter.go:115 level=info collector=schedstat 2023-06-03T19:32:15.342 INFO:journalctl@ceph.node-exporter.b.smithi164.stdout:Jun 03 19:32:14 smithi164 bash[46901]: ts=2023-06-03T19:32:14.881Z caller=node_exporter.go:115 level=info collector=sockstat 2023-06-03T19:32:15.342 INFO:journalctl@ceph.node-exporter.b.smithi164.stdout:Jun 03 19:32:14 smithi164 bash[46901]: ts=2023-06-03T19:32:14.881Z caller=node_exporter.go:115 level=info collector=softnet 2023-06-03T19:32:15.342 INFO:journalctl@ceph.node-exporter.b.smithi164.stdout:Jun 03 19:32:14 smithi164 bash[46901]: ts=2023-06-03T19:32:14.881Z caller=node_exporter.go:115 level=info collector=stat 2023-06-03T19:32:15.342 INFO:journalctl@ceph.node-exporter.b.smithi164.stdout:Jun 03 19:32:14 smithi164 bash[46901]: ts=2023-06-03T19:32:14.881Z caller=node_exporter.go:115 level=info collector=tapestats 2023-06-03T19:32:15.342 INFO:journalctl@ceph.node-exporter.b.smithi164.stdout:Jun 03 19:32:14 smithi164 bash[46901]: ts=2023-06-03T19:32:14.881Z caller=node_exporter.go:115 level=info collector=textfile 2023-06-03T19:32:15.343 INFO:journalctl@ceph.node-exporter.b.smithi164.stdout:Jun 03 19:32:14 smithi164 bash[46901]: ts=2023-06-03T19:32:14.881Z caller=node_exporter.go:115 level=info collector=thermal_zone 2023-06-03T19:32:15.343 INFO:journalctl@ceph.node-exporter.b.smithi164.stdout:Jun 03 19:32:14 smithi164 bash[46901]: ts=2023-06-03T19:32:14.881Z caller=node_exporter.go:115 level=info collector=time 2023-06-03T19:32:15.343 INFO:journalctl@ceph.node-exporter.b.smithi164.stdout:Jun 03 19:32:14 smithi164 bash[46901]: ts=2023-06-03T19:32:14.881Z caller=node_exporter.go:115 level=info collector=udp_queues 2023-06-03T19:32:15.343 INFO:journalctl@ceph.node-exporter.b.smithi164.stdout:Jun 03 19:32:14 smithi164 bash[46901]: ts=2023-06-03T19:32:14.881Z caller=node_exporter.go:115 level=info collector=uname 2023-06-03T19:32:15.343 INFO:journalctl@ceph.node-exporter.b.smithi164.stdout:Jun 03 19:32:14 smithi164 bash[46901]: ts=2023-06-03T19:32:14.881Z caller=node_exporter.go:115 level=info collector=vmstat 2023-06-03T19:32:15.343 INFO:journalctl@ceph.node-exporter.b.smithi164.stdout:Jun 03 19:32:14 smithi164 bash[46901]: ts=2023-06-03T19:32:14.881Z caller=node_exporter.go:115 level=info collector=xfs 2023-06-03T19:32:15.343 INFO:journalctl@ceph.node-exporter.b.smithi164.stdout:Jun 03 19:32:14 smithi164 bash[46901]: ts=2023-06-03T19:32:14.881Z caller=node_exporter.go:115 level=info collector=zfs 2023-06-03T19:32:15.344 INFO:journalctl@ceph.node-exporter.b.smithi164.stdout:Jun 03 19:32:14 smithi164 bash[46901]: ts=2023-06-03T19:32:14.881Z caller=node_exporter.go:199 level=info msg="Listening on" address=:9100 2023-06-03T19:32:15.344 INFO:journalctl@ceph.node-exporter.b.smithi164.stdout:Jun 03 19:32:14 smithi164 bash[46901]: ts=2023-06-03T19:32:14.881Z caller=tls_config.go:195 level=info msg="TLS is disabled." http2=false 2023-06-03T19:32:15.920 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:32:15 smithi089 bash[33173]: cluster 2023-06-03T19:32:14.379332+0000 mgr.y (mgr.14445) 28 : cluster [DBG] pgmap v16: 1 pgs: 1 active+clean; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:32:15.920 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:32:15 smithi089 bash[26572]: cluster 2023-06-03T19:32:14.379332+0000 mgr.y (mgr.14445) 28 : cluster [DBG] pgmap v16: 1 pgs: 1 active+clean; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:32:16.086 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:32:15 smithi164 bash[26855]: cluster 2023-06-03T19:32:14.379332+0000 mgr.y (mgr.14445) 28 : cluster [DBG] pgmap v16: 1 pgs: 1 active+clean; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:32:17.907 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:32:17 smithi089 bash[33173]: cluster 2023-06-03T19:32:16.379959+0000 mgr.y (mgr.14445) 29 : cluster [DBG] pgmap v17: 1 pgs: 1 active+clean; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:32:17.907 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:32:17 smithi089 bash[26572]: cluster 2023-06-03T19:32:16.379959+0000 mgr.y (mgr.14445) 29 : cluster [DBG] pgmap v17: 1 pgs: 1 active+clean; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:32:18.086 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:32:17 smithi164 bash[26855]: cluster 2023-06-03T19:32:16.379959+0000 mgr.y (mgr.14445) 29 : cluster [DBG] pgmap v17: 1 pgs: 1 active+clean; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:32:18.281 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 19:32:17 smithi089 bash[26845]: ::ffff:172.21.15.164 - - [03/Jun/2023:19:32:17] "GET /metrics HTTP/1.1" 200 149602 "" "Prometheus/2.33.4" 2023-06-03T19:32:19.799 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:32:19 smithi089 bash[26572]: cluster 2023-06-03T19:32:18.380528+0000 mgr.y (mgr.14445) 30 : cluster [DBG] pgmap v18: 1 pgs: 1 active+clean; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:32:19.799 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:32:19 smithi089 bash[33173]: cluster 2023-06-03T19:32:18.380528+0000 mgr.y (mgr.14445) 30 : cluster [DBG] pgmap v18: 1 pgs: 1 active+clean; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:32:20.086 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:32:19 smithi164 bash[26855]: cluster 2023-06-03T19:32:18.380528+0000 mgr.y (mgr.14445) 30 : cluster [DBG] pgmap v18: 1 pgs: 1 active+clean; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:32:20.713 INFO:teuthology.orchestra.run.smithi089.stderr:pool 'rbd' created 2023-06-03T19:32:21.031 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:32:20 smithi089 bash[26572]: audit 2023-06-03T19:32:20.219905+0000 mon.c (mon.2) 14 : audit [INF] from='client.? 172.21.15.89:0/999642200' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]: dispatch 2023-06-03T19:32:21.031 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:32:20 smithi089 bash[26572]: audit 2023-06-03T19:32:20.220673+0000 mon.a (mon.0) 597 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]: dispatch 2023-06-03T19:32:21.031 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:32:20 smithi089 bash[33173]: audit 2023-06-03T19:32:20.219905+0000 mon.c (mon.2) 14 : audit [INF] from='client.? 172.21.15.89:0/999642200' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]: dispatch 2023-06-03T19:32:21.031 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:32:20 smithi089 bash[33173]: audit 2023-06-03T19:32:20.220673+0000 mon.a (mon.0) 597 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]: dispatch 2023-06-03T19:32:21.086 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:32:20 smithi164 bash[26855]: audit 2023-06-03T19:32:20.219905+0000 mon.c (mon.2) 14 : audit [INF] from='client.? 172.21.15.89:0/999642200' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]: dispatch 2023-06-03T19:32:21.086 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:32:20 smithi164 bash[26855]: audit 2023-06-03T19:32:20.220673+0000 mon.a (mon.0) 597 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]: dispatch 2023-06-03T19:32:21.729 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:32:21 smithi089 bash[26572]: cluster 2023-06-03T19:32:20.380917+0000 mgr.y (mgr.14445) 31 : cluster [DBG] pgmap v19: 1 pgs: 1 active+clean; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:32:21.729 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:32:21 smithi089 bash[26572]: audit 2023-06-03T19:32:20.700900+0000 mon.a (mon.0) 598 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]': finished 2023-06-03T19:32:21.730 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:32:21 smithi089 bash[26572]: cluster 2023-06-03T19:32:20.700980+0000 mon.a (mon.0) 599 : cluster [DBG] osdmap e52: 8 total, 8 up, 8 in 2023-06-03T19:32:21.730 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:32:21 smithi089 bash[26572]: audit 2023-06-03T19:32:20.702086+0000 mon.a (mon.0) 600 : audit [INF] from='mgr.14445 172.21.15.89:0/573502486' entity='mgr.y' cmd=[{"prefix": "osd pool create", "format": "json", "pool": "device_health_metrics", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]: dispatch 2023-06-03T19:32:21.730 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:32:21 smithi089 bash[26572]: audit 2023-06-03T19:32:20.706033+0000 mon.a (mon.0) 601 : audit [INF] from='mgr.14445 172.21.15.89:0/573502486' entity='mgr.y' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": "device_health_metrics", "app": "mgr_devicehealth"}]: dispatch 2023-06-03T19:32:21.730 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:32:21 smithi089 bash[33173]: cluster 2023-06-03T19:32:20.380917+0000 mgr.y (mgr.14445) 31 : cluster [DBG] pgmap v19: 1 pgs: 1 active+clean; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:32:21.731 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:32:21 smithi089 bash[33173]: audit 2023-06-03T19:32:20.700900+0000 mon.a (mon.0) 598 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]': finished 2023-06-03T19:32:21.731 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:32:21 smithi089 bash[33173]: cluster 2023-06-03T19:32:20.700980+0000 mon.a (mon.0) 599 : cluster [DBG] osdmap e52: 8 total, 8 up, 8 in 2023-06-03T19:32:21.731 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:32:21 smithi089 bash[33173]: audit 2023-06-03T19:32:20.702086+0000 mon.a (mon.0) 600 : audit [INF] from='mgr.14445 172.21.15.89:0/573502486' entity='mgr.y' cmd=[{"prefix": "osd pool create", "format": "json", "pool": "device_health_metrics", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]: dispatch 2023-06-03T19:32:21.731 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:32:21 smithi089 bash[33173]: audit 2023-06-03T19:32:20.706033+0000 mon.a (mon.0) 601 : audit [INF] from='mgr.14445 172.21.15.89:0/573502486' entity='mgr.y' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": "device_health_metrics", "app": "mgr_devicehealth"}]: dispatch 2023-06-03T19:32:21.774 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- sudo ceph --cluster ceph osd pool application enable rbd rbd --yes-i-really-mean-it 2023-06-03T19:32:22.086 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:32:21 smithi164 bash[26855]: cluster 2023-06-03T19:32:20.380917+0000 mgr.y (mgr.14445) 31 : cluster [DBG] pgmap v19: 1 pgs: 1 active+clean; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:32:22.086 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:32:21 smithi164 bash[26855]: audit 2023-06-03T19:32:20.700900+0000 mon.a (mon.0) 598 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]': finished 2023-06-03T19:32:22.087 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:32:21 smithi164 bash[26855]: cluster 2023-06-03T19:32:20.700980+0000 mon.a (mon.0) 599 : cluster [DBG] osdmap e52: 8 total, 8 up, 8 in 2023-06-03T19:32:22.087 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:32:21 smithi164 bash[26855]: audit 2023-06-03T19:32:20.702086+0000 mon.a (mon.0) 600 : audit [INF] from='mgr.14445 172.21.15.89:0/573502486' entity='mgr.y' cmd=[{"prefix": "osd pool create", "format": "json", "pool": "device_health_metrics", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]: dispatch 2023-06-03T19:32:22.087 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:32:21 smithi164 bash[26855]: audit 2023-06-03T19:32:20.706033+0000 mon.a (mon.0) 601 : audit [INF] from='mgr.14445 172.21.15.89:0/573502486' entity='mgr.y' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": "device_health_metrics", "app": "mgr_devicehealth"}]: dispatch 2023-06-03T19:32:23.031 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:32:22 smithi089 bash[33173]: audit 2023-06-03T19:32:21.702669+0000 mon.a (mon.0) 602 : audit [INF] from='mgr.14445 172.21.15.89:0/573502486' entity='mgr.y' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": "device_health_metrics", "app": "mgr_devicehealth"}]': finished 2023-06-03T19:32:23.031 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:32:22 smithi089 bash[33173]: cluster 2023-06-03T19:32:21.702792+0000 mon.a (mon.0) 603 : cluster [DBG] osdmap e53: 8 total, 8 up, 8 in 2023-06-03T19:32:23.031 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:32:22 smithi089 bash[26572]: audit 2023-06-03T19:32:21.702669+0000 mon.a (mon.0) 602 : audit [INF] from='mgr.14445 172.21.15.89:0/573502486' entity='mgr.y' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": "device_health_metrics", "app": "mgr_devicehealth"}]': finished 2023-06-03T19:32:23.031 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:32:22 smithi089 bash[26572]: cluster 2023-06-03T19:32:21.702792+0000 mon.a (mon.0) 603 : cluster [DBG] osdmap e53: 8 total, 8 up, 8 in 2023-06-03T19:32:23.086 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:32:22 smithi164 bash[26855]: audit 2023-06-03T19:32:21.702669+0000 mon.a (mon.0) 602 : audit [INF] from='mgr.14445 172.21.15.89:0/573502486' entity='mgr.y' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": "device_health_metrics", "app": "mgr_devicehealth"}]': finished 2023-06-03T19:32:23.086 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:32:22 smithi164 bash[26855]: cluster 2023-06-03T19:32:21.702792+0000 mon.a (mon.0) 603 : cluster [DBG] osdmap e53: 8 total, 8 up, 8 in 2023-06-03T19:32:24.031 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:32:23 smithi089 bash[26572]: cluster 2023-06-03T19:32:22.381341+0000 mgr.y (mgr.14445) 32 : cluster [DBG] pgmap v22: 9 pgs: 8 unknown, 1 active+clean; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:32:24.031 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:32:23 smithi089 bash[26572]: cluster 2023-06-03T19:32:22.717190+0000 mon.a (mon.0) 604 : cluster [DBG] osdmap e54: 8 total, 8 up, 8 in 2023-06-03T19:32:24.031 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:32:23 smithi089 bash[26572]: audit 2023-06-03T19:32:23.369626+0000 mon.a (mon.0) 605 : audit [INF] from='mgr.14445 172.21.15.89:0/573502486' entity='mgr.y' 2023-06-03T19:32:24.032 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:32:23 smithi089 bash[26572]: audit 2023-06-03T19:32:23.371187+0000 mon.a (mon.0) 606 : audit [DBG] from='mgr.14445 172.21.15.89:0/573502486' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-03T19:32:24.032 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:32:23 smithi089 bash[26572]: audit 2023-06-03T19:32:23.617860+0000 mon.a (mon.0) 607 : audit [INF] from='mgr.14445 172.21.15.89:0/573502486' entity='mgr.y' 2023-06-03T19:32:24.032 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:32:23 smithi089 bash[33173]: cluster 2023-06-03T19:32:22.381341+0000 mgr.y (mgr.14445) 32 : cluster [DBG] pgmap v22: 9 pgs: 8 unknown, 1 active+clean; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:32:24.032 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:32:23 smithi089 bash[33173]: cluster 2023-06-03T19:32:22.717190+0000 mon.a (mon.0) 604 : cluster [DBG] osdmap e54: 8 total, 8 up, 8 in 2023-06-03T19:32:24.033 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:32:23 smithi089 bash[33173]: audit 2023-06-03T19:32:23.369626+0000 mon.a (mon.0) 605 : audit [INF] from='mgr.14445 172.21.15.89:0/573502486' entity='mgr.y' 2023-06-03T19:32:24.033 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:32:23 smithi089 bash[33173]: audit 2023-06-03T19:32:23.371187+0000 mon.a (mon.0) 606 : audit [DBG] from='mgr.14445 172.21.15.89:0/573502486' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-03T19:32:24.033 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:32:23 smithi089 bash[33173]: audit 2023-06-03T19:32:23.617860+0000 mon.a (mon.0) 607 : audit [INF] from='mgr.14445 172.21.15.89:0/573502486' entity='mgr.y' 2023-06-03T19:32:24.086 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:32:23 smithi164 bash[26855]: cluster 2023-06-03T19:32:22.381341+0000 mgr.y (mgr.14445) 32 : cluster [DBG] pgmap v22: 9 pgs: 8 unknown, 1 active+clean; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:32:24.086 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:32:23 smithi164 bash[26855]: cluster 2023-06-03T19:32:22.717190+0000 mon.a (mon.0) 604 : cluster [DBG] osdmap e54: 8 total, 8 up, 8 in 2023-06-03T19:32:24.087 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:32:23 smithi164 bash[26855]: audit 2023-06-03T19:32:23.369626+0000 mon.a (mon.0) 605 : audit [INF] from='mgr.14445 172.21.15.89:0/573502486' entity='mgr.y' 2023-06-03T19:32:24.087 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:32:23 smithi164 bash[26855]: audit 2023-06-03T19:32:23.371187+0000 mon.a (mon.0) 606 : audit [DBG] from='mgr.14445 172.21.15.89:0/573502486' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-03T19:32:24.087 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:32:23 smithi164 bash[26855]: audit 2023-06-03T19:32:23.617860+0000 mon.a (mon.0) 607 : audit [INF] from='mgr.14445 172.21.15.89:0/573502486' entity='mgr.y' 2023-06-03T19:32:25.948 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:32:25 smithi089 bash[26572]: cluster 2023-06-03T19:32:24.381859+0000 mgr.y (mgr.14445) 33 : cluster [DBG] pgmap v24: 9 pgs: 8 unknown, 1 active+clean; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:32:25.949 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:32:25 smithi089 bash[33173]: cluster 2023-06-03T19:32:24.381859+0000 mgr.y (mgr.14445) 33 : cluster [DBG] pgmap v24: 9 pgs: 8 unknown, 1 active+clean; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:32:25.949 INFO:journalctl@ceph.alertmanager.a.smithi089.stdout:Jun 03 19:32:25 smithi089 bash[52685]: level=info ts=2023-06-03T19:32:25.851Z caller=tls_config.go:191 msg="TLS is disabled." http2=false 2023-06-03T19:32:26.086 INFO:journalctl@ceph.mgr.x.smithi164.stdout:Jun 03 19:32:25 smithi164 bash[27934]: ::ffff:172.21.15.164 - - [03/Jun/2023:19:32:25] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2023-06-03T19:32:26.087 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:32:25 smithi164 bash[26855]: cluster 2023-06-03T19:32:24.381859+0000 mgr.y (mgr.14445) 33 : cluster [DBG] pgmap v24: 9 pgs: 8 unknown, 1 active+clean; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:32:26.746 INFO:teuthology.orchestra.run.smithi089.stderr:enabled application 'rbd' on pool 'rbd' 2023-06-03T19:32:27.031 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:32:26 smithi089 bash[33173]: audit 2023-06-03T19:32:26.315070+0000 mon.c (mon.2) 15 : audit [INF] from='client.? 172.21.15.89:0/285866441' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "rbd", "app": "rbd", "yes_i_really_mean_it": true}]: dispatch 2023-06-03T19:32:27.031 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:32:26 smithi089 bash[33173]: audit 2023-06-03T19:32:26.315829+0000 mon.a (mon.0) 608 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "rbd", "app": "rbd", "yes_i_really_mean_it": true}]: dispatch 2023-06-03T19:32:27.031 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:32:26 smithi089 bash[26572]: audit 2023-06-03T19:32:26.315070+0000 mon.c (mon.2) 15 : audit [INF] from='client.? 172.21.15.89:0/285866441' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "rbd", "app": "rbd", "yes_i_really_mean_it": true}]: dispatch 2023-06-03T19:32:27.032 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:32:26 smithi089 bash[26572]: audit 2023-06-03T19:32:26.315829+0000 mon.a (mon.0) 608 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "rbd", "app": "rbd", "yes_i_really_mean_it": true}]: dispatch 2023-06-03T19:32:27.086 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:32:26 smithi164 bash[26855]: audit 2023-06-03T19:32:26.315070+0000 mon.c (mon.2) 15 : audit [INF] from='client.? 172.21.15.89:0/285866441' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "rbd", "app": "rbd", "yes_i_really_mean_it": true}]: dispatch 2023-06-03T19:32:27.086 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:32:26 smithi164 bash[26855]: audit 2023-06-03T19:32:26.315829+0000 mon.a (mon.0) 608 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "rbd", "app": "rbd", "yes_i_really_mean_it": true}]: dispatch 2023-06-03T19:32:27.544 INFO:tasks.ceph:Waiting until ceph daemons up and pgs clean... 2023-06-03T19:32:27.545 INFO:tasks.cephadm.ceph_manager.ceph:waiting for mgr available 2023-06-03T19:32:27.546 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph mgr dump --format=json 2023-06-03T19:32:27.815 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:32:27 smithi089 bash[26572]: cluster 2023-06-03T19:32:26.382443+0000 mgr.y (mgr.14445) 34 : cluster [DBG] pgmap v25: 9 pgs: 9 active+clean; 0 B data, 43 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:32:27.815 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:32:27 smithi089 bash[26572]: audit 2023-06-03T19:32:26.732538+0000 mon.a (mon.0) 609 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "rbd", "app": "rbd", "yes_i_really_mean_it": true}]': finished 2023-06-03T19:32:27.816 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:32:27 smithi089 bash[26572]: cluster 2023-06-03T19:32:26.732616+0000 mon.a (mon.0) 610 : cluster [DBG] osdmap e55: 8 total, 8 up, 8 in 2023-06-03T19:32:27.816 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:32:27 smithi089 bash[33173]: cluster 2023-06-03T19:32:26.382443+0000 mgr.y (mgr.14445) 34 : cluster [DBG] pgmap v25: 9 pgs: 9 active+clean; 0 B data, 43 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:32:27.816 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:32:27 smithi089 bash[33173]: audit 2023-06-03T19:32:26.732538+0000 mon.a (mon.0) 609 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "rbd", "app": "rbd", "yes_i_really_mean_it": true}]': finished 2023-06-03T19:32:27.816 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:32:27 smithi089 bash[33173]: cluster 2023-06-03T19:32:26.732616+0000 mon.a (mon.0) 610 : cluster [DBG] osdmap e55: 8 total, 8 up, 8 in 2023-06-03T19:32:28.086 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:32:27 smithi164 bash[26855]: cluster 2023-06-03T19:32:26.382443+0000 mgr.y (mgr.14445) 34 : cluster [DBG] pgmap v25: 9 pgs: 9 active+clean; 0 B data, 43 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:32:28.086 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:32:27 smithi164 bash[26855]: audit 2023-06-03T19:32:26.732538+0000 mon.a (mon.0) 609 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "rbd", "app": "rbd", "yes_i_really_mean_it": true}]': finished 2023-06-03T19:32:28.086 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:32:27 smithi164 bash[26855]: cluster 2023-06-03T19:32:26.732616+0000 mon.a (mon.0) 610 : cluster [DBG] osdmap e55: 8 total, 8 up, 8 in 2023-06-03T19:32:28.281 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 19:32:27 smithi089 bash[26845]: ::ffff:172.21.15.164 - - [03/Jun/2023:19:32:27] "GET /metrics HTTP/1.1" 200 149600 "" "Prometheus/2.33.4" 2023-06-03T19:32:28.281 INFO:journalctl@ceph.alertmanager.a.smithi089.stdout:Jun 03 19:32:27 smithi089 bash[52685]: level=info ts=2023-06-03T19:32:27.808Z caller=cluster.go:696 component=cluster msg="gossip not settled" polls=0 before=0 now=1 elapsed=2.000758466s 2023-06-03T19:32:29.749 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 19:32:29 smithi089 bash[26845]: [03/Jun/2023:19:32:29] ENGINE Bus STOPPING 2023-06-03T19:32:29.749 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 19:32:29 smithi089 bash[26845]: [03/Jun/2023:19:32:29] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2023-06-03T19:32:29.749 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 19:32:29 smithi089 bash[26845]: [03/Jun/2023:19:32:29] ENGINE Bus STOPPED 2023-06-03T19:32:29.749 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 19:32:29 smithi089 bash[26845]: [03/Jun/2023:19:32:29] ENGINE Bus STARTING 2023-06-03T19:32:29.750 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 19:32:29 smithi089 bash[26845]: [03/Jun/2023:19:32:29] ENGINE Serving on http://:::9283 2023-06-03T19:32:29.750 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 19:32:29 smithi089 bash[26845]: [03/Jun/2023:19:32:29] ENGINE Bus STARTED 2023-06-03T19:32:30.031 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:32:29 smithi089 bash[26572]: cluster 2023-06-03T19:32:28.382991+0000 mgr.y (mgr.14445) 35 : cluster [DBG] pgmap v27: 9 pgs: 9 active+clean; 0 B data, 43 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:32:30.032 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:32:29 smithi089 bash[26572]: audit 2023-06-03T19:32:29.120054+0000 mon.a (mon.0) 611 : audit [INF] from='mgr.14445 172.21.15.89:0/573502486' entity='mgr.y' 2023-06-03T19:32:30.032 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:32:29 smithi089 bash[26572]: audit 2023-06-03T19:32:29.172094+0000 mon.a (mon.0) 612 : audit [INF] from='mgr.14445 172.21.15.89:0/573502486' entity='mgr.y' 2023-06-03T19:32:30.032 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:32:29 smithi089 bash[26572]: audit 2023-06-03T19:32:29.173590+0000 mon.a (mon.0) 613 : audit [DBG] from='mgr.14445 172.21.15.89:0/573502486' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-03T19:32:30.033 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:32:29 smithi089 bash[26572]: audit 2023-06-03T19:32:29.174778+0000 mon.a (mon.0) 614 : audit [INF] from='mgr.14445 172.21.15.89:0/573502486' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-03T19:32:30.033 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:32:29 smithi089 bash[26572]: audit 2023-06-03T19:32:29.183395+0000 mon.a (mon.0) 615 : audit [INF] from='mgr.14445 172.21.15.89:0/573502486' entity='mgr.y' 2023-06-03T19:32:30.033 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:32:29 smithi089 bash[26572]: audit 2023-06-03T19:32:29.243799+0000 mon.a (mon.0) 616 : audit [INF] from='mgr.14445 172.21.15.89:0/573502486' entity='mgr.y' 2023-06-03T19:32:30.034 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:32:29 smithi089 bash[26572]: audit 2023-06-03T19:32:29.250502+0000 mon.a (mon.0) 617 : audit [INF] from='mgr.14445 172.21.15.89:0/573502486' entity='mgr.y' 2023-06-03T19:32:30.034 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:32:29 smithi089 bash[26572]: audit 2023-06-03T19:32:29.256310+0000 mon.a (mon.0) 618 : audit [INF] from='mgr.14445 172.21.15.89:0/573502486' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2023-06-03T19:32:30.035 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:32:29 smithi089 bash[26572]: audit 2023-06-03T19:32:29.265209+0000 mon.a (mon.0) 619 : audit [INF] from='mgr.14445 172.21.15.89:0/573502486' entity='mgr.y' 2023-06-03T19:32:30.035 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:32:29 smithi089 bash[33173]: cluster 2023-06-03T19:32:28.382991+0000 mgr.y (mgr.14445) 35 : cluster [DBG] pgmap v27: 9 pgs: 9 active+clean; 0 B data, 43 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:32:30.036 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:32:29 smithi089 bash[33173]: audit 2023-06-03T19:32:29.120054+0000 mon.a (mon.0) 611 : audit [INF] from='mgr.14445 172.21.15.89:0/573502486' entity='mgr.y' 2023-06-03T19:32:30.036 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:32:29 smithi089 bash[33173]: audit 2023-06-03T19:32:29.172094+0000 mon.a (mon.0) 612 : audit [INF] from='mgr.14445 172.21.15.89:0/573502486' entity='mgr.y' 2023-06-03T19:32:30.036 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:32:29 smithi089 bash[33173]: audit 2023-06-03T19:32:29.173590+0000 mon.a (mon.0) 613 : audit [DBG] from='mgr.14445 172.21.15.89:0/573502486' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-03T19:32:30.037 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:32:29 smithi089 bash[33173]: audit 2023-06-03T19:32:29.174778+0000 mon.a (mon.0) 614 : audit [INF] from='mgr.14445 172.21.15.89:0/573502486' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-03T19:32:30.037 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:32:29 smithi089 bash[33173]: audit 2023-06-03T19:32:29.183395+0000 mon.a (mon.0) 615 : audit [INF] from='mgr.14445 172.21.15.89:0/573502486' entity='mgr.y' 2023-06-03T19:32:30.037 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:32:29 smithi089 bash[33173]: audit 2023-06-03T19:32:29.243799+0000 mon.a (mon.0) 616 : audit [INF] from='mgr.14445 172.21.15.89:0/573502486' entity='mgr.y' 2023-06-03T19:32:30.037 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:32:29 smithi089 bash[33173]: audit 2023-06-03T19:32:29.250502+0000 mon.a (mon.0) 617 : audit [INF] from='mgr.14445 172.21.15.89:0/573502486' entity='mgr.y' 2023-06-03T19:32:30.037 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:32:29 smithi089 bash[33173]: audit 2023-06-03T19:32:29.256310+0000 mon.a (mon.0) 618 : audit [INF] from='mgr.14445 172.21.15.89:0/573502486' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2023-06-03T19:32:30.038 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:32:29 smithi089 bash[33173]: audit 2023-06-03T19:32:29.265209+0000 mon.a (mon.0) 619 : audit [INF] from='mgr.14445 172.21.15.89:0/573502486' entity='mgr.y' 2023-06-03T19:32:30.086 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:32:29 smithi164 bash[26855]: cluster 2023-06-03T19:32:28.382991+0000 mgr.y (mgr.14445) 35 : cluster [DBG] pgmap v27: 9 pgs: 9 active+clean; 0 B data, 43 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:32:30.086 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:32:29 smithi164 bash[26855]: audit 2023-06-03T19:32:29.120054+0000 mon.a (mon.0) 611 : audit [INF] from='mgr.14445 172.21.15.89:0/573502486' entity='mgr.y' 2023-06-03T19:32:30.086 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:32:29 smithi164 bash[26855]: audit 2023-06-03T19:32:29.172094+0000 mon.a (mon.0) 612 : audit [INF] from='mgr.14445 172.21.15.89:0/573502486' entity='mgr.y' 2023-06-03T19:32:30.087 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:32:29 smithi164 bash[26855]: audit 2023-06-03T19:32:29.173590+0000 mon.a (mon.0) 613 : audit [DBG] from='mgr.14445 172.21.15.89:0/573502486' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-03T19:32:30.087 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:32:29 smithi164 bash[26855]: audit 2023-06-03T19:32:29.174778+0000 mon.a (mon.0) 614 : audit [INF] from='mgr.14445 172.21.15.89:0/573502486' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-03T19:32:30.087 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:32:29 smithi164 bash[26855]: audit 2023-06-03T19:32:29.183395+0000 mon.a (mon.0) 615 : audit [INF] from='mgr.14445 172.21.15.89:0/573502486' entity='mgr.y' 2023-06-03T19:32:30.087 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:32:29 smithi164 bash[26855]: audit 2023-06-03T19:32:29.243799+0000 mon.a (mon.0) 616 : audit [INF] from='mgr.14445 172.21.15.89:0/573502486' entity='mgr.y' 2023-06-03T19:32:30.087 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:32:29 smithi164 bash[26855]: audit 2023-06-03T19:32:29.250502+0000 mon.a (mon.0) 617 : audit [INF] from='mgr.14445 172.21.15.89:0/573502486' entity='mgr.y' 2023-06-03T19:32:30.087 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:32:29 smithi164 bash[26855]: audit 2023-06-03T19:32:29.256310+0000 mon.a (mon.0) 618 : audit [INF] from='mgr.14445 172.21.15.89:0/573502486' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2023-06-03T19:32:30.087 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:32:29 smithi164 bash[26855]: audit 2023-06-03T19:32:29.265209+0000 mon.a (mon.0) 619 : audit [INF] from='mgr.14445 172.21.15.89:0/573502486' entity='mgr.y' 2023-06-03T19:32:31.031 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:32:30 smithi089 bash[33173]: audit 2023-06-03T19:32:29.257102+0000 mgr.y (mgr.14445) 36 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2023-06-03T19:32:31.031 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:32:30 smithi089 bash[33173]: cephadm 2023-06-03T19:32:29.275519+0000 mgr.y (mgr.14445) 37 : cephadm [INF] Deploying daemon grafana.a on smithi164 2023-06-03T19:32:31.031 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:32:30 smithi089 bash[26572]: audit 2023-06-03T19:32:29.257102+0000 mgr.y (mgr.14445) 36 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2023-06-03T19:32:31.032 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:32:30 smithi089 bash[26572]: cephadm 2023-06-03T19:32:29.275519+0000 mgr.y (mgr.14445) 37 : cephadm [INF] Deploying daemon grafana.a on smithi164 2023-06-03T19:32:31.086 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:32:30 smithi164 bash[26855]: audit 2023-06-03T19:32:29.257102+0000 mgr.y (mgr.14445) 36 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2023-06-03T19:32:31.087 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:32:30 smithi164 bash[26855]: cephadm 2023-06-03T19:32:29.275519+0000 mgr.y (mgr.14445) 37 : cephadm [INF] Deploying daemon grafana.a on smithi164 2023-06-03T19:32:32.031 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:32:31 smithi089 bash[33173]: cluster 2023-06-03T19:32:30.383651+0000 mgr.y (mgr.14445) 38 : cluster [DBG] pgmap v28: 9 pgs: 9 active+clean; 0 B data, 43 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:32:32.031 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:32:31 smithi089 bash[26572]: cluster 2023-06-03T19:32:30.383651+0000 mgr.y (mgr.14445) 38 : cluster [DBG] pgmap v28: 9 pgs: 9 active+clean; 0 B data, 43 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:32:32.086 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:32:31 smithi164 bash[26855]: cluster 2023-06-03T19:32:30.383651+0000 mgr.y (mgr.14445) 38 : cluster [DBG] pgmap v28: 9 pgs: 9 active+clean; 0 B data, 43 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:32:32.241 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/mon.a/config 2023-06-03T19:32:34.031 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:32:33 smithi089 bash[26572]: cluster 2023-06-03T19:32:32.384158+0000 mgr.y (mgr.14445) 39 : cluster [DBG] pgmap v29: 9 pgs: 9 active+clean; 0 B data, 43 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:32:34.031 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:32:33 smithi089 bash[33173]: cluster 2023-06-03T19:32:32.384158+0000 mgr.y (mgr.14445) 39 : cluster [DBG] pgmap v29: 9 pgs: 9 active+clean; 0 B data, 43 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:32:34.086 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:32:33 smithi164 bash[26855]: cluster 2023-06-03T19:32:32.384158+0000 mgr.y (mgr.14445) 39 : cluster [DBG] pgmap v29: 9 pgs: 9 active+clean; 0 B data, 43 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:32:35.660 INFO:teuthology.orchestra.run.smithi089.stdout: 2023-06-03T19:32:36.031 INFO:journalctl@ceph.alertmanager.a.smithi089.stdout:Jun 03 19:32:35 smithi089 bash[52685]: level=info ts=2023-06-03T19:32:35.809Z caller=cluster.go:688 component=cluster msg="gossip settled; proceeding" elapsed=10.001894918s 2023-06-03T19:32:36.031 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:32:35 smithi089 bash[33173]: cluster 2023-06-03T19:32:34.384641+0000 mgr.y (mgr.14445) 40 : cluster [DBG] pgmap v30: 9 pgs: 9 active+clean; 0 B data, 43 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:32:36.031 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:32:35 smithi089 bash[33173]: audit 2023-06-03T19:32:35.645283+0000 mon.a (mon.0) 620 : audit [DBG] from='client.? 172.21.15.89:0/3334220620' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2023-06-03T19:32:36.032 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:32:35 smithi089 bash[26572]: cluster 2023-06-03T19:32:34.384641+0000 mgr.y (mgr.14445) 40 : cluster [DBG] pgmap v30: 9 pgs: 9 active+clean; 0 B data, 43 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:32:36.032 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:32:35 smithi089 bash[26572]: audit 2023-06-03T19:32:35.645283+0000 mon.a (mon.0) 620 : audit [DBG] from='client.? 172.21.15.89:0/3334220620' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2023-06-03T19:32:36.086 INFO:journalctl@ceph.mgr.x.smithi164.stdout:Jun 03 19:32:35 smithi164 bash[27934]: ::ffff:172.21.15.164 - - [03/Jun/2023:19:32:35] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2023-06-03T19:32:36.086 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:32:35 smithi164 bash[26855]: cluster 2023-06-03T19:32:34.384641+0000 mgr.y (mgr.14445) 40 : cluster [DBG] pgmap v30: 9 pgs: 9 active+clean; 0 B data, 43 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:32:36.086 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:32:35 smithi164 bash[26855]: audit 2023-06-03T19:32:35.645283+0000 mon.a (mon.0) 620 : audit [DBG] from='client.? 172.21.15.89:0/3334220620' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2023-06-03T19:32:36.420 INFO:teuthology.orchestra.run.smithi089.stdout:{"epoch":18,"active_gid":14445,"active_name":"y","active_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:6800","nonce":2009133143},{"type":"v1","addr":"172.21.15.89:6801","nonce":2009133143}]},"active_addr":"172.21.15.89:6801/2009133143","active_change":"2023-06-03T19:31:48.303850+0000","active_mgr_features":4540138314316775423,"available":true,"standbys":[{"gid":24355,"name":"x","mgr_features":4540138314316775423,"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2400","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"7","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","upmap"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.23.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/ceph-grafana:8.3.5","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"docker.io/library/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"docker.io/arcts/keepalived","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.3.1","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.33.4","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"cross_origin_url":{"name":"cross_origin_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"redirect_resolve_ip_addr":{"name":"redirect_resolve_ip_addr","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"str","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"str","level":"advanced","flags":0,"default_value":"30","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"str","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"str","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"noautoscale":{"name":"noautoscale","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"global autoscale flag","long_desc":"Option to turn on/off the autoscaler for all pools","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":1,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"str","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}]}],"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2400","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"7","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","upmap"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.23.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/ceph-grafana:8.3.5","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"docker.io/library/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"docker.io/arcts/keepalived","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.3.1","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.33.4","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"cross_origin_url":{"name":"cross_origin_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"redirect_resolve_ip_addr":{"name":"redirect_resolve_ip_addr","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"str","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"str","level":"advanced","flags":0,"default_value":"30","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"str","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"str","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"noautoscale":{"name":"noautoscale","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"global autoscale flag","long_desc":"Option to turn on/off the autoscaler for all pools","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":1,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"str","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}],"services":{"dashboard":"https://172.21.15.89:8443/","prometheus":"http://172.21.15.89:9283/"},"always_on_modules":{"nautilus":["balancer","crash","devicehealth","orchestrator_cli","progress","rbd_support","status","volumes"],"octopus":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"pacific":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"]},"last_failure_osd_epoch":51,"active_clients":[{"addrvec":[{"type":"v2","addr":"172.21.15.89:0","nonce":1591228500}]},{"addrvec":[{"type":"v2","addr":"172.21.15.89:0","nonce":2023636358}]}]} 2023-06-03T19:32:36.425 INFO:tasks.cephadm.ceph_manager.ceph:mgr available! 2023-06-03T19:32:36.426 INFO:tasks.cephadm.ceph_manager.ceph:waiting for all up 2023-06-03T19:32:36.426 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph osd dump --format=json 2023-06-03T19:32:38.031 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:32:37 smithi089 bash[26572]: cluster 2023-06-03T19:32:36.385242+0000 mgr.y (mgr.14445) 41 : cluster [DBG] pgmap v31: 9 pgs: 9 active+clean; 0 B data, 43 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:32:38.031 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 19:32:37 smithi089 bash[26845]: ::ffff:172.21.15.164 - - [03/Jun/2023:19:32:37] "GET /metrics HTTP/1.1" 200 152000 "" "Prometheus/2.33.4" 2023-06-03T19:32:38.032 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:32:37 smithi089 bash[33173]: cluster 2023-06-03T19:32:36.385242+0000 mgr.y (mgr.14445) 41 : cluster [DBG] pgmap v31: 9 pgs: 9 active+clean; 0 B data, 43 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:32:38.086 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:32:37 smithi164 bash[26855]: cluster 2023-06-03T19:32:36.385242+0000 mgr.y (mgr.14445) 41 : cluster [DBG] pgmap v31: 9 pgs: 9 active+clean; 0 B data, 43 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:32:40.031 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:32:39 smithi089 bash[33173]: cluster 2023-06-03T19:32:38.385837+0000 mgr.y (mgr.14445) 42 : cluster [DBG] pgmap v32: 9 pgs: 9 active+clean; 0 B data, 43 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:32:40.031 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:32:39 smithi089 bash[26572]: cluster 2023-06-03T19:32:38.385837+0000 mgr.y (mgr.14445) 42 : cluster [DBG] pgmap v32: 9 pgs: 9 active+clean; 0 B data, 43 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:32:40.086 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:32:39 smithi164 bash[26855]: cluster 2023-06-03T19:32:38.385837+0000 mgr.y (mgr.14445) 42 : cluster [DBG] pgmap v32: 9 pgs: 9 active+clean; 0 B data, 43 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:32:41.237 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/mon.a/config 2023-06-03T19:32:42.086 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:32:41 smithi164 bash[26855]: cluster 2023-06-03T19:32:40.386595+0000 mgr.y (mgr.14445) 43 : cluster [DBG] pgmap v33: 9 pgs: 9 active+clean; 0 B data, 43 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:32:42.281 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:32:41 smithi089 bash[26572]: cluster 2023-06-03T19:32:40.386595+0000 mgr.y (mgr.14445) 43 : cluster [DBG] pgmap v33: 9 pgs: 9 active+clean; 0 B data, 43 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:32:42.281 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:32:41 smithi089 bash[33173]: cluster 2023-06-03T19:32:40.386595+0000 mgr.y (mgr.14445) 43 : cluster [DBG] pgmap v33: 9 pgs: 9 active+clean; 0 B data, 43 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:32:44.086 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:32:43 smithi164 bash[26855]: cluster 2023-06-03T19:32:42.387198+0000 mgr.y (mgr.14445) 44 : cluster [DBG] pgmap v34: 9 pgs: 9 active+clean; 0 B data, 43 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:32:44.136 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:32:43 smithi089 bash[33173]: cluster 2023-06-03T19:32:42.387198+0000 mgr.y (mgr.14445) 44 : cluster [DBG] pgmap v34: 9 pgs: 9 active+clean; 0 B data, 43 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:32:44.137 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:32:43 smithi089 bash[26572]: cluster 2023-06-03T19:32:42.387198+0000 mgr.y (mgr.14445) 44 : cluster [DBG] pgmap v34: 9 pgs: 9 active+clean; 0 B data, 43 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:32:44.538 INFO:teuthology.orchestra.run.smithi089.stdout: 2023-06-03T19:32:44.539 INFO:teuthology.orchestra.run.smithi089.stdout:{"epoch":55,"fsid":"f1db3478-0243-11ee-9b1d-001a4aab830c","created":"2023-06-03T19:22:54.447184+0000","modified":"2023-06-03T19:32:26.720735+0000","last_up_change":"2023-06-03T19:31:13.424226+0000","last_in_change":"2023-06-03T19:30:46.909676+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":2,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"pacific","pools":[{"pool":1,"pool_name":"device_health_metrics","create_time":"2023-06-03T19:27:28.100610+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"53","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr_devicehealth":{}}},{"pool":2,"pool_name":"rbd","create_time":"2023-06-03T19:32:20.641846+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"55","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}}}],"osds":[{"osd":0,"uuid":"86063bfe-126e-46e7-93ed-c73813af165b","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":48,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:6802","nonce":1795715824},{"type":"v1","addr":"172.21.15.89:6803","nonce":1795715824}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:6804","nonce":1795715824},{"type":"v1","addr":"172.21.15.89:6805","nonce":1795715824}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:6808","nonce":1795715824},{"type":"v1","addr":"172.21.15.89:6809","nonce":1795715824}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:6806","nonce":1795715824},{"type":"v1","addr":"172.21.15.89:6807","nonce":1795715824}]},"public_addr":"172.21.15.89:6803/1795715824","cluster_addr":"172.21.15.89:6805/1795715824","heartbeat_back_addr":"172.21.15.89:6809/1795715824","heartbeat_front_addr":"172.21.15.89:6807/1795715824","state":["exists","up"]},{"osd":1,"uuid":"04783a81-6b8c-4657-8446-00efc872b313","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":52,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:6810","nonce":4020168282},{"type":"v1","addr":"172.21.15.89:6811","nonce":4020168282}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:6812","nonce":4020168282},{"type":"v1","addr":"172.21.15.89:6813","nonce":4020168282}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:6816","nonce":4020168282},{"type":"v1","addr":"172.21.15.89:6817","nonce":4020168282}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:6814","nonce":4020168282},{"type":"v1","addr":"172.21.15.89:6815","nonce":4020168282}]},"public_addr":"172.21.15.89:6811/4020168282","cluster_addr":"172.21.15.89:6813/4020168282","heartbeat_back_addr":"172.21.15.89:6817/4020168282","heartbeat_front_addr":"172.21.15.89:6815/4020168282","state":["exists","up"]},{"osd":2,"uuid":"d2d2c641-392a-444f-81bc-84eef6340f30","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":52,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:6818","nonce":2113694583},{"type":"v1","addr":"172.21.15.89:6819","nonce":2113694583}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:6820","nonce":2113694583},{"type":"v1","addr":"172.21.15.89:6821","nonce":2113694583}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:6824","nonce":2113694583},{"type":"v1","addr":"172.21.15.89:6825","nonce":2113694583}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:6822","nonce":2113694583},{"type":"v1","addr":"172.21.15.89:6823","nonce":2113694583}]},"public_addr":"172.21.15.89:6819/2113694583","cluster_addr":"172.21.15.89:6821/2113694583","heartbeat_back_addr":"172.21.15.89:6825/2113694583","heartbeat_front_addr":"172.21.15.89:6823/2113694583","state":["exists","up"]},{"osd":3,"uuid":"b38408eb-3acf-4a41-a314-72e4cd37b1ef","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":25,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:6826","nonce":3649114317},{"type":"v1","addr":"172.21.15.89:6827","nonce":3649114317}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:6828","nonce":3649114317},{"type":"v1","addr":"172.21.15.89:6829","nonce":3649114317}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:6832","nonce":3649114317},{"type":"v1","addr":"172.21.15.89:6833","nonce":3649114317}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:6830","nonce":3649114317},{"type":"v1","addr":"172.21.15.89:6831","nonce":3649114317}]},"public_addr":"172.21.15.89:6827/3649114317","cluster_addr":"172.21.15.89:6829/3649114317","heartbeat_back_addr":"172.21.15.89:6833/3649114317","heartbeat_front_addr":"172.21.15.89:6831/3649114317","state":["exists","up"]},{"osd":4,"uuid":"a94be785-9f54-4ab8-962a-9cf974a5b5e3","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":30,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.164:6800","nonce":538382489},{"type":"v1","addr":"172.21.15.164:6801","nonce":538382489}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.164:6802","nonce":538382489},{"type":"v1","addr":"172.21.15.164:6803","nonce":538382489}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.164:6806","nonce":538382489},{"type":"v1","addr":"172.21.15.164:6807","nonce":538382489}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.164:6804","nonce":538382489},{"type":"v1","addr":"172.21.15.164:6805","nonce":538382489}]},"public_addr":"172.21.15.164:6801/538382489","cluster_addr":"172.21.15.164:6803/538382489","heartbeat_back_addr":"172.21.15.164:6807/538382489","heartbeat_front_addr":"172.21.15.164:6805/538382489","state":["exists","up"]},{"osd":5,"uuid":"b8a5e8a1-23cc-442b-90aa-19bd0036dd3c","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":36,"up_thru":52,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.164:6808","nonce":3123006405},{"type":"v1","addr":"172.21.15.164:6809","nonce":3123006405}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.164:6810","nonce":3123006405},{"type":"v1","addr":"172.21.15.164:6811","nonce":3123006405}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.164:6814","nonce":3123006405},{"type":"v1","addr":"172.21.15.164:6815","nonce":3123006405}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.164:6812","nonce":3123006405},{"type":"v1","addr":"172.21.15.164:6813","nonce":3123006405}]},"public_addr":"172.21.15.164:6809/3123006405","cluster_addr":"172.21.15.164:6811/3123006405","heartbeat_back_addr":"172.21.15.164:6815/3123006405","heartbeat_front_addr":"172.21.15.164:6813/3123006405","state":["exists","up"]},{"osd":6,"uuid":"5c5edd3b-d92c-4f46-94ff-2d5828b2e732","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":42,"up_thru":52,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.164:6816","nonce":712403290},{"type":"v1","addr":"172.21.15.164:6817","nonce":712403290}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.164:6818","nonce":712403290},{"type":"v1","addr":"172.21.15.164:6819","nonce":712403290}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.164:6822","nonce":712403290},{"type":"v1","addr":"172.21.15.164:6823","nonce":712403290}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.164:6820","nonce":712403290},{"type":"v1","addr":"172.21.15.164:6821","nonce":712403290}]},"public_addr":"172.21.15.164:6817/712403290","cluster_addr":"172.21.15.164:6819/712403290","heartbeat_back_addr":"172.21.15.164:6823/712403290","heartbeat_front_addr":"172.21.15.164:6821/712403290","state":["exists","up"]},{"osd":7,"uuid":"bdff9290-e6c4-407e-9113-eb39696c0e27","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":48,"up_thru":52,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.164:6824","nonce":711997991},{"type":"v1","addr":"172.21.15.164:6825","nonce":711997991}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.164:6826","nonce":711997991},{"type":"v1","addr":"172.21.15.164:6827","nonce":711997991}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.164:6830","nonce":711997991},{"type":"v1","addr":"172.21.15.164:6831","nonce":711997991}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.164:6828","nonce":711997991},{"type":"v1","addr":"172.21.15.164:6829","nonce":711997991}]},"public_addr":"172.21.15.164:6825/711997991","cluster_addr":"172.21.15.164:6827/711997991","heartbeat_back_addr":"172.21.15.164:6831/711997991","heartbeat_front_addr":"172.21.15.164:6829/711997991","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-06-03T19:25:52.155714+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-06-03T19:26:38.554458+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-06-03T19:27:25.383237+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-06-03T19:28:13.272585+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-06-03T19:28:56.685710+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-06-03T19:29:41.572533+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-06-03T19:30:26.631983+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-06-03T19:31:12.018414+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.89:0/3944267183":"2023-06-04T19:23:55.626873+0000","172.21.15.89:0/470853232":"2023-06-04T19:31:48.303240+0000","172.21.15.89:6801/4116589411":"2023-06-04T19:23:55.626873+0000","172.21.15.89:6801/2256305550":"2023-06-04T19:31:48.303240+0000","172.21.15.89:6800/4116589411":"2023-06-04T19:23:55.626873+0000","172.21.15.89:0/274274920":"2023-06-04T19:23:55.626873+0000","172.21.15.89:0/1198702890":"2023-06-04T19:23:23.903333+0000","172.21.15.89:6800/1487566136":"2023-06-04T19:23:23.903333+0000","172.21.15.89:6800/2256305550":"2023-06-04T19:31:48.303240+0000","172.21.15.89:0/2392321634":"2023-06-04T19:31:48.303240+0000","172.21.15.89:6801/1487566136":"2023-06-04T19:23:23.903333+0000","172.21.15.89:0/811967393":"2023-06-04T19:23:23.903333+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2023-06-03T19:32:45.031 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:32:44 smithi089 bash[26572]: audit 2023-06-03T19:32:44.529777+0000 mon.c (mon.2) 16 : audit [DBG] from='client.? 172.21.15.89:0/1555516390' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-06-03T19:32:45.031 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:32:44 smithi089 bash[33173]: audit 2023-06-03T19:32:44.529777+0000 mon.c (mon.2) 16 : audit [DBG] from='client.? 172.21.15.89:0/1555516390' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-06-03T19:32:45.086 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:32:44 smithi164 bash[26855]: audit 2023-06-03T19:32:44.529777+0000 mon.c (mon.2) 16 : audit [DBG] from='client.? 172.21.15.89:0/1555516390' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-06-03T19:32:45.384 INFO:tasks.cephadm.ceph_manager.ceph:all up! 2023-06-03T19:32:45.385 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph osd dump --format=json 2023-06-03T19:32:46.086 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:32:45 smithi164 bash[26855]: cluster 2023-06-03T19:32:44.387909+0000 mgr.y (mgr.14445) 45 : cluster [DBG] pgmap v35: 9 pgs: 9 active+clean; 0 B data, 43 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:32:46.086 INFO:journalctl@ceph.mgr.x.smithi164.stdout:Jun 03 19:32:45 smithi164 bash[27934]: ::ffff:172.21.15.164 - - [03/Jun/2023:19:32:45] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2023-06-03T19:32:46.281 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:32:45 smithi089 bash[26572]: cluster 2023-06-03T19:32:44.387909+0000 mgr.y (mgr.14445) 45 : cluster [DBG] pgmap v35: 9 pgs: 9 active+clean; 0 B data, 43 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:32:46.281 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:32:45 smithi089 bash[33173]: cluster 2023-06-03T19:32:44.387909+0000 mgr.y (mgr.14445) 45 : cluster [DBG] pgmap v35: 9 pgs: 9 active+clean; 0 B data, 43 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:32:47.781 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:32:47 smithi089 bash[26572]: cluster 2023-06-03T19:32:46.388661+0000 mgr.y (mgr.14445) 46 : cluster [DBG] pgmap v36: 9 pgs: 9 active+clean; 0 B data, 43 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:32:47.781 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:32:47 smithi089 bash[33173]: cluster 2023-06-03T19:32:46.388661+0000 mgr.y (mgr.14445) 46 : cluster [DBG] pgmap v36: 9 pgs: 9 active+clean; 0 B data, 43 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:32:47.836 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:32:47 smithi164 bash[26855]: cluster 2023-06-03T19:32:46.388661+0000 mgr.y (mgr.14445) 46 : cluster [DBG] pgmap v36: 9 pgs: 9 active+clean; 0 B data, 43 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:32:48.281 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 19:32:47 smithi089 bash[26845]: ::ffff:172.21.15.164 - - [03/Jun/2023:19:32:47] "GET /metrics HTTP/1.1" 200 152000 "" "Prometheus/2.33.4" 2023-06-03T19:32:49.781 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:32:49 smithi089 bash[26572]: cluster 2023-06-03T19:32:48.389246+0000 mgr.y (mgr.14445) 47 : cluster [DBG] pgmap v37: 9 pgs: 9 active+clean; 0 B data, 43 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:32:49.781 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:32:49 smithi089 bash[33173]: cluster 2023-06-03T19:32:48.389246+0000 mgr.y (mgr.14445) 47 : cluster [DBG] pgmap v37: 9 pgs: 9 active+clean; 0 B data, 43 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:32:49.836 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:32:49 smithi164 bash[26855]: cluster 2023-06-03T19:32:48.389246+0000 mgr.y (mgr.14445) 47 : cluster [DBG] pgmap v37: 9 pgs: 9 active+clean; 0 B data, 43 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:32:50.196 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/mon.a/config 2023-06-03T19:32:51.781 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:32:51 smithi089 bash[26572]: cluster 2023-06-03T19:32:50.390065+0000 mgr.y (mgr.14445) 48 : cluster [DBG] pgmap v38: 9 pgs: 9 active+clean; 0 B data, 43 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:32:51.781 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:32:51 smithi089 bash[33173]: cluster 2023-06-03T19:32:50.390065+0000 mgr.y (mgr.14445) 48 : cluster [DBG] pgmap v38: 9 pgs: 9 active+clean; 0 B data, 43 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:32:51.836 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:32:51 smithi164 bash[26855]: cluster 2023-06-03T19:32:50.390065+0000 mgr.y (mgr.14445) 48 : cluster [DBG] pgmap v38: 9 pgs: 9 active+clean; 0 B data, 43 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:32:53.430 INFO:teuthology.orchestra.run.smithi089.stdout: 2023-06-03T19:32:53.430 INFO:teuthology.orchestra.run.smithi089.stdout:{"epoch":55,"fsid":"f1db3478-0243-11ee-9b1d-001a4aab830c","created":"2023-06-03T19:22:54.447184+0000","modified":"2023-06-03T19:32:26.720735+0000","last_up_change":"2023-06-03T19:31:13.424226+0000","last_in_change":"2023-06-03T19:30:46.909676+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":2,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"pacific","pools":[{"pool":1,"pool_name":"device_health_metrics","create_time":"2023-06-03T19:27:28.100610+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"53","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr_devicehealth":{}}},{"pool":2,"pool_name":"rbd","create_time":"2023-06-03T19:32:20.641846+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"55","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}}}],"osds":[{"osd":0,"uuid":"86063bfe-126e-46e7-93ed-c73813af165b","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":48,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:6802","nonce":1795715824},{"type":"v1","addr":"172.21.15.89:6803","nonce":1795715824}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:6804","nonce":1795715824},{"type":"v1","addr":"172.21.15.89:6805","nonce":1795715824}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:6808","nonce":1795715824},{"type":"v1","addr":"172.21.15.89:6809","nonce":1795715824}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:6806","nonce":1795715824},{"type":"v1","addr":"172.21.15.89:6807","nonce":1795715824}]},"public_addr":"172.21.15.89:6803/1795715824","cluster_addr":"172.21.15.89:6805/1795715824","heartbeat_back_addr":"172.21.15.89:6809/1795715824","heartbeat_front_addr":"172.21.15.89:6807/1795715824","state":["exists","up"]},{"osd":1,"uuid":"04783a81-6b8c-4657-8446-00efc872b313","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":52,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:6810","nonce":4020168282},{"type":"v1","addr":"172.21.15.89:6811","nonce":4020168282}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:6812","nonce":4020168282},{"type":"v1","addr":"172.21.15.89:6813","nonce":4020168282}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:6816","nonce":4020168282},{"type":"v1","addr":"172.21.15.89:6817","nonce":4020168282}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:6814","nonce":4020168282},{"type":"v1","addr":"172.21.15.89:6815","nonce":4020168282}]},"public_addr":"172.21.15.89:6811/4020168282","cluster_addr":"172.21.15.89:6813/4020168282","heartbeat_back_addr":"172.21.15.89:6817/4020168282","heartbeat_front_addr":"172.21.15.89:6815/4020168282","state":["exists","up"]},{"osd":2,"uuid":"d2d2c641-392a-444f-81bc-84eef6340f30","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":52,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:6818","nonce":2113694583},{"type":"v1","addr":"172.21.15.89:6819","nonce":2113694583}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:6820","nonce":2113694583},{"type":"v1","addr":"172.21.15.89:6821","nonce":2113694583}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:6824","nonce":2113694583},{"type":"v1","addr":"172.21.15.89:6825","nonce":2113694583}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:6822","nonce":2113694583},{"type":"v1","addr":"172.21.15.89:6823","nonce":2113694583}]},"public_addr":"172.21.15.89:6819/2113694583","cluster_addr":"172.21.15.89:6821/2113694583","heartbeat_back_addr":"172.21.15.89:6825/2113694583","heartbeat_front_addr":"172.21.15.89:6823/2113694583","state":["exists","up"]},{"osd":3,"uuid":"b38408eb-3acf-4a41-a314-72e4cd37b1ef","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":25,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:6826","nonce":3649114317},{"type":"v1","addr":"172.21.15.89:6827","nonce":3649114317}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:6828","nonce":3649114317},{"type":"v1","addr":"172.21.15.89:6829","nonce":3649114317}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:6832","nonce":3649114317},{"type":"v1","addr":"172.21.15.89:6833","nonce":3649114317}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:6830","nonce":3649114317},{"type":"v1","addr":"172.21.15.89:6831","nonce":3649114317}]},"public_addr":"172.21.15.89:6827/3649114317","cluster_addr":"172.21.15.89:6829/3649114317","heartbeat_back_addr":"172.21.15.89:6833/3649114317","heartbeat_front_addr":"172.21.15.89:6831/3649114317","state":["exists","up"]},{"osd":4,"uuid":"a94be785-9f54-4ab8-962a-9cf974a5b5e3","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":30,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.164:6800","nonce":538382489},{"type":"v1","addr":"172.21.15.164:6801","nonce":538382489}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.164:6802","nonce":538382489},{"type":"v1","addr":"172.21.15.164:6803","nonce":538382489}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.164:6806","nonce":538382489},{"type":"v1","addr":"172.21.15.164:6807","nonce":538382489}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.164:6804","nonce":538382489},{"type":"v1","addr":"172.21.15.164:6805","nonce":538382489}]},"public_addr":"172.21.15.164:6801/538382489","cluster_addr":"172.21.15.164:6803/538382489","heartbeat_back_addr":"172.21.15.164:6807/538382489","heartbeat_front_addr":"172.21.15.164:6805/538382489","state":["exists","up"]},{"osd":5,"uuid":"b8a5e8a1-23cc-442b-90aa-19bd0036dd3c","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":36,"up_thru":52,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.164:6808","nonce":3123006405},{"type":"v1","addr":"172.21.15.164:6809","nonce":3123006405}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.164:6810","nonce":3123006405},{"type":"v1","addr":"172.21.15.164:6811","nonce":3123006405}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.164:6814","nonce":3123006405},{"type":"v1","addr":"172.21.15.164:6815","nonce":3123006405}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.164:6812","nonce":3123006405},{"type":"v1","addr":"172.21.15.164:6813","nonce":3123006405}]},"public_addr":"172.21.15.164:6809/3123006405","cluster_addr":"172.21.15.164:6811/3123006405","heartbeat_back_addr":"172.21.15.164:6815/3123006405","heartbeat_front_addr":"172.21.15.164:6813/3123006405","state":["exists","up"]},{"osd":6,"uuid":"5c5edd3b-d92c-4f46-94ff-2d5828b2e732","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":42,"up_thru":52,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.164:6816","nonce":712403290},{"type":"v1","addr":"172.21.15.164:6817","nonce":712403290}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.164:6818","nonce":712403290},{"type":"v1","addr":"172.21.15.164:6819","nonce":712403290}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.164:6822","nonce":712403290},{"type":"v1","addr":"172.21.15.164:6823","nonce":712403290}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.164:6820","nonce":712403290},{"type":"v1","addr":"172.21.15.164:6821","nonce":712403290}]},"public_addr":"172.21.15.164:6817/712403290","cluster_addr":"172.21.15.164:6819/712403290","heartbeat_back_addr":"172.21.15.164:6823/712403290","heartbeat_front_addr":"172.21.15.164:6821/712403290","state":["exists","up"]},{"osd":7,"uuid":"bdff9290-e6c4-407e-9113-eb39696c0e27","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":48,"up_thru":52,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.164:6824","nonce":711997991},{"type":"v1","addr":"172.21.15.164:6825","nonce":711997991}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.164:6826","nonce":711997991},{"type":"v1","addr":"172.21.15.164:6827","nonce":711997991}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.164:6830","nonce":711997991},{"type":"v1","addr":"172.21.15.164:6831","nonce":711997991}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.164:6828","nonce":711997991},{"type":"v1","addr":"172.21.15.164:6829","nonce":711997991}]},"public_addr":"172.21.15.164:6825/711997991","cluster_addr":"172.21.15.164:6827/711997991","heartbeat_back_addr":"172.21.15.164:6831/711997991","heartbeat_front_addr":"172.21.15.164:6829/711997991","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-06-03T19:25:52.155714+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-06-03T19:26:38.554458+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-06-03T19:27:25.383237+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-06-03T19:28:13.272585+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-06-03T19:28:56.685710+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-06-03T19:29:41.572533+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-06-03T19:30:26.631983+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-06-03T19:31:12.018414+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.89:0/3944267183":"2023-06-04T19:23:55.626873+0000","172.21.15.89:0/470853232":"2023-06-04T19:31:48.303240+0000","172.21.15.89:6801/4116589411":"2023-06-04T19:23:55.626873+0000","172.21.15.89:6801/2256305550":"2023-06-04T19:31:48.303240+0000","172.21.15.89:6800/4116589411":"2023-06-04T19:23:55.626873+0000","172.21.15.89:0/274274920":"2023-06-04T19:23:55.626873+0000","172.21.15.89:0/1198702890":"2023-06-04T19:23:23.903333+0000","172.21.15.89:6800/1487566136":"2023-06-04T19:23:23.903333+0000","172.21.15.89:6800/2256305550":"2023-06-04T19:31:48.303240+0000","172.21.15.89:0/2392321634":"2023-06-04T19:31:48.303240+0000","172.21.15.89:6801/1487566136":"2023-06-04T19:23:23.903333+0000","172.21.15.89:0/811967393":"2023-06-04T19:23:23.903333+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2023-06-03T19:32:53.781 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:32:53 smithi089 bash[26572]: cluster 2023-06-03T19:32:52.390619+0000 mgr.y (mgr.14445) 49 : cluster [DBG] pgmap v39: 9 pgs: 9 active+clean; 0 B data, 43 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:32:53.781 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:32:53 smithi089 bash[26572]: audit 2023-06-03T19:32:53.421090+0000 mon.c (mon.2) 17 : audit [DBG] from='client.? 172.21.15.89:0/3914653768' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-06-03T19:32:53.782 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:32:53 smithi089 bash[33173]: cluster 2023-06-03T19:32:52.390619+0000 mgr.y (mgr.14445) 49 : cluster [DBG] pgmap v39: 9 pgs: 9 active+clean; 0 B data, 43 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:32:53.782 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:32:53 smithi089 bash[33173]: audit 2023-06-03T19:32:53.421090+0000 mon.c (mon.2) 17 : audit [DBG] from='client.? 172.21.15.89:0/3914653768' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-06-03T19:32:53.836 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:32:53 smithi164 bash[26855]: cluster 2023-06-03T19:32:52.390619+0000 mgr.y (mgr.14445) 49 : cluster [DBG] pgmap v39: 9 pgs: 9 active+clean; 0 B data, 43 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:32:53.836 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:32:53 smithi164 bash[26855]: audit 2023-06-03T19:32:53.421090+0000 mon.c (mon.2) 17 : audit [DBG] from='client.? 172.21.15.89:0/3914653768' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-06-03T19:32:54.137 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph tell osd.0 flush_pg_stats 2023-06-03T19:32:54.137 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph tell osd.1 flush_pg_stats 2023-06-03T19:32:54.138 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph tell osd.2 flush_pg_stats 2023-06-03T19:32:54.138 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph tell osd.3 flush_pg_stats 2023-06-03T19:32:54.139 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph tell osd.4 flush_pg_stats 2023-06-03T19:32:54.139 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph tell osd.5 flush_pg_stats 2023-06-03T19:32:54.140 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph tell osd.6 flush_pg_stats 2023-06-03T19:32:54.140 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph tell osd.7 flush_pg_stats 2023-06-03T19:32:55.781 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:32:55 smithi089 bash[33173]: cluster 2023-06-03T19:32:54.391019+0000 mgr.y (mgr.14445) 50 : cluster [DBG] pgmap v40: 9 pgs: 9 active+clean; 0 B data, 43 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:32:55.781 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:32:55 smithi089 bash[26572]: cluster 2023-06-03T19:32:54.391019+0000 mgr.y (mgr.14445) 50 : cluster [DBG] pgmap v40: 9 pgs: 9 active+clean; 0 B data, 43 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:32:55.836 INFO:journalctl@ceph.mgr.x.smithi164.stdout:Jun 03 19:32:55 smithi164 bash[27934]: ::ffff:172.21.15.164 - - [03/Jun/2023:19:32:55] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2023-06-03T19:32:55.836 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:32:55 smithi164 bash[26855]: cluster 2023-06-03T19:32:54.391019+0000 mgr.y (mgr.14445) 50 : cluster [DBG] pgmap v40: 9 pgs: 9 active+clean; 0 B data, 43 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:32:57.836 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:32:57 smithi164 bash[26855]: cluster 2023-06-03T19:32:56.391686+0000 mgr.y (mgr.14445) 51 : cluster [DBG] pgmap v41: 9 pgs: 9 active+clean; 0 B data, 43 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:32:57.908 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:32:57 smithi089 bash[33173]: cluster 2023-06-03T19:32:56.391686+0000 mgr.y (mgr.14445) 51 : cluster [DBG] pgmap v41: 9 pgs: 9 active+clean; 0 B data, 43 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:32:57.909 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:32:57 smithi089 bash[26572]: cluster 2023-06-03T19:32:56.391686+0000 mgr.y (mgr.14445) 51 : cluster [DBG] pgmap v41: 9 pgs: 9 active+clean; 0 B data, 43 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:32:58.281 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 19:32:57 smithi089 bash[26845]: ::ffff:172.21.15.164 - - [03/Jun/2023:19:32:57] "GET /metrics HTTP/1.1" 200 151997 "" "Prometheus/2.33.4" 2023-06-03T19:32:59.065 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/mon.a/config 2023-06-03T19:32:59.073 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/mon.a/config 2023-06-03T19:32:59.074 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/mon.a/config 2023-06-03T19:32:59.074 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/mon.a/config 2023-06-03T19:32:59.075 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/mon.a/config 2023-06-03T19:32:59.075 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/mon.a/config 2023-06-03T19:32:59.076 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/mon.a/config 2023-06-03T19:32:59.076 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/mon.a/config 2023-06-03T19:32:59.836 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:32:59 smithi164 bash[26855]: cluster 2023-06-03T19:32:58.392249+0000 mgr.y (mgr.14445) 52 : cluster [DBG] pgmap v42: 9 pgs: 9 active+clean; 0 B data, 43 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:33:00.031 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:32:59 smithi089 bash[33173]: cluster 2023-06-03T19:32:58.392249+0000 mgr.y (mgr.14445) 52 : cluster [DBG] pgmap v42: 9 pgs: 9 active+clean; 0 B data, 43 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:33:00.031 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:32:59 smithi089 bash[26572]: cluster 2023-06-03T19:32:58.392249+0000 mgr.y (mgr.14445) 52 : cluster [DBG] pgmap v42: 9 pgs: 9 active+clean; 0 B data, 43 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:33:01.836 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:33:01 smithi164 bash[26855]: cluster 2023-06-03T19:33:00.392777+0000 mgr.y (mgr.14445) 53 : cluster [DBG] pgmap v43: 9 pgs: 9 active+clean; 0 B data, 43 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:33:01.845 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:33:01 smithi089 bash[33173]: cluster 2023-06-03T19:33:00.392777+0000 mgr.y (mgr.14445) 53 : cluster [DBG] pgmap v43: 9 pgs: 9 active+clean; 0 B data, 43 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:33:01.846 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:33:01 smithi089 bash[26572]: cluster 2023-06-03T19:33:00.392777+0000 mgr.y (mgr.14445) 53 : cluster [DBG] pgmap v43: 9 pgs: 9 active+clean; 0 B data, 43 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:33:03.836 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:33:03 smithi164 bash[26855]: cluster 2023-06-03T19:33:02.393142+0000 mgr.y (mgr.14445) 54 : cluster [DBG] pgmap v44: 9 pgs: 9 active+clean; 0 B data, 43 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:33:03.843 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:33:03 smithi089 bash[26572]: cluster 2023-06-03T19:33:02.393142+0000 mgr.y (mgr.14445) 54 : cluster [DBG] pgmap v44: 9 pgs: 9 active+clean; 0 B data, 43 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:33:03.844 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:33:03 smithi089 bash[33173]: cluster 2023-06-03T19:33:02.393142+0000 mgr.y (mgr.14445) 54 : cluster [DBG] pgmap v44: 9 pgs: 9 active+clean; 0 B data, 43 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:33:05.836 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:33:05 smithi164 bash[26855]: cluster 2023-06-03T19:33:04.393663+0000 mgr.y (mgr.14445) 55 : cluster [DBG] pgmap v45: 9 pgs: 9 active+clean; 0 B data, 43 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:33:05.837 INFO:journalctl@ceph.mgr.x.smithi164.stdout:Jun 03 19:33:05 smithi164 bash[27934]: ::ffff:172.21.15.164 - - [03/Jun/2023:19:33:05] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2023-06-03T19:33:06.017 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:33:05 smithi089 bash[26572]: cluster 2023-06-03T19:33:04.393663+0000 mgr.y (mgr.14445) 55 : cluster [DBG] pgmap v45: 9 pgs: 9 active+clean; 0 B data, 43 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:33:06.018 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:33:05 smithi089 bash[33173]: cluster 2023-06-03T19:33:04.393663+0000 mgr.y (mgr.14445) 55 : cluster [DBG] pgmap v45: 9 pgs: 9 active+clean; 0 B data, 43 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:33:07.836 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:33:07 smithi164 bash[26855]: cluster 2023-06-03T19:33:06.394325+0000 mgr.y (mgr.14445) 56 : cluster [DBG] pgmap v46: 9 pgs: 9 active+clean; 0 B data, 43 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:33:07.908 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:33:07 smithi089 bash[26572]: cluster 2023-06-03T19:33:06.394325+0000 mgr.y (mgr.14445) 56 : cluster [DBG] pgmap v46: 9 pgs: 9 active+clean; 0 B data, 43 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:33:07.908 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:33:07 smithi089 bash[33173]: cluster 2023-06-03T19:33:06.394325+0000 mgr.y (mgr.14445) 56 : cluster [DBG] pgmap v46: 9 pgs: 9 active+clean; 0 B data, 43 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:33:08.281 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 19:33:07 smithi089 bash[26845]: ::ffff:172.21.15.164 - - [03/Jun/2023:19:33:07] "GET /metrics HTTP/1.1" 200 151993 "" "Prometheus/2.33.4" 2023-06-03T19:33:09.781 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:33:09 smithi089 bash[26572]: cluster 2023-06-03T19:33:08.394621+0000 mgr.y (mgr.14445) 57 : cluster [DBG] pgmap v47: 9 pgs: 9 active+clean; 0 B data, 43 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:33:09.781 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:33:09 smithi089 bash[26572]: audit 2023-06-03T19:33:09.293642+0000 mon.a (mon.0) 621 : audit [INF] from='mgr.14445 172.21.15.89:0/573502486' entity='mgr.y' 2023-06-03T19:33:09.781 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:33:09 smithi089 bash[26572]: audit 2023-06-03T19:33:09.296026+0000 mon.a (mon.0) 622 : audit [DBG] from='mgr.14445 172.21.15.89:0/573502486' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-03T19:33:09.782 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:33:09 smithi089 bash[33173]: cluster 2023-06-03T19:33:08.394621+0000 mgr.y (mgr.14445) 57 : cluster [DBG] pgmap v47: 9 pgs: 9 active+clean; 0 B data, 43 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:33:09.782 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:33:09 smithi089 bash[33173]: audit 2023-06-03T19:33:09.293642+0000 mon.a (mon.0) 621 : audit [INF] from='mgr.14445 172.21.15.89:0/573502486' entity='mgr.y' 2023-06-03T19:33:09.782 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:33:09 smithi089 bash[33173]: audit 2023-06-03T19:33:09.296026+0000 mon.a (mon.0) 622 : audit [DBG] from='mgr.14445 172.21.15.89:0/573502486' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-03T19:33:09.836 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:33:09 smithi164 bash[26855]: cluster 2023-06-03T19:33:08.394621+0000 mgr.y (mgr.14445) 57 : cluster [DBG] pgmap v47: 9 pgs: 9 active+clean; 0 B data, 43 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:33:09.836 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:33:09 smithi164 bash[26855]: audit 2023-06-03T19:33:09.293642+0000 mon.a (mon.0) 621 : audit [INF] from='mgr.14445 172.21.15.89:0/573502486' entity='mgr.y' 2023-06-03T19:33:09.836 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:33:09 smithi164 bash[26855]: audit 2023-06-03T19:33:09.296026+0000 mon.a (mon.0) 622 : audit [DBG] from='mgr.14445 172.21.15.89:0/573502486' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-03T19:33:10.682 INFO:journalctl@ceph.grafana.a.smithi164.stdout:Jun 03 19:33:10 smithi164 bash[47754]: t=2023-06-03T19:33:10+0000 lvl=info msg="Executing migration" logger=migrator id="create user table" 2023-06-03T19:33:10.959 INFO:journalctl@ceph.grafana.a.smithi164.stdout:Jun 03 19:33:10 smithi164 bash[47754]: t=2023-06-03T19:33:10+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index user.login" 2023-06-03T19:33:10.960 INFO:journalctl@ceph.grafana.a.smithi164.stdout:Jun 03 19:33:10 smithi164 bash[47754]: t=2023-06-03T19:33:10+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index user.email" 2023-06-03T19:33:10.960 INFO:journalctl@ceph.grafana.a.smithi164.stdout:Jun 03 19:33:10 smithi164 bash[47754]: t=2023-06-03T19:33:10+0000 lvl=info msg="Executing migration" logger=migrator id="drop index UQE_user_login - v1" 2023-06-03T19:33:10.960 INFO:journalctl@ceph.grafana.a.smithi164.stdout:Jun 03 19:33:10 smithi164 bash[47754]: t=2023-06-03T19:33:10+0000 lvl=info msg="Executing migration" logger=migrator id="drop index UQE_user_email - v1" 2023-06-03T19:33:10.960 INFO:journalctl@ceph.grafana.a.smithi164.stdout:Jun 03 19:33:10 smithi164 bash[47754]: t=2023-06-03T19:33:10+0000 lvl=info msg="Executing migration" logger=migrator id="Rename table user to user_v1 - v1" 2023-06-03T19:33:10.961 INFO:journalctl@ceph.grafana.a.smithi164.stdout:Jun 03 19:33:10 smithi164 bash[47754]: t=2023-06-03T19:33:10+0000 lvl=info msg="Executing migration" logger=migrator id="create user table v2" 2023-06-03T19:33:10.961 INFO:journalctl@ceph.grafana.a.smithi164.stdout:Jun 03 19:33:10 smithi164 bash[47754]: t=2023-06-03T19:33:10+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_user_login - v2" 2023-06-03T19:33:10.961 INFO:journalctl@ceph.grafana.a.smithi164.stdout:Jun 03 19:33:10 smithi164 bash[47754]: t=2023-06-03T19:33:10+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_user_email - v2" 2023-06-03T19:33:10.961 INFO:journalctl@ceph.grafana.a.smithi164.stdout:Jun 03 19:33:10 smithi164 bash[47754]: t=2023-06-03T19:33:10+0000 lvl=info msg="Executing migration" logger=migrator id="copy data_source v1 to v2" 2023-06-03T19:33:11.212 INFO:journalctl@ceph.grafana.a.smithi164.stdout:Jun 03 19:33:10 smithi164 bash[47754]: t=2023-06-03T19:33:10+0000 lvl=info msg="Executing migration" logger=migrator id="Drop old table user_v1" 2023-06-03T19:33:11.212 INFO:journalctl@ceph.grafana.a.smithi164.stdout:Jun 03 19:33:10 smithi164 bash[47754]: t=2023-06-03T19:33:10+0000 lvl=info msg="Executing migration" logger=migrator id="Add column help_flags1 to user table" 2023-06-03T19:33:11.212 INFO:journalctl@ceph.grafana.a.smithi164.stdout:Jun 03 19:33:11 smithi164 bash[47754]: t=2023-06-03T19:33:11+0000 lvl=info msg="Executing migration" logger=migrator id="Update user table charset" 2023-06-03T19:33:11.212 INFO:journalctl@ceph.grafana.a.smithi164.stdout:Jun 03 19:33:11 smithi164 bash[47754]: t=2023-06-03T19:33:11+0000 lvl=info msg="Executing migration" logger=migrator id="Add last_seen_at column to user" 2023-06-03T19:33:11.213 INFO:journalctl@ceph.grafana.a.smithi164.stdout:Jun 03 19:33:11 smithi164 bash[47754]: t=2023-06-03T19:33:11+0000 lvl=info msg="Executing migration" logger=migrator id="Add missing user data" 2023-06-03T19:33:11.213 INFO:journalctl@ceph.grafana.a.smithi164.stdout:Jun 03 19:33:11 smithi164 bash[47754]: t=2023-06-03T19:33:11+0000 lvl=info msg="Executing migration" logger=migrator id="Add is_disabled column to user" 2023-06-03T19:33:11.213 INFO:journalctl@ceph.grafana.a.smithi164.stdout:Jun 03 19:33:11 smithi164 bash[47754]: t=2023-06-03T19:33:11+0000 lvl=info msg="Executing migration" logger=migrator id="Add index user.login/user.email" 2023-06-03T19:33:11.213 INFO:journalctl@ceph.grafana.a.smithi164.stdout:Jun 03 19:33:11 smithi164 bash[47754]: t=2023-06-03T19:33:11+0000 lvl=info msg="Executing migration" logger=migrator id="Add is_service_account column to user" 2023-06-03T19:33:11.213 INFO:journalctl@ceph.grafana.a.smithi164.stdout:Jun 03 19:33:11 smithi164 bash[47754]: t=2023-06-03T19:33:11+0000 lvl=info msg="Executing migration" logger=migrator id="create temp user table v1-7" 2023-06-03T19:33:11.472 INFO:journalctl@ceph.grafana.a.smithi164.stdout:Jun 03 19:33:11 smithi164 bash[47754]: t=2023-06-03T19:33:11+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_temp_user_email - v1-7" 2023-06-03T19:33:11.473 INFO:journalctl@ceph.grafana.a.smithi164.stdout:Jun 03 19:33:11 smithi164 bash[47754]: t=2023-06-03T19:33:11+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_temp_user_org_id - v1-7" 2023-06-03T19:33:11.473 INFO:journalctl@ceph.grafana.a.smithi164.stdout:Jun 03 19:33:11 smithi164 bash[47754]: t=2023-06-03T19:33:11+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_temp_user_code - v1-7" 2023-06-03T19:33:11.473 INFO:journalctl@ceph.grafana.a.smithi164.stdout:Jun 03 19:33:11 smithi164 bash[47754]: t=2023-06-03T19:33:11+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_temp_user_status - v1-7" 2023-06-03T19:33:11.473 INFO:journalctl@ceph.grafana.a.smithi164.stdout:Jun 03 19:33:11 smithi164 bash[47754]: t=2023-06-03T19:33:11+0000 lvl=info msg="Executing migration" logger=migrator id="Update temp_user table charset" 2023-06-03T19:33:11.473 INFO:journalctl@ceph.grafana.a.smithi164.stdout:Jun 03 19:33:11 smithi164 bash[47754]: t=2023-06-03T19:33:11+0000 lvl=info msg="Executing migration" logger=migrator id="drop index IDX_temp_user_email - v1" 2023-06-03T19:33:11.473 INFO:journalctl@ceph.grafana.a.smithi164.stdout:Jun 03 19:33:11 smithi164 bash[47754]: t=2023-06-03T19:33:11+0000 lvl=info msg="Executing migration" logger=migrator id="drop index IDX_temp_user_org_id - v1" 2023-06-03T19:33:11.652 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:33:11 smithi089 bash[33173]: cluster 2023-06-03T19:33:10.395247+0000 mgr.y (mgr.14445) 58 : cluster [DBG] pgmap v48: 9 pgs: 9 active+clean; 0 B data, 43 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:33:11.652 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:33:11 smithi089 bash[26572]: cluster 2023-06-03T19:33:10.395247+0000 mgr.y (mgr.14445) 58 : cluster [DBG] pgmap v48: 9 pgs: 9 active+clean; 0 B data, 43 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:33:11.725 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:33:11 smithi164 bash[26855]: cluster 2023-06-03T19:33:10.395247+0000 mgr.y (mgr.14445) 58 : cluster [DBG] pgmap v48: 9 pgs: 9 active+clean; 0 B data, 43 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:33:11.725 INFO:journalctl@ceph.grafana.a.smithi164.stdout:Jun 03 19:33:11 smithi164 bash[47754]: t=2023-06-03T19:33:11+0000 lvl=info msg="Executing migration" logger=migrator id="drop index IDX_temp_user_code - v1" 2023-06-03T19:33:11.726 INFO:journalctl@ceph.grafana.a.smithi164.stdout:Jun 03 19:33:11 smithi164 bash[47754]: t=2023-06-03T19:33:11+0000 lvl=info msg="Executing migration" logger=migrator id="drop index IDX_temp_user_status - v1" 2023-06-03T19:33:11.726 INFO:journalctl@ceph.grafana.a.smithi164.stdout:Jun 03 19:33:11 smithi164 bash[47754]: t=2023-06-03T19:33:11+0000 lvl=info msg="Executing migration" logger=migrator id="Rename table temp_user to temp_user_tmp_qwerty - v1" 2023-06-03T19:33:11.726 INFO:journalctl@ceph.grafana.a.smithi164.stdout:Jun 03 19:33:11 smithi164 bash[47754]: t=2023-06-03T19:33:11+0000 lvl=info msg="Executing migration" logger=migrator id="create temp_user v2" 2023-06-03T19:33:11.726 INFO:journalctl@ceph.grafana.a.smithi164.stdout:Jun 03 19:33:11 smithi164 bash[47754]: t=2023-06-03T19:33:11+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_temp_user_email - v2" 2023-06-03T19:33:11.726 INFO:journalctl@ceph.grafana.a.smithi164.stdout:Jun 03 19:33:11 smithi164 bash[47754]: t=2023-06-03T19:33:11+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_temp_user_org_id - v2" 2023-06-03T19:33:11.726 INFO:journalctl@ceph.grafana.a.smithi164.stdout:Jun 03 19:33:11 smithi164 bash[47754]: t=2023-06-03T19:33:11+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_temp_user_code - v2" 2023-06-03T19:33:11.977 INFO:journalctl@ceph.grafana.a.smithi164.stdout:Jun 03 19:33:11 smithi164 bash[47754]: t=2023-06-03T19:33:11+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_temp_user_status - v2" 2023-06-03T19:33:11.977 INFO:journalctl@ceph.grafana.a.smithi164.stdout:Jun 03 19:33:11 smithi164 bash[47754]: t=2023-06-03T19:33:11+0000 lvl=info msg="Executing migration" logger=migrator id="copy temp_user v1 to v2" 2023-06-03T19:33:11.978 INFO:journalctl@ceph.grafana.a.smithi164.stdout:Jun 03 19:33:11 smithi164 bash[47754]: t=2023-06-03T19:33:11+0000 lvl=info msg="Executing migration" logger=migrator id="drop temp_user_tmp_qwerty" 2023-06-03T19:33:11.978 INFO:journalctl@ceph.grafana.a.smithi164.stdout:Jun 03 19:33:11 smithi164 bash[47754]: t=2023-06-03T19:33:11+0000 lvl=info msg="Executing migration" logger=migrator id="Set created for temp users that will otherwise prematurely expire" 2023-06-03T19:33:11.978 INFO:journalctl@ceph.grafana.a.smithi164.stdout:Jun 03 19:33:11 smithi164 bash[47754]: t=2023-06-03T19:33:11+0000 lvl=info msg="Executing migration" logger=migrator id="create star table" 2023-06-03T19:33:11.978 INFO:journalctl@ceph.grafana.a.smithi164.stdout:Jun 03 19:33:11 smithi164 bash[47754]: t=2023-06-03T19:33:11+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index star.user_id_dashboard_id" 2023-06-03T19:33:12.248 INFO:journalctl@ceph.grafana.a.smithi164.stdout:Jun 03 19:33:11 smithi164 bash[47754]: t=2023-06-03T19:33:11+0000 lvl=info msg="Executing migration" logger=migrator id="create org table v1" 2023-06-03T19:33:12.248 INFO:journalctl@ceph.grafana.a.smithi164.stdout:Jun 03 19:33:11 smithi164 bash[47754]: t=2023-06-03T19:33:11+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_org_name - v1" 2023-06-03T19:33:12.248 INFO:journalctl@ceph.grafana.a.smithi164.stdout:Jun 03 19:33:12 smithi164 bash[47754]: t=2023-06-03T19:33:12+0000 lvl=info msg="Executing migration" logger=migrator id="create org_user table v1" 2023-06-03T19:33:12.248 INFO:journalctl@ceph.grafana.a.smithi164.stdout:Jun 03 19:33:12 smithi164 bash[47754]: t=2023-06-03T19:33:12+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_org_user_org_id - v1" 2023-06-03T19:33:12.248 INFO:journalctl@ceph.grafana.a.smithi164.stdout:Jun 03 19:33:12 smithi164 bash[47754]: t=2023-06-03T19:33:12+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_org_user_org_id_user_id - v1" 2023-06-03T19:33:12.248 INFO:journalctl@ceph.grafana.a.smithi164.stdout:Jun 03 19:33:12 smithi164 bash[47754]: t=2023-06-03T19:33:12+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_org_user_user_id - v1" 2023-06-03T19:33:12.249 INFO:journalctl@ceph.grafana.a.smithi164.stdout:Jun 03 19:33:12 smithi164 bash[47754]: t=2023-06-03T19:33:12+0000 lvl=info msg="Executing migration" logger=migrator id="Update org table charset" 2023-06-03T19:33:12.249 INFO:journalctl@ceph.grafana.a.smithi164.stdout:Jun 03 19:33:12 smithi164 bash[47754]: t=2023-06-03T19:33:12+0000 lvl=info msg="Executing migration" logger=migrator id="Update org_user table charset" 2023-06-03T19:33:12.249 INFO:journalctl@ceph.grafana.a.smithi164.stdout:Jun 03 19:33:12 smithi164 bash[47754]: t=2023-06-03T19:33:12+0000 lvl=info msg="Executing migration" logger=migrator id="Migrate all Read Only Viewers to Viewers" 2023-06-03T19:33:12.249 INFO:journalctl@ceph.grafana.a.smithi164.stdout:Jun 03 19:33:12 smithi164 bash[47754]: t=2023-06-03T19:33:12+0000 lvl=info msg="Executing migration" logger=migrator id="create dashboard table" 2023-06-03T19:33:12.249 INFO:journalctl@ceph.grafana.a.smithi164.stdout:Jun 03 19:33:12 smithi164 bash[47754]: t=2023-06-03T19:33:12+0000 lvl=info msg="Executing migration" logger=migrator id="add index dashboard.account_id" 2023-06-03T19:33:12.249 INFO:journalctl@ceph.grafana.a.smithi164.stdout:Jun 03 19:33:12 smithi164 bash[47754]: t=2023-06-03T19:33:12+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index dashboard_account_id_slug" 2023-06-03T19:33:12.249 INFO:journalctl@ceph.grafana.a.smithi164.stdout:Jun 03 19:33:12 smithi164 bash[47754]: t=2023-06-03T19:33:12+0000 lvl=info msg="Executing migration" logger=migrator id="create dashboard_tag table" 2023-06-03T19:33:12.517 INFO:journalctl@ceph.grafana.a.smithi164.stdout:Jun 03 19:33:12 smithi164 bash[47754]: t=2023-06-03T19:33:12+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index dashboard_tag.dasboard_id_term" 2023-06-03T19:33:12.518 INFO:journalctl@ceph.grafana.a.smithi164.stdout:Jun 03 19:33:12 smithi164 bash[47754]: t=2023-06-03T19:33:12+0000 lvl=info msg="Executing migration" logger=migrator id="drop index UQE_dashboard_tag_dashboard_id_term - v1" 2023-06-03T19:33:12.518 INFO:journalctl@ceph.grafana.a.smithi164.stdout:Jun 03 19:33:12 smithi164 bash[47754]: t=2023-06-03T19:33:12+0000 lvl=info msg="Executing migration" logger=migrator id="Rename table dashboard to dashboard_v1 - v1" 2023-06-03T19:33:12.518 INFO:journalctl@ceph.grafana.a.smithi164.stdout:Jun 03 19:33:12 smithi164 bash[47754]: t=2023-06-03T19:33:12+0000 lvl=info msg="Executing migration" logger=migrator id="create dashboard v2" 2023-06-03T19:33:12.518 INFO:journalctl@ceph.grafana.a.smithi164.stdout:Jun 03 19:33:12 smithi164 bash[47754]: t=2023-06-03T19:33:12+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_dashboard_org_id - v2" 2023-06-03T19:33:12.518 INFO:journalctl@ceph.grafana.a.smithi164.stdout:Jun 03 19:33:12 smithi164 bash[47754]: t=2023-06-03T19:33:12+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_dashboard_org_id_slug - v2" 2023-06-03T19:33:12.518 INFO:journalctl@ceph.grafana.a.smithi164.stdout:Jun 03 19:33:12 smithi164 bash[47754]: t=2023-06-03T19:33:12+0000 lvl=info msg="Executing migration" logger=migrator id="copy dashboard v1 to v2" 2023-06-03T19:33:12.518 INFO:journalctl@ceph.grafana.a.smithi164.stdout:Jun 03 19:33:12 smithi164 bash[47754]: t=2023-06-03T19:33:12+0000 lvl=info msg="Executing migration" logger=migrator id="drop table dashboard_v1" 2023-06-03T19:33:12.803 INFO:journalctl@ceph.grafana.a.smithi164.stdout:Jun 03 19:33:12 smithi164 bash[47754]: t=2023-06-03T19:33:12+0000 lvl=info msg="Executing migration" logger=migrator id="alter dashboard.data to mediumtext v1" 2023-06-03T19:33:12.804 INFO:journalctl@ceph.grafana.a.smithi164.stdout:Jun 03 19:33:12 smithi164 bash[47754]: t=2023-06-03T19:33:12+0000 lvl=info msg="Executing migration" logger=migrator id="Add column updated_by in dashboard - v2" 2023-06-03T19:33:12.804 INFO:journalctl@ceph.grafana.a.smithi164.stdout:Jun 03 19:33:12 smithi164 bash[47754]: t=2023-06-03T19:33:12+0000 lvl=info msg="Executing migration" logger=migrator id="Add column created_by in dashboard - v2" 2023-06-03T19:33:12.804 INFO:journalctl@ceph.grafana.a.smithi164.stdout:Jun 03 19:33:12 smithi164 bash[47754]: t=2023-06-03T19:33:12+0000 lvl=info msg="Executing migration" logger=migrator id="Add column gnetId in dashboard" 2023-06-03T19:33:12.805 INFO:journalctl@ceph.grafana.a.smithi164.stdout:Jun 03 19:33:12 smithi164 bash[47754]: t=2023-06-03T19:33:12+0000 lvl=info msg="Executing migration" logger=migrator id="Add index for gnetId in dashboard" 2023-06-03T19:33:12.805 INFO:journalctl@ceph.grafana.a.smithi164.stdout:Jun 03 19:33:12 smithi164 bash[47754]: t=2023-06-03T19:33:12+0000 lvl=info msg="Executing migration" logger=migrator id="Add column plugin_id in dashboard" 2023-06-03T19:33:12.805 INFO:journalctl@ceph.grafana.a.smithi164.stdout:Jun 03 19:33:12 smithi164 bash[47754]: t=2023-06-03T19:33:12+0000 lvl=info msg="Executing migration" logger=migrator id="Add index for plugin_id in dashboard" 2023-06-03T19:33:12.805 INFO:journalctl@ceph.grafana.a.smithi164.stdout:Jun 03 19:33:12 smithi164 bash[47754]: t=2023-06-03T19:33:12+0000 lvl=info msg="Executing migration" logger=migrator id="Add index for dashboard_id in dashboard_tag" 2023-06-03T19:33:13.086 INFO:journalctl@ceph.grafana.a.smithi164.stdout:Jun 03 19:33:12 smithi164 bash[47754]: t=2023-06-03T19:33:12+0000 lvl=info msg="Executing migration" logger=migrator id="Update dashboard table charset" 2023-06-03T19:33:13.086 INFO:journalctl@ceph.grafana.a.smithi164.stdout:Jun 03 19:33:12 smithi164 bash[47754]: t=2023-06-03T19:33:12+0000 lvl=info msg="Executing migration" logger=migrator id="Update dashboard_tag table charset" 2023-06-03T19:33:13.087 INFO:journalctl@ceph.grafana.a.smithi164.stdout:Jun 03 19:33:12 smithi164 bash[47754]: t=2023-06-03T19:33:12+0000 lvl=info msg="Executing migration" logger=migrator id="Add column folder_id in dashboard" 2023-06-03T19:33:13.087 INFO:journalctl@ceph.grafana.a.smithi164.stdout:Jun 03 19:33:12 smithi164 bash[47754]: t=2023-06-03T19:33:12+0000 lvl=info msg="Executing migration" logger=migrator id="Add column isFolder in dashboard" 2023-06-03T19:33:13.087 INFO:journalctl@ceph.grafana.a.smithi164.stdout:Jun 03 19:33:12 smithi164 bash[47754]: t=2023-06-03T19:33:12+0000 lvl=info msg="Executing migration" logger=migrator id="Add column has_acl in dashboard" 2023-06-03T19:33:13.087 INFO:journalctl@ceph.grafana.a.smithi164.stdout:Jun 03 19:33:12 smithi164 bash[47754]: t=2023-06-03T19:33:12+0000 lvl=info msg="Executing migration" logger=migrator id="Add column uid in dashboard" 2023-06-03T19:33:13.087 INFO:journalctl@ceph.grafana.a.smithi164.stdout:Jun 03 19:33:12 smithi164 bash[47754]: t=2023-06-03T19:33:12+0000 lvl=info msg="Executing migration" logger=migrator id="Update uid column values in dashboard" 2023-06-03T19:33:13.087 INFO:journalctl@ceph.grafana.a.smithi164.stdout:Jun 03 19:33:13 smithi164 bash[47754]: t=2023-06-03T19:33:13+0000 lvl=info msg="Executing migration" logger=migrator id="Add unique index dashboard_org_id_uid" 2023-06-03T19:33:13.087 INFO:journalctl@ceph.grafana.a.smithi164.stdout:Jun 03 19:33:13 smithi164 bash[47754]: t=2023-06-03T19:33:13+0000 lvl=info msg="Executing migration" logger=migrator id="Remove unique index org_id_slug" 2023-06-03T19:33:13.358 INFO:journalctl@ceph.grafana.a.smithi164.stdout:Jun 03 19:33:13 smithi164 bash[47754]: t=2023-06-03T19:33:13+0000 lvl=info msg="Executing migration" logger=migrator id="Update dashboard title length" 2023-06-03T19:33:13.359 INFO:journalctl@ceph.grafana.a.smithi164.stdout:Jun 03 19:33:13 smithi164 bash[47754]: t=2023-06-03T19:33:13+0000 lvl=info msg="Executing migration" logger=migrator id="Add unique index for dashboard_org_id_title_folder_id" 2023-06-03T19:33:13.359 INFO:journalctl@ceph.grafana.a.smithi164.stdout:Jun 03 19:33:13 smithi164 bash[47754]: t=2023-06-03T19:33:13+0000 lvl=info msg="Executing migration" logger=migrator id="create dashboard_provisioning" 2023-06-03T19:33:13.360 INFO:journalctl@ceph.grafana.a.smithi164.stdout:Jun 03 19:33:13 smithi164 bash[47754]: t=2023-06-03T19:33:13+0000 lvl=info msg="Executing migration" logger=migrator id="Rename table dashboard_provisioning to dashboard_provisioning_tmp_qwerty - v1" 2023-06-03T19:33:13.360 INFO:journalctl@ceph.grafana.a.smithi164.stdout:Jun 03 19:33:13 smithi164 bash[47754]: t=2023-06-03T19:33:13+0000 lvl=info msg="Executing migration" logger=migrator id="create dashboard_provisioning v2" 2023-06-03T19:33:13.360 INFO:journalctl@ceph.grafana.a.smithi164.stdout:Jun 03 19:33:13 smithi164 bash[47754]: t=2023-06-03T19:33:13+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_dashboard_provisioning_dashboard_id - v2" 2023-06-03T19:33:13.360 INFO:journalctl@ceph.grafana.a.smithi164.stdout:Jun 03 19:33:13 smithi164 bash[47754]: t=2023-06-03T19:33:13+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_dashboard_provisioning_dashboard_id_name - v2" 2023-06-03T19:33:13.360 INFO:journalctl@ceph.grafana.a.smithi164.stdout:Jun 03 19:33:13 smithi164 bash[47754]: t=2023-06-03T19:33:13+0000 lvl=info msg="Executing migration" logger=migrator id="copy dashboard_provisioning v1 to v2" 2023-06-03T19:33:13.627 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:33:13 smithi164 bash[26855]: cluster 2023-06-03T19:33:12.395903+0000 mgr.y (mgr.14445) 59 : cluster [DBG] pgmap v49: 9 pgs: 9 active+clean; 0 B data, 43 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:33:13.628 INFO:journalctl@ceph.grafana.a.smithi164.stdout:Jun 03 19:33:13 smithi164 bash[47754]: t=2023-06-03T19:33:13+0000 lvl=info msg="Executing migration" logger=migrator id="drop dashboard_provisioning_tmp_qwerty" 2023-06-03T19:33:13.628 INFO:journalctl@ceph.grafana.a.smithi164.stdout:Jun 03 19:33:13 smithi164 bash[47754]: t=2023-06-03T19:33:13+0000 lvl=info msg="Executing migration" logger=migrator id="Add check_sum column" 2023-06-03T19:33:13.628 INFO:journalctl@ceph.grafana.a.smithi164.stdout:Jun 03 19:33:13 smithi164 bash[47754]: t=2023-06-03T19:33:13+0000 lvl=info msg="Executing migration" logger=migrator id="Add index for dashboard_title" 2023-06-03T19:33:13.628 INFO:journalctl@ceph.grafana.a.smithi164.stdout:Jun 03 19:33:13 smithi164 bash[47754]: t=2023-06-03T19:33:13+0000 lvl=info msg="Executing migration" logger=migrator id="delete tags for deleted dashboards" 2023-06-03T19:33:13.629 INFO:journalctl@ceph.grafana.a.smithi164.stdout:Jun 03 19:33:13 smithi164 bash[47754]: t=2023-06-03T19:33:13+0000 lvl=info msg="Executing migration" logger=migrator id="delete stars for deleted dashboards" 2023-06-03T19:33:13.629 INFO:journalctl@ceph.grafana.a.smithi164.stdout:Jun 03 19:33:13 smithi164 bash[47754]: t=2023-06-03T19:33:13+0000 lvl=info msg="Executing migration" logger=migrator id="Add index for dashboard_is_folder" 2023-06-03T19:33:13.629 INFO:journalctl@ceph.grafana.a.smithi164.stdout:Jun 03 19:33:13 smithi164 bash[47754]: t=2023-06-03T19:33:13+0000 lvl=info msg="Executing migration" logger=migrator id="create data_source table" 2023-06-03T19:33:13.629 INFO:journalctl@ceph.grafana.a.smithi164.stdout:Jun 03 19:33:13 smithi164 bash[47754]: t=2023-06-03T19:33:13+0000 lvl=info msg="Executing migration" logger=migrator id="add index data_source.account_id" 2023-06-03T19:33:13.662 INFO:teuthology.orchestra.run.smithi089.stdout:77309411398 2023-06-03T19:33:13.662 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph osd last-stat-seq osd.2 2023-06-03T19:33:13.888 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:33:13 smithi089 bash[33173]: cluster 2023-06-03T19:33:12.395903+0000 mgr.y (mgr.14445) 59 : cluster [DBG] pgmap v49: 9 pgs: 9 active+clean; 0 B data, 43 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:33:13.889 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:33:13 smithi089 bash[26572]: cluster 2023-06-03T19:33:12.395903+0000 mgr.y (mgr.14445) 59 : cluster [DBG] pgmap v49: 9 pgs: 9 active+clean; 0 B data, 43 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:33:13.890 INFO:teuthology.orchestra.run.smithi089.stdout:154618822700 2023-06-03T19:33:13.891 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph osd last-stat-seq osd.5 2023-06-03T19:33:13.905 INFO:journalctl@ceph.grafana.a.smithi164.stdout:Jun 03 19:33:13 smithi164 bash[47754]: t=2023-06-03T19:33:13+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index data_source.account_id_name" 2023-06-03T19:33:13.905 INFO:journalctl@ceph.grafana.a.smithi164.stdout:Jun 03 19:33:13 smithi164 bash[47754]: t=2023-06-03T19:33:13+0000 lvl=info msg="Executing migration" logger=migrator id="drop index IDX_data_source_account_id - v1" 2023-06-03T19:33:13.905 INFO:journalctl@ceph.grafana.a.smithi164.stdout:Jun 03 19:33:13 smithi164 bash[47754]: t=2023-06-03T19:33:13+0000 lvl=info msg="Executing migration" logger=migrator id="drop index UQE_data_source_account_id_name - v1" 2023-06-03T19:33:13.905 INFO:journalctl@ceph.grafana.a.smithi164.stdout:Jun 03 19:33:13 smithi164 bash[47754]: t=2023-06-03T19:33:13+0000 lvl=info msg="Executing migration" logger=migrator id="Rename table data_source to data_source_v1 - v1" 2023-06-03T19:33:13.906 INFO:journalctl@ceph.grafana.a.smithi164.stdout:Jun 03 19:33:13 smithi164 bash[47754]: t=2023-06-03T19:33:13+0000 lvl=info msg="Executing migration" logger=migrator id="create data_source table v2" 2023-06-03T19:33:13.906 INFO:journalctl@ceph.grafana.a.smithi164.stdout:Jun 03 19:33:13 smithi164 bash[47754]: t=2023-06-03T19:33:13+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_data_source_org_id - v2" 2023-06-03T19:33:13.906 INFO:journalctl@ceph.grafana.a.smithi164.stdout:Jun 03 19:33:13 smithi164 bash[47754]: t=2023-06-03T19:33:13+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_data_source_org_id_name - v2" 2023-06-03T19:33:13.906 INFO:journalctl@ceph.grafana.a.smithi164.stdout:Jun 03 19:33:13 smithi164 bash[47754]: t=2023-06-03T19:33:13+0000 lvl=info msg="Executing migration" logger=migrator id="copy data_source v1 to v2" 2023-06-03T19:33:13.907 INFO:journalctl@ceph.grafana.a.smithi164.stdout:Jun 03 19:33:13 smithi164 bash[47754]: t=2023-06-03T19:33:13+0000 lvl=info msg="Executing migration" logger=migrator id="Drop old table data_source_v1 #2" 2023-06-03T19:33:13.907 INFO:journalctl@ceph.grafana.a.smithi164.stdout:Jun 03 19:33:13 smithi164 bash[47754]: t=2023-06-03T19:33:13+0000 lvl=info msg="Executing migration" logger=migrator id="Add column with_credentials" 2023-06-03T19:33:14.049 INFO:teuthology.orchestra.run.smithi089.stdout:128849018933 2023-06-03T19:33:14.049 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph osd last-stat-seq osd.4 2023-06-03T19:33:14.166 INFO:journalctl@ceph.grafana.a.smithi164.stdout:Jun 03 19:33:13 smithi164 bash[47754]: t=2023-06-03T19:33:13+0000 lvl=info msg="Executing migration" logger=migrator id="Add secure json data column" 2023-06-03T19:33:14.166 INFO:journalctl@ceph.grafana.a.smithi164.stdout:Jun 03 19:33:13 smithi164 bash[47754]: t=2023-06-03T19:33:13+0000 lvl=info msg="Executing migration" logger=migrator id="Update data_source table charset" 2023-06-03T19:33:14.166 INFO:journalctl@ceph.grafana.a.smithi164.stdout:Jun 03 19:33:13 smithi164 bash[47754]: t=2023-06-03T19:33:13+0000 lvl=info msg="Executing migration" logger=migrator id="Update initial version to 1" 2023-06-03T19:33:14.166 INFO:journalctl@ceph.grafana.a.smithi164.stdout:Jun 03 19:33:14 smithi164 bash[47754]: t=2023-06-03T19:33:14+0000 lvl=info msg="Executing migration" logger=migrator id="Add read_only data column" 2023-06-03T19:33:14.167 INFO:journalctl@ceph.grafana.a.smithi164.stdout:Jun 03 19:33:14 smithi164 bash[47754]: t=2023-06-03T19:33:14+0000 lvl=info msg="Executing migration" logger=migrator id="Migrate logging ds to loki ds" 2023-06-03T19:33:14.167 INFO:journalctl@ceph.grafana.a.smithi164.stdout:Jun 03 19:33:14 smithi164 bash[47754]: t=2023-06-03T19:33:14+0000 lvl=info msg="Executing migration" logger=migrator id="Update json_data with nulls" 2023-06-03T19:33:14.167 INFO:journalctl@ceph.grafana.a.smithi164.stdout:Jun 03 19:33:14 smithi164 bash[47754]: t=2023-06-03T19:33:14+0000 lvl=info msg="Executing migration" logger=migrator id="Add uid column" 2023-06-03T19:33:14.167 INFO:journalctl@ceph.grafana.a.smithi164.stdout:Jun 03 19:33:14 smithi164 bash[47754]: t=2023-06-03T19:33:14+0000 lvl=info msg="Executing migration" logger=migrator id="Update uid value" 2023-06-03T19:33:14.167 INFO:journalctl@ceph.grafana.a.smithi164.stdout:Jun 03 19:33:14 smithi164 bash[47754]: t=2023-06-03T19:33:14+0000 lvl=info msg="Executing migration" logger=migrator id="Add unique index datasource_org_id_uid" 2023-06-03T19:33:14.435 INFO:journalctl@ceph.grafana.a.smithi164.stdout:Jun 03 19:33:14 smithi164 bash[47754]: t=2023-06-03T19:33:14+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index datasource_org_id_is_default" 2023-06-03T19:33:14.435 INFO:journalctl@ceph.grafana.a.smithi164.stdout:Jun 03 19:33:14 smithi164 bash[47754]: t=2023-06-03T19:33:14+0000 lvl=info msg="Executing migration" logger=migrator id="create api_key table" 2023-06-03T19:33:14.436 INFO:journalctl@ceph.grafana.a.smithi164.stdout:Jun 03 19:33:14 smithi164 bash[47754]: t=2023-06-03T19:33:14+0000 lvl=info msg="Executing migration" logger=migrator id="add index api_key.account_id" 2023-06-03T19:33:14.436 INFO:journalctl@ceph.grafana.a.smithi164.stdout:Jun 03 19:33:14 smithi164 bash[47754]: t=2023-06-03T19:33:14+0000 lvl=info msg="Executing migration" logger=migrator id="add index api_key.key" 2023-06-03T19:33:14.436 INFO:journalctl@ceph.grafana.a.smithi164.stdout:Jun 03 19:33:14 smithi164 bash[47754]: t=2023-06-03T19:33:14+0000 lvl=info msg="Executing migration" logger=migrator id="add index api_key.account_id_name" 2023-06-03T19:33:14.436 INFO:journalctl@ceph.grafana.a.smithi164.stdout:Jun 03 19:33:14 smithi164 bash[47754]: t=2023-06-03T19:33:14+0000 lvl=info msg="Executing migration" logger=migrator id="drop index IDX_api_key_account_id - v1" 2023-06-03T19:33:14.436 INFO:journalctl@ceph.grafana.a.smithi164.stdout:Jun 03 19:33:14 smithi164 bash[47754]: t=2023-06-03T19:33:14+0000 lvl=info msg="Executing migration" logger=migrator id="drop index UQE_api_key_key - v1" 2023-06-03T19:33:14.436 INFO:journalctl@ceph.grafana.a.smithi164.stdout:Jun 03 19:33:14 smithi164 bash[47754]: t=2023-06-03T19:33:14+0000 lvl=info msg="Executing migration" logger=migrator id="drop index UQE_api_key_account_id_name - v1" 2023-06-03T19:33:14.695 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:33:14 smithi164 bash[26855]: audit 2023-06-03T19:33:13.659613+0000 mon.a (mon.0) 623 : audit [INF] from='mgr.14445 172.21.15.89:0/573502486' entity='mgr.y' 2023-06-03T19:33:14.696 INFO:journalctl@ceph.grafana.a.smithi164.stdout:Jun 03 19:33:14 smithi164 bash[47754]: t=2023-06-03T19:33:14+0000 lvl=info msg="Executing migration" logger=migrator id="Rename table api_key to api_key_v1 - v1" 2023-06-03T19:33:14.696 INFO:journalctl@ceph.grafana.a.smithi164.stdout:Jun 03 19:33:14 smithi164 bash[47754]: t=2023-06-03T19:33:14+0000 lvl=info msg="Executing migration" logger=migrator id="create api_key table v2" 2023-06-03T19:33:14.696 INFO:journalctl@ceph.grafana.a.smithi164.stdout:Jun 03 19:33:14 smithi164 bash[47754]: t=2023-06-03T19:33:14+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_api_key_org_id - v2" 2023-06-03T19:33:14.696 INFO:journalctl@ceph.grafana.a.smithi164.stdout:Jun 03 19:33:14 smithi164 bash[47754]: t=2023-06-03T19:33:14+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_api_key_key - v2" 2023-06-03T19:33:14.696 INFO:journalctl@ceph.grafana.a.smithi164.stdout:Jun 03 19:33:14 smithi164 bash[47754]: t=2023-06-03T19:33:14+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_api_key_org_id_name - v2" 2023-06-03T19:33:14.697 INFO:journalctl@ceph.grafana.a.smithi164.stdout:Jun 03 19:33:14 smithi164 bash[47754]: t=2023-06-03T19:33:14+0000 lvl=info msg="Executing migration" logger=migrator id="copy api_key v1 to v2" 2023-06-03T19:33:14.697 INFO:journalctl@ceph.grafana.a.smithi164.stdout:Jun 03 19:33:14 smithi164 bash[47754]: t=2023-06-03T19:33:14+0000 lvl=info msg="Executing migration" logger=migrator id="Drop old table api_key_v1" 2023-06-03T19:33:14.860 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:33:14 smithi089 bash[33173]: audit 2023-06-03T19:33:13.659613+0000 mon.a (mon.0) 623 : audit [INF] from='mgr.14445 172.21.15.89:0/573502486' entity='mgr.y' 2023-06-03T19:33:14.860 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:33:14 smithi089 bash[26572]: audit 2023-06-03T19:33:13.659613+0000 mon.a (mon.0) 623 : audit [INF] from='mgr.14445 172.21.15.89:0/573502486' entity='mgr.y' 2023-06-03T19:33:14.948 INFO:journalctl@ceph.grafana.a.smithi164.stdout:Jun 03 19:33:14 smithi164 bash[47754]: t=2023-06-03T19:33:14+0000 lvl=info msg="Executing migration" logger=migrator id="Update api_key table charset" 2023-06-03T19:33:14.948 INFO:journalctl@ceph.grafana.a.smithi164.stdout:Jun 03 19:33:14 smithi164 bash[47754]: t=2023-06-03T19:33:14+0000 lvl=info msg="Executing migration" logger=migrator id="Add expires to api_key table" 2023-06-03T19:33:14.948 INFO:journalctl@ceph.grafana.a.smithi164.stdout:Jun 03 19:33:14 smithi164 bash[47754]: t=2023-06-03T19:33:14+0000 lvl=info msg="Executing migration" logger=migrator id="Add service account foreign key" 2023-06-03T19:33:14.949 INFO:journalctl@ceph.grafana.a.smithi164.stdout:Jun 03 19:33:14 smithi164 bash[47754]: t=2023-06-03T19:33:14+0000 lvl=info msg="Executing migration" logger=migrator id="create dashboard_snapshot table v4" 2023-06-03T19:33:14.949 INFO:journalctl@ceph.grafana.a.smithi164.stdout:Jun 03 19:33:14 smithi164 bash[47754]: t=2023-06-03T19:33:14+0000 lvl=info msg="Executing migration" logger=migrator id="drop table dashboard_snapshot_v4 #1" 2023-06-03T19:33:14.949 INFO:journalctl@ceph.grafana.a.smithi164.stdout:Jun 03 19:33:14 smithi164 bash[47754]: t=2023-06-03T19:33:14+0000 lvl=info msg="Executing migration" logger=migrator id="create dashboard_snapshot table v5 #2" 2023-06-03T19:33:14.949 INFO:journalctl@ceph.grafana.a.smithi164.stdout:Jun 03 19:33:14 smithi164 bash[47754]: t=2023-06-03T19:33:14+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_dashboard_snapshot_key - v5" 2023-06-03T19:33:14.949 INFO:journalctl@ceph.grafana.a.smithi164.stdout:Jun 03 19:33:14 smithi164 bash[47754]: t=2023-06-03T19:33:14+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_dashboard_snapshot_delete_key - v5" 2023-06-03T19:33:15.251 INFO:journalctl@ceph.grafana.a.smithi164.stdout:Jun 03 19:33:14 smithi164 bash[47754]: t=2023-06-03T19:33:14+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_dashboard_snapshot_user_id - v5" 2023-06-03T19:33:15.251 INFO:journalctl@ceph.grafana.a.smithi164.stdout:Jun 03 19:33:14 smithi164 bash[47754]: t=2023-06-03T19:33:14+0000 lvl=info msg="Executing migration" logger=migrator id="alter dashboard_snapshot to mediumtext v2" 2023-06-03T19:33:15.252 INFO:journalctl@ceph.grafana.a.smithi164.stdout:Jun 03 19:33:15 smithi164 bash[47754]: t=2023-06-03T19:33:15+0000 lvl=info msg="Executing migration" logger=migrator id="Update dashboard_snapshot table charset" 2023-06-03T19:33:15.252 INFO:journalctl@ceph.grafana.a.smithi164.stdout:Jun 03 19:33:15 smithi164 bash[47754]: t=2023-06-03T19:33:15+0000 lvl=info msg="Executing migration" logger=migrator id="Add column external_delete_url to dashboard_snapshots table" 2023-06-03T19:33:15.252 INFO:journalctl@ceph.grafana.a.smithi164.stdout:Jun 03 19:33:15 smithi164 bash[47754]: t=2023-06-03T19:33:15+0000 lvl=info msg="Executing migration" logger=migrator id="Add encrypted dashboard json column" 2023-06-03T19:33:15.252 INFO:journalctl@ceph.grafana.a.smithi164.stdout:Jun 03 19:33:15 smithi164 bash[47754]: t=2023-06-03T19:33:15+0000 lvl=info msg="Executing migration" logger=migrator id="Change dashboard_encrypted column to MEDIUMBLOB" 2023-06-03T19:33:15.252 INFO:journalctl@ceph.grafana.a.smithi164.stdout:Jun 03 19:33:15 smithi164 bash[47754]: t=2023-06-03T19:33:15+0000 lvl=info msg="Executing migration" logger=migrator id="create quota table v1" 2023-06-03T19:33:15.252 INFO:journalctl@ceph.grafana.a.smithi164.stdout:Jun 03 19:33:15 smithi164 bash[47754]: t=2023-06-03T19:33:15+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_quota_org_id_user_id_target - v1" 2023-06-03T19:33:15.253 INFO:journalctl@ceph.grafana.a.smithi164.stdout:Jun 03 19:33:15 smithi164 bash[47754]: t=2023-06-03T19:33:15+0000 lvl=info msg="Executing migration" logger=migrator id="Update quota table charset" 2023-06-03T19:33:15.528 INFO:journalctl@ceph.grafana.a.smithi164.stdout:Jun 03 19:33:15 smithi164 bash[47754]: t=2023-06-03T19:33:15+0000 lvl=info msg="Executing migration" logger=migrator id="create plugin_setting table" 2023-06-03T19:33:15.529 INFO:journalctl@ceph.grafana.a.smithi164.stdout:Jun 03 19:33:15 smithi164 bash[47754]: t=2023-06-03T19:33:15+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_plugin_setting_org_id_plugin_id - v1" 2023-06-03T19:33:15.529 INFO:journalctl@ceph.grafana.a.smithi164.stdout:Jun 03 19:33:15 smithi164 bash[47754]: t=2023-06-03T19:33:15+0000 lvl=info msg="Executing migration" logger=migrator id="Add column plugin_version to plugin_settings" 2023-06-03T19:33:15.529 INFO:journalctl@ceph.grafana.a.smithi164.stdout:Jun 03 19:33:15 smithi164 bash[47754]: t=2023-06-03T19:33:15+0000 lvl=info msg="Executing migration" logger=migrator id="Update plugin_setting table charset" 2023-06-03T19:33:15.529 INFO:journalctl@ceph.grafana.a.smithi164.stdout:Jun 03 19:33:15 smithi164 bash[47754]: t=2023-06-03T19:33:15+0000 lvl=info msg="Executing migration" logger=migrator id="create session table" 2023-06-03T19:33:15.529 INFO:journalctl@ceph.grafana.a.smithi164.stdout:Jun 03 19:33:15 smithi164 bash[47754]: t=2023-06-03T19:33:15+0000 lvl=info msg="Executing migration" logger=migrator id="Drop old table playlist table" 2023-06-03T19:33:15.814 INFO:journalctl@ceph.mgr.x.smithi164.stdout:Jun 03 19:33:15 smithi164 bash[27934]: ::ffff:172.21.15.164 - - [03/Jun/2023:19:33:15] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2023-06-03T19:33:15.815 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:33:15 smithi164 bash[26855]: cluster 2023-06-03T19:33:14.396283+0000 mgr.y (mgr.14445) 60 : cluster [DBG] pgmap v50: 9 pgs: 9 active+clean; 0 B data, 43 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:33:15.815 INFO:journalctl@ceph.grafana.a.smithi164.stdout:Jun 03 19:33:15 smithi164 bash[47754]: t=2023-06-03T19:33:15+0000 lvl=info msg="Executing migration" logger=migrator id="Drop old table playlist_item table" 2023-06-03T19:33:15.815 INFO:journalctl@ceph.grafana.a.smithi164.stdout:Jun 03 19:33:15 smithi164 bash[47754]: t=2023-06-03T19:33:15+0000 lvl=info msg="Executing migration" logger=migrator id="create playlist table v2" 2023-06-03T19:33:15.816 INFO:journalctl@ceph.grafana.a.smithi164.stdout:Jun 03 19:33:15 smithi164 bash[47754]: t=2023-06-03T19:33:15+0000 lvl=info msg="Executing migration" logger=migrator id="create playlist item table v2" 2023-06-03T19:33:15.816 INFO:journalctl@ceph.grafana.a.smithi164.stdout:Jun 03 19:33:15 smithi164 bash[47754]: t=2023-06-03T19:33:15+0000 lvl=info msg="Executing migration" logger=migrator id="Update playlist table charset" 2023-06-03T19:33:15.816 INFO:journalctl@ceph.grafana.a.smithi164.stdout:Jun 03 19:33:15 smithi164 bash[47754]: t=2023-06-03T19:33:15+0000 lvl=info msg="Executing migration" logger=migrator id="Update playlist_item table charset" 2023-06-03T19:33:15.816 INFO:journalctl@ceph.grafana.a.smithi164.stdout:Jun 03 19:33:15 smithi164 bash[47754]: t=2023-06-03T19:33:15+0000 lvl=info msg="Executing migration" logger=migrator id="drop preferences table v2" 2023-06-03T19:33:15.816 INFO:journalctl@ceph.grafana.a.smithi164.stdout:Jun 03 19:33:15 smithi164 bash[47754]: t=2023-06-03T19:33:15+0000 lvl=info msg="Executing migration" logger=migrator id="drop preferences table v3" 2023-06-03T19:33:16.031 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:33:15 smithi089 bash[33173]: cluster 2023-06-03T19:33:14.396283+0000 mgr.y (mgr.14445) 60 : cluster [DBG] pgmap v50: 9 pgs: 9 active+clean; 0 B data, 43 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:33:16.031 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:33:15 smithi089 bash[26572]: cluster 2023-06-03T19:33:14.396283+0000 mgr.y (mgr.14445) 60 : cluster [DBG] pgmap v50: 9 pgs: 9 active+clean; 0 B data, 43 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:33:16.068 INFO:journalctl@ceph.grafana.a.smithi164.stdout:Jun 03 19:33:15 smithi164 bash[47754]: t=2023-06-03T19:33:15+0000 lvl=info msg="Executing migration" logger=migrator id="create preferences table v3" 2023-06-03T19:33:16.068 INFO:journalctl@ceph.grafana.a.smithi164.stdout:Jun 03 19:33:15 smithi164 bash[47754]: t=2023-06-03T19:33:15+0000 lvl=info msg="Executing migration" logger=migrator id="Update preferences table charset" 2023-06-03T19:33:16.068 INFO:journalctl@ceph.grafana.a.smithi164.stdout:Jun 03 19:33:15 smithi164 bash[47754]: t=2023-06-03T19:33:15+0000 lvl=info msg="Executing migration" logger=migrator id="Add column team_id in preferences" 2023-06-03T19:33:16.068 INFO:journalctl@ceph.grafana.a.smithi164.stdout:Jun 03 19:33:15 smithi164 bash[47754]: t=2023-06-03T19:33:15+0000 lvl=info msg="Executing migration" logger=migrator id="Update team_id column values in preferences" 2023-06-03T19:33:16.068 INFO:journalctl@ceph.grafana.a.smithi164.stdout:Jun 03 19:33:15 smithi164 bash[47754]: t=2023-06-03T19:33:15+0000 lvl=info msg="Executing migration" logger=migrator id="Add column week_start in preferences" 2023-06-03T19:33:16.069 INFO:journalctl@ceph.grafana.a.smithi164.stdout:Jun 03 19:33:16 smithi164 bash[47754]: t=2023-06-03T19:33:16+0000 lvl=info msg="Executing migration" logger=migrator id="create alert table v1" 2023-06-03T19:33:16.069 INFO:journalctl@ceph.grafana.a.smithi164.stdout:Jun 03 19:33:16 smithi164 bash[47754]: t=2023-06-03T19:33:16+0000 lvl=info msg="Executing migration" logger=migrator id="add index alert org_id & id " 2023-06-03T19:33:16.336 INFO:journalctl@ceph.grafana.a.smithi164.stdout:Jun 03 19:33:16 smithi164 bash[47754]: t=2023-06-03T19:33:16+0000 lvl=info msg="Executing migration" logger=migrator id="add index alert state" 2023-06-03T19:33:16.336 INFO:journalctl@ceph.grafana.a.smithi164.stdout:Jun 03 19:33:16 smithi164 bash[47754]: t=2023-06-03T19:33:16+0000 lvl=info msg="Executing migration" logger=migrator id="add index alert dashboard_id" 2023-06-03T19:33:16.337 INFO:journalctl@ceph.grafana.a.smithi164.stdout:Jun 03 19:33:16 smithi164 bash[47754]: t=2023-06-03T19:33:16+0000 lvl=info msg="Executing migration" logger=migrator id="Create alert_rule_tag table v1" 2023-06-03T19:33:16.337 INFO:journalctl@ceph.grafana.a.smithi164.stdout:Jun 03 19:33:16 smithi164 bash[47754]: t=2023-06-03T19:33:16+0000 lvl=info msg="Executing migration" logger=migrator id="Add unique index alert_rule_tag.alert_id_tag_id" 2023-06-03T19:33:16.337 INFO:journalctl@ceph.grafana.a.smithi164.stdout:Jun 03 19:33:16 smithi164 bash[47754]: t=2023-06-03T19:33:16+0000 lvl=info msg="Executing migration" logger=migrator id="drop index UQE_alert_rule_tag_alert_id_tag_id - v1" 2023-06-03T19:33:16.337 INFO:journalctl@ceph.grafana.a.smithi164.stdout:Jun 03 19:33:16 smithi164 bash[47754]: t=2023-06-03T19:33:16+0000 lvl=info msg="Executing migration" logger=migrator id="Rename table alert_rule_tag to alert_rule_tag_v1 - v1" 2023-06-03T19:33:16.337 INFO:journalctl@ceph.grafana.a.smithi164.stdout:Jun 03 19:33:16 smithi164 bash[47754]: t=2023-06-03T19:33:16+0000 lvl=info msg="Executing migration" logger=migrator id="Create alert_rule_tag table v2" 2023-06-03T19:33:16.452 INFO:teuthology.orchestra.run.smithi089.stdout:55834574928 2023-06-03T19:33:16.452 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph osd last-stat-seq osd.1 2023-06-03T19:33:16.623 INFO:journalctl@ceph.grafana.a.smithi164.stdout:Jun 03 19:33:16 smithi164 bash[47754]: t=2023-06-03T19:33:16+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_alert_rule_tag_alert_id_tag_id - Add unique index alert_rule_tag.alert_id_tag_id V2" 2023-06-03T19:33:16.623 INFO:journalctl@ceph.grafana.a.smithi164.stdout:Jun 03 19:33:16 smithi164 bash[47754]: t=2023-06-03T19:33:16+0000 lvl=info msg="Executing migration" logger=migrator id="copy alert_rule_tag v1 to v2" 2023-06-03T19:33:16.624 INFO:journalctl@ceph.grafana.a.smithi164.stdout:Jun 03 19:33:16 smithi164 bash[47754]: t=2023-06-03T19:33:16+0000 lvl=info msg="Executing migration" logger=migrator id="drop table alert_rule_tag_v1" 2023-06-03T19:33:16.624 INFO:journalctl@ceph.grafana.a.smithi164.stdout:Jun 03 19:33:16 smithi164 bash[47754]: t=2023-06-03T19:33:16+0000 lvl=info msg="Executing migration" logger=migrator id="create alert_notification table v1" 2023-06-03T19:33:16.624 INFO:journalctl@ceph.grafana.a.smithi164.stdout:Jun 03 19:33:16 smithi164 bash[47754]: t=2023-06-03T19:33:16+0000 lvl=info msg="Executing migration" logger=migrator id="Add column is_default" 2023-06-03T19:33:16.624 INFO:journalctl@ceph.grafana.a.smithi164.stdout:Jun 03 19:33:16 smithi164 bash[47754]: t=2023-06-03T19:33:16+0000 lvl=info msg="Executing migration" logger=migrator id="Add column frequency" 2023-06-03T19:33:16.625 INFO:journalctl@ceph.grafana.a.smithi164.stdout:Jun 03 19:33:16 smithi164 bash[47754]: t=2023-06-03T19:33:16+0000 lvl=info msg="Executing migration" logger=migrator id="Add column send_reminder" 2023-06-03T19:33:16.625 INFO:journalctl@ceph.grafana.a.smithi164.stdout:Jun 03 19:33:16 smithi164 bash[47754]: t=2023-06-03T19:33:16+0000 lvl=info msg="Executing migration" logger=migrator id="Add column disable_resolve_message" 2023-06-03T19:33:16.625 INFO:journalctl@ceph.grafana.a.smithi164.stdout:Jun 03 19:33:16 smithi164 bash[47754]: t=2023-06-03T19:33:16+0000 lvl=info msg="Executing migration" logger=migrator id="add index alert_notification org_id & name" 2023-06-03T19:33:16.804 INFO:teuthology.orchestra.run.smithi089.stdout:34359738457 2023-06-03T19:33:16.805 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph osd last-stat-seq osd.0 2023-06-03T19:33:16.883 INFO:journalctl@ceph.grafana.a.smithi164.stdout:Jun 03 19:33:16 smithi164 bash[47754]: t=2023-06-03T19:33:16+0000 lvl=info msg="Executing migration" logger=migrator id="Update alert table charset" 2023-06-03T19:33:16.883 INFO:journalctl@ceph.grafana.a.smithi164.stdout:Jun 03 19:33:16 smithi164 bash[47754]: t=2023-06-03T19:33:16+0000 lvl=info msg="Executing migration" logger=migrator id="Update alert_notification table charset" 2023-06-03T19:33:16.884 INFO:journalctl@ceph.grafana.a.smithi164.stdout:Jun 03 19:33:16 smithi164 bash[47754]: t=2023-06-03T19:33:16+0000 lvl=info msg="Executing migration" logger=migrator id="create notification_journal table v1" 2023-06-03T19:33:16.884 INFO:journalctl@ceph.grafana.a.smithi164.stdout:Jun 03 19:33:16 smithi164 bash[47754]: t=2023-06-03T19:33:16+0000 lvl=info msg="Executing migration" logger=migrator id="add index notification_journal org_id & alert_id & notifier_id" 2023-06-03T19:33:16.884 INFO:journalctl@ceph.grafana.a.smithi164.stdout:Jun 03 19:33:16 smithi164 bash[47754]: t=2023-06-03T19:33:16+0000 lvl=info msg="Executing migration" logger=migrator id="drop alert_notification_journal" 2023-06-03T19:33:16.884 INFO:journalctl@ceph.grafana.a.smithi164.stdout:Jun 03 19:33:16 smithi164 bash[47754]: t=2023-06-03T19:33:16+0000 lvl=info msg="Executing migration" logger=migrator id="create alert_notification_state table v1" 2023-06-03T19:33:16.884 INFO:journalctl@ceph.grafana.a.smithi164.stdout:Jun 03 19:33:16 smithi164 bash[47754]: t=2023-06-03T19:33:16+0000 lvl=info msg="Executing migration" logger=migrator id="add index alert_notification_state org_id & alert_id & notifier_id" 2023-06-03T19:33:16.927 INFO:teuthology.orchestra.run.smithi089.stdout:206158430233 2023-06-03T19:33:16.927 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph osd last-stat-seq osd.7 2023-06-03T19:33:17.136 INFO:journalctl@ceph.grafana.a.smithi164.stdout:Jun 03 19:33:16 smithi164 bash[47754]: t=2023-06-03T19:33:16+0000 lvl=info msg="Executing migration" logger=migrator id="Add for to alert table" 2023-06-03T19:33:17.136 INFO:journalctl@ceph.grafana.a.smithi164.stdout:Jun 03 19:33:16 smithi164 bash[47754]: t=2023-06-03T19:33:16+0000 lvl=info msg="Executing migration" logger=migrator id="Add column uid in alert_notification" 2023-06-03T19:33:17.137 INFO:journalctl@ceph.grafana.a.smithi164.stdout:Jun 03 19:33:16 smithi164 bash[47754]: t=2023-06-03T19:33:16+0000 lvl=info msg="Executing migration" logger=migrator id="Update uid column values in alert_notification" 2023-06-03T19:33:17.137 INFO:journalctl@ceph.grafana.a.smithi164.stdout:Jun 03 19:33:16 smithi164 bash[47754]: t=2023-06-03T19:33:16+0000 lvl=info msg="Executing migration" logger=migrator id="Add unique index alert_notification_org_id_uid" 2023-06-03T19:33:17.137 INFO:journalctl@ceph.grafana.a.smithi164.stdout:Jun 03 19:33:17 smithi164 bash[47754]: t=2023-06-03T19:33:17+0000 lvl=info msg="Executing migration" logger=migrator id="Remove unique index org_id_name" 2023-06-03T19:33:17.137 INFO:journalctl@ceph.grafana.a.smithi164.stdout:Jun 03 19:33:17 smithi164 bash[47754]: t=2023-06-03T19:33:17+0000 lvl=info msg="Executing migration" logger=migrator id="Add column secure_settings in alert_notification" 2023-06-03T19:33:17.138 INFO:journalctl@ceph.grafana.a.smithi164.stdout:Jun 03 19:33:17 smithi164 bash[47754]: t=2023-06-03T19:33:17+0000 lvl=info msg="Executing migration" logger=migrator id="alter alert.settings to mediumtext" 2023-06-03T19:33:17.138 INFO:journalctl@ceph.grafana.a.smithi164.stdout:Jun 03 19:33:17 smithi164 bash[47754]: t=2023-06-03T19:33:17+0000 lvl=info msg="Executing migration" logger=migrator id="Add non-unique index alert_notification_state_alert_id" 2023-06-03T19:33:17.138 INFO:journalctl@ceph.grafana.a.smithi164.stdout:Jun 03 19:33:17 smithi164 bash[47754]: t=2023-06-03T19:33:17+0000 lvl=info msg="Executing migration" logger=migrator id="Add non-unique index alert_rule_tag_alert_id" 2023-06-03T19:33:17.405 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:33:17 smithi164 bash[26855]: audit 2023-06-03T19:33:16.176208+0000 mon.a (mon.0) 624 : audit [INF] from='mgr.14445 172.21.15.89:0/573502486' entity='mgr.y' 2023-06-03T19:33:17.405 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:33:17 smithi164 bash[26855]: cluster 2023-06-03T19:33:16.396788+0000 mgr.y (mgr.14445) 61 : cluster [DBG] pgmap v51: 9 pgs: 9 active+clean; 0 B data, 43 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:33:17.405 INFO:journalctl@ceph.grafana.a.smithi164.stdout:Jun 03 19:33:17 smithi164 bash[47754]: t=2023-06-03T19:33:17+0000 lvl=info msg="Executing migration" logger=migrator id="Drop old annotation table v4" 2023-06-03T19:33:17.405 INFO:journalctl@ceph.grafana.a.smithi164.stdout:Jun 03 19:33:17 smithi164 bash[47754]: t=2023-06-03T19:33:17+0000 lvl=info msg="Executing migration" logger=migrator id="create annotation table v5" 2023-06-03T19:33:17.406 INFO:journalctl@ceph.grafana.a.smithi164.stdout:Jun 03 19:33:17 smithi164 bash[47754]: t=2023-06-03T19:33:17+0000 lvl=info msg="Executing migration" logger=migrator id="add index annotation 0 v3" 2023-06-03T19:33:17.406 INFO:journalctl@ceph.grafana.a.smithi164.stdout:Jun 03 19:33:17 smithi164 bash[47754]: t=2023-06-03T19:33:17+0000 lvl=info msg="Executing migration" logger=migrator id="add index annotation 1 v3" 2023-06-03T19:33:17.406 INFO:journalctl@ceph.grafana.a.smithi164.stdout:Jun 03 19:33:17 smithi164 bash[47754]: t=2023-06-03T19:33:17+0000 lvl=info msg="Executing migration" logger=migrator id="add index annotation 2 v3" 2023-06-03T19:33:17.406 INFO:journalctl@ceph.grafana.a.smithi164.stdout:Jun 03 19:33:17 smithi164 bash[47754]: t=2023-06-03T19:33:17+0000 lvl=info msg="Executing migration" logger=migrator id="add index annotation 3 v3" 2023-06-03T19:33:17.406 INFO:journalctl@ceph.grafana.a.smithi164.stdout:Jun 03 19:33:17 smithi164 bash[47754]: t=2023-06-03T19:33:17+0000 lvl=info msg="Executing migration" logger=migrator id="add index annotation 4 v3" 2023-06-03T19:33:17.406 INFO:journalctl@ceph.grafana.a.smithi164.stdout:Jun 03 19:33:17 smithi164 bash[47754]: t=2023-06-03T19:33:17+0000 lvl=info msg="Executing migration" logger=migrator id="Update annotation table charset" 2023-06-03T19:33:17.531 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:33:17 smithi089 bash[33173]: audit 2023-06-03T19:33:16.176208+0000 mon.a (mon.0) 624 : audit [INF] from='mgr.14445 172.21.15.89:0/573502486' entity='mgr.y' 2023-06-03T19:33:17.532 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:33:17 smithi089 bash[33173]: cluster 2023-06-03T19:33:16.396788+0000 mgr.y (mgr.14445) 61 : cluster [DBG] pgmap v51: 9 pgs: 9 active+clean; 0 B data, 43 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:33:17.532 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:33:17 smithi089 bash[26572]: audit 2023-06-03T19:33:16.176208+0000 mon.a (mon.0) 624 : audit [INF] from='mgr.14445 172.21.15.89:0/573502486' entity='mgr.y' 2023-06-03T19:33:17.533 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:33:17 smithi089 bash[26572]: cluster 2023-06-03T19:33:16.396788+0000 mgr.y (mgr.14445) 61 : cluster [DBG] pgmap v51: 9 pgs: 9 active+clean; 0 B data, 43 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:33:17.666 INFO:journalctl@ceph.grafana.a.smithi164.stdout:Jun 03 19:33:17 smithi164 bash[47754]: t=2023-06-03T19:33:17+0000 lvl=info msg="Executing migration" logger=migrator id="Add column region_id to annotation table" 2023-06-03T19:33:17.666 INFO:journalctl@ceph.grafana.a.smithi164.stdout:Jun 03 19:33:17 smithi164 bash[47754]: t=2023-06-03T19:33:17+0000 lvl=info msg="Executing migration" logger=migrator id="Drop category_id index" 2023-06-03T19:33:17.666 INFO:journalctl@ceph.grafana.a.smithi164.stdout:Jun 03 19:33:17 smithi164 bash[47754]: t=2023-06-03T19:33:17+0000 lvl=info msg="Executing migration" logger=migrator id="Add column tags to annotation table" 2023-06-03T19:33:17.667 INFO:journalctl@ceph.grafana.a.smithi164.stdout:Jun 03 19:33:17 smithi164 bash[47754]: t=2023-06-03T19:33:17+0000 lvl=info msg="Executing migration" logger=migrator id="Create annotation_tag table v2" 2023-06-03T19:33:17.667 INFO:journalctl@ceph.grafana.a.smithi164.stdout:Jun 03 19:33:17 smithi164 bash[47754]: t=2023-06-03T19:33:17+0000 lvl=info msg="Executing migration" logger=migrator id="Add unique index annotation_tag.annotation_id_tag_id" 2023-06-03T19:33:17.667 INFO:journalctl@ceph.grafana.a.smithi164.stdout:Jun 03 19:33:17 smithi164 bash[47754]: t=2023-06-03T19:33:17+0000 lvl=info msg="Executing migration" logger=migrator id="drop index UQE_annotation_tag_annotation_id_tag_id - v2" 2023-06-03T19:33:17.667 INFO:journalctl@ceph.grafana.a.smithi164.stdout:Jun 03 19:33:17 smithi164 bash[47754]: t=2023-06-03T19:33:17+0000 lvl=info msg="Executing migration" logger=migrator id="Rename table annotation_tag to annotation_tag_v2 - v2" 2023-06-03T19:33:17.667 INFO:journalctl@ceph.grafana.a.smithi164.stdout:Jun 03 19:33:17 smithi164 bash[47754]: t=2023-06-03T19:33:17+0000 lvl=info msg="Executing migration" logger=migrator id="Create annotation_tag table v3" 2023-06-03T19:33:17.668 INFO:journalctl@ceph.grafana.a.smithi164.stdout:Jun 03 19:33:17 smithi164 bash[47754]: t=2023-06-03T19:33:17+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_annotation_tag_annotation_id_tag_id - Add unique index annotation_tag.annotation_id_tag_id V3" 2023-06-03T19:33:17.935 INFO:journalctl@ceph.grafana.a.smithi164.stdout:Jun 03 19:33:17 smithi164 bash[47754]: t=2023-06-03T19:33:17+0000 lvl=info msg="Executing migration" logger=migrator id="copy annotation_tag v2 to v3" 2023-06-03T19:33:17.935 INFO:journalctl@ceph.grafana.a.smithi164.stdout:Jun 03 19:33:17 smithi164 bash[47754]: t=2023-06-03T19:33:17+0000 lvl=info msg="Executing migration" logger=migrator id="drop table annotation_tag_v2" 2023-06-03T19:33:17.935 INFO:journalctl@ceph.grafana.a.smithi164.stdout:Jun 03 19:33:17 smithi164 bash[47754]: t=2023-06-03T19:33:17+0000 lvl=info msg="Executing migration" logger=migrator id="Update alert annotations and set TEXT to empty" 2023-06-03T19:33:17.936 INFO:journalctl@ceph.grafana.a.smithi164.stdout:Jun 03 19:33:17 smithi164 bash[47754]: t=2023-06-03T19:33:17+0000 lvl=info msg="Executing migration" logger=migrator id="Add created time to annotation table" 2023-06-03T19:33:17.936 INFO:journalctl@ceph.grafana.a.smithi164.stdout:Jun 03 19:33:17 smithi164 bash[47754]: t=2023-06-03T19:33:17+0000 lvl=info msg="Executing migration" logger=migrator id="Add updated time to annotation table" 2023-06-03T19:33:17.936 INFO:journalctl@ceph.grafana.a.smithi164.stdout:Jun 03 19:33:17 smithi164 bash[47754]: t=2023-06-03T19:33:17+0000 lvl=info msg="Executing migration" logger=migrator id="Add index for created in annotation table" 2023-06-03T19:33:17.936 INFO:journalctl@ceph.grafana.a.smithi164.stdout:Jun 03 19:33:17 smithi164 bash[47754]: t=2023-06-03T19:33:17+0000 lvl=info msg="Executing migration" logger=migrator id="Add index for updated in annotation table" 2023-06-03T19:33:17.936 INFO:journalctl@ceph.grafana.a.smithi164.stdout:Jun 03 19:33:17 smithi164 bash[47754]: t=2023-06-03T19:33:17+0000 lvl=info msg="Executing migration" logger=migrator id="Convert existing annotations from seconds to milliseconds" 2023-06-03T19:33:17.943 INFO:teuthology.orchestra.run.smithi089.stdout:107374182462 2023-06-03T19:33:17.944 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph osd last-stat-seq osd.3 2023-06-03T19:33:18.010 INFO:teuthology.orchestra.run.smithi089.stdout:180388626467 2023-06-03T19:33:18.011 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph osd last-stat-seq osd.6 2023-06-03T19:33:18.213 INFO:journalctl@ceph.grafana.a.smithi164.stdout:Jun 03 19:33:17 smithi164 bash[47754]: t=2023-06-03T19:33:17+0000 lvl=info msg="Executing migration" logger=migrator id="Add epoch_end column" 2023-06-03T19:33:18.213 INFO:journalctl@ceph.grafana.a.smithi164.stdout:Jun 03 19:33:17 smithi164 bash[47754]: t=2023-06-03T19:33:17+0000 lvl=info msg="Executing migration" logger=migrator id="Add index for epoch_end" 2023-06-03T19:33:18.213 INFO:journalctl@ceph.grafana.a.smithi164.stdout:Jun 03 19:33:18 smithi164 bash[47754]: t=2023-06-03T19:33:18+0000 lvl=info msg="Executing migration" logger=migrator id="Make epoch_end the same as epoch" 2023-06-03T19:33:18.213 INFO:journalctl@ceph.grafana.a.smithi164.stdout:Jun 03 19:33:18 smithi164 bash[47754]: t=2023-06-03T19:33:18+0000 lvl=info msg="Executing migration" logger=migrator id="Move region to single row" 2023-06-03T19:33:18.213 INFO:journalctl@ceph.grafana.a.smithi164.stdout:Jun 03 19:33:18 smithi164 bash[47754]: t=2023-06-03T19:33:18+0000 lvl=info msg="Executing migration" logger=migrator id="Remove index org_id_epoch from annotation table" 2023-06-03T19:33:18.214 INFO:journalctl@ceph.grafana.a.smithi164.stdout:Jun 03 19:33:18 smithi164 bash[47754]: t=2023-06-03T19:33:18+0000 lvl=info msg="Executing migration" logger=migrator id="Remove index org_id_dashboard_id_panel_id_epoch from annotation table" 2023-06-03T19:33:18.214 INFO:journalctl@ceph.grafana.a.smithi164.stdout:Jun 03 19:33:18 smithi164 bash[47754]: t=2023-06-03T19:33:18+0000 lvl=info msg="Executing migration" logger=migrator id="Add index for org_id_dashboard_id_epoch_end_epoch on annotation table" 2023-06-03T19:33:18.214 INFO:journalctl@ceph.grafana.a.smithi164.stdout:Jun 03 19:33:18 smithi164 bash[47754]: t=2023-06-03T19:33:18+0000 lvl=info msg="Executing migration" logger=migrator id="Add index for org_id_epoch_end_epoch on annotation table" 2023-06-03T19:33:18.281 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 19:33:17 smithi089 bash[26845]: ::ffff:172.21.15.164 - - [03/Jun/2023:19:33:17] "GET /metrics HTTP/1.1" 200 151993 "" "Prometheus/2.33.4" 2023-06-03T19:33:18.490 INFO:journalctl@ceph.grafana.a.smithi164.stdout:Jun 03 19:33:18 smithi164 bash[47754]: t=2023-06-03T19:33:18+0000 lvl=info msg="Executing migration" logger=migrator id="Remove index org_id_epoch_epoch_end from annotation table" 2023-06-03T19:33:18.490 INFO:journalctl@ceph.grafana.a.smithi164.stdout:Jun 03 19:33:18 smithi164 bash[47754]: t=2023-06-03T19:33:18+0000 lvl=info msg="Executing migration" logger=migrator id="Add index for alert_id on annotation table" 2023-06-03T19:33:18.490 INFO:journalctl@ceph.grafana.a.smithi164.stdout:Jun 03 19:33:18 smithi164 bash[47754]: t=2023-06-03T19:33:18+0000 lvl=info msg="Executing migration" logger=migrator id="create test_data table" 2023-06-03T19:33:18.491 INFO:journalctl@ceph.grafana.a.smithi164.stdout:Jun 03 19:33:18 smithi164 bash[47754]: t=2023-06-03T19:33:18+0000 lvl=info msg="Executing migration" logger=migrator id="create dashboard_version table v1" 2023-06-03T19:33:18.491 INFO:journalctl@ceph.grafana.a.smithi164.stdout:Jun 03 19:33:18 smithi164 bash[47754]: t=2023-06-03T19:33:18+0000 lvl=info msg="Executing migration" logger=migrator id="add index dashboard_version.dashboard_id" 2023-06-03T19:33:18.491 INFO:journalctl@ceph.grafana.a.smithi164.stdout:Jun 03 19:33:18 smithi164 bash[47754]: t=2023-06-03T19:33:18+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index dashboard_version.dashboard_id and dashboard_version.version" 2023-06-03T19:33:18.491 INFO:journalctl@ceph.grafana.a.smithi164.stdout:Jun 03 19:33:18 smithi164 bash[47754]: t=2023-06-03T19:33:18+0000 lvl=info msg="Executing migration" logger=migrator id="Set dashboard version to 1 where 0" 2023-06-03T19:33:18.491 INFO:journalctl@ceph.grafana.a.smithi164.stdout:Jun 03 19:33:18 smithi164 bash[47754]: t=2023-06-03T19:33:18+0000 lvl=info msg="Executing migration" logger=migrator id="save existing dashboard data in dashboard_version table v1" 2023-06-03T19:33:18.491 INFO:journalctl@ceph.grafana.a.smithi164.stdout:Jun 03 19:33:18 smithi164 bash[47754]: t=2023-06-03T19:33:18+0000 lvl=info msg="Executing migration" logger=migrator id="alter dashboard_version.data to mediumtext v1" 2023-06-03T19:33:18.742 INFO:journalctl@ceph.grafana.a.smithi164.stdout:Jun 03 19:33:18 smithi164 bash[47754]: t=2023-06-03T19:33:18+0000 lvl=info msg="Executing migration" logger=migrator id="create team table" 2023-06-03T19:33:18.743 INFO:journalctl@ceph.grafana.a.smithi164.stdout:Jun 03 19:33:18 smithi164 bash[47754]: t=2023-06-03T19:33:18+0000 lvl=info msg="Executing migration" logger=migrator id="add index team.org_id" 2023-06-03T19:33:18.743 INFO:journalctl@ceph.grafana.a.smithi164.stdout:Jun 03 19:33:18 smithi164 bash[47754]: t=2023-06-03T19:33:18+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index team_org_id_name" 2023-06-03T19:33:18.743 INFO:journalctl@ceph.grafana.a.smithi164.stdout:Jun 03 19:33:18 smithi164 bash[47754]: t=2023-06-03T19:33:18+0000 lvl=info msg="Executing migration" logger=migrator id="create team member table" 2023-06-03T19:33:18.743 INFO:journalctl@ceph.grafana.a.smithi164.stdout:Jun 03 19:33:18 smithi164 bash[47754]: t=2023-06-03T19:33:18+0000 lvl=info msg="Executing migration" logger=migrator id="add index team_member.org_id" 2023-06-03T19:33:18.743 INFO:journalctl@ceph.grafana.a.smithi164.stdout:Jun 03 19:33:18 smithi164 bash[47754]: t=2023-06-03T19:33:18+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index team_member_org_id_team_id_user_id" 2023-06-03T19:33:18.743 INFO:journalctl@ceph.grafana.a.smithi164.stdout:Jun 03 19:33:18 smithi164 bash[47754]: t=2023-06-03T19:33:18+0000 lvl=info msg="Executing migration" logger=migrator id="add index team_member.team_id" 2023-06-03T19:33:18.743 INFO:journalctl@ceph.grafana.a.smithi164.stdout:Jun 03 19:33:18 smithi164 bash[47754]: t=2023-06-03T19:33:18+0000 lvl=info msg="Executing migration" logger=migrator id="Add column email to team table" 2023-06-03T19:33:19.012 INFO:journalctl@ceph.grafana.a.smithi164.stdout:Jun 03 19:33:18 smithi164 bash[47754]: t=2023-06-03T19:33:18+0000 lvl=info msg="Executing migration" logger=migrator id="Add column external to team_member table" 2023-06-03T19:33:19.012 INFO:journalctl@ceph.grafana.a.smithi164.stdout:Jun 03 19:33:18 smithi164 bash[47754]: t=2023-06-03T19:33:18+0000 lvl=info msg="Executing migration" logger=migrator id="Add column permission to team_member table" 2023-06-03T19:33:19.012 INFO:journalctl@ceph.grafana.a.smithi164.stdout:Jun 03 19:33:18 smithi164 bash[47754]: t=2023-06-03T19:33:18+0000 lvl=info msg="Executing migration" logger=migrator id="create dashboard acl table" 2023-06-03T19:33:19.012 INFO:journalctl@ceph.grafana.a.smithi164.stdout:Jun 03 19:33:18 smithi164 bash[47754]: t=2023-06-03T19:33:18+0000 lvl=info msg="Executing migration" logger=migrator id="add index dashboard_acl_dashboard_id" 2023-06-03T19:33:19.013 INFO:journalctl@ceph.grafana.a.smithi164.stdout:Jun 03 19:33:18 smithi164 bash[47754]: t=2023-06-03T19:33:18+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index dashboard_acl_dashboard_id_user_id" 2023-06-03T19:33:19.013 INFO:journalctl@ceph.grafana.a.smithi164.stdout:Jun 03 19:33:18 smithi164 bash[47754]: t=2023-06-03T19:33:18+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index dashboard_acl_dashboard_id_team_id" 2023-06-03T19:33:19.013 INFO:journalctl@ceph.grafana.a.smithi164.stdout:Jun 03 19:33:18 smithi164 bash[47754]: t=2023-06-03T19:33:18+0000 lvl=info msg="Executing migration" logger=migrator id="add index dashboard_acl_user_id" 2023-06-03T19:33:19.013 INFO:journalctl@ceph.grafana.a.smithi164.stdout:Jun 03 19:33:18 smithi164 bash[47754]: t=2023-06-03T19:33:18+0000 lvl=info msg="Executing migration" logger=migrator id="add index dashboard_acl_team_id" 2023-06-03T19:33:19.013 INFO:journalctl@ceph.grafana.a.smithi164.stdout:Jun 03 19:33:18 smithi164 bash[47754]: t=2023-06-03T19:33:18+0000 lvl=info msg="Executing migration" logger=migrator id="add index dashboard_acl_org_id_role" 2023-06-03T19:33:19.013 INFO:journalctl@ceph.grafana.a.smithi164.stdout:Jun 03 19:33:19 smithi164 bash[47754]: t=2023-06-03T19:33:19+0000 lvl=info msg="Executing migration" logger=migrator id="add index dashboard_permission" 2023-06-03T19:33:19.264 INFO:journalctl@ceph.grafana.a.smithi164.stdout:Jun 03 19:33:19 smithi164 bash[47754]: t=2023-06-03T19:33:19+0000 lvl=info msg="Executing migration" logger=migrator id="save default acl rules in dashboard_acl table" 2023-06-03T19:33:19.265 INFO:journalctl@ceph.grafana.a.smithi164.stdout:Jun 03 19:33:19 smithi164 bash[47754]: t=2023-06-03T19:33:19+0000 lvl=info msg="Executing migration" logger=migrator id="delete acl rules for deleted dashboards and folders" 2023-06-03T19:33:19.265 INFO:journalctl@ceph.grafana.a.smithi164.stdout:Jun 03 19:33:19 smithi164 bash[47754]: t=2023-06-03T19:33:19+0000 lvl=info msg="Executing migration" logger=migrator id="create tag table" 2023-06-03T19:33:19.265 INFO:journalctl@ceph.grafana.a.smithi164.stdout:Jun 03 19:33:19 smithi164 bash[47754]: t=2023-06-03T19:33:19+0000 lvl=info msg="Executing migration" logger=migrator id="add index tag.key_value" 2023-06-03T19:33:19.265 INFO:journalctl@ceph.grafana.a.smithi164.stdout:Jun 03 19:33:19 smithi164 bash[47754]: t=2023-06-03T19:33:19+0000 lvl=info msg="Executing migration" logger=migrator id="create login attempt table" 2023-06-03T19:33:19.265 INFO:journalctl@ceph.grafana.a.smithi164.stdout:Jun 03 19:33:19 smithi164 bash[47754]: t=2023-06-03T19:33:19+0000 lvl=info msg="Executing migration" logger=migrator id="add index login_attempt.username" 2023-06-03T19:33:19.265 INFO:journalctl@ceph.grafana.a.smithi164.stdout:Jun 03 19:33:19 smithi164 bash[47754]: t=2023-06-03T19:33:19+0000 lvl=info msg="Executing migration" logger=migrator id="drop index IDX_login_attempt_username - v1" 2023-06-03T19:33:19.265 INFO:journalctl@ceph.grafana.a.smithi164.stdout:Jun 03 19:33:19 smithi164 bash[47754]: t=2023-06-03T19:33:19+0000 lvl=info msg="Executing migration" logger=migrator id="Rename table login_attempt to login_attempt_tmp_qwerty - v1" 2023-06-03T19:33:19.266 INFO:journalctl@ceph.grafana.a.smithi164.stdout:Jun 03 19:33:19 smithi164 bash[47754]: t=2023-06-03T19:33:19+0000 lvl=info msg="Executing migration" logger=migrator id="create login_attempt v2" 2023-06-03T19:33:19.551 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:33:19 smithi164 bash[26855]: cluster 2023-06-03T19:33:18.397325+0000 mgr.y (mgr.14445) 62 : cluster [DBG] pgmap v52: 9 pgs: 9 active+clean; 0 B data, 43 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:33:19.552 INFO:journalctl@ceph.grafana.a.smithi164.stdout:Jun 03 19:33:19 smithi164 bash[47754]: t=2023-06-03T19:33:19+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_login_attempt_username - v2" 2023-06-03T19:33:19.552 INFO:journalctl@ceph.grafana.a.smithi164.stdout:Jun 03 19:33:19 smithi164 bash[47754]: t=2023-06-03T19:33:19+0000 lvl=info msg="Executing migration" logger=migrator id="copy login_attempt v1 to v2" 2023-06-03T19:33:19.552 INFO:journalctl@ceph.grafana.a.smithi164.stdout:Jun 03 19:33:19 smithi164 bash[47754]: t=2023-06-03T19:33:19+0000 lvl=info msg="Executing migration" logger=migrator id="drop login_attempt_tmp_qwerty" 2023-06-03T19:33:19.552 INFO:journalctl@ceph.grafana.a.smithi164.stdout:Jun 03 19:33:19 smithi164 bash[47754]: t=2023-06-03T19:33:19+0000 lvl=info msg="Executing migration" logger=migrator id="create user auth table" 2023-06-03T19:33:19.552 INFO:journalctl@ceph.grafana.a.smithi164.stdout:Jun 03 19:33:19 smithi164 bash[47754]: t=2023-06-03T19:33:19+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_user_auth_auth_module_auth_id - v1" 2023-06-03T19:33:19.552 INFO:journalctl@ceph.grafana.a.smithi164.stdout:Jun 03 19:33:19 smithi164 bash[47754]: t=2023-06-03T19:33:19+0000 lvl=info msg="Executing migration" logger=migrator id="alter user_auth.auth_id to length 190" 2023-06-03T19:33:19.553 INFO:journalctl@ceph.grafana.a.smithi164.stdout:Jun 03 19:33:19 smithi164 bash[47754]: t=2023-06-03T19:33:19+0000 lvl=info msg="Executing migration" logger=migrator id="Add OAuth access token to user_auth" 2023-06-03T19:33:19.553 INFO:journalctl@ceph.grafana.a.smithi164.stdout:Jun 03 19:33:19 smithi164 bash[47754]: t=2023-06-03T19:33:19+0000 lvl=info msg="Executing migration" logger=migrator id="Add OAuth refresh token to user_auth" 2023-06-03T19:33:19.781 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:33:19 smithi089 bash[33173]: cluster 2023-06-03T19:33:18.397325+0000 mgr.y (mgr.14445) 62 : cluster [DBG] pgmap v52: 9 pgs: 9 active+clean; 0 B data, 43 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:33:19.782 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:33:19 smithi089 bash[26572]: cluster 2023-06-03T19:33:18.397325+0000 mgr.y (mgr.14445) 62 : cluster [DBG] pgmap v52: 9 pgs: 9 active+clean; 0 B data, 43 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:33:19.803 INFO:journalctl@ceph.grafana.a.smithi164.stdout:Jun 03 19:33:19 smithi164 bash[47754]: t=2023-06-03T19:33:19+0000 lvl=info msg="Executing migration" logger=migrator id="Add OAuth token type to user_auth" 2023-06-03T19:33:19.803 INFO:journalctl@ceph.grafana.a.smithi164.stdout:Jun 03 19:33:19 smithi164 bash[47754]: t=2023-06-03T19:33:19+0000 lvl=info msg="Executing migration" logger=migrator id="Add OAuth expiry to user_auth" 2023-06-03T19:33:19.803 INFO:journalctl@ceph.grafana.a.smithi164.stdout:Jun 03 19:33:19 smithi164 bash[47754]: t=2023-06-03T19:33:19+0000 lvl=info msg="Executing migration" logger=migrator id="Add index to user_id column in user_auth" 2023-06-03T19:33:19.804 INFO:journalctl@ceph.grafana.a.smithi164.stdout:Jun 03 19:33:19 smithi164 bash[47754]: t=2023-06-03T19:33:19+0000 lvl=info msg="Executing migration" logger=migrator id="create server_lock table" 2023-06-03T19:33:19.804 INFO:journalctl@ceph.grafana.a.smithi164.stdout:Jun 03 19:33:19 smithi164 bash[47754]: t=2023-06-03T19:33:19+0000 lvl=info msg="Executing migration" logger=migrator id="add index server_lock.operation_uid" 2023-06-03T19:33:19.804 INFO:journalctl@ceph.grafana.a.smithi164.stdout:Jun 03 19:33:19 smithi164 bash[47754]: t=2023-06-03T19:33:19+0000 lvl=info msg="Executing migration" logger=migrator id="create user auth token table" 2023-06-03T19:33:19.805 INFO:journalctl@ceph.grafana.a.smithi164.stdout:Jun 03 19:33:19 smithi164 bash[47754]: t=2023-06-03T19:33:19+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index user_auth_token.auth_token" 2023-06-03T19:33:20.064 INFO:journalctl@ceph.grafana.a.smithi164.stdout:Jun 03 19:33:19 smithi164 bash[47754]: t=2023-06-03T19:33:19+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index user_auth_token.prev_auth_token" 2023-06-03T19:33:20.064 INFO:journalctl@ceph.grafana.a.smithi164.stdout:Jun 03 19:33:19 smithi164 bash[47754]: t=2023-06-03T19:33:19+0000 lvl=info msg="Executing migration" logger=migrator id="add index user_auth_token.user_id" 2023-06-03T19:33:20.065 INFO:journalctl@ceph.grafana.a.smithi164.stdout:Jun 03 19:33:19 smithi164 bash[47754]: t=2023-06-03T19:33:19+0000 lvl=info msg="Executing migration" logger=migrator id="Add revoked_at to the user auth token" 2023-06-03T19:33:20.065 INFO:journalctl@ceph.grafana.a.smithi164.stdout:Jun 03 19:33:19 smithi164 bash[47754]: t=2023-06-03T19:33:19+0000 lvl=info msg="Executing migration" logger=migrator id="create cache_data table" 2023-06-03T19:33:20.065 INFO:journalctl@ceph.grafana.a.smithi164.stdout:Jun 03 19:33:19 smithi164 bash[47754]: t=2023-06-03T19:33:19+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index cache_data.cache_key" 2023-06-03T19:33:20.065 INFO:journalctl@ceph.grafana.a.smithi164.stdout:Jun 03 19:33:19 smithi164 bash[47754]: t=2023-06-03T19:33:19+0000 lvl=info msg="Executing migration" logger=migrator id="create short_url table v1" 2023-06-03T19:33:20.065 INFO:journalctl@ceph.grafana.a.smithi164.stdout:Jun 03 19:33:19 smithi164 bash[47754]: t=2023-06-03T19:33:19+0000 lvl=info msg="Executing migration" logger=migrator id="add index short_url.org_id-uid" 2023-06-03T19:33:20.065 INFO:journalctl@ceph.grafana.a.smithi164.stdout:Jun 03 19:33:20 smithi164 bash[47754]: t=2023-06-03T19:33:20+0000 lvl=info msg="Executing migration" logger=migrator id="delete alert_definition table" 2023-06-03T19:33:20.065 INFO:journalctl@ceph.grafana.a.smithi164.stdout:Jun 03 19:33:20 smithi164 bash[47754]: t=2023-06-03T19:33:20+0000 lvl=info msg="Executing migration" logger=migrator id="recreate alert_definition table" 2023-06-03T19:33:20.065 INFO:journalctl@ceph.grafana.a.smithi164.stdout:Jun 03 19:33:20 smithi164 bash[47754]: t=2023-06-03T19:33:20+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_definition on org_id and title columns" 2023-06-03T19:33:20.324 INFO:journalctl@ceph.grafana.a.smithi164.stdout:Jun 03 19:33:20 smithi164 bash[47754]: t=2023-06-03T19:33:20+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_definition on org_id and uid columns" 2023-06-03T19:33:20.325 INFO:journalctl@ceph.grafana.a.smithi164.stdout:Jun 03 19:33:20 smithi164 bash[47754]: t=2023-06-03T19:33:20+0000 lvl=info msg="Executing migration" logger=migrator id="alter alert_definition table data column to mediumtext in mysql" 2023-06-03T19:33:20.325 INFO:journalctl@ceph.grafana.a.smithi164.stdout:Jun 03 19:33:20 smithi164 bash[47754]: t=2023-06-03T19:33:20+0000 lvl=info msg="Executing migration" logger=migrator id="drop index in alert_definition on org_id and title columns" 2023-06-03T19:33:20.325 INFO:journalctl@ceph.grafana.a.smithi164.stdout:Jun 03 19:33:20 smithi164 bash[47754]: t=2023-06-03T19:33:20+0000 lvl=info msg="Executing migration" logger=migrator id="drop index in alert_definition on org_id and uid columns" 2023-06-03T19:33:20.325 INFO:journalctl@ceph.grafana.a.smithi164.stdout:Jun 03 19:33:20 smithi164 bash[47754]: t=2023-06-03T19:33:20+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index in alert_definition on org_id and title columns" 2023-06-03T19:33:20.325 INFO:journalctl@ceph.grafana.a.smithi164.stdout:Jun 03 19:33:20 smithi164 bash[47754]: t=2023-06-03T19:33:20+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index in alert_definition on org_id and uid columns" 2023-06-03T19:33:20.325 INFO:journalctl@ceph.grafana.a.smithi164.stdout:Jun 03 19:33:20 smithi164 bash[47754]: t=2023-06-03T19:33:20+0000 lvl=info msg="Executing migration" logger=migrator id="Add column paused in alert_definition" 2023-06-03T19:33:20.326 INFO:journalctl@ceph.grafana.a.smithi164.stdout:Jun 03 19:33:20 smithi164 bash[47754]: t=2023-06-03T19:33:20+0000 lvl=info msg="Executing migration" logger=migrator id="drop alert_definition table" 2023-06-03T19:33:20.586 INFO:journalctl@ceph.grafana.a.smithi164.stdout:Jun 03 19:33:20 smithi164 bash[47754]: t=2023-06-03T19:33:20+0000 lvl=info msg="Executing migration" logger=migrator id="delete alert_definition_version table" 2023-06-03T19:33:20.587 INFO:journalctl@ceph.grafana.a.smithi164.stdout:Jun 03 19:33:20 smithi164 bash[47754]: t=2023-06-03T19:33:20+0000 lvl=info msg="Executing migration" logger=migrator id="recreate alert_definition_version table" 2023-06-03T19:33:20.587 INFO:journalctl@ceph.grafana.a.smithi164.stdout:Jun 03 19:33:20 smithi164 bash[47754]: t=2023-06-03T19:33:20+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_definition_version table on alert_definition_id and version columns" 2023-06-03T19:33:20.587 INFO:journalctl@ceph.grafana.a.smithi164.stdout:Jun 03 19:33:20 smithi164 bash[47754]: t=2023-06-03T19:33:20+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_definition_version table on alert_definition_uid and version columns" 2023-06-03T19:33:20.587 INFO:journalctl@ceph.grafana.a.smithi164.stdout:Jun 03 19:33:20 smithi164 bash[47754]: t=2023-06-03T19:33:20+0000 lvl=info msg="Executing migration" logger=migrator id="alter alert_definition_version table data column to mediumtext in mysql" 2023-06-03T19:33:20.587 INFO:journalctl@ceph.grafana.a.smithi164.stdout:Jun 03 19:33:20 smithi164 bash[47754]: t=2023-06-03T19:33:20+0000 lvl=info msg="Executing migration" logger=migrator id="drop alert_definition_version table" 2023-06-03T19:33:20.587 INFO:journalctl@ceph.grafana.a.smithi164.stdout:Jun 03 19:33:20 smithi164 bash[47754]: t=2023-06-03T19:33:20+0000 lvl=info msg="Executing migration" logger=migrator id="create alert_instance table" 2023-06-03T19:33:20.588 INFO:journalctl@ceph.grafana.a.smithi164.stdout:Jun 03 19:33:20 smithi164 bash[47754]: t=2023-06-03T19:33:20+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_instance table on def_org_id, def_uid and current_state columns" 2023-06-03T19:33:20.663 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/mon.a/config 2023-06-03T19:33:20.668 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/mon.a/config 2023-06-03T19:33:20.668 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/mon.a/config 2023-06-03T19:33:20.669 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/mon.a/config 2023-06-03T19:33:20.669 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/mon.a/config 2023-06-03T19:33:20.855 INFO:journalctl@ceph.grafana.a.smithi164.stdout:Jun 03 19:33:20 smithi164 bash[47754]: t=2023-06-03T19:33:20+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_instance table on def_org_id, current_state columns" 2023-06-03T19:33:20.855 INFO:journalctl@ceph.grafana.a.smithi164.stdout:Jun 03 19:33:20 smithi164 bash[47754]: t=2023-06-03T19:33:20+0000 lvl=info msg="Executing migration" logger=migrator id="add column current_state_end to alert_instance" 2023-06-03T19:33:20.855 INFO:journalctl@ceph.grafana.a.smithi164.stdout:Jun 03 19:33:20 smithi164 bash[47754]: t=2023-06-03T19:33:20+0000 lvl=info msg="Executing migration" logger=migrator id="remove index def_org_id, def_uid, current_state on alert_instance" 2023-06-03T19:33:20.855 INFO:journalctl@ceph.grafana.a.smithi164.stdout:Jun 03 19:33:20 smithi164 bash[47754]: t=2023-06-03T19:33:20+0000 lvl=info msg="Executing migration" logger=migrator id="remove index def_org_id, current_state on alert_instance" 2023-06-03T19:33:20.856 INFO:journalctl@ceph.grafana.a.smithi164.stdout:Jun 03 19:33:20 smithi164 bash[47754]: t=2023-06-03T19:33:20+0000 lvl=info msg="Executing migration" logger=migrator id="rename def_org_id to rule_org_id in alert_instance" 2023-06-03T19:33:20.856 INFO:journalctl@ceph.grafana.a.smithi164.stdout:Jun 03 19:33:20 smithi164 bash[47754]: t=2023-06-03T19:33:20+0000 lvl=info msg="Executing migration" logger=migrator id="rename def_uid to rule_uid in alert_instance" 2023-06-03T19:33:20.856 INFO:journalctl@ceph.grafana.a.smithi164.stdout:Jun 03 19:33:20 smithi164 bash[47754]: t=2023-06-03T19:33:20+0000 lvl=info msg="Executing migration" logger=migrator id="add index rule_org_id, rule_uid, current_state on alert_instance" 2023-06-03T19:33:20.856 INFO:journalctl@ceph.grafana.a.smithi164.stdout:Jun 03 19:33:20 smithi164 bash[47754]: t=2023-06-03T19:33:20+0000 lvl=info msg="Executing migration" logger=migrator id="add index rule_org_id, current_state on alert_instance" 2023-06-03T19:33:20.856 INFO:journalctl@ceph.grafana.a.smithi164.stdout:Jun 03 19:33:20 smithi164 bash[47754]: t=2023-06-03T19:33:20+0000 lvl=info msg="Executing migration" logger=migrator id="create alert_rule table" 2023-06-03T19:33:20.856 INFO:journalctl@ceph.grafana.a.smithi164.stdout:Jun 03 19:33:20 smithi164 bash[47754]: t=2023-06-03T19:33:20+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_rule on org_id and title columns" 2023-06-03T19:33:21.124 INFO:journalctl@ceph.grafana.a.smithi164.stdout:Jun 03 19:33:20 smithi164 bash[47754]: t=2023-06-03T19:33:20+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_rule on org_id and uid columns" 2023-06-03T19:33:21.124 INFO:journalctl@ceph.grafana.a.smithi164.stdout:Jun 03 19:33:20 smithi164 bash[47754]: t=2023-06-03T19:33:20+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_rule on org_id, namespace_uid, group_uid columns" 2023-06-03T19:33:21.124 INFO:journalctl@ceph.grafana.a.smithi164.stdout:Jun 03 19:33:20 smithi164 bash[47754]: t=2023-06-03T19:33:20+0000 lvl=info msg="Executing migration" logger=migrator id="alter alert_rule table data column to mediumtext in mysql" 2023-06-03T19:33:21.124 INFO:journalctl@ceph.grafana.a.smithi164.stdout:Jun 03 19:33:20 smithi164 bash[47754]: t=2023-06-03T19:33:20+0000 lvl=info msg="Executing migration" logger=migrator id="add column for to alert_rule" 2023-06-03T19:33:21.125 INFO:journalctl@ceph.grafana.a.smithi164.stdout:Jun 03 19:33:20 smithi164 bash[47754]: t=2023-06-03T19:33:20+0000 lvl=info msg="Executing migration" logger=migrator id="add column annotations to alert_rule" 2023-06-03T19:33:21.125 INFO:journalctl@ceph.grafana.a.smithi164.stdout:Jun 03 19:33:21 smithi164 bash[47754]: t=2023-06-03T19:33:21+0000 lvl=info msg="Executing migration" logger=migrator id="add column labels to alert_rule" 2023-06-03T19:33:21.125 INFO:journalctl@ceph.grafana.a.smithi164.stdout:Jun 03 19:33:21 smithi164 bash[47754]: t=2023-06-03T19:33:21+0000 lvl=info msg="Executing migration" logger=migrator id="remove unique index from alert_rule on org_id, title columns" 2023-06-03T19:33:21.125 INFO:journalctl@ceph.grafana.a.smithi164.stdout:Jun 03 19:33:21 smithi164 bash[47754]: t=2023-06-03T19:33:21+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_rule on org_id, namespase_uid and title columns" 2023-06-03T19:33:21.125 INFO:journalctl@ceph.grafana.a.smithi164.stdout:Jun 03 19:33:21 smithi164 bash[47754]: t=2023-06-03T19:33:21+0000 lvl=info msg="Executing migration" logger=migrator id="add dashboard_uid column to alert_rule" 2023-06-03T19:33:21.125 INFO:journalctl@ceph.grafana.a.smithi164.stdout:Jun 03 19:33:21 smithi164 bash[47754]: t=2023-06-03T19:33:21+0000 lvl=info msg="Executing migration" logger=migrator id="add panel_id column to alert_rule" 2023-06-03T19:33:21.385 INFO:journalctl@ceph.grafana.a.smithi164.stdout:Jun 03 19:33:21 smithi164 bash[47754]: t=2023-06-03T19:33:21+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_rule on org_id, dashboard_uid and panel_id columns" 2023-06-03T19:33:21.385 INFO:journalctl@ceph.grafana.a.smithi164.stdout:Jun 03 19:33:21 smithi164 bash[47754]: t=2023-06-03T19:33:21+0000 lvl=info msg="Executing migration" logger=migrator id="create alert_rule_version table" 2023-06-03T19:33:21.385 INFO:journalctl@ceph.grafana.a.smithi164.stdout:Jun 03 19:33:21 smithi164 bash[47754]: t=2023-06-03T19:33:21+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_rule_version table on rule_org_id, rule_uid and version columns" 2023-06-03T19:33:21.385 INFO:journalctl@ceph.grafana.a.smithi164.stdout:Jun 03 19:33:21 smithi164 bash[47754]: t=2023-06-03T19:33:21+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_rule_version table on rule_org_id, rule_namespace_uid and rule_group columns" 2023-06-03T19:33:21.385 INFO:journalctl@ceph.grafana.a.smithi164.stdout:Jun 03 19:33:21 smithi164 bash[47754]: t=2023-06-03T19:33:21+0000 lvl=info msg="Executing migration" logger=migrator id="alter alert_rule_version table data column to mediumtext in mysql" 2023-06-03T19:33:21.386 INFO:journalctl@ceph.grafana.a.smithi164.stdout:Jun 03 19:33:21 smithi164 bash[47754]: t=2023-06-03T19:33:21+0000 lvl=info msg="Executing migration" logger=migrator id="add column for to alert_rule_version" 2023-06-03T19:33:21.386 INFO:journalctl@ceph.grafana.a.smithi164.stdout:Jun 03 19:33:21 smithi164 bash[47754]: t=2023-06-03T19:33:21+0000 lvl=info msg="Executing migration" logger=migrator id="add column annotations to alert_rule_version" 2023-06-03T19:33:21.646 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:33:21 smithi164 bash[26855]: cluster 2023-06-03T19:33:20.398041+0000 mgr.y (mgr.14445) 63 : cluster [DBG] pgmap v53: 9 pgs: 9 active+clean; 0 B data, 43 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:33:21.646 INFO:journalctl@ceph.grafana.a.smithi164.stdout:Jun 03 19:33:21 smithi164 bash[47754]: t=2023-06-03T19:33:21+0000 lvl=info msg="Executing migration" logger=migrator id="add column labels to alert_rule_version" 2023-06-03T19:33:21.646 INFO:journalctl@ceph.grafana.a.smithi164.stdout:Jun 03 19:33:21 smithi164 bash[47754]: t=2023-06-03T19:33:21+0000 lvl=info msg="Executing migration" logger=migrator id=create_alert_configuration_table 2023-06-03T19:33:21.646 INFO:journalctl@ceph.grafana.a.smithi164.stdout:Jun 03 19:33:21 smithi164 bash[47754]: t=2023-06-03T19:33:21+0000 lvl=info msg="Executing migration" logger=migrator id="Add column default in alert_configuration" 2023-06-03T19:33:21.647 INFO:journalctl@ceph.grafana.a.smithi164.stdout:Jun 03 19:33:21 smithi164 bash[47754]: t=2023-06-03T19:33:21+0000 lvl=info msg="Executing migration" logger=migrator id="alert alert_configuration alertmanager_configuration column from TEXT to MEDIUMTEXT if mysql" 2023-06-03T19:33:21.647 INFO:journalctl@ceph.grafana.a.smithi164.stdout:Jun 03 19:33:21 smithi164 bash[47754]: t=2023-06-03T19:33:21+0000 lvl=info msg="Executing migration" logger=migrator id="add column org_id in alert_configuration" 2023-06-03T19:33:21.647 INFO:journalctl@ceph.grafana.a.smithi164.stdout:Jun 03 19:33:21 smithi164 bash[47754]: t=2023-06-03T19:33:21+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_configuration table on org_id column" 2023-06-03T19:33:21.647 INFO:journalctl@ceph.grafana.a.smithi164.stdout:Jun 03 19:33:21 smithi164 bash[47754]: t=2023-06-03T19:33:21+0000 lvl=info msg="Executing migration" logger=migrator id=create_ngalert_configuration_table 2023-06-03T19:33:21.678 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/mon.a/config 2023-06-03T19:33:21.781 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:33:21 smithi089 bash[33173]: cluster 2023-06-03T19:33:20.398041+0000 mgr.y (mgr.14445) 63 : cluster [DBG] pgmap v53: 9 pgs: 9 active+clean; 0 B data, 43 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:33:21.781 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:33:21 smithi089 bash[26572]: cluster 2023-06-03T19:33:20.398041+0000 mgr.y (mgr.14445) 63 : cluster [DBG] pgmap v53: 9 pgs: 9 active+clean; 0 B data, 43 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:33:21.906 INFO:journalctl@ceph.grafana.a.smithi164.stdout:Jun 03 19:33:21 smithi164 bash[47754]: t=2023-06-03T19:33:21+0000 lvl=info msg="Executing migration" logger=migrator id="add index in ngalert_configuration on org_id column" 2023-06-03T19:33:21.906 INFO:journalctl@ceph.grafana.a.smithi164.stdout:Jun 03 19:33:21 smithi164 bash[47754]: t=2023-06-03T19:33:21+0000 lvl=info msg="Executing migration" logger=migrator id="clear migration entry \"remove unified alerting data\"" 2023-06-03T19:33:21.907 INFO:journalctl@ceph.grafana.a.smithi164.stdout:Jun 03 19:33:21 smithi164 bash[47754]: t=2023-06-03T19:33:21+0000 lvl=info msg="Executing migration" logger=migrator id="move dashboard alerts to unified alerting" 2023-06-03T19:33:21.907 INFO:journalctl@ceph.grafana.a.smithi164.stdout:Jun 03 19:33:21 smithi164 bash[47754]: t=2023-06-03T19:33:21+0000 lvl=info msg="Executing migration" logger=migrator id="create library_element table v1" 2023-06-03T19:33:21.907 INFO:journalctl@ceph.grafana.a.smithi164.stdout:Jun 03 19:33:21 smithi164 bash[47754]: t=2023-06-03T19:33:21+0000 lvl=info msg="Executing migration" logger=migrator id="add index library_element org_id-folder_id-name-kind" 2023-06-03T19:33:21.907 INFO:journalctl@ceph.grafana.a.smithi164.stdout:Jun 03 19:33:21 smithi164 bash[47754]: t=2023-06-03T19:33:21+0000 lvl=info msg="Executing migration" logger=migrator id="create library_element_connection table v1" 2023-06-03T19:33:21.907 INFO:journalctl@ceph.grafana.a.smithi164.stdout:Jun 03 19:33:21 smithi164 bash[47754]: t=2023-06-03T19:33:21+0000 lvl=info msg="Executing migration" logger=migrator id="add index library_element_connection element_id-kind-connection_id" 2023-06-03T19:33:21.907 INFO:journalctl@ceph.grafana.a.smithi164.stdout:Jun 03 19:33:21 smithi164 bash[47754]: t=2023-06-03T19:33:21+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index library_element org_id_uid" 2023-06-03T19:33:21.907 INFO:journalctl@ceph.grafana.a.smithi164.stdout:Jun 03 19:33:21 smithi164 bash[47754]: t=2023-06-03T19:33:21+0000 lvl=info msg="Executing migration" logger=migrator id="clone move dashboard alerts to unified alerting" 2023-06-03T19:33:22.192 INFO:journalctl@ceph.grafana.a.smithi164.stdout:Jun 03 19:33:21 smithi164 bash[47754]: t=2023-06-03T19:33:21+0000 lvl=info msg="Executing migration" logger=migrator id="create data_keys table" 2023-06-03T19:33:22.192 INFO:journalctl@ceph.grafana.a.smithi164.stdout:Jun 03 19:33:21 smithi164 bash[47754]: t=2023-06-03T19:33:21+0000 lvl=info msg="Executing migration" logger=migrator id="create kv_store table v1" 2023-06-03T19:33:22.193 INFO:journalctl@ceph.grafana.a.smithi164.stdout:Jun 03 19:33:21 smithi164 bash[47754]: t=2023-06-03T19:33:21+0000 lvl=info msg="Executing migration" logger=migrator id="add index kv_store.org_id-namespace-key" 2023-06-03T19:33:22.193 INFO:journalctl@ceph.grafana.a.smithi164.stdout:Jun 03 19:33:22 smithi164 bash[47754]: t=2023-06-03T19:33:22+0000 lvl=info msg="Executing migration" logger=migrator id="update dashboard_uid and panel_id from existing annotations" 2023-06-03T19:33:22.193 INFO:journalctl@ceph.grafana.a.smithi164.stdout:Jun 03 19:33:22 smithi164 bash[47754]: t=2023-06-03T19:33:22+0000 lvl=info msg="Executing migration" logger=migrator id="create permission table" 2023-06-03T19:33:22.193 INFO:journalctl@ceph.grafana.a.smithi164.stdout:Jun 03 19:33:22 smithi164 bash[47754]: t=2023-06-03T19:33:22+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index permission.role_id" 2023-06-03T19:33:22.193 INFO:journalctl@ceph.grafana.a.smithi164.stdout:Jun 03 19:33:22 smithi164 bash[47754]: t=2023-06-03T19:33:22+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index role_id_action_scope" 2023-06-03T19:33:22.193 INFO:journalctl@ceph.grafana.a.smithi164.stdout:Jun 03 19:33:22 smithi164 bash[47754]: t=2023-06-03T19:33:22+0000 lvl=info msg="Executing migration" logger=migrator id="create role table" 2023-06-03T19:33:22.193 INFO:journalctl@ceph.grafana.a.smithi164.stdout:Jun 03 19:33:22 smithi164 bash[47754]: t=2023-06-03T19:33:22+0000 lvl=info msg="Executing migration" logger=migrator id="add column display_name" 2023-06-03T19:33:22.193 INFO:journalctl@ceph.grafana.a.smithi164.stdout:Jun 03 19:33:22 smithi164 bash[47754]: t=2023-06-03T19:33:22+0000 lvl=info msg="Executing migration" logger=migrator id="add column group_name" 2023-06-03T19:33:22.444 INFO:journalctl@ceph.grafana.a.smithi164.stdout:Jun 03 19:33:22 smithi164 bash[47754]: t=2023-06-03T19:33:22+0000 lvl=info msg="Executing migration" logger=migrator id="add index role.org_id" 2023-06-03T19:33:22.445 INFO:journalctl@ceph.grafana.a.smithi164.stdout:Jun 03 19:33:22 smithi164 bash[47754]: t=2023-06-03T19:33:22+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index role_org_id_name" 2023-06-03T19:33:22.445 INFO:journalctl@ceph.grafana.a.smithi164.stdout:Jun 03 19:33:22 smithi164 bash[47754]: t=2023-06-03T19:33:22+0000 lvl=info msg="Executing migration" logger=migrator id="add index role_org_id_uid" 2023-06-03T19:33:22.445 INFO:journalctl@ceph.grafana.a.smithi164.stdout:Jun 03 19:33:22 smithi164 bash[47754]: t=2023-06-03T19:33:22+0000 lvl=info msg="Executing migration" logger=migrator id="create team role table" 2023-06-03T19:33:22.445 INFO:journalctl@ceph.grafana.a.smithi164.stdout:Jun 03 19:33:22 smithi164 bash[47754]: t=2023-06-03T19:33:22+0000 lvl=info msg="Executing migration" logger=migrator id="add index team_role.org_id" 2023-06-03T19:33:22.445 INFO:journalctl@ceph.grafana.a.smithi164.stdout:Jun 03 19:33:22 smithi164 bash[47754]: t=2023-06-03T19:33:22+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index team_role_org_id_team_id_role_id" 2023-06-03T19:33:22.445 INFO:journalctl@ceph.grafana.a.smithi164.stdout:Jun 03 19:33:22 smithi164 bash[47754]: t=2023-06-03T19:33:22+0000 lvl=info msg="Executing migration" logger=migrator id="add index team_role.team_id" 2023-06-03T19:33:22.694 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/mon.a/config 2023-06-03T19:33:22.695 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/mon.a/config 2023-06-03T19:33:22.696 INFO:journalctl@ceph.grafana.a.smithi164.stdout:Jun 03 19:33:22 smithi164 bash[47754]: t=2023-06-03T19:33:22+0000 lvl=info msg="Executing migration" logger=migrator id="create user role table" 2023-06-03T19:33:22.696 INFO:journalctl@ceph.grafana.a.smithi164.stdout:Jun 03 19:33:22 smithi164 bash[47754]: t=2023-06-03T19:33:22+0000 lvl=info msg="Executing migration" logger=migrator id="add index user_role.org_id" 2023-06-03T19:33:22.697 INFO:journalctl@ceph.grafana.a.smithi164.stdout:Jun 03 19:33:22 smithi164 bash[47754]: t=2023-06-03T19:33:22+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index user_role_org_id_user_id_role_id" 2023-06-03T19:33:22.697 INFO:journalctl@ceph.grafana.a.smithi164.stdout:Jun 03 19:33:22 smithi164 bash[47754]: t=2023-06-03T19:33:22+0000 lvl=info msg="Executing migration" logger=migrator id="add index user_role.user_id" 2023-06-03T19:33:22.697 INFO:journalctl@ceph.grafana.a.smithi164.stdout:Jun 03 19:33:22 smithi164 bash[47754]: t=2023-06-03T19:33:22+0000 lvl=info msg="Executing migration" logger=migrator id="create builtin role table" 2023-06-03T19:33:22.697 INFO:journalctl@ceph.grafana.a.smithi164.stdout:Jun 03 19:33:22 smithi164 bash[47754]: t=2023-06-03T19:33:22+0000 lvl=info msg="Executing migration" logger=migrator id="add index builtin_role.role_id" 2023-06-03T19:33:22.697 INFO:journalctl@ceph.grafana.a.smithi164.stdout:Jun 03 19:33:22 smithi164 bash[47754]: t=2023-06-03T19:33:22+0000 lvl=info msg="Executing migration" logger=migrator id="add index builtin_role.name" 2023-06-03T19:33:22.697 INFO:journalctl@ceph.grafana.a.smithi164.stdout:Jun 03 19:33:22 smithi164 bash[47754]: t=2023-06-03T19:33:22+0000 lvl=info msg="Executing migration" logger=migrator id="Add column org_id to builtin_role table" 2023-06-03T19:33:22.697 INFO:journalctl@ceph.grafana.a.smithi164.stdout:Jun 03 19:33:22 smithi164 bash[47754]: t=2023-06-03T19:33:22+0000 lvl=info msg="Executing migration" logger=migrator id="add index builtin_role.org_id" 2023-06-03T19:33:22.698 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:33:22 smithi164 bash[26855]: audit 2023-06-03T19:33:21.572791+0000 mon.a (mon.0) 625 : audit [INF] from='mgr.14445 172.21.15.89:0/573502486' entity='mgr.y' 2023-06-03T19:33:22.698 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:33:22 smithi164 bash[26855]: audit 2023-06-03T19:33:21.819601+0000 mon.a (mon.0) 626 : audit [DBG] from='mgr.14445 172.21.15.89:0/573502486' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-03T19:33:22.698 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:33:22 smithi164 bash[26855]: audit 2023-06-03T19:33:21.820940+0000 mon.a (mon.0) 627 : audit [INF] from='mgr.14445 172.21.15.89:0/573502486' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-03T19:33:22.698 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:33:22 smithi164 bash[26855]: audit 2023-06-03T19:33:21.829818+0000 mon.a (mon.0) 628 : audit [INF] from='mgr.14445 172.21.15.89:0/573502486' entity='mgr.y' 2023-06-03T19:33:22.971 INFO:journalctl@ceph.grafana.a.smithi164.stdout:Jun 03 19:33:22 smithi164 bash[47754]: t=2023-06-03T19:33:22+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index builtin_role_org_id_role_id_role" 2023-06-03T19:33:22.971 INFO:journalctl@ceph.grafana.a.smithi164.stdout:Jun 03 19:33:22 smithi164 bash[47754]: t=2023-06-03T19:33:22+0000 lvl=info msg="Executing migration" logger=migrator id="Remove unique index role_org_id_uid" 2023-06-03T19:33:22.971 INFO:journalctl@ceph.grafana.a.smithi164.stdout:Jun 03 19:33:22 smithi164 bash[47754]: t=2023-06-03T19:33:22+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index role.uid" 2023-06-03T19:33:22.972 INFO:journalctl@ceph.grafana.a.smithi164.stdout:Jun 03 19:33:22 smithi164 bash[47754]: t=2023-06-03T19:33:22+0000 lvl=info msg="Executing migration" logger=migrator id="create seed assignment table" 2023-06-03T19:33:22.972 INFO:journalctl@ceph.grafana.a.smithi164.stdout:Jun 03 19:33:22 smithi164 bash[47754]: t=2023-06-03T19:33:22+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index builtin_role_role_name" 2023-06-03T19:33:22.972 INFO:journalctl@ceph.grafana.a.smithi164.stdout:Jun 03 19:33:22 smithi164 bash[47754]: t=2023-06-03T19:33:22+0000 lvl=info msg="migrations completed" logger=migrator performed=381 skipped=0 duration=12.301091165s 2023-06-03T19:33:22.972 INFO:journalctl@ceph.grafana.a.smithi164.stdout:Jun 03 19:33:22 smithi164 bash[47754]: t=2023-06-03T19:33:22+0000 lvl=info msg="Created default organization" logger=sqlstore 2023-06-03T19:33:22.972 INFO:journalctl@ceph.grafana.a.smithi164.stdout:Jun 03 19:33:22 smithi164 bash[47754]: t=2023-06-03T19:33:22+0000 lvl=info msg="Initialising plugins" logger=plugin.manager 2023-06-03T19:33:22.972 INFO:journalctl@ceph.grafana.a.smithi164.stdout:Jun 03 19:33:22 smithi164 bash[47754]: t=2023-06-03T19:33:22+0000 lvl=info msg="Plugin registered" logger=plugin.manager pluginId=input 2023-06-03T19:33:22.972 INFO:journalctl@ceph.grafana.a.smithi164.stdout:Jun 03 19:33:22 smithi164 bash[47754]: t=2023-06-03T19:33:22+0000 lvl=info msg="Plugin registered" logger=plugin.manager pluginId=vonage-status-panel 2023-06-03T19:33:22.973 INFO:journalctl@ceph.grafana.a.smithi164.stdout:Jun 03 19:33:22 smithi164 bash[47754]: t=2023-06-03T19:33:22+0000 lvl=info msg="Plugin registered" logger=plugin.manager pluginId=grafana-piechart-panel 2023-06-03T19:33:23.031 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:33:22 smithi089 bash[33173]: audit 2023-06-03T19:33:21.572791+0000 mon.a (mon.0) 625 : audit [INF] from='mgr.14445 172.21.15.89:0/573502486' entity='mgr.y' 2023-06-03T19:33:23.032 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:33:22 smithi089 bash[33173]: audit 2023-06-03T19:33:21.819601+0000 mon.a (mon.0) 626 : audit [DBG] from='mgr.14445 172.21.15.89:0/573502486' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-03T19:33:23.032 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:33:22 smithi089 bash[33173]: audit 2023-06-03T19:33:21.820940+0000 mon.a (mon.0) 627 : audit [INF] from='mgr.14445 172.21.15.89:0/573502486' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-03T19:33:23.032 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:33:22 smithi089 bash[33173]: audit 2023-06-03T19:33:21.829818+0000 mon.a (mon.0) 628 : audit [INF] from='mgr.14445 172.21.15.89:0/573502486' entity='mgr.y' 2023-06-03T19:33:23.032 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:33:22 smithi089 bash[26572]: audit 2023-06-03T19:33:21.572791+0000 mon.a (mon.0) 625 : audit [INF] from='mgr.14445 172.21.15.89:0/573502486' entity='mgr.y' 2023-06-03T19:33:23.032 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:33:22 smithi089 bash[26572]: audit 2023-06-03T19:33:21.819601+0000 mon.a (mon.0) 626 : audit [DBG] from='mgr.14445 172.21.15.89:0/573502486' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-03T19:33:23.033 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:33:22 smithi089 bash[26572]: audit 2023-06-03T19:33:21.820940+0000 mon.a (mon.0) 627 : audit [INF] from='mgr.14445 172.21.15.89:0/573502486' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-03T19:33:23.033 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:33:22 smithi089 bash[26572]: audit 2023-06-03T19:33:21.829818+0000 mon.a (mon.0) 628 : audit [INF] from='mgr.14445 172.21.15.89:0/573502486' entity='mgr.y' 2023-06-03T19:33:23.336 INFO:journalctl@ceph.grafana.a.smithi164.stdout:Jun 03 19:33:22 smithi164 bash[47754]: t=2023-06-03T19:33:22+0000 lvl=info msg="Live Push Gateway initialization" logger=live.push_http 2023-06-03T19:33:23.336 INFO:journalctl@ceph.grafana.a.smithi164.stdout:Jun 03 19:33:23 smithi164 bash[47754]: t=2023-06-03T19:33:23+0000 lvl=warn msg="[Deprecated] the datasource provisioning config is outdated. please upgrade" logger=provisioning.datasources filename=/etc/grafana/provisioning/datasources/ceph-dashboard.yml 2023-06-03T19:33:23.336 INFO:journalctl@ceph.grafana.a.smithi164.stdout:Jun 03 19:33:23 smithi164 bash[47754]: t=2023-06-03T19:33:23+0000 lvl=info msg="inserting datasource from configuration " logger=provisioning.datasources name=Dashboard1 uid=P43CA22E17D0F9596 2023-06-03T19:33:23.337 INFO:journalctl@ceph.grafana.a.smithi164.stdout:Jun 03 19:33:23 smithi164 bash[47754]: t=2023-06-03T19:33:23+0000 lvl=info msg="warming cache for startup" logger=ngalert 2023-06-03T19:33:23.337 INFO:journalctl@ceph.grafana.a.smithi164.stdout:Jun 03 19:33:23 smithi164 bash[47754]: t=2023-06-03T19:33:23+0000 lvl=info msg="HTTP Server Listen" logger=http.server address=[::]:3000 protocol=https subUrl= socket= 2023-06-03T19:33:23.337 INFO:journalctl@ceph.grafana.a.smithi164.stdout:Jun 03 19:33:23 smithi164 bash[47754]: t=2023-06-03T19:33:23+0000 lvl=info msg="starting MultiOrg Alertmanager" logger=ngalert.multiorg.alertmanager 2023-06-03T19:33:23.781 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:33:23 smithi089 bash[33173]: cephadm 2023-06-03T19:33:21.835843+0000 mgr.y (mgr.14445) 64 : cephadm [INF] Reconfiguring alertmanager.a (dependencies changed)... 2023-06-03T19:33:23.782 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:33:23 smithi089 bash[33173]: cephadm 2023-06-03T19:33:21.839970+0000 mgr.y (mgr.14445) 65 : cephadm [INF] Reconfiguring daemon alertmanager.a on smithi089 2023-06-03T19:33:23.782 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:33:23 smithi089 bash[33173]: cluster 2023-06-03T19:33:22.398487+0000 mgr.y (mgr.14445) 66 : cluster [DBG] pgmap v54: 9 pgs: 9 active+clean; 0 B data, 43 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:33:23.782 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:33:23 smithi089 bash[26572]: cephadm 2023-06-03T19:33:21.835843+0000 mgr.y (mgr.14445) 64 : cephadm [INF] Reconfiguring alertmanager.a (dependencies changed)... 2023-06-03T19:33:23.782 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:33:23 smithi089 bash[26572]: cephadm 2023-06-03T19:33:21.839970+0000 mgr.y (mgr.14445) 65 : cephadm [INF] Reconfiguring daemon alertmanager.a on smithi089 2023-06-03T19:33:23.782 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:33:23 smithi089 bash[26572]: cluster 2023-06-03T19:33:22.398487+0000 mgr.y (mgr.14445) 66 : cluster [DBG] pgmap v54: 9 pgs: 9 active+clean; 0 B data, 43 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:33:23.836 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:33:23 smithi164 bash[26855]: cephadm 2023-06-03T19:33:21.835843+0000 mgr.y (mgr.14445) 64 : cephadm [INF] Reconfiguring alertmanager.a (dependencies changed)... 2023-06-03T19:33:23.836 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:33:23 smithi164 bash[26855]: cephadm 2023-06-03T19:33:21.839970+0000 mgr.y (mgr.14445) 65 : cephadm [INF] Reconfiguring daemon alertmanager.a on smithi089 2023-06-03T19:33:23.837 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:33:23 smithi164 bash[26855]: cluster 2023-06-03T19:33:22.398487+0000 mgr.y (mgr.14445) 66 : cluster [DBG] pgmap v54: 9 pgs: 9 active+clean; 0 B data, 43 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:33:25.654 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:33:25 smithi089 bash[26572]: cluster 2023-06-03T19:33:24.398918+0000 mgr.y (mgr.14445) 67 : cluster [DBG] pgmap v55: 9 pgs: 9 active+clean; 0 B data, 43 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:33:25.655 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:33:25 smithi089 bash[33173]: cluster 2023-06-03T19:33:24.398918+0000 mgr.y (mgr.14445) 67 : cluster [DBG] pgmap v55: 9 pgs: 9 active+clean; 0 B data, 43 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:33:26.086 INFO:journalctl@ceph.mgr.x.smithi164.stdout:Jun 03 19:33:25 smithi164 bash[27934]: ::ffff:172.21.15.164 - - [03/Jun/2023:19:33:25] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2023-06-03T19:33:26.087 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:33:25 smithi164 bash[26855]: cluster 2023-06-03T19:33:24.398918+0000 mgr.y (mgr.14445) 67 : cluster [DBG] pgmap v55: 9 pgs: 9 active+clean; 0 B data, 43 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:33:27.803 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:33:27 smithi089 bash[26572]: cluster 2023-06-03T19:33:26.399351+0000 mgr.y (mgr.14445) 68 : cluster [DBG] pgmap v56: 9 pgs: 9 active+clean; 0 B data, 43 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:33:27.804 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:33:27 smithi089 bash[33173]: cluster 2023-06-03T19:33:26.399351+0000 mgr.y (mgr.14445) 68 : cluster [DBG] pgmap v56: 9 pgs: 9 active+clean; 0 B data, 43 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:33:28.086 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:33:27 smithi164 bash[26855]: cluster 2023-06-03T19:33:26.399351+0000 mgr.y (mgr.14445) 68 : cluster [DBG] pgmap v56: 9 pgs: 9 active+clean; 0 B data, 43 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:33:28.237 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 19:33:27 smithi089 bash[26845]: ::ffff:172.21.15.164 - - [03/Jun/2023:19:33:27] "GET /metrics HTTP/1.1" 200 151997 "" "Prometheus/2.33.4" 2023-06-03T19:33:29.971 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:33:29 smithi089 bash[26572]: cluster 2023-06-03T19:33:28.399885+0000 mgr.y (mgr.14445) 69 : cluster [DBG] pgmap v57: 9 pgs: 9 active+clean; 0 B data, 43 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:33:29.972 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:33:29 smithi089 bash[33173]: cluster 2023-06-03T19:33:28.399885+0000 mgr.y (mgr.14445) 69 : cluster [DBG] pgmap v57: 9 pgs: 9 active+clean; 0 B data, 43 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:33:30.086 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:33:29 smithi164 bash[26855]: cluster 2023-06-03T19:33:28.399885+0000 mgr.y (mgr.14445) 69 : cluster [DBG] pgmap v57: 9 pgs: 9 active+clean; 0 B data, 43 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:33:32.031 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:33:31 smithi089 bash[26572]: cluster 2023-06-03T19:33:30.400348+0000 mgr.y (mgr.14445) 70 : cluster [DBG] pgmap v58: 9 pgs: 9 active+clean; 0 B data, 43 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:33:32.031 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:33:31 smithi089 bash[33173]: cluster 2023-06-03T19:33:30.400348+0000 mgr.y (mgr.14445) 70 : cluster [DBG] pgmap v58: 9 pgs: 9 active+clean; 0 B data, 43 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:33:32.079 INFO:teuthology.orchestra.run.smithi089.stdout:77309411402 2023-06-03T19:33:32.086 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:33:31 smithi164 bash[26855]: cluster 2023-06-03T19:33:30.400348+0000 mgr.y (mgr.14445) 70 : cluster [DBG] pgmap v58: 9 pgs: 9 active+clean; 0 B data, 43 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:33:32.881 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:33:32 smithi089 bash[26572]: audit 2023-06-03T19:33:32.072210+0000 mon.c (mon.2) 18 : audit [DBG] from='client.? 172.21.15.89:0/158474754' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2023-06-03T19:33:32.881 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:33:32 smithi089 bash[33173]: audit 2023-06-03T19:33:32.072210+0000 mon.c (mon.2) 18 : audit [DBG] from='client.? 172.21.15.89:0/158474754' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2023-06-03T19:33:33.086 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:33:32 smithi164 bash[26855]: audit 2023-06-03T19:33:32.072210+0000 mon.c (mon.2) 18 : audit [DBG] from='client.? 172.21.15.89:0/158474754' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2023-06-03T19:33:33.849 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:33:33 smithi089 bash[26572]: cluster 2023-06-03T19:33:32.400962+0000 mgr.y (mgr.14445) 71 : cluster [DBG] pgmap v59: 9 pgs: 9 active+clean; 0 B data, 43 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:33:33.849 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:33:33 smithi089 bash[33173]: cluster 2023-06-03T19:33:32.400962+0000 mgr.y (mgr.14445) 71 : cluster [DBG] pgmap v59: 9 pgs: 9 active+clean; 0 B data, 43 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:33:34.086 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:33:33 smithi164 bash[26855]: cluster 2023-06-03T19:33:32.400962+0000 mgr.y (mgr.14445) 71 : cluster [DBG] pgmap v59: 9 pgs: 9 active+clean; 0 B data, 43 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:33:35.352 INFO:tasks.cephadm.ceph_manager.ceph:need seq 77309411398 got 77309411402 for osd.2 2023-06-03T19:33:35.352 DEBUG:teuthology.parallel:result is None 2023-06-03T19:33:36.031 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:33:35 smithi089 bash[26572]: cluster 2023-06-03T19:33:34.401403+0000 mgr.y (mgr.14445) 72 : cluster [DBG] pgmap v60: 9 pgs: 9 active+clean; 0 B data, 43 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:33:36.032 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:33:35 smithi089 bash[33173]: cluster 2023-06-03T19:33:34.401403+0000 mgr.y (mgr.14445) 72 : cluster [DBG] pgmap v60: 9 pgs: 9 active+clean; 0 B data, 43 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:33:36.086 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:33:35 smithi164 bash[26855]: cluster 2023-06-03T19:33:34.401403+0000 mgr.y (mgr.14445) 72 : cluster [DBG] pgmap v60: 9 pgs: 9 active+clean; 0 B data, 43 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:33:36.087 INFO:journalctl@ceph.mgr.x.smithi164.stdout:Jun 03 19:33:35 smithi164 bash[27934]: ::ffff:172.21.15.164 - - [03/Jun/2023:19:33:35] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2023-06-03T19:33:36.856 INFO:teuthology.orchestra.run.smithi089.stdout:55834574933 2023-06-03T19:33:37.570 INFO:teuthology.orchestra.run.smithi089.stdout:154618822705 2023-06-03T19:33:37.908 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:33:37 smithi089 bash[26572]: cluster 2023-06-03T19:33:36.401785+0000 mgr.y (mgr.14445) 73 : cluster [DBG] pgmap v61: 9 pgs: 9 active+clean; 0 B data, 43 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:33:37.908 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:33:37 smithi089 bash[26572]: audit 2023-06-03T19:33:36.849773+0000 mon.a (mon.0) 629 : audit [DBG] from='client.? 172.21.15.89:0/1191714637' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2023-06-03T19:33:37.908 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:33:37 smithi089 bash[26572]: audit 2023-06-03T19:33:37.562959+0000 mon.c (mon.2) 19 : audit [DBG] from='client.? 172.21.15.89:0/188096100' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2023-06-03T19:33:37.909 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:33:37 smithi089 bash[33173]: cluster 2023-06-03T19:33:36.401785+0000 mgr.y (mgr.14445) 73 : cluster [DBG] pgmap v61: 9 pgs: 9 active+clean; 0 B data, 43 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:33:37.909 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:33:37 smithi089 bash[33173]: audit 2023-06-03T19:33:36.849773+0000 mon.a (mon.0) 629 : audit [DBG] from='client.? 172.21.15.89:0/1191714637' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2023-06-03T19:33:37.909 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:33:37 smithi089 bash[33173]: audit 2023-06-03T19:33:37.562959+0000 mon.c (mon.2) 19 : audit [DBG] from='client.? 172.21.15.89:0/188096100' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2023-06-03T19:33:38.086 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:33:37 smithi164 bash[26855]: cluster 2023-06-03T19:33:36.401785+0000 mgr.y (mgr.14445) 73 : cluster [DBG] pgmap v61: 9 pgs: 9 active+clean; 0 B data, 43 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:33:38.086 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:33:37 smithi164 bash[26855]: audit 2023-06-03T19:33:36.849773+0000 mon.a (mon.0) 629 : audit [DBG] from='client.? 172.21.15.89:0/1191714637' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2023-06-03T19:33:38.087 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:33:37 smithi164 bash[26855]: audit 2023-06-03T19:33:37.562959+0000 mon.c (mon.2) 19 : audit [DBG] from='client.? 172.21.15.89:0/188096100' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2023-06-03T19:33:38.281 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 19:33:37 smithi089 bash[26845]: ::ffff:172.21.15.164 - - [03/Jun/2023:19:33:37] "GET /metrics HTTP/1.1" 200 151995 "" "Prometheus/2.33.4" 2023-06-03T19:33:38.352 INFO:teuthology.orchestra.run.smithi089.stdout:206158430238 2023-06-03T19:33:38.380 INFO:teuthology.orchestra.run.smithi089.stdout:128849018938 2023-06-03T19:33:38.651 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:33:38 smithi089 bash[26572]: audit 2023-06-03T19:33:38.344950+0000 mon.c (mon.2) 20 : audit [DBG] from='client.? 172.21.15.89:0/3098278080' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2023-06-03T19:33:38.652 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:33:38 smithi089 bash[26572]: audit 2023-06-03T19:33:38.373182+0000 mon.a (mon.0) 630 : audit [DBG] from='client.? 172.21.15.89:0/1123912026' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2023-06-03T19:33:39.031 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:33:38 smithi089 bash[33173]: audit 2023-06-03T19:33:38.344950+0000 mon.c (mon.2) 20 : audit [DBG] from='client.? 172.21.15.89:0/3098278080' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2023-06-03T19:33:39.031 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:33:38 smithi089 bash[33173]: audit 2023-06-03T19:33:38.373182+0000 mon.a (mon.0) 630 : audit [DBG] from='client.? 172.21.15.89:0/1123912026' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2023-06-03T19:33:39.086 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:33:38 smithi164 bash[26855]: audit 2023-06-03T19:33:38.344950+0000 mon.c (mon.2) 20 : audit [DBG] from='client.? 172.21.15.89:0/3098278080' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2023-06-03T19:33:39.086 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:33:38 smithi164 bash[26855]: audit 2023-06-03T19:33:38.373182+0000 mon.a (mon.0) 630 : audit [DBG] from='client.? 172.21.15.89:0/1123912026' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2023-06-03T19:33:39.739 INFO:tasks.cephadm.ceph_manager.ceph:need seq 55834574928 got 55834574933 for osd.1 2023-06-03T19:33:39.740 DEBUG:teuthology.parallel:result is None 2023-06-03T19:33:40.031 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:33:39 smithi089 bash[26572]: cluster 2023-06-03T19:33:38.402202+0000 mgr.y (mgr.14445) 74 : cluster [DBG] pgmap v62: 9 pgs: 9 active+clean; 0 B data, 43 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:33:40.031 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:33:39 smithi089 bash[33173]: cluster 2023-06-03T19:33:38.402202+0000 mgr.y (mgr.14445) 74 : cluster [DBG] pgmap v62: 9 pgs: 9 active+clean; 0 B data, 43 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:33:40.066 INFO:teuthology.orchestra.run.smithi089.stdout:34359738462 2023-06-03T19:33:40.086 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:33:39 smithi164 bash[26855]: cluster 2023-06-03T19:33:38.402202+0000 mgr.y (mgr.14445) 74 : cluster [DBG] pgmap v62: 9 pgs: 9 active+clean; 0 B data, 43 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:33:40.486 INFO:tasks.cephadm.ceph_manager.ceph:need seq 154618822700 got 154618822705 for osd.5 2023-06-03T19:33:40.486 DEBUG:teuthology.parallel:result is None 2023-06-03T19:33:40.596 INFO:teuthology.orchestra.run.smithi089.stdout:180388626471 2023-06-03T19:33:40.781 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:33:40 smithi089 bash[26572]: audit 2023-06-03T19:33:40.058938+0000 mon.a (mon.0) 631 : audit [DBG] from='client.? 172.21.15.89:0/2984757070' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2023-06-03T19:33:40.781 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:33:40 smithi089 bash[26572]: audit 2023-06-03T19:33:40.589443+0000 mon.c (mon.2) 21 : audit [DBG] from='client.? 172.21.15.89:0/3660607757' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2023-06-03T19:33:40.782 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:33:40 smithi089 bash[33173]: audit 2023-06-03T19:33:40.058938+0000 mon.a (mon.0) 631 : audit [DBG] from='client.? 172.21.15.89:0/2984757070' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2023-06-03T19:33:40.782 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:33:40 smithi089 bash[33173]: audit 2023-06-03T19:33:40.589443+0000 mon.c (mon.2) 21 : audit [DBG] from='client.? 172.21.15.89:0/3660607757' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2023-06-03T19:33:41.086 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:33:40 smithi164 bash[26855]: audit 2023-06-03T19:33:40.058938+0000 mon.a (mon.0) 631 : audit [DBG] from='client.? 172.21.15.89:0/2984757070' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2023-06-03T19:33:41.087 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:33:40 smithi164 bash[26855]: audit 2023-06-03T19:33:40.589443+0000 mon.c (mon.2) 21 : audit [DBG] from='client.? 172.21.15.89:0/3660607757' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2023-06-03T19:33:41.383 INFO:tasks.cephadm.ceph_manager.ceph:need seq 128849018933 got 128849018938 for osd.4 2023-06-03T19:33:41.383 DEBUG:teuthology.parallel:result is None 2023-06-03T19:33:41.924 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:33:41 smithi089 bash[33173]: cluster 2023-06-03T19:33:40.402574+0000 mgr.y (mgr.14445) 75 : cluster [DBG] pgmap v63: 9 pgs: 9 active+clean; 0 B data, 43 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:33:41.924 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:33:41 smithi089 bash[26572]: cluster 2023-06-03T19:33:40.402574+0000 mgr.y (mgr.14445) 75 : cluster [DBG] pgmap v63: 9 pgs: 9 active+clean; 0 B data, 43 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:33:42.086 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:33:41 smithi164 bash[26855]: cluster 2023-06-03T19:33:40.402574+0000 mgr.y (mgr.14445) 75 : cluster [DBG] pgmap v63: 9 pgs: 9 active+clean; 0 B data, 43 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:33:43.039 INFO:tasks.cephadm.ceph_manager.ceph:need seq 34359738457 got 34359738462 for osd.0 2023-06-03T19:33:43.039 DEBUG:teuthology.parallel:result is None 2023-06-03T19:33:43.349 INFO:teuthology.orchestra.run.smithi089.stdout:107374182467 2023-06-03T19:33:43.781 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:33:43 smithi089 bash[33173]: cluster 2023-06-03T19:33:42.403034+0000 mgr.y (mgr.14445) 76 : cluster [DBG] pgmap v64: 9 pgs: 9 active+clean; 0 B data, 43 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:33:43.781 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:33:43 smithi089 bash[33173]: audit 2023-06-03T19:33:43.342818+0000 mon.c (mon.2) 22 : audit [DBG] from='client.? 172.21.15.89:0/3173368842' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2023-06-03T19:33:43.782 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:33:43 smithi089 bash[26572]: cluster 2023-06-03T19:33:42.403034+0000 mgr.y (mgr.14445) 76 : cluster [DBG] pgmap v64: 9 pgs: 9 active+clean; 0 B data, 43 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:33:43.782 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:33:43 smithi089 bash[26572]: audit 2023-06-03T19:33:43.342818+0000 mon.c (mon.2) 22 : audit [DBG] from='client.? 172.21.15.89:0/3173368842' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2023-06-03T19:33:44.086 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:33:43 smithi164 bash[26855]: cluster 2023-06-03T19:33:42.403034+0000 mgr.y (mgr.14445) 76 : cluster [DBG] pgmap v64: 9 pgs: 9 active+clean; 0 B data, 43 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:33:44.087 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:33:43 smithi164 bash[26855]: audit 2023-06-03T19:33:43.342818+0000 mon.c (mon.2) 22 : audit [DBG] from='client.? 172.21.15.89:0/3173368842' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2023-06-03T19:33:44.101 INFO:tasks.cephadm.ceph_manager.ceph:need seq 180388626467 got 180388626471 for osd.6 2023-06-03T19:33:44.101 DEBUG:teuthology.parallel:result is None 2023-06-03T19:33:44.190 INFO:tasks.cephadm.ceph_manager.ceph:need seq 206158430233 got 206158430238 for osd.7 2023-06-03T19:33:44.191 DEBUG:teuthology.parallel:result is None 2023-06-03T19:33:44.711 INFO:tasks.cephadm.ceph_manager.ceph:need seq 107374182462 got 107374182467 for osd.3 2023-06-03T19:33:44.712 DEBUG:teuthology.parallel:result is None 2023-06-03T19:33:44.712 INFO:tasks.cephadm.ceph_manager.ceph:waiting for clean 2023-06-03T19:33:44.713 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph pg dump --format=json 2023-06-03T19:33:44.882 INFO:journalctl@ceph.alertmanager.a.smithi089.stdout:Jun 03 19:33:44 smithi089 systemd[1]: Stopping Ceph alertmanager.a for f1db3478-0243-11ee-9b1d-001a4aab830c... 2023-06-03T19:33:44.882 INFO:journalctl@ceph.alertmanager.a.smithi089.stdout:Jun 03 19:33:44 smithi089 bash[57865]: Error response from daemon: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-alertmanager.a 2023-06-03T19:33:44.882 INFO:journalctl@ceph.alertmanager.a.smithi089.stdout:Jun 03 19:33:44 smithi089 bash[52685]: level=info ts=2023-06-03T19:33:44.801Z caller=main.go:557 msg="Received SIGTERM, exiting gracefully..." 2023-06-03T19:33:46.031 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:33:45 smithi089 bash[26572]: cluster 2023-06-03T19:33:44.403538+0000 mgr.y (mgr.14445) 77 : cluster [DBG] pgmap v65: 9 pgs: 9 active+clean; 0 B data, 43 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:33:46.032 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:33:45 smithi089 bash[26572]: audit 2023-06-03T19:33:45.658667+0000 mon.a (mon.0) 632 : audit [INF] from='mgr.14445 172.21.15.89:0/573502486' entity='mgr.y' 2023-06-03T19:33:46.032 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:33:45 smithi089 bash[33173]: cluster 2023-06-03T19:33:44.403538+0000 mgr.y (mgr.14445) 77 : cluster [DBG] pgmap v65: 9 pgs: 9 active+clean; 0 B data, 43 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:33:46.032 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:33:45 smithi089 bash[33173]: audit 2023-06-03T19:33:45.658667+0000 mon.a (mon.0) 632 : audit [INF] from='mgr.14445 172.21.15.89:0/573502486' entity='mgr.y' 2023-06-03T19:33:46.033 INFO:journalctl@ceph.alertmanager.a.smithi089.stdout:Jun 03 19:33:45 smithi089 bash[57906]: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-alertmanager-a 2023-06-03T19:33:46.033 INFO:journalctl@ceph.alertmanager.a.smithi089.stdout:Jun 03 19:33:45 smithi089 systemd[1]: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c@alertmanager.a.service: Succeeded. 2023-06-03T19:33:46.033 INFO:journalctl@ceph.alertmanager.a.smithi089.stdout:Jun 03 19:33:45 smithi089 systemd[1]: Stopped Ceph alertmanager.a for f1db3478-0243-11ee-9b1d-001a4aab830c. 2023-06-03T19:33:46.033 INFO:journalctl@ceph.alertmanager.a.smithi089.stdout:Jun 03 19:33:45 smithi089 systemd[1]: Started Ceph alertmanager.a for f1db3478-0243-11ee-9b1d-001a4aab830c. 2023-06-03T19:33:46.086 INFO:journalctl@ceph.mgr.x.smithi164.stdout:Jun 03 19:33:45 smithi164 bash[27934]: ::ffff:172.21.15.164 - - [03/Jun/2023:19:33:45] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2023-06-03T19:33:46.086 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:33:45 smithi164 bash[26855]: cluster 2023-06-03T19:33:44.403538+0000 mgr.y (mgr.14445) 77 : cluster [DBG] pgmap v65: 9 pgs: 9 active+clean; 0 B data, 43 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:33:46.087 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:33:45 smithi164 bash[26855]: audit 2023-06-03T19:33:45.658667+0000 mon.a (mon.0) 632 : audit [INF] from='mgr.14445 172.21.15.89:0/573502486' entity='mgr.y' 2023-06-03T19:33:47.020 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:33:46 smithi164 bash[26855]: cephadm 2023-06-03T19:33:45.660209+0000 mgr.y (mgr.14445) 78 : cephadm [INF] Reconfiguring prometheus.a (dependencies changed)... 2023-06-03T19:33:47.020 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:33:46 smithi164 bash[26855]: cephadm 2023-06-03T19:33:45.665583+0000 mgr.y (mgr.14445) 79 : cephadm [INF] Reconfiguring daemon prometheus.a on smithi164 2023-06-03T19:33:47.042 INFO:journalctl@ceph.alertmanager.a.smithi089.stdout:Jun 03 19:33:46 smithi089 bash[58026]: level=info ts=2023-06-03T19:33:46.825Z caller=main.go:225 msg="Starting Alertmanager" version="(version=0.23.0, branch=HEAD, revision=61046b17771a57cfd4c4a51be370ab930a4d7d54)" 2023-06-03T19:33:47.043 INFO:journalctl@ceph.alertmanager.a.smithi089.stdout:Jun 03 19:33:46 smithi089 bash[58026]: level=info ts=2023-06-03T19:33:46.825Z caller=main.go:226 build_context="(go=go1.16.7, user=root@e21a959be8d2, date=20210825-10:48:55)" 2023-06-03T19:33:47.043 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:33:46 smithi089 bash[26572]: cephadm 2023-06-03T19:33:45.660209+0000 mgr.y (mgr.14445) 78 : cephadm [INF] Reconfiguring prometheus.a (dependencies changed)... 2023-06-03T19:33:47.043 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:33:46 smithi089 bash[26572]: cephadm 2023-06-03T19:33:45.665583+0000 mgr.y (mgr.14445) 79 : cephadm [INF] Reconfiguring daemon prometheus.a on smithi164 2023-06-03T19:33:47.043 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:33:46 smithi089 bash[33173]: cephadm 2023-06-03T19:33:45.660209+0000 mgr.y (mgr.14445) 78 : cephadm [INF] Reconfiguring prometheus.a (dependencies changed)... 2023-06-03T19:33:47.044 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:33:46 smithi089 bash[33173]: cephadm 2023-06-03T19:33:45.665583+0000 mgr.y (mgr.14445) 79 : cephadm [INF] Reconfiguring daemon prometheus.a on smithi164 2023-06-03T19:33:47.531 INFO:journalctl@ceph.alertmanager.a.smithi089.stdout:Jun 03 19:33:47 smithi089 bash[58026]: level=info ts=2023-06-03T19:33:46.828Z caller=cluster.go:184 component=cluster msg="setting advertise address explicitly" addr=172.21.15.89 port=9094 2023-06-03T19:33:47.532 INFO:journalctl@ceph.alertmanager.a.smithi089.stdout:Jun 03 19:33:47 smithi089 bash[58026]: level=info ts=2023-06-03T19:33:46.829Z caller=cluster.go:671 component=cluster msg="Waiting for gossip to settle..." interval=2s 2023-06-03T19:33:47.532 INFO:journalctl@ceph.alertmanager.a.smithi089.stdout:Jun 03 19:33:47 smithi089 bash[58026]: level=info ts=2023-06-03T19:33:46.866Z caller=coordinator.go:113 component=configuration msg="Loading configuration file" file=/etc/alertmanager/alertmanager.yml 2023-06-03T19:33:47.532 INFO:journalctl@ceph.alertmanager.a.smithi089.stdout:Jun 03 19:33:47 smithi089 bash[58026]: level=info ts=2023-06-03T19:33:46.866Z caller=coordinator.go:126 component=configuration msg="Completed loading of configuration file" file=/etc/alertmanager/alertmanager.yml 2023-06-03T19:33:47.532 INFO:journalctl@ceph.alertmanager.a.smithi089.stdout:Jun 03 19:33:47 smithi089 bash[58026]: level=info ts=2023-06-03T19:33:46.871Z caller=main.go:518 msg=Listening address=:9093 2023-06-03T19:33:47.532 INFO:journalctl@ceph.alertmanager.a.smithi089.stdout:Jun 03 19:33:47 smithi089 bash[58026]: level=info ts=2023-06-03T19:33:46.871Z caller=tls_config.go:191 msg="TLS is disabled." http2=false 2023-06-03T19:33:48.031 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:33:47 smithi089 bash[26572]: cluster 2023-06-03T19:33:46.404138+0000 mgr.y (mgr.14445) 80 : cluster [DBG] pgmap v66: 9 pgs: 9 active+clean; 0 B data, 43 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:33:48.032 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 19:33:47 smithi089 bash[26845]: ::ffff:172.21.15.164 - - [03/Jun/2023:19:33:47] "GET /metrics HTTP/1.1" 200 151995 "" "Prometheus/2.33.4" 2023-06-03T19:33:48.032 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:33:47 smithi089 bash[33173]: cluster 2023-06-03T19:33:46.404138+0000 mgr.y (mgr.14445) 80 : cluster [DBG] pgmap v66: 9 pgs: 9 active+clean; 0 B data, 43 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:33:48.086 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:33:47 smithi164 bash[26855]: cluster 2023-06-03T19:33:46.404138+0000 mgr.y (mgr.14445) 80 : cluster [DBG] pgmap v66: 9 pgs: 9 active+clean; 0 B data, 43 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:33:49.281 INFO:journalctl@ceph.alertmanager.a.smithi089.stdout:Jun 03 19:33:48 smithi089 bash[58026]: level=info ts=2023-06-03T19:33:48.830Z caller=cluster.go:696 component=cluster msg="gossip not settled" polls=0 before=0 now=1 elapsed=2.000506468s 2023-06-03T19:33:49.506 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/mon.a/config 2023-06-03T19:33:49.940 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:33:49 smithi164 bash[26855]: cluster 2023-06-03T19:33:48.404745+0000 mgr.y (mgr.14445) 81 : cluster [DBG] pgmap v67: 9 pgs: 9 active+clean; 0 B data, 43 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:33:50.031 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:33:49 smithi089 bash[33173]: cluster 2023-06-03T19:33:48.404745+0000 mgr.y (mgr.14445) 81 : cluster [DBG] pgmap v67: 9 pgs: 9 active+clean; 0 B data, 43 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:33:50.032 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:33:49 smithi089 bash[26572]: cluster 2023-06-03T19:33:48.404745+0000 mgr.y (mgr.14445) 81 : cluster [DBG] pgmap v67: 9 pgs: 9 active+clean; 0 B data, 43 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:33:50.197 INFO:journalctl@ceph.prometheus.a.smithi164.stdout:Jun 03 19:33:49 smithi164 systemd[1]: Stopping Ceph prometheus.a for f1db3478-0243-11ee-9b1d-001a4aab830c... 2023-06-03T19:33:50.197 INFO:journalctl@ceph.prometheus.a.smithi164.stdout:Jun 03 19:33:49 smithi164 bash[48391]: Error response from daemon: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-prometheus.a 2023-06-03T19:33:50.198 INFO:journalctl@ceph.prometheus.a.smithi164.stdout:Jun 03 19:33:50 smithi164 bash[46758]: ts=2023-06-03T19:33:50.122Z caller=main.go:775 level=warn msg="Received SIGTERM, exiting gracefully..." 2023-06-03T19:33:50.198 INFO:journalctl@ceph.prometheus.a.smithi164.stdout:Jun 03 19:33:50 smithi164 bash[46758]: ts=2023-06-03T19:33:50.122Z caller=main.go:798 level=info msg="Stopping scrape discovery manager..." 2023-06-03T19:33:50.198 INFO:journalctl@ceph.prometheus.a.smithi164.stdout:Jun 03 19:33:50 smithi164 bash[46758]: ts=2023-06-03T19:33:50.122Z caller=main.go:812 level=info msg="Stopping notify discovery manager..." 2023-06-03T19:33:50.198 INFO:journalctl@ceph.prometheus.a.smithi164.stdout:Jun 03 19:33:50 smithi164 bash[46758]: ts=2023-06-03T19:33:50.122Z caller=main.go:834 level=info msg="Stopping scrape manager..." 2023-06-03T19:33:50.198 INFO:journalctl@ceph.prometheus.a.smithi164.stdout:Jun 03 19:33:50 smithi164 bash[46758]: ts=2023-06-03T19:33:50.122Z caller=main.go:794 level=info msg="Scrape discovery manager stopped" 2023-06-03T19:33:50.198 INFO:journalctl@ceph.prometheus.a.smithi164.stdout:Jun 03 19:33:50 smithi164 bash[46758]: ts=2023-06-03T19:33:50.122Z caller=main.go:808 level=info msg="Notify discovery manager stopped" 2023-06-03T19:33:50.198 INFO:journalctl@ceph.prometheus.a.smithi164.stdout:Jun 03 19:33:50 smithi164 bash[46758]: ts=2023-06-03T19:33:50.122Z caller=manager.go:945 level=info component="rule manager" msg="Stopping rule manager..." 2023-06-03T19:33:50.198 INFO:journalctl@ceph.prometheus.a.smithi164.stdout:Jun 03 19:33:50 smithi164 bash[46758]: ts=2023-06-03T19:33:50.122Z caller=main.go:828 level=info msg="Scrape manager stopped" 2023-06-03T19:33:50.199 INFO:journalctl@ceph.prometheus.a.smithi164.stdout:Jun 03 19:33:50 smithi164 bash[46758]: ts=2023-06-03T19:33:50.122Z caller=manager.go:955 level=info component="rule manager" msg="Rule manager stopped" 2023-06-03T19:33:50.199 INFO:journalctl@ceph.prometheus.a.smithi164.stdout:Jun 03 19:33:50 smithi164 bash[46758]: ts=2023-06-03T19:33:50.123Z caller=notifier.go:600 level=info component=notifier msg="Stopping notification manager..." 2023-06-03T19:33:50.199 INFO:journalctl@ceph.prometheus.a.smithi164.stdout:Jun 03 19:33:50 smithi164 bash[46758]: ts=2023-06-03T19:33:50.123Z caller=main.go:1054 level=info msg="Notifier manager stopped" 2023-06-03T19:33:50.199 INFO:journalctl@ceph.prometheus.a.smithi164.stdout:Jun 03 19:33:50 smithi164 bash[46758]: ts=2023-06-03T19:33:50.123Z caller=main.go:1066 level=info msg="See you next time!" 2023-06-03T19:33:51.314 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 19:33:51 smithi089 bash[26845]: [03/Jun/2023:19:33:51] ENGINE Bus STOPPING 2023-06-03T19:33:51.314 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 19:33:51 smithi089 bash[26845]: [03/Jun/2023:19:33:51] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2023-06-03T19:33:51.314 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 19:33:51 smithi089 bash[26845]: [03/Jun/2023:19:33:51] ENGINE Bus STOPPED 2023-06-03T19:33:51.314 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 19:33:51 smithi089 bash[26845]: [03/Jun/2023:19:33:51] ENGINE Bus STARTING 2023-06-03T19:33:51.315 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 19:33:51 smithi089 bash[26845]: [03/Jun/2023:19:33:51] ENGINE Serving on http://:::9283 2023-06-03T19:33:51.315 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 19:33:51 smithi089 bash[26845]: [03/Jun/2023:19:33:51] ENGINE Bus STARTED 2023-06-03T19:33:51.315 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 19:33:51 smithi089 bash[26845]: [03/Jun/2023:19:33:51] ENGINE Bus STOPPING 2023-06-03T19:33:51.315 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 19:33:51 smithi089 bash[26845]: [03/Jun/2023:19:33:51] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2023-06-03T19:33:51.315 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 19:33:51 smithi089 bash[26845]: [03/Jun/2023:19:33:51] ENGINE Bus STOPPED 2023-06-03T19:33:51.315 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 19:33:51 smithi089 bash[26845]: [03/Jun/2023:19:33:51] ENGINE Bus STARTING 2023-06-03T19:33:51.315 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 19:33:51 smithi089 bash[26845]: [03/Jun/2023:19:33:51] ENGINE Serving on http://:::9283 2023-06-03T19:33:51.316 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 19:33:51 smithi089 bash[26845]: [03/Jun/2023:19:33:51] ENGINE Bus STARTED 2023-06-03T19:33:51.336 INFO:journalctl@ceph.prometheus.a.smithi164.stdout:Jun 03 19:33:50 smithi164 bash[48433]: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-prometheus-a 2023-06-03T19:33:51.336 INFO:journalctl@ceph.prometheus.a.smithi164.stdout:Jun 03 19:33:50 smithi164 systemd[1]: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c@prometheus.a.service: Succeeded. 2023-06-03T19:33:51.337 INFO:journalctl@ceph.prometheus.a.smithi164.stdout:Jun 03 19:33:50 smithi164 systemd[1]: Stopped Ceph prometheus.a for f1db3478-0243-11ee-9b1d-001a4aab830c. 2023-06-03T19:33:51.337 INFO:journalctl@ceph.prometheus.a.smithi164.stdout:Jun 03 19:33:50 smithi164 systemd[1]: Started Ceph prometheus.a for f1db3478-0243-11ee-9b1d-001a4aab830c. 2023-06-03T19:33:51.459 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 19:33:51 smithi089 bash[26845]: [03/Jun/2023:19:33:51] ENGINE Bus STOPPING 2023-06-03T19:33:51.460 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 19:33:51 smithi089 bash[26845]: [03/Jun/2023:19:33:51] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2023-06-03T19:33:51.460 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 19:33:51 smithi089 bash[26845]: [03/Jun/2023:19:33:51] ENGINE Bus STOPPED 2023-06-03T19:33:51.460 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 19:33:51 smithi089 bash[26845]: [03/Jun/2023:19:33:51] ENGINE Bus STARTING 2023-06-03T19:33:51.460 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 19:33:51 smithi089 bash[26845]: [03/Jun/2023:19:33:51] ENGINE Serving on http://:::9283 2023-06-03T19:33:51.460 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 19:33:51 smithi089 bash[26845]: [03/Jun/2023:19:33:51] ENGINE Bus STARTED 2023-06-03T19:33:51.694 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:33:51 smithi089 bash[26572]: cluster 2023-06-03T19:33:50.405273+0000 mgr.y (mgr.14445) 82 : cluster [DBG] pgmap v68: 9 pgs: 9 active+clean; 0 B data, 43 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:33:51.695 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:33:51 smithi089 bash[26572]: audit 2023-06-03T19:33:51.019676+0000 mon.a (mon.0) 633 : audit [INF] from='mgr.14445 172.21.15.89:0/573502486' entity='mgr.y' 2023-06-03T19:33:51.695 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:33:51 smithi089 bash[26572]: audit 2023-06-03T19:33:51.027371+0000 mon.a (mon.0) 634 : audit [DBG] from='mgr.14445 172.21.15.89:0/573502486' entity='mgr.y' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2023-06-03T19:33:51.695 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:33:51 smithi089 bash[26572]: audit 2023-06-03T19:33:51.029314+0000 mon.a (mon.0) 635 : audit [INF] from='mgr.14445 172.21.15.89:0/573502486' entity='mgr.y' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi089.front.sepia.ceph.com:9093"}]: dispatch 2023-06-03T19:33:51.695 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:33:51 smithi089 bash[26572]: audit 2023-06-03T19:33:51.036914+0000 mon.a (mon.0) 636 : audit [INF] from='mgr.14445 172.21.15.89:0/573502486' entity='mgr.y' 2023-06-03T19:33:51.695 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:33:51 smithi089 bash[26572]: audit 2023-06-03T19:33:51.045413+0000 mon.a (mon.0) 637 : audit [DBG] from='mgr.14445 172.21.15.89:0/573502486' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-06-03T19:33:51.695 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:33:51 smithi089 bash[26572]: audit 2023-06-03T19:33:51.172153+0000 mon.a (mon.0) 638 : audit [INF] from='mgr.14445 172.21.15.89:0/573502486' entity='mgr.y' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi164.front.sepia.ceph.com:9095"}]: dispatch 2023-06-03T19:33:52.031 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:33:51 smithi089 bash[26572]: audit 2023-06-03T19:33:51.180773+0000 mon.a (mon.0) 639 : audit [INF] from='mgr.14445 172.21.15.89:0/573502486' entity='mgr.y' 2023-06-03T19:33:52.032 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:33:51 smithi089 bash[26572]: audit 2023-06-03T19:33:51.186591+0000 mon.a (mon.0) 640 : audit [DBG] from='mgr.14445 172.21.15.89:0/573502486' entity='mgr.y' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2023-06-03T19:33:52.032 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:33:51 smithi089 bash[26572]: audit 2023-06-03T19:33:51.310823+0000 mon.a (mon.0) 641 : audit [INF] from='mgr.14445 172.21.15.89:0/573502486' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi164.front.sepia.ceph.com:3000"}]: dispatch 2023-06-03T19:33:52.032 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:33:51 smithi089 bash[26572]: audit 2023-06-03T19:33:51.319887+0000 mon.a (mon.0) 642 : audit [INF] from='mgr.14445 172.21.15.89:0/573502486' entity='mgr.y' 2023-06-03T19:33:52.032 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:33:51 smithi089 bash[26572]: audit 2023-06-03T19:33:51.323095+0000 mon.a (mon.0) 643 : audit [DBG] from='mgr.14445 172.21.15.89:0/573502486' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-03T19:33:52.032 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:33:51 smithi089 bash[26572]: audit 2023-06-03T19:33:51.578208+0000 mon.a (mon.0) 644 : audit [INF] from='mgr.14445 172.21.15.89:0/573502486' entity='mgr.y' 2023-06-03T19:33:52.033 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:33:51 smithi089 bash[33173]: cluster 2023-06-03T19:33:50.405273+0000 mgr.y (mgr.14445) 82 : cluster [DBG] pgmap v68: 9 pgs: 9 active+clean; 0 B data, 43 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:33:52.033 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:33:51 smithi089 bash[33173]: audit 2023-06-03T19:33:51.019676+0000 mon.a (mon.0) 633 : audit [INF] from='mgr.14445 172.21.15.89:0/573502486' entity='mgr.y' 2023-06-03T19:33:52.033 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:33:51 smithi089 bash[33173]: audit 2023-06-03T19:33:51.027371+0000 mon.a (mon.0) 634 : audit [DBG] from='mgr.14445 172.21.15.89:0/573502486' entity='mgr.y' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2023-06-03T19:33:52.033 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:33:51 smithi089 bash[33173]: audit 2023-06-03T19:33:51.029314+0000 mon.a (mon.0) 635 : audit [INF] from='mgr.14445 172.21.15.89:0/573502486' entity='mgr.y' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi089.front.sepia.ceph.com:9093"}]: dispatch 2023-06-03T19:33:52.033 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:33:51 smithi089 bash[33173]: audit 2023-06-03T19:33:51.036914+0000 mon.a (mon.0) 636 : audit [INF] from='mgr.14445 172.21.15.89:0/573502486' entity='mgr.y' 2023-06-03T19:33:52.033 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:33:51 smithi089 bash[33173]: audit 2023-06-03T19:33:51.045413+0000 mon.a (mon.0) 637 : audit [DBG] from='mgr.14445 172.21.15.89:0/573502486' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-06-03T19:33:52.033 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:33:51 smithi089 bash[33173]: audit 2023-06-03T19:33:51.172153+0000 mon.a (mon.0) 638 : audit [INF] from='mgr.14445 172.21.15.89:0/573502486' entity='mgr.y' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi164.front.sepia.ceph.com:9095"}]: dispatch 2023-06-03T19:33:52.034 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:33:51 smithi089 bash[33173]: audit 2023-06-03T19:33:51.180773+0000 mon.a (mon.0) 639 : audit [INF] from='mgr.14445 172.21.15.89:0/573502486' entity='mgr.y' 2023-06-03T19:33:52.034 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:33:51 smithi089 bash[33173]: audit 2023-06-03T19:33:51.186591+0000 mon.a (mon.0) 640 : audit [DBG] from='mgr.14445 172.21.15.89:0/573502486' entity='mgr.y' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2023-06-03T19:33:52.034 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:33:51 smithi089 bash[33173]: audit 2023-06-03T19:33:51.310823+0000 mon.a (mon.0) 641 : audit [INF] from='mgr.14445 172.21.15.89:0/573502486' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi164.front.sepia.ceph.com:3000"}]: dispatch 2023-06-03T19:33:52.034 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:33:51 smithi089 bash[33173]: audit 2023-06-03T19:33:51.319887+0000 mon.a (mon.0) 642 : audit [INF] from='mgr.14445 172.21.15.89:0/573502486' entity='mgr.y' 2023-06-03T19:33:52.034 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:33:51 smithi089 bash[33173]: audit 2023-06-03T19:33:51.323095+0000 mon.a (mon.0) 643 : audit [DBG] from='mgr.14445 172.21.15.89:0/573502486' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-03T19:33:52.034 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:33:51 smithi089 bash[33173]: audit 2023-06-03T19:33:51.578208+0000 mon.a (mon.0) 644 : audit [INF] from='mgr.14445 172.21.15.89:0/573502486' entity='mgr.y' 2023-06-03T19:33:52.086 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:33:51 smithi164 bash[26855]: cluster 2023-06-03T19:33:50.405273+0000 mgr.y (mgr.14445) 82 : cluster [DBG] pgmap v68: 9 pgs: 9 active+clean; 0 B data, 43 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:33:52.087 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:33:51 smithi164 bash[26855]: audit 2023-06-03T19:33:51.019676+0000 mon.a (mon.0) 633 : audit [INF] from='mgr.14445 172.21.15.89:0/573502486' entity='mgr.y' 2023-06-03T19:33:52.087 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:33:51 smithi164 bash[26855]: audit 2023-06-03T19:33:51.027371+0000 mon.a (mon.0) 634 : audit [DBG] from='mgr.14445 172.21.15.89:0/573502486' entity='mgr.y' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2023-06-03T19:33:52.087 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:33:51 smithi164 bash[26855]: audit 2023-06-03T19:33:51.029314+0000 mon.a (mon.0) 635 : audit [INF] from='mgr.14445 172.21.15.89:0/573502486' entity='mgr.y' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi089.front.sepia.ceph.com:9093"}]: dispatch 2023-06-03T19:33:52.087 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:33:51 smithi164 bash[26855]: audit 2023-06-03T19:33:51.036914+0000 mon.a (mon.0) 636 : audit [INF] from='mgr.14445 172.21.15.89:0/573502486' entity='mgr.y' 2023-06-03T19:33:52.087 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:33:51 smithi164 bash[26855]: audit 2023-06-03T19:33:51.045413+0000 mon.a (mon.0) 637 : audit [DBG] from='mgr.14445 172.21.15.89:0/573502486' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-06-03T19:33:52.087 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:33:51 smithi164 bash[26855]: audit 2023-06-03T19:33:51.172153+0000 mon.a (mon.0) 638 : audit [INF] from='mgr.14445 172.21.15.89:0/573502486' entity='mgr.y' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi164.front.sepia.ceph.com:9095"}]: dispatch 2023-06-03T19:33:52.087 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:33:51 smithi164 bash[26855]: audit 2023-06-03T19:33:51.180773+0000 mon.a (mon.0) 639 : audit [INF] from='mgr.14445 172.21.15.89:0/573502486' entity='mgr.y' 2023-06-03T19:33:52.088 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:33:51 smithi164 bash[26855]: audit 2023-06-03T19:33:51.186591+0000 mon.a (mon.0) 640 : audit [DBG] from='mgr.14445 172.21.15.89:0/573502486' entity='mgr.y' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2023-06-03T19:33:52.088 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:33:51 smithi164 bash[26855]: audit 2023-06-03T19:33:51.310823+0000 mon.a (mon.0) 641 : audit [INF] from='mgr.14445 172.21.15.89:0/573502486' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi164.front.sepia.ceph.com:3000"}]: dispatch 2023-06-03T19:33:52.088 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:33:51 smithi164 bash[26855]: audit 2023-06-03T19:33:51.319887+0000 mon.a (mon.0) 642 : audit [INF] from='mgr.14445 172.21.15.89:0/573502486' entity='mgr.y' 2023-06-03T19:33:52.088 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:33:51 smithi164 bash[26855]: audit 2023-06-03T19:33:51.323095+0000 mon.a (mon.0) 643 : audit [DBG] from='mgr.14445 172.21.15.89:0/573502486' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-03T19:33:52.088 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:33:51 smithi164 bash[26855]: audit 2023-06-03T19:33:51.578208+0000 mon.a (mon.0) 644 : audit [INF] from='mgr.14445 172.21.15.89:0/573502486' entity='mgr.y' 2023-06-03T19:33:52.398 INFO:journalctl@ceph.prometheus.a.smithi164.stdout:Jun 03 19:33:52 smithi164 bash[48531]: ts=2023-06-03T19:33:52.237Z caller=main.go:512 level=info msg="Starting Prometheus" version="(version=2.33.4, branch=HEAD, revision=83032011a5d3e6102624fe58241a374a7201fee8)" 2023-06-03T19:33:52.398 INFO:journalctl@ceph.prometheus.a.smithi164.stdout:Jun 03 19:33:52 smithi164 bash[48531]: ts=2023-06-03T19:33:52.237Z caller=main.go:517 level=info build_context="(go=go1.17.7, user=root@d13bf69e7be8, date=20220222-16:51:28)" 2023-06-03T19:33:52.398 INFO:journalctl@ceph.prometheus.a.smithi164.stdout:Jun 03 19:33:52 smithi164 bash[48531]: ts=2023-06-03T19:33:52.237Z caller=main.go:518 level=info host_details="(Linux 5.15.0-73-generic #80~20.04.1-Ubuntu SMP Wed May 17 14:58:14 UTC 2023 x86_64 smithi164 (none))" 2023-06-03T19:33:52.398 INFO:journalctl@ceph.prometheus.a.smithi164.stdout:Jun 03 19:33:52 smithi164 bash[48531]: ts=2023-06-03T19:33:52.237Z caller=main.go:519 level=info fd_limits="(soft=1048576, hard=1048576)" 2023-06-03T19:33:52.398 INFO:journalctl@ceph.prometheus.a.smithi164.stdout:Jun 03 19:33:52 smithi164 bash[48531]: ts=2023-06-03T19:33:52.237Z caller=main.go:520 level=info vm_limits="(soft=unlimited, hard=unlimited)" 2023-06-03T19:33:52.398 INFO:journalctl@ceph.prometheus.a.smithi164.stdout:Jun 03 19:33:52 smithi164 bash[48531]: ts=2023-06-03T19:33:52.240Z caller=web.go:570 level=info component=web msg="Start listening for connections" address=:9095 2023-06-03T19:33:52.399 INFO:journalctl@ceph.prometheus.a.smithi164.stdout:Jun 03 19:33:52 smithi164 bash[48531]: ts=2023-06-03T19:33:52.241Z caller=main.go:923 level=info msg="Starting TSDB ..." 2023-06-03T19:33:52.399 INFO:journalctl@ceph.prometheus.a.smithi164.stdout:Jun 03 19:33:52 smithi164 bash[48531]: ts=2023-06-03T19:33:52.243Z caller=tls_config.go:195 level=info component=web msg="TLS is disabled." http2=false 2023-06-03T19:33:52.399 INFO:journalctl@ceph.prometheus.a.smithi164.stdout:Jun 03 19:33:52 smithi164 bash[48531]: ts=2023-06-03T19:33:52.245Z caller=head.go:493 level=info component=tsdb msg="Replaying on-disk memory mappable chunks if any" 2023-06-03T19:33:52.399 INFO:journalctl@ceph.prometheus.a.smithi164.stdout:Jun 03 19:33:52 smithi164 bash[48531]: ts=2023-06-03T19:33:52.245Z caller=head.go:527 level=info component=tsdb msg="On-disk memory mappable chunks replay completed" duration=2.113µs 2023-06-03T19:33:52.399 INFO:journalctl@ceph.prometheus.a.smithi164.stdout:Jun 03 19:33:52 smithi164 bash[48531]: ts=2023-06-03T19:33:52.245Z caller=head.go:533 level=info component=tsdb msg="Replaying WAL, this may take a while" 2023-06-03T19:33:52.399 INFO:journalctl@ceph.prometheus.a.smithi164.stdout:Jun 03 19:33:52 smithi164 bash[48531]: ts=2023-06-03T19:33:52.388Z caller=head.go:604 level=info component=tsdb msg="WAL segment loaded" segment=0 maxSegment=1 2023-06-03T19:33:52.704 INFO:journalctl@ceph.prometheus.a.smithi164.stdout:Jun 03 19:33:52 smithi164 bash[48531]: ts=2023-06-03T19:33:52.388Z caller=head.go:604 level=info component=tsdb msg="WAL segment loaded" segment=1 maxSegment=1 2023-06-03T19:33:52.704 INFO:journalctl@ceph.prometheus.a.smithi164.stdout:Jun 03 19:33:52 smithi164 bash[48531]: ts=2023-06-03T19:33:52.388Z caller=head.go:610 level=info component=tsdb msg="WAL replay completed" checkpoint_replay_duration=41.507µs wal_replay_duration=143.309214ms total_replay_duration=143.364999ms 2023-06-03T19:33:52.704 INFO:journalctl@ceph.prometheus.a.smithi164.stdout:Jun 03 19:33:52 smithi164 bash[48531]: ts=2023-06-03T19:33:52.392Z caller=main.go:944 level=info fs_type=XFS_SUPER_MAGIC 2023-06-03T19:33:52.705 INFO:journalctl@ceph.prometheus.a.smithi164.stdout:Jun 03 19:33:52 smithi164 bash[48531]: ts=2023-06-03T19:33:52.392Z caller=main.go:947 level=info msg="TSDB started" 2023-06-03T19:33:52.705 INFO:journalctl@ceph.prometheus.a.smithi164.stdout:Jun 03 19:33:52 smithi164 bash[48531]: ts=2023-06-03T19:33:52.392Z caller=main.go:1128 level=info msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml 2023-06-03T19:33:52.705 INFO:journalctl@ceph.prometheus.a.smithi164.stdout:Jun 03 19:33:52 smithi164 bash[48531]: ts=2023-06-03T19:33:52.418Z caller=main.go:1165 level=info msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=26.663274ms db_storage=1.405µs remote_storage=1.977µs web_handler=550ns query_engine=1.324µs scrape=532.194µs scrape_sd=48.891µs notify=56.088µs notify_sd=25.706µs rules=25.453536ms 2023-06-03T19:33:52.705 INFO:journalctl@ceph.prometheus.a.smithi164.stdout:Jun 03 19:33:52 smithi164 bash[48531]: ts=2023-06-03T19:33:52.419Z caller=main.go:896 level=info msg="Server is ready to receive web requests." 2023-06-03T19:33:53.031 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:33:52 smithi089 bash[33173]: audit 2023-06-03T19:33:51.027886+0000 mgr.y (mgr.14445) 83 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2023-06-03T19:33:53.032 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:33:52 smithi089 bash[33173]: audit 2023-06-03T19:33:51.029712+0000 mgr.y (mgr.14445) 84 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi089.front.sepia.ceph.com:9093"}]: dispatch 2023-06-03T19:33:53.032 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:33:52 smithi089 bash[33173]: audit 2023-06-03T19:33:51.045985+0000 mgr.y (mgr.14445) 85 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-06-03T19:33:53.032 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:33:52 smithi089 bash[33173]: audit 2023-06-03T19:33:51.173246+0000 mgr.y (mgr.14445) 86 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi164.front.sepia.ceph.com:9095"}]: dispatch 2023-06-03T19:33:53.032 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:33:52 smithi089 bash[33173]: audit 2023-06-03T19:33:51.187088+0000 mgr.y (mgr.14445) 87 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2023-06-03T19:33:53.032 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:33:52 smithi089 bash[33173]: audit 2023-06-03T19:33:51.311316+0000 mgr.y (mgr.14445) 88 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi164.front.sepia.ceph.com:3000"}]: dispatch 2023-06-03T19:33:53.033 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:33:52 smithi089 bash[26572]: audit 2023-06-03T19:33:51.027886+0000 mgr.y (mgr.14445) 83 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2023-06-03T19:33:53.033 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:33:52 smithi089 bash[26572]: audit 2023-06-03T19:33:51.029712+0000 mgr.y (mgr.14445) 84 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi089.front.sepia.ceph.com:9093"}]: dispatch 2023-06-03T19:33:53.033 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:33:52 smithi089 bash[26572]: audit 2023-06-03T19:33:51.045985+0000 mgr.y (mgr.14445) 85 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-06-03T19:33:53.033 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:33:52 smithi089 bash[26572]: audit 2023-06-03T19:33:51.173246+0000 mgr.y (mgr.14445) 86 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi164.front.sepia.ceph.com:9095"}]: dispatch 2023-06-03T19:33:53.033 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:33:52 smithi089 bash[26572]: audit 2023-06-03T19:33:51.187088+0000 mgr.y (mgr.14445) 87 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2023-06-03T19:33:53.033 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:33:52 smithi089 bash[26572]: audit 2023-06-03T19:33:51.311316+0000 mgr.y (mgr.14445) 88 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi164.front.sepia.ceph.com:3000"}]: dispatch 2023-06-03T19:33:53.086 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:33:52 smithi164 bash[26855]: audit 2023-06-03T19:33:51.027886+0000 mgr.y (mgr.14445) 83 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2023-06-03T19:33:53.087 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:33:52 smithi164 bash[26855]: audit 2023-06-03T19:33:51.029712+0000 mgr.y (mgr.14445) 84 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi089.front.sepia.ceph.com:9093"}]: dispatch 2023-06-03T19:33:53.087 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:33:52 smithi164 bash[26855]: audit 2023-06-03T19:33:51.045985+0000 mgr.y (mgr.14445) 85 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-06-03T19:33:53.087 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:33:52 smithi164 bash[26855]: audit 2023-06-03T19:33:51.173246+0000 mgr.y (mgr.14445) 86 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi164.front.sepia.ceph.com:9095"}]: dispatch 2023-06-03T19:33:53.087 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:33:52 smithi164 bash[26855]: audit 2023-06-03T19:33:51.187088+0000 mgr.y (mgr.14445) 87 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2023-06-03T19:33:53.087 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:33:52 smithi164 bash[26855]: audit 2023-06-03T19:33:51.311316+0000 mgr.y (mgr.14445) 88 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi164.front.sepia.ceph.com:3000"}]: dispatch 2023-06-03T19:33:53.128 INFO:teuthology.orchestra.run.smithi089.stdout: 2023-06-03T19:33:53.130 INFO:teuthology.orchestra.run.smithi089.stderr:dumped all 2023-06-03T19:33:53.899 INFO:teuthology.orchestra.run.smithi089.stdout:{"pg_ready":true,"pg_map":{"version":69,"stamp":"2023-06-03T19:33:52.405491+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":27,"acting":27,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":27,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":44208,"kb_used_data":4208,"kb_used_omap":0,"kb_used_meta":39936,"kb_avail":749917008,"statfs":{"total":767960285184,"available":767915016192,"internally_reserved":0,"allocated":4308992,"data_stored":1762828,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":40894464},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"0.000000"},"pg_stats":[{"pgid":"2.7","version":"0'0","reported_seq":11,"reported_epoch":54,"state":"active+clean","last_fresh":"2023-06-03T19:32:23.431446+0000","last_change":"2023-06-03T19:32:21.706362+0000","last_active":"2023-06-03T19:32:23.431446+0000","last_peered":"2023-06-03T19:32:23.431446+0000","last_clean":"2023-06-03T19:32:23.431446+0000","last_became_active":"2023-06-03T19:32:21.705855+0000","last_became_peered":"2023-06-03T19:32:21.705855+0000","last_unstale":"2023-06-03T19:32:23.431446+0000","last_undegraded":"2023-06-03T19:32:23.431446+0000","last_fullsized":"2023-06-03T19:32:23.431446+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:32:20.692043+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:32:20.692043+0000","last_clean_scrub_stamp":"2023-06-03T19:32:20.692043+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":11,"reported_epoch":54,"state":"active+clean","last_fresh":"2023-06-03T19:32:22.931122+0000","last_change":"2023-06-03T19:32:21.989748+0000","last_active":"2023-06-03T19:32:22.931122+0000","last_peered":"2023-06-03T19:32:22.931122+0000","last_clean":"2023-06-03T19:32:22.931122+0000","last_became_active":"2023-06-03T19:32:21.989310+0000","last_became_peered":"2023-06-03T19:32:21.989310+0000","last_unstale":"2023-06-03T19:32:22.931122+0000","last_undegraded":"2023-06-03T19:32:22.931122+0000","last_fullsized":"2023-06-03T19:32:22.931122+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:32:20.692043+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:32:20.692043+0000","last_clean_scrub_stamp":"2023-06-03T19:32:20.692043+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":11,"reported_epoch":54,"state":"active+clean","last_fresh":"2023-06-03T19:32:23.431073+0000","last_change":"2023-06-03T19:32:22.124191+0000","last_active":"2023-06-03T19:32:23.431073+0000","last_peered":"2023-06-03T19:32:23.431073+0000","last_clean":"2023-06-03T19:32:23.431073+0000","last_became_active":"2023-06-03T19:32:22.122933+0000","last_became_peered":"2023-06-03T19:32:22.122933+0000","last_unstale":"2023-06-03T19:32:23.431073+0000","last_undegraded":"2023-06-03T19:32:23.431073+0000","last_fullsized":"2023-06-03T19:32:23.431073+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:32:20.692043+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:32:20.692043+0000","last_clean_scrub_stamp":"2023-06-03T19:32:20.692043+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":11,"reported_epoch":54,"state":"active+clean","last_fresh":"2023-06-03T19:32:23.430935+0000","last_change":"2023-06-03T19:32:22.123780+0000","last_active":"2023-06-03T19:32:23.430935+0000","last_peered":"2023-06-03T19:32:23.430935+0000","last_clean":"2023-06-03T19:32:23.430935+0000","last_became_active":"2023-06-03T19:32:22.122852+0000","last_became_peered":"2023-06-03T19:32:22.122852+0000","last_unstale":"2023-06-03T19:32:23.430935+0000","last_undegraded":"2023-06-03T19:32:23.430935+0000","last_fullsized":"2023-06-03T19:32:23.430935+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:32:20.692043+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:32:20.692043+0000","last_clean_scrub_stamp":"2023-06-03T19:32:20.692043+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":15,"reported_epoch":54,"state":"active+clean","last_fresh":"2023-06-03T19:32:23.430917+0000","last_change":"2023-06-03T19:31:15.881174+0000","last_active":"2023-06-03T19:32:23.430917+0000","last_peered":"2023-06-03T19:32:23.430917+0000","last_clean":"2023-06-03T19:32:23.430917+0000","last_became_active":"2023-06-03T19:31:15.880415+0000","last_became_peered":"2023-06-03T19:31:15.880415+0000","last_unstale":"2023-06-03T19:32:23.430917+0000","last_undegraded":"2023-06-03T19:32:23.430917+0000","last_fullsized":"2023-06-03T19:32:23.430917+0000","mapping_epoch":49,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":50,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:27:28.100808+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:27:28.100808+0000","last_clean_scrub_stamp":"2023-06-03T19:27:28.100808+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":11,"reported_epoch":54,"state":"active+clean","last_fresh":"2023-06-03T19:32:23.344763+0000","last_change":"2023-06-03T19:32:21.707936+0000","last_active":"2023-06-03T19:32:23.344763+0000","last_peered":"2023-06-03T19:32:23.344763+0000","last_clean":"2023-06-03T19:32:23.344763+0000","last_became_active":"2023-06-03T19:32:21.707538+0000","last_became_peered":"2023-06-03T19:32:21.707538+0000","last_unstale":"2023-06-03T19:32:23.344763+0000","last_undegraded":"2023-06-03T19:32:23.344763+0000","last_fullsized":"2023-06-03T19:32:23.344763+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:32:20.692043+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:32:20.692043+0000","last_clean_scrub_stamp":"2023-06-03T19:32:20.692043+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":11,"reported_epoch":54,"state":"active+clean","last_fresh":"2023-06-03T19:32:22.716441+0000","last_change":"2023-06-03T19:32:22.123980+0000","last_active":"2023-06-03T19:32:22.716441+0000","last_peered":"2023-06-03T19:32:22.716441+0000","last_clean":"2023-06-03T19:32:22.716441+0000","last_became_active":"2023-06-03T19:32:22.123641+0000","last_became_peered":"2023-06-03T19:32:22.123641+0000","last_unstale":"2023-06-03T19:32:22.716441+0000","last_undegraded":"2023-06-03T19:32:22.716441+0000","last_fullsized":"2023-06-03T19:32:22.716441+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:32:20.692043+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:32:20.692043+0000","last_clean_scrub_stamp":"2023-06-03T19:32:20.692043+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":11,"reported_epoch":54,"state":"active+clean","last_fresh":"2023-06-03T19:32:23.344642+0000","last_change":"2023-06-03T19:32:21.709485+0000","last_active":"2023-06-03T19:32:23.344642+0000","last_peered":"2023-06-03T19:32:23.344642+0000","last_clean":"2023-06-03T19:32:23.344642+0000","last_became_active":"2023-06-03T19:32:21.708866+0000","last_became_peered":"2023-06-03T19:32:21.708866+0000","last_unstale":"2023-06-03T19:32:23.344642+0000","last_undegraded":"2023-06-03T19:32:23.344642+0000","last_fullsized":"2023-06-03T19:32:23.344642+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:32:20.692043+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:32:20.692043+0000","last_clean_scrub_stamp":"2023-06-03T19:32:20.692043+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":11,"reported_epoch":54,"state":"active+clean","last_fresh":"2023-06-03T19:32:22.931176+0000","last_change":"2023-06-03T19:32:22.125458+0000","last_active":"2023-06-03T19:32:22.931176+0000","last_peered":"2023-06-03T19:32:22.931176+0000","last_clean":"2023-06-03T19:32:22.931176+0000","last_became_active":"2023-06-03T19:32:22.124914+0000","last_became_peered":"2023-06-03T19:32:22.124914+0000","last_unstale":"2023-06-03T19:32:22.931176+0000","last_undegraded":"2023-06-03T19:32:22.931176+0000","last_fullsized":"2023-06-03T19:32:22.931176+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:32:20.692043+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:32:20.692043+0000","last_clean_scrub_stamp":"2023-06-03T19:32:20.692043+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":3,"acting":3,"num_store_stats":3}],"osd_stats":[{"osd":7,"up_from":48,"seq":206158430241,"num_pgs":6,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":5336,"kb_used_data":528,"kb_used_omap":0,"kb_used_meta":4800,"kb_avail":93739816,"statfs":{"total":95995035648,"available":95989571584,"internally_reserved":0,"allocated":540672,"data_stored":220511,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":4915200},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 3 19:33:21 2023","interfaces":[{"interface":"back","average":{"1min":0.83799999999999997,"5min":0.81299999999999994,"15min":0.80900000000000005},"min":{"1min":0.68000000000000005,"5min":0.57699999999999996,"15min":0.57699999999999996},"max":{"1min":1.1919999999999999,"5min":1.1919999999999999,"15min":1.1919999999999999},"last":0.90200000000000002},{"interface":"front","average":{"1min":0.94699999999999995,"5min":0.88600000000000001,"15min":0.876},"min":{"1min":0.72199999999999998,"5min":0.56499999999999995,"15min":0.56499999999999995},"max":{"1min":1.2569999999999999,"5min":1.2569999999999999,"15min":1.2569999999999999},"last":0.751}]},{"osd":1,"last update":"Sat Jun 3 19:33:21 2023","interfaces":[{"interface":"back","average":{"1min":0.871,"5min":0.82899999999999996,"15min":0.82199999999999995},"min":{"1min":0.65300000000000002,"5min":0.53100000000000003,"15min":0.53100000000000003},"max":{"1min":1.1739999999999999,"5min":1.202,"15min":1.202},"last":0.83799999999999997},{"interface":"front","average":{"1min":0.95299999999999996,"5min":0.88300000000000001,"15min":0.871},"min":{"1min":0.76800000000000002,"5min":0.61699999999999999,"15min":0.61699999999999999},"max":{"1min":1.095,"5min":1.127,"15min":1.127},"last":1.085}]},{"osd":2,"last update":"Sat Jun 3 19:33:21 2023","interfaces":[{"interface":"back","average":{"1min":0.90100000000000002,"5min":0.91300000000000003,"15min":0.91500000000000004},"min":{"1min":0.63300000000000001,"5min":0.63300000000000001,"15min":0.63300000000000001},"max":{"1min":1.21,"5min":1.538,"15min":1.538},"last":1.016},{"interface":"front","average":{"1min":0.96899999999999997,"5min":0.88100000000000001,"15min":0.86699999999999999},"min":{"1min":0.61499999999999999,"5min":0.60699999999999998,"15min":0.60699999999999998},"max":{"1min":1.24,"5min":1.24,"15min":1.24},"last":1.109}]},{"osd":3,"last update":"Sat Jun 3 19:33:21 2023","interfaces":[{"interface":"back","average":{"1min":0.95699999999999996,"5min":0.91600000000000004,"15min":0.90900000000000003},"min":{"1min":0.71299999999999997,"5min":0.53800000000000003,"15min":0.53800000000000003},"max":{"1min":1.1539999999999999,"5min":1.3069999999999999,"15min":1.3069999999999999},"last":1.032},{"interface":"front","average":{"1min":0.92600000000000005,"5min":0.92900000000000005,"15min":0.92900000000000005},"min":{"1min":0.63500000000000001,"5min":0.57199999999999995,"15min":0.57199999999999995},"max":{"1min":1.143,"5min":1.5069999999999999,"15min":1.5069999999999999},"last":0.94899999999999995}]},{"osd":4,"last update":"Sat Jun 3 19:33:21 2023","interfaces":[{"interface":"back","average":{"1min":0.89200000000000002,"5min":0.75800000000000001,"15min":0.73599999999999999},"min":{"1min":0.57399999999999995,"5min":0.51200000000000001,"15min":0.51200000000000001},"max":{"1min":1.2829999999999999,"5min":1.2829999999999999,"15min":1.2829999999999999},"last":0.82299999999999995},{"interface":"front","average":{"1min":0.873,"5min":0.82899999999999996,"15min":0.82199999999999995},"min":{"1min":0.53300000000000003,"5min":0.45500000000000002,"15min":0.45500000000000002},"max":{"1min":1.3009999999999999,"5min":1.351,"15min":1.351},"last":1.244}]},{"osd":5,"last update":"Sat Jun 3 19:33:21 2023","interfaces":[{"interface":"back","average":{"1min":0.90300000000000002,"5min":0.82699999999999996,"15min":0.81499999999999995},"min":{"1min":0.58399999999999996,"5min":0.48499999999999999,"15min":0.48499999999999999},"max":{"1min":1.3320000000000001,"5min":1.3320000000000001,"15min":1.3320000000000001},"last":1.1679999999999999},{"interface":"front","average":{"1min":0.98999999999999999,"5min":0.93799999999999994,"15min":0.93000000000000005},"min":{"1min":0.59299999999999997,"5min":0.437,"15min":0.437},"max":{"1min":1.5620000000000001,"5min":1.605,"15min":1.605},"last":1.2609999999999999}]},{"osd":6,"last update":"Sat Jun 3 19:33:21 2023","interfaces":[{"interface":"back","average":{"1min":1.083,"5min":0.98999999999999999,"15min":0.97399999999999998},"min":{"1min":0.66300000000000003,"5min":0.47899999999999998,"15min":0.47899999999999998},"max":{"1min":1.6080000000000001,"5min":1.6870000000000001,"15min":1.6870000000000001},"last":1.212},{"interface":"front","average":{"1min":1.01,"5min":0.95299999999999996,"15min":0.94299999999999995},"min":{"1min":0.61799999999999999,"5min":0.59299999999999997,"15min":0.59299999999999997},"max":{"1min":1.4319999999999999,"5min":1.4319999999999999,"15min":1.4319999999999999},"last":1.1859999999999999}]}]},{"osd":6,"up_from":42,"seq":180388626474,"num_pgs":4,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":5396,"kb_used_data":524,"kb_used_omap":0,"kb_used_meta":4864,"kb_avail":93739756,"statfs":{"total":95995035648,"available":95989510144,"internally_reserved":0,"allocated":536576,"data_stored":220196,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":4980736},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 3 19:33:34 2023","interfaces":[{"interface":"back","average":{"1min":0.84599999999999997,"5min":0.84099999999999997,"15min":0.82999999999999996},"min":{"1min":0.49099999999999999,"5min":0.49099999999999999,"15min":0.49099999999999999},"max":{"1min":1.2749999999999999,"5min":1.2749999999999999,"15min":1.2749999999999999},"last":0.58199999999999996},{"interface":"front","average":{"1min":0.84299999999999997,"5min":0.872,"15min":0.872},"min":{"1min":0.66200000000000003,"5min":0.57399999999999995,"15min":0.57399999999999995},"max":{"1min":1.087,"5min":1.284,"15min":1.284},"last":0.64400000000000002}]},{"osd":1,"last update":"Sat Jun 3 19:33:34 2023","interfaces":[{"interface":"back","average":{"1min":0.85799999999999998,"5min":0.875,"15min":0.87},"min":{"1min":0.50700000000000001,"5min":0.50700000000000001,"15min":0.50700000000000001},"max":{"1min":1.175,"5min":1.3759999999999999,"15min":1.3759999999999999},"last":0.61399999999999999},{"interface":"front","average":{"1min":0.872,"5min":0.93400000000000005,"15min":0.95199999999999996},"min":{"1min":0.56399999999999995,"5min":0.54100000000000004,"15min":0.54100000000000004},"max":{"1min":1.125,"5min":1.7769999999999999,"15min":1.7769999999999999},"last":0.71699999999999997}]},{"osd":2,"last update":"Sat Jun 3 19:33:34 2023","interfaces":[{"interface":"back","average":{"1min":0.85599999999999998,"5min":0.92000000000000004,"15min":0.92000000000000004},"min":{"1min":0.58499999999999996,"5min":0.58499999999999996,"15min":0.58499999999999996},"max":{"1min":1.2010000000000001,"5min":1.3069999999999999,"15min":1.3069999999999999},"last":0.68799999999999994},{"interface":"front","average":{"1min":0.93000000000000005,"5min":0.93400000000000005,"15min":0.93400000000000005},"min":{"1min":0.63900000000000001,"5min":0.63600000000000001,"15min":0.63600000000000001},"max":{"1min":1.2929999999999999,"5min":1.2929999999999999,"15min":1.2929999999999999},"last":0.86699999999999999}]},{"osd":3,"last update":"Sat Jun 3 19:33:34 2023","interfaces":[{"interface":"back","average":{"1min":0.90900000000000003,"5min":0.86199999999999999,"15min":0.84899999999999998},"min":{"1min":0.60199999999999998,"5min":0.60199999999999998,"15min":0.60199999999999998},"max":{"1min":1.2010000000000001,"5min":1.2290000000000001,"15min":1.2290000000000001},"last":0.83299999999999996},{"interface":"front","average":{"1min":0.92600000000000005,"5min":0.98899999999999999,"15min":1.002},"min":{"1min":0.64200000000000002,"5min":0.64200000000000002,"15min":0.64200000000000002},"max":{"1min":1.1950000000000001,"5min":1.905,"15min":1.905},"last":0.76200000000000001}]},{"osd":4,"last update":"Sat Jun 3 19:33:34 2023","interfaces":[{"interface":"back","average":{"1min":0.93600000000000005,"5min":0.85499999999999998,"15min":0.84099999999999997},"min":{"1min":0.53600000000000003,"5min":0.46899999999999997,"15min":0.46899999999999997},"max":{"1min":1.2490000000000001,"5min":1.3140000000000001,"15min":1.3140000000000001},"last":0.96199999999999997},{"interface":"front","average":{"1min":0.93500000000000005,"5min":0.88100000000000001,"15min":0.871},"min":{"1min":0.61799999999999999,"5min":0.434,"15min":0.434},"max":{"1min":1.218,"5min":1.218,"15min":1.218},"last":0.94799999999999995}]},{"osd":5,"last update":"Sat Jun 3 19:33:34 2023","interfaces":[{"interface":"back","average":{"1min":1.0429999999999999,"5min":0.93500000000000005,"15min":0.91900000000000004},"min":{"1min":0.60199999999999998,"5min":0.41599999999999998,"15min":0.41599999999999998},"max":{"1min":1.3109999999999999,"5min":1.3109999999999999,"15min":1.3109999999999999},"last":1.034},{"interface":"front","average":{"1min":1.02,"5min":0.93300000000000005,"15min":0.93500000000000005},"min":{"1min":0.64700000000000002,"5min":0.52500000000000002,"15min":0.52500000000000002},"max":{"1min":1.2310000000000001,"5min":1.284,"15min":1.284},"last":1.0189999999999999}]},{"osd":7,"last update":"Sat Jun 3 19:33:20 2023","interfaces":[{"interface":"back","average":{"1min":1.081,"5min":0.96299999999999997,"15min":0.94299999999999995},"min":{"1min":0.626,"5min":0.45700000000000002,"15min":0.45700000000000002},"max":{"1min":1.3919999999999999,"5min":1.3919999999999999,"15min":1.3919999999999999},"last":1.091},{"interface":"front","average":{"1min":1.0780000000000001,"5min":1.0109999999999999,"15min":1},"min":{"1min":0.63400000000000001,"5min":0.53600000000000003,"15min":0.53600000000000003},"max":{"1min":1.347,"5min":1.3839999999999999,"15min":1.3839999999999999},"last":0.91700000000000004}]}]},{"osd":1,"up_from":13,"seq":55834574936,"num_pgs":4,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":5652,"kb_used_data":524,"kb_used_omap":0,"kb_used_meta":5120,"kb_avail":93739500,"statfs":{"total":95995035648,"available":95989248000,"internally_reserved":0,"allocated":536576,"data_stored":220196,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5242880},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 3 19:33:48 2023","interfaces":[{"interface":"back","average":{"1min":0.55600000000000005,"5min":0.67100000000000004,"15min":0.56899999999999995},"min":{"1min":0.34200000000000003,"5min":0.34200000000000003,"15min":0.34200000000000003},"max":{"1min":0.77900000000000003,"5min":1.123,"15min":1.123},"last":0.64000000000000001},{"interface":"front","average":{"1min":0.626,"5min":0.72199999999999998,"15min":0.61599999999999999},"min":{"1min":0.30399999999999999,"5min":0.30399999999999999,"15min":0.29699999999999999},"max":{"1min":1.075,"5min":1.1559999999999999,"15min":1.1559999999999999},"last":0.59499999999999997}]},{"osd":2,"last update":"Sat Jun 3 19:33:38 2023","interfaces":[{"interface":"back","average":{"1min":0.65800000000000003,"5min":0.73199999999999998,"15min":0.628},"min":{"1min":0.41299999999999998,"5min":0.40899999999999997,"15min":0.27600000000000002},"max":{"1min":0.92600000000000005,"5min":1.2030000000000001,"15min":1.2030000000000001},"last":0.74399999999999999},{"interface":"front","average":{"1min":0.624,"5min":0.749,"15min":0.66200000000000003},"min":{"1min":0.42599999999999999,"5min":0.42599999999999999,"15min":0.32100000000000001},"max":{"1min":0.94299999999999995,"5min":1.2150000000000001,"15min":1.2150000000000001},"last":0.83799999999999997}]},{"osd":3,"last update":"Sat Jun 3 19:33:23 2023","interfaces":[{"interface":"back","average":{"1min":0.68000000000000005,"5min":0.74399999999999999,"15min":0.73099999999999998},"min":{"1min":0.42399999999999999,"5min":0.42399999999999999,"15min":0.42399999999999999},"max":{"1min":1.022,"5min":1.1100000000000001,"15min":1.1100000000000001},"last":0.69699999999999995},{"interface":"front","average":{"1min":0.77800000000000002,"5min":0.81999999999999995,"15min":0.78000000000000003},"min":{"1min":0.40600000000000003,"5min":0.40600000000000003,"15min":0.40600000000000003},"max":{"1min":1.0649999999999999,"5min":1.264,"15min":1.264},"last":0.95199999999999996}]},{"osd":4,"last update":"Sat Jun 3 19:33:04 2023","interfaces":[{"interface":"back","average":{"1min":0.89600000000000002,"5min":0.94299999999999995,"15min":0.94899999999999995},"min":{"1min":0.66700000000000004,"5min":0.66700000000000004,"15min":0.66700000000000004},"max":{"1min":1.26,"5min":1.2949999999999999,"15min":1.2949999999999999},"last":0.98799999999999999},{"interface":"front","average":{"1min":0.95099999999999996,"5min":0.93500000000000005,"15min":0.93300000000000005},"min":{"1min":0.65300000000000002,"5min":0.64500000000000002,"15min":0.64500000000000002},"max":{"1min":1.444,"5min":1.444,"15min":1.444},"last":1.0549999999999999}]},{"osd":5,"last update":"Sat Jun 3 19:33:48 2023","interfaces":[{"interface":"back","average":{"1min":0.93899999999999995,"5min":1.018,"15min":1.0389999999999999},"min":{"1min":0.68799999999999994,"5min":0.65200000000000002,"15min":0.65200000000000002},"max":{"1min":1.353,"5min":1.617,"15min":1.617},"last":1.3049999999999999},{"interface":"front","average":{"1min":0.96899999999999997,"5min":1.042,"15min":1.0720000000000001},"min":{"1min":0.69999999999999996,"5min":0.67900000000000005,"15min":0.67900000000000005},"max":{"1min":1.325,"5min":1.595,"15min":1.595},"last":1.121}]},{"osd":6,"last update":"Sat Jun 3 19:33:32 2023","interfaces":[{"interface":"back","average":{"1min":0.98299999999999998,"5min":1.016,"15min":1.0249999999999999},"min":{"1min":0.54500000000000004,"5min":0.54500000000000004,"15min":0.54500000000000004},"max":{"1min":1.4770000000000001,"5min":1.4770000000000001,"15min":1.4770000000000001},"last":1.161},{"interface":"front","average":{"1min":1.0169999999999999,"5min":1.0940000000000001,"15min":1.1120000000000001},"min":{"1min":0.76900000000000002,"5min":0.69499999999999995,"15min":0.69499999999999995},"max":{"1min":1.3859999999999999,"5min":1.3859999999999999,"15min":1.3859999999999999},"last":1.222}]},{"osd":7,"last update":"Sat Jun 3 19:33:18 2023","interfaces":[{"interface":"back","average":{"1min":0.99399999999999999,"5min":1.0760000000000001,"15min":1.0900000000000001},"min":{"1min":0.63700000000000001,"5min":0.63700000000000001,"15min":0.63700000000000001},"max":{"1min":1.369,"5min":1.4810000000000001,"15min":1.4810000000000001},"last":1.2569999999999999},{"interface":"front","average":{"1min":1.0469999999999999,"5min":1.149,"15min":1.1659999999999999},"min":{"1min":0.71199999999999997,"5min":0.71199999999999997,"15min":0.71199999999999997},"max":{"1min":1.54,"5min":1.54,"15min":1.54},"last":1.2769999999999999}]}]},{"osd":0,"up_from":8,"seq":34359738465,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":5716,"kb_used_data":524,"kb_used_omap":0,"kb_used_meta":5184,"kb_avail":93739436,"statfs":{"total":95995035648,"available":95989182464,"internally_reserved":0,"allocated":536576,"data_stored":220196,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5308416},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Jun 3 19:32:54 2023","interfaces":[{"interface":"back","average":{"1min":0.67200000000000004,"5min":0.69199999999999995,"15min":0.57899999999999996},"min":{"1min":0.39000000000000001,"5min":0.32300000000000001,"15min":0.32100000000000001},"max":{"1min":1.012,"5min":1.238,"15min":1.238},"last":0.59699999999999998},{"interface":"front","average":{"1min":0.72999999999999998,"5min":0.70299999999999996,"15min":0.59299999999999997},"min":{"1min":0.40000000000000002,"5min":0.40000000000000002,"15min":0.33600000000000002},"max":{"1min":0.995,"5min":1.0860000000000001,"15min":1.0860000000000001},"last":0.625}]},{"osd":2,"last update":"Sat Jun 3 19:33:36 2023","interfaces":[{"interface":"back","average":{"1min":0.66400000000000003,"5min":0.71799999999999997,"15min":0.65000000000000002},"min":{"1min":0.47399999999999998,"5min":0.35299999999999998,"15min":0.35299999999999998},"max":{"1min":0.94999999999999996,"5min":1.052,"15min":1.052},"last":0.71299999999999997},{"interface":"front","average":{"1min":0.69499999999999995,"5min":0.752,"15min":0.67100000000000004},"min":{"1min":0.39100000000000001,"5min":0.39100000000000001,"15min":0.39100000000000001},"max":{"1min":1.0980000000000001,"5min":1.1850000000000001,"15min":1.1850000000000001},"last":0.94499999999999995}]},{"osd":3,"last update":"Sat Jun 3 19:33:28 2023","interfaces":[{"interface":"back","average":{"1min":0.66100000000000003,"5min":0.71299999999999997,"15min":0.73999999999999999},"min":{"1min":0.373,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":1.0169999999999999,"5min":1.0860000000000001,"15min":1.0860000000000001},"last":0.67200000000000004},{"interface":"front","average":{"1min":0.73999999999999999,"5min":0.82099999999999995,"15min":0.80200000000000005},"min":{"1min":0.437,"5min":0.437,"15min":0.437},"max":{"1min":1.1200000000000001,"5min":1.1639999999999999,"15min":1.1639999999999999},"last":0.80100000000000005}]},{"osd":4,"last update":"Sat Jun 3 19:33:00 2023","interfaces":[{"interface":"back","average":{"1min":0.93999999999999995,"5min":0.92300000000000004,"15min":0.92400000000000004},"min":{"1min":0.71499999999999997,"5min":0.54200000000000004,"15min":0.54200000000000004},"max":{"1min":1.425,"5min":1.77,"15min":1.77},"last":0.84799999999999998},{"interface":"front","average":{"1min":0.97499999999999998,"5min":0.97099999999999997,"15min":0.93899999999999995},"min":{"1min":0.56599999999999995,"5min":0.56599999999999995,"15min":0.56599999999999995},"max":{"1min":1.4590000000000001,"5min":1.4590000000000001,"15min":1.4590000000000001},"last":0.77500000000000002}]},{"osd":5,"last update":"Sat Jun 3 19:32:49 2023","interfaces":[{"interface":"back","average":{"1min":0.97399999999999998,"5min":1.0620000000000001,"15min":1.0660000000000001},"min":{"1min":0.52700000000000002,"5min":0.52700000000000002,"15min":0.52700000000000002},"max":{"1min":1.5549999999999999,"5min":1.5549999999999999,"15min":1.5549999999999999},"last":0.755},{"interface":"front","average":{"1min":0.93799999999999994,"5min":0.97799999999999998,"15min":0.97799999999999998},"min":{"1min":0.42899999999999999,"5min":0.42899999999999999,"15min":0.42899999999999999},"max":{"1min":1.325,"5min":1.325,"15min":1.325},"last":0.81499999999999995}]},{"osd":6,"last update":"Sat Jun 3 19:33:36 2023","interfaces":[{"interface":"back","average":{"1min":0.98499999999999999,"5min":1.0760000000000001,"15min":1.0900000000000001},"min":{"1min":0.66300000000000003,"5min":0.65300000000000002,"15min":0.65300000000000002},"max":{"1min":1.5660000000000001,"5min":1.615,"15min":1.615},"last":0.88},{"interface":"front","average":{"1min":0.98799999999999999,"5min":1.0449999999999999,"15min":1.0549999999999999},"min":{"1min":0.54400000000000004,"5min":0.54400000000000004,"15min":0.54400000000000004},"max":{"1min":1.421,"5min":1.4490000000000001,"15min":1.4490000000000001},"last":1.018}]},{"osd":7,"last update":"Sat Jun 3 19:33:22 2023","interfaces":[{"interface":"back","average":{"1min":1.022,"5min":1.0720000000000001,"15min":1.0800000000000001},"min":{"1min":0.629,"5min":0.59499999999999997,"15min":0.59499999999999997},"max":{"1min":1.456,"5min":1.514,"15min":1.514},"last":1.1359999999999999},{"interface":"front","average":{"1min":0.96199999999999997,"5min":1.028,"15min":1.0389999999999999},"min":{"1min":0.57899999999999996,"5min":0.57899999999999996,"15min":0.57899999999999996},"max":{"1min":1.526,"5min":1.526,"15min":1.526},"last":1.091}]}]},{"osd":2,"up_from":18,"seq":77309411406,"num_pgs":3,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":5592,"kb_used_data":528,"kb_used_omap":0,"kb_used_meta":5056,"kb_avail":93739560,"statfs":{"total":95995035648,"available":95989309440,"internally_reserved":0,"allocated":540672,"data_stored":220511,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5177344},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 3 19:33:37 2023","interfaces":[{"interface":"back","average":{"1min":0.63800000000000001,"5min":0.69099999999999995,"15min":0.61099999999999999},"min":{"1min":0.314,"5min":0.314,"15min":0.314},"max":{"1min":0.97699999999999998,"5min":1.1819999999999999,"15min":1.1819999999999999},"last":0.64400000000000002},{"interface":"front","average":{"1min":0.754,"5min":0.70899999999999996,"15min":0.627},"min":{"1min":0.35499999999999998,"5min":0.35499999999999998,"15min":0.30199999999999999},"max":{"1min":3.004,"5min":3.004,"15min":3.004},"last":0.68000000000000005}]},{"osd":1,"last update":"Sat Jun 3 19:33:37 2023","interfaces":[{"interface":"back","average":{"1min":0.69699999999999995,"5min":0.72699999999999998,"15min":0.63300000000000001},"min":{"1min":0.39100000000000001,"5min":0.39100000000000001,"15min":0.39100000000000001},"max":{"1min":1.1339999999999999,"5min":1.139,"15min":1.139},"last":1.0900000000000001},{"interface":"front","average":{"1min":0.71499999999999997,"5min":0.77200000000000002,"15min":0.66800000000000004},"min":{"1min":0.41199999999999998,"5min":0.39300000000000002,"15min":0.33500000000000002},"max":{"1min":1.05,"5min":1.161,"15min":1.161},"last":1.1619999999999999}]},{"osd":3,"last update":"Sat Jun 3 19:33:29 2023","interfaces":[{"interface":"back","average":{"1min":0.71399999999999997,"5min":0.77000000000000002,"15min":0.749},"min":{"1min":0.44700000000000001,"5min":0.44700000000000001,"15min":0.44700000000000001},"max":{"1min":0.90600000000000003,"5min":1.0469999999999999,"15min":1.0469999999999999},"last":0.74099999999999999},{"interface":"front","average":{"1min":0.77700000000000002,"5min":0.82099999999999995,"15min":0.77300000000000002},"min":{"1min":0.36299999999999999,"5min":0.34399999999999997,"15min":0.34399999999999997},"max":{"1min":1.085,"5min":1.5840000000000001,"15min":1.5840000000000001},"last":0.82999999999999996}]},{"osd":4,"last update":"Sat Jun 3 19:33:09 2023","interfaces":[{"interface":"back","average":{"1min":0.91900000000000004,"5min":0.94799999999999995,"15min":0.93300000000000005},"min":{"1min":0.625,"5min":0.61499999999999999,"15min":0.61499999999999999},"max":{"1min":1.266,"5min":1.536,"15min":1.536},"last":1.2010000000000001},{"interface":"front","average":{"1min":0.95199999999999996,"5min":1.0860000000000001,"15min":1.012},"min":{"1min":0.57799999999999996,"5min":0.57799999999999996,"15min":0.57799999999999996},"max":{"1min":1.3260000000000001,"5min":8.3829999999999991,"15min":8.3829999999999991},"last":0.95399999999999996}]},{"osd":5,"last update":"Sat Jun 3 19:32:50 2023","interfaces":[{"interface":"back","average":{"1min":1.05,"5min":1.1020000000000001,"15min":1.097},"min":{"1min":0.79900000000000004,"5min":0.79900000000000004,"15min":0.79900000000000004},"max":{"1min":1.2490000000000001,"5min":1.7150000000000001,"15min":1.7150000000000001},"last":0.89700000000000002},{"interface":"front","average":{"1min":0.96099999999999997,"5min":0.97699999999999998,"15min":0.97399999999999998},"min":{"1min":0.74399999999999999,"5min":0.74399999999999999,"15min":0.74399999999999999},"max":{"1min":1.268,"5min":1.369,"15min":1.369},"last":1.1100000000000001}]},{"osd":6,"last update":"Sat Jun 3 19:33:36 2023","interfaces":[{"interface":"back","average":{"1min":1.026,"5min":1.048,"15min":1.054},"min":{"1min":0.71799999999999997,"5min":0.71799999999999997,"15min":0.71799999999999997},"max":{"1min":1.3700000000000001,"5min":1.446,"15min":1.446},"last":1.0109999999999999},{"interface":"front","average":{"1min":1.026,"5min":1.1279999999999999,"15min":1.1459999999999999},"min":{"1min":0.58799999999999997,"5min":0.58799999999999997,"15min":0.58799999999999997},"max":{"1min":1.2689999999999999,"5min":1.5940000000000001,"15min":1.5940000000000001},"last":1.2250000000000001}]},{"osd":7,"last update":"Sat Jun 3 19:33:16 2023","interfaces":[{"interface":"back","average":{"1min":0.98499999999999999,"5min":1.085,"15min":1.101},"min":{"1min":0.68999999999999995,"5min":0.68999999999999995,"15min":0.68999999999999995},"max":{"1min":1.4219999999999999,"5min":1.486,"15min":1.486},"last":1.0329999999999999},{"interface":"front","average":{"1min":1.0700000000000001,"5min":1.135,"15min":1.1459999999999999},"min":{"1min":0.69899999999999995,"5min":0.69899999999999995,"15min":0.69899999999999995},"max":{"1min":1.3660000000000001,"5min":1.5600000000000001,"15min":1.5600000000000001},"last":1.1339999999999999}]}]},{"osd":3,"up_from":25,"seq":107374182469,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":5528,"kb_used_data":528,"kb_used_omap":0,"kb_used_meta":4992,"kb_avail":93739624,"statfs":{"total":95995035648,"available":95989374976,"internally_reserved":0,"allocated":540672,"data_stored":220511,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5111808},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 3 19:33:23 2023","interfaces":[{"interface":"back","average":{"1min":0.67900000000000005,"5min":0.69199999999999995,"15min":0.67400000000000004},"min":{"1min":0.45700000000000002,"5min":0.377,"15min":0.377},"max":{"1min":0.93999999999999995,"5min":1.153,"15min":1.153},"last":0.56499999999999995},{"interface":"front","average":{"1min":0.82299999999999995,"5min":0.76000000000000001,"15min":0.73899999999999999},"min":{"1min":0.41899999999999998,"5min":0.41899999999999998,"15min":0.41899999999999998},"max":{"1min":1.8979999999999999,"5min":1.8979999999999999,"15min":1.8979999999999999},"last":0.63300000000000001}]},{"osd":1,"last update":"Sat Jun 3 19:33:23 2023","interfaces":[{"interface":"back","average":{"1min":0.73299999999999998,"5min":0.753,"15min":0.72299999999999998},"min":{"1min":0.47999999999999998,"5min":0.439,"15min":0.439},"max":{"1min":1.044,"5min":1.218,"15min":1.218},"last":1.095},{"interface":"front","average":{"1min":0.70799999999999996,"5min":0.76100000000000001,"15min":0.72899999999999998},"min":{"1min":0.53700000000000003,"5min":0.45000000000000001,"15min":0.45000000000000001},"max":{"1min":1.109,"5min":1.109,"15min":1.109},"last":0.80200000000000005}]},{"osd":2,"last update":"Sat Jun 3 19:33:23 2023","interfaces":[{"interface":"back","average":{"1min":0.80900000000000005,"5min":0.80500000000000005,"15min":0.77700000000000002},"min":{"1min":0.48899999999999999,"5min":0.48899999999999999,"15min":0.48899999999999999},"max":{"1min":1.1899999999999999,"5min":1.359,"15min":1.359},"last":0.71799999999999997},{"interface":"front","average":{"1min":0.72899999999999998,"5min":0.76200000000000001,"15min":0.755},"min":{"1min":0.504,"5min":0.42699999999999999,"15min":0.42699999999999999},"max":{"1min":1.091,"5min":1.355,"15min":1.355},"last":0.997}]},{"osd":4,"last update":"Sat Jun 3 19:33:03 2023","interfaces":[{"interface":"back","average":{"1min":0.97199999999999998,"5min":0.95899999999999996,"15min":0.95399999999999996},"min":{"1min":0.74399999999999999,"5min":0.59499999999999997,"15min":0.59499999999999997},"max":{"1min":1.3320000000000001,"5min":1.393,"15min":1.393},"last":0.90500000000000003},{"interface":"front","average":{"1min":1,"5min":1.0009999999999999,"15min":0.98999999999999999},"min":{"1min":0.83399999999999996,"5min":0.81200000000000006,"15min":0.81200000000000006},"max":{"1min":1.2969999999999999,"5min":1.528,"15min":1.528},"last":1.0660000000000001}]},{"osd":5,"last update":"Sat Jun 3 19:32:50 2023","interfaces":[{"interface":"back","average":{"1min":1.026,"5min":1.04,"15min":1.0600000000000001},"min":{"1min":0.76200000000000001,"5min":0.69299999999999995,"15min":0.69299999999999995},"max":{"1min":1.3460000000000001,"5min":1.48,"15min":1.48},"last":1.1899999999999999},{"interface":"front","average":{"1min":0.995,"5min":1.0629999999999999,"15min":1.079},"min":{"1min":0.68999999999999995,"5min":0.68400000000000005,"15min":0.68400000000000005},"max":{"1min":1.3360000000000001,"5min":1.452,"15min":1.452},"last":1.1479999999999999}]},{"osd":6,"last update":"Sat Jun 3 19:33:34 2023","interfaces":[{"interface":"back","average":{"1min":1.0329999999999999,"5min":1.0920000000000001,"15min":1.113},"min":{"1min":0.81100000000000005,"5min":0.745,"15min":0.745},"max":{"1min":1.2789999999999999,"5min":1.5029999999999999,"15min":1.5029999999999999},"last":0.82199999999999995},{"interface":"front","average":{"1min":0.98399999999999999,"5min":1.075,"15min":1.105},"min":{"1min":0.66900000000000004,"5min":0.66900000000000004,"15min":0.66900000000000004},"max":{"1min":1.369,"5min":1.474,"15min":1.474},"last":1.125}]},{"osd":7,"last update":"Sat Jun 3 19:33:19 2023","interfaces":[{"interface":"back","average":{"1min":1.0640000000000001,"5min":1.079,"15min":1.081},"min":{"1min":0.68700000000000006,"5min":0.66800000000000004,"15min":0.66800000000000004},"max":{"1min":1.643,"5min":1.643,"15min":1.643},"last":0.77100000000000002},{"interface":"front","average":{"1min":1.123,"5min":1.0960000000000001,"15min":1.0920000000000001},"min":{"1min":0.64900000000000002,"5min":0.64900000000000002,"15min":0.64900000000000002},"max":{"1min":1.52,"5min":1.52,"15min":1.52},"last":0.88200000000000001}]}]},{"osd":4,"up_from":30,"seq":128849018941,"num_pgs":2,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":5528,"kb_used_data":528,"kb_used_omap":0,"kb_used_meta":4992,"kb_avail":93739624,"statfs":{"total":95995035648,"available":95989374976,"internally_reserved":0,"allocated":540672,"data_stored":220511,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5111808},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 3 19:33:06 2023","interfaces":[{"interface":"back","average":{"1min":0.91000000000000003,"5min":0.86599999999999999,"15min":0.86699999999999999},"min":{"1min":0.64600000000000002,"5min":0.55300000000000005,"15min":0.55300000000000005},"max":{"1min":1.234,"5min":1.234,"15min":1.234},"last":0.72699999999999998},{"interface":"front","average":{"1min":0.92700000000000005,"5min":0.87,"15min":0.876},"min":{"1min":0.56999999999999995,"5min":0.46100000000000002,"15min":0.46100000000000002},"max":{"1min":1.218,"5min":1.218,"15min":1.218},"last":0.78600000000000003}]},{"osd":1,"last update":"Sat Jun 3 19:33:06 2023","interfaces":[{"interface":"back","average":{"1min":0.86299999999999999,"5min":0.86799999999999999,"15min":0.86599999999999999},"min":{"1min":0.61599999999999999,"5min":0.502,"15min":0.502},"max":{"1min":1.161,"5min":1.179,"15min":1.179},"last":0.76100000000000001},{"interface":"front","average":{"1min":0.90900000000000003,"5min":0.85799999999999998,"15min":0.85099999999999998},"min":{"1min":0.66600000000000004,"5min":0.47599999999999998,"15min":0.47599999999999998},"max":{"1min":1.2430000000000001,"5min":1.2430000000000001,"15min":1.2430000000000001},"last":0.80900000000000005}]},{"osd":2,"last update":"Sat Jun 3 19:33:06 2023","interfaces":[{"interface":"back","average":{"1min":0.94199999999999995,"5min":0.89400000000000002,"15min":0.88400000000000001},"min":{"1min":0.68300000000000005,"5min":0.65100000000000002,"15min":0.65100000000000002},"max":{"1min":1.1559999999999999,"5min":1.2190000000000001,"15min":1.2190000000000001},"last":0.70299999999999996},{"interface":"front","average":{"1min":0.85699999999999998,"5min":0.88100000000000001,"15min":0.88700000000000001},"min":{"1min":0.67300000000000004,"5min":0.60299999999999998,"15min":0.60299999999999998},"max":{"1min":1.0960000000000001,"5min":1.1679999999999999,"15min":1.1679999999999999},"last":0.88700000000000001}]},{"osd":3,"last update":"Sat Jun 3 19:33:06 2023","interfaces":[{"interface":"back","average":{"1min":0.88500000000000001,"5min":0.87,"15min":0.872},"min":{"1min":0.64000000000000001,"5min":0.56699999999999995,"15min":0.56699999999999995},"max":{"1min":1.071,"5min":1.2749999999999999,"15min":1.2749999999999999},"last":0.67800000000000005},{"interface":"front","average":{"1min":0.91900000000000004,"5min":0.93300000000000005,"15min":0.93999999999999995},"min":{"1min":0.67100000000000004,"5min":0.64300000000000002,"15min":0.64300000000000002},"max":{"1min":1.2669999999999999,"5min":1.2669999999999999,"15min":1.2669999999999999},"last":0.91700000000000004}]},{"osd":5,"last update":"Sat Jun 3 19:33:48 2023","interfaces":[{"interface":"back","average":{"1min":0.85099999999999998,"5min":0.79300000000000004,"15min":0.76300000000000001},"min":{"1min":0.52000000000000002,"5min":0.45500000000000002,"15min":0.45500000000000002},"max":{"1min":1.214,"5min":1.214,"15min":1.214},"last":0.62},{"interface":"front","average":{"1min":0.86699999999999999,"5min":0.84299999999999997,"15min":0.82699999999999996},"min":{"1min":0.46800000000000003,"5min":0.45500000000000002,"15min":0.45500000000000002},"max":{"1min":1.2769999999999999,"5min":1.298,"15min":1.298},"last":0.55900000000000005}]},{"osd":6,"last update":"Sat Jun 3 19:33:33 2023","interfaces":[{"interface":"back","average":{"1min":0.93000000000000005,"5min":0.91900000000000004,"15min":0.93500000000000005},"min":{"1min":0.53200000000000003,"5min":0.46899999999999997,"15min":0.46899999999999997},"max":{"1min":1.1930000000000001,"5min":1.226,"15min":1.226},"last":0.57799999999999996},{"interface":"front","average":{"1min":0.98899999999999999,"5min":0.93400000000000005,"15min":0.93999999999999995},"min":{"1min":0.65200000000000002,"5min":0.503,"15min":0.503},"max":{"1min":1.288,"5min":1.288,"15min":1.288},"last":0.81899999999999995}]},{"osd":7,"last update":"Sat Jun 3 19:33:17 2023","interfaces":[{"interface":"back","average":{"1min":1.0449999999999999,"5min":0.91000000000000003,"15min":0.88800000000000001},"min":{"1min":0.77600000000000002,"5min":0.47099999999999997,"15min":0.47099999999999997},"max":{"1min":1.3480000000000001,"5min":1.3480000000000001,"15min":1.3480000000000001},"last":0.87},{"interface":"front","average":{"1min":1.0900000000000001,"5min":0.94299999999999995,"15min":0.91900000000000004},"min":{"1min":0.73299999999999998,"5min":0.52300000000000002,"15min":0.52300000000000002},"max":{"1min":1.2989999999999999,"5min":1.2989999999999999,"15min":1.2989999999999999},"last":0.85599999999999998}]}]},{"osd":5,"up_from":36,"seq":154618822708,"num_pgs":2,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":5460,"kb_used_data":524,"kb_used_omap":0,"kb_used_meta":4928,"kb_avail":93739692,"statfs":{"total":95995035648,"available":95989444608,"internally_reserved":0,"allocated":536576,"data_stored":220196,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5046272},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 3 19:32:50 2023","interfaces":[{"interface":"back","average":{"1min":0.85399999999999998,"5min":0.875,"15min":0.88300000000000001},"min":{"1min":0.54700000000000004,"5min":0.51400000000000001,"15min":0.51400000000000001},"max":{"1min":1.4750000000000001,"5min":1.4750000000000001,"15min":1.4750000000000001},"last":0.69699999999999995},{"interface":"front","average":{"1min":0.89900000000000002,"5min":0.86599999999999999,"15min":0.84599999999999997},"min":{"1min":0.58999999999999997,"5min":0.58999999999999997,"15min":0.58999999999999997},"max":{"1min":1.9259999999999999,"5min":1.9259999999999999,"15min":1.9259999999999999},"last":1.008}]},{"osd":1,"last update":"Sat Jun 3 19:32:50 2023","interfaces":[{"interface":"back","average":{"1min":0.94299999999999995,"5min":0.89200000000000002,"15min":0.89000000000000001},"min":{"1min":0.61399999999999999,"5min":0.49399999999999999,"15min":0.49399999999999999},"max":{"1min":1.994,"5min":1.994,"15min":1.994},"last":1.1499999999999999},{"interface":"front","average":{"1min":0.93100000000000005,"5min":0.88900000000000001,"15min":0.86499999999999999},"min":{"1min":0.55400000000000005,"5min":0.55400000000000005,"15min":0.55400000000000005},"max":{"1min":1.742,"5min":1.742,"15min":1.742},"last":0.77000000000000002}]},{"osd":2,"last update":"Sat Jun 3 19:32:50 2023","interfaces":[{"interface":"back","average":{"1min":0.86599999999999999,"5min":0.88600000000000001,"15min":0.874},"min":{"1min":0.54200000000000004,"5min":0.54200000000000004,"15min":0.54200000000000004},"max":{"1min":1.595,"5min":1.708,"15min":1.708},"last":0.73399999999999999},{"interface":"front","average":{"1min":0.96599999999999997,"5min":0.91900000000000004,"15min":0.89700000000000002},"min":{"1min":0.56000000000000005,"5min":0.56000000000000005,"15min":0.56000000000000005},"max":{"1min":2.0840000000000001,"5min":2.0840000000000001,"15min":2.0840000000000001},"last":1.0349999999999999}]},{"osd":3,"last update":"Sat Jun 3 19:32:50 2023","interfaces":[{"interface":"back","average":{"1min":0.97999999999999998,"5min":0.95499999999999996,"15min":0.95899999999999996},"min":{"1min":0.61699999999999999,"5min":0.58199999999999996,"15min":0.58199999999999996},"max":{"1min":2.1699999999999999,"5min":2.1699999999999999,"15min":2.1699999999999999},"last":1.107},{"interface":"front","average":{"1min":0.90300000000000002,"5min":0.88500000000000001,"15min":0.86499999999999999},"min":{"1min":0.57899999999999996,"5min":0.57899999999999996,"15min":0.57899999999999996},"max":{"1min":1.5349999999999999,"5min":1.6459999999999999,"15min":1.6459999999999999},"last":0.80900000000000005}]},{"osd":4,"last update":"Sat Jun 3 19:32:50 2023","interfaces":[{"interface":"back","average":{"1min":0.93000000000000005,"5min":0.80400000000000005,"15min":0.77000000000000002},"min":{"1min":0.46400000000000002,"5min":0.39100000000000001,"15min":0.39100000000000001},"max":{"1min":2.1299999999999999,"5min":2.1299999999999999,"15min":2.1299999999999999},"last":0.88700000000000001},{"interface":"front","average":{"1min":1.006,"5min":0.80800000000000005,"15min":0.754},"min":{"1min":0.55300000000000005,"5min":0.38,"15min":0.38},"max":{"1min":2.0830000000000002,"5min":2.0830000000000002,"15min":2.0830000000000002},"last":0.52000000000000002}]},{"osd":6,"last update":"Sat Jun 3 19:33:35 2023","interfaces":[{"interface":"back","average":{"1min":0.96999999999999997,"5min":0.92200000000000004,"15min":0.90600000000000003},"min":{"1min":0.53600000000000003,"5min":0.46500000000000002,"15min":0.46500000000000002},"max":{"1min":1.2050000000000001,"5min":1.6599999999999999,"15min":1.6599999999999999},"last":0.80400000000000005},{"interface":"front","average":{"1min":1.052,"5min":0.998,"15min":0.97599999999999998},"min":{"1min":0.65900000000000003,"5min":0.56200000000000006,"15min":0.56200000000000006},"max":{"1min":1.2789999999999999,"5min":1.8380000000000001,"15min":1.8380000000000001},"last":0.96399999999999997}]},{"osd":7,"last update":"Sat Jun 3 19:33:18 2023","interfaces":[{"interface":"back","average":{"1min":1.048,"5min":1.0549999999999999,"15min":1.056},"min":{"1min":0.63200000000000001,"5min":0.63200000000000001,"15min":0.63200000000000001},"max":{"1min":1.292,"5min":2.004,"15min":2.004},"last":0.86699999999999999},{"interface":"front","average":{"1min":0.94599999999999995,"5min":0.94999999999999996,"15min":0.94999999999999996},"min":{"1min":0.52700000000000002,"5min":0.48099999999999998,"15min":0.48099999999999998},"max":{"1min":1.2509999999999999,"5min":1.6479999999999999,"15min":1.6479999999999999},"last":0.98399999999999999}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-06-03T19:33:53.902 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph pg dump --format=json 2023-06-03T19:33:54.031 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:33:53 smithi089 bash[26572]: cluster 2023-06-03T19:33:52.405653+0000 mgr.y (mgr.14445) 89 : cluster [DBG] pgmap v69: 9 pgs: 9 active+clean; 0 B data, 43 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:33:54.032 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:33:53 smithi089 bash[33173]: cluster 2023-06-03T19:33:52.405653+0000 mgr.y (mgr.14445) 89 : cluster [DBG] pgmap v69: 9 pgs: 9 active+clean; 0 B data, 43 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:33:54.086 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:33:53 smithi164 bash[26855]: cluster 2023-06-03T19:33:52.405653+0000 mgr.y (mgr.14445) 89 : cluster [DBG] pgmap v69: 9 pgs: 9 active+clean; 0 B data, 43 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:33:55.031 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:33:54 smithi089 bash[33173]: audit 2023-06-03T19:33:53.112581+0000 mgr.y (mgr.14445) 90 : audit [DBG] from='client.14616 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-06-03T19:33:55.032 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:33:54 smithi089 bash[26572]: audit 2023-06-03T19:33:53.112581+0000 mgr.y (mgr.14445) 90 : audit [DBG] from='client.14616 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-06-03T19:33:55.086 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:33:54 smithi164 bash[26855]: audit 2023-06-03T19:33:53.112581+0000 mgr.y (mgr.14445) 90 : audit [DBG] from='client.14616 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-06-03T19:33:56.031 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:33:55 smithi089 bash[26572]: cluster 2023-06-03T19:33:54.406223+0000 mgr.y (mgr.14445) 91 : cluster [DBG] pgmap v70: 9 pgs: 9 active+clean; 0 B data, 43 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:33:56.032 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:33:55 smithi089 bash[33173]: cluster 2023-06-03T19:33:54.406223+0000 mgr.y (mgr.14445) 91 : cluster [DBG] pgmap v70: 9 pgs: 9 active+clean; 0 B data, 43 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:33:56.086 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:33:55 smithi164 bash[26855]: cluster 2023-06-03T19:33:54.406223+0000 mgr.y (mgr.14445) 91 : cluster [DBG] pgmap v70: 9 pgs: 9 active+clean; 0 B data, 43 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:33:57.281 INFO:journalctl@ceph.alertmanager.a.smithi089.stdout:Jun 03 19:33:56 smithi089 bash[58026]: level=info ts=2023-06-03T19:33:56.833Z caller=cluster.go:688 component=cluster msg="gossip settled; proceeding" elapsed=10.003649721s 2023-06-03T19:33:58.031 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 19:33:57 smithi089 bash[26845]: ::ffff:172.21.15.164 - - [03/Jun/2023:19:33:57] "GET /metrics HTTP/1.1" 200 152009 "" "Prometheus/2.33.4" 2023-06-03T19:33:58.032 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:33:57 smithi089 bash[26572]: cluster 2023-06-03T19:33:56.406807+0000 mgr.y (mgr.14445) 92 : cluster [DBG] pgmap v71: 9 pgs: 9 active+clean; 0 B data, 43 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:33:58.032 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:33:57 smithi089 bash[26572]: audit 2023-06-03T19:33:57.351573+0000 mon.a (mon.0) 645 : audit [INF] from='mgr.14445 172.21.15.89:0/573502486' entity='mgr.y' 2023-06-03T19:33:58.033 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:33:57 smithi089 bash[26572]: audit 2023-06-03T19:33:57.400876+0000 mon.a (mon.0) 646 : audit [INF] from='mgr.14445 172.21.15.89:0/573502486' entity='mgr.y' 2023-06-03T19:33:58.033 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:33:57 smithi089 bash[26572]: audit 2023-06-03T19:33:57.402285+0000 mon.a (mon.0) 647 : audit [DBG] from='mgr.14445 172.21.15.89:0/573502486' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-03T19:33:58.033 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:33:57 smithi089 bash[26572]: audit 2023-06-03T19:33:57.403583+0000 mon.a (mon.0) 648 : audit [INF] from='mgr.14445 172.21.15.89:0/573502486' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-03T19:33:58.033 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:33:57 smithi089 bash[26572]: audit 2023-06-03T19:33:57.409824+0000 mon.a (mon.0) 649 : audit [INF] from='mgr.14445 172.21.15.89:0/573502486' entity='mgr.y' 2023-06-03T19:33:58.033 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:33:57 smithi089 bash[26572]: audit 2023-06-03T19:33:57.417424+0000 mon.a (mon.0) 650 : audit [DBG] from='mgr.14445 172.21.15.89:0/573502486' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-03T19:33:58.034 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:33:57 smithi089 bash[33173]: cluster 2023-06-03T19:33:56.406807+0000 mgr.y (mgr.14445) 92 : cluster [DBG] pgmap v71: 9 pgs: 9 active+clean; 0 B data, 43 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:33:58.034 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:33:57 smithi089 bash[33173]: audit 2023-06-03T19:33:57.351573+0000 mon.a (mon.0) 645 : audit [INF] from='mgr.14445 172.21.15.89:0/573502486' entity='mgr.y' 2023-06-03T19:33:58.034 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:33:57 smithi089 bash[33173]: audit 2023-06-03T19:33:57.400876+0000 mon.a (mon.0) 646 : audit [INF] from='mgr.14445 172.21.15.89:0/573502486' entity='mgr.y' 2023-06-03T19:33:58.034 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:33:57 smithi089 bash[33173]: audit 2023-06-03T19:33:57.402285+0000 mon.a (mon.0) 647 : audit [DBG] from='mgr.14445 172.21.15.89:0/573502486' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-03T19:33:58.034 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:33:57 smithi089 bash[33173]: audit 2023-06-03T19:33:57.403583+0000 mon.a (mon.0) 648 : audit [INF] from='mgr.14445 172.21.15.89:0/573502486' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-03T19:33:58.034 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:33:57 smithi089 bash[33173]: audit 2023-06-03T19:33:57.409824+0000 mon.a (mon.0) 649 : audit [INF] from='mgr.14445 172.21.15.89:0/573502486' entity='mgr.y' 2023-06-03T19:33:58.034 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:33:57 smithi089 bash[33173]: audit 2023-06-03T19:33:57.417424+0000 mon.a (mon.0) 650 : audit [DBG] from='mgr.14445 172.21.15.89:0/573502486' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-03T19:33:58.086 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:33:57 smithi164 bash[26855]: cluster 2023-06-03T19:33:56.406807+0000 mgr.y (mgr.14445) 92 : cluster [DBG] pgmap v71: 9 pgs: 9 active+clean; 0 B data, 43 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:33:58.087 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:33:57 smithi164 bash[26855]: audit 2023-06-03T19:33:57.351573+0000 mon.a (mon.0) 645 : audit [INF] from='mgr.14445 172.21.15.89:0/573502486' entity='mgr.y' 2023-06-03T19:33:58.087 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:33:57 smithi164 bash[26855]: audit 2023-06-03T19:33:57.400876+0000 mon.a (mon.0) 646 : audit [INF] from='mgr.14445 172.21.15.89:0/573502486' entity='mgr.y' 2023-06-03T19:33:58.087 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:33:57 smithi164 bash[26855]: audit 2023-06-03T19:33:57.402285+0000 mon.a (mon.0) 647 : audit [DBG] from='mgr.14445 172.21.15.89:0/573502486' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-03T19:33:58.087 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:33:57 smithi164 bash[26855]: audit 2023-06-03T19:33:57.403583+0000 mon.a (mon.0) 648 : audit [INF] from='mgr.14445 172.21.15.89:0/573502486' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-03T19:33:58.087 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:33:57 smithi164 bash[26855]: audit 2023-06-03T19:33:57.409824+0000 mon.a (mon.0) 649 : audit [INF] from='mgr.14445 172.21.15.89:0/573502486' entity='mgr.y' 2023-06-03T19:33:58.087 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:33:57 smithi164 bash[26855]: audit 2023-06-03T19:33:57.417424+0000 mon.a (mon.0) 650 : audit [DBG] from='mgr.14445 172.21.15.89:0/573502486' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-03T19:33:58.473 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/mon.a/config 2023-06-03T19:34:00.031 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:33:59 smithi089 bash[33173]: cluster 2023-06-03T19:33:58.407361+0000 mgr.y (mgr.14445) 93 : cluster [DBG] pgmap v72: 9 pgs: 9 active+clean; 0 B data, 43 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:34:00.032 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:33:59 smithi089 bash[26572]: cluster 2023-06-03T19:33:58.407361+0000 mgr.y (mgr.14445) 93 : cluster [DBG] pgmap v72: 9 pgs: 9 active+clean; 0 B data, 43 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:34:00.086 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:33:59 smithi164 bash[26855]: cluster 2023-06-03T19:33:58.407361+0000 mgr.y (mgr.14445) 93 : cluster [DBG] pgmap v72: 9 pgs: 9 active+clean; 0 B data, 43 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:34:01.664 INFO:teuthology.orchestra.run.smithi089.stdout: 2023-06-03T19:34:01.669 INFO:teuthology.orchestra.run.smithi089.stderr:dumped all 2023-06-03T19:34:02.031 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:34:01 smithi089 bash[33173]: cluster 2023-06-03T19:34:00.407933+0000 mgr.y (mgr.14445) 94 : cluster [DBG] pgmap v73: 9 pgs: 9 active+clean; 0 B data, 43 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:34:02.032 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:34:01 smithi089 bash[26572]: cluster 2023-06-03T19:34:00.407933+0000 mgr.y (mgr.14445) 94 : cluster [DBG] pgmap v73: 9 pgs: 9 active+clean; 0 B data, 43 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:34:02.086 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:34:01 smithi164 bash[26855]: cluster 2023-06-03T19:34:00.407933+0000 mgr.y (mgr.14445) 94 : cluster [DBG] pgmap v73: 9 pgs: 9 active+clean; 0 B data, 43 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:34:02.434 INFO:teuthology.orchestra.run.smithi089.stdout:{"pg_ready":true,"pg_map":{"version":73,"stamp":"2023-06-03T19:34:00.407650+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":27,"acting":27,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":27,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":44208,"kb_used_data":4208,"kb_used_omap":0,"kb_used_meta":39936,"kb_avail":749917008,"statfs":{"total":767960285184,"available":767915016192,"internally_reserved":0,"allocated":4308992,"data_stored":1762828,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":40894464},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"0.000000"},"pg_stats":[{"pgid":"2.7","version":"0'0","reported_seq":11,"reported_epoch":54,"state":"active+clean","last_fresh":"2023-06-03T19:32:23.431446+0000","last_change":"2023-06-03T19:32:21.706362+0000","last_active":"2023-06-03T19:32:23.431446+0000","last_peered":"2023-06-03T19:32:23.431446+0000","last_clean":"2023-06-03T19:32:23.431446+0000","last_became_active":"2023-06-03T19:32:21.705855+0000","last_became_peered":"2023-06-03T19:32:21.705855+0000","last_unstale":"2023-06-03T19:32:23.431446+0000","last_undegraded":"2023-06-03T19:32:23.431446+0000","last_fullsized":"2023-06-03T19:32:23.431446+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:32:20.692043+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:32:20.692043+0000","last_clean_scrub_stamp":"2023-06-03T19:32:20.692043+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":11,"reported_epoch":54,"state":"active+clean","last_fresh":"2023-06-03T19:32:22.931122+0000","last_change":"2023-06-03T19:32:21.989748+0000","last_active":"2023-06-03T19:32:22.931122+0000","last_peered":"2023-06-03T19:32:22.931122+0000","last_clean":"2023-06-03T19:32:22.931122+0000","last_became_active":"2023-06-03T19:32:21.989310+0000","last_became_peered":"2023-06-03T19:32:21.989310+0000","last_unstale":"2023-06-03T19:32:22.931122+0000","last_undegraded":"2023-06-03T19:32:22.931122+0000","last_fullsized":"2023-06-03T19:32:22.931122+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:32:20.692043+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:32:20.692043+0000","last_clean_scrub_stamp":"2023-06-03T19:32:20.692043+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":11,"reported_epoch":54,"state":"active+clean","last_fresh":"2023-06-03T19:32:23.431073+0000","last_change":"2023-06-03T19:32:22.124191+0000","last_active":"2023-06-03T19:32:23.431073+0000","last_peered":"2023-06-03T19:32:23.431073+0000","last_clean":"2023-06-03T19:32:23.431073+0000","last_became_active":"2023-06-03T19:32:22.122933+0000","last_became_peered":"2023-06-03T19:32:22.122933+0000","last_unstale":"2023-06-03T19:32:23.431073+0000","last_undegraded":"2023-06-03T19:32:23.431073+0000","last_fullsized":"2023-06-03T19:32:23.431073+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:32:20.692043+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:32:20.692043+0000","last_clean_scrub_stamp":"2023-06-03T19:32:20.692043+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":11,"reported_epoch":54,"state":"active+clean","last_fresh":"2023-06-03T19:32:23.430935+0000","last_change":"2023-06-03T19:32:22.123780+0000","last_active":"2023-06-03T19:32:23.430935+0000","last_peered":"2023-06-03T19:32:23.430935+0000","last_clean":"2023-06-03T19:32:23.430935+0000","last_became_active":"2023-06-03T19:32:22.122852+0000","last_became_peered":"2023-06-03T19:32:22.122852+0000","last_unstale":"2023-06-03T19:32:23.430935+0000","last_undegraded":"2023-06-03T19:32:23.430935+0000","last_fullsized":"2023-06-03T19:32:23.430935+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:32:20.692043+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:32:20.692043+0000","last_clean_scrub_stamp":"2023-06-03T19:32:20.692043+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":15,"reported_epoch":54,"state":"active+clean","last_fresh":"2023-06-03T19:32:23.430917+0000","last_change":"2023-06-03T19:31:15.881174+0000","last_active":"2023-06-03T19:32:23.430917+0000","last_peered":"2023-06-03T19:32:23.430917+0000","last_clean":"2023-06-03T19:32:23.430917+0000","last_became_active":"2023-06-03T19:31:15.880415+0000","last_became_peered":"2023-06-03T19:31:15.880415+0000","last_unstale":"2023-06-03T19:32:23.430917+0000","last_undegraded":"2023-06-03T19:32:23.430917+0000","last_fullsized":"2023-06-03T19:32:23.430917+0000","mapping_epoch":49,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":50,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:27:28.100808+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:27:28.100808+0000","last_clean_scrub_stamp":"2023-06-03T19:27:28.100808+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":11,"reported_epoch":54,"state":"active+clean","last_fresh":"2023-06-03T19:32:23.344763+0000","last_change":"2023-06-03T19:32:21.707936+0000","last_active":"2023-06-03T19:32:23.344763+0000","last_peered":"2023-06-03T19:32:23.344763+0000","last_clean":"2023-06-03T19:32:23.344763+0000","last_became_active":"2023-06-03T19:32:21.707538+0000","last_became_peered":"2023-06-03T19:32:21.707538+0000","last_unstale":"2023-06-03T19:32:23.344763+0000","last_undegraded":"2023-06-03T19:32:23.344763+0000","last_fullsized":"2023-06-03T19:32:23.344763+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:32:20.692043+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:32:20.692043+0000","last_clean_scrub_stamp":"2023-06-03T19:32:20.692043+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":11,"reported_epoch":54,"state":"active+clean","last_fresh":"2023-06-03T19:32:22.716441+0000","last_change":"2023-06-03T19:32:22.123980+0000","last_active":"2023-06-03T19:32:22.716441+0000","last_peered":"2023-06-03T19:32:22.716441+0000","last_clean":"2023-06-03T19:32:22.716441+0000","last_became_active":"2023-06-03T19:32:22.123641+0000","last_became_peered":"2023-06-03T19:32:22.123641+0000","last_unstale":"2023-06-03T19:32:22.716441+0000","last_undegraded":"2023-06-03T19:32:22.716441+0000","last_fullsized":"2023-06-03T19:32:22.716441+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:32:20.692043+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:32:20.692043+0000","last_clean_scrub_stamp":"2023-06-03T19:32:20.692043+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":11,"reported_epoch":54,"state":"active+clean","last_fresh":"2023-06-03T19:32:23.344642+0000","last_change":"2023-06-03T19:32:21.709485+0000","last_active":"2023-06-03T19:32:23.344642+0000","last_peered":"2023-06-03T19:32:23.344642+0000","last_clean":"2023-06-03T19:32:23.344642+0000","last_became_active":"2023-06-03T19:32:21.708866+0000","last_became_peered":"2023-06-03T19:32:21.708866+0000","last_unstale":"2023-06-03T19:32:23.344642+0000","last_undegraded":"2023-06-03T19:32:23.344642+0000","last_fullsized":"2023-06-03T19:32:23.344642+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:32:20.692043+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:32:20.692043+0000","last_clean_scrub_stamp":"2023-06-03T19:32:20.692043+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":11,"reported_epoch":54,"state":"active+clean","last_fresh":"2023-06-03T19:32:22.931176+0000","last_change":"2023-06-03T19:32:22.125458+0000","last_active":"2023-06-03T19:32:22.931176+0000","last_peered":"2023-06-03T19:32:22.931176+0000","last_clean":"2023-06-03T19:32:22.931176+0000","last_became_active":"2023-06-03T19:32:22.124914+0000","last_became_peered":"2023-06-03T19:32:22.124914+0000","last_unstale":"2023-06-03T19:32:22.931176+0000","last_undegraded":"2023-06-03T19:32:22.931176+0000","last_fullsized":"2023-06-03T19:32:22.931176+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:32:20.692043+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:32:20.692043+0000","last_clean_scrub_stamp":"2023-06-03T19:32:20.692043+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":3,"acting":3,"num_store_stats":3}],"osd_stats":[{"osd":7,"up_from":48,"seq":206158430243,"num_pgs":6,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":5336,"kb_used_data":528,"kb_used_omap":0,"kb_used_meta":4800,"kb_avail":93739816,"statfs":{"total":95995035648,"available":95989571584,"internally_reserved":0,"allocated":540672,"data_stored":220511,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":4915200},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 3 19:33:21 2023","interfaces":[{"interface":"back","average":{"1min":0.83799999999999997,"5min":0.81299999999999994,"15min":0.80900000000000005},"min":{"1min":0.68000000000000005,"5min":0.57699999999999996,"15min":0.57699999999999996},"max":{"1min":1.1919999999999999,"5min":1.1919999999999999,"15min":1.1919999999999999},"last":0.97199999999999998},{"interface":"front","average":{"1min":0.94699999999999995,"5min":0.88600000000000001,"15min":0.876},"min":{"1min":0.72199999999999998,"5min":0.56499999999999995,"15min":0.56499999999999995},"max":{"1min":1.2569999999999999,"5min":1.2569999999999999,"15min":1.2569999999999999},"last":0.63800000000000001}]},{"osd":1,"last update":"Sat Jun 3 19:33:21 2023","interfaces":[{"interface":"back","average":{"1min":0.871,"5min":0.82899999999999996,"15min":0.82199999999999995},"min":{"1min":0.65300000000000002,"5min":0.53100000000000003,"15min":0.53100000000000003},"max":{"1min":1.1739999999999999,"5min":1.202,"15min":1.202},"last":0.73999999999999999},{"interface":"front","average":{"1min":0.95299999999999996,"5min":0.88300000000000001,"15min":0.871},"min":{"1min":0.76800000000000002,"5min":0.61699999999999999,"15min":0.61699999999999999},"max":{"1min":1.095,"5min":1.127,"15min":1.127},"last":0.86799999999999999}]},{"osd":2,"last update":"Sat Jun 3 19:33:21 2023","interfaces":[{"interface":"back","average":{"1min":0.90100000000000002,"5min":0.91300000000000003,"15min":0.91500000000000004},"min":{"1min":0.63300000000000001,"5min":0.63300000000000001,"15min":0.63300000000000001},"max":{"1min":1.21,"5min":1.538,"15min":1.538},"last":0.58999999999999997},{"interface":"front","average":{"1min":0.96899999999999997,"5min":0.88100000000000001,"15min":0.86699999999999999},"min":{"1min":0.61499999999999999,"5min":0.60699999999999998,"15min":0.60699999999999998},"max":{"1min":1.24,"5min":1.24,"15min":1.24},"last":0.70399999999999996}]},{"osd":3,"last update":"Sat Jun 3 19:33:21 2023","interfaces":[{"interface":"back","average":{"1min":0.95699999999999996,"5min":0.91600000000000004,"15min":0.90900000000000003},"min":{"1min":0.71299999999999997,"5min":0.53800000000000003,"15min":0.53800000000000003},"max":{"1min":1.1539999999999999,"5min":1.3069999999999999,"15min":1.3069999999999999},"last":0.91600000000000004},{"interface":"front","average":{"1min":0.92600000000000005,"5min":0.92900000000000005,"15min":0.92900000000000005},"min":{"1min":0.63500000000000001,"5min":0.57199999999999995,"15min":0.57199999999999995},"max":{"1min":1.143,"5min":1.5069999999999999,"15min":1.5069999999999999},"last":0.78300000000000003}]},{"osd":4,"last update":"Sat Jun 3 19:33:21 2023","interfaces":[{"interface":"back","average":{"1min":0.89200000000000002,"5min":0.75800000000000001,"15min":0.73599999999999999},"min":{"1min":0.57399999999999995,"5min":0.51200000000000001,"15min":0.51200000000000001},"max":{"1min":1.2829999999999999,"5min":1.2829999999999999,"15min":1.2829999999999999},"last":0.75900000000000001},{"interface":"front","average":{"1min":0.873,"5min":0.82899999999999996,"15min":0.82199999999999995},"min":{"1min":0.53300000000000003,"5min":0.45500000000000002,"15min":0.45500000000000002},"max":{"1min":1.3009999999999999,"5min":1.351,"15min":1.351},"last":0.80200000000000005}]},{"osd":5,"last update":"Sat Jun 3 19:33:21 2023","interfaces":[{"interface":"back","average":{"1min":0.90300000000000002,"5min":0.82699999999999996,"15min":0.81499999999999995},"min":{"1min":0.58399999999999996,"5min":0.48499999999999999,"15min":0.48499999999999999},"max":{"1min":1.3320000000000001,"5min":1.3320000000000001,"15min":1.3320000000000001},"last":0.85199999999999998},{"interface":"front","average":{"1min":0.98999999999999999,"5min":0.93799999999999994,"15min":0.93000000000000005},"min":{"1min":0.59299999999999997,"5min":0.437,"15min":0.437},"max":{"1min":1.5620000000000001,"5min":1.605,"15min":1.605},"last":0.94399999999999995}]},{"osd":6,"last update":"Sat Jun 3 19:33:21 2023","interfaces":[{"interface":"back","average":{"1min":1.083,"5min":0.98999999999999999,"15min":0.97399999999999998},"min":{"1min":0.66300000000000003,"5min":0.47899999999999998,"15min":0.47899999999999998},"max":{"1min":1.6080000000000001,"5min":1.6870000000000001,"15min":1.6870000000000001},"last":1.0029999999999999},{"interface":"front","average":{"1min":1.01,"5min":0.95299999999999996,"15min":0.94299999999999995},"min":{"1min":0.61799999999999999,"5min":0.59299999999999997,"15min":0.59299999999999997},"max":{"1min":1.4319999999999999,"5min":1.4319999999999999,"15min":1.4319999999999999},"last":1.02}]}]},{"osd":6,"up_from":42,"seq":180388626476,"num_pgs":4,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":5396,"kb_used_data":524,"kb_used_omap":0,"kb_used_meta":4864,"kb_avail":93739756,"statfs":{"total":95995035648,"available":95989510144,"internally_reserved":0,"allocated":536576,"data_stored":220196,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":4980736},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 3 19:33:34 2023","interfaces":[{"interface":"back","average":{"1min":0.84599999999999997,"5min":0.84099999999999997,"15min":0.82999999999999996},"min":{"1min":0.49099999999999999,"5min":0.49099999999999999,"15min":0.49099999999999999},"max":{"1min":1.2749999999999999,"5min":1.2749999999999999,"15min":1.2749999999999999},"last":0.81999999999999995},{"interface":"front","average":{"1min":0.84299999999999997,"5min":0.872,"15min":0.872},"min":{"1min":0.66200000000000003,"5min":0.57399999999999995,"15min":0.57399999999999995},"max":{"1min":1.087,"5min":1.284,"15min":1.284},"last":0.88600000000000001}]},{"osd":1,"last update":"Sat Jun 3 19:33:34 2023","interfaces":[{"interface":"back","average":{"1min":0.85799999999999998,"5min":0.875,"15min":0.87},"min":{"1min":0.50700000000000001,"5min":0.50700000000000001,"15min":0.50700000000000001},"max":{"1min":1.175,"5min":1.3759999999999999,"15min":1.3759999999999999},"last":0.85299999999999998},{"interface":"front","average":{"1min":0.872,"5min":0.93400000000000005,"15min":0.95199999999999996},"min":{"1min":0.56399999999999995,"5min":0.54100000000000004,"15min":0.54100000000000004},"max":{"1min":1.125,"5min":1.7769999999999999,"15min":1.7769999999999999},"last":0.65200000000000002}]},{"osd":2,"last update":"Sat Jun 3 19:33:34 2023","interfaces":[{"interface":"back","average":{"1min":0.85599999999999998,"5min":0.92000000000000004,"15min":0.92000000000000004},"min":{"1min":0.58499999999999996,"5min":0.58499999999999996,"15min":0.58499999999999996},"max":{"1min":1.2010000000000001,"5min":1.3069999999999999,"15min":1.3069999999999999},"last":0.73899999999999999},{"interface":"front","average":{"1min":0.93000000000000005,"5min":0.93400000000000005,"15min":0.93400000000000005},"min":{"1min":0.63900000000000001,"5min":0.63600000000000001,"15min":0.63600000000000001},"max":{"1min":1.2929999999999999,"5min":1.2929999999999999,"15min":1.2929999999999999},"last":1.0269999999999999}]},{"osd":3,"last update":"Sat Jun 3 19:33:34 2023","interfaces":[{"interface":"back","average":{"1min":0.90900000000000003,"5min":0.86199999999999999,"15min":0.84899999999999998},"min":{"1min":0.60199999999999998,"5min":0.60199999999999998,"15min":0.60199999999999998},"max":{"1min":1.2010000000000001,"5min":1.2290000000000001,"15min":1.2290000000000001},"last":1.071},{"interface":"front","average":{"1min":0.92600000000000005,"5min":0.98899999999999999,"15min":1.002},"min":{"1min":0.64200000000000002,"5min":0.64200000000000002,"15min":0.64200000000000002},"max":{"1min":1.1950000000000001,"5min":1.905,"15min":1.905},"last":0.94099999999999995}]},{"osd":4,"last update":"Sat Jun 3 19:33:34 2023","interfaces":[{"interface":"back","average":{"1min":0.93600000000000005,"5min":0.85499999999999998,"15min":0.84099999999999997},"min":{"1min":0.53600000000000003,"5min":0.46899999999999997,"15min":0.46899999999999997},"max":{"1min":1.2490000000000001,"5min":1.3140000000000001,"15min":1.3140000000000001},"last":0.98299999999999998},{"interface":"front","average":{"1min":0.93500000000000005,"5min":0.88100000000000001,"15min":0.871},"min":{"1min":0.61799999999999999,"5min":0.434,"15min":0.434},"max":{"1min":1.218,"5min":1.218,"15min":1.218},"last":1.129}]},{"osd":5,"last update":"Sat Jun 3 19:33:34 2023","interfaces":[{"interface":"back","average":{"1min":1.0429999999999999,"5min":0.93500000000000005,"15min":0.91900000000000004},"min":{"1min":0.60199999999999998,"5min":0.41599999999999998,"15min":0.41599999999999998},"max":{"1min":1.3109999999999999,"5min":1.3109999999999999,"15min":1.3109999999999999},"last":1.1040000000000001},{"interface":"front","average":{"1min":1.02,"5min":0.93300000000000005,"15min":0.93500000000000005},"min":{"1min":0.64700000000000002,"5min":0.52500000000000002,"15min":0.52500000000000002},"max":{"1min":1.2310000000000001,"5min":1.284,"15min":1.284},"last":0.96099999999999997}]},{"osd":7,"last update":"Sat Jun 3 19:33:20 2023","interfaces":[{"interface":"back","average":{"1min":1.081,"5min":0.96299999999999997,"15min":0.94299999999999995},"min":{"1min":0.626,"5min":0.45700000000000002,"15min":0.45700000000000002},"max":{"1min":1.3919999999999999,"5min":1.3919999999999999,"15min":1.3919999999999999},"last":1.1579999999999999},{"interface":"front","average":{"1min":1.0780000000000001,"5min":1.0109999999999999,"15min":1},"min":{"1min":0.63400000000000001,"5min":0.53600000000000003,"15min":0.53600000000000003},"max":{"1min":1.347,"5min":1.3839999999999999,"15min":1.3839999999999999},"last":1.05}]}]},{"osd":1,"up_from":13,"seq":55834574938,"num_pgs":4,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":5652,"kb_used_data":524,"kb_used_omap":0,"kb_used_meta":5120,"kb_avail":93739500,"statfs":{"total":95995035648,"available":95989248000,"internally_reserved":0,"allocated":536576,"data_stored":220196,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5242880},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 3 19:33:48 2023","interfaces":[{"interface":"back","average":{"1min":0.55600000000000005,"5min":0.67100000000000004,"15min":0.56899999999999995},"min":{"1min":0.34200000000000003,"5min":0.34200000000000003,"15min":0.34200000000000003},"max":{"1min":0.77900000000000003,"5min":1.123,"15min":1.123},"last":0.71699999999999997},{"interface":"front","average":{"1min":0.626,"5min":0.72199999999999998,"15min":0.61599999999999999},"min":{"1min":0.30399999999999999,"5min":0.30399999999999999,"15min":0.29699999999999999},"max":{"1min":1.075,"5min":1.1559999999999999,"15min":1.1559999999999999},"last":0.76200000000000001}]},{"osd":2,"last update":"Sat Jun 3 19:33:38 2023","interfaces":[{"interface":"back","average":{"1min":0.65800000000000003,"5min":0.73199999999999998,"15min":0.628},"min":{"1min":0.41299999999999998,"5min":0.40899999999999997,"15min":0.27600000000000002},"max":{"1min":0.92600000000000005,"5min":1.2030000000000001,"15min":1.2030000000000001},"last":0.83799999999999997},{"interface":"front","average":{"1min":0.624,"5min":0.749,"15min":0.66200000000000003},"min":{"1min":0.42599999999999999,"5min":0.42599999999999999,"15min":0.32100000000000001},"max":{"1min":0.94299999999999995,"5min":1.2150000000000001,"15min":1.2150000000000001},"last":0.86599999999999999}]},{"osd":3,"last update":"Sat Jun 3 19:33:23 2023","interfaces":[{"interface":"back","average":{"1min":0.68000000000000005,"5min":0.74399999999999999,"15min":0.73099999999999998},"min":{"1min":0.42399999999999999,"5min":0.42399999999999999,"15min":0.42399999999999999},"max":{"1min":1.022,"5min":1.1100000000000001,"15min":1.1100000000000001},"last":0.81000000000000005},{"interface":"front","average":{"1min":0.77800000000000002,"5min":0.81999999999999995,"15min":0.78000000000000003},"min":{"1min":0.40600000000000003,"5min":0.40600000000000003,"15min":0.40600000000000003},"max":{"1min":1.0649999999999999,"5min":1.264,"15min":1.264},"last":0.996}]},{"osd":4,"last update":"Sat Jun 3 19:33:04 2023","interfaces":[{"interface":"back","average":{"1min":0.89600000000000002,"5min":0.94299999999999995,"15min":0.94899999999999995},"min":{"1min":0.66700000000000004,"5min":0.66700000000000004,"15min":0.66700000000000004},"max":{"1min":1.26,"5min":1.2949999999999999,"15min":1.2949999999999999},"last":1.0269999999999999},{"interface":"front","average":{"1min":0.95099999999999996,"5min":0.93500000000000005,"15min":0.93300000000000005},"min":{"1min":0.65300000000000002,"5min":0.64500000000000002,"15min":0.64500000000000002},"max":{"1min":1.444,"5min":1.444,"15min":1.444},"last":1.0569999999999999}]},{"osd":5,"last update":"Sat Jun 3 19:33:48 2023","interfaces":[{"interface":"back","average":{"1min":0.93899999999999995,"5min":1.018,"15min":1.0389999999999999},"min":{"1min":0.68799999999999994,"5min":0.65200000000000002,"15min":0.65200000000000002},"max":{"1min":1.353,"5min":1.617,"15min":1.617},"last":1.1850000000000001},{"interface":"front","average":{"1min":0.96899999999999997,"5min":1.042,"15min":1.0720000000000001},"min":{"1min":0.69999999999999996,"5min":0.67900000000000005,"15min":0.67900000000000005},"max":{"1min":1.325,"5min":1.595,"15min":1.595},"last":1.353}]},{"osd":6,"last update":"Sat Jun 3 19:33:32 2023","interfaces":[{"interface":"back","average":{"1min":0.98299999999999998,"5min":1.016,"15min":1.0249999999999999},"min":{"1min":0.54500000000000004,"5min":0.54500000000000004,"15min":0.54500000000000004},"max":{"1min":1.4770000000000001,"5min":1.4770000000000001,"15min":1.4770000000000001},"last":1.208},{"interface":"front","average":{"1min":1.0169999999999999,"5min":1.0940000000000001,"15min":1.1120000000000001},"min":{"1min":0.76900000000000002,"5min":0.69499999999999995,"15min":0.69499999999999995},"max":{"1min":1.3859999999999999,"5min":1.3859999999999999,"15min":1.3859999999999999},"last":1.2589999999999999}]},{"osd":7,"last update":"Sat Jun 3 19:33:18 2023","interfaces":[{"interface":"back","average":{"1min":0.99399999999999999,"5min":1.0760000000000001,"15min":1.0900000000000001},"min":{"1min":0.63700000000000001,"5min":0.63700000000000001,"15min":0.63700000000000001},"max":{"1min":1.369,"5min":1.4810000000000001,"15min":1.4810000000000001},"last":1.302},{"interface":"front","average":{"1min":1.0469999999999999,"5min":1.149,"15min":1.1659999999999999},"min":{"1min":0.71199999999999997,"5min":0.71199999999999997,"15min":0.71199999999999997},"max":{"1min":1.54,"5min":1.54,"15min":1.54},"last":1.3129999999999999}]}]},{"osd":0,"up_from":8,"seq":34359738467,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":5716,"kb_used_data":524,"kb_used_omap":0,"kb_used_meta":5184,"kb_avail":93739436,"statfs":{"total":95995035648,"available":95989182464,"internally_reserved":0,"allocated":536576,"data_stored":220196,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5308416},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Jun 3 19:33:56 2023","interfaces":[{"interface":"back","average":{"1min":0.627,"5min":0.69699999999999995,"15min":0.58599999999999997},"min":{"1min":0.28799999999999998,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":1.1459999999999999,"5min":1.238,"15min":1.238},"last":0.92000000000000004},{"interface":"front","average":{"1min":0.61299999999999999,"5min":0.70199999999999996,"15min":0.59799999999999998},"min":{"1min":0.40500000000000003,"5min":0.40000000000000002,"15min":0.33600000000000002},"max":{"1min":0.90100000000000002,"5min":1.0860000000000001,"15min":1.0860000000000001},"last":0.73299999999999998}]},{"osd":2,"last update":"Sat Jun 3 19:33:36 2023","interfaces":[{"interface":"back","average":{"1min":0.66400000000000003,"5min":0.71799999999999997,"15min":0.65000000000000002},"min":{"1min":0.47399999999999998,"5min":0.35299999999999998,"15min":0.35299999999999998},"max":{"1min":0.94999999999999996,"5min":1.052,"15min":1.052},"last":0.67400000000000004},{"interface":"front","average":{"1min":0.69499999999999995,"5min":0.752,"15min":0.67100000000000004},"min":{"1min":0.39100000000000001,"5min":0.39100000000000001,"15min":0.39100000000000001},"max":{"1min":1.0980000000000001,"5min":1.1850000000000001,"15min":1.1850000000000001},"last":0.872}]},{"osd":3,"last update":"Sat Jun 3 19:33:28 2023","interfaces":[{"interface":"back","average":{"1min":0.66100000000000003,"5min":0.71299999999999997,"15min":0.73999999999999999},"min":{"1min":0.373,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":1.0169999999999999,"5min":1.0860000000000001,"15min":1.0860000000000001},"last":0.70799999999999996},{"interface":"front","average":{"1min":0.73999999999999999,"5min":0.82099999999999995,"15min":0.80200000000000005},"min":{"1min":0.437,"5min":0.437,"15min":0.437},"max":{"1min":1.1200000000000001,"5min":1.1639999999999999,"15min":1.1639999999999999},"last":0.75700000000000001}]},{"osd":4,"last update":"Sat Jun 3 19:33:00 2023","interfaces":[{"interface":"back","average":{"1min":0.93999999999999995,"5min":0.92300000000000004,"15min":0.92400000000000004},"min":{"1min":0.71499999999999997,"5min":0.54200000000000004,"15min":0.54200000000000004},"max":{"1min":1.425,"5min":1.77,"15min":1.77},"last":1.141},{"interface":"front","average":{"1min":0.97499999999999998,"5min":0.97099999999999997,"15min":0.93899999999999995},"min":{"1min":0.56599999999999995,"5min":0.56599999999999995,"15min":0.56599999999999995},"max":{"1min":1.4590000000000001,"5min":1.4590000000000001,"15min":1.4590000000000001},"last":1.343}]},{"osd":5,"last update":"Sat Jun 3 19:33:52 2023","interfaces":[{"interface":"back","average":{"1min":0.91100000000000003,"5min":1.0309999999999999,"15min":1.056},"min":{"1min":0.60199999999999998,"5min":0.52700000000000002,"15min":0.52700000000000002},"max":{"1min":1.5900000000000001,"5min":1.5900000000000001,"15min":1.5900000000000001},"last":1.4039999999999999},{"interface":"front","average":{"1min":0.90800000000000003,"5min":0.96399999999999997,"15min":0.97299999999999998},"min":{"1min":0.59299999999999997,"5min":0.42899999999999999,"15min":0.42899999999999999},"max":{"1min":1.5169999999999999,"5min":1.5169999999999999,"15min":1.5169999999999999},"last":1.089}]},{"osd":6,"last update":"Sat Jun 3 19:33:36 2023","interfaces":[{"interface":"back","average":{"1min":0.98499999999999999,"5min":1.0760000000000001,"15min":1.0900000000000001},"min":{"1min":0.66300000000000003,"5min":0.65300000000000002,"15min":0.65300000000000002},"max":{"1min":1.5660000000000001,"5min":1.615,"15min":1.615},"last":1.206},{"interface":"front","average":{"1min":0.98799999999999999,"5min":1.0449999999999999,"15min":1.0549999999999999},"min":{"1min":0.54400000000000004,"5min":0.54400000000000004,"15min":0.54400000000000004},"max":{"1min":1.421,"5min":1.4490000000000001,"15min":1.4490000000000001},"last":1.302}]},{"osd":7,"last update":"Sat Jun 3 19:33:22 2023","interfaces":[{"interface":"back","average":{"1min":1.022,"5min":1.0720000000000001,"15min":1.0800000000000001},"min":{"1min":0.629,"5min":0.59499999999999997,"15min":0.59499999999999997},"max":{"1min":1.456,"5min":1.514,"15min":1.514},"last":1.3680000000000001},{"interface":"front","average":{"1min":0.96199999999999997,"5min":1.028,"15min":1.0389999999999999},"min":{"1min":0.57899999999999996,"5min":0.57899999999999996,"15min":0.57899999999999996},"max":{"1min":1.526,"5min":1.526,"15min":1.526},"last":1.228}]}]},{"osd":2,"up_from":18,"seq":77309411408,"num_pgs":3,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":5592,"kb_used_data":528,"kb_used_omap":0,"kb_used_meta":5056,"kb_avail":93739560,"statfs":{"total":95995035648,"available":95989309440,"internally_reserved":0,"allocated":540672,"data_stored":220511,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5177344},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 3 19:33:37 2023","interfaces":[{"interface":"back","average":{"1min":0.63800000000000001,"5min":0.69099999999999995,"15min":0.61099999999999999},"min":{"1min":0.314,"5min":0.314,"15min":0.314},"max":{"1min":0.97699999999999998,"5min":1.1819999999999999,"15min":1.1819999999999999},"last":0.67400000000000004},{"interface":"front","average":{"1min":0.754,"5min":0.70899999999999996,"15min":0.627},"min":{"1min":0.35499999999999998,"5min":0.35499999999999998,"15min":0.30199999999999999},"max":{"1min":3.004,"5min":3.004,"15min":3.004},"last":0.70399999999999996}]},{"osd":1,"last update":"Sat Jun 3 19:33:37 2023","interfaces":[{"interface":"back","average":{"1min":0.69699999999999995,"5min":0.72699999999999998,"15min":0.63300000000000001},"min":{"1min":0.39100000000000001,"5min":0.39100000000000001,"15min":0.39100000000000001},"max":{"1min":1.1339999999999999,"5min":1.139,"15min":1.139},"last":0.64600000000000002},{"interface":"front","average":{"1min":0.71499999999999997,"5min":0.77200000000000002,"15min":0.66800000000000004},"min":{"1min":0.41199999999999998,"5min":0.39300000000000002,"15min":0.33500000000000002},"max":{"1min":1.05,"5min":1.161,"15min":1.161},"last":0.59699999999999998}]},{"osd":3,"last update":"Sat Jun 3 19:33:29 2023","interfaces":[{"interface":"back","average":{"1min":0.71399999999999997,"5min":0.77000000000000002,"15min":0.749},"min":{"1min":0.44700000000000001,"5min":0.44700000000000001,"15min":0.44700000000000001},"max":{"1min":0.90600000000000003,"5min":1.0469999999999999,"15min":1.0469999999999999},"last":0.54100000000000004},{"interface":"front","average":{"1min":0.77700000000000002,"5min":0.82099999999999995,"15min":0.77300000000000002},"min":{"1min":0.36299999999999999,"5min":0.34399999999999997,"15min":0.34399999999999997},"max":{"1min":1.085,"5min":1.5840000000000001,"15min":1.5840000000000001},"last":0.72999999999999998}]},{"osd":4,"last update":"Sat Jun 3 19:33:09 2023","interfaces":[{"interface":"back","average":{"1min":0.91900000000000004,"5min":0.94799999999999995,"15min":0.93300000000000005},"min":{"1min":0.625,"5min":0.61499999999999999,"15min":0.61499999999999999},"max":{"1min":1.266,"5min":1.536,"15min":1.536},"last":0.79800000000000004},{"interface":"front","average":{"1min":0.95199999999999996,"5min":1.0860000000000001,"15min":1.012},"min":{"1min":0.57799999999999996,"5min":0.57799999999999996,"15min":0.57799999999999996},"max":{"1min":1.3260000000000001,"5min":8.3829999999999991,"15min":8.3829999999999991},"last":0.82699999999999996}]},{"osd":5,"last update":"Sat Jun 3 19:33:50 2023","interfaces":[{"interface":"back","average":{"1min":0.94399999999999995,"5min":1.0720000000000001,"15min":1.087},"min":{"1min":0.59899999999999998,"5min":0.59899999999999998,"15min":0.59899999999999998},"max":{"1min":1.2929999999999999,"5min":1.7150000000000001,"15min":1.7150000000000001},"last":0.89800000000000002},{"interface":"front","average":{"1min":0.95599999999999996,"5min":0.97399999999999998,"15min":0.97299999999999998},"min":{"1min":0.59699999999999998,"5min":0.59699999999999998,"15min":0.59699999999999998},"max":{"1min":1.2829999999999999,"5min":1.369,"15min":1.369},"last":0.96199999999999997}]},{"osd":6,"last update":"Sat Jun 3 19:33:36 2023","interfaces":[{"interface":"back","average":{"1min":1.026,"5min":1.048,"15min":1.054},"min":{"1min":0.71799999999999997,"5min":0.71799999999999997,"15min":0.71799999999999997},"max":{"1min":1.3700000000000001,"5min":1.446,"15min":1.446},"last":0.86699999999999999},{"interface":"front","average":{"1min":1.026,"5min":1.1279999999999999,"15min":1.1459999999999999},"min":{"1min":0.58799999999999997,"5min":0.58799999999999997,"15min":0.58799999999999997},"max":{"1min":1.2689999999999999,"5min":1.5940000000000001,"15min":1.5940000000000001},"last":1.0820000000000001}]},{"osd":7,"last update":"Sat Jun 3 19:33:16 2023","interfaces":[{"interface":"back","average":{"1min":0.98499999999999999,"5min":1.085,"15min":1.101},"min":{"1min":0.68999999999999995,"5min":0.68999999999999995,"15min":0.68999999999999995},"max":{"1min":1.4219999999999999,"5min":1.486,"15min":1.486},"last":0.90900000000000003},{"interface":"front","average":{"1min":1.0700000000000001,"5min":1.135,"15min":1.1459999999999999},"min":{"1min":0.69899999999999995,"5min":0.69899999999999995,"15min":0.69899999999999995},"max":{"1min":1.3660000000000001,"5min":1.5600000000000001,"15min":1.5600000000000001},"last":0.92400000000000004}]}]},{"osd":3,"up_from":25,"seq":107374182471,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":5528,"kb_used_data":528,"kb_used_omap":0,"kb_used_meta":4992,"kb_avail":93739624,"statfs":{"total":95995035648,"available":95989374976,"internally_reserved":0,"allocated":540672,"data_stored":220511,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5111808},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 3 19:33:23 2023","interfaces":[{"interface":"back","average":{"1min":0.67900000000000005,"5min":0.69199999999999995,"15min":0.67400000000000004},"min":{"1min":0.45700000000000002,"5min":0.377,"15min":0.377},"max":{"1min":0.93999999999999995,"5min":1.153,"15min":1.153},"last":0.65300000000000002},{"interface":"front","average":{"1min":0.82299999999999995,"5min":0.76000000000000001,"15min":0.73899999999999999},"min":{"1min":0.41899999999999998,"5min":0.41899999999999998,"15min":0.41899999999999998},"max":{"1min":1.8979999999999999,"5min":1.8979999999999999,"15min":1.8979999999999999},"last":0.71099999999999997}]},{"osd":1,"last update":"Sat Jun 3 19:33:23 2023","interfaces":[{"interface":"back","average":{"1min":0.73299999999999998,"5min":0.753,"15min":0.72299999999999998},"min":{"1min":0.47999999999999998,"5min":0.439,"15min":0.439},"max":{"1min":1.044,"5min":1.218,"15min":1.218},"last":0.81399999999999995},{"interface":"front","average":{"1min":0.70799999999999996,"5min":0.76100000000000001,"15min":0.72899999999999998},"min":{"1min":0.53700000000000003,"5min":0.45000000000000001,"15min":0.45000000000000001},"max":{"1min":1.109,"5min":1.109,"15min":1.109},"last":0.68000000000000005}]},{"osd":2,"last update":"Sat Jun 3 19:33:23 2023","interfaces":[{"interface":"back","average":{"1min":0.80900000000000005,"5min":0.80500000000000005,"15min":0.77700000000000002},"min":{"1min":0.48899999999999999,"5min":0.48899999999999999,"15min":0.48899999999999999},"max":{"1min":1.1899999999999999,"5min":1.359,"15min":1.359},"last":0.77200000000000002},{"interface":"front","average":{"1min":0.72899999999999998,"5min":0.76200000000000001,"15min":0.755},"min":{"1min":0.504,"5min":0.42699999999999999,"15min":0.42699999999999999},"max":{"1min":1.091,"5min":1.355,"15min":1.355},"last":0.74399999999999999}]},{"osd":4,"last update":"Sat Jun 3 19:33:03 2023","interfaces":[{"interface":"back","average":{"1min":0.97199999999999998,"5min":0.95899999999999996,"15min":0.95399999999999996},"min":{"1min":0.74399999999999999,"5min":0.59499999999999997,"15min":0.59499999999999997},"max":{"1min":1.3320000000000001,"5min":1.393,"15min":1.393},"last":1.02},{"interface":"front","average":{"1min":1,"5min":1.0009999999999999,"15min":0.98999999999999999},"min":{"1min":0.83399999999999996,"5min":0.81200000000000006,"15min":0.81200000000000006},"max":{"1min":1.2969999999999999,"5min":1.528,"15min":1.528},"last":1.071}]},{"osd":5,"last update":"Sat Jun 3 19:33:51 2023","interfaces":[{"interface":"back","average":{"1min":0.98399999999999999,"5min":1.0229999999999999,"15min":1.054},"min":{"1min":0.73399999999999999,"5min":0.69299999999999995,"15min":0.69299999999999995},"max":{"1min":1.272,"5min":1.48,"15min":1.48},"last":1.0980000000000001},{"interface":"front","average":{"1min":0.997,"5min":1.0449999999999999,"15min":1.073},"min":{"1min":0.70699999999999996,"5min":0.68400000000000005,"15min":0.68400000000000005},"max":{"1min":1.2170000000000001,"5min":1.452,"15min":1.452},"last":1.0329999999999999}]},{"osd":6,"last update":"Sat Jun 3 19:33:34 2023","interfaces":[{"interface":"back","average":{"1min":1.0329999999999999,"5min":1.0920000000000001,"15min":1.113},"min":{"1min":0.81100000000000005,"5min":0.745,"15min":0.745},"max":{"1min":1.2789999999999999,"5min":1.5029999999999999,"15min":1.5029999999999999},"last":0.93100000000000005},{"interface":"front","average":{"1min":0.98399999999999999,"5min":1.075,"15min":1.105},"min":{"1min":0.66900000000000004,"5min":0.66900000000000004,"15min":0.66900000000000004},"max":{"1min":1.369,"5min":1.474,"15min":1.474},"last":0.94299999999999995}]},{"osd":7,"last update":"Sat Jun 3 19:33:19 2023","interfaces":[{"interface":"back","average":{"1min":1.0640000000000001,"5min":1.079,"15min":1.081},"min":{"1min":0.68700000000000006,"5min":0.66800000000000004,"15min":0.66800000000000004},"max":{"1min":1.643,"5min":1.643,"15min":1.643},"last":1.1200000000000001},{"interface":"front","average":{"1min":1.123,"5min":1.0960000000000001,"15min":1.0920000000000001},"min":{"1min":0.64900000000000002,"5min":0.64900000000000002,"15min":0.64900000000000002},"max":{"1min":1.52,"5min":1.52,"15min":1.52},"last":0.98599999999999999}]}]},{"osd":4,"up_from":30,"seq":128849018943,"num_pgs":2,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":5528,"kb_used_data":528,"kb_used_omap":0,"kb_used_meta":4992,"kb_avail":93739624,"statfs":{"total":95995035648,"available":95989374976,"internally_reserved":0,"allocated":540672,"data_stored":220511,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5111808},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 3 19:33:06 2023","interfaces":[{"interface":"back","average":{"1min":0.91000000000000003,"5min":0.86599999999999999,"15min":0.86699999999999999},"min":{"1min":0.64600000000000002,"5min":0.55300000000000005,"15min":0.55300000000000005},"max":{"1min":1.234,"5min":1.234,"15min":1.234},"last":1.1579999999999999},{"interface":"front","average":{"1min":0.92700000000000005,"5min":0.87,"15min":0.876},"min":{"1min":0.56999999999999995,"5min":0.46100000000000002,"15min":0.46100000000000002},"max":{"1min":1.218,"5min":1.218,"15min":1.218},"last":0.92000000000000004}]},{"osd":1,"last update":"Sat Jun 3 19:33:06 2023","interfaces":[{"interface":"back","average":{"1min":0.86299999999999999,"5min":0.86799999999999999,"15min":0.86599999999999999},"min":{"1min":0.61599999999999999,"5min":0.502,"15min":0.502},"max":{"1min":1.161,"5min":1.179,"15min":1.179},"last":0.76800000000000002},{"interface":"front","average":{"1min":0.90900000000000003,"5min":0.85799999999999998,"15min":0.85099999999999998},"min":{"1min":0.66600000000000004,"5min":0.47599999999999998,"15min":0.47599999999999998},"max":{"1min":1.2430000000000001,"5min":1.2430000000000001,"15min":1.2430000000000001},"last":1.0980000000000001}]},{"osd":2,"last update":"Sat Jun 3 19:33:06 2023","interfaces":[{"interface":"back","average":{"1min":0.94199999999999995,"5min":0.89400000000000002,"15min":0.88400000000000001},"min":{"1min":0.68300000000000005,"5min":0.65100000000000002,"15min":0.65100000000000002},"max":{"1min":1.1559999999999999,"5min":1.2190000000000001,"15min":1.2190000000000001},"last":0.94699999999999995},{"interface":"front","average":{"1min":0.85699999999999998,"5min":0.88100000000000001,"15min":0.88700000000000001},"min":{"1min":0.67300000000000004,"5min":0.60299999999999998,"15min":0.60299999999999998},"max":{"1min":1.0960000000000001,"5min":1.1679999999999999,"15min":1.1679999999999999},"last":0.74199999999999999}]},{"osd":3,"last update":"Sat Jun 3 19:33:06 2023","interfaces":[{"interface":"back","average":{"1min":0.88500000000000001,"5min":0.87,"15min":0.872},"min":{"1min":0.64000000000000001,"5min":0.56699999999999995,"15min":0.56699999999999995},"max":{"1min":1.071,"5min":1.2749999999999999,"15min":1.2749999999999999},"last":0.98499999999999999},{"interface":"front","average":{"1min":0.91900000000000004,"5min":0.93300000000000005,"15min":0.93999999999999995},"min":{"1min":0.67100000000000004,"5min":0.64300000000000002,"15min":0.64300000000000002},"max":{"1min":1.2669999999999999,"5min":1.2669999999999999,"15min":1.2669999999999999},"last":0.68400000000000005}]},{"osd":5,"last update":"Sat Jun 3 19:33:48 2023","interfaces":[{"interface":"back","average":{"1min":0.85099999999999998,"5min":0.79300000000000004,"15min":0.76300000000000001},"min":{"1min":0.52000000000000002,"5min":0.45500000000000002,"15min":0.45500000000000002},"max":{"1min":1.214,"5min":1.214,"15min":1.214},"last":1.014},{"interface":"front","average":{"1min":0.86699999999999999,"5min":0.84299999999999997,"15min":0.82699999999999996},"min":{"1min":0.46800000000000003,"5min":0.45500000000000002,"15min":0.45500000000000002},"max":{"1min":1.2769999999999999,"5min":1.298,"15min":1.298},"last":0.71399999999999997}]},{"osd":6,"last update":"Sat Jun 3 19:33:33 2023","interfaces":[{"interface":"back","average":{"1min":0.93000000000000005,"5min":0.91900000000000004,"15min":0.93500000000000005},"min":{"1min":0.53200000000000003,"5min":0.46899999999999997,"15min":0.46899999999999997},"max":{"1min":1.1930000000000001,"5min":1.226,"15min":1.226},"last":0.84599999999999997},{"interface":"front","average":{"1min":0.98899999999999999,"5min":0.93400000000000005,"15min":0.93999999999999995},"min":{"1min":0.65200000000000002,"5min":0.503,"15min":0.503},"max":{"1min":1.288,"5min":1.288,"15min":1.288},"last":1.1870000000000001}]},{"osd":7,"last update":"Sat Jun 3 19:33:17 2023","interfaces":[{"interface":"back","average":{"1min":1.0449999999999999,"5min":0.91000000000000003,"15min":0.88800000000000001},"min":{"1min":0.77600000000000002,"5min":0.47099999999999997,"15min":0.47099999999999997},"max":{"1min":1.3480000000000001,"5min":1.3480000000000001,"15min":1.3480000000000001},"last":1.1319999999999999},{"interface":"front","average":{"1min":1.0900000000000001,"5min":0.94299999999999995,"15min":0.91900000000000004},"min":{"1min":0.73299999999999998,"5min":0.52300000000000002,"15min":0.52300000000000002},"max":{"1min":1.2989999999999999,"5min":1.2989999999999999,"15min":1.2989999999999999},"last":1.073}]}]},{"osd":5,"up_from":36,"seq":154618822710,"num_pgs":2,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":5460,"kb_used_data":524,"kb_used_omap":0,"kb_used_meta":4928,"kb_avail":93739692,"statfs":{"total":95995035648,"available":95989444608,"internally_reserved":0,"allocated":536576,"data_stored":220196,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5046272},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 3 19:33:51 2023","interfaces":[{"interface":"back","average":{"1min":0.877,"5min":0.873,"15min":0.88300000000000001},"min":{"1min":0.69699999999999995,"5min":0.51400000000000001,"15min":0.51400000000000001},"max":{"1min":1.1399999999999999,"5min":1.4750000000000001,"15min":1.4750000000000001},"last":0.72799999999999998},{"interface":"front","average":{"1min":0.76600000000000001,"5min":0.85199999999999998,"15min":0.84199999999999997},"min":{"1min":0.55200000000000005,"5min":0.55200000000000005,"15min":0.55200000000000005},"max":{"1min":1.0369999999999999,"5min":1.9259999999999999,"15min":1.9259999999999999},"last":1.107}]},{"osd":1,"last update":"Sat Jun 3 19:33:51 2023","interfaces":[{"interface":"back","average":{"1min":0.86399999999999999,"5min":0.88700000000000001,"15min":0.88900000000000001},"min":{"1min":0.60699999999999998,"5min":0.49399999999999999,"15min":0.49399999999999999},"max":{"1min":1.1499999999999999,"5min":1.994,"15min":1.994},"last":1.0249999999999999},{"interface":"front","average":{"1min":0.86699999999999999,"5min":0.89100000000000001,"15min":0.86599999999999999},"min":{"1min":0.68000000000000005,"5min":0.55400000000000005,"15min":0.55400000000000005},"max":{"1min":1.143,"5min":1.742,"15min":1.742},"last":0.97899999999999998}]},{"osd":2,"last update":"Sat Jun 3 19:33:51 2023","interfaces":[{"interface":"back","average":{"1min":0.80100000000000005,"5min":0.872,"15min":0.87},"min":{"1min":0.61699999999999999,"5min":0.54200000000000004,"15min":0.54200000000000004},"max":{"1min":1.071,"5min":1.708,"15min":1.708},"last":1.0680000000000001},{"interface":"front","average":{"1min":0.85799999999999998,"5min":0.91300000000000003,"15min":0.89500000000000002},"min":{"1min":0.60499999999999998,"5min":0.56000000000000005,"15min":0.56000000000000005},"max":{"1min":1.0700000000000001,"5min":2.0840000000000001,"15min":2.0840000000000001},"last":0.84499999999999997}]},{"osd":3,"last update":"Sat Jun 3 19:33:51 2023","interfaces":[{"interface":"back","average":{"1min":0.90300000000000002,"5min":0.94299999999999995,"15min":0.95499999999999996},"min":{"1min":0.65300000000000002,"5min":0.58199999999999996,"15min":0.58199999999999996},"max":{"1min":1.145,"5min":2.1699999999999999,"15min":2.1699999999999999},"last":0.91300000000000003},{"interface":"front","average":{"1min":0.90300000000000002,"5min":0.89400000000000002,"15min":0.86799999999999999},"min":{"1min":0.68100000000000005,"5min":0.57899999999999996,"15min":0.57899999999999996},"max":{"1min":1.2010000000000001,"5min":1.6459999999999999,"15min":1.6459999999999999},"last":1.173}]},{"osd":4,"last update":"Sat Jun 3 19:33:51 2023","interfaces":[{"interface":"back","average":{"1min":0.84599999999999997,"5min":0.82299999999999995,"15min":0.77700000000000002},"min":{"1min":0.55700000000000005,"5min":0.39100000000000001,"15min":0.39100000000000001},"max":{"1min":1.2370000000000001,"5min":2.1299999999999999,"15min":2.1299999999999999},"last":1.1419999999999999},{"interface":"front","average":{"1min":0.872,"5min":0.83699999999999997,"15min":0.76400000000000001},"min":{"1min":0.48499999999999999,"5min":0.38,"15min":0.38},"max":{"1min":1.2210000000000001,"5min":2.0830000000000002,"15min":2.0830000000000002},"last":1.246}]},{"osd":6,"last update":"Sat Jun 3 19:33:35 2023","interfaces":[{"interface":"back","average":{"1min":0.96999999999999997,"5min":0.92200000000000004,"15min":0.90600000000000003},"min":{"1min":0.53600000000000003,"5min":0.46500000000000002,"15min":0.46500000000000002},"max":{"1min":1.2050000000000001,"5min":1.6599999999999999,"15min":1.6599999999999999},"last":1.1919999999999999},{"interface":"front","average":{"1min":1.052,"5min":0.998,"15min":0.97599999999999998},"min":{"1min":0.65900000000000003,"5min":0.56200000000000006,"15min":0.56200000000000006},"max":{"1min":1.2789999999999999,"5min":1.8380000000000001,"15min":1.8380000000000001},"last":1.296}]},{"osd":7,"last update":"Sat Jun 3 19:33:18 2023","interfaces":[{"interface":"back","average":{"1min":1.048,"5min":1.0549999999999999,"15min":1.056},"min":{"1min":0.63200000000000001,"5min":0.63200000000000001,"15min":0.63200000000000001},"max":{"1min":1.292,"5min":2.004,"15min":2.004},"last":1.345},{"interface":"front","average":{"1min":0.94599999999999995,"5min":0.94999999999999996,"15min":0.94999999999999996},"min":{"1min":0.52700000000000002,"5min":0.48099999999999998,"15min":0.48099999999999998},"max":{"1min":1.2509999999999999,"5min":1.6479999999999999,"15min":1.6479999999999999},"last":1.268}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-06-03T19:34:02.437 INFO:tasks.cephadm.ceph_manager.ceph:clean! 2023-06-03T19:34:02.437 INFO:tasks.ceph:Waiting until ceph cluster ceph is healthy... 2023-06-03T19:34:02.437 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy 2023-06-03T19:34:02.438 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph health --format=json 2023-06-03T19:34:03.031 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:34:02 smithi089 bash[26572]: audit 2023-06-03T19:34:01.647873+0000 mgr.y (mgr.14445) 95 : audit [DBG] from='client.14622 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-06-03T19:34:03.032 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:34:02 smithi089 bash[33173]: audit 2023-06-03T19:34:01.647873+0000 mgr.y (mgr.14445) 95 : audit [DBG] from='client.14622 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-06-03T19:34:03.086 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:34:02 smithi164 bash[26855]: audit 2023-06-03T19:34:01.647873+0000 mgr.y (mgr.14445) 95 : audit [DBG] from='client.14622 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-06-03T19:34:04.031 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:34:03 smithi089 bash[26572]: cluster 2023-06-03T19:34:02.408528+0000 mgr.y (mgr.14445) 96 : cluster [DBG] pgmap v74: 9 pgs: 9 active+clean; 0 B data, 43 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:34:04.032 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:34:03 smithi089 bash[26572]: audit 2023-06-03T19:34:03.446090+0000 mon.a (mon.0) 651 : audit [INF] from='mgr.14445 172.21.15.89:0/573502486' entity='mgr.y' 2023-06-03T19:34:04.032 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:34:03 smithi089 bash[26572]: audit 2023-06-03T19:34:03.447722+0000 mon.a (mon.0) 652 : audit [DBG] from='mgr.14445 172.21.15.89:0/573502486' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-03T19:34:04.032 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:34:03 smithi089 bash[26572]: audit 2023-06-03T19:34:03.449218+0000 mon.a (mon.0) 653 : audit [INF] from='mgr.14445 172.21.15.89:0/573502486' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-03T19:34:04.032 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:34:03 smithi089 bash[26572]: audit 2023-06-03T19:34:03.457503+0000 mon.a (mon.0) 654 : audit [INF] from='mgr.14445 172.21.15.89:0/573502486' entity='mgr.y' 2023-06-03T19:34:04.032 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:34:03 smithi089 bash[33173]: cluster 2023-06-03T19:34:02.408528+0000 mgr.y (mgr.14445) 96 : cluster [DBG] pgmap v74: 9 pgs: 9 active+clean; 0 B data, 43 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:34:04.033 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:34:03 smithi089 bash[33173]: audit 2023-06-03T19:34:03.446090+0000 mon.a (mon.0) 651 : audit [INF] from='mgr.14445 172.21.15.89:0/573502486' entity='mgr.y' 2023-06-03T19:34:04.033 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:34:03 smithi089 bash[33173]: audit 2023-06-03T19:34:03.447722+0000 mon.a (mon.0) 652 : audit [DBG] from='mgr.14445 172.21.15.89:0/573502486' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-03T19:34:04.033 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:34:03 smithi089 bash[33173]: audit 2023-06-03T19:34:03.449218+0000 mon.a (mon.0) 653 : audit [INF] from='mgr.14445 172.21.15.89:0/573502486' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-03T19:34:04.033 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:34:03 smithi089 bash[33173]: audit 2023-06-03T19:34:03.457503+0000 mon.a (mon.0) 654 : audit [INF] from='mgr.14445 172.21.15.89:0/573502486' entity='mgr.y' 2023-06-03T19:34:04.086 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:34:03 smithi164 bash[26855]: cluster 2023-06-03T19:34:02.408528+0000 mgr.y (mgr.14445) 96 : cluster [DBG] pgmap v74: 9 pgs: 9 active+clean; 0 B data, 43 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:34:04.087 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:34:03 smithi164 bash[26855]: audit 2023-06-03T19:34:03.446090+0000 mon.a (mon.0) 651 : audit [INF] from='mgr.14445 172.21.15.89:0/573502486' entity='mgr.y' 2023-06-03T19:34:04.087 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:34:03 smithi164 bash[26855]: audit 2023-06-03T19:34:03.447722+0000 mon.a (mon.0) 652 : audit [DBG] from='mgr.14445 172.21.15.89:0/573502486' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-03T19:34:04.087 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:34:03 smithi164 bash[26855]: audit 2023-06-03T19:34:03.449218+0000 mon.a (mon.0) 653 : audit [INF] from='mgr.14445 172.21.15.89:0/573502486' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-03T19:34:04.087 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:34:03 smithi164 bash[26855]: audit 2023-06-03T19:34:03.457503+0000 mon.a (mon.0) 654 : audit [INF] from='mgr.14445 172.21.15.89:0/573502486' entity='mgr.y' 2023-06-03T19:34:06.031 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:34:05 smithi089 bash[33173]: cluster 2023-06-03T19:34:04.409252+0000 mgr.y (mgr.14445) 97 : cluster [DBG] pgmap v75: 9 pgs: 9 active+clean; 0 B data, 43 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:34:06.032 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:34:05 smithi089 bash[26572]: cluster 2023-06-03T19:34:04.409252+0000 mgr.y (mgr.14445) 97 : cluster [DBG] pgmap v75: 9 pgs: 9 active+clean; 0 B data, 43 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:34:06.086 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:34:05 smithi164 bash[26855]: cluster 2023-06-03T19:34:04.409252+0000 mgr.y (mgr.14445) 97 : cluster [DBG] pgmap v75: 9 pgs: 9 active+clean; 0 B data, 43 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:34:06.087 INFO:journalctl@ceph.mgr.x.smithi164.stdout:Jun 03 19:34:05 smithi164 bash[27934]: ::ffff:172.21.15.164 - - [03/Jun/2023:19:34:05] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2023-06-03T19:34:07.243 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/mon.a/config 2023-06-03T19:34:08.031 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:34:07 smithi089 bash[26572]: cluster 2023-06-03T19:34:06.409844+0000 mgr.y (mgr.14445) 98 : cluster [DBG] pgmap v76: 9 pgs: 9 active+clean; 0 B data, 43 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:34:08.032 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:34:07 smithi089 bash[33173]: cluster 2023-06-03T19:34:06.409844+0000 mgr.y (mgr.14445) 98 : cluster [DBG] pgmap v76: 9 pgs: 9 active+clean; 0 B data, 43 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:34:08.032 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 19:34:07 smithi089 bash[26845]: ::ffff:172.21.15.164 - - [03/Jun/2023:19:34:07] "GET /metrics HTTP/1.1" 200 152020 "" "Prometheus/2.33.4" 2023-06-03T19:34:08.086 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:34:07 smithi164 bash[26855]: cluster 2023-06-03T19:34:06.409844+0000 mgr.y (mgr.14445) 98 : cluster [DBG] pgmap v76: 9 pgs: 9 active+clean; 0 B data, 43 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:34:10.031 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:34:09 smithi089 bash[33173]: cluster 2023-06-03T19:34:08.410308+0000 mgr.y (mgr.14445) 99 : cluster [DBG] pgmap v77: 9 pgs: 9 active+clean; 0 B data, 43 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:34:10.032 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:34:09 smithi089 bash[26572]: cluster 2023-06-03T19:34:08.410308+0000 mgr.y (mgr.14445) 99 : cluster [DBG] pgmap v77: 9 pgs: 9 active+clean; 0 B data, 43 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:34:10.086 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:34:09 smithi164 bash[26855]: cluster 2023-06-03T19:34:08.410308+0000 mgr.y (mgr.14445) 99 : cluster [DBG] pgmap v77: 9 pgs: 9 active+clean; 0 B data, 43 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:34:10.525 INFO:teuthology.orchestra.run.smithi089.stdout: 2023-06-03T19:34:10.956 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:34:10 smithi089 bash[26572]: audit 2023-06-03T19:34:10.517608+0000 mon.a (mon.0) 655 : audit [DBG] from='client.? 172.21.15.89:0/1529694183' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2023-06-03T19:34:11.031 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:34:10 smithi089 bash[33173]: audit 2023-06-03T19:34:10.517608+0000 mon.a (mon.0) 655 : audit [DBG] from='client.? 172.21.15.89:0/1529694183' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2023-06-03T19:34:11.086 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:34:10 smithi164 bash[26855]: audit 2023-06-03T19:34:10.517608+0000 mon.a (mon.0) 655 : audit [DBG] from='client.? 172.21.15.89:0/1529694183' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2023-06-03T19:34:11.360 INFO:teuthology.orchestra.run.smithi089.stdout:{"status":"HEALTH_OK","checks":{},"mutes":[]} 2023-06-03T19:34:11.361 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy done 2023-06-03T19:34:11.361 INFO:tasks.cephadm:Setup complete, yielding 2023-06-03T19:34:11.361 INFO:teuthology.run_tasks:Running task cephadm.shell... 2023-06-03T19:34:11.371 INFO:tasks.cephadm:Running commands on role mon.a host ubuntu@smithi089.front.sepia.ceph.com 2023-06-03T19:34:11.372 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- bash -c 'ceph fs volume create foo' 2023-06-03T19:34:12.031 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:34:11 smithi089 bash[26572]: cluster 2023-06-03T19:34:10.410964+0000 mgr.y (mgr.14445) 100 : cluster [DBG] pgmap v78: 9 pgs: 9 active+clean; 0 B data, 43 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:34:12.032 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:34:11 smithi089 bash[33173]: cluster 2023-06-03T19:34:10.410964+0000 mgr.y (mgr.14445) 100 : cluster [DBG] pgmap v78: 9 pgs: 9 active+clean; 0 B data, 43 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:34:12.086 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:34:11 smithi164 bash[26855]: cluster 2023-06-03T19:34:10.410964+0000 mgr.y (mgr.14445) 100 : cluster [DBG] pgmap v78: 9 pgs: 9 active+clean; 0 B data, 43 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:34:14.031 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:34:13 smithi089 bash[26572]: cluster 2023-06-03T19:34:12.411498+0000 mgr.y (mgr.14445) 101 : cluster [DBG] pgmap v79: 9 pgs: 9 active+clean; 0 B data, 43 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:34:14.032 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:34:13 smithi089 bash[33173]: cluster 2023-06-03T19:34:12.411498+0000 mgr.y (mgr.14445) 101 : cluster [DBG] pgmap v79: 9 pgs: 9 active+clean; 0 B data, 43 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:34:14.086 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:34:13 smithi164 bash[26855]: cluster 2023-06-03T19:34:12.411498+0000 mgr.y (mgr.14445) 101 : cluster [DBG] pgmap v79: 9 pgs: 9 active+clean; 0 B data, 43 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:34:16.031 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:34:15 smithi089 bash[33173]: cluster 2023-06-03T19:34:14.412144+0000 mgr.y (mgr.14445) 102 : cluster [DBG] pgmap v80: 9 pgs: 9 active+clean; 0 B data, 43 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:34:16.032 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:34:15 smithi089 bash[33173]: audit 2023-06-03T19:34:15.106784+0000 mon.a (mon.0) 656 : audit [INF] from='mgr.14445 172.21.15.89:0/573502486' entity='mgr.y' cmd=[{"prefix": "osd pool create", "pool": "cephfs.foo.meta"}]: dispatch 2023-06-03T19:34:16.032 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:34:15 smithi089 bash[26572]: cluster 2023-06-03T19:34:14.412144+0000 mgr.y (mgr.14445) 102 : cluster [DBG] pgmap v80: 9 pgs: 9 active+clean; 0 B data, 43 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:34:16.032 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:34:15 smithi089 bash[26572]: audit 2023-06-03T19:34:15.106784+0000 mon.a (mon.0) 656 : audit [INF] from='mgr.14445 172.21.15.89:0/573502486' entity='mgr.y' cmd=[{"prefix": "osd pool create", "pool": "cephfs.foo.meta"}]: dispatch 2023-06-03T19:34:16.086 INFO:journalctl@ceph.mgr.x.smithi164.stdout:Jun 03 19:34:15 smithi164 bash[27934]: ::ffff:172.21.15.164 - - [03/Jun/2023:19:34:15] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2023-06-03T19:34:16.087 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:34:15 smithi164 bash[26855]: cluster 2023-06-03T19:34:14.412144+0000 mgr.y (mgr.14445) 102 : cluster [DBG] pgmap v80: 9 pgs: 9 active+clean; 0 B data, 43 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:34:16.087 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:34:15 smithi164 bash[26855]: audit 2023-06-03T19:34:15.106784+0000 mon.a (mon.0) 656 : audit [INF] from='mgr.14445 172.21.15.89:0/573502486' entity='mgr.y' cmd=[{"prefix": "osd pool create", "pool": "cephfs.foo.meta"}]: dispatch 2023-06-03T19:34:17.086 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:34:16 smithi164 bash[26855]: audit 2023-06-03T19:34:15.105957+0000 mgr.y (mgr.14445) 103 : audit [DBG] from='client.14634 -' entity='client.admin' cmd=[{"prefix": "fs volume create", "name": "foo", "target": ["mon-mgr", ""]}]: dispatch 2023-06-03T19:34:17.086 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:34:16 smithi164 bash[26855]: audit 2023-06-03T19:34:15.780798+0000 mon.a (mon.0) 657 : audit [INF] from='mgr.14445 172.21.15.89:0/573502486' entity='mgr.y' cmd='[{"prefix": "osd pool create", "pool": "cephfs.foo.meta"}]': finished 2023-06-03T19:34:17.087 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:34:16 smithi164 bash[26855]: cluster 2023-06-03T19:34:15.780916+0000 mon.a (mon.0) 658 : cluster [DBG] osdmap e56: 8 total, 8 up, 8 in 2023-06-03T19:34:17.087 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:34:16 smithi164 bash[26855]: audit 2023-06-03T19:34:15.785652+0000 mon.a (mon.0) 659 : audit [INF] from='mgr.14445 172.21.15.89:0/573502486' entity='mgr.y' cmd=[{"prefix": "osd pool create", "pool": "cephfs.foo.data"}]: dispatch 2023-06-03T19:34:17.281 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:34:16 smithi089 bash[26572]: audit 2023-06-03T19:34:15.105957+0000 mgr.y (mgr.14445) 103 : audit [DBG] from='client.14634 -' entity='client.admin' cmd=[{"prefix": "fs volume create", "name": "foo", "target": ["mon-mgr", ""]}]: dispatch 2023-06-03T19:34:17.281 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:34:16 smithi089 bash[26572]: audit 2023-06-03T19:34:15.780798+0000 mon.a (mon.0) 657 : audit [INF] from='mgr.14445 172.21.15.89:0/573502486' entity='mgr.y' cmd='[{"prefix": "osd pool create", "pool": "cephfs.foo.meta"}]': finished 2023-06-03T19:34:17.282 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:34:16 smithi089 bash[26572]: cluster 2023-06-03T19:34:15.780916+0000 mon.a (mon.0) 658 : cluster [DBG] osdmap e56: 8 total, 8 up, 8 in 2023-06-03T19:34:17.282 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:34:16 smithi089 bash[26572]: audit 2023-06-03T19:34:15.785652+0000 mon.a (mon.0) 659 : audit [INF] from='mgr.14445 172.21.15.89:0/573502486' entity='mgr.y' cmd=[{"prefix": "osd pool create", "pool": "cephfs.foo.data"}]: dispatch 2023-06-03T19:34:17.282 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:34:16 smithi089 bash[26572]: debug 2023-06-03T19:34:16.800+0000 7fc1ca855700 -1 log_channel(cluster) log [ERR] : Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2023-06-03T19:34:17.282 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:34:16 smithi089 bash[33173]: audit 2023-06-03T19:34:15.105957+0000 mgr.y (mgr.14445) 103 : audit [DBG] from='client.14634 -' entity='client.admin' cmd=[{"prefix": "fs volume create", "name": "foo", "target": ["mon-mgr", ""]}]: dispatch 2023-06-03T19:34:17.282 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:34:16 smithi089 bash[33173]: audit 2023-06-03T19:34:15.780798+0000 mon.a (mon.0) 657 : audit [INF] from='mgr.14445 172.21.15.89:0/573502486' entity='mgr.y' cmd='[{"prefix": "osd pool create", "pool": "cephfs.foo.meta"}]': finished 2023-06-03T19:34:17.282 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:34:16 smithi089 bash[33173]: cluster 2023-06-03T19:34:15.780916+0000 mon.a (mon.0) 658 : cluster [DBG] osdmap e56: 8 total, 8 up, 8 in 2023-06-03T19:34:17.283 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:34:16 smithi089 bash[33173]: audit 2023-06-03T19:34:15.785652+0000 mon.a (mon.0) 659 : audit [INF] from='mgr.14445 172.21.15.89:0/573502486' entity='mgr.y' cmd=[{"prefix": "osd pool create", "pool": "cephfs.foo.data"}]: dispatch 2023-06-03T19:34:17.957 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- bash -c 'ceph config set mon mon_warn_on_insecure_global_id_reclaim false --force' 2023-06-03T19:34:18.087 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:34:17 smithi164 bash[26855]: cluster 2023-06-03T19:34:16.412830+0000 mgr.y (mgr.14445) 104 : cluster [DBG] pgmap v82: 41 pgs: 32 unknown, 9 active+clean; 0 B data, 43 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:34:18.087 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:34:17 smithi164 bash[26855]: audit 2023-06-03T19:34:16.787194+0000 mon.a (mon.0) 660 : audit [INF] from='mgr.14445 172.21.15.89:0/573502486' entity='mgr.y' cmd='[{"prefix": "osd pool create", "pool": "cephfs.foo.data"}]': finished 2023-06-03T19:34:18.087 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:34:17 smithi164 bash[26855]: cluster 2023-06-03T19:34:16.787550+0000 mon.a (mon.0) 661 : cluster [DBG] osdmap e57: 8 total, 8 up, 8 in 2023-06-03T19:34:18.087 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:34:17 smithi164 bash[26855]: audit 2023-06-03T19:34:16.804142+0000 mon.a (mon.0) 662 : audit [INF] from='mgr.14445 172.21.15.89:0/573502486' entity='mgr.y' cmd=[{"prefix": "fs new", "fs_name": "foo", "metadata": "cephfs.foo.meta", "data": "cephfs.foo.data"}]: dispatch 2023-06-03T19:34:18.088 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:34:17 smithi164 bash[26855]: cluster 2023-06-03T19:34:16.805900+0000 mon.a (mon.0) 663 : cluster [ERR] Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2023-06-03T19:34:18.088 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:34:17 smithi164 bash[26855]: cluster 2023-06-03T19:34:16.805936+0000 mon.a (mon.0) 664 : cluster [WRN] Health check failed: 1 filesystem is online with fewer MDS than max_mds (MDS_UP_LESS_THAN_MAX) 2023-06-03T19:34:18.088 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:34:17 smithi164 bash[26855]: cluster 2023-06-03T19:34:16.818667+0000 mon.a (mon.0) 665 : cluster [DBG] osdmap e58: 8 total, 8 up, 8 in 2023-06-03T19:34:18.088 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:34:17 smithi164 bash[26855]: audit 2023-06-03T19:34:16.819863+0000 mon.a (mon.0) 666 : audit [INF] from='mgr.14445 172.21.15.89:0/573502486' entity='mgr.y' cmd='[{"prefix": "fs new", "fs_name": "foo", "metadata": "cephfs.foo.meta", "data": "cephfs.foo.data"}]': finished 2023-06-03T19:34:18.088 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:34:17 smithi164 bash[26855]: cluster 2023-06-03T19:34:16.820287+0000 mon.a (mon.0) 667 : cluster [DBG] fsmap foo:0 2023-06-03T19:34:18.088 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:34:17 smithi164 bash[26855]: audit 2023-06-03T19:34:16.826863+0000 mon.a (mon.0) 668 : audit [INF] from='mgr.14445 172.21.15.89:0/573502486' entity='mgr.y' 2023-06-03T19:34:18.088 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:34:17 smithi164 bash[26855]: audit 2023-06-03T19:34:16.861912+0000 mon.a (mon.0) 669 : audit [DBG] from='mgr.14445 172.21.15.89:0/573502486' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-03T19:34:18.088 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:34:17 smithi164 bash[26855]: audit 2023-06-03T19:34:17.158331+0000 mon.a (mon.0) 670 : audit [DBG] from='mgr.14445 172.21.15.89:0/573502486' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-03T19:34:18.089 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:34:17 smithi164 bash[26855]: audit 2023-06-03T19:34:17.160118+0000 mon.a (mon.0) 671 : audit [INF] from='mgr.14445 172.21.15.89:0/573502486' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-03T19:34:18.089 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:34:17 smithi164 bash[26855]: audit 2023-06-03T19:34:17.168808+0000 mon.a (mon.0) 672 : audit [INF] from='mgr.14445 172.21.15.89:0/573502486' entity='mgr.y' 2023-06-03T19:34:18.089 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:34:17 smithi164 bash[26855]: audit 2023-06-03T19:34:17.183119+0000 mon.a (mon.0) 673 : audit [INF] from='mgr.14445 172.21.15.89:0/573502486' entity='mgr.y' 2023-06-03T19:34:18.089 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:34:17 smithi164 bash[26855]: audit 2023-06-03T19:34:17.184474+0000 mon.a (mon.0) 674 : audit [INF] from='mgr.14445 172.21.15.89:0/573502486' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi089.giuxuj", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-06-03T19:34:18.089 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:34:17 smithi164 bash[26855]: audit 2023-06-03T19:34:17.190693+0000 mon.a (mon.0) 675 : audit [INF] from='mgr.14445 172.21.15.89:0/573502486' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi089.giuxuj", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2023-06-03T19:34:18.089 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:34:17 smithi164 bash[26855]: audit 2023-06-03T19:34:17.191781+0000 mon.a (mon.0) 676 : audit [DBG] from='mgr.14445 172.21.15.89:0/573502486' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-03T19:34:18.152 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 19:34:17 smithi089 bash[26845]: ::ffff:172.21.15.164 - - [03/Jun/2023:19:34:17] "GET /metrics HTTP/1.1" 200 152020 "" "Prometheus/2.33.4" 2023-06-03T19:34:18.152 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:34:17 smithi089 bash[33173]: cluster 2023-06-03T19:34:16.412830+0000 mgr.y (mgr.14445) 104 : cluster [DBG] pgmap v82: 41 pgs: 32 unknown, 9 active+clean; 0 B data, 43 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:34:18.152 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:34:17 smithi089 bash[33173]: audit 2023-06-03T19:34:16.787194+0000 mon.a (mon.0) 660 : audit [INF] from='mgr.14445 172.21.15.89:0/573502486' entity='mgr.y' cmd='[{"prefix": "osd pool create", "pool": "cephfs.foo.data"}]': finished 2023-06-03T19:34:18.153 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:34:17 smithi089 bash[33173]: cluster 2023-06-03T19:34:16.787550+0000 mon.a (mon.0) 661 : cluster [DBG] osdmap e57: 8 total, 8 up, 8 in 2023-06-03T19:34:18.153 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:34:17 smithi089 bash[33173]: audit 2023-06-03T19:34:16.804142+0000 mon.a (mon.0) 662 : audit [INF] from='mgr.14445 172.21.15.89:0/573502486' entity='mgr.y' cmd=[{"prefix": "fs new", "fs_name": "foo", "metadata": "cephfs.foo.meta", "data": "cephfs.foo.data"}]: dispatch 2023-06-03T19:34:18.153 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:34:17 smithi089 bash[33173]: cluster 2023-06-03T19:34:16.805900+0000 mon.a (mon.0) 663 : cluster [ERR] Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2023-06-03T19:34:18.153 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:34:17 smithi089 bash[33173]: cluster 2023-06-03T19:34:16.805936+0000 mon.a (mon.0) 664 : cluster [WRN] Health check failed: 1 filesystem is online with fewer MDS than max_mds (MDS_UP_LESS_THAN_MAX) 2023-06-03T19:34:18.153 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:34:17 smithi089 bash[33173]: cluster 2023-06-03T19:34:16.818667+0000 mon.a (mon.0) 665 : cluster [DBG] osdmap e58: 8 total, 8 up, 8 in 2023-06-03T19:34:18.153 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:34:17 smithi089 bash[33173]: audit 2023-06-03T19:34:16.819863+0000 mon.a (mon.0) 666 : audit [INF] from='mgr.14445 172.21.15.89:0/573502486' entity='mgr.y' cmd='[{"prefix": "fs new", "fs_name": "foo", "metadata": "cephfs.foo.meta", "data": "cephfs.foo.data"}]': finished 2023-06-03T19:34:18.153 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:34:17 smithi089 bash[33173]: cluster 2023-06-03T19:34:16.820287+0000 mon.a (mon.0) 667 : cluster [DBG] fsmap foo:0 2023-06-03T19:34:18.153 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:34:17 smithi089 bash[33173]: audit 2023-06-03T19:34:16.826863+0000 mon.a (mon.0) 668 : audit [INF] from='mgr.14445 172.21.15.89:0/573502486' entity='mgr.y' 2023-06-03T19:34:18.154 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:34:17 smithi089 bash[33173]: audit 2023-06-03T19:34:16.861912+0000 mon.a (mon.0) 669 : audit [DBG] from='mgr.14445 172.21.15.89:0/573502486' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-03T19:34:18.154 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:34:17 smithi089 bash[33173]: audit 2023-06-03T19:34:17.158331+0000 mon.a (mon.0) 670 : audit [DBG] from='mgr.14445 172.21.15.89:0/573502486' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-03T19:34:18.154 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:34:17 smithi089 bash[33173]: audit 2023-06-03T19:34:17.160118+0000 mon.a (mon.0) 671 : audit [INF] from='mgr.14445 172.21.15.89:0/573502486' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-03T19:34:18.154 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:34:17 smithi089 bash[33173]: audit 2023-06-03T19:34:17.168808+0000 mon.a (mon.0) 672 : audit [INF] from='mgr.14445 172.21.15.89:0/573502486' entity='mgr.y' 2023-06-03T19:34:18.154 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:34:17 smithi089 bash[33173]: audit 2023-06-03T19:34:17.183119+0000 mon.a (mon.0) 673 : audit [INF] from='mgr.14445 172.21.15.89:0/573502486' entity='mgr.y' 2023-06-03T19:34:18.154 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:34:17 smithi089 bash[33173]: audit 2023-06-03T19:34:17.184474+0000 mon.a (mon.0) 674 : audit [INF] from='mgr.14445 172.21.15.89:0/573502486' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi089.giuxuj", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-06-03T19:34:18.154 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:34:17 smithi089 bash[33173]: audit 2023-06-03T19:34:17.190693+0000 mon.a (mon.0) 675 : audit [INF] from='mgr.14445 172.21.15.89:0/573502486' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi089.giuxuj", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2023-06-03T19:34:18.155 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:34:17 smithi089 bash[33173]: audit 2023-06-03T19:34:17.191781+0000 mon.a (mon.0) 676 : audit [DBG] from='mgr.14445 172.21.15.89:0/573502486' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-03T19:34:18.155 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:34:17 smithi089 bash[26572]: cluster 2023-06-03T19:34:16.412830+0000 mgr.y (mgr.14445) 104 : cluster [DBG] pgmap v82: 41 pgs: 32 unknown, 9 active+clean; 0 B data, 43 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:34:18.155 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:34:17 smithi089 bash[26572]: audit 2023-06-03T19:34:16.787194+0000 mon.a (mon.0) 660 : audit [INF] from='mgr.14445 172.21.15.89:0/573502486' entity='mgr.y' cmd='[{"prefix": "osd pool create", "pool": "cephfs.foo.data"}]': finished 2023-06-03T19:34:18.155 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:34:17 smithi089 bash[26572]: cluster 2023-06-03T19:34:16.787550+0000 mon.a (mon.0) 661 : cluster [DBG] osdmap e57: 8 total, 8 up, 8 in 2023-06-03T19:34:18.155 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:34:17 smithi089 bash[26572]: audit 2023-06-03T19:34:16.804142+0000 mon.a (mon.0) 662 : audit [INF] from='mgr.14445 172.21.15.89:0/573502486' entity='mgr.y' cmd=[{"prefix": "fs new", "fs_name": "foo", "metadata": "cephfs.foo.meta", "data": "cephfs.foo.data"}]: dispatch 2023-06-03T19:34:18.155 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:34:17 smithi089 bash[26572]: cluster 2023-06-03T19:34:16.805900+0000 mon.a (mon.0) 663 : cluster [ERR] Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2023-06-03T19:34:18.155 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:34:17 smithi089 bash[26572]: cluster 2023-06-03T19:34:16.805936+0000 mon.a (mon.0) 664 : cluster [WRN] Health check failed: 1 filesystem is online with fewer MDS than max_mds (MDS_UP_LESS_THAN_MAX) 2023-06-03T19:34:18.156 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:34:17 smithi089 bash[26572]: cluster 2023-06-03T19:34:16.818667+0000 mon.a (mon.0) 665 : cluster [DBG] osdmap e58: 8 total, 8 up, 8 in 2023-06-03T19:34:18.156 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:34:17 smithi089 bash[26572]: audit 2023-06-03T19:34:16.819863+0000 mon.a (mon.0) 666 : audit [INF] from='mgr.14445 172.21.15.89:0/573502486' entity='mgr.y' cmd='[{"prefix": "fs new", "fs_name": "foo", "metadata": "cephfs.foo.meta", "data": "cephfs.foo.data"}]': finished 2023-06-03T19:34:18.156 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:34:17 smithi089 bash[26572]: cluster 2023-06-03T19:34:16.820287+0000 mon.a (mon.0) 667 : cluster [DBG] fsmap foo:0 2023-06-03T19:34:18.156 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:34:17 smithi089 bash[26572]: audit 2023-06-03T19:34:16.826863+0000 mon.a (mon.0) 668 : audit [INF] from='mgr.14445 172.21.15.89:0/573502486' entity='mgr.y' 2023-06-03T19:34:18.156 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:34:17 smithi089 bash[26572]: audit 2023-06-03T19:34:16.861912+0000 mon.a (mon.0) 669 : audit [DBG] from='mgr.14445 172.21.15.89:0/573502486' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-03T19:34:18.156 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:34:17 smithi089 bash[26572]: audit 2023-06-03T19:34:17.158331+0000 mon.a (mon.0) 670 : audit [DBG] from='mgr.14445 172.21.15.89:0/573502486' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-03T19:34:18.156 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:34:17 smithi089 bash[26572]: audit 2023-06-03T19:34:17.160118+0000 mon.a (mon.0) 671 : audit [INF] from='mgr.14445 172.21.15.89:0/573502486' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-03T19:34:18.157 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:34:17 smithi089 bash[26572]: audit 2023-06-03T19:34:17.168808+0000 mon.a (mon.0) 672 : audit [INF] from='mgr.14445 172.21.15.89:0/573502486' entity='mgr.y' 2023-06-03T19:34:18.157 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:34:17 smithi089 bash[26572]: audit 2023-06-03T19:34:17.183119+0000 mon.a (mon.0) 673 : audit [INF] from='mgr.14445 172.21.15.89:0/573502486' entity='mgr.y' 2023-06-03T19:34:18.157 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:34:17 smithi089 bash[26572]: audit 2023-06-03T19:34:17.184474+0000 mon.a (mon.0) 674 : audit [INF] from='mgr.14445 172.21.15.89:0/573502486' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi089.giuxuj", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-06-03T19:34:18.157 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:34:17 smithi089 bash[26572]: audit 2023-06-03T19:34:17.190693+0000 mon.a (mon.0) 675 : audit [INF] from='mgr.14445 172.21.15.89:0/573502486' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi089.giuxuj", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2023-06-03T19:34:18.157 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:34:17 smithi089 bash[26572]: audit 2023-06-03T19:34:17.191781+0000 mon.a (mon.0) 676 : audit [DBG] from='mgr.14445 172.21.15.89:0/573502486' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-03T19:34:19.086 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:34:18 smithi164 bash[26855]: cephadm 2023-06-03T19:34:16.823189+0000 mgr.y (mgr.14445) 105 : cephadm [INF] Saving service mds.foo spec with placement count:2 2023-06-03T19:34:19.087 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:34:18 smithi164 bash[26855]: cephadm 2023-06-03T19:34:17.192801+0000 mgr.y (mgr.14445) 106 : cephadm [INF] Deploying daemon mds.foo.smithi089.giuxuj on smithi089 2023-06-03T19:34:19.087 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:34:18 smithi164 bash[26855]: cluster 2023-06-03T19:34:17.819506+0000 mon.a (mon.0) 677 : cluster [DBG] osdmap e59: 8 total, 8 up, 8 in 2023-06-03T19:34:19.087 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:34:18 smithi164 bash[26855]: audit 2023-06-03T19:34:18.621165+0000 mon.a (mon.0) 678 : audit [INF] from='mgr.14445 172.21.15.89:0/573502486' entity='mgr.y' 2023-06-03T19:34:19.281 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:34:18 smithi089 bash[26572]: cephadm 2023-06-03T19:34:16.823189+0000 mgr.y (mgr.14445) 105 : cephadm [INF] Saving service mds.foo spec with placement count:2 2023-06-03T19:34:19.282 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:34:18 smithi089 bash[26572]: cephadm 2023-06-03T19:34:17.192801+0000 mgr.y (mgr.14445) 106 : cephadm [INF] Deploying daemon mds.foo.smithi089.giuxuj on smithi089 2023-06-03T19:34:19.282 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:34:18 smithi089 bash[26572]: cluster 2023-06-03T19:34:17.819506+0000 mon.a (mon.0) 677 : cluster [DBG] osdmap e59: 8 total, 8 up, 8 in 2023-06-03T19:34:19.282 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:34:18 smithi089 bash[26572]: audit 2023-06-03T19:34:18.621165+0000 mon.a (mon.0) 678 : audit [INF] from='mgr.14445 172.21.15.89:0/573502486' entity='mgr.y' 2023-06-03T19:34:19.282 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:34:18 smithi089 bash[33173]: cephadm 2023-06-03T19:34:16.823189+0000 mgr.y (mgr.14445) 105 : cephadm [INF] Saving service mds.foo spec with placement count:2 2023-06-03T19:34:19.282 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:34:18 smithi089 bash[33173]: cephadm 2023-06-03T19:34:17.192801+0000 mgr.y (mgr.14445) 106 : cephadm [INF] Deploying daemon mds.foo.smithi089.giuxuj on smithi089 2023-06-03T19:34:19.282 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:34:18 smithi089 bash[33173]: cluster 2023-06-03T19:34:17.819506+0000 mon.a (mon.0) 677 : cluster [DBG] osdmap e59: 8 total, 8 up, 8 in 2023-06-03T19:34:19.283 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:34:18 smithi089 bash[33173]: audit 2023-06-03T19:34:18.621165+0000 mon.a (mon.0) 678 : audit [INF] from='mgr.14445 172.21.15.89:0/573502486' entity='mgr.y' 2023-06-03T19:34:20.031 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:34:19 smithi089 bash[26572]: cluster 2023-06-03T19:34:18.413490+0000 mgr.y (mgr.14445) 107 : cluster [DBG] pgmap v86: 73 pgs: 64 unknown, 9 active+clean; 0 B data, 43 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:34:20.032 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:34:19 smithi089 bash[26572]: cluster 2023-06-03T19:34:18.826671+0000 mon.a (mon.0) 679 : cluster [DBG] osdmap e60: 8 total, 8 up, 8 in 2023-06-03T19:34:20.032 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:34:19 smithi089 bash[33173]: cluster 2023-06-03T19:34:18.413490+0000 mgr.y (mgr.14445) 107 : cluster [DBG] pgmap v86: 73 pgs: 64 unknown, 9 active+clean; 0 B data, 43 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:34:20.032 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:34:19 smithi089 bash[33173]: cluster 2023-06-03T19:34:18.826671+0000 mon.a (mon.0) 679 : cluster [DBG] osdmap e60: 8 total, 8 up, 8 in 2023-06-03T19:34:20.086 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:34:19 smithi164 bash[26855]: cluster 2023-06-03T19:34:18.413490+0000 mgr.y (mgr.14445) 107 : cluster [DBG] pgmap v86: 73 pgs: 64 unknown, 9 active+clean; 0 B data, 43 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:34:20.087 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:34:19 smithi164 bash[26855]: cluster 2023-06-03T19:34:18.826671+0000 mon.a (mon.0) 679 : cluster [DBG] osdmap e60: 8 total, 8 up, 8 in 2023-06-03T19:34:22.086 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:34:21 smithi164 bash[26855]: cluster 2023-06-03T19:34:20.414288+0000 mgr.y (mgr.14445) 108 : cluster [DBG] pgmap v88: 73 pgs: 73 active+clean; 0 B data, 45 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:34:22.087 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:34:21 smithi164 bash[26855]: audit 2023-06-03T19:34:21.506739+0000 mon.a (mon.0) 680 : audit [INF] from='mgr.14445 172.21.15.89:0/573502486' entity='mgr.y' 2023-06-03T19:34:22.087 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:34:21 smithi164 bash[26855]: audit 2023-06-03T19:34:21.508276+0000 mon.a (mon.0) 681 : audit [INF] from='mgr.14445 172.21.15.89:0/573502486' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi164.txgfgv", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-06-03T19:34:22.087 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:34:21 smithi164 bash[26855]: audit 2023-06-03T19:34:21.514144+0000 mon.a (mon.0) 682 : audit [INF] from='mgr.14445 172.21.15.89:0/573502486' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi164.txgfgv", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2023-06-03T19:34:22.088 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:34:21 smithi164 bash[26855]: audit 2023-06-03T19:34:21.515224+0000 mon.a (mon.0) 683 : audit [DBG] from='mgr.14445 172.21.15.89:0/573502486' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-03T19:34:22.281 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:34:21 smithi089 bash[26572]: cluster 2023-06-03T19:34:20.414288+0000 mgr.y (mgr.14445) 108 : cluster [DBG] pgmap v88: 73 pgs: 73 active+clean; 0 B data, 45 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:34:22.282 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:34:21 smithi089 bash[26572]: audit 2023-06-03T19:34:21.506739+0000 mon.a (mon.0) 680 : audit [INF] from='mgr.14445 172.21.15.89:0/573502486' entity='mgr.y' 2023-06-03T19:34:22.282 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:34:21 smithi089 bash[26572]: audit 2023-06-03T19:34:21.508276+0000 mon.a (mon.0) 681 : audit [INF] from='mgr.14445 172.21.15.89:0/573502486' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi164.txgfgv", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-06-03T19:34:22.282 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:34:21 smithi089 bash[26572]: audit 2023-06-03T19:34:21.514144+0000 mon.a (mon.0) 682 : audit [INF] from='mgr.14445 172.21.15.89:0/573502486' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi164.txgfgv", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2023-06-03T19:34:22.282 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:34:21 smithi089 bash[26572]: audit 2023-06-03T19:34:21.515224+0000 mon.a (mon.0) 683 : audit [DBG] from='mgr.14445 172.21.15.89:0/573502486' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-03T19:34:22.282 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:34:21 smithi089 bash[33173]: cluster 2023-06-03T19:34:20.414288+0000 mgr.y (mgr.14445) 108 : cluster [DBG] pgmap v88: 73 pgs: 73 active+clean; 0 B data, 45 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:34:22.283 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:34:21 smithi089 bash[33173]: audit 2023-06-03T19:34:21.506739+0000 mon.a (mon.0) 680 : audit [INF] from='mgr.14445 172.21.15.89:0/573502486' entity='mgr.y' 2023-06-03T19:34:22.283 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:34:21 smithi089 bash[33173]: audit 2023-06-03T19:34:21.508276+0000 mon.a (mon.0) 681 : audit [INF] from='mgr.14445 172.21.15.89:0/573502486' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi164.txgfgv", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-06-03T19:34:22.283 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:34:21 smithi089 bash[33173]: audit 2023-06-03T19:34:21.514144+0000 mon.a (mon.0) 682 : audit [INF] from='mgr.14445 172.21.15.89:0/573502486' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi164.txgfgv", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2023-06-03T19:34:22.283 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:34:21 smithi089 bash[33173]: audit 2023-06-03T19:34:21.515224+0000 mon.a (mon.0) 683 : audit [DBG] from='mgr.14445 172.21.15.89:0/573502486' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-03T19:34:23.086 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:34:22 smithi164 bash[26855]: cephadm 2023-06-03T19:34:21.516117+0000 mgr.y (mgr.14445) 109 : cephadm [INF] Deploying daemon mds.foo.smithi164.txgfgv on smithi164 2023-06-03T19:34:23.205 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:34:22 smithi089 bash[33173]: cephadm 2023-06-03T19:34:21.516117+0000 mgr.y (mgr.14445) 109 : cephadm [INF] Deploying daemon mds.foo.smithi164.txgfgv on smithi164 2023-06-03T19:34:23.206 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:34:22 smithi089 bash[26572]: cephadm 2023-06-03T19:34:21.516117+0000 mgr.y (mgr.14445) 109 : cephadm [INF] Deploying daemon mds.foo.smithi164.txgfgv on smithi164 2023-06-03T19:34:24.079 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:34:23 smithi164 bash[26855]: cluster 2023-06-03T19:34:22.414829+0000 mgr.y (mgr.14445) 110 : cluster [DBG] pgmap v89: 73 pgs: 73 active+clean; 0 B data, 45 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:34:24.281 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:34:23 smithi089 bash[26572]: cluster 2023-06-03T19:34:22.414829+0000 mgr.y (mgr.14445) 110 : cluster [DBG] pgmap v89: 73 pgs: 73 active+clean; 0 B data, 45 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:34:24.282 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:34:23 smithi089 bash[33173]: cluster 2023-06-03T19:34:22.414829+0000 mgr.y (mgr.14445) 110 : cluster [DBG] pgmap v89: 73 pgs: 73 active+clean; 0 B data, 45 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:34:24.515 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- bash -c 'ceph config set mon mon_warn_on_insecure_global_id_reclaim_allowed false --force' 2023-06-03T19:34:25.032 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:34:24 smithi089 bash[26572]: cluster 2023-06-03T19:34:23.824594+0000 mon.a (mon.0) 684 : cluster [DBG] mds.? [v2:172.21.15.89:6834/1926931511,v1:172.21.15.89:6835/1926931511] up:boot 2023-06-03T19:34:25.032 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:34:24 smithi089 bash[26572]: cluster 2023-06-03T19:34:23.824670+0000 mon.a (mon.0) 685 : cluster [INF] daemon mds.foo.smithi089.giuxuj assigned to filesystem foo as rank 0 (now has 1 ranks) 2023-06-03T19:34:25.032 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:34:24 smithi089 bash[26572]: cluster 2023-06-03T19:34:23.824774+0000 mon.a (mon.0) 686 : cluster [INF] Health check cleared: MDS_ALL_DOWN (was: 1 filesystem is offline) 2023-06-03T19:34:25.032 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:34:24 smithi089 bash[26572]: cluster 2023-06-03T19:34:23.824806+0000 mon.a (mon.0) 687 : cluster [INF] Health check cleared: MDS_UP_LESS_THAN_MAX (was: 1 filesystem is online with fewer MDS than max_mds) 2023-06-03T19:34:25.033 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:34:24 smithi089 bash[26572]: cluster 2023-06-03T19:34:23.824849+0000 mon.a (mon.0) 688 : cluster [INF] Cluster is now healthy 2023-06-03T19:34:25.033 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:34:24 smithi089 bash[26572]: cluster 2023-06-03T19:34:23.825171+0000 mon.a (mon.0) 689 : cluster [DBG] fsmap foo:0 1 up:standby 2023-06-03T19:34:25.033 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:34:24 smithi089 bash[26572]: audit 2023-06-03T19:34:23.826371+0000 mon.a (mon.0) 690 : audit [DBG] from='mgr.14445 172.21.15.89:0/573502486' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi089.giuxuj"}]: dispatch 2023-06-03T19:34:25.033 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:34:24 smithi089 bash[26572]: cluster 2023-06-03T19:34:23.834341+0000 mon.a (mon.0) 691 : cluster [DBG] fsmap foo:1 {0=foo.smithi089.giuxuj=up:creating} 2023-06-03T19:34:25.033 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:34:24 smithi089 bash[26572]: cluster 2023-06-03T19:34:23.849034+0000 mon.a (mon.0) 692 : cluster [INF] daemon mds.foo.smithi089.giuxuj is now active in filesystem foo as rank 0 2023-06-03T19:34:25.033 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:34:24 smithi089 bash[26572]: audit 2023-06-03T19:34:24.345648+0000 mon.a (mon.0) 693 : audit [INF] from='mgr.14445 172.21.15.89:0/573502486' entity='mgr.y' 2023-06-03T19:34:25.033 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:34:24 smithi089 bash[26572]: audit 2023-06-03T19:34:24.347456+0000 mon.a (mon.0) 694 : audit [DBG] from='mgr.14445 172.21.15.89:0/573502486' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-03T19:34:25.034 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:34:24 smithi089 bash[26572]: cluster 2023-06-03T19:34:24.415819+0000 mgr.y (mgr.14445) 111 : cluster [DBG] pgmap v90: 73 pgs: 73 active+clean; 2.3 KiB data, 45 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:34:25.034 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:34:24 smithi089 bash[33173]: cluster 2023-06-03T19:34:23.824594+0000 mon.a (mon.0) 684 : cluster [DBG] mds.? [v2:172.21.15.89:6834/1926931511,v1:172.21.15.89:6835/1926931511] up:boot 2023-06-03T19:34:25.034 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:34:24 smithi089 bash[33173]: cluster 2023-06-03T19:34:23.824670+0000 mon.a (mon.0) 685 : cluster [INF] daemon mds.foo.smithi089.giuxuj assigned to filesystem foo as rank 0 (now has 1 ranks) 2023-06-03T19:34:25.034 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:34:24 smithi089 bash[33173]: cluster 2023-06-03T19:34:23.824774+0000 mon.a (mon.0) 686 : cluster [INF] Health check cleared: MDS_ALL_DOWN (was: 1 filesystem is offline) 2023-06-03T19:34:25.035 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:34:24 smithi089 bash[33173]: cluster 2023-06-03T19:34:23.824806+0000 mon.a (mon.0) 687 : cluster [INF] Health check cleared: MDS_UP_LESS_THAN_MAX (was: 1 filesystem is online with fewer MDS than max_mds) 2023-06-03T19:34:25.035 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:34:24 smithi089 bash[33173]: cluster 2023-06-03T19:34:23.824849+0000 mon.a (mon.0) 688 : cluster [INF] Cluster is now healthy 2023-06-03T19:34:25.035 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:34:24 smithi089 bash[33173]: cluster 2023-06-03T19:34:23.825171+0000 mon.a (mon.0) 689 : cluster [DBG] fsmap foo:0 1 up:standby 2023-06-03T19:34:25.035 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:34:24 smithi089 bash[33173]: audit 2023-06-03T19:34:23.826371+0000 mon.a (mon.0) 690 : audit [DBG] from='mgr.14445 172.21.15.89:0/573502486' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi089.giuxuj"}]: dispatch 2023-06-03T19:34:25.035 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:34:24 smithi089 bash[33173]: cluster 2023-06-03T19:34:23.834341+0000 mon.a (mon.0) 691 : cluster [DBG] fsmap foo:1 {0=foo.smithi089.giuxuj=up:creating} 2023-06-03T19:34:25.035 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:34:24 smithi089 bash[33173]: cluster 2023-06-03T19:34:23.849034+0000 mon.a (mon.0) 692 : cluster [INF] daemon mds.foo.smithi089.giuxuj is now active in filesystem foo as rank 0 2023-06-03T19:34:25.035 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:34:24 smithi089 bash[33173]: audit 2023-06-03T19:34:24.345648+0000 mon.a (mon.0) 693 : audit [INF] from='mgr.14445 172.21.15.89:0/573502486' entity='mgr.y' 2023-06-03T19:34:25.036 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:34:24 smithi089 bash[33173]: audit 2023-06-03T19:34:24.347456+0000 mon.a (mon.0) 694 : audit [DBG] from='mgr.14445 172.21.15.89:0/573502486' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-03T19:34:25.036 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:34:24 smithi089 bash[33173]: cluster 2023-06-03T19:34:24.415819+0000 mgr.y (mgr.14445) 111 : cluster [DBG] pgmap v90: 73 pgs: 73 active+clean; 2.3 KiB data, 45 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:34:25.336 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:34:24 smithi164 bash[26855]: cluster 2023-06-03T19:34:23.824594+0000 mon.a (mon.0) 684 : cluster [DBG] mds.? [v2:172.21.15.89:6834/1926931511,v1:172.21.15.89:6835/1926931511] up:boot 2023-06-03T19:34:25.337 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:34:24 smithi164 bash[26855]: cluster 2023-06-03T19:34:23.824670+0000 mon.a (mon.0) 685 : cluster [INF] daemon mds.foo.smithi089.giuxuj assigned to filesystem foo as rank 0 (now has 1 ranks) 2023-06-03T19:34:25.337 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:34:24 smithi164 bash[26855]: cluster 2023-06-03T19:34:23.824774+0000 mon.a (mon.0) 686 : cluster [INF] Health check cleared: MDS_ALL_DOWN (was: 1 filesystem is offline) 2023-06-03T19:34:25.337 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:34:24 smithi164 bash[26855]: cluster 2023-06-03T19:34:23.824806+0000 mon.a (mon.0) 687 : cluster [INF] Health check cleared: MDS_UP_LESS_THAN_MAX (was: 1 filesystem is online with fewer MDS than max_mds) 2023-06-03T19:34:25.337 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:34:24 smithi164 bash[26855]: cluster 2023-06-03T19:34:23.824849+0000 mon.a (mon.0) 688 : cluster [INF] Cluster is now healthy 2023-06-03T19:34:25.337 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:34:24 smithi164 bash[26855]: cluster 2023-06-03T19:34:23.825171+0000 mon.a (mon.0) 689 : cluster [DBG] fsmap foo:0 1 up:standby 2023-06-03T19:34:25.337 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:34:24 smithi164 bash[26855]: audit 2023-06-03T19:34:23.826371+0000 mon.a (mon.0) 690 : audit [DBG] from='mgr.14445 172.21.15.89:0/573502486' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi089.giuxuj"}]: dispatch 2023-06-03T19:34:25.338 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:34:24 smithi164 bash[26855]: cluster 2023-06-03T19:34:23.834341+0000 mon.a (mon.0) 691 : cluster [DBG] fsmap foo:1 {0=foo.smithi089.giuxuj=up:creating} 2023-06-03T19:34:25.338 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:34:24 smithi164 bash[26855]: cluster 2023-06-03T19:34:23.849034+0000 mon.a (mon.0) 692 : cluster [INF] daemon mds.foo.smithi089.giuxuj is now active in filesystem foo as rank 0 2023-06-03T19:34:25.338 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:34:24 smithi164 bash[26855]: audit 2023-06-03T19:34:24.345648+0000 mon.a (mon.0) 693 : audit [INF] from='mgr.14445 172.21.15.89:0/573502486' entity='mgr.y' 2023-06-03T19:34:25.338 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:34:24 smithi164 bash[26855]: audit 2023-06-03T19:34:24.347456+0000 mon.a (mon.0) 694 : audit [DBG] from='mgr.14445 172.21.15.89:0/573502486' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-03T19:34:25.338 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:34:24 smithi164 bash[26855]: cluster 2023-06-03T19:34:24.415819+0000 mgr.y (mgr.14445) 111 : cluster [DBG] pgmap v90: 73 pgs: 73 active+clean; 2.3 KiB data, 45 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:34:25.836 INFO:journalctl@ceph.mgr.x.smithi164.stdout:Jun 03 19:34:25 smithi164 bash[27934]: ::ffff:172.21.15.164 - - [03/Jun/2023:19:34:25] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2023-06-03T19:34:26.105 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:34:25 smithi089 bash[26572]: cluster 2023-06-03T19:34:24.834821+0000 mon.a (mon.0) 695 : cluster [DBG] mds.? [v2:172.21.15.89:6834/1926931511,v1:172.21.15.89:6835/1926931511] up:active 2023-06-03T19:34:26.105 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:34:25 smithi089 bash[26572]: cluster 2023-06-03T19:34:24.834892+0000 mon.a (mon.0) 696 : cluster [DBG] fsmap foo:1 {0=foo.smithi089.giuxuj=up:active} 2023-06-03T19:34:26.105 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:34:25 smithi089 bash[33173]: cluster 2023-06-03T19:34:24.834821+0000 mon.a (mon.0) 695 : cluster [DBG] mds.? [v2:172.21.15.89:6834/1926931511,v1:172.21.15.89:6835/1926931511] up:active 2023-06-03T19:34:26.106 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:34:25 smithi089 bash[33173]: cluster 2023-06-03T19:34:24.834892+0000 mon.a (mon.0) 696 : cluster [DBG] fsmap foo:1 {0=foo.smithi089.giuxuj=up:active} 2023-06-03T19:34:26.336 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:34:25 smithi164 bash[26855]: cluster 2023-06-03T19:34:24.834821+0000 mon.a (mon.0) 695 : cluster [DBG] mds.? [v2:172.21.15.89:6834/1926931511,v1:172.21.15.89:6835/1926931511] up:active 2023-06-03T19:34:26.337 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:34:25 smithi164 bash[26855]: cluster 2023-06-03T19:34:24.834892+0000 mon.a (mon.0) 696 : cluster [DBG] fsmap foo:1 {0=foo.smithi089.giuxuj=up:active} 2023-06-03T19:34:27.738 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:34:27 smithi089 bash[26572]: cluster 2023-06-03T19:34:26.351382+0000 mon.a (mon.0) 697 : cluster [DBG] mds.? [v2:172.21.15.164:6832/2311698270,v1:172.21.15.164:6833/2311698270] up:boot 2023-06-03T19:34:27.738 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:34:27 smithi089 bash[26572]: cluster 2023-06-03T19:34:26.351993+0000 mon.a (mon.0) 698 : cluster [DBG] fsmap foo:1 {0=foo.smithi089.giuxuj=up:active} 1 up:standby 2023-06-03T19:34:27.738 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:34:27 smithi089 bash[26572]: audit 2023-06-03T19:34:26.352182+0000 mon.a (mon.0) 699 : audit [DBG] from='mgr.14445 172.21.15.89:0/573502486' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi164.txgfgv"}]: dispatch 2023-06-03T19:34:27.738 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:34:27 smithi089 bash[26572]: cluster 2023-06-03T19:34:26.355404+0000 mon.a (mon.0) 700 : cluster [DBG] fsmap foo:1 {0=foo.smithi089.giuxuj=up:active} 1 up:standby 2023-06-03T19:34:27.738 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:34:27 smithi089 bash[26572]: cluster 2023-06-03T19:34:26.416456+0000 mgr.y (mgr.14445) 112 : cluster [DBG] pgmap v91: 73 pgs: 73 active+clean; 2.3 KiB data, 45 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:34:27.739 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:34:27 smithi089 bash[33173]: cluster 2023-06-03T19:34:26.351382+0000 mon.a (mon.0) 697 : cluster [DBG] mds.? [v2:172.21.15.164:6832/2311698270,v1:172.21.15.164:6833/2311698270] up:boot 2023-06-03T19:34:27.739 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:34:27 smithi089 bash[33173]: cluster 2023-06-03T19:34:26.351993+0000 mon.a (mon.0) 698 : cluster [DBG] fsmap foo:1 {0=foo.smithi089.giuxuj=up:active} 1 up:standby 2023-06-03T19:34:27.739 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:34:27 smithi089 bash[33173]: audit 2023-06-03T19:34:26.352182+0000 mon.a (mon.0) 699 : audit [DBG] from='mgr.14445 172.21.15.89:0/573502486' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi164.txgfgv"}]: dispatch 2023-06-03T19:34:27.739 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:34:27 smithi089 bash[33173]: cluster 2023-06-03T19:34:26.355404+0000 mon.a (mon.0) 700 : cluster [DBG] fsmap foo:1 {0=foo.smithi089.giuxuj=up:active} 1 up:standby 2023-06-03T19:34:27.740 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:34:27 smithi089 bash[33173]: cluster 2023-06-03T19:34:26.416456+0000 mgr.y (mgr.14445) 112 : cluster [DBG] pgmap v91: 73 pgs: 73 active+clean; 2.3 KiB data, 45 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:34:27.836 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:34:27 smithi164 bash[26855]: cluster 2023-06-03T19:34:26.351382+0000 mon.a (mon.0) 697 : cluster [DBG] mds.? [v2:172.21.15.164:6832/2311698270,v1:172.21.15.164:6833/2311698270] up:boot 2023-06-03T19:34:27.837 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:34:27 smithi164 bash[26855]: cluster 2023-06-03T19:34:26.351993+0000 mon.a (mon.0) 698 : cluster [DBG] fsmap foo:1 {0=foo.smithi089.giuxuj=up:active} 1 up:standby 2023-06-03T19:34:27.837 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:34:27 smithi164 bash[26855]: audit 2023-06-03T19:34:26.352182+0000 mon.a (mon.0) 699 : audit [DBG] from='mgr.14445 172.21.15.89:0/573502486' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi164.txgfgv"}]: dispatch 2023-06-03T19:34:27.837 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:34:27 smithi164 bash[26855]: cluster 2023-06-03T19:34:26.355404+0000 mon.a (mon.0) 700 : cluster [DBG] fsmap foo:1 {0=foo.smithi089.giuxuj=up:active} 1 up:standby 2023-06-03T19:34:27.837 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:34:27 smithi164 bash[26855]: cluster 2023-06-03T19:34:26.416456+0000 mgr.y (mgr.14445) 112 : cluster [DBG] pgmap v91: 73 pgs: 73 active+clean; 2.3 KiB data, 45 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:34:28.031 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 19:34:27 smithi089 bash[26845]: ::ffff:172.21.15.164 - - [03/Jun/2023:19:34:27] "GET /metrics HTTP/1.1" 200 157043 "" "Prometheus/2.33.4" 2023-06-03T19:34:28.945 INFO:teuthology.run_tasks:Running task ceph.healthy... 2023-06-03T19:34:28.955 INFO:tasks.ceph:Waiting until ceph daemons up and pgs clean... 2023-06-03T19:34:28.955 INFO:tasks.cephadm.ceph_manager.ceph:waiting for mgr available 2023-06-03T19:34:28.955 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph mgr dump --format=json 2023-06-03T19:34:30.031 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:34:29 smithi089 bash[26572]: cluster 2023-06-03T19:34:28.417128+0000 mgr.y (mgr.14445) 113 : cluster [DBG] pgmap v92: 73 pgs: 73 active+clean; 2.3 KiB data, 45 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:34:30.032 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:34:29 smithi089 bash[26572]: audit 2023-06-03T19:34:28.706848+0000 mon.a (mon.0) 701 : audit [INF] from='mgr.14445 172.21.15.89:0/573502486' entity='mgr.y' 2023-06-03T19:34:30.032 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:34:29 smithi089 bash[33173]: cluster 2023-06-03T19:34:28.417128+0000 mgr.y (mgr.14445) 113 : cluster [DBG] pgmap v92: 73 pgs: 73 active+clean; 2.3 KiB data, 45 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:34:30.032 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:34:29 smithi089 bash[33173]: audit 2023-06-03T19:34:28.706848+0000 mon.a (mon.0) 701 : audit [INF] from='mgr.14445 172.21.15.89:0/573502486' entity='mgr.y' 2023-06-03T19:34:30.086 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:34:29 smithi164 bash[26855]: cluster 2023-06-03T19:34:28.417128+0000 mgr.y (mgr.14445) 113 : cluster [DBG] pgmap v92: 73 pgs: 73 active+clean; 2.3 KiB data, 45 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:34:30.087 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:34:29 smithi164 bash[26855]: audit 2023-06-03T19:34:28.706848+0000 mon.a (mon.0) 701 : audit [INF] from='mgr.14445 172.21.15.89:0/573502486' entity='mgr.y' 2023-06-03T19:34:31.782 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:34:31 smithi089 bash[26572]: cluster 2023-06-03T19:34:30.417966+0000 mgr.y (mgr.14445) 114 : cluster [DBG] pgmap v93: 73 pgs: 73 active+clean; 2.3 KiB data, 45 MiB used, 715 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2023-06-03T19:34:31.782 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:34:31 smithi089 bash[26572]: audit 2023-06-03T19:34:30.430036+0000 mon.a (mon.0) 702 : audit [INF] from='mgr.14445 172.21.15.89:0/573502486' entity='mgr.y' 2023-06-03T19:34:31.782 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:34:31 smithi089 bash[26572]: audit 2023-06-03T19:34:31.332785+0000 mon.a (mon.0) 703 : audit [INF] from='mgr.14445 172.21.15.89:0/573502486' entity='mgr.y' 2023-06-03T19:34:31.782 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:34:31 smithi089 bash[26572]: audit 2023-06-03T19:34:31.334312+0000 mon.a (mon.0) 704 : audit [DBG] from='mgr.14445 172.21.15.89:0/573502486' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-03T19:34:31.782 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:34:31 smithi089 bash[26572]: audit 2023-06-03T19:34:31.335443+0000 mon.a (mon.0) 705 : audit [INF] from='mgr.14445 172.21.15.89:0/573502486' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-03T19:34:31.783 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:34:31 smithi089 bash[26572]: audit 2023-06-03T19:34:31.341766+0000 mon.a (mon.0) 706 : audit [INF] from='mgr.14445 172.21.15.89:0/573502486' entity='mgr.y' 2023-06-03T19:34:31.783 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:34:31 smithi089 bash[26572]: audit 2023-06-03T19:34:31.350474+0000 mon.a (mon.0) 707 : audit [DBG] from='mgr.14445 172.21.15.89:0/573502486' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-03T19:34:31.783 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:34:31 smithi089 bash[33173]: cluster 2023-06-03T19:34:30.417966+0000 mgr.y (mgr.14445) 114 : cluster [DBG] pgmap v93: 73 pgs: 73 active+clean; 2.3 KiB data, 45 MiB used, 715 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2023-06-03T19:34:31.783 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:34:31 smithi089 bash[33173]: audit 2023-06-03T19:34:30.430036+0000 mon.a (mon.0) 702 : audit [INF] from='mgr.14445 172.21.15.89:0/573502486' entity='mgr.y' 2023-06-03T19:34:31.783 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:34:31 smithi089 bash[33173]: audit 2023-06-03T19:34:31.332785+0000 mon.a (mon.0) 703 : audit [INF] from='mgr.14445 172.21.15.89:0/573502486' entity='mgr.y' 2023-06-03T19:34:31.783 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:34:31 smithi089 bash[33173]: audit 2023-06-03T19:34:31.334312+0000 mon.a (mon.0) 704 : audit [DBG] from='mgr.14445 172.21.15.89:0/573502486' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-03T19:34:31.784 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:34:31 smithi089 bash[33173]: audit 2023-06-03T19:34:31.335443+0000 mon.a (mon.0) 705 : audit [INF] from='mgr.14445 172.21.15.89:0/573502486' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-03T19:34:31.784 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:34:31 smithi089 bash[33173]: audit 2023-06-03T19:34:31.341766+0000 mon.a (mon.0) 706 : audit [INF] from='mgr.14445 172.21.15.89:0/573502486' entity='mgr.y' 2023-06-03T19:34:31.784 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:34:31 smithi089 bash[33173]: audit 2023-06-03T19:34:31.350474+0000 mon.a (mon.0) 707 : audit [DBG] from='mgr.14445 172.21.15.89:0/573502486' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-03T19:34:31.836 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:34:31 smithi164 bash[26855]: cluster 2023-06-03T19:34:30.417966+0000 mgr.y (mgr.14445) 114 : cluster [DBG] pgmap v93: 73 pgs: 73 active+clean; 2.3 KiB data, 45 MiB used, 715 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2023-06-03T19:34:31.837 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:34:31 smithi164 bash[26855]: audit 2023-06-03T19:34:30.430036+0000 mon.a (mon.0) 702 : audit [INF] from='mgr.14445 172.21.15.89:0/573502486' entity='mgr.y' 2023-06-03T19:34:31.837 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:34:31 smithi164 bash[26855]: audit 2023-06-03T19:34:31.332785+0000 mon.a (mon.0) 703 : audit [INF] from='mgr.14445 172.21.15.89:0/573502486' entity='mgr.y' 2023-06-03T19:34:31.837 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:34:31 smithi164 bash[26855]: audit 2023-06-03T19:34:31.334312+0000 mon.a (mon.0) 704 : audit [DBG] from='mgr.14445 172.21.15.89:0/573502486' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-03T19:34:31.837 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:34:31 smithi164 bash[26855]: audit 2023-06-03T19:34:31.335443+0000 mon.a (mon.0) 705 : audit [INF] from='mgr.14445 172.21.15.89:0/573502486' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-03T19:34:31.837 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:34:31 smithi164 bash[26855]: audit 2023-06-03T19:34:31.341766+0000 mon.a (mon.0) 706 : audit [INF] from='mgr.14445 172.21.15.89:0/573502486' entity='mgr.y' 2023-06-03T19:34:31.837 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:34:31 smithi164 bash[26855]: audit 2023-06-03T19:34:31.350474+0000 mon.a (mon.0) 707 : audit [DBG] from='mgr.14445 172.21.15.89:0/573502486' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-03T19:34:33.031 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:34:32 smithi089 bash[33173]: audit 2023-06-03T19:34:31.622893+0000 mon.a (mon.0) 708 : audit [INF] from='mgr.14445 172.21.15.89:0/573502486' entity='mgr.y' 2023-06-03T19:34:33.032 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:34:32 smithi089 bash[33173]: audit 2023-06-03T19:34:31.624508+0000 mon.a (mon.0) 709 : audit [DBG] from='mgr.14445 172.21.15.89:0/573502486' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-03T19:34:33.032 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:34:32 smithi089 bash[33173]: audit 2023-06-03T19:34:31.625891+0000 mon.a (mon.0) 710 : audit [INF] from='mgr.14445 172.21.15.89:0/573502486' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-03T19:34:33.032 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:34:32 smithi089 bash[33173]: audit 2023-06-03T19:34:31.631878+0000 mon.a (mon.0) 711 : audit [INF] from='mgr.14445 172.21.15.89:0/573502486' entity='mgr.y' 2023-06-03T19:34:33.032 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:34:32 smithi089 bash[26572]: audit 2023-06-03T19:34:31.622893+0000 mon.a (mon.0) 708 : audit [INF] from='mgr.14445 172.21.15.89:0/573502486' entity='mgr.y' 2023-06-03T19:34:33.032 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:34:32 smithi089 bash[26572]: audit 2023-06-03T19:34:31.624508+0000 mon.a (mon.0) 709 : audit [DBG] from='mgr.14445 172.21.15.89:0/573502486' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-03T19:34:33.033 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:34:32 smithi089 bash[26572]: audit 2023-06-03T19:34:31.625891+0000 mon.a (mon.0) 710 : audit [INF] from='mgr.14445 172.21.15.89:0/573502486' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-03T19:34:33.033 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:34:32 smithi089 bash[26572]: audit 2023-06-03T19:34:31.631878+0000 mon.a (mon.0) 711 : audit [INF] from='mgr.14445 172.21.15.89:0/573502486' entity='mgr.y' 2023-06-03T19:34:33.086 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:34:32 smithi164 bash[26855]: audit 2023-06-03T19:34:31.622893+0000 mon.a (mon.0) 708 : audit [INF] from='mgr.14445 172.21.15.89:0/573502486' entity='mgr.y' 2023-06-03T19:34:33.087 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:34:32 smithi164 bash[26855]: audit 2023-06-03T19:34:31.624508+0000 mon.a (mon.0) 709 : audit [DBG] from='mgr.14445 172.21.15.89:0/573502486' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-03T19:34:33.087 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:34:32 smithi164 bash[26855]: audit 2023-06-03T19:34:31.625891+0000 mon.a (mon.0) 710 : audit [INF] from='mgr.14445 172.21.15.89:0/573502486' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-03T19:34:33.087 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:34:32 smithi164 bash[26855]: audit 2023-06-03T19:34:31.631878+0000 mon.a (mon.0) 711 : audit [INF] from='mgr.14445 172.21.15.89:0/573502486' entity='mgr.y' 2023-06-03T19:34:33.534 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/mon.a/config 2023-06-03T19:34:34.031 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:34:33 smithi089 bash[33173]: cluster 2023-06-03T19:34:32.418629+0000 mgr.y (mgr.14445) 115 : cluster [DBG] pgmap v94: 73 pgs: 73 active+clean; 2.3 KiB data, 45 MiB used, 715 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2023-06-03T19:34:34.032 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:34:33 smithi089 bash[33173]: audit 2023-06-03T19:34:33.622063+0000 mon.a (mon.0) 712 : audit [INF] from='mgr.14445 172.21.15.89:0/573502486' entity='mgr.y' 2023-06-03T19:34:34.032 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:34:33 smithi089 bash[26572]: cluster 2023-06-03T19:34:32.418629+0000 mgr.y (mgr.14445) 115 : cluster [DBG] pgmap v94: 73 pgs: 73 active+clean; 2.3 KiB data, 45 MiB used, 715 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2023-06-03T19:34:34.032 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:34:33 smithi089 bash[26572]: audit 2023-06-03T19:34:33.622063+0000 mon.a (mon.0) 712 : audit [INF] from='mgr.14445 172.21.15.89:0/573502486' entity='mgr.y' 2023-06-03T19:34:34.086 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:34:33 smithi164 bash[26855]: cluster 2023-06-03T19:34:32.418629+0000 mgr.y (mgr.14445) 115 : cluster [DBG] pgmap v94: 73 pgs: 73 active+clean; 2.3 KiB data, 45 MiB used, 715 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2023-06-03T19:34:34.087 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:34:33 smithi164 bash[26855]: audit 2023-06-03T19:34:33.622063+0000 mon.a (mon.0) 712 : audit [INF] from='mgr.14445 172.21.15.89:0/573502486' entity='mgr.y' 2023-06-03T19:34:36.031 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:34:35 smithi089 bash[33173]: cluster 2023-06-03T19:34:34.419570+0000 mgr.y (mgr.14445) 116 : cluster [DBG] pgmap v95: 73 pgs: 73 active+clean; 2.3 KiB data, 45 MiB used, 715 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2023-06-03T19:34:36.032 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:34:35 smithi089 bash[26572]: cluster 2023-06-03T19:34:34.419570+0000 mgr.y (mgr.14445) 116 : cluster [DBG] pgmap v95: 73 pgs: 73 active+clean; 2.3 KiB data, 45 MiB used, 715 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2023-06-03T19:34:36.087 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:34:35 smithi164 bash[26855]: cluster 2023-06-03T19:34:34.419570+0000 mgr.y (mgr.14445) 116 : cluster [DBG] pgmap v95: 73 pgs: 73 active+clean; 2.3 KiB data, 45 MiB used, 715 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2023-06-03T19:34:36.087 INFO:journalctl@ceph.mgr.x.smithi164.stdout:Jun 03 19:34:35 smithi164 bash[27934]: ::ffff:172.21.15.164 - - [03/Jun/2023:19:34:35] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2023-06-03T19:34:37.070 INFO:teuthology.orchestra.run.smithi089.stdout: 2023-06-03T19:34:37.816 INFO:teuthology.orchestra.run.smithi089.stdout:{"epoch":18,"active_gid":14445,"active_name":"y","active_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:6800","nonce":2009133143},{"type":"v1","addr":"172.21.15.89:6801","nonce":2009133143}]},"active_addr":"172.21.15.89:6801/2009133143","active_change":"2023-06-03T19:31:48.303850+0000","active_mgr_features":4540138314316775423,"available":true,"standbys":[{"gid":24355,"name":"x","mgr_features":4540138314316775423,"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2400","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"7","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","upmap"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.23.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/ceph-grafana:8.3.5","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"docker.io/library/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"docker.io/arcts/keepalived","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.3.1","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.33.4","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"cross_origin_url":{"name":"cross_origin_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"redirect_resolve_ip_addr":{"name":"redirect_resolve_ip_addr","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"str","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"str","level":"advanced","flags":0,"default_value":"30","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"str","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"str","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"noautoscale":{"name":"noautoscale","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"global autoscale flag","long_desc":"Option to turn on/off the autoscaler for all pools","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":1,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"str","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}]}],"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2400","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"7","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","upmap"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.23.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/ceph-grafana:8.3.5","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"docker.io/library/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"docker.io/arcts/keepalived","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.3.1","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.33.4","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"cross_origin_url":{"name":"cross_origin_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"redirect_resolve_ip_addr":{"name":"redirect_resolve_ip_addr","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"str","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"str","level":"advanced","flags":0,"default_value":"30","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"str","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"str","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"noautoscale":{"name":"noautoscale","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"global autoscale flag","long_desc":"Option to turn on/off the autoscaler for all pools","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":1,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"str","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}],"services":{"dashboard":"https://172.21.15.89:8443/","prometheus":"http://172.21.15.89:9283/"},"always_on_modules":{"nautilus":["balancer","crash","devicehealth","orchestrator_cli","progress","rbd_support","status","volumes"],"octopus":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"pacific":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"]},"last_failure_osd_epoch":51,"active_clients":[{"addrvec":[{"type":"v2","addr":"172.21.15.89:0","nonce":1591228500}]},{"addrvec":[{"type":"v2","addr":"172.21.15.89:0","nonce":2023636358}]}]} 2023-06-03T19:34:37.821 INFO:tasks.cephadm.ceph_manager.ceph:mgr available! 2023-06-03T19:34:37.821 INFO:tasks.cephadm.ceph_manager.ceph:waiting for all up 2023-06-03T19:34:37.822 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph osd dump --format=json 2023-06-03T19:34:37.907 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:34:37 smithi089 bash[33173]: cluster 2023-06-03T19:34:36.420218+0000 mgr.y (mgr.14445) 117 : cluster [DBG] pgmap v96: 73 pgs: 73 active+clean; 2.3 KiB data, 45 MiB used, 715 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2023-06-03T19:34:37.907 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:34:37 smithi089 bash[33173]: audit 2023-06-03T19:34:37.054054+0000 mon.a (mon.0) 713 : audit [DBG] from='client.? 172.21.15.89:0/376320047' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2023-06-03T19:34:37.907 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:34:37 smithi089 bash[26572]: cluster 2023-06-03T19:34:36.420218+0000 mgr.y (mgr.14445) 117 : cluster [DBG] pgmap v96: 73 pgs: 73 active+clean; 2.3 KiB data, 45 MiB used, 715 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2023-06-03T19:34:37.907 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:34:37 smithi089 bash[26572]: audit 2023-06-03T19:34:37.054054+0000 mon.a (mon.0) 713 : audit [DBG] from='client.? 172.21.15.89:0/376320047' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2023-06-03T19:34:38.086 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:34:37 smithi164 bash[26855]: cluster 2023-06-03T19:34:36.420218+0000 mgr.y (mgr.14445) 117 : cluster [DBG] pgmap v96: 73 pgs: 73 active+clean; 2.3 KiB data, 45 MiB used, 715 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2023-06-03T19:34:38.087 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:34:37 smithi164 bash[26855]: audit 2023-06-03T19:34:37.054054+0000 mon.a (mon.0) 713 : audit [DBG] from='client.? 172.21.15.89:0/376320047' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2023-06-03T19:34:38.281 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 19:34:37 smithi089 bash[26845]: ::ffff:172.21.15.164 - - [03/Jun/2023:19:34:37] "GET /metrics HTTP/1.1" 200 187844 "" "Prometheus/2.33.4" 2023-06-03T19:34:40.031 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:34:39 smithi089 bash[33173]: cluster 2023-06-03T19:34:38.420954+0000 mgr.y (mgr.14445) 118 : cluster [DBG] pgmap v97: 73 pgs: 73 active+clean; 2.3 KiB data, 45 MiB used, 715 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2023-06-03T19:34:40.032 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:34:39 smithi089 bash[26572]: cluster 2023-06-03T19:34:38.420954+0000 mgr.y (mgr.14445) 118 : cluster [DBG] pgmap v97: 73 pgs: 73 active+clean; 2.3 KiB data, 45 MiB used, 715 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2023-06-03T19:34:40.086 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:34:39 smithi164 bash[26855]: cluster 2023-06-03T19:34:38.420954+0000 mgr.y (mgr.14445) 118 : cluster [DBG] pgmap v97: 73 pgs: 73 active+clean; 2.3 KiB data, 45 MiB used, 715 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2023-06-03T19:34:42.031 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:34:41 smithi089 bash[33173]: cluster 2023-06-03T19:34:40.422070+0000 mgr.y (mgr.14445) 119 : cluster [DBG] pgmap v98: 73 pgs: 73 active+clean; 2.3 KiB data, 45 MiB used, 715 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2023-06-03T19:34:42.032 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:34:41 smithi089 bash[26572]: cluster 2023-06-03T19:34:40.422070+0000 mgr.y (mgr.14445) 119 : cluster [DBG] pgmap v98: 73 pgs: 73 active+clean; 2.3 KiB data, 45 MiB used, 715 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2023-06-03T19:34:42.086 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:34:41 smithi164 bash[26855]: cluster 2023-06-03T19:34:40.422070+0000 mgr.y (mgr.14445) 119 : cluster [DBG] pgmap v98: 73 pgs: 73 active+clean; 2.3 KiB data, 45 MiB used, 715 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2023-06-03T19:34:42.639 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/mon.a/config 2023-06-03T19:34:44.028 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:34:43 smithi089 bash[26572]: cluster 2023-06-03T19:34:42.422720+0000 mgr.y (mgr.14445) 120 : cluster [DBG] pgmap v99: 73 pgs: 73 active+clean; 2.3 KiB data, 45 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:34:44.029 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:34:43 smithi089 bash[33173]: cluster 2023-06-03T19:34:42.422720+0000 mgr.y (mgr.14445) 120 : cluster [DBG] pgmap v99: 73 pgs: 73 active+clean; 2.3 KiB data, 45 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:34:44.086 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:34:43 smithi164 bash[26855]: cluster 2023-06-03T19:34:42.422720+0000 mgr.y (mgr.14445) 120 : cluster [DBG] pgmap v99: 73 pgs: 73 active+clean; 2.3 KiB data, 45 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:34:45.781 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:34:45 smithi089 bash[26572]: cluster 2023-06-03T19:34:44.423741+0000 mgr.y (mgr.14445) 121 : cluster [DBG] pgmap v100: 73 pgs: 73 active+clean; 2.3 KiB data, 45 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:34:45.782 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:34:45 smithi089 bash[33173]: cluster 2023-06-03T19:34:44.423741+0000 mgr.y (mgr.14445) 121 : cluster [DBG] pgmap v100: 73 pgs: 73 active+clean; 2.3 KiB data, 45 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:34:45.949 INFO:teuthology.orchestra.run.smithi089.stdout: 2023-06-03T19:34:45.949 INFO:teuthology.orchestra.run.smithi089.stdout:{"epoch":60,"fsid":"f1db3478-0243-11ee-9b1d-001a4aab830c","created":"2023-06-03T19:22:54.447184+0000","modified":"2023-06-03T19:34:18.816360+0000","last_up_change":"2023-06-03T19:31:13.424226+0000","last_in_change":"2023-06-03T19:30:46.909676+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":4,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"pacific","pools":[{"pool":1,"pool_name":"device_health_metrics","create_time":"2023-06-03T19:27:28.100610+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"53","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr_devicehealth":{}}},{"pool":2,"pool_name":"rbd","create_time":"2023-06-03T19:32:20.641846+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"55","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}}},{"pool":3,"pool_name":"cephfs.foo.meta","create_time":"2023-06-03T19:34:15.554658+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"59","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"foo"}}},{"pool":4,"pool_name":"cephfs.foo.data","create_time":"2023-06-03T19:34:16.201294+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"60","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"cephfs":{"data":"foo"}}}],"osds":[{"osd":0,"uuid":"86063bfe-126e-46e7-93ed-c73813af165b","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":57,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:6802","nonce":1795715824},{"type":"v1","addr":"172.21.15.89:6803","nonce":1795715824}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:6804","nonce":1795715824},{"type":"v1","addr":"172.21.15.89:6805","nonce":1795715824}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:6808","nonce":1795715824},{"type":"v1","addr":"172.21.15.89:6809","nonce":1795715824}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:6806","nonce":1795715824},{"type":"v1","addr":"172.21.15.89:6807","nonce":1795715824}]},"public_addr":"172.21.15.89:6803/1795715824","cluster_addr":"172.21.15.89:6805/1795715824","heartbeat_back_addr":"172.21.15.89:6809/1795715824","heartbeat_front_addr":"172.21.15.89:6807/1795715824","state":["exists","up"]},{"osd":1,"uuid":"04783a81-6b8c-4657-8446-00efc872b313","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":57,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:6810","nonce":4020168282},{"type":"v1","addr":"172.21.15.89:6811","nonce":4020168282}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:6812","nonce":4020168282},{"type":"v1","addr":"172.21.15.89:6813","nonce":4020168282}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:6816","nonce":4020168282},{"type":"v1","addr":"172.21.15.89:6817","nonce":4020168282}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:6814","nonce":4020168282},{"type":"v1","addr":"172.21.15.89:6815","nonce":4020168282}]},"public_addr":"172.21.15.89:6811/4020168282","cluster_addr":"172.21.15.89:6813/4020168282","heartbeat_back_addr":"172.21.15.89:6817/4020168282","heartbeat_front_addr":"172.21.15.89:6815/4020168282","state":["exists","up"]},{"osd":2,"uuid":"d2d2c641-392a-444f-81bc-84eef6340f30","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":57,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:6818","nonce":2113694583},{"type":"v1","addr":"172.21.15.89:6819","nonce":2113694583}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:6820","nonce":2113694583},{"type":"v1","addr":"172.21.15.89:6821","nonce":2113694583}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:6824","nonce":2113694583},{"type":"v1","addr":"172.21.15.89:6825","nonce":2113694583}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:6822","nonce":2113694583},{"type":"v1","addr":"172.21.15.89:6823","nonce":2113694583}]},"public_addr":"172.21.15.89:6819/2113694583","cluster_addr":"172.21.15.89:6821/2113694583","heartbeat_back_addr":"172.21.15.89:6825/2113694583","heartbeat_front_addr":"172.21.15.89:6823/2113694583","state":["exists","up"]},{"osd":3,"uuid":"b38408eb-3acf-4a41-a314-72e4cd37b1ef","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":25,"up_thru":57,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:6826","nonce":3649114317},{"type":"v1","addr":"172.21.15.89:6827","nonce":3649114317}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:6828","nonce":3649114317},{"type":"v1","addr":"172.21.15.89:6829","nonce":3649114317}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:6832","nonce":3649114317},{"type":"v1","addr":"172.21.15.89:6833","nonce":3649114317}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:6830","nonce":3649114317},{"type":"v1","addr":"172.21.15.89:6831","nonce":3649114317}]},"public_addr":"172.21.15.89:6827/3649114317","cluster_addr":"172.21.15.89:6829/3649114317","heartbeat_back_addr":"172.21.15.89:6833/3649114317","heartbeat_front_addr":"172.21.15.89:6831/3649114317","state":["exists","up"]},{"osd":4,"uuid":"a94be785-9f54-4ab8-962a-9cf974a5b5e3","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":30,"up_thru":57,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.164:6800","nonce":538382489},{"type":"v1","addr":"172.21.15.164:6801","nonce":538382489}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.164:6802","nonce":538382489},{"type":"v1","addr":"172.21.15.164:6803","nonce":538382489}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.164:6806","nonce":538382489},{"type":"v1","addr":"172.21.15.164:6807","nonce":538382489}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.164:6804","nonce":538382489},{"type":"v1","addr":"172.21.15.164:6805","nonce":538382489}]},"public_addr":"172.21.15.164:6801/538382489","cluster_addr":"172.21.15.164:6803/538382489","heartbeat_back_addr":"172.21.15.164:6807/538382489","heartbeat_front_addr":"172.21.15.164:6805/538382489","state":["exists","up"]},{"osd":5,"uuid":"b8a5e8a1-23cc-442b-90aa-19bd0036dd3c","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":36,"up_thru":57,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.164:6808","nonce":3123006405},{"type":"v1","addr":"172.21.15.164:6809","nonce":3123006405}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.164:6810","nonce":3123006405},{"type":"v1","addr":"172.21.15.164:6811","nonce":3123006405}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.164:6814","nonce":3123006405},{"type":"v1","addr":"172.21.15.164:6815","nonce":3123006405}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.164:6812","nonce":3123006405},{"type":"v1","addr":"172.21.15.164:6813","nonce":3123006405}]},"public_addr":"172.21.15.164:6809/3123006405","cluster_addr":"172.21.15.164:6811/3123006405","heartbeat_back_addr":"172.21.15.164:6815/3123006405","heartbeat_front_addr":"172.21.15.164:6813/3123006405","state":["exists","up"]},{"osd":6,"uuid":"5c5edd3b-d92c-4f46-94ff-2d5828b2e732","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":42,"up_thru":57,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.164:6816","nonce":712403290},{"type":"v1","addr":"172.21.15.164:6817","nonce":712403290}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.164:6818","nonce":712403290},{"type":"v1","addr":"172.21.15.164:6819","nonce":712403290}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.164:6822","nonce":712403290},{"type":"v1","addr":"172.21.15.164:6823","nonce":712403290}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.164:6820","nonce":712403290},{"type":"v1","addr":"172.21.15.164:6821","nonce":712403290}]},"public_addr":"172.21.15.164:6817/712403290","cluster_addr":"172.21.15.164:6819/712403290","heartbeat_back_addr":"172.21.15.164:6823/712403290","heartbeat_front_addr":"172.21.15.164:6821/712403290","state":["exists","up"]},{"osd":7,"uuid":"bdff9290-e6c4-407e-9113-eb39696c0e27","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":48,"up_thru":56,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.164:6824","nonce":711997991},{"type":"v1","addr":"172.21.15.164:6825","nonce":711997991}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.164:6826","nonce":711997991},{"type":"v1","addr":"172.21.15.164:6827","nonce":711997991}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.164:6830","nonce":711997991},{"type":"v1","addr":"172.21.15.164:6831","nonce":711997991}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.164:6828","nonce":711997991},{"type":"v1","addr":"172.21.15.164:6829","nonce":711997991}]},"public_addr":"172.21.15.164:6825/711997991","cluster_addr":"172.21.15.164:6827/711997991","heartbeat_back_addr":"172.21.15.164:6831/711997991","heartbeat_front_addr":"172.21.15.164:6829/711997991","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-06-03T19:25:52.155714+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-06-03T19:26:38.554458+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-06-03T19:27:25.383237+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-06-03T19:28:13.272585+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-06-03T19:28:56.685710+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-06-03T19:29:41.572533+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-06-03T19:30:26.631983+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-06-03T19:31:12.018414+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.89:0/3944267183":"2023-06-04T19:23:55.626873+0000","172.21.15.89:0/470853232":"2023-06-04T19:31:48.303240+0000","172.21.15.89:6801/4116589411":"2023-06-04T19:23:55.626873+0000","172.21.15.89:6801/2256305550":"2023-06-04T19:31:48.303240+0000","172.21.15.89:6800/4116589411":"2023-06-04T19:23:55.626873+0000","172.21.15.89:0/274274920":"2023-06-04T19:23:55.626873+0000","172.21.15.89:0/1198702890":"2023-06-04T19:23:23.903333+0000","172.21.15.89:6800/1487566136":"2023-06-04T19:23:23.903333+0000","172.21.15.89:6800/2256305550":"2023-06-04T19:31:48.303240+0000","172.21.15.89:0/2392321634":"2023-06-04T19:31:48.303240+0000","172.21.15.89:6801/1487566136":"2023-06-04T19:23:23.903333+0000","172.21.15.89:0/811967393":"2023-06-04T19:23:23.903333+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2023-06-03T19:34:46.086 INFO:journalctl@ceph.mgr.x.smithi164.stdout:Jun 03 19:34:45 smithi164 bash[27934]: ::ffff:172.21.15.164 - - [03/Jun/2023:19:34:45] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2023-06-03T19:34:46.087 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:34:45 smithi164 bash[26855]: cluster 2023-06-03T19:34:44.423741+0000 mgr.y (mgr.14445) 121 : cluster [DBG] pgmap v100: 73 pgs: 73 active+clean; 2.3 KiB data, 45 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:34:46.705 INFO:tasks.cephadm.ceph_manager.ceph:all up! 2023-06-03T19:34:46.706 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph osd dump --format=json 2023-06-03T19:34:47.031 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:34:46 smithi089 bash[26572]: audit 2023-06-03T19:34:45.939313+0000 mon.c (mon.2) 23 : audit [DBG] from='client.? 172.21.15.89:0/1562446231' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-06-03T19:34:47.032 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:34:46 smithi089 bash[33173]: audit 2023-06-03T19:34:45.939313+0000 mon.c (mon.2) 23 : audit [DBG] from='client.? 172.21.15.89:0/1562446231' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-06-03T19:34:47.086 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:34:46 smithi164 bash[26855]: audit 2023-06-03T19:34:45.939313+0000 mon.c (mon.2) 23 : audit [DBG] from='client.? 172.21.15.89:0/1562446231' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-06-03T19:34:48.031 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:34:47 smithi089 bash[26572]: cluster 2023-06-03T19:34:46.424188+0000 mgr.y (mgr.14445) 122 : cluster [DBG] pgmap v101: 73 pgs: 73 active+clean; 2.3 KiB data, 45 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:34:48.032 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:34:47 smithi089 bash[33173]: cluster 2023-06-03T19:34:46.424188+0000 mgr.y (mgr.14445) 122 : cluster [DBG] pgmap v101: 73 pgs: 73 active+clean; 2.3 KiB data, 45 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:34:48.032 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 19:34:47 smithi089 bash[26845]: ::ffff:172.21.15.164 - - [03/Jun/2023:19:34:47] "GET /metrics HTTP/1.1" 200 187844 "" "Prometheus/2.33.4" 2023-06-03T19:34:48.086 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:34:47 smithi164 bash[26855]: cluster 2023-06-03T19:34:46.424188+0000 mgr.y (mgr.14445) 122 : cluster [DBG] pgmap v101: 73 pgs: 73 active+clean; 2.3 KiB data, 45 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:34:49.031 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:34:48 smithi089 bash[33173]: audit 2023-06-03T19:34:48.420254+0000 mon.a (mon.0) 714 : audit [INF] from='mgr.14445 172.21.15.89:0/573502486' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.9", "id": [1, 2]}]: dispatch 2023-06-03T19:34:49.032 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:34:48 smithi089 bash[33173]: audit 2023-06-03T19:34:48.421046+0000 mon.a (mon.0) 715 : audit [INF] from='mgr.14445 172.21.15.89:0/573502486' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.b", "id": [1, 2]}]: dispatch 2023-06-03T19:34:49.032 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:34:48 smithi089 bash[33173]: audit 2023-06-03T19:34:48.421676+0000 mon.a (mon.0) 716 : audit [INF] from='mgr.14445 172.21.15.89:0/573502486' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.d", "id": [1, 5]}]: dispatch 2023-06-03T19:34:49.032 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:34:48 smithi089 bash[26572]: audit 2023-06-03T19:34:48.420254+0000 mon.a (mon.0) 714 : audit [INF] from='mgr.14445 172.21.15.89:0/573502486' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.9", "id": [1, 2]}]: dispatch 2023-06-03T19:34:49.032 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:34:48 smithi089 bash[26572]: audit 2023-06-03T19:34:48.421046+0000 mon.a (mon.0) 715 : audit [INF] from='mgr.14445 172.21.15.89:0/573502486' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.b", "id": [1, 2]}]: dispatch 2023-06-03T19:34:49.032 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:34:48 smithi089 bash[26572]: audit 2023-06-03T19:34:48.421676+0000 mon.a (mon.0) 716 : audit [INF] from='mgr.14445 172.21.15.89:0/573502486' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.d", "id": [1, 5]}]: dispatch 2023-06-03T19:34:49.086 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:34:48 smithi164 bash[26855]: audit 2023-06-03T19:34:48.420254+0000 mon.a (mon.0) 714 : audit [INF] from='mgr.14445 172.21.15.89:0/573502486' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.9", "id": [1, 2]}]: dispatch 2023-06-03T19:34:49.087 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:34:48 smithi164 bash[26855]: audit 2023-06-03T19:34:48.421046+0000 mon.a (mon.0) 715 : audit [INF] from='mgr.14445 172.21.15.89:0/573502486' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.b", "id": [1, 2]}]: dispatch 2023-06-03T19:34:49.087 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:34:48 smithi164 bash[26855]: audit 2023-06-03T19:34:48.421676+0000 mon.a (mon.0) 716 : audit [INF] from='mgr.14445 172.21.15.89:0/573502486' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.d", "id": [1, 5]}]: dispatch 2023-06-03T19:34:50.031 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:34:49 smithi089 bash[26572]: cluster 2023-06-03T19:34:48.424836+0000 mgr.y (mgr.14445) 123 : cluster [DBG] pgmap v102: 73 pgs: 73 active+clean; 2.3 KiB data, 45 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:34:50.032 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:34:49 smithi089 bash[26572]: audit 2023-06-03T19:34:48.688135+0000 mon.a (mon.0) 717 : audit [INF] from='mgr.14445 172.21.15.89:0/573502486' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.9", "id": [1, 2]}]': finished 2023-06-03T19:34:50.032 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:34:49 smithi089 bash[26572]: audit 2023-06-03T19:34:48.688250+0000 mon.a (mon.0) 718 : audit [INF] from='mgr.14445 172.21.15.89:0/573502486' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.b", "id": [1, 2]}]': finished 2023-06-03T19:34:50.032 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:34:49 smithi089 bash[26572]: audit 2023-06-03T19:34:48.688341+0000 mon.a (mon.0) 719 : audit [INF] from='mgr.14445 172.21.15.89:0/573502486' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.d", "id": [1, 5]}]': finished 2023-06-03T19:34:50.032 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:34:49 smithi089 bash[26572]: cluster 2023-06-03T19:34:48.688423+0000 mon.a (mon.0) 720 : cluster [DBG] osdmap e61: 8 total, 8 up, 8 in 2023-06-03T19:34:50.032 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:34:49 smithi089 bash[33173]: cluster 2023-06-03T19:34:48.424836+0000 mgr.y (mgr.14445) 123 : cluster [DBG] pgmap v102: 73 pgs: 73 active+clean; 2.3 KiB data, 45 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:34:50.033 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:34:49 smithi089 bash[33173]: audit 2023-06-03T19:34:48.688135+0000 mon.a (mon.0) 717 : audit [INF] from='mgr.14445 172.21.15.89:0/573502486' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.9", "id": [1, 2]}]': finished 2023-06-03T19:34:50.033 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:34:49 smithi089 bash[33173]: audit 2023-06-03T19:34:48.688250+0000 mon.a (mon.0) 718 : audit [INF] from='mgr.14445 172.21.15.89:0/573502486' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.b", "id": [1, 2]}]': finished 2023-06-03T19:34:50.033 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:34:49 smithi089 bash[33173]: audit 2023-06-03T19:34:48.688341+0000 mon.a (mon.0) 719 : audit [INF] from='mgr.14445 172.21.15.89:0/573502486' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.d", "id": [1, 5]}]': finished 2023-06-03T19:34:50.033 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:34:49 smithi089 bash[33173]: cluster 2023-06-03T19:34:48.688423+0000 mon.a (mon.0) 720 : cluster [DBG] osdmap e61: 8 total, 8 up, 8 in 2023-06-03T19:34:50.086 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:34:49 smithi164 bash[26855]: cluster 2023-06-03T19:34:48.424836+0000 mgr.y (mgr.14445) 123 : cluster [DBG] pgmap v102: 73 pgs: 73 active+clean; 2.3 KiB data, 45 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:34:50.087 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:34:49 smithi164 bash[26855]: audit 2023-06-03T19:34:48.688135+0000 mon.a (mon.0) 717 : audit [INF] from='mgr.14445 172.21.15.89:0/573502486' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.9", "id": [1, 2]}]': finished 2023-06-03T19:34:50.087 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:34:49 smithi164 bash[26855]: audit 2023-06-03T19:34:48.688250+0000 mon.a (mon.0) 718 : audit [INF] from='mgr.14445 172.21.15.89:0/573502486' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.b", "id": [1, 2]}]': finished 2023-06-03T19:34:50.087 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:34:49 smithi164 bash[26855]: audit 2023-06-03T19:34:48.688341+0000 mon.a (mon.0) 719 : audit [INF] from='mgr.14445 172.21.15.89:0/573502486' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.d", "id": [1, 5]}]': finished 2023-06-03T19:34:50.087 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:34:49 smithi164 bash[26855]: cluster 2023-06-03T19:34:48.688423+0000 mon.a (mon.0) 720 : cluster [DBG] osdmap e61: 8 total, 8 up, 8 in 2023-06-03T19:34:51.031 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:34:50 smithi089 bash[33173]: cluster 2023-06-03T19:34:49.692353+0000 mon.a (mon.0) 721 : cluster [DBG] osdmap e62: 8 total, 8 up, 8 in 2023-06-03T19:34:51.032 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:34:50 smithi089 bash[26572]: cluster 2023-06-03T19:34:49.692353+0000 mon.a (mon.0) 721 : cluster [DBG] osdmap e62: 8 total, 8 up, 8 in 2023-06-03T19:34:51.087 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:34:50 smithi164 bash[26855]: cluster 2023-06-03T19:34:49.692353+0000 mon.a (mon.0) 721 : cluster [DBG] osdmap e62: 8 total, 8 up, 8 in 2023-06-03T19:34:51.523 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/mon.a/config 2023-06-03T19:34:52.031 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:34:51 smithi089 bash[33173]: cluster 2023-06-03T19:34:50.425626+0000 mgr.y (mgr.14445) 124 : cluster [DBG] pgmap v105: 73 pgs: 3 peering, 70 active+clean; 2.3 KiB data, 45 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:34:52.032 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:34:51 smithi089 bash[26572]: cluster 2023-06-03T19:34:50.425626+0000 mgr.y (mgr.14445) 124 : cluster [DBG] pgmap v105: 73 pgs: 3 peering, 70 active+clean; 2.3 KiB data, 45 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:34:52.086 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:34:51 smithi164 bash[26855]: cluster 2023-06-03T19:34:50.425626+0000 mgr.y (mgr.14445) 124 : cluster [DBG] pgmap v105: 73 pgs: 3 peering, 70 active+clean; 2.3 KiB data, 45 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:34:54.032 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:34:53 smithi089 bash[33173]: cluster 2023-06-03T19:34:52.426339+0000 mgr.y (mgr.14445) 125 : cluster [DBG] pgmap v106: 73 pgs: 3 peering, 70 active+clean; 2.3 KiB data, 45 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:34:54.032 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:34:53 smithi089 bash[26572]: cluster 2023-06-03T19:34:52.426339+0000 mgr.y (mgr.14445) 125 : cluster [DBG] pgmap v106: 73 pgs: 3 peering, 70 active+clean; 2.3 KiB data, 45 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:34:54.087 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:34:53 smithi164 bash[26855]: cluster 2023-06-03T19:34:52.426339+0000 mgr.y (mgr.14445) 125 : cluster [DBG] pgmap v106: 73 pgs: 3 peering, 70 active+clean; 2.3 KiB data, 45 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:34:55.131 INFO:teuthology.orchestra.run.smithi089.stdout: 2023-06-03T19:34:55.131 INFO:teuthology.orchestra.run.smithi089.stdout:{"epoch":62,"fsid":"f1db3478-0243-11ee-9b1d-001a4aab830c","created":"2023-06-03T19:22:54.447184+0000","modified":"2023-06-03T19:34:49.683813+0000","last_up_change":"2023-06-03T19:31:13.424226+0000","last_in_change":"2023-06-03T19:30:46.909676+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":4,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"luminous","require_osd_release":"pacific","pools":[{"pool":1,"pool_name":"device_health_metrics","create_time":"2023-06-03T19:27:28.100610+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"53","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr_devicehealth":{}}},{"pool":2,"pool_name":"rbd","create_time":"2023-06-03T19:32:20.641846+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"55","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}}},{"pool":3,"pool_name":"cephfs.foo.meta","create_time":"2023-06-03T19:34:15.554658+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"59","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"foo"}}},{"pool":4,"pool_name":"cephfs.foo.data","create_time":"2023-06-03T19:34:16.201294+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"60","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"cephfs":{"data":"foo"}}}],"osds":[{"osd":0,"uuid":"86063bfe-126e-46e7-93ed-c73813af165b","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":61,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:6802","nonce":1795715824},{"type":"v1","addr":"172.21.15.89:6803","nonce":1795715824}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:6804","nonce":1795715824},{"type":"v1","addr":"172.21.15.89:6805","nonce":1795715824}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:6808","nonce":1795715824},{"type":"v1","addr":"172.21.15.89:6809","nonce":1795715824}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:6806","nonce":1795715824},{"type":"v1","addr":"172.21.15.89:6807","nonce":1795715824}]},"public_addr":"172.21.15.89:6803/1795715824","cluster_addr":"172.21.15.89:6805/1795715824","heartbeat_back_addr":"172.21.15.89:6809/1795715824","heartbeat_front_addr":"172.21.15.89:6807/1795715824","state":["exists","up"]},{"osd":1,"uuid":"04783a81-6b8c-4657-8446-00efc872b313","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":57,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:6810","nonce":4020168282},{"type":"v1","addr":"172.21.15.89:6811","nonce":4020168282}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:6812","nonce":4020168282},{"type":"v1","addr":"172.21.15.89:6813","nonce":4020168282}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:6816","nonce":4020168282},{"type":"v1","addr":"172.21.15.89:6817","nonce":4020168282}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:6814","nonce":4020168282},{"type":"v1","addr":"172.21.15.89:6815","nonce":4020168282}]},"public_addr":"172.21.15.89:6811/4020168282","cluster_addr":"172.21.15.89:6813/4020168282","heartbeat_back_addr":"172.21.15.89:6817/4020168282","heartbeat_front_addr":"172.21.15.89:6815/4020168282","state":["exists","up"]},{"osd":2,"uuid":"d2d2c641-392a-444f-81bc-84eef6340f30","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":57,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:6818","nonce":2113694583},{"type":"v1","addr":"172.21.15.89:6819","nonce":2113694583}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:6820","nonce":2113694583},{"type":"v1","addr":"172.21.15.89:6821","nonce":2113694583}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:6824","nonce":2113694583},{"type":"v1","addr":"172.21.15.89:6825","nonce":2113694583}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:6822","nonce":2113694583},{"type":"v1","addr":"172.21.15.89:6823","nonce":2113694583}]},"public_addr":"172.21.15.89:6819/2113694583","cluster_addr":"172.21.15.89:6821/2113694583","heartbeat_back_addr":"172.21.15.89:6825/2113694583","heartbeat_front_addr":"172.21.15.89:6823/2113694583","state":["exists","up"]},{"osd":3,"uuid":"b38408eb-3acf-4a41-a314-72e4cd37b1ef","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":25,"up_thru":57,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:6826","nonce":3649114317},{"type":"v1","addr":"172.21.15.89:6827","nonce":3649114317}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:6828","nonce":3649114317},{"type":"v1","addr":"172.21.15.89:6829","nonce":3649114317}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:6832","nonce":3649114317},{"type":"v1","addr":"172.21.15.89:6833","nonce":3649114317}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:6830","nonce":3649114317},{"type":"v1","addr":"172.21.15.89:6831","nonce":3649114317}]},"public_addr":"172.21.15.89:6827/3649114317","cluster_addr":"172.21.15.89:6829/3649114317","heartbeat_back_addr":"172.21.15.89:6833/3649114317","heartbeat_front_addr":"172.21.15.89:6831/3649114317","state":["exists","up"]},{"osd":4,"uuid":"a94be785-9f54-4ab8-962a-9cf974a5b5e3","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":30,"up_thru":61,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.164:6800","nonce":538382489},{"type":"v1","addr":"172.21.15.164:6801","nonce":538382489}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.164:6802","nonce":538382489},{"type":"v1","addr":"172.21.15.164:6803","nonce":538382489}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.164:6806","nonce":538382489},{"type":"v1","addr":"172.21.15.164:6807","nonce":538382489}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.164:6804","nonce":538382489},{"type":"v1","addr":"172.21.15.164:6805","nonce":538382489}]},"public_addr":"172.21.15.164:6801/538382489","cluster_addr":"172.21.15.164:6803/538382489","heartbeat_back_addr":"172.21.15.164:6807/538382489","heartbeat_front_addr":"172.21.15.164:6805/538382489","state":["exists","up"]},{"osd":5,"uuid":"b8a5e8a1-23cc-442b-90aa-19bd0036dd3c","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":36,"up_thru":57,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.164:6808","nonce":3123006405},{"type":"v1","addr":"172.21.15.164:6809","nonce":3123006405}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.164:6810","nonce":3123006405},{"type":"v1","addr":"172.21.15.164:6811","nonce":3123006405}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.164:6814","nonce":3123006405},{"type":"v1","addr":"172.21.15.164:6815","nonce":3123006405}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.164:6812","nonce":3123006405},{"type":"v1","addr":"172.21.15.164:6813","nonce":3123006405}]},"public_addr":"172.21.15.164:6809/3123006405","cluster_addr":"172.21.15.164:6811/3123006405","heartbeat_back_addr":"172.21.15.164:6815/3123006405","heartbeat_front_addr":"172.21.15.164:6813/3123006405","state":["exists","up"]},{"osd":6,"uuid":"5c5edd3b-d92c-4f46-94ff-2d5828b2e732","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":42,"up_thru":57,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.164:6816","nonce":712403290},{"type":"v1","addr":"172.21.15.164:6817","nonce":712403290}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.164:6818","nonce":712403290},{"type":"v1","addr":"172.21.15.164:6819","nonce":712403290}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.164:6822","nonce":712403290},{"type":"v1","addr":"172.21.15.164:6823","nonce":712403290}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.164:6820","nonce":712403290},{"type":"v1","addr":"172.21.15.164:6821","nonce":712403290}]},"public_addr":"172.21.15.164:6817/712403290","cluster_addr":"172.21.15.164:6819/712403290","heartbeat_back_addr":"172.21.15.164:6823/712403290","heartbeat_front_addr":"172.21.15.164:6821/712403290","state":["exists","up"]},{"osd":7,"uuid":"bdff9290-e6c4-407e-9113-eb39696c0e27","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":48,"up_thru":56,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.164:6824","nonce":711997991},{"type":"v1","addr":"172.21.15.164:6825","nonce":711997991}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.164:6826","nonce":711997991},{"type":"v1","addr":"172.21.15.164:6827","nonce":711997991}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.164:6830","nonce":711997991},{"type":"v1","addr":"172.21.15.164:6831","nonce":711997991}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.164:6828","nonce":711997991},{"type":"v1","addr":"172.21.15.164:6829","nonce":711997991}]},"public_addr":"172.21.15.164:6825/711997991","cluster_addr":"172.21.15.164:6827/711997991","heartbeat_back_addr":"172.21.15.164:6831/711997991","heartbeat_front_addr":"172.21.15.164:6829/711997991","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-06-03T19:25:52.155714+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-06-03T19:26:38.554458+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-06-03T19:27:25.383237+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-06-03T19:28:13.272585+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-06-03T19:28:56.685710+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-06-03T19:29:41.572533+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-06-03T19:30:26.631983+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-06-03T19:31:12.018414+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[{"pgid":"4.9","mappings":[{"from":1,"to":2}]},{"pgid":"4.b","mappings":[{"from":1,"to":2}]},{"pgid":"4.d","mappings":[{"from":1,"to":5}]}],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.89:0/3944267183":"2023-06-04T19:23:55.626873+0000","172.21.15.89:0/470853232":"2023-06-04T19:31:48.303240+0000","172.21.15.89:6801/4116589411":"2023-06-04T19:23:55.626873+0000","172.21.15.89:6801/2256305550":"2023-06-04T19:31:48.303240+0000","172.21.15.89:6800/4116589411":"2023-06-04T19:23:55.626873+0000","172.21.15.89:0/274274920":"2023-06-04T19:23:55.626873+0000","172.21.15.89:0/1198702890":"2023-06-04T19:23:23.903333+0000","172.21.15.89:6800/1487566136":"2023-06-04T19:23:23.903333+0000","172.21.15.89:6800/2256305550":"2023-06-04T19:31:48.303240+0000","172.21.15.89:0/2392321634":"2023-06-04T19:31:48.303240+0000","172.21.15.89:6801/1487566136":"2023-06-04T19:23:23.903333+0000","172.21.15.89:0/811967393":"2023-06-04T19:23:23.903333+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2023-06-03T19:34:55.917 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph tell osd.0 flush_pg_stats 2023-06-03T19:34:55.918 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph tell osd.1 flush_pg_stats 2023-06-03T19:34:55.918 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph tell osd.2 flush_pg_stats 2023-06-03T19:34:55.919 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph tell osd.3 flush_pg_stats 2023-06-03T19:34:55.919 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph tell osd.4 flush_pg_stats 2023-06-03T19:34:55.919 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph tell osd.5 flush_pg_stats 2023-06-03T19:34:55.920 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph tell osd.6 flush_pg_stats 2023-06-03T19:34:55.920 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph tell osd.7 flush_pg_stats 2023-06-03T19:34:56.031 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:34:55 smithi089 bash[26572]: cluster 2023-06-03T19:34:54.427129+0000 mgr.y (mgr.14445) 126 : cluster [DBG] pgmap v107: 73 pgs: 73 active+clean; 2.3 KiB data, 45 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:34:56.032 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:34:55 smithi089 bash[26572]: audit 2023-06-03T19:34:55.121241+0000 mon.a (mon.0) 722 : audit [DBG] from='client.? 172.21.15.89:0/810196015' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-06-03T19:34:56.032 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:34:55 smithi089 bash[33173]: cluster 2023-06-03T19:34:54.427129+0000 mgr.y (mgr.14445) 126 : cluster [DBG] pgmap v107: 73 pgs: 73 active+clean; 2.3 KiB data, 45 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:34:56.032 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:34:55 smithi089 bash[33173]: audit 2023-06-03T19:34:55.121241+0000 mon.a (mon.0) 722 : audit [DBG] from='client.? 172.21.15.89:0/810196015' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-06-03T19:34:56.086 INFO:journalctl@ceph.mgr.x.smithi164.stdout:Jun 03 19:34:55 smithi164 bash[27934]: ::ffff:172.21.15.164 - - [03/Jun/2023:19:34:55] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2023-06-03T19:34:56.087 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:34:55 smithi164 bash[26855]: cluster 2023-06-03T19:34:54.427129+0000 mgr.y (mgr.14445) 126 : cluster [DBG] pgmap v107: 73 pgs: 73 active+clean; 2.3 KiB data, 45 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:34:56.087 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:34:55 smithi164 bash[26855]: audit 2023-06-03T19:34:55.121241+0000 mon.a (mon.0) 722 : audit [DBG] from='client.? 172.21.15.89:0/810196015' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-06-03T19:34:58.031 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 19:34:57 smithi089 bash[26845]: ::ffff:172.21.15.164 - - [03/Jun/2023:19:34:57] "GET /metrics HTTP/1.1" 200 187840 "" "Prometheus/2.33.4" 2023-06-03T19:34:58.032 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:34:57 smithi089 bash[33173]: cluster 2023-06-03T19:34:56.427686+0000 mgr.y (mgr.14445) 127 : cluster [DBG] pgmap v108: 73 pgs: 73 active+clean; 2.3 KiB data, 45 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:34:58.032 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:34:57 smithi089 bash[26572]: cluster 2023-06-03T19:34:56.427686+0000 mgr.y (mgr.14445) 127 : cluster [DBG] pgmap v108: 73 pgs: 73 active+clean; 2.3 KiB data, 45 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:34:58.086 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:34:57 smithi164 bash[26855]: cluster 2023-06-03T19:34:56.427686+0000 mgr.y (mgr.14445) 127 : cluster [DBG] pgmap v108: 73 pgs: 73 active+clean; 2.3 KiB data, 45 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:35:00.031 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:34:59 smithi089 bash[33173]: cluster 2023-06-03T19:34:58.428340+0000 mgr.y (mgr.14445) 128 : cluster [DBG] pgmap v109: 73 pgs: 73 active+clean; 2.3 KiB data, 45 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:35:00.032 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:34:59 smithi089 bash[26572]: cluster 2023-06-03T19:34:58.428340+0000 mgr.y (mgr.14445) 128 : cluster [DBG] pgmap v109: 73 pgs: 73 active+clean; 2.3 KiB data, 45 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:35:00.087 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:34:59 smithi164 bash[26855]: cluster 2023-06-03T19:34:58.428340+0000 mgr.y (mgr.14445) 128 : cluster [DBG] pgmap v109: 73 pgs: 73 active+clean; 2.3 KiB data, 45 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:35:00.852 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/mon.a/config 2023-06-03T19:35:00.853 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/mon.a/config 2023-06-03T19:35:00.853 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/mon.a/config 2023-06-03T19:35:00.854 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/mon.a/config 2023-06-03T19:35:00.854 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/mon.a/config 2023-06-03T19:35:00.854 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/mon.a/config 2023-06-03T19:35:00.855 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/mon.a/config 2023-06-03T19:35:00.859 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/mon.a/config 2023-06-03T19:35:02.031 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:35:01 smithi089 bash[26572]: cluster 2023-06-03T19:35:00.429499+0000 mgr.y (mgr.14445) 129 : cluster [DBG] pgmap v110: 73 pgs: 73 active+clean; 2.3 KiB data, 45 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:35:02.032 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:35:01 smithi089 bash[33173]: cluster 2023-06-03T19:35:00.429499+0000 mgr.y (mgr.14445) 129 : cluster [DBG] pgmap v110: 73 pgs: 73 active+clean; 2.3 KiB data, 45 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:35:02.087 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:35:01 smithi164 bash[26855]: cluster 2023-06-03T19:35:00.429499+0000 mgr.y (mgr.14445) 129 : cluster [DBG] pgmap v110: 73 pgs: 73 active+clean; 2.3 KiB data, 45 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:35:03.887 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:35:03 smithi089 bash[26572]: cluster 2023-06-03T19:35:02.430179+0000 mgr.y (mgr.14445) 130 : cluster [DBG] pgmap v111: 73 pgs: 73 active+clean; 2.3 KiB data, 45 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:35:03.887 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:35:03 smithi089 bash[33173]: cluster 2023-06-03T19:35:02.430179+0000 mgr.y (mgr.14445) 130 : cluster [DBG] pgmap v111: 73 pgs: 73 active+clean; 2.3 KiB data, 45 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:35:04.086 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:35:03 smithi164 bash[26855]: cluster 2023-06-03T19:35:02.430179+0000 mgr.y (mgr.14445) 130 : cluster [DBG] pgmap v111: 73 pgs: 73 active+clean; 2.3 KiB data, 45 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:35:06.031 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:35:05 smithi089 bash[33173]: cluster 2023-06-03T19:35:04.430808+0000 mgr.y (mgr.14445) 131 : cluster [DBG] pgmap v112: 73 pgs: 73 active+clean; 2.3 KiB data, 45 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:35:06.032 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:35:05 smithi089 bash[26572]: cluster 2023-06-03T19:35:04.430808+0000 mgr.y (mgr.14445) 131 : cluster [DBG] pgmap v112: 73 pgs: 73 active+clean; 2.3 KiB data, 45 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:35:06.086 INFO:journalctl@ceph.mgr.x.smithi164.stdout:Jun 03 19:35:05 smithi164 bash[27934]: ::ffff:172.21.15.164 - - [03/Jun/2023:19:35:05] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2023-06-03T19:35:06.087 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:35:05 smithi164 bash[26855]: cluster 2023-06-03T19:35:04.430808+0000 mgr.y (mgr.14445) 131 : cluster [DBG] pgmap v112: 73 pgs: 73 active+clean; 2.3 KiB data, 45 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:35:08.031 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:35:07 smithi089 bash[33173]: cluster 2023-06-03T19:35:06.431360+0000 mgr.y (mgr.14445) 132 : cluster [DBG] pgmap v113: 73 pgs: 73 active+clean; 2.3 KiB data, 45 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:35:08.032 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 19:35:07 smithi089 bash[26845]: ::ffff:172.21.15.164 - - [03/Jun/2023:19:35:07] "GET /metrics HTTP/1.1" 200 187864 "" "Prometheus/2.33.4" 2023-06-03T19:35:08.032 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:35:07 smithi089 bash[26572]: cluster 2023-06-03T19:35:06.431360+0000 mgr.y (mgr.14445) 132 : cluster [DBG] pgmap v113: 73 pgs: 73 active+clean; 2.3 KiB data, 45 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:35:08.086 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:35:07 smithi164 bash[26855]: cluster 2023-06-03T19:35:06.431360+0000 mgr.y (mgr.14445) 132 : cluster [DBG] pgmap v113: 73 pgs: 73 active+clean; 2.3 KiB data, 45 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:35:09.794 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:35:09 smithi089 bash[26572]: cluster 2023-06-03T19:35:08.431724+0000 mgr.y (mgr.14445) 133 : cluster [DBG] pgmap v114: 73 pgs: 73 active+clean; 2.3 KiB data, 45 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:35:10.031 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:35:09 smithi089 bash[33173]: cluster 2023-06-03T19:35:08.431724+0000 mgr.y (mgr.14445) 133 : cluster [DBG] pgmap v114: 73 pgs: 73 active+clean; 2.3 KiB data, 45 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:35:10.086 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:35:09 smithi164 bash[26855]: cluster 2023-06-03T19:35:08.431724+0000 mgr.y (mgr.14445) 133 : cluster [DBG] pgmap v114: 73 pgs: 73 active+clean; 2.3 KiB data, 45 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:35:12.032 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:35:11 smithi089 bash[26572]: cluster 2023-06-03T19:35:10.432828+0000 mgr.y (mgr.14445) 134 : cluster [DBG] pgmap v115: 73 pgs: 73 active+clean; 2.3 KiB data, 45 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:35:12.032 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:35:11 smithi089 bash[33173]: cluster 2023-06-03T19:35:10.432828+0000 mgr.y (mgr.14445) 134 : cluster [DBG] pgmap v115: 73 pgs: 73 active+clean; 2.3 KiB data, 45 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:35:12.086 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:35:11 smithi164 bash[26855]: cluster 2023-06-03T19:35:10.432828+0000 mgr.y (mgr.14445) 134 : cluster [DBG] pgmap v115: 73 pgs: 73 active+clean; 2.3 KiB data, 45 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:35:13.857 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:35:13 smithi089 bash[26572]: cluster 2023-06-03T19:35:12.433273+0000 mgr.y (mgr.14445) 135 : cluster [DBG] pgmap v116: 73 pgs: 73 active+clean; 2.3 KiB data, 45 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:35:14.087 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:35:13 smithi164 bash[26855]: cluster 2023-06-03T19:35:12.433273+0000 mgr.y (mgr.14445) 135 : cluster [DBG] pgmap v116: 73 pgs: 73 active+clean; 2.3 KiB data, 45 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:35:14.281 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:35:13 smithi089 bash[33173]: cluster 2023-06-03T19:35:12.433273+0000 mgr.y (mgr.14445) 135 : cluster [DBG] pgmap v116: 73 pgs: 73 active+clean; 2.3 KiB data, 45 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:35:15.592 INFO:teuthology.orchestra.run.smithi089.stdout:34359738482 2023-06-03T19:35:15.592 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph osd last-stat-seq osd.0 2023-06-03T19:35:16.032 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:35:15 smithi089 bash[33173]: cluster 2023-06-03T19:35:14.434253+0000 mgr.y (mgr.14445) 136 : cluster [DBG] pgmap v117: 73 pgs: 73 active+clean; 2.3 KiB data, 45 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:35:16.032 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:35:15 smithi089 bash[26572]: cluster 2023-06-03T19:35:14.434253+0000 mgr.y (mgr.14445) 136 : cluster [DBG] pgmap v117: 73 pgs: 73 active+clean; 2.3 KiB data, 45 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:35:16.087 INFO:journalctl@ceph.mgr.x.smithi164.stdout:Jun 03 19:35:15 smithi164 bash[27934]: ::ffff:172.21.15.164 - - [03/Jun/2023:19:35:15] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2023-06-03T19:35:16.087 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:35:15 smithi164 bash[26855]: cluster 2023-06-03T19:35:14.434253+0000 mgr.y (mgr.14445) 136 : cluster [DBG] pgmap v117: 73 pgs: 73 active+clean; 2.3 KiB data, 45 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:35:16.755 INFO:teuthology.orchestra.run.smithi089.stdout:107374182486 2023-06-03T19:35:16.755 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph osd last-stat-seq osd.3 2023-06-03T19:35:18.029 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:35:17 smithi089 bash[33173]: cluster 2023-06-03T19:35:16.434932+0000 mgr.y (mgr.14445) 137 : cluster [DBG] pgmap v118: 73 pgs: 73 active+clean; 2.3 KiB data, 45 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:35:18.029 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:35:17 smithi089 bash[26572]: cluster 2023-06-03T19:35:16.434932+0000 mgr.y (mgr.14445) 137 : cluster [DBG] pgmap v118: 73 pgs: 73 active+clean; 2.3 KiB data, 45 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:35:18.030 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 19:35:17 smithi089 bash[26845]: ::ffff:172.21.15.164 - - [03/Jun/2023:19:35:17] "GET /metrics HTTP/1.1" 200 187864 "" "Prometheus/2.33.4" 2023-06-03T19:35:18.086 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:35:17 smithi164 bash[26855]: cluster 2023-06-03T19:35:16.434932+0000 mgr.y (mgr.14445) 137 : cluster [DBG] pgmap v118: 73 pgs: 73 active+clean; 2.3 KiB data, 45 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:35:19.843 INFO:teuthology.orchestra.run.smithi089.stdout:206158430259 2023-06-03T19:35:19.844 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph osd last-stat-seq osd.7 2023-06-03T19:35:20.031 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:35:19 smithi089 bash[26572]: cluster 2023-06-03T19:35:18.435590+0000 mgr.y (mgr.14445) 138 : cluster [DBG] pgmap v119: 73 pgs: 73 active+clean; 2.3 KiB data, 45 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:35:20.032 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:35:19 smithi089 bash[33173]: cluster 2023-06-03T19:35:18.435590+0000 mgr.y (mgr.14445) 138 : cluster [DBG] pgmap v119: 73 pgs: 73 active+clean; 2.3 KiB data, 45 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:35:20.087 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:35:19 smithi164 bash[26855]: cluster 2023-06-03T19:35:18.435590+0000 mgr.y (mgr.14445) 138 : cluster [DBG] pgmap v119: 73 pgs: 73 active+clean; 2.3 KiB data, 45 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:35:20.968 INFO:teuthology.orchestra.run.smithi089.stdout:180388626492 2023-06-03T19:35:20.969 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph osd last-stat-seq osd.6 2023-06-03T19:35:21.339 INFO:teuthology.orchestra.run.smithi089.stdout:154618822726 2023-06-03T19:35:21.340 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph osd last-stat-seq osd.5 2023-06-03T19:35:21.728 INFO:teuthology.orchestra.run.smithi089.stdout:77309411424 2023-06-03T19:35:21.729 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph osd last-stat-seq osd.2 2023-06-03T19:35:21.894 INFO:teuthology.orchestra.run.smithi089.stdout:55834574954 2023-06-03T19:35:21.895 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph osd last-stat-seq osd.1 2023-06-03T19:35:21.981 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:35:21 smithi089 bash[33173]: cluster 2023-06-03T19:35:20.436650+0000 mgr.y (mgr.14445) 139 : cluster [DBG] pgmap v120: 73 pgs: 73 active+clean; 2.3 KiB data, 45 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:35:21.981 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:35:21 smithi089 bash[26572]: cluster 2023-06-03T19:35:20.436650+0000 mgr.y (mgr.14445) 139 : cluster [DBG] pgmap v120: 73 pgs: 73 active+clean; 2.3 KiB data, 45 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:35:21.983 INFO:teuthology.orchestra.run.smithi089.stdout:128849018959 2023-06-03T19:35:21.983 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph osd last-stat-seq osd.4 2023-06-03T19:35:22.087 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:35:21 smithi164 bash[26855]: cluster 2023-06-03T19:35:20.436650+0000 mgr.y (mgr.14445) 139 : cluster [DBG] pgmap v120: 73 pgs: 73 active+clean; 2.3 KiB data, 45 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:35:24.087 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:35:23 smithi164 bash[26855]: cluster 2023-06-03T19:35:22.437285+0000 mgr.y (mgr.14445) 140 : cluster [DBG] pgmap v121: 73 pgs: 73 active+clean; 2.3 KiB data, 45 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:35:24.281 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:35:23 smithi089 bash[33173]: cluster 2023-06-03T19:35:22.437285+0000 mgr.y (mgr.14445) 140 : cluster [DBG] pgmap v121: 73 pgs: 73 active+clean; 2.3 KiB data, 45 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:35:24.282 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:35:23 smithi089 bash[26572]: cluster 2023-06-03T19:35:22.437285+0000 mgr.y (mgr.14445) 140 : cluster [DBG] pgmap v121: 73 pgs: 73 active+clean; 2.3 KiB data, 45 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:35:24.636 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/mon.a/config 2023-06-03T19:35:24.636 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/mon.a/config 2023-06-03T19:35:24.637 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/mon.a/config 2023-06-03T19:35:25.658 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/mon.a/config 2023-06-03T19:35:25.658 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/mon.a/config 2023-06-03T19:35:25.659 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/mon.a/config 2023-06-03T19:35:26.031 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:35:25 smithi089 bash[26572]: cluster 2023-06-03T19:35:24.438324+0000 mgr.y (mgr.14445) 141 : cluster [DBG] pgmap v122: 73 pgs: 73 active+clean; 2.3 KiB data, 45 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:35:26.032 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:35:25 smithi089 bash[33173]: cluster 2023-06-03T19:35:24.438324+0000 mgr.y (mgr.14445) 141 : cluster [DBG] pgmap v122: 73 pgs: 73 active+clean; 2.3 KiB data, 45 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:35:26.086 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:35:25 smithi164 bash[26855]: cluster 2023-06-03T19:35:24.438324+0000 mgr.y (mgr.14445) 141 : cluster [DBG] pgmap v122: 73 pgs: 73 active+clean; 2.3 KiB data, 45 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:35:26.087 INFO:journalctl@ceph.mgr.x.smithi164.stdout:Jun 03 19:35:25 smithi164 bash[27934]: ::ffff:172.21.15.164 - - [03/Jun/2023:19:35:25] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2023-06-03T19:35:26.679 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/mon.a/config 2023-06-03T19:35:26.680 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/mon.a/config 2023-06-03T19:35:28.074 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:35:27 smithi089 bash[26572]: cluster 2023-06-03T19:35:26.439075+0000 mgr.y (mgr.14445) 142 : cluster [DBG] pgmap v123: 73 pgs: 73 active+clean; 2.3 KiB data, 45 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:35:28.075 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:35:27 smithi089 bash[33173]: cluster 2023-06-03T19:35:26.439075+0000 mgr.y (mgr.14445) 142 : cluster [DBG] pgmap v123: 73 pgs: 73 active+clean; 2.3 KiB data, 45 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:35:28.075 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 19:35:27 smithi089 bash[26845]: ::ffff:172.21.15.164 - - [03/Jun/2023:19:35:27] "GET /metrics HTTP/1.1" 200 187883 "" "Prometheus/2.33.4" 2023-06-03T19:35:28.086 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:35:27 smithi164 bash[26855]: cluster 2023-06-03T19:35:26.439075+0000 mgr.y (mgr.14445) 142 : cluster [DBG] pgmap v123: 73 pgs: 73 active+clean; 2.3 KiB data, 45 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:35:30.086 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:35:29 smithi164 bash[26855]: cluster 2023-06-03T19:35:28.439750+0000 mgr.y (mgr.14445) 143 : cluster [DBG] pgmap v124: 73 pgs: 73 active+clean; 2.3 KiB data, 45 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:35:30.099 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:35:29 smithi089 bash[33173]: cluster 2023-06-03T19:35:28.439750+0000 mgr.y (mgr.14445) 143 : cluster [DBG] pgmap v124: 73 pgs: 73 active+clean; 2.3 KiB data, 45 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:35:30.099 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:35:29 smithi089 bash[26572]: cluster 2023-06-03T19:35:28.439750+0000 mgr.y (mgr.14445) 143 : cluster [DBG] pgmap v124: 73 pgs: 73 active+clean; 2.3 KiB data, 45 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:35:32.087 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:35:31 smithi164 bash[26855]: cluster 2023-06-03T19:35:30.440397+0000 mgr.y (mgr.14445) 144 : cluster [DBG] pgmap v125: 73 pgs: 73 active+clean; 2.3 KiB data, 45 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:35:32.087 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:35:31 smithi164 bash[26855]: audit 2023-06-03T19:35:31.640202+0000 mon.a (mon.0) 723 : audit [DBG] from='mgr.14445 172.21.15.89:0/573502486' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-03T19:35:32.323 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:35:31 smithi089 bash[33173]: cluster 2023-06-03T19:35:30.440397+0000 mgr.y (mgr.14445) 144 : cluster [DBG] pgmap v125: 73 pgs: 73 active+clean; 2.3 KiB data, 45 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:35:32.324 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:35:31 smithi089 bash[33173]: audit 2023-06-03T19:35:31.640202+0000 mon.a (mon.0) 723 : audit [DBG] from='mgr.14445 172.21.15.89:0/573502486' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-03T19:35:32.324 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:35:31 smithi089 bash[26572]: cluster 2023-06-03T19:35:30.440397+0000 mgr.y (mgr.14445) 144 : cluster [DBG] pgmap v125: 73 pgs: 73 active+clean; 2.3 KiB data, 45 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:35:32.324 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:35:31 smithi089 bash[26572]: audit 2023-06-03T19:35:31.640202+0000 mon.a (mon.0) 723 : audit [DBG] from='mgr.14445 172.21.15.89:0/573502486' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-03T19:35:33.041 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:35:32 smithi089 bash[26572]: audit 2023-06-03T19:35:31.931617+0000 mon.a (mon.0) 724 : audit [DBG] from='mgr.14445 172.21.15.89:0/573502486' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-03T19:35:33.042 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:35:32 smithi089 bash[26572]: audit 2023-06-03T19:35:31.932594+0000 mon.a (mon.0) 725 : audit [INF] from='mgr.14445 172.21.15.89:0/573502486' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-03T19:35:33.042 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:35:32 smithi089 bash[26572]: audit 2023-06-03T19:35:31.938050+0000 mon.a (mon.0) 726 : audit [INF] from='mgr.14445 172.21.15.89:0/573502486' entity='mgr.y' 2023-06-03T19:35:33.042 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:35:32 smithi089 bash[33173]: audit 2023-06-03T19:35:31.931617+0000 mon.a (mon.0) 724 : audit [DBG] from='mgr.14445 172.21.15.89:0/573502486' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-03T19:35:33.042 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:35:32 smithi089 bash[33173]: audit 2023-06-03T19:35:31.932594+0000 mon.a (mon.0) 725 : audit [INF] from='mgr.14445 172.21.15.89:0/573502486' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-03T19:35:33.042 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:35:32 smithi089 bash[33173]: audit 2023-06-03T19:35:31.938050+0000 mon.a (mon.0) 726 : audit [INF] from='mgr.14445 172.21.15.89:0/573502486' entity='mgr.y' 2023-06-03T19:35:33.087 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:35:32 smithi164 bash[26855]: audit 2023-06-03T19:35:31.931617+0000 mon.a (mon.0) 724 : audit [DBG] from='mgr.14445 172.21.15.89:0/573502486' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-03T19:35:33.087 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:35:32 smithi164 bash[26855]: audit 2023-06-03T19:35:31.932594+0000 mon.a (mon.0) 725 : audit [INF] from='mgr.14445 172.21.15.89:0/573502486' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-03T19:35:33.087 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:35:32 smithi164 bash[26855]: audit 2023-06-03T19:35:31.938050+0000 mon.a (mon.0) 726 : audit [INF] from='mgr.14445 172.21.15.89:0/573502486' entity='mgr.y' 2023-06-03T19:35:34.087 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:35:33 smithi164 bash[26855]: cluster 2023-06-03T19:35:32.440986+0000 mgr.y (mgr.14445) 145 : cluster [DBG] pgmap v126: 73 pgs: 73 active+clean; 2.3 KiB data, 45 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:35:34.203 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:35:33 smithi089 bash[33173]: cluster 2023-06-03T19:35:32.440986+0000 mgr.y (mgr.14445) 145 : cluster [DBG] pgmap v126: 73 pgs: 73 active+clean; 2.3 KiB data, 45 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:35:34.203 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:35:33 smithi089 bash[26572]: cluster 2023-06-03T19:35:32.440986+0000 mgr.y (mgr.14445) 145 : cluster [DBG] pgmap v126: 73 pgs: 73 active+clean; 2.3 KiB data, 45 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:35:36.031 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:35:35 smithi089 bash[26572]: cluster 2023-06-03T19:35:34.441696+0000 mgr.y (mgr.14445) 146 : cluster [DBG] pgmap v127: 73 pgs: 73 active+clean; 2.3 KiB data, 45 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:35:36.032 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:35:35 smithi089 bash[33173]: cluster 2023-06-03T19:35:34.441696+0000 mgr.y (mgr.14445) 146 : cluster [DBG] pgmap v127: 73 pgs: 73 active+clean; 2.3 KiB data, 45 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:35:36.087 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:35:35 smithi164 bash[26855]: cluster 2023-06-03T19:35:34.441696+0000 mgr.y (mgr.14445) 146 : cluster [DBG] pgmap v127: 73 pgs: 73 active+clean; 2.3 KiB data, 45 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:35:36.087 INFO:journalctl@ceph.mgr.x.smithi164.stdout:Jun 03 19:35:35 smithi164 bash[27934]: ::ffff:172.21.15.164 - - [03/Jun/2023:19:35:35] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2023-06-03T19:35:36.610 INFO:teuthology.orchestra.run.smithi089.stdout:206158430263 2023-06-03T19:35:37.829 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:35:37 smithi089 bash[26572]: cluster 2023-06-03T19:35:36.442008+0000 mgr.y (mgr.14445) 147 : cluster [DBG] pgmap v128: 73 pgs: 73 active+clean; 2.3 KiB data, 45 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:35:37.830 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:35:37 smithi089 bash[26572]: audit 2023-06-03T19:35:36.602767+0000 mon.c (mon.2) 24 : audit [DBG] from='client.? 172.21.15.89:0/3116974946' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2023-06-03T19:35:37.830 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:35:37 smithi089 bash[33173]: cluster 2023-06-03T19:35:36.442008+0000 mgr.y (mgr.14445) 147 : cluster [DBG] pgmap v128: 73 pgs: 73 active+clean; 2.3 KiB data, 45 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:35:37.830 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:35:37 smithi089 bash[33173]: audit 2023-06-03T19:35:36.602767+0000 mon.c (mon.2) 24 : audit [DBG] from='client.? 172.21.15.89:0/3116974946' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2023-06-03T19:35:37.837 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:35:37 smithi164 bash[26855]: cluster 2023-06-03T19:35:36.442008+0000 mgr.y (mgr.14445) 147 : cluster [DBG] pgmap v128: 73 pgs: 73 active+clean; 2.3 KiB data, 45 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:35:37.837 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:35:37 smithi164 bash[26855]: audit 2023-06-03T19:35:36.602767+0000 mon.c (mon.2) 24 : audit [DBG] from='client.? 172.21.15.89:0/3116974946' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2023-06-03T19:35:38.031 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 19:35:37 smithi089 bash[26845]: ::ffff:172.21.15.164 - - [03/Jun/2023:19:35:37] "GET /metrics HTTP/1.1" 200 187881 "" "Prometheus/2.33.4" 2023-06-03T19:35:39.459 INFO:tasks.cephadm.ceph_manager.ceph:need seq 206158430259 got 206158430263 for osd.7 2023-06-03T19:35:39.460 DEBUG:teuthology.parallel:result is None 2023-06-03T19:35:39.781 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:35:39 smithi089 bash[26572]: cluster 2023-06-03T19:35:38.442616+0000 mgr.y (mgr.14445) 148 : cluster [DBG] pgmap v129: 73 pgs: 73 active+clean; 2.3 KiB data, 45 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:35:39.782 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:35:39 smithi089 bash[33173]: cluster 2023-06-03T19:35:38.442616+0000 mgr.y (mgr.14445) 148 : cluster [DBG] pgmap v129: 73 pgs: 73 active+clean; 2.3 KiB data, 45 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:35:39.837 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:35:39 smithi164 bash[26855]: cluster 2023-06-03T19:35:38.442616+0000 mgr.y (mgr.14445) 148 : cluster [DBG] pgmap v129: 73 pgs: 73 active+clean; 2.3 KiB data, 45 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:35:39.983 INFO:teuthology.orchestra.run.smithi089.stdout:107374182491 2023-06-03T19:35:40.559 INFO:teuthology.orchestra.run.smithi089.stdout:77309411428 2023-06-03T19:35:40.837 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:35:40 smithi164 bash[26855]: audit 2023-06-03T19:35:39.975673+0000 mon.a (mon.0) 727 : audit [DBG] from='client.? 172.21.15.89:0/3958858073' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2023-06-03T19:35:40.837 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:35:40 smithi164 bash[26855]: audit 2023-06-03T19:35:40.552157+0000 mon.c (mon.2) 25 : audit [DBG] from='client.? 172.21.15.89:0/247875943' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2023-06-03T19:35:40.898 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:35:40 smithi089 bash[33173]: audit 2023-06-03T19:35:39.975673+0000 mon.a (mon.0) 727 : audit [DBG] from='client.? 172.21.15.89:0/3958858073' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2023-06-03T19:35:40.898 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:35:40 smithi089 bash[33173]: audit 2023-06-03T19:35:40.552157+0000 mon.c (mon.2) 25 : audit [DBG] from='client.? 172.21.15.89:0/247875943' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2023-06-03T19:35:40.898 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:35:40 smithi089 bash[26572]: audit 2023-06-03T19:35:39.975673+0000 mon.a (mon.0) 727 : audit [DBG] from='client.? 172.21.15.89:0/3958858073' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2023-06-03T19:35:40.898 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:35:40 smithi089 bash[26572]: audit 2023-06-03T19:35:40.552157+0000 mon.c (mon.2) 25 : audit [DBG] from='client.? 172.21.15.89:0/247875943' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2023-06-03T19:35:41.837 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:35:41 smithi164 bash[26855]: cluster 2023-06-03T19:35:40.443668+0000 mgr.y (mgr.14445) 149 : cluster [DBG] pgmap v130: 73 pgs: 73 active+clean; 2.3 KiB data, 45 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:35:42.031 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:35:41 smithi089 bash[33173]: cluster 2023-06-03T19:35:40.443668+0000 mgr.y (mgr.14445) 149 : cluster [DBG] pgmap v130: 73 pgs: 73 active+clean; 2.3 KiB data, 45 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:35:42.032 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:35:41 smithi089 bash[26572]: cluster 2023-06-03T19:35:40.443668+0000 mgr.y (mgr.14445) 149 : cluster [DBG] pgmap v130: 73 pgs: 73 active+clean; 2.3 KiB data, 45 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:35:42.187 INFO:tasks.cephadm.ceph_manager.ceph:need seq 107374182486 got 107374182491 for osd.3 2023-06-03T19:35:42.187 DEBUG:teuthology.parallel:result is None 2023-06-03T19:35:43.085 INFO:tasks.cephadm.ceph_manager.ceph:need seq 77309411424 got 77309411428 for osd.2 2023-06-03T19:35:43.085 DEBUG:teuthology.parallel:result is None 2023-06-03T19:35:43.486 INFO:teuthology.orchestra.run.smithi089.stdout:34359738488 2023-06-03T19:35:43.737 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:35:43 smithi089 bash[26572]: cluster 2023-06-03T19:35:42.444429+0000 mgr.y (mgr.14445) 150 : cluster [DBG] pgmap v131: 73 pgs: 73 active+clean; 2.3 KiB data, 45 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:35:43.737 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:35:43 smithi089 bash[26572]: audit 2023-06-03T19:35:43.478750+0000 mon.c (mon.2) 26 : audit [DBG] from='client.? 172.21.15.89:0/356245026' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2023-06-03T19:35:43.738 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:35:43 smithi089 bash[33173]: cluster 2023-06-03T19:35:42.444429+0000 mgr.y (mgr.14445) 150 : cluster [DBG] pgmap v131: 73 pgs: 73 active+clean; 2.3 KiB data, 45 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:35:43.738 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:35:43 smithi089 bash[33173]: audit 2023-06-03T19:35:43.478750+0000 mon.c (mon.2) 26 : audit [DBG] from='client.? 172.21.15.89:0/356245026' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2023-06-03T19:35:43.763 INFO:teuthology.orchestra.run.smithi089.stdout:128849018964 2023-06-03T19:35:43.837 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:35:43 smithi164 bash[26855]: cluster 2023-06-03T19:35:42.444429+0000 mgr.y (mgr.14445) 150 : cluster [DBG] pgmap v131: 73 pgs: 73 active+clean; 2.3 KiB data, 45 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:35:43.837 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:35:43 smithi164 bash[26855]: audit 2023-06-03T19:35:43.478750+0000 mon.c (mon.2) 26 : audit [DBG] from='client.? 172.21.15.89:0/356245026' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2023-06-03T19:35:44.074 INFO:teuthology.orchestra.run.smithi089.stdout:154618822731 2023-06-03T19:35:44.481 INFO:teuthology.orchestra.run.smithi089.stdout:55834574959 2023-06-03T19:35:44.781 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:35:44 smithi089 bash[33173]: audit 2023-06-03T19:35:43.756021+0000 mon.a (mon.0) 728 : audit [DBG] from='client.? 172.21.15.89:0/3885482281' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2023-06-03T19:35:44.782 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:35:44 smithi089 bash[33173]: audit 2023-06-03T19:35:44.066977+0000 mon.a (mon.0) 729 : audit [DBG] from='client.? 172.21.15.89:0/3995119847' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2023-06-03T19:35:44.782 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:35:44 smithi089 bash[33173]: audit 2023-06-03T19:35:44.474009+0000 mon.c (mon.2) 27 : audit [DBG] from='client.? 172.21.15.89:0/858875427' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2023-06-03T19:35:44.782 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:35:44 smithi089 bash[26572]: audit 2023-06-03T19:35:43.756021+0000 mon.a (mon.0) 728 : audit [DBG] from='client.? 172.21.15.89:0/3885482281' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2023-06-03T19:35:44.782 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:35:44 smithi089 bash[26572]: audit 2023-06-03T19:35:44.066977+0000 mon.a (mon.0) 729 : audit [DBG] from='client.? 172.21.15.89:0/3995119847' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2023-06-03T19:35:44.783 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:35:44 smithi089 bash[26572]: audit 2023-06-03T19:35:44.474009+0000 mon.c (mon.2) 27 : audit [DBG] from='client.? 172.21.15.89:0/858875427' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2023-06-03T19:35:44.794 INFO:teuthology.orchestra.run.smithi089.stdout:180388626498 2023-06-03T19:35:45.087 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:35:44 smithi164 bash[26855]: audit 2023-06-03T19:35:43.756021+0000 mon.a (mon.0) 728 : audit [DBG] from='client.? 172.21.15.89:0/3885482281' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2023-06-03T19:35:45.087 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:35:44 smithi164 bash[26855]: audit 2023-06-03T19:35:44.066977+0000 mon.a (mon.0) 729 : audit [DBG] from='client.? 172.21.15.89:0/3995119847' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2023-06-03T19:35:45.087 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:35:44 smithi164 bash[26855]: audit 2023-06-03T19:35:44.474009+0000 mon.c (mon.2) 27 : audit [DBG] from='client.? 172.21.15.89:0/858875427' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2023-06-03T19:35:45.781 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:35:45 smithi089 bash[26572]: cluster 2023-06-03T19:35:44.444985+0000 mgr.y (mgr.14445) 151 : cluster [DBG] pgmap v132: 73 pgs: 73 active+clean; 2.3 KiB data, 45 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:35:45.782 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:35:45 smithi089 bash[26572]: audit 2023-06-03T19:35:44.787077+0000 mon.c (mon.2) 28 : audit [DBG] from='client.? 172.21.15.89:0/1764710124' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2023-06-03T19:35:45.782 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:35:45 smithi089 bash[33173]: cluster 2023-06-03T19:35:44.444985+0000 mgr.y (mgr.14445) 151 : cluster [DBG] pgmap v132: 73 pgs: 73 active+clean; 2.3 KiB data, 45 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:35:45.782 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:35:45 smithi089 bash[33173]: audit 2023-06-03T19:35:44.787077+0000 mon.c (mon.2) 28 : audit [DBG] from='client.? 172.21.15.89:0/1764710124' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2023-06-03T19:35:46.087 INFO:journalctl@ceph.mgr.x.smithi164.stdout:Jun 03 19:35:45 smithi164 bash[27934]: ::ffff:172.21.15.164 - - [03/Jun/2023:19:35:45] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2023-06-03T19:35:46.087 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:35:45 smithi164 bash[26855]: cluster 2023-06-03T19:35:44.444985+0000 mgr.y (mgr.14445) 151 : cluster [DBG] pgmap v132: 73 pgs: 73 active+clean; 2.3 KiB data, 45 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:35:46.087 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:35:45 smithi164 bash[26855]: audit 2023-06-03T19:35:44.787077+0000 mon.c (mon.2) 28 : audit [DBG] from='client.? 172.21.15.89:0/1764710124' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2023-06-03T19:35:46.227 INFO:tasks.cephadm.ceph_manager.ceph:need seq 34359738482 got 34359738488 for osd.0 2023-06-03T19:35:46.227 DEBUG:teuthology.parallel:result is None 2023-06-03T19:35:46.529 INFO:tasks.cephadm.ceph_manager.ceph:need seq 128849018959 got 128849018964 for osd.4 2023-06-03T19:35:46.530 DEBUG:teuthology.parallel:result is None 2023-06-03T19:35:47.266 INFO:tasks.cephadm.ceph_manager.ceph:need seq 154618822726 got 154618822731 for osd.5 2023-06-03T19:35:47.266 DEBUG:teuthology.parallel:result is None 2023-06-03T19:35:47.526 INFO:tasks.cephadm.ceph_manager.ceph:need seq 180388626492 got 180388626498 for osd.6 2023-06-03T19:35:47.526 DEBUG:teuthology.parallel:result is None 2023-06-03T19:35:47.565 INFO:tasks.cephadm.ceph_manager.ceph:need seq 55834574954 got 55834574959 for osd.1 2023-06-03T19:35:47.565 DEBUG:teuthology.parallel:result is None 2023-06-03T19:35:47.565 INFO:tasks.cephadm.ceph_manager.ceph:waiting for clean 2023-06-03T19:35:47.565 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph pg dump --format=json 2023-06-03T19:35:47.782 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:35:47 smithi089 bash[33173]: cluster 2023-06-03T19:35:46.445336+0000 mgr.y (mgr.14445) 152 : cluster [DBG] pgmap v133: 73 pgs: 73 active+clean; 2.3 KiB data, 45 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:35:47.782 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:35:47 smithi089 bash[26572]: cluster 2023-06-03T19:35:46.445336+0000 mgr.y (mgr.14445) 152 : cluster [DBG] pgmap v133: 73 pgs: 73 active+clean; 2.3 KiB data, 45 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:35:48.087 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:35:47 smithi164 bash[26855]: cluster 2023-06-03T19:35:46.445336+0000 mgr.y (mgr.14445) 152 : cluster [DBG] pgmap v133: 73 pgs: 73 active+clean; 2.3 KiB data, 45 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:35:48.281 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 19:35:47 smithi089 bash[26845]: ::ffff:172.21.15.164 - - [03/Jun/2023:19:35:47] "GET /metrics HTTP/1.1" 200 187881 "" "Prometheus/2.33.4" 2023-06-03T19:35:50.031 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:35:49 smithi089 bash[26572]: cluster 2023-06-03T19:35:48.445838+0000 mgr.y (mgr.14445) 153 : cluster [DBG] pgmap v134: 73 pgs: 73 active+clean; 2.3 KiB data, 45 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:35:50.032 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:35:49 smithi089 bash[33173]: cluster 2023-06-03T19:35:48.445838+0000 mgr.y (mgr.14445) 153 : cluster [DBG] pgmap v134: 73 pgs: 73 active+clean; 2.3 KiB data, 45 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:35:50.087 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:35:49 smithi164 bash[26855]: cluster 2023-06-03T19:35:48.445838+0000 mgr.y (mgr.14445) 153 : cluster [DBG] pgmap v134: 73 pgs: 73 active+clean; 2.3 KiB data, 45 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:35:52.032 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:35:51 smithi089 bash[33173]: cluster 2023-06-03T19:35:50.446964+0000 mgr.y (mgr.14445) 154 : cluster [DBG] pgmap v135: 73 pgs: 73 active+clean; 2.3 KiB data, 45 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:35:52.032 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:35:51 smithi089 bash[26572]: cluster 2023-06-03T19:35:50.446964+0000 mgr.y (mgr.14445) 154 : cluster [DBG] pgmap v135: 73 pgs: 73 active+clean; 2.3 KiB data, 45 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:35:52.087 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:35:51 smithi164 bash[26855]: cluster 2023-06-03T19:35:50.446964+0000 mgr.y (mgr.14445) 154 : cluster [DBG] pgmap v135: 73 pgs: 73 active+clean; 2.3 KiB data, 45 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:35:52.385 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/mon.a/config 2023-06-03T19:35:53.726 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:35:53 smithi089 bash[26572]: cluster 2023-06-03T19:35:52.447581+0000 mgr.y (mgr.14445) 155 : cluster [DBG] pgmap v136: 73 pgs: 73 active+clean; 2.3 KiB data, 45 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:35:53.727 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:35:53 smithi089 bash[33173]: cluster 2023-06-03T19:35:52.447581+0000 mgr.y (mgr.14445) 155 : cluster [DBG] pgmap v136: 73 pgs: 73 active+clean; 2.3 KiB data, 45 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:35:54.087 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:35:53 smithi164 bash[26855]: cluster 2023-06-03T19:35:52.447581+0000 mgr.y (mgr.14445) 155 : cluster [DBG] pgmap v136: 73 pgs: 73 active+clean; 2.3 KiB data, 45 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:35:55.781 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:35:55 smithi089 bash[33173]: cluster 2023-06-03T19:35:54.448327+0000 mgr.y (mgr.14445) 156 : cluster [DBG] pgmap v137: 73 pgs: 73 active+clean; 2.3 KiB data, 45 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:35:55.782 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:35:55 smithi089 bash[26572]: cluster 2023-06-03T19:35:54.448327+0000 mgr.y (mgr.14445) 156 : cluster [DBG] pgmap v137: 73 pgs: 73 active+clean; 2.3 KiB data, 45 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:35:55.932 INFO:teuthology.orchestra.run.smithi089.stdout: 2023-06-03T19:35:55.935 INFO:teuthology.orchestra.run.smithi089.stderr:dumped all 2023-06-03T19:35:56.087 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:35:55 smithi164 bash[26855]: cluster 2023-06-03T19:35:54.448327+0000 mgr.y (mgr.14445) 156 : cluster [DBG] pgmap v137: 73 pgs: 73 active+clean; 2.3 KiB data, 45 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:35:56.087 INFO:journalctl@ceph.mgr.x.smithi164.stdout:Jun 03 19:35:55 smithi164 bash[27934]: ::ffff:172.21.15.164 - - [03/Jun/2023:19:35:55] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2023-06-03T19:35:56.759 INFO:teuthology.orchestra.run.smithi089.stdout:{"pg_ready":true,"pg_map":{"version":137,"stamp":"2023-06-03T19:35:54.447815+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":2306,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":45,"num_write_kb":13,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":39,"ondisk_log_size":39,"up":219,"acting":219,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":219,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":46448,"kb_used_data":4976,"kb_used_omap":0,"kb_used_meta":41408,"kb_avail":749914768,"statfs":{"total":767960285184,"available":767912722432,"internally_reserved":0,"allocated":5095424,"data_stored":2165146,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":42401792},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.003785"},"pg_stats":[{"pgid":"4.18","version":"0'0","reported_seq":13,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-06-03T19:34:48.688273+0000","last_change":"2023-06-03T19:34:17.833986+0000","last_active":"2023-06-03T19:34:48.688273+0000","last_peered":"2023-06-03T19:34:48.688273+0000","last_clean":"2023-06-03T19:34:48.688273+0000","last_became_active":"2023-06-03T19:34:17.833598+0000","last_became_peered":"2023-06-03T19:34:17.833598+0000","last_unstale":"2023-06-03T19:34:48.688273+0000","last_undegraded":"2023-06-03T19:34:48.688273+0000","last_fullsized":"2023-06-03T19:34:48.688273+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"60'4","reported_seq":18,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-06-03T19:34:48.694317+0000","last_change":"2023-06-03T19:34:16.809349+0000","last_active":"2023-06-03T19:34:48.694317+0000","last_peered":"2023-06-03T19:34:48.694317+0000","last_clean":"2023-06-03T19:34:48.694317+0000","last_became_active":"2023-06-03T19:34:16.809012+0000","last_became_peered":"2023-06-03T19:34:16.809012+0000","last_unstale":"2023-06-03T19:34:48.694317+0000","last_undegraded":"2023-06-03T19:34:48.694317+0000","last_fullsized":"2023-06-03T19:34:48.694317+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","log_size":4,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":13,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-06-03T19:34:48.687439+0000","last_change":"2023-06-03T19:34:17.181723+0000","last_active":"2023-06-03T19:34:48.687439+0000","last_peered":"2023-06-03T19:34:48.687439+0000","last_clean":"2023-06-03T19:34:48.687439+0000","last_became_active":"2023-06-03T19:34:17.181302+0000","last_became_peered":"2023-06-03T19:34:17.181302+0000","last_unstale":"2023-06-03T19:34:48.687439+0000","last_undegraded":"2023-06-03T19:34:48.687439+0000","last_fullsized":"2023-06-03T19:34:48.687439+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"60'1","reported_seq":15,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-06-03T19:34:48.688731+0000","last_change":"2023-06-03T19:34:16.812230+0000","last_active":"2023-06-03T19:34:48.688731+0000","last_peered":"2023-06-03T19:34:48.688731+0000","last_clean":"2023-06-03T19:34:48.688731+0000","last_became_active":"2023-06-03T19:34:16.811860+0000","last_became_peered":"2023-06-03T19:34:16.811860+0000","last_unstale":"2023-06-03T19:34:48.688731+0000","last_undegraded":"2023-06-03T19:34:48.688731+0000","last_fullsized":"2023-06-03T19:34:48.688731+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","log_size":1,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":13,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-06-03T19:34:48.688562+0000","last_change":"2023-06-03T19:34:17.827691+0000","last_active":"2023-06-03T19:34:48.688562+0000","last_peered":"2023-06-03T19:34:48.688562+0000","last_clean":"2023-06-03T19:34:48.688562+0000","last_became_active":"2023-06-03T19:34:17.827421+0000","last_became_peered":"2023-06-03T19:34:17.827421+0000","last_unstale":"2023-06-03T19:34:48.688562+0000","last_undegraded":"2023-06-03T19:34:48.688562+0000","last_fullsized":"2023-06-03T19:34:48.688562+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"60'2","reported_seq":16,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-06-03T19:34:48.685939+0000","last_change":"2023-06-03T19:34:16.805013+0000","last_active":"2023-06-03T19:34:48.685939+0000","last_peered":"2023-06-03T19:34:48.685939+0000","last_clean":"2023-06-03T19:34:48.685939+0000","last_became_active":"2023-06-03T19:34:16.804707+0000","last_became_peered":"2023-06-03T19:34:16.804707+0000","last_unstale":"2023-06-03T19:34:48.685939+0000","last_undegraded":"2023-06-03T19:34:48.685939+0000","last_fullsized":"2023-06-03T19:34:48.685939+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","log_size":2,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":13,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-06-03T19:34:48.689486+0000","last_change":"2023-06-03T19:34:17.833202+0000","last_active":"2023-06-03T19:34:48.689486+0000","last_peered":"2023-06-03T19:34:48.689486+0000","last_clean":"2023-06-03T19:34:48.689486+0000","last_became_active":"2023-06-03T19:34:17.832695+0000","last_became_peered":"2023-06-03T19:34:17.832695+0000","last_unstale":"2023-06-03T19:34:48.689486+0000","last_undegraded":"2023-06-03T19:34:48.689486+0000","last_fullsized":"2023-06-03T19:34:48.689486+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"60'2","reported_seq":16,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-06-03T19:34:48.685655+0000","last_change":"2023-06-03T19:34:16.802522+0000","last_active":"2023-06-03T19:34:48.685655+0000","last_peered":"2023-06-03T19:34:48.685655+0000","last_clean":"2023-06-03T19:34:48.685655+0000","last_became_active":"2023-06-03T19:34:16.801282+0000","last_became_peered":"2023-06-03T19:34:16.801282+0000","last_unstale":"2023-06-03T19:34:48.685655+0000","last_undegraded":"2023-06-03T19:34:48.685655+0000","last_fullsized":"2023-06-03T19:34:48.685655+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","log_size":2,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":13,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-06-03T19:34:48.690498+0000","last_change":"2023-06-03T19:34:17.834796+0000","last_active":"2023-06-03T19:34:48.690498+0000","last_peered":"2023-06-03T19:34:48.690498+0000","last_clean":"2023-06-03T19:34:48.690498+0000","last_became_active":"2023-06-03T19:34:17.834464+0000","last_became_peered":"2023-06-03T19:34:17.834464+0000","last_unstale":"2023-06-03T19:34:48.690498+0000","last_undegraded":"2023-06-03T19:34:48.690498+0000","last_fullsized":"2023-06-03T19:34:48.690498+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1b","version":"60'1","reported_seq":15,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-06-03T19:34:48.694648+0000","last_change":"2023-06-03T19:34:16.810791+0000","last_active":"2023-06-03T19:34:48.694648+0000","last_peered":"2023-06-03T19:34:48.694648+0000","last_clean":"2023-06-03T19:34:48.694648+0000","last_became_active":"2023-06-03T19:34:16.810386+0000","last_became_peered":"2023-06-03T19:34:16.810386+0000","last_unstale":"2023-06-03T19:34:48.694648+0000","last_undegraded":"2023-06-03T19:34:48.694648+0000","last_fullsized":"2023-06-03T19:34:48.694648+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","log_size":1,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":13,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-06-03T19:34:48.689412+0000","last_change":"2023-06-03T19:34:17.830960+0000","last_active":"2023-06-03T19:34:48.689412+0000","last_peered":"2023-06-03T19:34:48.689412+0000","last_clean":"2023-06-03T19:34:48.689412+0000","last_became_active":"2023-06-03T19:34:17.830605+0000","last_became_peered":"2023-06-03T19:34:17.830605+0000","last_unstale":"2023-06-03T19:34:48.689412+0000","last_undegraded":"2023-06-03T19:34:48.689412+0000","last_fullsized":"2023-06-03T19:34:48.689412+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.6","version":"60'1","reported_seq":15,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-06-03T19:34:48.694150+0000","last_change":"2023-06-03T19:34:16.807099+0000","last_active":"2023-06-03T19:34:48.694150+0000","last_peered":"2023-06-03T19:34:48.694150+0000","last_clean":"2023-06-03T19:34:48.694150+0000","last_became_active":"2023-06-03T19:34:16.806765+0000","last_became_peered":"2023-06-03T19:34:16.806765+0000","last_unstale":"2023-06-03T19:34:48.694150+0000","last_undegraded":"2023-06-03T19:34:48.694150+0000","last_fullsized":"2023-06-03T19:34:48.694150+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","log_size":1,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":18,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-06-03T19:34:48.711665+0000","last_change":"2023-06-03T19:32:21.706362+0000","last_active":"2023-06-03T19:34:48.711665+0000","last_peered":"2023-06-03T19:34:48.711665+0000","last_clean":"2023-06-03T19:34:48.711665+0000","last_became_active":"2023-06-03T19:32:21.705855+0000","last_became_peered":"2023-06-03T19:32:21.705855+0000","last_unstale":"2023-06-03T19:34:48.711665+0000","last_undegraded":"2023-06-03T19:34:48.711665+0000","last_fullsized":"2023-06-03T19:34:48.711665+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:32:20.692043+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:32:20.692043+0000","last_clean_scrub_stamp":"2023-06-03T19:32:20.692043+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":13,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-06-03T19:34:48.686448+0000","last_change":"2023-06-03T19:34:16.918966+0000","last_active":"2023-06-03T19:34:48.686448+0000","last_peered":"2023-06-03T19:34:48.686448+0000","last_clean":"2023-06-03T19:34:48.686448+0000","last_became_active":"2023-06-03T19:34:16.918245+0000","last_became_peered":"2023-06-03T19:34:16.918245+0000","last_unstale":"2023-06-03T19:34:48.686448+0000","last_undegraded":"2023-06-03T19:34:48.686448+0000","last_fullsized":"2023-06-03T19:34:48.686448+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":18,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-06-03T19:34:48.695432+0000","last_change":"2023-06-03T19:32:21.989748+0000","last_active":"2023-06-03T19:34:48.695432+0000","last_peered":"2023-06-03T19:34:48.695432+0000","last_clean":"2023-06-03T19:34:48.695432+0000","last_became_active":"2023-06-03T19:32:21.989310+0000","last_became_peered":"2023-06-03T19:32:21.989310+0000","last_unstale":"2023-06-03T19:34:48.695432+0000","last_undegraded":"2023-06-03T19:34:48.695432+0000","last_fullsized":"2023-06-03T19:34:48.695432+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:32:20.692043+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:32:20.692043+0000","last_clean_scrub_stamp":"2023-06-03T19:32:20.692043+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.7","version":"60'2","reported_seq":16,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-06-03T19:34:48.689012+0000","last_change":"2023-06-03T19:34:16.811190+0000","last_active":"2023-06-03T19:34:48.689012+0000","last_peered":"2023-06-03T19:34:48.689012+0000","last_clean":"2023-06-03T19:34:48.689012+0000","last_became_active":"2023-06-03T19:34:16.810846+0000","last_became_peered":"2023-06-03T19:34:16.810846+0000","last_unstale":"2023-06-03T19:34:48.689012+0000","last_undegraded":"2023-06-03T19:34:48.689012+0000","last_fullsized":"2023-06-03T19:34:48.689012+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","log_size":2,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":13,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-06-03T19:34:48.694721+0000","last_change":"2023-06-03T19:34:17.849689+0000","last_active":"2023-06-03T19:34:48.694721+0000","last_peered":"2023-06-03T19:34:48.694721+0000","last_clean":"2023-06-03T19:34:48.694721+0000","last_became_active":"2023-06-03T19:34:17.849349+0000","last_became_peered":"2023-06-03T19:34:17.849349+0000","last_unstale":"2023-06-03T19:34:48.694721+0000","last_undegraded":"2023-06-03T19:34:48.694721+0000","last_fullsized":"2023-06-03T19:34:48.694721+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":14,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-06-03T19:34:48.694618+0000","last_change":"2023-06-03T19:34:16.803712+0000","last_active":"2023-06-03T19:34:48.694618+0000","last_peered":"2023-06-03T19:34:48.694618+0000","last_clean":"2023-06-03T19:34:48.694618+0000","last_became_active":"2023-06-03T19:34:16.802957+0000","last_became_peered":"2023-06-03T19:34:16.802957+0000","last_unstale":"2023-06-03T19:34:48.694618+0000","last_undegraded":"2023-06-03T19:34:48.694618+0000","last_fullsized":"2023-06-03T19:34:48.694618+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":18,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-06-03T19:34:48.712646+0000","last_change":"2023-06-03T19:32:22.124191+0000","last_active":"2023-06-03T19:34:48.712646+0000","last_peered":"2023-06-03T19:34:48.712646+0000","last_clean":"2023-06-03T19:34:48.712646+0000","last_became_active":"2023-06-03T19:32:22.122933+0000","last_became_peered":"2023-06-03T19:32:22.122933+0000","last_unstale":"2023-06-03T19:34:48.712646+0000","last_undegraded":"2023-06-03T19:34:48.712646+0000","last_fullsized":"2023-06-03T19:34:48.712646+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:32:20.692043+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:32:20.692043+0000","last_clean_scrub_stamp":"2023-06-03T19:32:20.692043+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":13,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-06-03T19:34:48.694182+0000","last_change":"2023-06-03T19:34:17.829675+0000","last_active":"2023-06-03T19:34:48.694182+0000","last_peered":"2023-06-03T19:34:48.694182+0000","last_clean":"2023-06-03T19:34:48.694182+0000","last_became_active":"2023-06-03T19:34:17.829308+0000","last_became_peered":"2023-06-03T19:34:17.829308+0000","last_unstale":"2023-06-03T19:34:48.694182+0000","last_undegraded":"2023-06-03T19:34:48.694182+0000","last_fullsized":"2023-06-03T19:34:48.694182+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":14,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-06-03T19:34:48.694276+0000","last_change":"2023-06-03T19:34:16.814714+0000","last_active":"2023-06-03T19:34:48.694276+0000","last_peered":"2023-06-03T19:34:48.694276+0000","last_clean":"2023-06-03T19:34:48.694276+0000","last_became_active":"2023-06-03T19:34:16.814075+0000","last_became_peered":"2023-06-03T19:34:16.814075+0000","last_unstale":"2023-06-03T19:34:48.694276+0000","last_undegraded":"2023-06-03T19:34:48.694276+0000","last_fullsized":"2023-06-03T19:34:48.694276+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":18,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-06-03T19:34:48.711992+0000","last_change":"2023-06-03T19:32:22.123780+0000","last_active":"2023-06-03T19:34:48.711992+0000","last_peered":"2023-06-03T19:34:48.711992+0000","last_clean":"2023-06-03T19:34:48.711992+0000","last_became_active":"2023-06-03T19:32:22.122852+0000","last_became_peered":"2023-06-03T19:32:22.122852+0000","last_unstale":"2023-06-03T19:34:48.711992+0000","last_undegraded":"2023-06-03T19:34:48.711992+0000","last_fullsized":"2023-06-03T19:34:48.711992+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:32:20.692043+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:32:20.692043+0000","last_clean_scrub_stamp":"2023-06-03T19:32:20.692043+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":13,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-06-03T19:34:48.711682+0000","last_change":"2023-06-03T19:34:17.848964+0000","last_active":"2023-06-03T19:34:48.711682+0000","last_peered":"2023-06-03T19:34:48.711682+0000","last_clean":"2023-06-03T19:34:48.711682+0000","last_became_active":"2023-06-03T19:34:17.848140+0000","last_became_peered":"2023-06-03T19:34:17.848140+0000","last_unstale":"2023-06-03T19:34:48.711682+0000","last_undegraded":"2023-06-03T19:34:48.711682+0000","last_fullsized":"2023-06-03T19:34:48.711682+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":14,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-06-03T19:34:48.687784+0000","last_change":"2023-06-03T19:34:16.813150+0000","last_active":"2023-06-03T19:34:48.687784+0000","last_peered":"2023-06-03T19:34:48.687784+0000","last_clean":"2023-06-03T19:34:48.687784+0000","last_became_active":"2023-06-03T19:34:16.812816+0000","last_became_peered":"2023-06-03T19:34:16.812816+0000","last_unstale":"2023-06-03T19:34:48.687784+0000","last_undegraded":"2023-06-03T19:34:48.687784+0000","last_fullsized":"2023-06-03T19:34:48.687784+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":22,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-06-03T19:34:48.712037+0000","last_change":"2023-06-03T19:31:15.881174+0000","last_active":"2023-06-03T19:34:48.712037+0000","last_peered":"2023-06-03T19:34:48.712037+0000","last_clean":"2023-06-03T19:34:48.712037+0000","last_became_active":"2023-06-03T19:31:15.880415+0000","last_became_peered":"2023-06-03T19:31:15.880415+0000","last_unstale":"2023-06-03T19:34:48.712037+0000","last_undegraded":"2023-06-03T19:34:48.712037+0000","last_fullsized":"2023-06-03T19:34:48.712037+0000","mapping_epoch":49,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":50,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:27:28.100808+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:27:28.100808+0000","last_clean_scrub_stamp":"2023-06-03T19:27:28.100808+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":18,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-06-03T19:34:48.685467+0000","last_change":"2023-06-03T19:32:21.707936+0000","last_active":"2023-06-03T19:34:48.685467+0000","last_peered":"2023-06-03T19:34:48.685467+0000","last_clean":"2023-06-03T19:34:48.685467+0000","last_became_active":"2023-06-03T19:32:21.707538+0000","last_became_peered":"2023-06-03T19:32:21.707538+0000","last_unstale":"2023-06-03T19:34:48.685467+0000","last_undegraded":"2023-06-03T19:34:48.685467+0000","last_fullsized":"2023-06-03T19:34:48.685467+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:32:20.692043+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:32:20.692043+0000","last_clean_scrub_stamp":"2023-06-03T19:32:20.692043+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":13,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-06-03T19:34:48.695348+0000","last_change":"2023-06-03T19:34:17.828260+0000","last_active":"2023-06-03T19:34:48.695348+0000","last_peered":"2023-06-03T19:34:48.695348+0000","last_clean":"2023-06-03T19:34:48.695348+0000","last_became_active":"2023-06-03T19:34:17.827781+0000","last_became_peered":"2023-06-03T19:34:17.827781+0000","last_unstale":"2023-06-03T19:34:48.695348+0000","last_undegraded":"2023-06-03T19:34:48.695348+0000","last_fullsized":"2023-06-03T19:34:48.695348+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":14,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-06-03T19:34:48.692754+0000","last_change":"2023-06-03T19:34:16.805748+0000","last_active":"2023-06-03T19:34:48.692754+0000","last_peered":"2023-06-03T19:34:48.692754+0000","last_clean":"2023-06-03T19:34:48.692754+0000","last_became_active":"2023-06-03T19:34:16.805383+0000","last_became_peered":"2023-06-03T19:34:16.805383+0000","last_unstale":"2023-06-03T19:34:48.692754+0000","last_undegraded":"2023-06-03T19:34:48.692754+0000","last_fullsized":"2023-06-03T19:34:48.692754+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":18,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-06-03T19:34:48.689191+0000","last_change":"2023-06-03T19:32:22.123980+0000","last_active":"2023-06-03T19:34:48.689191+0000","last_peered":"2023-06-03T19:34:48.689191+0000","last_clean":"2023-06-03T19:34:48.689191+0000","last_became_active":"2023-06-03T19:32:22.123641+0000","last_became_peered":"2023-06-03T19:32:22.123641+0000","last_unstale":"2023-06-03T19:34:48.689191+0000","last_undegraded":"2023-06-03T19:34:48.689191+0000","last_fullsized":"2023-06-03T19:34:48.689191+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:32:20.692043+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:32:20.692043+0000","last_clean_scrub_stamp":"2023-06-03T19:32:20.692043+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":13,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-06-03T19:34:48.694664+0000","last_change":"2023-06-03T19:34:17.829377+0000","last_active":"2023-06-03T19:34:48.694664+0000","last_peered":"2023-06-03T19:34:48.694664+0000","last_clean":"2023-06-03T19:34:48.694664+0000","last_became_active":"2023-06-03T19:34:17.829027+0000","last_became_peered":"2023-06-03T19:34:17.829027+0000","last_unstale":"2023-06-03T19:34:48.694664+0000","last_undegraded":"2023-06-03T19:34:48.694664+0000","last_fullsized":"2023-06-03T19:34:48.694664+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":14,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-06-03T19:34:48.688564+0000","last_change":"2023-06-03T19:34:16.804662+0000","last_active":"2023-06-03T19:34:48.688564+0000","last_peered":"2023-06-03T19:34:48.688564+0000","last_clean":"2023-06-03T19:34:48.688564+0000","last_became_active":"2023-06-03T19:34:16.804350+0000","last_became_peered":"2023-06-03T19:34:16.804350+0000","last_unstale":"2023-06-03T19:34:48.688564+0000","last_undegraded":"2023-06-03T19:34:48.688564+0000","last_fullsized":"2023-06-03T19:34:48.688564+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":18,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-06-03T19:34:48.685279+0000","last_change":"2023-06-03T19:32:21.709485+0000","last_active":"2023-06-03T19:34:48.685279+0000","last_peered":"2023-06-03T19:34:48.685279+0000","last_clean":"2023-06-03T19:34:48.685279+0000","last_became_active":"2023-06-03T19:32:21.708866+0000","last_became_peered":"2023-06-03T19:32:21.708866+0000","last_unstale":"2023-06-03T19:34:48.685279+0000","last_undegraded":"2023-06-03T19:34:48.685279+0000","last_fullsized":"2023-06-03T19:34:48.685279+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:32:20.692043+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:32:20.692043+0000","last_clean_scrub_stamp":"2023-06-03T19:32:20.692043+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":13,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-06-03T19:34:48.692657+0000","last_change":"2023-06-03T19:34:17.833656+0000","last_active":"2023-06-03T19:34:48.692657+0000","last_peered":"2023-06-03T19:34:48.692657+0000","last_clean":"2023-06-03T19:34:48.692657+0000","last_became_active":"2023-06-03T19:34:17.833319+0000","last_became_peered":"2023-06-03T19:34:17.833319+0000","last_unstale":"2023-06-03T19:34:48.692657+0000","last_undegraded":"2023-06-03T19:34:48.692657+0000","last_fullsized":"2023-06-03T19:34:48.692657+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":14,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-06-03T19:34:48.685848+0000","last_change":"2023-06-03T19:34:16.803369+0000","last_active":"2023-06-03T19:34:48.685848+0000","last_peered":"2023-06-03T19:34:48.685848+0000","last_clean":"2023-06-03T19:34:48.685848+0000","last_became_active":"2023-06-03T19:34:16.803020+0000","last_became_peered":"2023-06-03T19:34:16.803020+0000","last_unstale":"2023-06-03T19:34:48.685848+0000","last_undegraded":"2023-06-03T19:34:48.685848+0000","last_fullsized":"2023-06-03T19:34:48.685848+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":18,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-06-03T19:34:48.694479+0000","last_change":"2023-06-03T19:32:22.125458+0000","last_active":"2023-06-03T19:34:48.694479+0000","last_peered":"2023-06-03T19:34:48.694479+0000","last_clean":"2023-06-03T19:34:48.694479+0000","last_became_active":"2023-06-03T19:32:22.124914+0000","last_became_peered":"2023-06-03T19:32:22.124914+0000","last_unstale":"2023-06-03T19:34:48.694479+0000","last_undegraded":"2023-06-03T19:34:48.694479+0000","last_fullsized":"2023-06-03T19:34:48.694479+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:32:20.692043+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:32:20.692043+0000","last_clean_scrub_stamp":"2023-06-03T19:32:20.692043+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":13,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-06-03T19:34:48.695380+0000","last_change":"2023-06-03T19:34:17.833703+0000","last_active":"2023-06-03T19:34:48.695380+0000","last_peered":"2023-06-03T19:34:48.695380+0000","last_clean":"2023-06-03T19:34:48.695380+0000","last_became_active":"2023-06-03T19:34:17.833068+0000","last_became_peered":"2023-06-03T19:34:17.833068+0000","last_unstale":"2023-06-03T19:34:48.695380+0000","last_undegraded":"2023-06-03T19:34:48.695380+0000","last_fullsized":"2023-06-03T19:34:48.695380+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":14,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-06-03T19:34:48.686481+0000","last_change":"2023-06-03T19:34:16.810433+0000","last_active":"2023-06-03T19:34:48.686481+0000","last_peered":"2023-06-03T19:34:48.686481+0000","last_clean":"2023-06-03T19:34:48.686481+0000","last_became_active":"2023-06-03T19:34:16.810104+0000","last_became_peered":"2023-06-03T19:34:16.810104+0000","last_unstale":"2023-06-03T19:34:48.686481+0000","last_undegraded":"2023-06-03T19:34:48.686481+0000","last_fullsized":"2023-06-03T19:34:48.686481+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":13,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-06-03T19:34:48.689007+0000","last_change":"2023-06-03T19:34:17.831828+0000","last_active":"2023-06-03T19:34:48.689007+0000","last_peered":"2023-06-03T19:34:48.689007+0000","last_clean":"2023-06-03T19:34:48.689007+0000","last_became_active":"2023-06-03T19:34:17.831621+0000","last_became_peered":"2023-06-03T19:34:17.831621+0000","last_unstale":"2023-06-03T19:34:48.689007+0000","last_undegraded":"2023-06-03T19:34:48.689007+0000","last_fullsized":"2023-06-03T19:34:48.689007+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":14,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-06-03T19:34:48.689532+0000","last_change":"2023-06-03T19:34:16.800660+0000","last_active":"2023-06-03T19:34:48.689532+0000","last_peered":"2023-06-03T19:34:48.689532+0000","last_clean":"2023-06-03T19:34:48.689532+0000","last_became_active":"2023-06-03T19:34:16.800341+0000","last_became_peered":"2023-06-03T19:34:16.800341+0000","last_unstale":"2023-06-03T19:34:48.689532+0000","last_undegraded":"2023-06-03T19:34:48.689532+0000","last_fullsized":"2023-06-03T19:34:48.689532+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":22,"reported_epoch":62,"state":"active+clean","last_fresh":"2023-06-03T19:34:50.193972+0000","last_change":"2023-06-03T19:34:50.193972+0000","last_active":"2023-06-03T19:34:50.193972+0000","last_peered":"2023-06-03T19:34:50.193972+0000","last_clean":"2023-06-03T19:34:50.193972+0000","last_became_active":"2023-06-03T19:34:50.193301+0000","last_became_peered":"2023-06-03T19:34:50.193301+0000","last_unstale":"2023-06-03T19:34:50.193972+0000","last_undegraded":"2023-06-03T19:34:50.193972+0000","last_fullsized":"2023-06-03T19:34:50.193972+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,5],"acting":[4,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":14,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-06-03T19:34:48.711628+0000","last_change":"2023-06-03T19:34:16.804631+0000","last_active":"2023-06-03T19:34:48.711628+0000","last_peered":"2023-06-03T19:34:48.711628+0000","last_clean":"2023-06-03T19:34:48.711628+0000","last_became_active":"2023-06-03T19:34:16.804252+0000","last_became_peered":"2023-06-03T19:34:16.804252+0000","last_unstale":"2023-06-03T19:34:48.711628+0000","last_undegraded":"2023-06-03T19:34:48.711628+0000","last_fullsized":"2023-06-03T19:34:48.711628+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":13,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-06-03T19:34:48.689069+0000","last_change":"2023-06-03T19:34:17.833032+0000","last_active":"2023-06-03T19:34:48.689069+0000","last_peered":"2023-06-03T19:34:48.689069+0000","last_clean":"2023-06-03T19:34:48.689069+0000","last_became_active":"2023-06-03T19:34:17.832515+0000","last_became_peered":"2023-06-03T19:34:17.832515+0000","last_unstale":"2023-06-03T19:34:48.689069+0000","last_undegraded":"2023-06-03T19:34:48.689069+0000","last_fullsized":"2023-06-03T19:34:48.689069+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.b","version":"60'2","reported_seq":16,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-06-03T19:34:48.686810+0000","last_change":"2023-06-03T19:34:16.812257+0000","last_active":"2023-06-03T19:34:48.686810+0000","last_peered":"2023-06-03T19:34:48.686810+0000","last_clean":"2023-06-03T19:34:48.686810+0000","last_became_active":"2023-06-03T19:34:16.811863+0000","last_became_peered":"2023-06-03T19:34:16.811863+0000","last_unstale":"2023-06-03T19:34:48.686810+0000","last_undegraded":"2023-06-03T19:34:48.686810+0000","last_fullsized":"2023-06-03T19:34:48.686810+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","log_size":2,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":22,"reported_epoch":62,"state":"active+clean","last_fresh":"2023-06-03T19:34:50.186367+0000","last_change":"2023-06-03T19:34:50.186367+0000","last_active":"2023-06-03T19:34:50.186367+0000","last_peered":"2023-06-03T19:34:50.186367+0000","last_clean":"2023-06-03T19:34:50.186367+0000","last_became_active":"2023-06-03T19:34:50.185595+0000","last_became_peered":"2023-06-03T19:34:50.185595+0000","last_unstale":"2023-06-03T19:34:50.186367+0000","last_undegraded":"2023-06-03T19:34:50.186367+0000","last_fullsized":"2023-06-03T19:34:50.186367+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,4],"acting":[0,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":14,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-06-03T19:34:48.685592+0000","last_change":"2023-06-03T19:34:16.798436+0000","last_active":"2023-06-03T19:34:48.685592+0000","last_peered":"2023-06-03T19:34:48.685592+0000","last_clean":"2023-06-03T19:34:48.685592+0000","last_became_active":"2023-06-03T19:34:16.798053+0000","last_became_peered":"2023-06-03T19:34:16.798053+0000","last_unstale":"2023-06-03T19:34:48.685592+0000","last_undegraded":"2023-06-03T19:34:48.685592+0000","last_fullsized":"2023-06-03T19:34:48.685592+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":13,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-06-03T19:34:48.711807+0000","last_change":"2023-06-03T19:34:17.848644+0000","last_active":"2023-06-03T19:34:48.711807+0000","last_peered":"2023-06-03T19:34:48.711807+0000","last_clean":"2023-06-03T19:34:48.711807+0000","last_became_active":"2023-06-03T19:34:17.848051+0000","last_became_peered":"2023-06-03T19:34:17.848051+0000","last_unstale":"2023-06-03T19:34:48.711807+0000","last_undegraded":"2023-06-03T19:34:48.711807+0000","last_fullsized":"2023-06-03T19:34:48.711807+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"60'1","reported_seq":15,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-06-03T19:34:48.712805+0000","last_change":"2023-06-03T19:34:16.806559+0000","last_active":"2023-06-03T19:34:48.712805+0000","last_peered":"2023-06-03T19:34:48.712805+0000","last_clean":"2023-06-03T19:34:48.712805+0000","last_became_active":"2023-06-03T19:34:16.806149+0000","last_became_peered":"2023-06-03T19:34:16.806149+0000","last_unstale":"2023-06-03T19:34:48.712805+0000","last_undegraded":"2023-06-03T19:34:48.712805+0000","last_fullsized":"2023-06-03T19:34:48.712805+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","log_size":1,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":22,"reported_epoch":62,"state":"active+clean","last_fresh":"2023-06-03T19:34:50.184855+0000","last_change":"2023-06-03T19:34:50.184855+0000","last_active":"2023-06-03T19:34:50.184855+0000","last_peered":"2023-06-03T19:34:50.184855+0000","last_clean":"2023-06-03T19:34:50.184855+0000","last_became_active":"2023-06-03T19:34:50.184070+0000","last_became_peered":"2023-06-03T19:34:50.184070+0000","last_unstale":"2023-06-03T19:34:50.184855+0000","last_undegraded":"2023-06-03T19:34:50.184855+0000","last_fullsized":"2023-06-03T19:34:50.184855+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,3],"acting":[4,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.e","version":"60'2","reported_seq":16,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-06-03T19:34:48.712689+0000","last_change":"2023-06-03T19:34:16.800653+0000","last_active":"2023-06-03T19:34:48.712689+0000","last_peered":"2023-06-03T19:34:48.712689+0000","last_clean":"2023-06-03T19:34:48.712689+0000","last_became_active":"2023-06-03T19:34:16.800085+0000","last_became_peered":"2023-06-03T19:34:16.800085+0000","last_unstale":"2023-06-03T19:34:48.712689+0000","last_undegraded":"2023-06-03T19:34:48.712689+0000","last_fullsized":"2023-06-03T19:34:48.712689+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","log_size":2,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":13,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-06-03T19:34:48.685239+0000","last_change":"2023-06-03T19:34:17.848358+0000","last_active":"2023-06-03T19:34:48.685239+0000","last_peered":"2023-06-03T19:34:48.685239+0000","last_clean":"2023-06-03T19:34:48.685239+0000","last_became_active":"2023-06-03T19:34:17.847811+0000","last_became_peered":"2023-06-03T19:34:17.847811+0000","last_unstale":"2023-06-03T19:34:48.685239+0000","last_undegraded":"2023-06-03T19:34:48.685239+0000","last_fullsized":"2023-06-03T19:34:48.685239+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"60'1","reported_seq":15,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-06-03T19:34:48.712873+0000","last_change":"2023-06-03T19:34:16.807961+0000","last_active":"2023-06-03T19:34:48.712873+0000","last_peered":"2023-06-03T19:34:48.712873+0000","last_clean":"2023-06-03T19:34:48.712873+0000","last_became_active":"2023-06-03T19:34:16.805920+0000","last_became_peered":"2023-06-03T19:34:16.805920+0000","last_unstale":"2023-06-03T19:34:48.712873+0000","last_undegraded":"2023-06-03T19:34:48.712873+0000","last_fullsized":"2023-06-03T19:34:48.712873+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","log_size":1,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":13,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-06-03T19:34:48.689128+0000","last_change":"2023-06-03T19:34:16.896490+0000","last_active":"2023-06-03T19:34:48.689128+0000","last_peered":"2023-06-03T19:34:48.689128+0000","last_clean":"2023-06-03T19:34:48.689128+0000","last_became_active":"2023-06-03T19:34:16.896119+0000","last_became_peered":"2023-06-03T19:34:16.896119+0000","last_unstale":"2023-06-03T19:34:48.689128+0000","last_undegraded":"2023-06-03T19:34:48.689128+0000","last_fullsized":"2023-06-03T19:34:48.689128+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"60'8","reported_seq":22,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-06-03T19:34:48.711523+0000","last_change":"2023-06-03T19:34:16.803630+0000","last_active":"2023-06-03T19:34:48.711523+0000","last_peered":"2023-06-03T19:34:48.711523+0000","last_clean":"2023-06-03T19:34:48.711523+0000","last_became_active":"2023-06-03T19:34:16.803280+0000","last_became_peered":"2023-06-03T19:34:16.803280+0000","last_unstale":"2023-06-03T19:34:48.711523+0000","last_undegraded":"2023-06-03T19:34:48.711523+0000","last_fullsized":"2023-06-03T19:34:48.711523+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","log_size":8,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":13,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-06-03T19:34:48.693907+0000","last_change":"2023-06-03T19:34:17.850887+0000","last_active":"2023-06-03T19:34:48.693907+0000","last_peered":"2023-06-03T19:34:48.693907+0000","last_clean":"2023-06-03T19:34:48.693907+0000","last_became_active":"2023-06-03T19:34:17.850146+0000","last_became_peered":"2023-06-03T19:34:17.850146+0000","last_unstale":"2023-06-03T19:34:48.693907+0000","last_undegraded":"2023-06-03T19:34:48.693907+0000","last_fullsized":"2023-06-03T19:34:48.693907+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":14,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-06-03T19:34:48.712131+0000","last_change":"2023-06-03T19:34:16.806286+0000","last_active":"2023-06-03T19:34:48.712131+0000","last_peered":"2023-06-03T19:34:48.712131+0000","last_clean":"2023-06-03T19:34:48.712131+0000","last_became_active":"2023-06-03T19:34:16.805898+0000","last_became_peered":"2023-06-03T19:34:16.805898+0000","last_unstale":"2023-06-03T19:34:48.712131+0000","last_undegraded":"2023-06-03T19:34:48.712131+0000","last_fullsized":"2023-06-03T19:34:48.712131+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":13,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-06-03T19:34:48.685691+0000","last_change":"2023-06-03T19:34:17.848569+0000","last_active":"2023-06-03T19:34:48.685691+0000","last_peered":"2023-06-03T19:34:48.685691+0000","last_clean":"2023-06-03T19:34:48.685691+0000","last_became_active":"2023-06-03T19:34:17.848038+0000","last_became_peered":"2023-06-03T19:34:17.848038+0000","last_unstale":"2023-06-03T19:34:48.685691+0000","last_undegraded":"2023-06-03T19:34:48.685691+0000","last_fullsized":"2023-06-03T19:34:48.685691+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":14,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-06-03T19:34:48.694032+0000","last_change":"2023-06-03T19:34:16.814726+0000","last_active":"2023-06-03T19:34:48.694032+0000","last_peered":"2023-06-03T19:34:48.694032+0000","last_clean":"2023-06-03T19:34:48.694032+0000","last_became_active":"2023-06-03T19:34:16.814150+0000","last_became_peered":"2023-06-03T19:34:16.814150+0000","last_unstale":"2023-06-03T19:34:48.694032+0000","last_undegraded":"2023-06-03T19:34:48.694032+0000","last_fullsized":"2023-06-03T19:34:48.694032+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":13,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-06-03T19:34:48.688078+0000","last_change":"2023-06-03T19:34:16.896666+0000","last_active":"2023-06-03T19:34:48.688078+0000","last_peered":"2023-06-03T19:34:48.688078+0000","last_clean":"2023-06-03T19:34:48.688078+0000","last_became_active":"2023-06-03T19:34:16.895977+0000","last_became_peered":"2023-06-03T19:34:16.895977+0000","last_unstale":"2023-06-03T19:34:48.688078+0000","last_undegraded":"2023-06-03T19:34:48.688078+0000","last_fullsized":"2023-06-03T19:34:48.688078+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"60'2","reported_seq":16,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-06-03T19:34:48.712350+0000","last_change":"2023-06-03T19:34:16.800482+0000","last_active":"2023-06-03T19:34:48.712350+0000","last_peered":"2023-06-03T19:34:48.712350+0000","last_clean":"2023-06-03T19:34:48.712350+0000","last_became_active":"2023-06-03T19:34:16.799932+0000","last_became_peered":"2023-06-03T19:34:16.799932+0000","last_unstale":"2023-06-03T19:34:48.712350+0000","last_undegraded":"2023-06-03T19:34:48.712350+0000","last_fullsized":"2023-06-03T19:34:48.712350+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","log_size":2,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":13,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-06-03T19:34:48.688794+0000","last_change":"2023-06-03T19:34:17.832978+0000","last_active":"2023-06-03T19:34:48.688794+0000","last_peered":"2023-06-03T19:34:48.688794+0000","last_clean":"2023-06-03T19:34:48.688794+0000","last_became_active":"2023-06-03T19:34:17.832312+0000","last_became_peered":"2023-06-03T19:34:17.832312+0000","last_unstale":"2023-06-03T19:34:48.688794+0000","last_undegraded":"2023-06-03T19:34:48.688794+0000","last_fullsized":"2023-06-03T19:34:48.688794+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"60'6","reported_seq":20,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-06-03T19:34:48.688869+0000","last_change":"2023-06-03T19:34:16.800070+0000","last_active":"2023-06-03T19:34:48.688869+0000","last_peered":"2023-06-03T19:34:48.688869+0000","last_clean":"2023-06-03T19:34:48.688869+0000","last_became_active":"2023-06-03T19:34:16.798944+0000","last_became_peered":"2023-06-03T19:34:16.798944+0000","last_unstale":"2023-06-03T19:34:48.688869+0000","last_undegraded":"2023-06-03T19:34:48.688869+0000","last_fullsized":"2023-06-03T19:34:48.688869+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","log_size":6,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":1536,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6,"num_write_kb":3,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":13,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-06-03T19:34:48.692607+0000","last_change":"2023-06-03T19:34:17.829839+0000","last_active":"2023-06-03T19:34:48.692607+0000","last_peered":"2023-06-03T19:34:48.692607+0000","last_clean":"2023-06-03T19:34:48.692607+0000","last_became_active":"2023-06-03T19:34:17.829509+0000","last_became_peered":"2023-06-03T19:34:17.829509+0000","last_unstale":"2023-06-03T19:34:48.692607+0000","last_undegraded":"2023-06-03T19:34:48.692607+0000","last_fullsized":"2023-06-03T19:34:48.692607+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"60'3","reported_seq":17,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-06-03T19:34:48.712587+0000","last_change":"2023-06-03T19:34:16.808044+0000","last_active":"2023-06-03T19:34:48.712587+0000","last_peered":"2023-06-03T19:34:48.712587+0000","last_clean":"2023-06-03T19:34:48.712587+0000","last_became_active":"2023-06-03T19:34:16.807735+0000","last_became_peered":"2023-06-03T19:34:16.807735+0000","last_unstale":"2023-06-03T19:34:48.712587+0000","last_undegraded":"2023-06-03T19:34:48.712587+0000","last_fullsized":"2023-06-03T19:34:48.712587+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","log_size":3,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":13,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-06-03T19:34:48.687564+0000","last_change":"2023-06-03T19:34:17.293132+0000","last_active":"2023-06-03T19:34:48.687564+0000","last_peered":"2023-06-03T19:34:48.687564+0000","last_clean":"2023-06-03T19:34:48.687564+0000","last_became_active":"2023-06-03T19:34:17.292642+0000","last_became_peered":"2023-06-03T19:34:17.292642+0000","last_unstale":"2023-06-03T19:34:48.687564+0000","last_undegraded":"2023-06-03T19:34:48.687564+0000","last_fullsized":"2023-06-03T19:34:48.687564+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":14,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-06-03T19:34:48.685695+0000","last_change":"2023-06-03T19:34:16.800538+0000","last_active":"2023-06-03T19:34:48.685695+0000","last_peered":"2023-06-03T19:34:48.685695+0000","last_clean":"2023-06-03T19:34:48.685695+0000","last_became_active":"2023-06-03T19:34:16.799538+0000","last_became_peered":"2023-06-03T19:34:16.799538+0000","last_unstale":"2023-06-03T19:34:48.685695+0000","last_undegraded":"2023-06-03T19:34:48.685695+0000","last_fullsized":"2023-06-03T19:34:48.685695+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":13,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-06-03T19:34:48.686586+0000","last_change":"2023-06-03T19:34:17.293481+0000","last_active":"2023-06-03T19:34:48.686586+0000","last_peered":"2023-06-03T19:34:48.686586+0000","last_clean":"2023-06-03T19:34:48.686586+0000","last_became_active":"2023-06-03T19:34:17.292639+0000","last_became_peered":"2023-06-03T19:34:17.292639+0000","last_unstale":"2023-06-03T19:34:48.686586+0000","last_undegraded":"2023-06-03T19:34:48.686586+0000","last_fullsized":"2023-06-03T19:34:48.686586+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":14,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-06-03T19:34:48.694274+0000","last_change":"2023-06-03T19:34:16.814709+0000","last_active":"2023-06-03T19:34:48.694274+0000","last_peered":"2023-06-03T19:34:48.694274+0000","last_clean":"2023-06-03T19:34:48.694274+0000","last_became_active":"2023-06-03T19:34:16.814149+0000","last_became_peered":"2023-06-03T19:34:16.814149+0000","last_unstale":"2023-06-03T19:34:48.694274+0000","last_undegraded":"2023-06-03T19:34:48.694274+0000","last_fullsized":"2023-06-03T19:34:48.694274+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":13,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-06-03T19:34:48.711508+0000","last_change":"2023-06-03T19:34:17.825928+0000","last_active":"2023-06-03T19:34:48.711508+0000","last_peered":"2023-06-03T19:34:48.711508+0000","last_clean":"2023-06-03T19:34:48.711508+0000","last_became_active":"2023-06-03T19:34:17.824849+0000","last_became_peered":"2023-06-03T19:34:17.824849+0000","last_unstale":"2023-06-03T19:34:48.711508+0000","last_undegraded":"2023-06-03T19:34:48.711508+0000","last_fullsized":"2023-06-03T19:34:48.711508+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":14,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-06-03T19:34:48.686730+0000","last_change":"2023-06-03T19:34:16.810509+0000","last_active":"2023-06-03T19:34:48.686730+0000","last_peered":"2023-06-03T19:34:48.686730+0000","last_clean":"2023-06-03T19:34:48.686730+0000","last_became_active":"2023-06-03T19:34:16.810046+0000","last_became_peered":"2023-06-03T19:34:16.810046+0000","last_unstale":"2023-06-03T19:34:48.686730+0000","last_undegraded":"2023-06-03T19:34:48.686730+0000","last_fullsized":"2023-06-03T19:34:48.686730+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":13,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-06-03T19:34:48.694018+0000","last_change":"2023-06-03T19:34:17.850356+0000","last_active":"2023-06-03T19:34:48.694018+0000","last_peered":"2023-06-03T19:34:48.694018+0000","last_clean":"2023-06-03T19:34:48.694018+0000","last_became_active":"2023-06-03T19:34:17.849850+0000","last_became_peered":"2023-06-03T19:34:17.849850+0000","last_unstale":"2023-06-03T19:34:48.694018+0000","last_undegraded":"2023-06-03T19:34:48.694018+0000","last_fullsized":"2023-06-03T19:34:48.694018+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":14,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-06-03T19:34:48.693391+0000","last_change":"2023-06-03T19:34:16.805815+0000","last_active":"2023-06-03T19:34:48.693391+0000","last_peered":"2023-06-03T19:34:48.693391+0000","last_clean":"2023-06-03T19:34:48.693391+0000","last_became_active":"2023-06-03T19:34:16.805501+0000","last_became_peered":"2023-06-03T19:34:16.805501+0000","last_unstale":"2023-06-03T19:34:48.693391+0000","last_undegraded":"2023-06-03T19:34:48.693391+0000","last_fullsized":"2023-06-03T19:34:48.693391+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":13,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-06-03T19:34:48.688274+0000","last_change":"2023-06-03T19:34:16.822573+0000","last_active":"2023-06-03T19:34:48.688274+0000","last_peered":"2023-06-03T19:34:48.688274+0000","last_clean":"2023-06-03T19:34:48.688274+0000","last_became_active":"2023-06-03T19:34:16.822213+0000","last_became_peered":"2023-06-03T19:34:16.822213+0000","last_unstale":"2023-06-03T19:34:48.688274+0000","last_undegraded":"2023-06-03T19:34:48.688274+0000","last_fullsized":"2023-06-03T19:34:48.688274+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"60'1","reported_seq":15,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-06-03T19:34:48.688450+0000","last_change":"2023-06-03T19:34:16.804144+0000","last_active":"2023-06-03T19:34:48.688450+0000","last_peered":"2023-06-03T19:34:48.688450+0000","last_clean":"2023-06-03T19:34:48.688450+0000","last_became_active":"2023-06-03T19:34:16.803674+0000","last_became_peered":"2023-06-03T19:34:16.803674+0000","last_unstale":"2023-06-03T19:34:48.688450+0000","last_undegraded":"2023-06-03T19:34:48.688450+0000","last_fullsized":"2023-06-03T19:34:48.688450+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","log_size":1,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":2306,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":45,"num_write_kb":13,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":6918,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":39,"ondisk_log_size":39,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":3,"acting":3,"num_store_stats":3},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":48,"seq":206158430267,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":5704,"kb_used_data":640,"kb_used_omap":0,"kb_used_meta":5056,"kb_avail":93739448,"statfs":{"total":95995035648,"available":95989194752,"internally_reserved":0,"allocated":655360,"data_stored":271688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5177344},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 3 19:35:29 2023","interfaces":[{"interface":"back","average":{"1min":0.87,"5min":0.84599999999999997,"15min":0.81999999999999995},"min":{"1min":0.57399999999999995,"5min":0.54500000000000004,"15min":0.54500000000000004},"max":{"1min":1.0649999999999999,"5min":1.232,"15min":1.232},"last":1.0800000000000001},{"interface":"front","average":{"1min":0.95399999999999996,"5min":0.90800000000000003,"15min":0.88300000000000001},"min":{"1min":0.60299999999999998,"5min":0.56499999999999995,"15min":0.56499999999999995},"max":{"1min":1.2849999999999999,"5min":1.2849999999999999,"15min":1.2849999999999999},"last":1.0980000000000001}]},{"osd":1,"last update":"Sat Jun 3 19:35:29 2023","interfaces":[{"interface":"back","average":{"1min":0.89700000000000002,"5min":0.86099999999999999,"15min":0.83299999999999996},"min":{"1min":0.58999999999999997,"5min":0.51700000000000002,"15min":0.51700000000000002},"max":{"1min":1.155,"5min":1.2130000000000001,"15min":1.2130000000000001},"last":0.86499999999999999},{"interface":"front","average":{"1min":0.95399999999999996,"5min":0.90300000000000002,"15min":0.878},"min":{"1min":0.622,"5min":0.54500000000000004,"15min":0.54500000000000004},"max":{"1min":1.1879999999999999,"5min":1.228,"15min":1.228},"last":1.1120000000000001}]},{"osd":2,"last update":"Sat Jun 3 19:35:29 2023","interfaces":[{"interface":"back","average":{"1min":0.92500000000000004,"5min":0.93000000000000005,"15min":0.92100000000000004},"min":{"1min":0.63,"5min":0.58999999999999997,"15min":0.58999999999999997},"max":{"1min":1.131,"5min":1.538,"15min":1.538},"last":0.90000000000000002},{"interface":"front","average":{"1min":0.94499999999999995,"5min":0.90900000000000003,"15min":0.876},"min":{"1min":0.74399999999999999,"5min":0.60699999999999998,"15min":0.60699999999999998},"max":{"1min":1.1819999999999999,"5min":1.24,"15min":1.24},"last":1.1439999999999999}]},{"osd":3,"last update":"Sat Jun 3 19:35:29 2023","interfaces":[{"interface":"back","average":{"1min":0.94499999999999995,"5min":0.93300000000000005,"15min":0.91500000000000004},"min":{"1min":0.73099999999999998,"5min":0.53800000000000003,"15min":0.53800000000000003},"max":{"1min":1.22,"5min":1.3069999999999999,"15min":1.3069999999999999},"last":0.997},{"interface":"front","average":{"1min":0.94499999999999995,"5min":0.93600000000000005,"15min":0.93200000000000005},"min":{"1min":0.71499999999999997,"5min":0.57199999999999995,"15min":0.57199999999999995},"max":{"1min":1.1919999999999999,"5min":1.5069999999999999,"15min":1.5069999999999999},"last":1.024}]},{"osd":4,"last update":"Sat Jun 3 19:35:29 2023","interfaces":[{"interface":"back","average":{"1min":0.94899999999999995,"5min":0.83699999999999997,"15min":0.76200000000000001},"min":{"1min":0.69099999999999995,"5min":0.51200000000000001,"15min":0.51200000000000001},"max":{"1min":1.165,"5min":1.2829999999999999,"15min":1.2829999999999999},"last":0.78200000000000003},{"interface":"front","average":{"1min":0.94299999999999995,"5min":0.88500000000000001,"15min":0.84099999999999997},"min":{"1min":0.60699999999999998,"5min":0.45500000000000002,"15min":0.45500000000000002},"max":{"1min":1.2769999999999999,"5min":1.351,"15min":1.351},"last":1.2}]},{"osd":5,"last update":"Sat Jun 3 19:35:29 2023","interfaces":[{"interface":"back","average":{"1min":0.996,"5min":0.89200000000000002,"15min":0.83599999999999997},"min":{"1min":0.65300000000000002,"5min":0.48499999999999999,"15min":0.48499999999999999},"max":{"1min":1.3560000000000001,"5min":1.423,"15min":1.423},"last":0.94599999999999995},{"interface":"front","average":{"1min":1.0149999999999999,"5min":0.97099999999999997,"15min":0.94099999999999995},"min":{"1min":0.64700000000000002,"5min":0.437,"15min":0.437},"max":{"1min":1.268,"5min":1.605,"15min":1.605},"last":0.96699999999999997}]},{"osd":6,"last update":"Sat Jun 3 19:35:29 2023","interfaces":[{"interface":"back","average":{"1min":1.054,"5min":1.03,"15min":0.98799999999999999},"min":{"1min":0.55900000000000005,"5min":0.47899999999999998,"15min":0.47899999999999998},"max":{"1min":1.254,"5min":1.6870000000000001,"15min":1.6870000000000001},"last":1.0580000000000001},{"interface":"front","average":{"1min":1.0569999999999999,"5min":1.002,"15min":0.95999999999999996},"min":{"1min":0.73199999999999998,"5min":0.59299999999999997,"15min":0.59299999999999997},"max":{"1min":1.329,"5min":1.4319999999999999,"15min":1.4319999999999999},"last":1.248}]}]},{"osd":6,"up_from":42,"seq":180388626500,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":5688,"kb_used_data":624,"kb_used_omap":0,"kb_used_meta":5056,"kb_avail":93739464,"statfs":{"total":95995035648,"available":95989211136,"internally_reserved":0,"allocated":638976,"data_stored":271227,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5177344},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 3 19:35:38 2023","interfaces":[{"interface":"back","average":{"1min":0.84799999999999998,"5min":0.84599999999999997,"15min":0.83199999999999996},"min":{"1min":0.45400000000000001,"5min":0.45400000000000001,"15min":0.45400000000000001},"max":{"1min":1.2350000000000001,"5min":1.2749999999999999,"15min":1.2749999999999999},"last":0.76100000000000001},{"interface":"front","average":{"1min":0.84899999999999998,"5min":0.86099999999999999,"15min":0.86799999999999999},"min":{"1min":0.57699999999999996,"5min":0.52900000000000003,"15min":0.52900000000000003},"max":{"1min":1.2110000000000001,"5min":1.284,"15min":1.284},"last":1.2609999999999999}]},{"osd":1,"last update":"Sat Jun 3 19:35:38 2023","interfaces":[{"interface":"back","average":{"1min":0.878,"5min":0.873,"15min":0.86899999999999999},"min":{"1min":0.58099999999999996,"5min":0.50700000000000001,"15min":0.50700000000000001},"max":{"1min":1.143,"5min":1.3759999999999999,"15min":1.3759999999999999},"last":1.1699999999999999},{"interface":"front","average":{"1min":0.89000000000000001,"5min":0.89900000000000002,"15min":0.93999999999999995},"min":{"1min":0.66800000000000004,"5min":0.54100000000000004,"15min":0.54100000000000004},"max":{"1min":1.1299999999999999,"5min":1.7769999999999999,"15min":1.7769999999999999},"last":1.151}]},{"osd":2,"last update":"Sat Jun 3 19:35:38 2023","interfaces":[{"interface":"back","average":{"1min":0.874,"5min":0.89900000000000002,"15min":0.91300000000000003},"min":{"1min":0.68000000000000005,"5min":0.58199999999999996,"15min":0.58199999999999996},"max":{"1min":1.1220000000000001,"5min":1.3069999999999999,"15min":1.3069999999999999},"last":1.0409999999999999},{"interface":"front","average":{"1min":0.996,"5min":0.93899999999999995,"15min":0.93500000000000005},"min":{"1min":0.70199999999999996,"5min":0.63400000000000001,"15min":0.63400000000000001},"max":{"1min":1.2949999999999999,"5min":1.3400000000000001,"15min":1.3400000000000001},"last":1.119}]},{"osd":3,"last update":"Sat Jun 3 19:35:38 2023","interfaces":[{"interface":"back","average":{"1min":0.93999999999999995,"5min":0.88900000000000001,"15min":0.85799999999999998},"min":{"1min":0.65600000000000003,"5min":0.60199999999999998,"15min":0.60199999999999998},"max":{"1min":1.1919999999999999,"5min":1.2290000000000001,"15min":1.2290000000000001},"last":1.0740000000000001},{"interface":"front","average":{"1min":1.008,"5min":0.97699999999999998,"15min":0.998},"min":{"1min":0.76900000000000002,"5min":0.64200000000000002,"15min":0.64200000000000002},"max":{"1min":1.177,"5min":1.905,"15min":1.905},"last":1.095}]},{"osd":4,"last update":"Sat Jun 3 19:35:38 2023","interfaces":[{"interface":"back","average":{"1min":0.92800000000000005,"5min":0.88800000000000001,"15min":0.85199999999999998},"min":{"1min":0.59699999999999998,"5min":0.46899999999999997,"15min":0.46899999999999997},"max":{"1min":1.2250000000000001,"5min":1.3140000000000001,"15min":1.3140000000000001},"last":1.2030000000000001},{"interface":"front","average":{"1min":0.88300000000000001,"5min":0.90800000000000003,"15min":0.88},"min":{"1min":0.65200000000000002,"5min":0.434,"15min":0.434},"max":{"1min":1.284,"5min":1.3129999999999999,"15min":1.3129999999999999},"last":1.2390000000000001}]},{"osd":5,"last update":"Sat Jun 3 19:35:38 2023","interfaces":[{"interface":"back","average":{"1min":0.98399999999999999,"5min":0.96899999999999997,"15min":0.93100000000000005},"min":{"1min":0.64400000000000002,"5min":0.41599999999999998,"15min":0.41599999999999998},"max":{"1min":1.236,"5min":1.3109999999999999,"15min":1.3109999999999999},"last":1.3149999999999999},{"interface":"front","average":{"1min":0.99099999999999999,"5min":0.96199999999999997,"15min":0.94399999999999995},"min":{"1min":0.72799999999999998,"5min":0.52500000000000002,"15min":0.52500000000000002},"max":{"1min":1.236,"5min":1.284,"15min":1.284},"last":1.288}]},{"osd":7,"last update":"Sat Jun 3 19:35:22 2023","interfaces":[{"interface":"back","average":{"1min":0.99199999999999999,"5min":0.998,"15min":0.95499999999999996},"min":{"1min":0.70399999999999996,"5min":0.45700000000000002,"15min":0.45700000000000002},"max":{"1min":1.1839999999999999,"5min":1.403,"15min":1.403},"last":1.367},{"interface":"front","average":{"1min":1.0509999999999999,"5min":1.028,"15min":1.006},"min":{"1min":0.69199999999999995,"5min":0.53600000000000003,"15min":0.53600000000000003},"max":{"1min":1.3109999999999999,"5min":1.3839999999999999,"15min":1.3839999999999999},"last":1.222}]}]},{"osd":1,"up_from":13,"seq":55834574962,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":5928,"kb_used_data":608,"kb_used_omap":0,"kb_used_meta":5312,"kb_avail":93739224,"statfs":{"total":95995035648,"available":95988965376,"internally_reserved":0,"allocated":622592,"data_stored":269621,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5439488},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 3 19:35:51 2023","interfaces":[{"interface":"back","average":{"1min":0.64800000000000002,"5min":0.65900000000000003,"15min":0.58999999999999997},"min":{"1min":0.36699999999999999,"5min":0.34200000000000003,"15min":0.34200000000000003},"max":{"1min":1.0860000000000001,"5min":1.0920000000000001,"15min":1.123},"last":1.0640000000000001},{"interface":"front","average":{"1min":0.64500000000000002,"5min":0.68100000000000005,"15min":0.63200000000000001},"min":{"1min":0.33000000000000002,"5min":0.30399999999999999,"15min":0.29699999999999999},"max":{"1min":1.242,"5min":1.242,"15min":1.242},"last":0.65900000000000003}]},{"osd":2,"last update":"Sat Jun 3 19:35:42 2023","interfaces":[{"interface":"back","average":{"1min":0.66000000000000003,"5min":0.71099999999999997,"15min":0.64400000000000002},"min":{"1min":0.32800000000000001,"5min":0.32800000000000001,"15min":0.27600000000000002},"max":{"1min":1.159,"5min":1.2030000000000001,"15min":1.2030000000000001},"last":1.1319999999999999},{"interface":"front","average":{"1min":0.65600000000000003,"5min":0.69399999999999995,"15min":0.67100000000000004},"min":{"1min":0.378,"5min":0.378,"15min":0.32100000000000001},"max":{"1min":1.0389999999999999,"5min":1.0389999999999999,"15min":1.2150000000000001},"last":0.85899999999999999}]},{"osd":3,"last update":"Sat Jun 3 19:35:25 2023","interfaces":[{"interface":"back","average":{"1min":0.65400000000000003,"5min":0.69399999999999995,"15min":0.71899999999999997},"min":{"1min":0.378,"5min":0.378,"15min":0.378},"max":{"1min":0.878,"5min":1.1100000000000001,"15min":1.1100000000000001},"last":0.77900000000000003},{"interface":"front","average":{"1min":0.76900000000000002,"5min":0.79400000000000004,"15min":0.77900000000000003},"min":{"1min":0.41599999999999998,"5min":0.39300000000000002,"15min":0.39300000000000002},"max":{"1min":1.3049999999999999,"5min":1.3049999999999999,"15min":1.3049999999999999},"last":0.89300000000000002}]},{"osd":4,"last update":"Sat Jun 3 19:35:09 2023","interfaces":[{"interface":"back","average":{"1min":1.014,"5min":0.94099999999999995,"15min":0.94799999999999995},"min":{"1min":0.75,"5min":0.60999999999999999,"15min":0.60999999999999999},"max":{"1min":1.4950000000000001,"5min":1.4950000000000001,"15min":1.4950000000000001},"last":0.93899999999999995},{"interface":"front","average":{"1min":0.998,"5min":0.94899999999999995,"15min":0.93700000000000006},"min":{"1min":0.72399999999999998,"5min":0.63200000000000001,"15min":0.63200000000000001},"max":{"1min":1.2769999999999999,"5min":1.444,"15min":1.444},"last":1.0309999999999999}]},{"osd":5,"last update":"Sat Jun 3 19:35:51 2023","interfaces":[{"interface":"back","average":{"1min":1.0529999999999999,"5min":1.0229999999999999,"15min":1.0409999999999999},"min":{"1min":0.70399999999999996,"5min":0.65200000000000002,"15min":0.65200000000000002},"max":{"1min":1.5609999999999999,"5min":1.617,"15min":1.617},"last":1.5169999999999999},{"interface":"front","average":{"1min":1.016,"5min":1.0269999999999999,"15min":1.0669999999999999},"min":{"1min":0.71799999999999997,"5min":0.67900000000000005,"15min":0.67900000000000005},"max":{"1min":1.2849999999999999,"5min":1.595,"15min":1.595},"last":1.2849999999999999}]},{"osd":6,"last update":"Sat Jun 3 19:35:38 2023","interfaces":[{"interface":"back","average":{"1min":0.99199999999999999,"5min":0.998,"15min":1.0189999999999999},"min":{"1min":0.63500000000000001,"5min":0.54500000000000004,"15min":0.54500000000000004},"max":{"1min":1.333,"5min":1.4770000000000001,"15min":1.4770000000000001},"last":0.95799999999999996},{"interface":"front","average":{"1min":1.0469999999999999,"5min":1.0649999999999999,"15min":1.103},"min":{"1min":0.63300000000000001,"5min":0.57199999999999995,"15min":0.57199999999999995},"max":{"1min":1.4810000000000001,"5min":1.4810000000000001,"15min":1.4810000000000001},"last":1.5940000000000001}]},{"osd":7,"last update":"Sat Jun 3 19:35:20 2023","interfaces":[{"interface":"back","average":{"1min":1.0469999999999999,"5min":1.0569999999999999,"15min":1.083},"min":{"1min":0.60999999999999999,"5min":0.60999999999999999,"15min":0.60999999999999999},"max":{"1min":1.391,"5min":1.4810000000000001,"15min":1.4810000000000001},"last":1.198},{"interface":"front","average":{"1min":1.0629999999999999,"5min":1.1040000000000001,"15min":1.151},"min":{"1min":0.69499999999999995,"5min":0.69499999999999995,"15min":0.69499999999999995},"max":{"1min":1.4710000000000001,"5min":1.54,"15min":1.54},"last":1.089}]}]},{"osd":0,"up_from":8,"seq":34359738491,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6000,"kb_used_data":616,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":93739152,"statfs":{"total":95995035648,"available":95988891648,"internally_reserved":0,"allocated":630784,"data_stored":270197,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Jun 3 19:34:58 2023","interfaces":[{"interface":"back","average":{"1min":0.72899999999999998,"5min":0.69799999999999995,"15min":0.59999999999999998},"min":{"1min":0.439,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":1.5660000000000001,"5min":1.5660000000000001,"15min":1.5660000000000001},"last":0.90700000000000003},{"interface":"front","average":{"1min":0.71399999999999997,"5min":0.70799999999999996,"15min":0.60999999999999999},"min":{"1min":0.44500000000000001,"5min":0.40000000000000002,"15min":0.33600000000000002},"max":{"1min":1.228,"5min":1.228,"15min":1.228},"last":0.625}]},{"osd":2,"last update":"Sat Jun 3 19:35:43 2023","interfaces":[{"interface":"back","average":{"1min":0.621,"5min":0.68400000000000005,"15min":0.65700000000000003},"min":{"1min":0.28199999999999997,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":1.0900000000000001,"5min":1.1719999999999999,"15min":1.1719999999999999},"last":0.81799999999999995},{"interface":"front","average":{"1min":0.63500000000000001,"5min":0.72399999999999998,"15min":0.68000000000000005},"min":{"1min":0.39000000000000001,"5min":0.39000000000000001,"15min":0.39000000000000001},"max":{"1min":1.02,"5min":1.371,"15min":1.371},"last":0.79000000000000004}]},{"osd":3,"last update":"Sat Jun 3 19:35:32 2023","interfaces":[{"interface":"back","average":{"1min":0.621,"5min":0.66600000000000004,"15min":0.72099999999999997},"min":{"1min":0.38500000000000001,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":0.98299999999999998,"5min":1.0169999999999999,"15min":1.0860000000000001},"last":0.85999999999999999},{"interface":"front","average":{"1min":0.69899999999999995,"5min":0.79500000000000004,"15min":0.79600000000000004},"min":{"1min":0.40799999999999997,"5min":0.39500000000000002,"15min":0.39500000000000002},"max":{"1min":0.94999999999999996,"5min":1.7629999999999999,"15min":1.7629999999999999},"last":1.1120000000000001}]},{"osd":4,"last update":"Sat Jun 3 19:35:03 2023","interfaces":[{"interface":"back","average":{"1min":0.97499999999999998,"5min":0.93500000000000005,"15min":0.92800000000000005},"min":{"1min":0.69099999999999995,"5min":0.51900000000000002,"15min":0.51900000000000002},"max":{"1min":1.323,"5min":1.425,"15min":1.77},"last":0.996},{"interface":"front","average":{"1min":1.0980000000000001,"5min":0.995,"15min":0.94699999999999995},"min":{"1min":0.76900000000000002,"5min":0.56599999999999995,"15min":0.56599999999999995},"max":{"1min":1.637,"5min":1.637,"15min":1.637},"last":1.2330000000000001}]},{"osd":5,"last update":"Sat Jun 3 19:34:52 2023","interfaces":[{"interface":"back","average":{"1min":1.147,"5min":1.046,"15min":1.0609999999999999},"min":{"1min":0.75900000000000001,"5min":0.52700000000000002,"15min":0.52700000000000002},"max":{"1min":1.6799999999999999,"5min":1.6799999999999999,"15min":1.6799999999999999},"last":1.038},{"interface":"front","average":{"1min":0.97099999999999997,"5min":0.96299999999999997,"15min":0.97299999999999998},"min":{"1min":0.72299999999999998,"5min":0.42899999999999999,"15min":0.42899999999999999},"max":{"1min":1.3140000000000001,"5min":1.5169999999999999,"15min":1.5169999999999999},"last":1.05}]},{"osd":6,"last update":"Sat Jun 3 19:35:43 2023","interfaces":[{"interface":"back","average":{"1min":0.98799999999999999,"5min":1.056,"15min":1.083},"min":{"1min":0.56200000000000006,"5min":0.56200000000000006,"15min":0.56200000000000006},"max":{"1min":1.524,"5min":1.722,"15min":1.722},"last":1.1870000000000001},{"interface":"front","average":{"1min":1.0229999999999999,"5min":1.0489999999999999,"15min":1.0569999999999999},"min":{"1min":0.623,"5min":0.54400000000000004,"15min":0.54400000000000004},"max":{"1min":1.4379999999999999,"5min":1.4830000000000001,"15min":1.4830000000000001},"last":1.083}]},{"osd":7,"last update":"Sat Jun 3 19:35:24 2023","interfaces":[{"interface":"back","average":{"1min":0.96599999999999997,"5min":1.0669999999999999,"15min":1.079},"min":{"1min":0.64300000000000002,"5min":0.59499999999999997,"15min":0.59499999999999997},"max":{"1min":1.3360000000000001,"5min":1.514,"15min":1.514},"last":1.165},{"interface":"front","average":{"1min":0.94199999999999995,"5min":1.016,"15min":1.0349999999999999},"min":{"1min":0.65600000000000003,"5min":0.57899999999999996,"15min":0.57899999999999996},"max":{"1min":1.3859999999999999,"5min":1.786,"15min":1.786},"last":1.145}]}]},{"osd":2,"up_from":18,"seq":77309411432,"num_pgs":19,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":5812,"kb_used_data":620,"kb_used_omap":0,"kb_used_meta":5184,"kb_avail":93739340,"statfs":{"total":95995035648,"available":95989084160,"internally_reserved":0,"allocated":634880,"data_stored":270524,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5308416},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 3 19:35:42 2023","interfaces":[{"interface":"back","average":{"1min":0.64200000000000002,"5min":0.67500000000000004,"15min":0.622},"min":{"1min":0.36099999999999999,"5min":0.314,"15min":0.314},"max":{"1min":1.1160000000000001,"5min":1.1819999999999999,"15min":1.1819999999999999},"last":0.59499999999999997},{"interface":"front","average":{"1min":0.621,"5min":0.68200000000000005,"15min":0.63200000000000001},"min":{"1min":0.34399999999999997,"5min":0.34399999999999997,"15min":0.30199999999999999},"max":{"1min":0.872,"5min":3.004,"15min":3.004},"last":0.66000000000000003}]},{"osd":1,"last update":"Sat Jun 3 19:35:42 2023","interfaces":[{"interface":"back","average":{"1min":0.64300000000000002,"5min":0.70499999999999996,"15min":0.64700000000000002},"min":{"1min":0.375,"5min":0.375,"15min":0.375},"max":{"1min":1.0109999999999999,"5min":1.1339999999999999,"15min":1.139},"last":0.61899999999999999},{"interface":"front","average":{"1min":0.69099999999999995,"5min":0.75800000000000001,"15min":0.68300000000000005},"min":{"1min":0.38900000000000001,"5min":0.31,"15min":0.31},"max":{"1min":1.036,"5min":1.1619999999999999,"15min":1.1619999999999999},"last":0.747}]},{"osd":3,"last update":"Sat Jun 3 19:35:32 2023","interfaces":[{"interface":"back","average":{"1min":0.72599999999999998,"5min":0.749,"15min":0.746},"min":{"1min":0.49299999999999999,"5min":0.33800000000000002,"15min":0.33800000000000002},"max":{"1min":0.91900000000000004,"5min":1.323,"15min":1.323},"last":0.83399999999999996},{"interface":"front","average":{"1min":0.78400000000000003,"5min":0.81000000000000005,"15min":0.77800000000000002},"min":{"1min":0.45700000000000002,"5min":0.36299999999999999,"15min":0.34399999999999997},"max":{"1min":1.149,"5min":1.149,"15min":1.5840000000000001},"last":0.81499999999999995}]},{"osd":4,"last update":"Sat Jun 3 19:35:09 2023","interfaces":[{"interface":"back","average":{"1min":0.96999999999999997,"5min":0.96299999999999997,"15min":0.93799999999999994},"min":{"1min":0.72399999999999998,"5min":0.61199999999999999,"15min":0.61199999999999999},"max":{"1min":1.2430000000000001,"5min":1.554,"15min":1.554},"last":0.95299999999999996},{"interface":"front","average":{"1min":0.98399999999999999,"5min":1.093,"15min":1.014},"min":{"1min":0.52900000000000003,"5min":0.52900000000000003,"15min":0.52900000000000003},"max":{"1min":1.288,"5min":8.3829999999999991,"15min":8.3829999999999991},"last":1.1060000000000001}]},{"osd":5,"last update":"Sat Jun 3 19:35:53 2023","interfaces":[{"interface":"back","average":{"1min":0.96999999999999997,"5min":1.042,"15min":1.077},"min":{"1min":0.68200000000000005,"5min":0.59899999999999998,"15min":0.59899999999999998},"max":{"1min":1.2549999999999999,"5min":1.6479999999999999,"15min":1.7150000000000001},"last":1.04},{"interface":"front","average":{"1min":0.96299999999999997,"5min":0.96999999999999997,"15min":0.97199999999999998},"min":{"1min":0.67300000000000004,"5min":0.59699999999999998,"15min":0.59699999999999998},"max":{"1min":1.4339999999999999,"5min":1.4339999999999999,"15min":1.4339999999999999},"last":0.92400000000000004}]},{"osd":6,"last update":"Sat Jun 3 19:35:36 2023","interfaces":[{"interface":"back","average":{"1min":1.0720000000000001,"5min":1.044,"15min":1.0529999999999999},"min":{"1min":0.65500000000000003,"5min":0.65500000000000003,"15min":0.65500000000000003},"max":{"1min":1.446,"5min":1.502,"15min":1.502},"last":1.137},{"interface":"front","average":{"1min":1.0620000000000001,"5min":1.099,"15min":1.137},"min":{"1min":0.64700000000000002,"5min":0.58799999999999997,"15min":0.58799999999999997},"max":{"1min":1.4179999999999999,"5min":1.5940000000000001,"15min":1.5940000000000001},"last":1.1839999999999999}]},{"osd":7,"last update":"Sat Jun 3 19:35:22 2023","interfaces":[{"interface":"back","average":{"1min":1.0189999999999999,"5min":1.05,"15min":1.0900000000000001},"min":{"1min":0.65800000000000003,"5min":0.65600000000000003,"15min":0.65600000000000003},"max":{"1min":1.347,"5min":1.486,"15min":1.486},"last":0.96699999999999997},{"interface":"front","average":{"1min":1.103,"5min":1.1080000000000001,"15min":1.137},"min":{"1min":0.81499999999999995,"5min":0.67400000000000004,"15min":0.67400000000000004},"max":{"1min":1.431,"5min":1.5600000000000001,"15min":1.5600000000000001},"last":1.0760000000000001}]}]},{"osd":3,"up_from":25,"seq":107374182495,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":5808,"kb_used_data":616,"kb_used_omap":0,"kb_used_meta":5184,"kb_avail":93739344,"statfs":{"total":95995035648,"available":95989088256,"internally_reserved":0,"allocated":630784,"data_stored":270026,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5308416},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 3 19:35:24 2023","interfaces":[{"interface":"back","average":{"1min":0.64500000000000002,"5min":0.66900000000000004,"15min":0.67100000000000004},"min":{"1min":0.47699999999999998,"5min":0.314,"15min":0.314},"max":{"1min":0.95599999999999996,"5min":1.153,"15min":1.153},"last":0.74399999999999999},{"interface":"front","average":{"1min":0.68200000000000005,"5min":0.73199999999999998,"15min":0.73199999999999998},"min":{"1min":0.376,"5min":0.34499999999999997,"15min":0.34499999999999997},"max":{"1min":1.1419999999999999,"5min":1.8979999999999999,"15min":1.8979999999999999},"last":1.087}]},{"osd":1,"last update":"Sat Jun 3 19:35:24 2023","interfaces":[{"interface":"back","average":{"1min":0.72199999999999998,"5min":0.747,"15min":0.72399999999999998},"min":{"1min":0.48199999999999998,"5min":0.41799999999999998,"15min":0.41799999999999998},"max":{"1min":1.1919999999999999,"5min":1.218,"15min":1.218},"last":0.83699999999999997},{"interface":"front","average":{"1min":0.67600000000000005,"5min":0.72499999999999998,"15min":0.72299999999999998},"min":{"1min":0.38900000000000001,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":0.91000000000000003,"5min":1.109,"15min":1.109},"last":0.70499999999999996}]},{"osd":2,"last update":"Sat Jun 3 19:35:24 2023","interfaces":[{"interface":"back","average":{"1min":0.76300000000000001,"5min":0.77700000000000002,"15min":0.77100000000000002},"min":{"1min":0.47299999999999998,"5min":0.38300000000000001,"15min":0.38300000000000001},"max":{"1min":1.2749999999999999,"5min":1.359,"15min":1.359},"last":0.91300000000000003},{"interface":"front","average":{"1min":0.81499999999999995,"5min":0.76100000000000001,"15min":0.755},"min":{"1min":0.46999999999999997,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":1.5800000000000001,"5min":1.5800000000000001,"15min":1.5800000000000001},"last":0.92800000000000005}]},{"osd":4,"last update":"Sat Jun 3 19:35:09 2023","interfaces":[{"interface":"back","average":{"1min":0.90300000000000002,"5min":0.93999999999999995,"15min":0.94799999999999995},"min":{"1min":0.68400000000000005,"5min":0.59499999999999997,"15min":0.59499999999999997},"max":{"1min":1.2569999999999999,"5min":1.393,"15min":1.393},"last":1.1799999999999999},{"interface":"front","average":{"1min":0.96699999999999997,"5min":0.98999999999999999,"15min":0.98599999999999999},"min":{"1min":0.627,"5min":0.627,"15min":0.627},"max":{"1min":1.399,"5min":1.528,"15min":1.528},"last":0.97499999999999998}]},{"osd":5,"last update":"Sat Jun 3 19:34:53 2023","interfaces":[{"interface":"back","average":{"1min":1.0489999999999999,"5min":1.0189999999999999,"15min":1.0529999999999999},"min":{"1min":0.77800000000000002,"5min":0.69299999999999995,"15min":0.69299999999999995},"max":{"1min":1.3779999999999999,"5min":1.48,"15min":1.48},"last":1.0289999999999999},{"interface":"front","average":{"1min":1.05,"5min":1.0369999999999999,"15min":1.071},"min":{"1min":0.82699999999999996,"5min":0.68400000000000005,"15min":0.68400000000000005},"max":{"1min":1.292,"5min":1.452,"15min":1.452},"last":1.0609999999999999}]},{"osd":6,"last update":"Sat Jun 3 19:35:38 2023","interfaces":[{"interface":"back","average":{"1min":1.0449999999999999,"5min":1.0509999999999999,"15min":1.099},"min":{"1min":0.60899999999999999,"5min":0.60899999999999999,"15min":0.60899999999999999},"max":{"1min":1.9630000000000001,"5min":1.9630000000000001,"15min":1.9630000000000001},"last":1.23},{"interface":"front","average":{"1min":1.0600000000000001,"5min":1.046,"15min":1.0960000000000001},"min":{"1min":0.70999999999999996,"5min":0.65400000000000003,"15min":0.65400000000000003},"max":{"1min":1.5660000000000001,"5min":1.5660000000000001,"15min":1.5660000000000001},"last":1.194}]},{"osd":7,"last update":"Sat Jun 3 19:35:24 2023","interfaces":[{"interface":"back","average":{"1min":1.0720000000000001,"5min":1.0660000000000001,"15min":1.077},"min":{"1min":0.76400000000000001,"5min":0.66800000000000004,"15min":0.66800000000000004},"max":{"1min":1.6679999999999999,"5min":1.6679999999999999,"15min":1.6679999999999999},"last":1.2769999999999999},{"interface":"front","average":{"1min":1.0820000000000001,"5min":1.0860000000000001,"15min":1.0880000000000001},"min":{"1min":0.66700000000000004,"5min":0.64900000000000002,"15min":0.64900000000000002},"max":{"1min":1.5529999999999999,"5min":1.5529999999999999,"15min":1.5529999999999999},"last":1.3089999999999999}]}]},{"osd":4,"up_from":30,"seq":128849018967,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":5828,"kb_used_data":636,"kb_used_omap":0,"kb_used_meta":5184,"kb_avail":93739324,"statfs":{"total":95995035648,"available":95989067776,"internally_reserved":0,"allocated":651264,"data_stored":271618,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5308416},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 3 19:35:09 2023","interfaces":[{"interface":"back","average":{"1min":0.878,"5min":0.86299999999999999,"15min":0.86599999999999999},"min":{"1min":0.51600000000000001,"5min":0.51600000000000001,"15min":0.51600000000000001},"max":{"1min":1.1970000000000001,"5min":1.264,"15min":1.264},"last":0.58099999999999996},{"interface":"front","average":{"1min":0.93000000000000005,"5min":0.871,"15min":0.877},"min":{"1min":0.61199999999999999,"5min":0.46100000000000002,"15min":0.46100000000000002},"max":{"1min":1.331,"5min":1.331,"15min":1.331},"last":0.876}]},{"osd":1,"last update":"Sat Jun 3 19:35:09 2023","interfaces":[{"interface":"back","average":{"1min":0.92300000000000004,"5min":0.876,"15min":0.86899999999999999},"min":{"1min":0.65700000000000003,"5min":0.502,"15min":0.502},"max":{"1min":1.286,"5min":1.286,"15min":1.286},"last":0.68799999999999994},{"interface":"front","average":{"1min":0.87,"5min":0.875,"15min":0.85699999999999998},"min":{"1min":0.57899999999999996,"5min":0.47599999999999998,"15min":0.47599999999999998},"max":{"1min":1.2769999999999999,"5min":1.2769999999999999,"15min":1.2769999999999999},"last":0.98599999999999999}]},{"osd":2,"last update":"Sat Jun 3 19:35:09 2023","interfaces":[{"interface":"back","average":{"1min":0.98999999999999999,"5min":0.91200000000000003,"15min":0.89000000000000001},"min":{"1min":0.61899999999999999,"5min":0.61899999999999999,"15min":0.61899999999999999},"max":{"1min":1.1839999999999999,"5min":1.2190000000000001,"15min":1.2190000000000001},"last":0.748},{"interface":"front","average":{"1min":0.95599999999999996,"5min":0.89900000000000002,"15min":0.89300000000000002},"min":{"1min":0.65400000000000003,"5min":0.60299999999999998,"15min":0.60299999999999998},"max":{"1min":1.2310000000000001,"5min":1.284,"15min":1.284},"last":0.82799999999999996}]},{"osd":3,"last update":"Sat Jun 3 19:35:09 2023","interfaces":[{"interface":"back","average":{"1min":0.95799999999999996,"5min":0.89000000000000001,"15min":0.879},"min":{"1min":0.749,"5min":0.56699999999999995,"15min":0.56699999999999995},"max":{"1min":1.2150000000000001,"5min":1.345,"15min":1.345},"last":0.76600000000000001},{"interface":"front","average":{"1min":1.0429999999999999,"5min":0.94299999999999995,"15min":0.94299999999999995},"min":{"1min":0.747,"5min":0.58599999999999997,"15min":0.58599999999999997},"max":{"1min":1.371,"5min":1.371,"15min":1.371},"last":1.0149999999999999}]},{"osd":5,"last update":"Sat Jun 3 19:35:49 2023","interfaces":[{"interface":"back","average":{"1min":0.95499999999999996,"5min":0.88900000000000001,"15min":0.79500000000000004},"min":{"1min":0.53300000000000003,"5min":0.498,"15min":0.45500000000000002},"max":{"1min":1.196,"5min":1.46,"15min":1.46},"last":0.85799999999999998},{"interface":"front","average":{"1min":0.97499999999999998,"5min":0.90200000000000002,"15min":0.84599999999999997},"min":{"1min":0.63400000000000001,"5min":0.45500000000000002,"15min":0.45500000000000002},"max":{"1min":1.1930000000000001,"5min":1.3580000000000001,"15min":1.3580000000000001},"last":0.89000000000000001}]},{"osd":6,"last update":"Sat Jun 3 19:35:38 2023","interfaces":[{"interface":"back","average":{"1min":1,"5min":0.93400000000000005,"15min":0.93999999999999995},"min":{"1min":0.66400000000000003,"5min":0.46899999999999997,"15min":0.46899999999999997},"max":{"1min":1.2330000000000001,"5min":1.413,"15min":1.413},"last":0.92700000000000005},{"interface":"front","average":{"1min":1.044,"5min":0.96999999999999997,"15min":0.95199999999999996},"min":{"1min":0.83199999999999996,"5min":0.503,"15min":0.503},"max":{"1min":1.3160000000000001,"5min":1.3160000000000001,"15min":1.3160000000000001},"last":0.95099999999999996}]},{"osd":7,"last update":"Sat Jun 3 19:35:22 2023","interfaces":[{"interface":"back","average":{"1min":1.196,"5min":1.012,"15min":0.92200000000000004},"min":{"1min":0.97099999999999997,"5min":0.47099999999999997,"15min":0.47099999999999997},"max":{"1min":1.401,"5min":1.401,"15min":1.401},"last":1.0600000000000001},{"interface":"front","average":{"1min":1.1890000000000001,"5min":1.0409999999999999,"15min":0.95099999999999996},"min":{"1min":0.94899999999999995,"5min":0.52300000000000002,"15min":0.52300000000000002},"max":{"1min":1.357,"5min":1.712,"15min":1.712},"last":0.97399999999999998}]}]},{"osd":5,"up_from":36,"seq":154618822734,"num_pgs":21,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":5680,"kb_used_data":616,"kb_used_omap":0,"kb_used_meta":5056,"kb_avail":93739472,"statfs":{"total":95995035648,"available":95989219328,"internally_reserved":0,"allocated":630784,"data_stored":270245,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5177344},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 3 19:34:53 2023","interfaces":[{"interface":"back","average":{"1min":0.91400000000000003,"5min":0.878,"15min":0.88400000000000001},"min":{"1min":0.51800000000000002,"5min":0.51400000000000001,"15min":0.51400000000000001},"max":{"1min":1.2470000000000001,"5min":1.4750000000000001,"15min":1.4750000000000001},"last":0.73399999999999999},{"interface":"front","average":{"1min":0.91300000000000003,"5min":0.86699999999999999,"15min":0.84699999999999998},"min":{"1min":0.626,"5min":0.55200000000000005,"15min":0.55200000000000005},"max":{"1min":1.1890000000000001,"5min":1.9259999999999999,"15min":1.9259999999999999},"last":0.93899999999999995}]},{"osd":1,"last update":"Sat Jun 3 19:34:53 2023","interfaces":[{"interface":"back","average":{"1min":0.95299999999999996,"5min":0.90000000000000002,"15min":0.89300000000000002},"min":{"1min":0.63500000000000001,"5min":0.49399999999999999,"15min":0.49399999999999999},"max":{"1min":1.335,"5min":1.994,"15min":1.994},"last":0.98599999999999999},{"interface":"front","average":{"1min":0.94199999999999995,"5min":0.90900000000000003,"15min":0.872},"min":{"1min":0.66000000000000003,"5min":0.55400000000000005,"15min":0.55400000000000005},"max":{"1min":1.244,"5min":1.742,"15min":1.742},"last":1.256}]},{"osd":2,"last update":"Sat Jun 3 19:34:53 2023","interfaces":[{"interface":"back","average":{"1min":0.91400000000000003,"5min":0.88100000000000001,"15min":0.873},"min":{"1min":0.747,"5min":0.54200000000000004,"15min":0.54200000000000004},"max":{"1min":1.099,"5min":1.708,"15min":1.708},"last":1.145},{"interface":"front","average":{"1min":0.94699999999999995,"5min":0.92600000000000005,"15min":0.89900000000000002},"min":{"1min":0.70999999999999996,"5min":0.56000000000000005,"15min":0.56000000000000005},"max":{"1min":1.256,"5min":2.0840000000000001,"15min":2.0840000000000001},"last":1.121}]},{"osd":3,"last update":"Sat Jun 3 19:34:53 2023","interfaces":[{"interface":"back","average":{"1min":0.93799999999999994,"5min":0.93899999999999995,"15min":0.95399999999999996},"min":{"1min":0.68300000000000005,"5min":0.58199999999999996,"15min":0.58199999999999996},"max":{"1min":1.1759999999999999,"5min":2.1699999999999999,"15min":2.1699999999999999},"last":1.0620000000000001},{"interface":"front","average":{"1min":0.99199999999999999,"5min":0.92100000000000004,"15min":0.877},"min":{"1min":0.83799999999999997,"5min":0.57899999999999996,"15min":0.57899999999999996},"max":{"1min":1.173,"5min":1.6459999999999999,"15min":1.6459999999999999},"last":1.0129999999999999}]},{"osd":4,"last update":"Sat Jun 3 19:34:53 2023","interfaces":[{"interface":"back","average":{"1min":0.92500000000000004,"5min":0.85699999999999998,"15min":0.78800000000000003},"min":{"1min":0.68200000000000005,"5min":0.39100000000000001,"15min":0.39100000000000001},"max":{"1min":1.1930000000000001,"5min":2.1299999999999999,"15min":2.1299999999999999},"last":0.85799999999999998},{"interface":"front","average":{"1min":1.032,"5min":0.89800000000000002,"15min":0.78400000000000003},"min":{"1min":0.67500000000000004,"5min":0.38,"15min":0.38},"max":{"1min":1.246,"5min":2.0830000000000002,"15min":2.0830000000000002},"last":1.179}]},{"osd":6,"last update":"Sat Jun 3 19:35:38 2023","interfaces":[{"interface":"back","average":{"1min":1.0900000000000001,"5min":0.98299999999999998,"15min":0.92700000000000005},"min":{"1min":0.68899999999999995,"5min":0.46500000000000002,"15min":0.46500000000000002},"max":{"1min":1.337,"5min":1.6599999999999999,"15min":1.6599999999999999},"last":1.357},{"interface":"front","average":{"1min":1.1830000000000001,"5min":1.0720000000000001,"15min":1},"min":{"1min":0.96499999999999997,"5min":0.56200000000000006,"15min":0.56200000000000006},"max":{"1min":1.4630000000000001,"5min":1.8380000000000001,"15min":1.8380000000000001},"last":1.3169999999999999}]},{"osd":7,"last update":"Sat Jun 3 19:35:22 2023","interfaces":[{"interface":"back","average":{"1min":1.194,"5min":1.0960000000000001,"15min":1.0700000000000001},"min":{"1min":0.84099999999999997,"5min":0.63200000000000001,"15min":0.63200000000000001},"max":{"1min":1.417,"5min":2.004,"15min":2.004},"last":1.4039999999999999},{"interface":"front","average":{"1min":1.0760000000000001,"5min":0.995,"15min":0.96499999999999997},"min":{"1min":0.76800000000000002,"5min":0.48099999999999998,"15min":0.48099999999999998},"max":{"1min":1.29,"5min":1.6479999999999999,"15min":1.6479999999999999},"last":1.2390000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":1682,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":1606,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":1752,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-06-03T19:35:56.766 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph pg dump --format=json 2023-06-03T19:35:57.908 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:35:57 smithi089 bash[33173]: audit 2023-06-03T19:35:55.907246+0000 mgr.y (mgr.14445) 157 : audit [DBG] from='client.24689 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-06-03T19:35:57.908 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:35:57 smithi089 bash[33173]: cluster 2023-06-03T19:35:56.448939+0000 mgr.y (mgr.14445) 158 : cluster [DBG] pgmap v138: 73 pgs: 73 active+clean; 2.3 KiB data, 45 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:35:57.908 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:35:57 smithi089 bash[26572]: audit 2023-06-03T19:35:55.907246+0000 mgr.y (mgr.14445) 157 : audit [DBG] from='client.24689 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-06-03T19:35:57.909 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:35:57 smithi089 bash[26572]: cluster 2023-06-03T19:35:56.448939+0000 mgr.y (mgr.14445) 158 : cluster [DBG] pgmap v138: 73 pgs: 73 active+clean; 2.3 KiB data, 45 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:35:58.087 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:35:57 smithi164 bash[26855]: audit 2023-06-03T19:35:55.907246+0000 mgr.y (mgr.14445) 157 : audit [DBG] from='client.24689 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-06-03T19:35:58.087 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:35:57 smithi164 bash[26855]: cluster 2023-06-03T19:35:56.448939+0000 mgr.y (mgr.14445) 158 : cluster [DBG] pgmap v138: 73 pgs: 73 active+clean; 2.3 KiB data, 45 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:35:58.282 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 19:35:57 smithi089 bash[26845]: ::ffff:172.21.15.164 - - [03/Jun/2023:19:35:57] "GET /metrics HTTP/1.1" 200 187882 "" "Prometheus/2.33.4" 2023-06-03T19:36:00.032 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:35:59 smithi089 bash[26572]: cluster 2023-06-03T19:35:58.449507+0000 mgr.y (mgr.14445) 159 : cluster [DBG] pgmap v139: 73 pgs: 73 active+clean; 2.3 KiB data, 45 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:36:00.032 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:35:59 smithi089 bash[33173]: cluster 2023-06-03T19:35:58.449507+0000 mgr.y (mgr.14445) 159 : cluster [DBG] pgmap v139: 73 pgs: 73 active+clean; 2.3 KiB data, 45 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:36:00.087 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:35:59 smithi164 bash[26855]: cluster 2023-06-03T19:35:58.449507+0000 mgr.y (mgr.14445) 159 : cluster [DBG] pgmap v139: 73 pgs: 73 active+clean; 2.3 KiB data, 45 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:36:01.614 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/mon.a/config 2023-06-03T19:36:02.032 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:36:01 smithi089 bash[26572]: cluster 2023-06-03T19:36:00.450490+0000 mgr.y (mgr.14445) 160 : cluster [DBG] pgmap v140: 73 pgs: 73 active+clean; 2.3 KiB data, 45 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:36:02.032 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:36:01 smithi089 bash[33173]: cluster 2023-06-03T19:36:00.450490+0000 mgr.y (mgr.14445) 160 : cluster [DBG] pgmap v140: 73 pgs: 73 active+clean; 2.3 KiB data, 45 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:36:02.087 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:36:01 smithi164 bash[26855]: cluster 2023-06-03T19:36:00.450490+0000 mgr.y (mgr.14445) 160 : cluster [DBG] pgmap v140: 73 pgs: 73 active+clean; 2.3 KiB data, 45 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:36:04.032 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:36:03 smithi089 bash[33173]: cluster 2023-06-03T19:36:02.450950+0000 mgr.y (mgr.14445) 161 : cluster [DBG] pgmap v141: 73 pgs: 73 active+clean; 2.3 KiB data, 45 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:36:04.032 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:36:03 smithi089 bash[26572]: cluster 2023-06-03T19:36:02.450950+0000 mgr.y (mgr.14445) 161 : cluster [DBG] pgmap v141: 73 pgs: 73 active+clean; 2.3 KiB data, 45 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:36:04.087 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:36:03 smithi164 bash[26855]: cluster 2023-06-03T19:36:02.450950+0000 mgr.y (mgr.14445) 161 : cluster [DBG] pgmap v141: 73 pgs: 73 active+clean; 2.3 KiB data, 45 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:36:05.016 INFO:teuthology.orchestra.run.smithi089.stdout: 2023-06-03T19:36:05.017 INFO:teuthology.orchestra.run.smithi089.stderr:dumped all 2023-06-03T19:36:05.833 INFO:teuthology.orchestra.run.smithi089.stdout:{"pg_ready":true,"pg_map":{"version":142,"stamp":"2023-06-03T19:36:04.451212+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":2306,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":45,"num_write_kb":13,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":39,"ondisk_log_size":39,"up":219,"acting":219,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":219,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":46448,"kb_used_data":4976,"kb_used_omap":0,"kb_used_meta":41408,"kb_avail":749914768,"statfs":{"total":767960285184,"available":767912722432,"internally_reserved":0,"allocated":5095424,"data_stored":2165146,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":42401792},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.003956"},"pg_stats":[{"pgid":"4.18","version":"0'0","reported_seq":13,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-06-03T19:34:48.688273+0000","last_change":"2023-06-03T19:34:17.833986+0000","last_active":"2023-06-03T19:34:48.688273+0000","last_peered":"2023-06-03T19:34:48.688273+0000","last_clean":"2023-06-03T19:34:48.688273+0000","last_became_active":"2023-06-03T19:34:17.833598+0000","last_became_peered":"2023-06-03T19:34:17.833598+0000","last_unstale":"2023-06-03T19:34:48.688273+0000","last_undegraded":"2023-06-03T19:34:48.688273+0000","last_fullsized":"2023-06-03T19:34:48.688273+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"60'4","reported_seq":18,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-06-03T19:34:48.694317+0000","last_change":"2023-06-03T19:34:16.809349+0000","last_active":"2023-06-03T19:34:48.694317+0000","last_peered":"2023-06-03T19:34:48.694317+0000","last_clean":"2023-06-03T19:34:48.694317+0000","last_became_active":"2023-06-03T19:34:16.809012+0000","last_became_peered":"2023-06-03T19:34:16.809012+0000","last_unstale":"2023-06-03T19:34:48.694317+0000","last_undegraded":"2023-06-03T19:34:48.694317+0000","last_fullsized":"2023-06-03T19:34:48.694317+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","log_size":4,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":13,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-06-03T19:34:48.687439+0000","last_change":"2023-06-03T19:34:17.181723+0000","last_active":"2023-06-03T19:34:48.687439+0000","last_peered":"2023-06-03T19:34:48.687439+0000","last_clean":"2023-06-03T19:34:48.687439+0000","last_became_active":"2023-06-03T19:34:17.181302+0000","last_became_peered":"2023-06-03T19:34:17.181302+0000","last_unstale":"2023-06-03T19:34:48.687439+0000","last_undegraded":"2023-06-03T19:34:48.687439+0000","last_fullsized":"2023-06-03T19:34:48.687439+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"60'1","reported_seq":15,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-06-03T19:34:48.688731+0000","last_change":"2023-06-03T19:34:16.812230+0000","last_active":"2023-06-03T19:34:48.688731+0000","last_peered":"2023-06-03T19:34:48.688731+0000","last_clean":"2023-06-03T19:34:48.688731+0000","last_became_active":"2023-06-03T19:34:16.811860+0000","last_became_peered":"2023-06-03T19:34:16.811860+0000","last_unstale":"2023-06-03T19:34:48.688731+0000","last_undegraded":"2023-06-03T19:34:48.688731+0000","last_fullsized":"2023-06-03T19:34:48.688731+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","log_size":1,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":13,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-06-03T19:34:48.688562+0000","last_change":"2023-06-03T19:34:17.827691+0000","last_active":"2023-06-03T19:34:48.688562+0000","last_peered":"2023-06-03T19:34:48.688562+0000","last_clean":"2023-06-03T19:34:48.688562+0000","last_became_active":"2023-06-03T19:34:17.827421+0000","last_became_peered":"2023-06-03T19:34:17.827421+0000","last_unstale":"2023-06-03T19:34:48.688562+0000","last_undegraded":"2023-06-03T19:34:48.688562+0000","last_fullsized":"2023-06-03T19:34:48.688562+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"60'2","reported_seq":16,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-06-03T19:34:48.685939+0000","last_change":"2023-06-03T19:34:16.805013+0000","last_active":"2023-06-03T19:34:48.685939+0000","last_peered":"2023-06-03T19:34:48.685939+0000","last_clean":"2023-06-03T19:34:48.685939+0000","last_became_active":"2023-06-03T19:34:16.804707+0000","last_became_peered":"2023-06-03T19:34:16.804707+0000","last_unstale":"2023-06-03T19:34:48.685939+0000","last_undegraded":"2023-06-03T19:34:48.685939+0000","last_fullsized":"2023-06-03T19:34:48.685939+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","log_size":2,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":13,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-06-03T19:34:48.689486+0000","last_change":"2023-06-03T19:34:17.833202+0000","last_active":"2023-06-03T19:34:48.689486+0000","last_peered":"2023-06-03T19:34:48.689486+0000","last_clean":"2023-06-03T19:34:48.689486+0000","last_became_active":"2023-06-03T19:34:17.832695+0000","last_became_peered":"2023-06-03T19:34:17.832695+0000","last_unstale":"2023-06-03T19:34:48.689486+0000","last_undegraded":"2023-06-03T19:34:48.689486+0000","last_fullsized":"2023-06-03T19:34:48.689486+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"60'2","reported_seq":16,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-06-03T19:34:48.685655+0000","last_change":"2023-06-03T19:34:16.802522+0000","last_active":"2023-06-03T19:34:48.685655+0000","last_peered":"2023-06-03T19:34:48.685655+0000","last_clean":"2023-06-03T19:34:48.685655+0000","last_became_active":"2023-06-03T19:34:16.801282+0000","last_became_peered":"2023-06-03T19:34:16.801282+0000","last_unstale":"2023-06-03T19:34:48.685655+0000","last_undegraded":"2023-06-03T19:34:48.685655+0000","last_fullsized":"2023-06-03T19:34:48.685655+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","log_size":2,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":13,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-06-03T19:34:48.690498+0000","last_change":"2023-06-03T19:34:17.834796+0000","last_active":"2023-06-03T19:34:48.690498+0000","last_peered":"2023-06-03T19:34:48.690498+0000","last_clean":"2023-06-03T19:34:48.690498+0000","last_became_active":"2023-06-03T19:34:17.834464+0000","last_became_peered":"2023-06-03T19:34:17.834464+0000","last_unstale":"2023-06-03T19:34:48.690498+0000","last_undegraded":"2023-06-03T19:34:48.690498+0000","last_fullsized":"2023-06-03T19:34:48.690498+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1b","version":"60'1","reported_seq":15,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-06-03T19:34:48.694648+0000","last_change":"2023-06-03T19:34:16.810791+0000","last_active":"2023-06-03T19:34:48.694648+0000","last_peered":"2023-06-03T19:34:48.694648+0000","last_clean":"2023-06-03T19:34:48.694648+0000","last_became_active":"2023-06-03T19:34:16.810386+0000","last_became_peered":"2023-06-03T19:34:16.810386+0000","last_unstale":"2023-06-03T19:34:48.694648+0000","last_undegraded":"2023-06-03T19:34:48.694648+0000","last_fullsized":"2023-06-03T19:34:48.694648+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","log_size":1,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":13,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-06-03T19:34:48.689412+0000","last_change":"2023-06-03T19:34:17.830960+0000","last_active":"2023-06-03T19:34:48.689412+0000","last_peered":"2023-06-03T19:34:48.689412+0000","last_clean":"2023-06-03T19:34:48.689412+0000","last_became_active":"2023-06-03T19:34:17.830605+0000","last_became_peered":"2023-06-03T19:34:17.830605+0000","last_unstale":"2023-06-03T19:34:48.689412+0000","last_undegraded":"2023-06-03T19:34:48.689412+0000","last_fullsized":"2023-06-03T19:34:48.689412+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.6","version":"60'1","reported_seq":15,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-06-03T19:34:48.694150+0000","last_change":"2023-06-03T19:34:16.807099+0000","last_active":"2023-06-03T19:34:48.694150+0000","last_peered":"2023-06-03T19:34:48.694150+0000","last_clean":"2023-06-03T19:34:48.694150+0000","last_became_active":"2023-06-03T19:34:16.806765+0000","last_became_peered":"2023-06-03T19:34:16.806765+0000","last_unstale":"2023-06-03T19:34:48.694150+0000","last_undegraded":"2023-06-03T19:34:48.694150+0000","last_fullsized":"2023-06-03T19:34:48.694150+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","log_size":1,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":18,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-06-03T19:34:48.711665+0000","last_change":"2023-06-03T19:32:21.706362+0000","last_active":"2023-06-03T19:34:48.711665+0000","last_peered":"2023-06-03T19:34:48.711665+0000","last_clean":"2023-06-03T19:34:48.711665+0000","last_became_active":"2023-06-03T19:32:21.705855+0000","last_became_peered":"2023-06-03T19:32:21.705855+0000","last_unstale":"2023-06-03T19:34:48.711665+0000","last_undegraded":"2023-06-03T19:34:48.711665+0000","last_fullsized":"2023-06-03T19:34:48.711665+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:32:20.692043+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:32:20.692043+0000","last_clean_scrub_stamp":"2023-06-03T19:32:20.692043+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":13,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-06-03T19:34:48.686448+0000","last_change":"2023-06-03T19:34:16.918966+0000","last_active":"2023-06-03T19:34:48.686448+0000","last_peered":"2023-06-03T19:34:48.686448+0000","last_clean":"2023-06-03T19:34:48.686448+0000","last_became_active":"2023-06-03T19:34:16.918245+0000","last_became_peered":"2023-06-03T19:34:16.918245+0000","last_unstale":"2023-06-03T19:34:48.686448+0000","last_undegraded":"2023-06-03T19:34:48.686448+0000","last_fullsized":"2023-06-03T19:34:48.686448+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":18,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-06-03T19:34:48.695432+0000","last_change":"2023-06-03T19:32:21.989748+0000","last_active":"2023-06-03T19:34:48.695432+0000","last_peered":"2023-06-03T19:34:48.695432+0000","last_clean":"2023-06-03T19:34:48.695432+0000","last_became_active":"2023-06-03T19:32:21.989310+0000","last_became_peered":"2023-06-03T19:32:21.989310+0000","last_unstale":"2023-06-03T19:34:48.695432+0000","last_undegraded":"2023-06-03T19:34:48.695432+0000","last_fullsized":"2023-06-03T19:34:48.695432+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:32:20.692043+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:32:20.692043+0000","last_clean_scrub_stamp":"2023-06-03T19:32:20.692043+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.7","version":"60'2","reported_seq":16,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-06-03T19:34:48.689012+0000","last_change":"2023-06-03T19:34:16.811190+0000","last_active":"2023-06-03T19:34:48.689012+0000","last_peered":"2023-06-03T19:34:48.689012+0000","last_clean":"2023-06-03T19:34:48.689012+0000","last_became_active":"2023-06-03T19:34:16.810846+0000","last_became_peered":"2023-06-03T19:34:16.810846+0000","last_unstale":"2023-06-03T19:34:48.689012+0000","last_undegraded":"2023-06-03T19:34:48.689012+0000","last_fullsized":"2023-06-03T19:34:48.689012+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","log_size":2,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":13,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-06-03T19:34:48.694721+0000","last_change":"2023-06-03T19:34:17.849689+0000","last_active":"2023-06-03T19:34:48.694721+0000","last_peered":"2023-06-03T19:34:48.694721+0000","last_clean":"2023-06-03T19:34:48.694721+0000","last_became_active":"2023-06-03T19:34:17.849349+0000","last_became_peered":"2023-06-03T19:34:17.849349+0000","last_unstale":"2023-06-03T19:34:48.694721+0000","last_undegraded":"2023-06-03T19:34:48.694721+0000","last_fullsized":"2023-06-03T19:34:48.694721+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":14,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-06-03T19:34:48.694618+0000","last_change":"2023-06-03T19:34:16.803712+0000","last_active":"2023-06-03T19:34:48.694618+0000","last_peered":"2023-06-03T19:34:48.694618+0000","last_clean":"2023-06-03T19:34:48.694618+0000","last_became_active":"2023-06-03T19:34:16.802957+0000","last_became_peered":"2023-06-03T19:34:16.802957+0000","last_unstale":"2023-06-03T19:34:48.694618+0000","last_undegraded":"2023-06-03T19:34:48.694618+0000","last_fullsized":"2023-06-03T19:34:48.694618+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":18,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-06-03T19:34:48.712646+0000","last_change":"2023-06-03T19:32:22.124191+0000","last_active":"2023-06-03T19:34:48.712646+0000","last_peered":"2023-06-03T19:34:48.712646+0000","last_clean":"2023-06-03T19:34:48.712646+0000","last_became_active":"2023-06-03T19:32:22.122933+0000","last_became_peered":"2023-06-03T19:32:22.122933+0000","last_unstale":"2023-06-03T19:34:48.712646+0000","last_undegraded":"2023-06-03T19:34:48.712646+0000","last_fullsized":"2023-06-03T19:34:48.712646+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:32:20.692043+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:32:20.692043+0000","last_clean_scrub_stamp":"2023-06-03T19:32:20.692043+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":13,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-06-03T19:34:48.694182+0000","last_change":"2023-06-03T19:34:17.829675+0000","last_active":"2023-06-03T19:34:48.694182+0000","last_peered":"2023-06-03T19:34:48.694182+0000","last_clean":"2023-06-03T19:34:48.694182+0000","last_became_active":"2023-06-03T19:34:17.829308+0000","last_became_peered":"2023-06-03T19:34:17.829308+0000","last_unstale":"2023-06-03T19:34:48.694182+0000","last_undegraded":"2023-06-03T19:34:48.694182+0000","last_fullsized":"2023-06-03T19:34:48.694182+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":14,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-06-03T19:34:48.694276+0000","last_change":"2023-06-03T19:34:16.814714+0000","last_active":"2023-06-03T19:34:48.694276+0000","last_peered":"2023-06-03T19:34:48.694276+0000","last_clean":"2023-06-03T19:34:48.694276+0000","last_became_active":"2023-06-03T19:34:16.814075+0000","last_became_peered":"2023-06-03T19:34:16.814075+0000","last_unstale":"2023-06-03T19:34:48.694276+0000","last_undegraded":"2023-06-03T19:34:48.694276+0000","last_fullsized":"2023-06-03T19:34:48.694276+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":18,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-06-03T19:34:48.711992+0000","last_change":"2023-06-03T19:32:22.123780+0000","last_active":"2023-06-03T19:34:48.711992+0000","last_peered":"2023-06-03T19:34:48.711992+0000","last_clean":"2023-06-03T19:34:48.711992+0000","last_became_active":"2023-06-03T19:32:22.122852+0000","last_became_peered":"2023-06-03T19:32:22.122852+0000","last_unstale":"2023-06-03T19:34:48.711992+0000","last_undegraded":"2023-06-03T19:34:48.711992+0000","last_fullsized":"2023-06-03T19:34:48.711992+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:32:20.692043+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:32:20.692043+0000","last_clean_scrub_stamp":"2023-06-03T19:32:20.692043+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":13,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-06-03T19:34:48.711682+0000","last_change":"2023-06-03T19:34:17.848964+0000","last_active":"2023-06-03T19:34:48.711682+0000","last_peered":"2023-06-03T19:34:48.711682+0000","last_clean":"2023-06-03T19:34:48.711682+0000","last_became_active":"2023-06-03T19:34:17.848140+0000","last_became_peered":"2023-06-03T19:34:17.848140+0000","last_unstale":"2023-06-03T19:34:48.711682+0000","last_undegraded":"2023-06-03T19:34:48.711682+0000","last_fullsized":"2023-06-03T19:34:48.711682+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":14,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-06-03T19:34:48.687784+0000","last_change":"2023-06-03T19:34:16.813150+0000","last_active":"2023-06-03T19:34:48.687784+0000","last_peered":"2023-06-03T19:34:48.687784+0000","last_clean":"2023-06-03T19:34:48.687784+0000","last_became_active":"2023-06-03T19:34:16.812816+0000","last_became_peered":"2023-06-03T19:34:16.812816+0000","last_unstale":"2023-06-03T19:34:48.687784+0000","last_undegraded":"2023-06-03T19:34:48.687784+0000","last_fullsized":"2023-06-03T19:34:48.687784+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":22,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-06-03T19:34:48.712037+0000","last_change":"2023-06-03T19:31:15.881174+0000","last_active":"2023-06-03T19:34:48.712037+0000","last_peered":"2023-06-03T19:34:48.712037+0000","last_clean":"2023-06-03T19:34:48.712037+0000","last_became_active":"2023-06-03T19:31:15.880415+0000","last_became_peered":"2023-06-03T19:31:15.880415+0000","last_unstale":"2023-06-03T19:34:48.712037+0000","last_undegraded":"2023-06-03T19:34:48.712037+0000","last_fullsized":"2023-06-03T19:34:48.712037+0000","mapping_epoch":49,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":50,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:27:28.100808+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:27:28.100808+0000","last_clean_scrub_stamp":"2023-06-03T19:27:28.100808+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":18,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-06-03T19:34:48.685467+0000","last_change":"2023-06-03T19:32:21.707936+0000","last_active":"2023-06-03T19:34:48.685467+0000","last_peered":"2023-06-03T19:34:48.685467+0000","last_clean":"2023-06-03T19:34:48.685467+0000","last_became_active":"2023-06-03T19:32:21.707538+0000","last_became_peered":"2023-06-03T19:32:21.707538+0000","last_unstale":"2023-06-03T19:34:48.685467+0000","last_undegraded":"2023-06-03T19:34:48.685467+0000","last_fullsized":"2023-06-03T19:34:48.685467+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:32:20.692043+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:32:20.692043+0000","last_clean_scrub_stamp":"2023-06-03T19:32:20.692043+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":13,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-06-03T19:34:48.695348+0000","last_change":"2023-06-03T19:34:17.828260+0000","last_active":"2023-06-03T19:34:48.695348+0000","last_peered":"2023-06-03T19:34:48.695348+0000","last_clean":"2023-06-03T19:34:48.695348+0000","last_became_active":"2023-06-03T19:34:17.827781+0000","last_became_peered":"2023-06-03T19:34:17.827781+0000","last_unstale":"2023-06-03T19:34:48.695348+0000","last_undegraded":"2023-06-03T19:34:48.695348+0000","last_fullsized":"2023-06-03T19:34:48.695348+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":14,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-06-03T19:34:48.692754+0000","last_change":"2023-06-03T19:34:16.805748+0000","last_active":"2023-06-03T19:34:48.692754+0000","last_peered":"2023-06-03T19:34:48.692754+0000","last_clean":"2023-06-03T19:34:48.692754+0000","last_became_active":"2023-06-03T19:34:16.805383+0000","last_became_peered":"2023-06-03T19:34:16.805383+0000","last_unstale":"2023-06-03T19:34:48.692754+0000","last_undegraded":"2023-06-03T19:34:48.692754+0000","last_fullsized":"2023-06-03T19:34:48.692754+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":18,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-06-03T19:34:48.689191+0000","last_change":"2023-06-03T19:32:22.123980+0000","last_active":"2023-06-03T19:34:48.689191+0000","last_peered":"2023-06-03T19:34:48.689191+0000","last_clean":"2023-06-03T19:34:48.689191+0000","last_became_active":"2023-06-03T19:32:22.123641+0000","last_became_peered":"2023-06-03T19:32:22.123641+0000","last_unstale":"2023-06-03T19:34:48.689191+0000","last_undegraded":"2023-06-03T19:34:48.689191+0000","last_fullsized":"2023-06-03T19:34:48.689191+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:32:20.692043+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:32:20.692043+0000","last_clean_scrub_stamp":"2023-06-03T19:32:20.692043+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":13,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-06-03T19:34:48.694664+0000","last_change":"2023-06-03T19:34:17.829377+0000","last_active":"2023-06-03T19:34:48.694664+0000","last_peered":"2023-06-03T19:34:48.694664+0000","last_clean":"2023-06-03T19:34:48.694664+0000","last_became_active":"2023-06-03T19:34:17.829027+0000","last_became_peered":"2023-06-03T19:34:17.829027+0000","last_unstale":"2023-06-03T19:34:48.694664+0000","last_undegraded":"2023-06-03T19:34:48.694664+0000","last_fullsized":"2023-06-03T19:34:48.694664+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":14,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-06-03T19:34:48.688564+0000","last_change":"2023-06-03T19:34:16.804662+0000","last_active":"2023-06-03T19:34:48.688564+0000","last_peered":"2023-06-03T19:34:48.688564+0000","last_clean":"2023-06-03T19:34:48.688564+0000","last_became_active":"2023-06-03T19:34:16.804350+0000","last_became_peered":"2023-06-03T19:34:16.804350+0000","last_unstale":"2023-06-03T19:34:48.688564+0000","last_undegraded":"2023-06-03T19:34:48.688564+0000","last_fullsized":"2023-06-03T19:34:48.688564+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":18,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-06-03T19:34:48.685279+0000","last_change":"2023-06-03T19:32:21.709485+0000","last_active":"2023-06-03T19:34:48.685279+0000","last_peered":"2023-06-03T19:34:48.685279+0000","last_clean":"2023-06-03T19:34:48.685279+0000","last_became_active":"2023-06-03T19:32:21.708866+0000","last_became_peered":"2023-06-03T19:32:21.708866+0000","last_unstale":"2023-06-03T19:34:48.685279+0000","last_undegraded":"2023-06-03T19:34:48.685279+0000","last_fullsized":"2023-06-03T19:34:48.685279+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:32:20.692043+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:32:20.692043+0000","last_clean_scrub_stamp":"2023-06-03T19:32:20.692043+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":13,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-06-03T19:34:48.692657+0000","last_change":"2023-06-03T19:34:17.833656+0000","last_active":"2023-06-03T19:34:48.692657+0000","last_peered":"2023-06-03T19:34:48.692657+0000","last_clean":"2023-06-03T19:34:48.692657+0000","last_became_active":"2023-06-03T19:34:17.833319+0000","last_became_peered":"2023-06-03T19:34:17.833319+0000","last_unstale":"2023-06-03T19:34:48.692657+0000","last_undegraded":"2023-06-03T19:34:48.692657+0000","last_fullsized":"2023-06-03T19:34:48.692657+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":14,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-06-03T19:34:48.685848+0000","last_change":"2023-06-03T19:34:16.803369+0000","last_active":"2023-06-03T19:34:48.685848+0000","last_peered":"2023-06-03T19:34:48.685848+0000","last_clean":"2023-06-03T19:34:48.685848+0000","last_became_active":"2023-06-03T19:34:16.803020+0000","last_became_peered":"2023-06-03T19:34:16.803020+0000","last_unstale":"2023-06-03T19:34:48.685848+0000","last_undegraded":"2023-06-03T19:34:48.685848+0000","last_fullsized":"2023-06-03T19:34:48.685848+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":18,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-06-03T19:34:48.694479+0000","last_change":"2023-06-03T19:32:22.125458+0000","last_active":"2023-06-03T19:34:48.694479+0000","last_peered":"2023-06-03T19:34:48.694479+0000","last_clean":"2023-06-03T19:34:48.694479+0000","last_became_active":"2023-06-03T19:32:22.124914+0000","last_became_peered":"2023-06-03T19:32:22.124914+0000","last_unstale":"2023-06-03T19:34:48.694479+0000","last_undegraded":"2023-06-03T19:34:48.694479+0000","last_fullsized":"2023-06-03T19:34:48.694479+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:32:20.692043+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:32:20.692043+0000","last_clean_scrub_stamp":"2023-06-03T19:32:20.692043+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":13,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-06-03T19:34:48.695380+0000","last_change":"2023-06-03T19:34:17.833703+0000","last_active":"2023-06-03T19:34:48.695380+0000","last_peered":"2023-06-03T19:34:48.695380+0000","last_clean":"2023-06-03T19:34:48.695380+0000","last_became_active":"2023-06-03T19:34:17.833068+0000","last_became_peered":"2023-06-03T19:34:17.833068+0000","last_unstale":"2023-06-03T19:34:48.695380+0000","last_undegraded":"2023-06-03T19:34:48.695380+0000","last_fullsized":"2023-06-03T19:34:48.695380+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":14,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-06-03T19:34:48.686481+0000","last_change":"2023-06-03T19:34:16.810433+0000","last_active":"2023-06-03T19:34:48.686481+0000","last_peered":"2023-06-03T19:34:48.686481+0000","last_clean":"2023-06-03T19:34:48.686481+0000","last_became_active":"2023-06-03T19:34:16.810104+0000","last_became_peered":"2023-06-03T19:34:16.810104+0000","last_unstale":"2023-06-03T19:34:48.686481+0000","last_undegraded":"2023-06-03T19:34:48.686481+0000","last_fullsized":"2023-06-03T19:34:48.686481+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":13,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-06-03T19:34:48.689007+0000","last_change":"2023-06-03T19:34:17.831828+0000","last_active":"2023-06-03T19:34:48.689007+0000","last_peered":"2023-06-03T19:34:48.689007+0000","last_clean":"2023-06-03T19:34:48.689007+0000","last_became_active":"2023-06-03T19:34:17.831621+0000","last_became_peered":"2023-06-03T19:34:17.831621+0000","last_unstale":"2023-06-03T19:34:48.689007+0000","last_undegraded":"2023-06-03T19:34:48.689007+0000","last_fullsized":"2023-06-03T19:34:48.689007+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":14,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-06-03T19:34:48.689532+0000","last_change":"2023-06-03T19:34:16.800660+0000","last_active":"2023-06-03T19:34:48.689532+0000","last_peered":"2023-06-03T19:34:48.689532+0000","last_clean":"2023-06-03T19:34:48.689532+0000","last_became_active":"2023-06-03T19:34:16.800341+0000","last_became_peered":"2023-06-03T19:34:16.800341+0000","last_unstale":"2023-06-03T19:34:48.689532+0000","last_undegraded":"2023-06-03T19:34:48.689532+0000","last_fullsized":"2023-06-03T19:34:48.689532+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":22,"reported_epoch":62,"state":"active+clean","last_fresh":"2023-06-03T19:34:50.193972+0000","last_change":"2023-06-03T19:34:50.193972+0000","last_active":"2023-06-03T19:34:50.193972+0000","last_peered":"2023-06-03T19:34:50.193972+0000","last_clean":"2023-06-03T19:34:50.193972+0000","last_became_active":"2023-06-03T19:34:50.193301+0000","last_became_peered":"2023-06-03T19:34:50.193301+0000","last_unstale":"2023-06-03T19:34:50.193972+0000","last_undegraded":"2023-06-03T19:34:50.193972+0000","last_fullsized":"2023-06-03T19:34:50.193972+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,5],"acting":[4,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":14,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-06-03T19:34:48.711628+0000","last_change":"2023-06-03T19:34:16.804631+0000","last_active":"2023-06-03T19:34:48.711628+0000","last_peered":"2023-06-03T19:34:48.711628+0000","last_clean":"2023-06-03T19:34:48.711628+0000","last_became_active":"2023-06-03T19:34:16.804252+0000","last_became_peered":"2023-06-03T19:34:16.804252+0000","last_unstale":"2023-06-03T19:34:48.711628+0000","last_undegraded":"2023-06-03T19:34:48.711628+0000","last_fullsized":"2023-06-03T19:34:48.711628+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":13,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-06-03T19:34:48.689069+0000","last_change":"2023-06-03T19:34:17.833032+0000","last_active":"2023-06-03T19:34:48.689069+0000","last_peered":"2023-06-03T19:34:48.689069+0000","last_clean":"2023-06-03T19:34:48.689069+0000","last_became_active":"2023-06-03T19:34:17.832515+0000","last_became_peered":"2023-06-03T19:34:17.832515+0000","last_unstale":"2023-06-03T19:34:48.689069+0000","last_undegraded":"2023-06-03T19:34:48.689069+0000","last_fullsized":"2023-06-03T19:34:48.689069+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.b","version":"60'2","reported_seq":16,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-06-03T19:34:48.686810+0000","last_change":"2023-06-03T19:34:16.812257+0000","last_active":"2023-06-03T19:34:48.686810+0000","last_peered":"2023-06-03T19:34:48.686810+0000","last_clean":"2023-06-03T19:34:48.686810+0000","last_became_active":"2023-06-03T19:34:16.811863+0000","last_became_peered":"2023-06-03T19:34:16.811863+0000","last_unstale":"2023-06-03T19:34:48.686810+0000","last_undegraded":"2023-06-03T19:34:48.686810+0000","last_fullsized":"2023-06-03T19:34:48.686810+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","log_size":2,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":22,"reported_epoch":62,"state":"active+clean","last_fresh":"2023-06-03T19:34:50.186367+0000","last_change":"2023-06-03T19:34:50.186367+0000","last_active":"2023-06-03T19:34:50.186367+0000","last_peered":"2023-06-03T19:34:50.186367+0000","last_clean":"2023-06-03T19:34:50.186367+0000","last_became_active":"2023-06-03T19:34:50.185595+0000","last_became_peered":"2023-06-03T19:34:50.185595+0000","last_unstale":"2023-06-03T19:34:50.186367+0000","last_undegraded":"2023-06-03T19:34:50.186367+0000","last_fullsized":"2023-06-03T19:34:50.186367+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,4],"acting":[0,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":14,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-06-03T19:34:48.685592+0000","last_change":"2023-06-03T19:34:16.798436+0000","last_active":"2023-06-03T19:34:48.685592+0000","last_peered":"2023-06-03T19:34:48.685592+0000","last_clean":"2023-06-03T19:34:48.685592+0000","last_became_active":"2023-06-03T19:34:16.798053+0000","last_became_peered":"2023-06-03T19:34:16.798053+0000","last_unstale":"2023-06-03T19:34:48.685592+0000","last_undegraded":"2023-06-03T19:34:48.685592+0000","last_fullsized":"2023-06-03T19:34:48.685592+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":13,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-06-03T19:34:48.711807+0000","last_change":"2023-06-03T19:34:17.848644+0000","last_active":"2023-06-03T19:34:48.711807+0000","last_peered":"2023-06-03T19:34:48.711807+0000","last_clean":"2023-06-03T19:34:48.711807+0000","last_became_active":"2023-06-03T19:34:17.848051+0000","last_became_peered":"2023-06-03T19:34:17.848051+0000","last_unstale":"2023-06-03T19:34:48.711807+0000","last_undegraded":"2023-06-03T19:34:48.711807+0000","last_fullsized":"2023-06-03T19:34:48.711807+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"60'1","reported_seq":15,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-06-03T19:34:48.712805+0000","last_change":"2023-06-03T19:34:16.806559+0000","last_active":"2023-06-03T19:34:48.712805+0000","last_peered":"2023-06-03T19:34:48.712805+0000","last_clean":"2023-06-03T19:34:48.712805+0000","last_became_active":"2023-06-03T19:34:16.806149+0000","last_became_peered":"2023-06-03T19:34:16.806149+0000","last_unstale":"2023-06-03T19:34:48.712805+0000","last_undegraded":"2023-06-03T19:34:48.712805+0000","last_fullsized":"2023-06-03T19:34:48.712805+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","log_size":1,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":22,"reported_epoch":62,"state":"active+clean","last_fresh":"2023-06-03T19:34:50.184855+0000","last_change":"2023-06-03T19:34:50.184855+0000","last_active":"2023-06-03T19:34:50.184855+0000","last_peered":"2023-06-03T19:34:50.184855+0000","last_clean":"2023-06-03T19:34:50.184855+0000","last_became_active":"2023-06-03T19:34:50.184070+0000","last_became_peered":"2023-06-03T19:34:50.184070+0000","last_unstale":"2023-06-03T19:34:50.184855+0000","last_undegraded":"2023-06-03T19:34:50.184855+0000","last_fullsized":"2023-06-03T19:34:50.184855+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,3],"acting":[4,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.e","version":"60'2","reported_seq":16,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-06-03T19:34:48.712689+0000","last_change":"2023-06-03T19:34:16.800653+0000","last_active":"2023-06-03T19:34:48.712689+0000","last_peered":"2023-06-03T19:34:48.712689+0000","last_clean":"2023-06-03T19:34:48.712689+0000","last_became_active":"2023-06-03T19:34:16.800085+0000","last_became_peered":"2023-06-03T19:34:16.800085+0000","last_unstale":"2023-06-03T19:34:48.712689+0000","last_undegraded":"2023-06-03T19:34:48.712689+0000","last_fullsized":"2023-06-03T19:34:48.712689+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","log_size":2,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":13,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-06-03T19:34:48.685239+0000","last_change":"2023-06-03T19:34:17.848358+0000","last_active":"2023-06-03T19:34:48.685239+0000","last_peered":"2023-06-03T19:34:48.685239+0000","last_clean":"2023-06-03T19:34:48.685239+0000","last_became_active":"2023-06-03T19:34:17.847811+0000","last_became_peered":"2023-06-03T19:34:17.847811+0000","last_unstale":"2023-06-03T19:34:48.685239+0000","last_undegraded":"2023-06-03T19:34:48.685239+0000","last_fullsized":"2023-06-03T19:34:48.685239+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"60'1","reported_seq":15,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-06-03T19:34:48.712873+0000","last_change":"2023-06-03T19:34:16.807961+0000","last_active":"2023-06-03T19:34:48.712873+0000","last_peered":"2023-06-03T19:34:48.712873+0000","last_clean":"2023-06-03T19:34:48.712873+0000","last_became_active":"2023-06-03T19:34:16.805920+0000","last_became_peered":"2023-06-03T19:34:16.805920+0000","last_unstale":"2023-06-03T19:34:48.712873+0000","last_undegraded":"2023-06-03T19:34:48.712873+0000","last_fullsized":"2023-06-03T19:34:48.712873+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","log_size":1,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":13,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-06-03T19:34:48.689128+0000","last_change":"2023-06-03T19:34:16.896490+0000","last_active":"2023-06-03T19:34:48.689128+0000","last_peered":"2023-06-03T19:34:48.689128+0000","last_clean":"2023-06-03T19:34:48.689128+0000","last_became_active":"2023-06-03T19:34:16.896119+0000","last_became_peered":"2023-06-03T19:34:16.896119+0000","last_unstale":"2023-06-03T19:34:48.689128+0000","last_undegraded":"2023-06-03T19:34:48.689128+0000","last_fullsized":"2023-06-03T19:34:48.689128+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"60'8","reported_seq":22,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-06-03T19:34:48.711523+0000","last_change":"2023-06-03T19:34:16.803630+0000","last_active":"2023-06-03T19:34:48.711523+0000","last_peered":"2023-06-03T19:34:48.711523+0000","last_clean":"2023-06-03T19:34:48.711523+0000","last_became_active":"2023-06-03T19:34:16.803280+0000","last_became_peered":"2023-06-03T19:34:16.803280+0000","last_unstale":"2023-06-03T19:34:48.711523+0000","last_undegraded":"2023-06-03T19:34:48.711523+0000","last_fullsized":"2023-06-03T19:34:48.711523+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","log_size":8,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":13,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-06-03T19:34:48.693907+0000","last_change":"2023-06-03T19:34:17.850887+0000","last_active":"2023-06-03T19:34:48.693907+0000","last_peered":"2023-06-03T19:34:48.693907+0000","last_clean":"2023-06-03T19:34:48.693907+0000","last_became_active":"2023-06-03T19:34:17.850146+0000","last_became_peered":"2023-06-03T19:34:17.850146+0000","last_unstale":"2023-06-03T19:34:48.693907+0000","last_undegraded":"2023-06-03T19:34:48.693907+0000","last_fullsized":"2023-06-03T19:34:48.693907+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":14,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-06-03T19:34:48.712131+0000","last_change":"2023-06-03T19:34:16.806286+0000","last_active":"2023-06-03T19:34:48.712131+0000","last_peered":"2023-06-03T19:34:48.712131+0000","last_clean":"2023-06-03T19:34:48.712131+0000","last_became_active":"2023-06-03T19:34:16.805898+0000","last_became_peered":"2023-06-03T19:34:16.805898+0000","last_unstale":"2023-06-03T19:34:48.712131+0000","last_undegraded":"2023-06-03T19:34:48.712131+0000","last_fullsized":"2023-06-03T19:34:48.712131+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":13,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-06-03T19:34:48.685691+0000","last_change":"2023-06-03T19:34:17.848569+0000","last_active":"2023-06-03T19:34:48.685691+0000","last_peered":"2023-06-03T19:34:48.685691+0000","last_clean":"2023-06-03T19:34:48.685691+0000","last_became_active":"2023-06-03T19:34:17.848038+0000","last_became_peered":"2023-06-03T19:34:17.848038+0000","last_unstale":"2023-06-03T19:34:48.685691+0000","last_undegraded":"2023-06-03T19:34:48.685691+0000","last_fullsized":"2023-06-03T19:34:48.685691+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":14,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-06-03T19:34:48.694032+0000","last_change":"2023-06-03T19:34:16.814726+0000","last_active":"2023-06-03T19:34:48.694032+0000","last_peered":"2023-06-03T19:34:48.694032+0000","last_clean":"2023-06-03T19:34:48.694032+0000","last_became_active":"2023-06-03T19:34:16.814150+0000","last_became_peered":"2023-06-03T19:34:16.814150+0000","last_unstale":"2023-06-03T19:34:48.694032+0000","last_undegraded":"2023-06-03T19:34:48.694032+0000","last_fullsized":"2023-06-03T19:34:48.694032+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":13,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-06-03T19:34:48.688078+0000","last_change":"2023-06-03T19:34:16.896666+0000","last_active":"2023-06-03T19:34:48.688078+0000","last_peered":"2023-06-03T19:34:48.688078+0000","last_clean":"2023-06-03T19:34:48.688078+0000","last_became_active":"2023-06-03T19:34:16.895977+0000","last_became_peered":"2023-06-03T19:34:16.895977+0000","last_unstale":"2023-06-03T19:34:48.688078+0000","last_undegraded":"2023-06-03T19:34:48.688078+0000","last_fullsized":"2023-06-03T19:34:48.688078+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"60'2","reported_seq":16,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-06-03T19:34:48.712350+0000","last_change":"2023-06-03T19:34:16.800482+0000","last_active":"2023-06-03T19:34:48.712350+0000","last_peered":"2023-06-03T19:34:48.712350+0000","last_clean":"2023-06-03T19:34:48.712350+0000","last_became_active":"2023-06-03T19:34:16.799932+0000","last_became_peered":"2023-06-03T19:34:16.799932+0000","last_unstale":"2023-06-03T19:34:48.712350+0000","last_undegraded":"2023-06-03T19:34:48.712350+0000","last_fullsized":"2023-06-03T19:34:48.712350+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","log_size":2,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":13,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-06-03T19:34:48.688794+0000","last_change":"2023-06-03T19:34:17.832978+0000","last_active":"2023-06-03T19:34:48.688794+0000","last_peered":"2023-06-03T19:34:48.688794+0000","last_clean":"2023-06-03T19:34:48.688794+0000","last_became_active":"2023-06-03T19:34:17.832312+0000","last_became_peered":"2023-06-03T19:34:17.832312+0000","last_unstale":"2023-06-03T19:34:48.688794+0000","last_undegraded":"2023-06-03T19:34:48.688794+0000","last_fullsized":"2023-06-03T19:34:48.688794+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"60'6","reported_seq":20,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-06-03T19:34:48.688869+0000","last_change":"2023-06-03T19:34:16.800070+0000","last_active":"2023-06-03T19:34:48.688869+0000","last_peered":"2023-06-03T19:34:48.688869+0000","last_clean":"2023-06-03T19:34:48.688869+0000","last_became_active":"2023-06-03T19:34:16.798944+0000","last_became_peered":"2023-06-03T19:34:16.798944+0000","last_unstale":"2023-06-03T19:34:48.688869+0000","last_undegraded":"2023-06-03T19:34:48.688869+0000","last_fullsized":"2023-06-03T19:34:48.688869+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","log_size":6,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":1536,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6,"num_write_kb":3,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":13,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-06-03T19:34:48.692607+0000","last_change":"2023-06-03T19:34:17.829839+0000","last_active":"2023-06-03T19:34:48.692607+0000","last_peered":"2023-06-03T19:34:48.692607+0000","last_clean":"2023-06-03T19:34:48.692607+0000","last_became_active":"2023-06-03T19:34:17.829509+0000","last_became_peered":"2023-06-03T19:34:17.829509+0000","last_unstale":"2023-06-03T19:34:48.692607+0000","last_undegraded":"2023-06-03T19:34:48.692607+0000","last_fullsized":"2023-06-03T19:34:48.692607+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"60'3","reported_seq":17,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-06-03T19:34:48.712587+0000","last_change":"2023-06-03T19:34:16.808044+0000","last_active":"2023-06-03T19:34:48.712587+0000","last_peered":"2023-06-03T19:34:48.712587+0000","last_clean":"2023-06-03T19:34:48.712587+0000","last_became_active":"2023-06-03T19:34:16.807735+0000","last_became_peered":"2023-06-03T19:34:16.807735+0000","last_unstale":"2023-06-03T19:34:48.712587+0000","last_undegraded":"2023-06-03T19:34:48.712587+0000","last_fullsized":"2023-06-03T19:34:48.712587+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","log_size":3,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":13,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-06-03T19:34:48.687564+0000","last_change":"2023-06-03T19:34:17.293132+0000","last_active":"2023-06-03T19:34:48.687564+0000","last_peered":"2023-06-03T19:34:48.687564+0000","last_clean":"2023-06-03T19:34:48.687564+0000","last_became_active":"2023-06-03T19:34:17.292642+0000","last_became_peered":"2023-06-03T19:34:17.292642+0000","last_unstale":"2023-06-03T19:34:48.687564+0000","last_undegraded":"2023-06-03T19:34:48.687564+0000","last_fullsized":"2023-06-03T19:34:48.687564+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":14,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-06-03T19:34:48.685695+0000","last_change":"2023-06-03T19:34:16.800538+0000","last_active":"2023-06-03T19:34:48.685695+0000","last_peered":"2023-06-03T19:34:48.685695+0000","last_clean":"2023-06-03T19:34:48.685695+0000","last_became_active":"2023-06-03T19:34:16.799538+0000","last_became_peered":"2023-06-03T19:34:16.799538+0000","last_unstale":"2023-06-03T19:34:48.685695+0000","last_undegraded":"2023-06-03T19:34:48.685695+0000","last_fullsized":"2023-06-03T19:34:48.685695+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":13,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-06-03T19:34:48.686586+0000","last_change":"2023-06-03T19:34:17.293481+0000","last_active":"2023-06-03T19:34:48.686586+0000","last_peered":"2023-06-03T19:34:48.686586+0000","last_clean":"2023-06-03T19:34:48.686586+0000","last_became_active":"2023-06-03T19:34:17.292639+0000","last_became_peered":"2023-06-03T19:34:17.292639+0000","last_unstale":"2023-06-03T19:34:48.686586+0000","last_undegraded":"2023-06-03T19:34:48.686586+0000","last_fullsized":"2023-06-03T19:34:48.686586+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":14,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-06-03T19:34:48.694274+0000","last_change":"2023-06-03T19:34:16.814709+0000","last_active":"2023-06-03T19:34:48.694274+0000","last_peered":"2023-06-03T19:34:48.694274+0000","last_clean":"2023-06-03T19:34:48.694274+0000","last_became_active":"2023-06-03T19:34:16.814149+0000","last_became_peered":"2023-06-03T19:34:16.814149+0000","last_unstale":"2023-06-03T19:34:48.694274+0000","last_undegraded":"2023-06-03T19:34:48.694274+0000","last_fullsized":"2023-06-03T19:34:48.694274+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":13,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-06-03T19:34:48.711508+0000","last_change":"2023-06-03T19:34:17.825928+0000","last_active":"2023-06-03T19:34:48.711508+0000","last_peered":"2023-06-03T19:34:48.711508+0000","last_clean":"2023-06-03T19:34:48.711508+0000","last_became_active":"2023-06-03T19:34:17.824849+0000","last_became_peered":"2023-06-03T19:34:17.824849+0000","last_unstale":"2023-06-03T19:34:48.711508+0000","last_undegraded":"2023-06-03T19:34:48.711508+0000","last_fullsized":"2023-06-03T19:34:48.711508+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":14,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-06-03T19:34:48.686730+0000","last_change":"2023-06-03T19:34:16.810509+0000","last_active":"2023-06-03T19:34:48.686730+0000","last_peered":"2023-06-03T19:34:48.686730+0000","last_clean":"2023-06-03T19:34:48.686730+0000","last_became_active":"2023-06-03T19:34:16.810046+0000","last_became_peered":"2023-06-03T19:34:16.810046+0000","last_unstale":"2023-06-03T19:34:48.686730+0000","last_undegraded":"2023-06-03T19:34:48.686730+0000","last_fullsized":"2023-06-03T19:34:48.686730+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":13,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-06-03T19:34:48.694018+0000","last_change":"2023-06-03T19:34:17.850356+0000","last_active":"2023-06-03T19:34:48.694018+0000","last_peered":"2023-06-03T19:34:48.694018+0000","last_clean":"2023-06-03T19:34:48.694018+0000","last_became_active":"2023-06-03T19:34:17.849850+0000","last_became_peered":"2023-06-03T19:34:17.849850+0000","last_unstale":"2023-06-03T19:34:48.694018+0000","last_undegraded":"2023-06-03T19:34:48.694018+0000","last_fullsized":"2023-06-03T19:34:48.694018+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":14,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-06-03T19:34:48.693391+0000","last_change":"2023-06-03T19:34:16.805815+0000","last_active":"2023-06-03T19:34:48.693391+0000","last_peered":"2023-06-03T19:34:48.693391+0000","last_clean":"2023-06-03T19:34:48.693391+0000","last_became_active":"2023-06-03T19:34:16.805501+0000","last_became_peered":"2023-06-03T19:34:16.805501+0000","last_unstale":"2023-06-03T19:34:48.693391+0000","last_undegraded":"2023-06-03T19:34:48.693391+0000","last_fullsized":"2023-06-03T19:34:48.693391+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":13,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-06-03T19:34:48.688274+0000","last_change":"2023-06-03T19:34:16.822573+0000","last_active":"2023-06-03T19:34:48.688274+0000","last_peered":"2023-06-03T19:34:48.688274+0000","last_clean":"2023-06-03T19:34:48.688274+0000","last_became_active":"2023-06-03T19:34:16.822213+0000","last_became_peered":"2023-06-03T19:34:16.822213+0000","last_unstale":"2023-06-03T19:34:48.688274+0000","last_undegraded":"2023-06-03T19:34:48.688274+0000","last_fullsized":"2023-06-03T19:34:48.688274+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"60'1","reported_seq":15,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-06-03T19:34:48.688450+0000","last_change":"2023-06-03T19:34:16.804144+0000","last_active":"2023-06-03T19:34:48.688450+0000","last_peered":"2023-06-03T19:34:48.688450+0000","last_clean":"2023-06-03T19:34:48.688450+0000","last_became_active":"2023-06-03T19:34:16.803674+0000","last_became_peered":"2023-06-03T19:34:16.803674+0000","last_unstale":"2023-06-03T19:34:48.688450+0000","last_undegraded":"2023-06-03T19:34:48.688450+0000","last_fullsized":"2023-06-03T19:34:48.688450+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","log_size":1,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":2306,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":45,"num_write_kb":13,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":6918,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":39,"ondisk_log_size":39,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":3,"acting":3,"num_store_stats":3},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":48,"seq":206158430269,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":5704,"kb_used_data":640,"kb_used_omap":0,"kb_used_meta":5056,"kb_avail":93739448,"statfs":{"total":95995035648,"available":95989194752,"internally_reserved":0,"allocated":655360,"data_stored":271688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5177344},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 3 19:35:29 2023","interfaces":[{"interface":"back","average":{"1min":0.87,"5min":0.84599999999999997,"15min":0.81999999999999995},"min":{"1min":0.57399999999999995,"5min":0.54500000000000004,"15min":0.54500000000000004},"max":{"1min":1.0649999999999999,"5min":1.232,"15min":1.232},"last":1.073},{"interface":"front","average":{"1min":0.95399999999999996,"5min":0.90800000000000003,"15min":0.88300000000000001},"min":{"1min":0.60299999999999998,"5min":0.56499999999999995,"15min":0.56499999999999995},"max":{"1min":1.2849999999999999,"5min":1.2849999999999999,"15min":1.2849999999999999},"last":0.996}]},{"osd":1,"last update":"Sat Jun 3 19:35:29 2023","interfaces":[{"interface":"back","average":{"1min":0.89700000000000002,"5min":0.86099999999999999,"15min":0.83299999999999996},"min":{"1min":0.58999999999999997,"5min":0.51700000000000002,"15min":0.51700000000000002},"max":{"1min":1.155,"5min":1.2130000000000001,"15min":1.2130000000000001},"last":1.1870000000000001},{"interface":"front","average":{"1min":0.95399999999999996,"5min":0.90300000000000002,"15min":0.878},"min":{"1min":0.622,"5min":0.54500000000000004,"15min":0.54500000000000004},"max":{"1min":1.1879999999999999,"5min":1.228,"15min":1.228},"last":1.0409999999999999}]},{"osd":2,"last update":"Sat Jun 3 19:35:29 2023","interfaces":[{"interface":"back","average":{"1min":0.92500000000000004,"5min":0.93000000000000005,"15min":0.92100000000000004},"min":{"1min":0.63,"5min":0.58999999999999997,"15min":0.58999999999999997},"max":{"1min":1.131,"5min":1.538,"15min":1.538},"last":1.1379999999999999},{"interface":"front","average":{"1min":0.94499999999999995,"5min":0.90900000000000003,"15min":0.876},"min":{"1min":0.74399999999999999,"5min":0.60699999999999998,"15min":0.60699999999999998},"max":{"1min":1.1819999999999999,"5min":1.24,"15min":1.24},"last":0.93999999999999995}]},{"osd":3,"last update":"Sat Jun 3 19:35:29 2023","interfaces":[{"interface":"back","average":{"1min":0.94499999999999995,"5min":0.93300000000000005,"15min":0.91500000000000004},"min":{"1min":0.73099999999999998,"5min":0.53800000000000003,"15min":0.53800000000000003},"max":{"1min":1.22,"5min":1.3069999999999999,"15min":1.3069999999999999},"last":1.236},{"interface":"front","average":{"1min":0.94499999999999995,"5min":0.93600000000000005,"15min":0.93200000000000005},"min":{"1min":0.71499999999999997,"5min":0.57199999999999995,"15min":0.57199999999999995},"max":{"1min":1.1919999999999999,"5min":1.5069999999999999,"15min":1.5069999999999999},"last":0.75600000000000001}]},{"osd":4,"last update":"Sat Jun 3 19:35:29 2023","interfaces":[{"interface":"back","average":{"1min":0.94899999999999995,"5min":0.83699999999999997,"15min":0.76200000000000001},"min":{"1min":0.69099999999999995,"5min":0.51200000000000001,"15min":0.51200000000000001},"max":{"1min":1.165,"5min":1.2829999999999999,"15min":1.2829999999999999},"last":1.2130000000000001},{"interface":"front","average":{"1min":0.94299999999999995,"5min":0.88500000000000001,"15min":0.84099999999999997},"min":{"1min":0.60699999999999998,"5min":0.45500000000000002,"15min":0.45500000000000002},"max":{"1min":1.2769999999999999,"5min":1.351,"15min":1.351},"last":0.84799999999999998}]},{"osd":5,"last update":"Sat Jun 3 19:35:29 2023","interfaces":[{"interface":"back","average":{"1min":0.996,"5min":0.89200000000000002,"15min":0.83599999999999997},"min":{"1min":0.65300000000000002,"5min":0.48499999999999999,"15min":0.48499999999999999},"max":{"1min":1.3560000000000001,"5min":1.423,"15min":1.423},"last":1.113},{"interface":"front","average":{"1min":1.0149999999999999,"5min":0.97099999999999997,"15min":0.94099999999999995},"min":{"1min":0.64700000000000002,"5min":0.437,"15min":0.437},"max":{"1min":1.268,"5min":1.605,"15min":1.605},"last":1.2669999999999999}]},{"osd":6,"last update":"Sat Jun 3 19:35:29 2023","interfaces":[{"interface":"back","average":{"1min":1.054,"5min":1.03,"15min":0.98799999999999999},"min":{"1min":0.55900000000000005,"5min":0.47899999999999998,"15min":0.47899999999999998},"max":{"1min":1.254,"5min":1.6870000000000001,"15min":1.6870000000000001},"last":1.3069999999999999},{"interface":"front","average":{"1min":1.0569999999999999,"5min":1.002,"15min":0.95999999999999996},"min":{"1min":0.73199999999999998,"5min":0.59299999999999997,"15min":0.59299999999999997},"max":{"1min":1.329,"5min":1.4319999999999999,"15min":1.4319999999999999},"last":1.173}]}]},{"osd":6,"up_from":42,"seq":180388626502,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":5688,"kb_used_data":624,"kb_used_omap":0,"kb_used_meta":5056,"kb_avail":93739464,"statfs":{"total":95995035648,"available":95989211136,"internally_reserved":0,"allocated":638976,"data_stored":271227,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5177344},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 3 19:35:38 2023","interfaces":[{"interface":"back","average":{"1min":0.84799999999999998,"5min":0.84599999999999997,"15min":0.83199999999999996},"min":{"1min":0.45400000000000001,"5min":0.45400000000000001,"15min":0.45400000000000001},"max":{"1min":1.2350000000000001,"5min":1.2749999999999999,"15min":1.2749999999999999},"last":1.145},{"interface":"front","average":{"1min":0.84899999999999998,"5min":0.86099999999999999,"15min":0.86799999999999999},"min":{"1min":0.57699999999999996,"5min":0.52900000000000003,"15min":0.52900000000000003},"max":{"1min":1.2110000000000001,"5min":1.284,"15min":1.284},"last":1.0900000000000001}]},{"osd":1,"last update":"Sat Jun 3 19:35:38 2023","interfaces":[{"interface":"back","average":{"1min":0.878,"5min":0.873,"15min":0.86899999999999999},"min":{"1min":0.58099999999999996,"5min":0.50700000000000001,"15min":0.50700000000000001},"max":{"1min":1.143,"5min":1.3759999999999999,"15min":1.3759999999999999},"last":1.044},{"interface":"front","average":{"1min":0.89000000000000001,"5min":0.89900000000000002,"15min":0.93999999999999995},"min":{"1min":0.66800000000000004,"5min":0.54100000000000004,"15min":0.54100000000000004},"max":{"1min":1.1299999999999999,"5min":1.7769999999999999,"15min":1.7769999999999999},"last":1.417}]},{"osd":2,"last update":"Sat Jun 3 19:35:38 2023","interfaces":[{"interface":"back","average":{"1min":0.874,"5min":0.89900000000000002,"15min":0.91300000000000003},"min":{"1min":0.68000000000000005,"5min":0.58199999999999996,"15min":0.58199999999999996},"max":{"1min":1.1220000000000001,"5min":1.3069999999999999,"15min":1.3069999999999999},"last":0.90700000000000003},{"interface":"front","average":{"1min":0.996,"5min":0.93899999999999995,"15min":0.93500000000000005},"min":{"1min":0.70199999999999996,"5min":0.63400000000000001,"15min":0.63400000000000001},"max":{"1min":1.2949999999999999,"5min":1.3400000000000001,"15min":1.3400000000000001},"last":1.2270000000000001}]},{"osd":3,"last update":"Sat Jun 3 19:35:38 2023","interfaces":[{"interface":"back","average":{"1min":0.93999999999999995,"5min":0.88900000000000001,"15min":0.85799999999999998},"min":{"1min":0.65600000000000003,"5min":0.60199999999999998,"15min":0.60199999999999998},"max":{"1min":1.1919999999999999,"5min":1.2290000000000001,"15min":1.2290000000000001},"last":0.86699999999999999},{"interface":"front","average":{"1min":1.008,"5min":0.97699999999999998,"15min":0.998},"min":{"1min":0.76900000000000002,"5min":0.64200000000000002,"15min":0.64200000000000002},"max":{"1min":1.177,"5min":1.905,"15min":1.905},"last":1.302}]},{"osd":4,"last update":"Sat Jun 3 19:35:38 2023","interfaces":[{"interface":"back","average":{"1min":0.92800000000000005,"5min":0.88800000000000001,"15min":0.85199999999999998},"min":{"1min":0.59699999999999998,"5min":0.46899999999999997,"15min":0.46899999999999997},"max":{"1min":1.2250000000000001,"5min":1.3140000000000001,"15min":1.3140000000000001},"last":1.4379999999999999},{"interface":"front","average":{"1min":0.88300000000000001,"5min":0.90800000000000003,"15min":0.88},"min":{"1min":0.65200000000000002,"5min":0.434,"15min":0.434},"max":{"1min":1.284,"5min":1.3129999999999999,"15min":1.3129999999999999},"last":1.2849999999999999}]},{"osd":5,"last update":"Sat Jun 3 19:35:38 2023","interfaces":[{"interface":"back","average":{"1min":0.98399999999999999,"5min":0.96899999999999997,"15min":0.93100000000000005},"min":{"1min":0.64400000000000002,"5min":0.41599999999999998,"15min":0.41599999999999998},"max":{"1min":1.236,"5min":1.3109999999999999,"15min":1.3109999999999999},"last":1.391},{"interface":"front","average":{"1min":0.99099999999999999,"5min":0.96199999999999997,"15min":0.94399999999999995},"min":{"1min":0.72799999999999998,"5min":0.52500000000000002,"15min":0.52500000000000002},"max":{"1min":1.236,"5min":1.284,"15min":1.284},"last":1.1970000000000001}]},{"osd":7,"last update":"Sat Jun 3 19:35:22 2023","interfaces":[{"interface":"back","average":{"1min":0.99199999999999999,"5min":0.998,"15min":0.95499999999999996},"min":{"1min":0.70399999999999996,"5min":0.45700000000000002,"15min":0.45700000000000002},"max":{"1min":1.1839999999999999,"5min":1.403,"15min":1.403},"last":1.347},{"interface":"front","average":{"1min":1.0509999999999999,"5min":1.028,"15min":1.006},"min":{"1min":0.69199999999999995,"5min":0.53600000000000003,"15min":0.53600000000000003},"max":{"1min":1.3109999999999999,"5min":1.3839999999999999,"15min":1.3839999999999999},"last":1.4770000000000001}]}]},{"osd":1,"up_from":13,"seq":55834574964,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":5928,"kb_used_data":608,"kb_used_omap":0,"kb_used_meta":5312,"kb_avail":93739224,"statfs":{"total":95995035648,"available":95988965376,"internally_reserved":0,"allocated":622592,"data_stored":269621,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5439488},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 3 19:35:51 2023","interfaces":[{"interface":"back","average":{"1min":0.64800000000000002,"5min":0.65900000000000003,"15min":0.58999999999999997},"min":{"1min":0.36699999999999999,"5min":0.34200000000000003,"15min":0.34200000000000003},"max":{"1min":1.0860000000000001,"5min":1.0920000000000001,"15min":1.123},"last":0.66200000000000003},{"interface":"front","average":{"1min":0.64500000000000002,"5min":0.68100000000000005,"15min":0.63200000000000001},"min":{"1min":0.33000000000000002,"5min":0.30399999999999999,"15min":0.29699999999999999},"max":{"1min":1.242,"5min":1.242,"15min":1.242},"last":0.81299999999999994}]},{"osd":2,"last update":"Sat Jun 3 19:35:42 2023","interfaces":[{"interface":"back","average":{"1min":0.66000000000000003,"5min":0.71099999999999997,"15min":0.64400000000000002},"min":{"1min":0.32800000000000001,"5min":0.32800000000000001,"15min":0.27600000000000002},"max":{"1min":1.159,"5min":1.2030000000000001,"15min":1.2030000000000001},"last":0.748},{"interface":"front","average":{"1min":0.65600000000000003,"5min":0.69399999999999995,"15min":0.67100000000000004},"min":{"1min":0.378,"5min":0.378,"15min":0.32100000000000001},"max":{"1min":1.0389999999999999,"5min":1.0389999999999999,"15min":1.2150000000000001},"last":0.70999999999999996}]},{"osd":3,"last update":"Sat Jun 3 19:35:25 2023","interfaces":[{"interface":"back","average":{"1min":0.65400000000000003,"5min":0.69399999999999995,"15min":0.71899999999999997},"min":{"1min":0.378,"5min":0.378,"15min":0.378},"max":{"1min":0.878,"5min":1.1100000000000001,"15min":1.1100000000000001},"last":0.63},{"interface":"front","average":{"1min":0.76900000000000002,"5min":0.79400000000000004,"15min":0.77900000000000003},"min":{"1min":0.41599999999999998,"5min":0.39300000000000002,"15min":0.39300000000000002},"max":{"1min":1.3049999999999999,"5min":1.3049999999999999,"15min":1.3049999999999999},"last":0.79800000000000004}]},{"osd":4,"last update":"Sat Jun 3 19:35:09 2023","interfaces":[{"interface":"back","average":{"1min":1.014,"5min":0.94099999999999995,"15min":0.94799999999999995},"min":{"1min":0.75,"5min":0.60999999999999999,"15min":0.60999999999999999},"max":{"1min":1.4950000000000001,"5min":1.4950000000000001,"15min":1.4950000000000001},"last":1.0149999999999999},{"interface":"front","average":{"1min":0.998,"5min":0.94899999999999995,"15min":0.93700000000000006},"min":{"1min":0.72399999999999998,"5min":0.63200000000000001,"15min":0.63200000000000001},"max":{"1min":1.2769999999999999,"5min":1.444,"15min":1.444},"last":1.0049999999999999}]},{"osd":5,"last update":"Sat Jun 3 19:35:51 2023","interfaces":[{"interface":"back","average":{"1min":1.0529999999999999,"5min":1.0229999999999999,"15min":1.0409999999999999},"min":{"1min":0.70399999999999996,"5min":0.65200000000000002,"15min":0.65200000000000002},"max":{"1min":1.5609999999999999,"5min":1.617,"15min":1.617},"last":0.94499999999999995},{"interface":"front","average":{"1min":1.016,"5min":1.0269999999999999,"15min":1.0669999999999999},"min":{"1min":0.71799999999999997,"5min":0.67900000000000005,"15min":0.67900000000000005},"max":{"1min":1.2849999999999999,"5min":1.595,"15min":1.595},"last":1.169}]},{"osd":6,"last update":"Sat Jun 3 19:35:38 2023","interfaces":[{"interface":"back","average":{"1min":0.99199999999999999,"5min":0.998,"15min":1.0189999999999999},"min":{"1min":0.63500000000000001,"5min":0.54500000000000004,"15min":0.54500000000000004},"max":{"1min":1.333,"5min":1.4770000000000001,"15min":1.4770000000000001},"last":0.95699999999999996},{"interface":"front","average":{"1min":1.0469999999999999,"5min":1.0649999999999999,"15min":1.103},"min":{"1min":0.63300000000000001,"5min":0.57199999999999995,"15min":0.57199999999999995},"max":{"1min":1.4810000000000001,"5min":1.4810000000000001,"15min":1.4810000000000001},"last":0.98899999999999999}]},{"osd":7,"last update":"Sat Jun 3 19:35:20 2023","interfaces":[{"interface":"back","average":{"1min":1.0469999999999999,"5min":1.0569999999999999,"15min":1.083},"min":{"1min":0.60999999999999999,"5min":0.60999999999999999,"15min":0.60999999999999999},"max":{"1min":1.391,"5min":1.4810000000000001,"15min":1.4810000000000001},"last":1.101},{"interface":"front","average":{"1min":1.0629999999999999,"5min":1.1040000000000001,"15min":1.151},"min":{"1min":0.69499999999999995,"5min":0.69499999999999995,"15min":0.69499999999999995},"max":{"1min":1.4710000000000001,"5min":1.54,"15min":1.54},"last":0.85799999999999998}]}]},{"osd":0,"up_from":8,"seq":34359738493,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6000,"kb_used_data":616,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":93739152,"statfs":{"total":95995035648,"available":95988891648,"internally_reserved":0,"allocated":630784,"data_stored":270197,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Jun 3 19:36:01 2023","interfaces":[{"interface":"back","average":{"1min":0.61599999999999999,"5min":0.67800000000000005,"15min":0.60599999999999998},"min":{"1min":0.26000000000000001,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":0.94099999999999995,"5min":1.5660000000000001,"15min":1.5660000000000001},"last":0.747},{"interface":"front","average":{"1min":0.59699999999999998,"5min":0.67900000000000005,"15min":0.61299999999999999},"min":{"1min":0.40200000000000002,"5min":0.40000000000000002,"15min":0.33600000000000002},"max":{"1min":0.78200000000000003,"5min":1.228,"15min":1.228},"last":0.63900000000000001}]},{"osd":2,"last update":"Sat Jun 3 19:35:43 2023","interfaces":[{"interface":"back","average":{"1min":0.621,"5min":0.68400000000000005,"15min":0.65700000000000003},"min":{"1min":0.28199999999999997,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":1.0900000000000001,"5min":1.1719999999999999,"15min":1.1719999999999999},"last":0.60799999999999998},{"interface":"front","average":{"1min":0.63500000000000001,"5min":0.72399999999999998,"15min":0.68000000000000005},"min":{"1min":0.39000000000000001,"5min":0.39000000000000001,"15min":0.39000000000000001},"max":{"1min":1.02,"5min":1.371,"15min":1.371},"last":0.75600000000000001}]},{"osd":3,"last update":"Sat Jun 3 19:35:32 2023","interfaces":[{"interface":"back","average":{"1min":0.621,"5min":0.66600000000000004,"15min":0.72099999999999997},"min":{"1min":0.38500000000000001,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":0.98299999999999998,"5min":1.0169999999999999,"15min":1.0860000000000001},"last":0.66800000000000004},{"interface":"front","average":{"1min":0.69899999999999995,"5min":0.79500000000000004,"15min":0.79600000000000004},"min":{"1min":0.40799999999999997,"5min":0.39500000000000002,"15min":0.39500000000000002},"max":{"1min":0.94999999999999996,"5min":1.7629999999999999,"15min":1.7629999999999999},"last":0.69099999999999995}]},{"osd":4,"last update":"Sat Jun 3 19:35:03 2023","interfaces":[{"interface":"back","average":{"1min":0.97499999999999998,"5min":0.93500000000000005,"15min":0.92800000000000005},"min":{"1min":0.69099999999999995,"5min":0.51900000000000002,"15min":0.51900000000000002},"max":{"1min":1.323,"5min":1.425,"15min":1.77},"last":0.91300000000000003},{"interface":"front","average":{"1min":1.0980000000000001,"5min":0.995,"15min":0.94699999999999995},"min":{"1min":0.76900000000000002,"5min":0.56599999999999995,"15min":0.56599999999999995},"max":{"1min":1.637,"5min":1.637,"15min":1.637},"last":0.81000000000000005}]},{"osd":5,"last update":"Sat Jun 3 19:35:55 2023","interfaces":[{"interface":"back","average":{"1min":1.0740000000000001,"5min":1.0469999999999999,"15min":1.0609999999999999},"min":{"1min":0.74399999999999999,"5min":0.52700000000000002,"15min":0.52700000000000002},"max":{"1min":1.629,"5min":1.6799999999999999,"15min":1.6799999999999999},"last":0.78500000000000003},{"interface":"front","average":{"1min":1.0169999999999999,"5min":0.96999999999999997,"15min":0.97499999999999998},"min":{"1min":0.60899999999999999,"5min":0.42899999999999999,"15min":0.42899999999999999},"max":{"1min":1.5509999999999999,"5min":1.5509999999999999,"15min":1.5509999999999999},"last":1.014}]},{"osd":6,"last update":"Sat Jun 3 19:35:43 2023","interfaces":[{"interface":"back","average":{"1min":0.98799999999999999,"5min":1.056,"15min":1.083},"min":{"1min":0.56200000000000006,"5min":0.56200000000000006,"15min":0.56200000000000006},"max":{"1min":1.524,"5min":1.722,"15min":1.722},"last":1.0720000000000001},{"interface":"front","average":{"1min":1.0229999999999999,"5min":1.0489999999999999,"15min":1.0569999999999999},"min":{"1min":0.623,"5min":0.54400000000000004,"15min":0.54400000000000004},"max":{"1min":1.4379999999999999,"5min":1.4830000000000001,"15min":1.4830000000000001},"last":1.181}]},{"osd":7,"last update":"Sat Jun 3 19:35:24 2023","interfaces":[{"interface":"back","average":{"1min":0.96599999999999997,"5min":1.0669999999999999,"15min":1.079},"min":{"1min":0.64300000000000002,"5min":0.59499999999999997,"15min":0.59499999999999997},"max":{"1min":1.3360000000000001,"5min":1.514,"15min":1.514},"last":1.232},{"interface":"front","average":{"1min":0.94199999999999995,"5min":1.016,"15min":1.0349999999999999},"min":{"1min":0.65600000000000003,"5min":0.57899999999999996,"15min":0.57899999999999996},"max":{"1min":1.3859999999999999,"5min":1.786,"15min":1.786},"last":1.115}]}]},{"osd":2,"up_from":18,"seq":77309411434,"num_pgs":19,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":5812,"kb_used_data":620,"kb_used_omap":0,"kb_used_meta":5184,"kb_avail":93739340,"statfs":{"total":95995035648,"available":95989084160,"internally_reserved":0,"allocated":634880,"data_stored":270524,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5308416},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 3 19:35:42 2023","interfaces":[{"interface":"back","average":{"1min":0.64200000000000002,"5min":0.67500000000000004,"15min":0.622},"min":{"1min":0.36099999999999999,"5min":0.314,"15min":0.314},"max":{"1min":1.1160000000000001,"5min":1.1819999999999999,"15min":1.1819999999999999},"last":0.58899999999999997},{"interface":"front","average":{"1min":0.621,"5min":0.68200000000000005,"15min":0.63200000000000001},"min":{"1min":0.34399999999999997,"5min":0.34399999999999997,"15min":0.30199999999999999},"max":{"1min":0.872,"5min":3.004,"15min":3.004},"last":0.68999999999999995}]},{"osd":1,"last update":"Sat Jun 3 19:35:42 2023","interfaces":[{"interface":"back","average":{"1min":0.64300000000000002,"5min":0.70499999999999996,"15min":0.64700000000000002},"min":{"1min":0.375,"5min":0.375,"15min":0.375},"max":{"1min":1.0109999999999999,"5min":1.1339999999999999,"15min":1.139},"last":0.63700000000000001},{"interface":"front","average":{"1min":0.69099999999999995,"5min":0.75800000000000001,"15min":0.68300000000000005},"min":{"1min":0.38900000000000001,"5min":0.31,"15min":0.31},"max":{"1min":1.036,"5min":1.1619999999999999,"15min":1.1619999999999999},"last":0.91100000000000003}]},{"osd":3,"last update":"Sat Jun 3 19:35:32 2023","interfaces":[{"interface":"back","average":{"1min":0.72599999999999998,"5min":0.749,"15min":0.746},"min":{"1min":0.49299999999999999,"5min":0.33800000000000002,"15min":0.33800000000000002},"max":{"1min":0.91900000000000004,"5min":1.323,"15min":1.323},"last":0.746},{"interface":"front","average":{"1min":0.78400000000000003,"5min":0.81000000000000005,"15min":0.77800000000000002},"min":{"1min":0.45700000000000002,"5min":0.36299999999999999,"15min":0.34399999999999997},"max":{"1min":1.149,"5min":1.149,"15min":1.5840000000000001},"last":0.81899999999999995}]},{"osd":4,"last update":"Sat Jun 3 19:35:09 2023","interfaces":[{"interface":"back","average":{"1min":0.96999999999999997,"5min":0.96299999999999997,"15min":0.93799999999999994},"min":{"1min":0.72399999999999998,"5min":0.61199999999999999,"15min":0.61199999999999999},"max":{"1min":1.2430000000000001,"5min":1.554,"15min":1.554},"last":1.0209999999999999},{"interface":"front","average":{"1min":0.98399999999999999,"5min":1.093,"15min":1.014},"min":{"1min":0.52900000000000003,"5min":0.52900000000000003,"15min":0.52900000000000003},"max":{"1min":1.288,"5min":8.3829999999999991,"15min":8.3829999999999991},"last":1.0089999999999999}]},{"osd":5,"last update":"Sat Jun 3 19:35:53 2023","interfaces":[{"interface":"back","average":{"1min":0.96999999999999997,"5min":1.042,"15min":1.077},"min":{"1min":0.68200000000000005,"5min":0.59899999999999998,"15min":0.59899999999999998},"max":{"1min":1.2549999999999999,"5min":1.6479999999999999,"15min":1.7150000000000001},"last":1.0529999999999999},{"interface":"front","average":{"1min":0.96299999999999997,"5min":0.96999999999999997,"15min":0.97199999999999998},"min":{"1min":0.67300000000000004,"5min":0.59699999999999998,"15min":0.59699999999999998},"max":{"1min":1.4339999999999999,"5min":1.4339999999999999,"15min":1.4339999999999999},"last":1.175}]},{"osd":6,"last update":"Sat Jun 3 19:35:36 2023","interfaces":[{"interface":"back","average":{"1min":1.0720000000000001,"5min":1.044,"15min":1.0529999999999999},"min":{"1min":0.65500000000000003,"5min":0.65500000000000003,"15min":0.65500000000000003},"max":{"1min":1.446,"5min":1.502,"15min":1.502},"last":0.96299999999999997},{"interface":"front","average":{"1min":1.0620000000000001,"5min":1.099,"15min":1.137},"min":{"1min":0.64700000000000002,"5min":0.58799999999999997,"15min":0.58799999999999997},"max":{"1min":1.4179999999999999,"5min":1.5940000000000001,"15min":1.5940000000000001},"last":1.093}]},{"osd":7,"last update":"Sat Jun 3 19:35:22 2023","interfaces":[{"interface":"back","average":{"1min":1.0189999999999999,"5min":1.05,"15min":1.0900000000000001},"min":{"1min":0.65800000000000003,"5min":0.65600000000000003,"15min":0.65600000000000003},"max":{"1min":1.347,"5min":1.486,"15min":1.486},"last":1.282},{"interface":"front","average":{"1min":1.103,"5min":1.1080000000000001,"15min":1.137},"min":{"1min":0.81499999999999995,"5min":0.67400000000000004,"15min":0.67400000000000004},"max":{"1min":1.431,"5min":1.5600000000000001,"15min":1.5600000000000001},"last":1.26}]}]},{"osd":3,"up_from":25,"seq":107374182497,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":5808,"kb_used_data":616,"kb_used_omap":0,"kb_used_meta":5184,"kb_avail":93739344,"statfs":{"total":95995035648,"available":95989088256,"internally_reserved":0,"allocated":630784,"data_stored":270026,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5308416},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 3 19:35:24 2023","interfaces":[{"interface":"back","average":{"1min":0.64500000000000002,"5min":0.66900000000000004,"15min":0.67100000000000004},"min":{"1min":0.47699999999999998,"5min":0.314,"15min":0.314},"max":{"1min":0.95599999999999996,"5min":1.153,"15min":1.153},"last":0.66700000000000004},{"interface":"front","average":{"1min":0.68200000000000005,"5min":0.73199999999999998,"15min":0.73199999999999998},"min":{"1min":0.376,"5min":0.34499999999999997,"15min":0.34499999999999997},"max":{"1min":1.1419999999999999,"5min":1.8979999999999999,"15min":1.8979999999999999},"last":0.746}]},{"osd":1,"last update":"Sat Jun 3 19:35:24 2023","interfaces":[{"interface":"back","average":{"1min":0.72199999999999998,"5min":0.747,"15min":0.72399999999999998},"min":{"1min":0.48199999999999998,"5min":0.41799999999999998,"15min":0.41799999999999998},"max":{"1min":1.1919999999999999,"5min":1.218,"15min":1.218},"last":0.84499999999999997},{"interface":"front","average":{"1min":0.67600000000000005,"5min":0.72499999999999998,"15min":0.72299999999999998},"min":{"1min":0.38900000000000001,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":0.91000000000000003,"5min":1.109,"15min":1.109},"last":0.58599999999999997}]},{"osd":2,"last update":"Sat Jun 3 19:35:24 2023","interfaces":[{"interface":"back","average":{"1min":0.76300000000000001,"5min":0.77700000000000002,"15min":0.77100000000000002},"min":{"1min":0.47299999999999998,"5min":0.38300000000000001,"15min":0.38300000000000001},"max":{"1min":1.2749999999999999,"5min":1.359,"15min":1.359},"last":0.77500000000000002},{"interface":"front","average":{"1min":0.81499999999999995,"5min":0.76100000000000001,"15min":0.755},"min":{"1min":0.46999999999999997,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":1.5800000000000001,"5min":1.5800000000000001,"15min":1.5800000000000001},"last":0.54900000000000004}]},{"osd":4,"last update":"Sat Jun 3 19:35:09 2023","interfaces":[{"interface":"back","average":{"1min":0.90300000000000002,"5min":0.93999999999999995,"15min":0.94799999999999995},"min":{"1min":0.68400000000000005,"5min":0.59499999999999997,"15min":0.59499999999999997},"max":{"1min":1.2569999999999999,"5min":1.393,"15min":1.393},"last":0.90100000000000002},{"interface":"front","average":{"1min":0.96699999999999997,"5min":0.98999999999999999,"15min":0.98599999999999999},"min":{"1min":0.627,"5min":0.627,"15min":0.627},"max":{"1min":1.399,"5min":1.528,"15min":1.528},"last":0.97099999999999997}]},{"osd":5,"last update":"Sat Jun 3 19:35:56 2023","interfaces":[{"interface":"back","average":{"1min":0.998,"5min":1.0049999999999999,"15min":1.048},"min":{"1min":0.66500000000000004,"5min":0.66500000000000004,"15min":0.66500000000000004},"max":{"1min":1.921,"5min":1.921,"15min":1.921},"last":1.0049999999999999},{"interface":"front","average":{"1min":0.997,"5min":1.0189999999999999,"15min":1.0649999999999999},"min":{"1min":0.70699999999999996,"5min":0.68400000000000005,"15min":0.68400000000000005},"max":{"1min":1.984,"5min":1.984,"15min":1.984},"last":1.1000000000000001}]},{"osd":6,"last update":"Sat Jun 3 19:35:38 2023","interfaces":[{"interface":"back","average":{"1min":1.0449999999999999,"5min":1.0509999999999999,"15min":1.099},"min":{"1min":0.60899999999999999,"5min":0.60899999999999999,"15min":0.60899999999999999},"max":{"1min":1.9630000000000001,"5min":1.9630000000000001,"15min":1.9630000000000001},"last":0.94399999999999995},{"interface":"front","average":{"1min":1.0600000000000001,"5min":1.046,"15min":1.0960000000000001},"min":{"1min":0.70999999999999996,"5min":0.65400000000000003,"15min":0.65400000000000003},"max":{"1min":1.5660000000000001,"5min":1.5660000000000001,"15min":1.5660000000000001},"last":0.91000000000000003}]},{"osd":7,"last update":"Sat Jun 3 19:35:24 2023","interfaces":[{"interface":"back","average":{"1min":1.0720000000000001,"5min":1.0660000000000001,"15min":1.077},"min":{"1min":0.76400000000000001,"5min":0.66800000000000004,"15min":0.66800000000000004},"max":{"1min":1.6679999999999999,"5min":1.6679999999999999,"15min":1.6679999999999999},"last":0.82099999999999995},{"interface":"front","average":{"1min":1.0820000000000001,"5min":1.0860000000000001,"15min":1.0880000000000001},"min":{"1min":0.66700000000000004,"5min":0.64900000000000002,"15min":0.64900000000000002},"max":{"1min":1.5529999999999999,"5min":1.5529999999999999,"15min":1.5529999999999999},"last":1.04}]}]},{"osd":4,"up_from":30,"seq":128849018969,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":5828,"kb_used_data":636,"kb_used_omap":0,"kb_used_meta":5184,"kb_avail":93739324,"statfs":{"total":95995035648,"available":95989067776,"internally_reserved":0,"allocated":651264,"data_stored":271618,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5308416},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 3 19:35:09 2023","interfaces":[{"interface":"back","average":{"1min":0.878,"5min":0.86299999999999999,"15min":0.86599999999999999},"min":{"1min":0.51600000000000001,"5min":0.51600000000000001,"15min":0.51600000000000001},"max":{"1min":1.1970000000000001,"5min":1.264,"15min":1.264},"last":1.036},{"interface":"front","average":{"1min":0.93000000000000005,"5min":0.871,"15min":0.877},"min":{"1min":0.61199999999999999,"5min":0.46100000000000002,"15min":0.46100000000000002},"max":{"1min":1.331,"5min":1.331,"15min":1.331},"last":1.0229999999999999}]},{"osd":1,"last update":"Sat Jun 3 19:35:09 2023","interfaces":[{"interface":"back","average":{"1min":0.92300000000000004,"5min":0.876,"15min":0.86899999999999999},"min":{"1min":0.65700000000000003,"5min":0.502,"15min":0.502},"max":{"1min":1.286,"5min":1.286,"15min":1.286},"last":0.79600000000000004},{"interface":"front","average":{"1min":0.87,"5min":0.875,"15min":0.85699999999999998},"min":{"1min":0.57899999999999996,"5min":0.47599999999999998,"15min":0.47599999999999998},"max":{"1min":1.2769999999999999,"5min":1.2769999999999999,"15min":1.2769999999999999},"last":1.1850000000000001}]},{"osd":2,"last update":"Sat Jun 3 19:35:09 2023","interfaces":[{"interface":"back","average":{"1min":0.98999999999999999,"5min":0.91200000000000003,"15min":0.89000000000000001},"min":{"1min":0.61899999999999999,"5min":0.61899999999999999,"15min":0.61899999999999999},"max":{"1min":1.1839999999999999,"5min":1.2190000000000001,"15min":1.2190000000000001},"last":1.1200000000000001},{"interface":"front","average":{"1min":0.95599999999999996,"5min":0.89900000000000002,"15min":0.89300000000000002},"min":{"1min":0.65400000000000003,"5min":0.60299999999999998,"15min":0.60299999999999998},"max":{"1min":1.2310000000000001,"5min":1.284,"15min":1.284},"last":0.89100000000000001}]},{"osd":3,"last update":"Sat Jun 3 19:35:09 2023","interfaces":[{"interface":"back","average":{"1min":0.95799999999999996,"5min":0.89000000000000001,"15min":0.879},"min":{"1min":0.749,"5min":0.56699999999999995,"15min":0.56699999999999995},"max":{"1min":1.2150000000000001,"5min":1.345,"15min":1.345},"last":0.75800000000000001},{"interface":"front","average":{"1min":1.0429999999999999,"5min":0.94299999999999995,"15min":0.94299999999999995},"min":{"1min":0.747,"5min":0.58599999999999997,"15min":0.58599999999999997},"max":{"1min":1.371,"5min":1.371,"15min":1.371},"last":0.751}]},{"osd":5,"last update":"Sat Jun 3 19:35:49 2023","interfaces":[{"interface":"back","average":{"1min":0.95499999999999996,"5min":0.88900000000000001,"15min":0.79500000000000004},"min":{"1min":0.53300000000000003,"5min":0.498,"15min":0.45500000000000002},"max":{"1min":1.196,"5min":1.46,"15min":1.46},"last":0.72899999999999998},{"interface":"front","average":{"1min":0.97499999999999998,"5min":0.90200000000000002,"15min":0.84599999999999997},"min":{"1min":0.63400000000000001,"5min":0.45500000000000002,"15min":0.45500000000000002},"max":{"1min":1.1930000000000001,"5min":1.3580000000000001,"15min":1.3580000000000001},"last":0.91700000000000004}]},{"osd":6,"last update":"Sat Jun 3 19:35:38 2023","interfaces":[{"interface":"back","average":{"1min":1,"5min":0.93400000000000005,"15min":0.93999999999999995},"min":{"1min":0.66400000000000003,"5min":0.46899999999999997,"15min":0.46899999999999997},"max":{"1min":1.2330000000000001,"5min":1.413,"15min":1.413},"last":0.96899999999999997},{"interface":"front","average":{"1min":1.044,"5min":0.96999999999999997,"15min":0.95199999999999996},"min":{"1min":0.83199999999999996,"5min":0.503,"15min":0.503},"max":{"1min":1.3160000000000001,"5min":1.3160000000000001,"15min":1.3160000000000001},"last":0.68100000000000005}]},{"osd":7,"last update":"Sat Jun 3 19:35:22 2023","interfaces":[{"interface":"back","average":{"1min":1.196,"5min":1.012,"15min":0.92200000000000004},"min":{"1min":0.97099999999999997,"5min":0.47099999999999997,"15min":0.47099999999999997},"max":{"1min":1.401,"5min":1.401,"15min":1.401},"last":0.82699999999999996},{"interface":"front","average":{"1min":1.1890000000000001,"5min":1.0409999999999999,"15min":0.95099999999999996},"min":{"1min":0.94899999999999995,"5min":0.52300000000000002,"15min":0.52300000000000002},"max":{"1min":1.357,"5min":1.712,"15min":1.712},"last":1.0620000000000001}]}]},{"osd":5,"up_from":36,"seq":154618822736,"num_pgs":21,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":5680,"kb_used_data":616,"kb_used_omap":0,"kb_used_meta":5056,"kb_avail":93739472,"statfs":{"total":95995035648,"available":95989219328,"internally_reserved":0,"allocated":630784,"data_stored":270245,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5177344},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 3 19:35:54 2023","interfaces":[{"interface":"back","average":{"1min":0.78400000000000003,"5min":0.85699999999999998,"15min":0.877},"min":{"1min":0.51100000000000001,"5min":0.51100000000000001,"15min":0.51100000000000001},"max":{"1min":1.0469999999999999,"5min":1.4750000000000001,"15min":1.4750000000000001},"last":1.0669999999999999},{"interface":"front","average":{"1min":0.92100000000000004,"5min":0.88400000000000001,"15min":0.85199999999999998},"min":{"1min":0.621,"5min":0.55200000000000005,"15min":0.55200000000000005},"max":{"1min":1.236,"5min":1.9259999999999999,"15min":1.9259999999999999},"last":0.77400000000000002}]},{"osd":1,"last update":"Sat Jun 3 19:35:54 2023","interfaces":[{"interface":"back","average":{"1min":0.83599999999999997,"5min":0.88900000000000001,"15min":0.88900000000000001},"min":{"1min":0.59299999999999997,"5min":0.49399999999999999,"15min":0.49399999999999999},"max":{"1min":1.143,"5min":1.994,"15min":1.994},"last":1.147},{"interface":"front","average":{"1min":0.85099999999999998,"5min":0.90800000000000003,"15min":0.872},"min":{"1min":0.60699999999999998,"5min":0.55400000000000005,"15min":0.55400000000000005},"max":{"1min":1.256,"5min":1.742,"15min":1.742},"last":0.97799999999999998}]},{"osd":2,"last update":"Sat Jun 3 19:35:54 2023","interfaces":[{"interface":"back","average":{"1min":0.82199999999999995,"5min":0.872,"15min":0.87},"min":{"1min":0.57399999999999995,"5min":0.54200000000000004,"15min":0.54200000000000004},"max":{"1min":1.145,"5min":1.708,"15min":1.708},"last":0.95199999999999996},{"interface":"front","average":{"1min":0.92100000000000004,"5min":0.93300000000000005,"15min":0.90100000000000002},"min":{"1min":0.65200000000000002,"5min":0.56000000000000005,"15min":0.56000000000000005},"max":{"1min":1.2569999999999999,"5min":2.0840000000000001,"15min":2.0840000000000001},"last":0.88500000000000001}]},{"osd":3,"last update":"Sat Jun 3 19:35:54 2023","interfaces":[{"interface":"back","average":{"1min":0.94299999999999995,"5min":0.93500000000000005,"15min":0.95299999999999996},"min":{"1min":0.69899999999999995,"5min":0.58199999999999996,"15min":0.58199999999999996},"max":{"1min":1.202,"5min":2.1699999999999999,"15min":2.1699999999999999},"last":0.98999999999999999},{"interface":"front","average":{"1min":0.877,"5min":0.92600000000000005,"15min":0.879},"min":{"1min":0.72299999999999998,"5min":0.57899999999999996,"15min":0.57899999999999996},"max":{"1min":1.0129999999999999,"5min":1.6459999999999999,"15min":1.6459999999999999},"last":1.0429999999999999}]},{"osd":4,"last update":"Sat Jun 3 19:35:54 2023","interfaces":[{"interface":"back","average":{"1min":0.98299999999999998,"5min":0.90300000000000002,"15min":0.80300000000000005},"min":{"1min":0.70299999999999996,"5min":0.39100000000000001,"15min":0.39100000000000001},"max":{"1min":1.264,"5min":2.1299999999999999,"15min":2.1299999999999999},"last":0.68200000000000005},{"interface":"front","average":{"1min":1.044,"5min":0.96099999999999997,"15min":0.80500000000000005},"min":{"1min":0.69299999999999995,"5min":0.48499999999999999,"15min":0.38},"max":{"1min":1.214,"5min":2.0830000000000002,"15min":2.0830000000000002},"last":0.91800000000000004}]},{"osd":6,"last update":"Sat Jun 3 19:35:38 2023","interfaces":[{"interface":"back","average":{"1min":1.0900000000000001,"5min":0.98299999999999998,"15min":0.92700000000000005},"min":{"1min":0.68899999999999995,"5min":0.46500000000000002,"15min":0.46500000000000002},"max":{"1min":1.337,"5min":1.6599999999999999,"15min":1.6599999999999999},"last":0.83999999999999997},{"interface":"front","average":{"1min":1.1830000000000001,"5min":1.0720000000000001,"15min":1},"min":{"1min":0.96499999999999997,"5min":0.56200000000000006,"15min":0.56200000000000006},"max":{"1min":1.4630000000000001,"5min":1.8380000000000001,"15min":1.8380000000000001},"last":1.022}]},{"osd":7,"last update":"Sat Jun 3 19:35:22 2023","interfaces":[{"interface":"back","average":{"1min":1.194,"5min":1.0960000000000001,"15min":1.0700000000000001},"min":{"1min":0.84099999999999997,"5min":0.63200000000000001,"15min":0.63200000000000001},"max":{"1min":1.417,"5min":2.004,"15min":2.004},"last":1.103},{"interface":"front","average":{"1min":1.0760000000000001,"5min":0.995,"15min":0.96499999999999997},"min":{"1min":0.76800000000000002,"5min":0.48099999999999998,"15min":0.48099999999999998},"max":{"1min":1.29,"5min":1.6479999999999999,"15min":1.6479999999999999},"last":0.85799999999999998}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":1682,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":1606,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":1752,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-06-03T19:36:05.838 INFO:tasks.cephadm.ceph_manager.ceph:clean! 2023-06-03T19:36:05.839 INFO:tasks.ceph:Waiting until ceph cluster ceph is healthy... 2023-06-03T19:36:05.839 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy 2023-06-03T19:36:05.839 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph health --format=json 2023-06-03T19:36:06.032 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:36:05 smithi089 bash[33173]: cluster 2023-06-03T19:36:04.451984+0000 mgr.y (mgr.14445) 162 : cluster [DBG] pgmap v142: 73 pgs: 73 active+clean; 2.3 KiB data, 45 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:36:06.032 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:36:05 smithi089 bash[26572]: cluster 2023-06-03T19:36:04.451984+0000 mgr.y (mgr.14445) 162 : cluster [DBG] pgmap v142: 73 pgs: 73 active+clean; 2.3 KiB data, 45 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:36:06.087 INFO:journalctl@ceph.mgr.x.smithi164.stdout:Jun 03 19:36:05 smithi164 bash[27934]: ::ffff:172.21.15.164 - - [03/Jun/2023:19:36:05] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2023-06-03T19:36:06.087 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:36:05 smithi164 bash[26855]: cluster 2023-06-03T19:36:04.451984+0000 mgr.y (mgr.14445) 162 : cluster [DBG] pgmap v142: 73 pgs: 73 active+clean; 2.3 KiB data, 45 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:36:07.032 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:36:06 smithi089 bash[26572]: audit 2023-06-03T19:36:04.990208+0000 mgr.y (mgr.14445) 163 : audit [DBG] from='client.24695 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-06-03T19:36:07.032 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:36:06 smithi089 bash[33173]: audit 2023-06-03T19:36:04.990208+0000 mgr.y (mgr.14445) 163 : audit [DBG] from='client.24695 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-06-03T19:36:07.087 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:36:06 smithi164 bash[26855]: audit 2023-06-03T19:36:04.990208+0000 mgr.y (mgr.14445) 163 : audit [DBG] from='client.24695 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-06-03T19:36:07.908 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:36:07 smithi089 bash[26572]: cluster 2023-06-03T19:36:06.452632+0000 mgr.y (mgr.14445) 164 : cluster [DBG] pgmap v143: 73 pgs: 73 active+clean; 2.3 KiB data, 45 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:36:07.909 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:36:07 smithi089 bash[33173]: cluster 2023-06-03T19:36:06.452632+0000 mgr.y (mgr.14445) 164 : cluster [DBG] pgmap v143: 73 pgs: 73 active+clean; 2.3 KiB data, 45 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:36:08.087 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:36:07 smithi164 bash[26855]: cluster 2023-06-03T19:36:06.452632+0000 mgr.y (mgr.14445) 164 : cluster [DBG] pgmap v143: 73 pgs: 73 active+clean; 2.3 KiB data, 45 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:36:08.281 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 19:36:07 smithi089 bash[26845]: ::ffff:172.21.15.164 - - [03/Jun/2023:19:36:07] "GET /metrics HTTP/1.1" 200 187881 "" "Prometheus/2.33.4" 2023-06-03T19:36:10.032 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:36:09 smithi089 bash[26572]: cluster 2023-06-03T19:36:08.453275+0000 mgr.y (mgr.14445) 165 : cluster [DBG] pgmap v144: 73 pgs: 73 active+clean; 2.3 KiB data, 45 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:36:10.032 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:36:09 smithi089 bash[33173]: cluster 2023-06-03T19:36:08.453275+0000 mgr.y (mgr.14445) 165 : cluster [DBG] pgmap v144: 73 pgs: 73 active+clean; 2.3 KiB data, 45 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:36:10.087 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:36:09 smithi164 bash[26855]: cluster 2023-06-03T19:36:08.453275+0000 mgr.y (mgr.14445) 165 : cluster [DBG] pgmap v144: 73 pgs: 73 active+clean; 2.3 KiB data, 45 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:36:10.690 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/mon.a/config 2023-06-03T19:36:11.959 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:36:11 smithi089 bash[33173]: cluster 2023-06-03T19:36:10.454272+0000 mgr.y (mgr.14445) 166 : cluster [DBG] pgmap v145: 73 pgs: 73 active+clean; 2.3 KiB data, 45 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:36:11.959 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:36:11 smithi089 bash[26572]: cluster 2023-06-03T19:36:10.454272+0000 mgr.y (mgr.14445) 166 : cluster [DBG] pgmap v145: 73 pgs: 73 active+clean; 2.3 KiB data, 45 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:36:12.087 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:36:11 smithi164 bash[26855]: cluster 2023-06-03T19:36:10.454272+0000 mgr.y (mgr.14445) 166 : cluster [DBG] pgmap v145: 73 pgs: 73 active+clean; 2.3 KiB data, 45 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:36:14.032 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:36:13 smithi089 bash[33173]: cluster 2023-06-03T19:36:12.454945+0000 mgr.y (mgr.14445) 167 : cluster [DBG] pgmap v146: 73 pgs: 73 active+clean; 2.3 KiB data, 45 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:36:14.032 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:36:13 smithi089 bash[26572]: cluster 2023-06-03T19:36:12.454945+0000 mgr.y (mgr.14445) 167 : cluster [DBG] pgmap v146: 73 pgs: 73 active+clean; 2.3 KiB data, 45 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:36:14.087 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:36:13 smithi164 bash[26855]: cluster 2023-06-03T19:36:12.454945+0000 mgr.y (mgr.14445) 167 : cluster [DBG] pgmap v146: 73 pgs: 73 active+clean; 2.3 KiB data, 45 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:36:14.168 INFO:teuthology.orchestra.run.smithi089.stdout: 2023-06-03T19:36:14.957 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:36:14 smithi089 bash[33173]: audit 2023-06-03T19:36:14.160286+0000 mon.a (mon.0) 730 : audit [DBG] from='client.? 172.21.15.89:0/3040872429' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2023-06-03T19:36:14.958 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:36:14 smithi089 bash[26572]: audit 2023-06-03T19:36:14.160286+0000 mon.a (mon.0) 730 : audit [DBG] from='client.? 172.21.15.89:0/3040872429' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2023-06-03T19:36:14.958 INFO:teuthology.orchestra.run.smithi089.stdout:{"status":"HEALTH_OK","checks":{},"mutes":[]} 2023-06-03T19:36:14.959 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy done 2023-06-03T19:36:14.959 INFO:teuthology.run_tasks:Running task print... 2023-06-03T19:36:14.968 INFO:teuthology.task.print:**** upgrading first half of cluster, with stress **** 2023-06-03T19:36:14.969 INFO:teuthology.run_tasks:Running task parallel... 2023-06-03T19:36:15.030 INFO:teuthology.task.parallel:starting parallel... 2023-06-03T19:36:15.030 INFO:teuthology.task.parallel:In parallel, running task sequential... 2023-06-03T19:36:15.031 INFO:teuthology.task.sequential:In sequential, running task full_sequential... 2023-06-03T19:36:15.032 INFO:teuthology.task.full_sequential:In full_sequential, running task radosbench... 2023-06-03T19:36:15.033 INFO:tasks.radosbench:Beginning radosbench... 2023-06-03T19:36:15.034 INFO:tasks.cephadm.ceph_manager.ceph:creating pool_name unique_pool_0 2023-06-03T19:36:15.034 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph osd pool create unique_pool_0 16 2023-06-03T19:36:15.035 INFO:teuthology.task.parallel:In parallel, running task sequential... 2023-06-03T19:36:15.035 INFO:teuthology.task.sequential:In sequential, running task cephadm.shell... 2023-06-03T19:36:15.035 INFO:tasks.cephadm:Running commands on role mon.a host ubuntu@smithi089.front.sepia.ceph.com 2023-06-03T19:36:15.036 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -e sha1=8e24bcf462a1d94c2e12aa4e26419634883a1e88 -- bash -c 'ceph config set mgr mgr/cephadm/daemon_cache_timeout 60' 2023-06-03T19:36:15.087 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:36:14 smithi164 bash[26855]: audit 2023-06-03T19:36:14.160286+0000 mon.a (mon.0) 730 : audit [DBG] from='client.? 172.21.15.89:0/3040872429' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2023-06-03T19:36:16.032 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:36:15 smithi089 bash[26572]: cluster 2023-06-03T19:36:14.455505+0000 mgr.y (mgr.14445) 168 : cluster [DBG] pgmap v147: 73 pgs: 73 active+clean; 2.3 KiB data, 45 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:36:16.032 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:36:15 smithi089 bash[33173]: cluster 2023-06-03T19:36:14.455505+0000 mgr.y (mgr.14445) 168 : cluster [DBG] pgmap v147: 73 pgs: 73 active+clean; 2.3 KiB data, 45 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:36:16.087 INFO:journalctl@ceph.mgr.x.smithi164.stdout:Jun 03 19:36:15 smithi164 bash[27934]: ::ffff:172.21.15.164 - - [03/Jun/2023:19:36:15] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2023-06-03T19:36:16.087 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:36:15 smithi164 bash[26855]: cluster 2023-06-03T19:36:14.455505+0000 mgr.y (mgr.14445) 168 : cluster [DBG] pgmap v147: 73 pgs: 73 active+clean; 2.3 KiB data, 45 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:36:18.032 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 19:36:17 smithi089 bash[26845]: ::ffff:172.21.15.164 - - [03/Jun/2023:19:36:17] "GET /metrics HTTP/1.1" 200 187881 "" "Prometheus/2.33.4" 2023-06-03T19:36:18.032 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:36:17 smithi089 bash[33173]: cluster 2023-06-03T19:36:16.456120+0000 mgr.y (mgr.14445) 169 : cluster [DBG] pgmap v148: 73 pgs: 73 active+clean; 2.3 KiB data, 45 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:36:18.033 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:36:17 smithi089 bash[26572]: cluster 2023-06-03T19:36:16.456120+0000 mgr.y (mgr.14445) 169 : cluster [DBG] pgmap v148: 73 pgs: 73 active+clean; 2.3 KiB data, 45 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:36:18.087 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:36:17 smithi164 bash[26855]: cluster 2023-06-03T19:36:16.456120+0000 mgr.y (mgr.14445) 169 : cluster [DBG] pgmap v148: 73 pgs: 73 active+clean; 2.3 KiB data, 45 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:36:19.032 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 19:36:18 smithi089 bash[26845]: [03/Jun/2023:19:36:18] ENGINE Bus STOPPING 2023-06-03T19:36:19.032 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 19:36:18 smithi089 bash[26845]: [03/Jun/2023:19:36:18] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2023-06-03T19:36:19.033 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 19:36:18 smithi089 bash[26845]: [03/Jun/2023:19:36:18] ENGINE Bus STOPPED 2023-06-03T19:36:19.033 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 19:36:18 smithi089 bash[26845]: [03/Jun/2023:19:36:18] ENGINE Bus STARTING 2023-06-03T19:36:19.033 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 19:36:18 smithi089 bash[26845]: [03/Jun/2023:19:36:18] ENGINE Serving on http://:::9283 2023-06-03T19:36:19.033 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 19:36:18 smithi089 bash[26845]: [03/Jun/2023:19:36:18] ENGINE Bus STARTED 2023-06-03T19:36:19.545 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -e sha1=8e24bcf462a1d94c2e12aa4e26419634883a1e88 -- bash -c 'ceph config set global log_to_journald false --force' 2023-06-03T19:36:20.032 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:36:19 smithi089 bash[26572]: cluster 2023-06-03T19:36:18.456759+0000 mgr.y (mgr.14445) 170 : cluster [DBG] pgmap v149: 73 pgs: 73 active+clean; 2.3 KiB data, 45 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:36:20.032 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:36:19 smithi089 bash[26572]: audit 2023-06-03T19:36:18.761597+0000 mon.a (mon.0) 731 : audit [INF] from='client.? 172.21.15.89:0/3650841790' entity='client.admin' 2023-06-03T19:36:20.032 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:36:19 smithi089 bash[26572]: audit 2023-06-03T19:36:18.776901+0000 mon.a (mon.0) 732 : audit [DBG] from='mgr.14445 172.21.15.89:0/573502486' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-03T19:36:20.032 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:36:19 smithi089 bash[33173]: cluster 2023-06-03T19:36:18.456759+0000 mgr.y (mgr.14445) 170 : cluster [DBG] pgmap v149: 73 pgs: 73 active+clean; 2.3 KiB data, 45 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:36:20.032 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:36:19 smithi089 bash[33173]: audit 2023-06-03T19:36:18.761597+0000 mon.a (mon.0) 731 : audit [INF] from='client.? 172.21.15.89:0/3650841790' entity='client.admin' 2023-06-03T19:36:20.033 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:36:19 smithi089 bash[33173]: audit 2023-06-03T19:36:18.776901+0000 mon.a (mon.0) 732 : audit [DBG] from='mgr.14445 172.21.15.89:0/573502486' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-03T19:36:20.033 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/mon.a/config 2023-06-03T19:36:20.087 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:36:19 smithi164 bash[26855]: cluster 2023-06-03T19:36:18.456759+0000 mgr.y (mgr.14445) 170 : cluster [DBG] pgmap v149: 73 pgs: 73 active+clean; 2.3 KiB data, 45 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:36:20.087 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:36:19 smithi164 bash[26855]: audit 2023-06-03T19:36:18.761597+0000 mon.a (mon.0) 731 : audit [INF] from='client.? 172.21.15.89:0/3650841790' entity='client.admin' 2023-06-03T19:36:20.087 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:36:19 smithi164 bash[26855]: audit 2023-06-03T19:36:18.776901+0000 mon.a (mon.0) 732 : audit [DBG] from='mgr.14445 172.21.15.89:0/573502486' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-03T19:36:21.968 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:36:21 smithi089 bash[33173]: cluster 2023-06-03T19:36:20.457376+0000 mgr.y (mgr.14445) 171 : cluster [DBG] pgmap v150: 73 pgs: 73 active+clean; 2.3 KiB data, 45 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:36:21.968 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:36:21 smithi089 bash[26572]: cluster 2023-06-03T19:36:20.457376+0000 mgr.y (mgr.14445) 171 : cluster [DBG] pgmap v150: 73 pgs: 73 active+clean; 2.3 KiB data, 45 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:36:22.087 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:36:21 smithi164 bash[26855]: cluster 2023-06-03T19:36:20.457376+0000 mgr.y (mgr.14445) 171 : cluster [DBG] pgmap v150: 73 pgs: 73 active+clean; 2.3 KiB data, 45 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:36:24.032 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:36:23 smithi089 bash[26572]: cluster 2023-06-03T19:36:22.457847+0000 mgr.y (mgr.14445) 172 : cluster [DBG] pgmap v151: 73 pgs: 73 active+clean; 2.3 KiB data, 45 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:36:24.032 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:36:23 smithi089 bash[33173]: cluster 2023-06-03T19:36:22.457847+0000 mgr.y (mgr.14445) 172 : cluster [DBG] pgmap v151: 73 pgs: 73 active+clean; 2.3 KiB data, 45 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:36:24.087 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:36:23 smithi164 bash[26855]: cluster 2023-06-03T19:36:22.457847+0000 mgr.y (mgr.14445) 172 : cluster [DBG] pgmap v151: 73 pgs: 73 active+clean; 2.3 KiB data, 45 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:36:25.532 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 19:36:25 smithi089 bash[26845]: [03/Jun/2023:19:36:25] ENGINE Bus STOPPING 2023-06-03T19:36:25.532 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 19:36:25 smithi089 bash[26845]: [03/Jun/2023:19:36:25] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2023-06-03T19:36:25.532 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 19:36:25 smithi089 bash[26845]: [03/Jun/2023:19:36:25] ENGINE Bus STOPPED 2023-06-03T19:36:25.532 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 19:36:25 smithi089 bash[26845]: [03/Jun/2023:19:36:25] ENGINE Bus STARTING 2023-06-03T19:36:25.532 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 19:36:25 smithi089 bash[26845]: [03/Jun/2023:19:36:25] ENGINE Serving on http://:::9283 2023-06-03T19:36:25.533 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 19:36:25 smithi089 bash[26845]: [03/Jun/2023:19:36:25] ENGINE Bus STARTED 2023-06-03T19:36:25.989 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -e sha1=8e24bcf462a1d94c2e12aa4e26419634883a1e88 -- bash -c 'ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1' 2023-06-03T19:36:26.087 INFO:journalctl@ceph.mgr.x.smithi164.stdout:Jun 03 19:36:25 smithi164 bash[27934]: ::ffff:172.21.15.164 - - [03/Jun/2023:19:36:25] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2023-06-03T19:36:26.087 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:36:25 smithi164 bash[26855]: cluster 2023-06-03T19:36:24.458651+0000 mgr.y (mgr.14445) 173 : cluster [DBG] pgmap v152: 73 pgs: 73 active+clean; 2.3 KiB data, 45 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:36:26.087 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:36:25 smithi164 bash[26855]: audit 2023-06-03T19:36:24.786122+0000 mon.a (mon.0) 733 : audit [INF] from='mgr.14445 172.21.15.89:0/573502486' entity='mgr.y' 2023-06-03T19:36:26.088 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:36:25 smithi164 bash[26855]: audit 2023-06-03T19:36:24.829337+0000 mon.a (mon.0) 734 : audit [INF] from='mgr.14445 172.21.15.89:0/573502486' entity='mgr.y' 2023-06-03T19:36:26.088 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:36:25 smithi164 bash[26855]: audit 2023-06-03T19:36:24.830448+0000 mon.a (mon.0) 735 : audit [DBG] from='mgr.14445 172.21.15.89:0/573502486' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-03T19:36:26.088 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:36:25 smithi164 bash[26855]: audit 2023-06-03T19:36:24.831337+0000 mon.a (mon.0) 736 : audit [INF] from='mgr.14445 172.21.15.89:0/573502486' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-03T19:36:26.089 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:36:25 smithi164 bash[26855]: audit 2023-06-03T19:36:24.837805+0000 mon.a (mon.0) 737 : audit [INF] from='mgr.14445 172.21.15.89:0/573502486' entity='mgr.y' 2023-06-03T19:36:26.089 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:36:25 smithi164 bash[26855]: audit 2023-06-03T19:36:25.160813+0000 mon.a (mon.0) 738 : audit [INF] from='client.? ' entity='client.admin' 2023-06-03T19:36:26.089 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:36:25 smithi164 bash[26855]: audit 2023-06-03T19:36:25.210180+0000 mon.a (mon.0) 739 : audit [DBG] from='mgr.14445 172.21.15.89:0/573502486' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-03T19:36:26.089 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:36:25 smithi164 bash[26855]: audit 2023-06-03T19:36:25.213769+0000 mon.a (mon.0) 740 : audit [DBG] from='mgr.14445 172.21.15.89:0/573502486' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-03T19:36:26.090 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:36:25 smithi164 bash[26855]: audit 2023-06-03T19:36:25.215018+0000 mon.a (mon.0) 741 : audit [INF] from='mgr.14445 172.21.15.89:0/573502486' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-03T19:36:26.090 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:36:25 smithi164 bash[26855]: audit 2023-06-03T19:36:25.221241+0000 mon.a (mon.0) 742 : audit [INF] from='mgr.14445 172.21.15.89:0/573502486' entity='mgr.y' 2023-06-03T19:36:26.187 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:36:25 smithi089 bash[26572]: cluster 2023-06-03T19:36:24.458651+0000 mgr.y (mgr.14445) 173 : cluster [DBG] pgmap v152: 73 pgs: 73 active+clean; 2.3 KiB data, 45 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:36:26.187 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:36:25 smithi089 bash[26572]: audit 2023-06-03T19:36:24.786122+0000 mon.a (mon.0) 733 : audit [INF] from='mgr.14445 172.21.15.89:0/573502486' entity='mgr.y' 2023-06-03T19:36:26.187 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:36:25 smithi089 bash[26572]: audit 2023-06-03T19:36:24.829337+0000 mon.a (mon.0) 734 : audit [INF] from='mgr.14445 172.21.15.89:0/573502486' entity='mgr.y' 2023-06-03T19:36:26.188 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:36:25 smithi089 bash[26572]: audit 2023-06-03T19:36:24.830448+0000 mon.a (mon.0) 735 : audit [DBG] from='mgr.14445 172.21.15.89:0/573502486' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-03T19:36:26.188 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:36:25 smithi089 bash[26572]: audit 2023-06-03T19:36:24.831337+0000 mon.a (mon.0) 736 : audit [INF] from='mgr.14445 172.21.15.89:0/573502486' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-03T19:36:26.188 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:36:25 smithi089 bash[26572]: audit 2023-06-03T19:36:24.837805+0000 mon.a (mon.0) 737 : audit [INF] from='mgr.14445 172.21.15.89:0/573502486' entity='mgr.y' 2023-06-03T19:36:26.188 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:36:25 smithi089 bash[26572]: audit 2023-06-03T19:36:25.160813+0000 mon.a (mon.0) 738 : audit [INF] from='client.? ' entity='client.admin' 2023-06-03T19:36:26.188 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:36:25 smithi089 bash[26572]: audit 2023-06-03T19:36:25.210180+0000 mon.a (mon.0) 739 : audit [DBG] from='mgr.14445 172.21.15.89:0/573502486' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-03T19:36:26.189 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:36:25 smithi089 bash[26572]: audit 2023-06-03T19:36:25.213769+0000 mon.a (mon.0) 740 : audit [DBG] from='mgr.14445 172.21.15.89:0/573502486' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-03T19:36:26.189 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:36:25 smithi089 bash[26572]: audit 2023-06-03T19:36:25.215018+0000 mon.a (mon.0) 741 : audit [INF] from='mgr.14445 172.21.15.89:0/573502486' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-03T19:36:26.189 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:36:25 smithi089 bash[26572]: audit 2023-06-03T19:36:25.221241+0000 mon.a (mon.0) 742 : audit [INF] from='mgr.14445 172.21.15.89:0/573502486' entity='mgr.y' 2023-06-03T19:36:26.189 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:36:25 smithi089 bash[33173]: cluster 2023-06-03T19:36:24.458651+0000 mgr.y (mgr.14445) 173 : cluster [DBG] pgmap v152: 73 pgs: 73 active+clean; 2.3 KiB data, 45 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:36:26.189 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:36:25 smithi089 bash[33173]: audit 2023-06-03T19:36:24.786122+0000 mon.a (mon.0) 733 : audit [INF] from='mgr.14445 172.21.15.89:0/573502486' entity='mgr.y' 2023-06-03T19:36:26.189 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:36:25 smithi089 bash[33173]: audit 2023-06-03T19:36:24.829337+0000 mon.a (mon.0) 734 : audit [INF] from='mgr.14445 172.21.15.89:0/573502486' entity='mgr.y' 2023-06-03T19:36:26.190 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:36:25 smithi089 bash[33173]: audit 2023-06-03T19:36:24.830448+0000 mon.a (mon.0) 735 : audit [DBG] from='mgr.14445 172.21.15.89:0/573502486' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-03T19:36:26.190 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:36:25 smithi089 bash[33173]: audit 2023-06-03T19:36:24.831337+0000 mon.a (mon.0) 736 : audit [INF] from='mgr.14445 172.21.15.89:0/573502486' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-03T19:36:26.190 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:36:25 smithi089 bash[33173]: audit 2023-06-03T19:36:24.837805+0000 mon.a (mon.0) 737 : audit [INF] from='mgr.14445 172.21.15.89:0/573502486' entity='mgr.y' 2023-06-03T19:36:26.190 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:36:25 smithi089 bash[33173]: audit 2023-06-03T19:36:25.160813+0000 mon.a (mon.0) 738 : audit [INF] from='client.? ' entity='client.admin' 2023-06-03T19:36:26.190 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:36:25 smithi089 bash[33173]: audit 2023-06-03T19:36:25.210180+0000 mon.a (mon.0) 739 : audit [DBG] from='mgr.14445 172.21.15.89:0/573502486' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-03T19:36:26.190 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:36:25 smithi089 bash[33173]: audit 2023-06-03T19:36:25.213769+0000 mon.a (mon.0) 740 : audit [DBG] from='mgr.14445 172.21.15.89:0/573502486' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-03T19:36:26.190 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:36:25 smithi089 bash[33173]: audit 2023-06-03T19:36:25.215018+0000 mon.a (mon.0) 741 : audit [INF] from='mgr.14445 172.21.15.89:0/573502486' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-03T19:36:26.191 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:36:25 smithi089 bash[33173]: audit 2023-06-03T19:36:25.221241+0000 mon.a (mon.0) 742 : audit [INF] from='mgr.14445 172.21.15.89:0/573502486' entity='mgr.y' 2023-06-03T19:36:26.241 INFO:teuthology.orchestra.run.smithi089.stderr:pool 'unique_pool_0' created 2023-06-03T19:36:27.087 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:36:26 smithi164 bash[26855]: audit 2023-06-03T19:36:25.348469+0000 mon.a (mon.0) 743 : audit [INF] from='client.? 172.21.15.89:0/3544049557' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_0", "pg_num": 16}]: dispatch 2023-06-03T19:36:27.087 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:36:26 smithi164 bash[26855]: audit 2023-06-03T19:36:26.227059+0000 mon.a (mon.0) 744 : audit [INF] from='client.? 172.21.15.89:0/3544049557' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_0", "pg_num": 16}]': finished 2023-06-03T19:36:27.087 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:36:26 smithi164 bash[26855]: cluster 2023-06-03T19:36:26.227258+0000 mon.a (mon.0) 745 : cluster [DBG] osdmap e63: 8 total, 8 up, 8 in 2023-06-03T19:36:27.282 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:36:26 smithi089 bash[26572]: audit 2023-06-03T19:36:25.348469+0000 mon.a (mon.0) 743 : audit [INF] from='client.? 172.21.15.89:0/3544049557' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_0", "pg_num": 16}]: dispatch 2023-06-03T19:36:27.282 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:36:26 smithi089 bash[26572]: audit 2023-06-03T19:36:26.227059+0000 mon.a (mon.0) 744 : audit [INF] from='client.? 172.21.15.89:0/3544049557' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_0", "pg_num": 16}]': finished 2023-06-03T19:36:27.282 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:36:26 smithi089 bash[26572]: cluster 2023-06-03T19:36:26.227258+0000 mon.a (mon.0) 745 : cluster [DBG] osdmap e63: 8 total, 8 up, 8 in 2023-06-03T19:36:27.282 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:36:26 smithi089 bash[33173]: audit 2023-06-03T19:36:25.348469+0000 mon.a (mon.0) 743 : audit [INF] from='client.? 172.21.15.89:0/3544049557' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_0", "pg_num": 16}]: dispatch 2023-06-03T19:36:27.283 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:36:26 smithi089 bash[33173]: audit 2023-06-03T19:36:26.227059+0000 mon.a (mon.0) 744 : audit [INF] from='client.? 172.21.15.89:0/3544049557' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_0", "pg_num": 16}]': finished 2023-06-03T19:36:27.283 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:36:26 smithi089 bash[33173]: cluster 2023-06-03T19:36:26.227258+0000 mon.a (mon.0) 745 : cluster [DBG] osdmap e63: 8 total, 8 up, 8 in 2023-06-03T19:36:27.731 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph osd pool application enable unique_pool_0 rados --yes-i-really-mean-it || true 2023-06-03T19:36:28.030 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 19:36:27 smithi089 bash[26845]: ::ffff:172.21.15.164 - - [03/Jun/2023:19:36:27] "GET /metrics HTTP/1.1" 200 187882 "" "Prometheus/2.33.4" 2023-06-03T19:36:28.030 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:36:27 smithi089 bash[26572]: cluster 2023-06-03T19:36:26.459334+0000 mgr.y (mgr.14445) 174 : cluster [DBG] pgmap v154: 89 pgs: 16 unknown, 73 active+clean; 2.3 KiB data, 45 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:36:28.031 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:36:27 smithi089 bash[26572]: cluster 2023-06-03T19:36:27.235637+0000 mon.a (mon.0) 746 : cluster [DBG] osdmap e64: 8 total, 8 up, 8 in 2023-06-03T19:36:28.031 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:36:27 smithi089 bash[33173]: cluster 2023-06-03T19:36:26.459334+0000 mgr.y (mgr.14445) 174 : cluster [DBG] pgmap v154: 89 pgs: 16 unknown, 73 active+clean; 2.3 KiB data, 45 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:36:28.031 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:36:27 smithi089 bash[33173]: cluster 2023-06-03T19:36:27.235637+0000 mon.a (mon.0) 746 : cluster [DBG] osdmap e64: 8 total, 8 up, 8 in 2023-06-03T19:36:28.087 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:36:27 smithi164 bash[26855]: cluster 2023-06-03T19:36:26.459334+0000 mgr.y (mgr.14445) 174 : cluster [DBG] pgmap v154: 89 pgs: 16 unknown, 73 active+clean; 2.3 KiB data, 45 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:36:28.087 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:36:27 smithi164 bash[26855]: cluster 2023-06-03T19:36:27.235637+0000 mon.a (mon.0) 746 : cluster [DBG] osdmap e64: 8 total, 8 up, 8 in 2023-06-03T19:36:29.532 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:36:29 smithi089 bash[26572]: cluster 2023-06-03T19:36:28.238926+0000 mon.a (mon.0) 747 : cluster [DBG] osdmap e65: 8 total, 8 up, 8 in 2023-06-03T19:36:29.532 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:36:29 smithi089 bash[26572]: cluster 2023-06-03T19:36:28.459964+0000 mgr.y (mgr.14445) 175 : cluster [DBG] pgmap v157: 89 pgs: 16 unknown, 73 active+clean; 2.3 KiB data, 45 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:36:29.533 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:36:29 smithi089 bash[33173]: cluster 2023-06-03T19:36:28.238926+0000 mon.a (mon.0) 747 : cluster [DBG] osdmap e65: 8 total, 8 up, 8 in 2023-06-03T19:36:29.533 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:36:29 smithi089 bash[33173]: cluster 2023-06-03T19:36:28.459964+0000 mgr.y (mgr.14445) 175 : cluster [DBG] pgmap v157: 89 pgs: 16 unknown, 73 active+clean; 2.3 KiB data, 45 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:36:29.587 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:36:29 smithi164 bash[26855]: cluster 2023-06-03T19:36:28.238926+0000 mon.a (mon.0) 747 : cluster [DBG] osdmap e65: 8 total, 8 up, 8 in 2023-06-03T19:36:29.587 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:36:29 smithi164 bash[26855]: cluster 2023-06-03T19:36:28.459964+0000 mgr.y (mgr.14445) 175 : cluster [DBG] pgmap v157: 89 pgs: 16 unknown, 73 active+clean; 2.3 KiB data, 45 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:36:30.250 INFO:teuthology.orchestra.run.smithi089.stdout:Initiating upgrade to quay.ceph.io/ceph-ci/ceph:8e24bcf462a1d94c2e12aa4e26419634883a1e88 2023-06-03T19:36:31.204 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -e sha1=8e24bcf462a1d94c2e12aa4e26419634883a1e88 -- bash -c 'ceph orch ps' 2023-06-03T19:36:31.532 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:36:31 smithi089 bash[26572]: audit 2023-06-03T19:36:30.233906+0000 mgr.y (mgr.14445) 176 : audit [DBG] from='client.24725 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:8e24bcf462a1d94c2e12aa4e26419634883a1e88", "target": ["mon-mgr", ""]}]: dispatch 2023-06-03T19:36:31.532 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:36:31 smithi089 bash[26572]: cephadm 2023-06-03T19:36:30.235420+0000 mgr.y (mgr.14445) 177 : cephadm [INF] Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:8e24bcf462a1d94c2e12aa4e26419634883a1e88 2023-06-03T19:36:31.532 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:36:31 smithi089 bash[26572]: audit 2023-06-03T19:36:30.241885+0000 mon.a (mon.0) 748 : audit [INF] from='mgr.14445 172.21.15.89:0/573502486' entity='mgr.y' 2023-06-03T19:36:31.532 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:36:31 smithi089 bash[26572]: audit 2023-06-03T19:36:30.253612+0000 mon.a (mon.0) 749 : audit [DBG] from='mgr.14445 172.21.15.89:0/573502486' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-03T19:36:31.532 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:36:31 smithi089 bash[26572]: audit 2023-06-03T19:36:30.256215+0000 mon.a (mon.0) 750 : audit [DBG] from='mgr.14445 172.21.15.89:0/573502486' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-03T19:36:31.533 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:36:31 smithi089 bash[26572]: audit 2023-06-03T19:36:30.257497+0000 mon.a (mon.0) 751 : audit [INF] from='mgr.14445 172.21.15.89:0/573502486' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-03T19:36:31.533 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:36:31 smithi089 bash[26572]: audit 2023-06-03T19:36:30.265065+0000 mon.a (mon.0) 752 : audit [INF] from='mgr.14445 172.21.15.89:0/573502486' entity='mgr.y' 2023-06-03T19:36:31.533 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:36:31 smithi089 bash[26572]: cephadm 2023-06-03T19:36:30.270883+0000 mgr.y (mgr.14445) 178 : cephadm [INF] Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:8e24bcf462a1d94c2e12aa4e26419634883a1e88 2023-06-03T19:36:31.533 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:36:31 smithi089 bash[26572]: cluster 2023-06-03T19:36:30.461020+0000 mgr.y (mgr.14445) 179 : cluster [DBG] pgmap v158: 89 pgs: 89 active+clean; 2.3 KiB data, 46 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:36:31.533 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:36:31 smithi089 bash[33173]: audit 2023-06-03T19:36:30.233906+0000 mgr.y (mgr.14445) 176 : audit [DBG] from='client.24725 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:8e24bcf462a1d94c2e12aa4e26419634883a1e88", "target": ["mon-mgr", ""]}]: dispatch 2023-06-03T19:36:31.533 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:36:31 smithi089 bash[33173]: cephadm 2023-06-03T19:36:30.235420+0000 mgr.y (mgr.14445) 177 : cephadm [INF] Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:8e24bcf462a1d94c2e12aa4e26419634883a1e88 2023-06-03T19:36:31.534 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:36:31 smithi089 bash[33173]: audit 2023-06-03T19:36:30.241885+0000 mon.a (mon.0) 748 : audit [INF] from='mgr.14445 172.21.15.89:0/573502486' entity='mgr.y' 2023-06-03T19:36:31.534 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:36:31 smithi089 bash[33173]: audit 2023-06-03T19:36:30.253612+0000 mon.a (mon.0) 749 : audit [DBG] from='mgr.14445 172.21.15.89:0/573502486' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-03T19:36:31.534 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:36:31 smithi089 bash[33173]: audit 2023-06-03T19:36:30.256215+0000 mon.a (mon.0) 750 : audit [DBG] from='mgr.14445 172.21.15.89:0/573502486' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-03T19:36:31.534 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:36:31 smithi089 bash[33173]: audit 2023-06-03T19:36:30.257497+0000 mon.a (mon.0) 751 : audit [INF] from='mgr.14445 172.21.15.89:0/573502486' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-03T19:36:31.534 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:36:31 smithi089 bash[33173]: audit 2023-06-03T19:36:30.265065+0000 mon.a (mon.0) 752 : audit [INF] from='mgr.14445 172.21.15.89:0/573502486' entity='mgr.y' 2023-06-03T19:36:31.534 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:36:31 smithi089 bash[33173]: cephadm 2023-06-03T19:36:30.270883+0000 mgr.y (mgr.14445) 178 : cephadm [INF] Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:8e24bcf462a1d94c2e12aa4e26419634883a1e88 2023-06-03T19:36:31.534 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:36:31 smithi089 bash[33173]: cluster 2023-06-03T19:36:30.461020+0000 mgr.y (mgr.14445) 179 : cluster [DBG] pgmap v158: 89 pgs: 89 active+clean; 2.3 KiB data, 46 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:36:31.587 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:36:31 smithi164 bash[26855]: audit 2023-06-03T19:36:30.233906+0000 mgr.y (mgr.14445) 176 : audit [DBG] from='client.24725 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:8e24bcf462a1d94c2e12aa4e26419634883a1e88", "target": ["mon-mgr", ""]}]: dispatch 2023-06-03T19:36:31.587 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:36:31 smithi164 bash[26855]: cephadm 2023-06-03T19:36:30.235420+0000 mgr.y (mgr.14445) 177 : cephadm [INF] Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:8e24bcf462a1d94c2e12aa4e26419634883a1e88 2023-06-03T19:36:31.588 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:36:31 smithi164 bash[26855]: audit 2023-06-03T19:36:30.241885+0000 mon.a (mon.0) 748 : audit [INF] from='mgr.14445 172.21.15.89:0/573502486' entity='mgr.y' 2023-06-03T19:36:31.588 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:36:31 smithi164 bash[26855]: audit 2023-06-03T19:36:30.253612+0000 mon.a (mon.0) 749 : audit [DBG] from='mgr.14445 172.21.15.89:0/573502486' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-03T19:36:31.588 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:36:31 smithi164 bash[26855]: audit 2023-06-03T19:36:30.256215+0000 mon.a (mon.0) 750 : audit [DBG] from='mgr.14445 172.21.15.89:0/573502486' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-03T19:36:31.588 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:36:31 smithi164 bash[26855]: audit 2023-06-03T19:36:30.257497+0000 mon.a (mon.0) 751 : audit [INF] from='mgr.14445 172.21.15.89:0/573502486' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-03T19:36:31.588 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:36:31 smithi164 bash[26855]: audit 2023-06-03T19:36:30.265065+0000 mon.a (mon.0) 752 : audit [INF] from='mgr.14445 172.21.15.89:0/573502486' entity='mgr.y' 2023-06-03T19:36:31.589 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:36:31 smithi164 bash[26855]: cephadm 2023-06-03T19:36:30.270883+0000 mgr.y (mgr.14445) 178 : cephadm [INF] Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:8e24bcf462a1d94c2e12aa4e26419634883a1e88 2023-06-03T19:36:31.589 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:36:31 smithi164 bash[26855]: cluster 2023-06-03T19:36:30.461020+0000 mgr.y (mgr.14445) 179 : cluster [DBG] pgmap v158: 89 pgs: 89 active+clean; 2.3 KiB data, 46 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:36:33.722 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/mon.a/config 2023-06-03T19:36:33.837 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:36:33 smithi164 bash[26855]: cluster 2023-06-03T19:36:32.461458+0000 mgr.y (mgr.14445) 180 : cluster [DBG] pgmap v159: 89 pgs: 89 active+clean; 2.3 KiB data, 46 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:36:33.941 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:36:33 smithi089 bash[33173]: cluster 2023-06-03T19:36:32.461458+0000 mgr.y (mgr.14445) 180 : cluster [DBG] pgmap v159: 89 pgs: 89 active+clean; 2.3 KiB data, 46 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:36:33.942 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:36:33 smithi089 bash[26572]: cluster 2023-06-03T19:36:32.461458+0000 mgr.y (mgr.14445) 180 : cluster [DBG] pgmap v159: 89 pgs: 89 active+clean; 2.3 KiB data, 46 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:36:35.837 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:36:35 smithi164 bash[26855]: cluster 2023-06-03T19:36:34.462695+0000 mgr.y (mgr.14445) 181 : cluster [DBG] pgmap v160: 89 pgs: 89 active+clean; 2.3 KiB data, 46 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:36:35.837 INFO:journalctl@ceph.mgr.x.smithi164.stdout:Jun 03 19:36:35 smithi164 bash[27934]: ::ffff:172.21.15.164 - - [03/Jun/2023:19:36:35] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2023-06-03T19:36:36.032 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:36:35 smithi089 bash[26572]: cluster 2023-06-03T19:36:34.462695+0000 mgr.y (mgr.14445) 181 : cluster [DBG] pgmap v160: 89 pgs: 89 active+clean; 2.3 KiB data, 46 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:36:36.032 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:36:35 smithi089 bash[33173]: cluster 2023-06-03T19:36:34.462695+0000 mgr.y (mgr.14445) 181 : cluster [DBG] pgmap v160: 89 pgs: 89 active+clean; 2.3 KiB data, 46 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:36:37.837 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:36:37 smithi164 bash[26855]: cluster 2023-06-03T19:36:36.463325+0000 mgr.y (mgr.14445) 182 : cluster [DBG] pgmap v161: 89 pgs: 89 active+clean; 2.3 KiB data, 46 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:36:37.907 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:36:37 smithi089 bash[26572]: cluster 2023-06-03T19:36:36.463325+0000 mgr.y (mgr.14445) 182 : cluster [DBG] pgmap v161: 89 pgs: 89 active+clean; 2.3 KiB data, 46 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:36:37.908 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:36:37 smithi089 bash[33173]: cluster 2023-06-03T19:36:36.463325+0000 mgr.y (mgr.14445) 182 : cluster [DBG] pgmap v161: 89 pgs: 89 active+clean; 2.3 KiB data, 46 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:36:38.281 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 19:36:37 smithi089 bash[26845]: ::ffff:172.21.15.164 - - [03/Jun/2023:19:36:37] "GET /metrics HTTP/1.1" 200 190291 "" "Prometheus/2.33.4" 2023-06-03T19:36:40.032 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:36:39 smithi089 bash[26572]: cluster 2023-06-03T19:36:38.463888+0000 mgr.y (mgr.14445) 183 : cluster [DBG] pgmap v162: 89 pgs: 89 active+clean; 2.3 KiB data, 46 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:36:40.032 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:36:39 smithi089 bash[33173]: cluster 2023-06-03T19:36:38.463888+0000 mgr.y (mgr.14445) 183 : cluster [DBG] pgmap v162: 89 pgs: 89 active+clean; 2.3 KiB data, 46 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:36:40.087 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:36:39 smithi164 bash[26855]: cluster 2023-06-03T19:36:38.463888+0000 mgr.y (mgr.14445) 183 : cluster [DBG] pgmap v162: 89 pgs: 89 active+clean; 2.3 KiB data, 46 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:36:42.032 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:36:41 smithi089 bash[33173]: cluster 2023-06-03T19:36:40.464726+0000 mgr.y (mgr.14445) 184 : cluster [DBG] pgmap v163: 89 pgs: 89 active+clean; 2.3 KiB data, 46 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:36:42.032 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:36:41 smithi089 bash[26572]: cluster 2023-06-03T19:36:40.464726+0000 mgr.y (mgr.14445) 184 : cluster [DBG] pgmap v163: 89 pgs: 89 active+clean; 2.3 KiB data, 46 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:36:42.087 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:36:41 smithi164 bash[26855]: cluster 2023-06-03T19:36:40.464726+0000 mgr.y (mgr.14445) 184 : cluster [DBG] pgmap v163: 89 pgs: 89 active+clean; 2.3 KiB data, 46 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:36:44.032 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:36:43 smithi089 bash[33173]: cluster 2023-06-03T19:36:42.465193+0000 mgr.y (mgr.14445) 185 : cluster [DBG] pgmap v164: 89 pgs: 89 active+clean; 2.3 KiB data, 46 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:36:44.032 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:36:43 smithi089 bash[26572]: cluster 2023-06-03T19:36:42.465193+0000 mgr.y (mgr.14445) 185 : cluster [DBG] pgmap v164: 89 pgs: 89 active+clean; 2.3 KiB data, 46 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:36:44.087 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:36:43 smithi164 bash[26855]: cluster 2023-06-03T19:36:42.465193+0000 mgr.y (mgr.14445) 185 : cluster [DBG] pgmap v164: 89 pgs: 89 active+clean; 2.3 KiB data, 46 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:36:46.032 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:36:45 smithi089 bash[26572]: cluster 2023-06-03T19:36:44.465871+0000 mgr.y (mgr.14445) 186 : cluster [DBG] pgmap v165: 89 pgs: 89 active+clean; 2.3 KiB data, 46 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:36:46.032 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:36:45 smithi089 bash[33173]: cluster 2023-06-03T19:36:44.465871+0000 mgr.y (mgr.14445) 186 : cluster [DBG] pgmap v165: 89 pgs: 89 active+clean; 2.3 KiB data, 46 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:36:46.087 INFO:journalctl@ceph.mgr.x.smithi164.stdout:Jun 03 19:36:45 smithi164 bash[27934]: ::ffff:172.21.15.164 - - [03/Jun/2023:19:36:45] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2023-06-03T19:36:46.087 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:36:45 smithi164 bash[26855]: cluster 2023-06-03T19:36:44.465871+0000 mgr.y (mgr.14445) 186 : cluster [DBG] pgmap v165: 89 pgs: 89 active+clean; 2.3 KiB data, 46 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:36:47.907 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:36:47 smithi089 bash[33173]: cluster 2023-06-03T19:36:46.466506+0000 mgr.y (mgr.14445) 187 : cluster [DBG] pgmap v166: 89 pgs: 89 active+clean; 2.3 KiB data, 46 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:36:47.908 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:36:47 smithi089 bash[26572]: cluster 2023-06-03T19:36:46.466506+0000 mgr.y (mgr.14445) 187 : cluster [DBG] pgmap v166: 89 pgs: 89 active+clean; 2.3 KiB data, 46 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:36:48.087 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:36:47 smithi164 bash[26855]: cluster 2023-06-03T19:36:46.466506+0000 mgr.y (mgr.14445) 187 : cluster [DBG] pgmap v166: 89 pgs: 89 active+clean; 2.3 KiB data, 46 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:36:48.282 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 19:36:47 smithi089 bash[26845]: ::ffff:172.21.15.164 - - [03/Jun/2023:19:36:47] "GET /metrics HTTP/1.1" 200 190291 "" "Prometheus/2.33.4" 2023-06-03T19:36:49.908 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:36:49 smithi089 bash[33173]: cluster 2023-06-03T19:36:48.467264+0000 mgr.y (mgr.14445) 188 : cluster [DBG] pgmap v167: 89 pgs: 89 active+clean; 2.3 KiB data, 46 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:36:49.908 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:36:49 smithi089 bash[26572]: cluster 2023-06-03T19:36:48.467264+0000 mgr.y (mgr.14445) 188 : cluster [DBG] pgmap v167: 89 pgs: 89 active+clean; 2.3 KiB data, 46 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:36:50.087 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:36:49 smithi164 bash[26855]: cluster 2023-06-03T19:36:48.467264+0000 mgr.y (mgr.14445) 188 : cluster [DBG] pgmap v167: 89 pgs: 89 active+clean; 2.3 KiB data, 46 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:36:52.032 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:36:51 smithi089 bash[26572]: cluster 2023-06-03T19:36:50.468157+0000 mgr.y (mgr.14445) 189 : cluster [DBG] pgmap v168: 89 pgs: 89 active+clean; 2.3 KiB data, 46 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:36:52.032 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:36:51 smithi089 bash[33173]: cluster 2023-06-03T19:36:50.468157+0000 mgr.y (mgr.14445) 189 : cluster [DBG] pgmap v168: 89 pgs: 89 active+clean; 2.3 KiB data, 46 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:36:52.087 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:36:51 smithi164 bash[26855]: cluster 2023-06-03T19:36:50.468157+0000 mgr.y (mgr.14445) 189 : cluster [DBG] pgmap v168: 89 pgs: 89 active+clean; 2.3 KiB data, 46 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:36:53.337 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:36:53 smithi164 bash[26855]: audit 2023-06-03T19:36:52.040123+0000 mon.a (mon.0) 753 : audit [INF] from='mgr.14445 172.21.15.89:0/573502486' entity='mgr.y' 2023-06-03T19:36:53.337 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:36:53 smithi164 bash[26855]: cephadm 2023-06-03T19:36:52.040525+0000 mgr.y (mgr.14445) 190 : cephadm [INF] Upgrade: Target is version 18.0.0-3532-g8e24bcf4 (unknown) 2023-06-03T19:36:53.337 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:36:53 smithi164 bash[26855]: cephadm 2023-06-03T19:36:52.040612+0000 mgr.y (mgr.14445) 191 : cephadm [INF] Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:00b1b6d689278b4f97ff790045c8c51d8ca419ac7ee2ebdf11008f2e810a5c78, digests ['quay.ceph.io/ceph-ci/ceph@sha256:00b1b6d689278b4f97ff790045c8c51d8ca419ac7ee2ebdf11008f2e810a5c78'] 2023-06-03T19:36:53.338 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:36:53 smithi164 bash[26855]: audit 2023-06-03T19:36:52.041745+0000 mon.a (mon.0) 754 : audit [DBG] from='mgr.14445 172.21.15.89:0/573502486' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-03T19:36:53.338 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:36:53 smithi164 bash[26855]: audit 2023-06-03T19:36:52.048889+0000 mon.a (mon.0) 755 : audit [INF] from='mgr.14445 172.21.15.89:0/573502486' entity='mgr.y' 2023-06-03T19:36:53.338 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:36:53 smithi164 bash[26855]: cephadm 2023-06-03T19:36:52.049692+0000 mgr.y (mgr.14445) 192 : cephadm [INF] Upgrade: Need to upgrade myself (mgr.y) 2023-06-03T19:36:53.338 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:36:53 smithi164 bash[26855]: cephadm 2023-06-03T19:36:52.277729+0000 mgr.y (mgr.14445) 193 : cephadm [INF] Upgrade: Pulling quay.ceph.io/ceph-ci/ceph@sha256:00b1b6d689278b4f97ff790045c8c51d8ca419ac7ee2ebdf11008f2e810a5c78 on smithi164 2023-06-03T19:36:53.338 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:36:53 smithi164 bash[26855]: cluster 2023-06-03T19:36:52.468828+0000 mgr.y (mgr.14445) 194 : cluster [DBG] pgmap v169: 89 pgs: 89 active+clean; 2.3 KiB data, 46 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:36:53.441 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:36:53 smithi089 bash[26572]: audit 2023-06-03T19:36:52.040123+0000 mon.a (mon.0) 753 : audit [INF] from='mgr.14445 172.21.15.89:0/573502486' entity='mgr.y' 2023-06-03T19:36:53.441 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:36:53 smithi089 bash[26572]: cephadm 2023-06-03T19:36:52.040525+0000 mgr.y (mgr.14445) 190 : cephadm [INF] Upgrade: Target is version 18.0.0-3532-g8e24bcf4 (unknown) 2023-06-03T19:36:53.441 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:36:53 smithi089 bash[26572]: cephadm 2023-06-03T19:36:52.040612+0000 mgr.y (mgr.14445) 191 : cephadm [INF] Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:00b1b6d689278b4f97ff790045c8c51d8ca419ac7ee2ebdf11008f2e810a5c78, digests ['quay.ceph.io/ceph-ci/ceph@sha256:00b1b6d689278b4f97ff790045c8c51d8ca419ac7ee2ebdf11008f2e810a5c78'] 2023-06-03T19:36:53.441 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:36:53 smithi089 bash[26572]: audit 2023-06-03T19:36:52.041745+0000 mon.a (mon.0) 754 : audit [DBG] from='mgr.14445 172.21.15.89:0/573502486' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-03T19:36:53.442 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:36:53 smithi089 bash[26572]: audit 2023-06-03T19:36:52.048889+0000 mon.a (mon.0) 755 : audit [INF] from='mgr.14445 172.21.15.89:0/573502486' entity='mgr.y' 2023-06-03T19:36:53.442 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:36:53 smithi089 bash[26572]: cephadm 2023-06-03T19:36:52.049692+0000 mgr.y (mgr.14445) 192 : cephadm [INF] Upgrade: Need to upgrade myself (mgr.y) 2023-06-03T19:36:53.442 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:36:53 smithi089 bash[26572]: cephadm 2023-06-03T19:36:52.277729+0000 mgr.y (mgr.14445) 193 : cephadm [INF] Upgrade: Pulling quay.ceph.io/ceph-ci/ceph@sha256:00b1b6d689278b4f97ff790045c8c51d8ca419ac7ee2ebdf11008f2e810a5c78 on smithi164 2023-06-03T19:36:53.442 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:36:53 smithi089 bash[26572]: cluster 2023-06-03T19:36:52.468828+0000 mgr.y (mgr.14445) 194 : cluster [DBG] pgmap v169: 89 pgs: 89 active+clean; 2.3 KiB data, 46 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:36:53.442 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:36:53 smithi089 bash[33173]: audit 2023-06-03T19:36:52.040123+0000 mon.a (mon.0) 753 : audit [INF] from='mgr.14445 172.21.15.89:0/573502486' entity='mgr.y' 2023-06-03T19:36:53.442 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:36:53 smithi089 bash[33173]: cephadm 2023-06-03T19:36:52.040525+0000 mgr.y (mgr.14445) 190 : cephadm [INF] Upgrade: Target is version 18.0.0-3532-g8e24bcf4 (unknown) 2023-06-03T19:36:53.443 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:36:53 smithi089 bash[33173]: cephadm 2023-06-03T19:36:52.040612+0000 mgr.y (mgr.14445) 191 : cephadm [INF] Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:00b1b6d689278b4f97ff790045c8c51d8ca419ac7ee2ebdf11008f2e810a5c78, digests ['quay.ceph.io/ceph-ci/ceph@sha256:00b1b6d689278b4f97ff790045c8c51d8ca419ac7ee2ebdf11008f2e810a5c78'] 2023-06-03T19:36:53.443 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:36:53 smithi089 bash[33173]: audit 2023-06-03T19:36:52.041745+0000 mon.a (mon.0) 754 : audit [DBG] from='mgr.14445 172.21.15.89:0/573502486' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-03T19:36:53.443 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:36:53 smithi089 bash[33173]: audit 2023-06-03T19:36:52.048889+0000 mon.a (mon.0) 755 : audit [INF] from='mgr.14445 172.21.15.89:0/573502486' entity='mgr.y' 2023-06-03T19:36:53.443 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:36:53 smithi089 bash[33173]: cephadm 2023-06-03T19:36:52.049692+0000 mgr.y (mgr.14445) 192 : cephadm [INF] Upgrade: Need to upgrade myself (mgr.y) 2023-06-03T19:36:53.443 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:36:53 smithi089 bash[33173]: cephadm 2023-06-03T19:36:52.277729+0000 mgr.y (mgr.14445) 193 : cephadm [INF] Upgrade: Pulling quay.ceph.io/ceph-ci/ceph@sha256:00b1b6d689278b4f97ff790045c8c51d8ca419ac7ee2ebdf11008f2e810a5c78 on smithi164 2023-06-03T19:36:53.443 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:36:53 smithi089 bash[33173]: cluster 2023-06-03T19:36:52.468828+0000 mgr.y (mgr.14445) 194 : cluster [DBG] pgmap v169: 89 pgs: 89 active+clean; 2.3 KiB data, 46 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:36:53.963 INFO:teuthology.orchestra.run.smithi089.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2023-06-03T19:36:53.964 INFO:teuthology.orchestra.run.smithi089.stdout:alertmanager.a smithi089 *:9093,9094 running (3m) 29s ago 4m 13.2M - 0.23.0 ba2b418f427c 86721ff318d7 2023-06-03T19:36:53.964 INFO:teuthology.orchestra.run.smithi089.stdout:grafana.a smithi164 *:3000 running (3m) 29s ago 3m 43.1M - 8.3.5 dad864ee21e9 6be98e2ca11d 2023-06-03T19:36:53.964 INFO:teuthology.orchestra.run.smithi089.stdout:mds.foo.smithi089.giuxuj smithi089 running (2m) 29s ago 2m 13.2M - 16.2.13-257-gd8c5d349 b644dd95899d f0e9de24a3f8 2023-06-03T19:36:53.964 INFO:teuthology.orchestra.run.smithi089.stdout:mds.foo.smithi164.txgfgv smithi164 running (2m) 29s ago 2m 11.5M - 16.2.13-257-gd8c5d349 b644dd95899d 29da307aa3b4 2023-06-03T19:36:53.964 INFO:teuthology.orchestra.run.smithi089.stdout:mgr.x smithi164 *:8443 running (11m) 29s ago 11m 375M - 16.2.13-257-gd8c5d349 b644dd95899d f9d4342a0402 2023-06-03T19:36:53.964 INFO:teuthology.orchestra.run.smithi089.stdout:mgr.y smithi089 *:9283 running (13m) 29s ago 13m 425M - 16.2.13-257-gd8c5d349 b644dd95899d 323efd173e14 2023-06-03T19:36:53.965 INFO:teuthology.orchestra.run.smithi089.stdout:mon.a smithi089 running (13m) 29s ago 14m 49.4M 2048M 16.2.13-257-gd8c5d349 b644dd95899d 64c0c1397aaa 2023-06-03T19:36:53.965 INFO:teuthology.orchestra.run.smithi089.stdout:mon.b smithi164 running (12m) 29s ago 12m 38.9M 2048M 16.2.13-257-gd8c5d349 b644dd95899d 85a8c1795b60 2023-06-03T19:36:53.965 INFO:teuthology.orchestra.run.smithi089.stdout:mon.c smithi089 running (12m) 29s ago 12m 37.6M 2048M 16.2.13-257-gd8c5d349 b644dd95899d 268f878c5750 2023-06-03T19:36:53.965 INFO:teuthology.orchestra.run.smithi089.stdout:node-exporter.a smithi089 *:9100 running (4m) 29s ago 4m 9132k - 1.3.1 1dbe0e931976 22e0e7e9522e 2023-06-03T19:36:53.965 INFO:teuthology.orchestra.run.smithi089.stdout:node-exporter.b smithi164 *:9100 running (4m) 29s ago 4m 9455k - 1.3.1 1dbe0e931976 68274d505418 2023-06-03T19:36:53.965 INFO:teuthology.orchestra.run.smithi089.stdout:osd.0 smithi089 running (11m) 29s ago 11m 38.1M 4062M 16.2.13-257-gd8c5d349 b644dd95899d cb8bdf1749e4 2023-06-03T19:36:53.965 INFO:teuthology.orchestra.run.smithi089.stdout:osd.1 smithi089 running (10m) 29s ago 10m 37.2M 4062M 16.2.13-257-gd8c5d349 b644dd95899d d9e8a03d4689 2023-06-03T19:36:53.966 INFO:teuthology.orchestra.run.smithi089.stdout:osd.2 smithi089 running (9m) 29s ago 9m 36.7M 4062M 16.2.13-257-gd8c5d349 b644dd95899d f7fc93b466ec 2023-06-03T19:36:53.966 INFO:teuthology.orchestra.run.smithi089.stdout:osd.3 smithi089 running (8m) 29s ago 8m 38.3M 4062M 16.2.13-257-gd8c5d349 b644dd95899d 7e98b61121c6 2023-06-03T19:36:53.966 INFO:teuthology.orchestra.run.smithi089.stdout:osd.4 smithi164 running (8m) 29s ago 8m 37.3M 4318M 16.2.13-257-gd8c5d349 b644dd95899d 5c901af1a184 2023-06-03T19:36:53.966 INFO:teuthology.orchestra.run.smithi089.stdout:osd.5 smithi164 running (7m) 29s ago 7m 36.3M 4318M 16.2.13-257-gd8c5d349 b644dd95899d 13ddc40f04c4 2023-06-03T19:36:53.966 INFO:teuthology.orchestra.run.smithi089.stdout:osd.6 smithi164 running (6m) 29s ago 6m 37.1M 4318M 16.2.13-257-gd8c5d349 b644dd95899d cab2cb9998d7 2023-06-03T19:36:53.966 INFO:teuthology.orchestra.run.smithi089.stdout:osd.7 smithi164 running (5m) 29s ago 5m 37.7M 4318M 16.2.13-257-gd8c5d349 b644dd95899d b5fd865771b6 2023-06-03T19:36:53.966 INFO:teuthology.orchestra.run.smithi089.stdout:prometheus.a smithi164 *:9095 running (3m) 29s ago 4m 52.0M - 2.33.4 514e6a882f6e a04f33a7d1c4 2023-06-03T19:36:54.075 INFO:teuthology.orchestra.run.smithi089.stderr:enabled application 'rados' on pool 'unique_pool_0' 2023-06-03T19:36:54.282 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:36:54 smithi089 bash[26572]: audit 2023-06-03T19:36:53.883538+0000 mon.c (mon.2) 29 : audit [INF] from='client.? 172.21.15.89:0/3389753264' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_0", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-06-03T19:36:54.282 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:36:54 smithi089 bash[26572]: audit 2023-06-03T19:36:53.884392+0000 mon.a (mon.0) 756 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_0", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-06-03T19:36:54.282 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:36:54 smithi089 bash[33173]: audit 2023-06-03T19:36:53.883538+0000 mon.c (mon.2) 29 : audit [INF] from='client.? 172.21.15.89:0/3389753264' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_0", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-06-03T19:36:54.283 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:36:54 smithi089 bash[33173]: audit 2023-06-03T19:36:53.884392+0000 mon.a (mon.0) 756 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_0", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-06-03T19:36:54.337 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:36:54 smithi164 bash[26855]: audit 2023-06-03T19:36:53.883538+0000 mon.c (mon.2) 29 : audit [INF] from='client.? 172.21.15.89:0/3389753264' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_0", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-06-03T19:36:54.337 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:36:54 smithi164 bash[26855]: audit 2023-06-03T19:36:53.884392+0000 mon.a (mon.0) 756 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_0", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-06-03T19:36:55.337 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:36:55 smithi164 bash[26855]: audit 2023-06-03T19:36:53.870398+0000 mgr.y (mgr.14445) 195 : audit [DBG] from='client.24731 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-06-03T19:36:55.337 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:36:55 smithi164 bash[26855]: audit 2023-06-03T19:36:54.060615+0000 mon.a (mon.0) 757 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_0", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-06-03T19:36:55.338 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:36:55 smithi164 bash[26855]: cluster 2023-06-03T19:36:54.060683+0000 mon.a (mon.0) 758 : cluster [DBG] osdmap e66: 8 total, 8 up, 8 in 2023-06-03T19:36:55.338 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:36:55 smithi164 bash[26855]: cluster 2023-06-03T19:36:54.469922+0000 mgr.y (mgr.14445) 196 : cluster [DBG] pgmap v171: 89 pgs: 89 active+clean; 2.3 KiB data, 46 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:36:55.388 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:36:55 smithi089 bash[33173]: audit 2023-06-03T19:36:53.870398+0000 mgr.y (mgr.14445) 195 : audit [DBG] from='client.24731 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-06-03T19:36:55.388 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:36:55 smithi089 bash[33173]: audit 2023-06-03T19:36:54.060615+0000 mon.a (mon.0) 757 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_0", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-06-03T19:36:55.388 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:36:55 smithi089 bash[33173]: cluster 2023-06-03T19:36:54.060683+0000 mon.a (mon.0) 758 : cluster [DBG] osdmap e66: 8 total, 8 up, 8 in 2023-06-03T19:36:55.389 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:36:55 smithi089 bash[33173]: cluster 2023-06-03T19:36:54.469922+0000 mgr.y (mgr.14445) 196 : cluster [DBG] pgmap v171: 89 pgs: 89 active+clean; 2.3 KiB data, 46 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:36:55.389 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:36:55 smithi089 bash[26572]: audit 2023-06-03T19:36:53.870398+0000 mgr.y (mgr.14445) 195 : audit [DBG] from='client.24731 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-06-03T19:36:55.389 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:36:55 smithi089 bash[26572]: audit 2023-06-03T19:36:54.060615+0000 mon.a (mon.0) 757 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_0", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-06-03T19:36:55.389 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:36:55 smithi089 bash[26572]: cluster 2023-06-03T19:36:54.060683+0000 mon.a (mon.0) 758 : cluster [DBG] osdmap e66: 8 total, 8 up, 8 in 2023-06-03T19:36:55.389 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:36:55 smithi089 bash[26572]: cluster 2023-06-03T19:36:54.469922+0000 mgr.y (mgr.14445) 196 : cluster [DBG] pgmap v171: 89 pgs: 89 active+clean; 2.3 KiB data, 46 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:36:55.390 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -e sha1=8e24bcf462a1d94c2e12aa4e26419634883a1e88 -- bash -c 'echo wait for minority of mons to upgrade' 2023-06-03T19:36:56.087 INFO:journalctl@ceph.mgr.x.smithi164.stdout:Jun 03 19:36:55 smithi164 bash[27934]: ::ffff:172.21.15.164 - - [03/Jun/2023:19:36:55] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2023-06-03T19:36:56.437 DEBUG:teuthology.orchestra.run.smithi089:> /bin/sh -c 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage rados --no-log-to-stderr --name client.0 -b 65536 --object-size 65536 -p unique_pool_0 bench 90 write' 2023-06-03T19:36:56.440 INFO:tasks.radosbench:joining radosbench (timing out after 3000s) 2023-06-03T19:36:56.440 INFO:teuthology.orchestra.run:waiting for 3000 2023-06-03T19:36:56.482 INFO:tasks.radosbench.radosbench.0.smithi089.stdout:hints = 1 2023-06-03T19:36:56.483 INFO:tasks.radosbench.radosbench.0.smithi089.stdout:Maintaining 16 concurrent writes of 65536 bytes to objects of size 65536 for up to 90 seconds or 0 objects 2023-06-03T19:36:56.483 INFO:tasks.radosbench.radosbench.0.smithi089.stdout:Object prefix: benchmark_data_smithi089_66854 2023-06-03T19:36:56.483 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-06-03T19:36:56.483 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 0 0 0 0 0 0 - 0 2023-06-03T19:36:57.483 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 1 16 2723 2707 169.178 169.188 0.00389158 0.00589149 2023-06-03T19:36:57.837 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:36:57 smithi164 bash[26855]: cluster 2023-06-03T19:36:56.470496+0000 mgr.y (mgr.14445) 197 : cluster [DBG] pgmap v172: 89 pgs: 89 active+clean; 2.3 KiB data, 46 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:36:57.910 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 19:36:57 smithi089 bash[26845]: ::ffff:172.21.15.164 - - [03/Jun/2023:19:36:57] "GET /metrics HTTP/1.1" 200 190290 "" "Prometheus/2.33.4" 2023-06-03T19:36:57.910 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:36:57 smithi089 bash[33173]: cluster 2023-06-03T19:36:56.470496+0000 mgr.y (mgr.14445) 197 : cluster [DBG] pgmap v172: 89 pgs: 89 active+clean; 2.3 KiB data, 46 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:36:57.910 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:36:57 smithi089 bash[26572]: cluster 2023-06-03T19:36:56.470496+0000 mgr.y (mgr.14445) 197 : cluster [DBG] pgmap v172: 89 pgs: 89 active+clean; 2.3 KiB data, 46 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:36:58.483 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 2 15 5216 5201 162.518 155.875 0.00583071 0.00613643 2023-06-03T19:36:59.484 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 3 16 7535 7519 156.631 144.875 0.00988573 0.00637315 2023-06-03T19:36:59.592 INFO:teuthology.orchestra.run.smithi089.stdout:wait for minority of mons to upgrade 2023-06-03T19:36:59.781 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:36:59 smithi089 bash[26572]: cluster 2023-06-03T19:36:58.470892+0000 mgr.y (mgr.14445) 198 : cluster [DBG] pgmap v173: 89 pgs: 89 active+clean; 2.3 KiB data, 46 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:36:59.782 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:36:59 smithi089 bash[33173]: cluster 2023-06-03T19:36:58.470892+0000 mgr.y (mgr.14445) 198 : cluster [DBG] pgmap v173: 89 pgs: 89 active+clean; 2.3 KiB data, 46 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:36:59.837 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:36:59 smithi164 bash[26855]: cluster 2023-06-03T19:36:58.470892+0000 mgr.y (mgr.14445) 198 : cluster [DBG] pgmap v173: 89 pgs: 89 active+clean; 2.3 KiB data, 46 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:37:00.484 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 4 16 9803 9787 152.908 141.75 0.00502409 0.00652999 2023-06-03T19:37:00.979 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -e sha1=8e24bcf462a1d94c2e12aa4e26419634883a1e88 -- bash -c 'while ! ceph mon versions | grep $sha1 ; do sleep 2 ; done' 2023-06-03T19:37:01.484 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 5 16 12031 12015 150.174 139.25 0.00575839 0.00664859 2023-06-03T19:37:01.587 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:37:01 smithi089 bash[26572]: cluster 2023-06-03T19:37:00.471943+0000 mgr.y (mgr.14445) 199 : cluster [DBG] pgmap v174: 89 pgs: 89 active+clean; 463 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 46 MiB/s wr, 739 op/s 2023-06-03T19:37:02.032 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:37:01 smithi089 bash[33173]: cluster 2023-06-03T19:37:00.471943+0000 mgr.y (mgr.14445) 199 : cluster [DBG] pgmap v174: 89 pgs: 89 active+clean; 463 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 46 MiB/s wr, 739 op/s 2023-06-03T19:37:02.087 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:37:01 smithi164 bash[26855]: cluster 2023-06-03T19:37:00.471943+0000 mgr.y (mgr.14445) 199 : cluster [DBG] pgmap v174: 89 pgs: 89 active+clean; 463 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 46 MiB/s wr, 739 op/s 2023-06-03T19:37:02.484 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 6 16 14078 14062 146.466 127.938 0.00731097 0.00681929 2023-06-03T19:37:03.484 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 7 16 16070 16054 143.326 124.5 0.00269572 0.00697003 2023-06-03T19:37:03.594 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:37:03 smithi089 bash[26572]: cluster 2023-06-03T19:37:02.472452+0000 mgr.y (mgr.14445) 200 : cluster [DBG] pgmap v175: 89 pgs: 89 active+clean; 463 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 46 MiB/s wr, 739 op/s 2023-06-03T19:37:04.032 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:37:03 smithi089 bash[33173]: cluster 2023-06-03T19:37:02.472452+0000 mgr.y (mgr.14445) 200 : cluster [DBG] pgmap v175: 89 pgs: 89 active+clean; 463 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 46 MiB/s wr, 739 op/s 2023-06-03T19:37:04.088 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:37:03 smithi164 bash[26855]: cluster 2023-06-03T19:37:02.472452+0000 mgr.y (mgr.14445) 200 : cluster [DBG] pgmap v175: 89 pgs: 89 active+clean; 463 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 46 MiB/s wr, 739 op/s 2023-06-03T19:37:04.484 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 8 16 17993 17977 140.433 120.188 0.00789587 0.00711019 2023-06-03T19:37:05.484 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 9 16 19955 19939 138.453 122.625 0.0077786 0.00721607 2023-06-03T19:37:05.919 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:37:05 smithi089 bash[33173]: cluster 2023-06-03T19:37:04.473147+0000 mgr.y (mgr.14445) 201 : cluster [DBG] pgmap v176: 89 pgs: 89 active+clean; 1.1 GiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 107 MiB/s wr, 1.72k op/s 2023-06-03T19:37:05.920 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:37:05 smithi089 bash[26572]: cluster 2023-06-03T19:37:04.473147+0000 mgr.y (mgr.14445) 201 : cluster [DBG] pgmap v176: 89 pgs: 89 active+clean; 1.1 GiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 107 MiB/s wr, 1.72k op/s 2023-06-03T19:37:06.087 INFO:journalctl@ceph.mgr.x.smithi164.stdout:Jun 03 19:37:05 smithi164 bash[27934]: ::ffff:172.21.15.164 - - [03/Jun/2023:19:37:05] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2023-06-03T19:37:06.087 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:37:05 smithi164 bash[26855]: cluster 2023-06-03T19:37:04.473147+0000 mgr.y (mgr.14445) 201 : cluster [DBG] pgmap v176: 89 pgs: 89 active+clean; 1.1 GiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 107 MiB/s wr, 1.72k op/s 2023-06-03T19:37:06.484 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 10 16 22102 22086 138.025 134.188 0.0113962 0.00723931 2023-06-03T19:37:07.484 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 11 16 24047 24031 136.528 121.562 0.00896843 0.00731833 2023-06-03T19:37:07.909 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:37:07 smithi089 bash[26572]: cluster 2023-06-03T19:37:06.473589+0000 mgr.y (mgr.14445) 202 : cluster [DBG] pgmap v177: 89 pgs: 89 active+clean; 1.1 GiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 93 MiB/s wr, 1.49k op/s 2023-06-03T19:37:07.909 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:37:07 smithi089 bash[26572]: audit 2023-06-03T19:37:06.666600+0000 mon.c (mon.2) 30 : audit [DBG] from='client.? 172.21.15.89:0/2110305190' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-03T19:37:07.910 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 19:37:07 smithi089 bash[26845]: ::ffff:172.21.15.164 - - [03/Jun/2023:19:37:07] "GET /metrics HTTP/1.1" 200 190819 "" "Prometheus/2.33.4" 2023-06-03T19:37:07.910 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:37:07 smithi089 bash[33173]: cluster 2023-06-03T19:37:06.473589+0000 mgr.y (mgr.14445) 202 : cluster [DBG] pgmap v177: 89 pgs: 89 active+clean; 1.1 GiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 93 MiB/s wr, 1.49k op/s 2023-06-03T19:37:07.910 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:37:07 smithi089 bash[33173]: audit 2023-06-03T19:37:06.666600+0000 mon.c (mon.2) 30 : audit [DBG] from='client.? 172.21.15.89:0/2110305190' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-03T19:37:08.087 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:37:07 smithi164 bash[26855]: cluster 2023-06-03T19:37:06.473589+0000 mgr.y (mgr.14445) 202 : cluster [DBG] pgmap v177: 89 pgs: 89 active+clean; 1.1 GiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 93 MiB/s wr, 1.49k op/s 2023-06-03T19:37:08.087 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:37:07 smithi164 bash[26855]: audit 2023-06-03T19:37:06.666600+0000 mon.c (mon.2) 30 : audit [DBG] from='client.? 172.21.15.89:0/2110305190' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-03T19:37:08.484 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 12 16 25750 25734 134.019 106.438 0.00710916 0.00745535 2023-06-03T19:37:09.484 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 13 16 27667 27651 132.925 119.812 0.00467247 0.00751807 2023-06-03T19:37:10.032 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:37:09 smithi089 bash[26572]: cluster 2023-06-03T19:37:08.474124+0000 mgr.y (mgr.14445) 203 : cluster [DBG] pgmap v178: 89 pgs: 89 active+clean; 1.1 GiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 93 MiB/s wr, 1.49k op/s 2023-06-03T19:37:10.032 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:37:09 smithi089 bash[26572]: audit 2023-06-03T19:37:09.322983+0000 mon.a (mon.0) 759 : audit [DBG] from='client.? 172.21.15.89:0/193018612' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-03T19:37:10.033 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:37:09 smithi089 bash[33173]: cluster 2023-06-03T19:37:08.474124+0000 mgr.y (mgr.14445) 203 : cluster [DBG] pgmap v178: 89 pgs: 89 active+clean; 1.1 GiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 93 MiB/s wr, 1.49k op/s 2023-06-03T19:37:10.033 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:37:09 smithi089 bash[33173]: audit 2023-06-03T19:37:09.322983+0000 mon.a (mon.0) 759 : audit [DBG] from='client.? 172.21.15.89:0/193018612' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-03T19:37:10.087 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:37:09 smithi164 bash[26855]: cluster 2023-06-03T19:37:08.474124+0000 mgr.y (mgr.14445) 203 : cluster [DBG] pgmap v178: 89 pgs: 89 active+clean; 1.1 GiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 93 MiB/s wr, 1.49k op/s 2023-06-03T19:37:10.087 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:37:09 smithi164 bash[26855]: audit 2023-06-03T19:37:09.322983+0000 mon.a (mon.0) 759 : audit [DBG] from='client.? 172.21.15.89:0/193018612' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-03T19:37:10.484 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 14 16 29668 29652 132.363 125.062 0.00549184 0.00754937 2023-06-03T19:37:11.485 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 15 16 31632 31616 131.721 122.75 0.00417595 0.00758532 2023-06-03T19:37:12.032 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:37:11 smithi089 bash[26572]: cluster 2023-06-03T19:37:10.474848+0000 mgr.y (mgr.14445) 204 : cluster [DBG] pgmap v179: 89 pgs: 89 active+clean; 1.7 GiB data, 5.0 GiB used, 710 GiB / 715 GiB avail; 143 MiB/s wr, 2.29k op/s 2023-06-03T19:37:12.032 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:37:11 smithi089 bash[33173]: cluster 2023-06-03T19:37:10.474848+0000 mgr.y (mgr.14445) 204 : cluster [DBG] pgmap v179: 89 pgs: 89 active+clean; 1.7 GiB data, 5.0 GiB used, 710 GiB / 715 GiB avail; 143 MiB/s wr, 2.29k op/s 2023-06-03T19:37:12.087 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:37:11 smithi164 bash[26855]: cluster 2023-06-03T19:37:10.474848+0000 mgr.y (mgr.14445) 204 : cluster [DBG] pgmap v179: 89 pgs: 89 active+clean; 1.7 GiB data, 5.0 GiB used, 710 GiB / 715 GiB avail; 143 MiB/s wr, 2.29k op/s 2023-06-03T19:37:12.485 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 16 16 33447 33431 130.578 113.438 0.00356843 0.00765167 2023-06-03T19:37:13.032 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:37:12 smithi089 bash[26572]: audit 2023-06-03T19:37:11.946064+0000 mon.b (mon.1) 24 : audit [DBG] from='client.? 172.21.15.89:0/808671888' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-03T19:37:13.032 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:37:12 smithi089 bash[33173]: audit 2023-06-03T19:37:11.946064+0000 mon.b (mon.1) 24 : audit [DBG] from='client.? 172.21.15.89:0/808671888' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-03T19:37:13.087 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:37:12 smithi164 bash[26855]: audit 2023-06-03T19:37:11.946064+0000 mon.b (mon.1) 24 : audit [DBG] from='client.? 172.21.15.89:0/808671888' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-03T19:37:13.485 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 17 16 35312 35296 129.753 116.562 0.00358783 0.00770112 2023-06-03T19:37:14.032 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:37:13 smithi089 bash[33173]: cluster 2023-06-03T19:37:12.475317+0000 mgr.y (mgr.14445) 205 : cluster [DBG] pgmap v180: 89 pgs: 89 active+clean; 1.7 GiB data, 5.0 GiB used, 710 GiB / 715 GiB avail; 105 MiB/s wr, 1.68k op/s 2023-06-03T19:37:14.032 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:37:13 smithi089 bash[26572]: cluster 2023-06-03T19:37:12.475317+0000 mgr.y (mgr.14445) 205 : cluster [DBG] pgmap v180: 89 pgs: 89 active+clean; 1.7 GiB data, 5.0 GiB used, 710 GiB / 715 GiB avail; 105 MiB/s wr, 1.68k op/s 2023-06-03T19:37:14.087 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:37:13 smithi164 bash[26855]: cluster 2023-06-03T19:37:12.475317+0000 mgr.y (mgr.14445) 205 : cluster [DBG] pgmap v180: 89 pgs: 89 active+clean; 1.7 GiB data, 5.0 GiB used, 710 GiB / 715 GiB avail; 105 MiB/s wr, 1.68k op/s 2023-06-03T19:37:14.485 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 18 16 37020 37004 128.474 106.75 0.0176294 0.00777701 2023-06-03T19:37:15.032 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:37:14 smithi089 bash[33173]: audit 2023-06-03T19:37:14.560208+0000 mon.a (mon.0) 760 : audit [DBG] from='client.? 172.21.15.89:0/3401072703' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-03T19:37:15.032 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:37:14 smithi089 bash[26572]: audit 2023-06-03T19:37:14.560208+0000 mon.a (mon.0) 760 : audit [DBG] from='client.? 172.21.15.89:0/3401072703' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-03T19:37:15.087 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:37:14 smithi164 bash[26855]: audit 2023-06-03T19:37:14.560208+0000 mon.a (mon.0) 760 : audit [DBG] from='client.? 172.21.15.89:0/3401072703' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-03T19:37:15.485 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 19 16 38774 38758 127.482 109.625 0.00271092 0.00783695 2023-06-03T19:37:16.032 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:37:15 smithi089 bash[26572]: cluster 2023-06-03T19:37:14.476051+0000 mgr.y (mgr.14445) 206 : cluster [DBG] pgmap v181: 89 pgs: 89 active+clean; 2.3 GiB data, 6.8 GiB used, 708 GiB / 715 GiB avail; 154 MiB/s wr, 2.46k op/s 2023-06-03T19:37:16.032 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:37:15 smithi089 bash[33173]: cluster 2023-06-03T19:37:14.476051+0000 mgr.y (mgr.14445) 206 : cluster [DBG] pgmap v181: 89 pgs: 89 active+clean; 2.3 GiB data, 6.8 GiB used, 708 GiB / 715 GiB avail; 154 MiB/s wr, 2.46k op/s 2023-06-03T19:37:16.087 INFO:journalctl@ceph.mgr.x.smithi164.stdout:Jun 03 19:37:15 smithi164 bash[27934]: ::ffff:172.21.15.164 - - [03/Jun/2023:19:37:15] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2023-06-03T19:37:16.088 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:37:15 smithi164 bash[26855]: cluster 2023-06-03T19:37:14.476051+0000 mgr.y (mgr.14445) 206 : cluster [DBG] pgmap v181: 89 pgs: 89 active+clean; 2.3 GiB data, 6.8 GiB used, 708 GiB / 715 GiB avail; 154 MiB/s wr, 2.46k op/s 2023-06-03T19:37:16.485 INFO:tasks.radosbench.radosbench.0.smithi089.stdout:2023-06-03T19:37:16.478299+0000 min lat: 0.00134497 max lat: 0.133514 avg lat: 0.00789772 2023-06-03T19:37:16.486 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-06-03T19:37:16.486 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 20 16 40507 40491 126.523 108.312 0.00963948 0.00789772 2023-06-03T19:37:17.485 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 21 16 42099 42083 125.235 99.5 0.0112148 0.00797847 2023-06-03T19:37:17.909 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 19:37:17 smithi089 bash[26845]: ::ffff:172.21.15.164 - - [03/Jun/2023:19:37:17] "GET /metrics HTTP/1.1" 200 190819 "" "Prometheus/2.33.4" 2023-06-03T19:37:17.909 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:37:17 smithi089 bash[33173]: cluster 2023-06-03T19:37:16.476600+0000 mgr.y (mgr.14445) 207 : cluster [DBG] pgmap v182: 89 pgs: 89 active+clean; 2.3 GiB data, 6.8 GiB used, 708 GiB / 715 GiB avail; 99 MiB/s wr, 1.59k op/s 2023-06-03T19:37:17.909 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:37:17 smithi089 bash[33173]: audit 2023-06-03T19:37:17.209588+0000 mon.c (mon.2) 31 : audit [DBG] from='client.? 172.21.15.89:0/521828343' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-03T19:37:17.910 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:37:17 smithi089 bash[26572]: cluster 2023-06-03T19:37:16.476600+0000 mgr.y (mgr.14445) 207 : cluster [DBG] pgmap v182: 89 pgs: 89 active+clean; 2.3 GiB data, 6.8 GiB used, 708 GiB / 715 GiB avail; 99 MiB/s wr, 1.59k op/s 2023-06-03T19:37:17.910 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:37:17 smithi089 bash[26572]: audit 2023-06-03T19:37:17.209588+0000 mon.c (mon.2) 31 : audit [DBG] from='client.? 172.21.15.89:0/521828343' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-03T19:37:18.087 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:37:17 smithi164 bash[26855]: cluster 2023-06-03T19:37:16.476600+0000 mgr.y (mgr.14445) 207 : cluster [DBG] pgmap v182: 89 pgs: 89 active+clean; 2.3 GiB data, 6.8 GiB used, 708 GiB / 715 GiB avail; 99 MiB/s wr, 1.59k op/s 2023-06-03T19:37:18.087 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:37:17 smithi164 bash[26855]: audit 2023-06-03T19:37:17.209588+0000 mon.c (mon.2) 31 : audit [DBG] from='client.? 172.21.15.89:0/521828343' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-03T19:37:18.485 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 22 16 43744 43728 124.216 102.812 0.00395262 0.00804408 2023-06-03T19:37:19.485 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 23 16 45325 45309 123.111 98.8125 0.0116714 0.00811636 2023-06-03T19:37:20.032 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:37:19 smithi089 bash[33173]: cluster 2023-06-03T19:37:18.477027+0000 mgr.y (mgr.14445) 208 : cluster [DBG] pgmap v183: 89 pgs: 89 active+clean; 2.3 GiB data, 6.8 GiB used, 708 GiB / 715 GiB avail; 99 MiB/s wr, 1.59k op/s 2023-06-03T19:37:20.032 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:37:19 smithi089 bash[26572]: cluster 2023-06-03T19:37:18.477027+0000 mgr.y (mgr.14445) 208 : cluster [DBG] pgmap v183: 89 pgs: 89 active+clean; 2.3 GiB data, 6.8 GiB used, 708 GiB / 715 GiB avail; 99 MiB/s wr, 1.59k op/s 2023-06-03T19:37:20.087 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:37:19 smithi164 bash[26855]: cluster 2023-06-03T19:37:18.477027+0000 mgr.y (mgr.14445) 208 : cluster [DBG] pgmap v183: 89 pgs: 89 active+clean; 2.3 GiB data, 6.8 GiB used, 708 GiB / 715 GiB avail; 99 MiB/s wr, 1.59k op/s 2023-06-03T19:37:20.486 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 24 16 46862 46846 121.984 96.0625 0.00944177 0.00819278 2023-06-03T19:37:21.032 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:37:20 smithi089 bash[33173]: audit 2023-06-03T19:37:19.857001+0000 mon.a (mon.0) 761 : audit [DBG] from='client.? 172.21.15.89:0/3253820892' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-03T19:37:21.032 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:37:20 smithi089 bash[26572]: audit 2023-06-03T19:37:19.857001+0000 mon.a (mon.0) 761 : audit [DBG] from='client.? 172.21.15.89:0/3253820892' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-03T19:37:21.087 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:37:20 smithi164 bash[26855]: audit 2023-06-03T19:37:19.857001+0000 mon.a (mon.0) 761 : audit [DBG] from='client.? 172.21.15.89:0/3253820892' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-03T19:37:21.486 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 25 16 48415 48399 120.986 97.0625 0.0120305 0.00825974 2023-06-03T19:37:22.032 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:37:21 smithi089 bash[26572]: cluster 2023-06-03T19:37:20.477800+0000 mgr.y (mgr.14445) 209 : cluster [DBG] pgmap v184: 89 pgs: 89 active+clean; 2.8 GiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 143 MiB/s wr, 2.28k op/s 2023-06-03T19:37:22.032 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:37:21 smithi089 bash[33173]: cluster 2023-06-03T19:37:20.477800+0000 mgr.y (mgr.14445) 209 : cluster [DBG] pgmap v184: 89 pgs: 89 active+clean; 2.8 GiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 143 MiB/s wr, 2.28k op/s 2023-06-03T19:37:22.087 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:37:21 smithi164 bash[26855]: cluster 2023-06-03T19:37:20.477800+0000 mgr.y (mgr.14445) 209 : cluster [DBG] pgmap v184: 89 pgs: 89 active+clean; 2.8 GiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 143 MiB/s wr, 2.28k op/s 2023-06-03T19:37:22.486 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 26 16 49859 49843 119.804 90.25 0.0161413 0.00834061 2023-06-03T19:37:23.032 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:37:22 smithi089 bash[26572]: audit 2023-06-03T19:37:22.487353+0000 mon.c (mon.2) 32 : audit [DBG] from='client.? 172.21.15.89:0/3979258912' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-03T19:37:23.032 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:37:22 smithi089 bash[33173]: audit 2023-06-03T19:37:22.487353+0000 mon.c (mon.2) 32 : audit [DBG] from='client.? 172.21.15.89:0/3979258912' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-03T19:37:23.087 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:37:22 smithi164 bash[26855]: audit 2023-06-03T19:37:22.487353+0000 mon.c (mon.2) 32 : audit [DBG] from='client.? 172.21.15.89:0/3979258912' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-03T19:37:23.486 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 27 16 51309 51293 118.723 90.625 0.00665528 0.00841651 2023-06-03T19:37:23.801 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:37:23 smithi164 bash[26855]: cluster 2023-06-03T19:37:22.478244+0000 mgr.y (mgr.14445) 210 : cluster [DBG] pgmap v185: 89 pgs: 89 active+clean; 2.8 GiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 92 MiB/s wr, 1.47k op/s 2023-06-03T19:37:24.032 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:37:23 smithi089 bash[33173]: cluster 2023-06-03T19:37:22.478244+0000 mgr.y (mgr.14445) 210 : cluster [DBG] pgmap v185: 89 pgs: 89 active+clean; 2.8 GiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 92 MiB/s wr, 1.47k op/s 2023-06-03T19:37:24.032 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:37:23 smithi089 bash[26572]: cluster 2023-06-03T19:37:22.478244+0000 mgr.y (mgr.14445) 210 : cluster [DBG] pgmap v185: 89 pgs: 89 active+clean; 2.8 GiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 92 MiB/s wr, 1.47k op/s 2023-06-03T19:37:24.486 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 28 16 52669 52653 117.518 85 0.014548 0.00850337 2023-06-03T19:37:25.486 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 29 16 53940 53924 116.205 79.4375 0.0142432 0.00859944 2023-06-03T19:37:26.087 INFO:journalctl@ceph.mgr.x.smithi164.stdout:Jun 03 19:37:25 smithi164 bash[27934]: ::ffff:172.21.15.164 - - [03/Jun/2023:19:37:25] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2023-06-03T19:37:26.087 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:37:25 smithi164 bash[26855]: cluster 2023-06-03T19:37:24.479402+0000 mgr.y (mgr.14445) 211 : cluster [DBG] pgmap v186: 89 pgs: 89 active+clean; 3.2 GiB data, 9.7 GiB used, 705 GiB / 715 GiB avail; 130 MiB/s wr, 2.09k op/s 2023-06-03T19:37:26.087 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:37:25 smithi164 bash[26855]: cephadm 2023-06-03T19:37:24.809046+0000 mgr.y (mgr.14445) 212 : cephadm [INF] Upgrade: Updating mgr.x 2023-06-03T19:37:26.088 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:37:25 smithi164 bash[26855]: audit 2023-06-03T19:37:24.814522+0000 mon.a (mon.0) 762 : audit [INF] from='mgr.14445 172.21.15.89:0/573502486' entity='mgr.y' 2023-06-03T19:37:26.088 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:37:25 smithi164 bash[26855]: audit 2023-06-03T19:37:24.815568+0000 mon.a (mon.0) 763 : audit [INF] from='mgr.14445 172.21.15.89:0/573502486' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-06-03T19:37:26.088 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:37:25 smithi164 bash[26855]: audit 2023-06-03T19:37:24.816608+0000 mon.a (mon.0) 764 : audit [DBG] from='mgr.14445 172.21.15.89:0/573502486' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-06-03T19:37:26.088 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:37:25 smithi164 bash[26855]: audit 2023-06-03T19:37:24.817738+0000 mon.a (mon.0) 765 : audit [DBG] from='mgr.14445 172.21.15.89:0/573502486' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-03T19:37:26.088 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:37:25 smithi164 bash[26855]: cephadm 2023-06-03T19:37:24.818666+0000 mgr.y (mgr.14445) 213 : cephadm [INF] Deploying daemon mgr.x on smithi164 2023-06-03T19:37:26.088 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:37:25 smithi164 bash[26855]: audit 2023-06-03T19:37:25.085970+0000 mon.a (mon.0) 766 : audit [DBG] from='client.? 172.21.15.89:0/2736426489' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-03T19:37:26.282 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:37:25 smithi089 bash[33173]: cluster 2023-06-03T19:37:24.479402+0000 mgr.y (mgr.14445) 211 : cluster [DBG] pgmap v186: 89 pgs: 89 active+clean; 3.2 GiB data, 9.7 GiB used, 705 GiB / 715 GiB avail; 130 MiB/s wr, 2.09k op/s 2023-06-03T19:37:26.282 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:37:25 smithi089 bash[33173]: cephadm 2023-06-03T19:37:24.809046+0000 mgr.y (mgr.14445) 212 : cephadm [INF] Upgrade: Updating mgr.x 2023-06-03T19:37:26.282 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:37:25 smithi089 bash[33173]: audit 2023-06-03T19:37:24.814522+0000 mon.a (mon.0) 762 : audit [INF] from='mgr.14445 172.21.15.89:0/573502486' entity='mgr.y' 2023-06-03T19:37:26.282 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:37:25 smithi089 bash[33173]: audit 2023-06-03T19:37:24.815568+0000 mon.a (mon.0) 763 : audit [INF] from='mgr.14445 172.21.15.89:0/573502486' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-06-03T19:37:26.282 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:37:25 smithi089 bash[33173]: audit 2023-06-03T19:37:24.816608+0000 mon.a (mon.0) 764 : audit [DBG] from='mgr.14445 172.21.15.89:0/573502486' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-06-03T19:37:26.283 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:37:25 smithi089 bash[33173]: audit 2023-06-03T19:37:24.817738+0000 mon.a (mon.0) 765 : audit [DBG] from='mgr.14445 172.21.15.89:0/573502486' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-03T19:37:26.283 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:37:25 smithi089 bash[33173]: cephadm 2023-06-03T19:37:24.818666+0000 mgr.y (mgr.14445) 213 : cephadm [INF] Deploying daemon mgr.x on smithi164 2023-06-03T19:37:26.283 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:37:25 smithi089 bash[33173]: audit 2023-06-03T19:37:25.085970+0000 mon.a (mon.0) 766 : audit [DBG] from='client.? 172.21.15.89:0/2736426489' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-03T19:37:26.283 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:37:25 smithi089 bash[26572]: cluster 2023-06-03T19:37:24.479402+0000 mgr.y (mgr.14445) 211 : cluster [DBG] pgmap v186: 89 pgs: 89 active+clean; 3.2 GiB data, 9.7 GiB used, 705 GiB / 715 GiB avail; 130 MiB/s wr, 2.09k op/s 2023-06-03T19:37:26.283 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:37:25 smithi089 bash[26572]: cephadm 2023-06-03T19:37:24.809046+0000 mgr.y (mgr.14445) 212 : cephadm [INF] Upgrade: Updating mgr.x 2023-06-03T19:37:26.283 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:37:25 smithi089 bash[26572]: audit 2023-06-03T19:37:24.814522+0000 mon.a (mon.0) 762 : audit [INF] from='mgr.14445 172.21.15.89:0/573502486' entity='mgr.y' 2023-06-03T19:37:26.284 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:37:25 smithi089 bash[26572]: audit 2023-06-03T19:37:24.815568+0000 mon.a (mon.0) 763 : audit [INF] from='mgr.14445 172.21.15.89:0/573502486' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-06-03T19:37:26.284 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:37:25 smithi089 bash[26572]: audit 2023-06-03T19:37:24.816608+0000 mon.a (mon.0) 764 : audit [DBG] from='mgr.14445 172.21.15.89:0/573502486' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-06-03T19:37:26.284 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:37:25 smithi089 bash[26572]: audit 2023-06-03T19:37:24.817738+0000 mon.a (mon.0) 765 : audit [DBG] from='mgr.14445 172.21.15.89:0/573502486' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-03T19:37:26.284 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:37:25 smithi089 bash[26572]: cephadm 2023-06-03T19:37:24.818666+0000 mgr.y (mgr.14445) 213 : cephadm [INF] Deploying daemon mgr.x on smithi164 2023-06-03T19:37:26.284 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:37:25 smithi089 bash[26572]: audit 2023-06-03T19:37:25.085970+0000 mon.a (mon.0) 766 : audit [DBG] from='client.? 172.21.15.89:0/2736426489' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-03T19:37:26.486 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 30 16 55283 55267 115.129 83.9375 0.0179724 0.0086805 2023-06-03T19:37:27.486 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 31 16 56604 56588 114.078 82.5625 0.00972887 0.00875977 2023-06-03T19:37:28.087 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:37:27 smithi164 bash[26855]: cluster 2023-06-03T19:37:26.479834+0000 mgr.y (mgr.14445) 214 : cluster [DBG] pgmap v187: 89 pgs: 89 active+clean; 3.2 GiB data, 9.7 GiB used, 705 GiB / 715 GiB avail; 82 MiB/s wr, 1.30k op/s 2023-06-03T19:37:28.087 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:37:27 smithi164 bash[26855]: audit 2023-06-03T19:37:27.690071+0000 mon.a (mon.0) 767 : audit [DBG] from='client.? 172.21.15.89:0/2792340801' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-03T19:37:28.282 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 19:37:27 smithi089 bash[26845]: ::ffff:172.21.15.164 - - [03/Jun/2023:19:37:27] "GET /metrics HTTP/1.1" 200 191002 "" "Prometheus/2.33.4" 2023-06-03T19:37:28.282 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:37:27 smithi089 bash[33173]: cluster 2023-06-03T19:37:26.479834+0000 mgr.y (mgr.14445) 214 : cluster [DBG] pgmap v187: 89 pgs: 89 active+clean; 3.2 GiB data, 9.7 GiB used, 705 GiB / 715 GiB avail; 82 MiB/s wr, 1.30k op/s 2023-06-03T19:37:28.282 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:37:27 smithi089 bash[33173]: audit 2023-06-03T19:37:27.690071+0000 mon.a (mon.0) 767 : audit [DBG] from='client.? 172.21.15.89:0/2792340801' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-03T19:37:28.283 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:37:27 smithi089 bash[26572]: cluster 2023-06-03T19:37:26.479834+0000 mgr.y (mgr.14445) 214 : cluster [DBG] pgmap v187: 89 pgs: 89 active+clean; 3.2 GiB data, 9.7 GiB used, 705 GiB / 715 GiB avail; 82 MiB/s wr, 1.30k op/s 2023-06-03T19:37:28.283 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:37:27 smithi089 bash[26572]: audit 2023-06-03T19:37:27.690071+0000 mon.a (mon.0) 767 : audit [DBG] from='client.? 172.21.15.89:0/2792340801' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-03T19:37:28.411 INFO:journalctl@ceph.mgr.x.smithi164.stdout:Jun 03 19:37:28 smithi164 systemd[1]: Stopping Ceph mgr.x for f1db3478-0243-11ee-9b1d-001a4aab830c... 2023-06-03T19:37:28.411 INFO:journalctl@ceph.mgr.x.smithi164.stdout:Jun 03 19:37:28 smithi164 bash[50733]: Error response from daemon: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-mgr.x 2023-06-03T19:37:28.486 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 32 16 57839 57823 112.925 77.1875 0.00728575 0.00884967 2023-06-03T19:37:29.486 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 33 16 59171 59155 112.026 83.25 0.00604973 0.00892025 2023-06-03T19:37:29.815 INFO:journalctl@ceph.mgr.x.smithi164.stdout:Jun 03 19:37:29 smithi164 bash[50768]: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-mgr-x 2023-06-03T19:37:29.815 INFO:journalctl@ceph.mgr.x.smithi164.stdout:Jun 03 19:37:29 smithi164 systemd[1]: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c@mgr.x.service: Main process exited, code=exited, status=143/n/a 2023-06-03T19:37:29.816 INFO:journalctl@ceph.mgr.x.smithi164.stdout:Jun 03 19:37:29 smithi164 systemd[1]: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c@mgr.x.service: Failed with result 'exit-code'. 2023-06-03T19:37:29.816 INFO:journalctl@ceph.mgr.x.smithi164.stdout:Jun 03 19:37:29 smithi164 systemd[1]: Stopped Ceph mgr.x for f1db3478-0243-11ee-9b1d-001a4aab830c. 2023-06-03T19:37:29.816 INFO:journalctl@ceph.mgr.x.smithi164.stdout:Jun 03 19:37:29 smithi164 systemd[1]: Started Ceph mgr.x for f1db3478-0243-11ee-9b1d-001a4aab830c. 2023-06-03T19:37:30.087 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:37:29 smithi164 bash[26855]: cluster 2023-06-03T19:37:28.480215+0000 mgr.y (mgr.14445) 215 : cluster [DBG] pgmap v188: 89 pgs: 89 active+clean; 3.2 GiB data, 9.7 GiB used, 705 GiB / 715 GiB avail; 82 MiB/s wr, 1.30k op/s 2023-06-03T19:37:30.282 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:37:29 smithi089 bash[26572]: cluster 2023-06-03T19:37:28.480215+0000 mgr.y (mgr.14445) 215 : cluster [DBG] pgmap v188: 89 pgs: 89 active+clean; 3.2 GiB data, 9.7 GiB used, 705 GiB / 715 GiB avail; 82 MiB/s wr, 1.30k op/s 2023-06-03T19:37:30.282 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:37:29 smithi089 bash[33173]: cluster 2023-06-03T19:37:28.480215+0000 mgr.y (mgr.14445) 215 : cluster [DBG] pgmap v188: 89 pgs: 89 active+clean; 3.2 GiB data, 9.7 GiB used, 705 GiB / 715 GiB avail; 82 MiB/s wr, 1.30k op/s 2023-06-03T19:37:30.486 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 34 16 60327 60311 110.856 72.25 0.0121922 0.00901503 2023-06-03T19:37:31.281 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:37:30 smithi089 bash[26572]: audit 2023-06-03T19:37:29.859045+0000 mon.a (mon.0) 768 : audit [INF] from='mgr.14445 172.21.15.89:0/573502486' entity='mgr.y' 2023-06-03T19:37:31.282 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:37:30 smithi089 bash[26572]: audit 2023-06-03T19:37:29.860819+0000 mon.a (mon.0) 769 : audit [DBG] from='mgr.14445 172.21.15.89:0/573502486' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-03T19:37:31.282 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:37:30 smithi089 bash[26572]: audit 2023-06-03T19:37:30.306468+0000 mon.c (mon.2) 33 : audit [DBG] from='client.? 172.21.15.89:0/3283998597' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-03T19:37:31.282 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:37:30 smithi089 bash[26572]: cluster 2023-06-03T19:37:30.480822+0000 mgr.y (mgr.14445) 216 : cluster [DBG] pgmap v189: 89 pgs: 89 active+clean; 3.6 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 115 MiB/s wr, 1.85k op/s 2023-06-03T19:37:31.282 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:37:30 smithi089 bash[33173]: audit 2023-06-03T19:37:29.859045+0000 mon.a (mon.0) 768 : audit [INF] from='mgr.14445 172.21.15.89:0/573502486' entity='mgr.y' 2023-06-03T19:37:31.283 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:37:30 smithi089 bash[33173]: audit 2023-06-03T19:37:29.860819+0000 mon.a (mon.0) 769 : audit [DBG] from='mgr.14445 172.21.15.89:0/573502486' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-03T19:37:31.283 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:37:30 smithi089 bash[33173]: audit 2023-06-03T19:37:30.306468+0000 mon.c (mon.2) 33 : audit [DBG] from='client.? 172.21.15.89:0/3283998597' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-03T19:37:31.283 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:37:30 smithi089 bash[33173]: cluster 2023-06-03T19:37:30.480822+0000 mgr.y (mgr.14445) 216 : cluster [DBG] pgmap v189: 89 pgs: 89 active+clean; 3.6 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 115 MiB/s wr, 1.85k op/s 2023-06-03T19:37:31.287 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:37:30 smithi164 bash[26855]: audit 2023-06-03T19:37:29.859045+0000 mon.a (mon.0) 768 : audit [INF] from='mgr.14445 172.21.15.89:0/573502486' entity='mgr.y' 2023-06-03T19:37:31.287 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:37:30 smithi164 bash[26855]: audit 2023-06-03T19:37:29.860819+0000 mon.a (mon.0) 769 : audit [DBG] from='mgr.14445 172.21.15.89:0/573502486' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-03T19:37:31.288 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:37:30 smithi164 bash[26855]: audit 2023-06-03T19:37:30.306468+0000 mon.c (mon.2) 33 : audit [DBG] from='client.? 172.21.15.89:0/3283998597' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-03T19:37:31.288 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:37:30 smithi164 bash[26855]: cluster 2023-06-03T19:37:30.480822+0000 mgr.y (mgr.14445) 216 : cluster [DBG] pgmap v189: 89 pgs: 89 active+clean; 3.6 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 115 MiB/s wr, 1.85k op/s 2023-06-03T19:37:31.487 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 35 16 61600 61584 109.961 79.5625 0.00818197 0.0090882 2023-06-03T19:37:32.087 INFO:journalctl@ceph.mgr.x.smithi164.stdout:Jun 03 19:37:31 smithi164 bash[50883]: debug 2023-06-03T19:37:31.679+0000 7faa47d8e200 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2023-06-03T19:37:32.486 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 36 16 62734 62718 108.875 70.875 0.00637519 0.00917806 2023-06-03T19:37:33.334 INFO:journalctl@ceph.mgr.x.smithi164.stdout:Jun 03 19:37:33 smithi164 bash[50883]: debug 2023-06-03T19:37:33.011+0000 7faa47d8e200 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2023-06-03T19:37:33.334 INFO:journalctl@ceph.mgr.x.smithi164.stdout:Jun 03 19:37:33 smithi164 bash[50883]: debug 2023-06-03T19:37:33.163+0000 7faa47d8e200 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2023-06-03T19:37:33.334 INFO:journalctl@ceph.mgr.x.smithi164.stdout:Jun 03 19:37:33 smithi164 bash[50883]: debug 2023-06-03T19:37:33.319+0000 7faa47d8e200 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2023-06-03T19:37:33.487 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 37 16 63870 63854 107.852 71 0.0103407 0.00926638 2023-06-03T19:37:34.031 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:37:33 smithi089 bash[33173]: cluster 2023-06-03T19:37:32.481544+0000 mgr.y (mgr.14445) 217 : cluster [DBG] pgmap v190: 89 pgs: 89 active+clean; 3.6 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 72 MiB/s wr, 1.15k op/s 2023-06-03T19:37:34.032 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:37:33 smithi089 bash[33173]: audit 2023-06-03T19:37:32.945714+0000 mon.a (mon.0) 770 : audit [DBG] from='client.? 172.21.15.89:0/1664786268' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-03T19:37:34.032 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:37:33 smithi089 bash[26572]: cluster 2023-06-03T19:37:32.481544+0000 mgr.y (mgr.14445) 217 : cluster [DBG] pgmap v190: 89 pgs: 89 active+clean; 3.6 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 72 MiB/s wr, 1.15k op/s 2023-06-03T19:37:34.032 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:37:33 smithi089 bash[26572]: audit 2023-06-03T19:37:32.945714+0000 mon.a (mon.0) 770 : audit [DBG] from='client.? 172.21.15.89:0/1664786268' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-03T19:37:34.087 INFO:journalctl@ceph.mgr.x.smithi164.stdout:Jun 03 19:37:33 smithi164 bash[50883]: debug 2023-06-03T19:37:33.695+0000 7faa47d8e200 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2023-06-03T19:37:34.087 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:37:33 smithi164 bash[26855]: cluster 2023-06-03T19:37:32.481544+0000 mgr.y (mgr.14445) 217 : cluster [DBG] pgmap v190: 89 pgs: 89 active+clean; 3.6 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 72 MiB/s wr, 1.15k op/s 2023-06-03T19:37:34.088 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:37:33 smithi164 bash[26855]: audit 2023-06-03T19:37:32.945714+0000 mon.a (mon.0) 770 : audit [DBG] from='client.? 172.21.15.89:0/1664786268' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-03T19:37:34.487 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 38 16 65042 65026 106.941 73.25 0.00933414 0.00934547 2023-06-03T19:37:34.587 INFO:journalctl@ceph.mgr.x.smithi164.stdout:Jun 03 19:37:34 smithi164 bash[50883]: debug 2023-06-03T19:37:34.239+0000 7faa47d8e200 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2023-06-03T19:37:34.587 INFO:journalctl@ceph.mgr.x.smithi164.stdout:Jun 03 19:37:34 smithi164 bash[50883]: debug 2023-06-03T19:37:34.475+0000 7faa47d8e200 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2023-06-03T19:37:35.087 INFO:journalctl@ceph.mgr.x.smithi164.stdout:Jun 03 19:37:34 smithi164 bash[50883]: debug 2023-06-03T19:37:34.783+0000 7faa47d8e200 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2023-06-03T19:37:35.487 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 39 16 66085 66069 105.87 65.1875 0.015829 0.00943921 2023-06-03T19:37:36.032 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:37:35 smithi089 bash[26572]: cluster 2023-06-03T19:37:34.482292+0000 mgr.y (mgr.14445) 218 : cluster [DBG] pgmap v191: 89 pgs: 89 active+clean; 4.0 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 103 MiB/s wr, 1.64k op/s 2023-06-03T19:37:36.032 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:37:35 smithi089 bash[33173]: cluster 2023-06-03T19:37:34.482292+0000 mgr.y (mgr.14445) 218 : cluster [DBG] pgmap v191: 89 pgs: 89 active+clean; 4.0 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 103 MiB/s wr, 1.64k op/s 2023-06-03T19:37:36.087 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:37:35 smithi164 bash[26855]: cluster 2023-06-03T19:37:34.482292+0000 mgr.y (mgr.14445) 218 : cluster [DBG] pgmap v191: 89 pgs: 89 active+clean; 4.0 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 103 MiB/s wr, 1.64k op/s 2023-06-03T19:37:36.487 INFO:tasks.radosbench.radosbench.0.smithi089.stdout:2023-06-03T19:37:36.480175+0000 min lat: 0.00134497 max lat: 0.133514 avg lat: 0.00954478 2023-06-03T19:37:36.487 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-06-03T19:37:36.487 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 40 16 67027 67011 104.695 58.875 0.0471006 0.00954478 2023-06-03T19:37:36.587 INFO:journalctl@ceph.mgr.x.smithi164.stdout:Jun 03 19:37:36 smithi164 bash[50883]: debug 2023-06-03T19:37:36.291+0000 7faa47d8e200 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2023-06-03T19:37:36.587 INFO:journalctl@ceph.mgr.x.smithi164.stdout:Jun 03 19:37:36 smithi164 bash[50883]: debug 2023-06-03T19:37:36.471+0000 7faa47d8e200 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2023-06-03T19:37:37.031 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:37:36 smithi089 bash[26572]: audit 2023-06-03T19:37:35.649015+0000 mon.a (mon.0) 771 : audit [DBG] from='client.? 172.21.15.89:0/2979217844' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-03T19:37:37.032 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:37:36 smithi089 bash[26572]: audit 2023-06-03T19:37:35.919513+0000 mon.a (mon.0) 772 : audit [INF] from='mgr.14445 172.21.15.89:0/573502486' entity='mgr.y' 2023-06-03T19:37:37.032 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:37:36 smithi089 bash[33173]: audit 2023-06-03T19:37:35.649015+0000 mon.a (mon.0) 771 : audit [DBG] from='client.? 172.21.15.89:0/2979217844' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-03T19:37:37.032 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:37:36 smithi089 bash[33173]: audit 2023-06-03T19:37:35.919513+0000 mon.a (mon.0) 772 : audit [INF] from='mgr.14445 172.21.15.89:0/573502486' entity='mgr.y' 2023-06-03T19:37:37.087 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:37:36 smithi164 bash[26855]: audit 2023-06-03T19:37:35.649015+0000 mon.a (mon.0) 771 : audit [DBG] from='client.? 172.21.15.89:0/2979217844' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-03T19:37:37.088 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:37:36 smithi164 bash[26855]: audit 2023-06-03T19:37:35.919513+0000 mon.a (mon.0) 772 : audit [INF] from='mgr.14445 172.21.15.89:0/573502486' entity='mgr.y' 2023-06-03T19:37:37.088 INFO:journalctl@ceph.mgr.x.smithi164.stdout:Jun 03 19:37:36 smithi164 bash[50883]: debug 2023-06-03T19:37:36.659+0000 7faa47d8e200 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2023-06-03T19:37:37.487 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 41 16 67932 67916 103.521 56.5625 0.0508928 0.00965194 2023-06-03T19:37:37.836 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:37:37 smithi164 bash[26855]: cluster 2023-06-03T19:37:36.482722+0000 mgr.y (mgr.14445) 219 : cluster [DBG] pgmap v192: 89 pgs: 89 active+clean; 4.0 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 65 MiB/s wr, 1.03k op/s 2023-06-03T19:37:37.837 INFO:journalctl@ceph.mgr.x.smithi164.stdout:Jun 03 19:37:37 smithi164 bash[50883]: debug 2023-06-03T19:37:37.475+0000 7faa47d8e200 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2023-06-03T19:37:37.838 INFO:journalctl@ceph.mgr.x.smithi164.stdout:Jun 03 19:37:37 smithi164 bash[50883]: debug 2023-06-03T19:37:37.819+0000 7faa47d8e200 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2023-06-03T19:37:37.907 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:37:37 smithi089 bash[33173]: cluster 2023-06-03T19:37:36.482722+0000 mgr.y (mgr.14445) 219 : cluster [DBG] pgmap v192: 89 pgs: 89 active+clean; 4.0 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 65 MiB/s wr, 1.03k op/s 2023-06-03T19:37:37.908 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:37:37 smithi089 bash[26572]: cluster 2023-06-03T19:37:36.482722+0000 mgr.y (mgr.14445) 219 : cluster [DBG] pgmap v192: 89 pgs: 89 active+clean; 4.0 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 65 MiB/s wr, 1.03k op/s 2023-06-03T19:37:38.282 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 19:37:37 smithi089 bash[26845]: ::ffff:172.21.15.164 - - [03/Jun/2023:19:37:37] "GET /metrics HTTP/1.1" 200 191054 "" "Prometheus/2.33.4" 2023-06-03T19:37:38.487 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 42 16 68930 68914 102.541 62.375 0.0131032 0.00974651 2023-06-03T19:37:38.785 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:37:38 smithi164 bash[26855]: audit 2023-06-03T19:37:37.776829+0000 mon.a (mon.0) 773 : audit [INF] from='mgr.14445 172.21.15.89:0/573502486' entity='mgr.y' 2023-06-03T19:37:38.785 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:37:38 smithi164 bash[26855]: audit 2023-06-03T19:37:38.155245+0000 mon.a (mon.0) 774 : audit [DBG] from='mgr.14445 172.21.15.89:0/573502486' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-03T19:37:38.785 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:37:38 smithi164 bash[26855]: audit 2023-06-03T19:37:38.156497+0000 mon.a (mon.0) 775 : audit [INF] from='mgr.14445 172.21.15.89:0/573502486' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-03T19:37:38.786 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:37:38 smithi164 bash[26855]: audit 2023-06-03T19:37:38.163537+0000 mon.a (mon.0) 776 : audit [INF] from='mgr.14445 172.21.15.89:0/573502486' entity='mgr.y' 2023-06-03T19:37:38.786 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:37:38 smithi164 bash[26855]: audit 2023-06-03T19:37:38.173647+0000 mon.a (mon.0) 777 : audit [DBG] from='mgr.14445 172.21.15.89:0/573502486' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-03T19:37:38.786 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:37:38 smithi164 bash[26855]: audit 2023-06-03T19:37:38.185848+0000 mon.a (mon.0) 778 : audit [INF] from='mgr.14445 172.21.15.89:0/573502486' entity='mgr.y' cmd=[{"prefix": "mgr fail", "who": "y"}]: dispatch 2023-06-03T19:37:38.786 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:37:38 smithi164 bash[26855]: cluster 2023-06-03T19:37:38.192586+0000 mon.a (mon.0) 779 : cluster [DBG] osdmap e67: 8 total, 8 up, 8 in 2023-06-03T19:37:38.786 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:37:38 smithi164 bash[26855]: audit 2023-06-03T19:37:38.272738+0000 mon.c (mon.2) 34 : audit [DBG] from='client.? 172.21.15.89:0/2602486058' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-03T19:37:38.786 INFO:journalctl@ceph.mgr.x.smithi164.stdout:Jun 03 19:37:38 smithi164 bash[50883]: debug 2023-06-03T19:37:38.359+0000 7faa47d8e200 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2023-06-03T19:37:39.177 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:37:38 smithi089 bash[26572]: audit 2023-06-03T19:37:37.776829+0000 mon.a (mon.0) 773 : audit [INF] from='mgr.14445 172.21.15.89:0/573502486' entity='mgr.y' 2023-06-03T19:37:39.177 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:37:38 smithi089 bash[26572]: audit 2023-06-03T19:37:38.155245+0000 mon.a (mon.0) 774 : audit [DBG] from='mgr.14445 172.21.15.89:0/573502486' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-03T19:37:39.178 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:37:38 smithi089 bash[26572]: audit 2023-06-03T19:37:38.156497+0000 mon.a (mon.0) 775 : audit [INF] from='mgr.14445 172.21.15.89:0/573502486' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-03T19:37:39.178 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:37:38 smithi089 bash[26572]: audit 2023-06-03T19:37:38.163537+0000 mon.a (mon.0) 776 : audit [INF] from='mgr.14445 172.21.15.89:0/573502486' entity='mgr.y' 2023-06-03T19:37:39.178 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:37:38 smithi089 bash[26572]: audit 2023-06-03T19:37:38.173647+0000 mon.a (mon.0) 777 : audit [DBG] from='mgr.14445 172.21.15.89:0/573502486' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-03T19:37:39.178 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:37:38 smithi089 bash[26572]: audit 2023-06-03T19:37:38.185848+0000 mon.a (mon.0) 778 : audit [INF] from='mgr.14445 172.21.15.89:0/573502486' entity='mgr.y' cmd=[{"prefix": "mgr fail", "who": "y"}]: dispatch 2023-06-03T19:37:39.178 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:37:38 smithi089 bash[26572]: cluster 2023-06-03T19:37:38.192586+0000 mon.a (mon.0) 779 : cluster [DBG] osdmap e67: 8 total, 8 up, 8 in 2023-06-03T19:37:39.178 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:37:38 smithi089 bash[26572]: audit 2023-06-03T19:37:38.272738+0000 mon.c (mon.2) 34 : audit [DBG] from='client.? 172.21.15.89:0/2602486058' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-03T19:37:39.179 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 19:37:39 smithi089 bash[26845]: debug 2023-06-03T19:37:39.163+0000 7fc748ee1700 -1 mgr handle_mgr_map I was active but no longer am 2023-06-03T19:37:39.179 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:37:38 smithi089 bash[33173]: audit 2023-06-03T19:37:37.776829+0000 mon.a (mon.0) 773 : audit [INF] from='mgr.14445 172.21.15.89:0/573502486' entity='mgr.y' 2023-06-03T19:37:39.179 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:37:38 smithi089 bash[33173]: audit 2023-06-03T19:37:38.155245+0000 mon.a (mon.0) 774 : audit [DBG] from='mgr.14445 172.21.15.89:0/573502486' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-03T19:37:39.179 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:37:38 smithi089 bash[33173]: audit 2023-06-03T19:37:38.156497+0000 mon.a (mon.0) 775 : audit [INF] from='mgr.14445 172.21.15.89:0/573502486' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-03T19:37:39.179 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:37:38 smithi089 bash[33173]: audit 2023-06-03T19:37:38.163537+0000 mon.a (mon.0) 776 : audit [INF] from='mgr.14445 172.21.15.89:0/573502486' entity='mgr.y' 2023-06-03T19:37:39.179 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:37:38 smithi089 bash[33173]: audit 2023-06-03T19:37:38.173647+0000 mon.a (mon.0) 777 : audit [DBG] from='mgr.14445 172.21.15.89:0/573502486' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-03T19:37:39.180 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:37:38 smithi089 bash[33173]: audit 2023-06-03T19:37:38.185848+0000 mon.a (mon.0) 778 : audit [INF] from='mgr.14445 172.21.15.89:0/573502486' entity='mgr.y' cmd=[{"prefix": "mgr fail", "who": "y"}]: dispatch 2023-06-03T19:37:39.180 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:37:38 smithi089 bash[33173]: cluster 2023-06-03T19:37:38.192586+0000 mon.a (mon.0) 779 : cluster [DBG] osdmap e67: 8 total, 8 up, 8 in 2023-06-03T19:37:39.180 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:37:38 smithi089 bash[33173]: audit 2023-06-03T19:37:38.272738+0000 mon.c (mon.2) 34 : audit [DBG] from='client.? 172.21.15.89:0/2602486058' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-03T19:37:39.487 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 43 15 69998 69983 101.71 66.8125 0.0183206 0.00982644 2023-06-03T19:37:39.520 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 19:37:39 smithi089 bash[26845]: ignoring --setuser ceph since I am not root 2023-06-03T19:37:39.520 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 19:37:39 smithi089 bash[26845]: ignoring --setgroup ceph since I am not root 2023-06-03T19:37:39.521 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 19:37:39 smithi089 bash[26845]: debug 2023-06-03T19:37:39.507+0000 7f4b414aa680 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2023-06-03T19:37:39.587 INFO:journalctl@ceph.mgr.x.smithi164.stdout:Jun 03 19:37:39 smithi164 bash[50883]: debug 2023-06-03T19:37:39.083+0000 7faa47d8e200 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2023-06-03T19:37:39.782 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 19:37:39 smithi089 bash[26845]: debug 2023-06-03T19:37:39.663+0000 7f4b414aa680 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2023-06-03T19:37:39.789 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:37:39 smithi089 bash[26572]: cephadm 2023-06-03T19:37:38.180247+0000 mgr.y (mgr.14445) 220 : cephadm [INF] Upgrade: Need to upgrade myself (mgr.y) 2023-06-03T19:37:39.790 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:37:39 smithi089 bash[26572]: cephadm 2023-06-03T19:37:38.185217+0000 mgr.y (mgr.14445) 221 : cephadm [INF] Failing over to other MGR 2023-06-03T19:37:39.790 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:37:39 smithi089 bash[26572]: cluster 2023-06-03T19:37:38.483177+0000 mgr.y (mgr.14445) 222 : cluster [DBG] pgmap v194: 89 pgs: 89 active+clean; 4.0 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 77 MiB/s wr, 1.24k op/s 2023-06-03T19:37:39.790 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:37:39 smithi089 bash[26572]: cluster 2023-06-03T19:37:39.007272+0000 mon.a (mon.0) 780 : cluster [DBG] Standby manager daemon y started 2023-06-03T19:37:39.790 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:37:39 smithi089 bash[26572]: audit 2023-06-03T19:37:39.176463+0000 mon.a (mon.0) 781 : audit [INF] from='mgr.14445 172.21.15.89:0/573502486' entity='mgr.y' cmd='[{"prefix": "mgr fail", "who": "y"}]': finished 2023-06-03T19:37:39.790 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:37:39 smithi089 bash[26572]: cluster 2023-06-03T19:37:39.176532+0000 mon.a (mon.0) 782 : cluster [DBG] mgrmap e19: x(active, starting, since 0.989549s), standbys: y 2023-06-03T19:37:40.046 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:37:39 smithi164 bash[26855]: cephadm 2023-06-03T19:37:38.180247+0000 mgr.y (mgr.14445) 220 : cephadm [INF] Upgrade: Need to upgrade myself (mgr.y) 2023-06-03T19:37:40.047 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:37:39 smithi164 bash[26855]: cephadm 2023-06-03T19:37:38.185217+0000 mgr.y (mgr.14445) 221 : cephadm [INF] Failing over to other MGR 2023-06-03T19:37:40.047 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:37:39 smithi164 bash[26855]: cluster 2023-06-03T19:37:38.483177+0000 mgr.y (mgr.14445) 222 : cluster [DBG] pgmap v194: 89 pgs: 89 active+clean; 4.0 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 77 MiB/s wr, 1.24k op/s 2023-06-03T19:37:40.047 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:37:39 smithi164 bash[26855]: cluster 2023-06-03T19:37:39.007272+0000 mon.a (mon.0) 780 : cluster [DBG] Standby manager daemon y started 2023-06-03T19:37:40.047 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:37:39 smithi164 bash[26855]: audit 2023-06-03T19:37:39.176463+0000 mon.a (mon.0) 781 : audit [INF] from='mgr.14445 172.21.15.89:0/573502486' entity='mgr.y' cmd='[{"prefix": "mgr fail", "who": "y"}]': finished 2023-06-03T19:37:40.047 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:37:39 smithi164 bash[26855]: cluster 2023-06-03T19:37:39.176532+0000 mon.a (mon.0) 782 : cluster [DBG] mgrmap e19: x(active, starting, since 0.989549s), standbys: y 2023-06-03T19:37:40.048 INFO:journalctl@ceph.mgr.x.smithi164.stdout:Jun 03 19:37:39 smithi164 bash[50883]: debug 2023-06-03T19:37:39.875+0000 7faa47d8e200 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2023-06-03T19:37:40.048 INFO:journalctl@ceph.mgr.x.smithi164.stdout:Jun 03 19:37:40 smithi164 bash[50883]: debug 2023-06-03T19:37:40.031+0000 7faa47d8e200 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2023-06-03T19:37:40.282 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:37:39 smithi089 bash[33173]: cephadm 2023-06-03T19:37:38.180247+0000 mgr.y (mgr.14445) 220 : cephadm [INF] Upgrade: Need to upgrade myself (mgr.y) 2023-06-03T19:37:40.282 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:37:39 smithi089 bash[33173]: cephadm 2023-06-03T19:37:38.185217+0000 mgr.y (mgr.14445) 221 : cephadm [INF] Failing over to other MGR 2023-06-03T19:37:40.282 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:37:39 smithi089 bash[33173]: cluster 2023-06-03T19:37:38.483177+0000 mgr.y (mgr.14445) 222 : cluster [DBG] pgmap v194: 89 pgs: 89 active+clean; 4.0 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 77 MiB/s wr, 1.24k op/s 2023-06-03T19:37:40.282 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:37:39 smithi089 bash[33173]: cluster 2023-06-03T19:37:39.007272+0000 mon.a (mon.0) 780 : cluster [DBG] Standby manager daemon y started 2023-06-03T19:37:40.283 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:37:39 smithi089 bash[33173]: audit 2023-06-03T19:37:39.176463+0000 mon.a (mon.0) 781 : audit [INF] from='mgr.14445 172.21.15.89:0/573502486' entity='mgr.y' cmd='[{"prefix": "mgr fail", "who": "y"}]': finished 2023-06-03T19:37:40.283 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:37:39 smithi089 bash[33173]: cluster 2023-06-03T19:37:39.176532+0000 mon.a (mon.0) 782 : cluster [DBG] mgrmap e19: x(active, starting, since 0.989549s), standbys: y 2023-06-03T19:37:40.487 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 44 16 71027 71011 100.858 64.25 0.012412 0.00990886 2023-06-03T19:37:40.782 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 19:37:40 smithi089 bash[26845]: debug 2023-06-03T19:37:40.387+0000 7f4b414aa680 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2023-06-03T19:37:40.837 INFO:journalctl@ceph.mgr.x.smithi164.stdout:Jun 03 19:37:40 smithi164 bash[50883]: debug 2023-06-03T19:37:40.551+0000 7faa47d8e200 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2023-06-03T19:37:40.838 INFO:journalctl@ceph.mgr.x.smithi164.stdout:Jun 03 19:37:40 smithi164 bash[50883]: debug 2023-06-03T19:37:40.719+0000 7faa47d8e200 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2023-06-03T19:37:41.282 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:37:40 smithi089 bash[33173]: audit 2023-06-03T19:37:40.915001+0000 mon.c (mon.2) 35 : audit [DBG] from='client.? 172.21.15.89:0/81352651' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-03T19:37:41.282 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:37:40 smithi089 bash[26572]: audit 2023-06-03T19:37:40.915001+0000 mon.c (mon.2) 35 : audit [DBG] from='client.? 172.21.15.89:0/81352651' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-03T19:37:41.304 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:37:40 smithi164 bash[26855]: audit 2023-06-03T19:37:40.915001+0000 mon.c (mon.2) 35 : audit [DBG] from='client.? 172.21.15.89:0/81352651' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-03T19:37:41.305 INFO:journalctl@ceph.mgr.x.smithi164.stdout:Jun 03 19:37:40 smithi164 bash[50883]: debug 2023-06-03T19:37:40.931+0000 7faa47d8e200 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2023-06-03T19:37:41.305 INFO:journalctl@ceph.mgr.x.smithi164.stdout:Jun 03 19:37:41 smithi164 bash[50883]: debug 2023-06-03T19:37:41.103+0000 7faa47d8e200 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2023-06-03T19:37:41.305 INFO:journalctl@ceph.mgr.x.smithi164.stdout:Jun 03 19:37:41 smithi164 bash[50883]: debug 2023-06-03T19:37:41.291+0000 7faa47d8e200 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2023-06-03T19:37:41.487 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 45 15 72010 71995 99.9837 61.5 0.00917934 0.00999675 2023-06-03T19:37:42.032 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 19:37:41 smithi089 bash[26845]: debug 2023-06-03T19:37:41.763+0000 7f4b414aa680 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2023-06-03T19:37:42.032 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 19:37:41 smithi089 bash[26845]: debug 2023-06-03T19:37:41.907+0000 7f4b414aa680 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2023-06-03T19:37:42.087 INFO:journalctl@ceph.mgr.x.smithi164.stdout:Jun 03 19:37:41 smithi164 bash[50883]: debug 2023-06-03T19:37:41.575+0000 7faa47d8e200 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2023-06-03T19:37:42.087 INFO:journalctl@ceph.mgr.x.smithi164.stdout:Jun 03 19:37:41 smithi164 bash[50883]: debug 2023-06-03T19:37:41.747+0000 7faa47d8e200 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2023-06-03T19:37:42.487 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 46 16 73008 72992 99.1647 62.3125 0.0262776 0.0100783 2023-06-03T19:37:42.532 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 19:37:42 smithi089 bash[26845]: debug 2023-06-03T19:37:42.263+0000 7f4b414aa680 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2023-06-03T19:37:43.487 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 47 16 73999 73983 98.3725 61.9375 0.0153646 0.0101598 2023-06-03T19:37:43.837 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:37:43 smithi164 bash[26855]: cluster 2023-06-03T19:37:43.511120+0000 mon.a (mon.0) 783 : cluster [INF] Active manager daemon x restarted 2023-06-03T19:37:43.838 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:37:43 smithi164 bash[26855]: cluster 2023-06-03T19:37:43.511979+0000 mon.a (mon.0) 784 : cluster [INF] Activating manager daemon x 2023-06-03T19:37:43.838 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:37:43 smithi164 bash[26855]: audit 2023-06-03T19:37:43.519251+0000 mon.b (mon.1) 25 : audit [DBG] from='mgr.? 172.21.15.164:0/803880945' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-06-03T19:37:43.838 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:37:43 smithi164 bash[26855]: cluster 2023-06-03T19:37:43.521084+0000 mon.a (mon.0) 785 : cluster [DBG] osdmap e68: 8 total, 8 up, 8 in 2023-06-03T19:37:43.838 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:37:43 smithi164 bash[26855]: audit 2023-06-03T19:37:43.527408+0000 mon.b (mon.1) 26 : audit [DBG] from='mgr.? 172.21.15.164:0/803880945' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-06-03T19:37:43.838 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:37:43 smithi164 bash[26855]: audit 2023-06-03T19:37:43.528591+0000 mon.b (mon.1) 27 : audit [DBG] from='mgr.? 172.21.15.164:0/803880945' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-06-03T19:37:43.838 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:37:43 smithi164 bash[26855]: audit 2023-06-03T19:37:43.529066+0000 mon.b (mon.1) 28 : audit [DBG] from='mgr.? 172.21.15.164:0/803880945' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-06-03T19:37:43.838 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:37:43 smithi164 bash[26855]: audit 2023-06-03T19:37:43.529667+0000 mon.c (mon.2) 36 : audit [DBG] from='client.? 172.21.15.89:0/3055317719' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-03T19:37:43.839 INFO:journalctl@ceph.mgr.x.smithi164.stdout:Jun 03 19:37:43 smithi164 bash[50883]: debug 2023-06-03T19:37:43.483+0000 7faa47d8e200 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2023-06-03T19:37:43.839 INFO:journalctl@ceph.mgr.x.smithi164.stdout:Jun 03 19:37:43 smithi164 bash[50883]: [03/Jun/2023:19:37:43] ENGINE Bus STARTING 2023-06-03T19:37:43.839 INFO:journalctl@ceph.mgr.x.smithi164.stdout:Jun 03 19:37:43 smithi164 bash[50883]: CherryPy Checker: 2023-06-03T19:37:43.839 INFO:journalctl@ceph.mgr.x.smithi164.stdout:Jun 03 19:37:43 smithi164 bash[50883]: The Application mounted at '' has an empty config. 2023-06-03T19:37:43.839 INFO:journalctl@ceph.mgr.x.smithi164.stdout:Jun 03 19:37:43 smithi164 bash[50883]: [03/Jun/2023:19:37:43] ENGINE Serving on http://:::9283 2023-06-03T19:37:43.840 INFO:journalctl@ceph.mgr.x.smithi164.stdout:Jun 03 19:37:43 smithi164 bash[50883]: [03/Jun/2023:19:37:43] ENGINE Bus STARTED 2023-06-03T19:37:43.840 INFO:journalctl@ceph.mgr.x.smithi164.stdout:Jun 03 19:37:43 smithi164 bash[50883]: [03/Jun/2023:19:37:43] ENGINE Bus STOPPING 2023-06-03T19:37:43.840 INFO:journalctl@ceph.mgr.x.smithi164.stdout:Jun 03 19:37:43 smithi164 bash[50883]: [03/Jun/2023:19:37:43] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2023-06-03T19:37:43.840 INFO:journalctl@ceph.mgr.x.smithi164.stdout:Jun 03 19:37:43 smithi164 bash[50883]: [03/Jun/2023:19:37:43] ENGINE Bus STOPPED 2023-06-03T19:37:43.993 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:37:43 smithi089 bash[26572]: cluster 2023-06-03T19:37:43.511120+0000 mon.a (mon.0) 783 : cluster [INF] Active manager daemon x restarted 2023-06-03T19:37:43.993 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:37:43 smithi089 bash[26572]: cluster 2023-06-03T19:37:43.511979+0000 mon.a (mon.0) 784 : cluster [INF] Activating manager daemon x 2023-06-03T19:37:43.993 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:37:43 smithi089 bash[26572]: audit 2023-06-03T19:37:43.519251+0000 mon.b (mon.1) 25 : audit [DBG] from='mgr.? 172.21.15.164:0/803880945' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-06-03T19:37:43.993 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:37:43 smithi089 bash[26572]: cluster 2023-06-03T19:37:43.521084+0000 mon.a (mon.0) 785 : cluster [DBG] osdmap e68: 8 total, 8 up, 8 in 2023-06-03T19:37:43.994 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:37:43 smithi089 bash[26572]: audit 2023-06-03T19:37:43.527408+0000 mon.b (mon.1) 26 : audit [DBG] from='mgr.? 172.21.15.164:0/803880945' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-06-03T19:37:43.994 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:37:43 smithi089 bash[26572]: audit 2023-06-03T19:37:43.528591+0000 mon.b (mon.1) 27 : audit [DBG] from='mgr.? 172.21.15.164:0/803880945' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-06-03T19:37:43.994 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:37:43 smithi089 bash[26572]: audit 2023-06-03T19:37:43.529066+0000 mon.b (mon.1) 28 : audit [DBG] from='mgr.? 172.21.15.164:0/803880945' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-06-03T19:37:43.994 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:37:43 smithi089 bash[26572]: audit 2023-06-03T19:37:43.529667+0000 mon.c (mon.2) 36 : audit [DBG] from='client.? 172.21.15.89:0/3055317719' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-03T19:37:43.994 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:37:43 smithi089 bash[33173]: cluster 2023-06-03T19:37:43.511120+0000 mon.a (mon.0) 783 : cluster [INF] Active manager daemon x restarted 2023-06-03T19:37:43.994 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:37:43 smithi089 bash[33173]: cluster 2023-06-03T19:37:43.511979+0000 mon.a (mon.0) 784 : cluster [INF] Activating manager daemon x 2023-06-03T19:37:43.995 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:37:43 smithi089 bash[33173]: audit 2023-06-03T19:37:43.519251+0000 mon.b (mon.1) 25 : audit [DBG] from='mgr.? 172.21.15.164:0/803880945' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-06-03T19:37:43.995 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:37:43 smithi089 bash[33173]: cluster 2023-06-03T19:37:43.521084+0000 mon.a (mon.0) 785 : cluster [DBG] osdmap e68: 8 total, 8 up, 8 in 2023-06-03T19:37:43.995 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:37:43 smithi089 bash[33173]: audit 2023-06-03T19:37:43.527408+0000 mon.b (mon.1) 26 : audit [DBG] from='mgr.? 172.21.15.164:0/803880945' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-06-03T19:37:43.995 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:37:43 smithi089 bash[33173]: audit 2023-06-03T19:37:43.528591+0000 mon.b (mon.1) 27 : audit [DBG] from='mgr.? 172.21.15.164:0/803880945' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-06-03T19:37:43.995 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:37:43 smithi089 bash[33173]: audit 2023-06-03T19:37:43.529066+0000 mon.b (mon.1) 28 : audit [DBG] from='mgr.? 172.21.15.164:0/803880945' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-06-03T19:37:43.995 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:37:43 smithi089 bash[33173]: audit 2023-06-03T19:37:43.529667+0000 mon.c (mon.2) 36 : audit [DBG] from='client.? 172.21.15.89:0/3055317719' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-03T19:37:44.191 INFO:journalctl@ceph.mgr.x.smithi164.stdout:Jun 03 19:37:43 smithi164 bash[50883]: [03/Jun/2023:19:37:43] ENGINE Bus STARTING 2023-06-03T19:37:44.191 INFO:journalctl@ceph.mgr.x.smithi164.stdout:Jun 03 19:37:44 smithi164 bash[50883]: [03/Jun/2023:19:37:44] ENGINE Serving on http://:::9283 2023-06-03T19:37:44.191 INFO:journalctl@ceph.mgr.x.smithi164.stdout:Jun 03 19:37:44 smithi164 bash[50883]: [03/Jun/2023:19:37:44] ENGINE Bus STARTED 2023-06-03T19:37:44.191 INFO:journalctl@ceph.mgr.x.smithi164.stdout:Jun 03 19:37:44 smithi164 bash[50883]: debug 2023-06-03T19:37:44.135+0000 7fa9abc79700 -1 client.0 error registering admin socket command: (17) File exists 2023-06-03T19:37:44.192 INFO:journalctl@ceph.mgr.x.smithi164.stdout:Jun 03 19:37:44 smithi164 bash[50883]: debug 2023-06-03T19:37:44.135+0000 7fa9abc79700 -1 client.0 error registering admin socket command: (17) File exists 2023-06-03T19:37:44.192 INFO:journalctl@ceph.mgr.x.smithi164.stdout:Jun 03 19:37:44 smithi164 bash[50883]: debug 2023-06-03T19:37:44.135+0000 7fa9abc79700 -1 client.0 error registering admin socket command: (17) File exists 2023-06-03T19:37:44.192 INFO:journalctl@ceph.mgr.x.smithi164.stdout:Jun 03 19:37:44 smithi164 bash[50883]: debug 2023-06-03T19:37:44.135+0000 7fa9abc79700 -1 client.0 error registering admin socket command: (17) File exists 2023-06-03T19:37:44.192 INFO:journalctl@ceph.mgr.x.smithi164.stdout:Jun 03 19:37:44 smithi164 bash[50883]: debug 2023-06-03T19:37:44.135+0000 7fa9abc79700 -1 client.0 error registering admin socket command: (17) File exists 2023-06-03T19:37:44.192 INFO:journalctl@ceph.mgr.x.smithi164.stdout:Jun 03 19:37:44 smithi164 bash[50883]: debug 2023-06-03T19:37:44.139+0000 7fa9a7c71700 -1 client.0 error registering admin socket command: (17) File exists 2023-06-03T19:37:44.192 INFO:journalctl@ceph.mgr.x.smithi164.stdout:Jun 03 19:37:44 smithi164 bash[50883]: debug 2023-06-03T19:37:44.139+0000 7fa9a7c71700 -1 client.0 error registering admin socket command: (17) File exists 2023-06-03T19:37:44.193 INFO:journalctl@ceph.mgr.x.smithi164.stdout:Jun 03 19:37:44 smithi164 bash[50883]: debug 2023-06-03T19:37:44.139+0000 7fa9a7c71700 -1 client.0 error registering admin socket command: (17) File exists 2023-06-03T19:37:44.193 INFO:journalctl@ceph.mgr.x.smithi164.stdout:Jun 03 19:37:44 smithi164 bash[50883]: debug 2023-06-03T19:37:44.139+0000 7fa9a7c71700 -1 client.0 error registering admin socket command: (17) File exists 2023-06-03T19:37:44.193 INFO:journalctl@ceph.mgr.x.smithi164.stdout:Jun 03 19:37:44 smithi164 bash[50883]: debug 2023-06-03T19:37:44.139+0000 7fa9a7c71700 -1 client.0 error registering admin socket command: (17) File exists 2023-06-03T19:37:44.242 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 19:37:43 smithi089 bash[26845]: debug 2023-06-03T19:37:43.983+0000 7f4b414aa680 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2023-06-03T19:37:44.488 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 48 16 74939 74923 97.5469 58.75 0.00534925 0.0102449 2023-06-03T19:37:44.532 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 19:37:44 smithi089 bash[26845]: debug 2023-06-03T19:37:44.231+0000 7f4b414aa680 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2023-06-03T19:37:44.532 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 19:37:44 smithi089 bash[26845]: debug 2023-06-03T19:37:44.375+0000 7f4b414aa680 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2023-06-03T19:37:44.886 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 19:37:44 smithi089 bash[26845]: debug 2023-06-03T19:37:44.547+0000 7f4b414aa680 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2023-06-03T19:37:44.887 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 19:37:44 smithi089 bash[26845]: debug 2023-06-03T19:37:44.875+0000 7f4b414aa680 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2023-06-03T19:37:44.887 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:37:44 smithi089 bash[33173]: cluster 2023-06-03T19:37:43.577127+0000 mon.a (mon.0) 786 : cluster [DBG] mgrmap e20: x(active, starting, since 0.0653959s), standbys: y 2023-06-03T19:37:44.888 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:37:44 smithi089 bash[33173]: audit 2023-06-03T19:37:43.588855+0000 mon.a (mon.0) 787 : audit [DBG] from='mgr.24634 172.21.15.164:0/803880945' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-06-03T19:37:44.888 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:37:44 smithi089 bash[33173]: audit 2023-06-03T19:37:43.589161+0000 mon.a (mon.0) 788 : audit [DBG] from='mgr.24634 172.21.15.164:0/803880945' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-06-03T19:37:44.888 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:37:44 smithi089 bash[33173]: audit 2023-06-03T19:37:43.589341+0000 mon.a (mon.0) 789 : audit [DBG] from='mgr.24634 172.21.15.164:0/803880945' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-06-03T19:37:44.888 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:37:44 smithi089 bash[33173]: audit 2023-06-03T19:37:43.589525+0000 mon.a (mon.0) 790 : audit [DBG] from='mgr.24634 172.21.15.164:0/803880945' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "foo.smithi089.giuxuj"}]: dispatch 2023-06-03T19:37:44.888 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:37:44 smithi089 bash[33173]: audit 2023-06-03T19:37:43.589855+0000 mon.a (mon.0) 791 : audit [DBG] from='mgr.24634 172.21.15.164:0/803880945' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "foo.smithi164.txgfgv"}]: dispatch 2023-06-03T19:37:44.888 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:37:44 smithi089 bash[33173]: audit 2023-06-03T19:37:43.592236+0000 mon.a (mon.0) 792 : audit [DBG] from='mgr.24634 172.21.15.164:0/803880945' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2023-06-03T19:37:44.888 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:37:44 smithi089 bash[33173]: audit 2023-06-03T19:37:43.592426+0000 mon.a (mon.0) 793 : audit [DBG] from='mgr.24634 172.21.15.164:0/803880945' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2023-06-03T19:37:44.889 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:37:44 smithi089 bash[33173]: audit 2023-06-03T19:37:43.594539+0000 mon.a (mon.0) 794 : audit [DBG] from='mgr.24634 172.21.15.164:0/803880945' entity='mgr.x' cmd=[{"prefix": "mds metadata"}]: dispatch 2023-06-03T19:37:44.889 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:37:44 smithi089 bash[33173]: audit 2023-06-03T19:37:43.594880+0000 mon.a (mon.0) 795 : audit [DBG] from='mgr.24634 172.21.15.164:0/803880945' entity='mgr.x' cmd=[{"prefix": "osd metadata"}]: dispatch 2023-06-03T19:37:44.889 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:37:44 smithi089 bash[33173]: audit 2023-06-03T19:37:43.595661+0000 mon.a (mon.0) 796 : audit [DBG] from='mgr.24634 172.21.15.164:0/803880945' entity='mgr.x' cmd=[{"prefix": "mon metadata"}]: dispatch 2023-06-03T19:37:44.889 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:37:44 smithi089 bash[33173]: audit 2023-06-03T19:37:43.598820+0000 mon.a (mon.0) 797 : audit [DBG] from='mgr.24634 172.21.15.164:0/803880945' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-06-03T19:37:44.889 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:37:44 smithi089 bash[33173]: audit 2023-06-03T19:37:43.599179+0000 mon.a (mon.0) 798 : audit [DBG] from='mgr.24634 172.21.15.164:0/803880945' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-06-03T19:37:44.889 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:37:44 smithi089 bash[33173]: audit 2023-06-03T19:37:43.599460+0000 mon.a (mon.0) 799 : audit [DBG] from='mgr.24634 172.21.15.164:0/803880945' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-06-03T19:37:44.889 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:37:44 smithi089 bash[33173]: audit 2023-06-03T19:37:43.599742+0000 mon.a (mon.0) 800 : audit [DBG] from='mgr.24634 172.21.15.164:0/803880945' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-06-03T19:37:44.889 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:37:44 smithi089 bash[33173]: audit 2023-06-03T19:37:43.600017+0000 mon.a (mon.0) 801 : audit [DBG] from='mgr.24634 172.21.15.164:0/803880945' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-06-03T19:37:44.890 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:37:44 smithi089 bash[33173]: audit 2023-06-03T19:37:43.600292+0000 mon.a (mon.0) 802 : audit [DBG] from='mgr.24634 172.21.15.164:0/803880945' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-06-03T19:37:44.890 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:37:44 smithi089 bash[33173]: audit 2023-06-03T19:37:43.600740+0000 mon.a (mon.0) 803 : audit [DBG] from='mgr.24634 172.21.15.164:0/803880945' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-06-03T19:37:44.890 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:37:44 smithi089 bash[33173]: audit 2023-06-03T19:37:43.601055+0000 mon.a (mon.0) 804 : audit [DBG] from='mgr.24634 172.21.15.164:0/803880945' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-06-03T19:37:44.890 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:37:44 smithi089 bash[33173]: cluster 2023-06-03T19:37:43.732858+0000 mon.a (mon.0) 805 : cluster [INF] Manager daemon x is now available 2023-06-03T19:37:44.890 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:37:44 smithi089 bash[33173]: audit 2023-06-03T19:37:43.789924+0000 mon.a (mon.0) 806 : audit [INF] from='mgr.24634 172.21.15.164:0/803880945' entity='mgr.x' 2023-06-03T19:37:44.890 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:37:44 smithi089 bash[33173]: audit 2023-06-03T19:37:43.798997+0000 mon.a (mon.0) 807 : audit [DBG] from='mgr.24634 172.21.15.164:0/803880945' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-03T19:37:44.890 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:37:44 smithi089 bash[33173]: audit 2023-06-03T19:37:43.808562+0000 mon.a (mon.0) 808 : audit [DBG] from='mgr.24634 172.21.15.164:0/803880945' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-03T19:37:44.891 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:37:44 smithi089 bash[33173]: audit 2023-06-03T19:37:43.873549+0000 mon.a (mon.0) 809 : audit [INF] from='mgr.24634 172.21.15.164:0/803880945' entity='mgr.x' cmd=[{"prefix": "osd pool rename", "format": "json", "srcpool": "device_health_metrics", "destpool": ".mgr", "yes_i_really_mean_it": true}]: dispatch 2023-06-03T19:37:44.891 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:37:44 smithi089 bash[33173]: audit 2023-06-03T19:37:43.907341+0000 mon.a (mon.0) 810 : audit [DBG] from='mgr.24634 172.21.15.164:0/803880945' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T19:37:44.891 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:37:44 smithi089 bash[33173]: audit 2023-06-03T19:37:43.971506+0000 mon.a (mon.0) 811 : audit [INF] from='mgr.24634 172.21.15.164:0/803880945' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2023-06-03T19:37:44.891 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:37:44 smithi089 bash[33173]: audit 2023-06-03T19:37:43.999062+0000 mon.a (mon.0) 812 : audit [INF] from='mgr.24634 172.21.15.164:0/803880945' entity='mgr.x' 2023-06-03T19:37:44.891 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:37:44 smithi089 bash[33173]: audit 2023-06-03T19:37:44.114945+0000 mon.a (mon.0) 813 : audit [INF] from='mgr.24634 172.21.15.164:0/803880945' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2023-06-03T19:37:44.891 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:37:44 smithi089 bash[33173]: audit 2023-06-03T19:37:44.153897+0000 mon.a (mon.0) 814 : audit [INF] from='mgr.24634 172.21.15.164:0/803880945' entity='mgr.x' 2023-06-03T19:37:44.891 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:37:44 smithi089 bash[33173]: audit 2023-06-03T19:37:44.171332+0000 mon.a (mon.0) 815 : audit [INF] from='mgr.24634 172.21.15.164:0/803880945' entity='mgr.x' 2023-06-03T19:37:44.892 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:37:44 smithi089 bash[26572]: cluster 2023-06-03T19:37:43.577127+0000 mon.a (mon.0) 786 : cluster [DBG] mgrmap e20: x(active, starting, since 0.0653959s), standbys: y 2023-06-03T19:37:44.892 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:37:44 smithi089 bash[26572]: audit 2023-06-03T19:37:43.588855+0000 mon.a (mon.0) 787 : audit [DBG] from='mgr.24634 172.21.15.164:0/803880945' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-06-03T19:37:44.892 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:37:44 smithi089 bash[26572]: audit 2023-06-03T19:37:43.589161+0000 mon.a (mon.0) 788 : audit [DBG] from='mgr.24634 172.21.15.164:0/803880945' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-06-03T19:37:44.892 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:37:44 smithi089 bash[26572]: audit 2023-06-03T19:37:43.589341+0000 mon.a (mon.0) 789 : audit [DBG] from='mgr.24634 172.21.15.164:0/803880945' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-06-03T19:37:44.893 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:37:44 smithi089 bash[26572]: audit 2023-06-03T19:37:43.589525+0000 mon.a (mon.0) 790 : audit [DBG] from='mgr.24634 172.21.15.164:0/803880945' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "foo.smithi089.giuxuj"}]: dispatch 2023-06-03T19:37:44.893 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:37:44 smithi089 bash[26572]: audit 2023-06-03T19:37:43.589855+0000 mon.a (mon.0) 791 : audit [DBG] from='mgr.24634 172.21.15.164:0/803880945' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "foo.smithi164.txgfgv"}]: dispatch 2023-06-03T19:37:44.893 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:37:44 smithi089 bash[26572]: audit 2023-06-03T19:37:43.592236+0000 mon.a (mon.0) 792 : audit [DBG] from='mgr.24634 172.21.15.164:0/803880945' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2023-06-03T19:37:44.893 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:37:44 smithi089 bash[26572]: audit 2023-06-03T19:37:43.592426+0000 mon.a (mon.0) 793 : audit [DBG] from='mgr.24634 172.21.15.164:0/803880945' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2023-06-03T19:37:44.893 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:37:44 smithi089 bash[26572]: audit 2023-06-03T19:37:43.594539+0000 mon.a (mon.0) 794 : audit [DBG] from='mgr.24634 172.21.15.164:0/803880945' entity='mgr.x' cmd=[{"prefix": "mds metadata"}]: dispatch 2023-06-03T19:37:44.893 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:37:44 smithi089 bash[26572]: audit 2023-06-03T19:37:43.594880+0000 mon.a (mon.0) 795 : audit [DBG] from='mgr.24634 172.21.15.164:0/803880945' entity='mgr.x' cmd=[{"prefix": "osd metadata"}]: dispatch 2023-06-03T19:37:44.893 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:37:44 smithi089 bash[26572]: audit 2023-06-03T19:37:43.595661+0000 mon.a (mon.0) 796 : audit [DBG] from='mgr.24634 172.21.15.164:0/803880945' entity='mgr.x' cmd=[{"prefix": "mon metadata"}]: dispatch 2023-06-03T19:37:44.894 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:37:44 smithi089 bash[26572]: audit 2023-06-03T19:37:43.598820+0000 mon.a (mon.0) 797 : audit [DBG] from='mgr.24634 172.21.15.164:0/803880945' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-06-03T19:37:44.894 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:37:44 smithi089 bash[26572]: audit 2023-06-03T19:37:43.599179+0000 mon.a (mon.0) 798 : audit [DBG] from='mgr.24634 172.21.15.164:0/803880945' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-06-03T19:37:44.894 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:37:44 smithi089 bash[26572]: audit 2023-06-03T19:37:43.599460+0000 mon.a (mon.0) 799 : audit [DBG] from='mgr.24634 172.21.15.164:0/803880945' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-06-03T19:37:44.894 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:37:44 smithi089 bash[26572]: audit 2023-06-03T19:37:43.599742+0000 mon.a (mon.0) 800 : audit [DBG] from='mgr.24634 172.21.15.164:0/803880945' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-06-03T19:37:44.894 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:37:44 smithi089 bash[26572]: audit 2023-06-03T19:37:43.600017+0000 mon.a (mon.0) 801 : audit [DBG] from='mgr.24634 172.21.15.164:0/803880945' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-06-03T19:37:44.894 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:37:44 smithi089 bash[26572]: audit 2023-06-03T19:37:43.600292+0000 mon.a (mon.0) 802 : audit [DBG] from='mgr.24634 172.21.15.164:0/803880945' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-06-03T19:37:44.894 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:37:44 smithi089 bash[26572]: audit 2023-06-03T19:37:43.600740+0000 mon.a (mon.0) 803 : audit [DBG] from='mgr.24634 172.21.15.164:0/803880945' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-06-03T19:37:44.894 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:37:44 smithi089 bash[26572]: audit 2023-06-03T19:37:43.601055+0000 mon.a (mon.0) 804 : audit [DBG] from='mgr.24634 172.21.15.164:0/803880945' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-06-03T19:37:44.895 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:37:44 smithi089 bash[26572]: cluster 2023-06-03T19:37:43.732858+0000 mon.a (mon.0) 805 : cluster [INF] Manager daemon x is now available 2023-06-03T19:37:44.895 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:37:44 smithi089 bash[26572]: audit 2023-06-03T19:37:43.789924+0000 mon.a (mon.0) 806 : audit [INF] from='mgr.24634 172.21.15.164:0/803880945' entity='mgr.x' 2023-06-03T19:37:44.895 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:37:44 smithi089 bash[26572]: audit 2023-06-03T19:37:43.798997+0000 mon.a (mon.0) 807 : audit [DBG] from='mgr.24634 172.21.15.164:0/803880945' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-03T19:37:44.895 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:37:44 smithi089 bash[26572]: audit 2023-06-03T19:37:43.808562+0000 mon.a (mon.0) 808 : audit [DBG] from='mgr.24634 172.21.15.164:0/803880945' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-03T19:37:44.895 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:37:44 smithi089 bash[26572]: audit 2023-06-03T19:37:43.873549+0000 mon.a (mon.0) 809 : audit [INF] from='mgr.24634 172.21.15.164:0/803880945' entity='mgr.x' cmd=[{"prefix": "osd pool rename", "format": "json", "srcpool": "device_health_metrics", "destpool": ".mgr", "yes_i_really_mean_it": true}]: dispatch 2023-06-03T19:37:44.895 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:37:44 smithi089 bash[26572]: audit 2023-06-03T19:37:43.907341+0000 mon.a (mon.0) 810 : audit [DBG] from='mgr.24634 172.21.15.164:0/803880945' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T19:37:44.895 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:37:44 smithi089 bash[26572]: audit 2023-06-03T19:37:43.971506+0000 mon.a (mon.0) 811 : audit [INF] from='mgr.24634 172.21.15.164:0/803880945' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2023-06-03T19:37:44.896 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:37:44 smithi089 bash[26572]: audit 2023-06-03T19:37:43.999062+0000 mon.a (mon.0) 812 : audit [INF] from='mgr.24634 172.21.15.164:0/803880945' entity='mgr.x' 2023-06-03T19:37:44.896 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:37:44 smithi089 bash[26572]: audit 2023-06-03T19:37:44.114945+0000 mon.a (mon.0) 813 : audit [INF] from='mgr.24634 172.21.15.164:0/803880945' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2023-06-03T19:37:44.896 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:37:44 smithi089 bash[26572]: audit 2023-06-03T19:37:44.153897+0000 mon.a (mon.0) 814 : audit [INF] from='mgr.24634 172.21.15.164:0/803880945' entity='mgr.x' 2023-06-03T19:37:44.896 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:37:44 smithi089 bash[26572]: audit 2023-06-03T19:37:44.171332+0000 mon.a (mon.0) 815 : audit [INF] from='mgr.24634 172.21.15.164:0/803880945' entity='mgr.x' 2023-06-03T19:37:45.088 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:37:44 smithi164 bash[26855]: cluster 2023-06-03T19:37:43.577127+0000 mon.a (mon.0) 786 : cluster [DBG] mgrmap e20: x(active, starting, since 0.0653959s), standbys: y 2023-06-03T19:37:45.088 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:37:44 smithi164 bash[26855]: audit 2023-06-03T19:37:43.588855+0000 mon.a (mon.0) 787 : audit [DBG] from='mgr.24634 172.21.15.164:0/803880945' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-06-03T19:37:45.089 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:37:44 smithi164 bash[26855]: audit 2023-06-03T19:37:43.589161+0000 mon.a (mon.0) 788 : audit [DBG] from='mgr.24634 172.21.15.164:0/803880945' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-06-03T19:37:45.089 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:37:44 smithi164 bash[26855]: audit 2023-06-03T19:37:43.589341+0000 mon.a (mon.0) 789 : audit [DBG] from='mgr.24634 172.21.15.164:0/803880945' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-06-03T19:37:45.089 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:37:44 smithi164 bash[26855]: audit 2023-06-03T19:37:43.589525+0000 mon.a (mon.0) 790 : audit [DBG] from='mgr.24634 172.21.15.164:0/803880945' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "foo.smithi089.giuxuj"}]: dispatch 2023-06-03T19:37:45.089 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:37:44 smithi164 bash[26855]: audit 2023-06-03T19:37:43.589855+0000 mon.a (mon.0) 791 : audit [DBG] from='mgr.24634 172.21.15.164:0/803880945' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "foo.smithi164.txgfgv"}]: dispatch 2023-06-03T19:37:45.089 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:37:44 smithi164 bash[26855]: audit 2023-06-03T19:37:43.592236+0000 mon.a (mon.0) 792 : audit [DBG] from='mgr.24634 172.21.15.164:0/803880945' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2023-06-03T19:37:45.089 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:37:44 smithi164 bash[26855]: audit 2023-06-03T19:37:43.592426+0000 mon.a (mon.0) 793 : audit [DBG] from='mgr.24634 172.21.15.164:0/803880945' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2023-06-03T19:37:45.090 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:37:44 smithi164 bash[26855]: audit 2023-06-03T19:37:43.594539+0000 mon.a (mon.0) 794 : audit [DBG] from='mgr.24634 172.21.15.164:0/803880945' entity='mgr.x' cmd=[{"prefix": "mds metadata"}]: dispatch 2023-06-03T19:37:45.090 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:37:44 smithi164 bash[26855]: audit 2023-06-03T19:37:43.594880+0000 mon.a (mon.0) 795 : audit [DBG] from='mgr.24634 172.21.15.164:0/803880945' entity='mgr.x' cmd=[{"prefix": "osd metadata"}]: dispatch 2023-06-03T19:37:45.090 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:37:44 smithi164 bash[26855]: audit 2023-06-03T19:37:43.595661+0000 mon.a (mon.0) 796 : audit [DBG] from='mgr.24634 172.21.15.164:0/803880945' entity='mgr.x' cmd=[{"prefix": "mon metadata"}]: dispatch 2023-06-03T19:37:45.090 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:37:44 smithi164 bash[26855]: audit 2023-06-03T19:37:43.598820+0000 mon.a (mon.0) 797 : audit [DBG] from='mgr.24634 172.21.15.164:0/803880945' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-06-03T19:37:45.090 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:37:44 smithi164 bash[26855]: audit 2023-06-03T19:37:43.599179+0000 mon.a (mon.0) 798 : audit [DBG] from='mgr.24634 172.21.15.164:0/803880945' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-06-03T19:37:45.091 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:37:44 smithi164 bash[26855]: audit 2023-06-03T19:37:43.599460+0000 mon.a (mon.0) 799 : audit [DBG] from='mgr.24634 172.21.15.164:0/803880945' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-06-03T19:37:45.091 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:37:44 smithi164 bash[26855]: audit 2023-06-03T19:37:43.599742+0000 mon.a (mon.0) 800 : audit [DBG] from='mgr.24634 172.21.15.164:0/803880945' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-06-03T19:37:45.091 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:37:44 smithi164 bash[26855]: audit 2023-06-03T19:37:43.600017+0000 mon.a (mon.0) 801 : audit [DBG] from='mgr.24634 172.21.15.164:0/803880945' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-06-03T19:37:45.091 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:37:44 smithi164 bash[26855]: audit 2023-06-03T19:37:43.600292+0000 mon.a (mon.0) 802 : audit [DBG] from='mgr.24634 172.21.15.164:0/803880945' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-06-03T19:37:45.091 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:37:44 smithi164 bash[26855]: audit 2023-06-03T19:37:43.600740+0000 mon.a (mon.0) 803 : audit [DBG] from='mgr.24634 172.21.15.164:0/803880945' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-06-03T19:37:45.092 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:37:44 smithi164 bash[26855]: audit 2023-06-03T19:37:43.601055+0000 mon.a (mon.0) 804 : audit [DBG] from='mgr.24634 172.21.15.164:0/803880945' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-06-03T19:37:45.092 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:37:44 smithi164 bash[26855]: cluster 2023-06-03T19:37:43.732858+0000 mon.a (mon.0) 805 : cluster [INF] Manager daemon x is now available 2023-06-03T19:37:45.092 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:37:44 smithi164 bash[26855]: audit 2023-06-03T19:37:43.789924+0000 mon.a (mon.0) 806 : audit [INF] from='mgr.24634 172.21.15.164:0/803880945' entity='mgr.x' 2023-06-03T19:37:45.092 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:37:44 smithi164 bash[26855]: audit 2023-06-03T19:37:43.798997+0000 mon.a (mon.0) 807 : audit [DBG] from='mgr.24634 172.21.15.164:0/803880945' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-03T19:37:45.092 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:37:44 smithi164 bash[26855]: audit 2023-06-03T19:37:43.808562+0000 mon.a (mon.0) 808 : audit [DBG] from='mgr.24634 172.21.15.164:0/803880945' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-03T19:37:45.092 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:37:44 smithi164 bash[26855]: audit 2023-06-03T19:37:43.873549+0000 mon.a (mon.0) 809 : audit [INF] from='mgr.24634 172.21.15.164:0/803880945' entity='mgr.x' cmd=[{"prefix": "osd pool rename", "format": "json", "srcpool": "device_health_metrics", "destpool": ".mgr", "yes_i_really_mean_it": true}]: dispatch 2023-06-03T19:37:45.093 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:37:44 smithi164 bash[26855]: audit 2023-06-03T19:37:43.907341+0000 mon.a (mon.0) 810 : audit [DBG] from='mgr.24634 172.21.15.164:0/803880945' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T19:37:45.093 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:37:44 smithi164 bash[26855]: audit 2023-06-03T19:37:43.971506+0000 mon.a (mon.0) 811 : audit [INF] from='mgr.24634 172.21.15.164:0/803880945' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2023-06-03T19:37:45.093 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:37:44 smithi164 bash[26855]: audit 2023-06-03T19:37:43.999062+0000 mon.a (mon.0) 812 : audit [INF] from='mgr.24634 172.21.15.164:0/803880945' entity='mgr.x' 2023-06-03T19:37:45.093 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:37:44 smithi164 bash[26855]: audit 2023-06-03T19:37:44.114945+0000 mon.a (mon.0) 813 : audit [INF] from='mgr.24634 172.21.15.164:0/803880945' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2023-06-03T19:37:45.093 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:37:44 smithi164 bash[26855]: audit 2023-06-03T19:37:44.153897+0000 mon.a (mon.0) 814 : audit [INF] from='mgr.24634 172.21.15.164:0/803880945' entity='mgr.x' 2023-06-03T19:37:45.093 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:37:44 smithi164 bash[26855]: audit 2023-06-03T19:37:44.171332+0000 mon.a (mon.0) 815 : audit [INF] from='mgr.24634 172.21.15.164:0/803880945' entity='mgr.x' 2023-06-03T19:37:45.488 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 49 16 75877 75861 96.7525 58.625 0.0208019 0.0103297 2023-06-03T19:37:45.781 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 19:37:45 smithi089 bash[26845]: debug 2023-06-03T19:37:45.419+0000 7f4b414aa680 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2023-06-03T19:37:45.782 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 19:37:45 smithi089 bash[26845]: debug 2023-06-03T19:37:45.603+0000 7f4b414aa680 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2023-06-03T19:37:45.782 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:37:45 smithi089 bash[33173]: cluster 2023-06-03T19:37:44.606779+0000 mon.a (mon.0) 816 : cluster [DBG] mgrmap e21: x(active, since 1.09508s), standbys: y 2023-06-03T19:37:45.782 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:37:45 smithi089 bash[33173]: cephadm 2023-06-03T19:37:44.649784+0000 mgr.x (mgr.24634) 1 : cephadm [INF] Deploying cephadm binary to smithi089 2023-06-03T19:37:45.783 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:37:45 smithi089 bash[33173]: cluster 2023-06-03T19:37:44.670452+0000 mgr.x (mgr.24634) 2 : cluster [DBG] pgmap v3: 89 pgs: 89 active+clean; 4.6 GiB data, 14 GiB used, 701 GiB / 715 GiB avail 2023-06-03T19:37:45.783 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:37:45 smithi089 bash[33173]: cephadm 2023-06-03T19:37:44.693159+0000 mgr.x (mgr.24634) 3 : cephadm [INF] Deploying cephadm binary to smithi164 2023-06-03T19:37:45.783 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:37:45 smithi089 bash[33173]: cephadm 2023-06-03T19:37:44.720967+0000 mgr.x (mgr.24634) 4 : cephadm [INF] [03/Jun/2023:19:37:44] ENGINE Bus STARTING 2023-06-03T19:37:45.783 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:37:45 smithi089 bash[33173]: audit 2023-06-03T19:37:44.791993+0000 mon.a (mon.0) 817 : audit [INF] from='mgr.24634 172.21.15.164:0/803880945' entity='mgr.x' cmd='[{"prefix": "osd pool rename", "format": "json", "srcpool": "device_health_metrics", "destpool": ".mgr", "yes_i_really_mean_it": true}]': finished 2023-06-03T19:37:45.783 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:37:45 smithi089 bash[33173]: cluster 2023-06-03T19:37:44.792100+0000 mon.a (mon.0) 818 : cluster [DBG] osdmap e69: 8 total, 8 up, 8 in 2023-06-03T19:37:45.783 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:37:45 smithi089 bash[33173]: audit 2023-06-03T19:37:44.803739+0000 mon.a (mon.0) 819 : audit [INF] from='mgr.24634 172.21.15.164:0/803880945' entity='mgr.x' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]: dispatch 2023-06-03T19:37:45.783 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:37:45 smithi089 bash[33173]: cephadm 2023-06-03T19:37:44.831425+0000 mgr.x (mgr.24634) 5 : cephadm [INF] [03/Jun/2023:19:37:44] ENGINE Serving on http://172.21.15.164:8765 2023-06-03T19:37:45.784 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:37:45 smithi089 bash[33173]: cephadm 2023-06-03T19:37:45.106486+0000 mgr.x (mgr.24634) 6 : cephadm [INF] [03/Jun/2023:19:37:45] ENGINE Serving on https://172.21.15.164:7150 2023-06-03T19:37:45.784 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:37:45 smithi089 bash[33173]: cephadm 2023-06-03T19:37:45.106622+0000 mgr.x (mgr.24634) 7 : cephadm [INF] [03/Jun/2023:19:37:45] ENGINE Bus STARTED 2023-06-03T19:37:45.784 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:37:45 smithi089 bash[26572]: cluster 2023-06-03T19:37:44.606779+0000 mon.a (mon.0) 816 : cluster [DBG] mgrmap e21: x(active, since 1.09508s), standbys: y 2023-06-03T19:37:45.784 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:37:45 smithi089 bash[26572]: cephadm 2023-06-03T19:37:44.649784+0000 mgr.x (mgr.24634) 1 : cephadm [INF] Deploying cephadm binary to smithi089 2023-06-03T19:37:45.784 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:37:45 smithi089 bash[26572]: cluster 2023-06-03T19:37:44.670452+0000 mgr.x (mgr.24634) 2 : cluster [DBG] pgmap v3: 89 pgs: 89 active+clean; 4.6 GiB data, 14 GiB used, 701 GiB / 715 GiB avail 2023-06-03T19:37:45.784 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:37:45 smithi089 bash[26572]: cephadm 2023-06-03T19:37:44.693159+0000 mgr.x (mgr.24634) 3 : cephadm [INF] Deploying cephadm binary to smithi164 2023-06-03T19:37:45.785 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:37:45 smithi089 bash[26572]: cephadm 2023-06-03T19:37:44.720967+0000 mgr.x (mgr.24634) 4 : cephadm [INF] [03/Jun/2023:19:37:44] ENGINE Bus STARTING 2023-06-03T19:37:45.785 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:37:45 smithi089 bash[26572]: audit 2023-06-03T19:37:44.791993+0000 mon.a (mon.0) 817 : audit [INF] from='mgr.24634 172.21.15.164:0/803880945' entity='mgr.x' cmd='[{"prefix": "osd pool rename", "format": "json", "srcpool": "device_health_metrics", "destpool": ".mgr", "yes_i_really_mean_it": true}]': finished 2023-06-03T19:37:45.785 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:37:45 smithi089 bash[26572]: cluster 2023-06-03T19:37:44.792100+0000 mon.a (mon.0) 818 : cluster [DBG] osdmap e69: 8 total, 8 up, 8 in 2023-06-03T19:37:45.785 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:37:45 smithi089 bash[26572]: audit 2023-06-03T19:37:44.803739+0000 mon.a (mon.0) 819 : audit [INF] from='mgr.24634 172.21.15.164:0/803880945' entity='mgr.x' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]: dispatch 2023-06-03T19:37:45.785 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:37:45 smithi089 bash[26572]: cephadm 2023-06-03T19:37:44.831425+0000 mgr.x (mgr.24634) 5 : cephadm [INF] [03/Jun/2023:19:37:44] ENGINE Serving on http://172.21.15.164:8765 2023-06-03T19:37:45.785 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:37:45 smithi089 bash[26572]: cephadm 2023-06-03T19:37:45.106486+0000 mgr.x (mgr.24634) 6 : cephadm [INF] [03/Jun/2023:19:37:45] ENGINE Serving on https://172.21.15.164:7150 2023-06-03T19:37:45.785 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:37:45 smithi089 bash[26572]: cephadm 2023-06-03T19:37:45.106622+0000 mgr.x (mgr.24634) 7 : cephadm [INF] [03/Jun/2023:19:37:45] ENGINE Bus STARTED 2023-06-03T19:37:46.087 INFO:journalctl@ceph.mgr.x.smithi164.stdout:Jun 03 19:37:45 smithi164 bash[50883]: ::ffff:172.21.15.164 - - [03/Jun/2023:19:37:45] "GET /metrics HTTP/1.1" 200 33523 "" "Prometheus/2.33.4" 2023-06-03T19:37:46.088 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:37:45 smithi164 bash[26855]: cluster 2023-06-03T19:37:44.606779+0000 mon.a (mon.0) 816 : cluster [DBG] mgrmap e21: x(active, since 1.09508s), standbys: y 2023-06-03T19:37:46.088 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:37:45 smithi164 bash[26855]: cephadm 2023-06-03T19:37:44.649784+0000 mgr.x (mgr.24634) 1 : cephadm [INF] Deploying cephadm binary to smithi089 2023-06-03T19:37:46.088 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:37:45 smithi164 bash[26855]: cluster 2023-06-03T19:37:44.670452+0000 mgr.x (mgr.24634) 2 : cluster [DBG] pgmap v3: 89 pgs: 89 active+clean; 4.6 GiB data, 14 GiB used, 701 GiB / 715 GiB avail 2023-06-03T19:37:46.088 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:37:45 smithi164 bash[26855]: cephadm 2023-06-03T19:37:44.693159+0000 mgr.x (mgr.24634) 3 : cephadm [INF] Deploying cephadm binary to smithi164 2023-06-03T19:37:46.088 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:37:45 smithi164 bash[26855]: cephadm 2023-06-03T19:37:44.720967+0000 mgr.x (mgr.24634) 4 : cephadm [INF] [03/Jun/2023:19:37:44] ENGINE Bus STARTING 2023-06-03T19:37:46.088 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:37:45 smithi164 bash[26855]: audit 2023-06-03T19:37:44.791993+0000 mon.a (mon.0) 817 : audit [INF] from='mgr.24634 172.21.15.164:0/803880945' entity='mgr.x' cmd='[{"prefix": "osd pool rename", "format": "json", "srcpool": "device_health_metrics", "destpool": ".mgr", "yes_i_really_mean_it": true}]': finished 2023-06-03T19:37:46.089 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:37:45 smithi164 bash[26855]: cluster 2023-06-03T19:37:44.792100+0000 mon.a (mon.0) 818 : cluster [DBG] osdmap e69: 8 total, 8 up, 8 in 2023-06-03T19:37:46.089 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:37:45 smithi164 bash[26855]: audit 2023-06-03T19:37:44.803739+0000 mon.a (mon.0) 819 : audit [INF] from='mgr.24634 172.21.15.164:0/803880945' entity='mgr.x' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]: dispatch 2023-06-03T19:37:46.089 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:37:45 smithi164 bash[26855]: cephadm 2023-06-03T19:37:44.831425+0000 mgr.x (mgr.24634) 5 : cephadm [INF] [03/Jun/2023:19:37:44] ENGINE Serving on http://172.21.15.164:8765 2023-06-03T19:37:46.089 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:37:45 smithi164 bash[26855]: cephadm 2023-06-03T19:37:45.106486+0000 mgr.x (mgr.24634) 6 : cephadm [INF] [03/Jun/2023:19:37:45] ENGINE Serving on https://172.21.15.164:7150 2023-06-03T19:37:46.089 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:37:45 smithi164 bash[26855]: cephadm 2023-06-03T19:37:45.106622+0000 mgr.x (mgr.24634) 7 : cephadm [INF] [03/Jun/2023:19:37:45] ENGINE Bus STARTED 2023-06-03T19:37:46.488 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 50 16 76769 76753 95.9323 55.75 0.015809 0.0104184 2023-06-03T19:37:47.087 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:37:46 smithi164 bash[26855]: cluster 2023-06-03T19:37:45.592242+0000 mgr.x (mgr.24634) 8 : cluster [DBG] pgmap v5: 89 pgs: 89 active+clean; 4.6 GiB data, 14 GiB used, 701 GiB / 715 GiB avail 2023-06-03T19:37:47.087 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:37:46 smithi164 bash[26855]: audit 2023-06-03T19:37:45.796100+0000 mon.a (mon.0) 820 : audit [INF] from='mgr.24634 172.21.15.164:0/803880945' entity='mgr.x' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]': finished 2023-06-03T19:37:47.087 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:37:46 smithi164 bash[26855]: cluster 2023-06-03T19:37:45.796388+0000 mon.a (mon.0) 821 : cluster [DBG] osdmap e70: 8 total, 8 up, 8 in 2023-06-03T19:37:47.087 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:37:46 smithi164 bash[26855]: cluster 2023-06-03T19:37:45.815520+0000 mon.a (mon.0) 822 : cluster [DBG] mgrmap e22: x(active, since 2s), standbys: y 2023-06-03T19:37:47.088 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:37:46 smithi164 bash[26855]: audit 2023-06-03T19:37:46.246930+0000 mon.b (mon.1) 29 : audit [DBG] from='client.? 172.21.15.89:0/261153790' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-03T19:37:47.116 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:37:46 smithi089 bash[26572]: cluster 2023-06-03T19:37:45.592242+0000 mgr.x (mgr.24634) 8 : cluster [DBG] pgmap v5: 89 pgs: 89 active+clean; 4.6 GiB data, 14 GiB used, 701 GiB / 715 GiB avail 2023-06-03T19:37:47.117 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:37:46 smithi089 bash[26572]: audit 2023-06-03T19:37:45.796100+0000 mon.a (mon.0) 820 : audit [INF] from='mgr.24634 172.21.15.164:0/803880945' entity='mgr.x' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]': finished 2023-06-03T19:37:47.117 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:37:46 smithi089 bash[26572]: cluster 2023-06-03T19:37:45.796388+0000 mon.a (mon.0) 821 : cluster [DBG] osdmap e70: 8 total, 8 up, 8 in 2023-06-03T19:37:47.117 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:37:46 smithi089 bash[26572]: cluster 2023-06-03T19:37:45.815520+0000 mon.a (mon.0) 822 : cluster [DBG] mgrmap e22: x(active, since 2s), standbys: y 2023-06-03T19:37:47.117 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:37:46 smithi089 bash[26572]: audit 2023-06-03T19:37:46.246930+0000 mon.b (mon.1) 29 : audit [DBG] from='client.? 172.21.15.89:0/261153790' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-03T19:37:47.118 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 19:37:46 smithi089 bash[26845]: debug 2023-06-03T19:37:46.951+0000 7f4b414aa680 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2023-06-03T19:37:47.118 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 19:37:47 smithi089 bash[26845]: debug 2023-06-03T19:37:47.107+0000 7f4b414aa680 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2023-06-03T19:37:47.118 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:37:46 smithi089 bash[33173]: cluster 2023-06-03T19:37:45.592242+0000 mgr.x (mgr.24634) 8 : cluster [DBG] pgmap v5: 89 pgs: 89 active+clean; 4.6 GiB data, 14 GiB used, 701 GiB / 715 GiB avail 2023-06-03T19:37:47.118 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:37:46 smithi089 bash[33173]: audit 2023-06-03T19:37:45.796100+0000 mon.a (mon.0) 820 : audit [INF] from='mgr.24634 172.21.15.164:0/803880945' entity='mgr.x' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]': finished 2023-06-03T19:37:47.118 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:37:46 smithi089 bash[33173]: cluster 2023-06-03T19:37:45.796388+0000 mon.a (mon.0) 821 : cluster [DBG] osdmap e70: 8 total, 8 up, 8 in 2023-06-03T19:37:47.118 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:37:46 smithi089 bash[33173]: cluster 2023-06-03T19:37:45.815520+0000 mon.a (mon.0) 822 : cluster [DBG] mgrmap e22: x(active, since 2s), standbys: y 2023-06-03T19:37:47.119 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:37:46 smithi089 bash[33173]: audit 2023-06-03T19:37:46.246930+0000 mon.b (mon.1) 29 : audit [DBG] from='client.? 172.21.15.89:0/261153790' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-03T19:37:47.488 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 51 16 77649 77633 95.1296 55 0.00855633 0.0105054 2023-06-03T19:37:47.532 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 19:37:47 smithi089 bash[26845]: debug 2023-06-03T19:37:47.271+0000 7f4b414aa680 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2023-06-03T19:37:48.032 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 19:37:47 smithi089 bash[26845]: debug 2023-06-03T19:37:47.595+0000 7f4b414aa680 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2023-06-03T19:37:48.386 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 19:37:48 smithi089 bash[26845]: debug 2023-06-03T19:37:48.087+0000 7f4b414aa680 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2023-06-03T19:37:48.387 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 19:37:48 smithi089 bash[26845]: debug 2023-06-03T19:37:48.375+0000 7f4b414aa680 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2023-06-03T19:37:48.488 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 52 16 78488 78472 94.3085 52.4375 0.051819 0.0105968 2023-06-03T19:37:48.653 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 19:37:48 smithi089 bash[26845]: debug 2023-06-03T19:37:48.643+0000 7f4b414aa680 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2023-06-03T19:37:48.989 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:37:48 smithi089 bash[26572]: cluster 2023-06-03T19:37:47.592690+0000 mgr.x (mgr.24634) 9 : cluster [DBG] pgmap v7: 89 pgs: 89 active+clean; 4.6 GiB data, 14 GiB used, 701 GiB / 715 GiB avail 2023-06-03T19:37:48.990 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:37:48 smithi089 bash[26572]: cluster 2023-06-03T19:37:47.808458+0000 mon.a (mon.0) 823 : cluster [DBG] mgrmap e23: x(active, since 4s), standbys: y 2023-06-03T19:37:48.990 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:37:48 smithi089 bash[26572]: audit 2023-06-03T19:37:47.923661+0000 mon.a (mon.0) 824 : audit [INF] from='mgr.24634 172.21.15.164:0/803880945' entity='mgr.x' 2023-06-03T19:37:48.991 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 19:37:48 smithi089 bash[26845]: debug 2023-06-03T19:37:48.979+0000 7f4b414aa680 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2023-06-03T19:37:48.991 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:37:48 smithi089 bash[33173]: cluster 2023-06-03T19:37:47.592690+0000 mgr.x (mgr.24634) 9 : cluster [DBG] pgmap v7: 89 pgs: 89 active+clean; 4.6 GiB data, 14 GiB used, 701 GiB / 715 GiB avail 2023-06-03T19:37:48.992 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:37:48 smithi089 bash[33173]: cluster 2023-06-03T19:37:47.808458+0000 mon.a (mon.0) 823 : cluster [DBG] mgrmap e23: x(active, since 4s), standbys: y 2023-06-03T19:37:48.992 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:37:48 smithi089 bash[33173]: audit 2023-06-03T19:37:47.923661+0000 mon.a (mon.0) 824 : audit [INF] from='mgr.24634 172.21.15.164:0/803880945' entity='mgr.x' 2023-06-03T19:37:49.087 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:37:48 smithi164 bash[26855]: cluster 2023-06-03T19:37:47.592690+0000 mgr.x (mgr.24634) 9 : cluster [DBG] pgmap v7: 89 pgs: 89 active+clean; 4.6 GiB data, 14 GiB used, 701 GiB / 715 GiB avail 2023-06-03T19:37:49.087 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:37:48 smithi164 bash[26855]: cluster 2023-06-03T19:37:47.808458+0000 mon.a (mon.0) 823 : cluster [DBG] mgrmap e23: x(active, since 4s), standbys: y 2023-06-03T19:37:49.087 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:37:48 smithi164 bash[26855]: audit 2023-06-03T19:37:47.923661+0000 mon.a (mon.0) 824 : audit [INF] from='mgr.24634 172.21.15.164:0/803880945' entity='mgr.x' 2023-06-03T19:37:49.263 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 19:37:49 smithi089 bash[26845]: debug 2023-06-03T19:37:49.131+0000 7f4b414aa680 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2023-06-03T19:37:49.263 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 19:37:49 smithi089 bash[26845]: [03/Jun/2023:19:37:49] ENGINE Bus STARTING 2023-06-03T19:37:49.263 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 19:37:49 smithi089 bash[26845]: CherryPy Checker: 2023-06-03T19:37:49.264 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 19:37:49 smithi089 bash[26845]: The Application mounted at '' has an empty config. 2023-06-03T19:37:49.264 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 19:37:49 smithi089 bash[26845]: [03/Jun/2023:19:37:49] ENGINE Serving on http://:::9283 2023-06-03T19:37:49.264 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 19:37:49 smithi089 bash[26845]: [03/Jun/2023:19:37:49] ENGINE Bus STARTED 2023-06-03T19:37:49.488 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 53 16 79420 79404 93.628 58.25 0.00701294 0.0106751 2023-06-03T19:37:50.087 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:37:49 smithi164 bash[26855]: audit 2023-06-03T19:37:49.017825+0000 mon.a (mon.0) 825 : audit [DBG] from='client.? 172.21.15.89:0/2907829723' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-03T19:37:50.087 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:37:49 smithi164 bash[26855]: cluster 2023-06-03T19:37:49.138253+0000 mon.a (mon.0) 826 : cluster [DBG] Standby manager daemon y restarted 2023-06-03T19:37:50.088 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:37:49 smithi164 bash[26855]: cluster 2023-06-03T19:37:49.138591+0000 mon.a (mon.0) 827 : cluster [DBG] Standby manager daemon y started 2023-06-03T19:37:50.088 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:37:49 smithi164 bash[26855]: audit 2023-06-03T19:37:49.145496+0000 mon.a (mon.0) 828 : audit [DBG] from='mgr.? 172.21.15.89:0/1307086040' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2023-06-03T19:37:50.088 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:37:49 smithi164 bash[26855]: audit 2023-06-03T19:37:49.151833+0000 mon.a (mon.0) 829 : audit [DBG] from='mgr.? 172.21.15.89:0/1307086040' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-06-03T19:37:50.088 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:37:49 smithi164 bash[26855]: audit 2023-06-03T19:37:49.154562+0000 mon.a (mon.0) 830 : audit [DBG] from='mgr.? 172.21.15.89:0/1307086040' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2023-06-03T19:37:50.088 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:37:49 smithi164 bash[26855]: audit 2023-06-03T19:37:49.162277+0000 mon.a (mon.0) 831 : audit [DBG] from='mgr.? 172.21.15.89:0/1307086040' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-06-03T19:37:50.282 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:37:49 smithi089 bash[33173]: audit 2023-06-03T19:37:49.017825+0000 mon.a (mon.0) 825 : audit [DBG] from='client.? 172.21.15.89:0/2907829723' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-03T19:37:50.282 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:37:49 smithi089 bash[33173]: cluster 2023-06-03T19:37:49.138253+0000 mon.a (mon.0) 826 : cluster [DBG] Standby manager daemon y restarted 2023-06-03T19:37:50.282 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:37:49 smithi089 bash[33173]: cluster 2023-06-03T19:37:49.138591+0000 mon.a (mon.0) 827 : cluster [DBG] Standby manager daemon y started 2023-06-03T19:37:50.282 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:37:49 smithi089 bash[33173]: audit 2023-06-03T19:37:49.145496+0000 mon.a (mon.0) 828 : audit [DBG] from='mgr.? 172.21.15.89:0/1307086040' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2023-06-03T19:37:50.282 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:37:49 smithi089 bash[33173]: audit 2023-06-03T19:37:49.151833+0000 mon.a (mon.0) 829 : audit [DBG] from='mgr.? 172.21.15.89:0/1307086040' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-06-03T19:37:50.282 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:37:49 smithi089 bash[33173]: audit 2023-06-03T19:37:49.154562+0000 mon.a (mon.0) 830 : audit [DBG] from='mgr.? 172.21.15.89:0/1307086040' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2023-06-03T19:37:50.283 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:37:49 smithi089 bash[33173]: audit 2023-06-03T19:37:49.162277+0000 mon.a (mon.0) 831 : audit [DBG] from='mgr.? 172.21.15.89:0/1307086040' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-06-03T19:37:50.283 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:37:49 smithi089 bash[26572]: audit 2023-06-03T19:37:49.017825+0000 mon.a (mon.0) 825 : audit [DBG] from='client.? 172.21.15.89:0/2907829723' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-03T19:37:50.283 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:37:49 smithi089 bash[26572]: cluster 2023-06-03T19:37:49.138253+0000 mon.a (mon.0) 826 : cluster [DBG] Standby manager daemon y restarted 2023-06-03T19:37:50.283 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:37:49 smithi089 bash[26572]: cluster 2023-06-03T19:37:49.138591+0000 mon.a (mon.0) 827 : cluster [DBG] Standby manager daemon y started 2023-06-03T19:37:50.284 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:37:49 smithi089 bash[26572]: audit 2023-06-03T19:37:49.145496+0000 mon.a (mon.0) 828 : audit [DBG] from='mgr.? 172.21.15.89:0/1307086040' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2023-06-03T19:37:50.284 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:37:49 smithi089 bash[26572]: audit 2023-06-03T19:37:49.151833+0000 mon.a (mon.0) 829 : audit [DBG] from='mgr.? 172.21.15.89:0/1307086040' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-06-03T19:37:50.284 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:37:49 smithi089 bash[26572]: audit 2023-06-03T19:37:49.154562+0000 mon.a (mon.0) 830 : audit [DBG] from='mgr.? 172.21.15.89:0/1307086040' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2023-06-03T19:37:50.284 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:37:49 smithi089 bash[26572]: audit 2023-06-03T19:37:49.162277+0000 mon.a (mon.0) 831 : audit [DBG] from='mgr.? 172.21.15.89:0/1307086040' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-06-03T19:37:50.488 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 54 16 80441 80425 93.0758 63.8125 0.0188399 0.0107383 2023-06-03T19:37:51.087 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:37:50 smithi164 bash[26855]: cluster 2023-06-03T19:37:49.593109+0000 mgr.x (mgr.24634) 10 : cluster [DBG] pgmap v8: 89 pgs: 89 active+clean; 4.6 GiB data, 14 GiB used, 701 GiB / 715 GiB avail 2023-06-03T19:37:51.088 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:37:50 smithi164 bash[26855]: cluster 2023-06-03T19:37:49.827606+0000 mon.a (mon.0) 832 : cluster [DBG] mgrmap e24: x(active, since 6s), standbys: y 2023-06-03T19:37:51.282 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:37:50 smithi089 bash[26572]: cluster 2023-06-03T19:37:49.593109+0000 mgr.x (mgr.24634) 10 : cluster [DBG] pgmap v8: 89 pgs: 89 active+clean; 4.6 GiB data, 14 GiB used, 701 GiB / 715 GiB avail 2023-06-03T19:37:51.282 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:37:50 smithi089 bash[26572]: cluster 2023-06-03T19:37:49.827606+0000 mon.a (mon.0) 832 : cluster [DBG] mgrmap e24: x(active, since 6s), standbys: y 2023-06-03T19:37:51.283 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:37:50 smithi089 bash[33173]: cluster 2023-06-03T19:37:49.593109+0000 mgr.x (mgr.24634) 10 : cluster [DBG] pgmap v8: 89 pgs: 89 active+clean; 4.6 GiB data, 14 GiB used, 701 GiB / 715 GiB avail 2023-06-03T19:37:51.283 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:37:50 smithi089 bash[33173]: cluster 2023-06-03T19:37:49.827606+0000 mon.a (mon.0) 832 : cluster [DBG] mgrmap e24: x(active, since 6s), standbys: y 2023-06-03T19:37:51.488 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 55 16 81356 81340 92.4232 57.1875 0.00793865 0.0108128 2023-06-03T19:37:52.488 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 56 16 82292 82276 91.8173 58.5 0.0263228 0.0108848 2023-06-03T19:37:53.032 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:37:52 smithi089 bash[33173]: cluster 2023-06-03T19:37:51.593827+0000 mgr.x (mgr.24634) 11 : cluster [DBG] pgmap v9: 89 pgs: 89 active+clean; 4.9 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 12 KiB/s rd, 41 MiB/s wr, 684 op/s 2023-06-03T19:37:53.032 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:37:52 smithi089 bash[33173]: audit 2023-06-03T19:37:51.725497+0000 mon.c (mon.2) 37 : audit [DBG] from='client.? 172.21.15.89:0/858215745' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-03T19:37:53.032 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:37:52 smithi089 bash[26572]: cluster 2023-06-03T19:37:51.593827+0000 mgr.x (mgr.24634) 11 : cluster [DBG] pgmap v9: 89 pgs: 89 active+clean; 4.9 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 12 KiB/s rd, 41 MiB/s wr, 684 op/s 2023-06-03T19:37:53.032 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:37:52 smithi089 bash[26572]: audit 2023-06-03T19:37:51.725497+0000 mon.c (mon.2) 37 : audit [DBG] from='client.? 172.21.15.89:0/858215745' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-03T19:37:53.089 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:37:52 smithi164 bash[26855]: cluster 2023-06-03T19:37:51.593827+0000 mgr.x (mgr.24634) 11 : cluster [DBG] pgmap v9: 89 pgs: 89 active+clean; 4.9 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 12 KiB/s rd, 41 MiB/s wr, 684 op/s 2023-06-03T19:37:53.089 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:37:52 smithi164 bash[26855]: audit 2023-06-03T19:37:51.725497+0000 mon.c (mon.2) 37 : audit [DBG] from='client.? 172.21.15.89:0/858215745' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-03T19:37:53.488 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 57 16 83146 83130 91.1428 53.375 0.0193101 0.0109656 2023-06-03T19:37:54.489 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 58 16 83894 83878 90.3773 46.75 0.0212105 0.0110575 2023-06-03T19:37:54.782 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:37:54 smithi089 bash[33173]: audit 2023-06-03T19:37:53.405656+0000 mon.a (mon.0) 833 : audit [INF] from='mgr.24634 172.21.15.164:0/803880945' entity='mgr.x' 2023-06-03T19:37:54.782 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:37:54 smithi089 bash[33173]: audit 2023-06-03T19:37:54.207631+0000 mon.a (mon.0) 834 : audit [INF] from='mgr.24634 172.21.15.164:0/803880945' entity='mgr.x' 2023-06-03T19:37:54.782 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:37:54 smithi089 bash[33173]: audit 2023-06-03T19:37:54.222903+0000 mon.a (mon.0) 835 : audit [INF] from='mgr.24634 172.21.15.164:0/803880945' entity='mgr.x' 2023-06-03T19:37:54.782 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:37:54 smithi089 bash[33173]: audit 2023-06-03T19:37:54.401535+0000 mon.a (mon.0) 836 : audit [DBG] from='client.? 172.21.15.89:0/1891759176' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-03T19:37:54.782 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:37:54 smithi089 bash[26572]: audit 2023-06-03T19:37:53.405656+0000 mon.a (mon.0) 833 : audit [INF] from='mgr.24634 172.21.15.164:0/803880945' entity='mgr.x' 2023-06-03T19:37:54.783 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:37:54 smithi089 bash[26572]: audit 2023-06-03T19:37:54.207631+0000 mon.a (mon.0) 834 : audit [INF] from='mgr.24634 172.21.15.164:0/803880945' entity='mgr.x' 2023-06-03T19:37:54.783 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:37:54 smithi089 bash[26572]: audit 2023-06-03T19:37:54.222903+0000 mon.a (mon.0) 835 : audit [INF] from='mgr.24634 172.21.15.164:0/803880945' entity='mgr.x' 2023-06-03T19:37:54.783 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:37:54 smithi089 bash[26572]: audit 2023-06-03T19:37:54.401535+0000 mon.a (mon.0) 836 : audit [DBG] from='client.? 172.21.15.89:0/1891759176' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-03T19:37:54.837 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:37:54 smithi164 bash[26855]: audit 2023-06-03T19:37:53.405656+0000 mon.a (mon.0) 833 : audit [INF] from='mgr.24634 172.21.15.164:0/803880945' entity='mgr.x' 2023-06-03T19:37:54.837 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:37:54 smithi164 bash[26855]: audit 2023-06-03T19:37:54.207631+0000 mon.a (mon.0) 834 : audit [INF] from='mgr.24634 172.21.15.164:0/803880945' entity='mgr.x' 2023-06-03T19:37:54.838 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:37:54 smithi164 bash[26855]: audit 2023-06-03T19:37:54.222903+0000 mon.a (mon.0) 835 : audit [INF] from='mgr.24634 172.21.15.164:0/803880945' entity='mgr.x' 2023-06-03T19:37:54.838 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:37:54 smithi164 bash[26855]: audit 2023-06-03T19:37:54.401535+0000 mon.a (mon.0) 836 : audit [DBG] from='client.? 172.21.15.89:0/1891759176' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-03T19:37:55.489 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 59 16 84677 84661 89.6749 48.9375 0.0366268 0.0111445 2023-06-03T19:37:55.782 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:37:55 smithi089 bash[33173]: cluster 2023-06-03T19:37:53.594394+0000 mgr.x (mgr.24634) 12 : cluster [DBG] pgmap v10: 89 pgs: 89 active+clean; 4.9 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 9.3 KiB/s rd, 32 MiB/s wr, 538 op/s 2023-06-03T19:37:55.782 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:37:55 smithi089 bash[33173]: audit 2023-06-03T19:37:54.910500+0000 mon.a (mon.0) 837 : audit [INF] from='mgr.24634 172.21.15.164:0/803880945' entity='mgr.x' 2023-06-03T19:37:55.782 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:37:55 smithi089 bash[33173]: audit 2023-06-03T19:37:54.920297+0000 mon.a (mon.0) 838 : audit [INF] from='mgr.24634 172.21.15.164:0/803880945' entity='mgr.x' 2023-06-03T19:37:55.782 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:37:55 smithi089 bash[33173]: audit 2023-06-03T19:37:55.177256+0000 mon.a (mon.0) 839 : audit [INF] from='mgr.24634 172.21.15.164:0/803880945' entity='mgr.x' 2023-06-03T19:37:55.782 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:37:55 smithi089 bash[33173]: audit 2023-06-03T19:37:55.184357+0000 mon.a (mon.0) 840 : audit [INF] from='mgr.24634 172.21.15.164:0/803880945' entity='mgr.x' 2023-06-03T19:37:55.783 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:37:55 smithi089 bash[26572]: cluster 2023-06-03T19:37:53.594394+0000 mgr.x (mgr.24634) 12 : cluster [DBG] pgmap v10: 89 pgs: 89 active+clean; 4.9 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 9.3 KiB/s rd, 32 MiB/s wr, 538 op/s 2023-06-03T19:37:55.783 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:37:55 smithi089 bash[26572]: audit 2023-06-03T19:37:54.910500+0000 mon.a (mon.0) 837 : audit [INF] from='mgr.24634 172.21.15.164:0/803880945' entity='mgr.x' 2023-06-03T19:37:55.783 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:37:55 smithi089 bash[26572]: audit 2023-06-03T19:37:54.920297+0000 mon.a (mon.0) 838 : audit [INF] from='mgr.24634 172.21.15.164:0/803880945' entity='mgr.x' 2023-06-03T19:37:55.783 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:37:55 smithi089 bash[26572]: audit 2023-06-03T19:37:55.177256+0000 mon.a (mon.0) 839 : audit [INF] from='mgr.24634 172.21.15.164:0/803880945' entity='mgr.x' 2023-06-03T19:37:55.783 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:37:55 smithi089 bash[26572]: audit 2023-06-03T19:37:55.184357+0000 mon.a (mon.0) 840 : audit [INF] from='mgr.24634 172.21.15.164:0/803880945' entity='mgr.x' 2023-06-03T19:37:55.837 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:37:55 smithi164 bash[26855]: cluster 2023-06-03T19:37:53.594394+0000 mgr.x (mgr.24634) 12 : cluster [DBG] pgmap v10: 89 pgs: 89 active+clean; 4.9 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 9.3 KiB/s rd, 32 MiB/s wr, 538 op/s 2023-06-03T19:37:55.837 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:37:55 smithi164 bash[26855]: audit 2023-06-03T19:37:54.910500+0000 mon.a (mon.0) 837 : audit [INF] from='mgr.24634 172.21.15.164:0/803880945' entity='mgr.x' 2023-06-03T19:37:55.838 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:37:55 smithi164 bash[26855]: audit 2023-06-03T19:37:54.920297+0000 mon.a (mon.0) 838 : audit [INF] from='mgr.24634 172.21.15.164:0/803880945' entity='mgr.x' 2023-06-03T19:37:55.838 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:37:55 smithi164 bash[26855]: audit 2023-06-03T19:37:55.177256+0000 mon.a (mon.0) 839 : audit [INF] from='mgr.24634 172.21.15.164:0/803880945' entity='mgr.x' 2023-06-03T19:37:55.838 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:37:55 smithi164 bash[26855]: audit 2023-06-03T19:37:55.184357+0000 mon.a (mon.0) 840 : audit [INF] from='mgr.24634 172.21.15.164:0/803880945' entity='mgr.x' 2023-06-03T19:37:55.838 INFO:journalctl@ceph.mgr.x.smithi164.stdout:Jun 03 19:37:55 smithi164 bash[50883]: ::ffff:172.21.15.164 - - [03/Jun/2023:19:37:55] "GET /metrics HTTP/1.1" 200 33523 "" "Prometheus/2.33.4" 2023-06-03T19:37:56.489 INFO:tasks.radosbench.radosbench.0.smithi089.stdout:2023-06-03T19:37:56.482079+0000 min lat: 0.00134497 max lat: 0.133514 avg lat: 0.0112199 2023-06-03T19:37:56.489 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-06-03T19:37:56.489 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 60 16 85543 85527 89.0823 54.125 0.0149681 0.0112199 2023-06-03T19:37:57.489 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 61 16 86456 86440 88.5572 57.0625 0.00684914 0.0112863 2023-06-03T19:37:57.532 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:37:57 smithi089 bash[26572]: cluster 2023-06-03T19:37:55.595120+0000 mgr.x (mgr.24634) 13 : cluster [DBG] pgmap v11: 89 pgs: 89 active+clean; 5.1 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 9.1 KiB/s rd, 56 MiB/s wr, 919 op/s 2023-06-03T19:37:57.532 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:37:57 smithi089 bash[26572]: audit 2023-06-03T19:37:56.125950+0000 mon.a (mon.0) 841 : audit [INF] from='mgr.24634 172.21.15.164:0/803880945' entity='mgr.x' 2023-06-03T19:37:57.532 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:37:57 smithi089 bash[26572]: audit 2023-06-03T19:37:56.134371+0000 mon.a (mon.0) 842 : audit [INF] from='mgr.24634 172.21.15.164:0/803880945' entity='mgr.x' 2023-06-03T19:37:57.532 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:37:57 smithi089 bash[26572]: audit 2023-06-03T19:37:57.064858+0000 mon.c (mon.2) 38 : audit [DBG] from='client.? 172.21.15.89:0/4074596253' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-03T19:37:57.533 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:37:57 smithi089 bash[33173]: cluster 2023-06-03T19:37:55.595120+0000 mgr.x (mgr.24634) 13 : cluster [DBG] pgmap v11: 89 pgs: 89 active+clean; 5.1 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 9.1 KiB/s rd, 56 MiB/s wr, 919 op/s 2023-06-03T19:37:57.533 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:37:57 smithi089 bash[33173]: audit 2023-06-03T19:37:56.125950+0000 mon.a (mon.0) 841 : audit [INF] from='mgr.24634 172.21.15.164:0/803880945' entity='mgr.x' 2023-06-03T19:37:57.533 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:37:57 smithi089 bash[33173]: audit 2023-06-03T19:37:56.134371+0000 mon.a (mon.0) 842 : audit [INF] from='mgr.24634 172.21.15.164:0/803880945' entity='mgr.x' 2023-06-03T19:37:57.533 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:37:57 smithi089 bash[33173]: audit 2023-06-03T19:37:57.064858+0000 mon.c (mon.2) 38 : audit [DBG] from='client.? 172.21.15.89:0/4074596253' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-03T19:37:57.587 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:37:57 smithi164 bash[26855]: cluster 2023-06-03T19:37:55.595120+0000 mgr.x (mgr.24634) 13 : cluster [DBG] pgmap v11: 89 pgs: 89 active+clean; 5.1 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 9.1 KiB/s rd, 56 MiB/s wr, 919 op/s 2023-06-03T19:37:57.587 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:37:57 smithi164 bash[26855]: audit 2023-06-03T19:37:56.125950+0000 mon.a (mon.0) 841 : audit [INF] from='mgr.24634 172.21.15.164:0/803880945' entity='mgr.x' 2023-06-03T19:37:57.587 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:37:57 smithi164 bash[26855]: audit 2023-06-03T19:37:56.134371+0000 mon.a (mon.0) 842 : audit [INF] from='mgr.24634 172.21.15.164:0/803880945' entity='mgr.x' 2023-06-03T19:37:57.587 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:37:57 smithi164 bash[26855]: audit 2023-06-03T19:37:57.064858+0000 mon.c (mon.2) 38 : audit [DBG] from='client.? 172.21.15.89:0/4074596253' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-03T19:37:58.281 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 19:37:57 smithi089 bash[26845]: ::ffff:172.21.15.164 - - [03/Jun/2023:19:37:57] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2023-06-03T19:37:58.489 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 62 16 87388 87372 88.0683 58.25 0.0118527 0.0113501 2023-06-03T19:37:59.489 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 63 16 88268 88252 87.5433 55 0.00447079 0.0114161 2023-06-03T19:38:00.032 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:37:59 smithi089 bash[26572]: audit 2023-06-03T19:37:57.386715+0000 mon.a (mon.0) 843 : audit [INF] from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2023-06-03T19:38:00.032 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:37:59 smithi089 bash[26572]: cluster 2023-06-03T19:37:57.595550+0000 mgr.x (mgr.24634) 14 : cluster [DBG] pgmap v12: 89 pgs: 89 active+clean; 5.1 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 7.7 KiB/s rd, 48 MiB/s wr, 779 op/s 2023-06-03T19:38:00.032 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:37:59 smithi089 bash[26572]: audit 2023-06-03T19:37:58.503843+0000 mon.a (mon.0) 844 : audit [INF] from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2023-06-03T19:38:00.032 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:37:59 smithi089 bash[26572]: audit 2023-06-03T19:37:58.504803+0000 mon.a (mon.0) 845 : audit [DBG] from='mgr.24634 172.21.15.164:0/803880945' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-06-03T19:38:00.033 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:37:59 smithi089 bash[26572]: audit 2023-06-03T19:37:58.505390+0000 mon.a (mon.0) 846 : audit [DBG] from='mgr.24634 172.21.15.164:0/803880945' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-06-03T19:38:00.033 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:37:59 smithi089 bash[26572]: audit 2023-06-03T19:37:58.505719+0000 mon.a (mon.0) 847 : audit [DBG] from='mgr.24634 172.21.15.164:0/803880945' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-06-03T19:38:00.033 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:37:59 smithi089 bash[26572]: audit 2023-06-03T19:37:58.512352+0000 mon.a (mon.0) 848 : audit [DBG] from='mgr.24634 172.21.15.164:0/803880945' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-06-03T19:38:00.033 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:37:59 smithi089 bash[26572]: audit 2023-06-03T19:37:58.512577+0000 mon.a (mon.0) 849 : audit [DBG] from='mgr.24634 172.21.15.164:0/803880945' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-06-03T19:38:00.033 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:37:59 smithi089 bash[26572]: audit 2023-06-03T19:37:58.512818+0000 mon.a (mon.0) 850 : audit [DBG] from='mgr.24634 172.21.15.164:0/803880945' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-06-03T19:38:00.033 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:37:59 smithi089 bash[33173]: audit 2023-06-03T19:37:57.386715+0000 mon.a (mon.0) 843 : audit [INF] from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2023-06-03T19:38:00.034 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:37:59 smithi089 bash[33173]: cluster 2023-06-03T19:37:57.595550+0000 mgr.x (mgr.24634) 14 : cluster [DBG] pgmap v12: 89 pgs: 89 active+clean; 5.1 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 7.7 KiB/s rd, 48 MiB/s wr, 779 op/s 2023-06-03T19:38:00.034 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:37:59 smithi089 bash[33173]: audit 2023-06-03T19:37:58.503843+0000 mon.a (mon.0) 844 : audit [INF] from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2023-06-03T19:38:00.034 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:37:59 smithi089 bash[33173]: audit 2023-06-03T19:37:58.504803+0000 mon.a (mon.0) 845 : audit [DBG] from='mgr.24634 172.21.15.164:0/803880945' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-06-03T19:38:00.034 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:37:59 smithi089 bash[33173]: audit 2023-06-03T19:37:58.505390+0000 mon.a (mon.0) 846 : audit [DBG] from='mgr.24634 172.21.15.164:0/803880945' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-06-03T19:38:00.034 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:37:59 smithi089 bash[33173]: audit 2023-06-03T19:37:58.505719+0000 mon.a (mon.0) 847 : audit [DBG] from='mgr.24634 172.21.15.164:0/803880945' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-06-03T19:38:00.034 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:37:59 smithi089 bash[33173]: audit 2023-06-03T19:37:58.512352+0000 mon.a (mon.0) 848 : audit [DBG] from='mgr.24634 172.21.15.164:0/803880945' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-06-03T19:38:00.034 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:37:59 smithi089 bash[33173]: audit 2023-06-03T19:37:58.512577+0000 mon.a (mon.0) 849 : audit [DBG] from='mgr.24634 172.21.15.164:0/803880945' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-06-03T19:38:00.035 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:37:59 smithi089 bash[33173]: audit 2023-06-03T19:37:58.512818+0000 mon.a (mon.0) 850 : audit [DBG] from='mgr.24634 172.21.15.164:0/803880945' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-06-03T19:38:00.087 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:37:59 smithi164 bash[26855]: audit 2023-06-03T19:37:57.386715+0000 mon.a (mon.0) 843 : audit [INF] from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2023-06-03T19:38:00.087 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:37:59 smithi164 bash[26855]: cluster 2023-06-03T19:37:57.595550+0000 mgr.x (mgr.24634) 14 : cluster [DBG] pgmap v12: 89 pgs: 89 active+clean; 5.1 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 7.7 KiB/s rd, 48 MiB/s wr, 779 op/s 2023-06-03T19:38:00.087 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:37:59 smithi164 bash[26855]: audit 2023-06-03T19:37:58.503843+0000 mon.a (mon.0) 844 : audit [INF] from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2023-06-03T19:38:00.088 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:37:59 smithi164 bash[26855]: audit 2023-06-03T19:37:58.504803+0000 mon.a (mon.0) 845 : audit [DBG] from='mgr.24634 172.21.15.164:0/803880945' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-06-03T19:38:00.088 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:37:59 smithi164 bash[26855]: audit 2023-06-03T19:37:58.505390+0000 mon.a (mon.0) 846 : audit [DBG] from='mgr.24634 172.21.15.164:0/803880945' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-06-03T19:38:00.088 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:37:59 smithi164 bash[26855]: audit 2023-06-03T19:37:58.505719+0000 mon.a (mon.0) 847 : audit [DBG] from='mgr.24634 172.21.15.164:0/803880945' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-06-03T19:38:00.088 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:37:59 smithi164 bash[26855]: audit 2023-06-03T19:37:58.512352+0000 mon.a (mon.0) 848 : audit [DBG] from='mgr.24634 172.21.15.164:0/803880945' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-06-03T19:38:00.088 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:37:59 smithi164 bash[26855]: audit 2023-06-03T19:37:58.512577+0000 mon.a (mon.0) 849 : audit [DBG] from='mgr.24634 172.21.15.164:0/803880945' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-06-03T19:38:00.088 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:37:59 smithi164 bash[26855]: audit 2023-06-03T19:37:58.512818+0000 mon.a (mon.0) 850 : audit [DBG] from='mgr.24634 172.21.15.164:0/803880945' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-06-03T19:38:00.489 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 64 16 89193 89177 87.0787 57.8125 0.0203643 0.0114781 2023-06-03T19:38:01.032 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:38:00 smithi089 bash[26572]: audit 2023-06-03T19:37:58.509464+0000 mon.b (mon.1) 30 : audit [INF] from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2023-06-03T19:38:01.032 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:38:00 smithi089 bash[26572]: audit 2023-06-03T19:37:58.907384+0000 mon.a (mon.0) 851 : audit [DBG] from='mgr.24634 172.21.15.164:0/803880945' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T19:38:01.032 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:38:00 smithi089 bash[26572]: cluster 2023-06-03T19:37:59.595961+0000 mgr.x (mgr.24634) 15 : cluster [DBG] pgmap v13: 89 pgs: 89 active+clean; 5.1 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 7.6 KiB/s rd, 47 MiB/s wr, 766 op/s 2023-06-03T19:38:01.033 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:38:00 smithi089 bash[26572]: audit 2023-06-03T19:37:59.617705+0000 mon.b (mon.1) 31 : audit [INF] from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2023-06-03T19:38:01.033 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:38:00 smithi089 bash[26572]: audit 2023-06-03T19:37:59.643338+0000 mon.a (mon.0) 852 : audit [DBG] from='mgr.24634 172.21.15.164:0/803880945' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-06-03T19:38:01.033 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:38:00 smithi089 bash[26572]: audit 2023-06-03T19:37:59.643553+0000 mon.a (mon.0) 853 : audit [DBG] from='mgr.24634 172.21.15.164:0/803880945' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-06-03T19:38:01.033 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:38:00 smithi089 bash[26572]: audit 2023-06-03T19:37:59.643733+0000 mon.a (mon.0) 854 : audit [DBG] from='mgr.24634 172.21.15.164:0/803880945' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-06-03T19:38:01.034 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:38:00 smithi089 bash[33173]: audit 2023-06-03T19:37:58.509464+0000 mon.b (mon.1) 30 : audit [INF] from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2023-06-03T19:38:01.034 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:38:00 smithi089 bash[33173]: audit 2023-06-03T19:37:58.907384+0000 mon.a (mon.0) 851 : audit [DBG] from='mgr.24634 172.21.15.164:0/803880945' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T19:38:01.034 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:38:00 smithi089 bash[33173]: cluster 2023-06-03T19:37:59.595961+0000 mgr.x (mgr.24634) 15 : cluster [DBG] pgmap v13: 89 pgs: 89 active+clean; 5.1 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 7.6 KiB/s rd, 47 MiB/s wr, 766 op/s 2023-06-03T19:38:01.034 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:38:00 smithi089 bash[33173]: audit 2023-06-03T19:37:59.617705+0000 mon.b (mon.1) 31 : audit [INF] from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2023-06-03T19:38:01.034 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:38:00 smithi089 bash[33173]: audit 2023-06-03T19:37:59.643338+0000 mon.a (mon.0) 852 : audit [DBG] from='mgr.24634 172.21.15.164:0/803880945' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-06-03T19:38:01.034 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:38:00 smithi089 bash[33173]: audit 2023-06-03T19:37:59.643553+0000 mon.a (mon.0) 853 : audit [DBG] from='mgr.24634 172.21.15.164:0/803880945' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-06-03T19:38:01.035 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:38:00 smithi089 bash[33173]: audit 2023-06-03T19:37:59.643733+0000 mon.a (mon.0) 854 : audit [DBG] from='mgr.24634 172.21.15.164:0/803880945' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-06-03T19:38:01.087 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:38:00 smithi164 bash[26855]: audit 2023-06-03T19:37:58.509464+0000 mon.b (mon.1) 30 : audit [INF] from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2023-06-03T19:38:01.087 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:38:00 smithi164 bash[26855]: audit 2023-06-03T19:37:58.907384+0000 mon.a (mon.0) 851 : audit [DBG] from='mgr.24634 172.21.15.164:0/803880945' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T19:38:01.088 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:38:00 smithi164 bash[26855]: cluster 2023-06-03T19:37:59.595961+0000 mgr.x (mgr.24634) 15 : cluster [DBG] pgmap v13: 89 pgs: 89 active+clean; 5.1 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 7.6 KiB/s rd, 47 MiB/s wr, 766 op/s 2023-06-03T19:38:01.088 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:38:00 smithi164 bash[26855]: audit 2023-06-03T19:37:59.617705+0000 mon.b (mon.1) 31 : audit [INF] from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2023-06-03T19:38:01.088 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:38:00 smithi164 bash[26855]: audit 2023-06-03T19:37:59.643338+0000 mon.a (mon.0) 852 : audit [DBG] from='mgr.24634 172.21.15.164:0/803880945' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-06-03T19:38:01.088 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:38:00 smithi164 bash[26855]: audit 2023-06-03T19:37:59.643553+0000 mon.a (mon.0) 853 : audit [DBG] from='mgr.24634 172.21.15.164:0/803880945' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-06-03T19:38:01.088 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:38:00 smithi164 bash[26855]: audit 2023-06-03T19:37:59.643733+0000 mon.a (mon.0) 854 : audit [DBG] from='mgr.24634 172.21.15.164:0/803880945' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-06-03T19:38:01.489 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 65 16 90164 90148 86.6726 60.6875 0.00483078 0.011531 2023-06-03T19:38:02.031 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:38:01 smithi089 bash[33173]: audit 2023-06-03T19:37:59.639396+0000 mon.c (mon.2) 39 : audit [INF] from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2023-06-03T19:38:02.032 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:38:01 smithi089 bash[33173]: audit 2023-06-03T19:38:00.706418+0000 mon.c (mon.2) 40 : audit [INF] from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2023-06-03T19:38:02.032 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:38:01 smithi089 bash[33173]: audit 2023-06-03T19:38:00.711768+0000 mon.c (mon.2) 41 : audit [DBG] from='client.? 172.21.15.89:0/1298738931' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-03T19:38:02.032 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:38:01 smithi089 bash[26572]: audit 2023-06-03T19:37:59.639396+0000 mon.c (mon.2) 39 : audit [INF] from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2023-06-03T19:38:02.032 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:38:01 smithi089 bash[26572]: audit 2023-06-03T19:38:00.706418+0000 mon.c (mon.2) 40 : audit [INF] from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2023-06-03T19:38:02.032 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:38:01 smithi089 bash[26572]: audit 2023-06-03T19:38:00.711768+0000 mon.c (mon.2) 41 : audit [DBG] from='client.? 172.21.15.89:0/1298738931' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-03T19:38:02.087 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:38:01 smithi164 bash[26855]: audit 2023-06-03T19:37:59.639396+0000 mon.c (mon.2) 39 : audit [INF] from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2023-06-03T19:38:02.087 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:38:01 smithi164 bash[26855]: audit 2023-06-03T19:38:00.706418+0000 mon.c (mon.2) 40 : audit [INF] from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2023-06-03T19:38:02.087 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:38:01 smithi164 bash[26855]: audit 2023-06-03T19:38:00.711768+0000 mon.c (mon.2) 41 : audit [DBG] from='client.? 172.21.15.89:0/1298738931' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-03T19:38:02.489 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 66 16 91140 91124 86.2836 61 0.0128744 0.0115842 2023-06-03T19:38:02.587 INFO:journalctl@ceph.prometheus.a.smithi164.stdout:Jun 03 19:38:02 smithi164 bash[48531]: ts=2023-06-03T19:38:02.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi164.front.sepia.ceph.com:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.89:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-06-03T19:38:03.031 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:38:02 smithi089 bash[26572]: cluster 2023-06-03T19:38:01.596700+0000 mgr.x (mgr.24634) 16 : cluster [DBG] pgmap v14: 89 pgs: 89 active+clean; 5.4 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 7.6 KiB/s rd, 70 MiB/s wr, 1.13k op/s 2023-06-03T19:38:03.032 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:38:02 smithi089 bash[33173]: cluster 2023-06-03T19:38:01.596700+0000 mgr.x (mgr.24634) 16 : cluster [DBG] pgmap v14: 89 pgs: 89 active+clean; 5.4 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 7.6 KiB/s rd, 70 MiB/s wr, 1.13k op/s 2023-06-03T19:38:03.087 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:38:02 smithi164 bash[26855]: cluster 2023-06-03T19:38:01.596700+0000 mgr.x (mgr.24634) 16 : cluster [DBG] pgmap v14: 89 pgs: 89 active+clean; 5.4 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 7.6 KiB/s rd, 70 MiB/s wr, 1.13k op/s 2023-06-03T19:38:03.489 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 67 16 92089 92073 85.8809 59.3125 0.0234071 0.0116382 2023-06-03T19:38:03.781 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:38:03 smithi089 bash[26572]: audit 2023-06-03T19:38:03.406343+0000 mon.a (mon.0) 855 : audit [DBG] from='client.? 172.21.15.89:0/2653915133' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-03T19:38:03.782 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:38:03 smithi089 bash[33173]: audit 2023-06-03T19:38:03.406343+0000 mon.a (mon.0) 855 : audit [DBG] from='client.? 172.21.15.89:0/2653915133' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-03T19:38:03.837 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:38:03 smithi164 bash[26855]: audit 2023-06-03T19:38:03.406343+0000 mon.a (mon.0) 855 : audit [DBG] from='client.? 172.21.15.89:0/2653915133' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-03T19:38:04.490 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 68 16 93053 93037 85.5039 60.25 0.00972278 0.0116903 2023-06-03T19:38:05.031 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:38:04 smithi089 bash[33173]: cluster 2023-06-03T19:38:03.597182+0000 mgr.x (mgr.24634) 17 : cluster [DBG] pgmap v15: 89 pgs: 89 active+clean; 5.4 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 767 B/s rd, 46 MiB/s wr, 738 op/s 2023-06-03T19:38:05.032 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:38:04 smithi089 bash[26572]: cluster 2023-06-03T19:38:03.597182+0000 mgr.x (mgr.24634) 17 : cluster [DBG] pgmap v15: 89 pgs: 89 active+clean; 5.4 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 767 B/s rd, 46 MiB/s wr, 738 op/s 2023-06-03T19:38:05.087 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:38:04 smithi164 bash[26855]: cluster 2023-06-03T19:38:03.597182+0000 mgr.x (mgr.24634) 17 : cluster [DBG] pgmap v15: 89 pgs: 89 active+clean; 5.4 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 767 B/s rd, 46 MiB/s wr, 738 op/s 2023-06-03T19:38:05.489 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 69 16 94025 94009 85.1451 60.75 0.0359025 0.0117389 2023-06-03T19:38:06.087 INFO:journalctl@ceph.mgr.x.smithi164.stdout:Jun 03 19:38:05 smithi164 bash[50883]: ::ffff:172.21.15.164 - - [03/Jun/2023:19:38:05] "GET /metrics HTTP/1.1" 200 189935 "" "Prometheus/2.33.4" 2023-06-03T19:38:06.490 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 70 16 94889 94873 84.7001 54 0.0308438 0.0117997 2023-06-03T19:38:07.021 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:38:06 smithi089 bash[26572]: cluster 2023-06-03T19:38:05.598043+0000 mgr.x (mgr.24634) 18 : cluster [DBG] pgmap v16: 89 pgs: 89 active+clean; 5.7 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 1.5 KiB/s rd, 71 MiB/s wr, 1.14k op/s 2023-06-03T19:38:07.021 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:38:06 smithi089 bash[26572]: audit 2023-06-03T19:38:06.054414+0000 mon.a (mon.0) 856 : audit [DBG] from='client.? 172.21.15.89:0/1474667546' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-03T19:38:07.022 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:38:06 smithi089 bash[33173]: cluster 2023-06-03T19:38:05.598043+0000 mgr.x (mgr.24634) 18 : cluster [DBG] pgmap v16: 89 pgs: 89 active+clean; 5.7 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 1.5 KiB/s rd, 71 MiB/s wr, 1.14k op/s 2023-06-03T19:38:07.022 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:38:06 smithi089 bash[33173]: audit 2023-06-03T19:38:06.054414+0000 mon.a (mon.0) 856 : audit [DBG] from='client.? 172.21.15.89:0/1474667546' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-03T19:38:07.087 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:38:06 smithi164 bash[26855]: cluster 2023-06-03T19:38:05.598043+0000 mgr.x (mgr.24634) 18 : cluster [DBG] pgmap v16: 89 pgs: 89 active+clean; 5.7 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 1.5 KiB/s rd, 71 MiB/s wr, 1.14k op/s 2023-06-03T19:38:07.087 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:38:06 smithi164 bash[26855]: audit 2023-06-03T19:38:06.054414+0000 mon.a (mon.0) 856 : audit [DBG] from='client.? 172.21.15.89:0/1474667546' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-03T19:38:07.490 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 71 16 95811 95795 84.3187 57.625 0.0150197 0.0118536 2023-06-03T19:38:08.281 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 19:38:07 smithi089 bash[26845]: ::ffff:172.21.15.164 - - [03/Jun/2023:19:38:07] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2023-06-03T19:38:08.490 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 72 16 96704 96688 83.9227 55.8125 0.00971896 0.0119097 2023-06-03T19:38:09.032 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:38:08 smithi089 bash[26572]: cluster 2023-06-03T19:38:07.598510+0000 mgr.x (mgr.24634) 19 : cluster [DBG] pgmap v17: 89 pgs: 89 active+clean; 5.7 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 767 B/s rd, 48 MiB/s wr, 769 op/s 2023-06-03T19:38:09.033 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:38:08 smithi089 bash[26572]: audit 2023-06-03T19:38:07.726796+0000 mon.a (mon.0) 857 : audit [INF] from='mgr.24634 172.21.15.164:0/803880945' entity='mgr.x' 2023-06-03T19:38:09.033 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:38:08 smithi089 bash[26572]: audit 2023-06-03T19:38:07.735022+0000 mon.a (mon.0) 858 : audit [INF] from='mgr.24634 172.21.15.164:0/803880945' entity='mgr.x' 2023-06-03T19:38:09.033 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:38:08 smithi089 bash[26572]: audit 2023-06-03T19:38:07.743176+0000 mon.a (mon.0) 859 : audit [INF] from='mgr.24634 172.21.15.164:0/803880945' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-06-03T19:38:09.033 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:38:08 smithi089 bash[26572]: audit 2023-06-03T19:38:07.744332+0000 mon.a (mon.0) 860 : audit [INF] from='mgr.24634 172.21.15.164:0/803880945' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-06-03T19:38:09.033 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:38:08 smithi089 bash[26572]: cephadm 2023-06-03T19:38:07.744594+0000 mgr.x (mgr.24634) 20 : cephadm [INF] Adjusting osd_memory_target on smithi164 to 2526M 2023-06-03T19:38:09.034 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:38:08 smithi089 bash[26572]: audit 2023-06-03T19:38:07.745326+0000 mon.a (mon.0) 861 : audit [INF] from='mgr.24634 172.21.15.164:0/803880945' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2023-06-03T19:38:09.034 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:38:08 smithi089 bash[26572]: audit 2023-06-03T19:38:07.746232+0000 mon.a (mon.0) 862 : audit [INF] from='mgr.24634 172.21.15.164:0/803880945' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2023-06-03T19:38:09.034 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:38:08 smithi089 bash[26572]: audit 2023-06-03T19:38:07.753464+0000 mon.a (mon.0) 863 : audit [INF] from='mgr.24634 172.21.15.164:0/803880945' entity='mgr.x' 2023-06-03T19:38:09.034 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:38:08 smithi089 bash[26572]: audit 2023-06-03T19:38:08.080068+0000 mon.a (mon.0) 864 : audit [INF] from='mgr.24634 172.21.15.164:0/803880945' entity='mgr.x' 2023-06-03T19:38:09.034 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:38:08 smithi089 bash[26572]: audit 2023-06-03T19:38:08.090156+0000 mon.a (mon.0) 865 : audit [INF] from='mgr.24634 172.21.15.164:0/803880945' entity='mgr.x' 2023-06-03T19:38:09.034 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:38:08 smithi089 bash[26572]: audit 2023-06-03T19:38:08.091845+0000 mon.a (mon.0) 866 : audit [INF] from='mgr.24634 172.21.15.164:0/803880945' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-06-03T19:38:09.034 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:38:08 smithi089 bash[26572]: audit 2023-06-03T19:38:08.096003+0000 mon.a (mon.0) 867 : audit [INF] from='mgr.24634 172.21.15.164:0/803880945' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-06-03T19:38:09.034 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:38:08 smithi089 bash[26572]: cephadm 2023-06-03T19:38:08.097494+0000 mgr.x (mgr.24634) 21 : cephadm [INF] Adjusting osd_memory_target on smithi089 to 2526M 2023-06-03T19:38:09.035 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:38:08 smithi089 bash[26572]: audit 2023-06-03T19:38:08.097820+0000 mon.a (mon.0) 868 : audit [INF] from='mgr.24634 172.21.15.164:0/803880945' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2023-06-03T19:38:09.035 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:38:08 smithi089 bash[26572]: audit 2023-06-03T19:38:08.099029+0000 mon.a (mon.0) 869 : audit [INF] from='mgr.24634 172.21.15.164:0/803880945' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2023-06-03T19:38:09.035 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:38:08 smithi089 bash[26572]: audit 2023-06-03T19:38:08.108145+0000 mon.a (mon.0) 870 : audit [INF] from='mgr.24634 172.21.15.164:0/803880945' entity='mgr.x' 2023-06-03T19:38:09.035 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:38:08 smithi089 bash[26572]: audit 2023-06-03T19:38:08.110004+0000 mon.a (mon.0) 871 : audit [DBG] from='mgr.24634 172.21.15.164:0/803880945' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-03T19:38:09.035 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:38:08 smithi089 bash[26572]: cephadm 2023-06-03T19:38:08.111065+0000 mgr.x (mgr.24634) 22 : cephadm [INF] Updating smithi089:/etc/ceph/ceph.conf 2023-06-03T19:38:09.035 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:38:08 smithi089 bash[26572]: cephadm 2023-06-03T19:38:08.111333+0000 mgr.x (mgr.24634) 23 : cephadm [INF] Updating smithi164:/etc/ceph/ceph.conf 2023-06-03T19:38:09.035 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:38:08 smithi089 bash[26572]: audit 2023-06-03T19:38:08.111701+0000 mon.a (mon.0) 872 : audit [INF] from='mgr.24634 172.21.15.164:0/803880945' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-03T19:38:09.035 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:38:08 smithi089 bash[26572]: cephadm 2023-06-03T19:38:08.329138+0000 mgr.x (mgr.24634) 24 : cephadm [INF] Updating smithi164:/var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/config/ceph.conf 2023-06-03T19:38:09.036 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:38:08 smithi089 bash[26572]: cephadm 2023-06-03T19:38:08.330888+0000 mgr.x (mgr.24634) 25 : cephadm [INF] Updating smithi089:/var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/config/ceph.conf 2023-06-03T19:38:09.036 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:38:08 smithi089 bash[33173]: cluster 2023-06-03T19:38:07.598510+0000 mgr.x (mgr.24634) 19 : cluster [DBG] pgmap v17: 89 pgs: 89 active+clean; 5.7 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 767 B/s rd, 48 MiB/s wr, 769 op/s 2023-06-03T19:38:09.036 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:38:08 smithi089 bash[33173]: audit 2023-06-03T19:38:07.726796+0000 mon.a (mon.0) 857 : audit [INF] from='mgr.24634 172.21.15.164:0/803880945' entity='mgr.x' 2023-06-03T19:38:09.036 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:38:08 smithi089 bash[33173]: audit 2023-06-03T19:38:07.735022+0000 mon.a (mon.0) 858 : audit [INF] from='mgr.24634 172.21.15.164:0/803880945' entity='mgr.x' 2023-06-03T19:38:09.036 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:38:08 smithi089 bash[33173]: audit 2023-06-03T19:38:07.743176+0000 mon.a (mon.0) 859 : audit [INF] from='mgr.24634 172.21.15.164:0/803880945' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-06-03T19:38:09.037 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:38:08 smithi089 bash[33173]: audit 2023-06-03T19:38:07.744332+0000 mon.a (mon.0) 860 : audit [INF] from='mgr.24634 172.21.15.164:0/803880945' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-06-03T19:38:09.037 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:38:08 smithi089 bash[33173]: cephadm 2023-06-03T19:38:07.744594+0000 mgr.x (mgr.24634) 20 : cephadm [INF] Adjusting osd_memory_target on smithi164 to 2526M 2023-06-03T19:38:09.037 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:38:08 smithi089 bash[33173]: audit 2023-06-03T19:38:07.745326+0000 mon.a (mon.0) 861 : audit [INF] from='mgr.24634 172.21.15.164:0/803880945' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2023-06-03T19:38:09.037 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:38:08 smithi089 bash[33173]: audit 2023-06-03T19:38:07.746232+0000 mon.a (mon.0) 862 : audit [INF] from='mgr.24634 172.21.15.164:0/803880945' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2023-06-03T19:38:09.037 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:38:08 smithi089 bash[33173]: audit 2023-06-03T19:38:07.753464+0000 mon.a (mon.0) 863 : audit [INF] from='mgr.24634 172.21.15.164:0/803880945' entity='mgr.x' 2023-06-03T19:38:09.037 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:38:08 smithi089 bash[33173]: audit 2023-06-03T19:38:08.080068+0000 mon.a (mon.0) 864 : audit [INF] from='mgr.24634 172.21.15.164:0/803880945' entity='mgr.x' 2023-06-03T19:38:09.037 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:38:08 smithi089 bash[33173]: audit 2023-06-03T19:38:08.090156+0000 mon.a (mon.0) 865 : audit [INF] from='mgr.24634 172.21.15.164:0/803880945' entity='mgr.x' 2023-06-03T19:38:09.037 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:38:08 smithi089 bash[33173]: audit 2023-06-03T19:38:08.091845+0000 mon.a (mon.0) 866 : audit [INF] from='mgr.24634 172.21.15.164:0/803880945' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-06-03T19:38:09.038 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:38:08 smithi089 bash[33173]: audit 2023-06-03T19:38:08.096003+0000 mon.a (mon.0) 867 : audit [INF] from='mgr.24634 172.21.15.164:0/803880945' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-06-03T19:38:09.038 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:38:08 smithi089 bash[33173]: cephadm 2023-06-03T19:38:08.097494+0000 mgr.x (mgr.24634) 21 : cephadm [INF] Adjusting osd_memory_target on smithi089 to 2526M 2023-06-03T19:38:09.038 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:38:08 smithi089 bash[33173]: audit 2023-06-03T19:38:08.097820+0000 mon.a (mon.0) 868 : audit [INF] from='mgr.24634 172.21.15.164:0/803880945' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2023-06-03T19:38:09.038 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:38:08 smithi089 bash[33173]: audit 2023-06-03T19:38:08.099029+0000 mon.a (mon.0) 869 : audit [INF] from='mgr.24634 172.21.15.164:0/803880945' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2023-06-03T19:38:09.038 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:38:08 smithi089 bash[33173]: audit 2023-06-03T19:38:08.108145+0000 mon.a (mon.0) 870 : audit [INF] from='mgr.24634 172.21.15.164:0/803880945' entity='mgr.x' 2023-06-03T19:38:09.038 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:38:08 smithi089 bash[33173]: audit 2023-06-03T19:38:08.110004+0000 mon.a (mon.0) 871 : audit [DBG] from='mgr.24634 172.21.15.164:0/803880945' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-03T19:38:09.038 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:38:08 smithi089 bash[33173]: cephadm 2023-06-03T19:38:08.111065+0000 mgr.x (mgr.24634) 22 : cephadm [INF] Updating smithi089:/etc/ceph/ceph.conf 2023-06-03T19:38:09.038 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:38:08 smithi089 bash[33173]: cephadm 2023-06-03T19:38:08.111333+0000 mgr.x (mgr.24634) 23 : cephadm [INF] Updating smithi164:/etc/ceph/ceph.conf 2023-06-03T19:38:09.039 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:38:08 smithi089 bash[33173]: audit 2023-06-03T19:38:08.111701+0000 mon.a (mon.0) 872 : audit [INF] from='mgr.24634 172.21.15.164:0/803880945' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-03T19:38:09.039 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:38:08 smithi089 bash[33173]: cephadm 2023-06-03T19:38:08.329138+0000 mgr.x (mgr.24634) 24 : cephadm [INF] Updating smithi164:/var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/config/ceph.conf 2023-06-03T19:38:09.039 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:38:08 smithi089 bash[33173]: cephadm 2023-06-03T19:38:08.330888+0000 mgr.x (mgr.24634) 25 : cephadm [INF] Updating smithi089:/var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/config/ceph.conf 2023-06-03T19:38:09.087 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:38:08 smithi164 bash[26855]: cluster 2023-06-03T19:38:07.598510+0000 mgr.x (mgr.24634) 19 : cluster [DBG] pgmap v17: 89 pgs: 89 active+clean; 5.7 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 767 B/s rd, 48 MiB/s wr, 769 op/s 2023-06-03T19:38:09.088 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:38:08 smithi164 bash[26855]: audit 2023-06-03T19:38:07.726796+0000 mon.a (mon.0) 857 : audit [INF] from='mgr.24634 172.21.15.164:0/803880945' entity='mgr.x' 2023-06-03T19:38:09.088 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:38:08 smithi164 bash[26855]: audit 2023-06-03T19:38:07.735022+0000 mon.a (mon.0) 858 : audit [INF] from='mgr.24634 172.21.15.164:0/803880945' entity='mgr.x' 2023-06-03T19:38:09.088 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:38:08 smithi164 bash[26855]: audit 2023-06-03T19:38:07.743176+0000 mon.a (mon.0) 859 : audit [INF] from='mgr.24634 172.21.15.164:0/803880945' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-06-03T19:38:09.088 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:38:08 smithi164 bash[26855]: audit 2023-06-03T19:38:07.744332+0000 mon.a (mon.0) 860 : audit [INF] from='mgr.24634 172.21.15.164:0/803880945' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-06-03T19:38:09.088 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:38:08 smithi164 bash[26855]: cephadm 2023-06-03T19:38:07.744594+0000 mgr.x (mgr.24634) 20 : cephadm [INF] Adjusting osd_memory_target on smithi164 to 2526M 2023-06-03T19:38:09.088 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:38:08 smithi164 bash[26855]: audit 2023-06-03T19:38:07.745326+0000 mon.a (mon.0) 861 : audit [INF] from='mgr.24634 172.21.15.164:0/803880945' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2023-06-03T19:38:09.089 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:38:08 smithi164 bash[26855]: audit 2023-06-03T19:38:07.746232+0000 mon.a (mon.0) 862 : audit [INF] from='mgr.24634 172.21.15.164:0/803880945' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2023-06-03T19:38:09.089 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:38:08 smithi164 bash[26855]: audit 2023-06-03T19:38:07.753464+0000 mon.a (mon.0) 863 : audit [INF] from='mgr.24634 172.21.15.164:0/803880945' entity='mgr.x' 2023-06-03T19:38:09.089 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:38:08 smithi164 bash[26855]: audit 2023-06-03T19:38:08.080068+0000 mon.a (mon.0) 864 : audit [INF] from='mgr.24634 172.21.15.164:0/803880945' entity='mgr.x' 2023-06-03T19:38:09.089 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:38:08 smithi164 bash[26855]: audit 2023-06-03T19:38:08.090156+0000 mon.a (mon.0) 865 : audit [INF] from='mgr.24634 172.21.15.164:0/803880945' entity='mgr.x' 2023-06-03T19:38:09.089 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:38:08 smithi164 bash[26855]: audit 2023-06-03T19:38:08.091845+0000 mon.a (mon.0) 866 : audit [INF] from='mgr.24634 172.21.15.164:0/803880945' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-06-03T19:38:09.089 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:38:08 smithi164 bash[26855]: audit 2023-06-03T19:38:08.096003+0000 mon.a (mon.0) 867 : audit [INF] from='mgr.24634 172.21.15.164:0/803880945' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-06-03T19:38:09.089 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:38:08 smithi164 bash[26855]: cephadm 2023-06-03T19:38:08.097494+0000 mgr.x (mgr.24634) 21 : cephadm [INF] Adjusting osd_memory_target on smithi089 to 2526M 2023-06-03T19:38:09.089 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:38:08 smithi164 bash[26855]: audit 2023-06-03T19:38:08.097820+0000 mon.a (mon.0) 868 : audit [INF] from='mgr.24634 172.21.15.164:0/803880945' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2023-06-03T19:38:09.090 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:38:08 smithi164 bash[26855]: audit 2023-06-03T19:38:08.099029+0000 mon.a (mon.0) 869 : audit [INF] from='mgr.24634 172.21.15.164:0/803880945' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2023-06-03T19:38:09.090 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:38:08 smithi164 bash[26855]: audit 2023-06-03T19:38:08.108145+0000 mon.a (mon.0) 870 : audit [INF] from='mgr.24634 172.21.15.164:0/803880945' entity='mgr.x' 2023-06-03T19:38:09.090 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:38:08 smithi164 bash[26855]: audit 2023-06-03T19:38:08.110004+0000 mon.a (mon.0) 871 : audit [DBG] from='mgr.24634 172.21.15.164:0/803880945' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-03T19:38:09.090 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:38:08 smithi164 bash[26855]: cephadm 2023-06-03T19:38:08.111065+0000 mgr.x (mgr.24634) 22 : cephadm [INF] Updating smithi089:/etc/ceph/ceph.conf 2023-06-03T19:38:09.090 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:38:08 smithi164 bash[26855]: cephadm 2023-06-03T19:38:08.111333+0000 mgr.x (mgr.24634) 23 : cephadm [INF] Updating smithi164:/etc/ceph/ceph.conf 2023-06-03T19:38:09.090 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:38:08 smithi164 bash[26855]: audit 2023-06-03T19:38:08.111701+0000 mon.a (mon.0) 872 : audit [INF] from='mgr.24634 172.21.15.164:0/803880945' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-03T19:38:09.090 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:38:08 smithi164 bash[26855]: cephadm 2023-06-03T19:38:08.329138+0000 mgr.x (mgr.24634) 24 : cephadm [INF] Updating smithi164:/var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/config/ceph.conf 2023-06-03T19:38:09.091 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:38:08 smithi164 bash[26855]: cephadm 2023-06-03T19:38:08.330888+0000 mgr.x (mgr.24634) 25 : cephadm [INF] Updating smithi089:/var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/config/ceph.conf 2023-06-03T19:38:09.490 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 73 16 97539 97523 83.4879 52.1875 0.0144797 0.011972 2023-06-03T19:38:09.782 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:38:09 smithi089 bash[26572]: cephadm 2023-06-03T19:38:08.546032+0000 mgr.x (mgr.24634) 26 : cephadm [INF] Updating smithi164:/etc/ceph/ceph.client.admin.keyring 2023-06-03T19:38:09.782 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:38:09 smithi089 bash[26572]: cephadm 2023-06-03T19:38:08.549341+0000 mgr.x (mgr.24634) 27 : cephadm [INF] Updating smithi089:/etc/ceph/ceph.client.admin.keyring 2023-06-03T19:38:09.782 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:38:09 smithi089 bash[26572]: audit 2023-06-03T19:38:08.740754+0000 mon.c (mon.2) 42 : audit [DBG] from='client.? 172.21.15.89:0/2303508963' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-03T19:38:09.783 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:38:09 smithi089 bash[26572]: cephadm 2023-06-03T19:38:08.766428+0000 mgr.x (mgr.24634) 28 : cephadm [INF] Updating smithi164:/var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/config/ceph.client.admin.keyring 2023-06-03T19:38:09.783 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:38:09 smithi089 bash[26572]: cephadm 2023-06-03T19:38:08.818926+0000 mgr.x (mgr.24634) 29 : cephadm [INF] Updating smithi089:/var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/config/ceph.client.admin.keyring 2023-06-03T19:38:09.783 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:38:09 smithi089 bash[26572]: audit 2023-06-03T19:38:08.946985+0000 mon.a (mon.0) 873 : audit [INF] from='mgr.24634 172.21.15.164:0/803880945' entity='mgr.x' 2023-06-03T19:38:09.783 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:38:09 smithi089 bash[26572]: audit 2023-06-03T19:38:08.955255+0000 mon.a (mon.0) 874 : audit [INF] from='mgr.24634 172.21.15.164:0/803880945' entity='mgr.x' 2023-06-03T19:38:09.783 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:38:09 smithi089 bash[26572]: audit 2023-06-03T19:38:09.049818+0000 mon.a (mon.0) 875 : audit [INF] from='mgr.24634 172.21.15.164:0/803880945' entity='mgr.x' 2023-06-03T19:38:09.783 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:38:09 smithi089 bash[26572]: audit 2023-06-03T19:38:09.057068+0000 mon.a (mon.0) 876 : audit [INF] from='mgr.24634 172.21.15.164:0/803880945' entity='mgr.x' 2023-06-03T19:38:09.783 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:38:09 smithi089 bash[26572]: audit 2023-06-03T19:38:09.069078+0000 mon.a (mon.0) 877 : audit [INF] from='mgr.24634 172.21.15.164:0/803880945' entity='mgr.x' 2023-06-03T19:38:09.784 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:38:09 smithi089 bash[26572]: cephadm 2023-06-03T19:38:09.143428+0000 mgr.x (mgr.24634) 30 : cephadm [INF] Reconfiguring node-exporter.a (dependencies changed)... 2023-06-03T19:38:09.784 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:38:09 smithi089 bash[26572]: cephadm 2023-06-03T19:38:09.143826+0000 mgr.x (mgr.24634) 31 : cephadm [INF] Reconfiguring daemon node-exporter.a on smithi089 2023-06-03T19:38:09.784 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:38:09 smithi089 bash[33173]: cephadm 2023-06-03T19:38:08.546032+0000 mgr.x (mgr.24634) 26 : cephadm [INF] Updating smithi164:/etc/ceph/ceph.client.admin.keyring 2023-06-03T19:38:09.784 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:38:09 smithi089 bash[33173]: cephadm 2023-06-03T19:38:08.549341+0000 mgr.x (mgr.24634) 27 : cephadm [INF] Updating smithi089:/etc/ceph/ceph.client.admin.keyring 2023-06-03T19:38:09.785 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:38:09 smithi089 bash[33173]: audit 2023-06-03T19:38:08.740754+0000 mon.c (mon.2) 42 : audit [DBG] from='client.? 172.21.15.89:0/2303508963' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-03T19:38:09.785 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:38:09 smithi089 bash[33173]: cephadm 2023-06-03T19:38:08.766428+0000 mgr.x (mgr.24634) 28 : cephadm [INF] Updating smithi164:/var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/config/ceph.client.admin.keyring 2023-06-03T19:38:09.785 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:38:09 smithi089 bash[33173]: cephadm 2023-06-03T19:38:08.818926+0000 mgr.x (mgr.24634) 29 : cephadm [INF] Updating smithi089:/var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/config/ceph.client.admin.keyring 2023-06-03T19:38:09.785 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:38:09 smithi089 bash[33173]: audit 2023-06-03T19:38:08.946985+0000 mon.a (mon.0) 873 : audit [INF] from='mgr.24634 172.21.15.164:0/803880945' entity='mgr.x' 2023-06-03T19:38:09.786 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:38:09 smithi089 bash[33173]: audit 2023-06-03T19:38:08.955255+0000 mon.a (mon.0) 874 : audit [INF] from='mgr.24634 172.21.15.164:0/803880945' entity='mgr.x' 2023-06-03T19:38:09.786 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:38:09 smithi089 bash[33173]: audit 2023-06-03T19:38:09.049818+0000 mon.a (mon.0) 875 : audit [INF] from='mgr.24634 172.21.15.164:0/803880945' entity='mgr.x' 2023-06-03T19:38:09.786 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:38:09 smithi089 bash[33173]: audit 2023-06-03T19:38:09.057068+0000 mon.a (mon.0) 876 : audit [INF] from='mgr.24634 172.21.15.164:0/803880945' entity='mgr.x' 2023-06-03T19:38:09.786 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:38:09 smithi089 bash[33173]: audit 2023-06-03T19:38:09.069078+0000 mon.a (mon.0) 877 : audit [INF] from='mgr.24634 172.21.15.164:0/803880945' entity='mgr.x' 2023-06-03T19:38:09.786 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:38:09 smithi089 bash[33173]: cephadm 2023-06-03T19:38:09.143428+0000 mgr.x (mgr.24634) 30 : cephadm [INF] Reconfiguring node-exporter.a (dependencies changed)... 2023-06-03T19:38:09.787 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:38:09 smithi089 bash[33173]: cephadm 2023-06-03T19:38:09.143826+0000 mgr.x (mgr.24634) 31 : cephadm [INF] Reconfiguring daemon node-exporter.a on smithi089 2023-06-03T19:38:09.787 INFO:journalctl@ceph.node-exporter.a.smithi089.stdout:Jun 03 19:38:09 smithi089 systemd[1]: Stopping Ceph node-exporter.a for f1db3478-0243-11ee-9b1d-001a4aab830c... 2023-06-03T19:38:09.787 INFO:journalctl@ceph.node-exporter.a.smithi089.stdout:Jun 03 19:38:09 smithi089 bash[69347]: Error response from daemon: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-node-exporter.a 2023-06-03T19:38:10.087 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:38:09 smithi164 bash[26855]: cephadm 2023-06-03T19:38:08.546032+0000 mgr.x (mgr.24634) 26 : cephadm [INF] Updating smithi164:/etc/ceph/ceph.client.admin.keyring 2023-06-03T19:38:10.087 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:38:09 smithi164 bash[26855]: cephadm 2023-06-03T19:38:08.549341+0000 mgr.x (mgr.24634) 27 : cephadm [INF] Updating smithi089:/etc/ceph/ceph.client.admin.keyring 2023-06-03T19:38:10.088 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:38:09 smithi164 bash[26855]: audit 2023-06-03T19:38:08.740754+0000 mon.c (mon.2) 42 : audit [DBG] from='client.? 172.21.15.89:0/2303508963' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-03T19:38:10.088 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:38:09 smithi164 bash[26855]: cephadm 2023-06-03T19:38:08.766428+0000 mgr.x (mgr.24634) 28 : cephadm [INF] Updating smithi164:/var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/config/ceph.client.admin.keyring 2023-06-03T19:38:10.088 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:38:09 smithi164 bash[26855]: cephadm 2023-06-03T19:38:08.818926+0000 mgr.x (mgr.24634) 29 : cephadm [INF] Updating smithi089:/var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/config/ceph.client.admin.keyring 2023-06-03T19:38:10.088 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:38:09 smithi164 bash[26855]: audit 2023-06-03T19:38:08.946985+0000 mon.a (mon.0) 873 : audit [INF] from='mgr.24634 172.21.15.164:0/803880945' entity='mgr.x' 2023-06-03T19:38:10.088 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:38:09 smithi164 bash[26855]: audit 2023-06-03T19:38:08.955255+0000 mon.a (mon.0) 874 : audit [INF] from='mgr.24634 172.21.15.164:0/803880945' entity='mgr.x' 2023-06-03T19:38:10.088 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:38:09 smithi164 bash[26855]: audit 2023-06-03T19:38:09.049818+0000 mon.a (mon.0) 875 : audit [INF] from='mgr.24634 172.21.15.164:0/803880945' entity='mgr.x' 2023-06-03T19:38:10.088 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:38:09 smithi164 bash[26855]: audit 2023-06-03T19:38:09.057068+0000 mon.a (mon.0) 876 : audit [INF] from='mgr.24634 172.21.15.164:0/803880945' entity='mgr.x' 2023-06-03T19:38:10.089 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:38:09 smithi164 bash[26855]: audit 2023-06-03T19:38:09.069078+0000 mon.a (mon.0) 877 : audit [INF] from='mgr.24634 172.21.15.164:0/803880945' entity='mgr.x' 2023-06-03T19:38:10.089 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:38:09 smithi164 bash[26855]: cephadm 2023-06-03T19:38:09.143428+0000 mgr.x (mgr.24634) 30 : cephadm [INF] Reconfiguring node-exporter.a (dependencies changed)... 2023-06-03T19:38:10.089 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:38:09 smithi164 bash[26855]: cephadm 2023-06-03T19:38:09.143826+0000 mgr.x (mgr.24634) 31 : cephadm [INF] Reconfiguring daemon node-exporter.a on smithi089 2023-06-03T19:38:10.490 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 74 16 98456 98440 83.1341 57.3125 0.00921905 0.0120228 2023-06-03T19:38:11.281 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:38:11 smithi089 bash[26572]: cluster 2023-06-03T19:38:09.598947+0000 mgr.x (mgr.24634) 32 : cluster [DBG] pgmap v18: 89 pgs: 89 active+clean; 5.7 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 767 B/s rd, 48 MiB/s wr, 769 op/s 2023-06-03T19:38:11.282 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:38:11 smithi089 bash[33173]: cluster 2023-06-03T19:38:09.598947+0000 mgr.x (mgr.24634) 32 : cluster [DBG] pgmap v18: 89 pgs: 89 active+clean; 5.7 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 767 B/s rd, 48 MiB/s wr, 769 op/s 2023-06-03T19:38:11.282 INFO:journalctl@ceph.node-exporter.a.smithi089.stdout:Jun 03 19:38:11 smithi089 bash[69384]: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-node-exporter-a 2023-06-03T19:38:11.282 INFO:journalctl@ceph.node-exporter.a.smithi089.stdout:Jun 03 19:38:11 smithi089 systemd[1]: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c@node-exporter.a.service: Main process exited, code=exited, status=143/n/a 2023-06-03T19:38:11.283 INFO:journalctl@ceph.node-exporter.a.smithi089.stdout:Jun 03 19:38:11 smithi089 systemd[1]: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c@node-exporter.a.service: Failed with result 'exit-code'. 2023-06-03T19:38:11.283 INFO:journalctl@ceph.node-exporter.a.smithi089.stdout:Jun 03 19:38:11 smithi089 systemd[1]: Stopped Ceph node-exporter.a for f1db3478-0243-11ee-9b1d-001a4aab830c. 2023-06-03T19:38:11.283 INFO:journalctl@ceph.node-exporter.a.smithi089.stdout:Jun 03 19:38:11 smithi089 systemd[1]: Started Ceph node-exporter.a for f1db3478-0243-11ee-9b1d-001a4aab830c. 2023-06-03T19:38:11.337 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:38:11 smithi164 bash[26855]: cluster 2023-06-03T19:38:09.598947+0000 mgr.x (mgr.24634) 32 : cluster [DBG] pgmap v18: 89 pgs: 89 active+clean; 5.7 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 767 B/s rd, 48 MiB/s wr, 769 op/s 2023-06-03T19:38:11.490 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 75 16 99308 99292 82.7356 53.25 0.0161003 0.0120808 2023-06-03T19:38:12.490 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 76 16 100170 100154 82.3557 53.875 0.0201003 0.0121365 2023-06-03T19:38:12.532 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:38:12 smithi089 bash[26572]: audit 2023-06-03T19:38:11.142244+0000 mon.a (mon.0) 878 : audit [INF] from='mgr.24634 172.21.15.164:0/803880945' entity='mgr.x' 2023-06-03T19:38:12.532 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:38:12 smithi089 bash[26572]: audit 2023-06-03T19:38:11.157097+0000 mon.a (mon.0) 879 : audit [INF] from='mgr.24634 172.21.15.164:0/803880945' entity='mgr.x' 2023-06-03T19:38:12.532 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:38:12 smithi089 bash[26572]: cephadm 2023-06-03T19:38:11.171425+0000 mgr.x (mgr.24634) 33 : cephadm [INF] Reconfiguring alertmanager.a (dependencies changed)... 2023-06-03T19:38:12.532 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:38:12 smithi089 bash[26572]: cephadm 2023-06-03T19:38:11.191628+0000 mgr.x (mgr.24634) 34 : cephadm [INF] Reconfiguring daemon alertmanager.a on smithi089 2023-06-03T19:38:12.533 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:38:12 smithi089 bash[26572]: audit 2023-06-03T19:38:11.453557+0000 mon.c (mon.2) 43 : audit [DBG] from='client.? 172.21.15.89:0/3421743726' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-03T19:38:12.533 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:38:12 smithi089 bash[33173]: audit 2023-06-03T19:38:11.142244+0000 mon.a (mon.0) 878 : audit [INF] from='mgr.24634 172.21.15.164:0/803880945' entity='mgr.x' 2023-06-03T19:38:12.533 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:38:12 smithi089 bash[33173]: audit 2023-06-03T19:38:11.157097+0000 mon.a (mon.0) 879 : audit [INF] from='mgr.24634 172.21.15.164:0/803880945' entity='mgr.x' 2023-06-03T19:38:12.533 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:38:12 smithi089 bash[33173]: cephadm 2023-06-03T19:38:11.171425+0000 mgr.x (mgr.24634) 33 : cephadm [INF] Reconfiguring alertmanager.a (dependencies changed)... 2023-06-03T19:38:12.533 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:38:12 smithi089 bash[33173]: cephadm 2023-06-03T19:38:11.191628+0000 mgr.x (mgr.24634) 34 : cephadm [INF] Reconfiguring daemon alertmanager.a on smithi089 2023-06-03T19:38:12.533 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:38:12 smithi089 bash[33173]: audit 2023-06-03T19:38:11.453557+0000 mon.c (mon.2) 43 : audit [DBG] from='client.? 172.21.15.89:0/3421743726' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-03T19:38:12.587 INFO:journalctl@ceph.prometheus.a.smithi164.stdout:Jun 03 19:38:12 smithi164 bash[48531]: ts=2023-06-03T19:38:12.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi164.front.sepia.ceph.com:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.89:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-06-03T19:38:12.588 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:38:12 smithi164 bash[26855]: audit 2023-06-03T19:38:11.142244+0000 mon.a (mon.0) 878 : audit [INF] from='mgr.24634 172.21.15.164:0/803880945' entity='mgr.x' 2023-06-03T19:38:12.588 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:38:12 smithi164 bash[26855]: audit 2023-06-03T19:38:11.157097+0000 mon.a (mon.0) 879 : audit [INF] from='mgr.24634 172.21.15.164:0/803880945' entity='mgr.x' 2023-06-03T19:38:12.588 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:38:12 smithi164 bash[26855]: cephadm 2023-06-03T19:38:11.171425+0000 mgr.x (mgr.24634) 33 : cephadm [INF] Reconfiguring alertmanager.a (dependencies changed)... 2023-06-03T19:38:12.588 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:38:12 smithi164 bash[26855]: cephadm 2023-06-03T19:38:11.191628+0000 mgr.x (mgr.24634) 34 : cephadm [INF] Reconfiguring daemon alertmanager.a on smithi089 2023-06-03T19:38:12.588 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:38:12 smithi164 bash[26855]: audit 2023-06-03T19:38:11.453557+0000 mon.c (mon.2) 43 : audit [DBG] from='client.? 172.21.15.89:0/3421743726' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-03T19:38:13.154 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:38:13 smithi089 bash[26572]: cluster 2023-06-03T19:38:11.599712+0000 mgr.x (mgr.24634) 35 : cluster [DBG] pgmap v19: 89 pgs: 89 active+clean; 6.0 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 767 B/s rd, 71 MiB/s wr, 1.14k op/s 2023-06-03T19:38:13.155 INFO:journalctl@ceph.node-exporter.a.smithi089.stdout:Jun 03 19:38:12 smithi089 bash[69490]: ts=2023-06-03T19:38:12.934Z caller=node_exporter.go:182 level=info msg="Starting node_exporter" version="(version=1.3.1, branch=HEAD, revision=a2321e7b940ddcff26873612bccdf7cd4c42b6b6)" 2023-06-03T19:38:13.155 INFO:journalctl@ceph.node-exporter.a.smithi089.stdout:Jun 03 19:38:12 smithi089 bash[69490]: ts=2023-06-03T19:38:12.934Z caller=node_exporter.go:183 level=info msg="Build context" build_context="(go=go1.17.3, user=root@243aafa5525c, date=20211205-11:09:49)" 2023-06-03T19:38:13.155 INFO:journalctl@ceph.node-exporter.a.smithi089.stdout:Jun 03 19:38:12 smithi089 bash[69490]: ts=2023-06-03T19:38:12.934Z caller=filesystem_common.go:111 level=info collector=filesystem msg="Parsed flag --collector.filesystem.mount-points-exclude" flag=^/(dev|proc|run/credentials/.+|sys|var/lib/docker/.+)($|/) 2023-06-03T19:38:13.155 INFO:journalctl@ceph.node-exporter.a.smithi089.stdout:Jun 03 19:38:12 smithi089 bash[69490]: ts=2023-06-03T19:38:12.934Z caller=filesystem_common.go:113 level=info collector=filesystem msg="Parsed flag --collector.filesystem.fs-types-exclude" flag=^(autofs|binfmt_misc|bpf|cgroup2?|configfs|debugfs|devpts|devtmpfs|fusectl|hugetlbfs|iso9660|mqueue|nsfs|overlay|proc|procfs|pstore|rpc_pipefs|securityfs|selinuxfs|squashfs|sysfs|tracefs)$ 2023-06-03T19:38:13.155 INFO:journalctl@ceph.node-exporter.a.smithi089.stdout:Jun 03 19:38:12 smithi089 bash[69490]: ts=2023-06-03T19:38:12.936Z caller=node_exporter.go:108 level=info msg="Enabled collectors" 2023-06-03T19:38:13.156 INFO:journalctl@ceph.node-exporter.a.smithi089.stdout:Jun 03 19:38:12 smithi089 bash[69490]: ts=2023-06-03T19:38:12.936Z caller=node_exporter.go:115 level=info collector=arp 2023-06-03T19:38:13.156 INFO:journalctl@ceph.node-exporter.a.smithi089.stdout:Jun 03 19:38:12 smithi089 bash[69490]: ts=2023-06-03T19:38:12.936Z caller=node_exporter.go:115 level=info collector=bcache 2023-06-03T19:38:13.156 INFO:journalctl@ceph.node-exporter.a.smithi089.stdout:Jun 03 19:38:12 smithi089 bash[69490]: ts=2023-06-03T19:38:12.936Z caller=node_exporter.go:115 level=info collector=bonding 2023-06-03T19:38:13.156 INFO:journalctl@ceph.node-exporter.a.smithi089.stdout:Jun 03 19:38:12 smithi089 bash[69490]: ts=2023-06-03T19:38:12.936Z caller=node_exporter.go:115 level=info collector=btrfs 2023-06-03T19:38:13.156 INFO:journalctl@ceph.node-exporter.a.smithi089.stdout:Jun 03 19:38:12 smithi089 bash[69490]: ts=2023-06-03T19:38:12.936Z caller=node_exporter.go:115 level=info collector=conntrack 2023-06-03T19:38:13.156 INFO:journalctl@ceph.node-exporter.a.smithi089.stdout:Jun 03 19:38:12 smithi089 bash[69490]: ts=2023-06-03T19:38:12.936Z caller=node_exporter.go:115 level=info collector=cpu 2023-06-03T19:38:13.157 INFO:journalctl@ceph.node-exporter.a.smithi089.stdout:Jun 03 19:38:12 smithi089 bash[69490]: ts=2023-06-03T19:38:12.936Z caller=node_exporter.go:115 level=info collector=cpufreq 2023-06-03T19:38:13.157 INFO:journalctl@ceph.node-exporter.a.smithi089.stdout:Jun 03 19:38:12 smithi089 bash[69490]: ts=2023-06-03T19:38:12.936Z caller=node_exporter.go:115 level=info collector=diskstats 2023-06-03T19:38:13.157 INFO:journalctl@ceph.node-exporter.a.smithi089.stdout:Jun 03 19:38:12 smithi089 bash[69490]: ts=2023-06-03T19:38:12.937Z caller=node_exporter.go:115 level=info collector=dmi 2023-06-03T19:38:13.157 INFO:journalctl@ceph.node-exporter.a.smithi089.stdout:Jun 03 19:38:12 smithi089 bash[69490]: ts=2023-06-03T19:38:12.937Z caller=node_exporter.go:115 level=info collector=edac 2023-06-03T19:38:13.157 INFO:journalctl@ceph.node-exporter.a.smithi089.stdout:Jun 03 19:38:12 smithi089 bash[69490]: ts=2023-06-03T19:38:12.937Z caller=node_exporter.go:115 level=info collector=entropy 2023-06-03T19:38:13.157 INFO:journalctl@ceph.node-exporter.a.smithi089.stdout:Jun 03 19:38:12 smithi089 bash[69490]: ts=2023-06-03T19:38:12.937Z caller=node_exporter.go:115 level=info collector=fibrechannel 2023-06-03T19:38:13.157 INFO:journalctl@ceph.node-exporter.a.smithi089.stdout:Jun 03 19:38:12 smithi089 bash[69490]: ts=2023-06-03T19:38:12.937Z caller=node_exporter.go:115 level=info collector=filefd 2023-06-03T19:38:13.158 INFO:journalctl@ceph.node-exporter.a.smithi089.stdout:Jun 03 19:38:12 smithi089 bash[69490]: ts=2023-06-03T19:38:12.937Z caller=node_exporter.go:115 level=info collector=filesystem 2023-06-03T19:38:13.158 INFO:journalctl@ceph.node-exporter.a.smithi089.stdout:Jun 03 19:38:12 smithi089 bash[69490]: ts=2023-06-03T19:38:12.937Z caller=node_exporter.go:115 level=info collector=hwmon 2023-06-03T19:38:13.158 INFO:journalctl@ceph.node-exporter.a.smithi089.stdout:Jun 03 19:38:12 smithi089 bash[69490]: ts=2023-06-03T19:38:12.937Z caller=node_exporter.go:115 level=info collector=infiniband 2023-06-03T19:38:13.158 INFO:journalctl@ceph.node-exporter.a.smithi089.stdout:Jun 03 19:38:12 smithi089 bash[69490]: ts=2023-06-03T19:38:12.937Z caller=node_exporter.go:115 level=info collector=ipvs 2023-06-03T19:38:13.158 INFO:journalctl@ceph.node-exporter.a.smithi089.stdout:Jun 03 19:38:12 smithi089 bash[69490]: ts=2023-06-03T19:38:12.937Z caller=node_exporter.go:115 level=info collector=loadavg 2023-06-03T19:38:13.158 INFO:journalctl@ceph.node-exporter.a.smithi089.stdout:Jun 03 19:38:12 smithi089 bash[69490]: ts=2023-06-03T19:38:12.937Z caller=node_exporter.go:115 level=info collector=mdadm 2023-06-03T19:38:13.158 INFO:journalctl@ceph.node-exporter.a.smithi089.stdout:Jun 03 19:38:12 smithi089 bash[69490]: ts=2023-06-03T19:38:12.937Z caller=node_exporter.go:115 level=info collector=meminfo 2023-06-03T19:38:13.159 INFO:journalctl@ceph.node-exporter.a.smithi089.stdout:Jun 03 19:38:12 smithi089 bash[69490]: ts=2023-06-03T19:38:12.937Z caller=node_exporter.go:115 level=info collector=netclass 2023-06-03T19:38:13.159 INFO:journalctl@ceph.node-exporter.a.smithi089.stdout:Jun 03 19:38:12 smithi089 bash[69490]: ts=2023-06-03T19:38:12.937Z caller=node_exporter.go:115 level=info collector=netdev 2023-06-03T19:38:13.159 INFO:journalctl@ceph.node-exporter.a.smithi089.stdout:Jun 03 19:38:12 smithi089 bash[69490]: ts=2023-06-03T19:38:12.937Z caller=node_exporter.go:115 level=info collector=netstat 2023-06-03T19:38:13.159 INFO:journalctl@ceph.node-exporter.a.smithi089.stdout:Jun 03 19:38:12 smithi089 bash[69490]: ts=2023-06-03T19:38:12.938Z caller=node_exporter.go:115 level=info collector=nfs 2023-06-03T19:38:13.159 INFO:journalctl@ceph.node-exporter.a.smithi089.stdout:Jun 03 19:38:12 smithi089 bash[69490]: ts=2023-06-03T19:38:12.938Z caller=node_exporter.go:115 level=info collector=nfsd 2023-06-03T19:38:13.159 INFO:journalctl@ceph.node-exporter.a.smithi089.stdout:Jun 03 19:38:12 smithi089 bash[69490]: ts=2023-06-03T19:38:12.938Z caller=node_exporter.go:115 level=info collector=nvme 2023-06-03T19:38:13.159 INFO:journalctl@ceph.node-exporter.a.smithi089.stdout:Jun 03 19:38:12 smithi089 bash[69490]: ts=2023-06-03T19:38:12.938Z caller=node_exporter.go:115 level=info collector=os 2023-06-03T19:38:13.160 INFO:journalctl@ceph.node-exporter.a.smithi089.stdout:Jun 03 19:38:12 smithi089 bash[69490]: ts=2023-06-03T19:38:12.938Z caller=node_exporter.go:115 level=info collector=powersupplyclass 2023-06-03T19:38:13.160 INFO:journalctl@ceph.node-exporter.a.smithi089.stdout:Jun 03 19:38:12 smithi089 bash[69490]: ts=2023-06-03T19:38:12.938Z caller=node_exporter.go:115 level=info collector=pressure 2023-06-03T19:38:13.160 INFO:journalctl@ceph.node-exporter.a.smithi089.stdout:Jun 03 19:38:12 smithi089 bash[69490]: ts=2023-06-03T19:38:12.938Z caller=node_exporter.go:115 level=info collector=rapl 2023-06-03T19:38:13.160 INFO:journalctl@ceph.node-exporter.a.smithi089.stdout:Jun 03 19:38:12 smithi089 bash[69490]: ts=2023-06-03T19:38:12.938Z caller=node_exporter.go:115 level=info collector=schedstat 2023-06-03T19:38:13.160 INFO:journalctl@ceph.node-exporter.a.smithi089.stdout:Jun 03 19:38:12 smithi089 bash[69490]: ts=2023-06-03T19:38:12.938Z caller=node_exporter.go:115 level=info collector=sockstat 2023-06-03T19:38:13.160 INFO:journalctl@ceph.node-exporter.a.smithi089.stdout:Jun 03 19:38:12 smithi089 bash[69490]: ts=2023-06-03T19:38:12.938Z caller=node_exporter.go:115 level=info collector=softnet 2023-06-03T19:38:13.160 INFO:journalctl@ceph.node-exporter.a.smithi089.stdout:Jun 03 19:38:12 smithi089 bash[69490]: ts=2023-06-03T19:38:12.938Z caller=node_exporter.go:115 level=info collector=stat 2023-06-03T19:38:13.161 INFO:journalctl@ceph.node-exporter.a.smithi089.stdout:Jun 03 19:38:12 smithi089 bash[69490]: ts=2023-06-03T19:38:12.938Z caller=node_exporter.go:115 level=info collector=tapestats 2023-06-03T19:38:13.161 INFO:journalctl@ceph.node-exporter.a.smithi089.stdout:Jun 03 19:38:12 smithi089 bash[69490]: ts=2023-06-03T19:38:12.938Z caller=node_exporter.go:115 level=info collector=textfile 2023-06-03T19:38:13.161 INFO:journalctl@ceph.node-exporter.a.smithi089.stdout:Jun 03 19:38:12 smithi089 bash[69490]: ts=2023-06-03T19:38:12.938Z caller=node_exporter.go:115 level=info collector=thermal_zone 2023-06-03T19:38:13.161 INFO:journalctl@ceph.node-exporter.a.smithi089.stdout:Jun 03 19:38:12 smithi089 bash[69490]: ts=2023-06-03T19:38:12.938Z caller=node_exporter.go:115 level=info collector=time 2023-06-03T19:38:13.161 INFO:journalctl@ceph.node-exporter.a.smithi089.stdout:Jun 03 19:38:12 smithi089 bash[69490]: ts=2023-06-03T19:38:12.939Z caller=node_exporter.go:115 level=info collector=udp_queues 2023-06-03T19:38:13.161 INFO:journalctl@ceph.node-exporter.a.smithi089.stdout:Jun 03 19:38:12 smithi089 bash[69490]: ts=2023-06-03T19:38:12.939Z caller=node_exporter.go:115 level=info collector=uname 2023-06-03T19:38:13.161 INFO:journalctl@ceph.node-exporter.a.smithi089.stdout:Jun 03 19:38:12 smithi089 bash[69490]: ts=2023-06-03T19:38:12.939Z caller=node_exporter.go:115 level=info collector=vmstat 2023-06-03T19:38:13.161 INFO:journalctl@ceph.node-exporter.a.smithi089.stdout:Jun 03 19:38:12 smithi089 bash[69490]: ts=2023-06-03T19:38:12.939Z caller=node_exporter.go:115 level=info collector=xfs 2023-06-03T19:38:13.162 INFO:journalctl@ceph.node-exporter.a.smithi089.stdout:Jun 03 19:38:12 smithi089 bash[69490]: ts=2023-06-03T19:38:12.939Z caller=node_exporter.go:115 level=info collector=zfs 2023-06-03T19:38:13.162 INFO:journalctl@ceph.node-exporter.a.smithi089.stdout:Jun 03 19:38:12 smithi089 bash[69490]: ts=2023-06-03T19:38:12.939Z caller=node_exporter.go:199 level=info msg="Listening on" address=:9100 2023-06-03T19:38:13.162 INFO:journalctl@ceph.node-exporter.a.smithi089.stdout:Jun 03 19:38:12 smithi089 bash[69490]: ts=2023-06-03T19:38:12.939Z caller=tls_config.go:195 level=info msg="TLS is disabled." http2=false 2023-06-03T19:38:13.490 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 77 16 101026 101010 81.9809 53.5 0.0151848 0.0121912 2023-06-03T19:38:13.532 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:38:13 smithi089 bash[33173]: cluster 2023-06-03T19:38:11.599712+0000 mgr.x (mgr.24634) 35 : cluster [DBG] pgmap v19: 89 pgs: 89 active+clean; 6.0 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 767 B/s rd, 71 MiB/s wr, 1.14k op/s 2023-06-03T19:38:13.587 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:38:13 smithi164 bash[26855]: cluster 2023-06-03T19:38:11.599712+0000 mgr.x (mgr.24634) 35 : cluster [DBG] pgmap v19: 89 pgs: 89 active+clean; 6.0 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 767 B/s rd, 71 MiB/s wr, 1.14k op/s 2023-06-03T19:38:14.490 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 78 16 101809 101793 81.5572 48.9375 0.0179491 0.0122559 2023-06-03T19:38:14.531 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:38:14 smithi089 bash[26572]: audit 2023-06-03T19:38:13.910364+0000 mon.a (mon.0) 880 : audit [DBG] from='mgr.24634 172.21.15.164:0/803880945' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T19:38:14.532 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:38:14 smithi089 bash[33173]: audit 2023-06-03T19:38:13.910364+0000 mon.a (mon.0) 880 : audit [DBG] from='mgr.24634 172.21.15.164:0/803880945' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T19:38:14.587 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:38:14 smithi164 bash[26855]: audit 2023-06-03T19:38:13.910364+0000 mon.a (mon.0) 880 : audit [DBG] from='mgr.24634 172.21.15.164:0/803880945' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T19:38:15.491 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 79 16 102695 102679 81.2257 55.375 0.0198258 0.0123051 2023-06-03T19:38:15.531 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:38:15 smithi089 bash[33173]: cluster 2023-06-03T19:38:13.600168+0000 mgr.x (mgr.24634) 36 : cluster [DBG] pgmap v20: 89 pgs: 89 active+clean; 6.0 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 767 B/s rd, 48 MiB/s wr, 775 op/s 2023-06-03T19:38:15.532 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:38:15 smithi089 bash[33173]: audit 2023-06-03T19:38:14.185211+0000 mon.c (mon.2) 44 : audit [DBG] from='client.? 172.21.15.89:0/1434859296' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-03T19:38:15.532 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:38:15 smithi089 bash[26572]: cluster 2023-06-03T19:38:13.600168+0000 mgr.x (mgr.24634) 36 : cluster [DBG] pgmap v20: 89 pgs: 89 active+clean; 6.0 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 767 B/s rd, 48 MiB/s wr, 775 op/s 2023-06-03T19:38:15.533 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:38:15 smithi089 bash[26572]: audit 2023-06-03T19:38:14.185211+0000 mon.c (mon.2) 44 : audit [DBG] from='client.? 172.21.15.89:0/1434859296' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-03T19:38:15.587 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:38:15 smithi164 bash[26855]: cluster 2023-06-03T19:38:13.600168+0000 mgr.x (mgr.24634) 36 : cluster [DBG] pgmap v20: 89 pgs: 89 active+clean; 6.0 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 767 B/s rd, 48 MiB/s wr, 775 op/s 2023-06-03T19:38:15.587 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:38:15 smithi164 bash[26855]: audit 2023-06-03T19:38:14.185211+0000 mon.c (mon.2) 44 : audit [DBG] from='client.? 172.21.15.89:0/1434859296' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-03T19:38:16.087 INFO:journalctl@ceph.mgr.x.smithi164.stdout:Jun 03 19:38:15 smithi164 bash[50883]: ::ffff:172.21.15.164 - - [03/Jun/2023:19:38:15] "GET /metrics HTTP/1.1" 200 189944 "" "Prometheus/2.33.4" 2023-06-03T19:38:16.491 INFO:tasks.radosbench.radosbench.0.smithi089.stdout:2023-06-03T19:38:16.483972+0000 min lat: 0.00134497 max lat: 0.133514 avg lat: 0.0123537 2023-06-03T19:38:16.491 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-06-03T19:38:16.491 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 80 16 103586 103570 80.9065 55.6875 0.0168006 0.0123537 2023-06-03T19:38:17.491 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 81 16 104490 104474 80.6051 56.5 0.00792556 0.0124006 2023-06-03T19:38:17.531 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:38:17 smithi089 bash[26572]: cluster 2023-06-03T19:38:15.600950+0000 mgr.x (mgr.24634) 37 : cluster [DBG] pgmap v21: 89 pgs: 89 active+clean; 6.2 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 767 B/s rd, 71 MiB/s wr, 1.13k op/s 2023-06-03T19:38:17.532 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:38:17 smithi089 bash[26572]: audit 2023-06-03T19:38:16.826376+0000 mon.a (mon.0) 881 : audit [DBG] from='client.? 172.21.15.89:0/4014679470' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-03T19:38:17.532 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:38:17 smithi089 bash[33173]: cluster 2023-06-03T19:38:15.600950+0000 mgr.x (mgr.24634) 37 : cluster [DBG] pgmap v21: 89 pgs: 89 active+clean; 6.2 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 767 B/s rd, 71 MiB/s wr, 1.13k op/s 2023-06-03T19:38:17.532 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:38:17 smithi089 bash[33173]: audit 2023-06-03T19:38:16.826376+0000 mon.a (mon.0) 881 : audit [DBG] from='client.? 172.21.15.89:0/4014679470' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-03T19:38:17.588 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:38:17 smithi164 bash[26855]: cluster 2023-06-03T19:38:15.600950+0000 mgr.x (mgr.24634) 37 : cluster [DBG] pgmap v21: 89 pgs: 89 active+clean; 6.2 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 767 B/s rd, 71 MiB/s wr, 1.13k op/s 2023-06-03T19:38:17.589 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:38:17 smithi164 bash[26855]: audit 2023-06-03T19:38:16.826376+0000 mon.a (mon.0) 881 : audit [DBG] from='client.? 172.21.15.89:0/4014679470' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-03T19:38:18.213 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 19:38:17 smithi089 bash[26845]: ::ffff:172.21.15.164 - - [03/Jun/2023:19:38:17] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2023-06-03T19:38:18.491 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 82 16 105363 105347 80.2874 54.5625 0.00854335 0.0124497 2023-06-03T19:38:19.491 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 83 16 106211 106195 79.9586 53 0.0268028 0.0125009 2023-06-03T19:38:19.531 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:38:19 smithi089 bash[33173]: cluster 2023-06-03T19:38:17.601484+0000 mgr.x (mgr.24634) 38 : cluster [DBG] pgmap v22: 89 pgs: 89 active+clean; 6.2 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 45 MiB/s wr, 727 op/s 2023-06-03T19:38:19.532 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:38:19 smithi089 bash[26572]: cluster 2023-06-03T19:38:17.601484+0000 mgr.x (mgr.24634) 38 : cluster [DBG] pgmap v22: 89 pgs: 89 active+clean; 6.2 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 45 MiB/s wr, 727 op/s 2023-06-03T19:38:19.587 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:38:19 smithi164 bash[26855]: cluster 2023-06-03T19:38:17.601484+0000 mgr.x (mgr.24634) 38 : cluster [DBG] pgmap v22: 89 pgs: 89 active+clean; 6.2 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 45 MiB/s wr, 727 op/s 2023-06-03T19:38:20.449 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:38:20 smithi089 bash[33173]: audit 2023-06-03T19:38:19.520245+0000 mon.a (mon.0) 882 : audit [DBG] from='client.? 172.21.15.89:0/2506285418' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-03T19:38:20.450 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:38:20 smithi089 bash[26572]: audit 2023-06-03T19:38:19.520245+0000 mon.a (mon.0) 882 : audit [DBG] from='client.? 172.21.15.89:0/2506285418' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-03T19:38:20.491 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 84 16 107088 107072 79.6592 54.8125 0.0100142 0.0125477 2023-06-03T19:38:20.587 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:38:20 smithi164 bash[26855]: audit 2023-06-03T19:38:19.520245+0000 mon.a (mon.0) 882 : audit [DBG] from='client.? 172.21.15.89:0/2506285418' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-03T19:38:21.473 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:38:21 smithi089 bash[26572]: cluster 2023-06-03T19:38:19.601938+0000 mgr.x (mgr.24634) 39 : cluster [DBG] pgmap v23: 89 pgs: 89 active+clean; 6.2 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 45 MiB/s wr, 727 op/s 2023-06-03T19:38:21.474 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:38:21 smithi089 bash[33173]: cluster 2023-06-03T19:38:19.601938+0000 mgr.x (mgr.24634) 39 : cluster [DBG] pgmap v23: 89 pgs: 89 active+clean; 6.2 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 45 MiB/s wr, 727 op/s 2023-06-03T19:38:21.491 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 85 15 107966 107951 79.3683 54.9375 0.0156052 0.0125939 2023-06-03T19:38:21.587 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:38:21 smithi164 bash[26855]: cluster 2023-06-03T19:38:19.601938+0000 mgr.x (mgr.24634) 39 : cluster [DBG] pgmap v23: 89 pgs: 89 active+clean; 6.2 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 45 MiB/s wr, 727 op/s 2023-06-03T19:38:21.734 INFO:journalctl@ceph.alertmanager.a.smithi089.stdout:Jun 03 19:38:21 smithi089 systemd[1]: Stopping Ceph alertmanager.a for f1db3478-0243-11ee-9b1d-001a4aab830c... 2023-06-03T19:38:21.735 INFO:journalctl@ceph.alertmanager.a.smithi089.stdout:Jun 03 19:38:21 smithi089 bash[69883]: Error response from daemon: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-alertmanager.a 2023-06-03T19:38:21.735 INFO:journalctl@ceph.alertmanager.a.smithi089.stdout:Jun 03 19:38:21 smithi089 bash[58026]: level=info ts=2023-06-03T19:38:21.669Z caller=main.go:557 msg="Received SIGTERM, exiting gracefully..." 2023-06-03T19:38:22.491 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 86 16 108813 108797 79.0602 52.875 0.0269293 0.0126432 2023-06-03T19:38:22.587 INFO:journalctl@ceph.prometheus.a.smithi164.stdout:Jun 03 19:38:22 smithi164 bash[48531]: ts=2023-06-03T19:38:22.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi164.front.sepia.ceph.com:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.89:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-06-03T19:38:22.911 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:38:22 smithi089 bash[33173]: cluster 2023-06-03T19:38:21.602647+0000 mgr.x (mgr.24634) 40 : cluster [DBG] pgmap v24: 89 pgs: 89 active+clean; 6.5 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 69 MiB/s wr, 1.10k op/s 2023-06-03T19:38:22.911 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:38:22 smithi089 bash[33173]: audit 2023-06-03T19:38:22.227700+0000 mon.c (mon.2) 45 : audit [DBG] from='client.? 172.21.15.89:0/2119518019' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-03T19:38:22.911 INFO:journalctl@ceph.alertmanager.a.smithi089.stdout:Jun 03 19:38:22 smithi089 bash[69915]: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-alertmanager-a 2023-06-03T19:38:22.911 INFO:journalctl@ceph.alertmanager.a.smithi089.stdout:Jun 03 19:38:22 smithi089 systemd[1]: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c@alertmanager.a.service: Succeeded. 2023-06-03T19:38:22.912 INFO:journalctl@ceph.alertmanager.a.smithi089.stdout:Jun 03 19:38:22 smithi089 systemd[1]: Stopped Ceph alertmanager.a for f1db3478-0243-11ee-9b1d-001a4aab830c. 2023-06-03T19:38:22.912 INFO:journalctl@ceph.alertmanager.a.smithi089.stdout:Jun 03 19:38:22 smithi089 systemd[1]: Started Ceph alertmanager.a for f1db3478-0243-11ee-9b1d-001a4aab830c. 2023-06-03T19:38:22.912 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:38:22 smithi089 bash[26572]: cluster 2023-06-03T19:38:21.602647+0000 mgr.x (mgr.24634) 40 : cluster [DBG] pgmap v24: 89 pgs: 89 active+clean; 6.5 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 69 MiB/s wr, 1.10k op/s 2023-06-03T19:38:22.912 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:38:22 smithi089 bash[26572]: audit 2023-06-03T19:38:22.227700+0000 mon.c (mon.2) 45 : audit [DBG] from='client.? 172.21.15.89:0/2119518019' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-03T19:38:23.087 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:38:22 smithi164 bash[26855]: cluster 2023-06-03T19:38:21.602647+0000 mgr.x (mgr.24634) 40 : cluster [DBG] pgmap v24: 89 pgs: 89 active+clean; 6.5 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 69 MiB/s wr, 1.10k op/s 2023-06-03T19:38:23.087 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:38:22 smithi164 bash[26855]: audit 2023-06-03T19:38:22.227700+0000 mon.c (mon.2) 45 : audit [DBG] from='client.? 172.21.15.89:0/2119518019' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-03T19:38:23.491 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 87 16 109600 109584 78.7167 49.1875 0.00820917 0.0126969 2023-06-03T19:38:23.837 INFO:journalctl@ceph.prometheus.a.smithi164.stdout:Jun 03 19:38:23 smithi164 bash[48531]: ts=2023-06-03T19:38:23.508Z caller=notifier.go:526 level=error component=notifier alertmanager=http://smithi089.front.sepia.ceph.com:9093/api/v2/alerts count=1 msg="Error sending alert" err="Post \"http://smithi089.front.sepia.ceph.com:9093/api/v2/alerts\": dial tcp 172.21.15.89:9093: connect: connection refused" 2023-06-03T19:38:23.840 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:38:23 smithi089 bash[26572]: audit 2023-06-03T19:38:22.833416+0000 mon.a (mon.0) 883 : audit [INF] from='mgr.24634 172.21.15.164:0/803880945' entity='mgr.x' 2023-06-03T19:38:23.840 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:38:23 smithi089 bash[26572]: audit 2023-06-03T19:38:22.844909+0000 mon.a (mon.0) 884 : audit [INF] from='mgr.24634 172.21.15.164:0/803880945' entity='mgr.x' 2023-06-03T19:38:23.841 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:38:23 smithi089 bash[26572]: cephadm 2023-06-03T19:38:22.868726+0000 mgr.x (mgr.24634) 41 : cephadm [INF] Reconfiguring prometheus.a (dependencies changed)... 2023-06-03T19:38:23.841 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:38:23 smithi089 bash[26572]: cephadm 2023-06-03T19:38:23.128921+0000 mgr.x (mgr.24634) 42 : cephadm [INF] Reconfiguring daemon prometheus.a on smithi164 2023-06-03T19:38:24.281 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:38:23 smithi089 bash[33173]: audit 2023-06-03T19:38:22.833416+0000 mon.a (mon.0) 883 : audit [INF] from='mgr.24634 172.21.15.164:0/803880945' entity='mgr.x' 2023-06-03T19:38:24.282 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:38:23 smithi089 bash[33173]: audit 2023-06-03T19:38:22.844909+0000 mon.a (mon.0) 884 : audit [INF] from='mgr.24634 172.21.15.164:0/803880945' entity='mgr.x' 2023-06-03T19:38:24.282 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:38:23 smithi089 bash[33173]: cephadm 2023-06-03T19:38:22.868726+0000 mgr.x (mgr.24634) 41 : cephadm [INF] Reconfiguring prometheus.a (dependencies changed)... 2023-06-03T19:38:24.282 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:38:23 smithi089 bash[33173]: cephadm 2023-06-03T19:38:23.128921+0000 mgr.x (mgr.24634) 42 : cephadm [INF] Reconfiguring daemon prometheus.a on smithi164 2023-06-03T19:38:24.337 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:38:23 smithi164 bash[26855]: audit 2023-06-03T19:38:22.833416+0000 mon.a (mon.0) 883 : audit [INF] from='mgr.24634 172.21.15.164:0/803880945' entity='mgr.x' 2023-06-03T19:38:24.337 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:38:23 smithi164 bash[26855]: audit 2023-06-03T19:38:22.844909+0000 mon.a (mon.0) 884 : audit [INF] from='mgr.24634 172.21.15.164:0/803880945' entity='mgr.x' 2023-06-03T19:38:24.337 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:38:23 smithi164 bash[26855]: cephadm 2023-06-03T19:38:22.868726+0000 mgr.x (mgr.24634) 41 : cephadm [INF] Reconfiguring prometheus.a (dependencies changed)... 2023-06-03T19:38:24.337 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:38:23 smithi164 bash[26855]: cephadm 2023-06-03T19:38:23.128921+0000 mgr.x (mgr.24634) 42 : cephadm [INF] Reconfiguring daemon prometheus.a on smithi164 2023-06-03T19:38:24.491 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 88 16 110428 110412 78.4102 51.75 0.0186703 0.0127476 2023-06-03T19:38:24.623 INFO:journalctl@ceph.alertmanager.a.smithi089.stdout:Jun 03 19:38:24 smithi089 bash[70028]: level=info ts=2023-06-03T19:38:24.532Z caller=main.go:225 msg="Starting Alertmanager" version="(version=0.23.0, branch=HEAD, revision=61046b17771a57cfd4c4a51be370ab930a4d7d54)" 2023-06-03T19:38:24.623 INFO:journalctl@ceph.alertmanager.a.smithi089.stdout:Jun 03 19:38:24 smithi089 bash[70028]: level=info ts=2023-06-03T19:38:24.532Z caller=main.go:226 build_context="(go=go1.16.7, user=root@e21a959be8d2, date=20210825-10:48:55)" 2023-06-03T19:38:24.623 INFO:journalctl@ceph.alertmanager.a.smithi089.stdout:Jun 03 19:38:24 smithi089 bash[70028]: level=info ts=2023-06-03T19:38:24.546Z caller=cluster.go:184 component=cluster msg="setting advertise address explicitly" addr=172.21.15.89 port=9094 2023-06-03T19:38:24.623 INFO:journalctl@ceph.alertmanager.a.smithi089.stdout:Jun 03 19:38:24 smithi089 bash[70028]: level=info ts=2023-06-03T19:38:24.553Z caller=cluster.go:671 component=cluster msg="Waiting for gossip to settle..." interval=2s 2023-06-03T19:38:24.624 INFO:journalctl@ceph.alertmanager.a.smithi089.stdout:Jun 03 19:38:24 smithi089 bash[70028]: level=info ts=2023-06-03T19:38:24.615Z caller=coordinator.go:113 component=configuration msg="Loading configuration file" file=/etc/alertmanager/alertmanager.yml 2023-06-03T19:38:24.781 INFO:journalctl@ceph.alertmanager.a.smithi089.stdout:Jun 03 19:38:24 smithi089 bash[70028]: level=info ts=2023-06-03T19:38:24.615Z caller=coordinator.go:126 component=configuration msg="Completed loading of configuration file" file=/etc/alertmanager/alertmanager.yml 2023-06-03T19:38:24.782 INFO:journalctl@ceph.alertmanager.a.smithi089.stdout:Jun 03 19:38:24 smithi089 bash[70028]: level=info ts=2023-06-03T19:38:24.619Z caller=main.go:518 msg=Listening address=:9093 2023-06-03T19:38:24.782 INFO:journalctl@ceph.alertmanager.a.smithi089.stdout:Jun 03 19:38:24 smithi089 bash[70028]: level=info ts=2023-06-03T19:38:24.619Z caller=tls_config.go:191 msg="TLS is disabled." http2=false 2023-06-03T19:38:25.031 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:38:24 smithi089 bash[26572]: cluster 2023-06-03T19:38:23.603190+0000 mgr.x (mgr.24634) 43 : cluster [DBG] pgmap v25: 89 pgs: 89 active+clean; 6.5 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 45 MiB/s wr, 723 op/s 2023-06-03T19:38:25.032 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:38:24 smithi089 bash[33173]: cluster 2023-06-03T19:38:23.603190+0000 mgr.x (mgr.24634) 43 : cluster [DBG] pgmap v25: 89 pgs: 89 active+clean; 6.5 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 45 MiB/s wr, 723 op/s 2023-06-03T19:38:25.337 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:38:24 smithi164 bash[26855]: cluster 2023-06-03T19:38:23.603190+0000 mgr.x (mgr.24634) 43 : cluster [DBG] pgmap v25: 89 pgs: 89 active+clean; 6.5 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 45 MiB/s wr, 723 op/s 2023-06-03T19:38:25.491 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 89 16 111135 111119 78.0257 44.1875 0.024463 0.0128099 2023-06-03T19:38:26.087 INFO:journalctl@ceph.mgr.x.smithi164.stdout:Jun 03 19:38:25 smithi164 bash[50883]: ::ffff:172.21.15.164 - - [03/Jun/2023:19:38:25] "GET /metrics HTTP/1.1" 200 189944 "" "Prometheus/2.33.4" 2023-06-03T19:38:26.087 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:38:25 smithi164 bash[26855]: audit 2023-06-03T19:38:24.949527+0000 mon.c (mon.2) 46 : audit [DBG] from='client.? 172.21.15.89:0/2472369482' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-03T19:38:26.281 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:38:25 smithi089 bash[26572]: audit 2023-06-03T19:38:24.949527+0000 mon.c (mon.2) 46 : audit [DBG] from='client.? 172.21.15.89:0/2472369482' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-03T19:38:26.282 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:38:25 smithi089 bash[33173]: audit 2023-06-03T19:38:24.949527+0000 mon.c (mon.2) 46 : audit [DBG] from='client.? 172.21.15.89:0/2472369482' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-03T19:38:26.491 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 90 5 111824 111819 77.6448 43.75 0.00974856 0.0128735 2023-06-03T19:38:26.868 INFO:journalctl@ceph.alertmanager.a.smithi089.stdout:Jun 03 19:38:26 smithi089 bash[70028]: level=info ts=2023-06-03T19:38:26.553Z caller=cluster.go:696 component=cluster msg="gossip not settled" polls=0 before=0 now=1 elapsed=2.000252237s 2023-06-03T19:38:27.281 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:38:26 smithi089 bash[26572]: cluster 2023-06-03T19:38:25.605558+0000 mgr.x (mgr.24634) 44 : cluster [DBG] pgmap v26: 89 pgs: 89 active+clean; 6.8 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 67 MiB/s wr, 1.07k op/s 2023-06-03T19:38:27.282 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:38:26 smithi089 bash[33173]: cluster 2023-06-03T19:38:25.605558+0000 mgr.x (mgr.24634) 44 : cluster [DBG] pgmap v26: 89 pgs: 89 active+clean; 6.8 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 67 MiB/s wr, 1.07k op/s 2023-06-03T19:38:27.337 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:38:26 smithi164 bash[26855]: cluster 2023-06-03T19:38:25.605558+0000 mgr.x (mgr.24634) 44 : cluster [DBG] pgmap v26: 89 pgs: 89 active+clean; 6.8 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 67 MiB/s wr, 1.07k op/s 2023-06-03T19:38:27.492 INFO:tasks.radosbench.radosbench.0.smithi089.stdout:Total time run: 90.0108 2023-06-03T19:38:27.492 INFO:tasks.radosbench.radosbench.0.smithi089.stdout:Total writes made: 111824 2023-06-03T19:38:27.492 INFO:tasks.radosbench.radosbench.0.smithi089.stdout:Write size: 65536 2023-06-03T19:38:27.492 INFO:tasks.radosbench.radosbench.0.smithi089.stdout:Object size: 65536 2023-06-03T19:38:27.492 INFO:tasks.radosbench.radosbench.0.smithi089.stdout:Bandwidth (MB/sec): 77.6463 2023-06-03T19:38:27.493 INFO:tasks.radosbench.radosbench.0.smithi089.stdout:Stddev Bandwidth: 30.1811 2023-06-03T19:38:27.493 INFO:tasks.radosbench.radosbench.0.smithi089.stdout:Max bandwidth (MB/sec): 169.188 2023-06-03T19:38:27.493 INFO:tasks.radosbench.radosbench.0.smithi089.stdout:Min bandwidth (MB/sec): 43.75 2023-06-03T19:38:27.493 INFO:tasks.radosbench.radosbench.0.smithi089.stdout:Average IOPS: 1242 2023-06-03T19:38:27.493 INFO:tasks.radosbench.radosbench.0.smithi089.stdout:Stddev IOPS: 482.898 2023-06-03T19:38:27.493 INFO:tasks.radosbench.radosbench.0.smithi089.stdout:Max IOPS: 2707 2023-06-03T19:38:27.493 INFO:tasks.radosbench.radosbench.0.smithi089.stdout:Min IOPS: 700 2023-06-03T19:38:27.493 INFO:tasks.radosbench.radosbench.0.smithi089.stdout:Average Latency(s): 0.0128743 2023-06-03T19:38:27.494 INFO:tasks.radosbench.radosbench.0.smithi089.stdout:Stddev Latency(s): 0.0085188 2023-06-03T19:38:27.494 INFO:tasks.radosbench.radosbench.0.smithi089.stdout:Max latency(s): 0.133514 2023-06-03T19:38:27.494 INFO:tasks.radosbench.radosbench.0.smithi089.stdout:Min latency(s): 0.00134497 2023-06-03T19:38:27.501 INFO:tasks.radosbench.radosbench.0.smithi089.stdout:Cleaning up (deleting benchmark objects) 2023-06-03T19:38:28.281 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:38:27 smithi089 bash[26572]: audit 2023-06-03T19:38:27.408079+0000 mon.a (mon.0) 885 : audit [DBG] from='client.? 172.21.15.89:0/1559287608' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-03T19:38:28.282 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 19:38:27 smithi089 bash[26845]: ::ffff:172.21.15.164 - - [03/Jun/2023:19:38:27] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2023-06-03T19:38:28.282 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:38:27 smithi089 bash[33173]: audit 2023-06-03T19:38:27.408079+0000 mon.a (mon.0) 885 : audit [DBG] from='client.? 172.21.15.89:0/1559287608' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-03T19:38:28.337 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:38:27 smithi164 bash[26855]: audit 2023-06-03T19:38:27.408079+0000 mon.a (mon.0) 885 : audit [DBG] from='client.? 172.21.15.89:0/1559287608' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-03T19:38:29.281 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:38:28 smithi089 bash[33173]: cluster 2023-06-03T19:38:27.607579+0000 mgr.x (mgr.24634) 45 : cluster [DBG] pgmap v27: 89 pgs: 89 active+clean; 6.8 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 45 MiB/s wr, 717 op/s 2023-06-03T19:38:29.282 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:38:28 smithi089 bash[26572]: cluster 2023-06-03T19:38:27.607579+0000 mgr.x (mgr.24634) 45 : cluster [DBG] pgmap v27: 89 pgs: 89 active+clean; 6.8 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 45 MiB/s wr, 717 op/s 2023-06-03T19:38:29.337 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:38:28 smithi164 bash[26855]: cluster 2023-06-03T19:38:27.607579+0000 mgr.x (mgr.24634) 45 : cluster [DBG] pgmap v27: 89 pgs: 89 active+clean; 6.8 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 45 MiB/s wr, 717 op/s 2023-06-03T19:38:30.282 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:38:29 smithi089 bash[26572]: audit 2023-06-03T19:38:28.906872+0000 mon.a (mon.0) 886 : audit [DBG] from='mgr.24634 172.21.15.164:0/803880945' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T19:38:30.282 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:38:29 smithi089 bash[33173]: audit 2023-06-03T19:38:28.906872+0000 mon.a (mon.0) 886 : audit [DBG] from='mgr.24634 172.21.15.164:0/803880945' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T19:38:30.337 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:38:29 smithi164 bash[26855]: audit 2023-06-03T19:38:28.906872+0000 mon.a (mon.0) 886 : audit [DBG] from='mgr.24634 172.21.15.164:0/803880945' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T19:38:31.282 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:38:30 smithi089 bash[26572]: cluster 2023-06-03T19:38:29.608029+0000 mgr.x (mgr.24634) 46 : cluster [DBG] pgmap v28: 89 pgs: 89 active+clean; 6.8 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 45 MiB/s wr, 717 op/s 2023-06-03T19:38:31.282 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:38:30 smithi089 bash[26572]: audit 2023-06-03T19:38:30.058587+0000 mon.c (mon.2) 47 : audit [DBG] from='client.? 172.21.15.89:0/3253217763' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-03T19:38:31.282 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:38:30 smithi089 bash[33173]: cluster 2023-06-03T19:38:29.608029+0000 mgr.x (mgr.24634) 46 : cluster [DBG] pgmap v28: 89 pgs: 89 active+clean; 6.8 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 45 MiB/s wr, 717 op/s 2023-06-03T19:38:31.282 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:38:30 smithi089 bash[33173]: audit 2023-06-03T19:38:30.058587+0000 mon.c (mon.2) 47 : audit [DBG] from='client.? 172.21.15.89:0/3253217763' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-03T19:38:31.337 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:38:30 smithi164 bash[26855]: cluster 2023-06-03T19:38:29.608029+0000 mgr.x (mgr.24634) 46 : cluster [DBG] pgmap v28: 89 pgs: 89 active+clean; 6.8 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 45 MiB/s wr, 717 op/s 2023-06-03T19:38:31.337 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:38:30 smithi164 bash[26855]: audit 2023-06-03T19:38:30.058587+0000 mon.c (mon.2) 47 : audit [DBG] from='client.? 172.21.15.89:0/3253217763' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-03T19:38:32.587 INFO:journalctl@ceph.prometheus.a.smithi164.stdout:Jun 03 19:38:32 smithi164 bash[48531]: ts=2023-06-03T19:38:32.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi164.front.sepia.ceph.com:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.89:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-06-03T19:38:33.031 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:38:32 smithi089 bash[33173]: cluster 2023-06-03T19:38:31.608795+0000 mgr.x (mgr.24634) 47 : cluster [DBG] pgmap v29: 89 pgs: 89 active+clean; 6.7 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 85 B/s rd, 51 MiB/s wr, 971 op/s 2023-06-03T19:38:33.032 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:38:32 smithi089 bash[26572]: cluster 2023-06-03T19:38:31.608795+0000 mgr.x (mgr.24634) 47 : cluster [DBG] pgmap v29: 89 pgs: 89 active+clean; 6.7 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 85 B/s rd, 51 MiB/s wr, 971 op/s 2023-06-03T19:38:33.087 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:38:32 smithi164 bash[26855]: cluster 2023-06-03T19:38:31.608795+0000 mgr.x (mgr.24634) 47 : cluster [DBG] pgmap v29: 89 pgs: 89 active+clean; 6.7 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 85 B/s rd, 51 MiB/s wr, 971 op/s 2023-06-03T19:38:34.032 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:38:33 smithi089 bash[26572]: audit 2023-06-03T19:38:32.749012+0000 mon.a (mon.0) 887 : audit [DBG] from='client.? 172.21.15.89:0/1682223671' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-03T19:38:34.032 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:38:33 smithi089 bash[33173]: audit 2023-06-03T19:38:32.749012+0000 mon.a (mon.0) 887 : audit [DBG] from='client.? 172.21.15.89:0/1682223671' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-03T19:38:34.087 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:38:33 smithi164 bash[26855]: audit 2023-06-03T19:38:32.749012+0000 mon.a (mon.0) 887 : audit [DBG] from='client.? 172.21.15.89:0/1682223671' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-03T19:38:35.032 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:38:34 smithi089 bash[26572]: cluster 2023-06-03T19:38:33.609174+0000 mgr.x (mgr.24634) 48 : cluster [DBG] pgmap v30: 89 pgs: 89 active+clean; 6.7 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 85 B/s rd, 28 MiB/s wr, 602 op/s 2023-06-03T19:38:35.032 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:38:34 smithi089 bash[33173]: cluster 2023-06-03T19:38:33.609174+0000 mgr.x (mgr.24634) 48 : cluster [DBG] pgmap v30: 89 pgs: 89 active+clean; 6.7 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 85 B/s rd, 28 MiB/s wr, 602 op/s 2023-06-03T19:38:35.033 INFO:journalctl@ceph.alertmanager.a.smithi089.stdout:Jun 03 19:38:34 smithi089 bash[70028]: level=info ts=2023-06-03T19:38:34.555Z caller=cluster.go:688 component=cluster msg="gossip settled; proceeding" elapsed=10.00200678s 2023-06-03T19:38:35.087 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:38:34 smithi164 bash[26855]: cluster 2023-06-03T19:38:33.609174+0000 mgr.x (mgr.24634) 48 : cluster [DBG] pgmap v30: 89 pgs: 89 active+clean; 6.7 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 85 B/s rd, 28 MiB/s wr, 602 op/s 2023-06-03T19:38:35.954 INFO:journalctl@ceph.mgr.x.smithi164.stdout:Jun 03 19:38:35 smithi164 bash[50883]: ::ffff:172.21.15.164 - - [03/Jun/2023:19:38:35] "GET /metrics HTTP/1.1" 200 189959 "" "Prometheus/2.33.4" 2023-06-03T19:38:35.955 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:38:35 smithi164 bash[26855]: audit 2023-06-03T19:38:35.433013+0000 mon.c (mon.2) 48 : audit [DBG] from='client.? 172.21.15.89:0/2366638356' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-03T19:38:36.031 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:38:35 smithi089 bash[26572]: audit 2023-06-03T19:38:35.433013+0000 mon.c (mon.2) 48 : audit [DBG] from='client.? 172.21.15.89:0/2366638356' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-03T19:38:36.032 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:38:35 smithi089 bash[33173]: audit 2023-06-03T19:38:35.433013+0000 mon.c (mon.2) 48 : audit [DBG] from='client.? 172.21.15.89:0/2366638356' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-03T19:38:37.031 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:38:36 smithi089 bash[33173]: cluster 2023-06-03T19:38:35.609920+0000 mgr.x (mgr.24634) 49 : cluster [DBG] pgmap v31: 89 pgs: 89 active+clean; 6.4 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 85 B/s rd, 28 MiB/s wr, 965 op/s 2023-06-03T19:38:37.032 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:38:36 smithi089 bash[26572]: cluster 2023-06-03T19:38:35.609920+0000 mgr.x (mgr.24634) 49 : cluster [DBG] pgmap v31: 89 pgs: 89 active+clean; 6.4 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 85 B/s rd, 28 MiB/s wr, 965 op/s 2023-06-03T19:38:37.087 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:38:36 smithi164 bash[26855]: cluster 2023-06-03T19:38:35.609920+0000 mgr.x (mgr.24634) 49 : cluster [DBG] pgmap v31: 89 pgs: 89 active+clean; 6.4 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 85 B/s rd, 28 MiB/s wr, 965 op/s 2023-06-03T19:38:38.281 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 19:38:37 smithi089 bash[26845]: ::ffff:172.21.15.164 - - [03/Jun/2023:19:38:37] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2023-06-03T19:38:38.689 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:38:38 smithi164 bash[26855]: cluster 2023-06-03T19:38:37.610343+0000 mgr.x (mgr.24634) 50 : cluster [DBG] pgmap v32: 89 pgs: 89 active+clean; 6.4 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 85 B/s rd, 6.4 MiB/s wr, 616 op/s 2023-06-03T19:38:38.689 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:38:38 smithi164 bash[26855]: audit 2023-06-03T19:38:38.145155+0000 mon.a (mon.0) 888 : audit [DBG] from='client.? 172.21.15.89:0/1783444567' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-03T19:38:39.032 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:38:38 smithi089 bash[26572]: cluster 2023-06-03T19:38:37.610343+0000 mgr.x (mgr.24634) 50 : cluster [DBG] pgmap v32: 89 pgs: 89 active+clean; 6.4 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 85 B/s rd, 6.4 MiB/s wr, 616 op/s 2023-06-03T19:38:39.032 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:38:38 smithi089 bash[26572]: audit 2023-06-03T19:38:38.145155+0000 mon.a (mon.0) 888 : audit [DBG] from='client.? 172.21.15.89:0/1783444567' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-03T19:38:39.033 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:38:38 smithi089 bash[33173]: cluster 2023-06-03T19:38:37.610343+0000 mgr.x (mgr.24634) 50 : cluster [DBG] pgmap v32: 89 pgs: 89 active+clean; 6.4 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 85 B/s rd, 6.4 MiB/s wr, 616 op/s 2023-06-03T19:38:39.033 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:38:38 smithi089 bash[33173]: audit 2023-06-03T19:38:38.145155+0000 mon.a (mon.0) 888 : audit [DBG] from='client.? 172.21.15.89:0/1783444567' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-03T19:38:39.691 INFO:journalctl@ceph.prometheus.a.smithi164.stdout:Jun 03 19:38:39 smithi164 systemd[1]: Stopping Ceph prometheus.a for f1db3478-0243-11ee-9b1d-001a4aab830c... 2023-06-03T19:38:39.692 INFO:journalctl@ceph.prometheus.a.smithi164.stdout:Jun 03 19:38:39 smithi164 bash[53173]: Error response from daemon: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-prometheus.a 2023-06-03T19:38:39.692 INFO:journalctl@ceph.prometheus.a.smithi164.stdout:Jun 03 19:38:39 smithi164 bash[48531]: ts=2023-06-03T19:38:39.639Z caller=main.go:775 level=warn msg="Received SIGTERM, exiting gracefully..." 2023-06-03T19:38:39.693 INFO:journalctl@ceph.prometheus.a.smithi164.stdout:Jun 03 19:38:39 smithi164 bash[48531]: ts=2023-06-03T19:38:39.639Z caller=main.go:798 level=info msg="Stopping scrape discovery manager..." 2023-06-03T19:38:39.693 INFO:journalctl@ceph.prometheus.a.smithi164.stdout:Jun 03 19:38:39 smithi164 bash[48531]: ts=2023-06-03T19:38:39.639Z caller=main.go:812 level=info msg="Stopping notify discovery manager..." 2023-06-03T19:38:39.693 INFO:journalctl@ceph.prometheus.a.smithi164.stdout:Jun 03 19:38:39 smithi164 bash[48531]: ts=2023-06-03T19:38:39.639Z caller=main.go:834 level=info msg="Stopping scrape manager..." 2023-06-03T19:38:39.693 INFO:journalctl@ceph.prometheus.a.smithi164.stdout:Jun 03 19:38:39 smithi164 bash[48531]: ts=2023-06-03T19:38:39.639Z caller=main.go:794 level=info msg="Scrape discovery manager stopped" 2023-06-03T19:38:39.694 INFO:journalctl@ceph.prometheus.a.smithi164.stdout:Jun 03 19:38:39 smithi164 bash[48531]: ts=2023-06-03T19:38:39.639Z caller=main.go:808 level=info msg="Notify discovery manager stopped" 2023-06-03T19:38:39.694 INFO:journalctl@ceph.prometheus.a.smithi164.stdout:Jun 03 19:38:39 smithi164 bash[48531]: ts=2023-06-03T19:38:39.639Z caller=manager.go:945 level=info component="rule manager" msg="Stopping rule manager..." 2023-06-03T19:38:39.694 INFO:journalctl@ceph.prometheus.a.smithi164.stdout:Jun 03 19:38:39 smithi164 bash[48531]: ts=2023-06-03T19:38:39.639Z caller=manager.go:955 level=info component="rule manager" msg="Rule manager stopped" 2023-06-03T19:38:39.695 INFO:journalctl@ceph.prometheus.a.smithi164.stdout:Jun 03 19:38:39 smithi164 bash[48531]: ts=2023-06-03T19:38:39.640Z caller=main.go:828 level=info msg="Scrape manager stopped" 2023-06-03T19:38:39.695 INFO:journalctl@ceph.prometheus.a.smithi164.stdout:Jun 03 19:38:39 smithi164 bash[48531]: ts=2023-06-03T19:38:39.640Z caller=notifier.go:600 level=info component=notifier msg="Stopping notification manager..." 2023-06-03T19:38:39.695 INFO:journalctl@ceph.prometheus.a.smithi164.stdout:Jun 03 19:38:39 smithi164 bash[48531]: ts=2023-06-03T19:38:39.640Z caller=main.go:1054 level=info msg="Notifier manager stopped" 2023-06-03T19:38:39.695 INFO:journalctl@ceph.prometheus.a.smithi164.stdout:Jun 03 19:38:39 smithi164 bash[48531]: ts=2023-06-03T19:38:39.640Z caller=main.go:1066 level=info msg="See you next time!" 2023-06-03T19:38:40.837 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:38:40 smithi164 bash[26855]: cluster 2023-06-03T19:38:39.610758+0000 mgr.x (mgr.24634) 51 : cluster [DBG] pgmap v33: 89 pgs: 89 active+clean; 6.4 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 85 B/s rd, 6.4 MiB/s wr, 616 op/s 2023-06-03T19:38:40.837 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:38:40 smithi164 bash[26855]: audit 2023-06-03T19:38:40.624640+0000 mon.a (mon.0) 889 : audit [INF] from='mgr.24634 172.21.15.164:0/803880945' entity='mgr.x' 2023-06-03T19:38:40.837 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:38:40 smithi164 bash[26855]: audit 2023-06-03T19:38:40.641479+0000 mon.a (mon.0) 890 : audit [INF] from='mgr.24634 172.21.15.164:0/803880945' entity='mgr.x' 2023-06-03T19:38:40.838 INFO:journalctl@ceph.prometheus.a.smithi164.stdout:Jun 03 19:38:40 smithi164 bash[53209]: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-prometheus-a 2023-06-03T19:38:40.838 INFO:journalctl@ceph.prometheus.a.smithi164.stdout:Jun 03 19:38:40 smithi164 systemd[1]: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c@prometheus.a.service: Succeeded. 2023-06-03T19:38:40.838 INFO:journalctl@ceph.prometheus.a.smithi164.stdout:Jun 03 19:38:40 smithi164 systemd[1]: Stopped Ceph prometheus.a for f1db3478-0243-11ee-9b1d-001a4aab830c. 2023-06-03T19:38:40.838 INFO:journalctl@ceph.prometheus.a.smithi164.stdout:Jun 03 19:38:40 smithi164 systemd[1]: Started Ceph prometheus.a for f1db3478-0243-11ee-9b1d-001a4aab830c. 2023-06-03T19:38:41.031 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:38:40 smithi089 bash[33173]: cluster 2023-06-03T19:38:39.610758+0000 mgr.x (mgr.24634) 51 : cluster [DBG] pgmap v33: 89 pgs: 89 active+clean; 6.4 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 85 B/s rd, 6.4 MiB/s wr, 616 op/s 2023-06-03T19:38:41.032 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:38:40 smithi089 bash[33173]: audit 2023-06-03T19:38:40.624640+0000 mon.a (mon.0) 889 : audit [INF] from='mgr.24634 172.21.15.164:0/803880945' entity='mgr.x' 2023-06-03T19:38:41.032 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:38:40 smithi089 bash[33173]: audit 2023-06-03T19:38:40.641479+0000 mon.a (mon.0) 890 : audit [INF] from='mgr.24634 172.21.15.164:0/803880945' entity='mgr.x' 2023-06-03T19:38:41.032 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:38:40 smithi089 bash[26572]: cluster 2023-06-03T19:38:39.610758+0000 mgr.x (mgr.24634) 51 : cluster [DBG] pgmap v33: 89 pgs: 89 active+clean; 6.4 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 85 B/s rd, 6.4 MiB/s wr, 616 op/s 2023-06-03T19:38:41.032 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:38:40 smithi089 bash[26572]: audit 2023-06-03T19:38:40.624640+0000 mon.a (mon.0) 889 : audit [INF] from='mgr.24634 172.21.15.164:0/803880945' entity='mgr.x' 2023-06-03T19:38:41.033 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:38:40 smithi089 bash[26572]: audit 2023-06-03T19:38:40.641479+0000 mon.a (mon.0) 890 : audit [INF] from='mgr.24634 172.21.15.164:0/803880945' entity='mgr.x' 2023-06-03T19:38:41.053 INFO:journalctl@ceph.node-exporter.b.smithi164.stdout:Jun 03 19:38:41 smithi164 systemd[1]: Stopping Ceph node-exporter.b for f1db3478-0243-11ee-9b1d-001a4aab830c... 2023-06-03T19:38:41.336 INFO:journalctl@ceph.node-exporter.b.smithi164.stdout:Jun 03 19:38:41 smithi164 bash[53316]: Error response from daemon: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-node-exporter.b 2023-06-03T19:38:42.032 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:38:41 smithi089 bash[33173]: cephadm 2023-06-03T19:38:40.679194+0000 mgr.x (mgr.24634) 52 : cephadm [INF] Reconfiguring node-exporter.b (dependencies changed)... 2023-06-03T19:38:42.032 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:38:41 smithi089 bash[33173]: cephadm 2023-06-03T19:38:40.679930+0000 mgr.x (mgr.24634) 53 : cephadm [INF] Reconfiguring daemon node-exporter.b on smithi164 2023-06-03T19:38:42.032 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:38:41 smithi089 bash[33173]: audit 2023-06-03T19:38:40.798756+0000 mon.a (mon.0) 891 : audit [DBG] from='client.? 172.21.15.89:0/1870313026' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-03T19:38:42.033 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:38:41 smithi089 bash[26572]: cephadm 2023-06-03T19:38:40.679194+0000 mgr.x (mgr.24634) 52 : cephadm [INF] Reconfiguring node-exporter.b (dependencies changed)... 2023-06-03T19:38:42.033 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:38:41 smithi089 bash[26572]: cephadm 2023-06-03T19:38:40.679930+0000 mgr.x (mgr.24634) 53 : cephadm [INF] Reconfiguring daemon node-exporter.b on smithi164 2023-06-03T19:38:42.033 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:38:41 smithi089 bash[26572]: audit 2023-06-03T19:38:40.798756+0000 mon.a (mon.0) 891 : audit [DBG] from='client.? 172.21.15.89:0/1870313026' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-03T19:38:42.066 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:38:41 smithi164 bash[26855]: cephadm 2023-06-03T19:38:40.679194+0000 mgr.x (mgr.24634) 52 : cephadm [INF] Reconfiguring node-exporter.b (dependencies changed)... 2023-06-03T19:38:42.067 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:38:41 smithi164 bash[26855]: cephadm 2023-06-03T19:38:40.679930+0000 mgr.x (mgr.24634) 53 : cephadm [INF] Reconfiguring daemon node-exporter.b on smithi164 2023-06-03T19:38:42.067 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:38:41 smithi164 bash[26855]: audit 2023-06-03T19:38:40.798756+0000 mon.a (mon.0) 891 : audit [DBG] from='client.? 172.21.15.89:0/1870313026' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-03T19:38:42.986 INFO:journalctl@ceph.node-exporter.b.smithi164.stdout:Jun 03 19:38:42 smithi164 bash[53349]: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-node-exporter-b 2023-06-03T19:38:42.987 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:38:42 smithi164 bash[26855]: cluster 2023-06-03T19:38:41.611512+0000 mgr.x (mgr.24634) 54 : cluster [DBG] pgmap v34: 89 pgs: 89 active+clean; 6.2 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 85 B/s rd, 6.4 MiB/s wr, 963 op/s 2023-06-03T19:38:43.032 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:38:42 smithi089 bash[26572]: cluster 2023-06-03T19:38:41.611512+0000 mgr.x (mgr.24634) 54 : cluster [DBG] pgmap v34: 89 pgs: 89 active+clean; 6.2 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 85 B/s rd, 6.4 MiB/s wr, 963 op/s 2023-06-03T19:38:43.033 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:38:42 smithi089 bash[33173]: cluster 2023-06-03T19:38:41.611512+0000 mgr.x (mgr.24634) 54 : cluster [DBG] pgmap v34: 89 pgs: 89 active+clean; 6.2 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 85 B/s rd, 6.4 MiB/s wr, 963 op/s 2023-06-03T19:38:43.237 INFO:journalctl@ceph.prometheus.a.smithi164.stdout:Jun 03 19:38:43 smithi164 bash[53294]: ts=2023-06-03T19:38:43.222Z caller=main.go:512 level=info msg="Starting Prometheus" version="(version=2.33.4, branch=HEAD, revision=83032011a5d3e6102624fe58241a374a7201fee8)" 2023-06-03T19:38:43.238 INFO:journalctl@ceph.prometheus.a.smithi164.stdout:Jun 03 19:38:43 smithi164 bash[53294]: ts=2023-06-03T19:38:43.223Z caller=main.go:517 level=info build_context="(go=go1.17.7, user=root@d13bf69e7be8, date=20220222-16:51:28)" 2023-06-03T19:38:43.238 INFO:journalctl@ceph.prometheus.a.smithi164.stdout:Jun 03 19:38:43 smithi164 bash[53294]: ts=2023-06-03T19:38:43.223Z caller=main.go:518 level=info host_details="(Linux 5.15.0-73-generic #80~20.04.1-Ubuntu SMP Wed May 17 14:58:14 UTC 2023 x86_64 smithi164 (none))" 2023-06-03T19:38:43.238 INFO:journalctl@ceph.prometheus.a.smithi164.stdout:Jun 03 19:38:43 smithi164 bash[53294]: ts=2023-06-03T19:38:43.223Z caller=main.go:519 level=info fd_limits="(soft=1048576, hard=1048576)" 2023-06-03T19:38:43.238 INFO:journalctl@ceph.prometheus.a.smithi164.stdout:Jun 03 19:38:43 smithi164 bash[53294]: ts=2023-06-03T19:38:43.223Z caller=main.go:520 level=info vm_limits="(soft=unlimited, hard=unlimited)" 2023-06-03T19:38:43.238 INFO:journalctl@ceph.prometheus.a.smithi164.stdout:Jun 03 19:38:43 smithi164 bash[53294]: ts=2023-06-03T19:38:43.226Z caller=web.go:570 level=info component=web msg="Start listening for connections" address=:9095 2023-06-03T19:38:43.238 INFO:journalctl@ceph.prometheus.a.smithi164.stdout:Jun 03 19:38:43 smithi164 bash[53294]: ts=2023-06-03T19:38:43.226Z caller=main.go:923 level=info msg="Starting TSDB ..." 2023-06-03T19:38:43.239 INFO:journalctl@ceph.prometheus.a.smithi164.stdout:Jun 03 19:38:43 smithi164 bash[53294]: ts=2023-06-03T19:38:43.228Z caller=tls_config.go:195 level=info component=web msg="TLS is disabled." http2=false 2023-06-03T19:38:43.239 INFO:journalctl@ceph.node-exporter.b.smithi164.stdout:Jun 03 19:38:42 smithi164 systemd[1]: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c@node-exporter.b.service: Main process exited, code=exited, status=143/n/a 2023-06-03T19:38:43.239 INFO:journalctl@ceph.node-exporter.b.smithi164.stdout:Jun 03 19:38:43 smithi164 systemd[1]: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c@node-exporter.b.service: Failed with result 'exit-code'. 2023-06-03T19:38:43.239 INFO:journalctl@ceph.node-exporter.b.smithi164.stdout:Jun 03 19:38:43 smithi164 systemd[1]: Stopped Ceph node-exporter.b for f1db3478-0243-11ee-9b1d-001a4aab830c. 2023-06-03T19:38:43.239 INFO:journalctl@ceph.node-exporter.b.smithi164.stdout:Jun 03 19:38:43 smithi164 systemd[1]: Started Ceph node-exporter.b for f1db3478-0243-11ee-9b1d-001a4aab830c. 2023-06-03T19:38:43.337 INFO:journalctl@ceph.prometheus.a.smithi164.stdout:Jun 03 19:38:43 smithi164 bash[53294]: ts=2023-06-03T19:38:43.231Z caller=head.go:493 level=info component=tsdb msg="Replaying on-disk memory mappable chunks if any" 2023-06-03T19:38:43.337 INFO:journalctl@ceph.prometheus.a.smithi164.stdout:Jun 03 19:38:43 smithi164 bash[53294]: ts=2023-06-03T19:38:43.231Z caller=head.go:527 level=info component=tsdb msg="On-disk memory mappable chunks replay completed" duration=2.803µs 2023-06-03T19:38:43.337 INFO:journalctl@ceph.prometheus.a.smithi164.stdout:Jun 03 19:38:43 smithi164 bash[53294]: ts=2023-06-03T19:38:43.231Z caller=head.go:533 level=info component=tsdb msg="Replaying WAL, this may take a while" 2023-06-03T19:38:43.517 INFO:journalctl@ceph.prometheus.a.smithi164.stdout:Jun 03 19:38:43 smithi164 bash[53294]: ts=2023-06-03T19:38:43.507Z caller=head.go:604 level=info component=tsdb msg="WAL segment loaded" segment=0 maxSegment=2 2023-06-03T19:38:44.236 INFO:journalctl@ceph.prometheus.a.smithi164.stdout:Jun 03 19:38:44 smithi164 bash[53294]: ts=2023-06-03T19:38:44.172Z caller=head.go:604 level=info component=tsdb msg="WAL segment loaded" segment=1 maxSegment=2 2023-06-03T19:38:44.237 INFO:journalctl@ceph.prometheus.a.smithi164.stdout:Jun 03 19:38:44 smithi164 bash[53294]: ts=2023-06-03T19:38:44.172Z caller=head.go:604 level=info component=tsdb msg="WAL segment loaded" segment=2 maxSegment=2 2023-06-03T19:38:44.237 INFO:journalctl@ceph.prometheus.a.smithi164.stdout:Jun 03 19:38:44 smithi164 bash[53294]: ts=2023-06-03T19:38:44.172Z caller=head.go:610 level=info component=tsdb msg="WAL replay completed" checkpoint_replay_duration=59.308µs wal_replay_duration=941.338344ms total_replay_duration=941.42599ms 2023-06-03T19:38:44.237 INFO:journalctl@ceph.prometheus.a.smithi164.stdout:Jun 03 19:38:44 smithi164 bash[53294]: ts=2023-06-03T19:38:44.176Z caller=main.go:944 level=info fs_type=XFS_SUPER_MAGIC 2023-06-03T19:38:44.237 INFO:journalctl@ceph.prometheus.a.smithi164.stdout:Jun 03 19:38:44 smithi164 bash[53294]: ts=2023-06-03T19:38:44.176Z caller=main.go:947 level=info msg="TSDB started" 2023-06-03T19:38:44.237 INFO:journalctl@ceph.prometheus.a.smithi164.stdout:Jun 03 19:38:44 smithi164 bash[53294]: ts=2023-06-03T19:38:44.176Z caller=main.go:1128 level=info msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml 2023-06-03T19:38:44.532 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:38:44 smithi089 bash[26572]: audit 2023-06-03T19:38:43.136414+0000 mon.a (mon.0) 892 : audit [INF] from='mgr.24634 172.21.15.164:0/803880945' entity='mgr.x' 2023-06-03T19:38:44.532 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:38:44 smithi089 bash[26572]: audit 2023-06-03T19:38:43.147378+0000 mon.a (mon.0) 893 : audit [INF] from='mgr.24634 172.21.15.164:0/803880945' entity='mgr.x' 2023-06-03T19:38:44.532 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:38:44 smithi089 bash[26572]: cephadm 2023-06-03T19:38:43.164225+0000 mgr.x (mgr.24634) 55 : cephadm [INF] Reconfiguring grafana.a (dependencies changed)... 2023-06-03T19:38:44.532 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:38:44 smithi089 bash[26572]: cephadm 2023-06-03T19:38:43.186531+0000 mgr.x (mgr.24634) 56 : cephadm [INF] Reconfiguring daemon grafana.a on smithi164 2023-06-03T19:38:44.532 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:38:44 smithi089 bash[26572]: audit 2023-06-03T19:38:43.453589+0000 mon.c (mon.2) 49 : audit [DBG] from='client.? 172.21.15.89:0/1971474219' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-03T19:38:44.533 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:38:44 smithi089 bash[26572]: audit 2023-06-03T19:38:43.906827+0000 mon.a (mon.0) 894 : audit [DBG] from='mgr.24634 172.21.15.164:0/803880945' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T19:38:44.533 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:38:44 smithi089 bash[33173]: audit 2023-06-03T19:38:43.136414+0000 mon.a (mon.0) 892 : audit [INF] from='mgr.24634 172.21.15.164:0/803880945' entity='mgr.x' 2023-06-03T19:38:44.533 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:38:44 smithi089 bash[33173]: audit 2023-06-03T19:38:43.147378+0000 mon.a (mon.0) 893 : audit [INF] from='mgr.24634 172.21.15.164:0/803880945' entity='mgr.x' 2023-06-03T19:38:44.533 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:38:44 smithi089 bash[33173]: cephadm 2023-06-03T19:38:43.164225+0000 mgr.x (mgr.24634) 55 : cephadm [INF] Reconfiguring grafana.a (dependencies changed)... 2023-06-03T19:38:44.533 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:38:44 smithi089 bash[33173]: cephadm 2023-06-03T19:38:43.186531+0000 mgr.x (mgr.24634) 56 : cephadm [INF] Reconfiguring daemon grafana.a on smithi164 2023-06-03T19:38:44.533 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:38:44 smithi089 bash[33173]: audit 2023-06-03T19:38:43.453589+0000 mon.c (mon.2) 49 : audit [DBG] from='client.? 172.21.15.89:0/1971474219' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-03T19:38:44.534 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:38:44 smithi089 bash[33173]: audit 2023-06-03T19:38:43.906827+0000 mon.a (mon.0) 894 : audit [DBG] from='mgr.24634 172.21.15.164:0/803880945' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T19:38:44.587 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:38:44 smithi164 bash[26855]: audit 2023-06-03T19:38:43.136414+0000 mon.a (mon.0) 892 : audit [INF] from='mgr.24634 172.21.15.164:0/803880945' entity='mgr.x' 2023-06-03T19:38:44.587 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:38:44 smithi164 bash[26855]: audit 2023-06-03T19:38:43.147378+0000 mon.a (mon.0) 893 : audit [INF] from='mgr.24634 172.21.15.164:0/803880945' entity='mgr.x' 2023-06-03T19:38:44.587 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:38:44 smithi164 bash[26855]: cephadm 2023-06-03T19:38:43.164225+0000 mgr.x (mgr.24634) 55 : cephadm [INF] Reconfiguring grafana.a (dependencies changed)... 2023-06-03T19:38:44.587 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:38:44 smithi164 bash[26855]: cephadm 2023-06-03T19:38:43.186531+0000 mgr.x (mgr.24634) 56 : cephadm [INF] Reconfiguring daemon grafana.a on smithi164 2023-06-03T19:38:44.587 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:38:44 smithi164 bash[26855]: audit 2023-06-03T19:38:43.453589+0000 mon.c (mon.2) 49 : audit [DBG] from='client.? 172.21.15.89:0/1971474219' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-03T19:38:44.587 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:38:44 smithi164 bash[26855]: audit 2023-06-03T19:38:43.906827+0000 mon.a (mon.0) 894 : audit [DBG] from='mgr.24634 172.21.15.164:0/803880945' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T19:38:44.588 INFO:journalctl@ceph.prometheus.a.smithi164.stdout:Jun 03 19:38:44 smithi164 bash[53294]: ts=2023-06-03T19:38:44.260Z caller=main.go:1165 level=info msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=84.225201ms db_storage=737ns remote_storage=1.268µs web_handler=313ns query_engine=874ns scrape=48.52001ms scrape_sd=106.751µs notify=15.101µs notify_sd=9.728µs rules=35.28437ms 2023-06-03T19:38:44.588 INFO:journalctl@ceph.prometheus.a.smithi164.stdout:Jun 03 19:38:44 smithi164 bash[53294]: ts=2023-06-03T19:38:44.260Z caller=main.go:896 level=info msg="Server is ready to receive web requests." 2023-06-03T19:38:45.337 INFO:journalctl@ceph.node-exporter.b.smithi164.stdout:Jun 03 19:38:45 smithi164 bash[53477]: ts=2023-06-03T19:38:45.113Z caller=node_exporter.go:182 level=info msg="Starting node_exporter" version="(version=1.3.1, branch=HEAD, revision=a2321e7b940ddcff26873612bccdf7cd4c42b6b6)" 2023-06-03T19:38:45.338 INFO:journalctl@ceph.node-exporter.b.smithi164.stdout:Jun 03 19:38:45 smithi164 bash[53477]: ts=2023-06-03T19:38:45.113Z caller=node_exporter.go:183 level=info msg="Build context" build_context="(go=go1.17.3, user=root@243aafa5525c, date=20211205-11:09:49)" 2023-06-03T19:38:45.338 INFO:journalctl@ceph.node-exporter.b.smithi164.stdout:Jun 03 19:38:45 smithi164 bash[53477]: ts=2023-06-03T19:38:45.114Z caller=filesystem_common.go:111 level=info collector=filesystem msg="Parsed flag --collector.filesystem.mount-points-exclude" flag=^/(dev|proc|run/credentials/.+|sys|var/lib/docker/.+)($|/) 2023-06-03T19:38:45.338 INFO:journalctl@ceph.node-exporter.b.smithi164.stdout:Jun 03 19:38:45 smithi164 bash[53477]: ts=2023-06-03T19:38:45.114Z caller=filesystem_common.go:113 level=info collector=filesystem msg="Parsed flag --collector.filesystem.fs-types-exclude" flag=^(autofs|binfmt_misc|bpf|cgroup2?|configfs|debugfs|devpts|devtmpfs|fusectl|hugetlbfs|iso9660|mqueue|nsfs|overlay|proc|procfs|pstore|rpc_pipefs|securityfs|selinuxfs|squashfs|sysfs|tracefs)$ 2023-06-03T19:38:45.338 INFO:journalctl@ceph.node-exporter.b.smithi164.stdout:Jun 03 19:38:45 smithi164 bash[53477]: ts=2023-06-03T19:38:45.114Z caller=node_exporter.go:108 level=info msg="Enabled collectors" 2023-06-03T19:38:45.338 INFO:journalctl@ceph.node-exporter.b.smithi164.stdout:Jun 03 19:38:45 smithi164 bash[53477]: ts=2023-06-03T19:38:45.114Z caller=node_exporter.go:115 level=info collector=arp 2023-06-03T19:38:45.338 INFO:journalctl@ceph.node-exporter.b.smithi164.stdout:Jun 03 19:38:45 smithi164 bash[53477]: ts=2023-06-03T19:38:45.114Z caller=node_exporter.go:115 level=info collector=bcache 2023-06-03T19:38:45.339 INFO:journalctl@ceph.node-exporter.b.smithi164.stdout:Jun 03 19:38:45 smithi164 bash[53477]: ts=2023-06-03T19:38:45.114Z caller=node_exporter.go:115 level=info collector=bonding 2023-06-03T19:38:45.339 INFO:journalctl@ceph.node-exporter.b.smithi164.stdout:Jun 03 19:38:45 smithi164 bash[53477]: ts=2023-06-03T19:38:45.114Z caller=node_exporter.go:115 level=info collector=btrfs 2023-06-03T19:38:45.339 INFO:journalctl@ceph.node-exporter.b.smithi164.stdout:Jun 03 19:38:45 smithi164 bash[53477]: ts=2023-06-03T19:38:45.114Z caller=node_exporter.go:115 level=info collector=conntrack 2023-06-03T19:38:45.339 INFO:journalctl@ceph.node-exporter.b.smithi164.stdout:Jun 03 19:38:45 smithi164 bash[53477]: ts=2023-06-03T19:38:45.114Z caller=node_exporter.go:115 level=info collector=cpu 2023-06-03T19:38:45.339 INFO:journalctl@ceph.node-exporter.b.smithi164.stdout:Jun 03 19:38:45 smithi164 bash[53477]: ts=2023-06-03T19:38:45.114Z caller=node_exporter.go:115 level=info collector=cpufreq 2023-06-03T19:38:45.339 INFO:journalctl@ceph.node-exporter.b.smithi164.stdout:Jun 03 19:38:45 smithi164 bash[53477]: ts=2023-06-03T19:38:45.114Z caller=node_exporter.go:115 level=info collector=diskstats 2023-06-03T19:38:45.339 INFO:journalctl@ceph.node-exporter.b.smithi164.stdout:Jun 03 19:38:45 smithi164 bash[53477]: ts=2023-06-03T19:38:45.114Z caller=node_exporter.go:115 level=info collector=dmi 2023-06-03T19:38:45.340 INFO:journalctl@ceph.node-exporter.b.smithi164.stdout:Jun 03 19:38:45 smithi164 bash[53477]: ts=2023-06-03T19:38:45.114Z caller=node_exporter.go:115 level=info collector=edac 2023-06-03T19:38:45.340 INFO:journalctl@ceph.node-exporter.b.smithi164.stdout:Jun 03 19:38:45 smithi164 bash[53477]: ts=2023-06-03T19:38:45.114Z caller=node_exporter.go:115 level=info collector=entropy 2023-06-03T19:38:45.340 INFO:journalctl@ceph.node-exporter.b.smithi164.stdout:Jun 03 19:38:45 smithi164 bash[53477]: ts=2023-06-03T19:38:45.114Z caller=node_exporter.go:115 level=info collector=fibrechannel 2023-06-03T19:38:45.340 INFO:journalctl@ceph.node-exporter.b.smithi164.stdout:Jun 03 19:38:45 smithi164 bash[53477]: ts=2023-06-03T19:38:45.114Z caller=node_exporter.go:115 level=info collector=filefd 2023-06-03T19:38:45.340 INFO:journalctl@ceph.node-exporter.b.smithi164.stdout:Jun 03 19:38:45 smithi164 bash[53477]: ts=2023-06-03T19:38:45.114Z caller=node_exporter.go:115 level=info collector=filesystem 2023-06-03T19:38:45.340 INFO:journalctl@ceph.node-exporter.b.smithi164.stdout:Jun 03 19:38:45 smithi164 bash[53477]: ts=2023-06-03T19:38:45.114Z caller=node_exporter.go:115 level=info collector=hwmon 2023-06-03T19:38:45.341 INFO:journalctl@ceph.node-exporter.b.smithi164.stdout:Jun 03 19:38:45 smithi164 bash[53477]: ts=2023-06-03T19:38:45.114Z caller=node_exporter.go:115 level=info collector=infiniband 2023-06-03T19:38:45.341 INFO:journalctl@ceph.node-exporter.b.smithi164.stdout:Jun 03 19:38:45 smithi164 bash[53477]: ts=2023-06-03T19:38:45.115Z caller=node_exporter.go:115 level=info collector=ipvs 2023-06-03T19:38:45.341 INFO:journalctl@ceph.node-exporter.b.smithi164.stdout:Jun 03 19:38:45 smithi164 bash[53477]: ts=2023-06-03T19:38:45.115Z caller=node_exporter.go:115 level=info collector=loadavg 2023-06-03T19:38:45.341 INFO:journalctl@ceph.node-exporter.b.smithi164.stdout:Jun 03 19:38:45 smithi164 bash[53477]: ts=2023-06-03T19:38:45.115Z caller=node_exporter.go:115 level=info collector=mdadm 2023-06-03T19:38:45.341 INFO:journalctl@ceph.node-exporter.b.smithi164.stdout:Jun 03 19:38:45 smithi164 bash[53477]: ts=2023-06-03T19:38:45.115Z caller=node_exporter.go:115 level=info collector=meminfo 2023-06-03T19:38:45.341 INFO:journalctl@ceph.node-exporter.b.smithi164.stdout:Jun 03 19:38:45 smithi164 bash[53477]: ts=2023-06-03T19:38:45.115Z caller=node_exporter.go:115 level=info collector=netclass 2023-06-03T19:38:45.342 INFO:journalctl@ceph.node-exporter.b.smithi164.stdout:Jun 03 19:38:45 smithi164 bash[53477]: ts=2023-06-03T19:38:45.115Z caller=node_exporter.go:115 level=info collector=netdev 2023-06-03T19:38:45.342 INFO:journalctl@ceph.node-exporter.b.smithi164.stdout:Jun 03 19:38:45 smithi164 bash[53477]: ts=2023-06-03T19:38:45.115Z caller=node_exporter.go:115 level=info collector=netstat 2023-06-03T19:38:45.342 INFO:journalctl@ceph.node-exporter.b.smithi164.stdout:Jun 03 19:38:45 smithi164 bash[53477]: ts=2023-06-03T19:38:45.115Z caller=node_exporter.go:115 level=info collector=nfs 2023-06-03T19:38:45.342 INFO:journalctl@ceph.node-exporter.b.smithi164.stdout:Jun 03 19:38:45 smithi164 bash[53477]: ts=2023-06-03T19:38:45.115Z caller=node_exporter.go:115 level=info collector=nfsd 2023-06-03T19:38:45.342 INFO:journalctl@ceph.node-exporter.b.smithi164.stdout:Jun 03 19:38:45 smithi164 bash[53477]: ts=2023-06-03T19:38:45.115Z caller=node_exporter.go:115 level=info collector=nvme 2023-06-03T19:38:45.342 INFO:journalctl@ceph.node-exporter.b.smithi164.stdout:Jun 03 19:38:45 smithi164 bash[53477]: ts=2023-06-03T19:38:45.115Z caller=node_exporter.go:115 level=info collector=os 2023-06-03T19:38:45.342 INFO:journalctl@ceph.node-exporter.b.smithi164.stdout:Jun 03 19:38:45 smithi164 bash[53477]: ts=2023-06-03T19:38:45.115Z caller=node_exporter.go:115 level=info collector=powersupplyclass 2023-06-03T19:38:45.343 INFO:journalctl@ceph.node-exporter.b.smithi164.stdout:Jun 03 19:38:45 smithi164 bash[53477]: ts=2023-06-03T19:38:45.115Z caller=node_exporter.go:115 level=info collector=pressure 2023-06-03T19:38:45.343 INFO:journalctl@ceph.node-exporter.b.smithi164.stdout:Jun 03 19:38:45 smithi164 bash[53477]: ts=2023-06-03T19:38:45.115Z caller=node_exporter.go:115 level=info collector=rapl 2023-06-03T19:38:45.343 INFO:journalctl@ceph.node-exporter.b.smithi164.stdout:Jun 03 19:38:45 smithi164 bash[53477]: ts=2023-06-03T19:38:45.115Z caller=node_exporter.go:115 level=info collector=schedstat 2023-06-03T19:38:45.343 INFO:journalctl@ceph.node-exporter.b.smithi164.stdout:Jun 03 19:38:45 smithi164 bash[53477]: ts=2023-06-03T19:38:45.115Z caller=node_exporter.go:115 level=info collector=sockstat 2023-06-03T19:38:45.343 INFO:journalctl@ceph.node-exporter.b.smithi164.stdout:Jun 03 19:38:45 smithi164 bash[53477]: ts=2023-06-03T19:38:45.115Z caller=node_exporter.go:115 level=info collector=softnet 2023-06-03T19:38:45.343 INFO:journalctl@ceph.node-exporter.b.smithi164.stdout:Jun 03 19:38:45 smithi164 bash[53477]: ts=2023-06-03T19:38:45.115Z caller=node_exporter.go:115 level=info collector=stat 2023-06-03T19:38:45.344 INFO:journalctl@ceph.node-exporter.b.smithi164.stdout:Jun 03 19:38:45 smithi164 bash[53477]: ts=2023-06-03T19:38:45.115Z caller=node_exporter.go:115 level=info collector=tapestats 2023-06-03T19:38:45.344 INFO:journalctl@ceph.node-exporter.b.smithi164.stdout:Jun 03 19:38:45 smithi164 bash[53477]: ts=2023-06-03T19:38:45.115Z caller=node_exporter.go:115 level=info collector=textfile 2023-06-03T19:38:45.344 INFO:journalctl@ceph.node-exporter.b.smithi164.stdout:Jun 03 19:38:45 smithi164 bash[53477]: ts=2023-06-03T19:38:45.115Z caller=node_exporter.go:115 level=info collector=thermal_zone 2023-06-03T19:38:45.344 INFO:journalctl@ceph.node-exporter.b.smithi164.stdout:Jun 03 19:38:45 smithi164 bash[53477]: ts=2023-06-03T19:38:45.115Z caller=node_exporter.go:115 level=info collector=time 2023-06-03T19:38:45.344 INFO:journalctl@ceph.node-exporter.b.smithi164.stdout:Jun 03 19:38:45 smithi164 bash[53477]: ts=2023-06-03T19:38:45.115Z caller=node_exporter.go:115 level=info collector=udp_queues 2023-06-03T19:38:45.344 INFO:journalctl@ceph.node-exporter.b.smithi164.stdout:Jun 03 19:38:45 smithi164 bash[53477]: ts=2023-06-03T19:38:45.115Z caller=node_exporter.go:115 level=info collector=uname 2023-06-03T19:38:45.345 INFO:journalctl@ceph.node-exporter.b.smithi164.stdout:Jun 03 19:38:45 smithi164 bash[53477]: ts=2023-06-03T19:38:45.115Z caller=node_exporter.go:115 level=info collector=vmstat 2023-06-03T19:38:45.345 INFO:journalctl@ceph.node-exporter.b.smithi164.stdout:Jun 03 19:38:45 smithi164 bash[53477]: ts=2023-06-03T19:38:45.115Z caller=node_exporter.go:115 level=info collector=xfs 2023-06-03T19:38:45.345 INFO:journalctl@ceph.node-exporter.b.smithi164.stdout:Jun 03 19:38:45 smithi164 bash[53477]: ts=2023-06-03T19:38:45.115Z caller=node_exporter.go:115 level=info collector=zfs 2023-06-03T19:38:45.345 INFO:journalctl@ceph.node-exporter.b.smithi164.stdout:Jun 03 19:38:45 smithi164 bash[53477]: ts=2023-06-03T19:38:45.115Z caller=node_exporter.go:199 level=info msg="Listening on" address=:9100 2023-06-03T19:38:45.345 INFO:journalctl@ceph.node-exporter.b.smithi164.stdout:Jun 03 19:38:45 smithi164 bash[53477]: ts=2023-06-03T19:38:45.115Z caller=tls_config.go:195 level=info msg="TLS is disabled." http2=false 2023-06-03T19:38:45.345 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:38:45 smithi164 bash[26855]: cluster 2023-06-03T19:38:43.611952+0000 mgr.x (mgr.24634) 57 : cluster [DBG] pgmap v35: 89 pgs: 89 active+clean; 6.2 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 709 op/s 2023-06-03T19:38:45.531 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:38:45 smithi089 bash[33173]: cluster 2023-06-03T19:38:43.611952+0000 mgr.x (mgr.24634) 57 : cluster [DBG] pgmap v35: 89 pgs: 89 active+clean; 6.2 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 709 op/s 2023-06-03T19:38:45.532 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:38:45 smithi089 bash[26572]: cluster 2023-06-03T19:38:43.611952+0000 mgr.x (mgr.24634) 57 : cluster [DBG] pgmap v35: 89 pgs: 89 active+clean; 6.2 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 709 op/s 2023-06-03T19:38:46.532 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:38:46 smithi089 bash[26572]: audit 2023-06-03T19:38:46.111980+0000 mon.b (mon.1) 32 : audit [DBG] from='client.? 172.21.15.89:0/4026082490' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-03T19:38:46.532 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:38:46 smithi089 bash[33173]: audit 2023-06-03T19:38:46.111980+0000 mon.b (mon.1) 32 : audit [DBG] from='client.? 172.21.15.89:0/4026082490' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-03T19:38:46.586 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:38:46 smithi164 bash[26855]: audit 2023-06-03T19:38:46.111980+0000 mon.b (mon.1) 32 : audit [DBG] from='client.? 172.21.15.89:0/4026082490' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-03T19:38:47.531 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:38:47 smithi089 bash[33173]: cluster 2023-06-03T19:38:45.612768+0000 mgr.x (mgr.24634) 58 : cluster [DBG] pgmap v36: 89 pgs: 89 active+clean; 6.0 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 1.03k op/s 2023-06-03T19:38:47.532 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:38:47 smithi089 bash[26572]: cluster 2023-06-03T19:38:45.612768+0000 mgr.x (mgr.24634) 58 : cluster [DBG] pgmap v36: 89 pgs: 89 active+clean; 6.0 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 1.03k op/s 2023-06-03T19:38:47.587 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:38:47 smithi164 bash[26855]: cluster 2023-06-03T19:38:45.612768+0000 mgr.x (mgr.24634) 58 : cluster [DBG] pgmap v36: 89 pgs: 89 active+clean; 6.0 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 1.03k op/s 2023-06-03T19:38:49.531 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:38:49 smithi089 bash[26572]: cluster 2023-06-03T19:38:47.613224+0000 mgr.x (mgr.24634) 59 : cluster [DBG] pgmap v37: 89 pgs: 89 active+clean; 6.0 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 670 op/s 2023-06-03T19:38:49.532 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:38:49 smithi089 bash[26572]: audit 2023-06-03T19:38:48.773301+0000 mon.b (mon.1) 33 : audit [DBG] from='client.? 172.21.15.89:0/947500769' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-03T19:38:49.532 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:38:49 smithi089 bash[33173]: cluster 2023-06-03T19:38:47.613224+0000 mgr.x (mgr.24634) 59 : cluster [DBG] pgmap v37: 89 pgs: 89 active+clean; 6.0 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 670 op/s 2023-06-03T19:38:49.532 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:38:49 smithi089 bash[33173]: audit 2023-06-03T19:38:48.773301+0000 mon.b (mon.1) 33 : audit [DBG] from='client.? 172.21.15.89:0/947500769' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-03T19:38:49.587 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:38:49 smithi164 bash[26855]: cluster 2023-06-03T19:38:47.613224+0000 mgr.x (mgr.24634) 59 : cluster [DBG] pgmap v37: 89 pgs: 89 active+clean; 6.0 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 670 op/s 2023-06-03T19:38:49.587 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:38:49 smithi164 bash[26855]: audit 2023-06-03T19:38:48.773301+0000 mon.b (mon.1) 33 : audit [DBG] from='client.? 172.21.15.89:0/947500769' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-03T19:38:51.531 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:38:51 smithi089 bash[26572]: cluster 2023-06-03T19:38:49.613725+0000 mgr.x (mgr.24634) 60 : cluster [DBG] pgmap v38: 89 pgs: 89 active+clean; 6.0 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 670 op/s 2023-06-03T19:38:51.532 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:38:51 smithi089 bash[33173]: cluster 2023-06-03T19:38:49.613725+0000 mgr.x (mgr.24634) 60 : cluster [DBG] pgmap v38: 89 pgs: 89 active+clean; 6.0 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 670 op/s 2023-06-03T19:38:51.587 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:38:51 smithi164 bash[26855]: cluster 2023-06-03T19:38:49.613725+0000 mgr.x (mgr.24634) 60 : cluster [DBG] pgmap v38: 89 pgs: 89 active+clean; 6.0 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 670 op/s 2023-06-03T19:38:52.531 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:38:52 smithi089 bash[33173]: audit 2023-06-03T19:38:51.443492+0000 mon.a (mon.0) 895 : audit [DBG] from='client.? 172.21.15.89:0/2743432870' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-03T19:38:52.532 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:38:52 smithi089 bash[26572]: audit 2023-06-03T19:38:51.443492+0000 mon.a (mon.0) 895 : audit [DBG] from='client.? 172.21.15.89:0/2743432870' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-03T19:38:52.587 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:38:52 smithi164 bash[26855]: audit 2023-06-03T19:38:51.443492+0000 mon.a (mon.0) 895 : audit [DBG] from='client.? 172.21.15.89:0/2743432870' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-03T19:38:53.531 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:38:53 smithi089 bash[26572]: cluster 2023-06-03T19:38:51.614536+0000 mgr.x (mgr.24634) 61 : cluster [DBG] pgmap v39: 89 pgs: 89 active+clean; 5.7 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 1k op/s 2023-06-03T19:38:53.532 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:38:53 smithi089 bash[33173]: cluster 2023-06-03T19:38:51.614536+0000 mgr.x (mgr.24634) 61 : cluster [DBG] pgmap v39: 89 pgs: 89 active+clean; 5.7 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 1k op/s 2023-06-03T19:38:53.587 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:38:53 smithi164 bash[26855]: cluster 2023-06-03T19:38:51.614536+0000 mgr.x (mgr.24634) 61 : cluster [DBG] pgmap v39: 89 pgs: 89 active+clean; 5.7 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 1k op/s 2023-06-03T19:38:53.588 INFO:journalctl@ceph.mgr.x.smithi164.stdout:Jun 03 19:38:53 smithi164 bash[50883]: ::ffff:172.21.15.164 - - [03/Jun/2023:19:38:53] "GET /metrics HTTP/1.1" 200 189982 "" "Prometheus/2.33.4" 2023-06-03T19:38:54.535 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:38:54 smithi089 bash[33173]: audit 2023-06-03T19:38:54.101780+0000 mon.a (mon.0) 896 : audit [DBG] from='client.? 172.21.15.89:0/1032572078' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-03T19:38:54.536 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:38:54 smithi089 bash[26572]: audit 2023-06-03T19:38:54.101780+0000 mon.a (mon.0) 896 : audit [DBG] from='client.? 172.21.15.89:0/1032572078' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-03T19:38:54.587 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:38:54 smithi164 bash[26855]: audit 2023-06-03T19:38:54.101780+0000 mon.a (mon.0) 896 : audit [DBG] from='client.? 172.21.15.89:0/1032572078' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-03T19:38:55.531 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:38:55 smithi089 bash[26572]: cluster 2023-06-03T19:38:53.614973+0000 mgr.x (mgr.24634) 62 : cluster [DBG] pgmap v40: 89 pgs: 89 active+clean; 5.7 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 653 op/s 2023-06-03T19:38:55.532 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:38:55 smithi089 bash[33173]: cluster 2023-06-03T19:38:53.614973+0000 mgr.x (mgr.24634) 62 : cluster [DBG] pgmap v40: 89 pgs: 89 active+clean; 5.7 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 653 op/s 2023-06-03T19:38:55.586 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:38:55 smithi164 bash[26855]: cluster 2023-06-03T19:38:53.614973+0000 mgr.x (mgr.24634) 62 : cluster [DBG] pgmap v40: 89 pgs: 89 active+clean; 5.7 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 653 op/s 2023-06-03T19:38:57.531 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:38:57 smithi089 bash[26572]: cluster 2023-06-03T19:38:55.615606+0000 mgr.x (mgr.24634) 63 : cluster [DBG] pgmap v41: 89 pgs: 89 active+clean; 5.5 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 971 op/s 2023-06-03T19:38:57.531 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:38:57 smithi089 bash[26572]: audit 2023-06-03T19:38:56.797705+0000 mon.a (mon.0) 897 : audit [DBG] from='client.? 172.21.15.89:0/4052756474' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-03T19:38:57.532 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:38:57 smithi089 bash[33173]: cluster 2023-06-03T19:38:55.615606+0000 mgr.x (mgr.24634) 63 : cluster [DBG] pgmap v41: 89 pgs: 89 active+clean; 5.5 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 971 op/s 2023-06-03T19:38:57.532 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:38:57 smithi089 bash[33173]: audit 2023-06-03T19:38:56.797705+0000 mon.a (mon.0) 897 : audit [DBG] from='client.? 172.21.15.89:0/4052756474' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-03T19:38:57.586 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:38:57 smithi164 bash[26855]: cluster 2023-06-03T19:38:55.615606+0000 mgr.x (mgr.24634) 63 : cluster [DBG] pgmap v41: 89 pgs: 89 active+clean; 5.5 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 971 op/s 2023-06-03T19:38:57.587 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:38:57 smithi164 bash[26855]: audit 2023-06-03T19:38:56.797705+0000 mon.a (mon.0) 897 : audit [DBG] from='client.? 172.21.15.89:0/4052756474' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-03T19:38:59.087 INFO:journalctl@ceph.prometheus.a.smithi164.stdout:Jun 03 19:38:58 smithi164 bash[53294]: ts=2023-06-03T19:38:58.659Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.13-257-gd8c5d349 (d8c5d34975dce1c5eb0aa3a7979a4d9b9a99d1ec) pacific (stable)\", cluster_addr=\"172.21.15.89\", device_class=\"ssd\", hostname=\"smithi089\", instance=\"smithi164:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.89\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.13-257-gd8c5d349 (d8c5d34975dce1c5eb0aa3a7979a4d9b9a99d1ec) pacific (stable)\", cluster_addr=\"172.21.15.89\", device_class=\"ssd\", hostname=\"smithi089\", instance=\"smithi164.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.89\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-06-03T19:38:59.087 INFO:journalctl@ceph.prometheus.a.smithi164.stdout:Jun 03 19:38:58 smithi164 bash[53294]: ts=2023-06-03T19:38:58.661Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephPGImbalance\nexpr: abs(((ceph_osd_numpg > 0) - on(job) group_left() avg by(job) (ceph_osd_numpg\n > 0)) / on(job) group_left() avg by(job) (ceph_osd_numpg > 0)) * on(ceph_daemon)\n group_left(hostname) ceph_osd_metadata > 0.3\nfor: 5m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.5\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} deviates by\n more than 30% from average PG count.\n summary: PGs are not balanced across OSDs\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.13-257-gd8c5d349 (d8c5d34975dce1c5eb0aa3a7979a4d9b9a99d1ec) pacific (stable)\", cluster_addr=\"172.21.15.89\", device_class=\"ssd\", hostname=\"smithi089\", instance=\"smithi164:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.89\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.13-257-gd8c5d349 (d8c5d34975dce1c5eb0aa3a7979a4d9b9a99d1ec) pacific (stable)\", cluster_addr=\"172.21.15.89\", device_class=\"ssd\", hostname=\"smithi089\", instance=\"smithi164.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.89\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-06-03T19:38:59.531 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:38:59 smithi089 bash[26572]: cluster 2023-06-03T19:38:57.616054+0000 mgr.x (mgr.24634) 64 : cluster [DBG] pgmap v42: 89 pgs: 89 active+clean; 5.5 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 647 op/s 2023-06-03T19:38:59.532 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:38:59 smithi089 bash[26572]: audit 2023-06-03T19:38:58.907600+0000 mon.a (mon.0) 898 : audit [DBG] from='mgr.24634 172.21.15.164:0/803880945' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T19:38:59.532 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:38:59 smithi089 bash[33173]: cluster 2023-06-03T19:38:57.616054+0000 mgr.x (mgr.24634) 64 : cluster [DBG] pgmap v42: 89 pgs: 89 active+clean; 5.5 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 647 op/s 2023-06-03T19:38:59.532 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:38:59 smithi089 bash[33173]: audit 2023-06-03T19:38:58.907600+0000 mon.a (mon.0) 898 : audit [DBG] from='mgr.24634 172.21.15.164:0/803880945' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T19:38:59.587 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:38:59 smithi164 bash[26855]: cluster 2023-06-03T19:38:57.616054+0000 mgr.x (mgr.24634) 64 : cluster [DBG] pgmap v42: 89 pgs: 89 active+clean; 5.5 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 647 op/s 2023-06-03T19:38:59.587 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:38:59 smithi164 bash[26855]: audit 2023-06-03T19:38:58.907600+0000 mon.a (mon.0) 898 : audit [DBG] from='mgr.24634 172.21.15.164:0/803880945' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T19:39:00.532 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:39:00 smithi089 bash[26572]: audit 2023-06-03T19:38:59.475277+0000 mon.a (mon.0) 899 : audit [DBG] from='client.? 172.21.15.89:0/640026648' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-03T19:39:00.532 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:39:00 smithi089 bash[33173]: audit 2023-06-03T19:38:59.475277+0000 mon.a (mon.0) 899 : audit [DBG] from='client.? 172.21.15.89:0/640026648' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-03T19:39:00.587 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:39:00 smithi164 bash[26855]: audit 2023-06-03T19:38:59.475277+0000 mon.a (mon.0) 899 : audit [DBG] from='client.? 172.21.15.89:0/640026648' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-03T19:39:01.532 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:39:01 smithi089 bash[26572]: cluster 2023-06-03T19:38:59.616562+0000 mgr.x (mgr.24634) 65 : cluster [DBG] pgmap v43: 89 pgs: 89 active+clean; 5.5 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 647 op/s 2023-06-03T19:39:01.532 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:39:01 smithi089 bash[33173]: cluster 2023-06-03T19:38:59.616562+0000 mgr.x (mgr.24634) 65 : cluster [DBG] pgmap v43: 89 pgs: 89 active+clean; 5.5 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 647 op/s 2023-06-03T19:39:01.587 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:39:01 smithi164 bash[26855]: cluster 2023-06-03T19:38:59.616562+0000 mgr.x (mgr.24634) 65 : cluster [DBG] pgmap v43: 89 pgs: 89 active+clean; 5.5 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 647 op/s 2023-06-03T19:39:03.031 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:39:02 smithi089 bash[26572]: cluster 2023-06-03T19:39:01.617375+0000 mgr.x (mgr.24634) 66 : cluster [DBG] pgmap v44: 89 pgs: 89 active+clean; 5.3 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 962 op/s 2023-06-03T19:39:03.032 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:39:02 smithi089 bash[26572]: audit 2023-06-03T19:39:02.200397+0000 mon.c (mon.2) 50 : audit [DBG] from='client.? 172.21.15.89:0/547910305' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-03T19:39:03.032 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:39:02 smithi089 bash[33173]: cluster 2023-06-03T19:39:01.617375+0000 mgr.x (mgr.24634) 66 : cluster [DBG] pgmap v44: 89 pgs: 89 active+clean; 5.3 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 962 op/s 2023-06-03T19:39:03.032 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:39:02 smithi089 bash[33173]: audit 2023-06-03T19:39:02.200397+0000 mon.c (mon.2) 50 : audit [DBG] from='client.? 172.21.15.89:0/547910305' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-03T19:39:03.087 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:39:02 smithi164 bash[26855]: cluster 2023-06-03T19:39:01.617375+0000 mgr.x (mgr.24634) 66 : cluster [DBG] pgmap v44: 89 pgs: 89 active+clean; 5.3 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 962 op/s 2023-06-03T19:39:03.087 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:39:02 smithi164 bash[26855]: audit 2023-06-03T19:39:02.200397+0000 mon.c (mon.2) 50 : audit [DBG] from='client.? 172.21.15.89:0/547910305' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-03T19:39:03.587 INFO:journalctl@ceph.mgr.x.smithi164.stdout:Jun 03 19:39:03 smithi164 bash[50883]: ::ffff:172.21.15.164 - - [03/Jun/2023:19:39:03] "GET /metrics HTTP/1.1" 200 189983 "" "Prometheus/2.33.4" 2023-06-03T19:39:05.031 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:39:04 smithi089 bash[26572]: cluster 2023-06-03T19:39:03.617881+0000 mgr.x (mgr.24634) 67 : cluster [DBG] pgmap v45: 89 pgs: 89 active+clean; 5.3 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 631 op/s 2023-06-03T19:39:05.032 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:39:04 smithi089 bash[33173]: cluster 2023-06-03T19:39:03.617881+0000 mgr.x (mgr.24634) 67 : cluster [DBG] pgmap v45: 89 pgs: 89 active+clean; 5.3 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 631 op/s 2023-06-03T19:39:05.086 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:39:04 smithi164 bash[26855]: cluster 2023-06-03T19:39:03.617881+0000 mgr.x (mgr.24634) 67 : cluster [DBG] pgmap v45: 89 pgs: 89 active+clean; 5.3 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 631 op/s 2023-06-03T19:39:06.031 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:39:05 smithi089 bash[26572]: audit 2023-06-03T19:39:04.863249+0000 mon.a (mon.0) 900 : audit [DBG] from='client.? 172.21.15.89:0/793181137' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-03T19:39:06.032 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:39:05 smithi089 bash[33173]: audit 2023-06-03T19:39:04.863249+0000 mon.a (mon.0) 900 : audit [DBG] from='client.? 172.21.15.89:0/793181137' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-03T19:39:06.086 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:39:05 smithi164 bash[26855]: audit 2023-06-03T19:39:04.863249+0000 mon.a (mon.0) 900 : audit [DBG] from='client.? 172.21.15.89:0/793181137' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-03T19:39:07.032 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:39:06 smithi089 bash[26572]: cluster 2023-06-03T19:39:05.618518+0000 mgr.x (mgr.24634) 68 : cluster [DBG] pgmap v46: 89 pgs: 89 active+clean; 5.0 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 941 op/s 2023-06-03T19:39:07.032 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:39:06 smithi089 bash[33173]: cluster 2023-06-03T19:39:05.618518+0000 mgr.x (mgr.24634) 68 : cluster [DBG] pgmap v46: 89 pgs: 89 active+clean; 5.0 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 941 op/s 2023-06-03T19:39:07.086 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:39:06 smithi164 bash[26855]: cluster 2023-06-03T19:39:05.618518+0000 mgr.x (mgr.24634) 68 : cluster [DBG] pgmap v46: 89 pgs: 89 active+clean; 5.0 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 941 op/s 2023-06-03T19:39:08.031 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:39:07 smithi089 bash[33173]: audit 2023-06-03T19:39:07.498373+0000 mon.c (mon.2) 51 : audit [DBG] from='client.? 172.21.15.89:0/1714585332' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-03T19:39:08.032 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:39:07 smithi089 bash[26572]: audit 2023-06-03T19:39:07.498373+0000 mon.c (mon.2) 51 : audit [DBG] from='client.? 172.21.15.89:0/1714585332' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-03T19:39:08.086 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:39:07 smithi164 bash[26855]: audit 2023-06-03T19:39:07.498373+0000 mon.c (mon.2) 51 : audit [DBG] from='client.? 172.21.15.89:0/1714585332' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-03T19:39:09.031 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:39:08 smithi089 bash[33173]: cluster 2023-06-03T19:39:07.622031+0000 mgr.x (mgr.24634) 69 : cluster [DBG] pgmap v47: 89 pgs: 89 active+clean; 5.0 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 624 op/s 2023-06-03T19:39:09.032 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:39:08 smithi089 bash[26572]: cluster 2023-06-03T19:39:07.622031+0000 mgr.x (mgr.24634) 69 : cluster [DBG] pgmap v47: 89 pgs: 89 active+clean; 5.0 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 624 op/s 2023-06-03T19:39:09.087 INFO:journalctl@ceph.prometheus.a.smithi164.stdout:Jun 03 19:39:08 smithi164 bash[53294]: ts=2023-06-03T19:39:08.659Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.13-257-gd8c5d349 (d8c5d34975dce1c5eb0aa3a7979a4d9b9a99d1ec) pacific (stable)\", cluster_addr=\"172.21.15.89\", device_class=\"ssd\", hostname=\"smithi089\", instance=\"smithi164:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.89\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.13-257-gd8c5d349 (d8c5d34975dce1c5eb0aa3a7979a4d9b9a99d1ec) pacific (stable)\", cluster_addr=\"172.21.15.89\", device_class=\"ssd\", hostname=\"smithi089\", instance=\"smithi164.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.89\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-06-03T19:39:09.087 INFO:journalctl@ceph.prometheus.a.smithi164.stdout:Jun 03 19:39:08 smithi164 bash[53294]: ts=2023-06-03T19:39:08.661Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephPGImbalance\nexpr: abs(((ceph_osd_numpg > 0) - on(job) group_left() avg by(job) (ceph_osd_numpg\n > 0)) / on(job) group_left() avg by(job) (ceph_osd_numpg > 0)) * on(ceph_daemon)\n group_left(hostname) ceph_osd_metadata > 0.3\nfor: 5m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.5\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} deviates by\n more than 30% from average PG count.\n summary: PGs are not balanced across OSDs\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.13-257-gd8c5d349 (d8c5d34975dce1c5eb0aa3a7979a4d9b9a99d1ec) pacific (stable)\", cluster_addr=\"172.21.15.89\", device_class=\"ssd\", hostname=\"smithi089\", instance=\"smithi164:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.89\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.13-257-gd8c5d349 (d8c5d34975dce1c5eb0aa3a7979a4d9b9a99d1ec) pacific (stable)\", cluster_addr=\"172.21.15.89\", device_class=\"ssd\", hostname=\"smithi089\", instance=\"smithi164.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.89\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-06-03T19:39:09.087 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:39:08 smithi164 bash[26855]: cluster 2023-06-03T19:39:07.622031+0000 mgr.x (mgr.24634) 69 : cluster [DBG] pgmap v47: 89 pgs: 89 active+clean; 5.0 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 624 op/s 2023-06-03T19:39:11.031 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:39:10 smithi089 bash[26572]: cluster 2023-06-03T19:39:09.622475+0000 mgr.x (mgr.24634) 70 : cluster [DBG] pgmap v48: 89 pgs: 89 active+clean; 5.0 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 624 op/s 2023-06-03T19:39:11.032 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:39:10 smithi089 bash[26572]: audit 2023-06-03T19:39:10.172176+0000 mon.a (mon.0) 901 : audit [DBG] from='client.? 172.21.15.89:0/2422577657' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-03T19:39:11.032 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:39:10 smithi089 bash[33173]: cluster 2023-06-03T19:39:09.622475+0000 mgr.x (mgr.24634) 70 : cluster [DBG] pgmap v48: 89 pgs: 89 active+clean; 5.0 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 624 op/s 2023-06-03T19:39:11.032 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:39:10 smithi089 bash[33173]: audit 2023-06-03T19:39:10.172176+0000 mon.a (mon.0) 901 : audit [DBG] from='client.? 172.21.15.89:0/2422577657' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-03T19:39:11.086 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:39:10 smithi164 bash[26855]: cluster 2023-06-03T19:39:09.622475+0000 mgr.x (mgr.24634) 70 : cluster [DBG] pgmap v48: 89 pgs: 89 active+clean; 5.0 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 624 op/s 2023-06-03T19:39:11.087 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:39:10 smithi164 bash[26855]: audit 2023-06-03T19:39:10.172176+0000 mon.a (mon.0) 901 : audit [DBG] from='client.? 172.21.15.89:0/2422577657' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-03T19:39:13.031 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:39:12 smithi089 bash[33173]: cluster 2023-06-03T19:39:11.623177+0000 mgr.x (mgr.24634) 71 : cluster [DBG] pgmap v49: 89 pgs: 89 active+clean; 4.8 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 926 op/s 2023-06-03T19:39:13.032 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:39:12 smithi089 bash[26572]: cluster 2023-06-03T19:39:11.623177+0000 mgr.x (mgr.24634) 71 : cluster [DBG] pgmap v49: 89 pgs: 89 active+clean; 4.8 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 926 op/s 2023-06-03T19:39:13.086 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:39:12 smithi164 bash[26855]: cluster 2023-06-03T19:39:11.623177+0000 mgr.x (mgr.24634) 71 : cluster [DBG] pgmap v49: 89 pgs: 89 active+clean; 4.8 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 926 op/s 2023-06-03T19:39:13.586 INFO:journalctl@ceph.mgr.x.smithi164.stdout:Jun 03 19:39:13 smithi164 bash[50883]: ::ffff:172.21.15.164 - - [03/Jun/2023:19:39:13] "GET /metrics HTTP/1.1" 200 189983 "" "Prometheus/2.33.4" 2023-06-03T19:39:14.031 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:39:13 smithi089 bash[33173]: audit 2023-06-03T19:39:12.800220+0000 mon.a (mon.0) 902 : audit [DBG] from='client.? 172.21.15.89:0/635183578' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-03T19:39:14.032 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:39:13 smithi089 bash[26572]: audit 2023-06-03T19:39:12.800220+0000 mon.a (mon.0) 902 : audit [DBG] from='client.? 172.21.15.89:0/635183578' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-03T19:39:14.087 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:39:13 smithi164 bash[26855]: audit 2023-06-03T19:39:12.800220+0000 mon.a (mon.0) 902 : audit [DBG] from='client.? 172.21.15.89:0/635183578' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-03T19:39:15.031 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:39:14 smithi089 bash[33173]: cluster 2023-06-03T19:39:13.623637+0000 mgr.x (mgr.24634) 72 : cluster [DBG] pgmap v50: 89 pgs: 89 active+clean; 4.8 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 612 op/s 2023-06-03T19:39:15.031 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:39:14 smithi089 bash[33173]: audit 2023-06-03T19:39:13.907783+0000 mon.a (mon.0) 903 : audit [DBG] from='mgr.24634 172.21.15.164:0/803880945' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T19:39:15.032 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:39:14 smithi089 bash[26572]: cluster 2023-06-03T19:39:13.623637+0000 mgr.x (mgr.24634) 72 : cluster [DBG] pgmap v50: 89 pgs: 89 active+clean; 4.8 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 612 op/s 2023-06-03T19:39:15.032 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:39:14 smithi089 bash[26572]: audit 2023-06-03T19:39:13.907783+0000 mon.a (mon.0) 903 : audit [DBG] from='mgr.24634 172.21.15.164:0/803880945' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T19:39:15.086 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:39:14 smithi164 bash[26855]: cluster 2023-06-03T19:39:13.623637+0000 mgr.x (mgr.24634) 72 : cluster [DBG] pgmap v50: 89 pgs: 89 active+clean; 4.8 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 612 op/s 2023-06-03T19:39:15.087 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:39:14 smithi164 bash[26855]: audit 2023-06-03T19:39:13.907783+0000 mon.a (mon.0) 903 : audit [DBG] from='mgr.24634 172.21.15.164:0/803880945' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T19:39:16.031 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:39:15 smithi089 bash[33173]: cluster 2023-06-03T19:39:14.645498+0000 mon.a (mon.0) 904 : cluster [DBG] mgrmap e25: x(active, since 91s), standbys: y 2023-06-03T19:39:16.031 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:39:15 smithi089 bash[33173]: audit 2023-06-03T19:39:15.412096+0000 mon.a (mon.0) 905 : audit [DBG] from='client.? 172.21.15.89:0/2434740747' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-03T19:39:16.032 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:39:15 smithi089 bash[26572]: cluster 2023-06-03T19:39:14.645498+0000 mon.a (mon.0) 904 : cluster [DBG] mgrmap e25: x(active, since 91s), standbys: y 2023-06-03T19:39:16.032 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:39:15 smithi089 bash[26572]: audit 2023-06-03T19:39:15.412096+0000 mon.a (mon.0) 905 : audit [DBG] from='client.? 172.21.15.89:0/2434740747' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-03T19:39:16.086 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:39:15 smithi164 bash[26855]: cluster 2023-06-03T19:39:14.645498+0000 mon.a (mon.0) 904 : cluster [DBG] mgrmap e25: x(active, since 91s), standbys: y 2023-06-03T19:39:16.087 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:39:15 smithi164 bash[26855]: audit 2023-06-03T19:39:15.412096+0000 mon.a (mon.0) 905 : audit [DBG] from='client.? 172.21.15.89:0/2434740747' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-03T19:39:17.031 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:39:16 smithi089 bash[26572]: cluster 2023-06-03T19:39:15.624399+0000 mgr.x (mgr.24634) 73 : cluster [DBG] pgmap v51: 89 pgs: 89 active+clean; 4.6 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 255 B/s wr, 875 op/s 2023-06-03T19:39:17.032 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:39:16 smithi089 bash[33173]: cluster 2023-06-03T19:39:15.624399+0000 mgr.x (mgr.24634) 73 : cluster [DBG] pgmap v51: 89 pgs: 89 active+clean; 4.6 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 255 B/s wr, 875 op/s 2023-06-03T19:39:17.086 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:39:16 smithi164 bash[26855]: cluster 2023-06-03T19:39:15.624399+0000 mgr.x (mgr.24634) 73 : cluster [DBG] pgmap v51: 89 pgs: 89 active+clean; 4.6 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 255 B/s wr, 875 op/s 2023-06-03T19:39:19.032 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:39:18 smithi089 bash[33173]: cluster 2023-06-03T19:39:17.625326+0000 mgr.x (mgr.24634) 74 : cluster [DBG] pgmap v52: 89 pgs: 89 active+clean; 4.6 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 255 B/s wr, 565 op/s 2023-06-03T19:39:19.060 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:39:18 smithi089 bash[33173]: audit 2023-06-03T19:39:18.015717+0000 mon.a (mon.0) 906 : audit [DBG] from='client.? 172.21.15.89:0/127558756' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-03T19:39:19.061 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:39:18 smithi089 bash[26572]: cluster 2023-06-03T19:39:17.625326+0000 mgr.x (mgr.24634) 74 : cluster [DBG] pgmap v52: 89 pgs: 89 active+clean; 4.6 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 255 B/s wr, 565 op/s 2023-06-03T19:39:19.061 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:39:18 smithi089 bash[26572]: audit 2023-06-03T19:39:18.015717+0000 mon.a (mon.0) 906 : audit [DBG] from='client.? 172.21.15.89:0/127558756' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-03T19:39:19.086 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:39:18 smithi164 bash[26855]: cluster 2023-06-03T19:39:17.625326+0000 mgr.x (mgr.24634) 74 : cluster [DBG] pgmap v52: 89 pgs: 89 active+clean; 4.6 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 255 B/s wr, 565 op/s 2023-06-03T19:39:19.087 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:39:18 smithi164 bash[26855]: audit 2023-06-03T19:39:18.015717+0000 mon.a (mon.0) 906 : audit [DBG] from='client.? 172.21.15.89:0/127558756' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-03T19:39:19.087 INFO:journalctl@ceph.prometheus.a.smithi164.stdout:Jun 03 19:39:18 smithi164 bash[53294]: ts=2023-06-03T19:39:18.659Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.13-257-gd8c5d349 (d8c5d34975dce1c5eb0aa3a7979a4d9b9a99d1ec) pacific (stable)\", cluster_addr=\"172.21.15.89\", device_class=\"ssd\", hostname=\"smithi089\", instance=\"smithi164:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.89\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.13-257-gd8c5d349 (d8c5d34975dce1c5eb0aa3a7979a4d9b9a99d1ec) pacific (stable)\", cluster_addr=\"172.21.15.89\", device_class=\"ssd\", hostname=\"smithi089\", instance=\"smithi164.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.89\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-06-03T19:39:19.087 INFO:journalctl@ceph.prometheus.a.smithi164.stdout:Jun 03 19:39:18 smithi164 bash[53294]: ts=2023-06-03T19:39:18.661Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephPGImbalance\nexpr: abs(((ceph_osd_numpg > 0) - on(job) group_left() avg by(job) (ceph_osd_numpg\n > 0)) / on(job) group_left() avg by(job) (ceph_osd_numpg > 0)) * on(ceph_daemon)\n group_left(hostname) ceph_osd_metadata > 0.3\nfor: 5m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.5\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} deviates by\n more than 30% from average PG count.\n summary: PGs are not balanced across OSDs\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.13-257-gd8c5d349 (d8c5d34975dce1c5eb0aa3a7979a4d9b9a99d1ec) pacific (stable)\", cluster_addr=\"172.21.15.89\", device_class=\"ssd\", hostname=\"smithi089\", instance=\"smithi164:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.89\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.13-257-gd8c5d349 (d8c5d34975dce1c5eb0aa3a7979a4d9b9a99d1ec) pacific (stable)\", cluster_addr=\"172.21.15.89\", device_class=\"ssd\", hostname=\"smithi089\", instance=\"smithi164.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.89\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-06-03T19:39:21.031 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:39:20 smithi089 bash[33173]: cluster 2023-06-03T19:39:19.625676+0000 mgr.x (mgr.24634) 75 : cluster [DBG] pgmap v53: 89 pgs: 89 active+clean; 4.6 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 255 B/s wr, 565 op/s 2023-06-03T19:39:21.032 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:39:20 smithi089 bash[26572]: cluster 2023-06-03T19:39:19.625676+0000 mgr.x (mgr.24634) 75 : cluster [DBG] pgmap v53: 89 pgs: 89 active+clean; 4.6 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 255 B/s wr, 565 op/s 2023-06-03T19:39:21.086 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:39:20 smithi164 bash[26855]: cluster 2023-06-03T19:39:19.625676+0000 mgr.x (mgr.24634) 75 : cluster [DBG] pgmap v53: 89 pgs: 89 active+clean; 4.6 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 255 B/s wr, 565 op/s 2023-06-03T19:39:22.031 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:39:21 smithi089 bash[26572]: audit 2023-06-03T19:39:20.719073+0000 mon.a (mon.0) 907 : audit [DBG] from='client.? 172.21.15.89:0/2895987970' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-03T19:39:22.032 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:39:21 smithi089 bash[33173]: audit 2023-06-03T19:39:20.719073+0000 mon.a (mon.0) 907 : audit [DBG] from='client.? 172.21.15.89:0/2895987970' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-03T19:39:22.086 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:39:21 smithi164 bash[26855]: audit 2023-06-03T19:39:20.719073+0000 mon.a (mon.0) 907 : audit [DBG] from='client.? 172.21.15.89:0/2895987970' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-03T19:39:23.032 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:39:22 smithi089 bash[26572]: cluster 2023-06-03T19:39:21.626381+0000 mgr.x (mgr.24634) 76 : cluster [DBG] pgmap v54: 89 pgs: 89 active+clean; 4.4 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 255 B/s wr, 834 op/s 2023-06-03T19:39:23.032 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:39:22 smithi089 bash[33173]: cluster 2023-06-03T19:39:21.626381+0000 mgr.x (mgr.24634) 76 : cluster [DBG] pgmap v54: 89 pgs: 89 active+clean; 4.4 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 255 B/s wr, 834 op/s 2023-06-03T19:39:23.086 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:39:22 smithi164 bash[26855]: cluster 2023-06-03T19:39:21.626381+0000 mgr.x (mgr.24634) 76 : cluster [DBG] pgmap v54: 89 pgs: 89 active+clean; 4.4 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 255 B/s wr, 834 op/s 2023-06-03T19:39:23.586 INFO:journalctl@ceph.mgr.x.smithi164.stdout:Jun 03 19:39:23 smithi164 bash[50883]: ::ffff:172.21.15.164 - - [03/Jun/2023:19:39:23] "GET /metrics HTTP/1.1" 200 189999 "" "Prometheus/2.33.4" 2023-06-03T19:39:24.031 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:39:23 smithi089 bash[26572]: audit 2023-06-03T19:39:23.386295+0000 mon.a (mon.0) 908 : audit [DBG] from='client.? 172.21.15.89:0/3243780747' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-03T19:39:24.032 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:39:23 smithi089 bash[33173]: audit 2023-06-03T19:39:23.386295+0000 mon.a (mon.0) 908 : audit [DBG] from='client.? 172.21.15.89:0/3243780747' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-03T19:39:24.086 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:39:23 smithi164 bash[26855]: audit 2023-06-03T19:39:23.386295+0000 mon.a (mon.0) 908 : audit [DBG] from='client.? 172.21.15.89:0/3243780747' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-03T19:39:25.032 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:39:24 smithi089 bash[26572]: cluster 2023-06-03T19:39:23.626808+0000 mgr.x (mgr.24634) 77 : cluster [DBG] pgmap v55: 89 pgs: 89 active+clean; 4.4 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 255 B/s wr, 531 op/s 2023-06-03T19:39:25.032 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:39:24 smithi089 bash[33173]: cluster 2023-06-03T19:39:23.626808+0000 mgr.x (mgr.24634) 77 : cluster [DBG] pgmap v55: 89 pgs: 89 active+clean; 4.4 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 255 B/s wr, 531 op/s 2023-06-03T19:39:25.086 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:39:24 smithi164 bash[26855]: cluster 2023-06-03T19:39:23.626808+0000 mgr.x (mgr.24634) 77 : cluster [DBG] pgmap v55: 89 pgs: 89 active+clean; 4.4 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 255 B/s wr, 531 op/s 2023-06-03T19:39:26.968 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:39:26 smithi164 bash[26855]: cluster 2023-06-03T19:39:25.627380+0000 mgr.x (mgr.24634) 78 : cluster [DBG] pgmap v56: 89 pgs: 89 active+clean; 4.2 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 255 B/s wr, 838 op/s 2023-06-03T19:39:26.969 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:39:26 smithi164 bash[26855]: audit 2023-06-03T19:39:26.066447+0000 mon.a (mon.0) 909 : audit [DBG] from='client.? 172.21.15.89:0/2926431986' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-03T19:39:27.031 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:39:26 smithi089 bash[33173]: cluster 2023-06-03T19:39:25.627380+0000 mgr.x (mgr.24634) 78 : cluster [DBG] pgmap v56: 89 pgs: 89 active+clean; 4.2 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 255 B/s wr, 838 op/s 2023-06-03T19:39:27.032 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:39:26 smithi089 bash[33173]: audit 2023-06-03T19:39:26.066447+0000 mon.a (mon.0) 909 : audit [DBG] from='client.? 172.21.15.89:0/2926431986' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-03T19:39:27.032 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:39:26 smithi089 bash[26572]: cluster 2023-06-03T19:39:25.627380+0000 mgr.x (mgr.24634) 78 : cluster [DBG] pgmap v56: 89 pgs: 89 active+clean; 4.2 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 255 B/s wr, 838 op/s 2023-06-03T19:39:27.032 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:39:26 smithi089 bash[26572]: audit 2023-06-03T19:39:26.066447+0000 mon.a (mon.0) 909 : audit [DBG] from='client.? 172.21.15.89:0/2926431986' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-03T19:39:28.973 INFO:journalctl@ceph.prometheus.a.smithi164.stdout:Jun 03 19:39:28 smithi164 bash[53294]: ts=2023-06-03T19:39:28.659Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.13-257-gd8c5d349 (d8c5d34975dce1c5eb0aa3a7979a4d9b9a99d1ec) pacific (stable)\", cluster_addr=\"172.21.15.89\", device_class=\"ssd\", hostname=\"smithi089\", instance=\"smithi164:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.89\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.13-257-gd8c5d349 (d8c5d34975dce1c5eb0aa3a7979a4d9b9a99d1ec) pacific (stable)\", cluster_addr=\"172.21.15.89\", device_class=\"ssd\", hostname=\"smithi089\", instance=\"smithi164.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.89\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-06-03T19:39:28.974 INFO:journalctl@ceph.prometheus.a.smithi164.stdout:Jun 03 19:39:28 smithi164 bash[53294]: ts=2023-06-03T19:39:28.660Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephPGImbalance\nexpr: abs(((ceph_osd_numpg > 0) - on(job) group_left() avg by(job) (ceph_osd_numpg\n > 0)) / on(job) group_left() avg by(job) (ceph_osd_numpg > 0)) * on(ceph_daemon)\n group_left(hostname) ceph_osd_metadata > 0.3\nfor: 5m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.5\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} deviates by\n more than 30% from average PG count.\n summary: PGs are not balanced across OSDs\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.13-257-gd8c5d349 (d8c5d34975dce1c5eb0aa3a7979a4d9b9a99d1ec) pacific (stable)\", cluster_addr=\"172.21.15.89\", device_class=\"ssd\", hostname=\"smithi089\", instance=\"smithi164:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.89\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.13-257-gd8c5d349 (d8c5d34975dce1c5eb0aa3a7979a4d9b9a99d1ec) pacific (stable)\", cluster_addr=\"172.21.15.89\", device_class=\"ssd\", hostname=\"smithi089\", instance=\"smithi164.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.89\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-06-03T19:39:28.974 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:39:28 smithi164 bash[26855]: cluster 2023-06-03T19:39:27.627972+0000 mgr.x (mgr.24634) 79 : cluster [DBG] pgmap v57: 89 pgs: 89 active+clean; 4.2 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 576 op/s 2023-06-03T19:39:29.031 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:39:28 smithi089 bash[33173]: cluster 2023-06-03T19:39:27.627972+0000 mgr.x (mgr.24634) 79 : cluster [DBG] pgmap v57: 89 pgs: 89 active+clean; 4.2 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 576 op/s 2023-06-03T19:39:29.032 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:39:28 smithi089 bash[26572]: cluster 2023-06-03T19:39:27.627972+0000 mgr.x (mgr.24634) 79 : cluster [DBG] pgmap v57: 89 pgs: 89 active+clean; 4.2 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 576 op/s 2023-06-03T19:39:29.966 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:39:29 smithi164 bash[26855]: audit 2023-06-03T19:39:28.766951+0000 mon.a (mon.0) 910 : audit [DBG] from='client.? 172.21.15.89:0/3776855838' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-03T19:39:29.966 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:39:29 smithi164 bash[26855]: audit 2023-06-03T19:39:28.909617+0000 mon.a (mon.0) 911 : audit [DBG] from='mgr.24634 172.21.15.164:0/803880945' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T19:39:29.966 INFO:journalctl@ceph.grafana.a.smithi164.stdout:Jun 03 19:39:29 smithi164 systemd[1]: Stopping Ceph grafana.a for f1db3478-0243-11ee-9b1d-001a4aab830c... 2023-06-03T19:39:30.031 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:39:29 smithi089 bash[33173]: audit 2023-06-03T19:39:28.766951+0000 mon.a (mon.0) 910 : audit [DBG] from='client.? 172.21.15.89:0/3776855838' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-03T19:39:30.032 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:39:29 smithi089 bash[33173]: audit 2023-06-03T19:39:28.909617+0000 mon.a (mon.0) 911 : audit [DBG] from='mgr.24634 172.21.15.164:0/803880945' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T19:39:30.032 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:39:29 smithi089 bash[26572]: audit 2023-06-03T19:39:28.766951+0000 mon.a (mon.0) 910 : audit [DBG] from='client.? 172.21.15.89:0/3776855838' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-03T19:39:30.032 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:39:29 smithi089 bash[26572]: audit 2023-06-03T19:39:28.909617+0000 mon.a (mon.0) 911 : audit [DBG] from='mgr.24634 172.21.15.164:0/803880945' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T19:39:30.324 INFO:journalctl@ceph.grafana.a.smithi164.stdout:Jun 03 19:39:29 smithi164 bash[53744]: Error response from daemon: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-grafana.a 2023-06-03T19:39:30.325 INFO:journalctl@ceph.grafana.a.smithi164.stdout:Jun 03 19:39:30 smithi164 bash[47754]: t=2023-06-03T19:39:30+0000 lvl=info msg="Shutdown started" logger=server reason="System signal: terminated" 2023-06-03T19:39:30.325 INFO:journalctl@ceph.grafana.a.smithi164.stdout:Jun 03 19:39:30 smithi164 bash[47754]: t=2023-06-03T19:39:30+0000 lvl=info msg="Database locked, sleeping then retrying" logger=sqlstore error="database is locked" retry=0 2023-06-03T19:39:30.836 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:39:30 smithi164 bash[26855]: cluster 2023-06-03T19:39:29.628384+0000 mgr.x (mgr.24634) 80 : cluster [DBG] pgmap v58: 89 pgs: 89 active+clean; 4.2 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 576 op/s 2023-06-03T19:39:31.031 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:39:30 smithi089 bash[33173]: cluster 2023-06-03T19:39:29.628384+0000 mgr.x (mgr.24634) 80 : cluster [DBG] pgmap v58: 89 pgs: 89 active+clean; 4.2 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 576 op/s 2023-06-03T19:39:31.032 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:39:30 smithi089 bash[26572]: cluster 2023-06-03T19:39:29.628384+0000 mgr.x (mgr.24634) 80 : cluster [DBG] pgmap v58: 89 pgs: 89 active+clean; 4.2 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 576 op/s 2023-06-03T19:39:31.586 INFO:journalctl@ceph.grafana.a.smithi164.stdout:Jun 03 19:39:31 smithi164 bash[53772]: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-grafana-a 2023-06-03T19:39:31.586 INFO:journalctl@ceph.grafana.a.smithi164.stdout:Jun 03 19:39:31 smithi164 systemd[1]: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c@grafana.a.service: Succeeded. 2023-06-03T19:39:31.587 INFO:journalctl@ceph.grafana.a.smithi164.stdout:Jun 03 19:39:31 smithi164 systemd[1]: Stopped Ceph grafana.a for f1db3478-0243-11ee-9b1d-001a4aab830c. 2023-06-03T19:39:31.587 INFO:journalctl@ceph.grafana.a.smithi164.stdout:Jun 03 19:39:31 smithi164 systemd[1]: Started Ceph grafana.a for f1db3478-0243-11ee-9b1d-001a4aab830c. 2023-06-03T19:39:32.587 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:39:32 smithi164 bash[26855]: audit 2023-06-03T19:39:31.343348+0000 mon.a (mon.0) 912 : audit [INF] from='mgr.24634 172.21.15.164:0/803880945' entity='mgr.x' 2023-06-03T19:39:32.587 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:39:32 smithi164 bash[26855]: audit 2023-06-03T19:39:31.352525+0000 mon.a (mon.0) 913 : audit [INF] from='mgr.24634 172.21.15.164:0/803880945' entity='mgr.x' 2023-06-03T19:39:32.587 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:39:32 smithi164 bash[26855]: audit 2023-06-03T19:39:31.394976+0000 mgr.x (mgr.24634) 81 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2023-06-03T19:39:32.587 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:39:32 smithi164 bash[26855]: audit 2023-06-03T19:39:31.396240+0000 mon.a (mon.0) 914 : audit [DBG] from='mgr.24634 172.21.15.164:0/803880945' entity='mgr.x' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2023-06-03T19:39:32.587 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:39:32 smithi164 bash[26855]: audit 2023-06-03T19:39:31.401397+0000 mgr.x (mgr.24634) 82 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-06-03T19:39:32.588 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:39:32 smithi164 bash[26855]: audit 2023-06-03T19:39:31.402592+0000 mon.a (mon.0) 915 : audit [DBG] from='mgr.24634 172.21.15.164:0/803880945' entity='mgr.x' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-06-03T19:39:32.588 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:39:32 smithi164 bash[26855]: audit 2023-06-03T19:39:31.408178+0000 mgr.x (mgr.24634) 83 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2023-06-03T19:39:32.588 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:39:32 smithi164 bash[26855]: audit 2023-06-03T19:39:31.409522+0000 mon.a (mon.0) 916 : audit [DBG] from='mgr.24634 172.21.15.164:0/803880945' entity='mgr.x' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2023-06-03T19:39:32.588 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:39:32 smithi164 bash[26855]: audit 2023-06-03T19:39:31.413773+0000 mon.a (mon.0) 917 : audit [DBG] from='client.? 172.21.15.89:0/2689901945' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-03T19:39:32.588 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:39:32 smithi164 bash[26855]: audit 2023-06-03T19:39:31.422849+0000 mon.a (mon.0) 918 : audit [DBG] from='mgr.24634 172.21.15.164:0/803880945' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-03T19:39:32.588 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:39:32 smithi164 bash[26855]: cephadm 2023-06-03T19:39:31.429646+0000 mgr.x (mgr.24634) 84 : cephadm [INF] Upgrade: Need to upgrade myself (mgr.x) 2023-06-03T19:39:32.588 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:39:32 smithi164 bash[26855]: cephadm 2023-06-03T19:39:31.430542+0000 mgr.x (mgr.24634) 85 : cephadm [INF] Upgrade: Need to upgrade myself (mgr.x) 2023-06-03T19:39:32.782 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:39:32 smithi089 bash[33173]: audit 2023-06-03T19:39:31.343348+0000 mon.a (mon.0) 912 : audit [INF] from='mgr.24634 172.21.15.164:0/803880945' entity='mgr.x' 2023-06-03T19:39:32.782 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:39:32 smithi089 bash[33173]: audit 2023-06-03T19:39:31.352525+0000 mon.a (mon.0) 913 : audit [INF] from='mgr.24634 172.21.15.164:0/803880945' entity='mgr.x' 2023-06-03T19:39:32.782 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:39:32 smithi089 bash[33173]: audit 2023-06-03T19:39:31.394976+0000 mgr.x (mgr.24634) 81 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2023-06-03T19:39:32.782 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:39:32 smithi089 bash[33173]: audit 2023-06-03T19:39:31.396240+0000 mon.a (mon.0) 914 : audit [DBG] from='mgr.24634 172.21.15.164:0/803880945' entity='mgr.x' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2023-06-03T19:39:32.782 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:39:32 smithi089 bash[33173]: audit 2023-06-03T19:39:31.401397+0000 mgr.x (mgr.24634) 82 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-06-03T19:39:32.782 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:39:32 smithi089 bash[33173]: audit 2023-06-03T19:39:31.402592+0000 mon.a (mon.0) 915 : audit [DBG] from='mgr.24634 172.21.15.164:0/803880945' entity='mgr.x' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-06-03T19:39:32.783 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:39:32 smithi089 bash[33173]: audit 2023-06-03T19:39:31.408178+0000 mgr.x (mgr.24634) 83 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2023-06-03T19:39:32.783 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:39:32 smithi089 bash[33173]: audit 2023-06-03T19:39:31.409522+0000 mon.a (mon.0) 916 : audit [DBG] from='mgr.24634 172.21.15.164:0/803880945' entity='mgr.x' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2023-06-03T19:39:32.783 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:39:32 smithi089 bash[33173]: audit 2023-06-03T19:39:31.413773+0000 mon.a (mon.0) 917 : audit [DBG] from='client.? 172.21.15.89:0/2689901945' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-03T19:39:32.783 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:39:32 smithi089 bash[33173]: audit 2023-06-03T19:39:31.422849+0000 mon.a (mon.0) 918 : audit [DBG] from='mgr.24634 172.21.15.164:0/803880945' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-03T19:39:32.783 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:39:32 smithi089 bash[33173]: cephadm 2023-06-03T19:39:31.429646+0000 mgr.x (mgr.24634) 84 : cephadm [INF] Upgrade: Need to upgrade myself (mgr.x) 2023-06-03T19:39:32.783 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:39:32 smithi089 bash[33173]: cephadm 2023-06-03T19:39:31.430542+0000 mgr.x (mgr.24634) 85 : cephadm [INF] Upgrade: Need to upgrade myself (mgr.x) 2023-06-03T19:39:32.784 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:39:32 smithi089 bash[26572]: audit 2023-06-03T19:39:31.343348+0000 mon.a (mon.0) 912 : audit [INF] from='mgr.24634 172.21.15.164:0/803880945' entity='mgr.x' 2023-06-03T19:39:32.784 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:39:32 smithi089 bash[26572]: audit 2023-06-03T19:39:31.352525+0000 mon.a (mon.0) 913 : audit [INF] from='mgr.24634 172.21.15.164:0/803880945' entity='mgr.x' 2023-06-03T19:39:32.784 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:39:32 smithi089 bash[26572]: audit 2023-06-03T19:39:31.394976+0000 mgr.x (mgr.24634) 81 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2023-06-03T19:39:32.784 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:39:32 smithi089 bash[26572]: audit 2023-06-03T19:39:31.396240+0000 mon.a (mon.0) 914 : audit [DBG] from='mgr.24634 172.21.15.164:0/803880945' entity='mgr.x' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2023-06-03T19:39:32.784 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:39:32 smithi089 bash[26572]: audit 2023-06-03T19:39:31.401397+0000 mgr.x (mgr.24634) 82 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-06-03T19:39:32.784 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:39:32 smithi089 bash[26572]: audit 2023-06-03T19:39:31.402592+0000 mon.a (mon.0) 915 : audit [DBG] from='mgr.24634 172.21.15.164:0/803880945' entity='mgr.x' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-06-03T19:39:32.784 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:39:32 smithi089 bash[26572]: audit 2023-06-03T19:39:31.408178+0000 mgr.x (mgr.24634) 83 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2023-06-03T19:39:32.785 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:39:32 smithi089 bash[26572]: audit 2023-06-03T19:39:31.409522+0000 mon.a (mon.0) 916 : audit [DBG] from='mgr.24634 172.21.15.164:0/803880945' entity='mgr.x' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2023-06-03T19:39:32.785 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:39:32 smithi089 bash[26572]: audit 2023-06-03T19:39:31.413773+0000 mon.a (mon.0) 917 : audit [DBG] from='client.? 172.21.15.89:0/2689901945' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-03T19:39:32.785 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:39:32 smithi089 bash[26572]: audit 2023-06-03T19:39:31.422849+0000 mon.a (mon.0) 918 : audit [DBG] from='mgr.24634 172.21.15.164:0/803880945' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-03T19:39:32.785 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:39:32 smithi089 bash[26572]: cephadm 2023-06-03T19:39:31.429646+0000 mgr.x (mgr.24634) 84 : cephadm [INF] Upgrade: Need to upgrade myself (mgr.x) 2023-06-03T19:39:32.785 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:39:32 smithi089 bash[26572]: cephadm 2023-06-03T19:39:31.430542+0000 mgr.x (mgr.24634) 85 : cephadm [INF] Upgrade: Need to upgrade myself (mgr.x) 2023-06-03T19:39:32.959 INFO:journalctl@ceph.grafana.a.smithi164.stdout:Jun 03 19:39:32 smithi164 bash[53862]: t=2023-06-03T19:39:32+0000 lvl=info msg="The state of unified alerting is still not defined. The decision will be made during as we run the database migrations" logger=settings 2023-06-03T19:39:32.960 INFO:journalctl@ceph.grafana.a.smithi164.stdout:Jun 03 19:39:32 smithi164 bash[53862]: t=2023-06-03T19:39:32+0000 lvl=warn msg="falling back to legacy setting of 'min_interval_seconds'; please use the configuration option in the `unified_alerting` section if Grafana 8 alerts are enabled." logger=settings 2023-06-03T19:39:32.960 INFO:journalctl@ceph.grafana.a.smithi164.stdout:Jun 03 19:39:32 smithi164 bash[53862]: t=2023-06-03T19:39:32+0000 lvl=info msg="Config loaded from" logger=settings file=/usr/share/grafana/conf/defaults.ini 2023-06-03T19:39:32.960 INFO:journalctl@ceph.grafana.a.smithi164.stdout:Jun 03 19:39:32 smithi164 bash[53862]: t=2023-06-03T19:39:32+0000 lvl=info msg="Config loaded from" logger=settings file=/etc/grafana/grafana.ini 2023-06-03T19:39:32.960 INFO:journalctl@ceph.grafana.a.smithi164.stdout:Jun 03 19:39:32 smithi164 bash[53862]: t=2023-06-03T19:39:32+0000 lvl=info msg="Config overridden from Environment variable" logger=settings var="GF_PATHS_DATA=/var/lib/grafana" 2023-06-03T19:39:32.960 INFO:journalctl@ceph.grafana.a.smithi164.stdout:Jun 03 19:39:32 smithi164 bash[53862]: t=2023-06-03T19:39:32+0000 lvl=info msg="Config overridden from Environment variable" logger=settings var="GF_PATHS_LOGS=/var/log/grafana" 2023-06-03T19:39:32.960 INFO:journalctl@ceph.grafana.a.smithi164.stdout:Jun 03 19:39:32 smithi164 bash[53862]: t=2023-06-03T19:39:32+0000 lvl=info msg="Config overridden from Environment variable" logger=settings var="GF_PATHS_PLUGINS=/var/lib/grafana/plugins" 2023-06-03T19:39:32.960 INFO:journalctl@ceph.grafana.a.smithi164.stdout:Jun 03 19:39:32 smithi164 bash[53862]: t=2023-06-03T19:39:32+0000 lvl=info msg="Config overridden from Environment variable" logger=settings var="GF_PATHS_PROVISIONING=/etc/grafana/provisioning" 2023-06-03T19:39:32.961 INFO:journalctl@ceph.grafana.a.smithi164.stdout:Jun 03 19:39:32 smithi164 bash[53862]: t=2023-06-03T19:39:32+0000 lvl=info msg="Path Home" logger=settings path=/usr/share/grafana 2023-06-03T19:39:32.961 INFO:journalctl@ceph.grafana.a.smithi164.stdout:Jun 03 19:39:32 smithi164 bash[53862]: t=2023-06-03T19:39:32+0000 lvl=info msg="Path Data" logger=settings path=/var/lib/grafana 2023-06-03T19:39:32.961 INFO:journalctl@ceph.grafana.a.smithi164.stdout:Jun 03 19:39:32 smithi164 bash[53862]: t=2023-06-03T19:39:32+0000 lvl=info msg="Path Logs" logger=settings path=/var/log/grafana 2023-06-03T19:39:32.961 INFO:journalctl@ceph.grafana.a.smithi164.stdout:Jun 03 19:39:32 smithi164 bash[53862]: t=2023-06-03T19:39:32+0000 lvl=info msg="Path Plugins" logger=settings path=/var/lib/grafana/plugins 2023-06-03T19:39:32.961 INFO:journalctl@ceph.grafana.a.smithi164.stdout:Jun 03 19:39:32 smithi164 bash[53862]: t=2023-06-03T19:39:32+0000 lvl=info msg="Path Provisioning" logger=settings path=/etc/grafana/provisioning 2023-06-03T19:39:32.961 INFO:journalctl@ceph.grafana.a.smithi164.stdout:Jun 03 19:39:32 smithi164 bash[53862]: t=2023-06-03T19:39:32+0000 lvl=info msg="App mode production" logger=settings 2023-06-03T19:39:32.961 INFO:journalctl@ceph.grafana.a.smithi164.stdout:Jun 03 19:39:32 smithi164 bash[53862]: t=2023-06-03T19:39:32+0000 lvl=info msg="Connecting to DB" logger=sqlstore dbtype=sqlite3 2023-06-03T19:39:32.962 INFO:journalctl@ceph.grafana.a.smithi164.stdout:Jun 03 19:39:32 smithi164 bash[53862]: t=2023-06-03T19:39:32+0000 lvl=warn msg="SQLite database file has broader permissions than it should" logger=sqlstore path=/var/lib/grafana/grafana.db mode=-rw-r--r-- expected=-rw-r----- 2023-06-03T19:39:32.962 INFO:journalctl@ceph.grafana.a.smithi164.stdout:Jun 03 19:39:32 smithi164 bash[53862]: t=2023-06-03T19:39:32+0000 lvl=info msg="Starting DB migrations" logger=migrator 2023-06-03T19:39:32.962 INFO:journalctl@ceph.grafana.a.smithi164.stdout:Jun 03 19:39:32 smithi164 bash[53862]: t=2023-06-03T19:39:32+0000 lvl=info msg="migrations completed" logger=migrator performed=0 skipped=377 duration=1.111949ms 2023-06-03T19:39:32.962 INFO:journalctl@ceph.grafana.a.smithi164.stdout:Jun 03 19:39:32 smithi164 bash[53862]: t=2023-06-03T19:39:32+0000 lvl=info msg="Created default organization" logger=sqlstore 2023-06-03T19:39:32.962 INFO:journalctl@ceph.grafana.a.smithi164.stdout:Jun 03 19:39:32 smithi164 bash[53862]: t=2023-06-03T19:39:32+0000 lvl=info msg="Initialising plugins" logger=plugin.manager 2023-06-03T19:39:32.962 INFO:journalctl@ceph.grafana.a.smithi164.stdout:Jun 03 19:39:32 smithi164 bash[53862]: t=2023-06-03T19:39:32+0000 lvl=info msg="Plugin registered" logger=plugin.manager pluginId=input 2023-06-03T19:39:33.217 INFO:journalctl@ceph.mgr.x.smithi164.stdout:Jun 03 19:39:33 smithi164 bash[50883]: ::ffff:172.21.15.164 - - [03/Jun/2023:19:39:33] "GET /metrics HTTP/1.1" 200 190003 "" "Prometheus/2.33.4" 2023-06-03T19:39:33.217 INFO:journalctl@ceph.grafana.a.smithi164.stdout:Jun 03 19:39:32 smithi164 bash[53862]: t=2023-06-03T19:39:32+0000 lvl=info msg="Plugin registered" logger=plugin.manager pluginId=vonage-status-panel 2023-06-03T19:39:33.217 INFO:journalctl@ceph.grafana.a.smithi164.stdout:Jun 03 19:39:32 smithi164 bash[53862]: t=2023-06-03T19:39:32+0000 lvl=info msg="Plugin registered" logger=plugin.manager pluginId=grafana-piechart-panel 2023-06-03T19:39:33.217 INFO:journalctl@ceph.grafana.a.smithi164.stdout:Jun 03 19:39:32 smithi164 bash[53862]: t=2023-06-03T19:39:32+0000 lvl=info msg="Live Push Gateway initialization" logger=live.push_http 2023-06-03T19:39:33.217 INFO:journalctl@ceph.grafana.a.smithi164.stdout:Jun 03 19:39:33 smithi164 bash[53862]: t=2023-06-03T19:39:33+0000 lvl=info msg="deleted datasource based on configuration" logger=provisioning.datasources name=Dashboard1 2023-06-03T19:39:33.586 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:39:33 smithi164 bash[26855]: cluster 2023-06-03T19:39:31.629146+0000 mgr.x (mgr.24634) 86 : cluster [DBG] pgmap v59: 89 pgs: 89 active+clean; 4.0 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 881 op/s 2023-06-03T19:39:33.587 INFO:journalctl@ceph.grafana.a.smithi164.stdout:Jun 03 19:39:33 smithi164 bash[53862]: t=2023-06-03T19:39:33+0000 lvl=info msg="inserting datasource from configuration " logger=provisioning.datasources name=Dashboard1 uid=P43CA22E17D0F9596 2023-06-03T19:39:33.587 INFO:journalctl@ceph.grafana.a.smithi164.stdout:Jun 03 19:39:33 smithi164 bash[53862]: t=2023-06-03T19:39:33+0000 lvl=info msg="inserting datasource from configuration " logger=provisioning.datasources name=Loki uid=P8E80F9AEF21F6940 2023-06-03T19:39:33.587 INFO:journalctl@ceph.grafana.a.smithi164.stdout:Jun 03 19:39:33 smithi164 bash[53862]: t=2023-06-03T19:39:33+0000 lvl=info msg="warming cache for startup" logger=ngalert 2023-06-03T19:39:33.587 INFO:journalctl@ceph.grafana.a.smithi164.stdout:Jun 03 19:39:33 smithi164 bash[53862]: t=2023-06-03T19:39:33+0000 lvl=info msg="HTTP Server Listen" logger=http.server address=[::]:3000 protocol=https subUrl= socket= 2023-06-03T19:39:33.587 INFO:journalctl@ceph.grafana.a.smithi164.stdout:Jun 03 19:39:33 smithi164 bash[53862]: t=2023-06-03T19:39:33+0000 lvl=info msg="starting MultiOrg Alertmanager" logger=ngalert.multiorg.alertmanager 2023-06-03T19:39:33.781 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:39:33 smithi089 bash[33173]: cluster 2023-06-03T19:39:31.629146+0000 mgr.x (mgr.24634) 86 : cluster [DBG] pgmap v59: 89 pgs: 89 active+clean; 4.0 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 881 op/s 2023-06-03T19:39:33.782 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:39:33 smithi089 bash[26572]: cluster 2023-06-03T19:39:31.629146+0000 mgr.x (mgr.24634) 86 : cluster [DBG] pgmap v59: 89 pgs: 89 active+clean; 4.0 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 881 op/s 2023-06-03T19:39:34.416 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:39:34 smithi089 bash[33173]: audit 2023-06-03T19:39:34.063970+0000 mon.c (mon.2) 52 : audit [DBG] from='client.? 172.21.15.89:0/3050518945' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-03T19:39:34.417 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:39:34 smithi089 bash[26572]: audit 2023-06-03T19:39:34.063970+0000 mon.c (mon.2) 52 : audit [DBG] from='client.? 172.21.15.89:0/3050518945' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-03T19:39:34.836 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:39:34 smithi164 bash[26855]: audit 2023-06-03T19:39:34.063970+0000 mon.c (mon.2) 52 : audit [DBG] from='client.? 172.21.15.89:0/3050518945' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-03T19:39:35.719 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:39:35 smithi089 bash[33173]: cluster 2023-06-03T19:39:33.629542+0000 mgr.x (mgr.24634) 87 : cluster [DBG] pgmap v60: 89 pgs: 89 active+clean; 4.0 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 612 op/s 2023-06-03T19:39:35.719 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:39:35 smithi089 bash[33173]: audit 2023-06-03T19:39:35.329458+0000 mon.a (mon.0) 919 : audit [INF] from='mgr.24634 172.21.15.164:0/803880945' entity='mgr.x' 2023-06-03T19:39:35.719 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:39:35 smithi089 bash[33173]: audit 2023-06-03T19:39:35.331638+0000 mon.a (mon.0) 920 : audit [INF] from='mgr.24634 172.21.15.164:0/803880945' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-06-03T19:39:35.719 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:39:35 smithi089 bash[33173]: audit 2023-06-03T19:39:35.333799+0000 mon.a (mon.0) 921 : audit [DBG] from='mgr.24634 172.21.15.164:0/803880945' entity='mgr.x' cmd=[{"prefix": "mgr services"}]: dispatch 2023-06-03T19:39:35.720 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:39:35 smithi089 bash[33173]: audit 2023-06-03T19:39:35.334789+0000 mon.a (mon.0) 922 : audit [DBG] from='mgr.24634 172.21.15.164:0/803880945' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-03T19:39:35.720 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:39:35 smithi089 bash[26572]: cluster 2023-06-03T19:39:33.629542+0000 mgr.x (mgr.24634) 87 : cluster [DBG] pgmap v60: 89 pgs: 89 active+clean; 4.0 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 612 op/s 2023-06-03T19:39:35.721 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:39:35 smithi089 bash[26572]: audit 2023-06-03T19:39:35.329458+0000 mon.a (mon.0) 919 : audit [INF] from='mgr.24634 172.21.15.164:0/803880945' entity='mgr.x' 2023-06-03T19:39:35.721 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:39:35 smithi089 bash[26572]: audit 2023-06-03T19:39:35.331638+0000 mon.a (mon.0) 920 : audit [INF] from='mgr.24634 172.21.15.164:0/803880945' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-06-03T19:39:35.721 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:39:35 smithi089 bash[26572]: audit 2023-06-03T19:39:35.333799+0000 mon.a (mon.0) 921 : audit [DBG] from='mgr.24634 172.21.15.164:0/803880945' entity='mgr.x' cmd=[{"prefix": "mgr services"}]: dispatch 2023-06-03T19:39:35.721 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:39:35 smithi089 bash[26572]: audit 2023-06-03T19:39:35.334789+0000 mon.a (mon.0) 922 : audit [DBG] from='mgr.24634 172.21.15.164:0/803880945' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-03T19:39:35.836 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:39:35 smithi164 bash[26855]: cluster 2023-06-03T19:39:33.629542+0000 mgr.x (mgr.24634) 87 : cluster [DBG] pgmap v60: 89 pgs: 89 active+clean; 4.0 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 612 op/s 2023-06-03T19:39:35.837 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:39:35 smithi164 bash[26855]: audit 2023-06-03T19:39:35.329458+0000 mon.a (mon.0) 919 : audit [INF] from='mgr.24634 172.21.15.164:0/803880945' entity='mgr.x' 2023-06-03T19:39:35.837 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:39:35 smithi164 bash[26855]: audit 2023-06-03T19:39:35.331638+0000 mon.a (mon.0) 920 : audit [INF] from='mgr.24634 172.21.15.164:0/803880945' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-06-03T19:39:35.837 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:39:35 smithi164 bash[26855]: audit 2023-06-03T19:39:35.333799+0000 mon.a (mon.0) 921 : audit [DBG] from='mgr.24634 172.21.15.164:0/803880945' entity='mgr.x' cmd=[{"prefix": "mgr services"}]: dispatch 2023-06-03T19:39:35.838 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:39:35 smithi164 bash[26855]: audit 2023-06-03T19:39:35.334789+0000 mon.a (mon.0) 922 : audit [DBG] from='mgr.24634 172.21.15.164:0/803880945' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-03T19:39:36.781 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:39:36 smithi089 bash[33173]: cephadm 2023-06-03T19:39:35.321384+0000 mgr.x (mgr.24634) 88 : cephadm [INF] Upgrade: Updating mgr.y 2023-06-03T19:39:36.781 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:39:36 smithi089 bash[33173]: cephadm 2023-06-03T19:39:35.333923+0000 mgr.x (mgr.24634) 89 : cephadm [INF] Deploying daemon mgr.y on smithi089 2023-06-03T19:39:36.782 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:39:36 smithi089 bash[26572]: cephadm 2023-06-03T19:39:35.321384+0000 mgr.x (mgr.24634) 88 : cephadm [INF] Upgrade: Updating mgr.y 2023-06-03T19:39:36.782 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:39:36 smithi089 bash[26572]: cephadm 2023-06-03T19:39:35.333923+0000 mgr.x (mgr.24634) 89 : cephadm [INF] Deploying daemon mgr.y on smithi089 2023-06-03T19:39:36.836 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:39:36 smithi164 bash[26855]: cephadm 2023-06-03T19:39:35.321384+0000 mgr.x (mgr.24634) 88 : cephadm [INF] Upgrade: Updating mgr.y 2023-06-03T19:39:36.837 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:39:36 smithi164 bash[26855]: cephadm 2023-06-03T19:39:35.333923+0000 mgr.x (mgr.24634) 89 : cephadm [INF] Deploying daemon mgr.y on smithi089 2023-06-03T19:39:37.782 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:39:37 smithi089 bash[26572]: cluster 2023-06-03T19:39:35.630314+0000 mgr.x (mgr.24634) 90 : cluster [DBG] pgmap v61: 89 pgs: 89 active+clean; 3.8 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 889 op/s 2023-06-03T19:39:37.782 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:39:37 smithi089 bash[26572]: audit 2023-06-03T19:39:36.751032+0000 mon.c (mon.2) 53 : audit [DBG] from='client.? 172.21.15.89:0/96593572' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-03T19:39:37.782 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:39:37 smithi089 bash[33173]: cluster 2023-06-03T19:39:35.630314+0000 mgr.x (mgr.24634) 90 : cluster [DBG] pgmap v61: 89 pgs: 89 active+clean; 3.8 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 889 op/s 2023-06-03T19:39:37.782 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:39:37 smithi089 bash[33173]: audit 2023-06-03T19:39:36.751032+0000 mon.c (mon.2) 53 : audit [DBG] from='client.? 172.21.15.89:0/96593572' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-03T19:39:37.836 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:39:37 smithi164 bash[26855]: cluster 2023-06-03T19:39:35.630314+0000 mgr.x (mgr.24634) 90 : cluster [DBG] pgmap v61: 89 pgs: 89 active+clean; 3.8 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 889 op/s 2023-06-03T19:39:37.836 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:39:37 smithi164 bash[26855]: audit 2023-06-03T19:39:36.751032+0000 mon.c (mon.2) 53 : audit [DBG] from='client.? 172.21.15.89:0/96593572' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-03T19:39:38.724 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 19:39:38 smithi089 systemd[1]: Stopping Ceph mgr.y for f1db3478-0243-11ee-9b1d-001a4aab830c... 2023-06-03T19:39:39.086 INFO:journalctl@ceph.prometheus.a.smithi164.stdout:Jun 03 19:39:38 smithi164 bash[53294]: ts=2023-06-03T19:39:38.660Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.13-257-gd8c5d349 (d8c5d34975dce1c5eb0aa3a7979a4d9b9a99d1ec) pacific (stable)\", cluster_addr=\"172.21.15.89\", device_class=\"ssd\", hostname=\"smithi089\", instance=\"smithi164:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.89\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.13-257-gd8c5d349 (d8c5d34975dce1c5eb0aa3a7979a4d9b9a99d1ec) pacific (stable)\", cluster_addr=\"172.21.15.89\", device_class=\"ssd\", hostname=\"smithi089\", instance=\"smithi164.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.89\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-06-03T19:39:39.087 INFO:journalctl@ceph.prometheus.a.smithi164.stdout:Jun 03 19:39:38 smithi164 bash[53294]: ts=2023-06-03T19:39:38.661Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephPGImbalance\nexpr: abs(((ceph_osd_numpg > 0) - on(job) group_left() avg by(job) (ceph_osd_numpg\n > 0)) / on(job) group_left() avg by(job) (ceph_osd_numpg > 0)) * on(ceph_daemon)\n group_left(hostname) ceph_osd_metadata > 0.3\nfor: 5m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.5\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} deviates by\n more than 30% from average PG count.\n summary: PGs are not balanced across OSDs\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.13-257-gd8c5d349 (d8c5d34975dce1c5eb0aa3a7979a4d9b9a99d1ec) pacific (stable)\", cluster_addr=\"172.21.15.89\", device_class=\"ssd\", hostname=\"smithi089\", instance=\"smithi164:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.89\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.13-257-gd8c5d349 (d8c5d34975dce1c5eb0aa3a7979a4d9b9a99d1ec) pacific (stable)\", cluster_addr=\"172.21.15.89\", device_class=\"ssd\", hostname=\"smithi089\", instance=\"smithi164.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.89\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-06-03T19:39:39.720 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:39:39 smithi089 bash[26572]: cluster 2023-06-03T19:39:37.630778+0000 mgr.x (mgr.24634) 91 : cluster [DBG] pgmap v62: 89 pgs: 89 active+clean; 3.8 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 582 op/s 2023-06-03T19:39:39.721 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 19:39:39 smithi089 bash[71027]: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-mgr-y 2023-06-03T19:39:39.721 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 19:39:39 smithi089 systemd[1]: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c@mgr.y.service: Main process exited, code=exited, status=143/n/a 2023-06-03T19:39:39.721 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:39:39 smithi089 bash[33173]: cluster 2023-06-03T19:39:37.630778+0000 mgr.x (mgr.24634) 91 : cluster [DBG] pgmap v62: 89 pgs: 89 active+clean; 3.8 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 582 op/s 2023-06-03T19:39:39.836 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:39:39 smithi164 bash[26855]: cluster 2023-06-03T19:39:37.630778+0000 mgr.x (mgr.24634) 91 : cluster [DBG] pgmap v62: 89 pgs: 89 active+clean; 3.8 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 582 op/s 2023-06-03T19:39:40.031 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 19:39:39 smithi089 systemd[1]: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c@mgr.y.service: Failed with result 'exit-code'. 2023-06-03T19:39:40.032 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 19:39:39 smithi089 systemd[1]: Stopped Ceph mgr.y for f1db3478-0243-11ee-9b1d-001a4aab830c. 2023-06-03T19:39:40.392 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 19:39:40 smithi089 systemd[1]: Started Ceph mgr.y for f1db3478-0243-11ee-9b1d-001a4aab830c. 2023-06-03T19:39:40.781 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:39:40 smithi089 bash[26572]: audit 2023-06-03T19:39:39.409245+0000 mon.a (mon.0) 923 : audit [DBG] from='client.? 172.21.15.89:0/901826879' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-03T19:39:40.782 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:39:40 smithi089 bash[26572]: audit 2023-06-03T19:39:40.075609+0000 mon.a (mon.0) 924 : audit [INF] from='mgr.24634 172.21.15.164:0/803880945' entity='mgr.x' 2023-06-03T19:39:40.782 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:39:40 smithi089 bash[26572]: audit 2023-06-03T19:39:40.095929+0000 mon.a (mon.0) 925 : audit [INF] from='mgr.24634 172.21.15.164:0/803880945' entity='mgr.x' 2023-06-03T19:39:40.782 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:39:40 smithi089 bash[26572]: audit 2023-06-03T19:39:40.120957+0000 mon.a (mon.0) 926 : audit [DBG] from='mgr.24634 172.21.15.164:0/803880945' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-03T19:39:40.782 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:39:40 smithi089 bash[33173]: audit 2023-06-03T19:39:39.409245+0000 mon.a (mon.0) 923 : audit [DBG] from='client.? 172.21.15.89:0/901826879' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-03T19:39:40.782 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:39:40 smithi089 bash[33173]: audit 2023-06-03T19:39:40.075609+0000 mon.a (mon.0) 924 : audit [INF] from='mgr.24634 172.21.15.164:0/803880945' entity='mgr.x' 2023-06-03T19:39:40.782 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:39:40 smithi089 bash[33173]: audit 2023-06-03T19:39:40.095929+0000 mon.a (mon.0) 925 : audit [INF] from='mgr.24634 172.21.15.164:0/803880945' entity='mgr.x' 2023-06-03T19:39:40.783 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:39:40 smithi089 bash[33173]: audit 2023-06-03T19:39:40.120957+0000 mon.a (mon.0) 926 : audit [DBG] from='mgr.24634 172.21.15.164:0/803880945' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-03T19:39:40.836 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:39:40 smithi164 bash[26855]: audit 2023-06-03T19:39:39.409245+0000 mon.a (mon.0) 923 : audit [DBG] from='client.? 172.21.15.89:0/901826879' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-03T19:39:40.837 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:39:40 smithi164 bash[26855]: audit 2023-06-03T19:39:40.075609+0000 mon.a (mon.0) 924 : audit [INF] from='mgr.24634 172.21.15.164:0/803880945' entity='mgr.x' 2023-06-03T19:39:40.837 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:39:40 smithi164 bash[26855]: audit 2023-06-03T19:39:40.095929+0000 mon.a (mon.0) 925 : audit [INF] from='mgr.24634 172.21.15.164:0/803880945' entity='mgr.x' 2023-06-03T19:39:40.837 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:39:40 smithi164 bash[26855]: audit 2023-06-03T19:39:40.120957+0000 mon.a (mon.0) 926 : audit [DBG] from='mgr.24634 172.21.15.164:0/803880945' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-03T19:39:41.781 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:39:41 smithi089 bash[26572]: cluster 2023-06-03T19:39:39.631243+0000 mgr.x (mgr.24634) 92 : cluster [DBG] pgmap v63: 89 pgs: 89 active+clean; 3.8 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 582 op/s 2023-06-03T19:39:41.782 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:39:41 smithi089 bash[33173]: cluster 2023-06-03T19:39:39.631243+0000 mgr.x (mgr.24634) 92 : cluster [DBG] pgmap v63: 89 pgs: 89 active+clean; 3.8 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 582 op/s 2023-06-03T19:39:41.836 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:39:41 smithi164 bash[26855]: cluster 2023-06-03T19:39:39.631243+0000 mgr.x (mgr.24634) 92 : cluster [DBG] pgmap v63: 89 pgs: 89 active+clean; 3.8 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 582 op/s 2023-06-03T19:39:42.281 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 19:39:41 smithi089 bash[71163]: debug 2023-06-03T19:39:41.948+0000 7ff995dcc200 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2023-06-03T19:39:43.031 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:39:42 smithi089 bash[26572]: cluster 2023-06-03T19:39:41.631826+0000 mgr.x (mgr.24634) 93 : cluster [DBG] pgmap v64: 89 pgs: 89 active+clean; 3.5 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 876 op/s 2023-06-03T19:39:43.032 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:39:42 smithi089 bash[26572]: audit 2023-06-03T19:39:42.161935+0000 mon.b (mon.1) 34 : audit [DBG] from='client.? 172.21.15.89:0/4077921818' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-03T19:39:43.032 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:39:42 smithi089 bash[33173]: cluster 2023-06-03T19:39:41.631826+0000 mgr.x (mgr.24634) 93 : cluster [DBG] pgmap v64: 89 pgs: 89 active+clean; 3.5 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 876 op/s 2023-06-03T19:39:43.032 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:39:42 smithi089 bash[33173]: audit 2023-06-03T19:39:42.161935+0000 mon.b (mon.1) 34 : audit [DBG] from='client.? 172.21.15.89:0/4077921818' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-03T19:39:43.086 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:39:42 smithi164 bash[26855]: cluster 2023-06-03T19:39:41.631826+0000 mgr.x (mgr.24634) 93 : cluster [DBG] pgmap v64: 89 pgs: 89 active+clean; 3.5 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 876 op/s 2023-06-03T19:39:43.086 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:39:42 smithi164 bash[26855]: audit 2023-06-03T19:39:42.161935+0000 mon.b (mon.1) 34 : audit [DBG] from='client.? 172.21.15.89:0/4077921818' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-03T19:39:43.522 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 19:39:43 smithi089 bash[71163]: debug 2023-06-03T19:39:43.232+0000 7ff995dcc200 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2023-06-03T19:39:43.522 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 19:39:43 smithi089 bash[71163]: debug 2023-06-03T19:39:43.380+0000 7ff995dcc200 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2023-06-03T19:39:43.586 INFO:journalctl@ceph.mgr.x.smithi164.stdout:Jun 03 19:39:43 smithi164 bash[50883]: ::ffff:172.21.15.164 - - [03/Jun/2023:19:39:43] "GET /metrics HTTP/1.1" 200 190003 "" "Prometheus/2.33.4" 2023-06-03T19:39:43.782 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 19:39:43 smithi089 bash[71163]: debug 2023-06-03T19:39:43.520+0000 7ff995dcc200 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2023-06-03T19:39:43.782 INFO:journalctl@ceph.alertmanager.a.smithi089.stdout:Jun 03 19:39:43 smithi089 bash[70028]: level=warn ts=2023-06-03T19:39:43.514Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi089.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.89:8443: connect: connection refused" 2023-06-03T19:39:44.281 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 19:39:43 smithi089 bash[71163]: debug 2023-06-03T19:39:43.888+0000 7ff995dcc200 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2023-06-03T19:39:44.701 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 19:39:44 smithi089 bash[71163]: debug 2023-06-03T19:39:44.428+0000 7ff995dcc200 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2023-06-03T19:39:44.701 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 19:39:44 smithi089 bash[71163]: debug 2023-06-03T19:39:44.672+0000 7ff995dcc200 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2023-06-03T19:39:44.701 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:39:44 smithi089 bash[26572]: cluster 2023-06-03T19:39:43.632238+0000 mgr.x (mgr.24634) 94 : cluster [DBG] pgmap v65: 89 pgs: 89 active+clean; 3.5 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 571 op/s 2023-06-03T19:39:44.702 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:39:44 smithi089 bash[26572]: audit 2023-06-03T19:39:43.909697+0000 mon.a (mon.0) 927 : audit [DBG] from='mgr.24634 172.21.15.164:0/803880945' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T19:39:45.032 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:39:44 smithi089 bash[33173]: cluster 2023-06-03T19:39:43.632238+0000 mgr.x (mgr.24634) 94 : cluster [DBG] pgmap v65: 89 pgs: 89 active+clean; 3.5 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 571 op/s 2023-06-03T19:39:45.032 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:39:44 smithi089 bash[33173]: audit 2023-06-03T19:39:43.909697+0000 mon.a (mon.0) 927 : audit [DBG] from='mgr.24634 172.21.15.164:0/803880945' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T19:39:45.086 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:39:44 smithi164 bash[26855]: cluster 2023-06-03T19:39:43.632238+0000 mgr.x (mgr.24634) 94 : cluster [DBG] pgmap v65: 89 pgs: 89 active+clean; 3.5 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 571 op/s 2023-06-03T19:39:45.086 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:39:44 smithi164 bash[26855]: audit 2023-06-03T19:39:43.909697+0000 mon.a (mon.0) 927 : audit [DBG] from='mgr.24634 172.21.15.164:0/803880945' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T19:39:45.532 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 19:39:45 smithi089 bash[71163]: debug 2023-06-03T19:39:45.036+0000 7ff995dcc200 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2023-06-03T19:39:46.031 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:39:45 smithi089 bash[26572]: audit 2023-06-03T19:39:44.944996+0000 mon.a (mon.0) 928 : audit [DBG] from='client.? 172.21.15.89:0/238913599' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-03T19:39:46.032 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:39:45 smithi089 bash[33173]: audit 2023-06-03T19:39:44.944996+0000 mon.a (mon.0) 928 : audit [DBG] from='client.? 172.21.15.89:0/238913599' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-03T19:39:46.086 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:39:45 smithi164 bash[26855]: audit 2023-06-03T19:39:44.944996+0000 mon.a (mon.0) 928 : audit [DBG] from='client.? 172.21.15.89:0/238913599' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-03T19:39:46.781 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 19:39:46 smithi089 bash[71163]: debug 2023-06-03T19:39:46.468+0000 7ff995dcc200 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2023-06-03T19:39:46.782 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 19:39:46 smithi089 bash[71163]: debug 2023-06-03T19:39:46.636+0000 7ff995dcc200 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2023-06-03T19:39:46.782 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:39:46 smithi089 bash[33173]: cluster 2023-06-03T19:39:45.632912+0000 mgr.x (mgr.24634) 95 : cluster [DBG] pgmap v66: 89 pgs: 89 active+clean; 3.3 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 848 op/s 2023-06-03T19:39:46.782 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:39:46 smithi089 bash[26572]: cluster 2023-06-03T19:39:45.632912+0000 mgr.x (mgr.24634) 95 : cluster [DBG] pgmap v66: 89 pgs: 89 active+clean; 3.3 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 848 op/s 2023-06-03T19:39:46.836 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:39:46 smithi164 bash[26855]: cluster 2023-06-03T19:39:45.632912+0000 mgr.x (mgr.24634) 95 : cluster [DBG] pgmap v66: 89 pgs: 89 active+clean; 3.3 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 848 op/s 2023-06-03T19:39:47.281 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 19:39:46 smithi089 bash[71163]: debug 2023-06-03T19:39:46.808+0000 7ff995dcc200 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2023-06-03T19:39:47.931 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 19:39:47 smithi089 bash[71163]: debug 2023-06-03T19:39:47.568+0000 7ff995dcc200 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2023-06-03T19:39:47.931 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 19:39:47 smithi089 bash[71163]: debug 2023-06-03T19:39:47.920+0000 7ff995dcc200 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2023-06-03T19:39:48.282 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:39:47 smithi089 bash[26572]: audit 2023-06-03T19:39:46.955808+0000 mon.a (mon.0) 929 : audit [INF] from='mgr.24634 172.21.15.164:0/803880945' entity='mgr.x' 2023-06-03T19:39:48.282 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:39:47 smithi089 bash[26572]: audit 2023-06-03T19:39:46.969467+0000 mon.a (mon.0) 930 : audit [INF] from='mgr.24634 172.21.15.164:0/803880945' entity='mgr.x' 2023-06-03T19:39:48.283 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:39:47 smithi089 bash[26572]: audit 2023-06-03T19:39:47.677330+0000 mon.a (mon.0) 931 : audit [DBG] from='client.? 172.21.15.89:0/916838829' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-03T19:39:48.283 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:39:47 smithi089 bash[26572]: audit 2023-06-03T19:39:47.763386+0000 mon.a (mon.0) 932 : audit [INF] from='mgr.24634 172.21.15.164:0/803880945' entity='mgr.x' 2023-06-03T19:39:48.283 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:39:47 smithi089 bash[26572]: audit 2023-06-03T19:39:47.774256+0000 mon.a (mon.0) 933 : audit [INF] from='mgr.24634 172.21.15.164:0/803880945' entity='mgr.x' 2023-06-03T19:39:48.284 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:39:47 smithi089 bash[33173]: audit 2023-06-03T19:39:46.955808+0000 mon.a (mon.0) 929 : audit [INF] from='mgr.24634 172.21.15.164:0/803880945' entity='mgr.x' 2023-06-03T19:39:48.284 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:39:47 smithi089 bash[33173]: audit 2023-06-03T19:39:46.969467+0000 mon.a (mon.0) 930 : audit [INF] from='mgr.24634 172.21.15.164:0/803880945' entity='mgr.x' 2023-06-03T19:39:48.284 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:39:47 smithi089 bash[33173]: audit 2023-06-03T19:39:47.677330+0000 mon.a (mon.0) 931 : audit [DBG] from='client.? 172.21.15.89:0/916838829' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-03T19:39:48.285 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:39:47 smithi089 bash[33173]: audit 2023-06-03T19:39:47.763386+0000 mon.a (mon.0) 932 : audit [INF] from='mgr.24634 172.21.15.164:0/803880945' entity='mgr.x' 2023-06-03T19:39:48.285 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:39:47 smithi089 bash[33173]: audit 2023-06-03T19:39:47.774256+0000 mon.a (mon.0) 933 : audit [INF] from='mgr.24634 172.21.15.164:0/803880945' entity='mgr.x' 2023-06-03T19:39:48.336 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:39:47 smithi164 bash[26855]: audit 2023-06-03T19:39:46.955808+0000 mon.a (mon.0) 929 : audit [INF] from='mgr.24634 172.21.15.164:0/803880945' entity='mgr.x' 2023-06-03T19:39:48.337 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:39:47 smithi164 bash[26855]: audit 2023-06-03T19:39:46.969467+0000 mon.a (mon.0) 930 : audit [INF] from='mgr.24634 172.21.15.164:0/803880945' entity='mgr.x' 2023-06-03T19:39:48.337 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:39:47 smithi164 bash[26855]: audit 2023-06-03T19:39:47.677330+0000 mon.a (mon.0) 931 : audit [DBG] from='client.? 172.21.15.89:0/916838829' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-03T19:39:48.337 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:39:47 smithi164 bash[26855]: audit 2023-06-03T19:39:47.763386+0000 mon.a (mon.0) 932 : audit [INF] from='mgr.24634 172.21.15.164:0/803880945' entity='mgr.x' 2023-06-03T19:39:48.338 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:39:47 smithi164 bash[26855]: audit 2023-06-03T19:39:47.774256+0000 mon.a (mon.0) 933 : audit [INF] from='mgr.24634 172.21.15.164:0/803880945' entity='mgr.x' 2023-06-03T19:39:48.781 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 19:39:48 smithi089 bash[71163]: debug 2023-06-03T19:39:48.476+0000 7ff995dcc200 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2023-06-03T19:39:48.992 INFO:journalctl@ceph.prometheus.a.smithi164.stdout:Jun 03 19:39:48 smithi164 bash[53294]: ts=2023-06-03T19:39:48.660Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.13-257-gd8c5d349 (d8c5d34975dce1c5eb0aa3a7979a4d9b9a99d1ec) pacific (stable)\", cluster_addr=\"172.21.15.89\", device_class=\"ssd\", hostname=\"smithi089\", instance=\"smithi164:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.89\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.13-257-gd8c5d349 (d8c5d34975dce1c5eb0aa3a7979a4d9b9a99d1ec) pacific (stable)\", cluster_addr=\"172.21.15.89\", device_class=\"ssd\", hostname=\"smithi089\", instance=\"smithi164.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.89\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-06-03T19:39:48.992 INFO:journalctl@ceph.prometheus.a.smithi164.stdout:Jun 03 19:39:48 smithi164 bash[53294]: ts=2023-06-03T19:39:48.663Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephPGImbalance\nexpr: abs(((ceph_osd_numpg > 0) - on(job) group_left() avg by(job) (ceph_osd_numpg\n > 0)) / on(job) group_left() avg by(job) (ceph_osd_numpg > 0)) * on(ceph_daemon)\n group_left(hostname) ceph_osd_metadata > 0.3\nfor: 5m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.5\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} deviates by\n more than 30% from average PG count.\n summary: PGs are not balanced across OSDs\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.13-257-gd8c5d349 (d8c5d34975dce1c5eb0aa3a7979a4d9b9a99d1ec) pacific (stable)\", cluster_addr=\"172.21.15.89\", device_class=\"ssd\", hostname=\"smithi089\", instance=\"smithi164:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.89\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.13-257-gd8c5d349 (d8c5d34975dce1c5eb0aa3a7979a4d9b9a99d1ec) pacific (stable)\", cluster_addr=\"172.21.15.89\", device_class=\"ssd\", hostname=\"smithi089\", instance=\"smithi164.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.89\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-06-03T19:39:48.993 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:39:48 smithi164 bash[26855]: cluster 2023-06-03T19:39:47.633359+0000 mgr.x (mgr.24634) 96 : cluster [DBG] pgmap v67: 89 pgs: 89 active+clean; 3.3 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 571 op/s 2023-06-03T19:39:48.993 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:39:48 smithi164 bash[26855]: audit 2023-06-03T19:39:48.774493+0000 mon.a (mon.0) 934 : audit [INF] from='mgr.24634 172.21.15.164:0/803880945' entity='mgr.x' 2023-06-03T19:39:48.993 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:39:48 smithi164 bash[26855]: audit 2023-06-03T19:39:48.782344+0000 mon.a (mon.0) 935 : audit [INF] from='mgr.24634 172.21.15.164:0/803880945' entity='mgr.x' 2023-06-03T19:39:49.281 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 19:39:49 smithi089 bash[71163]: debug 2023-06-03T19:39:49.216+0000 7ff995dcc200 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2023-06-03T19:39:49.282 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:39:48 smithi089 bash[33173]: cluster 2023-06-03T19:39:47.633359+0000 mgr.x (mgr.24634) 96 : cluster [DBG] pgmap v67: 89 pgs: 89 active+clean; 3.3 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 571 op/s 2023-06-03T19:39:49.282 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:39:48 smithi089 bash[33173]: audit 2023-06-03T19:39:48.774493+0000 mon.a (mon.0) 934 : audit [INF] from='mgr.24634 172.21.15.164:0/803880945' entity='mgr.x' 2023-06-03T19:39:49.282 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:39:48 smithi089 bash[33173]: audit 2023-06-03T19:39:48.782344+0000 mon.a (mon.0) 935 : audit [INF] from='mgr.24634 172.21.15.164:0/803880945' entity='mgr.x' 2023-06-03T19:39:49.282 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:39:48 smithi089 bash[26572]: cluster 2023-06-03T19:39:47.633359+0000 mgr.x (mgr.24634) 96 : cluster [DBG] pgmap v67: 89 pgs: 89 active+clean; 3.3 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 571 op/s 2023-06-03T19:39:49.282 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:39:48 smithi089 bash[26572]: audit 2023-06-03T19:39:48.774493+0000 mon.a (mon.0) 934 : audit [INF] from='mgr.24634 172.21.15.164:0/803880945' entity='mgr.x' 2023-06-03T19:39:49.283 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:39:48 smithi089 bash[26572]: audit 2023-06-03T19:39:48.782344+0000 mon.a (mon.0) 935 : audit [INF] from='mgr.24634 172.21.15.164:0/803880945' entity='mgr.x' 2023-06-03T19:39:50.281 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 19:39:50 smithi089 bash[71163]: debug 2023-06-03T19:39:50.000+0000 7ff995dcc200 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2023-06-03T19:39:50.281 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 19:39:50 smithi089 bash[71163]: debug 2023-06-03T19:39:50.160+0000 7ff995dcc200 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2023-06-03T19:39:51.000 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 19:39:50 smithi089 bash[71163]: debug 2023-06-03T19:39:50.648+0000 7ff995dcc200 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2023-06-03T19:39:51.000 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 19:39:50 smithi089 bash[71163]: debug 2023-06-03T19:39:50.800+0000 7ff995dcc200 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2023-06-03T19:39:51.281 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:39:50 smithi089 bash[26572]: cluster 2023-06-03T19:39:49.633924+0000 mgr.x (mgr.24634) 97 : cluster [DBG] pgmap v68: 89 pgs: 89 active+clean; 3.3 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 571 op/s 2023-06-03T19:39:51.281 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:39:50 smithi089 bash[26572]: audit 2023-06-03T19:39:50.414725+0000 mon.c (mon.2) 54 : audit [DBG] from='client.? 172.21.15.89:0/3808389265' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-03T19:39:51.282 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 19:39:51 smithi089 bash[71163]: debug 2023-06-03T19:39:51.004+0000 7ff995dcc200 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2023-06-03T19:39:51.282 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 19:39:51 smithi089 bash[71163]: debug 2023-06-03T19:39:51.156+0000 7ff995dcc200 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2023-06-03T19:39:51.282 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:39:50 smithi089 bash[33173]: cluster 2023-06-03T19:39:49.633924+0000 mgr.x (mgr.24634) 97 : cluster [DBG] pgmap v68: 89 pgs: 89 active+clean; 3.3 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 571 op/s 2023-06-03T19:39:51.282 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:39:50 smithi089 bash[33173]: audit 2023-06-03T19:39:50.414725+0000 mon.c (mon.2) 54 : audit [DBG] from='client.? 172.21.15.89:0/3808389265' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-03T19:39:51.336 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:39:50 smithi164 bash[26855]: cluster 2023-06-03T19:39:49.633924+0000 mgr.x (mgr.24634) 97 : cluster [DBG] pgmap v68: 89 pgs: 89 active+clean; 3.3 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 571 op/s 2023-06-03T19:39:51.336 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:39:50 smithi164 bash[26855]: audit 2023-06-03T19:39:50.414725+0000 mon.c (mon.2) 54 : audit [DBG] from='client.? 172.21.15.89:0/3808389265' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-03T19:39:51.596 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 19:39:51 smithi089 bash[71163]: debug 2023-06-03T19:39:51.324+0000 7ff995dcc200 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2023-06-03T19:39:52.031 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 19:39:51 smithi089 bash[71163]: debug 2023-06-03T19:39:51.588+0000 7ff995dcc200 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2023-06-03T19:39:52.032 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 19:39:51 smithi089 bash[71163]: debug 2023-06-03T19:39:51.752+0000 7ff995dcc200 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2023-06-03T19:39:53.031 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:39:52 smithi089 bash[33173]: cluster 2023-06-03T19:39:51.636623+0000 mgr.x (mgr.24634) 98 : cluster [DBG] pgmap v69: 89 pgs: 89 active+clean; 3.2 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 829 op/s 2023-06-03T19:39:53.032 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:39:52 smithi089 bash[26572]: cluster 2023-06-03T19:39:51.636623+0000 mgr.x (mgr.24634) 98 : cluster [DBG] pgmap v69: 89 pgs: 89 active+clean; 3.2 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 829 op/s 2023-06-03T19:39:53.086 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:39:52 smithi164 bash[26855]: cluster 2023-06-03T19:39:51.636623+0000 mgr.x (mgr.24634) 98 : cluster [DBG] pgmap v69: 89 pgs: 89 active+clean; 3.2 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 829 op/s 2023-06-03T19:39:53.586 INFO:journalctl@ceph.mgr.x.smithi164.stdout:Jun 03 19:39:53 smithi164 bash[50883]: ::ffff:172.21.15.164 - - [03/Jun/2023:19:39:53] "GET /metrics HTTP/1.1" 200 190031 "" "Prometheus/2.33.4" 2023-06-03T19:39:53.781 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 19:39:53 smithi089 bash[71163]: debug 2023-06-03T19:39:53.552+0000 7ff995dcc200 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2023-06-03T19:39:53.781 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 19:39:53 smithi089 bash[71163]: [03/Jun/2023:19:39:53] ENGINE Bus STARTING 2023-06-03T19:39:53.782 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 19:39:53 smithi089 bash[71163]: CherryPy Checker: 2023-06-03T19:39:53.782 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 19:39:53 smithi089 bash[71163]: The Application mounted at '' has an empty config. 2023-06-03T19:39:53.782 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 19:39:53 smithi089 bash[71163]: [03/Jun/2023:19:39:53] ENGINE Serving on http://:::9283 2023-06-03T19:39:53.782 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 19:39:53 smithi089 bash[71163]: [03/Jun/2023:19:39:53] ENGINE Bus STARTED 2023-06-03T19:39:53.782 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:39:53 smithi089 bash[33173]: audit 2023-06-03T19:39:53.123272+0000 mon.c (mon.2) 55 : audit [DBG] from='client.? 172.21.15.89:0/3239118473' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-03T19:39:53.783 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:39:53 smithi089 bash[33173]: cluster 2023-06-03T19:39:53.582864+0000 mon.a (mon.0) 936 : cluster [DBG] Standby manager daemon y restarted 2023-06-03T19:39:53.783 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:39:53 smithi089 bash[33173]: cluster 2023-06-03T19:39:53.583008+0000 mon.a (mon.0) 937 : cluster [DBG] Standby manager daemon y started 2023-06-03T19:39:53.783 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:39:53 smithi089 bash[33173]: audit 2023-06-03T19:39:53.583062+0000 mon.c (mon.2) 56 : audit [DBG] from='mgr.? 172.21.15.89:0/2108989343' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2023-06-03T19:39:53.783 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:39:53 smithi089 bash[33173]: audit 2023-06-03T19:39:53.589184+0000 mon.c (mon.2) 57 : audit [DBG] from='mgr.? 172.21.15.89:0/2108989343' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-06-03T19:39:53.783 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:39:53 smithi089 bash[33173]: audit 2023-06-03T19:39:53.591121+0000 mon.c (mon.2) 58 : audit [DBG] from='mgr.? 172.21.15.89:0/2108989343' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2023-06-03T19:39:53.783 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:39:53 smithi089 bash[33173]: audit 2023-06-03T19:39:53.591694+0000 mon.c (mon.2) 59 : audit [DBG] from='mgr.? 172.21.15.89:0/2108989343' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-06-03T19:39:53.784 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:39:53 smithi089 bash[26572]: audit 2023-06-03T19:39:53.123272+0000 mon.c (mon.2) 55 : audit [DBG] from='client.? 172.21.15.89:0/3239118473' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-03T19:39:53.784 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:39:53 smithi089 bash[26572]: cluster 2023-06-03T19:39:53.582864+0000 mon.a (mon.0) 936 : cluster [DBG] Standby manager daemon y restarted 2023-06-03T19:39:53.784 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:39:53 smithi089 bash[26572]: cluster 2023-06-03T19:39:53.583008+0000 mon.a (mon.0) 937 : cluster [DBG] Standby manager daemon y started 2023-06-03T19:39:53.784 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:39:53 smithi089 bash[26572]: audit 2023-06-03T19:39:53.583062+0000 mon.c (mon.2) 56 : audit [DBG] from='mgr.? 172.21.15.89:0/2108989343' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2023-06-03T19:39:53.784 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:39:53 smithi089 bash[26572]: audit 2023-06-03T19:39:53.589184+0000 mon.c (mon.2) 57 : audit [DBG] from='mgr.? 172.21.15.89:0/2108989343' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-06-03T19:39:53.784 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:39:53 smithi089 bash[26572]: audit 2023-06-03T19:39:53.591121+0000 mon.c (mon.2) 58 : audit [DBG] from='mgr.? 172.21.15.89:0/2108989343' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2023-06-03T19:39:53.784 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:39:53 smithi089 bash[26572]: audit 2023-06-03T19:39:53.591694+0000 mon.c (mon.2) 59 : audit [DBG] from='mgr.? 172.21.15.89:0/2108989343' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-06-03T19:39:53.785 INFO:journalctl@ceph.alertmanager.a.smithi089.stdout:Jun 03 19:39:53 smithi089 bash[70028]: level=error ts=2023-06-03T19:39:53.514Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi089.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.89:8443: connect: connection refused" 2023-06-03T19:39:53.785 INFO:journalctl@ceph.alertmanager.a.smithi089.stdout:Jun 03 19:39:53 smithi089 bash[70028]: level=warn ts=2023-06-03T19:39:53.515Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi089.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.89:8443: connect: connection refused" 2023-06-03T19:39:54.086 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:39:53 smithi164 bash[26855]: audit 2023-06-03T19:39:53.123272+0000 mon.c (mon.2) 55 : audit [DBG] from='client.? 172.21.15.89:0/3239118473' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-03T19:39:54.087 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:39:53 smithi164 bash[26855]: cluster 2023-06-03T19:39:53.582864+0000 mon.a (mon.0) 936 : cluster [DBG] Standby manager daemon y restarted 2023-06-03T19:39:54.087 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:39:53 smithi164 bash[26855]: cluster 2023-06-03T19:39:53.583008+0000 mon.a (mon.0) 937 : cluster [DBG] Standby manager daemon y started 2023-06-03T19:39:54.087 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:39:53 smithi164 bash[26855]: audit 2023-06-03T19:39:53.583062+0000 mon.c (mon.2) 56 : audit [DBG] from='mgr.? 172.21.15.89:0/2108989343' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2023-06-03T19:39:54.087 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:39:53 smithi164 bash[26855]: audit 2023-06-03T19:39:53.589184+0000 mon.c (mon.2) 57 : audit [DBG] from='mgr.? 172.21.15.89:0/2108989343' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-06-03T19:39:54.087 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:39:53 smithi164 bash[26855]: audit 2023-06-03T19:39:53.591121+0000 mon.c (mon.2) 58 : audit [DBG] from='mgr.? 172.21.15.89:0/2108989343' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2023-06-03T19:39:54.087 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:39:53 smithi164 bash[26855]: audit 2023-06-03T19:39:53.591694+0000 mon.c (mon.2) 59 : audit [DBG] from='mgr.? 172.21.15.89:0/2108989343' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-06-03T19:39:54.281 INFO:journalctl@ceph.alertmanager.a.smithi089.stdout:Jun 03 19:39:53 smithi089 bash[70028]: level=info ts=2023-06-03T19:39:53.990Z caller=notify.go:735 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify success" attempts=2 2023-06-03T19:39:55.031 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:39:54 smithi089 bash[26572]: cluster 2023-06-03T19:39:53.637004+0000 mgr.x (mgr.24634) 99 : cluster [DBG] pgmap v70: 89 pgs: 89 active+clean; 3.2 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 535 op/s 2023-06-03T19:39:55.032 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:39:54 smithi089 bash[26572]: cluster 2023-06-03T19:39:53.637990+0000 mon.a (mon.0) 938 : cluster [DBG] mgrmap e26: x(active, since 2m), standbys: y 2023-06-03T19:39:55.032 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:39:54 smithi089 bash[33173]: cluster 2023-06-03T19:39:53.637004+0000 mgr.x (mgr.24634) 99 : cluster [DBG] pgmap v70: 89 pgs: 89 active+clean; 3.2 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 535 op/s 2023-06-03T19:39:55.032 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:39:54 smithi089 bash[33173]: cluster 2023-06-03T19:39:53.637990+0000 mon.a (mon.0) 938 : cluster [DBG] mgrmap e26: x(active, since 2m), standbys: y 2023-06-03T19:39:55.087 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:39:54 smithi164 bash[26855]: cluster 2023-06-03T19:39:53.637004+0000 mgr.x (mgr.24634) 99 : cluster [DBG] pgmap v70: 89 pgs: 89 active+clean; 3.2 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 535 op/s 2023-06-03T19:39:55.087 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:39:54 smithi164 bash[26855]: cluster 2023-06-03T19:39:53.637990+0000 mon.a (mon.0) 938 : cluster [DBG] mgrmap e26: x(active, since 2m), standbys: y 2023-06-03T19:39:56.781 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:39:56 smithi089 bash[33173]: cluster 2023-06-03T19:39:55.637815+0000 mgr.x (mgr.24634) 100 : cluster [DBG] pgmap v71: 89 pgs: 89 active+clean; 3.0 GiB data, 9.5 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 815 op/s 2023-06-03T19:39:56.782 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:39:56 smithi089 bash[33173]: audit 2023-06-03T19:39:55.783429+0000 mon.a (mon.0) 939 : audit [DBG] from='client.? 172.21.15.89:0/1561645357' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-03T19:39:56.782 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:39:56 smithi089 bash[26572]: cluster 2023-06-03T19:39:55.637815+0000 mgr.x (mgr.24634) 100 : cluster [DBG] pgmap v71: 89 pgs: 89 active+clean; 3.0 GiB data, 9.5 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 815 op/s 2023-06-03T19:39:56.783 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:39:56 smithi089 bash[26572]: audit 2023-06-03T19:39:55.783429+0000 mon.a (mon.0) 939 : audit [DBG] from='client.? 172.21.15.89:0/1561645357' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-03T19:39:57.086 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:39:56 smithi164 bash[26855]: cluster 2023-06-03T19:39:55.637815+0000 mgr.x (mgr.24634) 100 : cluster [DBG] pgmap v71: 89 pgs: 89 active+clean; 3.0 GiB data, 9.5 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 815 op/s 2023-06-03T19:39:57.086 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:39:56 smithi164 bash[26855]: audit 2023-06-03T19:39:55.783429+0000 mon.a (mon.0) 939 : audit [DBG] from='client.? 172.21.15.89:0/1561645357' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-03T19:39:59.031 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:39:58 smithi089 bash[33173]: cluster 2023-06-03T19:39:57.638245+0000 mgr.x (mgr.24634) 101 : cluster [DBG] pgmap v72: 89 pgs: 89 active+clean; 3.0 GiB data, 9.5 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 538 op/s 2023-06-03T19:39:59.032 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:39:58 smithi089 bash[33173]: audit 2023-06-03T19:39:58.467838+0000 mon.b (mon.1) 35 : audit [DBG] from='client.? 172.21.15.89:0/4150426962' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-03T19:39:59.032 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:39:58 smithi089 bash[26572]: cluster 2023-06-03T19:39:57.638245+0000 mgr.x (mgr.24634) 101 : cluster [DBG] pgmap v72: 89 pgs: 89 active+clean; 3.0 GiB data, 9.5 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 538 op/s 2023-06-03T19:39:59.032 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:39:58 smithi089 bash[26572]: audit 2023-06-03T19:39:58.467838+0000 mon.b (mon.1) 35 : audit [DBG] from='client.? 172.21.15.89:0/4150426962' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-03T19:39:59.086 INFO:journalctl@ceph.prometheus.a.smithi164.stdout:Jun 03 19:39:58 smithi164 bash[53294]: ts=2023-06-03T19:39:58.660Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.13-257-gd8c5d349 (d8c5d34975dce1c5eb0aa3a7979a4d9b9a99d1ec) pacific (stable)\", cluster_addr=\"172.21.15.89\", device_class=\"ssd\", hostname=\"smithi089\", instance=\"smithi164:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.89\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.13-257-gd8c5d349 (d8c5d34975dce1c5eb0aa3a7979a4d9b9a99d1ec) pacific (stable)\", cluster_addr=\"172.21.15.89\", device_class=\"ssd\", hostname=\"smithi089\", instance=\"smithi164.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.89\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-06-03T19:39:59.087 INFO:journalctl@ceph.prometheus.a.smithi164.stdout:Jun 03 19:39:58 smithi164 bash[53294]: ts=2023-06-03T19:39:58.661Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephPGImbalance\nexpr: abs(((ceph_osd_numpg > 0) - on(job) group_left() avg by(job) (ceph_osd_numpg\n > 0)) / on(job) group_left() avg by(job) (ceph_osd_numpg > 0)) * on(ceph_daemon)\n group_left(hostname) ceph_osd_metadata > 0.3\nfor: 5m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.5\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} deviates by\n more than 30% from average PG count.\n summary: PGs are not balanced across OSDs\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.13-257-gd8c5d349 (d8c5d34975dce1c5eb0aa3a7979a4d9b9a99d1ec) pacific (stable)\", cluster_addr=\"172.21.15.89\", device_class=\"ssd\", hostname=\"smithi089\", instance=\"smithi164:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.89\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.13-257-gd8c5d349 (d8c5d34975dce1c5eb0aa3a7979a4d9b9a99d1ec) pacific (stable)\", cluster_addr=\"172.21.15.89\", device_class=\"ssd\", hostname=\"smithi089\", instance=\"smithi164.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.89\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-06-03T19:39:59.087 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:39:58 smithi164 bash[26855]: cluster 2023-06-03T19:39:57.638245+0000 mgr.x (mgr.24634) 101 : cluster [DBG] pgmap v72: 89 pgs: 89 active+clean; 3.0 GiB data, 9.5 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 538 op/s 2023-06-03T19:39:59.087 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:39:58 smithi164 bash[26855]: audit 2023-06-03T19:39:58.467838+0000 mon.b (mon.1) 35 : audit [DBG] from='client.? 172.21.15.89:0/4150426962' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-03T19:40:00.031 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:39:59 smithi089 bash[26572]: audit 2023-06-03T19:39:58.913559+0000 mon.a (mon.0) 940 : audit [DBG] from='mgr.24634 172.21.15.164:0/803880945' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T19:40:00.032 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:39:59 smithi089 bash[33173]: audit 2023-06-03T19:39:58.913559+0000 mon.a (mon.0) 940 : audit [DBG] from='mgr.24634 172.21.15.164:0/803880945' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T19:40:00.086 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:39:59 smithi164 bash[26855]: audit 2023-06-03T19:39:58.913559+0000 mon.a (mon.0) 940 : audit [DBG] from='mgr.24634 172.21.15.164:0/803880945' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T19:40:01.032 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:40:00 smithi089 bash[26572]: cluster 2023-06-03T19:39:59.638668+0000 mgr.x (mgr.24634) 102 : cluster [DBG] pgmap v73: 89 pgs: 89 active+clean; 3.0 GiB data, 9.5 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 538 op/s 2023-06-03T19:40:01.032 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:40:00 smithi089 bash[26572]: cluster 2023-06-03T19:40:00.000112+0000 mon.a (mon.0) 941 : cluster [INF] overall HEALTH_OK 2023-06-03T19:40:01.032 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:40:00 smithi089 bash[33173]: cluster 2023-06-03T19:39:59.638668+0000 mgr.x (mgr.24634) 102 : cluster [DBG] pgmap v73: 89 pgs: 89 active+clean; 3.0 GiB data, 9.5 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 538 op/s 2023-06-03T19:40:01.033 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:40:00 smithi089 bash[33173]: cluster 2023-06-03T19:40:00.000112+0000 mon.a (mon.0) 941 : cluster [INF] overall HEALTH_OK 2023-06-03T19:40:01.086 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:40:00 smithi164 bash[26855]: cluster 2023-06-03T19:39:59.638668+0000 mgr.x (mgr.24634) 102 : cluster [DBG] pgmap v73: 89 pgs: 89 active+clean; 3.0 GiB data, 9.5 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 538 op/s 2023-06-03T19:40:01.087 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:40:00 smithi164 bash[26855]: cluster 2023-06-03T19:40:00.000112+0000 mon.a (mon.0) 941 : cluster [INF] overall HEALTH_OK 2023-06-03T19:40:02.282 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:40:01 smithi089 bash[26572]: audit 2023-06-03T19:40:00.966606+0000 mon.a (mon.0) 942 : audit [INF] from='mgr.24634 172.21.15.164:0/803880945' entity='mgr.x' 2023-06-03T19:40:02.282 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:40:01 smithi089 bash[26572]: audit 2023-06-03T19:40:00.984573+0000 mon.a (mon.0) 943 : audit [INF] from='mgr.24634 172.21.15.164:0/803880945' entity='mgr.x' 2023-06-03T19:40:02.282 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:40:01 smithi089 bash[26572]: audit 2023-06-03T19:40:00.986294+0000 mon.a (mon.0) 944 : audit [DBG] from='mgr.24634 172.21.15.164:0/803880945' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-03T19:40:02.283 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:40:01 smithi089 bash[26572]: audit 2023-06-03T19:40:00.987903+0000 mon.a (mon.0) 945 : audit [INF] from='mgr.24634 172.21.15.164:0/803880945' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-03T19:40:02.283 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:40:01 smithi089 bash[26572]: audit 2023-06-03T19:40:01.001563+0000 mon.a (mon.0) 946 : audit [INF] from='mgr.24634 172.21.15.164:0/803880945' entity='mgr.x' 2023-06-03T19:40:02.283 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:40:01 smithi089 bash[26572]: audit 2023-06-03T19:40:01.086525+0000 mon.a (mon.0) 947 : audit [DBG] from='mgr.24634 172.21.15.164:0/803880945' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-03T19:40:02.283 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:40:01 smithi089 bash[26572]: cephadm 2023-06-03T19:40:01.095102+0000 mgr.x (mgr.24634) 103 : cephadm [INF] Upgrade: Need to upgrade myself (mgr.x) 2023-06-03T19:40:02.284 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:40:01 smithi089 bash[26572]: cephadm 2023-06-03T19:40:01.095862+0000 mgr.x (mgr.24634) 104 : cephadm [INF] Upgrade: Need to upgrade myself (mgr.x) 2023-06-03T19:40:02.284 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:40:01 smithi089 bash[26572]: cephadm 2023-06-03T19:40:01.100557+0000 mgr.x (mgr.24634) 105 : cephadm [INF] Failing over to other MGR 2023-06-03T19:40:02.284 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:40:01 smithi089 bash[26572]: audit 2023-06-03T19:40:01.102854+0000 mon.a (mon.0) 948 : audit [INF] from='mgr.24634 172.21.15.164:0/803880945' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2023-06-03T19:40:02.284 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:40:01 smithi089 bash[26572]: cluster 2023-06-03T19:40:01.115896+0000 mon.a (mon.0) 949 : cluster [DBG] osdmap e71: 8 total, 8 up, 8 in 2023-06-03T19:40:02.285 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:40:01 smithi089 bash[26572]: audit 2023-06-03T19:40:01.144208+0000 mon.a (mon.0) 950 : audit [DBG] from='client.? 172.21.15.89:0/1837036081' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-03T19:40:02.285 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 19:40:02 smithi089 bash[71163]: [03/Jun/2023:19:40:02] ENGINE Bus STOPPING 2023-06-03T19:40:02.285 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 19:40:02 smithi089 bash[71163]: [03/Jun/2023:19:40:02] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2023-06-03T19:40:02.285 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 19:40:02 smithi089 bash[71163]: [03/Jun/2023:19:40:02] ENGINE Bus STOPPED 2023-06-03T19:40:02.285 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 19:40:02 smithi089 bash[71163]: [03/Jun/2023:19:40:02] ENGINE Bus STARTING 2023-06-03T19:40:02.286 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:40:01 smithi089 bash[33173]: audit 2023-06-03T19:40:00.966606+0000 mon.a (mon.0) 942 : audit [INF] from='mgr.24634 172.21.15.164:0/803880945' entity='mgr.x' 2023-06-03T19:40:02.286 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:40:01 smithi089 bash[33173]: audit 2023-06-03T19:40:00.984573+0000 mon.a (mon.0) 943 : audit [INF] from='mgr.24634 172.21.15.164:0/803880945' entity='mgr.x' 2023-06-03T19:40:02.286 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:40:01 smithi089 bash[33173]: audit 2023-06-03T19:40:00.986294+0000 mon.a (mon.0) 944 : audit [DBG] from='mgr.24634 172.21.15.164:0/803880945' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-03T19:40:02.286 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:40:01 smithi089 bash[33173]: audit 2023-06-03T19:40:00.987903+0000 mon.a (mon.0) 945 : audit [INF] from='mgr.24634 172.21.15.164:0/803880945' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-03T19:40:02.286 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:40:01 smithi089 bash[33173]: audit 2023-06-03T19:40:01.001563+0000 mon.a (mon.0) 946 : audit [INF] from='mgr.24634 172.21.15.164:0/803880945' entity='mgr.x' 2023-06-03T19:40:02.286 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:40:01 smithi089 bash[33173]: audit 2023-06-03T19:40:01.086525+0000 mon.a (mon.0) 947 : audit [DBG] from='mgr.24634 172.21.15.164:0/803880945' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-03T19:40:02.286 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:40:01 smithi089 bash[33173]: cephadm 2023-06-03T19:40:01.095102+0000 mgr.x (mgr.24634) 103 : cephadm [INF] Upgrade: Need to upgrade myself (mgr.x) 2023-06-03T19:40:02.287 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:40:01 smithi089 bash[33173]: cephadm 2023-06-03T19:40:01.095862+0000 mgr.x (mgr.24634) 104 : cephadm [INF] Upgrade: Need to upgrade myself (mgr.x) 2023-06-03T19:40:02.287 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:40:01 smithi089 bash[33173]: cephadm 2023-06-03T19:40:01.100557+0000 mgr.x (mgr.24634) 105 : cephadm [INF] Failing over to other MGR 2023-06-03T19:40:02.287 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:40:01 smithi089 bash[33173]: audit 2023-06-03T19:40:01.102854+0000 mon.a (mon.0) 948 : audit [INF] from='mgr.24634 172.21.15.164:0/803880945' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2023-06-03T19:40:02.287 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:40:01 smithi089 bash[33173]: cluster 2023-06-03T19:40:01.115896+0000 mon.a (mon.0) 949 : cluster [DBG] osdmap e71: 8 total, 8 up, 8 in 2023-06-03T19:40:02.287 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:40:01 smithi089 bash[33173]: audit 2023-06-03T19:40:01.144208+0000 mon.a (mon.0) 950 : audit [DBG] from='client.? 172.21.15.89:0/1837036081' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-03T19:40:02.326 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:40:01 smithi164 bash[26855]: audit 2023-06-03T19:40:00.966606+0000 mon.a (mon.0) 942 : audit [INF] from='mgr.24634 172.21.15.164:0/803880945' entity='mgr.x' 2023-06-03T19:40:02.326 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:40:01 smithi164 bash[26855]: audit 2023-06-03T19:40:00.984573+0000 mon.a (mon.0) 943 : audit [INF] from='mgr.24634 172.21.15.164:0/803880945' entity='mgr.x' 2023-06-03T19:40:02.326 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:40:01 smithi164 bash[26855]: audit 2023-06-03T19:40:00.986294+0000 mon.a (mon.0) 944 : audit [DBG] from='mgr.24634 172.21.15.164:0/803880945' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-03T19:40:02.327 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:40:01 smithi164 bash[26855]: audit 2023-06-03T19:40:00.987903+0000 mon.a (mon.0) 945 : audit [INF] from='mgr.24634 172.21.15.164:0/803880945' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-03T19:40:02.327 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:40:01 smithi164 bash[26855]: audit 2023-06-03T19:40:01.001563+0000 mon.a (mon.0) 946 : audit [INF] from='mgr.24634 172.21.15.164:0/803880945' entity='mgr.x' 2023-06-03T19:40:02.327 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:40:01 smithi164 bash[26855]: audit 2023-06-03T19:40:01.086525+0000 mon.a (mon.0) 947 : audit [DBG] from='mgr.24634 172.21.15.164:0/803880945' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-03T19:40:02.327 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:40:01 smithi164 bash[26855]: cephadm 2023-06-03T19:40:01.095102+0000 mgr.x (mgr.24634) 103 : cephadm [INF] Upgrade: Need to upgrade myself (mgr.x) 2023-06-03T19:40:02.327 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:40:01 smithi164 bash[26855]: cephadm 2023-06-03T19:40:01.095862+0000 mgr.x (mgr.24634) 104 : cephadm [INF] Upgrade: Need to upgrade myself (mgr.x) 2023-06-03T19:40:02.327 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:40:01 smithi164 bash[26855]: cephadm 2023-06-03T19:40:01.100557+0000 mgr.x (mgr.24634) 105 : cephadm [INF] Failing over to other MGR 2023-06-03T19:40:02.327 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:40:01 smithi164 bash[26855]: audit 2023-06-03T19:40:01.102854+0000 mon.a (mon.0) 948 : audit [INF] from='mgr.24634 172.21.15.164:0/803880945' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2023-06-03T19:40:02.328 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:40:01 smithi164 bash[26855]: cluster 2023-06-03T19:40:01.115896+0000 mon.a (mon.0) 949 : cluster [DBG] osdmap e71: 8 total, 8 up, 8 in 2023-06-03T19:40:02.328 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:40:01 smithi164 bash[26855]: audit 2023-06-03T19:40:01.144208+0000 mon.a (mon.0) 950 : audit [DBG] from='client.? 172.21.15.89:0/1837036081' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-03T19:40:02.328 INFO:journalctl@ceph.mgr.x.smithi164.stdout:Jun 03 19:40:02 smithi164 bash[50883]: debug 2023-06-03T19:40:02.001+0000 7faa3b43d700 -1 mgr handle_mgr_map I was active but no longer am 2023-06-03T19:40:02.328 INFO:journalctl@ceph.mgr.x.smithi164.stdout:Jun 03 19:40:02 smithi164 bash[50883]: ignoring --setuser ceph since I am not root 2023-06-03T19:40:02.328 INFO:journalctl@ceph.mgr.x.smithi164.stdout:Jun 03 19:40:02 smithi164 bash[50883]: ignoring --setgroup ceph since I am not root 2023-06-03T19:40:02.328 INFO:journalctl@ceph.mgr.x.smithi164.stdout:Jun 03 19:40:02 smithi164 bash[50883]: debug 2023-06-03T19:40:02.313+0000 7fb84046a200 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2023-06-03T19:40:02.782 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 19:40:02 smithi089 bash[71163]: debug 2023-06-03T19:40:02.372+0000 7ff8fcc7d700 -1 client.0 error registering admin socket command: (17) File exists 2023-06-03T19:40:02.782 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 19:40:02 smithi089 bash[71163]: debug 2023-06-03T19:40:02.372+0000 7ff8fcc7d700 -1 client.0 error registering admin socket command: (17) File exists 2023-06-03T19:40:02.782 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 19:40:02 smithi089 bash[71163]: debug 2023-06-03T19:40:02.372+0000 7ff8fcc7d700 -1 client.0 error registering admin socket command: (17) File exists 2023-06-03T19:40:02.782 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 19:40:02 smithi089 bash[71163]: debug 2023-06-03T19:40:02.372+0000 7ff8fcc7d700 -1 client.0 error registering admin socket command: (17) File exists 2023-06-03T19:40:02.782 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 19:40:02 smithi089 bash[71163]: debug 2023-06-03T19:40:02.372+0000 7ff8fcc7d700 -1 client.0 error registering admin socket command: (17) File exists 2023-06-03T19:40:02.783 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 19:40:02 smithi089 bash[71163]: debug 2023-06-03T19:40:02.384+0000 7ff8f5c6f700 -1 client.0 error registering admin socket command: (17) File exists 2023-06-03T19:40:02.783 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 19:40:02 smithi089 bash[71163]: debug 2023-06-03T19:40:02.384+0000 7ff8f5c6f700 -1 client.0 error registering admin socket command: (17) File exists 2023-06-03T19:40:02.783 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 19:40:02 smithi089 bash[71163]: debug 2023-06-03T19:40:02.388+0000 7ff8f5c6f700 -1 client.0 error registering admin socket command: (17) File exists 2023-06-03T19:40:02.783 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 19:40:02 smithi089 bash[71163]: debug 2023-06-03T19:40:02.388+0000 7ff8f5c6f700 -1 client.0 error registering admin socket command: (17) File exists 2023-06-03T19:40:02.783 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 19:40:02 smithi089 bash[71163]: debug 2023-06-03T19:40:02.388+0000 7ff8f5c6f700 -1 client.0 error registering admin socket command: (17) File exists 2023-06-03T19:40:02.783 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 19:40:02 smithi089 bash[71163]: [03/Jun/2023:19:40:02] ENGINE Serving on http://:::9283 2023-06-03T19:40:02.783 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 19:40:02 smithi089 bash[71163]: [03/Jun/2023:19:40:02] ENGINE Bus STARTED 2023-06-03T19:40:03.282 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:40:03 smithi089 bash[26572]: audit 2023-06-03T19:40:02.006300+0000 mon.a (mon.0) 951 : audit [INF] from='mgr.24634 172.21.15.164:0/803880945' entity='mgr.x' cmd='[{"prefix": "mgr fail", "who": "x"}]': finished 2023-06-03T19:40:03.283 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:40:03 smithi089 bash[26572]: cluster 2023-06-03T19:40:02.006375+0000 mon.a (mon.0) 952 : cluster [DBG] mgrmap e27: y(active, starting, since 0.902534s) 2023-06-03T19:40:03.283 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:40:03 smithi089 bash[26572]: audit 2023-06-03T19:40:02.015456+0000 mon.b (mon.1) 36 : audit [DBG] from='mgr.25070 172.21.15.89:0/2108989343' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-06-03T19:40:03.283 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:40:03 smithi089 bash[26572]: audit 2023-06-03T19:40:02.016040+0000 mon.b (mon.1) 37 : audit [DBG] from='mgr.25070 172.21.15.89:0/2108989343' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-06-03T19:40:03.284 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:40:03 smithi089 bash[26572]: audit 2023-06-03T19:40:02.016693+0000 mon.b (mon.1) 38 : audit [DBG] from='mgr.25070 172.21.15.89:0/2108989343' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-06-03T19:40:03.284 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:40:03 smithi089 bash[26572]: audit 2023-06-03T19:40:02.017366+0000 mon.b (mon.1) 39 : audit [DBG] from='mgr.25070 172.21.15.89:0/2108989343' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi089.giuxuj"}]: dispatch 2023-06-03T19:40:03.284 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:40:03 smithi089 bash[26572]: audit 2023-06-03T19:40:02.018260+0000 mon.b (mon.1) 40 : audit [DBG] from='mgr.25070 172.21.15.89:0/2108989343' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi164.txgfgv"}]: dispatch 2023-06-03T19:40:03.285 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:40:03 smithi089 bash[26572]: audit 2023-06-03T19:40:02.018820+0000 mon.b (mon.1) 41 : audit [DBG] from='mgr.25070 172.21.15.89:0/2108989343' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2023-06-03T19:40:03.285 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:40:03 smithi089 bash[26572]: audit 2023-06-03T19:40:02.019079+0000 mon.b (mon.1) 42 : audit [DBG] from='mgr.25070 172.21.15.89:0/2108989343' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-06-03T19:40:03.285 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:40:03 smithi089 bash[26572]: audit 2023-06-03T19:40:02.019328+0000 mon.b (mon.1) 43 : audit [DBG] from='mgr.25070 172.21.15.89:0/2108989343' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-06-03T19:40:03.285 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:40:03 smithi089 bash[26572]: audit 2023-06-03T19:40:02.019690+0000 mon.b (mon.1) 44 : audit [DBG] from='mgr.25070 172.21.15.89:0/2108989343' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-06-03T19:40:03.286 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:40:03 smithi089 bash[26572]: audit 2023-06-03T19:40:02.019932+0000 mon.b (mon.1) 45 : audit [DBG] from='mgr.25070 172.21.15.89:0/2108989343' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-06-03T19:40:03.286 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:40:03 smithi089 bash[26572]: audit 2023-06-03T19:40:02.020146+0000 mon.b (mon.1) 46 : audit [DBG] from='mgr.25070 172.21.15.89:0/2108989343' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-06-03T19:40:03.286 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:40:03 smithi089 bash[26572]: audit 2023-06-03T19:40:02.020371+0000 mon.b (mon.1) 47 : audit [DBG] from='mgr.25070 172.21.15.89:0/2108989343' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-06-03T19:40:03.287 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:40:03 smithi089 bash[26572]: audit 2023-06-03T19:40:02.020595+0000 mon.b (mon.1) 48 : audit [DBG] from='mgr.25070 172.21.15.89:0/2108989343' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-06-03T19:40:03.287 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:40:03 smithi089 bash[26572]: audit 2023-06-03T19:40:02.020829+0000 mon.b (mon.1) 49 : audit [DBG] from='mgr.25070 172.21.15.89:0/2108989343' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-06-03T19:40:03.287 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:40:03 smithi089 bash[26572]: audit 2023-06-03T19:40:02.021073+0000 mon.b (mon.1) 50 : audit [DBG] from='mgr.25070 172.21.15.89:0/2108989343' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2023-06-03T19:40:03.287 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:40:03 smithi089 bash[26572]: audit 2023-06-03T19:40:02.021354+0000 mon.b (mon.1) 51 : audit [DBG] from='mgr.25070 172.21.15.89:0/2108989343' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2023-06-03T19:40:03.288 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:40:03 smithi089 bash[26572]: audit 2023-06-03T19:40:02.022870+0000 mon.b (mon.1) 52 : audit [DBG] from='mgr.25070 172.21.15.89:0/2108989343' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2023-06-03T19:40:03.288 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:40:03 smithi089 bash[26572]: cluster 2023-06-03T19:40:02.068464+0000 mon.a (mon.0) 953 : cluster [INF] Manager daemon y is now available 2023-06-03T19:40:03.288 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:40:03 smithi089 bash[26572]: audit 2023-06-03T19:40:02.120915+0000 mon.b (mon.1) 53 : audit [DBG] from='mgr.25070 172.21.15.89:0/2108989343' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-03T19:40:03.289 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:40:03 smithi089 bash[26572]: audit 2023-06-03T19:40:02.123088+0000 mon.b (mon.1) 54 : audit [DBG] from='mgr.25070 172.21.15.89:0/2108989343' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-03T19:40:03.289 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:40:03 smithi089 bash[26572]: audit 2023-06-03T19:40:02.175308+0000 mon.b (mon.1) 55 : audit [INF] from='mgr.25070 172.21.15.89:0/2108989343' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2023-06-03T19:40:03.289 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:40:03 smithi089 bash[26572]: audit 2023-06-03T19:40:02.177688+0000 mon.a (mon.0) 954 : audit [INF] from='mgr.25070 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2023-06-03T19:40:03.290 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:40:03 smithi089 bash[26572]: audit 2023-06-03T19:40:02.203238+0000 mon.b (mon.1) 56 : audit [DBG] from='mgr.25070 172.21.15.89:0/2108989343' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T19:40:03.290 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:40:03 smithi089 bash[26572]: audit 2023-06-03T19:40:02.337941+0000 mon.b (mon.1) 57 : audit [INF] from='mgr.25070 172.21.15.89:0/2108989343' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2023-06-03T19:40:03.290 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:40:03 smithi089 bash[26572]: audit 2023-06-03T19:40:02.340466+0000 mon.a (mon.0) 955 : audit [INF] from='mgr.25070 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2023-06-03T19:40:03.291 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:40:03 smithi089 bash[33173]: audit 2023-06-03T19:40:02.006300+0000 mon.a (mon.0) 951 : audit [INF] from='mgr.24634 172.21.15.164:0/803880945' entity='mgr.x' cmd='[{"prefix": "mgr fail", "who": "x"}]': finished 2023-06-03T19:40:03.291 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:40:03 smithi089 bash[33173]: cluster 2023-06-03T19:40:02.006375+0000 mon.a (mon.0) 952 : cluster [DBG] mgrmap e27: y(active, starting, since 0.902534s) 2023-06-03T19:40:03.291 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:40:03 smithi089 bash[33173]: audit 2023-06-03T19:40:02.015456+0000 mon.b (mon.1) 36 : audit [DBG] from='mgr.25070 172.21.15.89:0/2108989343' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-06-03T19:40:03.291 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:40:03 smithi089 bash[33173]: audit 2023-06-03T19:40:02.016040+0000 mon.b (mon.1) 37 : audit [DBG] from='mgr.25070 172.21.15.89:0/2108989343' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-06-03T19:40:03.292 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:40:03 smithi089 bash[33173]: audit 2023-06-03T19:40:02.016693+0000 mon.b (mon.1) 38 : audit [DBG] from='mgr.25070 172.21.15.89:0/2108989343' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-06-03T19:40:03.292 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:40:03 smithi089 bash[33173]: audit 2023-06-03T19:40:02.017366+0000 mon.b (mon.1) 39 : audit [DBG] from='mgr.25070 172.21.15.89:0/2108989343' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi089.giuxuj"}]: dispatch 2023-06-03T19:40:03.292 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:40:03 smithi089 bash[33173]: audit 2023-06-03T19:40:02.018260+0000 mon.b (mon.1) 40 : audit [DBG] from='mgr.25070 172.21.15.89:0/2108989343' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi164.txgfgv"}]: dispatch 2023-06-03T19:40:03.292 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:40:03 smithi089 bash[33173]: audit 2023-06-03T19:40:02.018820+0000 mon.b (mon.1) 41 : audit [DBG] from='mgr.25070 172.21.15.89:0/2108989343' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2023-06-03T19:40:03.293 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:40:03 smithi089 bash[33173]: audit 2023-06-03T19:40:02.019079+0000 mon.b (mon.1) 42 : audit [DBG] from='mgr.25070 172.21.15.89:0/2108989343' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-06-03T19:40:03.293 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:40:03 smithi089 bash[33173]: audit 2023-06-03T19:40:02.019328+0000 mon.b (mon.1) 43 : audit [DBG] from='mgr.25070 172.21.15.89:0/2108989343' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-06-03T19:40:03.293 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:40:03 smithi089 bash[33173]: audit 2023-06-03T19:40:02.019690+0000 mon.b (mon.1) 44 : audit [DBG] from='mgr.25070 172.21.15.89:0/2108989343' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-06-03T19:40:03.293 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:40:03 smithi089 bash[33173]: audit 2023-06-03T19:40:02.019932+0000 mon.b (mon.1) 45 : audit [DBG] from='mgr.25070 172.21.15.89:0/2108989343' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-06-03T19:40:03.294 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:40:03 smithi089 bash[33173]: audit 2023-06-03T19:40:02.020146+0000 mon.b (mon.1) 46 : audit [DBG] from='mgr.25070 172.21.15.89:0/2108989343' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-06-03T19:40:03.294 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:40:03 smithi089 bash[33173]: audit 2023-06-03T19:40:02.020371+0000 mon.b (mon.1) 47 : audit [DBG] from='mgr.25070 172.21.15.89:0/2108989343' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-06-03T19:40:03.294 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:40:03 smithi089 bash[33173]: audit 2023-06-03T19:40:02.020595+0000 mon.b (mon.1) 48 : audit [DBG] from='mgr.25070 172.21.15.89:0/2108989343' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-06-03T19:40:03.294 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:40:03 smithi089 bash[33173]: audit 2023-06-03T19:40:02.020829+0000 mon.b (mon.1) 49 : audit [DBG] from='mgr.25070 172.21.15.89:0/2108989343' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-06-03T19:40:03.294 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:40:03 smithi089 bash[33173]: audit 2023-06-03T19:40:02.021073+0000 mon.b (mon.1) 50 : audit [DBG] from='mgr.25070 172.21.15.89:0/2108989343' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2023-06-03T19:40:03.294 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:40:03 smithi089 bash[33173]: audit 2023-06-03T19:40:02.021354+0000 mon.b (mon.1) 51 : audit [DBG] from='mgr.25070 172.21.15.89:0/2108989343' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2023-06-03T19:40:03.295 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:40:03 smithi089 bash[33173]: audit 2023-06-03T19:40:02.022870+0000 mon.b (mon.1) 52 : audit [DBG] from='mgr.25070 172.21.15.89:0/2108989343' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2023-06-03T19:40:03.295 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:40:03 smithi089 bash[33173]: cluster 2023-06-03T19:40:02.068464+0000 mon.a (mon.0) 953 : cluster [INF] Manager daemon y is now available 2023-06-03T19:40:03.295 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:40:03 smithi089 bash[33173]: audit 2023-06-03T19:40:02.120915+0000 mon.b (mon.1) 53 : audit [DBG] from='mgr.25070 172.21.15.89:0/2108989343' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-03T19:40:03.295 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:40:03 smithi089 bash[33173]: audit 2023-06-03T19:40:02.123088+0000 mon.b (mon.1) 54 : audit [DBG] from='mgr.25070 172.21.15.89:0/2108989343' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-03T19:40:03.295 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:40:03 smithi089 bash[33173]: audit 2023-06-03T19:40:02.175308+0000 mon.b (mon.1) 55 : audit [INF] from='mgr.25070 172.21.15.89:0/2108989343' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2023-06-03T19:40:03.295 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:40:03 smithi089 bash[33173]: audit 2023-06-03T19:40:02.177688+0000 mon.a (mon.0) 954 : audit [INF] from='mgr.25070 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2023-06-03T19:40:03.295 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:40:03 smithi089 bash[33173]: audit 2023-06-03T19:40:02.203238+0000 mon.b (mon.1) 56 : audit [DBG] from='mgr.25070 172.21.15.89:0/2108989343' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T19:40:03.296 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:40:03 smithi089 bash[33173]: audit 2023-06-03T19:40:02.337941+0000 mon.b (mon.1) 57 : audit [INF] from='mgr.25070 172.21.15.89:0/2108989343' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2023-06-03T19:40:03.296 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:40:03 smithi089 bash[33173]: audit 2023-06-03T19:40:02.340466+0000 mon.a (mon.0) 955 : audit [INF] from='mgr.25070 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2023-06-03T19:40:03.336 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:40:03 smithi164 bash[26855]: audit 2023-06-03T19:40:02.006300+0000 mon.a (mon.0) 951 : audit [INF] from='mgr.24634 172.21.15.164:0/803880945' entity='mgr.x' cmd='[{"prefix": "mgr fail", "who": "x"}]': finished 2023-06-03T19:40:03.337 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:40:03 smithi164 bash[26855]: cluster 2023-06-03T19:40:02.006375+0000 mon.a (mon.0) 952 : cluster [DBG] mgrmap e27: y(active, starting, since 0.902534s) 2023-06-03T19:40:03.337 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:40:03 smithi164 bash[26855]: audit 2023-06-03T19:40:02.015456+0000 mon.b (mon.1) 36 : audit [DBG] from='mgr.25070 172.21.15.89:0/2108989343' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-06-03T19:40:03.337 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:40:03 smithi164 bash[26855]: audit 2023-06-03T19:40:02.016040+0000 mon.b (mon.1) 37 : audit [DBG] from='mgr.25070 172.21.15.89:0/2108989343' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-06-03T19:40:03.337 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:40:03 smithi164 bash[26855]: audit 2023-06-03T19:40:02.016693+0000 mon.b (mon.1) 38 : audit [DBG] from='mgr.25070 172.21.15.89:0/2108989343' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-06-03T19:40:03.337 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:40:03 smithi164 bash[26855]: audit 2023-06-03T19:40:02.017366+0000 mon.b (mon.1) 39 : audit [DBG] from='mgr.25070 172.21.15.89:0/2108989343' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi089.giuxuj"}]: dispatch 2023-06-03T19:40:03.337 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:40:03 smithi164 bash[26855]: audit 2023-06-03T19:40:02.018260+0000 mon.b (mon.1) 40 : audit [DBG] from='mgr.25070 172.21.15.89:0/2108989343' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi164.txgfgv"}]: dispatch 2023-06-03T19:40:03.338 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:40:03 smithi164 bash[26855]: audit 2023-06-03T19:40:02.018820+0000 mon.b (mon.1) 41 : audit [DBG] from='mgr.25070 172.21.15.89:0/2108989343' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2023-06-03T19:40:03.338 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:40:03 smithi164 bash[26855]: audit 2023-06-03T19:40:02.019079+0000 mon.b (mon.1) 42 : audit [DBG] from='mgr.25070 172.21.15.89:0/2108989343' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-06-03T19:40:03.338 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:40:03 smithi164 bash[26855]: audit 2023-06-03T19:40:02.019328+0000 mon.b (mon.1) 43 : audit [DBG] from='mgr.25070 172.21.15.89:0/2108989343' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-06-03T19:40:03.338 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:40:03 smithi164 bash[26855]: audit 2023-06-03T19:40:02.019690+0000 mon.b (mon.1) 44 : audit [DBG] from='mgr.25070 172.21.15.89:0/2108989343' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-06-03T19:40:03.338 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:40:03 smithi164 bash[26855]: audit 2023-06-03T19:40:02.019932+0000 mon.b (mon.1) 45 : audit [DBG] from='mgr.25070 172.21.15.89:0/2108989343' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-06-03T19:40:03.338 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:40:03 smithi164 bash[26855]: audit 2023-06-03T19:40:02.020146+0000 mon.b (mon.1) 46 : audit [DBG] from='mgr.25070 172.21.15.89:0/2108989343' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-06-03T19:40:03.338 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:40:03 smithi164 bash[26855]: audit 2023-06-03T19:40:02.020371+0000 mon.b (mon.1) 47 : audit [DBG] from='mgr.25070 172.21.15.89:0/2108989343' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-06-03T19:40:03.339 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:40:03 smithi164 bash[26855]: audit 2023-06-03T19:40:02.020595+0000 mon.b (mon.1) 48 : audit [DBG] from='mgr.25070 172.21.15.89:0/2108989343' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-06-03T19:40:03.339 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:40:03 smithi164 bash[26855]: audit 2023-06-03T19:40:02.020829+0000 mon.b (mon.1) 49 : audit [DBG] from='mgr.25070 172.21.15.89:0/2108989343' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-06-03T19:40:03.339 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:40:03 smithi164 bash[26855]: audit 2023-06-03T19:40:02.021073+0000 mon.b (mon.1) 50 : audit [DBG] from='mgr.25070 172.21.15.89:0/2108989343' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2023-06-03T19:40:03.339 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:40:03 smithi164 bash[26855]: audit 2023-06-03T19:40:02.021354+0000 mon.b (mon.1) 51 : audit [DBG] from='mgr.25070 172.21.15.89:0/2108989343' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2023-06-03T19:40:03.339 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:40:03 smithi164 bash[26855]: audit 2023-06-03T19:40:02.022870+0000 mon.b (mon.1) 52 : audit [DBG] from='mgr.25070 172.21.15.89:0/2108989343' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2023-06-03T19:40:03.339 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:40:03 smithi164 bash[26855]: cluster 2023-06-03T19:40:02.068464+0000 mon.a (mon.0) 953 : cluster [INF] Manager daemon y is now available 2023-06-03T19:40:03.339 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:40:03 smithi164 bash[26855]: audit 2023-06-03T19:40:02.120915+0000 mon.b (mon.1) 53 : audit [DBG] from='mgr.25070 172.21.15.89:0/2108989343' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-03T19:40:03.340 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:40:03 smithi164 bash[26855]: audit 2023-06-03T19:40:02.123088+0000 mon.b (mon.1) 54 : audit [DBG] from='mgr.25070 172.21.15.89:0/2108989343' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-03T19:40:03.340 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:40:03 smithi164 bash[26855]: audit 2023-06-03T19:40:02.175308+0000 mon.b (mon.1) 55 : audit [INF] from='mgr.25070 172.21.15.89:0/2108989343' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2023-06-03T19:40:03.340 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:40:03 smithi164 bash[26855]: audit 2023-06-03T19:40:02.177688+0000 mon.a (mon.0) 954 : audit [INF] from='mgr.25070 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2023-06-03T19:40:03.340 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:40:03 smithi164 bash[26855]: audit 2023-06-03T19:40:02.203238+0000 mon.b (mon.1) 56 : audit [DBG] from='mgr.25070 172.21.15.89:0/2108989343' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T19:40:03.340 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:40:03 smithi164 bash[26855]: audit 2023-06-03T19:40:02.337941+0000 mon.b (mon.1) 57 : audit [INF] from='mgr.25070 172.21.15.89:0/2108989343' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2023-06-03T19:40:03.340 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:40:03 smithi164 bash[26855]: audit 2023-06-03T19:40:02.340466+0000 mon.a (mon.0) 955 : audit [INF] from='mgr.25070 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2023-06-03T19:40:03.962 INFO:journalctl@ceph.mgr.x.smithi164.stdout:Jun 03 19:40:03 smithi164 bash[50883]: debug 2023-06-03T19:40:03.637+0000 7fb84046a200 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2023-06-03T19:40:03.963 INFO:journalctl@ceph.mgr.x.smithi164.stdout:Jun 03 19:40:03 smithi164 bash[50883]: debug 2023-06-03T19:40:03.793+0000 7fb84046a200 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2023-06-03T19:40:03.963 INFO:journalctl@ceph.mgr.x.smithi164.stdout:Jun 03 19:40:03 smithi164 bash[50883]: debug 2023-06-03T19:40:03.949+0000 7fb84046a200 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2023-06-03T19:40:04.336 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:40:04 smithi164 bash[26855]: cluster 2023-06-03T19:40:03.032211+0000 mon.a (mon.0) 956 : cluster [DBG] mgrmap e28: y(active, since 1.92837s) 2023-06-03T19:40:04.337 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:40:04 smithi164 bash[26855]: cluster 2023-06-03T19:40:03.131022+0000 mgr.y (mgr.25070) 1 : cluster [DBG] pgmap v3: 89 pgs: 89 active+clean; 2.6 GiB data, 8.4 GiB used, 707 GiB / 715 GiB avail 2023-06-03T19:40:04.337 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:40:04 smithi164 bash[26855]: audit 2023-06-03T19:40:03.822180+0000 mon.b (mon.1) 58 : audit [DBG] from='client.? 172.21.15.89:0/331036908' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-03T19:40:04.531 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:40:04 smithi089 bash[33173]: cluster 2023-06-03T19:40:03.032211+0000 mon.a (mon.0) 956 : cluster [DBG] mgrmap e28: y(active, since 1.92837s) 2023-06-03T19:40:04.531 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:40:04 smithi089 bash[33173]: cluster 2023-06-03T19:40:03.131022+0000 mgr.y (mgr.25070) 1 : cluster [DBG] pgmap v3: 89 pgs: 89 active+clean; 2.6 GiB data, 8.4 GiB used, 707 GiB / 715 GiB avail 2023-06-03T19:40:04.532 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:40:04 smithi089 bash[33173]: audit 2023-06-03T19:40:03.822180+0000 mon.b (mon.1) 58 : audit [DBG] from='client.? 172.21.15.89:0/331036908' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-03T19:40:04.532 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:40:04 smithi089 bash[26572]: cluster 2023-06-03T19:40:03.032211+0000 mon.a (mon.0) 956 : cluster [DBG] mgrmap e28: y(active, since 1.92837s) 2023-06-03T19:40:04.532 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:40:04 smithi089 bash[26572]: cluster 2023-06-03T19:40:03.131022+0000 mgr.y (mgr.25070) 1 : cluster [DBG] pgmap v3: 89 pgs: 89 active+clean; 2.6 GiB data, 8.4 GiB used, 707 GiB / 715 GiB avail 2023-06-03T19:40:04.532 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:40:04 smithi089 bash[26572]: audit 2023-06-03T19:40:03.822180+0000 mon.b (mon.1) 58 : audit [DBG] from='client.? 172.21.15.89:0/331036908' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-03T19:40:04.836 INFO:journalctl@ceph.mgr.x.smithi164.stdout:Jun 03 19:40:04 smithi164 bash[50883]: debug 2023-06-03T19:40:04.341+0000 7fb84046a200 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2023-06-03T19:40:05.336 INFO:journalctl@ceph.mgr.x.smithi164.stdout:Jun 03 19:40:04 smithi164 bash[50883]: debug 2023-06-03T19:40:04.885+0000 7fb84046a200 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2023-06-03T19:40:05.336 INFO:journalctl@ceph.mgr.x.smithi164.stdout:Jun 03 19:40:05 smithi164 bash[50883]: debug 2023-06-03T19:40:05.125+0000 7fb84046a200 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2023-06-03T19:40:05.337 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:40:05 smithi164 bash[26855]: cluster 2023-06-03T19:40:04.046777+0000 mon.a (mon.0) 957 : cluster [DBG] mgrmap e29: y(active, since 2s) 2023-06-03T19:40:05.337 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:40:05 smithi164 bash[26855]: cephadm 2023-06-03T19:40:04.423040+0000 mgr.y (mgr.25070) 3 : cephadm [INF] [03/Jun/2023:19:40:04] ENGINE Bus STARTING 2023-06-03T19:40:05.337 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:40:05 smithi164 bash[26855]: cephadm 2023-06-03T19:40:04.545548+0000 mgr.y (mgr.25070) 4 : cephadm [INF] [03/Jun/2023:19:40:04] ENGINE Serving on https://172.21.15.89:7150 2023-06-03T19:40:05.337 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:40:05 smithi164 bash[26855]: cephadm 2023-06-03T19:40:04.647136+0000 mgr.y (mgr.25070) 5 : cephadm [INF] [03/Jun/2023:19:40:04] ENGINE Serving on http://172.21.15.89:8765 2023-06-03T19:40:05.337 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:40:05 smithi164 bash[26855]: cephadm 2023-06-03T19:40:04.647282+0000 mgr.y (mgr.25070) 6 : cephadm [INF] [03/Jun/2023:19:40:04] ENGINE Bus STARTED 2023-06-03T19:40:05.531 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:40:05 smithi089 bash[33173]: cluster 2023-06-03T19:40:04.046777+0000 mon.a (mon.0) 957 : cluster [DBG] mgrmap e29: y(active, since 2s) 2023-06-03T19:40:05.532 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:40:05 smithi089 bash[33173]: cephadm 2023-06-03T19:40:04.423040+0000 mgr.y (mgr.25070) 3 : cephadm [INF] [03/Jun/2023:19:40:04] ENGINE Bus STARTING 2023-06-03T19:40:05.532 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:40:05 smithi089 bash[33173]: cephadm 2023-06-03T19:40:04.545548+0000 mgr.y (mgr.25070) 4 : cephadm [INF] [03/Jun/2023:19:40:04] ENGINE Serving on https://172.21.15.89:7150 2023-06-03T19:40:05.532 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:40:05 smithi089 bash[33173]: cephadm 2023-06-03T19:40:04.647136+0000 mgr.y (mgr.25070) 5 : cephadm [INF] [03/Jun/2023:19:40:04] ENGINE Serving on http://172.21.15.89:8765 2023-06-03T19:40:05.532 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:40:05 smithi089 bash[33173]: cephadm 2023-06-03T19:40:04.647282+0000 mgr.y (mgr.25070) 6 : cephadm [INF] [03/Jun/2023:19:40:04] ENGINE Bus STARTED 2023-06-03T19:40:05.532 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:40:05 smithi089 bash[26572]: cluster 2023-06-03T19:40:04.046777+0000 mon.a (mon.0) 957 : cluster [DBG] mgrmap e29: y(active, since 2s) 2023-06-03T19:40:05.532 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:40:05 smithi089 bash[26572]: cephadm 2023-06-03T19:40:04.423040+0000 mgr.y (mgr.25070) 3 : cephadm [INF] [03/Jun/2023:19:40:04] ENGINE Bus STARTING 2023-06-03T19:40:05.533 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:40:05 smithi089 bash[26572]: cephadm 2023-06-03T19:40:04.545548+0000 mgr.y (mgr.25070) 4 : cephadm [INF] [03/Jun/2023:19:40:04] ENGINE Serving on https://172.21.15.89:7150 2023-06-03T19:40:05.533 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:40:05 smithi089 bash[26572]: cephadm 2023-06-03T19:40:04.647136+0000 mgr.y (mgr.25070) 5 : cephadm [INF] [03/Jun/2023:19:40:04] ENGINE Serving on http://172.21.15.89:8765 2023-06-03T19:40:05.533 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:40:05 smithi089 bash[26572]: cephadm 2023-06-03T19:40:04.647282+0000 mgr.y (mgr.25070) 6 : cephadm [INF] [03/Jun/2023:19:40:04] ENGINE Bus STARTED 2023-06-03T19:40:05.836 INFO:journalctl@ceph.mgr.x.smithi164.stdout:Jun 03 19:40:05 smithi164 bash[50883]: debug 2023-06-03T19:40:05.457+0000 7fb84046a200 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2023-06-03T19:40:07.280 INFO:journalctl@ceph.mgr.x.smithi164.stdout:Jun 03 19:40:06 smithi164 bash[50883]: debug 2023-06-03T19:40:06.917+0000 7fb84046a200 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2023-06-03T19:40:07.280 INFO:journalctl@ceph.mgr.x.smithi164.stdout:Jun 03 19:40:07 smithi164 bash[50883]: debug 2023-06-03T19:40:07.081+0000 7fb84046a200 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2023-06-03T19:40:07.280 INFO:journalctl@ceph.mgr.x.smithi164.stdout:Jun 03 19:40:07 smithi164 bash[50883]: debug 2023-06-03T19:40:07.269+0000 7fb84046a200 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2023-06-03T19:40:07.280 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:40:07 smithi164 bash[26855]: cluster 2023-06-03T19:40:06.020811+0000 mgr.y (mgr.25070) 7 : cluster [DBG] pgmap v5: 89 pgs: 89 active+clean; 2.6 GiB data, 8.4 GiB used, 707 GiB / 715 GiB avail 2023-06-03T19:40:07.281 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:40:07 smithi164 bash[26855]: audit 2023-06-03T19:40:06.463433+0000 mon.a (mon.0) 958 : audit [DBG] from='client.? 172.21.15.89:0/692291036' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-03T19:40:07.532 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:40:07 smithi089 bash[26572]: cluster 2023-06-03T19:40:06.020811+0000 mgr.y (mgr.25070) 7 : cluster [DBG] pgmap v5: 89 pgs: 89 active+clean; 2.6 GiB data, 8.4 GiB used, 707 GiB / 715 GiB avail 2023-06-03T19:40:07.532 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:40:07 smithi089 bash[26572]: audit 2023-06-03T19:40:06.463433+0000 mon.a (mon.0) 958 : audit [DBG] from='client.? 172.21.15.89:0/692291036' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-03T19:40:07.532 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:40:07 smithi089 bash[33173]: cluster 2023-06-03T19:40:06.020811+0000 mgr.y (mgr.25070) 7 : cluster [DBG] pgmap v5: 89 pgs: 89 active+clean; 2.6 GiB data, 8.4 GiB used, 707 GiB / 715 GiB avail 2023-06-03T19:40:07.532 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:40:07 smithi089 bash[33173]: audit 2023-06-03T19:40:06.463433+0000 mon.a (mon.0) 958 : audit [DBG] from='client.? 172.21.15.89:0/692291036' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-03T19:40:08.336 INFO:journalctl@ceph.mgr.x.smithi164.stdout:Jun 03 19:40:08 smithi164 bash[50883]: debug 2023-06-03T19:40:08.005+0000 7fb84046a200 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2023-06-03T19:40:08.838 INFO:journalctl@ceph.mgr.x.smithi164.stdout:Jun 03 19:40:08 smithi164 bash[50883]: debug 2023-06-03T19:40:08.357+0000 7fb84046a200 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2023-06-03T19:40:09.336 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:40:09 smithi164 bash[26855]: cluster 2023-06-03T19:40:08.021232+0000 mgr.y (mgr.25070) 8 : cluster [DBG] pgmap v6: 89 pgs: 89 active+clean; 2.6 GiB data, 8.4 GiB used, 707 GiB / 715 GiB avail 2023-06-03T19:40:09.337 INFO:journalctl@ceph.mgr.x.smithi164.stdout:Jun 03 19:40:08 smithi164 bash[50883]: debug 2023-06-03T19:40:08.945+0000 7fb84046a200 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2023-06-03T19:40:09.531 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:40:09 smithi089 bash[26572]: cluster 2023-06-03T19:40:08.021232+0000 mgr.y (mgr.25070) 8 : cluster [DBG] pgmap v6: 89 pgs: 89 active+clean; 2.6 GiB data, 8.4 GiB used, 707 GiB / 715 GiB avail 2023-06-03T19:40:09.532 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:40:09 smithi089 bash[33173]: cluster 2023-06-03T19:40:08.021232+0000 mgr.y (mgr.25070) 8 : cluster [DBG] pgmap v6: 89 pgs: 89 active+clean; 2.6 GiB data, 8.4 GiB used, 707 GiB / 715 GiB avail 2023-06-03T19:40:10.064 INFO:journalctl@ceph.mgr.x.smithi164.stdout:Jun 03 19:40:09 smithi164 bash[50883]: debug 2023-06-03T19:40:09.701+0000 7fb84046a200 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2023-06-03T19:40:10.336 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:40:10 smithi164 bash[26855]: audit 2023-06-03T19:40:09.141313+0000 mon.c (mon.2) 60 : audit [DBG] from='client.? 172.21.15.89:0/3436739850' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-03T19:40:10.336 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:40:10 smithi164 bash[26855]: audit 2023-06-03T19:40:09.326415+0000 mon.a (mon.0) 959 : audit [INF] from='mgr.25070 ' entity='mgr.y' 2023-06-03T19:40:10.337 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:40:10 smithi164 bash[26855]: audit 2023-06-03T19:40:09.338818+0000 mon.a (mon.0) 960 : audit [INF] from='mgr.25070 ' entity='mgr.y' 2023-06-03T19:40:10.337 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:40:10 smithi164 bash[26855]: audit 2023-06-03T19:40:09.692658+0000 mon.a (mon.0) 961 : audit [INF] from='mgr.25070 ' entity='mgr.y' 2023-06-03T19:40:10.337 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:40:10 smithi164 bash[26855]: audit 2023-06-03T19:40:09.706882+0000 mon.a (mon.0) 962 : audit [INF] from='mgr.25070 ' entity='mgr.y' 2023-06-03T19:40:10.531 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:40:10 smithi089 bash[26572]: audit 2023-06-03T19:40:09.141313+0000 mon.c (mon.2) 60 : audit [DBG] from='client.? 172.21.15.89:0/3436739850' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-03T19:40:10.532 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:40:10 smithi089 bash[26572]: audit 2023-06-03T19:40:09.326415+0000 mon.a (mon.0) 959 : audit [INF] from='mgr.25070 ' entity='mgr.y' 2023-06-03T19:40:10.532 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:40:10 smithi089 bash[26572]: audit 2023-06-03T19:40:09.338818+0000 mon.a (mon.0) 960 : audit [INF] from='mgr.25070 ' entity='mgr.y' 2023-06-03T19:40:10.532 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:40:10 smithi089 bash[26572]: audit 2023-06-03T19:40:09.692658+0000 mon.a (mon.0) 961 : audit [INF] from='mgr.25070 ' entity='mgr.y' 2023-06-03T19:40:10.532 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:40:10 smithi089 bash[26572]: audit 2023-06-03T19:40:09.706882+0000 mon.a (mon.0) 962 : audit [INF] from='mgr.25070 ' entity='mgr.y' 2023-06-03T19:40:10.532 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:40:10 smithi089 bash[33173]: audit 2023-06-03T19:40:09.141313+0000 mon.c (mon.2) 60 : audit [DBG] from='client.? 172.21.15.89:0/3436739850' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-03T19:40:10.533 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:40:10 smithi089 bash[33173]: audit 2023-06-03T19:40:09.326415+0000 mon.a (mon.0) 959 : audit [INF] from='mgr.25070 ' entity='mgr.y' 2023-06-03T19:40:10.533 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:40:10 smithi089 bash[33173]: audit 2023-06-03T19:40:09.338818+0000 mon.a (mon.0) 960 : audit [INF] from='mgr.25070 ' entity='mgr.y' 2023-06-03T19:40:10.533 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:40:10 smithi089 bash[33173]: audit 2023-06-03T19:40:09.692658+0000 mon.a (mon.0) 961 : audit [INF] from='mgr.25070 ' entity='mgr.y' 2023-06-03T19:40:10.533 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:40:10 smithi089 bash[33173]: audit 2023-06-03T19:40:09.706882+0000 mon.a (mon.0) 962 : audit [INF] from='mgr.25070 ' entity='mgr.y' 2023-06-03T19:40:10.836 INFO:journalctl@ceph.mgr.x.smithi164.stdout:Jun 03 19:40:10 smithi164 bash[50883]: debug 2023-06-03T19:40:10.501+0000 7fb84046a200 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2023-06-03T19:40:10.836 INFO:journalctl@ceph.mgr.x.smithi164.stdout:Jun 03 19:40:10 smithi164 bash[50883]: debug 2023-06-03T19:40:10.689+0000 7fb84046a200 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2023-06-03T19:40:11.559 INFO:journalctl@ceph.mgr.x.smithi164.stdout:Jun 03 19:40:11 smithi164 bash[50883]: debug 2023-06-03T19:40:11.185+0000 7fb84046a200 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2023-06-03T19:40:11.560 INFO:journalctl@ceph.mgr.x.smithi164.stdout:Jun 03 19:40:11 smithi164 bash[50883]: debug 2023-06-03T19:40:11.345+0000 7fb84046a200 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2023-06-03T19:40:11.560 INFO:journalctl@ceph.mgr.x.smithi164.stdout:Jun 03 19:40:11 smithi164 bash[50883]: debug 2023-06-03T19:40:11.549+0000 7fb84046a200 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2023-06-03T19:40:11.560 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:40:11 smithi164 bash[26855]: cluster 2023-06-03T19:40:10.021962+0000 mgr.y (mgr.25070) 9 : cluster [DBG] pgmap v7: 89 pgs: 89 active+clean; 2.4 GiB data, 7.9 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 461 op/s 2023-06-03T19:40:11.560 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:40:11 smithi164 bash[26855]: audit 2023-06-03T19:40:10.345325+0000 mon.a (mon.0) 963 : audit [INF] from='mgr.25070 ' entity='mgr.y' 2023-06-03T19:40:11.561 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:40:11 smithi164 bash[26855]: audit 2023-06-03T19:40:10.357125+0000 mon.a (mon.0) 964 : audit [INF] from='mgr.25070 ' entity='mgr.y' 2023-06-03T19:40:11.561 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:40:11 smithi164 bash[26855]: audit 2023-06-03T19:40:10.722022+0000 mon.a (mon.0) 965 : audit [INF] from='mgr.25070 ' entity='mgr.y' 2023-06-03T19:40:11.561 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:40:11 smithi164 bash[26855]: audit 2023-06-03T19:40:10.739391+0000 mon.a (mon.0) 966 : audit [INF] from='mgr.25070 ' entity='mgr.y' 2023-06-03T19:40:11.781 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:40:11 smithi089 bash[26572]: cluster 2023-06-03T19:40:10.021962+0000 mgr.y (mgr.25070) 9 : cluster [DBG] pgmap v7: 89 pgs: 89 active+clean; 2.4 GiB data, 7.9 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 461 op/s 2023-06-03T19:40:11.781 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:40:11 smithi089 bash[26572]: audit 2023-06-03T19:40:10.345325+0000 mon.a (mon.0) 963 : audit [INF] from='mgr.25070 ' entity='mgr.y' 2023-06-03T19:40:11.782 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:40:11 smithi089 bash[26572]: audit 2023-06-03T19:40:10.357125+0000 mon.a (mon.0) 964 : audit [INF] from='mgr.25070 ' entity='mgr.y' 2023-06-03T19:40:11.782 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:40:11 smithi089 bash[26572]: audit 2023-06-03T19:40:10.722022+0000 mon.a (mon.0) 965 : audit [INF] from='mgr.25070 ' entity='mgr.y' 2023-06-03T19:40:11.782 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:40:11 smithi089 bash[26572]: audit 2023-06-03T19:40:10.739391+0000 mon.a (mon.0) 966 : audit [INF] from='mgr.25070 ' entity='mgr.y' 2023-06-03T19:40:11.782 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:40:11 smithi089 bash[33173]: cluster 2023-06-03T19:40:10.021962+0000 mgr.y (mgr.25070) 9 : cluster [DBG] pgmap v7: 89 pgs: 89 active+clean; 2.4 GiB data, 7.9 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 461 op/s 2023-06-03T19:40:11.782 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:40:11 smithi089 bash[33173]: audit 2023-06-03T19:40:10.345325+0000 mon.a (mon.0) 963 : audit [INF] from='mgr.25070 ' entity='mgr.y' 2023-06-03T19:40:11.782 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:40:11 smithi089 bash[33173]: audit 2023-06-03T19:40:10.357125+0000 mon.a (mon.0) 964 : audit [INF] from='mgr.25070 ' entity='mgr.y' 2023-06-03T19:40:11.783 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:40:11 smithi089 bash[33173]: audit 2023-06-03T19:40:10.722022+0000 mon.a (mon.0) 965 : audit [INF] from='mgr.25070 ' entity='mgr.y' 2023-06-03T19:40:11.783 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:40:11 smithi089 bash[33173]: audit 2023-06-03T19:40:10.739391+0000 mon.a (mon.0) 966 : audit [INF] from='mgr.25070 ' entity='mgr.y' 2023-06-03T19:40:11.836 INFO:journalctl@ceph.mgr.x.smithi164.stdout:Jun 03 19:40:11 smithi164 bash[50883]: debug 2023-06-03T19:40:11.717+0000 7fb84046a200 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2023-06-03T19:40:12.203 INFO:journalctl@ceph.mgr.x.smithi164.stdout:Jun 03 19:40:11 smithi164 bash[50883]: debug 2023-06-03T19:40:11.901+0000 7fb84046a200 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2023-06-03T19:40:12.203 INFO:journalctl@ceph.mgr.x.smithi164.stdout:Jun 03 19:40:12 smithi164 bash[50883]: debug 2023-06-03T19:40:12.189+0000 7fb84046a200 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2023-06-03T19:40:12.586 INFO:journalctl@ceph.mgr.x.smithi164.stdout:Jun 03 19:40:12 smithi164 bash[50883]: debug 2023-06-03T19:40:12.377+0000 7fb84046a200 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2023-06-03T19:40:13.031 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:40:12 smithi089 bash[26572]: audit 2023-06-03T19:40:11.804932+0000 mon.a (mon.0) 967 : audit [DBG] from='client.? 172.21.15.89:0/2160543655' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-03T19:40:13.032 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:40:12 smithi089 bash[33173]: audit 2023-06-03T19:40:11.804932+0000 mon.a (mon.0) 967 : audit [DBG] from='client.? 172.21.15.89:0/2160543655' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-03T19:40:13.086 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:40:12 smithi164 bash[26855]: audit 2023-06-03T19:40:11.804932+0000 mon.a (mon.0) 967 : audit [DBG] from='client.? 172.21.15.89:0/2160543655' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-03T19:40:14.031 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:40:13 smithi089 bash[26572]: cluster 2023-06-03T19:40:12.022447+0000 mgr.y (mgr.25070) 10 : cluster [DBG] pgmap v8: 89 pgs: 89 active+clean; 2.4 GiB data, 7.9 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 357 op/s 2023-06-03T19:40:14.032 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:40:13 smithi089 bash[33173]: cluster 2023-06-03T19:40:12.022447+0000 mgr.y (mgr.25070) 10 : cluster [DBG] pgmap v8: 89 pgs: 89 active+clean; 2.4 GiB data, 7.9 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 357 op/s 2023-06-03T19:40:14.086 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:40:13 smithi164 bash[26855]: cluster 2023-06-03T19:40:12.022447+0000 mgr.y (mgr.25070) 10 : cluster [DBG] pgmap v8: 89 pgs: 89 active+clean; 2.4 GiB data, 7.9 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 357 op/s 2023-06-03T19:40:14.586 INFO:journalctl@ceph.mgr.x.smithi164.stdout:Jun 03 19:40:14 smithi164 bash[50883]: debug 2023-06-03T19:40:14.161+0000 7fb84046a200 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2023-06-03T19:40:14.586 INFO:journalctl@ceph.mgr.x.smithi164.stdout:Jun 03 19:40:14 smithi164 bash[50883]: [03/Jun/2023:19:40:14] ENGINE Bus STARTING 2023-06-03T19:40:14.586 INFO:journalctl@ceph.mgr.x.smithi164.stdout:Jun 03 19:40:14 smithi164 bash[50883]: CherryPy Checker: 2023-06-03T19:40:14.587 INFO:journalctl@ceph.mgr.x.smithi164.stdout:Jun 03 19:40:14 smithi164 bash[50883]: The Application mounted at '' has an empty config. 2023-06-03T19:40:14.587 INFO:journalctl@ceph.mgr.x.smithi164.stdout:Jun 03 19:40:14 smithi164 bash[50883]: [03/Jun/2023:19:40:14] ENGINE Serving on http://:::9283 2023-06-03T19:40:14.587 INFO:journalctl@ceph.mgr.x.smithi164.stdout:Jun 03 19:40:14 smithi164 bash[50883]: [03/Jun/2023:19:40:14] ENGINE Bus STARTED 2023-06-03T19:40:15.031 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:40:14 smithi089 bash[26572]: cluster 2023-06-03T19:40:14.190908+0000 mon.a (mon.0) 968 : cluster [DBG] Standby manager daemon x started 2023-06-03T19:40:15.031 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:40:14 smithi089 bash[26572]: audit 2023-06-03T19:40:14.195802+0000 mon.c (mon.2) 61 : audit [DBG] from='mgr.? 172.21.15.164:0/3419036054' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-06-03T19:40:15.032 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:40:14 smithi089 bash[26572]: audit 2023-06-03T19:40:14.198457+0000 mon.c (mon.2) 62 : audit [DBG] from='mgr.? 172.21.15.164:0/3419036054' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-06-03T19:40:15.032 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:40:14 smithi089 bash[26572]: audit 2023-06-03T19:40:14.201281+0000 mon.c (mon.2) 63 : audit [DBG] from='mgr.? 172.21.15.164:0/3419036054' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-06-03T19:40:15.032 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:40:14 smithi089 bash[26572]: audit 2023-06-03T19:40:14.202886+0000 mon.c (mon.2) 64 : audit [DBG] from='mgr.? 172.21.15.164:0/3419036054' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-06-03T19:40:15.032 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:40:14 smithi089 bash[26572]: audit 2023-06-03T19:40:14.472559+0000 mon.a (mon.0) 969 : audit [DBG] from='client.? 172.21.15.89:0/3562852268' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-03T19:40:15.032 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:40:14 smithi089 bash[33173]: cluster 2023-06-03T19:40:14.190908+0000 mon.a (mon.0) 968 : cluster [DBG] Standby manager daemon x started 2023-06-03T19:40:15.033 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:40:14 smithi089 bash[33173]: audit 2023-06-03T19:40:14.195802+0000 mon.c (mon.2) 61 : audit [DBG] from='mgr.? 172.21.15.164:0/3419036054' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-06-03T19:40:15.033 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:40:14 smithi089 bash[33173]: audit 2023-06-03T19:40:14.198457+0000 mon.c (mon.2) 62 : audit [DBG] from='mgr.? 172.21.15.164:0/3419036054' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-06-03T19:40:15.033 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:40:14 smithi089 bash[33173]: audit 2023-06-03T19:40:14.201281+0000 mon.c (mon.2) 63 : audit [DBG] from='mgr.? 172.21.15.164:0/3419036054' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-06-03T19:40:15.033 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:40:14 smithi089 bash[33173]: audit 2023-06-03T19:40:14.202886+0000 mon.c (mon.2) 64 : audit [DBG] from='mgr.? 172.21.15.164:0/3419036054' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-06-03T19:40:15.033 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:40:14 smithi089 bash[33173]: audit 2023-06-03T19:40:14.472559+0000 mon.a (mon.0) 969 : audit [DBG] from='client.? 172.21.15.89:0/3562852268' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-03T19:40:15.086 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:40:14 smithi164 bash[26855]: cluster 2023-06-03T19:40:14.190908+0000 mon.a (mon.0) 968 : cluster [DBG] Standby manager daemon x started 2023-06-03T19:40:15.086 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:40:14 smithi164 bash[26855]: audit 2023-06-03T19:40:14.195802+0000 mon.c (mon.2) 61 : audit [DBG] from='mgr.? 172.21.15.164:0/3419036054' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-06-03T19:40:15.086 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:40:14 smithi164 bash[26855]: audit 2023-06-03T19:40:14.198457+0000 mon.c (mon.2) 62 : audit [DBG] from='mgr.? 172.21.15.164:0/3419036054' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-06-03T19:40:15.087 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:40:14 smithi164 bash[26855]: audit 2023-06-03T19:40:14.201281+0000 mon.c (mon.2) 63 : audit [DBG] from='mgr.? 172.21.15.164:0/3419036054' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-06-03T19:40:15.087 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:40:14 smithi164 bash[26855]: audit 2023-06-03T19:40:14.202886+0000 mon.c (mon.2) 64 : audit [DBG] from='mgr.? 172.21.15.164:0/3419036054' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-06-03T19:40:15.087 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:40:14 smithi164 bash[26855]: audit 2023-06-03T19:40:14.472559+0000 mon.a (mon.0) 969 : audit [DBG] from='client.? 172.21.15.89:0/3562852268' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-03T19:40:15.782 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:40:15 smithi089 bash[26572]: cluster 2023-06-03T19:40:14.023266+0000 mgr.y (mgr.25070) 11 : cluster [DBG] pgmap v9: 89 pgs: 89 active+clean; 2.2 GiB data, 7.3 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 569 op/s 2023-06-03T19:40:15.782 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:40:15 smithi089 bash[26572]: cluster 2023-06-03T19:40:14.646010+0000 mon.a (mon.0) 970 : cluster [DBG] mgrmap e30: y(active, since 13s), standbys: x 2023-06-03T19:40:15.782 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:40:15 smithi089 bash[26572]: audit 2023-06-03T19:40:14.651996+0000 mon.b (mon.1) 59 : audit [DBG] from='mgr.25070 172.21.15.89:0/2108989343' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2023-06-03T19:40:15.782 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:40:15 smithi089 bash[33173]: cluster 2023-06-03T19:40:14.023266+0000 mgr.y (mgr.25070) 11 : cluster [DBG] pgmap v9: 89 pgs: 89 active+clean; 2.2 GiB data, 7.3 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 569 op/s 2023-06-03T19:40:15.783 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:40:15 smithi089 bash[33173]: cluster 2023-06-03T19:40:14.646010+0000 mon.a (mon.0) 970 : cluster [DBG] mgrmap e30: y(active, since 13s), standbys: x 2023-06-03T19:40:15.783 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:40:15 smithi089 bash[33173]: audit 2023-06-03T19:40:14.651996+0000 mon.b (mon.1) 59 : audit [DBG] from='mgr.25070 172.21.15.89:0/2108989343' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2023-06-03T19:40:16.086 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:40:15 smithi164 bash[26855]: cluster 2023-06-03T19:40:14.023266+0000 mgr.y (mgr.25070) 11 : cluster [DBG] pgmap v9: 89 pgs: 89 active+clean; 2.2 GiB data, 7.3 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 569 op/s 2023-06-03T19:40:16.086 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:40:15 smithi164 bash[26855]: cluster 2023-06-03T19:40:14.646010+0000 mon.a (mon.0) 970 : cluster [DBG] mgrmap e30: y(active, since 13s), standbys: x 2023-06-03T19:40:16.086 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:40:15 smithi164 bash[26855]: audit 2023-06-03T19:40:14.651996+0000 mon.b (mon.1) 59 : audit [DBG] from='mgr.25070 172.21.15.89:0/2108989343' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2023-06-03T19:40:18.031 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:40:17 smithi089 bash[33173]: cluster 2023-06-03T19:40:16.023731+0000 mgr.y (mgr.25070) 12 : cluster [DBG] pgmap v10: 89 pgs: 89 active+clean; 2.2 GiB data, 7.3 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 516 op/s 2023-06-03T19:40:18.031 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:40:17 smithi089 bash[33173]: audit 2023-06-03T19:40:17.159029+0000 mon.b (mon.1) 60 : audit [DBG] from='client.? 172.21.15.89:0/1187880508' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-03T19:40:18.032 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:40:17 smithi089 bash[33173]: audit 2023-06-03T19:40:17.203182+0000 mon.b (mon.1) 61 : audit [DBG] from='mgr.25070 172.21.15.89:0/2108989343' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T19:40:18.032 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:40:17 smithi089 bash[26572]: cluster 2023-06-03T19:40:16.023731+0000 mgr.y (mgr.25070) 12 : cluster [DBG] pgmap v10: 89 pgs: 89 active+clean; 2.2 GiB data, 7.3 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 516 op/s 2023-06-03T19:40:18.032 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:40:17 smithi089 bash[26572]: audit 2023-06-03T19:40:17.159029+0000 mon.b (mon.1) 60 : audit [DBG] from='client.? 172.21.15.89:0/1187880508' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-03T19:40:18.032 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:40:17 smithi089 bash[26572]: audit 2023-06-03T19:40:17.203182+0000 mon.b (mon.1) 61 : audit [DBG] from='mgr.25070 172.21.15.89:0/2108989343' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T19:40:18.086 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:40:17 smithi164 bash[26855]: cluster 2023-06-03T19:40:16.023731+0000 mgr.y (mgr.25070) 12 : cluster [DBG] pgmap v10: 89 pgs: 89 active+clean; 2.2 GiB data, 7.3 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 516 op/s 2023-06-03T19:40:18.087 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:40:17 smithi164 bash[26855]: audit 2023-06-03T19:40:17.159029+0000 mon.b (mon.1) 60 : audit [DBG] from='client.? 172.21.15.89:0/1187880508' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-03T19:40:18.087 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:40:17 smithi164 bash[26855]: audit 2023-06-03T19:40:17.203182+0000 mon.b (mon.1) 61 : audit [DBG] from='mgr.25070 172.21.15.89:0/2108989343' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T19:40:20.031 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:40:19 smithi089 bash[33173]: cluster 2023-06-03T19:40:18.024132+0000 mgr.y (mgr.25070) 13 : cluster [DBG] pgmap v11: 89 pgs: 89 active+clean; 2.2 GiB data, 7.3 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 516 op/s 2023-06-03T19:40:20.032 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:40:19 smithi089 bash[26572]: cluster 2023-06-03T19:40:18.024132+0000 mgr.y (mgr.25070) 13 : cluster [DBG] pgmap v11: 89 pgs: 89 active+clean; 2.2 GiB data, 7.3 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 516 op/s 2023-06-03T19:40:20.086 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:40:19 smithi164 bash[26855]: cluster 2023-06-03T19:40:18.024132+0000 mgr.y (mgr.25070) 13 : cluster [DBG] pgmap v11: 89 pgs: 89 active+clean; 2.2 GiB data, 7.3 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 516 op/s 2023-06-03T19:40:20.781 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:40:20 smithi089 bash[33173]: audit 2023-06-03T19:40:19.915073+0000 mon.c (mon.2) 65 : audit [DBG] from='client.? 172.21.15.89:0/3952017424' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-03T19:40:20.782 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:40:20 smithi089 bash[26572]: audit 2023-06-03T19:40:19.915073+0000 mon.c (mon.2) 65 : audit [DBG] from='client.? 172.21.15.89:0/3952017424' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-03T19:40:21.086 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:40:20 smithi164 bash[26855]: audit 2023-06-03T19:40:19.915073+0000 mon.c (mon.2) 65 : audit [DBG] from='client.? 172.21.15.89:0/3952017424' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-03T19:40:22.031 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:40:21 smithi089 bash[26572]: cluster 2023-06-03T19:40:20.024984+0000 mgr.y (mgr.25070) 14 : cluster [DBG] pgmap v12: 89 pgs: 89 active+clean; 2.0 GiB data, 6.7 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 805 op/s 2023-06-03T19:40:22.032 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:40:21 smithi089 bash[33173]: cluster 2023-06-03T19:40:20.024984+0000 mgr.y (mgr.25070) 14 : cluster [DBG] pgmap v12: 89 pgs: 89 active+clean; 2.0 GiB data, 6.7 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 805 op/s 2023-06-03T19:40:22.086 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:40:21 smithi164 bash[26855]: cluster 2023-06-03T19:40:20.024984+0000 mgr.y (mgr.25070) 14 : cluster [DBG] pgmap v12: 89 pgs: 89 active+clean; 2.0 GiB data, 6.7 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 805 op/s 2023-06-03T19:40:23.031 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:40:22 smithi089 bash[33173]: audit 2023-06-03T19:40:22.600934+0000 mon.a (mon.0) 971 : audit [DBG] from='client.? 172.21.15.89:0/3399870437' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-03T19:40:23.032 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:40:22 smithi089 bash[33173]: audit 2023-06-03T19:40:22.647028+0000 mon.a (mon.0) 972 : audit [INF] from='mgr.25070 ' entity='mgr.y' 2023-06-03T19:40:23.032 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:40:22 smithi089 bash[33173]: audit 2023-06-03T19:40:22.657455+0000 mon.a (mon.0) 973 : audit [INF] from='mgr.25070 ' entity='mgr.y' 2023-06-03T19:40:23.033 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:40:22 smithi089 bash[26572]: audit 2023-06-03T19:40:22.600934+0000 mon.a (mon.0) 971 : audit [DBG] from='client.? 172.21.15.89:0/3399870437' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-03T19:40:23.033 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:40:22 smithi089 bash[26572]: audit 2023-06-03T19:40:22.647028+0000 mon.a (mon.0) 972 : audit [INF] from='mgr.25070 ' entity='mgr.y' 2023-06-03T19:40:23.033 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:40:22 smithi089 bash[26572]: audit 2023-06-03T19:40:22.657455+0000 mon.a (mon.0) 973 : audit [INF] from='mgr.25070 ' entity='mgr.y' 2023-06-03T19:40:23.086 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:40:22 smithi164 bash[26855]: audit 2023-06-03T19:40:22.600934+0000 mon.a (mon.0) 971 : audit [DBG] from='client.? 172.21.15.89:0/3399870437' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-03T19:40:23.087 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:40:22 smithi164 bash[26855]: audit 2023-06-03T19:40:22.647028+0000 mon.a (mon.0) 972 : audit [INF] from='mgr.25070 ' entity='mgr.y' 2023-06-03T19:40:23.087 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:40:22 smithi164 bash[26855]: audit 2023-06-03T19:40:22.657455+0000 mon.a (mon.0) 973 : audit [INF] from='mgr.25070 ' entity='mgr.y' 2023-06-03T19:40:23.586 INFO:journalctl@ceph.mgr.x.smithi164.stdout:Jun 03 19:40:23 smithi164 bash[50883]: ::ffff:172.21.15.164 - - [03/Jun/2023:19:40:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2023-06-03T19:40:24.031 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:40:23 smithi089 bash[26572]: cluster 2023-06-03T19:40:22.025463+0000 mgr.y (mgr.25070) 15 : cluster [DBG] pgmap v13: 89 pgs: 89 active+clean; 2.0 GiB data, 6.7 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 540 op/s 2023-06-03T19:40:24.032 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:40:23 smithi089 bash[26572]: audit 2023-06-03T19:40:22.663854+0000 mon.b (mon.1) 62 : audit [DBG] from='mgr.25070 172.21.15.89:0/2108989343' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-03T19:40:24.032 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:40:23 smithi089 bash[26572]: audit 2023-06-03T19:40:22.664965+0000 mon.b (mon.1) 63 : audit [INF] from='mgr.25070 172.21.15.89:0/2108989343' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-03T19:40:24.032 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:40:23 smithi089 bash[26572]: cephadm 2023-06-03T19:40:22.668213+0000 mgr.y (mgr.25070) 16 : cephadm [INF] Updating smithi089:/etc/ceph/ceph.conf 2023-06-03T19:40:24.032 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:40:23 smithi089 bash[26572]: cephadm 2023-06-03T19:40:22.668444+0000 mgr.y (mgr.25070) 17 : cephadm [INF] Updating smithi164:/etc/ceph/ceph.conf 2023-06-03T19:40:24.032 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:40:23 smithi089 bash[26572]: audit 2023-06-03T19:40:23.548646+0000 mon.a (mon.0) 974 : audit [INF] from='mgr.25070 ' entity='mgr.y' 2023-06-03T19:40:24.033 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:40:23 smithi089 bash[26572]: audit 2023-06-03T19:40:23.562930+0000 mon.a (mon.0) 975 : audit [INF] from='mgr.25070 ' entity='mgr.y' 2023-06-03T19:40:24.033 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:40:23 smithi089 bash[26572]: audit 2023-06-03T19:40:23.570538+0000 mon.a (mon.0) 976 : audit [INF] from='mgr.25070 ' entity='mgr.y' 2023-06-03T19:40:24.033 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:40:23 smithi089 bash[26572]: audit 2023-06-03T19:40:23.587518+0000 mon.a (mon.0) 977 : audit [INF] from='mgr.25070 ' entity='mgr.y' 2023-06-03T19:40:24.033 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:40:23 smithi089 bash[26572]: audit 2023-06-03T19:40:23.599755+0000 mon.a (mon.0) 978 : audit [INF] from='mgr.25070 ' entity='mgr.y' 2023-06-03T19:40:24.034 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:40:23 smithi089 bash[33173]: cluster 2023-06-03T19:40:22.025463+0000 mgr.y (mgr.25070) 15 : cluster [DBG] pgmap v13: 89 pgs: 89 active+clean; 2.0 GiB data, 6.7 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 540 op/s 2023-06-03T19:40:24.034 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:40:23 smithi089 bash[33173]: audit 2023-06-03T19:40:22.663854+0000 mon.b (mon.1) 62 : audit [DBG] from='mgr.25070 172.21.15.89:0/2108989343' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-03T19:40:24.034 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:40:23 smithi089 bash[33173]: audit 2023-06-03T19:40:22.664965+0000 mon.b (mon.1) 63 : audit [INF] from='mgr.25070 172.21.15.89:0/2108989343' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-03T19:40:24.035 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:40:23 smithi089 bash[33173]: cephadm 2023-06-03T19:40:22.668213+0000 mgr.y (mgr.25070) 16 : cephadm [INF] Updating smithi089:/etc/ceph/ceph.conf 2023-06-03T19:40:24.035 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:40:23 smithi089 bash[33173]: cephadm 2023-06-03T19:40:22.668444+0000 mgr.y (mgr.25070) 17 : cephadm [INF] Updating smithi164:/etc/ceph/ceph.conf 2023-06-03T19:40:24.035 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:40:23 smithi089 bash[33173]: audit 2023-06-03T19:40:23.548646+0000 mon.a (mon.0) 974 : audit [INF] from='mgr.25070 ' entity='mgr.y' 2023-06-03T19:40:24.036 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:40:23 smithi089 bash[33173]: audit 2023-06-03T19:40:23.562930+0000 mon.a (mon.0) 975 : audit [INF] from='mgr.25070 ' entity='mgr.y' 2023-06-03T19:40:24.036 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:40:23 smithi089 bash[33173]: audit 2023-06-03T19:40:23.570538+0000 mon.a (mon.0) 976 : audit [INF] from='mgr.25070 ' entity='mgr.y' 2023-06-03T19:40:24.036 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:40:23 smithi089 bash[33173]: audit 2023-06-03T19:40:23.587518+0000 mon.a (mon.0) 977 : audit [INF] from='mgr.25070 ' entity='mgr.y' 2023-06-03T19:40:24.036 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:40:23 smithi089 bash[33173]: audit 2023-06-03T19:40:23.599755+0000 mon.a (mon.0) 978 : audit [INF] from='mgr.25070 ' entity='mgr.y' 2023-06-03T19:40:24.086 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:40:23 smithi164 bash[26855]: cluster 2023-06-03T19:40:22.025463+0000 mgr.y (mgr.25070) 15 : cluster [DBG] pgmap v13: 89 pgs: 89 active+clean; 2.0 GiB data, 6.7 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 540 op/s 2023-06-03T19:40:24.086 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:40:23 smithi164 bash[26855]: audit 2023-06-03T19:40:22.663854+0000 mon.b (mon.1) 62 : audit [DBG] from='mgr.25070 172.21.15.89:0/2108989343' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-03T19:40:24.086 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:40:23 smithi164 bash[26855]: audit 2023-06-03T19:40:22.664965+0000 mon.b (mon.1) 63 : audit [INF] from='mgr.25070 172.21.15.89:0/2108989343' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-03T19:40:24.087 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:40:23 smithi164 bash[26855]: cephadm 2023-06-03T19:40:22.668213+0000 mgr.y (mgr.25070) 16 : cephadm [INF] Updating smithi089:/etc/ceph/ceph.conf 2023-06-03T19:40:24.087 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:40:23 smithi164 bash[26855]: cephadm 2023-06-03T19:40:22.668444+0000 mgr.y (mgr.25070) 17 : cephadm [INF] Updating smithi164:/etc/ceph/ceph.conf 2023-06-03T19:40:24.087 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:40:23 smithi164 bash[26855]: audit 2023-06-03T19:40:23.548646+0000 mon.a (mon.0) 974 : audit [INF] from='mgr.25070 ' entity='mgr.y' 2023-06-03T19:40:24.087 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:40:23 smithi164 bash[26855]: audit 2023-06-03T19:40:23.562930+0000 mon.a (mon.0) 975 : audit [INF] from='mgr.25070 ' entity='mgr.y' 2023-06-03T19:40:24.087 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:40:23 smithi164 bash[26855]: audit 2023-06-03T19:40:23.570538+0000 mon.a (mon.0) 976 : audit [INF] from='mgr.25070 ' entity='mgr.y' 2023-06-03T19:40:24.087 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:40:23 smithi164 bash[26855]: audit 2023-06-03T19:40:23.587518+0000 mon.a (mon.0) 977 : audit [INF] from='mgr.25070 ' entity='mgr.y' 2023-06-03T19:40:24.087 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:40:23 smithi164 bash[26855]: audit 2023-06-03T19:40:23.599755+0000 mon.a (mon.0) 978 : audit [INF] from='mgr.25070 ' entity='mgr.y' 2023-06-03T19:40:25.031 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:40:24 smithi089 bash[26572]: cephadm 2023-06-03T19:40:22.884024+0000 mgr.y (mgr.25070) 18 : cephadm [INF] Updating smithi164:/var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/config/ceph.conf 2023-06-03T19:40:25.031 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:40:24 smithi089 bash[26572]: cephadm 2023-06-03T19:40:22.890312+0000 mgr.y (mgr.25070) 19 : cephadm [INF] Updating smithi089:/var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/config/ceph.conf 2023-06-03T19:40:25.032 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:40:24 smithi089 bash[26572]: cephadm 2023-06-03T19:40:23.108803+0000 mgr.y (mgr.25070) 20 : cephadm [INF] Updating smithi164:/etc/ceph/ceph.client.admin.keyring 2023-06-03T19:40:25.032 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:40:24 smithi089 bash[26572]: cephadm 2023-06-03T19:40:23.111951+0000 mgr.y (mgr.25070) 21 : cephadm [INF] Updating smithi089:/etc/ceph/ceph.client.admin.keyring 2023-06-03T19:40:25.032 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:40:24 smithi089 bash[26572]: cephadm 2023-06-03T19:40:23.325488+0000 mgr.y (mgr.25070) 22 : cephadm [INF] Updating smithi164:/var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/config/ceph.client.admin.keyring 2023-06-03T19:40:25.032 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:40:24 smithi089 bash[26572]: cephadm 2023-06-03T19:40:23.327566+0000 mgr.y (mgr.25070) 23 : cephadm [INF] Updating smithi089:/var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/config/ceph.client.admin.keyring 2023-06-03T19:40:25.032 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:40:24 smithi089 bash[26572]: cephadm 2023-06-03T19:40:23.671976+0000 mgr.y (mgr.25070) 24 : cephadm [INF] Reconfiguring prometheus.a (dependencies changed)... 2023-06-03T19:40:25.033 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:40:24 smithi089 bash[33173]: cephadm 2023-06-03T19:40:22.884024+0000 mgr.y (mgr.25070) 18 : cephadm [INF] Updating smithi164:/var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/config/ceph.conf 2023-06-03T19:40:25.033 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:40:24 smithi089 bash[33173]: cephadm 2023-06-03T19:40:22.890312+0000 mgr.y (mgr.25070) 19 : cephadm [INF] Updating smithi089:/var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/config/ceph.conf 2023-06-03T19:40:25.033 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:40:24 smithi089 bash[33173]: cephadm 2023-06-03T19:40:23.108803+0000 mgr.y (mgr.25070) 20 : cephadm [INF] Updating smithi164:/etc/ceph/ceph.client.admin.keyring 2023-06-03T19:40:25.033 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:40:24 smithi089 bash[33173]: cephadm 2023-06-03T19:40:23.111951+0000 mgr.y (mgr.25070) 21 : cephadm [INF] Updating smithi089:/etc/ceph/ceph.client.admin.keyring 2023-06-03T19:40:25.033 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:40:24 smithi089 bash[33173]: cephadm 2023-06-03T19:40:23.325488+0000 mgr.y (mgr.25070) 22 : cephadm [INF] Updating smithi164:/var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/config/ceph.client.admin.keyring 2023-06-03T19:40:25.033 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:40:24 smithi089 bash[33173]: cephadm 2023-06-03T19:40:23.327566+0000 mgr.y (mgr.25070) 23 : cephadm [INF] Updating smithi089:/var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/config/ceph.client.admin.keyring 2023-06-03T19:40:25.034 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:40:24 smithi089 bash[33173]: cephadm 2023-06-03T19:40:23.671976+0000 mgr.y (mgr.25070) 24 : cephadm [INF] Reconfiguring prometheus.a (dependencies changed)... 2023-06-03T19:40:25.086 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:40:24 smithi164 bash[26855]: cephadm 2023-06-03T19:40:22.884024+0000 mgr.y (mgr.25070) 18 : cephadm [INF] Updating smithi164:/var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/config/ceph.conf 2023-06-03T19:40:25.086 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:40:24 smithi164 bash[26855]: cephadm 2023-06-03T19:40:22.890312+0000 mgr.y (mgr.25070) 19 : cephadm [INF] Updating smithi089:/var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/config/ceph.conf 2023-06-03T19:40:25.086 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:40:24 smithi164 bash[26855]: cephadm 2023-06-03T19:40:23.108803+0000 mgr.y (mgr.25070) 20 : cephadm [INF] Updating smithi164:/etc/ceph/ceph.client.admin.keyring 2023-06-03T19:40:25.086 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:40:24 smithi164 bash[26855]: cephadm 2023-06-03T19:40:23.111951+0000 mgr.y (mgr.25070) 21 : cephadm [INF] Updating smithi089:/etc/ceph/ceph.client.admin.keyring 2023-06-03T19:40:25.087 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:40:24 smithi164 bash[26855]: cephadm 2023-06-03T19:40:23.325488+0000 mgr.y (mgr.25070) 22 : cephadm [INF] Updating smithi164:/var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/config/ceph.client.admin.keyring 2023-06-03T19:40:25.087 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:40:24 smithi164 bash[26855]: cephadm 2023-06-03T19:40:23.327566+0000 mgr.y (mgr.25070) 23 : cephadm [INF] Updating smithi089:/var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/config/ceph.client.admin.keyring 2023-06-03T19:40:25.087 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:40:24 smithi164 bash[26855]: cephadm 2023-06-03T19:40:23.671976+0000 mgr.y (mgr.25070) 24 : cephadm [INF] Reconfiguring prometheus.a (dependencies changed)... 2023-06-03T19:40:26.031 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:40:25 smithi089 bash[33173]: cephadm 2023-06-03T19:40:23.939308+0000 mgr.y (mgr.25070) 25 : cephadm [INF] Reconfiguring daemon prometheus.a on smithi164 2023-06-03T19:40:26.031 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:40:25 smithi089 bash[33173]: cluster 2023-06-03T19:40:24.026183+0000 mgr.y (mgr.25070) 26 : cluster [DBG] pgmap v14: 89 pgs: 89 active+clean; 1.8 GiB data, 6.1 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 827 op/s 2023-06-03T19:40:26.032 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:40:25 smithi089 bash[33173]: audit 2023-06-03T19:40:25.305460+0000 mon.c (mon.2) 66 : audit [DBG] from='client.? 172.21.15.89:0/2605629157' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-03T19:40:26.032 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:40:25 smithi089 bash[26572]: cephadm 2023-06-03T19:40:23.939308+0000 mgr.y (mgr.25070) 25 : cephadm [INF] Reconfiguring daemon prometheus.a on smithi164 2023-06-03T19:40:26.032 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:40:25 smithi089 bash[26572]: cluster 2023-06-03T19:40:24.026183+0000 mgr.y (mgr.25070) 26 : cluster [DBG] pgmap v14: 89 pgs: 89 active+clean; 1.8 GiB data, 6.1 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 827 op/s 2023-06-03T19:40:26.032 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:40:25 smithi089 bash[26572]: audit 2023-06-03T19:40:25.305460+0000 mon.c (mon.2) 66 : audit [DBG] from='client.? 172.21.15.89:0/2605629157' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-03T19:40:26.086 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:40:25 smithi164 bash[26855]: cephadm 2023-06-03T19:40:23.939308+0000 mgr.y (mgr.25070) 25 : cephadm [INF] Reconfiguring daemon prometheus.a on smithi164 2023-06-03T19:40:26.086 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:40:25 smithi164 bash[26855]: cluster 2023-06-03T19:40:24.026183+0000 mgr.y (mgr.25070) 26 : cluster [DBG] pgmap v14: 89 pgs: 89 active+clean; 1.8 GiB data, 6.1 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 827 op/s 2023-06-03T19:40:26.086 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:40:25 smithi164 bash[26855]: audit 2023-06-03T19:40:25.305460+0000 mon.c (mon.2) 66 : audit [DBG] from='client.? 172.21.15.89:0/2605629157' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-03T19:40:28.031 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:40:27 smithi089 bash[33173]: cluster 2023-06-03T19:40:26.026674+0000 mgr.y (mgr.25070) 27 : cluster [DBG] pgmap v15: 89 pgs: 89 active+clean; 1.8 GiB data, 6.1 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 576 op/s 2023-06-03T19:40:28.032 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:40:27 smithi089 bash[26572]: cluster 2023-06-03T19:40:26.026674+0000 mgr.y (mgr.25070) 27 : cluster [DBG] pgmap v15: 89 pgs: 89 active+clean; 1.8 GiB data, 6.1 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 576 op/s 2023-06-03T19:40:28.086 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:40:27 smithi164 bash[26855]: cluster 2023-06-03T19:40:26.026674+0000 mgr.y (mgr.25070) 27 : cluster [DBG] pgmap v15: 89 pgs: 89 active+clean; 1.8 GiB data, 6.1 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 576 op/s 2023-06-03T19:40:29.031 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:40:28 smithi089 bash[33173]: audit 2023-06-03T19:40:28.003445+0000 mon.b (mon.1) 64 : audit [DBG] from='client.? 172.21.15.89:0/3056482518' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-03T19:40:29.032 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:40:28 smithi089 bash[26572]: audit 2023-06-03T19:40:28.003445+0000 mon.b (mon.1) 64 : audit [DBG] from='client.? 172.21.15.89:0/3056482518' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-03T19:40:29.086 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:40:28 smithi164 bash[26855]: audit 2023-06-03T19:40:28.003445+0000 mon.b (mon.1) 64 : audit [DBG] from='client.? 172.21.15.89:0/3056482518' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-03T19:40:30.031 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:40:29 smithi089 bash[33173]: cluster 2023-06-03T19:40:28.027144+0000 mgr.y (mgr.25070) 28 : cluster [DBG] pgmap v16: 89 pgs: 89 active+clean; 1.8 GiB data, 6.1 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 576 op/s 2023-06-03T19:40:30.031 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:40:29 smithi089 bash[26572]: cluster 2023-06-03T19:40:28.027144+0000 mgr.y (mgr.25070) 28 : cluster [DBG] pgmap v16: 89 pgs: 89 active+clean; 1.8 GiB data, 6.1 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 576 op/s 2023-06-03T19:40:30.086 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:40:29 smithi164 bash[26855]: cluster 2023-06-03T19:40:28.027144+0000 mgr.y (mgr.25070) 28 : cluster [DBG] pgmap v16: 89 pgs: 89 active+clean; 1.8 GiB data, 6.1 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 576 op/s 2023-06-03T19:40:31.031 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:40:30 smithi089 bash[26572]: audit 2023-06-03T19:40:30.692696+0000 mon.a (mon.0) 979 : audit [DBG] from='client.? 172.21.15.89:0/1347502142' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-03T19:40:31.032 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:40:30 smithi089 bash[33173]: audit 2023-06-03T19:40:30.692696+0000 mon.a (mon.0) 979 : audit [DBG] from='client.? 172.21.15.89:0/1347502142' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-03T19:40:31.086 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:40:30 smithi164 bash[26855]: audit 2023-06-03T19:40:30.692696+0000 mon.a (mon.0) 979 : audit [DBG] from='client.? 172.21.15.89:0/1347502142' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-03T19:40:31.656 INFO:journalctl@ceph.prometheus.a.smithi164.stdout:Jun 03 19:40:31 smithi164 systemd[1]: Stopping Ceph prometheus.a for f1db3478-0243-11ee-9b1d-001a4aab830c... 2023-06-03T19:40:31.656 INFO:journalctl@ceph.prometheus.a.smithi164.stdout:Jun 03 19:40:31 smithi164 bash[55072]: Error response from daemon: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-prometheus.a 2023-06-03T19:40:31.657 INFO:journalctl@ceph.prometheus.a.smithi164.stdout:Jun 03 19:40:31 smithi164 bash[53294]: ts=2023-06-03T19:40:31.646Z caller=main.go:775 level=warn msg="Received SIGTERM, exiting gracefully..." 2023-06-03T19:40:31.657 INFO:journalctl@ceph.prometheus.a.smithi164.stdout:Jun 03 19:40:31 smithi164 bash[53294]: ts=2023-06-03T19:40:31.646Z caller=main.go:798 level=info msg="Stopping scrape discovery manager..." 2023-06-03T19:40:31.657 INFO:journalctl@ceph.prometheus.a.smithi164.stdout:Jun 03 19:40:31 smithi164 bash[53294]: ts=2023-06-03T19:40:31.646Z caller=main.go:812 level=info msg="Stopping notify discovery manager..." 2023-06-03T19:40:31.657 INFO:journalctl@ceph.prometheus.a.smithi164.stdout:Jun 03 19:40:31 smithi164 bash[53294]: ts=2023-06-03T19:40:31.646Z caller=main.go:834 level=info msg="Stopping scrape manager..." 2023-06-03T19:40:31.657 INFO:journalctl@ceph.prometheus.a.smithi164.stdout:Jun 03 19:40:31 smithi164 bash[53294]: ts=2023-06-03T19:40:31.646Z caller=main.go:794 level=info msg="Scrape discovery manager stopped" 2023-06-03T19:40:31.657 INFO:journalctl@ceph.prometheus.a.smithi164.stdout:Jun 03 19:40:31 smithi164 bash[53294]: ts=2023-06-03T19:40:31.646Z caller=main.go:808 level=info msg="Notify discovery manager stopped" 2023-06-03T19:40:31.657 INFO:journalctl@ceph.prometheus.a.smithi164.stdout:Jun 03 19:40:31 smithi164 bash[53294]: ts=2023-06-03T19:40:31.646Z caller=manager.go:945 level=info component="rule manager" msg="Stopping rule manager..." 2023-06-03T19:40:31.658 INFO:journalctl@ceph.prometheus.a.smithi164.stdout:Jun 03 19:40:31 smithi164 bash[53294]: ts=2023-06-03T19:40:31.646Z caller=manager.go:955 level=info component="rule manager" msg="Rule manager stopped" 2023-06-03T19:40:31.658 INFO:journalctl@ceph.prometheus.a.smithi164.stdout:Jun 03 19:40:31 smithi164 bash[53294]: ts=2023-06-03T19:40:31.647Z caller=notifier.go:600 level=info component=notifier msg="Stopping notification manager..." 2023-06-03T19:40:32.031 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:40:31 smithi089 bash[26572]: cluster 2023-06-03T19:40:30.027782+0000 mgr.y (mgr.25070) 29 : cluster [DBG] pgmap v17: 89 pgs: 89 active+clean; 1.6 GiB data, 5.5 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 865 op/s 2023-06-03T19:40:32.032 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:40:31 smithi089 bash[33173]: cluster 2023-06-03T19:40:30.027782+0000 mgr.y (mgr.25070) 29 : cluster [DBG] pgmap v17: 89 pgs: 89 active+clean; 1.6 GiB data, 5.5 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 865 op/s 2023-06-03T19:40:32.086 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:40:31 smithi164 bash[26855]: cluster 2023-06-03T19:40:30.027782+0000 mgr.y (mgr.25070) 29 : cluster [DBG] pgmap v17: 89 pgs: 89 active+clean; 1.6 GiB data, 5.5 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 865 op/s 2023-06-03T19:40:32.087 INFO:journalctl@ceph.prometheus.a.smithi164.stdout:Jun 03 19:40:31 smithi164 bash[53294]: ts=2023-06-03T19:40:31.647Z caller=main.go:1054 level=info msg="Notifier manager stopped" 2023-06-03T19:40:32.087 INFO:journalctl@ceph.prometheus.a.smithi164.stdout:Jun 03 19:40:31 smithi164 bash[53294]: ts=2023-06-03T19:40:31.647Z caller=main.go:828 level=info msg="Scrape manager stopped" 2023-06-03T19:40:32.088 INFO:journalctl@ceph.prometheus.a.smithi164.stdout:Jun 03 19:40:31 smithi164 bash[53294]: ts=2023-06-03T19:40:31.647Z caller=main.go:1066 level=info msg="See you next time!" 2023-06-03T19:40:33.031 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:40:32 smithi089 bash[26572]: cluster 2023-06-03T19:40:32.028239+0000 mgr.y (mgr.25070) 30 : cluster [DBG] pgmap v18: 89 pgs: 89 active+clean; 1.6 GiB data, 5.5 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 576 op/s 2023-06-03T19:40:33.032 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:40:32 smithi089 bash[26572]: audit 2023-06-03T19:40:32.204048+0000 mon.b (mon.1) 65 : audit [DBG] from='mgr.25070 172.21.15.89:0/2108989343' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T19:40:33.032 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:40:32 smithi089 bash[33173]: cluster 2023-06-03T19:40:32.028239+0000 mgr.y (mgr.25070) 30 : cluster [DBG] pgmap v18: 89 pgs: 89 active+clean; 1.6 GiB data, 5.5 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 576 op/s 2023-06-03T19:40:33.032 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:40:32 smithi089 bash[33173]: audit 2023-06-03T19:40:32.204048+0000 mon.b (mon.1) 65 : audit [DBG] from='mgr.25070 172.21.15.89:0/2108989343' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T19:40:33.086 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:40:32 smithi164 bash[26855]: cluster 2023-06-03T19:40:32.028239+0000 mgr.y (mgr.25070) 30 : cluster [DBG] pgmap v18: 89 pgs: 89 active+clean; 1.6 GiB data, 5.5 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 576 op/s 2023-06-03T19:40:33.086 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:40:32 smithi164 bash[26855]: audit 2023-06-03T19:40:32.204048+0000 mon.b (mon.1) 65 : audit [DBG] from='mgr.25070 172.21.15.89:0/2108989343' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T19:40:33.087 INFO:journalctl@ceph.prometheus.a.smithi164.stdout:Jun 03 19:40:32 smithi164 bash[55103]: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-prometheus-a 2023-06-03T19:40:33.087 INFO:journalctl@ceph.prometheus.a.smithi164.stdout:Jun 03 19:40:32 smithi164 systemd[1]: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c@prometheus.a.service: Succeeded. 2023-06-03T19:40:33.087 INFO:journalctl@ceph.prometheus.a.smithi164.stdout:Jun 03 19:40:32 smithi164 systemd[1]: Stopped Ceph prometheus.a for f1db3478-0243-11ee-9b1d-001a4aab830c. 2023-06-03T19:40:33.087 INFO:journalctl@ceph.prometheus.a.smithi164.stdout:Jun 03 19:40:32 smithi164 systemd[1]: Started Ceph prometheus.a for f1db3478-0243-11ee-9b1d-001a4aab830c. 2023-06-03T19:40:34.031 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:40:33 smithi089 bash[26572]: audit 2023-06-03T19:40:32.768903+0000 mon.a (mon.0) 980 : audit [INF] from='mgr.25070 ' entity='mgr.y' 2023-06-03T19:40:34.032 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:40:33 smithi089 bash[26572]: audit 2023-06-03T19:40:32.792662+0000 mon.a (mon.0) 981 : audit [INF] from='mgr.25070 ' entity='mgr.y' 2023-06-03T19:40:34.032 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:40:33 smithi089 bash[26572]: audit 2023-06-03T19:40:32.824181+0000 mon.b (mon.1) 66 : audit [DBG] from='mgr.25070 172.21.15.89:0/2108989343' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-06-03T19:40:34.032 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:40:33 smithi089 bash[26572]: audit 2023-06-03T19:40:32.829503+0000 mgr.y (mgr.25070) 31 : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-06-03T19:40:34.033 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:40:33 smithi089 bash[26572]: audit 2023-06-03T19:40:32.841169+0000 mon.b (mon.1) 67 : audit [DBG] from='mgr.25070 172.21.15.89:0/2108989343' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-03T19:40:34.033 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:40:33 smithi089 bash[26572]: audit 2023-06-03T19:40:33.362235+0000 mon.a (mon.0) 982 : audit [DBG] from='client.? 172.21.15.89:0/3915316242' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-03T19:40:34.033 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:40:33 smithi089 bash[33173]: audit 2023-06-03T19:40:32.768903+0000 mon.a (mon.0) 980 : audit [INF] from='mgr.25070 ' entity='mgr.y' 2023-06-03T19:40:34.034 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:40:33 smithi089 bash[33173]: audit 2023-06-03T19:40:32.792662+0000 mon.a (mon.0) 981 : audit [INF] from='mgr.25070 ' entity='mgr.y' 2023-06-03T19:40:34.034 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:40:33 smithi089 bash[33173]: audit 2023-06-03T19:40:32.824181+0000 mon.b (mon.1) 66 : audit [DBG] from='mgr.25070 172.21.15.89:0/2108989343' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-06-03T19:40:34.034 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:40:33 smithi089 bash[33173]: audit 2023-06-03T19:40:32.829503+0000 mgr.y (mgr.25070) 31 : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-06-03T19:40:34.034 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:40:33 smithi089 bash[33173]: audit 2023-06-03T19:40:32.841169+0000 mon.b (mon.1) 67 : audit [DBG] from='mgr.25070 172.21.15.89:0/2108989343' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-03T19:40:34.035 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:40:33 smithi089 bash[33173]: audit 2023-06-03T19:40:33.362235+0000 mon.a (mon.0) 982 : audit [DBG] from='client.? 172.21.15.89:0/3915316242' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-03T19:40:34.084 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:40:33 smithi164 bash[26855]: audit 2023-06-03T19:40:32.768903+0000 mon.a (mon.0) 980 : audit [INF] from='mgr.25070 ' entity='mgr.y' 2023-06-03T19:40:34.085 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:40:33 smithi164 bash[26855]: audit 2023-06-03T19:40:32.792662+0000 mon.a (mon.0) 981 : audit [INF] from='mgr.25070 ' entity='mgr.y' 2023-06-03T19:40:34.085 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:40:33 smithi164 bash[26855]: audit 2023-06-03T19:40:32.824181+0000 mon.b (mon.1) 66 : audit [DBG] from='mgr.25070 172.21.15.89:0/2108989343' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-06-03T19:40:34.085 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:40:33 smithi164 bash[26855]: audit 2023-06-03T19:40:32.829503+0000 mgr.y (mgr.25070) 31 : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-06-03T19:40:34.085 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:40:33 smithi164 bash[26855]: audit 2023-06-03T19:40:32.841169+0000 mon.b (mon.1) 67 : audit [DBG] from='mgr.25070 172.21.15.89:0/2108989343' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-03T19:40:34.086 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:40:33 smithi164 bash[26855]: audit 2023-06-03T19:40:33.362235+0000 mon.a (mon.0) 982 : audit [DBG] from='client.? 172.21.15.89:0/3915316242' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-03T19:40:35.082 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:40:34 smithi164 bash[26855]: cluster 2023-06-03T19:40:34.029106+0000 mgr.y (mgr.25070) 32 : cluster [DBG] pgmap v19: 89 pgs: 89 active+clean; 1.4 GiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 871 op/s 2023-06-03T19:40:35.281 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:40:34 smithi089 bash[33173]: cluster 2023-06-03T19:40:34.029106+0000 mgr.y (mgr.25070) 32 : cluster [DBG] pgmap v19: 89 pgs: 89 active+clean; 1.4 GiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 871 op/s 2023-06-03T19:40:35.281 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:40:34 smithi089 bash[26572]: cluster 2023-06-03T19:40:34.029106+0000 mgr.y (mgr.25070) 32 : cluster [DBG] pgmap v19: 89 pgs: 89 active+clean; 1.4 GiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 871 op/s 2023-06-03T19:40:35.336 INFO:journalctl@ceph.prometheus.a.smithi164.stdout:Jun 03 19:40:35 smithi164 bash[55188]: ts=2023-06-03T19:40:35.229Z caller=main.go:512 level=info msg="Starting Prometheus" version="(version=2.33.4, branch=HEAD, revision=83032011a5d3e6102624fe58241a374a7201fee8)" 2023-06-03T19:40:35.336 INFO:journalctl@ceph.prometheus.a.smithi164.stdout:Jun 03 19:40:35 smithi164 bash[55188]: ts=2023-06-03T19:40:35.229Z caller=main.go:517 level=info build_context="(go=go1.17.7, user=root@d13bf69e7be8, date=20220222-16:51:28)" 2023-06-03T19:40:35.336 INFO:journalctl@ceph.prometheus.a.smithi164.stdout:Jun 03 19:40:35 smithi164 bash[55188]: ts=2023-06-03T19:40:35.229Z caller=main.go:518 level=info host_details="(Linux 5.15.0-73-generic #80~20.04.1-Ubuntu SMP Wed May 17 14:58:14 UTC 2023 x86_64 smithi164 (none))" 2023-06-03T19:40:35.337 INFO:journalctl@ceph.prometheus.a.smithi164.stdout:Jun 03 19:40:35 smithi164 bash[55188]: ts=2023-06-03T19:40:35.229Z caller=main.go:519 level=info fd_limits="(soft=1048576, hard=1048576)" 2023-06-03T19:40:35.337 INFO:journalctl@ceph.prometheus.a.smithi164.stdout:Jun 03 19:40:35 smithi164 bash[55188]: ts=2023-06-03T19:40:35.229Z caller=main.go:520 level=info vm_limits="(soft=unlimited, hard=unlimited)" 2023-06-03T19:40:35.337 INFO:journalctl@ceph.prometheus.a.smithi164.stdout:Jun 03 19:40:35 smithi164 bash[55188]: ts=2023-06-03T19:40:35.234Z caller=web.go:570 level=info component=web msg="Start listening for connections" address=:9095 2023-06-03T19:40:35.337 INFO:journalctl@ceph.prometheus.a.smithi164.stdout:Jun 03 19:40:35 smithi164 bash[55188]: ts=2023-06-03T19:40:35.234Z caller=main.go:923 level=info msg="Starting TSDB ..." 2023-06-03T19:40:35.337 INFO:journalctl@ceph.prometheus.a.smithi164.stdout:Jun 03 19:40:35 smithi164 bash[55188]: ts=2023-06-03T19:40:35.236Z caller=tls_config.go:195 level=info component=web msg="TLS is disabled." http2=false 2023-06-03T19:40:35.337 INFO:journalctl@ceph.prometheus.a.smithi164.stdout:Jun 03 19:40:35 smithi164 bash[55188]: ts=2023-06-03T19:40:35.239Z caller=head.go:493 level=info component=tsdb msg="Replaying on-disk memory mappable chunks if any" 2023-06-03T19:40:35.337 INFO:journalctl@ceph.prometheus.a.smithi164.stdout:Jun 03 19:40:35 smithi164 bash[55188]: ts=2023-06-03T19:40:35.239Z caller=head.go:527 level=info component=tsdb msg="On-disk memory mappable chunks replay completed" duration=2.601µs 2023-06-03T19:40:35.337 INFO:journalctl@ceph.prometheus.a.smithi164.stdout:Jun 03 19:40:35 smithi164 bash[55188]: ts=2023-06-03T19:40:35.239Z caller=head.go:533 level=info component=tsdb msg="Replaying WAL, this may take a while" 2023-06-03T19:40:35.836 INFO:journalctl@ceph.prometheus.a.smithi164.stdout:Jun 03 19:40:35 smithi164 bash[55188]: ts=2023-06-03T19:40:35.498Z caller=head.go:604 level=info component=tsdb msg="WAL segment loaded" segment=0 maxSegment=3 2023-06-03T19:40:36.407 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:40:36 smithi164 bash[26855]: audit 2023-06-03T19:40:36.054687+0000 mon.b (mon.1) 68 : audit [DBG] from='client.? 172.21.15.89:0/1457798813' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-03T19:40:36.408 INFO:journalctl@ceph.prometheus.a.smithi164.stdout:Jun 03 19:40:36 smithi164 bash[55188]: ts=2023-06-03T19:40:36.116Z caller=head.go:604 level=info component=tsdb msg="WAL segment loaded" segment=1 maxSegment=3 2023-06-03T19:40:36.408 INFO:journalctl@ceph.prometheus.a.smithi164.stdout:Jun 03 19:40:36 smithi164 bash[55188]: ts=2023-06-03T19:40:36.397Z caller=head.go:604 level=info component=tsdb msg="WAL segment loaded" segment=2 maxSegment=3 2023-06-03T19:40:36.408 INFO:journalctl@ceph.prometheus.a.smithi164.stdout:Jun 03 19:40:36 smithi164 bash[55188]: ts=2023-06-03T19:40:36.398Z caller=head.go:604 level=info component=tsdb msg="WAL segment loaded" segment=3 maxSegment=3 2023-06-03T19:40:36.408 INFO:journalctl@ceph.prometheus.a.smithi164.stdout:Jun 03 19:40:36 smithi164 bash[55188]: ts=2023-06-03T19:40:36.398Z caller=head.go:610 level=info component=tsdb msg="WAL replay completed" checkpoint_replay_duration=58.954µs wal_replay_duration=1.158254917s total_replay_duration=1.158339284s 2023-06-03T19:40:36.531 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:40:36 smithi089 bash[26572]: audit 2023-06-03T19:40:36.054687+0000 mon.b (mon.1) 68 : audit [DBG] from='client.? 172.21.15.89:0/1457798813' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-03T19:40:36.532 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:40:36 smithi089 bash[33173]: audit 2023-06-03T19:40:36.054687+0000 mon.b (mon.1) 68 : audit [DBG] from='client.? 172.21.15.89:0/1457798813' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-03T19:40:36.806 INFO:journalctl@ceph.prometheus.a.smithi164.stdout:Jun 03 19:40:36 smithi164 bash[55188]: ts=2023-06-03T19:40:36.406Z caller=main.go:944 level=info fs_type=XFS_SUPER_MAGIC 2023-06-03T19:40:36.806 INFO:journalctl@ceph.prometheus.a.smithi164.stdout:Jun 03 19:40:36 smithi164 bash[55188]: ts=2023-06-03T19:40:36.406Z caller=main.go:947 level=info msg="TSDB started" 2023-06-03T19:40:36.806 INFO:journalctl@ceph.prometheus.a.smithi164.stdout:Jun 03 19:40:36 smithi164 bash[55188]: ts=2023-06-03T19:40:36.406Z caller=main.go:1128 level=info msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml 2023-06-03T19:40:36.806 INFO:journalctl@ceph.prometheus.a.smithi164.stdout:Jun 03 19:40:36 smithi164 bash[55188]: ts=2023-06-03T19:40:36.469Z caller=main.go:1165 level=info msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=63.671607ms db_storage=1.058µs remote_storage=2.664µs web_handler=553ns query_engine=1.044µs scrape=356.664µs scrape_sd=121.114µs notify=17.698µs notify_sd=15.85µs rules=61.355781ms 2023-06-03T19:40:36.806 INFO:journalctl@ceph.prometheus.a.smithi164.stdout:Jun 03 19:40:36 smithi164 bash[55188]: ts=2023-06-03T19:40:36.469Z caller=main.go:896 level=info msg="Server is ready to receive web requests." 2023-06-03T19:40:37.531 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:40:37 smithi089 bash[26572]: cluster 2023-06-03T19:40:36.029622+0000 mgr.y (mgr.25070) 33 : cluster [DBG] pgmap v20: 89 pgs: 89 active+clean; 1.4 GiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 583 op/s 2023-06-03T19:40:37.532 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:40:37 smithi089 bash[26572]: audit 2023-06-03T19:40:36.383080+0000 mon.b (mon.1) 69 : audit [INF] from='mgr.25070 172.21.15.89:0/2108989343' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-06-03T19:40:37.532 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:40:37 smithi089 bash[26572]: cephadm 2023-06-03T19:40:36.384079+0000 mgr.y (mgr.25070) 34 : cephadm [INF] Upgrade: Updating mgr.x 2023-06-03T19:40:37.532 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:40:37 smithi089 bash[26572]: audit 2023-06-03T19:40:36.385643+0000 mon.a (mon.0) 983 : audit [INF] from='mgr.25070 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-06-03T19:40:37.532 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:40:37 smithi089 bash[26572]: audit 2023-06-03T19:40:36.385774+0000 mon.b (mon.1) 70 : audit [DBG] from='mgr.25070 172.21.15.89:0/2108989343' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-06-03T19:40:37.532 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:40:37 smithi089 bash[26572]: audit 2023-06-03T19:40:36.387073+0000 mon.b (mon.1) 71 : audit [DBG] from='mgr.25070 172.21.15.89:0/2108989343' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-03T19:40:37.533 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:40:37 smithi089 bash[26572]: cephadm 2023-06-03T19:40:36.390094+0000 mgr.y (mgr.25070) 35 : cephadm [INF] Deploying daemon mgr.x on smithi164 2023-06-03T19:40:37.533 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:40:37 smithi089 bash[33173]: cluster 2023-06-03T19:40:36.029622+0000 mgr.y (mgr.25070) 33 : cluster [DBG] pgmap v20: 89 pgs: 89 active+clean; 1.4 GiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 583 op/s 2023-06-03T19:40:37.533 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:40:37 smithi089 bash[33173]: audit 2023-06-03T19:40:36.383080+0000 mon.b (mon.1) 69 : audit [INF] from='mgr.25070 172.21.15.89:0/2108989343' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-06-03T19:40:37.533 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:40:37 smithi089 bash[33173]: cephadm 2023-06-03T19:40:36.384079+0000 mgr.y (mgr.25070) 34 : cephadm [INF] Upgrade: Updating mgr.x 2023-06-03T19:40:37.534 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:40:37 smithi089 bash[33173]: audit 2023-06-03T19:40:36.385643+0000 mon.a (mon.0) 983 : audit [INF] from='mgr.25070 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-06-03T19:40:37.534 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:40:37 smithi089 bash[33173]: audit 2023-06-03T19:40:36.385774+0000 mon.b (mon.1) 70 : audit [DBG] from='mgr.25070 172.21.15.89:0/2108989343' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-06-03T19:40:37.534 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:40:37 smithi089 bash[33173]: audit 2023-06-03T19:40:36.387073+0000 mon.b (mon.1) 71 : audit [DBG] from='mgr.25070 172.21.15.89:0/2108989343' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-03T19:40:37.534 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:40:37 smithi089 bash[33173]: cephadm 2023-06-03T19:40:36.390094+0000 mgr.y (mgr.25070) 35 : cephadm [INF] Deploying daemon mgr.x on smithi164 2023-06-03T19:40:37.586 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:40:37 smithi164 bash[26855]: cluster 2023-06-03T19:40:36.029622+0000 mgr.y (mgr.25070) 33 : cluster [DBG] pgmap v20: 89 pgs: 89 active+clean; 1.4 GiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 583 op/s 2023-06-03T19:40:37.586 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:40:37 smithi164 bash[26855]: audit 2023-06-03T19:40:36.383080+0000 mon.b (mon.1) 69 : audit [INF] from='mgr.25070 172.21.15.89:0/2108989343' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-06-03T19:40:37.586 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:40:37 smithi164 bash[26855]: cephadm 2023-06-03T19:40:36.384079+0000 mgr.y (mgr.25070) 34 : cephadm [INF] Upgrade: Updating mgr.x 2023-06-03T19:40:37.586 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:40:37 smithi164 bash[26855]: audit 2023-06-03T19:40:36.385643+0000 mon.a (mon.0) 983 : audit [INF] from='mgr.25070 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-06-03T19:40:37.586 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:40:37 smithi164 bash[26855]: audit 2023-06-03T19:40:36.385774+0000 mon.b (mon.1) 70 : audit [DBG] from='mgr.25070 172.21.15.89:0/2108989343' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-06-03T19:40:37.587 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:40:37 smithi164 bash[26855]: audit 2023-06-03T19:40:36.387073+0000 mon.b (mon.1) 71 : audit [DBG] from='mgr.25070 172.21.15.89:0/2108989343' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-03T19:40:37.587 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:40:37 smithi164 bash[26855]: cephadm 2023-06-03T19:40:36.390094+0000 mgr.y (mgr.25070) 35 : cephadm [INF] Deploying daemon mgr.x on smithi164 2023-06-03T19:40:39.430 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:40:39 smithi164 bash[26855]: cluster 2023-06-03T19:40:38.030071+0000 mgr.y (mgr.25070) 36 : cluster [DBG] pgmap v21: 89 pgs: 89 active+clean; 1.4 GiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 583 op/s 2023-06-03T19:40:39.430 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:40:39 smithi164 bash[26855]: audit 2023-06-03T19:40:38.747682+0000 mon.c (mon.2) 67 : audit [DBG] from='client.? 172.21.15.89:0/3060279139' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-03T19:40:39.431 INFO:journalctl@ceph.mgr.x.smithi164.stdout:Jun 03 19:40:39 smithi164 systemd[1]: Stopping Ceph mgr.x for f1db3478-0243-11ee-9b1d-001a4aab830c... 2023-06-03T19:40:39.531 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:40:39 smithi089 bash[26572]: cluster 2023-06-03T19:40:38.030071+0000 mgr.y (mgr.25070) 36 : cluster [DBG] pgmap v21: 89 pgs: 89 active+clean; 1.4 GiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 583 op/s 2023-06-03T19:40:39.532 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:40:39 smithi089 bash[26572]: audit 2023-06-03T19:40:38.747682+0000 mon.c (mon.2) 67 : audit [DBG] from='client.? 172.21.15.89:0/3060279139' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-03T19:40:39.532 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:40:39 smithi089 bash[33173]: cluster 2023-06-03T19:40:38.030071+0000 mgr.y (mgr.25070) 36 : cluster [DBG] pgmap v21: 89 pgs: 89 active+clean; 1.4 GiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 583 op/s 2023-06-03T19:40:39.532 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:40:39 smithi089 bash[33173]: audit 2023-06-03T19:40:38.747682+0000 mon.c (mon.2) 67 : audit [DBG] from='client.? 172.21.15.89:0/3060279139' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-03T19:40:40.982 INFO:journalctl@ceph.mgr.x.smithi164.stdout:Jun 03 19:40:40 smithi164 bash[55476]: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-mgr-x 2023-06-03T19:40:40.982 INFO:journalctl@ceph.mgr.x.smithi164.stdout:Jun 03 19:40:40 smithi164 systemd[1]: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c@mgr.x.service: Main process exited, code=exited, status=143/n/a 2023-06-03T19:40:40.983 INFO:journalctl@ceph.mgr.x.smithi164.stdout:Jun 03 19:40:40 smithi164 systemd[1]: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c@mgr.x.service: Failed with result 'exit-code'. 2023-06-03T19:40:40.983 INFO:journalctl@ceph.mgr.x.smithi164.stdout:Jun 03 19:40:40 smithi164 systemd[1]: Stopped Ceph mgr.x for f1db3478-0243-11ee-9b1d-001a4aab830c. 2023-06-03T19:40:40.983 INFO:journalctl@ceph.mgr.x.smithi164.stdout:Jun 03 19:40:40 smithi164 systemd[1]: Started Ceph mgr.x for f1db3478-0243-11ee-9b1d-001a4aab830c. 2023-06-03T19:40:41.336 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:40:41 smithi164 bash[26855]: cluster 2023-06-03T19:40:40.033718+0000 mgr.y (mgr.25070) 37 : cluster [DBG] pgmap v22: 89 pgs: 89 active+clean; 1.2 GiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 874 op/s 2023-06-03T19:40:41.336 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:40:41 smithi164 bash[26855]: audit 2023-06-03T19:40:41.023328+0000 mon.a (mon.0) 984 : audit [INF] from='mgr.25070 ' entity='mgr.y' 2023-06-03T19:40:41.336 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:40:41 smithi164 bash[26855]: audit 2023-06-03T19:40:41.040015+0000 mon.a (mon.0) 985 : audit [INF] from='mgr.25070 ' entity='mgr.y' 2023-06-03T19:40:41.337 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:40:41 smithi164 bash[26855]: audit 2023-06-03T19:40:41.072518+0000 mon.b (mon.1) 72 : audit [DBG] from='mgr.25070 172.21.15.89:0/2108989343' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-03T19:40:41.531 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:40:41 smithi089 bash[26572]: cluster 2023-06-03T19:40:40.033718+0000 mgr.y (mgr.25070) 37 : cluster [DBG] pgmap v22: 89 pgs: 89 active+clean; 1.2 GiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 874 op/s 2023-06-03T19:40:41.532 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:40:41 smithi089 bash[26572]: audit 2023-06-03T19:40:41.023328+0000 mon.a (mon.0) 984 : audit [INF] from='mgr.25070 ' entity='mgr.y' 2023-06-03T19:40:41.532 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:40:41 smithi089 bash[26572]: audit 2023-06-03T19:40:41.040015+0000 mon.a (mon.0) 985 : audit [INF] from='mgr.25070 ' entity='mgr.y' 2023-06-03T19:40:41.532 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:40:41 smithi089 bash[26572]: audit 2023-06-03T19:40:41.072518+0000 mon.b (mon.1) 72 : audit [DBG] from='mgr.25070 172.21.15.89:0/2108989343' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-03T19:40:41.532 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:40:41 smithi089 bash[33173]: cluster 2023-06-03T19:40:40.033718+0000 mgr.y (mgr.25070) 37 : cluster [DBG] pgmap v22: 89 pgs: 89 active+clean; 1.2 GiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 874 op/s 2023-06-03T19:40:41.533 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:40:41 smithi089 bash[33173]: audit 2023-06-03T19:40:41.023328+0000 mon.a (mon.0) 984 : audit [INF] from='mgr.25070 ' entity='mgr.y' 2023-06-03T19:40:41.533 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:40:41 smithi089 bash[33173]: audit 2023-06-03T19:40:41.040015+0000 mon.a (mon.0) 985 : audit [INF] from='mgr.25070 ' entity='mgr.y' 2023-06-03T19:40:41.533 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:40:41 smithi089 bash[33173]: audit 2023-06-03T19:40:41.072518+0000 mon.b (mon.1) 72 : audit [DBG] from='mgr.25070 172.21.15.89:0/2108989343' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-03T19:40:42.493 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:40:42 smithi164 bash[26855]: audit 2023-06-03T19:40:41.436198+0000 mon.a (mon.0) 986 : audit [DBG] from='client.? 172.21.15.89:0/4101479108' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-03T19:40:42.531 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:40:42 smithi089 bash[26572]: audit 2023-06-03T19:40:41.436198+0000 mon.a (mon.0) 986 : audit [DBG] from='client.? 172.21.15.89:0/4101479108' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-03T19:40:42.532 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:40:42 smithi089 bash[33173]: audit 2023-06-03T19:40:41.436198+0000 mon.a (mon.0) 986 : audit [DBG] from='client.? 172.21.15.89:0/4101479108' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-03T19:40:43.336 INFO:journalctl@ceph.mgr.x.smithi164.stdout:Jun 03 19:40:42 smithi164 bash[55600]: debug 2023-06-03T19:40:42.896+0000 7ff247918200 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2023-06-03T19:40:44.031 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:40:43 smithi089 bash[33173]: cluster 2023-06-03T19:40:42.034281+0000 mgr.y (mgr.25070) 38 : cluster [DBG] pgmap v23: 89 pgs: 89 active+clean; 1.2 GiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 585 op/s 2023-06-03T19:40:44.031 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:40:43 smithi089 bash[26572]: cluster 2023-06-03T19:40:42.034281+0000 mgr.y (mgr.25070) 38 : cluster [DBG] pgmap v23: 89 pgs: 89 active+clean; 1.2 GiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 585 op/s 2023-06-03T19:40:44.086 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:40:43 smithi164 bash[26855]: cluster 2023-06-03T19:40:42.034281+0000 mgr.y (mgr.25070) 38 : cluster [DBG] pgmap v23: 89 pgs: 89 active+clean; 1.2 GiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 585 op/s 2023-06-03T19:40:44.471 INFO:journalctl@ceph.mgr.x.smithi164.stdout:Jun 03 19:40:44 smithi164 bash[55600]: debug 2023-06-03T19:40:44.152+0000 7ff247918200 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2023-06-03T19:40:44.471 INFO:journalctl@ceph.mgr.x.smithi164.stdout:Jun 03 19:40:44 smithi164 bash[55600]: debug 2023-06-03T19:40:44.300+0000 7ff247918200 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2023-06-03T19:40:44.471 INFO:journalctl@ceph.mgr.x.smithi164.stdout:Jun 03 19:40:44 smithi164 bash[55600]: debug 2023-06-03T19:40:44.460+0000 7ff247918200 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2023-06-03T19:40:44.836 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:40:44 smithi164 bash[26855]: audit 2023-06-03T19:40:44.118137+0000 mon.b (mon.1) 73 : audit [DBG] from='client.? 172.21.15.89:0/2477891359' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-03T19:40:45.031 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:40:44 smithi089 bash[33173]: audit 2023-06-03T19:40:44.118137+0000 mon.b (mon.1) 73 : audit [DBG] from='client.? 172.21.15.89:0/2477891359' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-03T19:40:45.031 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:40:44 smithi089 bash[26572]: audit 2023-06-03T19:40:44.118137+0000 mon.b (mon.1) 73 : audit [DBG] from='client.? 172.21.15.89:0/2477891359' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-03T19:40:45.336 INFO:journalctl@ceph.mgr.x.smithi164.stdout:Jun 03 19:40:44 smithi164 bash[55600]: debug 2023-06-03T19:40:44.848+0000 7ff247918200 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2023-06-03T19:40:45.649 INFO:journalctl@ceph.mgr.x.smithi164.stdout:Jun 03 19:40:45 smithi164 bash[55600]: debug 2023-06-03T19:40:45.380+0000 7ff247918200 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2023-06-03T19:40:45.649 INFO:journalctl@ceph.mgr.x.smithi164.stdout:Jun 03 19:40:45 smithi164 bash[55600]: debug 2023-06-03T19:40:45.612+0000 7ff247918200 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2023-06-03T19:40:45.940 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:40:45 smithi164 bash[26855]: cluster 2023-06-03T19:40:44.035163+0000 mgr.y (mgr.25070) 39 : cluster [DBG] pgmap v24: 89 pgs: 89 active+clean; 980 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 876 op/s 2023-06-03T19:40:45.940 INFO:journalctl@ceph.mgr.x.smithi164.stdout:Jun 03 19:40:45 smithi164 bash[55600]: debug 2023-06-03T19:40:45.928+0000 7ff247918200 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2023-06-03T19:40:46.031 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:40:45 smithi089 bash[33173]: cluster 2023-06-03T19:40:44.035163+0000 mgr.y (mgr.25070) 39 : cluster [DBG] pgmap v24: 89 pgs: 89 active+clean; 980 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 876 op/s 2023-06-03T19:40:46.031 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:40:45 smithi089 bash[26572]: cluster 2023-06-03T19:40:44.035163+0000 mgr.y (mgr.25070) 39 : cluster [DBG] pgmap v24: 89 pgs: 89 active+clean; 980 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 876 op/s 2023-06-03T19:40:46.531 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 19:40:46 smithi089 bash[71163]: ::ffff:172.21.15.164 - - [03/Jun/2023:19:40:46] "GET /metrics HTTP/1.1" 200 190031 "" "Prometheus/2.33.4" 2023-06-03T19:40:47.807 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:40:47 smithi164 bash[26855]: cluster 2023-06-03T19:40:46.035557+0000 mgr.y (mgr.25070) 40 : cluster [DBG] pgmap v25: 89 pgs: 89 active+clean; 980 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 582 op/s 2023-06-03T19:40:47.807 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:40:47 smithi164 bash[26855]: audit 2023-06-03T19:40:46.771584+0000 mon.b (mon.1) 74 : audit [DBG] from='client.? 172.21.15.89:0/2396555591' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-03T19:40:47.807 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:40:47 smithi164 bash[26855]: audit 2023-06-03T19:40:47.205151+0000 mon.b (mon.1) 75 : audit [DBG] from='mgr.25070 172.21.15.89:0/2108989343' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T19:40:47.808 INFO:journalctl@ceph.mgr.x.smithi164.stdout:Jun 03 19:40:47 smithi164 bash[55600]: debug 2023-06-03T19:40:47.440+0000 7ff247918200 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2023-06-03T19:40:47.808 INFO:journalctl@ceph.mgr.x.smithi164.stdout:Jun 03 19:40:47 smithi164 bash[55600]: debug 2023-06-03T19:40:47.620+0000 7ff247918200 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2023-06-03T19:40:47.808 INFO:journalctl@ceph.mgr.x.smithi164.stdout:Jun 03 19:40:47 smithi164 bash[55600]: debug 2023-06-03T19:40:47.792+0000 7ff247918200 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2023-06-03T19:40:48.031 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:40:47 smithi089 bash[33173]: cluster 2023-06-03T19:40:46.035557+0000 mgr.y (mgr.25070) 40 : cluster [DBG] pgmap v25: 89 pgs: 89 active+clean; 980 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 582 op/s 2023-06-03T19:40:48.031 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:40:47 smithi089 bash[33173]: audit 2023-06-03T19:40:46.771584+0000 mon.b (mon.1) 74 : audit [DBG] from='client.? 172.21.15.89:0/2396555591' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-03T19:40:48.032 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:40:47 smithi089 bash[33173]: audit 2023-06-03T19:40:47.205151+0000 mon.b (mon.1) 75 : audit [DBG] from='mgr.25070 172.21.15.89:0/2108989343' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T19:40:48.032 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:40:47 smithi089 bash[26572]: cluster 2023-06-03T19:40:46.035557+0000 mgr.y (mgr.25070) 40 : cluster [DBG] pgmap v25: 89 pgs: 89 active+clean; 980 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 582 op/s 2023-06-03T19:40:48.032 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:40:47 smithi089 bash[26572]: audit 2023-06-03T19:40:46.771584+0000 mon.b (mon.1) 74 : audit [DBG] from='client.? 172.21.15.89:0/2396555591' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-03T19:40:48.032 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:40:47 smithi089 bash[26572]: audit 2023-06-03T19:40:47.205151+0000 mon.b (mon.1) 75 : audit [DBG] from='mgr.25070 172.21.15.89:0/2108989343' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T19:40:48.836 INFO:journalctl@ceph.mgr.x.smithi164.stdout:Jun 03 19:40:48 smithi164 bash[55600]: debug 2023-06-03T19:40:48.528+0000 7ff247918200 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2023-06-03T19:40:48.836 INFO:journalctl@ceph.prometheus.a.smithi164.stdout:Jun 03 19:40:48 smithi164 bash[55188]: ts=2023-06-03T19:40:48.662Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.13-257-gd8c5d349 (d8c5d34975dce1c5eb0aa3a7979a4d9b9a99d1ec) pacific (stable)\", cluster_addr=\"172.21.15.89\", device_class=\"ssd\", hostname=\"smithi089\", instance=\"smithi164.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.89\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.13-257-gd8c5d349 (d8c5d34975dce1c5eb0aa3a7979a4d9b9a99d1ec) pacific (stable)\", cluster_addr=\"172.21.15.89\", device_class=\"ssd\", hostname=\"smithi089\", instance=\"smithi089:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.89\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-06-03T19:40:48.836 INFO:journalctl@ceph.prometheus.a.smithi164.stdout:Jun 03 19:40:48 smithi164 bash[55188]: ts=2023-06-03T19:40:48.664Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephPGImbalance\nexpr: abs(((ceph_osd_numpg > 0) - on(job) group_left() avg by(job) (ceph_osd_numpg\n > 0)) / on(job) group_left() avg by(job) (ceph_osd_numpg > 0)) * on(ceph_daemon)\n group_left(hostname) ceph_osd_metadata > 0.3\nfor: 5m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.5\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} deviates by\n more than 30% from average PG count.\n summary: PGs are not balanced across OSDs\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.13-257-gd8c5d349 (d8c5d34975dce1c5eb0aa3a7979a4d9b9a99d1ec) pacific (stable)\", cluster_addr=\"172.21.15.89\", device_class=\"ssd\", hostname=\"smithi089\", instance=\"smithi164.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.89\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.13-257-gd8c5d349 (d8c5d34975dce1c5eb0aa3a7979a4d9b9a99d1ec) pacific (stable)\", cluster_addr=\"172.21.15.89\", device_class=\"ssd\", hostname=\"smithi089\", instance=\"smithi089:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.89\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-06-03T19:40:49.335 INFO:journalctl@ceph.mgr.x.smithi164.stdout:Jun 03 19:40:48 smithi164 bash[55600]: debug 2023-06-03T19:40:48.908+0000 7ff247918200 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2023-06-03T19:40:49.836 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:40:49 smithi164 bash[26855]: cluster 2023-06-03T19:40:48.036075+0000 mgr.y (mgr.25070) 41 : cluster [DBG] pgmap v26: 89 pgs: 89 active+clean; 980 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 582 op/s 2023-06-03T19:40:49.837 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:40:49 smithi164 bash[26855]: audit 2023-06-03T19:40:49.136467+0000 mon.a (mon.0) 987 : audit [INF] from='mgr.25070 ' entity='mgr.y' 2023-06-03T19:40:49.837 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:40:49 smithi164 bash[26855]: audit 2023-06-03T19:40:49.150300+0000 mon.a (mon.0) 988 : audit [INF] from='mgr.25070 ' entity='mgr.y' 2023-06-03T19:40:49.837 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:40:49 smithi164 bash[26855]: audit 2023-06-03T19:40:49.394413+0000 mon.a (mon.0) 989 : audit [DBG] from='client.? 172.21.15.89:0/1105714080' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-03T19:40:49.838 INFO:journalctl@ceph.mgr.x.smithi164.stdout:Jun 03 19:40:49 smithi164 bash[55600]: debug 2023-06-03T19:40:49.472+0000 7ff247918200 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2023-06-03T19:40:50.031 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:40:49 smithi089 bash[26572]: cluster 2023-06-03T19:40:48.036075+0000 mgr.y (mgr.25070) 41 : cluster [DBG] pgmap v26: 89 pgs: 89 active+clean; 980 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 582 op/s 2023-06-03T19:40:50.031 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:40:49 smithi089 bash[26572]: audit 2023-06-03T19:40:49.136467+0000 mon.a (mon.0) 987 : audit [INF] from='mgr.25070 ' entity='mgr.y' 2023-06-03T19:40:50.032 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:40:49 smithi089 bash[26572]: audit 2023-06-03T19:40:49.150300+0000 mon.a (mon.0) 988 : audit [INF] from='mgr.25070 ' entity='mgr.y' 2023-06-03T19:40:50.032 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:40:49 smithi089 bash[26572]: audit 2023-06-03T19:40:49.394413+0000 mon.a (mon.0) 989 : audit [DBG] from='client.? 172.21.15.89:0/1105714080' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-03T19:40:50.032 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:40:49 smithi089 bash[33173]: cluster 2023-06-03T19:40:48.036075+0000 mgr.y (mgr.25070) 41 : cluster [DBG] pgmap v26: 89 pgs: 89 active+clean; 980 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 582 op/s 2023-06-03T19:40:50.032 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:40:49 smithi089 bash[33173]: audit 2023-06-03T19:40:49.136467+0000 mon.a (mon.0) 987 : audit [INF] from='mgr.25070 ' entity='mgr.y' 2023-06-03T19:40:50.032 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:40:49 smithi089 bash[33173]: audit 2023-06-03T19:40:49.150300+0000 mon.a (mon.0) 988 : audit [INF] from='mgr.25070 ' entity='mgr.y' 2023-06-03T19:40:50.032 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:40:49 smithi089 bash[33173]: audit 2023-06-03T19:40:49.394413+0000 mon.a (mon.0) 989 : audit [DBG] from='client.? 172.21.15.89:0/1105714080' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-03T19:40:50.586 INFO:journalctl@ceph.mgr.x.smithi164.stdout:Jun 03 19:40:50 smithi164 bash[55600]: debug 2023-06-03T19:40:50.204+0000 7ff247918200 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2023-06-03T19:40:51.336 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:40:51 smithi164 bash[26855]: cluster 2023-06-03T19:40:50.036782+0000 mgr.y (mgr.25070) 42 : cluster [DBG] pgmap v27: 89 pgs: 89 active+clean; 780 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 848 op/s 2023-06-03T19:40:51.336 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:40:51 smithi164 bash[26855]: audit 2023-06-03T19:40:50.098257+0000 mon.a (mon.0) 990 : audit [INF] from='mgr.25070 ' entity='mgr.y' 2023-06-03T19:40:51.336 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:40:51 smithi164 bash[26855]: audit 2023-06-03T19:40:50.112426+0000 mon.a (mon.0) 991 : audit [INF] from='mgr.25070 ' entity='mgr.y' 2023-06-03T19:40:51.336 INFO:journalctl@ceph.mgr.x.smithi164.stdout:Jun 03 19:40:51 smithi164 bash[55600]: debug 2023-06-03T19:40:51.008+0000 7ff247918200 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2023-06-03T19:40:51.337 INFO:journalctl@ceph.mgr.x.smithi164.stdout:Jun 03 19:40:51 smithi164 bash[55600]: debug 2023-06-03T19:40:51.164+0000 7ff247918200 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2023-06-03T19:40:51.531 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:40:51 smithi089 bash[26572]: cluster 2023-06-03T19:40:50.036782+0000 mgr.y (mgr.25070) 42 : cluster [DBG] pgmap v27: 89 pgs: 89 active+clean; 780 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 848 op/s 2023-06-03T19:40:51.531 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:40:51 smithi089 bash[26572]: audit 2023-06-03T19:40:50.098257+0000 mon.a (mon.0) 990 : audit [INF] from='mgr.25070 ' entity='mgr.y' 2023-06-03T19:40:51.532 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:40:51 smithi089 bash[26572]: audit 2023-06-03T19:40:50.112426+0000 mon.a (mon.0) 991 : audit [INF] from='mgr.25070 ' entity='mgr.y' 2023-06-03T19:40:51.532 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:40:51 smithi089 bash[33173]: cluster 2023-06-03T19:40:50.036782+0000 mgr.y (mgr.25070) 42 : cluster [DBG] pgmap v27: 89 pgs: 89 active+clean; 780 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 848 op/s 2023-06-03T19:40:51.532 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:40:51 smithi089 bash[33173]: audit 2023-06-03T19:40:50.098257+0000 mon.a (mon.0) 990 : audit [INF] from='mgr.25070 ' entity='mgr.y' 2023-06-03T19:40:51.532 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:40:51 smithi089 bash[33173]: audit 2023-06-03T19:40:50.112426+0000 mon.a (mon.0) 991 : audit [INF] from='mgr.25070 ' entity='mgr.y' 2023-06-03T19:40:52.048 INFO:journalctl@ceph.mgr.x.smithi164.stdout:Jun 03 19:40:51 smithi164 bash[55600]: debug 2023-06-03T19:40:51.660+0000 7ff247918200 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2023-06-03T19:40:52.048 INFO:journalctl@ceph.mgr.x.smithi164.stdout:Jun 03 19:40:51 smithi164 bash[55600]: debug 2023-06-03T19:40:51.836+0000 7ff247918200 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2023-06-03T19:40:52.048 INFO:journalctl@ceph.mgr.x.smithi164.stdout:Jun 03 19:40:52 smithi164 bash[55600]: debug 2023-06-03T19:40:52.036+0000 7ff247918200 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2023-06-03T19:40:52.336 INFO:journalctl@ceph.mgr.x.smithi164.stdout:Jun 03 19:40:52 smithi164 bash[55600]: debug 2023-06-03T19:40:52.204+0000 7ff247918200 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2023-06-03T19:40:52.654 INFO:journalctl@ceph.mgr.x.smithi164.stdout:Jun 03 19:40:52 smithi164 bash[55600]: debug 2023-06-03T19:40:52.376+0000 7ff247918200 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2023-06-03T19:40:53.031 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:40:52 smithi089 bash[33173]: audit 2023-06-03T19:40:52.050359+0000 mon.c (mon.2) 68 : audit [DBG] from='client.? 172.21.15.89:0/2364630483' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-03T19:40:53.031 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:40:52 smithi089 bash[26572]: audit 2023-06-03T19:40:52.050359+0000 mon.c (mon.2) 68 : audit [DBG] from='client.? 172.21.15.89:0/2364630483' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-03T19:40:53.086 INFO:journalctl@ceph.mgr.x.smithi164.stdout:Jun 03 19:40:52 smithi164 bash[55600]: debug 2023-06-03T19:40:52.664+0000 7ff247918200 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2023-06-03T19:40:53.086 INFO:journalctl@ceph.mgr.x.smithi164.stdout:Jun 03 19:40:52 smithi164 bash[55600]: debug 2023-06-03T19:40:52.836+0000 7ff247918200 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2023-06-03T19:40:53.086 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:40:52 smithi164 bash[26855]: audit 2023-06-03T19:40:52.050359+0000 mon.c (mon.2) 68 : audit [DBG] from='client.? 172.21.15.89:0/2364630483' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-03T19:40:54.031 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:40:53 smithi089 bash[26572]: cluster 2023-06-03T19:40:52.037217+0000 mgr.y (mgr.25070) 43 : cluster [DBG] pgmap v28: 89 pgs: 89 active+clean; 780 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 557 op/s 2023-06-03T19:40:54.032 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:40:53 smithi089 bash[33173]: cluster 2023-06-03T19:40:52.037217+0000 mgr.y (mgr.25070) 43 : cluster [DBG] pgmap v28: 89 pgs: 89 active+clean; 780 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 557 op/s 2023-06-03T19:40:54.086 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:40:53 smithi164 bash[26855]: cluster 2023-06-03T19:40:52.037217+0000 mgr.y (mgr.25070) 43 : cluster [DBG] pgmap v28: 89 pgs: 89 active+clean; 780 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 557 op/s 2023-06-03T19:40:55.031 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:40:54 smithi089 bash[26572]: cluster 2023-06-03T19:40:54.622523+0000 mon.a (mon.0) 992 : cluster [DBG] Standby manager daemon x restarted 2023-06-03T19:40:55.032 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:40:54 smithi089 bash[26572]: cluster 2023-06-03T19:40:54.622666+0000 mon.a (mon.0) 993 : cluster [DBG] Standby manager daemon x started 2023-06-03T19:40:55.032 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:40:54 smithi089 bash[26572]: audit 2023-06-03T19:40:54.628591+0000 mon.c (mon.2) 69 : audit [DBG] from='mgr.? 172.21.15.164:0/1707464496' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-06-03T19:40:55.032 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:40:54 smithi089 bash[26572]: audit 2023-06-03T19:40:54.633878+0000 mon.c (mon.2) 70 : audit [DBG] from='mgr.? 172.21.15.164:0/1707464496' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-06-03T19:40:55.032 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:40:54 smithi089 bash[26572]: audit 2023-06-03T19:40:54.637077+0000 mon.c (mon.2) 71 : audit [DBG] from='mgr.? 172.21.15.164:0/1707464496' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-06-03T19:40:55.032 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:40:54 smithi089 bash[26572]: audit 2023-06-03T19:40:54.637883+0000 mon.c (mon.2) 72 : audit [DBG] from='mgr.? 172.21.15.164:0/1707464496' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-06-03T19:40:55.033 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:40:54 smithi089 bash[33173]: cluster 2023-06-03T19:40:54.622523+0000 mon.a (mon.0) 992 : cluster [DBG] Standby manager daemon x restarted 2023-06-03T19:40:55.033 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:40:54 smithi089 bash[33173]: cluster 2023-06-03T19:40:54.622666+0000 mon.a (mon.0) 993 : cluster [DBG] Standby manager daemon x started 2023-06-03T19:40:55.033 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:40:54 smithi089 bash[33173]: audit 2023-06-03T19:40:54.628591+0000 mon.c (mon.2) 69 : audit [DBG] from='mgr.? 172.21.15.164:0/1707464496' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-06-03T19:40:55.033 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:40:54 smithi089 bash[33173]: audit 2023-06-03T19:40:54.633878+0000 mon.c (mon.2) 70 : audit [DBG] from='mgr.? 172.21.15.164:0/1707464496' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-06-03T19:40:55.033 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:40:54 smithi089 bash[33173]: audit 2023-06-03T19:40:54.637077+0000 mon.c (mon.2) 71 : audit [DBG] from='mgr.? 172.21.15.164:0/1707464496' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-06-03T19:40:55.033 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:40:54 smithi089 bash[33173]: audit 2023-06-03T19:40:54.637883+0000 mon.c (mon.2) 72 : audit [DBG] from='mgr.? 172.21.15.164:0/1707464496' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-06-03T19:40:55.086 INFO:journalctl@ceph.mgr.x.smithi164.stdout:Jun 03 19:40:54 smithi164 bash[55600]: debug 2023-06-03T19:40:54.596+0000 7ff247918200 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2023-06-03T19:40:55.086 INFO:journalctl@ceph.mgr.x.smithi164.stdout:Jun 03 19:40:54 smithi164 bash[55600]: [03/Jun/2023:19:40:54] ENGINE Bus STARTING 2023-06-03T19:40:55.086 INFO:journalctl@ceph.mgr.x.smithi164.stdout:Jun 03 19:40:54 smithi164 bash[55600]: CherryPy Checker: 2023-06-03T19:40:55.086 INFO:journalctl@ceph.mgr.x.smithi164.stdout:Jun 03 19:40:54 smithi164 bash[55600]: The Application mounted at '' has an empty config. 2023-06-03T19:40:55.086 INFO:journalctl@ceph.mgr.x.smithi164.stdout:Jun 03 19:40:54 smithi164 bash[55600]: [03/Jun/2023:19:40:54] ENGINE Serving on http://:::9283 2023-06-03T19:40:55.087 INFO:journalctl@ceph.mgr.x.smithi164.stdout:Jun 03 19:40:54 smithi164 bash[55600]: [03/Jun/2023:19:40:54] ENGINE Bus STARTED 2023-06-03T19:40:55.087 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:40:54 smithi164 bash[26855]: cluster 2023-06-03T19:40:54.622523+0000 mon.a (mon.0) 992 : cluster [DBG] Standby manager daemon x restarted 2023-06-03T19:40:55.087 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:40:54 smithi164 bash[26855]: cluster 2023-06-03T19:40:54.622666+0000 mon.a (mon.0) 993 : cluster [DBG] Standby manager daemon x started 2023-06-03T19:40:55.087 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:40:54 smithi164 bash[26855]: audit 2023-06-03T19:40:54.628591+0000 mon.c (mon.2) 69 : audit [DBG] from='mgr.? 172.21.15.164:0/1707464496' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-06-03T19:40:55.087 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:40:54 smithi164 bash[26855]: audit 2023-06-03T19:40:54.633878+0000 mon.c (mon.2) 70 : audit [DBG] from='mgr.? 172.21.15.164:0/1707464496' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-06-03T19:40:55.087 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:40:54 smithi164 bash[26855]: audit 2023-06-03T19:40:54.637077+0000 mon.c (mon.2) 71 : audit [DBG] from='mgr.? 172.21.15.164:0/1707464496' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-06-03T19:40:55.088 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:40:54 smithi164 bash[26855]: audit 2023-06-03T19:40:54.637883+0000 mon.c (mon.2) 72 : audit [DBG] from='mgr.? 172.21.15.164:0/1707464496' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-06-03T19:40:56.031 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:40:55 smithi089 bash[33173]: cluster 2023-06-03T19:40:54.041359+0000 mgr.y (mgr.25070) 44 : cluster [DBG] pgmap v29: 89 pgs: 89 active+clean; 586 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 816 op/s 2023-06-03T19:40:56.031 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:40:55 smithi089 bash[33173]: cluster 2023-06-03T19:40:54.687737+0000 mon.a (mon.0) 994 : cluster [DBG] mgrmap e31: y(active, since 53s), standbys: x 2023-06-03T19:40:56.032 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:40:55 smithi089 bash[33173]: audit 2023-06-03T19:40:54.733782+0000 mon.c (mon.2) 73 : audit [DBG] from='client.? 172.21.15.89:0/2482440746' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-03T19:40:56.032 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:40:55 smithi089 bash[26572]: cluster 2023-06-03T19:40:54.041359+0000 mgr.y (mgr.25070) 44 : cluster [DBG] pgmap v29: 89 pgs: 89 active+clean; 586 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 816 op/s 2023-06-03T19:40:56.032 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:40:55 smithi089 bash[26572]: cluster 2023-06-03T19:40:54.687737+0000 mon.a (mon.0) 994 : cluster [DBG] mgrmap e31: y(active, since 53s), standbys: x 2023-06-03T19:40:56.032 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:40:55 smithi089 bash[26572]: audit 2023-06-03T19:40:54.733782+0000 mon.c (mon.2) 73 : audit [DBG] from='client.? 172.21.15.89:0/2482440746' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-03T19:40:56.086 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:40:55 smithi164 bash[26855]: cluster 2023-06-03T19:40:54.041359+0000 mgr.y (mgr.25070) 44 : cluster [DBG] pgmap v29: 89 pgs: 89 active+clean; 586 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 816 op/s 2023-06-03T19:40:56.086 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:40:55 smithi164 bash[26855]: cluster 2023-06-03T19:40:54.687737+0000 mon.a (mon.0) 994 : cluster [DBG] mgrmap e31: y(active, since 53s), standbys: x 2023-06-03T19:40:56.086 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:40:55 smithi164 bash[26855]: audit 2023-06-03T19:40:54.733782+0000 mon.c (mon.2) 73 : audit [DBG] from='client.? 172.21.15.89:0/2482440746' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-03T19:40:56.531 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 19:40:56 smithi089 bash[71163]: ::ffff:172.21.15.164 - - [03/Jun/2023:19:40:56] "GET /metrics HTTP/1.1" 200 190042 "" "Prometheus/2.33.4" 2023-06-03T19:40:57.833 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:40:57 smithi164 bash[26855]: cluster 2023-06-03T19:40:56.041784+0000 mgr.y (mgr.25070) 45 : cluster [DBG] pgmap v30: 89 pgs: 89 active+clean; 586 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 525 op/s 2023-06-03T19:40:57.833 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:40:57 smithi164 bash[26855]: audit 2023-06-03T19:40:57.416048+0000 mon.c (mon.2) 74 : audit [DBG] from='client.? 172.21.15.89:0/145921194' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-03T19:40:58.031 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:40:57 smithi089 bash[33173]: cluster 2023-06-03T19:40:56.041784+0000 mgr.y (mgr.25070) 45 : cluster [DBG] pgmap v30: 89 pgs: 89 active+clean; 586 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 525 op/s 2023-06-03T19:40:58.031 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:40:57 smithi089 bash[33173]: audit 2023-06-03T19:40:57.416048+0000 mon.c (mon.2) 74 : audit [DBG] from='client.? 172.21.15.89:0/145921194' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-03T19:40:58.032 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:40:57 smithi089 bash[26572]: cluster 2023-06-03T19:40:56.041784+0000 mgr.y (mgr.25070) 45 : cluster [DBG] pgmap v30: 89 pgs: 89 active+clean; 586 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 525 op/s 2023-06-03T19:40:58.032 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:40:57 smithi089 bash[26572]: audit 2023-06-03T19:40:57.416048+0000 mon.c (mon.2) 74 : audit [DBG] from='client.? 172.21.15.89:0/145921194' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-03T19:40:59.086 INFO:journalctl@ceph.prometheus.a.smithi164.stdout:Jun 03 19:40:58 smithi164 bash[55188]: ts=2023-06-03T19:40:58.661Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.13-257-gd8c5d349 (d8c5d34975dce1c5eb0aa3a7979a4d9b9a99d1ec) pacific (stable)\", cluster_addr=\"172.21.15.89\", device_class=\"ssd\", hostname=\"smithi089\", instance=\"smithi164.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.89\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.13-257-gd8c5d349 (d8c5d34975dce1c5eb0aa3a7979a4d9b9a99d1ec) pacific (stable)\", cluster_addr=\"172.21.15.89\", device_class=\"ssd\", hostname=\"smithi089\", instance=\"smithi089:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.89\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-06-03T19:40:59.086 INFO:journalctl@ceph.prometheus.a.smithi164.stdout:Jun 03 19:40:58 smithi164 bash[55188]: ts=2023-06-03T19:40:58.662Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephPGImbalance\nexpr: abs(((ceph_osd_numpg > 0) - on(job) group_left() avg by(job) (ceph_osd_numpg\n > 0)) / on(job) group_left() avg by(job) (ceph_osd_numpg > 0)) * on(ceph_daemon)\n group_left(hostname) ceph_osd_metadata > 0.3\nfor: 5m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.5\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} deviates by\n more than 30% from average PG count.\n summary: PGs are not balanced across OSDs\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.13-257-gd8c5d349 (d8c5d34975dce1c5eb0aa3a7979a4d9b9a99d1ec) pacific (stable)\", cluster_addr=\"172.21.15.89\", device_class=\"ssd\", hostname=\"smithi089\", instance=\"smithi164.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.89\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.13-257-gd8c5d349 (d8c5d34975dce1c5eb0aa3a7979a4d9b9a99d1ec) pacific (stable)\", cluster_addr=\"172.21.15.89\", device_class=\"ssd\", hostname=\"smithi089\", instance=\"smithi089:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.89\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-06-03T19:41:00.031 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:40:59 smithi089 bash[26572]: cluster 2023-06-03T19:40:58.042236+0000 mgr.y (mgr.25070) 46 : cluster [DBG] pgmap v31: 89 pgs: 89 active+clean; 586 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 525 op/s 2023-06-03T19:41:00.032 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:40:59 smithi089 bash[33173]: cluster 2023-06-03T19:40:58.042236+0000 mgr.y (mgr.25070) 46 : cluster [DBG] pgmap v31: 89 pgs: 89 active+clean; 586 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 525 op/s 2023-06-03T19:41:00.086 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:40:59 smithi164 bash[26855]: cluster 2023-06-03T19:40:58.042236+0000 mgr.y (mgr.25070) 46 : cluster [DBG] pgmap v31: 89 pgs: 89 active+clean; 586 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 525 op/s 2023-06-03T19:41:01.031 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:41:00 smithi089 bash[33173]: audit 2023-06-03T19:41:00.155693+0000 mon.c (mon.2) 75 : audit [DBG] from='client.? 172.21.15.89:0/3993899662' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-03T19:41:01.032 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:41:00 smithi089 bash[26572]: audit 2023-06-03T19:41:00.155693+0000 mon.c (mon.2) 75 : audit [DBG] from='client.? 172.21.15.89:0/3993899662' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-03T19:41:01.041 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:41:00 smithi164 bash[26855]: audit 2023-06-03T19:41:00.155693+0000 mon.c (mon.2) 75 : audit [DBG] from='client.? 172.21.15.89:0/3993899662' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-03T19:41:02.031 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:41:01 smithi089 bash[33173]: cluster 2023-06-03T19:41:00.042961+0000 mgr.y (mgr.25070) 47 : cluster [DBG] pgmap v32: 89 pgs: 89 active+clean; 373 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 808 op/s 2023-06-03T19:41:02.032 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:41:01 smithi089 bash[26572]: cluster 2023-06-03T19:41:00.042961+0000 mgr.y (mgr.25070) 47 : cluster [DBG] pgmap v32: 89 pgs: 89 active+clean; 373 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 808 op/s 2023-06-03T19:41:02.086 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:41:01 smithi164 bash[26855]: cluster 2023-06-03T19:41:00.042961+0000 mgr.y (mgr.25070) 47 : cluster [DBG] pgmap v32: 89 pgs: 89 active+clean; 373 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 808 op/s 2023-06-03T19:41:03.284 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:41:02 smithi089 bash[26572]: audit 2023-06-03T19:41:01.873304+0000 mon.a (mon.0) 995 : audit [INF] from='mgr.25070 ' entity='mgr.y' 2023-06-03T19:41:03.284 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:41:02 smithi089 bash[26572]: audit 2023-06-03T19:41:01.889769+0000 mon.a (mon.0) 996 : audit [INF] from='mgr.25070 ' entity='mgr.y' 2023-06-03T19:41:03.285 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:41:02 smithi089 bash[26572]: audit 2023-06-03T19:41:01.894079+0000 mon.b (mon.1) 76 : audit [DBG] from='mgr.25070 172.21.15.89:0/2108989343' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-03T19:41:03.285 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:41:02 smithi089 bash[26572]: audit 2023-06-03T19:41:01.895837+0000 mon.b (mon.1) 77 : audit [INF] from='mgr.25070 172.21.15.89:0/2108989343' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-03T19:41:03.285 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:41:02 smithi089 bash[26572]: audit 2023-06-03T19:41:01.904957+0000 mon.a (mon.0) 997 : audit [INF] from='mgr.25070 ' entity='mgr.y' 2023-06-03T19:41:03.285 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:41:02 smithi089 bash[26572]: audit 2023-06-03T19:41:01.984958+0000 mon.b (mon.1) 78 : audit [DBG] from='mgr.25070 172.21.15.89:0/2108989343' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-03T19:41:03.285 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:41:02 smithi089 bash[26572]: audit 2023-06-03T19:41:01.995633+0000 mon.b (mon.1) 79 : audit [DBG] from='mgr.25070 172.21.15.89:0/2108989343' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-06-03T19:41:03.285 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:41:02 smithi089 bash[26572]: cephadm 2023-06-03T19:41:01.998403+0000 mgr.y (mgr.25070) 48 : cephadm [INF] Upgrade: Setting container_image for all mgr 2023-06-03T19:41:03.285 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:41:02 smithi089 bash[26572]: audit 2023-06-03T19:41:02.009504+0000 mon.a (mon.0) 998 : audit [INF] from='mgr.25070 ' entity='mgr.y' 2023-06-03T19:41:03.288 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:41:02 smithi089 bash[26572]: audit 2023-06-03T19:41:02.012218+0000 mon.b (mon.1) 80 : audit [INF] from='mgr.25070 172.21.15.89:0/2108989343' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]: dispatch 2023-06-03T19:41:03.288 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:41:02 smithi089 bash[26572]: audit 2023-06-03T19:41:02.014097+0000 mon.a (mon.0) 999 : audit [INF] from='mgr.25070 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]: dispatch 2023-06-03T19:41:03.288 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:41:02 smithi089 bash[26572]: audit 2023-06-03T19:41:02.020029+0000 mon.a (mon.0) 1000 : audit [INF] from='mgr.25070 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]': finished 2023-06-03T19:41:03.288 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:41:02 smithi089 bash[26572]: audit 2023-06-03T19:41:02.023813+0000 mon.b (mon.1) 81 : audit [INF] from='mgr.25070 172.21.15.89:0/2108989343' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]: dispatch 2023-06-03T19:41:03.289 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:41:02 smithi089 bash[26572]: audit 2023-06-03T19:41:02.025802+0000 mon.a (mon.0) 1001 : audit [INF] from='mgr.25070 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]: dispatch 2023-06-03T19:41:03.289 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:41:02 smithi089 bash[26572]: audit 2023-06-03T19:41:02.032537+0000 mon.a (mon.0) 1002 : audit [INF] from='mgr.25070 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]': finished 2023-06-03T19:41:03.289 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:41:02 smithi089 bash[26572]: audit 2023-06-03T19:41:02.036128+0000 mon.b (mon.1) 82 : audit [DBG] from='mgr.25070 172.21.15.89:0/2108989343' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2023-06-03T19:41:03.289 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:41:02 smithi089 bash[26572]: audit 2023-06-03T19:41:02.037257+0000 mon.b (mon.1) 83 : audit [DBG] from='mgr.25070 172.21.15.89:0/2108989343' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["a"]}]: dispatch 2023-06-03T19:41:03.289 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:41:02 smithi089 bash[26572]: cephadm 2023-06-03T19:41:02.039760+0000 mgr.y (mgr.25070) 49 : cephadm [INF] Upgrade: It appears safe to stop mon.a 2023-06-03T19:41:03.289 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:41:02 smithi089 bash[26572]: cluster 2023-06-03T19:41:02.043302+0000 mgr.y (mgr.25070) 50 : cluster [DBG] pgmap v33: 89 pgs: 89 active+clean; 373 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 542 op/s 2023-06-03T19:41:03.289 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:41:02 smithi089 bash[26572]: audit 2023-06-03T19:41:02.208955+0000 mon.b (mon.1) 84 : audit [DBG] from='mgr.25070 172.21.15.89:0/2108989343' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T19:41:03.290 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:41:02 smithi089 bash[33173]: audit 2023-06-03T19:41:01.873304+0000 mon.a (mon.0) 995 : audit [INF] from='mgr.25070 ' entity='mgr.y' 2023-06-03T19:41:03.290 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:41:02 smithi089 bash[33173]: audit 2023-06-03T19:41:01.889769+0000 mon.a (mon.0) 996 : audit [INF] from='mgr.25070 ' entity='mgr.y' 2023-06-03T19:41:03.290 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:41:02 smithi089 bash[33173]: audit 2023-06-03T19:41:01.894079+0000 mon.b (mon.1) 76 : audit [DBG] from='mgr.25070 172.21.15.89:0/2108989343' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-03T19:41:03.290 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:41:02 smithi089 bash[33173]: audit 2023-06-03T19:41:01.895837+0000 mon.b (mon.1) 77 : audit [INF] from='mgr.25070 172.21.15.89:0/2108989343' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-03T19:41:03.290 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:41:02 smithi089 bash[33173]: audit 2023-06-03T19:41:01.904957+0000 mon.a (mon.0) 997 : audit [INF] from='mgr.25070 ' entity='mgr.y' 2023-06-03T19:41:03.290 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:41:02 smithi089 bash[33173]: audit 2023-06-03T19:41:01.984958+0000 mon.b (mon.1) 78 : audit [DBG] from='mgr.25070 172.21.15.89:0/2108989343' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-03T19:41:03.291 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:41:02 smithi089 bash[33173]: audit 2023-06-03T19:41:01.995633+0000 mon.b (mon.1) 79 : audit [DBG] from='mgr.25070 172.21.15.89:0/2108989343' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-06-03T19:41:03.291 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:41:02 smithi089 bash[33173]: cephadm 2023-06-03T19:41:01.998403+0000 mgr.y (mgr.25070) 48 : cephadm [INF] Upgrade: Setting container_image for all mgr 2023-06-03T19:41:03.291 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:41:02 smithi089 bash[33173]: audit 2023-06-03T19:41:02.009504+0000 mon.a (mon.0) 998 : audit [INF] from='mgr.25070 ' entity='mgr.y' 2023-06-03T19:41:03.291 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:41:02 smithi089 bash[33173]: audit 2023-06-03T19:41:02.012218+0000 mon.b (mon.1) 80 : audit [INF] from='mgr.25070 172.21.15.89:0/2108989343' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]: dispatch 2023-06-03T19:41:03.291 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:41:02 smithi089 bash[33173]: audit 2023-06-03T19:41:02.014097+0000 mon.a (mon.0) 999 : audit [INF] from='mgr.25070 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]: dispatch 2023-06-03T19:41:03.291 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:41:02 smithi089 bash[33173]: audit 2023-06-03T19:41:02.020029+0000 mon.a (mon.0) 1000 : audit [INF] from='mgr.25070 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]': finished 2023-06-03T19:41:03.291 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:41:02 smithi089 bash[33173]: audit 2023-06-03T19:41:02.023813+0000 mon.b (mon.1) 81 : audit [INF] from='mgr.25070 172.21.15.89:0/2108989343' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]: dispatch 2023-06-03T19:41:03.292 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:41:02 smithi089 bash[33173]: audit 2023-06-03T19:41:02.025802+0000 mon.a (mon.0) 1001 : audit [INF] from='mgr.25070 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]: dispatch 2023-06-03T19:41:03.292 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:41:02 smithi089 bash[33173]: audit 2023-06-03T19:41:02.032537+0000 mon.a (mon.0) 1002 : audit [INF] from='mgr.25070 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]': finished 2023-06-03T19:41:03.292 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:41:02 smithi089 bash[33173]: audit 2023-06-03T19:41:02.036128+0000 mon.b (mon.1) 82 : audit [DBG] from='mgr.25070 172.21.15.89:0/2108989343' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2023-06-03T19:41:03.292 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:41:02 smithi089 bash[33173]: audit 2023-06-03T19:41:02.037257+0000 mon.b (mon.1) 83 : audit [DBG] from='mgr.25070 172.21.15.89:0/2108989343' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["a"]}]: dispatch 2023-06-03T19:41:03.292 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:41:02 smithi089 bash[33173]: cephadm 2023-06-03T19:41:02.039760+0000 mgr.y (mgr.25070) 49 : cephadm [INF] Upgrade: It appears safe to stop mon.a 2023-06-03T19:41:03.292 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:41:02 smithi089 bash[33173]: cluster 2023-06-03T19:41:02.043302+0000 mgr.y (mgr.25070) 50 : cluster [DBG] pgmap v33: 89 pgs: 89 active+clean; 373 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 542 op/s 2023-06-03T19:41:03.292 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:41:02 smithi089 bash[33173]: audit 2023-06-03T19:41:02.208955+0000 mon.b (mon.1) 84 : audit [DBG] from='mgr.25070 172.21.15.89:0/2108989343' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T19:41:03.336 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:41:02 smithi164 bash[26855]: audit 2023-06-03T19:41:01.873304+0000 mon.a (mon.0) 995 : audit [INF] from='mgr.25070 ' entity='mgr.y' 2023-06-03T19:41:03.336 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:41:02 smithi164 bash[26855]: audit 2023-06-03T19:41:01.889769+0000 mon.a (mon.0) 996 : audit [INF] from='mgr.25070 ' entity='mgr.y' 2023-06-03T19:41:03.337 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:41:02 smithi164 bash[26855]: audit 2023-06-03T19:41:01.894079+0000 mon.b (mon.1) 76 : audit [DBG] from='mgr.25070 172.21.15.89:0/2108989343' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-03T19:41:03.337 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:41:02 smithi164 bash[26855]: audit 2023-06-03T19:41:01.895837+0000 mon.b (mon.1) 77 : audit [INF] from='mgr.25070 172.21.15.89:0/2108989343' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-03T19:41:03.337 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:41:02 smithi164 bash[26855]: audit 2023-06-03T19:41:01.904957+0000 mon.a (mon.0) 997 : audit [INF] from='mgr.25070 ' entity='mgr.y' 2023-06-03T19:41:03.337 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:41:02 smithi164 bash[26855]: audit 2023-06-03T19:41:01.984958+0000 mon.b (mon.1) 78 : audit [DBG] from='mgr.25070 172.21.15.89:0/2108989343' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-03T19:41:03.337 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:41:02 smithi164 bash[26855]: audit 2023-06-03T19:41:01.995633+0000 mon.b (mon.1) 79 : audit [DBG] from='mgr.25070 172.21.15.89:0/2108989343' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-06-03T19:41:03.337 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:41:02 smithi164 bash[26855]: cephadm 2023-06-03T19:41:01.998403+0000 mgr.y (mgr.25070) 48 : cephadm [INF] Upgrade: Setting container_image for all mgr 2023-06-03T19:41:03.337 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:41:02 smithi164 bash[26855]: audit 2023-06-03T19:41:02.009504+0000 mon.a (mon.0) 998 : audit [INF] from='mgr.25070 ' entity='mgr.y' 2023-06-03T19:41:03.338 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:41:02 smithi164 bash[26855]: audit 2023-06-03T19:41:02.012218+0000 mon.b (mon.1) 80 : audit [INF] from='mgr.25070 172.21.15.89:0/2108989343' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]: dispatch 2023-06-03T19:41:03.338 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:41:02 smithi164 bash[26855]: audit 2023-06-03T19:41:02.014097+0000 mon.a (mon.0) 999 : audit [INF] from='mgr.25070 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]: dispatch 2023-06-03T19:41:03.338 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:41:02 smithi164 bash[26855]: audit 2023-06-03T19:41:02.020029+0000 mon.a (mon.0) 1000 : audit [INF] from='mgr.25070 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]': finished 2023-06-03T19:41:03.338 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:41:02 smithi164 bash[26855]: audit 2023-06-03T19:41:02.023813+0000 mon.b (mon.1) 81 : audit [INF] from='mgr.25070 172.21.15.89:0/2108989343' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]: dispatch 2023-06-03T19:41:03.338 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:41:02 smithi164 bash[26855]: audit 2023-06-03T19:41:02.025802+0000 mon.a (mon.0) 1001 : audit [INF] from='mgr.25070 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]: dispatch 2023-06-03T19:41:03.338 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:41:02 smithi164 bash[26855]: audit 2023-06-03T19:41:02.032537+0000 mon.a (mon.0) 1002 : audit [INF] from='mgr.25070 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]': finished 2023-06-03T19:41:03.338 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:41:02 smithi164 bash[26855]: audit 2023-06-03T19:41:02.036128+0000 mon.b (mon.1) 82 : audit [DBG] from='mgr.25070 172.21.15.89:0/2108989343' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2023-06-03T19:41:03.338 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:41:02 smithi164 bash[26855]: audit 2023-06-03T19:41:02.037257+0000 mon.b (mon.1) 83 : audit [DBG] from='mgr.25070 172.21.15.89:0/2108989343' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["a"]}]: dispatch 2023-06-03T19:41:03.339 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:41:02 smithi164 bash[26855]: cephadm 2023-06-03T19:41:02.039760+0000 mgr.y (mgr.25070) 49 : cephadm [INF] Upgrade: It appears safe to stop mon.a 2023-06-03T19:41:03.339 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:41:02 smithi164 bash[26855]: cluster 2023-06-03T19:41:02.043302+0000 mgr.y (mgr.25070) 50 : cluster [DBG] pgmap v33: 89 pgs: 89 active+clean; 373 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 542 op/s 2023-06-03T19:41:03.339 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:41:02 smithi164 bash[26855]: audit 2023-06-03T19:41:02.208955+0000 mon.b (mon.1) 84 : audit [DBG] from='mgr.25070 172.21.15.89:0/2108989343' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T19:41:04.281 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:41:03 smithi089 bash[26572]: audit 2023-06-03T19:41:02.886022+0000 mon.a (mon.0) 1003 : audit [DBG] from='client.? 172.21.15.89:0/2540754873' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-03T19:41:04.282 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:41:03 smithi089 bash[33173]: audit 2023-06-03T19:41:02.886022+0000 mon.a (mon.0) 1003 : audit [DBG] from='client.? 172.21.15.89:0/2540754873' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-03T19:41:04.336 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:41:03 smithi164 bash[26855]: audit 2023-06-03T19:41:02.886022+0000 mon.a (mon.0) 1003 : audit [DBG] from='client.? 172.21.15.89:0/2540754873' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-03T19:41:05.031 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:41:04 smithi089 bash[33173]: cluster 2023-06-03T19:41:04.044087+0000 mgr.y (mgr.25070) 51 : cluster [DBG] pgmap v34: 89 pgs: 89 active+clean; 159 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 0 B/s wr, 827 op/s 2023-06-03T19:41:05.032 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:41:04 smithi089 bash[26572]: cluster 2023-06-03T19:41:04.044087+0000 mgr.y (mgr.25070) 51 : cluster [DBG] pgmap v34: 89 pgs: 89 active+clean; 159 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 0 B/s wr, 827 op/s 2023-06-03T19:41:05.335 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:41:04 smithi164 bash[26855]: cluster 2023-06-03T19:41:04.044087+0000 mgr.y (mgr.25070) 51 : cluster [DBG] pgmap v34: 89 pgs: 89 active+clean; 159 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 0 B/s wr, 827 op/s 2023-06-03T19:41:06.234 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 19:41:06 smithi089 bash[71163]: ::ffff:172.21.15.164 - - [03/Jun/2023:19:41:06] "GET /metrics HTTP/1.1" 200 190054 "" "Prometheus/2.33.4" 2023-06-03T19:41:06.235 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:41:05 smithi089 bash[33173]: audit 2023-06-03T19:41:05.606640+0000 mon.c (mon.2) 76 : audit [DBG] from='client.? 172.21.15.89:0/2926559111' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-03T19:41:06.235 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:41:05 smithi089 bash[26572]: audit 2023-06-03T19:41:05.606640+0000 mon.c (mon.2) 76 : audit [DBG] from='client.? 172.21.15.89:0/2926559111' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-03T19:41:06.336 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:41:05 smithi164 bash[26855]: audit 2023-06-03T19:41:05.606640+0000 mon.c (mon.2) 76 : audit [DBG] from='client.? 172.21.15.89:0/2926559111' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-03T19:41:06.848 INFO:tasks.radosbench.radosbench.0.smithi089.stdout:Removed 111824 objects 2023-06-03T19:41:06.848 INFO:tasks.radosbench.radosbench.0.smithi089.stdout:Clean up completed and total clean up time :159.347 2023-06-03T19:41:08.031 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:41:07 smithi089 bash[33173]: cluster 2023-06-03T19:41:06.044521+0000 mgr.y (mgr.25070) 52 : cluster [DBG] pgmap v35: 89 pgs: 89 active+clean; 159 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 0 B/s wr, 568 op/s 2023-06-03T19:41:08.032 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:41:07 smithi089 bash[33173]: audit 2023-06-03T19:41:06.792379+0000 mon.a (mon.0) 1004 : audit [INF] from='mgr.25070 ' entity='mgr.y' 2023-06-03T19:41:08.032 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:41:07 smithi089 bash[33173]: audit 2023-06-03T19:41:06.796739+0000 mon.b (mon.1) 85 : audit [INF] from='mgr.25070 172.21.15.89:0/2108989343' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-06-03T19:41:08.032 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:41:07 smithi089 bash[33173]: audit 2023-06-03T19:41:06.797854+0000 mon.b (mon.1) 86 : audit [DBG] from='mgr.25070 172.21.15.89:0/2108989343' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-06-03T19:41:08.033 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:41:07 smithi089 bash[33173]: audit 2023-06-03T19:41:06.799478+0000 mon.b (mon.1) 87 : audit [DBG] from='mgr.25070 172.21.15.89:0/2108989343' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-03T19:41:08.033 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:41:07 smithi089 bash[26572]: cluster 2023-06-03T19:41:06.044521+0000 mgr.y (mgr.25070) 52 : cluster [DBG] pgmap v35: 89 pgs: 89 active+clean; 159 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 0 B/s wr, 568 op/s 2023-06-03T19:41:08.034 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:41:07 smithi089 bash[26572]: audit 2023-06-03T19:41:06.792379+0000 mon.a (mon.0) 1004 : audit [INF] from='mgr.25070 ' entity='mgr.y' 2023-06-03T19:41:08.034 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:41:07 smithi089 bash[26572]: audit 2023-06-03T19:41:06.796739+0000 mon.b (mon.1) 85 : audit [INF] from='mgr.25070 172.21.15.89:0/2108989343' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-06-03T19:41:08.034 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:41:07 smithi089 bash[26572]: audit 2023-06-03T19:41:06.797854+0000 mon.b (mon.1) 86 : audit [DBG] from='mgr.25070 172.21.15.89:0/2108989343' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-06-03T19:41:08.034 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:41:07 smithi089 bash[26572]: audit 2023-06-03T19:41:06.799478+0000 mon.b (mon.1) 87 : audit [DBG] from='mgr.25070 172.21.15.89:0/2108989343' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-03T19:41:08.086 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:41:07 smithi164 bash[26855]: cluster 2023-06-03T19:41:06.044521+0000 mgr.y (mgr.25070) 52 : cluster [DBG] pgmap v35: 89 pgs: 89 active+clean; 159 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 0 B/s wr, 568 op/s 2023-06-03T19:41:08.086 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:41:07 smithi164 bash[26855]: audit 2023-06-03T19:41:06.792379+0000 mon.a (mon.0) 1004 : audit [INF] from='mgr.25070 ' entity='mgr.y' 2023-06-03T19:41:08.086 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:41:07 smithi164 bash[26855]: audit 2023-06-03T19:41:06.796739+0000 mon.b (mon.1) 85 : audit [INF] from='mgr.25070 172.21.15.89:0/2108989343' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-06-03T19:41:08.087 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:41:07 smithi164 bash[26855]: audit 2023-06-03T19:41:06.797854+0000 mon.b (mon.1) 86 : audit [DBG] from='mgr.25070 172.21.15.89:0/2108989343' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-06-03T19:41:08.087 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:41:07 smithi164 bash[26855]: audit 2023-06-03T19:41:06.799478+0000 mon.b (mon.1) 87 : audit [DBG] from='mgr.25070 172.21.15.89:0/2108989343' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-03T19:41:09.034 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:41:08 smithi089 bash[33173]: cephadm 2023-06-03T19:41:06.781905+0000 mgr.y (mgr.25070) 53 : cephadm [INF] Upgrade: Updating mon.a 2023-06-03T19:41:09.034 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:41:08 smithi089 bash[33173]: cephadm 2023-06-03T19:41:06.802565+0000 mgr.y (mgr.25070) 54 : cephadm [INF] Deploying daemon mon.a on smithi089 2023-06-03T19:41:09.035 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:41:08 smithi089 bash[33173]: audit 2023-06-03T19:41:08.062245+0000 mon.a (mon.0) 1005 : audit [DBG] from='client.? 172.21.15.89:0/195531372' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-03T19:41:09.035 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:41:08 smithi089 bash[26572]: cephadm 2023-06-03T19:41:06.781905+0000 mgr.y (mgr.25070) 53 : cephadm [INF] Upgrade: Updating mon.a 2023-06-03T19:41:09.036 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:41:08 smithi089 bash[26572]: cephadm 2023-06-03T19:41:06.802565+0000 mgr.y (mgr.25070) 54 : cephadm [INF] Deploying daemon mon.a on smithi089 2023-06-03T19:41:09.036 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:41:08 smithi089 bash[26572]: audit 2023-06-03T19:41:08.062245+0000 mon.a (mon.0) 1005 : audit [DBG] from='client.? 172.21.15.89:0/195531372' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-03T19:41:09.086 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:41:08 smithi164 bash[26855]: cephadm 2023-06-03T19:41:06.781905+0000 mgr.y (mgr.25070) 53 : cephadm [INF] Upgrade: Updating mon.a 2023-06-03T19:41:09.086 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:41:08 smithi164 bash[26855]: cephadm 2023-06-03T19:41:06.802565+0000 mgr.y (mgr.25070) 54 : cephadm [INF] Deploying daemon mon.a on smithi089 2023-06-03T19:41:09.087 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:41:08 smithi164 bash[26855]: audit 2023-06-03T19:41:08.062245+0000 mon.a (mon.0) 1005 : audit [DBG] from='client.? 172.21.15.89:0/195531372' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-03T19:41:09.087 INFO:journalctl@ceph.prometheus.a.smithi164.stdout:Jun 03 19:41:08 smithi164 bash[55188]: ts=2023-06-03T19:41:08.663Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.13-257-gd8c5d349 (d8c5d34975dce1c5eb0aa3a7979a4d9b9a99d1ec) pacific (stable)\", cluster_addr=\"172.21.15.89\", device_class=\"ssd\", hostname=\"smithi089\", instance=\"smithi164.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.89\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.13-257-gd8c5d349 (d8c5d34975dce1c5eb0aa3a7979a4d9b9a99d1ec) pacific (stable)\", cluster_addr=\"172.21.15.89\", device_class=\"ssd\", hostname=\"smithi089\", instance=\"smithi089:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.89\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-06-03T19:41:09.087 INFO:journalctl@ceph.prometheus.a.smithi164.stdout:Jun 03 19:41:08 smithi164 bash[55188]: ts=2023-06-03T19:41:08.665Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephPGImbalance\nexpr: abs(((ceph_osd_numpg > 0) - on(job) group_left() avg by(job) (ceph_osd_numpg\n > 0)) / on(job) group_left() avg by(job) (ceph_osd_numpg > 0)) * on(ceph_daemon)\n group_left(hostname) ceph_osd_metadata > 0.3\nfor: 5m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.5\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} deviates by\n more than 30% from average PG count.\n summary: PGs are not balanced across OSDs\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.13-257-gd8c5d349 (d8c5d34975dce1c5eb0aa3a7979a4d9b9a99d1ec) pacific (stable)\", cluster_addr=\"172.21.15.89\", device_class=\"ssd\", hostname=\"smithi089\", instance=\"smithi164.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.89\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.13-257-gd8c5d349 (d8c5d34975dce1c5eb0aa3a7979a4d9b9a99d1ec) pacific (stable)\", cluster_addr=\"172.21.15.89\", device_class=\"ssd\", hostname=\"smithi089\", instance=\"smithi089:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.89\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-06-03T19:41:09.654 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:41:09 smithi089 systemd[1]: Stopping Ceph mon.a for f1db3478-0243-11ee-9b1d-001a4aab830c... 2023-06-03T19:41:09.918 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:41:09 smithi089 bash[33173]: cluster 2023-06-03T19:41:08.044892+0000 mgr.y (mgr.25070) 55 : cluster [DBG] pgmap v36: 89 pgs: 89 active+clean; 159 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 0 B/s wr, 568 op/s 2023-06-03T19:41:09.918 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:41:09 smithi089 bash[26572]: cluster 2023-06-03T19:41:08.044892+0000 mgr.y (mgr.25070) 55 : cluster [DBG] pgmap v36: 89 pgs: 89 active+clean; 159 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 0 B/s wr, 568 op/s 2023-06-03T19:41:09.918 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:41:09 smithi089 bash[26572]: debug 2023-06-03T19:41:09.694+0000 7fc1d1062700 -1 received signal: Terminated from /sbin/docker-init -- /usr/bin/ceph-mon -n mon.a -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-stderr=true (PID: 1) UID: 0 2023-06-03T19:41:09.918 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:41:09 smithi089 bash[26572]: debug 2023-06-03T19:41:09.694+0000 7fc1d1062700 -1 mon.a@0(leader) e3 *** Got Signal Terminated *** 2023-06-03T19:41:10.086 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:41:09 smithi164 bash[26855]: cluster 2023-06-03T19:41:08.044892+0000 mgr.y (mgr.25070) 55 : cluster [DBG] pgmap v36: 89 pgs: 89 active+clean; 159 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 0 B/s wr, 568 op/s 2023-06-03T19:41:10.781 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:41:10 smithi089 bash[74916]: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-mon-a 2023-06-03T19:41:10.781 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:41:10 smithi089 systemd[1]: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c@mon.a.service: Succeeded. 2023-06-03T19:41:10.782 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:41:10 smithi089 systemd[1]: Stopped Ceph mon.a for f1db3478-0243-11ee-9b1d-001a4aab830c. 2023-06-03T19:41:11.281 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:41:10 smithi089 systemd[1]: Started Ceph mon.a for f1db3478-0243-11ee-9b1d-001a4aab830c. 2023-06-03T19:41:12.282 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:41:12 smithi089 bash[75068]: debug 2023-06-03T19:41:12.046+0000 7f0ffccbac80 0 set uid:gid to 167:167 (ceph:ceph) 2023-06-03T19:41:12.282 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:41:12 smithi089 bash[75068]: debug 2023-06-03T19:41:12.046+0000 7f0ffccbac80 0 ceph version 18.0.0-3532-g8e24bcf4 (8e24bcf462a1d94c2e12aa4e26419634883a1e88) reef (rc), process ceph-mon, pid 7 2023-06-03T19:41:12.283 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:41:12 smithi089 bash[75068]: debug 2023-06-03T19:41:12.046+0000 7f0ffccbac80 0 pidfile_write: ignore empty --pid-file 2023-06-03T19:41:12.283 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:41:12 smithi089 bash[75068]: debug 2023-06-03T19:41:12.050+0000 7f0ffccbac80 0 load: jerasure load: lrc 2023-06-03T19:41:12.283 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:41:12 smithi089 bash[75068]: debug 2023-06-03T19:41:12.050+0000 7f0ffccbac80 4 rocksdb: RocksDB version: 7.9.2 2023-06-03T19:41:12.283 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:41:12 smithi089 bash[75068]: debug 2023-06-03T19:41:12.050+0000 7f0ffccbac80 4 rocksdb: Git sha 0 2023-06-03T19:41:12.283 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:41:12 smithi089 bash[75068]: debug 2023-06-03T19:41:12.050+0000 7f0ffccbac80 4 rocksdb: Compile date 2023-06-01 15:16:58 2023-06-03T19:41:12.283 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:41:12 smithi089 bash[75068]: debug 2023-06-03T19:41:12.050+0000 7f0ffccbac80 4 rocksdb: DB SUMMARY 2023-06-03T19:41:12.284 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:41:12 smithi089 bash[75068]: debug 2023-06-03T19:41:12.050+0000 7f0ffccbac80 4 rocksdb: DB Session ID: 919HVZ0G2IFV3IQWARLX 2023-06-03T19:41:12.284 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:41:12 smithi089 bash[75068]: debug 2023-06-03T19:41:12.050+0000 7f0ffccbac80 4 rocksdb: CURRENT file: CURRENT 2023-06-03T19:41:12.284 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:41:12 smithi089 bash[75068]: debug 2023-06-03T19:41:12.050+0000 7f0ffccbac80 4 rocksdb: IDENTITY file: IDENTITY 2023-06-03T19:41:12.284 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:41:12 smithi089 bash[75068]: debug 2023-06-03T19:41:12.050+0000 7f0ffccbac80 4 rocksdb: MANIFEST file: MANIFEST-000008 size: 1145 Bytes 2023-06-03T19:41:12.284 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:41:12 smithi089 bash[75068]: debug 2023-06-03T19:41:12.050+0000 7f0ffccbac80 4 rocksdb: SST files in /var/lib/ceph/mon/ceph-a/store.db dir, Total Num: 1, files: 000026.sst 2023-06-03T19:41:12.284 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:41:12 smithi089 bash[75068]: debug 2023-06-03T19:41:12.050+0000 7f0ffccbac80 4 rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-a/store.db: 000024.log size: 830924 ; 2023-06-03T19:41:12.284 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:41:12 smithi089 bash[75068]: debug 2023-06-03T19:41:12.050+0000 7f0ffccbac80 4 rocksdb: Options.error_if_exists: 0 2023-06-03T19:41:12.285 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:41:12 smithi089 bash[75068]: debug 2023-06-03T19:41:12.050+0000 7f0ffccbac80 4 rocksdb: Options.create_if_missing: 0 2023-06-03T19:41:12.285 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:41:12 smithi089 bash[75068]: debug 2023-06-03T19:41:12.050+0000 7f0ffccbac80 4 rocksdb: Options.paranoid_checks: 1 2023-06-03T19:41:12.285 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:41:12 smithi089 bash[75068]: debug 2023-06-03T19:41:12.050+0000 7f0ffccbac80 4 rocksdb: Options.flush_verify_memtable_count: 1 2023-06-03T19:41:12.285 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:41:12 smithi089 bash[75068]: debug 2023-06-03T19:41:12.050+0000 7f0ffccbac80 4 rocksdb: Options.track_and_verify_wals_in_manifest: 0 2023-06-03T19:41:12.285 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:41:12 smithi089 bash[75068]: debug 2023-06-03T19:41:12.050+0000 7f0ffccbac80 4 rocksdb: Options.verify_sst_unique_id_in_manifest: 1 2023-06-03T19:41:12.285 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:41:12 smithi089 bash[75068]: debug 2023-06-03T19:41:12.050+0000 7f0ffccbac80 4 rocksdb: Options.env: 0x56382295b720 2023-06-03T19:41:12.285 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:41:12 smithi089 bash[75068]: debug 2023-06-03T19:41:12.050+0000 7f0ffccbac80 4 rocksdb: Options.fs: PosixFileSystem 2023-06-03T19:41:12.285 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:41:12 smithi089 bash[75068]: debug 2023-06-03T19:41:12.050+0000 7f0ffccbac80 4 rocksdb: Options.info_log: 0x56382420d460 2023-06-03T19:41:12.286 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:41:12 smithi089 bash[75068]: debug 2023-06-03T19:41:12.050+0000 7f0ffccbac80 4 rocksdb: Options.max_file_opening_threads: 16 2023-06-03T19:41:12.286 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:41:12 smithi089 bash[75068]: debug 2023-06-03T19:41:12.050+0000 7f0ffccbac80 4 rocksdb: Options.statistics: (nil) 2023-06-03T19:41:12.286 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:41:12 smithi089 bash[75068]: debug 2023-06-03T19:41:12.050+0000 7f0ffccbac80 4 rocksdb: Options.use_fsync: 0 2023-06-03T19:41:12.286 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:41:12 smithi089 bash[75068]: debug 2023-06-03T19:41:12.050+0000 7f0ffccbac80 4 rocksdb: Options.max_log_file_size: 0 2023-06-03T19:41:12.288 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:41:12 smithi089 bash[75068]: debug 2023-06-03T19:41:12.050+0000 7f0ffccbac80 4 rocksdb: Options.max_manifest_file_size: 1073741824 2023-06-03T19:41:12.288 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:41:12 smithi089 bash[75068]: debug 2023-06-03T19:41:12.050+0000 7f0ffccbac80 4 rocksdb: Options.log_file_time_to_roll: 0 2023-06-03T19:41:12.289 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:41:12 smithi089 bash[75068]: debug 2023-06-03T19:41:12.050+0000 7f0ffccbac80 4 rocksdb: Options.keep_log_file_num: 1000 2023-06-03T19:41:12.289 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:41:12 smithi089 bash[75068]: debug 2023-06-03T19:41:12.050+0000 7f0ffccbac80 4 rocksdb: Options.recycle_log_file_num: 0 2023-06-03T19:41:12.289 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:41:12 smithi089 bash[75068]: debug 2023-06-03T19:41:12.050+0000 7f0ffccbac80 4 rocksdb: Options.allow_fallocate: 1 2023-06-03T19:41:12.289 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:41:12 smithi089 bash[75068]: debug 2023-06-03T19:41:12.050+0000 7f0ffccbac80 4 rocksdb: Options.allow_mmap_reads: 0 2023-06-03T19:41:12.289 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:41:12 smithi089 bash[75068]: debug 2023-06-03T19:41:12.050+0000 7f0ffccbac80 4 rocksdb: Options.allow_mmap_writes: 0 2023-06-03T19:41:12.289 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:41:12 smithi089 bash[75068]: debug 2023-06-03T19:41:12.050+0000 7f0ffccbac80 4 rocksdb: Options.use_direct_reads: 0 2023-06-03T19:41:12.289 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:41:12 smithi089 bash[75068]: debug 2023-06-03T19:41:12.050+0000 7f0ffccbac80 4 rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2023-06-03T19:41:12.290 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:41:12 smithi089 bash[75068]: debug 2023-06-03T19:41:12.050+0000 7f0ffccbac80 4 rocksdb: Options.create_missing_column_families: 0 2023-06-03T19:41:12.290 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:41:12 smithi089 bash[75068]: debug 2023-06-03T19:41:12.050+0000 7f0ffccbac80 4 rocksdb: Options.db_log_dir: 2023-06-03T19:41:12.290 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:41:12 smithi089 bash[75068]: debug 2023-06-03T19:41:12.050+0000 7f0ffccbac80 4 rocksdb: Options.wal_dir: 2023-06-03T19:41:12.290 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:41:12 smithi089 bash[75068]: debug 2023-06-03T19:41:12.050+0000 7f0ffccbac80 4 rocksdb: Options.table_cache_numshardbits: 6 2023-06-03T19:41:12.290 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:41:12 smithi089 bash[75068]: debug 2023-06-03T19:41:12.050+0000 7f0ffccbac80 4 rocksdb: Options.WAL_ttl_seconds: 0 2023-06-03T19:41:12.290 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:41:12 smithi089 bash[75068]: debug 2023-06-03T19:41:12.050+0000 7f0ffccbac80 4 rocksdb: Options.WAL_size_limit_MB: 0 2023-06-03T19:41:12.290 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:41:12 smithi089 bash[75068]: debug 2023-06-03T19:41:12.050+0000 7f0ffccbac80 4 rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2023-06-03T19:41:12.290 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:41:12 smithi089 bash[75068]: debug 2023-06-03T19:41:12.050+0000 7f0ffccbac80 4 rocksdb: Options.manifest_preallocation_size: 4194304 2023-06-03T19:41:12.291 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:41:12 smithi089 bash[75068]: debug 2023-06-03T19:41:12.050+0000 7f0ffccbac80 4 rocksdb: Options.is_fd_close_on_exec: 1 2023-06-03T19:41:12.291 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:41:12 smithi089 bash[75068]: debug 2023-06-03T19:41:12.050+0000 7f0ffccbac80 4 rocksdb: Options.advise_random_on_open: 1 2023-06-03T19:41:12.291 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:41:12 smithi089 bash[75068]: debug 2023-06-03T19:41:12.050+0000 7f0ffccbac80 4 rocksdb: Options.db_write_buffer_size: 0 2023-06-03T19:41:12.291 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:41:12 smithi089 bash[75068]: debug 2023-06-03T19:41:12.050+0000 7f0ffccbac80 4 rocksdb: Options.write_buffer_manager: 0x56382349ac80 2023-06-03T19:41:12.291 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:41:12 smithi089 bash[75068]: debug 2023-06-03T19:41:12.050+0000 7f0ffccbac80 4 rocksdb: Options.access_hint_on_compaction_start: 1 2023-06-03T19:41:12.291 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:41:12 smithi089 bash[75068]: debug 2023-06-03T19:41:12.050+0000 7f0ffccbac80 4 rocksdb: Options.random_access_max_buffer_size: 1048576 2023-06-03T19:41:12.291 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:41:12 smithi089 bash[75068]: debug 2023-06-03T19:41:12.050+0000 7f0ffccbac80 4 rocksdb: Options.use_adaptive_mutex: 0 2023-06-03T19:41:12.292 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:41:12 smithi089 bash[75068]: debug 2023-06-03T19:41:12.050+0000 7f0ffccbac80 4 rocksdb: Options.rate_limiter: (nil) 2023-06-03T19:41:12.292 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:41:12 smithi089 bash[75068]: debug 2023-06-03T19:41:12.050+0000 7f0ffccbac80 4 rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2023-06-03T19:41:12.292 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:41:12 smithi089 bash[75068]: debug 2023-06-03T19:41:12.050+0000 7f0ffccbac80 4 rocksdb: Options.wal_recovery_mode: 2 2023-06-03T19:41:12.292 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:41:12 smithi089 bash[75068]: debug 2023-06-03T19:41:12.050+0000 7f0ffccbac80 4 rocksdb: Options.enable_thread_tracking: 0 2023-06-03T19:41:12.292 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:41:12 smithi089 bash[75068]: debug 2023-06-03T19:41:12.050+0000 7f0ffccbac80 4 rocksdb: Options.enable_pipelined_write: 0 2023-06-03T19:41:12.292 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:41:12 smithi089 bash[75068]: debug 2023-06-03T19:41:12.050+0000 7f0ffccbac80 4 rocksdb: Options.unordered_write: 0 2023-06-03T19:41:12.292 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:41:12 smithi089 bash[75068]: debug 2023-06-03T19:41:12.050+0000 7f0ffccbac80 4 rocksdb: Options.allow_concurrent_memtable_write: 1 2023-06-03T19:41:12.293 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:41:12 smithi089 bash[75068]: debug 2023-06-03T19:41:12.050+0000 7f0ffccbac80 4 rocksdb: Options.enable_write_thread_adaptive_yield: 1 2023-06-03T19:41:12.293 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:41:12 smithi089 bash[75068]: debug 2023-06-03T19:41:12.050+0000 7f0ffccbac80 4 rocksdb: Options.write_thread_max_yield_usec: 100 2023-06-03T19:41:12.293 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:41:12 smithi089 bash[75068]: debug 2023-06-03T19:41:12.050+0000 7f0ffccbac80 4 rocksdb: Options.write_thread_slow_yield_usec: 3 2023-06-03T19:41:12.294 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:41:12 smithi089 bash[75068]: debug 2023-06-03T19:41:12.050+0000 7f0ffccbac80 4 rocksdb: Options.row_cache: None 2023-06-03T19:41:12.294 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:41:12 smithi089 bash[75068]: debug 2023-06-03T19:41:12.050+0000 7f0ffccbac80 4 rocksdb: Options.wal_filter: None 2023-06-03T19:41:12.294 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:41:12 smithi089 bash[75068]: debug 2023-06-03T19:41:12.050+0000 7f0ffccbac80 4 rocksdb: Options.avoid_flush_during_recovery: 0 2023-06-03T19:41:12.294 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:41:12 smithi089 bash[75068]: debug 2023-06-03T19:41:12.050+0000 7f0ffccbac80 4 rocksdb: Options.allow_ingest_behind: 0 2023-06-03T19:41:12.295 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:41:12 smithi089 bash[75068]: debug 2023-06-03T19:41:12.050+0000 7f0ffccbac80 4 rocksdb: Options.two_write_queues: 0 2023-06-03T19:41:12.295 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:41:12 smithi089 bash[75068]: debug 2023-06-03T19:41:12.050+0000 7f0ffccbac80 4 rocksdb: Options.manual_wal_flush: 0 2023-06-03T19:41:12.295 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:41:12 smithi089 bash[75068]: debug 2023-06-03T19:41:12.050+0000 7f0ffccbac80 4 rocksdb: Options.wal_compression: 0 2023-06-03T19:41:12.296 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:41:12 smithi089 bash[75068]: debug 2023-06-03T19:41:12.050+0000 7f0ffccbac80 4 rocksdb: Options.atomic_flush: 0 2023-06-03T19:41:12.296 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:41:12 smithi089 bash[75068]: debug 2023-06-03T19:41:12.050+0000 7f0ffccbac80 4 rocksdb: Options.avoid_unnecessary_blocking_io: 0 2023-06-03T19:41:12.296 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:41:12 smithi089 bash[75068]: debug 2023-06-03T19:41:12.050+0000 7f0ffccbac80 4 rocksdb: Options.persist_stats_to_disk: 0 2023-06-03T19:41:12.297 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:41:12 smithi089 bash[75068]: debug 2023-06-03T19:41:12.050+0000 7f0ffccbac80 4 rocksdb: Options.write_dbid_to_manifest: 0 2023-06-03T19:41:12.297 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:41:12 smithi089 bash[75068]: debug 2023-06-03T19:41:12.050+0000 7f0ffccbac80 4 rocksdb: Options.log_readahead_size: 0 2023-06-03T19:41:12.297 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:41:12 smithi089 bash[75068]: debug 2023-06-03T19:41:12.050+0000 7f0ffccbac80 4 rocksdb: Options.file_checksum_gen_factory: Unknown 2023-06-03T19:41:12.297 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:41:12 smithi089 bash[75068]: debug 2023-06-03T19:41:12.050+0000 7f0ffccbac80 4 rocksdb: Options.best_efforts_recovery: 0 2023-06-03T19:41:12.297 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:41:12 smithi089 bash[75068]: debug 2023-06-03T19:41:12.050+0000 7f0ffccbac80 4 rocksdb: Options.max_bgerror_resume_count: 2147483647 2023-06-03T19:41:12.297 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:41:12 smithi089 bash[75068]: debug 2023-06-03T19:41:12.050+0000 7f0ffccbac80 4 rocksdb: Options.bgerror_resume_retry_interval: 1000000 2023-06-03T19:41:12.298 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:41:12 smithi089 bash[75068]: debug 2023-06-03T19:41:12.050+0000 7f0ffccbac80 4 rocksdb: Options.allow_data_in_errors: 0 2023-06-03T19:41:12.298 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:41:12 smithi089 bash[75068]: debug 2023-06-03T19:41:12.050+0000 7f0ffccbac80 4 rocksdb: Options.db_host_id: __hostname__ 2023-06-03T19:41:12.298 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:41:12 smithi089 bash[75068]: debug 2023-06-03T19:41:12.050+0000 7f0ffccbac80 4 rocksdb: Options.enforce_single_del_contracts: true 2023-06-03T19:41:12.298 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:41:12 smithi089 bash[75068]: debug 2023-06-03T19:41:12.050+0000 7f0ffccbac80 4 rocksdb: Options.max_background_jobs: 2 2023-06-03T19:41:12.298 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:41:12 smithi089 bash[75068]: debug 2023-06-03T19:41:12.050+0000 7f0ffccbac80 4 rocksdb: Options.max_background_compactions: -1 2023-06-03T19:41:12.298 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:41:12 smithi089 bash[75068]: debug 2023-06-03T19:41:12.050+0000 7f0ffccbac80 4 rocksdb: Options.max_subcompactions: 1 2023-06-03T19:41:12.298 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:41:12 smithi089 bash[75068]: debug 2023-06-03T19:41:12.050+0000 7f0ffccbac80 4 rocksdb: Options.avoid_flush_during_shutdown: 0 2023-06-03T19:41:12.298 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:41:12 smithi089 bash[75068]: debug 2023-06-03T19:41:12.050+0000 7f0ffccbac80 4 rocksdb: Options.writable_file_max_buffer_size: 1048576 2023-06-03T19:41:12.299 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:41:12 smithi089 bash[75068]: debug 2023-06-03T19:41:12.050+0000 7f0ffccbac80 4 rocksdb: Options.delayed_write_rate : 16777216 2023-06-03T19:41:12.299 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:41:12 smithi089 bash[75068]: debug 2023-06-03T19:41:12.050+0000 7f0ffccbac80 4 rocksdb: Options.max_total_wal_size: 0 2023-06-03T19:41:12.299 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:41:12 smithi089 bash[75068]: debug 2023-06-03T19:41:12.050+0000 7f0ffccbac80 4 rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2023-06-03T19:41:12.300 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:41:12 smithi089 bash[75068]: debug 2023-06-03T19:41:12.050+0000 7f0ffccbac80 4 rocksdb: Options.stats_dump_period_sec: 600 2023-06-03T19:41:12.300 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:41:12 smithi089 bash[75068]: debug 2023-06-03T19:41:12.050+0000 7f0ffccbac80 4 rocksdb: Options.stats_persist_period_sec: 600 2023-06-03T19:41:12.300 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:41:12 smithi089 bash[75068]: debug 2023-06-03T19:41:12.050+0000 7f0ffccbac80 4 rocksdb: Options.stats_history_buffer_size: 1048576 2023-06-03T19:41:12.300 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:41:12 smithi089 bash[75068]: debug 2023-06-03T19:41:12.050+0000 7f0ffccbac80 4 rocksdb: Options.max_open_files: -1 2023-06-03T19:41:12.300 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:41:12 smithi089 bash[75068]: debug 2023-06-03T19:41:12.050+0000 7f0ffccbac80 4 rocksdb: Options.bytes_per_sync: 0 2023-06-03T19:41:12.300 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:41:12 smithi089 bash[75068]: debug 2023-06-03T19:41:12.050+0000 7f0ffccbac80 4 rocksdb: Options.wal_bytes_per_sync: 0 2023-06-03T19:41:12.301 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:41:12 smithi089 bash[75068]: debug 2023-06-03T19:41:12.050+0000 7f0ffccbac80 4 rocksdb: Options.strict_bytes_per_sync: 0 2023-06-03T19:41:12.301 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:41:12 smithi089 bash[75068]: debug 2023-06-03T19:41:12.050+0000 7f0ffccbac80 4 rocksdb: Options.compaction_readahead_size: 0 2023-06-03T19:41:12.301 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:41:12 smithi089 bash[75068]: debug 2023-06-03T19:41:12.050+0000 7f0ffccbac80 4 rocksdb: Options.max_background_flushes: -1 2023-06-03T19:41:12.301 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:41:12 smithi089 bash[75068]: debug 2023-06-03T19:41:12.050+0000 7f0ffccbac80 4 rocksdb: Compression algorithms supported: 2023-06-03T19:41:12.301 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:41:12 smithi089 bash[75068]: debug 2023-06-03T19:41:12.050+0000 7f0ffccbac80 4 rocksdb: kZSTDNotFinalCompression supported: 0 2023-06-03T19:41:12.301 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:41:12 smithi089 bash[75068]: debug 2023-06-03T19:41:12.050+0000 7f0ffccbac80 4 rocksdb: kZSTD supported: 0 2023-06-03T19:41:12.301 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:41:12 smithi089 bash[75068]: debug 2023-06-03T19:41:12.050+0000 7f0ffccbac80 4 rocksdb: kXpressCompression supported: 0 2023-06-03T19:41:12.302 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:41:12 smithi089 bash[75068]: debug 2023-06-03T19:41:12.050+0000 7f0ffccbac80 4 rocksdb: kLZ4HCCompression supported: 1 2023-06-03T19:41:12.302 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:41:12 smithi089 bash[75068]: debug 2023-06-03T19:41:12.050+0000 7f0ffccbac80 4 rocksdb: kZlibCompression supported: 1 2023-06-03T19:41:12.302 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:41:12 smithi089 bash[75068]: debug 2023-06-03T19:41:12.050+0000 7f0ffccbac80 4 rocksdb: kSnappyCompression supported: 1 2023-06-03T19:41:12.302 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:41:12 smithi089 bash[75068]: debug 2023-06-03T19:41:12.050+0000 7f0ffccbac80 4 rocksdb: kLZ4Compression supported: 1 2023-06-03T19:41:12.302 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:41:12 smithi089 bash[75068]: debug 2023-06-03T19:41:12.050+0000 7f0ffccbac80 4 rocksdb: kBZip2Compression supported: 0 2023-06-03T19:41:12.302 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:41:12 smithi089 bash[75068]: debug 2023-06-03T19:41:12.050+0000 7f0ffccbac80 4 rocksdb: Fast CRC32 supported: Supported on x86 2023-06-03T19:41:12.302 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:41:12 smithi089 bash[75068]: debug 2023-06-03T19:41:12.050+0000 7f0ffccbac80 4 rocksdb: DMutex implementation: pthread_mutex_t 2023-06-03T19:41:12.302 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:41:12 smithi089 bash[75068]: debug 2023-06-03T19:41:12.050+0000 7f0ffccbac80 4 rocksdb: [db/version_set.cc:5527] Recovering from manifest file: /var/lib/ceph/mon/ceph-a/store.db/MANIFEST-000008 2023-06-03T19:41:12.303 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:41:12 smithi089 bash[75068]: debug 2023-06-03T19:41:12.050+0000 7f0ffccbac80 4 rocksdb: [db/column_family.cc:630] --------------- Options for column family [default]: 2023-06-03T19:41:12.303 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:41:12 smithi089 bash[75068]: debug 2023-06-03T19:41:12.050+0000 7f0ffccbac80 4 rocksdb: Options.comparator: leveldb.BytewiseComparator 2023-06-03T19:41:12.303 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:41:12 smithi089 bash[75068]: debug 2023-06-03T19:41:12.050+0000 7f0ffccbac80 4 rocksdb: Options.merge_operator: 2023-06-03T19:41:12.303 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:41:12 smithi089 bash[75068]: debug 2023-06-03T19:41:12.050+0000 7f0ffccbac80 4 rocksdb: Options.compaction_filter: None 2023-06-03T19:41:12.303 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:41:12 smithi089 bash[75068]: debug 2023-06-03T19:41:12.050+0000 7f0ffccbac80 4 rocksdb: Options.compaction_filter_factory: None 2023-06-03T19:41:12.303 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:41:12 smithi089 bash[75068]: debug 2023-06-03T19:41:12.050+0000 7f0ffccbac80 4 rocksdb: Options.sst_partitioner_factory: None 2023-06-03T19:41:12.303 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:41:12 smithi089 bash[75068]: debug 2023-06-03T19:41:12.050+0000 7f0ffccbac80 4 rocksdb: Options.memtable_factory: SkipListFactory 2023-06-03T19:41:12.304 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:41:12 smithi089 bash[75068]: debug 2023-06-03T19:41:12.050+0000 7f0ffccbac80 4 rocksdb: Options.table_factory: BlockBasedTable 2023-06-03T19:41:12.304 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:41:12 smithi089 bash[75068]: debug 2023-06-03T19:41:12.050+0000 7f0ffccbac80 4 rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x56382420d560) 2023-06-03T19:41:12.305 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:41:12 smithi089 bash[75068]: cache_index_and_filter_blocks: 1 2023-06-03T19:41:12.305 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:41:12 smithi089 bash[75068]: cache_index_and_filter_blocks_with_high_priority: 0 2023-06-03T19:41:12.305 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:41:12 smithi089 bash[75068]: pin_l0_filter_and_index_blocks_in_cache: 0 2023-06-03T19:41:12.305 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:41:12 smithi089 bash[75068]: pin_top_level_index_and_filter: 1 2023-06-03T19:41:12.306 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:41:12 smithi089 bash[75068]: index_type: 0 2023-06-03T19:41:12.306 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:41:12 smithi089 bash[75068]: data_block_index_type: 0 2023-06-03T19:41:12.306 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:41:12 smithi089 bash[75068]: index_shortening: 1 2023-06-03T19:41:12.306 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:41:12 smithi089 bash[75068]: data_block_hash_table_util_ratio: 0.750000 2023-06-03T19:41:12.307 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:41:12 smithi089 bash[75068]: checksum: 4 2023-06-03T19:41:12.307 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:41:12 smithi089 bash[75068]: no_block_cache: 0 2023-06-03T19:41:12.307 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:41:12 smithi089 bash[75068]: block_cache: 0x56382351d350 2023-06-03T19:41:12.307 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:41:12 smithi089 bash[75068]: block_cache_name: BinnedLRUCache 2023-06-03T19:41:12.308 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:41:12 smithi089 bash[75068]: block_cache_options: 2023-06-03T19:41:12.308 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:41:12 smithi089 bash[75068]: capacity : 536870912 2023-06-03T19:41:12.308 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:41:12 smithi089 bash[75068]: num_shard_bits : 4 2023-06-03T19:41:12.308 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:41:12 smithi089 bash[75068]: strict_capacity_limit : 0 2023-06-03T19:41:12.309 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:41:12 smithi089 bash[75068]: high_pri_pool_ratio: 0.000 2023-06-03T19:41:12.309 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:41:12 smithi089 bash[75068]: block_cache_compressed: (nil) 2023-06-03T19:41:12.309 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:41:12 smithi089 bash[75068]: persistent_cache: (nil) 2023-06-03T19:41:12.309 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:41:12 smithi089 bash[75068]: block_size: 4096 2023-06-03T19:41:12.310 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:41:12 smithi089 bash[75068]: block_size_deviation: 10 2023-06-03T19:41:12.310 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:41:12 smithi089 bash[75068]: block_restart_interval: 16 2023-06-03T19:41:12.310 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:41:12 smithi089 bash[75068]: index_block_restart_interval: 1 2023-06-03T19:41:12.310 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:41:12 smithi089 bash[75068]: metadata_block_size: 4096 2023-06-03T19:41:12.311 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:41:12 smithi089 bash[75068]: partition_filters: 0 2023-06-03T19:41:12.311 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:41:12 smithi089 bash[75068]: use_delta_encoding: 1 2023-06-03T19:41:12.311 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:41:12 smithi089 bash[75068]: filter_policy: bloomfilter 2023-06-03T19:41:12.311 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:41:12 smithi089 bash[75068]: whole_key_filtering: 1 2023-06-03T19:41:12.312 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:41:12 smithi089 bash[75068]: verify_compression: 0 2023-06-03T19:41:12.312 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:41:12 smithi089 bash[75068]: read_amp_bytes_per_bit: 0 2023-06-03T19:41:12.312 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:41:12 smithi089 bash[75068]: format_version: 5 2023-06-03T19:41:12.312 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:41:12 smithi089 bash[75068]: enable_index_compression: 1 2023-06-03T19:41:12.313 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:41:12 smithi089 bash[75068]: block_align: 0 2023-06-03T19:41:12.313 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:41:12 smithi089 bash[75068]: max_auto_readahead_size: 262144 2023-06-03T19:41:12.313 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:41:12 smithi089 bash[75068]: prepopulate_block_cache: 0 2023-06-03T19:41:12.313 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:41:12 smithi089 bash[75068]: initial_auto_readahead_size: 8192 2023-06-03T19:41:12.313 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:41:12 smithi089 bash[75068]: num_file_reads_for_auto_readahead: 2 2023-06-03T19:41:12.313 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:41:12 smithi089 bash[75068]: debug 2023-06-03T19:41:12.050+0000 7f0ffccbac80 4 rocksdb: Options.write_buffer_size: 33554432 2023-06-03T19:41:12.314 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:41:12 smithi089 bash[75068]: debug 2023-06-03T19:41:12.050+0000 7f0ffccbac80 4 rocksdb: Options.max_write_buffer_number: 2 2023-06-03T19:41:12.314 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:41:12 smithi089 bash[75068]: debug 2023-06-03T19:41:12.050+0000 7f0ffccbac80 4 rocksdb: Options.compression: NoCompression 2023-06-03T19:41:12.314 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:41:12 smithi089 bash[75068]: debug 2023-06-03T19:41:12.050+0000 7f0ffccbac80 4 rocksdb: Options.bottommost_compression: Disabled 2023-06-03T19:41:12.314 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:41:12 smithi089 bash[75068]: debug 2023-06-03T19:41:12.050+0000 7f0ffccbac80 4 rocksdb: Options.prefix_extractor: nullptr 2023-06-03T19:41:12.314 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:41:12 smithi089 bash[75068]: debug 2023-06-03T19:41:12.050+0000 7f0ffccbac80 4 rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2023-06-03T19:41:12.314 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:41:12 smithi089 bash[75068]: debug 2023-06-03T19:41:12.050+0000 7f0ffccbac80 4 rocksdb: Options.num_levels: 7 2023-06-03T19:41:12.314 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:41:12 smithi089 bash[75068]: debug 2023-06-03T19:41:12.050+0000 7f0ffccbac80 4 rocksdb: Options.min_write_buffer_number_to_merge: 1 2023-06-03T19:41:12.314 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:41:12 smithi089 bash[75068]: debug 2023-06-03T19:41:12.050+0000 7f0ffccbac80 4 rocksdb: Options.max_write_buffer_number_to_maintain: 0 2023-06-03T19:41:12.315 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:41:12 smithi089 bash[75068]: debug 2023-06-03T19:41:12.050+0000 7f0ffccbac80 4 rocksdb: Options.max_write_buffer_size_to_maintain: 0 2023-06-03T19:41:12.315 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:41:12 smithi089 bash[75068]: debug 2023-06-03T19:41:12.050+0000 7f0ffccbac80 4 rocksdb: Options.bottommost_compression_opts.window_bits: -14 2023-06-03T19:41:12.315 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:41:12 smithi089 bash[75068]: debug 2023-06-03T19:41:12.050+0000 7f0ffccbac80 4 rocksdb: Options.bottommost_compression_opts.level: 32767 2023-06-03T19:41:12.315 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:41:12 smithi089 bash[75068]: debug 2023-06-03T19:41:12.050+0000 7f0ffccbac80 4 rocksdb: Options.bottommost_compression_opts.strategy: 0 2023-06-03T19:41:12.315 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:41:12 smithi089 bash[75068]: debug 2023-06-03T19:41:12.050+0000 7f0ffccbac80 4 rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2023-06-03T19:41:12.315 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:41:12 smithi089 bash[75068]: debug 2023-06-03T19:41:12.050+0000 7f0ffccbac80 4 rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2023-06-03T19:41:12.315 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:41:12 smithi089 bash[75068]: debug 2023-06-03T19:41:12.050+0000 7f0ffccbac80 4 rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2023-06-03T19:41:12.316 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:41:12 smithi089 bash[75068]: debug 2023-06-03T19:41:12.050+0000 7f0ffccbac80 4 rocksdb: Options.bottommost_compression_opts.enabled: false 2023-06-03T19:41:12.316 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:41:12 smithi089 bash[75068]: debug 2023-06-03T19:41:12.050+0000 7f0ffccbac80 4 rocksdb: Options.bottommost_compression_opts.max_dict_buffer_bytes: 0 2023-06-03T19:41:12.316 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:41:12 smithi089 bash[75068]: debug 2023-06-03T19:41:12.050+0000 7f0ffccbac80 4 rocksdb: Options.bottommost_compression_opts.use_zstd_dict_trainer: true 2023-06-03T19:41:12.316 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:41:12 smithi089 bash[75068]: debug 2023-06-03T19:41:12.050+0000 7f0ffccbac80 4 rocksdb: Options.compression_opts.window_bits: -14 2023-06-03T19:41:12.316 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:41:12 smithi089 bash[75068]: debug 2023-06-03T19:41:12.050+0000 7f0ffccbac80 4 rocksdb: Options.compression_opts.level: 32767 2023-06-03T19:41:12.316 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:41:12 smithi089 bash[75068]: debug 2023-06-03T19:41:12.050+0000 7f0ffccbac80 4 rocksdb: Options.compression_opts.strategy: 0 2023-06-03T19:41:12.316 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:41:12 smithi089 bash[75068]: debug 2023-06-03T19:41:12.050+0000 7f0ffccbac80 4 rocksdb: Options.compression_opts.max_dict_bytes: 0 2023-06-03T19:41:12.316 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:41:12 smithi089 bash[75068]: debug 2023-06-03T19:41:12.050+0000 7f0ffccbac80 4 rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2023-06-03T19:41:12.317 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:41:12 smithi089 bash[75068]: debug 2023-06-03T19:41:12.050+0000 7f0ffccbac80 4 rocksdb: Options.compression_opts.use_zstd_dict_trainer: true 2023-06-03T19:41:12.317 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:41:12 smithi089 bash[75068]: debug 2023-06-03T19:41:12.050+0000 7f0ffccbac80 4 rocksdb: Options.compression_opts.parallel_threads: 1 2023-06-03T19:41:12.317 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:41:12 smithi089 bash[75068]: debug 2023-06-03T19:41:12.050+0000 7f0ffccbac80 4 rocksdb: Options.compression_opts.enabled: false 2023-06-03T19:41:12.317 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:41:12 smithi089 bash[75068]: debug 2023-06-03T19:41:12.050+0000 7f0ffccbac80 4 rocksdb: Options.compression_opts.max_dict_buffer_bytes: 0 2023-06-03T19:41:12.317 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:41:12 smithi089 bash[75068]: debug 2023-06-03T19:41:12.050+0000 7f0ffccbac80 4 rocksdb: Options.level0_file_num_compaction_trigger: 4 2023-06-03T19:41:12.317 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:41:12 smithi089 bash[75068]: debug 2023-06-03T19:41:12.050+0000 7f0ffccbac80 4 rocksdb: Options.level0_slowdown_writes_trigger: 20 2023-06-03T19:41:12.317 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:41:12 smithi089 bash[75068]: debug 2023-06-03T19:41:12.050+0000 7f0ffccbac80 4 rocksdb: Options.level0_stop_writes_trigger: 36 2023-06-03T19:41:12.317 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:41:12 smithi089 bash[75068]: debug 2023-06-03T19:41:12.050+0000 7f0ffccbac80 4 rocksdb: Options.target_file_size_base: 67108864 2023-06-03T19:41:12.318 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:41:12 smithi089 bash[75068]: debug 2023-06-03T19:41:12.050+0000 7f0ffccbac80 4 rocksdb: Options.target_file_size_multiplier: 1 2023-06-03T19:41:12.318 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:41:12 smithi089 bash[75068]: debug 2023-06-03T19:41:12.050+0000 7f0ffccbac80 4 rocksdb: Options.max_bytes_for_level_base: 268435456 2023-06-03T19:41:12.318 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:41:12 smithi089 bash[75068]: debug 2023-06-03T19:41:12.050+0000 7f0ffccbac80 4 rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2023-06-03T19:41:12.318 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:41:12 smithi089 bash[75068]: debug 2023-06-03T19:41:12.050+0000 7f0ffccbac80 4 rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2023-06-03T19:41:12.318 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:41:12 smithi089 bash[75068]: debug 2023-06-03T19:41:12.050+0000 7f0ffccbac80 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2023-06-03T19:41:12.318 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:41:12 smithi089 bash[75068]: debug 2023-06-03T19:41:12.050+0000 7f0ffccbac80 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2023-06-03T19:41:12.318 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:41:12 smithi089 bash[75068]: debug 2023-06-03T19:41:12.050+0000 7f0ffccbac80 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2023-06-03T19:41:12.318 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:41:12 smithi089 bash[75068]: debug 2023-06-03T19:41:12.050+0000 7f0ffccbac80 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2023-06-03T19:41:12.319 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:41:12 smithi089 bash[75068]: debug 2023-06-03T19:41:12.050+0000 7f0ffccbac80 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2023-06-03T19:41:12.319 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:41:12 smithi089 bash[75068]: debug 2023-06-03T19:41:12.050+0000 7f0ffccbac80 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2023-06-03T19:41:12.319 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:41:12 smithi089 bash[75068]: debug 2023-06-03T19:41:12.050+0000 7f0ffccbac80 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2023-06-03T19:41:12.319 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:41:12 smithi089 bash[75068]: debug 2023-06-03T19:41:12.050+0000 7f0ffccbac80 4 rocksdb: Options.max_sequential_skip_in_iterations: 8 2023-06-03T19:41:12.319 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:41:12 smithi089 bash[75068]: debug 2023-06-03T19:41:12.050+0000 7f0ffccbac80 4 rocksdb: Options.max_compaction_bytes: 1677721600 2023-06-03T19:41:12.319 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:41:12 smithi089 bash[75068]: debug 2023-06-03T19:41:12.050+0000 7f0ffccbac80 4 rocksdb: Options.ignore_max_compaction_bytes_for_input: true 2023-06-03T19:41:12.319 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:41:12 smithi089 bash[75068]: debug 2023-06-03T19:41:12.050+0000 7f0ffccbac80 4 rocksdb: Options.arena_block_size: 1048576 2023-06-03T19:41:12.319 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:41:12 smithi089 bash[75068]: debug 2023-06-03T19:41:12.050+0000 7f0ffccbac80 4 rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2023-06-03T19:41:12.320 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:41:12 smithi089 bash[75068]: debug 2023-06-03T19:41:12.050+0000 7f0ffccbac80 4 rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2023-06-03T19:41:12.320 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:41:12 smithi089 bash[75068]: debug 2023-06-03T19:41:12.050+0000 7f0ffccbac80 4 rocksdb: Options.disable_auto_compactions: 0 2023-06-03T19:41:12.320 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:41:12 smithi089 bash[75068]: debug 2023-06-03T19:41:12.050+0000 7f0ffccbac80 4 rocksdb: Options.compaction_style: kCompactionStyleLevel 2023-06-03T19:41:12.320 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:41:12 smithi089 bash[75068]: debug 2023-06-03T19:41:12.050+0000 7f0ffccbac80 4 rocksdb: Options.compaction_pri: kMinOverlappingRatio 2023-06-03T19:41:12.320 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:41:12 smithi089 bash[75068]: debug 2023-06-03T19:41:12.050+0000 7f0ffccbac80 4 rocksdb: Options.compaction_options_universal.size_ratio: 1 2023-06-03T19:41:12.320 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:41:12 smithi089 bash[75068]: debug 2023-06-03T19:41:12.050+0000 7f0ffccbac80 4 rocksdb: Options.compaction_options_universal.min_merge_width: 2 2023-06-03T19:41:12.320 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:41:12 smithi089 bash[75068]: debug 2023-06-03T19:41:12.050+0000 7f0ffccbac80 4 rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2023-06-03T19:41:12.321 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:41:12 smithi089 bash[75068]: debug 2023-06-03T19:41:12.050+0000 7f0ffccbac80 4 rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2023-06-03T19:41:12.321 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:41:12 smithi089 bash[75068]: debug 2023-06-03T19:41:12.050+0000 7f0ffccbac80 4 rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2023-06-03T19:41:12.321 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:41:12 smithi089 bash[75068]: debug 2023-06-03T19:41:12.050+0000 7f0ffccbac80 4 rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2023-06-03T19:41:12.321 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:41:12 smithi089 bash[75068]: debug 2023-06-03T19:41:12.050+0000 7f0ffccbac80 4 rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2023-06-03T19:41:12.321 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:41:12 smithi089 bash[75068]: debug 2023-06-03T19:41:12.050+0000 7f0ffccbac80 4 rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2023-06-03T19:41:12.321 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:41:12 smithi089 bash[75068]: debug 2023-06-03T19:41:12.050+0000 7f0ffccbac80 4 rocksdb: Options.table_properties_collectors: 2023-06-03T19:41:12.321 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:41:12 smithi089 bash[75068]: debug 2023-06-03T19:41:12.050+0000 7f0ffccbac80 4 rocksdb: Options.inplace_update_support: 0 2023-06-03T19:41:12.321 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:41:12 smithi089 bash[75068]: debug 2023-06-03T19:41:12.050+0000 7f0ffccbac80 4 rocksdb: Options.inplace_update_num_locks: 10000 2023-06-03T19:41:12.322 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:41:12 smithi089 bash[75068]: debug 2023-06-03T19:41:12.050+0000 7f0ffccbac80 4 rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2023-06-03T19:41:12.322 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:41:12 smithi089 bash[75068]: debug 2023-06-03T19:41:12.050+0000 7f0ffccbac80 4 rocksdb: Options.memtable_whole_key_filtering: 0 2023-06-03T19:41:12.322 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:41:12 smithi089 bash[75068]: debug 2023-06-03T19:41:12.050+0000 7f0ffccbac80 4 rocksdb: Options.memtable_huge_page_size: 0 2023-06-03T19:41:12.322 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:41:12 smithi089 bash[75068]: debug 2023-06-03T19:41:12.050+0000 7f0ffccbac80 4 rocksdb: Options.bloom_locality: 0 2023-06-03T19:41:12.322 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:41:12 smithi089 bash[75068]: debug 2023-06-03T19:41:12.050+0000 7f0ffccbac80 4 rocksdb: Options.max_successive_merges: 0 2023-06-03T19:41:12.322 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:41:12 smithi089 bash[75068]: debug 2023-06-03T19:41:12.050+0000 7f0ffccbac80 4 rocksdb: Options.optimize_filters_for_hits: 0 2023-06-03T19:41:12.322 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:41:12 smithi089 bash[75068]: debug 2023-06-03T19:41:12.050+0000 7f0ffccbac80 4 rocksdb: Options.paranoid_file_checks: 0 2023-06-03T19:41:12.322 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:41:12 smithi089 bash[75068]: debug 2023-06-03T19:41:12.050+0000 7f0ffccbac80 4 rocksdb: Options.force_consistency_checks: 1 2023-06-03T19:41:12.323 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:41:12 smithi089 bash[75068]: debug 2023-06-03T19:41:12.050+0000 7f0ffccbac80 4 rocksdb: Options.report_bg_io_stats: 0 2023-06-03T19:41:12.323 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:41:12 smithi089 bash[75068]: debug 2023-06-03T19:41:12.050+0000 7f0ffccbac80 4 rocksdb: Options.ttl: 2592000 2023-06-03T19:41:12.323 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:41:12 smithi089 bash[75068]: debug 2023-06-03T19:41:12.050+0000 7f0ffccbac80 4 rocksdb: Options.periodic_compaction_seconds: 0 2023-06-03T19:41:12.323 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:41:12 smithi089 bash[75068]: debug 2023-06-03T19:41:12.050+0000 7f0ffccbac80 4 rocksdb: Options.preclude_last_level_data_seconds: 0 2023-06-03T19:41:12.323 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:41:12 smithi089 bash[75068]: debug 2023-06-03T19:41:12.050+0000 7f0ffccbac80 4 rocksdb: Options.preserve_internal_time_seconds: 0 2023-06-03T19:41:12.323 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:41:12 smithi089 bash[75068]: debug 2023-06-03T19:41:12.050+0000 7f0ffccbac80 4 rocksdb: Options.enable_blob_files: false 2023-06-03T19:41:12.323 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:41:12 smithi089 bash[75068]: debug 2023-06-03T19:41:12.050+0000 7f0ffccbac80 4 rocksdb: Options.min_blob_size: 0 2023-06-03T19:41:12.323 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:41:12 smithi089 bash[75068]: debug 2023-06-03T19:41:12.050+0000 7f0ffccbac80 4 rocksdb: Options.blob_file_size: 268435456 2023-06-03T19:41:12.324 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:41:12 smithi089 bash[75068]: debug 2023-06-03T19:41:12.050+0000 7f0ffccbac80 4 rocksdb: Options.blob_compression_type: NoCompression 2023-06-03T19:41:12.324 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:41:12 smithi089 bash[75068]: debug 2023-06-03T19:41:12.050+0000 7f0ffccbac80 4 rocksdb: Options.enable_blob_garbage_collection: false 2023-06-03T19:41:12.324 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:41:12 smithi089 bash[75068]: debug 2023-06-03T19:41:12.050+0000 7f0ffccbac80 4 rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2023-06-03T19:41:12.324 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:41:12 smithi089 bash[75068]: debug 2023-06-03T19:41:12.050+0000 7f0ffccbac80 4 rocksdb: Options.blob_garbage_collection_force_threshold: 1.000000 2023-06-03T19:41:12.324 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:41:12 smithi089 bash[75068]: debug 2023-06-03T19:41:12.050+0000 7f0ffccbac80 4 rocksdb: Options.blob_compaction_readahead_size: 0 2023-06-03T19:41:12.324 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:41:12 smithi089 bash[75068]: debug 2023-06-03T19:41:12.050+0000 7f0ffccbac80 4 rocksdb: Options.blob_file_starting_level: 0 2023-06-03T19:41:12.324 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:41:12 smithi089 bash[75068]: debug 2023-06-03T19:41:12.050+0000 7f0ffccbac80 4 rocksdb: Options.experimental_mempurge_threshold: 0.000000 2023-06-03T19:41:12.325 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:41:12 smithi089 bash[75068]: debug 2023-06-03T19:41:12.054+0000 7f0ff14d8700 3 rocksdb: [table/block_based/block_based_table_reader.cc:721] At least one SST file opened without unique ID to verify: 26.sst 2023-06-03T19:41:12.325 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:41:12 smithi089 bash[75068]: debug 2023-06-03T19:41:12.054+0000 7f0ffccbac80 4 rocksdb: [db/version_set.cc:5566] Recovered from manifest file:/var/lib/ceph/mon/ceph-a/store.db/MANIFEST-000008 succeeded,manifest_file_number is 8, next_file_number is 28, last_sequence is 15793, log_number is 24,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2023-06-03T19:41:12.325 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:41:12 smithi089 bash[75068]: debug 2023-06-03T19:41:12.054+0000 7f0ffccbac80 4 rocksdb: [db/version_set.cc:5581] Column family [default] (ID 0), log number is 24 2023-06-03T19:41:12.325 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:41:12 smithi089 bash[75068]: debug 2023-06-03T19:41:12.054+0000 7f0ffccbac80 4 rocksdb: [db/db_impl/db_impl_open.cc:539] DB ID: 0078eeb4-08ad-41b7-bb3b-92dda6327849 2023-06-03T19:41:12.325 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:41:12 smithi089 bash[75068]: debug 2023-06-03T19:41:12.054+0000 7f0ffccbac80 4 rocksdb: EVENT_LOG_v1 {"time_micros": 1685821272058107, "job": 1, "event": "recovery_started", "wal_files": [24]} 2023-06-03T19:41:12.325 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:41:12 smithi089 bash[75068]: debug 2023-06-03T19:41:12.054+0000 7f0ffccbac80 4 rocksdb: [db/db_impl/db_impl_open.cc:1043] Recovering log #24 mode 2 2023-06-03T19:41:12.325 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:41:12 smithi089 bash[75068]: debug 2023-06-03T19:41:12.058+0000 7f0ffccbac80 4 rocksdb: EVENT_LOG_v1 {"time_micros": 1685821272062875, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 29, "file_size": 828327, "file_checksum": "", "file_checksum_func_name": "Unknown", "smallest_seqno": 15794, "largest_seqno": 16021, "table_properties": {"data_size": 826091, "index_size": 968, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 261, "raw_key_size": 2296, "raw_average_key_size": 24, "raw_value_size": 823469, "raw_average_value_size": 8760, "num_data_blocks": 42, "num_entries": 94, "num_filter_entries": 94, "num_deletions": 2, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "bloomfilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; max_dict_buffer_bytes=0; use_zstd_dict_trainer=1; ", "creation_time": 1685821272, "oldest_key_time": 0, "file_creation_time": 0, "slow_compression_estimated_data_size": 0, "fast_compression_estimated_data_size": 0, "db_id": "0078eeb4-08ad-41b7-bb3b-92dda6327849", "db_session_id": "919HVZ0G2IFV3IQWARLX", "orig_file_number": 29, "seqno_to_time_mapping": "N/A"}} 2023-06-03T19:41:12.325 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:41:12 smithi089 bash[75068]: debug 2023-06-03T19:41:12.058+0000 7f0ffccbac80 4 rocksdb: EVENT_LOG_v1 {"time_micros": 1685821272062982, "job": 1, "event": "recovery_finished"} 2023-06-03T19:41:12.326 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:41:12 smithi089 bash[75068]: debug 2023-06-03T19:41:12.058+0000 7f0ffccbac80 4 rocksdb: [db/version_set.cc:5047] Creating manifest 31 2023-06-03T19:41:12.326 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:41:12 smithi089 bash[75068]: debug 2023-06-03T19:41:12.062+0000 7f0ffccbac80 4 rocksdb: [file/delete_scheduler.cc:74] Deleted file /var/lib/ceph/mon/ceph-a/store.db/000024.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2023-06-03T19:41:12.326 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:41:12 smithi089 bash[75068]: debug 2023-06-03T19:41:12.062+0000 7f0ffccbac80 4 rocksdb: [db/db_impl/db_impl_open.cc:1987] SstFileManager instance 0x5638235be000 2023-06-03T19:41:12.326 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:41:12 smithi089 bash[75068]: debug 2023-06-03T19:41:12.062+0000 7f0ffccbac80 4 rocksdb: DB pointer 0x5638235ac000 2023-06-03T19:41:12.326 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:41:12 smithi089 bash[75068]: debug 2023-06-03T19:41:12.062+0000 7f0feb4cc700 4 rocksdb: [db/db_impl/db_impl.cc:1109] ------- DUMPING STATS ------- 2023-06-03T19:41:12.326 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:41:12 smithi089 bash[75068]: debug 2023-06-03T19:41:12.062+0000 7f0feb4cc700 4 rocksdb: [db/db_impl/db_impl.cc:1111] 2023-06-03T19:41:12.326 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:41:12 smithi089 bash[75068]: ** DB Stats ** 2023-06-03T19:41:12.326 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:41:12 smithi089 bash[75068]: Uptime(secs): 0.0 total, 0.0 interval 2023-06-03T19:41:12.327 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:41:12 smithi089 bash[75068]: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2023-06-03T19:41:12.327 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:41:12 smithi089 bash[75068]: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2023-06-03T19:41:12.327 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:41:12 smithi089 bash[75068]: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2023-06-03T19:41:12.327 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:41:12 smithi089 bash[75068]: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2023-06-03T19:41:12.327 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:41:12 smithi089 bash[75068]: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2023-06-03T19:41:12.327 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:41:12 smithi089 bash[75068]: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2023-06-03T19:41:12.327 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:41:12 smithi089 bash[75068]: ** Compaction Stats [default] ** 2023-06-03T19:41:12.328 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:41:12 smithi089 bash[75068]: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2023-06-03T19:41:12.328 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:41:12 smithi089 bash[75068]: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ 2023-06-03T19:41:12.328 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:41:12 smithi089 bash[75068]: L0 1/0 808.91 KB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 267.0 0.00 0.00 1 0.003 0 0 0.0 0.0 2023-06-03T19:41:12.328 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:41:12 smithi089 bash[75068]: L6 1/0 31.98 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 0.0 0.0 2023-06-03T19:41:12.328 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:41:12 smithi089 bash[75068]: Sum 2/0 32.77 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 267.0 0.00 0.00 1 0.003 0 0 0.0 0.0 2023-06-03T19:41:12.328 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:41:12 smithi089 bash[75068]: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 267.0 0.00 0.00 1 0.003 0 0 0.0 0.0 2023-06-03T19:41:12.328 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:41:12 smithi089 bash[75068]: ** Compaction Stats [default] ** 2023-06-03T19:41:12.328 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:41:12 smithi089 bash[75068]: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2023-06-03T19:41:12.329 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:41:12 smithi089 bash[75068]: --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2023-06-03T19:41:12.329 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:41:12 smithi089 bash[75068]: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 267.0 0.00 0.00 1 0.003 0 0 0.0 0.0 2023-06-03T19:41:12.329 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:41:12 smithi089 bash[75068]: Blob file count: 0, total size: 0.0 GB, garbage size: 0.0 GB, space amp: 0.0 2023-06-03T19:41:12.329 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:41:12 smithi089 bash[75068]: Uptime(secs): 0.0 total, 0.0 interval 2023-06-03T19:41:12.329 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:41:12 smithi089 bash[75068]: Flush(GB): cumulative 0.001, interval 0.001 2023-06-03T19:41:12.329 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:41:12 smithi089 bash[75068]: AddFile(GB): cumulative 0.000, interval 0.000 2023-06-03T19:41:12.329 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:41:12 smithi089 bash[75068]: AddFile(Total Files): cumulative 0, interval 0 2023-06-03T19:41:12.329 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:41:12 smithi089 bash[75068]: AddFile(L0 Files): cumulative 0, interval 0 2023-06-03T19:41:12.330 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:41:12 smithi089 bash[75068]: AddFile(Keys): cumulative 0, interval 0 2023-06-03T19:41:12.330 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:41:12 smithi089 bash[75068]: Cumulative compaction: 0.00 GB write, 84.90 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-06-03T19:41:12.330 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:41:12 smithi089 bash[75068]: Interval compaction: 0.00 GB write, 84.90 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-06-03T19:41:12.330 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:41:12 smithi089 bash[75068]: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2023-06-03T19:41:12.330 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:41:12 smithi089 bash[75068]: Block cache BinnedLRUCache@0x56382351d350#7 capacity: 512.00 MB usage: 1.39 KB table_size: 0 occupancy: 18446744073709551615 collections: 1 last_copies: 0 last_secs: 2.2e-05 secs_since: 0 2023-06-03T19:41:12.330 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:41:12 smithi089 bash[75068]: Block cache entry stats(count,size,portion): FilterBlock(1,0.31 KB,5.96046e-05%) IndexBlock(1,1.08 KB,0.000205636%) Misc(1,0.00 KB,0%) 2023-06-03T19:41:12.330 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:41:12 smithi089 bash[75068]: ** File Read Latency Histogram By Level [default] ** 2023-06-03T19:41:12.330 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:41:12 smithi089 bash[75068]: debug 2023-06-03T19:41:12.062+0000 7f0ffccbac80 0 starting mon.a rank 0 at public addrs [v2:172.21.15.89:3300/0,v1:172.21.15.89:6789/0] at bind addrs [v2:172.21.15.89:3300/0,v1:172.21.15.89:6789/0] mon_data /var/lib/ceph/mon/ceph-a fsid f1db3478-0243-11ee-9b1d-001a4aab830c 2023-06-03T19:41:12.331 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:41:12 smithi089 bash[75068]: debug 2023-06-03T19:41:12.062+0000 7f0ffccbac80 1 mon.a@-1(???) e3 preinit fsid f1db3478-0243-11ee-9b1d-001a4aab830c 2023-06-03T19:41:12.331 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:41:12 smithi089 bash[75068]: debug 2023-06-03T19:41:12.062+0000 7f0ffccbac80 0 mon.a@-1(???).mds e7 new map 2023-06-03T19:41:12.331 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:41:12 smithi089 bash[75068]: debug 2023-06-03T19:41:12.062+0000 7f0ffccbac80 0 mon.a@-1(???).mds e7 print_map 2023-06-03T19:41:12.331 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:41:12 smithi089 bash[75068]: e7 2023-06-03T19:41:12.331 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:41:12 smithi089 bash[75068]: enable_multiple, ever_enabled_multiple: 1,1 2023-06-03T19:41:12.331 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:41:12 smithi089 bash[75068]: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2023-06-03T19:41:12.331 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:41:12 smithi089 bash[75068]: legacy client fscid: 1 2023-06-03T19:41:12.331 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:41:12 smithi089 bash[75068]: 2023-06-03T19:41:12.332 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:41:12 smithi089 bash[75068]: Filesystem 'foo' (1) 2023-06-03T19:41:12.332 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:41:12 smithi089 bash[75068]: fs_name foo 2023-06-03T19:41:12.332 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:41:12 smithi089 bash[75068]: epoch 5 2023-06-03T19:41:12.332 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:41:12 smithi089 bash[75068]: flags 12 joinable allow_snaps allow_multimds_snaps 2023-06-03T19:41:12.332 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:41:12 smithi089 bash[75068]: created 2023-06-03T19:34:16.805120+0000 2023-06-03T19:41:12.332 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:41:12 smithi089 bash[75068]: modified 2023-06-03T19:34:24.830695+0000 2023-06-03T19:41:12.332 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:41:12 smithi089 bash[75068]: tableserver 0 2023-06-03T19:41:12.333 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:41:12 smithi089 bash[75068]: root 0 2023-06-03T19:41:12.333 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:41:12 smithi089 bash[75068]: session_timeout 60 2023-06-03T19:41:12.333 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:41:12 smithi089 bash[75068]: session_autoclose 300 2023-06-03T19:41:12.333 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:41:12 smithi089 bash[75068]: max_file_size 1099511627776 2023-06-03T19:41:12.333 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:41:12 smithi089 bash[75068]: required_client_features {} 2023-06-03T19:41:12.333 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:41:12 smithi089 bash[75068]: last_failure 0 2023-06-03T19:41:12.333 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:41:12 smithi089 bash[75068]: last_failure_osd_epoch 0 2023-06-03T19:41:12.333 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:41:12 smithi089 bash[75068]: compat compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,7=mds uses inline data,8=no anchor table,9=file layout v2,10=snaprealm v2} 2023-06-03T19:41:12.334 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:41:12 smithi089 bash[75068]: max_mds 1 2023-06-03T19:41:12.334 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:41:12 smithi089 bash[75068]: in 0 2023-06-03T19:41:12.334 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:41:12 smithi089 bash[75068]: up {0=14646} 2023-06-03T19:41:12.334 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:41:12 smithi089 bash[75068]: failed 2023-06-03T19:41:12.334 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:41:12 smithi089 bash[75068]: damaged 2023-06-03T19:41:12.334 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:41:12 smithi089 bash[75068]: stopped 2023-06-03T19:41:12.334 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:41:12 smithi089 bash[75068]: data_pools [4] 2023-06-03T19:41:12.334 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:41:12 smithi089 bash[75068]: metadata_pool 3 2023-06-03T19:41:12.335 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:41:12 smithi089 bash[75068]: inline_data disabled 2023-06-03T19:41:12.335 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:41:12 smithi089 bash[75068]: balancer 2023-06-03T19:41:12.335 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:41:12 smithi089 bash[75068]: bal_rank_mask -1 2023-06-03T19:41:12.335 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:41:12 smithi089 bash[75068]: standby_count_wanted 1 2023-06-03T19:41:12.335 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:41:12 smithi089 bash[75068]: [mds.foo.smithi089.giuxuj{0:14646} state up:active seq 2 join_fscid=1 addr [v2:172.21.15.89:6834/1926931511,v1:172.21.15.89:6835/1926931511] compat {c=[1],r=[1],i=[7ff]}] 2023-06-03T19:41:12.335 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:41:12 smithi089 bash[75068]: 2023-06-03T19:41:12.335 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:41:12 smithi089 bash[75068]: 2023-06-03T19:41:12.335 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:41:12 smithi089 bash[75068]: Standby daemons: 2023-06-03T19:41:12.336 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:41:12 smithi089 bash[75068]: 2023-06-03T19:41:12.336 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:41:12 smithi089 bash[75068]: [mds.foo.smithi164.txgfgv{-1:24499} state up:standby seq 1 join_fscid=1 addr [v2:172.21.15.164:6832/2311698270,v1:172.21.15.164:6833/2311698270] compat {c=[1],r=[1],i=[7ff]}] 2023-06-03T19:41:12.336 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:41:12 smithi089 bash[75068]: debug 2023-06-03T19:41:12.062+0000 7f0ffccbac80 0 mon.a@-1(???).osd e71 crush map has features 3314933000854323200, adjusting msgr requires 2023-06-03T19:41:12.336 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:41:12 smithi089 bash[75068]: debug 2023-06-03T19:41:12.062+0000 7f0ffccbac80 0 mon.a@-1(???).osd e71 crush map has features 432629239337189376, adjusting msgr requires 2023-06-03T19:41:12.336 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:41:12 smithi089 bash[75068]: debug 2023-06-03T19:41:12.062+0000 7f0ffccbac80 0 mon.a@-1(???).osd e71 crush map has features 432629239337189376, adjusting msgr requires 2023-06-03T19:41:12.336 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:41:12 smithi089 bash[75068]: debug 2023-06-03T19:41:12.062+0000 7f0ffccbac80 0 mon.a@-1(???).osd e71 crush map has features 432629239337189376, adjusting msgr requires 2023-06-03T19:41:12.336 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:41:12 smithi089 bash[75068]: debug 2023-06-03T19:41:12.062+0000 7f0ffccbac80 1 mon.a@-1(???).paxosservice(auth 1..20) refresh upgraded, format 0 -> 3 2023-06-03T19:41:12.336 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:41:12 smithi089 bash[75068]: debug 2023-06-03T19:41:12.066+0000 7f0ffccbac80 20 expand_channel_meta expand map: {default=false} 2023-06-03T19:41:12.337 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:41:12 smithi089 bash[75068]: debug 2023-06-03T19:41:12.066+0000 7f0ffccbac80 20 expand_channel_meta from 'false' to 'false' 2023-06-03T19:41:12.337 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:41:12 smithi089 bash[75068]: debug 2023-06-03T19:41:12.066+0000 7f0ffccbac80 20 expand_channel_meta expanded map: {default=false} 2023-06-03T19:41:12.337 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:41:12 smithi089 bash[75068]: debug 2023-06-03T19:41:12.066+0000 7f0ffccbac80 20 expand_channel_meta expand map: {default=info} 2023-06-03T19:41:12.337 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:41:12 smithi089 bash[75068]: debug 2023-06-03T19:41:12.066+0000 7f0ffccbac80 20 expand_channel_meta from 'info' to 'info' 2023-06-03T19:41:12.337 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:41:12 smithi089 bash[75068]: debug 2023-06-03T19:41:12.066+0000 7f0ffccbac80 20 expand_channel_meta expanded map: {default=info} 2023-06-03T19:41:12.337 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:41:12 smithi089 bash[75068]: debug 2023-06-03T19:41:12.066+0000 7f0ffccbac80 20 expand_channel_meta expand map: {default=daemon} 2023-06-03T19:41:12.337 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:41:12 smithi089 bash[75068]: debug 2023-06-03T19:41:12.066+0000 7f0ffccbac80 20 expand_channel_meta from 'daemon' to 'daemon' 2023-06-03T19:41:12.337 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:41:12 smithi089 bash[75068]: debug 2023-06-03T19:41:12.066+0000 7f0ffccbac80 20 expand_channel_meta expanded map: {default=daemon} 2023-06-03T19:41:12.338 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:41:12 smithi089 bash[75068]: debug 2023-06-03T19:41:12.066+0000 7f0ffccbac80 20 expand_channel_meta expand map: {default=debug} 2023-06-03T19:41:12.338 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:41:12 smithi089 bash[75068]: debug 2023-06-03T19:41:12.066+0000 7f0ffccbac80 20 expand_channel_meta from 'debug' to 'debug' 2023-06-03T19:41:12.338 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:41:12 smithi089 bash[75068]: debug 2023-06-03T19:41:12.066+0000 7f0ffccbac80 20 expand_channel_meta expanded map: {default=debug} 2023-06-03T19:41:12.694 INFO:tasks.cephadm.ceph_manager.ceph:removing pool_name unique_pool_0 2023-06-03T19:41:12.695 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph osd pool rm unique_pool_0 unique_pool_0 --yes-i-really-really-mean-it 2023-06-03T19:41:13.836 INFO:journalctl@ceph.mgr.x.smithi164.stdout:Jun 03 19:41:13 smithi164 bash[55600]: ignoring --setuser ceph since I am not root 2023-06-03T19:41:13.836 INFO:journalctl@ceph.mgr.x.smithi164.stdout:Jun 03 19:41:13 smithi164 bash[55600]: ignoring --setgroup ceph since I am not root 2023-06-03T19:41:13.836 INFO:journalctl@ceph.mgr.x.smithi164.stdout:Jun 03 19:41:13 smithi164 bash[55600]: debug 2023-06-03T19:41:13.600+0000 7f09193ee700 1 -- 172.21.15.164:0/1969504910 <== mon.2 v2:172.21.15.89:3301/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x562dcfd05520 con 0x562dd0a84800 2023-06-03T19:41:13.836 INFO:journalctl@ceph.mgr.x.smithi164.stdout:Jun 03 19:41:13 smithi164 bash[55600]: debug 2023-06-03T19:41:13.756+0000 7f092311c200 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2023-06-03T19:41:13.837 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:41:13 smithi164 bash[26855]: cluster 2023-06-03T19:41:10.045587+0000 mgr.y (mgr.25070) 56 : cluster [DBG] pgmap v37: 89 pgs: 89 active+clean; 581 KiB data, 677 MiB used, 715 GiB / 715 GiB avail; 0 B/s wr, 780 op/s 2023-06-03T19:41:13.837 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:41:13 smithi164 bash[26855]: audit 2023-06-03T19:41:10.491911+0000 mon.c (mon.2) 77 : audit [DBG] from='client.? 172.21.15.89:0/2964839806' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-03T19:41:13.837 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:41:13 smithi164 bash[26855]: cluster 2023-06-03T19:41:12.046161+0000 mgr.y (mgr.25070) 57 : cluster [DBG] pgmap v38: 89 pgs: 89 active+clean; 581 KiB data, 677 MiB used, 715 GiB / 715 GiB avail; 0 B/s wr, 496 op/s 2023-06-03T19:41:13.837 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:41:13 smithi164 bash[26855]: audit 2023-06-03T19:41:12.954015+0000 mon.c (mon.2) 78 : audit [DBG] from='client.? 172.21.15.89:0/2315445979' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-03T19:41:13.837 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:41:13 smithi164 bash[26855]: cluster 2023-06-03T19:41:13.486622+0000 mon.a (mon.0) 1 : cluster [INF] mon.a calling monitor election 2023-06-03T19:41:13.837 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:41:13 smithi164 bash[26855]: cluster 2023-06-03T19:41:13.509128+0000 mon.a (mon.0) 2 : cluster [INF] mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2023-06-03T19:41:13.837 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:41:13 smithi164 bash[26855]: cluster 2023-06-03T19:41:13.518046+0000 mon.a (mon.0) 3 : cluster [DBG] monmap e3: 3 mons at {a=[v2:172.21.15.89:3300/0,v1:172.21.15.89:6789/0],b=[v2:172.21.15.164:3300/0,v1:172.21.15.164:6789/0],c=[v2:172.21.15.89:3301/0,v1:172.21.15.89:6790/0]} removed_ranks: {} 2023-06-03T19:41:13.838 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:41:13 smithi164 bash[26855]: cluster 2023-06-03T19:41:13.518961+0000 mon.a (mon.0) 4 : cluster [DBG] fsmap foo:1 {0=foo.smithi089.giuxuj=up:active} 1 up:standby 2023-06-03T19:41:13.838 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:41:13 smithi164 bash[26855]: cluster 2023-06-03T19:41:13.519014+0000 mon.a (mon.0) 5 : cluster [DBG] osdmap e71: 8 total, 8 up, 8 in 2023-06-03T19:41:13.838 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:41:13 smithi164 bash[26855]: cluster 2023-06-03T19:41:13.520429+0000 mon.a (mon.0) 6 : cluster [DBG] mgrmap e31: y(active, since 72s), standbys: x 2023-06-03T19:41:13.838 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:41:13 smithi164 bash[26855]: cluster 2023-06-03T19:41:13.546104+0000 mon.a (mon.0) 7 : cluster [DBG] mgrmap e32: y(active, since 72s), standbys: x 2023-06-03T19:41:13.838 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:41:13 smithi164 bash[26855]: cluster 2023-06-03T19:41:13.546148+0000 mon.a (mon.0) 8 : cluster [INF] overall HEALTH_OK 2023-06-03T19:41:13.838 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:41:13 smithi164 bash[26855]: audit 2023-06-03T19:41:13.546262+0000 mon.a (mon.0) 9 : audit [INF] from='mgr.25070 ' entity='' 2023-06-03T19:41:14.031 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 19:41:13 smithi089 bash[71163]: ignoring --setuser ceph since I am not root 2023-06-03T19:41:14.032 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 19:41:13 smithi089 bash[71163]: ignoring --setgroup ceph since I am not root 2023-06-03T19:41:14.032 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 19:41:13 smithi089 bash[71163]: debug 2023-06-03T19:41:13.606+0000 7fe4c46c4700 1 -- 172.21.15.89:0/353114219 <== mon.0 v2:172.21.15.89:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x56550be6f520 con 0x56550cbeec00 2023-06-03T19:41:14.032 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 19:41:13 smithi089 bash[71163]: debug 2023-06-03T19:41:13.762+0000 7fe4ce3f2200 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2023-06-03T19:41:14.032 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:41:13 smithi089 bash[33173]: cluster 2023-06-03T19:41:10.045587+0000 mgr.y (mgr.25070) 56 : cluster [DBG] pgmap v37: 89 pgs: 89 active+clean; 581 KiB data, 677 MiB used, 715 GiB / 715 GiB avail; 0 B/s wr, 780 op/s 2023-06-03T19:41:14.033 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:41:13 smithi089 bash[33173]: audit 2023-06-03T19:41:10.491911+0000 mon.c (mon.2) 77 : audit [DBG] from='client.? 172.21.15.89:0/2964839806' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-03T19:41:14.033 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:41:13 smithi089 bash[33173]: cluster 2023-06-03T19:41:12.046161+0000 mgr.y (mgr.25070) 57 : cluster [DBG] pgmap v38: 89 pgs: 89 active+clean; 581 KiB data, 677 MiB used, 715 GiB / 715 GiB avail; 0 B/s wr, 496 op/s 2023-06-03T19:41:14.033 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:41:13 smithi089 bash[33173]: audit 2023-06-03T19:41:12.954015+0000 mon.c (mon.2) 78 : audit [DBG] from='client.? 172.21.15.89:0/2315445979' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-03T19:41:14.033 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:41:13 smithi089 bash[33173]: cluster 2023-06-03T19:41:13.486622+0000 mon.a (mon.0) 1 : cluster [INF] mon.a calling monitor election 2023-06-03T19:41:14.033 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:41:13 smithi089 bash[33173]: cluster 2023-06-03T19:41:13.509128+0000 mon.a (mon.0) 2 : cluster [INF] mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2023-06-03T19:41:14.033 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:41:13 smithi089 bash[33173]: cluster 2023-06-03T19:41:13.518046+0000 mon.a (mon.0) 3 : cluster [DBG] monmap e3: 3 mons at {a=[v2:172.21.15.89:3300/0,v1:172.21.15.89:6789/0],b=[v2:172.21.15.164:3300/0,v1:172.21.15.164:6789/0],c=[v2:172.21.15.89:3301/0,v1:172.21.15.89:6790/0]} removed_ranks: {} 2023-06-03T19:41:14.033 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:41:13 smithi089 bash[33173]: cluster 2023-06-03T19:41:13.518961+0000 mon.a (mon.0) 4 : cluster [DBG] fsmap foo:1 {0=foo.smithi089.giuxuj=up:active} 1 up:standby 2023-06-03T19:41:14.034 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:41:13 smithi089 bash[33173]: cluster 2023-06-03T19:41:13.519014+0000 mon.a (mon.0) 5 : cluster [DBG] osdmap e71: 8 total, 8 up, 8 in 2023-06-03T19:41:14.034 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:41:13 smithi089 bash[33173]: cluster 2023-06-03T19:41:13.520429+0000 mon.a (mon.0) 6 : cluster [DBG] mgrmap e31: y(active, since 72s), standbys: x 2023-06-03T19:41:14.034 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:41:13 smithi089 bash[33173]: cluster 2023-06-03T19:41:13.546104+0000 mon.a (mon.0) 7 : cluster [DBG] mgrmap e32: y(active, since 72s), standbys: x 2023-06-03T19:41:14.034 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:41:13 smithi089 bash[33173]: cluster 2023-06-03T19:41:13.546148+0000 mon.a (mon.0) 8 : cluster [INF] overall HEALTH_OK 2023-06-03T19:41:14.034 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:41:13 smithi089 bash[33173]: audit 2023-06-03T19:41:13.546262+0000 mon.a (mon.0) 9 : audit [INF] from='mgr.25070 ' entity='' 2023-06-03T19:41:15.031 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 19:41:14 smithi089 bash[71163]: debug 2023-06-03T19:41:14.694+0000 7fe4ce3f2200 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2023-06-03T19:41:15.031 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 19:41:14 smithi089 bash[71163]: debug 2023-06-03T19:41:14.794+0000 7fe4ce3f2200 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2023-06-03T19:41:15.032 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 19:41:14 smithi089 bash[71163]: debug 2023-06-03T19:41:14.898+0000 7fe4ce3f2200 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2023-06-03T19:41:15.085 INFO:journalctl@ceph.mgr.x.smithi164.stdout:Jun 03 19:41:14 smithi164 bash[55600]: debug 2023-06-03T19:41:14.636+0000 7f092311c200 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2023-06-03T19:41:15.086 INFO:journalctl@ceph.mgr.x.smithi164.stdout:Jun 03 19:41:14 smithi164 bash[55600]: debug 2023-06-03T19:41:14.736+0000 7f092311c200 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2023-06-03T19:41:15.086 INFO:journalctl@ceph.mgr.x.smithi164.stdout:Jun 03 19:41:14 smithi164 bash[55600]: debug 2023-06-03T19:41:14.840+0000 7f092311c200 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2023-06-03T19:41:15.411 INFO:teuthology.orchestra.run.smithi089.stdout: "ceph version 18.0.0-3532-g8e24bcf4 (8e24bcf462a1d94c2e12aa4e26419634883a1e88) reef (rc)": 1 2023-06-03T19:41:15.478 INFO:journalctl@ceph.mgr.x.smithi164.stdout:Jun 03 19:41:15 smithi164 bash[55600]: debug 2023-06-03T19:41:15.104+0000 7f092311c200 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2023-06-03T19:41:15.531 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 19:41:15 smithi089 bash[71163]: debug 2023-06-03T19:41:15.186+0000 7fe4ce3f2200 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2023-06-03T19:41:15.835 INFO:journalctl@ceph.mgr.x.smithi164.stdout:Jun 03 19:41:15 smithi164 bash[55600]: debug 2023-06-03T19:41:15.464+0000 7f092311c200 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2023-06-03T19:41:15.836 INFO:journalctl@ceph.mgr.x.smithi164.stdout:Jun 03 19:41:15 smithi164 bash[55600]: debug 2023-06-03T19:41:15.628+0000 7f092311c200 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2023-06-03T19:41:15.836 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:41:15 smithi164 bash[26855]: cluster 2023-06-03T19:41:14.552585+0000 mon.a (mon.0) 10 : cluster [DBG] mgrmap e33: y(active, since 73s), standbys: x 2023-06-03T19:41:15.836 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:41:15 smithi164 bash[26855]: audit 2023-06-03T19:41:15.389612+0000 mon.c (mon.2) 79 : audit [DBG] from='client.? 172.21.15.89:0/2819795159' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-03T19:41:15.956 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 19:41:15 smithi089 bash[71163]: debug 2023-06-03T19:41:15.566+0000 7fe4ce3f2200 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2023-06-03T19:41:15.956 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 19:41:15 smithi089 bash[71163]: debug 2023-06-03T19:41:15.730+0000 7fe4ce3f2200 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2023-06-03T19:41:15.956 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:41:15 smithi089 bash[33173]: cluster 2023-06-03T19:41:14.552585+0000 mon.a (mon.0) 10 : cluster [DBG] mgrmap e33: y(active, since 73s), standbys: x 2023-06-03T19:41:15.956 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:41:15 smithi089 bash[33173]: audit 2023-06-03T19:41:15.389612+0000 mon.c (mon.2) 79 : audit [DBG] from='client.? 172.21.15.89:0/2819795159' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-03T19:41:16.281 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 19:41:15 smithi089 bash[71163]: debug 2023-06-03T19:41:15.946+0000 7fe4ce3f2200 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2023-06-03T19:41:16.303 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -e sha1=8e24bcf462a1d94c2e12aa4e26419634883a1e88 -- bash -c 'ceph orch ps' 2023-06-03T19:41:16.335 INFO:journalctl@ceph.mgr.x.smithi164.stdout:Jun 03 19:41:15 smithi164 bash[55600]: debug 2023-06-03T19:41:15.844+0000 7f092311c200 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2023-06-03T19:41:17.335 INFO:journalctl@ceph.mgr.x.smithi164.stdout:Jun 03 19:41:16 smithi164 bash[55600]: debug 2023-06-03T19:41:16.892+0000 7f092311c200 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2023-06-03T19:41:17.336 INFO:journalctl@ceph.mgr.x.smithi164.stdout:Jun 03 19:41:17 smithi164 bash[55600]: debug 2023-06-03T19:41:17.008+0000 7f092311c200 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2023-06-03T19:41:17.336 INFO:journalctl@ceph.mgr.x.smithi164.stdout:Jun 03 19:41:17 smithi164 bash[55600]: debug 2023-06-03T19:41:17.124+0000 7f092311c200 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2023-06-03T19:41:17.487 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/mon.a/config 2023-06-03T19:41:17.521 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 19:41:17 smithi089 bash[71163]: debug 2023-06-03T19:41:17.026+0000 7fe4ce3f2200 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2023-06-03T19:41:17.521 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 19:41:17 smithi089 bash[71163]: debug 2023-06-03T19:41:17.142+0000 7fe4ce3f2200 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2023-06-03T19:41:17.521 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 19:41:17 smithi089 bash[71163]: debug 2023-06-03T19:41:17.258+0000 7fe4ce3f2200 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2023-06-03T19:41:17.843 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 19:41:17 smithi089 bash[71163]: debug 2023-06-03T19:41:17.730+0000 7fe4ce3f2200 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2023-06-03T19:41:18.085 INFO:journalctl@ceph.mgr.x.smithi164.stdout:Jun 03 19:41:17 smithi164 bash[55600]: debug 2023-06-03T19:41:17.616+0000 7f092311c200 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2023-06-03T19:41:18.086 INFO:journalctl@ceph.mgr.x.smithi164.stdout:Jun 03 19:41:17 smithi164 bash[55600]: debug 2023-06-03T19:41:17.848+0000 7f092311c200 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2023-06-03T19:41:18.254 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 19:41:17 smithi089 bash[71163]: debug 2023-06-03T19:41:17.954+0000 7fe4ce3f2200 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2023-06-03T19:41:18.585 INFO:journalctl@ceph.mgr.x.smithi164.stdout:Jun 03 19:41:18 smithi164 bash[55600]: debug 2023-06-03T19:41:18.224+0000 7f092311c200 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2023-06-03T19:41:18.717 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 19:41:18 smithi089 bash[71163]: debug 2023-06-03T19:41:18.362+0000 7fe4ce3f2200 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2023-06-03T19:41:19.031 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 19:41:18 smithi089 bash[71163]: debug 2023-06-03T19:41:18.886+0000 7fe4ce3f2200 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2023-06-03T19:41:19.085 INFO:journalctl@ceph.mgr.x.smithi164.stdout:Jun 03 19:41:18 smithi164 bash[55600]: debug 2023-06-03T19:41:18.712+0000 7f092311c200 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2023-06-03T19:41:19.586 INFO:journalctl@ceph.mgr.x.smithi164.stdout:Jun 03 19:41:19 smithi164 bash[55600]: debug 2023-06-03T19:41:19.268+0000 7f092311c200 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2023-06-03T19:41:19.586 INFO:journalctl@ceph.mgr.x.smithi164.stdout:Jun 03 19:41:19 smithi164 bash[55600]: debug 2023-06-03T19:41:19.376+0000 7f092311c200 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2023-06-03T19:41:19.701 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 19:41:19 smithi089 bash[71163]: debug 2023-06-03T19:41:19.434+0000 7fe4ce3f2200 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2023-06-03T19:41:19.701 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 19:41:19 smithi089 bash[71163]: debug 2023-06-03T19:41:19.542+0000 7fe4ce3f2200 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2023-06-03T19:41:19.997 INFO:journalctl@ceph.mgr.x.smithi164.stdout:Jun 03 19:41:19 smithi164 bash[55600]: debug 2023-06-03T19:41:19.724+0000 7f092311c200 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2023-06-03T19:41:19.997 INFO:journalctl@ceph.mgr.x.smithi164.stdout:Jun 03 19:41:19 smithi164 bash[55600]: debug 2023-06-03T19:41:19.840+0000 7f092311c200 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2023-06-03T19:41:20.018 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 19:41:19 smithi089 bash[71163]: debug 2023-06-03T19:41:19.894+0000 7fe4ce3f2200 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2023-06-03T19:41:20.139 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 19:41:20 smithi089 bash[71163]: debug 2023-06-03T19:41:20.006+0000 7fe4ce3f2200 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2023-06-03T19:41:20.280 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 19:41:20 smithi089 bash[71163]: debug 2023-06-03T19:41:20.150+0000 7fe4ce3f2200 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2023-06-03T19:41:20.336 INFO:journalctl@ceph.mgr.x.smithi164.stdout:Jun 03 19:41:19 smithi164 bash[55600]: debug 2023-06-03T19:41:19.984+0000 7f092311c200 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2023-06-03T19:41:20.336 INFO:journalctl@ceph.mgr.x.smithi164.stdout:Jun 03 19:41:20 smithi164 bash[55600]: debug 2023-06-03T19:41:20.104+0000 7f092311c200 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2023-06-03T19:41:20.336 INFO:journalctl@ceph.mgr.x.smithi164.stdout:Jun 03 19:41:20 smithi164 bash[55600]: debug 2023-06-03T19:41:20.224+0000 7f092311c200 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2023-06-03T19:41:20.531 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 19:41:20 smithi089 bash[71163]: debug 2023-06-03T19:41:20.270+0000 7fe4ce3f2200 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2023-06-03T19:41:20.531 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 19:41:20 smithi089 bash[71163]: debug 2023-06-03T19:41:20.390+0000 7fe4ce3f2200 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2023-06-03T19:41:20.835 INFO:journalctl@ceph.mgr.x.smithi164.stdout:Jun 03 19:41:20 smithi164 bash[55600]: debug 2023-06-03T19:41:20.412+0000 7f092311c200 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2023-06-03T19:41:20.836 INFO:journalctl@ceph.mgr.x.smithi164.stdout:Jun 03 19:41:20 smithi164 bash[55600]: debug 2023-06-03T19:41:20.540+0000 7f092311c200 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2023-06-03T19:41:21.031 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 19:41:20 smithi089 bash[71163]: debug 2023-06-03T19:41:20.578+0000 7fe4ce3f2200 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2023-06-03T19:41:21.031 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 19:41:20 smithi089 bash[71163]: debug 2023-06-03T19:41:20.710+0000 7fe4ce3f2200 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2023-06-03T19:41:21.867 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:41:21 smithi089 bash[33173]: cluster 2023-06-03T19:41:21.795157+0000 mon.a (mon.0) 11 : cluster [DBG] Standby manager daemon x restarted 2023-06-03T19:41:21.867 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:41:21 smithi089 bash[33173]: cluster 2023-06-03T19:41:21.795238+0000 mon.a (mon.0) 12 : cluster [DBG] Standby manager daemon x started 2023-06-03T19:41:21.867 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:41:21 smithi089 bash[33173]: audit 2023-06-03T19:41:21.799645+0000 mon.b (mon.1) 88 : audit [DBG] from='mgr.? 172.21.15.164:0/3808758476' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-06-03T19:41:21.867 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:41:21 smithi089 bash[33173]: audit 2023-06-03T19:41:21.801274+0000 mon.b (mon.1) 89 : audit [DBG] from='mgr.? 172.21.15.164:0/3808758476' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-06-03T19:41:21.868 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:41:21 smithi089 bash[33173]: audit 2023-06-03T19:41:21.806608+0000 mon.b (mon.1) 90 : audit [DBG] from='mgr.? 172.21.15.164:0/3808758476' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-06-03T19:41:21.868 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:41:21 smithi089 bash[33173]: audit 2023-06-03T19:41:21.807119+0000 mon.b (mon.1) 91 : audit [DBG] from='mgr.? 172.21.15.164:0/3808758476' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-06-03T19:41:22.086 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:41:21 smithi164 bash[26855]: cluster 2023-06-03T19:41:21.795157+0000 mon.a (mon.0) 11 : cluster [DBG] Standby manager daemon x restarted 2023-06-03T19:41:22.086 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:41:21 smithi164 bash[26855]: cluster 2023-06-03T19:41:21.795238+0000 mon.a (mon.0) 12 : cluster [DBG] Standby manager daemon x started 2023-06-03T19:41:22.086 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:41:21 smithi164 bash[26855]: audit 2023-06-03T19:41:21.799645+0000 mon.b (mon.1) 88 : audit [DBG] from='mgr.? 172.21.15.164:0/3808758476' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-06-03T19:41:22.086 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:41:21 smithi164 bash[26855]: audit 2023-06-03T19:41:21.801274+0000 mon.b (mon.1) 89 : audit [DBG] from='mgr.? 172.21.15.164:0/3808758476' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-06-03T19:41:22.086 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:41:21 smithi164 bash[26855]: audit 2023-06-03T19:41:21.806608+0000 mon.b (mon.1) 90 : audit [DBG] from='mgr.? 172.21.15.164:0/3808758476' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-06-03T19:41:22.087 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:41:21 smithi164 bash[26855]: audit 2023-06-03T19:41:21.807119+0000 mon.b (mon.1) 91 : audit [DBG] from='mgr.? 172.21.15.164:0/3808758476' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-06-03T19:41:22.087 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:41:21 smithi164 bash[26855]: debug 2023-06-03T19:41:21.936+0000 7f532d3d6700 -1 mon.b@1(peon).osd e72 update_from_paxos full map CRC mismatch, resetting to canonical 2023-06-03T19:41:22.087 INFO:journalctl@ceph.mgr.x.smithi164.stdout:Jun 03 19:41:21 smithi164 bash[55600]: debug 2023-06-03T19:41:21.764+0000 7f092311c200 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2023-06-03T19:41:22.087 INFO:journalctl@ceph.mgr.x.smithi164.stdout:Jun 03 19:41:21 smithi164 bash[55600]: [03/Jun/2023:19:41:21] ENGINE Bus STARTING 2023-06-03T19:41:22.087 INFO:journalctl@ceph.mgr.x.smithi164.stdout:Jun 03 19:41:21 smithi164 bash[55600]: CherryPy Checker: 2023-06-03T19:41:22.087 INFO:journalctl@ceph.mgr.x.smithi164.stdout:Jun 03 19:41:21 smithi164 bash[55600]: The Application mounted at '' has an empty config. 2023-06-03T19:41:22.088 INFO:journalctl@ceph.mgr.x.smithi164.stdout:Jun 03 19:41:21 smithi164 bash[55600]: [03/Jun/2023:19:41:21] ENGINE Serving on http://:::9283 2023-06-03T19:41:22.088 INFO:journalctl@ceph.mgr.x.smithi164.stdout:Jun 03 19:41:21 smithi164 bash[55600]: [03/Jun/2023:19:41:21] ENGINE Bus STARTED 2023-06-03T19:41:22.160 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:41:21 smithi089 bash[33173]: debug 2023-06-03T19:41:21.938+0000 7f68c98aa700 -1 mon.c@2(peon).osd e72 update_from_paxos full map CRC mismatch, resetting to canonical 2023-06-03T19:41:22.161 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 19:41:21 smithi089 bash[71163]: debug 2023-06-03T19:41:21.930+0000 7fe4ce3f2200 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2023-06-03T19:41:22.161 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 19:41:22 smithi089 bash[71163]: [03/Jun/2023:19:41:22] ENGINE Bus STARTING 2023-06-03T19:41:22.161 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 19:41:22 smithi089 bash[71163]: debug 2023-06-03T19:41:22.146+0000 7fe42918b700 -1 client.0 error registering admin socket command: (17) File exists 2023-06-03T19:41:22.161 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 19:41:22 smithi089 bash[71163]: debug 2023-06-03T19:41:22.146+0000 7fe42918b700 -1 client.0 error registering admin socket command: (17) File exists 2023-06-03T19:41:22.161 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 19:41:22 smithi089 bash[71163]: debug 2023-06-03T19:41:22.146+0000 7fe42918b700 -1 client.0 error registering admin socket command: (17) File exists 2023-06-03T19:41:22.162 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 19:41:22 smithi089 bash[71163]: debug 2023-06-03T19:41:22.146+0000 7fe42918b700 -1 client.0 error registering admin socket command: (17) File exists 2023-06-03T19:41:22.162 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 19:41:22 smithi089 bash[71163]: debug 2023-06-03T19:41:22.146+0000 7fe42918b700 -1 client.0 error registering admin socket command: (17) File exists 2023-06-03T19:41:22.162 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 19:41:22 smithi089 bash[71163]: debug 2023-06-03T19:41:22.146+0000 7fe42c992700 -1 client.0 error registering admin socket command: (17) File exists 2023-06-03T19:41:22.162 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 19:41:22 smithi089 bash[71163]: debug 2023-06-03T19:41:22.150+0000 7fe42c992700 -1 client.0 error registering admin socket command: (17) File exists 2023-06-03T19:41:22.162 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 19:41:22 smithi089 bash[71163]: debug 2023-06-03T19:41:22.150+0000 7fe42c992700 -1 client.0 error registering admin socket command: (17) File exists 2023-06-03T19:41:22.162 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 19:41:22 smithi089 bash[71163]: debug 2023-06-03T19:41:22.150+0000 7fe42c992700 -1 client.0 error registering admin socket command: (17) File exists 2023-06-03T19:41:22.162 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 19:41:22 smithi089 bash[71163]: debug 2023-06-03T19:41:22.150+0000 7fe42c992700 -1 client.0 error registering admin socket command: (17) File exists 2023-06-03T19:41:22.531 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 19:41:22 smithi089 bash[71163]: CherryPy Checker: 2023-06-03T19:41:22.531 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 19:41:22 smithi089 bash[71163]: The Application mounted at '' has an empty config. 2023-06-03T19:41:22.531 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 19:41:22 smithi089 bash[71163]: [03/Jun/2023:19:41:22] ENGINE Serving on http://:::9283 2023-06-03T19:41:22.532 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 19:41:22 smithi089 bash[71163]: [03/Jun/2023:19:41:22] ENGINE Bus STARTED 2023-06-03T19:41:22.978 INFO:teuthology.orchestra.run.smithi089.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2023-06-03T19:41:22.979 INFO:teuthology.orchestra.run.smithi089.stdout:alertmanager.a smithi089 *:9093,9094 running (3m) 73s ago 8m 14.5M - 0.23.0 ba2b418f427c 0fad3b643b2e 2023-06-03T19:41:22.979 INFO:teuthology.orchestra.run.smithi089.stdout:grafana.a smithi164 *:3000 running (111s) 33s ago 8m 40.2M - 8.3.5 dad864ee21e9 64925a21f38b 2023-06-03T19:41:22.979 INFO:teuthology.orchestra.run.smithi089.stdout:mds.foo.smithi089.giuxuj smithi089 running (7m) 73s ago 7m 15.1M - 16.2.13-257-gd8c5d349 b644dd95899d f0e9de24a3f8 2023-06-03T19:41:22.979 INFO:teuthology.orchestra.run.smithi089.stdout:mds.foo.smithi164.txgfgv smithi164 running (6m) 33s ago 6m 13.2M - 16.2.13-257-gd8c5d349 b644dd95899d 29da307aa3b4 2023-06-03T19:41:22.979 INFO:teuthology.orchestra.run.smithi089.stdout:mgr.x smithi164 *:8443,9283,8765 running (41s) 33s ago 16m 80.8M - 18.0.0-3532-g8e24bcf4 a57053f96fe3 3ada341a2c72 2023-06-03T19:41:22.979 INFO:teuthology.orchestra.run.smithi089.stdout:mgr.y smithi089 *:8443,9283,8765 running (102s) 73s ago 18m 455M - 18.0.0-3532-g8e24bcf4 a57053f96fe3 8307fbe12a7d 2023-06-03T19:41:22.980 INFO:teuthology.orchestra.run.smithi089.stdout:mon.a smithi089 running (18m) 73s ago 18m 66.6M 2048M 16.2.13-257-gd8c5d349 b644dd95899d 64c0c1397aaa 2023-06-03T19:41:22.980 INFO:teuthology.orchestra.run.smithi089.stdout:mon.b smithi164 running (16m) 33s ago 16m 65.0M 2048M 16.2.13-257-gd8c5d349 b644dd95899d 85a8c1795b60 2023-06-03T19:41:22.980 INFO:teuthology.orchestra.run.smithi089.stdout:mon.c smithi089 running (16m) 73s ago 16m 58.4M 2048M 16.2.13-257-gd8c5d349 b644dd95899d 268f878c5750 2023-06-03T19:41:22.980 INFO:teuthology.orchestra.run.smithi089.stdout:node-exporter.a smithi089 *:9100 running (3m) 73s ago 9m 10.2M - 1.3.1 1dbe0e931976 f41a3d45da20 2023-06-03T19:41:22.980 INFO:teuthology.orchestra.run.smithi089.stdout:node-exporter.b smithi164 *:9100 running (2m) 33s ago 9m 11.2M - 1.3.1 1dbe0e931976 1e69859880c6 2023-06-03T19:41:22.980 INFO:teuthology.orchestra.run.smithi089.stdout:osd.0 smithi089 running (15m) 73s ago 15m 359M 2526M 16.2.13-257-gd8c5d349 b644dd95899d cb8bdf1749e4 2023-06-03T19:41:22.980 INFO:teuthology.orchestra.run.smithi089.stdout:osd.1 smithi089 running (14m) 73s ago 14m 236M 2526M 16.2.13-257-gd8c5d349 b644dd95899d d9e8a03d4689 2023-06-03T19:41:22.981 INFO:teuthology.orchestra.run.smithi089.stdout:osd.2 smithi089 running (14m) 73s ago 14m 323M 2526M 16.2.13-257-gd8c5d349 b644dd95899d f7fc93b466ec 2023-06-03T19:41:22.981 INFO:teuthology.orchestra.run.smithi089.stdout:osd.3 smithi089 running (13m) 73s ago 13m 163M 2526M 16.2.13-257-gd8c5d349 b644dd95899d 7e98b61121c6 2023-06-03T19:41:22.981 INFO:teuthology.orchestra.run.smithi089.stdout:osd.4 smithi164 running (12m) 33s ago 12m 363M 2526M 16.2.13-257-gd8c5d349 b644dd95899d 5c901af1a184 2023-06-03T19:41:22.981 INFO:teuthology.orchestra.run.smithi089.stdout:osd.5 smithi164 running (11m) 33s ago 11m 399M 2526M 16.2.13-257-gd8c5d349 b644dd95899d 13ddc40f04c4 2023-06-03T19:41:22.981 INFO:teuthology.orchestra.run.smithi089.stdout:osd.6 smithi164 running (11m) 33s ago 11m 205M 2526M 16.2.13-257-gd8c5d349 b644dd95899d cab2cb9998d7 2023-06-03T19:41:22.981 INFO:teuthology.orchestra.run.smithi089.stdout:osd.7 smithi164 running (10m) 33s ago 10m 257M 2526M 16.2.13-257-gd8c5d349 b644dd95899d b5fd865771b6 2023-06-03T19:41:22.981 INFO:teuthology.orchestra.run.smithi089.stdout:prometheus.a smithi164 *:9095 running (50s) 33s ago 9m 49.2M - 2.33.4 514e6a882f6e 3ff744731c36 2023-06-03T19:41:23.058 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 19:41:22 smithi089 bash[71163]: debug 2023-06-03T19:41:22.946+0000 7fe48642e700 -1 mgr.server handle_report got status from non-daemon mon.a 2023-06-03T19:41:23.058 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:41:22 smithi089 bash[33173]: cluster 2023-06-03T19:41:21.852831+0000 mon.a (mon.0) 13 : cluster [DBG] mgrmap e34: y(active, since 80s), standbys: x 2023-06-03T19:41:23.059 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:41:22 smithi089 bash[33173]: cluster 2023-06-03T19:41:21.936042+0000 mon.a (mon.0) 14 : cluster [INF] Active manager daemon y restarted 2023-06-03T19:41:23.059 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:41:22 smithi089 bash[33173]: cluster 2023-06-03T19:41:21.936378+0000 mon.a (mon.0) 15 : cluster [INF] Activating manager daemon y 2023-06-03T19:41:23.059 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:41:22 smithi089 bash[33173]: cluster 2023-06-03T19:41:21.948103+0000 osd.2 (osd.2) 3 : cluster [WRN] failed to encode map e72 with expected crc 2023-06-03T19:41:23.059 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:41:22 smithi089 bash[33173]: cluster 2023-06-03T19:41:21.948822+0000 mon.a (mon.0) 16 : cluster [DBG] osdmap e72: 8 total, 8 up, 8 in 2023-06-03T19:41:23.059 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:41:22 smithi089 bash[33173]: cluster 2023-06-03T19:41:21.949531+0000 mon.a (mon.0) 17 : cluster [DBG] mgrmap e35: y(active, starting, since 0.0132434s), standbys: x 2023-06-03T19:41:23.059 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:41:22 smithi089 bash[33173]: cluster 2023-06-03T19:41:21.950519+0000 osd.3 (osd.3) 3 : cluster [WRN] failed to encode map e72 with expected crc 2023-06-03T19:41:23.059 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:41:22 smithi089 bash[33173]: audit 2023-06-03T19:41:21.957865+0000 mon.c (mon.2) 80 : audit [DBG] from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-06-03T19:41:23.060 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:41:22 smithi089 bash[33173]: audit 2023-06-03T19:41:21.958132+0000 mon.c (mon.2) 81 : audit [DBG] from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-06-03T19:41:23.060 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:41:22 smithi089 bash[33173]: audit 2023-06-03T19:41:21.958377+0000 mon.c (mon.2) 82 : audit [DBG] from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-06-03T19:41:23.060 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:41:22 smithi089 bash[33173]: audit 2023-06-03T19:41:21.958639+0000 mon.c (mon.2) 83 : audit [DBG] from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi089.giuxuj"}]: dispatch 2023-06-03T19:41:23.060 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:41:22 smithi089 bash[33173]: audit 2023-06-03T19:41:21.959008+0000 mon.c (mon.2) 84 : audit [DBG] from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi164.txgfgv"}]: dispatch 2023-06-03T19:41:23.060 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:41:22 smithi089 bash[33173]: audit 2023-06-03T19:41:21.960298+0000 mon.c (mon.2) 85 : audit [DBG] from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2023-06-03T19:41:23.060 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:41:22 smithi089 bash[33173]: audit 2023-06-03T19:41:21.960530+0000 mon.c (mon.2) 86 : audit [DBG] from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2023-06-03T19:41:23.061 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:41:22 smithi089 bash[33173]: audit 2023-06-03T19:41:21.960884+0000 mon.c (mon.2) 87 : audit [DBG] from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-06-03T19:41:23.061 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:41:22 smithi089 bash[33173]: audit 2023-06-03T19:41:21.961193+0000 mon.c (mon.2) 88 : audit [DBG] from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-06-03T19:41:23.061 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:41:22 smithi089 bash[33173]: audit 2023-06-03T19:41:21.961519+0000 mon.c (mon.2) 89 : audit [DBG] from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-06-03T19:41:23.061 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:41:22 smithi089 bash[33173]: audit 2023-06-03T19:41:21.961889+0000 mon.c (mon.2) 90 : audit [DBG] from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-06-03T19:41:23.061 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:41:22 smithi089 bash[33173]: audit 2023-06-03T19:41:21.962252+0000 mon.c (mon.2) 91 : audit [DBG] from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-06-03T19:41:23.061 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:41:22 smithi089 bash[33173]: audit 2023-06-03T19:41:21.962535+0000 mon.c (mon.2) 92 : audit [DBG] from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-06-03T19:41:23.061 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:41:22 smithi089 bash[33173]: audit 2023-06-03T19:41:21.962835+0000 mon.c (mon.2) 93 : audit [DBG] from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-06-03T19:41:23.062 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:41:22 smithi089 bash[33173]: audit 2023-06-03T19:41:21.963118+0000 mon.c (mon.2) 94 : audit [DBG] from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-06-03T19:41:23.062 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:41:22 smithi089 bash[33173]: audit 2023-06-03T19:41:21.963438+0000 mon.c (mon.2) 95 : audit [DBG] from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2023-06-03T19:41:23.062 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:41:22 smithi089 bash[33173]: audit 2023-06-03T19:41:21.963783+0000 mon.c (mon.2) 96 : audit [DBG] from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2023-06-03T19:41:23.062 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:41:22 smithi089 bash[33173]: audit 2023-06-03T19:41:21.964327+0000 mon.c (mon.2) 97 : audit [DBG] from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2023-06-03T19:41:23.062 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:41:22 smithi089 bash[33173]: cluster 2023-06-03T19:41:21.996798+0000 mon.a (mon.0) 18 : cluster [INF] Manager daemon y is now available 2023-06-03T19:41:23.062 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:41:22 smithi089 bash[33173]: audit 2023-06-03T19:41:22.028279+0000 mon.c (mon.2) 98 : audit [DBG] from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-03T19:41:23.062 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:41:22 smithi089 bash[33173]: audit 2023-06-03T19:41:22.032453+0000 mon.c (mon.2) 99 : audit [DBG] from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-03T19:41:23.063 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:41:22 smithi089 bash[33173]: audit 2023-06-03T19:41:22.063352+0000 mon.c (mon.2) 100 : audit [DBG] from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T19:41:23.063 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:41:22 smithi089 bash[33173]: audit 2023-06-03T19:41:22.074604+0000 mon.c (mon.2) 101 : audit [INF] from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2023-06-03T19:41:23.063 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:41:22 smithi089 bash[33173]: audit 2023-06-03T19:41:22.075049+0000 mon.a (mon.0) 19 : audit [INF] from='mgr.34104 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2023-06-03T19:41:23.063 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:41:22 smithi089 bash[33173]: audit 2023-06-03T19:41:22.108637+0000 mon.c (mon.2) 102 : audit [INF] from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2023-06-03T19:41:23.063 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:41:22 smithi089 bash[33173]: audit 2023-06-03T19:41:22.109400+0000 mon.a (mon.0) 20 : audit [INF] from='mgr.34104 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2023-06-03T19:41:23.063 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:41:22 smithi089 bash[33173]: cephadm 2023-06-03T19:41:22.493770+0000 mgr.y (mgr.34104) 1 : cephadm [INF] [03/Jun/2023:19:41:22] ENGINE Bus STARTING 2023-06-03T19:41:23.063 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:41:22 smithi089 bash[33173]: cephadm 2023-06-03T19:41:22.610117+0000 mgr.y (mgr.34104) 2 : cephadm [INF] [03/Jun/2023:19:41:22] ENGINE Serving on https://172.21.15.89:7150 2023-06-03T19:41:23.336 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:41:22 smithi164 bash[26855]: cluster 2023-06-03T19:41:21.852831+0000 mon.a (mon.0) 13 : cluster [DBG] mgrmap e34: y(active, since 80s), standbys: x 2023-06-03T19:41:23.336 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:41:22 smithi164 bash[26855]: cluster 2023-06-03T19:41:21.936042+0000 mon.a (mon.0) 14 : cluster [INF] Active manager daemon y restarted 2023-06-03T19:41:23.337 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:41:22 smithi164 bash[26855]: cluster 2023-06-03T19:41:21.936378+0000 mon.a (mon.0) 15 : cluster [INF] Activating manager daemon y 2023-06-03T19:41:23.337 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:41:22 smithi164 bash[26855]: cluster 2023-06-03T19:41:21.948103+0000 osd.2 (osd.2) 3 : cluster [WRN] failed to encode map e72 with expected crc 2023-06-03T19:41:23.337 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:41:22 smithi164 bash[26855]: cluster 2023-06-03T19:41:21.948822+0000 mon.a (mon.0) 16 : cluster [DBG] osdmap e72: 8 total, 8 up, 8 in 2023-06-03T19:41:23.337 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:41:22 smithi164 bash[26855]: cluster 2023-06-03T19:41:21.949531+0000 mon.a (mon.0) 17 : cluster [DBG] mgrmap e35: y(active, starting, since 0.0132434s), standbys: x 2023-06-03T19:41:23.337 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:41:22 smithi164 bash[26855]: cluster 2023-06-03T19:41:21.950519+0000 osd.3 (osd.3) 3 : cluster [WRN] failed to encode map e72 with expected crc 2023-06-03T19:41:23.337 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:41:22 smithi164 bash[26855]: audit 2023-06-03T19:41:21.957865+0000 mon.c (mon.2) 80 : audit [DBG] from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-06-03T19:41:23.337 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:41:22 smithi164 bash[26855]: audit 2023-06-03T19:41:21.958132+0000 mon.c (mon.2) 81 : audit [DBG] from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-06-03T19:41:23.338 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:41:22 smithi164 bash[26855]: audit 2023-06-03T19:41:21.958377+0000 mon.c (mon.2) 82 : audit [DBG] from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-06-03T19:41:23.338 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:41:22 smithi164 bash[26855]: audit 2023-06-03T19:41:21.958639+0000 mon.c (mon.2) 83 : audit [DBG] from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi089.giuxuj"}]: dispatch 2023-06-03T19:41:23.338 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:41:22 smithi164 bash[26855]: audit 2023-06-03T19:41:21.959008+0000 mon.c (mon.2) 84 : audit [DBG] from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi164.txgfgv"}]: dispatch 2023-06-03T19:41:23.338 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:41:22 smithi164 bash[26855]: audit 2023-06-03T19:41:21.960298+0000 mon.c (mon.2) 85 : audit [DBG] from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2023-06-03T19:41:23.338 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:41:22 smithi164 bash[26855]: audit 2023-06-03T19:41:21.960530+0000 mon.c (mon.2) 86 : audit [DBG] from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2023-06-03T19:41:23.338 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:41:22 smithi164 bash[26855]: audit 2023-06-03T19:41:21.960884+0000 mon.c (mon.2) 87 : audit [DBG] from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-06-03T19:41:23.338 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:41:22 smithi164 bash[26855]: audit 2023-06-03T19:41:21.961193+0000 mon.c (mon.2) 88 : audit [DBG] from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-06-03T19:41:23.339 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:41:22 smithi164 bash[26855]: audit 2023-06-03T19:41:21.961519+0000 mon.c (mon.2) 89 : audit [DBG] from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-06-03T19:41:23.339 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:41:22 smithi164 bash[26855]: audit 2023-06-03T19:41:21.961889+0000 mon.c (mon.2) 90 : audit [DBG] from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-06-03T19:41:23.339 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:41:22 smithi164 bash[26855]: audit 2023-06-03T19:41:21.962252+0000 mon.c (mon.2) 91 : audit [DBG] from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-06-03T19:41:23.339 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:41:22 smithi164 bash[26855]: audit 2023-06-03T19:41:21.962535+0000 mon.c (mon.2) 92 : audit [DBG] from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-06-03T19:41:23.339 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:41:22 smithi164 bash[26855]: audit 2023-06-03T19:41:21.962835+0000 mon.c (mon.2) 93 : audit [DBG] from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-06-03T19:41:23.339 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:41:22 smithi164 bash[26855]: audit 2023-06-03T19:41:21.963118+0000 mon.c (mon.2) 94 : audit [DBG] from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-06-03T19:41:23.339 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:41:22 smithi164 bash[26855]: audit 2023-06-03T19:41:21.963438+0000 mon.c (mon.2) 95 : audit [DBG] from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2023-06-03T19:41:23.339 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:41:22 smithi164 bash[26855]: audit 2023-06-03T19:41:21.963783+0000 mon.c (mon.2) 96 : audit [DBG] from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2023-06-03T19:41:23.340 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:41:22 smithi164 bash[26855]: audit 2023-06-03T19:41:21.964327+0000 mon.c (mon.2) 97 : audit [DBG] from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2023-06-03T19:41:23.340 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:41:22 smithi164 bash[26855]: cluster 2023-06-03T19:41:21.996798+0000 mon.a (mon.0) 18 : cluster [INF] Manager daemon y is now available 2023-06-03T19:41:23.340 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:41:22 smithi164 bash[26855]: audit 2023-06-03T19:41:22.028279+0000 mon.c (mon.2) 98 : audit [DBG] from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-03T19:41:23.340 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:41:22 smithi164 bash[26855]: audit 2023-06-03T19:41:22.032453+0000 mon.c (mon.2) 99 : audit [DBG] from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-03T19:41:23.340 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:41:22 smithi164 bash[26855]: audit 2023-06-03T19:41:22.063352+0000 mon.c (mon.2) 100 : audit [DBG] from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T19:41:23.340 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:41:22 smithi164 bash[26855]: audit 2023-06-03T19:41:22.074604+0000 mon.c (mon.2) 101 : audit [INF] from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2023-06-03T19:41:23.340 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:41:22 smithi164 bash[26855]: audit 2023-06-03T19:41:22.075049+0000 mon.a (mon.0) 19 : audit [INF] from='mgr.34104 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2023-06-03T19:41:23.341 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:41:22 smithi164 bash[26855]: audit 2023-06-03T19:41:22.108637+0000 mon.c (mon.2) 102 : audit [INF] from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2023-06-03T19:41:23.341 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:41:22 smithi164 bash[26855]: audit 2023-06-03T19:41:22.109400+0000 mon.a (mon.0) 20 : audit [INF] from='mgr.34104 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2023-06-03T19:41:23.341 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:41:22 smithi164 bash[26855]: cephadm 2023-06-03T19:41:22.493770+0000 mgr.y (mgr.34104) 1 : cephadm [INF] [03/Jun/2023:19:41:22] ENGINE Bus STARTING 2023-06-03T19:41:23.341 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:41:22 smithi164 bash[26855]: cephadm 2023-06-03T19:41:22.610117+0000 mgr.y (mgr.34104) 2 : cephadm [INF] [03/Jun/2023:19:41:22] ENGINE Serving on https://172.21.15.89:7150 2023-06-03T19:41:23.874 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -e sha1=8e24bcf462a1d94c2e12aa4e26419634883a1e88 -- bash -c 'ceph orch upgrade pause' 2023-06-03T19:41:23.968 INFO:teuthology.orchestra.run.smithi089.stderr:pool 'unique_pool_0' removed 2023-06-03T19:41:24.281 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:41:23 smithi089 bash[33173]: cluster 2023-06-03T19:41:21.955022+0000 osd.4 (osd.4) 3 : cluster [WRN] failed to encode map e72 with expected crc 2023-06-03T19:41:24.281 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:41:23 smithi089 bash[33173]: cephadm 2023-06-03T19:41:22.711842+0000 mgr.y (mgr.34104) 3 : cephadm [INF] [03/Jun/2023:19:41:22] ENGINE Serving on http://172.21.15.89:8765 2023-06-03T19:41:24.282 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:41:23 smithi089 bash[33173]: cephadm 2023-06-03T19:41:22.712187+0000 mgr.y (mgr.34104) 4 : cephadm [INF] [03/Jun/2023:19:41:22] ENGINE Bus STARTED 2023-06-03T19:41:24.282 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:41:23 smithi089 bash[33173]: cluster 2023-06-03T19:41:22.950718+0000 mon.a (mon.0) 21 : cluster [DBG] mgrmap e36: y(active, since 1.0144s), standbys: x 2023-06-03T19:41:24.282 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:41:23 smithi089 bash[33173]: audit 2023-06-03T19:41:22.951664+0000 mgr.y (mgr.34104) 5 : audit [DBG] from='client.34128 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-06-03T19:41:24.282 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:41:23 smithi089 bash[33173]: cluster 2023-06-03T19:41:22.964550+0000 mgr.y (mgr.34104) 6 : cluster [DBG] pgmap v3: 89 pgs: 89 active+clean; 582 KiB data, 677 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:41:24.282 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:41:23 smithi089 bash[33173]: audit 2023-06-03T19:41:23.212292+0000 mon.a (mon.0) 22 : audit [INF] from='client.? 172.21.15.89:0/2756304252' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_0", "pool2": "unique_pool_0", "yes_i_really_really_mean_it": true}]: dispatch 2023-06-03T19:41:24.282 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:41:23 smithi089 bash[33173]: debug 2023-06-03T19:41:23.950+0000 7f68c98aa700 -1 mon.c@2(peon).osd e73 update_from_paxos full map CRC mismatch, resetting to canonical 2023-06-03T19:41:24.336 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:41:23 smithi164 bash[26855]: cluster 2023-06-03T19:41:21.955022+0000 osd.4 (osd.4) 3 : cluster [WRN] failed to encode map e72 with expected crc 2023-06-03T19:41:24.336 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:41:23 smithi164 bash[26855]: cephadm 2023-06-03T19:41:22.711842+0000 mgr.y (mgr.34104) 3 : cephadm [INF] [03/Jun/2023:19:41:22] ENGINE Serving on http://172.21.15.89:8765 2023-06-03T19:41:24.336 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:41:23 smithi164 bash[26855]: cephadm 2023-06-03T19:41:22.712187+0000 mgr.y (mgr.34104) 4 : cephadm [INF] [03/Jun/2023:19:41:22] ENGINE Bus STARTED 2023-06-03T19:41:24.337 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:41:23 smithi164 bash[26855]: cluster 2023-06-03T19:41:22.950718+0000 mon.a (mon.0) 21 : cluster [DBG] mgrmap e36: y(active, since 1.0144s), standbys: x 2023-06-03T19:41:24.337 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:41:23 smithi164 bash[26855]: audit 2023-06-03T19:41:22.951664+0000 mgr.y (mgr.34104) 5 : audit [DBG] from='client.34128 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-06-03T19:41:24.337 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:41:23 smithi164 bash[26855]: cluster 2023-06-03T19:41:22.964550+0000 mgr.y (mgr.34104) 6 : cluster [DBG] pgmap v3: 89 pgs: 89 active+clean; 582 KiB data, 677 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:41:24.337 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:41:23 smithi164 bash[26855]: audit 2023-06-03T19:41:23.212292+0000 mon.a (mon.0) 22 : audit [INF] from='client.? 172.21.15.89:0/2756304252' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_0", "pool2": "unique_pool_0", "yes_i_really_really_mean_it": true}]: dispatch 2023-06-03T19:41:24.337 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:41:23 smithi164 bash[26855]: debug 2023-06-03T19:41:23.948+0000 7f532d3d6700 -1 mon.b@1(peon).osd e73 update_from_paxos full map CRC mismatch, resetting to canonical 2023-06-03T19:41:25.146 INFO:teuthology.task.full_sequential:In full_sequential, running task radosbench... 2023-06-03T19:41:25.147 INFO:tasks.radosbench:Beginning radosbench... 2023-06-03T19:41:25.147 INFO:tasks.cephadm.ceph_manager.ceph:creating pool_name unique_pool_1 2023-06-03T19:41:25.147 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph osd pool create unique_pool_1 16 2023-06-03T19:41:25.281 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:41:24 smithi089 bash[33173]: audit 2023-06-03T19:41:23.955969+0000 mon.a (mon.0) 23 : audit [INF] from='client.? 172.21.15.89:0/2756304252' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_0", "pool2": "unique_pool_0", "yes_i_really_really_mean_it": true}]': finished 2023-06-03T19:41:25.281 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:41:24 smithi089 bash[33173]: cluster 2023-06-03T19:41:23.956053+0000 mon.a (mon.0) 24 : cluster [DBG] osdmap e73: 8 total, 8 up, 8 in 2023-06-03T19:41:25.281 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:41:24 smithi089 bash[33173]: cluster 2023-06-03T19:41:23.957258+0000 osd.6 (osd.6) 3 : cluster [WRN] failed to encode map e73 with expected crc 2023-06-03T19:41:25.282 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:41:24 smithi089 bash[33173]: cluster 2023-06-03T19:41:23.959437+0000 osd.1 (osd.1) 3 : cluster [WRN] failed to encode map e73 with expected crc 2023-06-03T19:41:25.282 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:41:24 smithi089 bash[33173]: cluster 2023-06-03T19:41:23.961992+0000 mgr.y (mgr.34104) 7 : cluster [DBG] pgmap v5: 73 pgs: 73 active+clean; 582 KiB data, 677 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:41:25.335 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:41:24 smithi164 bash[26855]: audit 2023-06-03T19:41:23.955969+0000 mon.a (mon.0) 23 : audit [INF] from='client.? 172.21.15.89:0/2756304252' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_0", "pool2": "unique_pool_0", "yes_i_really_really_mean_it": true}]': finished 2023-06-03T19:41:25.336 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:41:24 smithi164 bash[26855]: cluster 2023-06-03T19:41:23.956053+0000 mon.a (mon.0) 24 : cluster [DBG] osdmap e73: 8 total, 8 up, 8 in 2023-06-03T19:41:25.336 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:41:24 smithi164 bash[26855]: cluster 2023-06-03T19:41:23.957258+0000 osd.6 (osd.6) 3 : cluster [WRN] failed to encode map e73 with expected crc 2023-06-03T19:41:25.336 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:41:24 smithi164 bash[26855]: cluster 2023-06-03T19:41:23.959437+0000 osd.1 (osd.1) 3 : cluster [WRN] failed to encode map e73 with expected crc 2023-06-03T19:41:25.336 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:41:24 smithi164 bash[26855]: cluster 2023-06-03T19:41:23.961992+0000 mgr.y (mgr.34104) 7 : cluster [DBG] pgmap v5: 73 pgs: 73 active+clean; 582 KiB data, 677 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:41:26.237 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:41:25 smithi089 bash[33173]: cluster 2023-06-03T19:41:24.960023+0000 mon.a (mon.0) 25 : cluster [DBG] mgrmap e37: y(active, since 3s), standbys: x 2023-06-03T19:41:26.335 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:41:25 smithi164 bash[26855]: cluster 2023-06-03T19:41:24.960023+0000 mon.a (mon.0) 25 : cluster [DBG] mgrmap e37: y(active, since 3s), standbys: x 2023-06-03T19:41:26.531 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 19:41:26 smithi089 bash[71163]: ::ffff:172.21.15.164 - - [03/Jun/2023:19:41:26] "GET /metrics HTTP/1.1" 200 33495 "" "Prometheus/2.33.4" 2023-06-03T19:41:27.281 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:41:26 smithi089 bash[33173]: cluster 2023-06-03T19:41:25.962617+0000 mgr.y (mgr.34104) 8 : cluster [DBG] pgmap v6: 73 pgs: 73 active+clean; 582 KiB data, 677 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:41:27.335 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:41:26 smithi164 bash[26855]: cluster 2023-06-03T19:41:25.962617+0000 mgr.y (mgr.34104) 8 : cluster [DBG] pgmap v6: 73 pgs: 73 active+clean; 582 KiB data, 677 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:41:27.977 INFO:teuthology.orchestra.run.smithi089.stdout:Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:00b1b6d689278b4f97ff790045c8c51d8ca419ac7ee2ebdf11008f2e810a5c78 2023-06-03T19:41:28.815 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -e sha1=8e24bcf462a1d94c2e12aa4e26419634883a1e88 -- bash -c 'sleep 60' 2023-06-03T19:41:28.977 INFO:journalctl@ceph.prometheus.a.smithi164.stdout:Jun 03 19:41:28 smithi164 bash[55188]: ts=2023-06-03T19:41:28.662Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.13-257-gd8c5d349 (d8c5d34975dce1c5eb0aa3a7979a4d9b9a99d1ec) pacific (stable)\", cluster_addr=\"172.21.15.89\", device_class=\"ssd\", hostname=\"smithi089\", instance=\"smithi164.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.89\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.13-257-gd8c5d349 (d8c5d34975dce1c5eb0aa3a7979a4d9b9a99d1ec) pacific (stable)\", cluster_addr=\"172.21.15.89\", device_class=\"ssd\", hostname=\"smithi089\", instance=\"smithi089:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.89\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-06-03T19:41:28.977 INFO:journalctl@ceph.prometheus.a.smithi164.stdout:Jun 03 19:41:28 smithi164 bash[55188]: ts=2023-06-03T19:41:28.664Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephPGImbalance\nexpr: abs(((ceph_osd_numpg > 0) - on(job) group_left() avg by(job) (ceph_osd_numpg\n > 0)) / on(job) group_left() avg by(job) (ceph_osd_numpg > 0)) * on(ceph_daemon)\n group_left(hostname) ceph_osd_metadata > 0.3\nfor: 5m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.5\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} deviates by\n more than 30% from average PG count.\n summary: PGs are not balanced across OSDs\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.13-257-gd8c5d349 (d8c5d34975dce1c5eb0aa3a7979a4d9b9a99d1ec) pacific (stable)\", cluster_addr=\"172.21.15.89\", device_class=\"ssd\", hostname=\"smithi089\", instance=\"smithi164.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.89\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.13-257-gd8c5d349 (d8c5d34975dce1c5eb0aa3a7979a4d9b9a99d1ec) pacific (stable)\", cluster_addr=\"172.21.15.89\", device_class=\"ssd\", hostname=\"smithi089\", instance=\"smithi089:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.89\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-06-03T19:41:29.281 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:41:28 smithi089 bash[33173]: audit 2023-06-03T19:41:27.962952+0000 mgr.y (mgr.34104) 9 : audit [DBG] from='client.34158 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2023-06-03T19:41:29.281 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:41:28 smithi089 bash[33173]: cluster 2023-06-03T19:41:27.963394+0000 mgr.y (mgr.34104) 10 : cluster [DBG] pgmap v7: 73 pgs: 73 active+clean; 582 KiB data, 688 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:41:29.281 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:41:28 smithi089 bash[33173]: cephadm 2023-06-03T19:41:27.963772+0000 mgr.y (mgr.34104) 11 : cephadm [INF] Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:00b1b6d689278b4f97ff790045c8c51d8ca419ac7ee2ebdf11008f2e810a5c78 2023-06-03T19:41:29.282 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:41:28 smithi089 bash[33173]: audit 2023-06-03T19:41:27.969625+0000 mon.a (mon.0) 26 : audit [INF] from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:41:29.282 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:41:28 smithi089 bash[33173]: audit 2023-06-03T19:41:28.813707+0000 mon.a (mon.0) 27 : audit [INF] from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:41:29.282 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:41:28 smithi089 bash[33173]: audit 2023-06-03T19:41:28.825396+0000 mon.a (mon.0) 28 : audit [INF] from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:41:29.282 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:41:28 smithi089 bash[33173]: audit 2023-06-03T19:41:28.867992+0000 mon.a (mon.0) 29 : audit [INF] from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:41:29.282 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:41:28 smithi089 bash[33173]: audit 2023-06-03T19:41:28.879470+0000 mon.a (mon.0) 30 : audit [INF] from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:41:29.336 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:41:28 smithi164 bash[26855]: audit 2023-06-03T19:41:27.962952+0000 mgr.y (mgr.34104) 9 : audit [DBG] from='client.34158 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2023-06-03T19:41:29.336 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:41:28 smithi164 bash[26855]: cluster 2023-06-03T19:41:27.963394+0000 mgr.y (mgr.34104) 10 : cluster [DBG] pgmap v7: 73 pgs: 73 active+clean; 582 KiB data, 688 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:41:29.336 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:41:28 smithi164 bash[26855]: cephadm 2023-06-03T19:41:27.963772+0000 mgr.y (mgr.34104) 11 : cephadm [INF] Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:00b1b6d689278b4f97ff790045c8c51d8ca419ac7ee2ebdf11008f2e810a5c78 2023-06-03T19:41:29.336 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:41:28 smithi164 bash[26855]: audit 2023-06-03T19:41:27.969625+0000 mon.a (mon.0) 26 : audit [INF] from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:41:29.336 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:41:28 smithi164 bash[26855]: audit 2023-06-03T19:41:28.813707+0000 mon.a (mon.0) 27 : audit [INF] from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:41:29.337 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:41:28 smithi164 bash[26855]: audit 2023-06-03T19:41:28.825396+0000 mon.a (mon.0) 28 : audit [INF] from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:41:29.337 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:41:28 smithi164 bash[26855]: audit 2023-06-03T19:41:28.867992+0000 mon.a (mon.0) 29 : audit [INF] from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:41:29.337 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:41:28 smithi164 bash[26855]: audit 2023-06-03T19:41:28.879470+0000 mon.a (mon.0) 30 : audit [INF] from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:41:30.316 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/mon.a/config 2023-06-03T19:41:31.031 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:41:30 smithi089 bash[33173]: audit 2023-06-03T19:41:29.734112+0000 mon.a (mon.0) 31 : audit [INF] from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:41:31.031 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:41:30 smithi089 bash[33173]: audit 2023-06-03T19:41:29.737742+0000 mon.a (mon.0) 32 : audit [INF] from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:41:31.031 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:41:30 smithi089 bash[33173]: audit 2023-06-03T19:41:29.747974+0000 mon.a (mon.0) 33 : audit [INF] from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:41:31.032 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:41:30 smithi089 bash[33173]: audit 2023-06-03T19:41:29.754240+0000 mon.a (mon.0) 34 : audit [INF] from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:41:31.032 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:41:30 smithi089 bash[33173]: cluster 2023-06-03T19:41:29.964234+0000 mgr.y (mgr.34104) 12 : cluster [DBG] pgmap v8: 73 pgs: 73 active+clean; 582 KiB data, 696 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:41:31.086 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:41:30 smithi164 bash[26855]: audit 2023-06-03T19:41:29.734112+0000 mon.a (mon.0) 31 : audit [INF] from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:41:31.086 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:41:30 smithi164 bash[26855]: audit 2023-06-03T19:41:29.737742+0000 mon.a (mon.0) 32 : audit [INF] from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:41:31.086 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:41:30 smithi164 bash[26855]: audit 2023-06-03T19:41:29.747974+0000 mon.a (mon.0) 33 : audit [INF] from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:41:31.086 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:41:30 smithi164 bash[26855]: audit 2023-06-03T19:41:29.754240+0000 mon.a (mon.0) 34 : audit [INF] from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:41:31.087 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:41:30 smithi164 bash[26855]: cluster 2023-06-03T19:41:29.964234+0000 mgr.y (mgr.34104) 12 : cluster [DBG] pgmap v8: 73 pgs: 73 active+clean; 582 KiB data, 696 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:41:33.281 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:41:33 smithi089 bash[33173]: cluster 2023-06-03T19:41:31.964897+0000 mgr.y (mgr.34104) 13 : cluster [DBG] pgmap v9: 73 pgs: 73 active+clean; 582 KiB data, 696 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:41:33.586 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:41:33 smithi164 bash[26855]: cluster 2023-06-03T19:41:31.964897+0000 mgr.y (mgr.34104) 13 : cluster [DBG] pgmap v9: 73 pgs: 73 active+clean; 582 KiB data, 696 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:41:35.531 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:41:35 smithi089 bash[33173]: cluster 2023-06-03T19:41:33.965645+0000 mgr.y (mgr.34104) 14 : cluster [DBG] pgmap v10: 73 pgs: 73 active+clean; 582 KiB data, 696 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:41:35.585 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:41:35 smithi164 bash[26855]: cluster 2023-06-03T19:41:33.965645+0000 mgr.y (mgr.34104) 14 : cluster [DBG] pgmap v10: 73 pgs: 73 active+clean; 582 KiB data, 696 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:41:36.133 INFO:teuthology.orchestra.run.smithi089.stderr:pool 'unique_pool_1' created 2023-06-03T19:41:36.531 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 19:41:36 smithi089 bash[71163]: ::ffff:172.21.15.164 - - [03/Jun/2023:19:41:36] "GET /metrics HTTP/1.1" 200 33495 "" "Prometheus/2.33.4" 2023-06-03T19:41:36.531 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:41:36 smithi089 bash[33173]: debug 2023-06-03T19:41:36.106+0000 7f68c98aa700 -1 mon.c@2(peon).osd e74 update_from_paxos full map CRC mismatch, resetting to canonical 2023-06-03T19:41:36.531 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:41:36 smithi089 bash[33173]: audit 2023-06-03T19:41:35.650512+0000 mon.a (mon.0) 35 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_1", "pg_num": 16}]: dispatch 2023-06-03T19:41:36.586 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:41:36 smithi164 bash[26855]: debug 2023-06-03T19:41:36.104+0000 7f532d3d6700 -1 mon.b@1(peon).osd e74 update_from_paxos full map CRC mismatch, resetting to canonical 2023-06-03T19:41:36.586 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:41:36 smithi164 bash[26855]: audit 2023-06-03T19:41:35.650512+0000 mon.a (mon.0) 35 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_1", "pg_num": 16}]: dispatch 2023-06-03T19:41:36.969 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph osd pool application enable unique_pool_1 rados --yes-i-really-mean-it || true 2023-06-03T19:41:37.281 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:41:37 smithi089 bash[33173]: debug 2023-06-03T19:41:37.106+0000 7f68c98aa700 -1 mon.c@2(peon).osd e75 update_from_paxos full map CRC mismatch, resetting to canonical 2023-06-03T19:41:37.281 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:41:37 smithi089 bash[33173]: cluster 2023-06-03T19:41:35.966188+0000 mgr.y (mgr.34104) 15 : cluster [DBG] pgmap v11: 73 pgs: 73 active+clean; 582 KiB data, 696 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:41:37.282 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:41:37 smithi089 bash[33173]: audit 2023-06-03T19:41:36.110925+0000 mon.a (mon.0) 36 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_1", "pg_num": 16}]': finished 2023-06-03T19:41:37.282 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:41:37 smithi089 bash[33173]: cluster 2023-06-03T19:41:36.110995+0000 mon.a (mon.0) 37 : cluster [DBG] osdmap e74: 8 total, 8 up, 8 in 2023-06-03T19:41:37.282 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:41:37 smithi089 bash[33173]: cluster 2023-06-03T19:41:36.113210+0000 osd.4 (osd.4) 4 : cluster [WRN] failed to encode map e74 with expected crc 2023-06-03T19:41:37.282 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:41:37 smithi089 bash[33173]: cluster 2023-06-03T19:41:36.113585+0000 osd.0 (osd.0) 3 : cluster [WRN] failed to encode map e74 with expected crc 2023-06-03T19:41:37.282 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:41:37 smithi089 bash[33173]: audit 2023-06-03T19:41:37.061241+0000 mon.c (mon.2) 103 : audit [DBG] from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T19:41:37.585 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:41:37 smithi164 bash[26855]: debug 2023-06-03T19:41:37.108+0000 7f532d3d6700 -1 mon.b@1(peon).osd e75 update_from_paxos full map CRC mismatch, resetting to canonical 2023-06-03T19:41:37.586 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:41:37 smithi164 bash[26855]: cluster 2023-06-03T19:41:35.966188+0000 mgr.y (mgr.34104) 15 : cluster [DBG] pgmap v11: 73 pgs: 73 active+clean; 582 KiB data, 696 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:41:37.586 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:41:37 smithi164 bash[26855]: audit 2023-06-03T19:41:36.110925+0000 mon.a (mon.0) 36 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_1", "pg_num": 16}]': finished 2023-06-03T19:41:37.586 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:41:37 smithi164 bash[26855]: cluster 2023-06-03T19:41:36.110995+0000 mon.a (mon.0) 37 : cluster [DBG] osdmap e74: 8 total, 8 up, 8 in 2023-06-03T19:41:37.586 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:41:37 smithi164 bash[26855]: cluster 2023-06-03T19:41:36.113210+0000 osd.4 (osd.4) 4 : cluster [WRN] failed to encode map e74 with expected crc 2023-06-03T19:41:37.586 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:41:37 smithi164 bash[26855]: cluster 2023-06-03T19:41:36.113585+0000 osd.0 (osd.0) 3 : cluster [WRN] failed to encode map e74 with expected crc 2023-06-03T19:41:37.586 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:41:37 smithi164 bash[26855]: audit 2023-06-03T19:41:37.061241+0000 mon.c (mon.2) 103 : audit [DBG] from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T19:41:38.531 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:41:38 smithi089 bash[33173]: cluster 2023-06-03T19:41:37.114223+0000 mon.a (mon.0) 38 : cluster [DBG] osdmap e75: 8 total, 8 up, 8 in 2023-06-03T19:41:38.531 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:41:38 smithi089 bash[33173]: cluster 2023-06-03T19:41:37.118984+0000 osd.3 (osd.3) 4 : cluster [WRN] failed to encode map e75 with expected crc 2023-06-03T19:41:38.532 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:41:38 smithi089 bash[33173]: cluster 2023-06-03T19:41:37.119037+0000 osd.1 (osd.1) 4 : cluster [WRN] failed to encode map e75 with expected crc 2023-06-03T19:41:38.532 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:41:38 smithi089 bash[33173]: cluster 2023-06-03T19:41:37.120562+0000 osd.7 (osd.7) 3 : cluster [WRN] failed to encode map e75 with expected crc 2023-06-03T19:41:38.532 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:41:38 smithi089 bash[33173]: cluster 2023-06-03T19:41:37.120582+0000 osd.4 (osd.4) 5 : cluster [WRN] failed to encode map e75 with expected crc 2023-06-03T19:41:38.532 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:41:38 smithi089 bash[33173]: cluster 2023-06-03T19:41:37.120624+0000 osd.5 (osd.5) 3 : cluster [WRN] failed to encode map e75 with expected crc 2023-06-03T19:41:38.532 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:41:38 smithi089 bash[33173]: cluster 2023-06-03T19:41:37.121829+0000 osd.3 (osd.3) 5 : cluster [WRN] failed to encode map e75 with expected crc 2023-06-03T19:41:38.532 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:41:38 smithi089 bash[33173]: cluster 2023-06-03T19:41:37.122321+0000 osd.2 (osd.2) 4 : cluster [WRN] failed to encode map e75 with expected crc 2023-06-03T19:41:38.532 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:41:38 smithi089 bash[33173]: cluster 2023-06-03T19:41:37.122331+0000 osd.1 (osd.1) 5 : cluster [WRN] failed to encode map e75 with expected crc 2023-06-03T19:41:38.533 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:41:38 smithi089 bash[33173]: cluster 2023-06-03T19:41:37.124136+0000 osd.0 (osd.0) 4 : cluster [WRN] failed to encode map e75 with expected crc 2023-06-03T19:41:38.533 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:41:38 smithi089 bash[33173]: debug 2023-06-03T19:41:38.130+0000 7f68c98aa700 -1 mon.c@2(peon).osd e76 update_from_paxos full map CRC mismatch, resetting to canonical 2023-06-03T19:41:38.569 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:41:38 smithi164 bash[26855]: cluster 2023-06-03T19:41:37.114223+0000 mon.a (mon.0) 38 : cluster [DBG] osdmap e75: 8 total, 8 up, 8 in 2023-06-03T19:41:38.569 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:41:38 smithi164 bash[26855]: cluster 2023-06-03T19:41:37.118984+0000 osd.3 (osd.3) 4 : cluster [WRN] failed to encode map e75 with expected crc 2023-06-03T19:41:38.570 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:41:38 smithi164 bash[26855]: cluster 2023-06-03T19:41:37.119037+0000 osd.1 (osd.1) 4 : cluster [WRN] failed to encode map e75 with expected crc 2023-06-03T19:41:38.570 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:41:38 smithi164 bash[26855]: cluster 2023-06-03T19:41:37.120562+0000 osd.7 (osd.7) 3 : cluster [WRN] failed to encode map e75 with expected crc 2023-06-03T19:41:38.570 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:41:38 smithi164 bash[26855]: cluster 2023-06-03T19:41:37.120582+0000 osd.4 (osd.4) 5 : cluster [WRN] failed to encode map e75 with expected crc 2023-06-03T19:41:38.570 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:41:38 smithi164 bash[26855]: cluster 2023-06-03T19:41:37.120624+0000 osd.5 (osd.5) 3 : cluster [WRN] failed to encode map e75 with expected crc 2023-06-03T19:41:38.570 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:41:38 smithi164 bash[26855]: cluster 2023-06-03T19:41:37.121829+0000 osd.3 (osd.3) 5 : cluster [WRN] failed to encode map e75 with expected crc 2023-06-03T19:41:38.570 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:41:38 smithi164 bash[26855]: cluster 2023-06-03T19:41:37.122321+0000 osd.2 (osd.2) 4 : cluster [WRN] failed to encode map e75 with expected crc 2023-06-03T19:41:38.570 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:41:38 smithi164 bash[26855]: cluster 2023-06-03T19:41:37.122331+0000 osd.1 (osd.1) 5 : cluster [WRN] failed to encode map e75 with expected crc 2023-06-03T19:41:38.571 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:41:38 smithi164 bash[26855]: cluster 2023-06-03T19:41:37.124136+0000 osd.0 (osd.0) 4 : cluster [WRN] failed to encode map e75 with expected crc 2023-06-03T19:41:38.571 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:41:38 smithi164 bash[26855]: debug 2023-06-03T19:41:38.128+0000 7f532d3d6700 -1 mon.b@1(peon).osd e76 update_from_paxos full map CRC mismatch, resetting to canonical 2023-06-03T19:41:38.835 INFO:journalctl@ceph.prometheus.a.smithi164.stdout:Jun 03 19:41:38 smithi164 bash[55188]: ts=2023-06-03T19:41:38.661Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.13-257-gd8c5d349 (d8c5d34975dce1c5eb0aa3a7979a4d9b9a99d1ec) pacific (stable)\", cluster_addr=\"172.21.15.89\", device_class=\"ssd\", hostname=\"smithi089\", instance=\"smithi164.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.89\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.13-257-gd8c5d349 (d8c5d34975dce1c5eb0aa3a7979a4d9b9a99d1ec) pacific (stable)\", cluster_addr=\"172.21.15.89\", device_class=\"ssd\", hostname=\"smithi089\", instance=\"smithi089:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.89\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-06-03T19:41:38.836 INFO:journalctl@ceph.prometheus.a.smithi164.stdout:Jun 03 19:41:38 smithi164 bash[55188]: ts=2023-06-03T19:41:38.663Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephPGImbalance\nexpr: abs(((ceph_osd_numpg > 0) - on(job) group_left() avg by(job) (ceph_osd_numpg\n > 0)) / on(job) group_left() avg by(job) (ceph_osd_numpg > 0)) * on(ceph_daemon)\n group_left(hostname) ceph_osd_metadata > 0.3\nfor: 5m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.5\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} deviates by\n more than 30% from average PG count.\n summary: PGs are not balanced across OSDs\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.13-257-gd8c5d349 (d8c5d34975dce1c5eb0aa3a7979a4d9b9a99d1ec) pacific (stable)\", cluster_addr=\"172.21.15.89\", device_class=\"ssd\", hostname=\"smithi089\", instance=\"smithi164.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.89\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.13-257-gd8c5d349 (d8c5d34975dce1c5eb0aa3a7979a4d9b9a99d1ec) pacific (stable)\", cluster_addr=\"172.21.15.89\", device_class=\"ssd\", hostname=\"smithi089\", instance=\"smithi089:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.89\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-06-03T19:41:39.531 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:41:39 smithi089 bash[33173]: cluster 2023-06-03T19:41:37.967347+0000 mgr.y (mgr.34104) 16 : cluster [DBG] pgmap v14: 89 pgs: 89 active+clean; 582 KiB data, 696 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:41:39.531 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:41:39 smithi089 bash[33173]: cluster 2023-06-03T19:41:38.136378+0000 mon.a (mon.0) 39 : cluster [DBG] osdmap e76: 8 total, 8 up, 8 in 2023-06-03T19:41:39.532 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:41:39 smithi089 bash[33173]: cluster 2023-06-03T19:41:38.139122+0000 osd.3 (osd.3) 6 : cluster [WRN] failed to encode map e76 with expected crc 2023-06-03T19:41:39.532 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:41:39 smithi089 bash[33173]: cluster 2023-06-03T19:41:38.140694+0000 osd.2 (osd.2) 5 : cluster [WRN] failed to encode map e76 with expected crc 2023-06-03T19:41:39.532 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:41:39 smithi089 bash[33173]: cluster 2023-06-03T19:41:38.620416+0000 osd.5 (osd.5) 4 : cluster [WRN] failed to encode map e76 with expected crc 2023-06-03T19:41:39.586 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:41:39 smithi164 bash[26855]: cluster 2023-06-03T19:41:37.967347+0000 mgr.y (mgr.34104) 16 : cluster [DBG] pgmap v14: 89 pgs: 89 active+clean; 582 KiB data, 696 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:41:39.586 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:41:39 smithi164 bash[26855]: cluster 2023-06-03T19:41:38.136378+0000 mon.a (mon.0) 39 : cluster [DBG] osdmap e76: 8 total, 8 up, 8 in 2023-06-03T19:41:39.586 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:41:39 smithi164 bash[26855]: cluster 2023-06-03T19:41:38.139122+0000 osd.3 (osd.3) 6 : cluster [WRN] failed to encode map e76 with expected crc 2023-06-03T19:41:39.586 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:41:39 smithi164 bash[26855]: cluster 2023-06-03T19:41:38.140694+0000 osd.2 (osd.2) 5 : cluster [WRN] failed to encode map e76 with expected crc 2023-06-03T19:41:39.586 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:41:39 smithi164 bash[26855]: cluster 2023-06-03T19:41:38.620416+0000 osd.5 (osd.5) 4 : cluster [WRN] failed to encode map e76 with expected crc 2023-06-03T19:41:40.781 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:41:40 smithi089 bash[33173]: cluster 2023-06-03T19:41:38.489380+0000 osd.0 (osd.0) 5 : cluster [WRN] failed to encode map e76 with expected crc 2023-06-03T19:41:40.782 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:41:40 smithi089 bash[33173]: audit 2023-06-03T19:41:39.399943+0000 mon.a (mon.0) 40 : audit [INF] from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:41:40.782 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:41:40 smithi089 bash[33173]: audit 2023-06-03T19:41:39.415386+0000 mon.a (mon.0) 41 : audit [INF] from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:41:40.782 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:41:40 smithi089 bash[33173]: audit 2023-06-03T19:41:39.419367+0000 mon.c (mon.2) 104 : audit [DBG] from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-03T19:41:40.782 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:41:40 smithi089 bash[33173]: audit 2023-06-03T19:41:39.420764+0000 mon.c (mon.2) 105 : audit [INF] from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-03T19:41:40.782 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:41:40 smithi089 bash[33173]: cephadm 2023-06-03T19:41:39.422499+0000 mgr.y (mgr.34104) 17 : cephadm [INF] Updating smithi089:/etc/ceph/ceph.conf 2023-06-03T19:41:40.783 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:41:40 smithi089 bash[33173]: cephadm 2023-06-03T19:41:39.422890+0000 mgr.y (mgr.34104) 18 : cephadm [INF] Updating smithi164:/etc/ceph/ceph.conf 2023-06-03T19:41:40.783 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:41:40 smithi089 bash[33173]: cephadm 2023-06-03T19:41:39.587414+0000 mgr.y (mgr.34104) 19 : cephadm [INF] Updating smithi089:/var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/config/ceph.conf 2023-06-03T19:41:40.783 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:41:40 smithi089 bash[33173]: cephadm 2023-06-03T19:41:39.588588+0000 mgr.y (mgr.34104) 20 : cephadm [INF] Updating smithi164:/var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/config/ceph.conf 2023-06-03T19:41:40.783 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:41:40 smithi089 bash[33173]: audit 2023-06-03T19:41:40.092691+0000 mon.a (mon.0) 42 : audit [INF] from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:41:40.783 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:41:40 smithi089 bash[33173]: audit 2023-06-03T19:41:40.100444+0000 mon.a (mon.0) 43 : audit [INF] from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:41:40.783 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:41:40 smithi089 bash[33173]: audit 2023-06-03T19:41:40.107634+0000 mon.a (mon.0) 44 : audit [INF] from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:41:40.783 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:41:40 smithi089 bash[33173]: audit 2023-06-03T19:41:40.114500+0000 mon.a (mon.0) 45 : audit [INF] from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:41:40.783 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:41:40 smithi089 bash[33173]: audit 2023-06-03T19:41:40.128461+0000 mon.a (mon.0) 46 : audit [INF] from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:41:40.784 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:41:40 smithi089 bash[33173]: audit 2023-06-03T19:41:40.208816+0000 mon.c (mon.2) 106 : audit [DBG] from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-03T19:41:40.784 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:41:40 smithi089 bash[33173]: audit 2023-06-03T19:41:40.211353+0000 mon.c (mon.2) 107 : audit [DBG] from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-03T19:41:40.784 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:41:40 smithi089 bash[33173]: audit 2023-06-03T19:41:40.212505+0000 mon.c (mon.2) 108 : audit [INF] from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-03T19:41:40.784 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:41:40 smithi089 bash[33173]: audit 2023-06-03T19:41:40.220506+0000 mon.a (mon.0) 47 : audit [INF] from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:41:40.836 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:41:40 smithi164 bash[26855]: cluster 2023-06-03T19:41:38.489380+0000 osd.0 (osd.0) 5 : cluster [WRN] failed to encode map e76 with expected crc 2023-06-03T19:41:40.836 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:41:40 smithi164 bash[26855]: audit 2023-06-03T19:41:39.399943+0000 mon.a (mon.0) 40 : audit [INF] from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:41:40.836 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:41:40 smithi164 bash[26855]: audit 2023-06-03T19:41:39.415386+0000 mon.a (mon.0) 41 : audit [INF] from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:41:40.836 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:41:40 smithi164 bash[26855]: audit 2023-06-03T19:41:39.419367+0000 mon.c (mon.2) 104 : audit [DBG] from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-03T19:41:40.837 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:41:40 smithi164 bash[26855]: audit 2023-06-03T19:41:39.420764+0000 mon.c (mon.2) 105 : audit [INF] from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-03T19:41:40.837 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:41:40 smithi164 bash[26855]: cephadm 2023-06-03T19:41:39.422499+0000 mgr.y (mgr.34104) 17 : cephadm [INF] Updating smithi089:/etc/ceph/ceph.conf 2023-06-03T19:41:40.837 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:41:40 smithi164 bash[26855]: cephadm 2023-06-03T19:41:39.422890+0000 mgr.y (mgr.34104) 18 : cephadm [INF] Updating smithi164:/etc/ceph/ceph.conf 2023-06-03T19:41:40.837 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:41:40 smithi164 bash[26855]: cephadm 2023-06-03T19:41:39.587414+0000 mgr.y (mgr.34104) 19 : cephadm [INF] Updating smithi089:/var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/config/ceph.conf 2023-06-03T19:41:40.837 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:41:40 smithi164 bash[26855]: cephadm 2023-06-03T19:41:39.588588+0000 mgr.y (mgr.34104) 20 : cephadm [INF] Updating smithi164:/var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/config/ceph.conf 2023-06-03T19:41:40.837 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:41:40 smithi164 bash[26855]: audit 2023-06-03T19:41:40.092691+0000 mon.a (mon.0) 42 : audit [INF] from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:41:40.837 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:41:40 smithi164 bash[26855]: audit 2023-06-03T19:41:40.100444+0000 mon.a (mon.0) 43 : audit [INF] from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:41:40.838 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:41:40 smithi164 bash[26855]: audit 2023-06-03T19:41:40.107634+0000 mon.a (mon.0) 44 : audit [INF] from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:41:40.838 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:41:40 smithi164 bash[26855]: audit 2023-06-03T19:41:40.114500+0000 mon.a (mon.0) 45 : audit [INF] from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:41:40.838 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:41:40 smithi164 bash[26855]: audit 2023-06-03T19:41:40.128461+0000 mon.a (mon.0) 46 : audit [INF] from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:41:40.838 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:41:40 smithi164 bash[26855]: audit 2023-06-03T19:41:40.208816+0000 mon.c (mon.2) 106 : audit [DBG] from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-03T19:41:40.838 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:41:40 smithi164 bash[26855]: audit 2023-06-03T19:41:40.211353+0000 mon.c (mon.2) 107 : audit [DBG] from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-03T19:41:40.838 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:41:40 smithi164 bash[26855]: audit 2023-06-03T19:41:40.212505+0000 mon.c (mon.2) 108 : audit [INF] from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-03T19:41:40.838 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:41:40 smithi164 bash[26855]: audit 2023-06-03T19:41:40.220506+0000 mon.a (mon.0) 47 : audit [INF] from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:41:41.781 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:41:41 smithi089 bash[33173]: cephadm 2023-06-03T19:41:39.752932+0000 mgr.y (mgr.34104) 21 : cephadm [INF] Updating smithi089:/etc/ceph/ceph.client.admin.keyring 2023-06-03T19:41:41.781 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:41:41 smithi089 bash[33173]: cephadm 2023-06-03T19:41:39.754386+0000 mgr.y (mgr.34104) 22 : cephadm [INF] Updating smithi164:/etc/ceph/ceph.client.admin.keyring 2023-06-03T19:41:41.782 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:41:41 smithi089 bash[33173]: cephadm 2023-06-03T19:41:39.918698+0000 mgr.y (mgr.34104) 23 : cephadm [INF] Updating smithi089:/var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/config/ceph.client.admin.keyring 2023-06-03T19:41:41.782 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:41:41 smithi089 bash[33173]: cephadm 2023-06-03T19:41:39.919882+0000 mgr.y (mgr.34104) 24 : cephadm [INF] Updating smithi164:/var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/config/ceph.client.admin.keyring 2023-06-03T19:41:41.782 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:41:41 smithi089 bash[33173]: cluster 2023-06-03T19:41:39.967909+0000 mgr.y (mgr.34104) 25 : cluster [DBG] pgmap v16: 89 pgs: 89 active+clean; 582 KiB data, 696 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:41:41.793 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/mon.a/config 2023-06-03T19:41:41.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:41:41 smithi164 bash[26855]: cephadm 2023-06-03T19:41:39.752932+0000 mgr.y (mgr.34104) 21 : cephadm [INF] Updating smithi089:/etc/ceph/ceph.client.admin.keyring 2023-06-03T19:41:41.836 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:41:41 smithi164 bash[26855]: cephadm 2023-06-03T19:41:39.754386+0000 mgr.y (mgr.34104) 22 : cephadm [INF] Updating smithi164:/etc/ceph/ceph.client.admin.keyring 2023-06-03T19:41:41.836 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:41:41 smithi164 bash[26855]: cephadm 2023-06-03T19:41:39.918698+0000 mgr.y (mgr.34104) 23 : cephadm [INF] Updating smithi089:/var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/config/ceph.client.admin.keyring 2023-06-03T19:41:41.836 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:41:41 smithi164 bash[26855]: cephadm 2023-06-03T19:41:39.919882+0000 mgr.y (mgr.34104) 24 : cephadm [INF] Updating smithi164:/var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/config/ceph.client.admin.keyring 2023-06-03T19:41:41.836 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:41:41 smithi164 bash[26855]: cluster 2023-06-03T19:41:39.967909+0000 mgr.y (mgr.34104) 25 : cluster [DBG] pgmap v16: 89 pgs: 89 active+clean; 582 KiB data, 696 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:41:43.781 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:41:43 smithi089 bash[33173]: cluster 2023-06-03T19:41:41.968472+0000 mgr.y (mgr.34104) 26 : cluster [DBG] pgmap v17: 89 pgs: 89 active+clean; 582 KiB data, 696 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:41:43.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:41:43 smithi164 bash[26855]: cluster 2023-06-03T19:41:41.968472+0000 mgr.y (mgr.34104) 26 : cluster [DBG] pgmap v17: 89 pgs: 89 active+clean; 582 KiB data, 696 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:41:45.781 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:41:45 smithi089 bash[33173]: cluster 2023-06-03T19:41:43.969170+0000 mgr.y (mgr.34104) 27 : cluster [DBG] pgmap v18: 89 pgs: 89 active+clean; 582 KiB data, 696 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:41:45.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:41:45 smithi164 bash[26855]: cluster 2023-06-03T19:41:43.969170+0000 mgr.y (mgr.34104) 27 : cluster [DBG] pgmap v18: 89 pgs: 89 active+clean; 582 KiB data, 696 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:41:46.456 INFO:teuthology.orchestra.run.smithi089.stderr:enabled application 'rados' on pool 'unique_pool_1' 2023-06-03T19:41:46.510 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 19:41:46 smithi089 bash[71163]: ::ffff:172.21.15.164 - - [03/Jun/2023:19:41:46] "GET /metrics HTTP/1.1" 200 189898 "" "Prometheus/2.33.4" 2023-06-03T19:41:46.511 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:41:46 smithi089 bash[33173]: audit 2023-06-03T19:41:45.457549+0000 mon.a (mon.0) 48 : audit [INF] from='client.? 172.21.15.89:0/111199136' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_1", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-06-03T19:41:46.511 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:41:46 smithi089 bash[33173]: debug 2023-06-03T19:41:46.433+0000 7f68c98aa700 -1 mon.c@2(peon).osd e77 update_from_paxos full map CRC mismatch, resetting to canonical 2023-06-03T19:41:46.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:41:46 smithi164 bash[26855]: audit 2023-06-03T19:41:45.457549+0000 mon.a (mon.0) 48 : audit [INF] from='client.? 172.21.15.89:0/111199136' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_1", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-06-03T19:41:46.836 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:41:46 smithi164 bash[26855]: debug 2023-06-03T19:41:46.435+0000 7f532d3d6700 -1 mon.b@1(peon).osd e77 update_from_paxos full map CRC mismatch, resetting to canonical 2023-06-03T19:41:47.781 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:41:47 smithi089 bash[33173]: cluster 2023-06-03T19:41:45.969764+0000 mgr.y (mgr.34104) 28 : cluster [DBG] pgmap v19: 89 pgs: 89 active+clean; 582 KiB data, 696 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:41:47.781 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:41:47 smithi089 bash[33173]: audit 2023-06-03T19:41:46.442832+0000 mon.a (mon.0) 49 : audit [INF] from='client.? 172.21.15.89:0/111199136' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_1", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-06-03T19:41:47.782 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:41:47 smithi089 bash[33173]: cluster 2023-06-03T19:41:46.442914+0000 mon.a (mon.0) 50 : cluster [DBG] osdmap e77: 8 total, 8 up, 8 in 2023-06-03T19:41:47.782 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:41:47 smithi089 bash[33173]: cluster 2023-06-03T19:41:46.443841+0000 osd.5 (osd.5) 5 : cluster [WRN] failed to encode map e77 with expected crc 2023-06-03T19:41:47.782 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:41:47 smithi089 bash[33173]: cluster 2023-06-03T19:41:46.446238+0000 osd.0 (osd.0) 6 : cluster [WRN] failed to encode map e77 with expected crc 2023-06-03T19:41:47.836 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:41:47 smithi164 bash[26855]: cluster 2023-06-03T19:41:45.969764+0000 mgr.y (mgr.34104) 28 : cluster [DBG] pgmap v19: 89 pgs: 89 active+clean; 582 KiB data, 696 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:41:47.836 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:41:47 smithi164 bash[26855]: audit 2023-06-03T19:41:46.442832+0000 mon.a (mon.0) 49 : audit [INF] from='client.? 172.21.15.89:0/111199136' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_1", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-06-03T19:41:47.836 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:41:47 smithi164 bash[26855]: cluster 2023-06-03T19:41:46.442914+0000 mon.a (mon.0) 50 : cluster [DBG] osdmap e77: 8 total, 8 up, 8 in 2023-06-03T19:41:47.836 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:41:47 smithi164 bash[26855]: cluster 2023-06-03T19:41:46.443841+0000 osd.5 (osd.5) 5 : cluster [WRN] failed to encode map e77 with expected crc 2023-06-03T19:41:47.836 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:41:47 smithi164 bash[26855]: cluster 2023-06-03T19:41:46.446238+0000 osd.0 (osd.0) 6 : cluster [WRN] failed to encode map e77 with expected crc 2023-06-03T19:41:48.321 DEBUG:teuthology.orchestra.run.smithi089:> /bin/sh -c 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage rados --no-log-to-stderr --name client.0 -b 65536 --object-size 65536 -p unique_pool_1 bench 90 write' 2023-06-03T19:41:48.323 INFO:tasks.radosbench:joining radosbench (timing out after 3000s) 2023-06-03T19:41:48.324 INFO:teuthology.orchestra.run:waiting for 3000 2023-06-03T19:41:48.368 INFO:tasks.radosbench.radosbench.0.smithi089.stdout:hints = 1 2023-06-03T19:41:48.368 INFO:tasks.radosbench.radosbench.0.smithi089.stdout:Maintaining 16 concurrent writes of 65536 bytes to objects of size 65536 for up to 90 seconds or 0 objects 2023-06-03T19:41:48.369 INFO:tasks.radosbench.radosbench.0.smithi089.stdout:Object prefix: benchmark_data_smithi089_77191 2023-06-03T19:41:48.369 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-06-03T19:41:48.369 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 0 0 0 0 0 0 - 0 2023-06-03T19:41:49.085 INFO:journalctl@ceph.prometheus.a.smithi164.stdout:Jun 03 19:41:48 smithi164 bash[55188]: ts=2023-06-03T19:41:48.661Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.13-257-gd8c5d349 (d8c5d34975dce1c5eb0aa3a7979a4d9b9a99d1ec) pacific (stable)\", cluster_addr=\"172.21.15.89\", device_class=\"ssd\", hostname=\"smithi089\", instance=\"smithi164.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.89\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.13-257-gd8c5d349 (d8c5d34975dce1c5eb0aa3a7979a4d9b9a99d1ec) pacific (stable)\", cluster_addr=\"172.21.15.89\", device_class=\"ssd\", hostname=\"smithi089\", instance=\"smithi089:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.89\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-06-03T19:41:49.086 INFO:journalctl@ceph.prometheus.a.smithi164.stdout:Jun 03 19:41:48 smithi164 bash[55188]: ts=2023-06-03T19:41:48.663Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephPGImbalance\nexpr: abs(((ceph_osd_numpg > 0) - on(job) group_left() avg by(job) (ceph_osd_numpg\n > 0)) / on(job) group_left() avg by(job) (ceph_osd_numpg > 0)) * on(ceph_daemon)\n group_left(hostname) ceph_osd_metadata > 0.3\nfor: 5m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.5\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} deviates by\n more than 30% from average PG count.\n summary: PGs are not balanced across OSDs\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.13-257-gd8c5d349 (d8c5d34975dce1c5eb0aa3a7979a4d9b9a99d1ec) pacific (stable)\", cluster_addr=\"172.21.15.89\", device_class=\"ssd\", hostname=\"smithi089\", instance=\"smithi164.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.89\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.13-257-gd8c5d349 (d8c5d34975dce1c5eb0aa3a7979a4d9b9a99d1ec) pacific (stable)\", cluster_addr=\"172.21.15.89\", device_class=\"ssd\", hostname=\"smithi089\", instance=\"smithi089:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.89\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-06-03T19:41:49.369 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 1 16 3175 3159 197.426 197.438 0.00194803 0.00504635 2023-06-03T19:41:49.781 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:41:49 smithi089 bash[33173]: cluster 2023-06-03T19:41:47.970813+0000 mgr.y (mgr.34104) 29 : cluster [DBG] pgmap v21: 89 pgs: 89 active+clean; 582 KiB data, 696 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:41:49.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:41:49 smithi164 bash[26855]: cluster 2023-06-03T19:41:47.970813+0000 mgr.y (mgr.34104) 29 : cluster [DBG] pgmap v21: 89 pgs: 89 active+clean; 582 KiB data, 696 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:41:50.369 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 2 16 6122 6106 190.796 184.188 0.00562394 0.00522883 2023-06-03T19:41:51.369 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 3 16 8923 8907 185.546 175.062 0.00443365 0.00538057 2023-06-03T19:41:51.781 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:41:51 smithi089 bash[33173]: cluster 2023-06-03T19:41:49.971280+0000 mgr.y (mgr.34104) 30 : cluster [DBG] pgmap v22: 89 pgs: 89 active+clean; 582 KiB data, 696 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:41:51.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:41:51 smithi164 bash[26855]: cluster 2023-06-03T19:41:49.971280+0000 mgr.y (mgr.34104) 30 : cluster [DBG] pgmap v22: 89 pgs: 89 active+clean; 582 KiB data, 696 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:41:52.369 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 4 16 11531 11515 179.906 163 0.0031222 0.0055483 2023-06-03T19:41:52.781 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:41:52 smithi089 bash[33173]: audit 2023-06-03T19:41:52.061611+0000 mon.c (mon.2) 109 : audit [DBG] from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T19:41:52.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:41:52 smithi164 bash[26855]: audit 2023-06-03T19:41:52.061611+0000 mon.c (mon.2) 109 : audit [DBG] from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T19:41:53.370 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 5 16 13993 13977 174.696 153.875 0.00495398 0.00571826 2023-06-03T19:41:53.781 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:41:53 smithi089 bash[33173]: cluster 2023-06-03T19:41:51.971853+0000 mgr.y (mgr.34104) 31 : cluster [DBG] pgmap v23: 89 pgs: 89 active+clean; 582 KiB data, 696 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:41:53.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:41:53 smithi164 bash[26855]: cluster 2023-06-03T19:41:51.971853+0000 mgr.y (mgr.34104) 31 : cluster [DBG] pgmap v23: 89 pgs: 89 active+clean; 582 KiB data, 696 MiB used, 715 GiB / 715 GiB avail 2023-06-03T19:41:54.370 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 6 16 16364 16348 170.276 148.188 0.0107821 0.00586562 2023-06-03T19:41:55.370 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 7 16 18622 18606 166.11 141.125 0.00332356 0.00601254 2023-06-03T19:41:55.781 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:41:55 smithi089 bash[33173]: cluster 2023-06-03T19:41:53.972747+0000 mgr.y (mgr.34104) 32 : cluster [DBG] pgmap v24: 89 pgs: 89 active+clean; 813 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 81 MiB/s wr, 1.30k op/s 2023-06-03T19:41:55.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:41:55 smithi164 bash[26855]: cluster 2023-06-03T19:41:53.972747+0000 mgr.y (mgr.34104) 32 : cluster [DBG] pgmap v24: 89 pgs: 89 active+clean; 813 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 81 MiB/s wr, 1.30k op/s 2023-06-03T19:41:56.370 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 8 16 20824 20808 162.547 137.625 0.00720791 0.00614585 2023-06-03T19:41:56.531 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 19:41:56 smithi089 bash[71163]: ::ffff:172.21.15.164 - - [03/Jun/2023:19:41:56] "GET /metrics HTTP/1.1" 200 189899 "" "Prometheus/2.33.4" 2023-06-03T19:41:57.370 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 9 16 22939 22923 159.173 132.188 0.00368519 0.00627541 2023-06-03T19:41:57.781 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:41:57 smithi089 bash[33173]: cluster 2023-06-03T19:41:55.973202+0000 mgr.y (mgr.34104) 33 : cluster [DBG] pgmap v25: 89 pgs: 89 active+clean; 813 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 81 MiB/s wr, 1.30k op/s 2023-06-03T19:41:57.836 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:41:57 smithi164 bash[26855]: cluster 2023-06-03T19:41:55.973202+0000 mgr.y (mgr.34104) 33 : cluster [DBG] pgmap v25: 89 pgs: 89 active+clean; 813 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 81 MiB/s wr, 1.30k op/s 2023-06-03T19:41:58.370 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 10 16 24962 24946 155.898 126.438 0.00579436 0.0064075 2023-06-03T19:41:59.086 INFO:journalctl@ceph.prometheus.a.smithi164.stdout:Jun 03 19:41:58 smithi164 bash[55188]: ts=2023-06-03T19:41:58.661Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.13-257-gd8c5d349 (d8c5d34975dce1c5eb0aa3a7979a4d9b9a99d1ec) pacific (stable)\", cluster_addr=\"172.21.15.89\", device_class=\"ssd\", hostname=\"smithi089\", instance=\"smithi164.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.89\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.13-257-gd8c5d349 (d8c5d34975dce1c5eb0aa3a7979a4d9b9a99d1ec) pacific (stable)\", cluster_addr=\"172.21.15.89\", device_class=\"ssd\", hostname=\"smithi089\", instance=\"smithi089:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.89\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-06-03T19:41:59.086 INFO:journalctl@ceph.prometheus.a.smithi164.stdout:Jun 03 19:41:58 smithi164 bash[55188]: ts=2023-06-03T19:41:58.662Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephPGImbalance\nexpr: abs(((ceph_osd_numpg > 0) - on(job) group_left() avg by(job) (ceph_osd_numpg\n > 0)) / on(job) group_left() avg by(job) (ceph_osd_numpg > 0)) * on(ceph_daemon)\n group_left(hostname) ceph_osd_metadata > 0.3\nfor: 5m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.5\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} deviates by\n more than 30% from average PG count.\n summary: PGs are not balanced across OSDs\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.13-257-gd8c5d349 (d8c5d34975dce1c5eb0aa3a7979a4d9b9a99d1ec) pacific (stable)\", cluster_addr=\"172.21.15.89\", device_class=\"ssd\", hostname=\"smithi089\", instance=\"smithi164.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.89\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.13-257-gd8c5d349 (d8c5d34975dce1c5eb0aa3a7979a4d9b9a99d1ec) pacific (stable)\", cluster_addr=\"172.21.15.89\", device_class=\"ssd\", hostname=\"smithi089\", instance=\"smithi089:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.89\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-06-03T19:41:59.370 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 11 16 27028 27012 153.463 129.125 0.00582497 0.00650839 2023-06-03T19:41:59.781 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:41:59 smithi089 bash[33173]: cluster 2023-06-03T19:41:57.973759+0000 mgr.y (mgr.34104) 34 : cluster [DBG] pgmap v26: 89 pgs: 89 active+clean; 1.3 GiB data, 4.6 GiB used, 711 GiB / 715 GiB avail; 120 MiB/s wr, 1.91k op/s 2023-06-03T19:41:59.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:41:59 smithi164 bash[26855]: cluster 2023-06-03T19:41:57.973759+0000 mgr.y (mgr.34104) 34 : cluster [DBG] pgmap v26: 89 pgs: 89 active+clean; 1.3 GiB data, 4.6 GiB used, 711 GiB / 715 GiB avail; 120 MiB/s wr, 1.91k op/s 2023-06-03T19:42:00.370 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 12 16 28953 28937 150.7 120.312 0.00572141 0.00663039 2023-06-03T19:42:01.370 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 13 16 30878 30862 148.361 120.312 0.00397455 0.00673303 2023-06-03T19:42:01.781 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:42:01 smithi089 bash[33173]: cluster 2023-06-03T19:41:59.974343+0000 mgr.y (mgr.34104) 35 : cluster [DBG] pgmap v27: 89 pgs: 89 active+clean; 1.5 GiB data, 5.0 GiB used, 710 GiB / 715 GiB avail; 126 MiB/s wr, 2.01k op/s 2023-06-03T19:42:01.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:42:01 smithi164 bash[26855]: cluster 2023-06-03T19:41:59.974343+0000 mgr.y (mgr.34104) 35 : cluster [DBG] pgmap v27: 89 pgs: 89 active+clean; 1.5 GiB data, 5.0 GiB used, 710 GiB / 715 GiB avail; 126 MiB/s wr, 2.01k op/s 2023-06-03T19:42:02.370 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 14 16 32722 32706 145.995 115.25 0.00763645 0.0068429 2023-06-03T19:42:03.370 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 15 16 34452 34436 143.47 108.125 0.00534768 0.00696373 2023-06-03T19:42:03.781 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:42:03 smithi089 bash[33173]: cluster 2023-06-03T19:42:01.974810+0000 mgr.y (mgr.34104) 36 : cluster [DBG] pgmap v28: 89 pgs: 89 active+clean; 1.5 GiB data, 5.0 GiB used, 710 GiB / 715 GiB avail; 126 MiB/s wr, 2.01k op/s 2023-06-03T19:42:03.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:42:03 smithi164 bash[26855]: cluster 2023-06-03T19:42:01.974810+0000 mgr.y (mgr.34104) 36 : cluster [DBG] pgmap v28: 89 pgs: 89 active+clean; 1.5 GiB data, 5.0 GiB used, 710 GiB / 715 GiB avail; 126 MiB/s wr, 2.01k op/s 2023-06-03T19:42:04.371 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 16 16 36184 36168 141.268 108.25 0.00593814 0.00707281 2023-06-03T19:42:05.371 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 17 16 37850 37834 139.082 104.125 0.00739154 0.0071837 2023-06-03T19:42:05.781 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:42:05 smithi089 bash[33173]: cluster 2023-06-03T19:42:03.975818+0000 mgr.y (mgr.34104) 37 : cluster [DBG] pgmap v29: 89 pgs: 89 active+clean; 2.1 GiB data, 6.8 GiB used, 708 GiB / 715 GiB avail; 176 MiB/s wr, 2.81k op/s 2023-06-03T19:42:05.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:42:05 smithi164 bash[26855]: cluster 2023-06-03T19:42:03.975818+0000 mgr.y (mgr.34104) 37 : cluster [DBG] pgmap v29: 89 pgs: 89 active+clean; 2.1 GiB data, 6.8 GiB used, 708 GiB / 715 GiB avail; 176 MiB/s wr, 2.81k op/s 2023-06-03T19:42:06.371 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 18 16 39454 39438 136.924 100.25 0.00890795 0.00729627 2023-06-03T19:42:06.531 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 19:42:06 smithi089 bash[71163]: ::ffff:172.21.15.164 - - [03/Jun/2023:19:42:06] "GET /metrics HTTP/1.1" 200 189899 "" "Prometheus/2.33.4" 2023-06-03T19:42:07.371 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 19 16 41023 41007 134.878 98.0625 0.00453012 0.00740722 2023-06-03T19:42:07.781 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:42:07 smithi089 bash[33173]: cluster 2023-06-03T19:42:05.976312+0000 mgr.y (mgr.34104) 38 : cluster [DBG] pgmap v30: 89 pgs: 89 active+clean; 2.1 GiB data, 6.8 GiB used, 708 GiB / 715 GiB avail; 108 MiB/s wr, 1.73k op/s 2023-06-03T19:42:07.781 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:42:07 smithi089 bash[33173]: audit 2023-06-03T19:42:07.061846+0000 mon.c (mon.2) 110 : audit [DBG] from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T19:42:07.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:42:07 smithi164 bash[26855]: cluster 2023-06-03T19:42:05.976312+0000 mgr.y (mgr.34104) 38 : cluster [DBG] pgmap v30: 89 pgs: 89 active+clean; 2.1 GiB data, 6.8 GiB used, 708 GiB / 715 GiB avail; 108 MiB/s wr, 1.73k op/s 2023-06-03T19:42:07.836 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:42:07 smithi164 bash[26855]: audit 2023-06-03T19:42:07.061846+0000 mon.c (mon.2) 110 : audit [DBG] from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T19:42:08.371 INFO:tasks.radosbench.radosbench.0.smithi089.stdout:2023-06-03T19:42:08.364916+0000 min lat: 0.00131693 max lat: 0.0528736 avg lat: 0.00751535 2023-06-03T19:42:08.371 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-06-03T19:42:08.371 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 20 15 42561 42546 132.943 96.1875 0.0172319 0.00751535 2023-06-03T19:42:09.085 INFO:journalctl@ceph.prometheus.a.smithi164.stdout:Jun 03 19:42:08 smithi164 bash[55188]: ts=2023-06-03T19:42:08.661Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.13-257-gd8c5d349 (d8c5d34975dce1c5eb0aa3a7979a4d9b9a99d1ec) pacific (stable)\", cluster_addr=\"172.21.15.89\", device_class=\"ssd\", hostname=\"smithi089\", instance=\"smithi164.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.89\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.13-257-gd8c5d349 (d8c5d34975dce1c5eb0aa3a7979a4d9b9a99d1ec) pacific (stable)\", cluster_addr=\"172.21.15.89\", device_class=\"ssd\", hostname=\"smithi089\", instance=\"smithi089:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.89\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-06-03T19:42:09.086 INFO:journalctl@ceph.prometheus.a.smithi164.stdout:Jun 03 19:42:08 smithi164 bash[55188]: ts=2023-06-03T19:42:08.662Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephPGImbalance\nexpr: abs(((ceph_osd_numpg > 0) - on(job) group_left() avg by(job) (ceph_osd_numpg\n > 0)) / on(job) group_left() avg by(job) (ceph_osd_numpg > 0)) * on(ceph_daemon)\n group_left(hostname) ceph_osd_metadata > 0.3\nfor: 5m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.5\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} deviates by\n more than 30% from average PG count.\n summary: PGs are not balanced across OSDs\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.13-257-gd8c5d349 (d8c5d34975dce1c5eb0aa3a7979a4d9b9a99d1ec) pacific (stable)\", cluster_addr=\"172.21.15.89\", device_class=\"ssd\", hostname=\"smithi089\", instance=\"smithi164.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.89\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.13-257-gd8c5d349 (d8c5d34975dce1c5eb0aa3a7979a4d9b9a99d1ec) pacific (stable)\", cluster_addr=\"172.21.15.89\", device_class=\"ssd\", hostname=\"smithi089\", instance=\"smithi089:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.89\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-06-03T19:42:09.371 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 21 16 44098 44082 131.184 96 0.00905771 0.00761653 2023-06-03T19:42:09.781 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:42:09 smithi089 bash[33173]: cluster 2023-06-03T19:42:07.977007+0000 mgr.y (mgr.34104) 39 : cluster [DBG] pgmap v31: 89 pgs: 89 active+clean; 2.4 GiB data, 7.6 GiB used, 708 GiB / 715 GiB avail; 135 MiB/s wr, 2.15k op/s 2023-06-03T19:42:09.836 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:42:09 smithi164 bash[26855]: cluster 2023-06-03T19:42:07.977007+0000 mgr.y (mgr.34104) 39 : cluster [DBG] pgmap v31: 89 pgs: 89 active+clean; 2.4 GiB data, 7.6 GiB used, 708 GiB / 715 GiB avail; 135 MiB/s wr, 2.15k op/s 2023-06-03T19:42:10.371 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 22 16 45624 45608 129.556 95.375 0.0144522 0.00771351 2023-06-03T19:42:11.371 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 23 16 47114 47098 127.971 93.125 0.0077677 0.00780849 2023-06-03T19:42:11.781 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:42:11 smithi089 bash[33173]: cluster 2023-06-03T19:42:09.978197+0000 mgr.y (mgr.34104) 40 : cluster [DBG] pgmap v32: 89 pgs: 89 active+clean; 2.6 GiB data, 8.4 GiB used, 707 GiB / 715 GiB avail; 104 MiB/s wr, 1.66k op/s 2023-06-03T19:42:11.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:42:11 smithi164 bash[26855]: cluster 2023-06-03T19:42:09.978197+0000 mgr.y (mgr.34104) 40 : cluster [DBG] pgmap v32: 89 pgs: 89 active+clean; 2.6 GiB data, 8.4 GiB used, 707 GiB / 715 GiB avail; 104 MiB/s wr, 1.66k op/s 2023-06-03T19:42:12.371 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 24 16 48559 48543 126.402 90.3125 0.0090702 0.00790512 2023-06-03T19:42:13.371 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 25 16 49984 49968 124.908 89.0625 0.0216059 0.00800055 2023-06-03T19:42:13.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:42:13 smithi164 bash[26855]: cluster 2023-06-03T19:42:11.978677+0000 mgr.y (mgr.34104) 41 : cluster [DBG] pgmap v33: 89 pgs: 89 active+clean; 2.6 GiB data, 8.4 GiB used, 707 GiB / 715 GiB avail; 93 MiB/s wr, 1.48k op/s 2023-06-03T19:42:14.031 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:42:13 smithi089 bash[33173]: cluster 2023-06-03T19:42:11.978677+0000 mgr.y (mgr.34104) 41 : cluster [DBG] pgmap v33: 89 pgs: 89 active+clean; 2.6 GiB data, 8.4 GiB used, 707 GiB / 715 GiB avail; 93 MiB/s wr, 1.48k op/s 2023-06-03T19:42:14.371 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 26 16 51348 51332 123.382 85.25 0.00818093 0.00809876 2023-06-03T19:42:15.371 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 27 16 52701 52685 121.944 84.5625 0.0095421 0.00819498 2023-06-03T19:42:15.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:42:15 smithi164 bash[26855]: cluster 2023-06-03T19:42:13.979385+0000 mgr.y (mgr.34104) 42 : cluster [DBG] pgmap v34: 89 pgs: 89 active+clean; 3.0 GiB data, 9.7 GiB used, 705 GiB / 715 GiB avail; 131 MiB/s wr, 2.10k op/s 2023-06-03T19:42:16.031 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:42:15 smithi089 bash[33173]: cluster 2023-06-03T19:42:13.979385+0000 mgr.y (mgr.34104) 42 : cluster [DBG] pgmap v34: 89 pgs: 89 active+clean; 3.0 GiB data, 9.7 GiB used, 705 GiB / 715 GiB avail; 131 MiB/s wr, 2.10k op/s 2023-06-03T19:42:16.372 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 28 16 54040 54024 120.578 83.6875 0.0128021 0.00828802 2023-06-03T19:42:16.531 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 19:42:16 smithi089 bash[71163]: ::ffff:172.21.15.164 - - [03/Jun/2023:19:42:16] "GET /metrics HTTP/1.1" 200 189997 "" "Prometheus/2.33.4" 2023-06-03T19:42:17.372 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 29 16 55357 55341 119.258 82.3125 0.00553014 0.00837918 2023-06-03T19:42:17.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:42:17 smithi164 bash[26855]: cluster 2023-06-03T19:42:15.979895+0000 mgr.y (mgr.34104) 43 : cluster [DBG] pgmap v35: 89 pgs: 89 active+clean; 3.0 GiB data, 9.7 GiB used, 705 GiB / 715 GiB avail; 82 MiB/s wr, 1.30k op/s 2023-06-03T19:42:18.031 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:42:17 smithi089 bash[33173]: cluster 2023-06-03T19:42:15.979895+0000 mgr.y (mgr.34104) 43 : cluster [DBG] pgmap v35: 89 pgs: 89 active+clean; 3.0 GiB data, 9.7 GiB used, 705 GiB / 715 GiB avail; 82 MiB/s wr, 1.30k op/s 2023-06-03T19:42:18.372 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 30 16 56660 56644 117.997 81.4375 0.00780139 0.00846907 2023-06-03T19:42:19.085 INFO:journalctl@ceph.prometheus.a.smithi164.stdout:Jun 03 19:42:18 smithi164 bash[55188]: ts=2023-06-03T19:42:18.663Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.13-257-gd8c5d349 (d8c5d34975dce1c5eb0aa3a7979a4d9b9a99d1ec) pacific (stable)\", cluster_addr=\"172.21.15.89\", device_class=\"ssd\", hostname=\"smithi089\", instance=\"smithi164.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.89\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.13-257-gd8c5d349 (d8c5d34975dce1c5eb0aa3a7979a4d9b9a99d1ec) pacific (stable)\", cluster_addr=\"172.21.15.89\", device_class=\"ssd\", hostname=\"smithi089\", instance=\"smithi089:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.89\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-06-03T19:42:19.085 INFO:journalctl@ceph.prometheus.a.smithi164.stdout:Jun 03 19:42:18 smithi164 bash[55188]: ts=2023-06-03T19:42:18.664Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephPGImbalance\nexpr: abs(((ceph_osd_numpg > 0) - on(job) group_left() avg by(job) (ceph_osd_numpg\n > 0)) / on(job) group_left() avg by(job) (ceph_osd_numpg > 0)) * on(ceph_daemon)\n group_left(hostname) ceph_osd_metadata > 0.3\nfor: 5m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.5\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} deviates by\n more than 30% from average PG count.\n summary: PGs are not balanced across OSDs\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.13-257-gd8c5d349 (d8c5d34975dce1c5eb0aa3a7979a4d9b9a99d1ec) pacific (stable)\", cluster_addr=\"172.21.15.89\", device_class=\"ssd\", hostname=\"smithi089\", instance=\"smithi164.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.89\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.13-257-gd8c5d349 (d8c5d34975dce1c5eb0aa3a7979a4d9b9a99d1ec) pacific (stable)\", cluster_addr=\"172.21.15.89\", device_class=\"ssd\", hostname=\"smithi089\", instance=\"smithi089:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.89\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-06-03T19:42:19.372 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 31 16 57953 57937 116.797 80.8125 0.00546466 0.00855636 2023-06-03T19:42:19.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:42:19 smithi164 bash[26855]: cluster 2023-06-03T19:42:17.980620+0000 mgr.y (mgr.34104) 44 : cluster [DBG] pgmap v36: 89 pgs: 89 active+clean; 3.3 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 104 MiB/s wr, 1.66k op/s 2023-06-03T19:42:20.031 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:42:19 smithi089 bash[33173]: cluster 2023-06-03T19:42:17.980620+0000 mgr.y (mgr.34104) 44 : cluster [DBG] pgmap v36: 89 pgs: 89 active+clean; 3.3 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 104 MiB/s wr, 1.66k op/s 2023-06-03T19:42:20.372 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 32 16 59228 59212 115.637 79.6875 0.00721436 0.0086425 2023-06-03T19:42:21.372 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 33 16 60478 60462 114.5 78.125 0.00675836 0.00872734 2023-06-03T19:42:21.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:42:21 smithi164 bash[26855]: cluster 2023-06-03T19:42:19.981237+0000 mgr.y (mgr.34104) 45 : cluster [DBG] pgmap v37: 89 pgs: 89 active+clean; 3.4 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 90 MiB/s wr, 1.44k op/s 2023-06-03T19:42:22.031 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:42:21 smithi089 bash[33173]: cluster 2023-06-03T19:42:19.981237+0000 mgr.y (mgr.34104) 45 : cluster [DBG] pgmap v37: 89 pgs: 89 active+clean; 3.4 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 90 MiB/s wr, 1.44k op/s 2023-06-03T19:42:22.372 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 34 16 61605 61589 113.204 70.4375 0.0989311 0.00880864 2023-06-03T19:42:22.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:42:22 smithi164 bash[26855]: audit 2023-06-03T19:42:22.065282+0000 mon.c (mon.2) 111 : audit [DBG] from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T19:42:23.031 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:42:22 smithi089 bash[33173]: audit 2023-06-03T19:42:22.065282+0000 mon.c (mon.2) 111 : audit [DBG] from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T19:42:23.372 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 35 16 62829 62813 112.155 76.5 0.00728805 0.00891023 2023-06-03T19:42:23.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:42:23 smithi164 bash[26855]: cluster 2023-06-03T19:42:21.981708+0000 mgr.y (mgr.34104) 46 : cluster [DBG] pgmap v38: 89 pgs: 89 active+clean; 3.4 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 74 MiB/s wr, 1.18k op/s 2023-06-03T19:42:24.031 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:42:23 smithi089 bash[33173]: cluster 2023-06-03T19:42:21.981708+0000 mgr.y (mgr.34104) 46 : cluster [DBG] pgmap v38: 89 pgs: 89 active+clean; 3.4 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 74 MiB/s wr, 1.18k op/s 2023-06-03T19:42:24.372 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 36 16 64048 64032 111.156 76.1875 0.0158213 0.00899062 2023-06-03T19:42:25.372 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 37 16 65252 65236 110.185 75.25 0.0174665 0.00907001 2023-06-03T19:42:25.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:42:25 smithi164 bash[26855]: cluster 2023-06-03T19:42:23.982566+0000 mgr.y (mgr.34104) 47 : cluster [DBG] pgmap v39: 89 pgs: 89 active+clean; 3.8 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 106 MiB/s wr, 1.70k op/s 2023-06-03T19:42:26.031 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:42:25 smithi089 bash[33173]: cluster 2023-06-03T19:42:23.982566+0000 mgr.y (mgr.34104) 47 : cluster [DBG] pgmap v39: 89 pgs: 89 active+clean; 3.8 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 106 MiB/s wr, 1.70k op/s 2023-06-03T19:42:26.373 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 38 16 66411 66395 109.192 72.4375 0.0100308 0.00915209 2023-06-03T19:42:26.531 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 19:42:26 smithi089 bash[71163]: ::ffff:172.21.15.164 - - [03/Jun/2023:19:42:26] "GET /metrics HTTP/1.1" 200 190025 "" "Prometheus/2.33.4" 2023-06-03T19:42:27.373 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 39 16 67570 67554 108.249 72.4375 0.0103306 0.00923225 2023-06-03T19:42:27.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:42:27 smithi164 bash[26855]: cluster 2023-06-03T19:42:25.982984+0000 mgr.y (mgr.34104) 48 : cluster [DBG] pgmap v40: 89 pgs: 89 active+clean; 3.8 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 67 MiB/s wr, 1.08k op/s 2023-06-03T19:42:28.031 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:42:27 smithi089 bash[33173]: cluster 2023-06-03T19:42:25.982984+0000 mgr.y (mgr.34104) 48 : cluster [DBG] pgmap v40: 89 pgs: 89 active+clean; 3.8 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 67 MiB/s wr, 1.08k op/s 2023-06-03T19:42:28.373 INFO:tasks.radosbench.radosbench.0.smithi089.stdout:2023-06-03T19:42:28.366846+0000 min lat: 0.00131693 max lat: 0.110456 avg lat: 0.00931221 2023-06-03T19:42:28.373 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-06-03T19:42:28.373 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 40 16 68710 68694 107.324 71.25 0.0247027 0.00931221 2023-06-03T19:42:29.085 INFO:journalctl@ceph.prometheus.a.smithi164.stdout:Jun 03 19:42:28 smithi164 bash[55188]: ts=2023-06-03T19:42:28.662Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.13-257-gd8c5d349 (d8c5d34975dce1c5eb0aa3a7979a4d9b9a99d1ec) pacific (stable)\", cluster_addr=\"172.21.15.89\", device_class=\"ssd\", hostname=\"smithi089\", instance=\"smithi164.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.89\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.13-257-gd8c5d349 (d8c5d34975dce1c5eb0aa3a7979a4d9b9a99d1ec) pacific (stable)\", cluster_addr=\"172.21.15.89\", device_class=\"ssd\", hostname=\"smithi089\", instance=\"smithi089:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.89\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-06-03T19:42:29.086 INFO:journalctl@ceph.prometheus.a.smithi164.stdout:Jun 03 19:42:28 smithi164 bash[55188]: ts=2023-06-03T19:42:28.663Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephPGImbalance\nexpr: abs(((ceph_osd_numpg > 0) - on(job) group_left() avg by(job) (ceph_osd_numpg\n > 0)) / on(job) group_left() avg by(job) (ceph_osd_numpg > 0)) * on(ceph_daemon)\n group_left(hostname) ceph_osd_metadata > 0.3\nfor: 5m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.5\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} deviates by\n more than 30% from average PG count.\n summary: PGs are not balanced across OSDs\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.13-257-gd8c5d349 (d8c5d34975dce1c5eb0aa3a7979a4d9b9a99d1ec) pacific (stable)\", cluster_addr=\"172.21.15.89\", device_class=\"ssd\", hostname=\"smithi089\", instance=\"smithi164.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.89\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.13-257-gd8c5d349 (d8c5d34975dce1c5eb0aa3a7979a4d9b9a99d1ec) pacific (stable)\", cluster_addr=\"172.21.15.89\", device_class=\"ssd\", hostname=\"smithi089\", instance=\"smithi089:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.89\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-06-03T19:42:29.373 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 41 16 69894 69878 106.511 74 0.00956597 0.00938317 2023-06-03T19:42:29.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:42:29 smithi164 bash[26855]: cluster 2023-06-03T19:42:27.983736+0000 mgr.y (mgr.34104) 49 : cluster [DBG] pgmap v41: 89 pgs: 89 active+clean; 3.9 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 77 MiB/s wr, 1.23k op/s 2023-06-03T19:42:30.031 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:42:29 smithi089 bash[33173]: cluster 2023-06-03T19:42:27.983736+0000 mgr.y (mgr.34104) 49 : cluster [DBG] pgmap v41: 89 pgs: 89 active+clean; 3.9 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 77 MiB/s wr, 1.23k op/s 2023-06-03T19:42:30.373 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 42 16 71077 71061 105.735 73.9375 0.0100705 0.00945205 2023-06-03T19:42:31.373 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 43 16 72241 72225 104.968 72.75 0.0161156 0.00952093 2023-06-03T19:42:31.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:42:31 smithi164 bash[26855]: cluster 2023-06-03T19:42:29.984251+0000 mgr.y (mgr.34104) 50 : cluster [DBG] pgmap v42: 89 pgs: 89 active+clean; 4.2 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 76 MiB/s wr, 1.22k op/s 2023-06-03T19:42:32.031 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:42:31 smithi089 bash[33173]: cluster 2023-06-03T19:42:29.984251+0000 mgr.y (mgr.34104) 50 : cluster [DBG] pgmap v42: 89 pgs: 89 active+clean; 4.2 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 76 MiB/s wr, 1.22k op/s 2023-06-03T19:42:32.373 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 44 16 73374 73358 104.191 70.8125 0.00348936 0.00958769 2023-06-03T19:42:33.373 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 45 16 74458 74442 103.382 67.75 0.00872425 0.00966707 2023-06-03T19:42:34.031 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:42:33 smithi089 bash[33173]: cluster 2023-06-03T19:42:31.984694+0000 mgr.y (mgr.34104) 51 : cluster [DBG] pgmap v43: 89 pgs: 89 active+clean; 4.2 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 63 MiB/s wr, 1.01k op/s 2023-06-03T19:42:34.085 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:42:33 smithi164 bash[26855]: cluster 2023-06-03T19:42:31.984694+0000 mgr.y (mgr.34104) 51 : cluster [DBG] pgmap v43: 89 pgs: 89 active+clean; 4.2 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 63 MiB/s wr, 1.01k op/s 2023-06-03T19:42:34.373 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 46 16 75506 75490 102.558 65.5 0.0173438 0.00974441 2023-06-03T19:42:35.340 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -e sha1=8e24bcf462a1d94c2e12aa4e26419634883a1e88 -- bash -c 'ceph orch upgrade resume' 2023-06-03T19:42:35.373 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 47 16 76577 76561 101.8 66.9375 0.0146685 0.00981769 2023-06-03T19:42:36.031 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:42:35 smithi089 bash[33173]: cluster 2023-06-03T19:42:33.985433+0000 mgr.y (mgr.34104) 52 : cluster [DBG] pgmap v44: 89 pgs: 89 active+clean; 4.5 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 93 MiB/s wr, 1.49k op/s 2023-06-03T19:42:36.085 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:42:35 smithi164 bash[26855]: cluster 2023-06-03T19:42:33.985433+0000 mgr.y (mgr.34104) 52 : cluster [DBG] pgmap v44: 89 pgs: 89 active+clean; 4.5 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 93 MiB/s wr, 1.49k op/s 2023-06-03T19:42:36.373 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 48 16 77632 77616 101.053 65.9375 0.011227 0.00989026 2023-06-03T19:42:36.531 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 19:42:36 smithi089 bash[71163]: ::ffff:172.21.15.164 - - [03/Jun/2023:19:42:36] "GET /metrics HTTP/1.1" 200 190025 "" "Prometheus/2.33.4" 2023-06-03T19:42:37.373 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 49 16 78592 78576 100.215 60 0.00877908 0.00997234 2023-06-03T19:42:37.780 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:42:37 smithi089 bash[33173]: cluster 2023-06-03T19:42:35.985908+0000 mgr.y (mgr.34104) 53 : cluster [DBG] pgmap v45: 89 pgs: 89 active+clean; 4.5 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 61 MiB/s wr, 972 op/s 2023-06-03T19:42:37.781 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:42:37 smithi089 bash[33173]: audit 2023-06-03T19:42:37.062639+0000 mon.c (mon.2) 112 : audit [DBG] from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T19:42:38.085 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:42:37 smithi164 bash[26855]: cluster 2023-06-03T19:42:35.985908+0000 mgr.y (mgr.34104) 53 : cluster [DBG] pgmap v45: 89 pgs: 89 active+clean; 4.5 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 61 MiB/s wr, 972 op/s 2023-06-03T19:42:38.085 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:42:37 smithi164 bash[26855]: audit 2023-06-03T19:42:37.062639+0000 mon.c (mon.2) 112 : audit [DBG] from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T19:42:38.374 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 50 16 79609 79593 99.4815 63.5625 0.0189717 0.0100461 2023-06-03T19:42:39.085 INFO:journalctl@ceph.prometheus.a.smithi164.stdout:Jun 03 19:42:38 smithi164 bash[55188]: ts=2023-06-03T19:42:38.661Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.13-257-gd8c5d349 (d8c5d34975dce1c5eb0aa3a7979a4d9b9a99d1ec) pacific (stable)\", cluster_addr=\"172.21.15.89\", device_class=\"ssd\", hostname=\"smithi089\", instance=\"smithi164.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.89\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.13-257-gd8c5d349 (d8c5d34975dce1c5eb0aa3a7979a4d9b9a99d1ec) pacific (stable)\", cluster_addr=\"172.21.15.89\", device_class=\"ssd\", hostname=\"smithi089\", instance=\"smithi089:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.89\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-06-03T19:42:39.085 INFO:journalctl@ceph.prometheus.a.smithi164.stdout:Jun 03 19:42:38 smithi164 bash[55188]: ts=2023-06-03T19:42:38.662Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephPGImbalance\nexpr: abs(((ceph_osd_numpg > 0) - on(job) group_left() avg by(job) (ceph_osd_numpg\n > 0)) / on(job) group_left() avg by(job) (ceph_osd_numpg > 0)) * on(ceph_daemon)\n group_left(hostname) ceph_osd_metadata > 0.3\nfor: 5m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.5\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} deviates by\n more than 30% from average PG count.\n summary: PGs are not balanced across OSDs\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.13-257-gd8c5d349 (d8c5d34975dce1c5eb0aa3a7979a4d9b9a99d1ec) pacific (stable)\", cluster_addr=\"172.21.15.89\", device_class=\"ssd\", hostname=\"smithi089\", instance=\"smithi164.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.89\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.13-257-gd8c5d349 (d8c5d34975dce1c5eb0aa3a7979a4d9b9a99d1ec) pacific (stable)\", cluster_addr=\"172.21.15.89\", device_class=\"ssd\", hostname=\"smithi089\", instance=\"smithi089:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.89\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-06-03T19:42:39.374 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 51 16 80642 80626 98.7967 64.5625 0.00462733 0.0101164 2023-06-03T19:42:39.781 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:42:39 smithi089 bash[33173]: cluster 2023-06-03T19:42:37.986657+0000 mgr.y (mgr.34104) 54 : cluster [DBG] pgmap v46: 89 pgs: 89 active+clean; 4.7 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 78 MiB/s wr, 1.24k op/s 2023-06-03T19:42:40.085 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:42:39 smithi164 bash[26855]: cluster 2023-06-03T19:42:37.986657+0000 mgr.y (mgr.34104) 54 : cluster [DBG] pgmap v46: 89 pgs: 89 active+clean; 4.7 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 78 MiB/s wr, 1.24k op/s 2023-06-03T19:42:40.143 INFO:teuthology.orchestra.run.smithi089.stdout:Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:00b1b6d689278b4f97ff790045c8c51d8ca419ac7ee2ebdf11008f2e810a5c78 2023-06-03T19:42:40.374 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 52 16 81556 81540 97.9952 57.125 0.0325581 0.0101992 2023-06-03T19:42:41.243 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -e sha1=8e24bcf462a1d94c2e12aa4e26419634883a1e88 -- bash -c 'echo wait for majority of mons to upgrade' 2023-06-03T19:42:41.374 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 53 16 82505 82489 97.2653 59.3125 0.0132719 0.0102746 2023-06-03T19:42:41.515 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:42:41 smithi089 bash[33173]: cluster 2023-06-03T19:42:39.987171+0000 mgr.y (mgr.34104) 55 : cluster [DBG] pgmap v47: 89 pgs: 89 active+clean; 4.8 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 78 MiB/s wr, 1.25k op/s 2023-06-03T19:42:41.515 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:42:41 smithi089 bash[33173]: audit 2023-06-03T19:42:40.125785+0000 mgr.y (mgr.34104) 56 : audit [DBG] from='client.25355 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2023-06-03T19:42:41.515 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:42:41 smithi089 bash[33173]: cephadm 2023-06-03T19:42:40.127250+0000 mgr.y (mgr.34104) 57 : cephadm [INF] Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:00b1b6d689278b4f97ff790045c8c51d8ca419ac7ee2ebdf11008f2e810a5c78 2023-06-03T19:42:41.515 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:42:41 smithi089 bash[33173]: audit 2023-06-03T19:42:40.134535+0000 mon.a (mon.0) 51 : audit [INF] from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:42:41.516 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:42:41 smithi089 bash[33173]: audit 2023-06-03T19:42:40.136292+0000 mon.c (mon.2) 113 : audit [DBG] from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-03T19:42:41.585 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:42:41 smithi164 bash[26855]: cluster 2023-06-03T19:42:39.987171+0000 mgr.y (mgr.34104) 55 : cluster [DBG] pgmap v47: 89 pgs: 89 active+clean; 4.8 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 78 MiB/s wr, 1.25k op/s 2023-06-03T19:42:41.586 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:42:41 smithi164 bash[26855]: audit 2023-06-03T19:42:40.125785+0000 mgr.y (mgr.34104) 56 : audit [DBG] from='client.25355 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2023-06-03T19:42:41.586 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:42:41 smithi164 bash[26855]: cephadm 2023-06-03T19:42:40.127250+0000 mgr.y (mgr.34104) 57 : cephadm [INF] Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:00b1b6d689278b4f97ff790045c8c51d8ca419ac7ee2ebdf11008f2e810a5c78 2023-06-03T19:42:41.586 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:42:41 smithi164 bash[26855]: audit 2023-06-03T19:42:40.134535+0000 mon.a (mon.0) 51 : audit [INF] from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:42:41.586 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:42:41 smithi164 bash[26855]: audit 2023-06-03T19:42:40.136292+0000 mon.c (mon.2) 113 : audit [DBG] from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-03T19:42:42.374 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 54 16 83457 83441 96.5658 59.5 0.0100778 0.0103499 2023-06-03T19:42:43.374 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 55 16 84440 84424 95.927 61.4375 0.0150049 0.0104191 2023-06-03T19:42:43.462 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:42:43 smithi089 bash[33173]: cluster 2023-06-03T19:42:41.987606+0000 mgr.y (mgr.34104) 58 : cluster [DBG] pgmap v48: 89 pgs: 89 active+clean; 4.8 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 57 MiB/s wr, 913 op/s 2023-06-03T19:42:43.585 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:42:43 smithi164 bash[26855]: cluster 2023-06-03T19:42:41.987606+0000 mgr.y (mgr.34104) 58 : cluster [DBG] pgmap v48: 89 pgs: 89 active+clean; 4.8 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 57 MiB/s wr, 913 op/s 2023-06-03T19:42:44.374 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 56 16 85468 85452 95.3612 64.25 0.00825191 0.0104808 2023-06-03T19:42:45.374 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 57 16 86434 86418 94.7474 60.375 0.0209164 0.0105476 2023-06-03T19:42:45.531 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:42:45 smithi089 bash[33173]: cluster 2023-06-03T19:42:43.988332+0000 mgr.y (mgr.34104) 59 : cluster [DBG] pgmap v49: 89 pgs: 89 active+clean; 5.1 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 82 MiB/s wr, 1.32k op/s 2023-06-03T19:42:45.582 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:42:45 smithi164 bash[26855]: cluster 2023-06-03T19:42:43.988332+0000 mgr.y (mgr.34104) 59 : cluster [DBG] pgmap v49: 89 pgs: 89 active+clean; 5.1 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 82 MiB/s wr, 1.32k op/s 2023-06-03T19:42:45.634 INFO:teuthology.orchestra.run.smithi089.stdout:wait for majority of mons to upgrade 2023-06-03T19:42:46.234 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 19:42:46 smithi089 bash[71163]: ::ffff:172.21.15.164 - - [03/Jun/2023:19:42:46] "GET /metrics HTTP/1.1" 200 190029 "" "Prometheus/2.33.4" 2023-06-03T19:42:46.375 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 58 16 87249 87233 93.9913 50.9375 0.0197068 0.0106339 2023-06-03T19:42:46.685 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -e sha1=8e24bcf462a1d94c2e12aa4e26419634883a1e88 -- bash -c 'while ! ceph mon versions | grep $sha1 | egrep '"'"': [23]'"'"' ; do sleep 2 ; done' 2023-06-03T19:42:47.375 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 59 16 88094 88078 93.2932 52.8125 0.00696349 0.0107121 2023-06-03T19:42:47.781 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:42:47 smithi089 bash[33173]: cluster 2023-06-03T19:42:45.989293+0000 mgr.y (mgr.34104) 60 : cluster [DBG] pgmap v50: 89 pgs: 89 active+clean; 5.1 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 52 MiB/s wr, 838 op/s 2023-06-03T19:42:47.781 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:42:47 smithi089 bash[33173]: audit 2023-06-03T19:42:46.456451+0000 mon.a (mon.0) 52 : audit [INF] from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:42:47.781 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:42:47 smithi089 bash[33173]: audit 2023-06-03T19:42:46.479549+0000 mon.a (mon.0) 53 : audit [INF] from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:42:47.781 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:42:47 smithi089 bash[33173]: audit 2023-06-03T19:42:46.809080+0000 mon.a (mon.0) 54 : audit [INF] from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:42:47.781 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:42:47 smithi089 bash[33173]: audit 2023-06-03T19:42:46.823468+0000 mon.a (mon.0) 55 : audit [INF] from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:42:47.782 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:42:47 smithi089 bash[33173]: audit 2023-06-03T19:42:47.408124+0000 mon.c (mon.2) 114 : audit [DBG] from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-03T19:42:47.782 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:42:47 smithi089 bash[33173]: audit 2023-06-03T19:42:47.409694+0000 mon.c (mon.2) 115 : audit [INF] from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-03T19:42:47.782 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:42:47 smithi089 bash[33173]: audit 2023-06-03T19:42:47.422393+0000 mon.a (mon.0) 56 : audit [INF] from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:42:47.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:42:47 smithi164 bash[26855]: cluster 2023-06-03T19:42:45.989293+0000 mgr.y (mgr.34104) 60 : cluster [DBG] pgmap v50: 89 pgs: 89 active+clean; 5.1 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 52 MiB/s wr, 838 op/s 2023-06-03T19:42:47.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:42:47 smithi164 bash[26855]: audit 2023-06-03T19:42:46.456451+0000 mon.a (mon.0) 52 : audit [INF] from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:42:47.836 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:42:47 smithi164 bash[26855]: audit 2023-06-03T19:42:46.479549+0000 mon.a (mon.0) 53 : audit [INF] from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:42:47.836 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:42:47 smithi164 bash[26855]: audit 2023-06-03T19:42:46.809080+0000 mon.a (mon.0) 54 : audit [INF] from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:42:47.836 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:42:47 smithi164 bash[26855]: audit 2023-06-03T19:42:46.823468+0000 mon.a (mon.0) 55 : audit [INF] from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:42:47.836 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:42:47 smithi164 bash[26855]: audit 2023-06-03T19:42:47.408124+0000 mon.c (mon.2) 114 : audit [DBG] from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-03T19:42:47.836 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:42:47 smithi164 bash[26855]: audit 2023-06-03T19:42:47.409694+0000 mon.c (mon.2) 115 : audit [INF] from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-03T19:42:47.836 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:42:47 smithi164 bash[26855]: audit 2023-06-03T19:42:47.422393+0000 mon.a (mon.0) 56 : audit [INF] from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:42:48.375 INFO:tasks.radosbench.radosbench.0.smithi089.stdout:2023-06-03T19:42:48.369241+0000 min lat: 0.00131693 max lat: 0.110456 avg lat: 0.0107814 2023-06-03T19:42:48.375 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-06-03T19:42:48.375 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 60 16 89021 89005 92.7039 57.9375 0.0144728 0.0107814 2023-06-03T19:42:48.781 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:42:48 smithi089 bash[33173]: audit 2023-06-03T19:42:47.516975+0000 mon.c (mon.2) 116 : audit [DBG] from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-03T19:42:48.781 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:42:48 smithi089 bash[33173]: audit 2023-06-03T19:42:47.524491+0000 mon.c (mon.2) 117 : audit [DBG] from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-06-03T19:42:48.781 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:42:48 smithi089 bash[33173]: audit 2023-06-03T19:42:47.527332+0000 mon.c (mon.2) 118 : audit [DBG] from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2023-06-03T19:42:48.781 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:42:48 smithi089 bash[33173]: audit 2023-06-03T19:42:47.528693+0000 mon.c (mon.2) 119 : audit [DBG] from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["c"]}]: dispatch 2023-06-03T19:42:48.781 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:42:48 smithi089 bash[33173]: cephadm 2023-06-03T19:42:47.530094+0000 mgr.y (mgr.34104) 61 : cephadm [INF] Upgrade: It appears safe to stop mon.c 2023-06-03T19:42:48.835 INFO:journalctl@ceph.prometheus.a.smithi164.stdout:Jun 03 19:42:48 smithi164 bash[55188]: ts=2023-06-03T19:42:48.661Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.13-257-gd8c5d349 (d8c5d34975dce1c5eb0aa3a7979a4d9b9a99d1ec) pacific (stable)\", cluster_addr=\"172.21.15.89\", device_class=\"ssd\", hostname=\"smithi089\", instance=\"smithi164.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.89\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.13-257-gd8c5d349 (d8c5d34975dce1c5eb0aa3a7979a4d9b9a99d1ec) pacific (stable)\", cluster_addr=\"172.21.15.89\", device_class=\"ssd\", hostname=\"smithi089\", instance=\"smithi089:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.89\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-06-03T19:42:48.835 INFO:journalctl@ceph.prometheus.a.smithi164.stdout:Jun 03 19:42:48 smithi164 bash[55188]: ts=2023-06-03T19:42:48.662Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephPGImbalance\nexpr: abs(((ceph_osd_numpg > 0) - on(job) group_left() avg by(job) (ceph_osd_numpg\n > 0)) / on(job) group_left() avg by(job) (ceph_osd_numpg > 0)) * on(ceph_daemon)\n group_left(hostname) ceph_osd_metadata > 0.3\nfor: 5m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.5\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} deviates by\n more than 30% from average PG count.\n summary: PGs are not balanced across OSDs\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.13-257-gd8c5d349 (d8c5d34975dce1c5eb0aa3a7979a4d9b9a99d1ec) pacific (stable)\", cluster_addr=\"172.21.15.89\", device_class=\"ssd\", hostname=\"smithi089\", instance=\"smithi164.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.89\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.13-257-gd8c5d349 (d8c5d34975dce1c5eb0aa3a7979a4d9b9a99d1ec) pacific (stable)\", cluster_addr=\"172.21.15.89\", device_class=\"ssd\", hostname=\"smithi089\", instance=\"smithi089:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.89\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-06-03T19:42:48.836 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:42:48 smithi164 bash[26855]: audit 2023-06-03T19:42:47.516975+0000 mon.c (mon.2) 116 : audit [DBG] from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-03T19:42:48.836 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:42:48 smithi164 bash[26855]: audit 2023-06-03T19:42:47.524491+0000 mon.c (mon.2) 117 : audit [DBG] from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-06-03T19:42:48.836 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:42:48 smithi164 bash[26855]: audit 2023-06-03T19:42:47.527332+0000 mon.c (mon.2) 118 : audit [DBG] from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2023-06-03T19:42:48.836 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:42:48 smithi164 bash[26855]: audit 2023-06-03T19:42:47.528693+0000 mon.c (mon.2) 119 : audit [DBG] from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["c"]}]: dispatch 2023-06-03T19:42:48.836 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:42:48 smithi164 bash[26855]: cephadm 2023-06-03T19:42:47.530094+0000 mgr.y (mgr.34104) 61 : cephadm [INF] Upgrade: It appears safe to stop mon.c 2023-06-03T19:42:49.375 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 61 16 89834 89818 92.0171 50.8125 0.00876828 0.0108621 2023-06-03T19:42:49.747 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:42:49 smithi089 bash[33173]: cluster 2023-06-03T19:42:47.989793+0000 mgr.y (mgr.34104) 62 : cluster [DBG] pgmap v51: 89 pgs: 89 active+clean; 5.2 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 55 MiB/s wr, 886 op/s 2023-06-03T19:42:49.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:42:49 smithi164 bash[26855]: cluster 2023-06-03T19:42:47.989793+0000 mgr.y (mgr.34104) 62 : cluster [DBG] pgmap v51: 89 pgs: 89 active+clean; 5.2 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 55 MiB/s wr, 886 op/s 2023-06-03T19:42:50.375 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 62 16 90758 90742 91.4643 57.75 0.0170267 0.0109277 2023-06-03T19:42:51.375 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 63 16 91746 91730 90.9925 61.75 0.02538 0.0109842 2023-06-03T19:42:51.780 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:42:51 smithi089 bash[33173]: cluster 2023-06-03T19:42:49.990588+0000 mgr.y (mgr.34104) 63 : cluster [DBG] pgmap v52: 89 pgs: 89 active+clean; 5.4 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 59 MiB/s wr, 950 op/s 2023-06-03T19:42:51.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:42:51 smithi164 bash[26855]: cluster 2023-06-03T19:42:49.990588+0000 mgr.y (mgr.34104) 63 : cluster [DBG] pgmap v52: 89 pgs: 89 active+clean; 5.4 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 59 MiB/s wr, 950 op/s 2023-06-03T19:42:52.375 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 64 16 92697 92681 90.4994 59.4375 0.00857552 0.0110422 2023-06-03T19:42:52.780 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:42:52 smithi089 bash[33173]: audit 2023-06-03T19:42:52.062635+0000 mon.c (mon.2) 120 : audit [DBG] from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T19:42:52.781 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:42:52 smithi089 bash[33173]: audit 2023-06-03T19:42:52.296547+0000 mon.a (mon.0) 57 : audit [INF] from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:42:52.781 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:42:52 smithi089 bash[33173]: audit 2023-06-03T19:42:52.299361+0000 mon.c (mon.2) 121 : audit [INF] from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-06-03T19:42:52.781 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:42:52 smithi089 bash[33173]: audit 2023-06-03T19:42:52.305406+0000 mon.a (mon.0) 58 : audit [DBG] from='mgr.34104 ' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-06-03T19:42:52.781 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:42:52 smithi089 bash[33173]: audit 2023-06-03T19:42:52.310221+0000 mon.c (mon.2) 122 : audit [DBG] from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-03T19:42:52.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:42:52 smithi164 bash[26855]: audit 2023-06-03T19:42:52.062635+0000 mon.c (mon.2) 120 : audit [DBG] from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T19:42:52.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:42:52 smithi164 bash[26855]: audit 2023-06-03T19:42:52.296547+0000 mon.a (mon.0) 57 : audit [INF] from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:42:52.836 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:42:52 smithi164 bash[26855]: audit 2023-06-03T19:42:52.299361+0000 mon.c (mon.2) 121 : audit [INF] from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-06-03T19:42:52.836 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:42:52 smithi164 bash[26855]: audit 2023-06-03T19:42:52.305406+0000 mon.a (mon.0) 58 : audit [DBG] from='mgr.34104 ' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-06-03T19:42:52.836 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:42:52 smithi164 bash[26855]: audit 2023-06-03T19:42:52.310221+0000 mon.c (mon.2) 122 : audit [DBG] from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-03T19:42:53.375 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 65 16 93600 93584 89.9753 56.4375 0.0075796 0.0111085 2023-06-03T19:42:53.780 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:42:53 smithi089 bash[33173]: cluster 2023-06-03T19:42:51.991016+0000 mgr.y (mgr.34104) 64 : cluster [DBG] pgmap v53: 89 pgs: 89 active+clean; 5.4 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 49 MiB/s wr, 785 op/s 2023-06-03T19:42:53.781 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:42:53 smithi089 bash[33173]: cephadm 2023-06-03T19:42:52.279843+0000 mgr.y (mgr.34104) 65 : cephadm [INF] Upgrade: Updating mon.c 2023-06-03T19:42:53.781 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:42:53 smithi089 bash[33173]: cephadm 2023-06-03T19:42:52.311988+0000 mgr.y (mgr.34104) 66 : cephadm [INF] Deploying daemon mon.c on smithi089 2023-06-03T19:42:53.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:42:53 smithi164 bash[26855]: cluster 2023-06-03T19:42:51.991016+0000 mgr.y (mgr.34104) 64 : cluster [DBG] pgmap v53: 89 pgs: 89 active+clean; 5.4 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 49 MiB/s wr, 785 op/s 2023-06-03T19:42:53.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:42:53 smithi164 bash[26855]: cephadm 2023-06-03T19:42:52.279843+0000 mgr.y (mgr.34104) 65 : cephadm [INF] Upgrade: Updating mon.c 2023-06-03T19:42:53.836 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:42:53 smithi164 bash[26855]: cephadm 2023-06-03T19:42:52.311988+0000 mgr.y (mgr.34104) 66 : cephadm [INF] Deploying daemon mon.c on smithi089 2023-06-03T19:42:54.375 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 66 16 94487 94471 89.4519 55.4375 0.0256526 0.0111734 2023-06-03T19:42:54.805 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:42:54 smithi089 bash[33173]: cluster 2023-06-03T19:42:53.509411+0000 mon.a (mon.0) 59 : cluster [DBG] mgrmap e38: y(active, since 91s), standbys: x 2023-06-03T19:42:54.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:42:54 smithi164 bash[26855]: cluster 2023-06-03T19:42:53.509411+0000 mon.a (mon.0) 59 : cluster [DBG] mgrmap e38: y(active, since 91s), standbys: x 2023-06-03T19:42:55.376 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 67 16 95312 95296 88.8864 51.5625 0.0118826 0.0112448 2023-06-03T19:42:55.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:42:55 smithi164 bash[26855]: cluster 2023-06-03T19:42:53.991894+0000 mgr.y (mgr.34104) 67 : cluster [DBG] pgmap v54: 89 pgs: 89 active+clean; 5.7 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 73 MiB/s wr, 1.17k op/s 2023-06-03T19:42:55.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:42:55 smithi164 bash[26855]: audit 2023-06-03T19:42:55.065414+0000 mon.b (mon.1) 92 : audit [DBG] from='client.? 172.21.15.89:0/1498143111' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-03T19:42:56.010 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:42:55 smithi089 bash[33173]: cluster 2023-06-03T19:42:53.991894+0000 mgr.y (mgr.34104) 67 : cluster [DBG] pgmap v54: 89 pgs: 89 active+clean; 5.7 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 73 MiB/s wr, 1.17k op/s 2023-06-03T19:42:56.010 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:42:55 smithi089 bash[33173]: audit 2023-06-03T19:42:55.065414+0000 mon.b (mon.1) 92 : audit [DBG] from='client.? 172.21.15.89:0/1498143111' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-03T19:42:56.011 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:42:56 smithi089 systemd[1]: Stopping Ceph mon.c for f1db3478-0243-11ee-9b1d-001a4aab830c... 2023-06-03T19:42:56.281 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 19:42:56 smithi089 bash[71163]: [03/Jun/2023:19:42:56] ENGINE Bus STOPPING 2023-06-03T19:42:56.281 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 19:42:56 smithi089 bash[71163]: [03/Jun/2023:19:42:56] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2023-06-03T19:42:56.281 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 19:42:56 smithi089 bash[71163]: [03/Jun/2023:19:42:56] ENGINE Bus STOPPED 2023-06-03T19:42:56.281 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 19:42:56 smithi089 bash[71163]: [03/Jun/2023:19:42:56] ENGINE Bus STARTING 2023-06-03T19:42:56.281 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 19:42:56 smithi089 bash[71163]: ::ffff:172.21.15.164 - - [03/Jun/2023:19:42:56] "GET /metrics HTTP/1.1" 200 190051 "" "Prometheus/2.33.4" 2023-06-03T19:42:56.282 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:42:56 smithi089 bash[33173]: debug 2023-06-03T19:42:56.172+0000 7f68d00b7700 -1 received signal: Terminated from /sbin/docker-init -- /usr/bin/ceph-mon -n mon.c -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-stderr=true (PID: 1) UID: 0 2023-06-03T19:42:56.282 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:42:56 smithi089 bash[33173]: debug 2023-06-03T19:42:56.172+0000 7f68d00b7700 -1 mon.c@2(peon) e3 *** Got Signal Terminated *** 2023-06-03T19:42:56.376 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 68 16 96140 96124 88.3402 51.75 0.0150219 0.0113148 2023-06-03T19:42:56.780 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 19:42:56 smithi089 bash[71163]: [03/Jun/2023:19:42:56] ENGINE Serving on http://:::9283 2023-06-03T19:42:56.781 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 19:42:56 smithi089 bash[71163]: [03/Jun/2023:19:42:56] ENGINE Bus STARTED 2023-06-03T19:42:57.376 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 69 16 96965 96949 87.8068 51.5625 0.0432691 0.0113821 2023-06-03T19:42:57.530 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:42:57 smithi089 bash[78304]: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-mon-c 2023-06-03T19:42:57.531 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:42:57 smithi089 systemd[1]: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c@mon.c.service: Succeeded. 2023-06-03T19:42:57.531 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:42:57 smithi089 systemd[1]: Stopped Ceph mon.c for f1db3478-0243-11ee-9b1d-001a4aab830c. 2023-06-03T19:42:58.030 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:42:57 smithi089 systemd[1]: Started Ceph mon.c for f1db3478-0243-11ee-9b1d-001a4aab830c. 2023-06-03T19:42:58.376 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 70 16 97770 97754 87.2711 50.3125 0.0136829 0.0114533 2023-06-03T19:42:59.085 INFO:journalctl@ceph.prometheus.a.smithi164.stdout:Jun 03 19:42:58 smithi164 bash[55188]: ts=2023-06-03T19:42:58.660Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.13-257-gd8c5d349 (d8c5d34975dce1c5eb0aa3a7979a4d9b9a99d1ec) pacific (stable)\", cluster_addr=\"172.21.15.89\", device_class=\"ssd\", hostname=\"smithi089\", instance=\"smithi164.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.89\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.13-257-gd8c5d349 (d8c5d34975dce1c5eb0aa3a7979a4d9b9a99d1ec) pacific (stable)\", cluster_addr=\"172.21.15.89\", device_class=\"ssd\", hostname=\"smithi089\", instance=\"smithi089:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.89\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-06-03T19:42:59.085 INFO:journalctl@ceph.prometheus.a.smithi164.stdout:Jun 03 19:42:58 smithi164 bash[55188]: ts=2023-06-03T19:42:58.661Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephPGImbalance\nexpr: abs(((ceph_osd_numpg > 0) - on(job) group_left() avg by(job) (ceph_osd_numpg\n > 0)) / on(job) group_left() avg by(job) (ceph_osd_numpg > 0)) * on(ceph_daemon)\n group_left(hostname) ceph_osd_metadata > 0.3\nfor: 5m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.5\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} deviates by\n more than 30% from average PG count.\n summary: PGs are not balanced across OSDs\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.13-257-gd8c5d349 (d8c5d34975dce1c5eb0aa3a7979a4d9b9a99d1ec) pacific (stable)\", cluster_addr=\"172.21.15.89\", device_class=\"ssd\", hostname=\"smithi089\", instance=\"smithi164.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.89\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.13-257-gd8c5d349 (d8c5d34975dce1c5eb0aa3a7979a4d9b9a99d1ec) pacific (stable)\", cluster_addr=\"172.21.15.89\", device_class=\"ssd\", hostname=\"smithi089\", instance=\"smithi089:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.89\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-06-03T19:42:59.376 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 71 16 98686 98670 86.8482 57.25 0.0105811 0.0115084 2023-06-03T19:42:59.785 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:42:59 smithi089 bash[78452]: debug 2023-06-03T19:42:59.508+0000 7f9a21422c80 0 set uid:gid to 167:167 (ceph:ceph) 2023-06-03T19:42:59.785 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:42:59 smithi089 bash[78452]: debug 2023-06-03T19:42:59.508+0000 7f9a21422c80 0 ceph version 18.0.0-3532-g8e24bcf4 (8e24bcf462a1d94c2e12aa4e26419634883a1e88) reef (rc), process ceph-mon, pid 7 2023-06-03T19:42:59.785 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:42:59 smithi089 bash[78452]: debug 2023-06-03T19:42:59.508+0000 7f9a21422c80 0 pidfile_write: ignore empty --pid-file 2023-06-03T19:42:59.785 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:42:59 smithi089 bash[78452]: debug 2023-06-03T19:42:59.512+0000 7f9a21422c80 0 load: jerasure load: lrc 2023-06-03T19:42:59.786 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:42:59 smithi089 bash[78452]: debug 2023-06-03T19:42:59.516+0000 7f9a21422c80 4 rocksdb: RocksDB version: 7.9.2 2023-06-03T19:42:59.786 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:42:59 smithi089 bash[78452]: debug 2023-06-03T19:42:59.516+0000 7f9a21422c80 4 rocksdb: Git sha 0 2023-06-03T19:42:59.786 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:42:59 smithi089 bash[78452]: debug 2023-06-03T19:42:59.516+0000 7f9a21422c80 4 rocksdb: Compile date 2023-06-01 15:16:58 2023-06-03T19:42:59.786 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:42:59 smithi089 bash[78452]: debug 2023-06-03T19:42:59.516+0000 7f9a21422c80 4 rocksdb: DB SUMMARY 2023-06-03T19:42:59.786 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:42:59 smithi089 bash[78452]: debug 2023-06-03T19:42:59.516+0000 7f9a21422c80 4 rocksdb: DB Session ID: AQI6EUQ0185BYRBGXFII 2023-06-03T19:42:59.786 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:42:59 smithi089 bash[78452]: debug 2023-06-03T19:42:59.516+0000 7f9a21422c80 4 rocksdb: CURRENT file: CURRENT 2023-06-03T19:42:59.786 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:42:59 smithi089 bash[78452]: debug 2023-06-03T19:42:59.516+0000 7f9a21422c80 4 rocksdb: IDENTITY file: IDENTITY 2023-06-03T19:42:59.787 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:42:59 smithi089 bash[78452]: debug 2023-06-03T19:42:59.516+0000 7f9a21422c80 4 rocksdb: MANIFEST file: MANIFEST-000005 size: 1460 Bytes 2023-06-03T19:42:59.787 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:42:59 smithi089 bash[78452]: debug 2023-06-03T19:42:59.516+0000 7f9a21422c80 4 rocksdb: SST files in /var/lib/ceph/mon/ceph-c/store.db dir, Total Num: 1, files: 000029.sst 2023-06-03T19:42:59.787 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:42:59 smithi089 bash[78452]: debug 2023-06-03T19:42:59.516+0000 7f9a21422c80 4 rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-c/store.db: 000027.log size: 1795662 ; 2023-06-03T19:42:59.787 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:42:59 smithi089 bash[78452]: debug 2023-06-03T19:42:59.516+0000 7f9a21422c80 4 rocksdb: Options.error_if_exists: 0 2023-06-03T19:42:59.787 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:42:59 smithi089 bash[78452]: debug 2023-06-03T19:42:59.516+0000 7f9a21422c80 4 rocksdb: Options.create_if_missing: 0 2023-06-03T19:42:59.787 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:42:59 smithi089 bash[78452]: debug 2023-06-03T19:42:59.516+0000 7f9a21422c80 4 rocksdb: Options.paranoid_checks: 1 2023-06-03T19:42:59.787 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:42:59 smithi089 bash[78452]: debug 2023-06-03T19:42:59.516+0000 7f9a21422c80 4 rocksdb: Options.flush_verify_memtable_count: 1 2023-06-03T19:42:59.787 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:42:59 smithi089 bash[78452]: debug 2023-06-03T19:42:59.516+0000 7f9a21422c80 4 rocksdb: Options.track_and_verify_wals_in_manifest: 0 2023-06-03T19:42:59.788 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:42:59 smithi089 bash[78452]: debug 2023-06-03T19:42:59.516+0000 7f9a21422c80 4 rocksdb: Options.verify_sst_unique_id_in_manifest: 1 2023-06-03T19:42:59.788 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:42:59 smithi089 bash[78452]: debug 2023-06-03T19:42:59.516+0000 7f9a21422c80 4 rocksdb: Options.env: 0x56133852a720 2023-06-03T19:42:59.788 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:42:59 smithi089 bash[78452]: debug 2023-06-03T19:42:59.516+0000 7f9a21422c80 4 rocksdb: Options.fs: PosixFileSystem 2023-06-03T19:42:59.788 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:42:59 smithi089 bash[78452]: debug 2023-06-03T19:42:59.516+0000 7f9a21422c80 4 rocksdb: Options.info_log: 0x56133b821460 2023-06-03T19:42:59.788 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:42:59 smithi089 bash[78452]: debug 2023-06-03T19:42:59.516+0000 7f9a21422c80 4 rocksdb: Options.max_file_opening_threads: 16 2023-06-03T19:42:59.788 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:42:59 smithi089 bash[78452]: debug 2023-06-03T19:42:59.516+0000 7f9a21422c80 4 rocksdb: Options.statistics: (nil) 2023-06-03T19:42:59.788 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:42:59 smithi089 bash[78452]: debug 2023-06-03T19:42:59.516+0000 7f9a21422c80 4 rocksdb: Options.use_fsync: 0 2023-06-03T19:42:59.789 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:42:59 smithi089 bash[78452]: debug 2023-06-03T19:42:59.516+0000 7f9a21422c80 4 rocksdb: Options.max_log_file_size: 0 2023-06-03T19:42:59.789 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:42:59 smithi089 bash[78452]: debug 2023-06-03T19:42:59.516+0000 7f9a21422c80 4 rocksdb: Options.max_manifest_file_size: 1073741824 2023-06-03T19:42:59.789 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:42:59 smithi089 bash[78452]: debug 2023-06-03T19:42:59.516+0000 7f9a21422c80 4 rocksdb: Options.log_file_time_to_roll: 0 2023-06-03T19:42:59.789 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:42:59 smithi089 bash[78452]: debug 2023-06-03T19:42:59.516+0000 7f9a21422c80 4 rocksdb: Options.keep_log_file_num: 1000 2023-06-03T19:42:59.789 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:42:59 smithi089 bash[78452]: debug 2023-06-03T19:42:59.516+0000 7f9a21422c80 4 rocksdb: Options.recycle_log_file_num: 0 2023-06-03T19:42:59.789 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:42:59 smithi089 bash[78452]: debug 2023-06-03T19:42:59.516+0000 7f9a21422c80 4 rocksdb: Options.allow_fallocate: 1 2023-06-03T19:42:59.789 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:42:59 smithi089 bash[78452]: debug 2023-06-03T19:42:59.516+0000 7f9a21422c80 4 rocksdb: Options.allow_mmap_reads: 0 2023-06-03T19:42:59.789 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:42:59 smithi089 bash[78452]: debug 2023-06-03T19:42:59.516+0000 7f9a21422c80 4 rocksdb: Options.allow_mmap_writes: 0 2023-06-03T19:42:59.790 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:42:59 smithi089 bash[78452]: debug 2023-06-03T19:42:59.516+0000 7f9a21422c80 4 rocksdb: Options.use_direct_reads: 0 2023-06-03T19:42:59.790 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:42:59 smithi089 bash[78452]: debug 2023-06-03T19:42:59.516+0000 7f9a21422c80 4 rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2023-06-03T19:42:59.790 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:42:59 smithi089 bash[78452]: debug 2023-06-03T19:42:59.516+0000 7f9a21422c80 4 rocksdb: Options.create_missing_column_families: 0 2023-06-03T19:42:59.790 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:42:59 smithi089 bash[78452]: debug 2023-06-03T19:42:59.516+0000 7f9a21422c80 4 rocksdb: Options.db_log_dir: 2023-06-03T19:42:59.791 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:42:59 smithi089 bash[78452]: debug 2023-06-03T19:42:59.516+0000 7f9a21422c80 4 rocksdb: Options.wal_dir: 2023-06-03T19:42:59.791 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:42:59 smithi089 bash[78452]: debug 2023-06-03T19:42:59.516+0000 7f9a21422c80 4 rocksdb: Options.table_cache_numshardbits: 6 2023-06-03T19:42:59.791 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:42:59 smithi089 bash[78452]: debug 2023-06-03T19:42:59.516+0000 7f9a21422c80 4 rocksdb: Options.WAL_ttl_seconds: 0 2023-06-03T19:42:59.791 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:42:59 smithi089 bash[78452]: debug 2023-06-03T19:42:59.516+0000 7f9a21422c80 4 rocksdb: Options.WAL_size_limit_MB: 0 2023-06-03T19:42:59.792 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:42:59 smithi089 bash[78452]: debug 2023-06-03T19:42:59.516+0000 7f9a21422c80 4 rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2023-06-03T19:42:59.792 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:42:59 smithi089 bash[78452]: debug 2023-06-03T19:42:59.516+0000 7f9a21422c80 4 rocksdb: Options.manifest_preallocation_size: 4194304 2023-06-03T19:42:59.792 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:42:59 smithi089 bash[78452]: debug 2023-06-03T19:42:59.516+0000 7f9a21422c80 4 rocksdb: Options.is_fd_close_on_exec: 1 2023-06-03T19:42:59.792 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:42:59 smithi089 bash[78452]: debug 2023-06-03T19:42:59.516+0000 7f9a21422c80 4 rocksdb: Options.advise_random_on_open: 1 2023-06-03T19:42:59.792 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:42:59 smithi089 bash[78452]: debug 2023-06-03T19:42:59.516+0000 7f9a21422c80 4 rocksdb: Options.db_write_buffer_size: 0 2023-06-03T19:42:59.792 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:42:59 smithi089 bash[78452]: debug 2023-06-03T19:42:59.516+0000 7f9a21422c80 4 rocksdb: Options.write_buffer_manager: 0x56133aaaec80 2023-06-03T19:42:59.792 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:42:59 smithi089 bash[78452]: debug 2023-06-03T19:42:59.516+0000 7f9a21422c80 4 rocksdb: Options.access_hint_on_compaction_start: 1 2023-06-03T19:42:59.793 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:42:59 smithi089 bash[78452]: debug 2023-06-03T19:42:59.516+0000 7f9a21422c80 4 rocksdb: Options.random_access_max_buffer_size: 1048576 2023-06-03T19:42:59.793 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:42:59 smithi089 bash[78452]: debug 2023-06-03T19:42:59.516+0000 7f9a21422c80 4 rocksdb: Options.use_adaptive_mutex: 0 2023-06-03T19:42:59.793 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:42:59 smithi089 bash[78452]: debug 2023-06-03T19:42:59.516+0000 7f9a21422c80 4 rocksdb: Options.rate_limiter: (nil) 2023-06-03T19:42:59.793 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:42:59 smithi089 bash[78452]: debug 2023-06-03T19:42:59.516+0000 7f9a21422c80 4 rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2023-06-03T19:42:59.794 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:42:59 smithi089 bash[78452]: debug 2023-06-03T19:42:59.516+0000 7f9a21422c80 4 rocksdb: Options.wal_recovery_mode: 2 2023-06-03T19:42:59.794 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:42:59 smithi089 bash[78452]: debug 2023-06-03T19:42:59.516+0000 7f9a21422c80 4 rocksdb: Options.enable_thread_tracking: 0 2023-06-03T19:42:59.794 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:42:59 smithi089 bash[78452]: debug 2023-06-03T19:42:59.516+0000 7f9a21422c80 4 rocksdb: Options.enable_pipelined_write: 0 2023-06-03T19:42:59.794 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:42:59 smithi089 bash[78452]: debug 2023-06-03T19:42:59.516+0000 7f9a21422c80 4 rocksdb: Options.unordered_write: 0 2023-06-03T19:42:59.795 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:42:59 smithi089 bash[78452]: debug 2023-06-03T19:42:59.516+0000 7f9a21422c80 4 rocksdb: Options.allow_concurrent_memtable_write: 1 2023-06-03T19:42:59.795 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:42:59 smithi089 bash[78452]: debug 2023-06-03T19:42:59.516+0000 7f9a21422c80 4 rocksdb: Options.enable_write_thread_adaptive_yield: 1 2023-06-03T19:42:59.795 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:42:59 smithi089 bash[78452]: debug 2023-06-03T19:42:59.516+0000 7f9a21422c80 4 rocksdb: Options.write_thread_max_yield_usec: 100 2023-06-03T19:42:59.795 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:42:59 smithi089 bash[78452]: debug 2023-06-03T19:42:59.516+0000 7f9a21422c80 4 rocksdb: Options.write_thread_slow_yield_usec: 3 2023-06-03T19:42:59.795 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:42:59 smithi089 bash[78452]: debug 2023-06-03T19:42:59.516+0000 7f9a21422c80 4 rocksdb: Options.row_cache: None 2023-06-03T19:42:59.795 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:42:59 smithi089 bash[78452]: debug 2023-06-03T19:42:59.516+0000 7f9a21422c80 4 rocksdb: Options.wal_filter: None 2023-06-03T19:42:59.795 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:42:59 smithi089 bash[78452]: debug 2023-06-03T19:42:59.516+0000 7f9a21422c80 4 rocksdb: Options.avoid_flush_during_recovery: 0 2023-06-03T19:42:59.796 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:42:59 smithi089 bash[78452]: debug 2023-06-03T19:42:59.516+0000 7f9a21422c80 4 rocksdb: Options.allow_ingest_behind: 0 2023-06-03T19:42:59.796 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:42:59 smithi089 bash[78452]: debug 2023-06-03T19:42:59.516+0000 7f9a21422c80 4 rocksdb: Options.two_write_queues: 0 2023-06-03T19:42:59.796 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:42:59 smithi089 bash[78452]: debug 2023-06-03T19:42:59.516+0000 7f9a21422c80 4 rocksdb: Options.manual_wal_flush: 0 2023-06-03T19:42:59.796 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:42:59 smithi089 bash[78452]: debug 2023-06-03T19:42:59.516+0000 7f9a21422c80 4 rocksdb: Options.wal_compression: 0 2023-06-03T19:42:59.796 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:42:59 smithi089 bash[78452]: debug 2023-06-03T19:42:59.516+0000 7f9a21422c80 4 rocksdb: Options.atomic_flush: 0 2023-06-03T19:42:59.796 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:42:59 smithi089 bash[78452]: debug 2023-06-03T19:42:59.516+0000 7f9a21422c80 4 rocksdb: Options.avoid_unnecessary_blocking_io: 0 2023-06-03T19:42:59.796 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:42:59 smithi089 bash[78452]: debug 2023-06-03T19:42:59.516+0000 7f9a21422c80 4 rocksdb: Options.persist_stats_to_disk: 0 2023-06-03T19:42:59.796 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:42:59 smithi089 bash[78452]: debug 2023-06-03T19:42:59.516+0000 7f9a21422c80 4 rocksdb: Options.write_dbid_to_manifest: 0 2023-06-03T19:42:59.797 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:42:59 smithi089 bash[78452]: debug 2023-06-03T19:42:59.516+0000 7f9a21422c80 4 rocksdb: Options.log_readahead_size: 0 2023-06-03T19:42:59.797 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:42:59 smithi089 bash[78452]: debug 2023-06-03T19:42:59.516+0000 7f9a21422c80 4 rocksdb: Options.file_checksum_gen_factory: Unknown 2023-06-03T19:42:59.797 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:42:59 smithi089 bash[78452]: debug 2023-06-03T19:42:59.516+0000 7f9a21422c80 4 rocksdb: Options.best_efforts_recovery: 0 2023-06-03T19:42:59.797 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:42:59 smithi089 bash[78452]: debug 2023-06-03T19:42:59.516+0000 7f9a21422c80 4 rocksdb: Options.max_bgerror_resume_count: 2147483647 2023-06-03T19:42:59.797 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:42:59 smithi089 bash[78452]: debug 2023-06-03T19:42:59.516+0000 7f9a21422c80 4 rocksdb: Options.bgerror_resume_retry_interval: 1000000 2023-06-03T19:42:59.797 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:42:59 smithi089 bash[78452]: debug 2023-06-03T19:42:59.516+0000 7f9a21422c80 4 rocksdb: Options.allow_data_in_errors: 0 2023-06-03T19:42:59.797 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:42:59 smithi089 bash[78452]: debug 2023-06-03T19:42:59.516+0000 7f9a21422c80 4 rocksdb: Options.db_host_id: __hostname__ 2023-06-03T19:42:59.798 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:42:59 smithi089 bash[78452]: debug 2023-06-03T19:42:59.516+0000 7f9a21422c80 4 rocksdb: Options.enforce_single_del_contracts: true 2023-06-03T19:42:59.798 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:42:59 smithi089 bash[78452]: debug 2023-06-03T19:42:59.516+0000 7f9a21422c80 4 rocksdb: Options.max_background_jobs: 2 2023-06-03T19:42:59.798 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:42:59 smithi089 bash[78452]: debug 2023-06-03T19:42:59.516+0000 7f9a21422c80 4 rocksdb: Options.max_background_compactions: -1 2023-06-03T19:42:59.798 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:42:59 smithi089 bash[78452]: debug 2023-06-03T19:42:59.516+0000 7f9a21422c80 4 rocksdb: Options.max_subcompactions: 1 2023-06-03T19:42:59.798 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:42:59 smithi089 bash[78452]: debug 2023-06-03T19:42:59.516+0000 7f9a21422c80 4 rocksdb: Options.avoid_flush_during_shutdown: 0 2023-06-03T19:42:59.798 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:42:59 smithi089 bash[78452]: debug 2023-06-03T19:42:59.516+0000 7f9a21422c80 4 rocksdb: Options.writable_file_max_buffer_size: 1048576 2023-06-03T19:42:59.798 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:42:59 smithi089 bash[78452]: debug 2023-06-03T19:42:59.516+0000 7f9a21422c80 4 rocksdb: Options.delayed_write_rate : 16777216 2023-06-03T19:42:59.798 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:42:59 smithi089 bash[78452]: debug 2023-06-03T19:42:59.516+0000 7f9a21422c80 4 rocksdb: Options.max_total_wal_size: 0 2023-06-03T19:42:59.799 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:42:59 smithi089 bash[78452]: debug 2023-06-03T19:42:59.516+0000 7f9a21422c80 4 rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2023-06-03T19:42:59.799 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:42:59 smithi089 bash[78452]: debug 2023-06-03T19:42:59.516+0000 7f9a21422c80 4 rocksdb: Options.stats_dump_period_sec: 600 2023-06-03T19:42:59.799 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:42:59 smithi089 bash[78452]: debug 2023-06-03T19:42:59.516+0000 7f9a21422c80 4 rocksdb: Options.stats_persist_period_sec: 600 2023-06-03T19:42:59.799 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:42:59 smithi089 bash[78452]: debug 2023-06-03T19:42:59.516+0000 7f9a21422c80 4 rocksdb: Options.stats_history_buffer_size: 1048576 2023-06-03T19:42:59.799 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:42:59 smithi089 bash[78452]: debug 2023-06-03T19:42:59.516+0000 7f9a21422c80 4 rocksdb: Options.max_open_files: -1 2023-06-03T19:42:59.799 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:42:59 smithi089 bash[78452]: debug 2023-06-03T19:42:59.516+0000 7f9a21422c80 4 rocksdb: Options.bytes_per_sync: 0 2023-06-03T19:42:59.799 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:42:59 smithi089 bash[78452]: debug 2023-06-03T19:42:59.516+0000 7f9a21422c80 4 rocksdb: Options.wal_bytes_per_sync: 0 2023-06-03T19:42:59.800 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:42:59 smithi089 bash[78452]: debug 2023-06-03T19:42:59.516+0000 7f9a21422c80 4 rocksdb: Options.strict_bytes_per_sync: 0 2023-06-03T19:42:59.800 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:42:59 smithi089 bash[78452]: debug 2023-06-03T19:42:59.516+0000 7f9a21422c80 4 rocksdb: Options.compaction_readahead_size: 0 2023-06-03T19:42:59.800 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:42:59 smithi089 bash[78452]: debug 2023-06-03T19:42:59.516+0000 7f9a21422c80 4 rocksdb: Options.max_background_flushes: -1 2023-06-03T19:42:59.800 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:42:59 smithi089 bash[78452]: debug 2023-06-03T19:42:59.516+0000 7f9a21422c80 4 rocksdb: Compression algorithms supported: 2023-06-03T19:42:59.800 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:42:59 smithi089 bash[78452]: debug 2023-06-03T19:42:59.516+0000 7f9a21422c80 4 rocksdb: kZSTDNotFinalCompression supported: 0 2023-06-03T19:42:59.800 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:42:59 smithi089 bash[78452]: debug 2023-06-03T19:42:59.516+0000 7f9a21422c80 4 rocksdb: kZSTD supported: 0 2023-06-03T19:42:59.800 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:42:59 smithi089 bash[78452]: debug 2023-06-03T19:42:59.516+0000 7f9a21422c80 4 rocksdb: kXpressCompression supported: 0 2023-06-03T19:42:59.801 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:42:59 smithi089 bash[78452]: debug 2023-06-03T19:42:59.516+0000 7f9a21422c80 4 rocksdb: kLZ4HCCompression supported: 1 2023-06-03T19:42:59.801 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:42:59 smithi089 bash[78452]: debug 2023-06-03T19:42:59.516+0000 7f9a21422c80 4 rocksdb: kZlibCompression supported: 1 2023-06-03T19:42:59.801 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:42:59 smithi089 bash[78452]: debug 2023-06-03T19:42:59.516+0000 7f9a21422c80 4 rocksdb: kSnappyCompression supported: 1 2023-06-03T19:42:59.801 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:42:59 smithi089 bash[78452]: debug 2023-06-03T19:42:59.516+0000 7f9a21422c80 4 rocksdb: kLZ4Compression supported: 1 2023-06-03T19:42:59.801 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:42:59 smithi089 bash[78452]: debug 2023-06-03T19:42:59.516+0000 7f9a21422c80 4 rocksdb: kBZip2Compression supported: 0 2023-06-03T19:42:59.801 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:42:59 smithi089 bash[78452]: debug 2023-06-03T19:42:59.516+0000 7f9a21422c80 4 rocksdb: Fast CRC32 supported: Supported on x86 2023-06-03T19:42:59.801 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:42:59 smithi089 bash[78452]: debug 2023-06-03T19:42:59.516+0000 7f9a21422c80 4 rocksdb: DMutex implementation: pthread_mutex_t 2023-06-03T19:42:59.801 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:42:59 smithi089 bash[78452]: debug 2023-06-03T19:42:59.516+0000 7f9a21422c80 4 rocksdb: [db/version_set.cc:5527] Recovering from manifest file: /var/lib/ceph/mon/ceph-c/store.db/MANIFEST-000005 2023-06-03T19:42:59.802 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:42:59 smithi089 bash[78452]: debug 2023-06-03T19:42:59.516+0000 7f9a21422c80 4 rocksdb: [db/column_family.cc:630] --------------- Options for column family [default]: 2023-06-03T19:42:59.802 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:42:59 smithi089 bash[78452]: debug 2023-06-03T19:42:59.516+0000 7f9a21422c80 4 rocksdb: Options.comparator: leveldb.BytewiseComparator 2023-06-03T19:42:59.802 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:42:59 smithi089 bash[78452]: debug 2023-06-03T19:42:59.516+0000 7f9a21422c80 4 rocksdb: Options.merge_operator: 2023-06-03T19:42:59.802 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:42:59 smithi089 bash[78452]: debug 2023-06-03T19:42:59.516+0000 7f9a21422c80 4 rocksdb: Options.compaction_filter: None 2023-06-03T19:42:59.802 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:42:59 smithi089 bash[78452]: debug 2023-06-03T19:42:59.516+0000 7f9a21422c80 4 rocksdb: Options.compaction_filter_factory: None 2023-06-03T19:42:59.802 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:42:59 smithi089 bash[78452]: debug 2023-06-03T19:42:59.516+0000 7f9a21422c80 4 rocksdb: Options.sst_partitioner_factory: None 2023-06-03T19:42:59.802 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:42:59 smithi089 bash[78452]: debug 2023-06-03T19:42:59.516+0000 7f9a21422c80 4 rocksdb: Options.memtable_factory: SkipListFactory 2023-06-03T19:42:59.803 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:42:59 smithi089 bash[78452]: debug 2023-06-03T19:42:59.516+0000 7f9a21422c80 4 rocksdb: Options.table_factory: BlockBasedTable 2023-06-03T19:42:59.803 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:42:59 smithi089 bash[78452]: debug 2023-06-03T19:42:59.516+0000 7f9a21422c80 4 rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x56133b821560) 2023-06-03T19:42:59.803 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:42:59 smithi089 bash[78452]: cache_index_and_filter_blocks: 1 2023-06-03T19:42:59.803 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:42:59 smithi089 bash[78452]: cache_index_and_filter_blocks_with_high_priority: 0 2023-06-03T19:42:59.803 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:42:59 smithi089 bash[78452]: pin_l0_filter_and_index_blocks_in_cache: 0 2023-06-03T19:42:59.803 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:42:59 smithi089 bash[78452]: pin_top_level_index_and_filter: 1 2023-06-03T19:42:59.803 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:42:59 smithi089 bash[78452]: index_type: 0 2023-06-03T19:42:59.803 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:42:59 smithi089 bash[78452]: data_block_index_type: 0 2023-06-03T19:42:59.804 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:42:59 smithi089 bash[78452]: index_shortening: 1 2023-06-03T19:42:59.804 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:42:59 smithi089 bash[78452]: data_block_hash_table_util_ratio: 0.750000 2023-06-03T19:42:59.804 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:42:59 smithi089 bash[78452]: checksum: 4 2023-06-03T19:42:59.804 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:42:59 smithi089 bash[78452]: no_block_cache: 0 2023-06-03T19:42:59.804 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:42:59 smithi089 bash[78452]: block_cache: 0x56133ab311f0 2023-06-03T19:42:59.804 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:42:59 smithi089 bash[78452]: block_cache_name: BinnedLRUCache 2023-06-03T19:42:59.804 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:42:59 smithi089 bash[78452]: block_cache_options: 2023-06-03T19:42:59.804 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:42:59 smithi089 bash[78452]: capacity : 536870912 2023-06-03T19:42:59.805 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:42:59 smithi089 bash[78452]: num_shard_bits : 4 2023-06-03T19:42:59.805 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:42:59 smithi089 bash[78452]: strict_capacity_limit : 0 2023-06-03T19:42:59.805 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:42:59 smithi089 bash[78452]: high_pri_pool_ratio: 0.000 2023-06-03T19:42:59.805 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:42:59 smithi089 bash[78452]: block_cache_compressed: (nil) 2023-06-03T19:42:59.805 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:42:59 smithi089 bash[78452]: persistent_cache: (nil) 2023-06-03T19:42:59.805 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:42:59 smithi089 bash[78452]: block_size: 4096 2023-06-03T19:42:59.805 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:42:59 smithi089 bash[78452]: block_size_deviation: 10 2023-06-03T19:42:59.806 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:42:59 smithi089 bash[78452]: block_restart_interval: 16 2023-06-03T19:42:59.806 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:42:59 smithi089 bash[78452]: index_block_restart_interval: 1 2023-06-03T19:42:59.806 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:42:59 smithi089 bash[78452]: metadata_block_size: 4096 2023-06-03T19:42:59.806 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:42:59 smithi089 bash[78452]: partition_filters: 0 2023-06-03T19:42:59.807 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:42:59 smithi089 bash[78452]: use_delta_encoding: 1 2023-06-03T19:42:59.807 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:42:59 smithi089 bash[78452]: filter_policy: bloomfilter 2023-06-03T19:42:59.807 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:42:59 smithi089 bash[78452]: whole_key_filtering: 1 2023-06-03T19:42:59.807 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:42:59 smithi089 bash[78452]: verify_compression: 0 2023-06-03T19:42:59.807 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:42:59 smithi089 bash[78452]: read_amp_bytes_per_bit: 0 2023-06-03T19:42:59.808 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:42:59 smithi089 bash[78452]: format_version: 5 2023-06-03T19:42:59.808 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:42:59 smithi089 bash[78452]: enable_index_compression: 1 2023-06-03T19:42:59.808 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:42:59 smithi089 bash[78452]: block_align: 0 2023-06-03T19:42:59.808 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:42:59 smithi089 bash[78452]: max_auto_readahead_size: 262144 2023-06-03T19:42:59.808 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:42:59 smithi089 bash[78452]: prepopulate_block_cache: 0 2023-06-03T19:42:59.809 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:42:59 smithi089 bash[78452]: initial_auto_readahead_size: 8192 2023-06-03T19:42:59.809 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:42:59 smithi089 bash[78452]: num_file_reads_for_auto_readahead: 2 2023-06-03T19:42:59.809 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:42:59 smithi089 bash[78452]: debug 2023-06-03T19:42:59.516+0000 7f9a21422c80 4 rocksdb: Options.write_buffer_size: 33554432 2023-06-03T19:42:59.809 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:42:59 smithi089 bash[78452]: debug 2023-06-03T19:42:59.516+0000 7f9a21422c80 4 rocksdb: Options.max_write_buffer_number: 2 2023-06-03T19:42:59.809 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:42:59 smithi089 bash[78452]: debug 2023-06-03T19:42:59.516+0000 7f9a21422c80 4 rocksdb: Options.compression: NoCompression 2023-06-03T19:42:59.810 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:42:59 smithi089 bash[78452]: debug 2023-06-03T19:42:59.516+0000 7f9a21422c80 4 rocksdb: Options.bottommost_compression: Disabled 2023-06-03T19:42:59.810 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:42:59 smithi089 bash[78452]: debug 2023-06-03T19:42:59.516+0000 7f9a21422c80 4 rocksdb: Options.prefix_extractor: nullptr 2023-06-03T19:42:59.810 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:42:59 smithi089 bash[78452]: debug 2023-06-03T19:42:59.516+0000 7f9a21422c80 4 rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2023-06-03T19:42:59.810 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:42:59 smithi089 bash[78452]: debug 2023-06-03T19:42:59.516+0000 7f9a21422c80 4 rocksdb: Options.num_levels: 7 2023-06-03T19:42:59.810 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:42:59 smithi089 bash[78452]: debug 2023-06-03T19:42:59.516+0000 7f9a21422c80 4 rocksdb: Options.min_write_buffer_number_to_merge: 1 2023-06-03T19:42:59.811 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:42:59 smithi089 bash[78452]: debug 2023-06-03T19:42:59.516+0000 7f9a21422c80 4 rocksdb: Options.max_write_buffer_number_to_maintain: 0 2023-06-03T19:42:59.811 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:42:59 smithi089 bash[78452]: debug 2023-06-03T19:42:59.516+0000 7f9a21422c80 4 rocksdb: Options.max_write_buffer_size_to_maintain: 0 2023-06-03T19:42:59.811 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:42:59 smithi089 bash[78452]: debug 2023-06-03T19:42:59.516+0000 7f9a21422c80 4 rocksdb: Options.bottommost_compression_opts.window_bits: -14 2023-06-03T19:42:59.811 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:42:59 smithi089 bash[78452]: debug 2023-06-03T19:42:59.516+0000 7f9a21422c80 4 rocksdb: Options.bottommost_compression_opts.level: 32767 2023-06-03T19:42:59.811 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:42:59 smithi089 bash[78452]: debug 2023-06-03T19:42:59.516+0000 7f9a21422c80 4 rocksdb: Options.bottommost_compression_opts.strategy: 0 2023-06-03T19:42:59.812 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:42:59 smithi089 bash[78452]: debug 2023-06-03T19:42:59.516+0000 7f9a21422c80 4 rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2023-06-03T19:42:59.812 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:42:59 smithi089 bash[78452]: debug 2023-06-03T19:42:59.516+0000 7f9a21422c80 4 rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2023-06-03T19:42:59.812 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:42:59 smithi089 bash[78452]: debug 2023-06-03T19:42:59.516+0000 7f9a21422c80 4 rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2023-06-03T19:42:59.812 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:42:59 smithi089 bash[78452]: debug 2023-06-03T19:42:59.516+0000 7f9a21422c80 4 rocksdb: Options.bottommost_compression_opts.enabled: false 2023-06-03T19:42:59.812 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:42:59 smithi089 bash[78452]: debug 2023-06-03T19:42:59.516+0000 7f9a21422c80 4 rocksdb: Options.bottommost_compression_opts.max_dict_buffer_bytes: 0 2023-06-03T19:42:59.813 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:42:59 smithi089 bash[78452]: debug 2023-06-03T19:42:59.516+0000 7f9a21422c80 4 rocksdb: Options.bottommost_compression_opts.use_zstd_dict_trainer: true 2023-06-03T19:42:59.813 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:42:59 smithi089 bash[78452]: debug 2023-06-03T19:42:59.516+0000 7f9a21422c80 4 rocksdb: Options.compression_opts.window_bits: -14 2023-06-03T19:42:59.813 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:42:59 smithi089 bash[78452]: debug 2023-06-03T19:42:59.516+0000 7f9a21422c80 4 rocksdb: Options.compression_opts.level: 32767 2023-06-03T19:42:59.813 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:42:59 smithi089 bash[78452]: debug 2023-06-03T19:42:59.516+0000 7f9a21422c80 4 rocksdb: Options.compression_opts.strategy: 0 2023-06-03T19:42:59.813 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:42:59 smithi089 bash[78452]: debug 2023-06-03T19:42:59.516+0000 7f9a21422c80 4 rocksdb: Options.compression_opts.max_dict_bytes: 0 2023-06-03T19:42:59.814 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:42:59 smithi089 bash[78452]: debug 2023-06-03T19:42:59.516+0000 7f9a21422c80 4 rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2023-06-03T19:42:59.814 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:42:59 smithi089 bash[78452]: debug 2023-06-03T19:42:59.516+0000 7f9a21422c80 4 rocksdb: Options.compression_opts.use_zstd_dict_trainer: true 2023-06-03T19:42:59.814 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:42:59 smithi089 bash[78452]: debug 2023-06-03T19:42:59.516+0000 7f9a21422c80 4 rocksdb: Options.compression_opts.parallel_threads: 1 2023-06-03T19:42:59.814 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:42:59 smithi089 bash[78452]: debug 2023-06-03T19:42:59.516+0000 7f9a21422c80 4 rocksdb: Options.compression_opts.enabled: false 2023-06-03T19:42:59.814 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:42:59 smithi089 bash[78452]: debug 2023-06-03T19:42:59.516+0000 7f9a21422c80 4 rocksdb: Options.compression_opts.max_dict_buffer_bytes: 0 2023-06-03T19:42:59.815 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:42:59 smithi089 bash[78452]: debug 2023-06-03T19:42:59.516+0000 7f9a21422c80 4 rocksdb: Options.level0_file_num_compaction_trigger: 4 2023-06-03T19:42:59.815 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:42:59 smithi089 bash[78452]: debug 2023-06-03T19:42:59.516+0000 7f9a21422c80 4 rocksdb: Options.level0_slowdown_writes_trigger: 20 2023-06-03T19:42:59.815 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:42:59 smithi089 bash[78452]: debug 2023-06-03T19:42:59.516+0000 7f9a21422c80 4 rocksdb: Options.level0_stop_writes_trigger: 36 2023-06-03T19:42:59.815 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:42:59 smithi089 bash[78452]: debug 2023-06-03T19:42:59.516+0000 7f9a21422c80 4 rocksdb: Options.target_file_size_base: 67108864 2023-06-03T19:42:59.815 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:42:59 smithi089 bash[78452]: debug 2023-06-03T19:42:59.516+0000 7f9a21422c80 4 rocksdb: Options.target_file_size_multiplier: 1 2023-06-03T19:42:59.816 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:42:59 smithi089 bash[78452]: debug 2023-06-03T19:42:59.516+0000 7f9a21422c80 4 rocksdb: Options.max_bytes_for_level_base: 268435456 2023-06-03T19:42:59.816 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:42:59 smithi089 bash[78452]: debug 2023-06-03T19:42:59.516+0000 7f9a21422c80 4 rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2023-06-03T19:42:59.816 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:42:59 smithi089 bash[78452]: debug 2023-06-03T19:42:59.516+0000 7f9a21422c80 4 rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2023-06-03T19:42:59.816 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:42:59 smithi089 bash[78452]: debug 2023-06-03T19:42:59.516+0000 7f9a21422c80 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2023-06-03T19:42:59.816 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:42:59 smithi089 bash[78452]: debug 2023-06-03T19:42:59.516+0000 7f9a21422c80 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2023-06-03T19:42:59.817 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:42:59 smithi089 bash[78452]: debug 2023-06-03T19:42:59.516+0000 7f9a21422c80 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2023-06-03T19:42:59.817 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:42:59 smithi089 bash[78452]: debug 2023-06-03T19:42:59.516+0000 7f9a21422c80 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2023-06-03T19:42:59.817 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:42:59 smithi089 bash[78452]: debug 2023-06-03T19:42:59.516+0000 7f9a21422c80 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2023-06-03T19:42:59.817 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:42:59 smithi089 bash[78452]: debug 2023-06-03T19:42:59.516+0000 7f9a21422c80 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2023-06-03T19:42:59.817 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:42:59 smithi089 bash[78452]: debug 2023-06-03T19:42:59.516+0000 7f9a21422c80 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2023-06-03T19:42:59.818 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:42:59 smithi089 bash[78452]: debug 2023-06-03T19:42:59.516+0000 7f9a21422c80 4 rocksdb: Options.max_sequential_skip_in_iterations: 8 2023-06-03T19:42:59.818 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:42:59 smithi089 bash[78452]: debug 2023-06-03T19:42:59.516+0000 7f9a21422c80 4 rocksdb: Options.max_compaction_bytes: 1677721600 2023-06-03T19:42:59.818 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:42:59 smithi089 bash[78452]: debug 2023-06-03T19:42:59.516+0000 7f9a21422c80 4 rocksdb: Options.ignore_max_compaction_bytes_for_input: true 2023-06-03T19:42:59.818 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:42:59 smithi089 bash[78452]: debug 2023-06-03T19:42:59.516+0000 7f9a21422c80 4 rocksdb: Options.arena_block_size: 1048576 2023-06-03T19:42:59.818 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:42:59 smithi089 bash[78452]: debug 2023-06-03T19:42:59.516+0000 7f9a21422c80 4 rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2023-06-03T19:42:59.819 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:42:59 smithi089 bash[78452]: debug 2023-06-03T19:42:59.516+0000 7f9a21422c80 4 rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2023-06-03T19:42:59.819 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:42:59 smithi089 bash[78452]: debug 2023-06-03T19:42:59.516+0000 7f9a21422c80 4 rocksdb: Options.disable_auto_compactions: 0 2023-06-03T19:42:59.819 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:42:59 smithi089 bash[78452]: debug 2023-06-03T19:42:59.516+0000 7f9a21422c80 4 rocksdb: Options.compaction_style: kCompactionStyleLevel 2023-06-03T19:42:59.819 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:42:59 smithi089 bash[78452]: debug 2023-06-03T19:42:59.516+0000 7f9a21422c80 4 rocksdb: Options.compaction_pri: kMinOverlappingRatio 2023-06-03T19:42:59.819 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:42:59 smithi089 bash[78452]: debug 2023-06-03T19:42:59.516+0000 7f9a21422c80 4 rocksdb: Options.compaction_options_universal.size_ratio: 1 2023-06-03T19:42:59.820 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:42:59 smithi089 bash[78452]: debug 2023-06-03T19:42:59.516+0000 7f9a21422c80 4 rocksdb: Options.compaction_options_universal.min_merge_width: 2 2023-06-03T19:42:59.820 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:42:59 smithi089 bash[78452]: debug 2023-06-03T19:42:59.516+0000 7f9a21422c80 4 rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2023-06-03T19:42:59.820 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:42:59 smithi089 bash[78452]: debug 2023-06-03T19:42:59.516+0000 7f9a21422c80 4 rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2023-06-03T19:42:59.820 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:42:59 smithi089 bash[78452]: debug 2023-06-03T19:42:59.516+0000 7f9a21422c80 4 rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2023-06-03T19:42:59.820 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:42:59 smithi089 bash[78452]: debug 2023-06-03T19:42:59.516+0000 7f9a21422c80 4 rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2023-06-03T19:42:59.821 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:42:59 smithi089 bash[78452]: debug 2023-06-03T19:42:59.516+0000 7f9a21422c80 4 rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2023-06-03T19:42:59.821 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:42:59 smithi089 bash[78452]: debug 2023-06-03T19:42:59.516+0000 7f9a21422c80 4 rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2023-06-03T19:42:59.821 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:42:59 smithi089 bash[78452]: debug 2023-06-03T19:42:59.516+0000 7f9a21422c80 4 rocksdb: Options.table_properties_collectors: 2023-06-03T19:42:59.821 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:42:59 smithi089 bash[78452]: debug 2023-06-03T19:42:59.516+0000 7f9a21422c80 4 rocksdb: Options.inplace_update_support: 0 2023-06-03T19:42:59.821 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:42:59 smithi089 bash[78452]: debug 2023-06-03T19:42:59.516+0000 7f9a21422c80 4 rocksdb: Options.inplace_update_num_locks: 10000 2023-06-03T19:42:59.822 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:42:59 smithi089 bash[78452]: debug 2023-06-03T19:42:59.516+0000 7f9a21422c80 4 rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2023-06-03T19:42:59.822 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:42:59 smithi089 bash[78452]: debug 2023-06-03T19:42:59.516+0000 7f9a21422c80 4 rocksdb: Options.memtable_whole_key_filtering: 0 2023-06-03T19:42:59.822 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:42:59 smithi089 bash[78452]: debug 2023-06-03T19:42:59.516+0000 7f9a21422c80 4 rocksdb: Options.memtable_huge_page_size: 0 2023-06-03T19:42:59.822 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:42:59 smithi089 bash[78452]: debug 2023-06-03T19:42:59.516+0000 7f9a21422c80 4 rocksdb: Options.bloom_locality: 0 2023-06-03T19:42:59.822 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:42:59 smithi089 bash[78452]: debug 2023-06-03T19:42:59.516+0000 7f9a21422c80 4 rocksdb: Options.max_successive_merges: 0 2023-06-03T19:42:59.823 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:42:59 smithi089 bash[78452]: debug 2023-06-03T19:42:59.516+0000 7f9a21422c80 4 rocksdb: Options.optimize_filters_for_hits: 0 2023-06-03T19:42:59.823 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:42:59 smithi089 bash[78452]: debug 2023-06-03T19:42:59.516+0000 7f9a21422c80 4 rocksdb: Options.paranoid_file_checks: 0 2023-06-03T19:42:59.823 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:42:59 smithi089 bash[78452]: debug 2023-06-03T19:42:59.516+0000 7f9a21422c80 4 rocksdb: Options.force_consistency_checks: 1 2023-06-03T19:42:59.823 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:42:59 smithi089 bash[78452]: debug 2023-06-03T19:42:59.516+0000 7f9a21422c80 4 rocksdb: Options.report_bg_io_stats: 0 2023-06-03T19:42:59.823 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:42:59 smithi089 bash[78452]: debug 2023-06-03T19:42:59.516+0000 7f9a21422c80 4 rocksdb: Options.ttl: 2592000 2023-06-03T19:42:59.824 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:42:59 smithi089 bash[78452]: debug 2023-06-03T19:42:59.516+0000 7f9a21422c80 4 rocksdb: Options.periodic_compaction_seconds: 0 2023-06-03T19:42:59.824 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:42:59 smithi089 bash[78452]: debug 2023-06-03T19:42:59.516+0000 7f9a21422c80 4 rocksdb: Options.preclude_last_level_data_seconds: 0 2023-06-03T19:42:59.824 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:42:59 smithi089 bash[78452]: debug 2023-06-03T19:42:59.516+0000 7f9a21422c80 4 rocksdb: Options.preserve_internal_time_seconds: 0 2023-06-03T19:42:59.824 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:42:59 smithi089 bash[78452]: debug 2023-06-03T19:42:59.516+0000 7f9a21422c80 4 rocksdb: Options.enable_blob_files: false 2023-06-03T19:42:59.824 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:42:59 smithi089 bash[78452]: debug 2023-06-03T19:42:59.516+0000 7f9a21422c80 4 rocksdb: Options.min_blob_size: 0 2023-06-03T19:42:59.825 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:42:59 smithi089 bash[78452]: debug 2023-06-03T19:42:59.516+0000 7f9a21422c80 4 rocksdb: Options.blob_file_size: 268435456 2023-06-03T19:42:59.825 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:42:59 smithi089 bash[78452]: debug 2023-06-03T19:42:59.516+0000 7f9a21422c80 4 rocksdb: Options.blob_compression_type: NoCompression 2023-06-03T19:42:59.825 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:42:59 smithi089 bash[78452]: debug 2023-06-03T19:42:59.516+0000 7f9a21422c80 4 rocksdb: Options.enable_blob_garbage_collection: false 2023-06-03T19:42:59.825 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:42:59 smithi089 bash[78452]: debug 2023-06-03T19:42:59.516+0000 7f9a21422c80 4 rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2023-06-03T19:42:59.826 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:42:59 smithi089 bash[78452]: debug 2023-06-03T19:42:59.516+0000 7f9a21422c80 4 rocksdb: Options.blob_garbage_collection_force_threshold: 1.000000 2023-06-03T19:42:59.826 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:42:59 smithi089 bash[78452]: debug 2023-06-03T19:42:59.516+0000 7f9a21422c80 4 rocksdb: Options.blob_compaction_readahead_size: 0 2023-06-03T19:42:59.826 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:42:59 smithi089 bash[78452]: debug 2023-06-03T19:42:59.516+0000 7f9a21422c80 4 rocksdb: Options.blob_file_starting_level: 0 2023-06-03T19:42:59.826 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:42:59 smithi089 bash[78452]: debug 2023-06-03T19:42:59.516+0000 7f9a21422c80 4 rocksdb: Options.experimental_mempurge_threshold: 0.000000 2023-06-03T19:42:59.827 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:42:59 smithi089 bash[78452]: debug 2023-06-03T19:42:59.516+0000 7f9a16c42700 3 rocksdb: [table/block_based/block_based_table_reader.cc:721] At least one SST file opened without unique ID to verify: 29.sst 2023-06-03T19:42:59.827 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:42:59 smithi089 bash[78452]: debug 2023-06-03T19:42:59.520+0000 7f9a21422c80 4 rocksdb: [db/version_set.cc:5566] Recovered from manifest file:/var/lib/ceph/mon/ceph-c/store.db/MANIFEST-000005 succeeded,manifest_file_number is 5, next_file_number is 31, last_sequence is 19063, log_number is 27,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2023-06-03T19:42:59.827 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:42:59 smithi089 bash[78452]: debug 2023-06-03T19:42:59.520+0000 7f9a21422c80 4 rocksdb: [db/version_set.cc:5581] Column family [default] (ID 0), log number is 27 2023-06-03T19:42:59.827 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:42:59 smithi089 bash[78452]: debug 2023-06-03T19:42:59.524+0000 7f9a21422c80 4 rocksdb: [db/db_impl/db_impl_open.cc:539] DB ID: 58f31236-0b12-436d-9ffe-836294be1f51 2023-06-03T19:42:59.827 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:42:59 smithi089 bash[78452]: debug 2023-06-03T19:42:59.524+0000 7f9a21422c80 4 rocksdb: EVENT_LOG_v1 {"time_micros": 1685821379525444, "job": 1, "event": "recovery_started", "wal_files": [27]} 2023-06-03T19:42:59.828 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:42:59 smithi089 bash[78452]: debug 2023-06-03T19:42:59.524+0000 7f9a21422c80 4 rocksdb: [db/db_impl/db_impl_open.cc:1043] Recovering log #27 mode 2 2023-06-03T19:42:59.828 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:42:59 smithi089 bash[78452]: debug 2023-06-03T19:42:59.536+0000 7f9a21422c80 4 rocksdb: EVENT_LOG_v1 {"time_micros": 1685821379537550, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 32, "file_size": 1194548, "file_checksum": "", "file_checksum_func_name": "Unknown", "smallest_seqno": 19058, "largest_seqno": 19354, "table_properties": {"data_size": 1191899, "index_size": 1317, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 325, "raw_key_size": 2300, "raw_average_key_size": 22, "raw_value_size": 1188840, "raw_average_value_size": 11431, "num_data_blocks": 57, "num_entries": 104, "num_filter_entries": 104, "num_deletions": 0, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "bloomfilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; max_dict_buffer_bytes=0; use_zstd_dict_trainer=1; ", "creation_time": 1685821379, "oldest_key_time": 0, "file_creation_time": 0, "slow_compression_estimated_data_size": 0, "fast_compression_estimated_data_size": 0, "db_id": "58f31236-0b12-436d-9ffe-836294be1f51", "db_session_id": "AQI6EUQ0185BYRBGXFII", "orig_file_number": 32, "seqno_to_time_mapping": "N/A"}} 2023-06-03T19:42:59.828 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:42:59 smithi089 bash[78452]: debug 2023-06-03T19:42:59.536+0000 7f9a21422c80 4 rocksdb: EVENT_LOG_v1 {"time_micros": 1685821379537658, "job": 1, "event": "recovery_finished"} 2023-06-03T19:42:59.828 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:42:59 smithi089 bash[78452]: debug 2023-06-03T19:42:59.536+0000 7f9a21422c80 4 rocksdb: [db/version_set.cc:5047] Creating manifest 34 2023-06-03T19:42:59.829 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:42:59 smithi089 bash[78452]: debug 2023-06-03T19:42:59.536+0000 7f9a21422c80 4 rocksdb: [file/delete_scheduler.cc:74] Deleted file /var/lib/ceph/mon/ceph-c/store.db/000027.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2023-06-03T19:42:59.829 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:42:59 smithi089 bash[78452]: debug 2023-06-03T19:42:59.536+0000 7f9a21422c80 4 rocksdb: [db/db_impl/db_impl_open.cc:1987] SstFileManager instance 0x56133abd2000 2023-06-03T19:42:59.829 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:42:59 smithi089 bash[78452]: debug 2023-06-03T19:42:59.536+0000 7f9a21422c80 4 rocksdb: DB pointer 0x56133abc0000 2023-06-03T19:42:59.829 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:42:59 smithi089 bash[78452]: debug 2023-06-03T19:42:59.540+0000 7f9a0fc34700 4 rocksdb: [db/db_impl/db_impl.cc:1109] ------- DUMPING STATS ------- 2023-06-03T19:42:59.829 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:42:59 smithi089 bash[78452]: debug 2023-06-03T19:42:59.540+0000 7f9a0fc34700 4 rocksdb: [db/db_impl/db_impl.cc:1111] 2023-06-03T19:42:59.830 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:42:59 smithi089 bash[78452]: ** DB Stats ** 2023-06-03T19:42:59.830 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:42:59 smithi089 bash[78452]: Uptime(secs): 0.0 total, 0.0 interval 2023-06-03T19:42:59.830 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:42:59 smithi089 bash[78452]: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2023-06-03T19:42:59.830 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:42:59 smithi089 bash[78452]: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2023-06-03T19:42:59.830 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:42:59 smithi089 bash[78452]: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2023-06-03T19:42:59.831 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:42:59 smithi089 bash[78452]: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2023-06-03T19:42:59.831 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:42:59 smithi089 bash[78452]: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2023-06-03T19:42:59.831 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:42:59 smithi089 bash[78452]: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2023-06-03T19:42:59.831 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:42:59 smithi089 bash[78452]: ** Compaction Stats [default] ** 2023-06-03T19:42:59.831 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:42:59 smithi089 bash[78452]: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2023-06-03T19:42:59.832 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:42:59 smithi089 bash[78452]: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ 2023-06-03T19:42:59.832 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:42:59 smithi089 bash[78452]: L0 1/0 1.14 MB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 227.1 0.01 0.00 1 0.005 0 0 0.0 0.0 2023-06-03T19:42:59.832 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:42:59 smithi089 bash[78452]: L6 1/0 30.37 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 0.0 0.0 2023-06-03T19:42:59.832 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:42:59 smithi089 bash[78452]: Sum 2/0 31.51 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 227.1 0.01 0.00 1 0.005 0 0 0.0 0.0 2023-06-03T19:42:59.833 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:42:59 smithi089 bash[78452]: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 227.1 0.01 0.00 1 0.005 0 0 0.0 0.0 2023-06-03T19:42:59.833 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:42:59 smithi089 bash[78452]: ** Compaction Stats [default] ** 2023-06-03T19:42:59.833 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:42:59 smithi089 bash[78452]: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2023-06-03T19:42:59.833 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:42:59 smithi089 bash[78452]: --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2023-06-03T19:42:59.833 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:42:59 smithi089 bash[78452]: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 227.1 0.01 0.00 1 0.005 0 0 0.0 0.0 2023-06-03T19:42:59.834 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:42:59 smithi089 bash[78452]: Blob file count: 0, total size: 0.0 GB, garbage size: 0.0 GB, space amp: 0.0 2023-06-03T19:42:59.834 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:42:59 smithi089 bash[78452]: Uptime(secs): 0.0 total, 0.0 interval 2023-06-03T19:42:59.834 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:42:59 smithi089 bash[78452]: Flush(GB): cumulative 0.001, interval 0.001 2023-06-03T19:42:59.834 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:42:59 smithi089 bash[78452]: AddFile(GB): cumulative 0.000, interval 0.000 2023-06-03T19:42:59.834 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:42:59 smithi089 bash[78452]: AddFile(Total Files): cumulative 0, interval 0 2023-06-03T19:42:59.835 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:42:59 smithi089 bash[78452]: AddFile(L0 Files): cumulative 0, interval 0 2023-06-03T19:42:59.835 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:42:59 smithi089 bash[78452]: AddFile(Keys): cumulative 0, interval 0 2023-06-03T19:42:59.835 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:42:59 smithi089 bash[78452]: Cumulative compaction: 0.00 GB write, 48.45 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-06-03T19:42:59.835 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:42:59 smithi089 bash[78452]: Interval compaction: 0.00 GB write, 48.45 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-06-03T19:42:59.836 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:42:59 smithi089 bash[78452]: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2023-06-03T19:42:59.836 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:42:59 smithi089 bash[78452]: Block cache BinnedLRUCache@0x56133ab311f0#7 capacity: 512.00 MB usage: 109.17 KB table_size: 0 occupancy: 18446744073709551615 collections: 1 last_copies: 0 last_secs: 1.5e-05 secs_since: 0 2023-06-03T19:42:59.836 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:42:59 smithi089 bash[78452]: Block cache entry stats(count,size,portion): DataBlock(3,52.73 KB,0.0100583%) FilterBlock(2,6.91 KB,0.00131726%) IndexBlock(2,49.53 KB,0.00944734%) Misc(1,0.00 KB,0%) 2023-06-03T19:42:59.836 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:42:59 smithi089 bash[78452]: ** File Read Latency Histogram By Level [default] ** 2023-06-03T19:42:59.836 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:42:59 smithi089 bash[78452]: debug 2023-06-03T19:42:59.540+0000 7f9a21422c80 0 starting mon.c rank 2 at public addrs [v2:172.21.15.89:3301/0,v1:172.21.15.89:6790/0] at bind addrs [v2:172.21.15.89:3301/0,v1:172.21.15.89:6790/0] mon_data /var/lib/ceph/mon/ceph-c fsid f1db3478-0243-11ee-9b1d-001a4aab830c 2023-06-03T19:42:59.837 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:42:59 smithi089 bash[78452]: debug 2023-06-03T19:42:59.540+0000 7f9a21422c80 1 mon.c@-1(???) e3 preinit fsid f1db3478-0243-11ee-9b1d-001a4aab830c 2023-06-03T19:42:59.837 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:42:59 smithi089 bash[78452]: debug 2023-06-03T19:42:59.540+0000 7f9a21422c80 0 mon.c@-1(???).mds e7 new map 2023-06-03T19:42:59.837 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:42:59 smithi089 bash[78452]: debug 2023-06-03T19:42:59.540+0000 7f9a21422c80 0 mon.c@-1(???).mds e7 print_map 2023-06-03T19:42:59.837 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:42:59 smithi089 bash[78452]: e7 2023-06-03T19:42:59.837 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:42:59 smithi089 bash[78452]: enable_multiple, ever_enabled_multiple: 1,1 2023-06-03T19:42:59.838 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:42:59 smithi089 bash[78452]: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2023-06-03T19:42:59.838 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:42:59 smithi089 bash[78452]: legacy client fscid: 1 2023-06-03T19:42:59.838 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:42:59 smithi089 bash[78452]: 2023-06-03T19:42:59.838 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:42:59 smithi089 bash[78452]: Filesystem 'foo' (1) 2023-06-03T19:42:59.838 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:42:59 smithi089 bash[78452]: fs_name foo 2023-06-03T19:42:59.839 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:42:59 smithi089 bash[78452]: epoch 5 2023-06-03T19:42:59.839 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:42:59 smithi089 bash[78452]: flags 12 joinable allow_snaps allow_multimds_snaps 2023-06-03T19:42:59.839 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:42:59 smithi089 bash[78452]: created 2023-06-03T19:34:16.805120+0000 2023-06-03T19:42:59.839 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:42:59 smithi089 bash[78452]: modified 2023-06-03T19:34:24.830695+0000 2023-06-03T19:42:59.839 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:42:59 smithi089 bash[78452]: tableserver 0 2023-06-03T19:42:59.840 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:42:59 smithi089 bash[78452]: root 0 2023-06-03T19:42:59.840 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:42:59 smithi089 bash[78452]: session_timeout 60 2023-06-03T19:42:59.840 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:42:59 smithi089 bash[78452]: session_autoclose 300 2023-06-03T19:42:59.840 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:42:59 smithi089 bash[78452]: max_file_size 1099511627776 2023-06-03T19:42:59.841 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:42:59 smithi089 bash[78452]: required_client_features {} 2023-06-03T19:42:59.841 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:42:59 smithi089 bash[78452]: last_failure 0 2023-06-03T19:42:59.841 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:42:59 smithi089 bash[78452]: last_failure_osd_epoch 0 2023-06-03T19:42:59.841 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:42:59 smithi089 bash[78452]: compat compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,7=mds uses inline data,8=no anchor table,9=file layout v2,10=snaprealm v2} 2023-06-03T19:42:59.841 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:42:59 smithi089 bash[78452]: max_mds 1 2023-06-03T19:42:59.842 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:42:59 smithi089 bash[78452]: in 0 2023-06-03T19:42:59.842 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:42:59 smithi089 bash[78452]: up {0=14646} 2023-06-03T19:42:59.842 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:42:59 smithi089 bash[78452]: failed 2023-06-03T19:42:59.842 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:42:59 smithi089 bash[78452]: damaged 2023-06-03T19:42:59.842 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:42:59 smithi089 bash[78452]: stopped 2023-06-03T19:42:59.843 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:42:59 smithi089 bash[78452]: data_pools [4] 2023-06-03T19:42:59.843 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:42:59 smithi089 bash[78452]: metadata_pool 3 2023-06-03T19:42:59.843 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:42:59 smithi089 bash[78452]: inline_data disabled 2023-06-03T19:42:59.843 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:42:59 smithi089 bash[78452]: balancer 2023-06-03T19:42:59.843 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:42:59 smithi089 bash[78452]: bal_rank_mask -1 2023-06-03T19:42:59.844 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:42:59 smithi089 bash[78452]: standby_count_wanted 1 2023-06-03T19:42:59.844 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:42:59 smithi089 bash[78452]: [mds.foo.smithi089.giuxuj{0:14646} state up:active seq 2 join_fscid=1 addr [v2:172.21.15.89:6834/1926931511,v1:172.21.15.89:6835/1926931511] compat {c=[1],r=[1],i=[7ff]}] 2023-06-03T19:42:59.844 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:42:59 smithi089 bash[78452]: 2023-06-03T19:42:59.844 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:42:59 smithi089 bash[78452]: 2023-06-03T19:42:59.844 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:42:59 smithi089 bash[78452]: Standby daemons: 2023-06-03T19:42:59.845 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:42:59 smithi089 bash[78452]: 2023-06-03T19:42:59.845 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:42:59 smithi089 bash[78452]: [mds.foo.smithi164.txgfgv{-1:24499} state up:standby seq 1 join_fscid=1 addr [v2:172.21.15.164:6832/2311698270,v1:172.21.15.164:6833/2311698270] compat {c=[1],r=[1],i=[7ff]}] 2023-06-03T19:42:59.845 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:42:59 smithi089 bash[78452]: debug 2023-06-03T19:42:59.540+0000 7f9a21422c80 0 mon.c@-1(???).osd e77 crush map has features 3314933000854323200, adjusting msgr requires 2023-06-03T19:42:59.845 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:42:59 smithi089 bash[78452]: debug 2023-06-03T19:42:59.540+0000 7f9a21422c80 0 mon.c@-1(???).osd e77 crush map has features 432629239337189376, adjusting msgr requires 2023-06-03T19:42:59.846 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:42:59 smithi089 bash[78452]: debug 2023-06-03T19:42:59.540+0000 7f9a21422c80 0 mon.c@-1(???).osd e77 crush map has features 432629239337189376, adjusting msgr requires 2023-06-03T19:42:59.846 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:42:59 smithi089 bash[78452]: debug 2023-06-03T19:42:59.540+0000 7f9a21422c80 0 mon.c@-1(???).osd e77 crush map has features 432629239337189376, adjusting msgr requires 2023-06-03T19:42:59.846 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:42:59 smithi089 bash[78452]: debug 2023-06-03T19:42:59.544+0000 7f9a21422c80 1 mon.c@-1(???).paxosservice(auth 1..21) refresh upgraded, format 0 -> 3 2023-06-03T19:43:00.376 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 72 16 99576 99560 86.4145 55.625 0.0166675 0.0115665 2023-06-03T19:43:01.376 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 73 16 100490 100474 86.0132 57.125 0.0261532 0.0116203 2023-06-03T19:43:02.376 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 74 16 101380 101364 85.6025 55.625 0.0367449 0.0116763 2023-06-03T19:43:03.376 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 75 16 102254 102238 85.1894 54.625 0.00864003 0.0117326 2023-06-03T19:43:04.377 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 76 16 103117 103101 84.7781 53.9375 0.00808941 0.0117897 2023-06-03T19:43:05.377 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 77 16 103918 103902 84.3272 50.0625 0.0157201 0.0118534 2023-06-03T19:43:06.085 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:43:05 smithi164 bash[26855]: cluster 2023-06-03T19:42:59.563136+0000 mon.c (mon.2) 1 : cluster [INF] mon.c calling monitor election 2023-06-03T19:43:06.086 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:43:05 smithi164 bash[26855]: cluster 2023-06-03T19:43:05.569573+0000 mon.b (mon.1) 98 : cluster [INF] mon.b calling monitor election 2023-06-03T19:43:06.086 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:43:05 smithi164 bash[26855]: cluster 2023-06-03T19:43:05.572652+0000 mon.a (mon.0) 73 : cluster [INF] mon.a calling monitor election 2023-06-03T19:43:06.086 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:43:05 smithi164 bash[26855]: cluster 2023-06-03T19:43:05.577655+0000 mon.a (mon.0) 74 : cluster [INF] mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2023-06-03T19:43:06.086 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:43:05 smithi164 bash[26855]: cluster 2023-06-03T19:43:05.588676+0000 mon.a (mon.0) 75 : cluster [DBG] monmap e3: 3 mons at {a=[v2:172.21.15.89:3300/0,v1:172.21.15.89:6789/0],b=[v2:172.21.15.164:3300/0,v1:172.21.15.164:6789/0],c=[v2:172.21.15.89:3301/0,v1:172.21.15.89:6790/0]} removed_ranks: {} 2023-06-03T19:43:06.086 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:43:05 smithi164 bash[26855]: cluster 2023-06-03T19:43:05.589347+0000 mon.a (mon.0) 76 : cluster [DBG] fsmap foo:1 {0=foo.smithi089.giuxuj=up:active} 1 up:standby 2023-06-03T19:43:06.086 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:43:05 smithi164 bash[26855]: cluster 2023-06-03T19:43:05.589375+0000 mon.a (mon.0) 77 : cluster [DBG] osdmap e77: 8 total, 8 up, 8 in 2023-06-03T19:43:06.086 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:43:05 smithi164 bash[26855]: cluster 2023-06-03T19:43:05.590338+0000 mon.a (mon.0) 78 : cluster [DBG] mgrmap e38: y(active, since 103s), standbys: x 2023-06-03T19:43:06.087 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:43:05 smithi164 bash[26855]: cluster 2023-06-03T19:43:05.590484+0000 mon.a (mon.0) 79 : cluster [INF] Health check cleared: MON_DOWN (was: 1/3 mons down, quorum a,b) 2023-06-03T19:43:06.087 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:43:05 smithi164 bash[26855]: cluster 2023-06-03T19:43:05.590508+0000 mon.a (mon.0) 80 : cluster [INF] Cluster is now healthy 2023-06-03T19:43:06.087 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:43:05 smithi164 bash[26855]: cluster 2023-06-03T19:43:05.603762+0000 mon.a (mon.0) 81 : cluster [INF] overall HEALTH_OK 2023-06-03T19:43:06.377 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 78 16 104735 104719 83.9007 51.0625 0.0158596 0.0119131 2023-06-03T19:43:06.530 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 19:43:06 smithi089 bash[71163]: ::ffff:172.21.15.164 - - [03/Jun/2023:19:43:06] "GET /metrics HTTP/1.1" 200 190051 "" "Prometheus/2.33.4" 2023-06-03T19:43:06.639 INFO:teuthology.orchestra.run.smithi089.stdout: "ceph version 18.0.0-3532-g8e24bcf4 (8e24bcf462a1d94c2e12aa4e26419634883a1e88) reef (rc)": 2 2023-06-03T19:43:07.085 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:43:06 smithi164 bash[26855]: audit 2023-06-03T19:43:06.594358+0000 mon.a (mon.0) 82 : audit [DBG] from='client.? 172.21.15.89:0/3649894161' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-03T19:43:07.377 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 79 16 105546 105530 83.4802 50.6875 0.0183943 0.0119732 2023-06-03T19:43:08.085 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:43:07 smithi164 bash[26855]: cluster 2023-06-03T19:43:05.998598+0000 mgr.y (mgr.34104) 73 : cluster [DBG] pgmap v60: 89 pgs: 89 active+clean; 6.2 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 45 MiB/s wr, 723 op/s 2023-06-03T19:43:08.085 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:43:07 smithi164 bash[26855]: audit 2023-06-03T19:43:07.063322+0000 mon.b (mon.1) 99 : audit [DBG] from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T19:43:08.377 INFO:tasks.radosbench.radosbench.0.smithi089.stdout:2023-06-03T19:43:08.371303+0000 min lat: 0.00131693 max lat: 0.213513 avg lat: 0.012024 2023-06-03T19:43:08.377 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-06-03T19:43:08.377 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 80 16 106428 106412 83.1257 55.125 0.0100335 0.012024 2023-06-03T19:43:08.634 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -e sha1=8e24bcf462a1d94c2e12aa4e26419634883a1e88 -- bash -c 'ceph orch ps' 2023-06-03T19:43:09.085 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:43:08 smithi164 bash[26855]: cluster 2023-06-03T19:43:07.999125+0000 mgr.y (mgr.34104) 74 : cluster [DBG] pgmap v61: 89 pgs: 89 active+clean; 6.3 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 52 MiB/s wr, 831 op/s 2023-06-03T19:43:09.085 INFO:journalctl@ceph.prometheus.a.smithi164.stdout:Jun 03 19:43:08 smithi164 bash[55188]: ts=2023-06-03T19:43:08.660Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.13-257-gd8c5d349 (d8c5d34975dce1c5eb0aa3a7979a4d9b9a99d1ec) pacific (stable)\", cluster_addr=\"172.21.15.89\", device_class=\"ssd\", hostname=\"smithi089\", instance=\"smithi164.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.89\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.13-257-gd8c5d349 (d8c5d34975dce1c5eb0aa3a7979a4d9b9a99d1ec) pacific (stable)\", cluster_addr=\"172.21.15.89\", device_class=\"ssd\", hostname=\"smithi089\", instance=\"smithi089:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.89\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-06-03T19:43:09.086 INFO:journalctl@ceph.prometheus.a.smithi164.stdout:Jun 03 19:43:08 smithi164 bash[55188]: ts=2023-06-03T19:43:08.662Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephPGImbalance\nexpr: abs(((ceph_osd_numpg > 0) - on(job) group_left() avg by(job) (ceph_osd_numpg\n > 0)) / on(job) group_left() avg by(job) (ceph_osd_numpg > 0)) * on(ceph_daemon)\n group_left(hostname) ceph_osd_metadata > 0.3\nfor: 5m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.5\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} deviates by\n more than 30% from average PG count.\n summary: PGs are not balanced across OSDs\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.13-257-gd8c5d349 (d8c5d34975dce1c5eb0aa3a7979a4d9b9a99d1ec) pacific (stable)\", cluster_addr=\"172.21.15.89\", device_class=\"ssd\", hostname=\"smithi089\", instance=\"smithi164.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.89\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.13-257-gd8c5d349 (d8c5d34975dce1c5eb0aa3a7979a4d9b9a99d1ec) pacific (stable)\", cluster_addr=\"172.21.15.89\", device_class=\"ssd\", hostname=\"smithi089\", instance=\"smithi089:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.89\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-06-03T19:43:09.377 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 81 16 107260 107244 82.7414 52 0.0159514 0.0120804 2023-06-03T19:43:10.030 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 19:43:09 smithi089 bash[71163]: debug 2023-06-03T19:43:09.560+0000 7fe48642e700 -1 mgr.server handle_report got status from non-daemon mon.c 2023-06-03T19:43:10.377 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 82 16 108107 108091 82.3779 52.9375 0.0231866 0.0121331 2023-06-03T19:43:11.377 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 83 16 108931 108915 82.0058 51.5 0.0244863 0.0121893 2023-06-03T19:43:11.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:43:11 smithi164 bash[26855]: cluster 2023-06-03T19:43:09.999770+0000 mgr.y (mgr.34104) 75 : cluster [DBG] pgmap v62: 89 pgs: 89 active+clean; 6.5 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 53 MiB/s wr, 855 op/s 2023-06-03T19:43:12.378 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 84 16 109695 109679 81.597 47.75 0.0259764 0.0122491 2023-06-03T19:43:13.380 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 85 16 110337 110321 81.1072 40.125 0.0191761 0.0123226 2023-06-03T19:43:13.491 INFO:teuthology.orchestra.run.smithi089.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2023-06-03T19:43:13.491 INFO:teuthology.orchestra.run.smithi089.stdout:alertmanager.a smithi089 *:9093,9094 running (4m) 27s ago 10m 14.0M - 0.23.0 ba2b418f427c 0fad3b643b2e 2023-06-03T19:43:13.492 INFO:teuthology.orchestra.run.smithi089.stdout:grafana.a smithi164 *:3000 running (3m) 26s ago 10m 40.9M - 8.3.5 dad864ee21e9 64925a21f38b 2023-06-03T19:43:13.492 INFO:teuthology.orchestra.run.smithi089.stdout:mds.foo.smithi089.giuxuj smithi089 running (8m) 27s ago 8m 16.3M - 16.2.13-257-gd8c5d349 b644dd95899d f0e9de24a3f8 2023-06-03T19:43:13.492 INFO:teuthology.orchestra.run.smithi089.stdout:mds.foo.smithi164.txgfgv smithi164 running (8m) 26s ago 8m 13.7M - 16.2.13-257-gd8c5d349 b644dd95899d 29da307aa3b4 2023-06-03T19:43:13.492 INFO:teuthology.orchestra.run.smithi089.stdout:mgr.x smithi164 *:8443,9283,8765 running (2m) 26s ago 18m 423M - 18.0.0-3532-g8e24bcf4 a57053f96fe3 3ada341a2c72 2023-06-03T19:43:13.492 INFO:teuthology.orchestra.run.smithi089.stdout:mgr.y smithi089 *:8443,9283,8765 running (3m) 27s ago 20m 471M - 18.0.0-3532-g8e24bcf4 a57053f96fe3 8307fbe12a7d 2023-06-03T19:43:13.492 INFO:teuthology.orchestra.run.smithi089.stdout:mon.a smithi089 running (2m) 27s ago 20m 44.9M 2048M 18.0.0-3532-g8e24bcf4 a57053f96fe3 c5e2e407c45f 2023-06-03T19:43:13.492 INFO:teuthology.orchestra.run.smithi089.stdout:mon.b smithi164 running (18m) 26s ago 18m 52.1M 2048M 16.2.13-257-gd8c5d349 b644dd95899d 85a8c1795b60 2023-06-03T19:43:13.493 INFO:teuthology.orchestra.run.smithi089.stdout:mon.c smithi089 starting - - - 2048M 2023-06-03T19:43:13.493 INFO:teuthology.orchestra.run.smithi089.stdout:node-exporter.a smithi089 *:9100 running (5m) 27s ago 11m 12.1M - 1.3.1 1dbe0e931976 f41a3d45da20 2023-06-03T19:43:13.493 INFO:teuthology.orchestra.run.smithi089.stdout:node-exporter.b smithi164 *:9100 running (4m) 26s ago 11m 8928k - 1.3.1 1dbe0e931976 1e69859880c6 2023-06-03T19:43:13.493 INFO:teuthology.orchestra.run.smithi089.stdout:osd.0 smithi089 running (17m) 27s ago 17m 358M 2526M 16.2.13-257-gd8c5d349 b644dd95899d cb8bdf1749e4 2023-06-03T19:43:13.493 INFO:teuthology.orchestra.run.smithi089.stdout:osd.1 smithi089 running (16m) 27s ago 16m 283M 2526M 16.2.13-257-gd8c5d349 b644dd95899d d9e8a03d4689 2023-06-03T19:43:13.493 INFO:teuthology.orchestra.run.smithi089.stdout:osd.2 smithi089 running (15m) 27s ago 15m 409M 2526M 16.2.13-257-gd8c5d349 b644dd95899d f7fc93b466ec 2023-06-03T19:43:13.493 INFO:teuthology.orchestra.run.smithi089.stdout:osd.3 smithi089 running (15m) 27s ago 15m 367M 2526M 16.2.13-257-gd8c5d349 b644dd95899d 7e98b61121c6 2023-06-03T19:43:13.494 INFO:teuthology.orchestra.run.smithi089.stdout:osd.4 smithi164 running (14m) 26s ago 14m 373M 2526M 16.2.13-257-gd8c5d349 b644dd95899d 5c901af1a184 2023-06-03T19:43:13.494 INFO:teuthology.orchestra.run.smithi089.stdout:osd.5 smithi164 running (13m) 26s ago 13m 411M 2526M 16.2.13-257-gd8c5d349 b644dd95899d 13ddc40f04c4 2023-06-03T19:43:13.494 INFO:teuthology.orchestra.run.smithi089.stdout:osd.6 smithi164 running (12m) 26s ago 12m 248M 2526M 16.2.13-257-gd8c5d349 b644dd95899d cab2cb9998d7 2023-06-03T19:43:13.494 INFO:teuthology.orchestra.run.smithi089.stdout:osd.7 smithi164 running (12m) 26s ago 12m 331M 2526M 16.2.13-257-gd8c5d349 b644dd95899d b5fd865771b6 2023-06-03T19:43:13.494 INFO:teuthology.orchestra.run.smithi089.stdout:prometheus.a smithi164 *:9095 running (2m) 26s ago 11m 67.0M - 2.33.4 514e6a882f6e 3ff744731c36 2023-06-03T19:43:13.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:43:13 smithi164 bash[26855]: cluster 2023-06-03T19:43:12.000227+0000 mgr.y (mgr.34104) 76 : cluster [DBG] pgmap v63: 89 pgs: 89 active+clean; 6.5 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 45 MiB/s wr, 722 op/s 2023-06-03T19:43:14.380 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 86 16 111050 111034 80.6822 44.5625 0.00646691 0.0123881 2023-06-03T19:43:14.555 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -e sha1=8e24bcf462a1d94c2e12aa4e26419634883a1e88 -- bash -c 'ceph orch upgrade pause' 2023-06-03T19:43:15.085 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:43:14 smithi164 bash[26855]: audit 2023-06-03T19:43:13.465617+0000 mgr.y (mgr.34104) 77 : audit [DBG] from='client.44102 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-06-03T19:43:15.085 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:43:14 smithi164 bash[26855]: audit 2023-06-03T19:43:13.630914+0000 mon.a (mon.0) 83 : audit [INF] from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:43:15.085 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:43:14 smithi164 bash[26855]: audit 2023-06-03T19:43:13.649749+0000 mon.a (mon.0) 84 : audit [INF] from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:43:15.381 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 87 16 111774 111758 80.2749 45.25 0.013333 0.0124516 2023-06-03T19:43:16.085 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:43:15 smithi164 bash[26855]: cluster 2023-06-03T19:43:14.001060+0000 mgr.y (mgr.34104) 78 : cluster [DBG] pgmap v64: 89 pgs: 89 active+clean; 6.7 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 66 MiB/s wr, 1.06k op/s 2023-06-03T19:43:16.085 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:43:15 smithi164 bash[26855]: audit 2023-06-03T19:43:14.751773+0000 mon.a (mon.0) 85 : audit [INF] from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:43:16.085 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:43:15 smithi164 bash[26855]: audit 2023-06-03T19:43:14.768501+0000 mon.a (mon.0) 86 : audit [INF] from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:43:16.381 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 88 16 112608 112592 79.955 52.125 0.0153207 0.0125023 2023-06-03T19:43:16.530 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 19:43:16 smithi089 bash[71163]: ::ffff:172.21.15.164 - - [03/Jun/2023:19:43:16] "GET /metrics HTTP/1.1" 200 186995 "" "Prometheus/2.33.4" 2023-06-03T19:43:17.381 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 89 15 113451 113436 79.6493 52.75 0.0179037 0.0125491 2023-06-03T19:43:18.085 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:43:17 smithi164 bash[26855]: cluster 2023-06-03T19:43:16.001501+0000 mgr.y (mgr.34104) 79 : cluster [DBG] pgmap v65: 89 pgs: 89 active+clean; 6.7 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 43 MiB/s wr, 681 op/s 2023-06-03T19:43:18.381 INFO:tasks.radosbench.radosbench.0.smithi089.stdout:Total time run: 90.0085 2023-06-03T19:43:18.381 INFO:tasks.radosbench.radosbench.0.smithi089.stdout:Total writes made: 114272 2023-06-03T19:43:18.381 INFO:tasks.radosbench.radosbench.0.smithi089.stdout:Write size: 65536 2023-06-03T19:43:18.381 INFO:tasks.radosbench.radosbench.0.smithi089.stdout:Object size: 65536 2023-06-03T19:43:18.382 INFO:tasks.radosbench.radosbench.0.smithi089.stdout:Bandwidth (MB/sec): 79.3481 2023-06-03T19:43:18.382 INFO:tasks.radosbench.radosbench.0.smithi089.stdout:Stddev Bandwidth: 34.1239 2023-06-03T19:43:18.382 INFO:tasks.radosbench.radosbench.0.smithi089.stdout:Max bandwidth (MB/sec): 197.438 2023-06-03T19:43:18.382 INFO:tasks.radosbench.radosbench.0.smithi089.stdout:Min bandwidth (MB/sec): 40.125 2023-06-03T19:43:18.382 INFO:tasks.radosbench.radosbench.0.smithi089.stdout:Average IOPS: 1269 2023-06-03T19:43:18.382 INFO:tasks.radosbench.radosbench.0.smithi089.stdout:Stddev IOPS: 545.982 2023-06-03T19:43:18.382 INFO:tasks.radosbench.radosbench.0.smithi089.stdout:Max IOPS: 3159 2023-06-03T19:43:18.383 INFO:tasks.radosbench.radosbench.0.smithi089.stdout:Min IOPS: 642 2023-06-03T19:43:18.383 INFO:tasks.radosbench.radosbench.0.smithi089.stdout:Average Latency(s): 0.012598 2023-06-03T19:43:18.383 INFO:tasks.radosbench.radosbench.0.smithi089.stdout:Stddev Latency(s): 0.00856686 2023-06-03T19:43:18.383 INFO:tasks.radosbench.radosbench.0.smithi089.stdout:Max latency(s): 0.213513 2023-06-03T19:43:18.383 INFO:tasks.radosbench.radosbench.0.smithi089.stdout:Min latency(s): 0.00131693 2023-06-03T19:43:18.389 INFO:tasks.radosbench.radosbench.0.smithi089.stdout:Cleaning up (deleting benchmark objects) 2023-06-03T19:43:19.085 INFO:journalctl@ceph.prometheus.a.smithi164.stdout:Jun 03 19:43:18 smithi164 bash[55188]: ts=2023-06-03T19:43:18.661Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.13-257-gd8c5d349 (d8c5d34975dce1c5eb0aa3a7979a4d9b9a99d1ec) pacific (stable)\", cluster_addr=\"172.21.15.89\", device_class=\"ssd\", hostname=\"smithi089\", instance=\"smithi164.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.89\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.13-257-gd8c5d349 (d8c5d34975dce1c5eb0aa3a7979a4d9b9a99d1ec) pacific (stable)\", cluster_addr=\"172.21.15.89\", device_class=\"ssd\", hostname=\"smithi089\", instance=\"smithi089:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.89\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-06-03T19:43:19.086 INFO:journalctl@ceph.prometheus.a.smithi164.stdout:Jun 03 19:43:18 smithi164 bash[55188]: ts=2023-06-03T19:43:18.662Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephPGImbalance\nexpr: abs(((ceph_osd_numpg > 0) - on(job) group_left() avg by(job) (ceph_osd_numpg\n > 0)) / on(job) group_left() avg by(job) (ceph_osd_numpg > 0)) * on(ceph_daemon)\n group_left(hostname) ceph_osd_metadata > 0.3\nfor: 5m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.5\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} deviates by\n more than 30% from average PG count.\n summary: PGs are not balanced across OSDs\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.13-257-gd8c5d349 (d8c5d34975dce1c5eb0aa3a7979a4d9b9a99d1ec) pacific (stable)\", cluster_addr=\"172.21.15.89\", device_class=\"ssd\", hostname=\"smithi089\", instance=\"smithi164.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.89\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.13-257-gd8c5d349 (d8c5d34975dce1c5eb0aa3a7979a4d9b9a99d1ec) pacific (stable)\", cluster_addr=\"172.21.15.89\", device_class=\"ssd\", hostname=\"smithi089\", instance=\"smithi089:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.89\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-06-03T19:43:19.358 INFO:teuthology.orchestra.run.smithi089.stdout:Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:00b1b6d689278b4f97ff790045c8c51d8ca419ac7ee2ebdf11008f2e810a5c78 2023-06-03T19:43:20.085 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:43:19 smithi164 bash[26855]: cluster 2023-06-03T19:43:18.002088+0000 mgr.y (mgr.34104) 80 : cluster [DBG] pgmap v66: 89 pgs: 89 active+clean; 6.7 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 45 MiB/s wr, 718 op/s 2023-06-03T19:43:20.086 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:43:19 smithi164 bash[26855]: audit 2023-06-03T19:43:19.344722+0000 mon.a (mon.0) 87 : audit [INF] from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:43:20.341 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -e sha1=8e24bcf462a1d94c2e12aa4e26419634883a1e88 -- bash -c 'sleep 60' 2023-06-03T19:43:21.085 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:43:20 smithi164 bash[26855]: audit 2023-06-03T19:43:19.337970+0000 mgr.y (mgr.34104) 81 : audit [DBG] from='client.34209 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2023-06-03T19:43:21.085 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:43:20 smithi164 bash[26855]: cephadm 2023-06-03T19:43:19.338568+0000 mgr.y (mgr.34104) 82 : cephadm [INF] Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:00b1b6d689278b4f97ff790045c8c51d8ca419ac7ee2ebdf11008f2e810a5c78 2023-06-03T19:43:22.085 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:43:21 smithi164 bash[26855]: cluster 2023-06-03T19:43:20.002784+0000 mgr.y (mgr.34104) 83 : cluster [DBG] pgmap v67: 89 pgs: 89 active+clean; 7.0 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 56 MiB/s wr, 896 op/s 2023-06-03T19:43:23.085 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:43:22 smithi164 bash[26855]: cluster 2023-06-03T19:43:22.003206+0000 mgr.y (mgr.34104) 84 : cluster [DBG] pgmap v68: 89 pgs: 89 active+clean; 7.0 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 41 MiB/s wr, 657 op/s 2023-06-03T19:43:23.085 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:43:22 smithi164 bash[26855]: audit 2023-06-03T19:43:22.061219+0000 mon.b (mon.1) 100 : audit [DBG] from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T19:43:25.335 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:43:25 smithi164 bash[26855]: cluster 2023-06-03T19:43:24.006995+0000 mgr.y (mgr.34104) 85 : cluster [DBG] pgmap v69: 89 pgs: 89 active+clean; 6.7 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 85 B/s rd, 42 MiB/s wr, 1.00k op/s 2023-06-03T19:43:26.530 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 19:43:26 smithi089 bash[71163]: ::ffff:172.21.15.164 - - [03/Jun/2023:19:43:26] "GET /metrics HTTP/1.1" 200 187050 "" "Prometheus/2.33.4" 2023-06-03T19:43:27.335 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:43:27 smithi164 bash[26855]: cluster 2023-06-03T19:43:26.007473+0000 mgr.y (mgr.34104) 86 : cluster [DBG] pgmap v70: 89 pgs: 89 active+clean; 6.7 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 85 B/s rd, 22 MiB/s wr, 671 op/s 2023-06-03T19:43:29.085 INFO:journalctl@ceph.prometheus.a.smithi164.stdout:Jun 03 19:43:28 smithi164 bash[55188]: ts=2023-06-03T19:43:28.661Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.13-257-gd8c5d349 (d8c5d34975dce1c5eb0aa3a7979a4d9b9a99d1ec) pacific (stable)\", cluster_addr=\"172.21.15.89\", device_class=\"ssd\", hostname=\"smithi089\", instance=\"smithi164.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.89\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.13-257-gd8c5d349 (d8c5d34975dce1c5eb0aa3a7979a4d9b9a99d1ec) pacific (stable)\", cluster_addr=\"172.21.15.89\", device_class=\"ssd\", hostname=\"smithi089\", instance=\"smithi089:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.89\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-06-03T19:43:29.085 INFO:journalctl@ceph.prometheus.a.smithi164.stdout:Jun 03 19:43:28 smithi164 bash[55188]: ts=2023-06-03T19:43:28.662Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephPGImbalance\nexpr: abs(((ceph_osd_numpg > 0) - on(job) group_left() avg by(job) (ceph_osd_numpg\n > 0)) / on(job) group_left() avg by(job) (ceph_osd_numpg > 0)) * on(ceph_daemon)\n group_left(hostname) ceph_osd_metadata > 0.3\nfor: 5m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.5\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} deviates by\n more than 30% from average PG count.\n summary: PGs are not balanced across OSDs\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.13-257-gd8c5d349 (d8c5d34975dce1c5eb0aa3a7979a4d9b9a99d1ec) pacific (stable)\", cluster_addr=\"172.21.15.89\", device_class=\"ssd\", hostname=\"smithi089\", instance=\"smithi164.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.89\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.13-257-gd8c5d349 (d8c5d34975dce1c5eb0aa3a7979a4d9b9a99d1ec) pacific (stable)\", cluster_addr=\"172.21.15.89\", device_class=\"ssd\", hostname=\"smithi089\", instance=\"smithi089:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.89\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-06-03T19:43:29.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:43:29 smithi164 bash[26855]: cluster 2023-06-03T19:43:28.007863+0000 mgr.y (mgr.34104) 87 : cluster [DBG] pgmap v71: 89 pgs: 89 active+clean; 6.7 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 85 B/s rd, 22 MiB/s wr, 671 op/s 2023-06-03T19:43:29.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:43:29 smithi164 bash[26855]: audit 2023-06-03T19:43:28.426725+0000 mon.a (mon.0) 88 : audit [INF] from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:43:29.836 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:43:29 smithi164 bash[26855]: audit 2023-06-03T19:43:28.437499+0000 mon.a (mon.0) 89 : audit [INF] from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:43:29.836 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:43:29 smithi164 bash[26855]: audit 2023-06-03T19:43:28.438006+0000 mon.b (mon.1) 101 : audit [DBG] from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-03T19:43:29.836 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:43:29 smithi164 bash[26855]: audit 2023-06-03T19:43:28.443925+0000 mon.b (mon.1) 102 : audit [INF] from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-03T19:43:29.836 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:43:29 smithi164 bash[26855]: audit 2023-06-03T19:43:28.452624+0000 mon.a (mon.0) 90 : audit [INF] from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:43:29.836 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:43:29 smithi164 bash[26855]: audit 2023-06-03T19:43:28.538105+0000 mon.b (mon.1) 103 : audit [DBG] from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-03T19:43:29.836 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:43:29 smithi164 bash[26855]: audit 2023-06-03T19:43:28.548595+0000 mon.b (mon.1) 104 : audit [DBG] from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-03T19:43:29.836 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:43:29 smithi164 bash[26855]: audit 2023-06-03T19:43:28.549819+0000 mon.b (mon.1) 105 : audit [INF] from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-03T19:43:29.837 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:43:29 smithi164 bash[26855]: audit 2023-06-03T19:43:28.561362+0000 mon.a (mon.0) 91 : audit [INF] from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:43:31.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:43:31 smithi164 bash[26855]: cluster 2023-06-03T19:43:30.008610+0000 mgr.y (mgr.34104) 88 : cluster [DBG] pgmap v72: 89 pgs: 89 active+clean; 6.5 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 85 B/s rd, 19 MiB/s wr, 969 op/s 2023-06-03T19:43:33.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:43:33 smithi164 bash[26855]: cluster 2023-06-03T19:43:32.009033+0000 mgr.y (mgr.34104) 89 : cluster [DBG] pgmap v73: 89 pgs: 89 active+clean; 6.5 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 85 B/s rd, 1.4 MiB/s wr, 684 op/s 2023-06-03T19:43:35.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:43:35 smithi164 bash[26855]: cluster 2023-06-03T19:43:34.009793+0000 mgr.y (mgr.34104) 90 : cluster [DBG] pgmap v74: 89 pgs: 89 active+clean; 6.2 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 85 B/s rd, 1.4 MiB/s wr, 1.02k op/s 2023-06-03T19:43:36.530 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 19:43:36 smithi089 bash[71163]: ::ffff:172.21.15.164 - - [03/Jun/2023:19:43:36] "GET /metrics HTTP/1.1" 200 187050 "" "Prometheus/2.33.4" 2023-06-03T19:43:37.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:43:37 smithi164 bash[26855]: cluster 2023-06-03T19:43:36.010253+0000 mgr.y (mgr.34104) 91 : cluster [DBG] pgmap v75: 89 pgs: 89 active+clean; 6.2 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 673 op/s 2023-06-03T19:43:37.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:43:37 smithi164 bash[26855]: audit 2023-06-03T19:43:37.062010+0000 mon.b (mon.1) 106 : audit [DBG] from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T19:43:39.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:43:39 smithi164 bash[26855]: cluster 2023-06-03T19:43:38.010741+0000 mgr.y (mgr.34104) 92 : cluster [DBG] pgmap v76: 89 pgs: 89 active+clean; 6.2 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 673 op/s 2023-06-03T19:43:41.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:43:41 smithi164 bash[26855]: cluster 2023-06-03T19:43:40.011468+0000 mgr.y (mgr.34104) 93 : cluster [DBG] pgmap v77: 89 pgs: 89 active+clean; 6.0 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 998 op/s 2023-06-03T19:43:43.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:43:43 smithi164 bash[26855]: cluster 2023-06-03T19:43:42.011843+0000 mgr.y (mgr.34104) 94 : cluster [DBG] pgmap v78: 89 pgs: 89 active+clean; 6.0 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 662 op/s 2023-06-03T19:43:45.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:43:45 smithi164 bash[26855]: cluster 2023-06-03T19:43:44.013850+0000 mgr.y (mgr.34104) 95 : cluster [DBG] pgmap v79: 89 pgs: 89 active+clean; 5.8 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 984 op/s 2023-06-03T19:43:46.530 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 19:43:46 smithi089 bash[71163]: ::ffff:172.21.15.164 - - [03/Jun/2023:19:43:46] "GET /metrics HTTP/1.1" 200 187107 "" "Prometheus/2.33.4" 2023-06-03T19:43:47.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:43:47 smithi164 bash[26855]: cluster 2023-06-03T19:43:46.014374+0000 mgr.y (mgr.34104) 96 : cluster [DBG] pgmap v80: 89 pgs: 89 active+clean; 5.8 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 646 op/s 2023-06-03T19:43:49.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:43:49 smithi164 bash[26855]: cluster 2023-06-03T19:43:48.014731+0000 mgr.y (mgr.34104) 97 : cluster [DBG] pgmap v81: 89 pgs: 89 active+clean; 5.8 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 647 op/s 2023-06-03T19:43:51.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:43:51 smithi164 bash[26855]: cluster 2023-06-03T19:43:50.015406+0000 mgr.y (mgr.34104) 98 : cluster [DBG] pgmap v82: 89 pgs: 89 active+clean; 5.5 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 965 op/s 2023-06-03T19:43:52.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:43:52 smithi164 bash[26855]: audit 2023-06-03T19:43:52.062182+0000 mon.b (mon.1) 107 : audit [DBG] from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T19:43:53.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:43:53 smithi164 bash[26855]: cluster 2023-06-03T19:43:52.016589+0000 mgr.y (mgr.34104) 99 : cluster [DBG] pgmap v83: 89 pgs: 89 active+clean; 5.5 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 640 op/s 2023-06-03T19:43:55.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:43:55 smithi164 bash[26855]: cluster 2023-06-03T19:43:54.017331+0000 mgr.y (mgr.34104) 100 : cluster [DBG] pgmap v84: 89 pgs: 89 active+clean; 5.3 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 961 op/s 2023-06-03T19:43:56.530 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 19:43:56 smithi089 bash[71163]: ::ffff:172.21.15.164 - - [03/Jun/2023:19:43:56] "GET /metrics HTTP/1.1" 200 187110 "" "Prometheus/2.33.4" 2023-06-03T19:43:57.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:43:57 smithi164 bash[26855]: cluster 2023-06-03T19:43:56.017789+0000 mgr.y (mgr.34104) 101 : cluster [DBG] pgmap v85: 89 pgs: 89 active+clean; 5.3 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 639 op/s 2023-06-03T19:43:59.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:43:59 smithi164 bash[26855]: cluster 2023-06-03T19:43:58.020033+0000 mgr.y (mgr.34104) 102 : cluster [DBG] pgmap v86: 89 pgs: 89 active+clean; 5.3 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 639 op/s 2023-06-03T19:44:01.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:01 smithi164 bash[26855]: cluster 2023-06-03T19:44:00.020801+0000 mgr.y (mgr.34104) 103 : cluster [DBG] pgmap v87: 89 pgs: 89 active+clean; 5.1 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 951 op/s 2023-06-03T19:44:03.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:03 smithi164 bash[26855]: cluster 2023-06-03T19:44:02.021222+0000 mgr.y (mgr.34104) 104 : cluster [DBG] pgmap v88: 89 pgs: 89 active+clean; 5.1 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 632 op/s 2023-06-03T19:44:05.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:05 smithi164 bash[26855]: cluster 2023-06-03T19:44:04.025387+0000 mgr.y (mgr.34104) 105 : cluster [DBG] pgmap v89: 89 pgs: 89 active+clean; 4.8 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 942 op/s 2023-06-03T19:44:06.530 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 19:44:06 smithi089 bash[71163]: ::ffff:172.21.15.164 - - [03/Jun/2023:19:44:06] "GET /metrics HTTP/1.1" 200 187110 "" "Prometheus/2.33.4" 2023-06-03T19:44:07.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:07 smithi164 bash[26855]: cluster 2023-06-03T19:44:06.025820+0000 mgr.y (mgr.34104) 106 : cluster [DBG] pgmap v90: 89 pgs: 89 active+clean; 4.8 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 621 op/s 2023-06-03T19:44:07.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:07 smithi164 bash[26855]: audit 2023-06-03T19:44:07.062353+0000 mon.b (mon.1) 108 : audit [DBG] from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T19:44:09.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:09 smithi164 bash[26855]: cluster 2023-06-03T19:44:08.026225+0000 mgr.y (mgr.34104) 107 : cluster [DBG] pgmap v91: 89 pgs: 89 active+clean; 4.8 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 621 op/s 2023-06-03T19:44:11.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:11 smithi164 bash[26855]: cluster 2023-06-03T19:44:10.026843+0000 mgr.y (mgr.34104) 108 : cluster [DBG] pgmap v92: 89 pgs: 89 active+clean; 4.6 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 932 op/s 2023-06-03T19:44:13.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:13 smithi164 bash[26855]: cluster 2023-06-03T19:44:12.027279+0000 mgr.y (mgr.34104) 109 : cluster [DBG] pgmap v93: 89 pgs: 89 active+clean; 4.6 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 620 op/s 2023-06-03T19:44:15.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:15 smithi164 bash[26855]: cluster 2023-06-03T19:44:14.028044+0000 mgr.y (mgr.34104) 110 : cluster [DBG] pgmap v94: 89 pgs: 89 active+clean; 4.4 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 925 op/s 2023-06-03T19:44:16.530 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 19:44:16 smithi089 bash[71163]: ::ffff:172.21.15.164 - - [03/Jun/2023:19:44:16] "GET /metrics HTTP/1.1" 200 187117 "" "Prometheus/2.33.4" 2023-06-03T19:44:17.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:17 smithi164 bash[26855]: cluster 2023-06-03T19:44:16.028802+0000 mgr.y (mgr.34104) 111 : cluster [DBG] pgmap v95: 89 pgs: 89 active+clean; 4.4 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 615 op/s 2023-06-03T19:44:19.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:19 smithi164 bash[26855]: cluster 2023-06-03T19:44:18.029228+0000 mgr.y (mgr.34104) 112 : cluster [DBG] pgmap v96: 89 pgs: 89 active+clean; 4.4 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 615 op/s 2023-06-03T19:44:21.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:21 smithi164 bash[26855]: cluster 2023-06-03T19:44:20.030120+0000 mgr.y (mgr.34104) 113 : cluster [DBG] pgmap v97: 89 pgs: 89 active+clean; 4.2 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 920 op/s 2023-06-03T19:44:22.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:22 smithi164 bash[26855]: audit 2023-06-03T19:44:22.062698+0000 mon.b (mon.1) 109 : audit [DBG] from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T19:44:24.085 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:23 smithi164 bash[26855]: cluster 2023-06-03T19:44:22.031492+0000 mgr.y (mgr.34104) 114 : cluster [DBG] pgmap v98: 89 pgs: 89 active+clean; 4.2 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 609 op/s 2023-06-03T19:44:26.085 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:25 smithi164 bash[26855]: cluster 2023-06-03T19:44:24.032237+0000 mgr.y (mgr.34104) 115 : cluster [DBG] pgmap v99: 89 pgs: 89 active+clean; 3.9 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 917 op/s 2023-06-03T19:44:26.647 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 19:44:26 smithi089 bash[71163]: ::ffff:172.21.15.164 - - [03/Jun/2023:19:44:26] "GET /metrics HTTP/1.1" 200 187114 "" "Prometheus/2.33.4" 2023-06-03T19:44:27.917 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -e sha1=8e24bcf462a1d94c2e12aa4e26419634883a1e88 -- bash -c 'ceph orch upgrade resume' 2023-06-03T19:44:28.085 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:27 smithi164 bash[26855]: cluster 2023-06-03T19:44:26.032663+0000 mgr.y (mgr.34104) 116 : cluster [DBG] pgmap v100: 89 pgs: 89 active+clean; 3.9 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 612 op/s 2023-06-03T19:44:30.085 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:29 smithi164 bash[26855]: cluster 2023-06-03T19:44:28.033077+0000 mgr.y (mgr.34104) 117 : cluster [DBG] pgmap v101: 89 pgs: 89 active+clean; 3.9 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 612 op/s 2023-06-03T19:44:30.085 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:29 smithi164 bash[26855]: audit 2023-06-03T19:44:28.649932+0000 mon.b (mon.1) 110 : audit [DBG] from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-03T19:44:30.085 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:29 smithi164 bash[26855]: audit 2023-06-03T19:44:29.128473+0000 mon.a (mon.0) 92 : audit [INF] from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:44:30.086 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:29 smithi164 bash[26855]: audit 2023-06-03T19:44:29.150722+0000 mon.a (mon.0) 93 : audit [INF] from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:44:32.085 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:31 smithi164 bash[26855]: cluster 2023-06-03T19:44:30.033866+0000 mgr.y (mgr.34104) 118 : cluster [DBG] pgmap v102: 89 pgs: 89 active+clean; 3.7 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 911 op/s 2023-06-03T19:44:32.563 INFO:teuthology.orchestra.run.smithi089.stdout:Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:00b1b6d689278b4f97ff790045c8c51d8ca419ac7ee2ebdf11008f2e810a5c78 2023-06-03T19:44:33.581 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -e sha1=8e24bcf462a1d94c2e12aa4e26419634883a1e88 -- bash -c 'echo wait for all mons to upgrade' 2023-06-03T19:44:33.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:33 smithi164 bash[26855]: cluster 2023-06-03T19:44:32.034321+0000 mgr.y (mgr.34104) 119 : cluster [DBG] pgmap v103: 89 pgs: 89 active+clean; 3.7 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 607 op/s 2023-06-03T19:44:33.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:33 smithi164 bash[26855]: audit 2023-06-03T19:44:32.543854+0000 mgr.y (mgr.34104) 120 : audit [DBG] from='client.34215 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2023-06-03T19:44:33.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:33 smithi164 bash[26855]: cephadm 2023-06-03T19:44:32.544300+0000 mgr.y (mgr.34104) 121 : cephadm [INF] Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:00b1b6d689278b4f97ff790045c8c51d8ca419ac7ee2ebdf11008f2e810a5c78 2023-06-03T19:44:33.836 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:33 smithi164 bash[26855]: audit 2023-06-03T19:44:32.556028+0000 mon.a (mon.0) 94 : audit [INF] from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:44:35.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:35 smithi164 bash[26855]: cluster 2023-06-03T19:44:34.035017+0000 mgr.y (mgr.34104) 122 : cluster [DBG] pgmap v104: 89 pgs: 89 active+clean; 3.5 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 906 op/s 2023-06-03T19:44:35.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:35 smithi164 bash[26855]: audit 2023-06-03T19:44:35.368125+0000 mon.a (mon.0) 95 : audit [INF] from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:44:35.836 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:35 smithi164 bash[26855]: audit 2023-06-03T19:44:35.378597+0000 mon.a (mon.0) 96 : audit [INF] from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:44:36.530 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 19:44:36 smithi089 bash[71163]: ::ffff:172.21.15.164 - - [03/Jun/2023:19:44:36] "GET /metrics HTTP/1.1" 200 187114 "" "Prometheus/2.33.4" 2023-06-03T19:44:36.965 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:36 smithi164 bash[26855]: audit 2023-06-03T19:44:35.664963+0000 mon.a (mon.0) 97 : audit [INF] from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:44:36.965 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:36 smithi164 bash[26855]: audit 2023-06-03T19:44:35.676306+0000 mon.a (mon.0) 98 : audit [INF] from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:44:36.966 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:36 smithi164 bash[26855]: cluster 2023-06-03T19:44:36.035439+0000 mgr.y (mgr.34104) 123 : cluster [DBG] pgmap v105: 89 pgs: 89 active+clean; 3.5 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 598 op/s 2023-06-03T19:44:36.966 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:36 smithi164 bash[26855]: audit 2023-06-03T19:44:36.267128+0000 mon.b (mon.1) 111 : audit [DBG] from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-03T19:44:36.966 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:36 smithi164 bash[26855]: audit 2023-06-03T19:44:36.268790+0000 mon.b (mon.1) 112 : audit [INF] from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-03T19:44:36.967 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:36 smithi164 bash[26855]: audit 2023-06-03T19:44:36.280446+0000 mon.a (mon.0) 99 : audit [INF] from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:44:36.967 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:36 smithi164 bash[26855]: audit 2023-06-03T19:44:36.350930+0000 mon.b (mon.1) 113 : audit [DBG] from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-03T19:44:36.967 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:36 smithi164 bash[26855]: audit 2023-06-03T19:44:36.356554+0000 mon.b (mon.1) 114 : audit [DBG] from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-06-03T19:44:36.968 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:36 smithi164 bash[26855]: audit 2023-06-03T19:44:36.358926+0000 mon.b (mon.1) 115 : audit [DBG] from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2023-06-03T19:44:36.968 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:36 smithi164 bash[26855]: audit 2023-06-03T19:44:36.360274+0000 mon.b (mon.1) 116 : audit [DBG] from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["b"]}]: dispatch 2023-06-03T19:44:36.968 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:36 smithi164 bash[26855]: cephadm 2023-06-03T19:44:36.363231+0000 mgr.y (mgr.34104) 124 : cephadm [INF] Upgrade: It appears safe to stop mon.b 2023-06-03T19:44:37.510 INFO:teuthology.orchestra.run.smithi089.stdout:wait for all mons to upgrade 2023-06-03T19:44:38.085 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:37 smithi164 bash[26855]: audit 2023-06-03T19:44:37.063044+0000 mon.b (mon.1) 117 : audit [DBG] from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T19:44:38.471 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -e sha1=8e24bcf462a1d94c2e12aa4e26419634883a1e88 -- bash -c 'while ! ceph mon versions | grep $sha1 | grep '"'"': 3'"'"' ; do sleep 2 ; done' 2023-06-03T19:44:39.085 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:38 smithi164 bash[26855]: cluster 2023-06-03T19:44:38.035884+0000 mgr.y (mgr.34104) 125 : cluster [DBG] pgmap v106: 89 pgs: 89 active+clean; 3.5 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 598 op/s 2023-06-03T19:44:41.585 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:41 smithi164 bash[26855]: cluster 2023-06-03T19:44:40.036649+0000 mgr.y (mgr.34104) 126 : cluster [DBG] pgmap v107: 89 pgs: 89 active+clean; 3.3 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 875 op/s 2023-06-03T19:44:42.838 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:42 smithi164 bash[26855]: cephadm 2023-06-03T19:44:41.464377+0000 mgr.y (mgr.34104) 127 : cephadm [INF] Upgrade: Updating mon.b 2023-06-03T19:44:42.839 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:42 smithi164 bash[26855]: audit 2023-06-03T19:44:41.487627+0000 mon.b (mon.1) 118 : audit [INF] from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-06-03T19:44:42.839 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:42 smithi164 bash[26855]: audit 2023-06-03T19:44:41.488087+0000 mon.a (mon.0) 100 : audit [INF] from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:44:42.839 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:42 smithi164 bash[26855]: audit 2023-06-03T19:44:41.490109+0000 mon.b (mon.1) 119 : audit [DBG] from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-03T19:44:42.839 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:42 smithi164 bash[26855]: audit 2023-06-03T19:44:41.490937+0000 mon.a (mon.0) 101 : audit [DBG] from='mgr.34104 ' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-06-03T19:44:42.839 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:42 smithi164 bash[26855]: cephadm 2023-06-03T19:44:41.493306+0000 mgr.y (mgr.34104) 128 : cephadm [INF] Deploying daemon mon.b on smithi164 2023-06-03T19:44:43.495 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:43 smithi164 bash[26855]: cluster 2023-06-03T19:44:42.039697+0000 mgr.y (mgr.34104) 129 : cluster [DBG] pgmap v108: 89 pgs: 89 active+clean; 3.3 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 575 op/s 2023-06-03T19:44:43.495 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:43 smithi164 bash[26855]: audit 2023-06-03T19:44:42.979539+0000 mon.c (mon.2) 2 : audit [DBG] from='client.? 172.21.15.89:0/4084515067' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-03T19:44:44.583 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:44 smithi164 systemd[1]: Stopping Ceph mon.b for f1db3478-0243-11ee-9b1d-001a4aab830c... 2023-06-03T19:44:44.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:44 smithi164 bash[26855]: debug 2023-06-03T19:44:44.745+0000 7f5333be3700 -1 received signal: Terminated from /sbin/docker-init -- /usr/bin/ceph-mon -n mon.b -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-stderr=true (PID: 1) UID: 0 2023-06-03T19:44:44.836 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:44 smithi164 bash[26855]: debug 2023-06-03T19:44:44.745+0000 7f5333be3700 -1 mon.b@1(peon) e3 *** Got Signal Terminated *** 2023-06-03T19:44:45.030 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 19:44:44 smithi089 bash[71163]: [03/Jun/2023:19:44:44] ENGINE Bus STOPPING 2023-06-03T19:44:45.030 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 19:44:44 smithi089 bash[71163]: [03/Jun/2023:19:44:44] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2023-06-03T19:44:45.031 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 19:44:44 smithi089 bash[71163]: [03/Jun/2023:19:44:44] ENGINE Bus STOPPED 2023-06-03T19:44:45.031 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 19:44:44 smithi089 bash[71163]: [03/Jun/2023:19:44:44] ENGINE Bus STARTING 2023-06-03T19:44:45.031 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 19:44:44 smithi089 bash[71163]: [03/Jun/2023:19:44:44] ENGINE Serving on http://:::9283 2023-06-03T19:44:45.031 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 19:44:44 smithi089 bash[71163]: [03/Jun/2023:19:44:44] ENGINE Bus STARTED 2023-06-03T19:44:46.335 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:45 smithi164 bash[59351]: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-mon-b 2023-06-03T19:44:46.336 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:46 smithi164 systemd[1]: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c@mon.b.service: Succeeded. 2023-06-03T19:44:46.336 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:46 smithi164 systemd[1]: Stopped Ceph mon.b for f1db3478-0243-11ee-9b1d-001a4aab830c. 2023-06-03T19:44:46.530 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 19:44:46 smithi089 bash[71163]: ::ffff:172.21.15.164 - - [03/Jun/2023:19:44:46] "GET /metrics HTTP/1.1" 200 187129 "" "Prometheus/2.33.4" 2023-06-03T19:44:46.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:46 smithi164 systemd[1]: Started Ceph mon.b for f1db3478-0243-11ee-9b1d-001a4aab830c. 2023-06-03T19:44:48.086 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:47 smithi164 bash[59462]: debug 2023-06-03T19:44:47.828+0000 7fdac79e1c80 0 set uid:gid to 167:167 (ceph:ceph) 2023-06-03T19:44:48.086 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:47 smithi164 bash[59462]: debug 2023-06-03T19:44:47.828+0000 7fdac79e1c80 0 ceph version 18.0.0-3532-g8e24bcf4 (8e24bcf462a1d94c2e12aa4e26419634883a1e88) reef (rc), process ceph-mon, pid 7 2023-06-03T19:44:48.086 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:47 smithi164 bash[59462]: debug 2023-06-03T19:44:47.828+0000 7fdac79e1c80 0 pidfile_write: ignore empty --pid-file 2023-06-03T19:44:48.086 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:47 smithi164 bash[59462]: debug 2023-06-03T19:44:47.836+0000 7fdac79e1c80 0 load: jerasure load: lrc 2023-06-03T19:44:48.086 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:47 smithi164 bash[59462]: debug 2023-06-03T19:44:47.836+0000 7fdac79e1c80 4 rocksdb: RocksDB version: 7.9.2 2023-06-03T19:44:48.086 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:47 smithi164 bash[59462]: debug 2023-06-03T19:44:47.836+0000 7fdac79e1c80 4 rocksdb: Git sha 0 2023-06-03T19:44:48.087 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:47 smithi164 bash[59462]: debug 2023-06-03T19:44:47.836+0000 7fdac79e1c80 4 rocksdb: Compile date 2023-06-01 15:16:58 2023-06-03T19:44:48.087 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:47 smithi164 bash[59462]: debug 2023-06-03T19:44:47.836+0000 7fdac79e1c80 4 rocksdb: DB SUMMARY 2023-06-03T19:44:48.087 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:47 smithi164 bash[59462]: debug 2023-06-03T19:44:47.836+0000 7fdac79e1c80 4 rocksdb: DB Session ID: MER74KV56Z66HDSBFJCC 2023-06-03T19:44:48.087 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:47 smithi164 bash[59462]: debug 2023-06-03T19:44:47.836+0000 7fdac79e1c80 4 rocksdb: CURRENT file: CURRENT 2023-06-03T19:44:48.087 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:47 smithi164 bash[59462]: debug 2023-06-03T19:44:47.836+0000 7fdac79e1c80 4 rocksdb: IDENTITY file: IDENTITY 2023-06-03T19:44:48.087 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:47 smithi164 bash[59462]: debug 2023-06-03T19:44:47.836+0000 7fdac79e1c80 4 rocksdb: MANIFEST file: MANIFEST-000005 size: 1848 Bytes 2023-06-03T19:44:48.087 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:47 smithi164 bash[59462]: debug 2023-06-03T19:44:47.836+0000 7fdac79e1c80 4 rocksdb: SST files in /var/lib/ceph/mon/ceph-b/store.db dir, Total Num: 1, files: 000035.sst 2023-06-03T19:44:48.088 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:47 smithi164 bash[59462]: debug 2023-06-03T19:44:47.836+0000 7fdac79e1c80 4 rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-b/store.db: 000033.log size: 545399 ; 2023-06-03T19:44:48.088 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:47 smithi164 bash[59462]: debug 2023-06-03T19:44:47.836+0000 7fdac79e1c80 4 rocksdb: Options.error_if_exists: 0 2023-06-03T19:44:48.088 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:47 smithi164 bash[59462]: debug 2023-06-03T19:44:47.836+0000 7fdac79e1c80 4 rocksdb: Options.create_if_missing: 0 2023-06-03T19:44:48.088 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:47 smithi164 bash[59462]: debug 2023-06-03T19:44:47.836+0000 7fdac79e1c80 4 rocksdb: Options.paranoid_checks: 1 2023-06-03T19:44:48.088 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:47 smithi164 bash[59462]: debug 2023-06-03T19:44:47.836+0000 7fdac79e1c80 4 rocksdb: Options.flush_verify_memtable_count: 1 2023-06-03T19:44:48.088 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:47 smithi164 bash[59462]: debug 2023-06-03T19:44:47.836+0000 7fdac79e1c80 4 rocksdb: Options.track_and_verify_wals_in_manifest: 0 2023-06-03T19:44:48.088 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:47 smithi164 bash[59462]: debug 2023-06-03T19:44:47.836+0000 7fdac79e1c80 4 rocksdb: Options.verify_sst_unique_id_in_manifest: 1 2023-06-03T19:44:48.089 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:47 smithi164 bash[59462]: debug 2023-06-03T19:44:47.836+0000 7fdac79e1c80 4 rocksdb: Options.env: 0x55ce48250720 2023-06-03T19:44:48.089 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:47 smithi164 bash[59462]: debug 2023-06-03T19:44:47.836+0000 7fdac79e1c80 4 rocksdb: Options.fs: PosixFileSystem 2023-06-03T19:44:48.089 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:47 smithi164 bash[59462]: debug 2023-06-03T19:44:47.836+0000 7fdac79e1c80 4 rocksdb: Options.info_log: 0x55ce4b369460 2023-06-03T19:44:48.089 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:47 smithi164 bash[59462]: debug 2023-06-03T19:44:47.836+0000 7fdac79e1c80 4 rocksdb: Options.max_file_opening_threads: 16 2023-06-03T19:44:48.089 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:47 smithi164 bash[59462]: debug 2023-06-03T19:44:47.836+0000 7fdac79e1c80 4 rocksdb: Options.statistics: (nil) 2023-06-03T19:44:48.089 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:47 smithi164 bash[59462]: debug 2023-06-03T19:44:47.836+0000 7fdac79e1c80 4 rocksdb: Options.use_fsync: 0 2023-06-03T19:44:48.089 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:47 smithi164 bash[59462]: debug 2023-06-03T19:44:47.836+0000 7fdac79e1c80 4 rocksdb: Options.max_log_file_size: 0 2023-06-03T19:44:48.089 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:47 smithi164 bash[59462]: debug 2023-06-03T19:44:47.836+0000 7fdac79e1c80 4 rocksdb: Options.max_manifest_file_size: 1073741824 2023-06-03T19:44:48.090 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:47 smithi164 bash[59462]: debug 2023-06-03T19:44:47.836+0000 7fdac79e1c80 4 rocksdb: Options.log_file_time_to_roll: 0 2023-06-03T19:44:48.090 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:47 smithi164 bash[59462]: debug 2023-06-03T19:44:47.836+0000 7fdac79e1c80 4 rocksdb: Options.keep_log_file_num: 1000 2023-06-03T19:44:48.090 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:47 smithi164 bash[59462]: debug 2023-06-03T19:44:47.836+0000 7fdac79e1c80 4 rocksdb: Options.recycle_log_file_num: 0 2023-06-03T19:44:48.090 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:47 smithi164 bash[59462]: debug 2023-06-03T19:44:47.836+0000 7fdac79e1c80 4 rocksdb: Options.allow_fallocate: 1 2023-06-03T19:44:48.090 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:47 smithi164 bash[59462]: debug 2023-06-03T19:44:47.836+0000 7fdac79e1c80 4 rocksdb: Options.allow_mmap_reads: 0 2023-06-03T19:44:48.090 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:47 smithi164 bash[59462]: debug 2023-06-03T19:44:47.836+0000 7fdac79e1c80 4 rocksdb: Options.allow_mmap_writes: 0 2023-06-03T19:44:48.090 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:47 smithi164 bash[59462]: debug 2023-06-03T19:44:47.836+0000 7fdac79e1c80 4 rocksdb: Options.use_direct_reads: 0 2023-06-03T19:44:48.090 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:47 smithi164 bash[59462]: debug 2023-06-03T19:44:47.836+0000 7fdac79e1c80 4 rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2023-06-03T19:44:48.091 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:47 smithi164 bash[59462]: debug 2023-06-03T19:44:47.836+0000 7fdac79e1c80 4 rocksdb: Options.create_missing_column_families: 0 2023-06-03T19:44:48.091 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:47 smithi164 bash[59462]: debug 2023-06-03T19:44:47.836+0000 7fdac79e1c80 4 rocksdb: Options.db_log_dir: 2023-06-03T19:44:48.091 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:47 smithi164 bash[59462]: debug 2023-06-03T19:44:47.836+0000 7fdac79e1c80 4 rocksdb: Options.wal_dir: 2023-06-03T19:44:48.091 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:47 smithi164 bash[59462]: debug 2023-06-03T19:44:47.836+0000 7fdac79e1c80 4 rocksdb: Options.table_cache_numshardbits: 6 2023-06-03T19:44:48.091 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:47 smithi164 bash[59462]: debug 2023-06-03T19:44:47.836+0000 7fdac79e1c80 4 rocksdb: Options.WAL_ttl_seconds: 0 2023-06-03T19:44:48.091 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:47 smithi164 bash[59462]: debug 2023-06-03T19:44:47.836+0000 7fdac79e1c80 4 rocksdb: Options.WAL_size_limit_MB: 0 2023-06-03T19:44:48.091 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:47 smithi164 bash[59462]: debug 2023-06-03T19:44:47.836+0000 7fdac79e1c80 4 rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2023-06-03T19:44:48.092 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:47 smithi164 bash[59462]: debug 2023-06-03T19:44:47.836+0000 7fdac79e1c80 4 rocksdb: Options.manifest_preallocation_size: 4194304 2023-06-03T19:44:48.092 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:47 smithi164 bash[59462]: debug 2023-06-03T19:44:47.836+0000 7fdac79e1c80 4 rocksdb: Options.is_fd_close_on_exec: 1 2023-06-03T19:44:48.092 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:47 smithi164 bash[59462]: debug 2023-06-03T19:44:47.836+0000 7fdac79e1c80 4 rocksdb: Options.advise_random_on_open: 1 2023-06-03T19:44:48.092 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:47 smithi164 bash[59462]: debug 2023-06-03T19:44:47.836+0000 7fdac79e1c80 4 rocksdb: Options.db_write_buffer_size: 0 2023-06-03T19:44:48.092 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:47 smithi164 bash[59462]: debug 2023-06-03T19:44:47.836+0000 7fdac79e1c80 4 rocksdb: Options.write_buffer_manager: 0x55ce4a5f6c80 2023-06-03T19:44:48.092 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:47 smithi164 bash[59462]: debug 2023-06-03T19:44:47.836+0000 7fdac79e1c80 4 rocksdb: Options.access_hint_on_compaction_start: 1 2023-06-03T19:44:48.092 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:47 smithi164 bash[59462]: debug 2023-06-03T19:44:47.836+0000 7fdac79e1c80 4 rocksdb: Options.random_access_max_buffer_size: 1048576 2023-06-03T19:44:48.092 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:47 smithi164 bash[59462]: debug 2023-06-03T19:44:47.836+0000 7fdac79e1c80 4 rocksdb: Options.use_adaptive_mutex: 0 2023-06-03T19:44:48.093 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:47 smithi164 bash[59462]: debug 2023-06-03T19:44:47.836+0000 7fdac79e1c80 4 rocksdb: Options.rate_limiter: (nil) 2023-06-03T19:44:48.093 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:47 smithi164 bash[59462]: debug 2023-06-03T19:44:47.836+0000 7fdac79e1c80 4 rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2023-06-03T19:44:48.093 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:47 smithi164 bash[59462]: debug 2023-06-03T19:44:47.836+0000 7fdac79e1c80 4 rocksdb: Options.wal_recovery_mode: 2 2023-06-03T19:44:48.094 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:47 smithi164 bash[59462]: debug 2023-06-03T19:44:47.836+0000 7fdac79e1c80 4 rocksdb: Options.enable_thread_tracking: 0 2023-06-03T19:44:48.095 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:47 smithi164 bash[59462]: debug 2023-06-03T19:44:47.836+0000 7fdac79e1c80 4 rocksdb: Options.enable_pipelined_write: 0 2023-06-03T19:44:48.095 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:47 smithi164 bash[59462]: debug 2023-06-03T19:44:47.836+0000 7fdac79e1c80 4 rocksdb: Options.unordered_write: 0 2023-06-03T19:44:48.095 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:47 smithi164 bash[59462]: debug 2023-06-03T19:44:47.836+0000 7fdac79e1c80 4 rocksdb: Options.allow_concurrent_memtable_write: 1 2023-06-03T19:44:48.095 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:47 smithi164 bash[59462]: debug 2023-06-03T19:44:47.836+0000 7fdac79e1c80 4 rocksdb: Options.enable_write_thread_adaptive_yield: 1 2023-06-03T19:44:48.095 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:47 smithi164 bash[59462]: debug 2023-06-03T19:44:47.836+0000 7fdac79e1c80 4 rocksdb: Options.write_thread_max_yield_usec: 100 2023-06-03T19:44:48.095 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:47 smithi164 bash[59462]: debug 2023-06-03T19:44:47.836+0000 7fdac79e1c80 4 rocksdb: Options.write_thread_slow_yield_usec: 3 2023-06-03T19:44:48.096 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:47 smithi164 bash[59462]: debug 2023-06-03T19:44:47.836+0000 7fdac79e1c80 4 rocksdb: Options.row_cache: None 2023-06-03T19:44:48.096 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:47 smithi164 bash[59462]: debug 2023-06-03T19:44:47.836+0000 7fdac79e1c80 4 rocksdb: Options.wal_filter: None 2023-06-03T19:44:48.096 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:47 smithi164 bash[59462]: debug 2023-06-03T19:44:47.836+0000 7fdac79e1c80 4 rocksdb: Options.avoid_flush_during_recovery: 0 2023-06-03T19:44:48.096 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:47 smithi164 bash[59462]: debug 2023-06-03T19:44:47.836+0000 7fdac79e1c80 4 rocksdb: Options.allow_ingest_behind: 0 2023-06-03T19:44:48.096 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:47 smithi164 bash[59462]: debug 2023-06-03T19:44:47.836+0000 7fdac79e1c80 4 rocksdb: Options.two_write_queues: 0 2023-06-03T19:44:48.096 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:47 smithi164 bash[59462]: debug 2023-06-03T19:44:47.836+0000 7fdac79e1c80 4 rocksdb: Options.manual_wal_flush: 0 2023-06-03T19:44:48.096 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:47 smithi164 bash[59462]: debug 2023-06-03T19:44:47.836+0000 7fdac79e1c80 4 rocksdb: Options.wal_compression: 0 2023-06-03T19:44:48.096 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:47 smithi164 bash[59462]: debug 2023-06-03T19:44:47.836+0000 7fdac79e1c80 4 rocksdb: Options.atomic_flush: 0 2023-06-03T19:44:48.097 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:47 smithi164 bash[59462]: debug 2023-06-03T19:44:47.836+0000 7fdac79e1c80 4 rocksdb: Options.avoid_unnecessary_blocking_io: 0 2023-06-03T19:44:48.097 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:47 smithi164 bash[59462]: debug 2023-06-03T19:44:47.836+0000 7fdac79e1c80 4 rocksdb: Options.persist_stats_to_disk: 0 2023-06-03T19:44:48.097 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:47 smithi164 bash[59462]: debug 2023-06-03T19:44:47.836+0000 7fdac79e1c80 4 rocksdb: Options.write_dbid_to_manifest: 0 2023-06-03T19:44:48.097 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:47 smithi164 bash[59462]: debug 2023-06-03T19:44:47.836+0000 7fdac79e1c80 4 rocksdb: Options.log_readahead_size: 0 2023-06-03T19:44:48.097 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:47 smithi164 bash[59462]: debug 2023-06-03T19:44:47.836+0000 7fdac79e1c80 4 rocksdb: Options.file_checksum_gen_factory: Unknown 2023-06-03T19:44:48.097 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:47 smithi164 bash[59462]: debug 2023-06-03T19:44:47.836+0000 7fdac79e1c80 4 rocksdb: Options.best_efforts_recovery: 0 2023-06-03T19:44:48.097 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:47 smithi164 bash[59462]: debug 2023-06-03T19:44:47.836+0000 7fdac79e1c80 4 rocksdb: Options.max_bgerror_resume_count: 2147483647 2023-06-03T19:44:48.098 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:47 smithi164 bash[59462]: debug 2023-06-03T19:44:47.836+0000 7fdac79e1c80 4 rocksdb: Options.bgerror_resume_retry_interval: 1000000 2023-06-03T19:44:48.098 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:47 smithi164 bash[59462]: debug 2023-06-03T19:44:47.836+0000 7fdac79e1c80 4 rocksdb: Options.allow_data_in_errors: 0 2023-06-03T19:44:48.098 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:47 smithi164 bash[59462]: debug 2023-06-03T19:44:47.836+0000 7fdac79e1c80 4 rocksdb: Options.db_host_id: __hostname__ 2023-06-03T19:44:48.098 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:47 smithi164 bash[59462]: debug 2023-06-03T19:44:47.836+0000 7fdac79e1c80 4 rocksdb: Options.enforce_single_del_contracts: true 2023-06-03T19:44:48.098 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:47 smithi164 bash[59462]: debug 2023-06-03T19:44:47.836+0000 7fdac79e1c80 4 rocksdb: Options.max_background_jobs: 2 2023-06-03T19:44:48.098 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:47 smithi164 bash[59462]: debug 2023-06-03T19:44:47.836+0000 7fdac79e1c80 4 rocksdb: Options.max_background_compactions: -1 2023-06-03T19:44:48.098 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:47 smithi164 bash[59462]: debug 2023-06-03T19:44:47.836+0000 7fdac79e1c80 4 rocksdb: Options.max_subcompactions: 1 2023-06-03T19:44:48.098 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:47 smithi164 bash[59462]: debug 2023-06-03T19:44:47.836+0000 7fdac79e1c80 4 rocksdb: Options.avoid_flush_during_shutdown: 0 2023-06-03T19:44:48.099 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:47 smithi164 bash[59462]: debug 2023-06-03T19:44:47.836+0000 7fdac79e1c80 4 rocksdb: Options.writable_file_max_buffer_size: 1048576 2023-06-03T19:44:48.099 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:47 smithi164 bash[59462]: debug 2023-06-03T19:44:47.836+0000 7fdac79e1c80 4 rocksdb: Options.delayed_write_rate : 16777216 2023-06-03T19:44:48.099 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:47 smithi164 bash[59462]: debug 2023-06-03T19:44:47.836+0000 7fdac79e1c80 4 rocksdb: Options.max_total_wal_size: 0 2023-06-03T19:44:48.099 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:47 smithi164 bash[59462]: debug 2023-06-03T19:44:47.836+0000 7fdac79e1c80 4 rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2023-06-03T19:44:48.099 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:47 smithi164 bash[59462]: debug 2023-06-03T19:44:47.836+0000 7fdac79e1c80 4 rocksdb: Options.stats_dump_period_sec: 600 2023-06-03T19:44:48.099 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:47 smithi164 bash[59462]: debug 2023-06-03T19:44:47.836+0000 7fdac79e1c80 4 rocksdb: Options.stats_persist_period_sec: 600 2023-06-03T19:44:48.099 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:47 smithi164 bash[59462]: debug 2023-06-03T19:44:47.836+0000 7fdac79e1c80 4 rocksdb: Options.stats_history_buffer_size: 1048576 2023-06-03T19:44:48.099 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:47 smithi164 bash[59462]: debug 2023-06-03T19:44:47.836+0000 7fdac79e1c80 4 rocksdb: Options.max_open_files: -1 2023-06-03T19:44:48.100 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:47 smithi164 bash[59462]: debug 2023-06-03T19:44:47.836+0000 7fdac79e1c80 4 rocksdb: Options.bytes_per_sync: 0 2023-06-03T19:44:48.100 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:47 smithi164 bash[59462]: debug 2023-06-03T19:44:47.836+0000 7fdac79e1c80 4 rocksdb: Options.wal_bytes_per_sync: 0 2023-06-03T19:44:48.100 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:47 smithi164 bash[59462]: debug 2023-06-03T19:44:47.836+0000 7fdac79e1c80 4 rocksdb: Options.strict_bytes_per_sync: 0 2023-06-03T19:44:48.100 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:47 smithi164 bash[59462]: debug 2023-06-03T19:44:47.836+0000 7fdac79e1c80 4 rocksdb: Options.compaction_readahead_size: 0 2023-06-03T19:44:48.100 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:47 smithi164 bash[59462]: debug 2023-06-03T19:44:47.836+0000 7fdac79e1c80 4 rocksdb: Options.max_background_flushes: -1 2023-06-03T19:44:48.100 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:47 smithi164 bash[59462]: debug 2023-06-03T19:44:47.836+0000 7fdac79e1c80 4 rocksdb: Compression algorithms supported: 2023-06-03T19:44:48.100 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:47 smithi164 bash[59462]: debug 2023-06-03T19:44:47.836+0000 7fdac79e1c80 4 rocksdb: kZSTDNotFinalCompression supported: 0 2023-06-03T19:44:48.100 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:47 smithi164 bash[59462]: debug 2023-06-03T19:44:47.836+0000 7fdac79e1c80 4 rocksdb: kZSTD supported: 0 2023-06-03T19:44:48.101 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:47 smithi164 bash[59462]: debug 2023-06-03T19:44:47.836+0000 7fdac79e1c80 4 rocksdb: kXpressCompression supported: 0 2023-06-03T19:44:48.101 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:47 smithi164 bash[59462]: debug 2023-06-03T19:44:47.836+0000 7fdac79e1c80 4 rocksdb: kLZ4HCCompression supported: 1 2023-06-03T19:44:48.101 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:47 smithi164 bash[59462]: debug 2023-06-03T19:44:47.836+0000 7fdac79e1c80 4 rocksdb: kZlibCompression supported: 1 2023-06-03T19:44:48.101 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:47 smithi164 bash[59462]: debug 2023-06-03T19:44:47.836+0000 7fdac79e1c80 4 rocksdb: kSnappyCompression supported: 1 2023-06-03T19:44:48.101 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:47 smithi164 bash[59462]: debug 2023-06-03T19:44:47.836+0000 7fdac79e1c80 4 rocksdb: kLZ4Compression supported: 1 2023-06-03T19:44:48.101 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:47 smithi164 bash[59462]: debug 2023-06-03T19:44:47.836+0000 7fdac79e1c80 4 rocksdb: kBZip2Compression supported: 0 2023-06-03T19:44:48.101 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:47 smithi164 bash[59462]: debug 2023-06-03T19:44:47.836+0000 7fdac79e1c80 4 rocksdb: Fast CRC32 supported: Supported on x86 2023-06-03T19:44:48.101 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:47 smithi164 bash[59462]: debug 2023-06-03T19:44:47.836+0000 7fdac79e1c80 4 rocksdb: DMutex implementation: pthread_mutex_t 2023-06-03T19:44:48.102 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:47 smithi164 bash[59462]: debug 2023-06-03T19:44:47.836+0000 7fdac79e1c80 4 rocksdb: [db/version_set.cc:5527] Recovering from manifest file: /var/lib/ceph/mon/ceph-b/store.db/MANIFEST-000005 2023-06-03T19:44:48.102 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:47 smithi164 bash[59462]: debug 2023-06-03T19:44:47.836+0000 7fdac79e1c80 4 rocksdb: [db/column_family.cc:630] --------------- Options for column family [default]: 2023-06-03T19:44:48.102 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:47 smithi164 bash[59462]: debug 2023-06-03T19:44:47.836+0000 7fdac79e1c80 4 rocksdb: Options.comparator: leveldb.BytewiseComparator 2023-06-03T19:44:48.102 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:47 smithi164 bash[59462]: debug 2023-06-03T19:44:47.836+0000 7fdac79e1c80 4 rocksdb: Options.merge_operator: 2023-06-03T19:44:48.102 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:47 smithi164 bash[59462]: debug 2023-06-03T19:44:47.836+0000 7fdac79e1c80 4 rocksdb: Options.compaction_filter: None 2023-06-03T19:44:48.102 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:47 smithi164 bash[59462]: debug 2023-06-03T19:44:47.836+0000 7fdac79e1c80 4 rocksdb: Options.compaction_filter_factory: None 2023-06-03T19:44:48.102 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:47 smithi164 bash[59462]: debug 2023-06-03T19:44:47.836+0000 7fdac79e1c80 4 rocksdb: Options.sst_partitioner_factory: None 2023-06-03T19:44:48.103 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:47 smithi164 bash[59462]: debug 2023-06-03T19:44:47.836+0000 7fdac79e1c80 4 rocksdb: Options.memtable_factory: SkipListFactory 2023-06-03T19:44:48.103 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:47 smithi164 bash[59462]: debug 2023-06-03T19:44:47.836+0000 7fdac79e1c80 4 rocksdb: Options.table_factory: BlockBasedTable 2023-06-03T19:44:48.103 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:47 smithi164 bash[59462]: debug 2023-06-03T19:44:47.836+0000 7fdac79e1c80 4 rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x55ce4b369560) 2023-06-03T19:44:48.103 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:47 smithi164 bash[59462]: cache_index_and_filter_blocks: 1 2023-06-03T19:44:48.103 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:47 smithi164 bash[59462]: cache_index_and_filter_blocks_with_high_priority: 0 2023-06-03T19:44:48.103 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:47 smithi164 bash[59462]: pin_l0_filter_and_index_blocks_in_cache: 0 2023-06-03T19:44:48.103 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:47 smithi164 bash[59462]: pin_top_level_index_and_filter: 1 2023-06-03T19:44:48.103 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:47 smithi164 bash[59462]: index_type: 0 2023-06-03T19:44:48.104 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:47 smithi164 bash[59462]: data_block_index_type: 0 2023-06-03T19:44:48.104 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:47 smithi164 bash[59462]: index_shortening: 1 2023-06-03T19:44:48.104 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:47 smithi164 bash[59462]: data_block_hash_table_util_ratio: 0.750000 2023-06-03T19:44:48.104 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:47 smithi164 bash[59462]: checksum: 4 2023-06-03T19:44:48.104 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:47 smithi164 bash[59462]: no_block_cache: 0 2023-06-03T19:44:48.104 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:47 smithi164 bash[59462]: block_cache: 0x55ce4a6791f0 2023-06-03T19:44:48.104 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:47 smithi164 bash[59462]: block_cache_name: BinnedLRUCache 2023-06-03T19:44:48.105 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:47 smithi164 bash[59462]: block_cache_options: 2023-06-03T19:44:48.105 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:47 smithi164 bash[59462]: capacity : 536870912 2023-06-03T19:44:48.105 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:47 smithi164 bash[59462]: num_shard_bits : 4 2023-06-03T19:44:48.105 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:47 smithi164 bash[59462]: strict_capacity_limit : 0 2023-06-03T19:44:48.105 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:47 smithi164 bash[59462]: high_pri_pool_ratio: 0.000 2023-06-03T19:44:48.105 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:47 smithi164 bash[59462]: block_cache_compressed: (nil) 2023-06-03T19:44:48.105 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:47 smithi164 bash[59462]: persistent_cache: (nil) 2023-06-03T19:44:48.105 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:47 smithi164 bash[59462]: block_size: 4096 2023-06-03T19:44:48.106 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:47 smithi164 bash[59462]: block_size_deviation: 10 2023-06-03T19:44:48.106 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:47 smithi164 bash[59462]: block_restart_interval: 16 2023-06-03T19:44:48.106 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:47 smithi164 bash[59462]: index_block_restart_interval: 1 2023-06-03T19:44:48.106 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:47 smithi164 bash[59462]: metadata_block_size: 4096 2023-06-03T19:44:48.106 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:47 smithi164 bash[59462]: partition_filters: 0 2023-06-03T19:44:48.106 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:47 smithi164 bash[59462]: use_delta_encoding: 1 2023-06-03T19:44:48.106 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:47 smithi164 bash[59462]: filter_policy: bloomfilter 2023-06-03T19:44:48.106 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:47 smithi164 bash[59462]: whole_key_filtering: 1 2023-06-03T19:44:48.107 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:47 smithi164 bash[59462]: verify_compression: 0 2023-06-03T19:44:48.107 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:47 smithi164 bash[59462]: read_amp_bytes_per_bit: 0 2023-06-03T19:44:48.107 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:47 smithi164 bash[59462]: format_version: 5 2023-06-03T19:44:48.107 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:47 smithi164 bash[59462]: enable_index_compression: 1 2023-06-03T19:44:48.107 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:47 smithi164 bash[59462]: block_align: 0 2023-06-03T19:44:48.107 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:47 smithi164 bash[59462]: max_auto_readahead_size: 262144 2023-06-03T19:44:48.107 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:47 smithi164 bash[59462]: prepopulate_block_cache: 0 2023-06-03T19:44:48.107 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:47 smithi164 bash[59462]: initial_auto_readahead_size: 8192 2023-06-03T19:44:48.108 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:47 smithi164 bash[59462]: num_file_reads_for_auto_readahead: 2 2023-06-03T19:44:48.108 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:47 smithi164 bash[59462]: debug 2023-06-03T19:44:47.836+0000 7fdac79e1c80 4 rocksdb: Options.write_buffer_size: 33554432 2023-06-03T19:44:48.108 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:47 smithi164 bash[59462]: debug 2023-06-03T19:44:47.836+0000 7fdac79e1c80 4 rocksdb: Options.max_write_buffer_number: 2 2023-06-03T19:44:48.108 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:47 smithi164 bash[59462]: debug 2023-06-03T19:44:47.836+0000 7fdac79e1c80 4 rocksdb: Options.compression: NoCompression 2023-06-03T19:44:48.108 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:47 smithi164 bash[59462]: debug 2023-06-03T19:44:47.836+0000 7fdac79e1c80 4 rocksdb: Options.bottommost_compression: Disabled 2023-06-03T19:44:48.108 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:47 smithi164 bash[59462]: debug 2023-06-03T19:44:47.836+0000 7fdac79e1c80 4 rocksdb: Options.prefix_extractor: nullptr 2023-06-03T19:44:48.108 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:47 smithi164 bash[59462]: debug 2023-06-03T19:44:47.836+0000 7fdac79e1c80 4 rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2023-06-03T19:44:48.108 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:47 smithi164 bash[59462]: debug 2023-06-03T19:44:47.836+0000 7fdac79e1c80 4 rocksdb: Options.num_levels: 7 2023-06-03T19:44:48.109 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:47 smithi164 bash[59462]: debug 2023-06-03T19:44:47.836+0000 7fdac79e1c80 4 rocksdb: Options.min_write_buffer_number_to_merge: 1 2023-06-03T19:44:48.109 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:47 smithi164 bash[59462]: debug 2023-06-03T19:44:47.836+0000 7fdac79e1c80 4 rocksdb: Options.max_write_buffer_number_to_maintain: 0 2023-06-03T19:44:48.109 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:47 smithi164 bash[59462]: debug 2023-06-03T19:44:47.836+0000 7fdac79e1c80 4 rocksdb: Options.max_write_buffer_size_to_maintain: 0 2023-06-03T19:44:48.109 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:47 smithi164 bash[59462]: debug 2023-06-03T19:44:47.836+0000 7fdac79e1c80 4 rocksdb: Options.bottommost_compression_opts.window_bits: -14 2023-06-03T19:44:48.109 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:47 smithi164 bash[59462]: debug 2023-06-03T19:44:47.836+0000 7fdac79e1c80 4 rocksdb: Options.bottommost_compression_opts.level: 32767 2023-06-03T19:44:48.109 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:47 smithi164 bash[59462]: debug 2023-06-03T19:44:47.836+0000 7fdac79e1c80 4 rocksdb: Options.bottommost_compression_opts.strategy: 0 2023-06-03T19:44:48.109 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:47 smithi164 bash[59462]: debug 2023-06-03T19:44:47.836+0000 7fdac79e1c80 4 rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2023-06-03T19:44:48.109 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:47 smithi164 bash[59462]: debug 2023-06-03T19:44:47.836+0000 7fdac79e1c80 4 rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2023-06-03T19:44:48.110 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:47 smithi164 bash[59462]: debug 2023-06-03T19:44:47.836+0000 7fdac79e1c80 4 rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2023-06-03T19:44:48.110 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:47 smithi164 bash[59462]: debug 2023-06-03T19:44:47.836+0000 7fdac79e1c80 4 rocksdb: Options.bottommost_compression_opts.enabled: false 2023-06-03T19:44:48.110 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:47 smithi164 bash[59462]: debug 2023-06-03T19:44:47.836+0000 7fdac79e1c80 4 rocksdb: Options.bottommost_compression_opts.max_dict_buffer_bytes: 0 2023-06-03T19:44:48.110 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:47 smithi164 bash[59462]: debug 2023-06-03T19:44:47.836+0000 7fdac79e1c80 4 rocksdb: Options.bottommost_compression_opts.use_zstd_dict_trainer: true 2023-06-03T19:44:48.110 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:47 smithi164 bash[59462]: debug 2023-06-03T19:44:47.836+0000 7fdac79e1c80 4 rocksdb: Options.compression_opts.window_bits: -14 2023-06-03T19:44:48.110 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:47 smithi164 bash[59462]: debug 2023-06-03T19:44:47.836+0000 7fdac79e1c80 4 rocksdb: Options.compression_opts.level: 32767 2023-06-03T19:44:48.110 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:47 smithi164 bash[59462]: debug 2023-06-03T19:44:47.836+0000 7fdac79e1c80 4 rocksdb: Options.compression_opts.strategy: 0 2023-06-03T19:44:48.110 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:47 smithi164 bash[59462]: debug 2023-06-03T19:44:47.836+0000 7fdac79e1c80 4 rocksdb: Options.compression_opts.max_dict_bytes: 0 2023-06-03T19:44:48.111 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:47 smithi164 bash[59462]: debug 2023-06-03T19:44:47.836+0000 7fdac79e1c80 4 rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2023-06-03T19:44:48.111 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:47 smithi164 bash[59462]: debug 2023-06-03T19:44:47.836+0000 7fdac79e1c80 4 rocksdb: Options.compression_opts.use_zstd_dict_trainer: true 2023-06-03T19:44:48.111 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:47 smithi164 bash[59462]: debug 2023-06-03T19:44:47.836+0000 7fdac79e1c80 4 rocksdb: Options.compression_opts.parallel_threads: 1 2023-06-03T19:44:48.111 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:47 smithi164 bash[59462]: debug 2023-06-03T19:44:47.836+0000 7fdac79e1c80 4 rocksdb: Options.compression_opts.enabled: false 2023-06-03T19:44:48.111 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:47 smithi164 bash[59462]: debug 2023-06-03T19:44:47.836+0000 7fdac79e1c80 4 rocksdb: Options.compression_opts.max_dict_buffer_bytes: 0 2023-06-03T19:44:48.111 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:47 smithi164 bash[59462]: debug 2023-06-03T19:44:47.836+0000 7fdac79e1c80 4 rocksdb: Options.level0_file_num_compaction_trigger: 4 2023-06-03T19:44:48.111 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:47 smithi164 bash[59462]: debug 2023-06-03T19:44:47.836+0000 7fdac79e1c80 4 rocksdb: Options.level0_slowdown_writes_trigger: 20 2023-06-03T19:44:48.112 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:47 smithi164 bash[59462]: debug 2023-06-03T19:44:47.836+0000 7fdac79e1c80 4 rocksdb: Options.level0_stop_writes_trigger: 36 2023-06-03T19:44:48.112 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:47 smithi164 bash[59462]: debug 2023-06-03T19:44:47.836+0000 7fdac79e1c80 4 rocksdb: Options.target_file_size_base: 67108864 2023-06-03T19:44:48.112 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:47 smithi164 bash[59462]: debug 2023-06-03T19:44:47.836+0000 7fdac79e1c80 4 rocksdb: Options.target_file_size_multiplier: 1 2023-06-03T19:44:48.112 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:47 smithi164 bash[59462]: debug 2023-06-03T19:44:47.836+0000 7fdac79e1c80 4 rocksdb: Options.max_bytes_for_level_base: 268435456 2023-06-03T19:44:48.112 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:47 smithi164 bash[59462]: debug 2023-06-03T19:44:47.836+0000 7fdac79e1c80 4 rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2023-06-03T19:44:48.112 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:47 smithi164 bash[59462]: debug 2023-06-03T19:44:47.836+0000 7fdac79e1c80 4 rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2023-06-03T19:44:48.112 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:47 smithi164 bash[59462]: debug 2023-06-03T19:44:47.836+0000 7fdac79e1c80 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2023-06-03T19:44:48.112 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:47 smithi164 bash[59462]: debug 2023-06-03T19:44:47.836+0000 7fdac79e1c80 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2023-06-03T19:44:48.113 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:47 smithi164 bash[59462]: debug 2023-06-03T19:44:47.836+0000 7fdac79e1c80 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2023-06-03T19:44:48.113 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:47 smithi164 bash[59462]: debug 2023-06-03T19:44:47.836+0000 7fdac79e1c80 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2023-06-03T19:44:48.113 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:47 smithi164 bash[59462]: debug 2023-06-03T19:44:47.836+0000 7fdac79e1c80 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2023-06-03T19:44:48.113 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:47 smithi164 bash[59462]: debug 2023-06-03T19:44:47.836+0000 7fdac79e1c80 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2023-06-03T19:44:48.113 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:47 smithi164 bash[59462]: debug 2023-06-03T19:44:47.836+0000 7fdac79e1c80 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2023-06-03T19:44:48.113 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:47 smithi164 bash[59462]: debug 2023-06-03T19:44:47.836+0000 7fdac79e1c80 4 rocksdb: Options.max_sequential_skip_in_iterations: 8 2023-06-03T19:44:48.113 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:47 smithi164 bash[59462]: debug 2023-06-03T19:44:47.836+0000 7fdac79e1c80 4 rocksdb: Options.max_compaction_bytes: 1677721600 2023-06-03T19:44:48.113 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:47 smithi164 bash[59462]: debug 2023-06-03T19:44:47.836+0000 7fdac79e1c80 4 rocksdb: Options.ignore_max_compaction_bytes_for_input: true 2023-06-03T19:44:48.114 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:47 smithi164 bash[59462]: debug 2023-06-03T19:44:47.836+0000 7fdac79e1c80 4 rocksdb: Options.arena_block_size: 1048576 2023-06-03T19:44:48.114 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:47 smithi164 bash[59462]: debug 2023-06-03T19:44:47.836+0000 7fdac79e1c80 4 rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2023-06-03T19:44:48.114 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:47 smithi164 bash[59462]: debug 2023-06-03T19:44:47.836+0000 7fdac79e1c80 4 rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2023-06-03T19:44:48.114 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:47 smithi164 bash[59462]: debug 2023-06-03T19:44:47.836+0000 7fdac79e1c80 4 rocksdb: Options.disable_auto_compactions: 0 2023-06-03T19:44:48.114 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:47 smithi164 bash[59462]: debug 2023-06-03T19:44:47.836+0000 7fdac79e1c80 4 rocksdb: Options.compaction_style: kCompactionStyleLevel 2023-06-03T19:44:48.114 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:47 smithi164 bash[59462]: debug 2023-06-03T19:44:47.836+0000 7fdac79e1c80 4 rocksdb: Options.compaction_pri: kMinOverlappingRatio 2023-06-03T19:44:48.114 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:47 smithi164 bash[59462]: debug 2023-06-03T19:44:47.836+0000 7fdac79e1c80 4 rocksdb: Options.compaction_options_universal.size_ratio: 1 2023-06-03T19:44:48.114 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:47 smithi164 bash[59462]: debug 2023-06-03T19:44:47.836+0000 7fdac79e1c80 4 rocksdb: Options.compaction_options_universal.min_merge_width: 2 2023-06-03T19:44:48.115 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:47 smithi164 bash[59462]: debug 2023-06-03T19:44:47.836+0000 7fdac79e1c80 4 rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2023-06-03T19:44:48.115 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:47 smithi164 bash[59462]: debug 2023-06-03T19:44:47.836+0000 7fdac79e1c80 4 rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2023-06-03T19:44:48.115 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:47 smithi164 bash[59462]: debug 2023-06-03T19:44:47.836+0000 7fdac79e1c80 4 rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2023-06-03T19:44:48.115 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:47 smithi164 bash[59462]: debug 2023-06-03T19:44:47.836+0000 7fdac79e1c80 4 rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2023-06-03T19:44:48.115 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:47 smithi164 bash[59462]: debug 2023-06-03T19:44:47.836+0000 7fdac79e1c80 4 rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2023-06-03T19:44:48.115 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:47 smithi164 bash[59462]: debug 2023-06-03T19:44:47.836+0000 7fdac79e1c80 4 rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2023-06-03T19:44:48.115 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:47 smithi164 bash[59462]: debug 2023-06-03T19:44:47.836+0000 7fdac79e1c80 4 rocksdb: Options.table_properties_collectors: 2023-06-03T19:44:48.115 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:47 smithi164 bash[59462]: debug 2023-06-03T19:44:47.836+0000 7fdac79e1c80 4 rocksdb: Options.inplace_update_support: 0 2023-06-03T19:44:48.116 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:47 smithi164 bash[59462]: debug 2023-06-03T19:44:47.836+0000 7fdac79e1c80 4 rocksdb: Options.inplace_update_num_locks: 10000 2023-06-03T19:44:48.116 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:47 smithi164 bash[59462]: debug 2023-06-03T19:44:47.836+0000 7fdac79e1c80 4 rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2023-06-03T19:44:48.116 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:47 smithi164 bash[59462]: debug 2023-06-03T19:44:47.836+0000 7fdac79e1c80 4 rocksdb: Options.memtable_whole_key_filtering: 0 2023-06-03T19:44:48.116 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:47 smithi164 bash[59462]: debug 2023-06-03T19:44:47.836+0000 7fdac79e1c80 4 rocksdb: Options.memtable_huge_page_size: 0 2023-06-03T19:44:48.116 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:47 smithi164 bash[59462]: debug 2023-06-03T19:44:47.836+0000 7fdac79e1c80 4 rocksdb: Options.bloom_locality: 0 2023-06-03T19:44:48.116 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:47 smithi164 bash[59462]: debug 2023-06-03T19:44:47.836+0000 7fdac79e1c80 4 rocksdb: Options.max_successive_merges: 0 2023-06-03T19:44:48.116 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:47 smithi164 bash[59462]: debug 2023-06-03T19:44:47.836+0000 7fdac79e1c80 4 rocksdb: Options.optimize_filters_for_hits: 0 2023-06-03T19:44:48.116 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:47 smithi164 bash[59462]: debug 2023-06-03T19:44:47.836+0000 7fdac79e1c80 4 rocksdb: Options.paranoid_file_checks: 0 2023-06-03T19:44:48.117 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:47 smithi164 bash[59462]: debug 2023-06-03T19:44:47.836+0000 7fdac79e1c80 4 rocksdb: Options.force_consistency_checks: 1 2023-06-03T19:44:48.117 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:47 smithi164 bash[59462]: debug 2023-06-03T19:44:47.836+0000 7fdac79e1c80 4 rocksdb: Options.report_bg_io_stats: 0 2023-06-03T19:44:48.117 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:47 smithi164 bash[59462]: debug 2023-06-03T19:44:47.836+0000 7fdac79e1c80 4 rocksdb: Options.ttl: 2592000 2023-06-03T19:44:48.117 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:47 smithi164 bash[59462]: debug 2023-06-03T19:44:47.836+0000 7fdac79e1c80 4 rocksdb: Options.periodic_compaction_seconds: 0 2023-06-03T19:44:48.117 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:47 smithi164 bash[59462]: debug 2023-06-03T19:44:47.836+0000 7fdac79e1c80 4 rocksdb: Options.preclude_last_level_data_seconds: 0 2023-06-03T19:44:48.117 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:47 smithi164 bash[59462]: debug 2023-06-03T19:44:47.836+0000 7fdac79e1c80 4 rocksdb: Options.preserve_internal_time_seconds: 0 2023-06-03T19:44:48.117 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:47 smithi164 bash[59462]: debug 2023-06-03T19:44:47.836+0000 7fdac79e1c80 4 rocksdb: Options.enable_blob_files: false 2023-06-03T19:44:48.117 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:47 smithi164 bash[59462]: debug 2023-06-03T19:44:47.836+0000 7fdac79e1c80 4 rocksdb: Options.min_blob_size: 0 2023-06-03T19:44:48.118 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:47 smithi164 bash[59462]: debug 2023-06-03T19:44:47.836+0000 7fdac79e1c80 4 rocksdb: Options.blob_file_size: 268435456 2023-06-03T19:44:48.118 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:47 smithi164 bash[59462]: debug 2023-06-03T19:44:47.836+0000 7fdac79e1c80 4 rocksdb: Options.blob_compression_type: NoCompression 2023-06-03T19:44:48.118 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:47 smithi164 bash[59462]: debug 2023-06-03T19:44:47.836+0000 7fdac79e1c80 4 rocksdb: Options.enable_blob_garbage_collection: false 2023-06-03T19:44:48.118 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:47 smithi164 bash[59462]: debug 2023-06-03T19:44:47.836+0000 7fdac79e1c80 4 rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2023-06-03T19:44:48.118 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:47 smithi164 bash[59462]: debug 2023-06-03T19:44:47.836+0000 7fdac79e1c80 4 rocksdb: Options.blob_garbage_collection_force_threshold: 1.000000 2023-06-03T19:44:48.118 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:47 smithi164 bash[59462]: debug 2023-06-03T19:44:47.836+0000 7fdac79e1c80 4 rocksdb: Options.blob_compaction_readahead_size: 0 2023-06-03T19:44:48.118 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:47 smithi164 bash[59462]: debug 2023-06-03T19:44:47.836+0000 7fdac79e1c80 4 rocksdb: Options.blob_file_starting_level: 0 2023-06-03T19:44:48.119 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:47 smithi164 bash[59462]: debug 2023-06-03T19:44:47.836+0000 7fdac79e1c80 4 rocksdb: Options.experimental_mempurge_threshold: 0.000000 2023-06-03T19:44:48.119 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:47 smithi164 bash[59462]: debug 2023-06-03T19:44:47.836+0000 7fdabd201700 3 rocksdb: [table/block_based/block_based_table_reader.cc:721] At least one SST file opened without unique ID to verify: 35.sst 2023-06-03T19:44:48.119 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:47 smithi164 bash[59462]: debug 2023-06-03T19:44:47.836+0000 7fdac79e1c80 4 rocksdb: [db/version_set.cc:5566] Recovered from manifest file:/var/lib/ceph/mon/ceph-b/store.db/MANIFEST-000005 succeeded,manifest_file_number is 5, next_file_number is 37, last_sequence is 21177, log_number is 33,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2023-06-03T19:44:48.119 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:47 smithi164 bash[59462]: debug 2023-06-03T19:44:47.836+0000 7fdac79e1c80 4 rocksdb: [db/version_set.cc:5581] Column family [default] (ID 0), log number is 33 2023-06-03T19:44:48.119 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:47 smithi164 bash[59462]: debug 2023-06-03T19:44:47.836+0000 7fdac79e1c80 4 rocksdb: [db/db_impl/db_impl_open.cc:539] DB ID: 1fc6c658-3235-4cc0-bcf6-ebe52a05e6d4 2023-06-03T19:44:48.119 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:47 smithi164 bash[59462]: debug 2023-06-03T19:44:47.836+0000 7fdac79e1c80 4 rocksdb: EVENT_LOG_v1 {"time_micros": 1685821487841250, "job": 1, "event": "recovery_started", "wal_files": [33]} 2023-06-03T19:44:48.119 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:47 smithi164 bash[59462]: debug 2023-06-03T19:44:47.836+0000 7fdac79e1c80 4 rocksdb: [db/db_impl/db_impl_open.cc:1043] Recovering log #33 mode 2 2023-06-03T19:44:48.119 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:47 smithi164 bash[59462]: debug 2023-06-03T19:44:47.840+0000 7fdac79e1c80 4 rocksdb: EVENT_LOG_v1 {"time_micros": 1685821487844993, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 38, "file_size": 363968, "file_checksum": "", "file_checksum_func_name": "Unknown", "smallest_seqno": 21182, "largest_seqno": 21276, "table_properties": {"data_size": 362397, "index_size": 431, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 133, "raw_key_size": 1007, "raw_average_key_size": 22, "raw_value_size": 361145, "raw_average_value_size": 8207, "num_data_blocks": 19, "num_entries": 44, "num_filter_entries": 44, "num_deletions": 0, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "bloomfilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; max_dict_buffer_bytes=0; use_zstd_dict_trainer=1; ", "creation_time": 1685821487, "oldest_key_time": 0, "file_creation_time": 0, "slow_compression_estimated_data_size": 0, "fast_compression_estimated_data_size": 0, "db_id": "1fc6c658-3235-4cc0-bcf6-ebe52a05e6d4", "db_session_id": "MER74KV56Z66HDSBFJCC", "orig_file_number": 38, "seqno_to_time_mapping": "N/A"}} 2023-06-03T19:44:48.120 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:47 smithi164 bash[59462]: debug 2023-06-03T19:44:47.840+0000 7fdac79e1c80 4 rocksdb: EVENT_LOG_v1 {"time_micros": 1685821487845305, "job": 1, "event": "recovery_finished"} 2023-06-03T19:44:48.120 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:47 smithi164 bash[59462]: debug 2023-06-03T19:44:47.840+0000 7fdac79e1c80 4 rocksdb: [db/version_set.cc:5047] Creating manifest 40 2023-06-03T19:44:48.120 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:47 smithi164 bash[59462]: debug 2023-06-03T19:44:47.844+0000 7fdac79e1c80 4 rocksdb: [file/delete_scheduler.cc:74] Deleted file /var/lib/ceph/mon/ceph-b/store.db/000033.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2023-06-03T19:44:48.120 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:47 smithi164 bash[59462]: debug 2023-06-03T19:44:47.844+0000 7fdac79e1c80 4 rocksdb: [db/db_impl/db_impl_open.cc:1987] SstFileManager instance 0x55ce4a71a000 2023-06-03T19:44:48.120 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:47 smithi164 bash[59462]: debug 2023-06-03T19:44:47.844+0000 7fdac79e1c80 4 rocksdb: DB pointer 0x55ce4a708000 2023-06-03T19:44:48.120 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:47 smithi164 bash[59462]: debug 2023-06-03T19:44:47.844+0000 7fdab61f3700 4 rocksdb: [db/db_impl/db_impl.cc:1109] ------- DUMPING STATS ------- 2023-06-03T19:44:48.120 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:47 smithi164 bash[59462]: debug 2023-06-03T19:44:47.844+0000 7fdab61f3700 4 rocksdb: [db/db_impl/db_impl.cc:1111] 2023-06-03T19:44:48.121 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:47 smithi164 bash[59462]: ** DB Stats ** 2023-06-03T19:44:48.121 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:47 smithi164 bash[59462]: Uptime(secs): 0.0 total, 0.0 interval 2023-06-03T19:44:48.121 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:47 smithi164 bash[59462]: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2023-06-03T19:44:48.121 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:47 smithi164 bash[59462]: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2023-06-03T19:44:48.121 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:47 smithi164 bash[59462]: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2023-06-03T19:44:48.121 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:47 smithi164 bash[59462]: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2023-06-03T19:44:48.121 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:47 smithi164 bash[59462]: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2023-06-03T19:44:48.121 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:47 smithi164 bash[59462]: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2023-06-03T19:44:48.122 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:47 smithi164 bash[59462]: ** Compaction Stats [default] ** 2023-06-03T19:44:48.122 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:47 smithi164 bash[59462]: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2023-06-03T19:44:48.122 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:47 smithi164 bash[59462]: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ 2023-06-03T19:44:48.122 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:47 smithi164 bash[59462]: L0 1/0 355.44 KB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 214.0 0.00 0.00 1 0.002 0 0 0.0 0.0 2023-06-03T19:44:48.122 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:47 smithi164 bash[59462]: L6 1/0 30.23 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 0.0 0.0 2023-06-03T19:44:48.122 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:47 smithi164 bash[59462]: Sum 2/0 30.58 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 214.0 0.00 0.00 1 0.002 0 0 0.0 0.0 2023-06-03T19:44:48.122 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:47 smithi164 bash[59462]: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 214.0 0.00 0.00 1 0.002 0 0 0.0 0.0 2023-06-03T19:44:48.122 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:47 smithi164 bash[59462]: ** Compaction Stats [default] ** 2023-06-03T19:44:48.123 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:47 smithi164 bash[59462]: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2023-06-03T19:44:48.123 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:47 smithi164 bash[59462]: --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2023-06-03T19:44:48.123 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:47 smithi164 bash[59462]: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 214.0 0.00 0.00 1 0.002 0 0 0.0 0.0 2023-06-03T19:44:48.123 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:47 smithi164 bash[59462]: Blob file count: 0, total size: 0.0 GB, garbage size: 0.0 GB, space amp: 0.0 2023-06-03T19:44:48.123 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:47 smithi164 bash[59462]: Uptime(secs): 0.0 total, 0.0 interval 2023-06-03T19:44:48.123 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:47 smithi164 bash[59462]: Flush(GB): cumulative 0.000, interval 0.000 2023-06-03T19:44:48.123 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:47 smithi164 bash[59462]: AddFile(GB): cumulative 0.000, interval 0.000 2023-06-03T19:44:48.123 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:47 smithi164 bash[59462]: AddFile(Total Files): cumulative 0, interval 0 2023-06-03T19:44:48.124 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:47 smithi164 bash[59462]: AddFile(L0 Files): cumulative 0, interval 0 2023-06-03T19:44:48.124 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:47 smithi164 bash[59462]: AddFile(Keys): cumulative 0, interval 0 2023-06-03T19:44:48.124 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:47 smithi164 bash[59462]: Cumulative compaction: 0.00 GB write, 43.26 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-06-03T19:44:48.124 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:47 smithi164 bash[59462]: Interval compaction: 0.00 GB write, 43.26 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-06-03T19:44:48.124 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:47 smithi164 bash[59462]: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2023-06-03T19:44:48.124 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:47 smithi164 bash[59462]: Block cache BinnedLRUCache@0x55ce4a6791f0#7 capacity: 512.00 MB usage: 0.69 KB table_size: 0 occupancy: 18446744073709551615 collections: 1 last_copies: 0 last_secs: 1.3e-05 secs_since: 0 2023-06-03T19:44:48.124 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:47 smithi164 bash[59462]: Block cache entry stats(count,size,portion): FilterBlock(1,0.17 KB,3.27826e-05%) IndexBlock(1,0.52 KB,9.83477e-05%) Misc(1,0.00 KB,0%) 2023-06-03T19:44:48.124 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:47 smithi164 bash[59462]: ** File Read Latency Histogram By Level [default] ** 2023-06-03T19:44:48.125 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:47 smithi164 bash[59462]: debug 2023-06-03T19:44:47.844+0000 7fdac79e1c80 0 starting mon.b rank 1 at public addrs [v2:172.21.15.164:3300/0,v1:172.21.15.164:6789/0] at bind addrs [v2:172.21.15.164:3300/0,v1:172.21.15.164:6789/0] mon_data /var/lib/ceph/mon/ceph-b fsid f1db3478-0243-11ee-9b1d-001a4aab830c 2023-06-03T19:44:48.125 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:47 smithi164 bash[59462]: debug 2023-06-03T19:44:47.848+0000 7fdac79e1c80 1 mon.b@-1(???) e3 preinit fsid f1db3478-0243-11ee-9b1d-001a4aab830c 2023-06-03T19:44:48.125 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:47 smithi164 bash[59462]: debug 2023-06-03T19:44:47.848+0000 7fdac79e1c80 0 mon.b@-1(???).mds e7 new map 2023-06-03T19:44:48.125 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:47 smithi164 bash[59462]: debug 2023-06-03T19:44:47.848+0000 7fdac79e1c80 0 mon.b@-1(???).mds e7 print_map 2023-06-03T19:44:48.125 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:47 smithi164 bash[59462]: e7 2023-06-03T19:44:48.125 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:47 smithi164 bash[59462]: enable_multiple, ever_enabled_multiple: 1,1 2023-06-03T19:44:48.125 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:47 smithi164 bash[59462]: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2023-06-03T19:44:48.125 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:47 smithi164 bash[59462]: legacy client fscid: 1 2023-06-03T19:44:48.126 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:47 smithi164 bash[59462]: 2023-06-03T19:44:48.126 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:47 smithi164 bash[59462]: Filesystem 'foo' (1) 2023-06-03T19:44:48.126 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:47 smithi164 bash[59462]: fs_name foo 2023-06-03T19:44:48.126 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:47 smithi164 bash[59462]: epoch 5 2023-06-03T19:44:48.126 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:47 smithi164 bash[59462]: flags 12 joinable allow_snaps allow_multimds_snaps 2023-06-03T19:44:48.126 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:47 smithi164 bash[59462]: created 2023-06-03T19:34:16.805120+0000 2023-06-03T19:44:48.126 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:47 smithi164 bash[59462]: modified 2023-06-03T19:34:24.830695+0000 2023-06-03T19:44:48.127 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:47 smithi164 bash[59462]: tableserver 0 2023-06-03T19:44:48.127 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:47 smithi164 bash[59462]: root 0 2023-06-03T19:44:48.127 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:47 smithi164 bash[59462]: session_timeout 60 2023-06-03T19:44:48.127 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:47 smithi164 bash[59462]: session_autoclose 300 2023-06-03T19:44:48.127 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:47 smithi164 bash[59462]: max_file_size 1099511627776 2023-06-03T19:44:48.127 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:47 smithi164 bash[59462]: required_client_features {} 2023-06-03T19:44:48.127 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:47 smithi164 bash[59462]: last_failure 0 2023-06-03T19:44:48.127 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:47 smithi164 bash[59462]: last_failure_osd_epoch 0 2023-06-03T19:44:48.128 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:47 smithi164 bash[59462]: compat compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,7=mds uses inline data,8=no anchor table,9=file layout v2,10=snaprealm v2} 2023-06-03T19:44:48.128 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:47 smithi164 bash[59462]: max_mds 1 2023-06-03T19:44:48.128 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:47 smithi164 bash[59462]: in 0 2023-06-03T19:44:48.128 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:47 smithi164 bash[59462]: up {0=14646} 2023-06-03T19:44:48.128 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:47 smithi164 bash[59462]: failed 2023-06-03T19:44:48.128 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:47 smithi164 bash[59462]: damaged 2023-06-03T19:44:48.128 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:47 smithi164 bash[59462]: stopped 2023-06-03T19:44:48.128 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:47 smithi164 bash[59462]: data_pools [4] 2023-06-03T19:44:48.129 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:47 smithi164 bash[59462]: metadata_pool 3 2023-06-03T19:44:48.129 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:47 smithi164 bash[59462]: inline_data disabled 2023-06-03T19:44:48.129 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:47 smithi164 bash[59462]: balancer 2023-06-03T19:44:48.129 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:47 smithi164 bash[59462]: bal_rank_mask -1 2023-06-03T19:44:48.129 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:47 smithi164 bash[59462]: standby_count_wanted 1 2023-06-03T19:44:48.129 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:47 smithi164 bash[59462]: [mds.foo.smithi089.giuxuj{0:14646} state up:active seq 2 join_fscid=1 addr [v2:172.21.15.89:6834/1926931511,v1:172.21.15.89:6835/1926931511] compat {c=[1],r=[1],i=[7ff]}] 2023-06-03T19:44:48.129 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:47 smithi164 bash[59462]: 2023-06-03T19:44:48.129 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:47 smithi164 bash[59462]: 2023-06-03T19:44:48.130 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:47 smithi164 bash[59462]: Standby daemons: 2023-06-03T19:44:48.130 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:47 smithi164 bash[59462]: 2023-06-03T19:44:48.130 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:47 smithi164 bash[59462]: [mds.foo.smithi164.txgfgv{-1:24499} state up:standby seq 1 join_fscid=1 addr [v2:172.21.15.164:6832/2311698270,v1:172.21.15.164:6833/2311698270] compat {c=[1],r=[1],i=[7ff]}] 2023-06-03T19:44:48.130 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:47 smithi164 bash[59462]: debug 2023-06-03T19:44:47.848+0000 7fdac79e1c80 0 mon.b@-1(???).osd e77 crush map has features 3314933000854323200, adjusting msgr requires 2023-06-03T19:44:48.130 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:47 smithi164 bash[59462]: debug 2023-06-03T19:44:47.848+0000 7fdac79e1c80 0 mon.b@-1(???).osd e77 crush map has features 432629239337189376, adjusting msgr requires 2023-06-03T19:44:48.130 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:47 smithi164 bash[59462]: debug 2023-06-03T19:44:47.848+0000 7fdac79e1c80 0 mon.b@-1(???).osd e77 crush map has features 432629239337189376, adjusting msgr requires 2023-06-03T19:44:48.130 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:47 smithi164 bash[59462]: debug 2023-06-03T19:44:47.848+0000 7fdac79e1c80 0 mon.b@-1(???).osd e77 crush map has features 432629239337189376, adjusting msgr requires 2023-06-03T19:44:48.130 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:47 smithi164 bash[59462]: debug 2023-06-03T19:44:47.848+0000 7fdac79e1c80 1 mon.b@-1(???).paxosservice(auth 1..23) refresh upgraded, format 0 -> 3 2023-06-03T19:44:53.280 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 19:44:52 smithi089 bash[71163]: debug 2023-06-03T19:44:52.870+0000 7fe48642e700 -1 mgr.server handle_report got status from non-daemon mon.b 2023-06-03T19:44:54.033 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:44:53 smithi089 bash[75068]: cluster 1685820174.447331 mon.a (mon.0) 0 : 1 mkfs f1db3478-0243-11ee-9b1d-001a4aab830c 2023-06-03T19:44:54.033 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:44:53 smithi089 bash[75068]: cephadm 1685821123.1865313 mgr.x (mgr.24634) 56 : cephadm 1 Reconfiguring daemon grafana.a on smithi164 2023-06-03T19:44:54.033 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:44:53 smithi089 bash[75068]: cephadm 1685821171.429646 mgr.x (mgr.24634) 84 : cephadm 1 Upgrade: Need to upgrade myself (mgr.x) 2023-06-03T19:44:54.034 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:44:53 smithi089 bash[75068]: cephadm 1685821171.4305422 mgr.x (mgr.24634) 85 : cephadm 1 Upgrade: Need to upgrade myself (mgr.x) 2023-06-03T19:44:54.034 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:44:53 smithi089 bash[75068]: cephadm 1685821175.321385 mgr.x (mgr.24634) 88 : cephadm 1 Upgrade: Updating mgr.y 2023-06-03T19:44:54.034 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:44:53 smithi089 bash[75068]: cephadm 1685821175.3339238 mgr.x (mgr.24634) 89 : cephadm 1 Deploying daemon mgr.y on smithi089 2023-06-03T19:44:54.034 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:44:53 smithi089 bash[75068]: cephadm 1685821201.0951023 mgr.x (mgr.24634) 103 : cephadm 1 Upgrade: Need to upgrade myself (mgr.x) 2023-06-03T19:44:54.034 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:44:53 smithi089 bash[75068]: cephadm 1685821201.095862 mgr.x (mgr.24634) 104 : cephadm 1 Upgrade: Need to upgrade myself (mgr.x) 2023-06-03T19:44:54.034 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:44:53 smithi089 bash[75068]: cephadm 1685821201.100557 mgr.x (mgr.24634) 105 : cephadm 1 Failing over to other MGR 2023-06-03T19:44:54.034 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:44:53 smithi089 bash[75068]: cephadm 1685821204.4230404 mgr.y (mgr.25070) 3 : cephadm 1 [03/Jun/2023:19:40:04] ENGINE Bus STARTING 2023-06-03T19:44:54.035 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:44:53 smithi089 bash[75068]: cephadm 1685821204.5455484 mgr.y (mgr.25070) 4 : cephadm 1 [03/Jun/2023:19:40:04] ENGINE Serving on https://172.21.15.89:7150 2023-06-03T19:44:54.035 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:44:53 smithi089 bash[75068]: cephadm 1685821204.6471362 mgr.y (mgr.25070) 5 : cephadm 1 [03/Jun/2023:19:40:04] ENGINE Serving on http://172.21.15.89:8765 2023-06-03T19:44:54.035 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:44:53 smithi089 bash[75068]: cephadm 1685821204.6472821 mgr.y (mgr.25070) 6 : cephadm 1 [03/Jun/2023:19:40:04] ENGINE Bus STARTED 2023-06-03T19:44:54.035 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:44:53 smithi089 bash[75068]: cephadm 1685821222.6682131 mgr.y (mgr.25070) 16 : cephadm 1 Updating smithi089:/etc/ceph/ceph.conf 2023-06-03T19:44:54.035 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:44:53 smithi089 bash[75068]: cephadm 1685821222.6684442 mgr.y (mgr.25070) 17 : cephadm 1 Updating smithi164:/etc/ceph/ceph.conf 2023-06-03T19:44:54.035 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:44:53 smithi089 bash[75068]: cephadm 1685821222.8840244 mgr.y (mgr.25070) 18 : cephadm 1 Updating smithi164:/var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/config/ceph.conf 2023-06-03T19:44:54.035 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:44:53 smithi089 bash[75068]: cephadm 1685821222.890312 mgr.y (mgr.25070) 19 : cephadm 1 Updating smithi089:/var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/config/ceph.conf 2023-06-03T19:44:54.036 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:44:53 smithi089 bash[75068]: cephadm 1685821223.1088037 mgr.y (mgr.25070) 20 : cephadm 1 Updating smithi164:/etc/ceph/ceph.client.admin.keyring 2023-06-03T19:44:54.036 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:44:53 smithi089 bash[75068]: cephadm 1685821223.111952 mgr.y (mgr.25070) 21 : cephadm 1 Updating smithi089:/etc/ceph/ceph.client.admin.keyring 2023-06-03T19:44:54.036 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:44:53 smithi089 bash[75068]: cephadm 1685821223.3254883 mgr.y (mgr.25070) 22 : cephadm 1 Updating smithi164:/var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/config/ceph.client.admin.keyring 2023-06-03T19:44:54.036 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:44:53 smithi089 bash[75068]: cephadm 1685821223.3275669 mgr.y (mgr.25070) 23 : cephadm 1 Updating smithi089:/var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/config/ceph.client.admin.keyring 2023-06-03T19:44:54.036 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:44:53 smithi089 bash[75068]: cephadm 1685821223.671977 mgr.y (mgr.25070) 24 : cephadm 1 Reconfiguring prometheus.a (dependencies changed)... 2023-06-03T19:44:54.036 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:44:53 smithi089 bash[75068]: cephadm 1685821223.9393086 mgr.y (mgr.25070) 25 : cephadm 1 Reconfiguring daemon prometheus.a on smithi164 2023-06-03T19:44:54.036 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:44:53 smithi089 bash[75068]: cephadm 1685821236.3840792 mgr.y (mgr.25070) 34 : cephadm 1 Upgrade: Updating mgr.x 2023-06-03T19:44:54.037 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:44:53 smithi089 bash[75068]: cephadm 1685821236.390095 mgr.y (mgr.25070) 35 : cephadm 1 Deploying daemon mgr.x on smithi164 2023-06-03T19:44:54.037 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:44:53 smithi089 bash[75068]: cephadm 1685821261.9984038 mgr.y (mgr.25070) 48 : cephadm 1 Upgrade: Setting container_image for all mgr 2023-06-03T19:44:54.037 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:44:53 smithi089 bash[75068]: cephadm 1685821262.03976 mgr.y (mgr.25070) 49 : cephadm 1 Upgrade: It appears safe to stop mon.a 2023-06-03T19:44:54.037 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:44:53 smithi089 bash[75068]: cephadm 1685821266.7819054 mgr.y (mgr.25070) 53 : cephadm 1 Upgrade: Updating mon.a 2023-06-03T19:44:54.037 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:44:53 smithi089 bash[75068]: cephadm 1685821266.8025656 mgr.y (mgr.25070) 54 : cephadm 1 Deploying daemon mon.a on smithi089 2023-06-03T19:44:54.037 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:44:53 smithi089 bash[75068]: cephadm 1685821282.4937708 mgr.y (mgr.34104) 1 : cephadm 1 [03/Jun/2023:19:41:22] ENGINE Bus STARTING 2023-06-03T19:44:54.037 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:44:53 smithi089 bash[75068]: cephadm 1685821282.6101174 mgr.y (mgr.34104) 2 : cephadm 1 [03/Jun/2023:19:41:22] ENGINE Serving on https://172.21.15.89:7150 2023-06-03T19:44:54.038 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:44:53 smithi089 bash[75068]: cephadm 1685821282.7118428 mgr.y (mgr.34104) 3 : cephadm 1 [03/Jun/2023:19:41:22] ENGINE Serving on http://172.21.15.89:8765 2023-06-03T19:44:54.038 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:44:53 smithi089 bash[75068]: cephadm 1685821282.7121873 mgr.y (mgr.34104) 4 : cephadm 1 [03/Jun/2023:19:41:22] ENGINE Bus STARTED 2023-06-03T19:44:54.038 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:44:53 smithi089 bash[75068]: cephadm 1685821287.9637723 mgr.y (mgr.34104) 11 : cephadm 1 Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:00b1b6d689278b4f97ff790045c8c51d8ca419ac7ee2ebdf11008f2e810a5c78 2023-06-03T19:44:54.038 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:44:53 smithi089 bash[75068]: cephadm 1685821299.4224992 mgr.y (mgr.34104) 17 : cephadm 1 Updating smithi089:/etc/ceph/ceph.conf 2023-06-03T19:44:54.038 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:44:53 smithi089 bash[75068]: cephadm 1685821299.4228904 mgr.y (mgr.34104) 18 : cephadm 1 Updating smithi164:/etc/ceph/ceph.conf 2023-06-03T19:44:54.038 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:44:53 smithi089 bash[75068]: cephadm 1685821299.587414 mgr.y (mgr.34104) 19 : cephadm 1 Updating smithi089:/var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/config/ceph.conf 2023-06-03T19:44:54.038 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:44:53 smithi089 bash[75068]: cephadm 1685821299.5885887 mgr.y (mgr.34104) 20 : cephadm 1 Updating smithi164:/var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/config/ceph.conf 2023-06-03T19:44:54.039 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:44:53 smithi089 bash[75068]: cephadm 1685821299.7529323 mgr.y (mgr.34104) 21 : cephadm 1 Updating smithi089:/etc/ceph/ceph.client.admin.keyring 2023-06-03T19:44:54.039 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:44:53 smithi089 bash[75068]: cephadm 1685821299.7543862 mgr.y (mgr.34104) 22 : cephadm 1 Updating smithi164:/etc/ceph/ceph.client.admin.keyring 2023-06-03T19:44:54.039 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:44:53 smithi089 bash[75068]: cephadm 1685821299.9186985 mgr.y (mgr.34104) 23 : cephadm 1 Updating smithi089:/var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/config/ceph.client.admin.keyring 2023-06-03T19:44:54.039 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:44:53 smithi089 bash[75068]: cephadm 1685821299.919883 mgr.y (mgr.34104) 24 : cephadm 1 Updating smithi164:/var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/config/ceph.client.admin.keyring 2023-06-03T19:44:54.039 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:44:53 smithi089 bash[75068]: cephadm 1685821360.1272502 mgr.y (mgr.34104) 57 : cephadm 1 Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:00b1b6d689278b4f97ff790045c8c51d8ca419ac7ee2ebdf11008f2e810a5c78 2023-06-03T19:44:54.039 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:44:53 smithi089 bash[75068]: cephadm 1685821367.5300949 mgr.y (mgr.34104) 61 : cephadm 1 Upgrade: It appears safe to stop mon.c 2023-06-03T19:44:54.039 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:44:53 smithi089 bash[75068]: cephadm 1685821372.2798438 mgr.y (mgr.34104) 65 : cephadm 1 Upgrade: Updating mon.c 2023-06-03T19:44:54.039 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:44:53 smithi089 bash[75068]: audit 1685821372.3102217 mon.c (mon.2) 122 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-03T19:44:54.040 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:44:53 smithi089 bash[75068]: cephadm 1685821372.311988 mgr.y (mgr.34104) 66 : cephadm 1 Deploying daemon mon.c on smithi089 2023-06-03T19:44:54.040 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:44:53 smithi089 bash[75068]: audit 1685821375.0654147 mon.b (mon.1) 92 : audit 0 from='client.? 172.21.15.89:0/1498143111' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-03T19:44:54.040 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:44:53 smithi089 bash[75068]: cluster 1685821385.9985988 mgr.y (mgr.34104) 73 : cluster 0 pgmap v60: 89 pgs: 89 active+clean; 6.2 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 45 MiB/s wr, 723 op/s 2023-06-03T19:44:54.040 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:44:53 smithi089 bash[75068]: audit 1685821386.594359 mon.a (mon.0) 82 : audit 0 from='client.? 172.21.15.89:0/3649894161' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-03T19:44:54.040 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:44:53 smithi089 bash[75068]: audit 1685821387.063322 mon.b (mon.1) 99 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T19:44:54.040 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:44:53 smithi089 bash[75068]: cluster 1685821387.9991255 mgr.y (mgr.34104) 74 : cluster 0 pgmap v61: 89 pgs: 89 active+clean; 6.3 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 52 MiB/s wr, 831 op/s 2023-06-03T19:44:54.040 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:44:53 smithi089 bash[75068]: cluster 1685821389.9997709 mgr.y (mgr.34104) 75 : cluster 0 pgmap v62: 89 pgs: 89 active+clean; 6.5 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 53 MiB/s wr, 855 op/s 2023-06-03T19:44:54.041 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:44:53 smithi089 bash[75068]: cluster 1685821392.000228 mgr.y (mgr.34104) 76 : cluster 0 pgmap v63: 89 pgs: 89 active+clean; 6.5 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 45 MiB/s wr, 722 op/s 2023-06-03T19:44:54.041 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:44:53 smithi089 bash[75068]: audit 1685821393.4656174 mgr.y (mgr.34104) 77 : audit 0 from='client.44102 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-06-03T19:44:54.041 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:44:53 smithi089 bash[75068]: audit 1685821393.6309147 mon.a (mon.0) 83 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:44:54.041 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:44:53 smithi089 bash[75068]: audit 1685821393.6497493 mon.a (mon.0) 84 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:44:54.041 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:44:53 smithi089 bash[75068]: cluster 1685821394.00106 mgr.y (mgr.34104) 78 : cluster 0 pgmap v64: 89 pgs: 89 active+clean; 6.7 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 66 MiB/s wr, 1.06k op/s 2023-06-03T19:44:54.041 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:44:53 smithi089 bash[75068]: audit 1685821394.7517736 mon.a (mon.0) 85 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:44:54.041 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:44:53 smithi089 bash[75068]: audit 1685821394.768501 mon.a (mon.0) 86 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:44:54.042 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:44:53 smithi089 bash[75068]: cluster 1685821396.0015018 mgr.y (mgr.34104) 79 : cluster 0 pgmap v65: 89 pgs: 89 active+clean; 6.7 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 43 MiB/s wr, 681 op/s 2023-06-03T19:44:54.042 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:44:53 smithi089 bash[75068]: cluster 1685821398.002088 mgr.y (mgr.34104) 80 : cluster 0 pgmap v66: 89 pgs: 89 active+clean; 6.7 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 45 MiB/s wr, 718 op/s 2023-06-03T19:44:54.042 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:44:53 smithi089 bash[75068]: audit 1685821399.3379703 mgr.y (mgr.34104) 81 : audit 0 from='client.34209 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2023-06-03T19:44:54.042 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:44:53 smithi089 bash[75068]: cephadm 1685821399.3385687 mgr.y (mgr.34104) 82 : cephadm 1 Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:00b1b6d689278b4f97ff790045c8c51d8ca419ac7ee2ebdf11008f2e810a5c78 2023-06-03T19:44:54.042 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:44:53 smithi089 bash[75068]: audit 1685821399.344722 mon.a (mon.0) 87 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:44:54.042 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:44:53 smithi089 bash[75068]: cluster 1685821400.0027843 mgr.y (mgr.34104) 83 : cluster 0 pgmap v67: 89 pgs: 89 active+clean; 7.0 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 56 MiB/s wr, 896 op/s 2023-06-03T19:44:54.042 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:44:53 smithi089 bash[75068]: cluster 1685821402.0032067 mgr.y (mgr.34104) 84 : cluster 0 pgmap v68: 89 pgs: 89 active+clean; 7.0 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 41 MiB/s wr, 657 op/s 2023-06-03T19:44:54.043 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:44:53 smithi089 bash[75068]: audit 1685821402.0612197 mon.b (mon.1) 100 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T19:44:54.043 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:44:53 smithi089 bash[75068]: cluster 1685821404.0069954 mgr.y (mgr.34104) 85 : cluster 0 pgmap v69: 89 pgs: 89 active+clean; 6.7 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 85 B/s rd, 42 MiB/s wr, 1.00k op/s 2023-06-03T19:44:54.043 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:44:53 smithi089 bash[75068]: cluster 1685821406.007473 mgr.y (mgr.34104) 86 : cluster 0 pgmap v70: 89 pgs: 89 active+clean; 6.7 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 85 B/s rd, 22 MiB/s wr, 671 op/s 2023-06-03T19:44:54.043 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:44:53 smithi089 bash[75068]: cluster 1685821408.0078633 mgr.y (mgr.34104) 87 : cluster 0 pgmap v71: 89 pgs: 89 active+clean; 6.7 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 85 B/s rd, 22 MiB/s wr, 671 op/s 2023-06-03T19:44:54.043 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:44:53 smithi089 bash[75068]: audit 1685821408.4267251 mon.a (mon.0) 88 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:44:54.043 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:44:53 smithi089 bash[75068]: audit 1685821408.4374998 mon.a (mon.0) 89 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:44:54.043 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:44:53 smithi089 bash[75068]: audit 1685821408.4380069 mon.b (mon.1) 101 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-03T19:44:54.044 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:44:53 smithi089 bash[75068]: audit 1685821408.4439259 mon.b (mon.1) 102 : audit 1 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-03T19:44:54.044 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:44:53 smithi089 bash[75068]: audit 1685821408.452624 mon.a (mon.0) 90 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:44:54.044 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:44:53 smithi089 bash[75068]: audit 1685821408.5381055 mon.b (mon.1) 103 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-03T19:44:54.044 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:44:53 smithi089 bash[75068]: audit 1685821408.5485957 mon.b (mon.1) 104 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-03T19:44:54.044 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:44:53 smithi089 bash[75068]: audit 1685821408.5498192 mon.b (mon.1) 105 : audit 1 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-03T19:44:54.044 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:44:53 smithi089 bash[75068]: audit 1685821408.5613625 mon.a (mon.0) 91 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:44:54.044 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:44:53 smithi089 bash[75068]: cluster 1685821410.0086102 mgr.y (mgr.34104) 88 : cluster 0 pgmap v72: 89 pgs: 89 active+clean; 6.5 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 85 B/s rd, 19 MiB/s wr, 969 op/s 2023-06-03T19:44:54.045 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:44:53 smithi089 bash[75068]: cluster 1685821412.0090332 mgr.y (mgr.34104) 89 : cluster 0 pgmap v73: 89 pgs: 89 active+clean; 6.5 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 85 B/s rd, 1.4 MiB/s wr, 684 op/s 2023-06-03T19:44:54.045 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:44:53 smithi089 bash[75068]: cluster 1685821414.0097935 mgr.y (mgr.34104) 90 : cluster 0 pgmap v74: 89 pgs: 89 active+clean; 6.2 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 85 B/s rd, 1.4 MiB/s wr, 1.02k op/s 2023-06-03T19:44:54.045 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:44:53 smithi089 bash[75068]: cluster 1685821416.0102537 mgr.y (mgr.34104) 91 : cluster 0 pgmap v75: 89 pgs: 89 active+clean; 6.2 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 673 op/s 2023-06-03T19:44:54.045 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:44:53 smithi089 bash[75068]: audit 1685821417.0620108 mon.b (mon.1) 106 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T19:44:54.045 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:44:53 smithi089 bash[75068]: cluster 1685821418.0107417 mgr.y (mgr.34104) 92 : cluster 0 pgmap v76: 89 pgs: 89 active+clean; 6.2 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 673 op/s 2023-06-03T19:44:54.045 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:44:53 smithi089 bash[75068]: cluster 1685821420.0114686 mgr.y (mgr.34104) 93 : cluster 0 pgmap v77: 89 pgs: 89 active+clean; 6.0 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 998 op/s 2023-06-03T19:44:54.046 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:44:53 smithi089 bash[75068]: cluster 1685821422.011843 mgr.y (mgr.34104) 94 : cluster 0 pgmap v78: 89 pgs: 89 active+clean; 6.0 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 662 op/s 2023-06-03T19:44:54.046 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:44:53 smithi089 bash[75068]: cluster 1685821424.013851 mgr.y (mgr.34104) 95 : cluster 0 pgmap v79: 89 pgs: 89 active+clean; 5.8 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 984 op/s 2023-06-03T19:44:54.046 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:44:53 smithi089 bash[75068]: cluster 1685821426.0143747 mgr.y (mgr.34104) 96 : cluster 0 pgmap v80: 89 pgs: 89 active+clean; 5.8 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 646 op/s 2023-06-03T19:44:54.046 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:44:53 smithi089 bash[75068]: cluster 1685821428.0147314 mgr.y (mgr.34104) 97 : cluster 0 pgmap v81: 89 pgs: 89 active+clean; 5.8 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 647 op/s 2023-06-03T19:44:54.046 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:44:53 smithi089 bash[75068]: cluster 1685821430.0154066 mgr.y (mgr.34104) 98 : cluster 0 pgmap v82: 89 pgs: 89 active+clean; 5.5 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 965 op/s 2023-06-03T19:44:54.047 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:44:53 smithi089 bash[75068]: cluster 1685821432.0165899 mgr.y (mgr.34104) 99 : cluster 0 pgmap v83: 89 pgs: 89 active+clean; 5.5 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 640 op/s 2023-06-03T19:44:54.047 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:44:53 smithi089 bash[75068]: audit 1685821432.0621827 mon.b (mon.1) 107 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T19:44:54.047 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:44:53 smithi089 bash[75068]: cluster 1685821434.0173318 mgr.y (mgr.34104) 100 : cluster 0 pgmap v84: 89 pgs: 89 active+clean; 5.3 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 961 op/s 2023-06-03T19:44:54.047 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:44:53 smithi089 bash[75068]: cluster 1685821436.0177891 mgr.y (mgr.34104) 101 : cluster 0 pgmap v85: 89 pgs: 89 active+clean; 5.3 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 639 op/s 2023-06-03T19:44:54.047 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:44:53 smithi089 bash[75068]: cluster 1685821438.0200334 mgr.y (mgr.34104) 102 : cluster 0 pgmap v86: 89 pgs: 89 active+clean; 5.3 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 639 op/s 2023-06-03T19:44:54.047 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:44:53 smithi089 bash[75068]: cluster 1685821440.020801 mgr.y (mgr.34104) 103 : cluster 0 pgmap v87: 89 pgs: 89 active+clean; 5.1 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 951 op/s 2023-06-03T19:44:54.048 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:44:53 smithi089 bash[75068]: cluster 1685821442.0212224 mgr.y (mgr.34104) 104 : cluster 0 pgmap v88: 89 pgs: 89 active+clean; 5.1 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 632 op/s 2023-06-03T19:44:54.048 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:44:53 smithi089 bash[75068]: cluster 1685821444.0253873 mgr.y (mgr.34104) 105 : cluster 0 pgmap v89: 89 pgs: 89 active+clean; 4.8 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 942 op/s 2023-06-03T19:44:54.048 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:44:53 smithi089 bash[75068]: cluster 1685821446.0258205 mgr.y (mgr.34104) 106 : cluster 0 pgmap v90: 89 pgs: 89 active+clean; 4.8 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 621 op/s 2023-06-03T19:44:54.048 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:44:53 smithi089 bash[75068]: audit 1685821447.0623534 mon.b (mon.1) 108 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T19:44:54.049 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:44:53 smithi089 bash[75068]: cluster 1685821448.0262258 mgr.y (mgr.34104) 107 : cluster 0 pgmap v91: 89 pgs: 89 active+clean; 4.8 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 621 op/s 2023-06-03T19:44:54.049 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:44:53 smithi089 bash[75068]: cluster 1685821450.026844 mgr.y (mgr.34104) 108 : cluster 0 pgmap v92: 89 pgs: 89 active+clean; 4.6 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 932 op/s 2023-06-03T19:44:54.049 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:44:53 smithi089 bash[75068]: cluster 1685821452.0272791 mgr.y (mgr.34104) 109 : cluster 0 pgmap v93: 89 pgs: 89 active+clean; 4.6 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 620 op/s 2023-06-03T19:44:54.049 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:44:53 smithi089 bash[75068]: cluster 1685821454.0280442 mgr.y (mgr.34104) 110 : cluster 0 pgmap v94: 89 pgs: 89 active+clean; 4.4 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 925 op/s 2023-06-03T19:44:54.050 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:44:53 smithi089 bash[75068]: cluster 1685821456.0288029 mgr.y (mgr.34104) 111 : cluster 0 pgmap v95: 89 pgs: 89 active+clean; 4.4 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 615 op/s 2023-06-03T19:44:54.050 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:44:53 smithi089 bash[75068]: cluster 1685821458.0292284 mgr.y (mgr.34104) 112 : cluster 0 pgmap v96: 89 pgs: 89 active+clean; 4.4 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 615 op/s 2023-06-03T19:44:54.050 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:44:53 smithi089 bash[75068]: cluster 1685821460.0301201 mgr.y (mgr.34104) 113 : cluster 0 pgmap v97: 89 pgs: 89 active+clean; 4.2 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 920 op/s 2023-06-03T19:44:54.050 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:44:53 smithi089 bash[75068]: cluster 1685821462.0314925 mgr.y (mgr.34104) 114 : cluster 0 pgmap v98: 89 pgs: 89 active+clean; 4.2 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 609 op/s 2023-06-03T19:44:54.050 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:44:53 smithi089 bash[75068]: audit 1685821462.0626981 mon.b (mon.1) 109 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T19:44:54.050 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:44:53 smithi089 bash[75068]: cluster 1685821464.0322375 mgr.y (mgr.34104) 115 : cluster 0 pgmap v99: 89 pgs: 89 active+clean; 3.9 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 917 op/s 2023-06-03T19:44:54.050 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:44:53 smithi089 bash[75068]: cluster 1685821466.032663 mgr.y (mgr.34104) 116 : cluster 0 pgmap v100: 89 pgs: 89 active+clean; 3.9 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 612 op/s 2023-06-03T19:44:54.050 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:44:53 smithi089 bash[75068]: cluster 1685821468.0330772 mgr.y (mgr.34104) 117 : cluster 0 pgmap v101: 89 pgs: 89 active+clean; 3.9 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 612 op/s 2023-06-03T19:44:54.051 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:44:53 smithi089 bash[75068]: audit 1685821468.6499329 mon.b (mon.1) 110 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-03T19:44:54.051 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:44:53 smithi089 bash[75068]: audit 1685821469.128474 mon.a (mon.0) 92 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:44:54.051 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:44:53 smithi089 bash[75068]: audit 1685821469.150722 mon.a (mon.0) 93 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:44:54.051 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:44:53 smithi089 bash[75068]: cluster 1685821470.0338662 mgr.y (mgr.34104) 118 : cluster 0 pgmap v102: 89 pgs: 89 active+clean; 3.7 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 911 op/s 2023-06-03T19:44:54.051 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:44:53 smithi089 bash[75068]: cluster 1685821472.0343218 mgr.y (mgr.34104) 119 : cluster 0 pgmap v103: 89 pgs: 89 active+clean; 3.7 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 607 op/s 2023-06-03T19:44:54.051 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:44:53 smithi089 bash[75068]: audit 1685821472.543854 mgr.y (mgr.34104) 120 : audit 0 from='client.34215 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2023-06-03T19:44:54.051 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:44:53 smithi089 bash[75068]: cephadm 1685821472.5443 mgr.y (mgr.34104) 121 : cephadm 1 Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:00b1b6d689278b4f97ff790045c8c51d8ca419ac7ee2ebdf11008f2e810a5c78 2023-06-03T19:44:54.052 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:44:53 smithi089 bash[75068]: audit 1685821472.5560281 mon.a (mon.0) 94 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:44:54.052 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:44:53 smithi089 bash[75068]: cluster 1685821474.035018 mgr.y (mgr.34104) 122 : cluster 0 pgmap v104: 89 pgs: 89 active+clean; 3.5 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 906 op/s 2023-06-03T19:44:54.052 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:44:53 smithi089 bash[75068]: audit 1685821475.3681252 mon.a (mon.0) 95 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:44:54.052 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:44:53 smithi089 bash[75068]: audit 1685821475.3785973 mon.a (mon.0) 96 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:44:54.052 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:44:53 smithi089 bash[75068]: audit 1685821475.6649632 mon.a (mon.0) 97 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:44:54.052 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:44:53 smithi089 bash[75068]: audit 1685821475.6763065 mon.a (mon.0) 98 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:44:54.052 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:44:53 smithi089 bash[75068]: cluster 1685821476.03544 mgr.y (mgr.34104) 123 : cluster 0 pgmap v105: 89 pgs: 89 active+clean; 3.5 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 598 op/s 2023-06-03T19:44:54.053 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:44:53 smithi089 bash[75068]: audit 1685821476.2671282 mon.b (mon.1) 111 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-03T19:44:54.053 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:44:53 smithi089 bash[75068]: audit 1685821476.26879 mon.b (mon.1) 112 : audit 1 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-03T19:44:54.053 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:44:53 smithi089 bash[75068]: audit 1685821476.2804465 mon.a (mon.0) 99 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:44:54.053 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:44:53 smithi089 bash[75068]: audit 1685821476.3509307 mon.b (mon.1) 113 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-03T19:44:54.053 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:44:53 smithi089 bash[75068]: audit 1685821476.3565543 mon.b (mon.1) 114 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-06-03T19:44:54.053 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:44:53 smithi089 bash[75068]: audit 1685821476.3589268 mon.b (mon.1) 115 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2023-06-03T19:44:54.053 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:44:53 smithi089 bash[75068]: audit 1685821476.3602748 mon.b (mon.1) 116 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["b"]}]: dispatch 2023-06-03T19:44:54.054 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:44:53 smithi089 bash[75068]: cephadm 1685821476.3632317 mgr.y (mgr.34104) 124 : cephadm 1 Upgrade: It appears safe to stop mon.b 2023-06-03T19:44:54.054 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:44:53 smithi089 bash[75068]: audit 1685821477.0630448 mon.b (mon.1) 117 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T19:44:54.054 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:44:53 smithi089 bash[75068]: cluster 1685821478.0358849 mgr.y (mgr.34104) 125 : cluster 0 pgmap v106: 89 pgs: 89 active+clean; 3.5 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 598 op/s 2023-06-03T19:44:54.054 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:44:53 smithi089 bash[75068]: cluster 1685821480.0366492 mgr.y (mgr.34104) 126 : cluster 0 pgmap v107: 89 pgs: 89 active+clean; 3.3 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 875 op/s 2023-06-03T19:44:54.054 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:44:53 smithi089 bash[75068]: cephadm 1685821481.4643776 mgr.y (mgr.34104) 127 : cephadm 1 Upgrade: Updating mon.b 2023-06-03T19:44:54.054 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:44:53 smithi089 bash[75068]: audit 1685821481.4876273 mon.b (mon.1) 118 : audit 1 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-06-03T19:44:54.054 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:44:53 smithi089 bash[75068]: audit 1685821481.488088 mon.a (mon.0) 100 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:44:54.054 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:44:53 smithi089 bash[75068]: audit 1685821481.4901097 mon.b (mon.1) 119 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-03T19:44:54.055 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:44:53 smithi089 bash[75068]: audit 1685821481.490938 mon.a (mon.0) 101 : audit 0 from='mgr.34104 ' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-06-03T19:44:54.055 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:44:53 smithi089 bash[75068]: cephadm 1685821481.4933066 mgr.y (mgr.34104) 128 : cephadm 1 Deploying daemon mon.b on smithi164 2023-06-03T19:44:54.055 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:44:53 smithi089 bash[75068]: cluster 1685821482.0396972 mgr.y (mgr.34104) 129 : cluster 0 pgmap v108: 89 pgs: 89 active+clean; 3.3 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 575 op/s 2023-06-03T19:44:54.055 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:44:53 smithi089 bash[75068]: audit 1685821482.9795392 mon.c (mon.2) 2 : audit 0 from='client.? 172.21.15.89:0/4084515067' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-03T19:44:54.055 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:44:53 smithi089 bash[75068]: cluster 1685821484.0404198 mgr.y (mgr.34104) 130 : cluster 0 pgmap v109: 89 pgs: 89 active+clean; 3.1 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 876 op/s 2023-06-03T19:44:54.055 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:44:53 smithi089 bash[75068]: audit 1685821484.7535958 mon.c (mon.2) 3 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-06-03T19:44:54.055 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:44:53 smithi089 bash[75068]: audit 1685821484.7539053 mon.c (mon.2) 4 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-06-03T19:44:54.056 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:44:53 smithi089 bash[75068]: audit 1685821484.7541707 mon.c (mon.2) 5 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-06-03T19:44:54.056 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:44:53 smithi089 bash[75068]: cluster 1685821488.5692677 mon.a (mon.0) 108 : cluster 1 overall HEALTH_OK 2023-06-03T19:44:54.056 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:44:53 smithi089 bash[75068]: cluster 1685821488.5708764 mon.b (mon.1) 2 : cluster 1 mon.b calling monitor election 2023-06-03T19:44:54.056 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:44:53 smithi089 bash[75068]: cluster 1685821488.5728936 mon.a (mon.0) 109 : cluster 1 mon.a calling monitor election 2023-06-03T19:44:54.056 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:44:53 smithi089 bash[75068]: cluster 1685821493.5741582 mon.a (mon.0) 110 : cluster 1 mon.a is new leader, mons a,b in quorum (ranks 0,1) 2023-06-03T19:44:54.056 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:44:53 smithi089 bash[75068]: cluster 1685821493.5829618 mon.a (mon.0) 111 : cluster 0 monmap e4: 3 mons at {a=[v2:172.21.15.89:3300/0,v1:172.21.15.89:6789/0],b=[v2:172.21.15.164:3300/0,v1:172.21.15.164:6789/0],c=[v2:172.21.15.89:3301/0,v1:172.21.15.89:6790/0]} removed_ranks: {} 2023-06-03T19:44:54.057 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:44:53 smithi089 bash[75068]: cluster 1685821493.5836842 mon.a (mon.0) 112 : cluster 0 fsmap foo:1 {0=foo.smithi089.giuxuj=up:active} 1 up:standby 2023-06-03T19:44:54.057 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:44:53 smithi089 bash[75068]: cluster 1685821493.5837233 mon.a (mon.0) 113 : cluster 0 osdmap e77: 8 total, 8 up, 8 in 2023-06-03T19:44:54.057 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:44:53 smithi089 bash[75068]: cluster 1685821493.5842338 mon.a (mon.0) 114 : cluster 0 mgrmap e38: y(active, since 3m), standbys: x 2023-06-03T19:44:54.057 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:44:53 smithi089 bash[75068]: cluster 1685821493.5844698 mon.a (mon.0) 115 : cluster 3 Health check failed: 1/3 mons down, quorum a,b (MON_DOWN) 2023-06-03T19:44:54.057 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:44:53 smithi089 bash[75068]: cluster 1685821493.6041977 mon.a (mon.0) 116 : cluster 3 Health detail: HEALTH_WARN 1/3 mons down, quorum a,b 2023-06-03T19:44:54.057 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:44:53 smithi089 bash[75068]: cluster 1685821493.6043186 mon.a (mon.0) 117 : cluster 3 [WRN] MON_DOWN: 1/3 mons down, quorum a,b 2023-06-03T19:44:54.057 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:44:53 smithi089 bash[75068]: cluster 1685821493.6043558 mon.a (mon.0) 118 : cluster 3 mon.c (rank 2) addr [v2:172.21.15.89:3301/0,v1:172.21.15.89:6790/0] is down (out of quorum) 2023-06-03T19:44:54.088 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:53 smithi164 bash[59462]: cluster 1685820174.447331 mon.a (mon.0) 0 : 1 mkfs f1db3478-0243-11ee-9b1d-001a4aab830c 2023-06-03T19:44:54.088 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:53 smithi164 bash[59462]: cephadm 1685821123.1865313 mgr.x (mgr.24634) 56 : cephadm 1 Reconfiguring daemon grafana.a on smithi164 2023-06-03T19:44:54.088 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:53 smithi164 bash[59462]: cephadm 1685821171.429646 mgr.x (mgr.24634) 84 : cephadm 1 Upgrade: Need to upgrade myself (mgr.x) 2023-06-03T19:44:54.089 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:53 smithi164 bash[59462]: cephadm 1685821171.4305422 mgr.x (mgr.24634) 85 : cephadm 1 Upgrade: Need to upgrade myself (mgr.x) 2023-06-03T19:44:54.089 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:53 smithi164 bash[59462]: cephadm 1685821175.321385 mgr.x (mgr.24634) 88 : cephadm 1 Upgrade: Updating mgr.y 2023-06-03T19:44:54.089 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:53 smithi164 bash[59462]: cephadm 1685821175.3339238 mgr.x (mgr.24634) 89 : cephadm 1 Deploying daemon mgr.y on smithi089 2023-06-03T19:44:54.089 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:53 smithi164 bash[59462]: cephadm 1685821201.0951023 mgr.x (mgr.24634) 103 : cephadm 1 Upgrade: Need to upgrade myself (mgr.x) 2023-06-03T19:44:54.089 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:53 smithi164 bash[59462]: cephadm 1685821201.095862 mgr.x (mgr.24634) 104 : cephadm 1 Upgrade: Need to upgrade myself (mgr.x) 2023-06-03T19:44:54.089 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:53 smithi164 bash[59462]: cephadm 1685821201.100557 mgr.x (mgr.24634) 105 : cephadm 1 Failing over to other MGR 2023-06-03T19:44:54.089 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:53 smithi164 bash[59462]: cephadm 1685821204.4230404 mgr.y (mgr.25070) 3 : cephadm 1 [03/Jun/2023:19:40:04] ENGINE Bus STARTING 2023-06-03T19:44:54.089 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:53 smithi164 bash[59462]: cephadm 1685821204.5455484 mgr.y (mgr.25070) 4 : cephadm 1 [03/Jun/2023:19:40:04] ENGINE Serving on https://172.21.15.89:7150 2023-06-03T19:44:54.090 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:53 smithi164 bash[59462]: cephadm 1685821204.6471362 mgr.y (mgr.25070) 5 : cephadm 1 [03/Jun/2023:19:40:04] ENGINE Serving on http://172.21.15.89:8765 2023-06-03T19:44:54.090 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:53 smithi164 bash[59462]: cephadm 1685821204.6472821 mgr.y (mgr.25070) 6 : cephadm 1 [03/Jun/2023:19:40:04] ENGINE Bus STARTED 2023-06-03T19:44:54.090 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:53 smithi164 bash[59462]: cephadm 1685821222.6682131 mgr.y (mgr.25070) 16 : cephadm 1 Updating smithi089:/etc/ceph/ceph.conf 2023-06-03T19:44:54.090 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:53 smithi164 bash[59462]: cephadm 1685821222.6684442 mgr.y (mgr.25070) 17 : cephadm 1 Updating smithi164:/etc/ceph/ceph.conf 2023-06-03T19:44:54.090 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:53 smithi164 bash[59462]: cephadm 1685821222.8840244 mgr.y (mgr.25070) 18 : cephadm 1 Updating smithi164:/var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/config/ceph.conf 2023-06-03T19:44:54.090 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:53 smithi164 bash[59462]: cephadm 1685821222.890312 mgr.y (mgr.25070) 19 : cephadm 1 Updating smithi089:/var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/config/ceph.conf 2023-06-03T19:44:54.090 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:53 smithi164 bash[59462]: cephadm 1685821223.1088037 mgr.y (mgr.25070) 20 : cephadm 1 Updating smithi164:/etc/ceph/ceph.client.admin.keyring 2023-06-03T19:44:54.091 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:53 smithi164 bash[59462]: cephadm 1685821223.111952 mgr.y (mgr.25070) 21 : cephadm 1 Updating smithi089:/etc/ceph/ceph.client.admin.keyring 2023-06-03T19:44:54.091 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:53 smithi164 bash[59462]: cephadm 1685821223.3254883 mgr.y (mgr.25070) 22 : cephadm 1 Updating smithi164:/var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/config/ceph.client.admin.keyring 2023-06-03T19:44:54.091 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:53 smithi164 bash[59462]: cephadm 1685821223.3275669 mgr.y (mgr.25070) 23 : cephadm 1 Updating smithi089:/var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/config/ceph.client.admin.keyring 2023-06-03T19:44:54.091 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:53 smithi164 bash[59462]: cephadm 1685821223.671977 mgr.y (mgr.25070) 24 : cephadm 1 Reconfiguring prometheus.a (dependencies changed)... 2023-06-03T19:44:54.091 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:53 smithi164 bash[59462]: cephadm 1685821223.9393086 mgr.y (mgr.25070) 25 : cephadm 1 Reconfiguring daemon prometheus.a on smithi164 2023-06-03T19:44:54.091 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:53 smithi164 bash[59462]: cephadm 1685821236.3840792 mgr.y (mgr.25070) 34 : cephadm 1 Upgrade: Updating mgr.x 2023-06-03T19:44:54.091 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:53 smithi164 bash[59462]: cephadm 1685821236.390095 mgr.y (mgr.25070) 35 : cephadm 1 Deploying daemon mgr.x on smithi164 2023-06-03T19:44:54.091 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:53 smithi164 bash[59462]: cephadm 1685821261.9984038 mgr.y (mgr.25070) 48 : cephadm 1 Upgrade: Setting container_image for all mgr 2023-06-03T19:44:54.092 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:53 smithi164 bash[59462]: cephadm 1685821262.03976 mgr.y (mgr.25070) 49 : cephadm 1 Upgrade: It appears safe to stop mon.a 2023-06-03T19:44:54.092 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:53 smithi164 bash[59462]: cephadm 1685821266.7819054 mgr.y (mgr.25070) 53 : cephadm 1 Upgrade: Updating mon.a 2023-06-03T19:44:54.092 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:53 smithi164 bash[59462]: cephadm 1685821266.8025656 mgr.y (mgr.25070) 54 : cephadm 1 Deploying daemon mon.a on smithi089 2023-06-03T19:44:54.092 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:53 smithi164 bash[59462]: cephadm 1685821282.4937708 mgr.y (mgr.34104) 1 : cephadm 1 [03/Jun/2023:19:41:22] ENGINE Bus STARTING 2023-06-03T19:44:54.092 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:53 smithi164 bash[59462]: cephadm 1685821282.6101174 mgr.y (mgr.34104) 2 : cephadm 1 [03/Jun/2023:19:41:22] ENGINE Serving on https://172.21.15.89:7150 2023-06-03T19:44:54.092 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:53 smithi164 bash[59462]: cephadm 1685821282.7118428 mgr.y (mgr.34104) 3 : cephadm 1 [03/Jun/2023:19:41:22] ENGINE Serving on http://172.21.15.89:8765 2023-06-03T19:44:54.092 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:53 smithi164 bash[59462]: cephadm 1685821282.7121873 mgr.y (mgr.34104) 4 : cephadm 1 [03/Jun/2023:19:41:22] ENGINE Bus STARTED 2023-06-03T19:44:54.093 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:53 smithi164 bash[59462]: cephadm 1685821287.9637723 mgr.y (mgr.34104) 11 : cephadm 1 Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:00b1b6d689278b4f97ff790045c8c51d8ca419ac7ee2ebdf11008f2e810a5c78 2023-06-03T19:44:54.093 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:53 smithi164 bash[59462]: cephadm 1685821299.4224992 mgr.y (mgr.34104) 17 : cephadm 1 Updating smithi089:/etc/ceph/ceph.conf 2023-06-03T19:44:54.093 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:53 smithi164 bash[59462]: cephadm 1685821299.4228904 mgr.y (mgr.34104) 18 : cephadm 1 Updating smithi164:/etc/ceph/ceph.conf 2023-06-03T19:44:54.093 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:53 smithi164 bash[59462]: cephadm 1685821299.587414 mgr.y (mgr.34104) 19 : cephadm 1 Updating smithi089:/var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/config/ceph.conf 2023-06-03T19:44:54.093 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:53 smithi164 bash[59462]: cephadm 1685821299.5885887 mgr.y (mgr.34104) 20 : cephadm 1 Updating smithi164:/var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/config/ceph.conf 2023-06-03T19:44:54.093 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:53 smithi164 bash[59462]: cephadm 1685821299.7529323 mgr.y (mgr.34104) 21 : cephadm 1 Updating smithi089:/etc/ceph/ceph.client.admin.keyring 2023-06-03T19:44:54.093 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:53 smithi164 bash[59462]: cephadm 1685821299.7543862 mgr.y (mgr.34104) 22 : cephadm 1 Updating smithi164:/etc/ceph/ceph.client.admin.keyring 2023-06-03T19:44:54.093 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:53 smithi164 bash[59462]: cephadm 1685821299.9186985 mgr.y (mgr.34104) 23 : cephadm 1 Updating smithi089:/var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/config/ceph.client.admin.keyring 2023-06-03T19:44:54.094 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:53 smithi164 bash[59462]: cephadm 1685821299.919883 mgr.y (mgr.34104) 24 : cephadm 1 Updating smithi164:/var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/config/ceph.client.admin.keyring 2023-06-03T19:44:54.094 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:53 smithi164 bash[59462]: cephadm 1685821360.1272502 mgr.y (mgr.34104) 57 : cephadm 1 Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:00b1b6d689278b4f97ff790045c8c51d8ca419ac7ee2ebdf11008f2e810a5c78 2023-06-03T19:44:54.094 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:53 smithi164 bash[59462]: cephadm 1685821367.5300949 mgr.y (mgr.34104) 61 : cephadm 1 Upgrade: It appears safe to stop mon.c 2023-06-03T19:44:54.094 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:53 smithi164 bash[59462]: cephadm 1685821372.2798438 mgr.y (mgr.34104) 65 : cephadm 1 Upgrade: Updating mon.c 2023-06-03T19:44:54.094 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:53 smithi164 bash[59462]: audit 1685821372.3102217 mon.c (mon.2) 122 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-03T19:44:54.094 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:53 smithi164 bash[59462]: cephadm 1685821372.311988 mgr.y (mgr.34104) 66 : cephadm 1 Deploying daemon mon.c on smithi089 2023-06-03T19:44:54.094 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:53 smithi164 bash[59462]: audit 1685821375.0654147 mon.b (mon.1) 92 : audit 0 from='client.? 172.21.15.89:0/1498143111' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-03T19:44:54.095 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:53 smithi164 bash[59462]: cluster 1685821385.9985988 mgr.y (mgr.34104) 73 : cluster 0 pgmap v60: 89 pgs: 89 active+clean; 6.2 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 45 MiB/s wr, 723 op/s 2023-06-03T19:44:54.095 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:53 smithi164 bash[59462]: audit 1685821386.594359 mon.a (mon.0) 82 : audit 0 from='client.? 172.21.15.89:0/3649894161' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-03T19:44:54.095 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:53 smithi164 bash[59462]: audit 1685821387.063322 mon.b (mon.1) 99 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T19:44:54.095 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:53 smithi164 bash[59462]: cluster 1685821387.9991255 mgr.y (mgr.34104) 74 : cluster 0 pgmap v61: 89 pgs: 89 active+clean; 6.3 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 52 MiB/s wr, 831 op/s 2023-06-03T19:44:54.095 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:53 smithi164 bash[59462]: cluster 1685821389.9997709 mgr.y (mgr.34104) 75 : cluster 0 pgmap v62: 89 pgs: 89 active+clean; 6.5 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 53 MiB/s wr, 855 op/s 2023-06-03T19:44:54.095 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:53 smithi164 bash[59462]: cluster 1685821392.000228 mgr.y (mgr.34104) 76 : cluster 0 pgmap v63: 89 pgs: 89 active+clean; 6.5 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 45 MiB/s wr, 722 op/s 2023-06-03T19:44:54.095 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:53 smithi164 bash[59462]: audit 1685821393.4656174 mgr.y (mgr.34104) 77 : audit 0 from='client.44102 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-06-03T19:44:54.096 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:53 smithi164 bash[59462]: audit 1685821393.6309147 mon.a (mon.0) 83 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:44:54.096 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:53 smithi164 bash[59462]: audit 1685821393.6497493 mon.a (mon.0) 84 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:44:54.096 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:53 smithi164 bash[59462]: cluster 1685821394.00106 mgr.y (mgr.34104) 78 : cluster 0 pgmap v64: 89 pgs: 89 active+clean; 6.7 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 66 MiB/s wr, 1.06k op/s 2023-06-03T19:44:54.096 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:53 smithi164 bash[59462]: audit 1685821394.7517736 mon.a (mon.0) 85 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:44:54.096 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:53 smithi164 bash[59462]: audit 1685821394.768501 mon.a (mon.0) 86 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:44:54.096 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:53 smithi164 bash[59462]: cluster 1685821396.0015018 mgr.y (mgr.34104) 79 : cluster 0 pgmap v65: 89 pgs: 89 active+clean; 6.7 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 43 MiB/s wr, 681 op/s 2023-06-03T19:44:54.096 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:53 smithi164 bash[59462]: cluster 1685821398.002088 mgr.y (mgr.34104) 80 : cluster 0 pgmap v66: 89 pgs: 89 active+clean; 6.7 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 45 MiB/s wr, 718 op/s 2023-06-03T19:44:54.096 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:53 smithi164 bash[59462]: audit 1685821399.3379703 mgr.y (mgr.34104) 81 : audit 0 from='client.34209 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2023-06-03T19:44:54.097 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:53 smithi164 bash[59462]: cephadm 1685821399.3385687 mgr.y (mgr.34104) 82 : cephadm 1 Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:00b1b6d689278b4f97ff790045c8c51d8ca419ac7ee2ebdf11008f2e810a5c78 2023-06-03T19:44:54.097 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:53 smithi164 bash[59462]: audit 1685821399.344722 mon.a (mon.0) 87 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:44:54.097 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:53 smithi164 bash[59462]: cluster 1685821400.0027843 mgr.y (mgr.34104) 83 : cluster 0 pgmap v67: 89 pgs: 89 active+clean; 7.0 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 56 MiB/s wr, 896 op/s 2023-06-03T19:44:54.097 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:53 smithi164 bash[59462]: cluster 1685821402.0032067 mgr.y (mgr.34104) 84 : cluster 0 pgmap v68: 89 pgs: 89 active+clean; 7.0 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 41 MiB/s wr, 657 op/s 2023-06-03T19:44:54.097 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:53 smithi164 bash[59462]: audit 1685821402.0612197 mon.b (mon.1) 100 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T19:44:54.097 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:53 smithi164 bash[59462]: cluster 1685821404.0069954 mgr.y (mgr.34104) 85 : cluster 0 pgmap v69: 89 pgs: 89 active+clean; 6.7 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 85 B/s rd, 42 MiB/s wr, 1.00k op/s 2023-06-03T19:44:54.097 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:53 smithi164 bash[59462]: cluster 1685821406.007473 mgr.y (mgr.34104) 86 : cluster 0 pgmap v70: 89 pgs: 89 active+clean; 6.7 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 85 B/s rd, 22 MiB/s wr, 671 op/s 2023-06-03T19:44:54.097 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:53 smithi164 bash[59462]: cluster 1685821408.0078633 mgr.y (mgr.34104) 87 : cluster 0 pgmap v71: 89 pgs: 89 active+clean; 6.7 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 85 B/s rd, 22 MiB/s wr, 671 op/s 2023-06-03T19:44:54.098 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:53 smithi164 bash[59462]: audit 1685821408.4267251 mon.a (mon.0) 88 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:44:54.098 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:53 smithi164 bash[59462]: audit 1685821408.4374998 mon.a (mon.0) 89 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:44:54.098 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:53 smithi164 bash[59462]: audit 1685821408.4380069 mon.b (mon.1) 101 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-03T19:44:54.098 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:53 smithi164 bash[59462]: audit 1685821408.4439259 mon.b (mon.1) 102 : audit 1 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-03T19:44:54.098 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:53 smithi164 bash[59462]: audit 1685821408.452624 mon.a (mon.0) 90 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:44:54.098 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:53 smithi164 bash[59462]: audit 1685821408.5381055 mon.b (mon.1) 103 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-03T19:44:54.099 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:53 smithi164 bash[59462]: audit 1685821408.5485957 mon.b (mon.1) 104 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-03T19:44:54.099 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:53 smithi164 bash[59462]: audit 1685821408.5498192 mon.b (mon.1) 105 : audit 1 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-03T19:44:54.099 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:53 smithi164 bash[59462]: audit 1685821408.5613625 mon.a (mon.0) 91 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:44:54.099 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:53 smithi164 bash[59462]: cluster 1685821410.0086102 mgr.y (mgr.34104) 88 : cluster 0 pgmap v72: 89 pgs: 89 active+clean; 6.5 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 85 B/s rd, 19 MiB/s wr, 969 op/s 2023-06-03T19:44:54.099 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:53 smithi164 bash[59462]: cluster 1685821412.0090332 mgr.y (mgr.34104) 89 : cluster 0 pgmap v73: 89 pgs: 89 active+clean; 6.5 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 85 B/s rd, 1.4 MiB/s wr, 684 op/s 2023-06-03T19:44:54.100 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:53 smithi164 bash[59462]: cluster 1685821414.0097935 mgr.y (mgr.34104) 90 : cluster 0 pgmap v74: 89 pgs: 89 active+clean; 6.2 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 85 B/s rd, 1.4 MiB/s wr, 1.02k op/s 2023-06-03T19:44:54.100 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:53 smithi164 bash[59462]: cluster 1685821416.0102537 mgr.y (mgr.34104) 91 : cluster 0 pgmap v75: 89 pgs: 89 active+clean; 6.2 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 673 op/s 2023-06-03T19:44:54.100 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:53 smithi164 bash[59462]: audit 1685821417.0620108 mon.b (mon.1) 106 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T19:44:54.100 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:53 smithi164 bash[59462]: cluster 1685821418.0107417 mgr.y (mgr.34104) 92 : cluster 0 pgmap v76: 89 pgs: 89 active+clean; 6.2 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 673 op/s 2023-06-03T19:44:54.100 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:53 smithi164 bash[59462]: cluster 1685821420.0114686 mgr.y (mgr.34104) 93 : cluster 0 pgmap v77: 89 pgs: 89 active+clean; 6.0 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 998 op/s 2023-06-03T19:44:54.100 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:53 smithi164 bash[59462]: cluster 1685821422.011843 mgr.y (mgr.34104) 94 : cluster 0 pgmap v78: 89 pgs: 89 active+clean; 6.0 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 662 op/s 2023-06-03T19:44:54.101 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:53 smithi164 bash[59462]: cluster 1685821424.013851 mgr.y (mgr.34104) 95 : cluster 0 pgmap v79: 89 pgs: 89 active+clean; 5.8 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 984 op/s 2023-06-03T19:44:54.101 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:53 smithi164 bash[59462]: cluster 1685821426.0143747 mgr.y (mgr.34104) 96 : cluster 0 pgmap v80: 89 pgs: 89 active+clean; 5.8 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 646 op/s 2023-06-03T19:44:54.101 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:53 smithi164 bash[59462]: cluster 1685821428.0147314 mgr.y (mgr.34104) 97 : cluster 0 pgmap v81: 89 pgs: 89 active+clean; 5.8 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 647 op/s 2023-06-03T19:44:54.101 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:53 smithi164 bash[59462]: cluster 1685821430.0154066 mgr.y (mgr.34104) 98 : cluster 0 pgmap v82: 89 pgs: 89 active+clean; 5.5 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 965 op/s 2023-06-03T19:44:54.101 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:53 smithi164 bash[59462]: cluster 1685821432.0165899 mgr.y (mgr.34104) 99 : cluster 0 pgmap v83: 89 pgs: 89 active+clean; 5.5 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 640 op/s 2023-06-03T19:44:54.101 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:53 smithi164 bash[59462]: audit 1685821432.0621827 mon.b (mon.1) 107 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T19:44:54.101 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:53 smithi164 bash[59462]: cluster 1685821434.0173318 mgr.y (mgr.34104) 100 : cluster 0 pgmap v84: 89 pgs: 89 active+clean; 5.3 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 961 op/s 2023-06-03T19:44:54.102 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:53 smithi164 bash[59462]: cluster 1685821436.0177891 mgr.y (mgr.34104) 101 : cluster 0 pgmap v85: 89 pgs: 89 active+clean; 5.3 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 639 op/s 2023-06-03T19:44:54.102 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:53 smithi164 bash[59462]: cluster 1685821438.0200334 mgr.y (mgr.34104) 102 : cluster 0 pgmap v86: 89 pgs: 89 active+clean; 5.3 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 639 op/s 2023-06-03T19:44:54.102 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:53 smithi164 bash[59462]: cluster 1685821440.020801 mgr.y (mgr.34104) 103 : cluster 0 pgmap v87: 89 pgs: 89 active+clean; 5.1 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 951 op/s 2023-06-03T19:44:54.102 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:53 smithi164 bash[59462]: cluster 1685821442.0212224 mgr.y (mgr.34104) 104 : cluster 0 pgmap v88: 89 pgs: 89 active+clean; 5.1 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 632 op/s 2023-06-03T19:44:54.102 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:53 smithi164 bash[59462]: cluster 1685821444.0253873 mgr.y (mgr.34104) 105 : cluster 0 pgmap v89: 89 pgs: 89 active+clean; 4.8 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 942 op/s 2023-06-03T19:44:54.102 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:53 smithi164 bash[59462]: cluster 1685821446.0258205 mgr.y (mgr.34104) 106 : cluster 0 pgmap v90: 89 pgs: 89 active+clean; 4.8 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 621 op/s 2023-06-03T19:44:54.103 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:53 smithi164 bash[59462]: audit 1685821447.0623534 mon.b (mon.1) 108 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T19:44:54.103 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:53 smithi164 bash[59462]: cluster 1685821448.0262258 mgr.y (mgr.34104) 107 : cluster 0 pgmap v91: 89 pgs: 89 active+clean; 4.8 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 621 op/s 2023-06-03T19:44:54.103 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:53 smithi164 bash[59462]: cluster 1685821450.026844 mgr.y (mgr.34104) 108 : cluster 0 pgmap v92: 89 pgs: 89 active+clean; 4.6 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 932 op/s 2023-06-03T19:44:54.103 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:53 smithi164 bash[59462]: cluster 1685821452.0272791 mgr.y (mgr.34104) 109 : cluster 0 pgmap v93: 89 pgs: 89 active+clean; 4.6 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 620 op/s 2023-06-03T19:44:54.103 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:53 smithi164 bash[59462]: cluster 1685821454.0280442 mgr.y (mgr.34104) 110 : cluster 0 pgmap v94: 89 pgs: 89 active+clean; 4.4 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 925 op/s 2023-06-03T19:44:54.103 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:53 smithi164 bash[59462]: cluster 1685821456.0288029 mgr.y (mgr.34104) 111 : cluster 0 pgmap v95: 89 pgs: 89 active+clean; 4.4 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 615 op/s 2023-06-03T19:44:54.104 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:53 smithi164 bash[59462]: cluster 1685821458.0292284 mgr.y (mgr.34104) 112 : cluster 0 pgmap v96: 89 pgs: 89 active+clean; 4.4 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 615 op/s 2023-06-03T19:44:54.104 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:53 smithi164 bash[59462]: cluster 1685821460.0301201 mgr.y (mgr.34104) 113 : cluster 0 pgmap v97: 89 pgs: 89 active+clean; 4.2 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 920 op/s 2023-06-03T19:44:54.104 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:53 smithi164 bash[59462]: cluster 1685821462.0314925 mgr.y (mgr.34104) 114 : cluster 0 pgmap v98: 89 pgs: 89 active+clean; 4.2 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 609 op/s 2023-06-03T19:44:54.104 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:53 smithi164 bash[59462]: audit 1685821462.0626981 mon.b (mon.1) 109 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T19:44:54.104 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:53 smithi164 bash[59462]: cluster 1685821464.0322375 mgr.y (mgr.34104) 115 : cluster 0 pgmap v99: 89 pgs: 89 active+clean; 3.9 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 917 op/s 2023-06-03T19:44:54.104 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:53 smithi164 bash[59462]: cluster 1685821466.032663 mgr.y (mgr.34104) 116 : cluster 0 pgmap v100: 89 pgs: 89 active+clean; 3.9 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 612 op/s 2023-06-03T19:44:54.104 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:53 smithi164 bash[59462]: cluster 1685821468.0330772 mgr.y (mgr.34104) 117 : cluster 0 pgmap v101: 89 pgs: 89 active+clean; 3.9 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 612 op/s 2023-06-03T19:44:54.104 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:53 smithi164 bash[59462]: audit 1685821468.6499329 mon.b (mon.1) 110 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-03T19:44:54.105 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:53 smithi164 bash[59462]: audit 1685821469.128474 mon.a (mon.0) 92 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:44:54.105 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:53 smithi164 bash[59462]: audit 1685821469.150722 mon.a (mon.0) 93 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:44:54.105 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:53 smithi164 bash[59462]: cluster 1685821470.0338662 mgr.y (mgr.34104) 118 : cluster 0 pgmap v102: 89 pgs: 89 active+clean; 3.7 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 911 op/s 2023-06-03T19:44:54.105 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:53 smithi164 bash[59462]: cluster 1685821472.0343218 mgr.y (mgr.34104) 119 : cluster 0 pgmap v103: 89 pgs: 89 active+clean; 3.7 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 607 op/s 2023-06-03T19:44:54.105 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:53 smithi164 bash[59462]: audit 1685821472.543854 mgr.y (mgr.34104) 120 : audit 0 from='client.34215 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2023-06-03T19:44:54.105 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:53 smithi164 bash[59462]: cephadm 1685821472.5443 mgr.y (mgr.34104) 121 : cephadm 1 Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:00b1b6d689278b4f97ff790045c8c51d8ca419ac7ee2ebdf11008f2e810a5c78 2023-06-03T19:44:54.105 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:53 smithi164 bash[59462]: audit 1685821472.5560281 mon.a (mon.0) 94 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:44:54.106 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:53 smithi164 bash[59462]: cluster 1685821474.035018 mgr.y (mgr.34104) 122 : cluster 0 pgmap v104: 89 pgs: 89 active+clean; 3.5 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 906 op/s 2023-06-03T19:44:54.106 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:53 smithi164 bash[59462]: audit 1685821475.3681252 mon.a (mon.0) 95 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:44:54.106 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:53 smithi164 bash[59462]: audit 1685821475.3785973 mon.a (mon.0) 96 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:44:54.106 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:53 smithi164 bash[59462]: audit 1685821475.6649632 mon.a (mon.0) 97 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:44:54.106 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:53 smithi164 bash[59462]: audit 1685821475.6763065 mon.a (mon.0) 98 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:44:54.106 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:53 smithi164 bash[59462]: cluster 1685821476.03544 mgr.y (mgr.34104) 123 : cluster 0 pgmap v105: 89 pgs: 89 active+clean; 3.5 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 598 op/s 2023-06-03T19:44:54.106 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:53 smithi164 bash[59462]: audit 1685821476.2671282 mon.b (mon.1) 111 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-03T19:44:54.106 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:53 smithi164 bash[59462]: audit 1685821476.26879 mon.b (mon.1) 112 : audit 1 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-03T19:44:54.107 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:53 smithi164 bash[59462]: audit 1685821476.2804465 mon.a (mon.0) 99 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:44:54.107 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:53 smithi164 bash[59462]: audit 1685821476.3509307 mon.b (mon.1) 113 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-03T19:44:54.107 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:53 smithi164 bash[59462]: audit 1685821476.3565543 mon.b (mon.1) 114 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-06-03T19:44:54.107 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:53 smithi164 bash[59462]: audit 1685821476.3589268 mon.b (mon.1) 115 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2023-06-03T19:44:54.107 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:53 smithi164 bash[59462]: audit 1685821476.3602748 mon.b (mon.1) 116 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["b"]}]: dispatch 2023-06-03T19:44:54.107 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:53 smithi164 bash[59462]: cephadm 1685821476.3632317 mgr.y (mgr.34104) 124 : cephadm 1 Upgrade: It appears safe to stop mon.b 2023-06-03T19:44:54.108 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:53 smithi164 bash[59462]: audit 1685821477.0630448 mon.b (mon.1) 117 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T19:44:54.108 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:53 smithi164 bash[59462]: cluster 1685821478.0358849 mgr.y (mgr.34104) 125 : cluster 0 pgmap v106: 89 pgs: 89 active+clean; 3.5 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 598 op/s 2023-06-03T19:44:54.108 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:53 smithi164 bash[59462]: cluster 1685821480.0366492 mgr.y (mgr.34104) 126 : cluster 0 pgmap v107: 89 pgs: 89 active+clean; 3.3 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 875 op/s 2023-06-03T19:44:54.108 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:53 smithi164 bash[59462]: cephadm 1685821481.4643776 mgr.y (mgr.34104) 127 : cephadm 1 Upgrade: Updating mon.b 2023-06-03T19:44:54.108 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:53 smithi164 bash[59462]: audit 1685821481.4876273 mon.b (mon.1) 118 : audit 1 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-06-03T19:44:54.108 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:53 smithi164 bash[59462]: audit 1685821481.488088 mon.a (mon.0) 100 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:44:54.108 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:53 smithi164 bash[59462]: audit 1685821481.4901097 mon.b (mon.1) 119 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-03T19:44:54.108 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:53 smithi164 bash[59462]: audit 1685821481.490938 mon.a (mon.0) 101 : audit 0 from='mgr.34104 ' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-06-03T19:44:54.109 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:53 smithi164 bash[59462]: cephadm 1685821481.4933066 mgr.y (mgr.34104) 128 : cephadm 1 Deploying daemon mon.b on smithi164 2023-06-03T19:44:54.109 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:53 smithi164 bash[59462]: cluster 1685821482.0396972 mgr.y (mgr.34104) 129 : cluster 0 pgmap v108: 89 pgs: 89 active+clean; 3.3 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 575 op/s 2023-06-03T19:44:54.109 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:53 smithi164 bash[59462]: audit 1685821482.9795392 mon.c (mon.2) 2 : audit 0 from='client.? 172.21.15.89:0/4084515067' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-03T19:44:54.109 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:53 smithi164 bash[59462]: cluster 1685821484.0404198 mgr.y (mgr.34104) 130 : cluster 0 pgmap v109: 89 pgs: 89 active+clean; 3.1 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 876 op/s 2023-06-03T19:44:54.109 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:53 smithi164 bash[59462]: audit 1685821484.7535958 mon.c (mon.2) 3 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-06-03T19:44:54.109 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:53 smithi164 bash[59462]: audit 1685821484.7539053 mon.c (mon.2) 4 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-06-03T19:44:54.109 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:53 smithi164 bash[59462]: audit 1685821484.7541707 mon.c (mon.2) 5 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-06-03T19:44:54.109 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:53 smithi164 bash[59462]: cluster 1685821488.5692677 mon.a (mon.0) 108 : cluster 1 overall HEALTH_OK 2023-06-03T19:44:54.110 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:53 smithi164 bash[59462]: cluster 1685821488.5708764 mon.b (mon.1) 2 : cluster 1 mon.b calling monitor election 2023-06-03T19:44:54.110 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:53 smithi164 bash[59462]: cluster 1685821488.5728936 mon.a (mon.0) 109 : cluster 1 mon.a calling monitor election 2023-06-03T19:44:54.110 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:53 smithi164 bash[59462]: cluster 1685821493.5741582 mon.a (mon.0) 110 : cluster 1 mon.a is new leader, mons a,b in quorum (ranks 0,1) 2023-06-03T19:44:54.110 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:53 smithi164 bash[59462]: cluster 1685821493.5829618 mon.a (mon.0) 111 : cluster 0 monmap e4: 3 mons at {a=[v2:172.21.15.89:3300/0,v1:172.21.15.89:6789/0],b=[v2:172.21.15.164:3300/0,v1:172.21.15.164:6789/0],c=[v2:172.21.15.89:3301/0,v1:172.21.15.89:6790/0]} removed_ranks: {} 2023-06-03T19:44:54.110 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:53 smithi164 bash[59462]: cluster 1685821493.5836842 mon.a (mon.0) 112 : cluster 0 fsmap foo:1 {0=foo.smithi089.giuxuj=up:active} 1 up:standby 2023-06-03T19:44:54.110 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:53 smithi164 bash[59462]: cluster 1685821493.5837233 mon.a (mon.0) 113 : cluster 0 osdmap e77: 8 total, 8 up, 8 in 2023-06-03T19:44:54.111 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:53 smithi164 bash[59462]: cluster 1685821493.5842338 mon.a (mon.0) 114 : cluster 0 mgrmap e38: y(active, since 3m), standbys: x 2023-06-03T19:44:54.111 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:53 smithi164 bash[59462]: cluster 1685821493.5844698 mon.a (mon.0) 115 : cluster 3 Health check failed: 1/3 mons down, quorum a,b (MON_DOWN) 2023-06-03T19:44:54.111 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:53 smithi164 bash[59462]: cluster 1685821493.6041977 mon.a (mon.0) 116 : cluster 3 Health detail: HEALTH_WARN 1/3 mons down, quorum a,b 2023-06-03T19:44:54.111 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:53 smithi164 bash[59462]: cluster 1685821493.6043186 mon.a (mon.0) 117 : cluster 3 [WRN] MON_DOWN: 1/3 mons down, quorum a,b 2023-06-03T19:44:54.111 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:53 smithi164 bash[59462]: cluster 1685821493.6043558 mon.a (mon.0) 118 : cluster 3 mon.c (rank 2) addr [v2:172.21.15.89:3301/0,v1:172.21.15.89:6790/0] is down (out of quorum) 2023-06-03T19:44:54.245 INFO:teuthology.orchestra.run.smithi089.stdout: "ceph version 18.0.0-3532-g8e24bcf4 (8e24bcf462a1d94c2e12aa4e26419634883a1e88) reef (rc)": 3 2023-06-03T19:44:54.780 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:44:54 smithi089 bash[75068]: audit 1685821494.1984737 mon.a (mon.0) 119 : audit 0 from='client.? 172.21.15.89:0/3652051568' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-03T19:44:55.085 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:54 smithi164 bash[59462]: audit 1685821494.1984737 mon.a (mon.0) 119 : audit 0 from='client.? 172.21.15.89:0/3652051568' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-03T19:44:55.653 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:44:55 smithi089 bash[78452]: cluster 1685820174.447331 mon.a (mon.0) 0 : 1 mkfs f1db3478-0243-11ee-9b1d-001a4aab830c 2023-06-03T19:44:55.653 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:44:55 smithi089 bash[78452]: cephadm 1685821123.1865313 mgr.x (mgr.24634) 56 : cephadm 1 Reconfiguring daemon grafana.a on smithi164 2023-06-03T19:44:55.654 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:44:55 smithi089 bash[78452]: cephadm 1685821171.429646 mgr.x (mgr.24634) 84 : cephadm 1 Upgrade: Need to upgrade myself (mgr.x) 2023-06-03T19:44:55.654 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:44:55 smithi089 bash[78452]: cephadm 1685821171.4305422 mgr.x (mgr.24634) 85 : cephadm 1 Upgrade: Need to upgrade myself (mgr.x) 2023-06-03T19:44:55.654 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:44:55 smithi089 bash[78452]: cephadm 1685821175.321385 mgr.x (mgr.24634) 88 : cephadm 1 Upgrade: Updating mgr.y 2023-06-03T19:44:55.654 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:44:55 smithi089 bash[78452]: cephadm 1685821175.3339238 mgr.x (mgr.24634) 89 : cephadm 1 Deploying daemon mgr.y on smithi089 2023-06-03T19:44:55.654 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:44:55 smithi089 bash[78452]: cephadm 1685821201.0951023 mgr.x (mgr.24634) 103 : cephadm 1 Upgrade: Need to upgrade myself (mgr.x) 2023-06-03T19:44:55.654 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:44:55 smithi089 bash[78452]: cephadm 1685821201.095862 mgr.x (mgr.24634) 104 : cephadm 1 Upgrade: Need to upgrade myself (mgr.x) 2023-06-03T19:44:55.654 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:44:55 smithi089 bash[78452]: cephadm 1685821201.100557 mgr.x (mgr.24634) 105 : cephadm 1 Failing over to other MGR 2023-06-03T19:44:55.655 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:44:55 smithi089 bash[78452]: cephadm 1685821204.4230404 mgr.y (mgr.25070) 3 : cephadm 1 [03/Jun/2023:19:40:04] ENGINE Bus STARTING 2023-06-03T19:44:55.655 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:44:55 smithi089 bash[78452]: cephadm 1685821204.5455484 mgr.y (mgr.25070) 4 : cephadm 1 [03/Jun/2023:19:40:04] ENGINE Serving on https://172.21.15.89:7150 2023-06-03T19:44:55.655 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:44:55 smithi089 bash[78452]: cephadm 1685821204.6471362 mgr.y (mgr.25070) 5 : cephadm 1 [03/Jun/2023:19:40:04] ENGINE Serving on http://172.21.15.89:8765 2023-06-03T19:44:55.655 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:44:55 smithi089 bash[78452]: cephadm 1685821204.6472821 mgr.y (mgr.25070) 6 : cephadm 1 [03/Jun/2023:19:40:04] ENGINE Bus STARTED 2023-06-03T19:44:55.655 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:44:55 smithi089 bash[78452]: cephadm 1685821222.6682131 mgr.y (mgr.25070) 16 : cephadm 1 Updating smithi089:/etc/ceph/ceph.conf 2023-06-03T19:44:55.655 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:44:55 smithi089 bash[78452]: cephadm 1685821222.6684442 mgr.y (mgr.25070) 17 : cephadm 1 Updating smithi164:/etc/ceph/ceph.conf 2023-06-03T19:44:55.656 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:44:55 smithi089 bash[78452]: cephadm 1685821222.8840244 mgr.y (mgr.25070) 18 : cephadm 1 Updating smithi164:/var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/config/ceph.conf 2023-06-03T19:44:55.656 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:44:55 smithi089 bash[78452]: cephadm 1685821222.890312 mgr.y (mgr.25070) 19 : cephadm 1 Updating smithi089:/var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/config/ceph.conf 2023-06-03T19:44:55.656 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:44:55 smithi089 bash[78452]: cephadm 1685821223.1088037 mgr.y (mgr.25070) 20 : cephadm 1 Updating smithi164:/etc/ceph/ceph.client.admin.keyring 2023-06-03T19:44:55.656 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:44:55 smithi089 bash[78452]: cephadm 1685821223.111952 mgr.y (mgr.25070) 21 : cephadm 1 Updating smithi089:/etc/ceph/ceph.client.admin.keyring 2023-06-03T19:44:55.656 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:44:55 smithi089 bash[78452]: cephadm 1685821223.3254883 mgr.y (mgr.25070) 22 : cephadm 1 Updating smithi164:/var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/config/ceph.client.admin.keyring 2023-06-03T19:44:55.656 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:44:55 smithi089 bash[78452]: cephadm 1685821223.3275669 mgr.y (mgr.25070) 23 : cephadm 1 Updating smithi089:/var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/config/ceph.client.admin.keyring 2023-06-03T19:44:55.657 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:44:55 smithi089 bash[78452]: cephadm 1685821223.671977 mgr.y (mgr.25070) 24 : cephadm 1 Reconfiguring prometheus.a (dependencies changed)... 2023-06-03T19:44:55.657 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:44:55 smithi089 bash[78452]: cephadm 1685821223.9393086 mgr.y (mgr.25070) 25 : cephadm 1 Reconfiguring daemon prometheus.a on smithi164 2023-06-03T19:44:55.657 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:44:55 smithi089 bash[78452]: cephadm 1685821236.3840792 mgr.y (mgr.25070) 34 : cephadm 1 Upgrade: Updating mgr.x 2023-06-03T19:44:55.657 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:44:55 smithi089 bash[78452]: cephadm 1685821236.390095 mgr.y (mgr.25070) 35 : cephadm 1 Deploying daemon mgr.x on smithi164 2023-06-03T19:44:55.657 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:44:55 smithi089 bash[78452]: cephadm 1685821261.9984038 mgr.y (mgr.25070) 48 : cephadm 1 Upgrade: Setting container_image for all mgr 2023-06-03T19:44:55.658 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:44:55 smithi089 bash[78452]: cephadm 1685821262.03976 mgr.y (mgr.25070) 49 : cephadm 1 Upgrade: It appears safe to stop mon.a 2023-06-03T19:44:55.658 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:44:55 smithi089 bash[78452]: cephadm 1685821266.7819054 mgr.y (mgr.25070) 53 : cephadm 1 Upgrade: Updating mon.a 2023-06-03T19:44:55.658 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:44:55 smithi089 bash[78452]: cephadm 1685821266.8025656 mgr.y (mgr.25070) 54 : cephadm 1 Deploying daemon mon.a on smithi089 2023-06-03T19:44:55.658 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:44:55 smithi089 bash[78452]: cephadm 1685821282.4937708 mgr.y (mgr.34104) 1 : cephadm 1 [03/Jun/2023:19:41:22] ENGINE Bus STARTING 2023-06-03T19:44:55.658 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:44:55 smithi089 bash[78452]: cephadm 1685821282.6101174 mgr.y (mgr.34104) 2 : cephadm 1 [03/Jun/2023:19:41:22] ENGINE Serving on https://172.21.15.89:7150 2023-06-03T19:44:55.658 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:44:55 smithi089 bash[78452]: cephadm 1685821282.7118428 mgr.y (mgr.34104) 3 : cephadm 1 [03/Jun/2023:19:41:22] ENGINE Serving on http://172.21.15.89:8765 2023-06-03T19:44:55.658 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:44:55 smithi089 bash[78452]: cephadm 1685821282.7121873 mgr.y (mgr.34104) 4 : cephadm 1 [03/Jun/2023:19:41:22] ENGINE Bus STARTED 2023-06-03T19:44:55.659 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:44:55 smithi089 bash[78452]: cephadm 1685821287.9637723 mgr.y (mgr.34104) 11 : cephadm 1 Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:00b1b6d689278b4f97ff790045c8c51d8ca419ac7ee2ebdf11008f2e810a5c78 2023-06-03T19:44:55.659 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:44:55 smithi089 bash[78452]: cephadm 1685821299.4224992 mgr.y (mgr.34104) 17 : cephadm 1 Updating smithi089:/etc/ceph/ceph.conf 2023-06-03T19:44:55.659 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:44:55 smithi089 bash[78452]: cephadm 1685821299.4228904 mgr.y (mgr.34104) 18 : cephadm 1 Updating smithi164:/etc/ceph/ceph.conf 2023-06-03T19:44:55.659 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:44:55 smithi089 bash[78452]: cephadm 1685821299.587414 mgr.y (mgr.34104) 19 : cephadm 1 Updating smithi089:/var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/config/ceph.conf 2023-06-03T19:44:55.659 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:44:55 smithi089 bash[78452]: cephadm 1685821299.5885887 mgr.y (mgr.34104) 20 : cephadm 1 Updating smithi164:/var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/config/ceph.conf 2023-06-03T19:44:55.659 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:44:55 smithi089 bash[78452]: cephadm 1685821299.7529323 mgr.y (mgr.34104) 21 : cephadm 1 Updating smithi089:/etc/ceph/ceph.client.admin.keyring 2023-06-03T19:44:55.659 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:44:55 smithi089 bash[78452]: cephadm 1685821299.7543862 mgr.y (mgr.34104) 22 : cephadm 1 Updating smithi164:/etc/ceph/ceph.client.admin.keyring 2023-06-03T19:44:55.659 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:44:55 smithi089 bash[78452]: cephadm 1685821299.9186985 mgr.y (mgr.34104) 23 : cephadm 1 Updating smithi089:/var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/config/ceph.client.admin.keyring 2023-06-03T19:44:55.660 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:44:55 smithi089 bash[78452]: cephadm 1685821299.919883 mgr.y (mgr.34104) 24 : cephadm 1 Updating smithi164:/var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/config/ceph.client.admin.keyring 2023-06-03T19:44:55.660 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:44:55 smithi089 bash[78452]: cephadm 1685821360.1272502 mgr.y (mgr.34104) 57 : cephadm 1 Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:00b1b6d689278b4f97ff790045c8c51d8ca419ac7ee2ebdf11008f2e810a5c78 2023-06-03T19:44:55.660 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:44:55 smithi089 bash[78452]: cephadm 1685821367.5300949 mgr.y (mgr.34104) 61 : cephadm 1 Upgrade: It appears safe to stop mon.c 2023-06-03T19:44:55.660 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:44:55 smithi089 bash[78452]: cephadm 1685821372.2798438 mgr.y (mgr.34104) 65 : cephadm 1 Upgrade: Updating mon.c 2023-06-03T19:44:55.660 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:44:55 smithi089 bash[78452]: audit 1685821372.3102217 mon.c (mon.2) 122 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-03T19:44:55.660 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:44:55 smithi089 bash[78452]: cephadm 1685821372.311988 mgr.y (mgr.34104) 66 : cephadm 1 Deploying daemon mon.c on smithi089 2023-06-03T19:44:55.660 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:44:55 smithi089 bash[78452]: audit 1685821375.0654147 mon.b (mon.1) 92 : audit 0 from='client.? 172.21.15.89:0/1498143111' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-03T19:44:55.661 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:44:55 smithi089 bash[78452]: cluster 1685821385.9985988 mgr.y (mgr.34104) 73 : cluster 0 pgmap v60: 89 pgs: 89 active+clean; 6.2 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 45 MiB/s wr, 723 op/s 2023-06-03T19:44:55.661 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:44:55 smithi089 bash[78452]: audit 1685821386.594359 mon.a (mon.0) 82 : audit 0 from='client.? 172.21.15.89:0/3649894161' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-03T19:44:55.661 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:44:55 smithi089 bash[78452]: audit 1685821387.063322 mon.b (mon.1) 99 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T19:44:55.662 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:44:55 smithi089 bash[78452]: cluster 1685821387.9991255 mgr.y (mgr.34104) 74 : cluster 0 pgmap v61: 89 pgs: 89 active+clean; 6.3 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 52 MiB/s wr, 831 op/s 2023-06-03T19:44:55.662 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:44:55 smithi089 bash[78452]: cluster 1685821389.9997709 mgr.y (mgr.34104) 75 : cluster 0 pgmap v62: 89 pgs: 89 active+clean; 6.5 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 53 MiB/s wr, 855 op/s 2023-06-03T19:44:55.662 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:44:55 smithi089 bash[78452]: cluster 1685821392.000228 mgr.y (mgr.34104) 76 : cluster 0 pgmap v63: 89 pgs: 89 active+clean; 6.5 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 45 MiB/s wr, 722 op/s 2023-06-03T19:44:55.662 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:44:55 smithi089 bash[78452]: audit 1685821393.4656174 mgr.y (mgr.34104) 77 : audit 0 from='client.44102 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-06-03T19:44:55.663 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:44:55 smithi089 bash[78452]: audit 1685821393.6309147 mon.a (mon.0) 83 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:44:55.663 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:44:55 smithi089 bash[78452]: audit 1685821393.6497493 mon.a (mon.0) 84 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:44:55.663 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:44:55 smithi089 bash[78452]: cluster 1685821394.00106 mgr.y (mgr.34104) 78 : cluster 0 pgmap v64: 89 pgs: 89 active+clean; 6.7 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 66 MiB/s wr, 1.06k op/s 2023-06-03T19:44:55.663 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:44:55 smithi089 bash[78452]: audit 1685821394.7517736 mon.a (mon.0) 85 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:44:55.663 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:44:55 smithi089 bash[78452]: audit 1685821394.768501 mon.a (mon.0) 86 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:44:55.663 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:44:55 smithi089 bash[78452]: cluster 1685821396.0015018 mgr.y (mgr.34104) 79 : cluster 0 pgmap v65: 89 pgs: 89 active+clean; 6.7 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 43 MiB/s wr, 681 op/s 2023-06-03T19:44:55.663 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:44:55 smithi089 bash[78452]: cluster 1685821398.002088 mgr.y (mgr.34104) 80 : cluster 0 pgmap v66: 89 pgs: 89 active+clean; 6.7 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 45 MiB/s wr, 718 op/s 2023-06-03T19:44:55.663 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:44:55 smithi089 bash[78452]: audit 1685821399.3379703 mgr.y (mgr.34104) 81 : audit 0 from='client.34209 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2023-06-03T19:44:55.664 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:44:55 smithi089 bash[78452]: cephadm 1685821399.3385687 mgr.y (mgr.34104) 82 : cephadm 1 Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:00b1b6d689278b4f97ff790045c8c51d8ca419ac7ee2ebdf11008f2e810a5c78 2023-06-03T19:44:55.664 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:44:55 smithi089 bash[78452]: audit 1685821399.344722 mon.a (mon.0) 87 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:44:55.664 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:44:55 smithi089 bash[78452]: cluster 1685821400.0027843 mgr.y (mgr.34104) 83 : cluster 0 pgmap v67: 89 pgs: 89 active+clean; 7.0 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 56 MiB/s wr, 896 op/s 2023-06-03T19:44:55.664 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:44:55 smithi089 bash[78452]: cluster 1685821402.0032067 mgr.y (mgr.34104) 84 : cluster 0 pgmap v68: 89 pgs: 89 active+clean; 7.0 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 41 MiB/s wr, 657 op/s 2023-06-03T19:44:55.664 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:44:55 smithi089 bash[78452]: audit 1685821402.0612197 mon.b (mon.1) 100 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T19:44:55.664 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:44:55 smithi089 bash[78452]: cluster 1685821404.0069954 mgr.y (mgr.34104) 85 : cluster 0 pgmap v69: 89 pgs: 89 active+clean; 6.7 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 85 B/s rd, 42 MiB/s wr, 1.00k op/s 2023-06-03T19:44:55.664 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:44:55 smithi089 bash[78452]: cluster 1685821406.007473 mgr.y (mgr.34104) 86 : cluster 0 pgmap v70: 89 pgs: 89 active+clean; 6.7 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 85 B/s rd, 22 MiB/s wr, 671 op/s 2023-06-03T19:44:55.665 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:44:55 smithi089 bash[78452]: cluster 1685821408.0078633 mgr.y (mgr.34104) 87 : cluster 0 pgmap v71: 89 pgs: 89 active+clean; 6.7 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 85 B/s rd, 22 MiB/s wr, 671 op/s 2023-06-03T19:44:55.665 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:44:55 smithi089 bash[78452]: audit 1685821408.4267251 mon.a (mon.0) 88 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:44:55.665 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:44:55 smithi089 bash[78452]: audit 1685821408.4374998 mon.a (mon.0) 89 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:44:55.665 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:44:55 smithi089 bash[75068]: cluster 1685821486.040922 mgr.y (mgr.34104) 131 : cluster 0 pgmap v110: 89 pgs: 89 active+clean; 3.1 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 577 op/s 2023-06-03T19:44:55.665 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:44:55 smithi089 bash[75068]: cluster 1685821488.0413744 mgr.y (mgr.34104) 132 : cluster 0 pgmap v111: 89 pgs: 89 active+clean; 3.1 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 577 op/s 2023-06-03T19:44:55.666 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:44:55 smithi089 bash[75068]: cluster 1685821488.5762205 mon.c (mon.2) 8 : cluster 1 mon.c calling monitor election 2023-06-03T19:44:55.666 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:44:55 smithi089 bash[75068]: audit 1685821488.578151 mon.c (mon.2) 9 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-06-03T19:44:55.666 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:44:55 smithi089 bash[75068]: audit 1685821488.5789816 mon.c (mon.2) 10 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-06-03T19:44:55.666 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:44:55 smithi089 bash[75068]: audit 1685821488.5796552 mon.c (mon.2) 11 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-06-03T19:44:55.666 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:44:55 smithi089 bash[75068]: cluster 1685821490.0420983 mgr.y (mgr.34104) 133 : cluster 0 pgmap v112: 89 pgs: 89 active+clean; 2.9 GiB data, 9.8 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 868 op/s 2023-06-03T19:44:55.666 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:44:55 smithi089 bash[75068]: cluster 1685821492.0425348 mgr.y (mgr.34104) 134 : cluster 0 pgmap v113: 89 pgs: 89 active+clean; 2.9 GiB data, 9.8 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 591 op/s 2023-06-03T19:44:55.667 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:44:55 smithi089 bash[75068]: audit 1685821492.0652323 mon.c (mon.2) 12 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T19:44:55.667 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:44:55 smithi089 bash[75068]: cluster 1685821494.0432582 mgr.y (mgr.34104) 135 : cluster 0 pgmap v114: 89 pgs: 89 active+clean; 2.6 GiB data, 9.2 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 893 op/s 2023-06-03T19:44:55.667 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:44:55 smithi089 bash[75068]: cluster 1685821495.2954721 mon.b (mon.1) 3 : cluster 1 mon.b calling monitor election 2023-06-03T19:44:55.667 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:44:55 smithi089 bash[75068]: cluster 1685821495.2976289 mon.a (mon.0) 120 : cluster 1 mon.a calling monitor election 2023-06-03T19:44:55.667 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:44:55 smithi089 bash[75068]: cluster 1685821495.3215907 mon.a (mon.0) 121 : cluster 1 mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2023-06-03T19:44:55.667 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:44:55 smithi089 bash[75068]: cluster 1685821495.3283675 mon.a (mon.0) 122 : cluster 0 monmap e4: 3 mons at {a=[v2:172.21.15.89:3300/0,v1:172.21.15.89:6789/0],b=[v2:172.21.15.164:3300/0,v1:172.21.15.164:6789/0],c=[v2:172.21.15.89:3301/0,v1:172.21.15.89:6790/0]} removed_ranks: {} 2023-06-03T19:44:55.667 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:44:55 smithi089 bash[75068]: cluster 1685821495.3290384 mon.a (mon.0) 123 : cluster 0 fsmap foo:1 {0=foo.smithi089.giuxuj=up:active} 1 up:standby 2023-06-03T19:44:55.668 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:44:55 smithi089 bash[75068]: cluster 1685821495.3291001 mon.a (mon.0) 124 : cluster 0 osdmap e77: 8 total, 8 up, 8 in 2023-06-03T19:44:55.668 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:44:55 smithi089 bash[75068]: cluster 1685821495.331162 mon.a (mon.0) 125 : cluster 0 mgrmap e38: y(active, since 3m), standbys: x 2023-06-03T19:44:55.668 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:44:55 smithi089 bash[75068]: cluster 1685821495.3315442 mon.a (mon.0) 126 : cluster 1 Health check cleared: MON_DOWN (was: 1/3 mons down, quorum a,b) 2023-06-03T19:44:55.668 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:44:55 smithi089 bash[75068]: cluster 1685821495.3315997 mon.a (mon.0) 127 : cluster 1 Cluster is now healthy 2023-06-03T19:44:55.668 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:44:55 smithi089 bash[75068]: cluster 1685821495.3666618 mon.a (mon.0) 128 : cluster 1 overall HEALTH_OK 2023-06-03T19:44:55.668 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:44:55 smithi089 bash[75068]: audit 1685821495.3667607 mon.a (mon.0) 129 : audit 1 from='mgr.34104 ' entity='' 2023-06-03T19:44:55.668 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:44:55 smithi089 bash[75068]: audit 1685821495.383462 mon.a (mon.0) 130 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:44:55.669 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:44:55 smithi089 bash[75068]: audit 1685821495.4231832 mon.c (mon.2) 13 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-03T19:44:55.669 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:44:55 smithi089 bash[78452]: audit 1685821408.4380069 mon.b (mon.1) 101 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-03T19:44:55.669 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:44:55 smithi089 bash[78452]: audit 1685821408.4439259 mon.b (mon.1) 102 : audit 1 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-03T19:44:55.669 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:44:55 smithi089 bash[78452]: audit 1685821408.452624 mon.a (mon.0) 90 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:44:55.670 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:44:55 smithi089 bash[78452]: audit 1685821408.5381055 mon.b (mon.1) 103 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-03T19:44:55.670 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:44:55 smithi089 bash[78452]: audit 1685821408.5485957 mon.b (mon.1) 104 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-03T19:44:55.670 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:44:55 smithi089 bash[78452]: audit 1685821408.5498192 mon.b (mon.1) 105 : audit 1 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-03T19:44:55.670 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:44:55 smithi089 bash[78452]: audit 1685821408.5613625 mon.a (mon.0) 91 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:44:55.670 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:44:55 smithi089 bash[78452]: cluster 1685821410.0086102 mgr.y (mgr.34104) 88 : cluster 0 pgmap v72: 89 pgs: 89 active+clean; 6.5 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 85 B/s rd, 19 MiB/s wr, 969 op/s 2023-06-03T19:44:55.670 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:44:55 smithi089 bash[78452]: cluster 1685821412.0090332 mgr.y (mgr.34104) 89 : cluster 0 pgmap v73: 89 pgs: 89 active+clean; 6.5 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 85 B/s rd, 1.4 MiB/s wr, 684 op/s 2023-06-03T19:44:55.670 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:44:55 smithi089 bash[78452]: cluster 1685821414.0097935 mgr.y (mgr.34104) 90 : cluster 0 pgmap v74: 89 pgs: 89 active+clean; 6.2 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 85 B/s rd, 1.4 MiB/s wr, 1.02k op/s 2023-06-03T19:44:55.670 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:44:55 smithi089 bash[78452]: cluster 1685821416.0102537 mgr.y (mgr.34104) 91 : cluster 0 pgmap v75: 89 pgs: 89 active+clean; 6.2 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 673 op/s 2023-06-03T19:44:55.671 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:44:55 smithi089 bash[78452]: audit 1685821417.0620108 mon.b (mon.1) 106 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T19:44:55.671 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:44:55 smithi089 bash[78452]: cluster 1685821418.0107417 mgr.y (mgr.34104) 92 : cluster 0 pgmap v76: 89 pgs: 89 active+clean; 6.2 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 673 op/s 2023-06-03T19:44:55.671 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:44:55 smithi089 bash[78452]: cluster 1685821420.0114686 mgr.y (mgr.34104) 93 : cluster 0 pgmap v77: 89 pgs: 89 active+clean; 6.0 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 998 op/s 2023-06-03T19:44:55.671 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:44:55 smithi089 bash[78452]: cluster 1685821422.011843 mgr.y (mgr.34104) 94 : cluster 0 pgmap v78: 89 pgs: 89 active+clean; 6.0 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 662 op/s 2023-06-03T19:44:55.671 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:44:55 smithi089 bash[78452]: cluster 1685821424.013851 mgr.y (mgr.34104) 95 : cluster 0 pgmap v79: 89 pgs: 89 active+clean; 5.8 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 984 op/s 2023-06-03T19:44:55.671 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:44:55 smithi089 bash[78452]: cluster 1685821426.0143747 mgr.y (mgr.34104) 96 : cluster 0 pgmap v80: 89 pgs: 89 active+clean; 5.8 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 646 op/s 2023-06-03T19:44:55.671 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:44:55 smithi089 bash[78452]: cluster 1685821428.0147314 mgr.y (mgr.34104) 97 : cluster 0 pgmap v81: 89 pgs: 89 active+clean; 5.8 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 647 op/s 2023-06-03T19:44:55.672 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:44:55 smithi089 bash[78452]: cluster 1685821430.0154066 mgr.y (mgr.34104) 98 : cluster 0 pgmap v82: 89 pgs: 89 active+clean; 5.5 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 965 op/s 2023-06-03T19:44:55.672 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:44:55 smithi089 bash[78452]: cluster 1685821432.0165899 mgr.y (mgr.34104) 99 : cluster 0 pgmap v83: 89 pgs: 89 active+clean; 5.5 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 640 op/s 2023-06-03T19:44:55.672 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:44:55 smithi089 bash[78452]: audit 1685821432.0621827 mon.b (mon.1) 107 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T19:44:55.672 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:44:55 smithi089 bash[78452]: cluster 1685821434.0173318 mgr.y (mgr.34104) 100 : cluster 0 pgmap v84: 89 pgs: 89 active+clean; 5.3 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 961 op/s 2023-06-03T19:44:55.672 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:44:55 smithi089 bash[78452]: cluster 1685821436.0177891 mgr.y (mgr.34104) 101 : cluster 0 pgmap v85: 89 pgs: 89 active+clean; 5.3 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 639 op/s 2023-06-03T19:44:55.672 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:44:55 smithi089 bash[78452]: cluster 1685821438.0200334 mgr.y (mgr.34104) 102 : cluster 0 pgmap v86: 89 pgs: 89 active+clean; 5.3 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 639 op/s 2023-06-03T19:44:55.672 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:44:55 smithi089 bash[78452]: cluster 1685821440.020801 mgr.y (mgr.34104) 103 : cluster 0 pgmap v87: 89 pgs: 89 active+clean; 5.1 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 951 op/s 2023-06-03T19:44:55.672 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:44:55 smithi089 bash[78452]: cluster 1685821442.0212224 mgr.y (mgr.34104) 104 : cluster 0 pgmap v88: 89 pgs: 89 active+clean; 5.1 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 632 op/s 2023-06-03T19:44:55.673 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:44:55 smithi089 bash[78452]: cluster 1685821444.0253873 mgr.y (mgr.34104) 105 : cluster 0 pgmap v89: 89 pgs: 89 active+clean; 4.8 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 942 op/s 2023-06-03T19:44:55.673 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:44:55 smithi089 bash[78452]: cluster 1685821446.0258205 mgr.y (mgr.34104) 106 : cluster 0 pgmap v90: 89 pgs: 89 active+clean; 4.8 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 621 op/s 2023-06-03T19:44:55.673 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:44:55 smithi089 bash[78452]: audit 1685821447.0623534 mon.b (mon.1) 108 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T19:44:55.673 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:44:55 smithi089 bash[78452]: cluster 1685821448.0262258 mgr.y (mgr.34104) 107 : cluster 0 pgmap v91: 89 pgs: 89 active+clean; 4.8 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 621 op/s 2023-06-03T19:44:55.673 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:44:55 smithi089 bash[78452]: cluster 1685821450.026844 mgr.y (mgr.34104) 108 : cluster 0 pgmap v92: 89 pgs: 89 active+clean; 4.6 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 932 op/s 2023-06-03T19:44:55.673 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:44:55 smithi089 bash[78452]: cluster 1685821452.0272791 mgr.y (mgr.34104) 109 : cluster 0 pgmap v93: 89 pgs: 89 active+clean; 4.6 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 620 op/s 2023-06-03T19:44:55.673 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:44:55 smithi089 bash[78452]: cluster 1685821454.0280442 mgr.y (mgr.34104) 110 : cluster 0 pgmap v94: 89 pgs: 89 active+clean; 4.4 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 925 op/s 2023-06-03T19:44:55.674 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:44:55 smithi089 bash[78452]: cluster 1685821456.0288029 mgr.y (mgr.34104) 111 : cluster 0 pgmap v95: 89 pgs: 89 active+clean; 4.4 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 615 op/s 2023-06-03T19:44:55.674 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:44:55 smithi089 bash[78452]: cluster 1685821458.0292284 mgr.y (mgr.34104) 112 : cluster 0 pgmap v96: 89 pgs: 89 active+clean; 4.4 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 615 op/s 2023-06-03T19:44:55.674 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:44:55 smithi089 bash[78452]: cluster 1685821460.0301201 mgr.y (mgr.34104) 113 : cluster 0 pgmap v97: 89 pgs: 89 active+clean; 4.2 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 920 op/s 2023-06-03T19:44:55.674 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:44:55 smithi089 bash[78452]: cluster 1685821462.0314925 mgr.y (mgr.34104) 114 : cluster 0 pgmap v98: 89 pgs: 89 active+clean; 4.2 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 609 op/s 2023-06-03T19:44:55.674 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:44:55 smithi089 bash[78452]: audit 1685821462.0626981 mon.b (mon.1) 109 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T19:44:55.674 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:44:55 smithi089 bash[78452]: cluster 1685821464.0322375 mgr.y (mgr.34104) 115 : cluster 0 pgmap v99: 89 pgs: 89 active+clean; 3.9 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 917 op/s 2023-06-03T19:44:55.674 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:44:55 smithi089 bash[78452]: cluster 1685821466.032663 mgr.y (mgr.34104) 116 : cluster 0 pgmap v100: 89 pgs: 89 active+clean; 3.9 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 612 op/s 2023-06-03T19:44:55.674 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:44:55 smithi089 bash[78452]: cluster 1685821468.0330772 mgr.y (mgr.34104) 117 : cluster 0 pgmap v101: 89 pgs: 89 active+clean; 3.9 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 612 op/s 2023-06-03T19:44:55.675 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:44:55 smithi089 bash[78452]: audit 1685821468.6499329 mon.b (mon.1) 110 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-03T19:44:55.675 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:44:55 smithi089 bash[78452]: audit 1685821469.128474 mon.a (mon.0) 92 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:44:55.675 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:44:55 smithi089 bash[78452]: audit 1685821469.150722 mon.a (mon.0) 93 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:44:55.675 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:44:55 smithi089 bash[78452]: cluster 1685821470.0338662 mgr.y (mgr.34104) 118 : cluster 0 pgmap v102: 89 pgs: 89 active+clean; 3.7 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 911 op/s 2023-06-03T19:44:55.675 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:44:55 smithi089 bash[78452]: cluster 1685821472.0343218 mgr.y (mgr.34104) 119 : cluster 0 pgmap v103: 89 pgs: 89 active+clean; 3.7 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 607 op/s 2023-06-03T19:44:55.675 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:44:55 smithi089 bash[78452]: audit 1685821472.543854 mgr.y (mgr.34104) 120 : audit 0 from='client.34215 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2023-06-03T19:44:55.675 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:44:55 smithi089 bash[78452]: cephadm 1685821472.5443 mgr.y (mgr.34104) 121 : cephadm 1 Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:00b1b6d689278b4f97ff790045c8c51d8ca419ac7ee2ebdf11008f2e810a5c78 2023-06-03T19:44:55.676 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:44:55 smithi089 bash[78452]: audit 1685821472.5560281 mon.a (mon.0) 94 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:44:55.676 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:44:55 smithi089 bash[78452]: cluster 1685821474.035018 mgr.y (mgr.34104) 122 : cluster 0 pgmap v104: 89 pgs: 89 active+clean; 3.5 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 906 op/s 2023-06-03T19:44:55.676 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:44:55 smithi089 bash[78452]: audit 1685821475.3681252 mon.a (mon.0) 95 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:44:55.676 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:44:55 smithi089 bash[78452]: audit 1685821475.3785973 mon.a (mon.0) 96 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:44:55.676 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:44:55 smithi089 bash[78452]: audit 1685821475.6649632 mon.a (mon.0) 97 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:44:55.676 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:44:55 smithi089 bash[78452]: audit 1685821475.6763065 mon.a (mon.0) 98 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:44:55.676 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:44:55 smithi089 bash[78452]: cluster 1685821476.03544 mgr.y (mgr.34104) 123 : cluster 0 pgmap v105: 89 pgs: 89 active+clean; 3.5 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 598 op/s 2023-06-03T19:44:55.676 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:44:55 smithi089 bash[78452]: audit 1685821476.2671282 mon.b (mon.1) 111 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-03T19:44:55.677 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:44:55 smithi089 bash[78452]: audit 1685821476.26879 mon.b (mon.1) 112 : audit 1 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-03T19:44:55.677 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:44:55 smithi089 bash[78452]: audit 1685821476.2804465 mon.a (mon.0) 99 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:44:55.677 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:44:55 smithi089 bash[78452]: audit 1685821476.3509307 mon.b (mon.1) 113 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-03T19:44:55.677 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:44:55 smithi089 bash[78452]: audit 1685821476.3565543 mon.b (mon.1) 114 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-06-03T19:44:55.677 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:44:55 smithi089 bash[78452]: audit 1685821476.3589268 mon.b (mon.1) 115 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2023-06-03T19:44:55.677 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:44:55 smithi089 bash[78452]: audit 1685821476.3602748 mon.b (mon.1) 116 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["b"]}]: dispatch 2023-06-03T19:44:55.677 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:44:55 smithi089 bash[78452]: cephadm 1685821476.3632317 mgr.y (mgr.34104) 124 : cephadm 1 Upgrade: It appears safe to stop mon.b 2023-06-03T19:44:55.678 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:44:55 smithi089 bash[78452]: audit 1685821477.0630448 mon.b (mon.1) 117 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T19:44:55.678 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:44:55 smithi089 bash[78452]: cluster 1685821478.0358849 mgr.y (mgr.34104) 125 : cluster 0 pgmap v106: 89 pgs: 89 active+clean; 3.5 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 598 op/s 2023-06-03T19:44:55.678 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:44:55 smithi089 bash[78452]: cluster 1685821480.0366492 mgr.y (mgr.34104) 126 : cluster 0 pgmap v107: 89 pgs: 89 active+clean; 3.3 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 875 op/s 2023-06-03T19:44:55.678 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:44:55 smithi089 bash[78452]: cephadm 1685821481.4643776 mgr.y (mgr.34104) 127 : cephadm 1 Upgrade: Updating mon.b 2023-06-03T19:44:55.678 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:44:55 smithi089 bash[78452]: audit 1685821481.4876273 mon.b (mon.1) 118 : audit 1 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-06-03T19:44:55.678 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:44:55 smithi089 bash[78452]: audit 1685821481.488088 mon.a (mon.0) 100 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:44:55.679 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:44:55 smithi089 bash[78452]: audit 1685821481.4901097 mon.b (mon.1) 119 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-03T19:44:55.679 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:44:55 smithi089 bash[78452]: audit 1685821481.490938 mon.a (mon.0) 101 : audit 0 from='mgr.34104 ' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-06-03T19:44:55.679 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:44:55 smithi089 bash[78452]: cephadm 1685821481.4933066 mgr.y (mgr.34104) 128 : cephadm 1 Deploying daemon mon.b on smithi164 2023-06-03T19:44:55.679 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:44:55 smithi089 bash[78452]: cluster 1685821482.0396972 mgr.y (mgr.34104) 129 : cluster 0 pgmap v108: 89 pgs: 89 active+clean; 3.3 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 575 op/s 2023-06-03T19:44:55.679 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:44:55 smithi089 bash[78452]: audit 1685821482.9795392 mon.c (mon.2) 2 : audit 0 from='client.? 172.21.15.89:0/4084515067' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-03T19:44:55.679 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:44:55 smithi089 bash[78452]: cluster 1685821484.0404198 mgr.y (mgr.34104) 130 : cluster 0 pgmap v109: 89 pgs: 89 active+clean; 3.1 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 876 op/s 2023-06-03T19:44:55.680 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:44:55 smithi089 bash[78452]: audit 1685821484.7535958 mon.c (mon.2) 3 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-06-03T19:44:55.680 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:44:55 smithi089 bash[78452]: audit 1685821484.7539053 mon.c (mon.2) 4 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-06-03T19:44:55.680 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:44:55 smithi089 bash[78452]: audit 1685821484.7541707 mon.c (mon.2) 5 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-06-03T19:44:55.680 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:44:55 smithi089 bash[78452]: cluster 1685821488.5692677 mon.a (mon.0) 108 : cluster 1 overall HEALTH_OK 2023-06-03T19:44:55.680 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:44:55 smithi089 bash[78452]: cluster 1685821488.5708764 mon.b (mon.1) 2 : cluster 1 mon.b calling monitor election 2023-06-03T19:44:55.680 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:44:55 smithi089 bash[78452]: cluster 1685821488.5728936 mon.a (mon.0) 109 : cluster 1 mon.a calling monitor election 2023-06-03T19:44:55.681 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:44:55 smithi089 bash[78452]: cluster 1685821493.5741582 mon.a (mon.0) 110 : cluster 1 mon.a is new leader, mons a,b in quorum (ranks 0,1) 2023-06-03T19:44:55.681 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:44:55 smithi089 bash[78452]: cluster 1685821493.5829618 mon.a (mon.0) 111 : cluster 0 monmap e4: 3 mons at {a=[v2:172.21.15.89:3300/0,v1:172.21.15.89:6789/0],b=[v2:172.21.15.164:3300/0,v1:172.21.15.164:6789/0],c=[v2:172.21.15.89:3301/0,v1:172.21.15.89:6790/0]} removed_ranks: {} 2023-06-03T19:44:55.681 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:44:55 smithi089 bash[78452]: cluster 1685821493.5836842 mon.a (mon.0) 112 : cluster 0 fsmap foo:1 {0=foo.smithi089.giuxuj=up:active} 1 up:standby 2023-06-03T19:44:55.681 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:44:55 smithi089 bash[78452]: cluster 1685821493.5837233 mon.a (mon.0) 113 : cluster 0 osdmap e77: 8 total, 8 up, 8 in 2023-06-03T19:44:55.681 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:44:55 smithi089 bash[78452]: cluster 1685821493.5842338 mon.a (mon.0) 114 : cluster 0 mgrmap e38: y(active, since 3m), standbys: x 2023-06-03T19:44:55.681 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:44:55 smithi089 bash[78452]: cluster 1685821493.5844698 mon.a (mon.0) 115 : cluster 3 Health check failed: 1/3 mons down, quorum a,b (MON_DOWN) 2023-06-03T19:44:55.681 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:44:55 smithi089 bash[78452]: cluster 1685821493.6041977 mon.a (mon.0) 116 : cluster 3 Health detail: HEALTH_WARN 1/3 mons down, quorum a,b 2023-06-03T19:44:55.681 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:44:55 smithi089 bash[78452]: cluster 1685821493.6043186 mon.a (mon.0) 117 : cluster 3 [WRN] MON_DOWN: 1/3 mons down, quorum a,b 2023-06-03T19:44:55.682 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:44:55 smithi089 bash[78452]: cluster 1685821493.6043558 mon.a (mon.0) 118 : cluster 3 mon.c (rank 2) addr [v2:172.21.15.89:3301/0,v1:172.21.15.89:6790/0] is down (out of quorum) 2023-06-03T19:44:55.682 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:44:55 smithi089 bash[78452]: audit 1685821494.1984737 mon.a (mon.0) 119 : audit 0 from='client.? 172.21.15.89:0/3652051568' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-06-03T19:44:55.682 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:44:55 smithi089 bash[78452]: cluster 1685821486.040922 mgr.y (mgr.34104) 131 : cluster 0 pgmap v110: 89 pgs: 89 active+clean; 3.1 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 577 op/s 2023-06-03T19:44:55.682 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:44:55 smithi089 bash[78452]: cluster 1685821488.0413744 mgr.y (mgr.34104) 132 : cluster 0 pgmap v111: 89 pgs: 89 active+clean; 3.1 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 577 op/s 2023-06-03T19:44:55.682 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:44:55 smithi089 bash[78452]: cluster 1685821488.5762205 mon.c (mon.2) 8 : cluster 1 mon.c calling monitor election 2023-06-03T19:44:55.682 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:44:55 smithi089 bash[78452]: audit 1685821488.578151 mon.c (mon.2) 9 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-06-03T19:44:55.683 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:44:55 smithi089 bash[78452]: audit 1685821488.5789816 mon.c (mon.2) 10 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-06-03T19:44:55.683 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:44:55 smithi089 bash[78452]: audit 1685821488.5796552 mon.c (mon.2) 11 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-06-03T19:44:56.030 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:44:55 smithi089 bash[78452]: cluster 1685821490.0420983 mgr.y (mgr.34104) 133 : cluster 0 pgmap v112: 89 pgs: 89 active+clean; 2.9 GiB data, 9.8 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 868 op/s 2023-06-03T19:44:56.031 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:44:55 smithi089 bash[78452]: cluster 1685821492.0425348 mgr.y (mgr.34104) 134 : cluster 0 pgmap v113: 89 pgs: 89 active+clean; 2.9 GiB data, 9.8 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 591 op/s 2023-06-03T19:44:56.031 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:44:55 smithi089 bash[78452]: audit 1685821492.0652323 mon.c (mon.2) 12 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T19:44:56.031 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:44:55 smithi089 bash[78452]: cluster 1685821494.0432582 mgr.y (mgr.34104) 135 : cluster 0 pgmap v114: 89 pgs: 89 active+clean; 2.6 GiB data, 9.2 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 893 op/s 2023-06-03T19:44:56.031 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:44:55 smithi089 bash[78452]: cluster 1685821495.2954721 mon.b (mon.1) 3 : cluster 1 mon.b calling monitor election 2023-06-03T19:44:56.031 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:44:55 smithi089 bash[78452]: cluster 1685821495.2976289 mon.a (mon.0) 120 : cluster 1 mon.a calling monitor election 2023-06-03T19:44:56.031 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:44:55 smithi089 bash[78452]: cluster 1685821495.3215907 mon.a (mon.0) 121 : cluster 1 mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2023-06-03T19:44:56.032 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:44:55 smithi089 bash[78452]: cluster 1685821495.3283675 mon.a (mon.0) 122 : cluster 0 monmap e4: 3 mons at {a=[v2:172.21.15.89:3300/0,v1:172.21.15.89:6789/0],b=[v2:172.21.15.164:3300/0,v1:172.21.15.164:6789/0],c=[v2:172.21.15.89:3301/0,v1:172.21.15.89:6790/0]} removed_ranks: {} 2023-06-03T19:44:56.032 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:44:55 smithi089 bash[78452]: cluster 1685821495.3290384 mon.a (mon.0) 123 : cluster 0 fsmap foo:1 {0=foo.smithi089.giuxuj=up:active} 1 up:standby 2023-06-03T19:44:56.032 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:44:55 smithi089 bash[78452]: cluster 1685821495.3291001 mon.a (mon.0) 124 : cluster 0 osdmap e77: 8 total, 8 up, 8 in 2023-06-03T19:44:56.032 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:44:55 smithi089 bash[78452]: cluster 1685821495.331162 mon.a (mon.0) 125 : cluster 0 mgrmap e38: y(active, since 3m), standbys: x 2023-06-03T19:44:56.032 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:44:55 smithi089 bash[78452]: cluster 1685821495.3315442 mon.a (mon.0) 126 : cluster 1 Health check cleared: MON_DOWN (was: 1/3 mons down, quorum a,b) 2023-06-03T19:44:56.032 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:44:55 smithi089 bash[78452]: cluster 1685821495.3315997 mon.a (mon.0) 127 : cluster 1 Cluster is now healthy 2023-06-03T19:44:56.033 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:44:55 smithi089 bash[78452]: cluster 1685821495.3666618 mon.a (mon.0) 128 : cluster 1 overall HEALTH_OK 2023-06-03T19:44:56.033 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:44:55 smithi089 bash[78452]: audit 1685821495.3667607 mon.a (mon.0) 129 : audit 1 from='mgr.34104 ' entity='' 2023-06-03T19:44:56.033 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:44:55 smithi089 bash[78452]: audit 1685821495.383462 mon.a (mon.0) 130 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:44:56.033 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:44:55 smithi089 bash[78452]: audit 1685821495.4231832 mon.c (mon.2) 13 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-03T19:44:56.085 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:55 smithi164 bash[59462]: cluster 1685821486.040922 mgr.y (mgr.34104) 131 : cluster 0 pgmap v110: 89 pgs: 89 active+clean; 3.1 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 577 op/s 2023-06-03T19:44:56.086 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:55 smithi164 bash[59462]: cluster 1685821488.0413744 mgr.y (mgr.34104) 132 : cluster 0 pgmap v111: 89 pgs: 89 active+clean; 3.1 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 577 op/s 2023-06-03T19:44:56.086 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:55 smithi164 bash[59462]: cluster 1685821488.5762205 mon.c (mon.2) 8 : cluster 1 mon.c calling monitor election 2023-06-03T19:44:56.086 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:55 smithi164 bash[59462]: audit 1685821488.578151 mon.c (mon.2) 9 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-06-03T19:44:56.087 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:55 smithi164 bash[59462]: audit 1685821488.5789816 mon.c (mon.2) 10 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-06-03T19:44:56.087 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:55 smithi164 bash[59462]: audit 1685821488.5796552 mon.c (mon.2) 11 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-06-03T19:44:56.087 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:55 smithi164 bash[59462]: cluster 1685821490.0420983 mgr.y (mgr.34104) 133 : cluster 0 pgmap v112: 89 pgs: 89 active+clean; 2.9 GiB data, 9.8 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 868 op/s 2023-06-03T19:44:56.087 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:55 smithi164 bash[59462]: cluster 1685821492.0425348 mgr.y (mgr.34104) 134 : cluster 0 pgmap v113: 89 pgs: 89 active+clean; 2.9 GiB data, 9.8 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 591 op/s 2023-06-03T19:44:56.087 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:55 smithi164 bash[59462]: audit 1685821492.0652323 mon.c (mon.2) 12 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T19:44:56.087 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:55 smithi164 bash[59462]: cluster 1685821494.0432582 mgr.y (mgr.34104) 135 : cluster 0 pgmap v114: 89 pgs: 89 active+clean; 2.6 GiB data, 9.2 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 893 op/s 2023-06-03T19:44:56.087 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:55 smithi164 bash[59462]: cluster 1685821495.2954721 mon.b (mon.1) 3 : cluster 1 mon.b calling monitor election 2023-06-03T19:44:56.088 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:55 smithi164 bash[59462]: cluster 1685821495.2976289 mon.a (mon.0) 120 : cluster 1 mon.a calling monitor election 2023-06-03T19:44:56.088 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:55 smithi164 bash[59462]: cluster 1685821495.3215907 mon.a (mon.0) 121 : cluster 1 mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2023-06-03T19:44:56.088 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:55 smithi164 bash[59462]: cluster 1685821495.3283675 mon.a (mon.0) 122 : cluster 0 monmap e4: 3 mons at {a=[v2:172.21.15.89:3300/0,v1:172.21.15.89:6789/0],b=[v2:172.21.15.164:3300/0,v1:172.21.15.164:6789/0],c=[v2:172.21.15.89:3301/0,v1:172.21.15.89:6790/0]} removed_ranks: {} 2023-06-03T19:44:56.088 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:55 smithi164 bash[59462]: cluster 1685821495.3290384 mon.a (mon.0) 123 : cluster 0 fsmap foo:1 {0=foo.smithi089.giuxuj=up:active} 1 up:standby 2023-06-03T19:44:56.088 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:55 smithi164 bash[59462]: cluster 1685821495.3291001 mon.a (mon.0) 124 : cluster 0 osdmap e77: 8 total, 8 up, 8 in 2023-06-03T19:44:56.088 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:55 smithi164 bash[59462]: cluster 1685821495.331162 mon.a (mon.0) 125 : cluster 0 mgrmap e38: y(active, since 3m), standbys: x 2023-06-03T19:44:56.088 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:55 smithi164 bash[59462]: cluster 1685821495.3315442 mon.a (mon.0) 126 : cluster 1 Health check cleared: MON_DOWN (was: 1/3 mons down, quorum a,b) 2023-06-03T19:44:56.089 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:55 smithi164 bash[59462]: cluster 1685821495.3315997 mon.a (mon.0) 127 : cluster 1 Cluster is now healthy 2023-06-03T19:44:56.089 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:55 smithi164 bash[59462]: cluster 1685821495.3666618 mon.a (mon.0) 128 : cluster 1 overall HEALTH_OK 2023-06-03T19:44:56.089 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:55 smithi164 bash[59462]: audit 1685821495.3667607 mon.a (mon.0) 129 : audit 1 from='mgr.34104 ' entity='' 2023-06-03T19:44:56.089 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:55 smithi164 bash[59462]: audit 1685821495.383462 mon.a (mon.0) 130 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:44:56.089 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:55 smithi164 bash[59462]: audit 1685821495.4231832 mon.c (mon.2) 13 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-03T19:44:56.530 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 19:44:56 smithi089 bash[71163]: ::ffff:172.21.15.164 - - [03/Jun/2023:19:44:56] "GET /metrics HTTP/1.1" 200 184103 "" "Prometheus/2.33.4" 2023-06-03T19:44:56.553 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -e sha1=8e24bcf462a1d94c2e12aa4e26419634883a1e88 -- bash -c 'ceph orch ps' 2023-06-03T19:44:57.280 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:44:56 smithi089 bash[75068]: audit 1685821495.854694 mon.a (mon.0) 131 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:44:57.281 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:44:56 smithi089 bash[75068]: audit 1685821495.865627 mon.a (mon.0) 132 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:44:57.281 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:44:56 smithi089 bash[75068]: cluster 1685821496.0459461 mgr.y (mgr.34104) 136 : cluster 0 pgmap v115: 89 pgs: 89 active+clean; 2.6 GiB data, 9.2 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 592 op/s 2023-06-03T19:44:57.281 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:44:56 smithi089 bash[78452]: audit 1685821495.854694 mon.a (mon.0) 131 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:44:57.281 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:44:56 smithi089 bash[78452]: audit 1685821495.865627 mon.a (mon.0) 132 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:44:57.282 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:44:56 smithi089 bash[78452]: cluster 1685821496.0459461 mgr.y (mgr.34104) 136 : cluster 0 pgmap v115: 89 pgs: 89 active+clean; 2.6 GiB data, 9.2 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 592 op/s 2023-06-03T19:44:57.335 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:56 smithi164 bash[59462]: audit 1685821495.854694 mon.a (mon.0) 131 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:44:57.335 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:56 smithi164 bash[59462]: audit 1685821495.865627 mon.a (mon.0) 132 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:44:57.335 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:56 smithi164 bash[59462]: cluster 1685821496.0459461 mgr.y (mgr.34104) 136 : cluster 0 pgmap v115: 89 pgs: 89 active+clean; 2.6 GiB data, 9.2 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 592 op/s 2023-06-03T19:44:58.884 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:44:58 smithi089 bash[75068]: cluster 1685821498.047754 mgr.y (mgr.34104) 137 : cluster 0 pgmap v116: 89 pgs: 89 active+clean; 2.6 GiB data, 9.2 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 592 op/s 2023-06-03T19:44:59.280 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:44:58 smithi089 bash[78452]: cluster 1685821498.047754 mgr.y (mgr.34104) 137 : cluster 0 pgmap v116: 89 pgs: 89 active+clean; 2.6 GiB data, 9.2 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 592 op/s 2023-06-03T19:44:59.335 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:44:58 smithi164 bash[59462]: cluster 1685821498.047754 mgr.y (mgr.34104) 137 : cluster 0 pgmap v116: 89 pgs: 89 active+clean; 2.6 GiB data, 9.2 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 592 op/s 2023-06-03T19:45:01.371 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:45:01 smithi164 bash[59462]: cluster 1685821500.0484462 mgr.y (mgr.34104) 138 : cluster 0 pgmap v117: 89 pgs: 89 active+clean; 2.4 GiB data, 8.5 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 887 op/s 2023-06-03T19:45:01.403 INFO:teuthology.orchestra.run.smithi089.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2023-06-03T19:45:01.404 INFO:teuthology.orchestra.run.smithi089.stdout:alertmanager.a smithi089 *:9093,9094 running (6m) 25s ago 12m 14.4M - 0.23.0 ba2b418f427c 0fad3b643b2e 2023-06-03T19:45:01.404 INFO:teuthology.orchestra.run.smithi089.stdout:grafana.a smithi164 *:3000 running (5m) 26s ago 11m 40.5M - 8.3.5 dad864ee21e9 64925a21f38b 2023-06-03T19:45:01.404 INFO:teuthology.orchestra.run.smithi089.stdout:mds.foo.smithi089.giuxuj smithi089 running (10m) 25s ago 10m 16.9M - 16.2.13-257-gd8c5d349 b644dd95899d f0e9de24a3f8 2023-06-03T19:45:01.404 INFO:teuthology.orchestra.run.smithi089.stdout:mds.foo.smithi164.txgfgv smithi164 running (10m) 26s ago 10m 14.0M - 16.2.13-257-gd8c5d349 b644dd95899d 29da307aa3b4 2023-06-03T19:45:01.404 INFO:teuthology.orchestra.run.smithi089.stdout:mgr.x smithi164 *:8443,9283,8765 running (4m) 26s ago 19m 425M - 18.0.0-3532-g8e24bcf4 a57053f96fe3 3ada341a2c72 2023-06-03T19:45:01.404 INFO:teuthology.orchestra.run.smithi089.stdout:mgr.y smithi089 *:8443,9283,8765 running (5m) 25s ago 21m 475M - 18.0.0-3532-g8e24bcf4 a57053f96fe3 8307fbe12a7d 2023-06-03T19:45:01.405 INFO:teuthology.orchestra.run.smithi089.stdout:mon.a smithi089 running (3m) 25s ago 22m 53.9M 2048M 18.0.0-3532-g8e24bcf4 a57053f96fe3 c5e2e407c45f 2023-06-03T19:45:01.405 INFO:teuthology.orchestra.run.smithi089.stdout:mon.b smithi164 starting - - - 2048M 2023-06-03T19:45:01.405 INFO:teuthology.orchestra.run.smithi089.stdout:mon.c smithi089 running (2m) 25s ago 20m 38.5M 2048M 18.0.0-3532-g8e24bcf4 a57053f96fe3 f10053d47b03 2023-06-03T19:45:01.405 INFO:teuthology.orchestra.run.smithi089.stdout:node-exporter.a smithi089 *:9100 running (6m) 25s ago 12m 9.76M - 1.3.1 1dbe0e931976 f41a3d45da20 2023-06-03T19:45:01.405 INFO:teuthology.orchestra.run.smithi089.stdout:node-exporter.b smithi164 *:9100 running (6m) 26s ago 12m 12.0M - 1.3.1 1dbe0e931976 1e69859880c6 2023-06-03T19:45:01.405 INFO:teuthology.orchestra.run.smithi089.stdout:osd.0 smithi089 running (19m) 25s ago 19m 411M 2526M 16.2.13-257-gd8c5d349 b644dd95899d cb8bdf1749e4 2023-06-03T19:45:01.405 INFO:teuthology.orchestra.run.smithi089.stdout:osd.1 smithi089 running (18m) 25s ago 18m 350M 2526M 16.2.13-257-gd8c5d349 b644dd95899d d9e8a03d4689 2023-06-03T19:45:01.405 INFO:teuthology.orchestra.run.smithi089.stdout:osd.2 smithi089 running (17m) 25s ago 17m 511M 2526M 16.2.13-257-gd8c5d349 b644dd95899d f7fc93b466ec 2023-06-03T19:45:01.406 INFO:teuthology.orchestra.run.smithi089.stdout:osd.3 smithi089 running (16m) 25s ago 16m 500M 2526M 16.2.13-257-gd8c5d349 b644dd95899d 7e98b61121c6 2023-06-03T19:45:01.406 INFO:teuthology.orchestra.run.smithi089.stdout:osd.4 smithi164 running (16m) 26s ago 16m 439M 2526M 16.2.13-257-gd8c5d349 b644dd95899d 5c901af1a184 2023-06-03T19:45:01.406 INFO:teuthology.orchestra.run.smithi089.stdout:osd.5 smithi164 running (15m) 26s ago 15m 470M 2526M 16.2.13-257-gd8c5d349 b644dd95899d 13ddc40f04c4 2023-06-03T19:45:01.406 INFO:teuthology.orchestra.run.smithi089.stdout:osd.6 smithi164 running (14m) 26s ago 14m 310M 2526M 16.2.13-257-gd8c5d349 b644dd95899d cab2cb9998d7 2023-06-03T19:45:01.406 INFO:teuthology.orchestra.run.smithi089.stdout:osd.7 smithi164 running (13m) 26s ago 13m 417M 2526M 16.2.13-257-gd8c5d349 b644dd95899d b5fd865771b6 2023-06-03T19:45:01.406 INFO:teuthology.orchestra.run.smithi089.stdout:prometheus.a smithi164 *:9095 running (4m) 26s ago 12m 70.6M - 2.33.4 514e6a882f6e 3ff744731c36 2023-06-03T19:45:01.439 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:45:01 smithi089 bash[78452]: cluster 1685821500.0484462 mgr.y (mgr.34104) 138 : cluster 0 pgmap v117: 89 pgs: 89 active+clean; 2.4 GiB data, 8.5 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 887 op/s 2023-06-03T19:45:01.439 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:45:01 smithi089 bash[75068]: cluster 1685821500.0484462 mgr.y (mgr.34104) 138 : cluster 0 pgmap v117: 89 pgs: 89 active+clean; 2.4 GiB data, 8.5 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 887 op/s 2023-06-03T19:45:02.530 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:45:02 smithi089 bash[78452]: audit 1685821501.3791423 mgr.y (mgr.34104) 139 : audit 0 from='client.54103 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-06-03T19:45:02.530 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:45:02 smithi089 bash[75068]: audit 1685821501.3791423 mgr.y (mgr.34104) 139 : audit 0 from='client.54103 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-06-03T19:45:02.558 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -e sha1=8e24bcf462a1d94c2e12aa4e26419634883a1e88 -- bash -c 'ceph orch upgrade pause' 2023-06-03T19:45:02.585 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:45:02 smithi164 bash[59462]: audit 1685821501.3791423 mgr.y (mgr.34104) 139 : audit 0 from='client.54103 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-06-03T19:45:03.780 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:45:03 smithi089 bash[75068]: cluster 1685821502.0515554 mgr.y (mgr.34104) 140 : cluster 0 pgmap v118: 89 pgs: 89 active+clean; 2.4 GiB data, 8.5 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 596 op/s 2023-06-03T19:45:03.780 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:45:03 smithi089 bash[75068]: audit 1685821502.5479515 mon.a (mon.0) 133 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:45:03.781 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:45:03 smithi089 bash[75068]: audit 1685821502.5640914 mon.a (mon.0) 134 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:45:03.781 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:45:03 smithi089 bash[78452]: cluster 1685821502.0515554 mgr.y (mgr.34104) 140 : cluster 0 pgmap v118: 89 pgs: 89 active+clean; 2.4 GiB data, 8.5 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 596 op/s 2023-06-03T19:45:03.781 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:45:03 smithi089 bash[78452]: audit 1685821502.5479515 mon.a (mon.0) 133 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:45:03.781 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:45:03 smithi089 bash[78452]: audit 1685821502.5640914 mon.a (mon.0) 134 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:45:03.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:45:03 smithi164 bash[59462]: cluster 1685821502.0515554 mgr.y (mgr.34104) 140 : cluster 0 pgmap v118: 89 pgs: 89 active+clean; 2.4 GiB data, 8.5 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 596 op/s 2023-06-03T19:45:03.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:45:03 smithi164 bash[59462]: audit 1685821502.5479515 mon.a (mon.0) 133 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:45:03.836 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:45:03 smithi164 bash[59462]: audit 1685821502.5640914 mon.a (mon.0) 134 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:45:04.723 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:45:04 smithi089 bash[75068]: audit 1685821503.5796103 mon.a (mon.0) 135 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:45:04.723 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:45:04 smithi089 bash[75068]: audit 1685821503.593357 mon.a (mon.0) 136 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:45:04.723 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:45:04 smithi089 bash[78452]: audit 1685821503.5796103 mon.a (mon.0) 135 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:45:04.724 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:45:04 smithi089 bash[78452]: audit 1685821503.593357 mon.a (mon.0) 136 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:45:04.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:45:04 smithi164 bash[59462]: audit 1685821503.5796103 mon.a (mon.0) 135 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:45:04.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:45:04 smithi164 bash[59462]: audit 1685821503.593357 mon.a (mon.0) 136 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:45:06.030 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:45:05 smithi089 bash[78452]: cluster 1685821504.0522726 mgr.y (mgr.34104) 141 : cluster 0 pgmap v119: 89 pgs: 89 active+clean; 2.2 GiB data, 7.9 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 891 op/s 2023-06-03T19:45:06.030 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:45:05 smithi089 bash[75068]: cluster 1685821504.0522726 mgr.y (mgr.34104) 141 : cluster 0 pgmap v119: 89 pgs: 89 active+clean; 2.2 GiB data, 7.9 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 891 op/s 2023-06-03T19:45:06.085 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:45:05 smithi164 bash[59462]: cluster 1685821504.0522726 mgr.y (mgr.34104) 141 : cluster 0 pgmap v119: 89 pgs: 89 active+clean; 2.2 GiB data, 7.9 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 891 op/s 2023-06-03T19:45:06.530 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 19:45:06 smithi089 bash[71163]: ::ffff:172.21.15.164 - - [03/Jun/2023:19:45:06] "GET /metrics HTTP/1.1" 200 184103 "" "Prometheus/2.33.4" 2023-06-03T19:45:07.814 INFO:teuthology.orchestra.run.smithi089.stdout:Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:00b1b6d689278b4f97ff790045c8c51d8ca419ac7ee2ebdf11008f2e810a5c78 2023-06-03T19:45:07.990 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:45:07 smithi089 bash[78452]: cluster 1685821506.052761 mgr.y (mgr.34104) 142 : cluster 0 pgmap v120: 89 pgs: 89 active+clean; 2.2 GiB data, 7.9 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 589 op/s 2023-06-03T19:45:07.990 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:45:07 smithi089 bash[78452]: audit 1685821507.065696 mon.c (mon.2) 14 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T19:45:07.991 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:45:07 smithi089 bash[75068]: cluster 1685821506.052761 mgr.y (mgr.34104) 142 : cluster 0 pgmap v120: 89 pgs: 89 active+clean; 2.2 GiB data, 7.9 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 589 op/s 2023-06-03T19:45:07.991 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:45:07 smithi089 bash[75068]: audit 1685821507.065696 mon.c (mon.2) 14 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T19:45:08.085 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:45:07 smithi164 bash[59462]: cluster 1685821506.052761 mgr.y (mgr.34104) 142 : cluster 0 pgmap v120: 89 pgs: 89 active+clean; 2.2 GiB data, 7.9 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 589 op/s 2023-06-03T19:45:08.085 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:45:07 smithi164 bash[59462]: audit 1685821507.065696 mon.c (mon.2) 14 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T19:45:08.929 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -e sha1=8e24bcf462a1d94c2e12aa4e26419634883a1e88 -- bash -c 'sleep 60' 2023-06-03T19:45:09.085 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:45:08 smithi164 bash[59462]: audit 1685821507.7978117 mgr.y (mgr.34104) 143 : audit 0 from='client.44129 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2023-06-03T19:45:09.086 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:45:08 smithi164 bash[59462]: cephadm 1685821507.7982414 mgr.y (mgr.34104) 144 : cephadm 1 Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:00b1b6d689278b4f97ff790045c8c51d8ca419ac7ee2ebdf11008f2e810a5c78 2023-06-03T19:45:09.086 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:45:08 smithi164 bash[59462]: audit 1685821507.805424 mon.a (mon.0) 137 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:45:09.086 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:45:08 smithi164 bash[59462]: cluster 1685821508.053215 mgr.y (mgr.34104) 145 : cluster 0 pgmap v121: 89 pgs: 89 active+clean; 2.2 GiB data, 7.9 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 589 op/s 2023-06-03T19:45:09.192 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:45:08 smithi089 bash[75068]: audit 1685821507.7978117 mgr.y (mgr.34104) 143 : audit 0 from='client.44129 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2023-06-03T19:45:09.193 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:45:08 smithi089 bash[75068]: cephadm 1685821507.7982414 mgr.y (mgr.34104) 144 : cephadm 1 Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:00b1b6d689278b4f97ff790045c8c51d8ca419ac7ee2ebdf11008f2e810a5c78 2023-06-03T19:45:09.193 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:45:08 smithi089 bash[75068]: audit 1685821507.805424 mon.a (mon.0) 137 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:45:09.194 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:45:08 smithi089 bash[75068]: cluster 1685821508.053215 mgr.y (mgr.34104) 145 : cluster 0 pgmap v121: 89 pgs: 89 active+clean; 2.2 GiB data, 7.9 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 589 op/s 2023-06-03T19:45:09.194 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:45:08 smithi089 bash[78452]: audit 1685821507.7978117 mgr.y (mgr.34104) 143 : audit 0 from='client.44129 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2023-06-03T19:45:09.194 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:45:08 smithi089 bash[78452]: cephadm 1685821507.7982414 mgr.y (mgr.34104) 144 : cephadm 1 Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:00b1b6d689278b4f97ff790045c8c51d8ca419ac7ee2ebdf11008f2e810a5c78 2023-06-03T19:45:09.194 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:45:08 smithi089 bash[78452]: audit 1685821507.805424 mon.a (mon.0) 137 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:45:09.195 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:45:08 smithi089 bash[78452]: cluster 1685821508.053215 mgr.y (mgr.34104) 145 : cluster 0 pgmap v121: 89 pgs: 89 active+clean; 2.2 GiB data, 7.9 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 589 op/s 2023-06-03T19:45:11.280 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:45:11 smithi089 bash[78452]: cluster 1685821510.0539334 mgr.y (mgr.34104) 146 : cluster 0 pgmap v122: 89 pgs: 89 active+clean; 2.0 GiB data, 7.3 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 882 op/s 2023-06-03T19:45:11.280 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:45:11 smithi089 bash[75068]: cluster 1685821510.0539334 mgr.y (mgr.34104) 146 : cluster 0 pgmap v122: 89 pgs: 89 active+clean; 2.0 GiB data, 7.3 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 882 op/s 2023-06-03T19:45:11.585 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:45:11 smithi164 bash[59462]: cluster 1685821510.0539334 mgr.y (mgr.34104) 146 : cluster 0 pgmap v122: 89 pgs: 89 active+clean; 2.0 GiB data, 7.3 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 882 op/s 2023-06-03T19:45:13.280 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:45:13 smithi089 bash[78452]: cluster 1685821512.05441 mgr.y (mgr.34104) 147 : cluster 0 pgmap v123: 89 pgs: 89 active+clean; 2.0 GiB data, 7.3 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 588 op/s 2023-06-03T19:45:13.281 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:45:13 smithi089 bash[75068]: cluster 1685821512.05441 mgr.y (mgr.34104) 147 : cluster 0 pgmap v123: 89 pgs: 89 active+clean; 2.0 GiB data, 7.3 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 588 op/s 2023-06-03T19:45:13.585 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:45:13 smithi164 bash[59462]: cluster 1685821512.05441 mgr.y (mgr.34104) 147 : cluster 0 pgmap v123: 89 pgs: 89 active+clean; 2.0 GiB data, 7.3 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 588 op/s 2023-06-03T19:45:15.335 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:45:15 smithi164 bash[59462]: cluster 1685821514.0551696 mgr.y (mgr.34104) 148 : cluster 0 pgmap v124: 89 pgs: 89 active+clean; 1.8 GiB data, 6.6 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 886 op/s 2023-06-03T19:45:15.530 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:45:15 smithi089 bash[78452]: cluster 1685821514.0551696 mgr.y (mgr.34104) 148 : cluster 0 pgmap v124: 89 pgs: 89 active+clean; 1.8 GiB data, 6.6 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 886 op/s 2023-06-03T19:45:15.531 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:45:15 smithi089 bash[75068]: cluster 1685821514.0551696 mgr.y (mgr.34104) 148 : cluster 0 pgmap v124: 89 pgs: 89 active+clean; 1.8 GiB data, 6.6 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 886 op/s 2023-06-03T19:45:16.535 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 19:45:16 smithi089 bash[71163]: ::ffff:172.21.15.164 - - [03/Jun/2023:19:45:16] "GET /metrics HTTP/1.1" 200 184121 "" "Prometheus/2.33.4" 2023-06-03T19:45:17.010 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:45:17 smithi089 bash[75068]: audit 1685821516.0008087 mon.a (mon.0) 138 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:45:17.010 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:45:17 smithi089 bash[75068]: audit 1685821516.0182006 mon.a (mon.0) 139 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:45:17.010 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:45:17 smithi089 bash[75068]: audit 1685821516.0289488 mon.c (mon.2) 15 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-03T19:45:17.010 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:45:17 smithi089 bash[75068]: audit 1685821516.0301075 mon.c (mon.2) 16 : audit 1 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-03T19:45:17.011 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:45:17 smithi089 bash[75068]: audit 1685821516.0384972 mon.a (mon.0) 140 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:45:17.011 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:45:17 smithi089 bash[75068]: cluster 1685821516.0556495 mgr.y (mgr.34104) 149 : cluster 0 pgmap v125: 89 pgs: 89 active+clean; 1.8 GiB data, 6.6 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 591 op/s 2023-06-03T19:45:17.011 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:45:17 smithi089 bash[75068]: cephadm 1685821516.1095364 mgr.y (mgr.34104) 150 : cephadm 1 Reconfiguring mon.a (monmap changed)... 2023-06-03T19:45:17.280 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:45:17 smithi089 bash[75068]: audit 1685821516.1101098 mon.c (mon.2) 17 : audit 1 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-06-03T19:45:17.281 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:45:17 smithi089 bash[75068]: audit 1685821516.1114848 mon.c (mon.2) 18 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-06-03T19:45:17.281 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:45:17 smithi089 bash[75068]: audit 1685821516.112977 mon.c (mon.2) 19 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-03T19:45:17.281 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:45:17 smithi089 bash[75068]: cephadm 1685821516.1149404 mgr.y (mgr.34104) 151 : cephadm 1 Reconfiguring daemon mon.a on smithi089 2023-06-03T19:45:17.282 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:45:17 smithi089 bash[78452]: audit 1685821516.0008087 mon.a (mon.0) 138 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:45:17.282 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:45:17 smithi089 bash[78452]: audit 1685821516.0182006 mon.a (mon.0) 139 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:45:17.282 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:45:17 smithi089 bash[78452]: audit 1685821516.0289488 mon.c (mon.2) 15 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-03T19:45:17.282 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:45:17 smithi089 bash[78452]: audit 1685821516.0301075 mon.c (mon.2) 16 : audit 1 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-03T19:45:17.283 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:45:17 smithi089 bash[78452]: audit 1685821516.0384972 mon.a (mon.0) 140 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:45:17.283 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:45:17 smithi089 bash[78452]: cluster 1685821516.0556495 mgr.y (mgr.34104) 149 : cluster 0 pgmap v125: 89 pgs: 89 active+clean; 1.8 GiB data, 6.6 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 591 op/s 2023-06-03T19:45:17.283 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:45:17 smithi089 bash[78452]: cephadm 1685821516.1095364 mgr.y (mgr.34104) 150 : cephadm 1 Reconfiguring mon.a (monmap changed)... 2023-06-03T19:45:17.283 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:45:17 smithi089 bash[78452]: audit 1685821516.1101098 mon.c (mon.2) 17 : audit 1 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-06-03T19:45:17.283 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:45:17 smithi089 bash[78452]: audit 1685821516.1114848 mon.c (mon.2) 18 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-06-03T19:45:17.283 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:45:17 smithi089 bash[78452]: audit 1685821516.112977 mon.c (mon.2) 19 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-03T19:45:17.284 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:45:17 smithi089 bash[78452]: cephadm 1685821516.1149404 mgr.y (mgr.34104) 151 : cephadm 1 Reconfiguring daemon mon.a on smithi089 2023-06-03T19:45:17.335 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:45:17 smithi164 bash[59462]: audit 1685821516.0008087 mon.a (mon.0) 138 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:45:17.335 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:45:17 smithi164 bash[59462]: audit 1685821516.0182006 mon.a (mon.0) 139 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:45:17.335 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:45:17 smithi164 bash[59462]: audit 1685821516.0289488 mon.c (mon.2) 15 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-03T19:45:17.336 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:45:17 smithi164 bash[59462]: audit 1685821516.0301075 mon.c (mon.2) 16 : audit 1 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-03T19:45:17.336 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:45:17 smithi164 bash[59462]: audit 1685821516.0384972 mon.a (mon.0) 140 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:45:17.336 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:45:17 smithi164 bash[59462]: cluster 1685821516.0556495 mgr.y (mgr.34104) 149 : cluster 0 pgmap v125: 89 pgs: 89 active+clean; 1.8 GiB data, 6.6 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 591 op/s 2023-06-03T19:45:17.336 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:45:17 smithi164 bash[59462]: cephadm 1685821516.1095364 mgr.y (mgr.34104) 150 : cephadm 1 Reconfiguring mon.a (monmap changed)... 2023-06-03T19:45:17.336 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:45:17 smithi164 bash[59462]: audit 1685821516.1101098 mon.c (mon.2) 17 : audit 1 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-06-03T19:45:17.337 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:45:17 smithi164 bash[59462]: audit 1685821516.1114848 mon.c (mon.2) 18 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-06-03T19:45:17.337 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:45:17 smithi164 bash[59462]: audit 1685821516.112977 mon.c (mon.2) 19 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-03T19:45:17.337 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:45:17 smithi164 bash[59462]: cephadm 1685821516.1149404 mgr.y (mgr.34104) 151 : cephadm 1 Reconfiguring daemon mon.a on smithi089 2023-06-03T19:45:19.530 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:45:19 smithi089 bash[75068]: cluster 1685821518.0560942 mgr.y (mgr.34104) 152 : cluster 0 pgmap v126: 89 pgs: 89 active+clean; 1.8 GiB data, 6.6 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 591 op/s 2023-06-03T19:45:19.530 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:45:19 smithi089 bash[78452]: cluster 1685821518.0560942 mgr.y (mgr.34104) 152 : cluster 0 pgmap v126: 89 pgs: 89 active+clean; 1.8 GiB data, 6.6 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 591 op/s 2023-06-03T19:45:19.585 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:45:19 smithi164 bash[59462]: cluster 1685821518.0560942 mgr.y (mgr.34104) 152 : cluster 0 pgmap v126: 89 pgs: 89 active+clean; 1.8 GiB data, 6.6 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 591 op/s 2023-06-03T19:45:20.780 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:45:20 smithi089 bash[75068]: audit 1685821519.491266 mon.a (mon.0) 141 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:45:20.780 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:45:20 smithi089 bash[75068]: audit 1685821519.5086493 mon.a (mon.0) 142 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:45:20.780 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:45:20 smithi089 bash[75068]: cephadm 1685821519.5511167 mgr.y (mgr.34104) 153 : cephadm 1 Reconfiguring mgr.y (monmap changed)... 2023-06-03T19:45:20.781 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:45:20 smithi089 bash[75068]: audit 1685821519.5518568 mon.c (mon.2) 20 : audit 1 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-06-03T19:45:20.781 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:45:20 smithi089 bash[75068]: audit 1685821519.5528688 mon.a (mon.0) 143 : audit 1 from='mgr.34104 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-06-03T19:45:20.781 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:45:20 smithi089 bash[75068]: audit 1685821519.5548296 mon.c (mon.2) 21 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-06-03T19:45:20.781 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:45:20 smithi089 bash[75068]: audit 1685821519.5564575 mon.c (mon.2) 22 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-03T19:45:20.781 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:45:20 smithi089 bash[75068]: cephadm 1685821519.5577989 mgr.y (mgr.34104) 154 : cephadm 1 Reconfiguring daemon mgr.y on smithi089 2023-06-03T19:45:20.781 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:45:20 smithi089 bash[78452]: audit 1685821519.491266 mon.a (mon.0) 141 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:45:20.782 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:45:20 smithi089 bash[78452]: audit 1685821519.5086493 mon.a (mon.0) 142 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:45:20.782 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:45:20 smithi089 bash[78452]: cephadm 1685821519.5511167 mgr.y (mgr.34104) 153 : cephadm 1 Reconfiguring mgr.y (monmap changed)... 2023-06-03T19:45:20.782 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:45:20 smithi089 bash[78452]: audit 1685821519.5518568 mon.c (mon.2) 20 : audit 1 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-06-03T19:45:20.782 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:45:20 smithi089 bash[78452]: audit 1685821519.5528688 mon.a (mon.0) 143 : audit 1 from='mgr.34104 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-06-03T19:45:20.783 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:45:20 smithi089 bash[78452]: audit 1685821519.5548296 mon.c (mon.2) 21 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-06-03T19:45:20.783 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:45:20 smithi089 bash[78452]: audit 1685821519.5564575 mon.c (mon.2) 22 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-03T19:45:20.783 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:45:20 smithi089 bash[78452]: cephadm 1685821519.5577989 mgr.y (mgr.34104) 154 : cephadm 1 Reconfiguring daemon mgr.y on smithi089 2023-06-03T19:45:20.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:45:20 smithi164 bash[59462]: audit 1685821519.491266 mon.a (mon.0) 141 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:45:20.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:45:20 smithi164 bash[59462]: audit 1685821519.5086493 mon.a (mon.0) 142 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:45:20.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:45:20 smithi164 bash[59462]: cephadm 1685821519.5511167 mgr.y (mgr.34104) 153 : cephadm 1 Reconfiguring mgr.y (monmap changed)... 2023-06-03T19:45:20.836 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:45:20 smithi164 bash[59462]: audit 1685821519.5518568 mon.c (mon.2) 20 : audit 1 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-06-03T19:45:20.836 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:45:20 smithi164 bash[59462]: audit 1685821519.5528688 mon.a (mon.0) 143 : audit 1 from='mgr.34104 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-06-03T19:45:20.836 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:45:20 smithi164 bash[59462]: audit 1685821519.5548296 mon.c (mon.2) 21 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-06-03T19:45:20.836 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:45:20 smithi164 bash[59462]: audit 1685821519.5564575 mon.c (mon.2) 22 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-03T19:45:20.836 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:45:20 smithi164 bash[59462]: cephadm 1685821519.5577989 mgr.y (mgr.34104) 154 : cephadm 1 Reconfiguring daemon mgr.y on smithi089 2023-06-03T19:45:21.780 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:45:21 smithi089 bash[75068]: cluster 1685821520.056876 mgr.y (mgr.34104) 155 : cluster 0 pgmap v127: 89 pgs: 89 active+clean; 1.6 GiB data, 6.0 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 880 op/s 2023-06-03T19:45:21.780 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:45:21 smithi089 bash[78452]: cluster 1685821520.056876 mgr.y (mgr.34104) 155 : cluster 0 pgmap v127: 89 pgs: 89 active+clean; 1.6 GiB data, 6.0 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 880 op/s 2023-06-03T19:45:21.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:45:21 smithi164 bash[59462]: cluster 1685821520.056876 mgr.y (mgr.34104) 155 : cluster 0 pgmap v127: 89 pgs: 89 active+clean; 1.6 GiB data, 6.0 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 880 op/s 2023-06-03T19:45:22.781 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:45:22 smithi089 bash[75068]: audit 1685821522.0655384 mon.c (mon.2) 23 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T19:45:22.781 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:45:22 smithi089 bash[78452]: audit 1685821522.0655384 mon.c (mon.2) 23 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T19:45:22.836 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:45:22 smithi164 bash[59462]: audit 1685821522.0655384 mon.c (mon.2) 23 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T19:45:24.030 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:45:23 smithi089 bash[75068]: cluster 1685821522.0572965 mgr.y (mgr.34104) 156 : cluster 0 pgmap v128: 89 pgs: 89 active+clean; 1.6 GiB data, 6.0 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 587 op/s 2023-06-03T19:45:24.031 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:45:23 smithi089 bash[75068]: audit 1685821522.6202152 mon.a (mon.0) 144 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:45:24.031 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:45:23 smithi089 bash[75068]: audit 1685821522.634315 mon.a (mon.0) 145 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:45:24.031 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:45:23 smithi089 bash[75068]: cephadm 1685821522.638913 mgr.y (mgr.34104) 157 : cephadm 1 Reconfiguring mon.c (monmap changed)... 2023-06-03T19:45:24.031 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:45:23 smithi089 bash[75068]: audit 1685821522.6392944 mon.c (mon.2) 24 : audit 1 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-06-03T19:45:24.031 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:45:23 smithi089 bash[75068]: audit 1685821522.6403148 mon.c (mon.2) 25 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-06-03T19:45:24.031 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:45:23 smithi089 bash[75068]: audit 1685821522.6411843 mon.c (mon.2) 26 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-03T19:45:24.031 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:45:23 smithi089 bash[75068]: cephadm 1685821522.6423218 mgr.y (mgr.34104) 158 : cephadm 1 Reconfiguring daemon mon.c on smithi089 2023-06-03T19:45:24.032 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:45:23 smithi089 bash[78452]: cluster 1685821522.0572965 mgr.y (mgr.34104) 156 : cluster 0 pgmap v128: 89 pgs: 89 active+clean; 1.6 GiB data, 6.0 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 587 op/s 2023-06-03T19:45:24.032 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:45:23 smithi089 bash[78452]: audit 1685821522.6202152 mon.a (mon.0) 144 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:45:24.032 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:45:23 smithi089 bash[78452]: audit 1685821522.634315 mon.a (mon.0) 145 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:45:24.032 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:45:23 smithi089 bash[78452]: cephadm 1685821522.638913 mgr.y (mgr.34104) 157 : cephadm 1 Reconfiguring mon.c (monmap changed)... 2023-06-03T19:45:24.032 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:45:23 smithi089 bash[78452]: audit 1685821522.6392944 mon.c (mon.2) 24 : audit 1 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-06-03T19:45:24.032 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:45:23 smithi089 bash[78452]: audit 1685821522.6403148 mon.c (mon.2) 25 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-06-03T19:45:24.033 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:45:23 smithi089 bash[78452]: audit 1685821522.6411843 mon.c (mon.2) 26 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-03T19:45:24.033 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:45:23 smithi089 bash[78452]: cephadm 1685821522.6423218 mgr.y (mgr.34104) 158 : cephadm 1 Reconfiguring daemon mon.c on smithi089 2023-06-03T19:45:24.085 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:45:23 smithi164 bash[59462]: cluster 1685821522.0572965 mgr.y (mgr.34104) 156 : cluster 0 pgmap v128: 89 pgs: 89 active+clean; 1.6 GiB data, 6.0 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 587 op/s 2023-06-03T19:45:24.085 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:45:23 smithi164 bash[59462]: audit 1685821522.6202152 mon.a (mon.0) 144 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:45:24.085 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:45:23 smithi164 bash[59462]: audit 1685821522.634315 mon.a (mon.0) 145 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:45:24.086 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:45:23 smithi164 bash[59462]: cephadm 1685821522.638913 mgr.y (mgr.34104) 157 : cephadm 1 Reconfiguring mon.c (monmap changed)... 2023-06-03T19:45:24.086 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:45:23 smithi164 bash[59462]: audit 1685821522.6392944 mon.c (mon.2) 24 : audit 1 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-06-03T19:45:24.086 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:45:23 smithi164 bash[59462]: audit 1685821522.6403148 mon.c (mon.2) 25 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-06-03T19:45:24.086 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:45:23 smithi164 bash[59462]: audit 1685821522.6411843 mon.c (mon.2) 26 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-03T19:45:24.086 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:45:23 smithi164 bash[59462]: cephadm 1685821522.6423218 mgr.y (mgr.34104) 158 : cephadm 1 Reconfiguring daemon mon.c on smithi089 2023-06-03T19:45:25.955 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:45:25 smithi089 bash[75068]: cluster 1685821524.0580292 mgr.y (mgr.34104) 159 : cluster 0 pgmap v129: 89 pgs: 89 active+clean; 1.4 GiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 880 op/s 2023-06-03T19:45:25.956 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:45:25 smithi089 bash[75068]: audit 1685821525.5364637 mon.a (mon.0) 146 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:45:25.956 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:45:25 smithi089 bash[75068]: audit 1685821525.5444312 mon.a (mon.0) 147 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:45:25.956 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:45:25 smithi089 bash[75068]: audit 1685821525.57259 mon.c (mon.2) 27 : audit 1 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2023-06-03T19:45:25.956 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:45:25 smithi089 bash[75068]: audit 1685821525.574022 mon.c (mon.2) 28 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-03T19:45:25.956 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:45:25 smithi089 bash[78452]: cluster 1685821524.0580292 mgr.y (mgr.34104) 159 : cluster 0 pgmap v129: 89 pgs: 89 active+clean; 1.4 GiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 880 op/s 2023-06-03T19:45:25.957 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:45:25 smithi089 bash[78452]: audit 1685821525.5364637 mon.a (mon.0) 146 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:45:25.957 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:45:25 smithi089 bash[78452]: audit 1685821525.5444312 mon.a (mon.0) 147 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:45:25.957 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:45:25 smithi089 bash[78452]: audit 1685821525.57259 mon.c (mon.2) 27 : audit 1 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2023-06-03T19:45:25.957 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:45:25 smithi089 bash[78452]: audit 1685821525.574022 mon.c (mon.2) 28 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-03T19:45:26.085 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:45:25 smithi164 bash[59462]: cluster 1685821524.0580292 mgr.y (mgr.34104) 159 : cluster 0 pgmap v129: 89 pgs: 89 active+clean; 1.4 GiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 880 op/s 2023-06-03T19:45:26.085 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:45:25 smithi164 bash[59462]: audit 1685821525.5364637 mon.a (mon.0) 146 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:45:26.086 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:45:25 smithi164 bash[59462]: audit 1685821525.5444312 mon.a (mon.0) 147 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:45:26.086 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:45:25 smithi164 bash[59462]: audit 1685821525.57259 mon.c (mon.2) 27 : audit 1 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2023-06-03T19:45:26.086 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:45:25 smithi164 bash[59462]: audit 1685821525.574022 mon.c (mon.2) 28 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-03T19:45:26.233 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 19:45:26 smithi089 bash[71163]: ::ffff:172.21.15.164 - - [03/Jun/2023:19:45:26] "GET /metrics HTTP/1.1" 200 184136 "" "Prometheus/2.33.4" 2023-06-03T19:45:27.030 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:45:26 smithi089 bash[78452]: cephadm 1685821525.5721455 mgr.y (mgr.34104) 160 : cephadm 1 Reconfiguring osd.0 (monmap changed)... 2023-06-03T19:45:27.030 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:45:26 smithi089 bash[78452]: cephadm 1685821525.57779 mgr.y (mgr.34104) 161 : cephadm 1 Reconfiguring daemon osd.0 on smithi089 2023-06-03T19:45:27.030 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:45:26 smithi089 bash[75068]: cephadm 1685821525.5721455 mgr.y (mgr.34104) 160 : cephadm 1 Reconfiguring osd.0 (monmap changed)... 2023-06-03T19:45:27.031 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:45:26 smithi089 bash[75068]: cephadm 1685821525.57779 mgr.y (mgr.34104) 161 : cephadm 1 Reconfiguring daemon osd.0 on smithi089 2023-06-03T19:45:27.085 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:45:26 smithi164 bash[59462]: cephadm 1685821525.5721455 mgr.y (mgr.34104) 160 : cephadm 1 Reconfiguring osd.0 (monmap changed)... 2023-06-03T19:45:27.085 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:45:26 smithi164 bash[59462]: cephadm 1685821525.57779 mgr.y (mgr.34104) 161 : cephadm 1 Reconfiguring daemon osd.0 on smithi089 2023-06-03T19:45:28.030 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:45:27 smithi089 bash[75068]: cluster 1685821526.0610478 mgr.y (mgr.34104) 162 : cluster 0 pgmap v130: 89 pgs: 89 active+clean; 1.4 GiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 581 op/s 2023-06-03T19:45:28.030 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:45:27 smithi089 bash[78452]: cluster 1685821526.0610478 mgr.y (mgr.34104) 162 : cluster 0 pgmap v130: 89 pgs: 89 active+clean; 1.4 GiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 581 op/s 2023-06-03T19:45:28.085 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:45:27 smithi164 bash[59462]: cluster 1685821526.0610478 mgr.y (mgr.34104) 162 : cluster 0 pgmap v130: 89 pgs: 89 active+clean; 1.4 GiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 581 op/s 2023-06-03T19:45:30.030 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:45:29 smithi089 bash[75068]: cluster 1685821528.061531 mgr.y (mgr.34104) 163 : cluster 0 pgmap v131: 89 pgs: 89 active+clean; 1.4 GiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 581 op/s 2023-06-03T19:45:30.030 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:45:29 smithi089 bash[75068]: audit 1685821528.5955033 mon.a (mon.0) 148 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:45:30.030 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:45:29 smithi089 bash[75068]: audit 1685821528.6096437 mon.a (mon.0) 149 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:45:30.031 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:45:29 smithi089 bash[75068]: cephadm 1685821528.6511247 mgr.y (mgr.34104) 164 : cephadm 1 Reconfiguring osd.1 (monmap changed)... 2023-06-03T19:45:30.031 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:45:29 smithi089 bash[75068]: audit 1685821528.6517072 mon.c (mon.2) 29 : audit 1 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2023-06-03T19:45:30.031 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:45:29 smithi089 bash[75068]: audit 1685821528.6530156 mon.c (mon.2) 30 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-03T19:45:30.031 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:45:29 smithi089 bash[75068]: cephadm 1685821528.6633363 mgr.y (mgr.34104) 165 : cephadm 1 Reconfiguring daemon osd.1 on smithi089 2023-06-03T19:45:30.031 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:45:29 smithi089 bash[78452]: cluster 1685821528.061531 mgr.y (mgr.34104) 163 : cluster 0 pgmap v131: 89 pgs: 89 active+clean; 1.4 GiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 581 op/s 2023-06-03T19:45:30.031 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:45:29 smithi089 bash[78452]: audit 1685821528.5955033 mon.a (mon.0) 148 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:45:30.032 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:45:29 smithi089 bash[78452]: audit 1685821528.6096437 mon.a (mon.0) 149 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:45:30.032 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:45:29 smithi089 bash[78452]: cephadm 1685821528.6511247 mgr.y (mgr.34104) 164 : cephadm 1 Reconfiguring osd.1 (monmap changed)... 2023-06-03T19:45:30.032 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:45:29 smithi089 bash[78452]: audit 1685821528.6517072 mon.c (mon.2) 29 : audit 1 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2023-06-03T19:45:30.032 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:45:29 smithi089 bash[78452]: audit 1685821528.6530156 mon.c (mon.2) 30 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-03T19:45:30.032 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:45:29 smithi089 bash[78452]: cephadm 1685821528.6633363 mgr.y (mgr.34104) 165 : cephadm 1 Reconfiguring daemon osd.1 on smithi089 2023-06-03T19:45:30.085 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:45:29 smithi164 bash[59462]: cluster 1685821528.061531 mgr.y (mgr.34104) 163 : cluster 0 pgmap v131: 89 pgs: 89 active+clean; 1.4 GiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 581 op/s 2023-06-03T19:45:30.085 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:45:29 smithi164 bash[59462]: audit 1685821528.5955033 mon.a (mon.0) 148 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:45:30.085 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:45:29 smithi164 bash[59462]: audit 1685821528.6096437 mon.a (mon.0) 149 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:45:30.086 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:45:29 smithi164 bash[59462]: cephadm 1685821528.6511247 mgr.y (mgr.34104) 164 : cephadm 1 Reconfiguring osd.1 (monmap changed)... 2023-06-03T19:45:30.086 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:45:29 smithi164 bash[59462]: audit 1685821528.6517072 mon.c (mon.2) 29 : audit 1 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2023-06-03T19:45:30.086 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:45:29 smithi164 bash[59462]: audit 1685821528.6530156 mon.c (mon.2) 30 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-03T19:45:30.086 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:45:29 smithi164 bash[59462]: cephadm 1685821528.6633363 mgr.y (mgr.34104) 165 : cephadm 1 Reconfiguring daemon osd.1 on smithi089 2023-06-03T19:45:32.030 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:45:31 smithi089 bash[75068]: cluster 1685821530.0622435 mgr.y (mgr.34104) 166 : cluster 0 pgmap v132: 89 pgs: 89 active+clean; 1.1 GiB data, 4.7 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 876 op/s 2023-06-03T19:45:32.030 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:45:31 smithi089 bash[75068]: audit 1685821531.3334842 mon.a (mon.0) 150 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:45:32.031 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:45:31 smithi089 bash[75068]: audit 1685821531.343264 mon.a (mon.0) 151 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:45:32.031 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:45:31 smithi089 bash[75068]: audit 1685821531.3938987 mon.c (mon.2) 31 : audit 1 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2023-06-03T19:45:32.031 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:45:31 smithi089 bash[75068]: audit 1685821531.3952763 mon.c (mon.2) 32 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-03T19:45:32.031 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:45:31 smithi089 bash[78452]: cluster 1685821530.0622435 mgr.y (mgr.34104) 166 : cluster 0 pgmap v132: 89 pgs: 89 active+clean; 1.1 GiB data, 4.7 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 876 op/s 2023-06-03T19:45:32.031 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:45:31 smithi089 bash[78452]: audit 1685821531.3334842 mon.a (mon.0) 150 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:45:32.032 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:45:31 smithi089 bash[78452]: audit 1685821531.343264 mon.a (mon.0) 151 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:45:32.032 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:45:31 smithi089 bash[78452]: audit 1685821531.3938987 mon.c (mon.2) 31 : audit 1 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2023-06-03T19:45:32.032 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:45:31 smithi089 bash[78452]: audit 1685821531.3952763 mon.c (mon.2) 32 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-03T19:45:32.085 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:45:31 smithi164 bash[59462]: cluster 1685821530.0622435 mgr.y (mgr.34104) 166 : cluster 0 pgmap v132: 89 pgs: 89 active+clean; 1.1 GiB data, 4.7 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 876 op/s 2023-06-03T19:45:32.085 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:45:31 smithi164 bash[59462]: audit 1685821531.3334842 mon.a (mon.0) 150 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:45:32.085 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:45:31 smithi164 bash[59462]: audit 1685821531.343264 mon.a (mon.0) 151 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:45:32.086 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:45:31 smithi164 bash[59462]: audit 1685821531.3938987 mon.c (mon.2) 31 : audit 1 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2023-06-03T19:45:32.086 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:45:31 smithi164 bash[59462]: audit 1685821531.3952763 mon.c (mon.2) 32 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-03T19:45:32.780 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:45:32 smithi089 bash[75068]: cephadm 1685821531.3932848 mgr.y (mgr.34104) 167 : cephadm 1 Reconfiguring osd.2 (monmap changed)... 2023-06-03T19:45:32.780 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:45:32 smithi089 bash[75068]: cephadm 1685821531.3982794 mgr.y (mgr.34104) 168 : cephadm 1 Reconfiguring daemon osd.2 on smithi089 2023-06-03T19:45:32.780 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:45:32 smithi089 bash[78452]: cephadm 1685821531.3932848 mgr.y (mgr.34104) 167 : cephadm 1 Reconfiguring osd.2 (monmap changed)... 2023-06-03T19:45:32.781 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:45:32 smithi089 bash[78452]: cephadm 1685821531.3982794 mgr.y (mgr.34104) 168 : cephadm 1 Reconfiguring daemon osd.2 on smithi089 2023-06-03T19:45:33.085 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:45:32 smithi164 bash[59462]: cephadm 1685821531.3932848 mgr.y (mgr.34104) 167 : cephadm 1 Reconfiguring osd.2 (monmap changed)... 2023-06-03T19:45:33.085 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:45:32 smithi164 bash[59462]: cephadm 1685821531.3982794 mgr.y (mgr.34104) 168 : cephadm 1 Reconfiguring daemon osd.2 on smithi089 2023-06-03T19:45:33.780 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:45:33 smithi089 bash[75068]: cluster 1685821532.0626936 mgr.y (mgr.34104) 169 : cluster 0 pgmap v133: 89 pgs: 89 active+clean; 1.1 GiB data, 4.7 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 588 op/s 2023-06-03T19:45:33.780 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:45:33 smithi089 bash[78452]: cluster 1685821532.0626936 mgr.y (mgr.34104) 169 : cluster 0 pgmap v133: 89 pgs: 89 active+clean; 1.1 GiB data, 4.7 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 588 op/s 2023-06-03T19:45:34.085 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:45:33 smithi164 bash[59462]: cluster 1685821532.0626936 mgr.y (mgr.34104) 169 : cluster 0 pgmap v133: 89 pgs: 89 active+clean; 1.1 GiB data, 4.7 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 588 op/s 2023-06-03T19:45:35.530 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:45:35 smithi089 bash[75068]: cluster 1685821534.0633814 mgr.y (mgr.34104) 170 : cluster 0 pgmap v134: 89 pgs: 89 active+clean; 939 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 893 op/s 2023-06-03T19:45:35.530 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:45:35 smithi089 bash[75068]: audit 1685821534.2707565 mon.a (mon.0) 152 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:45:35.531 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:45:35 smithi089 bash[75068]: audit 1685821534.2825782 mon.a (mon.0) 153 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:45:35.531 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:45:35 smithi089 bash[75068]: cephadm 1685821534.3349037 mgr.y (mgr.34104) 171 : cephadm 1 Reconfiguring osd.3 (monmap changed)... 2023-06-03T19:45:35.531 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:45:35 smithi089 bash[75068]: audit 1685821534.335653 mon.c (mon.2) 33 : audit 1 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2023-06-03T19:45:35.531 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:45:35 smithi089 bash[75068]: audit 1685821534.3390431 mon.c (mon.2) 34 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-03T19:45:35.531 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:45:35 smithi089 bash[75068]: cephadm 1685821534.3418243 mgr.y (mgr.34104) 172 : cephadm 1 Reconfiguring daemon osd.3 on smithi089 2023-06-03T19:45:35.531 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:45:35 smithi089 bash[78452]: cluster 1685821534.0633814 mgr.y (mgr.34104) 170 : cluster 0 pgmap v134: 89 pgs: 89 active+clean; 939 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 893 op/s 2023-06-03T19:45:35.532 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:45:35 smithi089 bash[78452]: audit 1685821534.2707565 mon.a (mon.0) 152 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:45:35.532 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:45:35 smithi089 bash[78452]: audit 1685821534.2825782 mon.a (mon.0) 153 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:45:35.532 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:45:35 smithi089 bash[78452]: cephadm 1685821534.3349037 mgr.y (mgr.34104) 171 : cephadm 1 Reconfiguring osd.3 (monmap changed)... 2023-06-03T19:45:35.532 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:45:35 smithi089 bash[78452]: audit 1685821534.335653 mon.c (mon.2) 33 : audit 1 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2023-06-03T19:45:35.532 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:45:35 smithi089 bash[78452]: audit 1685821534.3390431 mon.c (mon.2) 34 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-03T19:45:35.532 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:45:35 smithi089 bash[78452]: cephadm 1685821534.3418243 mgr.y (mgr.34104) 172 : cephadm 1 Reconfiguring daemon osd.3 on smithi089 2023-06-03T19:45:35.585 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:45:35 smithi164 bash[59462]: cluster 1685821534.0633814 mgr.y (mgr.34104) 170 : cluster 0 pgmap v134: 89 pgs: 89 active+clean; 939 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 893 op/s 2023-06-03T19:45:35.585 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:45:35 smithi164 bash[59462]: audit 1685821534.2707565 mon.a (mon.0) 152 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:45:35.585 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:45:35 smithi164 bash[59462]: audit 1685821534.2825782 mon.a (mon.0) 153 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:45:35.586 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:45:35 smithi164 bash[59462]: cephadm 1685821534.3349037 mgr.y (mgr.34104) 171 : cephadm 1 Reconfiguring osd.3 (monmap changed)... 2023-06-03T19:45:35.586 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:45:35 smithi164 bash[59462]: audit 1685821534.335653 mon.c (mon.2) 33 : audit 1 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2023-06-03T19:45:35.586 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:45:35 smithi164 bash[59462]: audit 1685821534.3390431 mon.c (mon.2) 34 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-03T19:45:35.586 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:45:35 smithi164 bash[59462]: cephadm 1685821534.3418243 mgr.y (mgr.34104) 172 : cephadm 1 Reconfiguring daemon osd.3 on smithi089 2023-06-03T19:45:36.410 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 19:45:36 smithi089 bash[71163]: ::ffff:172.21.15.164 - - [03/Jun/2023:19:45:36] "GET /metrics HTTP/1.1" 200 184136 "" "Prometheus/2.33.4" 2023-06-03T19:45:37.530 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:45:37 smithi089 bash[75068]: cluster 1685821536.0637763 mgr.y (mgr.34104) 173 : cluster 0 pgmap v135: 89 pgs: 89 active+clean; 939 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 599 op/s 2023-06-03T19:45:37.530 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:45:37 smithi089 bash[75068]: audit 1685821537.0662372 mon.c (mon.2) 35 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T19:45:37.531 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:45:37 smithi089 bash[75068]: audit 1685821537.24901 mon.a (mon.0) 154 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:45:37.531 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:45:37 smithi089 bash[75068]: audit 1685821537.262848 mon.a (mon.0) 155 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:45:37.531 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:45:37 smithi089 bash[78452]: cluster 1685821536.0637763 mgr.y (mgr.34104) 173 : cluster 0 pgmap v135: 89 pgs: 89 active+clean; 939 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 599 op/s 2023-06-03T19:45:37.531 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:45:37 smithi089 bash[78452]: audit 1685821537.0662372 mon.c (mon.2) 35 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T19:45:37.531 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:45:37 smithi089 bash[78452]: audit 1685821537.24901 mon.a (mon.0) 154 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:45:37.532 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:45:37 smithi089 bash[78452]: audit 1685821537.262848 mon.a (mon.0) 155 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:45:37.585 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:45:37 smithi164 bash[59462]: cluster 1685821536.0637763 mgr.y (mgr.34104) 173 : cluster 0 pgmap v135: 89 pgs: 89 active+clean; 939 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 599 op/s 2023-06-03T19:45:37.585 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:45:37 smithi164 bash[59462]: audit 1685821537.0662372 mon.c (mon.2) 35 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T19:45:37.586 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:45:37 smithi164 bash[59462]: audit 1685821537.24901 mon.a (mon.0) 154 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:45:37.586 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:45:37 smithi164 bash[59462]: audit 1685821537.262848 mon.a (mon.0) 155 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:45:38.585 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:45:38 smithi164 bash[59462]: cephadm 1685821537.2897754 mgr.y (mgr.34104) 174 : cephadm 1 Reconfiguring mds.foo.smithi089.giuxuj (monmap changed)... 2023-06-03T19:45:38.585 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:45:38 smithi164 bash[59462]: audit 1685821537.290385 mon.c (mon.2) 36 : audit 1 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi089.giuxuj", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-06-03T19:45:38.585 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:45:38 smithi164 bash[59462]: audit 1685821537.2913063 mon.a (mon.0) 156 : audit 1 from='mgr.34104 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi089.giuxuj", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-06-03T19:45:38.586 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:45:38 smithi164 bash[59462]: audit 1685821537.2931647 mon.c (mon.2) 37 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-03T19:45:38.586 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:45:38 smithi164 bash[59462]: cephadm 1685821537.2944143 mgr.y (mgr.34104) 175 : cephadm 1 Reconfiguring daemon mds.foo.smithi089.giuxuj on smithi089 2023-06-03T19:45:38.780 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:45:38 smithi089 bash[75068]: cephadm 1685821537.2897754 mgr.y (mgr.34104) 174 : cephadm 1 Reconfiguring mds.foo.smithi089.giuxuj (monmap changed)... 2023-06-03T19:45:38.781 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:45:38 smithi089 bash[75068]: audit 1685821537.290385 mon.c (mon.2) 36 : audit 1 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi089.giuxuj", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-06-03T19:45:38.781 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:45:38 smithi089 bash[75068]: audit 1685821537.2913063 mon.a (mon.0) 156 : audit 1 from='mgr.34104 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi089.giuxuj", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-06-03T19:45:38.781 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:45:38 smithi089 bash[75068]: audit 1685821537.2931647 mon.c (mon.2) 37 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-03T19:45:38.781 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:45:38 smithi089 bash[75068]: cephadm 1685821537.2944143 mgr.y (mgr.34104) 175 : cephadm 1 Reconfiguring daemon mds.foo.smithi089.giuxuj on smithi089 2023-06-03T19:45:38.781 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:45:38 smithi089 bash[78452]: cephadm 1685821537.2897754 mgr.y (mgr.34104) 174 : cephadm 1 Reconfiguring mds.foo.smithi089.giuxuj (monmap changed)... 2023-06-03T19:45:38.782 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:45:38 smithi089 bash[78452]: audit 1685821537.290385 mon.c (mon.2) 36 : audit 1 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi089.giuxuj", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-06-03T19:45:38.782 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:45:38 smithi089 bash[78452]: audit 1685821537.2913063 mon.a (mon.0) 156 : audit 1 from='mgr.34104 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi089.giuxuj", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-06-03T19:45:38.782 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:45:38 smithi089 bash[78452]: audit 1685821537.2931647 mon.c (mon.2) 37 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-03T19:45:38.782 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:45:38 smithi089 bash[78452]: cephadm 1685821537.2944143 mgr.y (mgr.34104) 175 : cephadm 1 Reconfiguring daemon mds.foo.smithi089.giuxuj on smithi089 2023-06-03T19:45:39.530 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:45:39 smithi089 bash[75068]: cluster 1685821538.0643382 mgr.y (mgr.34104) 176 : cluster 0 pgmap v136: 89 pgs: 89 active+clean; 939 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 600 op/s 2023-06-03T19:45:39.531 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:45:39 smithi089 bash[78452]: cluster 1685821538.0643382 mgr.y (mgr.34104) 176 : cluster 0 pgmap v136: 89 pgs: 89 active+clean; 939 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 600 op/s 2023-06-03T19:45:39.585 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:45:39 smithi164 bash[59462]: cluster 1685821538.0643382 mgr.y (mgr.34104) 176 : cluster 0 pgmap v136: 89 pgs: 89 active+clean; 939 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 600 op/s 2023-06-03T19:45:41.780 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:45:41 smithi089 bash[75068]: cluster 1685821540.0649753 mgr.y (mgr.34104) 177 : cluster 0 pgmap v137: 89 pgs: 89 active+clean; 713 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 901 op/s 2023-06-03T19:45:41.780 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:45:41 smithi089 bash[75068]: audit 1685821540.3517659 mon.a (mon.0) 157 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:45:41.780 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:45:41 smithi089 bash[75068]: audit 1685821540.360252 mon.a (mon.0) 158 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:45:41.781 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:45:41 smithi089 bash[75068]: cephadm 1685821540.3727791 mgr.y (mgr.34104) 178 : cephadm 1 Reconfiguring mon.b (monmap changed)... 2023-06-03T19:45:41.781 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:45:41 smithi089 bash[75068]: audit 1685821540.3735204 mon.c (mon.2) 38 : audit 1 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-06-03T19:45:41.781 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:45:41 smithi089 bash[75068]: audit 1685821540.3753452 mon.c (mon.2) 39 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-06-03T19:45:41.781 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:45:41 smithi089 bash[75068]: audit 1685821540.3769286 mon.c (mon.2) 40 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-03T19:45:41.781 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:45:41 smithi089 bash[75068]: cephadm 1685821540.3781312 mgr.y (mgr.34104) 179 : cephadm 1 Reconfiguring daemon mon.b on smithi164 2023-06-03T19:45:41.781 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:45:41 smithi089 bash[78452]: cluster 1685821540.0649753 mgr.y (mgr.34104) 177 : cluster 0 pgmap v137: 89 pgs: 89 active+clean; 713 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 901 op/s 2023-06-03T19:45:41.782 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:45:41 smithi089 bash[78452]: audit 1685821540.3517659 mon.a (mon.0) 157 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:45:41.782 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:45:41 smithi089 bash[78452]: audit 1685821540.360252 mon.a (mon.0) 158 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:45:41.782 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:45:41 smithi089 bash[78452]: cephadm 1685821540.3727791 mgr.y (mgr.34104) 178 : cephadm 1 Reconfiguring mon.b (monmap changed)... 2023-06-03T19:45:41.782 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:45:41 smithi089 bash[78452]: audit 1685821540.3735204 mon.c (mon.2) 38 : audit 1 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-06-03T19:45:41.782 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:45:41 smithi089 bash[78452]: audit 1685821540.3753452 mon.c (mon.2) 39 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-06-03T19:45:41.782 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:45:41 smithi089 bash[78452]: audit 1685821540.3769286 mon.c (mon.2) 40 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-03T19:45:41.782 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:45:41 smithi089 bash[78452]: cephadm 1685821540.3781312 mgr.y (mgr.34104) 179 : cephadm 1 Reconfiguring daemon mon.b on smithi164 2023-06-03T19:45:41.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:45:41 smithi164 bash[59462]: cluster 1685821540.0649753 mgr.y (mgr.34104) 177 : cluster 0 pgmap v137: 89 pgs: 89 active+clean; 713 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 901 op/s 2023-06-03T19:45:41.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:45:41 smithi164 bash[59462]: audit 1685821540.3517659 mon.a (mon.0) 157 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:45:41.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:45:41 smithi164 bash[59462]: audit 1685821540.360252 mon.a (mon.0) 158 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:45:41.836 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:45:41 smithi164 bash[59462]: cephadm 1685821540.3727791 mgr.y (mgr.34104) 178 : cephadm 1 Reconfiguring mon.b (monmap changed)... 2023-06-03T19:45:41.836 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:45:41 smithi164 bash[59462]: audit 1685821540.3735204 mon.c (mon.2) 38 : audit 1 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-06-03T19:45:41.836 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:45:41 smithi164 bash[59462]: audit 1685821540.3753452 mon.c (mon.2) 39 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-06-03T19:45:41.836 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:45:41 smithi164 bash[59462]: audit 1685821540.3769286 mon.c (mon.2) 40 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-03T19:45:41.836 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:45:41 smithi164 bash[59462]: cephadm 1685821540.3781312 mgr.y (mgr.34104) 179 : cephadm 1 Reconfiguring daemon mon.b on smithi164 2023-06-03T19:45:43.780 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:45:43 smithi089 bash[78452]: cluster 1685821542.0654073 mgr.y (mgr.34104) 180 : cluster 0 pgmap v138: 89 pgs: 89 active+clean; 713 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 606 op/s 2023-06-03T19:45:43.780 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:45:43 smithi089 bash[75068]: cluster 1685821542.0654073 mgr.y (mgr.34104) 180 : cluster 0 pgmap v138: 89 pgs: 89 active+clean; 713 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 606 op/s 2023-06-03T19:45:43.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:45:43 smithi164 bash[59462]: cluster 1685821542.0654073 mgr.y (mgr.34104) 180 : cluster 0 pgmap v138: 89 pgs: 89 active+clean; 713 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 606 op/s 2023-06-03T19:45:45.717 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:45:45 smithi164 bash[59462]: cluster 1685821544.066128 mgr.y (mgr.34104) 181 : cluster 0 pgmap v139: 89 pgs: 89 active+clean; 487 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 907 op/s 2023-06-03T19:45:45.717 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:45:45 smithi164 bash[59462]: audit 1685821545.266193 mon.a (mon.0) 159 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:45:45.717 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:45:45 smithi164 bash[59462]: audit 1685821545.2792594 mon.a (mon.0) 160 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:45:45.718 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:45:45 smithi164 bash[59462]: audit 1685821545.3292835 mon.c (mon.2) 41 : audit 1 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-06-03T19:45:45.718 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:45:45 smithi164 bash[59462]: audit 1685821545.3299818 mon.a (mon.0) 161 : audit 1 from='mgr.34104 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-06-03T19:45:45.718 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:45:45 smithi164 bash[59462]: audit 1685821545.3319707 mon.c (mon.2) 42 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-06-03T19:45:45.718 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:45:45 smithi164 bash[59462]: audit 1685821545.3335545 mon.c (mon.2) 43 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-03T19:45:45.780 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:45:45 smithi089 bash[78452]: cluster 1685821544.066128 mgr.y (mgr.34104) 181 : cluster 0 pgmap v139: 89 pgs: 89 active+clean; 487 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 907 op/s 2023-06-03T19:45:45.780 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:45:45 smithi089 bash[78452]: audit 1685821545.266193 mon.a (mon.0) 159 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:45:45.780 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:45:45 smithi089 bash[78452]: audit 1685821545.2792594 mon.a (mon.0) 160 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:45:45.780 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:45:45 smithi089 bash[78452]: audit 1685821545.3292835 mon.c (mon.2) 41 : audit 1 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-06-03T19:45:45.781 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:45:45 smithi089 bash[78452]: audit 1685821545.3299818 mon.a (mon.0) 161 : audit 1 from='mgr.34104 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-06-03T19:45:45.781 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:45:45 smithi089 bash[78452]: audit 1685821545.3319707 mon.c (mon.2) 42 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-06-03T19:45:45.781 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:45:45 smithi089 bash[78452]: audit 1685821545.3335545 mon.c (mon.2) 43 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-03T19:45:45.781 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:45:45 smithi089 bash[75068]: cluster 1685821544.066128 mgr.y (mgr.34104) 181 : cluster 0 pgmap v139: 89 pgs: 89 active+clean; 487 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 907 op/s 2023-06-03T19:45:45.781 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:45:45 smithi089 bash[75068]: audit 1685821545.266193 mon.a (mon.0) 159 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:45:45.781 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:45:45 smithi089 bash[75068]: audit 1685821545.2792594 mon.a (mon.0) 160 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:45:45.782 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:45:45 smithi089 bash[75068]: audit 1685821545.3292835 mon.c (mon.2) 41 : audit 1 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-06-03T19:45:45.782 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:45:45 smithi089 bash[75068]: audit 1685821545.3299818 mon.a (mon.0) 161 : audit 1 from='mgr.34104 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-06-03T19:45:45.782 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:45:45 smithi089 bash[75068]: audit 1685821545.3319707 mon.c (mon.2) 42 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-06-03T19:45:45.782 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:45:45 smithi089 bash[75068]: audit 1685821545.3335545 mon.c (mon.2) 43 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-03T19:45:46.529 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:45:46 smithi089 bash[78452]: cephadm 1685821545.3285937 mgr.y (mgr.34104) 182 : cephadm 1 Reconfiguring mgr.x (monmap changed)... 2023-06-03T19:45:46.530 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:45:46 smithi089 bash[78452]: cephadm 1685821545.337017 mgr.y (mgr.34104) 183 : cephadm 1 Reconfiguring daemon mgr.x on smithi164 2023-06-03T19:45:46.530 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 19:45:46 smithi089 bash[71163]: ::ffff:172.21.15.164 - - [03/Jun/2023:19:45:46] "GET /metrics HTTP/1.1" 200 184141 "" "Prometheus/2.33.4" 2023-06-03T19:45:46.530 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:45:46 smithi089 bash[75068]: cephadm 1685821545.3285937 mgr.y (mgr.34104) 182 : cephadm 1 Reconfiguring mgr.x (monmap changed)... 2023-06-03T19:45:46.531 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:45:46 smithi089 bash[75068]: cephadm 1685821545.337017 mgr.y (mgr.34104) 183 : cephadm 1 Reconfiguring daemon mgr.x on smithi164 2023-06-03T19:45:46.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:45:46 smithi164 bash[59462]: cephadm 1685821545.3285937 mgr.y (mgr.34104) 182 : cephadm 1 Reconfiguring mgr.x (monmap changed)... 2023-06-03T19:45:46.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:45:46 smithi164 bash[59462]: cephadm 1685821545.337017 mgr.y (mgr.34104) 183 : cephadm 1 Reconfiguring daemon mgr.x on smithi164 2023-06-03T19:45:47.585 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:45:47 smithi164 bash[59462]: cluster 1685821546.066608 mgr.y (mgr.34104) 184 : cluster 0 pgmap v140: 89 pgs: 89 active+clean; 487 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 602 op/s 2023-06-03T19:45:47.779 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:45:47 smithi089 bash[78452]: cluster 1685821546.066608 mgr.y (mgr.34104) 184 : cluster 0 pgmap v140: 89 pgs: 89 active+clean; 487 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 602 op/s 2023-06-03T19:45:47.780 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:45:47 smithi089 bash[75068]: cluster 1685821546.066608 mgr.y (mgr.34104) 184 : cluster 0 pgmap v140: 89 pgs: 89 active+clean; 487 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 602 op/s 2023-06-03T19:45:49.335 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:45:49 smithi164 bash[59462]: audit 1685821548.0593257 mon.a (mon.0) 162 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:45:49.335 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:45:49 smithi164 bash[59462]: cluster 1685821548.0670319 mgr.y (mgr.34104) 185 : cluster 0 pgmap v141: 89 pgs: 89 active+clean; 487 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 602 op/s 2023-06-03T19:45:49.335 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:45:49 smithi164 bash[59462]: audit 1685821548.0750184 mon.a (mon.0) 163 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:45:49.336 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:45:49 smithi164 bash[59462]: cephadm 1685821548.114962 mgr.y (mgr.34104) 186 : cephadm 1 Reconfiguring osd.4 (monmap changed)... 2023-06-03T19:45:49.336 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:45:49 smithi164 bash[59462]: audit 1685821548.1154296 mon.c (mon.2) 44 : audit 1 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2023-06-03T19:45:49.336 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:45:49 smithi164 bash[59462]: audit 1685821548.1165738 mon.c (mon.2) 45 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-03T19:45:49.336 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:45:49 smithi164 bash[59462]: cephadm 1685821548.1197855 mgr.y (mgr.34104) 187 : cephadm 1 Reconfiguring daemon osd.4 on smithi164 2023-06-03T19:45:49.530 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:45:49 smithi089 bash[78452]: audit 1685821548.0593257 mon.a (mon.0) 162 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:45:49.530 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:45:49 smithi089 bash[78452]: cluster 1685821548.0670319 mgr.y (mgr.34104) 185 : cluster 0 pgmap v141: 89 pgs: 89 active+clean; 487 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 602 op/s 2023-06-03T19:45:49.530 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:45:49 smithi089 bash[78452]: audit 1685821548.0750184 mon.a (mon.0) 163 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:45:49.530 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:45:49 smithi089 bash[78452]: cephadm 1685821548.114962 mgr.y (mgr.34104) 186 : cephadm 1 Reconfiguring osd.4 (monmap changed)... 2023-06-03T19:45:49.530 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:45:49 smithi089 bash[78452]: audit 1685821548.1154296 mon.c (mon.2) 44 : audit 1 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2023-06-03T19:45:49.531 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:45:49 smithi089 bash[78452]: audit 1685821548.1165738 mon.c (mon.2) 45 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-03T19:45:49.531 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:45:49 smithi089 bash[78452]: cephadm 1685821548.1197855 mgr.y (mgr.34104) 187 : cephadm 1 Reconfiguring daemon osd.4 on smithi164 2023-06-03T19:45:49.531 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:45:49 smithi089 bash[75068]: audit 1685821548.0593257 mon.a (mon.0) 162 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:45:49.531 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:45:49 smithi089 bash[75068]: cluster 1685821548.0670319 mgr.y (mgr.34104) 185 : cluster 0 pgmap v141: 89 pgs: 89 active+clean; 487 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 602 op/s 2023-06-03T19:45:49.531 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:45:49 smithi089 bash[75068]: audit 1685821548.0750184 mon.a (mon.0) 163 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:45:49.531 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:45:49 smithi089 bash[75068]: cephadm 1685821548.114962 mgr.y (mgr.34104) 186 : cephadm 1 Reconfiguring osd.4 (monmap changed)... 2023-06-03T19:45:49.532 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:45:49 smithi089 bash[75068]: audit 1685821548.1154296 mon.c (mon.2) 44 : audit 1 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2023-06-03T19:45:49.532 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:45:49 smithi089 bash[75068]: audit 1685821548.1165738 mon.c (mon.2) 45 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-03T19:45:49.532 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:45:49 smithi089 bash[75068]: cephadm 1685821548.1197855 mgr.y (mgr.34104) 187 : cephadm 1 Reconfiguring daemon osd.4 on smithi164 2023-06-03T19:45:51.493 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:45:51 smithi164 bash[59462]: cluster 1685821550.0677698 mgr.y (mgr.34104) 188 : cluster 0 pgmap v142: 89 pgs: 89 active+clean; 269 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 893 op/s 2023-06-03T19:45:51.493 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:45:51 smithi164 bash[59462]: audit 1685821551.069194 mon.a (mon.0) 164 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:45:51.493 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:45:51 smithi164 bash[59462]: audit 1685821551.0822887 mon.a (mon.0) 165 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:45:51.493 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:45:51 smithi164 bash[59462]: audit 1685821551.097042 mon.c (mon.2) 46 : audit 1 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2023-06-03T19:45:51.493 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:45:51 smithi164 bash[59462]: audit 1685821551.1050475 mon.c (mon.2) 47 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-03T19:45:51.530 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:45:51 smithi089 bash[75068]: cluster 1685821550.0677698 mgr.y (mgr.34104) 188 : cluster 0 pgmap v142: 89 pgs: 89 active+clean; 269 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 893 op/s 2023-06-03T19:45:51.530 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:45:51 smithi089 bash[75068]: audit 1685821551.069194 mon.a (mon.0) 164 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:45:51.530 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:45:51 smithi089 bash[75068]: audit 1685821551.0822887 mon.a (mon.0) 165 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:45:51.530 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:45:51 smithi089 bash[75068]: audit 1685821551.097042 mon.c (mon.2) 46 : audit 1 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2023-06-03T19:45:51.531 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:45:51 smithi089 bash[75068]: audit 1685821551.1050475 mon.c (mon.2) 47 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-03T19:45:51.531 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:45:51 smithi089 bash[78452]: cluster 1685821550.0677698 mgr.y (mgr.34104) 188 : cluster 0 pgmap v142: 89 pgs: 89 active+clean; 269 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 893 op/s 2023-06-03T19:45:51.531 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:45:51 smithi089 bash[78452]: audit 1685821551.069194 mon.a (mon.0) 164 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:45:51.531 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:45:51 smithi089 bash[78452]: audit 1685821551.0822887 mon.a (mon.0) 165 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:45:51.531 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:45:51 smithi089 bash[78452]: audit 1685821551.097042 mon.c (mon.2) 46 : audit 1 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2023-06-03T19:45:51.531 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:45:51 smithi089 bash[78452]: audit 1685821551.1050475 mon.c (mon.2) 47 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-03T19:45:52.530 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:45:52 smithi089 bash[75068]: cephadm 1685821551.096442 mgr.y (mgr.34104) 189 : cephadm 1 Reconfiguring osd.5 (monmap changed)... 2023-06-03T19:45:52.530 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:45:52 smithi089 bash[75068]: cephadm 1685821551.1091695 mgr.y (mgr.34104) 190 : cephadm 1 Reconfiguring daemon osd.5 on smithi164 2023-06-03T19:45:52.530 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:45:52 smithi089 bash[75068]: audit 1685821552.066358 mon.c (mon.2) 48 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T19:45:52.531 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:45:52 smithi089 bash[78452]: cephadm 1685821551.096442 mgr.y (mgr.34104) 189 : cephadm 1 Reconfiguring osd.5 (monmap changed)... 2023-06-03T19:45:52.531 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:45:52 smithi089 bash[78452]: cephadm 1685821551.1091695 mgr.y (mgr.34104) 190 : cephadm 1 Reconfiguring daemon osd.5 on smithi164 2023-06-03T19:45:52.531 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:45:52 smithi089 bash[78452]: audit 1685821552.066358 mon.c (mon.2) 48 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T19:45:52.585 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:45:52 smithi164 bash[59462]: cephadm 1685821551.096442 mgr.y (mgr.34104) 189 : cephadm 1 Reconfiguring osd.5 (monmap changed)... 2023-06-03T19:45:52.585 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:45:52 smithi164 bash[59462]: cephadm 1685821551.1091695 mgr.y (mgr.34104) 190 : cephadm 1 Reconfiguring daemon osd.5 on smithi164 2023-06-03T19:45:52.585 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:45:52 smithi164 bash[59462]: audit 1685821552.066358 mon.c (mon.2) 48 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T19:45:53.335 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:45:53 smithi164 bash[59462]: cluster 1685821552.0681689 mgr.y (mgr.34104) 191 : cluster 0 pgmap v143: 89 pgs: 89 active+clean; 269 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 591 op/s 2023-06-03T19:45:53.530 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:45:53 smithi089 bash[75068]: cluster 1685821552.0681689 mgr.y (mgr.34104) 191 : cluster 0 pgmap v143: 89 pgs: 89 active+clean; 269 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 591 op/s 2023-06-03T19:45:53.530 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:45:53 smithi089 bash[78452]: cluster 1685821552.0681689 mgr.y (mgr.34104) 191 : cluster 0 pgmap v143: 89 pgs: 89 active+clean; 269 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 591 op/s 2023-06-03T19:45:54.422 INFO:tasks.radosbench.radosbench.0.smithi089.stdout:Removed 114272 objects 2023-06-03T19:45:54.422 INFO:tasks.radosbench.radosbench.0.smithi089.stdout:Clean up completed and total clean up time :156.033 2023-06-03T19:45:55.280 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:45:54 smithi089 bash[78452]: audit 1685821553.8843706 mon.a (mon.0) 166 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:45:55.280 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:45:54 smithi089 bash[78452]: audit 1685821553.904009 mon.a (mon.0) 167 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:45:55.280 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:45:54 smithi089 bash[78452]: cephadm 1685821553.9417014 mgr.y (mgr.34104) 192 : cephadm 1 Reconfiguring osd.6 (monmap changed)... 2023-06-03T19:45:55.280 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:45:54 smithi089 bash[78452]: audit 1685821553.9421246 mon.c (mon.2) 49 : audit 1 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2023-06-03T19:45:55.281 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:45:54 smithi089 bash[78452]: audit 1685821553.9439182 mon.c (mon.2) 50 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-03T19:45:55.281 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:45:54 smithi089 bash[78452]: cephadm 1685821553.947994 mgr.y (mgr.34104) 193 : cephadm 1 Reconfiguring daemon osd.6 on smithi164 2023-06-03T19:45:55.281 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:45:54 smithi089 bash[78452]: cluster 1685821554.0689485 mgr.y (mgr.34104) 194 : cluster 0 pgmap v144: 89 pgs: 89 active+clean; 49 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 0 B/s wr, 884 op/s 2023-06-03T19:45:55.282 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:45:54 smithi089 bash[75068]: audit 1685821553.8843706 mon.a (mon.0) 166 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:45:55.282 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:45:54 smithi089 bash[75068]: audit 1685821553.904009 mon.a (mon.0) 167 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:45:55.282 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:45:54 smithi089 bash[75068]: cephadm 1685821553.9417014 mgr.y (mgr.34104) 192 : cephadm 1 Reconfiguring osd.6 (monmap changed)... 2023-06-03T19:45:55.282 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:45:54 smithi089 bash[75068]: audit 1685821553.9421246 mon.c (mon.2) 49 : audit 1 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2023-06-03T19:45:55.282 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:45:54 smithi089 bash[75068]: audit 1685821553.9439182 mon.c (mon.2) 50 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-03T19:45:55.283 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:45:54 smithi089 bash[75068]: cephadm 1685821553.947994 mgr.y (mgr.34104) 193 : cephadm 1 Reconfiguring daemon osd.6 on smithi164 2023-06-03T19:45:55.283 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:45:54 smithi089 bash[75068]: cluster 1685821554.0689485 mgr.y (mgr.34104) 194 : cluster 0 pgmap v144: 89 pgs: 89 active+clean; 49 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 0 B/s wr, 884 op/s 2023-06-03T19:45:55.335 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:45:54 smithi164 bash[59462]: audit 1685821553.8843706 mon.a (mon.0) 166 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:45:55.335 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:45:54 smithi164 bash[59462]: audit 1685821553.904009 mon.a (mon.0) 167 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:45:55.336 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:45:54 smithi164 bash[59462]: cephadm 1685821553.9417014 mgr.y (mgr.34104) 192 : cephadm 1 Reconfiguring osd.6 (monmap changed)... 2023-06-03T19:45:55.336 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:45:54 smithi164 bash[59462]: audit 1685821553.9421246 mon.c (mon.2) 49 : audit 1 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2023-06-03T19:45:55.336 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:45:54 smithi164 bash[59462]: audit 1685821553.9439182 mon.c (mon.2) 50 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-03T19:45:55.336 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:45:54 smithi164 bash[59462]: cephadm 1685821553.947994 mgr.y (mgr.34104) 193 : cephadm 1 Reconfiguring daemon osd.6 on smithi164 2023-06-03T19:45:55.336 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:45:54 smithi164 bash[59462]: cluster 1685821554.0689485 mgr.y (mgr.34104) 194 : cluster 0 pgmap v144: 89 pgs: 89 active+clean; 49 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 0 B/s wr, 884 op/s 2023-06-03T19:45:56.530 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 19:45:56 smithi089 bash[71163]: ::ffff:172.21.15.164 - - [03/Jun/2023:19:45:56] "GET /metrics HTTP/1.1" 200 184151 "" "Prometheus/2.33.4" 2023-06-03T19:45:57.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:45:57 smithi164 bash[59462]: cluster 1685821556.0695822 mgr.y (mgr.34104) 195 : cluster 0 pgmap v145: 89 pgs: 89 active+clean; 49 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 0 B/s wr, 583 op/s 2023-06-03T19:45:57.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:45:57 smithi164 bash[59462]: audit 1685821556.579134 mon.a (mon.0) 168 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:45:57.836 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:45:57 smithi164 bash[59462]: audit 1685821556.590166 mon.a (mon.0) 169 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:45:57.836 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:45:57 smithi164 bash[59462]: cephadm 1685821556.6184633 mgr.y (mgr.34104) 196 : cephadm 1 Reconfiguring osd.7 (monmap changed)... 2023-06-03T19:45:57.836 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:45:57 smithi164 bash[59462]: audit 1685821556.6190665 mon.c (mon.2) 51 : audit 1 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2023-06-03T19:45:57.836 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:45:57 smithi164 bash[59462]: audit 1685821556.620623 mon.c (mon.2) 52 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-03T19:45:57.836 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:45:57 smithi164 bash[59462]: cephadm 1685821556.6248138 mgr.y (mgr.34104) 197 : cephadm 1 Reconfiguring daemon osd.7 on smithi164 2023-06-03T19:45:58.030 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:45:57 smithi089 bash[75068]: cluster 1685821556.0695822 mgr.y (mgr.34104) 195 : cluster 0 pgmap v145: 89 pgs: 89 active+clean; 49 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 0 B/s wr, 583 op/s 2023-06-03T19:45:58.031 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:45:57 smithi089 bash[75068]: audit 1685821556.579134 mon.a (mon.0) 168 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:45:58.031 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:45:57 smithi089 bash[75068]: audit 1685821556.590166 mon.a (mon.0) 169 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:45:58.031 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:45:57 smithi089 bash[75068]: cephadm 1685821556.6184633 mgr.y (mgr.34104) 196 : cephadm 1 Reconfiguring osd.7 (monmap changed)... 2023-06-03T19:45:58.031 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:45:57 smithi089 bash[75068]: audit 1685821556.6190665 mon.c (mon.2) 51 : audit 1 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2023-06-03T19:45:58.031 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:45:57 smithi089 bash[75068]: audit 1685821556.620623 mon.c (mon.2) 52 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-03T19:45:58.031 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:45:57 smithi089 bash[75068]: cephadm 1685821556.6248138 mgr.y (mgr.34104) 197 : cephadm 1 Reconfiguring daemon osd.7 on smithi164 2023-06-03T19:45:58.032 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:45:57 smithi089 bash[78452]: cluster 1685821556.0695822 mgr.y (mgr.34104) 195 : cluster 0 pgmap v145: 89 pgs: 89 active+clean; 49 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 0 B/s wr, 583 op/s 2023-06-03T19:45:58.032 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:45:57 smithi089 bash[78452]: audit 1685821556.579134 mon.a (mon.0) 168 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:45:58.032 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:45:57 smithi089 bash[78452]: audit 1685821556.590166 mon.a (mon.0) 169 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:45:58.032 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:45:57 smithi089 bash[78452]: cephadm 1685821556.6184633 mgr.y (mgr.34104) 196 : cephadm 1 Reconfiguring osd.7 (monmap changed)... 2023-06-03T19:45:58.033 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:45:57 smithi089 bash[78452]: audit 1685821556.6190665 mon.c (mon.2) 51 : audit 1 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2023-06-03T19:45:58.033 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:45:57 smithi089 bash[78452]: audit 1685821556.620623 mon.c (mon.2) 52 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-03T19:45:58.033 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:45:57 smithi089 bash[78452]: cephadm 1685821556.6248138 mgr.y (mgr.34104) 197 : cephadm 1 Reconfiguring daemon osd.7 on smithi164 2023-06-03T19:45:58.479 INFO:tasks.cephadm.ceph_manager.ceph:removing pool_name unique_pool_1 2023-06-03T19:45:58.480 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph osd pool rm unique_pool_1 unique_pool_1 --yes-i-really-really-mean-it 2023-06-03T19:46:00.030 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:45:59 smithi089 bash[75068]: cluster 1685821558.0702074 mgr.y (mgr.34104) 198 : cluster 0 pgmap v146: 89 pgs: 89 active+clean; 49 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 0 B/s wr, 583 op/s 2023-06-03T19:46:00.030 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:45:59 smithi089 bash[75068]: audit 1685821558.9297016 mon.a (mon.0) 170 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:46:00.031 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:45:59 smithi089 bash[75068]: audit 1685821558.94439 mon.a (mon.0) 171 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:46:00.031 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:45:59 smithi089 bash[75068]: audit 1685821558.9961882 mon.c (mon.2) 53 : audit 1 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi164.txgfgv", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-06-03T19:46:00.031 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:45:59 smithi089 bash[75068]: audit 1685821558.9969957 mon.a (mon.0) 172 : audit 1 from='mgr.34104 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi164.txgfgv", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-06-03T19:46:00.031 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:45:59 smithi089 bash[75068]: audit 1685821558.999318 mon.c (mon.2) 54 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-03T19:46:00.031 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:45:59 smithi089 bash[78452]: cluster 1685821558.0702074 mgr.y (mgr.34104) 198 : cluster 0 pgmap v146: 89 pgs: 89 active+clean; 49 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 0 B/s wr, 583 op/s 2023-06-03T19:46:00.031 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:45:59 smithi089 bash[78452]: audit 1685821558.9297016 mon.a (mon.0) 170 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:46:00.032 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:45:59 smithi089 bash[78452]: audit 1685821558.94439 mon.a (mon.0) 171 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:46:00.032 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:45:59 smithi089 bash[78452]: audit 1685821558.9961882 mon.c (mon.2) 53 : audit 1 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi164.txgfgv", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-06-03T19:46:00.032 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:45:59 smithi089 bash[78452]: audit 1685821558.9969957 mon.a (mon.0) 172 : audit 1 from='mgr.34104 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi164.txgfgv", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-06-03T19:46:00.032 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:45:59 smithi089 bash[78452]: audit 1685821558.999318 mon.c (mon.2) 54 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-03T19:46:00.085 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:45:59 smithi164 bash[59462]: cluster 1685821558.0702074 mgr.y (mgr.34104) 198 : cluster 0 pgmap v146: 89 pgs: 89 active+clean; 49 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 0 B/s wr, 583 op/s 2023-06-03T19:46:00.086 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:45:59 smithi164 bash[59462]: audit 1685821558.9297016 mon.a (mon.0) 170 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:46:00.086 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:45:59 smithi164 bash[59462]: audit 1685821558.94439 mon.a (mon.0) 171 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:46:00.086 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:45:59 smithi164 bash[59462]: audit 1685821558.9961882 mon.c (mon.2) 53 : audit 1 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi164.txgfgv", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-06-03T19:46:00.086 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:45:59 smithi164 bash[59462]: audit 1685821558.9969957 mon.a (mon.0) 172 : audit 1 from='mgr.34104 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi164.txgfgv", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-06-03T19:46:00.086 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:45:59 smithi164 bash[59462]: audit 1685821558.999318 mon.c (mon.2) 54 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-03T19:46:01.030 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:46:00 smithi089 bash[75068]: cephadm 1685821558.9954255 mgr.y (mgr.34104) 199 : cephadm 1 Reconfiguring mds.foo.smithi164.txgfgv (monmap changed)... 2023-06-03T19:46:01.030 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:46:00 smithi089 bash[75068]: cephadm 1685821559.0010974 mgr.y (mgr.34104) 200 : cephadm 1 Reconfiguring daemon mds.foo.smithi164.txgfgv on smithi164 2023-06-03T19:46:01.031 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:46:00 smithi089 bash[78452]: cephadm 1685821558.9954255 mgr.y (mgr.34104) 199 : cephadm 1 Reconfiguring mds.foo.smithi164.txgfgv (monmap changed)... 2023-06-03T19:46:01.031 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:46:00 smithi089 bash[78452]: cephadm 1685821559.0010974 mgr.y (mgr.34104) 200 : cephadm 1 Reconfiguring daemon mds.foo.smithi164.txgfgv on smithi164 2023-06-03T19:46:01.085 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:46:00 smithi164 bash[59462]: cephadm 1685821558.9954255 mgr.y (mgr.34104) 199 : cephadm 1 Reconfiguring mds.foo.smithi164.txgfgv (monmap changed)... 2023-06-03T19:46:01.086 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:46:00 smithi164 bash[59462]: cephadm 1685821559.0010974 mgr.y (mgr.34104) 200 : cephadm 1 Reconfiguring daemon mds.foo.smithi164.txgfgv on smithi164 2023-06-03T19:46:02.030 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:46:01 smithi089 bash[75068]: cluster 1685821560.0713115 mgr.y (mgr.34104) 201 : cluster 0 pgmap v147: 89 pgs: 89 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 0 B/s wr, 648 op/s 2023-06-03T19:46:02.030 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:46:01 smithi089 bash[75068]: audit 1685821561.2356272 mon.a (mon.0) 173 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:46:02.030 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:46:01 smithi089 bash[75068]: audit 1685821561.2478135 mon.a (mon.0) 174 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:46:02.030 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:46:01 smithi089 bash[75068]: audit 1685821561.280577 mon.c (mon.2) 55 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-03T19:46:02.031 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:46:01 smithi089 bash[78452]: cluster 1685821560.0713115 mgr.y (mgr.34104) 201 : cluster 0 pgmap v147: 89 pgs: 89 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 0 B/s wr, 648 op/s 2023-06-03T19:46:02.031 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:46:01 smithi089 bash[78452]: audit 1685821561.2356272 mon.a (mon.0) 173 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:46:02.031 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:46:01 smithi089 bash[78452]: audit 1685821561.2478135 mon.a (mon.0) 174 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:46:02.031 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:46:01 smithi089 bash[78452]: audit 1685821561.280577 mon.c (mon.2) 55 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-03T19:46:02.085 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:46:01 smithi164 bash[59462]: cluster 1685821560.0713115 mgr.y (mgr.34104) 201 : cluster 0 pgmap v147: 89 pgs: 89 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 0 B/s wr, 648 op/s 2023-06-03T19:46:02.086 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:46:01 smithi164 bash[59462]: audit 1685821561.2356272 mon.a (mon.0) 173 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:46:02.086 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:46:01 smithi164 bash[59462]: audit 1685821561.2478135 mon.a (mon.0) 174 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:46:02.086 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:46:01 smithi164 bash[59462]: audit 1685821561.280577 mon.c (mon.2) 55 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-03T19:46:03.301 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/mon.a/config 2023-06-03T19:46:04.030 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:46:03 smithi089 bash[78452]: cluster 1685821562.071953 mgr.y (mgr.34104) 202 : cluster 0 pgmap v148: 89 pgs: 89 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 0 B/s wr, 357 op/s 2023-06-03T19:46:04.030 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:46:03 smithi089 bash[75068]: cluster 1685821562.071953 mgr.y (mgr.34104) 202 : cluster 0 pgmap v148: 89 pgs: 89 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 0 B/s wr, 357 op/s 2023-06-03T19:46:04.085 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:46:03 smithi164 bash[59462]: cluster 1685821562.071953 mgr.y (mgr.34104) 202 : cluster 0 pgmap v148: 89 pgs: 89 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 0 B/s wr, 357 op/s 2023-06-03T19:46:05.860 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:46:05 smithi089 bash[78452]: cluster 1685821564.0730865 mgr.y (mgr.34104) 203 : cluster 0 pgmap v149: 89 pgs: 89 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 0 B/s wr, 357 op/s 2023-06-03T19:46:05.861 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:46:05 smithi089 bash[75068]: cluster 1685821564.0730865 mgr.y (mgr.34104) 203 : cluster 0 pgmap v149: 89 pgs: 89 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 0 B/s wr, 357 op/s 2023-06-03T19:46:06.085 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:46:05 smithi164 bash[59462]: cluster 1685821564.0730865 mgr.y (mgr.34104) 203 : cluster 0 pgmap v149: 89 pgs: 89 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 0 B/s wr, 357 op/s 2023-06-03T19:46:06.530 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 19:46:06 smithi089 bash[71163]: ::ffff:172.21.15.164 - - [03/Jun/2023:19:46:06] "GET /metrics HTTP/1.1" 200 184151 "" "Prometheus/2.33.4" 2023-06-03T19:46:07.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:46:07 smithi164 bash[59462]: cluster 1685821566.0736382 mgr.y (mgr.34104) 204 : cluster 0 pgmap v150: 89 pgs: 89 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 0 B/s wr, 64 op/s 2023-06-03T19:46:07.836 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:46:07 smithi164 bash[59462]: audit 1685821566.525769 mon.a (mon.0) 175 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:46:07.836 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:46:07 smithi164 bash[59462]: audit 1685821566.5357013 mon.a (mon.0) 176 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:46:07.836 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:46:07 smithi164 bash[59462]: audit 1685821567.0081172 mon.c (mon.2) 56 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-03T19:46:07.836 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:46:07 smithi164 bash[59462]: audit 1685821567.0096574 mon.c (mon.2) 57 : audit 1 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-03T19:46:07.836 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:46:07 smithi164 bash[59462]: audit 1685821567.0173519 mon.a (mon.0) 177 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:46:07.836 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:46:07 smithi164 bash[59462]: audit 1685821567.0664828 mon.c (mon.2) 58 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T19:46:07.853 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:46:07 smithi089 bash[78452]: cluster 1685821566.0736382 mgr.y (mgr.34104) 204 : cluster 0 pgmap v150: 89 pgs: 89 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 0 B/s wr, 64 op/s 2023-06-03T19:46:07.853 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:46:07 smithi089 bash[78452]: audit 1685821566.525769 mon.a (mon.0) 175 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:46:07.854 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:46:07 smithi089 bash[78452]: audit 1685821566.5357013 mon.a (mon.0) 176 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:46:07.854 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:46:07 smithi089 bash[78452]: audit 1685821567.0081172 mon.c (mon.2) 56 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-03T19:46:07.854 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:46:07 smithi089 bash[78452]: audit 1685821567.0096574 mon.c (mon.2) 57 : audit 1 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-03T19:46:07.854 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:46:07 smithi089 bash[78452]: audit 1685821567.0173519 mon.a (mon.0) 177 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:46:07.854 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:46:07 smithi089 bash[78452]: audit 1685821567.0664828 mon.c (mon.2) 58 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T19:46:07.854 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:46:07 smithi089 bash[75068]: cluster 1685821566.0736382 mgr.y (mgr.34104) 204 : cluster 0 pgmap v150: 89 pgs: 89 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 0 B/s wr, 64 op/s 2023-06-03T19:46:07.855 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:46:07 smithi089 bash[75068]: audit 1685821566.525769 mon.a (mon.0) 175 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:46:07.855 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:46:07 smithi089 bash[75068]: audit 1685821566.5357013 mon.a (mon.0) 176 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:46:07.855 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:46:07 smithi089 bash[75068]: audit 1685821567.0081172 mon.c (mon.2) 56 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-03T19:46:07.855 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:46:07 smithi089 bash[75068]: audit 1685821567.0096574 mon.c (mon.2) 57 : audit 1 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-03T19:46:07.855 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:46:07 smithi089 bash[75068]: audit 1685821567.0173519 mon.a (mon.0) 177 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:46:07.855 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:46:07 smithi089 bash[75068]: audit 1685821567.0664828 mon.c (mon.2) 58 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T19:46:08.555 INFO:teuthology.orchestra.run.smithi089.stderr:pool 'unique_pool_1' removed 2023-06-03T19:46:08.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:46:08 smithi164 bash[59462]: audit 1685821568.3056214 mon.c (mon.2) 59 : audit 1 from='client.? 172.21.15.89:0/2450499050' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_1", "pool2": "unique_pool_1", "yes_i_really_really_mean_it": true}]: dispatch 2023-06-03T19:46:08.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:46:08 smithi164 bash[59462]: audit 1685821568.306384 mon.a (mon.0) 178 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_1", "pool2": "unique_pool_1", "yes_i_really_really_mean_it": true}]: dispatch 2023-06-03T19:46:09.030 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:46:08 smithi089 bash[75068]: audit 1685821568.3056214 mon.c (mon.2) 59 : audit 1 from='client.? 172.21.15.89:0/2450499050' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_1", "pool2": "unique_pool_1", "yes_i_really_really_mean_it": true}]: dispatch 2023-06-03T19:46:09.030 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:46:08 smithi089 bash[75068]: audit 1685821568.306384 mon.a (mon.0) 178 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_1", "pool2": "unique_pool_1", "yes_i_really_really_mean_it": true}]: dispatch 2023-06-03T19:46:09.030 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:46:08 smithi089 bash[78452]: audit 1685821568.3056214 mon.c (mon.2) 59 : audit 1 from='client.? 172.21.15.89:0/2450499050' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_1", "pool2": "unique_pool_1", "yes_i_really_really_mean_it": true}]: dispatch 2023-06-03T19:46:09.030 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:46:08 smithi089 bash[78452]: audit 1685821568.306384 mon.a (mon.0) 178 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_1", "pool2": "unique_pool_1", "yes_i_really_really_mean_it": true}]: dispatch 2023-06-03T19:46:09.386 INFO:teuthology.task.full_sequential:In full_sequential, running task radosbench... 2023-06-03T19:46:09.387 INFO:tasks.radosbench:Beginning radosbench... 2023-06-03T19:46:09.388 INFO:tasks.cephadm.ceph_manager.ceph:creating pool_name unique_pool_2 2023-06-03T19:46:09.388 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph osd pool create unique_pool_2 16 2023-06-03T19:46:09.780 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:46:09 smithi089 bash[78452]: cluster 1685821568.0742583 mgr.y (mgr.34104) 205 : cluster 0 pgmap v151: 89 pgs: 89 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 0 B/s wr, 64 op/s 2023-06-03T19:46:09.780 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:46:09 smithi089 bash[78452]: cluster 1685821568.5328577 osd.7 (osd.7) 4 : cluster 3 failed to encode map e78 with expected crc 2023-06-03T19:46:09.780 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:46:09 smithi089 bash[78452]: cluster 1685821568.5358536 osd.2 (osd.2) 6 : cluster 3 failed to encode map e78 with expected crc 2023-06-03T19:46:09.780 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:46:09 smithi089 bash[78452]: audit 1685821568.5372217 mon.a (mon.0) 179 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_1", "pool2": "unique_pool_1", "yes_i_really_really_mean_it": true}]': finished 2023-06-03T19:46:09.781 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:46:09 smithi089 bash[78452]: cluster 1685821568.5373406 mon.a (mon.0) 180 : cluster 0 osdmap e78: 8 total, 8 up, 8 in 2023-06-03T19:46:09.781 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:46:09 smithi089 bash[75068]: cluster 1685821568.0742583 mgr.y (mgr.34104) 205 : cluster 0 pgmap v151: 89 pgs: 89 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 0 B/s wr, 64 op/s 2023-06-03T19:46:09.781 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:46:09 smithi089 bash[75068]: cluster 1685821568.5328577 osd.7 (osd.7) 4 : cluster 3 failed to encode map e78 with expected crc 2023-06-03T19:46:09.781 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:46:09 smithi089 bash[75068]: cluster 1685821568.5358536 osd.2 (osd.2) 6 : cluster 3 failed to encode map e78 with expected crc 2023-06-03T19:46:09.781 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:46:09 smithi089 bash[75068]: audit 1685821568.5372217 mon.a (mon.0) 179 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_1", "pool2": "unique_pool_1", "yes_i_really_really_mean_it": true}]': finished 2023-06-03T19:46:09.781 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:46:09 smithi089 bash[75068]: cluster 1685821568.5373406 mon.a (mon.0) 180 : cluster 0 osdmap e78: 8 total, 8 up, 8 in 2023-06-03T19:46:09.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:46:09 smithi164 bash[59462]: cluster 1685821568.0742583 mgr.y (mgr.34104) 205 : cluster 0 pgmap v151: 89 pgs: 89 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 0 B/s wr, 64 op/s 2023-06-03T19:46:09.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:46:09 smithi164 bash[59462]: cluster 1685821568.5328577 osd.7 (osd.7) 4 : cluster 3 failed to encode map e78 with expected crc 2023-06-03T19:46:09.836 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:46:09 smithi164 bash[59462]: cluster 1685821568.5358536 osd.2 (osd.2) 6 : cluster 3 failed to encode map e78 with expected crc 2023-06-03T19:46:09.836 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:46:09 smithi164 bash[59462]: audit 1685821568.5372217 mon.a (mon.0) 179 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_1", "pool2": "unique_pool_1", "yes_i_really_really_mean_it": true}]': finished 2023-06-03T19:46:09.836 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:46:09 smithi164 bash[59462]: cluster 1685821568.5373406 mon.a (mon.0) 180 : cluster 0 osdmap e78: 8 total, 8 up, 8 in 2023-06-03T19:46:11.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:46:11 smithi164 bash[59462]: cluster 1685821570.0750153 mgr.y (mgr.34104) 206 : cluster 0 pgmap v153: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2023-06-03T19:46:12.030 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:46:11 smithi089 bash[78452]: cluster 1685821570.0750153 mgr.y (mgr.34104) 206 : cluster 0 pgmap v153: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2023-06-03T19:46:12.030 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:46:11 smithi089 bash[75068]: cluster 1685821570.0750153 mgr.y (mgr.34104) 206 : cluster 0 pgmap v153: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2023-06-03T19:46:13.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:46:13 smithi164 bash[59462]: cluster 1685821572.0755777 mgr.y (mgr.34104) 207 : cluster 0 pgmap v154: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2023-06-03T19:46:13.888 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:46:13 smithi089 bash[78452]: cluster 1685821572.0755777 mgr.y (mgr.34104) 207 : cluster 0 pgmap v154: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2023-06-03T19:46:13.889 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:46:13 smithi089 bash[75068]: cluster 1685821572.0755777 mgr.y (mgr.34104) 207 : cluster 0 pgmap v154: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2023-06-03T19:46:13.890 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -e sha1=8e24bcf462a1d94c2e12aa4e26419634883a1e88 -- bash -c 'ceph orch upgrade resume' 2023-06-03T19:46:14.381 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/mon.a/config 2023-06-03T19:46:15.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:46:15 smithi164 bash[59462]: cluster 1685821574.0762053 mgr.y (mgr.34104) 208 : cluster 0 pgmap v155: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-06-03T19:46:16.024 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:46:15 smithi089 bash[78452]: cluster 1685821574.0762053 mgr.y (mgr.34104) 208 : cluster 0 pgmap v155: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-06-03T19:46:16.024 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:46:15 smithi089 bash[75068]: cluster 1685821574.0762053 mgr.y (mgr.34104) 208 : cluster 0 pgmap v155: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-06-03T19:46:16.276 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 19:46:16 smithi089 bash[71163]: ::ffff:172.21.15.164 - - [03/Jun/2023:19:46:16] "GET /metrics HTTP/1.1" 200 184099 "" "Prometheus/2.33.4" 2023-06-03T19:46:17.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:46:17 smithi164 bash[59462]: cluster 1685821576.0769591 mgr.y (mgr.34104) 209 : cluster 0 pgmap v156: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-06-03T19:46:18.030 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:46:17 smithi089 bash[78452]: cluster 1685821576.0769591 mgr.y (mgr.34104) 209 : cluster 0 pgmap v156: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-06-03T19:46:18.030 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:46:17 smithi089 bash[75068]: cluster 1685821576.0769591 mgr.y (mgr.34104) 209 : cluster 0 pgmap v156: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-06-03T19:46:19.780 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:46:19 smithi089 bash[75068]: cluster 1685821578.0775366 mgr.y (mgr.34104) 210 : cluster 0 pgmap v157: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-06-03T19:46:19.780 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:46:19 smithi089 bash[78452]: cluster 1685821578.0775366 mgr.y (mgr.34104) 210 : cluster 0 pgmap v157: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-06-03T19:46:19.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:46:19 smithi164 bash[59462]: cluster 1685821578.0775366 mgr.y (mgr.34104) 210 : cluster 0 pgmap v157: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-06-03T19:46:20.397 INFO:teuthology.orchestra.run.smithi089.stdout:Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:00b1b6d689278b4f97ff790045c8c51d8ca419ac7ee2ebdf11008f2e810a5c78 2023-06-03T19:46:20.597 INFO:teuthology.orchestra.run.smithi089.stderr:pool 'unique_pool_2' created 2023-06-03T19:46:20.780 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:46:20 smithi089 bash[75068]: audit 1685821579.9825482 mon.c (mon.2) 60 : audit 1 from='client.? 172.21.15.89:0/3771007441' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_2", "pg_num": 16}]: dispatch 2023-06-03T19:46:20.780 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:46:20 smithi089 bash[75068]: audit 1685821579.983263 mon.a (mon.0) 181 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_2", "pg_num": 16}]: dispatch 2023-06-03T19:46:20.780 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:46:20 smithi089 bash[75068]: audit 1685821580.390508 mon.a (mon.0) 182 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:46:20.780 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:46:20 smithi089 bash[75068]: audit 1685821580.430479 mon.c (mon.2) 61 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-03T19:46:20.781 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:46:20 smithi089 bash[78452]: audit 1685821579.9825482 mon.c (mon.2) 60 : audit 1 from='client.? 172.21.15.89:0/3771007441' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_2", "pg_num": 16}]: dispatch 2023-06-03T19:46:20.781 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:46:20 smithi089 bash[78452]: audit 1685821579.983263 mon.a (mon.0) 181 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_2", "pg_num": 16}]: dispatch 2023-06-03T19:46:20.781 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:46:20 smithi089 bash[78452]: audit 1685821580.390508 mon.a (mon.0) 182 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:46:20.781 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:46:20 smithi089 bash[78452]: audit 1685821580.430479 mon.c (mon.2) 61 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-03T19:46:20.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:46:20 smithi164 bash[59462]: audit 1685821579.9825482 mon.c (mon.2) 60 : audit 1 from='client.? 172.21.15.89:0/3771007441' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_2", "pg_num": 16}]: dispatch 2023-06-03T19:46:20.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:46:20 smithi164 bash[59462]: audit 1685821579.983263 mon.a (mon.0) 181 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_2", "pg_num": 16}]: dispatch 2023-06-03T19:46:20.836 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:46:20 smithi164 bash[59462]: audit 1685821580.390508 mon.a (mon.0) 182 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:46:20.836 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:46:20 smithi164 bash[59462]: audit 1685821580.430479 mon.c (mon.2) 61 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-03T19:46:21.630 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -e sha1=8e24bcf462a1d94c2e12aa4e26419634883a1e88 -- bash -c 'echo wait for half of osds to upgrade' 2023-06-03T19:46:21.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:46:21 smithi164 bash[59462]: audit 1685821580.0065253 mgr.y (mgr.34104) 211 : audit 0 from='client.34257 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2023-06-03T19:46:21.836 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:46:21 smithi164 bash[59462]: cephadm 1685821580.0071988 mgr.y (mgr.34104) 212 : cephadm 1 Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:00b1b6d689278b4f97ff790045c8c51d8ca419ac7ee2ebdf11008f2e810a5c78 2023-06-03T19:46:21.836 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:46:21 smithi164 bash[59462]: cluster 1685821580.0783858 mgr.y (mgr.34104) 213 : cluster 0 pgmap v158: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-06-03T19:46:21.836 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:46:21 smithi164 bash[59462]: cluster 1685821580.5738785 osd.6 (osd.6) 4 : cluster 3 failed to encode map e79 with expected crc 2023-06-03T19:46:21.836 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:46:21 smithi164 bash[59462]: cluster 1685821580.57707 osd.5 (osd.5) 6 : cluster 3 failed to encode map e79 with expected crc 2023-06-03T19:46:21.836 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:46:21 smithi164 bash[59462]: audit 1685821580.581358 mon.a (mon.0) 183 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_2", "pg_num": 16}]': finished 2023-06-03T19:46:21.836 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:46:21 smithi164 bash[59462]: cluster 1685821580.5814097 mon.a (mon.0) 184 : cluster 0 osdmap e79: 8 total, 8 up, 8 in 2023-06-03T19:46:21.928 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:46:21 smithi089 bash[75068]: audit 1685821580.0065253 mgr.y (mgr.34104) 211 : audit 0 from='client.34257 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2023-06-03T19:46:21.928 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:46:21 smithi089 bash[75068]: cephadm 1685821580.0071988 mgr.y (mgr.34104) 212 : cephadm 1 Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:00b1b6d689278b4f97ff790045c8c51d8ca419ac7ee2ebdf11008f2e810a5c78 2023-06-03T19:46:21.928 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:46:21 smithi089 bash[75068]: cluster 1685821580.0783858 mgr.y (mgr.34104) 213 : cluster 0 pgmap v158: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-06-03T19:46:21.928 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:46:21 smithi089 bash[75068]: cluster 1685821580.5738785 osd.6 (osd.6) 4 : cluster 3 failed to encode map e79 with expected crc 2023-06-03T19:46:21.929 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:46:21 smithi089 bash[75068]: cluster 1685821580.57707 osd.5 (osd.5) 6 : cluster 3 failed to encode map e79 with expected crc 2023-06-03T19:46:21.929 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:46:21 smithi089 bash[75068]: audit 1685821580.581358 mon.a (mon.0) 183 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_2", "pg_num": 16}]': finished 2023-06-03T19:46:21.929 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:46:21 smithi089 bash[75068]: cluster 1685821580.5814097 mon.a (mon.0) 184 : cluster 0 osdmap e79: 8 total, 8 up, 8 in 2023-06-03T19:46:21.929 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:46:21 smithi089 bash[78452]: audit 1685821580.0065253 mgr.y (mgr.34104) 211 : audit 0 from='client.34257 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2023-06-03T19:46:21.929 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:46:21 smithi089 bash[78452]: cephadm 1685821580.0071988 mgr.y (mgr.34104) 212 : cephadm 1 Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:00b1b6d689278b4f97ff790045c8c51d8ca419ac7ee2ebdf11008f2e810a5c78 2023-06-03T19:46:21.929 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:46:21 smithi089 bash[78452]: cluster 1685821580.0783858 mgr.y (mgr.34104) 213 : cluster 0 pgmap v158: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-06-03T19:46:21.930 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:46:21 smithi089 bash[78452]: cluster 1685821580.5738785 osd.6 (osd.6) 4 : cluster 3 failed to encode map e79 with expected crc 2023-06-03T19:46:21.930 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:46:21 smithi089 bash[78452]: cluster 1685821580.57707 osd.5 (osd.5) 6 : cluster 3 failed to encode map e79 with expected crc 2023-06-03T19:46:21.930 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:46:21 smithi089 bash[78452]: audit 1685821580.581358 mon.a (mon.0) 183 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_2", "pg_num": 16}]': finished 2023-06-03T19:46:21.930 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:46:21 smithi089 bash[78452]: cluster 1685821580.5814097 mon.a (mon.0) 184 : cluster 0 osdmap e79: 8 total, 8 up, 8 in 2023-06-03T19:46:22.584 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph osd pool application enable unique_pool_2 rados --yes-i-really-mean-it || true 2023-06-03T19:46:23.030 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:46:22 smithi089 bash[75068]: cluster 1685821581.587297 osd.5 (osd.5) 7 : cluster 3 failed to encode map e80 with expected crc 2023-06-03T19:46:23.030 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:46:22 smithi089 bash[75068]: cluster 1685821581.5896409 osd.6 (osd.6) 5 : cluster 3 failed to encode map e80 with expected crc 2023-06-03T19:46:23.030 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:46:22 smithi089 bash[75068]: cluster 1685821581.5897477 osd.1 (osd.1) 6 : cluster 3 failed to encode map e80 with expected crc 2023-06-03T19:46:23.030 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:46:22 smithi089 bash[75068]: cluster 1685821581.5899186 osd.7 (osd.7) 5 : cluster 3 failed to encode map e80 with expected crc 2023-06-03T19:46:23.031 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:46:22 smithi089 bash[75068]: cluster 1685821581.590003 osd.4 (osd.4) 6 : cluster 3 failed to encode map e80 with expected crc 2023-06-03T19:46:23.031 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:46:22 smithi089 bash[75068]: cluster 1685821581.590276 osd.5 (osd.5) 8 : cluster 3 failed to encode map e80 with expected crc 2023-06-03T19:46:23.031 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:46:22 smithi089 bash[75068]: cluster 1685821581.5918279 mon.a (mon.0) 185 : cluster 0 osdmap e80: 8 total, 8 up, 8 in 2023-06-03T19:46:23.031 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:46:22 smithi089 bash[75068]: cluster 1685821581.5921142 osd.3 (osd.3) 7 : cluster 3 failed to encode map e80 with expected crc 2023-06-03T19:46:23.031 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:46:22 smithi089 bash[75068]: cluster 1685821581.592869 osd.2 (osd.2) 7 : cluster 3 failed to encode map e80 with expected crc 2023-06-03T19:46:23.031 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:46:22 smithi089 bash[75068]: cluster 1685821581.5930102 osd.0 (osd.0) 7 : cluster 3 failed to encode map e80 with expected crc 2023-06-03T19:46:23.032 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:46:22 smithi089 bash[75068]: cluster 1685821581.5931983 osd.1 (osd.1) 7 : cluster 3 failed to encode map e80 with expected crc 2023-06-03T19:46:23.032 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:46:22 smithi089 bash[75068]: audit 1685821582.066948 mon.c (mon.2) 62 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T19:46:23.032 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:46:22 smithi089 bash[78452]: cluster 1685821581.587297 osd.5 (osd.5) 7 : cluster 3 failed to encode map e80 with expected crc 2023-06-03T19:46:23.032 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:46:22 smithi089 bash[78452]: cluster 1685821581.5896409 osd.6 (osd.6) 5 : cluster 3 failed to encode map e80 with expected crc 2023-06-03T19:46:23.032 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:46:22 smithi089 bash[78452]: cluster 1685821581.5897477 osd.1 (osd.1) 6 : cluster 3 failed to encode map e80 with expected crc 2023-06-03T19:46:23.032 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:46:22 smithi089 bash[78452]: cluster 1685821581.5899186 osd.7 (osd.7) 5 : cluster 3 failed to encode map e80 with expected crc 2023-06-03T19:46:23.033 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:46:22 smithi089 bash[78452]: cluster 1685821581.590003 osd.4 (osd.4) 6 : cluster 3 failed to encode map e80 with expected crc 2023-06-03T19:46:23.033 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:46:22 smithi089 bash[78452]: cluster 1685821581.590276 osd.5 (osd.5) 8 : cluster 3 failed to encode map e80 with expected crc 2023-06-03T19:46:23.033 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:46:22 smithi089 bash[78452]: cluster 1685821581.5918279 mon.a (mon.0) 185 : cluster 0 osdmap e80: 8 total, 8 up, 8 in 2023-06-03T19:46:23.033 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:46:22 smithi089 bash[78452]: cluster 1685821581.5921142 osd.3 (osd.3) 7 : cluster 3 failed to encode map e80 with expected crc 2023-06-03T19:46:23.033 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:46:22 smithi089 bash[78452]: cluster 1685821581.592869 osd.2 (osd.2) 7 : cluster 3 failed to encode map e80 with expected crc 2023-06-03T19:46:23.033 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:46:22 smithi089 bash[78452]: cluster 1685821581.5930102 osd.0 (osd.0) 7 : cluster 3 failed to encode map e80 with expected crc 2023-06-03T19:46:23.033 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:46:22 smithi089 bash[78452]: cluster 1685821581.5931983 osd.1 (osd.1) 7 : cluster 3 failed to encode map e80 with expected crc 2023-06-03T19:46:23.034 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:46:22 smithi089 bash[78452]: audit 1685821582.066948 mon.c (mon.2) 62 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T19:46:23.085 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:46:22 smithi164 bash[59462]: cluster 1685821581.587297 osd.5 (osd.5) 7 : cluster 3 failed to encode map e80 with expected crc 2023-06-03T19:46:23.086 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:46:22 smithi164 bash[59462]: cluster 1685821581.5896409 osd.6 (osd.6) 5 : cluster 3 failed to encode map e80 with expected crc 2023-06-03T19:46:23.086 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:46:22 smithi164 bash[59462]: cluster 1685821581.5897477 osd.1 (osd.1) 6 : cluster 3 failed to encode map e80 with expected crc 2023-06-03T19:46:23.086 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:46:22 smithi164 bash[59462]: cluster 1685821581.5899186 osd.7 (osd.7) 5 : cluster 3 failed to encode map e80 with expected crc 2023-06-03T19:46:23.086 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:46:22 smithi164 bash[59462]: cluster 1685821581.590003 osd.4 (osd.4) 6 : cluster 3 failed to encode map e80 with expected crc 2023-06-03T19:46:23.086 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:46:22 smithi164 bash[59462]: cluster 1685821581.590276 osd.5 (osd.5) 8 : cluster 3 failed to encode map e80 with expected crc 2023-06-03T19:46:23.087 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:46:22 smithi164 bash[59462]: cluster 1685821581.5918279 mon.a (mon.0) 185 : cluster 0 osdmap e80: 8 total, 8 up, 8 in 2023-06-03T19:46:23.087 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:46:22 smithi164 bash[59462]: cluster 1685821581.5921142 osd.3 (osd.3) 7 : cluster 3 failed to encode map e80 with expected crc 2023-06-03T19:46:23.087 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:46:22 smithi164 bash[59462]: cluster 1685821581.592869 osd.2 (osd.2) 7 : cluster 3 failed to encode map e80 with expected crc 2023-06-03T19:46:23.087 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:46:22 smithi164 bash[59462]: cluster 1685821581.5930102 osd.0 (osd.0) 7 : cluster 3 failed to encode map e80 with expected crc 2023-06-03T19:46:23.087 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:46:22 smithi164 bash[59462]: cluster 1685821581.5931983 osd.1 (osd.1) 7 : cluster 3 failed to encode map e80 with expected crc 2023-06-03T19:46:23.087 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:46:22 smithi164 bash[59462]: audit 1685821582.066948 mon.c (mon.2) 62 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T19:46:24.030 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:46:23 smithi089 bash[75068]: cluster 1685821582.0794983 mgr.y (mgr.34104) 214 : cluster 0 pgmap v161: 89 pgs: 16 unknown, 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-06-03T19:46:24.030 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:46:23 smithi089 bash[75068]: cluster 1685821582.5954318 osd.2 (osd.2) 8 : cluster 3 failed to encode map e81 with expected crc 2023-06-03T19:46:24.030 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:46:23 smithi089 bash[75068]: cluster 1685821582.5960069 osd.3 (osd.3) 8 : cluster 3 failed to encode map e81 with expected crc 2023-06-03T19:46:24.030 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:46:23 smithi089 bash[75068]: cluster 1685821582.5998309 mon.a (mon.0) 186 : cluster 0 osdmap e81: 8 total, 8 up, 8 in 2023-06-03T19:46:24.031 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:46:23 smithi089 bash[78452]: cluster 1685821582.0794983 mgr.y (mgr.34104) 214 : cluster 0 pgmap v161: 89 pgs: 16 unknown, 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-06-03T19:46:24.031 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:46:23 smithi089 bash[78452]: cluster 1685821582.5954318 osd.2 (osd.2) 8 : cluster 3 failed to encode map e81 with expected crc 2023-06-03T19:46:24.031 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:46:23 smithi089 bash[78452]: cluster 1685821582.5960069 osd.3 (osd.3) 8 : cluster 3 failed to encode map e81 with expected crc 2023-06-03T19:46:24.031 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:46:23 smithi089 bash[78452]: cluster 1685821582.5998309 mon.a (mon.0) 186 : cluster 0 osdmap e81: 8 total, 8 up, 8 in 2023-06-03T19:46:24.085 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:46:23 smithi164 bash[59462]: cluster 1685821582.0794983 mgr.y (mgr.34104) 214 : cluster 0 pgmap v161: 89 pgs: 16 unknown, 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-06-03T19:46:24.086 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:46:23 smithi164 bash[59462]: cluster 1685821582.5954318 osd.2 (osd.2) 8 : cluster 3 failed to encode map e81 with expected crc 2023-06-03T19:46:24.086 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:46:23 smithi164 bash[59462]: cluster 1685821582.5960069 osd.3 (osd.3) 8 : cluster 3 failed to encode map e81 with expected crc 2023-06-03T19:46:24.086 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:46:23 smithi164 bash[59462]: cluster 1685821582.5998309 mon.a (mon.0) 186 : cluster 0 osdmap e81: 8 total, 8 up, 8 in 2023-06-03T19:46:25.530 INFO:teuthology.orchestra.run.smithi089.stdout:wait for half of osds to upgrade 2023-06-03T19:46:25.717 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:46:25 smithi089 bash[78452]: cluster 1685821584.080596 mgr.y (mgr.34104) 215 : cluster 0 pgmap v163: 89 pgs: 89 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-06-03T19:46:25.718 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:46:25 smithi089 bash[75068]: cluster 1685821584.080596 mgr.y (mgr.34104) 215 : cluster 0 pgmap v163: 89 pgs: 89 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-06-03T19:46:26.085 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:46:25 smithi164 bash[59462]: cluster 1685821584.080596 mgr.y (mgr.34104) 215 : cluster 0 pgmap v163: 89 pgs: 89 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-06-03T19:46:26.365 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -e sha1=8e24bcf462a1d94c2e12aa4e26419634883a1e88 -- bash -c 'while ! ceph osd versions | grep $sha1 | egrep '"'"': [45678]'"'"'; do sleep 2 ; done' 2023-06-03T19:46:26.529 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 19:46:26 smithi089 bash[71163]: ::ffff:172.21.15.164 - - [03/Jun/2023:19:46:26] "GET /metrics HTTP/1.1" 200 189910 "" "Prometheus/2.33.4" 2023-06-03T19:46:27.364 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/mon.a/config 2023-06-03T19:46:27.741 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:46:27 smithi089 bash[78452]: cluster 1685821586.0812297 mgr.y (mgr.34104) 216 : cluster 0 pgmap v164: 89 pgs: 89 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-06-03T19:46:27.741 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:46:27 smithi089 bash[78452]: audit 1685821586.8334756 mon.a (mon.0) 187 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:46:27.741 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:46:27 smithi089 bash[78452]: audit 1685821586.8444798 mon.a (mon.0) 188 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:46:27.741 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:46:27 smithi089 bash[78452]: audit 1685821587.3176112 mon.c (mon.2) 63 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-03T19:46:27.741 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:46:27 smithi089 bash[78452]: audit 1685821587.3188236 mon.c (mon.2) 64 : audit 1 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-03T19:46:27.741 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:46:27 smithi089 bash[78452]: audit 1685821587.326851 mon.a (mon.0) 189 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:46:27.742 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:46:27 smithi089 bash[78452]: audit 1685821587.3952467 mon.c (mon.2) 65 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-03T19:46:27.742 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:46:27 smithi089 bash[78452]: audit 1685821587.3986435 mon.c (mon.2) 66 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-06-03T19:46:27.742 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:46:27 smithi089 bash[78452]: audit 1685821587.4004595 mon.c (mon.2) 67 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-06-03T19:46:27.742 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:46:27 smithi089 bash[78452]: audit 1685821587.4081738 mon.a (mon.0) 190 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:46:27.742 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:46:27 smithi089 bash[78452]: audit 1685821587.40956 mon.c (mon.2) 68 : audit 1 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2023-06-03T19:46:27.742 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:46:27 smithi089 bash[78452]: audit 1685821587.4098785 mon.a (mon.0) 191 : audit 1 from='mgr.34104 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2023-06-03T19:46:27.742 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:46:27 smithi089 bash[78452]: audit 1685821587.4139578 mon.a (mon.0) 192 : audit 1 from='mgr.34104 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]': finished 2023-06-03T19:46:27.743 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:46:27 smithi089 bash[78452]: audit 1685821587.4153779 mon.c (mon.2) 69 : audit 1 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2023-06-03T19:46:27.743 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:46:27 smithi089 bash[78452]: audit 1685821587.4158864 mon.a (mon.0) 193 : audit 1 from='mgr.34104 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2023-06-03T19:46:27.743 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:46:27 smithi089 bash[78452]: audit 1685821587.4206045 mon.a (mon.0) 194 : audit 1 from='mgr.34104 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]': finished 2023-06-03T19:46:27.743 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:46:27 smithi089 bash[78452]: audit 1685821587.424171 mon.c (mon.2) 70 : audit 1 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2023-06-03T19:46:27.743 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:46:27 smithi089 bash[78452]: audit 1685821587.4245667 mon.a (mon.0) 195 : audit 1 from='mgr.34104 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2023-06-03T19:46:27.743 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:46:27 smithi089 bash[78452]: audit 1685821587.4290278 mon.a (mon.0) 196 : audit 1 from='mgr.34104 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]': finished 2023-06-03T19:46:27.743 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:46:27 smithi089 bash[78452]: audit 1685821587.432667 mon.c (mon.2) 71 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-06-03T19:46:27.743 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:46:27 smithi089 bash[78452]: audit 1685821587.4391308 mon.a (mon.0) 197 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:46:27.744 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:46:27 smithi089 bash[78452]: audit 1685821587.4437964 mon.c (mon.2) 72 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2023-06-03T19:46:27.744 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:46:27 smithi089 bash[75068]: cluster 1685821586.0812297 mgr.y (mgr.34104) 216 : cluster 0 pgmap v164: 89 pgs: 89 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-06-03T19:46:27.744 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:46:27 smithi089 bash[75068]: audit 1685821586.8334756 mon.a (mon.0) 187 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:46:27.744 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:46:27 smithi089 bash[75068]: audit 1685821586.8444798 mon.a (mon.0) 188 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:46:27.744 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:46:27 smithi089 bash[75068]: audit 1685821587.3176112 mon.c (mon.2) 63 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-03T19:46:27.744 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:46:27 smithi089 bash[75068]: audit 1685821587.3188236 mon.c (mon.2) 64 : audit 1 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-03T19:46:27.745 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:46:27 smithi089 bash[75068]: audit 1685821587.326851 mon.a (mon.0) 189 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:46:27.745 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:46:27 smithi089 bash[75068]: audit 1685821587.3952467 mon.c (mon.2) 65 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-03T19:46:27.745 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:46:27 smithi089 bash[75068]: audit 1685821587.3986435 mon.c (mon.2) 66 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-06-03T19:46:27.745 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:46:27 smithi089 bash[75068]: audit 1685821587.4004595 mon.c (mon.2) 67 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-06-03T19:46:27.745 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:46:27 smithi089 bash[75068]: audit 1685821587.4081738 mon.a (mon.0) 190 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:46:27.745 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:46:27 smithi089 bash[75068]: audit 1685821587.40956 mon.c (mon.2) 68 : audit 1 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2023-06-03T19:46:27.745 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:46:27 smithi089 bash[75068]: audit 1685821587.4098785 mon.a (mon.0) 191 : audit 1 from='mgr.34104 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2023-06-03T19:46:27.746 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:46:27 smithi089 bash[75068]: audit 1685821587.4139578 mon.a (mon.0) 192 : audit 1 from='mgr.34104 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]': finished 2023-06-03T19:46:27.746 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:46:27 smithi089 bash[75068]: audit 1685821587.4153779 mon.c (mon.2) 69 : audit 1 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2023-06-03T19:46:27.746 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:46:27 smithi089 bash[75068]: audit 1685821587.4158864 mon.a (mon.0) 193 : audit 1 from='mgr.34104 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2023-06-03T19:46:27.746 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:46:27 smithi089 bash[75068]: audit 1685821587.4206045 mon.a (mon.0) 194 : audit 1 from='mgr.34104 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]': finished 2023-06-03T19:46:27.746 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:46:27 smithi089 bash[75068]: audit 1685821587.424171 mon.c (mon.2) 70 : audit 1 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2023-06-03T19:46:27.746 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:46:27 smithi089 bash[75068]: audit 1685821587.4245667 mon.a (mon.0) 195 : audit 1 from='mgr.34104 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2023-06-03T19:46:27.746 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:46:27 smithi089 bash[75068]: audit 1685821587.4290278 mon.a (mon.0) 196 : audit 1 from='mgr.34104 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]': finished 2023-06-03T19:46:27.746 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:46:27 smithi089 bash[75068]: audit 1685821587.432667 mon.c (mon.2) 71 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-06-03T19:46:27.747 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:46:27 smithi089 bash[75068]: audit 1685821587.4391308 mon.a (mon.0) 197 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:46:27.747 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:46:27 smithi089 bash[75068]: audit 1685821587.4437964 mon.c (mon.2) 72 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2023-06-03T19:46:28.086 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:46:27 smithi164 bash[59462]: cluster 1685821586.0812297 mgr.y (mgr.34104) 216 : cluster 0 pgmap v164: 89 pgs: 89 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-06-03T19:46:28.086 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:46:27 smithi164 bash[59462]: audit 1685821586.8334756 mon.a (mon.0) 187 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:46:28.086 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:46:27 smithi164 bash[59462]: audit 1685821586.8444798 mon.a (mon.0) 188 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:46:28.086 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:46:27 smithi164 bash[59462]: audit 1685821587.3176112 mon.c (mon.2) 63 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-03T19:46:28.086 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:46:27 smithi164 bash[59462]: audit 1685821587.3188236 mon.c (mon.2) 64 : audit 1 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-03T19:46:28.087 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:46:27 smithi164 bash[59462]: audit 1685821587.326851 mon.a (mon.0) 189 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:46:28.087 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:46:27 smithi164 bash[59462]: audit 1685821587.3952467 mon.c (mon.2) 65 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-03T19:46:28.087 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:46:27 smithi164 bash[59462]: audit 1685821587.3986435 mon.c (mon.2) 66 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-06-03T19:46:28.087 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:46:27 smithi164 bash[59462]: audit 1685821587.4004595 mon.c (mon.2) 67 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-06-03T19:46:28.087 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:46:27 smithi164 bash[59462]: audit 1685821587.4081738 mon.a (mon.0) 190 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:46:28.087 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:46:27 smithi164 bash[59462]: audit 1685821587.40956 mon.c (mon.2) 68 : audit 1 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2023-06-03T19:46:28.087 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:46:27 smithi164 bash[59462]: audit 1685821587.4098785 mon.a (mon.0) 191 : audit 1 from='mgr.34104 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2023-06-03T19:46:28.087 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:46:27 smithi164 bash[59462]: audit 1685821587.4139578 mon.a (mon.0) 192 : audit 1 from='mgr.34104 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]': finished 2023-06-03T19:46:28.088 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:46:27 smithi164 bash[59462]: audit 1685821587.4153779 mon.c (mon.2) 69 : audit 1 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2023-06-03T19:46:28.088 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:46:27 smithi164 bash[59462]: audit 1685821587.4158864 mon.a (mon.0) 193 : audit 1 from='mgr.34104 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2023-06-03T19:46:28.088 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:46:27 smithi164 bash[59462]: audit 1685821587.4206045 mon.a (mon.0) 194 : audit 1 from='mgr.34104 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]': finished 2023-06-03T19:46:28.088 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:46:27 smithi164 bash[59462]: audit 1685821587.424171 mon.c (mon.2) 70 : audit 1 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2023-06-03T19:46:28.088 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:46:27 smithi164 bash[59462]: audit 1685821587.4245667 mon.a (mon.0) 195 : audit 1 from='mgr.34104 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2023-06-03T19:46:28.088 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:46:27 smithi164 bash[59462]: audit 1685821587.4290278 mon.a (mon.0) 196 : audit 1 from='mgr.34104 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]': finished 2023-06-03T19:46:28.088 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:46:27 smithi164 bash[59462]: audit 1685821587.432667 mon.c (mon.2) 71 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-06-03T19:46:28.089 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:46:27 smithi164 bash[59462]: audit 1685821587.4391308 mon.a (mon.0) 197 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:46:28.089 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:46:27 smithi164 bash[59462]: audit 1685821587.4437964 mon.c (mon.2) 72 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2023-06-03T19:46:28.680 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:46:28 smithi089 bash[75068]: cephadm 1685821587.4012725 mgr.y (mgr.34104) 217 : cephadm 1 Upgrade: Setting container_image for all mon 2023-06-03T19:46:28.680 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:46:28 smithi089 bash[75068]: cephadm 1685821587.4335785 mgr.y (mgr.34104) 218 : cephadm 1 Upgrade: Setting container_image for all crash 2023-06-03T19:46:28.681 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:46:28 smithi089 bash[75068]: audit 1685821587.4440289 mgr.y (mgr.34104) 219 : audit 0 from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2023-06-03T19:46:28.681 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:46:28 smithi089 bash[75068]: cephadm 1685821587.4455655 mgr.y (mgr.34104) 220 : cephadm 1 Upgrade: osd.0 is safe to restart 2023-06-03T19:46:28.681 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:46:28 smithi089 bash[78452]: cephadm 1685821587.4012725 mgr.y (mgr.34104) 217 : cephadm 1 Upgrade: Setting container_image for all mon 2023-06-03T19:46:28.681 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:46:28 smithi089 bash[78452]: cephadm 1685821587.4335785 mgr.y (mgr.34104) 218 : cephadm 1 Upgrade: Setting container_image for all crash 2023-06-03T19:46:28.681 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:46:28 smithi089 bash[78452]: audit 1685821587.4440289 mgr.y (mgr.34104) 219 : audit 0 from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2023-06-03T19:46:28.681 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:46:28 smithi089 bash[78452]: cephadm 1685821587.4455655 mgr.y (mgr.34104) 220 : cephadm 1 Upgrade: osd.0 is safe to restart 2023-06-03T19:46:29.085 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:46:28 smithi164 bash[59462]: cephadm 1685821587.4012725 mgr.y (mgr.34104) 217 : cephadm 1 Upgrade: Setting container_image for all mon 2023-06-03T19:46:29.086 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:46:28 smithi164 bash[59462]: cephadm 1685821587.4335785 mgr.y (mgr.34104) 218 : cephadm 1 Upgrade: Setting container_image for all crash 2023-06-03T19:46:29.086 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:46:28 smithi164 bash[59462]: audit 1685821587.4440289 mgr.y (mgr.34104) 219 : audit 0 from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2023-06-03T19:46:29.086 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:46:28 smithi164 bash[59462]: cephadm 1685821587.4455655 mgr.y (mgr.34104) 220 : cephadm 1 Upgrade: osd.0 is safe to restart 2023-06-03T19:46:29.960 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:46:29 smithi089 bash[75068]: cluster 1685821588.0818036 mgr.y (mgr.34104) 221 : cluster 0 pgmap v165: 89 pgs: 89 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-06-03T19:46:29.961 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:46:29 smithi089 bash[78452]: cluster 1685821588.0818036 mgr.y (mgr.34104) 221 : cluster 0 pgmap v165: 89 pgs: 89 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-06-03T19:46:30.085 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:46:29 smithi164 bash[59462]: cluster 1685821588.0818036 mgr.y (mgr.34104) 221 : cluster 0 pgmap v165: 89 pgs: 89 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-06-03T19:46:32.036 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:46:31 smithi089 bash[78452]: cluster 1685821590.0830803 mgr.y (mgr.34104) 222 : cluster 0 pgmap v166: 89 pgs: 89 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-06-03T19:46:32.037 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:46:31 smithi089 bash[75068]: cluster 1685821590.0830803 mgr.y (mgr.34104) 222 : cluster 0 pgmap v166: 89 pgs: 89 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-06-03T19:46:32.085 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:46:31 smithi164 bash[59462]: cluster 1685821590.0830803 mgr.y (mgr.34104) 222 : cluster 0 pgmap v166: 89 pgs: 89 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-06-03T19:46:33.530 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:46:33 smithi089 bash[75068]: cluster 1685821592.0836153 mgr.y (mgr.34104) 223 : cluster 0 pgmap v167: 89 pgs: 89 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-06-03T19:46:33.530 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:46:33 smithi089 bash[75068]: audit 1685821592.1423032 mon.a (mon.0) 198 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:46:33.530 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:46:33 smithi089 bash[75068]: audit 1685821593.0332472 mon.a (mon.0) 199 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:46:33.530 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:46:33 smithi089 bash[75068]: audit 1685821593.0344758 mon.c (mon.2) 73 : audit 1 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2023-06-03T19:46:33.530 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:46:33 smithi089 bash[75068]: audit 1685821593.035276 mon.c (mon.2) 74 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-03T19:46:33.531 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:46:33 smithi089 bash[78452]: cluster 1685821592.0836153 mgr.y (mgr.34104) 223 : cluster 0 pgmap v167: 89 pgs: 89 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-06-03T19:46:33.531 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:46:33 smithi089 bash[78452]: audit 1685821592.1423032 mon.a (mon.0) 198 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:46:33.531 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:46:33 smithi089 bash[78452]: audit 1685821593.0332472 mon.a (mon.0) 199 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:46:33.531 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:46:33 smithi089 bash[78452]: audit 1685821593.0344758 mon.c (mon.2) 73 : audit 1 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2023-06-03T19:46:33.531 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:46:33 smithi089 bash[78452]: audit 1685821593.035276 mon.c (mon.2) 74 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-03T19:46:33.585 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:46:33 smithi164 bash[59462]: cluster 1685821592.0836153 mgr.y (mgr.34104) 223 : cluster 0 pgmap v167: 89 pgs: 89 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-06-03T19:46:33.586 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:46:33 smithi164 bash[59462]: audit 1685821592.1423032 mon.a (mon.0) 198 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:46:33.586 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:46:33 smithi164 bash[59462]: audit 1685821593.0332472 mon.a (mon.0) 199 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:46:33.586 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:46:33 smithi164 bash[59462]: audit 1685821593.0344758 mon.c (mon.2) 73 : audit 1 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2023-06-03T19:46:33.586 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:46:33 smithi164 bash[59462]: audit 1685821593.035276 mon.c (mon.2) 74 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-03T19:46:34.529 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:46:34 smithi089 bash[78452]: cephadm 1685821593.0240757 mgr.y (mgr.34104) 224 : cephadm 1 Upgrade: Updating osd.0 2023-06-03T19:46:34.530 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:46:34 smithi089 bash[78452]: cephadm 1685821593.0377517 mgr.y (mgr.34104) 225 : cephadm 1 Deploying daemon osd.0 on smithi089 2023-06-03T19:46:34.530 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:46:34 smithi089 bash[78452]: audit 1685821593.673984 mon.c (mon.2) 75 : audit 0 from='client.? 172.21.15.89:0/4030176247' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:46:34.530 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:46:34 smithi089 bash[75068]: cephadm 1685821593.0240757 mgr.y (mgr.34104) 224 : cephadm 1 Upgrade: Updating osd.0 2023-06-03T19:46:34.530 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:46:34 smithi089 bash[75068]: cephadm 1685821593.0377517 mgr.y (mgr.34104) 225 : cephadm 1 Deploying daemon osd.0 on smithi089 2023-06-03T19:46:34.531 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:46:34 smithi089 bash[75068]: audit 1685821593.673984 mon.c (mon.2) 75 : audit 0 from='client.? 172.21.15.89:0/4030176247' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:46:34.585 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:46:34 smithi164 bash[59462]: cephadm 1685821593.0240757 mgr.y (mgr.34104) 224 : cephadm 1 Upgrade: Updating osd.0 2023-06-03T19:46:34.586 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:46:34 smithi164 bash[59462]: cephadm 1685821593.0377517 mgr.y (mgr.34104) 225 : cephadm 1 Deploying daemon osd.0 on smithi089 2023-06-03T19:46:34.586 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:46:34 smithi164 bash[59462]: audit 1685821593.673984 mon.c (mon.2) 75 : audit 0 from='client.? 172.21.15.89:0/4030176247' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:46:35.279 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:46:35 smithi089 bash[78452]: cluster 1685821594.084671 mgr.y (mgr.34104) 226 : cluster 0 pgmap v168: 89 pgs: 89 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-06-03T19:46:35.280 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:46:35 smithi089 bash[75068]: cluster 1685821594.084671 mgr.y (mgr.34104) 226 : cluster 0 pgmap v168: 89 pgs: 89 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-06-03T19:46:35.585 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:46:35 smithi164 bash[59462]: cluster 1685821594.084671 mgr.y (mgr.34104) 226 : cluster 0 pgmap v168: 89 pgs: 89 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-06-03T19:46:36.176 INFO:teuthology.orchestra.run.smithi089.stderr:enabled application 'rados' on pool 'unique_pool_2' 2023-06-03T19:46:36.252 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 19:46:36 smithi089 bash[71163]: ::ffff:172.21.15.164 - - [03/Jun/2023:19:46:36] "GET /metrics HTTP/1.1" 200 189910 "" "Prometheus/2.33.4" 2023-06-03T19:46:36.252 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:46:36 smithi089 bash[75068]: audit 1685821595.195585 mon.a (mon.0) 200 : audit 1 from='client.? 172.21.15.89:0/2275598652' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_2", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-06-03T19:46:36.253 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:46:36 smithi089 bash[75068]: audit 1685821596.1196942 mon.a (mon.0) 201 : audit 0 from='client.? 172.21.15.89:0/1642393376' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:46:36.253 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:46:36 smithi089 bash[78452]: audit 1685821595.195585 mon.a (mon.0) 200 : audit 1 from='client.? 172.21.15.89:0/2275598652' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_2", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-06-03T19:46:36.253 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:46:36 smithi089 bash[78452]: audit 1685821596.1196942 mon.a (mon.0) 201 : audit 0 from='client.? 172.21.15.89:0/1642393376' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:46:36.585 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:46:36 smithi164 bash[59462]: audit 1685821595.195585 mon.a (mon.0) 200 : audit 1 from='client.? 172.21.15.89:0/2275598652' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_2", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-06-03T19:46:36.586 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:46:36 smithi164 bash[59462]: audit 1685821596.1196942 mon.a (mon.0) 201 : audit 0 from='client.? 172.21.15.89:0/1642393376' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:46:37.488 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:46:37 smithi089 bash[78452]: cluster 1685821596.0853367 mgr.y (mgr.34104) 227 : cluster 0 pgmap v169: 89 pgs: 89 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-06-03T19:46:37.488 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:46:37 smithi089 bash[78452]: cluster 1685821596.1620336 osd.5 (osd.5) 9 : cluster 3 failed to encode map e82 with expected crc 2023-06-03T19:46:37.489 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:46:37 smithi089 bash[78452]: cluster 1685821596.1641817 osd.1 (osd.1) 8 : cluster 3 failed to encode map e82 with expected crc 2023-06-03T19:46:37.489 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:46:37 smithi089 bash[78452]: audit 1685821596.166662 mon.a (mon.0) 202 : audit 1 from='client.? 172.21.15.89:0/2275598652' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_2", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-06-03T19:46:37.489 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:46:37 smithi089 bash[78452]: cluster 1685821596.1667328 mon.a (mon.0) 203 : cluster 0 osdmap e82: 8 total, 8 up, 8 in 2023-06-03T19:46:37.489 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:46:37 smithi089 bash[78452]: audit 1685821597.0675583 mon.c (mon.2) 76 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T19:46:37.489 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:46:37 smithi089 bash[75068]: cluster 1685821596.0853367 mgr.y (mgr.34104) 227 : cluster 0 pgmap v169: 89 pgs: 89 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-06-03T19:46:37.489 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:46:37 smithi089 bash[75068]: cluster 1685821596.1620336 osd.5 (osd.5) 9 : cluster 3 failed to encode map e82 with expected crc 2023-06-03T19:46:37.490 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:46:37 smithi089 bash[75068]: cluster 1685821596.1641817 osd.1 (osd.1) 8 : cluster 3 failed to encode map e82 with expected crc 2023-06-03T19:46:37.490 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:46:37 smithi089 bash[75068]: audit 1685821596.166662 mon.a (mon.0) 202 : audit 1 from='client.? 172.21.15.89:0/2275598652' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_2", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-06-03T19:46:37.490 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:46:37 smithi089 bash[75068]: cluster 1685821596.1667328 mon.a (mon.0) 203 : cluster 0 osdmap e82: 8 total, 8 up, 8 in 2023-06-03T19:46:37.490 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:46:37 smithi089 bash[75068]: audit 1685821597.0675583 mon.c (mon.2) 76 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T19:46:37.585 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:46:37 smithi164 bash[59462]: cluster 1685821596.0853367 mgr.y (mgr.34104) 227 : cluster 0 pgmap v169: 89 pgs: 89 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-06-03T19:46:37.586 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:46:37 smithi164 bash[59462]: cluster 1685821596.1620336 osd.5 (osd.5) 9 : cluster 3 failed to encode map e82 with expected crc 2023-06-03T19:46:37.586 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:46:37 smithi164 bash[59462]: cluster 1685821596.1641817 osd.1 (osd.1) 8 : cluster 3 failed to encode map e82 with expected crc 2023-06-03T19:46:37.586 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:46:37 smithi164 bash[59462]: audit 1685821596.166662 mon.a (mon.0) 202 : audit 1 from='client.? 172.21.15.89:0/2275598652' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_2", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-06-03T19:46:37.586 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:46:37 smithi164 bash[59462]: cluster 1685821596.1667328 mon.a (mon.0) 203 : cluster 0 osdmap e82: 8 total, 8 up, 8 in 2023-06-03T19:46:37.586 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:46:37 smithi164 bash[59462]: audit 1685821597.0675583 mon.c (mon.2) 76 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T19:46:38.491 DEBUG:teuthology.orchestra.run.smithi089:> /bin/sh -c 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage rados --no-log-to-stderr --name client.0 -b 65536 --object-size 65536 -p unique_pool_2 bench 90 write' 2023-06-03T19:46:38.493 INFO:tasks.radosbench:joining radosbench (timing out after 3000s) 2023-06-03T19:46:38.494 INFO:teuthology.orchestra.run:waiting for 3000 2023-06-03T19:46:38.529 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:46:38 smithi089 bash[75068]: cluster 1685821597.825833 osd.6 (osd.6) 6 : cluster 3 failed to encode map e82 with expected crc 2023-06-03T19:46:38.530 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:46:38 smithi089 bash[78452]: cluster 1685821597.825833 osd.6 (osd.6) 6 : cluster 3 failed to encode map e82 with expected crc 2023-06-03T19:46:38.534 INFO:tasks.radosbench.radosbench.0.smithi089.stdout:hints = 1 2023-06-03T19:46:38.534 INFO:tasks.radosbench.radosbench.0.smithi089.stdout:Maintaining 16 concurrent writes of 65536 bytes to objects of size 65536 for up to 90 seconds or 0 objects 2023-06-03T19:46:38.534 INFO:tasks.radosbench.radosbench.0.smithi089.stdout:Object prefix: benchmark_data_smithi089_84347 2023-06-03T19:46:38.535 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-06-03T19:46:38.535 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 0 0 0 0 0 0 - 0 2023-06-03T19:46:38.585 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:46:38 smithi164 bash[59462]: cluster 1685821597.825833 osd.6 (osd.6) 6 : cluster 3 failed to encode map e82 with expected crc 2023-06-03T19:46:39.398 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:46:39 smithi089 bash[78452]: cluster 1685821598.0559928 osd.3 (osd.3) 9 : cluster 3 failed to encode map e82 with expected crc 2023-06-03T19:46:39.398 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:46:39 smithi089 bash[78452]: cluster 1685821598.085814 mgr.y (mgr.34104) 228 : cluster 0 pgmap v171: 89 pgs: 89 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-06-03T19:46:39.398 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:46:39 smithi089 bash[78452]: cluster 1685821598.2209544 osd.4 (osd.4) 7 : cluster 3 failed to encode map e82 with expected crc 2023-06-03T19:46:39.399 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:46:39 smithi089 bash[78452]: cluster 1685821598.3390768 osd.7 (osd.7) 6 : cluster 3 failed to encode map e82 with expected crc 2023-06-03T19:46:39.399 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:46:39 smithi089 bash[78452]: audit 1685821598.4802344 mon.a (mon.0) 204 : audit 0 from='client.? 172.21.15.89:0/808587434' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:46:39.399 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:46:39 smithi089 bash[75068]: cluster 1685821598.0559928 osd.3 (osd.3) 9 : cluster 3 failed to encode map e82 with expected crc 2023-06-03T19:46:39.399 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:46:39 smithi089 bash[75068]: cluster 1685821598.085814 mgr.y (mgr.34104) 228 : cluster 0 pgmap v171: 89 pgs: 89 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-06-03T19:46:39.399 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:46:39 smithi089 bash[75068]: cluster 1685821598.2209544 osd.4 (osd.4) 7 : cluster 3 failed to encode map e82 with expected crc 2023-06-03T19:46:39.399 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:46:39 smithi089 bash[75068]: cluster 1685821598.3390768 osd.7 (osd.7) 6 : cluster 3 failed to encode map e82 with expected crc 2023-06-03T19:46:39.400 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:46:39 smithi089 bash[75068]: audit 1685821598.4802344 mon.a (mon.0) 204 : audit 0 from='client.? 172.21.15.89:0/808587434' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:46:39.535 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 1 16 2587 2571 160.679 160.688 0.00389393 0.0061871 2023-06-03T19:46:39.585 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:46:39 smithi164 bash[59462]: cluster 1685821598.0559928 osd.3 (osd.3) 9 : cluster 3 failed to encode map e82 with expected crc 2023-06-03T19:46:39.586 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:46:39 smithi164 bash[59462]: cluster 1685821598.085814 mgr.y (mgr.34104) 228 : cluster 0 pgmap v171: 89 pgs: 89 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-06-03T19:46:39.586 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:46:39 smithi164 bash[59462]: cluster 1685821598.2209544 osd.4 (osd.4) 7 : cluster 3 failed to encode map e82 with expected crc 2023-06-03T19:46:39.586 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:46:39 smithi164 bash[59462]: cluster 1685821598.3390768 osd.7 (osd.7) 6 : cluster 3 failed to encode map e82 with expected crc 2023-06-03T19:46:39.586 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:46:39 smithi164 bash[59462]: audit 1685821598.4802344 mon.a (mon.0) 204 : audit 0 from='client.? 172.21.15.89:0/808587434' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:46:39.779 INFO:journalctl@ceph.osd.0.smithi089.stdout:Jun 03 19:46:39 smithi089 systemd[1]: Stopping Ceph osd.0 for f1db3478-0243-11ee-9b1d-001a4aab830c... 2023-06-03T19:46:39.780 INFO:journalctl@ceph.osd.0.smithi089.stdout:Jun 03 19:46:39 smithi089 bash[36371]: debug 2023-06-03T19:46:39.552+0000 7efe3a797700 -1 received signal: Terminated from /sbin/docker-init -- /usr/bin/ceph-osd -n osd.0 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug (PID: 1) UID: 0 2023-06-03T19:46:39.780 INFO:journalctl@ceph.osd.0.smithi089.stdout:Jun 03 19:46:39 smithi089 bash[36371]: debug 2023-06-03T19:46:39.552+0000 7efe3a797700 -1 osd.0 82 *** Got signal Terminated *** 2023-06-03T19:46:39.780 INFO:journalctl@ceph.osd.0.smithi089.stdout:Jun 03 19:46:39 smithi089 bash[36371]: debug 2023-06-03T19:46:39.552+0000 7efe3a797700 -1 osd.0 82 *** Immediate shutdown (osd_fast_shutdown=true) *** 2023-06-03T19:46:40.530 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:46:40 smithi089 bash[78452]: cluster 1685821599.5564904 mon.a (mon.0) 205 : cluster 1 osd.0 marked itself down and dead 2023-06-03T19:46:40.530 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:46:40 smithi089 bash[75068]: cluster 1685821599.5564904 mon.a (mon.0) 205 : cluster 1 osd.0 marked itself down and dead 2023-06-03T19:46:40.537 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 2 16 4559 4543 141.958 123.25 0.00135761 0.0060325 2023-06-03T19:46:40.586 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:46:40 smithi164 bash[59462]: cluster 1685821599.5564904 mon.a (mon.0) 205 : cluster 1 osd.0 marked itself down and dead 2023-06-03T19:46:41.529 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:46:41 smithi089 bash[78452]: cluster 1685821600.0867298 mgr.y (mgr.34104) 229 : cluster 0 pgmap v172: 89 pgs: 89 active+clean; 9.3 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 896 KiB/s wr, 13 op/s 2023-06-03T19:46:41.530 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:46:41 smithi089 bash[78452]: cluster 1685821600.2123976 mon.a (mon.0) 206 : cluster 3 Health check failed: 1 osds down (OSD_DOWN) 2023-06-03T19:46:41.530 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:46:41 smithi089 bash[78452]: cluster 1685821600.224596 osd.7 (osd.7) 7 : cluster 3 failed to encode map e83 with expected crc 2023-06-03T19:46:41.530 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:46:41 smithi089 bash[78452]: cluster 1685821600.2290351 osd.2 (osd.2) 9 : cluster 3 failed to encode map e83 with expected crc 2023-06-03T19:46:41.530 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:46:41 smithi089 bash[78452]: cluster 1685821600.2373502 mon.a (mon.0) 207 : cluster 0 osdmap e83: 8 total, 7 up, 8 in 2023-06-03T19:46:41.530 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:46:41 smithi089 bash[78452]: audit 1685821600.8438818 mon.a (mon.0) 208 : audit 0 from='client.? 172.21.15.89:0/924585979' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:46:41.531 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:46:41 smithi089 bash[75068]: cluster 1685821600.0867298 mgr.y (mgr.34104) 229 : cluster 0 pgmap v172: 89 pgs: 89 active+clean; 9.3 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 896 KiB/s wr, 13 op/s 2023-06-03T19:46:41.531 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:46:41 smithi089 bash[75068]: cluster 1685821600.2123976 mon.a (mon.0) 206 : cluster 3 Health check failed: 1 osds down (OSD_DOWN) 2023-06-03T19:46:41.531 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:46:41 smithi089 bash[75068]: cluster 1685821600.224596 osd.7 (osd.7) 7 : cluster 3 failed to encode map e83 with expected crc 2023-06-03T19:46:41.531 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:46:41 smithi089 bash[75068]: cluster 1685821600.2290351 osd.2 (osd.2) 9 : cluster 3 failed to encode map e83 with expected crc 2023-06-03T19:46:41.531 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:46:41 smithi089 bash[75068]: cluster 1685821600.2373502 mon.a (mon.0) 207 : cluster 0 osdmap e83: 8 total, 7 up, 8 in 2023-06-03T19:46:41.531 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:46:41 smithi089 bash[75068]: audit 1685821600.8438818 mon.a (mon.0) 208 : audit 0 from='client.? 172.21.15.89:0/924585979' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:46:41.535 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 3 16 5537 5521 115.011 61.125 0.00338655 0.00868541 2023-06-03T19:46:41.585 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:46:41 smithi164 bash[59462]: cluster 1685821600.0867298 mgr.y (mgr.34104) 229 : cluster 0 pgmap v172: 89 pgs: 89 active+clean; 9.3 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 896 KiB/s wr, 13 op/s 2023-06-03T19:46:41.585 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:46:41 smithi164 bash[59462]: cluster 1685821600.2123976 mon.a (mon.0) 206 : cluster 3 Health check failed: 1 osds down (OSD_DOWN) 2023-06-03T19:46:41.586 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:46:41 smithi164 bash[59462]: cluster 1685821600.224596 osd.7 (osd.7) 7 : cluster 3 failed to encode map e83 with expected crc 2023-06-03T19:46:41.586 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:46:41 smithi164 bash[59462]: cluster 1685821600.2290351 osd.2 (osd.2) 9 : cluster 3 failed to encode map e83 with expected crc 2023-06-03T19:46:41.586 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:46:41 smithi164 bash[59462]: cluster 1685821600.2373502 mon.a (mon.0) 207 : cluster 0 osdmap e83: 8 total, 7 up, 8 in 2023-06-03T19:46:41.586 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:46:41 smithi164 bash[59462]: audit 1685821600.8438818 mon.a (mon.0) 208 : audit 0 from='client.? 172.21.15.89:0/924585979' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:46:42.529 INFO:journalctl@ceph.osd.0.smithi089.stdout:Jun 03 19:46:42 smithi089 bash[84425]: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd-0 2023-06-03T19:46:42.530 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:46:42 smithi089 bash[78452]: cluster 1685821601.2265432 osd.5 (osd.5) 10 : cluster 3 failed to encode map e84 with expected crc 2023-06-03T19:46:42.530 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:46:42 smithi089 bash[78452]: cluster 1685821601.2291055 osd.3 (osd.3) 10 : cluster 3 failed to encode map e84 with expected crc 2023-06-03T19:46:42.530 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:46:42 smithi089 bash[78452]: cluster 1685821601.2308202 osd.7 (osd.7) 8 : cluster 3 failed to encode map e84 with expected crc 2023-06-03T19:46:42.530 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:46:42 smithi089 bash[78452]: cluster 1685821601.2309356 osd.4 (osd.4) 8 : cluster 3 failed to encode map e84 with expected crc 2023-06-03T19:46:42.530 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:46:42 smithi089 bash[78452]: cluster 1685821601.231121 osd.6 (osd.6) 7 : cluster 3 failed to encode map e84 with expected crc 2023-06-03T19:46:42.531 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:46:42 smithi089 bash[78452]: cluster 1685821601.2330554 mon.a (mon.0) 209 : cluster 0 osdmap e84: 8 total, 7 up, 8 in 2023-06-03T19:46:42.531 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:46:42 smithi089 bash[78452]: cluster 1685821601.2334523 osd.3 (osd.3) 11 : cluster 3 failed to encode map e84 with expected crc 2023-06-03T19:46:42.531 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:46:42 smithi089 bash[78452]: cluster 1685821601.2336907 osd.1 (osd.1) 9 : cluster 3 failed to encode map e84 with expected crc 2023-06-03T19:46:42.531 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:46:42 smithi089 bash[78452]: cluster 1685821601.2337215 osd.2 (osd.2) 10 : cluster 3 failed to encode map e84 with expected crc 2023-06-03T19:46:42.531 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:46:42 smithi089 bash[75068]: cluster 1685821601.2265432 osd.5 (osd.5) 10 : cluster 3 failed to encode map e84 with expected crc 2023-06-03T19:46:42.531 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:46:42 smithi089 bash[75068]: cluster 1685821601.2291055 osd.3 (osd.3) 10 : cluster 3 failed to encode map e84 with expected crc 2023-06-03T19:46:42.531 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:46:42 smithi089 bash[75068]: cluster 1685821601.2308202 osd.7 (osd.7) 8 : cluster 3 failed to encode map e84 with expected crc 2023-06-03T19:46:42.532 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:46:42 smithi089 bash[75068]: cluster 1685821601.2309356 osd.4 (osd.4) 8 : cluster 3 failed to encode map e84 with expected crc 2023-06-03T19:46:42.532 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:46:42 smithi089 bash[75068]: cluster 1685821601.231121 osd.6 (osd.6) 7 : cluster 3 failed to encode map e84 with expected crc 2023-06-03T19:46:42.532 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:46:42 smithi089 bash[75068]: cluster 1685821601.2330554 mon.a (mon.0) 209 : cluster 0 osdmap e84: 8 total, 7 up, 8 in 2023-06-03T19:46:42.532 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:46:42 smithi089 bash[75068]: cluster 1685821601.2334523 osd.3 (osd.3) 11 : cluster 3 failed to encode map e84 with expected crc 2023-06-03T19:46:42.532 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:46:42 smithi089 bash[75068]: cluster 1685821601.2336907 osd.1 (osd.1) 9 : cluster 3 failed to encode map e84 with expected crc 2023-06-03T19:46:42.532 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:46:42 smithi089 bash[75068]: cluster 1685821601.2337215 osd.2 (osd.2) 10 : cluster 3 failed to encode map e84 with expected crc 2023-06-03T19:46:42.535 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 4 16 8716 8700 135.926 198.688 0.0057619 0.0073477 2023-06-03T19:46:42.585 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:46:42 smithi164 bash[59462]: cluster 1685821601.2265432 osd.5 (osd.5) 10 : cluster 3 failed to encode map e84 with expected crc 2023-06-03T19:46:42.586 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:46:42 smithi164 bash[59462]: cluster 1685821601.2291055 osd.3 (osd.3) 10 : cluster 3 failed to encode map e84 with expected crc 2023-06-03T19:46:42.586 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:46:42 smithi164 bash[59462]: cluster 1685821601.2308202 osd.7 (osd.7) 8 : cluster 3 failed to encode map e84 with expected crc 2023-06-03T19:46:42.586 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:46:42 smithi164 bash[59462]: cluster 1685821601.2309356 osd.4 (osd.4) 8 : cluster 3 failed to encode map e84 with expected crc 2023-06-03T19:46:42.586 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:46:42 smithi164 bash[59462]: cluster 1685821601.231121 osd.6 (osd.6) 7 : cluster 3 failed to encode map e84 with expected crc 2023-06-03T19:46:42.586 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:46:42 smithi164 bash[59462]: cluster 1685821601.2330554 mon.a (mon.0) 209 : cluster 0 osdmap e84: 8 total, 7 up, 8 in 2023-06-03T19:46:42.586 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:46:42 smithi164 bash[59462]: cluster 1685821601.2334523 osd.3 (osd.3) 11 : cluster 3 failed to encode map e84 with expected crc 2023-06-03T19:46:42.586 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:46:42 smithi164 bash[59462]: cluster 1685821601.2336907 osd.1 (osd.1) 9 : cluster 3 failed to encode map e84 with expected crc 2023-06-03T19:46:42.586 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:46:42 smithi164 bash[59462]: cluster 1685821601.2337215 osd.2 (osd.2) 10 : cluster 3 failed to encode map e84 with expected crc 2023-06-03T19:46:43.529 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:46:43 smithi089 bash[78452]: cluster 1685821602.087117 mgr.y (mgr.34104) 230 : cluster 0 pgmap v175: 89 pgs: 15 stale+active+clean, 74 active+clean; 9.3 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 1.5 MiB/s wr, 23 op/s 2023-06-03T19:46:43.530 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:46:43 smithi089 bash[75068]: cluster 1685821602.087117 mgr.y (mgr.34104) 230 : cluster 0 pgmap v175: 89 pgs: 15 stale+active+clean, 74 active+clean; 9.3 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 1.5 MiB/s wr, 23 op/s 2023-06-03T19:46:43.535 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 5 16 11534 11518 143.963 176.125 0.00382499 0.00693742 2023-06-03T19:46:43.585 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:46:43 smithi164 bash[59462]: cluster 1685821602.087117 mgr.y (mgr.34104) 230 : cluster 0 pgmap v175: 89 pgs: 15 stale+active+clean, 74 active+clean; 9.3 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 1.5 MiB/s wr, 23 op/s 2023-06-03T19:46:44.529 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:46:44 smithi089 bash[78452]: audit 1685821603.4215095 mon.a (mon.0) 210 : audit 0 from='client.? 172.21.15.89:0/2277230486' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:46:44.530 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:46:44 smithi089 bash[75068]: audit 1685821603.4215095 mon.a (mon.0) 210 : audit 0 from='client.? 172.21.15.89:0/2277230486' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:46:44.535 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 6 16 14366 14350 149.466 177 0.00499286 0.00668196 2023-06-03T19:46:44.585 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:46:44 smithi164 bash[59462]: audit 1685821603.4215095 mon.a (mon.0) 210 : audit 0 from='client.? 172.21.15.89:0/2277230486' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:46:45.529 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:46:45 smithi089 bash[78452]: cluster 1685821604.0879312 mgr.y (mgr.34104) 231 : cluster 0 pgmap v176: 89 pgs: 15 active+undersized+degraded, 19 active+undersized, 55 active+clean; 697 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 88 MiB/s wr, 1.41k op/s; 5420/33495 objects degraded (16.182%); 8.1 KiB/s, 0 objects/s recovering 2023-06-03T19:46:45.530 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:46:45 smithi089 bash[78452]: cluster 1685821604.2443764 mon.a (mon.0) 211 : cluster 3 Health check failed: Degraded data redundancy: 5420/33495 objects degraded (16.182%), 15 pgs degraded (PG_DEGRADED) 2023-06-03T19:46:45.530 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:46:45 smithi089 bash[75068]: cluster 1685821604.0879312 mgr.y (mgr.34104) 231 : cluster 0 pgmap v176: 89 pgs: 15 active+undersized+degraded, 19 active+undersized, 55 active+clean; 697 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 88 MiB/s wr, 1.41k op/s; 5420/33495 objects degraded (16.182%); 8.1 KiB/s, 0 objects/s recovering 2023-06-03T19:46:45.530 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:46:45 smithi089 bash[75068]: cluster 1685821604.2443764 mon.a (mon.0) 211 : cluster 3 Health check failed: Degraded data redundancy: 5420/33495 objects degraded (16.182%), 15 pgs degraded (PG_DEGRADED) 2023-06-03T19:46:45.535 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 7 16 16822 16806 150.04 153.5 0.00441674 0.00665843 2023-06-03T19:46:45.585 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:46:45 smithi164 bash[59462]: cluster 1685821604.0879312 mgr.y (mgr.34104) 231 : cluster 0 pgmap v176: 89 pgs: 15 active+undersized+degraded, 19 active+undersized, 55 active+clean; 697 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 88 MiB/s wr, 1.41k op/s; 5420/33495 objects degraded (16.182%); 8.1 KiB/s, 0 objects/s recovering 2023-06-03T19:46:45.586 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:46:45 smithi164 bash[59462]: cluster 1685821604.2443764 mon.a (mon.0) 211 : cluster 3 Health check failed: Degraded data redundancy: 5420/33495 objects degraded (16.182%), 15 pgs degraded (PG_DEGRADED) 2023-06-03T19:46:46.530 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:46:46 smithi089 bash[75068]: audit 1685821605.9573557 mon.c (mon.2) 77 : audit 0 from='client.? 172.21.15.89:0/4143335494' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:46:46.530 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 19:46:46 smithi089 bash[71163]: ::ffff:172.21.15.164 - - [03/Jun/2023:19:46:46] "GET /metrics HTTP/1.1" 200 189908 "" "Prometheus/2.33.4" 2023-06-03T19:46:46.530 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:46:46 smithi089 bash[78452]: audit 1685821605.9573557 mon.c (mon.2) 77 : audit 0 from='client.? 172.21.15.89:0/4143335494' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:46:46.536 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 8 16 18372 18356 143.393 96.875 0.00726955 0.00696692 2023-06-03T19:46:46.585 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:46:46 smithi164 bash[59462]: audit 1685821605.9573557 mon.c (mon.2) 77 : audit 0 from='client.? 172.21.15.89:0/4143335494' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:46:46.997 INFO:journalctl@ceph.osd.0.smithi089.stdout:Jun 03 19:46:46 smithi089 systemd[1]: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c@osd.0.service: Succeeded. 2023-06-03T19:46:46.997 INFO:journalctl@ceph.osd.0.smithi089.stdout:Jun 03 19:46:46 smithi089 systemd[1]: Stopped Ceph osd.0 for f1db3478-0243-11ee-9b1d-001a4aab830c. 2023-06-03T19:46:46.997 INFO:journalctl@ceph.osd.0.smithi089.stdout:Jun 03 19:46:46 smithi089 systemd[1]: Started Ceph osd.0 for f1db3478-0243-11ee-9b1d-001a4aab830c. 2023-06-03T19:46:47.280 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:46:47 smithi089 bash[78452]: cluster 1685821606.0884979 mgr.y (mgr.34104) 232 : cluster 0 pgmap v177: 89 pgs: 15 active+undersized+degraded, 19 active+undersized, 55 active+clean; 697 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 87 MiB/s wr, 1.39k op/s; 5420/33495 objects degraded (16.182%); 8.0 KiB/s, 0 objects/s recovering 2023-06-03T19:46:47.280 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:46:47 smithi089 bash[78452]: audit 1685821607.039108 mon.a (mon.0) 212 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:46:47.281 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:46:47 smithi089 bash[78452]: audit 1685821607.0500937 mon.a (mon.0) 213 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:46:47.281 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:46:47 smithi089 bash[78452]: audit 1685821607.0961177 mon.c (mon.2) 78 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-03T19:46:47.281 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:46:47 smithi089 bash[75068]: cluster 1685821606.0884979 mgr.y (mgr.34104) 232 : cluster 0 pgmap v177: 89 pgs: 15 active+undersized+degraded, 19 active+undersized, 55 active+clean; 697 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 87 MiB/s wr, 1.39k op/s; 5420/33495 objects degraded (16.182%); 8.0 KiB/s, 0 objects/s recovering 2023-06-03T19:46:47.282 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:46:47 smithi089 bash[75068]: audit 1685821607.039108 mon.a (mon.0) 212 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:46:47.282 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:46:47 smithi089 bash[75068]: audit 1685821607.0500937 mon.a (mon.0) 213 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:46:47.282 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:46:47 smithi089 bash[75068]: audit 1685821607.0961177 mon.c (mon.2) 78 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-03T19:46:47.536 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 9 15 20497 20482 142.223 132.875 0.00774709 0.00702434 2023-06-03T19:46:47.585 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:46:47 smithi164 bash[59462]: cluster 1685821606.0884979 mgr.y (mgr.34104) 232 : cluster 0 pgmap v177: 89 pgs: 15 active+undersized+degraded, 19 active+undersized, 55 active+clean; 697 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 87 MiB/s wr, 1.39k op/s; 5420/33495 objects degraded (16.182%); 8.0 KiB/s, 0 objects/s recovering 2023-06-03T19:46:47.586 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:46:47 smithi164 bash[59462]: audit 1685821607.039108 mon.a (mon.0) 212 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:46:47.586 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:46:47 smithi164 bash[59462]: audit 1685821607.0500937 mon.a (mon.0) 213 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:46:47.586 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:46:47 smithi164 bash[59462]: audit 1685821607.0961177 mon.c (mon.2) 78 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-03T19:46:48.536 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 10 16 22795 22779 142.356 143.562 0.00617866 0.0070183 2023-06-03T19:46:49.530 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:46:49 smithi089 bash[78452]: cluster 1685821608.0891662 mgr.y (mgr.34104) 233 : cluster 0 pgmap v178: 89 pgs: 15 active+undersized+degraded, 19 active+undersized, 55 active+clean; 697 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 86 MiB/s wr, 1.37k op/s; 5420/33495 objects degraded (16.182%); 8.0 KiB/s, 0 objects/s recovering 2023-06-03T19:46:49.530 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:46:49 smithi089 bash[78452]: audit 1685821608.533015 mon.a (mon.0) 214 : audit 0 from='client.? 172.21.15.89:0/4293346782' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:46:49.530 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:46:49 smithi089 bash[75068]: cluster 1685821608.0891662 mgr.y (mgr.34104) 233 : cluster 0 pgmap v178: 89 pgs: 15 active+undersized+degraded, 19 active+undersized, 55 active+clean; 697 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 86 MiB/s wr, 1.37k op/s; 5420/33495 objects degraded (16.182%); 8.0 KiB/s, 0 objects/s recovering 2023-06-03T19:46:49.530 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:46:49 smithi089 bash[75068]: audit 1685821608.533015 mon.a (mon.0) 214 : audit 0 from='client.? 172.21.15.89:0/4293346782' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:46:49.536 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 11 16 24979 24963 141.822 136.5 0.00936883 0.00704485 2023-06-03T19:46:49.585 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:46:49 smithi164 bash[59462]: cluster 1685821608.0891662 mgr.y (mgr.34104) 233 : cluster 0 pgmap v178: 89 pgs: 15 active+undersized+degraded, 19 active+undersized, 55 active+clean; 697 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 86 MiB/s wr, 1.37k op/s; 5420/33495 objects degraded (16.182%); 8.0 KiB/s, 0 objects/s recovering 2023-06-03T19:46:49.585 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:46:49 smithi164 bash[59462]: audit 1685821608.533015 mon.a (mon.0) 214 : audit 0 from='client.? 172.21.15.89:0/4293346782' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:46:50.280 INFO:journalctl@ceph.osd.0.smithi089.stdout:Jun 03 19:46:49 smithi089 bash[84715]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0 2023-06-03T19:46:50.280 INFO:journalctl@ceph.osd.0.smithi089.stdout:Jun 03 19:46:49 smithi089 bash[84715]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-0 --no-mon-config --dev /dev/mapper/vg_nvme-lv_4 2023-06-03T19:46:50.280 INFO:journalctl@ceph.osd.0.smithi089.stdout:Jun 03 19:46:49 smithi089 bash[84715]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_4 2023-06-03T19:46:50.280 INFO:journalctl@ceph.osd.0.smithi089.stdout:Jun 03 19:46:49 smithi089 bash[84715]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-3 2023-06-03T19:46:50.280 INFO:journalctl@ceph.osd.0.smithi089.stdout:Jun 03 19:46:49 smithi089 bash[84715]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_4 /var/lib/ceph/osd/ceph-0/block 2023-06-03T19:46:50.281 INFO:journalctl@ceph.osd.0.smithi089.stdout:Jun 03 19:46:49 smithi089 bash[84715]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0 2023-06-03T19:46:50.281 INFO:journalctl@ceph.osd.0.smithi089.stdout:Jun 03 19:46:49 smithi089 bash[84715]: --> ceph-volume raw activate successful for osd ID: 0 2023-06-03T19:46:50.530 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:46:50 smithi089 bash[75068]: cluster 1685821610.277751 mon.a (mon.0) 215 : cluster 3 Health check update: Degraded data redundancy: 11072/67470 objects degraded (16.410%), 15 pgs degraded (PG_DEGRADED) 2023-06-03T19:46:50.530 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:46:50 smithi089 bash[78452]: cluster 1685821610.277751 mon.a (mon.0) 215 : cluster 3 Health check update: Degraded data redundancy: 11072/67470 objects degraded (16.410%), 15 pgs degraded (PG_DEGRADED) 2023-06-03T19:46:50.536 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 12 16 27042 27026 140.748 128.938 0.011648 0.00709937 2023-06-03T19:46:50.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:46:50 smithi164 bash[59462]: cluster 1685821610.277751 mon.a (mon.0) 215 : cluster 3 Health check update: Degraded data redundancy: 11072/67470 objects degraded (16.410%), 15 pgs degraded (PG_DEGRADED) 2023-06-03T19:46:51.536 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 13 16 29129 29113 139.954 130.438 0.00718407 0.0071376 2023-06-03T19:46:51.780 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:46:51 smithi089 bash[78452]: cluster 1685821610.0899649 mgr.y (mgr.34104) 234 : cluster 0 pgmap v179: 89 pgs: 15 active+undersized+degraded, 19 active+undersized, 55 active+clean; 1.4 GiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 141 MiB/s wr, 2.26k op/s; 11072/67470 objects degraded (16.410%); 6.5 KiB/s, 0 objects/s recovering 2023-06-03T19:46:51.780 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:46:51 smithi089 bash[78452]: audit 1685821611.1139567 mon.a (mon.0) 216 : audit 0 from='client.? 172.21.15.89:0/3860416265' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:46:51.780 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:46:51 smithi089 bash[75068]: cluster 1685821610.0899649 mgr.y (mgr.34104) 234 : cluster 0 pgmap v179: 89 pgs: 15 active+undersized+degraded, 19 active+undersized, 55 active+clean; 1.4 GiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 141 MiB/s wr, 2.26k op/s; 11072/67470 objects degraded (16.410%); 6.5 KiB/s, 0 objects/s recovering 2023-06-03T19:46:51.780 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:46:51 smithi089 bash[75068]: audit 1685821611.1139567 mon.a (mon.0) 216 : audit 0 from='client.? 172.21.15.89:0/3860416265' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:46:51.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:46:51 smithi164 bash[59462]: cluster 1685821610.0899649 mgr.y (mgr.34104) 234 : cluster 0 pgmap v179: 89 pgs: 15 active+undersized+degraded, 19 active+undersized, 55 active+clean; 1.4 GiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 141 MiB/s wr, 2.26k op/s; 11072/67470 objects degraded (16.410%); 6.5 KiB/s, 0 objects/s recovering 2023-06-03T19:46:51.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:46:51 smithi164 bash[59462]: audit 1685821611.1139567 mon.a (mon.0) 216 : audit 0 from='client.? 172.21.15.89:0/3860416265' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:46:52.536 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 14 16 31220 31204 139.291 130.688 0.00618756 0.0071737 2023-06-03T19:46:53.357 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:46:53 smithi089 bash[78452]: audit 1685821612.0745053 mon.a (mon.0) 217 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:46:53.358 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:46:53 smithi089 bash[78452]: audit 1685821612.0790377 mon.c (mon.2) 79 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T19:46:53.358 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:46:53 smithi089 bash[78452]: cluster 1685821612.0948646 mgr.y (mgr.34104) 235 : cluster 0 pgmap v180: 89 pgs: 15 active+undersized+degraded, 19 active+undersized, 55 active+clean; 1.4 GiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 128 MiB/s wr, 2.05k op/s; 11072/67470 objects degraded (16.410%); 5.9 KiB/s, 0 objects/s recovering 2023-06-03T19:46:53.358 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:46:53 smithi089 bash[75068]: audit 1685821612.0745053 mon.a (mon.0) 217 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:46:53.358 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:46:53 smithi089 bash[75068]: audit 1685821612.0790377 mon.c (mon.2) 79 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T19:46:53.359 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:46:53 smithi089 bash[75068]: cluster 1685821612.0948646 mgr.y (mgr.34104) 235 : cluster 0 pgmap v180: 89 pgs: 15 active+undersized+degraded, 19 active+undersized, 55 active+clean; 1.4 GiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 128 MiB/s wr, 2.05k op/s; 11072/67470 objects degraded (16.410%); 5.9 KiB/s, 0 objects/s recovering 2023-06-03T19:46:53.536 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 15 16 33156 33140 138.071 121 0.00532964 0.00723543 2023-06-03T19:46:53.585 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:46:53 smithi164 bash[59462]: audit 1685821612.0745053 mon.a (mon.0) 217 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:46:53.586 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:46:53 smithi164 bash[59462]: audit 1685821612.0790377 mon.c (mon.2) 79 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T19:46:53.586 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:46:53 smithi164 bash[59462]: cluster 1685821612.0948646 mgr.y (mgr.34104) 235 : cluster 0 pgmap v180: 89 pgs: 15 active+undersized+degraded, 19 active+undersized, 55 active+clean; 1.4 GiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 128 MiB/s wr, 2.05k op/s; 11072/67470 objects degraded (16.410%); 5.9 KiB/s, 0 objects/s recovering 2023-06-03T19:46:53.779 INFO:journalctl@ceph.osd.0.smithi089.stdout:Jun 03 19:46:53 smithi089 bash[85044]: debug 2023-06-03T19:46:53.347+0000 7f48a6b5d700 1 -- 172.21.15.89:0/2761777004 <== mon.0 v2:172.21.15.89:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x55ac92eead00 con 0x55ac92f45000 2023-06-03T19:46:54.279 INFO:journalctl@ceph.osd.0.smithi089.stdout:Jun 03 19:46:53 smithi089 bash[85044]: debug 2023-06-03T19:46:53.915+0000 7f48afce3640 -1 Falling back to public interface 2023-06-03T19:46:54.280 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:46:54 smithi089 bash[75068]: audit 1685821613.7641943 mon.c (mon.2) 80 : audit 0 from='client.? 172.21.15.89:0/3015516870' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:46:54.281 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:46:54 smithi089 bash[78452]: audit 1685821613.7641943 mon.c (mon.2) 80 : audit 0 from='client.? 172.21.15.89:0/3015516870' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:46:54.537 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 16 16 35057 35041 136.866 118.812 0.00745914 0.00730081 2023-06-03T19:46:54.585 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:46:54 smithi164 bash[59462]: audit 1685821613.7641943 mon.c (mon.2) 80 : audit 0 from='client.? 172.21.15.89:0/3015516870' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:46:55.530 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:46:55 smithi089 bash[75068]: cluster 1685821614.09576 mgr.y (mgr.34104) 236 : cluster 0 pgmap v181: 89 pgs: 15 active+undersized+degraded, 19 active+undersized, 55 active+clean; 2.0 GiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 170 MiB/s wr, 2.73k op/s; 16371/98727 objects degraded (16.582%); 5.3 KiB/s, 0 objects/s recovering 2023-06-03T19:46:55.530 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:46:55 smithi089 bash[78452]: cluster 1685821614.09576 mgr.y (mgr.34104) 236 : cluster 0 pgmap v181: 89 pgs: 15 active+undersized+degraded, 19 active+undersized, 55 active+clean; 2.0 GiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 170 MiB/s wr, 2.73k op/s; 16371/98727 objects degraded (16.582%); 5.3 KiB/s, 0 objects/s recovering 2023-06-03T19:46:55.538 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 17 16 36798 36782 135.216 108.812 0.0147629 0.00738871 2023-06-03T19:46:55.585 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:46:55 smithi164 bash[59462]: cluster 1685821614.09576 mgr.y (mgr.34104) 236 : cluster 0 pgmap v181: 89 pgs: 15 active+undersized+degraded, 19 active+undersized, 55 active+clean; 2.0 GiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 170 MiB/s wr, 2.73k op/s; 16371/98727 objects degraded (16.582%); 5.3 KiB/s, 0 objects/s recovering 2023-06-03T19:46:56.530 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 19:46:56 smithi089 bash[71163]: ::ffff:172.21.15.164 - - [03/Jun/2023:19:46:56] "GET /metrics HTTP/1.1" 200 190057 "" "Prometheus/2.33.4" 2023-06-03T19:46:56.537 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 18 16 38261 38245 132.783 91.4375 0.0129022 0.00752522 2023-06-03T19:46:57.529 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:46:57 smithi089 bash[75068]: cluster 1685821616.0963624 mgr.y (mgr.34104) 237 : cluster 0 pgmap v182: 89 pgs: 15 active+undersized+degraded, 19 active+undersized, 55 active+clean; 2.0 GiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 113 MiB/s wr, 1.81k op/s; 16371/98727 objects degraded (16.582%) 2023-06-03T19:46:57.530 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:46:57 smithi089 bash[75068]: audit 1685821616.3134613 mon.b (mon.1) 4 : audit 0 from='client.? 172.21.15.89:0/3584850182' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:46:57.530 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:46:57 smithi089 bash[75068]: audit 1685821616.566244 mon.a (mon.0) 218 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:46:57.530 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:46:57 smithi089 bash[75068]: audit 1685821616.586652 mon.a (mon.0) 219 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:46:57.530 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:46:57 smithi089 bash[78452]: cluster 1685821616.0963624 mgr.y (mgr.34104) 237 : cluster 0 pgmap v182: 89 pgs: 15 active+undersized+degraded, 19 active+undersized, 55 active+clean; 2.0 GiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 113 MiB/s wr, 1.81k op/s; 16371/98727 objects degraded (16.582%) 2023-06-03T19:46:57.531 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:46:57 smithi089 bash[78452]: audit 1685821616.3134613 mon.b (mon.1) 4 : audit 0 from='client.? 172.21.15.89:0/3584850182' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:46:57.531 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:46:57 smithi089 bash[78452]: audit 1685821616.566244 mon.a (mon.0) 218 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:46:57.531 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:46:57 smithi089 bash[78452]: audit 1685821616.586652 mon.a (mon.0) 219 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:46:57.537 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 19 16 39809 39793 130.886 96.75 0.0118 0.00763383 2023-06-03T19:46:57.585 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:46:57 smithi164 bash[59462]: cluster 1685821616.0963624 mgr.y (mgr.34104) 237 : cluster 0 pgmap v182: 89 pgs: 15 active+undersized+degraded, 19 active+undersized, 55 active+clean; 2.0 GiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 113 MiB/s wr, 1.81k op/s; 16371/98727 objects degraded (16.582%) 2023-06-03T19:46:57.586 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:46:57 smithi164 bash[59462]: audit 1685821616.3134613 mon.b (mon.1) 4 : audit 0 from='client.? 172.21.15.89:0/3584850182' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:46:57.586 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:46:57 smithi164 bash[59462]: audit 1685821616.566244 mon.a (mon.0) 218 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:46:57.586 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:46:57 smithi164 bash[59462]: audit 1685821616.586652 mon.a (mon.0) 219 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:46:58.529 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:46:58 smithi089 bash[78452]: cluster 1685821617.2040706 mon.a (mon.0) 220 : cluster 3 Health check update: Degraded data redundancy: 16371/98727 objects degraded (16.582%), 15 pgs degraded (PG_DEGRADED) 2023-06-03T19:46:58.530 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:46:58 smithi089 bash[78452]: audit 1685821617.6373715 mon.a (mon.0) 221 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:46:58.530 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:46:58 smithi089 bash[78452]: audit 1685821617.6488867 mon.a (mon.0) 222 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:46:58.530 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:46:58 smithi089 bash[75068]: cluster 1685821617.2040706 mon.a (mon.0) 220 : cluster 3 Health check update: Degraded data redundancy: 16371/98727 objects degraded (16.582%), 15 pgs degraded (PG_DEGRADED) 2023-06-03T19:46:58.530 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:46:58 smithi089 bash[75068]: audit 1685821617.6373715 mon.a (mon.0) 221 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:46:58.530 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:46:58 smithi089 bash[75068]: audit 1685821617.6488867 mon.a (mon.0) 222 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:46:58.537 INFO:tasks.radosbench.radosbench.0.smithi089.stdout:2023-06-03T19:46:58.532266+0000 min lat: 0.00135761 max lat: 1.02386 avg lat: 0.00772416 2023-06-03T19:46:58.537 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-06-03T19:46:58.537 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 20 16 41412 41396 129.351 100.188 0.00873794 0.00772416 2023-06-03T19:46:58.585 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:46:58 smithi164 bash[59462]: cluster 1685821617.2040706 mon.a (mon.0) 220 : cluster 3 Health check update: Degraded data redundancy: 16371/98727 objects degraded (16.582%), 15 pgs degraded (PG_DEGRADED) 2023-06-03T19:46:58.585 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:46:58 smithi164 bash[59462]: audit 1685821617.6373715 mon.a (mon.0) 221 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:46:58.586 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:46:58 smithi164 bash[59462]: audit 1685821617.6488867 mon.a (mon.0) 222 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:46:59.537 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 21 16 43048 43032 128.059 102.25 0.0099247 0.00780315 2023-06-03T19:47:00.030 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:46:59 smithi089 bash[75068]: cluster 1685821618.0971181 mgr.y (mgr.34104) 238 : cluster 0 pgmap v183: 89 pgs: 15 active+undersized+degraded, 19 active+undersized, 55 active+clean; 2.0 GiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 113 MiB/s wr, 1.81k op/s; 16371/98727 objects degraded (16.582%) 2023-06-03T19:47:00.030 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:46:59 smithi089 bash[75068]: audit 1685821618.9562895 mon.c (mon.2) 81 : audit 0 from='client.? 172.21.15.89:0/2420936307' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:47:00.030 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:46:59 smithi089 bash[78452]: cluster 1685821618.0971181 mgr.y (mgr.34104) 238 : cluster 0 pgmap v183: 89 pgs: 15 active+undersized+degraded, 19 active+undersized, 55 active+clean; 2.0 GiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 113 MiB/s wr, 1.81k op/s; 16371/98727 objects degraded (16.582%) 2023-06-03T19:47:00.031 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:46:59 smithi089 bash[78452]: audit 1685821618.9562895 mon.c (mon.2) 81 : audit 0 from='client.? 172.21.15.89:0/2420936307' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:47:00.085 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:46:59 smithi164 bash[59462]: cluster 1685821618.0971181 mgr.y (mgr.34104) 238 : cluster 0 pgmap v183: 89 pgs: 15 active+undersized+degraded, 19 active+undersized, 55 active+clean; 2.0 GiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 113 MiB/s wr, 1.81k op/s; 16371/98727 objects degraded (16.582%) 2023-06-03T19:47:00.086 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:46:59 smithi164 bash[59462]: audit 1685821618.9562895 mon.c (mon.2) 81 : audit 0 from='client.? 172.21.15.89:0/2420936307' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:47:00.537 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 22 16 44703 44687 126.94 103.438 0.0132301 0.00787234 2023-06-03T19:47:01.537 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 23 16 46185 46169 125.447 92.625 0.00413225 0.00796124 2023-06-03T19:47:02.030 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:47:01 smithi089 bash[75068]: cluster 1685821620.098036 mgr.y (mgr.34104) 239 : cluster 0 pgmap v184: 89 pgs: 15 active+undersized+degraded, 19 active+undersized, 55 active+clean; 2.5 GiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 156 MiB/s wr, 2.50k op/s; 20490/123594 objects degraded (16.578%) 2023-06-03T19:47:02.030 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:47:01 smithi089 bash[75068]: audit 1685821621.5654209 mon.a (mon.0) 223 : audit 0 from='client.? 172.21.15.89:0/3195236100' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:47:02.031 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:47:01 smithi089 bash[78452]: cluster 1685821620.098036 mgr.y (mgr.34104) 239 : cluster 0 pgmap v184: 89 pgs: 15 active+undersized+degraded, 19 active+undersized, 55 active+clean; 2.5 GiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 156 MiB/s wr, 2.50k op/s; 20490/123594 objects degraded (16.578%) 2023-06-03T19:47:02.031 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:47:01 smithi089 bash[78452]: audit 1685821621.5654209 mon.a (mon.0) 223 : audit 0 from='client.? 172.21.15.89:0/3195236100' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:47:02.085 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:47:01 smithi164 bash[59462]: cluster 1685821620.098036 mgr.y (mgr.34104) 239 : cluster 0 pgmap v184: 89 pgs: 15 active+undersized+degraded, 19 active+undersized, 55 active+clean; 2.5 GiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 156 MiB/s wr, 2.50k op/s; 20490/123594 objects degraded (16.578%) 2023-06-03T19:47:02.085 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:47:01 smithi164 bash[59462]: audit 1685821621.5654209 mon.a (mon.0) 223 : audit 0 from='client.? 172.21.15.89:0/3195236100' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:47:02.537 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 24 16 47734 47718 124.254 96.8125 0.00330229 0.00804292 2023-06-03T19:47:03.029 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:47:02 smithi089 bash[78452]: cluster 1685821622.2060163 mon.a (mon.0) 224 : cluster 3 Health check update: Degraded data redundancy: 20490/123594 objects degraded (16.578%), 15 pgs degraded (PG_DEGRADED) 2023-06-03T19:47:03.030 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:47:02 smithi089 bash[75068]: cluster 1685821622.2060163 mon.a (mon.0) 224 : cluster 3 Health check update: Degraded data redundancy: 20490/123594 objects degraded (16.578%), 15 pgs degraded (PG_DEGRADED) 2023-06-03T19:47:03.085 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:47:02 smithi164 bash[59462]: cluster 1685821622.2060163 mon.a (mon.0) 224 : cluster 3 Health check update: Degraded data redundancy: 20490/123594 objects degraded (16.578%), 15 pgs degraded (PG_DEGRADED) 2023-06-03T19:47:03.537 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 25 16 49203 49187 122.956 91.8125 0.00625486 0.00812583 2023-06-03T19:47:04.029 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:47:03 smithi089 bash[75068]: cluster 1685821622.0986814 mgr.y (mgr.34104) 240 : cluster 0 pgmap v185: 89 pgs: 15 active+undersized+degraded, 19 active+undersized, 55 active+clean; 2.5 GiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 97 MiB/s wr, 1.56k op/s; 20490/123594 objects degraded (16.578%) 2023-06-03T19:47:04.030 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:47:03 smithi089 bash[78452]: cluster 1685821622.0986814 mgr.y (mgr.34104) 240 : cluster 0 pgmap v185: 89 pgs: 15 active+undersized+degraded, 19 active+undersized, 55 active+clean; 2.5 GiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 97 MiB/s wr, 1.56k op/s; 20490/123594 objects degraded (16.578%) 2023-06-03T19:47:04.085 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:47:03 smithi164 bash[59462]: cluster 1685821622.0986814 mgr.y (mgr.34104) 240 : cluster 0 pgmap v185: 89 pgs: 15 active+undersized+degraded, 19 active+undersized, 55 active+clean; 2.5 GiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 97 MiB/s wr, 1.56k op/s; 20490/123594 objects degraded (16.578%) 2023-06-03T19:47:04.537 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 26 16 50632 50616 121.662 89.3125 0.0102891 0.00821395 2023-06-03T19:47:04.685 INFO:journalctl@ceph.osd.0.smithi089.stdout:Jun 03 19:47:04 smithi089 bash[85044]: debug 2023-06-03T19:47:04.283+0000 7f48afce3640 -1 osd.0 82 log_to_monitors true 2023-06-03T19:47:05.029 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:47:04 smithi089 bash[75068]: audit 1685821624.204 mon.a (mon.0) 225 : audit 0 from='client.? 172.21.15.89:0/1661158640' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:47:05.030 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:47:04 smithi089 bash[75068]: audit 1685821624.2949932 mon.a (mon.0) 226 : audit 1 from='osd.0 [v2:172.21.15.89:6802/2920103307,v1:172.21.15.89:6803/2920103307]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2023-06-03T19:47:05.030 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:47:04 smithi089 bash[78452]: audit 1685821624.204 mon.a (mon.0) 225 : audit 0 from='client.? 172.21.15.89:0/1661158640' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:47:05.030 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:47:04 smithi089 bash[78452]: audit 1685821624.2949932 mon.a (mon.0) 226 : audit 1 from='osd.0 [v2:172.21.15.89:6802/2920103307,v1:172.21.15.89:6803/2920103307]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2023-06-03T19:47:05.031 INFO:journalctl@ceph.osd.0.smithi089.stdout:Jun 03 19:47:04 smithi089 bash[85044]: debug 2023-06-03T19:47:04.727+0000 7f489d147700 -1 osd.0 82 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2023-06-03T19:47:05.085 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:47:04 smithi164 bash[59462]: audit 1685821624.204 mon.a (mon.0) 225 : audit 0 from='client.? 172.21.15.89:0/1661158640' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:47:05.086 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:47:04 smithi164 bash[59462]: audit 1685821624.2949932 mon.a (mon.0) 226 : audit 1 from='osd.0 [v2:172.21.15.89:6802/2920103307,v1:172.21.15.89:6803/2920103307]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2023-06-03T19:47:05.537 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 27 16 52237 52221 120.871 100.312 0.00683611 0.00826841 2023-06-03T19:47:06.029 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:47:05 smithi089 bash[75068]: cluster 1685821624.099652 mgr.y (mgr.34104) 241 : cluster 0 pgmap v186: 89 pgs: 15 active+undersized+degraded, 19 active+undersized, 55 active+clean; 3.0 GiB data, 8.8 GiB used, 706 GiB / 715 GiB avail; 138 MiB/s wr, 2.21k op/s; 24410/147003 objects degraded (16.605%) 2023-06-03T19:47:06.030 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:47:05 smithi089 bash[75068]: cluster 1685821624.677357 osd.4 (osd.4) 9 : cluster 3 failed to encode map e85 with expected crc 2023-06-03T19:47:06.030 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:47:05 smithi089 bash[75068]: cluster 1685821624.686003 osd.2 (osd.2) 11 : cluster 3 failed to encode map e85 with expected crc 2023-06-03T19:47:06.030 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:47:05 smithi089 bash[75068]: audit 1685821624.688344 mon.a (mon.0) 227 : audit 1 from='osd.0 [v2:172.21.15.89:6802/2920103307,v1:172.21.15.89:6803/2920103307]' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2023-06-03T19:47:06.030 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:47:05 smithi089 bash[75068]: cluster 1685821624.6885774 mon.a (mon.0) 228 : cluster 0 osdmap e85: 8 total, 7 up, 8 in 2023-06-03T19:47:06.030 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:47:05 smithi089 bash[75068]: audit 1685821624.692304 mon.a (mon.0) 229 : audit 1 from='osd.0 [v2:172.21.15.89:6802/2920103307,v1:172.21.15.89:6803/2920103307]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi089", "root=default"]}]: dispatch 2023-06-03T19:47:06.030 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:47:05 smithi089 bash[75068]: cluster 1685821624.7048728 osd.1 (osd.1) 10 : cluster 3 failed to encode map e85 with expected crc 2023-06-03T19:47:06.031 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:47:05 smithi089 bash[78452]: cluster 1685821624.099652 mgr.y (mgr.34104) 241 : cluster 0 pgmap v186: 89 pgs: 15 active+undersized+degraded, 19 active+undersized, 55 active+clean; 3.0 GiB data, 8.8 GiB used, 706 GiB / 715 GiB avail; 138 MiB/s wr, 2.21k op/s; 24410/147003 objects degraded (16.605%) 2023-06-03T19:47:06.031 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:47:05 smithi089 bash[78452]: cluster 1685821624.677357 osd.4 (osd.4) 9 : cluster 3 failed to encode map e85 with expected crc 2023-06-03T19:47:06.031 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:47:05 smithi089 bash[78452]: cluster 1685821624.686003 osd.2 (osd.2) 11 : cluster 3 failed to encode map e85 with expected crc 2023-06-03T19:47:06.031 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:47:05 smithi089 bash[78452]: audit 1685821624.688344 mon.a (mon.0) 227 : audit 1 from='osd.0 [v2:172.21.15.89:6802/2920103307,v1:172.21.15.89:6803/2920103307]' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2023-06-03T19:47:06.031 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:47:05 smithi089 bash[78452]: cluster 1685821624.6885774 mon.a (mon.0) 228 : cluster 0 osdmap e85: 8 total, 7 up, 8 in 2023-06-03T19:47:06.031 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:47:05 smithi089 bash[78452]: audit 1685821624.692304 mon.a (mon.0) 229 : audit 1 from='osd.0 [v2:172.21.15.89:6802/2920103307,v1:172.21.15.89:6803/2920103307]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi089", "root=default"]}]: dispatch 2023-06-03T19:47:06.032 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:47:05 smithi089 bash[78452]: cluster 1685821624.7048728 osd.1 (osd.1) 10 : cluster 3 failed to encode map e85 with expected crc 2023-06-03T19:47:06.085 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:47:05 smithi164 bash[59462]: cluster 1685821624.099652 mgr.y (mgr.34104) 241 : cluster 0 pgmap v186: 89 pgs: 15 active+undersized+degraded, 19 active+undersized, 55 active+clean; 3.0 GiB data, 8.8 GiB used, 706 GiB / 715 GiB avail; 138 MiB/s wr, 2.21k op/s; 24410/147003 objects degraded (16.605%) 2023-06-03T19:47:06.086 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:47:05 smithi164 bash[59462]: cluster 1685821624.677357 osd.4 (osd.4) 9 : cluster 3 failed to encode map e85 with expected crc 2023-06-03T19:47:06.086 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:47:05 smithi164 bash[59462]: cluster 1685821624.686003 osd.2 (osd.2) 11 : cluster 3 failed to encode map e85 with expected crc 2023-06-03T19:47:06.086 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:47:05 smithi164 bash[59462]: audit 1685821624.688344 mon.a (mon.0) 227 : audit 1 from='osd.0 [v2:172.21.15.89:6802/2920103307,v1:172.21.15.89:6803/2920103307]' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2023-06-03T19:47:06.086 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:47:05 smithi164 bash[59462]: cluster 1685821624.6885774 mon.a (mon.0) 228 : cluster 0 osdmap e85: 8 total, 7 up, 8 in 2023-06-03T19:47:06.086 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:47:05 smithi164 bash[59462]: audit 1685821624.692304 mon.a (mon.0) 229 : audit 1 from='osd.0 [v2:172.21.15.89:6802/2920103307,v1:172.21.15.89:6803/2920103307]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi089", "root=default"]}]: dispatch 2023-06-03T19:47:06.086 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:47:05 smithi164 bash[59462]: cluster 1685821624.7048728 osd.1 (osd.1) 10 : cluster 3 failed to encode map e85 with expected crc 2023-06-03T19:47:06.529 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 19:47:06 smithi089 bash[71163]: ::ffff:172.21.15.164 - - [03/Jun/2023:19:47:06] "GET /metrics HTTP/1.1" 200 190057 "" "Prometheus/2.33.4" 2023-06-03T19:47:06.537 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 28 16 52549 52533 117.25 19.5 0.00436087 0.00827757 2023-06-03T19:47:07.029 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:47:06 smithi089 bash[78452]: cluster 1685821625.6801386 mon.a (mon.0) 230 : cluster 1 Health check cleared: OSD_DOWN (was: 1 osds down) 2023-06-03T19:47:07.029 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:47:06 smithi089 bash[78452]: cluster 1685821625.700591 osd.1 (osd.1) 11 : cluster 3 failed to encode map e86 with expected crc 2023-06-03T19:47:07.029 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:47:06 smithi089 bash[78452]: cluster 1685821625.7012258 osd.5 (osd.5) 11 : cluster 3 failed to encode map e86 with expected crc 2023-06-03T19:47:07.029 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:47:06 smithi089 bash[78452]: cluster 1685821625.7032588 mon.a (mon.0) 231 : cluster 1 osd.0 [v2:172.21.15.89:6802/2920103307,v1:172.21.15.89:6803/2920103307] boot 2023-06-03T19:47:07.029 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:47:06 smithi089 bash[78452]: cluster 1685821625.7033813 mon.a (mon.0) 232 : cluster 0 osdmap e86: 8 total, 8 up, 8 in 2023-06-03T19:47:07.030 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:47:06 smithi089 bash[78452]: audit 1685821625.7085419 mon.c (mon.2) 82 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-06-03T19:47:07.030 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:47:06 smithi089 bash[78452]: audit 1685821626.5673048 mon.c (mon.2) 83 : audit 0 from='client.? 172.21.15.89:0/1420563662' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:47:07.030 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:47:06 smithi089 bash[75068]: cluster 1685821625.6801386 mon.a (mon.0) 230 : cluster 1 Health check cleared: OSD_DOWN (was: 1 osds down) 2023-06-03T19:47:07.030 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:47:06 smithi089 bash[75068]: cluster 1685821625.700591 osd.1 (osd.1) 11 : cluster 3 failed to encode map e86 with expected crc 2023-06-03T19:47:07.030 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:47:06 smithi089 bash[75068]: cluster 1685821625.7012258 osd.5 (osd.5) 11 : cluster 3 failed to encode map e86 with expected crc 2023-06-03T19:47:07.030 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:47:06 smithi089 bash[75068]: cluster 1685821625.7032588 mon.a (mon.0) 231 : cluster 1 osd.0 [v2:172.21.15.89:6802/2920103307,v1:172.21.15.89:6803/2920103307] boot 2023-06-03T19:47:07.031 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:47:06 smithi089 bash[75068]: cluster 1685821625.7033813 mon.a (mon.0) 232 : cluster 0 osdmap e86: 8 total, 8 up, 8 in 2023-06-03T19:47:07.031 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:47:06 smithi089 bash[75068]: audit 1685821625.7085419 mon.c (mon.2) 82 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-06-03T19:47:07.031 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:47:06 smithi089 bash[75068]: audit 1685821626.5673048 mon.c (mon.2) 83 : audit 0 from='client.? 172.21.15.89:0/1420563662' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:47:07.085 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:47:06 smithi164 bash[59462]: cluster 1685821625.6801386 mon.a (mon.0) 230 : cluster 1 Health check cleared: OSD_DOWN (was: 1 osds down) 2023-06-03T19:47:07.086 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:47:06 smithi164 bash[59462]: cluster 1685821625.700591 osd.1 (osd.1) 11 : cluster 3 failed to encode map e86 with expected crc 2023-06-03T19:47:07.086 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:47:06 smithi164 bash[59462]: cluster 1685821625.7012258 osd.5 (osd.5) 11 : cluster 3 failed to encode map e86 with expected crc 2023-06-03T19:47:07.086 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:47:06 smithi164 bash[59462]: cluster 1685821625.7032588 mon.a (mon.0) 231 : cluster 1 osd.0 [v2:172.21.15.89:6802/2920103307,v1:172.21.15.89:6803/2920103307] boot 2023-06-03T19:47:07.087 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:47:06 smithi164 bash[59462]: cluster 1685821625.7033813 mon.a (mon.0) 232 : cluster 0 osdmap e86: 8 total, 8 up, 8 in 2023-06-03T19:47:07.087 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:47:06 smithi164 bash[59462]: audit 1685821625.7085419 mon.c (mon.2) 82 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-06-03T19:47:07.087 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:47:06 smithi164 bash[59462]: audit 1685821626.5673048 mon.c (mon.2) 83 : audit 0 from='client.? 172.21.15.89:0/1420563662' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:47:07.538 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 29 16 52549 52533 113.207 0 - 0.00827757 2023-06-03T19:47:08.030 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:47:07 smithi089 bash[78452]: cluster 1685821626.1001933 mgr.y (mgr.34104) 242 : cluster 0 pgmap v189: 89 pgs: 15 active+undersized+degraded, 19 active+undersized, 55 active+clean; 3.0 GiB data, 8.8 GiB used, 706 GiB / 715 GiB avail; 126 MiB/s wr, 2.01k op/s; 24410/147003 objects degraded (16.605%) 2023-06-03T19:47:08.030 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:47:07 smithi089 bash[78452]: cluster 1685821626.7115285 osd.6 (osd.6) 8 : cluster 3 failed to encode map e87 with expected crc 2023-06-03T19:47:08.030 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:47:07 smithi089 bash[78452]: cluster 1685821626.714167 osd.6 (osd.6) 9 : cluster 3 failed to encode map e87 with expected crc 2023-06-03T19:47:08.030 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:47:07 smithi089 bash[78452]: cluster 1685821626.7143366 osd.2 (osd.2) 12 : cluster 3 failed to encode map e87 with expected crc 2023-06-03T19:47:08.030 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:47:07 smithi089 bash[78452]: cluster 1685821626.7144568 osd.7 (osd.7) 9 : cluster 3 failed to encode map e87 with expected crc 2023-06-03T19:47:08.030 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:47:07 smithi089 bash[78452]: cluster 1685821626.714463 osd.4 (osd.4) 10 : cluster 3 failed to encode map e87 with expected crc 2023-06-03T19:47:08.031 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:47:07 smithi089 bash[78452]: cluster 1685821626.7167988 mon.a (mon.0) 233 : cluster 0 osdmap e87: 8 total, 8 up, 8 in 2023-06-03T19:47:08.031 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:47:07 smithi089 bash[78452]: cluster 1685821626.7173119 osd.1 (osd.1) 12 : cluster 3 failed to encode map e87 with expected crc 2023-06-03T19:47:08.031 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:47:07 smithi089 bash[78452]: cluster 1685821626.7193623 osd.7 (osd.7) 10 : cluster 3 failed to encode map e87 with expected crc 2023-06-03T19:47:08.031 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:47:07 smithi089 bash[78452]: cluster 1685821626.7195053 osd.4 (osd.4) 11 : cluster 3 failed to encode map e87 with expected crc 2023-06-03T19:47:08.031 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:47:07 smithi089 bash[78452]: cluster 1685821626.7197394 osd.2 (osd.2) 13 : cluster 3 failed to encode map e87 with expected crc 2023-06-03T19:47:08.031 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:47:07 smithi089 bash[78452]: cluster 1685821626.7226157 osd.7 (osd.7) 11 : cluster 3 failed to encode map e87 with expected crc 2023-06-03T19:47:08.031 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:47:07 smithi089 bash[78452]: audit 1685821627.0740595 mon.a (mon.0) 234 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:47:08.031 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:47:07 smithi089 bash[78452]: audit 1685821627.0760412 mon.c (mon.2) 84 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T19:47:08.032 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:47:07 smithi089 bash[78452]: cluster 1685821627.2073615 mon.a (mon.0) 235 : cluster 3 Health check update: Degraded data redundancy: 24410/147003 objects degraded (16.605%), 15 pgs degraded (PG_DEGRADED) 2023-06-03T19:47:08.032 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:47:07 smithi089 bash[75068]: cluster 1685821626.1001933 mgr.y (mgr.34104) 242 : cluster 0 pgmap v189: 89 pgs: 15 active+undersized+degraded, 19 active+undersized, 55 active+clean; 3.0 GiB data, 8.8 GiB used, 706 GiB / 715 GiB avail; 126 MiB/s wr, 2.01k op/s; 24410/147003 objects degraded (16.605%) 2023-06-03T19:47:08.032 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:47:07 smithi089 bash[75068]: cluster 1685821626.7115285 osd.6 (osd.6) 8 : cluster 3 failed to encode map e87 with expected crc 2023-06-03T19:47:08.032 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:47:07 smithi089 bash[75068]: cluster 1685821626.714167 osd.6 (osd.6) 9 : cluster 3 failed to encode map e87 with expected crc 2023-06-03T19:47:08.032 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:47:07 smithi089 bash[75068]: cluster 1685821626.7143366 osd.2 (osd.2) 12 : cluster 3 failed to encode map e87 with expected crc 2023-06-03T19:47:08.033 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:47:07 smithi089 bash[75068]: cluster 1685821626.7144568 osd.7 (osd.7) 9 : cluster 3 failed to encode map e87 with expected crc 2023-06-03T19:47:08.033 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:47:07 smithi089 bash[75068]: cluster 1685821626.714463 osd.4 (osd.4) 10 : cluster 3 failed to encode map e87 with expected crc 2023-06-03T19:47:08.033 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:47:07 smithi089 bash[75068]: cluster 1685821626.7167988 mon.a (mon.0) 233 : cluster 0 osdmap e87: 8 total, 8 up, 8 in 2023-06-03T19:47:08.033 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:47:07 smithi089 bash[75068]: cluster 1685821626.7173119 osd.1 (osd.1) 12 : cluster 3 failed to encode map e87 with expected crc 2023-06-03T19:47:08.033 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:47:07 smithi089 bash[75068]: cluster 1685821626.7193623 osd.7 (osd.7) 10 : cluster 3 failed to encode map e87 with expected crc 2023-06-03T19:47:08.033 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:47:07 smithi089 bash[75068]: cluster 1685821626.7195053 osd.4 (osd.4) 11 : cluster 3 failed to encode map e87 with expected crc 2023-06-03T19:47:08.033 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:47:07 smithi089 bash[75068]: cluster 1685821626.7197394 osd.2 (osd.2) 13 : cluster 3 failed to encode map e87 with expected crc 2023-06-03T19:47:08.033 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:47:07 smithi089 bash[75068]: cluster 1685821626.7226157 osd.7 (osd.7) 11 : cluster 3 failed to encode map e87 with expected crc 2023-06-03T19:47:08.034 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:47:07 smithi089 bash[75068]: audit 1685821627.0740595 mon.a (mon.0) 234 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:47:08.034 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:47:07 smithi089 bash[75068]: audit 1685821627.0760412 mon.c (mon.2) 84 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T19:47:08.034 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:47:07 smithi089 bash[75068]: cluster 1685821627.2073615 mon.a (mon.0) 235 : cluster 3 Health check update: Degraded data redundancy: 24410/147003 objects degraded (16.605%), 15 pgs degraded (PG_DEGRADED) 2023-06-03T19:47:08.086 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:47:07 smithi164 bash[59462]: cluster 1685821626.1001933 mgr.y (mgr.34104) 242 : cluster 0 pgmap v189: 89 pgs: 15 active+undersized+degraded, 19 active+undersized, 55 active+clean; 3.0 GiB data, 8.8 GiB used, 706 GiB / 715 GiB avail; 126 MiB/s wr, 2.01k op/s; 24410/147003 objects degraded (16.605%) 2023-06-03T19:47:08.086 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:47:07 smithi164 bash[59462]: cluster 1685821626.7115285 osd.6 (osd.6) 8 : cluster 3 failed to encode map e87 with expected crc 2023-06-03T19:47:08.086 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:47:07 smithi164 bash[59462]: cluster 1685821626.714167 osd.6 (osd.6) 9 : cluster 3 failed to encode map e87 with expected crc 2023-06-03T19:47:08.086 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:47:07 smithi164 bash[59462]: cluster 1685821626.7143366 osd.2 (osd.2) 12 : cluster 3 failed to encode map e87 with expected crc 2023-06-03T19:47:08.087 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:47:07 smithi164 bash[59462]: cluster 1685821626.7144568 osd.7 (osd.7) 9 : cluster 3 failed to encode map e87 with expected crc 2023-06-03T19:47:08.087 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:47:07 smithi164 bash[59462]: cluster 1685821626.714463 osd.4 (osd.4) 10 : cluster 3 failed to encode map e87 with expected crc 2023-06-03T19:47:08.087 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:47:07 smithi164 bash[59462]: cluster 1685821626.7167988 mon.a (mon.0) 233 : cluster 0 osdmap e87: 8 total, 8 up, 8 in 2023-06-03T19:47:08.087 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:47:07 smithi164 bash[59462]: cluster 1685821626.7173119 osd.1 (osd.1) 12 : cluster 3 failed to encode map e87 with expected crc 2023-06-03T19:47:08.087 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:47:07 smithi164 bash[59462]: cluster 1685821626.7193623 osd.7 (osd.7) 10 : cluster 3 failed to encode map e87 with expected crc 2023-06-03T19:47:08.087 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:47:07 smithi164 bash[59462]: cluster 1685821626.7195053 osd.4 (osd.4) 11 : cluster 3 failed to encode map e87 with expected crc 2023-06-03T19:47:08.087 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:47:07 smithi164 bash[59462]: cluster 1685821626.7197394 osd.2 (osd.2) 13 : cluster 3 failed to encode map e87 with expected crc 2023-06-03T19:47:08.087 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:47:07 smithi164 bash[59462]: cluster 1685821626.7226157 osd.7 (osd.7) 11 : cluster 3 failed to encode map e87 with expected crc 2023-06-03T19:47:08.088 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:47:07 smithi164 bash[59462]: audit 1685821627.0740595 mon.a (mon.0) 234 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:47:08.088 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:47:07 smithi164 bash[59462]: audit 1685821627.0760412 mon.c (mon.2) 84 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T19:47:08.088 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:47:07 smithi164 bash[59462]: cluster 1685821627.2073615 mon.a (mon.0) 235 : cluster 3 Health check update: Degraded data redundancy: 24410/147003 objects degraded (16.605%), 15 pgs degraded (PG_DEGRADED) 2023-06-03T19:47:08.538 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 30 16 52549 52533 109.433 0 - 0.00827757 2023-06-03T19:47:08.980 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:47:08 smithi089 bash[75068]: cluster 1685821626.717477 osd.3 (osd.3) 12 : cluster 3 failed to encode map e87 with expected crc 2023-06-03T19:47:08.980 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:47:08 smithi089 bash[75068]: cluster 1685821627.7204444 osd.2 (osd.2) 14 : cluster 3 failed to encode map e88 with expected crc 2023-06-03T19:47:08.981 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:47:08 smithi089 bash[75068]: cluster 1685821627.7205513 osd.7 (osd.7) 12 : cluster 3 failed to encode map e88 with expected crc 2023-06-03T19:47:08.981 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:47:08 smithi089 bash[75068]: cluster 1685821627.7206147 osd.6 (osd.6) 10 : cluster 3 failed to encode map e88 with expected crc 2023-06-03T19:47:08.981 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:47:08 smithi089 bash[75068]: cluster 1685821627.7206805 osd.4 (osd.4) 12 : cluster 3 failed to encode map e88 with expected crc 2023-06-03T19:47:08.981 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:47:08 smithi089 bash[75068]: cluster 1685821627.7229154 mon.a (mon.0) 236 : cluster 0 osdmap e88: 8 total, 8 up, 8 in 2023-06-03T19:47:08.981 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:47:08 smithi089 bash[75068]: cluster 1685821627.723332 osd.1 (osd.1) 13 : cluster 3 failed to encode map e88 with expected crc 2023-06-03T19:47:08.981 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:47:08 smithi089 bash[75068]: cluster 1685821628.1010656 mgr.y (mgr.34104) 243 : cluster 0 pgmap v192: 89 pgs: 15 active+undersized+degraded, 19 active+undersized, 55 active+clean; 3.0 GiB data, 8.8 GiB used, 706 GiB / 715 GiB avail; 24410/147003 objects degraded (16.605%) 2023-06-03T19:47:08.982 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:47:08 smithi089 bash[78452]: cluster 1685821626.717477 osd.3 (osd.3) 12 : cluster 3 failed to encode map e87 with expected crc 2023-06-03T19:47:08.982 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:47:08 smithi089 bash[78452]: cluster 1685821627.7204444 osd.2 (osd.2) 14 : cluster 3 failed to encode map e88 with expected crc 2023-06-03T19:47:08.982 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:47:08 smithi089 bash[78452]: cluster 1685821627.7205513 osd.7 (osd.7) 12 : cluster 3 failed to encode map e88 with expected crc 2023-06-03T19:47:08.982 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:47:08 smithi089 bash[78452]: cluster 1685821627.7206147 osd.6 (osd.6) 10 : cluster 3 failed to encode map e88 with expected crc 2023-06-03T19:47:08.982 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:47:08 smithi089 bash[78452]: cluster 1685821627.7206805 osd.4 (osd.4) 12 : cluster 3 failed to encode map e88 with expected crc 2023-06-03T19:47:08.982 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:47:08 smithi089 bash[78452]: cluster 1685821627.7229154 mon.a (mon.0) 236 : cluster 0 osdmap e88: 8 total, 8 up, 8 in 2023-06-03T19:47:08.982 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:47:08 smithi089 bash[78452]: cluster 1685821627.723332 osd.1 (osd.1) 13 : cluster 3 failed to encode map e88 with expected crc 2023-06-03T19:47:08.983 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:47:08 smithi089 bash[78452]: cluster 1685821628.1010656 mgr.y (mgr.34104) 243 : cluster 0 pgmap v192: 89 pgs: 15 active+undersized+degraded, 19 active+undersized, 55 active+clean; 3.0 GiB data, 8.8 GiB used, 706 GiB / 715 GiB avail; 24410/147003 objects degraded (16.605%) 2023-06-03T19:47:09.085 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:47:08 smithi164 bash[59462]: cluster 1685821626.717477 osd.3 (osd.3) 12 : cluster 3 failed to encode map e87 with expected crc 2023-06-03T19:47:09.085 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:47:08 smithi164 bash[59462]: cluster 1685821627.7204444 osd.2 (osd.2) 14 : cluster 3 failed to encode map e88 with expected crc 2023-06-03T19:47:09.086 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:47:08 smithi164 bash[59462]: cluster 1685821627.7205513 osd.7 (osd.7) 12 : cluster 3 failed to encode map e88 with expected crc 2023-06-03T19:47:09.086 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:47:08 smithi164 bash[59462]: cluster 1685821627.7206147 osd.6 (osd.6) 10 : cluster 3 failed to encode map e88 with expected crc 2023-06-03T19:47:09.086 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:47:08 smithi164 bash[59462]: cluster 1685821627.7206805 osd.4 (osd.4) 12 : cluster 3 failed to encode map e88 with expected crc 2023-06-03T19:47:09.086 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:47:08 smithi164 bash[59462]: cluster 1685821627.7229154 mon.a (mon.0) 236 : cluster 0 osdmap e88: 8 total, 8 up, 8 in 2023-06-03T19:47:09.086 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:47:08 smithi164 bash[59462]: cluster 1685821627.723332 osd.1 (osd.1) 13 : cluster 3 failed to encode map e88 with expected crc 2023-06-03T19:47:09.086 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:47:08 smithi164 bash[59462]: cluster 1685821628.1010656 mgr.y (mgr.34104) 243 : cluster 0 pgmap v192: 89 pgs: 15 active+undersized+degraded, 19 active+undersized, 55 active+clean; 3.0 GiB data, 8.8 GiB used, 706 GiB / 715 GiB avail; 24410/147003 objects degraded (16.605%) 2023-06-03T19:47:09.538 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 31 16 53212 53196 107.24 13.8125 0.00787127 0.00931626 2023-06-03T19:47:10.029 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:47:09 smithi089 bash[75068]: cluster 1685821627.7196429 osd.3 (osd.3) 13 : cluster 3 failed to encode map e88 with expected crc 2023-06-03T19:47:10.029 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:47:09 smithi089 bash[75068]: cluster 1685821627.7253125 osd.3 (osd.3) 14 : cluster 3 failed to encode map e88 with expected crc 2023-06-03T19:47:10.030 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:47:09 smithi089 bash[75068]: audit 1685821629.176908 mon.a (mon.0) 237 : audit 0 from='client.? 172.21.15.89:0/1870019530' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:47:10.030 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:47:09 smithi089 bash[78452]: cluster 1685821627.7196429 osd.3 (osd.3) 13 : cluster 3 failed to encode map e88 with expected crc 2023-06-03T19:47:10.030 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:47:09 smithi089 bash[78452]: cluster 1685821627.7253125 osd.3 (osd.3) 14 : cluster 3 failed to encode map e88 with expected crc 2023-06-03T19:47:10.030 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:47:09 smithi089 bash[78452]: audit 1685821629.176908 mon.a (mon.0) 237 : audit 0 from='client.? 172.21.15.89:0/1870019530' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:47:10.085 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:47:09 smithi164 bash[59462]: cluster 1685821627.7196429 osd.3 (osd.3) 13 : cluster 3 failed to encode map e88 with expected crc 2023-06-03T19:47:10.085 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:47:09 smithi164 bash[59462]: cluster 1685821627.7253125 osd.3 (osd.3) 14 : cluster 3 failed to encode map e88 with expected crc 2023-06-03T19:47:10.085 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:47:09 smithi164 bash[59462]: audit 1685821629.176908 mon.a (mon.0) 237 : audit 0 from='client.? 172.21.15.89:0/1870019530' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:47:10.538 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 32 16 54047 54031 105.519 52.1875 0.0295494 0.0094691 2023-06-03T19:47:11.029 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:47:10 smithi089 bash[78452]: cluster 1685821630.1038444 mgr.y (mgr.34104) 244 : cluster 0 pgmap v193: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 5 activating+undersized+degraded+remapped, 81 active+clean; 3.2 GiB data, 9.5 GiB used, 706 GiB / 715 GiB avail; 42 MiB/s wr, 664 op/s; 23971/157797 objects degraded (15.191%) 2023-06-03T19:47:11.030 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:47:10 smithi089 bash[75068]: cluster 1685821630.1038444 mgr.y (mgr.34104) 244 : cluster 0 pgmap v193: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 5 activating+undersized+degraded+remapped, 81 active+clean; 3.2 GiB data, 9.5 GiB used, 706 GiB / 715 GiB avail; 42 MiB/s wr, 664 op/s; 23971/157797 objects degraded (15.191%) 2023-06-03T19:47:11.085 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:47:10 smithi164 bash[59462]: cluster 1685821630.1038444 mgr.y (mgr.34104) 244 : cluster 0 pgmap v193: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 5 activating+undersized+degraded+remapped, 81 active+clean; 3.2 GiB data, 9.5 GiB used, 706 GiB / 715 GiB avail; 42 MiB/s wr, 664 op/s; 23971/157797 objects degraded (15.191%) 2023-06-03T19:47:11.538 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 33 16 54863 54847 103.867 51 0.0209719 0.00961702 2023-06-03T19:47:12.029 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 19:47:11 smithi089 bash[71163]: debug 2023-06-03T19:47:11.727+0000 7fe48642e700 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2023-06-03T19:47:12.538 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 34 16 55667 55651 102.29 50.25 0.0275116 0.00976784 2023-06-03T19:47:12.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:47:12 smithi164 bash[59462]: audit 1685821631.5800993 mon.a (mon.0) 238 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:47:12.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:47:12 smithi164 bash[59462]: audit 1685821631.601987 mon.a (mon.0) 239 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:47:12.836 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:47:12 smithi164 bash[59462]: audit 1685821631.6087887 mon.c (mon.2) 85 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-03T19:47:12.836 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:47:12 smithi164 bash[59462]: audit 1685821631.610922 mon.c (mon.2) 86 : audit 1 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-03T19:47:12.836 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:47:12 smithi164 bash[59462]: audit 1685821631.6228 mon.a (mon.0) 240 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:47:12.836 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:47:12 smithi164 bash[59462]: audit 1685821631.708907 mon.c (mon.2) 87 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-03T19:47:12.836 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:47:12 smithi164 bash[59462]: audit 1685821631.7155392 mon.c (mon.2) 88 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-06-03T19:47:12.836 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:47:12 smithi164 bash[59462]: audit 1685821631.72047 mon.c (mon.2) 89 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-06-03T19:47:12.836 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:47:12 smithi164 bash[59462]: audit 1685821631.72351 mon.c (mon.2) 90 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-06-03T19:47:12.837 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:47:12 smithi164 bash[59462]: audit 1685821631.7282429 mon.c (mon.2) 91 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-06-03T19:47:12.837 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:47:12 smithi164 bash[59462]: audit 1685821631.8137841 mon.a (mon.0) 241 : audit 0 from='client.? 172.21.15.89:0/2298151188' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:47:12.837 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:47:12 smithi164 bash[59462]: cluster 1685821632.2089553 mon.a (mon.0) 242 : cluster 3 Health check update: Degraded data redundancy: 23971/157797 objects degraded (15.191%), 8 pgs degraded (PG_DEGRADED) 2023-06-03T19:47:13.030 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:47:12 smithi089 bash[75068]: audit 1685821631.5800993 mon.a (mon.0) 238 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:47:13.030 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:47:12 smithi089 bash[75068]: audit 1685821631.601987 mon.a (mon.0) 239 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:47:13.030 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:47:12 smithi089 bash[75068]: audit 1685821631.6087887 mon.c (mon.2) 85 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-03T19:47:13.030 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:47:12 smithi089 bash[75068]: audit 1685821631.610922 mon.c (mon.2) 86 : audit 1 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-03T19:47:13.031 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:47:12 smithi089 bash[75068]: audit 1685821631.6228 mon.a (mon.0) 240 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:47:13.031 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:47:12 smithi089 bash[75068]: audit 1685821631.708907 mon.c (mon.2) 87 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-03T19:47:13.031 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:47:12 smithi089 bash[75068]: audit 1685821631.7155392 mon.c (mon.2) 88 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-06-03T19:47:13.031 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:47:12 smithi089 bash[75068]: audit 1685821631.72047 mon.c (mon.2) 89 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-06-03T19:47:13.031 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:47:12 smithi089 bash[75068]: audit 1685821631.72351 mon.c (mon.2) 90 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-06-03T19:47:13.031 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:47:12 smithi089 bash[75068]: audit 1685821631.7282429 mon.c (mon.2) 91 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-06-03T19:47:13.031 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:47:12 smithi089 bash[75068]: audit 1685821631.8137841 mon.a (mon.0) 241 : audit 0 from='client.? 172.21.15.89:0/2298151188' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:47:13.032 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:47:12 smithi089 bash[75068]: cluster 1685821632.2089553 mon.a (mon.0) 242 : cluster 3 Health check update: Degraded data redundancy: 23971/157797 objects degraded (15.191%), 8 pgs degraded (PG_DEGRADED) 2023-06-03T19:47:13.032 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:47:12 smithi089 bash[78452]: audit 1685821631.5800993 mon.a (mon.0) 238 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:47:13.032 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:47:12 smithi089 bash[78452]: audit 1685821631.601987 mon.a (mon.0) 239 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:47:13.032 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:47:12 smithi089 bash[78452]: audit 1685821631.6087887 mon.c (mon.2) 85 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-03T19:47:13.032 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:47:12 smithi089 bash[78452]: audit 1685821631.610922 mon.c (mon.2) 86 : audit 1 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-03T19:47:13.032 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:47:12 smithi089 bash[78452]: audit 1685821631.6228 mon.a (mon.0) 240 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:47:13.033 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:47:12 smithi089 bash[78452]: audit 1685821631.708907 mon.c (mon.2) 87 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-03T19:47:13.033 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:47:12 smithi089 bash[78452]: audit 1685821631.7155392 mon.c (mon.2) 88 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-06-03T19:47:13.033 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:47:12 smithi089 bash[78452]: audit 1685821631.72047 mon.c (mon.2) 89 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-06-03T19:47:13.033 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:47:12 smithi089 bash[78452]: audit 1685821631.72351 mon.c (mon.2) 90 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-06-03T19:47:13.033 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:47:12 smithi089 bash[78452]: audit 1685821631.7282429 mon.c (mon.2) 91 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-06-03T19:47:13.033 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:47:12 smithi089 bash[78452]: audit 1685821631.8137841 mon.a (mon.0) 241 : audit 0 from='client.? 172.21.15.89:0/2298151188' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:47:13.033 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:47:12 smithi089 bash[78452]: cluster 1685821632.2089553 mon.a (mon.0) 242 : cluster 3 Health check update: Degraded data redundancy: 23971/157797 objects degraded (15.191%), 8 pgs degraded (PG_DEGRADED) 2023-06-03T19:47:13.538 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 35 16 56526 56510 100.901 53.6875 0.0135183 0.00990239 2023-06-03T19:47:14.030 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:47:13 smithi089 bash[78452]: audit 1685821631.7292533 mgr.y (mgr.34104) 245 : audit 0 from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-06-03T19:47:14.030 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:47:13 smithi089 bash[78452]: cephadm 1685821631.7310798 mgr.y (mgr.34104) 246 : cephadm 1 Upgrade: unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2023-06-03T19:47:14.030 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:47:13 smithi089 bash[78452]: cluster 1685821632.104416 mgr.y (mgr.34104) 247 : cluster 0 pgmap v194: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 5 activating+undersized+degraded+remapped, 81 active+clean; 3.2 GiB data, 9.5 GiB used, 706 GiB / 715 GiB avail; 35 MiB/s wr, 562 op/s; 23971/157797 objects degraded (15.191%) 2023-06-03T19:47:14.031 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:47:13 smithi089 bash[75068]: audit 1685821631.7292533 mgr.y (mgr.34104) 245 : audit 0 from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-06-03T19:47:14.031 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:47:13 smithi089 bash[75068]: cephadm 1685821631.7310798 mgr.y (mgr.34104) 246 : cephadm 1 Upgrade: unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2023-06-03T19:47:14.031 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:47:13 smithi089 bash[75068]: cluster 1685821632.104416 mgr.y (mgr.34104) 247 : cluster 0 pgmap v194: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 5 activating+undersized+degraded+remapped, 81 active+clean; 3.2 GiB data, 9.5 GiB used, 706 GiB / 715 GiB avail; 35 MiB/s wr, 562 op/s; 23971/157797 objects degraded (15.191%) 2023-06-03T19:47:14.085 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:47:13 smithi164 bash[59462]: audit 1685821631.7292533 mgr.y (mgr.34104) 245 : audit 0 from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-06-03T19:47:14.085 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:47:13 smithi164 bash[59462]: cephadm 1685821631.7310798 mgr.y (mgr.34104) 246 : cephadm 1 Upgrade: unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2023-06-03T19:47:14.085 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:47:13 smithi164 bash[59462]: cluster 1685821632.104416 mgr.y (mgr.34104) 247 : cluster 0 pgmap v194: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 5 activating+undersized+degraded+remapped, 81 active+clean; 3.2 GiB data, 9.5 GiB used, 706 GiB / 715 GiB avail; 35 MiB/s wr, 562 op/s; 23971/157797 objects degraded (15.191%) 2023-06-03T19:47:14.538 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 36 16 57304 57288 99.449 48.625 0.0103716 0.0100486 2023-06-03T19:47:15.029 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:47:14 smithi089 bash[75068]: audit 1685821634.4307 mon.c (mon.2) 92 : audit 0 from='client.? 172.21.15.89:0/1187220109' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:47:15.030 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:47:14 smithi089 bash[78452]: audit 1685821634.4307 mon.c (mon.2) 92 : audit 0 from='client.? 172.21.15.89:0/1187220109' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:47:15.085 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:47:14 smithi164 bash[59462]: audit 1685821634.4307 mon.c (mon.2) 92 : audit 0 from='client.? 172.21.15.89:0/1187220109' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:47:15.538 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 37 16 58155 58139 98.1985 53.1875 0.0271341 0.0101766 2023-06-03T19:47:16.030 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:47:15 smithi089 bash[75068]: cluster 1685821634.1051698 mgr.y (mgr.34104) 248 : cluster 0 pgmap v195: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 81 active+clean; 3.4 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 58 MiB/s wr, 928 op/s; 23644/169290 objects degraded (13.967%); 2.5 MiB/s, 40 objects/s recovering 2023-06-03T19:47:16.030 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:47:15 smithi089 bash[78452]: cluster 1685821634.1051698 mgr.y (mgr.34104) 248 : cluster 0 pgmap v195: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 81 active+clean; 3.4 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 58 MiB/s wr, 928 op/s; 23644/169290 objects degraded (13.967%); 2.5 MiB/s, 40 objects/s recovering 2023-06-03T19:47:16.085 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:47:15 smithi164 bash[59462]: cluster 1685821634.1051698 mgr.y (mgr.34104) 248 : cluster 0 pgmap v195: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 81 active+clean; 3.4 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 58 MiB/s wr, 928 op/s; 23644/169290 objects degraded (13.967%); 2.5 MiB/s, 40 objects/s recovering 2023-06-03T19:47:16.529 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 19:47:16 smithi089 bash[71163]: ::ffff:172.21.15.164 - - [03/Jun/2023:19:47:16] "GET /metrics HTTP/1.1" 200 206881 "" "Prometheus/2.33.4" 2023-06-03T19:47:16.539 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 38 16 58990 58974 96.9876 52.1875 0.0200814 0.0103039 2023-06-03T19:47:17.539 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 39 16 59776 59760 95.7602 49.125 0.00846603 0.0104351 2023-06-03T19:47:18.029 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:47:17 smithi089 bash[78452]: cluster 1685821636.105831 mgr.y (mgr.34104) 249 : cluster 0 pgmap v196: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 81 active+clean; 3.4 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 49 MiB/s wr, 791 op/s; 23644/169290 objects degraded (13.967%); 2.2 MiB/s, 34 objects/s recovering 2023-06-03T19:47:18.030 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:47:17 smithi089 bash[78452]: audit 1685821637.0359354 mon.b (mon.1) 5 : audit 0 from='client.? 172.21.15.89:0/2300192237' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:47:18.030 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:47:17 smithi089 bash[78452]: cluster 1685821637.2103539 mon.a (mon.0) 243 : cluster 3 Health check update: Degraded data redundancy: 23644/169290 objects degraded (13.967%), 8 pgs degraded (PG_DEGRADED) 2023-06-03T19:47:18.030 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:47:17 smithi089 bash[75068]: cluster 1685821636.105831 mgr.y (mgr.34104) 249 : cluster 0 pgmap v196: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 81 active+clean; 3.4 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 49 MiB/s wr, 791 op/s; 23644/169290 objects degraded (13.967%); 2.2 MiB/s, 34 objects/s recovering 2023-06-03T19:47:18.030 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:47:17 smithi089 bash[75068]: audit 1685821637.0359354 mon.b (mon.1) 5 : audit 0 from='client.? 172.21.15.89:0/2300192237' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:47:18.030 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:47:17 smithi089 bash[75068]: cluster 1685821637.2103539 mon.a (mon.0) 243 : cluster 3 Health check update: Degraded data redundancy: 23644/169290 objects degraded (13.967%), 8 pgs degraded (PG_DEGRADED) 2023-06-03T19:47:18.085 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:47:17 smithi164 bash[59462]: cluster 1685821636.105831 mgr.y (mgr.34104) 249 : cluster 0 pgmap v196: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 81 active+clean; 3.4 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 49 MiB/s wr, 791 op/s; 23644/169290 objects degraded (13.967%); 2.2 MiB/s, 34 objects/s recovering 2023-06-03T19:47:18.085 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:47:17 smithi164 bash[59462]: audit 1685821637.0359354 mon.b (mon.1) 5 : audit 0 from='client.? 172.21.15.89:0/2300192237' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:47:18.086 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:47:17 smithi164 bash[59462]: cluster 1685821637.2103539 mon.a (mon.0) 243 : cluster 3 Health check update: Degraded data redundancy: 23644/169290 objects degraded (13.967%), 8 pgs degraded (PG_DEGRADED) 2023-06-03T19:47:18.539 INFO:tasks.radosbench.radosbench.0.smithi089.stdout:2023-06-03T19:47:18.534200+0000 min lat: 0.00135761 max lat: 2.96402 avg lat: 0.0105574 2023-06-03T19:47:18.539 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-06-03T19:47:18.539 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 40 16 60602 60586 94.6567 51.625 0.0338309 0.0105574 2023-06-03T19:47:19.539 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 41 16 61363 61347 93.5079 47.5625 0.0129067 0.0106888 2023-06-03T19:47:20.029 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:47:19 smithi089 bash[78452]: cluster 1685821638.1064231 mgr.y (mgr.34104) 250 : cluster 0 pgmap v197: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 81 active+clean; 3.4 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 45 MiB/s wr, 715 op/s; 23644/169290 objects degraded (13.967%); 2.0 MiB/s, 31 objects/s recovering 2023-06-03T19:47:20.030 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:47:19 smithi089 bash[75068]: cluster 1685821638.1064231 mgr.y (mgr.34104) 250 : cluster 0 pgmap v197: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 81 active+clean; 3.4 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 45 MiB/s wr, 715 op/s; 23644/169290 objects degraded (13.967%); 2.0 MiB/s, 31 objects/s recovering 2023-06-03T19:47:20.085 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:47:19 smithi164 bash[59462]: cluster 1685821638.1064231 mgr.y (mgr.34104) 250 : cluster 0 pgmap v197: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 81 active+clean; 3.4 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 45 MiB/s wr, 715 op/s; 23644/169290 objects degraded (13.967%); 2.0 MiB/s, 31 objects/s recovering 2023-06-03T19:47:20.539 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 42 16 62192 62176 92.515 51.8125 0.020929 0.0108017 2023-06-03T19:47:21.030 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:47:20 smithi089 bash[78452]: audit 1685821639.6401198 mon.c (mon.2) 93 : audit 0 from='client.? 172.21.15.89:0/1932914479' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:47:21.030 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:47:20 smithi089 bash[75068]: audit 1685821639.6401198 mon.c (mon.2) 93 : audit 0 from='client.? 172.21.15.89:0/1932914479' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:47:21.085 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:47:20 smithi164 bash[59462]: audit 1685821639.6401198 mon.c (mon.2) 93 : audit 0 from='client.? 172.21.15.89:0/1932914479' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:47:21.539 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 43 16 62997 62981 91.5334 50.3125 0.0126133 0.0109177 2023-06-03T19:47:22.029 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:47:21 smithi089 bash[78452]: cluster 1685821640.1073186 mgr.y (mgr.34104) 251 : cluster 0 pgmap v198: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 81 active+clean; 3.7 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 60 MiB/s wr, 959 op/s; 23259/181557 objects degraded (12.811%); 3.7 MiB/s, 59 objects/s recovering 2023-06-03T19:47:22.030 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:47:21 smithi089 bash[75068]: cluster 1685821640.1073186 mgr.y (mgr.34104) 251 : cluster 0 pgmap v198: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 81 active+clean; 3.7 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 60 MiB/s wr, 959 op/s; 23259/181557 objects degraded (12.811%); 3.7 MiB/s, 59 objects/s recovering 2023-06-03T19:47:22.086 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:47:21 smithi164 bash[59462]: cluster 1685821640.1073186 mgr.y (mgr.34104) 251 : cluster 0 pgmap v198: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 81 active+clean; 3.7 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 60 MiB/s wr, 959 op/s; 23259/181557 objects degraded (12.811%); 3.7 MiB/s, 59 objects/s recovering 2023-06-03T19:47:22.539 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 44 16 63728 63712 90.4914 45.6875 0.0341675 0.0110434 2023-06-03T19:47:23.029 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:47:22 smithi089 bash[78452]: audit 1685821642.0694644 mon.c (mon.2) 94 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T19:47:23.030 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:47:22 smithi089 bash[78452]: cluster 1685821642.2115293 mon.a (mon.0) 244 : cluster 3 Health check update: Degraded data redundancy: 23259/181557 objects degraded (12.811%), 8 pgs degraded (PG_DEGRADED) 2023-06-03T19:47:23.030 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:47:22 smithi089 bash[78452]: audit 1685821642.2432964 mon.b (mon.1) 6 : audit 0 from='client.? 172.21.15.89:0/3485666994' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:47:23.030 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:47:22 smithi089 bash[75068]: audit 1685821642.0694644 mon.c (mon.2) 94 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T19:47:23.031 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:47:22 smithi089 bash[75068]: cluster 1685821642.2115293 mon.a (mon.0) 244 : cluster 3 Health check update: Degraded data redundancy: 23259/181557 objects degraded (12.811%), 8 pgs degraded (PG_DEGRADED) 2023-06-03T19:47:23.031 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:47:22 smithi089 bash[75068]: audit 1685821642.2432964 mon.b (mon.1) 6 : audit 0 from='client.? 172.21.15.89:0/3485666994' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:47:23.085 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:47:22 smithi164 bash[59462]: audit 1685821642.0694644 mon.c (mon.2) 94 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T19:47:23.086 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:47:22 smithi164 bash[59462]: cluster 1685821642.2115293 mon.a (mon.0) 244 : cluster 3 Health check update: Degraded data redundancy: 23259/181557 objects degraded (12.811%), 8 pgs degraded (PG_DEGRADED) 2023-06-03T19:47:23.086 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:47:22 smithi164 bash[59462]: audit 1685821642.2432964 mon.b (mon.1) 6 : audit 0 from='client.? 172.21.15.89:0/3485666994' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:47:23.539 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 45 16 64549 64533 89.6206 51.3125 0.0263941 0.0111514 2023-06-03T19:47:24.029 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:47:23 smithi089 bash[78452]: cluster 1685821642.1079433 mgr.y (mgr.34104) 252 : cluster 0 pgmap v199: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 81 active+clean; 3.7 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 41 MiB/s wr, 659 op/s; 23259/181557 objects degraded (12.811%); 3.7 MiB/s, 59 objects/s recovering 2023-06-03T19:47:24.030 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:47:23 smithi089 bash[75068]: cluster 1685821642.1079433 mgr.y (mgr.34104) 252 : cluster 0 pgmap v199: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 81 active+clean; 3.7 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 41 MiB/s wr, 659 op/s; 23259/181557 objects degraded (12.811%); 3.7 MiB/s, 59 objects/s recovering 2023-06-03T19:47:24.085 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:47:23 smithi164 bash[59462]: cluster 1685821642.1079433 mgr.y (mgr.34104) 252 : cluster 0 pgmap v199: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 81 active+clean; 3.7 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 41 MiB/s wr, 659 op/s; 23259/181557 objects degraded (12.811%); 3.7 MiB/s, 59 objects/s recovering 2023-06-03T19:47:24.539 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 46 16 65362 65346 88.7769 50.8125 0.0132822 0.0112563 2023-06-03T19:47:25.539 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 47 16 66107 66091 87.8786 46.5625 0.0175528 0.0113712 2023-06-03T19:47:26.029 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:47:25 smithi089 bash[78452]: cluster 1685821644.1088152 mgr.y (mgr.34104) 253 : cluster 0 pgmap v200: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 81 active+clean; 3.9 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 62 MiB/s wr, 989 op/s; 22869/193431 objects degraded (11.823%); 5.7 MiB/s, 91 objects/s recovering 2023-06-03T19:47:26.030 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:47:25 smithi089 bash[78452]: audit 1685821644.8596623 mon.b (mon.1) 7 : audit 0 from='client.? 172.21.15.89:0/3380094812' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:47:26.030 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:47:25 smithi089 bash[75068]: cluster 1685821644.1088152 mgr.y (mgr.34104) 253 : cluster 0 pgmap v200: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 81 active+clean; 3.9 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 62 MiB/s wr, 989 op/s; 22869/193431 objects degraded (11.823%); 5.7 MiB/s, 91 objects/s recovering 2023-06-03T19:47:26.031 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:47:25 smithi089 bash[75068]: audit 1685821644.8596623 mon.b (mon.1) 7 : audit 0 from='client.? 172.21.15.89:0/3380094812' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:47:26.085 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:47:25 smithi164 bash[59462]: cluster 1685821644.1088152 mgr.y (mgr.34104) 253 : cluster 0 pgmap v200: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 81 active+clean; 3.9 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 62 MiB/s wr, 989 op/s; 22869/193431 objects degraded (11.823%); 5.7 MiB/s, 91 objects/s recovering 2023-06-03T19:47:26.085 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:47:25 smithi164 bash[59462]: audit 1685821644.8596623 mon.b (mon.1) 7 : audit 0 from='client.? 172.21.15.89:0/3380094812' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:47:26.529 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 19:47:26 smithi089 bash[71163]: ::ffff:172.21.15.164 - - [03/Jun/2023:19:47:26] "GET /metrics HTTP/1.1" 200 207195 "" "Prometheus/2.33.4" 2023-06-03T19:47:26.539 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 48 16 66907 66891 87.0893 50 0.0506035 0.0114758 2023-06-03T19:47:27.029 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 19:47:26 smithi089 bash[71163]: debug 2023-06-03T19:47:26.747+0000 7fe48642e700 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2023-06-03T19:47:27.539 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 49 16 67620 67604 86.2213 44.5625 0.00483033 0.0115862 2023-06-03T19:47:28.029 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:47:27 smithi089 bash[75068]: cluster 1685821646.1121953 mgr.y (mgr.34104) 254 : cluster 0 pgmap v201: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 81 active+clean; 3.9 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 42 MiB/s wr, 670 op/s; 22869/193431 objects degraded (11.823%); 4.0 MiB/s, 64 objects/s recovering 2023-06-03T19:47:28.030 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:47:27 smithi089 bash[75068]: audit 1685821646.7471082 mon.c (mon.2) 95 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-06-03T19:47:28.030 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:47:27 smithi089 bash[75068]: cluster 1685821647.2129123 mon.a (mon.0) 245 : cluster 3 Health check update: Degraded data redundancy: 22869/193431 objects degraded (11.823%), 8 pgs degraded (PG_DEGRADED) 2023-06-03T19:47:28.030 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:47:27 smithi089 bash[75068]: audit 1685821647.4844806 mon.a (mon.0) 246 : audit 0 from='client.? 172.21.15.89:0/232605276' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:47:28.030 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:47:27 smithi089 bash[78452]: cluster 1685821646.1121953 mgr.y (mgr.34104) 254 : cluster 0 pgmap v201: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 81 active+clean; 3.9 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 42 MiB/s wr, 670 op/s; 22869/193431 objects degraded (11.823%); 4.0 MiB/s, 64 objects/s recovering 2023-06-03T19:47:28.030 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:47:27 smithi089 bash[78452]: audit 1685821646.7471082 mon.c (mon.2) 95 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-06-03T19:47:28.030 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:47:27 smithi089 bash[78452]: cluster 1685821647.2129123 mon.a (mon.0) 245 : cluster 3 Health check update: Degraded data redundancy: 22869/193431 objects degraded (11.823%), 8 pgs degraded (PG_DEGRADED) 2023-06-03T19:47:28.031 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:47:27 smithi089 bash[78452]: audit 1685821647.4844806 mon.a (mon.0) 246 : audit 0 from='client.? 172.21.15.89:0/232605276' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:47:28.085 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:47:27 smithi164 bash[59462]: cluster 1685821646.1121953 mgr.y (mgr.34104) 254 : cluster 0 pgmap v201: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 81 active+clean; 3.9 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 42 MiB/s wr, 670 op/s; 22869/193431 objects degraded (11.823%); 4.0 MiB/s, 64 objects/s recovering 2023-06-03T19:47:28.085 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:47:27 smithi164 bash[59462]: audit 1685821646.7471082 mon.c (mon.2) 95 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-06-03T19:47:28.086 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:47:27 smithi164 bash[59462]: cluster 1685821647.2129123 mon.a (mon.0) 245 : cluster 3 Health check update: Degraded data redundancy: 22869/193431 objects degraded (11.823%), 8 pgs degraded (PG_DEGRADED) 2023-06-03T19:47:28.086 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:47:27 smithi164 bash[59462]: audit 1685821647.4844806 mon.a (mon.0) 246 : audit 0 from='client.? 172.21.15.89:0/232605276' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:47:28.540 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 50 16 68398 68382 85.4693 48.625 0.0185394 0.0116935 2023-06-03T19:47:29.029 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:47:28 smithi089 bash[78452]: audit 1685821646.7503443 mgr.y (mgr.34104) 255 : audit 0 from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-06-03T19:47:29.029 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:47:28 smithi089 bash[78452]: cephadm 1685821646.7545402 mgr.y (mgr.34104) 256 : cephadm 1 Upgrade: unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2023-06-03T19:47:29.030 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:47:28 smithi089 bash[75068]: audit 1685821646.7503443 mgr.y (mgr.34104) 255 : audit 0 from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-06-03T19:47:29.030 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:47:28 smithi089 bash[75068]: cephadm 1685821646.7545402 mgr.y (mgr.34104) 256 : cephadm 1 Upgrade: unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2023-06-03T19:47:29.085 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:47:28 smithi164 bash[59462]: audit 1685821646.7503443 mgr.y (mgr.34104) 255 : audit 0 from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-06-03T19:47:29.085 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:47:28 smithi164 bash[59462]: cephadm 1685821646.7545402 mgr.y (mgr.34104) 256 : cephadm 1 Upgrade: unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2023-06-03T19:47:29.540 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 51 16 69187 69171 84.7603 49.3125 0.00489534 0.0117869 2023-06-03T19:47:30.030 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:47:29 smithi089 bash[75068]: cluster 1685821648.1131916 mgr.y (mgr.34104) 257 : cluster 0 pgmap v202: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 81 active+clean; 3.9 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 42 MiB/s wr, 670 op/s; 22869/193431 objects degraded (11.823%); 4.0 MiB/s, 64 objects/s recovering 2023-06-03T19:47:30.030 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:47:29 smithi089 bash[78452]: cluster 1685821648.1131916 mgr.y (mgr.34104) 257 : cluster 0 pgmap v202: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 81 active+clean; 3.9 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 42 MiB/s wr, 670 op/s; 22869/193431 objects degraded (11.823%); 4.0 MiB/s, 64 objects/s recovering 2023-06-03T19:47:30.085 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:47:29 smithi164 bash[59462]: cluster 1685821648.1131916 mgr.y (mgr.34104) 257 : cluster 0 pgmap v202: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 81 active+clean; 3.9 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 42 MiB/s wr, 670 op/s; 22869/193431 objects degraded (11.823%); 4.0 MiB/s, 64 objects/s recovering 2023-06-03T19:47:30.540 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 52 16 69940 69924 84.0352 47.0625 0.0207126 0.0118907 2023-06-03T19:47:31.029 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:47:30 smithi089 bash[78452]: audit 1685821650.0875678 mon.c (mon.2) 96 : audit 0 from='client.? 172.21.15.89:0/1328812907' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:47:31.030 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:47:30 smithi089 bash[75068]: audit 1685821650.0875678 mon.c (mon.2) 96 : audit 0 from='client.? 172.21.15.89:0/1328812907' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:47:31.085 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:47:30 smithi164 bash[59462]: audit 1685821650.0875678 mon.c (mon.2) 96 : audit 0 from='client.? 172.21.15.89:0/1328812907' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:47:31.540 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 53 15 70716 70701 83.3658 48.5625 0.00575161 0.0119877 2023-06-03T19:47:32.029 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:47:31 smithi089 bash[75068]: cluster 1685821650.1140807 mgr.y (mgr.34104) 258 : cluster 0 pgmap v203: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 81 active+clean; 4.2 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 62 MiB/s wr, 991 op/s; 22446/204999 objects degraded (10.949%); 6.2 MiB/s, 99 objects/s recovering 2023-06-03T19:47:32.030 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:47:31 smithi089 bash[78452]: cluster 1685821650.1140807 mgr.y (mgr.34104) 258 : cluster 0 pgmap v203: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 81 active+clean; 4.2 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 62 MiB/s wr, 991 op/s; 22446/204999 objects degraded (10.949%); 6.2 MiB/s, 99 objects/s recovering 2023-06-03T19:47:32.085 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:47:31 smithi164 bash[59462]: cluster 1685821650.1140807 mgr.y (mgr.34104) 258 : cluster 0 pgmap v203: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 81 active+clean; 4.2 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 62 MiB/s wr, 991 op/s; 22446/204999 objects degraded (10.949%); 6.2 MiB/s, 99 objects/s recovering 2023-06-03T19:47:32.540 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 54 16 71476 71460 82.7004 47.4375 0.0107023 0.0120847 2023-06-03T19:47:33.029 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:47:32 smithi089 bash[75068]: cluster 1685821652.1146865 mgr.y (mgr.34104) 259 : cluster 0 pgmap v204: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 81 active+clean; 4.2 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 41 MiB/s wr, 650 op/s; 22446/204999 objects degraded (10.949%); 4.2 MiB/s, 67 objects/s recovering 2023-06-03T19:47:33.030 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:47:32 smithi089 bash[75068]: cluster 1685821652.2142818 mon.a (mon.0) 247 : cluster 3 Health check update: Degraded data redundancy: 22446/204999 objects degraded (10.949%), 8 pgs degraded (PG_DEGRADED) 2023-06-03T19:47:33.030 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:47:32 smithi089 bash[78452]: cluster 1685821652.1146865 mgr.y (mgr.34104) 259 : cluster 0 pgmap v204: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 81 active+clean; 4.2 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 41 MiB/s wr, 650 op/s; 22446/204999 objects degraded (10.949%); 4.2 MiB/s, 67 objects/s recovering 2023-06-03T19:47:33.030 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:47:32 smithi089 bash[78452]: cluster 1685821652.2142818 mon.a (mon.0) 247 : cluster 3 Health check update: Degraded data redundancy: 22446/204999 objects degraded (10.949%), 8 pgs degraded (PG_DEGRADED) 2023-06-03T19:47:33.085 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:47:32 smithi164 bash[59462]: cluster 1685821652.1146865 mgr.y (mgr.34104) 259 : cluster 0 pgmap v204: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 81 active+clean; 4.2 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 41 MiB/s wr, 650 op/s; 22446/204999 objects degraded (10.949%); 4.2 MiB/s, 67 objects/s recovering 2023-06-03T19:47:33.086 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:47:32 smithi164 bash[59462]: cluster 1685821652.2142818 mon.a (mon.0) 247 : cluster 3 Health check update: Degraded data redundancy: 22446/204999 objects degraded (10.949%), 8 pgs degraded (PG_DEGRADED) 2023-06-03T19:47:33.540 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 55 16 72250 72234 82.0762 48.375 0.0163005 0.0121765 2023-06-03T19:47:34.029 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:47:33 smithi089 bash[75068]: audit 1685821652.7312906 mon.c (mon.2) 97 : audit 0 from='client.? 172.21.15.89:0/262255015' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:47:34.030 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:47:33 smithi089 bash[78452]: audit 1685821652.7312906 mon.c (mon.2) 97 : audit 0 from='client.? 172.21.15.89:0/262255015' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:47:34.085 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:47:33 smithi164 bash[59462]: audit 1685821652.7312906 mon.c (mon.2) 97 : audit 0 from='client.? 172.21.15.89:0/262255015' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:47:34.540 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 56 16 73071 73055 81.5268 51.3125 0.0146666 0.0122593 2023-06-03T19:47:35.029 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:47:34 smithi089 bash[75068]: cluster 1685821654.1156342 mgr.y (mgr.34104) 260 : cluster 0 pgmap v205: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 81 active+clean; 4.4 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 61 MiB/s wr, 973 op/s; 22012/216615 objects degraded (10.162%); 6.5 MiB/s, 103 objects/s recovering 2023-06-03T19:47:35.030 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:47:34 smithi089 bash[78452]: cluster 1685821654.1156342 mgr.y (mgr.34104) 260 : cluster 0 pgmap v205: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 81 active+clean; 4.4 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 61 MiB/s wr, 973 op/s; 22012/216615 objects degraded (10.162%); 6.5 MiB/s, 103 objects/s recovering 2023-06-03T19:47:35.085 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:47:34 smithi164 bash[59462]: cluster 1685821654.1156342 mgr.y (mgr.34104) 260 : cluster 0 pgmap v205: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 81 active+clean; 4.4 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 61 MiB/s wr, 973 op/s; 22012/216615 objects degraded (10.162%); 6.5 MiB/s, 103 objects/s recovering 2023-06-03T19:47:35.540 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 57 16 73824 73808 80.9221 47.0625 0.0451215 0.0123494 2023-06-03T19:47:36.029 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:47:35 smithi089 bash[75068]: audit 1685821655.336687 mon.a (mon.0) 248 : audit 0 from='client.? 172.21.15.89:0/446395214' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:47:36.030 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:47:35 smithi089 bash[78452]: audit 1685821655.336687 mon.a (mon.0) 248 : audit 0 from='client.? 172.21.15.89:0/446395214' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:47:36.085 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:47:35 smithi164 bash[59462]: audit 1685821655.336687 mon.a (mon.0) 248 : audit 0 from='client.? 172.21.15.89:0/446395214' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:47:36.535 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 19:47:36 smithi089 bash[71163]: ::ffff:172.21.15.164 - - [03/Jun/2023:19:47:36] "GET /metrics HTTP/1.1" 200 207195 "" "Prometheus/2.33.4" 2023-06-03T19:47:36.540 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 58 16 74606 74590 80.3694 48.875 0.0485285 0.0124335 2023-06-03T19:47:37.029 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:47:36 smithi089 bash[75068]: cluster 1685821656.1163445 mgr.y (mgr.34104) 261 : cluster 0 pgmap v206: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 81 active+clean; 4.4 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 40 MiB/s wr, 643 op/s; 22012/216615 objects degraded (10.162%); 4.5 MiB/s, 71 objects/s recovering 2023-06-03T19:47:37.030 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:47:36 smithi089 bash[78452]: cluster 1685821656.1163445 mgr.y (mgr.34104) 261 : cluster 0 pgmap v206: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 81 active+clean; 4.4 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 40 MiB/s wr, 643 op/s; 22012/216615 objects degraded (10.162%); 4.5 MiB/s, 71 objects/s recovering 2023-06-03T19:47:37.085 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:47:36 smithi164 bash[59462]: cluster 1685821656.1163445 mgr.y (mgr.34104) 261 : cluster 0 pgmap v206: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 81 active+clean; 4.4 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 40 MiB/s wr, 643 op/s; 22012/216615 objects degraded (10.162%); 4.5 MiB/s, 71 objects/s recovering 2023-06-03T19:47:37.540 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 59 16 75394 75378 79.8419 49.25 0.0119482 0.0125173 2023-06-03T19:47:38.029 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:47:37 smithi089 bash[75068]: audit 1685821657.0693989 mon.c (mon.2) 98 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T19:47:38.030 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:47:37 smithi089 bash[75068]: cluster 1685821657.2155612 mon.a (mon.0) 249 : cluster 3 Health check update: Degraded data redundancy: 22012/216615 objects degraded (10.162%), 8 pgs degraded (PG_DEGRADED) 2023-06-03T19:47:38.030 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:47:37 smithi089 bash[78452]: audit 1685821657.0693989 mon.c (mon.2) 98 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T19:47:38.030 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:47:37 smithi089 bash[78452]: cluster 1685821657.2155612 mon.a (mon.0) 249 : cluster 3 Health check update: Degraded data redundancy: 22012/216615 objects degraded (10.162%), 8 pgs degraded (PG_DEGRADED) 2023-06-03T19:47:38.085 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:47:37 smithi164 bash[59462]: audit 1685821657.0693989 mon.c (mon.2) 98 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T19:47:38.085 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:47:37 smithi164 bash[59462]: cluster 1685821657.2155612 mon.a (mon.0) 249 : cluster 3 Health check update: Degraded data redundancy: 22012/216615 objects degraded (10.162%), 8 pgs degraded (PG_DEGRADED) 2023-06-03T19:47:38.541 INFO:tasks.radosbench.radosbench.0.smithi089.stdout:2023-06-03T19:47:38.536188+0000 min lat: 0.00135761 max lat: 2.96402 avg lat: 0.0125989 2023-06-03T19:47:38.541 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-06-03T19:47:38.541 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 60 16 76176 76160 79.3257 48.875 0.0106861 0.0125989 2023-06-03T19:47:39.029 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:47:38 smithi089 bash[75068]: audit 1685821657.8934019 mon.b (mon.1) 8 : audit 0 from='client.? 172.21.15.89:0/2234634427' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:47:39.030 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:47:38 smithi089 bash[75068]: cluster 1685821658.1168869 mgr.y (mgr.34104) 262 : cluster 0 pgmap v207: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 81 active+clean; 4.4 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 40 MiB/s wr, 643 op/s; 22012/216615 objects degraded (10.162%); 4.5 MiB/s, 71 objects/s recovering 2023-06-03T19:47:39.030 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:47:38 smithi089 bash[78452]: audit 1685821657.8934019 mon.b (mon.1) 8 : audit 0 from='client.? 172.21.15.89:0/2234634427' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:47:39.030 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:47:38 smithi089 bash[78452]: cluster 1685821658.1168869 mgr.y (mgr.34104) 262 : cluster 0 pgmap v207: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 81 active+clean; 4.4 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 40 MiB/s wr, 643 op/s; 22012/216615 objects degraded (10.162%); 4.5 MiB/s, 71 objects/s recovering 2023-06-03T19:47:39.085 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:47:38 smithi164 bash[59462]: audit 1685821657.8934019 mon.b (mon.1) 8 : audit 0 from='client.? 172.21.15.89:0/2234634427' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:47:39.086 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:47:38 smithi164 bash[59462]: cluster 1685821658.1168869 mgr.y (mgr.34104) 262 : cluster 0 pgmap v207: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 81 active+clean; 4.4 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 40 MiB/s wr, 643 op/s; 22012/216615 objects degraded (10.162%); 4.5 MiB/s, 71 objects/s recovering 2023-06-03T19:47:39.541 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 61 16 76968 76952 78.8367 49.5 0.0126469 0.012678 2023-06-03T19:47:40.541 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 62 16 77678 77662 78.2808 44.375 0.0476637 0.0127651 2023-06-03T19:47:41.529 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:47:41 smithi089 bash[78452]: cluster 1685821660.117783 mgr.y (mgr.34104) 263 : cluster 0 pgmap v208: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 81 active+clean; 4.6 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 61 MiB/s wr, 969 op/s; 21543/228336 objects degraded (9.435%); 6.9 MiB/s, 110 objects/s recovering 2023-06-03T19:47:41.530 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:47:41 smithi089 bash[78452]: audit 1685821660.4854858 mon.c (mon.2) 99 : audit 0 from='client.? 172.21.15.89:0/3750976612' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:47:41.530 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:47:41 smithi089 bash[75068]: cluster 1685821660.117783 mgr.y (mgr.34104) 263 : cluster 0 pgmap v208: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 81 active+clean; 4.6 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 61 MiB/s wr, 969 op/s; 21543/228336 objects degraded (9.435%); 6.9 MiB/s, 110 objects/s recovering 2023-06-03T19:47:41.530 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:47:41 smithi089 bash[75068]: audit 1685821660.4854858 mon.c (mon.2) 99 : audit 0 from='client.? 172.21.15.89:0/3750976612' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:47:41.541 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 63 16 78433 78417 77.7871 47.1875 0.018496 0.0128478 2023-06-03T19:47:41.587 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:47:41 smithi164 bash[59462]: cluster 1685821660.117783 mgr.y (mgr.34104) 263 : cluster 0 pgmap v208: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 81 active+clean; 4.6 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 61 MiB/s wr, 969 op/s; 21543/228336 objects degraded (9.435%); 6.9 MiB/s, 110 objects/s recovering 2023-06-03T19:47:41.587 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:47:41 smithi164 bash[59462]: audit 1685821660.4854858 mon.c (mon.2) 99 : audit 0 from='client.? 172.21.15.89:0/3750976612' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:47:42.029 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 19:47:41 smithi089 bash[71163]: debug 2023-06-03T19:47:41.766+0000 7fe48642e700 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2023-06-03T19:47:42.529 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:47:42 smithi089 bash[75068]: audit 1685821661.7693074 mon.c (mon.2) 100 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-06-03T19:47:42.530 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:47:42 smithi089 bash[78452]: audit 1685821661.7693074 mon.c (mon.2) 100 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-06-03T19:47:42.541 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 64 16 79172 79156 77.2933 46.1875 0.0350249 0.0129307 2023-06-03T19:47:42.585 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:47:42 smithi164 bash[59462]: audit 1685821661.7693074 mon.c (mon.2) 100 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-06-03T19:47:43.529 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:47:43 smithi089 bash[78452]: audit 1685821661.7695272 mgr.y (mgr.34104) 264 : audit 0 from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-06-03T19:47:43.530 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:47:43 smithi089 bash[78452]: cephadm 1685821661.7737074 mgr.y (mgr.34104) 265 : cephadm 1 Upgrade: unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2023-06-03T19:47:43.530 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:47:43 smithi089 bash[78452]: cluster 1685821662.1183152 mgr.y (mgr.34104) 266 : cluster 0 pgmap v209: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 81 active+clean; 4.6 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 41 MiB/s wr, 648 op/s; 21543/228336 objects degraded (9.435%); 4.7 MiB/s, 75 objects/s recovering 2023-06-03T19:47:43.530 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:47:43 smithi089 bash[78452]: cluster 1685821662.2167382 mon.a (mon.0) 250 : cluster 3 Health check update: Degraded data redundancy: 21543/228336 objects degraded (9.435%), 8 pgs degraded (PG_DEGRADED) 2023-06-03T19:47:43.530 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:47:43 smithi089 bash[78452]: audit 1685821663.0795405 mon.a (mon.0) 251 : audit 0 from='client.? 172.21.15.89:0/3277228166' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:47:43.530 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:47:43 smithi089 bash[75068]: audit 1685821661.7695272 mgr.y (mgr.34104) 264 : audit 0 from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-06-03T19:47:43.531 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:47:43 smithi089 bash[75068]: cephadm 1685821661.7737074 mgr.y (mgr.34104) 265 : cephadm 1 Upgrade: unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2023-06-03T19:47:43.531 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:47:43 smithi089 bash[75068]: cluster 1685821662.1183152 mgr.y (mgr.34104) 266 : cluster 0 pgmap v209: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 81 active+clean; 4.6 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 41 MiB/s wr, 648 op/s; 21543/228336 objects degraded (9.435%); 4.7 MiB/s, 75 objects/s recovering 2023-06-03T19:47:43.531 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:47:43 smithi089 bash[75068]: cluster 1685821662.2167382 mon.a (mon.0) 250 : cluster 3 Health check update: Degraded data redundancy: 21543/228336 objects degraded (9.435%), 8 pgs degraded (PG_DEGRADED) 2023-06-03T19:47:43.531 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:47:43 smithi089 bash[75068]: audit 1685821663.0795405 mon.a (mon.0) 251 : audit 0 from='client.? 172.21.15.89:0/3277228166' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:47:43.541 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 65 16 79836 79820 76.7426 41.5 0.0417377 0.0130223 2023-06-03T19:47:43.585 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:47:43 smithi164 bash[59462]: audit 1685821661.7695272 mgr.y (mgr.34104) 264 : audit 0 from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-06-03T19:47:43.585 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:47:43 smithi164 bash[59462]: cephadm 1685821661.7737074 mgr.y (mgr.34104) 265 : cephadm 1 Upgrade: unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2023-06-03T19:47:43.586 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:47:43 smithi164 bash[59462]: cluster 1685821662.1183152 mgr.y (mgr.34104) 266 : cluster 0 pgmap v209: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 81 active+clean; 4.6 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 41 MiB/s wr, 648 op/s; 21543/228336 objects degraded (9.435%); 4.7 MiB/s, 75 objects/s recovering 2023-06-03T19:47:43.586 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:47:43 smithi164 bash[59462]: cluster 1685821662.2167382 mon.a (mon.0) 250 : cluster 3 Health check update: Degraded data redundancy: 21543/228336 objects degraded (9.435%), 8 pgs degraded (PG_DEGRADED) 2023-06-03T19:47:43.586 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:47:43 smithi164 bash[59462]: audit 1685821663.0795405 mon.a (mon.0) 251 : audit 0 from='client.? 172.21.15.89:0/3277228166' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:47:44.541 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 66 16 80570 80554 76.2748 45.875 0.0240926 0.0131025 2023-06-03T19:47:45.529 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:47:45 smithi089 bash[78452]: cluster 1685821664.1191292 mgr.y (mgr.34104) 267 : cluster 0 pgmap v210: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 81 active+clean; 4.9 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 60 MiB/s wr, 954 op/s; 21054/239385 objects degraded (8.795%); 7.2 MiB/s, 115 objects/s recovering 2023-06-03T19:47:45.530 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:47:45 smithi089 bash[75068]: cluster 1685821664.1191292 mgr.y (mgr.34104) 267 : cluster 0 pgmap v210: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 81 active+clean; 4.9 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 60 MiB/s wr, 954 op/s; 21054/239385 objects degraded (8.795%); 7.2 MiB/s, 115 objects/s recovering 2023-06-03T19:47:45.541 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 67 16 81250 81234 75.7706 42.5 0.00610075 0.0131891 2023-06-03T19:47:45.585 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:47:45 smithi164 bash[59462]: cluster 1685821664.1191292 mgr.y (mgr.34104) 267 : cluster 0 pgmap v210: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 81 active+clean; 4.9 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 60 MiB/s wr, 954 op/s; 21054/239385 objects degraded (8.795%); 7.2 MiB/s, 115 objects/s recovering 2023-06-03T19:47:46.530 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:47:46 smithi089 bash[75068]: audit 1685821665.7165964 mon.a (mon.0) 252 : audit 0 from='client.? 172.21.15.89:0/3432712905' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:47:46.530 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 19:47:46 smithi089 bash[71163]: ::ffff:172.21.15.164 - - [03/Jun/2023:19:47:46] "GET /metrics HTTP/1.1" 200 207223 "" "Prometheus/2.33.4" 2023-06-03T19:47:46.530 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:47:46 smithi089 bash[78452]: audit 1685821665.7165964 mon.a (mon.0) 252 : audit 0 from='client.? 172.21.15.89:0/3432712905' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:47:46.541 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 68 15 81942 81927 75.2933 43.3125 0.0409949 0.0132753 2023-06-03T19:47:46.585 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:47:46 smithi164 bash[59462]: audit 1685821665.7165964 mon.a (mon.0) 252 : audit 0 from='client.? 172.21.15.89:0/3432712905' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:47:47.529 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:47:47 smithi089 bash[75068]: cluster 1685821666.1199083 mgr.y (mgr.34104) 268 : cluster 0 pgmap v211: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 81 active+clean; 4.9 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 40 MiB/s wr, 632 op/s; 21054/239385 objects degraded (8.795%); 5.0 MiB/s, 79 objects/s recovering 2023-06-03T19:47:47.530 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:47:47 smithi089 bash[78452]: cluster 1685821666.1199083 mgr.y (mgr.34104) 268 : cluster 0 pgmap v211: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 81 active+clean; 4.9 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 40 MiB/s wr, 632 op/s; 21054/239385 objects degraded (8.795%); 5.0 MiB/s, 79 objects/s recovering 2023-06-03T19:47:47.541 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 69 15 82648 82633 74.8415 44.125 0.0129158 0.0133547 2023-06-03T19:47:47.585 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:47:47 smithi164 bash[59462]: cluster 1685821666.1199083 mgr.y (mgr.34104) 268 : cluster 0 pgmap v211: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 81 active+clean; 4.9 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 40 MiB/s wr, 632 op/s; 21054/239385 objects degraded (8.795%); 5.0 MiB/s, 79 objects/s recovering 2023-06-03T19:47:48.531 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:47:48 smithi089 bash[75068]: cluster 1685821667.2179582 mon.a (mon.0) 253 : cluster 3 Health check update: Degraded data redundancy: 21054/239385 objects degraded (8.795%), 8 pgs degraded (PG_DEGRADED) 2023-06-03T19:47:48.531 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:47:48 smithi089 bash[78452]: cluster 1685821667.2179582 mon.a (mon.0) 253 : cluster 3 Health check update: Degraded data redundancy: 21054/239385 objects degraded (8.795%), 8 pgs degraded (PG_DEGRADED) 2023-06-03T19:47:48.542 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 70 16 83304 83288 74.3571 40.9375 0.0128664 0.01344 2023-06-03T19:47:48.585 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:47:48 smithi164 bash[59462]: cluster 1685821667.2179582 mon.a (mon.0) 253 : cluster 3 Health check update: Degraded data redundancy: 21054/239385 objects degraded (8.795%), 8 pgs degraded (PG_DEGRADED) 2023-06-03T19:47:49.529 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:47:49 smithi089 bash[75068]: cluster 1685821668.1204379 mgr.y (mgr.34104) 269 : cluster 0 pgmap v212: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 81 active+clean; 4.9 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 40 MiB/s wr, 632 op/s; 21054/239385 objects degraded (8.795%); 5.0 MiB/s, 79 objects/s recovering 2023-06-03T19:47:49.530 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:47:49 smithi089 bash[75068]: audit 1685821668.3393583 mon.c (mon.2) 101 : audit 0 from='client.? 172.21.15.89:0/2718083999' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:47:49.530 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:47:49 smithi089 bash[78452]: cluster 1685821668.1204379 mgr.y (mgr.34104) 269 : cluster 0 pgmap v212: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 81 active+clean; 4.9 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 40 MiB/s wr, 632 op/s; 21054/239385 objects degraded (8.795%); 5.0 MiB/s, 79 objects/s recovering 2023-06-03T19:47:49.530 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:47:49 smithi089 bash[78452]: audit 1685821668.3393583 mon.c (mon.2) 101 : audit 0 from='client.? 172.21.15.89:0/2718083999' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:47:49.542 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 71 16 84003 83987 73.9251 43.6875 0.00845335 0.0135198 2023-06-03T19:47:49.585 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:47:49 smithi164 bash[59462]: cluster 1685821668.1204379 mgr.y (mgr.34104) 269 : cluster 0 pgmap v212: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 81 active+clean; 4.9 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 40 MiB/s wr, 632 op/s; 21054/239385 objects degraded (8.795%); 5.0 MiB/s, 79 objects/s recovering 2023-06-03T19:47:49.585 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:47:49 smithi164 bash[59462]: audit 1685821668.3393583 mon.c (mon.2) 101 : audit 0 from='client.? 172.21.15.89:0/2718083999' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:47:50.542 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 72 16 84701 84685 73.5042 43.625 0.00804051 0.0135974 2023-06-03T19:47:51.529 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:47:51 smithi089 bash[75068]: cluster 1685821670.1218936 mgr.y (mgr.34104) 270 : cluster 0 pgmap v213: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 81 active+clean; 5.1 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 58 MiB/s wr, 922 op/s; 20527/249837 objects degraded (8.216%); 7.7 MiB/s, 123 objects/s recovering 2023-06-03T19:47:51.530 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:47:51 smithi089 bash[75068]: audit 1685821670.9668627 mon.c (mon.2) 102 : audit 0 from='client.? 172.21.15.89:0/2511150809' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:47:51.530 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:47:51 smithi089 bash[78452]: cluster 1685821670.1218936 mgr.y (mgr.34104) 270 : cluster 0 pgmap v213: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 81 active+clean; 5.1 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 58 MiB/s wr, 922 op/s; 20527/249837 objects degraded (8.216%); 7.7 MiB/s, 123 objects/s recovering 2023-06-03T19:47:51.530 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:47:51 smithi089 bash[78452]: audit 1685821670.9668627 mon.c (mon.2) 102 : audit 0 from='client.? 172.21.15.89:0/2511150809' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:47:51.542 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 73 16 85374 85358 73.0734 42.0625 0.0238495 0.013678 2023-06-03T19:47:51.585 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:47:51 smithi164 bash[59462]: cluster 1685821670.1218936 mgr.y (mgr.34104) 270 : cluster 0 pgmap v213: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 81 active+clean; 5.1 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 58 MiB/s wr, 922 op/s; 20527/249837 objects degraded (8.216%); 7.7 MiB/s, 123 objects/s recovering 2023-06-03T19:47:51.585 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:47:51 smithi164 bash[59462]: audit 1685821670.9668627 mon.c (mon.2) 102 : audit 0 from='client.? 172.21.15.89:0/2511150809' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:47:52.529 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:47:52 smithi089 bash[75068]: audit 1685821672.071354 mon.c (mon.2) 103 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T19:47:52.531 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:47:52 smithi089 bash[75068]: cluster 1685821672.21921 mon.a (mon.0) 254 : cluster 3 Health check update: Degraded data redundancy: 20527/249837 objects degraded (8.216%), 8 pgs degraded (PG_DEGRADED) 2023-06-03T19:47:52.532 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:47:52 smithi089 bash[78452]: audit 1685821672.071354 mon.c (mon.2) 103 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T19:47:52.532 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:47:52 smithi089 bash[78452]: cluster 1685821672.21921 mon.a (mon.0) 254 : cluster 3 Health check update: Degraded data redundancy: 20527/249837 objects degraded (8.216%), 8 pgs degraded (PG_DEGRADED) 2023-06-03T19:47:52.542 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 74 16 86055 86039 72.661 42.5625 0.00701083 0.0137543 2023-06-03T19:47:52.585 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:47:52 smithi164 bash[59462]: audit 1685821672.071354 mon.c (mon.2) 103 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T19:47:52.585 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:47:52 smithi164 bash[59462]: cluster 1685821672.21921 mon.a (mon.0) 254 : cluster 3 Health check update: Degraded data redundancy: 20527/249837 objects degraded (8.216%), 8 pgs degraded (PG_DEGRADED) 2023-06-03T19:47:53.529 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:47:53 smithi089 bash[75068]: cluster 1685821672.1224287 mgr.y (mgr.34104) 271 : cluster 0 pgmap v214: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 81 active+clean; 5.1 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 37 MiB/s wr, 596 op/s; 20527/249837 objects degraded (8.216%); 5.3 MiB/s, 84 objects/s recovering 2023-06-03T19:47:53.530 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:47:53 smithi089 bash[78452]: cluster 1685821672.1224287 mgr.y (mgr.34104) 271 : cluster 0 pgmap v214: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 81 active+clean; 5.1 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 37 MiB/s wr, 596 op/s; 20527/249837 objects degraded (8.216%); 5.3 MiB/s, 84 objects/s recovering 2023-06-03T19:47:53.542 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 75 16 86705 86689 72.2339 40.625 0.0124823 0.0138361 2023-06-03T19:47:53.585 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:47:53 smithi164 bash[59462]: cluster 1685821672.1224287 mgr.y (mgr.34104) 271 : cluster 0 pgmap v214: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 81 active+clean; 5.1 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 37 MiB/s wr, 596 op/s; 20527/249837 objects degraded (8.216%); 5.3 MiB/s, 84 objects/s recovering 2023-06-03T19:47:54.529 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:47:54 smithi089 bash[75068]: audit 1685821673.5950751 mon.c (mon.2) 104 : audit 0 from='client.? 172.21.15.89:0/3838809078' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:47:54.530 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:47:54 smithi089 bash[78452]: audit 1685821673.5950751 mon.c (mon.2) 104 : audit 0 from='client.? 172.21.15.89:0/3838809078' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:47:54.542 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 76 16 87380 87364 71.8385 42.1875 0.037652 0.0139121 2023-06-03T19:47:54.585 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:47:54 smithi164 bash[59462]: audit 1685821673.5950751 mon.c (mon.2) 104 : audit 0 from='client.? 172.21.15.89:0/3838809078' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:47:55.529 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:47:55 smithi089 bash[75068]: cluster 1685821674.1232796 mgr.y (mgr.34104) 272 : cluster 0 pgmap v215: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 81 active+clean; 5.3 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 55 MiB/s wr, 881 op/s; 19989/260067 objects degraded (7.686%); 8.1 MiB/s, 129 objects/s recovering 2023-06-03T19:47:55.530 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:47:55 smithi089 bash[78452]: cluster 1685821674.1232796 mgr.y (mgr.34104) 272 : cluster 0 pgmap v215: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 81 active+clean; 5.3 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 55 MiB/s wr, 881 op/s; 19989/260067 objects degraded (7.686%); 8.1 MiB/s, 129 objects/s recovering 2023-06-03T19:47:55.542 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 77 16 88055 88039 71.4533 42.1875 0.042513 0.0139876 2023-06-03T19:47:55.585 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:47:55 smithi164 bash[59462]: cluster 1685821674.1232796 mgr.y (mgr.34104) 272 : cluster 0 pgmap v215: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 81 active+clean; 5.3 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 55 MiB/s wr, 881 op/s; 19989/260067 objects degraded (7.686%); 8.1 MiB/s, 129 objects/s recovering 2023-06-03T19:47:56.529 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:47:56 smithi089 bash[75068]: audit 1685821676.2094169 mon.a (mon.0) 255 : audit 0 from='client.? 172.21.15.89:0/215452255' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:47:56.530 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 19:47:56 smithi089 bash[71163]: ::ffff:172.21.15.164 - - [03/Jun/2023:19:47:56] "GET /metrics HTTP/1.1" 200 207241 "" "Prometheus/2.33.4" 2023-06-03T19:47:56.530 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:47:56 smithi089 bash[78452]: audit 1685821676.2094169 mon.a (mon.0) 255 : audit 0 from='client.? 172.21.15.89:0/215452255' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:47:56.542 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 78 16 88675 88659 71.034 38.75 0.0292525 0.0140703 2023-06-03T19:47:56.585 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:47:56 smithi164 bash[59462]: audit 1685821676.2094169 mon.a (mon.0) 255 : audit 0 from='client.? 172.21.15.89:0/215452255' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:47:57.265 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 19:47:56 smithi089 bash[71163]: debug 2023-06-03T19:47:56.786+0000 7fe48642e700 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2023-06-03T19:47:57.530 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:47:57 smithi089 bash[78452]: cluster 1685821676.1238976 mgr.y (mgr.34104) 273 : cluster 0 pgmap v216: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 81 active+clean; 5.3 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 36 MiB/s wr, 574 op/s; 19989/260067 objects degraded (7.686%); 5.5 MiB/s, 88 objects/s recovering 2023-06-03T19:47:57.530 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:47:57 smithi089 bash[78452]: audit 1685821676.7893634 mon.c (mon.2) 105 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-06-03T19:47:57.530 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:47:57 smithi089 bash[78452]: cluster 1685821677.2204833 mon.a (mon.0) 256 : cluster 3 Health check update: Degraded data redundancy: 19989/260067 objects degraded (7.686%), 8 pgs degraded (PG_DEGRADED) 2023-06-03T19:47:57.530 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:47:57 smithi089 bash[75068]: cluster 1685821676.1238976 mgr.y (mgr.34104) 273 : cluster 0 pgmap v216: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 81 active+clean; 5.3 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 36 MiB/s wr, 574 op/s; 19989/260067 objects degraded (7.686%); 5.5 MiB/s, 88 objects/s recovering 2023-06-03T19:47:57.530 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:47:57 smithi089 bash[75068]: audit 1685821676.7893634 mon.c (mon.2) 105 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-06-03T19:47:57.531 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:47:57 smithi089 bash[75068]: cluster 1685821677.2204833 mon.a (mon.0) 256 : cluster 3 Health check update: Degraded data redundancy: 19989/260067 objects degraded (7.686%), 8 pgs degraded (PG_DEGRADED) 2023-06-03T19:47:57.542 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 79 16 89366 89350 70.6815 43.1875 0.0418943 0.0141403 2023-06-03T19:47:57.585 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:47:57 smithi164 bash[59462]: cluster 1685821676.1238976 mgr.y (mgr.34104) 273 : cluster 0 pgmap v216: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 81 active+clean; 5.3 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 36 MiB/s wr, 574 op/s; 19989/260067 objects degraded (7.686%); 5.5 MiB/s, 88 objects/s recovering 2023-06-03T19:47:57.585 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:47:57 smithi164 bash[59462]: audit 1685821676.7893634 mon.c (mon.2) 105 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-06-03T19:47:57.586 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:47:57 smithi164 bash[59462]: cluster 1685821677.2204833 mon.a (mon.0) 256 : cluster 3 Health check update: Degraded data redundancy: 19989/260067 objects degraded (7.686%), 8 pgs degraded (PG_DEGRADED) 2023-06-03T19:47:58.530 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:47:58 smithi089 bash[75068]: audit 1685821676.789635 mgr.y (mgr.34104) 274 : audit 0 from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-06-03T19:47:58.530 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:47:58 smithi089 bash[75068]: cephadm 1685821676.7912838 mgr.y (mgr.34104) 275 : cephadm 1 Upgrade: unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2023-06-03T19:47:58.530 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:47:58 smithi089 bash[78452]: audit 1685821676.789635 mgr.y (mgr.34104) 274 : audit 0 from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-06-03T19:47:58.531 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:47:58 smithi089 bash[78452]: cephadm 1685821676.7912838 mgr.y (mgr.34104) 275 : cephadm 1 Upgrade: unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2023-06-03T19:47:58.542 INFO:tasks.radosbench.radosbench.0.smithi089.stdout:2023-06-03T19:47:58.538149+0000 min lat: 0.00135761 max lat: 2.96402 avg lat: 0.0142097 2023-06-03T19:47:58.543 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-06-03T19:47:58.543 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 80 16 90056 90040 70.337 43.125 0.0146236 0.0142097 2023-06-03T19:47:58.585 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:47:58 smithi164 bash[59462]: audit 1685821676.789635 mgr.y (mgr.34104) 274 : audit 0 from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-06-03T19:47:58.585 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:47:58 smithi164 bash[59462]: cephadm 1685821676.7912838 mgr.y (mgr.34104) 275 : cephadm 1 Upgrade: unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2023-06-03T19:47:59.543 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 81 16 90745 90729 70.0002 43.0625 0.0282491 0.0142795 2023-06-03T19:47:59.585 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:47:59 smithi164 bash[59462]: cluster 1685821678.1248271 mgr.y (mgr.34104) 276 : cluster 0 pgmap v217: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 81 active+clean; 5.3 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 36 MiB/s wr, 574 op/s; 19989/260067 objects degraded (7.686%); 5.5 MiB/s, 88 objects/s recovering 2023-06-03T19:47:59.586 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:47:59 smithi164 bash[59462]: audit 1685821678.8360083 mon.a (mon.0) 257 : audit 0 from='client.? 172.21.15.89:0/3694000085' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:47:59.779 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:47:59 smithi089 bash[78452]: cluster 1685821678.1248271 mgr.y (mgr.34104) 276 : cluster 0 pgmap v217: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 81 active+clean; 5.3 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 36 MiB/s wr, 574 op/s; 19989/260067 objects degraded (7.686%); 5.5 MiB/s, 88 objects/s recovering 2023-06-03T19:47:59.780 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:47:59 smithi089 bash[78452]: audit 1685821678.8360083 mon.a (mon.0) 257 : audit 0 from='client.? 172.21.15.89:0/3694000085' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:47:59.780 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:47:59 smithi089 bash[75068]: cluster 1685821678.1248271 mgr.y (mgr.34104) 276 : cluster 0 pgmap v217: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 81 active+clean; 5.3 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 36 MiB/s wr, 574 op/s; 19989/260067 objects degraded (7.686%); 5.5 MiB/s, 88 objects/s recovering 2023-06-03T19:47:59.780 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:47:59 smithi089 bash[75068]: audit 1685821678.8360083 mon.a (mon.0) 257 : audit 0 from='client.? 172.21.15.89:0/3694000085' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:48:00.543 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 82 16 91458 91442 69.6899 44.5625 0.00655392 0.0143276 2023-06-03T19:48:01.543 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 83 16 91675 91659 69.0136 13.5625 0.0212458 0.0144838 2023-06-03T19:48:01.585 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:48:01 smithi164 bash[59462]: cluster 1685821680.1256218 mgr.y (mgr.34104) 277 : cluster 0 pgmap v218: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 81 active+clean; 5.5 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 53 MiB/s wr, 851 op/s; 19384/270057 objects degraded (7.178%); 8.7 MiB/s, 139 objects/s recovering 2023-06-03T19:48:01.586 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:48:01 smithi164 bash[59462]: cluster 1685821680.2884955 osd.3 (osd.3) 15 : cluster 3 failed to encode map e89 with expected crc 2023-06-03T19:48:01.586 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:48:01 smithi164 bash[59462]: cluster 1685821680.2898135 osd.4 (osd.4) 13 : cluster 3 failed to encode map e89 with expected crc 2023-06-03T19:48:01.586 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:48:01 smithi164 bash[59462]: cluster 1685821680.2910728 mon.a (mon.0) 258 : cluster 0 osdmap e89: 8 total, 8 up, 8 in 2023-06-03T19:48:01.586 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:48:01 smithi164 bash[59462]: cluster 1685821680.2960155 osd.2 (osd.2) 15 : cluster 3 failed to encode map e89 with expected crc 2023-06-03T19:48:01.586 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:48:01 smithi164 bash[59462]: audit 1685821681.2737625 mon.a (mon.0) 259 : audit 0 from='client.? 172.21.15.89:0/1228332509' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:48:01.779 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:48:01 smithi089 bash[78452]: cluster 1685821680.1256218 mgr.y (mgr.34104) 277 : cluster 0 pgmap v218: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 81 active+clean; 5.5 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 53 MiB/s wr, 851 op/s; 19384/270057 objects degraded (7.178%); 8.7 MiB/s, 139 objects/s recovering 2023-06-03T19:48:01.780 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:48:01 smithi089 bash[78452]: cluster 1685821680.2884955 osd.3 (osd.3) 15 : cluster 3 failed to encode map e89 with expected crc 2023-06-03T19:48:01.780 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:48:01 smithi089 bash[78452]: cluster 1685821680.2898135 osd.4 (osd.4) 13 : cluster 3 failed to encode map e89 with expected crc 2023-06-03T19:48:01.780 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:48:01 smithi089 bash[78452]: cluster 1685821680.2910728 mon.a (mon.0) 258 : cluster 0 osdmap e89: 8 total, 8 up, 8 in 2023-06-03T19:48:01.780 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:48:01 smithi089 bash[78452]: cluster 1685821680.2960155 osd.2 (osd.2) 15 : cluster 3 failed to encode map e89 with expected crc 2023-06-03T19:48:01.780 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:48:01 smithi089 bash[78452]: audit 1685821681.2737625 mon.a (mon.0) 259 : audit 0 from='client.? 172.21.15.89:0/1228332509' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:48:01.780 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:48:01 smithi089 bash[75068]: cluster 1685821680.1256218 mgr.y (mgr.34104) 277 : cluster 0 pgmap v218: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 81 active+clean; 5.5 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 53 MiB/s wr, 851 op/s; 19384/270057 objects degraded (7.178%); 8.7 MiB/s, 139 objects/s recovering 2023-06-03T19:48:01.781 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:48:01 smithi089 bash[75068]: cluster 1685821680.2884955 osd.3 (osd.3) 15 : cluster 3 failed to encode map e89 with expected crc 2023-06-03T19:48:01.781 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:48:01 smithi089 bash[75068]: cluster 1685821680.2898135 osd.4 (osd.4) 13 : cluster 3 failed to encode map e89 with expected crc 2023-06-03T19:48:01.781 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:48:01 smithi089 bash[75068]: cluster 1685821680.2910728 mon.a (mon.0) 258 : cluster 0 osdmap e89: 8 total, 8 up, 8 in 2023-06-03T19:48:01.781 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:48:01 smithi089 bash[75068]: cluster 1685821680.2960155 osd.2 (osd.2) 15 : cluster 3 failed to encode map e89 with expected crc 2023-06-03T19:48:01.781 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:48:01 smithi089 bash[75068]: audit 1685821681.2737625 mon.a (mon.0) 259 : audit 0 from='client.? 172.21.15.89:0/1228332509' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:48:02.543 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 84 16 92363 92347 68.7039 43 0.0109063 0.0145461 2023-06-03T19:48:02.585 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:48:02 smithi164 bash[59462]: cluster 1685821681.2973626 osd.2 (osd.2) 16 : cluster 3 failed to encode map e90 with expected crc 2023-06-03T19:48:02.586 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:48:02 smithi164 bash[59462]: cluster 1685821681.2981386 osd.3 (osd.3) 16 : cluster 3 failed to encode map e90 with expected crc 2023-06-03T19:48:02.586 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:48:02 smithi164 bash[59462]: cluster 1685821681.30048 mon.a (mon.0) 260 : cluster 0 osdmap e90: 8 total, 8 up, 8 in 2023-06-03T19:48:02.586 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:48:02 smithi164 bash[59462]: cluster 1685821681.302048 osd.4 (osd.4) 14 : cluster 3 failed to encode map e90 with expected crc 2023-06-03T19:48:02.586 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:48:02 smithi164 bash[59462]: cluster 1685821681.304436 osd.1 (osd.1) 14 : cluster 3 failed to encode map e90 with expected crc 2023-06-03T19:48:02.586 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:48:02 smithi164 bash[59462]: cluster 1685821681.331635 osd.6 (osd.6) 11 : cluster 3 failed to encode map e90 with expected crc 2023-06-03T19:48:02.586 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:48:02 smithi164 bash[59462]: cluster 1685821682.2217455 mon.a (mon.0) 261 : cluster 3 Health check update: Degraded data redundancy: 19384/270057 objects degraded (7.178%), 8 pgs degraded (PG_DEGRADED) 2023-06-03T19:48:02.779 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:48:02 smithi089 bash[75068]: cluster 1685821681.2973626 osd.2 (osd.2) 16 : cluster 3 failed to encode map e90 with expected crc 2023-06-03T19:48:02.780 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:48:02 smithi089 bash[75068]: cluster 1685821681.2981386 osd.3 (osd.3) 16 : cluster 3 failed to encode map e90 with expected crc 2023-06-03T19:48:02.780 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:48:02 smithi089 bash[75068]: cluster 1685821681.30048 mon.a (mon.0) 260 : cluster 0 osdmap e90: 8 total, 8 up, 8 in 2023-06-03T19:48:02.780 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:48:02 smithi089 bash[75068]: cluster 1685821681.302048 osd.4 (osd.4) 14 : cluster 3 failed to encode map e90 with expected crc 2023-06-03T19:48:02.780 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:48:02 smithi089 bash[75068]: cluster 1685821681.304436 osd.1 (osd.1) 14 : cluster 3 failed to encode map e90 with expected crc 2023-06-03T19:48:02.780 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:48:02 smithi089 bash[75068]: cluster 1685821681.331635 osd.6 (osd.6) 11 : cluster 3 failed to encode map e90 with expected crc 2023-06-03T19:48:02.781 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:48:02 smithi089 bash[75068]: cluster 1685821682.2217455 mon.a (mon.0) 261 : cluster 3 Health check update: Degraded data redundancy: 19384/270057 objects degraded (7.178%), 8 pgs degraded (PG_DEGRADED) 2023-06-03T19:48:02.781 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:48:02 smithi089 bash[78452]: cluster 1685821681.2973626 osd.2 (osd.2) 16 : cluster 3 failed to encode map e90 with expected crc 2023-06-03T19:48:02.781 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:48:02 smithi089 bash[78452]: cluster 1685821681.2981386 osd.3 (osd.3) 16 : cluster 3 failed to encode map e90 with expected crc 2023-06-03T19:48:02.781 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:48:02 smithi089 bash[78452]: cluster 1685821681.30048 mon.a (mon.0) 260 : cluster 0 osdmap e90: 8 total, 8 up, 8 in 2023-06-03T19:48:02.781 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:48:02 smithi089 bash[78452]: cluster 1685821681.302048 osd.4 (osd.4) 14 : cluster 3 failed to encode map e90 with expected crc 2023-06-03T19:48:02.781 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:48:02 smithi089 bash[78452]: cluster 1685821681.304436 osd.1 (osd.1) 14 : cluster 3 failed to encode map e90 with expected crc 2023-06-03T19:48:02.781 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:48:02 smithi089 bash[78452]: cluster 1685821681.331635 osd.6 (osd.6) 11 : cluster 3 failed to encode map e90 with expected crc 2023-06-03T19:48:02.782 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:48:02 smithi089 bash[78452]: cluster 1685821682.2217455 mon.a (mon.0) 261 : cluster 3 Health check update: Degraded data redundancy: 19384/270057 objects degraded (7.178%), 8 pgs degraded (PG_DEGRADED) 2023-06-03T19:48:03.543 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 85 16 93059 93043 68.4073 43.5 0.0584328 0.0146115 2023-06-03T19:48:03.585 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:48:03 smithi164 bash[59462]: cluster 1685821682.1260993 mgr.y (mgr.34104) 278 : cluster 0 pgmap v221: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 81 active+clean; 5.5 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 26 MiB/s wr, 416 op/s; 19384/270057 objects degraded (7.178%); 4.7 MiB/s, 75 objects/s recovering 2023-06-03T19:48:03.779 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:48:03 smithi089 bash[75068]: cluster 1685821682.1260993 mgr.y (mgr.34104) 278 : cluster 0 pgmap v221: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 81 active+clean; 5.5 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 26 MiB/s wr, 416 op/s; 19384/270057 objects degraded (7.178%); 4.7 MiB/s, 75 objects/s recovering 2023-06-03T19:48:03.780 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:48:03 smithi089 bash[78452]: cluster 1685821682.1260993 mgr.y (mgr.34104) 278 : cluster 0 pgmap v221: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 81 active+clean; 5.5 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 26 MiB/s wr, 416 op/s; 19384/270057 objects degraded (7.178%); 4.7 MiB/s, 75 objects/s recovering 2023-06-03T19:48:04.543 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 86 16 93742 93726 68.1082 42.6875 0.0403561 0.0146757 2023-06-03T19:48:04.585 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:48:04 smithi164 bash[59462]: audit 1685821683.8405025 mon.a (mon.0) 262 : audit 0 from='client.? 172.21.15.89:0/2734685957' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:48:04.779 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:48:04 smithi089 bash[75068]: audit 1685821683.8405025 mon.a (mon.0) 262 : audit 0 from='client.? 172.21.15.89:0/2734685957' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:48:04.780 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:48:04 smithi089 bash[78452]: audit 1685821683.8405025 mon.a (mon.0) 262 : audit 0 from='client.? 172.21.15.89:0/2734685957' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:48:05.543 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 87 16 94490 94474 67.8627 46.75 0.0179441 0.014729 2023-06-03T19:48:05.585 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:48:05 smithi164 bash[59462]: cluster 1685821684.1269298 mgr.y (mgr.34104) 279 : cluster 0 pgmap v222: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 3 active+recovering+undersized+degraded+remapped, 81 active+clean; 5.7 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 48 MiB/s wr, 769 op/s; 19040/278535 objects degraded (6.836%); 7.4 MiB/s, 118 objects/s recovering 2023-06-03T19:48:05.779 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:48:05 smithi089 bash[75068]: cluster 1685821684.1269298 mgr.y (mgr.34104) 279 : cluster 0 pgmap v222: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 3 active+recovering+undersized+degraded+remapped, 81 active+clean; 5.7 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 48 MiB/s wr, 769 op/s; 19040/278535 objects degraded (6.836%); 7.4 MiB/s, 118 objects/s recovering 2023-06-03T19:48:05.780 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:48:05 smithi089 bash[78452]: cluster 1685821684.1269298 mgr.y (mgr.34104) 279 : cluster 0 pgmap v222: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 3 active+recovering+undersized+degraded+remapped, 81 active+clean; 5.7 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 48 MiB/s wr, 769 op/s; 19040/278535 objects degraded (6.836%); 7.4 MiB/s, 118 objects/s recovering 2023-06-03T19:48:06.529 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 19:48:06 smithi089 bash[71163]: ::ffff:172.21.15.164 - - [03/Jun/2023:19:48:06] "GET /metrics HTTP/1.1" 200 207241 "" "Prometheus/2.33.4" 2023-06-03T19:48:06.543 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 88 16 95199 95183 67.595 44.3125 0.0197807 0.0147857 2023-06-03T19:48:07.543 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 89 16 95937 95921 67.3537 46.125 0.0139506 0.0148409 2023-06-03T19:48:07.585 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:48:07 smithi164 bash[59462]: cluster 1685821686.1275833 mgr.y (mgr.34104) 280 : cluster 0 pgmap v223: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.7 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 50 MiB/s wr, 799 op/s; 18867/279273 objects degraded (6.756%); 8.8 MiB/s, 140 objects/s recovering 2023-06-03T19:48:07.586 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:48:07 smithi164 bash[59462]: audit 1685821686.438032 mon.b (mon.1) 9 : audit 0 from='client.? 172.21.15.89:0/1811910893' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:48:07.586 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:48:07 smithi164 bash[59462]: audit 1685821687.069924 mon.c (mon.2) 106 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T19:48:07.586 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:48:07 smithi164 bash[59462]: cluster 1685821687.2232172 mon.a (mon.0) 263 : cluster 3 Health check update: Degraded data redundancy: 18867/279273 objects degraded (6.756%), 7 pgs degraded (PG_DEGRADED) 2023-06-03T19:48:07.779 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:48:07 smithi089 bash[75068]: cluster 1685821686.1275833 mgr.y (mgr.34104) 280 : cluster 0 pgmap v223: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.7 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 50 MiB/s wr, 799 op/s; 18867/279273 objects degraded (6.756%); 8.8 MiB/s, 140 objects/s recovering 2023-06-03T19:48:07.779 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:48:07 smithi089 bash[75068]: audit 1685821686.438032 mon.b (mon.1) 9 : audit 0 from='client.? 172.21.15.89:0/1811910893' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:48:07.780 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:48:07 smithi089 bash[75068]: audit 1685821687.069924 mon.c (mon.2) 106 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T19:48:07.780 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:48:07 smithi089 bash[75068]: cluster 1685821687.2232172 mon.a (mon.0) 263 : cluster 3 Health check update: Degraded data redundancy: 18867/279273 objects degraded (6.756%), 7 pgs degraded (PG_DEGRADED) 2023-06-03T19:48:07.780 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:48:07 smithi089 bash[78452]: cluster 1685821686.1275833 mgr.y (mgr.34104) 280 : cluster 0 pgmap v223: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.7 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 50 MiB/s wr, 799 op/s; 18867/279273 objects degraded (6.756%); 8.8 MiB/s, 140 objects/s recovering 2023-06-03T19:48:07.780 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:48:07 smithi089 bash[78452]: audit 1685821686.438032 mon.b (mon.1) 9 : audit 0 from='client.? 172.21.15.89:0/1811910893' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:48:07.780 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:48:07 smithi089 bash[78452]: audit 1685821687.069924 mon.c (mon.2) 106 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T19:48:07.781 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:48:07 smithi089 bash[78452]: cluster 1685821687.2232172 mon.a (mon.0) 263 : cluster 3 Health check update: Degraded data redundancy: 18867/279273 objects degraded (6.756%), 7 pgs degraded (PG_DEGRADED) 2023-06-03T19:48:08.543 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 90 13 96664 96651 67.1122 45.625 0.0233121 0.0148941 2023-06-03T19:48:09.544 INFO:tasks.radosbench.radosbench.0.smithi089.stdout:Total time run: 90.024 2023-06-03T19:48:09.544 INFO:tasks.radosbench.radosbench.0.smithi089.stdout:Total writes made: 96664 2023-06-03T19:48:09.544 INFO:tasks.radosbench.radosbench.0.smithi089.stdout:Write size: 65536 2023-06-03T19:48:09.544 INFO:tasks.radosbench.radosbench.0.smithi089.stdout:Object size: 65536 2023-06-03T19:48:09.545 INFO:tasks.radosbench.radosbench.0.smithi089.stdout:Bandwidth (MB/sec): 67.1099 2023-06-03T19:48:09.545 INFO:tasks.radosbench.radosbench.0.smithi089.stdout:Stddev Bandwidth: 40.6069 2023-06-03T19:48:09.545 INFO:tasks.radosbench.radosbench.0.smithi089.stdout:Max bandwidth (MB/sec): 198.688 2023-06-03T19:48:09.545 INFO:tasks.radosbench.radosbench.0.smithi089.stdout:Min bandwidth (MB/sec): 0 2023-06-03T19:48:09.545 INFO:tasks.radosbench.radosbench.0.smithi089.stdout:Average IOPS: 1073 2023-06-03T19:48:09.545 INFO:tasks.radosbench.radosbench.0.smithi089.stdout:Stddev IOPS: 649.71 2023-06-03T19:48:09.545 INFO:tasks.radosbench.radosbench.0.smithi089.stdout:Max IOPS: 3179 2023-06-03T19:48:09.545 INFO:tasks.radosbench.radosbench.0.smithi089.stdout:Min IOPS: 0 2023-06-03T19:48:09.546 INFO:tasks.radosbench.radosbench.0.smithi089.stdout:Average Latency(s): 0.014895 2023-06-03T19:48:09.546 INFO:tasks.radosbench.radosbench.0.smithi089.stdout:Stddev Latency(s): 0.0433309 2023-06-03T19:48:09.546 INFO:tasks.radosbench.radosbench.0.smithi089.stdout:Max latency(s): 2.96402 2023-06-03T19:48:09.546 INFO:tasks.radosbench.radosbench.0.smithi089.stdout:Min latency(s): 0.00135761 2023-06-03T19:48:09.560 INFO:tasks.radosbench.radosbench.0.smithi089.stdout:Cleaning up (deleting benchmark objects) 2023-06-03T19:48:09.779 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:48:09 smithi089 bash[78452]: cluster 1685821688.1281488 mgr.y (mgr.34104) 281 : cluster 0 pgmap v224: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.7 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 24 MiB/s wr, 383 op/s; 18867/279273 objects degraded (6.756%); 4.0 MiB/s, 64 objects/s recovering 2023-06-03T19:48:09.779 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:48:09 smithi089 bash[78452]: audit 1685821688.8611202 mon.c (mon.2) 107 : audit 0 from='client.? 172.21.15.89:0/1876810712' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:48:09.780 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:48:09 smithi089 bash[75068]: cluster 1685821688.1281488 mgr.y (mgr.34104) 281 : cluster 0 pgmap v224: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.7 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 24 MiB/s wr, 383 op/s; 18867/279273 objects degraded (6.756%); 4.0 MiB/s, 64 objects/s recovering 2023-06-03T19:48:09.780 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:48:09 smithi089 bash[75068]: audit 1685821688.8611202 mon.c (mon.2) 107 : audit 0 from='client.? 172.21.15.89:0/1876810712' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:48:09.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:48:09 smithi164 bash[59462]: cluster 1685821688.1281488 mgr.y (mgr.34104) 281 : cluster 0 pgmap v224: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.7 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 24 MiB/s wr, 383 op/s; 18867/279273 objects degraded (6.756%); 4.0 MiB/s, 64 objects/s recovering 2023-06-03T19:48:09.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:48:09 smithi164 bash[59462]: audit 1685821688.8611202 mon.c (mon.2) 107 : audit 0 from='client.? 172.21.15.89:0/1876810712' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:48:11.779 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:48:11 smithi089 bash[75068]: cluster 1685821690.129102 mgr.y (mgr.34104) 282 : cluster 0 pgmap v225: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.9 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 42 MiB/s wr, 667 op/s; 18533/289608 objects degraded (6.399%); 5.4 MiB/s, 86 objects/s recovering 2023-06-03T19:48:11.780 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:48:11 smithi089 bash[78452]: cluster 1685821690.129102 mgr.y (mgr.34104) 282 : cluster 0 pgmap v225: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.9 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 42 MiB/s wr, 667 op/s; 18533/289608 objects degraded (6.399%); 5.4 MiB/s, 86 objects/s recovering 2023-06-03T19:48:11.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:48:11 smithi164 bash[59462]: cluster 1685821690.129102 mgr.y (mgr.34104) 282 : cluster 0 pgmap v225: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.9 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 42 MiB/s wr, 667 op/s; 18533/289608 objects degraded (6.399%); 5.4 MiB/s, 86 objects/s recovering 2023-06-03T19:48:12.779 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:48:12 smithi089 bash[75068]: audit 1685821691.437392 mon.b (mon.1) 10 : audit 0 from='client.? 172.21.15.89:0/2530210860' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:48:12.780 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:48:12 smithi089 bash[75068]: audit 1685821691.8056538 mon.c (mon.2) 108 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-03T19:48:12.780 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:48:12 smithi089 bash[75068]: cluster 1685821692.2245677 mon.a (mon.0) 264 : cluster 3 Health check update: Degraded data redundancy: 18533/289608 objects degraded (6.399%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-06-03T19:48:12.780 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:48:12 smithi089 bash[78452]: audit 1685821691.437392 mon.b (mon.1) 10 : audit 0 from='client.? 172.21.15.89:0/2530210860' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:48:12.781 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:48:12 smithi089 bash[78452]: audit 1685821691.8056538 mon.c (mon.2) 108 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-03T19:48:12.781 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:48:12 smithi089 bash[78452]: cluster 1685821692.2245677 mon.a (mon.0) 264 : cluster 3 Health check update: Degraded data redundancy: 18533/289608 objects degraded (6.399%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-06-03T19:48:12.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:48:12 smithi164 bash[59462]: audit 1685821691.437392 mon.b (mon.1) 10 : audit 0 from='client.? 172.21.15.89:0/2530210860' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:48:12.836 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:48:12 smithi164 bash[59462]: audit 1685821691.8056538 mon.c (mon.2) 108 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-03T19:48:12.836 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:48:12 smithi164 bash[59462]: cluster 1685821692.2245677 mon.a (mon.0) 264 : cluster 3 Health check update: Degraded data redundancy: 18533/289608 objects degraded (6.399%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-06-03T19:48:13.779 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:48:13 smithi089 bash[78452]: cluster 1685821692.1295874 mgr.y (mgr.34104) 283 : cluster 0 pgmap v226: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.9 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 38 MiB/s wr, 606 op/s; 18533/289608 objects degraded (6.399%); 4.9 MiB/s, 78 objects/s recovering 2023-06-03T19:48:13.780 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:48:13 smithi089 bash[75068]: cluster 1685821692.1295874 mgr.y (mgr.34104) 283 : cluster 0 pgmap v226: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.9 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 38 MiB/s wr, 606 op/s; 18533/289608 objects degraded (6.399%); 4.9 MiB/s, 78 objects/s recovering 2023-06-03T19:48:13.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:48:13 smithi164 bash[59462]: cluster 1685821692.1295874 mgr.y (mgr.34104) 283 : cluster 0 pgmap v226: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.9 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 38 MiB/s wr, 606 op/s; 18533/289608 objects degraded (6.399%); 4.9 MiB/s, 78 objects/s recovering 2023-06-03T19:48:14.779 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:48:14 smithi089 bash[78452]: audit 1685821694.0529108 mon.c (mon.2) 109 : audit 0 from='client.? 172.21.15.89:0/2093609198' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:48:14.779 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:48:14 smithi089 bash[75068]: audit 1685821694.0529108 mon.c (mon.2) 109 : audit 0 from='client.? 172.21.15.89:0/2093609198' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:48:14.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:48:14 smithi164 bash[59462]: audit 1685821694.0529108 mon.c (mon.2) 109 : audit 0 from='client.? 172.21.15.89:0/2093609198' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:48:15.779 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:48:15 smithi089 bash[78452]: cluster 1685821694.1304908 mgr.y (mgr.34104) 284 : cluster 0 pgmap v227: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.7 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 85 B/s rd, 35 MiB/s wr, 774 op/s; 18068/282195 objects degraded (6.403%); 6.9 MiB/s, 109 objects/s recovering 2023-06-03T19:48:15.780 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:48:15 smithi089 bash[75068]: cluster 1685821694.1304908 mgr.y (mgr.34104) 284 : cluster 0 pgmap v227: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.7 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 85 B/s rd, 35 MiB/s wr, 774 op/s; 18068/282195 objects degraded (6.403%); 6.9 MiB/s, 109 objects/s recovering 2023-06-03T19:48:15.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:48:15 smithi164 bash[59462]: cluster 1685821694.1304908 mgr.y (mgr.34104) 284 : cluster 0 pgmap v227: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.7 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 85 B/s rd, 35 MiB/s wr, 774 op/s; 18068/282195 objects degraded (6.403%); 6.9 MiB/s, 109 objects/s recovering 2023-06-03T19:48:16.529 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 19:48:16 smithi089 bash[71163]: ::ffff:172.21.15.164 - - [03/Jun/2023:19:48:16] "GET /metrics HTTP/1.1" 200 207337 "" "Prometheus/2.33.4" 2023-06-03T19:48:17.779 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:48:17 smithi089 bash[78452]: cluster 1685821696.1311364 mgr.y (mgr.34104) 285 : cluster 0 pgmap v228: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.7 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 85 B/s rd, 20 MiB/s wr, 556 op/s; 18068/281544 objects degraded (6.417%); 5.1 MiB/s, 80 objects/s recovering 2023-06-03T19:48:17.780 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:48:17 smithi089 bash[78452]: audit 1685821696.6596038 mon.c (mon.2) 110 : audit 0 from='client.? 172.21.15.89:0/3777593749' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:48:17.780 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:48:17 smithi089 bash[78452]: cluster 1685821697.2283263 mon.a (mon.0) 265 : cluster 3 Health check update: Degraded data redundancy: 18068/281544 objects degraded (6.417%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-06-03T19:48:17.780 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:48:17 smithi089 bash[75068]: cluster 1685821696.1311364 mgr.y (mgr.34104) 285 : cluster 0 pgmap v228: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.7 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 85 B/s rd, 20 MiB/s wr, 556 op/s; 18068/281544 objects degraded (6.417%); 5.1 MiB/s, 80 objects/s recovering 2023-06-03T19:48:17.780 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:48:17 smithi089 bash[75068]: audit 1685821696.6596038 mon.c (mon.2) 110 : audit 0 from='client.? 172.21.15.89:0/3777593749' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:48:17.781 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:48:17 smithi089 bash[75068]: cluster 1685821697.2283263 mon.a (mon.0) 265 : cluster 3 Health check update: Degraded data redundancy: 18068/281544 objects degraded (6.417%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-06-03T19:48:17.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:48:17 smithi164 bash[59462]: cluster 1685821696.1311364 mgr.y (mgr.34104) 285 : cluster 0 pgmap v228: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.7 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 85 B/s rd, 20 MiB/s wr, 556 op/s; 18068/281544 objects degraded (6.417%); 5.1 MiB/s, 80 objects/s recovering 2023-06-03T19:48:17.836 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:48:17 smithi164 bash[59462]: audit 1685821696.6596038 mon.c (mon.2) 110 : audit 0 from='client.? 172.21.15.89:0/3777593749' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:48:17.836 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:48:17 smithi164 bash[59462]: cluster 1685821697.2283263 mon.a (mon.0) 265 : cluster 3 Health check update: Degraded data redundancy: 18068/281544 objects degraded (6.417%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-06-03T19:48:19.530 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:48:19 smithi089 bash[75068]: cluster 1685821698.1318185 mgr.y (mgr.34104) 286 : cluster 0 pgmap v229: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.7 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 85 B/s rd, 19 MiB/s wr, 536 op/s; 18068/281544 objects degraded (6.417%); 4.2 MiB/s, 66 objects/s recovering 2023-06-03T19:48:19.530 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:48:19 smithi089 bash[75068]: audit 1685821698.3300347 mon.a (mon.0) 266 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:48:19.530 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:48:19 smithi089 bash[75068]: audit 1685821698.3445835 mon.a (mon.0) 267 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:48:19.530 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:48:19 smithi089 bash[75068]: audit 1685821698.4243374 mon.a (mon.0) 268 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:48:19.530 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:48:19 smithi089 bash[75068]: audit 1685821698.4375117 mon.a (mon.0) 269 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:48:19.530 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:48:19 smithi089 bash[75068]: audit 1685821699.0757074 mon.c (mon.2) 111 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-03T19:48:19.531 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:48:19 smithi089 bash[75068]: audit 1685821699.0772598 mon.c (mon.2) 112 : audit 1 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-03T19:48:19.531 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:48:19 smithi089 bash[75068]: audit 1685821699.0900881 mon.a (mon.0) 270 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:48:19.531 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:48:19 smithi089 bash[75068]: audit 1685821699.177302 mon.c (mon.2) 113 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-03T19:48:19.531 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:48:19 smithi089 bash[75068]: audit 1685821699.185971 mon.c (mon.2) 114 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-06-03T19:48:19.531 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:48:19 smithi089 bash[75068]: audit 1685821699.1888857 mon.c (mon.2) 115 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-06-03T19:48:19.531 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:48:19 smithi089 bash[75068]: audit 1685821699.1913774 mon.c (mon.2) 116 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-06-03T19:48:19.531 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:48:19 smithi089 bash[75068]: audit 1685821699.1945446 mon.c (mon.2) 117 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-06-03T19:48:19.532 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:48:19 smithi089 bash[75068]: audit 1685821699.319309 mon.c (mon.2) 118 : audit 0 from='client.? 172.21.15.89:0/401886650' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:48:19.532 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 19:48:19 smithi089 bash[71163]: debug 2023-06-03T19:48:19.190+0000 7fe48642e700 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2023-06-03T19:48:19.532 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:48:19 smithi089 bash[78452]: cluster 1685821698.1318185 mgr.y (mgr.34104) 286 : cluster 0 pgmap v229: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.7 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 85 B/s rd, 19 MiB/s wr, 536 op/s; 18068/281544 objects degraded (6.417%); 4.2 MiB/s, 66 objects/s recovering 2023-06-03T19:48:19.532 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:48:19 smithi089 bash[78452]: audit 1685821698.3300347 mon.a (mon.0) 266 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:48:19.533 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:48:19 smithi089 bash[78452]: audit 1685821698.3445835 mon.a (mon.0) 267 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:48:19.533 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:48:19 smithi089 bash[78452]: audit 1685821698.4243374 mon.a (mon.0) 268 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:48:19.533 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:48:19 smithi089 bash[78452]: audit 1685821698.4375117 mon.a (mon.0) 269 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:48:19.533 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:48:19 smithi089 bash[78452]: audit 1685821699.0757074 mon.c (mon.2) 111 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-03T19:48:19.533 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:48:19 smithi089 bash[78452]: audit 1685821699.0772598 mon.c (mon.2) 112 : audit 1 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-03T19:48:19.533 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:48:19 smithi089 bash[78452]: audit 1685821699.0900881 mon.a (mon.0) 270 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:48:19.534 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:48:19 smithi089 bash[78452]: audit 1685821699.177302 mon.c (mon.2) 113 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-03T19:48:19.534 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:48:19 smithi089 bash[78452]: audit 1685821699.185971 mon.c (mon.2) 114 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-06-03T19:48:19.534 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:48:19 smithi089 bash[78452]: audit 1685821699.1888857 mon.c (mon.2) 115 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-06-03T19:48:19.534 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:48:19 smithi089 bash[78452]: audit 1685821699.1913774 mon.c (mon.2) 116 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-06-03T19:48:19.534 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:48:19 smithi089 bash[78452]: audit 1685821699.1945446 mon.c (mon.2) 117 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-06-03T19:48:19.534 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:48:19 smithi089 bash[78452]: audit 1685821699.319309 mon.c (mon.2) 118 : audit 0 from='client.? 172.21.15.89:0/401886650' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:48:19.585 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:48:19 smithi164 bash[59462]: cluster 1685821698.1318185 mgr.y (mgr.34104) 286 : cluster 0 pgmap v229: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.7 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 85 B/s rd, 19 MiB/s wr, 536 op/s; 18068/281544 objects degraded (6.417%); 4.2 MiB/s, 66 objects/s recovering 2023-06-03T19:48:19.586 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:48:19 smithi164 bash[59462]: audit 1685821698.3300347 mon.a (mon.0) 266 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:48:19.586 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:48:19 smithi164 bash[59462]: audit 1685821698.3445835 mon.a (mon.0) 267 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:48:19.586 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:48:19 smithi164 bash[59462]: audit 1685821698.4243374 mon.a (mon.0) 268 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:48:19.586 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:48:19 smithi164 bash[59462]: audit 1685821698.4375117 mon.a (mon.0) 269 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:48:19.586 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:48:19 smithi164 bash[59462]: audit 1685821699.0757074 mon.c (mon.2) 111 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-03T19:48:19.586 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:48:19 smithi164 bash[59462]: audit 1685821699.0772598 mon.c (mon.2) 112 : audit 1 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-03T19:48:19.587 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:48:19 smithi164 bash[59462]: audit 1685821699.0900881 mon.a (mon.0) 270 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:48:19.587 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:48:19 smithi164 bash[59462]: audit 1685821699.177302 mon.c (mon.2) 113 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-03T19:48:19.587 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:48:19 smithi164 bash[59462]: audit 1685821699.185971 mon.c (mon.2) 114 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-06-03T19:48:19.587 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:48:19 smithi164 bash[59462]: audit 1685821699.1888857 mon.c (mon.2) 115 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-06-03T19:48:19.587 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:48:19 smithi164 bash[59462]: audit 1685821699.1913774 mon.c (mon.2) 116 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-06-03T19:48:19.587 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:48:19 smithi164 bash[59462]: audit 1685821699.1945446 mon.c (mon.2) 117 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-06-03T19:48:19.587 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:48:19 smithi164 bash[59462]: audit 1685821699.319309 mon.c (mon.2) 118 : audit 0 from='client.? 172.21.15.89:0/401886650' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:48:20.779 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:48:20 smithi089 bash[75068]: audit 1685821699.194809 mgr.y (mgr.34104) 287 : audit 0 from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-06-03T19:48:20.779 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:48:20 smithi089 bash[75068]: cephadm 1685821699.1966903 mgr.y (mgr.34104) 288 : cephadm 1 Upgrade: unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2023-06-03T19:48:20.780 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:48:20 smithi089 bash[78452]: audit 1685821699.194809 mgr.y (mgr.34104) 287 : audit 0 from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-06-03T19:48:20.780 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:48:20 smithi089 bash[78452]: cephadm 1685821699.1966903 mgr.y (mgr.34104) 288 : cephadm 1 Upgrade: unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2023-06-03T19:48:20.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:48:20 smithi164 bash[59462]: audit 1685821699.194809 mgr.y (mgr.34104) 287 : audit 0 from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-06-03T19:48:20.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:48:20 smithi164 bash[59462]: cephadm 1685821699.1966903 mgr.y (mgr.34104) 288 : cephadm 1 Upgrade: unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2023-06-03T19:48:21.779 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:48:21 smithi089 bash[75068]: cluster 1685821700.1328473 mgr.y (mgr.34104) 289 : cluster 0 pgmap v230: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.5 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 85 B/s rd, 19 MiB/s wr, 812 op/s; 17705/271617 objects degraded (6.518%); 6.0 MiB/s, 96 objects/s recovering 2023-06-03T19:48:21.779 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:48:21 smithi089 bash[75068]: cluster 1685821700.3415534 osd.2 (osd.2) 17 : cluster 3 failed to encode map e91 with expected crc 2023-06-03T19:48:21.779 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:48:21 smithi089 bash[75068]: cluster 1685821700.3445535 osd.3 (osd.3) 17 : cluster 3 failed to encode map e91 with expected crc 2023-06-03T19:48:21.780 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:48:21 smithi089 bash[75068]: cluster 1685821700.3451202 mon.a (mon.0) 271 : cluster 0 osdmap e91: 8 total, 8 up, 8 in 2023-06-03T19:48:21.780 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:48:21 smithi089 bash[75068]: cluster 1685821700.3454456 osd.1 (osd.1) 15 : cluster 3 failed to encode map e91 with expected crc 2023-06-03T19:48:21.780 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:48:21 smithi089 bash[78452]: cluster 1685821700.1328473 mgr.y (mgr.34104) 289 : cluster 0 pgmap v230: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.5 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 85 B/s rd, 19 MiB/s wr, 812 op/s; 17705/271617 objects degraded (6.518%); 6.0 MiB/s, 96 objects/s recovering 2023-06-03T19:48:21.780 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:48:21 smithi089 bash[78452]: cluster 1685821700.3415534 osd.2 (osd.2) 17 : cluster 3 failed to encode map e91 with expected crc 2023-06-03T19:48:21.780 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:48:21 smithi089 bash[78452]: cluster 1685821700.3445535 osd.3 (osd.3) 17 : cluster 3 failed to encode map e91 with expected crc 2023-06-03T19:48:21.781 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:48:21 smithi089 bash[78452]: cluster 1685821700.3451202 mon.a (mon.0) 271 : cluster 0 osdmap e91: 8 total, 8 up, 8 in 2023-06-03T19:48:21.781 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:48:21 smithi089 bash[78452]: cluster 1685821700.3454456 osd.1 (osd.1) 15 : cluster 3 failed to encode map e91 with expected crc 2023-06-03T19:48:21.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:48:21 smithi164 bash[59462]: cluster 1685821700.1328473 mgr.y (mgr.34104) 289 : cluster 0 pgmap v230: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.5 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 85 B/s rd, 19 MiB/s wr, 812 op/s; 17705/271617 objects degraded (6.518%); 6.0 MiB/s, 96 objects/s recovering 2023-06-03T19:48:21.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:48:21 smithi164 bash[59462]: cluster 1685821700.3415534 osd.2 (osd.2) 17 : cluster 3 failed to encode map e91 with expected crc 2023-06-03T19:48:21.836 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:48:21 smithi164 bash[59462]: cluster 1685821700.3445535 osd.3 (osd.3) 17 : cluster 3 failed to encode map e91 with expected crc 2023-06-03T19:48:21.836 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:48:21 smithi164 bash[59462]: cluster 1685821700.3451202 mon.a (mon.0) 271 : cluster 0 osdmap e91: 8 total, 8 up, 8 in 2023-06-03T19:48:21.836 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:48:21 smithi164 bash[59462]: cluster 1685821700.3454456 osd.1 (osd.1) 15 : cluster 3 failed to encode map e91 with expected crc 2023-06-03T19:48:22.779 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:48:22 smithi089 bash[78452]: cluster 1685821701.3413253 osd.5 (osd.5) 12 : cluster 3 failed to encode map e92 with expected crc 2023-06-03T19:48:22.779 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:48:22 smithi089 bash[78452]: cluster 1685821701.3435307 osd.3 (osd.3) 18 : cluster 3 failed to encode map e92 with expected crc 2023-06-03T19:48:22.780 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:48:22 smithi089 bash[78452]: cluster 1685821701.3479946 mon.a (mon.0) 272 : cluster 0 osdmap e92: 8 total, 8 up, 8 in 2023-06-03T19:48:22.780 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:48:22 smithi089 bash[78452]: cluster 1685821701.348117 osd.4 (osd.4) 15 : cluster 3 failed to encode map e92 with expected crc 2023-06-03T19:48:22.780 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:48:22 smithi089 bash[78452]: cluster 1685821701.3518655 osd.1 (osd.1) 16 : cluster 3 failed to encode map e92 with expected crc 2023-06-03T19:48:22.780 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:48:22 smithi089 bash[78452]: cluster 1685821701.3620842 osd.6 (osd.6) 12 : cluster 3 failed to encode map e92 with expected crc 2023-06-03T19:48:22.780 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:48:22 smithi089 bash[78452]: audit 1685821701.8836913 mon.a (mon.0) 273 : audit 0 from='client.? 172.21.15.89:0/2787005835' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:48:22.780 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:48:22 smithi089 bash[78452]: audit 1685821702.072844 mon.c (mon.2) 119 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T19:48:22.780 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:48:22 smithi089 bash[78452]: cluster 1685821702.2307074 mon.a (mon.0) 274 : cluster 3 Health check update: Degraded data redundancy: 17705/271617 objects degraded (6.518%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-06-03T19:48:22.781 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:48:22 smithi089 bash[75068]: cluster 1685821701.3413253 osd.5 (osd.5) 12 : cluster 3 failed to encode map e92 with expected crc 2023-06-03T19:48:22.781 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:48:22 smithi089 bash[75068]: cluster 1685821701.3435307 osd.3 (osd.3) 18 : cluster 3 failed to encode map e92 with expected crc 2023-06-03T19:48:22.781 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:48:22 smithi089 bash[75068]: cluster 1685821701.3479946 mon.a (mon.0) 272 : cluster 0 osdmap e92: 8 total, 8 up, 8 in 2023-06-03T19:48:22.781 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:48:22 smithi089 bash[75068]: cluster 1685821701.348117 osd.4 (osd.4) 15 : cluster 3 failed to encode map e92 with expected crc 2023-06-03T19:48:22.781 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:48:22 smithi089 bash[75068]: cluster 1685821701.3518655 osd.1 (osd.1) 16 : cluster 3 failed to encode map e92 with expected crc 2023-06-03T19:48:22.781 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:48:22 smithi089 bash[75068]: cluster 1685821701.3620842 osd.6 (osd.6) 12 : cluster 3 failed to encode map e92 with expected crc 2023-06-03T19:48:22.782 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:48:22 smithi089 bash[75068]: audit 1685821701.8836913 mon.a (mon.0) 273 : audit 0 from='client.? 172.21.15.89:0/2787005835' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:48:22.782 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:48:22 smithi089 bash[75068]: audit 1685821702.072844 mon.c (mon.2) 119 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T19:48:22.782 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:48:22 smithi089 bash[75068]: cluster 1685821702.2307074 mon.a (mon.0) 274 : cluster 3 Health check update: Degraded data redundancy: 17705/271617 objects degraded (6.518%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-06-03T19:48:22.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:48:22 smithi164 bash[59462]: cluster 1685821701.3413253 osd.5 (osd.5) 12 : cluster 3 failed to encode map e92 with expected crc 2023-06-03T19:48:22.836 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:48:22 smithi164 bash[59462]: cluster 1685821701.3435307 osd.3 (osd.3) 18 : cluster 3 failed to encode map e92 with expected crc 2023-06-03T19:48:22.836 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:48:22 smithi164 bash[59462]: cluster 1685821701.3479946 mon.a (mon.0) 272 : cluster 0 osdmap e92: 8 total, 8 up, 8 in 2023-06-03T19:48:22.836 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:48:22 smithi164 bash[59462]: cluster 1685821701.348117 osd.4 (osd.4) 15 : cluster 3 failed to encode map e92 with expected crc 2023-06-03T19:48:22.836 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:48:22 smithi164 bash[59462]: cluster 1685821701.3518655 osd.1 (osd.1) 16 : cluster 3 failed to encode map e92 with expected crc 2023-06-03T19:48:22.836 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:48:22 smithi164 bash[59462]: cluster 1685821701.3620842 osd.6 (osd.6) 12 : cluster 3 failed to encode map e92 with expected crc 2023-06-03T19:48:22.836 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:48:22 smithi164 bash[59462]: audit 1685821701.8836913 mon.a (mon.0) 273 : audit 0 from='client.? 172.21.15.89:0/2787005835' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:48:22.837 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:48:22 smithi164 bash[59462]: audit 1685821702.072844 mon.c (mon.2) 119 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T19:48:22.837 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:48:22 smithi164 bash[59462]: cluster 1685821702.2307074 mon.a (mon.0) 274 : cluster 3 Health check update: Degraded data redundancy: 17705/271617 objects degraded (6.518%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-06-03T19:48:23.779 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:48:23 smithi089 bash[75068]: cluster 1685821702.1333897 mgr.y (mgr.34104) 290 : cluster 0 pgmap v233: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.5 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 440 op/s; 17705/271617 objects degraded (6.518%); 2.8 MiB/s, 45 objects/s recovering 2023-06-03T19:48:23.780 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:48:23 smithi089 bash[78452]: cluster 1685821702.1333897 mgr.y (mgr.34104) 290 : cluster 0 pgmap v233: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.5 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 440 op/s; 17705/271617 objects degraded (6.518%); 2.8 MiB/s, 45 objects/s recovering 2023-06-03T19:48:23.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:48:23 smithi164 bash[59462]: cluster 1685821702.1333897 mgr.y (mgr.34104) 290 : cluster 0 pgmap v233: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.5 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 440 op/s; 17705/271617 objects degraded (6.518%); 2.8 MiB/s, 45 objects/s recovering 2023-06-03T19:48:25.779 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:48:25 smithi089 bash[78452]: cluster 1685821704.1342003 mgr.y (mgr.34104) 291 : cluster 0 pgmap v234: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.4 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 756 op/s; 17618/263373 objects degraded (6.689%); 3.5 MiB/s, 56 objects/s recovering 2023-06-03T19:48:25.780 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:48:25 smithi089 bash[78452]: audit 1685821704.4491065 mon.c (mon.2) 120 : audit 0 from='client.? 172.21.15.89:0/2179549633' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:48:25.780 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:48:25 smithi089 bash[75068]: cluster 1685821704.1342003 mgr.y (mgr.34104) 291 : cluster 0 pgmap v234: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.4 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 756 op/s; 17618/263373 objects degraded (6.689%); 3.5 MiB/s, 56 objects/s recovering 2023-06-03T19:48:25.780 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:48:25 smithi089 bash[75068]: audit 1685821704.4491065 mon.c (mon.2) 120 : audit 0 from='client.? 172.21.15.89:0/2179549633' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:48:25.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:48:25 smithi164 bash[59462]: cluster 1685821704.1342003 mgr.y (mgr.34104) 291 : cluster 0 pgmap v234: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.4 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 756 op/s; 17618/263373 objects degraded (6.689%); 3.5 MiB/s, 56 objects/s recovering 2023-06-03T19:48:25.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:48:25 smithi164 bash[59462]: audit 1685821704.4491065 mon.c (mon.2) 120 : audit 0 from='client.? 172.21.15.89:0/2179549633' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:48:26.529 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 19:48:26 smithi089 bash[71163]: ::ffff:172.21.15.164 - - [03/Jun/2023:19:48:26] "GET /metrics HTTP/1.1" 200 207373 "" "Prometheus/2.33.4" 2023-06-03T19:48:27.779 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:48:27 smithi089 bash[78452]: cluster 1685821706.1348398 mgr.y (mgr.34104) 292 : cluster 0 pgmap v235: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 5.3 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 814 op/s; 17559/261987 objects degraded (6.702%); 4.0 MiB/s, 63 objects/s recovering 2023-06-03T19:48:27.780 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:48:27 smithi089 bash[78452]: audit 1685821707.0350127 mon.c (mon.2) 121 : audit 0 from='client.? 172.21.15.89:0/1496396553' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:48:27.780 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:48:27 smithi089 bash[78452]: cluster 1685821707.2322266 mon.a (mon.0) 275 : cluster 3 Health check update: Degraded data redundancy: 17559/261987 objects degraded (6.702%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-06-03T19:48:27.781 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:48:27 smithi089 bash[75068]: cluster 1685821706.1348398 mgr.y (mgr.34104) 292 : cluster 0 pgmap v235: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 5.3 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 814 op/s; 17559/261987 objects degraded (6.702%); 4.0 MiB/s, 63 objects/s recovering 2023-06-03T19:48:27.781 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:48:27 smithi089 bash[75068]: audit 1685821707.0350127 mon.c (mon.2) 121 : audit 0 from='client.? 172.21.15.89:0/1496396553' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:48:27.781 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:48:27 smithi089 bash[75068]: cluster 1685821707.2322266 mon.a (mon.0) 275 : cluster 3 Health check update: Degraded data redundancy: 17559/261987 objects degraded (6.702%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-06-03T19:48:27.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:48:27 smithi164 bash[59462]: cluster 1685821706.1348398 mgr.y (mgr.34104) 292 : cluster 0 pgmap v235: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 5.3 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 814 op/s; 17559/261987 objects degraded (6.702%); 4.0 MiB/s, 63 objects/s recovering 2023-06-03T19:48:27.836 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:48:27 smithi164 bash[59462]: audit 1685821707.0350127 mon.c (mon.2) 121 : audit 0 from='client.? 172.21.15.89:0/1496396553' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:48:27.836 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:48:27 smithi164 bash[59462]: cluster 1685821707.2322266 mon.a (mon.0) 275 : cluster 3 Health check update: Degraded data redundancy: 17559/261987 objects degraded (6.702%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-06-03T19:48:29.779 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:48:29 smithi089 bash[75068]: cluster 1685821708.1353831 mgr.y (mgr.34104) 293 : cluster 0 pgmap v236: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 5.3 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 401 op/s; 17559/261987 objects degraded (6.702%); 1.1 MiB/s, 18 objects/s recovering 2023-06-03T19:48:29.780 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:48:29 smithi089 bash[78452]: cluster 1685821708.1353831 mgr.y (mgr.34104) 293 : cluster 0 pgmap v236: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 5.3 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 401 op/s; 17559/261987 objects degraded (6.702%); 1.1 MiB/s, 18 objects/s recovering 2023-06-03T19:48:29.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:48:29 smithi164 bash[59462]: cluster 1685821708.1353831 mgr.y (mgr.34104) 293 : cluster 0 pgmap v236: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 5.3 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 401 op/s; 17559/261987 objects degraded (6.702%); 1.1 MiB/s, 18 objects/s recovering 2023-06-03T19:48:30.779 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:48:30 smithi089 bash[75068]: audit 1685821709.6087682 mon.a (mon.0) 276 : audit 0 from='client.? 172.21.15.89:0/363896658' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:48:30.780 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:48:30 smithi089 bash[78452]: audit 1685821709.6087682 mon.a (mon.0) 276 : audit 0 from='client.? 172.21.15.89:0/363896658' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:48:30.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:48:30 smithi164 bash[59462]: audit 1685821709.6087682 mon.a (mon.0) 276 : audit 0 from='client.? 172.21.15.89:0/363896658' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:48:31.779 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:48:31 smithi089 bash[75068]: cluster 1685821710.1363893 mgr.y (mgr.34104) 294 : cluster 0 pgmap v237: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 5.1 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 689 op/s; 17483/251370 objects degraded (6.955%); 1.4 MiB/s, 22 objects/s recovering 2023-06-03T19:48:31.780 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:48:31 smithi089 bash[78452]: cluster 1685821710.1363893 mgr.y (mgr.34104) 294 : cluster 0 pgmap v237: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 5.1 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 689 op/s; 17483/251370 objects degraded (6.955%); 1.4 MiB/s, 22 objects/s recovering 2023-06-03T19:48:31.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:48:31 smithi164 bash[59462]: cluster 1685821710.1363893 mgr.y (mgr.34104) 294 : cluster 0 pgmap v237: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 5.1 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 689 op/s; 17483/251370 objects degraded (6.955%); 1.4 MiB/s, 22 objects/s recovering 2023-06-03T19:48:32.779 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:48:32 smithi089 bash[75068]: audit 1685821712.2115176 mon.c (mon.2) 122 : audit 0 from='client.? 172.21.15.89:0/637026502' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:48:32.780 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:48:32 smithi089 bash[75068]: cluster 1685821712.2334363 mon.a (mon.0) 277 : cluster 3 Health check update: Degraded data redundancy: 17483/251370 objects degraded (6.955%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-06-03T19:48:32.780 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:48:32 smithi089 bash[78452]: audit 1685821712.2115176 mon.c (mon.2) 122 : audit 0 from='client.? 172.21.15.89:0/637026502' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:48:32.780 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:48:32 smithi089 bash[78452]: cluster 1685821712.2334363 mon.a (mon.0) 277 : cluster 3 Health check update: Degraded data redundancy: 17483/251370 objects degraded (6.955%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-06-03T19:48:32.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:48:32 smithi164 bash[59462]: audit 1685821712.2115176 mon.c (mon.2) 122 : audit 0 from='client.? 172.21.15.89:0/637026502' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:48:32.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:48:32 smithi164 bash[59462]: cluster 1685821712.2334363 mon.a (mon.0) 277 : cluster 3 Health check update: Degraded data redundancy: 17483/251370 objects degraded (6.955%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-06-03T19:48:33.779 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:48:33 smithi089 bash[75068]: cluster 1685821712.1398869 mgr.y (mgr.34104) 295 : cluster 0 pgmap v238: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 5.1 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 625 op/s; 17483/251370 objects degraded (6.955%); 1.3 MiB/s, 20 objects/s recovering 2023-06-03T19:48:33.780 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:48:33 smithi089 bash[78452]: cluster 1685821712.1398869 mgr.y (mgr.34104) 295 : cluster 0 pgmap v238: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 5.1 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 625 op/s; 17483/251370 objects degraded (6.955%); 1.3 MiB/s, 20 objects/s recovering 2023-06-03T19:48:33.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:48:33 smithi164 bash[59462]: cluster 1685821712.1398869 mgr.y (mgr.34104) 295 : cluster 0 pgmap v238: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 5.1 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 625 op/s; 17483/251370 objects degraded (6.955%); 1.3 MiB/s, 20 objects/s recovering 2023-06-03T19:48:34.529 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 19:48:34 smithi089 bash[71163]: debug 2023-06-03T19:48:34.205+0000 7fe48642e700 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (1 PGs are or would become offline) 2023-06-03T19:48:34.530 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:48:34 smithi089 bash[75068]: audit 1685821714.2092786 mon.c (mon.2) 123 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-06-03T19:48:34.530 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:48:34 smithi089 bash[78452]: audit 1685821714.2092786 mon.c (mon.2) 123 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-06-03T19:48:34.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:48:34 smithi164 bash[59462]: audit 1685821714.2092786 mon.c (mon.2) 123 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-06-03T19:48:35.779 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:48:35 smithi089 bash[75068]: cluster 1685821714.1406577 mgr.y (mgr.34104) 296 : cluster 0 pgmap v239: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 4.9 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 816 op/s; 17406/242214 objects degraded (7.186%); 1.6 MiB/s, 24 objects/s recovering 2023-06-03T19:48:35.779 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:48:35 smithi089 bash[75068]: audit 1685821714.2102902 mgr.y (mgr.34104) 297 : audit 0 from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-06-03T19:48:35.779 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:48:35 smithi089 bash[75068]: cephadm 1685821714.2114174 mgr.y (mgr.34104) 298 : cephadm 1 Upgrade: unsafe to stop osd(s) at this time (1 PGs are or would become offline) 2023-06-03T19:48:35.780 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:48:35 smithi089 bash[75068]: audit 1685821714.8027565 mon.a (mon.0) 278 : audit 0 from='client.? 172.21.15.89:0/100038224' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:48:35.780 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:48:35 smithi089 bash[78452]: cluster 1685821714.1406577 mgr.y (mgr.34104) 296 : cluster 0 pgmap v239: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 4.9 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 816 op/s; 17406/242214 objects degraded (7.186%); 1.6 MiB/s, 24 objects/s recovering 2023-06-03T19:48:35.780 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:48:35 smithi089 bash[78452]: audit 1685821714.2102902 mgr.y (mgr.34104) 297 : audit 0 from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-06-03T19:48:35.780 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:48:35 smithi089 bash[78452]: cephadm 1685821714.2114174 mgr.y (mgr.34104) 298 : cephadm 1 Upgrade: unsafe to stop osd(s) at this time (1 PGs are or would become offline) 2023-06-03T19:48:35.780 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:48:35 smithi089 bash[78452]: audit 1685821714.8027565 mon.a (mon.0) 278 : audit 0 from='client.? 172.21.15.89:0/100038224' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:48:35.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:48:35 smithi164 bash[59462]: cluster 1685821714.1406577 mgr.y (mgr.34104) 296 : cluster 0 pgmap v239: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 4.9 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 816 op/s; 17406/242214 objects degraded (7.186%); 1.6 MiB/s, 24 objects/s recovering 2023-06-03T19:48:35.836 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:48:35 smithi164 bash[59462]: audit 1685821714.2102902 mgr.y (mgr.34104) 297 : audit 0 from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-06-03T19:48:35.836 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:48:35 smithi164 bash[59462]: cephadm 1685821714.2114174 mgr.y (mgr.34104) 298 : cephadm 1 Upgrade: unsafe to stop osd(s) at this time (1 PGs are or would become offline) 2023-06-03T19:48:35.836 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:48:35 smithi164 bash[59462]: audit 1685821714.8027565 mon.a (mon.0) 278 : audit 0 from='client.? 172.21.15.89:0/100038224' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:48:36.529 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 19:48:36 smithi089 bash[71163]: ::ffff:172.21.15.164 - - [03/Jun/2023:19:48:36] "GET /metrics HTTP/1.1" 200 207373 "" "Prometheus/2.33.4" 2023-06-03T19:48:37.779 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:48:37 smithi089 bash[78452]: cluster 1685821716.1412737 mgr.y (mgr.34104) 299 : cluster 0 pgmap v240: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 4.9 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 623 op/s; 17406/240903 objects degraded (7.225%); 1.1 MiB/s, 17 objects/s recovering 2023-06-03T19:48:37.780 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:48:37 smithi089 bash[78452]: audit 1685821717.0711064 mon.c (mon.2) 124 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T19:48:37.780 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:48:37 smithi089 bash[78452]: cluster 1685821717.2353115 mon.a (mon.0) 279 : cluster 3 Health check update: Degraded data redundancy: 17406/240903 objects degraded (7.225%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-06-03T19:48:37.780 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:48:37 smithi089 bash[78452]: audit 1685821717.4227169 mon.a (mon.0) 280 : audit 0 from='client.? 172.21.15.89:0/3703085111' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:48:37.780 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:48:37 smithi089 bash[75068]: cluster 1685821716.1412737 mgr.y (mgr.34104) 299 : cluster 0 pgmap v240: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 4.9 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 623 op/s; 17406/240903 objects degraded (7.225%); 1.1 MiB/s, 17 objects/s recovering 2023-06-03T19:48:37.780 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:48:37 smithi089 bash[75068]: audit 1685821717.0711064 mon.c (mon.2) 124 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T19:48:37.781 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:48:37 smithi089 bash[75068]: cluster 1685821717.2353115 mon.a (mon.0) 279 : cluster 3 Health check update: Degraded data redundancy: 17406/240903 objects degraded (7.225%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-06-03T19:48:37.781 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:48:37 smithi089 bash[75068]: audit 1685821717.4227169 mon.a (mon.0) 280 : audit 0 from='client.? 172.21.15.89:0/3703085111' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:48:37.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:48:37 smithi164 bash[59462]: cluster 1685821716.1412737 mgr.y (mgr.34104) 299 : cluster 0 pgmap v240: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 4.9 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 623 op/s; 17406/240903 objects degraded (7.225%); 1.1 MiB/s, 17 objects/s recovering 2023-06-03T19:48:37.836 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:48:37 smithi164 bash[59462]: audit 1685821717.0711064 mon.c (mon.2) 124 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T19:48:37.836 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:48:37 smithi164 bash[59462]: cluster 1685821717.2353115 mon.a (mon.0) 279 : cluster 3 Health check update: Degraded data redundancy: 17406/240903 objects degraded (7.225%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-06-03T19:48:37.836 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:48:37 smithi164 bash[59462]: audit 1685821717.4227169 mon.a (mon.0) 280 : audit 0 from='client.? 172.21.15.89:0/3703085111' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:48:39.779 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:48:39 smithi089 bash[75068]: cluster 1685821718.1418893 mgr.y (mgr.34104) 300 : cluster 0 pgmap v241: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 4.9 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 585 op/s; 17406/240903 objects degraded (7.225%); 816 KiB/s, 12 objects/s recovering 2023-06-03T19:48:39.780 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:48:39 smithi089 bash[78452]: cluster 1685821718.1418893 mgr.y (mgr.34104) 300 : cluster 0 pgmap v241: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 4.9 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 585 op/s; 17406/240903 objects degraded (7.225%); 816 KiB/s, 12 objects/s recovering 2023-06-03T19:48:39.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:48:39 smithi164 bash[59462]: cluster 1685821718.1418893 mgr.y (mgr.34104) 300 : cluster 0 pgmap v241: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 4.9 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 585 op/s; 17406/240903 objects degraded (7.225%); 816 KiB/s, 12 objects/s recovering 2023-06-03T19:48:40.779 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:48:40 smithi089 bash[75068]: audit 1685821720.0082374 mon.c (mon.2) 125 : audit 0 from='client.? 172.21.15.89:0/3745594258' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:48:40.780 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:48:40 smithi089 bash[78452]: audit 1685821720.0082374 mon.c (mon.2) 125 : audit 0 from='client.? 172.21.15.89:0/3745594258' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:48:40.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:48:40 smithi164 bash[59462]: audit 1685821720.0082374 mon.c (mon.2) 125 : audit 0 from='client.? 172.21.15.89:0/3745594258' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:48:41.779 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:48:41 smithi089 bash[75068]: cluster 1685821720.1427743 mgr.y (mgr.34104) 301 : cluster 0 pgmap v242: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 4.7 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 866 op/s; 17332/230787 objects degraded (7.510%); 1.2 MiB/s, 18 objects/s recovering 2023-06-03T19:48:41.780 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:48:41 smithi089 bash[78452]: cluster 1685821720.1427743 mgr.y (mgr.34104) 301 : cluster 0 pgmap v242: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 4.7 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 866 op/s; 17332/230787 objects degraded (7.510%); 1.2 MiB/s, 18 objects/s recovering 2023-06-03T19:48:41.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:48:41 smithi164 bash[59462]: cluster 1685821720.1427743 mgr.y (mgr.34104) 301 : cluster 0 pgmap v242: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 4.7 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 866 op/s; 17332/230787 objects degraded (7.510%); 1.2 MiB/s, 18 objects/s recovering 2023-06-03T19:48:42.779 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:48:42 smithi089 bash[75068]: cluster 1685821722.2366471 mon.a (mon.0) 281 : cluster 3 Health check update: Degraded data redundancy: 17332/230787 objects degraded (7.510%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-06-03T19:48:42.780 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:48:42 smithi089 bash[78452]: cluster 1685821722.2366471 mon.a (mon.0) 281 : cluster 3 Health check update: Degraded data redundancy: 17332/230787 objects degraded (7.510%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-06-03T19:48:42.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:48:42 smithi164 bash[59462]: cluster 1685821722.2366471 mon.a (mon.0) 281 : cluster 3 Health check update: Degraded data redundancy: 17332/230787 objects degraded (7.510%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-06-03T19:48:43.779 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:48:43 smithi089 bash[78452]: cluster 1685821722.1433318 mgr.y (mgr.34104) 302 : cluster 0 pgmap v243: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 4.7 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 571 op/s; 17332/230787 objects degraded (7.510%); 805 KiB/s, 12 objects/s recovering 2023-06-03T19:48:43.779 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:48:43 smithi089 bash[78452]: audit 1685821722.5753543 mon.c (mon.2) 126 : audit 0 from='client.? 172.21.15.89:0/1818295398' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:48:43.780 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:48:43 smithi089 bash[75068]: cluster 1685821722.1433318 mgr.y (mgr.34104) 302 : cluster 0 pgmap v243: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 4.7 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 571 op/s; 17332/230787 objects degraded (7.510%); 805 KiB/s, 12 objects/s recovering 2023-06-03T19:48:43.780 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:48:43 smithi089 bash[75068]: audit 1685821722.5753543 mon.c (mon.2) 126 : audit 0 from='client.? 172.21.15.89:0/1818295398' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:48:43.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:48:43 smithi164 bash[59462]: cluster 1685821722.1433318 mgr.y (mgr.34104) 302 : cluster 0 pgmap v243: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 4.7 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 571 op/s; 17332/230787 objects degraded (7.510%); 805 KiB/s, 12 objects/s recovering 2023-06-03T19:48:43.836 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:48:43 smithi164 bash[59462]: audit 1685821722.5753543 mon.c (mon.2) 126 : audit 0 from='client.? 172.21.15.89:0/1818295398' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:48:45.779 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:48:45 smithi089 bash[75068]: cluster 1685821724.1441073 mgr.y (mgr.34104) 303 : cluster 0 pgmap v244: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 4.5 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 822 op/s; 17257/221754 objects degraded (7.782%); 1.2 MiB/s, 18 objects/s recovering 2023-06-03T19:48:45.780 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:48:45 smithi089 bash[75068]: audit 1685821725.1279085 mon.c (mon.2) 127 : audit 0 from='client.? 172.21.15.89:0/3917448942' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:48:45.780 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:48:45 smithi089 bash[78452]: cluster 1685821724.1441073 mgr.y (mgr.34104) 303 : cluster 0 pgmap v244: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 4.5 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 822 op/s; 17257/221754 objects degraded (7.782%); 1.2 MiB/s, 18 objects/s recovering 2023-06-03T19:48:45.780 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:48:45 smithi089 bash[78452]: audit 1685821725.1279085 mon.c (mon.2) 127 : audit 0 from='client.? 172.21.15.89:0/3917448942' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:48:45.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:48:45 smithi164 bash[59462]: cluster 1685821724.1441073 mgr.y (mgr.34104) 303 : cluster 0 pgmap v244: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 4.5 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 822 op/s; 17257/221754 objects degraded (7.782%); 1.2 MiB/s, 18 objects/s recovering 2023-06-03T19:48:45.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:48:45 smithi164 bash[59462]: audit 1685821725.1279085 mon.c (mon.2) 127 : audit 0 from='client.? 172.21.15.89:0/3917448942' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:48:46.529 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 19:48:46 smithi089 bash[71163]: ::ffff:172.21.15.164 - - [03/Jun/2023:19:48:46] "GET /metrics HTTP/1.1" 200 207392 "" "Prometheus/2.33.4" 2023-06-03T19:48:47.779 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:48:47 smithi089 bash[75068]: cluster 1685821726.1446357 mgr.y (mgr.34104) 304 : cluster 0 pgmap v245: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 4.5 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 603 op/s; 17257/220491 objects degraded (7.827%); 794 KiB/s, 12 objects/s recovering 2023-06-03T19:48:47.780 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:48:47 smithi089 bash[75068]: cluster 1685821727.2378721 mon.a (mon.0) 282 : cluster 3 Health check update: Degraded data redundancy: 17257/220491 objects degraded (7.827%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-06-03T19:48:47.780 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:48:47 smithi089 bash[78452]: cluster 1685821726.1446357 mgr.y (mgr.34104) 304 : cluster 0 pgmap v245: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 4.5 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 603 op/s; 17257/220491 objects degraded (7.827%); 794 KiB/s, 12 objects/s recovering 2023-06-03T19:48:47.780 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:48:47 smithi089 bash[78452]: cluster 1685821727.2378721 mon.a (mon.0) 282 : cluster 3 Health check update: Degraded data redundancy: 17257/220491 objects degraded (7.827%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-06-03T19:48:47.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:48:47 smithi164 bash[59462]: cluster 1685821726.1446357 mgr.y (mgr.34104) 304 : cluster 0 pgmap v245: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 4.5 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 603 op/s; 17257/220491 objects degraded (7.827%); 794 KiB/s, 12 objects/s recovering 2023-06-03T19:48:47.836 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:48:47 smithi164 bash[59462]: cluster 1685821727.2378721 mon.a (mon.0) 282 : cluster 3 Health check update: Degraded data redundancy: 17257/220491 objects degraded (7.827%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-06-03T19:48:48.779 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:48:48 smithi089 bash[75068]: audit 1685821727.7589731 mon.a (mon.0) 283 : audit 0 from='client.? 172.21.15.89:0/76691731' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:48:48.780 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:48:48 smithi089 bash[78452]: audit 1685821727.7589731 mon.a (mon.0) 283 : audit 0 from='client.? 172.21.15.89:0/76691731' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:48:48.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:48:48 smithi164 bash[59462]: audit 1685821727.7589731 mon.a (mon.0) 283 : audit 0 from='client.? 172.21.15.89:0/76691731' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:48:49.499 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 19:48:49 smithi089 bash[71163]: debug 2023-06-03T19:48:49.221+0000 7fe48642e700 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (1 PGs are or would become offline) 2023-06-03T19:48:49.779 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:48:49 smithi089 bash[78452]: cluster 1685821728.145175 mgr.y (mgr.34104) 305 : cluster 0 pgmap v246: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 4.5 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 566 op/s; 17257/220491 objects degraded (7.827%); 794 KiB/s, 12 objects/s recovering 2023-06-03T19:48:49.779 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:48:49 smithi089 bash[78452]: audit 1685821729.2253947 mon.c (mon.2) 128 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-06-03T19:48:49.780 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:48:49 smithi089 bash[75068]: cluster 1685821728.145175 mgr.y (mgr.34104) 305 : cluster 0 pgmap v246: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 4.5 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 566 op/s; 17257/220491 objects degraded (7.827%); 794 KiB/s, 12 objects/s recovering 2023-06-03T19:48:49.780 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:48:49 smithi089 bash[75068]: audit 1685821729.2253947 mon.c (mon.2) 128 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-06-03T19:48:49.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:48:49 smithi164 bash[59462]: cluster 1685821728.145175 mgr.y (mgr.34104) 305 : cluster 0 pgmap v246: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 4.5 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 566 op/s; 17257/220491 objects degraded (7.827%); 794 KiB/s, 12 objects/s recovering 2023-06-03T19:48:49.836 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:48:49 smithi164 bash[59462]: audit 1685821729.2253947 mon.c (mon.2) 128 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-06-03T19:48:50.779 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:48:50 smithi089 bash[78452]: audit 1685821729.2256641 mgr.y (mgr.34104) 306 : audit 0 from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-06-03T19:48:50.779 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:48:50 smithi089 bash[78452]: cephadm 1685821729.227408 mgr.y (mgr.34104) 307 : cephadm 1 Upgrade: unsafe to stop osd(s) at this time (1 PGs are or would become offline) 2023-06-03T19:48:50.780 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:48:50 smithi089 bash[78452]: audit 1685821730.361635 mon.c (mon.2) 129 : audit 0 from='client.? 172.21.15.89:0/288431290' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:48:50.780 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:48:50 smithi089 bash[75068]: audit 1685821729.2256641 mgr.y (mgr.34104) 306 : audit 0 from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-06-03T19:48:50.780 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:48:50 smithi089 bash[75068]: cephadm 1685821729.227408 mgr.y (mgr.34104) 307 : cephadm 1 Upgrade: unsafe to stop osd(s) at this time (1 PGs are or would become offline) 2023-06-03T19:48:50.780 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:48:50 smithi089 bash[75068]: audit 1685821730.361635 mon.c (mon.2) 129 : audit 0 from='client.? 172.21.15.89:0/288431290' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:48:50.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:48:50 smithi164 bash[59462]: audit 1685821729.2256641 mgr.y (mgr.34104) 306 : audit 0 from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-06-03T19:48:50.836 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:48:50 smithi164 bash[59462]: cephadm 1685821729.227408 mgr.y (mgr.34104) 307 : cephadm 1 Upgrade: unsafe to stop osd(s) at this time (1 PGs are or would become offline) 2023-06-03T19:48:50.836 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:48:50 smithi164 bash[59462]: audit 1685821730.361635 mon.c (mon.2) 129 : audit 0 from='client.? 172.21.15.89:0/288431290' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:48:51.779 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:48:51 smithi089 bash[75068]: cluster 1685821730.1461031 mgr.y (mgr.34104) 308 : cluster 0 pgmap v247: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 4.3 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 803 op/s; 17182/211971 objects degraded (8.106%); 1.2 MiB/s, 18 objects/s recovering 2023-06-03T19:48:51.780 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:48:51 smithi089 bash[78452]: cluster 1685821730.1461031 mgr.y (mgr.34104) 308 : cluster 0 pgmap v247: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 4.3 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 803 op/s; 17182/211971 objects degraded (8.106%); 1.2 MiB/s, 18 objects/s recovering 2023-06-03T19:48:51.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:48:51 smithi164 bash[59462]: cluster 1685821730.1461031 mgr.y (mgr.34104) 308 : cluster 0 pgmap v247: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 4.3 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 803 op/s; 17182/211971 objects degraded (8.106%); 1.2 MiB/s, 18 objects/s recovering 2023-06-03T19:48:52.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:48:52 smithi164 bash[59462]: audit 1685821732.0713794 mon.c (mon.2) 130 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T19:48:52.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:48:52 smithi164 bash[59462]: cluster 1685821732.2393942 mon.a (mon.0) 284 : cluster 3 Health check update: Degraded data redundancy: 17182/211971 objects degraded (8.106%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-06-03T19:48:53.029 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:48:52 smithi089 bash[75068]: audit 1685821732.0713794 mon.c (mon.2) 130 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T19:48:53.030 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:48:52 smithi089 bash[75068]: cluster 1685821732.2393942 mon.a (mon.0) 284 : cluster 3 Health check update: Degraded data redundancy: 17182/211971 objects degraded (8.106%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-06-03T19:48:53.030 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:48:52 smithi089 bash[78452]: audit 1685821732.0713794 mon.c (mon.2) 130 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T19:48:53.030 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:48:52 smithi089 bash[78452]: cluster 1685821732.2393942 mon.a (mon.0) 284 : cluster 3 Health check update: Degraded data redundancy: 17182/211971 objects degraded (8.106%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-06-03T19:48:53.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:48:53 smithi164 bash[59462]: cluster 1685821732.146776 mgr.y (mgr.34104) 309 : cluster 0 pgmap v248: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 4.3 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 561 op/s; 17182/210555 objects degraded (8.160%); 800 KiB/s, 12 objects/s recovering 2023-06-03T19:48:53.836 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:48:53 smithi164 bash[59462]: audit 1685821732.9379432 mon.c (mon.2) 131 : audit 0 from='client.? 172.21.15.89:0/3001021273' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:48:54.029 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:48:53 smithi089 bash[75068]: cluster 1685821732.146776 mgr.y (mgr.34104) 309 : cluster 0 pgmap v248: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 4.3 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 561 op/s; 17182/210555 objects degraded (8.160%); 800 KiB/s, 12 objects/s recovering 2023-06-03T19:48:54.030 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:48:53 smithi089 bash[75068]: audit 1685821732.9379432 mon.c (mon.2) 131 : audit 0 from='client.? 172.21.15.89:0/3001021273' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:48:54.030 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:48:53 smithi089 bash[78452]: cluster 1685821732.146776 mgr.y (mgr.34104) 309 : cluster 0 pgmap v248: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 4.3 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 561 op/s; 17182/210555 objects degraded (8.160%); 800 KiB/s, 12 objects/s recovering 2023-06-03T19:48:54.030 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:48:53 smithi089 bash[78452]: audit 1685821732.9379432 mon.c (mon.2) 131 : audit 0 from='client.? 172.21.15.89:0/3001021273' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:48:55.836 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:48:55 smithi164 bash[59462]: cluster 1685821734.1475956 mgr.y (mgr.34104) 310 : cluster 0 pgmap v249: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 4.1 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 795 op/s; 17109/202122 objects degraded (8.465%); 1.2 MiB/s, 18 objects/s recovering 2023-06-03T19:48:55.836 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:48:55 smithi164 bash[59462]: audit 1685821735.50405 mon.a (mon.0) 285 : audit 0 from='client.? 172.21.15.89:0/3233284335' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:48:56.029 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:48:55 smithi089 bash[75068]: cluster 1685821734.1475956 mgr.y (mgr.34104) 310 : cluster 0 pgmap v249: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 4.1 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 795 op/s; 17109/202122 objects degraded (8.465%); 1.2 MiB/s, 18 objects/s recovering 2023-06-03T19:48:56.030 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:48:55 smithi089 bash[75068]: audit 1685821735.50405 mon.a (mon.0) 285 : audit 0 from='client.? 172.21.15.89:0/3233284335' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:48:56.030 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:48:55 smithi089 bash[78452]: cluster 1685821734.1475956 mgr.y (mgr.34104) 310 : cluster 0 pgmap v249: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 4.1 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 795 op/s; 17109/202122 objects degraded (8.465%); 1.2 MiB/s, 18 objects/s recovering 2023-06-03T19:48:56.030 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:48:55 smithi089 bash[78452]: audit 1685821735.50405 mon.a (mon.0) 285 : audit 0 from='client.? 172.21.15.89:0/3233284335' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:48:56.529 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 19:48:56 smithi089 bash[71163]: ::ffff:172.21.15.164 - - [03/Jun/2023:19:48:56] "GET /metrics HTTP/1.1" 200 207399 "" "Prometheus/2.33.4" 2023-06-03T19:48:57.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:48:57 smithi164 bash[59462]: cluster 1685821736.1482303 mgr.y (mgr.34104) 311 : cluster 0 pgmap v250: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 4.1 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 580 op/s; 17109/200853 objects degraded (8.518%); 789 KiB/s, 12 objects/s recovering 2023-06-03T19:48:57.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:48:57 smithi164 bash[59462]: cluster 1685821737.2407308 mon.a (mon.0) 286 : cluster 3 Health check update: Degraded data redundancy: 17109/200853 objects degraded (8.518%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-06-03T19:48:58.029 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:48:57 smithi089 bash[78452]: cluster 1685821736.1482303 mgr.y (mgr.34104) 311 : cluster 0 pgmap v250: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 4.1 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 580 op/s; 17109/200853 objects degraded (8.518%); 789 KiB/s, 12 objects/s recovering 2023-06-03T19:48:58.029 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:48:57 smithi089 bash[78452]: cluster 1685821737.2407308 mon.a (mon.0) 286 : cluster 3 Health check update: Degraded data redundancy: 17109/200853 objects degraded (8.518%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-06-03T19:48:58.030 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:48:57 smithi089 bash[75068]: cluster 1685821736.1482303 mgr.y (mgr.34104) 311 : cluster 0 pgmap v250: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 4.1 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 580 op/s; 17109/200853 objects degraded (8.518%); 789 KiB/s, 12 objects/s recovering 2023-06-03T19:48:58.030 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:48:57 smithi089 bash[75068]: cluster 1685821737.2407308 mon.a (mon.0) 286 : cluster 3 Health check update: Degraded data redundancy: 17109/200853 objects degraded (8.518%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-06-03T19:48:58.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:48:58 smithi164 bash[59462]: audit 1685821738.1023254 mon.c (mon.2) 132 : audit 0 from='client.? 172.21.15.89:0/989950253' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:48:59.029 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:48:58 smithi089 bash[75068]: audit 1685821738.1023254 mon.c (mon.2) 132 : audit 0 from='client.? 172.21.15.89:0/989950253' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:48:59.030 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:48:58 smithi089 bash[78452]: audit 1685821738.1023254 mon.c (mon.2) 132 : audit 0 from='client.? 172.21.15.89:0/989950253' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:48:59.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:48:59 smithi164 bash[59462]: cluster 1685821738.1488159 mgr.y (mgr.34104) 312 : cluster 0 pgmap v251: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 4.1 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 545 op/s; 17109/200853 objects degraded (8.518%); 789 KiB/s, 12 objects/s recovering 2023-06-03T19:49:00.029 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:48:59 smithi089 bash[75068]: cluster 1685821738.1488159 mgr.y (mgr.34104) 312 : cluster 0 pgmap v251: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 4.1 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 545 op/s; 17109/200853 objects degraded (8.518%); 789 KiB/s, 12 objects/s recovering 2023-06-03T19:49:00.030 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:48:59 smithi089 bash[78452]: cluster 1685821738.1488159 mgr.y (mgr.34104) 312 : cluster 0 pgmap v251: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 4.1 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 545 op/s; 17109/200853 objects degraded (8.518%); 789 KiB/s, 12 objects/s recovering 2023-06-03T19:49:01.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:49:01 smithi164 bash[59462]: cluster 1685821740.1496525 mgr.y (mgr.34104) 313 : cluster 0 pgmap v252: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 3.9 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 776 op/s; 17033/192519 objects degraded (8.847%); 1.2 MiB/s, 18 objects/s recovering 2023-06-03T19:49:01.836 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:49:01 smithi164 bash[59462]: audit 1685821740.693452 mon.c (mon.2) 133 : audit 0 from='client.? 172.21.15.89:0/3850358326' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:49:02.029 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:49:01 smithi089 bash[75068]: cluster 1685821740.1496525 mgr.y (mgr.34104) 313 : cluster 0 pgmap v252: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 3.9 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 776 op/s; 17033/192519 objects degraded (8.847%); 1.2 MiB/s, 18 objects/s recovering 2023-06-03T19:49:02.030 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:49:01 smithi089 bash[75068]: audit 1685821740.693452 mon.c (mon.2) 133 : audit 0 from='client.? 172.21.15.89:0/3850358326' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:49:02.030 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:49:01 smithi089 bash[78452]: cluster 1685821740.1496525 mgr.y (mgr.34104) 313 : cluster 0 pgmap v252: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 3.9 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 776 op/s; 17033/192519 objects degraded (8.847%); 1.2 MiB/s, 18 objects/s recovering 2023-06-03T19:49:02.030 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:49:01 smithi089 bash[78452]: audit 1685821740.693452 mon.c (mon.2) 133 : audit 0 from='client.? 172.21.15.89:0/3850358326' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:49:02.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:49:02 smithi164 bash[59462]: cluster 1685821742.258556 mon.a (mon.0) 287 : cluster 3 Health check update: Degraded data redundancy: 17033/192519 objects degraded (8.847%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-06-03T19:49:03.029 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:49:02 smithi089 bash[78452]: cluster 1685821742.258556 mon.a (mon.0) 287 : cluster 3 Health check update: Degraded data redundancy: 17033/192519 objects degraded (8.847%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-06-03T19:49:03.029 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:49:02 smithi089 bash[75068]: cluster 1685821742.258556 mon.a (mon.0) 287 : cluster 3 Health check update: Degraded data redundancy: 17033/192519 objects degraded (8.847%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-06-03T19:49:03.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:49:03 smithi164 bash[59462]: cluster 1685821742.1502614 mgr.y (mgr.34104) 314 : cluster 0 pgmap v253: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 3.9 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 573 op/s; 17033/191319 objects degraded (8.903%); 794 KiB/s, 12 objects/s recovering 2023-06-03T19:49:03.836 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:49:03 smithi164 bash[59462]: audit 1685821743.3388321 mon.b (mon.1) 11 : audit 0 from='client.? 172.21.15.89:0/3756714238' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:49:04.029 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:49:03 smithi089 bash[78452]: cluster 1685821742.1502614 mgr.y (mgr.34104) 314 : cluster 0 pgmap v253: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 3.9 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 573 op/s; 17033/191319 objects degraded (8.903%); 794 KiB/s, 12 objects/s recovering 2023-06-03T19:49:04.029 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:49:03 smithi089 bash[78452]: audit 1685821743.3388321 mon.b (mon.1) 11 : audit 0 from='client.? 172.21.15.89:0/3756714238' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:49:04.030 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:49:03 smithi089 bash[75068]: cluster 1685821742.1502614 mgr.y (mgr.34104) 314 : cluster 0 pgmap v253: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 3.9 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 573 op/s; 17033/191319 objects degraded (8.903%); 794 KiB/s, 12 objects/s recovering 2023-06-03T19:49:04.030 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:49:03 smithi089 bash[75068]: audit 1685821743.3388321 mon.b (mon.1) 11 : audit 0 from='client.? 172.21.15.89:0/3756714238' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:49:04.529 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 19:49:04 smithi089 bash[71163]: debug 2023-06-03T19:49:04.237+0000 7fe48642e700 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (1 PGs are or would become offline) 2023-06-03T19:49:05.029 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:49:04 smithi089 bash[78452]: audit 1685821744.2414749 mon.c (mon.2) 134 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-06-03T19:49:05.029 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:49:04 smithi089 bash[75068]: audit 1685821744.2414749 mon.c (mon.2) 134 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-06-03T19:49:05.085 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:49:04 smithi164 bash[59462]: audit 1685821744.2414749 mon.c (mon.2) 134 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-06-03T19:49:06.029 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:49:05 smithi089 bash[75068]: cluster 1685821744.1511452 mgr.y (mgr.34104) 315 : cluster 0 pgmap v254: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 3.7 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 766 op/s; 16959/182958 objects degraded (9.269%); 1.2 MiB/s, 18 objects/s recovering 2023-06-03T19:49:06.029 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:49:05 smithi089 bash[75068]: audit 1685821744.2421324 mgr.y (mgr.34104) 316 : audit 0 from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-06-03T19:49:06.029 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:49:05 smithi089 bash[75068]: cephadm 1685821744.2435338 mgr.y (mgr.34104) 317 : cephadm 1 Upgrade: unsafe to stop osd(s) at this time (1 PGs are or would become offline) 2023-06-03T19:49:06.030 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:49:05 smithi089 bash[78452]: cluster 1685821744.1511452 mgr.y (mgr.34104) 315 : cluster 0 pgmap v254: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 3.7 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 766 op/s; 16959/182958 objects degraded (9.269%); 1.2 MiB/s, 18 objects/s recovering 2023-06-03T19:49:06.030 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:49:05 smithi089 bash[78452]: audit 1685821744.2421324 mgr.y (mgr.34104) 316 : audit 0 from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-06-03T19:49:06.030 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:49:05 smithi089 bash[78452]: cephadm 1685821744.2435338 mgr.y (mgr.34104) 317 : cephadm 1 Upgrade: unsafe to stop osd(s) at this time (1 PGs are or would become offline) 2023-06-03T19:49:06.085 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:49:05 smithi164 bash[59462]: cluster 1685821744.1511452 mgr.y (mgr.34104) 315 : cluster 0 pgmap v254: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 3.7 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 766 op/s; 16959/182958 objects degraded (9.269%); 1.2 MiB/s, 18 objects/s recovering 2023-06-03T19:49:06.085 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:49:05 smithi164 bash[59462]: audit 1685821744.2421324 mgr.y (mgr.34104) 316 : audit 0 from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-06-03T19:49:06.086 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:49:05 smithi164 bash[59462]: cephadm 1685821744.2435338 mgr.y (mgr.34104) 317 : cephadm 1 Upgrade: unsafe to stop osd(s) at this time (1 PGs are or would become offline) 2023-06-03T19:49:06.529 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 19:49:06 smithi089 bash[71163]: ::ffff:172.21.15.164 - - [03/Jun/2023:19:49:06] "GET /metrics HTTP/1.1" 200 207399 "" "Prometheus/2.33.4" 2023-06-03T19:49:07.029 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:49:06 smithi089 bash[75068]: audit 1685821745.9148648 mon.c (mon.2) 135 : audit 0 from='client.? 172.21.15.89:0/3723847167' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:49:07.029 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:49:06 smithi089 bash[78452]: audit 1685821745.9148648 mon.c (mon.2) 135 : audit 0 from='client.? 172.21.15.89:0/3723847167' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:49:07.085 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:49:06 smithi164 bash[59462]: audit 1685821745.9148648 mon.c (mon.2) 135 : audit 0 from='client.? 172.21.15.89:0/3723847167' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:49:08.029 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:49:07 smithi089 bash[75068]: cluster 1685821746.1517394 mgr.y (mgr.34104) 318 : cluster 0 pgmap v255: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 3.7 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 565 op/s; 16959/181749 objects degraded (9.331%); 800 KiB/s, 12 objects/s recovering 2023-06-03T19:49:08.030 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:49:07 smithi089 bash[75068]: audit 1685821747.071347 mon.c (mon.2) 136 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T19:49:08.030 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:49:07 smithi089 bash[75068]: cluster 1685821747.2595766 mon.a (mon.0) 288 : cluster 3 Health check update: Degraded data redundancy: 16959/181749 objects degraded (9.331%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-06-03T19:49:08.030 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:49:07 smithi089 bash[78452]: cluster 1685821746.1517394 mgr.y (mgr.34104) 318 : cluster 0 pgmap v255: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 3.7 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 565 op/s; 16959/181749 objects degraded (9.331%); 800 KiB/s, 12 objects/s recovering 2023-06-03T19:49:08.030 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:49:07 smithi089 bash[78452]: audit 1685821747.071347 mon.c (mon.2) 136 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T19:49:08.030 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:49:07 smithi089 bash[78452]: cluster 1685821747.2595766 mon.a (mon.0) 288 : cluster 3 Health check update: Degraded data redundancy: 16959/181749 objects degraded (9.331%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-06-03T19:49:08.085 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:49:07 smithi164 bash[59462]: cluster 1685821746.1517394 mgr.y (mgr.34104) 318 : cluster 0 pgmap v255: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 3.7 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 565 op/s; 16959/181749 objects degraded (9.331%); 800 KiB/s, 12 objects/s recovering 2023-06-03T19:49:08.086 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:49:07 smithi164 bash[59462]: audit 1685821747.071347 mon.c (mon.2) 136 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T19:49:08.086 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:49:07 smithi164 bash[59462]: cluster 1685821747.2595766 mon.a (mon.0) 288 : cluster 3 Health check update: Degraded data redundancy: 16959/181749 objects degraded (9.331%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-06-03T19:49:09.029 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:49:08 smithi089 bash[75068]: audit 1685821748.5468318 mon.a (mon.0) 289 : audit 0 from='client.? 172.21.15.89:0/1550098241' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:49:09.029 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:49:08 smithi089 bash[78452]: audit 1685821748.5468318 mon.a (mon.0) 289 : audit 0 from='client.? 172.21.15.89:0/1550098241' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:49:09.085 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:49:08 smithi164 bash[59462]: audit 1685821748.5468318 mon.a (mon.0) 289 : audit 0 from='client.? 172.21.15.89:0/1550098241' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:49:10.029 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:49:09 smithi089 bash[75068]: cluster 1685821748.152287 mgr.y (mgr.34104) 319 : cluster 0 pgmap v256: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 3.7 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 530 op/s; 16959/181749 objects degraded (9.331%); 800 KiB/s, 12 objects/s recovering 2023-06-03T19:49:10.030 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:49:09 smithi089 bash[78452]: cluster 1685821748.152287 mgr.y (mgr.34104) 319 : cluster 0 pgmap v256: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 3.7 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 530 op/s; 16959/181749 objects degraded (9.331%); 800 KiB/s, 12 objects/s recovering 2023-06-03T19:49:10.085 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:49:09 smithi164 bash[59462]: cluster 1685821748.152287 mgr.y (mgr.34104) 319 : cluster 0 pgmap v256: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 3.7 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 530 op/s; 16959/181749 objects degraded (9.331%); 800 KiB/s, 12 objects/s recovering 2023-06-03T19:49:12.029 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:49:11 smithi089 bash[75068]: cluster 1685821750.1532319 mgr.y (mgr.34104) 320 : cluster 0 pgmap v257: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 3.5 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 770 op/s; 16887/173103 objects degraded (9.755%); 1.2 MiB/s, 18 objects/s recovering 2023-06-03T19:49:12.030 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:49:11 smithi089 bash[75068]: audit 1685821751.1023204 mon.c (mon.2) 137 : audit 0 from='client.? 172.21.15.89:0/4103030103' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:49:12.030 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:49:11 smithi089 bash[78452]: cluster 1685821750.1532319 mgr.y (mgr.34104) 320 : cluster 0 pgmap v257: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 3.5 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 770 op/s; 16887/173103 objects degraded (9.755%); 1.2 MiB/s, 18 objects/s recovering 2023-06-03T19:49:12.030 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:49:11 smithi089 bash[78452]: audit 1685821751.1023204 mon.c (mon.2) 137 : audit 0 from='client.? 172.21.15.89:0/4103030103' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:49:12.085 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:49:11 smithi164 bash[59462]: cluster 1685821750.1532319 mgr.y (mgr.34104) 320 : cluster 0 pgmap v257: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 3.5 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 770 op/s; 16887/173103 objects degraded (9.755%); 1.2 MiB/s, 18 objects/s recovering 2023-06-03T19:49:12.085 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:49:11 smithi164 bash[59462]: audit 1685821751.1023204 mon.c (mon.2) 137 : audit 0 from='client.? 172.21.15.89:0/4103030103' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:49:13.029 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:49:12 smithi089 bash[78452]: cluster 1685821752.2613604 mon.a (mon.0) 290 : cluster 3 Health check update: Degraded data redundancy: 16887/173103 objects degraded (9.755%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-06-03T19:49:13.030 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:49:12 smithi089 bash[75068]: cluster 1685821752.2613604 mon.a (mon.0) 290 : cluster 3 Health check update: Degraded data redundancy: 16887/173103 objects degraded (9.755%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-06-03T19:49:13.085 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:49:12 smithi164 bash[59462]: cluster 1685821752.2613604 mon.a (mon.0) 290 : cluster 3 Health check update: Degraded data redundancy: 16887/173103 objects degraded (9.755%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-06-03T19:49:14.029 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:49:13 smithi089 bash[75068]: cluster 1685821752.1540844 mgr.y (mgr.34104) 321 : cluster 0 pgmap v258: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 3.5 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 572 op/s; 16887/171906 objects degraded (9.823%); 778 KiB/s, 12 objects/s recovering 2023-06-03T19:49:14.030 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:49:13 smithi089 bash[78452]: cluster 1685821752.1540844 mgr.y (mgr.34104) 321 : cluster 0 pgmap v258: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 3.5 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 572 op/s; 16887/171906 objects degraded (9.823%); 778 KiB/s, 12 objects/s recovering 2023-06-03T19:49:14.085 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:49:13 smithi164 bash[59462]: cluster 1685821752.1540844 mgr.y (mgr.34104) 321 : cluster 0 pgmap v258: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 3.5 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 572 op/s; 16887/171906 objects degraded (9.823%); 778 KiB/s, 12 objects/s recovering 2023-06-03T19:49:15.029 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:49:14 smithi089 bash[75068]: audit 1685821753.6830864 mon.c (mon.2) 138 : audit 0 from='client.? 172.21.15.89:0/393804920' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:49:15.030 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:49:14 smithi089 bash[78452]: audit 1685821753.6830864 mon.c (mon.2) 138 : audit 0 from='client.? 172.21.15.89:0/393804920' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:49:15.085 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:49:14 smithi164 bash[59462]: audit 1685821753.6830864 mon.c (mon.2) 138 : audit 0 from='client.? 172.21.15.89:0/393804920' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:49:16.029 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:49:15 smithi089 bash[75068]: cluster 1685821754.1548343 mgr.y (mgr.34104) 322 : cluster 0 pgmap v259: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 3.3 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 772 op/s; 16812/163515 objects degraded (10.282%); 1.2 MiB/s, 18 objects/s recovering 2023-06-03T19:49:16.029 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:49:15 smithi089 bash[78452]: cluster 1685821754.1548343 mgr.y (mgr.34104) 322 : cluster 0 pgmap v259: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 3.3 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 772 op/s; 16812/163515 objects degraded (10.282%); 1.2 MiB/s, 18 objects/s recovering 2023-06-03T19:49:16.085 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:49:15 smithi164 bash[59462]: cluster 1685821754.1548343 mgr.y (mgr.34104) 322 : cluster 0 pgmap v259: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 3.3 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 772 op/s; 16812/163515 objects degraded (10.282%); 1.2 MiB/s, 18 objects/s recovering 2023-06-03T19:49:16.529 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 19:49:16 smithi089 bash[71163]: ::ffff:172.21.15.164 - - [03/Jun/2023:19:49:16] "GET /metrics HTTP/1.1" 200 207398 "" "Prometheus/2.33.4" 2023-06-03T19:49:17.029 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:49:16 smithi089 bash[78452]: audit 1685821756.29453 mon.c (mon.2) 139 : audit 0 from='client.? 172.21.15.89:0/2171080940' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:49:17.029 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:49:16 smithi089 bash[75068]: audit 1685821756.29453 mon.c (mon.2) 139 : audit 0 from='client.? 172.21.15.89:0/2171080940' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:49:17.085 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:49:16 smithi164 bash[59462]: audit 1685821756.29453 mon.c (mon.2) 139 : audit 0 from='client.? 172.21.15.89:0/2171080940' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:49:18.029 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:49:17 smithi089 bash[78452]: cluster 1685821756.1554303 mgr.y (mgr.34104) 323 : cluster 0 pgmap v260: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 3.3 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 574 op/s; 16812/162273 objects degraded (10.360%); 784 KiB/s, 12 objects/s recovering 2023-06-03T19:49:18.029 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:49:17 smithi089 bash[78452]: cluster 1685821757.2626576 mon.a (mon.0) 291 : cluster 3 Health check update: Degraded data redundancy: 16812/162273 objects degraded (10.360%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-06-03T19:49:18.030 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:49:17 smithi089 bash[75068]: cluster 1685821756.1554303 mgr.y (mgr.34104) 323 : cluster 0 pgmap v260: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 3.3 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 574 op/s; 16812/162273 objects degraded (10.360%); 784 KiB/s, 12 objects/s recovering 2023-06-03T19:49:18.030 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:49:17 smithi089 bash[75068]: cluster 1685821757.2626576 mon.a (mon.0) 291 : cluster 3 Health check update: Degraded data redundancy: 16812/162273 objects degraded (10.360%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-06-03T19:49:18.085 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:49:17 smithi164 bash[59462]: cluster 1685821756.1554303 mgr.y (mgr.34104) 323 : cluster 0 pgmap v260: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 3.3 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 574 op/s; 16812/162273 objects degraded (10.360%); 784 KiB/s, 12 objects/s recovering 2023-06-03T19:49:18.085 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:49:17 smithi164 bash[59462]: cluster 1685821757.2626576 mon.a (mon.0) 291 : cluster 3 Health check update: Degraded data redundancy: 16812/162273 objects degraded (10.360%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-06-03T19:49:20.029 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:49:19 smithi089 bash[75068]: cluster 1685821758.1559904 mgr.y (mgr.34104) 324 : cluster 0 pgmap v261: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 3.3 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 540 op/s; 16812/162273 objects degraded (10.360%); 784 KiB/s, 12 objects/s recovering 2023-06-03T19:49:20.030 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:49:19 smithi089 bash[75068]: audit 1685821758.8967416 mon.a (mon.0) 292 : audit 0 from='client.? 172.21.15.89:0/2727598910' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:49:20.030 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:49:19 smithi089 bash[75068]: audit 1685821759.2576149 mon.c (mon.2) 140 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-03T19:49:20.030 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:49:19 smithi089 bash[78452]: cluster 1685821758.1559904 mgr.y (mgr.34104) 324 : cluster 0 pgmap v261: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 3.3 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 540 op/s; 16812/162273 objects degraded (10.360%); 784 KiB/s, 12 objects/s recovering 2023-06-03T19:49:20.031 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:49:19 smithi089 bash[78452]: audit 1685821758.8967416 mon.a (mon.0) 292 : audit 0 from='client.? 172.21.15.89:0/2727598910' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:49:20.031 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:49:19 smithi089 bash[78452]: audit 1685821759.2576149 mon.c (mon.2) 140 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-03T19:49:20.085 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:49:19 smithi164 bash[59462]: cluster 1685821758.1559904 mgr.y (mgr.34104) 324 : cluster 0 pgmap v261: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 3.3 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 540 op/s; 16812/162273 objects degraded (10.360%); 784 KiB/s, 12 objects/s recovering 2023-06-03T19:49:20.085 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:49:19 smithi164 bash[59462]: audit 1685821758.8967416 mon.a (mon.0) 292 : audit 0 from='client.? 172.21.15.89:0/2727598910' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:49:20.085 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:49:19 smithi164 bash[59462]: audit 1685821759.2576149 mon.c (mon.2) 140 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-03T19:49:22.029 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:49:21 smithi089 bash[75068]: cluster 1685821760.15807 mgr.y (mgr.34104) 325 : cluster 0 pgmap v262: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 3.1 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 771 op/s; 16739/153945 objects degraded (10.873%); 1.1 MiB/s, 18 objects/s recovering 2023-06-03T19:49:22.030 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:49:21 smithi089 bash[75068]: audit 1685821761.5075479 mon.a (mon.0) 293 : audit 0 from='client.? 172.21.15.89:0/655160942' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:49:22.030 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:49:21 smithi089 bash[78452]: cluster 1685821760.15807 mgr.y (mgr.34104) 325 : cluster 0 pgmap v262: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 3.1 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 771 op/s; 16739/153945 objects degraded (10.873%); 1.1 MiB/s, 18 objects/s recovering 2023-06-03T19:49:22.030 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:49:21 smithi089 bash[78452]: audit 1685821761.5075479 mon.a (mon.0) 293 : audit 0 from='client.? 172.21.15.89:0/655160942' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:49:22.085 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:49:21 smithi164 bash[59462]: cluster 1685821760.15807 mgr.y (mgr.34104) 325 : cluster 0 pgmap v262: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 3.1 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 771 op/s; 16739/153945 objects degraded (10.873%); 1.1 MiB/s, 18 objects/s recovering 2023-06-03T19:49:22.085 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:49:21 smithi164 bash[59462]: audit 1685821761.5075479 mon.a (mon.0) 293 : audit 0 from='client.? 172.21.15.89:0/655160942' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:49:23.029 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:49:22 smithi089 bash[78452]: audit 1685821762.072129 mon.c (mon.2) 141 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T19:49:23.029 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:49:22 smithi089 bash[78452]: cluster 1685821762.2638195 mon.a (mon.0) 294 : cluster 3 Health check update: Degraded data redundancy: 16739/153945 objects degraded (10.873%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-06-03T19:49:23.030 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:49:22 smithi089 bash[75068]: audit 1685821762.072129 mon.c (mon.2) 141 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T19:49:23.030 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:49:22 smithi089 bash[75068]: cluster 1685821762.2638195 mon.a (mon.0) 294 : cluster 3 Health check update: Degraded data redundancy: 16739/153945 objects degraded (10.873%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-06-03T19:49:23.085 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:49:22 smithi164 bash[59462]: audit 1685821762.072129 mon.c (mon.2) 141 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T19:49:23.085 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:49:22 smithi164 bash[59462]: cluster 1685821762.2638195 mon.a (mon.0) 294 : cluster 3 Health check update: Degraded data redundancy: 16739/153945 objects degraded (10.873%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-06-03T19:49:24.029 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:49:23 smithi089 bash[78452]: cluster 1685821762.1586869 mgr.y (mgr.34104) 326 : cluster 0 pgmap v263: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 3.1 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 564 op/s; 16739/152781 objects degraded (10.956%); 789 KiB/s, 12 objects/s recovering 2023-06-03T19:49:24.029 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:49:23 smithi089 bash[75068]: cluster 1685821762.1586869 mgr.y (mgr.34104) 326 : cluster 0 pgmap v263: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 3.1 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 564 op/s; 16739/152781 objects degraded (10.956%); 789 KiB/s, 12 objects/s recovering 2023-06-03T19:49:24.085 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:49:23 smithi164 bash[59462]: cluster 1685821762.1586869 mgr.y (mgr.34104) 326 : cluster 0 pgmap v263: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 3.1 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 564 op/s; 16739/152781 objects degraded (10.956%); 789 KiB/s, 12 objects/s recovering 2023-06-03T19:49:25.029 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:49:24 smithi089 bash[78452]: audit 1685821764.0771134 mon.c (mon.2) 142 : audit 0 from='client.? 172.21.15.89:0/3367197936' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:49:25.029 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:49:24 smithi089 bash[75068]: audit 1685821764.0771134 mon.c (mon.2) 142 : audit 0 from='client.? 172.21.15.89:0/3367197936' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:49:25.085 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:49:24 smithi164 bash[59462]: audit 1685821764.0771134 mon.c (mon.2) 142 : audit 0 from='client.? 172.21.15.89:0/3367197936' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:49:26.029 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:49:25 smithi089 bash[78452]: cluster 1685821764.15947 mgr.y (mgr.34104) 327 : cluster 0 pgmap v264: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 2.9 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 757 op/s; 16668/144627 objects degraded (11.525%); 1.1 MiB/s, 18 objects/s recovering 2023-06-03T19:49:26.029 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:49:25 smithi089 bash[75068]: cluster 1685821764.15947 mgr.y (mgr.34104) 327 : cluster 0 pgmap v264: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 2.9 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 757 op/s; 16668/144627 objects degraded (11.525%); 1.1 MiB/s, 18 objects/s recovering 2023-06-03T19:49:26.085 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:49:25 smithi164 bash[59462]: cluster 1685821764.15947 mgr.y (mgr.34104) 327 : cluster 0 pgmap v264: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 2.9 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 757 op/s; 16668/144627 objects degraded (11.525%); 1.1 MiB/s, 18 objects/s recovering 2023-06-03T19:49:26.529 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 19:49:26 smithi089 bash[71163]: ::ffff:172.21.15.164 - - [03/Jun/2023:19:49:26] "GET /metrics HTTP/1.1" 200 207411 "" "Prometheus/2.33.4" 2023-06-03T19:49:27.030 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:49:26 smithi089 bash[75068]: audit 1685821765.7958899 mon.a (mon.0) 295 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:49:27.030 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:49:26 smithi089 bash[75068]: audit 1685821765.810098 mon.a (mon.0) 296 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:49:27.030 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:49:26 smithi089 bash[75068]: audit 1685821765.8394642 mon.a (mon.0) 297 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:49:27.030 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:49:26 smithi089 bash[75068]: audit 1685821765.8584514 mon.a (mon.0) 298 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:49:27.030 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:49:26 smithi089 bash[75068]: cluster 1685821766.1600382 mgr.y (mgr.34104) 328 : cluster 0 pgmap v265: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 2.9 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 556 op/s; 16668/143457 objects degraded (11.619%); 768 KiB/s, 11 objects/s recovering 2023-06-03T19:49:27.030 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:49:26 smithi089 bash[75068]: audit 1685821766.5029275 mon.c (mon.2) 143 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-03T19:49:27.031 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:49:26 smithi089 bash[75068]: audit 1685821766.5049477 mon.c (mon.2) 144 : audit 1 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-03T19:49:27.031 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:49:26 smithi089 bash[75068]: audit 1685821766.5174754 mon.a (mon.0) 299 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:49:27.031 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:49:26 smithi089 bash[75068]: audit 1685821766.6038504 mon.c (mon.2) 145 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-03T19:49:27.031 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:49:26 smithi089 bash[75068]: audit 1685821766.6099381 mon.c (mon.2) 146 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-06-03T19:49:27.031 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:49:26 smithi089 bash[75068]: audit 1685821766.6134415 mon.c (mon.2) 147 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-06-03T19:49:27.031 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:49:26 smithi089 bash[75068]: audit 1685821766.619891 mon.c (mon.2) 148 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-06-03T19:49:27.031 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:49:26 smithi089 bash[75068]: audit 1685821766.6228826 mon.c (mon.2) 149 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-06-03T19:49:27.032 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:49:26 smithi089 bash[75068]: audit 1685821766.623153 mgr.y (mgr.34104) 329 : audit 0 from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-06-03T19:49:27.032 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:49:26 smithi089 bash[75068]: cephadm 1685821766.6244876 mgr.y (mgr.34104) 330 : cephadm 1 Upgrade: unsafe to stop osd(s) at this time (1 PGs are or would become offline) 2023-06-03T19:49:27.032 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:49:26 smithi089 bash[75068]: audit 1685821766.6768804 mon.c (mon.2) 150 : audit 0 from='client.? 172.21.15.89:0/41778720' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:49:27.032 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 19:49:26 smithi089 bash[71163]: debug 2023-06-03T19:49:26.620+0000 7fe48642e700 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (1 PGs are or would become offline) 2023-06-03T19:49:27.032 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:49:26 smithi089 bash[78452]: audit 1685821765.7958899 mon.a (mon.0) 295 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:49:27.033 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:49:26 smithi089 bash[78452]: audit 1685821765.810098 mon.a (mon.0) 296 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:49:27.033 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:49:26 smithi089 bash[78452]: audit 1685821765.8394642 mon.a (mon.0) 297 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:49:27.033 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:49:26 smithi089 bash[78452]: audit 1685821765.8584514 mon.a (mon.0) 298 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:49:27.033 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:49:26 smithi089 bash[78452]: cluster 1685821766.1600382 mgr.y (mgr.34104) 328 : cluster 0 pgmap v265: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 2.9 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 556 op/s; 16668/143457 objects degraded (11.619%); 768 KiB/s, 11 objects/s recovering 2023-06-03T19:49:27.033 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:49:26 smithi089 bash[78452]: audit 1685821766.5029275 mon.c (mon.2) 143 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-03T19:49:27.033 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:49:26 smithi089 bash[78452]: audit 1685821766.5049477 mon.c (mon.2) 144 : audit 1 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-03T19:49:27.033 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:49:26 smithi089 bash[78452]: audit 1685821766.5174754 mon.a (mon.0) 299 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:49:27.034 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:49:26 smithi089 bash[78452]: audit 1685821766.6038504 mon.c (mon.2) 145 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-03T19:49:27.034 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:49:26 smithi089 bash[78452]: audit 1685821766.6099381 mon.c (mon.2) 146 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-06-03T19:49:27.034 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:49:26 smithi089 bash[78452]: audit 1685821766.6134415 mon.c (mon.2) 147 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-06-03T19:49:27.034 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:49:26 smithi089 bash[78452]: audit 1685821766.619891 mon.c (mon.2) 148 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-06-03T19:49:27.034 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:49:26 smithi089 bash[78452]: audit 1685821766.6228826 mon.c (mon.2) 149 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-06-03T19:49:27.035 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:49:26 smithi089 bash[78452]: audit 1685821766.623153 mgr.y (mgr.34104) 329 : audit 0 from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-06-03T19:49:27.035 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:49:26 smithi089 bash[78452]: cephadm 1685821766.6244876 mgr.y (mgr.34104) 330 : cephadm 1 Upgrade: unsafe to stop osd(s) at this time (1 PGs are or would become offline) 2023-06-03T19:49:27.035 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:49:26 smithi089 bash[78452]: audit 1685821766.6768804 mon.c (mon.2) 150 : audit 0 from='client.? 172.21.15.89:0/41778720' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:49:27.085 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:49:26 smithi164 bash[59462]: audit 1685821765.7958899 mon.a (mon.0) 295 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:49:27.086 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:49:26 smithi164 bash[59462]: audit 1685821765.810098 mon.a (mon.0) 296 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:49:27.086 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:49:26 smithi164 bash[59462]: audit 1685821765.8394642 mon.a (mon.0) 297 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:49:27.086 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:49:26 smithi164 bash[59462]: audit 1685821765.8584514 mon.a (mon.0) 298 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:49:27.087 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:49:26 smithi164 bash[59462]: cluster 1685821766.1600382 mgr.y (mgr.34104) 328 : cluster 0 pgmap v265: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 2.9 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 556 op/s; 16668/143457 objects degraded (11.619%); 768 KiB/s, 11 objects/s recovering 2023-06-03T19:49:27.087 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:49:26 smithi164 bash[59462]: audit 1685821766.5029275 mon.c (mon.2) 143 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-03T19:49:27.087 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:49:26 smithi164 bash[59462]: audit 1685821766.5049477 mon.c (mon.2) 144 : audit 1 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-03T19:49:27.087 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:49:26 smithi164 bash[59462]: audit 1685821766.5174754 mon.a (mon.0) 299 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:49:27.088 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:49:26 smithi164 bash[59462]: audit 1685821766.6038504 mon.c (mon.2) 145 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-03T19:49:27.088 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:49:26 smithi164 bash[59462]: audit 1685821766.6099381 mon.c (mon.2) 146 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-06-03T19:49:27.088 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:49:26 smithi164 bash[59462]: audit 1685821766.6134415 mon.c (mon.2) 147 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-06-03T19:49:27.088 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:49:26 smithi164 bash[59462]: audit 1685821766.619891 mon.c (mon.2) 148 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-06-03T19:49:27.089 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:49:26 smithi164 bash[59462]: audit 1685821766.6228826 mon.c (mon.2) 149 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-06-03T19:49:27.089 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:49:26 smithi164 bash[59462]: audit 1685821766.623153 mgr.y (mgr.34104) 329 : audit 0 from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-06-03T19:49:27.089 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:49:26 smithi164 bash[59462]: cephadm 1685821766.6244876 mgr.y (mgr.34104) 330 : cephadm 1 Upgrade: unsafe to stop osd(s) at this time (1 PGs are or would become offline) 2023-06-03T19:49:27.089 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:49:26 smithi164 bash[59462]: audit 1685821766.6768804 mon.c (mon.2) 150 : audit 0 from='client.? 172.21.15.89:0/41778720' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:49:28.279 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:49:27 smithi089 bash[78452]: cluster 1685821767.2647977 mon.a (mon.0) 300 : cluster 3 Health check update: Degraded data redundancy: 16668/143457 objects degraded (11.619%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-06-03T19:49:28.279 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:49:27 smithi089 bash[75068]: cluster 1685821767.2647977 mon.a (mon.0) 300 : cluster 3 Health check update: Degraded data redundancy: 16668/143457 objects degraded (11.619%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-06-03T19:49:28.335 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:49:27 smithi164 bash[59462]: cluster 1685821767.2647977 mon.a (mon.0) 300 : cluster 3 Health check update: Degraded data redundancy: 16668/143457 objects degraded (11.619%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-06-03T19:49:29.279 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:49:28 smithi089 bash[75068]: cluster 1685821768.160613 mgr.y (mgr.34104) 331 : cluster 0 pgmap v266: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 2.9 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 522 op/s; 16668/143457 objects degraded (11.619%); 768 KiB/s, 11 objects/s recovering 2023-06-03T19:49:29.280 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:49:28 smithi089 bash[78452]: cluster 1685821768.160613 mgr.y (mgr.34104) 331 : cluster 0 pgmap v266: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 2.9 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 522 op/s; 16668/143457 objects degraded (11.619%); 768 KiB/s, 11 objects/s recovering 2023-06-03T19:49:29.335 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:49:28 smithi164 bash[59462]: cluster 1685821768.160613 mgr.y (mgr.34104) 331 : cluster 0 pgmap v266: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 2.9 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 522 op/s; 16668/143457 objects degraded (11.619%); 768 KiB/s, 11 objects/s recovering 2023-06-03T19:49:30.279 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:49:29 smithi089 bash[75068]: audit 1685821769.3101943 mon.a (mon.0) 301 : audit 0 from='client.? 172.21.15.89:0/3089455618' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:49:30.279 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:49:29 smithi089 bash[78452]: audit 1685821769.3101943 mon.a (mon.0) 301 : audit 0 from='client.? 172.21.15.89:0/3089455618' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:49:30.335 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:49:29 smithi164 bash[59462]: audit 1685821769.3101943 mon.a (mon.0) 301 : audit 0 from='client.? 172.21.15.89:0/3089455618' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:49:31.279 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:49:30 smithi089 bash[75068]: cluster 1685821770.1614265 mgr.y (mgr.34104) 332 : cluster 0 pgmap v267: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 2.8 GiB data, 9.9 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 738 op/s; 16597/135678 objects degraded (12.233%); 1.1 MiB/s, 17 objects/s recovering 2023-06-03T19:49:31.280 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:49:30 smithi089 bash[78452]: cluster 1685821770.1614265 mgr.y (mgr.34104) 332 : cluster 0 pgmap v267: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 2.8 GiB data, 9.9 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 738 op/s; 16597/135678 objects degraded (12.233%); 1.1 MiB/s, 17 objects/s recovering 2023-06-03T19:49:31.335 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:49:30 smithi164 bash[59462]: cluster 1685821770.1614265 mgr.y (mgr.34104) 332 : cluster 0 pgmap v267: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 2.8 GiB data, 9.9 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 738 op/s; 16597/135678 objects degraded (12.233%); 1.1 MiB/s, 17 objects/s recovering 2023-06-03T19:49:32.279 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:49:31 smithi089 bash[75068]: audit 1685821771.9451516 mon.c (mon.2) 151 : audit 0 from='client.? 172.21.15.89:0/2197780989' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:49:32.279 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:49:31 smithi089 bash[78452]: audit 1685821771.9451516 mon.c (mon.2) 151 : audit 0 from='client.? 172.21.15.89:0/2197780989' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:49:32.335 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:49:31 smithi164 bash[59462]: audit 1685821771.9451516 mon.c (mon.2) 151 : audit 0 from='client.? 172.21.15.89:0/2197780989' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:49:33.279 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:49:33 smithi089 bash[75068]: cluster 1685821772.162032 mgr.y (mgr.34104) 333 : cluster 0 pgmap v268: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 2.7 GiB data, 9.9 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 540 op/s; 16597/134478 objects degraded (12.342%); 757 KiB/s, 11 objects/s recovering 2023-06-03T19:49:33.280 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:49:33 smithi089 bash[75068]: cluster 1685821772.2660706 mon.a (mon.0) 302 : cluster 3 Health check update: Degraded data redundancy: 16597/135678 objects degraded (12.233%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-06-03T19:49:33.280 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:49:33 smithi089 bash[78452]: cluster 1685821772.162032 mgr.y (mgr.34104) 333 : cluster 0 pgmap v268: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 2.7 GiB data, 9.9 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 540 op/s; 16597/134478 objects degraded (12.342%); 757 KiB/s, 11 objects/s recovering 2023-06-03T19:49:33.280 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:49:33 smithi089 bash[78452]: cluster 1685821772.2660706 mon.a (mon.0) 302 : cluster 3 Health check update: Degraded data redundancy: 16597/135678 objects degraded (12.233%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-06-03T19:49:33.335 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:49:33 smithi164 bash[59462]: cluster 1685821772.162032 mgr.y (mgr.34104) 333 : cluster 0 pgmap v268: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 2.7 GiB data, 9.9 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 540 op/s; 16597/134478 objects degraded (12.342%); 757 KiB/s, 11 objects/s recovering 2023-06-03T19:49:33.336 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:49:33 smithi164 bash[59462]: cluster 1685821772.2660706 mon.a (mon.0) 302 : cluster 3 Health check update: Degraded data redundancy: 16597/135678 objects degraded (12.233%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-06-03T19:49:35.529 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:49:35 smithi089 bash[78452]: cluster 1685821774.1628683 mgr.y (mgr.34104) 334 : cluster 0 pgmap v269: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 2.6 GiB data, 9.5 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 661 op/s; 16499/128955 objects degraded (12.794%); 885 KiB/s, 19 objects/s recovering 2023-06-03T19:49:35.542 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:49:35 smithi089 bash[78452]: audit 1685821774.5072515 mon.a (mon.0) 303 : audit 0 from='client.? 172.21.15.89:0/1172158889' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:49:35.543 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:49:35 smithi089 bash[75068]: cluster 1685821774.1628683 mgr.y (mgr.34104) 334 : cluster 0 pgmap v269: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 2.6 GiB data, 9.5 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 661 op/s; 16499/128955 objects degraded (12.794%); 885 KiB/s, 19 objects/s recovering 2023-06-03T19:49:35.543 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:49:35 smithi089 bash[75068]: audit 1685821774.5072515 mon.a (mon.0) 303 : audit 0 from='client.? 172.21.15.89:0/1172158889' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:49:35.585 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:49:35 smithi164 bash[59462]: cluster 1685821774.1628683 mgr.y (mgr.34104) 334 : cluster 0 pgmap v269: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 2.6 GiB data, 9.5 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 661 op/s; 16499/128955 objects degraded (12.794%); 885 KiB/s, 19 objects/s recovering 2023-06-03T19:49:35.585 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:49:35 smithi164 bash[59462]: audit 1685821774.5072515 mon.a (mon.0) 303 : audit 0 from='client.? 172.21.15.89:0/1172158889' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:49:36.529 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 19:49:36 smithi089 bash[71163]: ::ffff:172.21.15.164 - - [03/Jun/2023:19:49:36] "GET /metrics HTTP/1.1" 200 207411 "" "Prometheus/2.33.4" 2023-06-03T19:49:37.529 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:49:37 smithi089 bash[75068]: cluster 1685821776.163569 mgr.y (mgr.34104) 335 : cluster 0 pgmap v270: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 2.5 GiB data, 9.3 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 551 op/s; 16499/124767 objects degraded (13.224%); 506 KiB/s, 14 objects/s recovering 2023-06-03T19:49:37.529 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:49:37 smithi089 bash[75068]: audit 1685821777.0559444 mon.b (mon.1) 12 : audit 0 from='client.? 172.21.15.89:0/3765761834' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:49:37.530 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:49:37 smithi089 bash[75068]: audit 1685821777.0768669 mon.c (mon.2) 152 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T19:49:37.530 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:49:37 smithi089 bash[78452]: cluster 1685821776.163569 mgr.y (mgr.34104) 335 : cluster 0 pgmap v270: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 2.5 GiB data, 9.3 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 551 op/s; 16499/124767 objects degraded (13.224%); 506 KiB/s, 14 objects/s recovering 2023-06-03T19:49:37.530 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:49:37 smithi089 bash[78452]: audit 1685821777.0559444 mon.b (mon.1) 12 : audit 0 from='client.? 172.21.15.89:0/3765761834' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:49:37.530 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:49:37 smithi089 bash[78452]: audit 1685821777.0768669 mon.c (mon.2) 152 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T19:49:37.585 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:49:37 smithi164 bash[59462]: cluster 1685821776.163569 mgr.y (mgr.34104) 335 : cluster 0 pgmap v270: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 2.5 GiB data, 9.3 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 551 op/s; 16499/124767 objects degraded (13.224%); 506 KiB/s, 14 objects/s recovering 2023-06-03T19:49:37.585 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:49:37 smithi164 bash[59462]: audit 1685821777.0559444 mon.b (mon.1) 12 : audit 0 from='client.? 172.21.15.89:0/3765761834' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:49:37.585 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:49:37 smithi164 bash[59462]: audit 1685821777.0768669 mon.c (mon.2) 152 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T19:49:38.529 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:49:38 smithi089 bash[75068]: cluster 1685821777.2672787 mon.a (mon.0) 304 : cluster 3 Health check update: Degraded data redundancy: 16499/124767 objects degraded (13.224%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-06-03T19:49:38.530 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:49:38 smithi089 bash[78452]: cluster 1685821777.2672787 mon.a (mon.0) 304 : cluster 3 Health check update: Degraded data redundancy: 16499/124767 objects degraded (13.224%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-06-03T19:49:38.585 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:49:38 smithi164 bash[59462]: cluster 1685821777.2672787 mon.a (mon.0) 304 : cluster 3 Health check update: Degraded data redundancy: 16499/124767 objects degraded (13.224%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-06-03T19:49:39.585 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:49:39 smithi164 bash[59462]: cluster 1685821778.164171 mgr.y (mgr.34104) 336 : cluster 0 pgmap v271: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 2.5 GiB data, 9.3 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 518 op/s; 16499/124767 objects degraded (13.224%); 506 KiB/s, 14 objects/s recovering 2023-06-03T19:49:39.779 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:49:39 smithi089 bash[75068]: cluster 1685821778.164171 mgr.y (mgr.34104) 336 : cluster 0 pgmap v271: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 2.5 GiB data, 9.3 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 518 op/s; 16499/124767 objects degraded (13.224%); 506 KiB/s, 14 objects/s recovering 2023-06-03T19:49:39.779 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:49:39 smithi089 bash[78452]: cluster 1685821778.164171 mgr.y (mgr.34104) 336 : cluster 0 pgmap v271: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 2.5 GiB data, 9.3 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 518 op/s; 16499/124767 objects degraded (13.224%); 506 KiB/s, 14 objects/s recovering 2023-06-03T19:49:40.585 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:49:40 smithi164 bash[59462]: audit 1685821779.6616209 mon.a (mon.0) 305 : audit 0 from='client.? 172.21.15.89:0/4052045493' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:49:40.779 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:49:40 smithi089 bash[75068]: audit 1685821779.6616209 mon.a (mon.0) 305 : audit 0 from='client.? 172.21.15.89:0/4052045493' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:49:40.779 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:49:40 smithi089 bash[78452]: audit 1685821779.6616209 mon.a (mon.0) 305 : audit 0 from='client.? 172.21.15.89:0/4052045493' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:49:41.585 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:49:41 smithi164 bash[59462]: cluster 1685821780.1649764 mgr.y (mgr.34104) 337 : cluster 0 pgmap v272: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 2.4 GiB data, 8.8 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 741 op/s; 16396/116769 objects degraded (14.041%); 506 KiB/s, 22 objects/s recovering 2023-06-03T19:49:41.644 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:49:41 smithi089 bash[75068]: cluster 1685821780.1649764 mgr.y (mgr.34104) 337 : cluster 0 pgmap v272: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 2.4 GiB data, 8.8 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 741 op/s; 16396/116769 objects degraded (14.041%); 506 KiB/s, 22 objects/s recovering 2023-06-03T19:49:41.644 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 19:49:41 smithi089 bash[71163]: debug 2023-06-03T19:49:41.636+0000 7fe48642e700 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (1 PGs are or would become offline) 2023-06-03T19:49:41.645 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:49:41 smithi089 bash[78452]: cluster 1685821780.1649764 mgr.y (mgr.34104) 337 : cluster 0 pgmap v272: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 2.4 GiB data, 8.8 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 741 op/s; 16396/116769 objects degraded (14.041%); 506 KiB/s, 22 objects/s recovering 2023-06-03T19:49:42.585 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:49:42 smithi164 bash[59462]: audit 1685821781.6374075 mon.c (mon.2) 153 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-06-03T19:49:42.585 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:49:42 smithi164 bash[59462]: audit 1685821781.6379583 mgr.y (mgr.34104) 338 : audit 0 from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-06-03T19:49:42.586 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:49:42 smithi164 bash[59462]: cephadm 1685821781.6397858 mgr.y (mgr.34104) 339 : cephadm 1 Upgrade: unsafe to stop osd(s) at this time (1 PGs are or would become offline) 2023-06-03T19:49:42.586 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:49:42 smithi164 bash[59462]: audit 1685821782.238447 mon.a (mon.0) 306 : audit 0 from='client.? 172.21.15.89:0/135117576' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:49:42.586 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:49:42 smithi164 bash[59462]: cluster 1685821782.2684038 mon.a (mon.0) 307 : cluster 3 Health check update: Degraded data redundancy: 16396/116769 objects degraded (14.041%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-06-03T19:49:42.779 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:49:42 smithi089 bash[75068]: audit 1685821781.6374075 mon.c (mon.2) 153 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-06-03T19:49:42.780 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:49:42 smithi089 bash[75068]: audit 1685821781.6379583 mgr.y (mgr.34104) 338 : audit 0 from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-06-03T19:49:42.780 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:49:42 smithi089 bash[75068]: cephadm 1685821781.6397858 mgr.y (mgr.34104) 339 : cephadm 1 Upgrade: unsafe to stop osd(s) at this time (1 PGs are or would become offline) 2023-06-03T19:49:42.780 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:49:42 smithi089 bash[75068]: audit 1685821782.238447 mon.a (mon.0) 306 : audit 0 from='client.? 172.21.15.89:0/135117576' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:49:42.781 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:49:42 smithi089 bash[75068]: cluster 1685821782.2684038 mon.a (mon.0) 307 : cluster 3 Health check update: Degraded data redundancy: 16396/116769 objects degraded (14.041%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-06-03T19:49:42.781 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:49:42 smithi089 bash[78452]: audit 1685821781.6374075 mon.c (mon.2) 153 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-06-03T19:49:42.781 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:49:42 smithi089 bash[78452]: audit 1685821781.6379583 mgr.y (mgr.34104) 338 : audit 0 from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-06-03T19:49:42.782 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:49:42 smithi089 bash[78452]: cephadm 1685821781.6397858 mgr.y (mgr.34104) 339 : cephadm 1 Upgrade: unsafe to stop osd(s) at this time (1 PGs are or would become offline) 2023-06-03T19:49:42.782 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:49:42 smithi089 bash[78452]: audit 1685821782.238447 mon.a (mon.0) 306 : audit 0 from='client.? 172.21.15.89:0/135117576' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:49:42.782 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:49:42 smithi089 bash[78452]: cluster 1685821782.2684038 mon.a (mon.0) 307 : cluster 3 Health check update: Degraded data redundancy: 16396/116769 objects degraded (14.041%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-06-03T19:49:43.585 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:49:43 smithi164 bash[59462]: cluster 1685821782.1656175 mgr.y (mgr.34104) 340 : cluster 0 pgmap v273: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 2.4 GiB data, 8.7 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 557 op/s; 16396/115590 objects degraded (14.185%); 128 KiB/s, 16 objects/s recovering 2023-06-03T19:49:43.779 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:49:43 smithi089 bash[75068]: cluster 1685821782.1656175 mgr.y (mgr.34104) 340 : cluster 0 pgmap v273: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 2.4 GiB data, 8.7 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 557 op/s; 16396/115590 objects degraded (14.185%); 128 KiB/s, 16 objects/s recovering 2023-06-03T19:49:43.779 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:49:43 smithi089 bash[78452]: cluster 1685821782.1656175 mgr.y (mgr.34104) 340 : cluster 0 pgmap v273: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 2.4 GiB data, 8.7 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 557 op/s; 16396/115590 objects degraded (14.185%); 128 KiB/s, 16 objects/s recovering 2023-06-03T19:49:45.585 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:49:45 smithi164 bash[59462]: cluster 1685821784.1663883 mgr.y (mgr.34104) 341 : cluster 0 pgmap v274: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 2.2 GiB data, 8.4 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 670 op/s; 16292/110319 objects degraded (14.768%); 128 KiB/s, 25 objects/s recovering 2023-06-03T19:49:45.585 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:49:45 smithi164 bash[59462]: audit 1685821784.8444643 mon.c (mon.2) 154 : audit 0 from='client.? 172.21.15.89:0/2995753031' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:49:45.779 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:49:45 smithi089 bash[75068]: cluster 1685821784.1663883 mgr.y (mgr.34104) 341 : cluster 0 pgmap v274: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 2.2 GiB data, 8.4 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 670 op/s; 16292/110319 objects degraded (14.768%); 128 KiB/s, 25 objects/s recovering 2023-06-03T19:49:45.779 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:49:45 smithi089 bash[75068]: audit 1685821784.8444643 mon.c (mon.2) 154 : audit 0 from='client.? 172.21.15.89:0/2995753031' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:49:45.780 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:49:45 smithi089 bash[78452]: cluster 1685821784.1663883 mgr.y (mgr.34104) 341 : cluster 0 pgmap v274: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 2.2 GiB data, 8.4 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 670 op/s; 16292/110319 objects degraded (14.768%); 128 KiB/s, 25 objects/s recovering 2023-06-03T19:49:45.780 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:49:45 smithi089 bash[78452]: audit 1685821784.8444643 mon.c (mon.2) 154 : audit 0 from='client.? 172.21.15.89:0/2995753031' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:49:46.529 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 19:49:46 smithi089 bash[71163]: ::ffff:172.21.15.164 - - [03/Jun/2023:19:49:46] "GET /metrics HTTP/1.1" 200 207411 "" "Prometheus/2.33.4" 2023-06-03T19:49:47.585 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:49:47 smithi164 bash[59462]: cluster 1685821786.1670535 mgr.y (mgr.34104) 342 : cluster 0 pgmap v275: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 2.2 GiB data, 8.1 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 633 op/s; 16292/106149 objects degraded (15.348%); 0 B/s, 17 objects/s recovering 2023-06-03T19:49:47.585 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:49:47 smithi164 bash[59462]: cluster 1685821787.272803 mon.a (mon.0) 308 : cluster 3 Health check update: Degraded data redundancy: 16292/106149 objects degraded (15.348%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-06-03T19:49:47.779 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:49:47 smithi089 bash[75068]: cluster 1685821786.1670535 mgr.y (mgr.34104) 342 : cluster 0 pgmap v275: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 2.2 GiB data, 8.1 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 633 op/s; 16292/106149 objects degraded (15.348%); 0 B/s, 17 objects/s recovering 2023-06-03T19:49:47.779 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:49:47 smithi089 bash[75068]: cluster 1685821787.272803 mon.a (mon.0) 308 : cluster 3 Health check update: Degraded data redundancy: 16292/106149 objects degraded (15.348%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-06-03T19:49:47.780 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:49:47 smithi089 bash[78452]: cluster 1685821786.1670535 mgr.y (mgr.34104) 342 : cluster 0 pgmap v275: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 2.2 GiB data, 8.1 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 633 op/s; 16292/106149 objects degraded (15.348%); 0 B/s, 17 objects/s recovering 2023-06-03T19:49:47.780 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:49:47 smithi089 bash[78452]: cluster 1685821787.272803 mon.a (mon.0) 308 : cluster 3 Health check update: Degraded data redundancy: 16292/106149 objects degraded (15.348%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-06-03T19:49:48.585 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:49:48 smithi164 bash[59462]: audit 1685821787.4070032 mon.a (mon.0) 309 : audit 0 from='client.? 172.21.15.89:0/1003692054' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:49:48.779 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:49:48 smithi089 bash[78452]: audit 1685821787.4070032 mon.a (mon.0) 309 : audit 0 from='client.? 172.21.15.89:0/1003692054' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:49:48.779 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:49:48 smithi089 bash[75068]: audit 1685821787.4070032 mon.a (mon.0) 309 : audit 0 from='client.? 172.21.15.89:0/1003692054' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:49:49.779 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:49:49 smithi089 bash[78452]: cluster 1685821788.1675525 mgr.y (mgr.34104) 343 : cluster 0 pgmap v276: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 2.2 GiB data, 8.1 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 516 op/s; 16292/106149 objects degraded (15.348%); 0 B/s, 17 objects/s recovering 2023-06-03T19:49:49.779 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:49:49 smithi089 bash[75068]: cluster 1685821788.1675525 mgr.y (mgr.34104) 343 : cluster 0 pgmap v276: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 2.2 GiB data, 8.1 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 516 op/s; 16292/106149 objects degraded (15.348%); 0 B/s, 17 objects/s recovering 2023-06-03T19:49:49.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:49:49 smithi164 bash[59462]: cluster 1685821788.1675525 mgr.y (mgr.34104) 343 : cluster 0 pgmap v276: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 2.2 GiB data, 8.1 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 516 op/s; 16292/106149 objects degraded (15.348%); 0 B/s, 17 objects/s recovering 2023-06-03T19:49:50.779 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:49:50 smithi089 bash[78452]: audit 1685821789.9663627 mon.b (mon.1) 13 : audit 0 from='client.? 172.21.15.89:0/558084403' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:49:50.779 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:49:50 smithi089 bash[75068]: audit 1685821789.9663627 mon.b (mon.1) 13 : audit 0 from='client.? 172.21.15.89:0/558084403' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:49:50.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:49:50 smithi164 bash[59462]: audit 1685821789.9663627 mon.b (mon.1) 13 : audit 0 from='client.? 172.21.15.89:0/558084403' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:49:51.779 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:49:51 smithi089 bash[75068]: cluster 1685821790.168326 mgr.y (mgr.34104) 344 : cluster 0 pgmap v277: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 2.0 GiB data, 7.7 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 746 op/s; 16188/97884 objects degraded (16.538%); 0 B/s, 25 objects/s recovering 2023-06-03T19:49:51.780 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:49:51 smithi089 bash[78452]: cluster 1685821790.168326 mgr.y (mgr.34104) 344 : cluster 0 pgmap v277: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 2.0 GiB data, 7.7 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 746 op/s; 16188/97884 objects degraded (16.538%); 0 B/s, 25 objects/s recovering 2023-06-03T19:49:51.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:49:51 smithi164 bash[59462]: cluster 1685821790.168326 mgr.y (mgr.34104) 344 : cluster 0 pgmap v277: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 2.0 GiB data, 7.7 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 746 op/s; 16188/97884 objects degraded (16.538%); 0 B/s, 25 objects/s recovering 2023-06-03T19:49:52.779 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:49:52 smithi089 bash[75068]: audit 1685821792.077313 mon.c (mon.2) 155 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T19:49:52.780 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:49:52 smithi089 bash[75068]: cluster 1685821792.274086 mon.a (mon.0) 310 : cluster 3 Health check update: Degraded data redundancy: 16188/97884 objects degraded (16.538%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-06-03T19:49:52.780 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:49:52 smithi089 bash[78452]: audit 1685821792.077313 mon.c (mon.2) 155 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T19:49:52.780 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:49:52 smithi089 bash[78452]: cluster 1685821792.274086 mon.a (mon.0) 310 : cluster 3 Health check update: Degraded data redundancy: 16188/97884 objects degraded (16.538%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-06-03T19:49:52.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:49:52 smithi164 bash[59462]: audit 1685821792.077313 mon.c (mon.2) 155 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T19:49:52.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:49:52 smithi164 bash[59462]: cluster 1685821792.274086 mon.a (mon.0) 310 : cluster 3 Health check update: Degraded data redundancy: 16188/97884 objects degraded (16.538%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-06-03T19:49:53.779 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:49:53 smithi089 bash[75068]: cluster 1685821792.1690285 mgr.y (mgr.34104) 345 : cluster 0 pgmap v278: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 2.0 GiB data, 7.6 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 554 op/s; 16188/96783 objects degraded (16.726%); 0 B/s, 17 objects/s recovering 2023-06-03T19:49:53.779 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:49:53 smithi089 bash[75068]: audit 1685821792.5613592 mon.a (mon.0) 311 : audit 0 from='client.? 172.21.15.89:0/3924082616' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:49:53.780 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:49:53 smithi089 bash[78452]: cluster 1685821792.1690285 mgr.y (mgr.34104) 345 : cluster 0 pgmap v278: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 2.0 GiB data, 7.6 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 554 op/s; 16188/96783 objects degraded (16.726%); 0 B/s, 17 objects/s recovering 2023-06-03T19:49:53.780 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:49:53 smithi089 bash[78452]: audit 1685821792.5613592 mon.a (mon.0) 311 : audit 0 from='client.? 172.21.15.89:0/3924082616' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:49:53.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:49:53 smithi164 bash[59462]: cluster 1685821792.1690285 mgr.y (mgr.34104) 345 : cluster 0 pgmap v278: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 2.0 GiB data, 7.6 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 554 op/s; 16188/96783 objects degraded (16.726%); 0 B/s, 17 objects/s recovering 2023-06-03T19:49:53.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:49:53 smithi164 bash[59462]: audit 1685821792.5613592 mon.a (mon.0) 311 : audit 0 from='client.? 172.21.15.89:0/3924082616' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:49:55.779 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:49:55 smithi089 bash[75068]: cluster 1685821794.1697805 mgr.y (mgr.34104) 346 : cluster 0 pgmap v279: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 1.9 GiB data, 7.2 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 667 op/s; 16083/91560 objects degraded (17.566%); 0 B/s, 26 objects/s recovering 2023-06-03T19:49:55.779 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:49:55 smithi089 bash[75068]: audit 1685821795.162267 mon.c (mon.2) 156 : audit 0 from='client.? 172.21.15.89:0/2700447439' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:49:55.779 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:49:55 smithi089 bash[78452]: cluster 1685821794.1697805 mgr.y (mgr.34104) 346 : cluster 0 pgmap v279: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 1.9 GiB data, 7.2 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 667 op/s; 16083/91560 objects degraded (17.566%); 0 B/s, 26 objects/s recovering 2023-06-03T19:49:55.780 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:49:55 smithi089 bash[78452]: audit 1685821795.162267 mon.c (mon.2) 156 : audit 0 from='client.? 172.21.15.89:0/2700447439' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:49:55.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:49:55 smithi164 bash[59462]: cluster 1685821794.1697805 mgr.y (mgr.34104) 346 : cluster 0 pgmap v279: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 1.9 GiB data, 7.2 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 667 op/s; 16083/91560 objects degraded (17.566%); 0 B/s, 26 objects/s recovering 2023-06-03T19:49:55.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:49:55 smithi164 bash[59462]: audit 1685821795.162267 mon.c (mon.2) 156 : audit 0 from='client.? 172.21.15.89:0/2700447439' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:49:56.529 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 19:49:56 smithi089 bash[71163]: ::ffff:172.21.15.164 - - [03/Jun/2023:19:49:56] "GET /metrics HTTP/1.1" 200 207397 "" "Prometheus/2.33.4" 2023-06-03T19:49:57.029 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 19:49:56 smithi089 bash[71163]: debug 2023-06-03T19:49:56.652+0000 7fe48642e700 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (1 PGs are or would become offline) 2023-06-03T19:49:57.779 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:49:57 smithi089 bash[75068]: cluster 1685821796.1704192 mgr.y (mgr.34104) 347 : cluster 0 pgmap v280: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 1.8 GiB data, 7.1 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 631 op/s; 16083/87567 objects degraded (18.367%); 0 B/s, 17 objects/s recovering 2023-06-03T19:49:57.779 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:49:57 smithi089 bash[75068]: audit 1685821796.6532285 mon.c (mon.2) 157 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-06-03T19:49:57.779 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:49:57 smithi089 bash[75068]: audit 1685821796.6534662 mgr.y (mgr.34104) 348 : audit 0 from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-06-03T19:49:57.780 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:49:57 smithi089 bash[75068]: cephadm 1685821796.6550148 mgr.y (mgr.34104) 349 : cephadm 1 Upgrade: unsafe to stop osd(s) at this time (1 PGs are or would become offline) 2023-06-03T19:49:57.780 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:49:57 smithi089 bash[75068]: cluster 1685821797.275416 mon.a (mon.0) 312 : cluster 3 Health check update: Degraded data redundancy: 16083/87567 objects degraded (18.367%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-06-03T19:49:57.780 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:49:57 smithi089 bash[78452]: cluster 1685821796.1704192 mgr.y (mgr.34104) 347 : cluster 0 pgmap v280: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 1.8 GiB data, 7.1 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 631 op/s; 16083/87567 objects degraded (18.367%); 0 B/s, 17 objects/s recovering 2023-06-03T19:49:57.780 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:49:57 smithi089 bash[78452]: audit 1685821796.6532285 mon.c (mon.2) 157 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-06-03T19:49:57.780 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:49:57 smithi089 bash[78452]: audit 1685821796.6534662 mgr.y (mgr.34104) 348 : audit 0 from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-06-03T19:49:57.781 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:49:57 smithi089 bash[78452]: cephadm 1685821796.6550148 mgr.y (mgr.34104) 349 : cephadm 1 Upgrade: unsafe to stop osd(s) at this time (1 PGs are or would become offline) 2023-06-03T19:49:57.781 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:49:57 smithi089 bash[78452]: cluster 1685821797.275416 mon.a (mon.0) 312 : cluster 3 Health check update: Degraded data redundancy: 16083/87567 objects degraded (18.367%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-06-03T19:49:57.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:49:57 smithi164 bash[59462]: cluster 1685821796.1704192 mgr.y (mgr.34104) 347 : cluster 0 pgmap v280: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 1.8 GiB data, 7.1 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 631 op/s; 16083/87567 objects degraded (18.367%); 0 B/s, 17 objects/s recovering 2023-06-03T19:49:57.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:49:57 smithi164 bash[59462]: audit 1685821796.6532285 mon.c (mon.2) 157 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-06-03T19:49:57.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:49:57 smithi164 bash[59462]: audit 1685821796.6534662 mgr.y (mgr.34104) 348 : audit 0 from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-06-03T19:49:57.836 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:49:57 smithi164 bash[59462]: cephadm 1685821796.6550148 mgr.y (mgr.34104) 349 : cephadm 1 Upgrade: unsafe to stop osd(s) at this time (1 PGs are or would become offline) 2023-06-03T19:49:57.836 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:49:57 smithi164 bash[59462]: cluster 1685821797.275416 mon.a (mon.0) 312 : cluster 3 Health check update: Degraded data redundancy: 16083/87567 objects degraded (18.367%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-06-03T19:49:58.779 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:49:58 smithi089 bash[75068]: audit 1685821797.8094778 mon.a (mon.0) 313 : audit 0 from='client.? 172.21.15.89:0/1842046730' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:49:58.779 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:49:58 smithi089 bash[78452]: audit 1685821797.8094778 mon.a (mon.0) 313 : audit 0 from='client.? 172.21.15.89:0/1842046730' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:49:58.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:49:58 smithi164 bash[59462]: audit 1685821797.8094778 mon.a (mon.0) 313 : audit 0 from='client.? 172.21.15.89:0/1842046730' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:49:59.779 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:49:59 smithi089 bash[75068]: cluster 1685821798.1709898 mgr.y (mgr.34104) 350 : cluster 0 pgmap v281: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 1.8 GiB data, 7.1 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 515 op/s; 16083/87567 objects degraded (18.367%); 0 B/s, 17 objects/s recovering 2023-06-03T19:49:59.779 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:49:59 smithi089 bash[78452]: cluster 1685821798.1709898 mgr.y (mgr.34104) 350 : cluster 0 pgmap v281: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 1.8 GiB data, 7.1 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 515 op/s; 16083/87567 objects degraded (18.367%); 0 B/s, 17 objects/s recovering 2023-06-03T19:49:59.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:49:59 smithi164 bash[59462]: cluster 1685821798.1709898 mgr.y (mgr.34104) 350 : cluster 0 pgmap v281: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 1.8 GiB data, 7.1 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 515 op/s; 16083/87567 objects degraded (18.367%); 0 B/s, 17 objects/s recovering 2023-06-03T19:50:00.779 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:50:00 smithi089 bash[75068]: cluster 1685821800.0001268 mon.a (mon.0) 314 : cluster 3 Health detail: HEALTH_WARN Degraded data redundancy: 16083/87567 objects degraded (18.367%), 6 pgs degraded, 6 pgs undersized 2023-06-03T19:50:00.780 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:50:00 smithi089 bash[75068]: cluster 1685821800.0001676 mon.a (mon.0) 315 : cluster 3 [WRN] PG_DEGRADED: Degraded data redundancy: 16083/87567 objects degraded (18.367%), 6 pgs degraded, 6 pgs undersized 2023-06-03T19:50:00.780 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:50:00 smithi089 bash[75068]: cluster 1685821800.0001855 mon.a (mon.0) 316 : cluster 3 pg 7.5 is stuck undersized for 2m, current state active+recovery_wait+undersized+degraded+remapped, last acting [7,2] 2023-06-03T19:50:00.780 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:50:00 smithi089 bash[75068]: cluster 1685821800.000204 mon.a (mon.0) 317 : cluster 3 pg 7.6 is stuck undersized for 2m, current state active+recovery_wait+undersized+degraded+remapped, last acting [3,7] 2023-06-03T19:50:00.781 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:50:00 smithi089 bash[75068]: cluster 1685821800.0002158 mon.a (mon.0) 318 : cluster 3 pg 7.7 is stuck undersized for 2m, current state active+recovery_wait+undersized+degraded+remapped, last acting [6,3] 2023-06-03T19:50:00.781 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:50:00 smithi089 bash[75068]: cluster 1685821800.0002267 mon.a (mon.0) 319 : cluster 3 pg 7.d is stuck undersized for 2m, current state active+recovering+undersized+degraded+remapped, last acting [6,1] 2023-06-03T19:50:00.781 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:50:00 smithi089 bash[75068]: cluster 1685821800.000237 mon.a (mon.0) 320 : cluster 3 pg 7.e is stuck undersized for 2m, current state active+recovery_wait+undersized+degraded+remapped, last acting [2,5] 2023-06-03T19:50:00.781 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:50:00 smithi089 bash[75068]: cluster 1685821800.0002496 mon.a (mon.0) 321 : cluster 3 pg 7.f is stuck undersized for 2m, current state active+recovery_wait+undersized+degraded+remapped, last acting [3,7] 2023-06-03T19:50:00.781 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:50:00 smithi089 bash[78452]: cluster 1685821800.0001268 mon.a (mon.0) 314 : cluster 3 Health detail: HEALTH_WARN Degraded data redundancy: 16083/87567 objects degraded (18.367%), 6 pgs degraded, 6 pgs undersized 2023-06-03T19:50:00.782 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:50:00 smithi089 bash[78452]: cluster 1685821800.0001676 mon.a (mon.0) 315 : cluster 3 [WRN] PG_DEGRADED: Degraded data redundancy: 16083/87567 objects degraded (18.367%), 6 pgs degraded, 6 pgs undersized 2023-06-03T19:50:00.782 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:50:00 smithi089 bash[78452]: cluster 1685821800.0001855 mon.a (mon.0) 316 : cluster 3 pg 7.5 is stuck undersized for 2m, current state active+recovery_wait+undersized+degraded+remapped, last acting [7,2] 2023-06-03T19:50:00.782 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:50:00 smithi089 bash[78452]: cluster 1685821800.000204 mon.a (mon.0) 317 : cluster 3 pg 7.6 is stuck undersized for 2m, current state active+recovery_wait+undersized+degraded+remapped, last acting [3,7] 2023-06-03T19:50:00.782 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:50:00 smithi089 bash[78452]: cluster 1685821800.0002158 mon.a (mon.0) 318 : cluster 3 pg 7.7 is stuck undersized for 2m, current state active+recovery_wait+undersized+degraded+remapped, last acting [6,3] 2023-06-03T19:50:00.782 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:50:00 smithi089 bash[78452]: cluster 1685821800.0002267 mon.a (mon.0) 319 : cluster 3 pg 7.d is stuck undersized for 2m, current state active+recovering+undersized+degraded+remapped, last acting [6,1] 2023-06-03T19:50:00.782 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:50:00 smithi089 bash[78452]: cluster 1685821800.000237 mon.a (mon.0) 320 : cluster 3 pg 7.e is stuck undersized for 2m, current state active+recovery_wait+undersized+degraded+remapped, last acting [2,5] 2023-06-03T19:50:00.782 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:50:00 smithi089 bash[78452]: cluster 1685821800.0002496 mon.a (mon.0) 321 : cluster 3 pg 7.f is stuck undersized for 2m, current state active+recovery_wait+undersized+degraded+remapped, last acting [3,7] 2023-06-03T19:50:00.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:50:00 smithi164 bash[59462]: cluster 1685821800.0001268 mon.a (mon.0) 314 : cluster 3 Health detail: HEALTH_WARN Degraded data redundancy: 16083/87567 objects degraded (18.367%), 6 pgs degraded, 6 pgs undersized 2023-06-03T19:50:00.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:50:00 smithi164 bash[59462]: cluster 1685821800.0001676 mon.a (mon.0) 315 : cluster 3 [WRN] PG_DEGRADED: Degraded data redundancy: 16083/87567 objects degraded (18.367%), 6 pgs degraded, 6 pgs undersized 2023-06-03T19:50:00.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:50:00 smithi164 bash[59462]: cluster 1685821800.0001855 mon.a (mon.0) 316 : cluster 3 pg 7.5 is stuck undersized for 2m, current state active+recovery_wait+undersized+degraded+remapped, last acting [7,2] 2023-06-03T19:50:00.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:50:00 smithi164 bash[59462]: cluster 1685821800.000204 mon.a (mon.0) 317 : cluster 3 pg 7.6 is stuck undersized for 2m, current state active+recovery_wait+undersized+degraded+remapped, last acting [3,7] 2023-06-03T19:50:00.836 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:50:00 smithi164 bash[59462]: cluster 1685821800.0002158 mon.a (mon.0) 318 : cluster 3 pg 7.7 is stuck undersized for 2m, current state active+recovery_wait+undersized+degraded+remapped, last acting [6,3] 2023-06-03T19:50:00.836 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:50:00 smithi164 bash[59462]: cluster 1685821800.0002267 mon.a (mon.0) 319 : cluster 3 pg 7.d is stuck undersized for 2m, current state active+recovering+undersized+degraded+remapped, last acting [6,1] 2023-06-03T19:50:00.836 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:50:00 smithi164 bash[59462]: cluster 1685821800.000237 mon.a (mon.0) 320 : cluster 3 pg 7.e is stuck undersized for 2m, current state active+recovery_wait+undersized+degraded+remapped, last acting [2,5] 2023-06-03T19:50:00.836 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:50:00 smithi164 bash[59462]: cluster 1685821800.0002496 mon.a (mon.0) 321 : cluster 3 pg 7.f is stuck undersized for 2m, current state active+recovery_wait+undersized+degraded+remapped, last acting [3,7] 2023-06-03T19:50:01.779 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:50:01 smithi089 bash[75068]: cluster 1685821800.171808 mgr.y (mgr.34104) 351 : cluster 0 pgmap v282: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 1.6 GiB data, 6.6 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 736 op/s; 15973/79623 objects degraded (20.061%); 0 B/s, 26 objects/s recovering 2023-06-03T19:50:01.780 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:50:01 smithi089 bash[75068]: audit 1685821800.488355 mon.b (mon.1) 14 : audit 0 from='client.? 172.21.15.89:0/3458549204' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:50:01.780 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:50:01 smithi089 bash[78452]: cluster 1685821800.171808 mgr.y (mgr.34104) 351 : cluster 0 pgmap v282: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 1.6 GiB data, 6.6 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 736 op/s; 15973/79623 objects degraded (20.061%); 0 B/s, 26 objects/s recovering 2023-06-03T19:50:01.780 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:50:01 smithi089 bash[78452]: audit 1685821800.488355 mon.b (mon.1) 14 : audit 0 from='client.? 172.21.15.89:0/3458549204' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:50:01.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:50:01 smithi164 bash[59462]: cluster 1685821800.171808 mgr.y (mgr.34104) 351 : cluster 0 pgmap v282: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 1.6 GiB data, 6.6 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 736 op/s; 15973/79623 objects degraded (20.061%); 0 B/s, 26 objects/s recovering 2023-06-03T19:50:01.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:50:01 smithi164 bash[59462]: audit 1685821800.488355 mon.b (mon.1) 14 : audit 0 from='client.? 172.21.15.89:0/3458549204' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:50:02.779 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:50:02 smithi089 bash[75068]: cluster 1685821802.281139 mon.a (mon.0) 322 : cluster 3 Health check update: Degraded data redundancy: 15973/79623 objects degraded (20.061%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-06-03T19:50:02.779 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:50:02 smithi089 bash[78452]: cluster 1685821802.281139 mon.a (mon.0) 322 : cluster 3 Health check update: Degraded data redundancy: 15973/79623 objects degraded (20.061%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-06-03T19:50:02.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:50:02 smithi164 bash[59462]: cluster 1685821802.281139 mon.a (mon.0) 322 : cluster 3 Health check update: Degraded data redundancy: 15973/79623 objects degraded (20.061%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-06-03T19:50:03.779 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:50:03 smithi089 bash[75068]: cluster 1685821802.1723962 mgr.y (mgr.34104) 352 : cluster 0 pgmap v283: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 1.6 GiB data, 6.5 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 540 op/s; 15973/78426 objects degraded (20.367%); 0 B/s, 17 objects/s recovering 2023-06-03T19:50:03.779 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:50:03 smithi089 bash[75068]: audit 1685821803.0978718 mon.c (mon.2) 158 : audit 0 from='client.? 172.21.15.89:0/1767681651' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:50:03.780 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:50:03 smithi089 bash[78452]: cluster 1685821802.1723962 mgr.y (mgr.34104) 352 : cluster 0 pgmap v283: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 1.6 GiB data, 6.5 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 540 op/s; 15973/78426 objects degraded (20.367%); 0 B/s, 17 objects/s recovering 2023-06-03T19:50:03.780 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:50:03 smithi089 bash[78452]: audit 1685821803.0978718 mon.c (mon.2) 158 : audit 0 from='client.? 172.21.15.89:0/1767681651' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:50:03.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:50:03 smithi164 bash[59462]: cluster 1685821802.1723962 mgr.y (mgr.34104) 352 : cluster 0 pgmap v283: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 1.6 GiB data, 6.5 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 540 op/s; 15973/78426 objects degraded (20.367%); 0 B/s, 17 objects/s recovering 2023-06-03T19:50:03.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:50:03 smithi164 bash[59462]: audit 1685821803.0978718 mon.c (mon.2) 158 : audit 0 from='client.? 172.21.15.89:0/1767681651' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:50:05.779 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:50:05 smithi089 bash[75068]: cluster 1685821804.1732337 mgr.y (mgr.34104) 353 : cluster 0 pgmap v284: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 1.5 GiB data, 6.1 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 648 op/s; 15867/73431 objects degraded (21.608%); 0 B/s, 26 objects/s recovering 2023-06-03T19:50:05.779 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:50:05 smithi089 bash[78452]: cluster 1685821804.1732337 mgr.y (mgr.34104) 353 : cluster 0 pgmap v284: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 1.5 GiB data, 6.1 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 648 op/s; 15867/73431 objects degraded (21.608%); 0 B/s, 26 objects/s recovering 2023-06-03T19:50:05.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:50:05 smithi164 bash[59462]: cluster 1685821804.1732337 mgr.y (mgr.34104) 353 : cluster 0 pgmap v284: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 1.5 GiB data, 6.1 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 648 op/s; 15867/73431 objects degraded (21.608%); 0 B/s, 26 objects/s recovering 2023-06-03T19:50:06.529 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:50:06 smithi089 bash[75068]: audit 1685821805.6926284 mon.c (mon.2) 159 : audit 0 from='client.? 172.21.15.89:0/784892634' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:50:06.530 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 19:50:06 smithi089 bash[71163]: ::ffff:172.21.15.164 - - [03/Jun/2023:19:50:06] "GET /metrics HTTP/1.1" 200 207397 "" "Prometheus/2.33.4" 2023-06-03T19:50:06.530 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:50:06 smithi089 bash[78452]: audit 1685821805.6926284 mon.c (mon.2) 159 : audit 0 from='client.? 172.21.15.89:0/784892634' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:50:06.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:50:06 smithi164 bash[59462]: audit 1685821805.6926284 mon.c (mon.2) 159 : audit 0 from='client.? 172.21.15.89:0/784892634' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:50:07.779 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:50:07 smithi089 bash[75068]: cluster 1685821806.1738756 mgr.y (mgr.34104) 354 : cluster 0 pgmap v285: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 1.4 GiB data, 5.9 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 610 op/s; 15867/69564 objects degraded (22.809%); 0 B/s, 17 objects/s recovering 2023-06-03T19:50:07.779 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:50:07 smithi089 bash[75068]: audit 1685821807.0786114 mon.c (mon.2) 160 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T19:50:07.780 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:50:07 smithi089 bash[75068]: cluster 1685821807.283071 mon.a (mon.0) 323 : cluster 3 Health check update: Degraded data redundancy: 15867/69564 objects degraded (22.809%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-06-03T19:50:07.780 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:50:07 smithi089 bash[78452]: cluster 1685821806.1738756 mgr.y (mgr.34104) 354 : cluster 0 pgmap v285: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 1.4 GiB data, 5.9 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 610 op/s; 15867/69564 objects degraded (22.809%); 0 B/s, 17 objects/s recovering 2023-06-03T19:50:07.780 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:50:07 smithi089 bash[78452]: audit 1685821807.0786114 mon.c (mon.2) 160 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T19:50:07.780 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:50:07 smithi089 bash[78452]: cluster 1685821807.283071 mon.a (mon.0) 323 : cluster 3 Health check update: Degraded data redundancy: 15867/69564 objects degraded (22.809%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-06-03T19:50:07.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:50:07 smithi164 bash[59462]: cluster 1685821806.1738756 mgr.y (mgr.34104) 354 : cluster 0 pgmap v285: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 1.4 GiB data, 5.9 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 610 op/s; 15867/69564 objects degraded (22.809%); 0 B/s, 17 objects/s recovering 2023-06-03T19:50:07.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:50:07 smithi164 bash[59462]: audit 1685821807.0786114 mon.c (mon.2) 160 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T19:50:07.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:50:07 smithi164 bash[59462]: cluster 1685821807.283071 mon.a (mon.0) 323 : cluster 3 Health check update: Degraded data redundancy: 15867/69564 objects degraded (22.809%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-06-03T19:50:08.779 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:50:08 smithi089 bash[75068]: audit 1685821808.2859733 mon.a (mon.0) 324 : audit 0 from='client.? 172.21.15.89:0/2120336258' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:50:08.779 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:50:08 smithi089 bash[78452]: audit 1685821808.2859733 mon.a (mon.0) 324 : audit 0 from='client.? 172.21.15.89:0/2120336258' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:50:08.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:50:08 smithi164 bash[59462]: audit 1685821808.2859733 mon.a (mon.0) 324 : audit 0 from='client.? 172.21.15.89:0/2120336258' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:50:09.779 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:50:09 smithi089 bash[75068]: cluster 1685821808.1745832 mgr.y (mgr.34104) 355 : cluster 0 pgmap v286: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 1.4 GiB data, 5.9 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 499 op/s; 15867/69564 objects degraded (22.809%); 0 B/s, 17 objects/s recovering 2023-06-03T19:50:09.779 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:50:09 smithi089 bash[78452]: cluster 1685821808.1745832 mgr.y (mgr.34104) 355 : cluster 0 pgmap v286: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 1.4 GiB data, 5.9 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 499 op/s; 15867/69564 objects degraded (22.809%); 0 B/s, 17 objects/s recovering 2023-06-03T19:50:09.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:50:09 smithi164 bash[59462]: cluster 1685821808.1745832 mgr.y (mgr.34104) 355 : cluster 0 pgmap v286: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 1.4 GiB data, 5.9 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 499 op/s; 15867/69564 objects degraded (22.809%); 0 B/s, 17 objects/s recovering 2023-06-03T19:50:11.779 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:50:11 smithi089 bash[75068]: cluster 1685821810.1753678 mgr.y (mgr.34104) 356 : cluster 0 pgmap v287: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 1.3 GiB data, 5.5 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 722 op/s; 15759/61554 objects degraded (25.602%); 0 B/s, 26 objects/s recovering 2023-06-03T19:50:11.779 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:50:11 smithi089 bash[75068]: audit 1685821810.8731966 mon.a (mon.0) 325 : audit 0 from='client.? 172.21.15.89:0/1105448423' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:50:11.780 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 19:50:11 smithi089 bash[71163]: debug 2023-06-03T19:50:11.671+0000 7fe48642e700 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (1 PGs are or would become offline) 2023-06-03T19:50:11.780 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:50:11 smithi089 bash[78452]: cluster 1685821810.1753678 mgr.y (mgr.34104) 356 : cluster 0 pgmap v287: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 1.3 GiB data, 5.5 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 722 op/s; 15759/61554 objects degraded (25.602%); 0 B/s, 26 objects/s recovering 2023-06-03T19:50:11.780 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:50:11 smithi089 bash[78452]: audit 1685821810.8731966 mon.a (mon.0) 325 : audit 0 from='client.? 172.21.15.89:0/1105448423' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:50:11.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:50:11 smithi164 bash[59462]: cluster 1685821810.1753678 mgr.y (mgr.34104) 356 : cluster 0 pgmap v287: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 1.3 GiB data, 5.5 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 722 op/s; 15759/61554 objects degraded (25.602%); 0 B/s, 26 objects/s recovering 2023-06-03T19:50:11.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:50:11 smithi164 bash[59462]: audit 1685821810.8731966 mon.a (mon.0) 325 : audit 0 from='client.? 172.21.15.89:0/1105448423' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:50:12.779 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:50:12 smithi089 bash[78452]: audit 1685821811.6693015 mon.c (mon.2) 161 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-06-03T19:50:12.780 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:50:12 smithi089 bash[78452]: audit 1685821811.6728628 mgr.y (mgr.34104) 357 : audit 0 from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-06-03T19:50:12.780 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:50:12 smithi089 bash[78452]: cephadm 1685821811.674029 mgr.y (mgr.34104) 358 : cephadm 1 Upgrade: unsafe to stop osd(s) at this time (1 PGs are or would become offline) 2023-06-03T19:50:12.780 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:50:12 smithi089 bash[78452]: cluster 1685821812.2841856 mon.a (mon.0) 326 : cluster 3 Health check update: Degraded data redundancy: 15759/61554 objects degraded (25.602%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-06-03T19:50:12.781 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:50:12 smithi089 bash[75068]: audit 1685821811.6693015 mon.c (mon.2) 161 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-06-03T19:50:12.781 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:50:12 smithi089 bash[75068]: audit 1685821811.6728628 mgr.y (mgr.34104) 357 : audit 0 from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-06-03T19:50:12.781 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:50:12 smithi089 bash[75068]: cephadm 1685821811.674029 mgr.y (mgr.34104) 358 : cephadm 1 Upgrade: unsafe to stop osd(s) at this time (1 PGs are or would become offline) 2023-06-03T19:50:12.781 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:50:12 smithi089 bash[75068]: cluster 1685821812.2841856 mon.a (mon.0) 326 : cluster 3 Health check update: Degraded data redundancy: 15759/61554 objects degraded (25.602%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-06-03T19:50:12.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:50:12 smithi164 bash[59462]: audit 1685821811.6693015 mon.c (mon.2) 161 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-06-03T19:50:12.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:50:12 smithi164 bash[59462]: audit 1685821811.6728628 mgr.y (mgr.34104) 357 : audit 0 from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-06-03T19:50:12.836 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:50:12 smithi164 bash[59462]: cephadm 1685821811.674029 mgr.y (mgr.34104) 358 : cephadm 1 Upgrade: unsafe to stop osd(s) at this time (1 PGs are or would become offline) 2023-06-03T19:50:12.836 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:50:12 smithi164 bash[59462]: cluster 1685821812.2841856 mon.a (mon.0) 326 : cluster 3 Health check update: Degraded data redundancy: 15759/61554 objects degraded (25.602%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-06-03T19:50:13.779 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:50:13 smithi089 bash[78452]: cluster 1685821812.1759374 mgr.y (mgr.34104) 359 : cluster 0 pgmap v288: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 1.2 GiB data, 5.4 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 535 op/s; 15759/60321 objects degraded (26.125%); 0 B/s, 17 objects/s recovering 2023-06-03T19:50:13.779 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:50:13 smithi089 bash[78452]: audit 1685821813.447851 mon.c (mon.2) 162 : audit 0 from='client.? 172.21.15.89:0/3725186179' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:50:13.780 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:50:13 smithi089 bash[75068]: cluster 1685821812.1759374 mgr.y (mgr.34104) 359 : cluster 0 pgmap v288: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 1.2 GiB data, 5.4 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 535 op/s; 15759/60321 objects degraded (26.125%); 0 B/s, 17 objects/s recovering 2023-06-03T19:50:13.780 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:50:13 smithi089 bash[75068]: audit 1685821813.447851 mon.c (mon.2) 162 : audit 0 from='client.? 172.21.15.89:0/3725186179' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:50:13.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:50:13 smithi164 bash[59462]: cluster 1685821812.1759374 mgr.y (mgr.34104) 359 : cluster 0 pgmap v288: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 1.2 GiB data, 5.4 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 535 op/s; 15759/60321 objects degraded (26.125%); 0 B/s, 17 objects/s recovering 2023-06-03T19:50:13.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:50:13 smithi164 bash[59462]: audit 1685821813.447851 mon.c (mon.2) 162 : audit 0 from='client.? 172.21.15.89:0/3725186179' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:50:15.779 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:50:15 smithi089 bash[78452]: cluster 1685821814.1765354 mgr.y (mgr.34104) 360 : cluster 0 pgmap v289: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 1.1 GiB data, 5.0 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 648 op/s; 15651/55074 objects degraded (28.418%); 0 B/s, 26 objects/s recovering 2023-06-03T19:50:15.779 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:50:15 smithi089 bash[75068]: cluster 1685821814.1765354 mgr.y (mgr.34104) 360 : cluster 0 pgmap v289: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 1.1 GiB data, 5.0 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 648 op/s; 15651/55074 objects degraded (28.418%); 0 B/s, 26 objects/s recovering 2023-06-03T19:50:15.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:50:15 smithi164 bash[59462]: cluster 1685821814.1765354 mgr.y (mgr.34104) 360 : cluster 0 pgmap v289: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 1.1 GiB data, 5.0 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 648 op/s; 15651/55074 objects degraded (28.418%); 0 B/s, 26 objects/s recovering 2023-06-03T19:50:16.518 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:50:16 smithi089 bash[75068]: audit 1685821816.020104 mon.a (mon.0) 327 : audit 0 from='client.? 172.21.15.89:0/2240155330' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:50:16.519 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 19:50:16 smithi089 bash[71163]: ::ffff:172.21.15.164 - - [03/Jun/2023:19:50:16] "GET /metrics HTTP/1.1" 200 207406 "" "Prometheus/2.33.4" 2023-06-03T19:50:16.779 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:50:16 smithi089 bash[78452]: audit 1685821816.020104 mon.a (mon.0) 327 : audit 0 from='client.? 172.21.15.89:0/2240155330' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:50:16.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:50:16 smithi164 bash[59462]: audit 1685821816.020104 mon.a (mon.0) 327 : audit 0 from='client.? 172.21.15.89:0/2240155330' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:50:17.779 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:50:17 smithi089 bash[75068]: cluster 1685821816.1772597 mgr.y (mgr.34104) 361 : cluster 0 pgmap v290: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 1.0 GiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 620 op/s; 15651/51072 objects degraded (30.645%); 0 B/s, 17 objects/s recovering 2023-06-03T19:50:17.779 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:50:17 smithi089 bash[75068]: cluster 1685821817.2874017 mon.a (mon.0) 328 : cluster 3 Health check update: Degraded data redundancy: 15651/51072 objects degraded (30.645%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-06-03T19:50:17.780 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:50:17 smithi089 bash[78452]: cluster 1685821816.1772597 mgr.y (mgr.34104) 361 : cluster 0 pgmap v290: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 1.0 GiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 620 op/s; 15651/51072 objects degraded (30.645%); 0 B/s, 17 objects/s recovering 2023-06-03T19:50:17.780 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:50:17 smithi089 bash[78452]: cluster 1685821817.2874017 mon.a (mon.0) 328 : cluster 3 Health check update: Degraded data redundancy: 15651/51072 objects degraded (30.645%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-06-03T19:50:17.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:50:17 smithi164 bash[59462]: cluster 1685821816.1772597 mgr.y (mgr.34104) 361 : cluster 0 pgmap v290: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 1.0 GiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 620 op/s; 15651/51072 objects degraded (30.645%); 0 B/s, 17 objects/s recovering 2023-06-03T19:50:17.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:50:17 smithi164 bash[59462]: cluster 1685821817.2874017 mon.a (mon.0) 328 : cluster 3 Health check update: Degraded data redundancy: 15651/51072 objects degraded (30.645%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-06-03T19:50:19.779 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:50:19 smithi089 bash[75068]: cluster 1685821818.1778166 mgr.y (mgr.34104) 362 : cluster 0 pgmap v291: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 1.0 GiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 513 op/s; 15651/51072 objects degraded (30.645%); 0 B/s, 17 objects/s recovering 2023-06-03T19:50:19.779 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:50:19 smithi089 bash[75068]: audit 1685821818.6250908 mon.a (mon.0) 329 : audit 0 from='client.? 172.21.15.89:0/186504697' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:50:19.780 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:50:19 smithi089 bash[78452]: cluster 1685821818.1778166 mgr.y (mgr.34104) 362 : cluster 0 pgmap v291: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 1.0 GiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 513 op/s; 15651/51072 objects degraded (30.645%); 0 B/s, 17 objects/s recovering 2023-06-03T19:50:19.780 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:50:19 smithi089 bash[78452]: audit 1685821818.6250908 mon.a (mon.0) 329 : audit 0 from='client.? 172.21.15.89:0/186504697' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:50:19.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:50:19 smithi164 bash[59462]: cluster 1685821818.1778166 mgr.y (mgr.34104) 362 : cluster 0 pgmap v291: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 1.0 GiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 513 op/s; 15651/51072 objects degraded (30.645%); 0 B/s, 17 objects/s recovering 2023-06-03T19:50:19.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:50:19 smithi164 bash[59462]: audit 1685821818.6250908 mon.a (mon.0) 329 : audit 0 from='client.? 172.21.15.89:0/186504697' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:50:21.780 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:50:21 smithi089 bash[75068]: cluster 1685821820.178666 mgr.y (mgr.34104) 363 : cluster 0 pgmap v292: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 893 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 739 op/s; 15541/42921 objects degraded (36.208%); 0 B/s, 27 objects/s recovering 2023-06-03T19:50:21.780 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:50:21 smithi089 bash[75068]: audit 1685821821.2448754 mon.b (mon.1) 15 : audit 0 from='client.? 172.21.15.89:0/2948784495' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:50:21.781 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:50:21 smithi089 bash[78452]: cluster 1685821820.178666 mgr.y (mgr.34104) 363 : cluster 0 pgmap v292: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 893 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 739 op/s; 15541/42921 objects degraded (36.208%); 0 B/s, 27 objects/s recovering 2023-06-03T19:50:21.781 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:50:21 smithi089 bash[78452]: audit 1685821821.2448754 mon.b (mon.1) 15 : audit 0 from='client.? 172.21.15.89:0/2948784495' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:50:21.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:50:21 smithi164 bash[59462]: cluster 1685821820.178666 mgr.y (mgr.34104) 363 : cluster 0 pgmap v292: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 893 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 739 op/s; 15541/42921 objects degraded (36.208%); 0 B/s, 27 objects/s recovering 2023-06-03T19:50:21.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:50:21 smithi164 bash[59462]: audit 1685821821.2448754 mon.b (mon.1) 15 : audit 0 from='client.? 172.21.15.89:0/2948784495' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:50:22.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:50:22 smithi164 bash[59462]: audit 1685821822.0778253 mon.c (mon.2) 163 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T19:50:22.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:50:22 smithi164 bash[59462]: cluster 1685821822.288681 mon.a (mon.0) 330 : cluster 3 Health check update: Degraded data redundancy: 15541/42921 objects degraded (36.208%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-06-03T19:50:23.029 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:50:22 smithi089 bash[78452]: audit 1685821822.0778253 mon.c (mon.2) 163 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T19:50:23.029 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:50:22 smithi089 bash[78452]: cluster 1685821822.288681 mon.a (mon.0) 330 : cluster 3 Health check update: Degraded data redundancy: 15541/42921 objects degraded (36.208%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-06-03T19:50:23.029 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:50:22 smithi089 bash[75068]: audit 1685821822.0778253 mon.c (mon.2) 163 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T19:50:23.030 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:50:22 smithi089 bash[75068]: cluster 1685821822.288681 mon.a (mon.0) 330 : cluster 3 Health check update: Degraded data redundancy: 15541/42921 objects degraded (36.208%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-06-03T19:50:23.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:50:23 smithi164 bash[59462]: cluster 1685821822.1792014 mgr.y (mgr.34104) 364 : cluster 0 pgmap v293: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 870 MiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 548 op/s; 15541/41808 objects degraded (37.172%); 0 B/s, 18 objects/s recovering 2023-06-03T19:50:24.029 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:50:23 smithi089 bash[75068]: cluster 1685821822.1792014 mgr.y (mgr.34104) 364 : cluster 0 pgmap v293: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 870 MiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 548 op/s; 15541/41808 objects degraded (37.172%); 0 B/s, 18 objects/s recovering 2023-06-03T19:50:24.029 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:50:23 smithi089 bash[78452]: cluster 1685821822.1792014 mgr.y (mgr.34104) 364 : cluster 0 pgmap v293: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 870 MiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 548 op/s; 15541/41808 objects degraded (37.172%); 0 B/s, 18 objects/s recovering 2023-06-03T19:50:24.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:50:24 smithi164 bash[59462]: audit 1685821823.823078 mon.a (mon.0) 331 : audit 0 from='client.? 172.21.15.89:0/255879736' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:50:25.029 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:50:24 smithi089 bash[78452]: audit 1685821823.823078 mon.a (mon.0) 331 : audit 0 from='client.? 172.21.15.89:0/255879736' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:50:25.029 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:50:24 smithi089 bash[75068]: audit 1685821823.823078 mon.a (mon.0) 331 : audit 0 from='client.? 172.21.15.89:0/255879736' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:50:25.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:50:25 smithi164 bash[59462]: cluster 1685821824.1799076 mgr.y (mgr.34104) 365 : cluster 0 pgmap v294: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 765 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 654 op/s; 15432/36768 objects degraded (41.971%); 0 B/s, 27 objects/s recovering 2023-06-03T19:50:26.029 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:50:25 smithi089 bash[78452]: cluster 1685821824.1799076 mgr.y (mgr.34104) 365 : cluster 0 pgmap v294: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 765 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 654 op/s; 15432/36768 objects degraded (41.971%); 0 B/s, 27 objects/s recovering 2023-06-03T19:50:26.029 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:50:25 smithi089 bash[75068]: cluster 1685821824.1799076 mgr.y (mgr.34104) 365 : cluster 0 pgmap v294: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 765 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 654 op/s; 15432/36768 objects degraded (41.971%); 0 B/s, 27 objects/s recovering 2023-06-03T19:50:26.529 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 19:50:26 smithi089 bash[71163]: ::ffff:172.21.15.164 - - [03/Jun/2023:19:50:26] "GET /metrics HTTP/1.1" 200 207403 "" "Prometheus/2.33.4" 2023-06-03T19:50:26.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:50:26 smithi164 bash[59462]: audit 1685821826.3875415 mon.c (mon.2) 164 : audit 0 from='client.? 172.21.15.89:0/3397671589' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:50:27.029 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:50:26 smithi089 bash[75068]: audit 1685821826.3875415 mon.c (mon.2) 164 : audit 0 from='client.? 172.21.15.89:0/3397671589' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:50:27.029 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:50:26 smithi089 bash[78452]: audit 1685821826.3875415 mon.c (mon.2) 164 : audit 0 from='client.? 172.21.15.89:0/3397671589' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:50:27.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:50:27 smithi164 bash[59462]: cluster 1685821826.1807358 mgr.y (mgr.34104) 366 : cluster 0 pgmap v295: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 682 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 618 op/s; 15432/32784 objects degraded (47.072%); 0 B/s, 18 objects/s recovering 2023-06-03T19:50:27.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:50:27 smithi164 bash[59462]: audit 1685821826.689403 mon.c (mon.2) 165 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-03T19:50:27.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:50:27 smithi164 bash[59462]: cluster 1685821827.2927816 mon.a (mon.0) 332 : cluster 3 Health check update: Degraded data redundancy: 15432/32784 objects degraded (47.072%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-06-03T19:50:28.029 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:50:27 smithi089 bash[75068]: cluster 1685821826.1807358 mgr.y (mgr.34104) 366 : cluster 0 pgmap v295: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 682 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 618 op/s; 15432/32784 objects degraded (47.072%); 0 B/s, 18 objects/s recovering 2023-06-03T19:50:28.029 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:50:27 smithi089 bash[75068]: audit 1685821826.689403 mon.c (mon.2) 165 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-03T19:50:28.029 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:50:27 smithi089 bash[75068]: cluster 1685821827.2927816 mon.a (mon.0) 332 : cluster 3 Health check update: Degraded data redundancy: 15432/32784 objects degraded (47.072%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-06-03T19:50:28.030 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:50:27 smithi089 bash[78452]: cluster 1685821826.1807358 mgr.y (mgr.34104) 366 : cluster 0 pgmap v295: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 682 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 618 op/s; 15432/32784 objects degraded (47.072%); 0 B/s, 18 objects/s recovering 2023-06-03T19:50:28.030 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:50:27 smithi089 bash[78452]: audit 1685821826.689403 mon.c (mon.2) 165 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-03T19:50:28.030 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:50:27 smithi089 bash[78452]: cluster 1685821827.2927816 mon.a (mon.0) 332 : cluster 3 Health check update: Degraded data redundancy: 15432/32784 objects degraded (47.072%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-06-03T19:50:29.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:50:29 smithi164 bash[59462]: cluster 1685821828.1815128 mgr.y (mgr.34104) 367 : cluster 0 pgmap v296: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 682 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 507 op/s; 15432/32784 objects degraded (47.072%); 0 B/s, 18 objects/s recovering 2023-06-03T19:50:29.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:50:29 smithi164 bash[59462]: audit 1685821828.9956179 mon.a (mon.0) 333 : audit 0 from='client.? 172.21.15.89:0/646444536' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:50:30.029 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:50:29 smithi089 bash[78452]: cluster 1685821828.1815128 mgr.y (mgr.34104) 367 : cluster 0 pgmap v296: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 682 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 507 op/s; 15432/32784 objects degraded (47.072%); 0 B/s, 18 objects/s recovering 2023-06-03T19:50:30.029 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:50:29 smithi089 bash[78452]: audit 1685821828.9956179 mon.a (mon.0) 333 : audit 0 from='client.? 172.21.15.89:0/646444536' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:50:30.029 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:50:29 smithi089 bash[75068]: cluster 1685821828.1815128 mgr.y (mgr.34104) 367 : cluster 0 pgmap v296: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 682 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 507 op/s; 15432/32784 objects degraded (47.072%); 0 B/s, 18 objects/s recovering 2023-06-03T19:50:30.030 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:50:29 smithi089 bash[75068]: audit 1685821828.9956179 mon.a (mon.0) 333 : audit 0 from='client.? 172.21.15.89:0/646444536' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:50:31.890 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:50:31 smithi164 bash[59462]: cluster 1685821830.1823256 mgr.y (mgr.34104) 368 : cluster 0 pgmap v297: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 516 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 729 op/s; 15325/24804 objects degraded (61.784%); 0 B/s, 27 objects/s recovering 2023-06-03T19:50:32.029 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:50:31 smithi089 bash[75068]: cluster 1685821830.1823256 mgr.y (mgr.34104) 368 : cluster 0 pgmap v297: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 516 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 729 op/s; 15325/24804 objects degraded (61.784%); 0 B/s, 27 objects/s recovering 2023-06-03T19:50:32.029 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:50:31 smithi089 bash[78452]: cluster 1685821830.1823256 mgr.y (mgr.34104) 368 : cluster 0 pgmap v297: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 516 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 729 op/s; 15325/24804 objects degraded (61.784%); 0 B/s, 27 objects/s recovering 2023-06-03T19:50:32.985 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:50:32 smithi164 bash[59462]: audit 1685821831.5952663 mon.c (mon.2) 166 : audit 0 from='client.? 172.21.15.89:0/3879097668' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:50:32.985 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:50:32 smithi164 bash[59462]: cluster 1685821832.2940428 mon.a (mon.0) 334 : cluster 3 Health check update: Degraded data redundancy: 15325/24804 objects degraded (61.784%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-06-03T19:50:33.029 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:50:32 smithi089 bash[75068]: audit 1685821831.5952663 mon.c (mon.2) 166 : audit 0 from='client.? 172.21.15.89:0/3879097668' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:50:33.029 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:50:32 smithi089 bash[75068]: cluster 1685821832.2940428 mon.a (mon.0) 334 : cluster 3 Health check update: Degraded data redundancy: 15325/24804 objects degraded (61.784%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-06-03T19:50:33.030 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:50:32 smithi089 bash[78452]: audit 1685821831.5952663 mon.c (mon.2) 166 : audit 0 from='client.? 172.21.15.89:0/3879097668' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:50:33.030 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:50:32 smithi089 bash[78452]: cluster 1685821832.2940428 mon.a (mon.0) 334 : cluster 3 Health check update: Degraded data redundancy: 15325/24804 objects degraded (61.784%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-06-03T19:50:34.029 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:50:33 smithi089 bash[75068]: cluster 1685821832.1828885 mgr.y (mgr.34104) 369 : cluster 0 pgmap v298: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 492 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 534 op/s; 15325/23667 objects degraded (64.753%); 0 B/s, 17 objects/s recovering 2023-06-03T19:50:34.030 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:50:33 smithi089 bash[75068]: audit 1685821833.206801 mon.a (mon.0) 335 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:50:34.030 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:50:33 smithi089 bash[75068]: audit 1685821833.239905 mon.a (mon.0) 336 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:50:34.030 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:50:33 smithi089 bash[75068]: audit 1685821833.2492428 mon.a (mon.0) 337 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:50:34.030 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:50:33 smithi089 bash[75068]: audit 1685821833.2662604 mon.a (mon.0) 338 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:50:34.031 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:50:33 smithi089 bash[78452]: cluster 1685821832.1828885 mgr.y (mgr.34104) 369 : cluster 0 pgmap v298: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 492 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 534 op/s; 15325/23667 objects degraded (64.753%); 0 B/s, 17 objects/s recovering 2023-06-03T19:50:34.031 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:50:33 smithi089 bash[78452]: audit 1685821833.206801 mon.a (mon.0) 335 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:50:34.031 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:50:33 smithi089 bash[78452]: audit 1685821833.239905 mon.a (mon.0) 336 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:50:34.032 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:50:33 smithi089 bash[78452]: audit 1685821833.2492428 mon.a (mon.0) 337 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:50:34.032 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:50:33 smithi089 bash[78452]: audit 1685821833.2662604 mon.a (mon.0) 338 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:50:34.085 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:50:33 smithi164 bash[59462]: cluster 1685821832.1828885 mgr.y (mgr.34104) 369 : cluster 0 pgmap v298: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 492 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 534 op/s; 15325/23667 objects degraded (64.753%); 0 B/s, 17 objects/s recovering 2023-06-03T19:50:34.085 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:50:33 smithi164 bash[59462]: audit 1685821833.206801 mon.a (mon.0) 335 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:50:34.085 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:50:33 smithi164 bash[59462]: audit 1685821833.239905 mon.a (mon.0) 336 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:50:34.086 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:50:33 smithi164 bash[59462]: audit 1685821833.2492428 mon.a (mon.0) 337 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:50:34.086 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:50:33 smithi164 bash[59462]: audit 1685821833.2662604 mon.a (mon.0) 338 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:50:34.529 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 19:50:34 smithi089 bash[71163]: debug 2023-06-03T19:50:34.075+0000 7fe48642e700 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (1 PGs are or would become offline) 2023-06-03T19:50:35.029 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:50:34 smithi089 bash[75068]: audit 1685821833.958343 mon.c (mon.2) 167 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-03T19:50:35.030 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:50:34 smithi089 bash[75068]: audit 1685821833.962842 mon.c (mon.2) 168 : audit 1 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-03T19:50:35.030 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:50:34 smithi089 bash[75068]: audit 1685821833.9745948 mon.a (mon.0) 339 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:50:35.030 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:50:34 smithi089 bash[75068]: audit 1685821834.0604033 mon.c (mon.2) 169 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-03T19:50:35.030 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:50:34 smithi089 bash[75068]: audit 1685821834.0684626 mon.c (mon.2) 170 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-06-03T19:50:35.030 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:50:34 smithi089 bash[75068]: audit 1685821834.0723047 mon.c (mon.2) 171 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-06-03T19:50:35.030 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:50:34 smithi089 bash[75068]: audit 1685821834.0745637 mon.c (mon.2) 172 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-06-03T19:50:35.031 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:50:34 smithi089 bash[75068]: audit 1685821834.079087 mon.c (mon.2) 173 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-06-03T19:50:35.031 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:50:34 smithi089 bash[75068]: audit 1685821834.2028196 mon.c (mon.2) 174 : audit 0 from='client.? 172.21.15.89:0/289979725' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:50:35.031 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:50:34 smithi089 bash[78452]: audit 1685821833.958343 mon.c (mon.2) 167 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-03T19:50:35.031 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:50:34 smithi089 bash[78452]: audit 1685821833.962842 mon.c (mon.2) 168 : audit 1 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-03T19:50:35.031 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:50:34 smithi089 bash[78452]: audit 1685821833.9745948 mon.a (mon.0) 339 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:50:35.031 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:50:34 smithi089 bash[78452]: audit 1685821834.0604033 mon.c (mon.2) 169 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-03T19:50:35.031 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:50:34 smithi089 bash[78452]: audit 1685821834.0684626 mon.c (mon.2) 170 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-06-03T19:50:35.032 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:50:34 smithi089 bash[78452]: audit 1685821834.0723047 mon.c (mon.2) 171 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-06-03T19:50:35.032 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:50:34 smithi089 bash[78452]: audit 1685821834.0745637 mon.c (mon.2) 172 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-06-03T19:50:35.032 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:50:34 smithi089 bash[78452]: audit 1685821834.079087 mon.c (mon.2) 173 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-06-03T19:50:35.032 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:50:34 smithi089 bash[78452]: audit 1685821834.2028196 mon.c (mon.2) 174 : audit 0 from='client.? 172.21.15.89:0/289979725' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:50:35.085 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:50:34 smithi164 bash[59462]: audit 1685821833.958343 mon.c (mon.2) 167 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-03T19:50:35.085 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:50:34 smithi164 bash[59462]: audit 1685821833.962842 mon.c (mon.2) 168 : audit 1 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-03T19:50:35.085 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:50:34 smithi164 bash[59462]: audit 1685821833.9745948 mon.a (mon.0) 339 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:50:35.085 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:50:34 smithi164 bash[59462]: audit 1685821834.0604033 mon.c (mon.2) 169 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-03T19:50:35.085 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:50:34 smithi164 bash[59462]: audit 1685821834.0684626 mon.c (mon.2) 170 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-06-03T19:50:35.086 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:50:34 smithi164 bash[59462]: audit 1685821834.0723047 mon.c (mon.2) 171 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-06-03T19:50:35.086 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:50:34 smithi164 bash[59462]: audit 1685821834.0745637 mon.c (mon.2) 172 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-06-03T19:50:35.086 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:50:34 smithi164 bash[59462]: audit 1685821834.079087 mon.c (mon.2) 173 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-06-03T19:50:35.086 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:50:34 smithi164 bash[59462]: audit 1685821834.2028196 mon.c (mon.2) 174 : audit 0 from='client.? 172.21.15.89:0/289979725' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:50:36.029 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:50:35 smithi089 bash[78452]: audit 1685821834.079678 mgr.y (mgr.34104) 370 : audit 0 from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-06-03T19:50:36.029 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:50:35 smithi089 bash[78452]: cephadm 1685821834.0820465 mgr.y (mgr.34104) 371 : cephadm 1 Upgrade: unsafe to stop osd(s) at this time (1 PGs are or would become offline) 2023-06-03T19:50:36.029 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:50:35 smithi089 bash[78452]: cluster 1685821834.1836395 mgr.y (mgr.34104) 372 : cluster 0 pgmap v299: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 387 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 643 op/s; 15215/18645 objects degraded (81.604%); 0 B/s, 27 objects/s recovering 2023-06-03T19:50:36.030 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:50:35 smithi089 bash[75068]: audit 1685821834.079678 mgr.y (mgr.34104) 370 : audit 0 from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-06-03T19:50:36.030 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:50:35 smithi089 bash[75068]: cephadm 1685821834.0820465 mgr.y (mgr.34104) 371 : cephadm 1 Upgrade: unsafe to stop osd(s) at this time (1 PGs are or would become offline) 2023-06-03T19:50:36.030 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:50:35 smithi089 bash[75068]: cluster 1685821834.1836395 mgr.y (mgr.34104) 372 : cluster 0 pgmap v299: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 387 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 643 op/s; 15215/18645 objects degraded (81.604%); 0 B/s, 27 objects/s recovering 2023-06-03T19:50:36.085 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:50:35 smithi164 bash[59462]: audit 1685821834.079678 mgr.y (mgr.34104) 370 : audit 0 from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-06-03T19:50:36.085 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:50:35 smithi164 bash[59462]: cephadm 1685821834.0820465 mgr.y (mgr.34104) 371 : cephadm 1 Upgrade: unsafe to stop osd(s) at this time (1 PGs are or would become offline) 2023-06-03T19:50:36.085 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:50:35 smithi164 bash[59462]: cluster 1685821834.1836395 mgr.y (mgr.34104) 372 : cluster 0 pgmap v299: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 387 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 643 op/s; 15215/18645 objects degraded (81.604%); 0 B/s, 27 objects/s recovering 2023-06-03T19:50:36.529 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 19:50:36 smithi089 bash[71163]: ::ffff:172.21.15.164 - - [03/Jun/2023:19:50:36] "GET /metrics HTTP/1.1" 200 207403 "" "Prometheus/2.33.4" 2023-06-03T19:50:38.029 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:50:37 smithi089 bash[75068]: cluster 1685821836.1842818 mgr.y (mgr.34104) 373 : cluster 0 pgmap v300: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 311 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 605 op/s; 15215/14970 objects degraded (101.637%); 0 B/s, 18 objects/s recovering 2023-06-03T19:50:38.029 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:50:37 smithi089 bash[75068]: audit 1685821836.7783566 mon.a (mon.0) 340 : audit 0 from='client.? 172.21.15.89:0/463104179' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:50:38.029 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:50:37 smithi089 bash[75068]: audit 1685821837.078554 mon.c (mon.2) 175 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T19:50:38.030 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:50:37 smithi089 bash[75068]: cluster 1685821837.2962792 mon.a (mon.0) 341 : cluster 3 Health check update: Degraded data redundancy: 15215/14970 objects degraded (101.637%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-06-03T19:50:38.030 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:50:37 smithi089 bash[78452]: cluster 1685821836.1842818 mgr.y (mgr.34104) 373 : cluster 0 pgmap v300: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 311 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 605 op/s; 15215/14970 objects degraded (101.637%); 0 B/s, 18 objects/s recovering 2023-06-03T19:50:38.030 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:50:37 smithi089 bash[78452]: audit 1685821836.7783566 mon.a (mon.0) 340 : audit 0 from='client.? 172.21.15.89:0/463104179' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:50:38.030 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:50:37 smithi089 bash[78452]: audit 1685821837.078554 mon.c (mon.2) 175 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T19:50:38.030 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:50:37 smithi089 bash[78452]: cluster 1685821837.2962792 mon.a (mon.0) 341 : cluster 3 Health check update: Degraded data redundancy: 15215/14970 objects degraded (101.637%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-06-03T19:50:38.085 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:50:37 smithi164 bash[59462]: cluster 1685821836.1842818 mgr.y (mgr.34104) 373 : cluster 0 pgmap v300: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 311 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 605 op/s; 15215/14970 objects degraded (101.637%); 0 B/s, 18 objects/s recovering 2023-06-03T19:50:38.085 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:50:37 smithi164 bash[59462]: audit 1685821836.7783566 mon.a (mon.0) 340 : audit 0 from='client.? 172.21.15.89:0/463104179' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:50:38.085 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:50:37 smithi164 bash[59462]: audit 1685821837.078554 mon.c (mon.2) 175 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T19:50:38.086 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:50:37 smithi164 bash[59462]: cluster 1685821837.2962792 mon.a (mon.0) 341 : cluster 3 Health check update: Degraded data redundancy: 15215/14970 objects degraded (101.637%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-06-03T19:50:40.029 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:50:39 smithi089 bash[75068]: cluster 1685821838.18483 mgr.y (mgr.34104) 374 : cluster 0 pgmap v301: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 311 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 494 op/s; 15215/14970 objects degraded (101.637%); 0 B/s, 18 objects/s recovering 2023-06-03T19:50:40.030 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:50:39 smithi089 bash[75068]: audit 1685821839.3393757 mon.a (mon.0) 342 : audit 0 from='client.? 172.21.15.89:0/3405039499' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:50:40.030 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:50:39 smithi089 bash[78452]: cluster 1685821838.18483 mgr.y (mgr.34104) 374 : cluster 0 pgmap v301: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 311 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 494 op/s; 15215/14970 objects degraded (101.637%); 0 B/s, 18 objects/s recovering 2023-06-03T19:50:40.031 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:50:39 smithi089 bash[78452]: audit 1685821839.3393757 mon.a (mon.0) 342 : audit 0 from='client.? 172.21.15.89:0/3405039499' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:50:40.085 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:50:39 smithi164 bash[59462]: cluster 1685821838.18483 mgr.y (mgr.34104) 374 : cluster 0 pgmap v301: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 311 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 494 op/s; 15215/14970 objects degraded (101.637%); 0 B/s, 18 objects/s recovering 2023-06-03T19:50:40.085 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:50:39 smithi164 bash[59462]: audit 1685821839.3393757 mon.a (mon.0) 342 : audit 0 from='client.? 172.21.15.89:0/3405039499' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:50:42.006 INFO:tasks.radosbench.radosbench.0.smithi089.stdout:Removed 96664 objects 2023-06-03T19:50:42.006 INFO:tasks.radosbench.radosbench.0.smithi089.stdout:Clean up completed and total clean up time :152.446 2023-06-03T19:50:42.029 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:50:41 smithi089 bash[78452]: cluster 1685821840.1856847 mgr.y (mgr.34104) 375 : cluster 0 pgmap v302: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 141 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 720 op/s; 15106/6828 objects degraded (221.236%); 0 B/s, 27 objects/s recovering 2023-06-03T19:50:42.029 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:50:41 smithi089 bash[75068]: cluster 1685821840.1856847 mgr.y (mgr.34104) 375 : cluster 0 pgmap v302: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 141 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 720 op/s; 15106/6828 objects degraded (221.236%); 0 B/s, 27 objects/s recovering 2023-06-03T19:50:42.085 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:50:41 smithi164 bash[59462]: cluster 1685821840.1856847 mgr.y (mgr.34104) 375 : cluster 0 pgmap v302: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 141 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 720 op/s; 15106/6828 objects degraded (221.236%); 0 B/s, 27 objects/s recovering 2023-06-03T19:50:42.558 INFO:tasks.cephadm.ceph_manager.ceph:removing pool_name unique_pool_2 2023-06-03T19:50:42.558 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph osd pool rm unique_pool_2 unique_pool_2 --yes-i-really-really-mean-it 2023-06-03T19:50:43.029 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:50:42 smithi089 bash[75068]: audit 1685821841.9098253 mon.b (mon.1) 16 : audit 0 from='client.? 172.21.15.89:0/372775665' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:50:43.030 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:50:42 smithi089 bash[75068]: cluster 1685821842.2974074 mon.a (mon.0) 343 : cluster 3 Health check update: Degraded data redundancy: 15106/6828 objects degraded (221.236%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-06-03T19:50:43.030 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:50:42 smithi089 bash[78452]: audit 1685821841.9098253 mon.b (mon.1) 16 : audit 0 from='client.? 172.21.15.89:0/372775665' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:50:43.030 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:50:42 smithi089 bash[78452]: cluster 1685821842.2974074 mon.a (mon.0) 343 : cluster 3 Health check update: Degraded data redundancy: 15106/6828 objects degraded (221.236%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-06-03T19:50:43.085 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:50:42 smithi164 bash[59462]: audit 1685821841.9098253 mon.b (mon.1) 16 : audit 0 from='client.? 172.21.15.89:0/372775665' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:50:43.085 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:50:42 smithi164 bash[59462]: cluster 1685821842.2974074 mon.a (mon.0) 343 : cluster 3 Health check update: Degraded data redundancy: 15106/6828 objects degraded (221.236%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-06-03T19:50:44.029 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:50:43 smithi089 bash[75068]: cluster 1685821842.186648 mgr.y (mgr.34104) 376 : cluster 0 pgmap v303: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 118 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 530 op/s; 15106/5697 objects degraded (265.157%); 0 B/s, 18 objects/s recovering 2023-06-03T19:50:44.029 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:50:43 smithi089 bash[78452]: cluster 1685821842.186648 mgr.y (mgr.34104) 376 : cluster 0 pgmap v303: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 118 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 530 op/s; 15106/5697 objects degraded (265.157%); 0 B/s, 18 objects/s recovering 2023-06-03T19:50:44.085 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:50:43 smithi164 bash[59462]: cluster 1685821842.186648 mgr.y (mgr.34104) 376 : cluster 0 pgmap v303: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 118 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 530 op/s; 15106/5697 objects degraded (265.157%); 0 B/s, 18 objects/s recovering 2023-06-03T19:50:45.029 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:50:44 smithi089 bash[78452]: audit 1685821844.2896159 mon.a (mon.0) 344 : audit 0 from='client.? 172.21.15.89:0/3970469988' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:50:45.029 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:50:44 smithi089 bash[75068]: audit 1685821844.2896159 mon.a (mon.0) 344 : audit 0 from='client.? 172.21.15.89:0/3970469988' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:50:45.085 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:50:44 smithi164 bash[59462]: audit 1685821844.2896159 mon.a (mon.0) 344 : audit 0 from='client.? 172.21.15.89:0/3970469988' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:50:46.029 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:50:45 smithi089 bash[78452]: cluster 1685821844.1879466 mgr.y (mgr.34104) 377 : cluster 0 pgmap v304: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 38 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 605 op/s; 14994/1857 objects degraded (807.431%); 0 B/s, 27 objects/s recovering 2023-06-03T19:50:46.029 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:50:45 smithi089 bash[75068]: cluster 1685821844.1879466 mgr.y (mgr.34104) 377 : cluster 0 pgmap v304: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 38 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 605 op/s; 14994/1857 objects degraded (807.431%); 0 B/s, 27 objects/s recovering 2023-06-03T19:50:46.085 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:50:45 smithi164 bash[59462]: cluster 1685821844.1879466 mgr.y (mgr.34104) 377 : cluster 0 pgmap v304: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 38 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 605 op/s; 14994/1857 objects degraded (807.431%); 0 B/s, 27 objects/s recovering 2023-06-03T19:50:46.529 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 19:50:46 smithi089 bash[71163]: ::ffff:172.21.15.164 - - [03/Jun/2023:19:50:46] "GET /metrics HTTP/1.1" 200 207426 "" "Prometheus/2.33.4" 2023-06-03T19:50:47.029 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:50:46 smithi089 bash[75068]: cluster 1685821845.6308506 osd.6 (osd.6) 13 : cluster 3 failed to encode map e93 with expected crc 2023-06-03T19:50:47.029 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:50:46 smithi089 bash[75068]: cluster 1685821845.6312516 osd.5 (osd.5) 13 : cluster 3 failed to encode map e93 with expected crc 2023-06-03T19:50:47.029 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:50:46 smithi089 bash[75068]: cluster 1685821845.633851 osd.6 (osd.6) 14 : cluster 3 failed to encode map e93 with expected crc 2023-06-03T19:50:47.030 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:50:46 smithi089 bash[75068]: cluster 1685821845.6361332 mon.a (mon.0) 345 : cluster 0 osdmap e93: 8 total, 8 up, 8 in 2023-06-03T19:50:47.030 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:50:46 smithi089 bash[75068]: cluster 1685821845.6864696 osd.4 (osd.4) 16 : cluster 3 failed to encode map e93 with expected crc 2023-06-03T19:50:47.030 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:50:46 smithi089 bash[78452]: cluster 1685821845.6308506 osd.6 (osd.6) 13 : cluster 3 failed to encode map e93 with expected crc 2023-06-03T19:50:47.030 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:50:46 smithi089 bash[78452]: cluster 1685821845.6312516 osd.5 (osd.5) 13 : cluster 3 failed to encode map e93 with expected crc 2023-06-03T19:50:47.030 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:50:46 smithi089 bash[78452]: cluster 1685821845.633851 osd.6 (osd.6) 14 : cluster 3 failed to encode map e93 with expected crc 2023-06-03T19:50:47.030 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:50:46 smithi089 bash[78452]: cluster 1685821845.6361332 mon.a (mon.0) 345 : cluster 0 osdmap e93: 8 total, 8 up, 8 in 2023-06-03T19:50:47.031 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:50:46 smithi089 bash[78452]: cluster 1685821845.6864696 osd.4 (osd.4) 16 : cluster 3 failed to encode map e93 with expected crc 2023-06-03T19:50:47.085 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:50:46 smithi164 bash[59462]: cluster 1685821845.6308506 osd.6 (osd.6) 13 : cluster 3 failed to encode map e93 with expected crc 2023-06-03T19:50:47.085 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:50:46 smithi164 bash[59462]: cluster 1685821845.6312516 osd.5 (osd.5) 13 : cluster 3 failed to encode map e93 with expected crc 2023-06-03T19:50:47.085 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:50:46 smithi164 bash[59462]: cluster 1685821845.633851 osd.6 (osd.6) 14 : cluster 3 failed to encode map e93 with expected crc 2023-06-03T19:50:47.085 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:50:46 smithi164 bash[59462]: cluster 1685821845.6361332 mon.a (mon.0) 345 : cluster 0 osdmap e93: 8 total, 8 up, 8 in 2023-06-03T19:50:47.086 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:50:46 smithi164 bash[59462]: cluster 1685821845.6864696 osd.4 (osd.4) 16 : cluster 3 failed to encode map e93 with expected crc 2023-06-03T19:50:47.374 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/mon.a/config 2023-06-03T19:50:47.779 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:50:47 smithi089 bash[78452]: cluster 1685821846.1890507 mgr.y (mgr.34104) 378 : cluster 0 pgmap v306: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 582 KiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 0 B/s wr, 496 op/s; 14994/72 objects degraded (20825.000%); 0 B/s, 22 objects/s recovering 2023-06-03T19:50:47.779 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:50:47 smithi089 bash[78452]: cluster 1685821846.645062 osd.4 (osd.4) 17 : cluster 3 failed to encode map e94 with expected crc 2023-06-03T19:50:47.780 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:50:47 smithi089 bash[78452]: cluster 1685821846.6486945 osd.6 (osd.6) 15 : cluster 3 failed to encode map e94 with expected crc 2023-06-03T19:50:47.780 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:50:47 smithi089 bash[78452]: cluster 1685821846.6491523 osd.2 (osd.2) 18 : cluster 3 failed to encode map e94 with expected crc 2023-06-03T19:50:47.780 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:50:47 smithi089 bash[78452]: cluster 1685821846.651346 mon.a (mon.0) 346 : cluster 0 osdmap e94: 8 total, 8 up, 8 in 2023-06-03T19:50:47.781 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:50:47 smithi089 bash[78452]: audit 1685821846.6913571 mon.c (mon.2) 176 : audit 0 from='client.? 172.21.15.89:0/2259499633' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:50:47.781 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:50:47 smithi089 bash[78452]: cluster 1685821847.2991395 mon.a (mon.0) 347 : cluster 3 Health check update: Degraded data redundancy: 14994/72 objects degraded (20825.000%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-06-03T19:50:47.781 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:50:47 smithi089 bash[75068]: cluster 1685821846.1890507 mgr.y (mgr.34104) 378 : cluster 0 pgmap v306: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 582 KiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 0 B/s wr, 496 op/s; 14994/72 objects degraded (20825.000%); 0 B/s, 22 objects/s recovering 2023-06-03T19:50:47.782 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:50:47 smithi089 bash[75068]: cluster 1685821846.645062 osd.4 (osd.4) 17 : cluster 3 failed to encode map e94 with expected crc 2023-06-03T19:50:47.782 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:50:47 smithi089 bash[75068]: cluster 1685821846.6486945 osd.6 (osd.6) 15 : cluster 3 failed to encode map e94 with expected crc 2023-06-03T19:50:47.782 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:50:47 smithi089 bash[75068]: cluster 1685821846.6491523 osd.2 (osd.2) 18 : cluster 3 failed to encode map e94 with expected crc 2023-06-03T19:50:47.782 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:50:47 smithi089 bash[75068]: cluster 1685821846.651346 mon.a (mon.0) 346 : cluster 0 osdmap e94: 8 total, 8 up, 8 in 2023-06-03T19:50:47.783 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:50:47 smithi089 bash[75068]: audit 1685821846.6913571 mon.c (mon.2) 176 : audit 0 from='client.? 172.21.15.89:0/2259499633' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:50:47.783 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:50:47 smithi089 bash[75068]: cluster 1685821847.2991395 mon.a (mon.0) 347 : cluster 3 Health check update: Degraded data redundancy: 14994/72 objects degraded (20825.000%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-06-03T19:50:48.085 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:50:47 smithi164 bash[59462]: cluster 1685821846.1890507 mgr.y (mgr.34104) 378 : cluster 0 pgmap v306: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 582 KiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 0 B/s wr, 496 op/s; 14994/72 objects degraded (20825.000%); 0 B/s, 22 objects/s recovering 2023-06-03T19:50:48.085 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:50:47 smithi164 bash[59462]: cluster 1685821846.645062 osd.4 (osd.4) 17 : cluster 3 failed to encode map e94 with expected crc 2023-06-03T19:50:48.085 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:50:47 smithi164 bash[59462]: cluster 1685821846.6486945 osd.6 (osd.6) 15 : cluster 3 failed to encode map e94 with expected crc 2023-06-03T19:50:48.085 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:50:47 smithi164 bash[59462]: cluster 1685821846.6491523 osd.2 (osd.2) 18 : cluster 3 failed to encode map e94 with expected crc 2023-06-03T19:50:48.086 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:50:47 smithi164 bash[59462]: cluster 1685821846.651346 mon.a (mon.0) 346 : cluster 0 osdmap e94: 8 total, 8 up, 8 in 2023-06-03T19:50:48.086 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:50:47 smithi164 bash[59462]: audit 1685821846.6913571 mon.c (mon.2) 176 : audit 0 from='client.? 172.21.15.89:0/2259499633' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:50:48.086 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:50:47 smithi164 bash[59462]: cluster 1685821847.2991395 mon.a (mon.0) 347 : cluster 3 Health check update: Degraded data redundancy: 14994/72 objects degraded (20825.000%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-06-03T19:50:49.529 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 19:50:49 smithi089 bash[71163]: debug 2023-06-03T19:50:49.094+0000 7fe48642e700 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (1 PGs are or would become offline) 2023-06-03T19:50:49.876 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:50:49 smithi089 bash[78452]: cluster 1685821848.1898715 mgr.y (mgr.34104) 379 : cluster 0 pgmap v308: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 582 KiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 0 B/s wr, 281 op/s; 14994/72 objects degraded (20825.000%); 0 B/s, 13 objects/s recovering 2023-06-03T19:50:49.876 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:50:49 smithi089 bash[78452]: audit 1685821849.0420272 mon.a (mon.0) 348 : audit 0 from='client.? 172.21.15.89:0/2735686585' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:50:49.877 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:50:49 smithi089 bash[78452]: audit 1685821849.097728 mon.c (mon.2) 177 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-06-03T19:50:49.877 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:50:49 smithi089 bash[75068]: cluster 1685821848.1898715 mgr.y (mgr.34104) 379 : cluster 0 pgmap v308: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 582 KiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 0 B/s wr, 281 op/s; 14994/72 objects degraded (20825.000%); 0 B/s, 13 objects/s recovering 2023-06-03T19:50:49.877 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:50:49 smithi089 bash[75068]: audit 1685821849.0420272 mon.a (mon.0) 348 : audit 0 from='client.? 172.21.15.89:0/2735686585' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:50:49.877 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:50:49 smithi089 bash[75068]: audit 1685821849.097728 mon.c (mon.2) 177 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-06-03T19:50:50.085 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:50:49 smithi164 bash[59462]: cluster 1685821848.1898715 mgr.y (mgr.34104) 379 : cluster 0 pgmap v308: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 582 KiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 0 B/s wr, 281 op/s; 14994/72 objects degraded (20825.000%); 0 B/s, 13 objects/s recovering 2023-06-03T19:50:50.085 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:50:49 smithi164 bash[59462]: audit 1685821849.0420272 mon.a (mon.0) 348 : audit 0 from='client.? 172.21.15.89:0/2735686585' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:50:50.085 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:50:49 smithi164 bash[59462]: audit 1685821849.097728 mon.c (mon.2) 177 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-06-03T19:50:51.029 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:50:50 smithi089 bash[75068]: audit 1685821849.0980332 mgr.y (mgr.34104) 380 : audit 0 from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-06-03T19:50:51.029 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:50:50 smithi089 bash[75068]: cephadm 1685821849.099391 mgr.y (mgr.34104) 381 : cephadm 1 Upgrade: unsafe to stop osd(s) at this time (1 PGs are or would become offline) 2023-06-03T19:50:51.029 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:50:50 smithi089 bash[78452]: audit 1685821849.0980332 mgr.y (mgr.34104) 380 : audit 0 from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-06-03T19:50:51.030 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:50:50 smithi089 bash[78452]: cephadm 1685821849.099391 mgr.y (mgr.34104) 381 : cephadm 1 Upgrade: unsafe to stop osd(s) at this time (1 PGs are or would become offline) 2023-06-03T19:50:51.085 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:50:50 smithi164 bash[59462]: audit 1685821849.0980332 mgr.y (mgr.34104) 380 : audit 0 from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-06-03T19:50:51.085 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:50:50 smithi164 bash[59462]: cephadm 1685821849.099391 mgr.y (mgr.34104) 381 : cephadm 1 Upgrade: unsafe to stop osd(s) at this time (1 PGs are or would become offline) 2023-06-03T19:50:51.960 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:50:51 smithi089 bash[75068]: cluster 1685821850.1905317 mgr.y (mgr.34104) 382 : cluster 0 pgmap v309: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 582 KiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 0 B/s wr, 234 op/s; 14854/72 objects degraded (20630.556%); 0 B/s, 31 objects/s recovering 2023-06-03T19:50:51.961 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:50:51 smithi089 bash[75068]: audit 1685821851.3885257 mon.c (mon.2) 178 : audit 0 from='client.? 172.21.15.89:0/2752031391' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:50:51.961 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:50:51 smithi089 bash[78452]: cluster 1685821850.1905317 mgr.y (mgr.34104) 382 : cluster 0 pgmap v309: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 582 KiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 0 B/s wr, 234 op/s; 14854/72 objects degraded (20630.556%); 0 B/s, 31 objects/s recovering 2023-06-03T19:50:51.961 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:50:51 smithi089 bash[78452]: audit 1685821851.3885257 mon.c (mon.2) 178 : audit 0 from='client.? 172.21.15.89:0/2752031391' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:50:52.085 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:50:51 smithi164 bash[59462]: cluster 1685821850.1905317 mgr.y (mgr.34104) 382 : cluster 0 pgmap v309: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 582 KiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 0 B/s wr, 234 op/s; 14854/72 objects degraded (20630.556%); 0 B/s, 31 objects/s recovering 2023-06-03T19:50:52.085 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:50:51 smithi164 bash[59462]: audit 1685821851.3885257 mon.c (mon.2) 178 : audit 0 from='client.? 172.21.15.89:0/2752031391' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:50:53.029 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:50:52 smithi089 bash[78452]: audit 1685821852.0795546 mon.c (mon.2) 179 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T19:50:53.029 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:50:52 smithi089 bash[78452]: cluster 1685821852.3007708 mon.a (mon.0) 349 : cluster 3 Health check update: Degraded data redundancy: 14854/72 objects degraded (20630.556%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-06-03T19:50:53.029 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:50:52 smithi089 bash[78452]: audit 1685821852.32766 mon.a (mon.0) 350 : audit 1 from='client.? 172.21.15.89:0/1476378566' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_2", "pool2": "unique_pool_2", "yes_i_really_really_mean_it": true}]: dispatch 2023-06-03T19:50:53.030 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:50:52 smithi089 bash[75068]: audit 1685821852.0795546 mon.c (mon.2) 179 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T19:50:53.030 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:50:52 smithi089 bash[75068]: cluster 1685821852.3007708 mon.a (mon.0) 349 : cluster 3 Health check update: Degraded data redundancy: 14854/72 objects degraded (20630.556%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-06-03T19:50:53.030 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:50:52 smithi089 bash[75068]: audit 1685821852.32766 mon.a (mon.0) 350 : audit 1 from='client.? 172.21.15.89:0/1476378566' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_2", "pool2": "unique_pool_2", "yes_i_really_really_mean_it": true}]: dispatch 2023-06-03T19:50:53.085 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:50:52 smithi164 bash[59462]: audit 1685821852.0795546 mon.c (mon.2) 179 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T19:50:53.085 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:50:52 smithi164 bash[59462]: cluster 1685821852.3007708 mon.a (mon.0) 349 : cluster 3 Health check update: Degraded data redundancy: 14854/72 objects degraded (20630.556%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-06-03T19:50:53.085 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:50:52 smithi164 bash[59462]: audit 1685821852.32766 mon.a (mon.0) 350 : audit 1 from='client.? 172.21.15.89:0/1476378566' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_2", "pool2": "unique_pool_2", "yes_i_really_really_mean_it": true}]: dispatch 2023-06-03T19:50:53.318 INFO:teuthology.orchestra.run.smithi089.stderr:pool 'unique_pool_2' removed 2023-06-03T19:50:53.779 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:50:53 smithi089 bash[75068]: cluster 1685821852.191395 mgr.y (mgr.34104) 383 : cluster 0 pgmap v310: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 582 KiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 0 B/s wr, 74 op/s; 14854/72 objects degraded (20630.556%); 0 B/s, 17 objects/s recovering 2023-06-03T19:50:53.780 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:50:53 smithi089 bash[75068]: audit 1685821853.308794 mon.a (mon.0) 351 : audit 1 from='client.? 172.21.15.89:0/1476378566' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_2", "pool2": "unique_pool_2", "yes_i_really_really_mean_it": true}]': finished 2023-06-03T19:50:53.780 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:50:53 smithi089 bash[75068]: cluster 1685821853.308864 mon.a (mon.0) 352 : cluster 0 osdmap e95: 8 total, 8 up, 8 in 2023-06-03T19:50:53.780 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:50:53 smithi089 bash[78452]: cluster 1685821852.191395 mgr.y (mgr.34104) 383 : cluster 0 pgmap v310: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 582 KiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 0 B/s wr, 74 op/s; 14854/72 objects degraded (20630.556%); 0 B/s, 17 objects/s recovering 2023-06-03T19:50:53.780 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:50:53 smithi089 bash[78452]: audit 1685821853.308794 mon.a (mon.0) 351 : audit 1 from='client.? 172.21.15.89:0/1476378566' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_2", "pool2": "unique_pool_2", "yes_i_really_really_mean_it": true}]': finished 2023-06-03T19:50:53.780 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:50:53 smithi089 bash[78452]: cluster 1685821853.308864 mon.a (mon.0) 352 : cluster 0 osdmap e95: 8 total, 8 up, 8 in 2023-06-03T19:50:54.085 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:50:53 smithi164 bash[59462]: cluster 1685821852.191395 mgr.y (mgr.34104) 383 : cluster 0 pgmap v310: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 582 KiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 0 B/s wr, 74 op/s; 14854/72 objects degraded (20630.556%); 0 B/s, 17 objects/s recovering 2023-06-03T19:50:54.086 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:50:53 smithi164 bash[59462]: audit 1685821853.308794 mon.a (mon.0) 351 : audit 1 from='client.? 172.21.15.89:0/1476378566' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_2", "pool2": "unique_pool_2", "yes_i_really_really_mean_it": true}]': finished 2023-06-03T19:50:54.086 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:50:53 smithi164 bash[59462]: cluster 1685821853.308864 mon.a (mon.0) 352 : cluster 0 osdmap e95: 8 total, 8 up, 8 in 2023-06-03T19:50:54.243 INFO:teuthology.task.full_sequential:In full_sequential, running task radosbench... 2023-06-03T19:50:54.243 INFO:tasks.radosbench:Beginning radosbench... 2023-06-03T19:50:54.244 INFO:tasks.cephadm.ceph_manager.ceph:creating pool_name unique_pool_3 2023-06-03T19:50:54.244 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph osd pool create unique_pool_3 16 2023-06-03T19:50:55.029 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:50:54 smithi089 bash[75068]: cluster 1685821853.3066766 osd.3 (osd.3) 19 : cluster 3 failed to encode map e95 with expected crc 2023-06-03T19:50:55.029 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:50:54 smithi089 bash[75068]: cluster 1685821853.3073823 osd.2 (osd.2) 19 : cluster 3 failed to encode map e95 with expected crc 2023-06-03T19:50:55.030 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:50:54 smithi089 bash[75068]: audit 1685821853.76106 mon.c (mon.2) 180 : audit 0 from='client.? 172.21.15.89:0/4124332014' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:50:55.030 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:50:54 smithi089 bash[75068]: cluster 1685821854.0054045 osd.7 (osd.7) 13 : cluster 3 failed to encode map e95 with expected crc 2023-06-03T19:50:55.030 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:50:54 smithi089 bash[75068]: cluster 1685821854.1647396 osd.1 (osd.1) 17 : cluster 3 failed to encode map e95 with expected crc 2023-06-03T19:50:55.030 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:50:54 smithi089 bash[78452]: cluster 1685821853.3066766 osd.3 (osd.3) 19 : cluster 3 failed to encode map e95 with expected crc 2023-06-03T19:50:55.030 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:50:54 smithi089 bash[78452]: cluster 1685821853.3073823 osd.2 (osd.2) 19 : cluster 3 failed to encode map e95 with expected crc 2023-06-03T19:50:55.030 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:50:54 smithi089 bash[78452]: audit 1685821853.76106 mon.c (mon.2) 180 : audit 0 from='client.? 172.21.15.89:0/4124332014' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:50:55.031 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:50:54 smithi089 bash[78452]: cluster 1685821854.0054045 osd.7 (osd.7) 13 : cluster 3 failed to encode map e95 with expected crc 2023-06-03T19:50:55.031 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:50:54 smithi089 bash[78452]: cluster 1685821854.1647396 osd.1 (osd.1) 17 : cluster 3 failed to encode map e95 with expected crc 2023-06-03T19:50:55.085 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:50:54 smithi164 bash[59462]: cluster 1685821853.3066766 osd.3 (osd.3) 19 : cluster 3 failed to encode map e95 with expected crc 2023-06-03T19:50:55.085 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:50:54 smithi164 bash[59462]: cluster 1685821853.3073823 osd.2 (osd.2) 19 : cluster 3 failed to encode map e95 with expected crc 2023-06-03T19:50:55.085 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:50:54 smithi164 bash[59462]: audit 1685821853.76106 mon.c (mon.2) 180 : audit 0 from='client.? 172.21.15.89:0/4124332014' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:50:55.086 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:50:54 smithi164 bash[59462]: cluster 1685821854.0054045 osd.7 (osd.7) 13 : cluster 3 failed to encode map e95 with expected crc 2023-06-03T19:50:55.086 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:50:54 smithi164 bash[59462]: cluster 1685821854.1647396 osd.1 (osd.1) 17 : cluster 3 failed to encode map e95 with expected crc 2023-06-03T19:50:56.029 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:50:55 smithi089 bash[78452]: cluster 1685821854.0979636 osd.6 (osd.6) 16 : cluster 3 failed to encode map e95 with expected crc 2023-06-03T19:50:56.029 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:50:55 smithi089 bash[78452]: cluster 1685821854.1919844 mgr.y (mgr.34104) 384 : cluster 0 pgmap v312: 73 pgs: 73 active+clean; 582 KiB data, 1.7 GiB used, 714 GiB / 715 GiB avail 2023-06-03T19:50:56.029 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:50:55 smithi089 bash[78452]: cluster 1685821854.6697762 mon.a (mon.0) 353 : cluster 1 Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 14854/72 objects degraded (20630.556%), 5 pgs degraded, 5 pgs undersized) 2023-06-03T19:50:56.029 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:50:55 smithi089 bash[78452]: cluster 1685821854.6698387 mon.a (mon.0) 354 : cluster 1 Cluster is now healthy 2023-06-03T19:50:56.030 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:50:55 smithi089 bash[75068]: cluster 1685821854.0979636 osd.6 (osd.6) 16 : cluster 3 failed to encode map e95 with expected crc 2023-06-03T19:50:56.030 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:50:55 smithi089 bash[75068]: cluster 1685821854.1919844 mgr.y (mgr.34104) 384 : cluster 0 pgmap v312: 73 pgs: 73 active+clean; 582 KiB data, 1.7 GiB used, 714 GiB / 715 GiB avail 2023-06-03T19:50:56.030 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:50:55 smithi089 bash[75068]: cluster 1685821854.6697762 mon.a (mon.0) 353 : cluster 1 Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 14854/72 objects degraded (20630.556%), 5 pgs degraded, 5 pgs undersized) 2023-06-03T19:50:56.030 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:50:55 smithi089 bash[75068]: cluster 1685821854.6698387 mon.a (mon.0) 354 : cluster 1 Cluster is now healthy 2023-06-03T19:50:56.085 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:50:55 smithi164 bash[59462]: cluster 1685821854.0979636 osd.6 (osd.6) 16 : cluster 3 failed to encode map e95 with expected crc 2023-06-03T19:50:56.085 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:50:55 smithi164 bash[59462]: cluster 1685821854.1919844 mgr.y (mgr.34104) 384 : cluster 0 pgmap v312: 73 pgs: 73 active+clean; 582 KiB data, 1.7 GiB used, 714 GiB / 715 GiB avail 2023-06-03T19:50:56.085 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:50:55 smithi164 bash[59462]: cluster 1685821854.6697762 mon.a (mon.0) 353 : cluster 1 Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 14854/72 objects degraded (20630.556%), 5 pgs degraded, 5 pgs undersized) 2023-06-03T19:50:56.085 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:50:55 smithi164 bash[59462]: cluster 1685821854.6698387 mon.a (mon.0) 354 : cluster 1 Cluster is now healthy 2023-06-03T19:50:56.529 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 19:50:56 smithi089 bash[71163]: ::ffff:172.21.15.164 - - [03/Jun/2023:19:50:56] "GET /metrics HTTP/1.1" 200 207348 "" "Prometheus/2.33.4" 2023-06-03T19:50:57.029 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:50:56 smithi089 bash[75068]: audit 1685821856.118658 mon.a (mon.0) 355 : audit 0 from='client.? 172.21.15.89:0/52646313' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:50:57.029 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:50:56 smithi089 bash[78452]: audit 1685821856.118658 mon.a (mon.0) 355 : audit 0 from='client.? 172.21.15.89:0/52646313' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:50:57.085 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:50:56 smithi164 bash[59462]: audit 1685821856.118658 mon.a (mon.0) 355 : audit 0 from='client.? 172.21.15.89:0/52646313' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:50:58.029 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:50:57 smithi089 bash[78452]: cluster 1685821856.1928208 mgr.y (mgr.34104) 385 : cluster 0 pgmap v313: 73 pgs: 73 active+clean; 582 KiB data, 1.7 GiB used, 714 GiB / 715 GiB avail 2023-06-03T19:50:58.029 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:50:57 smithi089 bash[75068]: cluster 1685821856.1928208 mgr.y (mgr.34104) 385 : cluster 0 pgmap v313: 73 pgs: 73 active+clean; 582 KiB data, 1.7 GiB used, 714 GiB / 715 GiB avail 2023-06-03T19:50:58.085 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:50:57 smithi164 bash[59462]: cluster 1685821856.1928208 mgr.y (mgr.34104) 385 : cluster 0 pgmap v313: 73 pgs: 73 active+clean; 582 KiB data, 1.7 GiB used, 714 GiB / 715 GiB avail 2023-06-03T19:50:59.029 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:50:58 smithi089 bash[75068]: audit 1685821858.4880645 mon.c (mon.2) 181 : audit 0 from='client.? 172.21.15.89:0/2284733562' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:50:59.029 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:50:58 smithi089 bash[78452]: audit 1685821858.4880645 mon.c (mon.2) 181 : audit 0 from='client.? 172.21.15.89:0/2284733562' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:50:59.057 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/mon.a/config 2023-06-03T19:50:59.085 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:50:58 smithi164 bash[59462]: audit 1685821858.4880645 mon.c (mon.2) 181 : audit 0 from='client.? 172.21.15.89:0/2284733562' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:51:00.029 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:50:59 smithi089 bash[75068]: cluster 1685821858.1934378 mgr.y (mgr.34104) 386 : cluster 0 pgmap v314: 73 pgs: 73 active+clean; 582 KiB data, 1.7 GiB used, 714 GiB / 715 GiB avail 2023-06-03T19:51:00.029 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:50:59 smithi089 bash[78452]: cluster 1685821858.1934378 mgr.y (mgr.34104) 386 : cluster 0 pgmap v314: 73 pgs: 73 active+clean; 582 KiB data, 1.7 GiB used, 714 GiB / 715 GiB avail 2023-06-03T19:51:00.085 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:50:59 smithi164 bash[59462]: cluster 1685821858.1934378 mgr.y (mgr.34104) 386 : cluster 0 pgmap v314: 73 pgs: 73 active+clean; 582 KiB data, 1.7 GiB used, 714 GiB / 715 GiB avail 2023-06-03T19:51:02.029 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:51:01 smithi089 bash[78452]: cluster 1685821860.1944296 mgr.y (mgr.34104) 387 : cluster 0 pgmap v315: 73 pgs: 73 active+clean; 582 KiB data, 1.7 GiB used, 713 GiB / 715 GiB avail 2023-06-03T19:51:02.029 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:51:01 smithi089 bash[78452]: audit 1685821860.8391633 mon.c (mon.2) 182 : audit 0 from='client.? 172.21.15.89:0/382025365' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:51:02.029 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:51:01 smithi089 bash[75068]: cluster 1685821860.1944296 mgr.y (mgr.34104) 387 : cluster 0 pgmap v315: 73 pgs: 73 active+clean; 582 KiB data, 1.7 GiB used, 713 GiB / 715 GiB avail 2023-06-03T19:51:02.029 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:51:01 smithi089 bash[75068]: audit 1685821860.8391633 mon.c (mon.2) 182 : audit 0 from='client.? 172.21.15.89:0/382025365' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:51:02.085 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:51:01 smithi164 bash[59462]: cluster 1685821860.1944296 mgr.y (mgr.34104) 387 : cluster 0 pgmap v315: 73 pgs: 73 active+clean; 582 KiB data, 1.7 GiB used, 713 GiB / 715 GiB avail 2023-06-03T19:51:02.085 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:51:01 smithi164 bash[59462]: audit 1685821860.8391633 mon.c (mon.2) 182 : audit 0 from='client.? 172.21.15.89:0/382025365' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:51:03.332 INFO:teuthology.orchestra.run.smithi089.stderr:pool 'unique_pool_3' created 2023-06-03T19:51:03.517 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:51:03 smithi089 bash[75068]: cluster 1685821862.194907 mgr.y (mgr.34104) 388 : cluster 0 pgmap v316: 73 pgs: 73 active+clean; 582 KiB data, 1.7 GiB used, 713 GiB / 715 GiB avail 2023-06-03T19:51:03.517 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:51:03 smithi089 bash[75068]: audit 1685821862.2264633 mon.a (mon.0) 356 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:51:03.518 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:51:03 smithi089 bash[75068]: audit 1685821862.7063596 mon.c (mon.2) 183 : audit 1 from='client.? 172.21.15.89:0/3518752543' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_3", "pg_num": 16}]: dispatch 2023-06-03T19:51:03.518 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:51:03 smithi089 bash[75068]: audit 1685821862.7071836 mon.a (mon.0) 357 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_3", "pg_num": 16}]: dispatch 2023-06-03T19:51:03.518 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:51:03 smithi089 bash[78452]: cluster 1685821862.194907 mgr.y (mgr.34104) 388 : cluster 0 pgmap v316: 73 pgs: 73 active+clean; 582 KiB data, 1.7 GiB used, 713 GiB / 715 GiB avail 2023-06-03T19:51:03.518 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:51:03 smithi089 bash[78452]: audit 1685821862.2264633 mon.a (mon.0) 356 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:51:03.518 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:51:03 smithi089 bash[78452]: audit 1685821862.7063596 mon.c (mon.2) 183 : audit 1 from='client.? 172.21.15.89:0/3518752543' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_3", "pg_num": 16}]: dispatch 2023-06-03T19:51:03.518 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:51:03 smithi089 bash[78452]: audit 1685821862.7071836 mon.a (mon.0) 357 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_3", "pg_num": 16}]: dispatch 2023-06-03T19:51:03.585 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:51:03 smithi164 bash[59462]: cluster 1685821862.194907 mgr.y (mgr.34104) 388 : cluster 0 pgmap v316: 73 pgs: 73 active+clean; 582 KiB data, 1.7 GiB used, 713 GiB / 715 GiB avail 2023-06-03T19:51:03.585 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:51:03 smithi164 bash[59462]: audit 1685821862.2264633 mon.a (mon.0) 356 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:51:03.585 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:51:03 smithi164 bash[59462]: audit 1685821862.7063596 mon.c (mon.2) 183 : audit 1 from='client.? 172.21.15.89:0/3518752543' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_3", "pg_num": 16}]: dispatch 2023-06-03T19:51:03.586 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:51:03 smithi164 bash[59462]: audit 1685821862.7071836 mon.a (mon.0) 357 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_3", "pg_num": 16}]: dispatch 2023-06-03T19:51:04.201 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph osd pool application enable unique_pool_3 rados --yes-i-really-mean-it || true 2023-06-03T19:51:04.529 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 19:51:04 smithi089 bash[71163]: debug 2023-06-03T19:51:04.110+0000 7fe48642e700 -1 mgr.server reply reply (11) Resource temporarily unavailable 16 pgs have unknown state; cannot draw any conclusions 2023-06-03T19:51:04.529 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 19:51:04 smithi089 bash[71163]: debug 2023-06-03T19:51:04.110+0000 7fe48642e700 -1 mgr.server reply reply (16) Device or resource busy 16 pgs have unknown state; cannot draw any conclusionsunsafe to stop osd(s) at this time (0 PGs are or would become offline) 2023-06-03T19:51:04.530 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:51:04 smithi089 bash[78452]: cluster 1685821863.3083317 osd.4 (osd.4) 18 : cluster 3 failed to encode map e96 with expected crc 2023-06-03T19:51:04.530 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:51:04 smithi089 bash[78452]: cluster 1685821863.3090408 osd.5 (osd.5) 14 : cluster 3 failed to encode map e96 with expected crc 2023-06-03T19:51:04.530 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:51:04 smithi089 bash[78452]: audit 1685821863.3143368 mon.a (mon.0) 358 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_3", "pg_num": 16}]': finished 2023-06-03T19:51:04.530 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:51:04 smithi089 bash[78452]: cluster 1685821863.3144495 mon.a (mon.0) 359 : cluster 0 osdmap e96: 8 total, 8 up, 8 in 2023-06-03T19:51:04.530 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:51:04 smithi089 bash[78452]: audit 1685821863.3865058 mon.a (mon.0) 360 : audit 0 from='client.? 172.21.15.89:0/3477108580' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:51:04.530 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:51:04 smithi089 bash[78452]: audit 1685821864.1152363 mon.c (mon.2) 184 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-06-03T19:51:04.531 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:51:04 smithi089 bash[75068]: cluster 1685821863.3083317 osd.4 (osd.4) 18 : cluster 3 failed to encode map e96 with expected crc 2023-06-03T19:51:04.531 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:51:04 smithi089 bash[75068]: cluster 1685821863.3090408 osd.5 (osd.5) 14 : cluster 3 failed to encode map e96 with expected crc 2023-06-03T19:51:04.531 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:51:04 smithi089 bash[75068]: audit 1685821863.3143368 mon.a (mon.0) 358 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_3", "pg_num": 16}]': finished 2023-06-03T19:51:04.531 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:51:04 smithi089 bash[75068]: cluster 1685821863.3144495 mon.a (mon.0) 359 : cluster 0 osdmap e96: 8 total, 8 up, 8 in 2023-06-03T19:51:04.531 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:51:04 smithi089 bash[75068]: audit 1685821863.3865058 mon.a (mon.0) 360 : audit 0 from='client.? 172.21.15.89:0/3477108580' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:51:04.531 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:51:04 smithi089 bash[75068]: audit 1685821864.1152363 mon.c (mon.2) 184 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-06-03T19:51:04.585 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:51:04 smithi164 bash[59462]: cluster 1685821863.3083317 osd.4 (osd.4) 18 : cluster 3 failed to encode map e96 with expected crc 2023-06-03T19:51:04.585 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:51:04 smithi164 bash[59462]: cluster 1685821863.3090408 osd.5 (osd.5) 14 : cluster 3 failed to encode map e96 with expected crc 2023-06-03T19:51:04.586 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:51:04 smithi164 bash[59462]: audit 1685821863.3143368 mon.a (mon.0) 358 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_3", "pg_num": 16}]': finished 2023-06-03T19:51:04.586 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:51:04 smithi164 bash[59462]: cluster 1685821863.3144495 mon.a (mon.0) 359 : cluster 0 osdmap e96: 8 total, 8 up, 8 in 2023-06-03T19:51:04.586 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:51:04 smithi164 bash[59462]: audit 1685821863.3865058 mon.a (mon.0) 360 : audit 0 from='client.? 172.21.15.89:0/3477108580' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:51:04.586 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:51:04 smithi164 bash[59462]: audit 1685821864.1152363 mon.c (mon.2) 184 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-06-03T19:51:05.779 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:51:05 smithi089 bash[75068]: audit 1685821864.1156025 mgr.y (mgr.34104) 389 : audit 0 from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-06-03T19:51:05.779 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:51:05 smithi089 bash[75068]: cephadm 1685821864.117438 mgr.y (mgr.34104) 390 : cephadm 1 Upgrade: 16 pgs have unknown state; cannot draw any conclusions 2023-06-03T19:51:05.779 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:51:05 smithi089 bash[75068]: cluster 1685821864.1959815 mgr.y (mgr.34104) 391 : cluster 0 pgmap v318: 89 pgs: 4 creating+peering, 12 unknown, 73 active+clean; 582 KiB data, 1.7 GiB used, 713 GiB / 715 GiB avail 2023-06-03T19:51:05.780 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:51:05 smithi089 bash[75068]: cluster 1685821864.311654 osd.5 (osd.5) 15 : cluster 3 failed to encode map e97 with expected crc 2023-06-03T19:51:05.780 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:51:05 smithi089 bash[75068]: cluster 1685821864.3158176 osd.4 (osd.4) 19 : cluster 3 failed to encode map e97 with expected crc 2023-06-03T19:51:05.780 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:51:05 smithi089 bash[75068]: cluster 1685821864.3161283 osd.6 (osd.6) 17 : cluster 3 failed to encode map e97 with expected crc 2023-06-03T19:51:05.780 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:51:05 smithi089 bash[75068]: cluster 1685821864.316906 osd.5 (osd.5) 16 : cluster 3 failed to encode map e97 with expected crc 2023-06-03T19:51:05.780 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:51:05 smithi089 bash[75068]: cluster 1685821864.3184 mon.a (mon.0) 361 : cluster 0 osdmap e97: 8 total, 8 up, 8 in 2023-06-03T19:51:05.780 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:51:05 smithi089 bash[75068]: cluster 1685821864.3186624 osd.3 (osd.3) 20 : cluster 3 failed to encode map e97 with expected crc 2023-06-03T19:51:05.780 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:51:05 smithi089 bash[75068]: cluster 1685821864.318771 osd.1 (osd.1) 18 : cluster 3 failed to encode map e97 with expected crc 2023-06-03T19:51:05.781 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:51:05 smithi089 bash[78452]: audit 1685821864.1156025 mgr.y (mgr.34104) 389 : audit 0 from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-06-03T19:51:05.781 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:51:05 smithi089 bash[78452]: cephadm 1685821864.117438 mgr.y (mgr.34104) 390 : cephadm 1 Upgrade: 16 pgs have unknown state; cannot draw any conclusions 2023-06-03T19:51:05.781 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:51:05 smithi089 bash[78452]: cluster 1685821864.1959815 mgr.y (mgr.34104) 391 : cluster 0 pgmap v318: 89 pgs: 4 creating+peering, 12 unknown, 73 active+clean; 582 KiB data, 1.7 GiB used, 713 GiB / 715 GiB avail 2023-06-03T19:51:05.781 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:51:05 smithi089 bash[78452]: cluster 1685821864.311654 osd.5 (osd.5) 15 : cluster 3 failed to encode map e97 with expected crc 2023-06-03T19:51:05.781 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:51:05 smithi089 bash[78452]: cluster 1685821864.3158176 osd.4 (osd.4) 19 : cluster 3 failed to encode map e97 with expected crc 2023-06-03T19:51:05.781 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:51:05 smithi089 bash[78452]: cluster 1685821864.3161283 osd.6 (osd.6) 17 : cluster 3 failed to encode map e97 with expected crc 2023-06-03T19:51:05.782 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:51:05 smithi089 bash[78452]: cluster 1685821864.316906 osd.5 (osd.5) 16 : cluster 3 failed to encode map e97 with expected crc 2023-06-03T19:51:05.782 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:51:05 smithi089 bash[78452]: cluster 1685821864.3184 mon.a (mon.0) 361 : cluster 0 osdmap e97: 8 total, 8 up, 8 in 2023-06-03T19:51:05.782 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:51:05 smithi089 bash[78452]: cluster 1685821864.3186624 osd.3 (osd.3) 20 : cluster 3 failed to encode map e97 with expected crc 2023-06-03T19:51:05.782 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:51:05 smithi089 bash[78452]: cluster 1685821864.318771 osd.1 (osd.1) 18 : cluster 3 failed to encode map e97 with expected crc 2023-06-03T19:51:05.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:51:05 smithi164 bash[59462]: audit 1685821864.1156025 mgr.y (mgr.34104) 389 : audit 0 from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-06-03T19:51:05.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:51:05 smithi164 bash[59462]: cephadm 1685821864.117438 mgr.y (mgr.34104) 390 : cephadm 1 Upgrade: 16 pgs have unknown state; cannot draw any conclusions 2023-06-03T19:51:05.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:51:05 smithi164 bash[59462]: cluster 1685821864.1959815 mgr.y (mgr.34104) 391 : cluster 0 pgmap v318: 89 pgs: 4 creating+peering, 12 unknown, 73 active+clean; 582 KiB data, 1.7 GiB used, 713 GiB / 715 GiB avail 2023-06-03T19:51:05.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:51:05 smithi164 bash[59462]: cluster 1685821864.311654 osd.5 (osd.5) 15 : cluster 3 failed to encode map e97 with expected crc 2023-06-03T19:51:05.836 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:51:05 smithi164 bash[59462]: cluster 1685821864.3158176 osd.4 (osd.4) 19 : cluster 3 failed to encode map e97 with expected crc 2023-06-03T19:51:05.836 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:51:05 smithi164 bash[59462]: cluster 1685821864.3161283 osd.6 (osd.6) 17 : cluster 3 failed to encode map e97 with expected crc 2023-06-03T19:51:05.836 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:51:05 smithi164 bash[59462]: cluster 1685821864.316906 osd.5 (osd.5) 16 : cluster 3 failed to encode map e97 with expected crc 2023-06-03T19:51:05.836 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:51:05 smithi164 bash[59462]: cluster 1685821864.3184 mon.a (mon.0) 361 : cluster 0 osdmap e97: 8 total, 8 up, 8 in 2023-06-03T19:51:05.836 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:51:05 smithi164 bash[59462]: cluster 1685821864.3186624 osd.3 (osd.3) 20 : cluster 3 failed to encode map e97 with expected crc 2023-06-03T19:51:05.836 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:51:05 smithi164 bash[59462]: cluster 1685821864.318771 osd.1 (osd.1) 18 : cluster 3 failed to encode map e97 with expected crc 2023-06-03T19:51:06.529 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 19:51:06 smithi089 bash[71163]: ::ffff:172.21.15.164 - - [03/Jun/2023:19:51:06] "GET /metrics HTTP/1.1" 200 207348 "" "Prometheus/2.33.4" 2023-06-03T19:51:06.530 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:51:06 smithi089 bash[78452]: cluster 1685821865.3223112 osd.5 (osd.5) 17 : cluster 3 failed to encode map e98 with expected crc 2023-06-03T19:51:06.530 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:51:06 smithi089 bash[78452]: cluster 1685821865.3243413 osd.3 (osd.3) 21 : cluster 3 failed to encode map e98 with expected crc 2023-06-03T19:51:06.530 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:51:06 smithi089 bash[78452]: cluster 1685821865.3283448 mon.a (mon.0) 362 : cluster 0 osdmap e98: 8 total, 8 up, 8 in 2023-06-03T19:51:06.530 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:51:06 smithi089 bash[78452]: audit 1685821865.758099 mon.c (mon.2) 185 : audit 0 from='client.? 172.21.15.89:0/413429768' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:51:06.530 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:51:06 smithi089 bash[75068]: cluster 1685821865.3223112 osd.5 (osd.5) 17 : cluster 3 failed to encode map e98 with expected crc 2023-06-03T19:51:06.531 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:51:06 smithi089 bash[75068]: cluster 1685821865.3243413 osd.3 (osd.3) 21 : cluster 3 failed to encode map e98 with expected crc 2023-06-03T19:51:06.531 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:51:06 smithi089 bash[75068]: cluster 1685821865.3283448 mon.a (mon.0) 362 : cluster 0 osdmap e98: 8 total, 8 up, 8 in 2023-06-03T19:51:06.531 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:51:06 smithi089 bash[75068]: audit 1685821865.758099 mon.c (mon.2) 185 : audit 0 from='client.? 172.21.15.89:0/413429768' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:51:06.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:51:06 smithi164 bash[59462]: cluster 1685821865.3223112 osd.5 (osd.5) 17 : cluster 3 failed to encode map e98 with expected crc 2023-06-03T19:51:06.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:51:06 smithi164 bash[59462]: cluster 1685821865.3243413 osd.3 (osd.3) 21 : cluster 3 failed to encode map e98 with expected crc 2023-06-03T19:51:06.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:51:06 smithi164 bash[59462]: cluster 1685821865.3283448 mon.a (mon.0) 362 : cluster 0 osdmap e98: 8 total, 8 up, 8 in 2023-06-03T19:51:06.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:51:06 smithi164 bash[59462]: audit 1685821865.758099 mon.c (mon.2) 185 : audit 0 from='client.? 172.21.15.89:0/413429768' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:51:07.779 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:51:07 smithi089 bash[78452]: cluster 1685821866.1969497 mgr.y (mgr.34104) 392 : cluster 0 pgmap v321: 89 pgs: 3 creating+activating, 4 creating+peering, 6 unknown, 76 active+clean; 582 KiB data, 1.7 GiB used, 713 GiB / 715 GiB avail 2023-06-03T19:51:07.779 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:51:07 smithi089 bash[78452]: audit 1685821867.0875268 mon.a (mon.0) 363 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:51:07.780 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:51:07 smithi089 bash[78452]: audit 1685821867.0904434 mon.c (mon.2) 186 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T19:51:07.780 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:51:07 smithi089 bash[75068]: cluster 1685821866.1969497 mgr.y (mgr.34104) 392 : cluster 0 pgmap v321: 89 pgs: 3 creating+activating, 4 creating+peering, 6 unknown, 76 active+clean; 582 KiB data, 1.7 GiB used, 713 GiB / 715 GiB avail 2023-06-03T19:51:07.780 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:51:07 smithi089 bash[75068]: audit 1685821867.0875268 mon.a (mon.0) 363 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:51:07.780 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:51:07 smithi089 bash[75068]: audit 1685821867.0904434 mon.c (mon.2) 186 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T19:51:07.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:51:07 smithi164 bash[59462]: cluster 1685821866.1969497 mgr.y (mgr.34104) 392 : cluster 0 pgmap v321: 89 pgs: 3 creating+activating, 4 creating+peering, 6 unknown, 76 active+clean; 582 KiB data, 1.7 GiB used, 713 GiB / 715 GiB avail 2023-06-03T19:51:07.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:51:07 smithi164 bash[59462]: audit 1685821867.0875268 mon.a (mon.0) 363 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:51:07.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:51:07 smithi164 bash[59462]: audit 1685821867.0904434 mon.c (mon.2) 186 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T19:51:08.779 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:51:08 smithi089 bash[78452]: audit 1685821868.122076 mon.a (mon.0) 364 : audit 0 from='client.? 172.21.15.89:0/2782521250' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:51:08.779 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:51:08 smithi089 bash[75068]: audit 1685821868.122076 mon.a (mon.0) 364 : audit 0 from='client.? 172.21.15.89:0/2782521250' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:51:08.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:51:08 smithi164 bash[59462]: audit 1685821868.122076 mon.a (mon.0) 364 : audit 0 from='client.? 172.21.15.89:0/2782521250' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:51:09.031 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/mon.a/config 2023-06-03T19:51:09.779 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:51:09 smithi089 bash[75068]: cluster 1685821868.1976588 mgr.y (mgr.34104) 393 : cluster 0 pgmap v322: 89 pgs: 3 creating+activating, 4 creating+peering, 6 unknown, 76 active+clean; 582 KiB data, 1.7 GiB used, 713 GiB / 715 GiB avail 2023-06-03T19:51:09.779 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:51:09 smithi089 bash[78452]: cluster 1685821868.1976588 mgr.y (mgr.34104) 393 : cluster 0 pgmap v322: 89 pgs: 3 creating+activating, 4 creating+peering, 6 unknown, 76 active+clean; 582 KiB data, 1.7 GiB used, 713 GiB / 715 GiB avail 2023-06-03T19:51:09.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:51:09 smithi164 bash[59462]: cluster 1685821868.1976588 mgr.y (mgr.34104) 393 : cluster 0 pgmap v322: 89 pgs: 3 creating+activating, 4 creating+peering, 6 unknown, 76 active+clean; 582 KiB data, 1.7 GiB used, 713 GiB / 715 GiB avail 2023-06-03T19:51:11.529 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:51:11 smithi089 bash[75068]: cluster 1685821870.1987474 mgr.y (mgr.34104) 394 : cluster 0 pgmap v323: 89 pgs: 89 active+clean; 582 KiB data, 1.7 GiB used, 713 GiB / 715 GiB avail 2023-06-03T19:51:11.529 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:51:11 smithi089 bash[75068]: audit 1685821870.4914377 mon.a (mon.0) 365 : audit 0 from='client.? 172.21.15.89:0/1181900313' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:51:11.529 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:51:11 smithi089 bash[78452]: cluster 1685821870.1987474 mgr.y (mgr.34104) 394 : cluster 0 pgmap v323: 89 pgs: 89 active+clean; 582 KiB data, 1.7 GiB used, 713 GiB / 715 GiB avail 2023-06-03T19:51:11.530 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:51:11 smithi089 bash[78452]: audit 1685821870.4914377 mon.a (mon.0) 365 : audit 0 from='client.? 172.21.15.89:0/1181900313' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:51:11.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:51:11 smithi164 bash[59462]: cluster 1685821870.1987474 mgr.y (mgr.34104) 394 : cluster 0 pgmap v323: 89 pgs: 89 active+clean; 582 KiB data, 1.7 GiB used, 713 GiB / 715 GiB avail 2023-06-03T19:51:11.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:51:11 smithi164 bash[59462]: audit 1685821870.4914377 mon.a (mon.0) 365 : audit 0 from='client.? 172.21.15.89:0/1181900313' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:51:13.388 INFO:teuthology.orchestra.run.smithi089.stderr:enabled application 'rados' on pool 'unique_pool_3' 2023-06-03T19:51:13.779 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:51:13 smithi089 bash[75068]: cluster 1685821872.1993122 mgr.y (mgr.34104) 395 : cluster 0 pgmap v324: 89 pgs: 89 active+clean; 582 KiB data, 1.7 GiB used, 713 GiB / 715 GiB avail 2023-06-03T19:51:13.779 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:51:13 smithi089 bash[75068]: audit 1685821872.68117 mon.a (mon.0) 366 : audit 1 from='client.? 172.21.15.89:0/2395770436' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_3", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-06-03T19:51:13.779 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:51:13 smithi089 bash[75068]: audit 1685821872.8469336 mon.c (mon.2) 187 : audit 0 from='client.? 172.21.15.89:0/1568980501' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:51:13.780 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:51:13 smithi089 bash[78452]: cluster 1685821872.1993122 mgr.y (mgr.34104) 395 : cluster 0 pgmap v324: 89 pgs: 89 active+clean; 582 KiB data, 1.7 GiB used, 713 GiB / 715 GiB avail 2023-06-03T19:51:13.780 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:51:13 smithi089 bash[78452]: audit 1685821872.68117 mon.a (mon.0) 366 : audit 1 from='client.? 172.21.15.89:0/2395770436' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_3", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-06-03T19:51:13.780 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:51:13 smithi089 bash[78452]: audit 1685821872.8469336 mon.c (mon.2) 187 : audit 0 from='client.? 172.21.15.89:0/1568980501' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:51:13.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:51:13 smithi164 bash[59462]: cluster 1685821872.1993122 mgr.y (mgr.34104) 395 : cluster 0 pgmap v324: 89 pgs: 89 active+clean; 582 KiB data, 1.7 GiB used, 713 GiB / 715 GiB avail 2023-06-03T19:51:13.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:51:13 smithi164 bash[59462]: audit 1685821872.68117 mon.a (mon.0) 366 : audit 1 from='client.? 172.21.15.89:0/2395770436' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_3", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-06-03T19:51:13.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:51:13 smithi164 bash[59462]: audit 1685821872.8469336 mon.c (mon.2) 187 : audit 0 from='client.? 172.21.15.89:0/1568980501' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:51:14.529 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:51:14 smithi089 bash[75068]: cluster 1685821873.3762426 osd.2 (osd.2) 20 : cluster 3 failed to encode map e99 with expected crc 2023-06-03T19:51:14.529 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:51:14 smithi089 bash[75068]: audit 1685821873.3786783 mon.a (mon.0) 367 : audit 1 from='client.? 172.21.15.89:0/2395770436' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_3", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-06-03T19:51:14.529 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:51:14 smithi089 bash[75068]: cluster 1685821873.378757 mon.a (mon.0) 368 : cluster 0 osdmap e99: 8 total, 8 up, 8 in 2023-06-03T19:51:14.530 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:51:14 smithi089 bash[78452]: cluster 1685821873.3762426 osd.2 (osd.2) 20 : cluster 3 failed to encode map e99 with expected crc 2023-06-03T19:51:14.530 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:51:14 smithi089 bash[78452]: audit 1685821873.3786783 mon.a (mon.0) 367 : audit 1 from='client.? 172.21.15.89:0/2395770436' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_3", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-06-03T19:51:14.530 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:51:14 smithi089 bash[78452]: cluster 1685821873.378757 mon.a (mon.0) 368 : cluster 0 osdmap e99: 8 total, 8 up, 8 in 2023-06-03T19:51:14.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:51:14 smithi164 bash[59462]: cluster 1685821873.3762426 osd.2 (osd.2) 20 : cluster 3 failed to encode map e99 with expected crc 2023-06-03T19:51:14.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:51:14 smithi164 bash[59462]: audit 1685821873.3786783 mon.a (mon.0) 367 : audit 1 from='client.? 172.21.15.89:0/2395770436' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_3", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-06-03T19:51:14.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:51:14 smithi164 bash[59462]: cluster 1685821873.378757 mon.a (mon.0) 368 : cluster 0 osdmap e99: 8 total, 8 up, 8 in 2023-06-03T19:51:15.276 DEBUG:teuthology.orchestra.run.smithi089:> /bin/sh -c 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage rados --no-log-to-stderr --name client.0 -b 65536 --object-size 65536 -p unique_pool_3 bench 90 write' 2023-06-03T19:51:15.279 INFO:tasks.radosbench:joining radosbench (timing out after 3000s) 2023-06-03T19:51:15.279 INFO:teuthology.orchestra.run:waiting for 3000 2023-06-03T19:51:15.323 INFO:tasks.radosbench.radosbench.0.smithi089.stdout:hints = 1 2023-06-03T19:51:15.323 INFO:tasks.radosbench.radosbench.0.smithi089.stdout:Maintaining 16 concurrent writes of 65536 bytes to objects of size 65536 for up to 90 seconds or 0 objects 2023-06-03T19:51:15.323 INFO:tasks.radosbench.radosbench.0.smithi089.stdout:Object prefix: benchmark_data_smithi089_91006 2023-06-03T19:51:15.324 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-06-03T19:51:15.324 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 0 0 0 0 0 0 - 0 2023-06-03T19:51:15.779 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:51:15 smithi089 bash[78452]: cluster 1685821874.200021 mgr.y (mgr.34104) 396 : cluster 0 pgmap v326: 89 pgs: 89 active+clean; 582 KiB data, 1.7 GiB used, 713 GiB / 715 GiB avail 2023-06-03T19:51:15.779 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:51:15 smithi089 bash[78452]: cluster 1685821874.295396 osd.1 (osd.1) 19 : cluster 3 failed to encode map e99 with expected crc 2023-06-03T19:51:15.779 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:51:15 smithi089 bash[78452]: cluster 1685821874.3477032 osd.3 (osd.3) 22 : cluster 3 failed to encode map e99 with expected crc 2023-06-03T19:51:15.779 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:51:15 smithi089 bash[78452]: audit 1685821875.2147303 mon.a (mon.0) 369 : audit 0 from='client.? 172.21.15.89:0/16568341' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:51:15.780 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:51:15 smithi089 bash[75068]: cluster 1685821874.200021 mgr.y (mgr.34104) 396 : cluster 0 pgmap v326: 89 pgs: 89 active+clean; 582 KiB data, 1.7 GiB used, 713 GiB / 715 GiB avail 2023-06-03T19:51:15.780 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:51:15 smithi089 bash[75068]: cluster 1685821874.295396 osd.1 (osd.1) 19 : cluster 3 failed to encode map e99 with expected crc 2023-06-03T19:51:15.780 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:51:15 smithi089 bash[75068]: cluster 1685821874.3477032 osd.3 (osd.3) 22 : cluster 3 failed to encode map e99 with expected crc 2023-06-03T19:51:15.780 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:51:15 smithi089 bash[75068]: audit 1685821875.2147303 mon.a (mon.0) 369 : audit 0 from='client.? 172.21.15.89:0/16568341' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:51:15.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:51:15 smithi164 bash[59462]: cluster 1685821874.200021 mgr.y (mgr.34104) 396 : cluster 0 pgmap v326: 89 pgs: 89 active+clean; 582 KiB data, 1.7 GiB used, 713 GiB / 715 GiB avail 2023-06-03T19:51:15.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:51:15 smithi164 bash[59462]: cluster 1685821874.295396 osd.1 (osd.1) 19 : cluster 3 failed to encode map e99 with expected crc 2023-06-03T19:51:15.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:51:15 smithi164 bash[59462]: cluster 1685821874.3477032 osd.3 (osd.3) 22 : cluster 3 failed to encode map e99 with expected crc 2023-06-03T19:51:15.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:51:15 smithi164 bash[59462]: audit 1685821875.2147303 mon.a (mon.0) 369 : audit 0 from='client.? 172.21.15.89:0/16568341' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:51:16.324 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 1 16 3124 3108 194.24 194.25 0.00499578 0.00513137 2023-06-03T19:51:16.528 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 19:51:16 smithi089 bash[71163]: ::ffff:172.21.15.164 - - [03/Jun/2023:19:51:16] "GET /metrics HTTP/1.1" 200 207308 "" "Prometheus/2.33.4" 2023-06-03T19:51:17.324 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 2 16 6003 5987 187.08 179.938 0.00605635 0.0053239 2023-06-03T19:51:17.779 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:51:17 smithi089 bash[75068]: cluster 1685821876.2006235 mgr.y (mgr.34104) 397 : cluster 0 pgmap v327: 89 pgs: 89 active+clean; 16 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 1.6 MiB/s wr, 24 op/s 2023-06-03T19:51:17.779 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:51:17 smithi089 bash[78452]: cluster 1685821876.2006235 mgr.y (mgr.34104) 397 : cluster 0 pgmap v327: 89 pgs: 89 active+clean; 16 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 1.6 MiB/s wr, 24 op/s 2023-06-03T19:51:17.834 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:51:17 smithi164 bash[59462]: cluster 1685821876.2006235 mgr.y (mgr.34104) 397 : cluster 0 pgmap v327: 89 pgs: 89 active+clean; 16 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 1.6 MiB/s wr, 24 op/s 2023-06-03T19:51:18.324 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 3 16 8553 8537 177.84 159.375 0.00415298 0.00561554 2023-06-03T19:51:18.779 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:51:18 smithi089 bash[75068]: audit 1685821877.821345 mon.b (mon.1) 17 : audit 0 from='client.? 172.21.15.89:0/3709105543' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:51:18.779 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:51:18 smithi089 bash[78452]: audit 1685821877.821345 mon.b (mon.1) 17 : audit 0 from='client.? 172.21.15.89:0/3709105543' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:51:18.834 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:51:18 smithi164 bash[59462]: audit 1685821877.821345 mon.b (mon.1) 17 : audit 0 from='client.? 172.21.15.89:0/3709105543' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:51:19.324 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 4 16 11083 11067 172.909 158.125 0.0120667 0.00577388 2023-06-03T19:51:19.779 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:51:19 smithi089 bash[75068]: cluster 1685821878.2009704 mgr.y (mgr.34104) 398 : cluster 0 pgmap v328: 89 pgs: 89 active+clean; 16 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 1.6 MiB/s wr, 24 op/s 2023-06-03T19:51:19.779 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:51:19 smithi089 bash[75068]: audit 1685821879.1292937 mon.c (mon.2) 188 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-06-03T19:51:19.779 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:51:19 smithi089 bash[78452]: cluster 1685821878.2009704 mgr.y (mgr.34104) 398 : cluster 0 pgmap v328: 89 pgs: 89 active+clean; 16 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 1.6 MiB/s wr, 24 op/s 2023-06-03T19:51:19.780 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:51:19 smithi089 bash[78452]: audit 1685821879.1292937 mon.c (mon.2) 188 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-06-03T19:51:19.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:51:19 smithi164 bash[59462]: cluster 1685821878.2009704 mgr.y (mgr.34104) 398 : cluster 0 pgmap v328: 89 pgs: 89 active+clean; 16 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 1.6 MiB/s wr, 24 op/s 2023-06-03T19:51:19.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:51:19 smithi164 bash[59462]: audit 1685821879.1292937 mon.c (mon.2) 188 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-06-03T19:51:20.324 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 5 16 13280 13264 165.787 137.312 0.0115739 0.00602181 2023-06-03T19:51:20.779 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:51:20 smithi089 bash[78452]: audit 1685821879.1295145 mgr.y (mgr.34104) 399 : audit 0 from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-06-03T19:51:20.779 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:51:20 smithi089 bash[78452]: cephadm 1685821879.1310966 mgr.y (mgr.34104) 400 : cephadm 1 Upgrade: osd.1 is safe to restart 2023-06-03T19:51:20.780 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:51:20 smithi089 bash[75068]: audit 1685821879.1295145 mgr.y (mgr.34104) 399 : audit 0 from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-06-03T19:51:20.780 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:51:20 smithi089 bash[75068]: cephadm 1685821879.1310966 mgr.y (mgr.34104) 400 : cephadm 1 Upgrade: osd.1 is safe to restart 2023-06-03T19:51:20.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:51:20 smithi164 bash[59462]: audit 1685821879.1295145 mgr.y (mgr.34104) 399 : audit 0 from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-06-03T19:51:20.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:51:20 smithi164 bash[59462]: cephadm 1685821879.1310966 mgr.y (mgr.34104) 400 : cephadm 1 Upgrade: osd.1 is safe to restart 2023-06-03T19:51:21.324 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 6 16 15660 15644 162.945 148.75 0.00638316 0.00612625 2023-06-03T19:51:21.779 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:51:21 smithi089 bash[78452]: cluster 1685821880.2016902 mgr.y (mgr.34104) 401 : cluster 0 pgmap v329: 89 pgs: 89 active+clean; 488 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 49 MiB/s wr, 779 op/s 2023-06-03T19:51:21.779 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:51:21 smithi089 bash[78452]: audit 1685821880.479252 mon.b (mon.1) 18 : audit 0 from='client.? 172.21.15.89:0/1270910496' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:51:21.779 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:51:21 smithi089 bash[75068]: cluster 1685821880.2016902 mgr.y (mgr.34104) 401 : cluster 0 pgmap v329: 89 pgs: 89 active+clean; 488 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 49 MiB/s wr, 779 op/s 2023-06-03T19:51:21.779 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:51:21 smithi089 bash[75068]: audit 1685821880.479252 mon.b (mon.1) 18 : audit 0 from='client.? 172.21.15.89:0/1270910496' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:51:21.834 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:51:21 smithi164 bash[59462]: cluster 1685821880.2016902 mgr.y (mgr.34104) 401 : cluster 0 pgmap v329: 89 pgs: 89 active+clean; 488 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 49 MiB/s wr, 779 op/s 2023-06-03T19:51:21.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:51:21 smithi164 bash[59462]: audit 1685821880.479252 mon.b (mon.1) 18 : audit 0 from='client.? 172.21.15.89:0/1270910496' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:51:22.324 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 7 16 17925 17909 159.889 141.562 0.00380939 0.0062467 2023-06-03T19:51:22.779 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:51:22 smithi089 bash[78452]: audit 1685821882.0795977 mon.c (mon.2) 189 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T19:51:22.779 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:51:22 smithi089 bash[78452]: audit 1685821882.2579467 mon.a (mon.0) 370 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:51:22.779 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:51:22 smithi089 bash[75068]: audit 1685821882.0795977 mon.c (mon.2) 189 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T19:51:22.780 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:51:22 smithi089 bash[75068]: audit 1685821882.2579467 mon.a (mon.0) 370 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:51:22.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:51:22 smithi164 bash[59462]: audit 1685821882.0795977 mon.c (mon.2) 189 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T19:51:22.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:51:22 smithi164 bash[59462]: audit 1685821882.2579467 mon.a (mon.0) 370 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:51:23.324 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 8 16 20070 20054 156.659 134.062 0.0121204 0.00637564 2023-06-03T19:51:23.696 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:51:23 smithi089 bash[75068]: cluster 1685821882.2022216 mgr.y (mgr.34104) 402 : cluster 0 pgmap v330: 89 pgs: 89 active+clean; 641 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 64 MiB/s wr, 1.02k op/s 2023-06-03T19:51:23.696 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:51:23 smithi089 bash[75068]: audit 1685821883.0297003 mon.b (mon.1) 19 : audit 0 from='client.? 172.21.15.89:0/792812524' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:51:23.697 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:51:23 smithi089 bash[78452]: cluster 1685821882.2022216 mgr.y (mgr.34104) 402 : cluster 0 pgmap v330: 89 pgs: 89 active+clean; 641 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 64 MiB/s wr, 1.02k op/s 2023-06-03T19:51:23.697 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:51:23 smithi089 bash[78452]: audit 1685821883.0297003 mon.b (mon.1) 19 : audit 0 from='client.? 172.21.15.89:0/792812524' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:51:23.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:51:23 smithi164 bash[59462]: cluster 1685821882.2022216 mgr.y (mgr.34104) 402 : cluster 0 pgmap v330: 89 pgs: 89 active+clean; 641 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 64 MiB/s wr, 1.02k op/s 2023-06-03T19:51:23.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:51:23 smithi164 bash[59462]: audit 1685821883.0297003 mon.b (mon.1) 19 : audit 0 from='client.? 172.21.15.89:0/792812524' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:51:24.324 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 9 16 22078 22062 153.195 125.5 0.00427467 0.00652144 2023-06-03T19:51:25.324 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 10 16 24045 24029 150.168 122.938 0.00995207 0.0066516 2023-06-03T19:51:25.779 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:51:25 smithi089 bash[75068]: cluster 1685821884.2030299 mgr.y (mgr.34104) 403 : cluster 0 pgmap v331: 89 pgs: 89 active+clean; 1.1 GiB data, 5.1 GiB used, 710 GiB / 715 GiB avail; 100 MiB/s wr, 1.60k op/s 2023-06-03T19:51:25.780 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:51:25 smithi089 bash[78452]: cluster 1685821884.2030299 mgr.y (mgr.34104) 403 : cluster 0 pgmap v331: 89 pgs: 89 active+clean; 1.1 GiB data, 5.1 GiB used, 710 GiB / 715 GiB avail; 100 MiB/s wr, 1.60k op/s 2023-06-03T19:51:25.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:51:25 smithi164 bash[59462]: cluster 1685821884.2030299 mgr.y (mgr.34104) 403 : cluster 0 pgmap v331: 89 pgs: 89 active+clean; 1.1 GiB data, 5.1 GiB used, 710 GiB / 715 GiB avail; 100 MiB/s wr, 1.60k op/s 2023-06-03T19:51:26.233 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 19:51:26 smithi089 bash[71163]: ::ffff:172.21.15.164 - - [03/Jun/2023:19:51:26] "GET /metrics HTTP/1.1" 200 207391 "" "Prometheus/2.33.4" 2023-06-03T19:51:26.325 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 11 16 25785 25769 146.402 108.75 0.00755588 0.00682438 2023-06-03T19:51:26.491 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:51:26 smithi089 bash[75068]: cephadm 1685821885.4790125 mgr.y (mgr.34104) 404 : cephadm 1 Upgrade: Updating osd.1 2023-06-03T19:51:26.492 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:51:26 smithi089 bash[75068]: audit 1685821885.4937565 mon.a (mon.0) 371 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:51:26.492 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:51:26 smithi089 bash[75068]: audit 1685821885.4961987 mon.c (mon.2) 190 : audit 1 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2023-06-03T19:51:26.493 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:51:26 smithi089 bash[75068]: audit 1685821885.4986632 mon.c (mon.2) 191 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-03T19:51:26.493 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:51:26 smithi089 bash[75068]: cephadm 1685821885.5020127 mgr.y (mgr.34104) 405 : cephadm 1 Deploying daemon osd.1 on smithi089 2023-06-03T19:51:26.493 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:51:26 smithi089 bash[75068]: audit 1685821885.6075447 mon.a (mon.0) 372 : audit 0 from='client.? 172.21.15.89:0/3632887365' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:51:26.493 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:51:26 smithi089 bash[78452]: cephadm 1685821885.4790125 mgr.y (mgr.34104) 404 : cephadm 1 Upgrade: Updating osd.1 2023-06-03T19:51:26.493 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:51:26 smithi089 bash[78452]: audit 1685821885.4937565 mon.a (mon.0) 371 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:51:26.494 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:51:26 smithi089 bash[78452]: audit 1685821885.4961987 mon.c (mon.2) 190 : audit 1 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2023-06-03T19:51:26.494 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:51:26 smithi089 bash[78452]: audit 1685821885.4986632 mon.c (mon.2) 191 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-03T19:51:26.494 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:51:26 smithi089 bash[78452]: cephadm 1685821885.5020127 mgr.y (mgr.34104) 405 : cephadm 1 Deploying daemon osd.1 on smithi089 2023-06-03T19:51:26.494 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:51:26 smithi089 bash[78452]: audit 1685821885.6075447 mon.a (mon.0) 372 : audit 0 from='client.? 172.21.15.89:0/3632887365' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:51:26.834 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:51:26 smithi164 bash[59462]: cephadm 1685821885.4790125 mgr.y (mgr.34104) 404 : cephadm 1 Upgrade: Updating osd.1 2023-06-03T19:51:26.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:51:26 smithi164 bash[59462]: audit 1685821885.4937565 mon.a (mon.0) 371 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:51:26.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:51:26 smithi164 bash[59462]: audit 1685821885.4961987 mon.c (mon.2) 190 : audit 1 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2023-06-03T19:51:26.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:51:26 smithi164 bash[59462]: audit 1685821885.4986632 mon.c (mon.2) 191 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-03T19:51:26.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:51:26 smithi164 bash[59462]: cephadm 1685821885.5020127 mgr.y (mgr.34104) 405 : cephadm 1 Deploying daemon osd.1 on smithi089 2023-06-03T19:51:26.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:51:26 smithi164 bash[59462]: audit 1685821885.6075447 mon.a (mon.0) 372 : audit 0 from='client.? 172.21.15.89:0/3632887365' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:51:27.325 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 12 16 27629 27613 143.805 115.25 0.00830261 0.00694743 2023-06-03T19:51:27.779 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:51:27 smithi089 bash[75068]: cluster 1685821886.2035794 mgr.y (mgr.34104) 406 : cluster 0 pgmap v332: 89 pgs: 89 active+clean; 1.3 GiB data, 5.6 GiB used, 710 GiB / 715 GiB avail; 111 MiB/s wr, 1.78k op/s 2023-06-03T19:51:27.779 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:51:27 smithi089 bash[78452]: cluster 1685821886.2035794 mgr.y (mgr.34104) 406 : cluster 0 pgmap v332: 89 pgs: 89 active+clean; 1.3 GiB data, 5.6 GiB used, 710 GiB / 715 GiB avail; 111 MiB/s wr, 1.78k op/s 2023-06-03T19:51:27.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:51:27 smithi164 bash[59462]: cluster 1685821886.2035794 mgr.y (mgr.34104) 406 : cluster 0 pgmap v332: 89 pgs: 89 active+clean; 1.3 GiB data, 5.6 GiB used, 710 GiB / 715 GiB avail; 111 MiB/s wr, 1.78k op/s 2023-06-03T19:51:28.325 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 13 15 29371 29356 141.122 108.938 0.00955301 0.00707927 2023-06-03T19:51:28.514 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:51:28 smithi089 bash[75068]: audit 1685821888.1658258 mon.c (mon.2) 192 : audit 0 from='client.? 172.21.15.89:0/2257438970' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:51:28.779 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:51:28 smithi089 bash[78452]: audit 1685821888.1658258 mon.c (mon.2) 192 : audit 0 from='client.? 172.21.15.89:0/2257438970' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:51:28.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:51:28 smithi164 bash[59462]: audit 1685821888.1658258 mon.c (mon.2) 192 : audit 0 from='client.? 172.21.15.89:0/2257438970' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:51:29.326 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 14 16 31131 31115 138.883 109.938 0.00654609 0.00719145 2023-06-03T19:51:29.779 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:51:29 smithi089 bash[78452]: cluster 1685821888.2039464 mgr.y (mgr.34104) 407 : cluster 0 pgmap v333: 89 pgs: 89 active+clean; 1.3 GiB data, 5.6 GiB used, 710 GiB / 715 GiB avail; 110 MiB/s wr, 1.76k op/s 2023-06-03T19:51:29.779 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:51:29 smithi089 bash[75068]: cluster 1685821888.2039464 mgr.y (mgr.34104) 407 : cluster 0 pgmap v333: 89 pgs: 89 active+clean; 1.3 GiB data, 5.6 GiB used, 710 GiB / 715 GiB avail; 110 MiB/s wr, 1.76k op/s 2023-06-03T19:51:29.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:51:29 smithi164 bash[59462]: cluster 1685821888.2039464 mgr.y (mgr.34104) 407 : cluster 0 pgmap v333: 89 pgs: 89 active+clean; 1.3 GiB data, 5.6 GiB used, 710 GiB / 715 GiB avail; 110 MiB/s wr, 1.76k op/s 2023-06-03T19:51:30.326 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 15 16 32813 32797 136.632 105.125 0.00846881 0.00731145 2023-06-03T19:51:31.326 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 16 16 34282 34266 133.83 91.8125 0.0052773 0.00746515 2023-06-03T19:51:31.523 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:51:31 smithi089 bash[75068]: cluster 1685821890.2047744 mgr.y (mgr.34104) 408 : cluster 0 pgmap v334: 89 pgs: 89 active+clean; 1.8 GiB data, 7.1 GiB used, 708 GiB / 715 GiB avail; 149 MiB/s wr, 2.39k op/s 2023-06-03T19:51:31.523 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:51:31 smithi089 bash[75068]: audit 1685821890.6943264 mon.a (mon.0) 373 : audit 0 from='client.? 172.21.15.89:0/1451675437' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:51:31.779 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:51:31 smithi089 bash[78452]: cluster 1685821890.2047744 mgr.y (mgr.34104) 408 : cluster 0 pgmap v334: 89 pgs: 89 active+clean; 1.8 GiB data, 7.1 GiB used, 708 GiB / 715 GiB avail; 149 MiB/s wr, 2.39k op/s 2023-06-03T19:51:31.779 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:51:31 smithi089 bash[78452]: audit 1685821890.6943264 mon.a (mon.0) 373 : audit 0 from='client.? 172.21.15.89:0/1451675437' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:51:31.834 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:51:31 smithi164 bash[59462]: cluster 1685821890.2047744 mgr.y (mgr.34104) 408 : cluster 0 pgmap v334: 89 pgs: 89 active+clean; 1.8 GiB data, 7.1 GiB used, 708 GiB / 715 GiB avail; 149 MiB/s wr, 2.39k op/s 2023-06-03T19:51:31.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:51:31 smithi164 bash[59462]: audit 1685821890.6943264 mon.a (mon.0) 373 : audit 0 from='client.? 172.21.15.89:0/1451675437' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:51:32.326 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 17 16 35904 35888 131.92 101.375 0.0084201 0.00757454 2023-06-03T19:51:33.267 INFO:journalctl@ceph.osd.1.smithi089.stdout:Jun 03 19:51:33 smithi089 systemd[1]: Stopping Ceph osd.1 for f1db3478-0243-11ee-9b1d-001a4aab830c... 2023-06-03T19:51:33.326 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 18 16 37286 37270 129.39 86.375 0.00378659 0.00771993 2023-06-03T19:51:33.526 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:51:33 smithi089 bash[75068]: cluster 1685821892.2053757 mgr.y (mgr.34104) 409 : cluster 0 pgmap v335: 89 pgs: 89 active+clean; 1.9 GiB data, 7.3 GiB used, 708 GiB / 715 GiB avail; 118 MiB/s wr, 1.89k op/s 2023-06-03T19:51:33.527 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:51:33 smithi089 bash[75068]: audit 1685821893.3357673 mon.c (mon.2) 193 : audit 0 from='client.? 172.21.15.89:0/3030577776' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:51:33.527 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:51:33 smithi089 bash[75068]: cluster 1685821893.452585 mon.a (mon.0) 374 : cluster 1 osd.1 marked itself down and dead 2023-06-03T19:51:33.527 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:51:33 smithi089 bash[78452]: cluster 1685821892.2053757 mgr.y (mgr.34104) 409 : cluster 0 pgmap v335: 89 pgs: 89 active+clean; 1.9 GiB data, 7.3 GiB used, 708 GiB / 715 GiB avail; 118 MiB/s wr, 1.89k op/s 2023-06-03T19:51:33.528 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:51:33 smithi089 bash[78452]: audit 1685821893.3357673 mon.c (mon.2) 193 : audit 0 from='client.? 172.21.15.89:0/3030577776' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:51:33.528 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:51:33 smithi089 bash[78452]: cluster 1685821893.452585 mon.a (mon.0) 374 : cluster 1 osd.1 marked itself down and dead 2023-06-03T19:51:33.528 INFO:journalctl@ceph.osd.1.smithi089.stdout:Jun 03 19:51:33 smithi089 bash[40456]: debug 2023-06-03T19:51:33.445+0000 7fe843f6b700 -1 received signal: Terminated from /sbin/docker-init -- /usr/bin/ceph-osd -n osd.1 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug (PID: 1) UID: 0 2023-06-03T19:51:33.529 INFO:journalctl@ceph.osd.1.smithi089.stdout:Jun 03 19:51:33 smithi089 bash[40456]: debug 2023-06-03T19:51:33.445+0000 7fe843f6b700 -1 osd.1 99 *** Got signal Terminated *** 2023-06-03T19:51:33.529 INFO:journalctl@ceph.osd.1.smithi089.stdout:Jun 03 19:51:33 smithi089 bash[40456]: debug 2023-06-03T19:51:33.445+0000 7fe843f6b700 -1 osd.1 99 *** Immediate shutdown (osd_fast_shutdown=true) *** 2023-06-03T19:51:33.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:51:33 smithi164 bash[59462]: cluster 1685821892.2053757 mgr.y (mgr.34104) 409 : cluster 0 pgmap v335: 89 pgs: 89 active+clean; 1.9 GiB data, 7.3 GiB used, 708 GiB / 715 GiB avail; 118 MiB/s wr, 1.89k op/s 2023-06-03T19:51:33.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:51:33 smithi164 bash[59462]: audit 1685821893.3357673 mon.c (mon.2) 193 : audit 0 from='client.? 172.21.15.89:0/3030577776' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:51:33.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:51:33 smithi164 bash[59462]: cluster 1685821893.452585 mon.a (mon.0) 374 : cluster 1 osd.1 marked itself down and dead 2023-06-03T19:51:34.326 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 19 16 37606 37590 123.633 20 0.00237313 0.00774938 2023-06-03T19:51:34.779 INFO:journalctl@ceph.osd.1.smithi089.stdout:Jun 03 19:51:34 smithi089 bash[91529]: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd-1 2023-06-03T19:51:34.779 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:51:34 smithi089 bash[75068]: cluster 1685821893.5192688 mon.a (mon.0) 375 : cluster 3 Health check failed: 1 osds down (OSD_DOWN) 2023-06-03T19:51:34.779 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:51:34 smithi089 bash[75068]: cluster 1685821893.5277002 osd.7 (osd.7) 14 : cluster 3 failed to encode map e100 with expected crc 2023-06-03T19:51:34.779 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:51:34 smithi089 bash[75068]: cluster 1685821893.5331438 mon.a (mon.0) 376 : cluster 0 osdmap e100: 8 total, 7 up, 8 in 2023-06-03T19:51:34.780 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:51:34 smithi089 bash[75068]: cluster 1685821893.5335288 osd.5 (osd.5) 18 : cluster 3 failed to encode map e100 with expected crc 2023-06-03T19:51:34.780 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:51:34 smithi089 bash[78452]: cluster 1685821893.5192688 mon.a (mon.0) 375 : cluster 3 Health check failed: 1 osds down (OSD_DOWN) 2023-06-03T19:51:34.780 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:51:34 smithi089 bash[78452]: cluster 1685821893.5277002 osd.7 (osd.7) 14 : cluster 3 failed to encode map e100 with expected crc 2023-06-03T19:51:34.780 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:51:34 smithi089 bash[78452]: cluster 1685821893.5331438 mon.a (mon.0) 376 : cluster 0 osdmap e100: 8 total, 7 up, 8 in 2023-06-03T19:51:34.780 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:51:34 smithi089 bash[78452]: cluster 1685821893.5335288 osd.5 (osd.5) 18 : cluster 3 failed to encode map e100 with expected crc 2023-06-03T19:51:34.834 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:51:34 smithi164 bash[59462]: cluster 1685821893.5192688 mon.a (mon.0) 375 : cluster 3 Health check failed: 1 osds down (OSD_DOWN) 2023-06-03T19:51:34.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:51:34 smithi164 bash[59462]: cluster 1685821893.5277002 osd.7 (osd.7) 14 : cluster 3 failed to encode map e100 with expected crc 2023-06-03T19:51:34.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:51:34 smithi164 bash[59462]: cluster 1685821893.5331438 mon.a (mon.0) 376 : cluster 0 osdmap e100: 8 total, 7 up, 8 in 2023-06-03T19:51:34.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:51:34 smithi164 bash[59462]: cluster 1685821893.5335288 osd.5 (osd.5) 18 : cluster 3 failed to encode map e100 with expected crc 2023-06-03T19:51:35.326 INFO:tasks.radosbench.radosbench.0.smithi089.stdout:2023-06-03T19:51:35.322609+0000 min lat: 0.00131775 max lat: 1.04723 avg lat: 0.00820646 2023-06-03T19:51:35.327 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-06-03T19:51:35.327 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 20 16 38989 38973 121.773 86.4375 0.00596666 0.00820646 2023-06-03T19:51:35.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:51:35 smithi164 bash[59462]: cluster 1685821894.2062635 mgr.y (mgr.34104) 410 : cluster 0 pgmap v337: 89 pgs: 5 peering, 10 stale+active+clean, 74 active+clean; 2.1 GiB data, 8.2 GiB used, 707 GiB / 715 GiB avail; 104 MiB/s wr, 1.66k op/s 2023-06-03T19:51:35.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:51:35 smithi164 bash[59462]: cluster 1685821894.529529 mon.a (mon.0) 377 : cluster 3 Health check failed: Reduced data availability: 3 pgs inactive, 3 pgs peering (PG_AVAILABILITY) 2023-06-03T19:51:35.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:51:35 smithi164 bash[59462]: cluster 1685821894.535116 osd.6 (osd.6) 18 : cluster 3 failed to encode map e101 with expected crc 2023-06-03T19:51:35.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:51:35 smithi164 bash[59462]: cluster 1685821894.5361207 osd.5 (osd.5) 19 : cluster 3 failed to encode map e101 with expected crc 2023-06-03T19:51:35.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:51:35 smithi164 bash[59462]: cluster 1685821894.53737 osd.4 (osd.4) 20 : cluster 3 failed to encode map e101 with expected crc 2023-06-03T19:51:35.836 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:51:35 smithi164 bash[59462]: cluster 1685821894.537561 osd.7 (osd.7) 15 : cluster 3 failed to encode map e101 with expected crc 2023-06-03T19:51:35.836 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:51:35 smithi164 bash[59462]: cluster 1685821894.5378792 osd.6 (osd.6) 19 : cluster 3 failed to encode map e101 with expected crc 2023-06-03T19:51:35.836 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:51:35 smithi164 bash[59462]: cluster 1685821894.5399015 mon.a (mon.0) 378 : cluster 0 osdmap e101: 8 total, 7 up, 8 in 2023-06-03T19:51:35.836 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:51:35 smithi164 bash[59462]: cluster 1685821894.5404232 osd.3 (osd.3) 23 : cluster 3 failed to encode map e101 with expected crc 2023-06-03T19:51:35.836 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:51:35 smithi164 bash[59462]: cluster 1685821894.5404785 osd.7 (osd.7) 16 : cluster 3 failed to encode map e101 with expected crc 2023-06-03T19:51:35.836 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:51:35 smithi164 bash[59462]: cluster 1685821894.5435631 osd.2 (osd.2) 21 : cluster 3 failed to encode map e101 with expected crc 2023-06-03T19:51:35.836 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:51:35 smithi164 bash[59462]: cluster 1685821894.5443268 osd.5 (osd.5) 20 : cluster 3 failed to encode map e101 with expected crc 2023-06-03T19:51:35.837 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:51:35 smithi164 bash[59462]: cluster 1685821894.5456257 osd.2 (osd.2) 22 : cluster 3 failed to encode map e101 with expected crc 2023-06-03T19:51:35.837 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:51:35 smithi164 bash[59462]: cluster 1685821894.5480561 osd.5 (osd.5) 21 : cluster 3 failed to encode map e101 with expected crc 2023-06-03T19:51:36.029 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:51:35 smithi089 bash[78452]: cluster 1685821894.2062635 mgr.y (mgr.34104) 410 : cluster 0 pgmap v337: 89 pgs: 5 peering, 10 stale+active+clean, 74 active+clean; 2.1 GiB data, 8.2 GiB used, 707 GiB / 715 GiB avail; 104 MiB/s wr, 1.66k op/s 2023-06-03T19:51:36.029 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:51:35 smithi089 bash[78452]: cluster 1685821894.529529 mon.a (mon.0) 377 : cluster 3 Health check failed: Reduced data availability: 3 pgs inactive, 3 pgs peering (PG_AVAILABILITY) 2023-06-03T19:51:36.029 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:51:35 smithi089 bash[78452]: cluster 1685821894.535116 osd.6 (osd.6) 18 : cluster 3 failed to encode map e101 with expected crc 2023-06-03T19:51:36.030 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:51:35 smithi089 bash[78452]: cluster 1685821894.5361207 osd.5 (osd.5) 19 : cluster 3 failed to encode map e101 with expected crc 2023-06-03T19:51:36.030 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:51:35 smithi089 bash[78452]: cluster 1685821894.53737 osd.4 (osd.4) 20 : cluster 3 failed to encode map e101 with expected crc 2023-06-03T19:51:36.030 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:51:35 smithi089 bash[78452]: cluster 1685821894.537561 osd.7 (osd.7) 15 : cluster 3 failed to encode map e101 with expected crc 2023-06-03T19:51:36.030 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:51:35 smithi089 bash[78452]: cluster 1685821894.5378792 osd.6 (osd.6) 19 : cluster 3 failed to encode map e101 with expected crc 2023-06-03T19:51:36.030 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:51:35 smithi089 bash[78452]: cluster 1685821894.5399015 mon.a (mon.0) 378 : cluster 0 osdmap e101: 8 total, 7 up, 8 in 2023-06-03T19:51:36.030 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:51:35 smithi089 bash[78452]: cluster 1685821894.5404232 osd.3 (osd.3) 23 : cluster 3 failed to encode map e101 with expected crc 2023-06-03T19:51:36.030 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:51:35 smithi089 bash[78452]: cluster 1685821894.5404785 osd.7 (osd.7) 16 : cluster 3 failed to encode map e101 with expected crc 2023-06-03T19:51:36.030 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:51:35 smithi089 bash[78452]: cluster 1685821894.5435631 osd.2 (osd.2) 21 : cluster 3 failed to encode map e101 with expected crc 2023-06-03T19:51:36.031 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:51:35 smithi089 bash[78452]: cluster 1685821894.5443268 osd.5 (osd.5) 20 : cluster 3 failed to encode map e101 with expected crc 2023-06-03T19:51:36.031 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:51:35 smithi089 bash[78452]: cluster 1685821894.5456257 osd.2 (osd.2) 22 : cluster 3 failed to encode map e101 with expected crc 2023-06-03T19:51:36.031 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:51:35 smithi089 bash[78452]: cluster 1685821894.5480561 osd.5 (osd.5) 21 : cluster 3 failed to encode map e101 with expected crc 2023-06-03T19:51:36.031 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:51:35 smithi089 bash[75068]: cluster 1685821894.2062635 mgr.y (mgr.34104) 410 : cluster 0 pgmap v337: 89 pgs: 5 peering, 10 stale+active+clean, 74 active+clean; 2.1 GiB data, 8.2 GiB used, 707 GiB / 715 GiB avail; 104 MiB/s wr, 1.66k op/s 2023-06-03T19:51:36.031 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:51:35 smithi089 bash[75068]: cluster 1685821894.529529 mon.a (mon.0) 377 : cluster 3 Health check failed: Reduced data availability: 3 pgs inactive, 3 pgs peering (PG_AVAILABILITY) 2023-06-03T19:51:36.031 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:51:35 smithi089 bash[75068]: cluster 1685821894.535116 osd.6 (osd.6) 18 : cluster 3 failed to encode map e101 with expected crc 2023-06-03T19:51:36.032 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:51:35 smithi089 bash[75068]: cluster 1685821894.5361207 osd.5 (osd.5) 19 : cluster 3 failed to encode map e101 with expected crc 2023-06-03T19:51:36.032 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:51:35 smithi089 bash[75068]: cluster 1685821894.53737 osd.4 (osd.4) 20 : cluster 3 failed to encode map e101 with expected crc 2023-06-03T19:51:36.032 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:51:35 smithi089 bash[75068]: cluster 1685821894.537561 osd.7 (osd.7) 15 : cluster 3 failed to encode map e101 with expected crc 2023-06-03T19:51:36.032 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:51:35 smithi089 bash[75068]: cluster 1685821894.5378792 osd.6 (osd.6) 19 : cluster 3 failed to encode map e101 with expected crc 2023-06-03T19:51:36.032 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:51:35 smithi089 bash[75068]: cluster 1685821894.5399015 mon.a (mon.0) 378 : cluster 0 osdmap e101: 8 total, 7 up, 8 in 2023-06-03T19:51:36.032 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:51:35 smithi089 bash[75068]: cluster 1685821894.5404232 osd.3 (osd.3) 23 : cluster 3 failed to encode map e101 with expected crc 2023-06-03T19:51:36.032 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:51:35 smithi089 bash[75068]: cluster 1685821894.5404785 osd.7 (osd.7) 16 : cluster 3 failed to encode map e101 with expected crc 2023-06-03T19:51:36.033 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:51:35 smithi089 bash[75068]: cluster 1685821894.5435631 osd.2 (osd.2) 21 : cluster 3 failed to encode map e101 with expected crc 2023-06-03T19:51:36.033 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:51:35 smithi089 bash[75068]: cluster 1685821894.5443268 osd.5 (osd.5) 20 : cluster 3 failed to encode map e101 with expected crc 2023-06-03T19:51:36.033 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:51:35 smithi089 bash[75068]: cluster 1685821894.5456257 osd.2 (osd.2) 22 : cluster 3 failed to encode map e101 with expected crc 2023-06-03T19:51:36.033 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:51:35 smithi089 bash[75068]: cluster 1685821894.5480561 osd.5 (osd.5) 21 : cluster 3 failed to encode map e101 with expected crc 2023-06-03T19:51:36.327 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 21 16 40753 40737 121.223 110.25 0.00889209 0.0082442 2023-06-03T19:51:36.501 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 19:51:36 smithi089 bash[71163]: ::ffff:172.21.15.164 - - [03/Jun/2023:19:51:36] "GET /metrics HTTP/1.1" 200 207391 "" "Prometheus/2.33.4" 2023-06-03T19:51:36.779 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:51:36 smithi089 bash[75068]: audit 1685821895.8957753 mon.b (mon.1) 20 : audit 0 from='client.? 172.21.15.89:0/2271140533' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:51:36.779 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:51:36 smithi089 bash[78452]: audit 1685821895.8957753 mon.b (mon.1) 20 : audit 0 from='client.? 172.21.15.89:0/2271140533' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:51:36.834 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:51:36 smithi164 bash[59462]: audit 1685821895.8957753 mon.b (mon.1) 20 : audit 0 from='client.? 172.21.15.89:0/2271140533' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:51:37.327 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 22 16 42441 42425 120.508 105.5 0.00784768 0.00829261 2023-06-03T19:51:37.834 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:51:37 smithi164 bash[59462]: cluster 1685821896.2068694 mgr.y (mgr.34104) 411 : cluster 0 pgmap v339: 89 pgs: 2 active+undersized+degraded, 17 peering, 5 stale+active+clean, 2 active+undersized, 63 active+clean; 2.3 GiB data, 8.5 GiB used, 707 GiB / 715 GiB avail; 126 MiB/s wr, 2.02k op/s; 2651/112692 objects degraded (2.352%) 2023-06-03T19:51:37.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:51:37 smithi164 bash[59462]: cluster 1685821896.5252194 mon.a (mon.0) 379 : cluster 3 Health check failed: Degraded data redundancy: 2651/112692 objects degraded (2.352%), 2 pgs degraded (PG_DEGRADED) 2023-06-03T19:51:37.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:51:37 smithi164 bash[59462]: audit 1685821897.0877872 mon.a (mon.0) 380 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:51:37.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:51:37 smithi164 bash[59462]: audit 1685821897.090821 mon.c (mon.2) 194 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T19:51:38.029 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:51:37 smithi089 bash[75068]: cluster 1685821896.2068694 mgr.y (mgr.34104) 411 : cluster 0 pgmap v339: 89 pgs: 2 active+undersized+degraded, 17 peering, 5 stale+active+clean, 2 active+undersized, 63 active+clean; 2.3 GiB data, 8.5 GiB used, 707 GiB / 715 GiB avail; 126 MiB/s wr, 2.02k op/s; 2651/112692 objects degraded (2.352%) 2023-06-03T19:51:38.029 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:51:37 smithi089 bash[75068]: cluster 1685821896.5252194 mon.a (mon.0) 379 : cluster 3 Health check failed: Degraded data redundancy: 2651/112692 objects degraded (2.352%), 2 pgs degraded (PG_DEGRADED) 2023-06-03T19:51:38.029 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:51:37 smithi089 bash[75068]: audit 1685821897.0877872 mon.a (mon.0) 380 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:51:38.029 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:51:37 smithi089 bash[75068]: audit 1685821897.090821 mon.c (mon.2) 194 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T19:51:38.030 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:51:37 smithi089 bash[78452]: cluster 1685821896.2068694 mgr.y (mgr.34104) 411 : cluster 0 pgmap v339: 89 pgs: 2 active+undersized+degraded, 17 peering, 5 stale+active+clean, 2 active+undersized, 63 active+clean; 2.3 GiB data, 8.5 GiB used, 707 GiB / 715 GiB avail; 126 MiB/s wr, 2.02k op/s; 2651/112692 objects degraded (2.352%) 2023-06-03T19:51:38.030 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:51:37 smithi089 bash[78452]: cluster 1685821896.5252194 mon.a (mon.0) 379 : cluster 3 Health check failed: Degraded data redundancy: 2651/112692 objects degraded (2.352%), 2 pgs degraded (PG_DEGRADED) 2023-06-03T19:51:38.030 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:51:37 smithi089 bash[78452]: audit 1685821897.0877872 mon.a (mon.0) 380 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:51:38.030 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:51:37 smithi089 bash[78452]: audit 1685821897.090821 mon.c (mon.2) 194 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T19:51:38.327 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 23 16 44114 44098 119.815 104.562 0.00657484 0.00834033 2023-06-03T19:51:38.680 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:51:38 smithi089 bash[75068]: audit 1685821898.456505 mon.a (mon.0) 381 : audit 0 from='client.? 172.21.15.89:0/1575176571' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:51:38.681 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:51:38 smithi089 bash[78452]: audit 1685821898.456505 mon.a (mon.0) 381 : audit 0 from='client.? 172.21.15.89:0/1575176571' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:51:38.681 INFO:journalctl@ceph.osd.1.smithi089.stdout:Jun 03 19:51:38 smithi089 systemd[1]: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c@osd.1.service: Succeeded. 2023-06-03T19:51:38.682 INFO:journalctl@ceph.osd.1.smithi089.stdout:Jun 03 19:51:38 smithi089 systemd[1]: Stopped Ceph osd.1 for f1db3478-0243-11ee-9b1d-001a4aab830c. 2023-06-03T19:51:38.682 INFO:journalctl@ceph.osd.1.smithi089.stdout:Jun 03 19:51:38 smithi089 systemd[1]: Started Ceph osd.1 for f1db3478-0243-11ee-9b1d-001a4aab830c. 2023-06-03T19:51:38.834 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:51:38 smithi164 bash[59462]: audit 1685821898.456505 mon.a (mon.0) 381 : audit 0 from='client.? 172.21.15.89:0/1575176571' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:51:39.327 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 24 16 45583 45567 118.647 91.8125 0.0118067 0.00842295 2023-06-03T19:51:40.030 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:51:39 smithi089 bash[75068]: cluster 1685821898.207435 mgr.y (mgr.34104) 412 : cluster 0 pgmap v340: 89 pgs: 2 active+undersized+degraded, 17 peering, 5 stale+active+clean, 2 active+undersized, 63 active+clean; 2.3 GiB data, 8.5 GiB used, 707 GiB / 715 GiB avail; 67 MiB/s wr, 1.08k op/s; 2651/112692 objects degraded (2.352%) 2023-06-03T19:51:40.030 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:51:39 smithi089 bash[75068]: audit 1685821898.731912 mon.a (mon.0) 382 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:51:40.030 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:51:39 smithi089 bash[75068]: audit 1685821898.7489002 mon.a (mon.0) 383 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:51:40.030 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:51:39 smithi089 bash[75068]: audit 1685821898.7692208 mon.c (mon.2) 195 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-03T19:51:40.031 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:51:39 smithi089 bash[78452]: cluster 1685821898.207435 mgr.y (mgr.34104) 412 : cluster 0 pgmap v340: 89 pgs: 2 active+undersized+degraded, 17 peering, 5 stale+active+clean, 2 active+undersized, 63 active+clean; 2.3 GiB data, 8.5 GiB used, 707 GiB / 715 GiB avail; 67 MiB/s wr, 1.08k op/s; 2651/112692 objects degraded (2.352%) 2023-06-03T19:51:40.031 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:51:39 smithi089 bash[78452]: audit 1685821898.731912 mon.a (mon.0) 382 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:51:40.031 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:51:39 smithi089 bash[78452]: audit 1685821898.7489002 mon.a (mon.0) 383 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:51:40.031 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:51:39 smithi089 bash[78452]: audit 1685821898.7692208 mon.c (mon.2) 195 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-03T19:51:40.085 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:51:39 smithi164 bash[59462]: cluster 1685821898.207435 mgr.y (mgr.34104) 412 : cluster 0 pgmap v340: 89 pgs: 2 active+undersized+degraded, 17 peering, 5 stale+active+clean, 2 active+undersized, 63 active+clean; 2.3 GiB data, 8.5 GiB used, 707 GiB / 715 GiB avail; 67 MiB/s wr, 1.08k op/s; 2651/112692 objects degraded (2.352%) 2023-06-03T19:51:40.085 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:51:39 smithi164 bash[59462]: audit 1685821898.731912 mon.a (mon.0) 382 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:51:40.085 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:51:39 smithi164 bash[59462]: audit 1685821898.7489002 mon.a (mon.0) 383 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:51:40.085 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:51:39 smithi164 bash[59462]: audit 1685821898.7692208 mon.c (mon.2) 195 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-03T19:51:40.327 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 25 16 47188 47172 117.914 100.312 0.0102112 0.00847462 2023-06-03T19:51:41.029 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:51:40 smithi089 bash[75068]: cluster 1685821900.7295294 mon.a (mon.0) 384 : cluster 1 Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 5 pgs inactive, 5 pgs peering) 2023-06-03T19:51:41.029 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:51:40 smithi089 bash[78452]: cluster 1685821900.7295294 mon.a (mon.0) 384 : cluster 1 Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 5 pgs inactive, 5 pgs peering) 2023-06-03T19:51:41.084 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:51:40 smithi164 bash[59462]: cluster 1685821900.7295294 mon.a (mon.0) 384 : cluster 1 Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 5 pgs inactive, 5 pgs peering) 2023-06-03T19:51:41.327 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 26 16 48695 48679 117.001 94.1875 0.00829355 0.00854072 2023-06-03T19:51:42.029 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:51:41 smithi089 bash[75068]: cluster 1685821900.208397 mgr.y (mgr.34104) 413 : cluster 0 pgmap v341: 89 pgs: 10 active+undersized+degraded, 29 active+undersized, 50 active+clean; 2.7 GiB data, 9.4 GiB used, 706 GiB / 715 GiB avail; 101 MiB/s wr, 1.61k op/s; 19610/130362 objects degraded (15.043%) 2023-06-03T19:51:42.029 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:51:41 smithi089 bash[75068]: audit 1685821901.0059667 mon.b (mon.1) 21 : audit 0 from='client.? 172.21.15.89:0/941856335' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:51:42.030 INFO:journalctl@ceph.osd.1.smithi089.stdout:Jun 03 19:51:41 smithi089 bash[91794]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-1 2023-06-03T19:51:42.030 INFO:journalctl@ceph.osd.1.smithi089.stdout:Jun 03 19:51:41 smithi089 bash[91794]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-1 --no-mon-config --dev /dev/mapper/vg_nvme-lv_3 2023-06-03T19:51:42.031 INFO:journalctl@ceph.osd.1.smithi089.stdout:Jun 03 19:51:41 smithi089 bash[91794]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_3 2023-06-03T19:51:42.031 INFO:journalctl@ceph.osd.1.smithi089.stdout:Jun 03 19:51:41 smithi089 bash[91794]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-2 2023-06-03T19:51:42.031 INFO:journalctl@ceph.osd.1.smithi089.stdout:Jun 03 19:51:41 smithi089 bash[91794]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_3 /var/lib/ceph/osd/ceph-1/block 2023-06-03T19:51:42.032 INFO:journalctl@ceph.osd.1.smithi089.stdout:Jun 03 19:51:41 smithi089 bash[91794]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-1 2023-06-03T19:51:42.032 INFO:journalctl@ceph.osd.1.smithi089.stdout:Jun 03 19:51:41 smithi089 bash[91794]: --> ceph-volume raw activate successful for osd ID: 1 2023-06-03T19:51:42.032 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:51:41 smithi089 bash[78452]: cluster 1685821900.208397 mgr.y (mgr.34104) 413 : cluster 0 pgmap v341: 89 pgs: 10 active+undersized+degraded, 29 active+undersized, 50 active+clean; 2.7 GiB data, 9.4 GiB used, 706 GiB / 715 GiB avail; 101 MiB/s wr, 1.61k op/s; 19610/130362 objects degraded (15.043%) 2023-06-03T19:51:42.032 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:51:41 smithi089 bash[78452]: audit 1685821901.0059667 mon.b (mon.1) 21 : audit 0 from='client.? 172.21.15.89:0/941856335' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:51:42.085 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:51:41 smithi164 bash[59462]: cluster 1685821900.208397 mgr.y (mgr.34104) 413 : cluster 0 pgmap v341: 89 pgs: 10 active+undersized+degraded, 29 active+undersized, 50 active+clean; 2.7 GiB data, 9.4 GiB used, 706 GiB / 715 GiB avail; 101 MiB/s wr, 1.61k op/s; 19610/130362 objects degraded (15.043%) 2023-06-03T19:51:42.085 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:51:41 smithi164 bash[59462]: audit 1685821901.0059667 mon.b (mon.1) 21 : audit 0 from='client.? 172.21.15.89:0/941856335' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:51:42.327 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 27 16 50275 50259 116.325 98.75 0.0152279 0.0085903 2023-06-03T19:51:43.085 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:51:42 smithi164 bash[59462]: cluster 1685821902.318682 mon.a (mon.0) 385 : cluster 3 Health check update: Degraded data redundancy: 19610/130362 objects degraded (15.043%), 10 pgs degraded (PG_DEGRADED) 2023-06-03T19:51:43.279 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:51:42 smithi089 bash[78452]: cluster 1685821902.318682 mon.a (mon.0) 385 : cluster 3 Health check update: Degraded data redundancy: 19610/130362 objects degraded (15.043%), 10 pgs degraded (PG_DEGRADED) 2023-06-03T19:51:43.279 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:51:42 smithi089 bash[75068]: cluster 1685821902.318682 mon.a (mon.0) 385 : cluster 3 Health check update: Degraded data redundancy: 19610/130362 objects degraded (15.043%), 10 pgs degraded (PG_DEGRADED) 2023-06-03T19:51:43.332 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 28 16 51804 51788 115.577 95.5625 0.0237838 0.00864523 2023-06-03T19:51:44.029 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:51:43 smithi089 bash[75068]: cluster 1685821902.2090364 mgr.y (mgr.34104) 414 : cluster 0 pgmap v342: 89 pgs: 10 active+undersized+degraded, 29 active+undersized, 50 active+clean; 2.8 GiB data, 9.6 GiB used, 706 GiB / 715 GiB avail; 86 MiB/s wr, 1.38k op/s; 20102/136476 objects degraded (14.729%) 2023-06-03T19:51:44.029 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:51:43 smithi089 bash[75068]: audit 1685821903.5339625 mon.c (mon.2) 196 : audit 0 from='client.? 172.21.15.89:0/3165113422' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:51:44.029 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:51:43 smithi089 bash[78452]: cluster 1685821902.2090364 mgr.y (mgr.34104) 414 : cluster 0 pgmap v342: 89 pgs: 10 active+undersized+degraded, 29 active+undersized, 50 active+clean; 2.8 GiB data, 9.6 GiB used, 706 GiB / 715 GiB avail; 86 MiB/s wr, 1.38k op/s; 20102/136476 objects degraded (14.729%) 2023-06-03T19:51:44.030 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:51:43 smithi089 bash[78452]: audit 1685821903.5339625 mon.c (mon.2) 196 : audit 0 from='client.? 172.21.15.89:0/3165113422' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:51:44.085 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:51:43 smithi164 bash[59462]: cluster 1685821902.2090364 mgr.y (mgr.34104) 414 : cluster 0 pgmap v342: 89 pgs: 10 active+undersized+degraded, 29 active+undersized, 50 active+clean; 2.8 GiB data, 9.6 GiB used, 706 GiB / 715 GiB avail; 86 MiB/s wr, 1.38k op/s; 20102/136476 objects degraded (14.729%) 2023-06-03T19:51:44.085 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:51:43 smithi164 bash[59462]: audit 1685821903.5339625 mon.c (mon.2) 196 : audit 0 from='client.? 172.21.15.89:0/3165113422' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:51:44.329 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 29 16 53198 53182 114.596 87.125 0.00739179 0.00871845 2023-06-03T19:51:44.779 INFO:journalctl@ceph.osd.1.smithi089.stdout:Jun 03 19:51:44 smithi089 bash[92089]: debug 2023-06-03T19:51:44.633+0000 7f44685de700 1 -- 172.21.15.89:0/2831005727 <== mon.2 v2:172.21.15.89:3301/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x564e81dd7600 con 0x564e81de6c00 2023-06-03T19:51:44.779 INFO:journalctl@ceph.osd.1.smithi089.stdout:Jun 03 19:51:44 smithi089 bash[92089]: debug 2023-06-03T19:51:44.633+0000 7f44685de700 1 -- 172.21.15.89:0/2831005727 <== mon.2 v2:172.21.15.89:3301/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x564e81d8cd00 con 0x564e81de6c00 2023-06-03T19:51:44.828 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:51:44 smithi089 bash[75068]: cluster 1685821904.2098582 mgr.y (mgr.34104) 415 : cluster 0 pgmap v343: 89 pgs: 10 active+undersized+degraded, 29 active+undersized, 50 active+clean; 3.0 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 96 MiB/s wr, 1.53k op/s; 21076/148020 objects degraded (14.239%) 2023-06-03T19:51:45.084 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:51:44 smithi164 bash[59462]: cluster 1685821904.2098582 mgr.y (mgr.34104) 415 : cluster 0 pgmap v343: 89 pgs: 10 active+undersized+degraded, 29 active+undersized, 50 active+clean; 3.0 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 96 MiB/s wr, 1.53k op/s; 21076/148020 objects degraded (14.239%) 2023-06-03T19:51:45.199 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:51:44 smithi089 bash[78452]: cluster 1685821904.2098582 mgr.y (mgr.34104) 415 : cluster 0 pgmap v343: 89 pgs: 10 active+undersized+degraded, 29 active+undersized, 50 active+clean; 3.0 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 96 MiB/s wr, 1.53k op/s; 21076/148020 objects degraded (14.239%) 2023-06-03T19:51:45.329 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 30 16 54465 54449 113.415 79.1875 0.00443311 0.00881107 2023-06-03T19:51:45.529 INFO:journalctl@ceph.osd.1.smithi089.stdout:Jun 03 19:51:45 smithi089 bash[92089]: debug 2023-06-03T19:51:45.189+0000 7f4471764640 -1 Falling back to public interface 2023-06-03T19:51:46.329 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 31 16 55802 55786 112.452 83.5625 0.00866115 0.00888693 2023-06-03T19:51:46.529 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 19:51:46 smithi089 bash[71163]: ::ffff:172.21.15.164 - - [03/Jun/2023:19:51:46] "GET /metrics HTTP/1.1" 200 207508 "" "Prometheus/2.33.4" 2023-06-03T19:51:46.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:51:46 smithi164 bash[59462]: audit 1685821905.5678504 mon.a (mon.0) 386 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:51:46.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:51:46 smithi164 bash[59462]: audit 1685821905.5879562 mon.a (mon.0) 387 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:51:46.836 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:51:46 smithi164 bash[59462]: audit 1685821906.0905423 mon.c (mon.2) 197 : audit 0 from='client.? 172.21.15.89:0/505549458' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:51:47.029 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:51:46 smithi089 bash[78452]: audit 1685821905.5678504 mon.a (mon.0) 386 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:51:47.029 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:51:46 smithi089 bash[78452]: audit 1685821905.5879562 mon.a (mon.0) 387 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:51:47.030 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:51:46 smithi089 bash[78452]: audit 1685821906.0905423 mon.c (mon.2) 197 : audit 0 from='client.? 172.21.15.89:0/505549458' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:51:47.030 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:51:46 smithi089 bash[75068]: audit 1685821905.5678504 mon.a (mon.0) 386 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:51:47.030 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:51:46 smithi089 bash[75068]: audit 1685821905.5879562 mon.a (mon.0) 387 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:51:47.030 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:51:46 smithi089 bash[75068]: audit 1685821906.0905423 mon.c (mon.2) 197 : audit 0 from='client.? 172.21.15.89:0/505549458' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:51:47.329 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 32 16 57134 57118 111.539 83.25 0.00908214 0.00895989 2023-06-03T19:51:48.029 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:51:47 smithi089 bash[78452]: cluster 1685821906.2106137 mgr.y (mgr.34104) 416 : cluster 0 pgmap v344: 89 pgs: 10 active+undersized+degraded, 29 active+undersized, 50 active+clean; 3.2 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 93 MiB/s wr, 1.49k op/s; 23398/159120 objects degraded (14.705%) 2023-06-03T19:51:48.030 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:51:47 smithi089 bash[78452]: audit 1685821907.278525 mon.a (mon.0) 388 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:51:48.030 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:51:47 smithi089 bash[78452]: cluster 1685821907.321346 mon.a (mon.0) 389 : cluster 3 Health check update: Degraded data redundancy: 23398/159120 objects degraded (14.705%), 10 pgs degraded (PG_DEGRADED) 2023-06-03T19:51:48.030 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:51:47 smithi089 bash[75068]: cluster 1685821906.2106137 mgr.y (mgr.34104) 416 : cluster 0 pgmap v344: 89 pgs: 10 active+undersized+degraded, 29 active+undersized, 50 active+clean; 3.2 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 93 MiB/s wr, 1.49k op/s; 23398/159120 objects degraded (14.705%) 2023-06-03T19:51:48.031 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:51:47 smithi089 bash[75068]: audit 1685821907.278525 mon.a (mon.0) 388 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:51:48.031 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:51:47 smithi089 bash[75068]: cluster 1685821907.321346 mon.a (mon.0) 389 : cluster 3 Health check update: Degraded data redundancy: 23398/159120 objects degraded (14.705%), 10 pgs degraded (PG_DEGRADED) 2023-06-03T19:51:48.085 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:51:47 smithi164 bash[59462]: cluster 1685821906.2106137 mgr.y (mgr.34104) 416 : cluster 0 pgmap v344: 89 pgs: 10 active+undersized+degraded, 29 active+undersized, 50 active+clean; 3.2 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 93 MiB/s wr, 1.49k op/s; 23398/159120 objects degraded (14.705%) 2023-06-03T19:51:48.086 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:51:47 smithi164 bash[59462]: audit 1685821907.278525 mon.a (mon.0) 388 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:51:48.086 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:51:47 smithi164 bash[59462]: cluster 1685821907.321346 mon.a (mon.0) 389 : cluster 3 Health check update: Degraded data redundancy: 23398/159120 objects degraded (14.705%), 10 pgs degraded (PG_DEGRADED) 2023-06-03T19:51:48.329 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 33 16 58350 58334 110.462 76 0.00999215 0.00904524 2023-06-03T19:51:49.085 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:51:48 smithi164 bash[59462]: audit 1685821907.82873 mon.a (mon.0) 390 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:51:49.086 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:51:48 smithi164 bash[59462]: audit 1685821907.8422844 mon.a (mon.0) 391 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:51:49.086 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:51:48 smithi164 bash[59462]: cluster 1685821908.2110567 mgr.y (mgr.34104) 417 : cluster 0 pgmap v345: 89 pgs: 10 active+undersized+degraded, 29 active+undersized, 50 active+clean; 3.2 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 81 MiB/s wr, 1.29k op/s; 23398/159120 objects degraded (14.705%) 2023-06-03T19:51:49.086 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:51:48 smithi164 bash[59462]: audit 1685821908.7158313 mon.c (mon.2) 198 : audit 0 from='client.? 172.21.15.89:0/1506274936' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:51:49.279 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:51:48 smithi089 bash[75068]: audit 1685821907.82873 mon.a (mon.0) 390 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:51:49.279 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:51:48 smithi089 bash[75068]: audit 1685821907.8422844 mon.a (mon.0) 391 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:51:49.280 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:51:48 smithi089 bash[75068]: cluster 1685821908.2110567 mgr.y (mgr.34104) 417 : cluster 0 pgmap v345: 89 pgs: 10 active+undersized+degraded, 29 active+undersized, 50 active+clean; 3.2 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 81 MiB/s wr, 1.29k op/s; 23398/159120 objects degraded (14.705%) 2023-06-03T19:51:49.280 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:51:48 smithi089 bash[75068]: audit 1685821908.7158313 mon.c (mon.2) 198 : audit 0 from='client.? 172.21.15.89:0/1506274936' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:51:49.280 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:51:48 smithi089 bash[78452]: audit 1685821907.82873 mon.a (mon.0) 390 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:51:49.280 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:51:48 smithi089 bash[78452]: audit 1685821907.8422844 mon.a (mon.0) 391 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:51:49.280 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:51:48 smithi089 bash[78452]: cluster 1685821908.2110567 mgr.y (mgr.34104) 417 : cluster 0 pgmap v345: 89 pgs: 10 active+undersized+degraded, 29 active+undersized, 50 active+clean; 3.2 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 81 MiB/s wr, 1.29k op/s; 23398/159120 objects degraded (14.705%) 2023-06-03T19:51:49.280 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:51:48 smithi089 bash[78452]: audit 1685821908.7158313 mon.c (mon.2) 198 : audit 0 from='client.? 172.21.15.89:0/1506274936' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:51:49.329 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 34 16 59608 59592 109.525 78.625 0.012892 0.00912488 2023-06-03T19:51:50.279 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:51:49 smithi089 bash[78452]: audit 1685821908.8916066 mon.a (mon.0) 392 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:51:50.279 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:51:49 smithi089 bash[78452]: audit 1685821908.9068336 mon.a (mon.0) 393 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:51:50.280 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:51:49 smithi089 bash[75068]: audit 1685821908.8916066 mon.a (mon.0) 392 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:51:50.280 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:51:49 smithi089 bash[75068]: audit 1685821908.9068336 mon.a (mon.0) 393 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:51:50.329 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 35 16 60979 60963 108.844 85.6875 0.0155656 0.00918228 2023-06-03T19:51:50.335 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:51:49 smithi164 bash[59462]: audit 1685821908.8916066 mon.a (mon.0) 392 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:51:50.335 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:51:49 smithi164 bash[59462]: audit 1685821908.9068336 mon.a (mon.0) 393 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:51:51.279 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:51:50 smithi089 bash[75068]: cluster 1685821910.2118952 mgr.y (mgr.34104) 418 : cluster 0 pgmap v346: 89 pgs: 10 active+undersized+degraded, 29 active+undersized, 50 active+clean; 3.5 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 105 MiB/s wr, 1.69k op/s; 25788/173475 objects degraded (14.866%) 2023-06-03T19:51:51.279 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:51:50 smithi089 bash[78452]: cluster 1685821910.2118952 mgr.y (mgr.34104) 418 : cluster 0 pgmap v346: 89 pgs: 10 active+undersized+degraded, 29 active+undersized, 50 active+clean; 3.5 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 105 MiB/s wr, 1.69k op/s; 25788/173475 objects degraded (14.866%) 2023-06-03T19:51:51.329 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 36 16 62298 62282 108.11 82.4375 0.00715224 0.00924427 2023-06-03T19:51:51.334 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:51:50 smithi164 bash[59462]: cluster 1685821910.2118952 mgr.y (mgr.34104) 418 : cluster 0 pgmap v346: 89 pgs: 10 active+undersized+degraded, 29 active+undersized, 50 active+clean; 3.5 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 105 MiB/s wr, 1.69k op/s; 25788/173475 objects degraded (14.866%) 2023-06-03T19:51:52.279 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:51:51 smithi089 bash[75068]: audit 1685821911.2745063 mon.c (mon.2) 199 : audit 0 from='client.? 172.21.15.89:0/2644543005' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:51:52.279 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:51:51 smithi089 bash[78452]: audit 1685821911.2745063 mon.c (mon.2) 199 : audit 0 from='client.? 172.21.15.89:0/2644543005' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:51:52.330 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 37 16 63639 63623 107.454 83.8125 0.00973446 0.0093006 2023-06-03T19:51:52.335 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:51:51 smithi164 bash[59462]: audit 1685821911.2745063 mon.c (mon.2) 199 : audit 0 from='client.? 172.21.15.89:0/2644543005' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:51:53.330 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 38 16 64978 64962 106.828 83.6875 0.00928003 0.0093553 2023-06-03T19:51:53.529 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:51:53 smithi089 bash[75068]: audit 1685821912.0891776 mon.a (mon.0) 394 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:51:53.529 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:51:53 smithi089 bash[75068]: audit 1685821912.0932827 mon.c (mon.2) 200 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T19:51:53.530 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:51:53 smithi089 bash[75068]: cluster 1685821912.2125082 mgr.y (mgr.34104) 419 : cluster 0 pgmap v347: 89 pgs: 10 active+undersized+degraded, 29 active+undersized, 50 active+clean; 3.6 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 84 MiB/s wr, 1.34k op/s; 26228/178506 objects degraded (14.693%) 2023-06-03T19:51:53.530 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:51:53 smithi089 bash[75068]: cluster 1685821912.322714 mon.a (mon.0) 395 : cluster 3 Health check update: Degraded data redundancy: 25788/173475 objects degraded (14.866%), 10 pgs degraded (PG_DEGRADED) 2023-06-03T19:51:53.530 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:51:53 smithi089 bash[78452]: audit 1685821912.0891776 mon.a (mon.0) 394 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:51:53.530 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:51:53 smithi089 bash[78452]: audit 1685821912.0932827 mon.c (mon.2) 200 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T19:51:53.530 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:51:53 smithi089 bash[78452]: cluster 1685821912.2125082 mgr.y (mgr.34104) 419 : cluster 0 pgmap v347: 89 pgs: 10 active+undersized+degraded, 29 active+undersized, 50 active+clean; 3.6 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 84 MiB/s wr, 1.34k op/s; 26228/178506 objects degraded (14.693%) 2023-06-03T19:51:53.531 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:51:53 smithi089 bash[78452]: cluster 1685821912.322714 mon.a (mon.0) 395 : cluster 3 Health check update: Degraded data redundancy: 25788/173475 objects degraded (14.866%), 10 pgs degraded (PG_DEGRADED) 2023-06-03T19:51:53.585 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:51:53 smithi164 bash[59462]: audit 1685821912.0891776 mon.a (mon.0) 394 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:51:53.585 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:51:53 smithi164 bash[59462]: audit 1685821912.0932827 mon.c (mon.2) 200 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T19:51:53.586 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:51:53 smithi164 bash[59462]: cluster 1685821912.2125082 mgr.y (mgr.34104) 419 : cluster 0 pgmap v347: 89 pgs: 10 active+undersized+degraded, 29 active+undersized, 50 active+clean; 3.6 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 84 MiB/s wr, 1.34k op/s; 26228/178506 objects degraded (14.693%) 2023-06-03T19:51:53.586 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:51:53 smithi164 bash[59462]: cluster 1685821912.322714 mon.a (mon.0) 395 : cluster 3 Health check update: Degraded data redundancy: 25788/173475 objects degraded (14.866%), 10 pgs degraded (PG_DEGRADED) 2023-06-03T19:51:54.279 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:51:54 smithi089 bash[75068]: audit 1685821913.8199084 mon.b (mon.1) 22 : audit 0 from='client.? 172.21.15.89:0/821009532' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:51:54.279 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:51:54 smithi089 bash[78452]: audit 1685821913.8199084 mon.b (mon.1) 22 : audit 0 from='client.? 172.21.15.89:0/821009532' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:51:54.330 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 39 16 66202 66186 106.05 76.5 0.00671165 0.009424 2023-06-03T19:51:54.585 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:51:54 smithi164 bash[59462]: audit 1685821913.8199084 mon.b (mon.1) 22 : audit 0 from='client.? 172.21.15.89:0/821009532' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:51:55.330 INFO:tasks.radosbench.radosbench.0.smithi089.stdout:2023-06-03T19:51:55.326013+0000 min lat: 0.00131775 max lat: 1.04723 avg lat: 0.00949233 2023-06-03T19:51:55.330 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-06-03T19:51:55.331 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 40 15 67409 67394 105.286 75.5 0.0092237 0.00949233 2023-06-03T19:51:55.529 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:51:55 smithi089 bash[75068]: cluster 1685821914.213465 mgr.y (mgr.34104) 420 : cluster 0 pgmap v348: 89 pgs: 10 active+undersized+degraded, 29 active+undersized, 50 active+clean; 3.8 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 90 MiB/s wr, 1.45k op/s; 27087/188604 objects degraded (14.362%) 2023-06-03T19:51:55.529 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:51:55 smithi089 bash[78452]: cluster 1685821914.213465 mgr.y (mgr.34104) 420 : cluster 0 pgmap v348: 89 pgs: 10 active+undersized+degraded, 29 active+undersized, 50 active+clean; 3.8 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 90 MiB/s wr, 1.45k op/s; 27087/188604 objects degraded (14.362%) 2023-06-03T19:51:55.585 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:51:55 smithi164 bash[59462]: cluster 1685821914.213465 mgr.y (mgr.34104) 420 : cluster 0 pgmap v348: 89 pgs: 10 active+undersized+degraded, 29 active+undersized, 50 active+clean; 3.8 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 90 MiB/s wr, 1.45k op/s; 27087/188604 objects degraded (14.362%) 2023-06-03T19:51:56.330 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 41 16 68580 68564 104.502 73.125 0.00981444 0.00956244 2023-06-03T19:51:56.529 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 19:51:56 smithi089 bash[71163]: ::ffff:172.21.15.164 - - [03/Jun/2023:19:51:56] "GET /metrics HTTP/1.1" 200 207517 "" "Prometheus/2.33.4" 2023-06-03T19:51:57.279 INFO:journalctl@ceph.osd.1.smithi089.stdout:Jun 03 19:51:56 smithi089 bash[92089]: debug 2023-06-03T19:51:56.781+0000 7f4471764640 -1 osd.1 99 log_to_monitors true 2023-06-03T19:51:57.330 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 42 16 69812 69796 103.847 77 0.0106705 0.00962359 2023-06-03T19:51:57.561 INFO:journalctl@ceph.osd.1.smithi089.stdout:Jun 03 19:51:57 smithi089 bash[92089]: debug 2023-06-03T19:51:57.337+0000 7f4460bcc700 -1 osd.1 99 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2023-06-03T19:51:57.562 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:51:57 smithi089 bash[78452]: cluster 1685821916.21414 mgr.y (mgr.34104) 421 : cluster 0 pgmap v349: 89 pgs: 10 active+undersized+degraded, 29 active+undersized, 50 active+clean; 4.0 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 87 MiB/s wr, 1.39k op/s; 29142/198150 objects degraded (14.707%) 2023-06-03T19:51:57.562 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:51:57 smithi089 bash[78452]: audit 1685821916.3889403 mon.a (mon.0) 396 : audit 0 from='client.? 172.21.15.89:0/919272783' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:51:57.562 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:51:57 smithi089 bash[78452]: audit 1685821916.7931297 mon.c (mon.2) 201 : audit 1 from='osd.1 [v2:172.21.15.89:6810/2092956116,v1:172.21.15.89:6811/2092956116]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2023-06-03T19:51:57.562 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:51:57 smithi089 bash[78452]: audit 1685821916.7938282 mon.a (mon.0) 397 : audit 1 from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2023-06-03T19:51:57.563 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:51:57 smithi089 bash[75068]: cluster 1685821916.21414 mgr.y (mgr.34104) 421 : cluster 0 pgmap v349: 89 pgs: 10 active+undersized+degraded, 29 active+undersized, 50 active+clean; 4.0 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 87 MiB/s wr, 1.39k op/s; 29142/198150 objects degraded (14.707%) 2023-06-03T19:51:57.563 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:51:57 smithi089 bash[75068]: audit 1685821916.3889403 mon.a (mon.0) 396 : audit 0 from='client.? 172.21.15.89:0/919272783' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:51:57.563 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:51:57 smithi089 bash[75068]: audit 1685821916.7931297 mon.c (mon.2) 201 : audit 1 from='osd.1 [v2:172.21.15.89:6810/2092956116,v1:172.21.15.89:6811/2092956116]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2023-06-03T19:51:57.563 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:51:57 smithi089 bash[75068]: audit 1685821916.7938282 mon.a (mon.0) 397 : audit 1 from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2023-06-03T19:51:57.585 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:51:57 smithi164 bash[59462]: cluster 1685821916.21414 mgr.y (mgr.34104) 421 : cluster 0 pgmap v349: 89 pgs: 10 active+undersized+degraded, 29 active+undersized, 50 active+clean; 4.0 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 87 MiB/s wr, 1.39k op/s; 29142/198150 objects degraded (14.707%) 2023-06-03T19:51:57.585 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:51:57 smithi164 bash[59462]: audit 1685821916.3889403 mon.a (mon.0) 396 : audit 0 from='client.? 172.21.15.89:0/919272783' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:51:57.585 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:51:57 smithi164 bash[59462]: audit 1685821916.7931297 mon.c (mon.2) 201 : audit 1 from='osd.1 [v2:172.21.15.89:6810/2092956116,v1:172.21.15.89:6811/2092956116]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2023-06-03T19:51:57.586 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:51:57 smithi164 bash[59462]: audit 1685821916.7938282 mon.a (mon.0) 397 : audit 1 from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2023-06-03T19:51:58.330 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 43 16 71099 71083 103.302 80.4375 0.0202676 0.00967462 2023-06-03T19:51:58.585 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:51:58 smithi164 bash[59462]: cluster 1685821917.2717288 osd.6 (osd.6) 20 : cluster 3 failed to encode map e102 with expected crc 2023-06-03T19:51:58.585 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:51:58 smithi164 bash[59462]: audit 1685821917.277474 mon.a (mon.0) 398 : audit 1 from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2023-06-03T19:51:58.585 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:51:58 smithi164 bash[59462]: cluster 1685821917.2775908 mon.a (mon.0) 399 : cluster 0 osdmap e102: 8 total, 7 up, 8 in 2023-06-03T19:51:58.585 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:51:58 smithi164 bash[59462]: cluster 1685821917.2781656 osd.2 (osd.2) 23 : cluster 3 failed to encode map e102 with expected crc 2023-06-03T19:51:58.586 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:51:58 smithi164 bash[59462]: audit 1685821917.2820659 mon.c (mon.2) 202 : audit 1 from='osd.1 [v2:172.21.15.89:6810/2092956116,v1:172.21.15.89:6811/2092956116]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi089", "root=default"]}]: dispatch 2023-06-03T19:51:58.586 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:51:58 smithi164 bash[59462]: audit 1685821917.2913465 mon.a (mon.0) 400 : audit 1 from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi089", "root=default"]}]: dispatch 2023-06-03T19:51:58.586 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:51:58 smithi164 bash[59462]: cluster 1685821917.32395 mon.a (mon.0) 401 : cluster 3 Health check update: Degraded data redundancy: 29142/198150 objects degraded (14.707%), 10 pgs degraded (PG_DEGRADED) 2023-06-03T19:51:58.750 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:51:58 smithi089 bash[78452]: cluster 1685821917.2717288 osd.6 (osd.6) 20 : cluster 3 failed to encode map e102 with expected crc 2023-06-03T19:51:58.751 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:51:58 smithi089 bash[78452]: audit 1685821917.277474 mon.a (mon.0) 398 : audit 1 from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2023-06-03T19:51:58.751 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:51:58 smithi089 bash[78452]: cluster 1685821917.2775908 mon.a (mon.0) 399 : cluster 0 osdmap e102: 8 total, 7 up, 8 in 2023-06-03T19:51:58.751 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:51:58 smithi089 bash[78452]: cluster 1685821917.2781656 osd.2 (osd.2) 23 : cluster 3 failed to encode map e102 with expected crc 2023-06-03T19:51:58.751 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:51:58 smithi089 bash[78452]: audit 1685821917.2820659 mon.c (mon.2) 202 : audit 1 from='osd.1 [v2:172.21.15.89:6810/2092956116,v1:172.21.15.89:6811/2092956116]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi089", "root=default"]}]: dispatch 2023-06-03T19:51:58.751 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:51:58 smithi089 bash[78452]: audit 1685821917.2913465 mon.a (mon.0) 400 : audit 1 from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi089", "root=default"]}]: dispatch 2023-06-03T19:51:58.751 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:51:58 smithi089 bash[78452]: cluster 1685821917.32395 mon.a (mon.0) 401 : cluster 3 Health check update: Degraded data redundancy: 29142/198150 objects degraded (14.707%), 10 pgs degraded (PG_DEGRADED) 2023-06-03T19:51:58.752 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:51:58 smithi089 bash[75068]: cluster 1685821917.2717288 osd.6 (osd.6) 20 : cluster 3 failed to encode map e102 with expected crc 2023-06-03T19:51:58.752 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:51:58 smithi089 bash[75068]: audit 1685821917.277474 mon.a (mon.0) 398 : audit 1 from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2023-06-03T19:51:58.752 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:51:58 smithi089 bash[75068]: cluster 1685821917.2775908 mon.a (mon.0) 399 : cluster 0 osdmap e102: 8 total, 7 up, 8 in 2023-06-03T19:51:58.752 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:51:58 smithi089 bash[75068]: cluster 1685821917.2781656 osd.2 (osd.2) 23 : cluster 3 failed to encode map e102 with expected crc 2023-06-03T19:51:58.753 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:51:58 smithi089 bash[75068]: audit 1685821917.2820659 mon.c (mon.2) 202 : audit 1 from='osd.1 [v2:172.21.15.89:6810/2092956116,v1:172.21.15.89:6811/2092956116]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi089", "root=default"]}]: dispatch 2023-06-03T19:51:58.753 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:51:58 smithi089 bash[75068]: audit 1685821917.2913465 mon.a (mon.0) 400 : audit 1 from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi089", "root=default"]}]: dispatch 2023-06-03T19:51:58.753 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:51:58 smithi089 bash[75068]: cluster 1685821917.32395 mon.a (mon.0) 401 : cluster 3 Health check update: Degraded data redundancy: 29142/198150 objects degraded (14.707%), 10 pgs degraded (PG_DEGRADED) 2023-06-03T19:51:59.330 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 44 16 71148 71132 101.024 3.0625 0.00390183 0.00967686 2023-06-03T19:51:59.585 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:51:59 smithi164 bash[59462]: cluster 1685821918.2146833 mgr.y (mgr.34104) 422 : cluster 0 pgmap v351: 89 pgs: 10 active+undersized+degraded, 29 active+undersized, 50 active+clean; 4.0 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 81 MiB/s wr, 1.30k op/s; 29142/198150 objects degraded (14.707%) 2023-06-03T19:51:59.585 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:51:59 smithi164 bash[59462]: cluster 1685821918.3279493 mon.a (mon.0) 402 : cluster 1 Health check cleared: OSD_DOWN (was: 1 osds down) 2023-06-03T19:51:59.586 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:51:59 smithi164 bash[59462]: cluster 1685821918.3311796 osd.3 (osd.3) 24 : cluster 3 failed to encode map e103 with expected crc 2023-06-03T19:51:59.586 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:51:59 smithi164 bash[59462]: cluster 1685821918.3339295 mon.a (mon.0) 403 : cluster 1 osd.1 [v2:172.21.15.89:6810/2092956116,v1:172.21.15.89:6811/2092956116] boot 2023-06-03T19:51:59.586 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:51:59 smithi164 bash[59462]: cluster 1685821918.3341198 mon.a (mon.0) 404 : cluster 0 osdmap e103: 8 total, 8 up, 8 in 2023-06-03T19:51:59.586 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:51:59 smithi164 bash[59462]: cluster 1685821918.335515 osd.2 (osd.2) 24 : cluster 3 failed to encode map e103 with expected crc 2023-06-03T19:51:59.586 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:51:59 smithi164 bash[59462]: audit 1685821918.3371387 mon.c (mon.2) 203 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-06-03T19:51:59.586 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:51:59 smithi164 bash[59462]: audit 1685821918.765977 mon.c (mon.2) 204 : audit 0 from='client.? 172.21.15.89:0/1659147531' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:51:59.780 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:51:59 smithi089 bash[75068]: cluster 1685821918.2146833 mgr.y (mgr.34104) 422 : cluster 0 pgmap v351: 89 pgs: 10 active+undersized+degraded, 29 active+undersized, 50 active+clean; 4.0 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 81 MiB/s wr, 1.30k op/s; 29142/198150 objects degraded (14.707%) 2023-06-03T19:51:59.780 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:51:59 smithi089 bash[75068]: cluster 1685821918.3279493 mon.a (mon.0) 402 : cluster 1 Health check cleared: OSD_DOWN (was: 1 osds down) 2023-06-03T19:51:59.781 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:51:59 smithi089 bash[75068]: cluster 1685821918.3311796 osd.3 (osd.3) 24 : cluster 3 failed to encode map e103 with expected crc 2023-06-03T19:51:59.781 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:51:59 smithi089 bash[75068]: cluster 1685821918.3339295 mon.a (mon.0) 403 : cluster 1 osd.1 [v2:172.21.15.89:6810/2092956116,v1:172.21.15.89:6811/2092956116] boot 2023-06-03T19:51:59.781 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:51:59 smithi089 bash[75068]: cluster 1685821918.3341198 mon.a (mon.0) 404 : cluster 0 osdmap e103: 8 total, 8 up, 8 in 2023-06-03T19:51:59.782 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:51:59 smithi089 bash[75068]: cluster 1685821918.335515 osd.2 (osd.2) 24 : cluster 3 failed to encode map e103 with expected crc 2023-06-03T19:51:59.782 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:51:59 smithi089 bash[75068]: audit 1685821918.3371387 mon.c (mon.2) 203 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-06-03T19:51:59.782 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:51:59 smithi089 bash[75068]: audit 1685821918.765977 mon.c (mon.2) 204 : audit 0 from='client.? 172.21.15.89:0/1659147531' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:51:59.783 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:51:59 smithi089 bash[78452]: cluster 1685821918.2146833 mgr.y (mgr.34104) 422 : cluster 0 pgmap v351: 89 pgs: 10 active+undersized+degraded, 29 active+undersized, 50 active+clean; 4.0 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 81 MiB/s wr, 1.30k op/s; 29142/198150 objects degraded (14.707%) 2023-06-03T19:51:59.783 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:51:59 smithi089 bash[78452]: cluster 1685821918.3279493 mon.a (mon.0) 402 : cluster 1 Health check cleared: OSD_DOWN (was: 1 osds down) 2023-06-03T19:51:59.783 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:51:59 smithi089 bash[78452]: cluster 1685821918.3311796 osd.3 (osd.3) 24 : cluster 3 failed to encode map e103 with expected crc 2023-06-03T19:51:59.783 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:51:59 smithi089 bash[78452]: cluster 1685821918.3339295 mon.a (mon.0) 403 : cluster 1 osd.1 [v2:172.21.15.89:6810/2092956116,v1:172.21.15.89:6811/2092956116] boot 2023-06-03T19:51:59.784 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:51:59 smithi089 bash[78452]: cluster 1685821918.3341198 mon.a (mon.0) 404 : cluster 0 osdmap e103: 8 total, 8 up, 8 in 2023-06-03T19:51:59.784 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:51:59 smithi089 bash[78452]: cluster 1685821918.335515 osd.2 (osd.2) 24 : cluster 3 failed to encode map e103 with expected crc 2023-06-03T19:51:59.784 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:51:59 smithi089 bash[78452]: audit 1685821918.3371387 mon.c (mon.2) 203 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-06-03T19:51:59.785 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:51:59 smithi089 bash[78452]: audit 1685821918.765977 mon.c (mon.2) 204 : audit 0 from='client.? 172.21.15.89:0/1659147531' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:52:00.331 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 45 16 71148 71132 98.7791 0 - 0.00967686 2023-06-03T19:52:00.529 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:52:00 smithi089 bash[78452]: cluster 1685821919.3394196 osd.6 (osd.6) 21 : cluster 3 failed to encode map e104 with expected crc 2023-06-03T19:52:00.529 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:52:00 smithi089 bash[78452]: cluster 1685821919.341892 osd.4 (osd.4) 21 : cluster 3 failed to encode map e104 with expected crc 2023-06-03T19:52:00.529 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:52:00 smithi089 bash[78452]: cluster 1685821919.3419325 osd.7 (osd.7) 17 : cluster 3 failed to encode map e104 with expected crc 2023-06-03T19:52:00.529 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:52:00 smithi089 bash[78452]: cluster 1685821919.343152 osd.6 (osd.6) 22 : cluster 3 failed to encode map e104 with expected crc 2023-06-03T19:52:00.530 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:52:00 smithi089 bash[78452]: cluster 1685821919.34327 osd.2 (osd.2) 25 : cluster 3 failed to encode map e104 with expected crc 2023-06-03T19:52:00.530 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:52:00 smithi089 bash[78452]: cluster 1685821919.3440852 mon.a (mon.0) 405 : cluster 0 osdmap e104: 8 total, 8 up, 8 in 2023-06-03T19:52:00.530 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:52:00 smithi089 bash[78452]: cluster 1685821919.3444493 osd.3 (osd.3) 25 : cluster 3 failed to encode map e104 with expected crc 2023-06-03T19:52:00.530 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:52:00 smithi089 bash[78452]: cluster 1685821919.3469987 osd.5 (osd.5) 22 : cluster 3 failed to encode map e104 with expected crc 2023-06-03T19:52:00.530 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:52:00 smithi089 bash[78452]: cluster 1685821919.3619237 osd.2 (osd.2) 26 : cluster 3 failed to encode map e104 with expected crc 2023-06-03T19:52:00.531 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:52:00 smithi089 bash[75068]: cluster 1685821919.3394196 osd.6 (osd.6) 21 : cluster 3 failed to encode map e104 with expected crc 2023-06-03T19:52:00.531 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:52:00 smithi089 bash[75068]: cluster 1685821919.341892 osd.4 (osd.4) 21 : cluster 3 failed to encode map e104 with expected crc 2023-06-03T19:52:00.531 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:52:00 smithi089 bash[75068]: cluster 1685821919.3419325 osd.7 (osd.7) 17 : cluster 3 failed to encode map e104 with expected crc 2023-06-03T19:52:00.531 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:52:00 smithi089 bash[75068]: cluster 1685821919.343152 osd.6 (osd.6) 22 : cluster 3 failed to encode map e104 with expected crc 2023-06-03T19:52:00.531 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:52:00 smithi089 bash[75068]: cluster 1685821919.34327 osd.2 (osd.2) 25 : cluster 3 failed to encode map e104 with expected crc 2023-06-03T19:52:00.531 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:52:00 smithi089 bash[75068]: cluster 1685821919.3440852 mon.a (mon.0) 405 : cluster 0 osdmap e104: 8 total, 8 up, 8 in 2023-06-03T19:52:00.532 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:52:00 smithi089 bash[75068]: cluster 1685821919.3444493 osd.3 (osd.3) 25 : cluster 3 failed to encode map e104 with expected crc 2023-06-03T19:52:00.532 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:52:00 smithi089 bash[75068]: cluster 1685821919.3469987 osd.5 (osd.5) 22 : cluster 3 failed to encode map e104 with expected crc 2023-06-03T19:52:00.532 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:52:00 smithi089 bash[75068]: cluster 1685821919.3619237 osd.2 (osd.2) 26 : cluster 3 failed to encode map e104 with expected crc 2023-06-03T19:52:00.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:52:00 smithi164 bash[59462]: cluster 1685821919.3394196 osd.6 (osd.6) 21 : cluster 3 failed to encode map e104 with expected crc 2023-06-03T19:52:00.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:52:00 smithi164 bash[59462]: cluster 1685821919.341892 osd.4 (osd.4) 21 : cluster 3 failed to encode map e104 with expected crc 2023-06-03T19:52:00.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:52:00 smithi164 bash[59462]: cluster 1685821919.3419325 osd.7 (osd.7) 17 : cluster 3 failed to encode map e104 with expected crc 2023-06-03T19:52:00.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:52:00 smithi164 bash[59462]: cluster 1685821919.343152 osd.6 (osd.6) 22 : cluster 3 failed to encode map e104 with expected crc 2023-06-03T19:52:00.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:52:00 smithi164 bash[59462]: cluster 1685821919.34327 osd.2 (osd.2) 25 : cluster 3 failed to encode map e104 with expected crc 2023-06-03T19:52:00.836 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:52:00 smithi164 bash[59462]: cluster 1685821919.3440852 mon.a (mon.0) 405 : cluster 0 osdmap e104: 8 total, 8 up, 8 in 2023-06-03T19:52:00.836 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:52:00 smithi164 bash[59462]: cluster 1685821919.3444493 osd.3 (osd.3) 25 : cluster 3 failed to encode map e104 with expected crc 2023-06-03T19:52:00.836 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:52:00 smithi164 bash[59462]: cluster 1685821919.3469987 osd.5 (osd.5) 22 : cluster 3 failed to encode map e104 with expected crc 2023-06-03T19:52:00.836 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:52:00 smithi164 bash[59462]: cluster 1685821919.3619237 osd.2 (osd.2) 26 : cluster 3 failed to encode map e104 with expected crc 2023-06-03T19:52:01.331 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 46 16 71742 71726 97.4388 18.5625 0.0108852 0.0102556 2023-06-03T19:52:01.779 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:52:01 smithi089 bash[78452]: cluster 1685821920.215561 mgr.y (mgr.34104) 423 : cluster 0 pgmap v354: 89 pgs: 4 remapped+peering, 6 activating, 3 peering, 5 active+undersized+degraded, 21 active+undersized, 50 active+clean; 4.3 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 74 MiB/s wr, 1.19k op/s; 12981/210051 objects degraded (6.180%) 2023-06-03T19:52:01.779 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:52:01 smithi089 bash[78452]: cluster 1685821920.3631573 osd.3 (osd.3) 26 : cluster 3 failed to encode map e105 with expected crc 2023-06-03T19:52:01.779 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:52:01 smithi089 bash[78452]: cluster 1685821920.3658612 osd.5 (osd.5) 23 : cluster 3 failed to encode map e105 with expected crc 2023-06-03T19:52:01.779 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:52:01 smithi089 bash[78452]: cluster 1685821920.3667166 mon.a (mon.0) 406 : cluster 0 osdmap e105: 8 total, 8 up, 8 in 2023-06-03T19:52:01.779 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:52:01 smithi089 bash[78452]: cluster 1685821920.3677678 osd.2 (osd.2) 27 : cluster 3 failed to encode map e105 with expected crc 2023-06-03T19:52:01.780 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:52:01 smithi089 bash[78452]: cluster 1685821920.368223 osd.3 (osd.3) 27 : cluster 3 failed to encode map e105 with expected crc 2023-06-03T19:52:01.780 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:52:01 smithi089 bash[78452]: cluster 1685821920.3684752 osd.5 (osd.5) 24 : cluster 3 failed to encode map e105 with expected crc 2023-06-03T19:52:01.780 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:52:01 smithi089 bash[78452]: cluster 1685821920.3697932 osd.2 (osd.2) 28 : cluster 3 failed to encode map e105 with expected crc 2023-06-03T19:52:01.780 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:52:01 smithi089 bash[78452]: cluster 1685821920.3710725 osd.3 (osd.3) 28 : cluster 3 failed to encode map e105 with expected crc 2023-06-03T19:52:01.780 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:52:01 smithi089 bash[78452]: audit 1685821921.3377614 mon.c (mon.2) 205 : audit 0 from='client.? 172.21.15.89:0/3545758544' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:52:01.780 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:52:01 smithi089 bash[75068]: cluster 1685821920.215561 mgr.y (mgr.34104) 423 : cluster 0 pgmap v354: 89 pgs: 4 remapped+peering, 6 activating, 3 peering, 5 active+undersized+degraded, 21 active+undersized, 50 active+clean; 4.3 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 74 MiB/s wr, 1.19k op/s; 12981/210051 objects degraded (6.180%) 2023-06-03T19:52:01.781 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:52:01 smithi089 bash[75068]: cluster 1685821920.3631573 osd.3 (osd.3) 26 : cluster 3 failed to encode map e105 with expected crc 2023-06-03T19:52:01.781 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:52:01 smithi089 bash[75068]: cluster 1685821920.3658612 osd.5 (osd.5) 23 : cluster 3 failed to encode map e105 with expected crc 2023-06-03T19:52:01.781 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:52:01 smithi089 bash[75068]: cluster 1685821920.3667166 mon.a (mon.0) 406 : cluster 0 osdmap e105: 8 total, 8 up, 8 in 2023-06-03T19:52:01.781 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:52:01 smithi089 bash[75068]: cluster 1685821920.3677678 osd.2 (osd.2) 27 : cluster 3 failed to encode map e105 with expected crc 2023-06-03T19:52:01.781 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:52:01 smithi089 bash[75068]: cluster 1685821920.368223 osd.3 (osd.3) 27 : cluster 3 failed to encode map e105 with expected crc 2023-06-03T19:52:01.781 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:52:01 smithi089 bash[75068]: cluster 1685821920.3684752 osd.5 (osd.5) 24 : cluster 3 failed to encode map e105 with expected crc 2023-06-03T19:52:01.781 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:52:01 smithi089 bash[75068]: cluster 1685821920.3697932 osd.2 (osd.2) 28 : cluster 3 failed to encode map e105 with expected crc 2023-06-03T19:52:01.781 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:52:01 smithi089 bash[75068]: cluster 1685821920.3710725 osd.3 (osd.3) 28 : cluster 3 failed to encode map e105 with expected crc 2023-06-03T19:52:01.782 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:52:01 smithi089 bash[75068]: audit 1685821921.3377614 mon.c (mon.2) 205 : audit 0 from='client.? 172.21.15.89:0/3545758544' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:52:01.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:52:01 smithi164 bash[59462]: cluster 1685821920.215561 mgr.y (mgr.34104) 423 : cluster 0 pgmap v354: 89 pgs: 4 remapped+peering, 6 activating, 3 peering, 5 active+undersized+degraded, 21 active+undersized, 50 active+clean; 4.3 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 74 MiB/s wr, 1.19k op/s; 12981/210051 objects degraded (6.180%) 2023-06-03T19:52:01.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:52:01 smithi164 bash[59462]: cluster 1685821920.3631573 osd.3 (osd.3) 26 : cluster 3 failed to encode map e105 with expected crc 2023-06-03T19:52:01.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:52:01 smithi164 bash[59462]: cluster 1685821920.3658612 osd.5 (osd.5) 23 : cluster 3 failed to encode map e105 with expected crc 2023-06-03T19:52:01.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:52:01 smithi164 bash[59462]: cluster 1685821920.3667166 mon.a (mon.0) 406 : cluster 0 osdmap e105: 8 total, 8 up, 8 in 2023-06-03T19:52:01.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:52:01 smithi164 bash[59462]: cluster 1685821920.3677678 osd.2 (osd.2) 27 : cluster 3 failed to encode map e105 with expected crc 2023-06-03T19:52:01.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:52:01 smithi164 bash[59462]: cluster 1685821920.368223 osd.3 (osd.3) 27 : cluster 3 failed to encode map e105 with expected crc 2023-06-03T19:52:01.836 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:52:01 smithi164 bash[59462]: cluster 1685821920.3684752 osd.5 (osd.5) 24 : cluster 3 failed to encode map e105 with expected crc 2023-06-03T19:52:01.836 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:52:01 smithi164 bash[59462]: cluster 1685821920.3697932 osd.2 (osd.2) 28 : cluster 3 failed to encode map e105 with expected crc 2023-06-03T19:52:01.836 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:52:01 smithi164 bash[59462]: cluster 1685821920.3710725 osd.3 (osd.3) 28 : cluster 3 failed to encode map e105 with expected crc 2023-06-03T19:52:01.836 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:52:01 smithi164 bash[59462]: audit 1685821921.3377614 mon.c (mon.2) 205 : audit 0 from='client.? 172.21.15.89:0/3545758544' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:52:02.331 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 47 16 72596 72580 96.5012 53.375 0.00842688 0.0103539 2023-06-03T19:52:02.779 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:52:02 smithi089 bash[78452]: cluster 1685821920.5106542 osd.7 (osd.7) 18 : cluster 3 failed to encode map e105 with expected crc 2023-06-03T19:52:02.779 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:52:02 smithi089 bash[78452]: cluster 1685821922.3251605 mon.a (mon.0) 407 : cluster 3 Health check update: Degraded data redundancy: 12981/210051 objects degraded (6.180%), 5 pgs degraded (PG_DEGRADED) 2023-06-03T19:52:02.780 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:52:02 smithi089 bash[75068]: cluster 1685821920.5106542 osd.7 (osd.7) 18 : cluster 3 failed to encode map e105 with expected crc 2023-06-03T19:52:02.780 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:52:02 smithi089 bash[75068]: cluster 1685821922.3251605 mon.a (mon.0) 407 : cluster 3 Health check update: Degraded data redundancy: 12981/210051 objects degraded (6.180%), 5 pgs degraded (PG_DEGRADED) 2023-06-03T19:52:02.846 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:52:02 smithi164 bash[59462]: cluster 1685821920.5106542 osd.7 (osd.7) 18 : cluster 3 failed to encode map e105 with expected crc 2023-06-03T19:52:02.847 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:52:02 smithi164 bash[59462]: cluster 1685821922.3251605 mon.a (mon.0) 407 : cluster 3 Health check update: Degraded data redundancy: 12981/210051 objects degraded (6.180%), 5 pgs degraded (PG_DEGRADED) 2023-06-03T19:52:03.279 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 19:52:02 smithi089 bash[71163]: debug 2023-06-03T19:52:02.833+0000 7fe48642e700 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (9 PGs are or would become offline) 2023-06-03T19:52:03.331 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 48 16 73485 73469 95.6483 55.5625 0.0125539 0.0104499 2023-06-03T19:52:04.029 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:52:03 smithi089 bash[75068]: cluster 1685821922.2162282 mgr.y (mgr.34104) 424 : cluster 0 pgmap v356: 89 pgs: 52 active+clean, 20 active+undersized, 1 active+recovering+undersized+degraded+remapped, 3 active+undersized+degraded, 3 peering, 6 activating, 4 remapped+peering; 4.3 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 61 MiB/s wr, 969 op/s; 10683/212514 objects degraded (5.027%); 143 KiB/s, 2 objects/s recovering 2023-06-03T19:52:04.030 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:52:03 smithi089 bash[75068]: audit 1685821922.701751 mon.a (mon.0) 408 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:52:04.030 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:52:03 smithi089 bash[75068]: audit 1685821922.7166746 mon.a (mon.0) 409 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:52:04.030 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:52:03 smithi089 bash[75068]: audit 1685821922.724922 mon.c (mon.2) 206 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-03T19:52:04.030 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:52:03 smithi089 bash[75068]: audit 1685821922.7268732 mon.c (mon.2) 207 : audit 1 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-03T19:52:04.030 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:52:03 smithi089 bash[75068]: audit 1685821922.7389832 mon.a (mon.0) 410 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:52:04.030 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:52:03 smithi089 bash[75068]: audit 1685821922.8221345 mon.c (mon.2) 208 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-03T19:52:04.030 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:52:03 smithi089 bash[75068]: audit 1685821922.8313363 mon.c (mon.2) 209 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-06-03T19:52:04.031 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:52:03 smithi089 bash[75068]: audit 1685821922.8341224 mon.c (mon.2) 210 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-06-03T19:52:04.031 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:52:03 smithi089 bash[75068]: audit 1685821922.8362215 mon.c (mon.2) 211 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-06-03T19:52:04.031 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:52:03 smithi089 bash[75068]: audit 1685821922.8390677 mon.c (mon.2) 212 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-06-03T19:52:04.031 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:52:03 smithi089 bash[78452]: cluster 1685821922.2162282 mgr.y (mgr.34104) 424 : cluster 0 pgmap v356: 89 pgs: 52 active+clean, 20 active+undersized, 1 active+recovering+undersized+degraded+remapped, 3 active+undersized+degraded, 3 peering, 6 activating, 4 remapped+peering; 4.3 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 61 MiB/s wr, 969 op/s; 10683/212514 objects degraded (5.027%); 143 KiB/s, 2 objects/s recovering 2023-06-03T19:52:04.031 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:52:03 smithi089 bash[78452]: audit 1685821922.701751 mon.a (mon.0) 408 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:52:04.031 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:52:03 smithi089 bash[78452]: audit 1685821922.7166746 mon.a (mon.0) 409 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:52:04.032 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:52:03 smithi089 bash[78452]: audit 1685821922.724922 mon.c (mon.2) 206 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-03T19:52:04.032 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:52:03 smithi089 bash[78452]: audit 1685821922.7268732 mon.c (mon.2) 207 : audit 1 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-03T19:52:04.032 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:52:03 smithi089 bash[78452]: audit 1685821922.7389832 mon.a (mon.0) 410 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:52:04.032 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:52:03 smithi089 bash[78452]: audit 1685821922.8221345 mon.c (mon.2) 208 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-03T19:52:04.032 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:52:03 smithi089 bash[78452]: audit 1685821922.8313363 mon.c (mon.2) 209 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-06-03T19:52:04.032 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:52:03 smithi089 bash[78452]: audit 1685821922.8341224 mon.c (mon.2) 210 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-06-03T19:52:04.033 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:52:03 smithi089 bash[78452]: audit 1685821922.8362215 mon.c (mon.2) 211 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-06-03T19:52:04.033 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:52:03 smithi089 bash[78452]: audit 1685821922.8390677 mon.c (mon.2) 212 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-06-03T19:52:04.085 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:52:03 smithi164 bash[59462]: cluster 1685821922.2162282 mgr.y (mgr.34104) 424 : cluster 0 pgmap v356: 89 pgs: 52 active+clean, 20 active+undersized, 1 active+recovering+undersized+degraded+remapped, 3 active+undersized+degraded, 3 peering, 6 activating, 4 remapped+peering; 4.3 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 61 MiB/s wr, 969 op/s; 10683/212514 objects degraded (5.027%); 143 KiB/s, 2 objects/s recovering 2023-06-03T19:52:04.085 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:52:03 smithi164 bash[59462]: audit 1685821922.701751 mon.a (mon.0) 408 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:52:04.085 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:52:03 smithi164 bash[59462]: audit 1685821922.7166746 mon.a (mon.0) 409 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:52:04.085 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:52:03 smithi164 bash[59462]: audit 1685821922.724922 mon.c (mon.2) 206 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-03T19:52:04.085 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:52:03 smithi164 bash[59462]: audit 1685821922.7268732 mon.c (mon.2) 207 : audit 1 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-03T19:52:04.086 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:52:03 smithi164 bash[59462]: audit 1685821922.7389832 mon.a (mon.0) 410 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:52:04.086 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:52:03 smithi164 bash[59462]: audit 1685821922.8221345 mon.c (mon.2) 208 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-03T19:52:04.086 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:52:03 smithi164 bash[59462]: audit 1685821922.8313363 mon.c (mon.2) 209 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-06-03T19:52:04.086 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:52:03 smithi164 bash[59462]: audit 1685821922.8341224 mon.c (mon.2) 210 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-06-03T19:52:04.086 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:52:03 smithi164 bash[59462]: audit 1685821922.8362215 mon.c (mon.2) 211 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-06-03T19:52:04.086 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:52:03 smithi164 bash[59462]: audit 1685821922.8390677 mon.c (mon.2) 212 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-06-03T19:52:04.331 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 49 16 74311 74295 94.7498 51.625 0.015645 0.0105481 2023-06-03T19:52:05.029 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:52:04 smithi089 bash[75068]: audit 1685821922.8393168 mgr.y (mgr.34104) 425 : audit 0 from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-06-03T19:52:05.029 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:52:04 smithi089 bash[75068]: cephadm 1685821922.841089 mgr.y (mgr.34104) 426 : cephadm 1 Upgrade: unsafe to stop osd(s) at this time (9 PGs are or would become offline) 2023-06-03T19:52:05.029 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:52:04 smithi089 bash[75068]: audit 1685821923.9222233 mon.b (mon.1) 23 : audit 0 from='client.? 172.21.15.89:0/3920843135' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:52:05.030 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:52:04 smithi089 bash[78452]: audit 1685821922.8393168 mgr.y (mgr.34104) 425 : audit 0 from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-06-03T19:52:05.030 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:52:04 smithi089 bash[78452]: cephadm 1685821922.841089 mgr.y (mgr.34104) 426 : cephadm 1 Upgrade: unsafe to stop osd(s) at this time (9 PGs are or would become offline) 2023-06-03T19:52:05.030 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:52:04 smithi089 bash[78452]: audit 1685821923.9222233 mon.b (mon.1) 23 : audit 0 from='client.? 172.21.15.89:0/3920843135' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:52:05.085 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:52:04 smithi164 bash[59462]: audit 1685821922.8393168 mgr.y (mgr.34104) 425 : audit 0 from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-06-03T19:52:05.085 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:52:04 smithi164 bash[59462]: cephadm 1685821922.841089 mgr.y (mgr.34104) 426 : cephadm 1 Upgrade: unsafe to stop osd(s) at this time (9 PGs are or would become offline) 2023-06-03T19:52:05.085 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:52:04 smithi164 bash[59462]: audit 1685821923.9222233 mon.b (mon.1) 23 : audit 0 from='client.? 172.21.15.89:0/3920843135' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:52:05.331 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 50 16 75187 75171 93.9497 54.75 0.0113203 0.0106386 2023-06-03T19:52:06.029 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:52:05 smithi089 bash[75068]: cluster 1685821924.2170696 mgr.y (mgr.34104) 427 : cluster 0 pgmap v357: 89 pgs: 76 active+clean, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 6 activating, 4 remapped+peering; 4.4 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 66 MiB/s wr, 1.06k op/s; 6357/217206 objects degraded (2.927%); 139 KiB/s, 2 objects/s recovering 2023-06-03T19:52:06.030 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:52:05 smithi089 bash[78452]: cluster 1685821924.2170696 mgr.y (mgr.34104) 427 : cluster 0 pgmap v357: 89 pgs: 76 active+clean, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 6 activating, 4 remapped+peering; 4.4 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 66 MiB/s wr, 1.06k op/s; 6357/217206 objects degraded (2.927%); 139 KiB/s, 2 objects/s recovering 2023-06-03T19:52:06.084 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:52:05 smithi164 bash[59462]: cluster 1685821924.2170696 mgr.y (mgr.34104) 427 : cluster 0 pgmap v357: 89 pgs: 76 active+clean, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 6 activating, 4 remapped+peering; 4.4 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 66 MiB/s wr, 1.06k op/s; 6357/217206 objects degraded (2.927%); 139 KiB/s, 2 objects/s recovering 2023-06-03T19:52:06.331 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 51 16 76003 75987 93.1075 51 0.0300578 0.0107323 2023-06-03T19:52:06.529 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 19:52:06 smithi089 bash[71163]: ::ffff:172.21.15.164 - - [03/Jun/2023:19:52:06] "GET /metrics HTTP/1.1" 200 207517 "" "Prometheus/2.33.4" 2023-06-03T19:52:07.029 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:52:06 smithi089 bash[75068]: audit 1685821926.5276449 mon.a (mon.0) 411 : audit 0 from='client.? 172.21.15.89:0/3641987346' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:52:07.029 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:52:06 smithi089 bash[78452]: audit 1685821926.5276449 mon.a (mon.0) 411 : audit 0 from='client.? 172.21.15.89:0/3641987346' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:52:07.084 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:52:06 smithi164 bash[59462]: audit 1685821926.5276449 mon.a (mon.0) 411 : audit 0 from='client.? 172.21.15.89:0/3641987346' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:52:07.331 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 52 16 76841 76825 92.3241 52.375 0.00804537 0.0108234 2023-06-03T19:52:08.029 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:52:07 smithi089 bash[75068]: cluster 1685821926.2178295 mgr.y (mgr.34104) 428 : cluster 0 pgmap v358: 89 pgs: 82 active+clean, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped; 4.5 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 44 MiB/s wr, 701 op/s; 14587/222861 objects degraded (6.545%); 1.3 MiB/s, 21 objects/s recovering 2023-06-03T19:52:08.029 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:52:07 smithi089 bash[75068]: audit 1685821927.0877798 mon.a (mon.0) 412 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:52:08.029 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:52:07 smithi089 bash[75068]: audit 1685821927.1022077 mon.c (mon.2) 213 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T19:52:08.030 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:52:07 smithi089 bash[75068]: cluster 1685821927.3264205 mon.a (mon.0) 413 : cluster 3 Health check update: Degraded data redundancy: 14587/222861 objects degraded (6.545%), 7 pgs degraded (PG_DEGRADED) 2023-06-03T19:52:08.030 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:52:07 smithi089 bash[78452]: cluster 1685821926.2178295 mgr.y (mgr.34104) 428 : cluster 0 pgmap v358: 89 pgs: 82 active+clean, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped; 4.5 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 44 MiB/s wr, 701 op/s; 14587/222861 objects degraded (6.545%); 1.3 MiB/s, 21 objects/s recovering 2023-06-03T19:52:08.030 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:52:07 smithi089 bash[78452]: audit 1685821927.0877798 mon.a (mon.0) 412 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:52:08.030 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:52:07 smithi089 bash[78452]: audit 1685821927.1022077 mon.c (mon.2) 213 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T19:52:08.030 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:52:07 smithi089 bash[78452]: cluster 1685821927.3264205 mon.a (mon.0) 413 : cluster 3 Health check update: Degraded data redundancy: 14587/222861 objects degraded (6.545%), 7 pgs degraded (PG_DEGRADED) 2023-06-03T19:52:08.085 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:52:07 smithi164 bash[59462]: cluster 1685821926.2178295 mgr.y (mgr.34104) 428 : cluster 0 pgmap v358: 89 pgs: 82 active+clean, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped; 4.5 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 44 MiB/s wr, 701 op/s; 14587/222861 objects degraded (6.545%); 1.3 MiB/s, 21 objects/s recovering 2023-06-03T19:52:08.085 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:52:07 smithi164 bash[59462]: audit 1685821927.0877798 mon.a (mon.0) 412 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:52:08.085 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:52:07 smithi164 bash[59462]: audit 1685821927.1022077 mon.c (mon.2) 213 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T19:52:08.085 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:52:07 smithi164 bash[59462]: cluster 1685821927.3264205 mon.a (mon.0) 413 : cluster 3 Health check update: Degraded data redundancy: 14587/222861 objects degraded (6.545%), 7 pgs degraded (PG_DEGRADED) 2023-06-03T19:52:08.331 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 53 16 77736 77720 91.6375 55.9375 0.0168937 0.0109068 2023-06-03T19:52:09.331 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 54 16 78537 78521 90.8676 50.0625 0.0380211 0.010999 2023-06-03T19:52:10.029 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:52:09 smithi089 bash[78452]: cluster 1685821928.2184267 mgr.y (mgr.34104) 429 : cluster 0 pgmap v359: 89 pgs: 82 active+clean, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped; 4.5 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 39 MiB/s wr, 623 op/s; 14587/222861 objects degraded (6.545%); 1.2 MiB/s, 19 objects/s recovering 2023-06-03T19:52:10.029 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:52:09 smithi089 bash[78452]: audit 1685821929.1125493 mon.c (mon.2) 214 : audit 0 from='client.? 172.21.15.89:0/2739503813' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:52:10.030 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:52:09 smithi089 bash[75068]: cluster 1685821928.2184267 mgr.y (mgr.34104) 429 : cluster 0 pgmap v359: 89 pgs: 82 active+clean, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped; 4.5 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 39 MiB/s wr, 623 op/s; 14587/222861 objects degraded (6.545%); 1.2 MiB/s, 19 objects/s recovering 2023-06-03T19:52:10.030 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:52:09 smithi089 bash[75068]: audit 1685821929.1125493 mon.c (mon.2) 214 : audit 0 from='client.? 172.21.15.89:0/2739503813' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:52:10.085 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:52:09 smithi164 bash[59462]: cluster 1685821928.2184267 mgr.y (mgr.34104) 429 : cluster 0 pgmap v359: 89 pgs: 82 active+clean, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped; 4.5 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 39 MiB/s wr, 623 op/s; 14587/222861 objects degraded (6.545%); 1.2 MiB/s, 19 objects/s recovering 2023-06-03T19:52:10.085 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:52:09 smithi164 bash[59462]: audit 1685821929.1125493 mon.c (mon.2) 214 : audit 0 from='client.? 172.21.15.89:0/2739503813' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:52:10.331 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 55 16 79399 79383 90.1949 53.875 0.0300285 0.0110809 2023-06-03T19:52:11.332 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 56 16 80206 80190 89.4849 50.4375 0.0289239 0.0111677 2023-06-03T19:52:12.029 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:52:11 smithi089 bash[75068]: cluster 1685821930.2192116 mgr.y (mgr.34104) 430 : cluster 0 pgmap v360: 89 pgs: 82 active+clean, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped; 4.7 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 47 MiB/s wr, 744 op/s; 14587/232389 objects degraded (6.277%); 1.1 MiB/s, 16 objects/s recovering 2023-06-03T19:52:12.030 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:52:11 smithi089 bash[75068]: audit 1685821931.6963274 mon.c (mon.2) 215 : audit 0 from='client.? 172.21.15.89:0/1620672962' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:52:12.031 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:52:11 smithi089 bash[78452]: cluster 1685821930.2192116 mgr.y (mgr.34104) 430 : cluster 0 pgmap v360: 89 pgs: 82 active+clean, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped; 4.7 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 47 MiB/s wr, 744 op/s; 14587/232389 objects degraded (6.277%); 1.1 MiB/s, 16 objects/s recovering 2023-06-03T19:52:12.031 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:52:11 smithi089 bash[78452]: audit 1685821931.6963274 mon.c (mon.2) 215 : audit 0 from='client.? 172.21.15.89:0/1620672962' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:52:12.085 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:52:11 smithi164 bash[59462]: cluster 1685821930.2192116 mgr.y (mgr.34104) 430 : cluster 0 pgmap v360: 89 pgs: 82 active+clean, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped; 4.7 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 47 MiB/s wr, 744 op/s; 14587/232389 objects degraded (6.277%); 1.1 MiB/s, 16 objects/s recovering 2023-06-03T19:52:12.085 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:52:11 smithi164 bash[59462]: audit 1685821931.6963274 mon.c (mon.2) 215 : audit 0 from='client.? 172.21.15.89:0/1620672962' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:52:12.332 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 57 16 81048 81032 88.8382 52.625 0.0263582 0.0112496 2023-06-03T19:52:13.029 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:52:12 smithi089 bash[75068]: cluster 1685821932.3278956 mon.a (mon.0) 414 : cluster 3 Health check update: Degraded data redundancy: 14587/232389 objects degraded (6.277%), 7 pgs degraded (PG_DEGRADED) 2023-06-03T19:52:13.029 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:52:12 smithi089 bash[78452]: cluster 1685821932.3278956 mon.a (mon.0) 414 : cluster 3 Health check update: Degraded data redundancy: 14587/232389 objects degraded (6.277%), 7 pgs degraded (PG_DEGRADED) 2023-06-03T19:52:13.084 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:52:12 smithi164 bash[59462]: cluster 1685821932.3278956 mon.a (mon.0) 414 : cluster 3 Health check update: Degraded data redundancy: 14587/232389 objects degraded (6.277%), 7 pgs degraded (PG_DEGRADED) 2023-06-03T19:52:13.332 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 58 16 81890 81874 88.2138 52.625 0.0141355 0.0113301 2023-06-03T19:52:14.029 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:52:13 smithi089 bash[75068]: cluster 1685821932.2198727 mgr.y (mgr.34104) 431 : cluster 0 pgmap v361: 89 pgs: 82 active+clean, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped; 4.8 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 45 MiB/s wr, 714 op/s; 14401/235464 objects degraded (6.116%); 1.9 MiB/s, 29 objects/s recovering 2023-06-03T19:52:14.029 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:52:13 smithi089 bash[78452]: cluster 1685821932.2198727 mgr.y (mgr.34104) 431 : cluster 0 pgmap v361: 89 pgs: 82 active+clean, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped; 4.8 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 45 MiB/s wr, 714 op/s; 14401/235464 objects degraded (6.116%); 1.9 MiB/s, 29 objects/s recovering 2023-06-03T19:52:14.084 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:52:13 smithi164 bash[59462]: cluster 1685821932.2198727 mgr.y (mgr.34104) 431 : cluster 0 pgmap v361: 89 pgs: 82 active+clean, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped; 4.8 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 45 MiB/s wr, 714 op/s; 14401/235464 objects degraded (6.116%); 1.9 MiB/s, 29 objects/s recovering 2023-06-03T19:52:14.332 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 59 16 82702 82686 87.5787 50.75 0.0236457 0.0114114 2023-06-03T19:52:15.029 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:52:14 smithi089 bash[75068]: audit 1685821934.302537 mon.a (mon.0) 415 : audit 0 from='client.? 172.21.15.89:0/394061013' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:52:15.029 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:52:14 smithi089 bash[78452]: audit 1685821934.302537 mon.a (mon.0) 415 : audit 0 from='client.? 172.21.15.89:0/394061013' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:52:15.085 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:52:14 smithi164 bash[59462]: audit 1685821934.302537 mon.a (mon.0) 415 : audit 0 from='client.? 172.21.15.89:0/394061013' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:52:15.332 INFO:tasks.radosbench.radosbench.0.smithi089.stdout:2023-06-03T19:52:15.328092+0000 min lat: 0.00131775 max lat: 2.24771 avg lat: 0.0114887 2023-06-03T19:52:15.333 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-06-03T19:52:15.333 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 60 16 83534 83518 86.9857 52 0.0224671 0.0114887 2023-06-03T19:52:16.030 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:52:15 smithi089 bash[75068]: cluster 1685821934.2206252 mgr.y (mgr.34104) 432 : cluster 0 pgmap v362: 89 pgs: 82 active+clean, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped; 4.9 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 51 MiB/s wr, 810 op/s; 14401/241716 objects degraded (5.958%); 1.8 MiB/s, 28 objects/s recovering 2023-06-03T19:52:16.030 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:52:15 smithi089 bash[78452]: cluster 1685821934.2206252 mgr.y (mgr.34104) 432 : cluster 0 pgmap v362: 89 pgs: 82 active+clean, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped; 4.9 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 51 MiB/s wr, 810 op/s; 14401/241716 objects degraded (5.958%); 1.8 MiB/s, 28 objects/s recovering 2023-06-03T19:52:16.085 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:52:15 smithi164 bash[59462]: cluster 1685821934.2206252 mgr.y (mgr.34104) 432 : cluster 0 pgmap v362: 89 pgs: 82 active+clean, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped; 4.9 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 51 MiB/s wr, 810 op/s; 14401/241716 objects degraded (5.958%); 1.8 MiB/s, 28 objects/s recovering 2023-06-03T19:52:16.332 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 61 16 84380 84364 86.4264 52.875 0.0243557 0.0115642 2023-06-03T19:52:16.529 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 19:52:16 smithi089 bash[71163]: ::ffff:172.21.15.164 - - [03/Jun/2023:19:52:16] "GET /metrics HTTP/1.1" 200 212621 "" "Prometheus/2.33.4" 2023-06-03T19:52:17.332 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 62 16 85182 85166 85.8409 50.125 0.0081683 0.0116434 2023-06-03T19:52:18.029 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:52:17 smithi089 bash[75068]: cluster 1685821936.2212293 mgr.y (mgr.34104) 433 : cluster 0 pgmap v363: 89 pgs: 82 active+clean, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped; 5.0 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 53 MiB/s wr, 853 op/s; 14210/247941 objects degraded (5.731%); 2.8 MiB/s, 44 objects/s recovering 2023-06-03T19:52:18.030 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:52:17 smithi089 bash[75068]: audit 1685821936.87761 mon.a (mon.0) 416 : audit 0 from='client.? 172.21.15.89:0/3670540284' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:52:18.030 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:52:17 smithi089 bash[75068]: cluster 1685821937.3291981 mon.a (mon.0) 417 : cluster 3 Health check update: Degraded data redundancy: 14210/247941 objects degraded (5.731%), 7 pgs degraded (PG_DEGRADED) 2023-06-03T19:52:18.030 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 19:52:17 smithi089 bash[71163]: debug 2023-06-03T19:52:17.849+0000 7fe48642e700 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2023-06-03T19:52:18.030 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:52:17 smithi089 bash[78452]: cluster 1685821936.2212293 mgr.y (mgr.34104) 433 : cluster 0 pgmap v363: 89 pgs: 82 active+clean, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped; 5.0 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 53 MiB/s wr, 853 op/s; 14210/247941 objects degraded (5.731%); 2.8 MiB/s, 44 objects/s recovering 2023-06-03T19:52:18.031 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:52:17 smithi089 bash[78452]: audit 1685821936.87761 mon.a (mon.0) 416 : audit 0 from='client.? 172.21.15.89:0/3670540284' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:52:18.031 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:52:17 smithi089 bash[78452]: cluster 1685821937.3291981 mon.a (mon.0) 417 : cluster 3 Health check update: Degraded data redundancy: 14210/247941 objects degraded (5.731%), 7 pgs degraded (PG_DEGRADED) 2023-06-03T19:52:18.084 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:52:17 smithi164 bash[59462]: cluster 1685821936.2212293 mgr.y (mgr.34104) 433 : cluster 0 pgmap v363: 89 pgs: 82 active+clean, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped; 5.0 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 53 MiB/s wr, 853 op/s; 14210/247941 objects degraded (5.731%); 2.8 MiB/s, 44 objects/s recovering 2023-06-03T19:52:18.085 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:52:17 smithi164 bash[59462]: audit 1685821936.87761 mon.a (mon.0) 416 : audit 0 from='client.? 172.21.15.89:0/3670540284' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:52:18.085 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:52:17 smithi164 bash[59462]: cluster 1685821937.3291981 mon.a (mon.0) 417 : cluster 3 Health check update: Degraded data redundancy: 14210/247941 objects degraded (5.731%), 7 pgs degraded (PG_DEGRADED) 2023-06-03T19:52:18.332 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 63 16 86003 85987 85.2927 51.3125 0.0130921 0.0117183 2023-06-03T19:52:19.085 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:52:18 smithi164 bash[59462]: audit 1685821937.8539333 mon.c (mon.2) 216 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-06-03T19:52:19.279 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:52:18 smithi089 bash[75068]: audit 1685821937.8539333 mon.c (mon.2) 216 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-06-03T19:52:19.279 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:52:18 smithi089 bash[78452]: audit 1685821937.8539333 mon.c (mon.2) 216 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-06-03T19:52:19.332 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 64 16 86793 86777 84.7315 49.375 0.0210253 0.0117948 2023-06-03T19:52:20.085 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:52:19 smithi164 bash[59462]: audit 1685821937.854266 mgr.y (mgr.34104) 434 : audit 0 from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-06-03T19:52:20.085 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:52:19 smithi164 bash[59462]: cephadm 1685821937.8564765 mgr.y (mgr.34104) 435 : cephadm 1 Upgrade: unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2023-06-03T19:52:20.085 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:52:19 smithi164 bash[59462]: cluster 1685821938.2229743 mgr.y (mgr.34104) 436 : cluster 0 pgmap v364: 89 pgs: 82 active+clean, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped; 5.0 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 44 MiB/s wr, 696 op/s; 14210/247941 objects degraded (5.731%); 2.0 MiB/s, 31 objects/s recovering 2023-06-03T19:52:20.085 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:52:19 smithi164 bash[59462]: audit 1685821939.510732 mon.b (mon.1) 24 : audit 0 from='client.? 172.21.15.89:0/1072202960' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:52:20.279 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:52:19 smithi089 bash[75068]: audit 1685821937.854266 mgr.y (mgr.34104) 434 : audit 0 from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-06-03T19:52:20.279 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:52:19 smithi089 bash[75068]: cephadm 1685821937.8564765 mgr.y (mgr.34104) 435 : cephadm 1 Upgrade: unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2023-06-03T19:52:20.280 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:52:19 smithi089 bash[75068]: cluster 1685821938.2229743 mgr.y (mgr.34104) 436 : cluster 0 pgmap v364: 89 pgs: 82 active+clean, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped; 5.0 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 44 MiB/s wr, 696 op/s; 14210/247941 objects degraded (5.731%); 2.0 MiB/s, 31 objects/s recovering 2023-06-03T19:52:20.280 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:52:19 smithi089 bash[75068]: audit 1685821939.510732 mon.b (mon.1) 24 : audit 0 from='client.? 172.21.15.89:0/1072202960' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:52:20.280 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:52:19 smithi089 bash[78452]: audit 1685821937.854266 mgr.y (mgr.34104) 434 : audit 0 from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-06-03T19:52:20.280 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:52:19 smithi089 bash[78452]: cephadm 1685821937.8564765 mgr.y (mgr.34104) 435 : cephadm 1 Upgrade: unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2023-06-03T19:52:20.280 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:52:19 smithi089 bash[78452]: cluster 1685821938.2229743 mgr.y (mgr.34104) 436 : cluster 0 pgmap v364: 89 pgs: 82 active+clean, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped; 5.0 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 44 MiB/s wr, 696 op/s; 14210/247941 objects degraded (5.731%); 2.0 MiB/s, 31 objects/s recovering 2023-06-03T19:52:20.281 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:52:19 smithi089 bash[78452]: audit 1685821939.510732 mon.b (mon.1) 24 : audit 0 from='client.? 172.21.15.89:0/1072202960' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:52:20.332 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 65 16 87602 87586 84.2057 50.5625 0.0147994 0.0118699 2023-06-03T19:52:21.333 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 66 16 88431 88415 83.7149 51.8125 0.0189367 0.0119391 2023-06-03T19:52:22.085 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:52:21 smithi164 bash[59462]: cluster 1685821940.2238202 mgr.y (mgr.34104) 437 : cluster 0 pgmap v365: 89 pgs: 82 active+clean, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped; 5.2 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 60 MiB/s wr, 954 op/s; 14210/257241 objects degraded (5.524%); 2.0 MiB/s, 31 objects/s recovering 2023-06-03T19:52:22.279 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:52:21 smithi089 bash[75068]: cluster 1685821940.2238202 mgr.y (mgr.34104) 437 : cluster 0 pgmap v365: 89 pgs: 82 active+clean, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped; 5.2 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 60 MiB/s wr, 954 op/s; 14210/257241 objects degraded (5.524%); 2.0 MiB/s, 31 objects/s recovering 2023-06-03T19:52:22.279 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:52:21 smithi089 bash[78452]: cluster 1685821940.2238202 mgr.y (mgr.34104) 437 : cluster 0 pgmap v365: 89 pgs: 82 active+clean, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped; 5.2 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 60 MiB/s wr, 954 op/s; 14210/257241 objects degraded (5.524%); 2.0 MiB/s, 31 objects/s recovering 2023-06-03T19:52:22.333 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 67 16 89175 89159 83.1594 46.5 0.0356658 0.0120186 2023-06-03T19:52:23.085 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:52:22 smithi164 bash[59462]: audit 1685821942.0811496 mon.c (mon.2) 217 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T19:52:23.085 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:52:22 smithi164 bash[59462]: audit 1685821942.1097958 mon.a (mon.0) 418 : audit 0 from='client.? 172.21.15.89:0/1950484334' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:52:23.085 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:52:22 smithi164 bash[59462]: cluster 1685821942.3305814 mon.a (mon.0) 419 : cluster 3 Health check update: Degraded data redundancy: 14210/257241 objects degraded (5.524%), 7 pgs degraded (PG_DEGRADED) 2023-06-03T19:52:23.279 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:52:22 smithi089 bash[75068]: audit 1685821942.0811496 mon.c (mon.2) 217 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T19:52:23.279 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:52:22 smithi089 bash[75068]: audit 1685821942.1097958 mon.a (mon.0) 418 : audit 0 from='client.? 172.21.15.89:0/1950484334' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:52:23.280 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:52:22 smithi089 bash[75068]: cluster 1685821942.3305814 mon.a (mon.0) 419 : cluster 3 Health check update: Degraded data redundancy: 14210/257241 objects degraded (5.524%), 7 pgs degraded (PG_DEGRADED) 2023-06-03T19:52:23.280 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:52:22 smithi089 bash[78452]: audit 1685821942.0811496 mon.c (mon.2) 217 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T19:52:23.280 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:52:22 smithi089 bash[78452]: audit 1685821942.1097958 mon.a (mon.0) 418 : audit 0 from='client.? 172.21.15.89:0/1950484334' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:52:23.281 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:52:22 smithi089 bash[78452]: cluster 1685821942.3305814 mon.a (mon.0) 419 : cluster 3 Health check update: Degraded data redundancy: 14210/257241 objects degraded (5.524%), 7 pgs degraded (PG_DEGRADED) 2023-06-03T19:52:23.333 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 68 16 90005 89989 82.6992 51.875 0.0147624 0.0120855 2023-06-03T19:52:24.084 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:52:23 smithi164 bash[59462]: cluster 1685821942.2249718 mgr.y (mgr.34104) 438 : cluster 0 pgmap v366: 89 pgs: 82 active+clean, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped; 5.3 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 48 MiB/s wr, 773 op/s; 14021/260244 objects degraded (5.388%); 2.9 MiB/s, 47 objects/s recovering 2023-06-03T19:52:24.279 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:52:23 smithi089 bash[75068]: cluster 1685821942.2249718 mgr.y (mgr.34104) 438 : cluster 0 pgmap v366: 89 pgs: 82 active+clean, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped; 5.3 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 48 MiB/s wr, 773 op/s; 14021/260244 objects degraded (5.388%); 2.9 MiB/s, 47 objects/s recovering 2023-06-03T19:52:24.280 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:52:23 smithi089 bash[78452]: cluster 1685821942.2249718 mgr.y (mgr.34104) 438 : cluster 0 pgmap v366: 89 pgs: 82 active+clean, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped; 5.3 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 48 MiB/s wr, 773 op/s; 14021/260244 objects degraded (5.388%); 2.9 MiB/s, 47 objects/s recovering 2023-06-03T19:52:24.333 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 69 16 90788 90772 82.2099 48.9375 0.0246727 0.012157 2023-06-03T19:52:25.085 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:52:24 smithi164 bash[59462]: cluster 1685821944.2257507 mgr.y (mgr.34104) 439 : cluster 0 pgmap v367: 89 pgs: 82 active+clean, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped; 5.4 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 54 MiB/s wr, 856 op/s; 14021/266298 objects degraded (5.265%); 2.0 MiB/s, 31 objects/s recovering 2023-06-03T19:52:25.085 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:52:24 smithi164 bash[59462]: audit 1685821944.6739795 mon.c (mon.2) 218 : audit 0 from='client.? 172.21.15.89:0/3899693292' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:52:25.279 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:52:24 smithi089 bash[75068]: cluster 1685821944.2257507 mgr.y (mgr.34104) 439 : cluster 0 pgmap v367: 89 pgs: 82 active+clean, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped; 5.4 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 54 MiB/s wr, 856 op/s; 14021/266298 objects degraded (5.265%); 2.0 MiB/s, 31 objects/s recovering 2023-06-03T19:52:25.279 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:52:24 smithi089 bash[75068]: audit 1685821944.6739795 mon.c (mon.2) 218 : audit 0 from='client.? 172.21.15.89:0/3899693292' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:52:25.279 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:52:24 smithi089 bash[78452]: cluster 1685821944.2257507 mgr.y (mgr.34104) 439 : cluster 0 pgmap v367: 89 pgs: 82 active+clean, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped; 5.4 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 54 MiB/s wr, 856 op/s; 14021/266298 objects degraded (5.265%); 2.0 MiB/s, 31 objects/s recovering 2023-06-03T19:52:25.280 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:52:24 smithi089 bash[78452]: audit 1685821944.6739795 mon.c (mon.2) 218 : audit 0 from='client.? 172.21.15.89:0/3899693292' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:52:25.333 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 70 16 91597 91581 81.7577 50.5625 0.0105348 0.0122252 2023-06-03T19:52:26.333 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 71 16 92387 92371 81.3016 49.375 0.0138118 0.0122937 2023-06-03T19:52:26.529 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 19:52:26 smithi089 bash[71163]: ::ffff:172.21.15.164 - - [03/Jun/2023:19:52:26] "GET /metrics HTTP/1.1" 200 212653 "" "Prometheus/2.33.4" 2023-06-03T19:52:27.333 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 72 16 93165 93149 80.8477 48.625 0.0175004 0.0123625 2023-06-03T19:52:27.584 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:52:27 smithi164 bash[59462]: cluster 1685821946.2263958 mgr.y (mgr.34104) 440 : cluster 0 pgmap v368: 89 pgs: 82 active+clean, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped; 5.5 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 53 MiB/s wr, 848 op/s; 13827/272286 objects degraded (5.078%); 3.0 MiB/s, 47 objects/s recovering 2023-06-03T19:52:27.585 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:52:27 smithi164 bash[59462]: audit 1685821947.2483704 mon.a (mon.0) 420 : audit 0 from='client.? 172.21.15.89:0/2548998147' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:52:27.779 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:52:27 smithi089 bash[75068]: cluster 1685821946.2263958 mgr.y (mgr.34104) 440 : cluster 0 pgmap v368: 89 pgs: 82 active+clean, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped; 5.5 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 53 MiB/s wr, 848 op/s; 13827/272286 objects degraded (5.078%); 3.0 MiB/s, 47 objects/s recovering 2023-06-03T19:52:27.779 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:52:27 smithi089 bash[75068]: audit 1685821947.2483704 mon.a (mon.0) 420 : audit 0 from='client.? 172.21.15.89:0/2548998147' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:52:27.779 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:52:27 smithi089 bash[78452]: cluster 1685821946.2263958 mgr.y (mgr.34104) 440 : cluster 0 pgmap v368: 89 pgs: 82 active+clean, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped; 5.5 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 53 MiB/s wr, 848 op/s; 13827/272286 objects degraded (5.078%); 3.0 MiB/s, 47 objects/s recovering 2023-06-03T19:52:27.780 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:52:27 smithi089 bash[78452]: audit 1685821947.2483704 mon.a (mon.0) 420 : audit 0 from='client.? 172.21.15.89:0/2548998147' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:52:28.334 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 73 16 93996 93980 80.4516 51.9375 0.0474504 0.0124223 2023-06-03T19:52:28.584 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:52:28 smithi164 bash[59462]: cluster 1685821947.3318918 mon.a (mon.0) 421 : cluster 3 Health check update: Degraded data redundancy: 13827/272286 objects degraded (5.078%), 7 pgs degraded (PG_DEGRADED) 2023-06-03T19:52:28.779 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:52:28 smithi089 bash[75068]: cluster 1685821947.3318918 mon.a (mon.0) 421 : cluster 3 Health check update: Degraded data redundancy: 13827/272286 objects degraded (5.078%), 7 pgs degraded (PG_DEGRADED) 2023-06-03T19:52:28.779 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:52:28 smithi089 bash[78452]: cluster 1685821947.3318918 mon.a (mon.0) 421 : cluster 3 Health check update: Degraded data redundancy: 13827/272286 objects degraded (5.078%), 7 pgs degraded (PG_DEGRADED) 2023-06-03T19:52:29.333 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 74 16 94778 94762 80.0249 48.875 0.0215962 0.0124899 2023-06-03T19:52:29.779 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:52:29 smithi089 bash[75068]: cluster 1685821948.2270317 mgr.y (mgr.34104) 441 : cluster 0 pgmap v369: 89 pgs: 82 active+clean, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped; 5.5 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 42 MiB/s wr, 675 op/s; 13827/272286 objects degraded (5.078%); 2.0 MiB/s, 31 objects/s recovering 2023-06-03T19:52:29.780 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:52:29 smithi089 bash[78452]: cluster 1685821948.2270317 mgr.y (mgr.34104) 441 : cluster 0 pgmap v369: 89 pgs: 82 active+clean, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped; 5.5 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 42 MiB/s wr, 675 op/s; 13827/272286 objects degraded (5.078%); 2.0 MiB/s, 31 objects/s recovering 2023-06-03T19:52:29.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:52:29 smithi164 bash[59462]: cluster 1685821948.2270317 mgr.y (mgr.34104) 441 : cluster 0 pgmap v369: 89 pgs: 82 active+clean, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped; 5.5 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 42 MiB/s wr, 675 op/s; 13827/272286 objects degraded (5.078%); 2.0 MiB/s, 31 objects/s recovering 2023-06-03T19:52:30.334 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 75 16 95542 95526 79.5945 47.75 0.0119755 0.0125575 2023-06-03T19:52:30.779 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:52:30 smithi089 bash[75068]: audit 1685821949.8570864 mon.b (mon.1) 25 : audit 0 from='client.? 172.21.15.89:0/2837136049' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:52:30.780 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:52:30 smithi089 bash[78452]: audit 1685821949.8570864 mon.b (mon.1) 25 : audit 0 from='client.? 172.21.15.89:0/2837136049' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:52:30.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:52:30 smithi164 bash[59462]: audit 1685821949.8570864 mon.b (mon.1) 25 : audit 0 from='client.? 172.21.15.89:0/2837136049' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:52:31.334 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 76 16 96365 96349 79.2239 51.4375 0.0206275 0.0126162 2023-06-03T19:52:31.779 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:52:31 smithi089 bash[75068]: cluster 1685821950.227719 mgr.y (mgr.34104) 442 : cluster 0 pgmap v370: 89 pgs: 82 active+clean, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped; 5.7 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 58 MiB/s wr, 929 op/s; 13827/281400 objects degraded (4.914%); 2.0 MiB/s, 31 objects/s recovering 2023-06-03T19:52:31.779 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:52:31 smithi089 bash[78452]: cluster 1685821950.227719 mgr.y (mgr.34104) 442 : cluster 0 pgmap v370: 89 pgs: 82 active+clean, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped; 5.7 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 58 MiB/s wr, 929 op/s; 13827/281400 objects degraded (4.914%); 2.0 MiB/s, 31 objects/s recovering 2023-06-03T19:52:31.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:52:31 smithi164 bash[59462]: cluster 1685821950.227719 mgr.y (mgr.34104) 442 : cluster 0 pgmap v370: 89 pgs: 82 active+clean, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped; 5.7 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 58 MiB/s wr, 929 op/s; 13827/281400 objects degraded (4.914%); 2.0 MiB/s, 31 objects/s recovering 2023-06-03T19:52:32.334 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 77 16 97103 97087 78.794 46.125 0.0144227 0.0126842 2023-06-03T19:52:32.780 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:52:32 smithi089 bash[78452]: cluster 1685821952.3332543 mon.a (mon.0) 422 : cluster 3 Health check update: Degraded data redundancy: 13827/281400 objects degraded (4.914%), 7 pgs degraded (PG_DEGRADED) 2023-06-03T19:52:32.780 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:52:32 smithi089 bash[75068]: cluster 1685821952.3332543 mon.a (mon.0) 422 : cluster 3 Health check update: Degraded data redundancy: 13827/281400 objects degraded (4.914%), 7 pgs degraded (PG_DEGRADED) 2023-06-03T19:52:32.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:52:32 smithi164 bash[59462]: cluster 1685821952.3332543 mon.a (mon.0) 422 : cluster 3 Health check update: Degraded data redundancy: 13827/281400 objects degraded (4.914%), 7 pgs degraded (PG_DEGRADED) 2023-06-03T19:52:33.279 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 19:52:32 smithi089 bash[71163]: debug 2023-06-03T19:52:32.868+0000 7fe48642e700 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2023-06-03T19:52:33.334 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 78 16 97865 97849 78.3943 47.625 0.0184945 0.0127495 2023-06-03T19:52:33.779 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:52:33 smithi089 bash[75068]: cluster 1685821952.2283318 mgr.y (mgr.34104) 443 : cluster 0 pgmap v371: 89 pgs: 82 active+clean, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped; 5.8 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 47 MiB/s wr, 748 op/s; 13593/284211 objects degraded (4.783%); 3.2 MiB/s, 51 objects/s recovering 2023-06-03T19:52:33.779 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:52:33 smithi089 bash[75068]: audit 1685821952.4828687 mon.b (mon.1) 26 : audit 0 from='client.? 172.21.15.89:0/1657874459' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:52:33.780 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:52:33 smithi089 bash[75068]: audit 1685821952.8718872 mon.c (mon.2) 219 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-06-03T19:52:33.780 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:52:33 smithi089 bash[78452]: cluster 1685821952.2283318 mgr.y (mgr.34104) 443 : cluster 0 pgmap v371: 89 pgs: 82 active+clean, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped; 5.8 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 47 MiB/s wr, 748 op/s; 13593/284211 objects degraded (4.783%); 3.2 MiB/s, 51 objects/s recovering 2023-06-03T19:52:33.780 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:52:33 smithi089 bash[78452]: audit 1685821952.4828687 mon.b (mon.1) 26 : audit 0 from='client.? 172.21.15.89:0/1657874459' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:52:33.780 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:52:33 smithi089 bash[78452]: audit 1685821952.8718872 mon.c (mon.2) 219 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-06-03T19:52:33.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:52:33 smithi164 bash[59462]: cluster 1685821952.2283318 mgr.y (mgr.34104) 443 : cluster 0 pgmap v371: 89 pgs: 82 active+clean, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped; 5.8 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 47 MiB/s wr, 748 op/s; 13593/284211 objects degraded (4.783%); 3.2 MiB/s, 51 objects/s recovering 2023-06-03T19:52:33.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:52:33 smithi164 bash[59462]: audit 1685821952.4828687 mon.b (mon.1) 26 : audit 0 from='client.? 172.21.15.89:0/1657874459' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:52:33.836 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:52:33 smithi164 bash[59462]: audit 1685821952.8718872 mon.c (mon.2) 219 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-06-03T19:52:34.334 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 79 16 98639 98623 78.0143 48.375 0.0100645 0.0128111 2023-06-03T19:52:34.780 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:52:34 smithi089 bash[75068]: audit 1685821952.8722305 mgr.y (mgr.34104) 444 : audit 0 from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-06-03T19:52:34.780 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:52:34 smithi089 bash[75068]: cephadm 1685821952.8742812 mgr.y (mgr.34104) 445 : cephadm 1 Upgrade: unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2023-06-03T19:52:34.780 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:52:34 smithi089 bash[78452]: audit 1685821952.8722305 mgr.y (mgr.34104) 444 : audit 0 from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-06-03T19:52:34.781 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:52:34 smithi089 bash[78452]: cephadm 1685821952.8742812 mgr.y (mgr.34104) 445 : cephadm 1 Upgrade: unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2023-06-03T19:52:34.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:52:34 smithi164 bash[59462]: audit 1685821952.8722305 mgr.y (mgr.34104) 444 : audit 0 from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-06-03T19:52:34.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:52:34 smithi164 bash[59462]: cephadm 1685821952.8742812 mgr.y (mgr.34104) 445 : cephadm 1 Upgrade: unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2023-06-03T19:52:35.334 INFO:tasks.radosbench.radosbench.0.smithi089.stdout:2023-06-03T19:52:35.330091+0000 min lat: 0.00131775 max lat: 2.24771 avg lat: 0.0128792 2023-06-03T19:52:35.334 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-06-03T19:52:35.334 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 80 16 99360 99344 77.6024 45.0625 0.0172507 0.0128792 2023-06-03T19:52:35.779 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:52:35 smithi089 bash[75068]: cluster 1685821954.2291336 mgr.y (mgr.34104) 446 : cluster 0 pgmap v372: 89 pgs: 82 active+clean, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped; 5.9 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 52 MiB/s wr, 828 op/s; 13593/290097 objects degraded (4.686%); 2.2 MiB/s, 35 objects/s recovering 2023-06-03T19:52:35.779 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:52:35 smithi089 bash[75068]: audit 1685821955.0537422 mon.a (mon.0) 423 : audit 0 from='client.? 172.21.15.89:0/3520885514' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:52:35.780 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:52:35 smithi089 bash[78452]: cluster 1685821954.2291336 mgr.y (mgr.34104) 446 : cluster 0 pgmap v372: 89 pgs: 82 active+clean, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped; 5.9 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 52 MiB/s wr, 828 op/s; 13593/290097 objects degraded (4.686%); 2.2 MiB/s, 35 objects/s recovering 2023-06-03T19:52:35.780 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:52:35 smithi089 bash[78452]: audit 1685821955.0537422 mon.a (mon.0) 423 : audit 0 from='client.? 172.21.15.89:0/3520885514' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:52:35.834 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:52:35 smithi164 bash[59462]: cluster 1685821954.2291336 mgr.y (mgr.34104) 446 : cluster 0 pgmap v372: 89 pgs: 82 active+clean, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped; 5.9 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 52 MiB/s wr, 828 op/s; 13593/290097 objects degraded (4.686%); 2.2 MiB/s, 35 objects/s recovering 2023-06-03T19:52:35.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:52:35 smithi164 bash[59462]: audit 1685821955.0537422 mon.a (mon.0) 423 : audit 0 from='client.? 172.21.15.89:0/3520885514' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:52:36.334 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 81 16 100107 100091 77.2206 46.6875 0.025615 0.0129437 2023-06-03T19:52:36.529 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 19:52:36 smithi089 bash[71163]: ::ffff:172.21.15.164 - - [03/Jun/2023:19:52:36] "GET /metrics HTTP/1.1" 200 212653 "" "Prometheus/2.33.4" 2023-06-03T19:52:37.334 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 82 16 100850 100834 76.8452 46.4375 0.0117076 0.0130069 2023-06-03T19:52:37.779 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:52:37 smithi089 bash[75068]: cluster 1685821956.2299404 mgr.y (mgr.34104) 447 : cluster 0 pgmap v373: 89 pgs: 82 active+clean, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped; 6.0 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 51 MiB/s wr, 821 op/s; 13352/295872 objects degraded (4.513%); 3.5 MiB/s, 55 objects/s recovering 2023-06-03T19:52:37.779 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:52:37 smithi089 bash[75068]: audit 1685821957.0814774 mon.c (mon.2) 220 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T19:52:37.779 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:52:37 smithi089 bash[75068]: cluster 1685821957.33438 mon.a (mon.0) 424 : cluster 3 Health check update: Degraded data redundancy: 13352/295872 objects degraded (4.513%), 7 pgs degraded (PG_DEGRADED) 2023-06-03T19:52:37.780 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:52:37 smithi089 bash[78452]: cluster 1685821956.2299404 mgr.y (mgr.34104) 447 : cluster 0 pgmap v373: 89 pgs: 82 active+clean, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped; 6.0 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 51 MiB/s wr, 821 op/s; 13352/295872 objects degraded (4.513%); 3.5 MiB/s, 55 objects/s recovering 2023-06-03T19:52:37.780 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:52:37 smithi089 bash[78452]: audit 1685821957.0814774 mon.c (mon.2) 220 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T19:52:37.780 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:52:37 smithi089 bash[78452]: cluster 1685821957.33438 mon.a (mon.0) 424 : cluster 3 Health check update: Degraded data redundancy: 13352/295872 objects degraded (4.513%), 7 pgs degraded (PG_DEGRADED) 2023-06-03T19:52:37.834 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:52:37 smithi164 bash[59462]: cluster 1685821956.2299404 mgr.y (mgr.34104) 447 : cluster 0 pgmap v373: 89 pgs: 82 active+clean, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped; 6.0 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 51 MiB/s wr, 821 op/s; 13352/295872 objects degraded (4.513%); 3.5 MiB/s, 55 objects/s recovering 2023-06-03T19:52:37.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:52:37 smithi164 bash[59462]: audit 1685821957.0814774 mon.c (mon.2) 220 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T19:52:37.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:52:37 smithi164 bash[59462]: cluster 1685821957.33438 mon.a (mon.0) 424 : cluster 3 Health check update: Degraded data redundancy: 13352/295872 objects degraded (4.513%), 7 pgs degraded (PG_DEGRADED) 2023-06-03T19:52:38.334 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 83 16 101606 101590 76.4886 47.25 0.0172024 0.0130663 2023-06-03T19:52:38.779 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:52:38 smithi089 bash[75068]: audit 1685821957.6491525 mon.a (mon.0) 425 : audit 0 from='client.? 172.21.15.89:0/90772274' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:52:38.779 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:52:38 smithi089 bash[78452]: audit 1685821957.6491525 mon.a (mon.0) 425 : audit 0 from='client.? 172.21.15.89:0/90772274' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:52:38.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:52:38 smithi164 bash[59462]: audit 1685821957.6491525 mon.a (mon.0) 425 : audit 0 from='client.? 172.21.15.89:0/90772274' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:52:39.334 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 84 16 102357 102341 76.1368 46.9375 0.0116595 0.0131281 2023-06-03T19:52:39.779 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:52:39 smithi089 bash[75068]: cluster 1685821958.2305698 mgr.y (mgr.34104) 448 : cluster 0 pgmap v374: 89 pgs: 82 active+clean, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped; 6.0 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 41 MiB/s wr, 654 op/s; 13352/295872 objects degraded (4.513%); 2.5 MiB/s, 39 objects/s recovering 2023-06-03T19:52:39.780 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:52:39 smithi089 bash[78452]: cluster 1685821958.2305698 mgr.y (mgr.34104) 448 : cluster 0 pgmap v374: 89 pgs: 82 active+clean, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped; 6.0 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 41 MiB/s wr, 654 op/s; 13352/295872 objects degraded (4.513%); 2.5 MiB/s, 39 objects/s recovering 2023-06-03T19:52:39.834 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:52:39 smithi164 bash[59462]: cluster 1685821958.2305698 mgr.y (mgr.34104) 448 : cluster 0 pgmap v374: 89 pgs: 82 active+clean, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped; 6.0 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 41 MiB/s wr, 654 op/s; 13352/295872 objects degraded (4.513%); 2.5 MiB/s, 39 objects/s recovering 2023-06-03T19:52:40.334 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 85 16 103060 103044 75.7579 43.9375 0.0408909 0.0131935 2023-06-03T19:52:40.779 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:52:40 smithi089 bash[78452]: audit 1685821960.266393 mon.a (mon.0) 426 : audit 0 from='client.? 172.21.15.89:0/11108154' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:52:40.779 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:52:40 smithi089 bash[75068]: audit 1685821960.266393 mon.a (mon.0) 426 : audit 0 from='client.? 172.21.15.89:0/11108154' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:52:40.834 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:52:40 smithi164 bash[59462]: audit 1685821960.266393 mon.a (mon.0) 426 : audit 0 from='client.? 172.21.15.89:0/11108154' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:52:41.335 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 86 16 103817 103801 75.4271 47.3125 0.0161091 0.0132505 2023-06-03T19:52:41.779 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:52:41 smithi089 bash[78452]: cluster 1685821960.2311893 mgr.y (mgr.34104) 449 : cluster 0 pgmap v375: 89 pgs: 82 active+clean, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped; 6.2 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 56 MiB/s wr, 889 op/s; 13352/304314 objects degraded (4.388%); 2.5 MiB/s, 39 objects/s recovering 2023-06-03T19:52:41.780 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:52:41 smithi089 bash[75068]: cluster 1685821960.2311893 mgr.y (mgr.34104) 449 : cluster 0 pgmap v375: 89 pgs: 82 active+clean, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped; 6.2 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 56 MiB/s wr, 889 op/s; 13352/304314 objects degraded (4.388%); 2.5 MiB/s, 39 objects/s recovering 2023-06-03T19:52:41.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:52:41 smithi164 bash[59462]: cluster 1685821960.2311893 mgr.y (mgr.34104) 449 : cluster 0 pgmap v375: 89 pgs: 82 active+clean, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped; 6.2 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 56 MiB/s wr, 889 op/s; 13352/304314 objects degraded (4.388%); 2.5 MiB/s, 39 objects/s recovering 2023-06-03T19:52:42.335 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 87 16 104567 104551 75.0989 46.875 0.0110505 0.0133078 2023-06-03T19:52:42.779 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:52:42 smithi089 bash[78452]: cluster 1685821962.335673 mon.a (mon.0) 427 : cluster 3 Health check update: Degraded data redundancy: 13352/304314 objects degraded (4.388%), 7 pgs degraded (PG_DEGRADED) 2023-06-03T19:52:42.779 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:52:42 smithi089 bash[75068]: cluster 1685821962.335673 mon.a (mon.0) 427 : cluster 3 Health check update: Degraded data redundancy: 13352/304314 objects degraded (4.388%), 7 pgs degraded (PG_DEGRADED) 2023-06-03T19:52:42.838 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:52:42 smithi164 bash[59462]: cluster 1685821962.335673 mon.a (mon.0) 427 : cluster 3 Health check update: Degraded data redundancy: 13352/304314 objects degraded (4.388%), 7 pgs degraded (PG_DEGRADED) 2023-06-03T19:52:43.335 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 88 16 105260 105244 74.7376 43.3125 0.013842 0.013374 2023-06-03T19:52:43.779 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:52:43 smithi089 bash[75068]: cluster 1685821962.2317023 mgr.y (mgr.34104) 450 : cluster 0 pgmap v376: 89 pgs: 82 active+clean, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped; 6.2 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 44 MiB/s wr, 710 op/s; 13082/306987 objects degraded (4.261%); 3.9 MiB/s, 62 objects/s recovering 2023-06-03T19:52:43.779 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:52:43 smithi089 bash[75068]: audit 1685821962.8749425 mon.b (mon.1) 27 : audit 0 from='client.? 172.21.15.89:0/2469423006' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:52:43.780 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:52:43 smithi089 bash[78452]: cluster 1685821962.2317023 mgr.y (mgr.34104) 450 : cluster 0 pgmap v376: 89 pgs: 82 active+clean, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped; 6.2 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 44 MiB/s wr, 710 op/s; 13082/306987 objects degraded (4.261%); 3.9 MiB/s, 62 objects/s recovering 2023-06-03T19:52:43.780 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:52:43 smithi089 bash[78452]: audit 1685821962.8749425 mon.b (mon.1) 27 : audit 0 from='client.? 172.21.15.89:0/2469423006' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:52:43.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:52:43 smithi164 bash[59462]: cluster 1685821962.2317023 mgr.y (mgr.34104) 450 : cluster 0 pgmap v376: 89 pgs: 82 active+clean, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped; 6.2 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 44 MiB/s wr, 710 op/s; 13082/306987 objects degraded (4.261%); 3.9 MiB/s, 62 objects/s recovering 2023-06-03T19:52:43.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:52:43 smithi164 bash[59462]: audit 1685821962.8749425 mon.b (mon.1) 27 : audit 0 from='client.? 172.21.15.89:0/2469423006' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:52:44.335 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 89 16 106004 105988 74.4203 46.5 0.0380424 0.0134317 2023-06-03T19:52:45.335 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 90 3 106681 106678 74.0725 43.125 0.0164912 0.0134946 2023-06-03T19:52:45.779 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:52:45 smithi089 bash[75068]: cluster 1685821964.2324903 mgr.y (mgr.34104) 451 : cluster 0 pgmap v377: 89 pgs: 82 active+clean, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped; 6.4 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 49 MiB/s wr, 781 op/s; 13082/312345 objects degraded (4.188%); 2.7 MiB/s, 42 objects/s recovering 2023-06-03T19:52:45.779 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:52:45 smithi089 bash[75068]: audit 1685821965.426147 mon.a (mon.0) 428 : audit 0 from='client.? 172.21.15.89:0/2135362615' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:52:45.780 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:52:45 smithi089 bash[78452]: cluster 1685821964.2324903 mgr.y (mgr.34104) 451 : cluster 0 pgmap v377: 89 pgs: 82 active+clean, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped; 6.4 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 49 MiB/s wr, 781 op/s; 13082/312345 objects degraded (4.188%); 2.7 MiB/s, 42 objects/s recovering 2023-06-03T19:52:45.780 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:52:45 smithi089 bash[78452]: audit 1685821965.426147 mon.a (mon.0) 428 : audit 0 from='client.? 172.21.15.89:0/2135362615' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:52:45.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:52:45 smithi164 bash[59462]: cluster 1685821964.2324903 mgr.y (mgr.34104) 451 : cluster 0 pgmap v377: 89 pgs: 82 active+clean, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped; 6.4 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 49 MiB/s wr, 781 op/s; 13082/312345 objects degraded (4.188%); 2.7 MiB/s, 42 objects/s recovering 2023-06-03T19:52:45.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:52:45 smithi164 bash[59462]: audit 1685821965.426147 mon.a (mon.0) 428 : audit 0 from='client.? 172.21.15.89:0/2135362615' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:52:46.335 INFO:tasks.radosbench.radosbench.0.smithi089.stdout:Total time run: 90.0149 2023-06-03T19:52:46.336 INFO:tasks.radosbench.radosbench.0.smithi089.stdout:Total writes made: 106681 2023-06-03T19:52:46.336 INFO:tasks.radosbench.radosbench.0.smithi089.stdout:Write size: 65536 2023-06-03T19:52:46.336 INFO:tasks.radosbench.radosbench.0.smithi089.stdout:Object size: 65536 2023-06-03T19:52:46.336 INFO:tasks.radosbench.radosbench.0.smithi089.stdout:Bandwidth (MB/sec): 74.0718 2023-06-03T19:52:46.336 INFO:tasks.radosbench.radosbench.0.smithi089.stdout:Stddev Bandwidth: 36.7564 2023-06-03T19:52:46.336 INFO:tasks.radosbench.radosbench.0.smithi089.stdout:Max bandwidth (MB/sec): 194.25 2023-06-03T19:52:46.336 INFO:tasks.radosbench.radosbench.0.smithi089.stdout:Min bandwidth (MB/sec): 0 2023-06-03T19:52:46.337 INFO:tasks.radosbench.radosbench.0.smithi089.stdout:Average IOPS: 1185 2023-06-03T19:52:46.337 INFO:tasks.radosbench.radosbench.0.smithi089.stdout:Stddev IOPS: 588.102 2023-06-03T19:52:46.337 INFO:tasks.radosbench.radosbench.0.smithi089.stdout:Max IOPS: 3108 2023-06-03T19:52:46.337 INFO:tasks.radosbench.radosbench.0.smithi089.stdout:Min IOPS: 0 2023-06-03T19:52:46.337 INFO:tasks.radosbench.radosbench.0.smithi089.stdout:Average Latency(s): 0.0134948 2023-06-03T19:52:46.337 INFO:tasks.radosbench.radosbench.0.smithi089.stdout:Stddev Latency(s): 0.0311269 2023-06-03T19:52:46.337 INFO:tasks.radosbench.radosbench.0.smithi089.stdout:Max latency(s): 2.24771 2023-06-03T19:52:46.337 INFO:tasks.radosbench.radosbench.0.smithi089.stdout:Min latency(s): 0.00131775 2023-06-03T19:52:46.347 INFO:tasks.radosbench.radosbench.0.smithi089.stdout:Cleaning up (deleting benchmark objects) 2023-06-03T19:52:46.529 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 19:52:46 smithi089 bash[71163]: ::ffff:172.21.15.164 - - [03/Jun/2023:19:52:46] "GET /metrics HTTP/1.1" 200 212696 "" "Prometheus/2.33.4" 2023-06-03T19:52:47.779 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:52:47 smithi089 bash[78452]: cluster 1685821966.2333117 mgr.y (mgr.34104) 452 : cluster 0 pgmap v378: 89 pgs: 82 active+clean, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped; 6.5 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 48 MiB/s wr, 766 op/s; 12708/317703 objects degraded (4.000%); 4.6 MiB/s, 73 objects/s recovering 2023-06-03T19:52:47.779 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:52:47 smithi089 bash[78452]: cluster 1685821967.3369777 mon.a (mon.0) 429 : cluster 3 Health check update: Degraded data redundancy: 12708/317703 objects degraded (4.000%), 7 pgs degraded (PG_DEGRADED) 2023-06-03T19:52:47.779 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:52:47 smithi089 bash[78452]: cluster 1685821967.346842 mon.a (mon.0) 430 : cluster 0 osdmap e106: 8 total, 8 up, 8 in 2023-06-03T19:52:47.779 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:52:47 smithi089 bash[75068]: cluster 1685821966.2333117 mgr.y (mgr.34104) 452 : cluster 0 pgmap v378: 89 pgs: 82 active+clean, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped; 6.5 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 48 MiB/s wr, 766 op/s; 12708/317703 objects degraded (4.000%); 4.6 MiB/s, 73 objects/s recovering 2023-06-03T19:52:47.780 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:52:47 smithi089 bash[75068]: cluster 1685821967.3369777 mon.a (mon.0) 429 : cluster 3 Health check update: Degraded data redundancy: 12708/317703 objects degraded (4.000%), 7 pgs degraded (PG_DEGRADED) 2023-06-03T19:52:47.780 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:52:47 smithi089 bash[75068]: cluster 1685821967.346842 mon.a (mon.0) 430 : cluster 0 osdmap e106: 8 total, 8 up, 8 in 2023-06-03T19:52:47.834 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:52:47 smithi164 bash[59462]: cluster 1685821966.2333117 mgr.y (mgr.34104) 452 : cluster 0 pgmap v378: 89 pgs: 82 active+clean, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped; 6.5 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 48 MiB/s wr, 766 op/s; 12708/317703 objects degraded (4.000%); 4.6 MiB/s, 73 objects/s recovering 2023-06-03T19:52:47.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:52:47 smithi164 bash[59462]: cluster 1685821967.3369777 mon.a (mon.0) 429 : cluster 3 Health check update: Degraded data redundancy: 12708/317703 objects degraded (4.000%), 7 pgs degraded (PG_DEGRADED) 2023-06-03T19:52:47.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:52:47 smithi164 bash[59462]: cluster 1685821967.346842 mon.a (mon.0) 430 : cluster 0 osdmap e106: 8 total, 8 up, 8 in 2023-06-03T19:52:48.279 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 19:52:47 smithi089 bash[71163]: debug 2023-06-03T19:52:47.888+0000 7fe48642e700 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2023-06-03T19:52:48.779 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:52:48 smithi089 bash[75068]: cluster 1685821967.3494353 osd.3 (osd.3) 29 : cluster 3 failed to encode map e106 with expected crc 2023-06-03T19:52:48.779 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:52:48 smithi089 bash[75068]: cluster 1685821967.3512943 osd.4 (osd.4) 22 : cluster 3 failed to encode map e106 with expected crc 2023-06-03T19:52:48.780 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:52:48 smithi089 bash[75068]: cluster 1685821967.3540344 osd.6 (osd.6) 23 : cluster 3 failed to encode map e106 with expected crc 2023-06-03T19:52:48.780 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:52:48 smithi089 bash[75068]: cluster 1685821967.356574 osd.7 (osd.7) 19 : cluster 3 failed to encode map e106 with expected crc 2023-06-03T19:52:48.780 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:52:48 smithi089 bash[75068]: cluster 1685821967.3604558 osd.3 (osd.3) 30 : cluster 3 failed to encode map e106 with expected crc 2023-06-03T19:52:48.780 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:52:48 smithi089 bash[75068]: cluster 1685821967.3608122 osd.2 (osd.2) 29 : cluster 3 failed to encode map e106 with expected crc 2023-06-03T19:52:48.780 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:52:48 smithi089 bash[75068]: cluster 1685821967.36478 osd.5 (osd.5) 25 : cluster 3 failed to encode map e106 with expected crc 2023-06-03T19:52:48.781 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:52:48 smithi089 bash[75068]: cluster 1685821967.3655434 osd.2 (osd.2) 30 : cluster 3 failed to encode map e106 with expected crc 2023-06-03T19:52:48.781 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:52:48 smithi089 bash[75068]: audit 1685821967.8694592 mon.b (mon.1) 28 : audit 0 from='client.? 172.21.15.89:0/1263555661' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:52:48.781 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:52:48 smithi089 bash[75068]: audit 1685821967.889432 mon.c (mon.2) 221 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-06-03T19:52:48.781 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:52:48 smithi089 bash[75068]: cluster 1685821968.3649707 mon.a (mon.0) 431 : cluster 0 osdmap e107: 8 total, 8 up, 8 in 2023-06-03T19:52:48.782 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:52:48 smithi089 bash[78452]: cluster 1685821967.3494353 osd.3 (osd.3) 29 : cluster 3 failed to encode map e106 with expected crc 2023-06-03T19:52:48.782 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:52:48 smithi089 bash[78452]: cluster 1685821967.3512943 osd.4 (osd.4) 22 : cluster 3 failed to encode map e106 with expected crc 2023-06-03T19:52:48.782 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:52:48 smithi089 bash[78452]: cluster 1685821967.3540344 osd.6 (osd.6) 23 : cluster 3 failed to encode map e106 with expected crc 2023-06-03T19:52:48.782 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:52:48 smithi089 bash[78452]: cluster 1685821967.356574 osd.7 (osd.7) 19 : cluster 3 failed to encode map e106 with expected crc 2023-06-03T19:52:48.783 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:52:48 smithi089 bash[78452]: cluster 1685821967.3604558 osd.3 (osd.3) 30 : cluster 3 failed to encode map e106 with expected crc 2023-06-03T19:52:48.783 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:52:48 smithi089 bash[78452]: cluster 1685821967.3608122 osd.2 (osd.2) 29 : cluster 3 failed to encode map e106 with expected crc 2023-06-03T19:52:48.784 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:52:48 smithi089 bash[78452]: cluster 1685821967.36478 osd.5 (osd.5) 25 : cluster 3 failed to encode map e106 with expected crc 2023-06-03T19:52:48.784 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:52:48 smithi089 bash[78452]: cluster 1685821967.3655434 osd.2 (osd.2) 30 : cluster 3 failed to encode map e106 with expected crc 2023-06-03T19:52:48.785 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:52:48 smithi089 bash[78452]: audit 1685821967.8694592 mon.b (mon.1) 28 : audit 0 from='client.? 172.21.15.89:0/1263555661' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:52:48.785 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:52:48 smithi089 bash[78452]: audit 1685821967.889432 mon.c (mon.2) 221 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-06-03T19:52:48.785 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:52:48 smithi089 bash[78452]: cluster 1685821968.3649707 mon.a (mon.0) 431 : cluster 0 osdmap e107: 8 total, 8 up, 8 in 2023-06-03T19:52:48.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:52:48 smithi164 bash[59462]: cluster 1685821967.3494353 osd.3 (osd.3) 29 : cluster 3 failed to encode map e106 with expected crc 2023-06-03T19:52:48.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:52:48 smithi164 bash[59462]: cluster 1685821967.3512943 osd.4 (osd.4) 22 : cluster 3 failed to encode map e106 with expected crc 2023-06-03T19:52:48.836 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:52:48 smithi164 bash[59462]: cluster 1685821967.3540344 osd.6 (osd.6) 23 : cluster 3 failed to encode map e106 with expected crc 2023-06-03T19:52:48.836 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:52:48 smithi164 bash[59462]: cluster 1685821967.356574 osd.7 (osd.7) 19 : cluster 3 failed to encode map e106 with expected crc 2023-06-03T19:52:48.836 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:52:48 smithi164 bash[59462]: cluster 1685821967.3604558 osd.3 (osd.3) 30 : cluster 3 failed to encode map e106 with expected crc 2023-06-03T19:52:48.836 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:52:48 smithi164 bash[59462]: cluster 1685821967.3608122 osd.2 (osd.2) 29 : cluster 3 failed to encode map e106 with expected crc 2023-06-03T19:52:48.837 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:52:48 smithi164 bash[59462]: cluster 1685821967.36478 osd.5 (osd.5) 25 : cluster 3 failed to encode map e106 with expected crc 2023-06-03T19:52:48.838 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:52:48 smithi164 bash[59462]: cluster 1685821967.3655434 osd.2 (osd.2) 30 : cluster 3 failed to encode map e106 with expected crc 2023-06-03T19:52:48.838 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:52:48 smithi164 bash[59462]: audit 1685821967.8694592 mon.b (mon.1) 28 : audit 0 from='client.? 172.21.15.89:0/1263555661' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:52:48.838 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:52:48 smithi164 bash[59462]: audit 1685821967.889432 mon.c (mon.2) 221 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-06-03T19:52:48.838 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:52:48 smithi164 bash[59462]: cluster 1685821968.3649707 mon.a (mon.0) 431 : cluster 0 osdmap e107: 8 total, 8 up, 8 in 2023-06-03T19:52:49.779 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:52:49 smithi089 bash[75068]: audit 1685821967.8897414 mgr.y (mgr.34104) 453 : audit 0 from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-06-03T19:52:49.779 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:52:49 smithi089 bash[75068]: cephadm 1685821967.8917909 mgr.y (mgr.34104) 454 : cephadm 1 Upgrade: unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2023-06-03T19:52:49.779 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:52:49 smithi089 bash[75068]: cluster 1685821968.234195 mgr.y (mgr.34104) 455 : cluster 0 pgmap v380: 89 pgs: 1 peering, 82 active+clean, 6 active+recovery_wait+undersized+degraded+remapped; 6.5 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 45 MiB/s wr, 732 op/s; 12610/317541 objects degraded (3.971%); 4.6 MiB/s, 74 objects/s recovering 2023-06-03T19:52:49.780 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:52:49 smithi089 bash[75068]: cluster 1685821968.357671 osd.7 (osd.7) 20 : cluster 3 failed to encode map e107 with expected crc 2023-06-03T19:52:49.780 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:52:49 smithi089 bash[75068]: cluster 1685821968.3652394 osd.2 (osd.2) 31 : cluster 3 failed to encode map e107 with expected crc 2023-06-03T19:52:49.780 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:52:49 smithi089 bash[75068]: cluster 1685821968.375911 osd.5 (osd.5) 26 : cluster 3 failed to encode map e107 with expected crc 2023-06-03T19:52:49.780 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:52:49 smithi089 bash[75068]: cluster 1685821968.376406 osd.3 (osd.3) 31 : cluster 3 failed to encode map e107 with expected crc 2023-06-03T19:52:49.780 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:52:49 smithi089 bash[78452]: audit 1685821967.8897414 mgr.y (mgr.34104) 453 : audit 0 from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-06-03T19:52:49.780 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:52:49 smithi089 bash[78452]: cephadm 1685821967.8917909 mgr.y (mgr.34104) 454 : cephadm 1 Upgrade: unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2023-06-03T19:52:49.781 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:52:49 smithi089 bash[78452]: cluster 1685821968.234195 mgr.y (mgr.34104) 455 : cluster 0 pgmap v380: 89 pgs: 1 peering, 82 active+clean, 6 active+recovery_wait+undersized+degraded+remapped; 6.5 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 45 MiB/s wr, 732 op/s; 12610/317541 objects degraded (3.971%); 4.6 MiB/s, 74 objects/s recovering 2023-06-03T19:52:49.781 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:52:49 smithi089 bash[78452]: cluster 1685821968.357671 osd.7 (osd.7) 20 : cluster 3 failed to encode map e107 with expected crc 2023-06-03T19:52:49.781 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:52:49 smithi089 bash[78452]: cluster 1685821968.3652394 osd.2 (osd.2) 31 : cluster 3 failed to encode map e107 with expected crc 2023-06-03T19:52:49.781 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:52:49 smithi089 bash[78452]: cluster 1685821968.375911 osd.5 (osd.5) 26 : cluster 3 failed to encode map e107 with expected crc 2023-06-03T19:52:49.781 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:52:49 smithi089 bash[78452]: cluster 1685821968.376406 osd.3 (osd.3) 31 : cluster 3 failed to encode map e107 with expected crc 2023-06-03T19:52:49.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:52:49 smithi164 bash[59462]: audit 1685821967.8897414 mgr.y (mgr.34104) 453 : audit 0 from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-06-03T19:52:49.836 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:52:49 smithi164 bash[59462]: cephadm 1685821967.8917909 mgr.y (mgr.34104) 454 : cephadm 1 Upgrade: unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2023-06-03T19:52:49.836 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:52:49 smithi164 bash[59462]: cluster 1685821968.234195 mgr.y (mgr.34104) 455 : cluster 0 pgmap v380: 89 pgs: 1 peering, 82 active+clean, 6 active+recovery_wait+undersized+degraded+remapped; 6.5 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 45 MiB/s wr, 732 op/s; 12610/317541 objects degraded (3.971%); 4.6 MiB/s, 74 objects/s recovering 2023-06-03T19:52:49.836 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:52:49 smithi164 bash[59462]: cluster 1685821968.357671 osd.7 (osd.7) 20 : cluster 3 failed to encode map e107 with expected crc 2023-06-03T19:52:49.836 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:52:49 smithi164 bash[59462]: cluster 1685821968.3652394 osd.2 (osd.2) 31 : cluster 3 failed to encode map e107 with expected crc 2023-06-03T19:52:49.836 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:52:49 smithi164 bash[59462]: cluster 1685821968.375911 osd.5 (osd.5) 26 : cluster 3 failed to encode map e107 with expected crc 2023-06-03T19:52:49.837 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:52:49 smithi164 bash[59462]: cluster 1685821968.376406 osd.3 (osd.3) 31 : cluster 3 failed to encode map e107 with expected crc 2023-06-03T19:52:50.779 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:52:50 smithi089 bash[78452]: audit 1685821970.4448524 mon.a (mon.0) 432 : audit 0 from='client.? 172.21.15.89:0/813482529' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:52:50.779 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:52:50 smithi089 bash[75068]: audit 1685821970.4448524 mon.a (mon.0) 432 : audit 0 from='client.? 172.21.15.89:0/813482529' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:52:50.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:52:50 smithi164 bash[59462]: audit 1685821970.4448524 mon.a (mon.0) 432 : audit 0 from='client.? 172.21.15.89:0/813482529' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:52:51.779 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:52:51 smithi089 bash[78452]: cluster 1685821970.2348726 mgr.y (mgr.34104) 456 : cluster 0 pgmap v382: 89 pgs: 1 peering, 82 active+clean, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped; 6.4 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 127 B/s rd, 34 MiB/s wr, 689 op/s; 12569/316695 objects degraded (3.969%); 4.0 MiB/s, 64 objects/s recovering 2023-06-03T19:52:51.780 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:52:51 smithi089 bash[75068]: cluster 1685821970.2348726 mgr.y (mgr.34104) 456 : cluster 0 pgmap v382: 89 pgs: 1 peering, 82 active+clean, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped; 6.4 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 127 B/s rd, 34 MiB/s wr, 689 op/s; 12569/316695 objects degraded (3.969%); 4.0 MiB/s, 64 objects/s recovering 2023-06-03T19:52:51.836 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:52:51 smithi164 bash[59462]: cluster 1685821970.2348726 mgr.y (mgr.34104) 456 : cluster 0 pgmap v382: 89 pgs: 1 peering, 82 active+clean, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped; 6.4 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 127 B/s rd, 34 MiB/s wr, 689 op/s; 12569/316695 objects degraded (3.969%); 4.0 MiB/s, 64 objects/s recovering 2023-06-03T19:52:52.779 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:52:52 smithi089 bash[75068]: audit 1685821972.082291 mon.c (mon.2) 222 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T19:52:52.779 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:52:52 smithi089 bash[75068]: cluster 1685821972.339668 mon.a (mon.0) 433 : cluster 3 Health check update: Degraded data redundancy: 12569/316695 objects degraded (3.969%), 6 pgs degraded (PG_DEGRADED) 2023-06-03T19:52:52.780 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:52:52 smithi089 bash[78452]: audit 1685821972.082291 mon.c (mon.2) 222 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T19:52:52.780 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:52:52 smithi089 bash[78452]: cluster 1685821972.339668 mon.a (mon.0) 433 : cluster 3 Health check update: Degraded data redundancy: 12569/316695 objects degraded (3.969%), 6 pgs degraded (PG_DEGRADED) 2023-06-03T19:52:52.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:52:52 smithi164 bash[59462]: audit 1685821972.082291 mon.c (mon.2) 222 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T19:52:52.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:52:52 smithi164 bash[59462]: cluster 1685821972.339668 mon.a (mon.0) 433 : cluster 3 Health check update: Degraded data redundancy: 12569/316695 objects degraded (3.969%), 6 pgs degraded (PG_DEGRADED) 2023-06-03T19:52:53.779 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:52:53 smithi089 bash[75068]: cluster 1685821972.2354543 mgr.y (mgr.34104) 457 : cluster 0 pgmap v383: 89 pgs: 1 peering, 82 active+clean, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped; 6.4 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 127 B/s rd, 20 MiB/s wr, 535 op/s; 12569/315036 objects degraded (3.990%); 4.0 MiB/s, 64 objects/s recovering 2023-06-03T19:52:53.779 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:52:53 smithi089 bash[75068]: audit 1685821973.0331917 mon.a (mon.0) 434 : audit 0 from='client.? 172.21.15.89:0/3810616677' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:52:53.780 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:52:53 smithi089 bash[78452]: cluster 1685821972.2354543 mgr.y (mgr.34104) 457 : cluster 0 pgmap v383: 89 pgs: 1 peering, 82 active+clean, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped; 6.4 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 127 B/s rd, 20 MiB/s wr, 535 op/s; 12569/315036 objects degraded (3.990%); 4.0 MiB/s, 64 objects/s recovering 2023-06-03T19:52:53.780 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:52:53 smithi089 bash[78452]: audit 1685821973.0331917 mon.a (mon.0) 434 : audit 0 from='client.? 172.21.15.89:0/3810616677' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:52:53.834 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:52:53 smithi164 bash[59462]: cluster 1685821972.2354543 mgr.y (mgr.34104) 457 : cluster 0 pgmap v383: 89 pgs: 1 peering, 82 active+clean, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped; 6.4 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 127 B/s rd, 20 MiB/s wr, 535 op/s; 12569/315036 objects degraded (3.990%); 4.0 MiB/s, 64 objects/s recovering 2023-06-03T19:52:53.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:52:53 smithi164 bash[59462]: audit 1685821973.0331917 mon.a (mon.0) 434 : audit 0 from='client.? 172.21.15.89:0/3810616677' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:52:55.779 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:52:55 smithi089 bash[78452]: cluster 1685821974.2360828 mgr.y (mgr.34104) 458 : cluster 0 pgmap v384: 89 pgs: 83 active+clean, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped; 6.3 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 127 B/s rd, 6.3 MiB/s wr, 567 op/s; 12459/308907 objects degraded (4.033%); 1.9 MiB/s, 31 objects/s recovering 2023-06-03T19:52:55.779 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:52:55 smithi089 bash[75068]: cluster 1685821974.2360828 mgr.y (mgr.34104) 458 : cluster 0 pgmap v384: 89 pgs: 83 active+clean, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped; 6.3 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 127 B/s rd, 6.3 MiB/s wr, 567 op/s; 12459/308907 objects degraded (4.033%); 1.9 MiB/s, 31 objects/s recovering 2023-06-03T19:52:55.834 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:52:55 smithi164 bash[59462]: cluster 1685821974.2360828 mgr.y (mgr.34104) 458 : cluster 0 pgmap v384: 89 pgs: 83 active+clean, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped; 6.3 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 127 B/s rd, 6.3 MiB/s wr, 567 op/s; 12459/308907 objects degraded (4.033%); 1.9 MiB/s, 31 objects/s recovering 2023-06-03T19:52:56.502 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 19:52:56 smithi089 bash[71163]: ::ffff:172.21.15.164 - - [03/Jun/2023:19:52:56] "GET /metrics HTTP/1.1" 200 212697 "" "Prometheus/2.33.4" 2023-06-03T19:52:56.503 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:52:56 smithi089 bash[75068]: audit 1685821975.5740871 mon.a (mon.0) 435 : audit 0 from='client.? 172.21.15.89:0/104371671' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:52:56.779 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:52:56 smithi089 bash[78452]: audit 1685821975.5740871 mon.a (mon.0) 435 : audit 0 from='client.? 172.21.15.89:0/104371671' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:52:56.834 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:52:56 smithi164 bash[59462]: audit 1685821975.5740871 mon.a (mon.0) 435 : audit 0 from='client.? 172.21.15.89:0/104371671' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:52:57.779 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:52:57 smithi089 bash[75068]: cluster 1685821976.2365768 mgr.y (mgr.34104) 459 : cluster 0 pgmap v385: 89 pgs: 83 active+clean, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped; 6.2 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 115 B/s rd, 5.7 MiB/s wr, 698 op/s; 12459/303894 objects degraded (4.100%); 1.8 MiB/s, 28 objects/s recovering 2023-06-03T19:52:57.779 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:52:57 smithi089 bash[75068]: cluster 1685821977.3408036 mon.a (mon.0) 436 : cluster 3 Health check update: Degraded data redundancy: 12459/303894 objects degraded (4.100%), 6 pgs degraded (PG_DEGRADED) 2023-06-03T19:52:57.780 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:52:57 smithi089 bash[78452]: cluster 1685821976.2365768 mgr.y (mgr.34104) 459 : cluster 0 pgmap v385: 89 pgs: 83 active+clean, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped; 6.2 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 115 B/s rd, 5.7 MiB/s wr, 698 op/s; 12459/303894 objects degraded (4.100%); 1.8 MiB/s, 28 objects/s recovering 2023-06-03T19:52:57.780 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:52:57 smithi089 bash[78452]: cluster 1685821977.3408036 mon.a (mon.0) 436 : cluster 3 Health check update: Degraded data redundancy: 12459/303894 objects degraded (4.100%), 6 pgs degraded (PG_DEGRADED) 2023-06-03T19:52:57.834 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:52:57 smithi164 bash[59462]: cluster 1685821976.2365768 mgr.y (mgr.34104) 459 : cluster 0 pgmap v385: 89 pgs: 83 active+clean, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped; 6.2 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 115 B/s rd, 5.7 MiB/s wr, 698 op/s; 12459/303894 objects degraded (4.100%); 1.8 MiB/s, 28 objects/s recovering 2023-06-03T19:52:57.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:52:57 smithi164 bash[59462]: cluster 1685821977.3408036 mon.a (mon.0) 436 : cluster 3 Health check update: Degraded data redundancy: 12459/303894 objects degraded (4.100%), 6 pgs degraded (PG_DEGRADED) 2023-06-03T19:52:58.779 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:52:58 smithi089 bash[75068]: audit 1685821978.1411917 mon.a (mon.0) 437 : audit 0 from='client.? 172.21.15.89:0/3459526158' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:52:58.779 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:52:58 smithi089 bash[78452]: audit 1685821978.1411917 mon.a (mon.0) 437 : audit 0 from='client.? 172.21.15.89:0/3459526158' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:52:58.834 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:52:58 smithi164 bash[59462]: audit 1685821978.1411917 mon.a (mon.0) 437 : audit 0 from='client.? 172.21.15.89:0/3459526158' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:52:59.779 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:52:59 smithi089 bash[75068]: cluster 1685821978.2407508 mgr.y (mgr.34104) 460 : cluster 0 pgmap v386: 89 pgs: 83 active+clean, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped; 6.2 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 102 B/s rd, 5.0 MiB/s wr, 641 op/s; 12459/303120 objects degraded (4.110%); 966 KiB/s, 15 objects/s recovering 2023-06-03T19:52:59.779 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:52:59 smithi089 bash[78452]: cluster 1685821978.2407508 mgr.y (mgr.34104) 460 : cluster 0 pgmap v386: 89 pgs: 83 active+clean, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped; 6.2 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 102 B/s rd, 5.0 MiB/s wr, 641 op/s; 12459/303120 objects degraded (4.110%); 966 KiB/s, 15 objects/s recovering 2023-06-03T19:52:59.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:52:59 smithi164 bash[59462]: cluster 1685821978.2407508 mgr.y (mgr.34104) 460 : cluster 0 pgmap v386: 89 pgs: 83 active+clean, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped; 6.2 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 102 B/s rd, 5.0 MiB/s wr, 641 op/s; 12459/303120 objects degraded (4.110%); 966 KiB/s, 15 objects/s recovering 2023-06-03T19:53:01.779 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:53:01 smithi089 bash[75068]: cluster 1685821980.2415876 mgr.y (mgr.34104) 461 : cluster 0 pgmap v387: 89 pgs: 83 active+clean, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped; 6.0 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 1.0 MiB/s wr, 685 op/s; 12344/294651 objects degraded (4.189%); 1.2 MiB/s, 18 objects/s recovering 2023-06-03T19:53:01.780 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:53:01 smithi089 bash[75068]: audit 1685821980.7082276 mon.b (mon.1) 29 : audit 0 from='client.? 172.21.15.89:0/3113942348' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:53:01.780 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:53:01 smithi089 bash[78452]: cluster 1685821980.2415876 mgr.y (mgr.34104) 461 : cluster 0 pgmap v387: 89 pgs: 83 active+clean, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped; 6.0 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 1.0 MiB/s wr, 685 op/s; 12344/294651 objects degraded (4.189%); 1.2 MiB/s, 18 objects/s recovering 2023-06-03T19:53:01.780 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:53:01 smithi089 bash[78452]: audit 1685821980.7082276 mon.b (mon.1) 29 : audit 0 from='client.? 172.21.15.89:0/3113942348' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:53:01.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:53:01 smithi164 bash[59462]: cluster 1685821980.2415876 mgr.y (mgr.34104) 461 : cluster 0 pgmap v387: 89 pgs: 83 active+clean, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped; 6.0 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 1.0 MiB/s wr, 685 op/s; 12344/294651 objects degraded (4.189%); 1.2 MiB/s, 18 objects/s recovering 2023-06-03T19:53:01.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:53:01 smithi164 bash[59462]: audit 1685821980.7082276 mon.b (mon.1) 29 : audit 0 from='client.? 172.21.15.89:0/3113942348' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:53:02.834 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:53:02 smithi164 bash[59462]: cluster 1685821982.342028 mon.a (mon.0) 438 : cluster 3 Health check update: Degraded data redundancy: 12344/294651 objects degraded (4.189%), 6 pgs degraded (PG_DEGRADED) 2023-06-03T19:53:03.029 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:53:02 smithi089 bash[75068]: cluster 1685821982.342028 mon.a (mon.0) 438 : cluster 3 Health check update: Degraded data redundancy: 12344/294651 objects degraded (4.189%), 6 pgs degraded (PG_DEGRADED) 2023-06-03T19:53:03.030 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:53:02 smithi089 bash[78452]: cluster 1685821982.342028 mon.a (mon.0) 438 : cluster 3 Health check update: Degraded data redundancy: 12344/294651 objects degraded (4.189%), 6 pgs degraded (PG_DEGRADED) 2023-06-03T19:53:03.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:53:03 smithi164 bash[59462]: cluster 1685821982.2422242 mgr.y (mgr.34104) 462 : cluster 0 pgmap v388: 89 pgs: 83 active+clean, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped; 5.9 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 673 op/s; 12344/292446 objects degraded (4.221%); 1.2 MiB/s, 18 objects/s recovering 2023-06-03T19:53:03.836 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:53:03 smithi164 bash[59462]: audit 1685821982.90645 mon.c (mon.2) 223 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-03T19:53:03.836 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:53:03 smithi164 bash[59462]: audit 1685821983.3018627 mon.b (mon.1) 30 : audit 0 from='client.? 172.21.15.89:0/2205567892' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:53:04.029 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:53:03 smithi089 bash[75068]: cluster 1685821982.2422242 mgr.y (mgr.34104) 462 : cluster 0 pgmap v388: 89 pgs: 83 active+clean, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped; 5.9 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 673 op/s; 12344/292446 objects degraded (4.221%); 1.2 MiB/s, 18 objects/s recovering 2023-06-03T19:53:04.029 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:53:03 smithi089 bash[75068]: audit 1685821982.90645 mon.c (mon.2) 223 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-03T19:53:04.030 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:53:03 smithi089 bash[75068]: audit 1685821983.3018627 mon.b (mon.1) 30 : audit 0 from='client.? 172.21.15.89:0/2205567892' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:53:04.030 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:53:03 smithi089 bash[78452]: cluster 1685821982.2422242 mgr.y (mgr.34104) 462 : cluster 0 pgmap v388: 89 pgs: 83 active+clean, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped; 5.9 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 673 op/s; 12344/292446 objects degraded (4.221%); 1.2 MiB/s, 18 objects/s recovering 2023-06-03T19:53:04.030 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:53:03 smithi089 bash[78452]: audit 1685821982.90645 mon.c (mon.2) 223 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-03T19:53:04.030 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:53:03 smithi089 bash[78452]: audit 1685821983.3018627 mon.b (mon.1) 30 : audit 0 from='client.? 172.21.15.89:0/2205567892' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:53:05.834 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:53:05 smithi164 bash[59462]: cluster 1685821984.2430778 mgr.y (mgr.34104) 463 : cluster 0 pgmap v389: 89 pgs: 83 active+clean, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped; 5.8 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 795 op/s; 12232/286365 objects degraded (4.271%); 1.8 MiB/s, 28 objects/s recovering 2023-06-03T19:53:06.029 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:53:05 smithi089 bash[75068]: cluster 1685821984.2430778 mgr.y (mgr.34104) 463 : cluster 0 pgmap v389: 89 pgs: 83 active+clean, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped; 5.8 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 795 op/s; 12232/286365 objects degraded (4.271%); 1.8 MiB/s, 28 objects/s recovering 2023-06-03T19:53:06.030 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:53:05 smithi089 bash[78452]: cluster 1685821984.2430778 mgr.y (mgr.34104) 463 : cluster 0 pgmap v389: 89 pgs: 83 active+clean, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped; 5.8 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 795 op/s; 12232/286365 objects degraded (4.271%); 1.8 MiB/s, 28 objects/s recovering 2023-06-03T19:53:06.529 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 19:53:06 smithi089 bash[71163]: ::ffff:172.21.15.164 - - [03/Jun/2023:19:53:06] "GET /metrics HTTP/1.1" 200 212697 "" "Prometheus/2.33.4" 2023-06-03T19:53:06.834 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:53:06 smithi164 bash[59462]: audit 1685821985.9203827 mon.c (mon.2) 224 : audit 0 from='client.? 172.21.15.89:0/3463794553' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:53:07.030 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:53:06 smithi089 bash[75068]: audit 1685821985.9203827 mon.c (mon.2) 224 : audit 0 from='client.? 172.21.15.89:0/3463794553' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:53:07.030 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:53:06 smithi089 bash[78452]: audit 1685821985.9203827 mon.c (mon.2) 224 : audit 0 from='client.? 172.21.15.89:0/3463794553' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:53:07.834 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:53:07 smithi164 bash[59462]: cluster 1685821986.2437801 mgr.y (mgr.34104) 464 : cluster 0 pgmap v390: 89 pgs: 83 active+clean, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped; 5.7 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 759 op/s; 12232/281556 objects degraded (4.344%); 1.2 MiB/s, 18 objects/s recovering 2023-06-03T19:53:07.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:53:07 smithi164 bash[59462]: audit 1685821987.0822623 mon.c (mon.2) 225 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T19:53:07.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:53:07 smithi164 bash[59462]: cluster 1685821987.3433228 mon.a (mon.0) 439 : cluster 3 Health check update: Degraded data redundancy: 12232/281556 objects degraded (4.344%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-06-03T19:53:08.029 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:53:07 smithi089 bash[75068]: cluster 1685821986.2437801 mgr.y (mgr.34104) 464 : cluster 0 pgmap v390: 89 pgs: 83 active+clean, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped; 5.7 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 759 op/s; 12232/281556 objects degraded (4.344%); 1.2 MiB/s, 18 objects/s recovering 2023-06-03T19:53:08.029 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:53:07 smithi089 bash[75068]: audit 1685821987.0822623 mon.c (mon.2) 225 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T19:53:08.030 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:53:07 smithi089 bash[75068]: cluster 1685821987.3433228 mon.a (mon.0) 439 : cluster 3 Health check update: Degraded data redundancy: 12232/281556 objects degraded (4.344%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-06-03T19:53:08.030 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:53:07 smithi089 bash[78452]: cluster 1685821986.2437801 mgr.y (mgr.34104) 464 : cluster 0 pgmap v390: 89 pgs: 83 active+clean, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped; 5.7 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 759 op/s; 12232/281556 objects degraded (4.344%); 1.2 MiB/s, 18 objects/s recovering 2023-06-03T19:53:08.030 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:53:07 smithi089 bash[78452]: audit 1685821987.0822623 mon.c (mon.2) 225 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T19:53:08.030 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:53:07 smithi089 bash[78452]: cluster 1685821987.3433228 mon.a (mon.0) 439 : cluster 3 Health check update: Degraded data redundancy: 12232/281556 objects degraded (4.344%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-06-03T19:53:09.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:53:09 smithi164 bash[59462]: cluster 1685821988.2444825 mgr.y (mgr.34104) 465 : cluster 0 pgmap v391: 89 pgs: 83 active+clean, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped; 5.7 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 640 op/s; 12232/280839 objects degraded (4.356%); 1.2 MiB/s, 18 objects/s recovering 2023-06-03T19:53:09.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:53:09 smithi164 bash[59462]: audit 1685821988.6003773 mon.b (mon.1) 31 : audit 0 from='client.? 172.21.15.89:0/3580636814' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:53:09.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:53:09 smithi164 bash[59462]: audit 1685821989.5487325 mon.a (mon.0) 440 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:53:10.029 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:53:09 smithi089 bash[75068]: cluster 1685821988.2444825 mgr.y (mgr.34104) 465 : cluster 0 pgmap v391: 89 pgs: 83 active+clean, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped; 5.7 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 640 op/s; 12232/280839 objects degraded (4.356%); 1.2 MiB/s, 18 objects/s recovering 2023-06-03T19:53:10.029 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:53:09 smithi089 bash[75068]: audit 1685821988.6003773 mon.b (mon.1) 31 : audit 0 from='client.? 172.21.15.89:0/3580636814' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:53:10.030 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:53:09 smithi089 bash[75068]: audit 1685821989.5487325 mon.a (mon.0) 440 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:53:10.030 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:53:09 smithi089 bash[78452]: cluster 1685821988.2444825 mgr.y (mgr.34104) 465 : cluster 0 pgmap v391: 89 pgs: 83 active+clean, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped; 5.7 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 640 op/s; 12232/280839 objects degraded (4.356%); 1.2 MiB/s, 18 objects/s recovering 2023-06-03T19:53:10.030 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:53:09 smithi089 bash[78452]: audit 1685821988.6003773 mon.b (mon.1) 31 : audit 0 from='client.? 172.21.15.89:0/3580636814' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:53:10.030 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:53:09 smithi089 bash[78452]: audit 1685821989.5487325 mon.a (mon.0) 440 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:53:10.779 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 19:53:10 smithi089 bash[71163]: debug 2023-06-03T19:53:10.375+0000 7fe48642e700 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2023-06-03T19:53:10.779 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:53:10 smithi089 bash[78452]: audit 1685821989.5637062 mon.a (mon.0) 441 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:53:10.779 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:53:10 smithi089 bash[78452]: audit 1685821989.5832522 mon.a (mon.0) 442 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:53:10.780 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:53:10 smithi089 bash[78452]: audit 1685821989.5961232 mon.a (mon.0) 443 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:53:10.780 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:53:10 smithi089 bash[78452]: audit 1685821990.2643244 mon.c (mon.2) 226 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-03T19:53:10.780 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:53:10 smithi089 bash[78452]: audit 1685821990.2663593 mon.c (mon.2) 227 : audit 1 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-03T19:53:10.780 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:53:10 smithi089 bash[78452]: audit 1685821990.2762911 mon.a (mon.0) 444 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:53:10.780 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:53:10 smithi089 bash[78452]: audit 1685821990.3621476 mon.c (mon.2) 228 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-03T19:53:10.780 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:53:10 smithi089 bash[78452]: audit 1685821990.3689606 mon.c (mon.2) 229 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-06-03T19:53:10.780 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:53:10 smithi089 bash[78452]: audit 1685821990.3729656 mon.c (mon.2) 230 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-06-03T19:53:10.781 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:53:10 smithi089 bash[78452]: audit 1685821990.3759727 mon.c (mon.2) 231 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-06-03T19:53:10.781 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:53:10 smithi089 bash[78452]: audit 1685821990.3798013 mon.c (mon.2) 232 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-06-03T19:53:10.781 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:53:10 smithi089 bash[75068]: audit 1685821989.5637062 mon.a (mon.0) 441 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:53:10.781 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:53:10 smithi089 bash[75068]: audit 1685821989.5832522 mon.a (mon.0) 442 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:53:10.781 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:53:10 smithi089 bash[75068]: audit 1685821989.5961232 mon.a (mon.0) 443 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:53:10.781 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:53:10 smithi089 bash[75068]: audit 1685821990.2643244 mon.c (mon.2) 226 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-03T19:53:10.782 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:53:10 smithi089 bash[75068]: audit 1685821990.2663593 mon.c (mon.2) 227 : audit 1 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-03T19:53:10.782 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:53:10 smithi089 bash[75068]: audit 1685821990.2762911 mon.a (mon.0) 444 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:53:10.782 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:53:10 smithi089 bash[75068]: audit 1685821990.3621476 mon.c (mon.2) 228 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-03T19:53:10.782 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:53:10 smithi089 bash[75068]: audit 1685821990.3689606 mon.c (mon.2) 229 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-06-03T19:53:10.782 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:53:10 smithi089 bash[75068]: audit 1685821990.3729656 mon.c (mon.2) 230 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-06-03T19:53:10.782 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:53:10 smithi089 bash[75068]: audit 1685821990.3759727 mon.c (mon.2) 231 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-06-03T19:53:10.782 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:53:10 smithi089 bash[75068]: audit 1685821990.3798013 mon.c (mon.2) 232 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-06-03T19:53:10.834 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:53:10 smithi164 bash[59462]: audit 1685821989.5637062 mon.a (mon.0) 441 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:53:10.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:53:10 smithi164 bash[59462]: audit 1685821989.5832522 mon.a (mon.0) 442 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:53:10.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:53:10 smithi164 bash[59462]: audit 1685821989.5961232 mon.a (mon.0) 443 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:53:10.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:53:10 smithi164 bash[59462]: audit 1685821990.2643244 mon.c (mon.2) 226 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-03T19:53:10.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:53:10 smithi164 bash[59462]: audit 1685821990.2663593 mon.c (mon.2) 227 : audit 1 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-03T19:53:10.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:53:10 smithi164 bash[59462]: audit 1685821990.2762911 mon.a (mon.0) 444 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:53:10.836 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:53:10 smithi164 bash[59462]: audit 1685821990.3621476 mon.c (mon.2) 228 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-03T19:53:10.836 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:53:10 smithi164 bash[59462]: audit 1685821990.3689606 mon.c (mon.2) 229 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-06-03T19:53:10.836 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:53:10 smithi164 bash[59462]: audit 1685821990.3729656 mon.c (mon.2) 230 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-06-03T19:53:10.836 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:53:10 smithi164 bash[59462]: audit 1685821990.3759727 mon.c (mon.2) 231 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-06-03T19:53:10.836 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:53:10 smithi164 bash[59462]: audit 1685821990.3798013 mon.c (mon.2) 232 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-06-03T19:53:12.029 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:53:11 smithi089 bash[75068]: cluster 1685821990.2455566 mgr.y (mgr.34104) 466 : cluster 0 pgmap v392: 89 pgs: 83 active+clean, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped; 5.6 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 831 op/s; 12122/273168 objects degraded (4.438%); 1.8 MiB/s, 28 objects/s recovering 2023-06-03T19:53:12.030 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:53:11 smithi089 bash[75068]: audit 1685821990.3804698 mgr.y (mgr.34104) 467 : audit 0 from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-06-03T19:53:12.030 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:53:11 smithi089 bash[75068]: cephadm 1685821990.3828418 mgr.y (mgr.34104) 468 : cephadm 1 Upgrade: unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2023-06-03T19:53:12.030 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:53:11 smithi089 bash[75068]: audit 1685821991.1632845 mon.c (mon.2) 233 : audit 0 from='client.? 172.21.15.89:0/2687867643' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:53:12.030 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:53:11 smithi089 bash[78452]: cluster 1685821990.2455566 mgr.y (mgr.34104) 466 : cluster 0 pgmap v392: 89 pgs: 83 active+clean, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped; 5.6 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 831 op/s; 12122/273168 objects degraded (4.438%); 1.8 MiB/s, 28 objects/s recovering 2023-06-03T19:53:12.030 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:53:11 smithi089 bash[78452]: audit 1685821990.3804698 mgr.y (mgr.34104) 467 : audit 0 from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-06-03T19:53:12.031 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:53:11 smithi089 bash[78452]: cephadm 1685821990.3828418 mgr.y (mgr.34104) 468 : cephadm 1 Upgrade: unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2023-06-03T19:53:12.031 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:53:11 smithi089 bash[78452]: audit 1685821991.1632845 mon.c (mon.2) 233 : audit 0 from='client.? 172.21.15.89:0/2687867643' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:53:12.085 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:53:11 smithi164 bash[59462]: cluster 1685821990.2455566 mgr.y (mgr.34104) 466 : cluster 0 pgmap v392: 89 pgs: 83 active+clean, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped; 5.6 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 831 op/s; 12122/273168 objects degraded (4.438%); 1.8 MiB/s, 28 objects/s recovering 2023-06-03T19:53:12.085 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:53:11 smithi164 bash[59462]: audit 1685821990.3804698 mgr.y (mgr.34104) 467 : audit 0 from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-06-03T19:53:12.085 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:53:11 smithi164 bash[59462]: cephadm 1685821990.3828418 mgr.y (mgr.34104) 468 : cephadm 1 Upgrade: unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2023-06-03T19:53:12.085 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:53:11 smithi164 bash[59462]: audit 1685821991.1632845 mon.c (mon.2) 233 : audit 0 from='client.? 172.21.15.89:0/2687867643' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:53:13.029 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:53:12 smithi089 bash[75068]: cluster 1685821992.3445625 mon.a (mon.0) 445 : cluster 3 Health check update: Degraded data redundancy: 12122/273168 objects degraded (4.438%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-06-03T19:53:13.029 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:53:12 smithi089 bash[78452]: cluster 1685821992.3445625 mon.a (mon.0) 445 : cluster 3 Health check update: Degraded data redundancy: 12122/273168 objects degraded (4.438%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-06-03T19:53:13.088 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:53:12 smithi164 bash[59462]: cluster 1685821992.3445625 mon.a (mon.0) 445 : cluster 3 Health check update: Degraded data redundancy: 12122/273168 objects degraded (4.438%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-06-03T19:53:14.029 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:53:13 smithi089 bash[75068]: cluster 1685821992.2462168 mgr.y (mgr.34104) 469 : cluster 0 pgmap v393: 89 pgs: 83 active+clean, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped; 5.5 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 646 op/s; 12122/271362 objects degraded (4.467%); 1.2 MiB/s, 18 objects/s recovering 2023-06-03T19:53:14.030 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:53:13 smithi089 bash[78452]: cluster 1685821992.2462168 mgr.y (mgr.34104) 469 : cluster 0 pgmap v393: 89 pgs: 83 active+clean, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped; 5.5 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 646 op/s; 12122/271362 objects degraded (4.467%); 1.2 MiB/s, 18 objects/s recovering 2023-06-03T19:53:14.085 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:53:13 smithi164 bash[59462]: cluster 1685821992.2462168 mgr.y (mgr.34104) 469 : cluster 0 pgmap v393: 89 pgs: 83 active+clean, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped; 5.5 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 646 op/s; 12122/271362 objects degraded (4.467%); 1.2 MiB/s, 18 objects/s recovering 2023-06-03T19:53:15.029 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:53:14 smithi089 bash[75068]: audit 1685821993.7819386 mon.b (mon.1) 32 : audit 0 from='client.? 172.21.15.89:0/2056981745' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:53:15.030 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:53:14 smithi089 bash[78452]: audit 1685821993.7819386 mon.b (mon.1) 32 : audit 0 from='client.? 172.21.15.89:0/2056981745' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:53:15.084 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:53:14 smithi164 bash[59462]: audit 1685821993.7819386 mon.b (mon.1) 32 : audit 0 from='client.? 172.21.15.89:0/2056981745' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:53:16.029 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:53:15 smithi089 bash[75068]: cluster 1685821994.247023 mgr.y (mgr.34104) 470 : cluster 0 pgmap v394: 89 pgs: 83 active+clean, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped; 5.4 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 739 op/s; 12010/265830 objects degraded (4.518%); 1.7 MiB/s, 27 objects/s recovering 2023-06-03T19:53:16.029 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:53:15 smithi089 bash[78452]: cluster 1685821994.247023 mgr.y (mgr.34104) 470 : cluster 0 pgmap v394: 89 pgs: 83 active+clean, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped; 5.4 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 739 op/s; 12010/265830 objects degraded (4.518%); 1.7 MiB/s, 27 objects/s recovering 2023-06-03T19:53:16.084 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:53:15 smithi164 bash[59462]: cluster 1685821994.247023 mgr.y (mgr.34104) 470 : cluster 0 pgmap v394: 89 pgs: 83 active+clean, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped; 5.4 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 739 op/s; 12010/265830 objects degraded (4.518%); 1.7 MiB/s, 27 objects/s recovering 2023-06-03T19:53:16.529 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 19:53:16 smithi089 bash[71163]: ::ffff:172.21.15.164 - - [03/Jun/2023:19:53:16] "GET /metrics HTTP/1.1" 200 212796 "" "Prometheus/2.33.4" 2023-06-03T19:53:17.029 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:53:16 smithi089 bash[78452]: audit 1685821996.3574204 mon.c (mon.2) 234 : audit 0 from='client.? 172.21.15.89:0/3666974618' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:53:17.029 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:53:16 smithi089 bash[75068]: audit 1685821996.3574204 mon.c (mon.2) 234 : audit 0 from='client.? 172.21.15.89:0/3666974618' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:53:17.084 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:53:16 smithi164 bash[59462]: audit 1685821996.3574204 mon.c (mon.2) 234 : audit 0 from='client.? 172.21.15.89:0/3666974618' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:53:18.029 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:53:17 smithi089 bash[78452]: cluster 1685821996.2477598 mgr.y (mgr.34104) 471 : cluster 0 pgmap v395: 89 pgs: 83 active+clean, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped; 5.3 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 699 op/s; 12010/261177 objects degraded (4.598%); 1.2 MiB/s, 18 objects/s recovering 2023-06-03T19:53:18.029 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:53:17 smithi089 bash[78452]: cluster 1685821997.3458343 mon.a (mon.0) 446 : cluster 3 Health check update: Degraded data redundancy: 12010/261177 objects degraded (4.598%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-06-03T19:53:18.029 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:53:17 smithi089 bash[75068]: cluster 1685821996.2477598 mgr.y (mgr.34104) 471 : cluster 0 pgmap v395: 89 pgs: 83 active+clean, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped; 5.3 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 699 op/s; 12010/261177 objects degraded (4.598%); 1.2 MiB/s, 18 objects/s recovering 2023-06-03T19:53:18.029 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:53:17 smithi089 bash[75068]: cluster 1685821997.3458343 mon.a (mon.0) 446 : cluster 3 Health check update: Degraded data redundancy: 12010/261177 objects degraded (4.598%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-06-03T19:53:18.084 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:53:17 smithi164 bash[59462]: cluster 1685821996.2477598 mgr.y (mgr.34104) 471 : cluster 0 pgmap v395: 89 pgs: 83 active+clean, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped; 5.3 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 699 op/s; 12010/261177 objects degraded (4.598%); 1.2 MiB/s, 18 objects/s recovering 2023-06-03T19:53:18.085 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:53:17 smithi164 bash[59462]: cluster 1685821997.3458343 mon.a (mon.0) 446 : cluster 3 Health check update: Degraded data redundancy: 12010/261177 objects degraded (4.598%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-06-03T19:53:20.029 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:53:19 smithi089 bash[78452]: cluster 1685821998.2483523 mgr.y (mgr.34104) 472 : cluster 0 pgmap v396: 89 pgs: 83 active+clean, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped; 5.3 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 585 op/s; 12010/260478 objects degraded (4.611%); 1.2 MiB/s, 18 objects/s recovering 2023-06-03T19:53:20.029 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:53:19 smithi089 bash[78452]: audit 1685821998.9663773 mon.c (mon.2) 235 : audit 0 from='client.? 172.21.15.89:0/3306954742' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:53:20.029 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:53:19 smithi089 bash[75068]: cluster 1685821998.2483523 mgr.y (mgr.34104) 472 : cluster 0 pgmap v396: 89 pgs: 83 active+clean, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped; 5.3 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 585 op/s; 12010/260478 objects degraded (4.611%); 1.2 MiB/s, 18 objects/s recovering 2023-06-03T19:53:20.030 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:53:19 smithi089 bash[75068]: audit 1685821998.9663773 mon.c (mon.2) 235 : audit 0 from='client.? 172.21.15.89:0/3306954742' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:53:20.085 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:53:19 smithi164 bash[59462]: cluster 1685821998.2483523 mgr.y (mgr.34104) 472 : cluster 0 pgmap v396: 89 pgs: 83 active+clean, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped; 5.3 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 585 op/s; 12010/260478 objects degraded (4.611%); 1.2 MiB/s, 18 objects/s recovering 2023-06-03T19:53:20.085 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:53:19 smithi164 bash[59462]: audit 1685821998.9663773 mon.c (mon.2) 235 : audit 0 from='client.? 172.21.15.89:0/3306954742' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:53:22.029 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:53:21 smithi089 bash[75068]: cluster 1685822000.249176 mgr.y (mgr.34104) 473 : cluster 0 pgmap v397: 89 pgs: 83 active+clean, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped; 5.1 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 786 op/s; 11896/252510 objects degraded (4.711%); 1.7 MiB/s, 27 objects/s recovering 2023-06-03T19:53:22.029 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:53:21 smithi089 bash[75068]: audit 1685822001.516558 mon.a (mon.0) 447 : audit 0 from='client.? 172.21.15.89:0/1015922100' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:53:22.030 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:53:21 smithi089 bash[78452]: cluster 1685822000.249176 mgr.y (mgr.34104) 473 : cluster 0 pgmap v397: 89 pgs: 83 active+clean, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped; 5.1 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 786 op/s; 11896/252510 objects degraded (4.711%); 1.7 MiB/s, 27 objects/s recovering 2023-06-03T19:53:22.030 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:53:21 smithi089 bash[78452]: audit 1685822001.516558 mon.a (mon.0) 447 : audit 0 from='client.? 172.21.15.89:0/1015922100' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:53:22.085 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:53:21 smithi164 bash[59462]: cluster 1685822000.249176 mgr.y (mgr.34104) 473 : cluster 0 pgmap v397: 89 pgs: 83 active+clean, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped; 5.1 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 786 op/s; 11896/252510 objects degraded (4.711%); 1.7 MiB/s, 27 objects/s recovering 2023-06-03T19:53:22.085 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:53:21 smithi164 bash[59462]: audit 1685822001.516558 mon.a (mon.0) 447 : audit 0 from='client.? 172.21.15.89:0/1015922100' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:53:23.029 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:53:22 smithi089 bash[75068]: audit 1685822002.0825481 mon.c (mon.2) 236 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T19:53:23.029 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:53:22 smithi089 bash[75068]: cluster 1685822002.3471224 mon.a (mon.0) 448 : cluster 3 Health check update: Degraded data redundancy: 11896/252510 objects degraded (4.711%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-06-03T19:53:23.030 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:53:22 smithi089 bash[78452]: audit 1685822002.0825481 mon.c (mon.2) 236 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T19:53:23.030 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:53:22 smithi089 bash[78452]: cluster 1685822002.3471224 mon.a (mon.0) 448 : cluster 3 Health check update: Degraded data redundancy: 11896/252510 objects degraded (4.711%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-06-03T19:53:23.084 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:53:22 smithi164 bash[59462]: audit 1685822002.0825481 mon.c (mon.2) 236 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T19:53:23.085 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:53:22 smithi164 bash[59462]: cluster 1685822002.3471224 mon.a (mon.0) 448 : cluster 3 Health check update: Degraded data redundancy: 11896/252510 objects degraded (4.711%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-06-03T19:53:24.029 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:53:23 smithi089 bash[78452]: cluster 1685822002.2498176 mgr.y (mgr.34104) 474 : cluster 0 pgmap v398: 89 pgs: 83 active+clean, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped; 5.1 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 628 op/s; 11896/250530 objects degraded (4.748%); 1.2 MiB/s, 18 objects/s recovering 2023-06-03T19:53:24.029 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:53:23 smithi089 bash[75068]: cluster 1685822002.2498176 mgr.y (mgr.34104) 474 : cluster 0 pgmap v398: 89 pgs: 83 active+clean, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped; 5.1 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 628 op/s; 11896/250530 objects degraded (4.748%); 1.2 MiB/s, 18 objects/s recovering 2023-06-03T19:53:24.084 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:53:23 smithi164 bash[59462]: cluster 1685822002.2498176 mgr.y (mgr.34104) 474 : cluster 0 pgmap v398: 89 pgs: 83 active+clean, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped; 5.1 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 628 op/s; 11896/250530 objects degraded (4.748%); 1.2 MiB/s, 18 objects/s recovering 2023-06-03T19:53:25.029 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:53:24 smithi089 bash[75068]: audit 1685822004.1162872 mon.a (mon.0) 449 : audit 0 from='client.? 172.21.15.89:0/1429483118' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:53:25.029 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:53:24 smithi089 bash[78452]: audit 1685822004.1162872 mon.a (mon.0) 449 : audit 0 from='client.? 172.21.15.89:0/1429483118' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:53:25.084 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:53:24 smithi164 bash[59462]: audit 1685822004.1162872 mon.a (mon.0) 449 : audit 0 from='client.? 172.21.15.89:0/1429483118' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:53:25.687 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:53:25 smithi089 bash[75068]: cluster 1685822004.2533612 mgr.y (mgr.34104) 475 : cluster 0 pgmap v399: 89 pgs: 83 active+clean, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped; 5.0 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 740 op/s; 11785/244689 objects degraded (4.816%); 1.8 MiB/s, 28 objects/s recovering 2023-06-03T19:53:25.687 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:53:25 smithi089 bash[75068]: audit 1685822005.39756 mon.c (mon.2) 237 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-06-03T19:53:25.688 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 19:53:25 smithi089 bash[71163]: debug 2023-06-03T19:53:25.395+0000 7fe48642e700 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2023-06-03T19:53:26.029 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:53:25 smithi089 bash[78452]: cluster 1685822004.2533612 mgr.y (mgr.34104) 475 : cluster 0 pgmap v399: 89 pgs: 83 active+clean, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped; 5.0 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 740 op/s; 11785/244689 objects degraded (4.816%); 1.8 MiB/s, 28 objects/s recovering 2023-06-03T19:53:26.030 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:53:25 smithi089 bash[78452]: audit 1685822005.39756 mon.c (mon.2) 237 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-06-03T19:53:26.085 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:53:25 smithi164 bash[59462]: cluster 1685822004.2533612 mgr.y (mgr.34104) 475 : cluster 0 pgmap v399: 89 pgs: 83 active+clean, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped; 5.0 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 740 op/s; 11785/244689 objects degraded (4.816%); 1.8 MiB/s, 28 objects/s recovering 2023-06-03T19:53:26.085 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:53:25 smithi164 bash[59462]: audit 1685822005.39756 mon.c (mon.2) 237 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-06-03T19:53:26.529 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 19:53:26 smithi089 bash[71163]: ::ffff:172.21.15.164 - - [03/Jun/2023:19:53:26] "GET /metrics HTTP/1.1" 200 212798 "" "Prometheus/2.33.4" 2023-06-03T19:53:27.030 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:53:26 smithi089 bash[75068]: audit 1685822005.398103 mgr.y (mgr.34104) 476 : audit 0 from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-06-03T19:53:27.030 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:53:26 smithi089 bash[75068]: cephadm 1685822005.4001572 mgr.y (mgr.34104) 477 : cephadm 1 Upgrade: unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2023-06-03T19:53:27.031 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:53:26 smithi089 bash[78452]: audit 1685822005.398103 mgr.y (mgr.34104) 476 : audit 0 from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-06-03T19:53:27.031 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:53:26 smithi089 bash[78452]: cephadm 1685822005.4001572 mgr.y (mgr.34104) 477 : cephadm 1 Upgrade: unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2023-06-03T19:53:27.085 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:53:26 smithi164 bash[59462]: audit 1685822005.398103 mgr.y (mgr.34104) 476 : audit 0 from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-06-03T19:53:27.085 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:53:26 smithi164 bash[59462]: cephadm 1685822005.4001572 mgr.y (mgr.34104) 477 : cephadm 1 Upgrade: unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2023-06-03T19:53:28.029 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:53:27 smithi089 bash[75068]: cluster 1685822006.2540982 mgr.y (mgr.34104) 478 : cluster 0 pgmap v400: 89 pgs: 83 active+clean, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped; 4.9 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 711 op/s; 11785/240186 objects degraded (4.907%); 1.2 MiB/s, 18 objects/s recovering 2023-06-03T19:53:28.029 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:53:27 smithi089 bash[75068]: audit 1685822006.6887631 mon.b (mon.1) 33 : audit 0 from='client.? 172.21.15.89:0/709244669' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:53:28.029 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:53:27 smithi089 bash[75068]: cluster 1685822007.348429 mon.a (mon.0) 450 : cluster 3 Health check update: Degraded data redundancy: 11785/240186 objects degraded (4.907%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-06-03T19:53:28.029 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:53:27 smithi089 bash[78452]: cluster 1685822006.2540982 mgr.y (mgr.34104) 478 : cluster 0 pgmap v400: 89 pgs: 83 active+clean, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped; 4.9 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 711 op/s; 11785/240186 objects degraded (4.907%); 1.2 MiB/s, 18 objects/s recovering 2023-06-03T19:53:28.030 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:53:27 smithi089 bash[78452]: audit 1685822006.6887631 mon.b (mon.1) 33 : audit 0 from='client.? 172.21.15.89:0/709244669' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:53:28.030 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:53:27 smithi089 bash[78452]: cluster 1685822007.348429 mon.a (mon.0) 450 : cluster 3 Health check update: Degraded data redundancy: 11785/240186 objects degraded (4.907%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-06-03T19:53:28.086 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:53:27 smithi164 bash[59462]: cluster 1685822006.2540982 mgr.y (mgr.34104) 478 : cluster 0 pgmap v400: 89 pgs: 83 active+clean, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped; 4.9 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 711 op/s; 11785/240186 objects degraded (4.907%); 1.2 MiB/s, 18 objects/s recovering 2023-06-03T19:53:28.086 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:53:27 smithi164 bash[59462]: audit 1685822006.6887631 mon.b (mon.1) 33 : audit 0 from='client.? 172.21.15.89:0/709244669' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:53:28.086 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:53:27 smithi164 bash[59462]: cluster 1685822007.348429 mon.a (mon.0) 450 : cluster 3 Health check update: Degraded data redundancy: 11785/240186 objects degraded (4.907%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-06-03T19:53:30.029 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:53:29 smithi089 bash[78452]: cluster 1685822008.2547317 mgr.y (mgr.34104) 479 : cluster 0 pgmap v401: 89 pgs: 83 active+clean, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped; 4.9 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 600 op/s; 11785/239541 objects degraded (4.920%); 1.2 MiB/s, 18 objects/s recovering 2023-06-03T19:53:30.029 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:53:29 smithi089 bash[78452]: audit 1685822009.2916698 mon.c (mon.2) 238 : audit 0 from='client.? 172.21.15.89:0/467348455' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:53:30.029 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:53:29 smithi089 bash[75068]: cluster 1685822008.2547317 mgr.y (mgr.34104) 479 : cluster 0 pgmap v401: 89 pgs: 83 active+clean, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped; 4.9 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 600 op/s; 11785/239541 objects degraded (4.920%); 1.2 MiB/s, 18 objects/s recovering 2023-06-03T19:53:30.030 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:53:29 smithi089 bash[75068]: audit 1685822009.2916698 mon.c (mon.2) 238 : audit 0 from='client.? 172.21.15.89:0/467348455' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:53:30.084 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:53:29 smithi164 bash[59462]: cluster 1685822008.2547317 mgr.y (mgr.34104) 479 : cluster 0 pgmap v401: 89 pgs: 83 active+clean, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped; 4.9 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 600 op/s; 11785/239541 objects degraded (4.920%); 1.2 MiB/s, 18 objects/s recovering 2023-06-03T19:53:30.085 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:53:29 smithi164 bash[59462]: audit 1685822009.2916698 mon.c (mon.2) 238 : audit 0 from='client.? 172.21.15.89:0/467348455' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:53:32.029 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:53:31 smithi089 bash[78452]: cluster 1685822010.255528 mgr.y (mgr.34104) 480 : cluster 0 pgmap v402: 89 pgs: 83 active+clean, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped; 4.7 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 793 op/s; 11674/231906 objects degraded (5.034%); 1.7 MiB/s, 27 objects/s recovering 2023-06-03T19:53:32.029 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:53:31 smithi089 bash[75068]: cluster 1685822010.255528 mgr.y (mgr.34104) 480 : cluster 0 pgmap v402: 89 pgs: 83 active+clean, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped; 4.7 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 793 op/s; 11674/231906 objects degraded (5.034%); 1.7 MiB/s, 27 objects/s recovering 2023-06-03T19:53:32.084 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:53:31 smithi164 bash[59462]: cluster 1685822010.255528 mgr.y (mgr.34104) 480 : cluster 0 pgmap v402: 89 pgs: 83 active+clean, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped; 4.7 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 793 op/s; 11674/231906 objects degraded (5.034%); 1.7 MiB/s, 27 objects/s recovering 2023-06-03T19:53:33.029 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:53:32 smithi089 bash[78452]: audit 1685822011.8950348 mon.c (mon.2) 239 : audit 0 from='client.? 172.21.15.89:0/3488968632' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:53:33.029 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:53:32 smithi089 bash[78452]: cluster 1685822012.3497827 mon.a (mon.0) 451 : cluster 3 Health check update: Degraded data redundancy: 11674/231906 objects degraded (5.034%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-06-03T19:53:33.030 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:53:32 smithi089 bash[75068]: audit 1685822011.8950348 mon.c (mon.2) 239 : audit 0 from='client.? 172.21.15.89:0/3488968632' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:53:33.030 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:53:32 smithi089 bash[75068]: cluster 1685822012.3497827 mon.a (mon.0) 451 : cluster 3 Health check update: Degraded data redundancy: 11674/231906 objects degraded (5.034%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-06-03T19:53:33.084 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:53:32 smithi164 bash[59462]: audit 1685822011.8950348 mon.c (mon.2) 239 : audit 0 from='client.? 172.21.15.89:0/3488968632' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:53:33.085 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:53:32 smithi164 bash[59462]: cluster 1685822012.3497827 mon.a (mon.0) 451 : cluster 3 Health check update: Degraded data redundancy: 11674/231906 objects degraded (5.034%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-06-03T19:53:34.029 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:53:33 smithi089 bash[75068]: cluster 1685822012.2562077 mgr.y (mgr.34104) 481 : cluster 0 pgmap v403: 89 pgs: 83 active+clean, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped; 4.7 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 627 op/s; 11674/229896 objects degraded (5.078%); 1.2 MiB/s, 18 objects/s recovering 2023-06-03T19:53:34.029 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:53:33 smithi089 bash[78452]: cluster 1685822012.2562077 mgr.y (mgr.34104) 481 : cluster 0 pgmap v403: 89 pgs: 83 active+clean, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped; 4.7 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 627 op/s; 11674/229896 objects degraded (5.078%); 1.2 MiB/s, 18 objects/s recovering 2023-06-03T19:53:34.084 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:53:33 smithi164 bash[59462]: cluster 1685822012.2562077 mgr.y (mgr.34104) 481 : cluster 0 pgmap v403: 89 pgs: 83 active+clean, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped; 4.7 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 627 op/s; 11674/229896 objects degraded (5.078%); 1.2 MiB/s, 18 objects/s recovering 2023-06-03T19:53:35.029 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:53:34 smithi089 bash[75068]: audit 1685822014.4625528 mon.b (mon.1) 34 : audit 0 from='client.? 172.21.15.89:0/3407794941' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:53:35.029 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:53:34 smithi089 bash[78452]: audit 1685822014.4625528 mon.b (mon.1) 34 : audit 0 from='client.? 172.21.15.89:0/3407794941' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:53:35.084 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:53:34 smithi164 bash[59462]: audit 1685822014.4625528 mon.b (mon.1) 34 : audit 0 from='client.? 172.21.15.89:0/3407794941' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:53:36.029 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:53:35 smithi089 bash[75068]: cluster 1685822014.2613988 mgr.y (mgr.34104) 482 : cluster 0 pgmap v404: 89 pgs: 83 active+clean, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped; 4.6 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 732 op/s; 11561/224136 objects degraded (5.158%); 1.7 MiB/s, 27 objects/s recovering 2023-06-03T19:53:36.029 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:53:35 smithi089 bash[78452]: cluster 1685822014.2613988 mgr.y (mgr.34104) 482 : cluster 0 pgmap v404: 89 pgs: 83 active+clean, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped; 4.6 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 732 op/s; 11561/224136 objects degraded (5.158%); 1.7 MiB/s, 27 objects/s recovering 2023-06-03T19:53:36.084 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:53:35 smithi164 bash[59462]: cluster 1685822014.2613988 mgr.y (mgr.34104) 482 : cluster 0 pgmap v404: 89 pgs: 83 active+clean, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped; 4.6 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 732 op/s; 11561/224136 objects degraded (5.158%); 1.7 MiB/s, 27 objects/s recovering 2023-06-03T19:53:36.529 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 19:53:36 smithi089 bash[71163]: ::ffff:172.21.15.164 - - [03/Jun/2023:19:53:36] "GET /metrics HTTP/1.1" 200 212798 "" "Prometheus/2.33.4" 2023-06-03T19:53:38.029 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:53:37 smithi089 bash[75068]: cluster 1685822016.2621803 mgr.y (mgr.34104) 483 : cluster 0 pgmap v405: 89 pgs: 83 active+clean, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped; 4.5 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 691 op/s; 11561/219771 objects degraded (5.260%); 1.2 MiB/s, 18 objects/s recovering 2023-06-03T19:53:38.030 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:53:37 smithi089 bash[75068]: audit 1685822017.0370297 mon.a (mon.0) 452 : audit 0 from='client.? 172.21.15.89:0/3057965472' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:53:38.030 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:53:37 smithi089 bash[75068]: audit 1685822017.083115 mon.c (mon.2) 240 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T19:53:38.030 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:53:37 smithi089 bash[75068]: cluster 1685822017.35097 mon.a (mon.0) 453 : cluster 3 Health check update: Degraded data redundancy: 11561/219771 objects degraded (5.260%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-06-03T19:53:38.031 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:53:37 smithi089 bash[78452]: cluster 1685822016.2621803 mgr.y (mgr.34104) 483 : cluster 0 pgmap v405: 89 pgs: 83 active+clean, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped; 4.5 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 691 op/s; 11561/219771 objects degraded (5.260%); 1.2 MiB/s, 18 objects/s recovering 2023-06-03T19:53:38.031 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:53:37 smithi089 bash[78452]: audit 1685822017.0370297 mon.a (mon.0) 452 : audit 0 from='client.? 172.21.15.89:0/3057965472' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:53:38.031 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:53:37 smithi089 bash[78452]: audit 1685822017.083115 mon.c (mon.2) 240 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T19:53:38.031 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:53:37 smithi089 bash[78452]: cluster 1685822017.35097 mon.a (mon.0) 453 : cluster 3 Health check update: Degraded data redundancy: 11561/219771 objects degraded (5.260%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-06-03T19:53:38.085 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:53:37 smithi164 bash[59462]: cluster 1685822016.2621803 mgr.y (mgr.34104) 483 : cluster 0 pgmap v405: 89 pgs: 83 active+clean, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped; 4.5 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 691 op/s; 11561/219771 objects degraded (5.260%); 1.2 MiB/s, 18 objects/s recovering 2023-06-03T19:53:38.085 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:53:37 smithi164 bash[59462]: audit 1685822017.0370297 mon.a (mon.0) 452 : audit 0 from='client.? 172.21.15.89:0/3057965472' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:53:38.085 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:53:37 smithi164 bash[59462]: audit 1685822017.083115 mon.c (mon.2) 240 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T19:53:38.086 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:53:37 smithi164 bash[59462]: cluster 1685822017.35097 mon.a (mon.0) 453 : cluster 3 Health check update: Degraded data redundancy: 11561/219771 objects degraded (5.260%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-06-03T19:53:40.029 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:53:39 smithi089 bash[75068]: cluster 1685822018.2629292 mgr.y (mgr.34104) 484 : cluster 0 pgmap v406: 89 pgs: 83 active+clean, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped; 4.5 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 585 op/s; 11561/219087 objects degraded (5.277%); 1.2 MiB/s, 18 objects/s recovering 2023-06-03T19:53:40.029 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:53:39 smithi089 bash[75068]: audit 1685822019.650052 mon.c (mon.2) 241 : audit 0 from='client.? 172.21.15.89:0/4226988267' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:53:40.030 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:53:39 smithi089 bash[78452]: cluster 1685822018.2629292 mgr.y (mgr.34104) 484 : cluster 0 pgmap v406: 89 pgs: 83 active+clean, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped; 4.5 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 585 op/s; 11561/219087 objects degraded (5.277%); 1.2 MiB/s, 18 objects/s recovering 2023-06-03T19:53:40.030 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:53:39 smithi089 bash[78452]: audit 1685822019.650052 mon.c (mon.2) 241 : audit 0 from='client.? 172.21.15.89:0/4226988267' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:53:40.084 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:53:39 smithi164 bash[59462]: cluster 1685822018.2629292 mgr.y (mgr.34104) 484 : cluster 0 pgmap v406: 89 pgs: 83 active+clean, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped; 4.5 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 585 op/s; 11561/219087 objects degraded (5.277%); 1.2 MiB/s, 18 objects/s recovering 2023-06-03T19:53:40.085 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:53:39 smithi164 bash[59462]: audit 1685822019.650052 mon.c (mon.2) 241 : audit 0 from='client.? 172.21.15.89:0/4226988267' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:53:40.766 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:53:40 smithi089 bash[75068]: audit 1685822020.41424 mon.c (mon.2) 242 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-06-03T19:53:40.766 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 19:53:40 smithi089 bash[71163]: debug 2023-06-03T19:53:40.411+0000 7fe48642e700 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2023-06-03T19:53:40.767 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:53:40 smithi089 bash[78452]: audit 1685822020.41424 mon.c (mon.2) 242 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-06-03T19:53:41.084 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:53:40 smithi164 bash[59462]: audit 1685822020.41424 mon.c (mon.2) 242 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-06-03T19:53:42.030 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:53:41 smithi089 bash[75068]: cluster 1685822020.2637522 mgr.y (mgr.34104) 485 : cluster 0 pgmap v407: 89 pgs: 83 active+clean, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped; 4.3 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 777 op/s; 11451/211527 objects degraded (5.413%); 1.7 MiB/s, 27 objects/s recovering 2023-06-03T19:53:42.098 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:53:41 smithi089 bash[75068]: audit 1685822020.4145482 mgr.y (mgr.34104) 486 : audit 0 from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-06-03T19:53:42.098 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:53:41 smithi089 bash[75068]: cephadm 1685822020.4165149 mgr.y (mgr.34104) 487 : cephadm 1 Upgrade: unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2023-06-03T19:53:42.099 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:53:41 smithi089 bash[78452]: cluster 1685822020.2637522 mgr.y (mgr.34104) 485 : cluster 0 pgmap v407: 89 pgs: 83 active+clean, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped; 4.3 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 777 op/s; 11451/211527 objects degraded (5.413%); 1.7 MiB/s, 27 objects/s recovering 2023-06-03T19:53:42.099 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:53:41 smithi089 bash[78452]: audit 1685822020.4145482 mgr.y (mgr.34104) 486 : audit 0 from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-06-03T19:53:42.099 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:53:41 smithi089 bash[78452]: cephadm 1685822020.4165149 mgr.y (mgr.34104) 487 : cephadm 1 Upgrade: unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2023-06-03T19:53:42.099 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:53:41 smithi164 bash[59462]: cluster 1685822020.2637522 mgr.y (mgr.34104) 485 : cluster 0 pgmap v407: 89 pgs: 83 active+clean, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped; 4.3 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 777 op/s; 11451/211527 objects degraded (5.413%); 1.7 MiB/s, 27 objects/s recovering 2023-06-03T19:53:42.100 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:53:41 smithi164 bash[59462]: audit 1685822020.4145482 mgr.y (mgr.34104) 486 : audit 0 from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-06-03T19:53:42.100 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:53:41 smithi164 bash[59462]: cephadm 1685822020.4165149 mgr.y (mgr.34104) 487 : cephadm 1 Upgrade: unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2023-06-03T19:53:43.085 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:53:42 smithi164 bash[59462]: audit 1685822022.2233493 mon.c (mon.2) 243 : audit 0 from='client.? 172.21.15.89:0/791685361' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:53:43.086 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:53:42 smithi164 bash[59462]: cluster 1685822022.3523383 mon.a (mon.0) 454 : cluster 3 Health check update: Degraded data redundancy: 11451/211527 objects degraded (5.413%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-06-03T19:53:43.279 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:53:42 smithi089 bash[78452]: audit 1685822022.2233493 mon.c (mon.2) 243 : audit 0 from='client.? 172.21.15.89:0/791685361' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:53:43.279 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:53:42 smithi089 bash[78452]: cluster 1685822022.3523383 mon.a (mon.0) 454 : cluster 3 Health check update: Degraded data redundancy: 11451/211527 objects degraded (5.413%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-06-03T19:53:43.280 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:53:42 smithi089 bash[75068]: audit 1685822022.2233493 mon.c (mon.2) 243 : audit 0 from='client.? 172.21.15.89:0/791685361' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:53:43.280 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:53:42 smithi089 bash[75068]: cluster 1685822022.3523383 mon.a (mon.0) 454 : cluster 3 Health check update: Degraded data redundancy: 11451/211527 objects degraded (5.413%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-06-03T19:53:44.084 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:53:43 smithi164 bash[59462]: cluster 1685822022.2689703 mgr.y (mgr.34104) 488 : cluster 0 pgmap v408: 89 pgs: 83 active+clean, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped; 4.3 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 616 op/s; 11451/209700 objects degraded (5.461%); 1.2 MiB/s, 18 objects/s recovering 2023-06-03T19:53:44.279 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:53:43 smithi089 bash[78452]: cluster 1685822022.2689703 mgr.y (mgr.34104) 488 : cluster 0 pgmap v408: 89 pgs: 83 active+clean, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped; 4.3 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 616 op/s; 11451/209700 objects degraded (5.461%); 1.2 MiB/s, 18 objects/s recovering 2023-06-03T19:53:44.279 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:53:43 smithi089 bash[75068]: cluster 1685822022.2689703 mgr.y (mgr.34104) 488 : cluster 0 pgmap v408: 89 pgs: 83 active+clean, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped; 4.3 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 616 op/s; 11451/209700 objects degraded (5.461%); 1.2 MiB/s, 18 objects/s recovering 2023-06-03T19:53:46.084 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:53:45 smithi164 bash[59462]: cluster 1685822024.269844 mgr.y (mgr.34104) 489 : cluster 0 pgmap v409: 89 pgs: 83 active+clean, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped; 4.2 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 715 op/s; 11338/204093 objects degraded (5.555%); 1.7 MiB/s, 27 objects/s recovering 2023-06-03T19:53:46.085 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:53:45 smithi164 bash[59462]: audit 1685822024.8280575 mon.c (mon.2) 244 : audit 0 from='client.? 172.21.15.89:0/471869854' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:53:46.233 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:53:45 smithi089 bash[75068]: cluster 1685822024.269844 mgr.y (mgr.34104) 489 : cluster 0 pgmap v409: 89 pgs: 83 active+clean, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped; 4.2 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 715 op/s; 11338/204093 objects degraded (5.555%); 1.7 MiB/s, 27 objects/s recovering 2023-06-03T19:53:46.233 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:53:45 smithi089 bash[75068]: audit 1685822024.8280575 mon.c (mon.2) 244 : audit 0 from='client.? 172.21.15.89:0/471869854' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:53:46.233 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 19:53:46 smithi089 bash[71163]: ::ffff:172.21.15.164 - - [03/Jun/2023:19:53:46] "GET /metrics HTTP/1.1" 200 212786 "" "Prometheus/2.33.4" 2023-06-03T19:53:46.234 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:53:45 smithi089 bash[78452]: cluster 1685822024.269844 mgr.y (mgr.34104) 489 : cluster 0 pgmap v409: 89 pgs: 83 active+clean, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped; 4.2 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 715 op/s; 11338/204093 objects degraded (5.555%); 1.7 MiB/s, 27 objects/s recovering 2023-06-03T19:53:46.234 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:53:45 smithi089 bash[78452]: audit 1685822024.8280575 mon.c (mon.2) 244 : audit 0 from='client.? 172.21.15.89:0/471869854' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:53:48.084 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:53:47 smithi164 bash[59462]: cluster 1685822026.2705715 mgr.y (mgr.34104) 490 : cluster 0 pgmap v410: 89 pgs: 83 active+clean, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped; 4.1 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 676 op/s; 11338/199767 objects degraded (5.676%); 1.2 MiB/s, 18 objects/s recovering 2023-06-03T19:53:48.085 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:53:47 smithi164 bash[59462]: cluster 1685822027.3535106 mon.a (mon.0) 455 : cluster 3 Health check update: Degraded data redundancy: 11338/199767 objects degraded (5.676%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-06-03T19:53:48.085 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:53:47 smithi164 bash[59462]: audit 1685822027.4151604 mon.c (mon.2) 245 : audit 0 from='client.? 172.21.15.89:0/1260940529' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:53:48.279 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:53:47 smithi089 bash[78452]: cluster 1685822026.2705715 mgr.y (mgr.34104) 490 : cluster 0 pgmap v410: 89 pgs: 83 active+clean, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped; 4.1 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 676 op/s; 11338/199767 objects degraded (5.676%); 1.2 MiB/s, 18 objects/s recovering 2023-06-03T19:53:48.279 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:53:47 smithi089 bash[78452]: cluster 1685822027.3535106 mon.a (mon.0) 455 : cluster 3 Health check update: Degraded data redundancy: 11338/199767 objects degraded (5.676%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-06-03T19:53:48.279 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:53:47 smithi089 bash[78452]: audit 1685822027.4151604 mon.c (mon.2) 245 : audit 0 from='client.? 172.21.15.89:0/1260940529' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:53:48.279 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:53:47 smithi089 bash[75068]: cluster 1685822026.2705715 mgr.y (mgr.34104) 490 : cluster 0 pgmap v410: 89 pgs: 83 active+clean, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped; 4.1 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 676 op/s; 11338/199767 objects degraded (5.676%); 1.2 MiB/s, 18 objects/s recovering 2023-06-03T19:53:48.280 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:53:47 smithi089 bash[75068]: cluster 1685822027.3535106 mon.a (mon.0) 455 : cluster 3 Health check update: Degraded data redundancy: 11338/199767 objects degraded (5.676%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-06-03T19:53:48.280 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:53:47 smithi089 bash[75068]: audit 1685822027.4151604 mon.c (mon.2) 245 : audit 0 from='client.? 172.21.15.89:0/1260940529' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:53:50.084 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:53:49 smithi164 bash[59462]: cluster 1685822028.2712598 mgr.y (mgr.34104) 491 : cluster 0 pgmap v411: 89 pgs: 83 active+clean, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped; 4.1 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 572 op/s; 11338/199143 objects degraded (5.693%); 1.2 MiB/s, 18 objects/s recovering 2023-06-03T19:53:50.279 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:53:49 smithi089 bash[78452]: cluster 1685822028.2712598 mgr.y (mgr.34104) 491 : cluster 0 pgmap v411: 89 pgs: 83 active+clean, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped; 4.1 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 572 op/s; 11338/199143 objects degraded (5.693%); 1.2 MiB/s, 18 objects/s recovering 2023-06-03T19:53:50.279 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:53:49 smithi089 bash[75068]: cluster 1685822028.2712598 mgr.y (mgr.34104) 491 : cluster 0 pgmap v411: 89 pgs: 83 active+clean, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped; 4.1 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 572 op/s; 11338/199143 objects degraded (5.693%); 1.2 MiB/s, 18 objects/s recovering 2023-06-03T19:53:51.084 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:53:50 smithi164 bash[59462]: audit 1685822029.9997733 mon.a (mon.0) 456 : audit 0 from='client.? 172.21.15.89:0/4025378870' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:53:51.279 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:53:50 smithi089 bash[78452]: audit 1685822029.9997733 mon.a (mon.0) 456 : audit 0 from='client.? 172.21.15.89:0/4025378870' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:53:51.279 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:53:50 smithi089 bash[75068]: audit 1685822029.9997733 mon.a (mon.0) 456 : audit 0 from='client.? 172.21.15.89:0/4025378870' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:53:52.084 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:53:51 smithi164 bash[59462]: cluster 1685822030.2720659 mgr.y (mgr.34104) 492 : cluster 0 pgmap v412: 89 pgs: 83 active+clean, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped; 3.9 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 756 op/s; 11229/191847 objects degraded (5.853%); 1.7 MiB/s, 27 objects/s recovering 2023-06-03T19:53:52.279 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:53:51 smithi089 bash[75068]: cluster 1685822030.2720659 mgr.y (mgr.34104) 492 : cluster 0 pgmap v412: 89 pgs: 83 active+clean, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped; 3.9 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 756 op/s; 11229/191847 objects degraded (5.853%); 1.7 MiB/s, 27 objects/s recovering 2023-06-03T19:53:52.279 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:53:51 smithi089 bash[78452]: cluster 1685822030.2720659 mgr.y (mgr.34104) 492 : cluster 0 pgmap v412: 89 pgs: 83 active+clean, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped; 3.9 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 756 op/s; 11229/191847 objects degraded (5.853%); 1.7 MiB/s, 27 objects/s recovering 2023-06-03T19:53:53.084 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:53:52 smithi164 bash[59462]: audit 1685822032.0836017 mon.c (mon.2) 246 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T19:53:53.084 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:53:52 smithi164 bash[59462]: cluster 1685822032.3547132 mon.a (mon.0) 457 : cluster 3 Health check update: Degraded data redundancy: 11229/191847 objects degraded (5.853%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-06-03T19:53:53.085 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:53:52 smithi164 bash[59462]: audit 1685822032.595543 mon.b (mon.1) 35 : audit 0 from='client.? 172.21.15.89:0/1069938951' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:53:53.279 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:53:52 smithi089 bash[75068]: audit 1685822032.0836017 mon.c (mon.2) 246 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T19:53:53.279 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:53:52 smithi089 bash[75068]: cluster 1685822032.3547132 mon.a (mon.0) 457 : cluster 3 Health check update: Degraded data redundancy: 11229/191847 objects degraded (5.853%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-06-03T19:53:53.279 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:53:52 smithi089 bash[75068]: audit 1685822032.595543 mon.b (mon.1) 35 : audit 0 from='client.? 172.21.15.89:0/1069938951' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:53:53.280 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:53:52 smithi089 bash[78452]: audit 1685822032.0836017 mon.c (mon.2) 246 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T19:53:53.281 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:53:52 smithi089 bash[78452]: cluster 1685822032.3547132 mon.a (mon.0) 457 : cluster 3 Health check update: Degraded data redundancy: 11229/191847 objects degraded (5.853%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-06-03T19:53:53.281 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:53:52 smithi089 bash[78452]: audit 1685822032.595543 mon.b (mon.1) 35 : audit 0 from='client.? 172.21.15.89:0/1069938951' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:53:54.084 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:53:53 smithi164 bash[59462]: cluster 1685822032.2727678 mgr.y (mgr.34104) 493 : cluster 0 pgmap v413: 89 pgs: 83 active+clean, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped; 3.9 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 596 op/s; 11229/190044 objects degraded (5.909%); 1.2 MiB/s, 18 objects/s recovering 2023-06-03T19:53:54.279 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:53:53 smithi089 bash[75068]: cluster 1685822032.2727678 mgr.y (mgr.34104) 493 : cluster 0 pgmap v413: 89 pgs: 83 active+clean, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped; 3.9 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 596 op/s; 11229/190044 objects degraded (5.909%); 1.2 MiB/s, 18 objects/s recovering 2023-06-03T19:53:54.279 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:53:53 smithi089 bash[78452]: cluster 1685822032.2727678 mgr.y (mgr.34104) 493 : cluster 0 pgmap v413: 89 pgs: 83 active+clean, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped; 3.9 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 596 op/s; 11229/190044 objects degraded (5.909%); 1.2 MiB/s, 18 objects/s recovering 2023-06-03T19:53:55.279 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:53:54 smithi089 bash[75068]: cluster 1685822034.2735658 mgr.y (mgr.34104) 494 : cluster 0 pgmap v414: 89 pgs: 83 active+clean, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped; 3.8 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 698 op/s; 11114/184536 objects degraded (6.023%); 1.8 MiB/s, 28 objects/s recovering 2023-06-03T19:53:55.279 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:53:54 smithi089 bash[78452]: cluster 1685822034.2735658 mgr.y (mgr.34104) 494 : cluster 0 pgmap v414: 89 pgs: 83 active+clean, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped; 3.8 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 698 op/s; 11114/184536 objects degraded (6.023%); 1.8 MiB/s, 28 objects/s recovering 2023-06-03T19:53:55.334 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:53:54 smithi164 bash[59462]: cluster 1685822034.2735658 mgr.y (mgr.34104) 494 : cluster 0 pgmap v414: 89 pgs: 83 active+clean, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped; 3.8 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 698 op/s; 11114/184536 objects degraded (6.023%); 1.8 MiB/s, 28 objects/s recovering 2023-06-03T19:53:55.779 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 19:53:55 smithi089 bash[71163]: debug 2023-06-03T19:53:55.426+0000 7fe48642e700 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2023-06-03T19:53:56.231 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:53:55 smithi089 bash[78452]: audit 1685822035.2098708 mon.c (mon.2) 247 : audit 0 from='client.? 172.21.15.89:0/3031397860' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:53:56.231 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:53:55 smithi089 bash[78452]: audit 1685822035.4294062 mon.c (mon.2) 248 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-06-03T19:53:56.231 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:53:55 smithi089 bash[78452]: audit 1685822035.4296436 mgr.y (mgr.34104) 495 : audit 0 from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-06-03T19:53:56.232 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:53:55 smithi089 bash[78452]: cephadm 1685822035.4331574 mgr.y (mgr.34104) 496 : cephadm 1 Upgrade: unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2023-06-03T19:53:56.232 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:53:55 smithi089 bash[75068]: audit 1685822035.2098708 mon.c (mon.2) 247 : audit 0 from='client.? 172.21.15.89:0/3031397860' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:53:56.232 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:53:55 smithi089 bash[75068]: audit 1685822035.4294062 mon.c (mon.2) 248 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-06-03T19:53:56.232 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:53:55 smithi089 bash[75068]: audit 1685822035.4296436 mgr.y (mgr.34104) 495 : audit 0 from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-06-03T19:53:56.233 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:53:55 smithi089 bash[75068]: cephadm 1685822035.4331574 mgr.y (mgr.34104) 496 : cephadm 1 Upgrade: unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2023-06-03T19:53:56.334 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:53:55 smithi164 bash[59462]: audit 1685822035.2098708 mon.c (mon.2) 247 : audit 0 from='client.? 172.21.15.89:0/3031397860' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:53:56.335 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:53:55 smithi164 bash[59462]: audit 1685822035.4294062 mon.c (mon.2) 248 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-06-03T19:53:56.335 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:53:55 smithi164 bash[59462]: audit 1685822035.4296436 mgr.y (mgr.34104) 495 : audit 0 from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-06-03T19:53:56.336 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:53:55 smithi164 bash[59462]: cephadm 1685822035.4331574 mgr.y (mgr.34104) 496 : cephadm 1 Upgrade: unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2023-06-03T19:53:56.528 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 19:53:56 smithi089 bash[71163]: ::ffff:172.21.15.164 - - [03/Jun/2023:19:53:56] "GET /metrics HTTP/1.1" 200 212800 "" "Prometheus/2.33.4" 2023-06-03T19:53:57.279 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:53:56 smithi089 bash[75068]: cluster 1685822036.274192 mgr.y (mgr.34104) 497 : cluster 0 pgmap v415: 89 pgs: 83 active+clean, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped; 3.7 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 664 op/s; 11114/180174 objects degraded (6.168%); 1.2 MiB/s, 18 objects/s recovering 2023-06-03T19:53:57.279 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:53:56 smithi089 bash[78452]: cluster 1685822036.274192 mgr.y (mgr.34104) 497 : cluster 0 pgmap v415: 89 pgs: 83 active+clean, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped; 3.7 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 664 op/s; 11114/180174 objects degraded (6.168%); 1.2 MiB/s, 18 objects/s recovering 2023-06-03T19:53:57.336 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:53:56 smithi164 bash[59462]: cluster 1685822036.274192 mgr.y (mgr.34104) 497 : cluster 0 pgmap v415: 89 pgs: 83 active+clean, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped; 3.7 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 664 op/s; 11114/180174 objects degraded (6.168%); 1.2 MiB/s, 18 objects/s recovering 2023-06-03T19:53:58.279 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:53:57 smithi089 bash[75068]: cluster 1685822037.35599 mon.a (mon.0) 458 : cluster 3 Health check update: Degraded data redundancy: 11114/180174 objects degraded (6.168%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-06-03T19:53:58.279 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:53:57 smithi089 bash[75068]: audit 1685822037.764353 mon.c (mon.2) 249 : audit 0 from='client.? 172.21.15.89:0/25405078' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:53:58.280 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:53:57 smithi089 bash[78452]: cluster 1685822037.35599 mon.a (mon.0) 458 : cluster 3 Health check update: Degraded data redundancy: 11114/180174 objects degraded (6.168%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-06-03T19:53:58.280 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:53:57 smithi089 bash[78452]: audit 1685822037.764353 mon.c (mon.2) 249 : audit 0 from='client.? 172.21.15.89:0/25405078' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:53:58.334 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:53:57 smithi164 bash[59462]: cluster 1685822037.35599 mon.a (mon.0) 458 : cluster 3 Health check update: Degraded data redundancy: 11114/180174 objects degraded (6.168%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-06-03T19:53:58.334 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:53:57 smithi164 bash[59462]: audit 1685822037.764353 mon.c (mon.2) 249 : audit 0 from='client.? 172.21.15.89:0/25405078' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:53:59.279 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:53:58 smithi089 bash[78452]: cluster 1685822038.2747762 mgr.y (mgr.34104) 498 : cluster 0 pgmap v416: 89 pgs: 83 active+clean, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped; 3.7 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 560 op/s; 11114/179586 objects degraded (6.189%); 1.2 MiB/s, 18 objects/s recovering 2023-06-03T19:53:59.279 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:53:58 smithi089 bash[75068]: cluster 1685822038.2747762 mgr.y (mgr.34104) 498 : cluster 0 pgmap v416: 89 pgs: 83 active+clean, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped; 3.7 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 560 op/s; 11114/179586 objects degraded (6.189%); 1.2 MiB/s, 18 objects/s recovering 2023-06-03T19:53:59.334 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:53:58 smithi164 bash[59462]: cluster 1685822038.2747762 mgr.y (mgr.34104) 498 : cluster 0 pgmap v416: 89 pgs: 83 active+clean, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped; 3.7 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 560 op/s; 11114/179586 objects degraded (6.189%); 1.2 MiB/s, 18 objects/s recovering 2023-06-03T19:54:01.779 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:54:01 smithi089 bash[75068]: cluster 1685822040.2756054 mgr.y (mgr.34104) 499 : cluster 0 pgmap v417: 89 pgs: 83 active+clean, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped; 3.5 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 741 op/s; 11002/172425 objects degraded (6.381%); 1.7 MiB/s, 27 objects/s recovering 2023-06-03T19:54:01.779 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:54:01 smithi089 bash[75068]: audit 1685822040.3427565 mon.a (mon.0) 459 : audit 0 from='client.? 172.21.15.89:0/3074573016' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:54:01.779 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:54:01 smithi089 bash[78452]: cluster 1685822040.2756054 mgr.y (mgr.34104) 499 : cluster 0 pgmap v417: 89 pgs: 83 active+clean, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped; 3.5 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 741 op/s; 11002/172425 objects degraded (6.381%); 1.7 MiB/s, 27 objects/s recovering 2023-06-03T19:54:01.779 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:54:01 smithi089 bash[78452]: audit 1685822040.3427565 mon.a (mon.0) 459 : audit 0 from='client.? 172.21.15.89:0/3074573016' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:54:01.834 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:54:01 smithi164 bash[59462]: cluster 1685822040.2756054 mgr.y (mgr.34104) 499 : cluster 0 pgmap v417: 89 pgs: 83 active+clean, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped; 3.5 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 741 op/s; 11002/172425 objects degraded (6.381%); 1.7 MiB/s, 27 objects/s recovering 2023-06-03T19:54:01.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:54:01 smithi164 bash[59462]: audit 1685822040.3427565 mon.a (mon.0) 459 : audit 0 from='client.? 172.21.15.89:0/3074573016' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:54:03.779 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:54:03 smithi089 bash[75068]: cluster 1685822042.2762175 mgr.y (mgr.34104) 500 : cluster 0 pgmap v418: 89 pgs: 83 active+clean, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped; 3.5 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 591 op/s; 11002/170556 objects degraded (6.451%); 1.2 MiB/s, 18 objects/s recovering 2023-06-03T19:54:03.779 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:54:03 smithi089 bash[75068]: cluster 1685822042.3610687 mon.a (mon.0) 460 : cluster 3 Health check update: Degraded data redundancy: 11002/170556 objects degraded (6.451%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-06-03T19:54:03.779 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:54:03 smithi089 bash[75068]: audit 1685822042.9396286 mon.b (mon.1) 36 : audit 0 from='client.? 172.21.15.89:0/1560206889' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:54:03.780 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:54:03 smithi089 bash[78452]: cluster 1685822042.2762175 mgr.y (mgr.34104) 500 : cluster 0 pgmap v418: 89 pgs: 83 active+clean, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped; 3.5 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 591 op/s; 11002/170556 objects degraded (6.451%); 1.2 MiB/s, 18 objects/s recovering 2023-06-03T19:54:03.780 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:54:03 smithi089 bash[78452]: cluster 1685822042.3610687 mon.a (mon.0) 460 : cluster 3 Health check update: Degraded data redundancy: 11002/170556 objects degraded (6.451%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-06-03T19:54:03.780 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:54:03 smithi089 bash[78452]: audit 1685822042.9396286 mon.b (mon.1) 36 : audit 0 from='client.? 172.21.15.89:0/1560206889' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:54:03.834 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:54:03 smithi164 bash[59462]: cluster 1685822042.2762175 mgr.y (mgr.34104) 500 : cluster 0 pgmap v418: 89 pgs: 83 active+clean, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped; 3.5 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 591 op/s; 11002/170556 objects degraded (6.451%); 1.2 MiB/s, 18 objects/s recovering 2023-06-03T19:54:03.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:54:03 smithi164 bash[59462]: cluster 1685822042.3610687 mon.a (mon.0) 460 : cluster 3 Health check update: Degraded data redundancy: 11002/170556 objects degraded (6.451%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-06-03T19:54:03.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:54:03 smithi164 bash[59462]: audit 1685822042.9396286 mon.b (mon.1) 36 : audit 0 from='client.? 172.21.15.89:0/1560206889' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:54:05.779 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:54:05 smithi089 bash[75068]: cluster 1685822044.2770596 mgr.y (mgr.34104) 501 : cluster 0 pgmap v419: 89 pgs: 83 active+clean, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped; 3.4 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 690 op/s; 10888/165186 objects degraded (6.591%); 1.8 MiB/s, 28 objects/s recovering 2023-06-03T19:54:05.779 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:54:05 smithi089 bash[78452]: cluster 1685822044.2770596 mgr.y (mgr.34104) 501 : cluster 0 pgmap v419: 89 pgs: 83 active+clean, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped; 3.4 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 690 op/s; 10888/165186 objects degraded (6.591%); 1.8 MiB/s, 28 objects/s recovering 2023-06-03T19:54:05.834 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:54:05 smithi164 bash[59462]: cluster 1685822044.2770596 mgr.y (mgr.34104) 501 : cluster 0 pgmap v419: 89 pgs: 83 active+clean, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped; 3.4 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 690 op/s; 10888/165186 objects degraded (6.591%); 1.8 MiB/s, 28 objects/s recovering 2023-06-03T19:54:06.529 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:54:06 smithi089 bash[75068]: audit 1685822045.488493 mon.b (mon.1) 37 : audit 0 from='client.? 172.21.15.89:0/2377839852' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:54:06.529 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 19:54:06 smithi089 bash[71163]: ::ffff:172.21.15.164 - - [03/Jun/2023:19:54:06] "GET /metrics HTTP/1.1" 200 212800 "" "Prometheus/2.33.4" 2023-06-03T19:54:06.530 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:54:06 smithi089 bash[78452]: audit 1685822045.488493 mon.b (mon.1) 37 : audit 0 from='client.? 172.21.15.89:0/2377839852' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:54:06.834 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:54:06 smithi164 bash[59462]: audit 1685822045.488493 mon.b (mon.1) 37 : audit 0 from='client.? 172.21.15.89:0/2377839852' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:54:07.779 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:54:07 smithi089 bash[75068]: cluster 1685822046.2776265 mgr.y (mgr.34104) 502 : cluster 0 pgmap v420: 89 pgs: 83 active+clean, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped; 3.3 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 658 op/s; 10888/160818 objects degraded (6.770%); 1.2 MiB/s, 18 objects/s recovering 2023-06-03T19:54:07.779 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:54:07 smithi089 bash[75068]: audit 1685822047.0834787 mon.c (mon.2) 250 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T19:54:07.779 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:54:07 smithi089 bash[78452]: cluster 1685822046.2776265 mgr.y (mgr.34104) 502 : cluster 0 pgmap v420: 89 pgs: 83 active+clean, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped; 3.3 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 658 op/s; 10888/160818 objects degraded (6.770%); 1.2 MiB/s, 18 objects/s recovering 2023-06-03T19:54:07.779 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:54:07 smithi089 bash[78452]: audit 1685822047.0834787 mon.c (mon.2) 250 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T19:54:07.834 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:54:07 smithi164 bash[59462]: cluster 1685822046.2776265 mgr.y (mgr.34104) 502 : cluster 0 pgmap v420: 89 pgs: 83 active+clean, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped; 3.3 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 658 op/s; 10888/160818 objects degraded (6.770%); 1.2 MiB/s, 18 objects/s recovering 2023-06-03T19:54:07.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:54:07 smithi164 bash[59462]: audit 1685822047.0834787 mon.c (mon.2) 250 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T19:54:08.779 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:54:08 smithi089 bash[75068]: cluster 1685822047.3622525 mon.a (mon.0) 461 : cluster 3 Health check update: Degraded data redundancy: 10888/160818 objects degraded (6.770%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-06-03T19:54:08.779 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:54:08 smithi089 bash[75068]: audit 1685822048.0677273 mon.c (mon.2) 251 : audit 0 from='client.? 172.21.15.89:0/891994105' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:54:08.779 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:54:08 smithi089 bash[78452]: cluster 1685822047.3622525 mon.a (mon.0) 461 : cluster 3 Health check update: Degraded data redundancy: 10888/160818 objects degraded (6.770%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-06-03T19:54:08.780 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:54:08 smithi089 bash[78452]: audit 1685822048.0677273 mon.c (mon.2) 251 : audit 0 from='client.? 172.21.15.89:0/891994105' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:54:08.834 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:54:08 smithi164 bash[59462]: cluster 1685822047.3622525 mon.a (mon.0) 461 : cluster 3 Health check update: Degraded data redundancy: 10888/160818 objects degraded (6.770%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-06-03T19:54:08.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:54:08 smithi164 bash[59462]: audit 1685822048.0677273 mon.c (mon.2) 251 : audit 0 from='client.? 172.21.15.89:0/891994105' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:54:09.779 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:54:09 smithi089 bash[75068]: cluster 1685822048.2781024 mgr.y (mgr.34104) 503 : cluster 0 pgmap v421: 89 pgs: 83 active+clean, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped; 3.3 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 554 op/s; 10888/160200 objects degraded (6.797%); 1.2 MiB/s, 18 objects/s recovering 2023-06-03T19:54:09.779 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:54:09 smithi089 bash[78452]: cluster 1685822048.2781024 mgr.y (mgr.34104) 503 : cluster 0 pgmap v421: 89 pgs: 83 active+clean, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped; 3.3 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 554 op/s; 10888/160200 objects degraded (6.797%); 1.2 MiB/s, 18 objects/s recovering 2023-06-03T19:54:09.834 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:54:09 smithi164 bash[59462]: cluster 1685822048.2781024 mgr.y (mgr.34104) 503 : cluster 0 pgmap v421: 89 pgs: 83 active+clean, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped; 3.3 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 554 op/s; 10888/160200 objects degraded (6.797%); 1.2 MiB/s, 18 objects/s recovering 2023-06-03T19:54:11.779 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:54:11 smithi089 bash[75068]: cluster 1685822050.278854 mgr.y (mgr.34104) 504 : cluster 0 pgmap v422: 89 pgs: 83 active+clean, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped; 3.1 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 744 op/s; 10774/152793 objects degraded (7.051%); 1.8 MiB/s, 28 objects/s recovering 2023-06-03T19:54:11.779 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:54:11 smithi089 bash[75068]: audit 1685822050.447581 mon.c (mon.2) 252 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-03T19:54:11.779 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:54:11 smithi089 bash[75068]: audit 1685822050.688556 mon.a (mon.0) 462 : audit 0 from='client.? 172.21.15.89:0/1662013698' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:54:11.780 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:54:11 smithi089 bash[78452]: cluster 1685822050.278854 mgr.y (mgr.34104) 504 : cluster 0 pgmap v422: 89 pgs: 83 active+clean, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped; 3.1 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 744 op/s; 10774/152793 objects degraded (7.051%); 1.8 MiB/s, 28 objects/s recovering 2023-06-03T19:54:11.780 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:54:11 smithi089 bash[78452]: audit 1685822050.447581 mon.c (mon.2) 252 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-03T19:54:11.780 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:54:11 smithi089 bash[78452]: audit 1685822050.688556 mon.a (mon.0) 462 : audit 0 from='client.? 172.21.15.89:0/1662013698' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:54:11.834 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:54:11 smithi164 bash[59462]: cluster 1685822050.278854 mgr.y (mgr.34104) 504 : cluster 0 pgmap v422: 89 pgs: 83 active+clean, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped; 3.1 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 744 op/s; 10774/152793 objects degraded (7.051%); 1.8 MiB/s, 28 objects/s recovering 2023-06-03T19:54:11.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:54:11 smithi164 bash[59462]: audit 1685822050.447581 mon.c (mon.2) 252 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-03T19:54:11.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:54:11 smithi164 bash[59462]: audit 1685822050.688556 mon.a (mon.0) 462 : audit 0 from='client.? 172.21.15.89:0/1662013698' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:54:12.779 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:54:12 smithi089 bash[75068]: cluster 1685822052.378143 mon.a (mon.0) 463 : cluster 3 Health check update: Degraded data redundancy: 10774/152793 objects degraded (7.051%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-06-03T19:54:12.779 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:54:12 smithi089 bash[78452]: cluster 1685822052.378143 mon.a (mon.0) 463 : cluster 3 Health check update: Degraded data redundancy: 10774/152793 objects degraded (7.051%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-06-03T19:54:12.834 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:54:12 smithi164 bash[59462]: cluster 1685822052.378143 mon.a (mon.0) 463 : cluster 3 Health check update: Degraded data redundancy: 10774/152793 objects degraded (7.051%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-06-03T19:54:13.779 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:54:13 smithi089 bash[75068]: cluster 1685822052.2794807 mgr.y (mgr.34104) 505 : cluster 0 pgmap v423: 89 pgs: 83 active+clean, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped; 3.1 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 599 op/s; 10774/150849 objects degraded (7.142%); 1.2 MiB/s, 18 objects/s recovering 2023-06-03T19:54:13.779 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:54:13 smithi089 bash[75068]: audit 1685822053.2520099 mon.a (mon.0) 464 : audit 0 from='client.? 172.21.15.89:0/3204895397' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:54:13.780 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:54:13 smithi089 bash[78452]: cluster 1685822052.2794807 mgr.y (mgr.34104) 505 : cluster 0 pgmap v423: 89 pgs: 83 active+clean, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped; 3.1 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 599 op/s; 10774/150849 objects degraded (7.142%); 1.2 MiB/s, 18 objects/s recovering 2023-06-03T19:54:13.780 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:54:13 smithi089 bash[78452]: audit 1685822053.2520099 mon.a (mon.0) 464 : audit 0 from='client.? 172.21.15.89:0/3204895397' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:54:13.834 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:54:13 smithi164 bash[59462]: cluster 1685822052.2794807 mgr.y (mgr.34104) 505 : cluster 0 pgmap v423: 89 pgs: 83 active+clean, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped; 3.1 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 599 op/s; 10774/150849 objects degraded (7.142%); 1.2 MiB/s, 18 objects/s recovering 2023-06-03T19:54:13.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:54:13 smithi164 bash[59462]: audit 1685822053.2520099 mon.a (mon.0) 464 : audit 0 from='client.? 172.21.15.89:0/3204895397' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:54:15.779 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:54:15 smithi089 bash[75068]: cluster 1685822054.280264 mgr.y (mgr.34104) 506 : cluster 0 pgmap v424: 89 pgs: 83 active+clean, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped; 3.0 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 695 op/s; 10659/145509 objects degraded (7.325%); 1.8 MiB/s, 28 objects/s recovering 2023-06-03T19:54:15.779 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:54:15 smithi089 bash[78452]: cluster 1685822054.280264 mgr.y (mgr.34104) 506 : cluster 0 pgmap v424: 89 pgs: 83 active+clean, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped; 3.0 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 695 op/s; 10659/145509 objects degraded (7.325%); 1.8 MiB/s, 28 objects/s recovering 2023-06-03T19:54:15.834 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:54:15 smithi164 bash[59462]: cluster 1685822054.280264 mgr.y (mgr.34104) 506 : cluster 0 pgmap v424: 89 pgs: 83 active+clean, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped; 3.0 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 695 op/s; 10659/145509 objects degraded (7.325%); 1.8 MiB/s, 28 objects/s recovering 2023-06-03T19:54:16.529 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 19:54:16 smithi089 bash[71163]: ::ffff:172.21.15.164 - - [03/Jun/2023:19:54:16] "GET /metrics HTTP/1.1" 200 212814 "" "Prometheus/2.33.4" 2023-06-03T19:54:16.529 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:54:16 smithi089 bash[78452]: audit 1685822055.8730433 mon.c (mon.2) 253 : audit 0 from='client.? 172.21.15.89:0/3904165059' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:54:16.530 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:54:16 smithi089 bash[75068]: audit 1685822055.8730433 mon.c (mon.2) 253 : audit 0 from='client.? 172.21.15.89:0/3904165059' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:54:16.834 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:54:16 smithi164 bash[59462]: audit 1685822055.8730433 mon.c (mon.2) 253 : audit 0 from='client.? 172.21.15.89:0/3904165059' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:54:17.779 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:54:17 smithi089 bash[75068]: cluster 1685822056.2809455 mgr.y (mgr.34104) 507 : cluster 0 pgmap v425: 89 pgs: 83 active+clean, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped; 2.9 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 660 op/s; 10659/141384 objects degraded (7.539%); 1.2 MiB/s, 19 objects/s recovering 2023-06-03T19:54:17.779 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:54:17 smithi089 bash[75068]: audit 1685822057.1213884 mon.a (mon.0) 465 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:54:17.779 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:54:17 smithi089 bash[75068]: audit 1685822057.1317718 mon.a (mon.0) 466 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:54:17.779 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:54:17 smithi089 bash[75068]: audit 1685822057.1628892 mon.a (mon.0) 467 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:54:17.779 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:54:17 smithi089 bash[75068]: audit 1685822057.1825964 mon.a (mon.0) 468 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:54:17.780 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:54:17 smithi089 bash[75068]: cluster 1685822057.379643 mon.a (mon.0) 469 : cluster 3 Health check update: Degraded data redundancy: 10659/141384 objects degraded (7.539%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-06-03T19:54:17.780 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:54:17 smithi089 bash[78452]: cluster 1685822056.2809455 mgr.y (mgr.34104) 507 : cluster 0 pgmap v425: 89 pgs: 83 active+clean, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped; 2.9 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 660 op/s; 10659/141384 objects degraded (7.539%); 1.2 MiB/s, 19 objects/s recovering 2023-06-03T19:54:17.780 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:54:17 smithi089 bash[78452]: audit 1685822057.1213884 mon.a (mon.0) 465 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:54:17.780 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:54:17 smithi089 bash[78452]: audit 1685822057.1317718 mon.a (mon.0) 466 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:54:17.780 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:54:17 smithi089 bash[78452]: audit 1685822057.1628892 mon.a (mon.0) 467 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:54:17.780 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:54:17 smithi089 bash[78452]: audit 1685822057.1825964 mon.a (mon.0) 468 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:54:17.781 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:54:17 smithi089 bash[78452]: cluster 1685822057.379643 mon.a (mon.0) 469 : cluster 3 Health check update: Degraded data redundancy: 10659/141384 objects degraded (7.539%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-06-03T19:54:17.834 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:54:17 smithi164 bash[59462]: cluster 1685822056.2809455 mgr.y (mgr.34104) 507 : cluster 0 pgmap v425: 89 pgs: 83 active+clean, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped; 2.9 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 660 op/s; 10659/141384 objects degraded (7.539%); 1.2 MiB/s, 19 objects/s recovering 2023-06-03T19:54:17.834 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:54:17 smithi164 bash[59462]: audit 1685822057.1213884 mon.a (mon.0) 465 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:54:17.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:54:17 smithi164 bash[59462]: audit 1685822057.1317718 mon.a (mon.0) 466 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:54:17.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:54:17 smithi164 bash[59462]: audit 1685822057.1628892 mon.a (mon.0) 467 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:54:17.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:54:17 smithi164 bash[59462]: audit 1685822057.1825964 mon.a (mon.0) 468 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:54:17.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:54:17 smithi164 bash[59462]: cluster 1685822057.379643 mon.a (mon.0) 469 : cluster 3 Health check update: Degraded data redundancy: 10659/141384 objects degraded (7.539%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-06-03T19:54:18.278 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 19:54:17 smithi089 bash[71163]: debug 2023-06-03T19:54:17.970+0000 7fe48642e700 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2023-06-03T19:54:18.779 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:54:18 smithi089 bash[75068]: audit 1685822057.8685381 mon.c (mon.2) 254 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-03T19:54:18.779 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:54:18 smithi089 bash[75068]: audit 1685822057.8697906 mon.c (mon.2) 255 : audit 1 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-03T19:54:18.779 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:54:18 smithi089 bash[75068]: audit 1685822057.878018 mon.a (mon.0) 470 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:54:18.780 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:54:18 smithi089 bash[75068]: audit 1685822057.958049 mon.c (mon.2) 256 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-03T19:54:18.780 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:54:18 smithi089 bash[75068]: audit 1685822057.9638102 mon.c (mon.2) 257 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-06-03T19:54:18.780 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:54:18 smithi089 bash[75068]: audit 1685822057.9673684 mon.c (mon.2) 258 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-06-03T19:54:18.780 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:54:18 smithi089 bash[75068]: audit 1685822057.9697688 mon.c (mon.2) 259 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-06-03T19:54:18.780 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:54:18 smithi089 bash[75068]: audit 1685822057.9730287 mon.c (mon.2) 260 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-06-03T19:54:18.781 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:54:18 smithi089 bash[78452]: audit 1685822057.8685381 mon.c (mon.2) 254 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-03T19:54:18.781 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:54:18 smithi089 bash[78452]: audit 1685822057.8697906 mon.c (mon.2) 255 : audit 1 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-03T19:54:18.781 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:54:18 smithi089 bash[78452]: audit 1685822057.878018 mon.a (mon.0) 470 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:54:18.781 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:54:18 smithi089 bash[78452]: audit 1685822057.958049 mon.c (mon.2) 256 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-03T19:54:18.781 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:54:18 smithi089 bash[78452]: audit 1685822057.9638102 mon.c (mon.2) 257 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-06-03T19:54:18.782 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:54:18 smithi089 bash[78452]: audit 1685822057.9673684 mon.c (mon.2) 258 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-06-03T19:54:18.782 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:54:18 smithi089 bash[78452]: audit 1685822057.9697688 mon.c (mon.2) 259 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-06-03T19:54:18.782 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:54:18 smithi089 bash[78452]: audit 1685822057.9730287 mon.c (mon.2) 260 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-06-03T19:54:18.834 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:54:18 smithi164 bash[59462]: audit 1685822057.8685381 mon.c (mon.2) 254 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-03T19:54:18.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:54:18 smithi164 bash[59462]: audit 1685822057.8697906 mon.c (mon.2) 255 : audit 1 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-03T19:54:18.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:54:18 smithi164 bash[59462]: audit 1685822057.878018 mon.a (mon.0) 470 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:54:18.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:54:18 smithi164 bash[59462]: audit 1685822057.958049 mon.c (mon.2) 256 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-03T19:54:18.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:54:18 smithi164 bash[59462]: audit 1685822057.9638102 mon.c (mon.2) 257 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-06-03T19:54:18.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:54:18 smithi164 bash[59462]: audit 1685822057.9673684 mon.c (mon.2) 258 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-06-03T19:54:18.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:54:18 smithi164 bash[59462]: audit 1685822057.9697688 mon.c (mon.2) 259 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-06-03T19:54:18.836 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:54:18 smithi164 bash[59462]: audit 1685822057.9730287 mon.c (mon.2) 260 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-06-03T19:54:19.779 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:54:19 smithi089 bash[75068]: audit 1685822057.9732528 mgr.y (mgr.34104) 508 : audit 0 from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-06-03T19:54:19.779 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:54:19 smithi089 bash[75068]: cephadm 1685822057.9778957 mgr.y (mgr.34104) 509 : cephadm 1 Upgrade: unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2023-06-03T19:54:19.779 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:54:19 smithi089 bash[75068]: cluster 1685822058.2815614 mgr.y (mgr.34104) 510 : cluster 0 pgmap v426: 89 pgs: 83 active+clean, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped; 2.9 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 555 op/s; 10659/140829 objects degraded (7.569%); 1.2 MiB/s, 19 objects/s recovering 2023-06-03T19:54:19.779 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:54:19 smithi089 bash[75068]: audit 1685822058.4709883 mon.a (mon.0) 471 : audit 0 from='client.? 172.21.15.89:0/2947589337' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:54:19.780 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:54:19 smithi089 bash[78452]: audit 1685822057.9732528 mgr.y (mgr.34104) 508 : audit 0 from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-06-03T19:54:19.780 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:54:19 smithi089 bash[78452]: cephadm 1685822057.9778957 mgr.y (mgr.34104) 509 : cephadm 1 Upgrade: unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2023-06-03T19:54:19.780 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:54:19 smithi089 bash[78452]: cluster 1685822058.2815614 mgr.y (mgr.34104) 510 : cluster 0 pgmap v426: 89 pgs: 83 active+clean, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped; 2.9 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 555 op/s; 10659/140829 objects degraded (7.569%); 1.2 MiB/s, 19 objects/s recovering 2023-06-03T19:54:19.780 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:54:19 smithi089 bash[78452]: audit 1685822058.4709883 mon.a (mon.0) 471 : audit 0 from='client.? 172.21.15.89:0/2947589337' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:54:19.834 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:54:19 smithi164 bash[59462]: audit 1685822057.9732528 mgr.y (mgr.34104) 508 : audit 0 from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-06-03T19:54:19.834 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:54:19 smithi164 bash[59462]: cephadm 1685822057.9778957 mgr.y (mgr.34104) 509 : cephadm 1 Upgrade: unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2023-06-03T19:54:19.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:54:19 smithi164 bash[59462]: cluster 1685822058.2815614 mgr.y (mgr.34104) 510 : cluster 0 pgmap v426: 89 pgs: 83 active+clean, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped; 2.9 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 555 op/s; 10659/140829 objects degraded (7.569%); 1.2 MiB/s, 19 objects/s recovering 2023-06-03T19:54:19.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:54:19 smithi164 bash[59462]: audit 1685822058.4709883 mon.a (mon.0) 471 : audit 0 from='client.? 172.21.15.89:0/2947589337' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:54:21.779 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:54:21 smithi089 bash[75068]: cluster 1685822060.2824209 mgr.y (mgr.34104) 511 : cluster 0 pgmap v427: 89 pgs: 83 active+clean, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped; 2.7 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 725 op/s; 10546/134079 objects degraded (7.866%); 1.8 MiB/s, 28 objects/s recovering 2023-06-03T19:54:21.779 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:54:21 smithi089 bash[75068]: audit 1685822061.0273468 mon.a (mon.0) 472 : audit 0 from='client.? 172.21.15.89:0/971874372' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:54:21.779 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:54:21 smithi089 bash[78452]: cluster 1685822060.2824209 mgr.y (mgr.34104) 511 : cluster 0 pgmap v427: 89 pgs: 83 active+clean, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped; 2.7 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 725 op/s; 10546/134079 objects degraded (7.866%); 1.8 MiB/s, 28 objects/s recovering 2023-06-03T19:54:21.779 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:54:21 smithi089 bash[78452]: audit 1685822061.0273468 mon.a (mon.0) 472 : audit 0 from='client.? 172.21.15.89:0/971874372' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:54:21.834 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:54:21 smithi164 bash[59462]: cluster 1685822060.2824209 mgr.y (mgr.34104) 511 : cluster 0 pgmap v427: 89 pgs: 83 active+clean, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped; 2.7 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 725 op/s; 10546/134079 objects degraded (7.866%); 1.8 MiB/s, 28 objects/s recovering 2023-06-03T19:54:21.834 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:54:21 smithi164 bash[59462]: audit 1685822061.0273468 mon.a (mon.0) 472 : audit 0 from='client.? 172.21.15.89:0/971874372' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:54:22.779 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:54:22 smithi089 bash[75068]: audit 1685822062.0838892 mon.c (mon.2) 261 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T19:54:22.779 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:54:22 smithi089 bash[75068]: cluster 1685822062.3809133 mon.a (mon.0) 473 : cluster 3 Health check update: Degraded data redundancy: 10546/134079 objects degraded (7.866%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-06-03T19:54:22.779 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:54:22 smithi089 bash[75068]: cluster 1685822062.397881 mon.a (mon.0) 474 : cluster 0 osdmap e108: 8 total, 8 up, 8 in 2023-06-03T19:54:22.779 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:54:22 smithi089 bash[78452]: audit 1685822062.0838892 mon.c (mon.2) 261 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T19:54:22.779 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:54:22 smithi089 bash[78452]: cluster 1685822062.3809133 mon.a (mon.0) 473 : cluster 3 Health check update: Degraded data redundancy: 10546/134079 objects degraded (7.866%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-06-03T19:54:22.780 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:54:22 smithi089 bash[78452]: cluster 1685822062.397881 mon.a (mon.0) 474 : cluster 0 osdmap e108: 8 total, 8 up, 8 in 2023-06-03T19:54:22.834 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:54:22 smithi164 bash[59462]: audit 1685822062.0838892 mon.c (mon.2) 261 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T19:54:22.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:54:22 smithi164 bash[59462]: cluster 1685822062.3809133 mon.a (mon.0) 473 : cluster 3 Health check update: Degraded data redundancy: 10546/134079 objects degraded (7.866%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-06-03T19:54:22.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:54:22 smithi164 bash[59462]: cluster 1685822062.397881 mon.a (mon.0) 474 : cluster 0 osdmap e108: 8 total, 8 up, 8 in 2023-06-03T19:54:23.779 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:54:23 smithi089 bash[78452]: cluster 1685822062.283104 mgr.y (mgr.34104) 512 : cluster 0 pgmap v428: 89 pgs: 83 active+clean, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped; 2.7 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 563 op/s; 10546/132483 objects degraded (7.960%); 1.2 MiB/s, 18 objects/s recovering 2023-06-03T19:54:23.779 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:54:23 smithi089 bash[78452]: cluster 1685822062.3877068 osd.3 (osd.3) 32 : cluster 3 failed to encode map e108 with expected crc 2023-06-03T19:54:23.779 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:54:23 smithi089 bash[78452]: cluster 1685822062.3953862 osd.5 (osd.5) 27 : cluster 3 failed to encode map e108 with expected crc 2023-06-03T19:54:23.780 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:54:23 smithi089 bash[78452]: cluster 1685822062.3966732 osd.5 (osd.5) 28 : cluster 3 failed to encode map e108 with expected crc 2023-06-03T19:54:23.780 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:54:23 smithi089 bash[78452]: cluster 1685822063.4058907 mon.a (mon.0) 475 : cluster 0 osdmap e109: 8 total, 8 up, 8 in 2023-06-03T19:54:23.780 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:54:23 smithi089 bash[78452]: audit 1685822063.4590147 mon.a (mon.0) 476 : audit 0 from='client.? 172.21.15.89:0/2446157541' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:54:23.781 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:54:23 smithi089 bash[75068]: cluster 1685822062.283104 mgr.y (mgr.34104) 512 : cluster 0 pgmap v428: 89 pgs: 83 active+clean, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped; 2.7 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 563 op/s; 10546/132483 objects degraded (7.960%); 1.2 MiB/s, 18 objects/s recovering 2023-06-03T19:54:23.781 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:54:23 smithi089 bash[75068]: cluster 1685822062.3877068 osd.3 (osd.3) 32 : cluster 3 failed to encode map e108 with expected crc 2023-06-03T19:54:23.781 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:54:23 smithi089 bash[75068]: cluster 1685822062.3953862 osd.5 (osd.5) 27 : cluster 3 failed to encode map e108 with expected crc 2023-06-03T19:54:23.781 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:54:23 smithi089 bash[75068]: cluster 1685822062.3966732 osd.5 (osd.5) 28 : cluster 3 failed to encode map e108 with expected crc 2023-06-03T19:54:23.782 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:54:23 smithi089 bash[75068]: cluster 1685822063.4058907 mon.a (mon.0) 475 : cluster 0 osdmap e109: 8 total, 8 up, 8 in 2023-06-03T19:54:23.782 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:54:23 smithi089 bash[75068]: audit 1685822063.4590147 mon.a (mon.0) 476 : audit 0 from='client.? 172.21.15.89:0/2446157541' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:54:23.834 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:54:23 smithi164 bash[59462]: cluster 1685822062.283104 mgr.y (mgr.34104) 512 : cluster 0 pgmap v428: 89 pgs: 83 active+clean, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped; 2.7 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 563 op/s; 10546/132483 objects degraded (7.960%); 1.2 MiB/s, 18 objects/s recovering 2023-06-03T19:54:23.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:54:23 smithi164 bash[59462]: cluster 1685822062.3877068 osd.3 (osd.3) 32 : cluster 3 failed to encode map e108 with expected crc 2023-06-03T19:54:23.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:54:23 smithi164 bash[59462]: cluster 1685822062.3953862 osd.5 (osd.5) 27 : cluster 3 failed to encode map e108 with expected crc 2023-06-03T19:54:23.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:54:23 smithi164 bash[59462]: cluster 1685822062.3966732 osd.5 (osd.5) 28 : cluster 3 failed to encode map e108 with expected crc 2023-06-03T19:54:23.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:54:23 smithi164 bash[59462]: cluster 1685822063.4058907 mon.a (mon.0) 475 : cluster 0 osdmap e109: 8 total, 8 up, 8 in 2023-06-03T19:54:23.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:54:23 smithi164 bash[59462]: audit 1685822063.4590147 mon.a (mon.0) 476 : audit 0 from='client.? 172.21.15.89:0/2446157541' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:54:24.479 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:54:24 smithi164 bash[59462]: cluster 1685822063.402836 osd.3 (osd.3) 33 : cluster 3 failed to encode map e109 with expected crc 2023-06-03T19:54:24.479 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:54:24 smithi164 bash[59462]: cluster 1685822063.4045868 osd.6 (osd.6) 24 : cluster 3 failed to encode map e109 with expected crc 2023-06-03T19:54:24.479 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:54:24 smithi164 bash[59462]: cluster 1685822063.4047518 osd.5 (osd.5) 29 : cluster 3 failed to encode map e109 with expected crc 2023-06-03T19:54:24.479 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:54:24 smithi164 bash[59462]: cluster 1685822063.4536436 osd.2 (osd.2) 32 : cluster 3 failed to encode map e109 with expected crc 2023-06-03T19:54:24.479 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:54:24 smithi164 bash[59462]: cluster 1685822063.5054836 osd.7 (osd.7) 21 : cluster 3 failed to encode map e109 with expected crc 2023-06-03T19:54:24.779 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:54:24 smithi089 bash[78452]: cluster 1685822063.402836 osd.3 (osd.3) 33 : cluster 3 failed to encode map e109 with expected crc 2023-06-03T19:54:24.779 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:54:24 smithi089 bash[78452]: cluster 1685822063.4045868 osd.6 (osd.6) 24 : cluster 3 failed to encode map e109 with expected crc 2023-06-03T19:54:24.779 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:54:24 smithi089 bash[78452]: cluster 1685822063.4047518 osd.5 (osd.5) 29 : cluster 3 failed to encode map e109 with expected crc 2023-06-03T19:54:24.779 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:54:24 smithi089 bash[78452]: cluster 1685822063.4536436 osd.2 (osd.2) 32 : cluster 3 failed to encode map e109 with expected crc 2023-06-03T19:54:24.779 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:54:24 smithi089 bash[78452]: cluster 1685822063.5054836 osd.7 (osd.7) 21 : cluster 3 failed to encode map e109 with expected crc 2023-06-03T19:54:24.780 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:54:24 smithi089 bash[75068]: cluster 1685822063.402836 osd.3 (osd.3) 33 : cluster 3 failed to encode map e109 with expected crc 2023-06-03T19:54:24.780 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:54:24 smithi089 bash[75068]: cluster 1685822063.4045868 osd.6 (osd.6) 24 : cluster 3 failed to encode map e109 with expected crc 2023-06-03T19:54:24.780 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:54:24 smithi089 bash[75068]: cluster 1685822063.4047518 osd.5 (osd.5) 29 : cluster 3 failed to encode map e109 with expected crc 2023-06-03T19:54:24.780 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:54:24 smithi089 bash[75068]: cluster 1685822063.4536436 osd.2 (osd.2) 32 : cluster 3 failed to encode map e109 with expected crc 2023-06-03T19:54:24.780 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:54:24 smithi089 bash[75068]: cluster 1685822063.5054836 osd.7 (osd.7) 21 : cluster 3 failed to encode map e109 with expected crc 2023-06-03T19:54:25.778 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:54:25 smithi089 bash[75068]: cluster 1685822064.283856 mgr.y (mgr.34104) 513 : cluster 0 pgmap v431: 89 pgs: 1 peering, 83 active+clean, 5 active+recovery_wait+undersized+degraded+remapped; 2.6 GiB data, 9.7 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 567 op/s; 10476/127752 objects degraded (8.200%); 1.4 MiB/s, 22 objects/s recovering 2023-06-03T19:54:25.779 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:54:25 smithi089 bash[78452]: cluster 1685822064.283856 mgr.y (mgr.34104) 513 : cluster 0 pgmap v431: 89 pgs: 1 peering, 83 active+clean, 5 active+recovery_wait+undersized+degraded+remapped; 2.6 GiB data, 9.7 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 567 op/s; 10476/127752 objects degraded (8.200%); 1.4 MiB/s, 22 objects/s recovering 2023-06-03T19:54:25.834 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:54:25 smithi164 bash[59462]: cluster 1685822064.283856 mgr.y (mgr.34104) 513 : cluster 0 pgmap v431: 89 pgs: 1 peering, 83 active+clean, 5 active+recovery_wait+undersized+degraded+remapped; 2.6 GiB data, 9.7 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 567 op/s; 10476/127752 objects degraded (8.200%); 1.4 MiB/s, 22 objects/s recovering 2023-06-03T19:54:26.485 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:54:26 smithi089 bash[75068]: audit 1685822066.0230908 mon.a (mon.0) 477 : audit 0 from='client.? 172.21.15.89:0/3656218541' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:54:26.485 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 19:54:26 smithi089 bash[71163]: ::ffff:172.21.15.164 - - [03/Jun/2023:19:54:26] "GET /metrics HTTP/1.1" 200 212831 "" "Prometheus/2.33.4" 2023-06-03T19:54:26.779 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:54:26 smithi089 bash[78452]: audit 1685822066.0230908 mon.a (mon.0) 477 : audit 0 from='client.? 172.21.15.89:0/3656218541' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:54:26.834 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:54:26 smithi164 bash[59462]: audit 1685822066.0230908 mon.a (mon.0) 477 : audit 0 from='client.? 172.21.15.89:0/3656218541' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:54:27.779 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:54:27 smithi089 bash[75068]: cluster 1685822066.2844946 mgr.y (mgr.34104) 514 : cluster 0 pgmap v432: 89 pgs: 1 peering, 83 active+clean, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped; 2.5 GiB data, 9.6 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 689 op/s; 10400/124272 objects degraded (8.369%); 2.0 MiB/s, 32 objects/s recovering 2023-06-03T19:54:27.779 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:54:27 smithi089 bash[75068]: cluster 1685822067.3887796 mon.a (mon.0) 478 : cluster 3 Health check update: Degraded data redundancy: 10400/124272 objects degraded (8.369%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-06-03T19:54:27.779 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:54:27 smithi089 bash[78452]: cluster 1685822066.2844946 mgr.y (mgr.34104) 514 : cluster 0 pgmap v432: 89 pgs: 1 peering, 83 active+clean, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped; 2.5 GiB data, 9.6 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 689 op/s; 10400/124272 objects degraded (8.369%); 2.0 MiB/s, 32 objects/s recovering 2023-06-03T19:54:27.780 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:54:27 smithi089 bash[78452]: cluster 1685822067.3887796 mon.a (mon.0) 478 : cluster 3 Health check update: Degraded data redundancy: 10400/124272 objects degraded (8.369%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-06-03T19:54:27.834 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:54:27 smithi164 bash[59462]: cluster 1685822066.2844946 mgr.y (mgr.34104) 514 : cluster 0 pgmap v432: 89 pgs: 1 peering, 83 active+clean, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped; 2.5 GiB data, 9.6 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 689 op/s; 10400/124272 objects degraded (8.369%); 2.0 MiB/s, 32 objects/s recovering 2023-06-03T19:54:27.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:54:27 smithi164 bash[59462]: cluster 1685822067.3887796 mon.a (mon.0) 478 : cluster 3 Health check update: Degraded data redundancy: 10400/124272 objects degraded (8.369%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-06-03T19:54:29.779 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:54:29 smithi089 bash[78452]: cluster 1685822068.2851634 mgr.y (mgr.34104) 515 : cluster 0 pgmap v433: 89 pgs: 84 active+clean, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped; 2.5 GiB data, 9.6 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 454 op/s; 10400/123159 objects degraded (8.444%); 1.1 MiB/s, 18 objects/s recovering 2023-06-03T19:54:29.779 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:54:29 smithi089 bash[78452]: audit 1685822068.606349 mon.c (mon.2) 262 : audit 0 from='client.? 172.21.15.89:0/2391264375' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:54:29.779 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:54:29 smithi089 bash[75068]: cluster 1685822068.2851634 mgr.y (mgr.34104) 515 : cluster 0 pgmap v433: 89 pgs: 84 active+clean, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped; 2.5 GiB data, 9.6 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 454 op/s; 10400/123159 objects degraded (8.444%); 1.1 MiB/s, 18 objects/s recovering 2023-06-03T19:54:29.779 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:54:29 smithi089 bash[75068]: audit 1685822068.606349 mon.c (mon.2) 262 : audit 0 from='client.? 172.21.15.89:0/2391264375' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:54:29.834 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:54:29 smithi164 bash[59462]: cluster 1685822068.2851634 mgr.y (mgr.34104) 515 : cluster 0 pgmap v433: 89 pgs: 84 active+clean, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped; 2.5 GiB data, 9.6 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 454 op/s; 10400/123159 objects degraded (8.444%); 1.1 MiB/s, 18 objects/s recovering 2023-06-03T19:54:29.834 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:54:29 smithi164 bash[59462]: audit 1685822068.606349 mon.c (mon.2) 262 : audit 0 from='client.? 172.21.15.89:0/2391264375' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:54:31.779 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:54:31 smithi089 bash[75068]: cluster 1685822070.2860343 mgr.y (mgr.34104) 516 : cluster 0 pgmap v434: 89 pgs: 84 active+clean, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped; 2.4 GiB data, 9.2 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 668 op/s; 10293/116436 objects degraded (8.840%); 2.0 MiB/s, 31 objects/s recovering 2023-06-03T19:54:31.779 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:54:31 smithi089 bash[75068]: audit 1685822071.197114 mon.a (mon.0) 479 : audit 0 from='client.? 172.21.15.89:0/2581346923' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:54:31.780 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:54:31 smithi089 bash[78452]: cluster 1685822070.2860343 mgr.y (mgr.34104) 516 : cluster 0 pgmap v434: 89 pgs: 84 active+clean, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped; 2.4 GiB data, 9.2 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 668 op/s; 10293/116436 objects degraded (8.840%); 2.0 MiB/s, 31 objects/s recovering 2023-06-03T19:54:31.780 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:54:31 smithi089 bash[78452]: audit 1685822071.197114 mon.a (mon.0) 479 : audit 0 from='client.? 172.21.15.89:0/2581346923' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:54:31.834 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:54:31 smithi164 bash[59462]: cluster 1685822070.2860343 mgr.y (mgr.34104) 516 : cluster 0 pgmap v434: 89 pgs: 84 active+clean, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped; 2.4 GiB data, 9.2 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 668 op/s; 10293/116436 objects degraded (8.840%); 2.0 MiB/s, 31 objects/s recovering 2023-06-03T19:54:31.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:54:31 smithi164 bash[59462]: audit 1685822071.197114 mon.a (mon.0) 479 : audit 0 from='client.? 172.21.15.89:0/2581346923' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:54:32.779 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:54:32 smithi089 bash[75068]: cluster 1685822072.3902504 mon.a (mon.0) 480 : cluster 3 Health check update: Degraded data redundancy: 10293/116436 objects degraded (8.840%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-06-03T19:54:32.779 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:54:32 smithi089 bash[78452]: cluster 1685822072.3902504 mon.a (mon.0) 480 : cluster 3 Health check update: Degraded data redundancy: 10293/116436 objects degraded (8.840%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-06-03T19:54:32.834 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:54:32 smithi164 bash[59462]: cluster 1685822072.3902504 mon.a (mon.0) 480 : cluster 3 Health check update: Degraded data redundancy: 10293/116436 objects degraded (8.840%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-06-03T19:54:33.279 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 19:54:32 smithi089 bash[71163]: debug 2023-06-03T19:54:32.990+0000 7fe48642e700 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2023-06-03T19:54:33.779 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:54:33 smithi089 bash[75068]: cluster 1685822072.28667 mgr.y (mgr.34104) 517 : cluster 0 pgmap v435: 89 pgs: 84 active+clean, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped; 2.3 GiB data, 9.1 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 597 op/s; 10293/114744 objects degraded (8.970%); 1.6 MiB/s, 25 objects/s recovering 2023-06-03T19:54:33.779 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:54:33 smithi089 bash[75068]: audit 1685822072.9932723 mon.c (mon.2) 263 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-06-03T19:54:33.779 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:54:33 smithi089 bash[78452]: cluster 1685822072.28667 mgr.y (mgr.34104) 517 : cluster 0 pgmap v435: 89 pgs: 84 active+clean, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped; 2.3 GiB data, 9.1 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 597 op/s; 10293/114744 objects degraded (8.970%); 1.6 MiB/s, 25 objects/s recovering 2023-06-03T19:54:33.780 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:54:33 smithi089 bash[78452]: audit 1685822072.9932723 mon.c (mon.2) 263 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-06-03T19:54:33.834 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:54:33 smithi164 bash[59462]: cluster 1685822072.28667 mgr.y (mgr.34104) 517 : cluster 0 pgmap v435: 89 pgs: 84 active+clean, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped; 2.3 GiB data, 9.1 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 597 op/s; 10293/114744 objects degraded (8.970%); 1.6 MiB/s, 25 objects/s recovering 2023-06-03T19:54:33.834 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:54:33 smithi164 bash[59462]: audit 1685822072.9932723 mon.c (mon.2) 263 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-06-03T19:54:34.834 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:54:34 smithi164 bash[59462]: audit 1685822072.9936152 mgr.y (mgr.34104) 518 : audit 0 from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-06-03T19:54:34.834 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:54:34 smithi164 bash[59462]: cephadm 1685822072.9951935 mgr.y (mgr.34104) 519 : cephadm 1 Upgrade: unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2023-06-03T19:54:34.834 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:54:34 smithi164 bash[59462]: audit 1685822073.7732668 mon.c (mon.2) 264 : audit 0 from='client.? 172.21.15.89:0/2438101391' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:54:35.029 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:54:34 smithi089 bash[75068]: audit 1685822072.9936152 mgr.y (mgr.34104) 518 : audit 0 from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-06-03T19:54:35.029 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:54:34 smithi089 bash[75068]: cephadm 1685822072.9951935 mgr.y (mgr.34104) 519 : cephadm 1 Upgrade: unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2023-06-03T19:54:35.029 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:54:34 smithi089 bash[75068]: audit 1685822073.7732668 mon.c (mon.2) 264 : audit 0 from='client.? 172.21.15.89:0/2438101391' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:54:35.030 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:54:34 smithi089 bash[78452]: audit 1685822072.9936152 mgr.y (mgr.34104) 518 : audit 0 from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-06-03T19:54:35.030 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:54:34 smithi089 bash[78452]: cephadm 1685822072.9951935 mgr.y (mgr.34104) 519 : cephadm 1 Upgrade: unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2023-06-03T19:54:35.030 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:54:34 smithi089 bash[78452]: audit 1685822073.7732668 mon.c (mon.2) 264 : audit 0 from='client.? 172.21.15.89:0/2438101391' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:54:35.834 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:54:35 smithi164 bash[59462]: cluster 1685822074.2874331 mgr.y (mgr.34104) 520 : cluster 0 pgmap v436: 89 pgs: 84 active+clean, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped; 2.2 GiB data, 8.7 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 657 op/s; 10293/109380 objects degraded (9.410%); 1.1 MiB/s, 16 objects/s recovering 2023-06-03T19:54:36.029 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:54:35 smithi089 bash[75068]: cluster 1685822074.2874331 mgr.y (mgr.34104) 520 : cluster 0 pgmap v436: 89 pgs: 84 active+clean, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped; 2.2 GiB data, 8.7 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 657 op/s; 10293/109380 objects degraded (9.410%); 1.1 MiB/s, 16 objects/s recovering 2023-06-03T19:54:36.029 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:54:35 smithi089 bash[78452]: cluster 1685822074.2874331 mgr.y (mgr.34104) 520 : cluster 0 pgmap v436: 89 pgs: 84 active+clean, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped; 2.2 GiB data, 8.7 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 657 op/s; 10293/109380 objects degraded (9.410%); 1.1 MiB/s, 16 objects/s recovering 2023-06-03T19:54:36.529 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 19:54:36 smithi089 bash[71163]: ::ffff:172.21.15.164 - - [03/Jun/2023:19:54:36] "GET /metrics HTTP/1.1" 200 212831 "" "Prometheus/2.33.4" 2023-06-03T19:54:36.834 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:54:36 smithi164 bash[59462]: audit 1685822076.3677979 mon.c (mon.2) 265 : audit 0 from='client.? 172.21.15.89:0/1290296749' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:54:37.032 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:54:36 smithi089 bash[75068]: audit 1685822076.3677979 mon.c (mon.2) 265 : audit 0 from='client.? 172.21.15.89:0/1290296749' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:54:37.032 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:54:36 smithi089 bash[78452]: audit 1685822076.3677979 mon.c (mon.2) 265 : audit 0 from='client.? 172.21.15.89:0/1290296749' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:54:37.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:54:37 smithi164 bash[59462]: cluster 1685822076.2881708 mgr.y (mgr.34104) 521 : cluster 0 pgmap v437: 89 pgs: 84 active+clean, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped; 2.1 GiB data, 8.6 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 628 op/s; 10181/105135 objects degraded (9.684%); 1.4 MiB/s, 24 objects/s recovering 2023-06-03T19:54:37.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:54:37 smithi164 bash[59462]: audit 1685822077.0846202 mon.c (mon.2) 266 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T19:54:37.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:54:37 smithi164 bash[59462]: cluster 1685822077.397039 mon.a (mon.0) 481 : cluster 3 Health check update: Degraded data redundancy: 10181/105135 objects degraded (9.684%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-06-03T19:54:38.029 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:54:37 smithi089 bash[78452]: cluster 1685822076.2881708 mgr.y (mgr.34104) 521 : cluster 0 pgmap v437: 89 pgs: 84 active+clean, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped; 2.1 GiB data, 8.6 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 628 op/s; 10181/105135 objects degraded (9.684%); 1.4 MiB/s, 24 objects/s recovering 2023-06-03T19:54:38.029 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:54:37 smithi089 bash[78452]: audit 1685822077.0846202 mon.c (mon.2) 266 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T19:54:38.029 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:54:37 smithi089 bash[78452]: cluster 1685822077.397039 mon.a (mon.0) 481 : cluster 3 Health check update: Degraded data redundancy: 10181/105135 objects degraded (9.684%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-06-03T19:54:38.030 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:54:37 smithi089 bash[75068]: cluster 1685822076.2881708 mgr.y (mgr.34104) 521 : cluster 0 pgmap v437: 89 pgs: 84 active+clean, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped; 2.1 GiB data, 8.6 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 628 op/s; 10181/105135 objects degraded (9.684%); 1.4 MiB/s, 24 objects/s recovering 2023-06-03T19:54:38.030 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:54:37 smithi089 bash[75068]: audit 1685822077.0846202 mon.c (mon.2) 266 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T19:54:38.030 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:54:37 smithi089 bash[75068]: cluster 1685822077.397039 mon.a (mon.0) 481 : cluster 3 Health check update: Degraded data redundancy: 10181/105135 objects degraded (9.684%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-06-03T19:54:39.834 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:54:39 smithi164 bash[59462]: cluster 1685822078.2889755 mgr.y (mgr.34104) 522 : cluster 0 pgmap v438: 89 pgs: 84 active+clean, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped; 2.1 GiB data, 8.5 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 566 op/s; 10181/103884 objects degraded (9.800%); 981 KiB/s, 18 objects/s recovering 2023-06-03T19:54:39.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:54:39 smithi164 bash[59462]: audit 1685822078.9247944 mon.b (mon.1) 38 : audit 0 from='client.? 172.21.15.89:0/2372348630' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:54:40.029 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:54:39 smithi089 bash[75068]: cluster 1685822078.2889755 mgr.y (mgr.34104) 522 : cluster 0 pgmap v438: 89 pgs: 84 active+clean, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped; 2.1 GiB data, 8.5 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 566 op/s; 10181/103884 objects degraded (9.800%); 981 KiB/s, 18 objects/s recovering 2023-06-03T19:54:40.029 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:54:39 smithi089 bash[75068]: audit 1685822078.9247944 mon.b (mon.1) 38 : audit 0 from='client.? 172.21.15.89:0/2372348630' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:54:40.030 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:54:39 smithi089 bash[78452]: cluster 1685822078.2889755 mgr.y (mgr.34104) 522 : cluster 0 pgmap v438: 89 pgs: 84 active+clean, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped; 2.1 GiB data, 8.5 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 566 op/s; 10181/103884 objects degraded (9.800%); 981 KiB/s, 18 objects/s recovering 2023-06-03T19:54:40.030 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:54:39 smithi089 bash[78452]: audit 1685822078.9247944 mon.b (mon.1) 38 : audit 0 from='client.? 172.21.15.89:0/2372348630' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:54:41.834 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:54:41 smithi164 bash[59462]: cluster 1685822080.2898424 mgr.y (mgr.34104) 523 : cluster 0 pgmap v439: 89 pgs: 84 active+clean, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped; 2.0 GiB data, 8.0 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 717 op/s; 10071/97323 objects degraded (10.348%); 981 KiB/s, 27 objects/s recovering 2023-06-03T19:54:41.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:54:41 smithi164 bash[59462]: audit 1685822081.512833 mon.c (mon.2) 267 : audit 0 from='client.? 172.21.15.89:0/3920506671' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:54:42.029 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:54:41 smithi089 bash[75068]: cluster 1685822080.2898424 mgr.y (mgr.34104) 523 : cluster 0 pgmap v439: 89 pgs: 84 active+clean, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped; 2.0 GiB data, 8.0 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 717 op/s; 10071/97323 objects degraded (10.348%); 981 KiB/s, 27 objects/s recovering 2023-06-03T19:54:42.029 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:54:41 smithi089 bash[75068]: audit 1685822081.512833 mon.c (mon.2) 267 : audit 0 from='client.? 172.21.15.89:0/3920506671' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:54:42.029 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:54:41 smithi089 bash[78452]: cluster 1685822080.2898424 mgr.y (mgr.34104) 523 : cluster 0 pgmap v439: 89 pgs: 84 active+clean, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped; 2.0 GiB data, 8.0 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 717 op/s; 10071/97323 objects degraded (10.348%); 981 KiB/s, 27 objects/s recovering 2023-06-03T19:54:42.030 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:54:41 smithi089 bash[78452]: audit 1685822081.512833 mon.c (mon.2) 267 : audit 0 from='client.? 172.21.15.89:0/3920506671' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:54:42.834 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:54:42 smithi164 bash[59462]: cluster 1685822082.398194 mon.a (mon.0) 482 : cluster 3 Health check update: Degraded data redundancy: 10071/97323 objects degraded (10.348%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-06-03T19:54:43.029 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:54:42 smithi089 bash[75068]: cluster 1685822082.398194 mon.a (mon.0) 482 : cluster 3 Health check update: Degraded data redundancy: 10071/97323 objects degraded (10.348%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-06-03T19:54:43.029 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:54:42 smithi089 bash[78452]: cluster 1685822082.398194 mon.a (mon.0) 482 : cluster 3 Health check update: Degraded data redundancy: 10071/97323 objects degraded (10.348%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-06-03T19:54:43.834 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:54:43 smithi164 bash[59462]: cluster 1685822082.2904432 mgr.y (mgr.34104) 524 : cluster 0 pgmap v440: 89 pgs: 84 active+clean, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped; 1.9 GiB data, 8.0 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 582 op/s; 10071/95454 objects degraded (10.551%); 411 KiB/s, 18 objects/s recovering 2023-06-03T19:54:44.028 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:54:43 smithi089 bash[78452]: cluster 1685822082.2904432 mgr.y (mgr.34104) 524 : cluster 0 pgmap v440: 89 pgs: 84 active+clean, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped; 1.9 GiB data, 8.0 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 582 op/s; 10071/95454 objects degraded (10.551%); 411 KiB/s, 18 objects/s recovering 2023-06-03T19:54:44.029 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:54:43 smithi089 bash[75068]: cluster 1685822082.2904432 mgr.y (mgr.34104) 524 : cluster 0 pgmap v440: 89 pgs: 84 active+clean, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped; 1.9 GiB data, 8.0 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 582 op/s; 10071/95454 objects degraded (10.551%); 411 KiB/s, 18 objects/s recovering 2023-06-03T19:54:44.834 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:54:44 smithi164 bash[59462]: audit 1685822084.1049705 mon.c (mon.2) 268 : audit 0 from='client.? 172.21.15.89:0/471740441' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:54:45.029 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:54:44 smithi089 bash[75068]: audit 1685822084.1049705 mon.c (mon.2) 268 : audit 0 from='client.? 172.21.15.89:0/471740441' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:54:45.029 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:54:44 smithi089 bash[78452]: audit 1685822084.1049705 mon.c (mon.2) 268 : audit 0 from='client.? 172.21.15.89:0/471740441' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:54:45.834 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:54:45 smithi164 bash[59462]: cluster 1685822084.291223 mgr.y (mgr.34104) 525 : cluster 0 pgmap v441: 89 pgs: 84 active+clean, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped; 1.8 GiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 695 op/s; 10071/89706 objects degraded (11.227%); 411 KiB/s, 18 objects/s recovering 2023-06-03T19:54:46.029 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:54:45 smithi089 bash[75068]: cluster 1685822084.291223 mgr.y (mgr.34104) 525 : cluster 0 pgmap v441: 89 pgs: 84 active+clean, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped; 1.8 GiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 695 op/s; 10071/89706 objects degraded (11.227%); 411 KiB/s, 18 objects/s recovering 2023-06-03T19:54:46.029 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:54:45 smithi089 bash[78452]: cluster 1685822084.291223 mgr.y (mgr.34104) 525 : cluster 0 pgmap v441: 89 pgs: 84 active+clean, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped; 1.8 GiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 695 op/s; 10071/89706 objects degraded (11.227%); 411 KiB/s, 18 objects/s recovering 2023-06-03T19:54:46.529 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 19:54:46 smithi089 bash[71163]: ::ffff:172.21.15.164 - - [03/Jun/2023:19:54:46] "GET /metrics HTTP/1.1" 200 212846 "" "Prometheus/2.33.4" 2023-06-03T19:54:48.014 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:54:47 smithi089 bash[75068]: cluster 1685822086.2919116 mgr.y (mgr.34104) 526 : cluster 0 pgmap v442: 89 pgs: 84 active+clean, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped; 1.7 GiB data, 7.4 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 665 op/s; 9962/85395 objects degraded (11.666%); 411 KiB/s, 27 objects/s recovering 2023-06-03T19:54:48.014 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:54:47 smithi089 bash[75068]: audit 1685822086.7192938 mon.c (mon.2) 269 : audit 0 from='client.? 172.21.15.89:0/4168129951' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:54:48.015 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:54:47 smithi089 bash[75068]: cluster 1685822087.3992212 mon.a (mon.0) 483 : cluster 3 Health check update: Degraded data redundancy: 9962/85395 objects degraded (11.666%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-06-03T19:54:48.015 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:54:47 smithi089 bash[78452]: cluster 1685822086.2919116 mgr.y (mgr.34104) 526 : cluster 0 pgmap v442: 89 pgs: 84 active+clean, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped; 1.7 GiB data, 7.4 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 665 op/s; 9962/85395 objects degraded (11.666%); 411 KiB/s, 27 objects/s recovering 2023-06-03T19:54:48.015 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:54:47 smithi089 bash[78452]: audit 1685822086.7192938 mon.c (mon.2) 269 : audit 0 from='client.? 172.21.15.89:0/4168129951' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:54:48.015 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:54:47 smithi089 bash[78452]: cluster 1685822087.3992212 mon.a (mon.0) 483 : cluster 3 Health check update: Degraded data redundancy: 9962/85395 objects degraded (11.666%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-06-03T19:54:48.084 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:54:47 smithi164 bash[59462]: cluster 1685822086.2919116 mgr.y (mgr.34104) 526 : cluster 0 pgmap v442: 89 pgs: 84 active+clean, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped; 1.7 GiB data, 7.4 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 665 op/s; 9962/85395 objects degraded (11.666%); 411 KiB/s, 27 objects/s recovering 2023-06-03T19:54:48.084 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:54:47 smithi164 bash[59462]: audit 1685822086.7192938 mon.c (mon.2) 269 : audit 0 from='client.? 172.21.15.89:0/4168129951' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:54:48.085 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:54:47 smithi164 bash[59462]: cluster 1685822087.3992212 mon.a (mon.0) 483 : cluster 3 Health check update: Degraded data redundancy: 9962/85395 objects degraded (11.666%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-06-03T19:54:48.279 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 19:54:48 smithi089 bash[71163]: debug 2023-06-03T19:54:48.005+0000 7fe48642e700 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2023-06-03T19:54:49.029 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:54:48 smithi089 bash[75068]: audit 1685822088.0093343 mon.c (mon.2) 270 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-06-03T19:54:49.029 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:54:48 smithi089 bash[78452]: audit 1685822088.0093343 mon.c (mon.2) 270 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-06-03T19:54:49.084 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:54:48 smithi164 bash[59462]: audit 1685822088.0093343 mon.c (mon.2) 270 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-06-03T19:54:50.029 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:54:49 smithi089 bash[75068]: audit 1685822088.009586 mgr.y (mgr.34104) 527 : audit 0 from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-06-03T19:54:50.029 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:54:49 smithi089 bash[75068]: cephadm 1685822088.0116198 mgr.y (mgr.34104) 528 : cephadm 1 Upgrade: unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2023-06-03T19:54:50.029 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:54:49 smithi089 bash[75068]: cluster 1685822088.2925482 mgr.y (mgr.34104) 529 : cluster 0 pgmap v443: 89 pgs: 84 active+clean, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped; 1.7 GiB data, 7.3 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 581 op/s; 9962/84186 objects degraded (11.833%); 0 B/s, 18 objects/s recovering 2023-06-03T19:54:50.030 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:54:49 smithi089 bash[75068]: audit 1685822089.3077376 mon.b (mon.1) 39 : audit 0 from='client.? 172.21.15.89:0/724520745' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:54:50.030 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:54:49 smithi089 bash[78452]: audit 1685822088.009586 mgr.y (mgr.34104) 527 : audit 0 from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-06-03T19:54:50.030 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:54:49 smithi089 bash[78452]: cephadm 1685822088.0116198 mgr.y (mgr.34104) 528 : cephadm 1 Upgrade: unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2023-06-03T19:54:50.031 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:54:49 smithi089 bash[78452]: cluster 1685822088.2925482 mgr.y (mgr.34104) 529 : cluster 0 pgmap v443: 89 pgs: 84 active+clean, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped; 1.7 GiB data, 7.3 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 581 op/s; 9962/84186 objects degraded (11.833%); 0 B/s, 18 objects/s recovering 2023-06-03T19:54:50.031 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:54:49 smithi089 bash[78452]: audit 1685822089.3077376 mon.b (mon.1) 39 : audit 0 from='client.? 172.21.15.89:0/724520745' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:54:50.084 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:54:49 smithi164 bash[59462]: audit 1685822088.009586 mgr.y (mgr.34104) 527 : audit 0 from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-06-03T19:54:50.084 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:54:49 smithi164 bash[59462]: cephadm 1685822088.0116198 mgr.y (mgr.34104) 528 : cephadm 1 Upgrade: unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2023-06-03T19:54:50.085 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:54:49 smithi164 bash[59462]: cluster 1685822088.2925482 mgr.y (mgr.34104) 529 : cluster 0 pgmap v443: 89 pgs: 84 active+clean, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped; 1.7 GiB data, 7.3 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 581 op/s; 9962/84186 objects degraded (11.833%); 0 B/s, 18 objects/s recovering 2023-06-03T19:54:50.085 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:54:49 smithi164 bash[59462]: audit 1685822089.3077376 mon.b (mon.1) 39 : audit 0 from='client.? 172.21.15.89:0/724520745' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:54:52.029 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:54:51 smithi089 bash[75068]: cluster 1685822090.2934182 mgr.y (mgr.34104) 530 : cluster 0 pgmap v444: 89 pgs: 84 active+clean, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped; 1.6 GiB data, 6.9 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 730 op/s; 9849/77562 objects degraded (12.698%); 0 B/s, 27 objects/s recovering 2023-06-03T19:54:52.029 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:54:51 smithi089 bash[78452]: cluster 1685822090.2934182 mgr.y (mgr.34104) 530 : cluster 0 pgmap v444: 89 pgs: 84 active+clean, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped; 1.6 GiB data, 6.9 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 730 op/s; 9849/77562 objects degraded (12.698%); 0 B/s, 27 objects/s recovering 2023-06-03T19:54:52.084 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:54:51 smithi164 bash[59462]: cluster 1685822090.2934182 mgr.y (mgr.34104) 530 : cluster 0 pgmap v444: 89 pgs: 84 active+clean, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped; 1.6 GiB data, 6.9 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 730 op/s; 9849/77562 objects degraded (12.698%); 0 B/s, 27 objects/s recovering 2023-06-03T19:54:53.029 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:54:52 smithi089 bash[75068]: audit 1685822091.9133725 mon.c (mon.2) 271 : audit 0 from='client.? 172.21.15.89:0/3720984212' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:54:53.029 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:54:52 smithi089 bash[75068]: audit 1685822092.084797 mon.c (mon.2) 272 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T19:54:53.029 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:54:52 smithi089 bash[75068]: cluster 1685822092.405016 mon.a (mon.0) 484 : cluster 3 Health check update: Degraded data redundancy: 9849/77562 objects degraded (12.698%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-06-03T19:54:53.030 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:54:52 smithi089 bash[78452]: audit 1685822091.9133725 mon.c (mon.2) 271 : audit 0 from='client.? 172.21.15.89:0/3720984212' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:54:53.030 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:54:52 smithi089 bash[78452]: audit 1685822092.084797 mon.c (mon.2) 272 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T19:54:53.030 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:54:52 smithi089 bash[78452]: cluster 1685822092.405016 mon.a (mon.0) 484 : cluster 3 Health check update: Degraded data redundancy: 9849/77562 objects degraded (12.698%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-06-03T19:54:53.084 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:54:52 smithi164 bash[59462]: audit 1685822091.9133725 mon.c (mon.2) 271 : audit 0 from='client.? 172.21.15.89:0/3720984212' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:54:53.084 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:54:52 smithi164 bash[59462]: audit 1685822092.084797 mon.c (mon.2) 272 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T19:54:53.085 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:54:52 smithi164 bash[59462]: cluster 1685822092.405016 mon.a (mon.0) 484 : cluster 3 Health check update: Degraded data redundancy: 9849/77562 objects degraded (12.698%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-06-03T19:54:54.028 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:54:53 smithi089 bash[78452]: cluster 1685822092.2940364 mgr.y (mgr.34104) 531 : cluster 0 pgmap v445: 89 pgs: 84 active+clean, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped; 1.5 GiB data, 6.8 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 599 op/s; 9849/75747 objects degraded (13.002%); 0 B/s, 18 objects/s recovering 2023-06-03T19:54:54.029 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:54:53 smithi089 bash[75068]: cluster 1685822092.2940364 mgr.y (mgr.34104) 531 : cluster 0 pgmap v445: 89 pgs: 84 active+clean, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped; 1.5 GiB data, 6.8 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 599 op/s; 9849/75747 objects degraded (13.002%); 0 B/s, 18 objects/s recovering 2023-06-03T19:54:54.084 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:54:53 smithi164 bash[59462]: cluster 1685822092.2940364 mgr.y (mgr.34104) 531 : cluster 0 pgmap v445: 89 pgs: 84 active+clean, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped; 1.5 GiB data, 6.8 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 599 op/s; 9849/75747 objects degraded (13.002%); 0 B/s, 18 objects/s recovering 2023-06-03T19:54:55.029 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:54:54 smithi089 bash[78452]: audit 1685822094.490983 mon.c (mon.2) 273 : audit 0 from='client.? 172.21.15.89:0/4040112368' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:54:55.029 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:54:54 smithi089 bash[75068]: audit 1685822094.490983 mon.c (mon.2) 273 : audit 0 from='client.? 172.21.15.89:0/4040112368' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:54:55.084 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:54:54 smithi164 bash[59462]: audit 1685822094.490983 mon.c (mon.2) 273 : audit 0 from='client.? 172.21.15.89:0/4040112368' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:54:56.029 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:54:55 smithi089 bash[75068]: cluster 1685822094.2958083 mgr.y (mgr.34104) 532 : cluster 0 pgmap v446: 89 pgs: 84 active+clean, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped; 1.4 GiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 695 op/s; 9849/70389 objects degraded (13.992%); 0 B/s, 18 objects/s recovering 2023-06-03T19:54:56.029 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:54:55 smithi089 bash[78452]: cluster 1685822094.2958083 mgr.y (mgr.34104) 532 : cluster 0 pgmap v446: 89 pgs: 84 active+clean, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped; 1.4 GiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 695 op/s; 9849/70389 objects degraded (13.992%); 0 B/s, 18 objects/s recovering 2023-06-03T19:54:56.084 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:54:55 smithi164 bash[59462]: cluster 1685822094.2958083 mgr.y (mgr.34104) 532 : cluster 0 pgmap v446: 89 pgs: 84 active+clean, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped; 1.4 GiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 695 op/s; 9849/70389 objects degraded (13.992%); 0 B/s, 18 objects/s recovering 2023-06-03T19:54:56.529 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 19:54:56 smithi089 bash[71163]: ::ffff:172.21.15.164 - - [03/Jun/2023:19:54:56] "GET /metrics HTTP/1.1" 200 212810 "" "Prometheus/2.33.4" 2023-06-03T19:54:58.029 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:54:57 smithi089 bash[75068]: cluster 1685822096.2964911 mgr.y (mgr.34104) 533 : cluster 0 pgmap v447: 89 pgs: 84 active+clean, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped; 1.3 GiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 649 op/s; 9737/66297 objects degraded (14.687%); 0 B/s, 27 objects/s recovering 2023-06-03T19:54:58.029 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:54:57 smithi089 bash[75068]: audit 1685822097.0698147 mon.b (mon.1) 40 : audit 0 from='client.? 172.21.15.89:0/961819167' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:54:58.029 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:54:57 smithi089 bash[75068]: cluster 1685822097.4062304 mon.a (mon.0) 485 : cluster 3 Health check update: Degraded data redundancy: 9737/66297 objects degraded (14.687%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-06-03T19:54:58.029 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:54:57 smithi089 bash[78452]: cluster 1685822096.2964911 mgr.y (mgr.34104) 533 : cluster 0 pgmap v447: 89 pgs: 84 active+clean, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped; 1.3 GiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 649 op/s; 9737/66297 objects degraded (14.687%); 0 B/s, 27 objects/s recovering 2023-06-03T19:54:58.029 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:54:57 smithi089 bash[78452]: audit 1685822097.0698147 mon.b (mon.1) 40 : audit 0 from='client.? 172.21.15.89:0/961819167' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:54:58.030 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:54:57 smithi089 bash[78452]: cluster 1685822097.4062304 mon.a (mon.0) 485 : cluster 3 Health check update: Degraded data redundancy: 9737/66297 objects degraded (14.687%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-06-03T19:54:58.084 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:54:57 smithi164 bash[59462]: cluster 1685822096.2964911 mgr.y (mgr.34104) 533 : cluster 0 pgmap v447: 89 pgs: 84 active+clean, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped; 1.3 GiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 649 op/s; 9737/66297 objects degraded (14.687%); 0 B/s, 27 objects/s recovering 2023-06-03T19:54:58.084 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:54:57 smithi164 bash[59462]: audit 1685822097.0698147 mon.b (mon.1) 40 : audit 0 from='client.? 172.21.15.89:0/961819167' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:54:58.085 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:54:57 smithi164 bash[59462]: cluster 1685822097.4062304 mon.a (mon.0) 485 : cluster 3 Health check update: Degraded data redundancy: 9737/66297 objects degraded (14.687%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-06-03T19:55:00.029 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:54:59 smithi089 bash[75068]: cluster 1685822098.2971036 mgr.y (mgr.34104) 534 : cluster 0 pgmap v448: 89 pgs: 84 active+clean, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped; 1.3 GiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 564 op/s; 9737/65064 objects degraded (14.965%); 0 B/s, 18 objects/s recovering 2023-06-03T19:55:00.029 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:54:59 smithi089 bash[75068]: audit 1685822099.6450691 mon.b (mon.1) 41 : audit 0 from='client.? 172.21.15.89:0/2130612525' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:55:00.029 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:54:59 smithi089 bash[78452]: cluster 1685822098.2971036 mgr.y (mgr.34104) 534 : cluster 0 pgmap v448: 89 pgs: 84 active+clean, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped; 1.3 GiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 564 op/s; 9737/65064 objects degraded (14.965%); 0 B/s, 18 objects/s recovering 2023-06-03T19:55:00.030 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:54:59 smithi089 bash[78452]: audit 1685822099.6450691 mon.b (mon.1) 41 : audit 0 from='client.? 172.21.15.89:0/2130612525' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:55:00.084 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:54:59 smithi164 bash[59462]: cluster 1685822098.2971036 mgr.y (mgr.34104) 534 : cluster 0 pgmap v448: 89 pgs: 84 active+clean, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped; 1.3 GiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 564 op/s; 9737/65064 objects degraded (14.965%); 0 B/s, 18 objects/s recovering 2023-06-03T19:55:00.085 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:54:59 smithi164 bash[59462]: audit 1685822099.6450691 mon.b (mon.1) 41 : audit 0 from='client.? 172.21.15.89:0/2130612525' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:55:01.779 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:55:01 smithi089 bash[75068]: cluster 1685822100.3013759 mgr.y (mgr.34104) 535 : cluster 0 pgmap v449: 89 pgs: 84 active+clean, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped; 1.2 GiB data, 5.7 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 717 op/s; 9623/58326 objects degraded (16.499%); 0 B/s, 28 objects/s recovering 2023-06-03T19:55:01.779 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:55:01 smithi089 bash[78452]: cluster 1685822100.3013759 mgr.y (mgr.34104) 535 : cluster 0 pgmap v449: 89 pgs: 84 active+clean, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped; 1.2 GiB data, 5.7 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 717 op/s; 9623/58326 objects degraded (16.499%); 0 B/s, 28 objects/s recovering 2023-06-03T19:55:02.084 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:55:01 smithi164 bash[59462]: cluster 1685822100.3013759 mgr.y (mgr.34104) 535 : cluster 0 pgmap v449: 89 pgs: 84 active+clean, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped; 1.2 GiB data, 5.7 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 717 op/s; 9623/58326 objects degraded (16.499%); 0 B/s, 28 objects/s recovering 2023-06-03T19:55:03.039 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 19:55:03 smithi089 bash[71163]: debug 2023-06-03T19:55:03.021+0000 7fe48642e700 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2023-06-03T19:55:03.039 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:55:02 smithi089 bash[75068]: audit 1685822102.2320168 mon.a (mon.0) 486 : audit 0 from='client.? 172.21.15.89:0/3569372997' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:55:03.039 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:55:02 smithi089 bash[75068]: cluster 1685822102.407441 mon.a (mon.0) 487 : cluster 3 Health check update: Degraded data redundancy: 9623/58326 objects degraded (16.499%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-06-03T19:55:03.040 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:55:02 smithi089 bash[78452]: audit 1685822102.2320168 mon.a (mon.0) 486 : audit 0 from='client.? 172.21.15.89:0/3569372997' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:55:03.040 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:55:02 smithi089 bash[78452]: cluster 1685822102.407441 mon.a (mon.0) 487 : cluster 3 Health check update: Degraded data redundancy: 9623/58326 objects degraded (16.499%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-06-03T19:55:03.084 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:55:02 smithi164 bash[59462]: audit 1685822102.2320168 mon.a (mon.0) 486 : audit 0 from='client.? 172.21.15.89:0/3569372997' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:55:03.085 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:55:02 smithi164 bash[59462]: cluster 1685822102.407441 mon.a (mon.0) 487 : cluster 3 Health check update: Degraded data redundancy: 9623/58326 objects degraded (16.499%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-06-03T19:55:04.029 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:55:03 smithi089 bash[75068]: cluster 1685822102.3020267 mgr.y (mgr.34104) 536 : cluster 0 pgmap v450: 89 pgs: 84 active+clean, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped; 1.1 GiB data, 5.7 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 583 op/s; 9623/56535 objects degraded (17.021%); 0 B/s, 18 objects/s recovering 2023-06-03T19:55:04.029 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:55:03 smithi089 bash[75068]: audit 1685822103.0252695 mon.c (mon.2) 274 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-06-03T19:55:04.029 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:55:03 smithi089 bash[78452]: cluster 1685822102.3020267 mgr.y (mgr.34104) 536 : cluster 0 pgmap v450: 89 pgs: 84 active+clean, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped; 1.1 GiB data, 5.7 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 583 op/s; 9623/56535 objects degraded (17.021%); 0 B/s, 18 objects/s recovering 2023-06-03T19:55:04.029 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:55:03 smithi089 bash[78452]: audit 1685822103.0252695 mon.c (mon.2) 274 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-06-03T19:55:04.084 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:55:03 smithi164 bash[59462]: cluster 1685822102.3020267 mgr.y (mgr.34104) 536 : cluster 0 pgmap v450: 89 pgs: 84 active+clean, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped; 1.1 GiB data, 5.7 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 583 op/s; 9623/56535 objects degraded (17.021%); 0 B/s, 18 objects/s recovering 2023-06-03T19:55:04.084 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:55:03 smithi164 bash[59462]: audit 1685822103.0252695 mon.c (mon.2) 274 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-06-03T19:55:05.029 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:55:04 smithi089 bash[75068]: audit 1685822103.025508 mgr.y (mgr.34104) 537 : audit 0 from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-06-03T19:55:05.029 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:55:04 smithi089 bash[75068]: cephadm 1685822103.026568 mgr.y (mgr.34104) 538 : cephadm 1 Upgrade: unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2023-06-03T19:55:05.029 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:55:04 smithi089 bash[78452]: audit 1685822103.025508 mgr.y (mgr.34104) 537 : audit 0 from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-06-03T19:55:05.029 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:55:04 smithi089 bash[78452]: cephadm 1685822103.026568 mgr.y (mgr.34104) 538 : cephadm 1 Upgrade: unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2023-06-03T19:55:05.084 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:55:04 smithi164 bash[59462]: audit 1685822103.025508 mgr.y (mgr.34104) 537 : audit 0 from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-06-03T19:55:05.085 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:55:04 smithi164 bash[59462]: cephadm 1685822103.026568 mgr.y (mgr.34104) 538 : cephadm 1 Upgrade: unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2023-06-03T19:55:06.029 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:55:05 smithi089 bash[78452]: cluster 1685822104.3028128 mgr.y (mgr.34104) 539 : cluster 0 pgmap v451: 89 pgs: 84 active+clean, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped; 1.0 GiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 683 op/s; 9623/51111 objects degraded (18.828%); 0 B/s, 18 objects/s recovering 2023-06-03T19:55:06.029 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:55:05 smithi089 bash[78452]: audit 1685822104.9051063 mon.c (mon.2) 275 : audit 0 from='client.? 172.21.15.89:0/239224781' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:55:06.029 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:55:05 smithi089 bash[75068]: cluster 1685822104.3028128 mgr.y (mgr.34104) 539 : cluster 0 pgmap v451: 89 pgs: 84 active+clean, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped; 1.0 GiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 683 op/s; 9623/51111 objects degraded (18.828%); 0 B/s, 18 objects/s recovering 2023-06-03T19:55:06.030 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:55:05 smithi089 bash[75068]: audit 1685822104.9051063 mon.c (mon.2) 275 : audit 0 from='client.? 172.21.15.89:0/239224781' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:55:06.084 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:55:05 smithi164 bash[59462]: cluster 1685822104.3028128 mgr.y (mgr.34104) 539 : cluster 0 pgmap v451: 89 pgs: 84 active+clean, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped; 1.0 GiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 683 op/s; 9623/51111 objects degraded (18.828%); 0 B/s, 18 objects/s recovering 2023-06-03T19:55:06.085 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:55:05 smithi164 bash[59462]: audit 1685822104.9051063 mon.c (mon.2) 275 : audit 0 from='client.? 172.21.15.89:0/239224781' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:55:06.528 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 19:55:06 smithi089 bash[71163]: ::ffff:172.21.15.164 - - [03/Jun/2023:19:55:06] "GET /metrics HTTP/1.1" 200 212810 "" "Prometheus/2.33.4" 2023-06-03T19:55:08.029 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:55:07 smithi089 bash[78452]: cluster 1685822106.3034973 mgr.y (mgr.34104) 540 : cluster 0 pgmap v452: 89 pgs: 84 active+clean, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped; 980 MiB data, 5.1 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 646 op/s; 9515/47109 objects degraded (20.198%); 0 B/s, 27 objects/s recovering 2023-06-03T19:55:08.029 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:55:07 smithi089 bash[78452]: audit 1685822107.0854828 mon.c (mon.2) 276 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T19:55:08.029 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:55:07 smithi089 bash[78452]: cluster 1685822107.4084103 mon.a (mon.0) 488 : cluster 3 Health check update: Degraded data redundancy: 9515/47109 objects degraded (20.198%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-06-03T19:55:08.029 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:55:07 smithi089 bash[78452]: audit 1685822107.4786515 mon.c (mon.2) 277 : audit 0 from='client.? 172.21.15.89:0/3350586668' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:55:08.030 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:55:07 smithi089 bash[75068]: cluster 1685822106.3034973 mgr.y (mgr.34104) 540 : cluster 0 pgmap v452: 89 pgs: 84 active+clean, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped; 980 MiB data, 5.1 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 646 op/s; 9515/47109 objects degraded (20.198%); 0 B/s, 27 objects/s recovering 2023-06-03T19:55:08.030 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:55:07 smithi089 bash[75068]: audit 1685822107.0854828 mon.c (mon.2) 276 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T19:55:08.030 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:55:07 smithi089 bash[75068]: cluster 1685822107.4084103 mon.a (mon.0) 488 : cluster 3 Health check update: Degraded data redundancy: 9515/47109 objects degraded (20.198%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-06-03T19:55:08.030 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:55:07 smithi089 bash[75068]: audit 1685822107.4786515 mon.c (mon.2) 277 : audit 0 from='client.? 172.21.15.89:0/3350586668' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:55:08.084 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:55:07 smithi164 bash[59462]: cluster 1685822106.3034973 mgr.y (mgr.34104) 540 : cluster 0 pgmap v452: 89 pgs: 84 active+clean, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped; 980 MiB data, 5.1 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 646 op/s; 9515/47109 objects degraded (20.198%); 0 B/s, 27 objects/s recovering 2023-06-03T19:55:08.084 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:55:07 smithi164 bash[59462]: audit 1685822107.0854828 mon.c (mon.2) 276 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T19:55:08.085 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:55:07 smithi164 bash[59462]: cluster 1685822107.4084103 mon.a (mon.0) 488 : cluster 3 Health check update: Degraded data redundancy: 9515/47109 objects degraded (20.198%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-06-03T19:55:08.085 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:55:07 smithi164 bash[59462]: audit 1685822107.4786515 mon.c (mon.2) 277 : audit 0 from='client.? 172.21.15.89:0/3350586668' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:55:10.028 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:55:09 smithi089 bash[75068]: cluster 1685822108.3039684 mgr.y (mgr.34104) 541 : cluster 0 pgmap v453: 89 pgs: 84 active+clean, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped; 956 MiB data, 5.0 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 565 op/s; 9515/45915 objects degraded (20.723%); 0 B/s, 18 objects/s recovering 2023-06-03T19:55:10.029 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:55:09 smithi089 bash[78452]: cluster 1685822108.3039684 mgr.y (mgr.34104) 541 : cluster 0 pgmap v453: 89 pgs: 84 active+clean, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped; 956 MiB data, 5.0 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 565 op/s; 9515/45915 objects degraded (20.723%); 0 B/s, 18 objects/s recovering 2023-06-03T19:55:10.084 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:55:09 smithi164 bash[59462]: cluster 1685822108.3039684 mgr.y (mgr.34104) 541 : cluster 0 pgmap v453: 89 pgs: 84 active+clean, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped; 956 MiB data, 5.0 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 565 op/s; 9515/45915 objects degraded (20.723%); 0 B/s, 18 objects/s recovering 2023-06-03T19:55:11.028 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:55:10 smithi089 bash[78452]: audit 1685822110.0507612 mon.a (mon.0) 489 : audit 0 from='client.? 172.21.15.89:0/2978633287' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:55:11.029 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:55:10 smithi089 bash[75068]: audit 1685822110.0507612 mon.a (mon.0) 489 : audit 0 from='client.? 172.21.15.89:0/2978633287' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:55:11.084 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:55:10 smithi164 bash[59462]: audit 1685822110.0507612 mon.a (mon.0) 489 : audit 0 from='client.? 172.21.15.89:0/2978633287' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:55:12.028 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:55:11 smithi089 bash[75068]: cluster 1685822110.304871 mgr.y (mgr.34104) 542 : cluster 0 pgmap v454: 89 pgs: 84 active+clean, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped; 814 MiB data, 4.6 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 719 op/s; 9400/39132 objects degraded (24.021%); 0 B/s, 28 objects/s recovering 2023-06-03T19:55:12.029 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:55:11 smithi089 bash[78452]: cluster 1685822110.304871 mgr.y (mgr.34104) 542 : cluster 0 pgmap v454: 89 pgs: 84 active+clean, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped; 814 MiB data, 4.6 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 719 op/s; 9400/39132 objects degraded (24.021%); 0 B/s, 28 objects/s recovering 2023-06-03T19:55:12.084 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:55:11 smithi164 bash[59462]: cluster 1685822110.304871 mgr.y (mgr.34104) 542 : cluster 0 pgmap v454: 89 pgs: 84 active+clean, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped; 814 MiB data, 4.6 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 719 op/s; 9400/39132 objects degraded (24.021%); 0 B/s, 28 objects/s recovering 2023-06-03T19:55:13.029 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:55:12 smithi089 bash[75068]: cluster 1685822112.4103622 mon.a (mon.0) 490 : cluster 3 Health check update: Degraded data redundancy: 9400/39132 objects degraded (24.021%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-06-03T19:55:13.029 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:55:12 smithi089 bash[75068]: audit 1685822112.6628704 mon.a (mon.0) 491 : audit 0 from='client.? 172.21.15.89:0/933606797' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:55:13.029 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:55:12 smithi089 bash[78452]: cluster 1685822112.4103622 mon.a (mon.0) 490 : cluster 3 Health check update: Degraded data redundancy: 9400/39132 objects degraded (24.021%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-06-03T19:55:13.030 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:55:12 smithi089 bash[78452]: audit 1685822112.6628704 mon.a (mon.0) 491 : audit 0 from='client.? 172.21.15.89:0/933606797' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:55:13.084 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:55:12 smithi164 bash[59462]: cluster 1685822112.4103622 mon.a (mon.0) 490 : cluster 3 Health check update: Degraded data redundancy: 9400/39132 objects degraded (24.021%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-06-03T19:55:13.085 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:55:12 smithi164 bash[59462]: audit 1685822112.6628704 mon.a (mon.0) 491 : audit 0 from='client.? 172.21.15.89:0/933606797' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:55:14.029 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:55:13 smithi089 bash[78452]: cluster 1685822112.3055906 mgr.y (mgr.34104) 543 : cluster 0 pgmap v455: 89 pgs: 84 active+clean, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped; 779 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 579 op/s; 9400/37443 objects degraded (25.105%); 0 B/s, 18 objects/s recovering 2023-06-03T19:55:14.029 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:55:13 smithi089 bash[75068]: cluster 1685822112.3055906 mgr.y (mgr.34104) 543 : cluster 0 pgmap v455: 89 pgs: 84 active+clean, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped; 779 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 579 op/s; 9400/37443 objects degraded (25.105%); 0 B/s, 18 objects/s recovering 2023-06-03T19:55:14.084 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:55:13 smithi164 bash[59462]: cluster 1685822112.3055906 mgr.y (mgr.34104) 543 : cluster 0 pgmap v455: 89 pgs: 84 active+clean, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped; 779 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 579 op/s; 9400/37443 objects degraded (25.105%); 0 B/s, 18 objects/s recovering 2023-06-03T19:55:16.029 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:55:15 smithi089 bash[75068]: cluster 1685822114.306417 mgr.y (mgr.34104) 544 : cluster 0 pgmap v456: 89 pgs: 84 active+clean, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped; 662 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 686 op/s; 9400/31812 objects degraded (29.549%); 0 B/s, 18 objects/s recovering 2023-06-03T19:55:16.029 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:55:15 smithi089 bash[75068]: audit 1685822115.2769501 mon.c (mon.2) 278 : audit 0 from='client.? 172.21.15.89:0/2558652728' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:55:16.030 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:55:15 smithi089 bash[78452]: cluster 1685822114.306417 mgr.y (mgr.34104) 544 : cluster 0 pgmap v456: 89 pgs: 84 active+clean, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped; 662 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 686 op/s; 9400/31812 objects degraded (29.549%); 0 B/s, 18 objects/s recovering 2023-06-03T19:55:16.030 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:55:15 smithi089 bash[78452]: audit 1685822115.2769501 mon.c (mon.2) 278 : audit 0 from='client.? 172.21.15.89:0/2558652728' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:55:16.084 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:55:15 smithi164 bash[59462]: cluster 1685822114.306417 mgr.y (mgr.34104) 544 : cluster 0 pgmap v456: 89 pgs: 84 active+clean, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped; 662 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 686 op/s; 9400/31812 objects degraded (29.549%); 0 B/s, 18 objects/s recovering 2023-06-03T19:55:16.085 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:55:15 smithi164 bash[59462]: audit 1685822115.2769501 mon.c (mon.2) 278 : audit 0 from='client.? 172.21.15.89:0/2558652728' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:55:16.528 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 19:55:16 smithi089 bash[71163]: ::ffff:172.21.15.164 - - [03/Jun/2023:19:55:16] "GET /metrics HTTP/1.1" 200 212807 "" "Prometheus/2.33.4" 2023-06-03T19:55:18.029 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:55:17 smithi089 bash[75068]: cluster 1685822116.3070612 mgr.y (mgr.34104) 545 : cluster 0 pgmap v457: 89 pgs: 84 active+clean, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped; 573 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 654 op/s; 9288/27558 objects degraded (33.703%); 0 B/s, 27 objects/s recovering 2023-06-03T19:55:18.029 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:55:17 smithi089 bash[75068]: cluster 1685822117.4118955 mon.a (mon.0) 492 : cluster 3 Health check update: Degraded data redundancy: 9288/27558 objects degraded (33.703%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-06-03T19:55:18.029 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:55:17 smithi089 bash[78452]: cluster 1685822116.3070612 mgr.y (mgr.34104) 545 : cluster 0 pgmap v457: 89 pgs: 84 active+clean, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped; 573 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 654 op/s; 9288/27558 objects degraded (33.703%); 0 B/s, 27 objects/s recovering 2023-06-03T19:55:18.030 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:55:17 smithi089 bash[78452]: cluster 1685822117.4118955 mon.a (mon.0) 492 : cluster 3 Health check update: Degraded data redundancy: 9288/27558 objects degraded (33.703%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-06-03T19:55:18.084 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:55:17 smithi164 bash[59462]: cluster 1685822116.3070612 mgr.y (mgr.34104) 545 : cluster 0 pgmap v457: 89 pgs: 84 active+clean, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped; 573 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 654 op/s; 9288/27558 objects degraded (33.703%); 0 B/s, 27 objects/s recovering 2023-06-03T19:55:18.084 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:55:17 smithi164 bash[59462]: cluster 1685822117.4118955 mon.a (mon.0) 492 : cluster 3 Health check update: Degraded data redundancy: 9288/27558 objects degraded (33.703%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-06-03T19:55:19.029 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:55:18 smithi089 bash[78452]: audit 1685822117.8501287 mon.a (mon.0) 493 : audit 0 from='client.? 172.21.15.89:0/1407036001' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:55:19.029 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:55:18 smithi089 bash[78452]: audit 1685822118.041503 mon.c (mon.2) 279 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-03T19:55:19.029 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:55:18 smithi089 bash[78452]: audit 1685822118.5214815 mon.a (mon.0) 494 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:55:19.030 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:55:18 smithi089 bash[78452]: audit 1685822118.5408745 mon.a (mon.0) 495 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:55:19.030 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:55:18 smithi089 bash[78452]: audit 1685822118.6226 mon.a (mon.0) 496 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:55:19.030 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:55:18 smithi089 bash[78452]: audit 1685822118.6466987 mon.a (mon.0) 497 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:55:19.030 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:55:18 smithi089 bash[75068]: audit 1685822117.8501287 mon.a (mon.0) 493 : audit 0 from='client.? 172.21.15.89:0/1407036001' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:55:19.031 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:55:18 smithi089 bash[75068]: audit 1685822118.041503 mon.c (mon.2) 279 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-03T19:55:19.031 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:55:18 smithi089 bash[75068]: audit 1685822118.5214815 mon.a (mon.0) 494 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:55:19.032 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:55:18 smithi089 bash[75068]: audit 1685822118.5408745 mon.a (mon.0) 495 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:55:19.032 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:55:18 smithi089 bash[75068]: audit 1685822118.6226 mon.a (mon.0) 496 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:55:19.032 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:55:18 smithi089 bash[75068]: audit 1685822118.6466987 mon.a (mon.0) 497 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:55:19.084 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:55:18 smithi164 bash[59462]: audit 1685822117.8501287 mon.a (mon.0) 493 : audit 0 from='client.? 172.21.15.89:0/1407036001' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:55:19.085 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:55:18 smithi164 bash[59462]: audit 1685822118.041503 mon.c (mon.2) 279 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-03T19:55:19.085 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:55:18 smithi164 bash[59462]: audit 1685822118.5214815 mon.a (mon.0) 494 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:55:19.085 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:55:18 smithi164 bash[59462]: audit 1685822118.5408745 mon.a (mon.0) 495 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:55:19.085 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:55:18 smithi164 bash[59462]: audit 1685822118.6226 mon.a (mon.0) 496 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:55:19.086 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:55:18 smithi164 bash[59462]: audit 1685822118.6466987 mon.a (mon.0) 497 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:55:20.029 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:55:19 smithi089 bash[75068]: cluster 1685822118.307673 mgr.y (mgr.34104) 546 : cluster 0 pgmap v458: 89 pgs: 84 active+clean, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped; 547 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 578 op/s; 9288/26292 objects degraded (35.326%); 0 B/s, 18 objects/s recovering 2023-06-03T19:55:20.029 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:55:19 smithi089 bash[78452]: cluster 1685822118.307673 mgr.y (mgr.34104) 546 : cluster 0 pgmap v458: 89 pgs: 84 active+clean, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped; 547 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 578 op/s; 9288/26292 objects degraded (35.326%); 0 B/s, 18 objects/s recovering 2023-06-03T19:55:20.084 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:55:19 smithi164 bash[59462]: cluster 1685822118.307673 mgr.y (mgr.34104) 546 : cluster 0 pgmap v458: 89 pgs: 84 active+clean, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped; 547 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 578 op/s; 9288/26292 objects degraded (35.326%); 0 B/s, 18 objects/s recovering 2023-06-03T19:55:21.029 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:55:20 smithi089 bash[78452]: audit 1685822120.4354868 mon.b (mon.1) 42 : audit 0 from='client.? 172.21.15.89:0/2386407278' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:55:21.029 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:55:20 smithi089 bash[75068]: audit 1685822120.4354868 mon.b (mon.1) 42 : audit 0 from='client.? 172.21.15.89:0/2386407278' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:55:21.084 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:55:20 smithi164 bash[59462]: audit 1685822120.4354868 mon.b (mon.1) 42 : audit 0 from='client.? 172.21.15.89:0/2386407278' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:55:22.029 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:55:21 smithi089 bash[78452]: cluster 1685822120.3087788 mgr.y (mgr.34104) 547 : cluster 0 pgmap v459: 89 pgs: 84 active+clean, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped; 411 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 725 op/s; 9172/19791 objects degraded (46.344%); 0 B/s, 28 objects/s recovering 2023-06-03T19:55:22.029 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:55:21 smithi089 bash[75068]: cluster 1685822120.3087788 mgr.y (mgr.34104) 547 : cluster 0 pgmap v459: 89 pgs: 84 active+clean, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped; 411 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 725 op/s; 9172/19791 objects degraded (46.344%); 0 B/s, 28 objects/s recovering 2023-06-03T19:55:22.084 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:55:21 smithi164 bash[59462]: cluster 1685822120.3087788 mgr.y (mgr.34104) 547 : cluster 0 pgmap v459: 89 pgs: 84 active+clean, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped; 411 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 725 op/s; 9172/19791 objects degraded (46.344%); 0 B/s, 28 objects/s recovering 2023-06-03T19:55:23.084 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:55:22 smithi164 bash[59462]: audit 1685822122.0855143 mon.c (mon.2) 280 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T19:55:23.084 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:55:22 smithi164 bash[59462]: cluster 1685822122.4139988 mon.a (mon.0) 498 : cluster 3 Health check update: Degraded data redundancy: 9172/19791 objects degraded (46.344%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-06-03T19:55:23.279 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:55:22 smithi089 bash[75068]: audit 1685822122.0855143 mon.c (mon.2) 280 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T19:55:23.279 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:55:22 smithi089 bash[75068]: cluster 1685822122.4139988 mon.a (mon.0) 498 : cluster 3 Health check update: Degraded data redundancy: 9172/19791 objects degraded (46.344%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-06-03T19:55:23.279 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:55:22 smithi089 bash[78452]: audit 1685822122.0855143 mon.c (mon.2) 280 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T19:55:23.279 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:55:22 smithi089 bash[78452]: cluster 1685822122.4139988 mon.a (mon.0) 498 : cluster 3 Health check update: Degraded data redundancy: 9172/19791 objects degraded (46.344%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-06-03T19:55:24.084 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:55:23 smithi164 bash[59462]: cluster 1685822122.3093872 mgr.y (mgr.34104) 548 : cluster 0 pgmap v460: 89 pgs: 84 active+clean, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped; 373 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 587 op/s; 9172/17973 objects degraded (51.032%); 0 B/s, 18 objects/s recovering 2023-06-03T19:55:24.084 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:55:23 smithi164 bash[59462]: audit 1685822123.0184305 mon.b (mon.1) 43 : audit 0 from='client.? 172.21.15.89:0/4099904226' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:55:24.279 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:55:23 smithi089 bash[78452]: cluster 1685822122.3093872 mgr.y (mgr.34104) 548 : cluster 0 pgmap v460: 89 pgs: 84 active+clean, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped; 373 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 587 op/s; 9172/17973 objects degraded (51.032%); 0 B/s, 18 objects/s recovering 2023-06-03T19:55:24.279 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:55:23 smithi089 bash[78452]: audit 1685822123.0184305 mon.b (mon.1) 43 : audit 0 from='client.? 172.21.15.89:0/4099904226' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:55:24.279 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:55:23 smithi089 bash[75068]: cluster 1685822122.3093872 mgr.y (mgr.34104) 548 : cluster 0 pgmap v460: 89 pgs: 84 active+clean, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped; 373 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 587 op/s; 9172/17973 objects degraded (51.032%); 0 B/s, 18 objects/s recovering 2023-06-03T19:55:24.279 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:55:23 smithi089 bash[75068]: audit 1685822123.0184305 mon.b (mon.1) 43 : audit 0 from='client.? 172.21.15.89:0/4099904226' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:55:26.084 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:55:25 smithi164 bash[59462]: cluster 1685822124.3101888 mgr.y (mgr.34104) 549 : cluster 0 pgmap v461: 89 pgs: 84 active+clean, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped; 273 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 674 op/s; 9172/13146 objects degraded (69.770%); 0 B/s, 18 objects/s recovering 2023-06-03T19:55:26.084 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:55:25 smithi164 bash[59462]: audit 1685822125.1763527 mon.a (mon.0) 499 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:55:26.085 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:55:25 smithi164 bash[59462]: audit 1685822125.1952941 mon.a (mon.0) 500 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:55:26.085 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:55:25 smithi164 bash[59462]: audit 1685822125.3955882 mon.a (mon.0) 501 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:55:26.085 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:55:25 smithi164 bash[59462]: audit 1685822125.4122493 mon.a (mon.0) 502 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:55:26.085 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:55:25 smithi164 bash[59462]: audit 1685822125.6229522 mon.c (mon.2) 281 : audit 0 from='client.? 172.21.15.89:0/3732058306' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:55:26.186 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:55:25 smithi089 bash[75068]: cluster 1685822124.3101888 mgr.y (mgr.34104) 549 : cluster 0 pgmap v461: 89 pgs: 84 active+clean, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped; 273 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 674 op/s; 9172/13146 objects degraded (69.770%); 0 B/s, 18 objects/s recovering 2023-06-03T19:55:26.187 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:55:25 smithi089 bash[75068]: audit 1685822125.1763527 mon.a (mon.0) 499 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:55:26.187 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:55:25 smithi089 bash[75068]: audit 1685822125.1952941 mon.a (mon.0) 500 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:55:26.187 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:55:25 smithi089 bash[75068]: audit 1685822125.3955882 mon.a (mon.0) 501 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:55:26.187 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:55:25 smithi089 bash[75068]: audit 1685822125.4122493 mon.a (mon.0) 502 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:55:26.187 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:55:25 smithi089 bash[75068]: audit 1685822125.6229522 mon.c (mon.2) 281 : audit 0 from='client.? 172.21.15.89:0/3732058306' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:55:26.188 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:55:25 smithi089 bash[78452]: cluster 1685822124.3101888 mgr.y (mgr.34104) 549 : cluster 0 pgmap v461: 89 pgs: 84 active+clean, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped; 273 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 674 op/s; 9172/13146 objects degraded (69.770%); 0 B/s, 18 objects/s recovering 2023-06-03T19:55:26.188 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:55:25 smithi089 bash[78452]: audit 1685822125.1763527 mon.a (mon.0) 499 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:55:26.188 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:55:25 smithi089 bash[78452]: audit 1685822125.1952941 mon.a (mon.0) 500 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:55:26.188 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:55:25 smithi089 bash[78452]: audit 1685822125.3955882 mon.a (mon.0) 501 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:55:26.188 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:55:25 smithi089 bash[78452]: audit 1685822125.4122493 mon.a (mon.0) 502 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:55:26.188 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:55:25 smithi089 bash[78452]: audit 1685822125.6229522 mon.c (mon.2) 281 : audit 0 from='client.? 172.21.15.89:0/3732058306' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:55:26.528 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 19:55:26 smithi089 bash[71163]: debug 2023-06-03T19:55:26.176+0000 7fe48642e700 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2023-06-03T19:55:26.529 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 19:55:26 smithi089 bash[71163]: ::ffff:172.21.15.164 - - [03/Jun/2023:19:55:26] "GET /metrics HTTP/1.1" 200 212799 "" "Prometheus/2.33.4" 2023-06-03T19:55:27.084 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:55:26 smithi164 bash[59462]: audit 1685822126.064837 mon.c (mon.2) 282 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-03T19:55:27.084 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:55:26 smithi164 bash[59462]: audit 1685822126.0665066 mon.c (mon.2) 283 : audit 1 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-03T19:55:27.085 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:55:26 smithi164 bash[59462]: audit 1685822126.074772 mon.a (mon.0) 503 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:55:27.085 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:55:26 smithi164 bash[59462]: audit 1685822126.1619596 mon.c (mon.2) 284 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-03T19:55:27.085 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:55:26 smithi164 bash[59462]: audit 1685822126.1692655 mon.c (mon.2) 285 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-06-03T19:55:27.085 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:55:26 smithi164 bash[59462]: audit 1685822126.1724305 mon.c (mon.2) 286 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-06-03T19:55:27.085 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:55:26 smithi164 bash[59462]: audit 1685822126.178519 mon.c (mon.2) 287 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-06-03T19:55:27.085 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:55:26 smithi164 bash[59462]: audit 1685822126.181523 mon.c (mon.2) 288 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-06-03T19:55:27.279 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:55:26 smithi089 bash[78452]: audit 1685822126.064837 mon.c (mon.2) 282 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-03T19:55:27.279 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:55:26 smithi089 bash[78452]: audit 1685822126.0665066 mon.c (mon.2) 283 : audit 1 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-03T19:55:27.279 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:55:26 smithi089 bash[78452]: audit 1685822126.074772 mon.a (mon.0) 503 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:55:27.279 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:55:26 smithi089 bash[78452]: audit 1685822126.1619596 mon.c (mon.2) 284 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-03T19:55:27.279 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:55:26 smithi089 bash[78452]: audit 1685822126.1692655 mon.c (mon.2) 285 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-06-03T19:55:27.279 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:55:26 smithi089 bash[78452]: audit 1685822126.1724305 mon.c (mon.2) 286 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-06-03T19:55:27.280 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:55:26 smithi089 bash[78452]: audit 1685822126.178519 mon.c (mon.2) 287 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-06-03T19:55:27.280 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:55:26 smithi089 bash[78452]: audit 1685822126.181523 mon.c (mon.2) 288 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-06-03T19:55:27.280 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:55:26 smithi089 bash[75068]: audit 1685822126.064837 mon.c (mon.2) 282 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-03T19:55:27.280 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:55:26 smithi089 bash[75068]: audit 1685822126.0665066 mon.c (mon.2) 283 : audit 1 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-03T19:55:27.280 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:55:26 smithi089 bash[75068]: audit 1685822126.074772 mon.a (mon.0) 503 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:55:27.280 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:55:26 smithi089 bash[75068]: audit 1685822126.1619596 mon.c (mon.2) 284 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-03T19:55:27.281 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:55:26 smithi089 bash[75068]: audit 1685822126.1692655 mon.c (mon.2) 285 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-06-03T19:55:27.281 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:55:26 smithi089 bash[75068]: audit 1685822126.1724305 mon.c (mon.2) 286 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-06-03T19:55:27.281 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:55:26 smithi089 bash[75068]: audit 1685822126.178519 mon.c (mon.2) 287 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-06-03T19:55:27.281 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:55:26 smithi089 bash[75068]: audit 1685822126.181523 mon.c (mon.2) 288 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-06-03T19:55:28.084 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:55:27 smithi164 bash[59462]: audit 1685822126.181808 mgr.y (mgr.34104) 550 : audit 0 from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-06-03T19:55:28.084 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:55:27 smithi164 bash[59462]: cephadm 1685822126.1833754 mgr.y (mgr.34104) 551 : cephadm 1 Upgrade: unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2023-06-03T19:55:28.085 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:55:27 smithi164 bash[59462]: cluster 1685822126.3108878 mgr.y (mgr.34104) 552 : cluster 0 pgmap v462: 89 pgs: 84 active+clean, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped; 180 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 642 op/s; 9062/8670 objects degraded (104.521%); 0 B/s, 28 objects/s recovering 2023-06-03T19:55:28.085 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:55:27 smithi164 bash[59462]: cluster 1685822127.4186244 mon.a (mon.0) 504 : cluster 3 Health check update: Degraded data redundancy: 9062/8670 objects degraded (104.521%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-06-03T19:55:28.279 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:55:27 smithi089 bash[75068]: audit 1685822126.181808 mgr.y (mgr.34104) 550 : audit 0 from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-06-03T19:55:28.279 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:55:27 smithi089 bash[75068]: cephadm 1685822126.1833754 mgr.y (mgr.34104) 551 : cephadm 1 Upgrade: unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2023-06-03T19:55:28.279 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:55:27 smithi089 bash[75068]: cluster 1685822126.3108878 mgr.y (mgr.34104) 552 : cluster 0 pgmap v462: 89 pgs: 84 active+clean, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped; 180 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 642 op/s; 9062/8670 objects degraded (104.521%); 0 B/s, 28 objects/s recovering 2023-06-03T19:55:28.279 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:55:27 smithi089 bash[75068]: cluster 1685822127.4186244 mon.a (mon.0) 504 : cluster 3 Health check update: Degraded data redundancy: 9062/8670 objects degraded (104.521%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-06-03T19:55:28.280 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:55:27 smithi089 bash[78452]: audit 1685822126.181808 mgr.y (mgr.34104) 550 : audit 0 from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-06-03T19:55:28.280 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:55:27 smithi089 bash[78452]: cephadm 1685822126.1833754 mgr.y (mgr.34104) 551 : cephadm 1 Upgrade: unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2023-06-03T19:55:28.280 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:55:27 smithi089 bash[78452]: cluster 1685822126.3108878 mgr.y (mgr.34104) 552 : cluster 0 pgmap v462: 89 pgs: 84 active+clean, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped; 180 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 642 op/s; 9062/8670 objects degraded (104.521%); 0 B/s, 28 objects/s recovering 2023-06-03T19:55:28.280 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:55:27 smithi089 bash[78452]: cluster 1685822127.4186244 mon.a (mon.0) 504 : cluster 3 Health check update: Degraded data redundancy: 9062/8670 objects degraded (104.521%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-06-03T19:55:29.065 INFO:tasks.radosbench.radosbench.0.smithi089.stdout:Removed 106681 objects 2023-06-03T19:55:29.071 INFO:tasks.radosbench.radosbench.0.smithi089.stdout:Clean up completed and total clean up time :162.718 2023-06-03T19:55:29.084 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:55:28 smithi164 bash[59462]: audit 1685822128.1825678 mon.c (mon.2) 289 : audit 0 from='client.? 172.21.15.89:0/2264747924' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:55:29.278 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:55:28 smithi089 bash[78452]: audit 1685822128.1825678 mon.c (mon.2) 289 : audit 0 from='client.? 172.21.15.89:0/2264747924' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:55:29.279 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:55:28 smithi089 bash[75068]: audit 1685822128.1825678 mon.c (mon.2) 289 : audit 0 from='client.? 172.21.15.89:0/2264747924' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:55:30.084 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:55:29 smithi164 bash[59462]: cluster 1685822128.3114603 mgr.y (mgr.34104) 553 : cluster 0 pgmap v463: 89 pgs: 84 active+clean, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped; 157 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 555 op/s; 9062/7566 objects degraded (119.773%); 0 B/s, 18 objects/s recovering 2023-06-03T19:55:30.278 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:55:29 smithi089 bash[78452]: cluster 1685822128.3114603 mgr.y (mgr.34104) 553 : cluster 0 pgmap v463: 89 pgs: 84 active+clean, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped; 157 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 555 op/s; 9062/7566 objects degraded (119.773%); 0 B/s, 18 objects/s recovering 2023-06-03T19:55:30.279 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:55:29 smithi089 bash[75068]: cluster 1685822128.3114603 mgr.y (mgr.34104) 553 : cluster 0 pgmap v463: 89 pgs: 84 active+clean, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped; 157 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 555 op/s; 9062/7566 objects degraded (119.773%); 0 B/s, 18 objects/s recovering 2023-06-03T19:55:31.084 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:55:30 smithi164 bash[59462]: audit 1685822130.7657614 mon.c (mon.2) 290 : audit 0 from='client.? 172.21.15.89:0/4114850297' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:55:31.279 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:55:30 smithi089 bash[75068]: audit 1685822130.7657614 mon.c (mon.2) 290 : audit 0 from='client.? 172.21.15.89:0/4114850297' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:55:31.279 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:55:30 smithi089 bash[78452]: audit 1685822130.7657614 mon.c (mon.2) 290 : audit 0 from='client.? 172.21.15.89:0/4114850297' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:55:31.533 INFO:tasks.cephadm.ceph_manager.ceph:removing pool_name unique_pool_3 2023-06-03T19:55:31.534 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph osd pool rm unique_pool_3 unique_pool_3 --yes-i-really-really-mean-it 2023-06-03T19:55:32.029 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:55:31 smithi089 bash[78452]: cluster 1685822130.3125372 mgr.y (mgr.34104) 554 : cluster 0 pgmap v464: 89 pgs: 84 active+clean, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped; 38 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 677 op/s; 8949/1884 objects degraded (475.000%); 0 B/s, 28 objects/s recovering 2023-06-03T19:55:32.029 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:55:31 smithi089 bash[75068]: cluster 1685822130.3125372 mgr.y (mgr.34104) 554 : cluster 0 pgmap v464: 89 pgs: 84 active+clean, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped; 38 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 677 op/s; 8949/1884 objects degraded (475.000%); 0 B/s, 28 objects/s recovering 2023-06-03T19:55:32.084 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:55:31 smithi164 bash[59462]: cluster 1685822130.3125372 mgr.y (mgr.34104) 554 : cluster 0 pgmap v464: 89 pgs: 84 active+clean, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped; 38 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 677 op/s; 8949/1884 objects degraded (475.000%); 0 B/s, 28 objects/s recovering 2023-06-03T19:55:33.084 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:55:32 smithi164 bash[59462]: cluster 1685822132.4199517 mon.a (mon.0) 505 : cluster 3 Health check update: Degraded data redundancy: 8949/1884 objects degraded (475.000%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-06-03T19:55:33.278 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:55:32 smithi089 bash[75068]: cluster 1685822132.4199517 mon.a (mon.0) 505 : cluster 3 Health check update: Degraded data redundancy: 8949/1884 objects degraded (475.000%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-06-03T19:55:33.279 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:55:32 smithi089 bash[78452]: cluster 1685822132.4199517 mon.a (mon.0) 505 : cluster 3 Health check update: Degraded data redundancy: 8949/1884 objects degraded (475.000%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-06-03T19:55:34.085 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:55:33 smithi164 bash[59462]: cluster 1685822132.3134625 mgr.y (mgr.34104) 555 : cluster 0 pgmap v465: 89 pgs: 84 active+clean, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped; 15 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 528 op/s; 8949/762 objects degraded (1174.409%); 0 B/s, 18 objects/s recovering 2023-06-03T19:55:34.496 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:55:33 smithi164 bash[59462]: audit 1685822133.22368 mon.a (mon.0) 506 : audit 0 from='client.? 172.21.15.89:0/3713016745' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:55:34.497 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:55:33 smithi089 bash[75068]: cluster 1685822132.3134625 mgr.y (mgr.34104) 555 : cluster 0 pgmap v465: 89 pgs: 84 active+clean, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped; 15 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 528 op/s; 8949/762 objects degraded (1174.409%); 0 B/s, 18 objects/s recovering 2023-06-03T19:55:34.497 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:55:33 smithi089 bash[75068]: audit 1685822133.22368 mon.a (mon.0) 506 : audit 0 from='client.? 172.21.15.89:0/3713016745' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:55:34.497 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:55:33 smithi089 bash[78452]: cluster 1685822132.3134625 mgr.y (mgr.34104) 555 : cluster 0 pgmap v465: 89 pgs: 84 active+clean, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped; 15 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 528 op/s; 8949/762 objects degraded (1174.409%); 0 B/s, 18 objects/s recovering 2023-06-03T19:55:34.498 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:55:33 smithi089 bash[78452]: audit 1685822133.22368 mon.a (mon.0) 506 : audit 0 from='client.? 172.21.15.89:0/3713016745' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:55:36.232 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:55:35 smithi089 bash[78452]: cluster 1685822134.3146129 mgr.y (mgr.34104) 556 : cluster 0 pgmap v466: 89 pgs: 84 active+clean, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped; 582 KiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 497 op/s; 8949/72 objects degraded (12429.167%); 0 B/s, 18 objects/s recovering 2023-06-03T19:55:36.233 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:55:35 smithi089 bash[78452]: audit 1685822135.641108 mon.c (mon.2) 291 : audit 0 from='client.? 172.21.15.89:0/794732982' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:55:36.233 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:55:35 smithi089 bash[75068]: cluster 1685822134.3146129 mgr.y (mgr.34104) 556 : cluster 0 pgmap v466: 89 pgs: 84 active+clean, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped; 582 KiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 497 op/s; 8949/72 objects degraded (12429.167%); 0 B/s, 18 objects/s recovering 2023-06-03T19:55:36.233 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:55:35 smithi089 bash[75068]: audit 1685822135.641108 mon.c (mon.2) 291 : audit 0 from='client.? 172.21.15.89:0/794732982' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:55:36.334 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:55:35 smithi164 bash[59462]: cluster 1685822134.3146129 mgr.y (mgr.34104) 556 : cluster 0 pgmap v466: 89 pgs: 84 active+clean, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped; 582 KiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 497 op/s; 8949/72 objects degraded (12429.167%); 0 B/s, 18 objects/s recovering 2023-06-03T19:55:36.334 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:55:35 smithi164 bash[59462]: audit 1685822135.641108 mon.c (mon.2) 291 : audit 0 from='client.? 172.21.15.89:0/794732982' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:55:36.417 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/mon.a/config 2023-06-03T19:55:36.528 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 19:55:36 smithi089 bash[71163]: ::ffff:172.21.15.164 - - [03/Jun/2023:19:55:36] "GET /metrics HTTP/1.1" 200 212799 "" "Prometheus/2.33.4" 2023-06-03T19:55:38.190 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:55:37 smithi089 bash[78452]: cluster 1685822136.3152878 mgr.y (mgr.34104) 557 : cluster 0 pgmap v467: 89 pgs: 84 active+clean, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped; 582 KiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 363 op/s; 8833/72 objects degraded (12268.056%); 0 B/s, 28 objects/s recovering 2023-06-03T19:55:38.190 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:55:37 smithi089 bash[78452]: audit 1685822137.0856316 mon.c (mon.2) 292 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T19:55:38.190 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:55:37 smithi089 bash[78452]: cluster 1685822137.421592 mon.a (mon.0) 507 : cluster 3 Health check update: Degraded data redundancy: 8833/72 objects degraded (12268.056%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-06-03T19:55:38.190 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:55:37 smithi089 bash[75068]: cluster 1685822136.3152878 mgr.y (mgr.34104) 557 : cluster 0 pgmap v467: 89 pgs: 84 active+clean, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped; 582 KiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 363 op/s; 8833/72 objects degraded (12268.056%); 0 B/s, 28 objects/s recovering 2023-06-03T19:55:38.191 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:55:37 smithi089 bash[75068]: audit 1685822137.0856316 mon.c (mon.2) 292 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T19:55:38.191 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:55:37 smithi089 bash[75068]: cluster 1685822137.421592 mon.a (mon.0) 507 : cluster 3 Health check update: Degraded data redundancy: 8833/72 objects degraded (12268.056%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-06-03T19:55:38.334 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:55:37 smithi164 bash[59462]: cluster 1685822136.3152878 mgr.y (mgr.34104) 557 : cluster 0 pgmap v467: 89 pgs: 84 active+clean, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped; 582 KiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 363 op/s; 8833/72 objects degraded (12268.056%); 0 B/s, 28 objects/s recovering 2023-06-03T19:55:38.335 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:55:37 smithi164 bash[59462]: audit 1685822137.0856316 mon.c (mon.2) 292 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T19:55:38.335 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:55:37 smithi164 bash[59462]: cluster 1685822137.421592 mon.a (mon.0) 507 : cluster 3 Health check update: Degraded data redundancy: 8833/72 objects degraded (12268.056%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-06-03T19:55:39.278 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:55:38 smithi089 bash[75068]: audit 1685822137.9987726 mon.c (mon.2) 293 : audit 0 from='client.? 172.21.15.89:0/3917683474' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:55:39.279 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:55:38 smithi089 bash[78452]: audit 1685822137.9987726 mon.c (mon.2) 293 : audit 0 from='client.? 172.21.15.89:0/3917683474' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:55:39.334 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:55:38 smithi164 bash[59462]: audit 1685822137.9987726 mon.c (mon.2) 293 : audit 0 from='client.? 172.21.15.89:0/3917683474' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:55:40.155 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:55:39 smithi089 bash[75068]: cluster 1685822138.3159237 mgr.y (mgr.34104) 558 : cluster 0 pgmap v468: 89 pgs: 84 active+clean, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped; 582 KiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 238 op/s; 8833/72 objects degraded (12268.056%); 0 B/s, 19 objects/s recovering 2023-06-03T19:55:40.155 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:55:39 smithi089 bash[78452]: cluster 1685822138.3159237 mgr.y (mgr.34104) 558 : cluster 0 pgmap v468: 89 pgs: 84 active+clean, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped; 582 KiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 238 op/s; 8833/72 objects degraded (12268.056%); 0 B/s, 19 objects/s recovering 2023-06-03T19:55:40.334 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:55:39 smithi164 bash[59462]: cluster 1685822138.3159237 mgr.y (mgr.34104) 558 : cluster 0 pgmap v468: 89 pgs: 84 active+clean, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped; 582 KiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 238 op/s; 8833/72 objects degraded (12268.056%); 0 B/s, 19 objects/s recovering 2023-06-03T19:55:40.869 INFO:teuthology.orchestra.run.smithi089.stderr:pool 'unique_pool_3' removed 2023-06-03T19:55:41.278 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:55:40 smithi089 bash[75068]: audit 1685822140.3835316 mon.c (mon.2) 294 : audit 0 from='client.? 172.21.15.89:0/1378682310' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:55:41.279 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:55:40 smithi089 bash[75068]: audit 1685822140.5239851 mon.c (mon.2) 295 : audit 1 from='client.? 172.21.15.89:0/3372133755' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_3", "pool2": "unique_pool_3", "yes_i_really_really_mean_it": true}]: dispatch 2023-06-03T19:55:41.279 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:55:40 smithi089 bash[75068]: audit 1685822140.5244715 mon.a (mon.0) 508 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_3", "pool2": "unique_pool_3", "yes_i_really_really_mean_it": true}]: dispatch 2023-06-03T19:55:41.279 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:55:40 smithi089 bash[78452]: audit 1685822140.3835316 mon.c (mon.2) 294 : audit 0 from='client.? 172.21.15.89:0/1378682310' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:55:41.279 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:55:40 smithi089 bash[78452]: audit 1685822140.5239851 mon.c (mon.2) 295 : audit 1 from='client.? 172.21.15.89:0/3372133755' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_3", "pool2": "unique_pool_3", "yes_i_really_really_mean_it": true}]: dispatch 2023-06-03T19:55:41.280 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:55:40 smithi089 bash[78452]: audit 1685822140.5244715 mon.a (mon.0) 508 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_3", "pool2": "unique_pool_3", "yes_i_really_really_mean_it": true}]: dispatch 2023-06-03T19:55:41.334 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:55:40 smithi164 bash[59462]: audit 1685822140.3835316 mon.c (mon.2) 294 : audit 0 from='client.? 172.21.15.89:0/1378682310' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:55:41.334 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:55:40 smithi164 bash[59462]: audit 1685822140.5239851 mon.c (mon.2) 295 : audit 1 from='client.? 172.21.15.89:0/3372133755' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_3", "pool2": "unique_pool_3", "yes_i_really_really_mean_it": true}]: dispatch 2023-06-03T19:55:41.335 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:55:40 smithi164 bash[59462]: audit 1685822140.5244715 mon.a (mon.0) 508 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_3", "pool2": "unique_pool_3", "yes_i_really_really_mean_it": true}]: dispatch 2023-06-03T19:55:41.846 INFO:teuthology.task.sequential:In sequential, running task print... 2023-06-03T19:55:41.846 INFO:teuthology.task.print:**** done end radosbench.yaml 2023-06-03T19:55:41.846 DEBUG:teuthology.parallel:result is None 2023-06-03T19:55:42.279 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:55:41 smithi089 bash[78452]: cluster 1685822140.3171194 mgr.y (mgr.34104) 559 : cluster 0 pgmap v469: 89 pgs: 84 active+clean, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped; 582 KiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 208 op/s; 8716/72 objects degraded (12105.556%); 0 B/s, 28 objects/s recovering 2023-06-03T19:55:42.279 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:55:41 smithi089 bash[78452]: cluster 1685822140.8480265 osd.6 (osd.6) 25 : cluster 3 failed to encode map e110 with expected crc 2023-06-03T19:55:42.279 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:55:41 smithi089 bash[78452]: audit 1685822140.8557591 mon.a (mon.0) 509 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_3", "pool2": "unique_pool_3", "yes_i_really_really_mean_it": true}]': finished 2023-06-03T19:55:42.279 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:55:41 smithi089 bash[78452]: cluster 1685822140.8558443 mon.a (mon.0) 510 : cluster 0 osdmap e110: 8 total, 8 up, 8 in 2023-06-03T19:55:42.279 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:55:41 smithi089 bash[78452]: cluster 1685822140.961269 osd.3 (osd.3) 34 : cluster 3 failed to encode map e110 with expected crc 2023-06-03T19:55:42.280 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:55:41 smithi089 bash[78452]: audit 1685822141.1992574 mon.c (mon.2) 296 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-06-03T19:55:42.280 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:55:41 smithi089 bash[75068]: cluster 1685822140.3171194 mgr.y (mgr.34104) 559 : cluster 0 pgmap v469: 89 pgs: 84 active+clean, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped; 582 KiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 208 op/s; 8716/72 objects degraded (12105.556%); 0 B/s, 28 objects/s recovering 2023-06-03T19:55:42.280 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:55:41 smithi089 bash[75068]: cluster 1685822140.8480265 osd.6 (osd.6) 25 : cluster 3 failed to encode map e110 with expected crc 2023-06-03T19:55:42.280 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:55:41 smithi089 bash[75068]: audit 1685822140.8557591 mon.a (mon.0) 509 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_3", "pool2": "unique_pool_3", "yes_i_really_really_mean_it": true}]': finished 2023-06-03T19:55:42.280 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:55:41 smithi089 bash[75068]: cluster 1685822140.8558443 mon.a (mon.0) 510 : cluster 0 osdmap e110: 8 total, 8 up, 8 in 2023-06-03T19:55:42.280 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:55:41 smithi089 bash[75068]: cluster 1685822140.961269 osd.3 (osd.3) 34 : cluster 3 failed to encode map e110 with expected crc 2023-06-03T19:55:42.281 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:55:41 smithi089 bash[75068]: audit 1685822141.1992574 mon.c (mon.2) 296 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-06-03T19:55:42.334 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:55:41 smithi164 bash[59462]: cluster 1685822140.3171194 mgr.y (mgr.34104) 559 : cluster 0 pgmap v469: 89 pgs: 84 active+clean, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped; 582 KiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 208 op/s; 8716/72 objects degraded (12105.556%); 0 B/s, 28 objects/s recovering 2023-06-03T19:55:42.334 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:55:41 smithi164 bash[59462]: cluster 1685822140.8480265 osd.6 (osd.6) 25 : cluster 3 failed to encode map e110 with expected crc 2023-06-03T19:55:42.335 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:55:41 smithi164 bash[59462]: audit 1685822140.8557591 mon.a (mon.0) 509 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_3", "pool2": "unique_pool_3", "yes_i_really_really_mean_it": true}]': finished 2023-06-03T19:55:42.335 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:55:41 smithi164 bash[59462]: cluster 1685822140.8558443 mon.a (mon.0) 510 : cluster 0 osdmap e110: 8 total, 8 up, 8 in 2023-06-03T19:55:42.335 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:55:41 smithi164 bash[59462]: cluster 1685822140.961269 osd.3 (osd.3) 34 : cluster 3 failed to encode map e110 with expected crc 2023-06-03T19:55:42.335 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:55:41 smithi164 bash[59462]: audit 1685822141.1992574 mon.c (mon.2) 296 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-06-03T19:55:42.940 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:55:42 smithi089 bash[78452]: audit 1685822141.1997027 mgr.y (mgr.34104) 560 : audit 0 from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-06-03T19:55:42.940 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:55:42 smithi089 bash[78452]: cephadm 1685822141.201748 mgr.y (mgr.34104) 561 : cephadm 1 Upgrade: osd.2 is safe to restart 2023-06-03T19:55:42.940 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:55:42 smithi089 bash[78452]: cluster 1685822141.6105757 osd.5 (osd.5) 30 : cluster 3 failed to encode map e110 with expected crc 2023-06-03T19:55:42.940 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:55:42 smithi089 bash[78452]: cluster 1685822141.9011002 osd.4 (osd.4) 23 : cluster 3 failed to encode map e110 with expected crc 2023-06-03T19:55:42.941 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:55:42 smithi089 bash[78452]: cluster 1685822141.9015539 osd.7 (osd.7) 22 : cluster 3 failed to encode map e110 with expected crc 2023-06-03T19:55:42.941 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:55:42 smithi089 bash[78452]: cluster 1685822141.9035916 osd.2 (osd.2) 33 : cluster 3 failed to encode map e110 with expected crc 2023-06-03T19:55:42.941 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:55:42 smithi089 bash[78452]: cluster 1685822142.422584 mon.a (mon.0) 511 : cluster 3 Health check update: Degraded data redundancy: 8716/72 objects degraded (12105.556%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-06-03T19:55:42.941 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:55:42 smithi089 bash[78452]: audit 1685822142.7840126 mon.c (mon.2) 297 : audit 0 from='client.? 172.21.15.89:0/915413222' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:55:42.941 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:55:42 smithi089 bash[75068]: audit 1685822141.1997027 mgr.y (mgr.34104) 560 : audit 0 from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-06-03T19:55:42.941 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:55:42 smithi089 bash[75068]: cephadm 1685822141.201748 mgr.y (mgr.34104) 561 : cephadm 1 Upgrade: osd.2 is safe to restart 2023-06-03T19:55:42.941 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:55:42 smithi089 bash[75068]: cluster 1685822141.6105757 osd.5 (osd.5) 30 : cluster 3 failed to encode map e110 with expected crc 2023-06-03T19:55:42.942 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:55:42 smithi089 bash[75068]: cluster 1685822141.9011002 osd.4 (osd.4) 23 : cluster 3 failed to encode map e110 with expected crc 2023-06-03T19:55:42.942 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:55:42 smithi089 bash[75068]: cluster 1685822141.9015539 osd.7 (osd.7) 22 : cluster 3 failed to encode map e110 with expected crc 2023-06-03T19:55:42.942 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:55:42 smithi089 bash[75068]: cluster 1685822141.9035916 osd.2 (osd.2) 33 : cluster 3 failed to encode map e110 with expected crc 2023-06-03T19:55:42.942 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:55:42 smithi089 bash[75068]: cluster 1685822142.422584 mon.a (mon.0) 511 : cluster 3 Health check update: Degraded data redundancy: 8716/72 objects degraded (12105.556%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-06-03T19:55:42.942 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:55:42 smithi089 bash[75068]: audit 1685822142.7840126 mon.c (mon.2) 297 : audit 0 from='client.? 172.21.15.89:0/915413222' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:55:43.334 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:55:42 smithi164 bash[59462]: audit 1685822141.1997027 mgr.y (mgr.34104) 560 : audit 0 from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-06-03T19:55:43.335 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:55:42 smithi164 bash[59462]: cephadm 1685822141.201748 mgr.y (mgr.34104) 561 : cephadm 1 Upgrade: osd.2 is safe to restart 2023-06-03T19:55:43.335 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:55:42 smithi164 bash[59462]: cluster 1685822141.6105757 osd.5 (osd.5) 30 : cluster 3 failed to encode map e110 with expected crc 2023-06-03T19:55:43.335 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:55:42 smithi164 bash[59462]: cluster 1685822141.9011002 osd.4 (osd.4) 23 : cluster 3 failed to encode map e110 with expected crc 2023-06-03T19:55:43.335 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:55:42 smithi164 bash[59462]: cluster 1685822141.9015539 osd.7 (osd.7) 22 : cluster 3 failed to encode map e110 with expected crc 2023-06-03T19:55:43.335 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:55:42 smithi164 bash[59462]: cluster 1685822141.9035916 osd.2 (osd.2) 33 : cluster 3 failed to encode map e110 with expected crc 2023-06-03T19:55:43.335 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:55:42 smithi164 bash[59462]: cluster 1685822142.422584 mon.a (mon.0) 511 : cluster 3 Health check update: Degraded data redundancy: 8716/72 objects degraded (12105.556%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-06-03T19:55:43.335 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:55:42 smithi164 bash[59462]: audit 1685822142.7840126 mon.c (mon.2) 297 : audit 0 from='client.? 172.21.15.89:0/915413222' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:55:44.144 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:55:43 smithi089 bash[75068]: cluster 1685822142.3178458 mgr.y (mgr.34104) 562 : cluster 0 pgmap v471: 73 pgs: 73 active+clean; 582 KiB data, 2.2 GiB used, 713 GiB / 715 GiB avail 2023-06-03T19:55:44.144 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:55:43 smithi089 bash[75068]: cluster 1685822142.8516483 mon.a (mon.0) 512 : cluster 1 Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 8716/72 objects degraded (12105.556%), 5 pgs degraded, 5 pgs undersized) 2023-06-03T19:55:44.144 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:55:43 smithi089 bash[75068]: cluster 1685822142.8517013 mon.a (mon.0) 513 : cluster 1 Cluster is now healthy 2023-06-03T19:55:44.145 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:55:43 smithi089 bash[75068]: cephadm 1685822143.8217788 mgr.y (mgr.34104) 563 : cephadm 1 Upgrade: Updating osd.2 2023-06-03T19:55:44.145 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:55:43 smithi089 bash[75068]: audit 1685822143.8313262 mon.a (mon.0) 514 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:55:44.145 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:55:43 smithi089 bash[75068]: audit 1685822143.8330262 mon.c (mon.2) 298 : audit 1 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2023-06-03T19:55:44.145 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:55:43 smithi089 bash[75068]: audit 1685822143.8341343 mon.c (mon.2) 299 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-03T19:55:44.145 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:55:43 smithi089 bash[75068]: cephadm 1685822143.8367808 mgr.y (mgr.34104) 564 : cephadm 1 Deploying daemon osd.2 on smithi089 2023-06-03T19:55:44.145 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:55:43 smithi089 bash[78452]: cluster 1685822142.3178458 mgr.y (mgr.34104) 562 : cluster 0 pgmap v471: 73 pgs: 73 active+clean; 582 KiB data, 2.2 GiB used, 713 GiB / 715 GiB avail 2023-06-03T19:55:44.146 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:55:43 smithi089 bash[78452]: cluster 1685822142.8516483 mon.a (mon.0) 512 : cluster 1 Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 8716/72 objects degraded (12105.556%), 5 pgs degraded, 5 pgs undersized) 2023-06-03T19:55:44.146 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:55:43 smithi089 bash[78452]: cluster 1685822142.8517013 mon.a (mon.0) 513 : cluster 1 Cluster is now healthy 2023-06-03T19:55:44.146 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:55:43 smithi089 bash[78452]: cephadm 1685822143.8217788 mgr.y (mgr.34104) 563 : cephadm 1 Upgrade: Updating osd.2 2023-06-03T19:55:44.146 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:55:43 smithi089 bash[78452]: audit 1685822143.8313262 mon.a (mon.0) 514 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:55:44.146 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:55:43 smithi089 bash[78452]: audit 1685822143.8330262 mon.c (mon.2) 298 : audit 1 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2023-06-03T19:55:44.146 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:55:43 smithi089 bash[78452]: audit 1685822143.8341343 mon.c (mon.2) 299 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-03T19:55:44.146 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:55:43 smithi089 bash[78452]: cephadm 1685822143.8367808 mgr.y (mgr.34104) 564 : cephadm 1 Deploying daemon osd.2 on smithi089 2023-06-03T19:55:44.334 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:55:43 smithi164 bash[59462]: cluster 1685822142.3178458 mgr.y (mgr.34104) 562 : cluster 0 pgmap v471: 73 pgs: 73 active+clean; 582 KiB data, 2.2 GiB used, 713 GiB / 715 GiB avail 2023-06-03T19:55:44.335 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:55:43 smithi164 bash[59462]: cluster 1685822142.8516483 mon.a (mon.0) 512 : cluster 1 Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 8716/72 objects degraded (12105.556%), 5 pgs degraded, 5 pgs undersized) 2023-06-03T19:55:44.335 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:55:43 smithi164 bash[59462]: cluster 1685822142.8517013 mon.a (mon.0) 513 : cluster 1 Cluster is now healthy 2023-06-03T19:55:44.335 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:55:43 smithi164 bash[59462]: cephadm 1685822143.8217788 mgr.y (mgr.34104) 563 : cephadm 1 Upgrade: Updating osd.2 2023-06-03T19:55:44.335 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:55:43 smithi164 bash[59462]: audit 1685822143.8313262 mon.a (mon.0) 514 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:55:44.335 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:55:43 smithi164 bash[59462]: audit 1685822143.8330262 mon.c (mon.2) 298 : audit 1 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2023-06-03T19:55:44.335 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:55:43 smithi164 bash[59462]: audit 1685822143.8341343 mon.c (mon.2) 299 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-03T19:55:44.335 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:55:43 smithi164 bash[59462]: cephadm 1685822143.8367808 mgr.y (mgr.34104) 564 : cephadm 1 Deploying daemon osd.2 on smithi089 2023-06-03T19:55:45.279 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:55:44 smithi089 bash[75068]: cluster 1685822144.318489 mgr.y (mgr.34104) 565 : cluster 0 pgmap v472: 73 pgs: 73 active+clean; 582 KiB data, 2.2 GiB used, 713 GiB / 715 GiB avail 2023-06-03T19:55:45.280 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:55:44 smithi089 bash[78452]: cluster 1685822144.318489 mgr.y (mgr.34104) 565 : cluster 0 pgmap v472: 73 pgs: 73 active+clean; 582 KiB data, 2.2 GiB used, 713 GiB / 715 GiB avail 2023-06-03T19:55:45.334 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:55:44 smithi164 bash[59462]: cluster 1685822144.318489 mgr.y (mgr.34104) 565 : cluster 0 pgmap v472: 73 pgs: 73 active+clean; 582 KiB data, 2.2 GiB used, 713 GiB / 715 GiB avail 2023-06-03T19:55:46.233 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:55:45 smithi089 bash[78452]: audit 1685822145.1371531 mon.a (mon.0) 515 : audit 0 from='client.? 172.21.15.89:0/4089351520' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:55:46.234 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:55:45 smithi089 bash[75068]: audit 1685822145.1371531 mon.a (mon.0) 515 : audit 0 from='client.? 172.21.15.89:0/4089351520' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:55:46.334 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:55:45 smithi164 bash[59462]: audit 1685822145.1371531 mon.a (mon.0) 515 : audit 0 from='client.? 172.21.15.89:0/4089351520' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:55:46.529 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 19:55:46 smithi089 bash[71163]: ::ffff:172.21.15.164 - - [03/Jun/2023:19:55:46] "GET /metrics HTTP/1.1" 200 212728 "" "Prometheus/2.33.4" 2023-06-03T19:55:47.219 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:55:46 smithi089 bash[78452]: cluster 1685822146.3191953 mgr.y (mgr.34104) 566 : cluster 0 pgmap v473: 73 pgs: 73 active+clean; 582 KiB data, 2.2 GiB used, 713 GiB / 715 GiB avail 2023-06-03T19:55:47.220 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:55:46 smithi089 bash[75068]: cluster 1685822146.3191953 mgr.y (mgr.34104) 566 : cluster 0 pgmap v473: 73 pgs: 73 active+clean; 582 KiB data, 2.2 GiB used, 713 GiB / 715 GiB avail 2023-06-03T19:55:47.334 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:55:46 smithi164 bash[59462]: cluster 1685822146.3191953 mgr.y (mgr.34104) 566 : cluster 0 pgmap v473: 73 pgs: 73 active+clean; 582 KiB data, 2.2 GiB used, 713 GiB / 715 GiB avail 2023-06-03T19:55:48.722 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:55:48 smithi089 bash[78452]: audit 1685822147.3368716 mon.a (mon.0) 516 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:55:48.723 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:55:48 smithi089 bash[78452]: audit 1685822147.5033345 mon.c (mon.2) 300 : audit 0 from='client.? 172.21.15.89:0/298323765' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:55:48.723 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:55:48 smithi089 bash[75068]: audit 1685822147.3368716 mon.a (mon.0) 516 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:55:48.723 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:55:48 smithi089 bash[75068]: audit 1685822147.5033345 mon.c (mon.2) 300 : audit 0 from='client.? 172.21.15.89:0/298323765' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:55:48.834 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:55:48 smithi164 bash[59462]: audit 1685822147.3368716 mon.a (mon.0) 516 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:55:48.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:55:48 smithi164 bash[59462]: audit 1685822147.5033345 mon.c (mon.2) 300 : audit 0 from='client.? 172.21.15.89:0/298323765' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:55:49.029 INFO:journalctl@ceph.osd.2.smithi089.stdout:Jun 03 19:55:48 smithi089 systemd[1]: Stopping Ceph osd.2 for f1db3478-0243-11ee-9b1d-001a4aab830c... 2023-06-03T19:55:49.029 INFO:journalctl@ceph.osd.2.smithi089.stdout:Jun 03 19:55:48 smithi089 bash[44697]: debug 2023-06-03T19:55:48.908+0000 7f2e16e99700 -1 received signal: Terminated from /sbin/docker-init -- /usr/bin/ceph-osd -n osd.2 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug (PID: 1) UID: 0 2023-06-03T19:55:49.029 INFO:journalctl@ceph.osd.2.smithi089.stdout:Jun 03 19:55:48 smithi089 bash[44697]: debug 2023-06-03T19:55:48.908+0000 7f2e16e99700 -1 osd.2 110 *** Got signal Terminated *** 2023-06-03T19:55:49.029 INFO:journalctl@ceph.osd.2.smithi089.stdout:Jun 03 19:55:48 smithi089 bash[44697]: debug 2023-06-03T19:55:48.908+0000 7f2e16e99700 -1 osd.2 110 *** Immediate shutdown (osd_fast_shutdown=true) *** 2023-06-03T19:55:49.778 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:55:49 smithi089 bash[75068]: cluster 1685822148.3199418 mgr.y (mgr.34104) 567 : cluster 0 pgmap v474: 73 pgs: 73 active+clean; 582 KiB data, 2.2 GiB used, 713 GiB / 715 GiB avail 2023-06-03T19:55:49.779 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:55:49 smithi089 bash[75068]: cluster 1685822148.9108808 mon.a (mon.0) 517 : cluster 1 osd.2 marked itself down and dead 2023-06-03T19:55:49.779 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:55:49 smithi089 bash[78452]: cluster 1685822148.3199418 mgr.y (mgr.34104) 567 : cluster 0 pgmap v474: 73 pgs: 73 active+clean; 582 KiB data, 2.2 GiB used, 713 GiB / 715 GiB avail 2023-06-03T19:55:49.779 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:55:49 smithi089 bash[78452]: cluster 1685822148.9108808 mon.a (mon.0) 517 : cluster 1 osd.2 marked itself down and dead 2023-06-03T19:55:49.834 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:55:49 smithi164 bash[59462]: cluster 1685822148.3199418 mgr.y (mgr.34104) 567 : cluster 0 pgmap v474: 73 pgs: 73 active+clean; 582 KiB data, 2.2 GiB used, 713 GiB / 715 GiB avail 2023-06-03T19:55:49.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:55:49 smithi164 bash[59462]: cluster 1685822148.9108808 mon.a (mon.0) 517 : cluster 1 osd.2 marked itself down and dead 2023-06-03T19:55:50.779 INFO:journalctl@ceph.osd.2.smithi089.stdout:Jun 03 19:55:50 smithi089 bash[97887]: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd-2 2023-06-03T19:55:50.779 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:55:50 smithi089 bash[78452]: cluster 1685822149.430043 mon.a (mon.0) 518 : cluster 3 Health check failed: 1 osds down (OSD_DOWN) 2023-06-03T19:55:50.780 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:55:50 smithi089 bash[78452]: cluster 1685822149.4398828 osd.3 (osd.3) 35 : cluster 3 failed to encode map e111 with expected crc 2023-06-03T19:55:50.780 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:55:50 smithi089 bash[78452]: cluster 1685822149.4404998 osd.4 (osd.4) 24 : cluster 3 failed to encode map e111 with expected crc 2023-06-03T19:55:50.780 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:55:50 smithi089 bash[78452]: cluster 1685822149.4411604 osd.6 (osd.6) 26 : cluster 3 failed to encode map e111 with expected crc 2023-06-03T19:55:50.780 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:55:50 smithi089 bash[78452]: cluster 1685822149.4422512 mon.a (mon.0) 519 : cluster 0 osdmap e111: 8 total, 7 up, 8 in 2023-06-03T19:55:50.780 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:55:50 smithi089 bash[78452]: cluster 1685822149.4426196 osd.5 (osd.5) 31 : cluster 3 failed to encode map e111 with expected crc 2023-06-03T19:55:50.780 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:55:50 smithi089 bash[78452]: audit 1685822149.8689492 mon.c (mon.2) 301 : audit 0 from='client.? 172.21.15.89:0/1082583437' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:55:50.781 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:55:50 smithi089 bash[75068]: cluster 1685822149.430043 mon.a (mon.0) 518 : cluster 3 Health check failed: 1 osds down (OSD_DOWN) 2023-06-03T19:55:50.781 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:55:50 smithi089 bash[75068]: cluster 1685822149.4398828 osd.3 (osd.3) 35 : cluster 3 failed to encode map e111 with expected crc 2023-06-03T19:55:50.781 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:55:50 smithi089 bash[75068]: cluster 1685822149.4404998 osd.4 (osd.4) 24 : cluster 3 failed to encode map e111 with expected crc 2023-06-03T19:55:50.781 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:55:50 smithi089 bash[75068]: cluster 1685822149.4411604 osd.6 (osd.6) 26 : cluster 3 failed to encode map e111 with expected crc 2023-06-03T19:55:50.781 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:55:50 smithi089 bash[75068]: cluster 1685822149.4422512 mon.a (mon.0) 519 : cluster 0 osdmap e111: 8 total, 7 up, 8 in 2023-06-03T19:55:50.781 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:55:50 smithi089 bash[75068]: cluster 1685822149.4426196 osd.5 (osd.5) 31 : cluster 3 failed to encode map e111 with expected crc 2023-06-03T19:55:50.782 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:55:50 smithi089 bash[75068]: audit 1685822149.8689492 mon.c (mon.2) 301 : audit 0 from='client.? 172.21.15.89:0/1082583437' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:55:50.834 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:55:50 smithi164 bash[59462]: cluster 1685822149.430043 mon.a (mon.0) 518 : cluster 3 Health check failed: 1 osds down (OSD_DOWN) 2023-06-03T19:55:50.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:55:50 smithi164 bash[59462]: cluster 1685822149.4398828 osd.3 (osd.3) 35 : cluster 3 failed to encode map e111 with expected crc 2023-06-03T19:55:50.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:55:50 smithi164 bash[59462]: cluster 1685822149.4404998 osd.4 (osd.4) 24 : cluster 3 failed to encode map e111 with expected crc 2023-06-03T19:55:50.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:55:50 smithi164 bash[59462]: cluster 1685822149.4411604 osd.6 (osd.6) 26 : cluster 3 failed to encode map e111 with expected crc 2023-06-03T19:55:50.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:55:50 smithi164 bash[59462]: cluster 1685822149.4422512 mon.a (mon.0) 519 : cluster 0 osdmap e111: 8 total, 7 up, 8 in 2023-06-03T19:55:50.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:55:50 smithi164 bash[59462]: cluster 1685822149.4426196 osd.5 (osd.5) 31 : cluster 3 failed to encode map e111 with expected crc 2023-06-03T19:55:50.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:55:50 smithi164 bash[59462]: audit 1685822149.8689492 mon.c (mon.2) 301 : audit 0 from='client.? 172.21.15.89:0/1082583437' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:55:51.747 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:55:51 smithi089 bash[78452]: cluster 1685822150.3207736 mgr.y (mgr.34104) 568 : cluster 0 pgmap v476: 73 pgs: 3 peering, 1 stale+active+clean, 69 active+clean; 582 KiB data, 2.2 GiB used, 713 GiB / 715 GiB avail 2023-06-03T19:55:51.747 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:55:51 smithi089 bash[78452]: cluster 1685822150.4388487 mon.a (mon.0) 520 : cluster 3 Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2023-06-03T19:55:51.748 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:55:51 smithi089 bash[78452]: cluster 1685822150.4423387 osd.6 (osd.6) 27 : cluster 3 failed to encode map e112 with expected crc 2023-06-03T19:55:51.748 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:55:51 smithi089 bash[78452]: cluster 1685822150.4423807 osd.5 (osd.5) 32 : cluster 3 failed to encode map e112 with expected crc 2023-06-03T19:55:51.748 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:55:51 smithi089 bash[78452]: cluster 1685822150.4462721 osd.6 (osd.6) 28 : cluster 3 failed to encode map e112 with expected crc 2023-06-03T19:55:51.748 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:55:51 smithi089 bash[78452]: cluster 1685822150.4464796 osd.4 (osd.4) 25 : cluster 3 failed to encode map e112 with expected crc 2023-06-03T19:55:51.748 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:55:51 smithi089 bash[78452]: cluster 1685822150.4465778 osd.7 (osd.7) 23 : cluster 3 failed to encode map e112 with expected crc 2023-06-03T19:55:51.748 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:55:51 smithi089 bash[78452]: cluster 1685822150.448462 mon.a (mon.0) 521 : cluster 0 osdmap e112: 8 total, 7 up, 8 in 2023-06-03T19:55:51.748 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:55:51 smithi089 bash[78452]: cluster 1685822150.4489102 osd.3 (osd.3) 36 : cluster 3 failed to encode map e112 with expected crc 2023-06-03T19:55:51.749 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:55:51 smithi089 bash[75068]: cluster 1685822150.3207736 mgr.y (mgr.34104) 568 : cluster 0 pgmap v476: 73 pgs: 3 peering, 1 stale+active+clean, 69 active+clean; 582 KiB data, 2.2 GiB used, 713 GiB / 715 GiB avail 2023-06-03T19:55:51.749 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:55:51 smithi089 bash[75068]: cluster 1685822150.4388487 mon.a (mon.0) 520 : cluster 3 Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2023-06-03T19:55:51.749 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:55:51 smithi089 bash[75068]: cluster 1685822150.4423387 osd.6 (osd.6) 27 : cluster 3 failed to encode map e112 with expected crc 2023-06-03T19:55:51.749 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:55:51 smithi089 bash[75068]: cluster 1685822150.4423807 osd.5 (osd.5) 32 : cluster 3 failed to encode map e112 with expected crc 2023-06-03T19:55:51.749 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:55:51 smithi089 bash[75068]: cluster 1685822150.4462721 osd.6 (osd.6) 28 : cluster 3 failed to encode map e112 with expected crc 2023-06-03T19:55:51.750 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:55:51 smithi089 bash[75068]: cluster 1685822150.4464796 osd.4 (osd.4) 25 : cluster 3 failed to encode map e112 with expected crc 2023-06-03T19:55:51.750 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:55:51 smithi089 bash[75068]: cluster 1685822150.4465778 osd.7 (osd.7) 23 : cluster 3 failed to encode map e112 with expected crc 2023-06-03T19:55:51.750 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:55:51 smithi089 bash[75068]: cluster 1685822150.448462 mon.a (mon.0) 521 : cluster 0 osdmap e112: 8 total, 7 up, 8 in 2023-06-03T19:55:51.750 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:55:51 smithi089 bash[75068]: cluster 1685822150.4489102 osd.3 (osd.3) 36 : cluster 3 failed to encode map e112 with expected crc 2023-06-03T19:55:51.834 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:55:51 smithi164 bash[59462]: cluster 1685822150.3207736 mgr.y (mgr.34104) 568 : cluster 0 pgmap v476: 73 pgs: 3 peering, 1 stale+active+clean, 69 active+clean; 582 KiB data, 2.2 GiB used, 713 GiB / 715 GiB avail 2023-06-03T19:55:51.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:55:51 smithi164 bash[59462]: cluster 1685822150.4388487 mon.a (mon.0) 520 : cluster 3 Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2023-06-03T19:55:51.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:55:51 smithi164 bash[59462]: cluster 1685822150.4423387 osd.6 (osd.6) 27 : cluster 3 failed to encode map e112 with expected crc 2023-06-03T19:55:51.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:55:51 smithi164 bash[59462]: cluster 1685822150.4423807 osd.5 (osd.5) 32 : cluster 3 failed to encode map e112 with expected crc 2023-06-03T19:55:51.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:55:51 smithi164 bash[59462]: cluster 1685822150.4462721 osd.6 (osd.6) 28 : cluster 3 failed to encode map e112 with expected crc 2023-06-03T19:55:51.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:55:51 smithi164 bash[59462]: cluster 1685822150.4464796 osd.4 (osd.4) 25 : cluster 3 failed to encode map e112 with expected crc 2023-06-03T19:55:51.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:55:51 smithi164 bash[59462]: cluster 1685822150.4465778 osd.7 (osd.7) 23 : cluster 3 failed to encode map e112 with expected crc 2023-06-03T19:55:51.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:55:51 smithi164 bash[59462]: cluster 1685822150.448462 mon.a (mon.0) 521 : cluster 0 osdmap e112: 8 total, 7 up, 8 in 2023-06-03T19:55:51.836 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:55:51 smithi164 bash[59462]: cluster 1685822150.4489102 osd.3 (osd.3) 36 : cluster 3 failed to encode map e112 with expected crc 2023-06-03T19:55:53.203 INFO:journalctl@ceph.osd.2.smithi089.stdout:Jun 03 19:55:52 smithi089 systemd[1]: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c@osd.2.service: Succeeded. 2023-06-03T19:55:53.204 INFO:journalctl@ceph.osd.2.smithi089.stdout:Jun 03 19:55:52 smithi089 systemd[1]: Stopped Ceph osd.2 for f1db3478-0243-11ee-9b1d-001a4aab830c. 2023-06-03T19:55:53.204 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:55:53 smithi089 bash[75068]: audit 1685822152.0929627 mon.a (mon.0) 522 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:55:53.205 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:55:53 smithi089 bash[75068]: audit 1685822152.0956104 mon.c (mon.2) 302 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T19:55:53.205 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:55:53 smithi089 bash[75068]: audit 1685822152.2265897 mon.c (mon.2) 303 : audit 0 from='client.? 172.21.15.89:0/1516265346' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:55:53.205 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:55:53 smithi089 bash[75068]: cluster 1685822152.321558 mgr.y (mgr.34104) 569 : cluster 0 pgmap v478: 73 pgs: 2 active+undersized, 1 active+undersized+degraded, 3 peering, 1 stale+active+clean, 66 active+clean; 582 KiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 2/72 objects degraded (2.778%) 2023-06-03T19:55:53.205 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:55:53 smithi089 bash[78452]: audit 1685822152.0929627 mon.a (mon.0) 522 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:55:53.205 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:55:53 smithi089 bash[78452]: audit 1685822152.0956104 mon.c (mon.2) 302 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T19:55:53.205 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:55:53 smithi089 bash[78452]: audit 1685822152.2265897 mon.c (mon.2) 303 : audit 0 from='client.? 172.21.15.89:0/1516265346' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:55:53.206 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:55:53 smithi089 bash[78452]: cluster 1685822152.321558 mgr.y (mgr.34104) 569 : cluster 0 pgmap v478: 73 pgs: 2 active+undersized, 1 active+undersized+degraded, 3 peering, 1 stale+active+clean, 66 active+clean; 582 KiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 2/72 objects degraded (2.778%) 2023-06-03T19:55:53.528 INFO:journalctl@ceph.osd.2.smithi089.stdout:Jun 03 19:55:53 smithi089 systemd[1]: Started Ceph osd.2 for f1db3478-0243-11ee-9b1d-001a4aab830c. 2023-06-03T19:55:53.584 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:55:53 smithi164 bash[59462]: audit 1685822152.0929627 mon.a (mon.0) 522 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:55:53.585 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:55:53 smithi164 bash[59462]: audit 1685822152.0956104 mon.c (mon.2) 302 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T19:55:53.585 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:55:53 smithi164 bash[59462]: audit 1685822152.2265897 mon.c (mon.2) 303 : audit 0 from='client.? 172.21.15.89:0/1516265346' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:55:53.585 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:55:53 smithi164 bash[59462]: cluster 1685822152.321558 mgr.y (mgr.34104) 569 : cluster 0 pgmap v478: 73 pgs: 2 active+undersized, 1 active+undersized+degraded, 3 peering, 1 stale+active+clean, 66 active+clean; 582 KiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 2/72 objects degraded (2.778%) 2023-06-03T19:55:54.392 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:55:54 smithi089 bash[78452]: cluster 1685822153.089716 mon.a (mon.0) 523 : cluster 3 Health check failed: Degraded data redundancy: 2/72 objects degraded (2.778%), 1 pg degraded (PG_DEGRADED) 2023-06-03T19:55:54.392 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:55:54 smithi089 bash[78452]: audit 1685822153.2370002 mon.a (mon.0) 524 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:55:54.392 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:55:54 smithi089 bash[78452]: audit 1685822153.2487793 mon.a (mon.0) 525 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:55:54.392 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:55:54 smithi089 bash[78452]: audit 1685822153.278019 mon.c (mon.2) 304 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-03T19:55:54.393 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:55:54 smithi089 bash[75068]: cluster 1685822153.089716 mon.a (mon.0) 523 : cluster 3 Health check failed: Degraded data redundancy: 2/72 objects degraded (2.778%), 1 pg degraded (PG_DEGRADED) 2023-06-03T19:55:54.393 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:55:54 smithi089 bash[75068]: audit 1685822153.2370002 mon.a (mon.0) 524 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:55:54.393 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:55:54 smithi089 bash[75068]: audit 1685822153.2487793 mon.a (mon.0) 525 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:55:54.393 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:55:54 smithi089 bash[75068]: audit 1685822153.278019 mon.c (mon.2) 304 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-03T19:55:54.584 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:55:54 smithi164 bash[59462]: cluster 1685822153.089716 mon.a (mon.0) 523 : cluster 3 Health check failed: Degraded data redundancy: 2/72 objects degraded (2.778%), 1 pg degraded (PG_DEGRADED) 2023-06-03T19:55:54.584 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:55:54 smithi164 bash[59462]: audit 1685822153.2370002 mon.a (mon.0) 524 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:55:54.585 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:55:54 smithi164 bash[59462]: audit 1685822153.2487793 mon.a (mon.0) 525 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:55:54.585 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:55:54 smithi164 bash[59462]: audit 1685822153.278019 mon.c (mon.2) 304 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-03T19:55:55.529 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:55:55 smithi089 bash[75068]: cluster 1685822154.3225684 mgr.y (mgr.34104) 570 : cluster 0 pgmap v479: 73 pgs: 14 active+undersized, 2 active+undersized+degraded, 3 peering, 54 active+clean; 582 KiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2023-06-03T19:55:55.529 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:55:55 smithi089 bash[75068]: audit 1685822154.603224 mon.a (mon.0) 526 : audit 0 from='client.? 172.21.15.89:0/3974241177' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:55:55.529 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:55:55 smithi089 bash[78452]: cluster 1685822154.3225684 mgr.y (mgr.34104) 570 : cluster 0 pgmap v479: 73 pgs: 14 active+undersized, 2 active+undersized+degraded, 3 peering, 54 active+clean; 582 KiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2023-06-03T19:55:55.529 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:55:55 smithi089 bash[78452]: audit 1685822154.603224 mon.a (mon.0) 526 : audit 0 from='client.? 172.21.15.89:0/3974241177' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:55:55.530 INFO:journalctl@ceph.osd.2.smithi089.stdout:Jun 03 19:55:55 smithi089 bash[98184]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-2 2023-06-03T19:55:55.530 INFO:journalctl@ceph.osd.2.smithi089.stdout:Jun 03 19:55:55 smithi089 bash[98184]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-2 --no-mon-config --dev /dev/mapper/vg_nvme-lv_2 2023-06-03T19:55:55.530 INFO:journalctl@ceph.osd.2.smithi089.stdout:Jun 03 19:55:55 smithi089 bash[98184]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_2 2023-06-03T19:55:55.530 INFO:journalctl@ceph.osd.2.smithi089.stdout:Jun 03 19:55:55 smithi089 bash[98184]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-1 2023-06-03T19:55:55.530 INFO:journalctl@ceph.osd.2.smithi089.stdout:Jun 03 19:55:55 smithi089 bash[98184]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_2 /var/lib/ceph/osd/ceph-2/block 2023-06-03T19:55:55.530 INFO:journalctl@ceph.osd.2.smithi089.stdout:Jun 03 19:55:55 smithi089 bash[98184]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-2 2023-06-03T19:55:55.531 INFO:journalctl@ceph.osd.2.smithi089.stdout:Jun 03 19:55:55 smithi089 bash[98184]: --> ceph-volume raw activate successful for osd ID: 2 2023-06-03T19:55:55.584 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:55:55 smithi164 bash[59462]: cluster 1685822154.3225684 mgr.y (mgr.34104) 570 : cluster 0 pgmap v479: 73 pgs: 14 active+undersized, 2 active+undersized+degraded, 3 peering, 54 active+clean; 582 KiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2023-06-03T19:55:55.584 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:55:55 smithi164 bash[59462]: audit 1685822154.603224 mon.a (mon.0) 526 : audit 0 from='client.? 172.21.15.89:0/3974241177' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:55:56.529 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:55:56 smithi089 bash[78452]: cluster 1685822156.3756733 mon.a (mon.0) 527 : cluster 1 Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2023-06-03T19:55:56.529 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:55:56 smithi089 bash[75068]: cluster 1685822156.3756733 mon.a (mon.0) 527 : cluster 1 Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2023-06-03T19:55:56.530 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 19:55:56 smithi089 bash[71163]: ::ffff:172.21.15.164 - - [03/Jun/2023:19:55:56] "GET /metrics HTTP/1.1" 200 210402 "" "Prometheus/2.33.4" 2023-06-03T19:55:56.834 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:55:56 smithi164 bash[59462]: cluster 1685822156.3756733 mon.a (mon.0) 527 : cluster 1 Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2023-06-03T19:55:57.279 INFO:journalctl@ceph.osd.2.smithi089.stdout:Jun 03 19:55:57 smithi089 bash[98494]: debug 2023-06-03T19:55:57.020+0000 7f63cd3ae700 1 -- 172.21.15.89:0/2402949382 <== mon.2 v2:172.21.15.89:3301/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x5558a7212d00 con 0x5558a726d000 2023-06-03T19:55:57.779 INFO:journalctl@ceph.osd.2.smithi089.stdout:Jun 03 19:55:57 smithi089 bash[98494]: debug 2023-06-03T19:55:57.572+0000 7f63d6534640 -1 Falling back to public interface 2023-06-03T19:55:57.779 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:55:57 smithi089 bash[75068]: cluster 1685822156.3235006 mgr.y (mgr.34104) 571 : cluster 0 pgmap v480: 73 pgs: 17 active+undersized, 2 active+undersized+degraded, 54 active+clean; 582 KiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2023-06-03T19:55:57.780 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:55:57 smithi089 bash[75068]: audit 1685822156.96444 mon.c (mon.2) 305 : audit 0 from='client.? 172.21.15.89:0/3640170357' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:55:57.780 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:55:57 smithi089 bash[78452]: cluster 1685822156.3235006 mgr.y (mgr.34104) 571 : cluster 0 pgmap v480: 73 pgs: 17 active+undersized, 2 active+undersized+degraded, 54 active+clean; 582 KiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2023-06-03T19:55:57.780 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:55:57 smithi089 bash[78452]: audit 1685822156.96444 mon.c (mon.2) 305 : audit 0 from='client.? 172.21.15.89:0/3640170357' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:55:57.834 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:55:57 smithi164 bash[59462]: cluster 1685822156.3235006 mgr.y (mgr.34104) 571 : cluster 0 pgmap v480: 73 pgs: 17 active+undersized, 2 active+undersized+degraded, 54 active+clean; 582 KiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2023-06-03T19:55:57.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:55:57 smithi164 bash[59462]: audit 1685822156.96444 mon.c (mon.2) 305 : audit 0 from='client.? 172.21.15.89:0/3640170357' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:55:59.779 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:55:59 smithi089 bash[78452]: cluster 1685822158.3243275 mgr.y (mgr.34104) 572 : cluster 0 pgmap v481: 73 pgs: 17 active+undersized, 2 active+undersized+degraded, 54 active+clean; 582 KiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2023-06-03T19:55:59.779 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:55:59 smithi089 bash[78452]: audit 1685822159.3227093 mon.a (mon.0) 528 : audit 0 from='client.? 172.21.15.89:0/1316930540' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:55:59.779 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:55:59 smithi089 bash[75068]: cluster 1685822158.3243275 mgr.y (mgr.34104) 572 : cluster 0 pgmap v481: 73 pgs: 17 active+undersized, 2 active+undersized+degraded, 54 active+clean; 582 KiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2023-06-03T19:55:59.780 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:55:59 smithi089 bash[75068]: audit 1685822159.3227093 mon.a (mon.0) 528 : audit 0 from='client.? 172.21.15.89:0/1316930540' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:55:59.834 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:55:59 smithi164 bash[59462]: cluster 1685822158.3243275 mgr.y (mgr.34104) 572 : cluster 0 pgmap v481: 73 pgs: 17 active+undersized, 2 active+undersized+degraded, 54 active+clean; 582 KiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2023-06-03T19:55:59.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:55:59 smithi164 bash[59462]: audit 1685822159.3227093 mon.a (mon.0) 528 : audit 0 from='client.? 172.21.15.89:0/1316930540' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:56:02.029 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:56:01 smithi089 bash[78452]: cluster 1685822160.325284 mgr.y (mgr.34104) 573 : cluster 0 pgmap v482: 73 pgs: 17 active+undersized, 2 active+undersized+degraded, 54 active+clean; 582 KiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2023-06-03T19:56:02.029 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:56:01 smithi089 bash[78452]: audit 1685822160.6741185 mon.a (mon.0) 529 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:56:02.029 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:56:01 smithi089 bash[78452]: audit 1685822160.6851702 mon.a (mon.0) 530 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:56:02.029 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:56:01 smithi089 bash[78452]: audit 1685822161.4833524 mon.a (mon.0) 531 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:56:02.029 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:56:01 smithi089 bash[78452]: audit 1685822161.4955363 mon.a (mon.0) 532 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:56:02.030 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:56:01 smithi089 bash[75068]: cluster 1685822160.325284 mgr.y (mgr.34104) 573 : cluster 0 pgmap v482: 73 pgs: 17 active+undersized, 2 active+undersized+degraded, 54 active+clean; 582 KiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2023-06-03T19:56:02.030 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:56:01 smithi089 bash[75068]: audit 1685822160.6741185 mon.a (mon.0) 529 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:56:02.030 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:56:01 smithi089 bash[75068]: audit 1685822160.6851702 mon.a (mon.0) 530 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:56:02.030 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:56:01 smithi089 bash[75068]: audit 1685822161.4833524 mon.a (mon.0) 531 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:56:02.030 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:56:01 smithi089 bash[75068]: audit 1685822161.4955363 mon.a (mon.0) 532 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:56:02.084 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:56:01 smithi164 bash[59462]: cluster 1685822160.325284 mgr.y (mgr.34104) 573 : cluster 0 pgmap v482: 73 pgs: 17 active+undersized, 2 active+undersized+degraded, 54 active+clean; 582 KiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2023-06-03T19:56:02.085 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:56:01 smithi164 bash[59462]: audit 1685822160.6741185 mon.a (mon.0) 529 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:56:02.085 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:56:01 smithi164 bash[59462]: audit 1685822160.6851702 mon.a (mon.0) 530 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:56:02.085 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:56:01 smithi164 bash[59462]: audit 1685822161.4833524 mon.a (mon.0) 531 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:56:02.085 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:56:01 smithi164 bash[59462]: audit 1685822161.4955363 mon.a (mon.0) 532 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:56:03.029 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:56:02 smithi089 bash[78452]: audit 1685822161.6873772 mon.a (mon.0) 533 : audit 0 from='client.? 172.21.15.89:0/2340682593' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:56:03.029 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:56:02 smithi089 bash[78452]: audit 1685822162.3545723 mon.a (mon.0) 534 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:56:03.029 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:56:02 smithi089 bash[78452]: cluster 1685822162.4296257 mon.a (mon.0) 535 : cluster 3 Health check update: Degraded data redundancy: 4/72 objects degraded (5.556%), 2 pgs degraded (PG_DEGRADED) 2023-06-03T19:56:03.029 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:56:02 smithi089 bash[75068]: audit 1685822161.6873772 mon.a (mon.0) 533 : audit 0 from='client.? 172.21.15.89:0/2340682593' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:56:03.030 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:56:02 smithi089 bash[75068]: audit 1685822162.3545723 mon.a (mon.0) 534 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:56:03.030 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:56:02 smithi089 bash[75068]: cluster 1685822162.4296257 mon.a (mon.0) 535 : cluster 3 Health check update: Degraded data redundancy: 4/72 objects degraded (5.556%), 2 pgs degraded (PG_DEGRADED) 2023-06-03T19:56:03.084 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:56:02 smithi164 bash[59462]: audit 1685822161.6873772 mon.a (mon.0) 533 : audit 0 from='client.? 172.21.15.89:0/2340682593' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:56:03.084 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:56:02 smithi164 bash[59462]: audit 1685822162.3545723 mon.a (mon.0) 534 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:56:03.085 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:56:02 smithi164 bash[59462]: cluster 1685822162.4296257 mon.a (mon.0) 535 : cluster 3 Health check update: Degraded data redundancy: 4/72 objects degraded (5.556%), 2 pgs degraded (PG_DEGRADED) 2023-06-03T19:56:04.029 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:56:03 smithi089 bash[75068]: cluster 1685822162.3260658 mgr.y (mgr.34104) 574 : cluster 0 pgmap v483: 73 pgs: 17 active+undersized, 2 active+undersized+degraded, 54 active+clean; 582 KiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2023-06-03T19:56:04.029 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:56:03 smithi089 bash[78452]: cluster 1685822162.3260658 mgr.y (mgr.34104) 574 : cluster 0 pgmap v483: 73 pgs: 17 active+undersized, 2 active+undersized+degraded, 54 active+clean; 582 KiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2023-06-03T19:56:04.085 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:56:03 smithi164 bash[59462]: cluster 1685822162.3260658 mgr.y (mgr.34104) 574 : cluster 0 pgmap v483: 73 pgs: 17 active+undersized, 2 active+undersized+degraded, 54 active+clean; 582 KiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2023-06-03T19:56:05.029 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:56:04 smithi089 bash[75068]: audit 1685822164.0689151 mon.c (mon.2) 306 : audit 0 from='client.? 172.21.15.89:0/460976980' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:56:05.029 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:56:04 smithi089 bash[78452]: audit 1685822164.0689151 mon.c (mon.2) 306 : audit 0 from='client.? 172.21.15.89:0/460976980' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:56:05.084 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:56:04 smithi164 bash[59462]: audit 1685822164.0689151 mon.c (mon.2) 306 : audit 0 from='client.? 172.21.15.89:0/460976980' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:56:06.029 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:56:05 smithi089 bash[75068]: cluster 1685822164.3269584 mgr.y (mgr.34104) 575 : cluster 0 pgmap v484: 73 pgs: 17 active+undersized, 2 active+undersized+degraded, 54 active+clean; 582 KiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2023-06-03T19:56:06.029 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:56:05 smithi089 bash[78452]: cluster 1685822164.3269584 mgr.y (mgr.34104) 575 : cluster 0 pgmap v484: 73 pgs: 17 active+undersized, 2 active+undersized+degraded, 54 active+clean; 582 KiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2023-06-03T19:56:06.084 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:56:05 smithi164 bash[59462]: cluster 1685822164.3269584 mgr.y (mgr.34104) 575 : cluster 0 pgmap v484: 73 pgs: 17 active+undersized, 2 active+undersized+degraded, 54 active+clean; 582 KiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2023-06-03T19:56:06.528 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 19:56:06 smithi089 bash[71163]: ::ffff:172.21.15.164 - - [03/Jun/2023:19:56:06] "GET /metrics HTTP/1.1" 200 210402 "" "Prometheus/2.33.4" 2023-06-03T19:56:07.028 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:56:06 smithi089 bash[78452]: audit 1685822166.41938 mon.c (mon.2) 307 : audit 0 from='client.? 172.21.15.89:0/1107261391' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:56:07.029 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:56:06 smithi089 bash[75068]: audit 1685822166.41938 mon.c (mon.2) 307 : audit 0 from='client.? 172.21.15.89:0/1107261391' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:56:07.084 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:56:06 smithi164 bash[59462]: audit 1685822166.41938 mon.c (mon.2) 307 : audit 0 from='client.? 172.21.15.89:0/1107261391' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:56:07.960 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:56:07 smithi089 bash[75068]: cluster 1685822166.327492 mgr.y (mgr.34104) 576 : cluster 0 pgmap v485: 73 pgs: 17 active+undersized, 2 active+undersized+degraded, 54 active+clean; 582 KiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2023-06-03T19:56:07.961 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:56:07 smithi089 bash[75068]: audit 1685822167.0917785 mon.a (mon.0) 536 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:56:07.961 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:56:07 smithi089 bash[75068]: audit 1685822167.114762 mon.c (mon.2) 308 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T19:56:07.961 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:56:07 smithi089 bash[78452]: cluster 1685822166.327492 mgr.y (mgr.34104) 576 : cluster 0 pgmap v485: 73 pgs: 17 active+undersized, 2 active+undersized+degraded, 54 active+clean; 582 KiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2023-06-03T19:56:07.961 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:56:07 smithi089 bash[78452]: audit 1685822167.0917785 mon.a (mon.0) 536 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:56:07.961 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:56:07 smithi089 bash[78452]: audit 1685822167.114762 mon.c (mon.2) 308 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T19:56:08.084 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:56:07 smithi164 bash[59462]: cluster 1685822166.327492 mgr.y (mgr.34104) 576 : cluster 0 pgmap v485: 73 pgs: 17 active+undersized, 2 active+undersized+degraded, 54 active+clean; 582 KiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2023-06-03T19:56:08.084 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:56:07 smithi164 bash[59462]: audit 1685822167.0917785 mon.a (mon.0) 536 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:56:08.085 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:56:07 smithi164 bash[59462]: audit 1685822167.114762 mon.c (mon.2) 308 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T19:56:09.718 INFO:journalctl@ceph.osd.2.smithi089.stdout:Jun 03 19:56:09 smithi089 bash[98494]: debug 2023-06-03T19:56:09.447+0000 7f63d6534640 -1 osd.2 110 log_to_monitors true 2023-06-03T19:56:10.028 INFO:journalctl@ceph.osd.2.smithi089.stdout:Jun 03 19:56:09 smithi089 bash[98494]: debug 2023-06-03T19:56:09.763+0000 7f63c619d700 -1 osd.2 110 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2023-06-03T19:56:10.029 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:56:09 smithi089 bash[78452]: cluster 1685822168.328176 mgr.y (mgr.34104) 577 : cluster 0 pgmap v486: 73 pgs: 17 active+undersized, 2 active+undersized+degraded, 54 active+clean; 582 KiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2023-06-03T19:56:10.029 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:56:09 smithi089 bash[78452]: audit 1685822168.7768745 mon.a (mon.0) 537 : audit 0 from='client.? 172.21.15.89:0/1088072332' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:56:10.029 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:56:09 smithi089 bash[78452]: audit 1685822169.4560761 mon.c (mon.2) 309 : audit 1 from='osd.2 [v2:172.21.15.89:6818/1439643284,v1:172.21.15.89:6819/1439643284]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2023-06-03T19:56:10.029 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:56:09 smithi089 bash[78452]: audit 1685822169.456583 mon.a (mon.0) 538 : audit 1 from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2023-06-03T19:56:10.030 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:56:09 smithi089 bash[75068]: cluster 1685822168.328176 mgr.y (mgr.34104) 577 : cluster 0 pgmap v486: 73 pgs: 17 active+undersized, 2 active+undersized+degraded, 54 active+clean; 582 KiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2023-06-03T19:56:10.030 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:56:09 smithi089 bash[75068]: audit 1685822168.7768745 mon.a (mon.0) 537 : audit 0 from='client.? 172.21.15.89:0/1088072332' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:56:10.030 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:56:09 smithi089 bash[75068]: audit 1685822169.4560761 mon.c (mon.2) 309 : audit 1 from='osd.2 [v2:172.21.15.89:6818/1439643284,v1:172.21.15.89:6819/1439643284]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2023-06-03T19:56:10.030 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:56:09 smithi089 bash[75068]: audit 1685822169.456583 mon.a (mon.0) 538 : audit 1 from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2023-06-03T19:56:10.084 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:56:09 smithi164 bash[59462]: cluster 1685822168.328176 mgr.y (mgr.34104) 577 : cluster 0 pgmap v486: 73 pgs: 17 active+undersized, 2 active+undersized+degraded, 54 active+clean; 582 KiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2023-06-03T19:56:10.085 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:56:09 smithi164 bash[59462]: audit 1685822168.7768745 mon.a (mon.0) 537 : audit 0 from='client.? 172.21.15.89:0/1088072332' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:56:10.085 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:56:09 smithi164 bash[59462]: audit 1685822169.4560761 mon.c (mon.2) 309 : audit 1 from='osd.2 [v2:172.21.15.89:6818/1439643284,v1:172.21.15.89:6819/1439643284]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2023-06-03T19:56:10.085 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:56:09 smithi164 bash[59462]: audit 1685822169.456583 mon.a (mon.0) 538 : audit 1 from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2023-06-03T19:56:11.028 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:56:10 smithi089 bash[75068]: cluster 1685822169.7191718 osd.7 (osd.7) 24 : cluster 3 failed to encode map e113 with expected crc 2023-06-03T19:56:11.029 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:56:10 smithi089 bash[75068]: cluster 1685822169.719263 osd.5 (osd.5) 33 : cluster 3 failed to encode map e113 with expected crc 2023-06-03T19:56:11.029 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:56:10 smithi089 bash[75068]: audit 1685822169.7247672 mon.a (mon.0) 539 : audit 1 from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2023-06-03T19:56:11.029 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:56:10 smithi089 bash[75068]: cluster 1685822169.724915 mon.a (mon.0) 540 : cluster 0 osdmap e113: 8 total, 7 up, 8 in 2023-06-03T19:56:11.029 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:56:10 smithi089 bash[75068]: audit 1685822169.7256503 mon.c (mon.2) 310 : audit 1 from='osd.2 [v2:172.21.15.89:6818/1439643284,v1:172.21.15.89:6819/1439643284]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi089", "root=default"]}]: dispatch 2023-06-03T19:56:11.029 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:56:10 smithi089 bash[75068]: audit 1685822169.7260158 mon.a (mon.0) 541 : audit 1 from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi089", "root=default"]}]: dispatch 2023-06-03T19:56:11.030 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:56:10 smithi089 bash[78452]: cluster 1685822169.7191718 osd.7 (osd.7) 24 : cluster 3 failed to encode map e113 with expected crc 2023-06-03T19:56:11.030 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:56:10 smithi089 bash[78452]: cluster 1685822169.719263 osd.5 (osd.5) 33 : cluster 3 failed to encode map e113 with expected crc 2023-06-03T19:56:11.030 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:56:10 smithi089 bash[78452]: audit 1685822169.7247672 mon.a (mon.0) 539 : audit 1 from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2023-06-03T19:56:11.030 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:56:10 smithi089 bash[78452]: cluster 1685822169.724915 mon.a (mon.0) 540 : cluster 0 osdmap e113: 8 total, 7 up, 8 in 2023-06-03T19:56:11.030 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:56:10 smithi089 bash[78452]: audit 1685822169.7256503 mon.c (mon.2) 310 : audit 1 from='osd.2 [v2:172.21.15.89:6818/1439643284,v1:172.21.15.89:6819/1439643284]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi089", "root=default"]}]: dispatch 2023-06-03T19:56:11.031 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:56:10 smithi089 bash[78452]: audit 1685822169.7260158 mon.a (mon.0) 541 : audit 1 from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi089", "root=default"]}]: dispatch 2023-06-03T19:56:11.084 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:56:10 smithi164 bash[59462]: cluster 1685822169.7191718 osd.7 (osd.7) 24 : cluster 3 failed to encode map e113 with expected crc 2023-06-03T19:56:11.085 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:56:10 smithi164 bash[59462]: cluster 1685822169.719263 osd.5 (osd.5) 33 : cluster 3 failed to encode map e113 with expected crc 2023-06-03T19:56:11.085 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:56:10 smithi164 bash[59462]: audit 1685822169.7247672 mon.a (mon.0) 539 : audit 1 from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2023-06-03T19:56:11.085 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:56:10 smithi164 bash[59462]: cluster 1685822169.724915 mon.a (mon.0) 540 : cluster 0 osdmap e113: 8 total, 7 up, 8 in 2023-06-03T19:56:11.085 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:56:10 smithi164 bash[59462]: audit 1685822169.7256503 mon.c (mon.2) 310 : audit 1 from='osd.2 [v2:172.21.15.89:6818/1439643284,v1:172.21.15.89:6819/1439643284]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi089", "root=default"]}]: dispatch 2023-06-03T19:56:11.085 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:56:10 smithi164 bash[59462]: audit 1685822169.7260158 mon.a (mon.0) 541 : audit 1 from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi089", "root=default"]}]: dispatch 2023-06-03T19:56:11.746 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 19:56:11 smithi089 bash[71163]: debug 2023-06-03T19:56:11.411+0000 7fe48642e700 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (7 PGs are or would become offline) 2023-06-03T19:56:12.029 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:56:11 smithi089 bash[78452]: cluster 1685822170.329035 mgr.y (mgr.34104) 578 : cluster 0 pgmap v488: 73 pgs: 17 active+undersized, 2 active+undersized+degraded, 54 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2023-06-03T19:56:12.030 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:56:11 smithi089 bash[78452]: cluster 1685822170.7212777 mon.a (mon.0) 542 : cluster 1 Health check cleared: OSD_DOWN (was: 1 osds down) 2023-06-03T19:56:12.030 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:56:11 smithi089 bash[78452]: cluster 1685822170.7293122 osd.3 (osd.3) 37 : cluster 3 failed to encode map e114 with expected crc 2023-06-03T19:56:12.030 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:56:11 smithi089 bash[78452]: cluster 1685822170.7343345 osd.4 (osd.4) 26 : cluster 3 failed to encode map e113 with expected crc 2023-06-03T19:56:12.030 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:56:11 smithi089 bash[78452]: cluster 1685822170.7365906 mon.a (mon.0) 543 : cluster 1 osd.2 [v2:172.21.15.89:6818/1439643284,v1:172.21.15.89:6819/1439643284] boot 2023-06-03T19:56:12.030 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:56:11 smithi089 bash[78452]: cluster 1685822170.7366817 mon.a (mon.0) 544 : cluster 0 osdmap e114: 8 total, 8 up, 8 in 2023-06-03T19:56:12.031 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:56:11 smithi089 bash[78452]: audit 1685822170.7383852 mon.c (mon.2) 311 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-06-03T19:56:12.031 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:56:11 smithi089 bash[78452]: cluster 1685822170.7395978 osd.5 (osd.5) 34 : cluster 3 failed to encode map e114 with expected crc 2023-06-03T19:56:12.031 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:56:11 smithi089 bash[78452]: audit 1685822171.12176 mon.a (mon.0) 545 : audit 0 from='client.? 172.21.15.89:0/1256387735' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:56:12.031 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:56:11 smithi089 bash[78452]: audit 1685822171.2928212 mon.a (mon.0) 546 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:56:12.031 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:56:11 smithi089 bash[78452]: audit 1685822171.3054724 mon.a (mon.0) 547 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:56:12.031 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:56:11 smithi089 bash[78452]: audit 1685822171.3083057 mon.c (mon.2) 312 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-03T19:56:12.031 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:56:11 smithi089 bash[78452]: audit 1685822171.3097641 mon.c (mon.2) 313 : audit 1 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-03T19:56:12.032 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:56:11 smithi089 bash[78452]: audit 1685822171.3173888 mon.a (mon.0) 548 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:56:12.032 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:56:11 smithi089 bash[78452]: audit 1685822171.4027512 mon.c (mon.2) 314 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-03T19:56:12.032 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:56:11 smithi089 bash[78452]: audit 1685822171.408049 mon.c (mon.2) 315 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-06-03T19:56:12.032 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:56:11 smithi089 bash[78452]: audit 1685822171.410833 mon.c (mon.2) 316 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-06-03T19:56:12.032 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:56:11 smithi089 bash[78452]: audit 1685822171.4127972 mon.c (mon.2) 317 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-06-03T19:56:12.032 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:56:11 smithi089 bash[78452]: audit 1685822171.415356 mon.c (mon.2) 318 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2023-06-03T19:56:12.033 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:56:11 smithi089 bash[75068]: cluster 1685822170.329035 mgr.y (mgr.34104) 578 : cluster 0 pgmap v488: 73 pgs: 17 active+undersized, 2 active+undersized+degraded, 54 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2023-06-03T19:56:12.033 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:56:11 smithi089 bash[75068]: cluster 1685822170.7212777 mon.a (mon.0) 542 : cluster 1 Health check cleared: OSD_DOWN (was: 1 osds down) 2023-06-03T19:56:12.033 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:56:11 smithi089 bash[75068]: cluster 1685822170.7293122 osd.3 (osd.3) 37 : cluster 3 failed to encode map e114 with expected crc 2023-06-03T19:56:12.033 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:56:11 smithi089 bash[75068]: cluster 1685822170.7343345 osd.4 (osd.4) 26 : cluster 3 failed to encode map e113 with expected crc 2023-06-03T19:56:12.033 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:56:11 smithi089 bash[75068]: cluster 1685822170.7365906 mon.a (mon.0) 543 : cluster 1 osd.2 [v2:172.21.15.89:6818/1439643284,v1:172.21.15.89:6819/1439643284] boot 2023-06-03T19:56:12.033 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:56:11 smithi089 bash[75068]: cluster 1685822170.7366817 mon.a (mon.0) 544 : cluster 0 osdmap e114: 8 total, 8 up, 8 in 2023-06-03T19:56:12.033 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:56:11 smithi089 bash[75068]: audit 1685822170.7383852 mon.c (mon.2) 311 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-06-03T19:56:12.033 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:56:11 smithi089 bash[75068]: cluster 1685822170.7395978 osd.5 (osd.5) 34 : cluster 3 failed to encode map e114 with expected crc 2023-06-03T19:56:12.034 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:56:11 smithi089 bash[75068]: audit 1685822171.12176 mon.a (mon.0) 545 : audit 0 from='client.? 172.21.15.89:0/1256387735' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:56:12.034 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:56:11 smithi089 bash[75068]: audit 1685822171.2928212 mon.a (mon.0) 546 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:56:12.034 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:56:11 smithi089 bash[75068]: audit 1685822171.3054724 mon.a (mon.0) 547 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:56:12.034 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:56:11 smithi089 bash[75068]: audit 1685822171.3083057 mon.c (mon.2) 312 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-03T19:56:12.034 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:56:11 smithi089 bash[75068]: audit 1685822171.3097641 mon.c (mon.2) 313 : audit 1 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-03T19:56:12.034 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:56:11 smithi089 bash[75068]: audit 1685822171.3173888 mon.a (mon.0) 548 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:56:12.034 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:56:11 smithi089 bash[75068]: audit 1685822171.4027512 mon.c (mon.2) 314 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-03T19:56:12.035 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:56:11 smithi089 bash[75068]: audit 1685822171.408049 mon.c (mon.2) 315 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-06-03T19:56:12.035 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:56:11 smithi089 bash[75068]: audit 1685822171.410833 mon.c (mon.2) 316 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-06-03T19:56:12.035 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:56:11 smithi089 bash[75068]: audit 1685822171.4127972 mon.c (mon.2) 317 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-06-03T19:56:12.035 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:56:11 smithi089 bash[75068]: audit 1685822171.415356 mon.c (mon.2) 318 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2023-06-03T19:56:12.085 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:56:11 smithi164 bash[59462]: cluster 1685822170.329035 mgr.y (mgr.34104) 578 : cluster 0 pgmap v488: 73 pgs: 17 active+undersized, 2 active+undersized+degraded, 54 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2023-06-03T19:56:12.085 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:56:11 smithi164 bash[59462]: cluster 1685822170.7212777 mon.a (mon.0) 542 : cluster 1 Health check cleared: OSD_DOWN (was: 1 osds down) 2023-06-03T19:56:12.085 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:56:11 smithi164 bash[59462]: cluster 1685822170.7293122 osd.3 (osd.3) 37 : cluster 3 failed to encode map e114 with expected crc 2023-06-03T19:56:12.085 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:56:11 smithi164 bash[59462]: cluster 1685822170.7343345 osd.4 (osd.4) 26 : cluster 3 failed to encode map e113 with expected crc 2023-06-03T19:56:12.086 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:56:11 smithi164 bash[59462]: cluster 1685822170.7365906 mon.a (mon.0) 543 : cluster 1 osd.2 [v2:172.21.15.89:6818/1439643284,v1:172.21.15.89:6819/1439643284] boot 2023-06-03T19:56:12.086 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:56:11 smithi164 bash[59462]: cluster 1685822170.7366817 mon.a (mon.0) 544 : cluster 0 osdmap e114: 8 total, 8 up, 8 in 2023-06-03T19:56:12.086 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:56:11 smithi164 bash[59462]: audit 1685822170.7383852 mon.c (mon.2) 311 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-06-03T19:56:12.086 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:56:11 smithi164 bash[59462]: cluster 1685822170.7395978 osd.5 (osd.5) 34 : cluster 3 failed to encode map e114 with expected crc 2023-06-03T19:56:12.086 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:56:11 smithi164 bash[59462]: audit 1685822171.12176 mon.a (mon.0) 545 : audit 0 from='client.? 172.21.15.89:0/1256387735' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:56:12.086 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:56:11 smithi164 bash[59462]: audit 1685822171.2928212 mon.a (mon.0) 546 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:56:12.086 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:56:11 smithi164 bash[59462]: audit 1685822171.3054724 mon.a (mon.0) 547 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:56:12.087 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:56:11 smithi164 bash[59462]: audit 1685822171.3083057 mon.c (mon.2) 312 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-03T19:56:12.087 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:56:11 smithi164 bash[59462]: audit 1685822171.3097641 mon.c (mon.2) 313 : audit 1 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-03T19:56:12.087 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:56:11 smithi164 bash[59462]: audit 1685822171.3173888 mon.a (mon.0) 548 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:56:12.087 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:56:11 smithi164 bash[59462]: audit 1685822171.4027512 mon.c (mon.2) 314 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-03T19:56:12.087 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:56:11 smithi164 bash[59462]: audit 1685822171.408049 mon.c (mon.2) 315 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-06-03T19:56:12.087 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:56:11 smithi164 bash[59462]: audit 1685822171.410833 mon.c (mon.2) 316 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-06-03T19:56:12.087 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:56:11 smithi164 bash[59462]: audit 1685822171.4127972 mon.c (mon.2) 317 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-06-03T19:56:12.087 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:56:11 smithi164 bash[59462]: audit 1685822171.415356 mon.c (mon.2) 318 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2023-06-03T19:56:13.029 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:56:12 smithi089 bash[75068]: audit 1685822171.4156227 mgr.y (mgr.34104) 579 : audit 0 from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2023-06-03T19:56:13.029 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:56:12 smithi089 bash[75068]: cephadm 1685822171.4169183 mgr.y (mgr.34104) 580 : cephadm 1 Upgrade: unsafe to stop osd(s) at this time (7 PGs are or would become offline) 2023-06-03T19:56:13.029 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:56:12 smithi089 bash[75068]: cluster 1685822171.7314317 osd.5 (osd.5) 35 : cluster 3 failed to encode map e115 with expected crc 2023-06-03T19:56:13.030 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:56:12 smithi089 bash[75068]: cluster 1685822171.7345092 osd.5 (osd.5) 36 : cluster 3 failed to encode map e115 with expected crc 2023-06-03T19:56:13.030 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:56:12 smithi089 bash[75068]: cluster 1685822171.736145 osd.4 (osd.4) 27 : cluster 3 failed to encode map e115 with expected crc 2023-06-03T19:56:13.030 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:56:12 smithi089 bash[75068]: cluster 1685822171.7362285 osd.3 (osd.3) 38 : cluster 3 failed to encode map e115 with expected crc 2023-06-03T19:56:13.030 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:56:12 smithi089 bash[75068]: cluster 1685822171.736326 osd.7 (osd.7) 25 : cluster 3 failed to encode map e115 with expected crc 2023-06-03T19:56:13.030 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:56:12 smithi089 bash[75068]: cluster 1685822171.7379823 osd.6 (osd.6) 29 : cluster 3 failed to encode map e115 with expected crc 2023-06-03T19:56:13.030 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:56:12 smithi089 bash[75068]: cluster 1685822171.7384076 mon.a (mon.0) 549 : cluster 0 osdmap e115: 8 total, 8 up, 8 in 2023-06-03T19:56:13.030 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:56:12 smithi089 bash[75068]: cluster 1685822171.7402396 osd.4 (osd.4) 28 : cluster 3 failed to encode map e115 with expected crc 2023-06-03T19:56:13.031 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:56:12 smithi089 bash[75068]: cluster 1685822171.7418954 osd.6 (osd.6) 30 : cluster 3 failed to encode map e115 with expected crc 2023-06-03T19:56:13.031 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:56:12 smithi089 bash[75068]: cluster 1685822171.7420068 osd.7 (osd.7) 26 : cluster 3 failed to encode map e115 with expected crc 2023-06-03T19:56:13.031 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:56:12 smithi089 bash[75068]: cluster 1685822171.7421906 osd.4 (osd.4) 29 : cluster 3 failed to encode map e115 with expected crc 2023-06-03T19:56:13.031 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:56:12 smithi089 bash[75068]: cluster 1685822171.7642097 osd.3 (osd.3) 39 : cluster 3 failed to encode map e115 with expected crc 2023-06-03T19:56:13.031 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:56:12 smithi089 bash[75068]: cluster 1685822171.7658498 osd.3 (osd.3) 40 : cluster 3 failed to encode map e115 with expected crc 2023-06-03T19:56:13.031 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:56:12 smithi089 bash[78452]: audit 1685822171.4156227 mgr.y (mgr.34104) 579 : audit 0 from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2023-06-03T19:56:13.032 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:56:12 smithi089 bash[78452]: cephadm 1685822171.4169183 mgr.y (mgr.34104) 580 : cephadm 1 Upgrade: unsafe to stop osd(s) at this time (7 PGs are or would become offline) 2023-06-03T19:56:13.032 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:56:12 smithi089 bash[78452]: cluster 1685822171.7314317 osd.5 (osd.5) 35 : cluster 3 failed to encode map e115 with expected crc 2023-06-03T19:56:13.032 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:56:12 smithi089 bash[78452]: cluster 1685822171.7345092 osd.5 (osd.5) 36 : cluster 3 failed to encode map e115 with expected crc 2023-06-03T19:56:13.032 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:56:12 smithi089 bash[78452]: cluster 1685822171.736145 osd.4 (osd.4) 27 : cluster 3 failed to encode map e115 with expected crc 2023-06-03T19:56:13.032 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:56:12 smithi089 bash[78452]: cluster 1685822171.7362285 osd.3 (osd.3) 38 : cluster 3 failed to encode map e115 with expected crc 2023-06-03T19:56:13.032 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:56:12 smithi089 bash[78452]: cluster 1685822171.736326 osd.7 (osd.7) 25 : cluster 3 failed to encode map e115 with expected crc 2023-06-03T19:56:13.032 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:56:12 smithi089 bash[78452]: cluster 1685822171.7379823 osd.6 (osd.6) 29 : cluster 3 failed to encode map e115 with expected crc 2023-06-03T19:56:13.033 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:56:12 smithi089 bash[78452]: cluster 1685822171.7384076 mon.a (mon.0) 549 : cluster 0 osdmap e115: 8 total, 8 up, 8 in 2023-06-03T19:56:13.033 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:56:12 smithi089 bash[78452]: cluster 1685822171.7402396 osd.4 (osd.4) 28 : cluster 3 failed to encode map e115 with expected crc 2023-06-03T19:56:13.033 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:56:12 smithi089 bash[78452]: cluster 1685822171.7418954 osd.6 (osd.6) 30 : cluster 3 failed to encode map e115 with expected crc 2023-06-03T19:56:13.033 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:56:12 smithi089 bash[78452]: cluster 1685822171.7420068 osd.7 (osd.7) 26 : cluster 3 failed to encode map e115 with expected crc 2023-06-03T19:56:13.033 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:56:12 smithi089 bash[78452]: cluster 1685822171.7421906 osd.4 (osd.4) 29 : cluster 3 failed to encode map e115 with expected crc 2023-06-03T19:56:13.033 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:56:12 smithi089 bash[78452]: cluster 1685822171.7642097 osd.3 (osd.3) 39 : cluster 3 failed to encode map e115 with expected crc 2023-06-03T19:56:13.033 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:56:12 smithi089 bash[78452]: cluster 1685822171.7658498 osd.3 (osd.3) 40 : cluster 3 failed to encode map e115 with expected crc 2023-06-03T19:56:13.084 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:56:12 smithi164 bash[59462]: audit 1685822171.4156227 mgr.y (mgr.34104) 579 : audit 0 from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2023-06-03T19:56:13.085 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:56:12 smithi164 bash[59462]: cephadm 1685822171.4169183 mgr.y (mgr.34104) 580 : cephadm 1 Upgrade: unsafe to stop osd(s) at this time (7 PGs are or would become offline) 2023-06-03T19:56:13.085 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:56:12 smithi164 bash[59462]: cluster 1685822171.7314317 osd.5 (osd.5) 35 : cluster 3 failed to encode map e115 with expected crc 2023-06-03T19:56:13.085 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:56:12 smithi164 bash[59462]: cluster 1685822171.7345092 osd.5 (osd.5) 36 : cluster 3 failed to encode map e115 with expected crc 2023-06-03T19:56:13.085 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:56:12 smithi164 bash[59462]: cluster 1685822171.736145 osd.4 (osd.4) 27 : cluster 3 failed to encode map e115 with expected crc 2023-06-03T19:56:13.085 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:56:12 smithi164 bash[59462]: cluster 1685822171.7362285 osd.3 (osd.3) 38 : cluster 3 failed to encode map e115 with expected crc 2023-06-03T19:56:13.086 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:56:12 smithi164 bash[59462]: cluster 1685822171.736326 osd.7 (osd.7) 25 : cluster 3 failed to encode map e115 with expected crc 2023-06-03T19:56:13.086 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:56:12 smithi164 bash[59462]: cluster 1685822171.7379823 osd.6 (osd.6) 29 : cluster 3 failed to encode map e115 with expected crc 2023-06-03T19:56:13.086 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:56:12 smithi164 bash[59462]: cluster 1685822171.7384076 mon.a (mon.0) 549 : cluster 0 osdmap e115: 8 total, 8 up, 8 in 2023-06-03T19:56:13.086 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:56:12 smithi164 bash[59462]: cluster 1685822171.7402396 osd.4 (osd.4) 28 : cluster 3 failed to encode map e115 with expected crc 2023-06-03T19:56:13.086 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:56:12 smithi164 bash[59462]: cluster 1685822171.7418954 osd.6 (osd.6) 30 : cluster 3 failed to encode map e115 with expected crc 2023-06-03T19:56:13.086 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:56:12 smithi164 bash[59462]: cluster 1685822171.7420068 osd.7 (osd.7) 26 : cluster 3 failed to encode map e115 with expected crc 2023-06-03T19:56:13.086 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:56:12 smithi164 bash[59462]: cluster 1685822171.7421906 osd.4 (osd.4) 29 : cluster 3 failed to encode map e115 with expected crc 2023-06-03T19:56:13.086 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:56:12 smithi164 bash[59462]: cluster 1685822171.7642097 osd.3 (osd.3) 39 : cluster 3 failed to encode map e115 with expected crc 2023-06-03T19:56:13.087 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:56:12 smithi164 bash[59462]: cluster 1685822171.7658498 osd.3 (osd.3) 40 : cluster 3 failed to encode map e115 with expected crc 2023-06-03T19:56:14.029 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:56:13 smithi089 bash[78452]: cluster 1685822172.3297627 mgr.y (mgr.34104) 581 : cluster 0 pgmap v491: 73 pgs: 3 peering, 15 active+undersized, 1 active+undersized+degraded, 54 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 2/72 objects degraded (2.778%) 2023-06-03T19:56:14.029 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:56:13 smithi089 bash[78452]: cluster 1685822172.7412057 mon.a (mon.0) 550 : cluster 3 Health check update: Degraded data redundancy: 2/72 objects degraded (2.778%), 1 pg degraded (PG_DEGRADED) 2023-06-03T19:56:14.029 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:56:13 smithi089 bash[78452]: audit 1685822173.488055 mon.a (mon.0) 551 : audit 0 from='client.? 172.21.15.89:0/3291706098' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:56:14.029 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:56:13 smithi089 bash[75068]: cluster 1685822172.3297627 mgr.y (mgr.34104) 581 : cluster 0 pgmap v491: 73 pgs: 3 peering, 15 active+undersized, 1 active+undersized+degraded, 54 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 2/72 objects degraded (2.778%) 2023-06-03T19:56:14.030 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:56:13 smithi089 bash[75068]: cluster 1685822172.7412057 mon.a (mon.0) 550 : cluster 3 Health check update: Degraded data redundancy: 2/72 objects degraded (2.778%), 1 pg degraded (PG_DEGRADED) 2023-06-03T19:56:14.030 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:56:13 smithi089 bash[75068]: audit 1685822173.488055 mon.a (mon.0) 551 : audit 0 from='client.? 172.21.15.89:0/3291706098' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:56:14.084 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:56:13 smithi164 bash[59462]: cluster 1685822172.3297627 mgr.y (mgr.34104) 581 : cluster 0 pgmap v491: 73 pgs: 3 peering, 15 active+undersized, 1 active+undersized+degraded, 54 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 2/72 objects degraded (2.778%) 2023-06-03T19:56:14.084 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:56:13 smithi164 bash[59462]: cluster 1685822172.7412057 mon.a (mon.0) 550 : cluster 3 Health check update: Degraded data redundancy: 2/72 objects degraded (2.778%), 1 pg degraded (PG_DEGRADED) 2023-06-03T19:56:14.085 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:56:13 smithi164 bash[59462]: audit 1685822173.488055 mon.a (mon.0) 551 : audit 0 from='client.? 172.21.15.89:0/3291706098' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:56:15.084 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:56:14 smithi164 bash[59462]: cluster 1685822174.752779 mon.a (mon.0) 552 : cluster 1 Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 2/72 objects degraded (2.778%), 1 pg degraded) 2023-06-03T19:56:15.085 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:56:14 smithi164 bash[59462]: cluster 1685822174.7528384 mon.a (mon.0) 553 : cluster 1 Cluster is now healthy 2023-06-03T19:56:15.279 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:56:14 smithi089 bash[75068]: cluster 1685822174.752779 mon.a (mon.0) 552 : cluster 1 Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 2/72 objects degraded (2.778%), 1 pg degraded) 2023-06-03T19:56:15.279 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:56:14 smithi089 bash[75068]: cluster 1685822174.7528384 mon.a (mon.0) 553 : cluster 1 Cluster is now healthy 2023-06-03T19:56:15.279 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:56:14 smithi089 bash[78452]: cluster 1685822174.752779 mon.a (mon.0) 552 : cluster 1 Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 2/72 objects degraded (2.778%), 1 pg degraded) 2023-06-03T19:56:15.279 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:56:14 smithi089 bash[78452]: cluster 1685822174.7528384 mon.a (mon.0) 553 : cluster 1 Cluster is now healthy 2023-06-03T19:56:16.084 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:56:15 smithi164 bash[59462]: cluster 1685822174.3307886 mgr.y (mgr.34104) 582 : cluster 0 pgmap v492: 73 pgs: 3 peering, 4 active+undersized, 66 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-06-03T19:56:16.233 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:56:15 smithi089 bash[78452]: cluster 1685822174.3307886 mgr.y (mgr.34104) 582 : cluster 0 pgmap v492: 73 pgs: 3 peering, 4 active+undersized, 66 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-06-03T19:56:16.233 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:56:15 smithi089 bash[75068]: cluster 1685822174.3307886 mgr.y (mgr.34104) 582 : cluster 0 pgmap v492: 73 pgs: 3 peering, 4 active+undersized, 66 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-06-03T19:56:16.532 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 19:56:16 smithi089 bash[71163]: ::ffff:172.21.15.164 - - [03/Jun/2023:19:56:16] "GET /metrics HTTP/1.1" 200 210404 "" "Prometheus/2.33.4" 2023-06-03T19:56:17.084 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:56:16 smithi164 bash[59462]: audit 1685822175.8525743 mon.a (mon.0) 554 : audit 0 from='client.? 172.21.15.89:0/118691977' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:56:17.279 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:56:16 smithi089 bash[75068]: audit 1685822175.8525743 mon.a (mon.0) 554 : audit 0 from='client.? 172.21.15.89:0/118691977' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:56:17.279 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:56:16 smithi089 bash[78452]: audit 1685822175.8525743 mon.a (mon.0) 554 : audit 0 from='client.? 172.21.15.89:0/118691977' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:56:18.084 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:56:17 smithi164 bash[59462]: cluster 1685822176.3316367 mgr.y (mgr.34104) 583 : cluster 0 pgmap v493: 73 pgs: 73 active+clean; 582 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail 2023-06-03T19:56:18.279 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:56:17 smithi089 bash[78452]: cluster 1685822176.3316367 mgr.y (mgr.34104) 583 : cluster 0 pgmap v493: 73 pgs: 73 active+clean; 582 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail 2023-06-03T19:56:18.279 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:56:17 smithi089 bash[75068]: cluster 1685822176.3316367 mgr.y (mgr.34104) 583 : cluster 0 pgmap v493: 73 pgs: 73 active+clean; 582 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail 2023-06-03T19:56:19.084 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:56:18 smithi164 bash[59462]: audit 1685822178.2152157 mon.a (mon.0) 555 : audit 0 from='client.? 172.21.15.89:0/2936992498' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:56:19.279 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:56:18 smithi089 bash[75068]: audit 1685822178.2152157 mon.a (mon.0) 555 : audit 0 from='client.? 172.21.15.89:0/2936992498' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:56:19.279 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:56:18 smithi089 bash[78452]: audit 1685822178.2152157 mon.a (mon.0) 555 : audit 0 from='client.? 172.21.15.89:0/2936992498' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:56:20.278 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:56:19 smithi089 bash[75068]: cluster 1685822178.332368 mgr.y (mgr.34104) 584 : cluster 0 pgmap v494: 73 pgs: 73 active+clean; 582 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail 2023-06-03T19:56:20.279 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:56:19 smithi089 bash[78452]: cluster 1685822178.332368 mgr.y (mgr.34104) 584 : cluster 0 pgmap v494: 73 pgs: 73 active+clean; 582 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail 2023-06-03T19:56:20.334 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:56:19 smithi164 bash[59462]: cluster 1685822178.332368 mgr.y (mgr.34104) 584 : cluster 0 pgmap v494: 73 pgs: 73 active+clean; 582 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail 2023-06-03T19:56:21.279 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:56:20 smithi089 bash[75068]: audit 1685822180.5803602 mon.c (mon.2) 319 : audit 0 from='client.? 172.21.15.89:0/902615800' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:56:21.279 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:56:20 smithi089 bash[78452]: audit 1685822180.5803602 mon.c (mon.2) 319 : audit 0 from='client.? 172.21.15.89:0/902615800' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:56:21.334 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:56:20 smithi164 bash[59462]: audit 1685822180.5803602 mon.c (mon.2) 319 : audit 0 from='client.? 172.21.15.89:0/902615800' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:56:22.279 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:56:21 smithi089 bash[75068]: cluster 1685822180.3332143 mgr.y (mgr.34104) 585 : cluster 0 pgmap v495: 73 pgs: 73 active+clean; 582 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail 2023-06-03T19:56:22.279 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:56:21 smithi089 bash[78452]: cluster 1685822180.3332143 mgr.y (mgr.34104) 585 : cluster 0 pgmap v495: 73 pgs: 73 active+clean; 582 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail 2023-06-03T19:56:22.334 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:56:21 smithi164 bash[59462]: cluster 1685822180.3332143 mgr.y (mgr.34104) 585 : cluster 0 pgmap v495: 73 pgs: 73 active+clean; 582 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail 2023-06-03T19:56:23.529 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:56:23 smithi089 bash[78452]: audit 1685822182.0968006 mon.a (mon.0) 556 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:56:23.529 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:56:23 smithi089 bash[78452]: audit 1685822182.098581 mon.c (mon.2) 320 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T19:56:23.529 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:56:23 smithi089 bash[78452]: cluster 1685822182.3339176 mgr.y (mgr.34104) 586 : cluster 0 pgmap v496: 73 pgs: 73 active+clean; 582 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail 2023-06-03T19:56:23.529 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:56:23 smithi089 bash[78452]: audit 1685822182.379372 mon.a (mon.0) 557 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:56:23.529 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:56:23 smithi089 bash[78452]: audit 1685822182.94541 mon.c (mon.2) 321 : audit 0 from='client.? 172.21.15.89:0/3747629573' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:56:23.530 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:56:23 smithi089 bash[75068]: audit 1685822182.0968006 mon.a (mon.0) 556 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:56:23.530 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:56:23 smithi089 bash[75068]: audit 1685822182.098581 mon.c (mon.2) 320 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T19:56:23.530 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:56:23 smithi089 bash[75068]: cluster 1685822182.3339176 mgr.y (mgr.34104) 586 : cluster 0 pgmap v496: 73 pgs: 73 active+clean; 582 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail 2023-06-03T19:56:23.530 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:56:23 smithi089 bash[75068]: audit 1685822182.379372 mon.a (mon.0) 557 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:56:23.530 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:56:23 smithi089 bash[75068]: audit 1685822182.94541 mon.c (mon.2) 321 : audit 0 from='client.? 172.21.15.89:0/3747629573' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:56:23.584 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:56:23 smithi164 bash[59462]: audit 1685822182.0968006 mon.a (mon.0) 556 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:56:23.585 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:56:23 smithi164 bash[59462]: audit 1685822182.098581 mon.c (mon.2) 320 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T19:56:23.585 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:56:23 smithi164 bash[59462]: cluster 1685822182.3339176 mgr.y (mgr.34104) 586 : cluster 0 pgmap v496: 73 pgs: 73 active+clean; 582 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail 2023-06-03T19:56:23.585 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:56:23 smithi164 bash[59462]: audit 1685822182.379372 mon.a (mon.0) 557 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:56:23.585 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:56:23 smithi164 bash[59462]: audit 1685822182.94541 mon.c (mon.2) 321 : audit 0 from='client.? 172.21.15.89:0/3747629573' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:56:25.779 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:56:25 smithi089 bash[78452]: cluster 1685822184.33487 mgr.y (mgr.34104) 587 : cluster 0 pgmap v497: 73 pgs: 73 active+clean; 582 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail 2023-06-03T19:56:25.779 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:56:25 smithi089 bash[78452]: audit 1685822185.3135743 mon.a (mon.0) 558 : audit 0 from='client.? 172.21.15.89:0/2253872567' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:56:25.779 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:56:25 smithi089 bash[75068]: cluster 1685822184.33487 mgr.y (mgr.34104) 587 : cluster 0 pgmap v497: 73 pgs: 73 active+clean; 582 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail 2023-06-03T19:56:25.780 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:56:25 smithi089 bash[75068]: audit 1685822185.3135743 mon.a (mon.0) 558 : audit 0 from='client.? 172.21.15.89:0/2253872567' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:56:25.834 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:56:25 smithi164 bash[59462]: cluster 1685822184.33487 mgr.y (mgr.34104) 587 : cluster 0 pgmap v497: 73 pgs: 73 active+clean; 582 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail 2023-06-03T19:56:25.834 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:56:25 smithi164 bash[59462]: audit 1685822185.3135743 mon.a (mon.0) 558 : audit 0 from='client.? 172.21.15.89:0/2253872567' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:56:26.529 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 19:56:26 smithi089 bash[71163]: ::ffff:172.21.15.164 - - [03/Jun/2023:19:56:26] "GET /metrics HTTP/1.1" 200 215374 "" "Prometheus/2.33.4" 2023-06-03T19:56:27.779 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:56:27 smithi089 bash[75068]: cluster 1685822186.3358006 mgr.y (mgr.34104) 588 : cluster 0 pgmap v498: 73 pgs: 73 active+clean; 582 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail 2023-06-03T19:56:27.779 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:56:27 smithi089 bash[75068]: audit 1685822186.4210389 mon.c (mon.2) 322 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2023-06-03T19:56:27.779 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:56:27 smithi089 bash[75068]: audit 1685822186.4213355 mgr.y (mgr.34104) 589 : audit 0 from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2023-06-03T19:56:27.779 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:56:27 smithi089 bash[75068]: cephadm 1685822186.4232275 mgr.y (mgr.34104) 590 : cephadm 1 Upgrade: osd.3 is safe to restart 2023-06-03T19:56:27.779 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:56:27 smithi089 bash[78452]: cluster 1685822186.3358006 mgr.y (mgr.34104) 588 : cluster 0 pgmap v498: 73 pgs: 73 active+clean; 582 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail 2023-06-03T19:56:27.780 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:56:27 smithi089 bash[78452]: audit 1685822186.4210389 mon.c (mon.2) 322 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2023-06-03T19:56:27.780 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:56:27 smithi089 bash[78452]: audit 1685822186.4213355 mgr.y (mgr.34104) 589 : audit 0 from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2023-06-03T19:56:27.780 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:56:27 smithi089 bash[78452]: cephadm 1685822186.4232275 mgr.y (mgr.34104) 590 : cephadm 1 Upgrade: osd.3 is safe to restart 2023-06-03T19:56:27.834 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:56:27 smithi164 bash[59462]: cluster 1685822186.3358006 mgr.y (mgr.34104) 588 : cluster 0 pgmap v498: 73 pgs: 73 active+clean; 582 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail 2023-06-03T19:56:27.834 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:56:27 smithi164 bash[59462]: audit 1685822186.4210389 mon.c (mon.2) 322 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2023-06-03T19:56:27.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:56:27 smithi164 bash[59462]: audit 1685822186.4213355 mgr.y (mgr.34104) 589 : audit 0 from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2023-06-03T19:56:27.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:56:27 smithi164 bash[59462]: cephadm 1685822186.4232275 mgr.y (mgr.34104) 590 : cephadm 1 Upgrade: osd.3 is safe to restart 2023-06-03T19:56:28.779 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:56:28 smithi089 bash[75068]: audit 1685822187.670619 mon.a (mon.0) 559 : audit 0 from='client.? 172.21.15.89:0/3543027963' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:56:28.779 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:56:28 smithi089 bash[78452]: audit 1685822187.670619 mon.a (mon.0) 559 : audit 0 from='client.? 172.21.15.89:0/3543027963' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:56:28.834 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:56:28 smithi164 bash[59462]: audit 1685822187.670619 mon.a (mon.0) 559 : audit 0 from='client.? 172.21.15.89:0/3543027963' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:56:29.778 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:56:29 smithi089 bash[75068]: cluster 1685822188.336589 mgr.y (mgr.34104) 591 : cluster 0 pgmap v499: 73 pgs: 73 active+clean; 582 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail 2023-06-03T19:56:29.779 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:56:29 smithi089 bash[75068]: audit 1685822189.1512327 mon.a (mon.0) 560 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:56:29.779 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:56:29 smithi089 bash[75068]: audit 1685822189.1548479 mon.c (mon.2) 323 : audit 1 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2023-06-03T19:56:29.779 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:56:29 smithi089 bash[75068]: audit 1685822189.155885 mon.c (mon.2) 324 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-03T19:56:29.779 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:56:29 smithi089 bash[78452]: cluster 1685822188.336589 mgr.y (mgr.34104) 591 : cluster 0 pgmap v499: 73 pgs: 73 active+clean; 582 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail 2023-06-03T19:56:29.780 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:56:29 smithi089 bash[78452]: audit 1685822189.1512327 mon.a (mon.0) 560 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:56:29.780 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:56:29 smithi089 bash[78452]: audit 1685822189.1548479 mon.c (mon.2) 323 : audit 1 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2023-06-03T19:56:29.780 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:56:29 smithi089 bash[78452]: audit 1685822189.155885 mon.c (mon.2) 324 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-03T19:56:29.834 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:56:29 smithi164 bash[59462]: cluster 1685822188.336589 mgr.y (mgr.34104) 591 : cluster 0 pgmap v499: 73 pgs: 73 active+clean; 582 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail 2023-06-03T19:56:29.834 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:56:29 smithi164 bash[59462]: audit 1685822189.1512327 mon.a (mon.0) 560 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:56:29.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:56:29 smithi164 bash[59462]: audit 1685822189.1548479 mon.c (mon.2) 323 : audit 1 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2023-06-03T19:56:29.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:56:29 smithi164 bash[59462]: audit 1685822189.155885 mon.c (mon.2) 324 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-03T19:56:30.732 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:56:30 smithi089 bash[75068]: cephadm 1685822189.1436534 mgr.y (mgr.34104) 592 : cephadm 1 Upgrade: Updating osd.3 2023-06-03T19:56:30.732 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:56:30 smithi089 bash[75068]: cephadm 1685822189.1586149 mgr.y (mgr.34104) 593 : cephadm 1 Deploying daemon osd.3 on smithi089 2023-06-03T19:56:30.732 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:56:30 smithi089 bash[75068]: audit 1685822190.028242 mon.c (mon.2) 325 : audit 0 from='client.? 172.21.15.89:0/2301756987' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:56:30.733 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:56:30 smithi089 bash[78452]: cephadm 1685822189.1436534 mgr.y (mgr.34104) 592 : cephadm 1 Upgrade: Updating osd.3 2023-06-03T19:56:30.733 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:56:30 smithi089 bash[78452]: cephadm 1685822189.1586149 mgr.y (mgr.34104) 593 : cephadm 1 Deploying daemon osd.3 on smithi089 2023-06-03T19:56:30.733 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:56:30 smithi089 bash[78452]: audit 1685822190.028242 mon.c (mon.2) 325 : audit 0 from='client.? 172.21.15.89:0/2301756987' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:56:30.834 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:56:30 smithi164 bash[59462]: cephadm 1685822189.1436534 mgr.y (mgr.34104) 592 : cephadm 1 Upgrade: Updating osd.3 2023-06-03T19:56:30.834 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:56:30 smithi164 bash[59462]: cephadm 1685822189.1586149 mgr.y (mgr.34104) 593 : cephadm 1 Deploying daemon osd.3 on smithi089 2023-06-03T19:56:30.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:56:30 smithi164 bash[59462]: audit 1685822190.028242 mon.c (mon.2) 325 : audit 0 from='client.? 172.21.15.89:0/2301756987' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:56:31.779 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:56:31 smithi089 bash[78452]: cluster 1685822190.3373861 mgr.y (mgr.34104) 594 : cluster 0 pgmap v500: 73 pgs: 73 active+clean; 582 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail 2023-06-03T19:56:31.779 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:56:31 smithi089 bash[75068]: cluster 1685822190.3373861 mgr.y (mgr.34104) 594 : cluster 0 pgmap v500: 73 pgs: 73 active+clean; 582 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail 2023-06-03T19:56:31.834 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:56:31 smithi164 bash[59462]: cluster 1685822190.3373861 mgr.y (mgr.34104) 594 : cluster 0 pgmap v500: 73 pgs: 73 active+clean; 582 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail 2023-06-03T19:56:32.779 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:56:32 smithi089 bash[78452]: audit 1685822192.3875155 mon.a (mon.0) 561 : audit 0 from='client.? 172.21.15.89:0/2513608436' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:56:32.779 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:56:32 smithi089 bash[75068]: audit 1685822192.3875155 mon.a (mon.0) 561 : audit 0 from='client.? 172.21.15.89:0/2513608436' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:56:32.834 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:56:32 smithi164 bash[59462]: audit 1685822192.3875155 mon.a (mon.0) 561 : audit 0 from='client.? 172.21.15.89:0/2513608436' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:56:33.778 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:56:33 smithi089 bash[75068]: cluster 1685822192.3379097 mgr.y (mgr.34104) 595 : cluster 0 pgmap v501: 73 pgs: 73 active+clean; 582 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail 2023-06-03T19:56:33.779 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:56:33 smithi089 bash[78452]: cluster 1685822192.3379097 mgr.y (mgr.34104) 595 : cluster 0 pgmap v501: 73 pgs: 73 active+clean; 582 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail 2023-06-03T19:56:33.834 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:56:33 smithi164 bash[59462]: cluster 1685822192.3379097 mgr.y (mgr.34104) 595 : cluster 0 pgmap v501: 73 pgs: 73 active+clean; 582 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail 2023-06-03T19:56:34.270 INFO:journalctl@ceph.osd.3.smithi089.stdout:Jun 03 19:56:34 smithi089 systemd[1]: Stopping Ceph osd.3 for f1db3478-0243-11ee-9b1d-001a4aab830c... 2023-06-03T19:56:34.528 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:56:34 smithi089 bash[75068]: cluster 1685822194.26617 mon.a (mon.0) 562 : cluster 1 osd.3 marked itself down and dead 2023-06-03T19:56:34.529 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:56:34 smithi089 bash[78452]: cluster 1685822194.26617 mon.a (mon.0) 562 : cluster 1 osd.3 marked itself down and dead 2023-06-03T19:56:34.529 INFO:journalctl@ceph.osd.3.smithi089.stdout:Jun 03 19:56:34 smithi089 bash[48915]: debug 2023-06-03T19:56:34.263+0000 7f39aa8bf700 -1 received signal: Terminated from /sbin/docker-init -- /usr/bin/ceph-osd -n osd.3 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug (PID: 1) UID: 0 2023-06-03T19:56:34.529 INFO:journalctl@ceph.osd.3.smithi089.stdout:Jun 03 19:56:34 smithi089 bash[48915]: debug 2023-06-03T19:56:34.263+0000 7f39aa8bf700 -1 osd.3 115 *** Got signal Terminated *** 2023-06-03T19:56:34.530 INFO:journalctl@ceph.osd.3.smithi089.stdout:Jun 03 19:56:34 smithi089 bash[48915]: debug 2023-06-03T19:56:34.263+0000 7f39aa8bf700 -1 osd.3 115 *** Immediate shutdown (osd_fast_shutdown=true) *** 2023-06-03T19:56:34.834 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:56:34 smithi164 bash[59462]: cluster 1685822194.26617 mon.a (mon.0) 562 : cluster 1 osd.3 marked itself down and dead 2023-06-03T19:56:35.758 INFO:journalctl@ceph.osd.3.smithi089.stdout:Jun 03 19:56:35 smithi089 bash[100990]: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd-3 2023-06-03T19:56:35.758 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:56:35 smithi089 bash[78452]: cluster 1685822194.338948 mgr.y (mgr.34104) 596 : cluster 0 pgmap v502: 73 pgs: 73 active+clean; 582 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail 2023-06-03T19:56:35.758 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:56:35 smithi089 bash[78452]: cluster 1685822194.4770262 mon.a (mon.0) 563 : cluster 3 Health check failed: 1 osds down (OSD_DOWN) 2023-06-03T19:56:35.759 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:56:35 smithi089 bash[78452]: cluster 1685822194.487364 osd.4 (osd.4) 30 : cluster 3 failed to encode map e116 with expected crc 2023-06-03T19:56:35.759 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:56:35 smithi089 bash[78452]: cluster 1685822194.4875393 osd.7 (osd.7) 27 : cluster 3 failed to encode map e116 with expected crc 2023-06-03T19:56:35.759 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:56:35 smithi089 bash[78452]: cluster 1685822194.4918816 osd.6 (osd.6) 31 : cluster 3 failed to encode map e116 with expected crc 2023-06-03T19:56:35.759 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:56:35 smithi089 bash[78452]: cluster 1685822194.4933589 osd.5 (osd.5) 37 : cluster 3 failed to encode map e116 with expected crc 2023-06-03T19:56:35.759 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:56:35 smithi089 bash[78452]: cluster 1685822194.4940383 osd.4 (osd.4) 31 : cluster 3 failed to encode map e116 with expected crc 2023-06-03T19:56:35.759 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:56:35 smithi089 bash[78452]: cluster 1685822194.4947705 mon.a (mon.0) 564 : cluster 0 osdmap e116: 8 total, 7 up, 8 in 2023-06-03T19:56:35.760 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:56:35 smithi089 bash[78452]: cluster 1685822194.4970386 osd.5 (osd.5) 38 : cluster 3 failed to encode map e116 with expected crc 2023-06-03T19:56:35.760 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:56:35 smithi089 bash[78452]: cluster 1685822194.499785 osd.4 (osd.4) 32 : cluster 3 failed to encode map e116 with expected crc 2023-06-03T19:56:35.760 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:56:35 smithi089 bash[78452]: audit 1685822194.772263 mon.c (mon.2) 326 : audit 0 from='client.? 172.21.15.89:0/4203182123' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:56:35.760 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:56:35 smithi089 bash[75068]: cluster 1685822194.338948 mgr.y (mgr.34104) 596 : cluster 0 pgmap v502: 73 pgs: 73 active+clean; 582 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail 2023-06-03T19:56:35.760 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:56:35 smithi089 bash[75068]: cluster 1685822194.4770262 mon.a (mon.0) 563 : cluster 3 Health check failed: 1 osds down (OSD_DOWN) 2023-06-03T19:56:35.761 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:56:35 smithi089 bash[75068]: cluster 1685822194.487364 osd.4 (osd.4) 30 : cluster 3 failed to encode map e116 with expected crc 2023-06-03T19:56:35.761 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:56:35 smithi089 bash[75068]: cluster 1685822194.4875393 osd.7 (osd.7) 27 : cluster 3 failed to encode map e116 with expected crc 2023-06-03T19:56:35.761 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:56:35 smithi089 bash[75068]: cluster 1685822194.4918816 osd.6 (osd.6) 31 : cluster 3 failed to encode map e116 with expected crc 2023-06-03T19:56:35.761 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:56:35 smithi089 bash[75068]: cluster 1685822194.4933589 osd.5 (osd.5) 37 : cluster 3 failed to encode map e116 with expected crc 2023-06-03T19:56:35.761 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:56:35 smithi089 bash[75068]: cluster 1685822194.4940383 osd.4 (osd.4) 31 : cluster 3 failed to encode map e116 with expected crc 2023-06-03T19:56:35.761 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:56:35 smithi089 bash[75068]: cluster 1685822194.4947705 mon.a (mon.0) 564 : cluster 0 osdmap e116: 8 total, 7 up, 8 in 2023-06-03T19:56:35.761 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:56:35 smithi089 bash[75068]: cluster 1685822194.4970386 osd.5 (osd.5) 38 : cluster 3 failed to encode map e116 with expected crc 2023-06-03T19:56:35.762 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:56:35 smithi089 bash[75068]: cluster 1685822194.499785 osd.4 (osd.4) 32 : cluster 3 failed to encode map e116 with expected crc 2023-06-03T19:56:35.762 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:56:35 smithi089 bash[75068]: audit 1685822194.772263 mon.c (mon.2) 326 : audit 0 from='client.? 172.21.15.89:0/4203182123' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:56:35.834 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:56:35 smithi164 bash[59462]: cluster 1685822194.338948 mgr.y (mgr.34104) 596 : cluster 0 pgmap v502: 73 pgs: 73 active+clean; 582 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail 2023-06-03T19:56:35.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:56:35 smithi164 bash[59462]: cluster 1685822194.4770262 mon.a (mon.0) 563 : cluster 3 Health check failed: 1 osds down (OSD_DOWN) 2023-06-03T19:56:35.836 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:56:35 smithi164 bash[59462]: cluster 1685822194.487364 osd.4 (osd.4) 30 : cluster 3 failed to encode map e116 with expected crc 2023-06-03T19:56:35.836 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:56:35 smithi164 bash[59462]: cluster 1685822194.4875393 osd.7 (osd.7) 27 : cluster 3 failed to encode map e116 with expected crc 2023-06-03T19:56:35.836 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:56:35 smithi164 bash[59462]: cluster 1685822194.4918816 osd.6 (osd.6) 31 : cluster 3 failed to encode map e116 with expected crc 2023-06-03T19:56:35.836 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:56:35 smithi164 bash[59462]: cluster 1685822194.4933589 osd.5 (osd.5) 37 : cluster 3 failed to encode map e116 with expected crc 2023-06-03T19:56:35.837 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:56:35 smithi164 bash[59462]: cluster 1685822194.4940383 osd.4 (osd.4) 31 : cluster 3 failed to encode map e116 with expected crc 2023-06-03T19:56:35.837 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:56:35 smithi164 bash[59462]: cluster 1685822194.4947705 mon.a (mon.0) 564 : cluster 0 osdmap e116: 8 total, 7 up, 8 in 2023-06-03T19:56:35.837 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:56:35 smithi164 bash[59462]: cluster 1685822194.4970386 osd.5 (osd.5) 38 : cluster 3 failed to encode map e116 with expected crc 2023-06-03T19:56:35.837 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:56:35 smithi164 bash[59462]: cluster 1685822194.499785 osd.4 (osd.4) 32 : cluster 3 failed to encode map e116 with expected crc 2023-06-03T19:56:35.838 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:56:35 smithi164 bash[59462]: audit 1685822194.772263 mon.c (mon.2) 326 : audit 0 from='client.? 172.21.15.89:0/4203182123' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:56:36.516 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 19:56:36 smithi089 bash[71163]: ::ffff:172.21.15.164 - - [03/Jun/2023:19:56:36] "GET /metrics HTTP/1.1" 200 215374 "" "Prometheus/2.33.4" 2023-06-03T19:56:36.779 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:56:36 smithi089 bash[78452]: cluster 1685822195.4863124 osd.4 (osd.4) 33 : cluster 3 failed to encode map e117 with expected crc 2023-06-03T19:56:36.779 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:56:36 smithi089 bash[78452]: cluster 1685822195.4870522 osd.5 (osd.5) 39 : cluster 3 failed to encode map e117 with expected crc 2023-06-03T19:56:36.779 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:56:36 smithi089 bash[78452]: cluster 1685822195.490927 osd.4 (osd.4) 34 : cluster 3 failed to encode map e117 with expected crc 2023-06-03T19:56:36.779 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:56:36 smithi089 bash[78452]: cluster 1685822195.4909668 osd.7 (osd.7) 28 : cluster 3 failed to encode map e117 with expected crc 2023-06-03T19:56:36.779 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:56:36 smithi089 bash[78452]: cluster 1685822195.4910755 osd.6 (osd.6) 32 : cluster 3 failed to encode map e117 with expected crc 2023-06-03T19:56:36.779 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:56:36 smithi089 bash[78452]: cluster 1685822195.4919064 osd.5 (osd.5) 40 : cluster 3 failed to encode map e117 with expected crc 2023-06-03T19:56:36.780 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:56:36 smithi089 bash[78452]: cluster 1685822195.4929438 mon.a (mon.0) 565 : cluster 0 osdmap e117: 8 total, 7 up, 8 in 2023-06-03T19:56:36.780 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:56:36 smithi089 bash[78452]: cluster 1685822195.4949925 osd.6 (osd.6) 33 : cluster 3 failed to encode map e117 with expected crc 2023-06-03T19:56:36.780 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:56:36 smithi089 bash[78452]: cluster 1685822195.495695 osd.7 (osd.7) 29 : cluster 3 failed to encode map e117 with expected crc 2023-06-03T19:56:36.780 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:56:36 smithi089 bash[78452]: cluster 1685822195.496243 osd.5 (osd.5) 41 : cluster 3 failed to encode map e117 with expected crc 2023-06-03T19:56:36.780 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:56:36 smithi089 bash[75068]: cluster 1685822195.4863124 osd.4 (osd.4) 33 : cluster 3 failed to encode map e117 with expected crc 2023-06-03T19:56:36.780 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:56:36 smithi089 bash[75068]: cluster 1685822195.4870522 osd.5 (osd.5) 39 : cluster 3 failed to encode map e117 with expected crc 2023-06-03T19:56:36.781 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:56:36 smithi089 bash[75068]: cluster 1685822195.490927 osd.4 (osd.4) 34 : cluster 3 failed to encode map e117 with expected crc 2023-06-03T19:56:36.781 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:56:36 smithi089 bash[75068]: cluster 1685822195.4909668 osd.7 (osd.7) 28 : cluster 3 failed to encode map e117 with expected crc 2023-06-03T19:56:36.781 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:56:36 smithi089 bash[75068]: cluster 1685822195.4910755 osd.6 (osd.6) 32 : cluster 3 failed to encode map e117 with expected crc 2023-06-03T19:56:36.781 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:56:36 smithi089 bash[75068]: cluster 1685822195.4919064 osd.5 (osd.5) 40 : cluster 3 failed to encode map e117 with expected crc 2023-06-03T19:56:36.781 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:56:36 smithi089 bash[75068]: cluster 1685822195.4929438 mon.a (mon.0) 565 : cluster 0 osdmap e117: 8 total, 7 up, 8 in 2023-06-03T19:56:36.781 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:56:36 smithi089 bash[75068]: cluster 1685822195.4949925 osd.6 (osd.6) 33 : cluster 3 failed to encode map e117 with expected crc 2023-06-03T19:56:36.781 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:56:36 smithi089 bash[75068]: cluster 1685822195.495695 osd.7 (osd.7) 29 : cluster 3 failed to encode map e117 with expected crc 2023-06-03T19:56:36.781 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:56:36 smithi089 bash[75068]: cluster 1685822195.496243 osd.5 (osd.5) 41 : cluster 3 failed to encode map e117 with expected crc 2023-06-03T19:56:36.834 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:56:36 smithi164 bash[59462]: cluster 1685822195.4863124 osd.4 (osd.4) 33 : cluster 3 failed to encode map e117 with expected crc 2023-06-03T19:56:36.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:56:36 smithi164 bash[59462]: cluster 1685822195.4870522 osd.5 (osd.5) 39 : cluster 3 failed to encode map e117 with expected crc 2023-06-03T19:56:36.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:56:36 smithi164 bash[59462]: cluster 1685822195.490927 osd.4 (osd.4) 34 : cluster 3 failed to encode map e117 with expected crc 2023-06-03T19:56:36.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:56:36 smithi164 bash[59462]: cluster 1685822195.4909668 osd.7 (osd.7) 28 : cluster 3 failed to encode map e117 with expected crc 2023-06-03T19:56:36.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:56:36 smithi164 bash[59462]: cluster 1685822195.4910755 osd.6 (osd.6) 32 : cluster 3 failed to encode map e117 with expected crc 2023-06-03T19:56:36.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:56:36 smithi164 bash[59462]: cluster 1685822195.4919064 osd.5 (osd.5) 40 : cluster 3 failed to encode map e117 with expected crc 2023-06-03T19:56:36.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:56:36 smithi164 bash[59462]: cluster 1685822195.4929438 mon.a (mon.0) 565 : cluster 0 osdmap e117: 8 total, 7 up, 8 in 2023-06-03T19:56:36.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:56:36 smithi164 bash[59462]: cluster 1685822195.4949925 osd.6 (osd.6) 33 : cluster 3 failed to encode map e117 with expected crc 2023-06-03T19:56:36.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:56:36 smithi164 bash[59462]: cluster 1685822195.495695 osd.7 (osd.7) 29 : cluster 3 failed to encode map e117 with expected crc 2023-06-03T19:56:36.836 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:56:36 smithi164 bash[59462]: cluster 1685822195.496243 osd.5 (osd.5) 41 : cluster 3 failed to encode map e117 with expected crc 2023-06-03T19:56:37.779 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:56:37 smithi089 bash[75068]: cluster 1685822196.3397663 mgr.y (mgr.34104) 597 : cluster 0 pgmap v505: 73 pgs: 1 active+undersized+degraded, 6 active+undersized, 3 peering, 10 stale+active+clean, 53 active+clean; 582 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 1/72 objects degraded (1.389%) 2023-06-03T19:56:37.779 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:56:37 smithi089 bash[75068]: cluster 1685822196.5009935 mon.a (mon.0) 566 : cluster 3 Health check failed: Degraded data redundancy: 1/72 objects degraded (1.389%), 1 pg degraded (PG_DEGRADED) 2023-06-03T19:56:37.779 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:56:37 smithi089 bash[75068]: audit 1685822197.0932071 mon.a (mon.0) 567 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:56:37.780 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:56:37 smithi089 bash[75068]: audit 1685822197.0946753 mon.c (mon.2) 327 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T19:56:37.780 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:56:37 smithi089 bash[75068]: audit 1685822197.1712554 mon.a (mon.0) 568 : audit 0 from='client.? 172.21.15.89:0/3718211493' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:56:37.780 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:56:37 smithi089 bash[78452]: cluster 1685822196.3397663 mgr.y (mgr.34104) 597 : cluster 0 pgmap v505: 73 pgs: 1 active+undersized+degraded, 6 active+undersized, 3 peering, 10 stale+active+clean, 53 active+clean; 582 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 1/72 objects degraded (1.389%) 2023-06-03T19:56:37.781 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:56:37 smithi089 bash[78452]: cluster 1685822196.5009935 mon.a (mon.0) 566 : cluster 3 Health check failed: Degraded data redundancy: 1/72 objects degraded (1.389%), 1 pg degraded (PG_DEGRADED) 2023-06-03T19:56:37.781 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:56:37 smithi089 bash[78452]: audit 1685822197.0932071 mon.a (mon.0) 567 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:56:37.781 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:56:37 smithi089 bash[78452]: audit 1685822197.0946753 mon.c (mon.2) 327 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T19:56:37.781 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:56:37 smithi089 bash[78452]: audit 1685822197.1712554 mon.a (mon.0) 568 : audit 0 from='client.? 172.21.15.89:0/3718211493' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:56:37.834 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:56:37 smithi164 bash[59462]: cluster 1685822196.3397663 mgr.y (mgr.34104) 597 : cluster 0 pgmap v505: 73 pgs: 1 active+undersized+degraded, 6 active+undersized, 3 peering, 10 stale+active+clean, 53 active+clean; 582 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 1/72 objects degraded (1.389%) 2023-06-03T19:56:37.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:56:37 smithi164 bash[59462]: cluster 1685822196.5009935 mon.a (mon.0) 566 : cluster 3 Health check failed: Degraded data redundancy: 1/72 objects degraded (1.389%), 1 pg degraded (PG_DEGRADED) 2023-06-03T19:56:37.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:56:37 smithi164 bash[59462]: audit 1685822197.0932071 mon.a (mon.0) 567 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:56:37.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:56:37 smithi164 bash[59462]: audit 1685822197.0946753 mon.c (mon.2) 327 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T19:56:37.836 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:56:37 smithi164 bash[59462]: audit 1685822197.1712554 mon.a (mon.0) 568 : audit 0 from='client.? 172.21.15.89:0/3718211493' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:56:38.503 INFO:journalctl@ceph.osd.3.smithi089.stdout:Jun 03 19:56:38 smithi089 systemd[1]: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c@osd.3.service: Succeeded. 2023-06-03T19:56:38.504 INFO:journalctl@ceph.osd.3.smithi089.stdout:Jun 03 19:56:38 smithi089 systemd[1]: Stopped Ceph osd.3 for f1db3478-0243-11ee-9b1d-001a4aab830c. 2023-06-03T19:56:38.779 INFO:journalctl@ceph.osd.3.smithi089.stdout:Jun 03 19:56:38 smithi089 systemd[1]: Started Ceph osd.3 for f1db3478-0243-11ee-9b1d-001a4aab830c. 2023-06-03T19:56:39.834 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:56:39 smithi164 bash[59462]: cluster 1685822198.3409598 mgr.y (mgr.34104) 598 : cluster 0 pgmap v506: 73 pgs: 3 active+undersized+degraded, 25 active+undersized, 3 peering, 42 active+clean; 582 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2023-06-03T19:56:39.834 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:56:39 smithi164 bash[59462]: audit 1685822198.539328 mon.a (mon.0) 569 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:56:39.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:56:39 smithi164 bash[59462]: audit 1685822198.5542293 mon.a (mon.0) 570 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:56:39.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:56:39 smithi164 bash[59462]: audit 1685822198.600998 mon.c (mon.2) 328 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-03T19:56:40.029 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:56:39 smithi089 bash[75068]: cluster 1685822198.3409598 mgr.y (mgr.34104) 598 : cluster 0 pgmap v506: 73 pgs: 3 active+undersized+degraded, 25 active+undersized, 3 peering, 42 active+clean; 582 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2023-06-03T19:56:40.029 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:56:39 smithi089 bash[75068]: audit 1685822198.539328 mon.a (mon.0) 569 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:56:40.029 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:56:39 smithi089 bash[75068]: audit 1685822198.5542293 mon.a (mon.0) 570 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:56:40.029 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:56:39 smithi089 bash[75068]: audit 1685822198.600998 mon.c (mon.2) 328 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-03T19:56:40.030 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:56:39 smithi089 bash[78452]: cluster 1685822198.3409598 mgr.y (mgr.34104) 598 : cluster 0 pgmap v506: 73 pgs: 3 active+undersized+degraded, 25 active+undersized, 3 peering, 42 active+clean; 582 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2023-06-03T19:56:40.030 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:56:39 smithi089 bash[78452]: audit 1685822198.539328 mon.a (mon.0) 569 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:56:40.030 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:56:39 smithi089 bash[78452]: audit 1685822198.5542293 mon.a (mon.0) 570 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:56:40.030 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:56:39 smithi089 bash[78452]: audit 1685822198.600998 mon.c (mon.2) 328 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-03T19:56:40.779 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:56:40 smithi089 bash[75068]: audit 1685822199.5423727 mon.c (mon.2) 329 : audit 0 from='client.? 172.21.15.89:0/2518053462' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:56:40.779 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:56:40 smithi089 bash[78452]: audit 1685822199.5423727 mon.c (mon.2) 329 : audit 0 from='client.? 172.21.15.89:0/2518053462' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:56:40.779 INFO:journalctl@ceph.osd.3.smithi089.stdout:Jun 03 19:56:40 smithi089 bash[101275]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-3 2023-06-03T19:56:40.780 INFO:journalctl@ceph.osd.3.smithi089.stdout:Jun 03 19:56:40 smithi089 bash[101275]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-3 --no-mon-config --dev /dev/mapper/vg_nvme-lv_1 2023-06-03T19:56:40.780 INFO:journalctl@ceph.osd.3.smithi089.stdout:Jun 03 19:56:40 smithi089 bash[101275]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_1 2023-06-03T19:56:40.780 INFO:journalctl@ceph.osd.3.smithi089.stdout:Jun 03 19:56:40 smithi089 bash[101275]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-0 2023-06-03T19:56:40.780 INFO:journalctl@ceph.osd.3.smithi089.stdout:Jun 03 19:56:40 smithi089 bash[101275]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_1 /var/lib/ceph/osd/ceph-3/block 2023-06-03T19:56:40.780 INFO:journalctl@ceph.osd.3.smithi089.stdout:Jun 03 19:56:40 smithi089 bash[101275]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-3 2023-06-03T19:56:40.780 INFO:journalctl@ceph.osd.3.smithi089.stdout:Jun 03 19:56:40 smithi089 bash[101275]: --> ceph-volume raw activate successful for osd ID: 3 2023-06-03T19:56:40.834 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:56:40 smithi164 bash[59462]: audit 1685822199.5423727 mon.c (mon.2) 329 : audit 0 from='client.? 172.21.15.89:0/2518053462' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:56:41.778 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:56:41 smithi089 bash[75068]: cluster 1685822200.341664 mgr.y (mgr.34104) 599 : cluster 0 pgmap v507: 73 pgs: 3 active+undersized+degraded, 28 active+undersized, 42 active+clean; 582 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2023-06-03T19:56:41.779 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:56:41 smithi089 bash[78452]: cluster 1685822200.341664 mgr.y (mgr.34104) 599 : cluster 0 pgmap v507: 73 pgs: 3 active+undersized+degraded, 28 active+undersized, 42 active+clean; 582 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2023-06-03T19:56:41.834 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:56:41 smithi164 bash[59462]: cluster 1685822200.341664 mgr.y (mgr.34104) 599 : cluster 0 pgmap v507: 73 pgs: 3 active+undersized+degraded, 28 active+undersized, 42 active+clean; 582 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2023-06-03T19:56:42.779 INFO:journalctl@ceph.osd.3.smithi089.stdout:Jun 03 19:56:42 smithi089 bash[101590]: debug 2023-06-03T19:56:42.471+0000 7f5a403c2700 1 -- 172.21.15.89:0/964461422 <== mon.2 v2:172.21.15.89:3301/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x55856b3a0d00 con 0x55856b3fb000 2023-06-03T19:56:42.779 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:56:42 smithi089 bash[78452]: audit 1685822201.900991 mon.a (mon.0) 571 : audit 0 from='client.? 172.21.15.89:0/3933658627' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:56:42.779 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:56:42 smithi089 bash[78452]: cluster 1685822202.4428685 mon.a (mon.0) 572 : cluster 3 Health check update: Degraded data redundancy: 4/72 objects degraded (5.556%), 3 pgs degraded (PG_DEGRADED) 2023-06-03T19:56:42.780 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:56:42 smithi089 bash[75068]: audit 1685822201.900991 mon.a (mon.0) 571 : audit 0 from='client.? 172.21.15.89:0/3933658627' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:56:42.780 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:56:42 smithi089 bash[75068]: cluster 1685822202.4428685 mon.a (mon.0) 572 : cluster 3 Health check update: Degraded data redundancy: 4/72 objects degraded (5.556%), 3 pgs degraded (PG_DEGRADED) 2023-06-03T19:56:42.834 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:56:42 smithi164 bash[59462]: audit 1685822201.900991 mon.a (mon.0) 571 : audit 0 from='client.? 172.21.15.89:0/3933658627' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:56:42.834 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:56:42 smithi164 bash[59462]: cluster 1685822202.4428685 mon.a (mon.0) 572 : cluster 3 Health check update: Degraded data redundancy: 4/72 objects degraded (5.556%), 3 pgs degraded (PG_DEGRADED) 2023-06-03T19:56:43.278 INFO:journalctl@ceph.osd.3.smithi089.stdout:Jun 03 19:56:43 smithi089 bash[101590]: debug 2023-06-03T19:56:43.007+0000 7f5a49548640 -1 Falling back to public interface 2023-06-03T19:56:43.834 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:56:43 smithi164 bash[59462]: cluster 1685822202.3423662 mgr.y (mgr.34104) 600 : cluster 0 pgmap v508: 73 pgs: 3 active+undersized+degraded, 28 active+undersized, 42 active+clean; 582 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2023-06-03T19:56:44.029 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:56:43 smithi089 bash[75068]: cluster 1685822202.3423662 mgr.y (mgr.34104) 600 : cluster 0 pgmap v508: 73 pgs: 3 active+undersized+degraded, 28 active+undersized, 42 active+clean; 582 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2023-06-03T19:56:44.029 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:56:43 smithi089 bash[78452]: cluster 1685822202.3423662 mgr.y (mgr.34104) 600 : cluster 0 pgmap v508: 73 pgs: 3 active+undersized+degraded, 28 active+undersized, 42 active+clean; 582 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2023-06-03T19:56:44.834 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:56:44 smithi164 bash[59462]: audit 1685822204.2667701 mon.a (mon.0) 573 : audit 0 from='client.? 172.21.15.89:0/1067246896' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:56:45.029 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:56:44 smithi089 bash[78452]: audit 1685822204.2667701 mon.a (mon.0) 573 : audit 0 from='client.? 172.21.15.89:0/1067246896' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:56:45.029 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:56:44 smithi089 bash[75068]: audit 1685822204.2667701 mon.a (mon.0) 573 : audit 0 from='client.? 172.21.15.89:0/1067246896' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:56:45.834 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:56:45 smithi164 bash[59462]: cluster 1685822204.3434274 mgr.y (mgr.34104) 601 : cluster 0 pgmap v509: 73 pgs: 3 active+undersized+degraded, 28 active+undersized, 42 active+clean; 582 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2023-06-03T19:56:45.834 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:56:45 smithi164 bash[59462]: audit 1685822205.00678 mon.a (mon.0) 574 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:56:45.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:56:45 smithi164 bash[59462]: audit 1685822205.0171924 mon.a (mon.0) 575 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:56:46.028 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:56:45 smithi089 bash[78452]: cluster 1685822204.3434274 mgr.y (mgr.34104) 601 : cluster 0 pgmap v509: 73 pgs: 3 active+undersized+degraded, 28 active+undersized, 42 active+clean; 582 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2023-06-03T19:56:46.029 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:56:45 smithi089 bash[78452]: audit 1685822205.00678 mon.a (mon.0) 574 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:56:46.029 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:56:45 smithi089 bash[78452]: audit 1685822205.0171924 mon.a (mon.0) 575 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:56:46.029 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:56:45 smithi089 bash[75068]: cluster 1685822204.3434274 mgr.y (mgr.34104) 601 : cluster 0 pgmap v509: 73 pgs: 3 active+undersized+degraded, 28 active+undersized, 42 active+clean; 582 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2023-06-03T19:56:46.029 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:56:45 smithi089 bash[75068]: audit 1685822205.00678 mon.a (mon.0) 574 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:56:46.030 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:56:45 smithi089 bash[75068]: audit 1685822205.0171924 mon.a (mon.0) 575 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:56:46.530 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 19:56:46 smithi089 bash[71163]: ::ffff:172.21.15.164 - - [03/Jun/2023:19:56:46] "GET /metrics HTTP/1.1" 200 215377 "" "Prometheus/2.33.4" 2023-06-03T19:56:47.529 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:56:47 smithi089 bash[75068]: audit 1685822206.1109161 mon.a (mon.0) 576 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:56:47.529 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:56:47 smithi089 bash[75068]: audit 1685822206.1216528 mon.a (mon.0) 577 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:56:47.529 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:56:47 smithi089 bash[75068]: cluster 1685822206.3440332 mgr.y (mgr.34104) 602 : cluster 0 pgmap v510: 73 pgs: 3 active+undersized+degraded, 28 active+undersized, 42 active+clean; 582 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2023-06-03T19:56:47.529 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:56:47 smithi089 bash[75068]: audit 1685822206.6304555 mon.a (mon.0) 578 : audit 0 from='client.? 172.21.15.89:0/1247371912' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:56:47.529 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:56:47 smithi089 bash[75068]: audit 1685822206.9652355 mon.a (mon.0) 579 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:56:47.530 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:56:47 smithi089 bash[75068]: audit 1685822206.9774723 mon.a (mon.0) 580 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:56:47.530 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:56:47 smithi089 bash[78452]: audit 1685822206.1109161 mon.a (mon.0) 576 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:56:47.530 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:56:47 smithi089 bash[78452]: audit 1685822206.1216528 mon.a (mon.0) 577 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:56:47.530 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:56:47 smithi089 bash[78452]: cluster 1685822206.3440332 mgr.y (mgr.34104) 602 : cluster 0 pgmap v510: 73 pgs: 3 active+undersized+degraded, 28 active+undersized, 42 active+clean; 582 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2023-06-03T19:56:47.530 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:56:47 smithi089 bash[78452]: audit 1685822206.6304555 mon.a (mon.0) 578 : audit 0 from='client.? 172.21.15.89:0/1247371912' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:56:47.530 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:56:47 smithi089 bash[78452]: audit 1685822206.9652355 mon.a (mon.0) 579 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:56:47.531 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:56:47 smithi089 bash[78452]: audit 1685822206.9774723 mon.a (mon.0) 580 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:56:47.584 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:56:47 smithi164 bash[59462]: audit 1685822206.1109161 mon.a (mon.0) 576 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:56:47.585 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:56:47 smithi164 bash[59462]: audit 1685822206.1216528 mon.a (mon.0) 577 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:56:47.585 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:56:47 smithi164 bash[59462]: cluster 1685822206.3440332 mgr.y (mgr.34104) 602 : cluster 0 pgmap v510: 73 pgs: 3 active+undersized+degraded, 28 active+undersized, 42 active+clean; 582 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2023-06-03T19:56:47.585 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:56:47 smithi164 bash[59462]: audit 1685822206.6304555 mon.a (mon.0) 578 : audit 0 from='client.? 172.21.15.89:0/1247371912' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:56:47.585 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:56:47 smithi164 bash[59462]: audit 1685822206.9652355 mon.a (mon.0) 579 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:56:47.585 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:56:47 smithi164 bash[59462]: audit 1685822206.9774723 mon.a (mon.0) 580 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:56:48.778 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:56:48 smithi089 bash[75068]: audit 1685822207.4020712 mon.a (mon.0) 581 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:56:48.779 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:56:48 smithi089 bash[78452]: audit 1685822207.4020712 mon.a (mon.0) 581 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:56:48.834 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:56:48 smithi164 bash[59462]: audit 1685822207.4020712 mon.a (mon.0) 581 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:56:49.779 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:56:49 smithi089 bash[78452]: cluster 1685822208.3453093 mgr.y (mgr.34104) 603 : cluster 0 pgmap v511: 73 pgs: 3 active+undersized+degraded, 28 active+undersized, 42 active+clean; 582 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2023-06-03T19:56:49.779 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:56:49 smithi089 bash[78452]: audit 1685822208.972938 mon.a (mon.0) 582 : audit 0 from='client.? 172.21.15.89:0/1042621724' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:56:49.779 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:56:49 smithi089 bash[75068]: cluster 1685822208.3453093 mgr.y (mgr.34104) 603 : cluster 0 pgmap v511: 73 pgs: 3 active+undersized+degraded, 28 active+undersized, 42 active+clean; 582 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2023-06-03T19:56:49.779 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:56:49 smithi089 bash[75068]: audit 1685822208.972938 mon.a (mon.0) 582 : audit 0 from='client.? 172.21.15.89:0/1042621724' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:56:49.834 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:56:49 smithi164 bash[59462]: cluster 1685822208.3453093 mgr.y (mgr.34104) 603 : cluster 0 pgmap v511: 73 pgs: 3 active+undersized+degraded, 28 active+undersized, 42 active+clean; 582 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2023-06-03T19:56:49.834 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:56:49 smithi164 bash[59462]: audit 1685822208.972938 mon.a (mon.0) 582 : audit 0 from='client.? 172.21.15.89:0/1042621724' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:56:51.779 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:56:51 smithi089 bash[78452]: cluster 1685822210.3460763 mgr.y (mgr.34104) 604 : cluster 0 pgmap v512: 73 pgs: 3 active+undersized+degraded, 28 active+undersized, 42 active+clean; 582 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2023-06-03T19:56:51.779 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:56:51 smithi089 bash[78452]: audit 1685822211.323549 mon.a (mon.0) 583 : audit 0 from='client.? 172.21.15.89:0/3861313481' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:56:51.779 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:56:51 smithi089 bash[75068]: cluster 1685822210.3460763 mgr.y (mgr.34104) 604 : cluster 0 pgmap v512: 73 pgs: 3 active+undersized+degraded, 28 active+undersized, 42 active+clean; 582 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2023-06-03T19:56:51.779 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:56:51 smithi089 bash[75068]: audit 1685822211.323549 mon.a (mon.0) 583 : audit 0 from='client.? 172.21.15.89:0/3861313481' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:56:51.834 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:56:51 smithi164 bash[59462]: cluster 1685822210.3460763 mgr.y (mgr.34104) 604 : cluster 0 pgmap v512: 73 pgs: 3 active+undersized+degraded, 28 active+undersized, 42 active+clean; 582 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2023-06-03T19:56:51.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:56:51 smithi164 bash[59462]: audit 1685822211.323549 mon.a (mon.0) 583 : audit 0 from='client.? 172.21.15.89:0/3861313481' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:56:52.521 INFO:journalctl@ceph.osd.3.smithi089.stdout:Jun 03 19:56:52 smithi089 bash[101590]: debug 2023-06-03T19:56:52.230+0000 7f5a49548640 -1 osd.3 115 log_to_monitors true 2023-06-03T19:56:52.522 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:56:52 smithi089 bash[75068]: audit 1685822212.0876658 mon.c (mon.2) 330 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T19:56:52.522 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:56:52 smithi089 bash[75068]: audit 1685822212.2389982 mon.c (mon.2) 331 : audit 1 from='osd.3 [v2:172.21.15.89:6826/1558463544,v1:172.21.15.89:6827/1558463544]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2023-06-03T19:56:52.522 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:56:52 smithi089 bash[75068]: audit 1685822212.239394 mon.a (mon.0) 584 : audit 1 from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2023-06-03T19:56:52.523 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:56:52 smithi089 bash[78452]: audit 1685822212.0876658 mon.c (mon.2) 330 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T19:56:52.523 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:56:52 smithi089 bash[78452]: audit 1685822212.2389982 mon.c (mon.2) 331 : audit 1 from='osd.3 [v2:172.21.15.89:6826/1558463544,v1:172.21.15.89:6827/1558463544]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2023-06-03T19:56:52.523 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:56:52 smithi089 bash[78452]: audit 1685822212.239394 mon.a (mon.0) 584 : audit 1 from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2023-06-03T19:56:52.778 INFO:journalctl@ceph.osd.3.smithi089.stdout:Jun 03 19:56:52 smithi089 bash[101590]: debug 2023-06-03T19:56:52.514+0000 7f5a391b1700 -1 osd.3 115 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2023-06-03T19:56:52.834 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:56:52 smithi164 bash[59462]: audit 1685822212.0876658 mon.c (mon.2) 330 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T19:56:52.834 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:56:52 smithi164 bash[59462]: audit 1685822212.2389982 mon.c (mon.2) 331 : audit 1 from='osd.3 [v2:172.21.15.89:6826/1558463544,v1:172.21.15.89:6827/1558463544]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2023-06-03T19:56:52.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:56:52 smithi164 bash[59462]: audit 1685822212.239394 mon.a (mon.0) 584 : audit 1 from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2023-06-03T19:56:53.528 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:56:53 smithi089 bash[78452]: cluster 1685822212.3469481 mgr.y (mgr.34104) 605 : cluster 0 pgmap v513: 73 pgs: 3 active+undersized+degraded, 28 active+undersized, 42 active+clean; 582 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2023-06-03T19:56:53.529 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:56:53 smithi089 bash[78452]: cluster 1685822212.4603236 osd.6 (osd.6) 34 : cluster 3 failed to encode map e118 with expected crc 2023-06-03T19:56:53.529 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:56:53 smithi089 bash[78452]: audit 1685822212.4648304 mon.a (mon.0) 585 : audit 1 from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2023-06-03T19:56:53.529 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:56:53 smithi089 bash[78452]: cluster 1685822212.4649184 mon.a (mon.0) 586 : cluster 0 osdmap e118: 8 total, 7 up, 8 in 2023-06-03T19:56:53.529 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:56:53 smithi089 bash[78452]: audit 1685822212.4668672 mon.c (mon.2) 332 : audit 1 from='osd.3 [v2:172.21.15.89:6826/1558463544,v1:172.21.15.89:6827/1558463544]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi089", "root=default"]}]: dispatch 2023-06-03T19:56:53.529 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:56:53 smithi089 bash[78452]: audit 1685822212.4671597 mon.a (mon.0) 587 : audit 1 from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi089", "root=default"]}]: dispatch 2023-06-03T19:56:53.530 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:56:53 smithi089 bash[75068]: cluster 1685822212.3469481 mgr.y (mgr.34104) 605 : cluster 0 pgmap v513: 73 pgs: 3 active+undersized+degraded, 28 active+undersized, 42 active+clean; 582 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2023-06-03T19:56:53.530 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:56:53 smithi089 bash[75068]: cluster 1685822212.4603236 osd.6 (osd.6) 34 : cluster 3 failed to encode map e118 with expected crc 2023-06-03T19:56:53.530 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:56:53 smithi089 bash[75068]: audit 1685822212.4648304 mon.a (mon.0) 585 : audit 1 from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2023-06-03T19:56:53.530 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:56:53 smithi089 bash[75068]: cluster 1685822212.4649184 mon.a (mon.0) 586 : cluster 0 osdmap e118: 8 total, 7 up, 8 in 2023-06-03T19:56:53.530 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:56:53 smithi089 bash[75068]: audit 1685822212.4668672 mon.c (mon.2) 332 : audit 1 from='osd.3 [v2:172.21.15.89:6826/1558463544,v1:172.21.15.89:6827/1558463544]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi089", "root=default"]}]: dispatch 2023-06-03T19:56:53.530 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:56:53 smithi089 bash[75068]: audit 1685822212.4671597 mon.a (mon.0) 587 : audit 1 from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi089", "root=default"]}]: dispatch 2023-06-03T19:56:53.726 INFO:teuthology.orchestra.run.smithi089.stdout: "ceph version 18.0.0-3532-g8e24bcf4 (8e24bcf462a1d94c2e12aa4e26419634883a1e88) reef (rc)": 4 2023-06-03T19:56:53.834 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:56:53 smithi164 bash[59462]: cluster 1685822212.3469481 mgr.y (mgr.34104) 605 : cluster 0 pgmap v513: 73 pgs: 3 active+undersized+degraded, 28 active+undersized, 42 active+clean; 582 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2023-06-03T19:56:53.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:56:53 smithi164 bash[59462]: cluster 1685822212.4603236 osd.6 (osd.6) 34 : cluster 3 failed to encode map e118 with expected crc 2023-06-03T19:56:53.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:56:53 smithi164 bash[59462]: audit 1685822212.4648304 mon.a (mon.0) 585 : audit 1 from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2023-06-03T19:56:53.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:56:53 smithi164 bash[59462]: cluster 1685822212.4649184 mon.a (mon.0) 586 : cluster 0 osdmap e118: 8 total, 7 up, 8 in 2023-06-03T19:56:53.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:56:53 smithi164 bash[59462]: audit 1685822212.4668672 mon.c (mon.2) 332 : audit 1 from='osd.3 [v2:172.21.15.89:6826/1558463544,v1:172.21.15.89:6827/1558463544]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi089", "root=default"]}]: dispatch 2023-06-03T19:56:53.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:56:53 smithi164 bash[59462]: audit 1685822212.4671597 mon.a (mon.0) 587 : audit 1 from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi089", "root=default"]}]: dispatch 2023-06-03T19:56:54.528 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:56:54 smithi089 bash[75068]: cluster 1685822213.4615984 mon.a (mon.0) 588 : cluster 1 Health check cleared: OSD_DOWN (was: 1 osds down) 2023-06-03T19:56:54.529 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:56:54 smithi089 bash[75068]: cluster 1685822213.4642725 osd.6 (osd.6) 35 : cluster 3 failed to encode map e119 with expected crc 2023-06-03T19:56:54.529 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:56:54 smithi089 bash[75068]: cluster 1685822213.469391 mon.a (mon.0) 589 : cluster 1 osd.3 [v2:172.21.15.89:6826/1558463544,v1:172.21.15.89:6827/1558463544] boot 2023-06-03T19:56:54.529 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:56:54 smithi089 bash[75068]: cluster 1685822213.469518 mon.a (mon.0) 590 : cluster 0 osdmap e119: 8 total, 8 up, 8 in 2023-06-03T19:56:54.530 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:56:54 smithi089 bash[75068]: audit 1685822213.4727387 mon.c (mon.2) 333 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-06-03T19:56:54.530 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:56:54 smithi089 bash[75068]: audit 1685822213.7065575 mon.c (mon.2) 334 : audit 0 from='client.? 172.21.15.89:0/3249000352' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:56:54.530 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:56:54 smithi089 bash[78452]: cluster 1685822213.4615984 mon.a (mon.0) 588 : cluster 1 Health check cleared: OSD_DOWN (was: 1 osds down) 2023-06-03T19:56:54.531 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:56:54 smithi089 bash[78452]: cluster 1685822213.4642725 osd.6 (osd.6) 35 : cluster 3 failed to encode map e119 with expected crc 2023-06-03T19:56:54.531 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:56:54 smithi089 bash[78452]: cluster 1685822213.469391 mon.a (mon.0) 589 : cluster 1 osd.3 [v2:172.21.15.89:6826/1558463544,v1:172.21.15.89:6827/1558463544] boot 2023-06-03T19:56:54.531 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:56:54 smithi089 bash[78452]: cluster 1685822213.469518 mon.a (mon.0) 590 : cluster 0 osdmap e119: 8 total, 8 up, 8 in 2023-06-03T19:56:54.531 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:56:54 smithi089 bash[78452]: audit 1685822213.4727387 mon.c (mon.2) 333 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-06-03T19:56:54.532 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:56:54 smithi089 bash[78452]: audit 1685822213.7065575 mon.c (mon.2) 334 : audit 0 from='client.? 172.21.15.89:0/3249000352' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:56:54.836 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:56:54 smithi164 bash[59462]: cluster 1685822213.4615984 mon.a (mon.0) 588 : cluster 1 Health check cleared: OSD_DOWN (was: 1 osds down) 2023-06-03T19:56:54.837 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:56:54 smithi164 bash[59462]: cluster 1685822213.4642725 osd.6 (osd.6) 35 : cluster 3 failed to encode map e119 with expected crc 2023-06-03T19:56:54.837 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:56:54 smithi164 bash[59462]: cluster 1685822213.469391 mon.a (mon.0) 589 : cluster 1 osd.3 [v2:172.21.15.89:6826/1558463544,v1:172.21.15.89:6827/1558463544] boot 2023-06-03T19:56:54.837 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:56:54 smithi164 bash[59462]: cluster 1685822213.469518 mon.a (mon.0) 590 : cluster 0 osdmap e119: 8 total, 8 up, 8 in 2023-06-03T19:56:54.838 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:56:54 smithi164 bash[59462]: audit 1685822213.4727387 mon.c (mon.2) 333 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-06-03T19:56:54.838 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:56:54 smithi164 bash[59462]: audit 1685822213.7065575 mon.c (mon.2) 334 : audit 0 from='client.? 172.21.15.89:0/3249000352' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-06-03T19:56:55.659 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -e sha1=8e24bcf462a1d94c2e12aa4e26419634883a1e88 -- bash -c 'ceph orch upgrade pause' 2023-06-03T19:56:55.779 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:56:55 smithi089 bash[75068]: cluster 1685822214.3476532 mgr.y (mgr.34104) 606 : cluster 0 pgmap v516: 73 pgs: 3 active+undersized+degraded, 28 active+undersized, 42 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2023-06-03T19:56:55.779 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:56:55 smithi089 bash[75068]: cluster 1685822214.489591 osd.6 (osd.6) 36 : cluster 3 failed to encode map e120 with expected crc 2023-06-03T19:56:55.779 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:56:55 smithi089 bash[75068]: cluster 1685822214.4920099 osd.4 (osd.4) 35 : cluster 3 failed to encode map e120 with expected crc 2023-06-03T19:56:55.779 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:56:55 smithi089 bash[75068]: cluster 1685822214.4920297 osd.7 (osd.7) 30 : cluster 3 failed to encode map e120 with expected crc 2023-06-03T19:56:55.779 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:56:55 smithi089 bash[75068]: cluster 1685822214.4930975 osd.5 (osd.5) 42 : cluster 3 failed to encode map e120 with expected crc 2023-06-03T19:56:55.780 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:56:55 smithi089 bash[75068]: cluster 1685822214.4940863 mon.a (mon.0) 591 : cluster 0 osdmap e120: 8 total, 8 up, 8 in 2023-06-03T19:56:55.780 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:56:55 smithi089 bash[75068]: cluster 1685822214.4976947 osd.7 (osd.7) 31 : cluster 3 failed to encode map e120 with expected crc 2023-06-03T19:56:55.780 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:56:55 smithi089 bash[75068]: cluster 1685822214.4985864 osd.5 (osd.5) 43 : cluster 3 failed to encode map e120 with expected crc 2023-06-03T19:56:55.780 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:56:55 smithi089 bash[75068]: cluster 1685822214.5016613 osd.5 (osd.5) 44 : cluster 3 failed to encode map e120 with expected crc 2023-06-03T19:56:55.780 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:56:55 smithi089 bash[78452]: cluster 1685822214.3476532 mgr.y (mgr.34104) 606 : cluster 0 pgmap v516: 73 pgs: 3 active+undersized+degraded, 28 active+undersized, 42 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2023-06-03T19:56:55.780 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:56:55 smithi089 bash[78452]: cluster 1685822214.489591 osd.6 (osd.6) 36 : cluster 3 failed to encode map e120 with expected crc 2023-06-03T19:56:55.781 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:56:55 smithi089 bash[78452]: cluster 1685822214.4920099 osd.4 (osd.4) 35 : cluster 3 failed to encode map e120 with expected crc 2023-06-03T19:56:55.781 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:56:55 smithi089 bash[78452]: cluster 1685822214.4920297 osd.7 (osd.7) 30 : cluster 3 failed to encode map e120 with expected crc 2023-06-03T19:56:55.781 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:56:55 smithi089 bash[78452]: cluster 1685822214.4930975 osd.5 (osd.5) 42 : cluster 3 failed to encode map e120 with expected crc 2023-06-03T19:56:55.781 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:56:55 smithi089 bash[78452]: cluster 1685822214.4940863 mon.a (mon.0) 591 : cluster 0 osdmap e120: 8 total, 8 up, 8 in 2023-06-03T19:56:55.781 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:56:55 smithi089 bash[78452]: cluster 1685822214.4976947 osd.7 (osd.7) 31 : cluster 3 failed to encode map e120 with expected crc 2023-06-03T19:56:55.781 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:56:55 smithi089 bash[78452]: cluster 1685822214.4985864 osd.5 (osd.5) 43 : cluster 3 failed to encode map e120 with expected crc 2023-06-03T19:56:55.781 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:56:55 smithi089 bash[78452]: cluster 1685822214.5016613 osd.5 (osd.5) 44 : cluster 3 failed to encode map e120 with expected crc 2023-06-03T19:56:55.834 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:56:55 smithi164 bash[59462]: cluster 1685822214.3476532 mgr.y (mgr.34104) 606 : cluster 0 pgmap v516: 73 pgs: 3 active+undersized+degraded, 28 active+undersized, 42 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2023-06-03T19:56:55.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:56:55 smithi164 bash[59462]: cluster 1685822214.489591 osd.6 (osd.6) 36 : cluster 3 failed to encode map e120 with expected crc 2023-06-03T19:56:55.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:56:55 smithi164 bash[59462]: cluster 1685822214.4920099 osd.4 (osd.4) 35 : cluster 3 failed to encode map e120 with expected crc 2023-06-03T19:56:55.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:56:55 smithi164 bash[59462]: cluster 1685822214.4920297 osd.7 (osd.7) 30 : cluster 3 failed to encode map e120 with expected crc 2023-06-03T19:56:55.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:56:55 smithi164 bash[59462]: cluster 1685822214.4930975 osd.5 (osd.5) 42 : cluster 3 failed to encode map e120 with expected crc 2023-06-03T19:56:55.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:56:55 smithi164 bash[59462]: cluster 1685822214.4940863 mon.a (mon.0) 591 : cluster 0 osdmap e120: 8 total, 8 up, 8 in 2023-06-03T19:56:55.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:56:55 smithi164 bash[59462]: cluster 1685822214.4976947 osd.7 (osd.7) 31 : cluster 3 failed to encode map e120 with expected crc 2023-06-03T19:56:55.836 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:56:55 smithi164 bash[59462]: cluster 1685822214.4985864 osd.5 (osd.5) 43 : cluster 3 failed to encode map e120 with expected crc 2023-06-03T19:56:55.836 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:56:55 smithi164 bash[59462]: cluster 1685822214.5016613 osd.5 (osd.5) 44 : cluster 3 failed to encode map e120 with expected crc 2023-06-03T19:56:56.529 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 19:56:56 smithi089 bash[71163]: ::ffff:172.21.15.164 - - [03/Jun/2023:19:56:56] "GET /metrics HTTP/1.1" 200 215385 "" "Prometheus/2.33.4" 2023-06-03T19:56:57.747 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:56:57 smithi089 bash[75068]: cluster 1685822216.3481991 mgr.y (mgr.34104) 607 : cluster 0 pgmap v518: 73 pgs: 3 active+undersized+degraded, 21 active+undersized, 49 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2023-06-03T19:56:57.748 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:56:57 smithi089 bash[78452]: cluster 1685822216.3481991 mgr.y (mgr.34104) 607 : cluster 0 pgmap v518: 73 pgs: 3 active+undersized+degraded, 21 active+undersized, 49 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2023-06-03T19:56:57.834 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:56:57 smithi164 bash[59462]: cluster 1685822216.3481991 mgr.y (mgr.34104) 607 : cluster 0 pgmap v518: 73 pgs: 3 active+undersized+degraded, 21 active+undersized, 49 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2023-06-03T19:56:58.778 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:56:58 smithi089 bash[78452]: cluster 1685822218.4918673 mon.a (mon.0) 592 : cluster 1 Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 4/72 objects degraded (5.556%), 3 pgs degraded) 2023-06-03T19:56:58.779 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:56:58 smithi089 bash[78452]: cluster 1685822218.4919248 mon.a (mon.0) 593 : cluster 1 Cluster is now healthy 2023-06-03T19:56:58.779 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:56:58 smithi089 bash[75068]: cluster 1685822218.4918673 mon.a (mon.0) 592 : cluster 1 Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 4/72 objects degraded (5.556%), 3 pgs degraded) 2023-06-03T19:56:58.779 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:56:58 smithi089 bash[75068]: cluster 1685822218.4919248 mon.a (mon.0) 593 : cluster 1 Cluster is now healthy 2023-06-03T19:56:58.834 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:56:58 smithi164 bash[59462]: cluster 1685822218.4918673 mon.a (mon.0) 592 : cluster 1 Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 4/72 objects degraded (5.556%), 3 pgs degraded) 2023-06-03T19:56:58.834 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:56:58 smithi164 bash[59462]: cluster 1685822218.4919248 mon.a (mon.0) 593 : cluster 1 Cluster is now healthy 2023-06-03T19:56:59.584 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:56:59 smithi164 bash[59462]: cluster 1685822218.34885 mgr.y (mgr.34104) 608 : cluster 0 pgmap v519: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2023-06-03T19:56:59.585 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:56:59 smithi164 bash[59462]: audit 1685822218.8815618 mon.a (mon.0) 594 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:56:59.585 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:56:59 smithi164 bash[59462]: audit 1685822218.8946097 mon.a (mon.0) 595 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:56:59.585 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:56:59 smithi164 bash[59462]: audit 1685822218.8979516 mon.c (mon.2) 335 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-03T19:56:59.585 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:56:59 smithi164 bash[59462]: audit 1685822218.8994336 mon.c (mon.2) 336 : audit 1 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-03T19:56:59.585 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:56:59 smithi164 bash[59462]: audit 1685822218.907493 mon.a (mon.0) 596 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:56:59.585 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:56:59 smithi164 bash[59462]: audit 1685822218.986236 mon.c (mon.2) 337 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-03T19:56:59.585 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:56:59 smithi164 bash[59462]: audit 1685822218.989956 mon.c (mon.2) 338 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-06-03T19:56:59.586 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:56:59 smithi164 bash[59462]: audit 1685822218.9920926 mon.c (mon.2) 339 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-06-03T19:56:59.586 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:56:59 smithi164 bash[59462]: audit 1685822218.9936802 mon.c (mon.2) 340 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-06-03T19:56:59.586 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:56:59 smithi164 bash[59462]: audit 1685822218.9957485 mon.c (mon.2) 341 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2023-06-03T19:57:00.029 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:56:59 smithi089 bash[75068]: cluster 1685822218.34885 mgr.y (mgr.34104) 608 : cluster 0 pgmap v519: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2023-06-03T19:57:00.029 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:56:59 smithi089 bash[75068]: audit 1685822218.8815618 mon.a (mon.0) 594 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:57:00.029 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:56:59 smithi089 bash[75068]: audit 1685822218.8946097 mon.a (mon.0) 595 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:57:00.029 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:56:59 smithi089 bash[75068]: audit 1685822218.8979516 mon.c (mon.2) 335 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-03T19:57:00.029 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:56:59 smithi089 bash[75068]: audit 1685822218.8994336 mon.c (mon.2) 336 : audit 1 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-03T19:57:00.030 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:56:59 smithi089 bash[75068]: audit 1685822218.907493 mon.a (mon.0) 596 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:57:00.030 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:56:59 smithi089 bash[75068]: audit 1685822218.986236 mon.c (mon.2) 337 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-03T19:57:00.030 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:56:59 smithi089 bash[75068]: audit 1685822218.989956 mon.c (mon.2) 338 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-06-03T19:57:00.030 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:56:59 smithi089 bash[75068]: audit 1685822218.9920926 mon.c (mon.2) 339 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-06-03T19:57:00.030 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:56:59 smithi089 bash[75068]: audit 1685822218.9936802 mon.c (mon.2) 340 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-06-03T19:57:00.030 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:56:59 smithi089 bash[75068]: audit 1685822218.9957485 mon.c (mon.2) 341 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2023-06-03T19:57:00.031 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:56:59 smithi089 bash[78452]: cluster 1685822218.34885 mgr.y (mgr.34104) 608 : cluster 0 pgmap v519: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2023-06-03T19:57:00.031 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:56:59 smithi089 bash[78452]: audit 1685822218.8815618 mon.a (mon.0) 594 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:57:00.031 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:56:59 smithi089 bash[78452]: audit 1685822218.8946097 mon.a (mon.0) 595 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:57:00.031 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:56:59 smithi089 bash[78452]: audit 1685822218.8979516 mon.c (mon.2) 335 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-03T19:57:00.031 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:56:59 smithi089 bash[78452]: audit 1685822218.8994336 mon.c (mon.2) 336 : audit 1 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-03T19:57:00.031 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:56:59 smithi089 bash[78452]: audit 1685822218.907493 mon.a (mon.0) 596 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:57:00.031 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:56:59 smithi089 bash[78452]: audit 1685822218.986236 mon.c (mon.2) 337 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-03T19:57:00.032 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:56:59 smithi089 bash[78452]: audit 1685822218.989956 mon.c (mon.2) 338 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-06-03T19:57:00.032 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:56:59 smithi089 bash[78452]: audit 1685822218.9920926 mon.c (mon.2) 339 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-06-03T19:57:00.032 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:56:59 smithi089 bash[78452]: audit 1685822218.9936802 mon.c (mon.2) 340 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-06-03T19:57:00.032 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:56:59 smithi089 bash[78452]: audit 1685822218.9957485 mon.c (mon.2) 341 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2023-06-03T19:57:00.747 INFO:teuthology.orchestra.run.smithi089.stdout:Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:00b1b6d689278b4f97ff790045c8c51d8ca419ac7ee2ebdf11008f2e810a5c78 2023-06-03T19:57:00.811 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:57:00 smithi089 bash[75068]: audit 1685822218.9959996 mgr.y (mgr.34104) 609 : audit 0 from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2023-06-03T19:57:00.811 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:57:00 smithi089 bash[75068]: cephadm 1685822218.997248 mgr.y (mgr.34104) 610 : cephadm 1 Upgrade: osd.4 is safe to restart 2023-06-03T19:57:00.812 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:57:00 smithi089 bash[78452]: audit 1685822218.9959996 mgr.y (mgr.34104) 609 : audit 0 from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2023-06-03T19:57:00.812 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:57:00 smithi089 bash[78452]: cephadm 1685822218.997248 mgr.y (mgr.34104) 610 : cephadm 1 Upgrade: osd.4 is safe to restart 2023-06-03T19:57:00.834 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:57:00 smithi164 bash[59462]: audit 1685822218.9959996 mgr.y (mgr.34104) 609 : audit 0 from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2023-06-03T19:57:00.834 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:57:00 smithi164 bash[59462]: cephadm 1685822218.997248 mgr.y (mgr.34104) 610 : cephadm 1 Upgrade: osd.4 is safe to restart 2023-06-03T19:57:01.586 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -e sha1=8e24bcf462a1d94c2e12aa4e26419634883a1e88 -- bash -c 'ceph orch ps' 2023-06-03T19:57:02.028 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:57:01 smithi089 bash[75068]: cluster 1685822220.349447 mgr.y (mgr.34104) 611 : cluster 0 pgmap v520: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2023-06-03T19:57:02.029 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:57:01 smithi089 bash[75068]: audit 1685822220.7334573 mgr.y (mgr.34104) 612 : audit 0 from='client.45473 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2023-06-03T19:57:02.029 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:57:01 smithi089 bash[75068]: cephadm 1685822220.734099 mgr.y (mgr.34104) 613 : cephadm 1 Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:00b1b6d689278b4f97ff790045c8c51d8ca419ac7ee2ebdf11008f2e810a5c78 2023-06-03T19:57:02.029 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:57:01 smithi089 bash[75068]: audit 1685822220.7413616 mon.a (mon.0) 597 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:57:02.029 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:57:01 smithi089 bash[78452]: cluster 1685822220.349447 mgr.y (mgr.34104) 611 : cluster 0 pgmap v520: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2023-06-03T19:57:02.030 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:57:01 smithi089 bash[78452]: audit 1685822220.7334573 mgr.y (mgr.34104) 612 : audit 0 from='client.45473 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2023-06-03T19:57:02.030 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:57:01 smithi089 bash[78452]: cephadm 1685822220.734099 mgr.y (mgr.34104) 613 : cephadm 1 Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:00b1b6d689278b4f97ff790045c8c51d8ca419ac7ee2ebdf11008f2e810a5c78 2023-06-03T19:57:02.030 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:57:01 smithi089 bash[78452]: audit 1685822220.7413616 mon.a (mon.0) 597 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:57:02.084 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:57:01 smithi164 bash[59462]: cluster 1685822220.349447 mgr.y (mgr.34104) 611 : cluster 0 pgmap v520: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2023-06-03T19:57:02.085 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:57:01 smithi164 bash[59462]: audit 1685822220.7334573 mgr.y (mgr.34104) 612 : audit 0 from='client.45473 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2023-06-03T19:57:02.085 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:57:01 smithi164 bash[59462]: cephadm 1685822220.734099 mgr.y (mgr.34104) 613 : cephadm 1 Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:00b1b6d689278b4f97ff790045c8c51d8ca419ac7ee2ebdf11008f2e810a5c78 2023-06-03T19:57:02.085 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:57:01 smithi164 bash[59462]: audit 1685822220.7413616 mon.a (mon.0) 597 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:57:03.136 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:57:02 smithi089 bash[75068]: cephadm 1685822221.968614 mgr.y (mgr.34104) 614 : cephadm 1 Upgrade: Updating osd.4 2023-06-03T19:57:03.137 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:57:02 smithi089 bash[75068]: audit 1685822221.9750984 mon.a (mon.0) 598 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:57:03.137 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:57:02 smithi089 bash[75068]: audit 1685822221.9793162 mon.c (mon.2) 342 : audit 1 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2023-06-03T19:57:03.137 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:57:02 smithi089 bash[75068]: audit 1685822221.9804282 mon.c (mon.2) 343 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-03T19:57:03.137 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:57:02 smithi089 bash[75068]: cephadm 1685822221.9837224 mgr.y (mgr.34104) 615 : cephadm 1 Deploying daemon osd.4 on smithi164 2023-06-03T19:57:03.137 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:57:02 smithi089 bash[75068]: cluster 1685822222.3500152 mgr.y (mgr.34104) 616 : cluster 0 pgmap v521: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2023-06-03T19:57:03.137 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:57:02 smithi089 bash[75068]: audit 1685822222.4204335 mon.a (mon.0) 599 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:57:03.137 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:57:02 smithi089 bash[78452]: cephadm 1685822221.968614 mgr.y (mgr.34104) 614 : cephadm 1 Upgrade: Updating osd.4 2023-06-03T19:57:03.138 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:57:02 smithi089 bash[78452]: audit 1685822221.9750984 mon.a (mon.0) 598 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:57:03.138 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:57:02 smithi089 bash[78452]: audit 1685822221.9793162 mon.c (mon.2) 342 : audit 1 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2023-06-03T19:57:03.138 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:57:02 smithi089 bash[78452]: audit 1685822221.9804282 mon.c (mon.2) 343 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-03T19:57:03.138 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:57:02 smithi089 bash[78452]: cephadm 1685822221.9837224 mgr.y (mgr.34104) 615 : cephadm 1 Deploying daemon osd.4 on smithi164 2023-06-03T19:57:03.138 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:57:02 smithi089 bash[78452]: cluster 1685822222.3500152 mgr.y (mgr.34104) 616 : cluster 0 pgmap v521: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2023-06-03T19:57:03.138 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:57:02 smithi089 bash[78452]: audit 1685822222.4204335 mon.a (mon.0) 599 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:57:03.334 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:57:02 smithi164 bash[59462]: cephadm 1685822221.968614 mgr.y (mgr.34104) 614 : cephadm 1 Upgrade: Updating osd.4 2023-06-03T19:57:03.334 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:57:02 smithi164 bash[59462]: audit 1685822221.9750984 mon.a (mon.0) 598 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:57:03.335 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:57:02 smithi164 bash[59462]: audit 1685822221.9793162 mon.c (mon.2) 342 : audit 1 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2023-06-03T19:57:03.335 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:57:02 smithi164 bash[59462]: audit 1685822221.9804282 mon.c (mon.2) 343 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-03T19:57:03.335 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:57:02 smithi164 bash[59462]: cephadm 1685822221.9837224 mgr.y (mgr.34104) 615 : cephadm 1 Deploying daemon osd.4 on smithi164 2023-06-03T19:57:03.335 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:57:02 smithi164 bash[59462]: cluster 1685822222.3500152 mgr.y (mgr.34104) 616 : cluster 0 pgmap v521: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2023-06-03T19:57:03.335 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:57:02 smithi164 bash[59462]: audit 1685822222.4204335 mon.a (mon.0) 599 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:57:05.172 INFO:teuthology.orchestra.run.smithi089.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2023-06-03T19:57:05.172 INFO:teuthology.orchestra.run.smithi089.stdout:alertmanager.a smithi089 *:9093,9094 running (18m) 19s ago 24m 14.6M - 0.23.0 ba2b418f427c 0fad3b643b2e 2023-06-03T19:57:05.172 INFO:teuthology.orchestra.run.smithi089.stdout:grafana.a smithi164 *:3000 running (17m) 20s ago 23m 41.5M - 8.3.5 dad864ee21e9 64925a21f38b 2023-06-03T19:57:05.172 INFO:teuthology.orchestra.run.smithi089.stdout:mds.foo.smithi089.giuxuj smithi089 running (22m) 19s ago 22m 20.2M - 16.2.13-257-gd8c5d349 b644dd95899d f0e9de24a3f8 2023-06-03T19:57:05.172 INFO:teuthology.orchestra.run.smithi089.stdout:mds.foo.smithi164.txgfgv smithi164 running (22m) 20s ago 22m 16.5M - 16.2.13-257-gd8c5d349 b644dd95899d 29da307aa3b4 2023-06-03T19:57:05.173 INFO:teuthology.orchestra.run.smithi089.stdout:mgr.x smithi164 *:8443,9283,8765 running (16m) 20s ago 31m 430M - 18.0.0-3532-g8e24bcf4 a57053f96fe3 3ada341a2c72 2023-06-03T19:57:05.173 INFO:teuthology.orchestra.run.smithi089.stdout:mgr.y smithi089 *:8443,9283,8765 running (17m) 19s ago 34m 492M - 18.0.0-3532-g8e24bcf4 a57053f96fe3 8307fbe12a7d 2023-06-03T19:57:05.173 INFO:teuthology.orchestra.run.smithi089.stdout:mon.a smithi089 running (15m) 19s ago 34m 98.2M 2048M 18.0.0-3532-g8e24bcf4 a57053f96fe3 c5e2e407c45f 2023-06-03T19:57:05.173 INFO:teuthology.orchestra.run.smithi089.stdout:mon.b smithi164 running (12m) 20s ago 32m 51.7M 2048M 18.0.0-3532-g8e24bcf4 a57053f96fe3 c0685be70224 2023-06-03T19:57:05.173 INFO:teuthology.orchestra.run.smithi089.stdout:mon.c smithi089 running (14m) 19s ago 32m 61.0M 2048M 18.0.0-3532-g8e24bcf4 a57053f96fe3 f10053d47b03 2023-06-03T19:57:05.173 INFO:teuthology.orchestra.run.smithi089.stdout:node-exporter.a smithi089 *:9100 running (18m) 19s ago 24m 9900k - 1.3.1 1dbe0e931976 f41a3d45da20 2023-06-03T19:57:05.173 INFO:teuthology.orchestra.run.smithi089.stdout:node-exporter.b smithi164 *:9100 running (18m) 20s ago 24m 9704k - 1.3.1 1dbe0e931976 1e69859880c6 2023-06-03T19:57:05.174 INFO:teuthology.orchestra.run.smithi089.stdout:osd.0 smithi089 running (10m) 19s ago 31m 751M 2526M 18.0.0-3532-g8e24bcf4 a57053f96fe3 53d61ab317ba 2023-06-03T19:57:05.174 INFO:teuthology.orchestra.run.smithi089.stdout:osd.1 smithi089 running (5m) 19s ago 30m 542M 2526M 18.0.0-3532-g8e24bcf4 a57053f96fe3 6566cc8d9582 2023-06-03T19:57:05.174 INFO:teuthology.orchestra.run.smithi089.stdout:osd.2 smithi089 running (69s) 19s ago 29m 491M 2526M 18.0.0-3532-g8e24bcf4 a57053f96fe3 19ba71d65561 2023-06-03T19:57:05.174 INFO:teuthology.orchestra.run.smithi089.stdout:osd.3 smithi089 running (23s) 19s ago 29m - 2526M 18.0.0-3532-g8e24bcf4 a57053f96fe3 1bfa13c225ba 2023-06-03T19:57:05.174 INFO:teuthology.orchestra.run.smithi089.stdout:osd.4 smithi164 running (28m) 20s ago 28m 1126M 2526M 16.2.13-257-gd8c5d349 b644dd95899d 5c901af1a184 2023-06-03T19:57:05.174 INFO:teuthology.orchestra.run.smithi089.stdout:osd.5 smithi164 running (27m) 20s ago 27m 1030M 2526M 16.2.13-257-gd8c5d349 b644dd95899d 13ddc40f04c4 2023-06-03T19:57:05.174 INFO:teuthology.orchestra.run.smithi089.stdout:osd.6 smithi164 running (26m) 20s ago 26m 1061M 2526M 16.2.13-257-gd8c5d349 b644dd95899d cab2cb9998d7 2023-06-03T19:57:05.175 INFO:teuthology.orchestra.run.smithi089.stdout:osd.7 smithi164 running (25m) 20s ago 26m 595M 2526M 16.2.13-257-gd8c5d349 b644dd95899d b5fd865771b6 2023-06-03T19:57:05.175 INFO:teuthology.orchestra.run.smithi089.stdout:prometheus.a smithi164 *:9095 running (16m) 20s ago 24m 77.1M - 2.33.4 514e6a882f6e 3ff744731c36 2023-06-03T19:57:05.584 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:57:05 smithi089 bash[75068]: cluster 1685822224.350872 mgr.y (mgr.34104) 617 : cluster 0 pgmap v522: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2023-06-03T19:57:05.834 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:57:05 smithi164 bash[59462]: cluster 1685822224.350872 mgr.y (mgr.34104) 617 : cluster 0 pgmap v522: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2023-06-03T19:57:05.986 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:57:05 smithi089 bash[78452]: cluster 1685822224.350872 mgr.y (mgr.34104) 617 : cluster 0 pgmap v522: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2023-06-03T19:57:05.987 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -e sha1=8e24bcf462a1d94c2e12aa4e26419634883a1e88 -- bash -c 'ceph orch ps' 2023-06-03T19:57:06.279 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 19:57:06 smithi089 bash[71163]: ::ffff:172.21.15.164 - - [03/Jun/2023:19:57:06] "GET /metrics HTTP/1.1" 200 215385 "" "Prometheus/2.33.4" 2023-06-03T19:57:06.529 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:57:06 smithi089 bash[78452]: audit 1685822225.1477952 mgr.y (mgr.34104) 618 : audit 0 from='client.45479 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-06-03T19:57:06.529 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:57:06 smithi089 bash[75068]: audit 1685822225.1477952 mgr.y (mgr.34104) 618 : audit 0 from='client.45479 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-06-03T19:57:06.802 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:57:06 smithi164 bash[59462]: audit 1685822225.1477952 mgr.y (mgr.34104) 618 : audit 0 from='client.45479 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-06-03T19:57:07.085 INFO:journalctl@ceph.osd.4.smithi164.stdout:Jun 03 19:57:07 smithi164 systemd[1]: Stopping Ceph osd.4 for f1db3478-0243-11ee-9b1d-001a4aab830c... 2023-06-03T19:57:07.584 INFO:journalctl@ceph.osd.4.smithi164.stdout:Jun 03 19:57:07 smithi164 bash[30695]: debug 2023-06-03T19:57:07.177+0000 7f7d543cd700 -1 received signal: Terminated from /sbin/docker-init -- /usr/bin/ceph-osd -n osd.4 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug (PID: 1) UID: 0 2023-06-03T19:57:07.585 INFO:journalctl@ceph.osd.4.smithi164.stdout:Jun 03 19:57:07 smithi164 bash[30695]: debug 2023-06-03T19:57:07.177+0000 7f7d543cd700 -1 osd.4 120 *** Got signal Terminated *** 2023-06-03T19:57:07.585 INFO:journalctl@ceph.osd.4.smithi164.stdout:Jun 03 19:57:07 smithi164 bash[30695]: debug 2023-06-03T19:57:07.177+0000 7f7d543cd700 -1 osd.4 120 *** Immediate shutdown (osd_fast_shutdown=true) *** 2023-06-03T19:57:07.585 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:57:07 smithi164 bash[59462]: cluster 1685822226.351583 mgr.y (mgr.34104) 619 : cluster 0 pgmap v523: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2023-06-03T19:57:07.586 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:57:07 smithi164 bash[59462]: audit 1685822227.0957832 mon.a (mon.0) 600 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:57:07.586 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:57:07 smithi164 bash[59462]: audit 1685822227.0976214 mon.c (mon.2) 344 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T19:57:07.586 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:57:07 smithi164 bash[59462]: cluster 1685822227.1827335 mon.a (mon.0) 601 : cluster 1 osd.4 marked itself down and dead 2023-06-03T19:57:07.702 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:57:07 smithi089 bash[78452]: cluster 1685822226.351583 mgr.y (mgr.34104) 619 : cluster 0 pgmap v523: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2023-06-03T19:57:07.702 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:57:07 smithi089 bash[78452]: audit 1685822227.0957832 mon.a (mon.0) 600 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:57:07.702 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:57:07 smithi089 bash[78452]: audit 1685822227.0976214 mon.c (mon.2) 344 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T19:57:07.702 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:57:07 smithi089 bash[78452]: cluster 1685822227.1827335 mon.a (mon.0) 601 : cluster 1 osd.4 marked itself down and dead 2023-06-03T19:57:07.703 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:57:07 smithi089 bash[75068]: cluster 1685822226.351583 mgr.y (mgr.34104) 619 : cluster 0 pgmap v523: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2023-06-03T19:57:07.703 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:57:07 smithi089 bash[75068]: audit 1685822227.0957832 mon.a (mon.0) 600 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:57:07.703 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:57:07 smithi089 bash[75068]: audit 1685822227.0976214 mon.c (mon.2) 344 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T19:57:07.703 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:57:07 smithi089 bash[75068]: cluster 1685822227.1827335 mon.a (mon.0) 601 : cluster 1 osd.4 marked itself down and dead 2023-06-03T19:57:08.584 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:57:08 smithi164 bash[59462]: cluster 1685822228.0925558 mon.a (mon.0) 602 : cluster 3 Health check failed: 1 osds down (OSD_DOWN) 2023-06-03T19:57:08.584 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:57:08 smithi164 bash[59462]: cluster 1685822228.0979905 osd.5 (osd.5) 45 : cluster 3 failed to encode map e121 with expected crc 2023-06-03T19:57:08.585 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:57:08 smithi164 bash[59462]: cluster 1685822228.1020133 mon.a (mon.0) 603 : cluster 0 osdmap e121: 8 total, 7 up, 8 in 2023-06-03T19:57:08.779 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:57:08 smithi089 bash[75068]: cluster 1685822228.0925558 mon.a (mon.0) 602 : cluster 3 Health check failed: 1 osds down (OSD_DOWN) 2023-06-03T19:57:08.779 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:57:08 smithi089 bash[75068]: cluster 1685822228.0979905 osd.5 (osd.5) 45 : cluster 3 failed to encode map e121 with expected crc 2023-06-03T19:57:08.779 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:57:08 smithi089 bash[75068]: cluster 1685822228.1020133 mon.a (mon.0) 603 : cluster 0 osdmap e121: 8 total, 7 up, 8 in 2023-06-03T19:57:08.779 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:57:08 smithi089 bash[78452]: cluster 1685822228.0925558 mon.a (mon.0) 602 : cluster 3 Health check failed: 1 osds down (OSD_DOWN) 2023-06-03T19:57:08.779 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:57:08 smithi089 bash[78452]: cluster 1685822228.0979905 osd.5 (osd.5) 45 : cluster 3 failed to encode map e121 with expected crc 2023-06-03T19:57:08.780 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:57:08 smithi089 bash[78452]: cluster 1685822228.1020133 mon.a (mon.0) 603 : cluster 0 osdmap e121: 8 total, 7 up, 8 in 2023-06-03T19:57:09.465 INFO:journalctl@ceph.osd.4.smithi164.stdout:Jun 03 19:57:09 smithi164 bash[65528]: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd-4 2023-06-03T19:57:09.779 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:57:09 smithi089 bash[75068]: cluster 1685822228.0955915 osd.7 (osd.7) 32 : cluster 3 failed to encode map e121 with expected crc 2023-06-03T19:57:09.779 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:57:09 smithi089 bash[75068]: cluster 1685822228.3523693 mgr.y (mgr.34104) 620 : cluster 0 pgmap v525: 73 pgs: 13 stale+active+clean, 60 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2023-06-03T19:57:09.779 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:57:09 smithi089 bash[75068]: cluster 1685822229.1036365 osd.5 (osd.5) 46 : cluster 3 failed to encode map e122 with expected crc 2023-06-03T19:57:09.779 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:57:09 smithi089 bash[75068]: cluster 1685822229.1036992 osd.6 (osd.6) 37 : cluster 3 failed to encode map e122 with expected crc 2023-06-03T19:57:09.779 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:57:09 smithi089 bash[75068]: cluster 1685822229.1056564 mon.a (mon.0) 604 : cluster 0 osdmap e122: 8 total, 7 up, 8 in 2023-06-03T19:57:09.780 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:57:09 smithi089 bash[78452]: cluster 1685822228.0955915 osd.7 (osd.7) 32 : cluster 3 failed to encode map e121 with expected crc 2023-06-03T19:57:09.780 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:57:09 smithi089 bash[78452]: cluster 1685822228.3523693 mgr.y (mgr.34104) 620 : cluster 0 pgmap v525: 73 pgs: 13 stale+active+clean, 60 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2023-06-03T19:57:09.780 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:57:09 smithi089 bash[78452]: cluster 1685822229.1036365 osd.5 (osd.5) 46 : cluster 3 failed to encode map e122 with expected crc 2023-06-03T19:57:09.780 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:57:09 smithi089 bash[78452]: cluster 1685822229.1036992 osd.6 (osd.6) 37 : cluster 3 failed to encode map e122 with expected crc 2023-06-03T19:57:09.780 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:57:09 smithi089 bash[78452]: cluster 1685822229.1056564 mon.a (mon.0) 604 : cluster 0 osdmap e122: 8 total, 7 up, 8 in 2023-06-03T19:57:09.834 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:57:09 smithi164 bash[59462]: cluster 1685822228.0955915 osd.7 (osd.7) 32 : cluster 3 failed to encode map e121 with expected crc 2023-06-03T19:57:09.834 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:57:09 smithi164 bash[59462]: cluster 1685822228.3523693 mgr.y (mgr.34104) 620 : cluster 0 pgmap v525: 73 pgs: 13 stale+active+clean, 60 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2023-06-03T19:57:09.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:57:09 smithi164 bash[59462]: cluster 1685822229.1036365 osd.5 (osd.5) 46 : cluster 3 failed to encode map e122 with expected crc 2023-06-03T19:57:09.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:57:09 smithi164 bash[59462]: cluster 1685822229.1036992 osd.6 (osd.6) 37 : cluster 3 failed to encode map e122 with expected crc 2023-06-03T19:57:09.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:57:09 smithi164 bash[59462]: cluster 1685822229.1056564 mon.a (mon.0) 604 : cluster 0 osdmap e122: 8 total, 7 up, 8 in 2023-06-03T19:57:10.065 INFO:teuthology.orchestra.run.smithi089.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2023-06-03T19:57:10.066 INFO:teuthology.orchestra.run.smithi089.stdout:alertmanager.a smithi089 *:9093,9094 running (18m) 23s ago 24m 14.6M - 0.23.0 ba2b418f427c 0fad3b643b2e 2023-06-03T19:57:10.066 INFO:teuthology.orchestra.run.smithi089.stdout:grafana.a smithi164 *:3000 running (17m) 25s ago 24m 41.5M - 8.3.5 dad864ee21e9 64925a21f38b 2023-06-03T19:57:10.066 INFO:teuthology.orchestra.run.smithi089.stdout:mds.foo.smithi089.giuxuj smithi089 running (22m) 23s ago 22m 20.2M - 16.2.13-257-gd8c5d349 b644dd95899d f0e9de24a3f8 2023-06-03T19:57:10.066 INFO:teuthology.orchestra.run.smithi089.stdout:mds.foo.smithi164.txgfgv smithi164 running (22m) 25s ago 22m 16.5M - 16.2.13-257-gd8c5d349 b644dd95899d 29da307aa3b4 2023-06-03T19:57:10.066 INFO:teuthology.orchestra.run.smithi089.stdout:mgr.x smithi164 *:8443,9283,8765 running (16m) 25s ago 32m 430M - 18.0.0-3532-g8e24bcf4 a57053f96fe3 3ada341a2c72 2023-06-03T19:57:10.067 INFO:teuthology.orchestra.run.smithi089.stdout:mgr.y smithi089 *:8443,9283,8765 running (17m) 23s ago 34m 492M - 18.0.0-3532-g8e24bcf4 a57053f96fe3 8307fbe12a7d 2023-06-03T19:57:10.067 INFO:teuthology.orchestra.run.smithi089.stdout:mon.a smithi089 running (15m) 23s ago 34m 98.2M 2048M 18.0.0-3532-g8e24bcf4 a57053f96fe3 c5e2e407c45f 2023-06-03T19:57:10.067 INFO:teuthology.orchestra.run.smithi089.stdout:mon.b smithi164 running (12m) 25s ago 32m 51.7M 2048M 18.0.0-3532-g8e24bcf4 a57053f96fe3 c0685be70224 2023-06-03T19:57:10.067 INFO:teuthology.orchestra.run.smithi089.stdout:mon.c smithi089 running (14m) 23s ago 32m 61.0M 2048M 18.0.0-3532-g8e24bcf4 a57053f96fe3 f10053d47b03 2023-06-03T19:57:10.067 INFO:teuthology.orchestra.run.smithi089.stdout:node-exporter.a smithi089 *:9100 running (18m) 23s ago 25m 9900k - 1.3.1 1dbe0e931976 f41a3d45da20 2023-06-03T19:57:10.067 INFO:teuthology.orchestra.run.smithi089.stdout:node-exporter.b smithi164 *:9100 running (18m) 25s ago 25m 9704k - 1.3.1 1dbe0e931976 1e69859880c6 2023-06-03T19:57:10.068 INFO:teuthology.orchestra.run.smithi089.stdout:osd.0 smithi089 running (10m) 23s ago 31m 751M 2526M 18.0.0-3532-g8e24bcf4 a57053f96fe3 53d61ab317ba 2023-06-03T19:57:10.068 INFO:teuthology.orchestra.run.smithi089.stdout:osd.1 smithi089 running (5m) 23s ago 30m 542M 2526M 18.0.0-3532-g8e24bcf4 a57053f96fe3 6566cc8d9582 2023-06-03T19:57:10.068 INFO:teuthology.orchestra.run.smithi089.stdout:osd.2 smithi089 running (74s) 23s ago 29m 491M 2526M 18.0.0-3532-g8e24bcf4 a57053f96fe3 19ba71d65561 2023-06-03T19:57:10.068 INFO:teuthology.orchestra.run.smithi089.stdout:osd.3 smithi089 running (28s) 23s ago 29m - 2526M 18.0.0-3532-g8e24bcf4 a57053f96fe3 1bfa13c225ba 2023-06-03T19:57:10.068 INFO:teuthology.orchestra.run.smithi089.stdout:osd.4 smithi164 running (28m) 25s ago 28m 1126M 2526M 16.2.13-257-gd8c5d349 b644dd95899d 5c901af1a184 2023-06-03T19:57:10.068 INFO:teuthology.orchestra.run.smithi089.stdout:osd.5 smithi164 running (27m) 25s ago 27m 1030M 2526M 16.2.13-257-gd8c5d349 b644dd95899d 13ddc40f04c4 2023-06-03T19:57:10.068 INFO:teuthology.orchestra.run.smithi089.stdout:osd.6 smithi164 running (26m) 25s ago 26m 1061M 2526M 16.2.13-257-gd8c5d349 b644dd95899d cab2cb9998d7 2023-06-03T19:57:10.069 INFO:teuthology.orchestra.run.smithi089.stdout:osd.7 smithi164 running (26m) 25s ago 26m 595M 2526M 16.2.13-257-gd8c5d349 b644dd95899d b5fd865771b6 2023-06-03T19:57:10.069 INFO:teuthology.orchestra.run.smithi089.stdout:prometheus.a smithi164 *:9095 running (16m) 25s ago 25m 77.1M - 2.33.4 514e6a882f6e 3ff744731c36 2023-06-03T19:57:10.779 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:57:10 smithi089 bash[75068]: cluster 1685822229.105018 osd.7 (osd.7) 33 : cluster 3 failed to encode map e122 with expected crc 2023-06-03T19:57:10.779 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:57:10 smithi089 bash[75068]: cluster 1685822229.1064012 osd.7 (osd.7) 34 : cluster 3 failed to encode map e122 with expected crc 2023-06-03T19:57:10.779 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:57:10 smithi089 bash[75068]: cluster 1685822229.108402 osd.7 (osd.7) 35 : cluster 3 failed to encode map e122 with expected crc 2023-06-03T19:57:10.779 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:57:10 smithi089 bash[78452]: cluster 1685822229.105018 osd.7 (osd.7) 33 : cluster 3 failed to encode map e122 with expected crc 2023-06-03T19:57:10.780 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:57:10 smithi089 bash[78452]: cluster 1685822229.1064012 osd.7 (osd.7) 34 : cluster 3 failed to encode map e122 with expected crc 2023-06-03T19:57:10.780 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:57:10 smithi089 bash[78452]: cluster 1685822229.108402 osd.7 (osd.7) 35 : cluster 3 failed to encode map e122 with expected crc 2023-06-03T19:57:10.834 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:57:10 smithi164 bash[59462]: cluster 1685822229.105018 osd.7 (osd.7) 33 : cluster 3 failed to encode map e122 with expected crc 2023-06-03T19:57:10.834 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:57:10 smithi164 bash[59462]: cluster 1685822229.1064012 osd.7 (osd.7) 34 : cluster 3 failed to encode map e122 with expected crc 2023-06-03T19:57:10.834 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:57:10 smithi164 bash[59462]: cluster 1685822229.108402 osd.7 (osd.7) 35 : cluster 3 failed to encode map e122 with expected crc 2023-06-03T19:57:10.906 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -e sha1=8e24bcf462a1d94c2e12aa4e26419634883a1e88 -- bash -c 'ceph versions' 2023-06-03T19:57:11.779 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:57:11 smithi089 bash[78452]: audit 1685822230.0410464 mgr.y (mgr.34104) 621 : audit 0 from='client.45485 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-06-03T19:57:11.779 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:57:11 smithi089 bash[78452]: cluster 1685822230.3532507 mgr.y (mgr.34104) 622 : cluster 0 pgmap v527: 73 pgs: 3 active+undersized, 10 stale+active+clean, 60 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2023-06-03T19:57:11.779 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:57:11 smithi089 bash[75068]: audit 1685822230.0410464 mgr.y (mgr.34104) 621 : audit 0 from='client.45485 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-06-03T19:57:11.779 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:57:11 smithi089 bash[75068]: cluster 1685822230.3532507 mgr.y (mgr.34104) 622 : cluster 0 pgmap v527: 73 pgs: 3 active+undersized, 10 stale+active+clean, 60 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2023-06-03T19:57:11.834 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:57:11 smithi164 bash[59462]: audit 1685822230.0410464 mgr.y (mgr.34104) 621 : audit 0 from='client.45485 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-06-03T19:57:11.834 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:57:11 smithi164 bash[59462]: cluster 1685822230.3532507 mgr.y (mgr.34104) 622 : cluster 0 pgmap v527: 73 pgs: 3 active+undersized, 10 stale+active+clean, 60 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2023-06-03T19:57:11.835 INFO:journalctl@ceph.osd.4.smithi164.stdout:Jun 03 19:57:11 smithi164 systemd[1]: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c@osd.4.service: Succeeded. 2023-06-03T19:57:11.835 INFO:journalctl@ceph.osd.4.smithi164.stdout:Jun 03 19:57:11 smithi164 systemd[1]: Stopped Ceph osd.4 for f1db3478-0243-11ee-9b1d-001a4aab830c. 2023-06-03T19:57:12.334 INFO:journalctl@ceph.osd.4.smithi164.stdout:Jun 03 19:57:11 smithi164 systemd[1]: Started Ceph osd.4 for f1db3478-0243-11ee-9b1d-001a4aab830c. 2023-06-03T19:57:13.029 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:57:12 smithi089 bash[75068]: audit 1685822231.9411561 mon.a (mon.0) 605 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:57:13.029 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:57:12 smithi089 bash[75068]: audit 1685822231.9504094 mon.a (mon.0) 606 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:57:13.029 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:57:12 smithi089 bash[75068]: audit 1685822231.974176 mon.c (mon.2) 345 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-03T19:57:13.029 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:57:12 smithi089 bash[75068]: cluster 1685822232.353814 mgr.y (mgr.34104) 623 : cluster 0 pgmap v528: 73 pgs: 1 active+undersized+degraded, 7 active+undersized, 9 stale+active+clean, 56 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 1/72 objects degraded (1.389%) 2023-06-03T19:57:13.030 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:57:12 smithi089 bash[78452]: audit 1685822231.9411561 mon.a (mon.0) 605 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:57:13.030 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:57:12 smithi089 bash[78452]: audit 1685822231.9504094 mon.a (mon.0) 606 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:57:13.030 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:57:12 smithi089 bash[78452]: audit 1685822231.974176 mon.c (mon.2) 345 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-03T19:57:13.030 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:57:12 smithi089 bash[78452]: cluster 1685822232.353814 mgr.y (mgr.34104) 623 : cluster 0 pgmap v528: 73 pgs: 1 active+undersized+degraded, 7 active+undersized, 9 stale+active+clean, 56 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 1/72 objects degraded (1.389%) 2023-06-03T19:57:13.334 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:57:12 smithi164 bash[59462]: audit 1685822231.9411561 mon.a (mon.0) 605 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:57:13.334 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:57:12 smithi164 bash[59462]: audit 1685822231.9504094 mon.a (mon.0) 606 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:57:13.335 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:57:12 smithi164 bash[59462]: audit 1685822231.974176 mon.c (mon.2) 345 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-03T19:57:13.335 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:57:12 smithi164 bash[59462]: cluster 1685822232.353814 mgr.y (mgr.34104) 623 : cluster 0 pgmap v528: 73 pgs: 1 active+undersized+degraded, 7 active+undersized, 9 stale+active+clean, 56 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 1/72 objects degraded (1.389%) 2023-06-03T19:57:14.084 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:57:13 smithi164 bash[59462]: cluster 1685822232.94757 mon.a (mon.0) 607 : cluster 3 Health check failed: Degraded data redundancy: 1/72 objects degraded (1.389%), 1 pg degraded (PG_DEGRADED) 2023-06-03T19:57:14.085 INFO:journalctl@ceph.osd.4.smithi164.stdout:Jun 03 19:57:13 smithi164 bash[65773]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-4 2023-06-03T19:57:14.085 INFO:journalctl@ceph.osd.4.smithi164.stdout:Jun 03 19:57:13 smithi164 bash[65773]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-4 --no-mon-config --dev /dev/mapper/vg_nvme-lv_4 2023-06-03T19:57:14.085 INFO:journalctl@ceph.osd.4.smithi164.stdout:Jun 03 19:57:13 smithi164 bash[65773]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_4 2023-06-03T19:57:14.085 INFO:journalctl@ceph.osd.4.smithi164.stdout:Jun 03 19:57:13 smithi164 bash[65773]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-3 2023-06-03T19:57:14.085 INFO:journalctl@ceph.osd.4.smithi164.stdout:Jun 03 19:57:13 smithi164 bash[65773]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_4 /var/lib/ceph/osd/ceph-4/block 2023-06-03T19:57:14.085 INFO:journalctl@ceph.osd.4.smithi164.stdout:Jun 03 19:57:13 smithi164 bash[65773]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-4 2023-06-03T19:57:14.085 INFO:journalctl@ceph.osd.4.smithi164.stdout:Jun 03 19:57:13 smithi164 bash[65773]: --> ceph-volume raw activate successful for osd ID: 4 2023-06-03T19:57:14.279 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:57:13 smithi089 bash[75068]: cluster 1685822232.94757 mon.a (mon.0) 607 : cluster 3 Health check failed: Degraded data redundancy: 1/72 objects degraded (1.389%), 1 pg degraded (PG_DEGRADED) 2023-06-03T19:57:14.279 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:57:13 smithi089 bash[78452]: cluster 1685822232.94757 mon.a (mon.0) 607 : cluster 3 Health check failed: Degraded data redundancy: 1/72 objects degraded (1.389%), 1 pg degraded (PG_DEGRADED) 2023-06-03T19:57:15.028 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:57:14 smithi089 bash[75068]: cluster 1685822234.3545308 mgr.y (mgr.34104) 624 : cluster 0 pgmap v529: 73 pgs: 9 active+undersized+degraded, 24 active+undersized, 40 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 14/72 objects degraded (19.444%) 2023-06-03T19:57:15.029 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:57:14 smithi089 bash[78452]: cluster 1685822234.3545308 mgr.y (mgr.34104) 624 : cluster 0 pgmap v529: 73 pgs: 9 active+undersized+degraded, 24 active+undersized, 40 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 14/72 objects degraded (19.444%) 2023-06-03T19:57:15.252 INFO:teuthology.orchestra.run.smithi089.stdout:{ 2023-06-03T19:57:15.253 INFO:teuthology.orchestra.run.smithi089.stdout: "mon": { 2023-06-03T19:57:15.253 INFO:teuthology.orchestra.run.smithi089.stdout: "ceph version 18.0.0-3532-g8e24bcf4 (8e24bcf462a1d94c2e12aa4e26419634883a1e88) reef (rc)": 3 2023-06-03T19:57:15.253 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T19:57:15.253 INFO:teuthology.orchestra.run.smithi089.stdout: "mgr": { 2023-06-03T19:57:15.253 INFO:teuthology.orchestra.run.smithi089.stdout: "ceph version 18.0.0-3532-g8e24bcf4 (8e24bcf462a1d94c2e12aa4e26419634883a1e88) reef (rc)": 2 2023-06-03T19:57:15.254 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T19:57:15.254 INFO:teuthology.orchestra.run.smithi089.stdout: "osd": { 2023-06-03T19:57:15.254 INFO:teuthology.orchestra.run.smithi089.stdout: "ceph version 16.2.13-257-gd8c5d349 (d8c5d34975dce1c5eb0aa3a7979a4d9b9a99d1ec) pacific (stable)": 3, 2023-06-03T19:57:15.254 INFO:teuthology.orchestra.run.smithi089.stdout: "ceph version 18.0.0-3532-g8e24bcf4 (8e24bcf462a1d94c2e12aa4e26419634883a1e88) reef (rc)": 4 2023-06-03T19:57:15.254 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T19:57:15.254 INFO:teuthology.orchestra.run.smithi089.stdout: "mds": { 2023-06-03T19:57:15.254 INFO:teuthology.orchestra.run.smithi089.stdout: "ceph version 16.2.13-257-gd8c5d349 (d8c5d34975dce1c5eb0aa3a7979a4d9b9a99d1ec) pacific (stable)": 2 2023-06-03T19:57:15.254 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T19:57:15.255 INFO:teuthology.orchestra.run.smithi089.stdout: "overall": { 2023-06-03T19:57:15.255 INFO:teuthology.orchestra.run.smithi089.stdout: "ceph version 16.2.13-257-gd8c5d349 (d8c5d34975dce1c5eb0aa3a7979a4d9b9a99d1ec) pacific (stable)": 5, 2023-06-03T19:57:15.255 INFO:teuthology.orchestra.run.smithi089.stdout: "ceph version 18.0.0-3532-g8e24bcf4 (8e24bcf462a1d94c2e12aa4e26419634883a1e88) reef (rc)": 9 2023-06-03T19:57:15.255 INFO:teuthology.orchestra.run.smithi089.stdout: } 2023-06-03T19:57:15.255 INFO:teuthology.orchestra.run.smithi089.stdout:} 2023-06-03T19:57:15.335 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:57:14 smithi164 bash[59462]: cluster 1685822234.3545308 mgr.y (mgr.34104) 624 : cluster 0 pgmap v529: 73 pgs: 9 active+undersized+degraded, 24 active+undersized, 40 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 14/72 objects degraded (19.444%) 2023-06-03T19:57:15.955 INFO:journalctl@ceph.osd.4.smithi164.stdout:Jun 03 19:57:15 smithi164 bash[66063]: debug 2023-06-03T19:57:15.805+0000 7fb9ab04c700 1 -- 172.21.15.164:0/537282325 <== mon.2 v2:172.21.15.89:3301/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x55a49eb2cd00 con 0x55a49eb86c00 2023-06-03T19:57:16.081 DEBUG:teuthology.parallel:result is None 2023-06-03T19:57:16.081 INFO:teuthology.run_tasks:Running task print... 2023-06-03T19:57:16.090 INFO:teuthology.task.print:**** done upgrading first half of cluster **** 2023-06-03T19:57:16.091 INFO:teuthology.run_tasks:Running task ceph.healthy... 2023-06-03T19:57:16.106 INFO:tasks.ceph:Waiting until ceph daemons up and pgs clean... 2023-06-03T19:57:16.106 INFO:tasks.cephadm.ceph_manager.ceph:waiting for mgr available 2023-06-03T19:57:16.107 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph mgr dump --format=json 2023-06-03T19:57:16.232 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:57:15 smithi089 bash[78452]: audit 1685822235.247759 mon.c (mon.2) 346 : audit 0 from='client.? 172.21.15.89:0/1701540533' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2023-06-03T19:57:16.233 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:57:15 smithi089 bash[75068]: audit 1685822235.247759 mon.c (mon.2) 346 : audit 0 from='client.? 172.21.15.89:0/1701540533' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2023-06-03T19:57:16.334 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:57:15 smithi164 bash[59462]: audit 1685822235.247759 mon.c (mon.2) 346 : audit 0 from='client.? 172.21.15.89:0/1701540533' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2023-06-03T19:57:16.529 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 19:57:16 smithi089 bash[71163]: ::ffff:172.21.15.164 - - [03/Jun/2023:19:57:16] "GET /metrics HTTP/1.1" 200 220345 "" "Prometheus/2.33.4" 2023-06-03T19:57:16.834 INFO:journalctl@ceph.osd.4.smithi164.stdout:Jun 03 19:57:16 smithi164 bash[66063]: debug 2023-06-03T19:57:16.369+0000 7fb9b41d2640 -1 Falling back to public interface 2023-06-03T19:57:17.279 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:57:16 smithi089 bash[75068]: cluster 1685822236.3553686 mgr.y (mgr.34104) 625 : cluster 0 pgmap v530: 73 pgs: 9 active+undersized+degraded, 24 active+undersized, 40 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 14/72 objects degraded (19.444%) 2023-06-03T19:57:17.279 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:57:16 smithi089 bash[78452]: cluster 1685822236.3553686 mgr.y (mgr.34104) 625 : cluster 0 pgmap v530: 73 pgs: 9 active+undersized+degraded, 24 active+undersized, 40 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 14/72 objects degraded (19.444%) 2023-06-03T19:57:17.334 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:57:16 smithi164 bash[59462]: cluster 1685822236.3553686 mgr.y (mgr.34104) 625 : cluster 0 pgmap v530: 73 pgs: 9 active+undersized+degraded, 24 active+undersized, 40 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 14/72 objects degraded (19.444%) 2023-06-03T19:57:19.779 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:57:19 smithi089 bash[75068]: cluster 1685822238.3563516 mgr.y (mgr.34104) 626 : cluster 0 pgmap v531: 73 pgs: 9 active+undersized+degraded, 24 active+undersized, 40 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 14/72 objects degraded (19.444%) 2023-06-03T19:57:19.779 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:57:19 smithi089 bash[78452]: cluster 1685822238.3563516 mgr.y (mgr.34104) 626 : cluster 0 pgmap v531: 73 pgs: 9 active+undersized+degraded, 24 active+undersized, 40 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 14/72 objects degraded (19.444%) 2023-06-03T19:57:19.834 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:57:19 smithi164 bash[59462]: cluster 1685822238.3563516 mgr.y (mgr.34104) 626 : cluster 0 pgmap v531: 73 pgs: 9 active+undersized+degraded, 24 active+undersized, 40 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 14/72 objects degraded (19.444%) 2023-06-03T19:57:20.834 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:57:20 smithi164 bash[59462]: audit 1685822239.5729628 mon.a (mon.0) 608 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:57:20.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:57:20 smithi164 bash[59462]: audit 1685822239.5827496 mon.a (mon.0) 609 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:57:20.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:57:20 smithi164 bash[59462]: audit 1685822240.3859284 mon.a (mon.0) 610 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:57:20.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:57:20 smithi164 bash[59462]: audit 1685822240.397763 mon.a (mon.0) 611 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:57:20.922 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/mon.a/config 2023-06-03T19:57:20.965 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:57:20 smithi089 bash[75068]: audit 1685822239.5729628 mon.a (mon.0) 608 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:57:20.965 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:57:20 smithi089 bash[75068]: audit 1685822239.5827496 mon.a (mon.0) 609 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:57:20.966 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:57:20 smithi089 bash[75068]: audit 1685822240.3859284 mon.a (mon.0) 610 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:57:20.966 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:57:20 smithi089 bash[75068]: audit 1685822240.397763 mon.a (mon.0) 611 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:57:20.966 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:57:20 smithi089 bash[78452]: audit 1685822239.5729628 mon.a (mon.0) 608 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:57:20.966 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:57:20 smithi089 bash[78452]: audit 1685822239.5827496 mon.a (mon.0) 609 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:57:20.966 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:57:20 smithi089 bash[78452]: audit 1685822240.3859284 mon.a (mon.0) 610 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:57:20.966 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:57:20 smithi089 bash[78452]: audit 1685822240.397763 mon.a (mon.0) 611 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:57:22.029 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:57:21 smithi089 bash[75068]: cluster 1685822240.3573277 mgr.y (mgr.34104) 627 : cluster 0 pgmap v532: 73 pgs: 9 active+undersized+degraded, 24 active+undersized, 40 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 14/72 objects degraded (19.444%) 2023-06-03T19:57:22.029 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:57:21 smithi089 bash[78452]: cluster 1685822240.3573277 mgr.y (mgr.34104) 627 : cluster 0 pgmap v532: 73 pgs: 9 active+undersized+degraded, 24 active+undersized, 40 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 14/72 objects degraded (19.444%) 2023-06-03T19:57:22.084 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:57:21 smithi164 bash[59462]: cluster 1685822240.3573277 mgr.y (mgr.34104) 627 : cluster 0 pgmap v532: 73 pgs: 9 active+undersized+degraded, 24 active+undersized, 40 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 14/72 objects degraded (19.444%) 2023-06-03T19:57:23.423 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:57:23 smithi089 bash[78452]: audit 1685822242.0960848 mon.a (mon.0) 612 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:57:23.423 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:57:23 smithi089 bash[78452]: audit 1685822242.0982835 mon.c (mon.2) 347 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T19:57:23.424 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:57:23 smithi089 bash[78452]: cluster 1685822242.358289 mgr.y (mgr.34104) 628 : cluster 0 pgmap v533: 73 pgs: 9 active+undersized+degraded, 24 active+undersized, 40 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 14/72 objects degraded (19.444%) 2023-06-03T19:57:23.424 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:57:23 smithi089 bash[78452]: audit 1685822242.4456348 mon.a (mon.0) 613 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:57:23.424 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:57:23 smithi089 bash[78452]: cluster 1685822242.4568312 mon.a (mon.0) 614 : cluster 3 Health check update: Degraded data redundancy: 14/72 objects degraded (19.444%), 9 pgs degraded (PG_DEGRADED) 2023-06-03T19:57:23.424 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:57:23 smithi089 bash[75068]: audit 1685822242.0960848 mon.a (mon.0) 612 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:57:23.424 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:57:23 smithi089 bash[75068]: audit 1685822242.0982835 mon.c (mon.2) 347 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T19:57:23.425 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:57:23 smithi089 bash[75068]: cluster 1685822242.358289 mgr.y (mgr.34104) 628 : cluster 0 pgmap v533: 73 pgs: 9 active+undersized+degraded, 24 active+undersized, 40 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 14/72 objects degraded (19.444%) 2023-06-03T19:57:23.425 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:57:23 smithi089 bash[75068]: audit 1685822242.4456348 mon.a (mon.0) 613 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:57:23.425 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:57:23 smithi089 bash[75068]: cluster 1685822242.4568312 mon.a (mon.0) 614 : cluster 3 Health check update: Degraded data redundancy: 14/72 objects degraded (19.444%), 9 pgs degraded (PG_DEGRADED) 2023-06-03T19:57:23.584 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:57:23 smithi164 bash[59462]: audit 1685822242.0960848 mon.a (mon.0) 612 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:57:23.584 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:57:23 smithi164 bash[59462]: audit 1685822242.0982835 mon.c (mon.2) 347 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T19:57:23.585 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:57:23 smithi164 bash[59462]: cluster 1685822242.358289 mgr.y (mgr.34104) 628 : cluster 0 pgmap v533: 73 pgs: 9 active+undersized+degraded, 24 active+undersized, 40 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 14/72 objects degraded (19.444%) 2023-06-03T19:57:23.585 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:57:23 smithi164 bash[59462]: audit 1685822242.4456348 mon.a (mon.0) 613 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:57:23.585 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:57:23 smithi164 bash[59462]: cluster 1685822242.4568312 mon.a (mon.0) 614 : cluster 3 Health check update: Degraded data redundancy: 14/72 objects degraded (19.444%), 9 pgs degraded (PG_DEGRADED) 2023-06-03T19:57:25.162 INFO:teuthology.orchestra.run.smithi089.stdout: 2023-06-03T19:57:25.529 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:57:25 smithi089 bash[78452]: cluster 1685822244.3594444 mgr.y (mgr.34104) 629 : cluster 0 pgmap v534: 73 pgs: 9 active+undersized+degraded, 24 active+undersized, 40 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 14/72 objects degraded (19.444%) 2023-06-03T19:57:25.529 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:57:25 smithi089 bash[78452]: audit 1685822245.1486142 mon.c (mon.2) 348 : audit 0 from='client.? 172.21.15.89:0/728369854' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2023-06-03T19:57:25.529 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:57:25 smithi089 bash[75068]: cluster 1685822244.3594444 mgr.y (mgr.34104) 629 : cluster 0 pgmap v534: 73 pgs: 9 active+undersized+degraded, 24 active+undersized, 40 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 14/72 objects degraded (19.444%) 2023-06-03T19:57:25.530 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:57:25 smithi089 bash[75068]: audit 1685822245.1486142 mon.c (mon.2) 348 : audit 0 from='client.? 172.21.15.89:0/728369854' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2023-06-03T19:57:25.584 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:57:25 smithi164 bash[59462]: cluster 1685822244.3594444 mgr.y (mgr.34104) 629 : cluster 0 pgmap v534: 73 pgs: 9 active+undersized+degraded, 24 active+undersized, 40 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 14/72 objects degraded (19.444%) 2023-06-03T19:57:25.585 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:57:25 smithi164 bash[59462]: audit 1685822245.1486142 mon.c (mon.2) 348 : audit 0 from='client.? 172.21.15.89:0/728369854' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2023-06-03T19:57:25.905 INFO:teuthology.orchestra.run.smithi089.stdout:{"epoch":38,"active_gid":34104,"active_name":"y","active_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:6800","nonce":105304515},{"type":"v1","addr":"172.21.15.89:6801","nonce":105304515}]},"active_addr":"172.21.15.89:6801/105304515","active_change":"2023-06-03T19:41:21.936275+0000","active_mgr_features":4540138322906710015,"available":true,"standbys":[{"gid":25042,"name":"x","mgr_features":4540138322906710015,"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2359","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","upmap"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"alertmanager_web_password":{"name":"alertmanager_web_password","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"Alertmanager web password","long_desc":"","tags":[],"see_also":[]},"alertmanager_web_user":{"name":"alertmanager_web_user","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"Alertmanager web user","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"cgroups_split":{"name":"cgroups_split","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Pass --cgroups=split when cephadm creates containers (currently podman only)","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.25.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_elasticsearch":{"name":"container_image_elasticsearch","type":"str","level":"advanced","flags":0,"default_value":"quay.io/omrizeneva/elasticsearch:6.8.23","min":"","max":"","enum_allowed":[],"desc":"elasticsearch container image","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/ceph-grafana:9.4.7","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_agent":{"name":"container_image_jaeger_agent","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-agent:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger agent container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_collector":{"name":"container_image_jaeger_collector","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-collector:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger collector container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_query":{"name":"container_image_jaeger_query","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-query:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger query container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/keepalived:2.2.4","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_loki":{"name":"container_image_loki","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/loki:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Loki container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.5.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.43.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_promtail":{"name":"container_image_promtail","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/promtail:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Promtail container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_cephadm_command_timeout":{"name":"default_cephadm_command_timeout","type":"secs","level":"advanced","flags":0,"default_value":"900","min":"","max":"","enum_allowed":[],"desc":"Default timeout applied to cephadm commands run directly on the host (in seconds)","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_refresh_metadata":{"name":"log_refresh_metadata","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Log all refresh metadata. Includes daemon, device, and host info collected regularly. Only has effect if logging at debug level","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"prometheus_web_password":{"name":"prometheus_web_password","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"Prometheus web password","long_desc":"","tags":[],"see_also":[]},"prometheus_web_user":{"name":"prometheus_web_user","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"Prometheus web user","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"secure_monitoring_stack":{"name":"secure_monitoring_stack","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable TLS security for all the monitoring stack daemons","long_desc":"","tags":[],"see_also":[]},"service_discovery_port":{"name":"service_discovery_port","type":"int","level":"advanced","flags":0,"default_value":"8765","min":"","max":"","enum_allowed":[],"desc":"cephadm service discovery port","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_DASHBOARD":{"name":"FEATURE_TOGGLE_DASHBOARD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"cross_origin_url":{"name":"cross_origin_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"redirect_resolve_ip_addr":{"name":"redirect_resolve_ip_addr","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"fail_fs":{"name":"fail_fs","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Fail filesystem for rapid multi-rank mds upgrade","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"noautoscale":{"name":"noautoscale","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"global autoscale flag","long_desc":"Option to turn on/off the autoscaler for all pools","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":1,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rgw","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"drive_group_interval":{"name":"drive_group_interval","type":"float","level":"advanced","flags":0,"default_value":"300.0","min":"","max":"","enum_allowed":[],"desc":"interval in seconds between re-application of applied drive_groups","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard_description":{"name":"leaderboard_description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}]}],"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2359","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","upmap"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"alertmanager_web_password":{"name":"alertmanager_web_password","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"Alertmanager web password","long_desc":"","tags":[],"see_also":[]},"alertmanager_web_user":{"name":"alertmanager_web_user","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"Alertmanager web user","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"cgroups_split":{"name":"cgroups_split","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Pass --cgroups=split when cephadm creates containers (currently podman only)","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.25.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_elasticsearch":{"name":"container_image_elasticsearch","type":"str","level":"advanced","flags":0,"default_value":"quay.io/omrizeneva/elasticsearch:6.8.23","min":"","max":"","enum_allowed":[],"desc":"elasticsearch container image","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/ceph-grafana:9.4.7","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_agent":{"name":"container_image_jaeger_agent","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-agent:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger agent container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_collector":{"name":"container_image_jaeger_collector","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-collector:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger collector container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_query":{"name":"container_image_jaeger_query","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-query:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger query container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/keepalived:2.2.4","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_loki":{"name":"container_image_loki","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/loki:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Loki container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.5.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.43.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_promtail":{"name":"container_image_promtail","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/promtail:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Promtail container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_cephadm_command_timeout":{"name":"default_cephadm_command_timeout","type":"secs","level":"advanced","flags":0,"default_value":"900","min":"","max":"","enum_allowed":[],"desc":"Default timeout applied to cephadm commands run directly on the host (in seconds)","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_refresh_metadata":{"name":"log_refresh_metadata","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Log all refresh metadata. Includes daemon, device, and host info collected regularly. Only has effect if logging at debug level","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"prometheus_web_password":{"name":"prometheus_web_password","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"Prometheus web password","long_desc":"","tags":[],"see_also":[]},"prometheus_web_user":{"name":"prometheus_web_user","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"Prometheus web user","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"secure_monitoring_stack":{"name":"secure_monitoring_stack","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable TLS security for all the monitoring stack daemons","long_desc":"","tags":[],"see_also":[]},"service_discovery_port":{"name":"service_discovery_port","type":"int","level":"advanced","flags":0,"default_value":"8765","min":"","max":"","enum_allowed":[],"desc":"cephadm service discovery port","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_DASHBOARD":{"name":"FEATURE_TOGGLE_DASHBOARD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"cross_origin_url":{"name":"cross_origin_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"redirect_resolve_ip_addr":{"name":"redirect_resolve_ip_addr","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"fail_fs":{"name":"fail_fs","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Fail filesystem for rapid multi-rank mds upgrade","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"noautoscale":{"name":"noautoscale","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"global autoscale flag","long_desc":"Option to turn on/off the autoscaler for all pools","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":1,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rgw","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"drive_group_interval":{"name":"drive_group_interval","type":"float","level":"advanced","flags":0,"default_value":"300.0","min":"","max":"","enum_allowed":[],"desc":"interval in seconds between re-application of applied drive_groups","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard_description":{"name":"leaderboard_description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}],"services":{"dashboard":"https://172.21.15.89:8443/","prometheus":"http://172.21.15.89:9283/"},"always_on_modules":{"octopus":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"pacific":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"quincy":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"reef":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"]},"last_failure_osd_epoch":72,"active_clients":[{"name":"devicehealth","addrvec":[{"type":"v2","addr":"172.21.15.89:0","nonce":3935629605}]},{"name":"libcephsqlite","addrvec":[{"type":"v2","addr":"172.21.15.89:0","nonce":1507780492}]},{"name":"rbd_support","addrvec":[{"type":"v2","addr":"172.21.15.89:0","nonce":637907048}]},{"name":"volumes","addrvec":[{"type":"v2","addr":"172.21.15.89:0","nonce":3658418426}]}]} 2023-06-03T19:57:25.910 INFO:tasks.cephadm.ceph_manager.ceph:mgr available! 2023-06-03T19:57:25.910 INFO:tasks.cephadm.ceph_manager.ceph:waiting for all up 2023-06-03T19:57:25.911 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph osd dump --format=json 2023-06-03T19:57:26.529 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 19:57:26 smithi089 bash[71163]: ::ffff:172.21.15.164 - - [03/Jun/2023:19:57:26] "GET /metrics HTTP/1.1" 200 220351 "" "Prometheus/2.33.4" 2023-06-03T19:57:26.529 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:57:26 smithi089 bash[78452]: audit 1685822246.226973 mon.b (mon.1) 44 : audit 1 from='osd.4 [v2:172.21.15.164:6800/3822759075,v1:172.21.15.164:6801/3822759075]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2023-06-03T19:57:26.529 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:57:26 smithi089 bash[78452]: audit 1685822246.2301204 mon.a (mon.0) 615 : audit 1 from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2023-06-03T19:57:26.530 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:57:26 smithi089 bash[75068]: audit 1685822246.226973 mon.b (mon.1) 44 : audit 1 from='osd.4 [v2:172.21.15.164:6800/3822759075,v1:172.21.15.164:6801/3822759075]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2023-06-03T19:57:26.530 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:57:26 smithi089 bash[75068]: audit 1685822246.2301204 mon.a (mon.0) 615 : audit 1 from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2023-06-03T19:57:26.533 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:57:26 smithi164 bash[59462]: audit 1685822246.226973 mon.b (mon.1) 44 : audit 1 from='osd.4 [v2:172.21.15.164:6800/3822759075,v1:172.21.15.164:6801/3822759075]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2023-06-03T19:57:26.533 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:57:26 smithi164 bash[59462]: audit 1685822246.2301204 mon.a (mon.0) 615 : audit 1 from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2023-06-03T19:57:26.533 INFO:journalctl@ceph.osd.4.smithi164.stdout:Jun 03 19:57:26 smithi164 bash[66063]: debug 2023-06-03T19:57:26.221+0000 7fb9b41d2640 -1 osd.4 120 log_to_monitors true 2023-06-03T19:57:26.534 INFO:journalctl@ceph.osd.4.smithi164.stdout:Jun 03 19:57:26 smithi164 bash[66063]: debug 2023-06-03T19:57:26.469+0000 7fb9a4e3d700 -1 osd.4 120 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2023-06-03T19:57:27.779 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:57:27 smithi089 bash[75068]: cluster 1685822246.3603156 mgr.y (mgr.34104) 630 : cluster 0 pgmap v535: 73 pgs: 9 active+undersized+degraded, 24 active+undersized, 40 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 14/72 objects degraded (19.444%) 2023-06-03T19:57:27.779 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:57:27 smithi089 bash[75068]: cluster 1685822246.4236212 osd.6 (osd.6) 38 : cluster 3 failed to encode map e123 with expected crc 2023-06-03T19:57:27.779 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:57:27 smithi089 bash[75068]: audit 1685822246.4274907 mon.b (mon.1) 45 : audit 1 from='osd.4 [v2:172.21.15.164:6800/3822759075,v1:172.21.15.164:6801/3822759075]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi164", "root=default"]}]: dispatch 2023-06-03T19:57:27.779 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:57:27 smithi089 bash[75068]: audit 1685822246.4286346 mon.a (mon.0) 616 : audit 1 from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2023-06-03T19:57:27.780 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:57:27 smithi089 bash[75068]: cluster 1685822246.4287658 mon.a (mon.0) 617 : cluster 0 osdmap e123: 8 total, 7 up, 8 in 2023-06-03T19:57:27.780 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:57:27 smithi089 bash[75068]: audit 1685822246.4306962 mon.a (mon.0) 618 : audit 1 from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi164", "root=default"]}]: dispatch 2023-06-03T19:57:27.780 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:57:27 smithi089 bash[78452]: cluster 1685822246.3603156 mgr.y (mgr.34104) 630 : cluster 0 pgmap v535: 73 pgs: 9 active+undersized+degraded, 24 active+undersized, 40 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 14/72 objects degraded (19.444%) 2023-06-03T19:57:27.780 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:57:27 smithi089 bash[78452]: cluster 1685822246.4236212 osd.6 (osd.6) 38 : cluster 3 failed to encode map e123 with expected crc 2023-06-03T19:57:27.780 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:57:27 smithi089 bash[78452]: audit 1685822246.4274907 mon.b (mon.1) 45 : audit 1 from='osd.4 [v2:172.21.15.164:6800/3822759075,v1:172.21.15.164:6801/3822759075]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi164", "root=default"]}]: dispatch 2023-06-03T19:57:27.780 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:57:27 smithi089 bash[78452]: audit 1685822246.4286346 mon.a (mon.0) 616 : audit 1 from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2023-06-03T19:57:27.781 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:57:27 smithi089 bash[78452]: cluster 1685822246.4287658 mon.a (mon.0) 617 : cluster 0 osdmap e123: 8 total, 7 up, 8 in 2023-06-03T19:57:27.781 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:57:27 smithi089 bash[78452]: audit 1685822246.4306962 mon.a (mon.0) 618 : audit 1 from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi164", "root=default"]}]: dispatch 2023-06-03T19:57:27.834 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:57:27 smithi164 bash[59462]: cluster 1685822246.3603156 mgr.y (mgr.34104) 630 : cluster 0 pgmap v535: 73 pgs: 9 active+undersized+degraded, 24 active+undersized, 40 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 14/72 objects degraded (19.444%) 2023-06-03T19:57:27.834 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:57:27 smithi164 bash[59462]: cluster 1685822246.4236212 osd.6 (osd.6) 38 : cluster 3 failed to encode map e123 with expected crc 2023-06-03T19:57:27.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:57:27 smithi164 bash[59462]: audit 1685822246.4274907 mon.b (mon.1) 45 : audit 1 from='osd.4 [v2:172.21.15.164:6800/3822759075,v1:172.21.15.164:6801/3822759075]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi164", "root=default"]}]: dispatch 2023-06-03T19:57:27.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:57:27 smithi164 bash[59462]: audit 1685822246.4286346 mon.a (mon.0) 616 : audit 1 from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2023-06-03T19:57:27.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:57:27 smithi164 bash[59462]: cluster 1685822246.4287658 mon.a (mon.0) 617 : cluster 0 osdmap e123: 8 total, 7 up, 8 in 2023-06-03T19:57:27.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:57:27 smithi164 bash[59462]: audit 1685822246.4306962 mon.a (mon.0) 618 : audit 1 from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi164", "root=default"]}]: dispatch 2023-06-03T19:57:28.726 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:57:28 smithi164 bash[59462]: cluster 1685822247.4258053 mon.a (mon.0) 619 : cluster 1 Health check cleared: OSD_DOWN (was: 1 osds down) 2023-06-03T19:57:28.727 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:57:28 smithi164 bash[59462]: cluster 1685822247.4283845 osd.6 (osd.6) 39 : cluster 3 failed to encode map e124 with expected crc 2023-06-03T19:57:28.727 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:57:28 smithi164 bash[59462]: cluster 1685822247.4346378 mon.a (mon.0) 620 : cluster 1 osd.4 [v2:172.21.15.164:6800/3822759075,v1:172.21.15.164:6801/3822759075] boot 2023-06-03T19:57:28.727 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:57:28 smithi164 bash[59462]: cluster 1685822247.434745 mon.a (mon.0) 621 : cluster 0 osdmap e124: 8 total, 8 up, 8 in 2023-06-03T19:57:28.727 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:57:28 smithi164 bash[59462]: cluster 1685822247.4351778 osd.7 (osd.7) 36 : cluster 3 failed to encode map e124 with expected crc 2023-06-03T19:57:28.727 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:57:28 smithi164 bash[59462]: audit 1685822247.4371576 mon.c (mon.2) 349 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-06-03T19:57:28.779 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:57:28 smithi089 bash[78452]: cluster 1685822247.4258053 mon.a (mon.0) 619 : cluster 1 Health check cleared: OSD_DOWN (was: 1 osds down) 2023-06-03T19:57:28.779 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:57:28 smithi089 bash[78452]: cluster 1685822247.4283845 osd.6 (osd.6) 39 : cluster 3 failed to encode map e124 with expected crc 2023-06-03T19:57:28.779 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:57:28 smithi089 bash[78452]: cluster 1685822247.4346378 mon.a (mon.0) 620 : cluster 1 osd.4 [v2:172.21.15.164:6800/3822759075,v1:172.21.15.164:6801/3822759075] boot 2023-06-03T19:57:28.779 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:57:28 smithi089 bash[78452]: cluster 1685822247.434745 mon.a (mon.0) 621 : cluster 0 osdmap e124: 8 total, 8 up, 8 in 2023-06-03T19:57:28.779 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:57:28 smithi089 bash[78452]: cluster 1685822247.4351778 osd.7 (osd.7) 36 : cluster 3 failed to encode map e124 with expected crc 2023-06-03T19:57:28.780 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:57:28 smithi089 bash[78452]: audit 1685822247.4371576 mon.c (mon.2) 349 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-06-03T19:57:28.780 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:57:28 smithi089 bash[75068]: cluster 1685822247.4258053 mon.a (mon.0) 619 : cluster 1 Health check cleared: OSD_DOWN (was: 1 osds down) 2023-06-03T19:57:28.780 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:57:28 smithi089 bash[75068]: cluster 1685822247.4283845 osd.6 (osd.6) 39 : cluster 3 failed to encode map e124 with expected crc 2023-06-03T19:57:28.780 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:57:28 smithi089 bash[75068]: cluster 1685822247.4346378 mon.a (mon.0) 620 : cluster 1 osd.4 [v2:172.21.15.164:6800/3822759075,v1:172.21.15.164:6801/3822759075] boot 2023-06-03T19:57:28.780 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:57:28 smithi089 bash[75068]: cluster 1685822247.434745 mon.a (mon.0) 621 : cluster 0 osdmap e124: 8 total, 8 up, 8 in 2023-06-03T19:57:28.780 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:57:28 smithi089 bash[75068]: cluster 1685822247.4351778 osd.7 (osd.7) 36 : cluster 3 failed to encode map e124 with expected crc 2023-06-03T19:57:28.780 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:57:28 smithi089 bash[75068]: audit 1685822247.4371576 mon.c (mon.2) 349 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-06-03T19:57:29.779 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:57:29 smithi089 bash[75068]: cluster 1685822248.3613927 mgr.y (mgr.34104) 631 : cluster 0 pgmap v538: 73 pgs: 16 peering, 2 active+undersized+degraded, 15 active+undersized, 40 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 5/72 objects degraded (6.944%) 2023-06-03T19:57:29.779 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:57:29 smithi089 bash[75068]: cluster 1685822248.4404597 osd.6 (osd.6) 40 : cluster 3 failed to encode map e125 with expected crc 2023-06-03T19:57:29.779 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:57:29 smithi089 bash[75068]: cluster 1685822248.4449213 osd.5 (osd.5) 47 : cluster 3 failed to encode map e125 with expected crc 2023-06-03T19:57:29.779 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:57:29 smithi089 bash[75068]: cluster 1685822248.4450183 osd.7 (osd.7) 37 : cluster 3 failed to encode map e125 with expected crc 2023-06-03T19:57:29.780 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:57:29 smithi089 bash[75068]: cluster 1685822248.4452114 osd.6 (osd.6) 41 : cluster 3 failed to encode map e125 with expected crc 2023-06-03T19:57:29.780 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:57:29 smithi089 bash[75068]: cluster 1685822248.4470158 osd.6 (osd.6) 42 : cluster 3 failed to encode map e125 with expected crc 2023-06-03T19:57:29.780 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:57:29 smithi089 bash[75068]: cluster 1685822248.4471407 mon.a (mon.0) 622 : cluster 0 osdmap e125: 8 total, 8 up, 8 in 2023-06-03T19:57:29.780 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:57:29 smithi089 bash[75068]: cluster 1685822248.447962 osd.5 (osd.5) 48 : cluster 3 failed to encode map e125 with expected crc 2023-06-03T19:57:29.780 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:57:29 smithi089 bash[75068]: cluster 1685822248.4483685 osd.7 (osd.7) 38 : cluster 3 failed to encode map e125 with expected crc 2023-06-03T19:57:29.780 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:57:29 smithi089 bash[75068]: cluster 1685822248.4498909 osd.7 (osd.7) 39 : cluster 3 failed to encode map e125 with expected crc 2023-06-03T19:57:29.780 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:57:29 smithi089 bash[75068]: cluster 1685822248.4515996 osd.7 (osd.7) 40 : cluster 3 failed to encode map e125 with expected crc 2023-06-03T19:57:29.781 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:57:29 smithi089 bash[75068]: cluster 1685822248.459414 mon.a (mon.0) 623 : cluster 3 Health check update: Degraded data redundancy: 5/72 objects degraded (6.944%), 2 pgs degraded (PG_DEGRADED) 2023-06-03T19:57:29.781 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:57:29 smithi089 bash[78452]: cluster 1685822248.3613927 mgr.y (mgr.34104) 631 : cluster 0 pgmap v538: 73 pgs: 16 peering, 2 active+undersized+degraded, 15 active+undersized, 40 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 5/72 objects degraded (6.944%) 2023-06-03T19:57:29.781 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:57:29 smithi089 bash[78452]: cluster 1685822248.4404597 osd.6 (osd.6) 40 : cluster 3 failed to encode map e125 with expected crc 2023-06-03T19:57:29.781 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:57:29 smithi089 bash[78452]: cluster 1685822248.4449213 osd.5 (osd.5) 47 : cluster 3 failed to encode map e125 with expected crc 2023-06-03T19:57:29.781 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:57:29 smithi089 bash[78452]: cluster 1685822248.4450183 osd.7 (osd.7) 37 : cluster 3 failed to encode map e125 with expected crc 2023-06-03T19:57:29.781 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:57:29 smithi089 bash[78452]: cluster 1685822248.4452114 osd.6 (osd.6) 41 : cluster 3 failed to encode map e125 with expected crc 2023-06-03T19:57:29.781 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:57:29 smithi089 bash[78452]: cluster 1685822248.4470158 osd.6 (osd.6) 42 : cluster 3 failed to encode map e125 with expected crc 2023-06-03T19:57:29.782 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:57:29 smithi089 bash[78452]: cluster 1685822248.4471407 mon.a (mon.0) 622 : cluster 0 osdmap e125: 8 total, 8 up, 8 in 2023-06-03T19:57:29.782 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:57:29 smithi089 bash[78452]: cluster 1685822248.447962 osd.5 (osd.5) 48 : cluster 3 failed to encode map e125 with expected crc 2023-06-03T19:57:29.782 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:57:29 smithi089 bash[78452]: cluster 1685822248.4483685 osd.7 (osd.7) 38 : cluster 3 failed to encode map e125 with expected crc 2023-06-03T19:57:29.782 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:57:29 smithi089 bash[78452]: cluster 1685822248.4498909 osd.7 (osd.7) 39 : cluster 3 failed to encode map e125 with expected crc 2023-06-03T19:57:29.782 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:57:29 smithi089 bash[78452]: cluster 1685822248.4515996 osd.7 (osd.7) 40 : cluster 3 failed to encode map e125 with expected crc 2023-06-03T19:57:29.782 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:57:29 smithi089 bash[78452]: cluster 1685822248.459414 mon.a (mon.0) 623 : cluster 3 Health check update: Degraded data redundancy: 5/72 objects degraded (6.944%), 2 pgs degraded (PG_DEGRADED) 2023-06-03T19:57:29.823 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:57:29 smithi164 bash[59462]: cluster 1685822248.3613927 mgr.y (mgr.34104) 631 : cluster 0 pgmap v538: 73 pgs: 16 peering, 2 active+undersized+degraded, 15 active+undersized, 40 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 5/72 objects degraded (6.944%) 2023-06-03T19:57:29.823 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:57:29 smithi164 bash[59462]: cluster 1685822248.4404597 osd.6 (osd.6) 40 : cluster 3 failed to encode map e125 with expected crc 2023-06-03T19:57:29.823 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:57:29 smithi164 bash[59462]: cluster 1685822248.4449213 osd.5 (osd.5) 47 : cluster 3 failed to encode map e125 with expected crc 2023-06-03T19:57:29.824 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:57:29 smithi164 bash[59462]: cluster 1685822248.4450183 osd.7 (osd.7) 37 : cluster 3 failed to encode map e125 with expected crc 2023-06-03T19:57:29.824 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:57:29 smithi164 bash[59462]: cluster 1685822248.4452114 osd.6 (osd.6) 41 : cluster 3 failed to encode map e125 with expected crc 2023-06-03T19:57:29.824 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:57:29 smithi164 bash[59462]: cluster 1685822248.4470158 osd.6 (osd.6) 42 : cluster 3 failed to encode map e125 with expected crc 2023-06-03T19:57:29.824 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:57:29 smithi164 bash[59462]: cluster 1685822248.4471407 mon.a (mon.0) 622 : cluster 0 osdmap e125: 8 total, 8 up, 8 in 2023-06-03T19:57:29.824 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:57:29 smithi164 bash[59462]: cluster 1685822248.447962 osd.5 (osd.5) 48 : cluster 3 failed to encode map e125 with expected crc 2023-06-03T19:57:29.824 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:57:29 smithi164 bash[59462]: cluster 1685822248.4483685 osd.7 (osd.7) 38 : cluster 3 failed to encode map e125 with expected crc 2023-06-03T19:57:29.824 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:57:29 smithi164 bash[59462]: cluster 1685822248.4498909 osd.7 (osd.7) 39 : cluster 3 failed to encode map e125 with expected crc 2023-06-03T19:57:29.825 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:57:29 smithi164 bash[59462]: cluster 1685822248.4515996 osd.7 (osd.7) 40 : cluster 3 failed to encode map e125 with expected crc 2023-06-03T19:57:29.825 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:57:29 smithi164 bash[59462]: cluster 1685822248.459414 mon.a (mon.0) 623 : cluster 3 Health check update: Degraded data redundancy: 5/72 objects degraded (6.944%), 2 pgs degraded (PG_DEGRADED) 2023-06-03T19:57:30.729 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/mon.a/config 2023-06-03T19:57:31.779 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:57:31 smithi089 bash[75068]: cluster 1685822250.3622622 mgr.y (mgr.34104) 632 : cluster 0 pgmap v540: 73 pgs: 16 peering, 2 active+undersized+degraded, 15 active+undersized, 40 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 5/72 objects degraded (6.944%) 2023-06-03T19:57:31.779 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:57:31 smithi089 bash[75068]: audit 1685822250.4541147 mon.a (mon.0) 624 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:57:31.779 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:57:31 smithi089 bash[75068]: audit 1685822250.4664097 mon.a (mon.0) 625 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:57:31.779 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:57:31 smithi089 bash[75068]: audit 1685822250.468567 mon.c (mon.2) 350 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-03T19:57:31.780 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:57:31 smithi089 bash[75068]: audit 1685822250.4699132 mon.c (mon.2) 351 : audit 1 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-03T19:57:31.780 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:57:31 smithi089 bash[75068]: audit 1685822250.4780576 mon.a (mon.0) 626 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:57:31.780 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:57:31 smithi089 bash[75068]: audit 1685822250.5596414 mon.c (mon.2) 352 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-03T19:57:31.780 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:57:31 smithi089 bash[75068]: audit 1685822250.5627906 mon.c (mon.2) 353 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-03T19:57:31.780 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:57:31 smithi089 bash[75068]: audit 1685822250.56418 mon.c (mon.2) 354 : audit 1 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-03T19:57:31.780 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:57:31 smithi089 bash[75068]: audit 1685822250.5731342 mon.a (mon.0) 627 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:57:31.781 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:57:31 smithi089 bash[78452]: cluster 1685822250.3622622 mgr.y (mgr.34104) 632 : cluster 0 pgmap v540: 73 pgs: 16 peering, 2 active+undersized+degraded, 15 active+undersized, 40 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 5/72 objects degraded (6.944%) 2023-06-03T19:57:31.781 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:57:31 smithi089 bash[78452]: audit 1685822250.4541147 mon.a (mon.0) 624 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:57:31.781 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:57:31 smithi089 bash[78452]: audit 1685822250.4664097 mon.a (mon.0) 625 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:57:31.781 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:57:31 smithi089 bash[78452]: audit 1685822250.468567 mon.c (mon.2) 350 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-03T19:57:31.781 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:57:31 smithi089 bash[78452]: audit 1685822250.4699132 mon.c (mon.2) 351 : audit 1 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-03T19:57:31.781 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:57:31 smithi089 bash[78452]: audit 1685822250.4780576 mon.a (mon.0) 626 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:57:31.782 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:57:31 smithi089 bash[78452]: audit 1685822250.5596414 mon.c (mon.2) 352 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-03T19:57:31.782 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:57:31 smithi089 bash[78452]: audit 1685822250.5627906 mon.c (mon.2) 353 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-03T19:57:31.782 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:57:31 smithi089 bash[78452]: audit 1685822250.56418 mon.c (mon.2) 354 : audit 1 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-03T19:57:31.782 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:57:31 smithi089 bash[78452]: audit 1685822250.5731342 mon.a (mon.0) 627 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:57:31.834 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:57:31 smithi164 bash[59462]: cluster 1685822250.3622622 mgr.y (mgr.34104) 632 : cluster 0 pgmap v540: 73 pgs: 16 peering, 2 active+undersized+degraded, 15 active+undersized, 40 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 5/72 objects degraded (6.944%) 2023-06-03T19:57:31.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:57:31 smithi164 bash[59462]: audit 1685822250.4541147 mon.a (mon.0) 624 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:57:31.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:57:31 smithi164 bash[59462]: audit 1685822250.4664097 mon.a (mon.0) 625 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:57:31.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:57:31 smithi164 bash[59462]: audit 1685822250.468567 mon.c (mon.2) 350 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-03T19:57:31.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:57:31 smithi164 bash[59462]: audit 1685822250.4699132 mon.c (mon.2) 351 : audit 1 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-03T19:57:31.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:57:31 smithi164 bash[59462]: audit 1685822250.4780576 mon.a (mon.0) 626 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:57:31.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:57:31 smithi164 bash[59462]: audit 1685822250.5596414 mon.c (mon.2) 352 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-03T19:57:31.836 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:57:31 smithi164 bash[59462]: audit 1685822250.5627906 mon.c (mon.2) 353 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-03T19:57:31.836 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:57:31 smithi164 bash[59462]: audit 1685822250.56418 mon.c (mon.2) 354 : audit 1 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-03T19:57:31.836 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:57:31 smithi164 bash[59462]: audit 1685822250.5731342 mon.a (mon.0) 627 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:57:32.779 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:57:32 smithi089 bash[75068]: cluster 1685822252.4516516 mon.a (mon.0) 628 : cluster 1 Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 5/72 objects degraded (6.944%), 2 pgs degraded) 2023-06-03T19:57:32.779 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:57:32 smithi089 bash[75068]: cluster 1685822252.4517093 mon.a (mon.0) 629 : cluster 1 Cluster is now healthy 2023-06-03T19:57:32.779 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:57:32 smithi089 bash[78452]: cluster 1685822252.4516516 mon.a (mon.0) 628 : cluster 1 Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 5/72 objects degraded (6.944%), 2 pgs degraded) 2023-06-03T19:57:32.779 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:57:32 smithi089 bash[78452]: cluster 1685822252.4517093 mon.a (mon.0) 629 : cluster 1 Cluster is now healthy 2023-06-03T19:57:32.834 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:57:32 smithi164 bash[59462]: cluster 1685822252.4516516 mon.a (mon.0) 628 : cluster 1 Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 5/72 objects degraded (6.944%), 2 pgs degraded) 2023-06-03T19:57:32.834 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:57:32 smithi164 bash[59462]: cluster 1685822252.4517093 mon.a (mon.0) 629 : cluster 1 Cluster is now healthy 2023-06-03T19:57:33.778 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:57:33 smithi089 bash[75068]: cluster 1685822252.362876 mgr.y (mgr.34104) 633 : cluster 0 pgmap v541: 73 pgs: 16 peering, 57 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-06-03T19:57:33.779 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:57:33 smithi089 bash[78452]: cluster 1685822252.362876 mgr.y (mgr.34104) 633 : cluster 0 pgmap v541: 73 pgs: 16 peering, 57 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-06-03T19:57:33.834 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:57:33 smithi164 bash[59462]: cluster 1685822252.362876 mgr.y (mgr.34104) 633 : cluster 0 pgmap v541: 73 pgs: 16 peering, 57 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-06-03T19:57:34.827 INFO:teuthology.orchestra.run.smithi089.stdout: 2023-06-03T19:57:34.828 INFO:teuthology.orchestra.run.smithi089.stdout:{"epoch":125,"fsid":"f1db3478-0243-11ee-9b1d-001a4aab830c","created":"2023-06-03T19:22:54.447184+0000","modified":"2023-06-03T19:57:28.437574+0000","last_up_change":"2023-06-03T19:57:27.425070+0000","last_in_change":"2023-06-03T19:30:46.909676+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":23,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":8,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"luminous","require_osd_release":"pacific","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2023-06-03T19:27:28.100610+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"70","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{},"mgr_devicehealth":{}},"read_balance":{"score_acting":7.8899998664855957,"score_stable":7.8899998664855957,"optimal_score":0.37999999523162842,"raw_score_acting":3,"raw_score_stable":3,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2023-06-03T19:32:20.641846+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"55","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_acting":2,"score_stable":2,"optimal_score":1,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":3,"pool_name":"cephfs.foo.meta","create_time":"2023-06-03T19:34:15.554658+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"59","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"foo"}},"read_balance":{"score_acting":1.5,"score_stable":1.5,"optimal_score":1,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":4,"pool_name":"cephfs.foo.data","create_time":"2023-06-03T19:34:16.201294+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"60","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"cephfs":{"data":"foo"}},"read_balance":{"score_acting":2.25,"score_stable":2.25,"optimal_score":1,"raw_score_acting":2.25,"raw_score_stable":2.25,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"86063bfe-126e-46e7-93ed-c73813af165b","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":8,"last_clean_end":82,"up_from":86,"up_thru":124,"down_at":83,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:6802","nonce":2920103307},{"type":"v1","addr":"172.21.15.89:6803","nonce":2920103307}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:6804","nonce":2920103307},{"type":"v1","addr":"172.21.15.89:6805","nonce":2920103307}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:6808","nonce":2920103307},{"type":"v1","addr":"172.21.15.89:6809","nonce":2920103307}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:6806","nonce":2920103307},{"type":"v1","addr":"172.21.15.89:6807","nonce":2920103307}]},"public_addr":"172.21.15.89:6803/2920103307","cluster_addr":"172.21.15.89:6805/2920103307","heartbeat_back_addr":"172.21.15.89:6809/2920103307","heartbeat_front_addr":"172.21.15.89:6807/2920103307","state":["exists","up"]},{"osd":1,"uuid":"04783a81-6b8c-4657-8446-00efc872b313","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":13,"last_clean_end":99,"up_from":103,"up_thru":124,"down_at":100,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:6810","nonce":2092956116},{"type":"v1","addr":"172.21.15.89:6811","nonce":2092956116}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:6812","nonce":2092956116},{"type":"v1","addr":"172.21.15.89:6813","nonce":2092956116}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:6816","nonce":2092956116},{"type":"v1","addr":"172.21.15.89:6817","nonce":2092956116}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:6814","nonce":2092956116},{"type":"v1","addr":"172.21.15.89:6815","nonce":2092956116}]},"public_addr":"172.21.15.89:6811/2092956116","cluster_addr":"172.21.15.89:6813/2092956116","heartbeat_back_addr":"172.21.15.89:6817/2092956116","heartbeat_front_addr":"172.21.15.89:6815/2092956116","state":["exists","up"]},{"osd":2,"uuid":"d2d2c641-392a-444f-81bc-84eef6340f30","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":18,"last_clean_end":110,"up_from":114,"up_thru":121,"down_at":111,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:6818","nonce":1439643284},{"type":"v1","addr":"172.21.15.89:6819","nonce":1439643284}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:6820","nonce":1439643284},{"type":"v1","addr":"172.21.15.89:6821","nonce":1439643284}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:6824","nonce":1439643284},{"type":"v1","addr":"172.21.15.89:6825","nonce":1439643284}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:6822","nonce":1439643284},{"type":"v1","addr":"172.21.15.89:6823","nonce":1439643284}]},"public_addr":"172.21.15.89:6819/1439643284","cluster_addr":"172.21.15.89:6821/1439643284","heartbeat_back_addr":"172.21.15.89:6825/1439643284","heartbeat_front_addr":"172.21.15.89:6823/1439643284","state":["exists","up"]},{"osd":3,"uuid":"b38408eb-3acf-4a41-a314-72e4cd37b1ef","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":25,"last_clean_end":115,"up_from":119,"up_thru":124,"down_at":116,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:6826","nonce":1558463544},{"type":"v1","addr":"172.21.15.89:6827","nonce":1558463544}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:6828","nonce":1558463544},{"type":"v1","addr":"172.21.15.89:6829","nonce":1558463544}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:6832","nonce":1558463544},{"type":"v1","addr":"172.21.15.89:6833","nonce":1558463544}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:6830","nonce":1558463544},{"type":"v1","addr":"172.21.15.89:6831","nonce":1558463544}]},"public_addr":"172.21.15.89:6827/1558463544","cluster_addr":"172.21.15.89:6829/1558463544","heartbeat_back_addr":"172.21.15.89:6833/1558463544","heartbeat_front_addr":"172.21.15.89:6831/1558463544","state":["exists","up"]},{"osd":4,"uuid":"a94be785-9f54-4ab8-962a-9cf974a5b5e3","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":30,"last_clean_end":120,"up_from":124,"up_thru":124,"down_at":121,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.164:6800","nonce":3822759075},{"type":"v1","addr":"172.21.15.164:6801","nonce":3822759075}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.164:6802","nonce":3822759075},{"type":"v1","addr":"172.21.15.164:6803","nonce":3822759075}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.164:6806","nonce":3822759075},{"type":"v1","addr":"172.21.15.164:6807","nonce":3822759075}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.164:6804","nonce":3822759075},{"type":"v1","addr":"172.21.15.164:6805","nonce":3822759075}]},"public_addr":"172.21.15.164:6801/3822759075","cluster_addr":"172.21.15.164:6803/3822759075","heartbeat_back_addr":"172.21.15.164:6807/3822759075","heartbeat_front_addr":"172.21.15.164:6805/3822759075","state":["exists","up"]},{"osd":5,"uuid":"b8a5e8a1-23cc-442b-90aa-19bd0036dd3c","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":36,"up_thru":124,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.164:6808","nonce":3123006405},{"type":"v1","addr":"172.21.15.164:6809","nonce":3123006405}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.164:6810","nonce":3123006405},{"type":"v1","addr":"172.21.15.164:6811","nonce":3123006405}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.164:6814","nonce":3123006405},{"type":"v1","addr":"172.21.15.164:6815","nonce":3123006405}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.164:6812","nonce":3123006405},{"type":"v1","addr":"172.21.15.164:6813","nonce":3123006405}]},"public_addr":"172.21.15.164:6809/3123006405","cluster_addr":"172.21.15.164:6811/3123006405","heartbeat_back_addr":"172.21.15.164:6815/3123006405","heartbeat_front_addr":"172.21.15.164:6813/3123006405","state":["exists","up"]},{"osd":6,"uuid":"5c5edd3b-d92c-4f46-94ff-2d5828b2e732","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":42,"up_thru":124,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.164:6816","nonce":712403290},{"type":"v1","addr":"172.21.15.164:6817","nonce":712403290}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.164:6818","nonce":712403290},{"type":"v1","addr":"172.21.15.164:6819","nonce":712403290}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.164:6822","nonce":712403290},{"type":"v1","addr":"172.21.15.164:6823","nonce":712403290}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.164:6820","nonce":712403290},{"type":"v1","addr":"172.21.15.164:6821","nonce":712403290}]},"public_addr":"172.21.15.164:6817/712403290","cluster_addr":"172.21.15.164:6819/712403290","heartbeat_back_addr":"172.21.15.164:6823/712403290","heartbeat_front_addr":"172.21.15.164:6821/712403290","state":["exists","up"]},{"osd":7,"uuid":"bdff9290-e6c4-407e-9113-eb39696c0e27","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":48,"up_thru":124,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.164:6824","nonce":711997991},{"type":"v1","addr":"172.21.15.164:6825","nonce":711997991}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.164:6826","nonce":711997991},{"type":"v1","addr":"172.21.15.164:6827","nonce":711997991}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.164:6830","nonce":711997991},{"type":"v1","addr":"172.21.15.164:6831","nonce":711997991}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.164:6828","nonce":711997991},{"type":"v1","addr":"172.21.15.164:6829","nonce":711997991}]},"public_addr":"172.21.15.164:6825/711997991","cluster_addr":"172.21.15.164:6827/711997991","heartbeat_back_addr":"172.21.15.164:6831/711997991","heartbeat_front_addr":"172.21.15.164:6829/711997991","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-06-03T19:25:52.155714+0000","dead_epoch":82},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-06-03T19:26:38.554458+0000","dead_epoch":99},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-06-03T19:27:25.383237+0000","dead_epoch":110},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-06-03T19:28:13.272585+0000","dead_epoch":115},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-06-03T19:28:56.685710+0000","dead_epoch":120},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-06-03T19:29:41.572533+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-06-03T19:30:26.631983+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-06-03T19:31:12.018414+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[{"pgid":"4.9","mappings":[{"from":1,"to":2}]},{"pgid":"4.b","mappings":[{"from":1,"to":2}]},{"pgid":"4.d","mappings":[{"from":1,"to":5}]}],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.89:0/3543817158":"2023-06-04T19:41:21.936058+0000","172.21.15.164:0/2793695939":"2023-06-04T19:40:01.102939+0000","172.21.15.89:0/3944267183":"2023-06-04T19:23:55.626873+0000","172.21.15.89:0/2890034071":"2023-06-04T19:41:21.936058+0000","172.21.15.89:6801/3918707857":"2023-06-04T19:41:21.936058+0000","172.21.15.89:6800/3918707857":"2023-06-04T19:41:21.936058+0000","172.21.15.89:6800/2256305550":"2023-06-04T19:31:48.303240+0000","172.21.15.89:0/2023636358":"2023-06-04T19:37:38.185918+0000","172.21.15.89:0/470853232":"2023-06-04T19:31:48.303240+0000","172.21.15.164:6835/9946422":"2023-06-04T19:40:01.102939+0000","172.21.15.89:0/811967393":"2023-06-04T19:23:23.903333+0000","172.21.15.89:0/3044409358":"2023-06-04T19:41:21.936058+0000","172.21.15.164:0/3249546020":"2023-06-04T19:40:01.102939+0000","172.21.15.89:0/1198702890":"2023-06-04T19:23:23.903333+0000","172.21.15.164:0/1307416622":"2023-06-04T19:40:01.102939+0000","172.21.15.89:6801/2009133143":"2023-06-04T19:37:38.185918+0000","172.21.15.164:6834/9946422":"2023-06-04T19:40:01.102939+0000","172.21.15.89:6800/1487566136":"2023-06-04T19:23:23.903333+0000","172.21.15.89:6801/1487566136":"2023-06-04T19:23:23.903333+0000","172.21.15.89:0/274274920":"2023-06-04T19:23:55.626873+0000","172.21.15.89:0/2392321634":"2023-06-04T19:31:48.303240+0000","172.21.15.89:0/101502033":"2023-06-04T19:41:21.936058+0000","172.21.15.164:0/3011990159":"2023-06-04T19:40:01.102939+0000","172.21.15.89:6801/2256305550":"2023-06-04T19:31:48.303240+0000","172.21.15.89:6800/4116589411":"2023-06-04T19:23:55.626873+0000","172.21.15.89:0/1591228500":"2023-06-04T19:37:38.185918+0000","172.21.15.89:0/949193420":"2023-06-04T19:41:21.936058+0000","172.21.15.89:6800/2009133143":"2023-06-04T19:37:38.185918+0000","172.21.15.89:0/3513444952":"2023-06-04T19:41:21.936058+0000","172.21.15.89:6801/4116589411":"2023-06-04T19:23:55.626873+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2023-06-03T19:57:35.673 INFO:tasks.cephadm.ceph_manager.ceph:all up! 2023-06-03T19:57:35.674 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph osd dump --format=json 2023-06-03T19:57:35.779 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:57:35 smithi089 bash[75068]: cluster 1685822254.3636794 mgr.y (mgr.34104) 634 : cluster 0 pgmap v542: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-06-03T19:57:35.779 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:57:35 smithi089 bash[75068]: audit 1685822254.819749 mon.c (mon.2) 355 : audit 0 from='client.? 172.21.15.89:0/2892934451' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-06-03T19:57:35.779 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:57:35 smithi089 bash[78452]: cluster 1685822254.3636794 mgr.y (mgr.34104) 634 : cluster 0 pgmap v542: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-06-03T19:57:35.779 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:57:35 smithi089 bash[78452]: audit 1685822254.819749 mon.c (mon.2) 355 : audit 0 from='client.? 172.21.15.89:0/2892934451' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-06-03T19:57:35.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:57:35 smithi164 bash[59462]: cluster 1685822254.3636794 mgr.y (mgr.34104) 634 : cluster 0 pgmap v542: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-06-03T19:57:35.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:57:35 smithi164 bash[59462]: audit 1685822254.819749 mon.c (mon.2) 355 : audit 0 from='client.? 172.21.15.89:0/2892934451' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-06-03T19:57:36.528 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 19:57:36 smithi089 bash[71163]: ::ffff:172.21.15.164 - - [03/Jun/2023:19:57:36] "GET /metrics HTTP/1.1" 200 220351 "" "Prometheus/2.33.4" 2023-06-03T19:57:37.779 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:57:37 smithi089 bash[75068]: cluster 1685822256.3641891 mgr.y (mgr.34104) 635 : cluster 0 pgmap v543: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-06-03T19:57:37.779 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:57:37 smithi089 bash[75068]: audit 1685822257.0968223 mon.a (mon.0) 630 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:57:37.779 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:57:37 smithi089 bash[75068]: audit 1685822257.1010594 mon.c (mon.2) 356 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T19:57:37.780 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:57:37 smithi089 bash[78452]: cluster 1685822256.3641891 mgr.y (mgr.34104) 635 : cluster 0 pgmap v543: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-06-03T19:57:37.780 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:57:37 smithi089 bash[78452]: audit 1685822257.0968223 mon.a (mon.0) 630 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:57:37.780 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:57:37 smithi089 bash[78452]: audit 1685822257.1010594 mon.c (mon.2) 356 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T19:57:37.834 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:57:37 smithi164 bash[59462]: cluster 1685822256.3641891 mgr.y (mgr.34104) 635 : cluster 0 pgmap v543: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-06-03T19:57:37.834 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:57:37 smithi164 bash[59462]: audit 1685822257.0968223 mon.a (mon.0) 630 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:57:37.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:57:37 smithi164 bash[59462]: audit 1685822257.1010594 mon.c (mon.2) 356 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T19:57:39.778 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:57:39 smithi089 bash[78452]: cluster 1685822258.365211 mgr.y (mgr.34104) 636 : cluster 0 pgmap v544: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-06-03T19:57:39.779 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:57:39 smithi089 bash[75068]: cluster 1685822258.365211 mgr.y (mgr.34104) 636 : cluster 0 pgmap v544: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-06-03T19:57:39.834 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:57:39 smithi164 bash[59462]: cluster 1685822258.365211 mgr.y (mgr.34104) 636 : cluster 0 pgmap v544: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-06-03T19:57:40.481 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/mon.a/config 2023-06-03T19:57:41.781 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:57:41 smithi089 bash[75068]: cluster 1685822260.365851 mgr.y (mgr.34104) 637 : cluster 0 pgmap v545: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-06-03T19:57:41.782 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:57:41 smithi089 bash[78452]: cluster 1685822260.365851 mgr.y (mgr.34104) 637 : cluster 0 pgmap v545: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-06-03T19:57:41.834 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:57:41 smithi164 bash[59462]: cluster 1685822260.365851 mgr.y (mgr.34104) 637 : cluster 0 pgmap v545: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-06-03T19:57:43.767 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:57:43 smithi089 bash[75068]: cluster 1685822262.3666923 mgr.y (mgr.34104) 638 : cluster 0 pgmap v546: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-06-03T19:57:43.767 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:57:43 smithi089 bash[75068]: audit 1685822262.462633 mon.a (mon.0) 631 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:57:43.767 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:57:43 smithi089 bash[78452]: cluster 1685822262.3666923 mgr.y (mgr.34104) 638 : cluster 0 pgmap v546: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-06-03T19:57:43.767 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:57:43 smithi089 bash[78452]: audit 1685822262.462633 mon.a (mon.0) 631 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:57:43.834 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:57:43 smithi164 bash[59462]: cluster 1685822262.3666923 mgr.y (mgr.34104) 638 : cluster 0 pgmap v546: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-06-03T19:57:43.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:57:43 smithi164 bash[59462]: audit 1685822262.462633 mon.a (mon.0) 631 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:57:44.148 INFO:teuthology.orchestra.run.smithi089.stdout: 2023-06-03T19:57:44.149 INFO:teuthology.orchestra.run.smithi089.stdout:{"epoch":125,"fsid":"f1db3478-0243-11ee-9b1d-001a4aab830c","created":"2023-06-03T19:22:54.447184+0000","modified":"2023-06-03T19:57:28.437574+0000","last_up_change":"2023-06-03T19:57:27.425070+0000","last_in_change":"2023-06-03T19:30:46.909676+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":23,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":8,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"luminous","require_osd_release":"pacific","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2023-06-03T19:27:28.100610+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"70","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{},"mgr_devicehealth":{}},"read_balance":{"score_acting":7.8899998664855957,"score_stable":7.8899998664855957,"optimal_score":0.37999999523162842,"raw_score_acting":3,"raw_score_stable":3,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2023-06-03T19:32:20.641846+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"55","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_acting":2,"score_stable":2,"optimal_score":1,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":3,"pool_name":"cephfs.foo.meta","create_time":"2023-06-03T19:34:15.554658+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"59","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"foo"}},"read_balance":{"score_acting":1.5,"score_stable":1.5,"optimal_score":1,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":4,"pool_name":"cephfs.foo.data","create_time":"2023-06-03T19:34:16.201294+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"60","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"cephfs":{"data":"foo"}},"read_balance":{"score_acting":2.25,"score_stable":2.25,"optimal_score":1,"raw_score_acting":2.25,"raw_score_stable":2.25,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"86063bfe-126e-46e7-93ed-c73813af165b","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":8,"last_clean_end":82,"up_from":86,"up_thru":124,"down_at":83,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:6802","nonce":2920103307},{"type":"v1","addr":"172.21.15.89:6803","nonce":2920103307}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:6804","nonce":2920103307},{"type":"v1","addr":"172.21.15.89:6805","nonce":2920103307}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:6808","nonce":2920103307},{"type":"v1","addr":"172.21.15.89:6809","nonce":2920103307}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:6806","nonce":2920103307},{"type":"v1","addr":"172.21.15.89:6807","nonce":2920103307}]},"public_addr":"172.21.15.89:6803/2920103307","cluster_addr":"172.21.15.89:6805/2920103307","heartbeat_back_addr":"172.21.15.89:6809/2920103307","heartbeat_front_addr":"172.21.15.89:6807/2920103307","state":["exists","up"]},{"osd":1,"uuid":"04783a81-6b8c-4657-8446-00efc872b313","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":13,"last_clean_end":99,"up_from":103,"up_thru":124,"down_at":100,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:6810","nonce":2092956116},{"type":"v1","addr":"172.21.15.89:6811","nonce":2092956116}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:6812","nonce":2092956116},{"type":"v1","addr":"172.21.15.89:6813","nonce":2092956116}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:6816","nonce":2092956116},{"type":"v1","addr":"172.21.15.89:6817","nonce":2092956116}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:6814","nonce":2092956116},{"type":"v1","addr":"172.21.15.89:6815","nonce":2092956116}]},"public_addr":"172.21.15.89:6811/2092956116","cluster_addr":"172.21.15.89:6813/2092956116","heartbeat_back_addr":"172.21.15.89:6817/2092956116","heartbeat_front_addr":"172.21.15.89:6815/2092956116","state":["exists","up"]},{"osd":2,"uuid":"d2d2c641-392a-444f-81bc-84eef6340f30","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":18,"last_clean_end":110,"up_from":114,"up_thru":121,"down_at":111,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:6818","nonce":1439643284},{"type":"v1","addr":"172.21.15.89:6819","nonce":1439643284}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:6820","nonce":1439643284},{"type":"v1","addr":"172.21.15.89:6821","nonce":1439643284}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:6824","nonce":1439643284},{"type":"v1","addr":"172.21.15.89:6825","nonce":1439643284}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:6822","nonce":1439643284},{"type":"v1","addr":"172.21.15.89:6823","nonce":1439643284}]},"public_addr":"172.21.15.89:6819/1439643284","cluster_addr":"172.21.15.89:6821/1439643284","heartbeat_back_addr":"172.21.15.89:6825/1439643284","heartbeat_front_addr":"172.21.15.89:6823/1439643284","state":["exists","up"]},{"osd":3,"uuid":"b38408eb-3acf-4a41-a314-72e4cd37b1ef","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":25,"last_clean_end":115,"up_from":119,"up_thru":124,"down_at":116,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:6826","nonce":1558463544},{"type":"v1","addr":"172.21.15.89:6827","nonce":1558463544}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:6828","nonce":1558463544},{"type":"v1","addr":"172.21.15.89:6829","nonce":1558463544}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:6832","nonce":1558463544},{"type":"v1","addr":"172.21.15.89:6833","nonce":1558463544}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:6830","nonce":1558463544},{"type":"v1","addr":"172.21.15.89:6831","nonce":1558463544}]},"public_addr":"172.21.15.89:6827/1558463544","cluster_addr":"172.21.15.89:6829/1558463544","heartbeat_back_addr":"172.21.15.89:6833/1558463544","heartbeat_front_addr":"172.21.15.89:6831/1558463544","state":["exists","up"]},{"osd":4,"uuid":"a94be785-9f54-4ab8-962a-9cf974a5b5e3","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":30,"last_clean_end":120,"up_from":124,"up_thru":124,"down_at":121,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.164:6800","nonce":3822759075},{"type":"v1","addr":"172.21.15.164:6801","nonce":3822759075}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.164:6802","nonce":3822759075},{"type":"v1","addr":"172.21.15.164:6803","nonce":3822759075}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.164:6806","nonce":3822759075},{"type":"v1","addr":"172.21.15.164:6807","nonce":3822759075}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.164:6804","nonce":3822759075},{"type":"v1","addr":"172.21.15.164:6805","nonce":3822759075}]},"public_addr":"172.21.15.164:6801/3822759075","cluster_addr":"172.21.15.164:6803/3822759075","heartbeat_back_addr":"172.21.15.164:6807/3822759075","heartbeat_front_addr":"172.21.15.164:6805/3822759075","state":["exists","up"]},{"osd":5,"uuid":"b8a5e8a1-23cc-442b-90aa-19bd0036dd3c","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":36,"up_thru":124,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.164:6808","nonce":3123006405},{"type":"v1","addr":"172.21.15.164:6809","nonce":3123006405}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.164:6810","nonce":3123006405},{"type":"v1","addr":"172.21.15.164:6811","nonce":3123006405}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.164:6814","nonce":3123006405},{"type":"v1","addr":"172.21.15.164:6815","nonce":3123006405}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.164:6812","nonce":3123006405},{"type":"v1","addr":"172.21.15.164:6813","nonce":3123006405}]},"public_addr":"172.21.15.164:6809/3123006405","cluster_addr":"172.21.15.164:6811/3123006405","heartbeat_back_addr":"172.21.15.164:6815/3123006405","heartbeat_front_addr":"172.21.15.164:6813/3123006405","state":["exists","up"]},{"osd":6,"uuid":"5c5edd3b-d92c-4f46-94ff-2d5828b2e732","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":42,"up_thru":124,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.164:6816","nonce":712403290},{"type":"v1","addr":"172.21.15.164:6817","nonce":712403290}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.164:6818","nonce":712403290},{"type":"v1","addr":"172.21.15.164:6819","nonce":712403290}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.164:6822","nonce":712403290},{"type":"v1","addr":"172.21.15.164:6823","nonce":712403290}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.164:6820","nonce":712403290},{"type":"v1","addr":"172.21.15.164:6821","nonce":712403290}]},"public_addr":"172.21.15.164:6817/712403290","cluster_addr":"172.21.15.164:6819/712403290","heartbeat_back_addr":"172.21.15.164:6823/712403290","heartbeat_front_addr":"172.21.15.164:6821/712403290","state":["exists","up"]},{"osd":7,"uuid":"bdff9290-e6c4-407e-9113-eb39696c0e27","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":48,"up_thru":124,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.164:6824","nonce":711997991},{"type":"v1","addr":"172.21.15.164:6825","nonce":711997991}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.164:6826","nonce":711997991},{"type":"v1","addr":"172.21.15.164:6827","nonce":711997991}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.164:6830","nonce":711997991},{"type":"v1","addr":"172.21.15.164:6831","nonce":711997991}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.164:6828","nonce":711997991},{"type":"v1","addr":"172.21.15.164:6829","nonce":711997991}]},"public_addr":"172.21.15.164:6825/711997991","cluster_addr":"172.21.15.164:6827/711997991","heartbeat_back_addr":"172.21.15.164:6831/711997991","heartbeat_front_addr":"172.21.15.164:6829/711997991","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-06-03T19:25:52.155714+0000","dead_epoch":82},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-06-03T19:26:38.554458+0000","dead_epoch":99},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-06-03T19:27:25.383237+0000","dead_epoch":110},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-06-03T19:28:13.272585+0000","dead_epoch":115},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-06-03T19:28:56.685710+0000","dead_epoch":120},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-06-03T19:29:41.572533+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-06-03T19:30:26.631983+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-06-03T19:31:12.018414+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[{"pgid":"4.9","mappings":[{"from":1,"to":2}]},{"pgid":"4.b","mappings":[{"from":1,"to":2}]},{"pgid":"4.d","mappings":[{"from":1,"to":5}]}],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.89:0/3543817158":"2023-06-04T19:41:21.936058+0000","172.21.15.164:0/2793695939":"2023-06-04T19:40:01.102939+0000","172.21.15.89:0/3944267183":"2023-06-04T19:23:55.626873+0000","172.21.15.89:0/2890034071":"2023-06-04T19:41:21.936058+0000","172.21.15.89:6801/3918707857":"2023-06-04T19:41:21.936058+0000","172.21.15.89:6800/3918707857":"2023-06-04T19:41:21.936058+0000","172.21.15.89:6800/2256305550":"2023-06-04T19:31:48.303240+0000","172.21.15.89:0/2023636358":"2023-06-04T19:37:38.185918+0000","172.21.15.89:0/470853232":"2023-06-04T19:31:48.303240+0000","172.21.15.164:6835/9946422":"2023-06-04T19:40:01.102939+0000","172.21.15.89:0/811967393":"2023-06-04T19:23:23.903333+0000","172.21.15.89:0/3044409358":"2023-06-04T19:41:21.936058+0000","172.21.15.164:0/3249546020":"2023-06-04T19:40:01.102939+0000","172.21.15.89:0/1198702890":"2023-06-04T19:23:23.903333+0000","172.21.15.164:0/1307416622":"2023-06-04T19:40:01.102939+0000","172.21.15.89:6801/2009133143":"2023-06-04T19:37:38.185918+0000","172.21.15.164:6834/9946422":"2023-06-04T19:40:01.102939+0000","172.21.15.89:6800/1487566136":"2023-06-04T19:23:23.903333+0000","172.21.15.89:6801/1487566136":"2023-06-04T19:23:23.903333+0000","172.21.15.89:0/274274920":"2023-06-04T19:23:55.626873+0000","172.21.15.89:0/2392321634":"2023-06-04T19:31:48.303240+0000","172.21.15.89:0/101502033":"2023-06-04T19:41:21.936058+0000","172.21.15.164:0/3011990159":"2023-06-04T19:40:01.102939+0000","172.21.15.89:6801/2256305550":"2023-06-04T19:31:48.303240+0000","172.21.15.89:6800/4116589411":"2023-06-04T19:23:55.626873+0000","172.21.15.89:0/1591228500":"2023-06-04T19:37:38.185918+0000","172.21.15.89:0/949193420":"2023-06-04T19:41:21.936058+0000","172.21.15.89:6800/2009133143":"2023-06-04T19:37:38.185918+0000","172.21.15.89:0/3513444952":"2023-06-04T19:41:21.936058+0000","172.21.15.89:6801/4116589411":"2023-06-04T19:23:55.626873+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2023-06-03T19:57:44.779 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:57:44 smithi089 bash[78452]: audit 1685822264.1410477 mon.c (mon.2) 357 : audit 0 from='client.? 172.21.15.89:0/767147459' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-06-03T19:57:44.779 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:57:44 smithi089 bash[75068]: audit 1685822264.1410477 mon.c (mon.2) 357 : audit 0 from='client.? 172.21.15.89:0/767147459' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-06-03T19:57:44.834 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:57:44 smithi164 bash[59462]: audit 1685822264.1410477 mon.c (mon.2) 357 : audit 0 from='client.? 172.21.15.89:0/767147459' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-06-03T19:57:44.991 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph tell osd.0 flush_pg_stats 2023-06-03T19:57:44.992 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph tell osd.1 flush_pg_stats 2023-06-03T19:57:44.993 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph tell osd.2 flush_pg_stats 2023-06-03T19:57:44.993 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph tell osd.3 flush_pg_stats 2023-06-03T19:57:44.994 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph tell osd.4 flush_pg_stats 2023-06-03T19:57:44.994 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph tell osd.5 flush_pg_stats 2023-06-03T19:57:44.995 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph tell osd.6 flush_pg_stats 2023-06-03T19:57:44.995 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph tell osd.7 flush_pg_stats 2023-06-03T19:57:45.779 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:57:45 smithi089 bash[78452]: cluster 1685822264.367501 mgr.y (mgr.34104) 639 : cluster 0 pgmap v547: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-06-03T19:57:45.779 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:57:45 smithi089 bash[75068]: cluster 1685822264.367501 mgr.y (mgr.34104) 639 : cluster 0 pgmap v547: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-06-03T19:57:45.834 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:57:45 smithi164 bash[59462]: cluster 1685822264.367501 mgr.y (mgr.34104) 639 : cluster 0 pgmap v547: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-06-03T19:57:46.529 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 19:57:46 smithi089 bash[71163]: ::ffff:172.21.15.164 - - [03/Jun/2023:19:57:46] "GET /metrics HTTP/1.1" 200 225339 "" "Prometheus/2.33.4" 2023-06-03T19:57:47.779 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:57:47 smithi089 bash[78452]: cluster 1685822266.36829 mgr.y (mgr.34104) 640 : cluster 0 pgmap v548: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-06-03T19:57:47.779 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:57:47 smithi089 bash[75068]: cluster 1685822266.36829 mgr.y (mgr.34104) 640 : cluster 0 pgmap v548: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-06-03T19:57:47.834 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:57:47 smithi164 bash[59462]: cluster 1685822266.36829 mgr.y (mgr.34104) 640 : cluster 0 pgmap v548: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-06-03T19:57:49.779 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:57:49 smithi089 bash[75068]: cluster 1685822268.3691502 mgr.y (mgr.34104) 641 : cluster 0 pgmap v549: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-06-03T19:57:49.779 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:57:49 smithi089 bash[78452]: cluster 1685822268.3691502 mgr.y (mgr.34104) 641 : cluster 0 pgmap v549: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-06-03T19:57:49.834 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:57:49 smithi164 bash[59462]: cluster 1685822268.3691502 mgr.y (mgr.34104) 641 : cluster 0 pgmap v549: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-06-03T19:57:49.921 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/mon.a/config 2023-06-03T19:57:49.922 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/mon.a/config 2023-06-03T19:57:49.924 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/mon.a/config 2023-06-03T19:57:49.928 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/mon.a/config 2023-06-03T19:57:49.929 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/mon.a/config 2023-06-03T19:57:49.929 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/mon.a/config 2023-06-03T19:57:49.929 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/mon.a/config 2023-06-03T19:57:49.930 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/mon.a/config 2023-06-03T19:57:51.778 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:57:51 smithi089 bash[75068]: cluster 1685822270.369725 mgr.y (mgr.34104) 642 : cluster 0 pgmap v550: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-06-03T19:57:51.779 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:57:51 smithi089 bash[78452]: cluster 1685822270.369725 mgr.y (mgr.34104) 642 : cluster 0 pgmap v550: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-06-03T19:57:51.834 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:57:51 smithi164 bash[59462]: cluster 1685822270.369725 mgr.y (mgr.34104) 642 : cluster 0 pgmap v550: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-06-03T19:57:52.778 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:57:52 smithi089 bash[75068]: audit 1685822272.0900478 mon.c (mon.2) 358 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T19:57:52.779 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:57:52 smithi089 bash[78452]: audit 1685822272.0900478 mon.c (mon.2) 358 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T19:57:52.834 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:57:52 smithi164 bash[59462]: audit 1685822272.0900478 mon.c (mon.2) 358 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T19:57:53.779 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:57:53 smithi089 bash[78452]: cluster 1685822272.3703723 mgr.y (mgr.34104) 643 : cluster 0 pgmap v551: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-06-03T19:57:53.779 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:57:53 smithi089 bash[75068]: cluster 1685822272.3703723 mgr.y (mgr.34104) 643 : cluster 0 pgmap v551: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-06-03T19:57:53.834 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:57:53 smithi164 bash[59462]: cluster 1685822272.3703723 mgr.y (mgr.34104) 643 : cluster 0 pgmap v551: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-06-03T19:57:55.779 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:57:55 smithi089 bash[78452]: cluster 1685822274.3711355 mgr.y (mgr.34104) 644 : cluster 0 pgmap v552: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-06-03T19:57:55.779 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:57:55 smithi089 bash[75068]: cluster 1685822274.3711355 mgr.y (mgr.34104) 644 : cluster 0 pgmap v552: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-06-03T19:57:55.834 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:57:55 smithi164 bash[59462]: cluster 1685822274.3711355 mgr.y (mgr.34104) 644 : cluster 0 pgmap v552: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-06-03T19:57:56.529 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 19:57:56 smithi089 bash[71163]: ::ffff:172.21.15.164 - - [03/Jun/2023:19:57:56] "GET /metrics HTTP/1.1" 200 225323 "" "Prometheus/2.33.4" 2023-06-03T19:57:57.779 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:57:57 smithi089 bash[75068]: cluster 1685822276.3718612 mgr.y (mgr.34104) 645 : cluster 0 pgmap v553: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-06-03T19:57:57.779 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:57:57 smithi089 bash[78452]: cluster 1685822276.3718612 mgr.y (mgr.34104) 645 : cluster 0 pgmap v553: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-06-03T19:57:57.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:57:57 smithi164 bash[59462]: cluster 1685822276.3718612 mgr.y (mgr.34104) 645 : cluster 0 pgmap v553: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-06-03T19:57:59.778 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:57:59 smithi089 bash[75068]: cluster 1685822278.372972 mgr.y (mgr.34104) 646 : cluster 0 pgmap v554: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-06-03T19:57:59.779 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:57:59 smithi089 bash[78452]: cluster 1685822278.372972 mgr.y (mgr.34104) 646 : cluster 0 pgmap v554: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-06-03T19:57:59.834 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:57:59 smithi164 bash[59462]: cluster 1685822278.372972 mgr.y (mgr.34104) 646 : cluster 0 pgmap v554: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-06-03T19:58:01.778 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:58:01 smithi089 bash[75068]: cluster 1685822280.373572 mgr.y (mgr.34104) 647 : cluster 0 pgmap v555: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-06-03T19:58:01.779 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:58:01 smithi089 bash[78452]: cluster 1685822280.373572 mgr.y (mgr.34104) 647 : cluster 0 pgmap v555: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-06-03T19:58:01.834 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:58:01 smithi164 bash[59462]: cluster 1685822280.373572 mgr.y (mgr.34104) 647 : cluster 0 pgmap v555: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-06-03T19:58:03.779 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:58:03 smithi089 bash[75068]: cluster 1685822282.3742259 mgr.y (mgr.34104) 648 : cluster 0 pgmap v556: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-06-03T19:58:03.779 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:58:03 smithi089 bash[78452]: cluster 1685822282.3742259 mgr.y (mgr.34104) 648 : cluster 0 pgmap v556: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-06-03T19:58:03.834 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:58:03 smithi164 bash[59462]: cluster 1685822282.3742259 mgr.y (mgr.34104) 648 : cluster 0 pgmap v556: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-06-03T19:58:05.773 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:58:05 smithi089 bash[75068]: cluster 1685822284.3748026 mgr.y (mgr.34104) 649 : cluster 0 pgmap v557: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-06-03T19:58:05.774 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:58:05 smithi089 bash[78452]: cluster 1685822284.3748026 mgr.y (mgr.34104) 649 : cluster 0 pgmap v557: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-06-03T19:58:05.834 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:58:05 smithi164 bash[59462]: cluster 1685822284.3748026 mgr.y (mgr.34104) 649 : cluster 0 pgmap v557: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-06-03T19:58:05.950 INFO:teuthology.orchestra.run.smithi089.stdout:180388626764 2023-06-03T19:58:05.951 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph osd last-stat-seq osd.6 2023-06-03T19:58:06.596 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 19:58:06 smithi089 bash[71163]: ::ffff:172.21.15.164 - - [03/Jun/2023:19:58:06] "GET /metrics HTTP/1.1" 200 225323 "" "Prometheus/2.33.4" 2023-06-03T19:58:07.288 INFO:teuthology.orchestra.run.smithi089.stdout:489626271768 2023-06-03T19:58:07.288 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph osd last-stat-seq osd.2 2023-06-03T19:58:07.644 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:58:07 smithi089 bash[75068]: cluster 1685822286.3751996 mgr.y (mgr.34104) 650 : cluster 0 pgmap v558: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-06-03T19:58:07.644 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:58:07 smithi089 bash[75068]: audit 1685822287.0906494 mon.c (mon.2) 359 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T19:58:07.645 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:58:07 smithi089 bash[78452]: cluster 1685822286.3751996 mgr.y (mgr.34104) 650 : cluster 0 pgmap v558: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-06-03T19:58:07.645 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:58:07 smithi089 bash[78452]: audit 1685822287.0906494 mon.c (mon.2) 359 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T19:58:07.646 INFO:teuthology.orchestra.run.smithi089.stdout:206158430532 2023-06-03T19:58:07.646 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph osd last-stat-seq osd.7 2023-06-03T19:58:07.834 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:58:07 smithi164 bash[59462]: cluster 1685822286.3751996 mgr.y (mgr.34104) 650 : cluster 0 pgmap v558: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-06-03T19:58:07.834 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:58:07 smithi164 bash[59462]: audit 1685822287.0906494 mon.c (mon.2) 359 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T19:58:09.817 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:58:09 smithi089 bash[78452]: cluster 1685822288.375751 mgr.y (mgr.34104) 651 : cluster 0 pgmap v559: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-06-03T19:58:09.818 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:58:09 smithi089 bash[75068]: cluster 1685822288.375751 mgr.y (mgr.34104) 651 : cluster 0 pgmap v559: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-06-03T19:58:09.834 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:58:09 smithi164 bash[59462]: cluster 1685822288.375751 mgr.y (mgr.34104) 651 : cluster 0 pgmap v559: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-06-03T19:58:11.118 INFO:teuthology.orchestra.run.smithi089.stdout:532575944714 2023-06-03T19:58:11.119 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph osd last-stat-seq osd.4 2023-06-03T19:58:11.557 INFO:teuthology.orchestra.run.smithi089.stdout:369367187590 2023-06-03T19:58:11.557 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph osd last-stat-seq osd.0 2023-06-03T19:58:11.558 INFO:teuthology.orchestra.run.smithi089.stdout:154618823000 2023-06-03T19:58:11.559 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph osd last-stat-seq osd.5 2023-06-03T19:58:11.834 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:58:11 smithi164 bash[59462]: cluster 1685822290.376514 mgr.y (mgr.34104) 652 : cluster 0 pgmap v560: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-06-03T19:58:11.872 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:58:11 smithi089 bash[75068]: cluster 1685822290.376514 mgr.y (mgr.34104) 652 : cluster 0 pgmap v560: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-06-03T19:58:11.872 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:58:11 smithi089 bash[78452]: cluster 1685822290.376514 mgr.y (mgr.34104) 652 : cluster 0 pgmap v560: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-06-03T19:58:11.873 INFO:teuthology.orchestra.run.smithi089.stdout:511101108241 2023-06-03T19:58:11.873 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph osd last-stat-seq osd.3 2023-06-03T19:58:11.892 INFO:teuthology.orchestra.run.smithi089.stdout:442381631564 2023-06-03T19:58:11.893 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph osd last-stat-seq osd.1 2023-06-03T19:58:13.834 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:58:13 smithi164 bash[59462]: cluster 1685822292.377043 mgr.y (mgr.34104) 653 : cluster 0 pgmap v561: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-06-03T19:58:14.029 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:58:13 smithi089 bash[78452]: cluster 1685822292.377043 mgr.y (mgr.34104) 653 : cluster 0 pgmap v561: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-06-03T19:58:14.029 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:58:13 smithi089 bash[75068]: cluster 1685822292.377043 mgr.y (mgr.34104) 653 : cluster 0 pgmap v561: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-06-03T19:58:14.428 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/mon.a/config 2023-06-03T19:58:14.430 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/mon.a/config 2023-06-03T19:58:14.434 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/mon.a/config 2023-06-03T19:58:15.834 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:58:15 smithi164 bash[59462]: cluster 1685822294.3779764 mgr.y (mgr.34104) 654 : cluster 0 pgmap v562: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-06-03T19:58:16.028 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:58:15 smithi089 bash[75068]: cluster 1685822294.3779764 mgr.y (mgr.34104) 654 : cluster 0 pgmap v562: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-06-03T19:58:16.029 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:58:15 smithi089 bash[78452]: cluster 1685822294.3779764 mgr.y (mgr.34104) 654 : cluster 0 pgmap v562: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-06-03T19:58:16.463 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/mon.a/config 2023-06-03T19:58:16.464 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/mon.a/config 2023-06-03T19:58:16.465 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/mon.a/config 2023-06-03T19:58:16.465 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/mon.a/config 2023-06-03T19:58:16.468 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/mon.a/config 2023-06-03T19:58:16.495 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 19:58:16 smithi089 bash[71163]: ::ffff:172.21.15.164 - - [03/Jun/2023:19:58:16] "GET /metrics HTTP/1.1" 200 225300 "" "Prometheus/2.33.4" 2023-06-03T19:58:17.778 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:58:17 smithi089 bash[78452]: cluster 1685822296.37873 mgr.y (mgr.34104) 655 : cluster 0 pgmap v563: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-06-03T19:58:17.779 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:58:17 smithi089 bash[75068]: cluster 1685822296.37873 mgr.y (mgr.34104) 655 : cluster 0 pgmap v563: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-06-03T19:58:17.834 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:58:17 smithi164 bash[59462]: cluster 1685822296.37873 mgr.y (mgr.34104) 655 : cluster 0 pgmap v563: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-06-03T19:58:19.694 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:58:19 smithi089 bash[75068]: cluster 1685822298.3796313 mgr.y (mgr.34104) 656 : cluster 0 pgmap v564: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-06-03T19:58:19.694 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:58:19 smithi089 bash[78452]: cluster 1685822298.3796313 mgr.y (mgr.34104) 656 : cluster 0 pgmap v564: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-06-03T19:58:19.834 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:58:19 smithi164 bash[59462]: cluster 1685822298.3796313 mgr.y (mgr.34104) 656 : cluster 0 pgmap v564: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-06-03T19:58:21.822 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:58:21 smithi089 bash[78452]: cluster 1685822300.3802805 mgr.y (mgr.34104) 657 : cluster 0 pgmap v565: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-06-03T19:58:21.822 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:58:21 smithi089 bash[75068]: cluster 1685822300.3802805 mgr.y (mgr.34104) 657 : cluster 0 pgmap v565: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-06-03T19:58:21.834 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:58:21 smithi164 bash[59462]: cluster 1685822300.3802805 mgr.y (mgr.34104) 657 : cluster 0 pgmap v565: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-06-03T19:58:22.779 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:58:22 smithi089 bash[75068]: audit 1685822302.0897357 mon.c (mon.2) 360 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T19:58:22.780 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:58:22 smithi089 bash[78452]: audit 1685822302.0897357 mon.c (mon.2) 360 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T19:58:22.834 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:58:22 smithi164 bash[59462]: audit 1685822302.0897357 mon.c (mon.2) 360 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T19:58:23.822 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:58:23 smithi089 bash[75068]: cluster 1685822302.3810782 mgr.y (mgr.34104) 658 : cluster 0 pgmap v566: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-06-03T19:58:23.823 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:58:23 smithi089 bash[78452]: cluster 1685822302.3810782 mgr.y (mgr.34104) 658 : cluster 0 pgmap v566: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-06-03T19:58:23.834 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:58:23 smithi164 bash[59462]: cluster 1685822302.3810782 mgr.y (mgr.34104) 658 : cluster 0 pgmap v566: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-06-03T19:58:25.778 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:58:25 smithi089 bash[78452]: cluster 1685822304.3817353 mgr.y (mgr.34104) 659 : cluster 0 pgmap v567: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-06-03T19:58:25.779 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:58:25 smithi089 bash[75068]: cluster 1685822304.3817353 mgr.y (mgr.34104) 659 : cluster 0 pgmap v567: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-06-03T19:58:25.834 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:58:25 smithi164 bash[59462]: cluster 1685822304.3817353 mgr.y (mgr.34104) 659 : cluster 0 pgmap v567: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-06-03T19:58:26.563 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 19:58:26 smithi089 bash[71163]: ::ffff:172.21.15.164 - - [03/Jun/2023:19:58:26] "GET /metrics HTTP/1.1" 200 225304 "" "Prometheus/2.33.4" 2023-06-03T19:58:27.834 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:58:27 smithi164 bash[59462]: cluster 1685822306.3821497 mgr.y (mgr.34104) 660 : cluster 0 pgmap v568: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-06-03T19:58:28.029 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:58:27 smithi089 bash[75068]: cluster 1685822306.3821497 mgr.y (mgr.34104) 660 : cluster 0 pgmap v568: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-06-03T19:58:28.030 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:58:27 smithi089 bash[78452]: cluster 1685822306.3821497 mgr.y (mgr.34104) 660 : cluster 0 pgmap v568: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-06-03T19:58:29.834 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:58:29 smithi164 bash[59462]: cluster 1685822308.3829641 mgr.y (mgr.34104) 661 : cluster 0 pgmap v569: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-06-03T19:58:30.028 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:58:29 smithi089 bash[75068]: cluster 1685822308.3829641 mgr.y (mgr.34104) 661 : cluster 0 pgmap v569: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-06-03T19:58:30.029 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:58:29 smithi089 bash[78452]: cluster 1685822308.3829641 mgr.y (mgr.34104) 661 : cluster 0 pgmap v569: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-06-03T19:58:30.551 INFO:teuthology.orchestra.run.smithi089.stdout:489626271773 2023-06-03T19:58:30.601 INFO:teuthology.orchestra.run.smithi089.stdout:206158430537 2023-06-03T19:58:30.641 INFO:teuthology.orchestra.run.smithi089.stdout:180388626770 2023-06-03T19:58:30.830 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:58:30 smithi089 bash[75068]: audit 1685822310.547216 mon.a (mon.0) 632 : audit 0 from='client.? 172.21.15.89:0/2994234659' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2023-06-03T19:58:30.831 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:58:30 smithi089 bash[78452]: audit 1685822310.547216 mon.a (mon.0) 632 : audit 0 from='client.? 172.21.15.89:0/2994234659' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2023-06-03T19:58:30.834 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:58:30 smithi164 bash[59462]: audit 1685822310.547216 mon.a (mon.0) 632 : audit 0 from='client.? 172.21.15.89:0/2994234659' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2023-06-03T19:58:31.484 INFO:teuthology.orchestra.run.smithi089.stdout:511101108245 2023-06-03T19:58:31.778 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:58:31 smithi089 bash[75068]: cluster 1685822310.3833137 mgr.y (mgr.34104) 662 : cluster 0 pgmap v570: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-06-03T19:58:31.779 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:58:31 smithi089 bash[75068]: audit 1685822310.596933 mon.a (mon.0) 633 : audit 0 from='client.? 172.21.15.89:0/3843701443' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2023-06-03T19:58:31.779 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:58:31 smithi089 bash[75068]: audit 1685822310.6370392 mon.a (mon.0) 634 : audit 0 from='client.? 172.21.15.89:0/2784752003' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2023-06-03T19:58:31.779 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:58:31 smithi089 bash[75068]: audit 1685822310.657385 mon.c (mon.2) 361 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-03T19:58:31.779 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:58:31 smithi089 bash[75068]: audit 1685822311.4803414 mon.a (mon.0) 635 : audit 0 from='client.? 172.21.15.89:0/1982426580' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2023-06-03T19:58:31.779 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:58:31 smithi089 bash[78452]: cluster 1685822310.3833137 mgr.y (mgr.34104) 662 : cluster 0 pgmap v570: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-06-03T19:58:31.780 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:58:31 smithi089 bash[78452]: audit 1685822310.596933 mon.a (mon.0) 633 : audit 0 from='client.? 172.21.15.89:0/3843701443' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2023-06-03T19:58:31.780 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:58:31 smithi089 bash[78452]: audit 1685822310.6370392 mon.a (mon.0) 634 : audit 0 from='client.? 172.21.15.89:0/2784752003' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2023-06-03T19:58:31.780 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:58:31 smithi089 bash[78452]: audit 1685822310.657385 mon.c (mon.2) 361 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-03T19:58:31.780 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:58:31 smithi089 bash[78452]: audit 1685822311.4803414 mon.a (mon.0) 635 : audit 0 from='client.? 172.21.15.89:0/1982426580' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2023-06-03T19:58:31.834 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:58:31 smithi164 bash[59462]: cluster 1685822310.3833137 mgr.y (mgr.34104) 662 : cluster 0 pgmap v570: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-06-03T19:58:31.834 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:58:31 smithi164 bash[59462]: audit 1685822310.596933 mon.a (mon.0) 633 : audit 0 from='client.? 172.21.15.89:0/3843701443' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2023-06-03T19:58:31.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:58:31 smithi164 bash[59462]: audit 1685822310.6370392 mon.a (mon.0) 634 : audit 0 from='client.? 172.21.15.89:0/2784752003' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2023-06-03T19:58:31.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:58:31 smithi164 bash[59462]: audit 1685822310.657385 mon.c (mon.2) 361 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-03T19:58:31.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:58:31 smithi164 bash[59462]: audit 1685822311.4803414 mon.a (mon.0) 635 : audit 0 from='client.? 172.21.15.89:0/1982426580' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2023-06-03T19:58:33.117 INFO:tasks.cephadm.ceph_manager.ceph:need seq 180388626764 got 180388626770 for osd.6 2023-06-03T19:58:33.118 DEBUG:teuthology.parallel:result is None 2023-06-03T19:58:33.779 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:58:33 smithi089 bash[75068]: cluster 1685822312.3840673 mgr.y (mgr.34104) 663 : cluster 0 pgmap v571: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-06-03T19:58:33.779 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:58:33 smithi089 bash[78452]: cluster 1685822312.3840673 mgr.y (mgr.34104) 663 : cluster 0 pgmap v571: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-06-03T19:58:34.084 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:58:33 smithi164 bash[59462]: cluster 1685822312.3840673 mgr.y (mgr.34104) 663 : cluster 0 pgmap v571: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-06-03T19:58:34.470 INFO:tasks.cephadm.ceph_manager.ceph:need seq 206158430532 got 206158430537 for osd.7 2023-06-03T19:58:34.470 DEBUG:teuthology.parallel:result is None 2023-06-03T19:58:34.653 INFO:teuthology.orchestra.run.smithi089.stdout:154618823005 2023-06-03T19:58:35.180 INFO:tasks.cephadm.ceph_manager.ceph:need seq 489626271768 got 489626271773 for osd.2 2023-06-03T19:58:35.180 DEBUG:teuthology.parallel:result is None 2023-06-03T19:58:35.330 INFO:tasks.cephadm.ceph_manager.ceph:need seq 511101108241 got 511101108245 for osd.3 2023-06-03T19:58:35.330 DEBUG:teuthology.parallel:result is None 2023-06-03T19:58:35.642 INFO:teuthology.orchestra.run.smithi089.stdout:369367187595 2023-06-03T19:58:35.778 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:58:35 smithi089 bash[75068]: cluster 1685822314.3846636 mgr.y (mgr.34104) 664 : cluster 0 pgmap v572: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-06-03T19:58:35.779 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:58:35 smithi089 bash[75068]: audit 1685822314.6493173 mon.c (mon.2) 362 : audit 0 from='client.? 172.21.15.89:0/1073286370' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2023-06-03T19:58:35.779 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:58:35 smithi089 bash[78452]: cluster 1685822314.3846636 mgr.y (mgr.34104) 664 : cluster 0 pgmap v572: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-06-03T19:58:35.779 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:58:35 smithi089 bash[78452]: audit 1685822314.6493173 mon.c (mon.2) 362 : audit 0 from='client.? 172.21.15.89:0/1073286370' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2023-06-03T19:58:36.084 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:58:35 smithi164 bash[59462]: cluster 1685822314.3846636 mgr.y (mgr.34104) 664 : cluster 0 pgmap v572: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-06-03T19:58:36.084 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:58:35 smithi164 bash[59462]: audit 1685822314.6493173 mon.c (mon.2) 362 : audit 0 from='client.? 172.21.15.89:0/1073286370' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2023-06-03T19:58:36.483 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 19:58:36 smithi089 bash[71163]: ::ffff:172.21.15.164 - - [03/Jun/2023:19:58:36] "GET /metrics HTTP/1.1" 200 225304 "" "Prometheus/2.33.4" 2023-06-03T19:58:36.829 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:58:36 smithi089 bash[78452]: audit 1685822315.6383665 mon.a (mon.0) 636 : audit 0 from='client.? 172.21.15.89:0/1199552170' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2023-06-03T19:58:36.829 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:58:36 smithi089 bash[75068]: audit 1685822315.6383665 mon.a (mon.0) 636 : audit 0 from='client.? 172.21.15.89:0/1199552170' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2023-06-03T19:58:37.084 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:58:36 smithi164 bash[59462]: audit 1685822315.6383665 mon.a (mon.0) 636 : audit 0 from='client.? 172.21.15.89:0/1199552170' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2023-06-03T19:58:37.107 INFO:teuthology.orchestra.run.smithi089.stdout:442381631569 2023-06-03T19:58:37.261 INFO:teuthology.orchestra.run.smithi089.stdout:532575944719 2023-06-03T19:58:37.480 INFO:tasks.cephadm.ceph_manager.ceph:need seq 154618823000 got 154618823005 for osd.5 2023-06-03T19:58:37.481 DEBUG:teuthology.parallel:result is None 2023-06-03T19:58:37.731 INFO:tasks.cephadm.ceph_manager.ceph:need seq 369367187590 got 369367187595 for osd.0 2023-06-03T19:58:37.732 DEBUG:teuthology.parallel:result is None 2023-06-03T19:58:37.737 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:58:37 smithi089 bash[75068]: cluster 1685822316.385106 mgr.y (mgr.34104) 665 : cluster 0 pgmap v573: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-06-03T19:58:37.737 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:58:37 smithi089 bash[75068]: audit 1685822317.09081 mon.c (mon.2) 363 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T19:58:37.738 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:58:37 smithi089 bash[75068]: audit 1685822317.097744 mon.a (mon.0) 637 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:58:37.738 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:58:37 smithi089 bash[75068]: audit 1685822317.1030664 mon.a (mon.0) 638 : audit 0 from='client.? 172.21.15.89:0/575635686' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2023-06-03T19:58:37.738 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:58:37 smithi089 bash[75068]: audit 1685822317.113977 mon.a (mon.0) 639 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:58:37.738 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:58:37 smithi089 bash[75068]: audit 1685822317.2567573 mon.a (mon.0) 640 : audit 0 from='client.? 172.21.15.89:0/1138082717' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2023-06-03T19:58:37.738 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:58:37 smithi089 bash[78452]: cluster 1685822316.385106 mgr.y (mgr.34104) 665 : cluster 0 pgmap v573: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-06-03T19:58:37.738 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:58:37 smithi089 bash[78452]: audit 1685822317.09081 mon.c (mon.2) 363 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T19:58:37.739 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:58:37 smithi089 bash[78452]: audit 1685822317.097744 mon.a (mon.0) 637 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:58:37.739 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:58:37 smithi089 bash[78452]: audit 1685822317.1030664 mon.a (mon.0) 638 : audit 0 from='client.? 172.21.15.89:0/575635686' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2023-06-03T19:58:37.739 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:58:37 smithi089 bash[78452]: audit 1685822317.113977 mon.a (mon.0) 639 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:58:37.739 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:58:37 smithi089 bash[78452]: audit 1685822317.2567573 mon.a (mon.0) 640 : audit 0 from='client.? 172.21.15.89:0/1138082717' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2023-06-03T19:58:38.084 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:58:37 smithi164 bash[59462]: cluster 1685822316.385106 mgr.y (mgr.34104) 665 : cluster 0 pgmap v573: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-06-03T19:58:38.085 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:58:37 smithi164 bash[59462]: audit 1685822317.09081 mon.c (mon.2) 363 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T19:58:38.085 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:58:37 smithi164 bash[59462]: audit 1685822317.097744 mon.a (mon.0) 637 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:58:38.085 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:58:37 smithi164 bash[59462]: audit 1685822317.1030664 mon.a (mon.0) 638 : audit 0 from='client.? 172.21.15.89:0/575635686' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2023-06-03T19:58:38.085 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:58:37 smithi164 bash[59462]: audit 1685822317.113977 mon.a (mon.0) 639 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:58:38.085 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:58:37 smithi164 bash[59462]: audit 1685822317.2567573 mon.a (mon.0) 640 : audit 0 from='client.? 172.21.15.89:0/1138082717' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2023-06-03T19:58:38.716 INFO:tasks.cephadm.ceph_manager.ceph:need seq 442381631564 got 442381631569 for osd.1 2023-06-03T19:58:38.717 DEBUG:teuthology.parallel:result is None 2023-06-03T19:58:38.733 INFO:tasks.cephadm.ceph_manager.ceph:need seq 532575944714 got 532575944719 for osd.4 2023-06-03T19:58:38.734 DEBUG:teuthology.parallel:result is None 2023-06-03T19:58:38.734 INFO:tasks.cephadm.ceph_manager.ceph:waiting for clean 2023-06-03T19:58:38.734 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph pg dump --format=json 2023-06-03T19:58:40.029 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:58:39 smithi089 bash[75068]: cluster 1685822318.385999 mgr.y (mgr.34104) 666 : cluster 0 pgmap v574: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-06-03T19:58:40.029 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:58:39 smithi089 bash[78452]: cluster 1685822318.385999 mgr.y (mgr.34104) 666 : cluster 0 pgmap v574: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-06-03T19:58:40.084 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:58:39 smithi164 bash[59462]: cluster 1685822318.385999 mgr.y (mgr.34104) 666 : cluster 0 pgmap v574: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-06-03T19:58:42.029 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:58:41 smithi089 bash[75068]: cluster 1685822320.3866282 mgr.y (mgr.34104) 667 : cluster 0 pgmap v575: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-06-03T19:58:42.029 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:58:41 smithi089 bash[78452]: cluster 1685822320.3866282 mgr.y (mgr.34104) 667 : cluster 0 pgmap v575: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-06-03T19:58:42.084 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:58:41 smithi164 bash[59462]: cluster 1685822320.3866282 mgr.y (mgr.34104) 667 : cluster 0 pgmap v575: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-06-03T19:58:43.302 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/mon.a/config 2023-06-03T19:58:43.779 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:58:43 smithi089 bash[78452]: cluster 1685822322.3875077 mgr.y (mgr.34104) 668 : cluster 0 pgmap v576: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-06-03T19:58:43.779 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:58:43 smithi089 bash[78452]: audit 1685822322.5655725 mon.a (mon.0) 641 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:58:43.779 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:58:43 smithi089 bash[78452]: audit 1685822322.5749393 mon.a (mon.0) 642 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:58:43.779 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:58:43 smithi089 bash[78452]: audit 1685822323.0969138 mon.c (mon.2) 364 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-03T19:58:43.779 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:58:43 smithi089 bash[78452]: audit 1685822323.0988374 mon.c (mon.2) 365 : audit 1 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-03T19:58:43.780 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:58:43 smithi089 bash[78452]: audit 1685822323.1077642 mon.a (mon.0) 643 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:58:43.780 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:58:43 smithi089 bash[75068]: cluster 1685822322.3875077 mgr.y (mgr.34104) 668 : cluster 0 pgmap v576: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-06-03T19:58:43.780 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:58:43 smithi089 bash[75068]: audit 1685822322.5655725 mon.a (mon.0) 641 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:58:43.780 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:58:43 smithi089 bash[75068]: audit 1685822322.5749393 mon.a (mon.0) 642 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:58:43.781 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:58:43 smithi089 bash[75068]: audit 1685822323.0969138 mon.c (mon.2) 364 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-03T19:58:43.781 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:58:43 smithi089 bash[75068]: audit 1685822323.0988374 mon.c (mon.2) 365 : audit 1 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-03T19:58:43.781 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:58:43 smithi089 bash[75068]: audit 1685822323.1077642 mon.a (mon.0) 643 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:58:43.834 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:58:43 smithi164 bash[59462]: cluster 1685822322.3875077 mgr.y (mgr.34104) 668 : cluster 0 pgmap v576: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-06-03T19:58:43.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:58:43 smithi164 bash[59462]: audit 1685822322.5655725 mon.a (mon.0) 641 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:58:43.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:58:43 smithi164 bash[59462]: audit 1685822322.5749393 mon.a (mon.0) 642 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:58:43.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:58:43 smithi164 bash[59462]: audit 1685822323.0969138 mon.c (mon.2) 364 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-03T19:58:43.836 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:58:43 smithi164 bash[59462]: audit 1685822323.0988374 mon.c (mon.2) 365 : audit 1 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-03T19:58:43.836 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:58:43 smithi164 bash[59462]: audit 1685822323.1077642 mon.a (mon.0) 643 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:58:45.834 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:58:45 smithi164 bash[59462]: cluster 1685822324.3881428 mgr.y (mgr.34104) 669 : cluster 0 pgmap v577: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-06-03T19:58:46.029 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:58:45 smithi089 bash[75068]: cluster 1685822324.3881428 mgr.y (mgr.34104) 669 : cluster 0 pgmap v577: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-06-03T19:58:46.029 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:58:45 smithi089 bash[78452]: cluster 1685822324.3881428 mgr.y (mgr.34104) 669 : cluster 0 pgmap v577: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-06-03T19:58:46.528 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 19:58:46 smithi089 bash[71163]: ::ffff:172.21.15.164 - - [03/Jun/2023:19:58:46] "GET /metrics HTTP/1.1" 200 225309 "" "Prometheus/2.33.4" 2023-06-03T19:58:47.251 INFO:teuthology.orchestra.run.smithi089.stdout: 2023-06-03T19:58:47.254 INFO:teuthology.orchestra.run.smithi089.stderr:dumped all 2023-06-03T19:58:47.779 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:58:47 smithi089 bash[75068]: cluster 1685822326.38889 mgr.y (mgr.34104) 670 : cluster 0 pgmap v578: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-06-03T19:58:47.779 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:58:47 smithi089 bash[78452]: cluster 1685822326.38889 mgr.y (mgr.34104) 670 : cluster 0 pgmap v578: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-06-03T19:58:47.834 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:58:47 smithi164 bash[59462]: cluster 1685822326.38889 mgr.y (mgr.34104) 670 : cluster 0 pgmap v578: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-06-03T19:58:48.103 INFO:teuthology.orchestra.run.smithi089.stdout:{"pg_ready":true,"pg_map":{"version":578,"stamp":"2023-06-03T19:58:46.388401+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":595854,"num_objects":24,"num_object_clones":0,"num_object_copies":72,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":24,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":261,"num_write_kb":2446,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":195,"ondisk_log_size":195,"up":219,"acting":219,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":219,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":1315012,"kb_used_data":11460,"kb_used_omap":30,"kb_used_meta":1303457,"kb_avail":748646204,"statfs":{"total":767960285184,"available":766613712896,"internally_reserved":0,"allocated":11735040,"data_stored":7866659,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":31502,"internal_metadata":1334740210},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.004073"},"pg_stats":[{"pgid":"3.1a","version":"60'1","reported_seq":110,"reported_epoch":125,"state":"active+clean","last_fresh":"2023-06-03T19:57:28.467559+0000","last_change":"2023-06-03T19:57:28.467559+0000","last_active":"2023-06-03T19:57:28.467559+0000","last_peered":"2023-06-03T19:57:28.467559+0000","last_clean":"2023-06-03T19:57:28.467559+0000","last_became_active":"2023-06-03T19:57:28.467077+0000","last_became_peered":"2023-06-03T19:57:28.467077+0000","last_unstale":"2023-06-03T19:57:28.467559+0000","last_undegraded":"2023-06-03T19:57:28.467559+0000","last_fullsized":"2023-06-03T19:57:28.467559+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T21:46:02.795576+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":103,"reported_epoch":125,"state":"active+clean","last_fresh":"2023-06-03T19:57:28.470287+0000","last_change":"2023-06-03T19:57:28.470287+0000","last_active":"2023-06-03T19:57:28.470287+0000","last_peered":"2023-06-03T19:57:28.470287+0000","last_clean":"2023-06-03T19:57:28.470287+0000","last_became_active":"2023-06-03T19:57:28.469477+0000","last_became_peered":"2023-06-03T19:57:28.469477+0000","last_unstale":"2023-06-03T19:57:28.470287+0000","last_undegraded":"2023-06-03T19:57:28.470287+0000","last_fullsized":"2023-06-03T19:57:28.470287+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T06:48:19.229190+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":116,"reported_epoch":125,"state":"active+clean","last_fresh":"2023-06-03T19:57:28.473350+0000","last_change":"2023-06-03T19:57:28.473350+0000","last_active":"2023-06-03T19:57:28.473350+0000","last_peered":"2023-06-03T19:57:28.473350+0000","last_clean":"2023-06-03T19:57:28.473350+0000","last_became_active":"2023-06-03T19:57:28.472757+0000","last_became_peered":"2023-06-03T19:57:28.472757+0000","last_unstale":"2023-06-03T19:57:28.473350+0000","last_undegraded":"2023-06-03T19:57:28.473350+0000","last_fullsized":"2023-06-03T19:57:28.473350+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T19:58:10.432188+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":80,"reported_epoch":124,"state":"active+clean","last_fresh":"2023-06-03T19:57:27.441485+0000","last_change":"2023-06-03T19:56:54.507653+0000","last_active":"2023-06-03T19:57:27.441485+0000","last_peered":"2023-06-03T19:57:27.441485+0000","last_clean":"2023-06-03T19:57:27.441485+0000","last_became_active":"2023-06-03T19:56:54.507350+0000","last_became_peered":"2023-06-03T19:56:54.507350+0000","last_unstale":"2023-06-03T19:57:27.441485+0000","last_undegraded":"2023-06-03T19:57:27.441485+0000","last_fullsized":"2023-06-03T19:57:27.441485+0000","mapping_epoch":119,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T22:33:20.065844+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":104,"reported_epoch":124,"state":"active+clean","last_fresh":"2023-06-03T19:57:27.446904+0000","last_change":"2023-06-03T19:56:54.512932+0000","last_active":"2023-06-03T19:57:27.446904+0000","last_peered":"2023-06-03T19:57:27.446904+0000","last_clean":"2023-06-03T19:57:27.446904+0000","last_became_active":"2023-06-03T19:56:54.511992+0000","last_became_peered":"2023-06-03T19:56:54.511992+0000","last_unstale":"2023-06-03T19:57:27.446904+0000","last_undegraded":"2023-06-03T19:57:27.446904+0000","last_fullsized":"2023-06-03T19:57:27.446904+0000","mapping_epoch":119,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T21:01:35.069911+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":91,"reported_epoch":124,"state":"active+clean","last_fresh":"2023-06-03T19:57:27.437336+0000","last_change":"2023-06-03T19:51:59.359522+0000","last_active":"2023-06-03T19:57:27.437336+0000","last_peered":"2023-06-03T19:57:27.437336+0000","last_clean":"2023-06-03T19:57:27.437336+0000","last_became_active":"2023-06-03T19:51:59.359202+0000","last_became_peered":"2023-06-03T19:51:59.359202+0000","last_unstale":"2023-06-03T19:57:27.437336+0000","last_undegraded":"2023-06-03T19:57:27.437336+0000","last_fullsized":"2023-06-03T19:57:27.437336+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":78,"reported_epoch":124,"state":"active+clean","last_fresh":"2023-06-03T19:57:27.441034+0000","last_change":"2023-06-03T19:56:54.509490+0000","last_active":"2023-06-03T19:57:27.441034+0000","last_peered":"2023-06-03T19:57:27.441034+0000","last_clean":"2023-06-03T19:57:27.441034+0000","last_became_active":"2023-06-03T19:56:54.509145+0000","last_became_peered":"2023-06-03T19:56:54.509145+0000","last_unstale":"2023-06-03T19:57:27.441034+0000","last_undegraded":"2023-06-03T19:57:27.441034+0000","last_fullsized":"2023-06-03T19:57:27.441034+0000","mapping_epoch":119,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T07:12:03.964110+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":88,"reported_epoch":124,"state":"active+clean","last_fresh":"2023-06-03T19:57:27.446812+0000","last_change":"2023-06-03T19:56:54.511325+0000","last_active":"2023-06-03T19:57:27.446812+0000","last_peered":"2023-06-03T19:57:27.446812+0000","last_clean":"2023-06-03T19:57:27.446812+0000","last_became_active":"2023-06-03T19:56:54.511033+0000","last_became_peered":"2023-06-03T19:56:54.511033+0000","last_unstale":"2023-06-03T19:57:27.446812+0000","last_undegraded":"2023-06-03T19:57:27.446812+0000","last_fullsized":"2023-06-03T19:57:27.446812+0000","mapping_epoch":119,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T23:50:08.711478+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":92,"reported_epoch":124,"state":"active+clean","last_fresh":"2023-06-03T19:57:27.439764+0000","last_change":"2023-06-03T19:51:59.374143+0000","last_active":"2023-06-03T19:57:27.439764+0000","last_peered":"2023-06-03T19:57:27.439764+0000","last_clean":"2023-06-03T19:57:27.439764+0000","last_became_active":"2023-06-03T19:51:59.373146+0000","last_became_peered":"2023-06-03T19:51:59.373146+0000","last_unstale":"2023-06-03T19:57:27.439764+0000","last_undegraded":"2023-06-03T19:57:27.439764+0000","last_fullsized":"2023-06-03T19:57:27.439764+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":69,"reported_epoch":124,"state":"active+clean","last_fresh":"2023-06-03T19:57:27.449274+0000","last_change":"2023-06-03T19:56:54.510462+0000","last_active":"2023-06-03T19:57:27.449274+0000","last_peered":"2023-06-03T19:57:27.449274+0000","last_clean":"2023-06-03T19:57:27.449274+0000","last_became_active":"2023-06-03T19:56:54.509783+0000","last_became_peered":"2023-06-03T19:56:54.509783+0000","last_unstale":"2023-06-03T19:57:27.449274+0000","last_undegraded":"2023-06-03T19:57:27.449274+0000","last_fullsized":"2023-06-03T19:57:27.449274+0000","mapping_epoch":119,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T00:16:09.624276+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"60'3","reported_seq":110,"reported_epoch":125,"state":"active+clean","last_fresh":"2023-06-03T19:57:28.466401+0000","last_change":"2023-06-03T19:57:28.466401+0000","last_active":"2023-06-03T19:57:28.466401+0000","last_peered":"2023-06-03T19:57:28.466401+0000","last_clean":"2023-06-03T19:57:28.466401+0000","last_became_active":"2023-06-03T19:57:28.465452+0000","last_became_peered":"2023-06-03T19:57:28.465452+0000","last_unstale":"2023-06-03T19:57:28.466401+0000","last_undegraded":"2023-06-03T19:57:28.466401+0000","last_fullsized":"2023-06-03T19:57:28.466401+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":60,"reported_epoch":124,"state":"active+clean","last_fresh":"2023-06-03T19:57:27.446119+0000","last_change":"2023-06-03T19:56:11.753995+0000","last_active":"2023-06-03T19:57:27.446119+0000","last_peered":"2023-06-03T19:57:27.446119+0000","last_clean":"2023-06-03T19:57:27.446119+0000","last_became_active":"2023-06-03T19:56:11.753676+0000","last_became_peered":"2023-06-03T19:56:11.753676+0000","last_unstale":"2023-06-03T19:57:27.446119+0000","last_undegraded":"2023-06-03T19:57:27.446119+0000","last_fullsized":"2023-06-03T19:57:27.446119+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":115,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T01:00:40.343647+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"77'23","reported_seq":106,"reported_epoch":125,"state":"active+clean","last_fresh":"2023-06-03T19:57:28.462522+0000","last_change":"2023-06-03T19:57:28.462522+0000","last_active":"2023-06-03T19:57:28.462522+0000","last_peered":"2023-06-03T19:57:28.462522+0000","last_clean":"2023-06-03T19:57:28.462522+0000","last_became_active":"2023-06-03T19:57:28.462048+0000","last_became_peered":"2023-06-03T19:57:28.462048+0000","last_unstale":"2023-06-03T19:57:28.462522+0000","last_undegraded":"2023-06-03T19:57:28.462522+0000","last_fullsized":"2023-06-03T19:57:28.462522+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":23,"log_dups_size":0,"ondisk_log_size":23,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T23:23:56.697534+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4716,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":23,"num_write_kb":20,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":83,"reported_epoch":125,"state":"active+clean","last_fresh":"2023-06-03T19:57:28.469676+0000","last_change":"2023-06-03T19:57:28.469676+0000","last_active":"2023-06-03T19:57:28.469676+0000","last_peered":"2023-06-03T19:57:28.469676+0000","last_clean":"2023-06-03T19:57:28.469676+0000","last_became_active":"2023-06-03T19:57:28.469328+0000","last_became_peered":"2023-06-03T19:57:28.469328+0000","last_unstale":"2023-06-03T19:57:28.469676+0000","last_undegraded":"2023-06-03T19:57:28.469676+0000","last_fullsized":"2023-06-03T19:57:28.469676+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T23:25:26.041930+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"60'2","reported_seq":109,"reported_epoch":125,"state":"active+clean","last_fresh":"2023-06-03T19:57:28.467711+0000","last_change":"2023-06-03T19:57:28.467711+0000","last_active":"2023-06-03T19:57:28.467711+0000","last_peered":"2023-06-03T19:57:28.467711+0000","last_clean":"2023-06-03T19:57:28.467711+0000","last_became_active":"2023-06-03T19:57:28.467234+0000","last_became_peered":"2023-06-03T19:57:28.467234+0000","last_unstale":"2023-06-03T19:57:28.467711+0000","last_undegraded":"2023-06-03T19:57:28.467711+0000","last_fullsized":"2023-06-03T19:57:28.467711+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":88,"reported_epoch":124,"state":"active+clean","last_fresh":"2023-06-03T19:57:27.444818+0000","last_change":"2023-06-03T19:56:54.514616+0000","last_active":"2023-06-03T19:57:27.444818+0000","last_peered":"2023-06-03T19:57:27.444818+0000","last_clean":"2023-06-03T19:57:27.444818+0000","last_became_active":"2023-06-03T19:56:54.513918+0000","last_became_peered":"2023-06-03T19:56:54.513918+0000","last_unstale":"2023-06-03T19:57:27.444818+0000","last_undegraded":"2023-06-03T19:57:27.444818+0000","last_fullsized":"2023-06-03T19:57:27.444818+0000","mapping_epoch":119,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T19:36:10.967386+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":79,"reported_epoch":125,"state":"active+clean","last_fresh":"2023-06-03T19:57:28.450526+0000","last_change":"2023-06-03T19:56:54.507479+0000","last_active":"2023-06-03T19:57:28.450526+0000","last_peered":"2023-06-03T19:57:28.450526+0000","last_clean":"2023-06-03T19:57:28.450526+0000","last_became_active":"2023-06-03T19:56:54.507149+0000","last_became_peered":"2023-06-03T19:56:54.507149+0000","last_unstale":"2023-06-03T19:57:28.450526+0000","last_undegraded":"2023-06-03T19:57:28.450526+0000","last_fullsized":"2023-06-03T19:57:28.450526+0000","mapping_epoch":119,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T20:22:36.676317+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":91,"reported_epoch":124,"state":"active+clean","last_fresh":"2023-06-03T19:57:27.442389+0000","last_change":"2023-06-03T19:56:54.512472+0000","last_active":"2023-06-03T19:57:27.442389+0000","last_peered":"2023-06-03T19:57:27.442389+0000","last_clean":"2023-06-03T19:57:27.442389+0000","last_became_active":"2023-06-03T19:56:54.512051+0000","last_became_peered":"2023-06-03T19:56:54.512051+0000","last_unstale":"2023-06-03T19:57:27.442389+0000","last_undegraded":"2023-06-03T19:57:27.442389+0000","last_fullsized":"2023-06-03T19:57:27.442389+0000","mapping_epoch":119,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":92,"reported_epoch":125,"state":"active+clean","last_fresh":"2023-06-03T19:57:28.462780+0000","last_change":"2023-06-03T19:57:28.462780+0000","last_active":"2023-06-03T19:57:28.462780+0000","last_peered":"2023-06-03T19:57:28.462780+0000","last_clean":"2023-06-03T19:57:28.462780+0000","last_became_active":"2023-06-03T19:57:28.462298+0000","last_became_peered":"2023-06-03T19:57:28.462298+0000","last_unstale":"2023-06-03T19:57:28.462780+0000","last_undegraded":"2023-06-03T19:57:28.462780+0000","last_fullsized":"2023-06-03T19:57:28.462780+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":79,"reported_epoch":124,"state":"active+clean","last_fresh":"2023-06-03T19:57:27.443519+0000","last_change":"2023-06-03T19:56:54.510289+0000","last_active":"2023-06-03T19:57:27.443519+0000","last_peered":"2023-06-03T19:57:27.443519+0000","last_clean":"2023-06-03T19:57:27.443519+0000","last_became_active":"2023-06-03T19:56:54.509905+0000","last_became_peered":"2023-06-03T19:56:54.509905+0000","last_unstale":"2023-06-03T19:57:27.443519+0000","last_undegraded":"2023-06-03T19:57:27.443519+0000","last_fullsized":"2023-06-03T19:57:27.443519+0000","mapping_epoch":119,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T02:54:26.792401+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"60'8","reported_seq":100,"reported_epoch":124,"state":"active+clean","last_fresh":"2023-06-03T19:57:27.435852+0000","last_change":"2023-06-03T19:47:06.727778+0000","last_active":"2023-06-03T19:57:27.435852+0000","last_peered":"2023-06-03T19:57:27.435852+0000","last_clean":"2023-06-03T19:57:27.435852+0000","last_became_active":"2023-06-03T19:47:06.727305+0000","last_became_peered":"2023-06-03T19:47:06.727305+0000","last_unstale":"2023-06-03T19:57:27.435852+0000","last_undegraded":"2023-06-03T19:57:27.435852+0000","last_fullsized":"2023-06-03T19:57:27.435852+0000","mapping_epoch":86,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":87,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":88,"reported_epoch":124,"state":"active+clean","last_fresh":"2023-06-03T19:57:27.443855+0000","last_change":"2023-06-03T19:56:54.512994+0000","last_active":"2023-06-03T19:57:27.443855+0000","last_peered":"2023-06-03T19:57:27.443855+0000","last_clean":"2023-06-03T19:57:27.443855+0000","last_became_active":"2023-06-03T19:56:54.512579+0000","last_became_peered":"2023-06-03T19:56:54.512579+0000","last_unstale":"2023-06-03T19:57:27.443855+0000","last_undegraded":"2023-06-03T19:57:27.443855+0000","last_fullsized":"2023-06-03T19:57:27.443855+0000","mapping_epoch":119,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T07:02:45.187961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.f","version":"60'1","reported_seq":108,"reported_epoch":125,"state":"active+clean","last_fresh":"2023-06-03T19:57:28.464482+0000","last_change":"2023-06-03T19:57:28.464482+0000","last_active":"2023-06-03T19:57:28.464482+0000","last_peered":"2023-06-03T19:57:28.464482+0000","last_clean":"2023-06-03T19:57:28.464482+0000","last_became_active":"2023-06-03T19:57:28.464081+0000","last_became_peered":"2023-06-03T19:57:28.464081+0000","last_unstale":"2023-06-03T19:57:28.464482+0000","last_undegraded":"2023-06-03T19:57:28.464482+0000","last_fullsized":"2023-06-03T19:57:28.464482+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":76,"reported_epoch":124,"state":"active+clean","last_fresh":"2023-06-03T19:57:27.441050+0000","last_change":"2023-06-03T19:34:17.848358+0000","last_active":"2023-06-03T19:57:27.441050+0000","last_peered":"2023-06-03T19:57:27.441050+0000","last_clean":"2023-06-03T19:57:27.441050+0000","last_became_active":"2023-06-03T19:34:17.847811+0000","last_became_peered":"2023-06-03T19:34:17.847811+0000","last_unstale":"2023-06-03T19:57:27.441050+0000","last_undegraded":"2023-06-03T19:57:27.441050+0000","last_fullsized":"2023-06-03T19:57:27.441050+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.e","version":"60'2","reported_seq":109,"reported_epoch":125,"state":"active+clean","last_fresh":"2023-06-03T19:57:28.468330+0000","last_change":"2023-06-03T19:57:28.468330+0000","last_active":"2023-06-03T19:57:28.468330+0000","last_peered":"2023-06-03T19:57:28.468330+0000","last_clean":"2023-06-03T19:57:28.468330+0000","last_became_active":"2023-06-03T19:57:28.467823+0000","last_became_peered":"2023-06-03T19:57:28.467823+0000","last_unstale":"2023-06-03T19:57:28.468330+0000","last_undegraded":"2023-06-03T19:57:28.468330+0000","last_fullsized":"2023-06-03T19:57:28.468330+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":121,"reported_epoch":125,"state":"active+clean","last_fresh":"2023-06-03T19:57:28.461449+0000","last_change":"2023-06-03T19:57:28.461449+0000","last_active":"2023-06-03T19:57:28.461449+0000","last_peered":"2023-06-03T19:57:28.461449+0000","last_clean":"2023-06-03T19:57:28.461449+0000","last_became_active":"2023-06-03T19:57:28.460745+0000","last_became_peered":"2023-06-03T19:57:28.460745+0000","last_unstale":"2023-06-03T19:57:28.461449+0000","last_undegraded":"2023-06-03T19:57:28.461449+0000","last_fullsized":"2023-06-03T19:57:28.461449+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T20:41:19.361638+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,3],"acting":[4,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.d","version":"60'1","reported_seq":78,"reported_epoch":124,"state":"active+clean","last_fresh":"2023-06-03T19:57:27.445291+0000","last_change":"2023-06-03T19:34:16.806559+0000","last_active":"2023-06-03T19:57:27.445291+0000","last_peered":"2023-06-03T19:57:27.445291+0000","last_clean":"2023-06-03T19:57:27.445291+0000","last_became_active":"2023-06-03T19:34:16.806149+0000","last_became_peered":"2023-06-03T19:34:16.806149+0000","last_unstale":"2023-06-03T19:57:27.445291+0000","last_undegraded":"2023-06-03T19:57:27.445291+0000","last_fullsized":"2023-06-03T19:57:27.445291+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":91,"reported_epoch":124,"state":"active+clean","last_fresh":"2023-06-03T19:57:27.437525+0000","last_change":"2023-06-03T19:51:59.360650+0000","last_active":"2023-06-03T19:57:27.437525+0000","last_peered":"2023-06-03T19:57:27.437525+0000","last_clean":"2023-06-03T19:57:27.437525+0000","last_became_active":"2023-06-03T19:51:59.360237+0000","last_became_peered":"2023-06-03T19:51:59.360237+0000","last_unstale":"2023-06-03T19:57:27.437525+0000","last_undegraded":"2023-06-03T19:57:27.437525+0000","last_fullsized":"2023-06-03T19:57:27.437525+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":92,"reported_epoch":124,"state":"active+clean","last_fresh":"2023-06-03T19:57:27.440288+0000","last_change":"2023-06-03T19:56:54.507432+0000","last_active":"2023-06-03T19:57:27.440288+0000","last_peered":"2023-06-03T19:57:27.440288+0000","last_clean":"2023-06-03T19:57:27.440288+0000","last_became_active":"2023-06-03T19:56:54.507012+0000","last_became_peered":"2023-06-03T19:56:54.507012+0000","last_unstale":"2023-06-03T19:57:27.440288+0000","last_undegraded":"2023-06-03T19:57:27.440288+0000","last_fullsized":"2023-06-03T19:57:27.440288+0000","mapping_epoch":119,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":107,"reported_epoch":125,"state":"active+clean","last_fresh":"2023-06-03T19:57:28.465221+0000","last_change":"2023-06-03T19:57:28.464153+0000","last_active":"2023-06-03T19:57:28.465221+0000","last_peered":"2023-06-03T19:57:28.465221+0000","last_clean":"2023-06-03T19:57:28.465221+0000","last_became_active":"2023-06-03T19:57:28.463502+0000","last_became_peered":"2023-06-03T19:57:28.463502+0000","last_unstale":"2023-06-03T19:57:28.465221+0000","last_undegraded":"2023-06-03T19:57:28.465221+0000","last_fullsized":"2023-06-03T19:57:28.465221+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T06:11:00.999526+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,4],"acting":[0,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.b","version":"60'2","reported_seq":89,"reported_epoch":125,"state":"active+clean","last_fresh":"2023-06-03T19:57:28.470310+0000","last_change":"2023-06-03T19:57:28.470310+0000","last_active":"2023-06-03T19:57:28.470310+0000","last_peered":"2023-06-03T19:57:28.470310+0000","last_clean":"2023-06-03T19:57:28.470310+0000","last_became_active":"2023-06-03T19:57:28.469887+0000","last_became_peered":"2023-06-03T19:57:28.469887+0000","last_unstale":"2023-06-03T19:57:28.470310+0000","last_undegraded":"2023-06-03T19:57:28.470310+0000","last_fullsized":"2023-06-03T19:57:28.470310+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T21:58:53.387131+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":97,"reported_epoch":125,"state":"active+clean","last_fresh":"2023-06-03T19:57:28.465140+0000","last_change":"2023-06-03T19:57:28.465140+0000","last_active":"2023-06-03T19:57:28.465140+0000","last_peered":"2023-06-03T19:57:28.465140+0000","last_clean":"2023-06-03T19:57:28.465140+0000","last_became_active":"2023-06-03T19:57:28.464629+0000","last_became_peered":"2023-06-03T19:57:28.464629+0000","last_unstale":"2023-06-03T19:57:28.465140+0000","last_undegraded":"2023-06-03T19:57:28.465140+0000","last_fullsized":"2023-06-03T19:57:28.465140+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T21:47:11.619583+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":107,"reported_epoch":125,"state":"active+clean","last_fresh":"2023-06-03T19:57:28.461401+0000","last_change":"2023-06-03T19:57:28.461401+0000","last_active":"2023-06-03T19:57:28.461401+0000","last_peered":"2023-06-03T19:57:28.461401+0000","last_clean":"2023-06-03T19:57:28.461401+0000","last_became_active":"2023-06-03T19:57:28.461057+0000","last_became_peered":"2023-06-03T19:57:28.461057+0000","last_unstale":"2023-06-03T19:57:28.461401+0000","last_undegraded":"2023-06-03T19:57:28.461401+0000","last_fullsized":"2023-06-03T19:57:28.461401+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":102,"reported_epoch":125,"state":"active+clean","last_fresh":"2023-06-03T19:57:28.456750+0000","last_change":"2023-06-03T19:57:28.456750+0000","last_active":"2023-06-03T19:57:28.456750+0000","last_peered":"2023-06-03T19:57:28.456750+0000","last_clean":"2023-06-03T19:57:28.456750+0000","last_became_active":"2023-06-03T19:57:28.456436+0000","last_became_peered":"2023-06-03T19:57:28.456436+0000","last_unstale":"2023-06-03T19:57:28.456750+0000","last_undegraded":"2023-06-03T19:57:28.456750+0000","last_fullsized":"2023-06-03T19:57:28.456750+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T01:49:25.481554+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,5],"acting":[4,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":94,"reported_epoch":125,"state":"active+clean","last_fresh":"2023-06-03T19:57:28.462344+0000","last_change":"2023-06-03T19:57:28.462344+0000","last_active":"2023-06-03T19:57:28.462344+0000","last_peered":"2023-06-03T19:57:28.462344+0000","last_clean":"2023-06-03T19:57:28.462344+0000","last_became_active":"2023-06-03T19:57:28.462001+0000","last_became_peered":"2023-06-03T19:57:28.462001+0000","last_unstale":"2023-06-03T19:57:28.462344+0000","last_undegraded":"2023-06-03T19:57:28.462344+0000","last_fullsized":"2023-06-03T19:57:28.462344+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T00:45:13.789650+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":83,"reported_epoch":125,"state":"active+clean","last_fresh":"2023-06-03T19:57:28.467917+0000","last_change":"2023-06-03T19:57:28.467917+0000","last_active":"2023-06-03T19:57:28.467917+0000","last_peered":"2023-06-03T19:57:28.467917+0000","last_clean":"2023-06-03T19:57:28.467917+0000","last_became_active":"2023-06-03T19:57:28.467617+0000","last_became_peered":"2023-06-03T19:57:28.467617+0000","last_unstale":"2023-06-03T19:57:28.467917+0000","last_undegraded":"2023-06-03T19:57:28.467917+0000","last_fullsized":"2023-06-03T19:57:28.467917+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T23:34:23.840582+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":89,"reported_epoch":124,"state":"active+clean","last_fresh":"2023-06-03T19:57:27.447053+0000","last_change":"2023-06-03T19:56:54.512219+0000","last_active":"2023-06-03T19:57:27.447053+0000","last_peered":"2023-06-03T19:57:27.447053+0000","last_clean":"2023-06-03T19:57:27.447053+0000","last_became_active":"2023-06-03T19:56:54.511771+0000","last_became_peered":"2023-06-03T19:56:54.511771+0000","last_unstale":"2023-06-03T19:57:27.447053+0000","last_undegraded":"2023-06-03T19:57:27.447053+0000","last_fullsized":"2023-06-03T19:57:27.447053+0000","mapping_epoch":119,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T19:35:47.796520+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":76,"reported_epoch":125,"state":"active+clean","last_fresh":"2023-06-03T19:57:28.472005+0000","last_change":"2023-06-03T19:57:28.471023+0000","last_active":"2023-06-03T19:57:28.472005+0000","last_peered":"2023-06-03T19:57:28.472005+0000","last_clean":"2023-06-03T19:57:28.472005+0000","last_became_active":"2023-06-03T19:57:28.470452+0000","last_became_peered":"2023-06-03T19:57:28.470452+0000","last_unstale":"2023-06-03T19:57:28.472005+0000","last_undegraded":"2023-06-03T19:57:28.472005+0000","last_fullsized":"2023-06-03T19:57:28.472005+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T05:58:33.305030+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":94,"reported_epoch":125,"state":"active+clean","last_fresh":"2023-06-03T19:57:28.462051+0000","last_change":"2023-06-03T19:51:59.392796+0000","last_active":"2023-06-03T19:57:28.462051+0000","last_peered":"2023-06-03T19:57:28.462051+0000","last_clean":"2023-06-03T19:57:28.462051+0000","last_became_active":"2023-06-03T19:51:59.380207+0000","last_became_peered":"2023-06-03T19:51:59.380207+0000","last_unstale":"2023-06-03T19:57:28.462051+0000","last_undegraded":"2023-06-03T19:57:28.462051+0000","last_fullsized":"2023-06-03T19:57:28.462051+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:32:20.692043+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:32:20.692043+0000","last_clean_scrub_stamp":"2023-06-03T19:32:20.692043+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T00:28:05.866873+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":107,"reported_epoch":124,"state":"active+clean","last_fresh":"2023-06-03T19:57:27.442422+0000","last_change":"2023-06-03T19:56:54.512607+0000","last_active":"2023-06-03T19:57:27.442422+0000","last_peered":"2023-06-03T19:57:27.442422+0000","last_clean":"2023-06-03T19:57:27.442422+0000","last_became_active":"2023-06-03T19:56:54.512282+0000","last_became_peered":"2023-06-03T19:56:54.512282+0000","last_unstale":"2023-06-03T19:57:27.442422+0000","last_undegraded":"2023-06-03T19:57:27.442422+0000","last_fullsized":"2023-06-03T19:57:27.442422+0000","mapping_epoch":119,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":61,"reported_epoch":125,"state":"active+clean","last_fresh":"2023-06-03T19:57:28.471196+0000","last_change":"2023-06-03T19:57:28.470295+0000","last_active":"2023-06-03T19:57:28.471196+0000","last_peered":"2023-06-03T19:57:28.471196+0000","last_clean":"2023-06-03T19:57:28.471196+0000","last_became_active":"2023-06-03T19:57:28.469383+0000","last_became_peered":"2023-06-03T19:57:28.469383+0000","last_unstale":"2023-06-03T19:57:28.471196+0000","last_undegraded":"2023-06-03T19:57:28.471196+0000","last_fullsized":"2023-06-03T19:57:28.471196+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T05:35:19.667447+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":96,"reported_epoch":124,"state":"active+clean","last_fresh":"2023-06-03T19:57:27.441028+0000","last_change":"2023-06-03T19:51:59.373777+0000","last_active":"2023-06-03T19:57:27.441028+0000","last_peered":"2023-06-03T19:57:27.441028+0000","last_clean":"2023-06-03T19:57:27.441028+0000","last_became_active":"2023-06-03T19:51:59.372988+0000","last_became_peered":"2023-06-03T19:51:59.372988+0000","last_unstale":"2023-06-03T19:57:27.441028+0000","last_undegraded":"2023-06-03T19:57:27.441028+0000","last_fullsized":"2023-06-03T19:57:27.441028+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:32:20.692043+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:32:20.692043+0000","last_clean_scrub_stamp":"2023-06-03T19:32:20.692043+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":67,"reported_epoch":125,"state":"active+clean","last_fresh":"2023-06-03T19:57:28.460195+0000","last_change":"2023-06-03T19:57:28.460195+0000","last_active":"2023-06-03T19:57:28.460195+0000","last_peered":"2023-06-03T19:57:28.460195+0000","last_clean":"2023-06-03T19:57:28.460195+0000","last_became_active":"2023-06-03T19:57:28.459823+0000","last_became_peered":"2023-06-03T19:57:28.459823+0000","last_unstale":"2023-06-03T19:57:28.460195+0000","last_undegraded":"2023-06-03T19:57:28.460195+0000","last_fullsized":"2023-06-03T19:57:28.460195+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T20:08:48.686380+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":76,"reported_epoch":125,"state":"active+clean","last_fresh":"2023-06-03T19:57:28.462134+0000","last_change":"2023-06-03T19:56:54.516859+0000","last_active":"2023-06-03T19:57:28.462134+0000","last_peered":"2023-06-03T19:57:28.462134+0000","last_clean":"2023-06-03T19:57:28.462134+0000","last_became_active":"2023-06-03T19:56:54.516521+0000","last_became_peered":"2023-06-03T19:56:54.516521+0000","last_unstale":"2023-06-03T19:57:28.462134+0000","last_undegraded":"2023-06-03T19:57:28.462134+0000","last_fullsized":"2023-06-03T19:57:28.462134+0000","mapping_epoch":119,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T20:03:25.227382+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":97,"reported_epoch":125,"state":"active+clean","last_fresh":"2023-06-03T19:57:28.449619+0000","last_change":"2023-06-03T19:56:54.504859+0000","last_active":"2023-06-03T19:57:28.449619+0000","last_peered":"2023-06-03T19:57:28.449619+0000","last_clean":"2023-06-03T19:57:28.449619+0000","last_became_active":"2023-06-03T19:56:54.504455+0000","last_became_peered":"2023-06-03T19:56:54.504455+0000","last_unstale":"2023-06-03T19:57:28.449619+0000","last_undegraded":"2023-06-03T19:57:28.449619+0000","last_fullsized":"2023-06-03T19:57:28.449619+0000","mapping_epoch":119,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:32:20.692043+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:32:20.692043+0000","last_clean_scrub_stamp":"2023-06-03T19:32:20.692043+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T06:25:45.522100+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":102,"reported_epoch":125,"state":"active+clean","last_fresh":"2023-06-03T19:57:28.458987+0000","last_change":"2023-06-03T19:56:11.754048+0000","last_active":"2023-06-03T19:57:28.458987+0000","last_peered":"2023-06-03T19:57:28.458987+0000","last_clean":"2023-06-03T19:57:28.458987+0000","last_became_active":"2023-06-03T19:56:11.753687+0000","last_became_peered":"2023-06-03T19:56:11.753687+0000","last_unstale":"2023-06-03T19:57:28.458987+0000","last_undegraded":"2023-06-03T19:57:28.458987+0000","last_fullsized":"2023-06-03T19:57:28.458987+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":115,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T23:29:41.511496+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":89,"reported_epoch":125,"state":"active+clean","last_fresh":"2023-06-03T19:57:28.459616+0000","last_change":"2023-06-03T19:51:59.396573+0000","last_active":"2023-06-03T19:57:28.459616+0000","last_peered":"2023-06-03T19:57:28.459616+0000","last_clean":"2023-06-03T19:57:28.459616+0000","last_became_active":"2023-06-03T19:51:59.381593+0000","last_became_peered":"2023-06-03T19:51:59.381593+0000","last_unstale":"2023-06-03T19:57:28.459616+0000","last_undegraded":"2023-06-03T19:57:28.459616+0000","last_fullsized":"2023-06-03T19:57:28.459616+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T21:33:23.043772+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":96,"reported_epoch":124,"state":"active+clean","last_fresh":"2023-06-03T19:57:27.439804+0000","last_change":"2023-06-03T19:56:11.752151+0000","last_active":"2023-06-03T19:57:27.439804+0000","last_peered":"2023-06-03T19:57:27.439804+0000","last_clean":"2023-06-03T19:57:27.439804+0000","last_became_active":"2023-06-03T19:56:11.751767+0000","last_became_peered":"2023-06-03T19:56:11.751767+0000","last_unstale":"2023-06-03T19:57:27.439804+0000","last_undegraded":"2023-06-03T19:57:27.439804+0000","last_fullsized":"2023-06-03T19:57:27.439804+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":115,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:32:20.692043+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:32:20.692043+0000","last_clean_scrub_stamp":"2023-06-03T19:32:20.692043+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"72'139","reported_seq":328,"reported_epoch":124,"state":"active+clean","last_fresh":"2023-06-03T19:57:27.443270+0000","last_change":"2023-06-03T19:47:06.731872+0000","last_active":"2023-06-03T19:57:27.443270+0000","last_peered":"2023-06-03T19:57:27.443270+0000","last_clean":"2023-06-03T19:57:27.443270+0000","last_became_active":"2023-06-03T19:47:06.731175+0000","last_became_peered":"2023-06-03T19:47:06.731175+0000","last_unstale":"2023-06-03T19:57:27.443270+0000","last_undegraded":"2023-06-03T19:57:27.443270+0000","last_fullsized":"2023-06-03T19:57:27.443270+0000","mapping_epoch":86,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":87,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:27:28.100808+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:27:28.100808+0000","last_clean_scrub_stamp":"2023-06-03T19:27:28.100808+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":69,"reported_epoch":124,"state":"active+clean","last_fresh":"2023-06-03T19:57:27.443994+0000","last_change":"2023-06-03T19:56:54.509571+0000","last_active":"2023-06-03T19:57:27.443994+0000","last_peered":"2023-06-03T19:57:27.443994+0000","last_clean":"2023-06-03T19:57:27.443994+0000","last_became_active":"2023-06-03T19:56:54.509213+0000","last_became_peered":"2023-06-03T19:56:54.509213+0000","last_unstale":"2023-06-03T19:57:27.443994+0000","last_undegraded":"2023-06-03T19:57:27.443994+0000","last_fullsized":"2023-06-03T19:57:27.443994+0000","mapping_epoch":119,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T21:26:56.003203+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":91,"reported_epoch":125,"state":"active+clean","last_fresh":"2023-06-03T19:57:28.459664+0000","last_change":"2023-06-03T19:57:28.459664+0000","last_active":"2023-06-03T19:57:28.459664+0000","last_peered":"2023-06-03T19:57:28.459664+0000","last_clean":"2023-06-03T19:57:28.459664+0000","last_became_active":"2023-06-03T19:57:28.459289+0000","last_became_peered":"2023-06-03T19:57:28.459289+0000","last_unstale":"2023-06-03T19:57:28.459664+0000","last_undegraded":"2023-06-03T19:57:28.459664+0000","last_fullsized":"2023-06-03T19:57:28.459664+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":111,"reported_epoch":124,"state":"active+clean","last_fresh":"2023-06-03T19:57:27.443526+0000","last_change":"2023-06-03T19:51:59.358157+0000","last_active":"2023-06-03T19:57:27.443526+0000","last_peered":"2023-06-03T19:57:27.443526+0000","last_clean":"2023-06-03T19:57:27.443526+0000","last_became_active":"2023-06-03T19:51:59.357363+0000","last_became_peered":"2023-06-03T19:51:59.357363+0000","last_unstale":"2023-06-03T19:57:27.443526+0000","last_undegraded":"2023-06-03T19:57:27.443526+0000","last_fullsized":"2023-06-03T19:57:27.443526+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:32:20.692043+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:32:20.692043+0000","last_clean_scrub_stamp":"2023-06-03T19:32:20.692043+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":94,"reported_epoch":125,"state":"active+clean","last_fresh":"2023-06-03T19:57:28.459955+0000","last_change":"2023-06-03T19:57:28.459611+0000","last_active":"2023-06-03T19:57:28.459955+0000","last_peered":"2023-06-03T19:57:28.459955+0000","last_clean":"2023-06-03T19:57:28.459955+0000","last_became_active":"2023-06-03T19:57:28.459264+0000","last_became_peered":"2023-06-03T19:57:28.459264+0000","last_unstale":"2023-06-03T19:57:28.459955+0000","last_undegraded":"2023-06-03T19:57:28.459955+0000","last_fullsized":"2023-06-03T19:57:28.459955+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T22:15:06.557231+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":95,"reported_epoch":125,"state":"active+clean","last_fresh":"2023-06-03T19:57:28.451064+0000","last_change":"2023-06-03T19:56:11.745678+0000","last_active":"2023-06-03T19:57:28.451064+0000","last_peered":"2023-06-03T19:57:28.451064+0000","last_clean":"2023-06-03T19:57:28.451064+0000","last_became_active":"2023-06-03T19:56:11.744966+0000","last_became_peered":"2023-06-03T19:56:11.744966+0000","last_unstale":"2023-06-03T19:57:28.451064+0000","last_undegraded":"2023-06-03T19:57:28.451064+0000","last_fullsized":"2023-06-03T19:57:28.451064+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":115,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T21:12:06.830295+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":111,"reported_epoch":125,"state":"active+clean","last_fresh":"2023-06-03T19:57:28.466096+0000","last_change":"2023-06-03T19:57:28.466096+0000","last_active":"2023-06-03T19:57:28.466096+0000","last_peered":"2023-06-03T19:57:28.466096+0000","last_clean":"2023-06-03T19:57:28.466096+0000","last_became_active":"2023-06-03T19:57:28.464954+0000","last_became_peered":"2023-06-03T19:57:28.464954+0000","last_unstale":"2023-06-03T19:57:28.466096+0000","last_undegraded":"2023-06-03T19:57:28.466096+0000","last_fullsized":"2023-06-03T19:57:28.466096+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:32:20.692043+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:32:20.692043+0000","last_clean_scrub_stamp":"2023-06-03T19:32:20.692043+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":102,"reported_epoch":125,"state":"active+clean","last_fresh":"2023-06-03T19:57:28.462221+0000","last_change":"2023-06-03T19:56:11.750241+0000","last_active":"2023-06-03T19:57:28.462221+0000","last_peered":"2023-06-03T19:57:28.462221+0000","last_clean":"2023-06-03T19:57:28.462221+0000","last_became_active":"2023-06-03T19:56:11.749687+0000","last_became_peered":"2023-06-03T19:56:11.749687+0000","last_unstale":"2023-06-03T19:57:28.462221+0000","last_undegraded":"2023-06-03T19:57:28.462221+0000","last_fullsized":"2023-06-03T19:57:28.462221+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":115,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T22:38:25.819929+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":65,"reported_epoch":125,"state":"active+clean","last_fresh":"2023-06-03T19:57:28.458317+0000","last_change":"2023-06-03T19:47:06.733532+0000","last_active":"2023-06-03T19:57:28.458317+0000","last_peered":"2023-06-03T19:57:28.458317+0000","last_clean":"2023-06-03T19:57:28.458317+0000","last_became_active":"2023-06-03T19:47:06.733131+0000","last_became_peered":"2023-06-03T19:47:06.733131+0000","last_unstale":"2023-06-03T19:57:28.458317+0000","last_undegraded":"2023-06-03T19:57:28.458317+0000","last_fullsized":"2023-06-03T19:57:28.458317+0000","mapping_epoch":86,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":87,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T23:46:03.788646+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.7","version":"60'2","reported_seq":74,"reported_epoch":124,"state":"active+clean","last_fresh":"2023-06-03T19:57:27.444053+0000","last_change":"2023-06-03T19:56:54.510148+0000","last_active":"2023-06-03T19:57:27.444053+0000","last_peered":"2023-06-03T19:57:27.444053+0000","last_clean":"2023-06-03T19:57:27.444053+0000","last_became_active":"2023-06-03T19:56:54.509771+0000","last_became_peered":"2023-06-03T19:56:54.509771+0000","last_unstale":"2023-06-03T19:57:27.444053+0000","last_undegraded":"2023-06-03T19:57:27.444053+0000","last_fullsized":"2023-06-03T19:57:27.444053+0000","mapping_epoch":119,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T00:52:42.807432+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":102,"reported_epoch":125,"state":"active+clean","last_fresh":"2023-06-03T19:57:28.472011+0000","last_change":"2023-06-03T19:57:28.470954+0000","last_active":"2023-06-03T19:57:28.472011+0000","last_peered":"2023-06-03T19:57:28.472011+0000","last_clean":"2023-06-03T19:57:28.472011+0000","last_became_active":"2023-06-03T19:57:28.470524+0000","last_became_peered":"2023-06-03T19:57:28.470524+0000","last_unstale":"2023-06-03T19:57:28.472011+0000","last_undegraded":"2023-06-03T19:57:28.472011+0000","last_fullsized":"2023-06-03T19:57:28.472011+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:32:20.692043+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:32:20.692043+0000","last_clean_scrub_stamp":"2023-06-03T19:32:20.692043+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T19:39:19.983271+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":71,"reported_epoch":124,"state":"active+clean","last_fresh":"2023-06-03T19:57:27.446982+0000","last_change":"2023-06-03T19:56:54.507670+0000","last_active":"2023-06-03T19:57:27.446982+0000","last_peered":"2023-06-03T19:57:27.446982+0000","last_clean":"2023-06-03T19:57:27.446982+0000","last_became_active":"2023-06-03T19:56:54.507293+0000","last_became_peered":"2023-06-03T19:56:54.507293+0000","last_unstale":"2023-06-03T19:57:27.446982+0000","last_undegraded":"2023-06-03T19:57:27.446982+0000","last_fullsized":"2023-06-03T19:57:27.446982+0000","mapping_epoch":119,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T01:49:19.966054+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":96,"reported_epoch":124,"state":"active+clean","last_fresh":"2023-06-03T19:57:27.437244+0000","last_change":"2023-06-03T19:56:11.752730+0000","last_active":"2023-06-03T19:57:27.437244+0000","last_peered":"2023-06-03T19:57:27.437244+0000","last_clean":"2023-06-03T19:57:27.437244+0000","last_became_active":"2023-06-03T19:56:11.752441+0000","last_became_peered":"2023-06-03T19:56:11.752441+0000","last_unstale":"2023-06-03T19:57:27.437244+0000","last_undegraded":"2023-06-03T19:57:27.437244+0000","last_fullsized":"2023-06-03T19:57:27.437244+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":115,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:32:20.692043+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:32:20.692043+0000","last_clean_scrub_stamp":"2023-06-03T19:32:20.692043+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"60'1","reported_seq":98,"reported_epoch":125,"state":"active+clean","last_fresh":"2023-06-03T19:57:28.464648+0000","last_change":"2023-06-03T19:57:28.463999+0000","last_active":"2023-06-03T19:57:28.464648+0000","last_peered":"2023-06-03T19:57:28.464648+0000","last_clean":"2023-06-03T19:57:28.464648+0000","last_became_active":"2023-06-03T19:57:28.463588+0000","last_became_peered":"2023-06-03T19:57:28.463588+0000","last_unstale":"2023-06-03T19:57:28.464648+0000","last_undegraded":"2023-06-03T19:57:28.464648+0000","last_fullsized":"2023-06-03T19:57:28.464648+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T21:44:11.969059+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":65,"reported_epoch":125,"state":"active+clean","last_fresh":"2023-06-03T19:57:28.456467+0000","last_change":"2023-06-03T19:57:28.456467+0000","last_active":"2023-06-03T19:57:28.456467+0000","last_peered":"2023-06-03T19:57:28.456467+0000","last_clean":"2023-06-03T19:57:28.456467+0000","last_became_active":"2023-06-03T19:57:28.456120+0000","last_became_peered":"2023-06-03T19:57:28.456120+0000","last_unstale":"2023-06-03T19:57:28.456467+0000","last_undegraded":"2023-06-03T19:57:28.456467+0000","last_fullsized":"2023-06-03T19:57:28.456467+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T07:28:14.306624+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"60'1","reported_seq":82,"reported_epoch":125,"state":"active+clean","last_fresh":"2023-06-03T19:57:28.465124+0000","last_change":"2023-06-03T19:57:28.465124+0000","last_active":"2023-06-03T19:57:28.465124+0000","last_peered":"2023-06-03T19:57:28.465124+0000","last_clean":"2023-06-03T19:57:28.465124+0000","last_became_active":"2023-06-03T19:57:28.464654+0000","last_became_peered":"2023-06-03T19:57:28.464654+0000","last_unstale":"2023-06-03T19:57:28.465124+0000","last_undegraded":"2023-06-03T19:57:28.465124+0000","last_fullsized":"2023-06-03T19:57:28.465124+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T01:12:32.305507+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":109,"reported_epoch":125,"state":"active+clean","last_fresh":"2023-06-03T19:57:28.448170+0000","last_change":"2023-06-03T19:56:54.511760+0000","last_active":"2023-06-03T19:57:28.448170+0000","last_peered":"2023-06-03T19:57:28.448170+0000","last_clean":"2023-06-03T19:57:28.448170+0000","last_became_active":"2023-06-03T19:56:54.511429+0000","last_became_peered":"2023-06-03T19:56:54.511429+0000","last_unstale":"2023-06-03T19:57:28.448170+0000","last_undegraded":"2023-06-03T19:57:28.448170+0000","last_fullsized":"2023-06-03T19:57:28.448170+0000","mapping_epoch":119,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T05:42:46.254343+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"60'2","reported_seq":109,"reported_epoch":125,"state":"active+clean","last_fresh":"2023-06-03T19:57:28.462252+0000","last_change":"2023-06-03T19:57:28.462252+0000","last_active":"2023-06-03T19:57:28.462252+0000","last_peered":"2023-06-03T19:57:28.462252+0000","last_clean":"2023-06-03T19:57:28.462252+0000","last_became_active":"2023-06-03T19:57:28.461844+0000","last_became_peered":"2023-06-03T19:57:28.461844+0000","last_unstale":"2023-06-03T19:57:28.462252+0000","last_undegraded":"2023-06-03T19:57:28.462252+0000","last_fullsized":"2023-06-03T19:57:28.462252+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":112,"reported_epoch":125,"state":"active+clean","last_fresh":"2023-06-03T19:57:28.469248+0000","last_change":"2023-06-03T19:57:28.469248+0000","last_active":"2023-06-03T19:57:28.469248+0000","last_peered":"2023-06-03T19:57:28.469248+0000","last_clean":"2023-06-03T19:57:28.469248+0000","last_became_active":"2023-06-03T19:57:28.468903+0000","last_became_peered":"2023-06-03T19:57:28.468903+0000","last_unstale":"2023-06-03T19:57:28.469248+0000","last_undegraded":"2023-06-03T19:57:28.469248+0000","last_fullsized":"2023-06-03T19:57:28.469248+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T06:08:20.956098+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"60'2","reported_seq":94,"reported_epoch":125,"state":"active+clean","last_fresh":"2023-06-03T19:57:28.462610+0000","last_change":"2023-06-03T19:57:28.462610+0000","last_active":"2023-06-03T19:57:28.462610+0000","last_peered":"2023-06-03T19:57:28.462610+0000","last_clean":"2023-06-03T19:57:28.462610+0000","last_became_active":"2023-06-03T19:57:28.460655+0000","last_became_peered":"2023-06-03T19:57:28.460655+0000","last_unstale":"2023-06-03T19:57:28.462610+0000","last_undegraded":"2023-06-03T19:57:28.462610+0000","last_fullsized":"2023-06-03T19:57:28.462610+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":112,"reported_epoch":125,"state":"active+clean","last_fresh":"2023-06-03T19:57:28.464690+0000","last_change":"2023-06-03T19:57:28.464690+0000","last_active":"2023-06-03T19:57:28.464690+0000","last_peered":"2023-06-03T19:57:28.464690+0000","last_clean":"2023-06-03T19:57:28.464690+0000","last_became_active":"2023-06-03T19:57:28.463856+0000","last_became_peered":"2023-06-03T19:57:28.463856+0000","last_unstale":"2023-06-03T19:57:28.464690+0000","last_undegraded":"2023-06-03T19:57:28.464690+0000","last_fullsized":"2023-06-03T19:57:28.464690+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T00:11:00.991088+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"60'1","reported_seq":100,"reported_epoch":124,"state":"active+clean","last_fresh":"2023-06-03T19:57:27.444017+0000","last_change":"2023-06-03T19:56:54.505096+0000","last_active":"2023-06-03T19:57:27.444017+0000","last_peered":"2023-06-03T19:57:27.444017+0000","last_clean":"2023-06-03T19:57:27.444017+0000","last_became_active":"2023-06-03T19:56:54.504255+0000","last_became_peered":"2023-06-03T19:56:54.504255+0000","last_unstale":"2023-06-03T19:57:27.444017+0000","last_undegraded":"2023-06-03T19:57:27.444017+0000","last_fullsized":"2023-06-03T19:57:27.444017+0000","mapping_epoch":119,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T19:52:25.922308+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":113,"reported_epoch":124,"state":"active+clean","last_fresh":"2023-06-03T19:57:27.449326+0000","last_change":"2023-06-03T19:56:54.503416+0000","last_active":"2023-06-03T19:57:27.449326+0000","last_peered":"2023-06-03T19:57:27.449326+0000","last_clean":"2023-06-03T19:57:27.449326+0000","last_became_active":"2023-06-03T19:56:54.503018+0000","last_became_peered":"2023-06-03T19:56:54.503018+0000","last_unstale":"2023-06-03T19:57:27.449326+0000","last_undegraded":"2023-06-03T19:57:27.449326+0000","last_fullsized":"2023-06-03T19:57:27.449326+0000","mapping_epoch":119,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T20:14:00.330616+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"60'4","reported_seq":83,"reported_epoch":125,"state":"active+clean","last_fresh":"2023-06-03T19:57:28.451241+0000","last_change":"2023-06-03T19:56:11.748116+0000","last_active":"2023-06-03T19:57:28.451241+0000","last_peered":"2023-06-03T19:57:28.451241+0000","last_clean":"2023-06-03T19:57:28.451241+0000","last_became_active":"2023-06-03T19:56:11.747636+0000","last_became_peered":"2023-06-03T19:56:11.747636+0000","last_unstale":"2023-06-03T19:57:28.451241+0000","last_undegraded":"2023-06-03T19:57:28.451241+0000","last_fullsized":"2023-06-03T19:57:28.451241+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":115,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T06:57:48.052055+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":83,"reported_epoch":125,"state":"active+clean","last_fresh":"2023-06-03T19:57:28.459234+0000","last_change":"2023-06-03T19:57:28.459234+0000","last_active":"2023-06-03T19:57:28.459234+0000","last_peered":"2023-06-03T19:57:28.459234+0000","last_clean":"2023-06-03T19:57:28.459234+0000","last_became_active":"2023-06-03T19:57:28.458896+0000","last_became_peered":"2023-06-03T19:57:28.458896+0000","last_unstale":"2023-06-03T19:57:28.459234+0000","last_undegraded":"2023-06-03T19:57:28.459234+0000","last_fullsized":"2023-06-03T19:57:28.459234+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T21:18:37.142280+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5486,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":62,"num_write_kb":30,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16458,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":30319,"internal_metadata":0},"log_size":56,"ondisk_log_size":56,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":3,"num_store_stats":3},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":48,"seq":206158430540,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":287928,"kb_used_data":1776,"kb_used_omap":0,"kb_used_meta":286144,"kb_avail":93457224,"statfs":{"total":95995035648,"available":95700197376,"internally_reserved":0,"allocated":1818624,"data_stored":1348675,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":293011456},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 3 19:58:34 2023","interfaces":[{"interface":"back","average":{"1min":0.86599999999999999,"5min":1.1080000000000001,"15min":1.2729999999999999},"min":{"1min":0.48199999999999998,"5min":0.27500000000000002,"15min":0.253},"max":{"1min":1.3180000000000001,"5min":4.806,"15min":8.6099999999999994},"last":0.73099999999999998},{"interface":"front","average":{"1min":0.91100000000000003,"5min":1.218,"15min":1.506},"min":{"1min":0.57399999999999995,"5min":0.33200000000000002,"15min":0.28499999999999998},"max":{"1min":1.2390000000000001,"5min":6.6139999999999999,"15min":15.220000000000001},"last":0.93700000000000006}]},{"osd":1,"last update":"Sat Jun 3 19:58:12 2023","interfaces":[{"interface":"back","average":{"1min":0.84999999999999998,"5min":1.3879999999999999,"15min":1.8320000000000001},"min":{"1min":0.44400000000000001,"5min":0.315,"15min":0.315},"max":{"1min":1.385,"5min":8.1750000000000007,"15min":10.211},"last":1.0289999999999999},{"interface":"front","average":{"1min":1.0640000000000001,"5min":1.415,"15min":1.623},"min":{"1min":0.75800000000000001,"5min":0.36199999999999999,"15min":0.34799999999999998},"max":{"1min":2.0790000000000002,"5min":8.7360000000000007,"15min":11.635},"last":0.76700000000000002}]},{"osd":2,"last update":"Sat Jun 3 19:58:12 2023","interfaces":[{"interface":"back","average":{"1min":1.1080000000000001,"5min":1.032,"15min":1.0189999999999999},"min":{"1min":0.66100000000000003,"5min":0.66100000000000003,"15min":0.66100000000000003},"max":{"1min":2.9969999999999999,"5min":2.9969999999999999,"15min":2.9969999999999999},"last":1.1240000000000001},{"interface":"front","average":{"1min":1.0880000000000001,"5min":1,"15min":0.98499999999999999},"min":{"1min":0.63100000000000001,"5min":0.63100000000000001,"15min":0.63100000000000001},"max":{"1min":2.0299999999999998,"5min":2.0299999999999998,"15min":2.0299999999999998},"last":1.1619999999999999}]},{"osd":3,"last update":"Sat Jun 3 19:57:56 2023","interfaces":[{"interface":"back","average":{"1min":1.1559999999999999,"5min":1.1559999999999999,"15min":1.1559999999999999},"min":{"1min":0.65400000000000003,"5min":0.65400000000000003,"15min":0.65400000000000003},"max":{"1min":2.9399999999999999,"5min":2.9399999999999999,"15min":2.9399999999999999},"last":0.90800000000000003},{"interface":"front","average":{"1min":1.2,"5min":1.2,"15min":1.2},"min":{"1min":0.72399999999999998,"5min":0.72399999999999998,"15min":0.72399999999999998},"max":{"1min":3.085,"5min":3.085,"15min":3.085},"last":1.0669999999999999}]},{"osd":4,"last update":"Sat Jun 3 19:58:28 2023","interfaces":[{"interface":"back","average":{"1min":1.0509999999999999,"5min":1.0509999999999999,"15min":1.0509999999999999},"min":{"1min":0.66900000000000004,"5min":0.66900000000000004,"15min":0.66900000000000004},"max":{"1min":1.552,"5min":1.552,"15min":1.552},"last":0.82899999999999996},{"interface":"front","average":{"1min":1.119,"5min":1.119,"15min":1.119},"min":{"1min":0.77100000000000002,"5min":0.77100000000000002,"15min":0.77100000000000002},"max":{"1min":1.476,"5min":1.476,"15min":1.476},"last":1.2230000000000001}]},{"osd":5,"last update":"Sat Jun 3 19:58:07 2023","interfaces":[{"interface":"back","average":{"1min":0.97999999999999998,"5min":1.1240000000000001,"15min":1.2270000000000001},"min":{"1min":0.52100000000000002,"5min":0.33200000000000002,"15min":0.30199999999999999},"max":{"1min":1.659,"5min":10.151999999999999,"15min":12.778},"last":0.98299999999999998},{"interface":"front","average":{"1min":1.1140000000000001,"5min":1.1599999999999999,"15min":1.3440000000000001},"min":{"1min":0.622,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":1.5269999999999999,"5min":5.3840000000000003,"15min":9.1669999999999998},"last":1.274}]},{"osd":6,"last update":"Sat Jun 3 19:58:07 2023","interfaces":[{"interface":"back","average":{"1min":1.1000000000000001,"5min":1.2010000000000001,"15min":1.339},"min":{"1min":0.57699999999999996,"5min":0.36499999999999999,"15min":0.35899999999999999},"max":{"1min":1.5860000000000001,"5min":5.4429999999999996,"15min":9.3520000000000003},"last":1.3160000000000001},{"interface":"front","average":{"1min":0.92900000000000005,"5min":1.202,"15min":1.2230000000000001},"min":{"1min":0.57499999999999996,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":1.4910000000000001,"5min":9.0269999999999992,"15min":9.0269999999999992},"last":1.1000000000000001}]}]},{"osd":6,"up_from":42,"seq":180388626773,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":270504,"kb_used_data":1760,"kb_used_omap":0,"kb_used_meta":268736,"kb_avail":93474648,"statfs":{"total":95995035648,"available":95718039552,"internally_reserved":0,"allocated":1802240,"data_stored":1348214,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":275185664},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 3 19:58:35 2023","interfaces":[{"interface":"back","average":{"1min":0.98499999999999999,"5min":1.5009999999999999,"15min":1.5629999999999999},"min":{"1min":0.70199999999999996,"5min":0.28199999999999997,"15min":0.27500000000000002},"max":{"1min":1.3320000000000001,"5min":8.1890000000000001,"15min":11.944000000000001},"last":0.746},{"interface":"front","average":{"1min":0.94699999999999995,"5min":1.6739999999999999,"15min":1.659},"min":{"1min":0.56100000000000005,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":1.1259999999999999,"5min":10.835000000000001,"15min":12.502000000000001},"last":0.97299999999999998}]},{"osd":1,"last update":"Sat Jun 3 19:58:10 2023","interfaces":[{"interface":"back","average":{"1min":0.94099999999999995,"5min":1.393,"15min":1.589},"min":{"1min":0.57699999999999996,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":1.5720000000000001,"5min":6.8049999999999997,"15min":6.8049999999999997},"last":0.93200000000000005},{"interface":"front","average":{"1min":0.97699999999999998,"5min":1.5649999999999999,"15min":1.2210000000000001},"min":{"1min":0.74399999999999999,"5min":0.39100000000000001,"15min":0.38500000000000001},"max":{"1min":1.3240000000000001,"5min":8.9120000000000008,"15min":8.9120000000000008},"last":1.097}]},{"osd":2,"last update":"Sat Jun 3 19:58:12 2023","interfaces":[{"interface":"back","average":{"1min":0.89200000000000002,"5min":0.84999999999999998,"15min":0.84299999999999997},"min":{"1min":0.57399999999999995,"5min":0.57399999999999995,"15min":0.57399999999999995},"max":{"1min":1.4490000000000001,"5min":1.4490000000000001,"15min":1.4490000000000001},"last":1.02},{"interface":"front","average":{"1min":0.96699999999999997,"5min":0.94699999999999995,"15min":0.94399999999999995},"min":{"1min":0.52700000000000002,"5min":0.52700000000000002,"15min":0.52700000000000002},"max":{"1min":1.2989999999999999,"5min":1.5469999999999999,"15min":1.5469999999999999},"last":1.1739999999999999}]},{"osd":3,"last update":"Sat Jun 3 19:57:59 2023","interfaces":[{"interface":"back","average":{"1min":0.85399999999999998,"5min":0.85399999999999998,"15min":0.85399999999999998},"min":{"1min":0.623,"5min":0.623,"15min":0.623},"max":{"1min":1.1819999999999999,"5min":1.1819999999999999,"15min":1.1819999999999999},"last":0.77300000000000002},{"interface":"front","average":{"1min":0.99399999999999999,"5min":0.99399999999999999,"15min":0.99399999999999999},"min":{"1min":0.65300000000000002,"5min":0.65300000000000002,"15min":0.65300000000000002},"max":{"1min":1.4279999999999999,"5min":1.4279999999999999,"15min":1.4279999999999999},"last":1.0600000000000001}]},{"osd":4,"last update":"Sat Jun 3 19:58:29 2023","interfaces":[{"interface":"back","average":{"1min":0.999,"5min":0.999,"15min":0.999},"min":{"1min":0.751,"5min":0.751,"15min":0.751},"max":{"1min":1.1719999999999999,"5min":1.1719999999999999,"15min":1.1719999999999999},"last":1.131},{"interface":"front","average":{"1min":1.181,"5min":1.181,"15min":1.181},"min":{"1min":0.92000000000000004,"5min":0.92000000000000004,"15min":0.92000000000000004},"max":{"1min":1.46,"5min":1.46,"15min":1.46},"last":0.85299999999999998}]},{"osd":5,"last update":"Sat Jun 3 19:58:25 2023","interfaces":[{"interface":"back","average":{"1min":1.1679999999999999,"5min":1.385,"15min":1.2749999999999999},"min":{"1min":0.83299999999999996,"5min":0.34100000000000003,"15min":0.30499999999999999},"max":{"1min":1.393,"5min":11.083,"15min":11.083},"last":1.367},{"interface":"front","average":{"1min":1.089,"5min":1.4099999999999999,"15min":1.3220000000000001},"min":{"1min":0.80000000000000004,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":1.367,"5min":10.154,"15min":10.154},"last":1.234}]},{"osd":7,"last update":"Sat Jun 3 19:58:02 2023","interfaces":[{"interface":"back","average":{"1min":1.002,"5min":1.333,"15min":1.407},"min":{"1min":0.59199999999999997,"5min":0.313,"15min":0.313},"max":{"1min":1.3879999999999999,"5min":6.3029999999999999,"15min":9.5510000000000002},"last":1.29},{"interface":"front","average":{"1min":0.95799999999999996,"5min":1.528,"15min":1.4570000000000001},"min":{"1min":0.52800000000000002,"5min":0.35099999999999998,"15min":0.29999999999999999},"max":{"1min":1.292,"5min":12.407999999999999,"15min":12.407999999999999},"last":1.2649999999999999}]}]},{"osd":1,"up_from":103,"seq":442381631571,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":129700,"kb_used_data":1244,"kb_used_omap":4,"kb_used_meta":128443,"kb_avail":93615452,"statfs":{"total":95995035648,"available":95862222848,"internally_reserved":0,"allocated":1273856,"data_stored":767105,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":131526542},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 3 19:58:10 2023","interfaces":[{"interface":"back","average":{"1min":0.63100000000000001,"5min":1.2470000000000001,"15min":1.861},"min":{"1min":0.29299999999999998,"5min":0.187,"15min":0.187},"max":{"1min":1.0940000000000001,"5min":10.653,"15min":10.653},"last":0.629},{"interface":"front","average":{"1min":0.72999999999999998,"5min":1.6220000000000001,"15min":1.9239999999999999},"min":{"1min":0.39600000000000002,"5min":0.192,"15min":0.192},"max":{"1min":1.24,"5min":13.750999999999999,"15min":13.750999999999999},"last":0.53200000000000003}]},{"osd":2,"last update":"Sat Jun 3 19:58:16 2023","interfaces":[{"interface":"back","average":{"1min":0.71199999999999997,"5min":0.71499999999999997,"15min":0.71499999999999997},"min":{"1min":0.34599999999999997,"5min":0.34599999999999997,"15min":0.34599999999999997},"max":{"1min":1.135,"5min":1.135,"15min":1.135},"last":0.58199999999999996},{"interface":"front","average":{"1min":0.69799999999999995,"5min":0.73699999999999999,"15min":0.74299999999999999},"min":{"1min":0.36499999999999999,"5min":0.36499999999999999,"15min":0.36499999999999999},"max":{"1min":1.2250000000000001,"5min":1.548,"15min":1.548},"last":0.68200000000000005}]},{"osd":3,"last update":"Sat Jun 3 19:57:55 2023","interfaces":[{"interface":"back","average":{"1min":0.73799999999999999,"5min":0.73799999999999999,"15min":0.73799999999999999},"min":{"1min":0.36599999999999999,"5min":0.36599999999999999,"15min":0.36599999999999999},"max":{"1min":1.2769999999999999,"5min":1.2769999999999999,"15min":1.2769999999999999},"last":1.2},{"interface":"front","average":{"1min":0.86899999999999999,"5min":0.86899999999999999,"15min":0.86899999999999999},"min":{"1min":0.41299999999999998,"5min":0.41299999999999998,"15min":0.41299999999999998},"max":{"1min":1.3740000000000001,"5min":1.3740000000000001,"15min":1.3740000000000001},"last":1.036}]},{"osd":4,"last update":"Sat Jun 3 19:58:28 2023","interfaces":[{"interface":"back","average":{"1min":0.99299999999999999,"5min":0.99299999999999999,"15min":0.99299999999999999},"min":{"1min":0.80400000000000005,"5min":0.80400000000000005,"15min":0.80400000000000005},"max":{"1min":1.466,"5min":1.466,"15min":1.466},"last":0.89200000000000002},{"interface":"front","average":{"1min":0.95699999999999996,"5min":0.95699999999999996,"15min":0.95699999999999996},"min":{"1min":0.66900000000000004,"5min":0.66900000000000004,"15min":0.66900000000000004},"max":{"1min":1.591,"5min":1.591,"15min":1.591},"last":0.81100000000000005}]},{"osd":5,"last update":"Sat Jun 3 19:58:10 2023","interfaces":[{"interface":"back","average":{"1min":0.97499999999999998,"5min":1.7350000000000001,"15min":2.2370000000000001},"min":{"1min":0.69499999999999995,"5min":0.379,"15min":0.373},"max":{"1min":1.867,"5min":7.9400000000000004,"15min":7.9400000000000004},"last":0.93999999999999995},{"interface":"front","average":{"1min":1.0449999999999999,"5min":1.9299999999999999,"15min":2.7010000000000001},"min":{"1min":0.749,"5min":0.40000000000000002,"15min":0.40000000000000002},"max":{"1min":1.8899999999999999,"5min":13.797000000000001,"15min":13.797000000000001},"last":0.73499999999999999}]},{"osd":6,"last update":"Sat Jun 3 19:58:10 2023","interfaces":[{"interface":"back","average":{"1min":1.024,"5min":1.9430000000000001,"15min":2.3460000000000001},"min":{"1min":0.65500000000000003,"5min":0.38300000000000001,"15min":0.38300000000000001},"max":{"1min":1.8169999999999999,"5min":13.941000000000001,"15min":13.941000000000001},"last":0.78000000000000003},{"interface":"front","average":{"1min":1.139,"5min":1.879,"15min":2.859},"min":{"1min":0.77300000000000002,"5min":0.38700000000000001,"15min":0.38700000000000001},"max":{"1min":1.6559999999999999,"5min":9.4060000000000006,"15min":14.847},"last":1.1220000000000001}]},{"osd":7,"last update":"Sat Jun 3 19:58:10 2023","interfaces":[{"interface":"back","average":{"1min":1.099,"5min":1.6890000000000001,"15min":2.5059999999999998},"min":{"1min":0.67900000000000005,"5min":0.47899999999999998,"15min":0.46000000000000002},"max":{"1min":1.7629999999999999,"5min":7.6580000000000004,"15min":14.977},"last":1.2749999999999999},{"interface":"front","average":{"1min":1.048,"5min":1.4970000000000001,"15min":1.77},"min":{"1min":0.59399999999999997,"5min":0.371,"15min":0.371},"max":{"1min":1.786,"5min":7.6020000000000003,"15min":7.6020000000000003},"last":1.089}]}]},{"osd":0,"up_from":86,"seq":369367187598,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":219664,"kb_used_data":1864,"kb_used_omap":5,"kb_used_meta":217786,"kb_avail":93525488,"statfs":{"total":95995035648,"available":95770099712,"internally_reserved":0,"allocated":1908736,"data_stored":1361683,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":223013498},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Jun 3 19:58:11 2023","interfaces":[{"interface":"back","average":{"1min":0.73799999999999999,"5min":1.4219999999999999,"15min":2.194},"min":{"1min":0.42299999999999999,"5min":0.217,"15min":0.217},"max":{"1min":1.0449999999999999,"5min":10.441000000000001,"15min":13.760999999999999},"last":0.91200000000000003},{"interface":"front","average":{"1min":0.67000000000000004,"5min":1.24,"15min":1.8120000000000001},"min":{"1min":0.39700000000000002,"5min":0.26200000000000001,"15min":0.253},"max":{"1min":1.044,"5min":9.1280000000000001,"15min":9.1280000000000001},"last":0.51300000000000001}]},{"osd":2,"last update":"Sat Jun 3 19:58:14 2023","interfaces":[{"interface":"back","average":{"1min":0.69799999999999995,"5min":0.66600000000000004,"15min":0.66100000000000003},"min":{"1min":0.32300000000000001,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":1.077,"5min":1.077,"15min":1.077},"last":0.40400000000000003},{"interface":"front","average":{"1min":0.79400000000000004,"5min":0.75600000000000001,"15min":0.75},"min":{"1min":0.48799999999999999,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":1.1890000000000001,"5min":1.1890000000000001,"15min":1.1890000000000001},"last":0.66200000000000003}]},{"osd":3,"last update":"Sat Jun 3 19:57:55 2023","interfaces":[{"interface":"back","average":{"1min":0.81599999999999995,"5min":0.81599999999999995,"15min":0.81599999999999995},"min":{"1min":0.437,"5min":0.437,"15min":0.437},"max":{"1min":1.052,"5min":1.052,"15min":1.052},"last":0.45400000000000001},{"interface":"front","average":{"1min":0.76100000000000001,"5min":0.76100000000000001,"15min":0.76100000000000001},"min":{"1min":0.40400000000000003,"5min":0.40400000000000003,"15min":0.40400000000000003},"max":{"1min":1.1819999999999999,"5min":1.1819999999999999,"15min":1.1819999999999999},"last":0.70799999999999996}]},{"osd":4,"last update":"Sat Jun 3 19:58:31 2023","interfaces":[{"interface":"back","average":{"1min":1.0960000000000001,"5min":1.0960000000000001,"15min":1.0960000000000001},"min":{"1min":0.84099999999999997,"5min":0.84099999999999997,"15min":0.84099999999999997},"max":{"1min":1.474,"5min":1.474,"15min":1.474},"last":0.82599999999999996},{"interface":"front","average":{"1min":1.1180000000000001,"5min":1.1180000000000001,"15min":1.1180000000000001},"min":{"1min":0.73099999999999998,"5min":0.73099999999999998,"15min":0.73099999999999998},"max":{"1min":1.597,"5min":1.597,"15min":1.597},"last":0.84999999999999998}]},{"osd":5,"last update":"Sat Jun 3 19:58:31 2023","interfaces":[{"interface":"back","average":{"1min":0.99299999999999999,"5min":1.522,"15min":2.1280000000000001},"min":{"1min":0.623,"5min":0.35999999999999999,"15min":0.33900000000000002},"max":{"1min":1.2909999999999999,"5min":9.2059999999999995,"15min":16.312000000000001},"last":1.01},{"interface":"front","average":{"1min":1.1279999999999999,"5min":1.722,"15min":1.8080000000000001},"min":{"1min":0.70499999999999996,"5min":0.40500000000000003,"15min":0.32500000000000001},"max":{"1min":1.5449999999999999,"5min":9.3330000000000002,"15min":24.661000000000001},"last":0.92600000000000005}]},{"osd":6,"last update":"Sat Jun 3 19:58:31 2023","interfaces":[{"interface":"back","average":{"1min":1.1599999999999999,"5min":1.7310000000000001,"15min":1.778},"min":{"1min":0.63900000000000001,"5min":0.36099999999999999,"15min":0.33500000000000002},"max":{"1min":1.538,"5min":9.3629999999999995,"15min":24.599},"last":0.86199999999999999},{"interface":"front","average":{"1min":1.1839999999999999,"5min":1.6839999999999999,"15min":2.1869999999999998},"min":{"1min":0.85299999999999998,"5min":0.42999999999999999,"15min":0.42999999999999999},"max":{"1min":1.6299999999999999,"5min":10.459,"15min":13.601000000000001},"last":0.93700000000000006}]},{"osd":7,"last update":"Sat Jun 3 19:58:31 2023","interfaces":[{"interface":"back","average":{"1min":1.1579999999999999,"5min":1.7170000000000001,"15min":1.7},"min":{"1min":0.66900000000000004,"5min":0.39900000000000002,"15min":0.39900000000000002},"max":{"1min":1.605,"5min":9.4190000000000005,"15min":24.634},"last":0.77100000000000002},{"interface":"front","average":{"1min":1.1639999999999999,"5min":1.5600000000000001,"15min":2.0009999999999999},"min":{"1min":0.89300000000000002,"5min":0.45800000000000002,"15min":0.40699999999999997},"max":{"1min":1.573,"5min":12.621,"15min":12.621},"last":0.98399999999999999}]}]},{"osd":2,"up_from":114,"seq":489626271777,"num_pgs":19,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":15572,"kb_used_data":1228,"kb_used_omap":7,"kb_used_meta":14328,"kb_avail":93729580,"statfs":{"total":95995035648,"available":95979089920,"internally_reserved":0,"allocated":1257472,"data_stored":764442,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8111,"internal_metadata":14671953},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 3 19:58:16 2023","interfaces":[{"interface":"back","average":{"1min":0.59399999999999997,"5min":0.61899999999999999,"15min":0.623},"min":{"1min":0.35899999999999999,"5min":0.314,"15min":0.314},"max":{"1min":0.90300000000000002,"5min":0.90300000000000002,"15min":0.90300000000000002},"last":0.54300000000000004},{"interface":"front","average":{"1min":0.71199999999999997,"5min":0.70899999999999996,"15min":0.70899999999999996},"min":{"1min":0.42099999999999999,"5min":0.42099999999999999,"15min":0.42099999999999999},"max":{"1min":1.04,"5min":1.083,"15min":1.083},"last":0.504}]},{"osd":1,"last update":"Sat Jun 3 19:58:16 2023","interfaces":[{"interface":"back","average":{"1min":0.68100000000000005,"5min":0.71299999999999997,"15min":0.71799999999999997},"min":{"1min":0.36499999999999999,"5min":0.36499999999999999,"15min":0.36499999999999999},"max":{"1min":0.97699999999999998,"5min":1.1020000000000001,"15min":1.1020000000000001},"last":0.64300000000000002},{"interface":"front","average":{"1min":0.65700000000000003,"5min":0.67200000000000004,"15min":0.67400000000000004},"min":{"1min":0.49299999999999999,"5min":0.443,"15min":0.443},"max":{"1min":1.0580000000000001,"5min":1.0580000000000001,"15min":1.0580000000000001},"last":0.60299999999999998}]},{"osd":3,"last update":"Sat Jun 3 19:57:58 2023","interfaces":[{"interface":"back","average":{"1min":0.751,"5min":0.751,"15min":0.751},"min":{"1min":0.44700000000000001,"5min":0.44700000000000001,"15min":0.44700000000000001},"max":{"1min":1.0409999999999999,"5min":1.0409999999999999,"15min":1.0409999999999999},"last":0.69699999999999995},{"interface":"front","average":{"1min":0.75800000000000001,"5min":0.75800000000000001,"15min":0.75800000000000001},"min":{"1min":0.53900000000000003,"5min":0.53900000000000003,"15min":0.53900000000000003},"max":{"1min":0.95799999999999996,"5min":0.95799999999999996,"15min":0.95799999999999996},"last":0.72999999999999998}]},{"osd":4,"last update":"Sat Jun 3 19:58:31 2023","interfaces":[{"interface":"back","average":{"1min":0.92100000000000004,"5min":0.92100000000000004,"15min":0.92100000000000004},"min":{"1min":0.70699999999999996,"5min":0.70699999999999996,"15min":0.70699999999999996},"max":{"1min":1.2090000000000001,"5min":1.2090000000000001,"15min":1.2090000000000001},"last":0.67900000000000005},{"interface":"front","average":{"1min":0.97499999999999998,"5min":0.97499999999999998,"15min":0.97499999999999998},"min":{"1min":0.68100000000000005,"5min":0.68100000000000005,"15min":0.68100000000000005},"max":{"1min":1.3069999999999999,"5min":1.3069999999999999,"15min":1.3069999999999999},"last":1.03}]},{"osd":5,"last update":"Sat Jun 3 19:58:16 2023","interfaces":[{"interface":"back","average":{"1min":1.0049999999999999,"5min":1.0309999999999999,"15min":1.0349999999999999},"min":{"1min":0.77200000000000002,"5min":0.65500000000000003,"15min":0.65500000000000003},"max":{"1min":1.3979999999999999,"5min":1.3979999999999999,"15min":1.3979999999999999},"last":0.92000000000000004},{"interface":"front","average":{"1min":1.028,"5min":1.044,"15min":1.0469999999999999},"min":{"1min":0.65100000000000002,"5min":0.65100000000000002,"15min":0.65100000000000002},"max":{"1min":1.335,"5min":1.369,"15min":1.369},"last":1.1220000000000001}]},{"osd":6,"last update":"Sat Jun 3 19:58:16 2023","interfaces":[{"interface":"back","average":{"1min":1.0409999999999999,"5min":1.036,"15min":1.0349999999999999},"min":{"1min":0.71699999999999997,"5min":0.71699999999999997,"15min":0.71699999999999997},"max":{"1min":1.3899999999999999,"5min":1.3899999999999999,"15min":1.3899999999999999},"last":1.0049999999999999},{"interface":"front","average":{"1min":1.0780000000000001,"5min":1.052,"15min":1.048},"min":{"1min":0.73899999999999999,"5min":0.68100000000000005,"15min":0.68100000000000005},"max":{"1min":1.3919999999999999,"5min":1.508,"15min":1.508},"last":1.05}]},{"osd":7,"last update":"Sat Jun 3 19:58:16 2023","interfaces":[{"interface":"back","average":{"1min":1.1160000000000001,"5min":1.167,"15min":1.175},"min":{"1min":0.81999999999999995,"5min":0.77100000000000002,"15min":0.77100000000000002},"max":{"1min":1.456,"5min":1.456,"15min":1.456},"last":1.1850000000000001},{"interface":"front","average":{"1min":1.1220000000000001,"5min":1.151,"15min":1.1559999999999999},"min":{"1min":0.78600000000000003,"5min":0.78600000000000003,"15min":0.78600000000000003},"max":{"1min":1.3879999999999999,"5min":1.6930000000000001,"15min":1.6930000000000001},"last":1.151}]}]},{"osd":3,"up_from":119,"seq":511101108248,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":18108,"kb_used_data":1204,"kb_used_omap":2,"kb_used_meta":16893,"kb_avail":93727044,"statfs":{"total":95995035648,"available":95976493056,"internally_reserved":0,"allocated":1232896,"data_stored":760088,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":17298485},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 3 19:57:55 2023","interfaces":[{"interface":"back","average":{"1min":0.63200000000000001,"5min":0.63200000000000001,"15min":0.63200000000000001},"min":{"1min":0.433,"5min":0.433,"15min":0.433},"max":{"1min":0.84699999999999998,"5min":0.84699999999999998,"15min":0.84699999999999998},"last":0.46800000000000003},{"interface":"front","average":{"1min":0.73099999999999998,"5min":0.73099999999999998,"15min":0.73099999999999998},"min":{"1min":0.40699999999999997,"5min":0.40699999999999997,"15min":0.40699999999999997},"max":{"1min":1,"5min":1,"15min":1},"last":0.51300000000000001}]},{"osd":1,"last update":"Sat Jun 3 19:57:55 2023","interfaces":[{"interface":"back","average":{"1min":0.71099999999999997,"5min":0.71099999999999997,"15min":0.71099999999999997},"min":{"1min":0.39300000000000002,"5min":0.39300000000000002,"15min":0.39300000000000002},"max":{"1min":1.028,"5min":1.028,"15min":1.028},"last":0.40600000000000003},{"interface":"front","average":{"1min":0.76100000000000001,"5min":0.76100000000000001,"15min":0.76100000000000001},"min":{"1min":0.46800000000000003,"5min":0.46800000000000003,"15min":0.46800000000000003},"max":{"1min":1.01,"5min":1.01,"15min":1.01},"last":0.54600000000000004}]},{"osd":2,"last update":"Sat Jun 3 19:57:55 2023","interfaces":[{"interface":"back","average":{"1min":0.64400000000000002,"5min":0.64400000000000002,"15min":0.64400000000000002},"min":{"1min":0.40500000000000003,"5min":0.40500000000000003,"15min":0.40500000000000003},"max":{"1min":0.89100000000000001,"5min":0.89100000000000001,"15min":0.89100000000000001},"last":0.433},{"interface":"front","average":{"1min":0.76300000000000001,"5min":0.76300000000000001,"15min":0.76300000000000001},"min":{"1min":0.501,"5min":0.501,"15min":0.501},"max":{"1min":1.1140000000000001,"5min":1.1140000000000001,"15min":1.1140000000000001},"last":0.56799999999999995}]},{"osd":4,"last update":"Sat Jun 3 19:58:28 2023","interfaces":[{"interface":"back","average":{"1min":1,"5min":1,"15min":1},"min":{"1min":0.71499999999999997,"5min":0.71499999999999997,"15min":0.71499999999999997},"max":{"1min":1.3819999999999999,"5min":1.3819999999999999,"15min":1.3819999999999999},"last":0.95599999999999996},{"interface":"front","average":{"1min":1.0109999999999999,"5min":1.0109999999999999,"15min":1.0109999999999999},"min":{"1min":0.78200000000000003,"5min":0.78200000000000003,"15min":0.78200000000000003},"max":{"1min":1.2689999999999999,"5min":1.2689999999999999,"15min":1.2689999999999999},"last":0.85399999999999998}]},{"osd":5,"last update":"Sat Jun 3 19:57:55 2023","interfaces":[{"interface":"back","average":{"1min":0.98699999999999999,"5min":0.98699999999999999,"15min":0.98699999999999999},"min":{"1min":0.73499999999999999,"5min":0.73499999999999999,"15min":0.73499999999999999},"max":{"1min":1.429,"5min":1.429,"15min":1.429},"last":0.77700000000000002},{"interface":"front","average":{"1min":1.0249999999999999,"5min":1.0249999999999999,"15min":1.0249999999999999},"min":{"1min":0.754,"5min":0.754,"15min":0.754},"max":{"1min":1.5900000000000001,"5min":1.5900000000000001,"15min":1.5900000000000001},"last":0.65500000000000003}]},{"osd":6,"last update":"Sat Jun 3 19:57:55 2023","interfaces":[{"interface":"back","average":{"1min":1.0329999999999999,"5min":1.0329999999999999,"15min":1.0329999999999999},"min":{"1min":0.67100000000000004,"5min":0.67100000000000004,"15min":0.67100000000000004},"max":{"1min":1.6559999999999999,"5min":1.6559999999999999,"15min":1.6559999999999999},"last":0.90200000000000002},{"interface":"front","average":{"1min":1.0580000000000001,"5min":1.0580000000000001,"15min":1.0580000000000001},"min":{"1min":0.752,"5min":0.752,"15min":0.752},"max":{"1min":1.3340000000000001,"5min":1.3340000000000001,"15min":1.3340000000000001},"last":0.92000000000000004}]},{"osd":7,"last update":"Sat Jun 3 19:57:55 2023","interfaces":[{"interface":"back","average":{"1min":1.044,"5min":1.044,"15min":1.044},"min":{"1min":0.66600000000000004,"5min":0.66600000000000004,"15min":0.66600000000000004},"max":{"1min":1.3300000000000001,"5min":1.3300000000000001,"15min":1.3300000000000001},"last":0.878},{"interface":"front","average":{"1min":1.028,"5min":1.028,"15min":1.028},"min":{"1min":0.69599999999999995,"5min":0.69599999999999995,"15min":0.69599999999999995},"max":{"1min":1.4610000000000001,"5min":1.4610000000000001,"15min":1.4610000000000001},"last":0.73399999999999999}]}]},{"osd":4,"up_from":124,"seq":532575944721,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":15492,"kb_used_data":1212,"kb_used_omap":10,"kb_used_meta":14261,"kb_avail":93729660,"statfs":{"total":95995035648,"available":95979171840,"internally_reserved":0,"allocated":1241088,"data_stored":762768,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":10652,"internal_metadata":14603876},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 3 19:58:29 2023","interfaces":[{"interface":"back","average":{"1min":0.97499999999999998,"5min":0.97499999999999998,"15min":0.97499999999999998},"min":{"1min":0.55800000000000005,"5min":0.55800000000000005,"15min":0.55800000000000005},"max":{"1min":1.3240000000000001,"5min":1.3240000000000001,"15min":1.3240000000000001},"last":1.0629999999999999},{"interface":"front","average":{"1min":1.0309999999999999,"5min":1.0309999999999999,"15min":1.0309999999999999},"min":{"1min":0.78200000000000003,"5min":0.78200000000000003,"15min":0.78200000000000003},"max":{"1min":1.2250000000000001,"5min":1.2250000000000001,"15min":1.2250000000000001},"last":1.1970000000000001}]},{"osd":1,"last update":"Sat Jun 3 19:58:29 2023","interfaces":[{"interface":"back","average":{"1min":0.871,"5min":0.871,"15min":0.871},"min":{"1min":0.60999999999999999,"5min":0.60999999999999999,"15min":0.60999999999999999},"max":{"1min":1.2549999999999999,"5min":1.2549999999999999,"15min":1.2549999999999999},"last":0.73899999999999999},{"interface":"front","average":{"1min":0.86299999999999999,"5min":0.86299999999999999,"15min":0.86299999999999999},"min":{"1min":0.54900000000000004,"5min":0.54900000000000004,"15min":0.54900000000000004},"max":{"1min":1.278,"5min":1.278,"15min":1.278},"last":0.96299999999999997}]},{"osd":2,"last update":"Sat Jun 3 19:58:29 2023","interfaces":[{"interface":"back","average":{"1min":0.871,"5min":0.871,"15min":0.871},"min":{"1min":0.52800000000000002,"5min":0.52800000000000002,"15min":0.52800000000000002},"max":{"1min":1.196,"5min":1.196,"15min":1.196},"last":0.82499999999999996},{"interface":"front","average":{"1min":0.93799999999999994,"5min":0.93799999999999994,"15min":0.93799999999999994},"min":{"1min":0.63700000000000001,"5min":0.63700000000000001,"15min":0.63700000000000001},"max":{"1min":1.2769999999999999,"5min":1.2769999999999999,"15min":1.2769999999999999},"last":0.875}]},{"osd":3,"last update":"Sat Jun 3 19:58:29 2023","interfaces":[{"interface":"back","average":{"1min":0.96199999999999997,"5min":0.96199999999999997,"15min":0.96199999999999997},"min":{"1min":0.68700000000000006,"5min":0.68700000000000006,"15min":0.68700000000000006},"max":{"1min":1.238,"5min":1.238,"15min":1.238},"last":1.038},{"interface":"front","average":{"1min":1.056,"5min":1.056,"15min":1.056},"min":{"1min":0.71899999999999997,"5min":0.71899999999999997,"15min":0.71899999999999997},"max":{"1min":1.28,"5min":1.28,"15min":1.28},"last":1.113}]},{"osd":5,"last update":"Sat Jun 3 19:58:29 2023","interfaces":[{"interface":"back","average":{"1min":0.95299999999999996,"5min":0.95299999999999996,"15min":0.95299999999999996},"min":{"1min":0.72399999999999998,"5min":0.72399999999999998,"15min":0.72399999999999998},"max":{"1min":1.234,"5min":1.234,"15min":1.234},"last":1.081},{"interface":"front","average":{"1min":0.98799999999999999,"5min":0.98799999999999999,"15min":0.98799999999999999},"min":{"1min":0.77200000000000002,"5min":0.77200000000000002,"15min":0.77200000000000002},"max":{"1min":1.24,"5min":1.24,"15min":1.24},"last":0.92600000000000005}]},{"osd":6,"last update":"Sat Jun 3 19:58:29 2023","interfaces":[{"interface":"back","average":{"1min":1.0449999999999999,"5min":1.0449999999999999,"15min":1.0449999999999999},"min":{"1min":0.51300000000000001,"5min":0.51300000000000001,"15min":0.51300000000000001},"max":{"1min":1.323,"5min":1.323,"15min":1.323},"last":1.0109999999999999},{"interface":"front","average":{"1min":1.079,"5min":1.079,"15min":1.079},"min":{"1min":0.73399999999999999,"5min":0.73399999999999999,"15min":0.73399999999999999},"max":{"1min":1.494,"5min":1.494,"15min":1.494},"last":1.2390000000000001}]},{"osd":7,"last update":"Sat Jun 3 19:58:29 2023","interfaces":[{"interface":"back","average":{"1min":1.1379999999999999,"5min":1.1379999999999999,"15min":1.1379999999999999},"min":{"1min":0.77100000000000002,"5min":0.77100000000000002,"15min":0.77100000000000002},"max":{"1min":1.4490000000000001,"5min":1.4490000000000001,"15min":1.4490000000000001},"last":1.1379999999999999},{"interface":"front","average":{"1min":1.171,"5min":1.171,"15min":1.171},"min":{"1min":0.90300000000000002,"5min":0.90300000000000002,"15min":0.90300000000000002},"max":{"1min":1.383,"5min":1.383,"15min":1.383},"last":1.1699999999999999}]}]},{"osd":5,"up_from":36,"seq":154618823007,"num_pgs":21,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":358044,"kb_used_data":1172,"kb_used_omap":0,"kb_used_meta":356864,"kb_avail":93387108,"statfs":{"total":95995035648,"available":95628398592,"internally_reserved":0,"allocated":1200128,"data_stored":753684,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":365428736},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 3 19:58:19 2023","interfaces":[{"interface":"back","average":{"1min":0.91200000000000003,"5min":1.3029999999999999,"15min":1.4890000000000001},"min":{"1min":0.63600000000000001,"5min":0.26700000000000002,"15min":0.26600000000000001},"max":{"1min":1.147,"5min":7.54,"15min":12.489000000000001},"last":1.1970000000000001},{"interface":"front","average":{"1min":0.98199999999999998,"5min":1.6259999999999999,"15min":1.7310000000000001},"min":{"1min":0.82699999999999996,"5min":0.37,"15min":0.31},"max":{"1min":1.133,"5min":10.403,"15min":12.542999999999999},"last":1.0580000000000001}]},{"osd":1,"last update":"Sat Jun 3 19:58:14 2023","interfaces":[{"interface":"back","average":{"1min":0.998,"5min":1.6719999999999999,"15min":1.8500000000000001},"min":{"1min":0.76100000000000001,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":1.4059999999999999,"5min":11.27,"15min":11.27},"last":0.877},{"interface":"front","average":{"1min":0.86099999999999999,"5min":1.7849999999999999,"15min":1.9330000000000001},"min":{"1min":0.56599999999999995,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":1.0860000000000001,"5min":11.34,"15min":11.34},"last":0.92400000000000004}]},{"osd":2,"last update":"Sat Jun 3 19:58:14 2023","interfaces":[{"interface":"back","average":{"1min":0.91500000000000004,"5min":0.876,"15min":0.87},"min":{"1min":0.66500000000000004,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":1.169,"5min":1.169,"15min":1.169},"last":1.1000000000000001},{"interface":"front","average":{"1min":0.94399999999999995,"5min":0.96999999999999997,"15min":0.97399999999999998},"min":{"1min":0.73399999999999999,"5min":0.48799999999999999,"15min":0.48799999999999999},"max":{"1min":1.232,"5min":1.335,"15min":1.335},"last":1.1539999999999999}]},{"osd":3,"last update":"Sat Jun 3 19:57:53 2023","interfaces":[{"interface":"back","average":{"1min":1.018,"5min":1.018,"15min":1.018},"min":{"1min":0.38,"5min":0.38,"15min":0.38},"max":{"1min":1.2709999999999999,"5min":1.2709999999999999,"15min":1.2709999999999999},"last":1.3779999999999999},{"interface":"front","average":{"1min":0.99099999999999999,"5min":0.99099999999999999,"15min":0.99099999999999999},"min":{"1min":0.50700000000000001,"5min":0.50700000000000001,"15min":0.50700000000000001},"max":{"1min":1.363,"5min":1.363,"15min":1.363},"last":1.361}]},{"osd":4,"last update":"Sat Jun 3 19:58:30 2023","interfaces":[{"interface":"back","average":{"1min":1.002,"5min":1.002,"15min":1.002},"min":{"1min":0.60299999999999998,"5min":0.60299999999999998,"15min":0.60299999999999998},"max":{"1min":1.3500000000000001,"5min":1.3500000000000001,"15min":1.3500000000000001},"last":1.4750000000000001},{"interface":"front","average":{"1min":1.1879999999999999,"5min":1.1879999999999999,"15min":1.1879999999999999},"min":{"1min":1.022,"5min":1.022,"15min":1.022},"max":{"1min":1.522,"5min":1.522,"15min":1.522},"last":1.3160000000000001}]},{"osd":6,"last update":"Sat Jun 3 19:58:19 2023","interfaces":[{"interface":"back","average":{"1min":1.1000000000000001,"5min":1.6419999999999999,"15min":1.1970000000000001},"min":{"1min":0.80900000000000005,"5min":0.33000000000000002,"15min":0.33000000000000002},"max":{"1min":1.474,"5min":12.731999999999999,"15min":12.731999999999999},"last":1.431},{"interface":"front","average":{"1min":1.153,"5min":1.698,"15min":1.327},"min":{"1min":0.88700000000000001,"5min":0.42899999999999999,"15min":0.32800000000000001},"max":{"1min":1.4339999999999999,"5min":12.779,"15min":12.779},"last":1.3320000000000001}]},{"osd":7,"last update":"Sat Jun 3 19:58:01 2023","interfaces":[{"interface":"back","average":{"1min":1.0469999999999999,"5min":1.806,"15min":1.3759999999999999},"min":{"1min":0.47699999999999998,"5min":0.45700000000000002,"15min":0.379},"max":{"1min":1.371,"5min":11.297000000000001,"15min":11.297000000000001},"last":1.5229999999999999},{"interface":"front","average":{"1min":0.97199999999999998,"5min":1.4139999999999999,"15min":1.216},"min":{"1min":0.39600000000000002,"5min":0.39600000000000002,"15min":0.32100000000000001},"max":{"1min":1.26,"5min":7.4720000000000004,"15min":7.4720000000000004},"last":1.2649999999999999}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6928,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4862,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":10652,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4786,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4932,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-06-03T19:58:48.110 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph pg dump --format=json 2023-06-03T19:58:49.029 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:58:48 smithi089 bash[75068]: audit 1685822327.2347455 mgr.y (mgr.34104) 671 : audit 0 from='client.35670 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-06-03T19:58:49.029 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:58:48 smithi089 bash[78452]: audit 1685822327.2347455 mgr.y (mgr.34104) 671 : audit 0 from='client.35670 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-06-03T19:58:49.084 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:58:48 smithi164 bash[59462]: audit 1685822327.2347455 mgr.y (mgr.34104) 671 : audit 0 from='client.35670 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-06-03T19:58:50.029 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:58:49 smithi089 bash[75068]: cluster 1685822328.3898857 mgr.y (mgr.34104) 672 : cluster 0 pgmap v579: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-06-03T19:58:50.029 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:58:49 smithi089 bash[78452]: cluster 1685822328.3898857 mgr.y (mgr.34104) 672 : cluster 0 pgmap v579: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-06-03T19:58:50.084 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:58:49 smithi164 bash[59462]: cluster 1685822328.3898857 mgr.y (mgr.34104) 672 : cluster 0 pgmap v579: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-06-03T19:58:52.029 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:58:51 smithi089 bash[75068]: cluster 1685822330.3905659 mgr.y (mgr.34104) 673 : cluster 0 pgmap v580: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-06-03T19:58:52.029 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:58:51 smithi089 bash[78452]: cluster 1685822330.3905659 mgr.y (mgr.34104) 673 : cluster 0 pgmap v580: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-06-03T19:58:52.084 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:58:51 smithi164 bash[59462]: cluster 1685822330.3905659 mgr.y (mgr.34104) 673 : cluster 0 pgmap v580: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-06-03T19:58:52.965 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/mon.a/config 2023-06-03T19:58:53.002 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:58:52 smithi089 bash[75068]: audit 1685822332.0918157 mon.c (mon.2) 366 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T19:58:53.003 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:58:52 smithi089 bash[78452]: audit 1685822332.0918157 mon.c (mon.2) 366 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T19:58:53.084 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:58:52 smithi164 bash[59462]: audit 1685822332.0918157 mon.c (mon.2) 366 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T19:58:54.029 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:58:53 smithi089 bash[78452]: cluster 1685822332.3914113 mgr.y (mgr.34104) 674 : cluster 0 pgmap v581: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-06-03T19:58:54.030 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:58:53 smithi089 bash[75068]: cluster 1685822332.3914113 mgr.y (mgr.34104) 674 : cluster 0 pgmap v581: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-06-03T19:58:54.084 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:58:53 smithi164 bash[59462]: cluster 1685822332.3914113 mgr.y (mgr.34104) 674 : cluster 0 pgmap v581: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-06-03T19:58:56.029 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:58:55 smithi089 bash[75068]: cluster 1685822334.3923385 mgr.y (mgr.34104) 675 : cluster 0 pgmap v582: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-06-03T19:58:56.029 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:58:55 smithi089 bash[78452]: cluster 1685822334.3923385 mgr.y (mgr.34104) 675 : cluster 0 pgmap v582: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-06-03T19:58:56.084 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:58:55 smithi164 bash[59462]: cluster 1685822334.3923385 mgr.y (mgr.34104) 675 : cluster 0 pgmap v582: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-06-03T19:58:56.528 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 19:58:56 smithi089 bash[71163]: ::ffff:172.21.15.164 - - [03/Jun/2023:19:58:56] "GET /metrics HTTP/1.1" 200 225310 "" "Prometheus/2.33.4" 2023-06-03T19:58:56.644 INFO:teuthology.orchestra.run.smithi089.stdout: 2023-06-03T19:58:56.646 INFO:teuthology.orchestra.run.smithi089.stderr:dumped all 2023-06-03T19:58:57.544 INFO:teuthology.orchestra.run.smithi089.stdout:{"pg_ready":true,"pg_map":{"version":583,"stamp":"2023-06-03T19:58:56.392542+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":595854,"num_objects":24,"num_object_clones":0,"num_object_copies":72,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":24,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":261,"num_write_kb":2446,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":195,"ondisk_log_size":195,"up":219,"acting":219,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":219,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":1315012,"kb_used_data":11460,"kb_used_omap":30,"kb_used_meta":1303457,"kb_avail":748646204,"statfs":{"total":767960285184,"available":766613712896,"internally_reserved":0,"allocated":11735040,"data_stored":7866659,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":31502,"internal_metadata":1334740210},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.004755"},"pg_stats":[{"pgid":"3.1a","version":"60'1","reported_seq":110,"reported_epoch":125,"state":"active+clean","last_fresh":"2023-06-03T19:57:28.467559+0000","last_change":"2023-06-03T19:57:28.467559+0000","last_active":"2023-06-03T19:57:28.467559+0000","last_peered":"2023-06-03T19:57:28.467559+0000","last_clean":"2023-06-03T19:57:28.467559+0000","last_became_active":"2023-06-03T19:57:28.467077+0000","last_became_peered":"2023-06-03T19:57:28.467077+0000","last_unstale":"2023-06-03T19:57:28.467559+0000","last_undegraded":"2023-06-03T19:57:28.467559+0000","last_fullsized":"2023-06-03T19:57:28.467559+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T21:46:02.795576+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":103,"reported_epoch":125,"state":"active+clean","last_fresh":"2023-06-03T19:57:28.470287+0000","last_change":"2023-06-03T19:57:28.470287+0000","last_active":"2023-06-03T19:57:28.470287+0000","last_peered":"2023-06-03T19:57:28.470287+0000","last_clean":"2023-06-03T19:57:28.470287+0000","last_became_active":"2023-06-03T19:57:28.469477+0000","last_became_peered":"2023-06-03T19:57:28.469477+0000","last_unstale":"2023-06-03T19:57:28.470287+0000","last_undegraded":"2023-06-03T19:57:28.470287+0000","last_fullsized":"2023-06-03T19:57:28.470287+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T06:48:19.229190+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":116,"reported_epoch":125,"state":"active+clean","last_fresh":"2023-06-03T19:57:28.473350+0000","last_change":"2023-06-03T19:57:28.473350+0000","last_active":"2023-06-03T19:57:28.473350+0000","last_peered":"2023-06-03T19:57:28.473350+0000","last_clean":"2023-06-03T19:57:28.473350+0000","last_became_active":"2023-06-03T19:57:28.472757+0000","last_became_peered":"2023-06-03T19:57:28.472757+0000","last_unstale":"2023-06-03T19:57:28.473350+0000","last_undegraded":"2023-06-03T19:57:28.473350+0000","last_fullsized":"2023-06-03T19:57:28.473350+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T19:58:10.432188+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":80,"reported_epoch":124,"state":"active+clean","last_fresh":"2023-06-03T19:57:27.441485+0000","last_change":"2023-06-03T19:56:54.507653+0000","last_active":"2023-06-03T19:57:27.441485+0000","last_peered":"2023-06-03T19:57:27.441485+0000","last_clean":"2023-06-03T19:57:27.441485+0000","last_became_active":"2023-06-03T19:56:54.507350+0000","last_became_peered":"2023-06-03T19:56:54.507350+0000","last_unstale":"2023-06-03T19:57:27.441485+0000","last_undegraded":"2023-06-03T19:57:27.441485+0000","last_fullsized":"2023-06-03T19:57:27.441485+0000","mapping_epoch":119,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T22:33:20.065844+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":104,"reported_epoch":124,"state":"active+clean","last_fresh":"2023-06-03T19:57:27.446904+0000","last_change":"2023-06-03T19:56:54.512932+0000","last_active":"2023-06-03T19:57:27.446904+0000","last_peered":"2023-06-03T19:57:27.446904+0000","last_clean":"2023-06-03T19:57:27.446904+0000","last_became_active":"2023-06-03T19:56:54.511992+0000","last_became_peered":"2023-06-03T19:56:54.511992+0000","last_unstale":"2023-06-03T19:57:27.446904+0000","last_undegraded":"2023-06-03T19:57:27.446904+0000","last_fullsized":"2023-06-03T19:57:27.446904+0000","mapping_epoch":119,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T21:01:35.069911+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":91,"reported_epoch":124,"state":"active+clean","last_fresh":"2023-06-03T19:57:27.437336+0000","last_change":"2023-06-03T19:51:59.359522+0000","last_active":"2023-06-03T19:57:27.437336+0000","last_peered":"2023-06-03T19:57:27.437336+0000","last_clean":"2023-06-03T19:57:27.437336+0000","last_became_active":"2023-06-03T19:51:59.359202+0000","last_became_peered":"2023-06-03T19:51:59.359202+0000","last_unstale":"2023-06-03T19:57:27.437336+0000","last_undegraded":"2023-06-03T19:57:27.437336+0000","last_fullsized":"2023-06-03T19:57:27.437336+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":78,"reported_epoch":124,"state":"active+clean","last_fresh":"2023-06-03T19:57:27.441034+0000","last_change":"2023-06-03T19:56:54.509490+0000","last_active":"2023-06-03T19:57:27.441034+0000","last_peered":"2023-06-03T19:57:27.441034+0000","last_clean":"2023-06-03T19:57:27.441034+0000","last_became_active":"2023-06-03T19:56:54.509145+0000","last_became_peered":"2023-06-03T19:56:54.509145+0000","last_unstale":"2023-06-03T19:57:27.441034+0000","last_undegraded":"2023-06-03T19:57:27.441034+0000","last_fullsized":"2023-06-03T19:57:27.441034+0000","mapping_epoch":119,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T07:12:03.964110+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":88,"reported_epoch":124,"state":"active+clean","last_fresh":"2023-06-03T19:57:27.446812+0000","last_change":"2023-06-03T19:56:54.511325+0000","last_active":"2023-06-03T19:57:27.446812+0000","last_peered":"2023-06-03T19:57:27.446812+0000","last_clean":"2023-06-03T19:57:27.446812+0000","last_became_active":"2023-06-03T19:56:54.511033+0000","last_became_peered":"2023-06-03T19:56:54.511033+0000","last_unstale":"2023-06-03T19:57:27.446812+0000","last_undegraded":"2023-06-03T19:57:27.446812+0000","last_fullsized":"2023-06-03T19:57:27.446812+0000","mapping_epoch":119,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T23:50:08.711478+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":92,"reported_epoch":124,"state":"active+clean","last_fresh":"2023-06-03T19:57:27.439764+0000","last_change":"2023-06-03T19:51:59.374143+0000","last_active":"2023-06-03T19:57:27.439764+0000","last_peered":"2023-06-03T19:57:27.439764+0000","last_clean":"2023-06-03T19:57:27.439764+0000","last_became_active":"2023-06-03T19:51:59.373146+0000","last_became_peered":"2023-06-03T19:51:59.373146+0000","last_unstale":"2023-06-03T19:57:27.439764+0000","last_undegraded":"2023-06-03T19:57:27.439764+0000","last_fullsized":"2023-06-03T19:57:27.439764+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":69,"reported_epoch":124,"state":"active+clean","last_fresh":"2023-06-03T19:57:27.449274+0000","last_change":"2023-06-03T19:56:54.510462+0000","last_active":"2023-06-03T19:57:27.449274+0000","last_peered":"2023-06-03T19:57:27.449274+0000","last_clean":"2023-06-03T19:57:27.449274+0000","last_became_active":"2023-06-03T19:56:54.509783+0000","last_became_peered":"2023-06-03T19:56:54.509783+0000","last_unstale":"2023-06-03T19:57:27.449274+0000","last_undegraded":"2023-06-03T19:57:27.449274+0000","last_fullsized":"2023-06-03T19:57:27.449274+0000","mapping_epoch":119,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T00:16:09.624276+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"60'3","reported_seq":110,"reported_epoch":125,"state":"active+clean","last_fresh":"2023-06-03T19:57:28.466401+0000","last_change":"2023-06-03T19:57:28.466401+0000","last_active":"2023-06-03T19:57:28.466401+0000","last_peered":"2023-06-03T19:57:28.466401+0000","last_clean":"2023-06-03T19:57:28.466401+0000","last_became_active":"2023-06-03T19:57:28.465452+0000","last_became_peered":"2023-06-03T19:57:28.465452+0000","last_unstale":"2023-06-03T19:57:28.466401+0000","last_undegraded":"2023-06-03T19:57:28.466401+0000","last_fullsized":"2023-06-03T19:57:28.466401+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":60,"reported_epoch":124,"state":"active+clean","last_fresh":"2023-06-03T19:57:27.446119+0000","last_change":"2023-06-03T19:56:11.753995+0000","last_active":"2023-06-03T19:57:27.446119+0000","last_peered":"2023-06-03T19:57:27.446119+0000","last_clean":"2023-06-03T19:57:27.446119+0000","last_became_active":"2023-06-03T19:56:11.753676+0000","last_became_peered":"2023-06-03T19:56:11.753676+0000","last_unstale":"2023-06-03T19:57:27.446119+0000","last_undegraded":"2023-06-03T19:57:27.446119+0000","last_fullsized":"2023-06-03T19:57:27.446119+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":115,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T01:00:40.343647+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"77'23","reported_seq":106,"reported_epoch":125,"state":"active+clean","last_fresh":"2023-06-03T19:57:28.462522+0000","last_change":"2023-06-03T19:57:28.462522+0000","last_active":"2023-06-03T19:57:28.462522+0000","last_peered":"2023-06-03T19:57:28.462522+0000","last_clean":"2023-06-03T19:57:28.462522+0000","last_became_active":"2023-06-03T19:57:28.462048+0000","last_became_peered":"2023-06-03T19:57:28.462048+0000","last_unstale":"2023-06-03T19:57:28.462522+0000","last_undegraded":"2023-06-03T19:57:28.462522+0000","last_fullsized":"2023-06-03T19:57:28.462522+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":23,"log_dups_size":0,"ondisk_log_size":23,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T23:23:56.697534+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4716,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":23,"num_write_kb":20,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":83,"reported_epoch":125,"state":"active+clean","last_fresh":"2023-06-03T19:57:28.469676+0000","last_change":"2023-06-03T19:57:28.469676+0000","last_active":"2023-06-03T19:57:28.469676+0000","last_peered":"2023-06-03T19:57:28.469676+0000","last_clean":"2023-06-03T19:57:28.469676+0000","last_became_active":"2023-06-03T19:57:28.469328+0000","last_became_peered":"2023-06-03T19:57:28.469328+0000","last_unstale":"2023-06-03T19:57:28.469676+0000","last_undegraded":"2023-06-03T19:57:28.469676+0000","last_fullsized":"2023-06-03T19:57:28.469676+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T23:25:26.041930+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"60'2","reported_seq":109,"reported_epoch":125,"state":"active+clean","last_fresh":"2023-06-03T19:57:28.467711+0000","last_change":"2023-06-03T19:57:28.467711+0000","last_active":"2023-06-03T19:57:28.467711+0000","last_peered":"2023-06-03T19:57:28.467711+0000","last_clean":"2023-06-03T19:57:28.467711+0000","last_became_active":"2023-06-03T19:57:28.467234+0000","last_became_peered":"2023-06-03T19:57:28.467234+0000","last_unstale":"2023-06-03T19:57:28.467711+0000","last_undegraded":"2023-06-03T19:57:28.467711+0000","last_fullsized":"2023-06-03T19:57:28.467711+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":88,"reported_epoch":124,"state":"active+clean","last_fresh":"2023-06-03T19:57:27.444818+0000","last_change":"2023-06-03T19:56:54.514616+0000","last_active":"2023-06-03T19:57:27.444818+0000","last_peered":"2023-06-03T19:57:27.444818+0000","last_clean":"2023-06-03T19:57:27.444818+0000","last_became_active":"2023-06-03T19:56:54.513918+0000","last_became_peered":"2023-06-03T19:56:54.513918+0000","last_unstale":"2023-06-03T19:57:27.444818+0000","last_undegraded":"2023-06-03T19:57:27.444818+0000","last_fullsized":"2023-06-03T19:57:27.444818+0000","mapping_epoch":119,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T19:36:10.967386+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":79,"reported_epoch":125,"state":"active+clean","last_fresh":"2023-06-03T19:57:28.450526+0000","last_change":"2023-06-03T19:56:54.507479+0000","last_active":"2023-06-03T19:57:28.450526+0000","last_peered":"2023-06-03T19:57:28.450526+0000","last_clean":"2023-06-03T19:57:28.450526+0000","last_became_active":"2023-06-03T19:56:54.507149+0000","last_became_peered":"2023-06-03T19:56:54.507149+0000","last_unstale":"2023-06-03T19:57:28.450526+0000","last_undegraded":"2023-06-03T19:57:28.450526+0000","last_fullsized":"2023-06-03T19:57:28.450526+0000","mapping_epoch":119,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T20:22:36.676317+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":91,"reported_epoch":124,"state":"active+clean","last_fresh":"2023-06-03T19:57:27.442389+0000","last_change":"2023-06-03T19:56:54.512472+0000","last_active":"2023-06-03T19:57:27.442389+0000","last_peered":"2023-06-03T19:57:27.442389+0000","last_clean":"2023-06-03T19:57:27.442389+0000","last_became_active":"2023-06-03T19:56:54.512051+0000","last_became_peered":"2023-06-03T19:56:54.512051+0000","last_unstale":"2023-06-03T19:57:27.442389+0000","last_undegraded":"2023-06-03T19:57:27.442389+0000","last_fullsized":"2023-06-03T19:57:27.442389+0000","mapping_epoch":119,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":92,"reported_epoch":125,"state":"active+clean","last_fresh":"2023-06-03T19:57:28.462780+0000","last_change":"2023-06-03T19:57:28.462780+0000","last_active":"2023-06-03T19:57:28.462780+0000","last_peered":"2023-06-03T19:57:28.462780+0000","last_clean":"2023-06-03T19:57:28.462780+0000","last_became_active":"2023-06-03T19:57:28.462298+0000","last_became_peered":"2023-06-03T19:57:28.462298+0000","last_unstale":"2023-06-03T19:57:28.462780+0000","last_undegraded":"2023-06-03T19:57:28.462780+0000","last_fullsized":"2023-06-03T19:57:28.462780+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":79,"reported_epoch":124,"state":"active+clean","last_fresh":"2023-06-03T19:57:27.443519+0000","last_change":"2023-06-03T19:56:54.510289+0000","last_active":"2023-06-03T19:57:27.443519+0000","last_peered":"2023-06-03T19:57:27.443519+0000","last_clean":"2023-06-03T19:57:27.443519+0000","last_became_active":"2023-06-03T19:56:54.509905+0000","last_became_peered":"2023-06-03T19:56:54.509905+0000","last_unstale":"2023-06-03T19:57:27.443519+0000","last_undegraded":"2023-06-03T19:57:27.443519+0000","last_fullsized":"2023-06-03T19:57:27.443519+0000","mapping_epoch":119,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T02:54:26.792401+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"60'8","reported_seq":100,"reported_epoch":124,"state":"active+clean","last_fresh":"2023-06-03T19:57:27.435852+0000","last_change":"2023-06-03T19:47:06.727778+0000","last_active":"2023-06-03T19:57:27.435852+0000","last_peered":"2023-06-03T19:57:27.435852+0000","last_clean":"2023-06-03T19:57:27.435852+0000","last_became_active":"2023-06-03T19:47:06.727305+0000","last_became_peered":"2023-06-03T19:47:06.727305+0000","last_unstale":"2023-06-03T19:57:27.435852+0000","last_undegraded":"2023-06-03T19:57:27.435852+0000","last_fullsized":"2023-06-03T19:57:27.435852+0000","mapping_epoch":86,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":87,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":88,"reported_epoch":124,"state":"active+clean","last_fresh":"2023-06-03T19:57:27.443855+0000","last_change":"2023-06-03T19:56:54.512994+0000","last_active":"2023-06-03T19:57:27.443855+0000","last_peered":"2023-06-03T19:57:27.443855+0000","last_clean":"2023-06-03T19:57:27.443855+0000","last_became_active":"2023-06-03T19:56:54.512579+0000","last_became_peered":"2023-06-03T19:56:54.512579+0000","last_unstale":"2023-06-03T19:57:27.443855+0000","last_undegraded":"2023-06-03T19:57:27.443855+0000","last_fullsized":"2023-06-03T19:57:27.443855+0000","mapping_epoch":119,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T07:02:45.187961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.f","version":"60'1","reported_seq":108,"reported_epoch":125,"state":"active+clean","last_fresh":"2023-06-03T19:57:28.464482+0000","last_change":"2023-06-03T19:57:28.464482+0000","last_active":"2023-06-03T19:57:28.464482+0000","last_peered":"2023-06-03T19:57:28.464482+0000","last_clean":"2023-06-03T19:57:28.464482+0000","last_became_active":"2023-06-03T19:57:28.464081+0000","last_became_peered":"2023-06-03T19:57:28.464081+0000","last_unstale":"2023-06-03T19:57:28.464482+0000","last_undegraded":"2023-06-03T19:57:28.464482+0000","last_fullsized":"2023-06-03T19:57:28.464482+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":76,"reported_epoch":124,"state":"active+clean","last_fresh":"2023-06-03T19:57:27.441050+0000","last_change":"2023-06-03T19:34:17.848358+0000","last_active":"2023-06-03T19:57:27.441050+0000","last_peered":"2023-06-03T19:57:27.441050+0000","last_clean":"2023-06-03T19:57:27.441050+0000","last_became_active":"2023-06-03T19:34:17.847811+0000","last_became_peered":"2023-06-03T19:34:17.847811+0000","last_unstale":"2023-06-03T19:57:27.441050+0000","last_undegraded":"2023-06-03T19:57:27.441050+0000","last_fullsized":"2023-06-03T19:57:27.441050+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.e","version":"60'2","reported_seq":109,"reported_epoch":125,"state":"active+clean","last_fresh":"2023-06-03T19:57:28.468330+0000","last_change":"2023-06-03T19:57:28.468330+0000","last_active":"2023-06-03T19:57:28.468330+0000","last_peered":"2023-06-03T19:57:28.468330+0000","last_clean":"2023-06-03T19:57:28.468330+0000","last_became_active":"2023-06-03T19:57:28.467823+0000","last_became_peered":"2023-06-03T19:57:28.467823+0000","last_unstale":"2023-06-03T19:57:28.468330+0000","last_undegraded":"2023-06-03T19:57:28.468330+0000","last_fullsized":"2023-06-03T19:57:28.468330+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":121,"reported_epoch":125,"state":"active+clean","last_fresh":"2023-06-03T19:57:28.461449+0000","last_change":"2023-06-03T19:57:28.461449+0000","last_active":"2023-06-03T19:57:28.461449+0000","last_peered":"2023-06-03T19:57:28.461449+0000","last_clean":"2023-06-03T19:57:28.461449+0000","last_became_active":"2023-06-03T19:57:28.460745+0000","last_became_peered":"2023-06-03T19:57:28.460745+0000","last_unstale":"2023-06-03T19:57:28.461449+0000","last_undegraded":"2023-06-03T19:57:28.461449+0000","last_fullsized":"2023-06-03T19:57:28.461449+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T20:41:19.361638+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,3],"acting":[4,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.d","version":"60'1","reported_seq":78,"reported_epoch":124,"state":"active+clean","last_fresh":"2023-06-03T19:57:27.445291+0000","last_change":"2023-06-03T19:34:16.806559+0000","last_active":"2023-06-03T19:57:27.445291+0000","last_peered":"2023-06-03T19:57:27.445291+0000","last_clean":"2023-06-03T19:57:27.445291+0000","last_became_active":"2023-06-03T19:34:16.806149+0000","last_became_peered":"2023-06-03T19:34:16.806149+0000","last_unstale":"2023-06-03T19:57:27.445291+0000","last_undegraded":"2023-06-03T19:57:27.445291+0000","last_fullsized":"2023-06-03T19:57:27.445291+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":91,"reported_epoch":124,"state":"active+clean","last_fresh":"2023-06-03T19:57:27.437525+0000","last_change":"2023-06-03T19:51:59.360650+0000","last_active":"2023-06-03T19:57:27.437525+0000","last_peered":"2023-06-03T19:57:27.437525+0000","last_clean":"2023-06-03T19:57:27.437525+0000","last_became_active":"2023-06-03T19:51:59.360237+0000","last_became_peered":"2023-06-03T19:51:59.360237+0000","last_unstale":"2023-06-03T19:57:27.437525+0000","last_undegraded":"2023-06-03T19:57:27.437525+0000","last_fullsized":"2023-06-03T19:57:27.437525+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":92,"reported_epoch":124,"state":"active+clean","last_fresh":"2023-06-03T19:57:27.440288+0000","last_change":"2023-06-03T19:56:54.507432+0000","last_active":"2023-06-03T19:57:27.440288+0000","last_peered":"2023-06-03T19:57:27.440288+0000","last_clean":"2023-06-03T19:57:27.440288+0000","last_became_active":"2023-06-03T19:56:54.507012+0000","last_became_peered":"2023-06-03T19:56:54.507012+0000","last_unstale":"2023-06-03T19:57:27.440288+0000","last_undegraded":"2023-06-03T19:57:27.440288+0000","last_fullsized":"2023-06-03T19:57:27.440288+0000","mapping_epoch":119,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":107,"reported_epoch":125,"state":"active+clean","last_fresh":"2023-06-03T19:57:28.465221+0000","last_change":"2023-06-03T19:57:28.464153+0000","last_active":"2023-06-03T19:57:28.465221+0000","last_peered":"2023-06-03T19:57:28.465221+0000","last_clean":"2023-06-03T19:57:28.465221+0000","last_became_active":"2023-06-03T19:57:28.463502+0000","last_became_peered":"2023-06-03T19:57:28.463502+0000","last_unstale":"2023-06-03T19:57:28.465221+0000","last_undegraded":"2023-06-03T19:57:28.465221+0000","last_fullsized":"2023-06-03T19:57:28.465221+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T06:11:00.999526+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,4],"acting":[0,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.b","version":"60'2","reported_seq":89,"reported_epoch":125,"state":"active+clean","last_fresh":"2023-06-03T19:57:28.470310+0000","last_change":"2023-06-03T19:57:28.470310+0000","last_active":"2023-06-03T19:57:28.470310+0000","last_peered":"2023-06-03T19:57:28.470310+0000","last_clean":"2023-06-03T19:57:28.470310+0000","last_became_active":"2023-06-03T19:57:28.469887+0000","last_became_peered":"2023-06-03T19:57:28.469887+0000","last_unstale":"2023-06-03T19:57:28.470310+0000","last_undegraded":"2023-06-03T19:57:28.470310+0000","last_fullsized":"2023-06-03T19:57:28.470310+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T21:58:53.387131+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":97,"reported_epoch":125,"state":"active+clean","last_fresh":"2023-06-03T19:57:28.465140+0000","last_change":"2023-06-03T19:57:28.465140+0000","last_active":"2023-06-03T19:57:28.465140+0000","last_peered":"2023-06-03T19:57:28.465140+0000","last_clean":"2023-06-03T19:57:28.465140+0000","last_became_active":"2023-06-03T19:57:28.464629+0000","last_became_peered":"2023-06-03T19:57:28.464629+0000","last_unstale":"2023-06-03T19:57:28.465140+0000","last_undegraded":"2023-06-03T19:57:28.465140+0000","last_fullsized":"2023-06-03T19:57:28.465140+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T21:47:11.619583+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":107,"reported_epoch":125,"state":"active+clean","last_fresh":"2023-06-03T19:57:28.461401+0000","last_change":"2023-06-03T19:57:28.461401+0000","last_active":"2023-06-03T19:57:28.461401+0000","last_peered":"2023-06-03T19:57:28.461401+0000","last_clean":"2023-06-03T19:57:28.461401+0000","last_became_active":"2023-06-03T19:57:28.461057+0000","last_became_peered":"2023-06-03T19:57:28.461057+0000","last_unstale":"2023-06-03T19:57:28.461401+0000","last_undegraded":"2023-06-03T19:57:28.461401+0000","last_fullsized":"2023-06-03T19:57:28.461401+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":102,"reported_epoch":125,"state":"active+clean","last_fresh":"2023-06-03T19:57:28.456750+0000","last_change":"2023-06-03T19:57:28.456750+0000","last_active":"2023-06-03T19:57:28.456750+0000","last_peered":"2023-06-03T19:57:28.456750+0000","last_clean":"2023-06-03T19:57:28.456750+0000","last_became_active":"2023-06-03T19:57:28.456436+0000","last_became_peered":"2023-06-03T19:57:28.456436+0000","last_unstale":"2023-06-03T19:57:28.456750+0000","last_undegraded":"2023-06-03T19:57:28.456750+0000","last_fullsized":"2023-06-03T19:57:28.456750+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T01:49:25.481554+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,5],"acting":[4,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":94,"reported_epoch":125,"state":"active+clean","last_fresh":"2023-06-03T19:57:28.462344+0000","last_change":"2023-06-03T19:57:28.462344+0000","last_active":"2023-06-03T19:57:28.462344+0000","last_peered":"2023-06-03T19:57:28.462344+0000","last_clean":"2023-06-03T19:57:28.462344+0000","last_became_active":"2023-06-03T19:57:28.462001+0000","last_became_peered":"2023-06-03T19:57:28.462001+0000","last_unstale":"2023-06-03T19:57:28.462344+0000","last_undegraded":"2023-06-03T19:57:28.462344+0000","last_fullsized":"2023-06-03T19:57:28.462344+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T00:45:13.789650+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":83,"reported_epoch":125,"state":"active+clean","last_fresh":"2023-06-03T19:57:28.467917+0000","last_change":"2023-06-03T19:57:28.467917+0000","last_active":"2023-06-03T19:57:28.467917+0000","last_peered":"2023-06-03T19:57:28.467917+0000","last_clean":"2023-06-03T19:57:28.467917+0000","last_became_active":"2023-06-03T19:57:28.467617+0000","last_became_peered":"2023-06-03T19:57:28.467617+0000","last_unstale":"2023-06-03T19:57:28.467917+0000","last_undegraded":"2023-06-03T19:57:28.467917+0000","last_fullsized":"2023-06-03T19:57:28.467917+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T23:34:23.840582+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":89,"reported_epoch":124,"state":"active+clean","last_fresh":"2023-06-03T19:57:27.447053+0000","last_change":"2023-06-03T19:56:54.512219+0000","last_active":"2023-06-03T19:57:27.447053+0000","last_peered":"2023-06-03T19:57:27.447053+0000","last_clean":"2023-06-03T19:57:27.447053+0000","last_became_active":"2023-06-03T19:56:54.511771+0000","last_became_peered":"2023-06-03T19:56:54.511771+0000","last_unstale":"2023-06-03T19:57:27.447053+0000","last_undegraded":"2023-06-03T19:57:27.447053+0000","last_fullsized":"2023-06-03T19:57:27.447053+0000","mapping_epoch":119,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T19:35:47.796520+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":76,"reported_epoch":125,"state":"active+clean","last_fresh":"2023-06-03T19:57:28.472005+0000","last_change":"2023-06-03T19:57:28.471023+0000","last_active":"2023-06-03T19:57:28.472005+0000","last_peered":"2023-06-03T19:57:28.472005+0000","last_clean":"2023-06-03T19:57:28.472005+0000","last_became_active":"2023-06-03T19:57:28.470452+0000","last_became_peered":"2023-06-03T19:57:28.470452+0000","last_unstale":"2023-06-03T19:57:28.472005+0000","last_undegraded":"2023-06-03T19:57:28.472005+0000","last_fullsized":"2023-06-03T19:57:28.472005+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T05:58:33.305030+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":94,"reported_epoch":125,"state":"active+clean","last_fresh":"2023-06-03T19:57:28.462051+0000","last_change":"2023-06-03T19:51:59.392796+0000","last_active":"2023-06-03T19:57:28.462051+0000","last_peered":"2023-06-03T19:57:28.462051+0000","last_clean":"2023-06-03T19:57:28.462051+0000","last_became_active":"2023-06-03T19:51:59.380207+0000","last_became_peered":"2023-06-03T19:51:59.380207+0000","last_unstale":"2023-06-03T19:57:28.462051+0000","last_undegraded":"2023-06-03T19:57:28.462051+0000","last_fullsized":"2023-06-03T19:57:28.462051+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:32:20.692043+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:32:20.692043+0000","last_clean_scrub_stamp":"2023-06-03T19:32:20.692043+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T00:28:05.866873+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":107,"reported_epoch":124,"state":"active+clean","last_fresh":"2023-06-03T19:57:27.442422+0000","last_change":"2023-06-03T19:56:54.512607+0000","last_active":"2023-06-03T19:57:27.442422+0000","last_peered":"2023-06-03T19:57:27.442422+0000","last_clean":"2023-06-03T19:57:27.442422+0000","last_became_active":"2023-06-03T19:56:54.512282+0000","last_became_peered":"2023-06-03T19:56:54.512282+0000","last_unstale":"2023-06-03T19:57:27.442422+0000","last_undegraded":"2023-06-03T19:57:27.442422+0000","last_fullsized":"2023-06-03T19:57:27.442422+0000","mapping_epoch":119,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":61,"reported_epoch":125,"state":"active+clean","last_fresh":"2023-06-03T19:57:28.471196+0000","last_change":"2023-06-03T19:57:28.470295+0000","last_active":"2023-06-03T19:57:28.471196+0000","last_peered":"2023-06-03T19:57:28.471196+0000","last_clean":"2023-06-03T19:57:28.471196+0000","last_became_active":"2023-06-03T19:57:28.469383+0000","last_became_peered":"2023-06-03T19:57:28.469383+0000","last_unstale":"2023-06-03T19:57:28.471196+0000","last_undegraded":"2023-06-03T19:57:28.471196+0000","last_fullsized":"2023-06-03T19:57:28.471196+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T05:35:19.667447+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":96,"reported_epoch":124,"state":"active+clean","last_fresh":"2023-06-03T19:57:27.441028+0000","last_change":"2023-06-03T19:51:59.373777+0000","last_active":"2023-06-03T19:57:27.441028+0000","last_peered":"2023-06-03T19:57:27.441028+0000","last_clean":"2023-06-03T19:57:27.441028+0000","last_became_active":"2023-06-03T19:51:59.372988+0000","last_became_peered":"2023-06-03T19:51:59.372988+0000","last_unstale":"2023-06-03T19:57:27.441028+0000","last_undegraded":"2023-06-03T19:57:27.441028+0000","last_fullsized":"2023-06-03T19:57:27.441028+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:32:20.692043+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:32:20.692043+0000","last_clean_scrub_stamp":"2023-06-03T19:32:20.692043+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":67,"reported_epoch":125,"state":"active+clean","last_fresh":"2023-06-03T19:57:28.460195+0000","last_change":"2023-06-03T19:57:28.460195+0000","last_active":"2023-06-03T19:57:28.460195+0000","last_peered":"2023-06-03T19:57:28.460195+0000","last_clean":"2023-06-03T19:57:28.460195+0000","last_became_active":"2023-06-03T19:57:28.459823+0000","last_became_peered":"2023-06-03T19:57:28.459823+0000","last_unstale":"2023-06-03T19:57:28.460195+0000","last_undegraded":"2023-06-03T19:57:28.460195+0000","last_fullsized":"2023-06-03T19:57:28.460195+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T20:08:48.686380+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":76,"reported_epoch":125,"state":"active+clean","last_fresh":"2023-06-03T19:57:28.462134+0000","last_change":"2023-06-03T19:56:54.516859+0000","last_active":"2023-06-03T19:57:28.462134+0000","last_peered":"2023-06-03T19:57:28.462134+0000","last_clean":"2023-06-03T19:57:28.462134+0000","last_became_active":"2023-06-03T19:56:54.516521+0000","last_became_peered":"2023-06-03T19:56:54.516521+0000","last_unstale":"2023-06-03T19:57:28.462134+0000","last_undegraded":"2023-06-03T19:57:28.462134+0000","last_fullsized":"2023-06-03T19:57:28.462134+0000","mapping_epoch":119,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T20:03:25.227382+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":97,"reported_epoch":125,"state":"active+clean","last_fresh":"2023-06-03T19:57:28.449619+0000","last_change":"2023-06-03T19:56:54.504859+0000","last_active":"2023-06-03T19:57:28.449619+0000","last_peered":"2023-06-03T19:57:28.449619+0000","last_clean":"2023-06-03T19:57:28.449619+0000","last_became_active":"2023-06-03T19:56:54.504455+0000","last_became_peered":"2023-06-03T19:56:54.504455+0000","last_unstale":"2023-06-03T19:57:28.449619+0000","last_undegraded":"2023-06-03T19:57:28.449619+0000","last_fullsized":"2023-06-03T19:57:28.449619+0000","mapping_epoch":119,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:32:20.692043+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:32:20.692043+0000","last_clean_scrub_stamp":"2023-06-03T19:32:20.692043+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T06:25:45.522100+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":102,"reported_epoch":125,"state":"active+clean","last_fresh":"2023-06-03T19:57:28.458987+0000","last_change":"2023-06-03T19:56:11.754048+0000","last_active":"2023-06-03T19:57:28.458987+0000","last_peered":"2023-06-03T19:57:28.458987+0000","last_clean":"2023-06-03T19:57:28.458987+0000","last_became_active":"2023-06-03T19:56:11.753687+0000","last_became_peered":"2023-06-03T19:56:11.753687+0000","last_unstale":"2023-06-03T19:57:28.458987+0000","last_undegraded":"2023-06-03T19:57:28.458987+0000","last_fullsized":"2023-06-03T19:57:28.458987+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":115,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T23:29:41.511496+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":89,"reported_epoch":125,"state":"active+clean","last_fresh":"2023-06-03T19:57:28.459616+0000","last_change":"2023-06-03T19:51:59.396573+0000","last_active":"2023-06-03T19:57:28.459616+0000","last_peered":"2023-06-03T19:57:28.459616+0000","last_clean":"2023-06-03T19:57:28.459616+0000","last_became_active":"2023-06-03T19:51:59.381593+0000","last_became_peered":"2023-06-03T19:51:59.381593+0000","last_unstale":"2023-06-03T19:57:28.459616+0000","last_undegraded":"2023-06-03T19:57:28.459616+0000","last_fullsized":"2023-06-03T19:57:28.459616+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T21:33:23.043772+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":96,"reported_epoch":124,"state":"active+clean","last_fresh":"2023-06-03T19:57:27.439804+0000","last_change":"2023-06-03T19:56:11.752151+0000","last_active":"2023-06-03T19:57:27.439804+0000","last_peered":"2023-06-03T19:57:27.439804+0000","last_clean":"2023-06-03T19:57:27.439804+0000","last_became_active":"2023-06-03T19:56:11.751767+0000","last_became_peered":"2023-06-03T19:56:11.751767+0000","last_unstale":"2023-06-03T19:57:27.439804+0000","last_undegraded":"2023-06-03T19:57:27.439804+0000","last_fullsized":"2023-06-03T19:57:27.439804+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":115,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:32:20.692043+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:32:20.692043+0000","last_clean_scrub_stamp":"2023-06-03T19:32:20.692043+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"72'139","reported_seq":328,"reported_epoch":124,"state":"active+clean","last_fresh":"2023-06-03T19:57:27.443270+0000","last_change":"2023-06-03T19:47:06.731872+0000","last_active":"2023-06-03T19:57:27.443270+0000","last_peered":"2023-06-03T19:57:27.443270+0000","last_clean":"2023-06-03T19:57:27.443270+0000","last_became_active":"2023-06-03T19:47:06.731175+0000","last_became_peered":"2023-06-03T19:47:06.731175+0000","last_unstale":"2023-06-03T19:57:27.443270+0000","last_undegraded":"2023-06-03T19:57:27.443270+0000","last_fullsized":"2023-06-03T19:57:27.443270+0000","mapping_epoch":86,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":87,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:27:28.100808+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:27:28.100808+0000","last_clean_scrub_stamp":"2023-06-03T19:27:28.100808+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":69,"reported_epoch":124,"state":"active+clean","last_fresh":"2023-06-03T19:57:27.443994+0000","last_change":"2023-06-03T19:56:54.509571+0000","last_active":"2023-06-03T19:57:27.443994+0000","last_peered":"2023-06-03T19:57:27.443994+0000","last_clean":"2023-06-03T19:57:27.443994+0000","last_became_active":"2023-06-03T19:56:54.509213+0000","last_became_peered":"2023-06-03T19:56:54.509213+0000","last_unstale":"2023-06-03T19:57:27.443994+0000","last_undegraded":"2023-06-03T19:57:27.443994+0000","last_fullsized":"2023-06-03T19:57:27.443994+0000","mapping_epoch":119,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T21:26:56.003203+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":91,"reported_epoch":125,"state":"active+clean","last_fresh":"2023-06-03T19:57:28.459664+0000","last_change":"2023-06-03T19:57:28.459664+0000","last_active":"2023-06-03T19:57:28.459664+0000","last_peered":"2023-06-03T19:57:28.459664+0000","last_clean":"2023-06-03T19:57:28.459664+0000","last_became_active":"2023-06-03T19:57:28.459289+0000","last_became_peered":"2023-06-03T19:57:28.459289+0000","last_unstale":"2023-06-03T19:57:28.459664+0000","last_undegraded":"2023-06-03T19:57:28.459664+0000","last_fullsized":"2023-06-03T19:57:28.459664+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":111,"reported_epoch":124,"state":"active+clean","last_fresh":"2023-06-03T19:57:27.443526+0000","last_change":"2023-06-03T19:51:59.358157+0000","last_active":"2023-06-03T19:57:27.443526+0000","last_peered":"2023-06-03T19:57:27.443526+0000","last_clean":"2023-06-03T19:57:27.443526+0000","last_became_active":"2023-06-03T19:51:59.357363+0000","last_became_peered":"2023-06-03T19:51:59.357363+0000","last_unstale":"2023-06-03T19:57:27.443526+0000","last_undegraded":"2023-06-03T19:57:27.443526+0000","last_fullsized":"2023-06-03T19:57:27.443526+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:32:20.692043+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:32:20.692043+0000","last_clean_scrub_stamp":"2023-06-03T19:32:20.692043+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":94,"reported_epoch":125,"state":"active+clean","last_fresh":"2023-06-03T19:57:28.459955+0000","last_change":"2023-06-03T19:57:28.459611+0000","last_active":"2023-06-03T19:57:28.459955+0000","last_peered":"2023-06-03T19:57:28.459955+0000","last_clean":"2023-06-03T19:57:28.459955+0000","last_became_active":"2023-06-03T19:57:28.459264+0000","last_became_peered":"2023-06-03T19:57:28.459264+0000","last_unstale":"2023-06-03T19:57:28.459955+0000","last_undegraded":"2023-06-03T19:57:28.459955+0000","last_fullsized":"2023-06-03T19:57:28.459955+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T22:15:06.557231+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":95,"reported_epoch":125,"state":"active+clean","last_fresh":"2023-06-03T19:57:28.451064+0000","last_change":"2023-06-03T19:56:11.745678+0000","last_active":"2023-06-03T19:57:28.451064+0000","last_peered":"2023-06-03T19:57:28.451064+0000","last_clean":"2023-06-03T19:57:28.451064+0000","last_became_active":"2023-06-03T19:56:11.744966+0000","last_became_peered":"2023-06-03T19:56:11.744966+0000","last_unstale":"2023-06-03T19:57:28.451064+0000","last_undegraded":"2023-06-03T19:57:28.451064+0000","last_fullsized":"2023-06-03T19:57:28.451064+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":115,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T21:12:06.830295+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":111,"reported_epoch":125,"state":"active+clean","last_fresh":"2023-06-03T19:57:28.466096+0000","last_change":"2023-06-03T19:57:28.466096+0000","last_active":"2023-06-03T19:57:28.466096+0000","last_peered":"2023-06-03T19:57:28.466096+0000","last_clean":"2023-06-03T19:57:28.466096+0000","last_became_active":"2023-06-03T19:57:28.464954+0000","last_became_peered":"2023-06-03T19:57:28.464954+0000","last_unstale":"2023-06-03T19:57:28.466096+0000","last_undegraded":"2023-06-03T19:57:28.466096+0000","last_fullsized":"2023-06-03T19:57:28.466096+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:32:20.692043+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:32:20.692043+0000","last_clean_scrub_stamp":"2023-06-03T19:32:20.692043+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":102,"reported_epoch":125,"state":"active+clean","last_fresh":"2023-06-03T19:57:28.462221+0000","last_change":"2023-06-03T19:56:11.750241+0000","last_active":"2023-06-03T19:57:28.462221+0000","last_peered":"2023-06-03T19:57:28.462221+0000","last_clean":"2023-06-03T19:57:28.462221+0000","last_became_active":"2023-06-03T19:56:11.749687+0000","last_became_peered":"2023-06-03T19:56:11.749687+0000","last_unstale":"2023-06-03T19:57:28.462221+0000","last_undegraded":"2023-06-03T19:57:28.462221+0000","last_fullsized":"2023-06-03T19:57:28.462221+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":115,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T22:38:25.819929+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":65,"reported_epoch":125,"state":"active+clean","last_fresh":"2023-06-03T19:57:28.458317+0000","last_change":"2023-06-03T19:47:06.733532+0000","last_active":"2023-06-03T19:57:28.458317+0000","last_peered":"2023-06-03T19:57:28.458317+0000","last_clean":"2023-06-03T19:57:28.458317+0000","last_became_active":"2023-06-03T19:47:06.733131+0000","last_became_peered":"2023-06-03T19:47:06.733131+0000","last_unstale":"2023-06-03T19:57:28.458317+0000","last_undegraded":"2023-06-03T19:57:28.458317+0000","last_fullsized":"2023-06-03T19:57:28.458317+0000","mapping_epoch":86,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":87,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T23:46:03.788646+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.7","version":"60'2","reported_seq":74,"reported_epoch":124,"state":"active+clean","last_fresh":"2023-06-03T19:57:27.444053+0000","last_change":"2023-06-03T19:56:54.510148+0000","last_active":"2023-06-03T19:57:27.444053+0000","last_peered":"2023-06-03T19:57:27.444053+0000","last_clean":"2023-06-03T19:57:27.444053+0000","last_became_active":"2023-06-03T19:56:54.509771+0000","last_became_peered":"2023-06-03T19:56:54.509771+0000","last_unstale":"2023-06-03T19:57:27.444053+0000","last_undegraded":"2023-06-03T19:57:27.444053+0000","last_fullsized":"2023-06-03T19:57:27.444053+0000","mapping_epoch":119,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T00:52:42.807432+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":102,"reported_epoch":125,"state":"active+clean","last_fresh":"2023-06-03T19:57:28.472011+0000","last_change":"2023-06-03T19:57:28.470954+0000","last_active":"2023-06-03T19:57:28.472011+0000","last_peered":"2023-06-03T19:57:28.472011+0000","last_clean":"2023-06-03T19:57:28.472011+0000","last_became_active":"2023-06-03T19:57:28.470524+0000","last_became_peered":"2023-06-03T19:57:28.470524+0000","last_unstale":"2023-06-03T19:57:28.472011+0000","last_undegraded":"2023-06-03T19:57:28.472011+0000","last_fullsized":"2023-06-03T19:57:28.472011+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:32:20.692043+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:32:20.692043+0000","last_clean_scrub_stamp":"2023-06-03T19:32:20.692043+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T19:39:19.983271+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":72,"reported_epoch":125,"state":"active+clean","last_fresh":"2023-06-03T19:57:28.452994+0000","last_change":"2023-06-03T19:56:54.507670+0000","last_active":"2023-06-03T19:57:28.452994+0000","last_peered":"2023-06-03T19:57:28.452994+0000","last_clean":"2023-06-03T19:57:28.452994+0000","last_became_active":"2023-06-03T19:56:54.507293+0000","last_became_peered":"2023-06-03T19:56:54.507293+0000","last_unstale":"2023-06-03T19:57:28.452994+0000","last_undegraded":"2023-06-03T19:57:28.452994+0000","last_fullsized":"2023-06-03T19:57:28.452994+0000","mapping_epoch":119,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T01:49:19.966054+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":96,"reported_epoch":124,"state":"active+clean","last_fresh":"2023-06-03T19:57:27.437244+0000","last_change":"2023-06-03T19:56:11.752730+0000","last_active":"2023-06-03T19:57:27.437244+0000","last_peered":"2023-06-03T19:57:27.437244+0000","last_clean":"2023-06-03T19:57:27.437244+0000","last_became_active":"2023-06-03T19:56:11.752441+0000","last_became_peered":"2023-06-03T19:56:11.752441+0000","last_unstale":"2023-06-03T19:57:27.437244+0000","last_undegraded":"2023-06-03T19:57:27.437244+0000","last_fullsized":"2023-06-03T19:57:27.437244+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":115,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:32:20.692043+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:32:20.692043+0000","last_clean_scrub_stamp":"2023-06-03T19:32:20.692043+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"60'1","reported_seq":98,"reported_epoch":125,"state":"active+clean","last_fresh":"2023-06-03T19:57:28.464648+0000","last_change":"2023-06-03T19:57:28.463999+0000","last_active":"2023-06-03T19:57:28.464648+0000","last_peered":"2023-06-03T19:57:28.464648+0000","last_clean":"2023-06-03T19:57:28.464648+0000","last_became_active":"2023-06-03T19:57:28.463588+0000","last_became_peered":"2023-06-03T19:57:28.463588+0000","last_unstale":"2023-06-03T19:57:28.464648+0000","last_undegraded":"2023-06-03T19:57:28.464648+0000","last_fullsized":"2023-06-03T19:57:28.464648+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T21:44:11.969059+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":65,"reported_epoch":125,"state":"active+clean","last_fresh":"2023-06-03T19:57:28.456467+0000","last_change":"2023-06-03T19:57:28.456467+0000","last_active":"2023-06-03T19:57:28.456467+0000","last_peered":"2023-06-03T19:57:28.456467+0000","last_clean":"2023-06-03T19:57:28.456467+0000","last_became_active":"2023-06-03T19:57:28.456120+0000","last_became_peered":"2023-06-03T19:57:28.456120+0000","last_unstale":"2023-06-03T19:57:28.456467+0000","last_undegraded":"2023-06-03T19:57:28.456467+0000","last_fullsized":"2023-06-03T19:57:28.456467+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T07:28:14.306624+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"60'1","reported_seq":82,"reported_epoch":125,"state":"active+clean","last_fresh":"2023-06-03T19:57:28.465124+0000","last_change":"2023-06-03T19:57:28.465124+0000","last_active":"2023-06-03T19:57:28.465124+0000","last_peered":"2023-06-03T19:57:28.465124+0000","last_clean":"2023-06-03T19:57:28.465124+0000","last_became_active":"2023-06-03T19:57:28.464654+0000","last_became_peered":"2023-06-03T19:57:28.464654+0000","last_unstale":"2023-06-03T19:57:28.465124+0000","last_undegraded":"2023-06-03T19:57:28.465124+0000","last_fullsized":"2023-06-03T19:57:28.465124+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T01:12:32.305507+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":109,"reported_epoch":125,"state":"active+clean","last_fresh":"2023-06-03T19:57:28.448170+0000","last_change":"2023-06-03T19:56:54.511760+0000","last_active":"2023-06-03T19:57:28.448170+0000","last_peered":"2023-06-03T19:57:28.448170+0000","last_clean":"2023-06-03T19:57:28.448170+0000","last_became_active":"2023-06-03T19:56:54.511429+0000","last_became_peered":"2023-06-03T19:56:54.511429+0000","last_unstale":"2023-06-03T19:57:28.448170+0000","last_undegraded":"2023-06-03T19:57:28.448170+0000","last_fullsized":"2023-06-03T19:57:28.448170+0000","mapping_epoch":119,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T05:42:46.254343+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"60'2","reported_seq":109,"reported_epoch":125,"state":"active+clean","last_fresh":"2023-06-03T19:57:28.462252+0000","last_change":"2023-06-03T19:57:28.462252+0000","last_active":"2023-06-03T19:57:28.462252+0000","last_peered":"2023-06-03T19:57:28.462252+0000","last_clean":"2023-06-03T19:57:28.462252+0000","last_became_active":"2023-06-03T19:57:28.461844+0000","last_became_peered":"2023-06-03T19:57:28.461844+0000","last_unstale":"2023-06-03T19:57:28.462252+0000","last_undegraded":"2023-06-03T19:57:28.462252+0000","last_fullsized":"2023-06-03T19:57:28.462252+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":112,"reported_epoch":125,"state":"active+clean","last_fresh":"2023-06-03T19:57:28.469248+0000","last_change":"2023-06-03T19:57:28.469248+0000","last_active":"2023-06-03T19:57:28.469248+0000","last_peered":"2023-06-03T19:57:28.469248+0000","last_clean":"2023-06-03T19:57:28.469248+0000","last_became_active":"2023-06-03T19:57:28.468903+0000","last_became_peered":"2023-06-03T19:57:28.468903+0000","last_unstale":"2023-06-03T19:57:28.469248+0000","last_undegraded":"2023-06-03T19:57:28.469248+0000","last_fullsized":"2023-06-03T19:57:28.469248+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T06:08:20.956098+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"60'2","reported_seq":94,"reported_epoch":125,"state":"active+clean","last_fresh":"2023-06-03T19:57:28.462610+0000","last_change":"2023-06-03T19:57:28.462610+0000","last_active":"2023-06-03T19:57:28.462610+0000","last_peered":"2023-06-03T19:57:28.462610+0000","last_clean":"2023-06-03T19:57:28.462610+0000","last_became_active":"2023-06-03T19:57:28.460655+0000","last_became_peered":"2023-06-03T19:57:28.460655+0000","last_unstale":"2023-06-03T19:57:28.462610+0000","last_undegraded":"2023-06-03T19:57:28.462610+0000","last_fullsized":"2023-06-03T19:57:28.462610+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":112,"reported_epoch":125,"state":"active+clean","last_fresh":"2023-06-03T19:57:28.464690+0000","last_change":"2023-06-03T19:57:28.464690+0000","last_active":"2023-06-03T19:57:28.464690+0000","last_peered":"2023-06-03T19:57:28.464690+0000","last_clean":"2023-06-03T19:57:28.464690+0000","last_became_active":"2023-06-03T19:57:28.463856+0000","last_became_peered":"2023-06-03T19:57:28.463856+0000","last_unstale":"2023-06-03T19:57:28.464690+0000","last_undegraded":"2023-06-03T19:57:28.464690+0000","last_fullsized":"2023-06-03T19:57:28.464690+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T00:11:00.991088+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"60'1","reported_seq":101,"reported_epoch":125,"state":"active+clean","last_fresh":"2023-06-03T19:57:28.457091+0000","last_change":"2023-06-03T19:56:54.505096+0000","last_active":"2023-06-03T19:57:28.457091+0000","last_peered":"2023-06-03T19:57:28.457091+0000","last_clean":"2023-06-03T19:57:28.457091+0000","last_became_active":"2023-06-03T19:56:54.504255+0000","last_became_peered":"2023-06-03T19:56:54.504255+0000","last_unstale":"2023-06-03T19:57:28.457091+0000","last_undegraded":"2023-06-03T19:57:28.457091+0000","last_fullsized":"2023-06-03T19:57:28.457091+0000","mapping_epoch":119,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T19:52:25.922308+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":113,"reported_epoch":124,"state":"active+clean","last_fresh":"2023-06-03T19:57:27.449326+0000","last_change":"2023-06-03T19:56:54.503416+0000","last_active":"2023-06-03T19:57:27.449326+0000","last_peered":"2023-06-03T19:57:27.449326+0000","last_clean":"2023-06-03T19:57:27.449326+0000","last_became_active":"2023-06-03T19:56:54.503018+0000","last_became_peered":"2023-06-03T19:56:54.503018+0000","last_unstale":"2023-06-03T19:57:27.449326+0000","last_undegraded":"2023-06-03T19:57:27.449326+0000","last_fullsized":"2023-06-03T19:57:27.449326+0000","mapping_epoch":119,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T20:14:00.330616+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"60'4","reported_seq":83,"reported_epoch":125,"state":"active+clean","last_fresh":"2023-06-03T19:57:28.451241+0000","last_change":"2023-06-03T19:56:11.748116+0000","last_active":"2023-06-03T19:57:28.451241+0000","last_peered":"2023-06-03T19:57:28.451241+0000","last_clean":"2023-06-03T19:57:28.451241+0000","last_became_active":"2023-06-03T19:56:11.747636+0000","last_became_peered":"2023-06-03T19:56:11.747636+0000","last_unstale":"2023-06-03T19:57:28.451241+0000","last_undegraded":"2023-06-03T19:57:28.451241+0000","last_fullsized":"2023-06-03T19:57:28.451241+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":115,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T06:57:48.052055+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":83,"reported_epoch":125,"state":"active+clean","last_fresh":"2023-06-03T19:57:28.459234+0000","last_change":"2023-06-03T19:57:28.459234+0000","last_active":"2023-06-03T19:57:28.459234+0000","last_peered":"2023-06-03T19:57:28.459234+0000","last_clean":"2023-06-03T19:57:28.459234+0000","last_became_active":"2023-06-03T19:57:28.458896+0000","last_became_peered":"2023-06-03T19:57:28.458896+0000","last_unstale":"2023-06-03T19:57:28.459234+0000","last_undegraded":"2023-06-03T19:57:28.459234+0000","last_fullsized":"2023-06-03T19:57:28.459234+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T21:18:37.142280+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5486,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":62,"num_write_kb":30,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16458,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":30319,"internal_metadata":0},"log_size":56,"ondisk_log_size":56,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":3,"num_store_stats":3},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":48,"seq":206158430542,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":287928,"kb_used_data":1776,"kb_used_omap":0,"kb_used_meta":286144,"kb_avail":93457224,"statfs":{"total":95995035648,"available":95700197376,"internally_reserved":0,"allocated":1818624,"data_stored":1348675,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":293011456},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 3 19:58:34 2023","interfaces":[{"interface":"back","average":{"1min":0.86599999999999999,"5min":1.1080000000000001,"15min":1.2729999999999999},"min":{"1min":0.48199999999999998,"5min":0.27500000000000002,"15min":0.253},"max":{"1min":1.3180000000000001,"5min":4.806,"15min":8.6099999999999994},"last":0.79100000000000004},{"interface":"front","average":{"1min":0.91100000000000003,"5min":1.218,"15min":1.506},"min":{"1min":0.57399999999999995,"5min":0.33200000000000002,"15min":0.28499999999999998},"max":{"1min":1.2390000000000001,"5min":6.6139999999999999,"15min":15.220000000000001},"last":0.76800000000000002}]},{"osd":1,"last update":"Sat Jun 3 19:58:12 2023","interfaces":[{"interface":"back","average":{"1min":0.84999999999999998,"5min":1.3879999999999999,"15min":1.8320000000000001},"min":{"1min":0.44400000000000001,"5min":0.315,"15min":0.315},"max":{"1min":1.385,"5min":8.1750000000000007,"15min":10.211},"last":0.68400000000000005},{"interface":"front","average":{"1min":1.0640000000000001,"5min":1.415,"15min":1.623},"min":{"1min":0.75800000000000001,"5min":0.36199999999999999,"15min":0.34799999999999998},"max":{"1min":2.0790000000000002,"5min":8.7360000000000007,"15min":11.635},"last":0.93999999999999995}]},{"osd":2,"last update":"Sat Jun 3 19:58:12 2023","interfaces":[{"interface":"back","average":{"1min":1.1080000000000001,"5min":1.032,"15min":1.0189999999999999},"min":{"1min":0.66100000000000003,"5min":0.66100000000000003,"15min":0.66100000000000003},"max":{"1min":2.9969999999999999,"5min":2.9969999999999999,"15min":2.9969999999999999},"last":0.96599999999999997},{"interface":"front","average":{"1min":1.0880000000000001,"5min":1,"15min":0.98499999999999999},"min":{"1min":0.63100000000000001,"5min":0.63100000000000001,"15min":0.63100000000000001},"max":{"1min":2.0299999999999998,"5min":2.0299999999999998,"15min":2.0299999999999998},"last":0.87}]},{"osd":3,"last update":"Sat Jun 3 19:57:56 2023","interfaces":[{"interface":"back","average":{"1min":1.1559999999999999,"5min":1.1559999999999999,"15min":1.1559999999999999},"min":{"1min":0.65400000000000003,"5min":0.65400000000000003,"15min":0.65400000000000003},"max":{"1min":2.9399999999999999,"5min":2.9399999999999999,"15min":2.9399999999999999},"last":1.0640000000000001},{"interface":"front","average":{"1min":1.2,"5min":1.2,"15min":1.2},"min":{"1min":0.72399999999999998,"5min":0.72399999999999998,"15min":0.72399999999999998},"max":{"1min":3.085,"5min":3.085,"15min":3.085},"last":1.091}]},{"osd":4,"last update":"Sat Jun 3 19:58:28 2023","interfaces":[{"interface":"back","average":{"1min":1.0509999999999999,"5min":1.0509999999999999,"15min":1.0509999999999999},"min":{"1min":0.66900000000000004,"5min":0.66900000000000004,"15min":0.66900000000000004},"max":{"1min":1.552,"5min":1.552,"15min":1.552},"last":1.1739999999999999},{"interface":"front","average":{"1min":1.119,"5min":1.119,"15min":1.119},"min":{"1min":0.77100000000000002,"5min":0.77100000000000002,"15min":0.77100000000000002},"max":{"1min":1.476,"5min":1.476,"15min":1.476},"last":1.0189999999999999}]},{"osd":5,"last update":"Sat Jun 3 19:58:07 2023","interfaces":[{"interface":"back","average":{"1min":0.97999999999999998,"5min":1.1240000000000001,"15min":1.2270000000000001},"min":{"1min":0.52100000000000002,"5min":0.33200000000000002,"15min":0.30199999999999999},"max":{"1min":1.659,"5min":10.151999999999999,"15min":12.778},"last":0.89100000000000001},{"interface":"front","average":{"1min":1.1140000000000001,"5min":1.1599999999999999,"15min":1.3440000000000001},"min":{"1min":0.622,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":1.5269999999999999,"5min":5.3840000000000003,"15min":9.1669999999999998},"last":1.1930000000000001}]},{"osd":6,"last update":"Sat Jun 3 19:58:07 2023","interfaces":[{"interface":"back","average":{"1min":1.1000000000000001,"5min":1.2010000000000001,"15min":1.339},"min":{"1min":0.57699999999999996,"5min":0.36499999999999999,"15min":0.35899999999999999},"max":{"1min":1.5860000000000001,"5min":5.4429999999999996,"15min":9.3520000000000003},"last":1.149},{"interface":"front","average":{"1min":0.92900000000000005,"5min":1.202,"15min":1.2230000000000001},"min":{"1min":0.57499999999999996,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":1.4910000000000001,"5min":9.0269999999999992,"15min":9.0269999999999992},"last":1.1180000000000001}]}]},{"osd":6,"up_from":42,"seq":180388626775,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":270504,"kb_used_data":1760,"kb_used_omap":0,"kb_used_meta":268736,"kb_avail":93474648,"statfs":{"total":95995035648,"available":95718039552,"internally_reserved":0,"allocated":1802240,"data_stored":1348214,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":275185664},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 3 19:58:35 2023","interfaces":[{"interface":"back","average":{"1min":0.98499999999999999,"5min":1.5009999999999999,"15min":1.5629999999999999},"min":{"1min":0.70199999999999996,"5min":0.28199999999999997,"15min":0.27500000000000002},"max":{"1min":1.3320000000000001,"5min":8.1890000000000001,"15min":11.944000000000001},"last":1.179},{"interface":"front","average":{"1min":0.94699999999999995,"5min":1.6739999999999999,"15min":1.659},"min":{"1min":0.56100000000000005,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":1.1259999999999999,"5min":10.835000000000001,"15min":12.502000000000001},"last":0.98399999999999999}]},{"osd":1,"last update":"Sat Jun 3 19:58:10 2023","interfaces":[{"interface":"back","average":{"1min":0.94099999999999995,"5min":1.393,"15min":1.589},"min":{"1min":0.57699999999999996,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":1.5720000000000001,"5min":6.8049999999999997,"15min":6.8049999999999997},"last":0.94399999999999995},{"interface":"front","average":{"1min":0.97699999999999998,"5min":1.5649999999999999,"15min":1.2210000000000001},"min":{"1min":0.74399999999999999,"5min":0.39100000000000001,"15min":0.38500000000000001},"max":{"1min":1.3240000000000001,"5min":8.9120000000000008,"15min":8.9120000000000008},"last":0.88100000000000001}]},{"osd":2,"last update":"Sat Jun 3 19:58:12 2023","interfaces":[{"interface":"back","average":{"1min":0.89200000000000002,"5min":0.84999999999999998,"15min":0.84299999999999997},"min":{"1min":0.57399999999999995,"5min":0.57399999999999995,"15min":0.57399999999999995},"max":{"1min":1.4490000000000001,"5min":1.4490000000000001,"15min":1.4490000000000001},"last":1.01},{"interface":"front","average":{"1min":0.96699999999999997,"5min":0.94699999999999995,"15min":0.94399999999999995},"min":{"1min":0.52700000000000002,"5min":0.52700000000000002,"15min":0.52700000000000002},"max":{"1min":1.2989999999999999,"5min":1.5469999999999999,"15min":1.5469999999999999},"last":1.028}]},{"osd":3,"last update":"Sat Jun 3 19:57:59 2023","interfaces":[{"interface":"back","average":{"1min":0.85399999999999998,"5min":0.85399999999999998,"15min":0.85399999999999998},"min":{"1min":0.623,"5min":0.623,"15min":0.623},"max":{"1min":1.1819999999999999,"5min":1.1819999999999999,"15min":1.1819999999999999},"last":1.0680000000000001},{"interface":"front","average":{"1min":0.99399999999999999,"5min":0.99399999999999999,"15min":0.99399999999999999},"min":{"1min":0.65300000000000002,"5min":0.65300000000000002,"15min":0.65300000000000002},"max":{"1min":1.4279999999999999,"5min":1.4279999999999999,"15min":1.4279999999999999},"last":1.137}]},{"osd":4,"last update":"Sat Jun 3 19:58:29 2023","interfaces":[{"interface":"back","average":{"1min":0.999,"5min":0.999,"15min":0.999},"min":{"1min":0.751,"5min":0.751,"15min":0.751},"max":{"1min":1.1719999999999999,"5min":1.1719999999999999,"15min":1.1719999999999999},"last":0.90400000000000003},{"interface":"front","average":{"1min":1.181,"5min":1.181,"15min":1.181},"min":{"1min":0.92000000000000004,"5min":0.92000000000000004,"15min":0.92000000000000004},"max":{"1min":1.46,"5min":1.46,"15min":1.46},"last":0.86599999999999999}]},{"osd":5,"last update":"Sat Jun 3 19:58:25 2023","interfaces":[{"interface":"back","average":{"1min":1.1679999999999999,"5min":1.385,"15min":1.2749999999999999},"min":{"1min":0.83299999999999996,"5min":0.34100000000000003,"15min":0.30499999999999999},"max":{"1min":1.393,"5min":11.083,"15min":11.083},"last":1.087},{"interface":"front","average":{"1min":1.089,"5min":1.4099999999999999,"15min":1.3220000000000001},"min":{"1min":0.80000000000000004,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":1.367,"5min":10.154,"15min":10.154},"last":0.76500000000000001}]},{"osd":7,"last update":"Sat Jun 3 19:58:02 2023","interfaces":[{"interface":"back","average":{"1min":1.002,"5min":1.333,"15min":1.407},"min":{"1min":0.59199999999999997,"5min":0.313,"15min":0.313},"max":{"1min":1.3879999999999999,"5min":6.3029999999999999,"15min":9.5510000000000002},"last":0.79100000000000004},{"interface":"front","average":{"1min":0.95799999999999996,"5min":1.528,"15min":1.4570000000000001},"min":{"1min":0.52800000000000002,"5min":0.35099999999999998,"15min":0.29999999999999999},"max":{"1min":1.292,"5min":12.407999999999999,"15min":12.407999999999999},"last":1.111}]}]},{"osd":1,"up_from":103,"seq":442381631573,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":129700,"kb_used_data":1244,"kb_used_omap":4,"kb_used_meta":128443,"kb_avail":93615452,"statfs":{"total":95995035648,"available":95862222848,"internally_reserved":0,"allocated":1273856,"data_stored":767105,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":131526542},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 3 19:58:10 2023","interfaces":[{"interface":"back","average":{"1min":0.63100000000000001,"5min":1.2470000000000001,"15min":1.861},"min":{"1min":0.29299999999999998,"5min":0.187,"15min":0.187},"max":{"1min":1.0940000000000001,"5min":10.653,"15min":10.653},"last":0.74299999999999999},{"interface":"front","average":{"1min":0.72999999999999998,"5min":1.6220000000000001,"15min":1.9239999999999999},"min":{"1min":0.39600000000000002,"5min":0.192,"15min":0.192},"max":{"1min":1.24,"5min":13.750999999999999,"15min":13.750999999999999},"last":0.871}]},{"osd":2,"last update":"Sat Jun 3 19:58:16 2023","interfaces":[{"interface":"back","average":{"1min":0.71199999999999997,"5min":0.71499999999999997,"15min":0.71499999999999997},"min":{"1min":0.34599999999999997,"5min":0.34599999999999997,"15min":0.34599999999999997},"max":{"1min":1.135,"5min":1.135,"15min":1.135},"last":1.1539999999999999},{"interface":"front","average":{"1min":0.69799999999999995,"5min":0.73699999999999999,"15min":0.74299999999999999},"min":{"1min":0.36499999999999999,"5min":0.36499999999999999,"15min":0.36499999999999999},"max":{"1min":1.2250000000000001,"5min":1.548,"15min":1.548},"last":0.84299999999999997}]},{"osd":3,"last update":"Sat Jun 3 19:57:55 2023","interfaces":[{"interface":"back","average":{"1min":0.73799999999999999,"5min":0.73799999999999999,"15min":0.73799999999999999},"min":{"1min":0.36599999999999999,"5min":0.36599999999999999,"15min":0.36599999999999999},"max":{"1min":1.2769999999999999,"5min":1.2769999999999999,"15min":1.2769999999999999},"last":0.63400000000000001},{"interface":"front","average":{"1min":0.86899999999999999,"5min":0.86899999999999999,"15min":0.86899999999999999},"min":{"1min":0.41299999999999998,"5min":0.41299999999999998,"15min":0.41299999999999998},"max":{"1min":1.3740000000000001,"5min":1.3740000000000001,"15min":1.3740000000000001},"last":0.91600000000000004}]},{"osd":4,"last update":"Sat Jun 3 19:58:28 2023","interfaces":[{"interface":"back","average":{"1min":0.99299999999999999,"5min":0.99299999999999999,"15min":0.99299999999999999},"min":{"1min":0.80400000000000005,"5min":0.80400000000000005,"15min":0.80400000000000005},"max":{"1min":1.466,"5min":1.466,"15min":1.466},"last":1.008},{"interface":"front","average":{"1min":0.95699999999999996,"5min":0.95699999999999996,"15min":0.95699999999999996},"min":{"1min":0.66900000000000004,"5min":0.66900000000000004,"15min":0.66900000000000004},"max":{"1min":1.591,"5min":1.591,"15min":1.591},"last":1.1899999999999999}]},{"osd":5,"last update":"Sat Jun 3 19:58:10 2023","interfaces":[{"interface":"back","average":{"1min":0.97499999999999998,"5min":1.7350000000000001,"15min":2.2370000000000001},"min":{"1min":0.69499999999999995,"5min":0.379,"15min":0.373},"max":{"1min":1.867,"5min":7.9400000000000004,"15min":7.9400000000000004},"last":1.111},{"interface":"front","average":{"1min":1.0449999999999999,"5min":1.9299999999999999,"15min":2.7010000000000001},"min":{"1min":0.749,"5min":0.40000000000000002,"15min":0.40000000000000002},"max":{"1min":1.8899999999999999,"5min":13.797000000000001,"15min":13.797000000000001},"last":1.042}]},{"osd":6,"last update":"Sat Jun 3 19:58:10 2023","interfaces":[{"interface":"back","average":{"1min":1.024,"5min":1.9430000000000001,"15min":2.3460000000000001},"min":{"1min":0.65500000000000003,"5min":0.38300000000000001,"15min":0.38300000000000001},"max":{"1min":1.8169999999999999,"5min":13.941000000000001,"15min":13.941000000000001},"last":0.97499999999999998},{"interface":"front","average":{"1min":1.139,"5min":1.879,"15min":2.859},"min":{"1min":0.77300000000000002,"5min":0.38700000000000001,"15min":0.38700000000000001},"max":{"1min":1.6559999999999999,"5min":9.4060000000000006,"15min":14.847},"last":1.083}]},{"osd":7,"last update":"Sat Jun 3 19:58:10 2023","interfaces":[{"interface":"back","average":{"1min":1.099,"5min":1.6890000000000001,"15min":2.5059999999999998},"min":{"1min":0.67900000000000005,"5min":0.47899999999999998,"15min":0.46000000000000002},"max":{"1min":1.7629999999999999,"5min":7.6580000000000004,"15min":14.977},"last":1.26},{"interface":"front","average":{"1min":1.048,"5min":1.4970000000000001,"15min":1.77},"min":{"1min":0.59399999999999997,"5min":0.371,"15min":0.371},"max":{"1min":1.786,"5min":7.6020000000000003,"15min":7.6020000000000003},"last":1.242}]}]},{"osd":0,"up_from":86,"seq":369367187600,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":219664,"kb_used_data":1864,"kb_used_omap":5,"kb_used_meta":217786,"kb_avail":93525488,"statfs":{"total":95995035648,"available":95770099712,"internally_reserved":0,"allocated":1908736,"data_stored":1361683,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":223013498},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Jun 3 19:58:11 2023","interfaces":[{"interface":"back","average":{"1min":0.73799999999999999,"5min":1.4219999999999999,"15min":2.194},"min":{"1min":0.42299999999999999,"5min":0.217,"15min":0.217},"max":{"1min":1.0449999999999999,"5min":10.441000000000001,"15min":13.760999999999999},"last":0.78500000000000003},{"interface":"front","average":{"1min":0.67000000000000004,"5min":1.24,"15min":1.8120000000000001},"min":{"1min":0.39700000000000002,"5min":0.26200000000000001,"15min":0.253},"max":{"1min":1.044,"5min":9.1280000000000001,"15min":9.1280000000000001},"last":0.60699999999999998}]},{"osd":2,"last update":"Sat Jun 3 19:58:14 2023","interfaces":[{"interface":"back","average":{"1min":0.69799999999999995,"5min":0.66600000000000004,"15min":0.66100000000000003},"min":{"1min":0.32300000000000001,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":1.077,"5min":1.077,"15min":1.077},"last":0.65000000000000002},{"interface":"front","average":{"1min":0.79400000000000004,"5min":0.75600000000000001,"15min":0.75},"min":{"1min":0.48799999999999999,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":1.1890000000000001,"5min":1.1890000000000001,"15min":1.1890000000000001},"last":0.88900000000000001}]},{"osd":3,"last update":"Sat Jun 3 19:57:55 2023","interfaces":[{"interface":"back","average":{"1min":0.81599999999999995,"5min":0.81599999999999995,"15min":0.81599999999999995},"min":{"1min":0.437,"5min":0.437,"15min":0.437},"max":{"1min":1.052,"5min":1.052,"15min":1.052},"last":1.0249999999999999},{"interface":"front","average":{"1min":0.76100000000000001,"5min":0.76100000000000001,"15min":0.76100000000000001},"min":{"1min":0.40400000000000003,"5min":0.40400000000000003,"15min":0.40400000000000003},"max":{"1min":1.1819999999999999,"5min":1.1819999999999999,"15min":1.1819999999999999},"last":0.98999999999999999}]},{"osd":4,"last update":"Sat Jun 3 19:58:31 2023","interfaces":[{"interface":"back","average":{"1min":1.0960000000000001,"5min":1.0960000000000001,"15min":1.0960000000000001},"min":{"1min":0.84099999999999997,"5min":0.84099999999999997,"15min":0.84099999999999997},"max":{"1min":1.474,"5min":1.474,"15min":1.474},"last":1.3089999999999999},{"interface":"front","average":{"1min":1.1180000000000001,"5min":1.1180000000000001,"15min":1.1180000000000001},"min":{"1min":0.73099999999999998,"5min":0.73099999999999998,"15min":0.73099999999999998},"max":{"1min":1.597,"5min":1.597,"15min":1.597},"last":1.163}]},{"osd":5,"last update":"Sat Jun 3 19:58:31 2023","interfaces":[{"interface":"back","average":{"1min":0.99299999999999999,"5min":1.522,"15min":2.1280000000000001},"min":{"1min":0.623,"5min":0.35999999999999999,"15min":0.33900000000000002},"max":{"1min":1.2909999999999999,"5min":9.2059999999999995,"15min":16.312000000000001},"last":1.0880000000000001},{"interface":"front","average":{"1min":1.1279999999999999,"5min":1.722,"15min":1.8080000000000001},"min":{"1min":0.70499999999999996,"5min":0.40500000000000003,"15min":0.32500000000000001},"max":{"1min":1.5449999999999999,"5min":9.3330000000000002,"15min":24.661000000000001},"last":1.1200000000000001}]},{"osd":6,"last update":"Sat Jun 3 19:58:31 2023","interfaces":[{"interface":"back","average":{"1min":1.1599999999999999,"5min":1.7310000000000001,"15min":1.778},"min":{"1min":0.63900000000000001,"5min":0.36099999999999999,"15min":0.33500000000000002},"max":{"1min":1.538,"5min":9.3629999999999995,"15min":24.599},"last":1.196},{"interface":"front","average":{"1min":1.1839999999999999,"5min":1.6839999999999999,"15min":2.1869999999999998},"min":{"1min":0.85299999999999998,"5min":0.42999999999999999,"15min":0.42999999999999999},"max":{"1min":1.6299999999999999,"5min":10.459,"15min":13.601000000000001},"last":1.514}]},{"osd":7,"last update":"Sat Jun 3 19:58:31 2023","interfaces":[{"interface":"back","average":{"1min":1.1579999999999999,"5min":1.7170000000000001,"15min":1.7},"min":{"1min":0.66900000000000004,"5min":0.39900000000000002,"15min":0.39900000000000002},"max":{"1min":1.605,"5min":9.4190000000000005,"15min":24.634},"last":1.2629999999999999},{"interface":"front","average":{"1min":1.1639999999999999,"5min":1.5600000000000001,"15min":2.0009999999999999},"min":{"1min":0.89300000000000002,"5min":0.45800000000000002,"15min":0.40699999999999997},"max":{"1min":1.573,"5min":12.621,"15min":12.621},"last":1.4570000000000001}]}]},{"osd":2,"up_from":114,"seq":489626271779,"num_pgs":19,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":15572,"kb_used_data":1228,"kb_used_omap":7,"kb_used_meta":14328,"kb_avail":93729580,"statfs":{"total":95995035648,"available":95979089920,"internally_reserved":0,"allocated":1257472,"data_stored":764442,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8111,"internal_metadata":14671953},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 3 19:58:16 2023","interfaces":[{"interface":"back","average":{"1min":0.59399999999999997,"5min":0.61899999999999999,"15min":0.623},"min":{"1min":0.35899999999999999,"5min":0.314,"15min":0.314},"max":{"1min":0.90300000000000002,"5min":0.90300000000000002,"15min":0.90300000000000002},"last":0.57199999999999995},{"interface":"front","average":{"1min":0.71199999999999997,"5min":0.70899999999999996,"15min":0.70899999999999996},"min":{"1min":0.42099999999999999,"5min":0.42099999999999999,"15min":0.42099999999999999},"max":{"1min":1.04,"5min":1.083,"15min":1.083},"last":0.67600000000000005}]},{"osd":1,"last update":"Sat Jun 3 19:58:16 2023","interfaces":[{"interface":"back","average":{"1min":0.68100000000000005,"5min":0.71299999999999997,"15min":0.71799999999999997},"min":{"1min":0.36499999999999999,"5min":0.36499999999999999,"15min":0.36499999999999999},"max":{"1min":0.97699999999999998,"5min":1.1020000000000001,"15min":1.1020000000000001},"last":0.81200000000000006},{"interface":"front","average":{"1min":0.65700000000000003,"5min":0.67200000000000004,"15min":0.67400000000000004},"min":{"1min":0.49299999999999999,"5min":0.443,"15min":0.443},"max":{"1min":1.0580000000000001,"5min":1.0580000000000001,"15min":1.0580000000000001},"last":0.72399999999999998}]},{"osd":3,"last update":"Sat Jun 3 19:57:58 2023","interfaces":[{"interface":"back","average":{"1min":0.751,"5min":0.751,"15min":0.751},"min":{"1min":0.44700000000000001,"5min":0.44700000000000001,"15min":0.44700000000000001},"max":{"1min":1.0409999999999999,"5min":1.0409999999999999,"15min":1.0409999999999999},"last":0.76100000000000001},{"interface":"front","average":{"1min":0.75800000000000001,"5min":0.75800000000000001,"15min":0.75800000000000001},"min":{"1min":0.53900000000000003,"5min":0.53900000000000003,"15min":0.53900000000000003},"max":{"1min":0.95799999999999996,"5min":0.95799999999999996,"15min":0.95799999999999996},"last":1.056}]},{"osd":4,"last update":"Sat Jun 3 19:58:31 2023","interfaces":[{"interface":"back","average":{"1min":0.92100000000000004,"5min":0.92100000000000004,"15min":0.92100000000000004},"min":{"1min":0.70699999999999996,"5min":0.70699999999999996,"15min":0.70699999999999996},"max":{"1min":1.2090000000000001,"5min":1.2090000000000001,"15min":1.2090000000000001},"last":0.91700000000000004},{"interface":"front","average":{"1min":0.97499999999999998,"5min":0.97499999999999998,"15min":0.97499999999999998},"min":{"1min":0.68100000000000005,"5min":0.68100000000000005,"15min":0.68100000000000005},"max":{"1min":1.3069999999999999,"5min":1.3069999999999999,"15min":1.3069999999999999},"last":1.1899999999999999}]},{"osd":5,"last update":"Sat Jun 3 19:58:16 2023","interfaces":[{"interface":"back","average":{"1min":1.0049999999999999,"5min":1.0309999999999999,"15min":1.0349999999999999},"min":{"1min":0.77200000000000002,"5min":0.65500000000000003,"15min":0.65500000000000003},"max":{"1min":1.3979999999999999,"5min":1.3979999999999999,"15min":1.3979999999999999},"last":1.004},{"interface":"front","average":{"1min":1.028,"5min":1.044,"15min":1.0469999999999999},"min":{"1min":0.65100000000000002,"5min":0.65100000000000002,"15min":0.65100000000000002},"max":{"1min":1.335,"5min":1.369,"15min":1.369},"last":1.1659999999999999}]},{"osd":6,"last update":"Sat Jun 3 19:58:16 2023","interfaces":[{"interface":"back","average":{"1min":1.0409999999999999,"5min":1.036,"15min":1.0349999999999999},"min":{"1min":0.71699999999999997,"5min":0.71699999999999997,"15min":0.71699999999999997},"max":{"1min":1.3899999999999999,"5min":1.3899999999999999,"15min":1.3899999999999999},"last":0.94499999999999995},{"interface":"front","average":{"1min":1.0780000000000001,"5min":1.052,"15min":1.048},"min":{"1min":0.73899999999999999,"5min":0.68100000000000005,"15min":0.68100000000000005},"max":{"1min":1.3919999999999999,"5min":1.508,"15min":1.508},"last":1.222}]},{"osd":7,"last update":"Sat Jun 3 19:58:16 2023","interfaces":[{"interface":"back","average":{"1min":1.1160000000000001,"5min":1.167,"15min":1.175},"min":{"1min":0.81999999999999995,"5min":0.77100000000000002,"15min":0.77100000000000002},"max":{"1min":1.456,"5min":1.456,"15min":1.456},"last":1.1339999999999999},{"interface":"front","average":{"1min":1.1220000000000001,"5min":1.151,"15min":1.1559999999999999},"min":{"1min":0.78600000000000003,"5min":0.78600000000000003,"15min":0.78600000000000003},"max":{"1min":1.3879999999999999,"5min":1.6930000000000001,"15min":1.6930000000000001},"last":1.268}]}]},{"osd":3,"up_from":119,"seq":511101108250,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":18108,"kb_used_data":1204,"kb_used_omap":2,"kb_used_meta":16893,"kb_avail":93727044,"statfs":{"total":95995035648,"available":95976493056,"internally_reserved":0,"allocated":1232896,"data_stored":760088,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":17298485},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 3 19:57:55 2023","interfaces":[{"interface":"back","average":{"1min":0.63200000000000001,"5min":0.63200000000000001,"15min":0.63200000000000001},"min":{"1min":0.433,"5min":0.433,"15min":0.433},"max":{"1min":0.84699999999999998,"5min":0.84699999999999998,"15min":0.84699999999999998},"last":0.878},{"interface":"front","average":{"1min":0.73099999999999998,"5min":0.73099999999999998,"15min":0.73099999999999998},"min":{"1min":0.40699999999999997,"5min":0.40699999999999997,"15min":0.40699999999999997},"max":{"1min":1,"5min":1,"15min":1},"last":0.79200000000000004}]},{"osd":1,"last update":"Sat Jun 3 19:57:55 2023","interfaces":[{"interface":"back","average":{"1min":0.71099999999999997,"5min":0.71099999999999997,"15min":0.71099999999999997},"min":{"1min":0.39300000000000002,"5min":0.39300000000000002,"15min":0.39300000000000002},"max":{"1min":1.028,"5min":1.028,"15min":1.028},"last":0.83499999999999996},{"interface":"front","average":{"1min":0.76100000000000001,"5min":0.76100000000000001,"15min":0.76100000000000001},"min":{"1min":0.46800000000000003,"5min":0.46800000000000003,"15min":0.46800000000000003},"max":{"1min":1.01,"5min":1.01,"15min":1.01},"last":1.018}]},{"osd":2,"last update":"Sat Jun 3 19:57:55 2023","interfaces":[{"interface":"back","average":{"1min":0.64400000000000002,"5min":0.64400000000000002,"15min":0.64400000000000002},"min":{"1min":0.40500000000000003,"5min":0.40500000000000003,"15min":0.40500000000000003},"max":{"1min":0.89100000000000001,"5min":0.89100000000000001,"15min":0.89100000000000001},"last":0.68300000000000005},{"interface":"front","average":{"1min":0.76300000000000001,"5min":0.76300000000000001,"15min":0.76300000000000001},"min":{"1min":0.501,"5min":0.501,"15min":0.501},"max":{"1min":1.1140000000000001,"5min":1.1140000000000001,"15min":1.1140000000000001},"last":1.046}]},{"osd":4,"last update":"Sat Jun 3 19:58:28 2023","interfaces":[{"interface":"back","average":{"1min":1,"5min":1,"15min":1},"min":{"1min":0.71499999999999997,"5min":0.71499999999999997,"15min":0.71499999999999997},"max":{"1min":1.3819999999999999,"5min":1.3819999999999999,"15min":1.3819999999999999},"last":0.94899999999999995},{"interface":"front","average":{"1min":1.0109999999999999,"5min":1.0109999999999999,"15min":1.0109999999999999},"min":{"1min":0.78200000000000003,"5min":0.78200000000000003,"15min":0.78200000000000003},"max":{"1min":1.2689999999999999,"5min":1.2689999999999999,"15min":1.2689999999999999},"last":1.1279999999999999}]},{"osd":5,"last update":"Sat Jun 3 19:57:55 2023","interfaces":[{"interface":"back","average":{"1min":0.98699999999999999,"5min":0.98699999999999999,"15min":0.98699999999999999},"min":{"1min":0.73499999999999999,"5min":0.73499999999999999,"15min":0.73499999999999999},"max":{"1min":1.429,"5min":1.429,"15min":1.429},"last":1.0760000000000001},{"interface":"front","average":{"1min":1.0249999999999999,"5min":1.0249999999999999,"15min":1.0249999999999999},"min":{"1min":0.754,"5min":0.754,"15min":0.754},"max":{"1min":1.5900000000000001,"5min":1.5900000000000001,"15min":1.5900000000000001},"last":0.997}]},{"osd":6,"last update":"Sat Jun 3 19:57:55 2023","interfaces":[{"interface":"back","average":{"1min":1.0329999999999999,"5min":1.0329999999999999,"15min":1.0329999999999999},"min":{"1min":0.67100000000000004,"5min":0.67100000000000004,"15min":0.67100000000000004},"max":{"1min":1.6559999999999999,"5min":1.6559999999999999,"15min":1.6559999999999999},"last":1.1000000000000001},{"interface":"front","average":{"1min":1.0580000000000001,"5min":1.0580000000000001,"15min":1.0580000000000001},"min":{"1min":0.752,"5min":0.752,"15min":0.752},"max":{"1min":1.3340000000000001,"5min":1.3340000000000001,"15min":1.3340000000000001},"last":1.2170000000000001}]},{"osd":7,"last update":"Sat Jun 3 19:57:55 2023","interfaces":[{"interface":"back","average":{"1min":1.044,"5min":1.044,"15min":1.044},"min":{"1min":0.66600000000000004,"5min":0.66600000000000004,"15min":0.66600000000000004},"max":{"1min":1.3300000000000001,"5min":1.3300000000000001,"15min":1.3300000000000001},"last":1.153},{"interface":"front","average":{"1min":1.028,"5min":1.028,"15min":1.028},"min":{"1min":0.69599999999999995,"5min":0.69599999999999995,"15min":0.69599999999999995},"max":{"1min":1.4610000000000001,"5min":1.4610000000000001,"15min":1.4610000000000001},"last":1.2829999999999999}]}]},{"osd":4,"up_from":124,"seq":532575944723,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":15492,"kb_used_data":1212,"kb_used_omap":10,"kb_used_meta":14261,"kb_avail":93729660,"statfs":{"total":95995035648,"available":95979171840,"internally_reserved":0,"allocated":1241088,"data_stored":762768,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":10652,"internal_metadata":14603876},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 3 19:58:29 2023","interfaces":[{"interface":"back","average":{"1min":0.97499999999999998,"5min":0.97499999999999998,"15min":0.97499999999999998},"min":{"1min":0.55800000000000005,"5min":0.55800000000000005,"15min":0.55800000000000005},"max":{"1min":1.3240000000000001,"5min":1.3240000000000001,"15min":1.3240000000000001},"last":0.92200000000000004},{"interface":"front","average":{"1min":1.0309999999999999,"5min":1.0309999999999999,"15min":1.0309999999999999},"min":{"1min":0.78200000000000003,"5min":0.78200000000000003,"15min":0.78200000000000003},"max":{"1min":1.2250000000000001,"5min":1.2250000000000001,"15min":1.2250000000000001},"last":1.2549999999999999}]},{"osd":1,"last update":"Sat Jun 3 19:58:29 2023","interfaces":[{"interface":"back","average":{"1min":0.871,"5min":0.871,"15min":0.871},"min":{"1min":0.60999999999999999,"5min":0.60999999999999999,"15min":0.60999999999999999},"max":{"1min":1.2549999999999999,"5min":1.2549999999999999,"15min":1.2549999999999999},"last":1.165},{"interface":"front","average":{"1min":0.86299999999999999,"5min":0.86299999999999999,"15min":0.86299999999999999},"min":{"1min":0.54900000000000004,"5min":0.54900000000000004,"15min":0.54900000000000004},"max":{"1min":1.278,"5min":1.278,"15min":1.278},"last":0.80700000000000005}]},{"osd":2,"last update":"Sat Jun 3 19:58:29 2023","interfaces":[{"interface":"back","average":{"1min":0.871,"5min":0.871,"15min":0.871},"min":{"1min":0.52800000000000002,"5min":0.52800000000000002,"15min":0.52800000000000002},"max":{"1min":1.196,"5min":1.196,"15min":1.196},"last":1.0880000000000001},{"interface":"front","average":{"1min":0.93799999999999994,"5min":0.93799999999999994,"15min":0.93799999999999994},"min":{"1min":0.63700000000000001,"5min":0.63700000000000001,"15min":0.63700000000000001},"max":{"1min":1.2769999999999999,"5min":1.2769999999999999,"15min":1.2769999999999999},"last":0.84199999999999997}]},{"osd":3,"last update":"Sat Jun 3 19:58:29 2023","interfaces":[{"interface":"back","average":{"1min":0.96199999999999997,"5min":0.96199999999999997,"15min":0.96199999999999997},"min":{"1min":0.68700000000000006,"5min":0.68700000000000006,"15min":0.68700000000000006},"max":{"1min":1.238,"5min":1.238,"15min":1.238},"last":1.03},{"interface":"front","average":{"1min":1.056,"5min":1.056,"15min":1.056},"min":{"1min":0.71899999999999997,"5min":0.71899999999999997,"15min":0.71899999999999997},"max":{"1min":1.28,"5min":1.28,"15min":1.28},"last":0.71499999999999997}]},{"osd":5,"last update":"Sat Jun 3 19:58:29 2023","interfaces":[{"interface":"back","average":{"1min":0.95299999999999996,"5min":0.95299999999999996,"15min":0.95299999999999996},"min":{"1min":0.72399999999999998,"5min":0.72399999999999998,"15min":0.72399999999999998},"max":{"1min":1.234,"5min":1.234,"15min":1.234},"last":0.96999999999999997},{"interface":"front","average":{"1min":0.98799999999999999,"5min":0.98799999999999999,"15min":0.98799999999999999},"min":{"1min":0.77200000000000002,"5min":0.77200000000000002,"15min":0.77200000000000002},"max":{"1min":1.24,"5min":1.24,"15min":1.24},"last":1.2270000000000001}]},{"osd":6,"last update":"Sat Jun 3 19:58:29 2023","interfaces":[{"interface":"back","average":{"1min":1.0449999999999999,"5min":1.0449999999999999,"15min":1.0449999999999999},"min":{"1min":0.51300000000000001,"5min":0.51300000000000001,"15min":0.51300000000000001},"max":{"1min":1.323,"5min":1.323,"15min":1.323},"last":1.1259999999999999},{"interface":"front","average":{"1min":1.079,"5min":1.079,"15min":1.079},"min":{"1min":0.73399999999999999,"5min":0.73399999999999999,"15min":0.73399999999999999},"max":{"1min":1.494,"5min":1.494,"15min":1.494},"last":1.2889999999999999}]},{"osd":7,"last update":"Sat Jun 3 19:58:29 2023","interfaces":[{"interface":"back","average":{"1min":1.1379999999999999,"5min":1.1379999999999999,"15min":1.1379999999999999},"min":{"1min":0.77100000000000002,"5min":0.77100000000000002,"15min":0.77100000000000002},"max":{"1min":1.4490000000000001,"5min":1.4490000000000001,"15min":1.4490000000000001},"last":1.339},{"interface":"front","average":{"1min":1.171,"5min":1.171,"15min":1.171},"min":{"1min":0.90300000000000002,"5min":0.90300000000000002,"15min":0.90300000000000002},"max":{"1min":1.383,"5min":1.383,"15min":1.383},"last":1.1990000000000001}]}]},{"osd":5,"up_from":36,"seq":154618823009,"num_pgs":21,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":358044,"kb_used_data":1172,"kb_used_omap":0,"kb_used_meta":356864,"kb_avail":93387108,"statfs":{"total":95995035648,"available":95628398592,"internally_reserved":0,"allocated":1200128,"data_stored":753684,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":365428736},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 3 19:58:19 2023","interfaces":[{"interface":"back","average":{"1min":0.91200000000000003,"5min":1.3029999999999999,"15min":1.4890000000000001},"min":{"1min":0.63600000000000001,"5min":0.26700000000000002,"15min":0.26600000000000001},"max":{"1min":1.147,"5min":7.54,"15min":12.489000000000001},"last":1.0529999999999999},{"interface":"front","average":{"1min":0.98199999999999998,"5min":1.6259999999999999,"15min":1.7310000000000001},"min":{"1min":0.82699999999999996,"5min":0.37,"15min":0.31},"max":{"1min":1.133,"5min":10.403,"15min":12.542999999999999},"last":0.871}]},{"osd":1,"last update":"Sat Jun 3 19:58:14 2023","interfaces":[{"interface":"back","average":{"1min":0.998,"5min":1.6719999999999999,"15min":1.8500000000000001},"min":{"1min":0.76100000000000001,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":1.4059999999999999,"5min":11.27,"15min":11.27},"last":0.82999999999999996},{"interface":"front","average":{"1min":0.86099999999999999,"5min":1.7849999999999999,"15min":1.9330000000000001},"min":{"1min":0.56599999999999995,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":1.0860000000000001,"5min":11.34,"15min":11.34},"last":0.70499999999999996}]},{"osd":2,"last update":"Sat Jun 3 19:58:14 2023","interfaces":[{"interface":"back","average":{"1min":0.91500000000000004,"5min":0.876,"15min":0.87},"min":{"1min":0.66500000000000004,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":1.169,"5min":1.169,"15min":1.169},"last":1.2749999999999999},{"interface":"front","average":{"1min":0.94399999999999995,"5min":0.96999999999999997,"15min":0.97399999999999998},"min":{"1min":0.73399999999999999,"5min":0.48799999999999999,"15min":0.48799999999999999},"max":{"1min":1.232,"5min":1.335,"15min":1.335},"last":0.98099999999999998}]},{"osd":3,"last update":"Sat Jun 3 19:57:53 2023","interfaces":[{"interface":"back","average":{"1min":1.018,"5min":1.018,"15min":1.018},"min":{"1min":0.38,"5min":0.38,"15min":0.38},"max":{"1min":1.2709999999999999,"5min":1.2709999999999999,"15min":1.2709999999999999},"last":1.02},{"interface":"front","average":{"1min":0.99099999999999999,"5min":0.99099999999999999,"15min":0.99099999999999999},"min":{"1min":0.50700000000000001,"5min":0.50700000000000001,"15min":0.50700000000000001},"max":{"1min":1.363,"5min":1.363,"15min":1.363},"last":1.1859999999999999}]},{"osd":4,"last update":"Sat Jun 3 19:58:30 2023","interfaces":[{"interface":"back","average":{"1min":1.002,"5min":1.002,"15min":1.002},"min":{"1min":0.60299999999999998,"5min":0.60299999999999998,"15min":0.60299999999999998},"max":{"1min":1.3500000000000001,"5min":1.3500000000000001,"15min":1.3500000000000001},"last":1.22},{"interface":"front","average":{"1min":1.1879999999999999,"5min":1.1879999999999999,"15min":1.1879999999999999},"min":{"1min":1.022,"5min":1.022,"15min":1.022},"max":{"1min":1.522,"5min":1.522,"15min":1.522},"last":1.431}]},{"osd":6,"last update":"Sat Jun 3 19:58:19 2023","interfaces":[{"interface":"back","average":{"1min":1.1000000000000001,"5min":1.6419999999999999,"15min":1.1970000000000001},"min":{"1min":0.80900000000000005,"5min":0.33000000000000002,"15min":0.33000000000000002},"max":{"1min":1.474,"5min":12.731999999999999,"15min":12.731999999999999},"last":1.123},{"interface":"front","average":{"1min":1.153,"5min":1.698,"15min":1.327},"min":{"1min":0.88700000000000001,"5min":0.42899999999999999,"15min":0.32800000000000001},"max":{"1min":1.4339999999999999,"5min":12.779,"15min":12.779},"last":1.304}]},{"osd":7,"last update":"Sat Jun 3 19:58:01 2023","interfaces":[{"interface":"back","average":{"1min":1.0469999999999999,"5min":1.806,"15min":1.3759999999999999},"min":{"1min":0.47699999999999998,"5min":0.45700000000000002,"15min":0.379},"max":{"1min":1.371,"5min":11.297000000000001,"15min":11.297000000000001},"last":1.3799999999999999},{"interface":"front","average":{"1min":0.97199999999999998,"5min":1.4139999999999999,"15min":1.216},"min":{"1min":0.39600000000000002,"5min":0.39600000000000002,"15min":0.32100000000000001},"max":{"1min":1.26,"5min":7.4720000000000004,"15min":7.4720000000000004},"last":1.357}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6928,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4862,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":10652,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4786,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4932,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-06-03T19:58:57.550 INFO:tasks.cephadm.ceph_manager.ceph:clean! 2023-06-03T19:58:57.550 INFO:tasks.ceph:Waiting until ceph cluster ceph is healthy... 2023-06-03T19:58:57.551 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy 2023-06-03T19:58:57.551 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph health --format=json 2023-06-03T19:58:58.029 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:58:57 smithi089 bash[78452]: cluster 1685822336.3927877 mgr.y (mgr.34104) 676 : cluster 0 pgmap v583: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-06-03T19:58:58.029 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:58:57 smithi089 bash[78452]: audit 1685822336.623824 mgr.y (mgr.34104) 677 : audit 0 from='client.45620 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-06-03T19:58:58.029 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:58:57 smithi089 bash[75068]: cluster 1685822336.3927877 mgr.y (mgr.34104) 676 : cluster 0 pgmap v583: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-06-03T19:58:58.030 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:58:57 smithi089 bash[75068]: audit 1685822336.623824 mgr.y (mgr.34104) 677 : audit 0 from='client.45620 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-06-03T19:58:58.084 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:58:57 smithi164 bash[59462]: cluster 1685822336.3927877 mgr.y (mgr.34104) 676 : cluster 0 pgmap v583: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-06-03T19:58:58.085 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:58:57 smithi164 bash[59462]: audit 1685822336.623824 mgr.y (mgr.34104) 677 : audit 0 from='client.45620 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-06-03T19:59:00.029 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:58:59 smithi089 bash[78452]: cluster 1685822338.3936985 mgr.y (mgr.34104) 678 : cluster 0 pgmap v584: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-06-03T19:59:00.029 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:58:59 smithi089 bash[75068]: cluster 1685822338.3936985 mgr.y (mgr.34104) 678 : cluster 0 pgmap v584: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-06-03T19:59:00.084 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:58:59 smithi164 bash[59462]: cluster 1685822338.3936985 mgr.y (mgr.34104) 678 : cluster 0 pgmap v584: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-06-03T19:59:02.029 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:59:01 smithi089 bash[75068]: cluster 1685822340.394273 mgr.y (mgr.34104) 679 : cluster 0 pgmap v585: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-06-03T19:59:02.029 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:59:01 smithi089 bash[78452]: cluster 1685822340.394273 mgr.y (mgr.34104) 679 : cluster 0 pgmap v585: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-06-03T19:59:02.084 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:59:01 smithi164 bash[59462]: cluster 1685822340.394273 mgr.y (mgr.34104) 679 : cluster 0 pgmap v585: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-06-03T19:59:02.399 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/mon.a/config 2023-06-03T19:59:03.922 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:59:03 smithi089 bash[75068]: cluster 1685822342.3949587 mgr.y (mgr.34104) 680 : cluster 0 pgmap v586: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-06-03T19:59:03.923 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:59:03 smithi089 bash[78452]: cluster 1685822342.3949587 mgr.y (mgr.34104) 680 : cluster 0 pgmap v586: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-06-03T19:59:04.084 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:59:03 smithi164 bash[59462]: cluster 1685822342.3949587 mgr.y (mgr.34104) 680 : cluster 0 pgmap v586: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-06-03T19:59:05.901 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:59:05 smithi089 bash[75068]: cluster 1685822344.395931 mgr.y (mgr.34104) 681 : cluster 0 pgmap v587: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-06-03T19:59:05.901 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:59:05 smithi089 bash[78452]: cluster 1685822344.395931 mgr.y (mgr.34104) 681 : cluster 0 pgmap v587: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-06-03T19:59:06.084 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:59:05 smithi164 bash[59462]: cluster 1685822344.395931 mgr.y (mgr.34104) 681 : cluster 0 pgmap v587: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-06-03T19:59:06.308 INFO:teuthology.orchestra.run.smithi089.stdout: 2023-06-03T19:59:06.308 INFO:teuthology.orchestra.run.smithi089.stdout:{"status":"HEALTH_OK","checks":{},"mutes":[]} 2023-06-03T19:59:06.508 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 19:59:06 smithi089 bash[71163]: ::ffff:172.21.15.164 - - [03/Jun/2023:19:59:06] "GET /metrics HTTP/1.1" 200 225310 "" "Prometheus/2.33.4" 2023-06-03T19:59:06.779 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:59:06 smithi089 bash[75068]: audit 1685822346.3036146 mon.a (mon.0) 644 : audit 0 from='client.? 172.21.15.89:0/2978543227' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2023-06-03T19:59:06.779 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:59:06 smithi089 bash[78452]: audit 1685822346.3036146 mon.a (mon.0) 644 : audit 0 from='client.? 172.21.15.89:0/2978543227' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2023-06-03T19:59:07.084 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:59:06 smithi164 bash[59462]: audit 1685822346.3036146 mon.a (mon.0) 644 : audit 0 from='client.? 172.21.15.89:0/2978543227' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2023-06-03T19:59:07.210 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy done 2023-06-03T19:59:07.210 INFO:teuthology.run_tasks:Running task print... 2023-06-03T19:59:07.221 INFO:teuthology.task.print:**** applying stress + thrashing to mixed-version cluster **** 2023-06-03T19:59:07.222 INFO:teuthology.run_tasks:Running task parallel... 2023-06-03T19:59:07.271 INFO:teuthology.task.parallel:starting parallel... 2023-06-03T19:59:07.271 INFO:teuthology.task.parallel:In parallel, running task sequential... 2023-06-03T19:59:07.272 INFO:teuthology.task.sequential:In sequential, running task thrashosds... 2023-06-03T19:59:07.274 INFO:tasks.thrashosds:config is {'aggressive_pg_num_changes': False, 'chance_force_recovery': 0, 'chance_pgnum_grow': 1, 'chance_pgpnum_fix': 1, 'chance_thrash_cluster_full': 0, 'chance_thrash_pg_upmap': 0, 'chance_thrash_pg_upmap_items': 0, 'disable_objectstore_tool_tests': True, 'timeout': 1200, 'sighup_delay': 0.1, 'optrack_toggle_delay': 2.0, 'dump_ops_enable': 'true', 'noscrub_toggle_delay': 2.0, 'random_eio': 0.0} 2023-06-03T19:59:07.274 INFO:tasks.thrashosds:overrides is {'admin_socket': {'branch': 'reef'}, 'ceph': {'conf': {'global': {'mon election default strategy': 1}, 'mgr': {'debug mgr': 20, 'debug ms': 1}, 'mon': {'debug mon': 20, 'debug ms': 1, 'debug paxos': 20}, 'osd': {'debug ms': 1, 'debug osd': 20, 'osd shutdown pgref assert': True}}, 'create_rbd_pool': True, 'flavor': 'default', 'log-ignorelist': ['\\(MDS_ALL_DOWN\\)', '\\(MDS_UP_LESS_THAN_MAX\\)'], 'sha1': '8e24bcf462a1d94c2e12aa4e26419634883a1e88'}, 'ceph-deploy': {'conf': {'client': {'log file': '/var/log/ceph/ceph-$name.$pid.log'}, 'mon': {'osd default pool size': 2}}}, 'install': {'ceph': {'flavor': 'default', 'sha1': '8e24bcf462a1d94c2e12aa4e26419634883a1e88'}}, 'kernel': {'hwe': True}, 'workunit': {'branch': 'reef', 'sha1': 'dbb22f29210f43ee95e67f3b727f470d3faecd40'}} 2023-06-03T19:59:07.274 INFO:tasks.thrashosds:config is {'aggressive_pg_num_changes': False, 'chance_force_recovery': 0, 'chance_pgnum_grow': 1, 'chance_pgpnum_fix': 1, 'chance_thrash_cluster_full': 0, 'chance_thrash_pg_upmap': 0, 'chance_thrash_pg_upmap_items': 0, 'disable_objectstore_tool_tests': True, 'timeout': 1200, 'sighup_delay': 0.1, 'optrack_toggle_delay': 2.0, 'dump_ops_enable': 'true', 'noscrub_toggle_delay': 2.0, 'random_eio': 0.0} 2023-06-03T19:59:07.275 INFO:tasks.thrashosds:Beginning thrashosds... 2023-06-03T19:59:07.275 INFO:tasks.cephadm.ceph_manager.ceph:waiting for clean 2023-06-03T19:59:07.275 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph pg dump --format=json 2023-06-03T19:59:08.029 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:59:07 smithi089 bash[75068]: cluster 1685822346.3964074 mgr.y (mgr.34104) 682 : cluster 0 pgmap v588: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-06-03T19:59:08.029 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:59:07 smithi089 bash[75068]: audit 1685822347.0924277 mon.c (mon.2) 367 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T19:59:08.029 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:59:07 smithi089 bash[78452]: cluster 1685822346.3964074 mgr.y (mgr.34104) 682 : cluster 0 pgmap v588: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-06-03T19:59:08.029 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:59:07 smithi089 bash[78452]: audit 1685822347.0924277 mon.c (mon.2) 367 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T19:59:08.084 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:59:07 smithi164 bash[59462]: cluster 1685822346.3964074 mgr.y (mgr.34104) 682 : cluster 0 pgmap v588: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-06-03T19:59:08.084 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:59:07 smithi164 bash[59462]: audit 1685822347.0924277 mon.c (mon.2) 367 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T19:59:10.028 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:59:09 smithi089 bash[75068]: cluster 1685822348.3973796 mgr.y (mgr.34104) 683 : cluster 0 pgmap v589: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-06-03T19:59:10.029 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:59:09 smithi089 bash[78452]: cluster 1685822348.3973796 mgr.y (mgr.34104) 683 : cluster 0 pgmap v589: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-06-03T19:59:10.084 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:59:09 smithi164 bash[59462]: cluster 1685822348.3973796 mgr.y (mgr.34104) 683 : cluster 0 pgmap v589: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-06-03T19:59:12.029 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:59:11 smithi089 bash[78452]: cluster 1685822350.398066 mgr.y (mgr.34104) 684 : cluster 0 pgmap v590: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-06-03T19:59:12.029 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:59:11 smithi089 bash[75068]: cluster 1685822350.398066 mgr.y (mgr.34104) 684 : cluster 0 pgmap v590: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-06-03T19:59:12.084 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:59:11 smithi164 bash[59462]: cluster 1685822350.398066 mgr.y (mgr.34104) 684 : cluster 0 pgmap v590: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-06-03T19:59:12.091 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/mon.a/config 2023-06-03T19:59:14.029 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:59:13 smithi089 bash[78452]: cluster 1685822352.3988454 mgr.y (mgr.34104) 685 : cluster 0 pgmap v591: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-06-03T19:59:14.029 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:59:13 smithi089 bash[75068]: cluster 1685822352.3988454 mgr.y (mgr.34104) 685 : cluster 0 pgmap v591: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-06-03T19:59:14.084 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:59:13 smithi164 bash[59462]: cluster 1685822352.3988454 mgr.y (mgr.34104) 685 : cluster 0 pgmap v591: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-06-03T19:59:15.778 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:59:15 smithi089 bash[78452]: cluster 1685822354.3994281 mgr.y (mgr.34104) 686 : cluster 0 pgmap v592: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-06-03T19:59:15.779 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:59:15 smithi089 bash[75068]: cluster 1685822354.3994281 mgr.y (mgr.34104) 686 : cluster 0 pgmap v592: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-06-03T19:59:15.866 INFO:teuthology.orchestra.run.smithi089.stdout: 2023-06-03T19:59:15.870 INFO:teuthology.orchestra.run.smithi089.stderr:dumped all 2023-06-03T19:59:16.084 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:59:15 smithi164 bash[59462]: cluster 1685822354.3994281 mgr.y (mgr.34104) 686 : cluster 0 pgmap v592: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-06-03T19:59:16.278 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 19:59:16 smithi089 bash[71163]: ::ffff:172.21.15.164 - - [03/Jun/2023:19:59:16] "GET /metrics HTTP/1.1" 200 225315 "" "Prometheus/2.33.4" 2023-06-03T19:59:16.712 INFO:teuthology.orchestra.run.smithi089.stdout:{"pg_ready":true,"pg_map":{"version":592,"stamp":"2023-06-03T19:59:14.399053+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":595854,"num_objects":24,"num_object_clones":0,"num_object_copies":72,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":24,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":261,"num_write_kb":2446,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":195,"ondisk_log_size":195,"up":219,"acting":219,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":219,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":1315012,"kb_used_data":11460,"kb_used_omap":30,"kb_used_meta":1303457,"kb_avail":748646204,"statfs":{"total":767960285184,"available":766613712896,"internally_reserved":0,"allocated":11735040,"data_stored":7866659,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":31502,"internal_metadata":1334740210},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.004556"},"pg_stats":[{"pgid":"3.1a","version":"60'1","reported_seq":110,"reported_epoch":125,"state":"active+clean","last_fresh":"2023-06-03T19:57:28.467559+0000","last_change":"2023-06-03T19:57:28.467559+0000","last_active":"2023-06-03T19:57:28.467559+0000","last_peered":"2023-06-03T19:57:28.467559+0000","last_clean":"2023-06-03T19:57:28.467559+0000","last_became_active":"2023-06-03T19:57:28.467077+0000","last_became_peered":"2023-06-03T19:57:28.467077+0000","last_unstale":"2023-06-03T19:57:28.467559+0000","last_undegraded":"2023-06-03T19:57:28.467559+0000","last_fullsized":"2023-06-03T19:57:28.467559+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T21:46:02.795576+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":103,"reported_epoch":125,"state":"active+clean","last_fresh":"2023-06-03T19:57:28.470287+0000","last_change":"2023-06-03T19:57:28.470287+0000","last_active":"2023-06-03T19:57:28.470287+0000","last_peered":"2023-06-03T19:57:28.470287+0000","last_clean":"2023-06-03T19:57:28.470287+0000","last_became_active":"2023-06-03T19:57:28.469477+0000","last_became_peered":"2023-06-03T19:57:28.469477+0000","last_unstale":"2023-06-03T19:57:28.470287+0000","last_undegraded":"2023-06-03T19:57:28.470287+0000","last_fullsized":"2023-06-03T19:57:28.470287+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T06:48:19.229190+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":117,"reported_epoch":125,"state":"active+clean","last_fresh":"2023-06-03T19:57:28.473690+0000","last_change":"2023-06-03T19:57:28.473350+0000","last_active":"2023-06-03T19:57:28.473690+0000","last_peered":"2023-06-03T19:57:28.473690+0000","last_clean":"2023-06-03T19:57:28.473690+0000","last_became_active":"2023-06-03T19:57:28.472757+0000","last_became_peered":"2023-06-03T19:57:28.472757+0000","last_unstale":"2023-06-03T19:57:28.473690+0000","last_undegraded":"2023-06-03T19:57:28.473690+0000","last_fullsized":"2023-06-03T19:57:28.473690+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T19:58:10.432188+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":81,"reported_epoch":125,"state":"active+clean","last_fresh":"2023-06-03T19:57:28.450909+0000","last_change":"2023-06-03T19:56:54.507653+0000","last_active":"2023-06-03T19:57:28.450909+0000","last_peered":"2023-06-03T19:57:28.450909+0000","last_clean":"2023-06-03T19:57:28.450909+0000","last_became_active":"2023-06-03T19:56:54.507350+0000","last_became_peered":"2023-06-03T19:56:54.507350+0000","last_unstale":"2023-06-03T19:57:28.450909+0000","last_undegraded":"2023-06-03T19:57:28.450909+0000","last_fullsized":"2023-06-03T19:57:28.450909+0000","mapping_epoch":119,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T22:33:20.065844+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":104,"reported_epoch":124,"state":"active+clean","last_fresh":"2023-06-03T19:57:27.446904+0000","last_change":"2023-06-03T19:56:54.512932+0000","last_active":"2023-06-03T19:57:27.446904+0000","last_peered":"2023-06-03T19:57:27.446904+0000","last_clean":"2023-06-03T19:57:27.446904+0000","last_became_active":"2023-06-03T19:56:54.511992+0000","last_became_peered":"2023-06-03T19:56:54.511992+0000","last_unstale":"2023-06-03T19:57:27.446904+0000","last_undegraded":"2023-06-03T19:57:27.446904+0000","last_fullsized":"2023-06-03T19:57:27.446904+0000","mapping_epoch":119,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T21:01:35.069911+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":91,"reported_epoch":124,"state":"active+clean","last_fresh":"2023-06-03T19:57:27.437336+0000","last_change":"2023-06-03T19:51:59.359522+0000","last_active":"2023-06-03T19:57:27.437336+0000","last_peered":"2023-06-03T19:57:27.437336+0000","last_clean":"2023-06-03T19:57:27.437336+0000","last_became_active":"2023-06-03T19:51:59.359202+0000","last_became_peered":"2023-06-03T19:51:59.359202+0000","last_unstale":"2023-06-03T19:57:27.437336+0000","last_undegraded":"2023-06-03T19:57:27.437336+0000","last_fullsized":"2023-06-03T19:57:27.437336+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":79,"reported_epoch":125,"state":"active+clean","last_fresh":"2023-06-03T19:57:28.451456+0000","last_change":"2023-06-03T19:56:54.509490+0000","last_active":"2023-06-03T19:57:28.451456+0000","last_peered":"2023-06-03T19:57:28.451456+0000","last_clean":"2023-06-03T19:57:28.451456+0000","last_became_active":"2023-06-03T19:56:54.509145+0000","last_became_peered":"2023-06-03T19:56:54.509145+0000","last_unstale":"2023-06-03T19:57:28.451456+0000","last_undegraded":"2023-06-03T19:57:28.451456+0000","last_fullsized":"2023-06-03T19:57:28.451456+0000","mapping_epoch":119,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T07:12:03.964110+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":89,"reported_epoch":125,"state":"active+clean","last_fresh":"2023-06-03T19:57:28.452799+0000","last_change":"2023-06-03T19:56:54.511325+0000","last_active":"2023-06-03T19:57:28.452799+0000","last_peered":"2023-06-03T19:57:28.452799+0000","last_clean":"2023-06-03T19:57:28.452799+0000","last_became_active":"2023-06-03T19:56:54.511033+0000","last_became_peered":"2023-06-03T19:56:54.511033+0000","last_unstale":"2023-06-03T19:57:28.452799+0000","last_undegraded":"2023-06-03T19:57:28.452799+0000","last_fullsized":"2023-06-03T19:57:28.452799+0000","mapping_epoch":119,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T23:50:08.711478+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":92,"reported_epoch":124,"state":"active+clean","last_fresh":"2023-06-03T19:57:27.439764+0000","last_change":"2023-06-03T19:51:59.374143+0000","last_active":"2023-06-03T19:57:27.439764+0000","last_peered":"2023-06-03T19:57:27.439764+0000","last_clean":"2023-06-03T19:57:27.439764+0000","last_became_active":"2023-06-03T19:51:59.373146+0000","last_became_peered":"2023-06-03T19:51:59.373146+0000","last_unstale":"2023-06-03T19:57:27.439764+0000","last_undegraded":"2023-06-03T19:57:27.439764+0000","last_fullsized":"2023-06-03T19:57:27.439764+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":70,"reported_epoch":125,"state":"active+clean","last_fresh":"2023-06-03T19:57:28.457731+0000","last_change":"2023-06-03T19:56:54.510462+0000","last_active":"2023-06-03T19:57:28.457731+0000","last_peered":"2023-06-03T19:57:28.457731+0000","last_clean":"2023-06-03T19:57:28.457731+0000","last_became_active":"2023-06-03T19:56:54.509783+0000","last_became_peered":"2023-06-03T19:56:54.509783+0000","last_unstale":"2023-06-03T19:57:28.457731+0000","last_undegraded":"2023-06-03T19:57:28.457731+0000","last_fullsized":"2023-06-03T19:57:28.457731+0000","mapping_epoch":119,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T00:16:09.624276+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"60'3","reported_seq":110,"reported_epoch":125,"state":"active+clean","last_fresh":"2023-06-03T19:57:28.466401+0000","last_change":"2023-06-03T19:57:28.466401+0000","last_active":"2023-06-03T19:57:28.466401+0000","last_peered":"2023-06-03T19:57:28.466401+0000","last_clean":"2023-06-03T19:57:28.466401+0000","last_became_active":"2023-06-03T19:57:28.465452+0000","last_became_peered":"2023-06-03T19:57:28.465452+0000","last_unstale":"2023-06-03T19:57:28.466401+0000","last_undegraded":"2023-06-03T19:57:28.466401+0000","last_fullsized":"2023-06-03T19:57:28.466401+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":61,"reported_epoch":125,"state":"active+clean","last_fresh":"2023-06-03T19:57:28.458983+0000","last_change":"2023-06-03T19:56:11.753995+0000","last_active":"2023-06-03T19:57:28.458983+0000","last_peered":"2023-06-03T19:57:28.458983+0000","last_clean":"2023-06-03T19:57:28.458983+0000","last_became_active":"2023-06-03T19:56:11.753676+0000","last_became_peered":"2023-06-03T19:56:11.753676+0000","last_unstale":"2023-06-03T19:57:28.458983+0000","last_undegraded":"2023-06-03T19:57:28.458983+0000","last_fullsized":"2023-06-03T19:57:28.458983+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":115,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T01:00:40.343647+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"77'23","reported_seq":106,"reported_epoch":125,"state":"active+clean","last_fresh":"2023-06-03T19:57:28.462522+0000","last_change":"2023-06-03T19:57:28.462522+0000","last_active":"2023-06-03T19:57:28.462522+0000","last_peered":"2023-06-03T19:57:28.462522+0000","last_clean":"2023-06-03T19:57:28.462522+0000","last_became_active":"2023-06-03T19:57:28.462048+0000","last_became_peered":"2023-06-03T19:57:28.462048+0000","last_unstale":"2023-06-03T19:57:28.462522+0000","last_undegraded":"2023-06-03T19:57:28.462522+0000","last_fullsized":"2023-06-03T19:57:28.462522+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":23,"log_dups_size":0,"ondisk_log_size":23,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T23:23:56.697534+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4716,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":23,"num_write_kb":20,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":83,"reported_epoch":125,"state":"active+clean","last_fresh":"2023-06-03T19:57:28.469676+0000","last_change":"2023-06-03T19:57:28.469676+0000","last_active":"2023-06-03T19:57:28.469676+0000","last_peered":"2023-06-03T19:57:28.469676+0000","last_clean":"2023-06-03T19:57:28.469676+0000","last_became_active":"2023-06-03T19:57:28.469328+0000","last_became_peered":"2023-06-03T19:57:28.469328+0000","last_unstale":"2023-06-03T19:57:28.469676+0000","last_undegraded":"2023-06-03T19:57:28.469676+0000","last_fullsized":"2023-06-03T19:57:28.469676+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T23:25:26.041930+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"60'2","reported_seq":109,"reported_epoch":125,"state":"active+clean","last_fresh":"2023-06-03T19:57:28.467711+0000","last_change":"2023-06-03T19:57:28.467711+0000","last_active":"2023-06-03T19:57:28.467711+0000","last_peered":"2023-06-03T19:57:28.467711+0000","last_clean":"2023-06-03T19:57:28.467711+0000","last_became_active":"2023-06-03T19:57:28.467234+0000","last_became_peered":"2023-06-03T19:57:28.467234+0000","last_unstale":"2023-06-03T19:57:28.467711+0000","last_undegraded":"2023-06-03T19:57:28.467711+0000","last_fullsized":"2023-06-03T19:57:28.467711+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":89,"reported_epoch":125,"state":"active+clean","last_fresh":"2023-06-03T19:57:28.457345+0000","last_change":"2023-06-03T19:56:54.514616+0000","last_active":"2023-06-03T19:57:28.457345+0000","last_peered":"2023-06-03T19:57:28.457345+0000","last_clean":"2023-06-03T19:57:28.457345+0000","last_became_active":"2023-06-03T19:56:54.513918+0000","last_became_peered":"2023-06-03T19:56:54.513918+0000","last_unstale":"2023-06-03T19:57:28.457345+0000","last_undegraded":"2023-06-03T19:57:28.457345+0000","last_fullsized":"2023-06-03T19:57:28.457345+0000","mapping_epoch":119,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T19:36:10.967386+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":79,"reported_epoch":125,"state":"active+clean","last_fresh":"2023-06-03T19:57:28.450526+0000","last_change":"2023-06-03T19:56:54.507479+0000","last_active":"2023-06-03T19:57:28.450526+0000","last_peered":"2023-06-03T19:57:28.450526+0000","last_clean":"2023-06-03T19:57:28.450526+0000","last_became_active":"2023-06-03T19:56:54.507149+0000","last_became_peered":"2023-06-03T19:56:54.507149+0000","last_unstale":"2023-06-03T19:57:28.450526+0000","last_undegraded":"2023-06-03T19:57:28.450526+0000","last_fullsized":"2023-06-03T19:57:28.450526+0000","mapping_epoch":119,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T20:22:36.676317+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":91,"reported_epoch":124,"state":"active+clean","last_fresh":"2023-06-03T19:57:27.442389+0000","last_change":"2023-06-03T19:56:54.512472+0000","last_active":"2023-06-03T19:57:27.442389+0000","last_peered":"2023-06-03T19:57:27.442389+0000","last_clean":"2023-06-03T19:57:27.442389+0000","last_became_active":"2023-06-03T19:56:54.512051+0000","last_became_peered":"2023-06-03T19:56:54.512051+0000","last_unstale":"2023-06-03T19:57:27.442389+0000","last_undegraded":"2023-06-03T19:57:27.442389+0000","last_fullsized":"2023-06-03T19:57:27.442389+0000","mapping_epoch":119,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":92,"reported_epoch":125,"state":"active+clean","last_fresh":"2023-06-03T19:57:28.462780+0000","last_change":"2023-06-03T19:57:28.462780+0000","last_active":"2023-06-03T19:57:28.462780+0000","last_peered":"2023-06-03T19:57:28.462780+0000","last_clean":"2023-06-03T19:57:28.462780+0000","last_became_active":"2023-06-03T19:57:28.462298+0000","last_became_peered":"2023-06-03T19:57:28.462298+0000","last_unstale":"2023-06-03T19:57:28.462780+0000","last_undegraded":"2023-06-03T19:57:28.462780+0000","last_fullsized":"2023-06-03T19:57:28.462780+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":80,"reported_epoch":125,"state":"active+clean","last_fresh":"2023-06-03T19:57:28.451126+0000","last_change":"2023-06-03T19:56:54.510289+0000","last_active":"2023-06-03T19:57:28.451126+0000","last_peered":"2023-06-03T19:57:28.451126+0000","last_clean":"2023-06-03T19:57:28.451126+0000","last_became_active":"2023-06-03T19:56:54.509905+0000","last_became_peered":"2023-06-03T19:56:54.509905+0000","last_unstale":"2023-06-03T19:57:28.451126+0000","last_undegraded":"2023-06-03T19:57:28.451126+0000","last_fullsized":"2023-06-03T19:57:28.451126+0000","mapping_epoch":119,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T02:54:26.792401+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"60'8","reported_seq":100,"reported_epoch":124,"state":"active+clean","last_fresh":"2023-06-03T19:57:27.435852+0000","last_change":"2023-06-03T19:47:06.727778+0000","last_active":"2023-06-03T19:57:27.435852+0000","last_peered":"2023-06-03T19:57:27.435852+0000","last_clean":"2023-06-03T19:57:27.435852+0000","last_became_active":"2023-06-03T19:47:06.727305+0000","last_became_peered":"2023-06-03T19:47:06.727305+0000","last_unstale":"2023-06-03T19:57:27.435852+0000","last_undegraded":"2023-06-03T19:57:27.435852+0000","last_fullsized":"2023-06-03T19:57:27.435852+0000","mapping_epoch":86,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":87,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":88,"reported_epoch":124,"state":"active+clean","last_fresh":"2023-06-03T19:57:27.443855+0000","last_change":"2023-06-03T19:56:54.512994+0000","last_active":"2023-06-03T19:57:27.443855+0000","last_peered":"2023-06-03T19:57:27.443855+0000","last_clean":"2023-06-03T19:57:27.443855+0000","last_became_active":"2023-06-03T19:56:54.512579+0000","last_became_peered":"2023-06-03T19:56:54.512579+0000","last_unstale":"2023-06-03T19:57:27.443855+0000","last_undegraded":"2023-06-03T19:57:27.443855+0000","last_fullsized":"2023-06-03T19:57:27.443855+0000","mapping_epoch":119,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T07:02:45.187961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.f","version":"60'1","reported_seq":108,"reported_epoch":125,"state":"active+clean","last_fresh":"2023-06-03T19:57:28.464482+0000","last_change":"2023-06-03T19:57:28.464482+0000","last_active":"2023-06-03T19:57:28.464482+0000","last_peered":"2023-06-03T19:57:28.464482+0000","last_clean":"2023-06-03T19:57:28.464482+0000","last_became_active":"2023-06-03T19:57:28.464081+0000","last_became_peered":"2023-06-03T19:57:28.464081+0000","last_unstale":"2023-06-03T19:57:28.464482+0000","last_undegraded":"2023-06-03T19:57:28.464482+0000","last_fullsized":"2023-06-03T19:57:28.464482+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":76,"reported_epoch":124,"state":"active+clean","last_fresh":"2023-06-03T19:57:27.441050+0000","last_change":"2023-06-03T19:34:17.848358+0000","last_active":"2023-06-03T19:57:27.441050+0000","last_peered":"2023-06-03T19:57:27.441050+0000","last_clean":"2023-06-03T19:57:27.441050+0000","last_became_active":"2023-06-03T19:34:17.847811+0000","last_became_peered":"2023-06-03T19:34:17.847811+0000","last_unstale":"2023-06-03T19:57:27.441050+0000","last_undegraded":"2023-06-03T19:57:27.441050+0000","last_fullsized":"2023-06-03T19:57:27.441050+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.e","version":"60'2","reported_seq":109,"reported_epoch":125,"state":"active+clean","last_fresh":"2023-06-03T19:57:28.468330+0000","last_change":"2023-06-03T19:57:28.468330+0000","last_active":"2023-06-03T19:57:28.468330+0000","last_peered":"2023-06-03T19:57:28.468330+0000","last_clean":"2023-06-03T19:57:28.468330+0000","last_became_active":"2023-06-03T19:57:28.467823+0000","last_became_peered":"2023-06-03T19:57:28.467823+0000","last_unstale":"2023-06-03T19:57:28.468330+0000","last_undegraded":"2023-06-03T19:57:28.468330+0000","last_fullsized":"2023-06-03T19:57:28.468330+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":121,"reported_epoch":125,"state":"active+clean","last_fresh":"2023-06-03T19:57:28.461449+0000","last_change":"2023-06-03T19:57:28.461449+0000","last_active":"2023-06-03T19:57:28.461449+0000","last_peered":"2023-06-03T19:57:28.461449+0000","last_clean":"2023-06-03T19:57:28.461449+0000","last_became_active":"2023-06-03T19:57:28.460745+0000","last_became_peered":"2023-06-03T19:57:28.460745+0000","last_unstale":"2023-06-03T19:57:28.461449+0000","last_undegraded":"2023-06-03T19:57:28.461449+0000","last_fullsized":"2023-06-03T19:57:28.461449+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T20:41:19.361638+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,3],"acting":[4,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.d","version":"60'1","reported_seq":78,"reported_epoch":124,"state":"active+clean","last_fresh":"2023-06-03T19:57:27.445291+0000","last_change":"2023-06-03T19:34:16.806559+0000","last_active":"2023-06-03T19:57:27.445291+0000","last_peered":"2023-06-03T19:57:27.445291+0000","last_clean":"2023-06-03T19:57:27.445291+0000","last_became_active":"2023-06-03T19:34:16.806149+0000","last_became_peered":"2023-06-03T19:34:16.806149+0000","last_unstale":"2023-06-03T19:57:27.445291+0000","last_undegraded":"2023-06-03T19:57:27.445291+0000","last_fullsized":"2023-06-03T19:57:27.445291+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":91,"reported_epoch":124,"state":"active+clean","last_fresh":"2023-06-03T19:57:27.437525+0000","last_change":"2023-06-03T19:51:59.360650+0000","last_active":"2023-06-03T19:57:27.437525+0000","last_peered":"2023-06-03T19:57:27.437525+0000","last_clean":"2023-06-03T19:57:27.437525+0000","last_became_active":"2023-06-03T19:51:59.360237+0000","last_became_peered":"2023-06-03T19:51:59.360237+0000","last_unstale":"2023-06-03T19:57:27.437525+0000","last_undegraded":"2023-06-03T19:57:27.437525+0000","last_fullsized":"2023-06-03T19:57:27.437525+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":92,"reported_epoch":124,"state":"active+clean","last_fresh":"2023-06-03T19:57:27.440288+0000","last_change":"2023-06-03T19:56:54.507432+0000","last_active":"2023-06-03T19:57:27.440288+0000","last_peered":"2023-06-03T19:57:27.440288+0000","last_clean":"2023-06-03T19:57:27.440288+0000","last_became_active":"2023-06-03T19:56:54.507012+0000","last_became_peered":"2023-06-03T19:56:54.507012+0000","last_unstale":"2023-06-03T19:57:27.440288+0000","last_undegraded":"2023-06-03T19:57:27.440288+0000","last_fullsized":"2023-06-03T19:57:27.440288+0000","mapping_epoch":119,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":107,"reported_epoch":125,"state":"active+clean","last_fresh":"2023-06-03T19:57:28.465221+0000","last_change":"2023-06-03T19:57:28.464153+0000","last_active":"2023-06-03T19:57:28.465221+0000","last_peered":"2023-06-03T19:57:28.465221+0000","last_clean":"2023-06-03T19:57:28.465221+0000","last_became_active":"2023-06-03T19:57:28.463502+0000","last_became_peered":"2023-06-03T19:57:28.463502+0000","last_unstale":"2023-06-03T19:57:28.465221+0000","last_undegraded":"2023-06-03T19:57:28.465221+0000","last_fullsized":"2023-06-03T19:57:28.465221+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T06:11:00.999526+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,4],"acting":[0,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.b","version":"60'2","reported_seq":90,"reported_epoch":125,"state":"active+clean","last_fresh":"2023-06-03T19:57:28.471501+0000","last_change":"2023-06-03T19:57:28.470310+0000","last_active":"2023-06-03T19:57:28.471501+0000","last_peered":"2023-06-03T19:57:28.471501+0000","last_clean":"2023-06-03T19:57:28.471501+0000","last_became_active":"2023-06-03T19:57:28.469887+0000","last_became_peered":"2023-06-03T19:57:28.469887+0000","last_unstale":"2023-06-03T19:57:28.471501+0000","last_undegraded":"2023-06-03T19:57:28.471501+0000","last_fullsized":"2023-06-03T19:57:28.471501+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T21:58:53.387131+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":97,"reported_epoch":125,"state":"active+clean","last_fresh":"2023-06-03T19:57:28.465140+0000","last_change":"2023-06-03T19:57:28.465140+0000","last_active":"2023-06-03T19:57:28.465140+0000","last_peered":"2023-06-03T19:57:28.465140+0000","last_clean":"2023-06-03T19:57:28.465140+0000","last_became_active":"2023-06-03T19:57:28.464629+0000","last_became_peered":"2023-06-03T19:57:28.464629+0000","last_unstale":"2023-06-03T19:57:28.465140+0000","last_undegraded":"2023-06-03T19:57:28.465140+0000","last_fullsized":"2023-06-03T19:57:28.465140+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T21:47:11.619583+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":107,"reported_epoch":125,"state":"active+clean","last_fresh":"2023-06-03T19:57:28.461401+0000","last_change":"2023-06-03T19:57:28.461401+0000","last_active":"2023-06-03T19:57:28.461401+0000","last_peered":"2023-06-03T19:57:28.461401+0000","last_clean":"2023-06-03T19:57:28.461401+0000","last_became_active":"2023-06-03T19:57:28.461057+0000","last_became_peered":"2023-06-03T19:57:28.461057+0000","last_unstale":"2023-06-03T19:57:28.461401+0000","last_undegraded":"2023-06-03T19:57:28.461401+0000","last_fullsized":"2023-06-03T19:57:28.461401+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":102,"reported_epoch":125,"state":"active+clean","last_fresh":"2023-06-03T19:57:28.456750+0000","last_change":"2023-06-03T19:57:28.456750+0000","last_active":"2023-06-03T19:57:28.456750+0000","last_peered":"2023-06-03T19:57:28.456750+0000","last_clean":"2023-06-03T19:57:28.456750+0000","last_became_active":"2023-06-03T19:57:28.456436+0000","last_became_peered":"2023-06-03T19:57:28.456436+0000","last_unstale":"2023-06-03T19:57:28.456750+0000","last_undegraded":"2023-06-03T19:57:28.456750+0000","last_fullsized":"2023-06-03T19:57:28.456750+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T01:49:25.481554+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,5],"acting":[4,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":94,"reported_epoch":125,"state":"active+clean","last_fresh":"2023-06-03T19:57:28.462344+0000","last_change":"2023-06-03T19:57:28.462344+0000","last_active":"2023-06-03T19:57:28.462344+0000","last_peered":"2023-06-03T19:57:28.462344+0000","last_clean":"2023-06-03T19:57:28.462344+0000","last_became_active":"2023-06-03T19:57:28.462001+0000","last_became_peered":"2023-06-03T19:57:28.462001+0000","last_unstale":"2023-06-03T19:57:28.462344+0000","last_undegraded":"2023-06-03T19:57:28.462344+0000","last_fullsized":"2023-06-03T19:57:28.462344+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T00:45:13.789650+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":83,"reported_epoch":125,"state":"active+clean","last_fresh":"2023-06-03T19:57:28.467917+0000","last_change":"2023-06-03T19:57:28.467917+0000","last_active":"2023-06-03T19:57:28.467917+0000","last_peered":"2023-06-03T19:57:28.467917+0000","last_clean":"2023-06-03T19:57:28.467917+0000","last_became_active":"2023-06-03T19:57:28.467617+0000","last_became_peered":"2023-06-03T19:57:28.467617+0000","last_unstale":"2023-06-03T19:57:28.467917+0000","last_undegraded":"2023-06-03T19:57:28.467917+0000","last_fullsized":"2023-06-03T19:57:28.467917+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T23:34:23.840582+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":90,"reported_epoch":125,"state":"active+clean","last_fresh":"2023-06-03T19:57:28.453087+0000","last_change":"2023-06-03T19:56:54.512219+0000","last_active":"2023-06-03T19:57:28.453087+0000","last_peered":"2023-06-03T19:57:28.453087+0000","last_clean":"2023-06-03T19:57:28.453087+0000","last_became_active":"2023-06-03T19:56:54.511771+0000","last_became_peered":"2023-06-03T19:56:54.511771+0000","last_unstale":"2023-06-03T19:57:28.453087+0000","last_undegraded":"2023-06-03T19:57:28.453087+0000","last_fullsized":"2023-06-03T19:57:28.453087+0000","mapping_epoch":119,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T19:35:47.796520+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":76,"reported_epoch":125,"state":"active+clean","last_fresh":"2023-06-03T19:57:28.472005+0000","last_change":"2023-06-03T19:57:28.471023+0000","last_active":"2023-06-03T19:57:28.472005+0000","last_peered":"2023-06-03T19:57:28.472005+0000","last_clean":"2023-06-03T19:57:28.472005+0000","last_became_active":"2023-06-03T19:57:28.470452+0000","last_became_peered":"2023-06-03T19:57:28.470452+0000","last_unstale":"2023-06-03T19:57:28.472005+0000","last_undegraded":"2023-06-03T19:57:28.472005+0000","last_fullsized":"2023-06-03T19:57:28.472005+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T05:58:33.305030+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":94,"reported_epoch":125,"state":"active+clean","last_fresh":"2023-06-03T19:57:28.462051+0000","last_change":"2023-06-03T19:51:59.392796+0000","last_active":"2023-06-03T19:57:28.462051+0000","last_peered":"2023-06-03T19:57:28.462051+0000","last_clean":"2023-06-03T19:57:28.462051+0000","last_became_active":"2023-06-03T19:51:59.380207+0000","last_became_peered":"2023-06-03T19:51:59.380207+0000","last_unstale":"2023-06-03T19:57:28.462051+0000","last_undegraded":"2023-06-03T19:57:28.462051+0000","last_fullsized":"2023-06-03T19:57:28.462051+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:32:20.692043+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:32:20.692043+0000","last_clean_scrub_stamp":"2023-06-03T19:32:20.692043+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T00:28:05.866873+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":107,"reported_epoch":124,"state":"active+clean","last_fresh":"2023-06-03T19:57:27.442422+0000","last_change":"2023-06-03T19:56:54.512607+0000","last_active":"2023-06-03T19:57:27.442422+0000","last_peered":"2023-06-03T19:57:27.442422+0000","last_clean":"2023-06-03T19:57:27.442422+0000","last_became_active":"2023-06-03T19:56:54.512282+0000","last_became_peered":"2023-06-03T19:56:54.512282+0000","last_unstale":"2023-06-03T19:57:27.442422+0000","last_undegraded":"2023-06-03T19:57:27.442422+0000","last_fullsized":"2023-06-03T19:57:27.442422+0000","mapping_epoch":119,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":61,"reported_epoch":125,"state":"active+clean","last_fresh":"2023-06-03T19:57:28.471196+0000","last_change":"2023-06-03T19:57:28.470295+0000","last_active":"2023-06-03T19:57:28.471196+0000","last_peered":"2023-06-03T19:57:28.471196+0000","last_clean":"2023-06-03T19:57:28.471196+0000","last_became_active":"2023-06-03T19:57:28.469383+0000","last_became_peered":"2023-06-03T19:57:28.469383+0000","last_unstale":"2023-06-03T19:57:28.471196+0000","last_undegraded":"2023-06-03T19:57:28.471196+0000","last_fullsized":"2023-06-03T19:57:28.471196+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T05:35:19.667447+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":96,"reported_epoch":124,"state":"active+clean","last_fresh":"2023-06-03T19:57:27.441028+0000","last_change":"2023-06-03T19:51:59.373777+0000","last_active":"2023-06-03T19:57:27.441028+0000","last_peered":"2023-06-03T19:57:27.441028+0000","last_clean":"2023-06-03T19:57:27.441028+0000","last_became_active":"2023-06-03T19:51:59.372988+0000","last_became_peered":"2023-06-03T19:51:59.372988+0000","last_unstale":"2023-06-03T19:57:27.441028+0000","last_undegraded":"2023-06-03T19:57:27.441028+0000","last_fullsized":"2023-06-03T19:57:27.441028+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:32:20.692043+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:32:20.692043+0000","last_clean_scrub_stamp":"2023-06-03T19:32:20.692043+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":67,"reported_epoch":125,"state":"active+clean","last_fresh":"2023-06-03T19:57:28.460195+0000","last_change":"2023-06-03T19:57:28.460195+0000","last_active":"2023-06-03T19:57:28.460195+0000","last_peered":"2023-06-03T19:57:28.460195+0000","last_clean":"2023-06-03T19:57:28.460195+0000","last_became_active":"2023-06-03T19:57:28.459823+0000","last_became_peered":"2023-06-03T19:57:28.459823+0000","last_unstale":"2023-06-03T19:57:28.460195+0000","last_undegraded":"2023-06-03T19:57:28.460195+0000","last_fullsized":"2023-06-03T19:57:28.460195+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T20:08:48.686380+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":76,"reported_epoch":125,"state":"active+clean","last_fresh":"2023-06-03T19:57:28.462134+0000","last_change":"2023-06-03T19:56:54.516859+0000","last_active":"2023-06-03T19:57:28.462134+0000","last_peered":"2023-06-03T19:57:28.462134+0000","last_clean":"2023-06-03T19:57:28.462134+0000","last_became_active":"2023-06-03T19:56:54.516521+0000","last_became_peered":"2023-06-03T19:56:54.516521+0000","last_unstale":"2023-06-03T19:57:28.462134+0000","last_undegraded":"2023-06-03T19:57:28.462134+0000","last_fullsized":"2023-06-03T19:57:28.462134+0000","mapping_epoch":119,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T20:03:25.227382+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":97,"reported_epoch":125,"state":"active+clean","last_fresh":"2023-06-03T19:57:28.449619+0000","last_change":"2023-06-03T19:56:54.504859+0000","last_active":"2023-06-03T19:57:28.449619+0000","last_peered":"2023-06-03T19:57:28.449619+0000","last_clean":"2023-06-03T19:57:28.449619+0000","last_became_active":"2023-06-03T19:56:54.504455+0000","last_became_peered":"2023-06-03T19:56:54.504455+0000","last_unstale":"2023-06-03T19:57:28.449619+0000","last_undegraded":"2023-06-03T19:57:28.449619+0000","last_fullsized":"2023-06-03T19:57:28.449619+0000","mapping_epoch":119,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:32:20.692043+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:32:20.692043+0000","last_clean_scrub_stamp":"2023-06-03T19:32:20.692043+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T06:25:45.522100+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":102,"reported_epoch":125,"state":"active+clean","last_fresh":"2023-06-03T19:57:28.458987+0000","last_change":"2023-06-03T19:56:11.754048+0000","last_active":"2023-06-03T19:57:28.458987+0000","last_peered":"2023-06-03T19:57:28.458987+0000","last_clean":"2023-06-03T19:57:28.458987+0000","last_became_active":"2023-06-03T19:56:11.753687+0000","last_became_peered":"2023-06-03T19:56:11.753687+0000","last_unstale":"2023-06-03T19:57:28.458987+0000","last_undegraded":"2023-06-03T19:57:28.458987+0000","last_fullsized":"2023-06-03T19:57:28.458987+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":115,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T23:29:41.511496+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":89,"reported_epoch":125,"state":"active+clean","last_fresh":"2023-06-03T19:57:28.459616+0000","last_change":"2023-06-03T19:51:59.396573+0000","last_active":"2023-06-03T19:57:28.459616+0000","last_peered":"2023-06-03T19:57:28.459616+0000","last_clean":"2023-06-03T19:57:28.459616+0000","last_became_active":"2023-06-03T19:51:59.381593+0000","last_became_peered":"2023-06-03T19:51:59.381593+0000","last_unstale":"2023-06-03T19:57:28.459616+0000","last_undegraded":"2023-06-03T19:57:28.459616+0000","last_fullsized":"2023-06-03T19:57:28.459616+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T21:33:23.043772+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":96,"reported_epoch":124,"state":"active+clean","last_fresh":"2023-06-03T19:57:27.439804+0000","last_change":"2023-06-03T19:56:11.752151+0000","last_active":"2023-06-03T19:57:27.439804+0000","last_peered":"2023-06-03T19:57:27.439804+0000","last_clean":"2023-06-03T19:57:27.439804+0000","last_became_active":"2023-06-03T19:56:11.751767+0000","last_became_peered":"2023-06-03T19:56:11.751767+0000","last_unstale":"2023-06-03T19:57:27.439804+0000","last_undegraded":"2023-06-03T19:57:27.439804+0000","last_fullsized":"2023-06-03T19:57:27.439804+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":115,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:32:20.692043+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:32:20.692043+0000","last_clean_scrub_stamp":"2023-06-03T19:32:20.692043+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"72'139","reported_seq":328,"reported_epoch":124,"state":"active+clean","last_fresh":"2023-06-03T19:57:27.443270+0000","last_change":"2023-06-03T19:47:06.731872+0000","last_active":"2023-06-03T19:57:27.443270+0000","last_peered":"2023-06-03T19:57:27.443270+0000","last_clean":"2023-06-03T19:57:27.443270+0000","last_became_active":"2023-06-03T19:47:06.731175+0000","last_became_peered":"2023-06-03T19:47:06.731175+0000","last_unstale":"2023-06-03T19:57:27.443270+0000","last_undegraded":"2023-06-03T19:57:27.443270+0000","last_fullsized":"2023-06-03T19:57:27.443270+0000","mapping_epoch":86,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":87,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:27:28.100808+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:27:28.100808+0000","last_clean_scrub_stamp":"2023-06-03T19:27:28.100808+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":70,"reported_epoch":125,"state":"active+clean","last_fresh":"2023-06-03T19:57:28.457052+0000","last_change":"2023-06-03T19:56:54.509571+0000","last_active":"2023-06-03T19:57:28.457052+0000","last_peered":"2023-06-03T19:57:28.457052+0000","last_clean":"2023-06-03T19:57:28.457052+0000","last_became_active":"2023-06-03T19:56:54.509213+0000","last_became_peered":"2023-06-03T19:56:54.509213+0000","last_unstale":"2023-06-03T19:57:28.457052+0000","last_undegraded":"2023-06-03T19:57:28.457052+0000","last_fullsized":"2023-06-03T19:57:28.457052+0000","mapping_epoch":119,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T21:26:56.003203+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":91,"reported_epoch":125,"state":"active+clean","last_fresh":"2023-06-03T19:57:28.459664+0000","last_change":"2023-06-03T19:57:28.459664+0000","last_active":"2023-06-03T19:57:28.459664+0000","last_peered":"2023-06-03T19:57:28.459664+0000","last_clean":"2023-06-03T19:57:28.459664+0000","last_became_active":"2023-06-03T19:57:28.459289+0000","last_became_peered":"2023-06-03T19:57:28.459289+0000","last_unstale":"2023-06-03T19:57:28.459664+0000","last_undegraded":"2023-06-03T19:57:28.459664+0000","last_fullsized":"2023-06-03T19:57:28.459664+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":111,"reported_epoch":124,"state":"active+clean","last_fresh":"2023-06-03T19:57:27.443526+0000","last_change":"2023-06-03T19:51:59.358157+0000","last_active":"2023-06-03T19:57:27.443526+0000","last_peered":"2023-06-03T19:57:27.443526+0000","last_clean":"2023-06-03T19:57:27.443526+0000","last_became_active":"2023-06-03T19:51:59.357363+0000","last_became_peered":"2023-06-03T19:51:59.357363+0000","last_unstale":"2023-06-03T19:57:27.443526+0000","last_undegraded":"2023-06-03T19:57:27.443526+0000","last_fullsized":"2023-06-03T19:57:27.443526+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:32:20.692043+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:32:20.692043+0000","last_clean_scrub_stamp":"2023-06-03T19:32:20.692043+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":94,"reported_epoch":125,"state":"active+clean","last_fresh":"2023-06-03T19:57:28.459955+0000","last_change":"2023-06-03T19:57:28.459611+0000","last_active":"2023-06-03T19:57:28.459955+0000","last_peered":"2023-06-03T19:57:28.459955+0000","last_clean":"2023-06-03T19:57:28.459955+0000","last_became_active":"2023-06-03T19:57:28.459264+0000","last_became_peered":"2023-06-03T19:57:28.459264+0000","last_unstale":"2023-06-03T19:57:28.459955+0000","last_undegraded":"2023-06-03T19:57:28.459955+0000","last_fullsized":"2023-06-03T19:57:28.459955+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T22:15:06.557231+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":95,"reported_epoch":125,"state":"active+clean","last_fresh":"2023-06-03T19:57:28.451064+0000","last_change":"2023-06-03T19:56:11.745678+0000","last_active":"2023-06-03T19:57:28.451064+0000","last_peered":"2023-06-03T19:57:28.451064+0000","last_clean":"2023-06-03T19:57:28.451064+0000","last_became_active":"2023-06-03T19:56:11.744966+0000","last_became_peered":"2023-06-03T19:56:11.744966+0000","last_unstale":"2023-06-03T19:57:28.451064+0000","last_undegraded":"2023-06-03T19:57:28.451064+0000","last_fullsized":"2023-06-03T19:57:28.451064+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":115,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T21:12:06.830295+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":111,"reported_epoch":125,"state":"active+clean","last_fresh":"2023-06-03T19:57:28.466096+0000","last_change":"2023-06-03T19:57:28.466096+0000","last_active":"2023-06-03T19:57:28.466096+0000","last_peered":"2023-06-03T19:57:28.466096+0000","last_clean":"2023-06-03T19:57:28.466096+0000","last_became_active":"2023-06-03T19:57:28.464954+0000","last_became_peered":"2023-06-03T19:57:28.464954+0000","last_unstale":"2023-06-03T19:57:28.466096+0000","last_undegraded":"2023-06-03T19:57:28.466096+0000","last_fullsized":"2023-06-03T19:57:28.466096+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:32:20.692043+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:32:20.692043+0000","last_clean_scrub_stamp":"2023-06-03T19:32:20.692043+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":102,"reported_epoch":125,"state":"active+clean","last_fresh":"2023-06-03T19:57:28.462221+0000","last_change":"2023-06-03T19:56:11.750241+0000","last_active":"2023-06-03T19:57:28.462221+0000","last_peered":"2023-06-03T19:57:28.462221+0000","last_clean":"2023-06-03T19:57:28.462221+0000","last_became_active":"2023-06-03T19:56:11.749687+0000","last_became_peered":"2023-06-03T19:56:11.749687+0000","last_unstale":"2023-06-03T19:57:28.462221+0000","last_undegraded":"2023-06-03T19:57:28.462221+0000","last_fullsized":"2023-06-03T19:57:28.462221+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":115,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T22:38:25.819929+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":65,"reported_epoch":125,"state":"active+clean","last_fresh":"2023-06-03T19:57:28.458317+0000","last_change":"2023-06-03T19:47:06.733532+0000","last_active":"2023-06-03T19:57:28.458317+0000","last_peered":"2023-06-03T19:57:28.458317+0000","last_clean":"2023-06-03T19:57:28.458317+0000","last_became_active":"2023-06-03T19:47:06.733131+0000","last_became_peered":"2023-06-03T19:47:06.733131+0000","last_unstale":"2023-06-03T19:57:28.458317+0000","last_undegraded":"2023-06-03T19:57:28.458317+0000","last_fullsized":"2023-06-03T19:57:28.458317+0000","mapping_epoch":86,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":87,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T23:46:03.788646+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.7","version":"60'2","reported_seq":75,"reported_epoch":125,"state":"active+clean","last_fresh":"2023-06-03T19:57:28.457418+0000","last_change":"2023-06-03T19:56:54.510148+0000","last_active":"2023-06-03T19:57:28.457418+0000","last_peered":"2023-06-03T19:57:28.457418+0000","last_clean":"2023-06-03T19:57:28.457418+0000","last_became_active":"2023-06-03T19:56:54.509771+0000","last_became_peered":"2023-06-03T19:56:54.509771+0000","last_unstale":"2023-06-03T19:57:28.457418+0000","last_undegraded":"2023-06-03T19:57:28.457418+0000","last_fullsized":"2023-06-03T19:57:28.457418+0000","mapping_epoch":119,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T00:52:42.807432+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":102,"reported_epoch":125,"state":"active+clean","last_fresh":"2023-06-03T19:57:28.472011+0000","last_change":"2023-06-03T19:57:28.470954+0000","last_active":"2023-06-03T19:57:28.472011+0000","last_peered":"2023-06-03T19:57:28.472011+0000","last_clean":"2023-06-03T19:57:28.472011+0000","last_became_active":"2023-06-03T19:57:28.470524+0000","last_became_peered":"2023-06-03T19:57:28.470524+0000","last_unstale":"2023-06-03T19:57:28.472011+0000","last_undegraded":"2023-06-03T19:57:28.472011+0000","last_fullsized":"2023-06-03T19:57:28.472011+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:32:20.692043+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:32:20.692043+0000","last_clean_scrub_stamp":"2023-06-03T19:32:20.692043+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T19:39:19.983271+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":72,"reported_epoch":125,"state":"active+clean","last_fresh":"2023-06-03T19:57:28.452994+0000","last_change":"2023-06-03T19:56:54.507670+0000","last_active":"2023-06-03T19:57:28.452994+0000","last_peered":"2023-06-03T19:57:28.452994+0000","last_clean":"2023-06-03T19:57:28.452994+0000","last_became_active":"2023-06-03T19:56:54.507293+0000","last_became_peered":"2023-06-03T19:56:54.507293+0000","last_unstale":"2023-06-03T19:57:28.452994+0000","last_undegraded":"2023-06-03T19:57:28.452994+0000","last_fullsized":"2023-06-03T19:57:28.452994+0000","mapping_epoch":119,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T01:49:19.966054+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":96,"reported_epoch":124,"state":"active+clean","last_fresh":"2023-06-03T19:57:27.437244+0000","last_change":"2023-06-03T19:56:11.752730+0000","last_active":"2023-06-03T19:57:27.437244+0000","last_peered":"2023-06-03T19:57:27.437244+0000","last_clean":"2023-06-03T19:57:27.437244+0000","last_became_active":"2023-06-03T19:56:11.752441+0000","last_became_peered":"2023-06-03T19:56:11.752441+0000","last_unstale":"2023-06-03T19:57:27.437244+0000","last_undegraded":"2023-06-03T19:57:27.437244+0000","last_fullsized":"2023-06-03T19:57:27.437244+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":115,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:32:20.692043+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:32:20.692043+0000","last_clean_scrub_stamp":"2023-06-03T19:32:20.692043+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"60'1","reported_seq":98,"reported_epoch":125,"state":"active+clean","last_fresh":"2023-06-03T19:57:28.464648+0000","last_change":"2023-06-03T19:57:28.463999+0000","last_active":"2023-06-03T19:57:28.464648+0000","last_peered":"2023-06-03T19:57:28.464648+0000","last_clean":"2023-06-03T19:57:28.464648+0000","last_became_active":"2023-06-03T19:57:28.463588+0000","last_became_peered":"2023-06-03T19:57:28.463588+0000","last_unstale":"2023-06-03T19:57:28.464648+0000","last_undegraded":"2023-06-03T19:57:28.464648+0000","last_fullsized":"2023-06-03T19:57:28.464648+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T21:44:11.969059+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":65,"reported_epoch":125,"state":"active+clean","last_fresh":"2023-06-03T19:57:28.456467+0000","last_change":"2023-06-03T19:57:28.456467+0000","last_active":"2023-06-03T19:57:28.456467+0000","last_peered":"2023-06-03T19:57:28.456467+0000","last_clean":"2023-06-03T19:57:28.456467+0000","last_became_active":"2023-06-03T19:57:28.456120+0000","last_became_peered":"2023-06-03T19:57:28.456120+0000","last_unstale":"2023-06-03T19:57:28.456467+0000","last_undegraded":"2023-06-03T19:57:28.456467+0000","last_fullsized":"2023-06-03T19:57:28.456467+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T07:28:14.306624+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"60'1","reported_seq":82,"reported_epoch":125,"state":"active+clean","last_fresh":"2023-06-03T19:57:28.465124+0000","last_change":"2023-06-03T19:57:28.465124+0000","last_active":"2023-06-03T19:57:28.465124+0000","last_peered":"2023-06-03T19:57:28.465124+0000","last_clean":"2023-06-03T19:57:28.465124+0000","last_became_active":"2023-06-03T19:57:28.464654+0000","last_became_peered":"2023-06-03T19:57:28.464654+0000","last_unstale":"2023-06-03T19:57:28.465124+0000","last_undegraded":"2023-06-03T19:57:28.465124+0000","last_fullsized":"2023-06-03T19:57:28.465124+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T01:12:32.305507+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":109,"reported_epoch":125,"state":"active+clean","last_fresh":"2023-06-03T19:57:28.448170+0000","last_change":"2023-06-03T19:56:54.511760+0000","last_active":"2023-06-03T19:57:28.448170+0000","last_peered":"2023-06-03T19:57:28.448170+0000","last_clean":"2023-06-03T19:57:28.448170+0000","last_became_active":"2023-06-03T19:56:54.511429+0000","last_became_peered":"2023-06-03T19:56:54.511429+0000","last_unstale":"2023-06-03T19:57:28.448170+0000","last_undegraded":"2023-06-03T19:57:28.448170+0000","last_fullsized":"2023-06-03T19:57:28.448170+0000","mapping_epoch":119,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T05:42:46.254343+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"60'2","reported_seq":109,"reported_epoch":125,"state":"active+clean","last_fresh":"2023-06-03T19:57:28.462252+0000","last_change":"2023-06-03T19:57:28.462252+0000","last_active":"2023-06-03T19:57:28.462252+0000","last_peered":"2023-06-03T19:57:28.462252+0000","last_clean":"2023-06-03T19:57:28.462252+0000","last_became_active":"2023-06-03T19:57:28.461844+0000","last_became_peered":"2023-06-03T19:57:28.461844+0000","last_unstale":"2023-06-03T19:57:28.462252+0000","last_undegraded":"2023-06-03T19:57:28.462252+0000","last_fullsized":"2023-06-03T19:57:28.462252+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":112,"reported_epoch":125,"state":"active+clean","last_fresh":"2023-06-03T19:57:28.469248+0000","last_change":"2023-06-03T19:57:28.469248+0000","last_active":"2023-06-03T19:57:28.469248+0000","last_peered":"2023-06-03T19:57:28.469248+0000","last_clean":"2023-06-03T19:57:28.469248+0000","last_became_active":"2023-06-03T19:57:28.468903+0000","last_became_peered":"2023-06-03T19:57:28.468903+0000","last_unstale":"2023-06-03T19:57:28.469248+0000","last_undegraded":"2023-06-03T19:57:28.469248+0000","last_fullsized":"2023-06-03T19:57:28.469248+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T06:08:20.956098+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"60'2","reported_seq":94,"reported_epoch":125,"state":"active+clean","last_fresh":"2023-06-03T19:57:28.462610+0000","last_change":"2023-06-03T19:57:28.462610+0000","last_active":"2023-06-03T19:57:28.462610+0000","last_peered":"2023-06-03T19:57:28.462610+0000","last_clean":"2023-06-03T19:57:28.462610+0000","last_became_active":"2023-06-03T19:57:28.460655+0000","last_became_peered":"2023-06-03T19:57:28.460655+0000","last_unstale":"2023-06-03T19:57:28.462610+0000","last_undegraded":"2023-06-03T19:57:28.462610+0000","last_fullsized":"2023-06-03T19:57:28.462610+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":112,"reported_epoch":125,"state":"active+clean","last_fresh":"2023-06-03T19:57:28.464690+0000","last_change":"2023-06-03T19:57:28.464690+0000","last_active":"2023-06-03T19:57:28.464690+0000","last_peered":"2023-06-03T19:57:28.464690+0000","last_clean":"2023-06-03T19:57:28.464690+0000","last_became_active":"2023-06-03T19:57:28.463856+0000","last_became_peered":"2023-06-03T19:57:28.463856+0000","last_unstale":"2023-06-03T19:57:28.464690+0000","last_undegraded":"2023-06-03T19:57:28.464690+0000","last_fullsized":"2023-06-03T19:57:28.464690+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T00:11:00.991088+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"60'1","reported_seq":101,"reported_epoch":125,"state":"active+clean","last_fresh":"2023-06-03T19:57:28.457091+0000","last_change":"2023-06-03T19:56:54.505096+0000","last_active":"2023-06-03T19:57:28.457091+0000","last_peered":"2023-06-03T19:57:28.457091+0000","last_clean":"2023-06-03T19:57:28.457091+0000","last_became_active":"2023-06-03T19:56:54.504255+0000","last_became_peered":"2023-06-03T19:56:54.504255+0000","last_unstale":"2023-06-03T19:57:28.457091+0000","last_undegraded":"2023-06-03T19:57:28.457091+0000","last_fullsized":"2023-06-03T19:57:28.457091+0000","mapping_epoch":119,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T19:52:25.922308+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":113,"reported_epoch":124,"state":"active+clean","last_fresh":"2023-06-03T19:57:27.449326+0000","last_change":"2023-06-03T19:56:54.503416+0000","last_active":"2023-06-03T19:57:27.449326+0000","last_peered":"2023-06-03T19:57:27.449326+0000","last_clean":"2023-06-03T19:57:27.449326+0000","last_became_active":"2023-06-03T19:56:54.503018+0000","last_became_peered":"2023-06-03T19:56:54.503018+0000","last_unstale":"2023-06-03T19:57:27.449326+0000","last_undegraded":"2023-06-03T19:57:27.449326+0000","last_fullsized":"2023-06-03T19:57:27.449326+0000","mapping_epoch":119,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T20:14:00.330616+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"60'4","reported_seq":83,"reported_epoch":125,"state":"active+clean","last_fresh":"2023-06-03T19:57:28.451241+0000","last_change":"2023-06-03T19:56:11.748116+0000","last_active":"2023-06-03T19:57:28.451241+0000","last_peered":"2023-06-03T19:57:28.451241+0000","last_clean":"2023-06-03T19:57:28.451241+0000","last_became_active":"2023-06-03T19:56:11.747636+0000","last_became_peered":"2023-06-03T19:56:11.747636+0000","last_unstale":"2023-06-03T19:57:28.451241+0000","last_undegraded":"2023-06-03T19:57:28.451241+0000","last_fullsized":"2023-06-03T19:57:28.451241+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":115,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T06:57:48.052055+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":83,"reported_epoch":125,"state":"active+clean","last_fresh":"2023-06-03T19:57:28.459234+0000","last_change":"2023-06-03T19:57:28.459234+0000","last_active":"2023-06-03T19:57:28.459234+0000","last_peered":"2023-06-03T19:57:28.459234+0000","last_clean":"2023-06-03T19:57:28.459234+0000","last_became_active":"2023-06-03T19:57:28.458896+0000","last_became_peered":"2023-06-03T19:57:28.458896+0000","last_unstale":"2023-06-03T19:57:28.459234+0000","last_undegraded":"2023-06-03T19:57:28.459234+0000","last_fullsized":"2023-06-03T19:57:28.459234+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T21:18:37.142280+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5486,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":62,"num_write_kb":30,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16458,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":30319,"internal_metadata":0},"log_size":56,"ondisk_log_size":56,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":3,"num_store_stats":3},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":48,"seq":206158430546,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":287928,"kb_used_data":1776,"kb_used_omap":0,"kb_used_meta":286144,"kb_avail":93457224,"statfs":{"total":95995035648,"available":95700197376,"internally_reserved":0,"allocated":1818624,"data_stored":1348675,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":293011456},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 3 19:58:34 2023","interfaces":[{"interface":"back","average":{"1min":0.86599999999999999,"5min":1.1080000000000001,"15min":1.2729999999999999},"min":{"1min":0.48199999999999998,"5min":0.27500000000000002,"15min":0.253},"max":{"1min":1.3180000000000001,"5min":4.806,"15min":8.6099999999999994},"last":1.2629999999999999},{"interface":"front","average":{"1min":0.91100000000000003,"5min":1.218,"15min":1.506},"min":{"1min":0.57399999999999995,"5min":0.33200000000000002,"15min":0.28499999999999998},"max":{"1min":1.2390000000000001,"5min":6.6139999999999999,"15min":15.220000000000001},"last":0.89200000000000002}]},{"osd":1,"last update":"Sat Jun 3 19:58:12 2023","interfaces":[{"interface":"back","average":{"1min":0.84999999999999998,"5min":1.3879999999999999,"15min":1.8320000000000001},"min":{"1min":0.44400000000000001,"5min":0.315,"15min":0.315},"max":{"1min":1.385,"5min":8.1750000000000007,"15min":10.211},"last":1.216},{"interface":"front","average":{"1min":1.0640000000000001,"5min":1.415,"15min":1.623},"min":{"1min":0.75800000000000001,"5min":0.36199999999999999,"15min":0.34799999999999998},"max":{"1min":2.0790000000000002,"5min":8.7360000000000007,"15min":11.635},"last":1.079}]},{"osd":2,"last update":"Sat Jun 3 19:58:12 2023","interfaces":[{"interface":"back","average":{"1min":1.1080000000000001,"5min":1.032,"15min":1.0189999999999999},"min":{"1min":0.66100000000000003,"5min":0.66100000000000003,"15min":0.66100000000000003},"max":{"1min":2.9969999999999999,"5min":2.9969999999999999,"15min":2.9969999999999999},"last":1.0349999999999999},{"interface":"front","average":{"1min":1.0880000000000001,"5min":1,"15min":0.98499999999999999},"min":{"1min":0.63100000000000001,"5min":0.63100000000000001,"15min":0.63100000000000001},"max":{"1min":2.0299999999999998,"5min":2.0299999999999998,"15min":2.0299999999999998},"last":1.2310000000000001}]},{"osd":3,"last update":"Sat Jun 3 19:59:00 2023","interfaces":[{"interface":"back","average":{"1min":0.97799999999999998,"5min":1.1200000000000001,"15min":1.1439999999999999},"min":{"1min":0.70599999999999996,"5min":0.65400000000000003,"15min":0.65400000000000003},"max":{"1min":1.371,"5min":2.9399999999999999,"15min":2.9399999999999999},"last":0.93300000000000005},{"interface":"front","average":{"1min":1.0580000000000001,"5min":1.171,"15min":1.1899999999999999},"min":{"1min":0.66500000000000004,"5min":0.66500000000000004,"15min":0.66500000000000004},"max":{"1min":1.391,"5min":3.085,"15min":3.085},"last":1.181}]},{"osd":4,"last update":"Sat Jun 3 19:58:28 2023","interfaces":[{"interface":"back","average":{"1min":1.0509999999999999,"5min":1.0509999999999999,"15min":1.0509999999999999},"min":{"1min":0.66900000000000004,"5min":0.66900000000000004,"15min":0.66900000000000004},"max":{"1min":1.552,"5min":1.552,"15min":1.552},"last":1.1020000000000001},{"interface":"front","average":{"1min":1.119,"5min":1.119,"15min":1.119},"min":{"1min":0.77100000000000002,"5min":0.77100000000000002,"15min":0.77100000000000002},"max":{"1min":1.476,"5min":1.476,"15min":1.476},"last":1.2909999999999999}]},{"osd":5,"last update":"Sat Jun 3 19:59:09 2023","interfaces":[{"interface":"back","average":{"1min":1.054,"5min":1.1000000000000001,"15min":1.1879999999999999},"min":{"1min":0.80400000000000005,"5min":0.33200000000000002,"15min":0.30199999999999999},"max":{"1min":1.304,"5min":10.151999999999999,"15min":12.778},"last":1.1539999999999999},{"interface":"front","average":{"1min":1.157,"5min":1.089,"15min":1.3120000000000001},"min":{"1min":0.88100000000000001,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":1.417,"5min":5.3840000000000003,"15min":9.1669999999999998},"last":1.0129999999999999}]},{"osd":6,"last update":"Sat Jun 3 19:59:09 2023","interfaces":[{"interface":"back","average":{"1min":1.212,"5min":1.1719999999999999,"15min":1.325},"min":{"1min":1.0620000000000001,"5min":0.36499999999999999,"15min":0.35899999999999999},"max":{"1min":1.387,"5min":5.4429999999999996,"15min":9.3520000000000003},"last":1.127},{"interface":"front","average":{"1min":1.0509999999999999,"5min":1.169,"15min":1.1919999999999999},"min":{"1min":0.83699999999999997,"5min":0.41099999999999998,"15min":0.32900000000000001},"max":{"1min":1.274,"5min":9.0269999999999992,"15min":9.0269999999999992},"last":0.84399999999999997}]}]},{"osd":6,"up_from":42,"seq":180388626779,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":270504,"kb_used_data":1760,"kb_used_omap":0,"kb_used_meta":268736,"kb_avail":93474648,"statfs":{"total":95995035648,"available":95718039552,"internally_reserved":0,"allocated":1802240,"data_stored":1348214,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":275185664},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 3 19:58:35 2023","interfaces":[{"interface":"back","average":{"1min":0.98499999999999999,"5min":1.5009999999999999,"15min":1.5629999999999999},"min":{"1min":0.70199999999999996,"5min":0.28199999999999997,"15min":0.27500000000000002},"max":{"1min":1.3320000000000001,"5min":8.1890000000000001,"15min":11.944000000000001},"last":0.47099999999999997},{"interface":"front","average":{"1min":0.94699999999999995,"5min":1.6739999999999999,"15min":1.659},"min":{"1min":0.56100000000000005,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":1.1259999999999999,"5min":10.835000000000001,"15min":12.502000000000001},"last":0.60299999999999998}]},{"osd":1,"last update":"Sat Jun 3 19:58:10 2023","interfaces":[{"interface":"back","average":{"1min":0.94099999999999995,"5min":1.393,"15min":1.589},"min":{"1min":0.57699999999999996,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":1.5720000000000001,"5min":6.8049999999999997,"15min":6.8049999999999997},"last":0.64000000000000001},{"interface":"front","average":{"1min":0.97699999999999998,"5min":1.5649999999999999,"15min":1.2210000000000001},"min":{"1min":0.74399999999999999,"5min":0.39100000000000001,"15min":0.38500000000000001},"max":{"1min":1.3240000000000001,"5min":8.9120000000000008,"15min":8.9120000000000008},"last":0.79500000000000004}]},{"osd":2,"last update":"Sat Jun 3 19:58:12 2023","interfaces":[{"interface":"back","average":{"1min":0.89200000000000002,"5min":0.84999999999999998,"15min":0.84299999999999997},"min":{"1min":0.57399999999999995,"5min":0.57399999999999995,"15min":0.57399999999999995},"max":{"1min":1.4490000000000001,"5min":1.4490000000000001,"15min":1.4490000000000001},"last":0.69599999999999995},{"interface":"front","average":{"1min":0.96699999999999997,"5min":0.94699999999999995,"15min":0.94399999999999995},"min":{"1min":0.52700000000000002,"5min":0.52700000000000002,"15min":0.52700000000000002},"max":{"1min":1.2989999999999999,"5min":1.5469999999999999,"15min":1.5469999999999999},"last":0.72699999999999998}]},{"osd":3,"last update":"Sat Jun 3 19:59:00 2023","interfaces":[{"interface":"back","average":{"1min":0.89900000000000002,"5min":0.86299999999999999,"15min":0.85699999999999998},"min":{"1min":0.64500000000000002,"5min":0.623,"15min":0.623},"max":{"1min":1.173,"5min":1.1819999999999999,"15min":1.1819999999999999},"last":0.82899999999999996},{"interface":"front","average":{"1min":1.1100000000000001,"5min":1.0169999999999999,"15min":1.0009999999999999},"min":{"1min":0.76800000000000002,"5min":0.65300000000000002,"15min":0.65300000000000002},"max":{"1min":1.482,"5min":1.482,"15min":1.482},"last":0.85899999999999999}]},{"osd":4,"last update":"Sat Jun 3 19:58:29 2023","interfaces":[{"interface":"back","average":{"1min":0.999,"5min":0.999,"15min":0.999},"min":{"1min":0.751,"5min":0.751,"15min":0.751},"max":{"1min":1.1719999999999999,"5min":1.1719999999999999,"15min":1.1719999999999999},"last":0.95799999999999996},{"interface":"front","average":{"1min":1.181,"5min":1.181,"15min":1.181},"min":{"1min":0.92000000000000004,"5min":0.92000000000000004,"15min":0.92000000000000004},"max":{"1min":1.46,"5min":1.46,"15min":1.46},"last":0.94099999999999995}]},{"osd":5,"last update":"Sat Jun 3 19:58:25 2023","interfaces":[{"interface":"back","average":{"1min":1.1679999999999999,"5min":1.385,"15min":1.2749999999999999},"min":{"1min":0.83299999999999996,"5min":0.34100000000000003,"15min":0.30499999999999999},"max":{"1min":1.393,"5min":11.083,"15min":11.083},"last":0.90400000000000003},{"interface":"front","average":{"1min":1.089,"5min":1.4099999999999999,"15min":1.3220000000000001},"min":{"1min":0.80000000000000004,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":1.367,"5min":10.154,"15min":10.154},"last":0.98399999999999999}]},{"osd":7,"last update":"Sat Jun 3 19:59:04 2023","interfaces":[{"interface":"back","average":{"1min":1.252,"5min":1.3280000000000001,"15min":1.3540000000000001},"min":{"1min":0.79100000000000004,"5min":0.313,"15min":0.313},"max":{"1min":1.536,"5min":6.3029999999999999,"15min":8.0419999999999998},"last":1.0049999999999999},{"interface":"front","average":{"1min":1.121,"5min":1.5289999999999999,"15min":1.4059999999999999},"min":{"1min":0.76900000000000002,"5min":0.35099999999999998,"15min":0.29999999999999999},"max":{"1min":1.4059999999999999,"5min":12.407999999999999,"15min":12.407999999999999},"last":0.91600000000000004}]}]},{"osd":1,"up_from":103,"seq":442381631577,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":129700,"kb_used_data":1244,"kb_used_omap":4,"kb_used_meta":128443,"kb_avail":93615452,"statfs":{"total":95995035648,"available":95862222848,"internally_reserved":0,"allocated":1273856,"data_stored":767105,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":131526542},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 3 19:59:12 2023","interfaces":[{"interface":"back","average":{"1min":0.627,"5min":0.749,"15min":1.7589999999999999},"min":{"1min":0.38600000000000001,"5min":0.187,"15min":0.187},"max":{"1min":0.80000000000000004,"5min":4.1539999999999999,"15min":10.653},"last":0.76900000000000002},{"interface":"front","average":{"1min":0.68400000000000005,"5min":1.1319999999999999,"15min":1.831},"min":{"1min":0.32500000000000001,"5min":0.192,"15min":0.192},"max":{"1min":0.90900000000000003,"5min":13.750999999999999,"15min":13.750999999999999},"last":0.85099999999999998}]},{"osd":2,"last update":"Sat Jun 3 19:58:16 2023","interfaces":[{"interface":"back","average":{"1min":0.71199999999999997,"5min":0.71499999999999997,"15min":0.71499999999999997},"min":{"1min":0.34599999999999997,"5min":0.34599999999999997,"15min":0.34599999999999997},"max":{"1min":1.135,"5min":1.135,"15min":1.135},"last":0.64000000000000001},{"interface":"front","average":{"1min":0.69799999999999995,"5min":0.73699999999999999,"15min":0.74299999999999999},"min":{"1min":0.36499999999999999,"5min":0.36499999999999999,"15min":0.36499999999999999},"max":{"1min":1.2250000000000001,"5min":1.548,"15min":1.548},"last":0.88900000000000001}]},{"osd":3,"last update":"Sat Jun 3 19:58:59 2023","interfaces":[{"interface":"back","average":{"1min":0.66800000000000004,"5min":0.72399999999999998,"15min":0.73299999999999998},"min":{"1min":0.42999999999999999,"5min":0.36599999999999999,"15min":0.36599999999999999},"max":{"1min":1.2,"5min":1.2769999999999999,"15min":1.2769999999999999},"last":0.71599999999999997},{"interface":"front","average":{"1min":0.77800000000000002,"5min":0.84999999999999998,"15min":0.86199999999999999},"min":{"1min":0.40799999999999997,"5min":0.40799999999999997,"15min":0.40799999999999997},"max":{"1min":1.101,"5min":1.3740000000000001,"15min":1.3740000000000001},"last":0.80100000000000005}]},{"osd":4,"last update":"Sat Jun 3 19:58:28 2023","interfaces":[{"interface":"back","average":{"1min":0.99299999999999999,"5min":0.99299999999999999,"15min":0.99299999999999999},"min":{"1min":0.80400000000000005,"5min":0.80400000000000005,"15min":0.80400000000000005},"max":{"1min":1.466,"5min":1.466,"15min":1.466},"last":0.94799999999999995},{"interface":"front","average":{"1min":0.95699999999999996,"5min":0.95699999999999996,"15min":0.95699999999999996},"min":{"1min":0.66900000000000004,"5min":0.66900000000000004,"15min":0.66900000000000004},"max":{"1min":1.591,"5min":1.591,"15min":1.591},"last":1.264}]},{"osd":5,"last update":"Sat Jun 3 19:59:12 2023","interfaces":[{"interface":"back","average":{"1min":0.97799999999999998,"5min":1.395,"15min":2.1360000000000001},"min":{"1min":0.73899999999999999,"5min":0.437,"15min":0.373},"max":{"1min":1.2789999999999999,"5min":7.9400000000000004,"15min":7.9400000000000004},"last":1.1140000000000001},{"interface":"front","average":{"1min":0.98299999999999998,"5min":1.6950000000000001,"15min":2.5609999999999999},"min":{"1min":0.73499999999999999,"5min":0.40000000000000002,"15min":0.40000000000000002},"max":{"1min":1.198,"5min":13.797000000000001,"15min":13.797000000000001},"last":1.1419999999999999}]},{"osd":6,"last update":"Sat Jun 3 19:59:12 2023","interfaces":[{"interface":"back","average":{"1min":0.98799999999999999,"5min":1.657,"15min":2.242},"min":{"1min":0.78000000000000003,"5min":0.41399999999999998,"15min":0.38300000000000001},"max":{"1min":1.2310000000000001,"5min":13.941000000000001,"15min":13.941000000000001},"last":1.2310000000000001},{"interface":"front","average":{"1min":1.05,"5min":1.5389999999999999,"15min":2.706},"min":{"1min":0.72099999999999997,"5min":0.40100000000000002,"15min":0.38700000000000001},"max":{"1min":1.2989999999999999,"5min":9.4060000000000006,"15min":14.847},"last":1.0680000000000001}]},{"osd":7,"last update":"Sat Jun 3 19:59:12 2023","interfaces":[{"interface":"back","average":{"1min":1.083,"5min":1.415,"15min":2.3839999999999999},"min":{"1min":0.76400000000000001,"5min":0.48499999999999999,"15min":0.46000000000000002},"max":{"1min":1.375,"5min":7.056,"15min":14.977},"last":0.98699999999999999},{"interface":"front","average":{"1min":1.0649999999999999,"5min":1.165,"15min":1.714},"min":{"1min":0.76700000000000002,"5min":0.371,"15min":0.371},"max":{"1min":1.444,"5min":6.3789999999999996,"15min":7.6020000000000003},"last":1.198}]}]},{"osd":0,"up_from":86,"seq":369367187603,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":219664,"kb_used_data":1864,"kb_used_omap":5,"kb_used_meta":217786,"kb_avail":93525488,"statfs":{"total":95995035648,"available":95770099712,"internally_reserved":0,"allocated":1908736,"data_stored":1361683,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":223013498},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Jun 3 19:58:11 2023","interfaces":[{"interface":"back","average":{"1min":0.73799999999999999,"5min":1.4219999999999999,"15min":2.194},"min":{"1min":0.42299999999999999,"5min":0.217,"15min":0.217},"max":{"1min":1.0449999999999999,"5min":10.441000000000001,"15min":13.760999999999999},"last":0.68200000000000005},{"interface":"front","average":{"1min":0.67000000000000004,"5min":1.24,"15min":1.8120000000000001},"min":{"1min":0.39700000000000002,"5min":0.26200000000000001,"15min":0.253},"max":{"1min":1.044,"5min":9.1280000000000001,"15min":9.1280000000000001},"last":0.995}]},{"osd":2,"last update":"Sat Jun 3 19:58:14 2023","interfaces":[{"interface":"back","average":{"1min":0.69799999999999995,"5min":0.66600000000000004,"15min":0.66100000000000003},"min":{"1min":0.32300000000000001,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":1.077,"5min":1.077,"15min":1.077},"last":0.64900000000000002},{"interface":"front","average":{"1min":0.79400000000000004,"5min":0.75600000000000001,"15min":0.75},"min":{"1min":0.48799999999999999,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":1.1890000000000001,"5min":1.1890000000000001,"15min":1.1890000000000001},"last":0.88300000000000001}]},{"osd":3,"last update":"Sat Jun 3 19:58:59 2023","interfaces":[{"interface":"back","average":{"1min":0.70299999999999996,"5min":0.79300000000000004,"15min":0.80800000000000005},"min":{"1min":0.42599999999999999,"5min":0.42599999999999999,"15min":0.42599999999999999},"max":{"1min":1.0249999999999999,"5min":1.052,"15min":1.052},"last":0.82399999999999995},{"interface":"front","average":{"1min":0.77000000000000002,"5min":0.76200000000000001,"15min":0.76100000000000001},"min":{"1min":0.35399999999999998,"5min":0.35399999999999998,"15min":0.35399999999999998},"max":{"1min":1.1100000000000001,"5min":1.1819999999999999,"15min":1.1819999999999999},"last":0.95699999999999996}]},{"osd":4,"last update":"Sat Jun 3 19:58:31 2023","interfaces":[{"interface":"back","average":{"1min":1.0960000000000001,"5min":1.0960000000000001,"15min":1.0960000000000001},"min":{"1min":0.84099999999999997,"5min":0.84099999999999997,"15min":0.84099999999999997},"max":{"1min":1.474,"5min":1.474,"15min":1.474},"last":1.1519999999999999},{"interface":"front","average":{"1min":1.1180000000000001,"5min":1.1180000000000001,"15min":1.1180000000000001},"min":{"1min":0.73099999999999998,"5min":0.73099999999999998,"15min":0.73099999999999998},"max":{"1min":1.597,"5min":1.597,"15min":1.597},"last":1.1799999999999999}]},{"osd":5,"last update":"Sat Jun 3 19:58:31 2023","interfaces":[{"interface":"back","average":{"1min":0.99299999999999999,"5min":1.522,"15min":2.1280000000000001},"min":{"1min":0.623,"5min":0.35999999999999999,"15min":0.33900000000000002},"max":{"1min":1.2909999999999999,"5min":9.2059999999999995,"15min":16.312000000000001},"last":1.0760000000000001},{"interface":"front","average":{"1min":1.1279999999999999,"5min":1.722,"15min":1.8080000000000001},"min":{"1min":0.70499999999999996,"5min":0.40500000000000003,"15min":0.32500000000000001},"max":{"1min":1.5449999999999999,"5min":9.3330000000000002,"15min":24.661000000000001},"last":1.214}]},{"osd":6,"last update":"Sat Jun 3 19:58:31 2023","interfaces":[{"interface":"back","average":{"1min":1.1599999999999999,"5min":1.7310000000000001,"15min":1.778},"min":{"1min":0.63900000000000001,"5min":0.36099999999999999,"15min":0.33500000000000002},"max":{"1min":1.538,"5min":9.3629999999999995,"15min":24.599},"last":1.0389999999999999},{"interface":"front","average":{"1min":1.1839999999999999,"5min":1.6839999999999999,"15min":2.1869999999999998},"min":{"1min":0.85299999999999998,"5min":0.42999999999999999,"15min":0.42999999999999999},"max":{"1min":1.6299999999999999,"5min":10.459,"15min":13.601000000000001},"last":1.27}]},{"osd":7,"last update":"Sat Jun 3 19:58:31 2023","interfaces":[{"interface":"back","average":{"1min":1.1579999999999999,"5min":1.7170000000000001,"15min":1.7},"min":{"1min":0.66900000000000004,"5min":0.39900000000000002,"15min":0.39900000000000002},"max":{"1min":1.605,"5min":9.4190000000000005,"15min":24.634},"last":1.462},{"interface":"front","average":{"1min":1.1639999999999999,"5min":1.5600000000000001,"15min":2.0009999999999999},"min":{"1min":0.89300000000000002,"5min":0.45800000000000002,"15min":0.40699999999999997},"max":{"1min":1.573,"5min":12.621,"15min":12.621},"last":1.3680000000000001}]}]},{"osd":2,"up_from":114,"seq":489626271782,"num_pgs":19,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":15572,"kb_used_data":1228,"kb_used_omap":7,"kb_used_meta":14328,"kb_avail":93729580,"statfs":{"total":95995035648,"available":95979089920,"internally_reserved":0,"allocated":1257472,"data_stored":764442,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8111,"internal_metadata":14671953},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 3 19:58:16 2023","interfaces":[{"interface":"back","average":{"1min":0.59399999999999997,"5min":0.61899999999999999,"15min":0.623},"min":{"1min":0.35899999999999999,"5min":0.314,"15min":0.314},"max":{"1min":0.90300000000000002,"5min":0.90300000000000002,"15min":0.90300000000000002},"last":0.627},{"interface":"front","average":{"1min":0.71199999999999997,"5min":0.70899999999999996,"15min":0.70899999999999996},"min":{"1min":0.42099999999999999,"5min":0.42099999999999999,"15min":0.42099999999999999},"max":{"1min":1.04,"5min":1.083,"15min":1.083},"last":0.82599999999999996}]},{"osd":1,"last update":"Sat Jun 3 19:58:16 2023","interfaces":[{"interface":"back","average":{"1min":0.68100000000000005,"5min":0.71299999999999997,"15min":0.71799999999999997},"min":{"1min":0.36499999999999999,"5min":0.36499999999999999,"15min":0.36499999999999999},"max":{"1min":0.97699999999999998,"5min":1.1020000000000001,"15min":1.1020000000000001},"last":0.58999999999999997},{"interface":"front","average":{"1min":0.65700000000000003,"5min":0.67200000000000004,"15min":0.67400000000000004},"min":{"1min":0.49299999999999999,"5min":0.443,"15min":0.443},"max":{"1min":1.0580000000000001,"5min":1.0580000000000001,"15min":1.0580000000000001},"last":0.85999999999999999}]},{"osd":3,"last update":"Sat Jun 3 19:59:02 2023","interfaces":[{"interface":"back","average":{"1min":0.65100000000000002,"5min":0.73099999999999998,"15min":0.74399999999999999},"min":{"1min":0.39400000000000002,"5min":0.39400000000000002,"15min":0.39400000000000002},"max":{"1min":0.98399999999999999,"5min":1.0409999999999999,"15min":1.0409999999999999},"last":0.55600000000000005},{"interface":"front","average":{"1min":0.70999999999999996,"5min":0.748,"15min":0.754},"min":{"1min":0.38,"5min":0.38,"15min":0.38},"max":{"1min":1.155,"5min":1.155,"15min":1.155},"last":0.75800000000000001}]},{"osd":4,"last update":"Sat Jun 3 19:58:31 2023","interfaces":[{"interface":"back","average":{"1min":0.92100000000000004,"5min":0.92100000000000004,"15min":0.92100000000000004},"min":{"1min":0.70699999999999996,"5min":0.70699999999999996,"15min":0.70699999999999996},"max":{"1min":1.2090000000000001,"5min":1.2090000000000001,"15min":1.2090000000000001},"last":0.90000000000000002},{"interface":"front","average":{"1min":0.97499999999999998,"5min":0.97499999999999998,"15min":0.97499999999999998},"min":{"1min":0.68100000000000005,"5min":0.68100000000000005,"15min":0.68100000000000005},"max":{"1min":1.3069999999999999,"5min":1.3069999999999999,"15min":1.3069999999999999},"last":0.91500000000000004}]},{"osd":5,"last update":"Sat Jun 3 19:58:16 2023","interfaces":[{"interface":"back","average":{"1min":1.0049999999999999,"5min":1.0309999999999999,"15min":1.0349999999999999},"min":{"1min":0.77200000000000002,"5min":0.65500000000000003,"15min":0.65500000000000003},"max":{"1min":1.3979999999999999,"5min":1.3979999999999999,"15min":1.3979999999999999},"last":1.107},{"interface":"front","average":{"1min":1.028,"5min":1.044,"15min":1.0469999999999999},"min":{"1min":0.65100000000000002,"5min":0.65100000000000002,"15min":0.65100000000000002},"max":{"1min":1.335,"5min":1.369,"15min":1.369},"last":1.044}]},{"osd":6,"last update":"Sat Jun 3 19:58:16 2023","interfaces":[{"interface":"back","average":{"1min":1.0409999999999999,"5min":1.036,"15min":1.0349999999999999},"min":{"1min":0.71699999999999997,"5min":0.71699999999999997,"15min":0.71699999999999997},"max":{"1min":1.3899999999999999,"5min":1.3899999999999999,"15min":1.3899999999999999},"last":1.2450000000000001},{"interface":"front","average":{"1min":1.0780000000000001,"5min":1.052,"15min":1.048},"min":{"1min":0.73899999999999999,"5min":0.68100000000000005,"15min":0.68100000000000005},"max":{"1min":1.3919999999999999,"5min":1.508,"15min":1.508},"last":1.1399999999999999}]},{"osd":7,"last update":"Sat Jun 3 19:58:16 2023","interfaces":[{"interface":"back","average":{"1min":1.1160000000000001,"5min":1.167,"15min":1.175},"min":{"1min":0.81999999999999995,"5min":0.77100000000000002,"15min":0.77100000000000002},"max":{"1min":1.456,"5min":1.456,"15min":1.456},"last":1.3220000000000001},{"interface":"front","average":{"1min":1.1220000000000001,"5min":1.151,"15min":1.1559999999999999},"min":{"1min":0.78600000000000003,"5min":0.78600000000000003,"15min":0.78600000000000003},"max":{"1min":1.3879999999999999,"5min":1.6930000000000001,"15min":1.6930000000000001},"last":1.2909999999999999}]}]},{"osd":3,"up_from":119,"seq":511101108254,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":18108,"kb_used_data":1204,"kb_used_omap":2,"kb_used_meta":16893,"kb_avail":93727044,"statfs":{"total":95995035648,"available":95976493056,"internally_reserved":0,"allocated":1232896,"data_stored":760088,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":17298485},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 3 19:58:58 2023","interfaces":[{"interface":"back","average":{"1min":0.57999999999999996,"5min":0.621,"15min":0.628},"min":{"1min":0.33300000000000002,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":0.96799999999999997,"5min":0.96799999999999997,"15min":0.96799999999999997},"last":1.512},{"interface":"front","average":{"1min":0.58599999999999997,"5min":0.70199999999999996,"15min":0.72099999999999997},"min":{"1min":0.23799999999999999,"5min":0.23799999999999999,"15min":0.23799999999999999},"max":{"1min":0.85999999999999999,"5min":1,"15min":1},"last":1.1859999999999999}]},{"osd":1,"last update":"Sat Jun 3 19:58:58 2023","interfaces":[{"interface":"back","average":{"1min":0.59899999999999998,"5min":0.68799999999999994,"15min":0.70299999999999996},"min":{"1min":0.314,"5min":0.314,"15min":0.314},"max":{"1min":0.93000000000000005,"5min":1.028,"15min":1.028},"last":1.6220000000000001},{"interface":"front","average":{"1min":0.67600000000000005,"5min":0.74399999999999999,"15min":0.755},"min":{"1min":0.28499999999999998,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":1.018,"5min":1.018,"15min":1.018},"last":1.5640000000000001}]},{"osd":2,"last update":"Sat Jun 3 19:58:58 2023","interfaces":[{"interface":"back","average":{"1min":0.64800000000000002,"5min":0.64400000000000002,"15min":0.64400000000000002},"min":{"1min":0.32400000000000001,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":1.03,"5min":1.03,"15min":1.03},"last":0.59099999999999997},{"interface":"front","average":{"1min":0.65100000000000002,"5min":0.73999999999999999,"15min":0.755},"min":{"1min":0.34799999999999998,"5min":0.34799999999999998,"15min":0.34799999999999998},"max":{"1min":1.046,"5min":1.1140000000000001,"15min":1.1140000000000001},"last":1.8939999999999999}]},{"osd":4,"last update":"Sat Jun 3 19:58:28 2023","interfaces":[{"interface":"back","average":{"1min":1,"5min":1,"15min":1},"min":{"1min":0.71499999999999997,"5min":0.71499999999999997,"15min":0.71499999999999997},"max":{"1min":1.3819999999999999,"5min":1.3819999999999999,"15min":1.3819999999999999},"last":1.575},{"interface":"front","average":{"1min":1.0109999999999999,"5min":1.0109999999999999,"15min":1.0109999999999999},"min":{"1min":0.78200000000000003,"5min":0.78200000000000003,"15min":0.78200000000000003},"max":{"1min":1.2689999999999999,"5min":1.2689999999999999,"15min":1.2689999999999999},"last":2.21}]},{"osd":5,"last update":"Sat Jun 3 19:58:58 2023","interfaces":[{"interface":"back","average":{"1min":0.97899999999999998,"5min":0.98499999999999999,"15min":0.98599999999999999},"min":{"1min":0.69499999999999995,"5min":0.69499999999999995,"15min":0.69499999999999995},"max":{"1min":1.2929999999999999,"5min":1.429,"15min":1.429},"last":2.1459999999999999},{"interface":"front","average":{"1min":0.95699999999999996,"5min":1.0109999999999999,"15min":1.02},"min":{"1min":0.56200000000000006,"5min":0.56200000000000006,"15min":0.56200000000000006},"max":{"1min":1.343,"5min":1.5900000000000001,"15min":1.5900000000000001},"last":0.98899999999999999}]},{"osd":6,"last update":"Sat Jun 3 19:58:58 2023","interfaces":[{"interface":"back","average":{"1min":1.089,"5min":1.044,"15min":1.036},"min":{"1min":0.74099999999999999,"5min":0.67100000000000004,"15min":0.67100000000000004},"max":{"1min":1.351,"5min":1.6559999999999999,"15min":1.6559999999999999},"last":0.871},{"interface":"front","average":{"1min":1.04,"5min":1.054,"15min":1.056},"min":{"1min":0.58299999999999996,"5min":0.58299999999999996,"15min":0.58299999999999996},"max":{"1min":1.369,"5min":1.369,"15min":1.369},"last":2.0920000000000001}]},{"osd":7,"last update":"Sat Jun 3 19:58:58 2023","interfaces":[{"interface":"back","average":{"1min":0.98899999999999999,"5min":1.0329999999999999,"15min":1.04},"min":{"1min":0.62,"5min":0.62,"15min":0.62},"max":{"1min":1.351,"5min":1.351,"15min":1.351},"last":2.23},{"interface":"front","average":{"1min":0.97899999999999998,"5min":1.018,"15min":1.024},"min":{"1min":0.53600000000000003,"5min":0.53600000000000003,"15min":0.53600000000000003},"max":{"1min":1.3129999999999999,"5min":1.4610000000000001,"15min":1.4610000000000001},"last":2.0699999999999998}]}]},{"osd":4,"up_from":124,"seq":532575944727,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":15492,"kb_used_data":1212,"kb_used_omap":10,"kb_used_meta":14261,"kb_avail":93729660,"statfs":{"total":95995035648,"available":95979171840,"internally_reserved":0,"allocated":1241088,"data_stored":762768,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":10652,"internal_metadata":14603876},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 3 19:58:29 2023","interfaces":[{"interface":"back","average":{"1min":0.97499999999999998,"5min":0.97499999999999998,"15min":0.97499999999999998},"min":{"1min":0.55800000000000005,"5min":0.55800000000000005,"15min":0.55800000000000005},"max":{"1min":1.3240000000000001,"5min":1.3240000000000001,"15min":1.3240000000000001},"last":1.006},{"interface":"front","average":{"1min":1.0309999999999999,"5min":1.0309999999999999,"15min":1.0309999999999999},"min":{"1min":0.78200000000000003,"5min":0.78200000000000003,"15min":0.78200000000000003},"max":{"1min":1.2250000000000001,"5min":1.2250000000000001,"15min":1.2250000000000001},"last":1.1850000000000001}]},{"osd":1,"last update":"Sat Jun 3 19:58:29 2023","interfaces":[{"interface":"back","average":{"1min":0.871,"5min":0.871,"15min":0.871},"min":{"1min":0.60999999999999999,"5min":0.60999999999999999,"15min":0.60999999999999999},"max":{"1min":1.2549999999999999,"5min":1.2549999999999999,"15min":1.2549999999999999},"last":1.22},{"interface":"front","average":{"1min":0.86299999999999999,"5min":0.86299999999999999,"15min":0.86299999999999999},"min":{"1min":0.54900000000000004,"5min":0.54900000000000004,"15min":0.54900000000000004},"max":{"1min":1.278,"5min":1.278,"15min":1.278},"last":0.91900000000000004}]},{"osd":2,"last update":"Sat Jun 3 19:58:29 2023","interfaces":[{"interface":"back","average":{"1min":0.871,"5min":0.871,"15min":0.871},"min":{"1min":0.52800000000000002,"5min":0.52800000000000002,"15min":0.52800000000000002},"max":{"1min":1.196,"5min":1.196,"15min":1.196},"last":1.0700000000000001},{"interface":"front","average":{"1min":0.93799999999999994,"5min":0.93799999999999994,"15min":0.93799999999999994},"min":{"1min":0.63700000000000001,"5min":0.63700000000000001,"15min":0.63700000000000001},"max":{"1min":1.2769999999999999,"5min":1.2769999999999999,"15min":1.2769999999999999},"last":1.0960000000000001}]},{"osd":3,"last update":"Sat Jun 3 19:58:29 2023","interfaces":[{"interface":"back","average":{"1min":0.96199999999999997,"5min":0.96199999999999997,"15min":0.96199999999999997},"min":{"1min":0.68700000000000006,"5min":0.68700000000000006,"15min":0.68700000000000006},"max":{"1min":1.238,"5min":1.238,"15min":1.238},"last":0.84499999999999997},{"interface":"front","average":{"1min":1.056,"5min":1.056,"15min":1.056},"min":{"1min":0.71899999999999997,"5min":0.71899999999999997,"15min":0.71899999999999997},"max":{"1min":1.28,"5min":1.28,"15min":1.28},"last":1.1619999999999999}]},{"osd":5,"last update":"Sat Jun 3 19:58:29 2023","interfaces":[{"interface":"back","average":{"1min":0.95299999999999996,"5min":0.95299999999999996,"15min":0.95299999999999996},"min":{"1min":0.72399999999999998,"5min":0.72399999999999998,"15min":0.72399999999999998},"max":{"1min":1.234,"5min":1.234,"15min":1.234},"last":0.76800000000000002},{"interface":"front","average":{"1min":0.98799999999999999,"5min":0.98799999999999999,"15min":0.98799999999999999},"min":{"1min":0.77200000000000002,"5min":0.77200000000000002,"15min":0.77200000000000002},"max":{"1min":1.24,"5min":1.24,"15min":1.24},"last":1.137}]},{"osd":6,"last update":"Sat Jun 3 19:58:29 2023","interfaces":[{"interface":"back","average":{"1min":1.0449999999999999,"5min":1.0449999999999999,"15min":1.0449999999999999},"min":{"1min":0.51300000000000001,"5min":0.51300000000000001,"15min":0.51300000000000001},"max":{"1min":1.323,"5min":1.323,"15min":1.323},"last":0.97399999999999998},{"interface":"front","average":{"1min":1.079,"5min":1.079,"15min":1.079},"min":{"1min":0.73399999999999999,"5min":0.73399999999999999,"15min":0.73399999999999999},"max":{"1min":1.494,"5min":1.494,"15min":1.494},"last":1.0249999999999999}]},{"osd":7,"last update":"Sat Jun 3 19:58:29 2023","interfaces":[{"interface":"back","average":{"1min":1.1379999999999999,"5min":1.1379999999999999,"15min":1.1379999999999999},"min":{"1min":0.77100000000000002,"5min":0.77100000000000002,"15min":0.77100000000000002},"max":{"1min":1.4490000000000001,"5min":1.4490000000000001,"15min":1.4490000000000001},"last":1.27},{"interface":"front","average":{"1min":1.171,"5min":1.171,"15min":1.171},"min":{"1min":0.90300000000000002,"5min":0.90300000000000002,"15min":0.90300000000000002},"max":{"1min":1.383,"5min":1.383,"15min":1.383},"last":0.71699999999999997}]}]},{"osd":5,"up_from":36,"seq":154618823013,"num_pgs":21,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":358044,"kb_used_data":1172,"kb_used_omap":0,"kb_used_meta":356864,"kb_avail":93387108,"statfs":{"total":95995035648,"available":95628398592,"internally_reserved":0,"allocated":1200128,"data_stored":753684,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":365428736},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 3 19:58:19 2023","interfaces":[{"interface":"back","average":{"1min":0.91200000000000003,"5min":1.3029999999999999,"15min":1.4890000000000001},"min":{"1min":0.63600000000000001,"5min":0.26700000000000002,"15min":0.26600000000000001},"max":{"1min":1.147,"5min":7.54,"15min":12.489000000000001},"last":0.88700000000000001},{"interface":"front","average":{"1min":0.98199999999999998,"5min":1.6259999999999999,"15min":1.7310000000000001},"min":{"1min":0.82699999999999996,"5min":0.37,"15min":0.31},"max":{"1min":1.133,"5min":10.403,"15min":12.542999999999999},"last":1.036}]},{"osd":1,"last update":"Sat Jun 3 19:58:14 2023","interfaces":[{"interface":"back","average":{"1min":0.998,"5min":1.6719999999999999,"15min":1.8500000000000001},"min":{"1min":0.76100000000000001,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":1.4059999999999999,"5min":11.27,"15min":11.27},"last":1.272},{"interface":"front","average":{"1min":0.86099999999999999,"5min":1.7849999999999999,"15min":1.9330000000000001},"min":{"1min":0.56599999999999995,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":1.0860000000000001,"5min":11.34,"15min":11.34},"last":1.101}]},{"osd":2,"last update":"Sat Jun 3 19:58:14 2023","interfaces":[{"interface":"back","average":{"1min":0.91500000000000004,"5min":0.876,"15min":0.87},"min":{"1min":0.66500000000000004,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":1.169,"5min":1.169,"15min":1.169},"last":1.141},{"interface":"front","average":{"1min":0.94399999999999995,"5min":0.96999999999999997,"15min":0.97399999999999998},"min":{"1min":0.73399999999999999,"5min":0.48799999999999999,"15min":0.48799999999999999},"max":{"1min":1.232,"5min":1.335,"15min":1.335},"last":0.94099999999999995}]},{"osd":3,"last update":"Sat Jun 3 19:58:55 2023","interfaces":[{"interface":"back","average":{"1min":1.004,"5min":1.0149999999999999,"15min":1.0169999999999999},"min":{"1min":0.66600000000000004,"5min":0.38,"15min":0.38},"max":{"1min":1.3779999999999999,"5min":1.3779999999999999,"15min":1.3779999999999999},"last":1.389},{"interface":"front","average":{"1min":1.0920000000000001,"5min":1.0109999999999999,"15min":0.997},"min":{"1min":0.81799999999999995,"5min":0.50700000000000001,"15min":0.50700000000000001},"max":{"1min":1.361,"5min":1.363,"15min":1.363},"last":1.0049999999999999}]},{"osd":4,"last update":"Sat Jun 3 19:58:30 2023","interfaces":[{"interface":"back","average":{"1min":1.002,"5min":1.002,"15min":1.002},"min":{"1min":0.60299999999999998,"5min":0.60299999999999998,"15min":0.60299999999999998},"max":{"1min":1.3500000000000001,"5min":1.3500000000000001,"15min":1.3500000000000001},"last":1.1120000000000001},{"interface":"front","average":{"1min":1.1879999999999999,"5min":1.1879999999999999,"15min":1.1879999999999999},"min":{"1min":1.022,"5min":1.022,"15min":1.022},"max":{"1min":1.522,"5min":1.522,"15min":1.522},"last":1.294}]},{"osd":6,"last update":"Sat Jun 3 19:58:19 2023","interfaces":[{"interface":"back","average":{"1min":1.1000000000000001,"5min":1.6419999999999999,"15min":1.1970000000000001},"min":{"1min":0.80900000000000005,"5min":0.33000000000000002,"15min":0.33000000000000002},"max":{"1min":1.474,"5min":12.731999999999999,"15min":12.731999999999999},"last":0.96999999999999997},{"interface":"front","average":{"1min":1.153,"5min":1.698,"15min":1.327},"min":{"1min":0.88700000000000001,"5min":0.42899999999999999,"15min":0.32800000000000001},"max":{"1min":1.4339999999999999,"5min":12.779,"15min":12.779},"last":1.343}]},{"osd":7,"last update":"Sat Jun 3 19:59:06 2023","interfaces":[{"interface":"back","average":{"1min":1.198,"5min":1.605,"15min":1.3979999999999999},"min":{"1min":0.68799999999999994,"5min":0.45700000000000002,"15min":0.379},"max":{"1min":1.5229999999999999,"5min":11.297000000000001,"15min":11.297000000000001},"last":1.198},{"interface":"front","average":{"1min":1.1950000000000001,"5min":1.2909999999999999,"15min":1.252},"min":{"1min":0.89300000000000002,"5min":0.39600000000000002,"15min":0.32100000000000001},"max":{"1min":1.468,"5min":5.9240000000000004,"15min":7.4720000000000004},"last":1.228}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6928,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4862,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":10652,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4786,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4932,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-06-03T19:59:16.718 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph pg dump --format=json 2023-06-03T19:59:17.029 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:59:16 smithi089 bash[75068]: audit 1685822355.8450582 mgr.y (mgr.34104) 687 : audit 0 from='client.45632 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-06-03T19:59:17.029 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:59:16 smithi089 bash[78452]: audit 1685822355.8450582 mgr.y (mgr.34104) 687 : audit 0 from='client.45632 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-06-03T19:59:17.084 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:59:16 smithi164 bash[59462]: audit 1685822355.8450582 mgr.y (mgr.34104) 687 : audit 0 from='client.45632 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-06-03T19:59:18.028 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:59:17 smithi089 bash[75068]: cluster 1685822356.4001806 mgr.y (mgr.34104) 688 : cluster 0 pgmap v593: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-06-03T19:59:18.029 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:59:17 smithi089 bash[78452]: cluster 1685822356.4001806 mgr.y (mgr.34104) 688 : cluster 0 pgmap v593: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-06-03T19:59:18.084 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:59:17 smithi164 bash[59462]: cluster 1685822356.4001806 mgr.y (mgr.34104) 688 : cluster 0 pgmap v593: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-06-03T19:59:20.028 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:59:19 smithi089 bash[78452]: cluster 1685822358.4012868 mgr.y (mgr.34104) 689 : cluster 0 pgmap v594: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-06-03T19:59:20.029 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:59:19 smithi089 bash[75068]: cluster 1685822358.4012868 mgr.y (mgr.34104) 689 : cluster 0 pgmap v594: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-06-03T19:59:20.084 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:59:19 smithi164 bash[59462]: cluster 1685822358.4012868 mgr.y (mgr.34104) 689 : cluster 0 pgmap v594: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-06-03T19:59:21.579 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/mon.a/config 2023-06-03T19:59:22.029 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:59:21 smithi089 bash[75068]: cluster 1685822360.4019768 mgr.y (mgr.34104) 690 : cluster 0 pgmap v595: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-06-03T19:59:22.029 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:59:21 smithi089 bash[78452]: cluster 1685822360.4019768 mgr.y (mgr.34104) 690 : cluster 0 pgmap v595: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-06-03T19:59:22.084 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:59:21 smithi164 bash[59462]: cluster 1685822360.4019768 mgr.y (mgr.34104) 690 : cluster 0 pgmap v595: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-06-03T19:59:22.940 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:59:22 smithi089 bash[78452]: audit 1685822362.0928037 mon.c (mon.2) 368 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T19:59:22.940 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:59:22 smithi089 bash[75068]: audit 1685822362.0928037 mon.c (mon.2) 368 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T19:59:23.084 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:59:22 smithi164 bash[59462]: audit 1685822362.0928037 mon.c (mon.2) 368 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T19:59:24.031 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:59:23 smithi089 bash[75068]: cluster 1685822362.402753 mgr.y (mgr.34104) 691 : cluster 0 pgmap v596: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-06-03T19:59:24.031 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:59:23 smithi089 bash[78452]: cluster 1685822362.402753 mgr.y (mgr.34104) 691 : cluster 0 pgmap v596: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-06-03T19:59:24.084 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:59:23 smithi164 bash[59462]: cluster 1685822362.402753 mgr.y (mgr.34104) 691 : cluster 0 pgmap v596: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-06-03T19:59:25.282 INFO:teuthology.orchestra.run.smithi089.stdout: 2023-06-03T19:59:25.286 INFO:teuthology.orchestra.run.smithi089.stderr:dumped all 2023-06-03T19:59:26.028 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:59:25 smithi089 bash[75068]: cluster 1685822364.4036427 mgr.y (mgr.34104) 692 : cluster 0 pgmap v597: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-06-03T19:59:26.029 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:59:25 smithi089 bash[78452]: cluster 1685822364.4036427 mgr.y (mgr.34104) 692 : cluster 0 pgmap v597: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-06-03T19:59:26.049 INFO:teuthology.orchestra.run.smithi089.stdout:{"pg_ready":true,"pg_map":{"version":597,"stamp":"2023-06-03T19:59:24.403064+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":595854,"num_objects":24,"num_object_clones":0,"num_object_copies":72,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":24,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":261,"num_write_kb":2446,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":195,"ondisk_log_size":195,"up":219,"acting":219,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":219,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":1315012,"kb_used_data":11460,"kb_used_omap":30,"kb_used_meta":1303457,"kb_avail":748646204,"statfs":{"total":767960285184,"available":766613712896,"internally_reserved":0,"allocated":11735040,"data_stored":7866659,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":31502,"internal_metadata":1334740210},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.004735"},"pg_stats":[{"pgid":"3.1a","version":"60'1","reported_seq":110,"reported_epoch":125,"state":"active+clean","last_fresh":"2023-06-03T19:57:28.467559+0000","last_change":"2023-06-03T19:57:28.467559+0000","last_active":"2023-06-03T19:57:28.467559+0000","last_peered":"2023-06-03T19:57:28.467559+0000","last_clean":"2023-06-03T19:57:28.467559+0000","last_became_active":"2023-06-03T19:57:28.467077+0000","last_became_peered":"2023-06-03T19:57:28.467077+0000","last_unstale":"2023-06-03T19:57:28.467559+0000","last_undegraded":"2023-06-03T19:57:28.467559+0000","last_fullsized":"2023-06-03T19:57:28.467559+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T21:46:02.795576+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":104,"reported_epoch":125,"state":"active+clean","last_fresh":"2023-06-03T19:57:28.471575+0000","last_change":"2023-06-03T19:57:28.470287+0000","last_active":"2023-06-03T19:57:28.471575+0000","last_peered":"2023-06-03T19:57:28.471575+0000","last_clean":"2023-06-03T19:57:28.471575+0000","last_became_active":"2023-06-03T19:57:28.469477+0000","last_became_peered":"2023-06-03T19:57:28.469477+0000","last_unstale":"2023-06-03T19:57:28.471575+0000","last_undegraded":"2023-06-03T19:57:28.471575+0000","last_fullsized":"2023-06-03T19:57:28.471575+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T06:48:19.229190+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":117,"reported_epoch":125,"state":"active+clean","last_fresh":"2023-06-03T19:57:28.473690+0000","last_change":"2023-06-03T19:57:28.473350+0000","last_active":"2023-06-03T19:57:28.473690+0000","last_peered":"2023-06-03T19:57:28.473690+0000","last_clean":"2023-06-03T19:57:28.473690+0000","last_became_active":"2023-06-03T19:57:28.472757+0000","last_became_peered":"2023-06-03T19:57:28.472757+0000","last_unstale":"2023-06-03T19:57:28.473690+0000","last_undegraded":"2023-06-03T19:57:28.473690+0000","last_fullsized":"2023-06-03T19:57:28.473690+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T19:58:10.432188+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":81,"reported_epoch":125,"state":"active+clean","last_fresh":"2023-06-03T19:57:28.450909+0000","last_change":"2023-06-03T19:56:54.507653+0000","last_active":"2023-06-03T19:57:28.450909+0000","last_peered":"2023-06-03T19:57:28.450909+0000","last_clean":"2023-06-03T19:57:28.450909+0000","last_became_active":"2023-06-03T19:56:54.507350+0000","last_became_peered":"2023-06-03T19:56:54.507350+0000","last_unstale":"2023-06-03T19:57:28.450909+0000","last_undegraded":"2023-06-03T19:57:28.450909+0000","last_fullsized":"2023-06-03T19:57:28.450909+0000","mapping_epoch":119,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T22:33:20.065844+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":105,"reported_epoch":125,"state":"active+clean","last_fresh":"2023-06-03T19:57:28.452903+0000","last_change":"2023-06-03T19:56:54.512932+0000","last_active":"2023-06-03T19:57:28.452903+0000","last_peered":"2023-06-03T19:57:28.452903+0000","last_clean":"2023-06-03T19:57:28.452903+0000","last_became_active":"2023-06-03T19:56:54.511992+0000","last_became_peered":"2023-06-03T19:56:54.511992+0000","last_unstale":"2023-06-03T19:57:28.452903+0000","last_undegraded":"2023-06-03T19:57:28.452903+0000","last_fullsized":"2023-06-03T19:57:28.452903+0000","mapping_epoch":119,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T21:01:35.069911+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":91,"reported_epoch":124,"state":"active+clean","last_fresh":"2023-06-03T19:57:27.437336+0000","last_change":"2023-06-03T19:51:59.359522+0000","last_active":"2023-06-03T19:57:27.437336+0000","last_peered":"2023-06-03T19:57:27.437336+0000","last_clean":"2023-06-03T19:57:27.437336+0000","last_became_active":"2023-06-03T19:51:59.359202+0000","last_became_peered":"2023-06-03T19:51:59.359202+0000","last_unstale":"2023-06-03T19:57:27.437336+0000","last_undegraded":"2023-06-03T19:57:27.437336+0000","last_fullsized":"2023-06-03T19:57:27.437336+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":79,"reported_epoch":125,"state":"active+clean","last_fresh":"2023-06-03T19:57:28.451456+0000","last_change":"2023-06-03T19:56:54.509490+0000","last_active":"2023-06-03T19:57:28.451456+0000","last_peered":"2023-06-03T19:57:28.451456+0000","last_clean":"2023-06-03T19:57:28.451456+0000","last_became_active":"2023-06-03T19:56:54.509145+0000","last_became_peered":"2023-06-03T19:56:54.509145+0000","last_unstale":"2023-06-03T19:57:28.451456+0000","last_undegraded":"2023-06-03T19:57:28.451456+0000","last_fullsized":"2023-06-03T19:57:28.451456+0000","mapping_epoch":119,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T07:12:03.964110+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":89,"reported_epoch":125,"state":"active+clean","last_fresh":"2023-06-03T19:57:28.452799+0000","last_change":"2023-06-03T19:56:54.511325+0000","last_active":"2023-06-03T19:57:28.452799+0000","last_peered":"2023-06-03T19:57:28.452799+0000","last_clean":"2023-06-03T19:57:28.452799+0000","last_became_active":"2023-06-03T19:56:54.511033+0000","last_became_peered":"2023-06-03T19:56:54.511033+0000","last_unstale":"2023-06-03T19:57:28.452799+0000","last_undegraded":"2023-06-03T19:57:28.452799+0000","last_fullsized":"2023-06-03T19:57:28.452799+0000","mapping_epoch":119,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T23:50:08.711478+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":92,"reported_epoch":124,"state":"active+clean","last_fresh":"2023-06-03T19:57:27.439764+0000","last_change":"2023-06-03T19:51:59.374143+0000","last_active":"2023-06-03T19:57:27.439764+0000","last_peered":"2023-06-03T19:57:27.439764+0000","last_clean":"2023-06-03T19:57:27.439764+0000","last_became_active":"2023-06-03T19:51:59.373146+0000","last_became_peered":"2023-06-03T19:51:59.373146+0000","last_unstale":"2023-06-03T19:57:27.439764+0000","last_undegraded":"2023-06-03T19:57:27.439764+0000","last_fullsized":"2023-06-03T19:57:27.439764+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":70,"reported_epoch":125,"state":"active+clean","last_fresh":"2023-06-03T19:57:28.457731+0000","last_change":"2023-06-03T19:56:54.510462+0000","last_active":"2023-06-03T19:57:28.457731+0000","last_peered":"2023-06-03T19:57:28.457731+0000","last_clean":"2023-06-03T19:57:28.457731+0000","last_became_active":"2023-06-03T19:56:54.509783+0000","last_became_peered":"2023-06-03T19:56:54.509783+0000","last_unstale":"2023-06-03T19:57:28.457731+0000","last_undegraded":"2023-06-03T19:57:28.457731+0000","last_fullsized":"2023-06-03T19:57:28.457731+0000","mapping_epoch":119,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T00:16:09.624276+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"60'3","reported_seq":110,"reported_epoch":125,"state":"active+clean","last_fresh":"2023-06-03T19:57:28.466401+0000","last_change":"2023-06-03T19:57:28.466401+0000","last_active":"2023-06-03T19:57:28.466401+0000","last_peered":"2023-06-03T19:57:28.466401+0000","last_clean":"2023-06-03T19:57:28.466401+0000","last_became_active":"2023-06-03T19:57:28.465452+0000","last_became_peered":"2023-06-03T19:57:28.465452+0000","last_unstale":"2023-06-03T19:57:28.466401+0000","last_undegraded":"2023-06-03T19:57:28.466401+0000","last_fullsized":"2023-06-03T19:57:28.466401+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":61,"reported_epoch":125,"state":"active+clean","last_fresh":"2023-06-03T19:57:28.458983+0000","last_change":"2023-06-03T19:56:11.753995+0000","last_active":"2023-06-03T19:57:28.458983+0000","last_peered":"2023-06-03T19:57:28.458983+0000","last_clean":"2023-06-03T19:57:28.458983+0000","last_became_active":"2023-06-03T19:56:11.753676+0000","last_became_peered":"2023-06-03T19:56:11.753676+0000","last_unstale":"2023-06-03T19:57:28.458983+0000","last_undegraded":"2023-06-03T19:57:28.458983+0000","last_fullsized":"2023-06-03T19:57:28.458983+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":115,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T01:00:40.343647+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"77'23","reported_seq":106,"reported_epoch":125,"state":"active+clean","last_fresh":"2023-06-03T19:57:28.462522+0000","last_change":"2023-06-03T19:57:28.462522+0000","last_active":"2023-06-03T19:57:28.462522+0000","last_peered":"2023-06-03T19:57:28.462522+0000","last_clean":"2023-06-03T19:57:28.462522+0000","last_became_active":"2023-06-03T19:57:28.462048+0000","last_became_peered":"2023-06-03T19:57:28.462048+0000","last_unstale":"2023-06-03T19:57:28.462522+0000","last_undegraded":"2023-06-03T19:57:28.462522+0000","last_fullsized":"2023-06-03T19:57:28.462522+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":23,"log_dups_size":0,"ondisk_log_size":23,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T23:23:56.697534+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4716,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":23,"num_write_kb":20,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":83,"reported_epoch":125,"state":"active+clean","last_fresh":"2023-06-03T19:57:28.469676+0000","last_change":"2023-06-03T19:57:28.469676+0000","last_active":"2023-06-03T19:57:28.469676+0000","last_peered":"2023-06-03T19:57:28.469676+0000","last_clean":"2023-06-03T19:57:28.469676+0000","last_became_active":"2023-06-03T19:57:28.469328+0000","last_became_peered":"2023-06-03T19:57:28.469328+0000","last_unstale":"2023-06-03T19:57:28.469676+0000","last_undegraded":"2023-06-03T19:57:28.469676+0000","last_fullsized":"2023-06-03T19:57:28.469676+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T23:25:26.041930+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"60'2","reported_seq":109,"reported_epoch":125,"state":"active+clean","last_fresh":"2023-06-03T19:57:28.467711+0000","last_change":"2023-06-03T19:57:28.467711+0000","last_active":"2023-06-03T19:57:28.467711+0000","last_peered":"2023-06-03T19:57:28.467711+0000","last_clean":"2023-06-03T19:57:28.467711+0000","last_became_active":"2023-06-03T19:57:28.467234+0000","last_became_peered":"2023-06-03T19:57:28.467234+0000","last_unstale":"2023-06-03T19:57:28.467711+0000","last_undegraded":"2023-06-03T19:57:28.467711+0000","last_fullsized":"2023-06-03T19:57:28.467711+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":89,"reported_epoch":125,"state":"active+clean","last_fresh":"2023-06-03T19:57:28.457345+0000","last_change":"2023-06-03T19:56:54.514616+0000","last_active":"2023-06-03T19:57:28.457345+0000","last_peered":"2023-06-03T19:57:28.457345+0000","last_clean":"2023-06-03T19:57:28.457345+0000","last_became_active":"2023-06-03T19:56:54.513918+0000","last_became_peered":"2023-06-03T19:56:54.513918+0000","last_unstale":"2023-06-03T19:57:28.457345+0000","last_undegraded":"2023-06-03T19:57:28.457345+0000","last_fullsized":"2023-06-03T19:57:28.457345+0000","mapping_epoch":119,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T19:36:10.967386+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":79,"reported_epoch":125,"state":"active+clean","last_fresh":"2023-06-03T19:57:28.450526+0000","last_change":"2023-06-03T19:56:54.507479+0000","last_active":"2023-06-03T19:57:28.450526+0000","last_peered":"2023-06-03T19:57:28.450526+0000","last_clean":"2023-06-03T19:57:28.450526+0000","last_became_active":"2023-06-03T19:56:54.507149+0000","last_became_peered":"2023-06-03T19:56:54.507149+0000","last_unstale":"2023-06-03T19:57:28.450526+0000","last_undegraded":"2023-06-03T19:57:28.450526+0000","last_fullsized":"2023-06-03T19:57:28.450526+0000","mapping_epoch":119,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T20:22:36.676317+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":91,"reported_epoch":124,"state":"active+clean","last_fresh":"2023-06-03T19:57:27.442389+0000","last_change":"2023-06-03T19:56:54.512472+0000","last_active":"2023-06-03T19:57:27.442389+0000","last_peered":"2023-06-03T19:57:27.442389+0000","last_clean":"2023-06-03T19:57:27.442389+0000","last_became_active":"2023-06-03T19:56:54.512051+0000","last_became_peered":"2023-06-03T19:56:54.512051+0000","last_unstale":"2023-06-03T19:57:27.442389+0000","last_undegraded":"2023-06-03T19:57:27.442389+0000","last_fullsized":"2023-06-03T19:57:27.442389+0000","mapping_epoch":119,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":92,"reported_epoch":125,"state":"active+clean","last_fresh":"2023-06-03T19:57:28.462780+0000","last_change":"2023-06-03T19:57:28.462780+0000","last_active":"2023-06-03T19:57:28.462780+0000","last_peered":"2023-06-03T19:57:28.462780+0000","last_clean":"2023-06-03T19:57:28.462780+0000","last_became_active":"2023-06-03T19:57:28.462298+0000","last_became_peered":"2023-06-03T19:57:28.462298+0000","last_unstale":"2023-06-03T19:57:28.462780+0000","last_undegraded":"2023-06-03T19:57:28.462780+0000","last_fullsized":"2023-06-03T19:57:28.462780+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":80,"reported_epoch":125,"state":"active+clean","last_fresh":"2023-06-03T19:57:28.451126+0000","last_change":"2023-06-03T19:56:54.510289+0000","last_active":"2023-06-03T19:57:28.451126+0000","last_peered":"2023-06-03T19:57:28.451126+0000","last_clean":"2023-06-03T19:57:28.451126+0000","last_became_active":"2023-06-03T19:56:54.509905+0000","last_became_peered":"2023-06-03T19:56:54.509905+0000","last_unstale":"2023-06-03T19:57:28.451126+0000","last_undegraded":"2023-06-03T19:57:28.451126+0000","last_fullsized":"2023-06-03T19:57:28.451126+0000","mapping_epoch":119,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T02:54:26.792401+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"60'8","reported_seq":100,"reported_epoch":124,"state":"active+clean","last_fresh":"2023-06-03T19:57:27.435852+0000","last_change":"2023-06-03T19:47:06.727778+0000","last_active":"2023-06-03T19:57:27.435852+0000","last_peered":"2023-06-03T19:57:27.435852+0000","last_clean":"2023-06-03T19:57:27.435852+0000","last_became_active":"2023-06-03T19:47:06.727305+0000","last_became_peered":"2023-06-03T19:47:06.727305+0000","last_unstale":"2023-06-03T19:57:27.435852+0000","last_undegraded":"2023-06-03T19:57:27.435852+0000","last_fullsized":"2023-06-03T19:57:27.435852+0000","mapping_epoch":86,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":87,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":89,"reported_epoch":125,"state":"active+clean","last_fresh":"2023-06-03T19:57:28.457328+0000","last_change":"2023-06-03T19:56:54.512994+0000","last_active":"2023-06-03T19:57:28.457328+0000","last_peered":"2023-06-03T19:57:28.457328+0000","last_clean":"2023-06-03T19:57:28.457328+0000","last_became_active":"2023-06-03T19:56:54.512579+0000","last_became_peered":"2023-06-03T19:56:54.512579+0000","last_unstale":"2023-06-03T19:57:28.457328+0000","last_undegraded":"2023-06-03T19:57:28.457328+0000","last_fullsized":"2023-06-03T19:57:28.457328+0000","mapping_epoch":119,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T07:02:45.187961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.f","version":"60'1","reported_seq":108,"reported_epoch":125,"state":"active+clean","last_fresh":"2023-06-03T19:57:28.464482+0000","last_change":"2023-06-03T19:57:28.464482+0000","last_active":"2023-06-03T19:57:28.464482+0000","last_peered":"2023-06-03T19:57:28.464482+0000","last_clean":"2023-06-03T19:57:28.464482+0000","last_became_active":"2023-06-03T19:57:28.464081+0000","last_became_peered":"2023-06-03T19:57:28.464081+0000","last_unstale":"2023-06-03T19:57:28.464482+0000","last_undegraded":"2023-06-03T19:57:28.464482+0000","last_fullsized":"2023-06-03T19:57:28.464482+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":76,"reported_epoch":124,"state":"active+clean","last_fresh":"2023-06-03T19:57:27.441050+0000","last_change":"2023-06-03T19:34:17.848358+0000","last_active":"2023-06-03T19:57:27.441050+0000","last_peered":"2023-06-03T19:57:27.441050+0000","last_clean":"2023-06-03T19:57:27.441050+0000","last_became_active":"2023-06-03T19:34:17.847811+0000","last_became_peered":"2023-06-03T19:34:17.847811+0000","last_unstale":"2023-06-03T19:57:27.441050+0000","last_undegraded":"2023-06-03T19:57:27.441050+0000","last_fullsized":"2023-06-03T19:57:27.441050+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.e","version":"60'2","reported_seq":109,"reported_epoch":125,"state":"active+clean","last_fresh":"2023-06-03T19:57:28.468330+0000","last_change":"2023-06-03T19:57:28.468330+0000","last_active":"2023-06-03T19:57:28.468330+0000","last_peered":"2023-06-03T19:57:28.468330+0000","last_clean":"2023-06-03T19:57:28.468330+0000","last_became_active":"2023-06-03T19:57:28.467823+0000","last_became_peered":"2023-06-03T19:57:28.467823+0000","last_unstale":"2023-06-03T19:57:28.468330+0000","last_undegraded":"2023-06-03T19:57:28.468330+0000","last_fullsized":"2023-06-03T19:57:28.468330+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":121,"reported_epoch":125,"state":"active+clean","last_fresh":"2023-06-03T19:57:28.461449+0000","last_change":"2023-06-03T19:57:28.461449+0000","last_active":"2023-06-03T19:57:28.461449+0000","last_peered":"2023-06-03T19:57:28.461449+0000","last_clean":"2023-06-03T19:57:28.461449+0000","last_became_active":"2023-06-03T19:57:28.460745+0000","last_became_peered":"2023-06-03T19:57:28.460745+0000","last_unstale":"2023-06-03T19:57:28.461449+0000","last_undegraded":"2023-06-03T19:57:28.461449+0000","last_fullsized":"2023-06-03T19:57:28.461449+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T20:41:19.361638+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,3],"acting":[4,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.d","version":"60'1","reported_seq":78,"reported_epoch":124,"state":"active+clean","last_fresh":"2023-06-03T19:57:27.445291+0000","last_change":"2023-06-03T19:34:16.806559+0000","last_active":"2023-06-03T19:57:27.445291+0000","last_peered":"2023-06-03T19:57:27.445291+0000","last_clean":"2023-06-03T19:57:27.445291+0000","last_became_active":"2023-06-03T19:34:16.806149+0000","last_became_peered":"2023-06-03T19:34:16.806149+0000","last_unstale":"2023-06-03T19:57:27.445291+0000","last_undegraded":"2023-06-03T19:57:27.445291+0000","last_fullsized":"2023-06-03T19:57:27.445291+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":91,"reported_epoch":124,"state":"active+clean","last_fresh":"2023-06-03T19:57:27.437525+0000","last_change":"2023-06-03T19:51:59.360650+0000","last_active":"2023-06-03T19:57:27.437525+0000","last_peered":"2023-06-03T19:57:27.437525+0000","last_clean":"2023-06-03T19:57:27.437525+0000","last_became_active":"2023-06-03T19:51:59.360237+0000","last_became_peered":"2023-06-03T19:51:59.360237+0000","last_unstale":"2023-06-03T19:57:27.437525+0000","last_undegraded":"2023-06-03T19:57:27.437525+0000","last_fullsized":"2023-06-03T19:57:27.437525+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":92,"reported_epoch":124,"state":"active+clean","last_fresh":"2023-06-03T19:57:27.440288+0000","last_change":"2023-06-03T19:56:54.507432+0000","last_active":"2023-06-03T19:57:27.440288+0000","last_peered":"2023-06-03T19:57:27.440288+0000","last_clean":"2023-06-03T19:57:27.440288+0000","last_became_active":"2023-06-03T19:56:54.507012+0000","last_became_peered":"2023-06-03T19:56:54.507012+0000","last_unstale":"2023-06-03T19:57:27.440288+0000","last_undegraded":"2023-06-03T19:57:27.440288+0000","last_fullsized":"2023-06-03T19:57:27.440288+0000","mapping_epoch":119,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":107,"reported_epoch":125,"state":"active+clean","last_fresh":"2023-06-03T19:57:28.465221+0000","last_change":"2023-06-03T19:57:28.464153+0000","last_active":"2023-06-03T19:57:28.465221+0000","last_peered":"2023-06-03T19:57:28.465221+0000","last_clean":"2023-06-03T19:57:28.465221+0000","last_became_active":"2023-06-03T19:57:28.463502+0000","last_became_peered":"2023-06-03T19:57:28.463502+0000","last_unstale":"2023-06-03T19:57:28.465221+0000","last_undegraded":"2023-06-03T19:57:28.465221+0000","last_fullsized":"2023-06-03T19:57:28.465221+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T06:11:00.999526+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,4],"acting":[0,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.b","version":"60'2","reported_seq":90,"reported_epoch":125,"state":"active+clean","last_fresh":"2023-06-03T19:57:28.471501+0000","last_change":"2023-06-03T19:57:28.470310+0000","last_active":"2023-06-03T19:57:28.471501+0000","last_peered":"2023-06-03T19:57:28.471501+0000","last_clean":"2023-06-03T19:57:28.471501+0000","last_became_active":"2023-06-03T19:57:28.469887+0000","last_became_peered":"2023-06-03T19:57:28.469887+0000","last_unstale":"2023-06-03T19:57:28.471501+0000","last_undegraded":"2023-06-03T19:57:28.471501+0000","last_fullsized":"2023-06-03T19:57:28.471501+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T21:58:53.387131+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":97,"reported_epoch":125,"state":"active+clean","last_fresh":"2023-06-03T19:57:28.465140+0000","last_change":"2023-06-03T19:57:28.465140+0000","last_active":"2023-06-03T19:57:28.465140+0000","last_peered":"2023-06-03T19:57:28.465140+0000","last_clean":"2023-06-03T19:57:28.465140+0000","last_became_active":"2023-06-03T19:57:28.464629+0000","last_became_peered":"2023-06-03T19:57:28.464629+0000","last_unstale":"2023-06-03T19:57:28.465140+0000","last_undegraded":"2023-06-03T19:57:28.465140+0000","last_fullsized":"2023-06-03T19:57:28.465140+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T21:47:11.619583+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":107,"reported_epoch":125,"state":"active+clean","last_fresh":"2023-06-03T19:57:28.461401+0000","last_change":"2023-06-03T19:57:28.461401+0000","last_active":"2023-06-03T19:57:28.461401+0000","last_peered":"2023-06-03T19:57:28.461401+0000","last_clean":"2023-06-03T19:57:28.461401+0000","last_became_active":"2023-06-03T19:57:28.461057+0000","last_became_peered":"2023-06-03T19:57:28.461057+0000","last_unstale":"2023-06-03T19:57:28.461401+0000","last_undegraded":"2023-06-03T19:57:28.461401+0000","last_fullsized":"2023-06-03T19:57:28.461401+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":102,"reported_epoch":125,"state":"active+clean","last_fresh":"2023-06-03T19:57:28.456750+0000","last_change":"2023-06-03T19:57:28.456750+0000","last_active":"2023-06-03T19:57:28.456750+0000","last_peered":"2023-06-03T19:57:28.456750+0000","last_clean":"2023-06-03T19:57:28.456750+0000","last_became_active":"2023-06-03T19:57:28.456436+0000","last_became_peered":"2023-06-03T19:57:28.456436+0000","last_unstale":"2023-06-03T19:57:28.456750+0000","last_undegraded":"2023-06-03T19:57:28.456750+0000","last_fullsized":"2023-06-03T19:57:28.456750+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T01:49:25.481554+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,5],"acting":[4,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":94,"reported_epoch":125,"state":"active+clean","last_fresh":"2023-06-03T19:57:28.462344+0000","last_change":"2023-06-03T19:57:28.462344+0000","last_active":"2023-06-03T19:57:28.462344+0000","last_peered":"2023-06-03T19:57:28.462344+0000","last_clean":"2023-06-03T19:57:28.462344+0000","last_became_active":"2023-06-03T19:57:28.462001+0000","last_became_peered":"2023-06-03T19:57:28.462001+0000","last_unstale":"2023-06-03T19:57:28.462344+0000","last_undegraded":"2023-06-03T19:57:28.462344+0000","last_fullsized":"2023-06-03T19:57:28.462344+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T00:45:13.789650+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":83,"reported_epoch":125,"state":"active+clean","last_fresh":"2023-06-03T19:57:28.467917+0000","last_change":"2023-06-03T19:57:28.467917+0000","last_active":"2023-06-03T19:57:28.467917+0000","last_peered":"2023-06-03T19:57:28.467917+0000","last_clean":"2023-06-03T19:57:28.467917+0000","last_became_active":"2023-06-03T19:57:28.467617+0000","last_became_peered":"2023-06-03T19:57:28.467617+0000","last_unstale":"2023-06-03T19:57:28.467917+0000","last_undegraded":"2023-06-03T19:57:28.467917+0000","last_fullsized":"2023-06-03T19:57:28.467917+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T23:34:23.840582+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":90,"reported_epoch":125,"state":"active+clean","last_fresh":"2023-06-03T19:57:28.453087+0000","last_change":"2023-06-03T19:56:54.512219+0000","last_active":"2023-06-03T19:57:28.453087+0000","last_peered":"2023-06-03T19:57:28.453087+0000","last_clean":"2023-06-03T19:57:28.453087+0000","last_became_active":"2023-06-03T19:56:54.511771+0000","last_became_peered":"2023-06-03T19:56:54.511771+0000","last_unstale":"2023-06-03T19:57:28.453087+0000","last_undegraded":"2023-06-03T19:57:28.453087+0000","last_fullsized":"2023-06-03T19:57:28.453087+0000","mapping_epoch":119,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T19:35:47.796520+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":76,"reported_epoch":125,"state":"active+clean","last_fresh":"2023-06-03T19:57:28.472005+0000","last_change":"2023-06-03T19:57:28.471023+0000","last_active":"2023-06-03T19:57:28.472005+0000","last_peered":"2023-06-03T19:57:28.472005+0000","last_clean":"2023-06-03T19:57:28.472005+0000","last_became_active":"2023-06-03T19:57:28.470452+0000","last_became_peered":"2023-06-03T19:57:28.470452+0000","last_unstale":"2023-06-03T19:57:28.472005+0000","last_undegraded":"2023-06-03T19:57:28.472005+0000","last_fullsized":"2023-06-03T19:57:28.472005+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T05:58:33.305030+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":94,"reported_epoch":125,"state":"active+clean","last_fresh":"2023-06-03T19:57:28.462051+0000","last_change":"2023-06-03T19:51:59.392796+0000","last_active":"2023-06-03T19:57:28.462051+0000","last_peered":"2023-06-03T19:57:28.462051+0000","last_clean":"2023-06-03T19:57:28.462051+0000","last_became_active":"2023-06-03T19:51:59.380207+0000","last_became_peered":"2023-06-03T19:51:59.380207+0000","last_unstale":"2023-06-03T19:57:28.462051+0000","last_undegraded":"2023-06-03T19:57:28.462051+0000","last_fullsized":"2023-06-03T19:57:28.462051+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:32:20.692043+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:32:20.692043+0000","last_clean_scrub_stamp":"2023-06-03T19:32:20.692043+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T00:28:05.866873+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":107,"reported_epoch":124,"state":"active+clean","last_fresh":"2023-06-03T19:57:27.442422+0000","last_change":"2023-06-03T19:56:54.512607+0000","last_active":"2023-06-03T19:57:27.442422+0000","last_peered":"2023-06-03T19:57:27.442422+0000","last_clean":"2023-06-03T19:57:27.442422+0000","last_became_active":"2023-06-03T19:56:54.512282+0000","last_became_peered":"2023-06-03T19:56:54.512282+0000","last_unstale":"2023-06-03T19:57:27.442422+0000","last_undegraded":"2023-06-03T19:57:27.442422+0000","last_fullsized":"2023-06-03T19:57:27.442422+0000","mapping_epoch":119,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":61,"reported_epoch":125,"state":"active+clean","last_fresh":"2023-06-03T19:57:28.471196+0000","last_change":"2023-06-03T19:57:28.470295+0000","last_active":"2023-06-03T19:57:28.471196+0000","last_peered":"2023-06-03T19:57:28.471196+0000","last_clean":"2023-06-03T19:57:28.471196+0000","last_became_active":"2023-06-03T19:57:28.469383+0000","last_became_peered":"2023-06-03T19:57:28.469383+0000","last_unstale":"2023-06-03T19:57:28.471196+0000","last_undegraded":"2023-06-03T19:57:28.471196+0000","last_fullsized":"2023-06-03T19:57:28.471196+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T05:35:19.667447+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":96,"reported_epoch":124,"state":"active+clean","last_fresh":"2023-06-03T19:57:27.441028+0000","last_change":"2023-06-03T19:51:59.373777+0000","last_active":"2023-06-03T19:57:27.441028+0000","last_peered":"2023-06-03T19:57:27.441028+0000","last_clean":"2023-06-03T19:57:27.441028+0000","last_became_active":"2023-06-03T19:51:59.372988+0000","last_became_peered":"2023-06-03T19:51:59.372988+0000","last_unstale":"2023-06-03T19:57:27.441028+0000","last_undegraded":"2023-06-03T19:57:27.441028+0000","last_fullsized":"2023-06-03T19:57:27.441028+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:32:20.692043+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:32:20.692043+0000","last_clean_scrub_stamp":"2023-06-03T19:32:20.692043+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":67,"reported_epoch":125,"state":"active+clean","last_fresh":"2023-06-03T19:57:28.460195+0000","last_change":"2023-06-03T19:57:28.460195+0000","last_active":"2023-06-03T19:57:28.460195+0000","last_peered":"2023-06-03T19:57:28.460195+0000","last_clean":"2023-06-03T19:57:28.460195+0000","last_became_active":"2023-06-03T19:57:28.459823+0000","last_became_peered":"2023-06-03T19:57:28.459823+0000","last_unstale":"2023-06-03T19:57:28.460195+0000","last_undegraded":"2023-06-03T19:57:28.460195+0000","last_fullsized":"2023-06-03T19:57:28.460195+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T20:08:48.686380+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":76,"reported_epoch":125,"state":"active+clean","last_fresh":"2023-06-03T19:57:28.462134+0000","last_change":"2023-06-03T19:56:54.516859+0000","last_active":"2023-06-03T19:57:28.462134+0000","last_peered":"2023-06-03T19:57:28.462134+0000","last_clean":"2023-06-03T19:57:28.462134+0000","last_became_active":"2023-06-03T19:56:54.516521+0000","last_became_peered":"2023-06-03T19:56:54.516521+0000","last_unstale":"2023-06-03T19:57:28.462134+0000","last_undegraded":"2023-06-03T19:57:28.462134+0000","last_fullsized":"2023-06-03T19:57:28.462134+0000","mapping_epoch":119,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T20:03:25.227382+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":97,"reported_epoch":125,"state":"active+clean","last_fresh":"2023-06-03T19:57:28.449619+0000","last_change":"2023-06-03T19:56:54.504859+0000","last_active":"2023-06-03T19:57:28.449619+0000","last_peered":"2023-06-03T19:57:28.449619+0000","last_clean":"2023-06-03T19:57:28.449619+0000","last_became_active":"2023-06-03T19:56:54.504455+0000","last_became_peered":"2023-06-03T19:56:54.504455+0000","last_unstale":"2023-06-03T19:57:28.449619+0000","last_undegraded":"2023-06-03T19:57:28.449619+0000","last_fullsized":"2023-06-03T19:57:28.449619+0000","mapping_epoch":119,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:32:20.692043+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:32:20.692043+0000","last_clean_scrub_stamp":"2023-06-03T19:32:20.692043+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T06:25:45.522100+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":102,"reported_epoch":125,"state":"active+clean","last_fresh":"2023-06-03T19:57:28.458987+0000","last_change":"2023-06-03T19:56:11.754048+0000","last_active":"2023-06-03T19:57:28.458987+0000","last_peered":"2023-06-03T19:57:28.458987+0000","last_clean":"2023-06-03T19:57:28.458987+0000","last_became_active":"2023-06-03T19:56:11.753687+0000","last_became_peered":"2023-06-03T19:56:11.753687+0000","last_unstale":"2023-06-03T19:57:28.458987+0000","last_undegraded":"2023-06-03T19:57:28.458987+0000","last_fullsized":"2023-06-03T19:57:28.458987+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":115,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T23:29:41.511496+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":89,"reported_epoch":125,"state":"active+clean","last_fresh":"2023-06-03T19:57:28.459616+0000","last_change":"2023-06-03T19:51:59.396573+0000","last_active":"2023-06-03T19:57:28.459616+0000","last_peered":"2023-06-03T19:57:28.459616+0000","last_clean":"2023-06-03T19:57:28.459616+0000","last_became_active":"2023-06-03T19:51:59.381593+0000","last_became_peered":"2023-06-03T19:51:59.381593+0000","last_unstale":"2023-06-03T19:57:28.459616+0000","last_undegraded":"2023-06-03T19:57:28.459616+0000","last_fullsized":"2023-06-03T19:57:28.459616+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T21:33:23.043772+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":96,"reported_epoch":124,"state":"active+clean","last_fresh":"2023-06-03T19:57:27.439804+0000","last_change":"2023-06-03T19:56:11.752151+0000","last_active":"2023-06-03T19:57:27.439804+0000","last_peered":"2023-06-03T19:57:27.439804+0000","last_clean":"2023-06-03T19:57:27.439804+0000","last_became_active":"2023-06-03T19:56:11.751767+0000","last_became_peered":"2023-06-03T19:56:11.751767+0000","last_unstale":"2023-06-03T19:57:27.439804+0000","last_undegraded":"2023-06-03T19:57:27.439804+0000","last_fullsized":"2023-06-03T19:57:27.439804+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":115,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:32:20.692043+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:32:20.692043+0000","last_clean_scrub_stamp":"2023-06-03T19:32:20.692043+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"72'139","reported_seq":328,"reported_epoch":124,"state":"active+clean","last_fresh":"2023-06-03T19:57:27.443270+0000","last_change":"2023-06-03T19:47:06.731872+0000","last_active":"2023-06-03T19:57:27.443270+0000","last_peered":"2023-06-03T19:57:27.443270+0000","last_clean":"2023-06-03T19:57:27.443270+0000","last_became_active":"2023-06-03T19:47:06.731175+0000","last_became_peered":"2023-06-03T19:47:06.731175+0000","last_unstale":"2023-06-03T19:57:27.443270+0000","last_undegraded":"2023-06-03T19:57:27.443270+0000","last_fullsized":"2023-06-03T19:57:27.443270+0000","mapping_epoch":86,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":87,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:27:28.100808+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:27:28.100808+0000","last_clean_scrub_stamp":"2023-06-03T19:27:28.100808+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":70,"reported_epoch":125,"state":"active+clean","last_fresh":"2023-06-03T19:57:28.457052+0000","last_change":"2023-06-03T19:56:54.509571+0000","last_active":"2023-06-03T19:57:28.457052+0000","last_peered":"2023-06-03T19:57:28.457052+0000","last_clean":"2023-06-03T19:57:28.457052+0000","last_became_active":"2023-06-03T19:56:54.509213+0000","last_became_peered":"2023-06-03T19:56:54.509213+0000","last_unstale":"2023-06-03T19:57:28.457052+0000","last_undegraded":"2023-06-03T19:57:28.457052+0000","last_fullsized":"2023-06-03T19:57:28.457052+0000","mapping_epoch":119,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T21:26:56.003203+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":91,"reported_epoch":125,"state":"active+clean","last_fresh":"2023-06-03T19:57:28.459664+0000","last_change":"2023-06-03T19:57:28.459664+0000","last_active":"2023-06-03T19:57:28.459664+0000","last_peered":"2023-06-03T19:57:28.459664+0000","last_clean":"2023-06-03T19:57:28.459664+0000","last_became_active":"2023-06-03T19:57:28.459289+0000","last_became_peered":"2023-06-03T19:57:28.459289+0000","last_unstale":"2023-06-03T19:57:28.459664+0000","last_undegraded":"2023-06-03T19:57:28.459664+0000","last_fullsized":"2023-06-03T19:57:28.459664+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":111,"reported_epoch":124,"state":"active+clean","last_fresh":"2023-06-03T19:57:27.443526+0000","last_change":"2023-06-03T19:51:59.358157+0000","last_active":"2023-06-03T19:57:27.443526+0000","last_peered":"2023-06-03T19:57:27.443526+0000","last_clean":"2023-06-03T19:57:27.443526+0000","last_became_active":"2023-06-03T19:51:59.357363+0000","last_became_peered":"2023-06-03T19:51:59.357363+0000","last_unstale":"2023-06-03T19:57:27.443526+0000","last_undegraded":"2023-06-03T19:57:27.443526+0000","last_fullsized":"2023-06-03T19:57:27.443526+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:32:20.692043+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:32:20.692043+0000","last_clean_scrub_stamp":"2023-06-03T19:32:20.692043+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":94,"reported_epoch":125,"state":"active+clean","last_fresh":"2023-06-03T19:57:28.459955+0000","last_change":"2023-06-03T19:57:28.459611+0000","last_active":"2023-06-03T19:57:28.459955+0000","last_peered":"2023-06-03T19:57:28.459955+0000","last_clean":"2023-06-03T19:57:28.459955+0000","last_became_active":"2023-06-03T19:57:28.459264+0000","last_became_peered":"2023-06-03T19:57:28.459264+0000","last_unstale":"2023-06-03T19:57:28.459955+0000","last_undegraded":"2023-06-03T19:57:28.459955+0000","last_fullsized":"2023-06-03T19:57:28.459955+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T22:15:06.557231+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":95,"reported_epoch":125,"state":"active+clean","last_fresh":"2023-06-03T19:57:28.451064+0000","last_change":"2023-06-03T19:56:11.745678+0000","last_active":"2023-06-03T19:57:28.451064+0000","last_peered":"2023-06-03T19:57:28.451064+0000","last_clean":"2023-06-03T19:57:28.451064+0000","last_became_active":"2023-06-03T19:56:11.744966+0000","last_became_peered":"2023-06-03T19:56:11.744966+0000","last_unstale":"2023-06-03T19:57:28.451064+0000","last_undegraded":"2023-06-03T19:57:28.451064+0000","last_fullsized":"2023-06-03T19:57:28.451064+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":115,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T21:12:06.830295+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":111,"reported_epoch":125,"state":"active+clean","last_fresh":"2023-06-03T19:57:28.466096+0000","last_change":"2023-06-03T19:57:28.466096+0000","last_active":"2023-06-03T19:57:28.466096+0000","last_peered":"2023-06-03T19:57:28.466096+0000","last_clean":"2023-06-03T19:57:28.466096+0000","last_became_active":"2023-06-03T19:57:28.464954+0000","last_became_peered":"2023-06-03T19:57:28.464954+0000","last_unstale":"2023-06-03T19:57:28.466096+0000","last_undegraded":"2023-06-03T19:57:28.466096+0000","last_fullsized":"2023-06-03T19:57:28.466096+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:32:20.692043+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:32:20.692043+0000","last_clean_scrub_stamp":"2023-06-03T19:32:20.692043+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":102,"reported_epoch":125,"state":"active+clean","last_fresh":"2023-06-03T19:57:28.462221+0000","last_change":"2023-06-03T19:56:11.750241+0000","last_active":"2023-06-03T19:57:28.462221+0000","last_peered":"2023-06-03T19:57:28.462221+0000","last_clean":"2023-06-03T19:57:28.462221+0000","last_became_active":"2023-06-03T19:56:11.749687+0000","last_became_peered":"2023-06-03T19:56:11.749687+0000","last_unstale":"2023-06-03T19:57:28.462221+0000","last_undegraded":"2023-06-03T19:57:28.462221+0000","last_fullsized":"2023-06-03T19:57:28.462221+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":115,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T22:38:25.819929+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":65,"reported_epoch":125,"state":"active+clean","last_fresh":"2023-06-03T19:57:28.458317+0000","last_change":"2023-06-03T19:47:06.733532+0000","last_active":"2023-06-03T19:57:28.458317+0000","last_peered":"2023-06-03T19:57:28.458317+0000","last_clean":"2023-06-03T19:57:28.458317+0000","last_became_active":"2023-06-03T19:47:06.733131+0000","last_became_peered":"2023-06-03T19:47:06.733131+0000","last_unstale":"2023-06-03T19:57:28.458317+0000","last_undegraded":"2023-06-03T19:57:28.458317+0000","last_fullsized":"2023-06-03T19:57:28.458317+0000","mapping_epoch":86,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":87,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T23:46:03.788646+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.7","version":"60'2","reported_seq":75,"reported_epoch":125,"state":"active+clean","last_fresh":"2023-06-03T19:57:28.457418+0000","last_change":"2023-06-03T19:56:54.510148+0000","last_active":"2023-06-03T19:57:28.457418+0000","last_peered":"2023-06-03T19:57:28.457418+0000","last_clean":"2023-06-03T19:57:28.457418+0000","last_became_active":"2023-06-03T19:56:54.509771+0000","last_became_peered":"2023-06-03T19:56:54.509771+0000","last_unstale":"2023-06-03T19:57:28.457418+0000","last_undegraded":"2023-06-03T19:57:28.457418+0000","last_fullsized":"2023-06-03T19:57:28.457418+0000","mapping_epoch":119,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T00:52:42.807432+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":102,"reported_epoch":125,"state":"active+clean","last_fresh":"2023-06-03T19:57:28.472011+0000","last_change":"2023-06-03T19:57:28.470954+0000","last_active":"2023-06-03T19:57:28.472011+0000","last_peered":"2023-06-03T19:57:28.472011+0000","last_clean":"2023-06-03T19:57:28.472011+0000","last_became_active":"2023-06-03T19:57:28.470524+0000","last_became_peered":"2023-06-03T19:57:28.470524+0000","last_unstale":"2023-06-03T19:57:28.472011+0000","last_undegraded":"2023-06-03T19:57:28.472011+0000","last_fullsized":"2023-06-03T19:57:28.472011+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:32:20.692043+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:32:20.692043+0000","last_clean_scrub_stamp":"2023-06-03T19:32:20.692043+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T19:39:19.983271+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":72,"reported_epoch":125,"state":"active+clean","last_fresh":"2023-06-03T19:57:28.452994+0000","last_change":"2023-06-03T19:56:54.507670+0000","last_active":"2023-06-03T19:57:28.452994+0000","last_peered":"2023-06-03T19:57:28.452994+0000","last_clean":"2023-06-03T19:57:28.452994+0000","last_became_active":"2023-06-03T19:56:54.507293+0000","last_became_peered":"2023-06-03T19:56:54.507293+0000","last_unstale":"2023-06-03T19:57:28.452994+0000","last_undegraded":"2023-06-03T19:57:28.452994+0000","last_fullsized":"2023-06-03T19:57:28.452994+0000","mapping_epoch":119,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T01:49:19.966054+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":96,"reported_epoch":124,"state":"active+clean","last_fresh":"2023-06-03T19:57:27.437244+0000","last_change":"2023-06-03T19:56:11.752730+0000","last_active":"2023-06-03T19:57:27.437244+0000","last_peered":"2023-06-03T19:57:27.437244+0000","last_clean":"2023-06-03T19:57:27.437244+0000","last_became_active":"2023-06-03T19:56:11.752441+0000","last_became_peered":"2023-06-03T19:56:11.752441+0000","last_unstale":"2023-06-03T19:57:27.437244+0000","last_undegraded":"2023-06-03T19:57:27.437244+0000","last_fullsized":"2023-06-03T19:57:27.437244+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":115,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:32:20.692043+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:32:20.692043+0000","last_clean_scrub_stamp":"2023-06-03T19:32:20.692043+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"60'1","reported_seq":98,"reported_epoch":125,"state":"active+clean","last_fresh":"2023-06-03T19:57:28.464648+0000","last_change":"2023-06-03T19:57:28.463999+0000","last_active":"2023-06-03T19:57:28.464648+0000","last_peered":"2023-06-03T19:57:28.464648+0000","last_clean":"2023-06-03T19:57:28.464648+0000","last_became_active":"2023-06-03T19:57:28.463588+0000","last_became_peered":"2023-06-03T19:57:28.463588+0000","last_unstale":"2023-06-03T19:57:28.464648+0000","last_undegraded":"2023-06-03T19:57:28.464648+0000","last_fullsized":"2023-06-03T19:57:28.464648+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T21:44:11.969059+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":65,"reported_epoch":125,"state":"active+clean","last_fresh":"2023-06-03T19:57:28.456467+0000","last_change":"2023-06-03T19:57:28.456467+0000","last_active":"2023-06-03T19:57:28.456467+0000","last_peered":"2023-06-03T19:57:28.456467+0000","last_clean":"2023-06-03T19:57:28.456467+0000","last_became_active":"2023-06-03T19:57:28.456120+0000","last_became_peered":"2023-06-03T19:57:28.456120+0000","last_unstale":"2023-06-03T19:57:28.456467+0000","last_undegraded":"2023-06-03T19:57:28.456467+0000","last_fullsized":"2023-06-03T19:57:28.456467+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T07:28:14.306624+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"60'1","reported_seq":82,"reported_epoch":125,"state":"active+clean","last_fresh":"2023-06-03T19:57:28.465124+0000","last_change":"2023-06-03T19:57:28.465124+0000","last_active":"2023-06-03T19:57:28.465124+0000","last_peered":"2023-06-03T19:57:28.465124+0000","last_clean":"2023-06-03T19:57:28.465124+0000","last_became_active":"2023-06-03T19:57:28.464654+0000","last_became_peered":"2023-06-03T19:57:28.464654+0000","last_unstale":"2023-06-03T19:57:28.465124+0000","last_undegraded":"2023-06-03T19:57:28.465124+0000","last_fullsized":"2023-06-03T19:57:28.465124+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T01:12:32.305507+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":109,"reported_epoch":125,"state":"active+clean","last_fresh":"2023-06-03T19:57:28.448170+0000","last_change":"2023-06-03T19:56:54.511760+0000","last_active":"2023-06-03T19:57:28.448170+0000","last_peered":"2023-06-03T19:57:28.448170+0000","last_clean":"2023-06-03T19:57:28.448170+0000","last_became_active":"2023-06-03T19:56:54.511429+0000","last_became_peered":"2023-06-03T19:56:54.511429+0000","last_unstale":"2023-06-03T19:57:28.448170+0000","last_undegraded":"2023-06-03T19:57:28.448170+0000","last_fullsized":"2023-06-03T19:57:28.448170+0000","mapping_epoch":119,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T05:42:46.254343+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"60'2","reported_seq":109,"reported_epoch":125,"state":"active+clean","last_fresh":"2023-06-03T19:57:28.462252+0000","last_change":"2023-06-03T19:57:28.462252+0000","last_active":"2023-06-03T19:57:28.462252+0000","last_peered":"2023-06-03T19:57:28.462252+0000","last_clean":"2023-06-03T19:57:28.462252+0000","last_became_active":"2023-06-03T19:57:28.461844+0000","last_became_peered":"2023-06-03T19:57:28.461844+0000","last_unstale":"2023-06-03T19:57:28.462252+0000","last_undegraded":"2023-06-03T19:57:28.462252+0000","last_fullsized":"2023-06-03T19:57:28.462252+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":112,"reported_epoch":125,"state":"active+clean","last_fresh":"2023-06-03T19:57:28.469248+0000","last_change":"2023-06-03T19:57:28.469248+0000","last_active":"2023-06-03T19:57:28.469248+0000","last_peered":"2023-06-03T19:57:28.469248+0000","last_clean":"2023-06-03T19:57:28.469248+0000","last_became_active":"2023-06-03T19:57:28.468903+0000","last_became_peered":"2023-06-03T19:57:28.468903+0000","last_unstale":"2023-06-03T19:57:28.469248+0000","last_undegraded":"2023-06-03T19:57:28.469248+0000","last_fullsized":"2023-06-03T19:57:28.469248+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T06:08:20.956098+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"60'2","reported_seq":94,"reported_epoch":125,"state":"active+clean","last_fresh":"2023-06-03T19:57:28.462610+0000","last_change":"2023-06-03T19:57:28.462610+0000","last_active":"2023-06-03T19:57:28.462610+0000","last_peered":"2023-06-03T19:57:28.462610+0000","last_clean":"2023-06-03T19:57:28.462610+0000","last_became_active":"2023-06-03T19:57:28.460655+0000","last_became_peered":"2023-06-03T19:57:28.460655+0000","last_unstale":"2023-06-03T19:57:28.462610+0000","last_undegraded":"2023-06-03T19:57:28.462610+0000","last_fullsized":"2023-06-03T19:57:28.462610+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":112,"reported_epoch":125,"state":"active+clean","last_fresh":"2023-06-03T19:57:28.464690+0000","last_change":"2023-06-03T19:57:28.464690+0000","last_active":"2023-06-03T19:57:28.464690+0000","last_peered":"2023-06-03T19:57:28.464690+0000","last_clean":"2023-06-03T19:57:28.464690+0000","last_became_active":"2023-06-03T19:57:28.463856+0000","last_became_peered":"2023-06-03T19:57:28.463856+0000","last_unstale":"2023-06-03T19:57:28.464690+0000","last_undegraded":"2023-06-03T19:57:28.464690+0000","last_fullsized":"2023-06-03T19:57:28.464690+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T00:11:00.991088+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"60'1","reported_seq":101,"reported_epoch":125,"state":"active+clean","last_fresh":"2023-06-03T19:57:28.457091+0000","last_change":"2023-06-03T19:56:54.505096+0000","last_active":"2023-06-03T19:57:28.457091+0000","last_peered":"2023-06-03T19:57:28.457091+0000","last_clean":"2023-06-03T19:57:28.457091+0000","last_became_active":"2023-06-03T19:56:54.504255+0000","last_became_peered":"2023-06-03T19:56:54.504255+0000","last_unstale":"2023-06-03T19:57:28.457091+0000","last_undegraded":"2023-06-03T19:57:28.457091+0000","last_fullsized":"2023-06-03T19:57:28.457091+0000","mapping_epoch":119,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T19:52:25.922308+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":114,"reported_epoch":125,"state":"active+clean","last_fresh":"2023-06-03T19:57:28.457839+0000","last_change":"2023-06-03T19:56:54.503416+0000","last_active":"2023-06-03T19:57:28.457839+0000","last_peered":"2023-06-03T19:57:28.457839+0000","last_clean":"2023-06-03T19:57:28.457839+0000","last_became_active":"2023-06-03T19:56:54.503018+0000","last_became_peered":"2023-06-03T19:56:54.503018+0000","last_unstale":"2023-06-03T19:57:28.457839+0000","last_undegraded":"2023-06-03T19:57:28.457839+0000","last_fullsized":"2023-06-03T19:57:28.457839+0000","mapping_epoch":119,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T20:14:00.330616+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"60'4","reported_seq":83,"reported_epoch":125,"state":"active+clean","last_fresh":"2023-06-03T19:57:28.451241+0000","last_change":"2023-06-03T19:56:11.748116+0000","last_active":"2023-06-03T19:57:28.451241+0000","last_peered":"2023-06-03T19:57:28.451241+0000","last_clean":"2023-06-03T19:57:28.451241+0000","last_became_active":"2023-06-03T19:56:11.747636+0000","last_became_peered":"2023-06-03T19:56:11.747636+0000","last_unstale":"2023-06-03T19:57:28.451241+0000","last_undegraded":"2023-06-03T19:57:28.451241+0000","last_fullsized":"2023-06-03T19:57:28.451241+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":115,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T06:57:48.052055+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":83,"reported_epoch":125,"state":"active+clean","last_fresh":"2023-06-03T19:57:28.459234+0000","last_change":"2023-06-03T19:57:28.459234+0000","last_active":"2023-06-03T19:57:28.459234+0000","last_peered":"2023-06-03T19:57:28.459234+0000","last_clean":"2023-06-03T19:57:28.459234+0000","last_became_active":"2023-06-03T19:57:28.458896+0000","last_became_peered":"2023-06-03T19:57:28.458896+0000","last_unstale":"2023-06-03T19:57:28.459234+0000","last_undegraded":"2023-06-03T19:57:28.459234+0000","last_fullsized":"2023-06-03T19:57:28.459234+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T21:18:37.142280+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5486,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":62,"num_write_kb":30,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16458,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":30319,"internal_metadata":0},"log_size":56,"ondisk_log_size":56,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":3,"num_store_stats":3},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":48,"seq":206158430548,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":287928,"kb_used_data":1776,"kb_used_omap":0,"kb_used_meta":286144,"kb_avail":93457224,"statfs":{"total":95995035648,"available":95700197376,"internally_reserved":0,"allocated":1818624,"data_stored":1348675,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":293011456},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 3 19:58:34 2023","interfaces":[{"interface":"back","average":{"1min":0.86599999999999999,"5min":1.1080000000000001,"15min":1.2729999999999999},"min":{"1min":0.48199999999999998,"5min":0.27500000000000002,"15min":0.253},"max":{"1min":1.3180000000000001,"5min":4.806,"15min":8.6099999999999994},"last":1.1180000000000001},{"interface":"front","average":{"1min":0.91100000000000003,"5min":1.218,"15min":1.506},"min":{"1min":0.57399999999999995,"5min":0.33200000000000002,"15min":0.28499999999999998},"max":{"1min":1.2390000000000001,"5min":6.6139999999999999,"15min":15.220000000000001},"last":0.92600000000000005}]},{"osd":1,"last update":"Sat Jun 3 19:59:14 2023","interfaces":[{"interface":"back","average":{"1min":0.98999999999999999,"5min":1.21,"15min":1.7609999999999999},"min":{"1min":0.63900000000000001,"5min":0.315,"15min":0.315},"max":{"1min":1.319,"5min":8.1750000000000007,"15min":10.211},"last":1.169},{"interface":"front","average":{"1min":0.96999999999999997,"5min":1.1339999999999999,"15min":1.5720000000000001},"min":{"1min":0.68400000000000005,"5min":0.36199999999999999,"15min":0.34799999999999998},"max":{"1min":1.3620000000000001,"5min":8.4260000000000002,"15min":11.635},"last":1.0269999999999999}]},{"osd":2,"last update":"Sat Jun 3 19:59:14 2023","interfaces":[{"interface":"back","average":{"1min":0.98199999999999998,"5min":1.0249999999999999,"15min":1.0169999999999999},"min":{"1min":0.58299999999999996,"5min":0.58299999999999996,"15min":0.58299999999999996},"max":{"1min":1.8740000000000001,"5min":2.9969999999999999,"15min":2.9969999999999999},"last":0.90300000000000002},{"interface":"front","average":{"1min":1.01,"5min":1.006,"15min":0.98699999999999999},"min":{"1min":0.63500000000000001,"5min":0.63100000000000001,"15min":0.63100000000000001},"max":{"1min":1.3160000000000001,"5min":2.0299999999999998,"15min":2.0299999999999998},"last":0.97499999999999998}]},{"osd":3,"last update":"Sat Jun 3 19:59:00 2023","interfaces":[{"interface":"back","average":{"1min":0.97799999999999998,"5min":1.1200000000000001,"15min":1.1439999999999999},"min":{"1min":0.70599999999999996,"5min":0.65400000000000003,"15min":0.65400000000000003},"max":{"1min":1.371,"5min":2.9399999999999999,"15min":2.9399999999999999},"last":1.0580000000000001},{"interface":"front","average":{"1min":1.0580000000000001,"5min":1.171,"15min":1.1899999999999999},"min":{"1min":0.66500000000000004,"5min":0.66500000000000004,"15min":0.66500000000000004},"max":{"1min":1.391,"5min":3.085,"15min":3.085},"last":1.1819999999999999}]},{"osd":4,"last update":"Sat Jun 3 19:58:28 2023","interfaces":[{"interface":"back","average":{"1min":1.0509999999999999,"5min":1.0509999999999999,"15min":1.0509999999999999},"min":{"1min":0.66900000000000004,"5min":0.66900000000000004,"15min":0.66900000000000004},"max":{"1min":1.552,"5min":1.552,"15min":1.552},"last":0.79200000000000004},{"interface":"front","average":{"1min":1.119,"5min":1.119,"15min":1.119},"min":{"1min":0.77100000000000002,"5min":0.77100000000000002,"15min":0.77100000000000002},"max":{"1min":1.476,"5min":1.476,"15min":1.476},"last":0.81899999999999995}]},{"osd":5,"last update":"Sat Jun 3 19:59:09 2023","interfaces":[{"interface":"back","average":{"1min":1.054,"5min":1.1000000000000001,"15min":1.1879999999999999},"min":{"1min":0.80400000000000005,"5min":0.33200000000000002,"15min":0.30199999999999999},"max":{"1min":1.304,"5min":10.151999999999999,"15min":12.778},"last":1.0029999999999999},{"interface":"front","average":{"1min":1.157,"5min":1.089,"15min":1.3120000000000001},"min":{"1min":0.88100000000000001,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":1.417,"5min":5.3840000000000003,"15min":9.1669999999999998},"last":1.0840000000000001}]},{"osd":6,"last update":"Sat Jun 3 19:59:09 2023","interfaces":[{"interface":"back","average":{"1min":1.212,"5min":1.1719999999999999,"15min":1.325},"min":{"1min":1.0620000000000001,"5min":0.36499999999999999,"15min":0.35899999999999999},"max":{"1min":1.387,"5min":5.4429999999999996,"15min":9.3520000000000003},"last":1.137},{"interface":"front","average":{"1min":1.0509999999999999,"5min":1.169,"15min":1.1919999999999999},"min":{"1min":0.83699999999999997,"5min":0.41099999999999998,"15min":0.32900000000000001},"max":{"1min":1.274,"5min":9.0269999999999992,"15min":9.0269999999999992},"last":0.871}]}]},{"osd":6,"up_from":42,"seq":180388626781,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":270504,"kb_used_data":1760,"kb_used_omap":0,"kb_used_meta":268736,"kb_avail":93474648,"statfs":{"total":95995035648,"available":95718039552,"internally_reserved":0,"allocated":1802240,"data_stored":1348214,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":275185664},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 3 19:58:35 2023","interfaces":[{"interface":"back","average":{"1min":0.98499999999999999,"5min":1.5009999999999999,"15min":1.5629999999999999},"min":{"1min":0.70199999999999996,"5min":0.28199999999999997,"15min":0.27500000000000002},"max":{"1min":1.3320000000000001,"5min":8.1890000000000001,"15min":11.944000000000001},"last":1.0660000000000001},{"interface":"front","average":{"1min":0.94699999999999995,"5min":1.6739999999999999,"15min":1.659},"min":{"1min":0.56100000000000005,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":1.1259999999999999,"5min":10.835000000000001,"15min":12.502000000000001},"last":0.97699999999999998}]},{"osd":1,"last update":"Sat Jun 3 19:59:13 2023","interfaces":[{"interface":"back","average":{"1min":0.98199999999999998,"5min":1.3520000000000001,"15min":1.542},"min":{"1min":0.58399999999999996,"5min":0.35599999999999998,"15min":0.30499999999999999},"max":{"1min":1.4430000000000001,"5min":6.8049999999999997,"15min":6.8049999999999997},"last":1.2749999999999999},{"interface":"front","average":{"1min":0.93600000000000005,"5min":1.383,"15min":1.2130000000000001},"min":{"1min":0.69899999999999995,"5min":0.39100000000000001,"15min":0.38500000000000001},"max":{"1min":1.1759999999999999,"5min":6.7880000000000003,"15min":8.9120000000000008},"last":1.1040000000000001}]},{"osd":2,"last update":"Sat Jun 3 19:59:13 2023","interfaces":[{"interface":"back","average":{"1min":0.91300000000000003,"5min":0.86499999999999999,"15min":0.84799999999999998},"min":{"1min":0.65900000000000003,"5min":0.57399999999999995,"15min":0.57399999999999995},"max":{"1min":1.3120000000000001,"5min":1.4490000000000001,"15min":1.4490000000000001},"last":1.008},{"interface":"front","average":{"1min":0.96599999999999997,"5min":0.95199999999999996,"15min":0.94599999999999995},"min":{"1min":0.60899999999999999,"5min":0.52700000000000002,"15min":0.52700000000000002},"max":{"1min":1.3120000000000001,"5min":1.5469999999999999,"15min":1.5469999999999999},"last":0.81200000000000006}]},{"osd":3,"last update":"Sat Jun 3 19:59:00 2023","interfaces":[{"interface":"back","average":{"1min":0.89900000000000002,"5min":0.86299999999999999,"15min":0.85699999999999998},"min":{"1min":0.64500000000000002,"5min":0.623,"15min":0.623},"max":{"1min":1.173,"5min":1.1819999999999999,"15min":1.1819999999999999},"last":1.2450000000000001},{"interface":"front","average":{"1min":1.1100000000000001,"5min":1.0169999999999999,"15min":1.0009999999999999},"min":{"1min":0.76800000000000002,"5min":0.65300000000000002,"15min":0.65300000000000002},"max":{"1min":1.482,"5min":1.482,"15min":1.482},"last":1.413}]},{"osd":4,"last update":"Sat Jun 3 19:58:29 2023","interfaces":[{"interface":"back","average":{"1min":0.999,"5min":0.999,"15min":0.999},"min":{"1min":0.751,"5min":0.751,"15min":0.751},"max":{"1min":1.1719999999999999,"5min":1.1719999999999999,"15min":1.1719999999999999},"last":1.1499999999999999},{"interface":"front","average":{"1min":1.181,"5min":1.181,"15min":1.181},"min":{"1min":0.92000000000000004,"5min":0.92000000000000004,"15min":0.92000000000000004},"max":{"1min":1.46,"5min":1.46,"15min":1.46},"last":1.4650000000000001}]},{"osd":5,"last update":"Sat Jun 3 19:58:25 2023","interfaces":[{"interface":"back","average":{"1min":1.1679999999999999,"5min":1.385,"15min":1.2749999999999999},"min":{"1min":0.83299999999999996,"5min":0.34100000000000003,"15min":0.30499999999999999},"max":{"1min":1.393,"5min":11.083,"15min":11.083},"last":1.361},{"interface":"front","average":{"1min":1.089,"5min":1.4099999999999999,"15min":1.3220000000000001},"min":{"1min":0.80000000000000004,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":1.367,"5min":10.154,"15min":10.154},"last":1.212}]},{"osd":7,"last update":"Sat Jun 3 19:59:04 2023","interfaces":[{"interface":"back","average":{"1min":1.252,"5min":1.3280000000000001,"15min":1.3540000000000001},"min":{"1min":0.79100000000000004,"5min":0.313,"15min":0.313},"max":{"1min":1.536,"5min":6.3029999999999999,"15min":8.0419999999999998},"last":1.1870000000000001},{"interface":"front","average":{"1min":1.121,"5min":1.5289999999999999,"15min":1.4059999999999999},"min":{"1min":0.76900000000000002,"5min":0.35099999999999998,"15min":0.29999999999999999},"max":{"1min":1.4059999999999999,"5min":12.407999999999999,"15min":12.407999999999999},"last":1.331}]}]},{"osd":1,"up_from":103,"seq":442381631579,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":129700,"kb_used_data":1244,"kb_used_omap":4,"kb_used_meta":128443,"kb_avail":93615452,"statfs":{"total":95995035648,"available":95862222848,"internally_reserved":0,"allocated":1273856,"data_stored":767105,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":131526542},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 3 19:59:12 2023","interfaces":[{"interface":"back","average":{"1min":0.627,"5min":0.749,"15min":1.7589999999999999},"min":{"1min":0.38600000000000001,"5min":0.187,"15min":0.187},"max":{"1min":0.80000000000000004,"5min":4.1539999999999999,"15min":10.653},"last":1.077},{"interface":"front","average":{"1min":0.68400000000000005,"5min":1.1319999999999999,"15min":1.831},"min":{"1min":0.32500000000000001,"5min":0.192,"15min":0.192},"max":{"1min":0.90900000000000003,"5min":13.750999999999999,"15min":13.750999999999999},"last":0.879}]},{"osd":2,"last update":"Sat Jun 3 19:59:20 2023","interfaces":[{"interface":"back","average":{"1min":0.64900000000000002,"5min":0.70099999999999996,"15min":0.71099999999999997},"min":{"1min":0.36499999999999999,"5min":0.34599999999999997,"15min":0.34599999999999997},"max":{"1min":1.1539999999999999,"5min":1.1539999999999999,"15min":1.1539999999999999},"last":0.747},{"interface":"front","average":{"1min":0.65500000000000003,"5min":0.71799999999999997,"15min":0.73699999999999999},"min":{"1min":0.38700000000000001,"5min":0.36499999999999999,"15min":0.36499999999999999},"max":{"1min":0.88900000000000001,"5min":1.548,"15min":1.548},"last":0.70599999999999996}]},{"osd":3,"last update":"Sat Jun 3 19:58:59 2023","interfaces":[{"interface":"back","average":{"1min":0.66800000000000004,"5min":0.72399999999999998,"15min":0.73299999999999998},"min":{"1min":0.42999999999999999,"5min":0.36599999999999999,"15min":0.36599999999999999},"max":{"1min":1.2,"5min":1.2769999999999999,"15min":1.2769999999999999},"last":0.84299999999999997},{"interface":"front","average":{"1min":0.77800000000000002,"5min":0.84999999999999998,"15min":0.86199999999999999},"min":{"1min":0.40799999999999997,"5min":0.40799999999999997,"15min":0.40799999999999997},"max":{"1min":1.101,"5min":1.3740000000000001,"15min":1.3740000000000001},"last":1.2350000000000001}]},{"osd":4,"last update":"Sat Jun 3 19:58:28 2023","interfaces":[{"interface":"back","average":{"1min":0.99299999999999999,"5min":0.99299999999999999,"15min":0.99299999999999999},"min":{"1min":0.80400000000000005,"5min":0.80400000000000005,"15min":0.80400000000000005},"max":{"1min":1.466,"5min":1.466,"15min":1.466},"last":1.1160000000000001},{"interface":"front","average":{"1min":0.95699999999999996,"5min":0.95699999999999996,"15min":0.95699999999999996},"min":{"1min":0.66900000000000004,"5min":0.66900000000000004,"15min":0.66900000000000004},"max":{"1min":1.591,"5min":1.591,"15min":1.591},"last":0.78900000000000003}]},{"osd":5,"last update":"Sat Jun 3 19:59:12 2023","interfaces":[{"interface":"back","average":{"1min":0.97799999999999998,"5min":1.395,"15min":2.1360000000000001},"min":{"1min":0.73899999999999999,"5min":0.437,"15min":0.373},"max":{"1min":1.2789999999999999,"5min":7.9400000000000004,"15min":7.9400000000000004},"last":0.93300000000000005},{"interface":"front","average":{"1min":0.98299999999999998,"5min":1.6950000000000001,"15min":2.5609999999999999},"min":{"1min":0.73499999999999999,"5min":0.40000000000000002,"15min":0.40000000000000002},"max":{"1min":1.198,"5min":13.797000000000001,"15min":13.797000000000001},"last":1.034}]},{"osd":6,"last update":"Sat Jun 3 19:59:12 2023","interfaces":[{"interface":"back","average":{"1min":0.98799999999999999,"5min":1.657,"15min":2.242},"min":{"1min":0.78000000000000003,"5min":0.41399999999999998,"15min":0.38300000000000001},"max":{"1min":1.2310000000000001,"5min":13.941000000000001,"15min":13.941000000000001},"last":0.96399999999999997},{"interface":"front","average":{"1min":1.05,"5min":1.5389999999999999,"15min":2.706},"min":{"1min":0.72099999999999997,"5min":0.40100000000000002,"15min":0.38700000000000001},"max":{"1min":1.2989999999999999,"5min":9.4060000000000006,"15min":14.847},"last":1.3120000000000001}]},{"osd":7,"last update":"Sat Jun 3 19:59:12 2023","interfaces":[{"interface":"back","average":{"1min":1.083,"5min":1.415,"15min":2.3839999999999999},"min":{"1min":0.76400000000000001,"5min":0.48499999999999999,"15min":0.46000000000000002},"max":{"1min":1.375,"5min":7.056,"15min":14.977},"last":0.99299999999999999},{"interface":"front","average":{"1min":1.0649999999999999,"5min":1.165,"15min":1.714},"min":{"1min":0.76700000000000002,"5min":0.371,"15min":0.371},"max":{"1min":1.444,"5min":6.3789999999999996,"15min":7.6020000000000003},"last":1.272}]}]},{"osd":0,"up_from":86,"seq":369367187605,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":219664,"kb_used_data":1864,"kb_used_omap":5,"kb_used_meta":217786,"kb_avail":93525488,"statfs":{"total":95995035648,"available":95770099712,"internally_reserved":0,"allocated":1908736,"data_stored":1361683,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":223013498},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Jun 3 19:59:12 2023","interfaces":[{"interface":"back","average":{"1min":0.64400000000000002,"5min":1.024,"15min":2.0649999999999999},"min":{"1min":0.26900000000000002,"5min":0.217,"15min":0.217},"max":{"1min":0.91700000000000004,"5min":8.2579999999999991,"15min":13.760999999999999},"last":0.81999999999999995},{"interface":"front","average":{"1min":0.68600000000000005,"5min":1.0700000000000001,"15min":1.7170000000000001},"min":{"1min":0.44900000000000001,"5min":0.26200000000000001,"15min":0.253},"max":{"1min":1.026,"5min":9.1280000000000001,"15min":9.1280000000000001},"last":0.77400000000000002}]},{"osd":2,"last update":"Sat Jun 3 19:59:14 2023","interfaces":[{"interface":"back","average":{"1min":0.59799999999999998,"5min":0.65400000000000003,"15min":0.65700000000000003},"min":{"1min":0.36899999999999999,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":0.876,"5min":1.077,"15min":1.077},"last":0.54800000000000004},{"interface":"front","average":{"1min":0.77000000000000002,"5min":0.76100000000000001,"15min":0.751},"min":{"1min":0.32000000000000001,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":1.1890000000000001,"5min":1.1890000000000001,"15min":1.1890000000000001},"last":1.1040000000000001}]},{"osd":3,"last update":"Sat Jun 3 19:58:59 2023","interfaces":[{"interface":"back","average":{"1min":0.70299999999999996,"5min":0.79300000000000004,"15min":0.80800000000000005},"min":{"1min":0.42599999999999999,"5min":0.42599999999999999,"15min":0.42599999999999999},"max":{"1min":1.0249999999999999,"5min":1.052,"15min":1.052},"last":0.86899999999999999},{"interface":"front","average":{"1min":0.77000000000000002,"5min":0.76200000000000001,"15min":0.76100000000000001},"min":{"1min":0.35399999999999998,"5min":0.35399999999999998,"15min":0.35399999999999998},"max":{"1min":1.1100000000000001,"5min":1.1819999999999999,"15min":1.1819999999999999},"last":1.0209999999999999}]},{"osd":4,"last update":"Sat Jun 3 19:58:31 2023","interfaces":[{"interface":"back","average":{"1min":1.0960000000000001,"5min":1.0960000000000001,"15min":1.0960000000000001},"min":{"1min":0.84099999999999997,"5min":0.84099999999999997,"15min":0.84099999999999997},"max":{"1min":1.474,"5min":1.474,"15min":1.474},"last":1.0600000000000001},{"interface":"front","average":{"1min":1.1180000000000001,"5min":1.1180000000000001,"15min":1.1180000000000001},"min":{"1min":0.73099999999999998,"5min":0.73099999999999998,"15min":0.73099999999999998},"max":{"1min":1.597,"5min":1.597,"15min":1.597},"last":1.2330000000000001}]},{"osd":5,"last update":"Sat Jun 3 19:58:31 2023","interfaces":[{"interface":"back","average":{"1min":0.99299999999999999,"5min":1.522,"15min":2.1280000000000001},"min":{"1min":0.623,"5min":0.35999999999999999,"15min":0.33900000000000002},"max":{"1min":1.2909999999999999,"5min":9.2059999999999995,"15min":16.312000000000001},"last":0.92400000000000004},{"interface":"front","average":{"1min":1.1279999999999999,"5min":1.722,"15min":1.8080000000000001},"min":{"1min":0.70499999999999996,"5min":0.40500000000000003,"15min":0.32500000000000001},"max":{"1min":1.5449999999999999,"5min":9.3330000000000002,"15min":24.661000000000001},"last":1.1319999999999999}]},{"osd":6,"last update":"Sat Jun 3 19:58:31 2023","interfaces":[{"interface":"back","average":{"1min":1.1599999999999999,"5min":1.7310000000000001,"15min":1.778},"min":{"1min":0.63900000000000001,"5min":0.36099999999999999,"15min":0.33500000000000002},"max":{"1min":1.538,"5min":9.3629999999999995,"15min":24.599},"last":1.1910000000000001},{"interface":"front","average":{"1min":1.1839999999999999,"5min":1.6839999999999999,"15min":2.1869999999999998},"min":{"1min":0.85299999999999998,"5min":0.42999999999999999,"15min":0.42999999999999999},"max":{"1min":1.6299999999999999,"5min":10.459,"15min":13.601000000000001},"last":1.48}]},{"osd":7,"last update":"Sat Jun 3 19:58:31 2023","interfaces":[{"interface":"back","average":{"1min":1.1579999999999999,"5min":1.7170000000000001,"15min":1.7},"min":{"1min":0.66900000000000004,"5min":0.39900000000000002,"15min":0.39900000000000002},"max":{"1min":1.605,"5min":9.4190000000000005,"15min":24.634},"last":1.2569999999999999},{"interface":"front","average":{"1min":1.1639999999999999,"5min":1.5600000000000001,"15min":2.0009999999999999},"min":{"1min":0.89300000000000002,"5min":0.45800000000000002,"15min":0.40699999999999997},"max":{"1min":1.573,"5min":12.621,"15min":12.621},"last":1.4319999999999999}]}]},{"osd":2,"up_from":114,"seq":489626271784,"num_pgs":19,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":15572,"kb_used_data":1228,"kb_used_omap":7,"kb_used_meta":14328,"kb_avail":93729580,"statfs":{"total":95995035648,"available":95979089920,"internally_reserved":0,"allocated":1257472,"data_stored":764442,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8111,"internal_metadata":14671953},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 3 19:59:16 2023","interfaces":[{"interface":"back","average":{"1min":0.65700000000000003,"5min":0.625,"15min":0.625},"min":{"1min":0.45100000000000001,"5min":0.314,"15min":0.314},"max":{"1min":0.93500000000000005,"5min":0.93500000000000005,"15min":0.93500000000000005},"last":0.78200000000000003},{"interface":"front","average":{"1min":0.71199999999999997,"5min":0.70999999999999996,"15min":0.70899999999999996},"min":{"1min":0.41299999999999998,"5min":0.41299999999999998,"15min":0.41299999999999998},"max":{"1min":1,"5min":1.083,"15min":1.083},"last":0.61399999999999999}]},{"osd":1,"last update":"Sat Jun 3 19:59:16 2023","interfaces":[{"interface":"back","average":{"1min":0.65100000000000002,"5min":0.69899999999999995,"15min":0.71299999999999997},"min":{"1min":0.48299999999999998,"5min":0.36499999999999999,"15min":0.36499999999999999},"max":{"1min":0.85499999999999998,"5min":1.1020000000000001,"15min":1.1020000000000001},"last":0.92700000000000005},{"interface":"front","average":{"1min":0.68200000000000005,"5min":0.67300000000000004,"15min":0.67500000000000004},"min":{"1min":0.36399999999999999,"5min":0.36399999999999999,"15min":0.36399999999999999},"max":{"1min":1.0469999999999999,"5min":1.0580000000000001,"15min":1.0580000000000001},"last":0.73799999999999999}]},{"osd":3,"last update":"Sat Jun 3 19:59:02 2023","interfaces":[{"interface":"back","average":{"1min":0.65100000000000002,"5min":0.73099999999999998,"15min":0.74399999999999999},"min":{"1min":0.39400000000000002,"5min":0.39400000000000002,"15min":0.39400000000000002},"max":{"1min":0.98399999999999999,"5min":1.0409999999999999,"15min":1.0409999999999999},"last":0.67700000000000005},{"interface":"front","average":{"1min":0.70999999999999996,"5min":0.748,"15min":0.754},"min":{"1min":0.38,"5min":0.38,"15min":0.38},"max":{"1min":1.155,"5min":1.155,"15min":1.155},"last":0.89100000000000001}]},{"osd":4,"last update":"Sat Jun 3 19:58:31 2023","interfaces":[{"interface":"back","average":{"1min":0.92100000000000004,"5min":0.92100000000000004,"15min":0.92100000000000004},"min":{"1min":0.70699999999999996,"5min":0.70699999999999996,"15min":0.70699999999999996},"max":{"1min":1.2090000000000001,"5min":1.2090000000000001,"15min":1.2090000000000001},"last":1.0700000000000001},{"interface":"front","average":{"1min":0.97499999999999998,"5min":0.97499999999999998,"15min":0.97499999999999998},"min":{"1min":0.68100000000000005,"5min":0.68100000000000005,"15min":0.68100000000000005},"max":{"1min":1.3069999999999999,"5min":1.3069999999999999,"15min":1.3069999999999999},"last":1.0429999999999999}]},{"osd":5,"last update":"Sat Jun 3 19:59:16 2023","interfaces":[{"interface":"back","average":{"1min":0.96599999999999997,"5min":1.0169999999999999,"15min":1.0309999999999999},"min":{"1min":0.629,"5min":0.629,"15min":0.629},"max":{"1min":1.3680000000000001,"5min":1.3979999999999999,"15min":1.3979999999999999},"last":1.1040000000000001},{"interface":"front","average":{"1min":1.024,"5min":1.0389999999999999,"15min":1.0449999999999999},"min":{"1min":0.73999999999999999,"5min":0.65100000000000002,"15min":0.65100000000000002},"max":{"1min":1.3779999999999999,"5min":1.3779999999999999,"15min":1.3779999999999999},"last":1.137}]},{"osd":6,"last update":"Sat Jun 3 19:59:16 2023","interfaces":[{"interface":"back","average":{"1min":0.996,"5min":1.028,"15min":1.032},"min":{"1min":0.70199999999999996,"5min":0.70199999999999996,"15min":0.70199999999999996},"max":{"1min":1.2889999999999999,"5min":1.3899999999999999,"15min":1.3899999999999999},"last":0.84999999999999998},{"interface":"front","average":{"1min":0.98499999999999999,"5min":1.04,"15min":1.044},"min":{"1min":0.69299999999999995,"5min":0.68100000000000005,"15min":0.68100000000000005},"max":{"1min":1.3109999999999999,"5min":1.508,"15min":1.508},"last":1.1850000000000001}]},{"osd":7,"last update":"Sat Jun 3 19:59:16 2023","interfaces":[{"interface":"back","average":{"1min":1.1040000000000001,"5min":1.1519999999999999,"15min":1.1699999999999999},"min":{"1min":0.76100000000000001,"5min":0.76100000000000001,"15min":0.76100000000000001},"max":{"1min":1.4470000000000001,"5min":1.456,"15min":1.456},"last":1.008},{"interface":"front","average":{"1min":1.1220000000000001,"5min":1.1439999999999999,"15min":1.1539999999999999},"min":{"1min":0.81599999999999995,"5min":0.78600000000000003,"15min":0.78600000000000003},"max":{"1min":1.4419999999999999,"5min":1.6930000000000001,"15min":1.6930000000000001},"last":1.232}]}]},{"osd":3,"up_from":119,"seq":511101108256,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":18108,"kb_used_data":1204,"kb_used_omap":2,"kb_used_meta":16893,"kb_avail":93727044,"statfs":{"total":95995035648,"available":95976493056,"internally_reserved":0,"allocated":1232896,"data_stored":760088,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":17298485},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 3 19:58:58 2023","interfaces":[{"interface":"back","average":{"1min":0.57999999999999996,"5min":0.621,"15min":0.628},"min":{"1min":0.33300000000000002,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":0.96799999999999997,"5min":0.96799999999999997,"15min":0.96799999999999997},"last":0.61199999999999999},{"interface":"front","average":{"1min":0.58599999999999997,"5min":0.70199999999999996,"15min":0.72099999999999997},"min":{"1min":0.23799999999999999,"5min":0.23799999999999999,"15min":0.23799999999999999},"max":{"1min":0.85999999999999999,"5min":1,"15min":1},"last":0.78100000000000003}]},{"osd":1,"last update":"Sat Jun 3 19:58:58 2023","interfaces":[{"interface":"back","average":{"1min":0.59899999999999998,"5min":0.68799999999999994,"15min":0.70299999999999996},"min":{"1min":0.314,"5min":0.314,"15min":0.314},"max":{"1min":0.93000000000000005,"5min":1.028,"15min":1.028},"last":0.70799999999999996},{"interface":"front","average":{"1min":0.67600000000000005,"5min":0.74399999999999999,"15min":0.755},"min":{"1min":0.28499999999999998,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":1.018,"5min":1.018,"15min":1.018},"last":0.83299999999999996}]},{"osd":2,"last update":"Sat Jun 3 19:58:58 2023","interfaces":[{"interface":"back","average":{"1min":0.64800000000000002,"5min":0.64400000000000002,"15min":0.64400000000000002},"min":{"1min":0.32400000000000001,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":1.03,"5min":1.03,"15min":1.03},"last":0.94699999999999995},{"interface":"front","average":{"1min":0.65100000000000002,"5min":0.73999999999999999,"15min":0.755},"min":{"1min":0.34799999999999998,"5min":0.34799999999999998,"15min":0.34799999999999998},"max":{"1min":1.046,"5min":1.1140000000000001,"15min":1.1140000000000001},"last":0.91600000000000004}]},{"osd":4,"last update":"Sat Jun 3 19:58:28 2023","interfaces":[{"interface":"back","average":{"1min":1,"5min":1,"15min":1},"min":{"1min":0.71499999999999997,"5min":0.71499999999999997,"15min":0.71499999999999997},"max":{"1min":1.3819999999999999,"5min":1.3819999999999999,"15min":1.3819999999999999},"last":1.163},{"interface":"front","average":{"1min":1.0109999999999999,"5min":1.0109999999999999,"15min":1.0109999999999999},"min":{"1min":0.78200000000000003,"5min":0.78200000000000003,"15min":0.78200000000000003},"max":{"1min":1.2689999999999999,"5min":1.2689999999999999,"15min":1.2689999999999999},"last":1.1859999999999999}]},{"osd":5,"last update":"Sat Jun 3 19:58:58 2023","interfaces":[{"interface":"back","average":{"1min":0.97899999999999998,"5min":0.98499999999999999,"15min":0.98599999999999999},"min":{"1min":0.69499999999999995,"5min":0.69499999999999995,"15min":0.69499999999999995},"max":{"1min":1.2929999999999999,"5min":1.429,"15min":1.429},"last":1.2789999999999999},{"interface":"front","average":{"1min":0.95699999999999996,"5min":1.0109999999999999,"15min":1.02},"min":{"1min":0.56200000000000006,"5min":0.56200000000000006,"15min":0.56200000000000006},"max":{"1min":1.343,"5min":1.5900000000000001,"15min":1.5900000000000001},"last":1.091}]},{"osd":6,"last update":"Sat Jun 3 19:58:58 2023","interfaces":[{"interface":"back","average":{"1min":1.089,"5min":1.044,"15min":1.036},"min":{"1min":0.74099999999999999,"5min":0.67100000000000004,"15min":0.67100000000000004},"max":{"1min":1.351,"5min":1.6559999999999999,"15min":1.6559999999999999},"last":1.2170000000000001},{"interface":"front","average":{"1min":1.04,"5min":1.054,"15min":1.056},"min":{"1min":0.58299999999999996,"5min":0.58299999999999996,"15min":0.58299999999999996},"max":{"1min":1.369,"5min":1.369,"15min":1.369},"last":1.323}]},{"osd":7,"last update":"Sat Jun 3 19:58:58 2023","interfaces":[{"interface":"back","average":{"1min":0.98899999999999999,"5min":1.0329999999999999,"15min":1.04},"min":{"1min":0.62,"5min":0.62,"15min":0.62},"max":{"1min":1.351,"5min":1.351,"15min":1.351},"last":1.246},{"interface":"front","average":{"1min":0.97899999999999998,"5min":1.018,"15min":1.024},"min":{"1min":0.53600000000000003,"5min":0.53600000000000003,"15min":0.53600000000000003},"max":{"1min":1.3129999999999999,"5min":1.4610000000000001,"15min":1.4610000000000001},"last":1.129}]}]},{"osd":4,"up_from":124,"seq":532575944729,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":15492,"kb_used_data":1212,"kb_used_omap":10,"kb_used_meta":14261,"kb_avail":93729660,"statfs":{"total":95995035648,"available":95979171840,"internally_reserved":0,"allocated":1241088,"data_stored":762768,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":10652,"internal_metadata":14603876},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 3 19:58:29 2023","interfaces":[{"interface":"back","average":{"1min":0.97499999999999998,"5min":0.97499999999999998,"15min":0.97499999999999998},"min":{"1min":0.55800000000000005,"5min":0.55800000000000005,"15min":0.55800000000000005},"max":{"1min":1.3240000000000001,"5min":1.3240000000000001,"15min":1.3240000000000001},"last":0.78100000000000003},{"interface":"front","average":{"1min":1.0309999999999999,"5min":1.0309999999999999,"15min":1.0309999999999999},"min":{"1min":0.78200000000000003,"5min":0.78200000000000003,"15min":0.78200000000000003},"max":{"1min":1.2250000000000001,"5min":1.2250000000000001,"15min":1.2250000000000001},"last":1.1319999999999999}]},{"osd":1,"last update":"Sat Jun 3 19:58:29 2023","interfaces":[{"interface":"back","average":{"1min":0.871,"5min":0.871,"15min":0.871},"min":{"1min":0.60999999999999999,"5min":0.60999999999999999,"15min":0.60999999999999999},"max":{"1min":1.2549999999999999,"5min":1.2549999999999999,"15min":1.2549999999999999},"last":1.0600000000000001},{"interface":"front","average":{"1min":0.86299999999999999,"5min":0.86299999999999999,"15min":0.86299999999999999},"min":{"1min":0.54900000000000004,"5min":0.54900000000000004,"15min":0.54900000000000004},"max":{"1min":1.278,"5min":1.278,"15min":1.278},"last":1.1599999999999999}]},{"osd":2,"last update":"Sat Jun 3 19:58:29 2023","interfaces":[{"interface":"back","average":{"1min":0.871,"5min":0.871,"15min":0.871},"min":{"1min":0.52800000000000002,"5min":0.52800000000000002,"15min":0.52800000000000002},"max":{"1min":1.196,"5min":1.196,"15min":1.196},"last":0.99299999999999999},{"interface":"front","average":{"1min":0.93799999999999994,"5min":0.93799999999999994,"15min":0.93799999999999994},"min":{"1min":0.63700000000000001,"5min":0.63700000000000001,"15min":0.63700000000000001},"max":{"1min":1.2769999999999999,"5min":1.2769999999999999,"15min":1.2769999999999999},"last":1.0229999999999999}]},{"osd":3,"last update":"Sat Jun 3 19:58:29 2023","interfaces":[{"interface":"back","average":{"1min":0.96199999999999997,"5min":0.96199999999999997,"15min":0.96199999999999997},"min":{"1min":0.68700000000000006,"5min":0.68700000000000006,"15min":0.68700000000000006},"max":{"1min":1.238,"5min":1.238,"15min":1.238},"last":1.1060000000000001},{"interface":"front","average":{"1min":1.056,"5min":1.056,"15min":1.056},"min":{"1min":0.71899999999999997,"5min":0.71899999999999997,"15min":0.71899999999999997},"max":{"1min":1.28,"5min":1.28,"15min":1.28},"last":1.0840000000000001}]},{"osd":5,"last update":"Sat Jun 3 19:58:29 2023","interfaces":[{"interface":"back","average":{"1min":0.95299999999999996,"5min":0.95299999999999996,"15min":0.95299999999999996},"min":{"1min":0.72399999999999998,"5min":0.72399999999999998,"15min":0.72399999999999998},"max":{"1min":1.234,"5min":1.234,"15min":1.234},"last":1.224},{"interface":"front","average":{"1min":0.98799999999999999,"5min":0.98799999999999999,"15min":0.98799999999999999},"min":{"1min":0.77200000000000002,"5min":0.77200000000000002,"15min":0.77200000000000002},"max":{"1min":1.24,"5min":1.24,"15min":1.24},"last":0.871}]},{"osd":6,"last update":"Sat Jun 3 19:58:29 2023","interfaces":[{"interface":"back","average":{"1min":1.0449999999999999,"5min":1.0449999999999999,"15min":1.0449999999999999},"min":{"1min":0.51300000000000001,"5min":0.51300000000000001,"15min":0.51300000000000001},"max":{"1min":1.323,"5min":1.323,"15min":1.323},"last":0.90700000000000003},{"interface":"front","average":{"1min":1.079,"5min":1.079,"15min":1.079},"min":{"1min":0.73399999999999999,"5min":0.73399999999999999,"15min":0.73399999999999999},"max":{"1min":1.494,"5min":1.494,"15min":1.494},"last":1.196}]},{"osd":7,"last update":"Sat Jun 3 19:58:29 2023","interfaces":[{"interface":"back","average":{"1min":1.1379999999999999,"5min":1.1379999999999999,"15min":1.1379999999999999},"min":{"1min":0.77100000000000002,"5min":0.77100000000000002,"15min":0.77100000000000002},"max":{"1min":1.4490000000000001,"5min":1.4490000000000001,"15min":1.4490000000000001},"last":0.94999999999999996},{"interface":"front","average":{"1min":1.171,"5min":1.171,"15min":1.171},"min":{"1min":0.90300000000000002,"5min":0.90300000000000002,"15min":0.90300000000000002},"max":{"1min":1.383,"5min":1.383,"15min":1.383},"last":1.2629999999999999}]}]},{"osd":5,"up_from":36,"seq":154618823015,"num_pgs":21,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":358044,"kb_used_data":1172,"kb_used_omap":0,"kb_used_meta":356864,"kb_avail":93387108,"statfs":{"total":95995035648,"available":95628398592,"internally_reserved":0,"allocated":1200128,"data_stored":753684,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":365428736},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 3 19:59:19 2023","interfaces":[{"interface":"back","average":{"1min":0.878,"5min":1.026,"15min":1.4530000000000001},"min":{"1min":0.55100000000000005,"5min":0.26700000000000002,"15min":0.26600000000000001},"max":{"1min":1.1970000000000001,"5min":4.5419999999999998,"15min":12.489000000000001},"last":1.0069999999999999},{"interface":"front","average":{"1min":0.98499999999999999,"5min":1.2669999999999999,"15min":1.6990000000000001},"min":{"1min":0.76700000000000002,"5min":0.39700000000000002,"15min":0.31},"max":{"1min":1.4059999999999999,"5min":6.0720000000000001,"15min":12.542999999999999},"last":1.05}]},{"osd":1,"last update":"Sat Jun 3 19:59:16 2023","interfaces":[{"interface":"back","average":{"1min":0.90600000000000003,"5min":1.2889999999999999,"15min":1.7809999999999999},"min":{"1min":0.64300000000000002,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":1.272,"5min":11.27,"15min":11.27},"last":1.0309999999999999},{"interface":"front","average":{"1min":0.91500000000000004,"5min":1.528,"15min":1.8600000000000001},"min":{"1min":0.58999999999999997,"5min":0.32100000000000001,"15min":0.29999999999999999},"max":{"1min":1.2989999999999999,"5min":11.34,"15min":11.34},"last":0.80800000000000005}]},{"osd":2,"last update":"Sat Jun 3 19:59:16 2023","interfaces":[{"interface":"back","average":{"1min":1.0029999999999999,"5min":0.90300000000000002,"15min":0.879},"min":{"1min":0.70099999999999996,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":1.3560000000000001,"5min":1.3560000000000001,"15min":1.3560000000000001},"last":1.107},{"interface":"front","average":{"1min":0.97099999999999997,"5min":0.96899999999999997,"15min":0.97399999999999998},"min":{"1min":0.72199999999999998,"5min":0.48799999999999999,"15min":0.48799999999999999},"max":{"1min":1.1899999999999999,"5min":1.335,"15min":1.335},"last":0.97899999999999998}]},{"osd":3,"last update":"Sat Jun 3 19:58:55 2023","interfaces":[{"interface":"back","average":{"1min":1.004,"5min":1.0149999999999999,"15min":1.0169999999999999},"min":{"1min":0.66600000000000004,"5min":0.38,"15min":0.38},"max":{"1min":1.3779999999999999,"5min":1.3779999999999999,"15min":1.3779999999999999},"last":1.173},{"interface":"front","average":{"1min":1.0920000000000001,"5min":1.0109999999999999,"15min":0.997},"min":{"1min":0.81799999999999995,"5min":0.50700000000000001,"15min":0.50700000000000001},"max":{"1min":1.361,"5min":1.363,"15min":1.363},"last":0.93700000000000006}]},{"osd":4,"last update":"Sat Jun 3 19:58:30 2023","interfaces":[{"interface":"back","average":{"1min":1.002,"5min":1.002,"15min":1.002},"min":{"1min":0.60299999999999998,"5min":0.60299999999999998,"15min":0.60299999999999998},"max":{"1min":1.3500000000000001,"5min":1.3500000000000001,"15min":1.3500000000000001},"last":0.75800000000000001},{"interface":"front","average":{"1min":1.1879999999999999,"5min":1.1879999999999999,"15min":1.1879999999999999},"min":{"1min":1.022,"5min":1.022,"15min":1.022},"max":{"1min":1.522,"5min":1.522,"15min":1.522},"last":0.85399999999999998}]},{"osd":6,"last update":"Sat Jun 3 19:59:19 2023","interfaces":[{"interface":"back","average":{"1min":1.052,"5min":1.403,"15min":1.21},"min":{"1min":0.67400000000000004,"5min":0.33000000000000002,"15min":0.33000000000000002},"max":{"1min":1.431,"5min":12.731999999999999,"15min":12.731999999999999},"last":0.95399999999999996},{"interface":"front","average":{"1min":1.175,"5min":1.4379999999999999,"15min":1.3169999999999999},"min":{"1min":0.71699999999999997,"5min":0.42899999999999999,"15min":0.32800000000000001},"max":{"1min":1.504,"5min":12.779,"15min":12.779},"last":1.0820000000000001}]},{"osd":7,"last update":"Sat Jun 3 19:59:06 2023","interfaces":[{"interface":"back","average":{"1min":1.198,"5min":1.605,"15min":1.3979999999999999},"min":{"1min":0.68799999999999994,"5min":0.45700000000000002,"15min":0.379},"max":{"1min":1.5229999999999999,"5min":11.297000000000001,"15min":11.297000000000001},"last":1.139},{"interface":"front","average":{"1min":1.1950000000000001,"5min":1.2909999999999999,"15min":1.252},"min":{"1min":0.89300000000000002,"5min":0.39600000000000002,"15min":0.32100000000000001},"max":{"1min":1.468,"5min":5.9240000000000004,"15min":7.4720000000000004},"last":1.1619999999999999}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6928,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4862,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":10652,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4786,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4932,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-06-03T19:59:26.054 INFO:tasks.cephadm.ceph_manager.ceph:clean! 2023-06-03T19:59:26.055 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph osd dump 2023-06-03T19:59:26.084 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:59:25 smithi164 bash[59462]: cluster 1685822364.4036427 mgr.y (mgr.34104) 692 : cluster 0 pgmap v597: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-06-03T19:59:26.528 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 19:59:26 smithi089 bash[71163]: ::ffff:172.21.15.164 - - [03/Jun/2023:19:59:26] "GET /metrics HTTP/1.1" 200 225309 "" "Prometheus/2.33.4" 2023-06-03T19:59:27.029 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:59:26 smithi089 bash[75068]: audit 1685822365.2617035 mgr.y (mgr.34104) 693 : audit 0 from='client.35694 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-06-03T19:59:27.029 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:59:26 smithi089 bash[78452]: audit 1685822365.2617035 mgr.y (mgr.34104) 693 : audit 0 from='client.35694 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-06-03T19:59:27.084 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:59:26 smithi164 bash[59462]: audit 1685822365.2617035 mgr.y (mgr.34104) 693 : audit 0 from='client.35694 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-06-03T19:59:28.029 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:59:27 smithi089 bash[78452]: cluster 1685822366.404365 mgr.y (mgr.34104) 694 : cluster 0 pgmap v598: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-06-03T19:59:28.029 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:59:27 smithi089 bash[75068]: cluster 1685822366.404365 mgr.y (mgr.34104) 694 : cluster 0 pgmap v598: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-06-03T19:59:28.084 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:59:27 smithi164 bash[59462]: cluster 1685822366.404365 mgr.y (mgr.34104) 694 : cluster 0 pgmap v598: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-06-03T19:59:30.029 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:59:29 smithi089 bash[78452]: cluster 1685822368.4054303 mgr.y (mgr.34104) 695 : cluster 0 pgmap v599: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-06-03T19:59:30.029 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:59:29 smithi089 bash[75068]: cluster 1685822368.4054303 mgr.y (mgr.34104) 695 : cluster 0 pgmap v599: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-06-03T19:59:30.084 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:59:29 smithi164 bash[59462]: cluster 1685822368.4054303 mgr.y (mgr.34104) 695 : cluster 0 pgmap v599: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-06-03T19:59:30.910 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/mon.a/config 2023-06-03T19:59:32.028 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:59:31 smithi089 bash[78452]: cluster 1685822370.4060352 mgr.y (mgr.34104) 696 : cluster 0 pgmap v600: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-06-03T19:59:32.029 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:59:31 smithi089 bash[75068]: cluster 1685822370.4060352 mgr.y (mgr.34104) 696 : cluster 0 pgmap v600: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-06-03T19:59:32.084 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:59:31 smithi164 bash[59462]: cluster 1685822370.4060352 mgr.y (mgr.34104) 696 : cluster 0 pgmap v600: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-06-03T19:59:34.028 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:59:33 smithi089 bash[75068]: cluster 1685822372.4068406 mgr.y (mgr.34104) 697 : cluster 0 pgmap v601: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-06-03T19:59:34.029 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:59:33 smithi089 bash[78452]: cluster 1685822372.4068406 mgr.y (mgr.34104) 697 : cluster 0 pgmap v601: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-06-03T19:59:34.084 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:59:33 smithi164 bash[59462]: cluster 1685822372.4068406 mgr.y (mgr.34104) 697 : cluster 0 pgmap v601: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-06-03T19:59:34.679 INFO:teuthology.orchestra.run.smithi089.stdout:epoch 125 2023-06-03T19:59:34.680 INFO:teuthology.orchestra.run.smithi089.stdout:fsid f1db3478-0243-11ee-9b1d-001a4aab830c 2023-06-03T19:59:34.680 INFO:teuthology.orchestra.run.smithi089.stdout:created 2023-06-03T19:22:54.447184+0000 2023-06-03T19:59:34.680 INFO:teuthology.orchestra.run.smithi089.stdout:modified 2023-06-03T19:57:28.437574+0000 2023-06-03T19:59:34.680 INFO:teuthology.orchestra.run.smithi089.stdout:flags sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit 2023-06-03T19:59:34.680 INFO:teuthology.orchestra.run.smithi089.stdout:crush_version 23 2023-06-03T19:59:34.680 INFO:teuthology.orchestra.run.smithi089.stdout:full_ratio 0.95 2023-06-03T19:59:34.681 INFO:teuthology.orchestra.run.smithi089.stdout:backfillfull_ratio 0.9 2023-06-03T19:59:34.681 INFO:teuthology.orchestra.run.smithi089.stdout:nearfull_ratio 0.85 2023-06-03T19:59:34.681 INFO:teuthology.orchestra.run.smithi089.stdout:require_min_compat_client luminous 2023-06-03T19:59:34.681 INFO:teuthology.orchestra.run.smithi089.stdout:min_compat_client luminous 2023-06-03T19:59:34.681 INFO:teuthology.orchestra.run.smithi089.stdout:require_osd_release pacific 2023-06-03T19:59:34.681 INFO:teuthology.orchestra.run.smithi089.stdout:stretch_mode_enabled false 2023-06-03T19:59:34.681 INFO:teuthology.orchestra.run.smithi089.stdout:pool 1 '.mgr' replicated size 3 min_size 2 crush_rule 0 object_hash rjenkins pg_num 1 pgp_num 1 autoscale_mode off last_change 70 flags hashpspool stripe_width 0 pg_num_max 32 pg_num_min 1 application mgr,mgr_devicehealth read_balance_score 7.89 2023-06-03T19:59:34.681 INFO:teuthology.orchestra.run.smithi089.stdout:pool 2 'rbd' replicated size 3 min_size 2 crush_rule 0 object_hash rjenkins pg_num 8 pgp_num 8 autoscale_mode off last_change 55 flags hashpspool stripe_width 0 application rbd read_balance_score 2.00 2023-06-03T19:59:34.682 INFO:teuthology.orchestra.run.smithi089.stdout:pool 3 'cephfs.foo.meta' replicated size 3 min_size 2 crush_rule 0 object_hash rjenkins pg_num 32 pgp_num 32 autoscale_mode off last_change 59 flags hashpspool stripe_width 0 pg_autoscale_bias 4 pg_num_min 16 recovery_priority 5 application cephfs read_balance_score 1.50 2023-06-03T19:59:34.682 INFO:teuthology.orchestra.run.smithi089.stdout:pool 4 'cephfs.foo.data' replicated size 3 min_size 2 crush_rule 0 object_hash rjenkins pg_num 32 pgp_num 32 autoscale_mode off last_change 60 flags hashpspool stripe_width 0 application cephfs read_balance_score 2.25 2023-06-03T19:59:34.682 INFO:teuthology.orchestra.run.smithi089.stdout:max_osd 8 2023-06-03T19:59:34.682 INFO:teuthology.orchestra.run.smithi089.stdout:osd.0 up in weight 1 up_from 86 up_thru 124 down_at 83 last_clean_interval [8,82) [v2:172.21.15.89:6802/2920103307,v1:172.21.15.89:6803/2920103307] [v2:172.21.15.89:6804/2920103307,v1:172.21.15.89:6805/2920103307] exists,up 86063bfe-126e-46e7-93ed-c73813af165b 2023-06-03T19:59:34.682 INFO:teuthology.orchestra.run.smithi089.stdout:osd.1 up in weight 1 up_from 103 up_thru 124 down_at 100 last_clean_interval [13,99) [v2:172.21.15.89:6810/2092956116,v1:172.21.15.89:6811/2092956116] [v2:172.21.15.89:6812/2092956116,v1:172.21.15.89:6813/2092956116] exists,up 04783a81-6b8c-4657-8446-00efc872b313 2023-06-03T19:59:34.682 INFO:teuthology.orchestra.run.smithi089.stdout:osd.2 up in weight 1 up_from 114 up_thru 121 down_at 111 last_clean_interval [18,110) [v2:172.21.15.89:6818/1439643284,v1:172.21.15.89:6819/1439643284] [v2:172.21.15.89:6820/1439643284,v1:172.21.15.89:6821/1439643284] exists,up d2d2c641-392a-444f-81bc-84eef6340f30 2023-06-03T19:59:34.682 INFO:teuthology.orchestra.run.smithi089.stdout:osd.3 up in weight 1 up_from 119 up_thru 124 down_at 116 last_clean_interval [25,115) [v2:172.21.15.89:6826/1558463544,v1:172.21.15.89:6827/1558463544] [v2:172.21.15.89:6828/1558463544,v1:172.21.15.89:6829/1558463544] exists,up b38408eb-3acf-4a41-a314-72e4cd37b1ef 2023-06-03T19:59:34.683 INFO:teuthology.orchestra.run.smithi089.stdout:osd.4 up in weight 1 up_from 124 up_thru 124 down_at 121 last_clean_interval [30,120) [v2:172.21.15.164:6800/3822759075,v1:172.21.15.164:6801/3822759075] [v2:172.21.15.164:6802/3822759075,v1:172.21.15.164:6803/3822759075] exists,up a94be785-9f54-4ab8-962a-9cf974a5b5e3 2023-06-03T19:59:34.683 INFO:teuthology.orchestra.run.smithi089.stdout:osd.5 up in weight 1 up_from 36 up_thru 124 down_at 0 last_clean_interval [0,0) [v2:172.21.15.164:6808/3123006405,v1:172.21.15.164:6809/3123006405] [v2:172.21.15.164:6810/3123006405,v1:172.21.15.164:6811/3123006405] exists,up b8a5e8a1-23cc-442b-90aa-19bd0036dd3c 2023-06-03T19:59:34.683 INFO:teuthology.orchestra.run.smithi089.stdout:osd.6 up in weight 1 up_from 42 up_thru 124 down_at 0 last_clean_interval [0,0) [v2:172.21.15.164:6816/712403290,v1:172.21.15.164:6817/712403290] [v2:172.21.15.164:6818/712403290,v1:172.21.15.164:6819/712403290] exists,up 5c5edd3b-d92c-4f46-94ff-2d5828b2e732 2023-06-03T19:59:34.683 INFO:teuthology.orchestra.run.smithi089.stdout:osd.7 up in weight 1 up_from 48 up_thru 124 down_at 0 last_clean_interval [0,0) [v2:172.21.15.164:6824/711997991,v1:172.21.15.164:6825/711997991] [v2:172.21.15.164:6826/711997991,v1:172.21.15.164:6827/711997991] exists,up bdff9290-e6c4-407e-9113-eb39696c0e27 2023-06-03T19:59:34.683 INFO:teuthology.orchestra.run.smithi089.stdout:pg_upmap_items 4.9 [1,2] 2023-06-03T19:59:34.683 INFO:teuthology.orchestra.run.smithi089.stdout:pg_upmap_items 4.b [1,2] 2023-06-03T19:59:34.684 INFO:teuthology.orchestra.run.smithi089.stdout:pg_upmap_items 4.d [1,5] 2023-06-03T19:59:34.684 INFO:teuthology.orchestra.run.smithi089.stdout:blocklist 172.21.15.89:0/3543817158 expires 2023-06-04T19:41:21.936058+0000 2023-06-03T19:59:34.684 INFO:teuthology.orchestra.run.smithi089.stdout:blocklist 172.21.15.164:0/2793695939 expires 2023-06-04T19:40:01.102939+0000 2023-06-03T19:59:34.684 INFO:teuthology.orchestra.run.smithi089.stdout:blocklist 172.21.15.89:0/3944267183 expires 2023-06-04T19:23:55.626873+0000 2023-06-03T19:59:34.684 INFO:teuthology.orchestra.run.smithi089.stdout:blocklist 172.21.15.89:0/2890034071 expires 2023-06-04T19:41:21.936058+0000 2023-06-03T19:59:34.684 INFO:teuthology.orchestra.run.smithi089.stdout:blocklist 172.21.15.89:6801/3918707857 expires 2023-06-04T19:41:21.936058+0000 2023-06-03T19:59:34.684 INFO:teuthology.orchestra.run.smithi089.stdout:blocklist 172.21.15.89:6800/3918707857 expires 2023-06-04T19:41:21.936058+0000 2023-06-03T19:59:34.685 INFO:teuthology.orchestra.run.smithi089.stdout:blocklist 172.21.15.89:6800/2256305550 expires 2023-06-04T19:31:48.303240+0000 2023-06-03T19:59:34.685 INFO:teuthology.orchestra.run.smithi089.stdout:blocklist 172.21.15.89:0/2023636358 expires 2023-06-04T19:37:38.185918+0000 2023-06-03T19:59:34.685 INFO:teuthology.orchestra.run.smithi089.stdout:blocklist 172.21.15.89:0/470853232 expires 2023-06-04T19:31:48.303240+0000 2023-06-03T19:59:34.685 INFO:teuthology.orchestra.run.smithi089.stdout:blocklist 172.21.15.164:6835/9946422 expires 2023-06-04T19:40:01.102939+0000 2023-06-03T19:59:34.686 INFO:teuthology.orchestra.run.smithi089.stdout:blocklist 172.21.15.89:0/811967393 expires 2023-06-04T19:23:23.903333+0000 2023-06-03T19:59:34.686 INFO:teuthology.orchestra.run.smithi089.stdout:blocklist 172.21.15.89:0/3044409358 expires 2023-06-04T19:41:21.936058+0000 2023-06-03T19:59:34.686 INFO:teuthology.orchestra.run.smithi089.stdout:blocklist 172.21.15.164:0/3249546020 expires 2023-06-04T19:40:01.102939+0000 2023-06-03T19:59:34.687 INFO:teuthology.orchestra.run.smithi089.stdout:blocklist 172.21.15.89:0/1198702890 expires 2023-06-04T19:23:23.903333+0000 2023-06-03T19:59:34.687 INFO:teuthology.orchestra.run.smithi089.stdout:blocklist 172.21.15.164:0/1307416622 expires 2023-06-04T19:40:01.102939+0000 2023-06-03T19:59:34.687 INFO:teuthology.orchestra.run.smithi089.stdout:blocklist 172.21.15.89:6801/2009133143 expires 2023-06-04T19:37:38.185918+0000 2023-06-03T19:59:34.687 INFO:teuthology.orchestra.run.smithi089.stdout:blocklist 172.21.15.164:6834/9946422 expires 2023-06-04T19:40:01.102939+0000 2023-06-03T19:59:34.687 INFO:teuthology.orchestra.run.smithi089.stdout:blocklist 172.21.15.89:6800/1487566136 expires 2023-06-04T19:23:23.903333+0000 2023-06-03T19:59:34.688 INFO:teuthology.orchestra.run.smithi089.stdout:blocklist 172.21.15.89:6801/1487566136 expires 2023-06-04T19:23:23.903333+0000 2023-06-03T19:59:34.688 INFO:teuthology.orchestra.run.smithi089.stdout:blocklist 172.21.15.89:0/274274920 expires 2023-06-04T19:23:55.626873+0000 2023-06-03T19:59:34.688 INFO:teuthology.orchestra.run.smithi089.stdout:blocklist 172.21.15.89:0/2392321634 expires 2023-06-04T19:31:48.303240+0000 2023-06-03T19:59:34.689 INFO:teuthology.orchestra.run.smithi089.stdout:blocklist 172.21.15.89:0/101502033 expires 2023-06-04T19:41:21.936058+0000 2023-06-03T19:59:34.689 INFO:teuthology.orchestra.run.smithi089.stdout:blocklist 172.21.15.164:0/3011990159 expires 2023-06-04T19:40:01.102939+0000 2023-06-03T19:59:34.689 INFO:teuthology.orchestra.run.smithi089.stdout:blocklist 172.21.15.89:6801/2256305550 expires 2023-06-04T19:31:48.303240+0000 2023-06-03T19:59:34.690 INFO:teuthology.orchestra.run.smithi089.stdout:blocklist 172.21.15.89:6800/4116589411 expires 2023-06-04T19:23:55.626873+0000 2023-06-03T19:59:34.690 INFO:teuthology.orchestra.run.smithi089.stdout:blocklist 172.21.15.89:0/1591228500 expires 2023-06-04T19:37:38.185918+0000 2023-06-03T19:59:34.690 INFO:teuthology.orchestra.run.smithi089.stdout:blocklist 172.21.15.89:0/949193420 expires 2023-06-04T19:41:21.936058+0000 2023-06-03T19:59:34.690 INFO:teuthology.orchestra.run.smithi089.stdout:blocklist 172.21.15.89:6800/2009133143 expires 2023-06-04T19:37:38.185918+0000 2023-06-03T19:59:34.690 INFO:teuthology.orchestra.run.smithi089.stdout:blocklist 172.21.15.89:0/3513444952 expires 2023-06-04T19:41:21.936058+0000 2023-06-03T19:59:34.690 INFO:teuthology.orchestra.run.smithi089.stdout:blocklist 172.21.15.89:6801/4116589411 expires 2023-06-04T19:23:55.626873+0000 2023-06-03T19:59:35.029 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:59:34 smithi089 bash[75068]: audit 1685822374.6735773 mon.c (mon.2) 369 : audit 0 from='client.? 172.21.15.89:0/1402923439' entity='client.admin' cmd=[{"prefix": "osd dump"}]: dispatch 2023-06-03T19:59:35.029 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:59:34 smithi089 bash[78452]: audit 1685822374.6735773 mon.c (mon.2) 369 : audit 0 from='client.? 172.21.15.89:0/1402923439' entity='client.admin' cmd=[{"prefix": "osd dump"}]: dispatch 2023-06-03T19:59:35.084 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:59:34 smithi164 bash[59462]: audit 1685822374.6735773 mon.c (mon.2) 369 : audit 0 from='client.? 172.21.15.89:0/1402923439' entity='client.admin' cmd=[{"prefix": "osd dump"}]: dispatch 2023-06-03T19:59:35.559 INFO:tasks.cephadm.ceph_manager.ceph:['osd.0 up in weight 1 up_from 86 up_thru 124 down_at 83 last_clean_interval [8,82) [v2:172.21.15.89:6802/2920103307,v1:172.21.15.89:6803/2920103307] [v2:172.21.15.89:6804/2920103307,v1:172.21.15.89:6805/2920103307] exists,up 86063bfe-126e-46e7-93ed-c73813af165b', 'osd.1 up in weight 1 up_from 103 up_thru 124 down_at 100 last_clean_interval [13,99) [v2:172.21.15.89:6810/2092956116,v1:172.21.15.89:6811/2092956116] [v2:172.21.15.89:6812/2092956116,v1:172.21.15.89:6813/2092956116] exists,up 04783a81-6b8c-4657-8446-00efc872b313', 'osd.2 up in weight 1 up_from 114 up_thru 121 down_at 111 last_clean_interval [18,110) [v2:172.21.15.89:6818/1439643284,v1:172.21.15.89:6819/1439643284] [v2:172.21.15.89:6820/1439643284,v1:172.21.15.89:6821/1439643284] exists,up d2d2c641-392a-444f-81bc-84eef6340f30', 'osd.3 up in weight 1 up_from 119 up_thru 124 down_at 116 last_clean_interval [25,115) [v2:172.21.15.89:6826/1558463544,v1:172.21.15.89:6827/1558463544] [v2:172.21.15.89:6828/1558463544,v1:172.21.15.89:6829/1558463544] exists,up b38408eb-3acf-4a41-a314-72e4cd37b1ef', 'osd.4 up in weight 1 up_from 124 up_thru 124 down_at 121 last_clean_interval [30,120) [v2:172.21.15.164:6800/3822759075,v1:172.21.15.164:6801/3822759075] [v2:172.21.15.164:6802/3822759075,v1:172.21.15.164:6803/3822759075] exists,up a94be785-9f54-4ab8-962a-9cf974a5b5e3', 'osd.5 up in weight 1 up_from 36 up_thru 124 down_at 0 last_clean_interval [0,0) [v2:172.21.15.164:6808/3123006405,v1:172.21.15.164:6809/3123006405] [v2:172.21.15.164:6810/3123006405,v1:172.21.15.164:6811/3123006405] exists,up b8a5e8a1-23cc-442b-90aa-19bd0036dd3c', 'osd.6 up in weight 1 up_from 42 up_thru 124 down_at 0 last_clean_interval [0,0) [v2:172.21.15.164:6816/712403290,v1:172.21.15.164:6817/712403290] [v2:172.21.15.164:6818/712403290,v1:172.21.15.164:6819/712403290] exists,up 5c5edd3b-d92c-4f46-94ff-2d5828b2e732', 'osd.7 up in weight 1 up_from 48 up_thru 124 down_at 0 last_clean_interval [0,0) [v2:172.21.15.164:6824/711997991,v1:172.21.15.164:6825/711997991] [v2:172.21.15.164:6826/711997991,v1:172.21.15.164:6827/711997991] exists,up bdff9290-e6c4-407e-9113-eb39696c0e27'] 2023-06-03T19:59:35.560 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph daemon mon.a config show 2023-06-03T19:59:36.029 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:59:35 smithi089 bash[78452]: cluster 1685822374.4075272 mgr.y (mgr.34104) 698 : cluster 0 pgmap v602: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-06-03T19:59:36.029 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:59:35 smithi089 bash[75068]: cluster 1685822374.4075272 mgr.y (mgr.34104) 698 : cluster 0 pgmap v602: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-06-03T19:59:36.084 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:59:35 smithi164 bash[59462]: cluster 1685822374.4075272 mgr.y (mgr.34104) 698 : cluster 0 pgmap v602: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-06-03T19:59:36.528 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 19:59:36 smithi089 bash[71163]: ::ffff:172.21.15.164 - - [03/Jun/2023:19:59:36] "GET /metrics HTTP/1.1" 200 225309 "" "Prometheus/2.33.4" 2023-06-03T19:59:38.029 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:59:37 smithi089 bash[78452]: cluster 1685822376.4082282 mgr.y (mgr.34104) 699 : cluster 0 pgmap v603: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-06-03T19:59:38.029 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:59:37 smithi089 bash[78452]: audit 1685822377.093254 mon.c (mon.2) 370 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T19:59:38.029 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:59:37 smithi089 bash[75068]: cluster 1685822376.4082282 mgr.y (mgr.34104) 699 : cluster 0 pgmap v603: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-06-03T19:59:38.029 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:59:37 smithi089 bash[75068]: audit 1685822377.093254 mon.c (mon.2) 370 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T19:59:38.084 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:59:37 smithi164 bash[59462]: cluster 1685822376.4082282 mgr.y (mgr.34104) 699 : cluster 0 pgmap v603: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-06-03T19:59:38.084 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:59:37 smithi164 bash[59462]: audit 1685822377.093254 mon.c (mon.2) 370 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T19:59:40.028 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:59:39 smithi089 bash[78452]: cluster 1685822378.409325 mgr.y (mgr.34104) 700 : cluster 0 pgmap v604: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-06-03T19:59:40.029 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:59:39 smithi089 bash[75068]: cluster 1685822378.409325 mgr.y (mgr.34104) 700 : cluster 0 pgmap v604: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-06-03T19:59:40.084 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:59:39 smithi164 bash[59462]: cluster 1685822378.409325 mgr.y (mgr.34104) 700 : cluster 0 pgmap v604: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-06-03T19:59:40.379 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/mon.a/config 2023-06-03T19:59:41.895 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:59:41 smithi089 bash[75068]: cluster 1685822380.4099565 mgr.y (mgr.34104) 701 : cluster 0 pgmap v605: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-06-03T19:59:41.895 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:59:41 smithi089 bash[78452]: cluster 1685822380.4099565 mgr.y (mgr.34104) 701 : cluster 0 pgmap v605: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-06-03T19:59:42.084 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:59:41 smithi164 bash[59462]: cluster 1685822380.4099565 mgr.y (mgr.34104) 701 : cluster 0 pgmap v605: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-06-03T19:59:43.901 INFO:teuthology.orchestra.run.smithi089.stdout:{ 2023-06-03T19:59:43.901 INFO:teuthology.orchestra.run.smithi089.stdout: "name": "mon.a", 2023-06-03T19:59:43.901 INFO:teuthology.orchestra.run.smithi089.stdout: "cluster": "ceph", 2023-06-03T19:59:43.901 INFO:teuthology.orchestra.run.smithi089.stdout: "admin_socket": "/var/run/ceph/ceph-mon.a.asok", 2023-06-03T19:59:43.901 INFO:teuthology.orchestra.run.smithi089.stdout: "admin_socket_mode": "", 2023-06-03T19:59:43.902 INFO:teuthology.orchestra.run.smithi089.stdout: "auth_allow_insecure_global_id_reclaim": "false", 2023-06-03T19:59:43.902 INFO:teuthology.orchestra.run.smithi089.stdout: "auth_client_required": "cephx, none", 2023-06-03T19:59:43.902 INFO:teuthology.orchestra.run.smithi089.stdout: "auth_cluster_required": "cephx", 2023-06-03T19:59:43.902 INFO:teuthology.orchestra.run.smithi089.stdout: "auth_debug": "true", 2023-06-03T19:59:43.902 INFO:teuthology.orchestra.run.smithi089.stdout: "auth_expose_insecure_global_id_reclaim": "true", 2023-06-03T19:59:43.902 INFO:teuthology.orchestra.run.smithi089.stdout: "auth_mon_ticket_ttl": "660.000000", 2023-06-03T19:59:43.902 INFO:teuthology.orchestra.run.smithi089.stdout: "auth_service_required": "cephx", 2023-06-03T19:59:43.903 INFO:teuthology.orchestra.run.smithi089.stdout: "auth_service_ticket_ttl": "240.000000", 2023-06-03T19:59:43.903 INFO:teuthology.orchestra.run.smithi089.stdout: "auth_supported": "", 2023-06-03T19:59:43.903 INFO:teuthology.orchestra.run.smithi089.stdout: "bdev_aio": "true", 2023-06-03T19:59:43.903 INFO:teuthology.orchestra.run.smithi089.stdout: "bdev_aio_max_queue_depth": "1024", 2023-06-03T19:59:43.903 INFO:teuthology.orchestra.run.smithi089.stdout: "bdev_aio_poll_ms": "250", 2023-06-03T19:59:43.903 INFO:teuthology.orchestra.run.smithi089.stdout: "bdev_aio_reap_max": "16", 2023-06-03T19:59:43.903 INFO:teuthology.orchestra.run.smithi089.stdout: "bdev_async_discard": "false", 2023-06-03T19:59:43.903 INFO:teuthology.orchestra.run.smithi089.stdout: "bdev_block_size": "4096", 2023-06-03T19:59:43.904 INFO:teuthology.orchestra.run.smithi089.stdout: "bdev_debug_aio": "false", 2023-06-03T19:59:43.904 INFO:teuthology.orchestra.run.smithi089.stdout: "bdev_debug_aio_log_age": "5.000000", 2023-06-03T19:59:43.904 INFO:teuthology.orchestra.run.smithi089.stdout: "bdev_debug_aio_suicide_timeout": "60.000000", 2023-06-03T19:59:43.904 INFO:teuthology.orchestra.run.smithi089.stdout: "bdev_debug_inflight_ios": "false", 2023-06-03T19:59:43.904 INFO:teuthology.orchestra.run.smithi089.stdout: "bdev_enable_discard": "false", 2023-06-03T19:59:43.904 INFO:teuthology.orchestra.run.smithi089.stdout: "bdev_flock_retry": "3", 2023-06-03T19:59:43.905 INFO:teuthology.orchestra.run.smithi089.stdout: "bdev_flock_retry_interval": "0.100000", 2023-06-03T19:59:43.905 INFO:teuthology.orchestra.run.smithi089.stdout: "bdev_inject_crash": "0", 2023-06-03T19:59:43.905 INFO:teuthology.orchestra.run.smithi089.stdout: "bdev_inject_crash_flush_delay": "2", 2023-06-03T19:59:43.906 INFO:teuthology.orchestra.run.smithi089.stdout: "bdev_ioring": "false", 2023-06-03T19:59:43.906 INFO:teuthology.orchestra.run.smithi089.stdout: "bdev_ioring_hipri": "false", 2023-06-03T19:59:43.906 INFO:teuthology.orchestra.run.smithi089.stdout: "bdev_ioring_sqthread_poll": "false", 2023-06-03T19:59:43.906 INFO:teuthology.orchestra.run.smithi089.stdout: "bdev_nvme_unbind_from_kernel": "false", 2023-06-03T19:59:43.906 INFO:teuthology.orchestra.run.smithi089.stdout: "bdev_read_buffer_alignment": "4096", 2023-06-03T19:59:43.906 INFO:teuthology.orchestra.run.smithi089.stdout: "bdev_read_preallocated_huge_buffers": "", 2023-06-03T19:59:43.906 INFO:teuthology.orchestra.run.smithi089.stdout: "bdev_type": "", 2023-06-03T19:59:43.906 INFO:teuthology.orchestra.run.smithi089.stdout: "bluefs_alloc_size": "1048576", 2023-06-03T19:59:43.907 INFO:teuthology.orchestra.run.smithi089.stdout: "bluefs_allocator": "hybrid", 2023-06-03T19:59:43.907 INFO:teuthology.orchestra.run.smithi089.stdout: "bluefs_buffered_io": "true", 2023-06-03T19:59:43.907 INFO:teuthology.orchestra.run.smithi089.stdout: "bluefs_check_for_zeros": "false", 2023-06-03T19:59:43.907 INFO:teuthology.orchestra.run.smithi089.stdout: "bluefs_check_volume_selector_often": "false", 2023-06-03T19:59:43.907 INFO:teuthology.orchestra.run.smithi089.stdout: "bluefs_check_volume_selector_on_umount": "false", 2023-06-03T19:59:43.907 INFO:teuthology.orchestra.run.smithi089.stdout: "bluefs_compact_log_sync": "false", 2023-06-03T19:59:43.907 INFO:teuthology.orchestra.run.smithi089.stdout: "bluefs_failed_shared_alloc_cooldown": "600.000000", 2023-06-03T19:59:43.908 INFO:teuthology.orchestra.run.smithi089.stdout: "bluefs_log_compact_min_ratio": "5.000000", 2023-06-03T19:59:43.908 INFO:teuthology.orchestra.run.smithi089.stdout: "bluefs_log_compact_min_size": "16777216", 2023-06-03T19:59:43.908 INFO:teuthology.orchestra.run.smithi089.stdout: "bluefs_log_replay_check_allocations": "true", 2023-06-03T19:59:43.908 INFO:teuthology.orchestra.run.smithi089.stdout: "bluefs_max_log_runway": "4194304", 2023-06-03T19:59:43.908 INFO:teuthology.orchestra.run.smithi089.stdout: "bluefs_max_prefetch": "1048576", 2023-06-03T19:59:43.908 INFO:teuthology.orchestra.run.smithi089.stdout: "bluefs_min_flush_size": "524288", 2023-06-03T19:59:43.908 INFO:teuthology.orchestra.run.smithi089.stdout: "bluefs_min_log_runway": "1048576", 2023-06-03T19:59:43.909 INFO:teuthology.orchestra.run.smithi089.stdout: "bluefs_replay_recovery": "false", 2023-06-03T19:59:43.909 INFO:teuthology.orchestra.run.smithi089.stdout: "bluefs_replay_recovery_disable_compact": "false", 2023-06-03T19:59:43.909 INFO:teuthology.orchestra.run.smithi089.stdout: "bluefs_shared_alloc_size": "65536", 2023-06-03T19:59:43.909 INFO:teuthology.orchestra.run.smithi089.stdout: "bluefs_sync_write": "false", 2023-06-03T19:59:43.909 INFO:teuthology.orchestra.run.smithi089.stdout: "bluestore_2q_cache_kin_ratio": "0.500000", 2023-06-03T19:59:43.909 INFO:teuthology.orchestra.run.smithi089.stdout: "bluestore_2q_cache_kout_ratio": "0.500000", 2023-06-03T19:59:43.909 INFO:teuthology.orchestra.run.smithi089.stdout: "bluestore_alloc_stats_dump_interval": "86400.000000", 2023-06-03T19:59:43.909 INFO:teuthology.orchestra.run.smithi089.stdout: "bluestore_allocation_from_file": "true", 2023-06-03T19:59:43.910 INFO:teuthology.orchestra.run.smithi089.stdout: "bluestore_allocator": "hybrid", 2023-06-03T19:59:43.910 INFO:teuthology.orchestra.run.smithi089.stdout: "bluestore_avl_alloc_bf_free_pct": "4", 2023-06-03T19:59:43.910 INFO:teuthology.orchestra.run.smithi089.stdout: "bluestore_avl_alloc_bf_threshold": "131072", 2023-06-03T19:59:43.910 INFO:teuthology.orchestra.run.smithi089.stdout: "bluestore_avl_alloc_ff_max_search_bytes": "16777216", 2023-06-03T19:59:43.910 INFO:teuthology.orchestra.run.smithi089.stdout: "bluestore_avl_alloc_ff_max_search_count": "100", 2023-06-03T19:59:43.910 INFO:teuthology.orchestra.run.smithi089.stdout: "bluestore_bitmapallocator_blocks_per_zone": "1024", 2023-06-03T19:59:43.910 INFO:teuthology.orchestra.run.smithi089.stdout: "bluestore_bitmapallocator_span_size": "1024", 2023-06-03T19:59:43.910 INFO:teuthology.orchestra.run.smithi089.stdout: "bluestore_blobid_prealloc": "10240", 2023-06-03T19:59:43.911 INFO:teuthology.orchestra.run.smithi089.stdout: "bluestore_block_create": "true", 2023-06-03T19:59:43.911 INFO:teuthology.orchestra.run.smithi089.stdout: "bluestore_block_db_create": "false", 2023-06-03T19:59:43.911 INFO:teuthology.orchestra.run.smithi089.stdout: "bluestore_block_db_path": "", 2023-06-03T19:59:43.911 INFO:teuthology.orchestra.run.smithi089.stdout: "bluestore_block_db_size": "0", 2023-06-03T19:59:43.911 INFO:teuthology.orchestra.run.smithi089.stdout: "bluestore_block_path": "", 2023-06-03T19:59:43.911 INFO:teuthology.orchestra.run.smithi089.stdout: "bluestore_block_preallocate_file": "false", 2023-06-03T19:59:43.911 INFO:teuthology.orchestra.run.smithi089.stdout: "bluestore_block_size": "107374182400", 2023-06-03T19:59:43.911 INFO:teuthology.orchestra.run.smithi089.stdout: "bluestore_block_wal_create": "false", 2023-06-03T19:59:43.912 INFO:teuthology.orchestra.run.smithi089.stdout: "bluestore_block_wal_path": "", 2023-06-03T19:59:43.912 INFO:teuthology.orchestra.run.smithi089.stdout: "bluestore_block_wal_size": "100663296", 2023-06-03T19:59:43.912 INFO:teuthology.orchestra.run.smithi089.stdout: "bluestore_bluefs": "true", 2023-06-03T19:59:43.912 INFO:teuthology.orchestra.run.smithi089.stdout: "bluestore_bluefs_alloc_failure_dump_interval": "0.000000", 2023-06-03T19:59:43.912 INFO:teuthology.orchestra.run.smithi089.stdout: "bluestore_bluefs_env_mirror": "false", 2023-06-03T19:59:43.912 INFO:teuthology.orchestra.run.smithi089.stdout: "bluestore_bluefs_max_free": "10737418240", 2023-06-03T19:59:43.912 INFO:teuthology.orchestra.run.smithi089.stdout: "bluestore_cache_age_bin_interval": "1.000000", 2023-06-03T19:59:43.912 INFO:teuthology.orchestra.run.smithi089.stdout: "bluestore_cache_age_bins_data": "1 2 6 24 120 720 0 0 0 0", 2023-06-03T19:59:43.913 INFO:teuthology.orchestra.run.smithi089.stdout: "bluestore_cache_age_bins_kv": "1 2 6 24 120 720 0 0 0 0", 2023-06-03T19:59:43.913 INFO:teuthology.orchestra.run.smithi089.stdout: "bluestore_cache_age_bins_kv_onode": "0 0 0 0 0 0 0 0 0 720", 2023-06-03T19:59:43.913 INFO:teuthology.orchestra.run.smithi089.stdout: "bluestore_cache_age_bins_meta": "1 2 6 24 120 720 0 0 0 0", 2023-06-03T19:59:43.913 INFO:teuthology.orchestra.run.smithi089.stdout: "bluestore_cache_autotune": "true", 2023-06-03T19:59:43.913 INFO:teuthology.orchestra.run.smithi089.stdout: "bluestore_cache_autotune_interval": "5.000000", 2023-06-03T19:59:43.913 INFO:teuthology.orchestra.run.smithi089.stdout: "bluestore_cache_kv_onode_ratio": "0.040000", 2023-06-03T19:59:43.913 INFO:teuthology.orchestra.run.smithi089.stdout: "bluestore_cache_kv_ratio": "0.450000", 2023-06-03T19:59:43.914 INFO:teuthology.orchestra.run.smithi089.stdout: "bluestore_cache_meta_ratio": "0.450000", 2023-06-03T19:59:43.914 INFO:teuthology.orchestra.run.smithi089.stdout: "bluestore_cache_size": "0", 2023-06-03T19:59:43.914 INFO:teuthology.orchestra.run.smithi089.stdout: "bluestore_cache_size_hdd": "1073741824", 2023-06-03T19:59:43.914 INFO:teuthology.orchestra.run.smithi089.stdout: "bluestore_cache_size_ssd": "3221225472", 2023-06-03T19:59:43.914 INFO:teuthology.orchestra.run.smithi089.stdout: "bluestore_cache_trim_interval": "0.050000", 2023-06-03T19:59:43.914 INFO:teuthology.orchestra.run.smithi089.stdout: "bluestore_cache_trim_max_skip_pinned": "1000", 2023-06-03T19:59:43.914 INFO:teuthology.orchestra.run.smithi089.stdout: "bluestore_cache_type": "2q", 2023-06-03T19:59:43.914 INFO:teuthology.orchestra.run.smithi089.stdout: "bluestore_cleaner_sleep_interval": "5.000000", 2023-06-03T19:59:43.915 INFO:teuthology.orchestra.run.smithi089.stdout: "bluestore_clone_cow": "true", 2023-06-03T19:59:43.915 INFO:teuthology.orchestra.run.smithi089.stdout: "bluestore_compression_algorithm": "snappy", 2023-06-03T19:59:43.915 INFO:teuthology.orchestra.run.smithi089.stdout: "bluestore_compression_max_blob_size": "0", 2023-06-03T19:59:43.915 INFO:teuthology.orchestra.run.smithi089.stdout: "bluestore_compression_max_blob_size_hdd": "65536", 2023-06-03T19:59:43.915 INFO:teuthology.orchestra.run.smithi089.stdout: "bluestore_compression_max_blob_size_ssd": "65536", 2023-06-03T19:59:43.915 INFO:teuthology.orchestra.run.smithi089.stdout: "bluestore_compression_min_blob_size": "0", 2023-06-03T19:59:43.915 INFO:teuthology.orchestra.run.smithi089.stdout: "bluestore_compression_min_blob_size_hdd": "8192", 2023-06-03T19:59:43.915 INFO:teuthology.orchestra.run.smithi089.stdout: "bluestore_compression_min_blob_size_ssd": "65536", 2023-06-03T19:59:43.916 INFO:teuthology.orchestra.run.smithi089.stdout: "bluestore_compression_mode": "none", 2023-06-03T19:59:43.916 INFO:teuthology.orchestra.run.smithi089.stdout: "bluestore_compression_required_ratio": "0.875000", 2023-06-03T19:59:43.916 INFO:teuthology.orchestra.run.smithi089.stdout: "bluestore_csum_type": "crc32c", 2023-06-03T19:59:43.916 INFO:teuthology.orchestra.run.smithi089.stdout: "bluestore_debug_enforce_settings": "default", 2023-06-03T19:59:43.916 INFO:teuthology.orchestra.run.smithi089.stdout: "bluestore_debug_freelist": "false", 2023-06-03T19:59:43.916 INFO:teuthology.orchestra.run.smithi089.stdout: "bluestore_debug_fsck_abort": "false", 2023-06-03T19:59:43.916 INFO:teuthology.orchestra.run.smithi089.stdout: "bluestore_debug_inject_allocation_from_file_failure": "0.000000", 2023-06-03T19:59:43.916 INFO:teuthology.orchestra.run.smithi089.stdout: "bluestore_debug_inject_bug21040": "false", 2023-06-03T19:59:43.917 INFO:teuthology.orchestra.run.smithi089.stdout: "bluestore_debug_inject_csum_err_probability": "0.000000", 2023-06-03T19:59:43.917 INFO:teuthology.orchestra.run.smithi089.stdout: "bluestore_debug_inject_read_err": "false", 2023-06-03T19:59:43.917 INFO:teuthology.orchestra.run.smithi089.stdout: "bluestore_debug_legacy_omap": "false", 2023-06-03T19:59:43.917 INFO:teuthology.orchestra.run.smithi089.stdout: "bluestore_debug_no_reuse_blocks": "false", 2023-06-03T19:59:43.917 INFO:teuthology.orchestra.run.smithi089.stdout: "bluestore_debug_omit_block_device_write": "false", 2023-06-03T19:59:43.917 INFO:teuthology.orchestra.run.smithi089.stdout: "bluestore_debug_omit_kv_commit": "false", 2023-06-03T19:59:43.917 INFO:teuthology.orchestra.run.smithi089.stdout: "bluestore_debug_permit_any_bdev_label": "false", 2023-06-03T19:59:43.917 INFO:teuthology.orchestra.run.smithi089.stdout: "bluestore_debug_prefill": "0.000000", 2023-06-03T19:59:43.918 INFO:teuthology.orchestra.run.smithi089.stdout: "bluestore_debug_prefragment_max": "1048576", 2023-06-03T19:59:43.918 INFO:teuthology.orchestra.run.smithi089.stdout: "bluestore_debug_random_read_err": "0.000000", 2023-06-03T19:59:43.918 INFO:teuthology.orchestra.run.smithi089.stdout: "bluestore_debug_randomize_serial_transaction": "0", 2023-06-03T19:59:43.918 INFO:teuthology.orchestra.run.smithi089.stdout: "bluestore_debug_small_allocations": "0", 2023-06-03T19:59:43.918 INFO:teuthology.orchestra.run.smithi089.stdout: "bluestore_debug_too_many_blobs_threshold": "24576", 2023-06-03T19:59:43.918 INFO:teuthology.orchestra.run.smithi089.stdout: "bluestore_default_buffered_read": "true", 2023-06-03T19:59:43.918 INFO:teuthology.orchestra.run.smithi089.stdout: "bluestore_default_buffered_write": "false", 2023-06-03T19:59:43.918 INFO:teuthology.orchestra.run.smithi089.stdout: "bluestore_deferred_batch_ops": "0", 2023-06-03T19:59:43.919 INFO:teuthology.orchestra.run.smithi089.stdout: "bluestore_deferred_batch_ops_hdd": "64", 2023-06-03T19:59:43.919 INFO:teuthology.orchestra.run.smithi089.stdout: "bluestore_deferred_batch_ops_ssd": "16", 2023-06-03T19:59:43.919 INFO:teuthology.orchestra.run.smithi089.stdout: "bluestore_extent_map_inline_shard_prealloc_size": "256", 2023-06-03T19:59:43.919 INFO:teuthology.orchestra.run.smithi089.stdout: "bluestore_extent_map_shard_max_size": "1200", 2023-06-03T19:59:43.919 INFO:teuthology.orchestra.run.smithi089.stdout: "bluestore_extent_map_shard_min_size": "150", 2023-06-03T19:59:43.919 INFO:teuthology.orchestra.run.smithi089.stdout: "bluestore_extent_map_shard_target_size": "500", 2023-06-03T19:59:43.919 INFO:teuthology.orchestra.run.smithi089.stdout: "bluestore_extent_map_shard_target_size_slop": "0.200000", 2023-06-03T19:59:43.919 INFO:teuthology.orchestra.run.smithi089.stdout: "bluestore_fail_eio": "false", 2023-06-03T19:59:43.920 INFO:teuthology.orchestra.run.smithi089.stdout: "bluestore_freelist_blocks_per_key": "128", 2023-06-03T19:59:43.920 INFO:teuthology.orchestra.run.smithi089.stdout: "bluestore_fsck_error_on_no_per_pg_omap": "false", 2023-06-03T19:59:43.920 INFO:teuthology.orchestra.run.smithi089.stdout: "bluestore_fsck_error_on_no_per_pool_omap": "false", 2023-06-03T19:59:43.920 INFO:teuthology.orchestra.run.smithi089.stdout: "bluestore_fsck_error_on_no_per_pool_stats": "false", 2023-06-03T19:59:43.920 INFO:teuthology.orchestra.run.smithi089.stdout: "bluestore_fsck_on_mkfs": "true", 2023-06-03T19:59:43.920 INFO:teuthology.orchestra.run.smithi089.stdout: "bluestore_fsck_on_mkfs_deep": "false", 2023-06-03T19:59:43.920 INFO:teuthology.orchestra.run.smithi089.stdout: "bluestore_fsck_on_mount": "false", 2023-06-03T19:59:43.921 INFO:teuthology.orchestra.run.smithi089.stdout: "bluestore_fsck_on_mount_deep": "false", 2023-06-03T19:59:43.921 INFO:teuthology.orchestra.run.smithi089.stdout: "bluestore_fsck_on_umount": "false", 2023-06-03T19:59:43.921 INFO:teuthology.orchestra.run.smithi089.stdout: "bluestore_fsck_on_umount_deep": "false", 2023-06-03T19:59:43.921 INFO:teuthology.orchestra.run.smithi089.stdout: "bluestore_fsck_quick_fix_on_mount": "false", 2023-06-03T19:59:43.921 INFO:teuthology.orchestra.run.smithi089.stdout: "bluestore_fsck_quick_fix_threads": "2", 2023-06-03T19:59:43.921 INFO:teuthology.orchestra.run.smithi089.stdout: "bluestore_fsck_read_bytes_cap": "67108864", 2023-06-03T19:59:43.921 INFO:teuthology.orchestra.run.smithi089.stdout: "bluestore_fsck_shared_blob_tracker_size": "0.031250", 2023-06-03T19:59:43.921 INFO:teuthology.orchestra.run.smithi089.stdout: "bluestore_gc_enable_blob_threshold": "0", 2023-06-03T19:59:43.922 INFO:teuthology.orchestra.run.smithi089.stdout: "bluestore_gc_enable_total_threshold": "0", 2023-06-03T19:59:43.922 INFO:teuthology.orchestra.run.smithi089.stdout: "bluestore_hybrid_alloc_mem_cap": "67108864", 2023-06-03T19:59:43.922 INFO:teuthology.orchestra.run.smithi089.stdout: "bluestore_ignore_data_csum": "false", 2023-06-03T19:59:43.922 INFO:teuthology.orchestra.run.smithi089.stdout: "bluestore_kv_sync_util_logging_s": "10.000000", 2023-06-03T19:59:43.922 INFO:teuthology.orchestra.run.smithi089.stdout: "bluestore_kvbackend": "rocksdb", 2023-06-03T19:59:43.922 INFO:teuthology.orchestra.run.smithi089.stdout: "bluestore_log_collection_list_age": "60.000000", 2023-06-03T19:59:43.922 INFO:teuthology.orchestra.run.smithi089.stdout: "bluestore_log_omap_iterator_age": "5.000000", 2023-06-03T19:59:43.922 INFO:teuthology.orchestra.run.smithi089.stdout: "bluestore_log_op_age": "5.000000", 2023-06-03T19:59:43.923 INFO:teuthology.orchestra.run.smithi089.stdout: "bluestore_max_alloc_size": "0", 2023-06-03T19:59:43.923 INFO:teuthology.orchestra.run.smithi089.stdout: "bluestore_max_blob_size": "0", 2023-06-03T19:59:43.923 INFO:teuthology.orchestra.run.smithi089.stdout: "bluestore_max_blob_size_hdd": "65536", 2023-06-03T19:59:43.923 INFO:teuthology.orchestra.run.smithi089.stdout: "bluestore_max_blob_size_ssd": "65536", 2023-06-03T19:59:43.923 INFO:teuthology.orchestra.run.smithi089.stdout: "bluestore_max_defer_interval": "3.000000", 2023-06-03T19:59:43.923 INFO:teuthology.orchestra.run.smithi089.stdout: "bluestore_max_deferred_txc": "32", 2023-06-03T19:59:43.923 INFO:teuthology.orchestra.run.smithi089.stdout: "bluestore_min_alloc_size": "0", 2023-06-03T19:59:43.923 INFO:teuthology.orchestra.run.smithi089.stdout: "bluestore_min_alloc_size_hdd": "4096", 2023-06-03T19:59:43.924 INFO:teuthology.orchestra.run.smithi089.stdout: "bluestore_min_alloc_size_ssd": "4096", 2023-06-03T19:59:43.924 INFO:teuthology.orchestra.run.smithi089.stdout: "bluestore_nid_prealloc": "1024", 2023-06-03T19:59:43.924 INFO:teuthology.orchestra.run.smithi089.stdout: "bluestore_prefer_deferred_size": "0", 2023-06-03T19:59:43.924 INFO:teuthology.orchestra.run.smithi089.stdout: "bluestore_prefer_deferred_size_hdd": "65536", 2023-06-03T19:59:43.924 INFO:teuthology.orchestra.run.smithi089.stdout: "bluestore_prefer_deferred_size_ssd": "0", 2023-06-03T19:59:43.924 INFO:teuthology.orchestra.run.smithi089.stdout: "bluestore_qfsck_on_mount": "true", 2023-06-03T19:59:43.924 INFO:teuthology.orchestra.run.smithi089.stdout: "bluestore_retry_disk_reads": "3", 2023-06-03T19:59:43.925 INFO:teuthology.orchestra.run.smithi089.stdout: "bluestore_rocksdb_cf": "true", 2023-06-03T19:59:43.925 INFO:teuthology.orchestra.run.smithi089.stdout: "bluestore_rocksdb_cfs": "m(3) p(3,0-12) O(3,0-13)=block_cache={type=binned_lru} L P", 2023-06-03T19:59:43.925 INFO:teuthology.orchestra.run.smithi089.stdout: "bluestore_rocksdb_options": "compression=kNoCompression,max_write_buffer_number=128,min_write_buffer_number_to_merge=16,compaction_style=kCompactionStyleLevel,write_buffer_size=8388608,max_background_jobs=4,level0_file_num_compaction_trigger=8,max_bytes_for_level_base=1073741824,max_bytes_for_level_multiplier=8,compaction_readahead_size=2MB,max_total_wal_size=1073741824,writable_file_max_buffer_size=0,ttl=21600", 2023-06-03T19:59:43.925 INFO:teuthology.orchestra.run.smithi089.stdout: "bluestore_rocksdb_options_annex": "", 2023-06-03T19:59:43.925 INFO:teuthology.orchestra.run.smithi089.stdout: "bluestore_spdk_coremask": "0x1", 2023-06-03T19:59:43.925 INFO:teuthology.orchestra.run.smithi089.stdout: "bluestore_spdk_io_sleep": "5", 2023-06-03T19:59:43.925 INFO:teuthology.orchestra.run.smithi089.stdout: "bluestore_spdk_max_io_completion": "0", 2023-06-03T19:59:43.925 INFO:teuthology.orchestra.run.smithi089.stdout: "bluestore_spdk_mem": "512", 2023-06-03T19:59:43.926 INFO:teuthology.orchestra.run.smithi089.stdout: "bluestore_sync_submit_transaction": "false", 2023-06-03T19:59:43.926 INFO:teuthology.orchestra.run.smithi089.stdout: "bluestore_throttle_bytes": "67108864", 2023-06-03T19:59:43.926 INFO:teuthology.orchestra.run.smithi089.stdout: "bluestore_throttle_cost_per_io": "0", 2023-06-03T19:59:43.926 INFO:teuthology.orchestra.run.smithi089.stdout: "bluestore_throttle_cost_per_io_hdd": "670000", 2023-06-03T19:59:43.926 INFO:teuthology.orchestra.run.smithi089.stdout: "bluestore_throttle_cost_per_io_ssd": "4000", 2023-06-03T19:59:43.926 INFO:teuthology.orchestra.run.smithi089.stdout: "bluestore_throttle_deferred_bytes": "134217728", 2023-06-03T19:59:43.926 INFO:teuthology.orchestra.run.smithi089.stdout: "bluestore_throttle_trace_rate": "0.000000", 2023-06-03T19:59:43.926 INFO:teuthology.orchestra.run.smithi089.stdout: "bluestore_tracing": "false", 2023-06-03T19:59:43.927 INFO:teuthology.orchestra.run.smithi089.stdout: "bluestore_use_optimal_io_size_for_min_alloc_size": "false", 2023-06-03T19:59:43.927 INFO:teuthology.orchestra.run.smithi089.stdout: "bluestore_volume_selection_policy": "use_some_extra", 2023-06-03T19:59:43.927 INFO:teuthology.orchestra.run.smithi089.stdout: "bluestore_volume_selection_reserved": "0", 2023-06-03T19:59:43.927 INFO:teuthology.orchestra.run.smithi089.stdout: "bluestore_volume_selection_reserved_factor": "2.000000", 2023-06-03T19:59:43.927 INFO:teuthology.orchestra.run.smithi089.stdout: "bluestore_warn_on_bluefs_spillover": "true", 2023-06-03T19:59:43.927 INFO:teuthology.orchestra.run.smithi089.stdout: "bluestore_warn_on_legacy_statfs": "true", 2023-06-03T19:59:43.927 INFO:teuthology.orchestra.run.smithi089.stdout: "bluestore_warn_on_no_per_pg_omap": "false", 2023-06-03T19:59:43.928 INFO:teuthology.orchestra.run.smithi089.stdout: "bluestore_warn_on_no_per_pool_omap": "true", 2023-06-03T19:59:43.928 INFO:teuthology.orchestra.run.smithi089.stdout: "bluestore_warn_on_spurious_read_errors": "true", 2023-06-03T19:59:43.928 INFO:teuthology.orchestra.run.smithi089.stdout: "bluestore_zero_block_detection": "false", 2023-06-03T19:59:43.928 INFO:teuthology.orchestra.run.smithi089.stdout: "cephadm_path": "/usr/sbin/cephadm", 2023-06-03T19:59:43.928 INFO:teuthology.orchestra.run.smithi089.stdout: "cephfs_mirror_action_update_interval": "2", 2023-06-03T19:59:43.928 INFO:teuthology.orchestra.run.smithi089.stdout: "cephfs_mirror_directory_scan_interval": "10", 2023-06-03T19:59:43.928 INFO:teuthology.orchestra.run.smithi089.stdout: "cephfs_mirror_max_concurrent_directory_syncs": "3", 2023-06-03T19:59:43.928 INFO:teuthology.orchestra.run.smithi089.stdout: "cephfs_mirror_max_consecutive_failures_per_directory": "10", 2023-06-03T19:59:43.929 INFO:teuthology.orchestra.run.smithi089.stdout: "cephfs_mirror_max_snapshot_sync_per_cycle": "3", 2023-06-03T19:59:43.929 INFO:teuthology.orchestra.run.smithi089.stdout: "cephfs_mirror_mount_timeout": "10", 2023-06-03T19:59:43.929 INFO:teuthology.orchestra.run.smithi089.stdout: "cephfs_mirror_restart_mirror_on_blocklist_interval": "30", 2023-06-03T19:59:43.929 INFO:teuthology.orchestra.run.smithi089.stdout: "cephfs_mirror_restart_mirror_on_failure_interval": "20", 2023-06-03T19:59:43.929 INFO:teuthology.orchestra.run.smithi089.stdout: "cephfs_mirror_retry_failed_directories_interval": "60", 2023-06-03T19:59:43.929 INFO:teuthology.orchestra.run.smithi089.stdout: "cephsqlite_blocklist_dead_locker": "true", 2023-06-03T19:59:43.929 INFO:teuthology.orchestra.run.smithi089.stdout: "cephsqlite_lock_renewal_interval": "2000", 2023-06-03T19:59:43.929 INFO:teuthology.orchestra.run.smithi089.stdout: "cephsqlite_lock_renewal_timeout": "30000", 2023-06-03T19:59:43.930 INFO:teuthology.orchestra.run.smithi089.stdout: "cephx_cluster_require_signatures": "false", 2023-06-03T19:59:43.930 INFO:teuthology.orchestra.run.smithi089.stdout: "cephx_cluster_require_version": "2", 2023-06-03T19:59:43.930 INFO:teuthology.orchestra.run.smithi089.stdout: "cephx_require_signatures": "false", 2023-06-03T19:59:43.930 INFO:teuthology.orchestra.run.smithi089.stdout: "cephx_require_version": "2", 2023-06-03T19:59:43.930 INFO:teuthology.orchestra.run.smithi089.stdout: "cephx_service_require_signatures": "false", 2023-06-03T19:59:43.930 INFO:teuthology.orchestra.run.smithi089.stdout: "cephx_service_require_version": "2", 2023-06-03T19:59:43.930 INFO:teuthology.orchestra.run.smithi089.stdout: "cephx_sign_messages": "true", 2023-06-03T19:59:43.930 INFO:teuthology.orchestra.run.smithi089.stdout: "chdir": "", 2023-06-03T19:59:43.931 INFO:teuthology.orchestra.run.smithi089.stdout: "client_acl_type": "", 2023-06-03T19:59:43.931 INFO:teuthology.orchestra.run.smithi089.stdout: "client_asio_thread_count": "2", 2023-06-03T19:59:43.931 INFO:teuthology.orchestra.run.smithi089.stdout: "client_cache_mid": "0.750000", 2023-06-03T19:59:43.931 INFO:teuthology.orchestra.run.smithi089.stdout: "client_cache_size": "16384", 2023-06-03T19:59:43.931 INFO:teuthology.orchestra.run.smithi089.stdout: "client_caps_release_delay": "5", 2023-06-03T19:59:43.931 INFO:teuthology.orchestra.run.smithi089.stdout: "client_check_pool_perm": "true", 2023-06-03T19:59:43.931 INFO:teuthology.orchestra.run.smithi089.stdout: "client_collect_and_send_global_metrics": "false", 2023-06-03T19:59:43.932 INFO:teuthology.orchestra.run.smithi089.stdout: "client_debug_force_sync_read": "false", 2023-06-03T19:59:43.932 INFO:teuthology.orchestra.run.smithi089.stdout: "client_debug_getattr_caps": "false", 2023-06-03T19:59:43.932 INFO:teuthology.orchestra.run.smithi089.stdout: "client_debug_inject_tick_delay": "0", 2023-06-03T19:59:43.932 INFO:teuthology.orchestra.run.smithi089.stdout: "client_die_on_failed_dentry_invalidate": "true", 2023-06-03T19:59:43.932 INFO:teuthology.orchestra.run.smithi089.stdout: "client_die_on_failed_remount": "false", 2023-06-03T19:59:43.932 INFO:teuthology.orchestra.run.smithi089.stdout: "client_dirsize_rbytes": "true", 2023-06-03T19:59:43.932 INFO:teuthology.orchestra.run.smithi089.stdout: "client_force_lazyio": "false", 2023-06-03T19:59:43.932 INFO:teuthology.orchestra.run.smithi089.stdout: "client_fs": "", 2023-06-03T19:59:43.933 INFO:teuthology.orchestra.run.smithi089.stdout: "client_inject_fixed_oldest_tid": "false", 2023-06-03T19:59:43.933 INFO:teuthology.orchestra.run.smithi089.stdout: "client_inject_release_failure": "false", 2023-06-03T19:59:43.933 INFO:teuthology.orchestra.run.smithi089.stdout: "client_max_inline_size": "4096", 2023-06-03T19:59:43.933 INFO:teuthology.orchestra.run.smithi089.stdout: "client_max_retries_on_remount_failure": "5", 2023-06-03T19:59:43.933 INFO:teuthology.orchestra.run.smithi089.stdout: "client_mds_namespace": "", 2023-06-03T19:59:43.933 INFO:teuthology.orchestra.run.smithi089.stdout: "client_metadata": "", 2023-06-03T19:59:43.933 INFO:teuthology.orchestra.run.smithi089.stdout: "client_mount_gid": "-1", 2023-06-03T19:59:43.933 INFO:teuthology.orchestra.run.smithi089.stdout: "client_mount_timeout": "300", 2023-06-03T19:59:43.934 INFO:teuthology.orchestra.run.smithi089.stdout: "client_mount_uid": "-1", 2023-06-03T19:59:43.934 INFO:teuthology.orchestra.run.smithi089.stdout: "client_mountpoint": "/", 2023-06-03T19:59:43.934 INFO:teuthology.orchestra.run.smithi089.stdout: "client_notify_timeout": "10", 2023-06-03T19:59:43.934 INFO:teuthology.orchestra.run.smithi089.stdout: "client_oc": "true", 2023-06-03T19:59:43.934 INFO:teuthology.orchestra.run.smithi089.stdout: "client_oc_max_dirty": "104857600", 2023-06-03T19:59:43.934 INFO:teuthology.orchestra.run.smithi089.stdout: "client_oc_max_dirty_age": "5.000000", 2023-06-03T19:59:43.934 INFO:teuthology.orchestra.run.smithi089.stdout: "client_oc_max_objects": "1000", 2023-06-03T19:59:43.934 INFO:teuthology.orchestra.run.smithi089.stdout: "client_oc_size": "209715200", 2023-06-03T19:59:43.935 INFO:teuthology.orchestra.run.smithi089.stdout: "client_oc_target_dirty": "8388608", 2023-06-03T19:59:43.935 INFO:teuthology.orchestra.run.smithi089.stdout: "client_permissions": "true", 2023-06-03T19:59:43.935 INFO:teuthology.orchestra.run.smithi089.stdout: "client_quota": "true", 2023-06-03T19:59:43.935 INFO:teuthology.orchestra.run.smithi089.stdout: "client_quota_df": "true", 2023-06-03T19:59:43.935 INFO:teuthology.orchestra.run.smithi089.stdout: "client_readahead_max_bytes": "0", 2023-06-03T19:59:43.935 INFO:teuthology.orchestra.run.smithi089.stdout: "client_readahead_max_periods": "4", 2023-06-03T19:59:43.935 INFO:teuthology.orchestra.run.smithi089.stdout: "client_readahead_min": "131072", 2023-06-03T19:59:43.935 INFO:teuthology.orchestra.run.smithi089.stdout: "client_reconnect_stale": "false", 2023-06-03T19:59:43.936 INFO:teuthology.orchestra.run.smithi089.stdout: "client_shutdown_timeout": "30", 2023-06-03T19:59:43.936 INFO:teuthology.orchestra.run.smithi089.stdout: "client_snapdir": ".snap", 2023-06-03T19:59:43.936 INFO:teuthology.orchestra.run.smithi089.stdout: "client_tick_interval": "1", 2023-06-03T19:59:43.936 INFO:teuthology.orchestra.run.smithi089.stdout: "client_trace": "", 2023-06-03T19:59:43.936 INFO:teuthology.orchestra.run.smithi089.stdout: "client_try_dentry_invalidate": "false", 2023-06-03T19:59:43.936 INFO:teuthology.orchestra.run.smithi089.stdout: "client_use_faked_inos": "false", 2023-06-03T19:59:43.936 INFO:teuthology.orchestra.run.smithi089.stdout: "client_use_random_mds": "false", 2023-06-03T19:59:43.937 INFO:teuthology.orchestra.run.smithi089.stdout: "clog_to_graylog": "false", 2023-06-03T19:59:43.937 INFO:teuthology.orchestra.run.smithi089.stdout: "clog_to_graylog_host": "127.0.0.1", 2023-06-03T19:59:43.937 INFO:teuthology.orchestra.run.smithi089.stdout: "clog_to_graylog_port": "12201", 2023-06-03T19:59:43.937 INFO:teuthology.orchestra.run.smithi089.stdout: "clog_to_monitors": "default=true", 2023-06-03T19:59:43.937 INFO:teuthology.orchestra.run.smithi089.stdout: "clog_to_syslog": "false", 2023-06-03T19:59:43.937 INFO:teuthology.orchestra.run.smithi089.stdout: "clog_to_syslog_facility": "default=daemon audit=local0", 2023-06-03T19:59:43.937 INFO:teuthology.orchestra.run.smithi089.stdout: "clog_to_syslog_level": "info", 2023-06-03T19:59:43.937 INFO:teuthology.orchestra.run.smithi089.stdout: "cluster_addr": "-", 2023-06-03T19:59:43.938 INFO:teuthology.orchestra.run.smithi089.stdout: "cluster_network": "", 2023-06-03T19:59:43.938 INFO:teuthology.orchestra.run.smithi089.stdout: "cluster_network_interface": "", 2023-06-03T19:59:43.938 INFO:teuthology.orchestra.run.smithi089.stdout: "compressor_zlib_isal": "false", 2023-06-03T19:59:43.938 INFO:teuthology.orchestra.run.smithi089.stdout: "compressor_zlib_level": "5", 2023-06-03T19:59:43.938 INFO:teuthology.orchestra.run.smithi089.stdout: "compressor_zlib_winsize": "-15", 2023-06-03T19:59:43.938 INFO:teuthology.orchestra.run.smithi089.stdout: "compressor_zstd_level": "1", 2023-06-03T19:59:43.938 INFO:teuthology.orchestra.run.smithi089.stdout: "container_image": "quay.ceph.io/ceph-ci/ceph@sha256:00b1b6d689278b4f97ff790045c8c51d8ca419ac7ee2ebdf11008f2e810a5c78", 2023-06-03T19:59:43.938 INFO:teuthology.orchestra.run.smithi089.stdout: "crash_dir": "/var/lib/ceph/crash", 2023-06-03T19:59:43.939 INFO:teuthology.orchestra.run.smithi089.stdout: "crimson_alien_op_num_threads": "6", 2023-06-03T19:59:43.939 INFO:teuthology.orchestra.run.smithi089.stdout: "crimson_alien_thread_cpu_cores": "", 2023-06-03T19:59:43.939 INFO:teuthology.orchestra.run.smithi089.stdout: "crimson_osd_obc_lru_size": "10", 2023-06-03T19:59:43.939 INFO:teuthology.orchestra.run.smithi089.stdout: "crimson_osd_scheduler_concurrency": "0", 2023-06-03T19:59:43.939 INFO:teuthology.orchestra.run.smithi089.stdout: "crimson_seastar_smp": "1", 2023-06-03T19:59:43.939 INFO:teuthology.orchestra.run.smithi089.stdout: "crush_location": "", 2023-06-03T19:59:43.939 INFO:teuthology.orchestra.run.smithi089.stdout: "crush_location_hook": "", 2023-06-03T19:59:43.939 INFO:teuthology.orchestra.run.smithi089.stdout: "crush_location_hook_timeout": "10", 2023-06-03T19:59:43.940 INFO:teuthology.orchestra.run.smithi089.stdout: "daemonize": "false", 2023-06-03T19:59:43.940 INFO:teuthology.orchestra.run.smithi089.stdout: "daos_pool": "tank", 2023-06-03T19:59:43.940 INFO:teuthology.orchestra.run.smithi089.stdout: "dbstore_config_uri": "file:/var/lib/ceph/radosgw/dbstore-config.db", 2023-06-03T19:59:43.940 INFO:teuthology.orchestra.run.smithi089.stdout: "dbstore_db_dir": "/var/lib/ceph/radosgw", 2023-06-03T19:59:43.940 INFO:teuthology.orchestra.run.smithi089.stdout: "dbstore_db_name_prefix": "dbstore", 2023-06-03T19:59:43.940 INFO:teuthology.orchestra.run.smithi089.stdout: "debug_alienstore": "0/5", 2023-06-03T19:59:43.940 INFO:teuthology.orchestra.run.smithi089.stdout: "debug_allow_any_pool_priority": "false", 2023-06-03T19:59:43.940 INFO:teuthology.orchestra.run.smithi089.stdout: "debug_asok": "1/5", 2023-06-03T19:59:43.941 INFO:teuthology.orchestra.run.smithi089.stdout: "debug_asok_assert_abort": "false", 2023-06-03T19:59:43.941 INFO:teuthology.orchestra.run.smithi089.stdout: "debug_asserts_on_shutdown": "true", 2023-06-03T19:59:43.941 INFO:teuthology.orchestra.run.smithi089.stdout: "debug_auth": "1/5", 2023-06-03T19:59:43.941 INFO:teuthology.orchestra.run.smithi089.stdout: "debug_bdev": "1/3", 2023-06-03T19:59:43.941 INFO:teuthology.orchestra.run.smithi089.stdout: "debug_bluefs": "1/5", 2023-06-03T19:59:43.941 INFO:teuthology.orchestra.run.smithi089.stdout: "debug_bluestore": "1/5", 2023-06-03T19:59:43.941 INFO:teuthology.orchestra.run.smithi089.stdout: "debug_buffer": "0/1", 2023-06-03T19:59:43.941 INFO:teuthology.orchestra.run.smithi089.stdout: "debug_ceph_exporter": "1/5", 2023-06-03T19:59:43.942 INFO:teuthology.orchestra.run.smithi089.stdout: "debug_cephfs_mirror": "0/5", 2023-06-03T19:59:43.942 INFO:teuthology.orchestra.run.smithi089.stdout: "debug_cephsqlite": "0/5", 2023-06-03T19:59:43.942 INFO:teuthology.orchestra.run.smithi089.stdout: "debug_client": "0/5", 2023-06-03T19:59:43.942 INFO:teuthology.orchestra.run.smithi089.stdout: "debug_compressor": "1/5", 2023-06-03T19:59:43.942 INFO:teuthology.orchestra.run.smithi089.stdout: "debug_context": "0/1", 2023-06-03T19:59:43.942 INFO:teuthology.orchestra.run.smithi089.stdout: "debug_crush": "1/1", 2023-06-03T19:59:43.942 INFO:teuthology.orchestra.run.smithi089.stdout: "debug_crypto": "1/5", 2023-06-03T19:59:43.943 INFO:teuthology.orchestra.run.smithi089.stdout: "debug_cyanstore": "0/5", 2023-06-03T19:59:43.943 INFO:teuthology.orchestra.run.smithi089.stdout: "debug_deliberately_leak_memory": "false", 2023-06-03T19:59:43.943 INFO:teuthology.orchestra.run.smithi089.stdout: "debug_disable_randomized_ping": "false", 2023-06-03T19:59:43.943 INFO:teuthology.orchestra.run.smithi089.stdout: "debug_dpdk": "1/5", 2023-06-03T19:59:43.943 INFO:teuthology.orchestra.run.smithi089.stdout: "debug_eventtrace": "1/5", 2023-06-03T19:59:43.943 INFO:teuthology.orchestra.run.smithi089.stdout: "debug_filer": "0/1", 2023-06-03T19:59:43.943 INFO:teuthology.orchestra.run.smithi089.stdout: "debug_filestore": "1/3", 2023-06-03T19:59:43.943 INFO:teuthology.orchestra.run.smithi089.stdout: "debug_finisher": "1/1", 2023-06-03T19:59:43.944 INFO:teuthology.orchestra.run.smithi089.stdout: "debug_fuse": "1/5", 2023-06-03T19:59:43.944 INFO:teuthology.orchestra.run.smithi089.stdout: "debug_heartbeat_testing_span": "0", 2023-06-03T19:59:43.944 INFO:teuthology.orchestra.run.smithi089.stdout: "debug_heartbeatmap": "1/5", 2023-06-03T19:59:43.944 INFO:teuthology.orchestra.run.smithi089.stdout: "debug_immutable_obj_cache": "0/5", 2023-06-03T19:59:43.944 INFO:teuthology.orchestra.run.smithi089.stdout: "debug_javaclient": "1/5", 2023-06-03T19:59:43.944 INFO:teuthology.orchestra.run.smithi089.stdout: "debug_journal": "1/3", 2023-06-03T19:59:43.944 INFO:teuthology.orchestra.run.smithi089.stdout: "debug_journaler": "0/5", 2023-06-03T19:59:43.944 INFO:teuthology.orchestra.run.smithi089.stdout: "debug_kstore": "1/5", 2023-06-03T19:59:43.945 INFO:teuthology.orchestra.run.smithi089.stdout: "debug_leveldb": "4/5", 2023-06-03T19:59:43.945 INFO:teuthology.orchestra.run.smithi089.stdout: "debug_lockdep": "0/1", 2023-06-03T19:59:43.945 INFO:teuthology.orchestra.run.smithi089.stdout: "debug_mclock": "1/5", 2023-06-03T19:59:43.945 INFO:teuthology.orchestra.run.smithi089.stdout: "debug_mds": "1/5", 2023-06-03T19:59:43.945 INFO:teuthology.orchestra.run.smithi089.stdout: "debug_mds_balancer": "1/5", 2023-06-03T19:59:43.945 INFO:teuthology.orchestra.run.smithi089.stdout: "debug_mds_locker": "1/5", 2023-06-03T19:59:43.945 INFO:teuthology.orchestra.run.smithi089.stdout: "debug_mds_log": "1/5", 2023-06-03T19:59:43.945 INFO:teuthology.orchestra.run.smithi089.stdout: "debug_mds_log_expire": "1/5", 2023-06-03T19:59:43.946 INFO:teuthology.orchestra.run.smithi089.stdout: "debug_mds_migrator": "1/5", 2023-06-03T19:59:43.946 INFO:teuthology.orchestra.run.smithi089.stdout: "debug_memstore": "1/5", 2023-06-03T19:59:43.946 INFO:teuthology.orchestra.run.smithi089.stdout: "debug_mgr": "2/5", 2023-06-03T19:59:43.946 INFO:teuthology.orchestra.run.smithi089.stdout: "debug_mgrc": "1/5", 2023-06-03T19:59:43.946 INFO:teuthology.orchestra.run.smithi089.stdout: "debug_mon": "20/20", 2023-06-03T19:59:43.946 INFO:teuthology.orchestra.run.smithi089.stdout: "debug_monc": "0/10", 2023-06-03T19:59:43.946 INFO:teuthology.orchestra.run.smithi089.stdout: "debug_ms": "1/1", 2023-06-03T19:59:43.946 INFO:teuthology.orchestra.run.smithi089.stdout: "debug_none": "0/5", 2023-06-03T19:59:43.947 INFO:teuthology.orchestra.run.smithi089.stdout: "debug_objclass": "0/5", 2023-06-03T19:59:43.947 INFO:teuthology.orchestra.run.smithi089.stdout: "debug_objectcacher": "0/5", 2023-06-03T19:59:43.947 INFO:teuthology.orchestra.run.smithi089.stdout: "debug_objecter": "0/1", 2023-06-03T19:59:43.947 INFO:teuthology.orchestra.run.smithi089.stdout: "debug_optracker": "0/5", 2023-06-03T19:59:43.947 INFO:teuthology.orchestra.run.smithi089.stdout: "debug_osd": "1/5", 2023-06-03T19:59:43.947 INFO:teuthology.orchestra.run.smithi089.stdout: "debug_paxos": "20/20", 2023-06-03T19:59:43.947 INFO:teuthology.orchestra.run.smithi089.stdout: "debug_perfcounter": "1/5", 2023-06-03T19:59:43.947 INFO:teuthology.orchestra.run.smithi089.stdout: "debug_prioritycache": "1/5", 2023-06-03T19:59:43.948 INFO:teuthology.orchestra.run.smithi089.stdout: "debug_rados": "0/5", 2023-06-03T19:59:43.948 INFO:teuthology.orchestra.run.smithi089.stdout: "debug_rbd": "0/5", 2023-06-03T19:59:43.948 INFO:teuthology.orchestra.run.smithi089.stdout: "debug_rbd_mirror": "0/5", 2023-06-03T19:59:43.948 INFO:teuthology.orchestra.run.smithi089.stdout: "debug_rbd_pwl": "0/5", 2023-06-03T19:59:43.948 INFO:teuthology.orchestra.run.smithi089.stdout: "debug_rbd_replay": "0/5", 2023-06-03T19:59:43.948 INFO:teuthology.orchestra.run.smithi089.stdout: "debug_refs": "0/0", 2023-06-03T19:59:43.948 INFO:teuthology.orchestra.run.smithi089.stdout: "debug_reserver": "1/1", 2023-06-03T19:59:43.948 INFO:teuthology.orchestra.run.smithi089.stdout: "debug_rgw": "1/5", 2023-06-03T19:59:43.949 INFO:teuthology.orchestra.run.smithi089.stdout: "debug_rgw_access": "1/5", 2023-06-03T19:59:43.949 INFO:teuthology.orchestra.run.smithi089.stdout: "debug_rgw_datacache": "1/5", 2023-06-03T19:59:43.949 INFO:teuthology.orchestra.run.smithi089.stdout: "debug_rgw_dbstore": "1/5", 2023-06-03T19:59:43.949 INFO:teuthology.orchestra.run.smithi089.stdout: "debug_rgw_flight": "1/5", 2023-06-03T19:59:43.949 INFO:teuthology.orchestra.run.smithi089.stdout: "debug_rgw_sync": "1/5", 2023-06-03T19:59:43.949 INFO:teuthology.orchestra.run.smithi089.stdout: "debug_rocksdb": "4/5", 2023-06-03T19:59:43.949 INFO:teuthology.orchestra.run.smithi089.stdout: "debug_seastore": "0/5", 2023-06-03T19:59:43.949 INFO:teuthology.orchestra.run.smithi089.stdout: "debug_seastore_backref": "0/5", 2023-06-03T19:59:43.950 INFO:teuthology.orchestra.run.smithi089.stdout: "debug_seastore_cache": "0/5", 2023-06-03T19:59:43.950 INFO:teuthology.orchestra.run.smithi089.stdout: "debug_seastore_cleaner": "0/5", 2023-06-03T19:59:43.950 INFO:teuthology.orchestra.run.smithi089.stdout: "debug_seastore_device": "0/5", 2023-06-03T19:59:43.950 INFO:teuthology.orchestra.run.smithi089.stdout: "debug_seastore_epm": "0/5", 2023-06-03T19:59:43.950 INFO:teuthology.orchestra.run.smithi089.stdout: "debug_seastore_fixedkv_tree": "0/5", 2023-06-03T19:59:43.950 INFO:teuthology.orchestra.run.smithi089.stdout: "debug_seastore_journal": "0/5", 2023-06-03T19:59:43.950 INFO:teuthology.orchestra.run.smithi089.stdout: "debug_seastore_lba": "0/5", 2023-06-03T19:59:43.951 INFO:teuthology.orchestra.run.smithi089.stdout: "debug_seastore_odata": "0/5", 2023-06-03T19:59:43.951 INFO:teuthology.orchestra.run.smithi089.stdout: "debug_seastore_omap": "0/5", 2023-06-03T19:59:43.951 INFO:teuthology.orchestra.run.smithi089.stdout: "debug_seastore_onode": "0/5", 2023-06-03T19:59:43.951 INFO:teuthology.orchestra.run.smithi089.stdout: "debug_seastore_t": "0/5", 2023-06-03T19:59:43.951 INFO:teuthology.orchestra.run.smithi089.stdout: "debug_seastore_tm": "0/5", 2023-06-03T19:59:43.951 INFO:teuthology.orchestra.run.smithi089.stdout: "debug_striper": "0/1", 2023-06-03T19:59:43.951 INFO:teuthology.orchestra.run.smithi089.stdout: "debug_test": "0/5", 2023-06-03T19:59:43.951 INFO:teuthology.orchestra.run.smithi089.stdout: "debug_throttle": "1/1", 2023-06-03T19:59:43.952 INFO:teuthology.orchestra.run.smithi089.stdout: "debug_timer": "0/1", 2023-06-03T19:59:43.952 INFO:teuthology.orchestra.run.smithi089.stdout: "debug_tp": "0/5", 2023-06-03T19:59:43.952 INFO:teuthology.orchestra.run.smithi089.stdout: "device_failure_prediction_mode": "none", 2023-06-03T19:59:43.952 INFO:teuthology.orchestra.run.smithi089.stdout: "enable_experimental_unrecoverable_data_corrupting_features": "", 2023-06-03T19:59:43.952 INFO:teuthology.orchestra.run.smithi089.stdout: "erasure_code_dir": "/usr/lib64/ceph/erasure-code", 2023-06-03T19:59:43.952 INFO:teuthology.orchestra.run.smithi089.stdout: "err_to_graylog": "false", 2023-06-03T19:59:43.952 INFO:teuthology.orchestra.run.smithi089.stdout: "err_to_journald": "false", 2023-06-03T19:59:43.952 INFO:teuthology.orchestra.run.smithi089.stdout: "err_to_stderr": "true", 2023-06-03T19:59:43.953 INFO:teuthology.orchestra.run.smithi089.stdout: "err_to_syslog": "false", 2023-06-03T19:59:43.953 INFO:teuthology.orchestra.run.smithi089.stdout: "event_tracing": "false", 2023-06-03T19:59:43.953 INFO:teuthology.orchestra.run.smithi089.stdout: "exporter_addr": "0.0.0.0", 2023-06-03T19:59:43.953 INFO:teuthology.orchestra.run.smithi089.stdout: "exporter_http_port": "9926", 2023-06-03T19:59:43.953 INFO:teuthology.orchestra.run.smithi089.stdout: "exporter_prio_limit": "5", 2023-06-03T19:59:43.953 INFO:teuthology.orchestra.run.smithi089.stdout: "exporter_sock_dir": "/var/run/ceph/", 2023-06-03T19:59:43.953 INFO:teuthology.orchestra.run.smithi089.stdout: "exporter_sort_metrics": "true", 2023-06-03T19:59:43.953 INFO:teuthology.orchestra.run.smithi089.stdout: "exporter_stats_period": "5", 2023-06-03T19:59:43.954 INFO:teuthology.orchestra.run.smithi089.stdout: "fake_statfs_for_testing": "0", 2023-06-03T19:59:43.954 INFO:teuthology.orchestra.run.smithi089.stdout: "fatal_signal_handlers": "true", 2023-06-03T19:59:43.954 INFO:teuthology.orchestra.run.smithi089.stdout: "filer_max_purge_ops": "10", 2023-06-03T19:59:43.954 INFO:teuthology.orchestra.run.smithi089.stdout: "filer_max_truncate_ops": "128", 2023-06-03T19:59:43.954 INFO:teuthology.orchestra.run.smithi089.stdout: "filestore_apply_finisher_threads": "1", 2023-06-03T19:59:43.954 INFO:teuthology.orchestra.run.smithi089.stdout: "filestore_blackhole": "false", 2023-06-03T19:59:43.954 INFO:teuthology.orchestra.run.smithi089.stdout: "filestore_btrfs_clone_range": "true", 2023-06-03T19:59:43.954 INFO:teuthology.orchestra.run.smithi089.stdout: "filestore_btrfs_snap": "true", 2023-06-03T19:59:43.955 INFO:teuthology.orchestra.run.smithi089.stdout: "filestore_caller_concurrency": "10", 2023-06-03T19:59:43.955 INFO:teuthology.orchestra.run.smithi089.stdout: "filestore_collect_device_partition_information": "true", 2023-06-03T19:59:43.955 INFO:teuthology.orchestra.run.smithi089.stdout: "filestore_commit_timeout": "600.000000", 2023-06-03T19:59:43.955 INFO:teuthology.orchestra.run.smithi089.stdout: "filestore_debug_inject_read_err": "false", 2023-06-03T19:59:43.955 INFO:teuthology.orchestra.run.smithi089.stdout: "filestore_debug_omap_check": "false", 2023-06-03T19:59:43.955 INFO:teuthology.orchestra.run.smithi089.stdout: "filestore_debug_random_read_err": "0.000000", 2023-06-03T19:59:43.955 INFO:teuthology.orchestra.run.smithi089.stdout: "filestore_debug_verify_split": "false", 2023-06-03T19:59:43.955 INFO:teuthology.orchestra.run.smithi089.stdout: "filestore_dump_file": "", 2023-06-03T19:59:43.956 INFO:teuthology.orchestra.run.smithi089.stdout: "filestore_expected_throughput_bytes": "209715200.000000", 2023-06-03T19:59:43.956 INFO:teuthology.orchestra.run.smithi089.stdout: "filestore_expected_throughput_ops": "200.000000", 2023-06-03T19:59:43.956 INFO:teuthology.orchestra.run.smithi089.stdout: "filestore_fadvise": "true", 2023-06-03T19:59:43.956 INFO:teuthology.orchestra.run.smithi089.stdout: "filestore_fail_eio": "true", 2023-06-03T19:59:43.956 INFO:teuthology.orchestra.run.smithi089.stdout: "filestore_fd_cache_shards": "16", 2023-06-03T19:59:43.956 INFO:teuthology.orchestra.run.smithi089.stdout: "filestore_fd_cache_size": "128", 2023-06-03T19:59:43.956 INFO:teuthology.orchestra.run.smithi089.stdout: "filestore_fiemap": "false", 2023-06-03T19:59:43.957 INFO:teuthology.orchestra.run.smithi089.stdout: "filestore_fiemap_threshold": "4096", 2023-06-03T19:59:43.957 INFO:teuthology.orchestra.run.smithi089.stdout: "filestore_fsync_flushes_journal_data": "false", 2023-06-03T19:59:43.957 INFO:teuthology.orchestra.run.smithi089.stdout: "filestore_index_retry_probability": "0.000000", 2023-06-03T19:59:43.957 INFO:teuthology.orchestra.run.smithi089.stdout: "filestore_inject_stall": "0", 2023-06-03T19:59:43.957 INFO:teuthology.orchestra.run.smithi089.stdout: "filestore_journal_parallel": "false", 2023-06-03T19:59:43.957 INFO:teuthology.orchestra.run.smithi089.stdout: "filestore_journal_trailing": "false", 2023-06-03T19:59:43.957 INFO:teuthology.orchestra.run.smithi089.stdout: "filestore_journal_writeahead": "false", 2023-06-03T19:59:43.957 INFO:teuthology.orchestra.run.smithi089.stdout: "filestore_kill_at": "0", 2023-06-03T19:59:43.958 INFO:teuthology.orchestra.run.smithi089.stdout: "filestore_max_alloc_hint_size": "1048576", 2023-06-03T19:59:43.958 INFO:teuthology.orchestra.run.smithi089.stdout: "filestore_max_inline_xattr_size": "0", 2023-06-03T19:59:43.958 INFO:teuthology.orchestra.run.smithi089.stdout: "filestore_max_inline_xattr_size_btrfs": "2048", 2023-06-03T19:59:43.958 INFO:teuthology.orchestra.run.smithi089.stdout: "filestore_max_inline_xattr_size_other": "512", 2023-06-03T19:59:43.958 INFO:teuthology.orchestra.run.smithi089.stdout: "filestore_max_inline_xattr_size_xfs": "65536", 2023-06-03T19:59:43.958 INFO:teuthology.orchestra.run.smithi089.stdout: "filestore_max_inline_xattrs": "0", 2023-06-03T19:59:43.958 INFO:teuthology.orchestra.run.smithi089.stdout: "filestore_max_inline_xattrs_btrfs": "10", 2023-06-03T19:59:43.958 INFO:teuthology.orchestra.run.smithi089.stdout: "filestore_max_inline_xattrs_other": "2", 2023-06-03T19:59:43.959 INFO:teuthology.orchestra.run.smithi089.stdout: "filestore_max_inline_xattrs_xfs": "10", 2023-06-03T19:59:43.959 INFO:teuthology.orchestra.run.smithi089.stdout: "filestore_max_sync_interval": "5.000000", 2023-06-03T19:59:43.959 INFO:teuthology.orchestra.run.smithi089.stdout: "filestore_max_xattr_value_size": "0", 2023-06-03T19:59:43.959 INFO:teuthology.orchestra.run.smithi089.stdout: "filestore_max_xattr_value_size_btrfs": "65536", 2023-06-03T19:59:43.959 INFO:teuthology.orchestra.run.smithi089.stdout: "filestore_max_xattr_value_size_other": "1024", 2023-06-03T19:59:43.959 INFO:teuthology.orchestra.run.smithi089.stdout: "filestore_max_xattr_value_size_xfs": "65536", 2023-06-03T19:59:43.959 INFO:teuthology.orchestra.run.smithi089.stdout: "filestore_merge_threshold": "-10", 2023-06-03T19:59:43.959 INFO:teuthology.orchestra.run.smithi089.stdout: "filestore_min_sync_interval": "0.010000", 2023-06-03T19:59:43.960 INFO:teuthology.orchestra.run.smithi089.stdout: "filestore_odsync_write": "false", 2023-06-03T19:59:43.960 INFO:teuthology.orchestra.run.smithi089.stdout: "filestore_omap_backend": "rocksdb", 2023-06-03T19:59:43.960 INFO:teuthology.orchestra.run.smithi089.stdout: "filestore_omap_backend_path": "", 2023-06-03T19:59:43.960 INFO:teuthology.orchestra.run.smithi089.stdout: "filestore_omap_header_cache_size": "1024", 2023-06-03T19:59:43.960 INFO:teuthology.orchestra.run.smithi089.stdout: "filestore_ondisk_finisher_threads": "1", 2023-06-03T19:59:43.960 INFO:teuthology.orchestra.run.smithi089.stdout: "filestore_op_thread_suicide_timeout": "180", 2023-06-03T19:59:43.960 INFO:teuthology.orchestra.run.smithi089.stdout: "filestore_op_thread_timeout": "60", 2023-06-03T19:59:43.961 INFO:teuthology.orchestra.run.smithi089.stdout: "filestore_op_threads": "2", 2023-06-03T19:59:43.961 INFO:teuthology.orchestra.run.smithi089.stdout: "filestore_punch_hole": "false", 2023-06-03T19:59:43.961 INFO:teuthology.orchestra.run.smithi089.stdout: "filestore_queue_high_delay_multiple": "0.000000", 2023-06-03T19:59:43.961 INFO:teuthology.orchestra.run.smithi089.stdout: "filestore_queue_high_delay_multiple_bytes": "0.000000", 2023-06-03T19:59:43.961 INFO:teuthology.orchestra.run.smithi089.stdout: "filestore_queue_high_delay_multiple_ops": "0.000000", 2023-06-03T19:59:43.961 INFO:teuthology.orchestra.run.smithi089.stdout: "filestore_queue_high_threshhold": "0.900000", 2023-06-03T19:59:43.961 INFO:teuthology.orchestra.run.smithi089.stdout: "filestore_queue_low_threshhold": "0.300000", 2023-06-03T19:59:43.961 INFO:teuthology.orchestra.run.smithi089.stdout: "filestore_queue_max_bytes": "104857600", 2023-06-03T19:59:43.962 INFO:teuthology.orchestra.run.smithi089.stdout: "filestore_queue_max_delay_multiple": "0.000000", 2023-06-03T19:59:43.962 INFO:teuthology.orchestra.run.smithi089.stdout: "filestore_queue_max_delay_multiple_bytes": "0.000000", 2023-06-03T19:59:43.962 INFO:teuthology.orchestra.run.smithi089.stdout: "filestore_queue_max_delay_multiple_ops": "0.000000", 2023-06-03T19:59:43.962 INFO:teuthology.orchestra.run.smithi089.stdout: "filestore_queue_max_ops": "50", 2023-06-03T19:59:43.962 INFO:teuthology.orchestra.run.smithi089.stdout: "filestore_rocksdb_options": "max_background_jobs=10,compaction_readahead_size=2097152,compression=kNoCompression", 2023-06-03T19:59:43.962 INFO:teuthology.orchestra.run.smithi089.stdout: "filestore_seek_data_hole": "false", 2023-06-03T19:59:43.962 INFO:teuthology.orchestra.run.smithi089.stdout: "filestore_sloppy_crc": "false", 2023-06-03T19:59:43.962 INFO:teuthology.orchestra.run.smithi089.stdout: "filestore_sloppy_crc_block_size": "65536", 2023-06-03T19:59:43.963 INFO:teuthology.orchestra.run.smithi089.stdout: "filestore_splice": "false", 2023-06-03T19:59:43.963 INFO:teuthology.orchestra.run.smithi089.stdout: "filestore_split_multiple": "2", 2023-06-03T19:59:43.963 INFO:teuthology.orchestra.run.smithi089.stdout: "filestore_split_rand_factor": "20", 2023-06-03T19:59:43.963 INFO:teuthology.orchestra.run.smithi089.stdout: "filestore_update_to": "1000", 2023-06-03T19:59:43.963 INFO:teuthology.orchestra.run.smithi089.stdout: "filestore_wbthrottle_btrfs_bytes_hard_limit": "419430400", 2023-06-03T19:59:43.963 INFO:teuthology.orchestra.run.smithi089.stdout: "filestore_wbthrottle_btrfs_bytes_start_flusher": "41943040", 2023-06-03T19:59:43.963 INFO:teuthology.orchestra.run.smithi089.stdout: "filestore_wbthrottle_btrfs_inodes_hard_limit": "5000", 2023-06-03T19:59:43.963 INFO:teuthology.orchestra.run.smithi089.stdout: "filestore_wbthrottle_btrfs_inodes_start_flusher": "500", 2023-06-03T19:59:43.964 INFO:teuthology.orchestra.run.smithi089.stdout: "filestore_wbthrottle_btrfs_ios_hard_limit": "5000", 2023-06-03T19:59:43.964 INFO:teuthology.orchestra.run.smithi089.stdout: "filestore_wbthrottle_btrfs_ios_start_flusher": "500", 2023-06-03T19:59:43.964 INFO:teuthology.orchestra.run.smithi089.stdout: "filestore_wbthrottle_enable": "true", 2023-06-03T19:59:43.964 INFO:teuthology.orchestra.run.smithi089.stdout: "filestore_wbthrottle_xfs_bytes_hard_limit": "419430400", 2023-06-03T19:59:43.964 INFO:teuthology.orchestra.run.smithi089.stdout: "filestore_wbthrottle_xfs_bytes_start_flusher": "41943040", 2023-06-03T19:59:43.964 INFO:teuthology.orchestra.run.smithi089.stdout: "filestore_wbthrottle_xfs_inodes_hard_limit": "5000", 2023-06-03T19:59:43.964 INFO:teuthology.orchestra.run.smithi089.stdout: "filestore_wbthrottle_xfs_inodes_start_flusher": "500", 2023-06-03T19:59:43.965 INFO:teuthology.orchestra.run.smithi089.stdout: "filestore_wbthrottle_xfs_ios_hard_limit": "5000", 2023-06-03T19:59:43.965 INFO:teuthology.orchestra.run.smithi089.stdout: "filestore_wbthrottle_xfs_ios_start_flusher": "500", 2023-06-03T19:59:43.965 INFO:teuthology.orchestra.run.smithi089.stdout: "filestore_xfs_extsize": "false", 2023-06-03T19:59:43.965 INFO:teuthology.orchestra.run.smithi089.stdout: "filestore_zfs_snap": "false", 2023-06-03T19:59:43.965 INFO:teuthology.orchestra.run.smithi089.stdout: "fio_dir": "/tmp/fio", 2023-06-03T19:59:43.965 INFO:teuthology.orchestra.run.smithi089.stdout: "fsid": "f1db3478-0243-11ee-9b1d-001a4aab830c", 2023-06-03T19:59:43.965 INFO:teuthology.orchestra.run.smithi089.stdout: "fuse_allow_other": "true", 2023-06-03T19:59:43.965 INFO:teuthology.orchestra.run.smithi089.stdout: "fuse_atomic_o_trunc": "true", 2023-06-03T19:59:43.966 INFO:teuthology.orchestra.run.smithi089.stdout: "fuse_big_writes": "true", 2023-06-03T19:59:43.966 INFO:teuthology.orchestra.run.smithi089.stdout: "fuse_debug": "false", 2023-06-03T19:59:43.966 INFO:teuthology.orchestra.run.smithi089.stdout: "fuse_default_permissions": "false", 2023-06-03T19:59:43.966 INFO:teuthology.orchestra.run.smithi089.stdout: "fuse_disable_pagecache": "false", 2023-06-03T19:59:43.966 INFO:teuthology.orchestra.run.smithi089.stdout: "fuse_max_write": "0", 2023-06-03T19:59:43.966 INFO:teuthology.orchestra.run.smithi089.stdout: "fuse_multithreaded": "true", 2023-06-03T19:59:43.966 INFO:teuthology.orchestra.run.smithi089.stdout: "fuse_require_active_mds": "true", 2023-06-03T19:59:43.966 INFO:teuthology.orchestra.run.smithi089.stdout: "fuse_set_user_groups": "true", 2023-06-03T19:59:43.967 INFO:teuthology.orchestra.run.smithi089.stdout: "fuse_splice_move": "true", 2023-06-03T19:59:43.967 INFO:teuthology.orchestra.run.smithi089.stdout: "fuse_splice_read": "true", 2023-06-03T19:59:43.967 INFO:teuthology.orchestra.run.smithi089.stdout: "fuse_splice_write": "true", 2023-06-03T19:59:43.967 INFO:teuthology.orchestra.run.smithi089.stdout: "fuse_syncfs_on_mksnap": "true", 2023-06-03T19:59:43.967 INFO:teuthology.orchestra.run.smithi089.stdout: "fuse_use_invalidate_cb": "true", 2023-06-03T19:59:43.967 INFO:teuthology.orchestra.run.smithi089.stdout: "gss_ktab_client_file": "/var/lib/ceph/mon.a/gss_client_mon.a.ktab", 2023-06-03T19:59:43.967 INFO:teuthology.orchestra.run.smithi089.stdout: "gss_target_name": "ceph", 2023-06-03T19:59:43.967 INFO:teuthology.orchestra.run.smithi089.stdout: "heartbeat_file": "", 2023-06-03T19:59:43.968 INFO:teuthology.orchestra.run.smithi089.stdout: "heartbeat_inject_failure": "0", 2023-06-03T19:59:43.968 INFO:teuthology.orchestra.run.smithi089.stdout: "heartbeat_interval": "5", 2023-06-03T19:59:43.968 INFO:teuthology.orchestra.run.smithi089.stdout: "host": "smithi089", 2023-06-03T19:59:43.968 INFO:teuthology.orchestra.run.smithi089.stdout: "immutable_object_cache_client_dedicated_thread_num": "2", 2023-06-03T19:59:43.968 INFO:teuthology.orchestra.run.smithi089.stdout: "immutable_object_cache_max_inflight_ops": "128", 2023-06-03T19:59:43.968 INFO:teuthology.orchestra.run.smithi089.stdout: "immutable_object_cache_max_size": "1073741824", 2023-06-03T19:59:43.968 INFO:teuthology.orchestra.run.smithi089.stdout: "immutable_object_cache_path": "/tmp/ceph_immutable_object_cache", 2023-06-03T19:59:43.968 INFO:teuthology.orchestra.run.smithi089.stdout: "immutable_object_cache_qos_bps_burst": "0", 2023-06-03T19:59:43.969 INFO:teuthology.orchestra.run.smithi089.stdout: "immutable_object_cache_qos_bps_burst_seconds": "1", 2023-06-03T19:59:43.969 INFO:teuthology.orchestra.run.smithi089.stdout: "immutable_object_cache_qos_bps_limit": "0", 2023-06-03T19:59:43.969 INFO:teuthology.orchestra.run.smithi089.stdout: "immutable_object_cache_qos_iops_burst": "0", 2023-06-03T19:59:43.969 INFO:teuthology.orchestra.run.smithi089.stdout: "immutable_object_cache_qos_iops_burst_seconds": "1", 2023-06-03T19:59:43.969 INFO:teuthology.orchestra.run.smithi089.stdout: "immutable_object_cache_qos_iops_limit": "0", 2023-06-03T19:59:43.969 INFO:teuthology.orchestra.run.smithi089.stdout: "immutable_object_cache_qos_schedule_tick_min": "50", 2023-06-03T19:59:43.969 INFO:teuthology.orchestra.run.smithi089.stdout: "immutable_object_cache_sock": "/var/run/ceph/immutable_object_cache_sock", 2023-06-03T19:59:43.969 INFO:teuthology.orchestra.run.smithi089.stdout: "immutable_object_cache_watermark": "0.900000", 2023-06-03T19:59:43.970 INFO:teuthology.orchestra.run.smithi089.stdout: "inject_early_sigterm": "false", 2023-06-03T19:59:43.970 INFO:teuthology.orchestra.run.smithi089.stdout: "jaeger_agent_port": "6799", 2023-06-03T19:59:43.970 INFO:teuthology.orchestra.run.smithi089.stdout: "jaeger_tracing_enable": "false", 2023-06-03T19:59:43.970 INFO:teuthology.orchestra.run.smithi089.stdout: "journal_aio": "true", 2023-06-03T19:59:43.970 INFO:teuthology.orchestra.run.smithi089.stdout: "journal_align_min_size": "65536", 2023-06-03T19:59:43.970 INFO:teuthology.orchestra.run.smithi089.stdout: "journal_block_align": "true", 2023-06-03T19:59:43.970 INFO:teuthology.orchestra.run.smithi089.stdout: "journal_block_size": "4096", 2023-06-03T19:59:43.970 INFO:teuthology.orchestra.run.smithi089.stdout: "journal_dio": "true", 2023-06-03T19:59:43.971 INFO:teuthology.orchestra.run.smithi089.stdout: "journal_discard": "false", 2023-06-03T19:59:43.971 INFO:teuthology.orchestra.run.smithi089.stdout: "journal_force_aio": "false", 2023-06-03T19:59:43.971 INFO:teuthology.orchestra.run.smithi089.stdout: "journal_ignore_corruption": "false", 2023-06-03T19:59:43.971 INFO:teuthology.orchestra.run.smithi089.stdout: "journal_max_write_bytes": "10485760", 2023-06-03T19:59:43.971 INFO:teuthology.orchestra.run.smithi089.stdout: "journal_max_write_entries": "100", 2023-06-03T19:59:43.971 INFO:teuthology.orchestra.run.smithi089.stdout: "journal_replay_from": "0", 2023-06-03T19:59:43.971 INFO:teuthology.orchestra.run.smithi089.stdout: "journal_throttle_high_multiple": "0.000000", 2023-06-03T19:59:43.971 INFO:teuthology.orchestra.run.smithi089.stdout: "journal_throttle_high_threshhold": "0.900000", 2023-06-03T19:59:43.972 INFO:teuthology.orchestra.run.smithi089.stdout: "journal_throttle_low_threshhold": "0.600000", 2023-06-03T19:59:43.972 INFO:teuthology.orchestra.run.smithi089.stdout: "journal_throttle_max_multiple": "0.000000", 2023-06-03T19:59:43.972 INFO:teuthology.orchestra.run.smithi089.stdout: "journal_write_header_frequency": "0", 2023-06-03T19:59:43.972 INFO:teuthology.orchestra.run.smithi089.stdout: "journal_zero_on_create": "false", 2023-06-03T19:59:43.972 INFO:teuthology.orchestra.run.smithi089.stdout: "journaler_prefetch_periods": "10", 2023-06-03T19:59:43.972 INFO:teuthology.orchestra.run.smithi089.stdout: "journaler_prezero_periods": "5", 2023-06-03T19:59:43.972 INFO:teuthology.orchestra.run.smithi089.stdout: "journaler_write_head_interval": "15", 2023-06-03T19:59:43.973 INFO:teuthology.orchestra.run.smithi089.stdout: "key": "", 2023-06-03T19:59:43.973 INFO:teuthology.orchestra.run.smithi089.stdout: "keyfile": "", 2023-06-03T19:59:43.973 INFO:teuthology.orchestra.run.smithi089.stdout: "keyring": "/var/lib/ceph/mon/ceph-a/keyring", 2023-06-03T19:59:43.973 INFO:teuthology.orchestra.run.smithi089.stdout: "kstore_backend": "rocksdb", 2023-06-03T19:59:43.973 INFO:teuthology.orchestra.run.smithi089.stdout: "kstore_default_stripe_size": "65536", 2023-06-03T19:59:43.973 INFO:teuthology.orchestra.run.smithi089.stdout: "kstore_fsck_on_mount": "false", 2023-06-03T19:59:43.973 INFO:teuthology.orchestra.run.smithi089.stdout: "kstore_fsck_on_mount_deep": "true", 2023-06-03T19:59:43.973 INFO:teuthology.orchestra.run.smithi089.stdout: "kstore_max_bytes": "67108864", 2023-06-03T19:59:43.974 INFO:teuthology.orchestra.run.smithi089.stdout: "kstore_max_ops": "512", 2023-06-03T19:59:43.974 INFO:teuthology.orchestra.run.smithi089.stdout: "kstore_nid_prealloc": "1024", 2023-06-03T19:59:43.974 INFO:teuthology.orchestra.run.smithi089.stdout: "kstore_onode_map_size": "1024", 2023-06-03T19:59:43.974 INFO:teuthology.orchestra.run.smithi089.stdout: "kstore_rocksdb_options": "compression=kNoCompression", 2023-06-03T19:59:43.974 INFO:teuthology.orchestra.run.smithi089.stdout: "kstore_sync_submit_transaction": "false", 2023-06-03T19:59:43.974 INFO:teuthology.orchestra.run.smithi089.stdout: "kstore_sync_transaction": "false", 2023-06-03T19:59:43.974 INFO:teuthology.orchestra.run.smithi089.stdout: "leveldb_block_size": "65536", 2023-06-03T19:59:43.974 INFO:teuthology.orchestra.run.smithi089.stdout: "leveldb_bloom_size": "0", 2023-06-03T19:59:43.975 INFO:teuthology.orchestra.run.smithi089.stdout: "leveldb_cache_size": "536870912", 2023-06-03T19:59:43.975 INFO:teuthology.orchestra.run.smithi089.stdout: "leveldb_compact_on_mount": "false", 2023-06-03T19:59:43.975 INFO:teuthology.orchestra.run.smithi089.stdout: "leveldb_compression": "false", 2023-06-03T19:59:43.975 INFO:teuthology.orchestra.run.smithi089.stdout: "leveldb_log": "", 2023-06-03T19:59:43.975 INFO:teuthology.orchestra.run.smithi089.stdout: "leveldb_log_to_ceph_log": "true", 2023-06-03T19:59:43.975 INFO:teuthology.orchestra.run.smithi089.stdout: "leveldb_max_open_files": "0", 2023-06-03T19:59:43.975 INFO:teuthology.orchestra.run.smithi089.stdout: "leveldb_paranoid": "false", 2023-06-03T19:59:43.975 INFO:teuthology.orchestra.run.smithi089.stdout: "leveldb_write_buffer_size": "33554432", 2023-06-03T19:59:43.976 INFO:teuthology.orchestra.run.smithi089.stdout: "librados_thread_count": "2", 2023-06-03T19:59:43.976 INFO:teuthology.orchestra.run.smithi089.stdout: "lockdep": "false", 2023-06-03T19:59:43.976 INFO:teuthology.orchestra.run.smithi089.stdout: "lockdep_force_backtrace": "false", 2023-06-03T19:59:43.976 INFO:teuthology.orchestra.run.smithi089.stdout: "log_coarse_timestamps": "true", 2023-06-03T19:59:43.976 INFO:teuthology.orchestra.run.smithi089.stdout: "log_file": "/var/log/ceph/ceph-mon.a.log", 2023-06-03T19:59:43.976 INFO:teuthology.orchestra.run.smithi089.stdout: "log_flush_on_exit": "false", 2023-06-03T19:59:43.976 INFO:teuthology.orchestra.run.smithi089.stdout: "log_graylog_host": "127.0.0.1", 2023-06-03T19:59:43.976 INFO:teuthology.orchestra.run.smithi089.stdout: "log_graylog_port": "12201", 2023-06-03T19:59:43.977 INFO:teuthology.orchestra.run.smithi089.stdout: "log_max_new": "1000", 2023-06-03T19:59:43.977 INFO:teuthology.orchestra.run.smithi089.stdout: "log_max_recent": "10000", 2023-06-03T19:59:43.977 INFO:teuthology.orchestra.run.smithi089.stdout: "log_stderr_prefix": "debug ", 2023-06-03T19:59:43.977 INFO:teuthology.orchestra.run.smithi089.stdout: "log_stop_at_utilization": "0.970000", 2023-06-03T19:59:43.977 INFO:teuthology.orchestra.run.smithi089.stdout: "log_to_file": "true", 2023-06-03T19:59:43.977 INFO:teuthology.orchestra.run.smithi089.stdout: "log_to_graylog": "false", 2023-06-03T19:59:43.977 INFO:teuthology.orchestra.run.smithi089.stdout: "log_to_journald": "false", 2023-06-03T19:59:43.977 INFO:teuthology.orchestra.run.smithi089.stdout: "log_to_stderr": "false", 2023-06-03T19:59:43.978 INFO:teuthology.orchestra.run.smithi089.stdout: "log_to_syslog": "false", 2023-06-03T19:59:43.978 INFO:teuthology.orchestra.run.smithi089.stdout: "max_rotating_auth_attempts": "10", 2023-06-03T19:59:43.978 INFO:teuthology.orchestra.run.smithi089.stdout: "mds_abort_on_newly_corrupt_dentry": "true", 2023-06-03T19:59:43.978 INFO:teuthology.orchestra.run.smithi089.stdout: "mds_action_on_write_error": "1", 2023-06-03T19:59:43.978 INFO:teuthology.orchestra.run.smithi089.stdout: "mds_alternate_name_max": "8192", 2023-06-03T19:59:43.978 INFO:teuthology.orchestra.run.smithi089.stdout: "mds_asio_thread_count": "2", 2023-06-03T19:59:43.978 INFO:teuthology.orchestra.run.smithi089.stdout: "mds_bal_export_pin": "true", 2023-06-03T19:59:43.978 INFO:teuthology.orchestra.run.smithi089.stdout: "mds_bal_fragment_dirs": "true", 2023-06-03T19:59:43.979 INFO:teuthology.orchestra.run.smithi089.stdout: "mds_bal_fragment_fast_factor": "1.500000", 2023-06-03T19:59:43.979 INFO:teuthology.orchestra.run.smithi089.stdout: "mds_bal_fragment_interval": "5", 2023-06-03T19:59:43.979 INFO:teuthology.orchestra.run.smithi089.stdout: "mds_bal_fragment_size_max": "100000", 2023-06-03T19:59:43.979 INFO:teuthology.orchestra.run.smithi089.stdout: "mds_bal_idle_threshold": "0.000000", 2023-06-03T19:59:43.979 INFO:teuthology.orchestra.run.smithi089.stdout: "mds_bal_interval": "10", 2023-06-03T19:59:43.979 INFO:teuthology.orchestra.run.smithi089.stdout: "mds_bal_max": "-1", 2023-06-03T19:59:43.979 INFO:teuthology.orchestra.run.smithi089.stdout: "mds_bal_max_until": "-1", 2023-06-03T19:59:43.980 INFO:teuthology.orchestra.run.smithi089.stdout: "mds_bal_merge_size": "50", 2023-06-03T19:59:43.980 INFO:teuthology.orchestra.run.smithi089.stdout: "mds_bal_midchunk": "0.300000", 2023-06-03T19:59:43.980 INFO:teuthology.orchestra.run.smithi089.stdout: "mds_bal_min_rebalance": "0.100000", 2023-06-03T19:59:43.980 INFO:teuthology.orchestra.run.smithi089.stdout: "mds_bal_min_start": "0.200000", 2023-06-03T19:59:43.980 INFO:teuthology.orchestra.run.smithi089.stdout: "mds_bal_minchunk": "0.001000", 2023-06-03T19:59:43.980 INFO:teuthology.orchestra.run.smithi089.stdout: "mds_bal_mode": "0", 2023-06-03T19:59:43.980 INFO:teuthology.orchestra.run.smithi089.stdout: "mds_bal_need_max": "1.200000", 2023-06-03T19:59:43.980 INFO:teuthology.orchestra.run.smithi089.stdout: "mds_bal_need_min": "0.800000", 2023-06-03T19:59:43.981 INFO:teuthology.orchestra.run.smithi089.stdout: "mds_bal_replicate_threshold": "8000.000000", 2023-06-03T19:59:43.981 INFO:teuthology.orchestra.run.smithi089.stdout: "mds_bal_sample_interval": "3.000000", 2023-06-03T19:59:43.981 INFO:teuthology.orchestra.run.smithi089.stdout: "mds_bal_split_bits": "3", 2023-06-03T19:59:43.981 INFO:teuthology.orchestra.run.smithi089.stdout: "mds_bal_split_rd": "25000.000000", 2023-06-03T19:59:43.981 INFO:teuthology.orchestra.run.smithi089.stdout: "mds_bal_split_size": "10000", 2023-06-03T19:59:43.981 INFO:teuthology.orchestra.run.smithi089.stdout: "mds_bal_split_wr": "10000.000000", 2023-06-03T19:59:43.981 INFO:teuthology.orchestra.run.smithi089.stdout: "mds_bal_target_decay": "10.000000", 2023-06-03T19:59:43.981 INFO:teuthology.orchestra.run.smithi089.stdout: "mds_bal_unreplicate_threshold": "0.000000", 2023-06-03T19:59:43.982 INFO:teuthology.orchestra.run.smithi089.stdout: "mds_beacon_grace": "15.000000", 2023-06-03T19:59:43.982 INFO:teuthology.orchestra.run.smithi089.stdout: "mds_beacon_interval": "4.000000", 2023-06-03T19:59:43.982 INFO:teuthology.orchestra.run.smithi089.stdout: "mds_beacon_mon_down_grace": "60", 2023-06-03T19:59:43.982 INFO:teuthology.orchestra.run.smithi089.stdout: "mds_cache_memory_limit": "4294967296", 2023-06-03T19:59:43.982 INFO:teuthology.orchestra.run.smithi089.stdout: "mds_cache_mid": "0.700000", 2023-06-03T19:59:43.982 INFO:teuthology.orchestra.run.smithi089.stdout: "mds_cache_release_free_interval": "10", 2023-06-03T19:59:43.982 INFO:teuthology.orchestra.run.smithi089.stdout: "mds_cache_reservation": "0.050000", 2023-06-03T19:59:43.982 INFO:teuthology.orchestra.run.smithi089.stdout: "mds_cache_trim_decay_rate": "1.000000", 2023-06-03T19:59:43.983 INFO:teuthology.orchestra.run.smithi089.stdout: "mds_cache_trim_interval": "1", 2023-06-03T19:59:43.983 INFO:teuthology.orchestra.run.smithi089.stdout: "mds_cache_trim_threshold": "262144", 2023-06-03T19:59:43.983 INFO:teuthology.orchestra.run.smithi089.stdout: "mds_cap_acquisition_throttle_retry_request_timeout": "0.500000", 2023-06-03T19:59:43.983 INFO:teuthology.orchestra.run.smithi089.stdout: "mds_cap_revoke_eviction_timeout": "0.000000", 2023-06-03T19:59:43.983 INFO:teuthology.orchestra.run.smithi089.stdout: "mds_client_delegate_inos_pct": "50", 2023-06-03T19:59:43.983 INFO:teuthology.orchestra.run.smithi089.stdout: "mds_client_prealloc_inos": "1000", 2023-06-03T19:59:43.983 INFO:teuthology.orchestra.run.smithi089.stdout: "mds_client_writeable_range_max_inc_objs": "1024", 2023-06-03T19:59:43.983 INFO:teuthology.orchestra.run.smithi089.stdout: "mds_connect_bootstrapping": "false", 2023-06-03T19:59:43.984 INFO:teuthology.orchestra.run.smithi089.stdout: "mds_damage_table_max_entries": "10000", 2023-06-03T19:59:43.984 INFO:teuthology.orchestra.run.smithi089.stdout: "mds_data": "/var/lib/ceph/mds/ceph-a", 2023-06-03T19:59:43.984 INFO:teuthology.orchestra.run.smithi089.stdout: "mds_debug_auth_pins": "false", 2023-06-03T19:59:43.984 INFO:teuthology.orchestra.run.smithi089.stdout: "mds_debug_frag": "false", 2023-06-03T19:59:43.984 INFO:teuthology.orchestra.run.smithi089.stdout: "mds_debug_scatterstat": "false", 2023-06-03T19:59:43.984 INFO:teuthology.orchestra.run.smithi089.stdout: "mds_debug_subtrees": "false", 2023-06-03T19:59:43.984 INFO:teuthology.orchestra.run.smithi089.stdout: "mds_decay_halflife": "5.000000", 2023-06-03T19:59:43.984 INFO:teuthology.orchestra.run.smithi089.stdout: "mds_default_dir_hash": "2", 2023-06-03T19:59:43.985 INFO:teuthology.orchestra.run.smithi089.stdout: "mds_defer_session_stale": "true", 2023-06-03T19:59:43.985 INFO:teuthology.orchestra.run.smithi089.stdout: "mds_deny_all_reconnect": "false", 2023-06-03T19:59:43.985 INFO:teuthology.orchestra.run.smithi089.stdout: "mds_dir_keys_per_op": "16384", 2023-06-03T19:59:43.985 INFO:teuthology.orchestra.run.smithi089.stdout: "mds_dir_max_commit_size": "10", 2023-06-03T19:59:43.985 INFO:teuthology.orchestra.run.smithi089.stdout: "mds_dir_max_entries": "0", 2023-06-03T19:59:43.985 INFO:teuthology.orchestra.run.smithi089.stdout: "mds_dir_prefetch": "true", 2023-06-03T19:59:43.985 INFO:teuthology.orchestra.run.smithi089.stdout: "mds_dirstat_min_interval": "1.000000", 2023-06-03T19:59:43.986 INFO:teuthology.orchestra.run.smithi089.stdout: "mds_dump_cache_after_rejoin": "false", 2023-06-03T19:59:43.986 INFO:teuthology.orchestra.run.smithi089.stdout: "mds_dump_cache_on_map": "false", 2023-06-03T19:59:43.986 INFO:teuthology.orchestra.run.smithi089.stdout: "mds_dump_cache_threshold_file": "0", 2023-06-03T19:59:43.986 INFO:teuthology.orchestra.run.smithi089.stdout: "mds_dump_cache_threshold_formatter": "1073741824", 2023-06-03T19:59:43.986 INFO:teuthology.orchestra.run.smithi089.stdout: "mds_early_reply": "true", 2023-06-03T19:59:43.986 INFO:teuthology.orchestra.run.smithi089.stdout: "mds_enable_op_tracker": "true", 2023-06-03T19:59:43.986 INFO:teuthology.orchestra.run.smithi089.stdout: "mds_enforce_unique_name": "true", 2023-06-03T19:59:43.986 INFO:teuthology.orchestra.run.smithi089.stdout: "mds_export_ephemeral_distributed": "true", 2023-06-03T19:59:43.987 INFO:teuthology.orchestra.run.smithi089.stdout: "mds_export_ephemeral_distributed_factor": "2.000000", 2023-06-03T19:59:43.987 INFO:teuthology.orchestra.run.smithi089.stdout: "mds_export_ephemeral_random": "true", 2023-06-03T19:59:43.987 INFO:teuthology.orchestra.run.smithi089.stdout: "mds_export_ephemeral_random_max": "0.010000", 2023-06-03T19:59:43.987 INFO:teuthology.orchestra.run.smithi089.stdout: "mds_extraordinary_events_dump_interval": "0", 2023-06-03T19:59:43.987 INFO:teuthology.orchestra.run.smithi089.stdout: "mds_forward_all_requests_to_auth": "false", 2023-06-03T19:59:43.987 INFO:teuthology.orchestra.run.smithi089.stdout: "mds_freeze_tree_timeout": "30.000000", 2023-06-03T19:59:43.987 INFO:teuthology.orchestra.run.smithi089.stdout: "mds_fscrypt_last_block_max_size": "4096", 2023-06-03T19:59:43.987 INFO:teuthology.orchestra.run.smithi089.stdout: "mds_go_bad_corrupt_dentry": "true", 2023-06-03T19:59:43.988 INFO:teuthology.orchestra.run.smithi089.stdout: "mds_hack_allow_loading_invalid_metadata": "false", 2023-06-03T19:59:43.988 INFO:teuthology.orchestra.run.smithi089.stdout: "mds_health_cache_threshold": "1.500000", 2023-06-03T19:59:43.988 INFO:teuthology.orchestra.run.smithi089.stdout: "mds_health_summarize_threshold": "10", 2023-06-03T19:59:43.988 INFO:teuthology.orchestra.run.smithi089.stdout: "mds_heartbeat_grace": "15.000000", 2023-06-03T19:59:43.988 INFO:teuthology.orchestra.run.smithi089.stdout: "mds_heartbeat_reset_grace": "1000", 2023-06-03T19:59:43.988 INFO:teuthology.orchestra.run.smithi089.stdout: "mds_inject_health_dummy": "false", 2023-06-03T19:59:43.988 INFO:teuthology.orchestra.run.smithi089.stdout: "mds_inject_journal_corrupt_dentry_first": "0.000000", 2023-06-03T19:59:43.988 INFO:teuthology.orchestra.run.smithi089.stdout: "mds_inject_migrator_session_race": "false", 2023-06-03T19:59:43.989 INFO:teuthology.orchestra.run.smithi089.stdout: "mds_inject_rename_corrupt_dentry_first": "0.000000", 2023-06-03T19:59:43.989 INFO:teuthology.orchestra.run.smithi089.stdout: "mds_inject_traceless_reply_probability": "0.000000", 2023-06-03T19:59:43.989 INFO:teuthology.orchestra.run.smithi089.stdout: "mds_join_fs": "", 2023-06-03T19:59:43.989 INFO:teuthology.orchestra.run.smithi089.stdout: "mds_journal_format": "1", 2023-06-03T19:59:43.989 INFO:teuthology.orchestra.run.smithi089.stdout: "mds_kill_create_at": "0", 2023-06-03T19:59:43.989 INFO:teuthology.orchestra.run.smithi089.stdout: "mds_kill_export_at": "0", 2023-06-03T19:59:43.989 INFO:teuthology.orchestra.run.smithi089.stdout: "mds_kill_import_at": "0", 2023-06-03T19:59:43.989 INFO:teuthology.orchestra.run.smithi089.stdout: "mds_kill_journal_at": "0", 2023-06-03T19:59:43.990 INFO:teuthology.orchestra.run.smithi089.stdout: "mds_kill_journal_expire_at": "0", 2023-06-03T19:59:43.990 INFO:teuthology.orchestra.run.smithi089.stdout: "mds_kill_journal_replay_at": "0", 2023-06-03T19:59:43.990 INFO:teuthology.orchestra.run.smithi089.stdout: "mds_kill_link_at": "0", 2023-06-03T19:59:43.990 INFO:teuthology.orchestra.run.smithi089.stdout: "mds_kill_mdstable_at": "0", 2023-06-03T19:59:43.990 INFO:teuthology.orchestra.run.smithi089.stdout: "mds_kill_openc_at": "0", 2023-06-03T19:59:43.990 INFO:teuthology.orchestra.run.smithi089.stdout: "mds_kill_rename_at": "0", 2023-06-03T19:59:43.990 INFO:teuthology.orchestra.run.smithi089.stdout: "mds_log_events_per_segment": "1024", 2023-06-03T19:59:43.990 INFO:teuthology.orchestra.run.smithi089.stdout: "mds_log_max_events": "-1", 2023-06-03T19:59:43.991 INFO:teuthology.orchestra.run.smithi089.stdout: "mds_log_max_segments": "128", 2023-06-03T19:59:43.991 INFO:teuthology.orchestra.run.smithi089.stdout: "mds_log_pause": "false", 2023-06-03T19:59:43.991 INFO:teuthology.orchestra.run.smithi089.stdout: "mds_log_segment_size": "0", 2023-06-03T19:59:43.991 INFO:teuthology.orchestra.run.smithi089.stdout: "mds_log_skip_corrupt_events": "false", 2023-06-03T19:59:43.991 INFO:teuthology.orchestra.run.smithi089.stdout: "mds_log_warn_factor": "2.000000", 2023-06-03T19:59:43.991 INFO:teuthology.orchestra.run.smithi089.stdout: "mds_max_caps_per_client": "1048576", 2023-06-03T19:59:43.991 INFO:teuthology.orchestra.run.smithi089.stdout: "mds_max_completed_flushes": "100000", 2023-06-03T19:59:43.991 INFO:teuthology.orchestra.run.smithi089.stdout: "mds_max_completed_requests": "100000", 2023-06-03T19:59:43.992 INFO:teuthology.orchestra.run.smithi089.stdout: "mds_max_export_size": "20971520", 2023-06-03T19:59:43.992 INFO:teuthology.orchestra.run.smithi089.stdout: "mds_max_file_recover": "32", 2023-06-03T19:59:43.992 INFO:teuthology.orchestra.run.smithi089.stdout: "mds_max_purge_files": "64", 2023-06-03T19:59:43.992 INFO:teuthology.orchestra.run.smithi089.stdout: "mds_max_purge_ops": "8192", 2023-06-03T19:59:43.992 INFO:teuthology.orchestra.run.smithi089.stdout: "mds_max_purge_ops_per_pg": "0.500000", 2023-06-03T19:59:43.992 INFO:teuthology.orchestra.run.smithi089.stdout: "mds_max_scrub_ops_in_progress": "5", 2023-06-03T19:59:43.992 INFO:teuthology.orchestra.run.smithi089.stdout: "mds_max_snaps_per_dir": "100", 2023-06-03T19:59:43.993 INFO:teuthology.orchestra.run.smithi089.stdout: "mds_max_xattr_pairs_size": "65536", 2023-06-03T19:59:43.993 INFO:teuthology.orchestra.run.smithi089.stdout: "mds_metrics_update_interval": "2", 2023-06-03T19:59:43.993 INFO:teuthology.orchestra.run.smithi089.stdout: "mds_min_caps_per_client": "100", 2023-06-03T19:59:43.993 INFO:teuthology.orchestra.run.smithi089.stdout: "mds_min_caps_working_set": "10000", 2023-06-03T19:59:43.993 INFO:teuthology.orchestra.run.smithi089.stdout: "mds_mon_shutdown_timeout": "5.000000", 2023-06-03T19:59:43.993 INFO:teuthology.orchestra.run.smithi089.stdout: "mds_numa_node": "-1", 2023-06-03T19:59:43.993 INFO:teuthology.orchestra.run.smithi089.stdout: "mds_oft_prefetch_dirfrags": "false", 2023-06-03T19:59:43.993 INFO:teuthology.orchestra.run.smithi089.stdout: "mds_op_complaint_time": "30.000000", 2023-06-03T19:59:43.994 INFO:teuthology.orchestra.run.smithi089.stdout: "mds_op_history_duration": "600", 2023-06-03T19:59:43.994 INFO:teuthology.orchestra.run.smithi089.stdout: "mds_op_history_size": "20", 2023-06-03T19:59:43.994 INFO:teuthology.orchestra.run.smithi089.stdout: "mds_op_log_threshold": "5", 2023-06-03T19:59:43.994 INFO:teuthology.orchestra.run.smithi089.stdout: "mds_ping_grace": "15", 2023-06-03T19:59:43.994 INFO:teuthology.orchestra.run.smithi089.stdout: "mds_ping_interval": "5", 2023-06-03T19:59:43.994 INFO:teuthology.orchestra.run.smithi089.stdout: "mds_purge_queue_busy_flush_period": "1.000000", 2023-06-03T19:59:43.994 INFO:teuthology.orchestra.run.smithi089.stdout: "mds_recall_global_max_decay_threshold": "131072", 2023-06-03T19:59:43.994 INFO:teuthology.orchestra.run.smithi089.stdout: "mds_recall_max_caps": "30000", 2023-06-03T19:59:43.995 INFO:teuthology.orchestra.run.smithi089.stdout: "mds_recall_max_decay_rate": "1.500000", 2023-06-03T19:59:43.995 INFO:teuthology.orchestra.run.smithi089.stdout: "mds_recall_max_decay_threshold": "131072", 2023-06-03T19:59:43.995 INFO:teuthology.orchestra.run.smithi089.stdout: "mds_recall_warning_decay_rate": "60.000000", 2023-06-03T19:59:43.995 INFO:teuthology.orchestra.run.smithi089.stdout: "mds_recall_warning_threshold": "262144", 2023-06-03T19:59:43.995 INFO:teuthology.orchestra.run.smithi089.stdout: "mds_reconnect_timeout": "45.000000", 2023-06-03T19:59:43.995 INFO:teuthology.orchestra.run.smithi089.stdout: "mds_replay_interval": "1.000000", 2023-06-03T19:59:43.995 INFO:teuthology.orchestra.run.smithi089.stdout: "mds_replay_unsafe_with_closed_session": "false", 2023-06-03T19:59:43.995 INFO:teuthology.orchestra.run.smithi089.stdout: "mds_request_load_average_decay_rate": "60.000000", 2023-06-03T19:59:43.996 INFO:teuthology.orchestra.run.smithi089.stdout: "mds_root_ino_gid": "0", 2023-06-03T19:59:43.996 INFO:teuthology.orchestra.run.smithi089.stdout: "mds_root_ino_uid": "0", 2023-06-03T19:59:43.996 INFO:teuthology.orchestra.run.smithi089.stdout: "mds_scatter_nudge_interval": "5.000000", 2023-06-03T19:59:43.996 INFO:teuthology.orchestra.run.smithi089.stdout: "mds_session_blocklist_on_evict": "true", 2023-06-03T19:59:43.996 INFO:teuthology.orchestra.run.smithi089.stdout: "mds_session_blocklist_on_timeout": "true", 2023-06-03T19:59:43.996 INFO:teuthology.orchestra.run.smithi089.stdout: "mds_session_cache_liveness_decay_rate": "300.000000", 2023-06-03T19:59:43.996 INFO:teuthology.orchestra.run.smithi089.stdout: "mds_session_cache_liveness_magnitude": "10", 2023-06-03T19:59:43.996 INFO:teuthology.orchestra.run.smithi089.stdout: "mds_session_cap_acquisition_decay_rate": "10.000000", 2023-06-03T19:59:43.997 INFO:teuthology.orchestra.run.smithi089.stdout: "mds_session_cap_acquisition_throttle": "500000", 2023-06-03T19:59:43.997 INFO:teuthology.orchestra.run.smithi089.stdout: "mds_session_max_caps_throttle_ratio": "1.100000", 2023-06-03T19:59:43.997 INFO:teuthology.orchestra.run.smithi089.stdout: "mds_sessionmap_keys_per_op": "1024", 2023-06-03T19:59:43.997 INFO:teuthology.orchestra.run.smithi089.stdout: "mds_shutdown_check": "0", 2023-06-03T19:59:43.997 INFO:teuthology.orchestra.run.smithi089.stdout: "mds_skip_ino": "0", 2023-06-03T19:59:43.997 INFO:teuthology.orchestra.run.smithi089.stdout: "mds_sleep_rank_change": "0.000000", 2023-06-03T19:59:43.997 INFO:teuthology.orchestra.run.smithi089.stdout: "mds_snap_max_uid": "4294967294", 2023-06-03T19:59:43.997 INFO:teuthology.orchestra.run.smithi089.stdout: "mds_snap_min_uid": "0", 2023-06-03T19:59:43.998 INFO:teuthology.orchestra.run.smithi089.stdout: "mds_snap_rstat": "false", 2023-06-03T19:59:43.998 INFO:teuthology.orchestra.run.smithi089.stdout: "mds_standby_replay_damaged": "false", 2023-06-03T19:59:43.998 INFO:teuthology.orchestra.run.smithi089.stdout: "mds_symlink_recovery": "true", 2023-06-03T19:59:43.998 INFO:teuthology.orchestra.run.smithi089.stdout: "mds_task_status_update_interval": "2.000000", 2023-06-03T19:59:43.998 INFO:teuthology.orchestra.run.smithi089.stdout: "mds_thrash_exports": "0", 2023-06-03T19:59:43.998 INFO:teuthology.orchestra.run.smithi089.stdout: "mds_thrash_fragments": "0", 2023-06-03T19:59:43.998 INFO:teuthology.orchestra.run.smithi089.stdout: "mds_tick_interval": "5.000000", 2023-06-03T19:59:43.998 INFO:teuthology.orchestra.run.smithi089.stdout: "mds_valgrind_exit": "false", 2023-06-03T19:59:43.999 INFO:teuthology.orchestra.run.smithi089.stdout: "mds_verify_backtrace": "1", 2023-06-03T19:59:43.999 INFO:teuthology.orchestra.run.smithi089.stdout: "mds_verify_scatter": "false", 2023-06-03T19:59:43.999 INFO:teuthology.orchestra.run.smithi089.stdout: "mds_wipe_ino_prealloc": "false", 2023-06-03T19:59:43.999 INFO:teuthology.orchestra.run.smithi089.stdout: "mds_wipe_sessions": "false", 2023-06-03T19:59:43.999 INFO:teuthology.orchestra.run.smithi089.stdout: "mempool_debug": "false", 2023-06-03T19:59:43.999 INFO:teuthology.orchestra.run.smithi089.stdout: "memstore_debug_omit_block_device_write": "false", 2023-06-03T19:59:43.999 INFO:teuthology.orchestra.run.smithi089.stdout: "memstore_device_bytes": "1073741824", 2023-06-03T19:59:43.999 INFO:teuthology.orchestra.run.smithi089.stdout: "memstore_page_set": "false", 2023-06-03T19:59:44.000 INFO:teuthology.orchestra.run.smithi089.stdout: "memstore_page_size": "65536", 2023-06-03T19:59:44.000 INFO:teuthology.orchestra.run.smithi089.stdout: "mgr_client_bytes": "134217728", 2023-06-03T19:59:44.000 INFO:teuthology.orchestra.run.smithi089.stdout: "mgr_client_messages": "512", 2023-06-03T19:59:44.000 INFO:teuthology.orchestra.run.smithi089.stdout: "mgr_client_service_daemon_unregister_timeout": "1.000000", 2023-06-03T19:59:44.000 INFO:teuthology.orchestra.run.smithi089.stdout: "mgr_connect_retry_interval": "1.000000", 2023-06-03T19:59:44.000 INFO:teuthology.orchestra.run.smithi089.stdout: "mgr_data": "/var/lib/ceph/mgr/ceph-a", 2023-06-03T19:59:44.000 INFO:teuthology.orchestra.run.smithi089.stdout: "mgr_debug_aggressive_pg_num_changes": "false", 2023-06-03T19:59:44.000 INFO:teuthology.orchestra.run.smithi089.stdout: "mgr_disabled_modules": "", 2023-06-03T19:59:44.001 INFO:teuthology.orchestra.run.smithi089.stdout: "mgr_initial_modules": "restful iostat nfs", 2023-06-03T19:59:44.001 INFO:teuthology.orchestra.run.smithi089.stdout: "mgr_max_pg_num_change": "128", 2023-06-03T19:59:44.001 INFO:teuthology.orchestra.run.smithi089.stdout: "mgr_mds_bytes": "134217728", 2023-06-03T19:59:44.001 INFO:teuthology.orchestra.run.smithi089.stdout: "mgr_mds_messages": "128", 2023-06-03T19:59:44.001 INFO:teuthology.orchestra.run.smithi089.stdout: "mgr_module_path": "/usr/share/ceph/mgr", 2023-06-03T19:59:44.001 INFO:teuthology.orchestra.run.smithi089.stdout: "mgr_mon_bytes": "134217728", 2023-06-03T19:59:44.001 INFO:teuthology.orchestra.run.smithi089.stdout: "mgr_mon_messages": "128", 2023-06-03T19:59:44.001 INFO:teuthology.orchestra.run.smithi089.stdout: "mgr_osd_bytes": "536870912", 2023-06-03T19:59:44.002 INFO:teuthology.orchestra.run.smithi089.stdout: "mgr_osd_messages": "8192", 2023-06-03T19:59:44.002 INFO:teuthology.orchestra.run.smithi089.stdout: "mgr_pool": "true", 2023-06-03T19:59:44.002 INFO:teuthology.orchestra.run.smithi089.stdout: "mgr_service_beacon_grace": "60.000000", 2023-06-03T19:59:44.002 INFO:teuthology.orchestra.run.smithi089.stdout: "mgr_standby_modules": "true", 2023-06-03T19:59:44.002 INFO:teuthology.orchestra.run.smithi089.stdout: "mgr_stats_period": "5", 2023-06-03T19:59:44.002 INFO:teuthology.orchestra.run.smithi089.stdout: "mgr_stats_threshold": "5", 2023-06-03T19:59:44.002 INFO:teuthology.orchestra.run.smithi089.stdout: "mgr_tick_period": "2", 2023-06-03T19:59:44.002 INFO:teuthology.orchestra.run.smithi089.stdout: "mgr_ttl_cache_expire_seconds": "0", 2023-06-03T19:59:44.003 INFO:teuthology.orchestra.run.smithi089.stdout: "mon_accept_timeout_factor": "2.000000", 2023-06-03T19:59:44.003 INFO:teuthology.orchestra.run.smithi089.stdout: "mon_allow_pool_delete": "true", 2023-06-03T19:59:44.003 INFO:teuthology.orchestra.run.smithi089.stdout: "mon_allow_pool_size_one": "false", 2023-06-03T19:59:44.003 INFO:teuthology.orchestra.run.smithi089.stdout: "mon_auth_validate_all_caps": "true", 2023-06-03T19:59:44.003 INFO:teuthology.orchestra.run.smithi089.stdout: "mon_cache_target_full_warn_ratio": "0.660000", 2023-06-03T19:59:44.003 INFO:teuthology.orchestra.run.smithi089.stdout: "mon_clean_pg_upmaps_per_chunk": "256", 2023-06-03T19:59:44.003 INFO:teuthology.orchestra.run.smithi089.stdout: "mon_client_bytes": "104857600", 2023-06-03T19:59:44.004 INFO:teuthology.orchestra.run.smithi089.stdout: "mon_client_directed_command_retry": "2", 2023-06-03T19:59:44.004 INFO:teuthology.orchestra.run.smithi089.stdout: "mon_client_hunt_interval": "3.000000", 2023-06-03T19:59:44.004 INFO:teuthology.orchestra.run.smithi089.stdout: "mon_client_hunt_interval_backoff": "1.500000", 2023-06-03T19:59:44.004 INFO:teuthology.orchestra.run.smithi089.stdout: "mon_client_hunt_interval_max_multiple": "10.000000", 2023-06-03T19:59:44.004 INFO:teuthology.orchestra.run.smithi089.stdout: "mon_client_hunt_interval_min_multiple": "1.000000", 2023-06-03T19:59:44.004 INFO:teuthology.orchestra.run.smithi089.stdout: "mon_client_hunt_parallel": "3", 2023-06-03T19:59:44.004 INFO:teuthology.orchestra.run.smithi089.stdout: "mon_client_log_interval": "1.000000", 2023-06-03T19:59:44.004 INFO:teuthology.orchestra.run.smithi089.stdout: "mon_client_max_log_entries_per_message": "1000", 2023-06-03T19:59:44.005 INFO:teuthology.orchestra.run.smithi089.stdout: "mon_client_ping_interval": "10.000000", 2023-06-03T19:59:44.005 INFO:teuthology.orchestra.run.smithi089.stdout: "mon_client_ping_timeout": "30.000000", 2023-06-03T19:59:44.005 INFO:teuthology.orchestra.run.smithi089.stdout: "mon_clock_drift_allowed": "1.000000", 2023-06-03T19:59:44.005 INFO:teuthology.orchestra.run.smithi089.stdout: "mon_clock_drift_warn_backoff": "5.000000", 2023-06-03T19:59:44.005 INFO:teuthology.orchestra.run.smithi089.stdout: "mon_cluster_log_file": "default=/var/log/ceph/ceph.$channel.log cluster=/var/log/ceph/ceph.log", 2023-06-03T19:59:44.005 INFO:teuthology.orchestra.run.smithi089.stdout: "mon_cluster_log_file_level": "debug", 2023-06-03T19:59:44.005 INFO:teuthology.orchestra.run.smithi089.stdout: "mon_cluster_log_to_file": "false", 2023-06-03T19:59:44.005 INFO:teuthology.orchestra.run.smithi089.stdout: "mon_cluster_log_to_graylog": "false", 2023-06-03T19:59:44.006 INFO:teuthology.orchestra.run.smithi089.stdout: "mon_cluster_log_to_graylog_host": "127.0.0.1", 2023-06-03T19:59:44.006 INFO:teuthology.orchestra.run.smithi089.stdout: "mon_cluster_log_to_graylog_port": "12201", 2023-06-03T19:59:44.006 INFO:teuthology.orchestra.run.smithi089.stdout: "mon_cluster_log_to_journald": "false", 2023-06-03T19:59:44.006 INFO:teuthology.orchestra.run.smithi089.stdout: "mon_cluster_log_to_stderr": "true", 2023-06-03T19:59:44.006 INFO:teuthology.orchestra.run.smithi089.stdout: "mon_cluster_log_to_syslog": "default=false", 2023-06-03T19:59:44.006 INFO:teuthology.orchestra.run.smithi089.stdout: "mon_cluster_log_to_syslog_facility": "daemon", 2023-06-03T19:59:44.006 INFO:teuthology.orchestra.run.smithi089.stdout: "mon_cluster_log_to_syslog_level": "info", 2023-06-03T19:59:44.006 INFO:teuthology.orchestra.run.smithi089.stdout: "mon_compact_on_bootstrap": "false", 2023-06-03T19:59:44.007 INFO:teuthology.orchestra.run.smithi089.stdout: "mon_compact_on_start": "false", 2023-06-03T19:59:44.007 INFO:teuthology.orchestra.run.smithi089.stdout: "mon_compact_on_trim": "true", 2023-06-03T19:59:44.007 INFO:teuthology.orchestra.run.smithi089.stdout: "mon_con_tracker_persist_interval": "10", 2023-06-03T19:59:44.007 INFO:teuthology.orchestra.run.smithi089.stdout: "mon_con_tracker_score_halflife": "43200", 2023-06-03T19:59:44.007 INFO:teuthology.orchestra.run.smithi089.stdout: "mon_config_key_max_entry_size": "65536", 2023-06-03T19:59:44.007 INFO:teuthology.orchestra.run.smithi089.stdout: "mon_cpu_threads": "4", 2023-06-03T19:59:44.007 INFO:teuthology.orchestra.run.smithi089.stdout: "mon_crush_min_required_version": "hammer", 2023-06-03T19:59:44.008 INFO:teuthology.orchestra.run.smithi089.stdout: "mon_daemon_bytes": "419430400", 2023-06-03T19:59:44.008 INFO:teuthology.orchestra.run.smithi089.stdout: "mon_data": "/var/lib/ceph/mon/ceph-a", 2023-06-03T19:59:44.008 INFO:teuthology.orchestra.run.smithi089.stdout: "mon_data_avail_crit": "5", 2023-06-03T19:59:44.008 INFO:teuthology.orchestra.run.smithi089.stdout: "mon_data_avail_warn": "5", 2023-06-03T19:59:44.008 INFO:teuthology.orchestra.run.smithi089.stdout: "mon_data_size_warn": "16106127360", 2023-06-03T19:59:44.008 INFO:teuthology.orchestra.run.smithi089.stdout: "mon_debug_block_osdmap_trim": "false", 2023-06-03T19:59:44.008 INFO:teuthology.orchestra.run.smithi089.stdout: "mon_debug_deprecated_as_obsolete": "false", 2023-06-03T19:59:44.008 INFO:teuthology.orchestra.run.smithi089.stdout: "mon_debug_dump_json": "false", 2023-06-03T19:59:44.009 INFO:teuthology.orchestra.run.smithi089.stdout: "mon_debug_dump_location": "/var/log/ceph/ceph-mon.a.tdump", 2023-06-03T19:59:44.009 INFO:teuthology.orchestra.run.smithi089.stdout: "mon_debug_dump_transactions": "false", 2023-06-03T19:59:44.009 INFO:teuthology.orchestra.run.smithi089.stdout: "mon_debug_extra_checks": "false", 2023-06-03T19:59:44.009 INFO:teuthology.orchestra.run.smithi089.stdout: "mon_debug_no_initial_persistent_features": "false", 2023-06-03T19:59:44.009 INFO:teuthology.orchestra.run.smithi089.stdout: "mon_debug_no_require_bluestore_for_ec_overwrites": "false", 2023-06-03T19:59:44.009 INFO:teuthology.orchestra.run.smithi089.stdout: "mon_debug_no_require_quincy": "false", 2023-06-03T19:59:44.009 INFO:teuthology.orchestra.run.smithi089.stdout: "mon_debug_no_require_reef": "false", 2023-06-03T19:59:44.009 INFO:teuthology.orchestra.run.smithi089.stdout: "mon_debug_unsafe_allow_tier_with_nonempty_snaps": "false", 2023-06-03T19:59:44.010 INFO:teuthology.orchestra.run.smithi089.stdout: "mon_delta_reset_interval": "10.000000", 2023-06-03T19:59:44.010 INFO:teuthology.orchestra.run.smithi089.stdout: "mon_dns_srv_name": "ceph-mon", 2023-06-03T19:59:44.010 INFO:teuthology.orchestra.run.smithi089.stdout: "mon_down_mkfs_grace": "60", 2023-06-03T19:59:44.010 INFO:teuthology.orchestra.run.smithi089.stdout: "mon_election_default_strategy": "1", 2023-06-03T19:59:44.010 INFO:teuthology.orchestra.run.smithi089.stdout: "mon_election_timeout": "5.000000", 2023-06-03T19:59:44.010 INFO:teuthology.orchestra.run.smithi089.stdout: "mon_elector_ignore_propose_margin": "0.000500", 2023-06-03T19:59:44.010 INFO:teuthology.orchestra.run.smithi089.stdout: "mon_elector_ping_divisor": "2", 2023-06-03T19:59:44.010 INFO:teuthology.orchestra.run.smithi089.stdout: "mon_elector_ping_timeout": "2.000000", 2023-06-03T19:59:44.011 INFO:teuthology.orchestra.run.smithi089.stdout: "mon_enable_op_tracker": "true", 2023-06-03T19:59:44.011 INFO:teuthology.orchestra.run.smithi089.stdout: "mon_fake_pool_delete": "false", 2023-06-03T19:59:44.011 INFO:teuthology.orchestra.run.smithi089.stdout: "mon_force_quorum_join": "false", 2023-06-03T19:59:44.011 INFO:teuthology.orchestra.run.smithi089.stdout: "mon_globalid_prealloc": "10000", 2023-06-03T19:59:44.011 INFO:teuthology.orchestra.run.smithi089.stdout: "mon_health_detail_to_clog": "true", 2023-06-03T19:59:44.011 INFO:teuthology.orchestra.run.smithi089.stdout: "mon_health_log_update_period": "5", 2023-06-03T19:59:44.011 INFO:teuthology.orchestra.run.smithi089.stdout: "mon_health_max_detail": "50", 2023-06-03T19:59:44.011 INFO:teuthology.orchestra.run.smithi089.stdout: "mon_health_to_clog": "true", 2023-06-03T19:59:44.012 INFO:teuthology.orchestra.run.smithi089.stdout: "mon_health_to_clog_interval": "600", 2023-06-03T19:59:44.012 INFO:teuthology.orchestra.run.smithi089.stdout: "mon_health_to_clog_tick_interval": "60.000000", 2023-06-03T19:59:44.012 INFO:teuthology.orchestra.run.smithi089.stdout: "mon_host": "[v2:172.21.15.89:3300/0,v1:172.21.15.89:6789/0] [v2:172.21.15.164:3300/0,v1:172.21.15.164:6789/0] [v2:172.21.15.89:3301/0,v1:172.21.15.89:6790/0]", 2023-06-03T19:59:44.012 INFO:teuthology.orchestra.run.smithi089.stdout: "mon_host_override": "", 2023-06-03T19:59:44.012 INFO:teuthology.orchestra.run.smithi089.stdout: "mon_initial_members": "", 2023-06-03T19:59:44.012 INFO:teuthology.orchestra.run.smithi089.stdout: "mon_inject_pg_merge_bounce_probability": "0.000000", 2023-06-03T19:59:44.012 INFO:teuthology.orchestra.run.smithi089.stdout: "mon_inject_sync_get_chunk_delay": "0.000000", 2023-06-03T19:59:44.013 INFO:teuthology.orchestra.run.smithi089.stdout: "mon_inject_transaction_delay_max": "10.000000", 2023-06-03T19:59:44.013 INFO:teuthology.orchestra.run.smithi089.stdout: "mon_inject_transaction_delay_probability": "0.000000", 2023-06-03T19:59:44.013 INFO:teuthology.orchestra.run.smithi089.stdout: "mon_keyvaluedb": "rocksdb", 2023-06-03T19:59:44.013 INFO:teuthology.orchestra.run.smithi089.stdout: "mon_lease": "5.000000", 2023-06-03T19:59:44.013 INFO:teuthology.orchestra.run.smithi089.stdout: "mon_lease_ack_timeout_factor": "2.000000", 2023-06-03T19:59:44.013 INFO:teuthology.orchestra.run.smithi089.stdout: "mon_lease_renew_interval_factor": "0.600000", 2023-06-03T19:59:44.013 INFO:teuthology.orchestra.run.smithi089.stdout: "mon_log_full_interval": "50", 2023-06-03T19:59:44.013 INFO:teuthology.orchestra.run.smithi089.stdout: "mon_log_max": "10000", 2023-06-03T19:59:44.014 INFO:teuthology.orchestra.run.smithi089.stdout: "mon_log_max_summary": "50", 2023-06-03T19:59:44.014 INFO:teuthology.orchestra.run.smithi089.stdout: "mon_max_log_entries_per_event": "4096", 2023-06-03T19:59:44.014 INFO:teuthology.orchestra.run.smithi089.stdout: "mon_max_log_epochs": "500", 2023-06-03T19:59:44.014 INFO:teuthology.orchestra.run.smithi089.stdout: "mon_max_mdsmap_epochs": "500", 2023-06-03T19:59:44.014 INFO:teuthology.orchestra.run.smithi089.stdout: "mon_max_mgrmap_epochs": "500", 2023-06-03T19:59:44.014 INFO:teuthology.orchestra.run.smithi089.stdout: "mon_max_osd": "10000", 2023-06-03T19:59:44.014 INFO:teuthology.orchestra.run.smithi089.stdout: "mon_max_pg_per_osd": "10000", 2023-06-03T19:59:44.015 INFO:teuthology.orchestra.run.smithi089.stdout: "mon_max_pool_pg_num": "65536", 2023-06-03T19:59:44.015 INFO:teuthology.orchestra.run.smithi089.stdout: "mon_max_snap_prune_per_epoch": "100", 2023-06-03T19:59:44.015 INFO:teuthology.orchestra.run.smithi089.stdout: "mon_mds_blocklist_interval": "86400.000000", 2023-06-03T19:59:44.015 INFO:teuthology.orchestra.run.smithi089.stdout: "mon_mds_force_trim_to": "0", 2023-06-03T19:59:44.015 INFO:teuthology.orchestra.run.smithi089.stdout: "mon_mds_skip_sanity": "true", 2023-06-03T19:59:44.015 INFO:teuthology.orchestra.run.smithi089.stdout: "mon_memory_autotune": "true", 2023-06-03T19:59:44.015 INFO:teuthology.orchestra.run.smithi089.stdout: "mon_memory_target": "2147483648", 2023-06-03T19:59:44.015 INFO:teuthology.orchestra.run.smithi089.stdout: "mon_mgr_beacon_grace": "30", 2023-06-03T19:59:44.016 INFO:teuthology.orchestra.run.smithi089.stdout: "mon_mgr_blocklist_interval": "86400.000000", 2023-06-03T19:59:44.016 INFO:teuthology.orchestra.run.smithi089.stdout: "mon_mgr_digest_period": "5", 2023-06-03T19:59:44.016 INFO:teuthology.orchestra.run.smithi089.stdout: "mon_mgr_inactive_grace": "60", 2023-06-03T19:59:44.016 INFO:teuthology.orchestra.run.smithi089.stdout: "mon_mgr_mkfs_grace": "240", 2023-06-03T19:59:44.016 INFO:teuthology.orchestra.run.smithi089.stdout: "mon_mgr_proxy_client_bytes_ratio": "0.300000", 2023-06-03T19:59:44.016 INFO:teuthology.orchestra.run.smithi089.stdout: "mon_min_osdmap_epochs": "500", 2023-06-03T19:59:44.016 INFO:teuthology.orchestra.run.smithi089.stdout: "mon_op_complaint_time": "30", 2023-06-03T19:59:44.016 INFO:teuthology.orchestra.run.smithi089.stdout: "mon_op_history_duration": "600", 2023-06-03T19:59:44.017 INFO:teuthology.orchestra.run.smithi089.stdout: "mon_op_history_size": "20", 2023-06-03T19:59:44.017 INFO:teuthology.orchestra.run.smithi089.stdout: "mon_op_history_slow_op_size": "20", 2023-06-03T19:59:44.017 INFO:teuthology.orchestra.run.smithi089.stdout: "mon_op_history_slow_op_threshold": "10", 2023-06-03T19:59:44.017 INFO:teuthology.orchestra.run.smithi089.stdout: "mon_op_log_threshold": "5", 2023-06-03T19:59:44.017 INFO:teuthology.orchestra.run.smithi089.stdout: "mon_osd_adjust_down_out_interval": "true", 2023-06-03T19:59:44.017 INFO:teuthology.orchestra.run.smithi089.stdout: "mon_osd_adjust_heartbeat_grace": "true", 2023-06-03T19:59:44.017 INFO:teuthology.orchestra.run.smithi089.stdout: "mon_osd_auto_mark_auto_out_in": "true", 2023-06-03T19:59:44.017 INFO:teuthology.orchestra.run.smithi089.stdout: "mon_osd_auto_mark_in": "false", 2023-06-03T19:59:44.018 INFO:teuthology.orchestra.run.smithi089.stdout: "mon_osd_auto_mark_new_in": "true", 2023-06-03T19:59:44.018 INFO:teuthology.orchestra.run.smithi089.stdout: "mon_osd_backfillfull_ratio": "0.900000", 2023-06-03T19:59:44.018 INFO:teuthology.orchestra.run.smithi089.stdout: "mon_osd_blocklist_default_expire": "3600.000000", 2023-06-03T19:59:44.018 INFO:teuthology.orchestra.run.smithi089.stdout: "mon_osd_cache_size": "500", 2023-06-03T19:59:44.018 INFO:teuthology.orchestra.run.smithi089.stdout: "mon_osd_cache_size_min": "134217728", 2023-06-03T19:59:44.018 INFO:teuthology.orchestra.run.smithi089.stdout: "mon_osd_crush_smoke_test": "true", 2023-06-03T19:59:44.018 INFO:teuthology.orchestra.run.smithi089.stdout: "mon_osd_destroyed_out_interval": "600", 2023-06-03T19:59:44.018 INFO:teuthology.orchestra.run.smithi089.stdout: "mon_osd_down_out_interval": "600", 2023-06-03T19:59:44.019 INFO:teuthology.orchestra.run.smithi089.stdout: "mon_osd_down_out_subtree_limit": "rack", 2023-06-03T19:59:44.019 INFO:teuthology.orchestra.run.smithi089.stdout: "mon_osd_err_op_age_ratio": "128.000000", 2023-06-03T19:59:44.019 INFO:teuthology.orchestra.run.smithi089.stdout: "mon_osd_force_trim_to": "0", 2023-06-03T19:59:44.019 INFO:teuthology.orchestra.run.smithi089.stdout: "mon_osd_full_ratio": "0.950000", 2023-06-03T19:59:44.019 INFO:teuthology.orchestra.run.smithi089.stdout: "mon_osd_initial_require_min_compat_client": "luminous", 2023-06-03T19:59:44.019 INFO:teuthology.orchestra.run.smithi089.stdout: "mon_osd_laggy_halflife": "3600", 2023-06-03T19:59:44.019 INFO:teuthology.orchestra.run.smithi089.stdout: "mon_osd_laggy_max_interval": "300", 2023-06-03T19:59:44.019 INFO:teuthology.orchestra.run.smithi089.stdout: "mon_osd_laggy_weight": "0.300000", 2023-06-03T19:59:44.020 INFO:teuthology.orchestra.run.smithi089.stdout: "mon_osd_mapping_pgs_per_chunk": "4096", 2023-06-03T19:59:44.020 INFO:teuthology.orchestra.run.smithi089.stdout: "mon_osd_max_creating_pgs": "1024", 2023-06-03T19:59:44.020 INFO:teuthology.orchestra.run.smithi089.stdout: "mon_osd_max_initial_pgs": "1024", 2023-06-03T19:59:44.020 INFO:teuthology.orchestra.run.smithi089.stdout: "mon_osd_min_down_reporters": "2", 2023-06-03T19:59:44.020 INFO:teuthology.orchestra.run.smithi089.stdout: "mon_osd_min_in_ratio": "0.750000", 2023-06-03T19:59:44.020 INFO:teuthology.orchestra.run.smithi089.stdout: "mon_osd_min_up_ratio": "0.300000", 2023-06-03T19:59:44.020 INFO:teuthology.orchestra.run.smithi089.stdout: "mon_osd_nearfull_ratio": "0.850000", 2023-06-03T19:59:44.021 INFO:teuthology.orchestra.run.smithi089.stdout: "mon_osd_prime_pg_temp": "true", 2023-06-03T19:59:44.021 INFO:teuthology.orchestra.run.smithi089.stdout: "mon_osd_prime_pg_temp_max_estimate": "0.250000", 2023-06-03T19:59:44.021 INFO:teuthology.orchestra.run.smithi089.stdout: "mon_osd_prime_pg_temp_max_time": "0.500000", 2023-06-03T19:59:44.021 INFO:teuthology.orchestra.run.smithi089.stdout: "mon_osd_report_timeout": "900", 2023-06-03T19:59:44.021 INFO:teuthology.orchestra.run.smithi089.stdout: "mon_osd_reporter_subtree_level": "osd", 2023-06-03T19:59:44.021 INFO:teuthology.orchestra.run.smithi089.stdout: "mon_osd_snap_trim_queue_warn_on": "32768", 2023-06-03T19:59:44.021 INFO:teuthology.orchestra.run.smithi089.stdout: "mon_osd_warn_num_repaired": "10", 2023-06-03T19:59:44.021 INFO:teuthology.orchestra.run.smithi089.stdout: "mon_osd_warn_op_age": "32.000000", 2023-06-03T19:59:44.022 INFO:teuthology.orchestra.run.smithi089.stdout: "mon_osdmap_full_prune_enabled": "true", 2023-06-03T19:59:44.022 INFO:teuthology.orchestra.run.smithi089.stdout: "mon_osdmap_full_prune_interval": "10", 2023-06-03T19:59:44.022 INFO:teuthology.orchestra.run.smithi089.stdout: "mon_osdmap_full_prune_min": "10000", 2023-06-03T19:59:44.022 INFO:teuthology.orchestra.run.smithi089.stdout: "mon_osdmap_full_prune_txsize": "100", 2023-06-03T19:59:44.022 INFO:teuthology.orchestra.run.smithi089.stdout: "mon_pg_check_down_all_threshold": "0.500000", 2023-06-03T19:59:44.022 INFO:teuthology.orchestra.run.smithi089.stdout: "mon_pg_stuck_threshold": "60", 2023-06-03T19:59:44.022 INFO:teuthology.orchestra.run.smithi089.stdout: "mon_pg_warn_max_object_skew": "0.000000", 2023-06-03T19:59:44.022 INFO:teuthology.orchestra.run.smithi089.stdout: "mon_pg_warn_min_objects": "10000", 2023-06-03T19:59:44.023 INFO:teuthology.orchestra.run.smithi089.stdout: "mon_pg_warn_min_per_osd": "0", 2023-06-03T19:59:44.023 INFO:teuthology.orchestra.run.smithi089.stdout: "mon_pg_warn_min_pool_objects": "1000", 2023-06-03T19:59:44.023 INFO:teuthology.orchestra.run.smithi089.stdout: "mon_pool_quota_crit_threshold": "0", 2023-06-03T19:59:44.023 INFO:teuthology.orchestra.run.smithi089.stdout: "mon_pool_quota_warn_threshold": "0", 2023-06-03T19:59:44.023 INFO:teuthology.orchestra.run.smithi089.stdout: "mon_probe_timeout": "2.000000", 2023-06-03T19:59:44.023 INFO:teuthology.orchestra.run.smithi089.stdout: "mon_reweight_max_change": "0.050000", 2023-06-03T19:59:44.023 INFO:teuthology.orchestra.run.smithi089.stdout: "mon_reweight_max_osds": "4", 2023-06-03T19:59:44.023 INFO:teuthology.orchestra.run.smithi089.stdout: "mon_reweight_min_bytes_per_osd": "10", 2023-06-03T19:59:44.024 INFO:teuthology.orchestra.run.smithi089.stdout: "mon_reweight_min_pgs_per_osd": "4", 2023-06-03T19:59:44.024 INFO:teuthology.orchestra.run.smithi089.stdout: "mon_rocksdb_options": "write_buffer_size=33554432,compression=kNoCompression,level_compaction_dynamic_level_bytes=true", 2023-06-03T19:59:44.024 INFO:teuthology.orchestra.run.smithi089.stdout: "mon_scrub_inject_crc_mismatch": "0.000000", 2023-06-03T19:59:44.024 INFO:teuthology.orchestra.run.smithi089.stdout: "mon_scrub_inject_missing_keys": "0.000000", 2023-06-03T19:59:44.024 INFO:teuthology.orchestra.run.smithi089.stdout: "mon_scrub_interval": "86400", 2023-06-03T19:59:44.024 INFO:teuthology.orchestra.run.smithi089.stdout: "mon_scrub_max_keys": "100", 2023-06-03T19:59:44.024 INFO:teuthology.orchestra.run.smithi089.stdout: "mon_scrub_timeout": "300", 2023-06-03T19:59:44.024 INFO:teuthology.orchestra.run.smithi089.stdout: "mon_session_timeout": "300", 2023-06-03T19:59:44.025 INFO:teuthology.orchestra.run.smithi089.stdout: "mon_smart_report_timeout": "5", 2023-06-03T19:59:44.025 INFO:teuthology.orchestra.run.smithi089.stdout: "mon_stat_smooth_intervals": "6", 2023-06-03T19:59:44.025 INFO:teuthology.orchestra.run.smithi089.stdout: "mon_stretch_cluster_recovery_ratio": "0.600000", 2023-06-03T19:59:44.025 INFO:teuthology.orchestra.run.smithi089.stdout: "mon_stretch_pool_min_size": "2", 2023-06-03T19:59:44.025 INFO:teuthology.orchestra.run.smithi089.stdout: "mon_stretch_pool_size": "4", 2023-06-03T19:59:44.025 INFO:teuthology.orchestra.run.smithi089.stdout: "mon_stretch_recovery_min_wait": "15.000000", 2023-06-03T19:59:44.025 INFO:teuthology.orchestra.run.smithi089.stdout: "mon_subscribe_interval": "86400.000000", 2023-06-03T19:59:44.025 INFO:teuthology.orchestra.run.smithi089.stdout: "mon_sync_debug": "false", 2023-06-03T19:59:44.026 INFO:teuthology.orchestra.run.smithi089.stdout: "mon_sync_max_payload_keys": "2000", 2023-06-03T19:59:44.026 INFO:teuthology.orchestra.run.smithi089.stdout: "mon_sync_max_payload_size": "1048576", 2023-06-03T19:59:44.026 INFO:teuthology.orchestra.run.smithi089.stdout: "mon_sync_provider_kill_at": "0", 2023-06-03T19:59:44.026 INFO:teuthology.orchestra.run.smithi089.stdout: "mon_sync_requester_kill_at": "0", 2023-06-03T19:59:44.026 INFO:teuthology.orchestra.run.smithi089.stdout: "mon_sync_timeout": "60.000000", 2023-06-03T19:59:44.026 INFO:teuthology.orchestra.run.smithi089.stdout: "mon_target_pg_per_osd": "100", 2023-06-03T19:59:44.026 INFO:teuthology.orchestra.run.smithi089.stdout: "mon_tick_interval": "5", 2023-06-03T19:59:44.026 INFO:teuthology.orchestra.run.smithi089.stdout: "mon_timecheck_interval": "300.000000", 2023-06-03T19:59:44.027 INFO:teuthology.orchestra.run.smithi089.stdout: "mon_timecheck_skew_interval": "30.000000", 2023-06-03T19:59:44.027 INFO:teuthology.orchestra.run.smithi089.stdout: "mon_use_min_delay_socket": "false", 2023-06-03T19:59:44.027 INFO:teuthology.orchestra.run.smithi089.stdout: "mon_warn_older_version_delay": "604800", 2023-06-03T19:59:44.027 INFO:teuthology.orchestra.run.smithi089.stdout: "mon_warn_on_cache_pools_without_hit_sets": "true", 2023-06-03T19:59:44.027 INFO:teuthology.orchestra.run.smithi089.stdout: "mon_warn_on_crush_straw_calc_version_zero": "false", 2023-06-03T19:59:44.027 INFO:teuthology.orchestra.run.smithi089.stdout: "mon_warn_on_degraded_stretch_mode": "true", 2023-06-03T19:59:44.027 INFO:teuthology.orchestra.run.smithi089.stdout: "mon_warn_on_filestore_osds": "true", 2023-06-03T19:59:44.028 INFO:teuthology.orchestra.run.smithi089.stdout: "mon_warn_on_insecure_global_id_reclaim": "false", 2023-06-03T19:59:44.028 INFO:teuthology.orchestra.run.smithi089.stdout: "mon_warn_on_insecure_global_id_reclaim_allowed": "false", 2023-06-03T19:59:44.028 INFO:teuthology.orchestra.run.smithi089.stdout: "mon_warn_on_legacy_crush_tunables": "false", 2023-06-03T19:59:44.028 INFO:teuthology.orchestra.run.smithi089.stdout: "mon_warn_on_misplaced": "false", 2023-06-03T19:59:44.028 INFO:teuthology.orchestra.run.smithi089.stdout: "mon_warn_on_msgr2_not_enabled": "true", 2023-06-03T19:59:44.028 INFO:teuthology.orchestra.run.smithi089.stdout: "mon_warn_on_older_version": "true", 2023-06-03T19:59:44.028 INFO:teuthology.orchestra.run.smithi089.stdout: "mon_warn_on_osd_down_out_interval_zero": "false", 2023-06-03T19:59:44.028 INFO:teuthology.orchestra.run.smithi089.stdout: "mon_warn_on_pool_no_app": "true", 2023-06-03T19:59:44.029 INFO:teuthology.orchestra.run.smithi089.stdout: "mon_warn_on_pool_no_redundancy": "true", 2023-06-03T19:59:44.029 INFO:teuthology.orchestra.run.smithi089.stdout: "mon_warn_on_pool_pg_num_not_power_of_two": "false", 2023-06-03T19:59:44.029 INFO:teuthology.orchestra.run.smithi089.stdout: "mon_warn_on_slow_ping_ratio": "0.050000", 2023-06-03T19:59:44.029 INFO:teuthology.orchestra.run.smithi089.stdout: "mon_warn_on_slow_ping_time": "0.000000", 2023-06-03T19:59:44.029 INFO:teuthology.orchestra.run.smithi089.stdout: "mon_warn_on_too_few_osds": "false", 2023-06-03T19:59:44.029 INFO:teuthology.orchestra.run.smithi089.stdout: "mon_warn_pg_not_deep_scrubbed_ratio": "0.750000", 2023-06-03T19:59:44.029 INFO:teuthology.orchestra.run.smithi089.stdout: "mon_warn_pg_not_scrubbed_ratio": "0.500000", 2023-06-03T19:59:44.029 INFO:teuthology.orchestra.run.smithi089.stdout: "monmap": "", 2023-06-03T19:59:44.030 INFO:teuthology.orchestra.run.smithi089.stdout: "motr_admin_endpoint": "192.168.180.182@tcp:12345:4:1", 2023-06-03T19:59:44.030 INFO:teuthology.orchestra.run.smithi089.stdout: "motr_admin_fid": "0x7200000000000001:0x0", 2023-06-03T19:59:44.030 INFO:teuthology.orchestra.run.smithi089.stdout: "motr_ha_endpoint": "192.168.180.182@tcp:12345:1:1", 2023-06-03T19:59:44.030 INFO:teuthology.orchestra.run.smithi089.stdout: "motr_my_endpoint": "192.168.180.182@tcp:12345:4:1", 2023-06-03T19:59:44.030 INFO:teuthology.orchestra.run.smithi089.stdout: "motr_my_fid": "0x7200000000000001:0x0", 2023-06-03T19:59:44.030 INFO:teuthology.orchestra.run.smithi089.stdout: "motr_profile_fid": "0x7000000000000001:0x0", 2023-06-03T19:59:44.030 INFO:teuthology.orchestra.run.smithi089.stdout: "motr_tracing_enabled": "false", 2023-06-03T19:59:44.030 INFO:teuthology.orchestra.run.smithi089.stdout: "ms_async_op_threads": "3", 2023-06-03T19:59:44.031 INFO:teuthology.orchestra.run.smithi089.stdout: "ms_async_rdma_buffer_size": "131072", 2023-06-03T19:59:44.031 INFO:teuthology.orchestra.run.smithi089.stdout: "ms_async_rdma_cm": "false", 2023-06-03T19:59:44.031 INFO:teuthology.orchestra.run.smithi089.stdout: "ms_async_rdma_device_name": "", 2023-06-03T19:59:44.031 INFO:teuthology.orchestra.run.smithi089.stdout: "ms_async_rdma_dscp": "96", 2023-06-03T19:59:44.031 INFO:teuthology.orchestra.run.smithi089.stdout: "ms_async_rdma_enable_hugepage": "false", 2023-06-03T19:59:44.031 INFO:teuthology.orchestra.run.smithi089.stdout: "ms_async_rdma_gid_idx": "0", 2023-06-03T19:59:44.031 INFO:teuthology.orchestra.run.smithi089.stdout: "ms_async_rdma_local_gid": "", 2023-06-03T19:59:44.031 INFO:teuthology.orchestra.run.smithi089.stdout: "ms_async_rdma_polling_us": "1000", 2023-06-03T19:59:44.032 INFO:teuthology.orchestra.run.smithi089.stdout: "ms_async_rdma_port_num": "1", 2023-06-03T19:59:44.032 INFO:teuthology.orchestra.run.smithi089.stdout: "ms_async_rdma_receive_buffers": "32768", 2023-06-03T19:59:44.032 INFO:teuthology.orchestra.run.smithi089.stdout: "ms_async_rdma_receive_queue_len": "4096", 2023-06-03T19:59:44.032 INFO:teuthology.orchestra.run.smithi089.stdout: "ms_async_rdma_roce_ver": "1", 2023-06-03T19:59:44.032 INFO:teuthology.orchestra.run.smithi089.stdout: "ms_async_rdma_send_buffers": "1024", 2023-06-03T19:59:44.032 INFO:teuthology.orchestra.run.smithi089.stdout: "ms_async_rdma_sl": "3", 2023-06-03T19:59:44.032 INFO:teuthology.orchestra.run.smithi089.stdout: "ms_async_rdma_support_srq": "true", 2023-06-03T19:59:44.033 INFO:teuthology.orchestra.run.smithi089.stdout: "ms_async_rdma_type": "ib", 2023-06-03T19:59:44.033 INFO:teuthology.orchestra.run.smithi089.stdout: "ms_async_reap_threshold": "5", 2023-06-03T19:59:44.033 INFO:teuthology.orchestra.run.smithi089.stdout: "ms_bind_before_connect": "false", 2023-06-03T19:59:44.033 INFO:teuthology.orchestra.run.smithi089.stdout: "ms_bind_ipv4": "true", 2023-06-03T19:59:44.033 INFO:teuthology.orchestra.run.smithi089.stdout: "ms_bind_ipv6": "false", 2023-06-03T19:59:44.033 INFO:teuthology.orchestra.run.smithi089.stdout: "ms_bind_msgr1": "true", 2023-06-03T19:59:44.033 INFO:teuthology.orchestra.run.smithi089.stdout: "ms_bind_msgr2": "true", 2023-06-03T19:59:44.033 INFO:teuthology.orchestra.run.smithi089.stdout: "ms_bind_port_max": "7568", 2023-06-03T19:59:44.034 INFO:teuthology.orchestra.run.smithi089.stdout: "ms_bind_port_min": "6800", 2023-06-03T19:59:44.034 INFO:teuthology.orchestra.run.smithi089.stdout: "ms_bind_prefer_ipv4": "false", 2023-06-03T19:59:44.034 INFO:teuthology.orchestra.run.smithi089.stdout: "ms_bind_retry_count": "3", 2023-06-03T19:59:44.034 INFO:teuthology.orchestra.run.smithi089.stdout: "ms_bind_retry_delay": "5", 2023-06-03T19:59:44.034 INFO:teuthology.orchestra.run.smithi089.stdout: "ms_blackhole_client": "false", 2023-06-03T19:59:44.034 INFO:teuthology.orchestra.run.smithi089.stdout: "ms_blackhole_mds": "false", 2023-06-03T19:59:44.034 INFO:teuthology.orchestra.run.smithi089.stdout: "ms_blackhole_mgr": "false", 2023-06-03T19:59:44.034 INFO:teuthology.orchestra.run.smithi089.stdout: "ms_blackhole_mon": "false", 2023-06-03T19:59:44.035 INFO:teuthology.orchestra.run.smithi089.stdout: "ms_blackhole_osd": "false", 2023-06-03T19:59:44.035 INFO:teuthology.orchestra.run.smithi089.stdout: "ms_client_mode": "crc secure", 2023-06-03T19:59:44.035 INFO:teuthology.orchestra.run.smithi089.stdout: "ms_cluster_mode": "crc secure", 2023-06-03T19:59:44.035 INFO:teuthology.orchestra.run.smithi089.stdout: "ms_cluster_type": "", 2023-06-03T19:59:44.035 INFO:teuthology.orchestra.run.smithi089.stdout: "ms_compress_secure": "false", 2023-06-03T19:59:44.035 INFO:teuthology.orchestra.run.smithi089.stdout: "ms_connection_idle_timeout": "900", 2023-06-03T19:59:44.035 INFO:teuthology.orchestra.run.smithi089.stdout: "ms_connection_ready_timeout": "10", 2023-06-03T19:59:44.035 INFO:teuthology.orchestra.run.smithi089.stdout: "ms_crc_data": "true", 2023-06-03T19:59:44.036 INFO:teuthology.orchestra.run.smithi089.stdout: "ms_crc_header": "true", 2023-06-03T19:59:44.036 INFO:teuthology.orchestra.run.smithi089.stdout: "ms_die_on_bad_msg": "false", 2023-06-03T19:59:44.036 INFO:teuthology.orchestra.run.smithi089.stdout: "ms_die_on_bug": "true", 2023-06-03T19:59:44.036 INFO:teuthology.orchestra.run.smithi089.stdout: "ms_die_on_old_message": "true", 2023-06-03T19:59:44.036 INFO:teuthology.orchestra.run.smithi089.stdout: "ms_die_on_skipped_message": "false", 2023-06-03T19:59:44.036 INFO:teuthology.orchestra.run.smithi089.stdout: "ms_die_on_unhandled_msg": "false", 2023-06-03T19:59:44.036 INFO:teuthology.orchestra.run.smithi089.stdout: "ms_dispatch_throttle_bytes": "104857600", 2023-06-03T19:59:44.036 INFO:teuthology.orchestra.run.smithi089.stdout: "ms_dpdk_coremask": "0xF", 2023-06-03T19:59:44.037 INFO:teuthology.orchestra.run.smithi089.stdout: "ms_dpdk_debug_allow_loopback": "false", 2023-06-03T19:59:44.037 INFO:teuthology.orchestra.run.smithi089.stdout: "ms_dpdk_devs_allowlist": "", 2023-06-03T19:59:44.037 INFO:teuthology.orchestra.run.smithi089.stdout: "ms_dpdk_enable_tso": "true", 2023-06-03T19:59:44.037 INFO:teuthology.orchestra.run.smithi089.stdout: "ms_dpdk_gateway_ipv4_addr": "", 2023-06-03T19:59:44.037 INFO:teuthology.orchestra.run.smithi089.stdout: "ms_dpdk_host_ipv4_addr": "", 2023-06-03T19:59:44.037 INFO:teuthology.orchestra.run.smithi089.stdout: "ms_dpdk_hugepages": "", 2023-06-03T19:59:44.037 INFO:teuthology.orchestra.run.smithi089.stdout: "ms_dpdk_hw_flow_control": "true", 2023-06-03T19:59:44.037 INFO:teuthology.orchestra.run.smithi089.stdout: "ms_dpdk_hw_queue_weight": "1.000000", 2023-06-03T19:59:44.038 INFO:teuthology.orchestra.run.smithi089.stdout: "ms_dpdk_lro": "true", 2023-06-03T19:59:44.038 INFO:teuthology.orchestra.run.smithi089.stdout: "ms_dpdk_memory_channel": "4", 2023-06-03T19:59:44.038 INFO:teuthology.orchestra.run.smithi089.stdout: "ms_dpdk_netmask_ipv4_addr": "", 2023-06-03T19:59:44.038 INFO:teuthology.orchestra.run.smithi089.stdout: "ms_dpdk_pmd": "", 2023-06-03T19:59:44.038 INFO:teuthology.orchestra.run.smithi089.stdout: "ms_dpdk_port_id": "0", 2023-06-03T19:59:44.038 INFO:teuthology.orchestra.run.smithi089.stdout: "ms_dpdk_rx_buffer_count_per_core": "8192", 2023-06-03T19:59:44.038 INFO:teuthology.orchestra.run.smithi089.stdout: "ms_dump_corrupt_message_level": "1", 2023-06-03T19:59:44.038 INFO:teuthology.orchestra.run.smithi089.stdout: "ms_dump_on_send": "false", 2023-06-03T19:59:44.039 INFO:teuthology.orchestra.run.smithi089.stdout: "ms_initial_backoff": "0.200000", 2023-06-03T19:59:44.039 INFO:teuthology.orchestra.run.smithi089.stdout: "ms_inject_delay_max": "1.000000", 2023-06-03T19:59:44.039 INFO:teuthology.orchestra.run.smithi089.stdout: "ms_inject_delay_probability": "0.000000", 2023-06-03T19:59:44.039 INFO:teuthology.orchestra.run.smithi089.stdout: "ms_inject_delay_type": "", 2023-06-03T19:59:44.039 INFO:teuthology.orchestra.run.smithi089.stdout: "ms_inject_internal_delays": "0.000000", 2023-06-03T19:59:44.039 INFO:teuthology.orchestra.run.smithi089.stdout: "ms_inject_network_congestion": "0", 2023-06-03T19:59:44.039 INFO:teuthology.orchestra.run.smithi089.stdout: "ms_inject_socket_failures": "0", 2023-06-03T19:59:44.039 INFO:teuthology.orchestra.run.smithi089.stdout: "ms_learn_addr_from_peer": "true", 2023-06-03T19:59:44.040 INFO:teuthology.orchestra.run.smithi089.stdout: "ms_max_accept_failures": "4", 2023-06-03T19:59:44.040 INFO:teuthology.orchestra.run.smithi089.stdout: "ms_max_backoff": "15.000000", 2023-06-03T19:59:44.040 INFO:teuthology.orchestra.run.smithi089.stdout: "ms_mon_client_mode": "secure crc", 2023-06-03T19:59:44.040 INFO:teuthology.orchestra.run.smithi089.stdout: "ms_mon_cluster_mode": "secure crc", 2023-06-03T19:59:44.040 INFO:teuthology.orchestra.run.smithi089.stdout: "ms_mon_service_mode": "secure crc", 2023-06-03T19:59:44.040 INFO:teuthology.orchestra.run.smithi089.stdout: "ms_osd_compress_min_size": "1024", 2023-06-03T19:59:44.040 INFO:teuthology.orchestra.run.smithi089.stdout: "ms_osd_compress_mode": "none", 2023-06-03T19:59:44.041 INFO:teuthology.orchestra.run.smithi089.stdout: "ms_osd_compression_algorithm": "snappy", 2023-06-03T19:59:44.041 INFO:teuthology.orchestra.run.smithi089.stdout: "ms_pq_max_tokens_per_priority": "16777216", 2023-06-03T19:59:44.041 INFO:teuthology.orchestra.run.smithi089.stdout: "ms_pq_min_cost": "65536", 2023-06-03T19:59:44.041 INFO:teuthology.orchestra.run.smithi089.stdout: "ms_public_type": "", 2023-06-03T19:59:44.041 INFO:teuthology.orchestra.run.smithi089.stdout: "ms_service_mode": "crc secure", 2023-06-03T19:59:44.041 INFO:teuthology.orchestra.run.smithi089.stdout: "ms_tcp_listen_backlog": "512", 2023-06-03T19:59:44.041 INFO:teuthology.orchestra.run.smithi089.stdout: "ms_tcp_nodelay": "true", 2023-06-03T19:59:44.041 INFO:teuthology.orchestra.run.smithi089.stdout: "ms_tcp_prefetch_max_size": "4096", 2023-06-03T19:59:44.042 INFO:teuthology.orchestra.run.smithi089.stdout: "ms_tcp_rcvbuf": "0", 2023-06-03T19:59:44.042 INFO:teuthology.orchestra.run.smithi089.stdout: "ms_type": "async+posix", 2023-06-03T19:59:44.042 INFO:teuthology.orchestra.run.smithi089.stdout: "no_config_file": "false", 2023-06-03T19:59:44.042 INFO:teuthology.orchestra.run.smithi089.stdout: "objecter_completion_locks_per_session": "32", 2023-06-03T19:59:44.042 INFO:teuthology.orchestra.run.smithi089.stdout: "objecter_debug_inject_relock_delay": "false", 2023-06-03T19:59:44.042 INFO:teuthology.orchestra.run.smithi089.stdout: "objecter_inflight_op_bytes": "104857600", 2023-06-03T19:59:44.042 INFO:teuthology.orchestra.run.smithi089.stdout: "objecter_inflight_ops": "1024", 2023-06-03T19:59:44.042 INFO:teuthology.orchestra.run.smithi089.stdout: "objecter_inject_no_watch_ping": "false", 2023-06-03T19:59:44.043 INFO:teuthology.orchestra.run.smithi089.stdout: "objecter_retry_writes_after_first_reply": "false", 2023-06-03T19:59:44.043 INFO:teuthology.orchestra.run.smithi089.stdout: "objecter_tick_interval": "5.000000", 2023-06-03T19:59:44.043 INFO:teuthology.orchestra.run.smithi089.stdout: "objecter_timeout": "10.000000", 2023-06-03T19:59:44.043 INFO:teuthology.orchestra.run.smithi089.stdout: "objectstore_blackhole": "false", 2023-06-03T19:59:44.043 INFO:teuthology.orchestra.run.smithi089.stdout: "openssl_engine_opts": "", 2023-06-03T19:59:44.043 INFO:teuthology.orchestra.run.smithi089.stdout: "osd_agent_delay_time": "5.000000", 2023-06-03T19:59:44.043 INFO:teuthology.orchestra.run.smithi089.stdout: "osd_agent_hist_halflife": "1000", 2023-06-03T19:59:44.043 INFO:teuthology.orchestra.run.smithi089.stdout: "osd_agent_max_low_ops": "2", 2023-06-03T19:59:44.044 INFO:teuthology.orchestra.run.smithi089.stdout: "osd_agent_max_ops": "4", 2023-06-03T19:59:44.044 INFO:teuthology.orchestra.run.smithi089.stdout: "osd_agent_min_evict_effort": "0.100000", 2023-06-03T19:59:44.044 INFO:teuthology.orchestra.run.smithi089.stdout: "osd_agent_quantize_effort": "0.100000", 2023-06-03T19:59:44.044 INFO:teuthology.orchestra.run.smithi089.stdout: "osd_agent_slop": "0.020000", 2023-06-03T19:59:44.044 INFO:teuthology.orchestra.run.smithi089.stdout: "osd_aggregated_slow_ops_logging": "true", 2023-06-03T19:59:44.044 INFO:teuthology.orchestra.run.smithi089.stdout: "osd_allow_recovery_below_min_size": "true", 2023-06-03T19:59:44.044 INFO:teuthology.orchestra.run.smithi089.stdout: "osd_asio_thread_count": "2", 2023-06-03T19:59:44.044 INFO:teuthology.orchestra.run.smithi089.stdout: "osd_async_recovery_min_cost": "100", 2023-06-03T19:59:44.045 INFO:teuthology.orchestra.run.smithi089.stdout: "osd_auto_mark_unfound_lost": "false", 2023-06-03T19:59:44.045 INFO:teuthology.orchestra.run.smithi089.stdout: "osd_backfill_retry_interval": "30.000000", 2023-06-03T19:59:44.045 INFO:teuthology.orchestra.run.smithi089.stdout: "osd_backfill_scan_max": "512", 2023-06-03T19:59:44.045 INFO:teuthology.orchestra.run.smithi089.stdout: "osd_backfill_scan_min": "64", 2023-06-03T19:59:44.045 INFO:teuthology.orchestra.run.smithi089.stdout: "osd_backoff_on_degraded": "false", 2023-06-03T19:59:44.045 INFO:teuthology.orchestra.run.smithi089.stdout: "osd_backoff_on_peering": "false", 2023-06-03T19:59:44.045 INFO:teuthology.orchestra.run.smithi089.stdout: "osd_backoff_on_unfound": "true", 2023-06-03T19:59:44.045 INFO:teuthology.orchestra.run.smithi089.stdout: "osd_beacon_report_interval": "300", 2023-06-03T19:59:44.046 INFO:teuthology.orchestra.run.smithi089.stdout: "osd_bench_duration": "30", 2023-06-03T19:59:44.046 INFO:teuthology.orchestra.run.smithi089.stdout: "osd_bench_large_size_max_throughput": "104857600", 2023-06-03T19:59:44.046 INFO:teuthology.orchestra.run.smithi089.stdout: "osd_bench_max_block_size": "67108864", 2023-06-03T19:59:44.046 INFO:teuthology.orchestra.run.smithi089.stdout: "osd_bench_small_size_max_iops": "100", 2023-06-03T19:59:44.046 INFO:teuthology.orchestra.run.smithi089.stdout: "osd_blkin_trace_all": "false", 2023-06-03T19:59:44.046 INFO:teuthology.orchestra.run.smithi089.stdout: "osd_blocked_scrub_grace_period": "120", 2023-06-03T19:59:44.046 INFO:teuthology.orchestra.run.smithi089.stdout: "osd_calc_pg_upmaps_aggressively": "true", 2023-06-03T19:59:44.046 INFO:teuthology.orchestra.run.smithi089.stdout: "osd_calc_pg_upmaps_aggressively_fast": "true", 2023-06-03T19:59:44.047 INFO:teuthology.orchestra.run.smithi089.stdout: "osd_calc_pg_upmaps_local_fallback_retries": "100", 2023-06-03T19:59:44.047 INFO:teuthology.orchestra.run.smithi089.stdout: "osd_check_for_log_corruption": "false", 2023-06-03T19:59:44.047 INFO:teuthology.orchestra.run.smithi089.stdout: "osd_check_max_object_name_len_on_startup": "true", 2023-06-03T19:59:44.047 INFO:teuthology.orchestra.run.smithi089.stdout: "osd_class_default_list": "cephfs hello journal lock log numops otp rbd refcount rgw rgw_gc timeindex user version cas cmpomap queue 2pc_queue fifo", 2023-06-03T19:59:44.047 INFO:teuthology.orchestra.run.smithi089.stdout: "osd_class_dir": "/usr/lib64/rados-classes", 2023-06-03T19:59:44.047 INFO:teuthology.orchestra.run.smithi089.stdout: "osd_class_load_list": "cephfs hello journal lock log numops otp rbd refcount rgw rgw_gc timeindex user version cas cmpomap queue 2pc_queue fifo", 2023-06-03T19:59:44.047 INFO:teuthology.orchestra.run.smithi089.stdout: "osd_class_update_on_start": "true", 2023-06-03T19:59:44.047 INFO:teuthology.orchestra.run.smithi089.stdout: "osd_client_message_cap": "256", 2023-06-03T19:59:44.048 INFO:teuthology.orchestra.run.smithi089.stdout: "osd_client_message_size_cap": "524288000", 2023-06-03T19:59:44.048 INFO:teuthology.orchestra.run.smithi089.stdout: "osd_client_op_priority": "63", 2023-06-03T19:59:44.048 INFO:teuthology.orchestra.run.smithi089.stdout: "osd_client_watch_timeout": "30", 2023-06-03T19:59:44.048 INFO:teuthology.orchestra.run.smithi089.stdout: "osd_command_max_records": "256", 2023-06-03T19:59:44.048 INFO:teuthology.orchestra.run.smithi089.stdout: "osd_compact_on_start": "false", 2023-06-03T19:59:44.048 INFO:teuthology.orchestra.run.smithi089.stdout: "osd_copyfrom_max_chunk": "8388608", 2023-06-03T19:59:44.048 INFO:teuthology.orchestra.run.smithi089.stdout: "osd_crush_chooseleaf_type": "1", 2023-06-03T19:59:44.049 INFO:teuthology.orchestra.run.smithi089.stdout: "osd_crush_initial_weight": "-1.000000", 2023-06-03T19:59:44.049 INFO:teuthology.orchestra.run.smithi089.stdout: "osd_crush_update_on_start": "true", 2023-06-03T19:59:44.049 INFO:teuthology.orchestra.run.smithi089.stdout: "osd_crush_update_weight_set": "true", 2023-06-03T19:59:44.049 INFO:teuthology.orchestra.run.smithi089.stdout: "osd_data": "/var/lib/ceph/osd/ceph-a", 2023-06-03T19:59:44.049 INFO:teuthology.orchestra.run.smithi089.stdout: "osd_debug_crash_on_ignored_backoff": "false", 2023-06-03T19:59:44.049 INFO:teuthology.orchestra.run.smithi089.stdout: "osd_debug_deep_scrub_sleep": "0.000000", 2023-06-03T19:59:44.049 INFO:teuthology.orchestra.run.smithi089.stdout: "osd_debug_drop_ping_duration": "0", 2023-06-03T19:59:44.049 INFO:teuthology.orchestra.run.smithi089.stdout: "osd_debug_drop_ping_probability": "0.000000", 2023-06-03T19:59:44.050 INFO:teuthology.orchestra.run.smithi089.stdout: "osd_debug_feed_pullee": "-1", 2023-06-03T19:59:44.050 INFO:teuthology.orchestra.run.smithi089.stdout: "osd_debug_inject_copyfrom_error": "false", 2023-06-03T19:59:44.050 INFO:teuthology.orchestra.run.smithi089.stdout: "osd_debug_inject_dispatch_delay_duration": "0.100000", 2023-06-03T19:59:44.050 INFO:teuthology.orchestra.run.smithi089.stdout: "osd_debug_inject_dispatch_delay_probability": "0.000000", 2023-06-03T19:59:44.050 INFO:teuthology.orchestra.run.smithi089.stdout: "osd_debug_misdirected_ops": "false", 2023-06-03T19:59:44.050 INFO:teuthology.orchestra.run.smithi089.stdout: "osd_debug_no_acting_change": "false", 2023-06-03T19:59:44.050 INFO:teuthology.orchestra.run.smithi089.stdout: "osd_debug_no_purge_strays": "false", 2023-06-03T19:59:44.050 INFO:teuthology.orchestra.run.smithi089.stdout: "osd_debug_op_order": "false", 2023-06-03T19:59:44.051 INFO:teuthology.orchestra.run.smithi089.stdout: "osd_debug_pg_log_writeout": "false", 2023-06-03T19:59:44.051 INFO:teuthology.orchestra.run.smithi089.stdout: "osd_debug_pretend_recovery_active": "false", 2023-06-03T19:59:44.051 INFO:teuthology.orchestra.run.smithi089.stdout: "osd_debug_random_push_read_error": "0.000000", 2023-06-03T19:59:44.051 INFO:teuthology.orchestra.run.smithi089.stdout: "osd_debug_reject_backfill_probability": "0.000000", 2023-06-03T19:59:44.051 INFO:teuthology.orchestra.run.smithi089.stdout: "osd_debug_shutdown": "false", 2023-06-03T19:59:44.051 INFO:teuthology.orchestra.run.smithi089.stdout: "osd_debug_skip_full_check_in_backfill_reservation": "false", 2023-06-03T19:59:44.051 INFO:teuthology.orchestra.run.smithi089.stdout: "osd_debug_skip_full_check_in_recovery": "false", 2023-06-03T19:59:44.051 INFO:teuthology.orchestra.run.smithi089.stdout: "osd_debug_verify_cached_snaps": "false", 2023-06-03T19:59:44.052 INFO:teuthology.orchestra.run.smithi089.stdout: "osd_debug_verify_missing_on_start": "false", 2023-06-03T19:59:44.052 INFO:teuthology.orchestra.run.smithi089.stdout: "osd_debug_verify_snaps": "false", 2023-06-03T19:59:44.052 INFO:teuthology.orchestra.run.smithi089.stdout: "osd_debug_verify_stray_on_activate": "false", 2023-06-03T19:59:44.052 INFO:teuthology.orchestra.run.smithi089.stdout: "osd_deep_scrub_interval": "604800.000000", 2023-06-03T19:59:44.052 INFO:teuthology.orchestra.run.smithi089.stdout: "osd_deep_scrub_keys": "1024", 2023-06-03T19:59:44.052 INFO:teuthology.orchestra.run.smithi089.stdout: "osd_deep_scrub_large_omap_object_key_threshold": "200000", 2023-06-03T19:59:44.052 INFO:teuthology.orchestra.run.smithi089.stdout: "osd_deep_scrub_large_omap_object_value_sum_threshold": "1073741824", 2023-06-03T19:59:44.053 INFO:teuthology.orchestra.run.smithi089.stdout: "osd_deep_scrub_randomize_ratio": "0.150000", 2023-06-03T19:59:44.053 INFO:teuthology.orchestra.run.smithi089.stdout: "osd_deep_scrub_stride": "524288", 2023-06-03T19:59:44.053 INFO:teuthology.orchestra.run.smithi089.stdout: "osd_deep_scrub_update_digest_min_age": "7200", 2023-06-03T19:59:44.053 INFO:teuthology.orchestra.run.smithi089.stdout: "osd_default_data_pool_replay_window": "45", 2023-06-03T19:59:44.053 INFO:teuthology.orchestra.run.smithi089.stdout: "osd_default_notify_timeout": "30", 2023-06-03T19:59:44.053 INFO:teuthology.orchestra.run.smithi089.stdout: "osd_delete_sleep": "0.000000", 2023-06-03T19:59:44.053 INFO:teuthology.orchestra.run.smithi089.stdout: "osd_delete_sleep_hdd": "5.000000", 2023-06-03T19:59:44.053 INFO:teuthology.orchestra.run.smithi089.stdout: "osd_delete_sleep_hybrid": "1.000000", 2023-06-03T19:59:44.054 INFO:teuthology.orchestra.run.smithi089.stdout: "osd_delete_sleep_ssd": "1.000000", 2023-06-03T19:59:44.054 INFO:teuthology.orchestra.run.smithi089.stdout: "osd_discard_disconnected_ops": "true", 2023-06-03T19:59:44.054 INFO:teuthology.orchestra.run.smithi089.stdout: "osd_enable_op_tracker": "true", 2023-06-03T19:59:44.054 INFO:teuthology.orchestra.run.smithi089.stdout: "osd_erasure_code_plugins": "jerasure lrc", 2023-06-03T19:59:44.054 INFO:teuthology.orchestra.run.smithi089.stdout: "osd_extblkdev_plugins": "vdo", 2023-06-03T19:59:44.054 INFO:teuthology.orchestra.run.smithi089.stdout: "osd_failsafe_full_ratio": "0.970000", 2023-06-03T19:59:44.054 INFO:teuthology.orchestra.run.smithi089.stdout: "osd_fast_fail_on_connection_refused": "true", 2023-06-03T19:59:44.054 INFO:teuthology.orchestra.run.smithi089.stdout: "osd_fast_info": "true", 2023-06-03T19:59:44.055 INFO:teuthology.orchestra.run.smithi089.stdout: "osd_fast_shutdown": "true", 2023-06-03T19:59:44.055 INFO:teuthology.orchestra.run.smithi089.stdout: "osd_fast_shutdown_notify_mon": "true", 2023-06-03T19:59:44.055 INFO:teuthology.orchestra.run.smithi089.stdout: "osd_fast_shutdown_timeout": "15", 2023-06-03T19:59:44.055 INFO:teuthology.orchestra.run.smithi089.stdout: "osd_find_best_info_ignore_history_les": "false", 2023-06-03T19:59:44.055 INFO:teuthology.orchestra.run.smithi089.stdout: "osd_force_auth_primary_missing_objects": "100", 2023-06-03T19:59:44.055 INFO:teuthology.orchestra.run.smithi089.stdout: "osd_force_recovery_pg_log_entries_factor": "1.300000", 2023-06-03T19:59:44.055 INFO:teuthology.orchestra.run.smithi089.stdout: "osd_function_tracing": "false", 2023-06-03T19:59:44.055 INFO:teuthology.orchestra.run.smithi089.stdout: "osd_heartbeat_grace": "20", 2023-06-03T19:59:44.056 INFO:teuthology.orchestra.run.smithi089.stdout: "osd_heartbeat_interval": "6", 2023-06-03T19:59:44.056 INFO:teuthology.orchestra.run.smithi089.stdout: "osd_heartbeat_min_healthy_ratio": "0.330000", 2023-06-03T19:59:44.056 INFO:teuthology.orchestra.run.smithi089.stdout: "osd_heartbeat_min_peers": "10", 2023-06-03T19:59:44.056 INFO:teuthology.orchestra.run.smithi089.stdout: "osd_heartbeat_min_size": "2000", 2023-06-03T19:59:44.056 INFO:teuthology.orchestra.run.smithi089.stdout: "osd_heartbeat_stale": "600", 2023-06-03T19:59:44.056 INFO:teuthology.orchestra.run.smithi089.stdout: "osd_heartbeat_use_min_delay_socket": "false", 2023-06-03T19:59:44.056 INFO:teuthology.orchestra.run.smithi089.stdout: "osd_hit_set_max_size": "100000", 2023-06-03T19:59:44.057 INFO:teuthology.orchestra.run.smithi089.stdout: "osd_hit_set_min_size": "1000", 2023-06-03T19:59:44.057 INFO:teuthology.orchestra.run.smithi089.stdout: "osd_hit_set_namespace": ".ceph-internal", 2023-06-03T19:59:44.057 INFO:teuthology.orchestra.run.smithi089.stdout: "osd_ignore_stale_divergent_priors": "false", 2023-06-03T19:59:44.057 INFO:teuthology.orchestra.run.smithi089.stdout: "osd_inject_bad_map_crc_probability": "0.000000", 2023-06-03T19:59:44.057 INFO:teuthology.orchestra.run.smithi089.stdout: "osd_inject_failure_on_pg_removal": "false", 2023-06-03T19:59:44.057 INFO:teuthology.orchestra.run.smithi089.stdout: "osd_journal": "/var/lib/ceph/osd/ceph-a/journal", 2023-06-03T19:59:44.057 INFO:teuthology.orchestra.run.smithi089.stdout: "osd_journal_flush_on_shutdown": "true", 2023-06-03T19:59:44.057 INFO:teuthology.orchestra.run.smithi089.stdout: "osd_journal_size": "5120", 2023-06-03T19:59:44.058 INFO:teuthology.orchestra.run.smithi089.stdout: "osd_kill_backfill_at": "0", 2023-06-03T19:59:44.058 INFO:teuthology.orchestra.run.smithi089.stdout: "osd_loop_before_reset_tphandle": "64", 2023-06-03T19:59:44.058 INFO:teuthology.orchestra.run.smithi089.stdout: "osd_map_cache_size": "50", 2023-06-03T19:59:44.058 INFO:teuthology.orchestra.run.smithi089.stdout: "osd_map_dedup": "true", 2023-06-03T19:59:44.058 INFO:teuthology.orchestra.run.smithi089.stdout: "osd_map_message_max": "40", 2023-06-03T19:59:44.058 INFO:teuthology.orchestra.run.smithi089.stdout: "osd_map_message_max_bytes": "10485760", 2023-06-03T19:59:44.058 INFO:teuthology.orchestra.run.smithi089.stdout: "osd_map_share_max_epochs": "40", 2023-06-03T19:59:44.058 INFO:teuthology.orchestra.run.smithi089.stdout: "osd_max_attr_name_len": "100", 2023-06-03T19:59:44.059 INFO:teuthology.orchestra.run.smithi089.stdout: "osd_max_attr_size": "0", 2023-06-03T19:59:44.059 INFO:teuthology.orchestra.run.smithi089.stdout: "osd_max_backfills": "1", 2023-06-03T19:59:44.059 INFO:teuthology.orchestra.run.smithi089.stdout: "osd_max_markdown_count": "5", 2023-06-03T19:59:44.059 INFO:teuthology.orchestra.run.smithi089.stdout: "osd_max_markdown_period": "600", 2023-06-03T19:59:44.059 INFO:teuthology.orchestra.run.smithi089.stdout: "osd_max_object_name_len": "2048", 2023-06-03T19:59:44.059 INFO:teuthology.orchestra.run.smithi089.stdout: "osd_max_object_namespace_len": "256", 2023-06-03T19:59:44.059 INFO:teuthology.orchestra.run.smithi089.stdout: "osd_max_object_size": "134217728", 2023-06-03T19:59:44.059 INFO:teuthology.orchestra.run.smithi089.stdout: "osd_max_omap_bytes_per_request": "1073741824", 2023-06-03T19:59:44.060 INFO:teuthology.orchestra.run.smithi089.stdout: "osd_max_omap_entries_per_request": "1024", 2023-06-03T19:59:44.060 INFO:teuthology.orchestra.run.smithi089.stdout: "osd_max_pg_blocked_by": "16", 2023-06-03T19:59:44.060 INFO:teuthology.orchestra.run.smithi089.stdout: "osd_max_pg_log_entries": "10000", 2023-06-03T19:59:44.060 INFO:teuthology.orchestra.run.smithi089.stdout: "osd_max_pg_per_osd_hard_ratio": "3.000000", 2023-06-03T19:59:44.060 INFO:teuthology.orchestra.run.smithi089.stdout: "osd_max_pgls": "1024", 2023-06-03T19:59:44.060 INFO:teuthology.orchestra.run.smithi089.stdout: "osd_max_push_cost": "8388608", 2023-06-03T19:59:44.060 INFO:teuthology.orchestra.run.smithi089.stdout: "osd_max_push_objects": "10", 2023-06-03T19:59:44.060 INFO:teuthology.orchestra.run.smithi089.stdout: "osd_max_scrubs": "1", 2023-06-03T19:59:44.061 INFO:teuthology.orchestra.run.smithi089.stdout: "osd_max_snap_prune_intervals_per_epoch": "512", 2023-06-03T19:59:44.061 INFO:teuthology.orchestra.run.smithi089.stdout: "osd_max_trimming_pgs": "2", 2023-06-03T19:59:44.061 INFO:teuthology.orchestra.run.smithi089.stdout: "osd_max_write_op_reply_len": "64", 2023-06-03T19:59:44.061 INFO:teuthology.orchestra.run.smithi089.stdout: "osd_max_write_size": "90", 2023-06-03T19:59:44.061 INFO:teuthology.orchestra.run.smithi089.stdout: "osd_mclock_force_run_benchmark_on_init": "false", 2023-06-03T19:59:44.061 INFO:teuthology.orchestra.run.smithi089.stdout: "osd_mclock_iops_capacity_threshold_hdd": "500.000000", 2023-06-03T19:59:44.061 INFO:teuthology.orchestra.run.smithi089.stdout: "osd_mclock_iops_capacity_threshold_ssd": "80000.000000", 2023-06-03T19:59:44.061 INFO:teuthology.orchestra.run.smithi089.stdout: "osd_mclock_max_capacity_iops_hdd": "315.000000", 2023-06-03T19:59:44.062 INFO:teuthology.orchestra.run.smithi089.stdout: "osd_mclock_max_capacity_iops_ssd": "21500.000000", 2023-06-03T19:59:44.062 INFO:teuthology.orchestra.run.smithi089.stdout: "osd_mclock_max_sequential_bandwidth_hdd": "157286400", 2023-06-03T19:59:44.062 INFO:teuthology.orchestra.run.smithi089.stdout: "osd_mclock_max_sequential_bandwidth_ssd": "1258291200", 2023-06-03T19:59:44.062 INFO:teuthology.orchestra.run.smithi089.stdout: "osd_mclock_override_recovery_settings": "false", 2023-06-03T19:59:44.062 INFO:teuthology.orchestra.run.smithi089.stdout: "osd_mclock_profile": "balanced", 2023-06-03T19:59:44.062 INFO:teuthology.orchestra.run.smithi089.stdout: "osd_mclock_scheduler_anticipation_timeout": "0.000000", 2023-06-03T19:59:44.062 INFO:teuthology.orchestra.run.smithi089.stdout: "osd_mclock_scheduler_background_best_effort_lim": "0.000000", 2023-06-03T19:59:44.062 INFO:teuthology.orchestra.run.smithi089.stdout: "osd_mclock_scheduler_background_best_effort_res": "0.000000", 2023-06-03T19:59:44.063 INFO:teuthology.orchestra.run.smithi089.stdout: "osd_mclock_scheduler_background_best_effort_wgt": "1", 2023-06-03T19:59:44.063 INFO:teuthology.orchestra.run.smithi089.stdout: "osd_mclock_scheduler_background_recovery_lim": "0.000000", 2023-06-03T19:59:44.063 INFO:teuthology.orchestra.run.smithi089.stdout: "osd_mclock_scheduler_background_recovery_res": "0.000000", 2023-06-03T19:59:44.063 INFO:teuthology.orchestra.run.smithi089.stdout: "osd_mclock_scheduler_background_recovery_wgt": "1", 2023-06-03T19:59:44.063 INFO:teuthology.orchestra.run.smithi089.stdout: "osd_mclock_scheduler_client_lim": "0.000000", 2023-06-03T19:59:44.063 INFO:teuthology.orchestra.run.smithi089.stdout: "osd_mclock_scheduler_client_res": "0.000000", 2023-06-03T19:59:44.063 INFO:teuthology.orchestra.run.smithi089.stdout: "osd_mclock_scheduler_client_wgt": "1", 2023-06-03T19:59:44.063 INFO:teuthology.orchestra.run.smithi089.stdout: "osd_mclock_skip_benchmark": "false", 2023-06-03T19:59:44.064 INFO:teuthology.orchestra.run.smithi089.stdout: "osd_memory_base": "805306368", 2023-06-03T19:59:44.064 INFO:teuthology.orchestra.run.smithi089.stdout: "osd_memory_cache_min": "134217728", 2023-06-03T19:59:44.064 INFO:teuthology.orchestra.run.smithi089.stdout: "osd_memory_cache_resize_interval": "1.000000", 2023-06-03T19:59:44.064 INFO:teuthology.orchestra.run.smithi089.stdout: "osd_memory_expected_fragmentation": "0.150000", 2023-06-03T19:59:44.064 INFO:teuthology.orchestra.run.smithi089.stdout: "osd_memory_target": "4294967296", 2023-06-03T19:59:44.064 INFO:teuthology.orchestra.run.smithi089.stdout: "osd_memory_target_autotune": "false", 2023-06-03T19:59:44.064 INFO:teuthology.orchestra.run.smithi089.stdout: "osd_memory_target_cgroup_limit_ratio": "0.800000", 2023-06-03T19:59:44.065 INFO:teuthology.orchestra.run.smithi089.stdout: "osd_min_pg_log_entries": "250", 2023-06-03T19:59:44.065 INFO:teuthology.orchestra.run.smithi089.stdout: "osd_min_recovery_priority": "0", 2023-06-03T19:59:44.065 INFO:teuthology.orchestra.run.smithi089.stdout: "osd_mon_heartbeat_interval": "30", 2023-06-03T19:59:44.065 INFO:teuthology.orchestra.run.smithi089.stdout: "osd_mon_heartbeat_stat_stale": "3600", 2023-06-03T19:59:44.065 INFO:teuthology.orchestra.run.smithi089.stdout: "osd_mon_report_interval": "5", 2023-06-03T19:59:44.065 INFO:teuthology.orchestra.run.smithi089.stdout: "osd_mon_report_max_in_flight": "2", 2023-06-03T19:59:44.065 INFO:teuthology.orchestra.run.smithi089.stdout: "osd_mon_shutdown_timeout": "5.000000", 2023-06-03T19:59:44.065 INFO:teuthology.orchestra.run.smithi089.stdout: "osd_num_cache_shards": "32", 2023-06-03T19:59:44.066 INFO:teuthology.orchestra.run.smithi089.stdout: "osd_num_op_tracker_shard": "32", 2023-06-03T19:59:44.066 INFO:teuthology.orchestra.run.smithi089.stdout: "osd_numa_auto_affinity": "true", 2023-06-03T19:59:44.066 INFO:teuthology.orchestra.run.smithi089.stdout: "osd_numa_node": "-1", 2023-06-03T19:59:44.066 INFO:teuthology.orchestra.run.smithi089.stdout: "osd_numa_prefer_iface": "true", 2023-06-03T19:59:44.066 INFO:teuthology.orchestra.run.smithi089.stdout: "osd_object_clean_region_max_num_intervals": "10", 2023-06-03T19:59:44.066 INFO:teuthology.orchestra.run.smithi089.stdout: "osd_objecter_finishers": "1", 2023-06-03T19:59:44.066 INFO:teuthology.orchestra.run.smithi089.stdout: "osd_objectstore": "bluestore", 2023-06-03T19:59:44.066 INFO:teuthology.orchestra.run.smithi089.stdout: "osd_objectstore_fuse": "false", 2023-06-03T19:59:44.067 INFO:teuthology.orchestra.run.smithi089.stdout: "osd_objectstore_tracing": "false", 2023-06-03T19:59:44.067 INFO:teuthology.orchestra.run.smithi089.stdout: "osd_op_complaint_time": "30.000000", 2023-06-03T19:59:44.067 INFO:teuthology.orchestra.run.smithi089.stdout: "osd_op_history_duration": "600", 2023-06-03T19:59:44.067 INFO:teuthology.orchestra.run.smithi089.stdout: "osd_op_history_size": "20", 2023-06-03T19:59:44.067 INFO:teuthology.orchestra.run.smithi089.stdout: "osd_op_history_slow_op_size": "20", 2023-06-03T19:59:44.067 INFO:teuthology.orchestra.run.smithi089.stdout: "osd_op_history_slow_op_threshold": "10.000000", 2023-06-03T19:59:44.067 INFO:teuthology.orchestra.run.smithi089.stdout: "osd_op_log_threshold": "5", 2023-06-03T19:59:44.067 INFO:teuthology.orchestra.run.smithi089.stdout: "osd_op_num_shards": "0", 2023-06-03T19:59:44.068 INFO:teuthology.orchestra.run.smithi089.stdout: "osd_op_num_shards_hdd": "5", 2023-06-03T19:59:44.068 INFO:teuthology.orchestra.run.smithi089.stdout: "osd_op_num_shards_ssd": "8", 2023-06-03T19:59:44.068 INFO:teuthology.orchestra.run.smithi089.stdout: "osd_op_num_threads_per_shard": "0", 2023-06-03T19:59:44.068 INFO:teuthology.orchestra.run.smithi089.stdout: "osd_op_num_threads_per_shard_hdd": "1", 2023-06-03T19:59:44.068 INFO:teuthology.orchestra.run.smithi089.stdout: "osd_op_num_threads_per_shard_ssd": "2", 2023-06-03T19:59:44.068 INFO:teuthology.orchestra.run.smithi089.stdout: "osd_op_pq_max_tokens_per_priority": "4194304", 2023-06-03T19:59:44.068 INFO:teuthology.orchestra.run.smithi089.stdout: "osd_op_pq_min_cost": "65536", 2023-06-03T19:59:44.068 INFO:teuthology.orchestra.run.smithi089.stdout: "osd_op_queue": "mclock_scheduler", 2023-06-03T19:59:44.069 INFO:teuthology.orchestra.run.smithi089.stdout: "osd_op_queue_cut_off": "high", 2023-06-03T19:59:44.069 INFO:teuthology.orchestra.run.smithi089.stdout: "osd_op_thread_suicide_timeout": "150", 2023-06-03T19:59:44.069 INFO:teuthology.orchestra.run.smithi089.stdout: "osd_op_thread_timeout": "15", 2023-06-03T19:59:44.069 INFO:teuthology.orchestra.run.smithi089.stdout: "osd_open_classes_on_start": "true", 2023-06-03T19:59:44.069 INFO:teuthology.orchestra.run.smithi089.stdout: "osd_os_flags": "0", 2023-06-03T19:59:44.069 INFO:teuthology.orchestra.run.smithi089.stdout: "osd_peering_op_priority": "255", 2023-06-03T19:59:44.069 INFO:teuthology.orchestra.run.smithi089.stdout: "osd_pg_delete_cost": "1048576", 2023-06-03T19:59:44.069 INFO:teuthology.orchestra.run.smithi089.stdout: "osd_pg_delete_priority": "5", 2023-06-03T19:59:44.070 INFO:teuthology.orchestra.run.smithi089.stdout: "osd_pg_epoch_max_lag_factor": "2.000000", 2023-06-03T19:59:44.070 INFO:teuthology.orchestra.run.smithi089.stdout: "osd_pg_epoch_persisted_max_stale": "40", 2023-06-03T19:59:44.070 INFO:teuthology.orchestra.run.smithi089.stdout: "osd_pg_log_dups_tracked": "3000", 2023-06-03T19:59:44.070 INFO:teuthology.orchestra.run.smithi089.stdout: "osd_pg_log_trim_max": "10000", 2023-06-03T19:59:44.070 INFO:teuthology.orchestra.run.smithi089.stdout: "osd_pg_log_trim_min": "100", 2023-06-03T19:59:44.070 INFO:teuthology.orchestra.run.smithi089.stdout: "osd_pg_max_concurrent_snap_trims": "2", 2023-06-03T19:59:44.070 INFO:teuthology.orchestra.run.smithi089.stdout: "osd_pg_object_context_cache_count": "64", 2023-06-03T19:59:44.070 INFO:teuthology.orchestra.run.smithi089.stdout: "osd_pg_stat_report_interval_max": "500", 2023-06-03T19:59:44.071 INFO:teuthology.orchestra.run.smithi089.stdout: "osd_pool_default_cache_max_evict_check_size": "10", 2023-06-03T19:59:44.071 INFO:teuthology.orchestra.run.smithi089.stdout: "osd_pool_default_cache_min_evict_age": "0", 2023-06-03T19:59:44.071 INFO:teuthology.orchestra.run.smithi089.stdout: "osd_pool_default_cache_min_flush_age": "0", 2023-06-03T19:59:44.071 INFO:teuthology.orchestra.run.smithi089.stdout: "osd_pool_default_cache_target_dirty_high_ratio": "0.600000", 2023-06-03T19:59:44.071 INFO:teuthology.orchestra.run.smithi089.stdout: "osd_pool_default_cache_target_dirty_ratio": "0.400000", 2023-06-03T19:59:44.071 INFO:teuthology.orchestra.run.smithi089.stdout: "osd_pool_default_cache_target_full_ratio": "0.800000", 2023-06-03T19:59:44.071 INFO:teuthology.orchestra.run.smithi089.stdout: "osd_pool_default_crimson": "false", 2023-06-03T19:59:44.071 INFO:teuthology.orchestra.run.smithi089.stdout: "osd_pool_default_crush_rule": "-1", 2023-06-03T19:59:44.072 INFO:teuthology.orchestra.run.smithi089.stdout: "osd_pool_default_ec_fast_read": "false", 2023-06-03T19:59:44.072 INFO:teuthology.orchestra.run.smithi089.stdout: "osd_pool_default_erasure_code_profile": "plugin=jerasure technique=reed_sol_van k=2 m=1 crush-failure-domain=osd", 2023-06-03T19:59:44.072 INFO:teuthology.orchestra.run.smithi089.stdout: "osd_pool_default_flag_bulk": "false", 2023-06-03T19:59:44.072 INFO:teuthology.orchestra.run.smithi089.stdout: "osd_pool_default_flag_hashpspool": "true", 2023-06-03T19:59:44.072 INFO:teuthology.orchestra.run.smithi089.stdout: "osd_pool_default_flag_nodelete": "false", 2023-06-03T19:59:44.072 INFO:teuthology.orchestra.run.smithi089.stdout: "osd_pool_default_flag_nopgchange": "false", 2023-06-03T19:59:44.072 INFO:teuthology.orchestra.run.smithi089.stdout: "osd_pool_default_flag_nosizechange": "false", 2023-06-03T19:59:44.073 INFO:teuthology.orchestra.run.smithi089.stdout: "osd_pool_default_flags": "0", 2023-06-03T19:59:44.073 INFO:teuthology.orchestra.run.smithi089.stdout: "osd_pool_default_hit_set_bloom_fpp": "0.050000", 2023-06-03T19:59:44.073 INFO:teuthology.orchestra.run.smithi089.stdout: "osd_pool_default_min_size": "0", 2023-06-03T19:59:44.073 INFO:teuthology.orchestra.run.smithi089.stdout: "osd_pool_default_pg_autoscale_mode": "off", 2023-06-03T19:59:44.073 INFO:teuthology.orchestra.run.smithi089.stdout: "osd_pool_default_pg_num": "32", 2023-06-03T19:59:44.073 INFO:teuthology.orchestra.run.smithi089.stdout: "osd_pool_default_pgp_num": "0", 2023-06-03T19:59:44.073 INFO:teuthology.orchestra.run.smithi089.stdout: "osd_pool_default_read_lease_ratio": "0.800000", 2023-06-03T19:59:44.073 INFO:teuthology.orchestra.run.smithi089.stdout: "osd_pool_default_size": "3", 2023-06-03T19:59:44.074 INFO:teuthology.orchestra.run.smithi089.stdout: "osd_pool_default_type": "replicated", 2023-06-03T19:59:44.074 INFO:teuthology.orchestra.run.smithi089.stdout: "osd_pool_erasure_code_stripe_unit": "4096", 2023-06-03T19:59:44.074 INFO:teuthology.orchestra.run.smithi089.stdout: "osd_pool_use_gmt_hitset": "true", 2023-06-03T19:59:44.074 INFO:teuthology.orchestra.run.smithi089.stdout: "osd_push_per_object_cost": "1000", 2023-06-03T19:59:44.074 INFO:teuthology.orchestra.run.smithi089.stdout: "osd_read_ec_check_for_errors": "false", 2023-06-03T19:59:44.074 INFO:teuthology.orchestra.run.smithi089.stdout: "osd_recover_clone_overlap": "true", 2023-06-03T19:59:44.074 INFO:teuthology.orchestra.run.smithi089.stdout: "osd_recover_clone_overlap_limit": "10", 2023-06-03T19:59:44.074 INFO:teuthology.orchestra.run.smithi089.stdout: "osd_recovery_cost": "20971520", 2023-06-03T19:59:44.075 INFO:teuthology.orchestra.run.smithi089.stdout: "osd_recovery_delay_start": "0.000000", 2023-06-03T19:59:44.075 INFO:teuthology.orchestra.run.smithi089.stdout: "osd_recovery_max_active": "0", 2023-06-03T19:59:44.075 INFO:teuthology.orchestra.run.smithi089.stdout: "osd_recovery_max_active_hdd": "3", 2023-06-03T19:59:44.075 INFO:teuthology.orchestra.run.smithi089.stdout: "osd_recovery_max_active_ssd": "10", 2023-06-03T19:59:44.075 INFO:teuthology.orchestra.run.smithi089.stdout: "osd_recovery_max_chunk": "8388608", 2023-06-03T19:59:44.075 INFO:teuthology.orchestra.run.smithi089.stdout: "osd_recovery_max_omap_entries_per_chunk": "8096", 2023-06-03T19:59:44.075 INFO:teuthology.orchestra.run.smithi089.stdout: "osd_recovery_max_single_start": "1", 2023-06-03T19:59:44.075 INFO:teuthology.orchestra.run.smithi089.stdout: "osd_recovery_op_priority": "3", 2023-06-03T19:59:44.076 INFO:teuthology.orchestra.run.smithi089.stdout: "osd_recovery_op_warn_multiple": "16", 2023-06-03T19:59:44.076 INFO:teuthology.orchestra.run.smithi089.stdout: "osd_recovery_priority": "5", 2023-06-03T19:59:44.076 INFO:teuthology.orchestra.run.smithi089.stdout: "osd_recovery_retry_interval": "30.000000", 2023-06-03T19:59:44.076 INFO:teuthology.orchestra.run.smithi089.stdout: "osd_recovery_sleep": "0.000000", 2023-06-03T19:59:44.076 INFO:teuthology.orchestra.run.smithi089.stdout: "osd_recovery_sleep_hdd": "0.100000", 2023-06-03T19:59:44.076 INFO:teuthology.orchestra.run.smithi089.stdout: "osd_recovery_sleep_hybrid": "0.025000", 2023-06-03T19:59:44.076 INFO:teuthology.orchestra.run.smithi089.stdout: "osd_recovery_sleep_ssd": "0.000000", 2023-06-03T19:59:44.076 INFO:teuthology.orchestra.run.smithi089.stdout: "osd_repair_during_recovery": "false", 2023-06-03T19:59:44.077 INFO:teuthology.orchestra.run.smithi089.stdout: "osd_requested_scrub_priority": "120", 2023-06-03T19:59:44.077 INFO:teuthology.orchestra.run.smithi089.stdout: "osd_rocksdb_iterator_bounds_enabled": "true", 2023-06-03T19:59:44.077 INFO:teuthology.orchestra.run.smithi089.stdout: "osd_rollback_to_cluster_snap": "", 2023-06-03T19:59:44.077 INFO:teuthology.orchestra.run.smithi089.stdout: "osd_scrub_auto_repair": "false", 2023-06-03T19:59:44.077 INFO:teuthology.orchestra.run.smithi089.stdout: "osd_scrub_auto_repair_num_errors": "5", 2023-06-03T19:59:44.077 INFO:teuthology.orchestra.run.smithi089.stdout: "osd_scrub_backoff_ratio": "0.660000", 2023-06-03T19:59:44.077 INFO:teuthology.orchestra.run.smithi089.stdout: "osd_scrub_begin_hour": "0", 2023-06-03T19:59:44.077 INFO:teuthology.orchestra.run.smithi089.stdout: "osd_scrub_begin_week_day": "0", 2023-06-03T19:59:44.078 INFO:teuthology.orchestra.run.smithi089.stdout: "osd_scrub_chunk_max": "25", 2023-06-03T19:59:44.078 INFO:teuthology.orchestra.run.smithi089.stdout: "osd_scrub_chunk_min": "5", 2023-06-03T19:59:44.078 INFO:teuthology.orchestra.run.smithi089.stdout: "osd_scrub_cost": "52428800", 2023-06-03T19:59:44.078 INFO:teuthology.orchestra.run.smithi089.stdout: "osd_scrub_during_recovery": "false", 2023-06-03T19:59:44.078 INFO:teuthology.orchestra.run.smithi089.stdout: "osd_scrub_end_hour": "0", 2023-06-03T19:59:44.078 INFO:teuthology.orchestra.run.smithi089.stdout: "osd_scrub_end_week_day": "0", 2023-06-03T19:59:44.078 INFO:teuthology.orchestra.run.smithi089.stdout: "osd_scrub_event_cost": "4096", 2023-06-03T19:59:44.078 INFO:teuthology.orchestra.run.smithi089.stdout: "osd_scrub_extended_sleep": "0.000000", 2023-06-03T19:59:44.079 INFO:teuthology.orchestra.run.smithi089.stdout: "osd_scrub_interval_randomize_ratio": "0.500000", 2023-06-03T19:59:44.079 INFO:teuthology.orchestra.run.smithi089.stdout: "osd_scrub_invalid_stats": "true", 2023-06-03T19:59:44.079 INFO:teuthology.orchestra.run.smithi089.stdout: "osd_scrub_load_threshold": "0.500000", 2023-06-03T19:59:44.079 INFO:teuthology.orchestra.run.smithi089.stdout: "osd_scrub_max_interval": "604800.000000", 2023-06-03T19:59:44.079 INFO:teuthology.orchestra.run.smithi089.stdout: "osd_scrub_max_preemptions": "5", 2023-06-03T19:59:44.079 INFO:teuthology.orchestra.run.smithi089.stdout: "osd_scrub_min_interval": "86400.000000", 2023-06-03T19:59:44.079 INFO:teuthology.orchestra.run.smithi089.stdout: "osd_scrub_priority": "5", 2023-06-03T19:59:44.080 INFO:teuthology.orchestra.run.smithi089.stdout: "osd_scrub_reservation_timeout": "5000", 2023-06-03T19:59:44.080 INFO:teuthology.orchestra.run.smithi089.stdout: "osd_scrub_sleep": "0.000000", 2023-06-03T19:59:44.080 INFO:teuthology.orchestra.run.smithi089.stdout: "osd_scrub_slow_reservation_response": "2200", 2023-06-03T19:59:44.080 INFO:teuthology.orchestra.run.smithi089.stdout: "osd_shallow_scrub_chunk_max": "100", 2023-06-03T19:59:44.080 INFO:teuthology.orchestra.run.smithi089.stdout: "osd_shallow_scrub_chunk_min": "50", 2023-06-03T19:59:44.080 INFO:teuthology.orchestra.run.smithi089.stdout: "osd_shutdown_pgref_assert": "false", 2023-06-03T19:59:44.080 INFO:teuthology.orchestra.run.smithi089.stdout: "osd_skip_data_digest": "false", 2023-06-03T19:59:44.080 INFO:teuthology.orchestra.run.smithi089.stdout: "osd_smart_report_timeout": "5", 2023-06-03T19:59:44.081 INFO:teuthology.orchestra.run.smithi089.stdout: "osd_snap_trim_cost": "1048576", 2023-06-03T19:59:44.081 INFO:teuthology.orchestra.run.smithi089.stdout: "osd_snap_trim_priority": "5", 2023-06-03T19:59:44.081 INFO:teuthology.orchestra.run.smithi089.stdout: "osd_snap_trim_sleep": "0.000000", 2023-06-03T19:59:44.081 INFO:teuthology.orchestra.run.smithi089.stdout: "osd_snap_trim_sleep_hdd": "5.000000", 2023-06-03T19:59:44.081 INFO:teuthology.orchestra.run.smithi089.stdout: "osd_snap_trim_sleep_hybrid": "2.000000", 2023-06-03T19:59:44.081 INFO:teuthology.orchestra.run.smithi089.stdout: "osd_snap_trim_sleep_ssd": "0.000000", 2023-06-03T19:59:44.081 INFO:teuthology.orchestra.run.smithi089.stdout: "osd_stats_update_period_not_scrubbing": "120", 2023-06-03T19:59:44.081 INFO:teuthology.orchestra.run.smithi089.stdout: "osd_stats_update_period_scrubbing": "15", 2023-06-03T19:59:44.082 INFO:teuthology.orchestra.run.smithi089.stdout: "osd_target_pg_log_entries_per_osd": "300000", 2023-06-03T19:59:44.082 INFO:teuthology.orchestra.run.smithi089.stdout: "osd_target_transaction_size": "30", 2023-06-03T19:59:44.082 INFO:teuthology.orchestra.run.smithi089.stdout: "osd_tier_default_cache_hit_set_count": "4", 2023-06-03T19:59:44.082 INFO:teuthology.orchestra.run.smithi089.stdout: "osd_tier_default_cache_hit_set_grade_decay_rate": "20", 2023-06-03T19:59:44.082 INFO:teuthology.orchestra.run.smithi089.stdout: "osd_tier_default_cache_hit_set_period": "1200", 2023-06-03T19:59:44.082 INFO:teuthology.orchestra.run.smithi089.stdout: "osd_tier_default_cache_hit_set_search_last_n": "1", 2023-06-03T19:59:44.082 INFO:teuthology.orchestra.run.smithi089.stdout: "osd_tier_default_cache_hit_set_type": "bloom", 2023-06-03T19:59:44.082 INFO:teuthology.orchestra.run.smithi089.stdout: "osd_tier_default_cache_min_read_recency_for_promote": "1", 2023-06-03T19:59:44.083 INFO:teuthology.orchestra.run.smithi089.stdout: "osd_tier_default_cache_min_write_recency_for_promote": "1", 2023-06-03T19:59:44.083 INFO:teuthology.orchestra.run.smithi089.stdout: "osd_tier_default_cache_mode": "writeback", 2023-06-03T19:59:44.083 INFO:teuthology.orchestra.run.smithi089.stdout: "osd_tier_promote_max_bytes_sec": "5242880", 2023-06-03T19:59:44.083 INFO:teuthology.orchestra.run.smithi089.stdout: "osd_tier_promote_max_objects_sec": "25", 2023-06-03T19:59:44.083 INFO:teuthology.orchestra.run.smithi089.stdout: "osd_tracing": "false", 2023-06-03T19:59:44.083 INFO:teuthology.orchestra.run.smithi089.stdout: "osd_use_stale_snap": "false", 2023-06-03T19:59:44.083 INFO:teuthology.orchestra.run.smithi089.stdout: "osd_uuid": "00000000-0000-0000-0000-000000000000", 2023-06-03T19:59:44.084 INFO:teuthology.orchestra.run.smithi089.stdout: "osdc_blkin_trace_all": "false", 2023-06-03T19:59:44.084 INFO:teuthology.orchestra.run.smithi089.stdout: "paxos_kill_at": "0", 2023-06-03T19:59:44.084 INFO:teuthology.orchestra.run.smithi089.stdout: "paxos_max_join_drift": "10", 2023-06-03T19:59:44.084 INFO:teuthology.orchestra.run.smithi089.stdout: "paxos_min": "500", 2023-06-03T19:59:44.084 INFO:teuthology.orchestra.run.smithi089.stdout: "paxos_min_wait": "0.050000", 2023-06-03T19:59:44.084 INFO:teuthology.orchestra.run.smithi089.stdout: "paxos_propose_interval": "1.000000", 2023-06-03T19:59:44.084 INFO:teuthology.orchestra.run.smithi089.stdout: "paxos_service_trim_max": "500", 2023-06-03T19:59:44.084 INFO:teuthology.orchestra.run.smithi089.stdout: "paxos_service_trim_max_multiplier": "20", 2023-06-03T19:59:44.085 INFO:teuthology.orchestra.run.smithi089.stdout: "paxos_service_trim_min": "250", 2023-06-03T19:59:44.085 INFO:teuthology.orchestra.run.smithi089.stdout: "paxos_stash_full_interval": "25", 2023-06-03T19:59:44.085 INFO:teuthology.orchestra.run.smithi089.stdout: "paxos_trim_max": "500", 2023-06-03T19:59:44.085 INFO:teuthology.orchestra.run.smithi089.stdout: "paxos_trim_min": "250", 2023-06-03T19:59:44.085 INFO:teuthology.orchestra.run.smithi089.stdout: "perf": "true", 2023-06-03T19:59:44.085 INFO:teuthology.orchestra.run.smithi089.stdout: "pid_file": "", 2023-06-03T19:59:44.085 INFO:teuthology.orchestra.run.smithi089.stdout: "plugin_crypto_accelerator": "crypto_isal", 2023-06-03T19:59:44.085 INFO:teuthology.orchestra.run.smithi089.stdout: "plugin_dir": "/usr/lib64/ceph", 2023-06-03T19:59:44.086 INFO:teuthology.orchestra.run.smithi089.stdout: "public_addr": "-", 2023-06-03T19:59:44.086 INFO:teuthology.orchestra.run.smithi089.stdout: "public_addrv": "", 2023-06-03T19:59:44.086 INFO:teuthology.orchestra.run.smithi089.stdout: "public_bind_addr": "-", 2023-06-03T19:59:44.086 INFO:teuthology.orchestra.run.smithi089.stdout: "public_network": "172.21.15.254/32,172.21.0.0/20", 2023-06-03T19:59:44.086 INFO:teuthology.orchestra.run.smithi089.stdout: "public_network_interface": "", 2023-06-03T19:59:44.086 INFO:teuthology.orchestra.run.smithi089.stdout: "qat_compressor_enabled": "false", 2023-06-03T19:59:44.086 INFO:teuthology.orchestra.run.smithi089.stdout: "qat_compressor_session_max_number": "256", 2023-06-03T19:59:44.086 INFO:teuthology.orchestra.run.smithi089.stdout: "rados_mon_op_timeout": "0", 2023-06-03T19:59:44.087 INFO:teuthology.orchestra.run.smithi089.stdout: "rados_osd_op_timeout": "0", 2023-06-03T19:59:44.087 INFO:teuthology.orchestra.run.smithi089.stdout: "rados_tracing": "false", 2023-06-03T19:59:44.087 INFO:teuthology.orchestra.run.smithi089.stdout: "rbd_atime_update_interval": "60", 2023-06-03T19:59:44.087 INFO:teuthology.orchestra.run.smithi089.stdout: "rbd_auto_exclusive_lock_until_manual_request": "true", 2023-06-03T19:59:44.087 INFO:teuthology.orchestra.run.smithi089.stdout: "rbd_balance_parent_reads": "false", 2023-06-03T19:59:44.087 INFO:teuthology.orchestra.run.smithi089.stdout: "rbd_balance_snap_reads": "false", 2023-06-03T19:59:44.087 INFO:teuthology.orchestra.run.smithi089.stdout: "rbd_blkin_trace_all": "false", 2023-06-03T19:59:44.087 INFO:teuthology.orchestra.run.smithi089.stdout: "rbd_blocklist_expire_seconds": "0", 2023-06-03T19:59:44.088 INFO:teuthology.orchestra.run.smithi089.stdout: "rbd_blocklist_on_break_lock": "true", 2023-06-03T19:59:44.088 INFO:teuthology.orchestra.run.smithi089.stdout: "rbd_cache": "true", 2023-06-03T19:59:44.088 INFO:teuthology.orchestra.run.smithi089.stdout: "rbd_cache_block_writes_upfront": "false", 2023-06-03T19:59:44.088 INFO:teuthology.orchestra.run.smithi089.stdout: "rbd_cache_max_dirty": "25165824", 2023-06-03T19:59:44.088 INFO:teuthology.orchestra.run.smithi089.stdout: "rbd_cache_max_dirty_age": "1.000000", 2023-06-03T19:59:44.088 INFO:teuthology.orchestra.run.smithi089.stdout: "rbd_cache_max_dirty_object": "0", 2023-06-03T19:59:44.088 INFO:teuthology.orchestra.run.smithi089.stdout: "rbd_cache_policy": "writearound", 2023-06-03T19:59:44.088 INFO:teuthology.orchestra.run.smithi089.stdout: "rbd_cache_size": "33554432", 2023-06-03T19:59:44.089 INFO:teuthology.orchestra.run.smithi089.stdout: "rbd_cache_target_dirty": "16777216", 2023-06-03T19:59:44.089 INFO:teuthology.orchestra.run.smithi089.stdout: "rbd_cache_writethrough_until_flush": "true", 2023-06-03T19:59:44.089 INFO:teuthology.orchestra.run.smithi089.stdout: "rbd_clone_copy_on_read": "false", 2023-06-03T19:59:44.089 INFO:teuthology.orchestra.run.smithi089.stdout: "rbd_compression_hint": "none", 2023-06-03T19:59:44.089 INFO:teuthology.orchestra.run.smithi089.stdout: "rbd_concurrent_management_ops": "10", 2023-06-03T19:59:44.089 INFO:teuthology.orchestra.run.smithi089.stdout: "rbd_config_pool_override_update_timestamp": "0", 2023-06-03T19:59:44.089 INFO:teuthology.orchestra.run.smithi089.stdout: "rbd_default_clone_format": "auto", 2023-06-03T19:59:44.089 INFO:teuthology.orchestra.run.smithi089.stdout: "rbd_default_data_pool": "", 2023-06-03T19:59:44.090 INFO:teuthology.orchestra.run.smithi089.stdout: "rbd_default_features": "61", 2023-06-03T19:59:44.090 INFO:teuthology.orchestra.run.smithi089.stdout: "rbd_default_format": "2", 2023-06-03T19:59:44.090 INFO:teuthology.orchestra.run.smithi089.stdout: "rbd_default_map_options": "", 2023-06-03T19:59:44.090 INFO:teuthology.orchestra.run.smithi089.stdout: "rbd_default_order": "22", 2023-06-03T19:59:44.090 INFO:teuthology.orchestra.run.smithi089.stdout: "rbd_default_pool": "rbd", 2023-06-03T19:59:44.090 INFO:teuthology.orchestra.run.smithi089.stdout: "rbd_default_snapshot_quiesce_mode": "required", 2023-06-03T19:59:44.090 INFO:teuthology.orchestra.run.smithi089.stdout: "rbd_default_stripe_count": "0", 2023-06-03T19:59:44.090 INFO:teuthology.orchestra.run.smithi089.stdout: "rbd_default_stripe_unit": "0", 2023-06-03T19:59:44.091 INFO:teuthology.orchestra.run.smithi089.stdout: "rbd_disable_zero_copy_writes": "true", 2023-06-03T19:59:44.091 INFO:teuthology.orchestra.run.smithi089.stdout: "rbd_discard_granularity_bytes": "65536", 2023-06-03T19:59:44.091 INFO:teuthology.orchestra.run.smithi089.stdout: "rbd_discard_on_zeroed_write_same": "true", 2023-06-03T19:59:44.091 INFO:teuthology.orchestra.run.smithi089.stdout: "rbd_enable_alloc_hint": "true", 2023-06-03T19:59:44.091 INFO:teuthology.orchestra.run.smithi089.stdout: "rbd_invalidate_object_map_on_timeout": "true", 2023-06-03T19:59:44.091 INFO:teuthology.orchestra.run.smithi089.stdout: "rbd_io_scheduler": "simple", 2023-06-03T19:59:44.091 INFO:teuthology.orchestra.run.smithi089.stdout: "rbd_io_scheduler_simple_max_delay": "0", 2023-06-03T19:59:44.092 INFO:teuthology.orchestra.run.smithi089.stdout: "rbd_journal_commit_age": "5.000000", 2023-06-03T19:59:44.092 INFO:teuthology.orchestra.run.smithi089.stdout: "rbd_journal_max_concurrent_object_sets": "0", 2023-06-03T19:59:44.092 INFO:teuthology.orchestra.run.smithi089.stdout: "rbd_journal_max_payload_bytes": "16384", 2023-06-03T19:59:44.092 INFO:teuthology.orchestra.run.smithi089.stdout: "rbd_journal_object_flush_age": "0.000000", 2023-06-03T19:59:44.092 INFO:teuthology.orchestra.run.smithi089.stdout: "rbd_journal_object_flush_bytes": "1048576", 2023-06-03T19:59:44.092 INFO:teuthology.orchestra.run.smithi089.stdout: "rbd_journal_object_flush_interval": "0", 2023-06-03T19:59:44.092 INFO:teuthology.orchestra.run.smithi089.stdout: "rbd_journal_object_max_in_flight_appends": "0", 2023-06-03T19:59:44.092 INFO:teuthology.orchestra.run.smithi089.stdout: "rbd_journal_object_writethrough_until_flush": "true", 2023-06-03T19:59:44.093 INFO:teuthology.orchestra.run.smithi089.stdout: "rbd_journal_order": "24", 2023-06-03T19:59:44.093 INFO:teuthology.orchestra.run.smithi089.stdout: "rbd_journal_pool": "", 2023-06-03T19:59:44.093 INFO:teuthology.orchestra.run.smithi089.stdout: "rbd_journal_splay_width": "4", 2023-06-03T19:59:44.093 INFO:teuthology.orchestra.run.smithi089.stdout: "rbd_localize_parent_reads": "false", 2023-06-03T19:59:44.093 INFO:teuthology.orchestra.run.smithi089.stdout: "rbd_localize_snap_reads": "false", 2023-06-03T19:59:44.093 INFO:teuthology.orchestra.run.smithi089.stdout: "rbd_mirror_concurrent_image_deletions": "1", 2023-06-03T19:59:44.093 INFO:teuthology.orchestra.run.smithi089.stdout: "rbd_mirror_concurrent_image_syncs": "5", 2023-06-03T19:59:44.093 INFO:teuthology.orchestra.run.smithi089.stdout: "rbd_mirror_delete_retry_interval": "30.000000", 2023-06-03T19:59:44.094 INFO:teuthology.orchestra.run.smithi089.stdout: "rbd_mirror_image_perf_stats_prio": "5", 2023-06-03T19:59:44.094 INFO:teuthology.orchestra.run.smithi089.stdout: "rbd_mirror_image_policy_migration_throttle": "300", 2023-06-03T19:59:44.094 INFO:teuthology.orchestra.run.smithi089.stdout: "rbd_mirror_image_policy_rebalance_timeout": "0.000000", 2023-06-03T19:59:44.094 INFO:teuthology.orchestra.run.smithi089.stdout: "rbd_mirror_image_policy_type": "simple", 2023-06-03T19:59:44.094 INFO:teuthology.orchestra.run.smithi089.stdout: "rbd_mirror_image_policy_update_throttle_interval": "1.000000", 2023-06-03T19:59:44.094 INFO:teuthology.orchestra.run.smithi089.stdout: "rbd_mirror_image_state_check_interval": "30", 2023-06-03T19:59:44.094 INFO:teuthology.orchestra.run.smithi089.stdout: "rbd_mirror_journal_commit_age": "5.000000", 2023-06-03T19:59:44.094 INFO:teuthology.orchestra.run.smithi089.stdout: "rbd_mirror_journal_poll_age": "5.000000", 2023-06-03T19:59:44.095 INFO:teuthology.orchestra.run.smithi089.stdout: "rbd_mirror_leader_heartbeat_interval": "5", 2023-06-03T19:59:44.095 INFO:teuthology.orchestra.run.smithi089.stdout: "rbd_mirror_leader_max_acquire_attempts_before_break": "3", 2023-06-03T19:59:44.095 INFO:teuthology.orchestra.run.smithi089.stdout: "rbd_mirror_leader_max_missed_heartbeats": "2", 2023-06-03T19:59:44.095 INFO:teuthology.orchestra.run.smithi089.stdout: "rbd_mirror_memory_autotune": "true", 2023-06-03T19:59:44.095 INFO:teuthology.orchestra.run.smithi089.stdout: "rbd_mirror_memory_base": "805306368", 2023-06-03T19:59:44.095 INFO:teuthology.orchestra.run.smithi089.stdout: "rbd_mirror_memory_cache_autotune_interval": "30.000000", 2023-06-03T19:59:44.095 INFO:teuthology.orchestra.run.smithi089.stdout: "rbd_mirror_memory_cache_min": "134217728", 2023-06-03T19:59:44.095 INFO:teuthology.orchestra.run.smithi089.stdout: "rbd_mirror_memory_cache_resize_interval": "5.000000", 2023-06-03T19:59:44.096 INFO:teuthology.orchestra.run.smithi089.stdout: "rbd_mirror_memory_expected_fragmentation": "0.150000", 2023-06-03T19:59:44.096 INFO:teuthology.orchestra.run.smithi089.stdout: "rbd_mirror_memory_target": "4294967296", 2023-06-03T19:59:44.096 INFO:teuthology.orchestra.run.smithi089.stdout: "rbd_mirror_perf_stats_prio": "5", 2023-06-03T19:59:44.096 INFO:teuthology.orchestra.run.smithi089.stdout: "rbd_mirror_pool_replayers_refresh_interval": "30", 2023-06-03T19:59:44.096 INFO:teuthology.orchestra.run.smithi089.stdout: "rbd_mirror_sync_point_update_age": "30.000000", 2023-06-03T19:59:44.096 INFO:teuthology.orchestra.run.smithi089.stdout: "rbd_mirroring_delete_delay": "0", 2023-06-03T19:59:44.096 INFO:teuthology.orchestra.run.smithi089.stdout: "rbd_mirroring_max_mirroring_snapshots": "5", 2023-06-03T19:59:44.097 INFO:teuthology.orchestra.run.smithi089.stdout: "rbd_mirroring_replay_delay": "0", 2023-06-03T19:59:44.097 INFO:teuthology.orchestra.run.smithi089.stdout: "rbd_mirroring_resync_after_disconnect": "false", 2023-06-03T19:59:44.097 INFO:teuthology.orchestra.run.smithi089.stdout: "rbd_move_parent_to_trash_on_remove": "false", 2023-06-03T19:59:44.097 INFO:teuthology.orchestra.run.smithi089.stdout: "rbd_move_to_trash_on_remove": "false", 2023-06-03T19:59:44.097 INFO:teuthology.orchestra.run.smithi089.stdout: "rbd_move_to_trash_on_remove_expire_seconds": "0", 2023-06-03T19:59:44.097 INFO:teuthology.orchestra.run.smithi089.stdout: "rbd_mtime_update_interval": "60", 2023-06-03T19:59:44.097 INFO:teuthology.orchestra.run.smithi089.stdout: "rbd_non_blocking_aio": "true", 2023-06-03T19:59:44.097 INFO:teuthology.orchestra.run.smithi089.stdout: "rbd_op_thread_timeout": "60", 2023-06-03T19:59:44.098 INFO:teuthology.orchestra.run.smithi089.stdout: "rbd_op_threads": "1", 2023-06-03T19:59:44.098 INFO:teuthology.orchestra.run.smithi089.stdout: "rbd_parent_cache_enabled": "false", 2023-06-03T19:59:44.098 INFO:teuthology.orchestra.run.smithi089.stdout: "rbd_persistent_cache_mode": "disabled", 2023-06-03T19:59:44.098 INFO:teuthology.orchestra.run.smithi089.stdout: "rbd_persistent_cache_path": "/tmp", 2023-06-03T19:59:44.098 INFO:teuthology.orchestra.run.smithi089.stdout: "rbd_persistent_cache_size": "1073741824", 2023-06-03T19:59:44.098 INFO:teuthology.orchestra.run.smithi089.stdout: "rbd_plugins": "", 2023-06-03T19:59:44.098 INFO:teuthology.orchestra.run.smithi089.stdout: "rbd_qos_bps_burst": "0", 2023-06-03T19:59:44.098 INFO:teuthology.orchestra.run.smithi089.stdout: "rbd_qos_bps_burst_seconds": "1", 2023-06-03T19:59:44.099 INFO:teuthology.orchestra.run.smithi089.stdout: "rbd_qos_bps_limit": "0", 2023-06-03T19:59:44.099 INFO:teuthology.orchestra.run.smithi089.stdout: "rbd_qos_exclude_ops": "0", 2023-06-03T19:59:44.099 INFO:teuthology.orchestra.run.smithi089.stdout: "rbd_qos_iops_burst": "0", 2023-06-03T19:59:44.099 INFO:teuthology.orchestra.run.smithi089.stdout: "rbd_qos_iops_burst_seconds": "1", 2023-06-03T19:59:44.099 INFO:teuthology.orchestra.run.smithi089.stdout: "rbd_qos_iops_limit": "0", 2023-06-03T19:59:44.099 INFO:teuthology.orchestra.run.smithi089.stdout: "rbd_qos_read_bps_burst": "0", 2023-06-03T19:59:44.099 INFO:teuthology.orchestra.run.smithi089.stdout: "rbd_qos_read_bps_burst_seconds": "1", 2023-06-03T19:59:44.099 INFO:teuthology.orchestra.run.smithi089.stdout: "rbd_qos_read_bps_limit": "0", 2023-06-03T19:59:44.100 INFO:teuthology.orchestra.run.smithi089.stdout: "rbd_qos_read_iops_burst": "0", 2023-06-03T19:59:44.100 INFO:teuthology.orchestra.run.smithi089.stdout: "rbd_qos_read_iops_burst_seconds": "1", 2023-06-03T19:59:44.100 INFO:teuthology.orchestra.run.smithi089.stdout: "rbd_qos_read_iops_limit": "0", 2023-06-03T19:59:44.100 INFO:teuthology.orchestra.run.smithi089.stdout: "rbd_qos_schedule_tick_min": "50", 2023-06-03T19:59:44.100 INFO:teuthology.orchestra.run.smithi089.stdout: "rbd_qos_write_bps_burst": "0", 2023-06-03T19:59:44.100 INFO:teuthology.orchestra.run.smithi089.stdout: "rbd_qos_write_bps_burst_seconds": "1", 2023-06-03T19:59:44.100 INFO:teuthology.orchestra.run.smithi089.stdout: "rbd_qos_write_bps_limit": "0", 2023-06-03T19:59:44.100 INFO:teuthology.orchestra.run.smithi089.stdout: "rbd_qos_write_iops_burst": "0", 2023-06-03T19:59:44.101 INFO:teuthology.orchestra.run.smithi089.stdout: "rbd_qos_write_iops_burst_seconds": "1", 2023-06-03T19:59:44.101 INFO:teuthology.orchestra.run.smithi089.stdout: "rbd_qos_write_iops_limit": "0", 2023-06-03T19:59:44.101 INFO:teuthology.orchestra.run.smithi089.stdout: "rbd_quiesce_notification_attempts": "10", 2023-06-03T19:59:44.101 INFO:teuthology.orchestra.run.smithi089.stdout: "rbd_read_from_replica_policy": "default", 2023-06-03T19:59:44.101 INFO:teuthology.orchestra.run.smithi089.stdout: "rbd_readahead_disable_after_bytes": "52428800", 2023-06-03T19:59:44.101 INFO:teuthology.orchestra.run.smithi089.stdout: "rbd_readahead_max_bytes": "524288", 2023-06-03T19:59:44.101 INFO:teuthology.orchestra.run.smithi089.stdout: "rbd_readahead_trigger_requests": "10", 2023-06-03T19:59:44.101 INFO:teuthology.orchestra.run.smithi089.stdout: "rbd_request_timed_out_seconds": "30", 2023-06-03T19:59:44.102 INFO:teuthology.orchestra.run.smithi089.stdout: "rbd_skip_partial_discard": "true", 2023-06-03T19:59:44.102 INFO:teuthology.orchestra.run.smithi089.stdout: "rbd_sparse_read_threshold_bytes": "65536", 2023-06-03T19:59:44.102 INFO:teuthology.orchestra.run.smithi089.stdout: "rbd_tracing": "false", 2023-06-03T19:59:44.102 INFO:teuthology.orchestra.run.smithi089.stdout: "rbd_validate_names": "true", 2023-06-03T19:59:44.102 INFO:teuthology.orchestra.run.smithi089.stdout: "rbd_validate_pool": "true", 2023-06-03T19:59:44.102 INFO:teuthology.orchestra.run.smithi089.stdout: "restapi_base_url": "", 2023-06-03T19:59:44.102 INFO:teuthology.orchestra.run.smithi089.stdout: "restapi_log_level": "", 2023-06-03T19:59:44.102 INFO:teuthology.orchestra.run.smithi089.stdout: "rgw_acl_grants_max_num": "100", 2023-06-03T19:59:44.103 INFO:teuthology.orchestra.run.smithi089.stdout: "rgw_admin_entry": "admin", 2023-06-03T19:59:44.103 INFO:teuthology.orchestra.run.smithi089.stdout: "rgw_allow_notification_secrets_in_cleartext": "false", 2023-06-03T19:59:44.103 INFO:teuthology.orchestra.run.smithi089.stdout: "rgw_backend_store": "rados", 2023-06-03T19:59:44.103 INFO:teuthology.orchestra.run.smithi089.stdout: "rgw_barbican_url": "", 2023-06-03T19:59:44.103 INFO:teuthology.orchestra.run.smithi089.stdout: "rgw_beast_enable_async": "true", 2023-06-03T19:59:44.103 INFO:teuthology.orchestra.run.smithi089.stdout: "rgw_bucket_default_quota_max_objects": "-1", 2023-06-03T19:59:44.103 INFO:teuthology.orchestra.run.smithi089.stdout: "rgw_bucket_default_quota_max_size": "-1", 2023-06-03T19:59:44.104 INFO:teuthology.orchestra.run.smithi089.stdout: "rgw_bucket_index_max_aio": "128", 2023-06-03T19:59:44.104 INFO:teuthology.orchestra.run.smithi089.stdout: "rgw_bucket_index_transaction_instrumentation": "false", 2023-06-03T19:59:44.104 INFO:teuthology.orchestra.run.smithi089.stdout: "rgw_bucket_quota_cache_size": "10000", 2023-06-03T19:59:44.104 INFO:teuthology.orchestra.run.smithi089.stdout: "rgw_bucket_quota_ttl": "600", 2023-06-03T19:59:44.104 INFO:teuthology.orchestra.run.smithi089.stdout: "rgw_bucket_sync_spawn_window": "20", 2023-06-03T19:59:44.104 INFO:teuthology.orchestra.run.smithi089.stdout: "rgw_cache_enabled": "true", 2023-06-03T19:59:44.104 INFO:teuthology.orchestra.run.smithi089.stdout: "rgw_cache_expiry_interval": "900", 2023-06-03T19:59:44.104 INFO:teuthology.orchestra.run.smithi089.stdout: "rgw_cache_lru_size": "10000", 2023-06-03T19:59:44.105 INFO:teuthology.orchestra.run.smithi089.stdout: "rgw_config_store": "rados", 2023-06-03T19:59:44.105 INFO:teuthology.orchestra.run.smithi089.stdout: "rgw_content_length_compat": "false", 2023-06-03T19:59:44.105 INFO:teuthology.orchestra.run.smithi089.stdout: "rgw_copy_obj_progress": "true", 2023-06-03T19:59:44.105 INFO:teuthology.orchestra.run.smithi089.stdout: "rgw_copy_obj_progress_every_bytes": "1048576", 2023-06-03T19:59:44.105 INFO:teuthology.orchestra.run.smithi089.stdout: "rgw_cors_rules_max_num": "100", 2023-06-03T19:59:44.105 INFO:teuthology.orchestra.run.smithi089.stdout: "rgw_cross_domain_policy": "", 2023-06-03T19:59:44.105 INFO:teuthology.orchestra.run.smithi089.stdout: "rgw_crypt_default_encryption_key": "", 2023-06-03T19:59:44.105 INFO:teuthology.orchestra.run.smithi089.stdout: "rgw_crypt_kmip_addr": "", 2023-06-03T19:59:44.106 INFO:teuthology.orchestra.run.smithi089.stdout: "rgw_crypt_kmip_ca_path": "", 2023-06-03T19:59:44.106 INFO:teuthology.orchestra.run.smithi089.stdout: "rgw_crypt_kmip_client_cert": "", 2023-06-03T19:59:44.106 INFO:teuthology.orchestra.run.smithi089.stdout: "rgw_crypt_kmip_client_key": "", 2023-06-03T19:59:44.106 INFO:teuthology.orchestra.run.smithi089.stdout: "rgw_crypt_kmip_kms_key_template": "", 2023-06-03T19:59:44.106 INFO:teuthology.orchestra.run.smithi089.stdout: "rgw_crypt_kmip_password": "", 2023-06-03T19:59:44.106 INFO:teuthology.orchestra.run.smithi089.stdout: "rgw_crypt_kmip_s3_key_template": "$keyid", 2023-06-03T19:59:44.106 INFO:teuthology.orchestra.run.smithi089.stdout: "rgw_crypt_kmip_username": "", 2023-06-03T19:59:44.106 INFO:teuthology.orchestra.run.smithi089.stdout: "rgw_crypt_require_ssl": "true", 2023-06-03T19:59:44.107 INFO:teuthology.orchestra.run.smithi089.stdout: "rgw_crypt_s3_kms_backend": "barbican", 2023-06-03T19:59:44.107 INFO:teuthology.orchestra.run.smithi089.stdout: "rgw_crypt_s3_kms_encryption_keys": "", 2023-06-03T19:59:44.107 INFO:teuthology.orchestra.run.smithi089.stdout: "rgw_crypt_sse_s3_backend": "vault", 2023-06-03T19:59:44.107 INFO:teuthology.orchestra.run.smithi089.stdout: "rgw_crypt_sse_s3_key_template": "%bucket_id", 2023-06-03T19:59:44.107 INFO:teuthology.orchestra.run.smithi089.stdout: "rgw_crypt_sse_s3_vault_addr": "", 2023-06-03T19:59:44.107 INFO:teuthology.orchestra.run.smithi089.stdout: "rgw_crypt_sse_s3_vault_auth": "token", 2023-06-03T19:59:44.107 INFO:teuthology.orchestra.run.smithi089.stdout: "rgw_crypt_sse_s3_vault_namespace": "", 2023-06-03T19:59:44.107 INFO:teuthology.orchestra.run.smithi089.stdout: "rgw_crypt_sse_s3_vault_prefix": "", 2023-06-03T19:59:44.108 INFO:teuthology.orchestra.run.smithi089.stdout: "rgw_crypt_sse_s3_vault_secret_engine": "transit", 2023-06-03T19:59:44.108 INFO:teuthology.orchestra.run.smithi089.stdout: "rgw_crypt_sse_s3_vault_ssl_cacert": "", 2023-06-03T19:59:44.108 INFO:teuthology.orchestra.run.smithi089.stdout: "rgw_crypt_sse_s3_vault_ssl_clientcert": "", 2023-06-03T19:59:44.108 INFO:teuthology.orchestra.run.smithi089.stdout: "rgw_crypt_sse_s3_vault_ssl_clientkey": "", 2023-06-03T19:59:44.108 INFO:teuthology.orchestra.run.smithi089.stdout: "rgw_crypt_sse_s3_vault_token_file": "", 2023-06-03T19:59:44.108 INFO:teuthology.orchestra.run.smithi089.stdout: "rgw_crypt_sse_s3_vault_verify_ssl": "true", 2023-06-03T19:59:44.108 INFO:teuthology.orchestra.run.smithi089.stdout: "rgw_crypt_suppress_logs": "true", 2023-06-03T19:59:44.109 INFO:teuthology.orchestra.run.smithi089.stdout: "rgw_crypt_vault_addr": "", 2023-06-03T19:59:44.109 INFO:teuthology.orchestra.run.smithi089.stdout: "rgw_crypt_vault_auth": "token", 2023-06-03T19:59:44.109 INFO:teuthology.orchestra.run.smithi089.stdout: "rgw_crypt_vault_namespace": "", 2023-06-03T19:59:44.109 INFO:teuthology.orchestra.run.smithi089.stdout: "rgw_crypt_vault_prefix": "", 2023-06-03T19:59:44.109 INFO:teuthology.orchestra.run.smithi089.stdout: "rgw_crypt_vault_secret_engine": "transit", 2023-06-03T19:59:44.109 INFO:teuthology.orchestra.run.smithi089.stdout: "rgw_crypt_vault_ssl_cacert": "", 2023-06-03T19:59:44.109 INFO:teuthology.orchestra.run.smithi089.stdout: "rgw_crypt_vault_ssl_clientcert": "", 2023-06-03T19:59:44.109 INFO:teuthology.orchestra.run.smithi089.stdout: "rgw_crypt_vault_ssl_clientkey": "", 2023-06-03T19:59:44.110 INFO:teuthology.orchestra.run.smithi089.stdout: "rgw_crypt_vault_token_file": "", 2023-06-03T19:59:44.110 INFO:teuthology.orchestra.run.smithi089.stdout: "rgw_crypt_vault_verify_ssl": "true", 2023-06-03T19:59:44.110 INFO:teuthology.orchestra.run.smithi089.stdout: "rgw_curl_buffersize": "524288", 2023-06-03T19:59:44.110 INFO:teuthology.orchestra.run.smithi089.stdout: "rgw_curl_low_speed_limit": "1024", 2023-06-03T19:59:44.110 INFO:teuthology.orchestra.run.smithi089.stdout: "rgw_curl_low_speed_time": "300", 2023-06-03T19:59:44.110 INFO:teuthology.orchestra.run.smithi089.stdout: "rgw_curl_tcp_keepalive": "0", 2023-06-03T19:59:44.110 INFO:teuthology.orchestra.run.smithi089.stdout: "rgw_curl_wait_timeout_ms": "1000", 2023-06-03T19:59:44.110 INFO:teuthology.orchestra.run.smithi089.stdout: "rgw_d3n_l1_datacache_persistent_path": "/tmp/rgw_datacache/", 2023-06-03T19:59:44.111 INFO:teuthology.orchestra.run.smithi089.stdout: "rgw_d3n_l1_datacache_size": "1073741824", 2023-06-03T19:59:44.111 INFO:teuthology.orchestra.run.smithi089.stdout: "rgw_d3n_l1_evict_cache_on_start": "true", 2023-06-03T19:59:44.111 INFO:teuthology.orchestra.run.smithi089.stdout: "rgw_d3n_l1_eviction_policy": "lru", 2023-06-03T19:59:44.111 INFO:teuthology.orchestra.run.smithi089.stdout: "rgw_d3n_l1_fadvise": "4", 2023-06-03T19:59:44.111 INFO:teuthology.orchestra.run.smithi089.stdout: "rgw_d3n_l1_local_datacache_enabled": "false", 2023-06-03T19:59:44.111 INFO:teuthology.orchestra.run.smithi089.stdout: "rgw_d3n_libaio_aio_num": "64", 2023-06-03T19:59:44.111 INFO:teuthology.orchestra.run.smithi089.stdout: "rgw_d3n_libaio_aio_threads": "20", 2023-06-03T19:59:44.111 INFO:teuthology.orchestra.run.smithi089.stdout: "rgw_data": "/var/lib/ceph/radosgw/ceph-a", 2023-06-03T19:59:44.112 INFO:teuthology.orchestra.run.smithi089.stdout: "rgw_data_log_changes_size": "1000", 2023-06-03T19:59:44.112 INFO:teuthology.orchestra.run.smithi089.stdout: "rgw_data_log_num_shards": "128", 2023-06-03T19:59:44.112 INFO:teuthology.orchestra.run.smithi089.stdout: "rgw_data_log_obj_prefix": "data_log", 2023-06-03T19:59:44.112 INFO:teuthology.orchestra.run.smithi089.stdout: "rgw_data_log_window": "30", 2023-06-03T19:59:44.112 INFO:teuthology.orchestra.run.smithi089.stdout: "rgw_data_notify_interval_msec": "0", 2023-06-03T19:59:44.112 INFO:teuthology.orchestra.run.smithi089.stdout: "rgw_data_sync_poll_interval": "20", 2023-06-03T19:59:44.112 INFO:teuthology.orchestra.run.smithi089.stdout: "rgw_data_sync_spawn_window": "20", 2023-06-03T19:59:44.113 INFO:teuthology.orchestra.run.smithi089.stdout: "rgw_default_data_log_backing": "fifo", 2023-06-03T19:59:44.113 INFO:teuthology.orchestra.run.smithi089.stdout: "rgw_default_realm_info_oid": "default.realm", 2023-06-03T19:59:44.113 INFO:teuthology.orchestra.run.smithi089.stdout: "rgw_default_region_info_oid": "default.region", 2023-06-03T19:59:44.113 INFO:teuthology.orchestra.run.smithi089.stdout: "rgw_default_zone_info_oid": "default.zone", 2023-06-03T19:59:44.113 INFO:teuthology.orchestra.run.smithi089.stdout: "rgw_default_zonegroup_info_oid": "default.zonegroup", 2023-06-03T19:59:44.113 INFO:teuthology.orchestra.run.smithi089.stdout: "rgw_defer_to_bucket_acls": "", 2023-06-03T19:59:44.113 INFO:teuthology.orchestra.run.smithi089.stdout: "rgw_delete_multi_obj_max_num": "1000", 2023-06-03T19:59:44.113 INFO:teuthology.orchestra.run.smithi089.stdout: "rgw_dmclock_admin_lim": "0.000000", 2023-06-03T19:59:44.114 INFO:teuthology.orchestra.run.smithi089.stdout: "rgw_dmclock_admin_res": "100.000000", 2023-06-03T19:59:44.114 INFO:teuthology.orchestra.run.smithi089.stdout: "rgw_dmclock_admin_wgt": "100.000000", 2023-06-03T19:59:44.114 INFO:teuthology.orchestra.run.smithi089.stdout: "rgw_dmclock_auth_lim": "0.000000", 2023-06-03T19:59:44.114 INFO:teuthology.orchestra.run.smithi089.stdout: "rgw_dmclock_auth_res": "200.000000", 2023-06-03T19:59:44.114 INFO:teuthology.orchestra.run.smithi089.stdout: "rgw_dmclock_auth_wgt": "100.000000", 2023-06-03T19:59:44.114 INFO:teuthology.orchestra.run.smithi089.stdout: "rgw_dmclock_data_lim": "0.000000", 2023-06-03T19:59:44.114 INFO:teuthology.orchestra.run.smithi089.stdout: "rgw_dmclock_data_res": "500.000000", 2023-06-03T19:59:44.114 INFO:teuthology.orchestra.run.smithi089.stdout: "rgw_dmclock_data_wgt": "500.000000", 2023-06-03T19:59:44.115 INFO:teuthology.orchestra.run.smithi089.stdout: "rgw_dmclock_metadata_lim": "0.000000", 2023-06-03T19:59:44.115 INFO:teuthology.orchestra.run.smithi089.stdout: "rgw_dmclock_metadata_res": "500.000000", 2023-06-03T19:59:44.115 INFO:teuthology.orchestra.run.smithi089.stdout: "rgw_dmclock_metadata_wgt": "500.000000", 2023-06-03T19:59:44.115 INFO:teuthology.orchestra.run.smithi089.stdout: "rgw_dns_name": "", 2023-06-03T19:59:44.115 INFO:teuthology.orchestra.run.smithi089.stdout: "rgw_dns_s3website_name": "", 2023-06-03T19:59:44.115 INFO:teuthology.orchestra.run.smithi089.stdout: "rgw_dynamic_resharding": "true", 2023-06-03T19:59:44.115 INFO:teuthology.orchestra.run.smithi089.stdout: "rgw_enable_apis": "s3, s3website, swift, swift_auth, admin, sts, iam, notifications", 2023-06-03T19:59:44.115 INFO:teuthology.orchestra.run.smithi089.stdout: "rgw_enable_gc_threads": "true", 2023-06-03T19:59:44.116 INFO:teuthology.orchestra.run.smithi089.stdout: "rgw_enable_lc_threads": "true", 2023-06-03T19:59:44.116 INFO:teuthology.orchestra.run.smithi089.stdout: "rgw_enable_ops_log": "false", 2023-06-03T19:59:44.116 INFO:teuthology.orchestra.run.smithi089.stdout: "rgw_enable_quota_threads": "true", 2023-06-03T19:59:44.116 INFO:teuthology.orchestra.run.smithi089.stdout: "rgw_enable_static_website": "false", 2023-06-03T19:59:44.116 INFO:teuthology.orchestra.run.smithi089.stdout: "rgw_enable_usage_log": "false", 2023-06-03T19:59:44.116 INFO:teuthology.orchestra.run.smithi089.stdout: "rgw_enforce_swift_acls": "true", 2023-06-03T19:59:44.116 INFO:teuthology.orchestra.run.smithi089.stdout: "rgw_exit_timeout_secs": "120", 2023-06-03T19:59:44.116 INFO:teuthology.orchestra.run.smithi089.stdout: "rgw_expose_bucket": "false", 2023-06-03T19:59:44.117 INFO:teuthology.orchestra.run.smithi089.stdout: "rgw_extended_http_attrs": "", 2023-06-03T19:59:44.117 INFO:teuthology.orchestra.run.smithi089.stdout: "rgw_filter": "none", 2023-06-03T19:59:44.117 INFO:teuthology.orchestra.run.smithi089.stdout: "rgw_frontend_defaults": "beast ssl_certificate=config://rgw/cert/$realm/$zone.crt ssl_private_key=config://rgw/cert/$realm/$zone.key", 2023-06-03T19:59:44.117 INFO:teuthology.orchestra.run.smithi089.stdout: "rgw_frontends": "beast port=7480", 2023-06-03T19:59:44.117 INFO:teuthology.orchestra.run.smithi089.stdout: "rgw_gc_max_concurrent_io": "10", 2023-06-03T19:59:44.117 INFO:teuthology.orchestra.run.smithi089.stdout: "rgw_gc_max_deferred": "50", 2023-06-03T19:59:44.117 INFO:teuthology.orchestra.run.smithi089.stdout: "rgw_gc_max_deferred_entries_size": "3072", 2023-06-03T19:59:44.117 INFO:teuthology.orchestra.run.smithi089.stdout: "rgw_gc_max_objs": "32", 2023-06-03T19:59:44.118 INFO:teuthology.orchestra.run.smithi089.stdout: "rgw_gc_max_queue_size": "134213632", 2023-06-03T19:59:44.118 INFO:teuthology.orchestra.run.smithi089.stdout: "rgw_gc_max_trim_chunk": "16", 2023-06-03T19:59:44.118 INFO:teuthology.orchestra.run.smithi089.stdout: "rgw_gc_obj_min_wait": "7200", 2023-06-03T19:59:44.118 INFO:teuthology.orchestra.run.smithi089.stdout: "rgw_gc_processor_max_time": "3600", 2023-06-03T19:59:44.118 INFO:teuthology.orchestra.run.smithi089.stdout: "rgw_gc_processor_period": "3600", 2023-06-03T19:59:44.118 INFO:teuthology.orchestra.run.smithi089.stdout: "rgw_get_obj_max_req_size": "4194304", 2023-06-03T19:59:44.118 INFO:teuthology.orchestra.run.smithi089.stdout: "rgw_get_obj_window_size": "16777216", 2023-06-03T19:59:44.118 INFO:teuthology.orchestra.run.smithi089.stdout: "rgw_healthcheck_disabling_path": "", 2023-06-03T19:59:44.119 INFO:teuthology.orchestra.run.smithi089.stdout: "rgw_ignore_get_invalid_range": "false", 2023-06-03T19:59:44.119 INFO:teuthology.orchestra.run.smithi089.stdout: "rgw_init_timeout": "300", 2023-06-03T19:59:44.119 INFO:teuthology.orchestra.run.smithi089.stdout: "rgw_inject_notify_timeout_probability": "0.000000", 2023-06-03T19:59:44.119 INFO:teuthology.orchestra.run.smithi089.stdout: "rgw_json_config": "/var/lib/ceph/radosgw/config.json", 2023-06-03T19:59:44.119 INFO:teuthology.orchestra.run.smithi089.stdout: "rgw_keystone_accepted_admin_roles": "", 2023-06-03T19:59:44.119 INFO:teuthology.orchestra.run.smithi089.stdout: "rgw_keystone_accepted_roles": "Member, admin", 2023-06-03T19:59:44.119 INFO:teuthology.orchestra.run.smithi089.stdout: "rgw_keystone_admin_domain": "", 2023-06-03T19:59:44.120 INFO:teuthology.orchestra.run.smithi089.stdout: "rgw_keystone_admin_password": "", 2023-06-03T19:59:44.120 INFO:teuthology.orchestra.run.smithi089.stdout: "rgw_keystone_admin_password_path": "", 2023-06-03T19:59:44.120 INFO:teuthology.orchestra.run.smithi089.stdout: "rgw_keystone_admin_project": "", 2023-06-03T19:59:44.120 INFO:teuthology.orchestra.run.smithi089.stdout: "rgw_keystone_admin_tenant": "", 2023-06-03T19:59:44.120 INFO:teuthology.orchestra.run.smithi089.stdout: "rgw_keystone_admin_token": "", 2023-06-03T19:59:44.120 INFO:teuthology.orchestra.run.smithi089.stdout: "rgw_keystone_admin_token_path": "", 2023-06-03T19:59:44.120 INFO:teuthology.orchestra.run.smithi089.stdout: "rgw_keystone_admin_user": "", 2023-06-03T19:59:44.120 INFO:teuthology.orchestra.run.smithi089.stdout: "rgw_keystone_api_version": "2", 2023-06-03T19:59:44.121 INFO:teuthology.orchestra.run.smithi089.stdout: "rgw_keystone_barbican_domain": "", 2023-06-03T19:59:44.121 INFO:teuthology.orchestra.run.smithi089.stdout: "rgw_keystone_barbican_password": "", 2023-06-03T19:59:44.121 INFO:teuthology.orchestra.run.smithi089.stdout: "rgw_keystone_barbican_project": "", 2023-06-03T19:59:44.121 INFO:teuthology.orchestra.run.smithi089.stdout: "rgw_keystone_barbican_tenant": "", 2023-06-03T19:59:44.121 INFO:teuthology.orchestra.run.smithi089.stdout: "rgw_keystone_barbican_user": "", 2023-06-03T19:59:44.121 INFO:teuthology.orchestra.run.smithi089.stdout: "rgw_keystone_expired_token_cache_expiration": "3600", 2023-06-03T19:59:44.121 INFO:teuthology.orchestra.run.smithi089.stdout: "rgw_keystone_implicit_tenants": "false", 2023-06-03T19:59:44.121 INFO:teuthology.orchestra.run.smithi089.stdout: "rgw_keystone_service_token_accepted_roles": "admin", 2023-06-03T19:59:44.122 INFO:teuthology.orchestra.run.smithi089.stdout: "rgw_keystone_service_token_enabled": "false", 2023-06-03T19:59:44.122 INFO:teuthology.orchestra.run.smithi089.stdout: "rgw_keystone_token_cache_size": "10000", 2023-06-03T19:59:44.122 INFO:teuthology.orchestra.run.smithi089.stdout: "rgw_keystone_url": "", 2023-06-03T19:59:44.122 INFO:teuthology.orchestra.run.smithi089.stdout: "rgw_keystone_verify_ssl": "true", 2023-06-03T19:59:44.122 INFO:teuthology.orchestra.run.smithi089.stdout: "rgw_lc_debug_interval": "-1", 2023-06-03T19:59:44.122 INFO:teuthology.orchestra.run.smithi089.stdout: "rgw_lc_lock_max_time": "90", 2023-06-03T19:59:44.122 INFO:teuthology.orchestra.run.smithi089.stdout: "rgw_lc_max_objs": "32", 2023-06-03T19:59:44.122 INFO:teuthology.orchestra.run.smithi089.stdout: "rgw_lc_max_rules": "1000", 2023-06-03T19:59:44.123 INFO:teuthology.orchestra.run.smithi089.stdout: "rgw_lc_max_worker": "3", 2023-06-03T19:59:44.123 INFO:teuthology.orchestra.run.smithi089.stdout: "rgw_lc_max_wp_worker": "3", 2023-06-03T19:59:44.123 INFO:teuthology.orchestra.run.smithi089.stdout: "rgw_lc_thread_delay": "0", 2023-06-03T19:59:44.123 INFO:teuthology.orchestra.run.smithi089.stdout: "rgw_ldap_binddn": "uid=admin,cn=users,dc=example,dc=com", 2023-06-03T19:59:44.123 INFO:teuthology.orchestra.run.smithi089.stdout: "rgw_ldap_dnattr": "uid", 2023-06-03T19:59:44.123 INFO:teuthology.orchestra.run.smithi089.stdout: "rgw_ldap_searchdn": "cn=users,cn=accounts,dc=example,dc=com", 2023-06-03T19:59:44.123 INFO:teuthology.orchestra.run.smithi089.stdout: "rgw_ldap_searchfilter": "", 2023-06-03T19:59:44.123 INFO:teuthology.orchestra.run.smithi089.stdout: "rgw_ldap_secret": "/etc/openldap/secret", 2023-06-03T19:59:44.124 INFO:teuthology.orchestra.run.smithi089.stdout: "rgw_ldap_uri": "ldaps://", 2023-06-03T19:59:44.124 INFO:teuthology.orchestra.run.smithi089.stdout: "rgw_lifecycle_work_time": "00:00-06:00", 2023-06-03T19:59:44.124 INFO:teuthology.orchestra.run.smithi089.stdout: "rgw_list_bucket_min_readahead": "1000", 2023-06-03T19:59:44.124 INFO:teuthology.orchestra.run.smithi089.stdout: "rgw_list_buckets_max_chunk": "1000", 2023-06-03T19:59:44.124 INFO:teuthology.orchestra.run.smithi089.stdout: "rgw_log_http_headers": "", 2023-06-03T19:59:44.124 INFO:teuthology.orchestra.run.smithi089.stdout: "rgw_log_nonexistent_bucket": "false", 2023-06-03T19:59:44.124 INFO:teuthology.orchestra.run.smithi089.stdout: "rgw_log_object_name": "%Y-%m-%d-%H-%i-%n", 2023-06-03T19:59:44.125 INFO:teuthology.orchestra.run.smithi089.stdout: "rgw_log_object_name_utc": "false", 2023-06-03T19:59:44.125 INFO:teuthology.orchestra.run.smithi089.stdout: "rgw_luarocks_location": "/tmp/luarocks", 2023-06-03T19:59:44.125 INFO:teuthology.orchestra.run.smithi089.stdout: "rgw_max_attr_name_len": "0", 2023-06-03T19:59:44.125 INFO:teuthology.orchestra.run.smithi089.stdout: "rgw_max_attr_size": "0", 2023-06-03T19:59:44.125 INFO:teuthology.orchestra.run.smithi089.stdout: "rgw_max_attrs_num_in_req": "0", 2023-06-03T19:59:44.125 INFO:teuthology.orchestra.run.smithi089.stdout: "rgw_max_chunk_size": "4194304", 2023-06-03T19:59:44.125 INFO:teuthology.orchestra.run.smithi089.stdout: "rgw_max_concurrent_requests": "1024", 2023-06-03T19:59:44.125 INFO:teuthology.orchestra.run.smithi089.stdout: "rgw_max_copy_obj_concurrent_io": "10", 2023-06-03T19:59:44.126 INFO:teuthology.orchestra.run.smithi089.stdout: "rgw_max_dynamic_shards": "1999", 2023-06-03T19:59:44.126 INFO:teuthology.orchestra.run.smithi089.stdout: "rgw_max_listing_results": "1000", 2023-06-03T19:59:44.126 INFO:teuthology.orchestra.run.smithi089.stdout: "rgw_max_notify_retries": "3", 2023-06-03T19:59:44.126 INFO:teuthology.orchestra.run.smithi089.stdout: "rgw_max_objs_per_shard": "100000", 2023-06-03T19:59:44.126 INFO:teuthology.orchestra.run.smithi089.stdout: "rgw_max_put_param_size": "1048576", 2023-06-03T19:59:44.126 INFO:teuthology.orchestra.run.smithi089.stdout: "rgw_max_put_size": "5368709120", 2023-06-03T19:59:44.126 INFO:teuthology.orchestra.run.smithi089.stdout: "rgw_max_slo_entries": "1000", 2023-06-03T19:59:44.126 INFO:teuthology.orchestra.run.smithi089.stdout: "rgw_md_log_max_shards": "64", 2023-06-03T19:59:44.127 INFO:teuthology.orchestra.run.smithi089.stdout: "rgw_md_notify_interval_msec": "200", 2023-06-03T19:59:44.127 INFO:teuthology.orchestra.run.smithi089.stdout: "rgw_meta_sync_poll_interval": "20", 2023-06-03T19:59:44.127 INFO:teuthology.orchestra.run.smithi089.stdout: "rgw_meta_sync_spawn_window": "20", 2023-06-03T19:59:44.127 INFO:teuthology.orchestra.run.smithi089.stdout: "rgw_mime_types_file": "/etc/mime.types", 2023-06-03T19:59:44.127 INFO:teuthology.orchestra.run.smithi089.stdout: "rgw_mp_lock_max_time": "600", 2023-06-03T19:59:44.127 INFO:teuthology.orchestra.run.smithi089.stdout: "rgw_multi_obj_del_max_aio": "16", 2023-06-03T19:59:44.127 INFO:teuthology.orchestra.run.smithi089.stdout: "rgw_multipart_min_part_size": "5242880", 2023-06-03T19:59:44.127 INFO:teuthology.orchestra.run.smithi089.stdout: "rgw_multipart_part_upload_limit": "10000", 2023-06-03T19:59:44.128 INFO:teuthology.orchestra.run.smithi089.stdout: "rgw_nfs_fhcache_partitions": "3", 2023-06-03T19:59:44.128 INFO:teuthology.orchestra.run.smithi089.stdout: "rgw_nfs_fhcache_size": "2017", 2023-06-03T19:59:44.128 INFO:teuthology.orchestra.run.smithi089.stdout: "rgw_nfs_frontends": "rgw-nfs", 2023-06-03T19:59:44.128 INFO:teuthology.orchestra.run.smithi089.stdout: "rgw_nfs_lru_lane_hiwat": "911", 2023-06-03T19:59:44.128 INFO:teuthology.orchestra.run.smithi089.stdout: "rgw_nfs_lru_lanes": "5", 2023-06-03T19:59:44.128 INFO:teuthology.orchestra.run.smithi089.stdout: "rgw_nfs_max_gc": "300", 2023-06-03T19:59:44.128 INFO:teuthology.orchestra.run.smithi089.stdout: "rgw_nfs_namespace_expire_secs": "300", 2023-06-03T19:59:44.129 INFO:teuthology.orchestra.run.smithi089.stdout: "rgw_nfs_run_gc_threads": "false", 2023-06-03T19:59:44.129 INFO:teuthology.orchestra.run.smithi089.stdout: "rgw_nfs_run_lc_threads": "false", 2023-06-03T19:59:44.129 INFO:teuthology.orchestra.run.smithi089.stdout: "rgw_nfs_run_quota_threads": "false", 2023-06-03T19:59:44.129 INFO:teuthology.orchestra.run.smithi089.stdout: "rgw_nfs_run_sync_thread": "false", 2023-06-03T19:59:44.129 INFO:teuthology.orchestra.run.smithi089.stdout: "rgw_nfs_s3_fast_attrs": "false", 2023-06-03T19:59:44.129 INFO:teuthology.orchestra.run.smithi089.stdout: "rgw_nfs_write_completion_interval_s": "10", 2023-06-03T19:59:44.129 INFO:teuthology.orchestra.run.smithi089.stdout: "rgw_num_async_rados_threads": "32", 2023-06-03T19:59:44.129 INFO:teuthology.orchestra.run.smithi089.stdout: "rgw_num_control_oids": "8", 2023-06-03T19:59:44.130 INFO:teuthology.orchestra.run.smithi089.stdout: "rgw_numa_node": "-1", 2023-06-03T19:59:44.130 INFO:teuthology.orchestra.run.smithi089.stdout: "rgw_obj_stripe_size": "4194304", 2023-06-03T19:59:44.130 INFO:teuthology.orchestra.run.smithi089.stdout: "rgw_obj_tombstone_cache_size": "1000", 2023-06-03T19:59:44.130 INFO:teuthology.orchestra.run.smithi089.stdout: "rgw_objexp_chunk_size": "100", 2023-06-03T19:59:44.130 INFO:teuthology.orchestra.run.smithi089.stdout: "rgw_objexp_gc_interval": "600", 2023-06-03T19:59:44.130 INFO:teuthology.orchestra.run.smithi089.stdout: "rgw_objexp_hints_num_shards": "127", 2023-06-03T19:59:44.130 INFO:teuthology.orchestra.run.smithi089.stdout: "rgw_olh_pending_timeout_sec": "3600", 2023-06-03T19:59:44.130 INFO:teuthology.orchestra.run.smithi089.stdout: "rgw_op_thread_suicide_timeout": "0", 2023-06-03T19:59:44.131 INFO:teuthology.orchestra.run.smithi089.stdout: "rgw_op_thread_timeout": "600", 2023-06-03T19:59:44.131 INFO:teuthology.orchestra.run.smithi089.stdout: "rgw_op_tracing": "false", 2023-06-03T19:59:44.131 INFO:teuthology.orchestra.run.smithi089.stdout: "rgw_opa_token": "", 2023-06-03T19:59:44.131 INFO:teuthology.orchestra.run.smithi089.stdout: "rgw_opa_url": "", 2023-06-03T19:59:44.131 INFO:teuthology.orchestra.run.smithi089.stdout: "rgw_opa_verify_ssl": "true", 2023-06-03T19:59:44.131 INFO:teuthology.orchestra.run.smithi089.stdout: "rgw_ops_log_data_backlog": "5242880", 2023-06-03T19:59:44.131 INFO:teuthology.orchestra.run.smithi089.stdout: "rgw_ops_log_file_path": "/var/log/ceph/ops-log-ceph-mon.a.log", 2023-06-03T19:59:44.131 INFO:teuthology.orchestra.run.smithi089.stdout: "rgw_ops_log_rados": "false", 2023-06-03T19:59:44.132 INFO:teuthology.orchestra.run.smithi089.stdout: "rgw_ops_log_socket_path": "", 2023-06-03T19:59:44.132 INFO:teuthology.orchestra.run.smithi089.stdout: "rgw_override_bucket_index_max_shards": "0", 2023-06-03T19:59:44.132 INFO:teuthology.orchestra.run.smithi089.stdout: "rgw_pending_bucket_index_op_expiration": "120", 2023-06-03T19:59:44.132 INFO:teuthology.orchestra.run.smithi089.stdout: "rgw_period_latest_epoch_info_oid": ".latest_epoch", 2023-06-03T19:59:44.132 INFO:teuthology.orchestra.run.smithi089.stdout: "rgw_period_push_interval": "2.000000", 2023-06-03T19:59:44.132 INFO:teuthology.orchestra.run.smithi089.stdout: "rgw_period_push_interval_max": "30.000000", 2023-06-03T19:59:44.132 INFO:teuthology.orchestra.run.smithi089.stdout: "rgw_period_root_pool": ".rgw.root", 2023-06-03T19:59:44.132 INFO:teuthology.orchestra.run.smithi089.stdout: "rgw_policy_reject_invalid_principals": "true", 2023-06-03T19:59:44.133 INFO:teuthology.orchestra.run.smithi089.stdout: "rgw_print_continue": "true", 2023-06-03T19:59:44.133 INFO:teuthology.orchestra.run.smithi089.stdout: "rgw_print_prohibited_content_length": "false", 2023-06-03T19:59:44.133 INFO:teuthology.orchestra.run.smithi089.stdout: "rgw_put_obj_max_window_size": "67108864", 2023-06-03T19:59:44.133 INFO:teuthology.orchestra.run.smithi089.stdout: "rgw_put_obj_min_window_size": "16777216", 2023-06-03T19:59:44.133 INFO:teuthology.orchestra.run.smithi089.stdout: "rgw_rados_pool_autoscale_bias": "4.000000", 2023-06-03T19:59:44.133 INFO:teuthology.orchestra.run.smithi089.stdout: "rgw_rados_pool_recovery_priority": "5", 2023-06-03T19:59:44.133 INFO:teuthology.orchestra.run.smithi089.stdout: "rgw_rados_tracing": "false", 2023-06-03T19:59:44.134 INFO:teuthology.orchestra.run.smithi089.stdout: "rgw_realm": "", 2023-06-03T19:59:44.134 INFO:teuthology.orchestra.run.smithi089.stdout: "rgw_realm_id": "", 2023-06-03T19:59:44.134 INFO:teuthology.orchestra.run.smithi089.stdout: "rgw_realm_root_pool": ".rgw.root", 2023-06-03T19:59:44.134 INFO:teuthology.orchestra.run.smithi089.stdout: "rgw_region": "", 2023-06-03T19:59:44.134 INFO:teuthology.orchestra.run.smithi089.stdout: "rgw_region_root_pool": ".rgw.root", 2023-06-03T19:59:44.134 INFO:teuthology.orchestra.run.smithi089.stdout: "rgw_relaxed_region_enforcement": "false", 2023-06-03T19:59:44.134 INFO:teuthology.orchestra.run.smithi089.stdout: "rgw_relaxed_s3_bucket_names": "false", 2023-06-03T19:59:44.134 INFO:teuthology.orchestra.run.smithi089.stdout: "rgw_remote_addr_param": "REMOTE_ADDR", 2023-06-03T19:59:44.135 INFO:teuthology.orchestra.run.smithi089.stdout: "rgw_request_uri": "", 2023-06-03T19:59:44.135 INFO:teuthology.orchestra.run.smithi089.stdout: "rgw_reshard_batch_size": "64", 2023-06-03T19:59:44.135 INFO:teuthology.orchestra.run.smithi089.stdout: "rgw_reshard_bucket_lock_duration": "360", 2023-06-03T19:59:44.135 INFO:teuthology.orchestra.run.smithi089.stdout: "rgw_reshard_max_aio": "128", 2023-06-03T19:59:44.135 INFO:teuthology.orchestra.run.smithi089.stdout: "rgw_reshard_num_logs": "16", 2023-06-03T19:59:44.135 INFO:teuthology.orchestra.run.smithi089.stdout: "rgw_reshard_thread_interval": "600", 2023-06-03T19:59:44.135 INFO:teuthology.orchestra.run.smithi089.stdout: "rgw_resolve_cname": "false", 2023-06-03T19:59:44.135 INFO:teuthology.orchestra.run.smithi089.stdout: "rgw_rest_getusage_op_compat": "false", 2023-06-03T19:59:44.136 INFO:teuthology.orchestra.run.smithi089.stdout: "rgw_run_sync_thread": "true", 2023-06-03T19:59:44.136 INFO:teuthology.orchestra.run.smithi089.stdout: "rgw_s3_auth_order": "sts, external, local", 2023-06-03T19:59:44.136 INFO:teuthology.orchestra.run.smithi089.stdout: "rgw_s3_auth_use_keystone": "false", 2023-06-03T19:59:44.136 INFO:teuthology.orchestra.run.smithi089.stdout: "rgw_s3_auth_use_ldap": "false", 2023-06-03T19:59:44.136 INFO:teuthology.orchestra.run.smithi089.stdout: "rgw_s3_auth_use_rados": "true", 2023-06-03T19:59:44.136 INFO:teuthology.orchestra.run.smithi089.stdout: "rgw_s3_auth_use_sts": "false", 2023-06-03T19:59:44.136 INFO:teuthology.orchestra.run.smithi089.stdout: "rgw_s3_client_max_sig_ver": "-1", 2023-06-03T19:59:44.136 INFO:teuthology.orchestra.run.smithi089.stdout: "rgw_s3_success_create_obj_status": "0", 2023-06-03T19:59:44.137 INFO:teuthology.orchestra.run.smithi089.stdout: "rgw_safe_max_objects_per_shard": "102400", 2023-06-03T19:59:44.137 INFO:teuthology.orchestra.run.smithi089.stdout: "rgw_scheduler_type": "throttler", 2023-06-03T19:59:44.137 INFO:teuthology.orchestra.run.smithi089.stdout: "rgw_script_uri": "", 2023-06-03T19:59:44.137 INFO:teuthology.orchestra.run.smithi089.stdout: "rgw_service_provider_name": "", 2023-06-03T19:59:44.137 INFO:teuthology.orchestra.run.smithi089.stdout: "rgw_shard_warning_threshold": "90.000000", 2023-06-03T19:59:44.137 INFO:teuthology.orchestra.run.smithi089.stdout: "rgw_sts_client_id": "", 2023-06-03T19:59:44.137 INFO:teuthology.orchestra.run.smithi089.stdout: "rgw_sts_client_secret": "", 2023-06-03T19:59:44.137 INFO:teuthology.orchestra.run.smithi089.stdout: "rgw_sts_entry": "sts", 2023-06-03T19:59:44.138 INFO:teuthology.orchestra.run.smithi089.stdout: "rgw_sts_key": "sts", 2023-06-03T19:59:44.138 INFO:teuthology.orchestra.run.smithi089.stdout: "rgw_sts_max_session_duration": "43200", 2023-06-03T19:59:44.138 INFO:teuthology.orchestra.run.smithi089.stdout: "rgw_sts_min_session_duration": "900", 2023-06-03T19:59:44.138 INFO:teuthology.orchestra.run.smithi089.stdout: "rgw_sts_token_introspection_url": "", 2023-06-03T19:59:44.138 INFO:teuthology.orchestra.run.smithi089.stdout: "rgw_swift_account_in_url": "false", 2023-06-03T19:59:44.138 INFO:teuthology.orchestra.run.smithi089.stdout: "rgw_swift_auth_entry": "auth", 2023-06-03T19:59:44.138 INFO:teuthology.orchestra.run.smithi089.stdout: "rgw_swift_auth_url": "", 2023-06-03T19:59:44.138 INFO:teuthology.orchestra.run.smithi089.stdout: "rgw_swift_custom_header": "", 2023-06-03T19:59:44.139 INFO:teuthology.orchestra.run.smithi089.stdout: "rgw_swift_enforce_content_length": "false", 2023-06-03T19:59:44.139 INFO:teuthology.orchestra.run.smithi089.stdout: "rgw_swift_need_stats": "true", 2023-06-03T19:59:44.139 INFO:teuthology.orchestra.run.smithi089.stdout: "rgw_swift_tenant_name": "", 2023-06-03T19:59:44.139 INFO:teuthology.orchestra.run.smithi089.stdout: "rgw_swift_token_expiration": "86400", 2023-06-03T19:59:44.139 INFO:teuthology.orchestra.run.smithi089.stdout: "rgw_swift_url": "", 2023-06-03T19:59:44.139 INFO:teuthology.orchestra.run.smithi089.stdout: "rgw_swift_url_prefix": "swift", 2023-06-03T19:59:44.139 INFO:teuthology.orchestra.run.smithi089.stdout: "rgw_swift_versioning_enabled": "false", 2023-06-03T19:59:44.140 INFO:teuthology.orchestra.run.smithi089.stdout: "rgw_sync_data_full_inject_err_probability": "0.000000", 2023-06-03T19:59:44.140 INFO:teuthology.orchestra.run.smithi089.stdout: "rgw_sync_data_inject_err_probability": "0.000000", 2023-06-03T19:59:44.140 INFO:teuthology.orchestra.run.smithi089.stdout: "rgw_sync_lease_period": "120", 2023-06-03T19:59:44.140 INFO:teuthology.orchestra.run.smithi089.stdout: "rgw_sync_log_trim_concurrent_buckets": "4", 2023-06-03T19:59:44.140 INFO:teuthology.orchestra.run.smithi089.stdout: "rgw_sync_log_trim_interval": "1200", 2023-06-03T19:59:44.140 INFO:teuthology.orchestra.run.smithi089.stdout: "rgw_sync_log_trim_max_buckets": "16", 2023-06-03T19:59:44.140 INFO:teuthology.orchestra.run.smithi089.stdout: "rgw_sync_log_trim_min_cold_buckets": "4", 2023-06-03T19:59:44.140 INFO:teuthology.orchestra.run.smithi089.stdout: "rgw_sync_meta_inject_err_probability": "0.000000", 2023-06-03T19:59:44.141 INFO:teuthology.orchestra.run.smithi089.stdout: "rgw_sync_obj_etag_verify": "false", 2023-06-03T19:59:44.141 INFO:teuthology.orchestra.run.smithi089.stdout: "rgw_sync_trace_history_size": "4096", 2023-06-03T19:59:44.141 INFO:teuthology.orchestra.run.smithi089.stdout: "rgw_sync_trace_per_node_log_size": "32", 2023-06-03T19:59:44.141 INFO:teuthology.orchestra.run.smithi089.stdout: "rgw_sync_trace_servicemap_update_interval": "10", 2023-06-03T19:59:44.141 INFO:teuthology.orchestra.run.smithi089.stdout: "rgw_thread_pool_size": "512", 2023-06-03T19:59:44.141 INFO:teuthology.orchestra.run.smithi089.stdout: "rgw_torrent_comment": "", 2023-06-03T19:59:44.141 INFO:teuthology.orchestra.run.smithi089.stdout: "rgw_torrent_createby": "", 2023-06-03T19:59:44.141 INFO:teuthology.orchestra.run.smithi089.stdout: "rgw_torrent_encoding": "", 2023-06-03T19:59:44.142 INFO:teuthology.orchestra.run.smithi089.stdout: "rgw_torrent_flag": "false", 2023-06-03T19:59:44.142 INFO:teuthology.orchestra.run.smithi089.stdout: "rgw_torrent_origin": "", 2023-06-03T19:59:44.142 INFO:teuthology.orchestra.run.smithi089.stdout: "rgw_torrent_sha_unit": "524288", 2023-06-03T19:59:44.142 INFO:teuthology.orchestra.run.smithi089.stdout: "rgw_torrent_tracker": "", 2023-06-03T19:59:44.142 INFO:teuthology.orchestra.run.smithi089.stdout: "rgw_trust_forwarded_https": "false", 2023-06-03T19:59:44.142 INFO:teuthology.orchestra.run.smithi089.stdout: "rgw_usage_log_flush_threshold": "1024", 2023-06-03T19:59:44.142 INFO:teuthology.orchestra.run.smithi089.stdout: "rgw_usage_log_tick_interval": "30", 2023-06-03T19:59:44.142 INFO:teuthology.orchestra.run.smithi089.stdout: "rgw_usage_max_shards": "32", 2023-06-03T19:59:44.143 INFO:teuthology.orchestra.run.smithi089.stdout: "rgw_usage_max_user_shards": "1", 2023-06-03T19:59:44.143 INFO:teuthology.orchestra.run.smithi089.stdout: "rgw_use_opa_authz": "false", 2023-06-03T19:59:44.143 INFO:teuthology.orchestra.run.smithi089.stdout: "rgw_user_default_quota_max_objects": "-1", 2023-06-03T19:59:44.143 INFO:teuthology.orchestra.run.smithi089.stdout: "rgw_user_default_quota_max_size": "-1", 2023-06-03T19:59:44.143 INFO:teuthology.orchestra.run.smithi089.stdout: "rgw_user_max_buckets": "1000", 2023-06-03T19:59:44.143 INFO:teuthology.orchestra.run.smithi089.stdout: "rgw_user_policies_max_num": "100", 2023-06-03T19:59:44.143 INFO:teuthology.orchestra.run.smithi089.stdout: "rgw_user_quota_bucket_sync_interval": "180", 2023-06-03T19:59:44.143 INFO:teuthology.orchestra.run.smithi089.stdout: "rgw_user_quota_sync_idle_users": "false", 2023-06-03T19:59:44.144 INFO:teuthology.orchestra.run.smithi089.stdout: "rgw_user_quota_sync_interval": "86400", 2023-06-03T19:59:44.144 INFO:teuthology.orchestra.run.smithi089.stdout: "rgw_user_quota_sync_wait_time": "86400", 2023-06-03T19:59:44.144 INFO:teuthology.orchestra.run.smithi089.stdout: "rgw_user_unique_email": "true", 2023-06-03T19:59:44.144 INFO:teuthology.orchestra.run.smithi089.stdout: "rgw_verify_ssl": "true", 2023-06-03T19:59:44.144 INFO:teuthology.orchestra.run.smithi089.stdout: "rgw_website_routing_rules_max_num": "50", 2023-06-03T19:59:44.144 INFO:teuthology.orchestra.run.smithi089.stdout: "rgw_zone": "", 2023-06-03T19:59:44.144 INFO:teuthology.orchestra.run.smithi089.stdout: "rgw_zone_id": "", 2023-06-03T19:59:44.145 INFO:teuthology.orchestra.run.smithi089.stdout: "rgw_zone_root_pool": ".rgw.root", 2023-06-03T19:59:44.145 INFO:teuthology.orchestra.run.smithi089.stdout: "rgw_zonegroup": "", 2023-06-03T19:59:44.145 INFO:teuthology.orchestra.run.smithi089.stdout: "rgw_zonegroup_id": "", 2023-06-03T19:59:44.145 INFO:teuthology.orchestra.run.smithi089.stdout: "rgw_zonegroup_root_pool": ".rgw.root", 2023-06-03T19:59:44.145 INFO:teuthology.orchestra.run.smithi089.stdout: "rgwlc_auto_session_clear": "true", 2023-06-03T19:59:44.145 INFO:teuthology.orchestra.run.smithi089.stdout: "rgwlc_skip_bucket_step": "false", 2023-06-03T19:59:44.145 INFO:teuthology.orchestra.run.smithi089.stdout: "rocksdb_block_size": "4096", 2023-06-03T19:59:44.146 INFO:teuthology.orchestra.run.smithi089.stdout: "rocksdb_bloom_bits_per_key": "20", 2023-06-03T19:59:44.146 INFO:teuthology.orchestra.run.smithi089.stdout: "rocksdb_cache_index_and_filter_blocks": "true", 2023-06-03T19:59:44.146 INFO:teuthology.orchestra.run.smithi089.stdout: "rocksdb_cache_index_and_filter_blocks_with_high_priority": "false", 2023-06-03T19:59:44.146 INFO:teuthology.orchestra.run.smithi089.stdout: "rocksdb_cache_row_ratio": "0.000000", 2023-06-03T19:59:44.146 INFO:teuthology.orchestra.run.smithi089.stdout: "rocksdb_cache_shard_bits": "4", 2023-06-03T19:59:44.146 INFO:teuthology.orchestra.run.smithi089.stdout: "rocksdb_cache_size": "536870912", 2023-06-03T19:59:44.146 INFO:teuthology.orchestra.run.smithi089.stdout: "rocksdb_cache_type": "binned_lru", 2023-06-03T19:59:44.147 INFO:teuthology.orchestra.run.smithi089.stdout: "rocksdb_cf_compact_on_deletion": "true", 2023-06-03T19:59:44.147 INFO:teuthology.orchestra.run.smithi089.stdout: "rocksdb_cf_compact_on_deletion_sliding_window": "32768", 2023-06-03T19:59:44.147 INFO:teuthology.orchestra.run.smithi089.stdout: "rocksdb_cf_compact_on_deletion_trigger": "16384", 2023-06-03T19:59:44.147 INFO:teuthology.orchestra.run.smithi089.stdout: "rocksdb_collect_compaction_stats": "false", 2023-06-03T19:59:44.147 INFO:teuthology.orchestra.run.smithi089.stdout: "rocksdb_collect_extended_stats": "false", 2023-06-03T19:59:44.147 INFO:teuthology.orchestra.run.smithi089.stdout: "rocksdb_collect_memory_stats": "false", 2023-06-03T19:59:44.147 INFO:teuthology.orchestra.run.smithi089.stdout: "rocksdb_delete_range_threshold": "1048576", 2023-06-03T19:59:44.148 INFO:teuthology.orchestra.run.smithi089.stdout: "rocksdb_index_type": "binary_search", 2023-06-03T19:59:44.148 INFO:teuthology.orchestra.run.smithi089.stdout: "rocksdb_log_to_ceph_log": "true", 2023-06-03T19:59:44.148 INFO:teuthology.orchestra.run.smithi089.stdout: "rocksdb_metadata_block_size": "4096", 2023-06-03T19:59:44.148 INFO:teuthology.orchestra.run.smithi089.stdout: "rocksdb_partition_filters": "false", 2023-06-03T19:59:44.148 INFO:teuthology.orchestra.run.smithi089.stdout: "rocksdb_perf": "false", 2023-06-03T19:59:44.148 INFO:teuthology.orchestra.run.smithi089.stdout: "rocksdb_pin_l0_filter_and_index_blocks_in_cache": "false", 2023-06-03T19:59:44.149 INFO:teuthology.orchestra.run.smithi089.stdout: "rotating_keys_bootstrap_timeout": "30", 2023-06-03T19:59:44.149 INFO:teuthology.orchestra.run.smithi089.stdout: "rotating_keys_renewal_timeout": "10", 2023-06-03T19:59:44.149 INFO:teuthology.orchestra.run.smithi089.stdout: "run_dir": "/var/run/ceph", 2023-06-03T19:59:44.149 INFO:teuthology.orchestra.run.smithi089.stdout: "seastore_block_create": "true", 2023-06-03T19:59:44.149 INFO:teuthology.orchestra.run.smithi089.stdout: "seastore_cache_lru_size": "67108864", 2023-06-03T19:59:44.149 INFO:teuthology.orchestra.run.smithi089.stdout: "seastore_cbjournal_size": "5368709120", 2023-06-03T19:59:44.150 INFO:teuthology.orchestra.run.smithi089.stdout: "seastore_default_max_object_size": "16777216", 2023-06-03T19:59:44.150 INFO:teuthology.orchestra.run.smithi089.stdout: "seastore_default_object_metadata_reservation": "16777216", 2023-06-03T19:59:44.150 INFO:teuthology.orchestra.run.smithi089.stdout: "seastore_device_size": "53687091200", 2023-06-03T19:59:44.150 INFO:teuthology.orchestra.run.smithi089.stdout: "seastore_journal_batch_capacity": "16", 2023-06-03T19:59:44.150 INFO:teuthology.orchestra.run.smithi089.stdout: "seastore_journal_batch_flush_size": "16777216", 2023-06-03T19:59:44.150 INFO:teuthology.orchestra.run.smithi089.stdout: "seastore_journal_batch_preferred_fullness": "0.950000", 2023-06-03T19:59:44.151 INFO:teuthology.orchestra.run.smithi089.stdout: "seastore_journal_iodepth_limit": "5", 2023-06-03T19:59:44.151 INFO:teuthology.orchestra.run.smithi089.stdout: "seastore_main_device_type": "SSD", 2023-06-03T19:59:44.151 INFO:teuthology.orchestra.run.smithi089.stdout: "seastore_max_concurrent_transactions": "8", 2023-06-03T19:59:44.151 INFO:teuthology.orchestra.run.smithi089.stdout: "seastore_multiple_tiers_default_evict_ratio": "0.600000", 2023-06-03T19:59:44.151 INFO:teuthology.orchestra.run.smithi089.stdout: "seastore_multiple_tiers_fast_evict_ratio": "0.700000", 2023-06-03T19:59:44.151 INFO:teuthology.orchestra.run.smithi089.stdout: "seastore_multiple_tiers_stop_evict_ratio": "0.500000", 2023-06-03T19:59:44.152 INFO:teuthology.orchestra.run.smithi089.stdout: "seastore_obj_data_write_amplification": "1.250000", 2023-06-03T19:59:44.152 INFO:teuthology.orchestra.run.smithi089.stdout: "seastore_segment_size": "67108864", 2023-06-03T19:59:44.152 INFO:teuthology.orchestra.run.smithi089.stdout: "set_keepcaps": "false", 2023-06-03T19:59:44.152 INFO:teuthology.orchestra.run.smithi089.stdout: "setgroup": "ceph", 2023-06-03T19:59:44.152 INFO:teuthology.orchestra.run.smithi089.stdout: "setuser": "ceph", 2023-06-03T19:59:44.152 INFO:teuthology.orchestra.run.smithi089.stdout: "setuser_match_path": "", 2023-06-03T19:59:44.153 INFO:teuthology.orchestra.run.smithi089.stdout: "target_max_misplaced_ratio": "0.050000", 2023-06-03T19:59:44.153 INFO:teuthology.orchestra.run.smithi089.stdout: "thp": "false", 2023-06-03T19:59:44.153 INFO:teuthology.orchestra.run.smithi089.stdout: "threadpool_default_timeout": "60", 2023-06-03T19:59:44.153 INFO:teuthology.orchestra.run.smithi089.stdout: "threadpool_empty_queue_max_wait": "2", 2023-06-03T19:59:44.153 INFO:teuthology.orchestra.run.smithi089.stdout: "throttler_perf_counter": "true" 2023-06-03T19:59:44.154 INFO:teuthology.orchestra.run.smithi089.stdout:} 2023-06-03T19:59:44.154 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:59:43 smithi164 bash[59462]: cluster 1685822382.4107969 mgr.y (mgr.34104) 702 : cluster 0 pgmap v606: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-06-03T19:59:44.155 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:59:43 smithi164 bash[59462]: audit 1685822383.1908371 mon.c (mon.2) 371 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-03T19:59:44.155 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:59:43 smithi089 bash[78452]: cluster 1685822382.4107969 mgr.y (mgr.34104) 702 : cluster 0 pgmap v606: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-06-03T19:59:44.155 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:59:43 smithi089 bash[78452]: audit 1685822383.1908371 mon.c (mon.2) 371 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-03T19:59:44.155 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:59:43 smithi089 bash[75068]: cluster 1685822382.4107969 mgr.y (mgr.34104) 702 : cluster 0 pgmap v606: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-06-03T19:59:44.155 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:59:43 smithi089 bash[75068]: audit 1685822383.1908371 mon.c (mon.2) 371 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-03T19:59:44.786 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph -- tell 'mon.*' injectargs --mon_osd_down_out_interval=0 2023-06-03T19:59:46.029 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:59:45 smithi089 bash[75068]: cluster 1685822384.4118435 mgr.y (mgr.34104) 703 : cluster 0 pgmap v607: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-06-03T19:59:46.029 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:59:45 smithi089 bash[78452]: cluster 1685822384.4118435 mgr.y (mgr.34104) 703 : cluster 0 pgmap v607: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-06-03T19:59:46.084 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:59:45 smithi164 bash[59462]: cluster 1685822384.4118435 mgr.y (mgr.34104) 703 : cluster 0 pgmap v607: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-06-03T19:59:46.529 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 19:59:46 smithi089 bash[71163]: ::ffff:172.21.15.164 - - [03/Jun/2023:19:59:46] "GET /metrics HTTP/1.1" 200 225296 "" "Prometheus/2.33.4" 2023-06-03T19:59:48.029 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:59:47 smithi089 bash[78452]: cluster 1685822386.4126544 mgr.y (mgr.34104) 704 : cluster 0 pgmap v608: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-06-03T19:59:48.029 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:59:47 smithi089 bash[75068]: cluster 1685822386.4126544 mgr.y (mgr.34104) 704 : cluster 0 pgmap v608: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-06-03T19:59:48.084 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:59:47 smithi164 bash[59462]: cluster 1685822386.4126544 mgr.y (mgr.34104) 704 : cluster 0 pgmap v608: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-06-03T19:59:49.354 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/mon.a/config 2023-06-03T19:59:50.029 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:59:49 smithi089 bash[75068]: cluster 1685822388.4133968 mgr.y (mgr.34104) 705 : cluster 0 pgmap v609: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-06-03T19:59:50.029 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:59:49 smithi089 bash[75068]: audit 1685822389.6376467 mon.a (mon.0) 645 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:59:50.029 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:59:49 smithi089 bash[75068]: audit 1685822389.6491282 mon.a (mon.0) 646 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:59:50.029 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:59:49 smithi089 bash[75068]: audit 1685822389.6934133 mon.a (mon.0) 647 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:59:50.029 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:59:49 smithi089 bash[75068]: audit 1685822389.7046247 mon.a (mon.0) 648 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:59:50.030 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:59:49 smithi089 bash[78452]: cluster 1685822388.4133968 mgr.y (mgr.34104) 705 : cluster 0 pgmap v609: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-06-03T19:59:50.030 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:59:49 smithi089 bash[78452]: audit 1685822389.6376467 mon.a (mon.0) 645 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:59:50.030 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:59:49 smithi089 bash[78452]: audit 1685822389.6491282 mon.a (mon.0) 646 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:59:50.030 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:59:49 smithi089 bash[78452]: audit 1685822389.6934133 mon.a (mon.0) 647 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:59:50.030 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:59:49 smithi089 bash[78452]: audit 1685822389.7046247 mon.a (mon.0) 648 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:59:50.084 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:59:49 smithi164 bash[59462]: cluster 1685822388.4133968 mgr.y (mgr.34104) 705 : cluster 0 pgmap v609: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-06-03T19:59:50.084 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:59:49 smithi164 bash[59462]: audit 1685822389.6376467 mon.a (mon.0) 645 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:59:50.084 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:59:49 smithi164 bash[59462]: audit 1685822389.6491282 mon.a (mon.0) 646 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:59:50.084 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:59:49 smithi164 bash[59462]: audit 1685822389.6934133 mon.a (mon.0) 647 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:59:50.085 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:59:49 smithi164 bash[59462]: audit 1685822389.7046247 mon.a (mon.0) 648 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:59:51.028 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:59:50 smithi089 bash[75068]: audit 1685822390.1894808 mon.c (mon.2) 372 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-03T19:59:51.029 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:59:50 smithi089 bash[75068]: audit 1685822390.1913123 mon.c (mon.2) 373 : audit 1 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-03T19:59:51.029 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:59:50 smithi089 bash[75068]: audit 1685822390.2004976 mon.a (mon.0) 649 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:59:51.029 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:59:50 smithi089 bash[78452]: audit 1685822390.1894808 mon.c (mon.2) 372 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-03T19:59:51.030 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:59:50 smithi089 bash[78452]: audit 1685822390.1913123 mon.c (mon.2) 373 : audit 1 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-03T19:59:51.030 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:59:50 smithi089 bash[78452]: audit 1685822390.2004976 mon.a (mon.0) 649 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:59:51.084 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:59:50 smithi164 bash[59462]: audit 1685822390.1894808 mon.c (mon.2) 372 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-03T19:59:51.084 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:59:50 smithi164 bash[59462]: audit 1685822390.1913123 mon.c (mon.2) 373 : audit 1 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-03T19:59:51.084 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:59:50 smithi164 bash[59462]: audit 1685822390.2004976 mon.a (mon.0) 649 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T19:59:52.028 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:59:51 smithi089 bash[78452]: cluster 1685822390.41383 mgr.y (mgr.34104) 706 : cluster 0 pgmap v610: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-06-03T19:59:52.029 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:59:51 smithi089 bash[75068]: cluster 1685822390.41383 mgr.y (mgr.34104) 706 : cluster 0 pgmap v610: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-06-03T19:59:52.084 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:59:51 smithi164 bash[59462]: cluster 1685822390.41383 mgr.y (mgr.34104) 706 : cluster 0 pgmap v610: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-06-03T19:59:52.962 INFO:teuthology.orchestra.run.smithi089.stdout:mon.a: {} 2023-06-03T19:59:52.984 INFO:teuthology.orchestra.run.smithi089.stdout:mon.b: {} 2023-06-03T19:59:53.003 INFO:teuthology.orchestra.run.smithi089.stdout:mon.c: {} 2023-06-03T19:59:53.005 INFO:teuthology.orchestra.run.smithi089.stderr:mon.a: mon_osd_down_out_interval = '0' (not observed, change may require restart) 2023-06-03T19:59:53.005 INFO:teuthology.orchestra.run.smithi089.stderr:mon.b: mon_osd_down_out_interval = '0' (not observed, change may require restart) 2023-06-03T19:59:53.005 INFO:teuthology.orchestra.run.smithi089.stderr:mon.c: mon_osd_down_out_interval = '0' (not observed, change may require restart) 2023-06-03T19:59:53.028 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:59:52 smithi089 bash[75068]: audit 1685822392.0937476 mon.c (mon.2) 374 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T19:59:53.029 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:59:52 smithi089 bash[78452]: audit 1685822392.0937476 mon.c (mon.2) 374 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T19:59:53.084 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:59:52 smithi164 bash[59462]: audit 1685822392.0937476 mon.c (mon.2) 374 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T19:59:53.885 INFO:tasks.thrashosds.thrasher:Unable to test ceph-objectstore-tool, not available on all OSD nodes 2023-06-03T19:59:53.886 INFO:teuthology.task.sequential:In sequential, running task full_sequential... 2023-06-03T19:59:53.886 INFO:teuthology.task.full_sequential:In full_sequential, running task radosbench... 2023-06-03T19:59:53.887 INFO:tasks.radosbench:Beginning radosbench... 2023-06-03T19:59:53.887 INFO:tasks.cephadm.ceph_manager.ceph:creating pool_name unique_pool_4 2023-06-03T19:59:53.887 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph osd pool create unique_pool_4 16 2023-06-03T19:59:53.889 INFO:tasks.thrashosds.thrasher:starting do_thrash 2023-06-03T19:59:53.889 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 5, 6, 7] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-06-03T19:59:53.889 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph osd reweight 0 1 2023-06-03T19:59:53.890 INFO:tasks.thrashosds.thrasher:starting do_sighup with a delay of 0.1 2023-06-03T19:59:53.890 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T19:59:53.890 INFO:tasks.thrashosds.thrasher:starting do_optrack_toggle with a delay of 2.0 2023-06-03T19:59:53.891 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-06-03T19:59:53.891 INFO:tasks.thrashosds.thrasher:starting do_dump_ops 2023-06-03T19:59:53.891 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph daemon osd.0 dump_ops_in_flight 2023-06-03T19:59:53.892 INFO:tasks.thrashosds.thrasher:starting do_noscrub_toggle with a delay of 2.0 2023-06-03T19:59:53.892 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph osd set noscrub 2023-06-03T19:59:53.933 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T19:59:53.936 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T19:59:53.937 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T19:59:54.038 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T19:59:54.048 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:59:53 smithi089 bash[75068]: cluster 1685822392.4145663 mgr.y (mgr.34104) 707 : cluster 0 pgmap v611: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-06-03T19:59:54.048 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:59:53 smithi089 bash[75068]: audit 1685822392.942688 mon.c (mon.2) 375 : audit 0 from='client.? 172.21.15.89:0/850673529' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-06-03T19:59:54.049 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:59:53 smithi089 bash[78452]: cluster 1685822392.4145663 mgr.y (mgr.34104) 707 : cluster 0 pgmap v611: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-06-03T19:59:54.049 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:59:53 smithi089 bash[78452]: audit 1685822392.942688 mon.c (mon.2) 375 : audit 0 from='client.? 172.21.15.89:0/850673529' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-06-03T19:59:54.071 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T19:59:54.073 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T19:59:54.073 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T19:59:54.084 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:59:53 smithi164 bash[59462]: cluster 1685822392.4145663 mgr.y (mgr.34104) 707 : cluster 0 pgmap v611: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-06-03T19:59:54.084 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:59:53 smithi164 bash[59462]: audit 1685822392.942688 mon.c (mon.2) 375 : audit 0 from='client.? 172.21.15.89:0/850673529' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-06-03T19:59:54.175 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T19:59:54.206 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T19:59:54.209 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T19:59:54.209 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T19:59:54.310 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T19:59:54.347 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T19:59:54.351 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T19:59:54.351 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T19:59:54.452 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T19:59:54.487 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T19:59:54.490 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T19:59:54.490 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T19:59:54.591 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T19:59:54.632 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T19:59:54.635 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T19:59:54.635 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T19:59:54.736 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T19:59:54.770 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T19:59:54.772 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T19:59:54.773 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T19:59:54.875 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T19:59:54.917 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T19:59:54.921 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T19:59:54.922 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T19:59:55.022 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T19:59:55.059 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T19:59:55.063 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T19:59:55.064 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T19:59:55.164 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T19:59:55.199 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T19:59:55.203 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T19:59:55.203 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T19:59:55.304 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T19:59:55.338 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T19:59:55.341 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T19:59:55.342 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T19:59:55.442 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T19:59:55.482 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T19:59:55.484 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T19:59:55.485 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T19:59:55.586 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T19:59:55.625 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T19:59:55.627 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T19:59:55.628 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T19:59:55.729 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T19:59:55.767 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T19:59:55.770 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T19:59:55.771 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T19:59:55.872 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T19:59:55.906 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T19:59:55.909 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T19:59:55.911 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T19:59:56.012 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T19:59:56.049 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T19:59:56.051 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:59:55 smithi089 bash[78452]: cluster 1685822394.4155736 mgr.y (mgr.34104) 708 : cluster 0 pgmap v612: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-06-03T19:59:56.051 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:59:55 smithi089 bash[75068]: cluster 1685822394.4155736 mgr.y (mgr.34104) 708 : cluster 0 pgmap v612: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-06-03T19:59:56.052 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T19:59:56.052 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T19:59:56.084 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:59:55 smithi164 bash[59462]: cluster 1685822394.4155736 mgr.y (mgr.34104) 708 : cluster 0 pgmap v612: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-06-03T19:59:56.153 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T19:59:56.184 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T19:59:56.188 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T19:59:56.188 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T19:59:56.289 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T19:59:56.303 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 19:59:56 smithi089 bash[71163]: ::ffff:172.21.15.164 - - [03/Jun/2023:19:59:56] "GET /metrics HTTP/1.1" 200 225291 "" "Prometheus/2.33.4" 2023-06-03T19:59:56.326 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T19:59:56.329 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T19:59:56.329 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T19:59:56.430 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T19:59:56.473 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T19:59:56.475 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T19:59:56.476 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T19:59:56.577 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T19:59:56.611 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T19:59:56.614 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T19:59:56.615 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T19:59:56.716 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T19:59:56.749 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T19:59:56.751 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T19:59:56.752 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T19:59:56.853 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T19:59:56.890 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T19:59:56.894 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T19:59:56.894 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T19:59:56.995 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T19:59:57.038 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T19:59:57.042 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T19:59:57.043 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T19:59:57.144 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T19:59:57.182 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T19:59:57.184 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T19:59:57.185 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T19:59:57.286 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T19:59:57.318 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T19:59:57.320 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T19:59:57.321 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T19:59:57.422 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T19:59:57.456 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T19:59:57.459 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T19:59:57.460 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T19:59:57.561 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T19:59:57.598 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T19:59:57.601 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T19:59:57.602 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T19:59:57.703 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T19:59:57.741 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T19:59:57.745 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T19:59:57.745 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T19:59:57.834 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:59:57 smithi164 bash[59462]: cluster 1685822396.416355 mgr.y (mgr.34104) 709 : cluster 0 pgmap v613: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-06-03T19:59:57.846 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T19:59:57.883 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T19:59:57.886 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T19:59:57.887 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T19:59:57.988 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T19:59:58.024 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T19:59:58.027 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T19:59:58.027 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T19:59:58.128 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T19:59:58.140 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:59:57 smithi089 bash[78452]: cluster 1685822396.416355 mgr.y (mgr.34104) 709 : cluster 0 pgmap v613: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-06-03T19:59:58.140 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:59:57 smithi089 bash[75068]: cluster 1685822396.416355 mgr.y (mgr.34104) 709 : cluster 0 pgmap v613: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-06-03T19:59:58.162 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T19:59:58.166 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T19:59:58.166 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T19:59:58.267 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T19:59:58.308 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T19:59:58.311 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T19:59:58.312 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T19:59:58.412 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T19:59:58.451 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T19:59:58.453 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T19:59:58.454 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T19:59:58.555 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T19:59:58.596 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T19:59:58.599 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T19:59:58.600 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T19:59:58.701 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T19:59:58.710 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/mon.a/config 2023-06-03T19:59:58.713 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/mon.a/config 2023-06-03T19:59:58.713 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/mon.a/config 2023-06-03T19:59:58.713 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/mon.a/config 2023-06-03T19:59:58.714 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/mon.a/config 2023-06-03T19:59:58.742 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T19:59:58.745 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T19:59:58.746 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T19:59:58.847 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T19:59:58.879 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T19:59:58.881 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T19:59:58.881 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T19:59:58.982 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T19:59:59.014 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T19:59:59.016 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T19:59:59.017 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T19:59:59.118 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T19:59:59.151 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T19:59:59.153 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T19:59:59.154 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T19:59:59.255 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T19:59:59.289 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T19:59:59.291 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T19:59:59.292 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T19:59:59.393 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T19:59:59.427 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T19:59:59.431 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T19:59:59.431 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T19:59:59.532 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T19:59:59.565 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T19:59:59.567 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T19:59:59.568 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T19:59:59.669 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T19:59:59.707 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T19:59:59.711 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T19:59:59.711 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T19:59:59.812 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T19:59:59.844 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T19:59:59.847 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T19:59:59.847 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T19:59:59.948 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T19:59:59.960 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 19:59:59 smithi164 bash[59462]: cluster 1685822398.417433 mgr.y (mgr.34104) 710 : cluster 0 pgmap v614: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-06-03T19:59:59.986 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T19:59:59.988 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T19:59:59.989 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:00:00.090 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:00:00.102 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 19:59:59 smithi089 bash[75068]: cluster 1685822398.417433 mgr.y (mgr.34104) 710 : cluster 0 pgmap v614: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-06-03T20:00:00.102 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 19:59:59 smithi089 bash[78452]: cluster 1685822398.417433 mgr.y (mgr.34104) 710 : cluster 0 pgmap v614: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-06-03T20:00:00.121 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:00:00.124 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:00:00.124 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:00:00.225 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:00:00.255 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:00:00.258 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:00:00.259 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:00:00.360 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:00:00.397 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:00:00.400 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:00:00.400 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:00:00.501 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:00:00.537 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:00:00.540 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:00:00.541 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:00:00.642 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:00:00.679 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:00:00.682 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:00:00.682 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:00:00.783 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:00:00.816 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:00:00.818 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:00:00.819 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:00:00.920 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:00:00.948 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:00:00.950 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:00:00.951 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:00:01.052 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:00:01.064 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:00:00 smithi089 bash[75068]: cluster 1685822400.000167 mon.a (mon.0) 650 : cluster 1 overall HEALTH_OK 2023-06-03T20:00:01.064 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:00:00 smithi089 bash[78452]: cluster 1685822400.000167 mon.a (mon.0) 650 : cluster 1 overall HEALTH_OK 2023-06-03T20:00:01.083 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:00:01.084 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:00:00 smithi164 bash[59462]: cluster 1685822400.000167 mon.a (mon.0) 650 : cluster 1 overall HEALTH_OK 2023-06-03T20:00:01.085 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:00:01.086 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:00:01.187 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:00:01.230 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:00:01.233 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:00:01.234 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:00:01.335 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:00:01.360 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:00:01.362 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:00:01.363 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:00:01.464 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:00:01.494 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:00:01.496 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:00:01.497 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:00:01.597 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:00:01.624 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:00:01.626 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:00:01.627 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:00:01.728 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:00:01.765 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:00:01.768 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:00:01.769 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:00:01.870 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:00:01.910 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:00:01.913 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:00:01.914 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:00:01.955 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:00:01 smithi089 bash[75068]: cluster 1685822400.4181163 mgr.y (mgr.34104) 711 : cluster 0 pgmap v615: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-06-03T20:00:01.956 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:00:01 smithi089 bash[78452]: cluster 1685822400.4181163 mgr.y (mgr.34104) 711 : cluster 0 pgmap v615: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-06-03T20:00:02.014 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:00:02.042 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:00:02.044 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:00:02.045 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:00:02.084 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:00:01 smithi164 bash[59462]: cluster 1685822400.4181163 mgr.y (mgr.34104) 711 : cluster 0 pgmap v615: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-06-03T20:00:02.145 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:00:02.187 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:00:02.189 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:00:02.191 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:00:02.292 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:00:02.325 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:00:02.327 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:00:02.328 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:00:02.430 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:00:02.461 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:00:02.464 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:00:02.464 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:00:02.565 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:00:02.603 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:00:02.606 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:00:02.606 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:00:02.707 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:00:02.736 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:00:02.739 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:00:02.740 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:00:02.841 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:00:02.878 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:00:02.880 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:00:02.881 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:00:02.982 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:00:03.022 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:00:03.025 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:00:03.026 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:00:03.127 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:00:03.156 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:00:03.159 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:00:03.159 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:00:03.260 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:00:03.289 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:00:03.292 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:00:03.292 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:00:03.393 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:00:03.435 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:00:03.437 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:00:03.438 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:00:03.539 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:00:03.568 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:00:03.571 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:00:03.572 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:00:03.672 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:00:03.697 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:00:03.699 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:00:03.700 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:00:03.800 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:00:03.839 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:00:03.841 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:00:03.842 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:00:03.943 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:00:03.980 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:00:03.983 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:00:03.983 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:00:04.084 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:00:03 smithi164 bash[59462]: cluster 1685822402.4187841 mgr.y (mgr.34104) 712 : cluster 0 pgmap v616: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-06-03T20:00:04.084 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:00:04.087 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:00:03 smithi089 bash[75068]: cluster 1685822402.4187841 mgr.y (mgr.34104) 712 : cluster 0 pgmap v616: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-06-03T20:00:04.087 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:00:03 smithi089 bash[78452]: cluster 1685822402.4187841 mgr.y (mgr.34104) 712 : cluster 0 pgmap v616: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-06-03T20:00:04.108 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:00:04.111 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:00:04.111 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:00:04.212 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:00:04.251 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:00:04.254 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:00:04.255 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:00:04.356 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:00:04.381 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:00:04.384 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:00:04.385 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:00:04.485 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:00:04.517 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:00:04.519 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:00:04.520 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:00:04.621 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:00:04.651 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:00:04.653 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:00:04.653 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:00:04.754 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:00:04.791 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:00:04.793 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:00:04.794 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:00:04.894 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:00:04.935 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:00:04.937 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:00:04.938 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:00:05.039 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:00:05.073 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:00:05.077 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:00:05.078 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:00:05.179 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:00:05.208 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:00:05.210 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:00:05.211 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:00:05.311 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:00:05.347 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:00:05.349 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:00:05.350 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:00:05.451 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:00:05.486 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:00:05.488 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:00:05.490 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:00:05.591 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:00:05.625 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:00:05.627 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:00:05.628 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:00:05.729 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:00:05.766 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:00:05.768 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:00:05.768 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:00:05.869 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:00:05.877 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:00:05 smithi089 bash[75068]: cluster 1685822404.4195616 mgr.y (mgr.34104) 713 : cluster 0 pgmap v617: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-06-03T20:00:05.877 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:00:05 smithi089 bash[78452]: cluster 1685822404.4195616 mgr.y (mgr.34104) 713 : cluster 0 pgmap v617: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-06-03T20:00:05.895 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:00:05.899 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:00:05.900 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:00:06.001 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:00:06.031 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:00:06.033 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:00:06.034 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:00:06.084 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:00:05 smithi164 bash[59462]: cluster 1685822404.4195616 mgr.y (mgr.34104) 713 : cluster 0 pgmap v617: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-06-03T20:00:06.135 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:00:06.170 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:00:06.173 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:00:06.174 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:00:06.275 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:00:06.311 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:00:06.313 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:00:06.314 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:00:06.415 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:00:06.423 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 20:00:06 smithi089 bash[71163]: ::ffff:172.21.15.164 - - [03/Jun/2023:20:00:06] "GET /metrics HTTP/1.1" 200 225291 "" "Prometheus/2.33.4" 2023-06-03T20:00:06.443 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:00:06.446 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:00:06.446 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:00:06.547 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:00:06.581 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:00:06.584 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:00:06.585 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:00:06.686 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:00:06.716 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:00:06.718 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:00:06.719 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:00:06.819 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:00:06.857 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:00:06.858 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:00:06.859 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:00:06.960 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:00:06.996 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:00:06.999 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:00:06.999 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:00:07.100 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:00:07.126 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:00:07.129 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:00:07.130 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:00:07.230 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:00:07.257 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:00:07.259 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:00:07.260 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:00:07.361 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:00:07.397 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:00:07.399 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:00:07.400 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:00:07.501 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:00:07.539 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:00:07.542 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:00:07.543 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:00:07.643 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:00:07.673 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:00:07.676 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:00:07.677 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:00:07.777 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:00:07.817 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:00:07.820 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:00:07.820 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:00:07.921 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:00:07.959 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:00:07.962 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:00:07.963 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:00:08.028 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:00:07 smithi089 bash[75068]: cluster 1685822406.4200222 mgr.y (mgr.34104) 714 : cluster 0 pgmap v618: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-06-03T20:00:08.029 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:00:07 smithi089 bash[75068]: audit 1685822407.0935872 mon.c (mon.2) 376 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T20:00:08.029 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:00:07 smithi089 bash[78452]: cluster 1685822406.4200222 mgr.y (mgr.34104) 714 : cluster 0 pgmap v618: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-06-03T20:00:08.029 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:00:07 smithi089 bash[78452]: audit 1685822407.0935872 mon.c (mon.2) 376 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T20:00:08.064 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:00:08.077 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:00:07 smithi164 bash[59462]: cluster 1685822406.4200222 mgr.y (mgr.34104) 714 : cluster 0 pgmap v618: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-06-03T20:00:08.077 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:00:07 smithi164 bash[59462]: audit 1685822407.0935872 mon.c (mon.2) 376 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T20:00:08.095 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:00:08.097 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:00:08.098 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:00:08.199 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:00:08.228 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:00:08.231 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:00:08.231 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:00:08.332 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:00:08.366 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:00:08.372 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:00:08.373 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:00:08.474 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:00:08.503 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:00:08.505 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:00:08.506 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:00:08.607 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:00:08.644 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:00:08.646 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:00:08.647 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:00:08.748 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:00:08.783 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:00:08.787 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:00:08.788 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:00:08.889 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:00:08.918 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:00:08.921 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:00:08.921 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:00:09.022 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:00:09.058 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:00:09.060 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:00:09.061 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:00:09.162 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:00:09.198 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:00:09.201 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:00:09.202 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:00:09.303 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:00:09.333 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:00:09.334 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:00:09.335 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:00:09.436 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:00:09.470 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:00:09.473 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:00:09.474 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:00:09.575 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:00:09.604 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:00:09.605 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:00:09.606 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:00:09.707 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:00:09.745 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:00:09.749 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:00:09.749 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:00:09.850 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:00:09.866 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:00:09 smithi089 bash[78452]: cluster 1685822408.4211843 mgr.y (mgr.34104) 715 : cluster 0 pgmap v619: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-06-03T20:00:09.866 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:00:09 smithi089 bash[75068]: cluster 1685822408.4211843 mgr.y (mgr.34104) 715 : cluster 0 pgmap v619: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-06-03T20:00:09.889 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:00:09.892 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:00:09.893 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:00:09.994 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:00:10.018 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:00:10.020 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:00:10.021 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:00:10.084 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:00:09 smithi164 bash[59462]: cluster 1685822408.4211843 mgr.y (mgr.34104) 715 : cluster 0 pgmap v619: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-06-03T20:00:10.121 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:00:10.151 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:00:10.153 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:00:10.153 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:00:10.229 INFO:teuthology.orchestra.run.smithi089.stdout:{ 2023-06-03T20:00:10.229 INFO:teuthology.orchestra.run.smithi089.stdout: "ops": [], 2023-06-03T20:00:10.229 INFO:teuthology.orchestra.run.smithi089.stdout: "num_ops": 0 2023-06-03T20:00:10.229 INFO:teuthology.orchestra.run.smithi089.stdout:} 2023-06-03T20:00:10.255 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:00:10.288 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:00:10.291 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:00:10.292 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:00:10.392 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:00:10.428 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:00:10.430 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:00:10.431 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:00:10.532 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:00:10.562 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:00:10.563 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:00:10.564 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:00:10.665 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:00:10.695 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:00:10.698 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:00:10.699 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:00:10.800 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:00:10.832 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:00:10.835 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:00:10.836 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:00:10.837 INFO:teuthology.orchestra.run.smithi089.stderr:noscrub is set 2023-06-03T20:00:10.938 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:00:10.980 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:00:10.983 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:00:10.984 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:00:11.068 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:00:10 smithi089 bash[75068]: audit 1685822410.2506335 mon.c (mon.2) 377 : audit 1 from='client.? 172.21.15.89:0/3577473766' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-06-03T20:00:11.068 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:00:10 smithi089 bash[75068]: audit 1685822410.2511334 mon.a (mon.0) 651 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-06-03T20:00:11.069 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:00:10 smithi089 bash[78452]: audit 1685822410.2506335 mon.c (mon.2) 377 : audit 1 from='client.? 172.21.15.89:0/3577473766' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-06-03T20:00:11.069 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:00:10 smithi089 bash[78452]: audit 1685822410.2511334 mon.a (mon.0) 651 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-06-03T20:00:11.084 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:00:10 smithi164 bash[59462]: audit 1685822410.2506335 mon.c (mon.2) 377 : audit 1 from='client.? 172.21.15.89:0/3577473766' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-06-03T20:00:11.084 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:00:10 smithi164 bash[59462]: audit 1685822410.2511334 mon.a (mon.0) 651 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-06-03T20:00:11.085 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:00:11.121 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:00:11.125 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:00:11.126 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:00:11.227 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:00:11.248 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:00:11.251 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:00:11.251 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:00:11.352 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:00:11.374 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:00:11.376 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:00:11.377 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:00:11.478 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:00:11.501 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:00:11.504 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:00:11.505 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:00:11.537 INFO:teuthology.orchestra.run.smithi089.stdout:osd.0: {} 2023-06-03T20:00:11.558 INFO:teuthology.orchestra.run.smithi089.stdout:osd.1: {} 2023-06-03T20:00:11.579 INFO:teuthology.orchestra.run.smithi089.stdout:osd.2: {} 2023-06-03T20:00:11.600 INFO:teuthology.orchestra.run.smithi089.stdout:osd.3: {} 2023-06-03T20:00:11.605 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:00:11.623 INFO:teuthology.orchestra.run.smithi089.stdout:osd.4: {} 2023-06-03T20:00:11.640 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:00:11.643 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:00:11.644 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:00:11.645 INFO:teuthology.orchestra.run.smithi089.stdout:osd.5: {} 2023-06-03T20:00:11.669 INFO:teuthology.orchestra.run.smithi089.stdout:osd.6: {} 2023-06-03T20:00:11.694 INFO:teuthology.orchestra.run.smithi089.stdout:osd.7: {} 2023-06-03T20:00:11.696 INFO:teuthology.orchestra.run.smithi089.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-06-03T20:00:11.696 INFO:teuthology.orchestra.run.smithi089.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-06-03T20:00:11.696 INFO:teuthology.orchestra.run.smithi089.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-06-03T20:00:11.696 INFO:teuthology.orchestra.run.smithi089.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-06-03T20:00:11.696 INFO:teuthology.orchestra.run.smithi089.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-06-03T20:00:11.696 INFO:teuthology.orchestra.run.smithi089.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-06-03T20:00:11.697 INFO:teuthology.orchestra.run.smithi089.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-06-03T20:00:11.697 INFO:teuthology.orchestra.run.smithi089.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-06-03T20:00:11.745 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:00:11.780 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:00:11.783 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:00:11.784 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:00:11.884 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:00:11.895 INFO:teuthology.orchestra.run.smithi089.stderr:reweighted osd.0 to 1 (10000) 2023-06-03T20:00:11.896 INFO:teuthology.orchestra.run.smithi089.stderr:pool 'unique_pool_4' created 2023-06-03T20:00:11.917 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:00:11.920 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:00:11.920 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:00:12.001 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:00:11 smithi089 bash[75068]: cluster 1685822410.4219227 mgr.y (mgr.34104) 716 : cluster 0 pgmap v620: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-06-03T20:00:12.001 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:00:11 smithi089 bash[75068]: cluster 1685822410.8145 mon.a (mon.0) 652 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-06-03T20:00:12.002 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:00:11 smithi089 bash[75068]: cluster 1685822410.8228629 osd.6 (osd.6) 43 : cluster 3 failed to encode map e126 with expected crc 2023-06-03T20:00:12.002 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:00:11 smithi089 bash[75068]: cluster 1685822410.8232145 osd.5 (osd.5) 49 : cluster 3 failed to encode map e126 with expected crc 2023-06-03T20:00:12.002 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:00:11 smithi089 bash[75068]: audit 1685822410.826917 mon.a (mon.0) 653 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-06-03T20:00:12.002 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:00:11 smithi089 bash[75068]: cluster 1685822410.8269923 mon.a (mon.0) 654 : cluster 0 osdmap e126: 8 total, 8 up, 8 in 2023-06-03T20:00:12.002 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:00:11 smithi089 bash[75068]: audit 1685822411.2884452 mon.c (mon.2) 378 : audit 1 from='client.? 172.21.15.89:0/2227696954' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-06-03T20:00:12.002 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:00:11 smithi089 bash[75068]: audit 1685822411.2888875 mon.a (mon.0) 655 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-06-03T20:00:12.002 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:00:11 smithi089 bash[78452]: cluster 1685822410.4219227 mgr.y (mgr.34104) 716 : cluster 0 pgmap v620: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-06-03T20:00:12.003 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:00:11 smithi089 bash[78452]: cluster 1685822410.8145 mon.a (mon.0) 652 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-06-03T20:00:12.003 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:00:11 smithi089 bash[78452]: cluster 1685822410.8228629 osd.6 (osd.6) 43 : cluster 3 failed to encode map e126 with expected crc 2023-06-03T20:00:12.003 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:00:11 smithi089 bash[78452]: cluster 1685822410.8232145 osd.5 (osd.5) 49 : cluster 3 failed to encode map e126 with expected crc 2023-06-03T20:00:12.003 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:00:11 smithi089 bash[78452]: audit 1685822410.826917 mon.a (mon.0) 653 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-06-03T20:00:12.003 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:00:11 smithi089 bash[78452]: cluster 1685822410.8269923 mon.a (mon.0) 654 : cluster 0 osdmap e126: 8 total, 8 up, 8 in 2023-06-03T20:00:12.003 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:00:11 smithi089 bash[78452]: audit 1685822411.2884452 mon.c (mon.2) 378 : audit 1 from='client.? 172.21.15.89:0/2227696954' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-06-03T20:00:12.003 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:00:11 smithi089 bash[78452]: audit 1685822411.2888875 mon.a (mon.0) 655 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-06-03T20:00:12.022 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:00:12.046 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:00:12.049 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:00:12.050 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:00:12.151 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:00:12.183 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:00:12.186 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:00:12.187 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:00:12.287 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:00:12.316 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:00:12.317 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:00:12.318 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:00:12.334 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:00:11 smithi164 bash[59462]: cluster 1685822410.4219227 mgr.y (mgr.34104) 716 : cluster 0 pgmap v620: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-06-03T20:00:12.334 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:00:11 smithi164 bash[59462]: cluster 1685822410.8145 mon.a (mon.0) 652 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-06-03T20:00:12.334 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:00:11 smithi164 bash[59462]: cluster 1685822410.8228629 osd.6 (osd.6) 43 : cluster 3 failed to encode map e126 with expected crc 2023-06-03T20:00:12.335 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:00:11 smithi164 bash[59462]: cluster 1685822410.8232145 osd.5 (osd.5) 49 : cluster 3 failed to encode map e126 with expected crc 2023-06-03T20:00:12.335 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:00:11 smithi164 bash[59462]: audit 1685822410.826917 mon.a (mon.0) 653 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-06-03T20:00:12.335 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:00:11 smithi164 bash[59462]: cluster 1685822410.8269923 mon.a (mon.0) 654 : cluster 0 osdmap e126: 8 total, 8 up, 8 in 2023-06-03T20:00:12.335 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:00:11 smithi164 bash[59462]: audit 1685822411.2884452 mon.c (mon.2) 378 : audit 1 from='client.? 172.21.15.89:0/2227696954' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-06-03T20:00:12.335 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:00:11 smithi164 bash[59462]: audit 1685822411.2888875 mon.a (mon.0) 655 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-06-03T20:00:12.419 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:00:12.456 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:00:12.458 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:00:12.459 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:00:12.560 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:00:12.593 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:00:12.595 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:00:12.596 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:00:12.696 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:00:12.729 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:00:12.732 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:00:12.733 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:00:12.834 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:00:12.860 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:00:12.863 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:00:12.863 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:00:12.964 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:00:12.992 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph daemon osd.0 dump_blocked_ops 2023-06-03T20:00:12.993 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:00:12.995 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:00:12.995 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:00:13.084 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:00:12 smithi164 bash[59462]: audit 1685822411.428906 mon.a (mon.0) 656 : audit 1 from='client.? 172.21.15.89:0/1211844435' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_4", "pg_num": 16}]: dispatch 2023-06-03T20:00:13.084 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:00:12 smithi164 bash[59462]: audit 1685822411.5064573 mon.c (mon.2) 379 : audit 0 from='client.? 172.21.15.89:0/1793077484' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-03T20:00:13.085 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:00:12 smithi164 bash[59462]: cluster 1685822411.8795052 osd.6 (osd.6) 44 : cluster 3 failed to encode map e127 with expected crc 2023-06-03T20:00:13.085 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:00:12 smithi164 bash[59462]: cluster 1685822411.883123 osd.6 (osd.6) 45 : cluster 3 failed to encode map e127 with expected crc 2023-06-03T20:00:13.085 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:00:12 smithi164 bash[59462]: audit 1685822411.884509 mon.a (mon.0) 657 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2023-06-03T20:00:13.085 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:00:12 smithi164 bash[59462]: audit 1685822411.8846583 mon.a (mon.0) 658 : audit 1 from='client.? 172.21.15.89:0/1211844435' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_4", "pg_num": 16}]': finished 2023-06-03T20:00:13.085 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:00:12 smithi164 bash[59462]: cluster 1685822411.884737 osd.5 (osd.5) 50 : cluster 3 failed to encode map e127 with expected crc 2023-06-03T20:00:13.085 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:00:12 smithi164 bash[59462]: cluster 1685822411.8848076 osd.7 (osd.7) 41 : cluster 3 failed to encode map e127 with expected crc 2023-06-03T20:00:13.085 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:00:12 smithi164 bash[59462]: cluster 1685822411.8848784 mon.a (mon.0) 659 : cluster 0 osdmap e127: 8 total, 8 up, 8 in 2023-06-03T20:00:13.085 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:00:12 smithi164 bash[59462]: cluster 1685822411.8850665 osd.6 (osd.6) 46 : cluster 3 failed to encode map e127 with expected crc 2023-06-03T20:00:13.096 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:00:13.123 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:00:12 smithi089 bash[78452]: audit 1685822411.428906 mon.a (mon.0) 656 : audit 1 from='client.? 172.21.15.89:0/1211844435' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_4", "pg_num": 16}]: dispatch 2023-06-03T20:00:13.123 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:00:12 smithi089 bash[78452]: audit 1685822411.5064573 mon.c (mon.2) 379 : audit 0 from='client.? 172.21.15.89:0/1793077484' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-03T20:00:13.123 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:00:12 smithi089 bash[78452]: cluster 1685822411.8795052 osd.6 (osd.6) 44 : cluster 3 failed to encode map e127 with expected crc 2023-06-03T20:00:13.123 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:00:12 smithi089 bash[78452]: cluster 1685822411.883123 osd.6 (osd.6) 45 : cluster 3 failed to encode map e127 with expected crc 2023-06-03T20:00:13.123 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:00:12 smithi089 bash[78452]: audit 1685822411.884509 mon.a (mon.0) 657 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2023-06-03T20:00:13.124 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:00:12 smithi089 bash[78452]: audit 1685822411.8846583 mon.a (mon.0) 658 : audit 1 from='client.? 172.21.15.89:0/1211844435' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_4", "pg_num": 16}]': finished 2023-06-03T20:00:13.124 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:00:12 smithi089 bash[78452]: cluster 1685822411.884737 osd.5 (osd.5) 50 : cluster 3 failed to encode map e127 with expected crc 2023-06-03T20:00:13.124 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:00:12 smithi089 bash[78452]: cluster 1685822411.8848076 osd.7 (osd.7) 41 : cluster 3 failed to encode map e127 with expected crc 2023-06-03T20:00:13.124 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:00:12 smithi089 bash[78452]: cluster 1685822411.8848784 mon.a (mon.0) 659 : cluster 0 osdmap e127: 8 total, 8 up, 8 in 2023-06-03T20:00:13.124 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:00:12 smithi089 bash[78452]: cluster 1685822411.8850665 osd.6 (osd.6) 46 : cluster 3 failed to encode map e127 with expected crc 2023-06-03T20:00:13.124 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:00:12 smithi089 bash[75068]: audit 1685822411.428906 mon.a (mon.0) 656 : audit 1 from='client.? 172.21.15.89:0/1211844435' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_4", "pg_num": 16}]: dispatch 2023-06-03T20:00:13.125 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:00:12 smithi089 bash[75068]: audit 1685822411.5064573 mon.c (mon.2) 379 : audit 0 from='client.? 172.21.15.89:0/1793077484' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-03T20:00:13.125 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:00:12 smithi089 bash[75068]: cluster 1685822411.8795052 osd.6 (osd.6) 44 : cluster 3 failed to encode map e127 with expected crc 2023-06-03T20:00:13.125 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:00:12 smithi089 bash[75068]: cluster 1685822411.883123 osd.6 (osd.6) 45 : cluster 3 failed to encode map e127 with expected crc 2023-06-03T20:00:13.125 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:00:12 smithi089 bash[75068]: audit 1685822411.884509 mon.a (mon.0) 657 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2023-06-03T20:00:13.125 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:00:12 smithi089 bash[75068]: audit 1685822411.8846583 mon.a (mon.0) 658 : audit 1 from='client.? 172.21.15.89:0/1211844435' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_4", "pg_num": 16}]': finished 2023-06-03T20:00:13.125 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:00:12 smithi089 bash[75068]: cluster 1685822411.884737 osd.5 (osd.5) 50 : cluster 3 failed to encode map e127 with expected crc 2023-06-03T20:00:13.125 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:00:12 smithi089 bash[75068]: cluster 1685822411.8848076 osd.7 (osd.7) 41 : cluster 3 failed to encode map e127 with expected crc 2023-06-03T20:00:13.126 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:00:12 smithi089 bash[75068]: cluster 1685822411.8848784 mon.a (mon.0) 659 : cluster 0 osdmap e127: 8 total, 8 up, 8 in 2023-06-03T20:00:13.126 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:00:12 smithi089 bash[75068]: cluster 1685822411.8850665 osd.6 (osd.6) 46 : cluster 3 failed to encode map e127 with expected crc 2023-06-03T20:00:13.130 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:00:13.133 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:00:13.133 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:00:13.234 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:00:13.266 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:00:13.268 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:00:13.268 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:00:13.369 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:00:13.398 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:00:13.401 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:00:13.401 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:00:13.502 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:00:13.534 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:00:13.536 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:00:13.537 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:00:13.637 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:00:13.667 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:00:13.669 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:00:13.670 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:00:13.771 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:00:13.798 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:00:13.801 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:00:13.802 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:00:13.902 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:00:13.929 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:00:13.932 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:00:13.933 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:00:14.034 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:00:14.071 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:00:14.073 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:00:14.074 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:00:14.175 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:00:14.187 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:00:13 smithi164 bash[59462]: cluster 1685822412.422522 mgr.y (mgr.34104) 717 : cluster 0 pgmap v623: 89 pgs: 16 unknown, 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-06-03T20:00:14.187 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:00:13 smithi164 bash[59462]: cluster 1685822412.8936224 osd.6 (osd.6) 47 : cluster 3 failed to encode map e128 with expected crc 2023-06-03T20:00:14.188 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:00:13 smithi164 bash[59462]: cluster 1685822412.8939834 osd.5 (osd.5) 51 : cluster 3 failed to encode map e128 with expected crc 2023-06-03T20:00:14.188 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:00:13 smithi164 bash[59462]: cluster 1685822412.8957262 osd.7 (osd.7) 42 : cluster 3 failed to encode map e128 with expected crc 2023-06-03T20:00:14.188 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:00:13 smithi164 bash[59462]: cluster 1685822412.8972886 osd.5 (osd.5) 52 : cluster 3 failed to encode map e128 with expected crc 2023-06-03T20:00:14.188 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:00:13 smithi164 bash[59462]: cluster 1685822412.8976004 mon.a (mon.0) 660 : cluster 0 osdmap e128: 8 total, 8 up, 8 in 2023-06-03T20:00:14.209 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:00:14.212 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:00:14.213 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:00:14.279 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:00:13 smithi089 bash[75068]: cluster 1685822412.422522 mgr.y (mgr.34104) 717 : cluster 0 pgmap v623: 89 pgs: 16 unknown, 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-06-03T20:00:14.279 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:00:13 smithi089 bash[75068]: cluster 1685822412.8936224 osd.6 (osd.6) 47 : cluster 3 failed to encode map e128 with expected crc 2023-06-03T20:00:14.279 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:00:13 smithi089 bash[75068]: cluster 1685822412.8939834 osd.5 (osd.5) 51 : cluster 3 failed to encode map e128 with expected crc 2023-06-03T20:00:14.279 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:00:13 smithi089 bash[75068]: cluster 1685822412.8957262 osd.7 (osd.7) 42 : cluster 3 failed to encode map e128 with expected crc 2023-06-03T20:00:14.279 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:00:13 smithi089 bash[75068]: cluster 1685822412.8972886 osd.5 (osd.5) 52 : cluster 3 failed to encode map e128 with expected crc 2023-06-03T20:00:14.280 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:00:13 smithi089 bash[75068]: cluster 1685822412.8976004 mon.a (mon.0) 660 : cluster 0 osdmap e128: 8 total, 8 up, 8 in 2023-06-03T20:00:14.280 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:00:13 smithi089 bash[78452]: cluster 1685822412.422522 mgr.y (mgr.34104) 717 : cluster 0 pgmap v623: 89 pgs: 16 unknown, 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-06-03T20:00:14.280 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:00:13 smithi089 bash[78452]: cluster 1685822412.8936224 osd.6 (osd.6) 47 : cluster 3 failed to encode map e128 with expected crc 2023-06-03T20:00:14.280 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:00:13 smithi089 bash[78452]: cluster 1685822412.8939834 osd.5 (osd.5) 51 : cluster 3 failed to encode map e128 with expected crc 2023-06-03T20:00:14.280 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:00:13 smithi089 bash[78452]: cluster 1685822412.8957262 osd.7 (osd.7) 42 : cluster 3 failed to encode map e128 with expected crc 2023-06-03T20:00:14.280 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:00:13 smithi089 bash[78452]: cluster 1685822412.8972886 osd.5 (osd.5) 52 : cluster 3 failed to encode map e128 with expected crc 2023-06-03T20:00:14.281 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:00:13 smithi089 bash[78452]: cluster 1685822412.8976004 mon.a (mon.0) 660 : cluster 0 osdmap e128: 8 total, 8 up, 8 in 2023-06-03T20:00:14.313 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:00:14.348 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:00:14.350 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:00:14.350 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:00:14.451 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:00:14.480 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:00:14.483 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:00:14.483 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:00:14.584 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:00:14.615 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:00:14.618 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:00:14.618 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:00:14.719 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:00:14.744 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:00:14.747 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:00:14.747 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:00:14.848 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:00:14.874 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:00:14.876 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:00:14.877 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:00:14.978 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:00:15.019 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:00:15.022 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:00:15.022 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:00:15.124 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:00:15.159 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:00:15.163 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:00:15.163 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:00:15.264 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:00:15.277 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:00:14 smithi164 bash[59462]: cluster 1685822413.9006498 osd.6 (osd.6) 48 : cluster 3 failed to encode map e129 with expected crc 2023-06-03T20:00:15.277 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:00:14 smithi164 bash[59462]: cluster 1685822413.9009569 osd.5 (osd.5) 53 : cluster 3 failed to encode map e129 with expected crc 2023-06-03T20:00:15.278 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:00:14 smithi164 bash[59462]: cluster 1685822413.905524 mon.a (mon.0) 661 : cluster 0 osdmap e129: 8 total, 8 up, 8 in 2023-06-03T20:00:15.278 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:00:14 smithi089 bash[75068]: cluster 1685822413.9006498 osd.6 (osd.6) 48 : cluster 3 failed to encode map e129 with expected crc 2023-06-03T20:00:15.279 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:00:14 smithi089 bash[75068]: cluster 1685822413.9009569 osd.5 (osd.5) 53 : cluster 3 failed to encode map e129 with expected crc 2023-06-03T20:00:15.279 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:00:14 smithi089 bash[75068]: cluster 1685822413.905524 mon.a (mon.0) 661 : cluster 0 osdmap e129: 8 total, 8 up, 8 in 2023-06-03T20:00:15.279 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:00:14 smithi089 bash[78452]: cluster 1685822413.9006498 osd.6 (osd.6) 48 : cluster 3 failed to encode map e129 with expected crc 2023-06-03T20:00:15.279 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:00:14 smithi089 bash[78452]: cluster 1685822413.9009569 osd.5 (osd.5) 53 : cluster 3 failed to encode map e129 with expected crc 2023-06-03T20:00:15.279 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:00:14 smithi089 bash[78452]: cluster 1685822413.905524 mon.a (mon.0) 661 : cluster 0 osdmap e129: 8 total, 8 up, 8 in 2023-06-03T20:00:15.300 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:00:15.302 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:00:15.303 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:00:15.400 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph osd reweight 1 1 2023-06-03T20:00:15.404 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:00:15.436 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:00:15.440 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:00:15.440 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:00:15.442 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph osd pool application enable unique_pool_4 rados --yes-i-really-mean-it || true 2023-06-03T20:00:15.474 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph osd set nodeep-scrub 2023-06-03T20:00:15.541 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:00:15.578 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:00:15.582 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:00:15.582 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:00:15.683 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:00:15.722 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:00:15.725 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:00:15.725 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:00:15.826 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:00:15.859 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:00:15.862 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:00:15.863 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:00:15.964 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:00:16.003 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:00:16.016 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:00:16.017 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:00:16.118 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:00:16.158 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:00:16.160 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:00:16.161 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:00:16.233 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:00:15 smithi089 bash[78452]: cluster 1685822414.4229953 mgr.y (mgr.34104) 718 : cluster 0 pgmap v626: 89 pgs: 2 creating+activating, 4 creating+peering, 3 unknown, 80 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-06-03T20:00:16.233 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:00:15 smithi089 bash[75068]: cluster 1685822414.4229953 mgr.y (mgr.34104) 718 : cluster 0 pgmap v626: 89 pgs: 2 creating+activating, 4 creating+peering, 3 unknown, 80 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-06-03T20:00:16.263 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:00:16.276 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:00:15 smithi164 bash[59462]: cluster 1685822414.4229953 mgr.y (mgr.34104) 718 : cluster 0 pgmap v626: 89 pgs: 2 creating+activating, 4 creating+peering, 3 unknown, 80 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-06-03T20:00:16.298 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:00:16.302 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:00:16.302 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:00:16.403 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:00:16.440 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:00:16.443 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:00:16.444 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:00:16.528 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 20:00:16 smithi089 bash[71163]: ::ffff:172.21.15.164 - - [03/Jun/2023:20:00:16] "GET /metrics HTTP/1.1" 200 225294 "" "Prometheus/2.33.4" 2023-06-03T20:00:16.544 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:00:16.582 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:00:16.585 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:00:16.586 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:00:16.687 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:00:16.724 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:00:16.727 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:00:16.728 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:00:16.829 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:00:16.867 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:00:16.870 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:00:16.871 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:00:16.972 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:00:17.014 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:00:17.017 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:00:17.018 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:00:17.119 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:00:17.158 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:00:17.161 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:00:17.162 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:00:17.262 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:00:17.275 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:00:16 smithi164 bash[59462]: cluster 1685822416.4237752 mgr.y (mgr.34104) 719 : cluster 0 pgmap v627: 89 pgs: 2 creating+activating, 4 creating+peering, 83 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-06-03T20:00:17.278 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:00:16 smithi089 bash[78452]: cluster 1685822416.4237752 mgr.y (mgr.34104) 719 : cluster 0 pgmap v627: 89 pgs: 2 creating+activating, 4 creating+peering, 83 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-06-03T20:00:17.279 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:00:16 smithi089 bash[75068]: cluster 1685822416.4237752 mgr.y (mgr.34104) 719 : cluster 0 pgmap v627: 89 pgs: 2 creating+activating, 4 creating+peering, 83 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-06-03T20:00:17.300 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:00:17.303 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:00:17.303 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:00:17.328 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-06-03T20:00:17.405 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:00:17.439 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:00:17.442 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:00:17.442 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:00:17.543 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:00:17.580 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:00:17.583 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:00:17.583 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:00:17.684 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:00:17.725 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:00:17.727 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:00:17.728 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:00:17.824 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/mon.a/config 2023-06-03T20:00:17.829 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:00:17.864 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:00:17.867 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:00:17.867 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:00:17.968 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:00:18.002 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:00:18.006 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:00:18.006 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:00:18.107 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:00:18.141 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:00:18.142 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:00:18.143 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:00:18.244 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:00:18.278 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:00:18.280 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:00:18.281 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:00:18.382 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:00:18.420 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:00:18.423 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:00:18.424 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:00:18.525 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:00:18.565 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:00:18.568 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:00:18.568 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:00:18.669 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:00:18.700 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:00:18.702 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:00:18.703 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:00:18.804 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:00:18.838 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:00:18.841 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:00:18.842 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:00:18.943 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:00:18.975 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:00:18.977 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:00:18.978 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:00:19.079 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:00:19.115 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:00:19.117 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:00:19.118 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:00:19.219 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:00:19.255 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:00:19.257 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:00:19.258 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:00:19.359 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:00:19.391 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:00:19.395 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:00:19.395 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:00:19.497 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:00:19.534 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:00:19.537 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:00:19.538 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:00:19.639 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:00:19.676 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:00:19.678 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:00:19.679 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:00:19.779 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:00:19 smithi089 bash[75068]: cluster 1685822418.4244757 mgr.y (mgr.34104) 720 : cluster 0 pgmap v628: 89 pgs: 89 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-06-03T20:00:19.779 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:00:19 smithi089 bash[78452]: cluster 1685822418.4244757 mgr.y (mgr.34104) 720 : cluster 0 pgmap v628: 89 pgs: 89 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-06-03T20:00:19.780 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:00:19.791 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:00:19 smithi164 bash[59462]: cluster 1685822418.4244757 mgr.y (mgr.34104) 720 : cluster 0 pgmap v628: 89 pgs: 89 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-06-03T20:00:19.813 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:00:19.816 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:00:19.816 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:00:19.873 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/mon.a/config 2023-06-03T20:00:19.874 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/mon.a/config 2023-06-03T20:00:19.876 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/mon.a/config 2023-06-03T20:00:19.917 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:00:19.958 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:00:19.960 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:00:19.961 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:00:20.062 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:00:20.091 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:00:20.093 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:00:20.094 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:00:20.195 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:00:20.227 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:00:20.229 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:00:20.230 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:00:20.331 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:00:20.368 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:00:20.371 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:00:20.372 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:00:20.473 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:00:20.515 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:00:20.517 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:00:20.518 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:00:20.619 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:00:20.648 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:00:20.650 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:00:20.651 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:00:20.752 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:00:20.788 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:00:20.791 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:00:20.792 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:00:20.893 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:00:20.929 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:00:20.931 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:00:20.931 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:00:21.032 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:00:21.063 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:00:21.065 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:00:21.066 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:00:21.167 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:00:21.202 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:00:21.205 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:00:21.206 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:00:21.307 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:00:21.337 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:00:21.339 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:00:21.340 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:00:21.441 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:00:21.473 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:00:21.475 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:00:21.476 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:00:21.578 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:00:21.618 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:00:21.621 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:00:21.622 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:00:21.723 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:00:21.731 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:00:21 smithi089 bash[78452]: cluster 1685822420.4251137 mgr.y (mgr.34104) 721 : cluster 0 pgmap v629: 89 pgs: 89 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-06-03T20:00:21.732 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:00:21 smithi089 bash[75068]: cluster 1685822420.4251137 mgr.y (mgr.34104) 721 : cluster 0 pgmap v629: 89 pgs: 89 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-06-03T20:00:21.756 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:00:21.759 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:00:21.760 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:00:21.834 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:00:21 smithi164 bash[59462]: cluster 1685822420.4251137 mgr.y (mgr.34104) 721 : cluster 0 pgmap v629: 89 pgs: 89 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-06-03T20:00:21.860 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:00:21.889 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:00:21.893 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:00:21.894 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:00:21.911 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/mon.a/config 2023-06-03T20:00:21.995 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:00:22.029 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:00:22.032 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:00:22.033 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:00:22.134 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:00:22.167 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:00:22.170 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:00:22.170 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:00:22.271 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:00:22.305 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:00:22.309 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:00:22.310 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:00:22.411 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:00:22.447 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:00:22.451 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:00:22.452 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:00:22.553 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:00:22.588 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:00:22.590 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:00:22.591 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:00:22.692 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:00:22.724 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:00:22.727 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:00:22.728 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:00:22.829 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:00:22.869 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:00:22.871 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:00:22.872 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:00:22.973 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:00:23.002 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:00:23.005 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:00:23.005 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:00:23.106 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:00:23.139 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:00:23.143 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:00:23.143 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:00:23.244 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:00:23.257 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:00:23 smithi089 bash[75068]: audit 1685822422.1021502 mon.a (mon.0) 662 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:00:23.257 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:00:23 smithi089 bash[75068]: audit 1685822422.1036487 mon.c (mon.2) 380 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T20:00:23.257 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:00:23 smithi089 bash[75068]: cluster 1685822422.4256935 mgr.y (mgr.34104) 722 : cluster 0 pgmap v630: 89 pgs: 89 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-06-03T20:00:23.258 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:00:23 smithi089 bash[75068]: audit 1685822422.5000913 mon.a (mon.0) 663 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:00:23.258 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:00:23 smithi089 bash[78452]: audit 1685822422.1021502 mon.a (mon.0) 662 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:00:23.258 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:00:23 smithi089 bash[78452]: audit 1685822422.1036487 mon.c (mon.2) 380 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T20:00:23.258 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:00:23 smithi089 bash[78452]: cluster 1685822422.4256935 mgr.y (mgr.34104) 722 : cluster 0 pgmap v630: 89 pgs: 89 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-06-03T20:00:23.258 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:00:23 smithi089 bash[78452]: audit 1685822422.5000913 mon.a (mon.0) 663 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:00:23.277 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:00:23.280 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:00:23.281 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:00:23.382 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:00:23.411 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:00:23.413 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:00:23.414 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:00:23.515 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:00:23.526 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:00:23 smithi164 bash[59462]: audit 1685822422.1021502 mon.a (mon.0) 662 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:00:23.527 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:00:23 smithi164 bash[59462]: audit 1685822422.1036487 mon.c (mon.2) 380 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T20:00:23.527 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:00:23 smithi164 bash[59462]: cluster 1685822422.4256935 mgr.y (mgr.34104) 722 : cluster 0 pgmap v630: 89 pgs: 89 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-06-03T20:00:23.527 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:00:23 smithi164 bash[59462]: audit 1685822422.5000913 mon.a (mon.0) 663 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:00:23.551 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:00:23.554 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:00:23.555 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:00:23.655 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:00:23.683 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:00:23.685 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:00:23.686 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:00:23.787 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:00:23.823 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:00:23.826 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:00:23.827 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:00:23.928 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:00:23.966 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:00:23.968 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:00:23.969 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:00:24.070 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:00:24.100 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:00:24.103 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:00:24.104 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:00:24.205 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:00:24.238 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:00:24.240 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:00:24.241 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:00:24.342 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:00:24.368 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:00:24.370 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:00:24.371 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:00:24.472 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:00:24.504 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:00:24.506 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:00:24.507 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:00:24.608 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:00:24.641 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:00:24.647 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:00:24.647 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:00:24.736 INFO:teuthology.orchestra.run.smithi089.stdout:ERROR: (22) Invalid argument 2023-06-03T20:00:24.736 INFO:teuthology.orchestra.run.smithi089.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-06-03T20:00:24.749 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:00:24.785 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:00:24.787 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:00:24.788 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:00:24.888 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:00:24.924 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:00:24.926 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:00:24.927 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:00:25.028 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:00:25.059 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:00:25.061 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:00:25.061 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:00:25.163 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:00:25.199 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:00:25.202 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:00:25.203 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:00:25.303 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:00:25.334 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:00:25.335 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:00:25.336 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:00:25.437 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:00:25.470 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:00:25.472 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:00:25.473 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:00:25.574 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:00:25.582 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:00:25 smithi089 bash[78452]: cluster 1685822424.4266553 mgr.y (mgr.34104) 723 : cluster 0 pgmap v631: 89 pgs: 89 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-06-03T20:00:25.582 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:00:25 smithi089 bash[75068]: cluster 1685822424.4266553 mgr.y (mgr.34104) 723 : cluster 0 pgmap v631: 89 pgs: 89 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-06-03T20:00:25.603 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:00:25.605 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:00:25.606 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:00:25.706 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:00:25.745 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:00:25.748 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:00:25 smithi164 bash[59462]: cluster 1685822424.4266553 mgr.y (mgr.34104) 723 : cluster 0 pgmap v631: 89 pgs: 89 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-06-03T20:00:25.748 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:00:25.749 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:00:25.850 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:00:25.884 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:00:25.887 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:00:25.888 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:00:25.989 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:00:26.017 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:00:26.018 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:00:26.019 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:00:26.120 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:00:26.145 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:00:26.147 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:00:26.147 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:00:26.248 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:00:26.285 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:00:26.288 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:00:26.289 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:00:26.390 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:00:26.399 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 20:00:26 smithi089 bash[71163]: ::ffff:172.21.15.164 - - [03/Jun/2023:20:00:26] "GET /metrics HTTP/1.1" 200 227688 "" "Prometheus/2.33.4" 2023-06-03T20:00:26.416 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:00:26.419 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:00:26.420 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:00:26.521 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:00:26.546 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:00:26.549 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:00:26.549 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:00:26.650 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:00:26.682 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:00:26.685 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:00:26.686 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:00:26.787 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:00:26.818 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:00:26.819 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:00:26.820 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:00:26.921 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:00:26.946 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:00:26.948 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:00:26.949 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:00:27.049 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:00:27.087 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:00:27.089 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:00:27.090 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:00:27.191 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:00:27.229 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:00:27.232 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:00:27.233 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:00:27.334 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:00:27.368 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:00:27.371 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:00:27.372 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:00:27.473 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:00:27.505 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:00:27.508 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:00:27.508 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:00:27.609 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:00:27.623 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:00:27 smithi164 bash[59462]: cluster 1685822426.4273412 mgr.y (mgr.34104) 724 : cluster 0 pgmap v632: 89 pgs: 89 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-06-03T20:00:27.646 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:00:27.649 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:00:27.650 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:00:27.751 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:00:27.764 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:00:27 smithi089 bash[75068]: cluster 1685822426.4273412 mgr.y (mgr.34104) 724 : cluster 0 pgmap v632: 89 pgs: 89 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-06-03T20:00:27.765 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:00:27 smithi089 bash[78452]: cluster 1685822426.4273412 mgr.y (mgr.34104) 724 : cluster 0 pgmap v632: 89 pgs: 89 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-06-03T20:00:27.782 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:00:27.785 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:00:27.785 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:00:27.886 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:00:27.914 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:00:27.916 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:00:27.917 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:00:28.018 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:00:28.044 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:00:28.047 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:00:28.049 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:00:28.150 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:00:28.175 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:00:28.177 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:00:28.178 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:00:28.279 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:00:28.319 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:00:28.322 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:00:28.323 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:00:28.424 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:00:28.460 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:00:28.463 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:00:28.464 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:00:28.565 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:00:28.596 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:00:28.600 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:00:28.600 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:00:28.700 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph daemon osd.0 dump_historic_ops 2023-06-03T20:00:28.701 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:00:28.739 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:00:28.741 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:00:28.742 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:00:28.843 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:00:28.875 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:00:28.877 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:00:28.878 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:00:28.979 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:00:29.014 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:00:29.016 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:00:29.016 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:00:29.117 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:00:29.142 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:00:29.144 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:00:29.145 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:00:29.246 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:00:29.272 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:00:29.274 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:00:29.275 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:00:29.376 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:00:29.412 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:00:29.414 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:00:29.415 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:00:29.516 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:00:29.553 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:00:29.555 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:00:29.556 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:00:29.657 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:00:29.671 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:00:29 smithi164 bash[59462]: cluster 1685822428.4281714 mgr.y (mgr.34104) 725 : cluster 0 pgmap v633: 89 pgs: 89 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-06-03T20:00:29.689 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:00:29.692 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:00:29.693 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:00:29.794 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:00:29.835 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:00:29.837 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:00:29.838 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:00:29.938 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:00:29.976 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:00:29.980 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:00:29.980 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:00:30.028 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:00:29 smithi089 bash[78452]: cluster 1685822428.4281714 mgr.y (mgr.34104) 725 : cluster 0 pgmap v633: 89 pgs: 89 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-06-03T20:00:30.029 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:00:29 smithi089 bash[75068]: cluster 1685822428.4281714 mgr.y (mgr.34104) 725 : cluster 0 pgmap v633: 89 pgs: 89 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-06-03T20:00:30.081 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:00:30.105 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:00:30.107 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:00:30.108 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:00:30.209 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:00:30.246 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:00:30.249 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:00:30.250 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:00:30.351 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:00:30.382 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:00:30.385 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:00:30.386 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:00:30.486 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:00:30.520 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:00:30.523 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:00:30.524 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:00:30.624 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:00:30.656 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:00:30.661 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:00:30.662 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:00:30.763 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:00:30.789 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:00:30.792 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:00:30.792 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:00:30.893 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:00:30.924 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:00:30.927 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:00:30.928 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:00:31.029 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:00:31.069 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:00:31.071 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:00:31.072 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:00:31.173 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:00:31.204 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:00:31.206 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:00:31.207 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:00:31.308 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:00:31.351 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:00:31.354 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:00:31.355 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:00:31.456 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:00:31.487 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:00:31.489 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:00:31.490 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:00:31.584 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:00:31 smithi164 bash[59462]: cluster 1685822430.428907 mgr.y (mgr.34104) 726 : cluster 0 pgmap v634: 89 pgs: 89 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-06-03T20:00:31.591 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:00:31.617 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:00:31.621 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:00:31.622 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:00:31.723 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:00:31.736 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:00:31 smithi089 bash[75068]: cluster 1685822430.428907 mgr.y (mgr.34104) 726 : cluster 0 pgmap v634: 89 pgs: 89 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-06-03T20:00:31.737 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:00:31 smithi089 bash[78452]: cluster 1685822430.428907 mgr.y (mgr.34104) 726 : cluster 0 pgmap v634: 89 pgs: 89 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-06-03T20:00:31.761 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:00:31.764 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:00:31.765 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:00:31.865 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:00:31.904 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:00:31.906 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:00:31.906 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:00:32.007 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:00:32.036 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:00:32.038 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:00:32.039 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:00:32.140 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:00:32.171 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:00:32.173 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:00:32.174 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:00:32.275 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:00:32.313 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:00:32.316 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:00:32.317 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:00:32.418 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:00:32.462 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:00:32.465 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:00:32.466 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:00:32.567 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:00:32.602 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:00:32.604 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:00:32.604 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:00:32.705 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:00:32.736 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:00:32.738 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:00:32.739 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:00:32.840 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:00:32.879 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:00:32.881 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:00:32.882 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:00:32.983 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:00:33.023 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:00:33.026 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:00:33.026 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:00:33.127 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:00:33.155 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:00:33.156 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:00:33.156 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:00:33.257 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:00:33.299 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:00:33.302 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:00:33.302 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:00:33.403 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:00:33.427 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:00:33.429 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:00:33.430 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:00:33.521 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/mon.a/config 2023-06-03T20:00:33.531 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:00:33.550 INFO:teuthology.orchestra.run.smithi089.stderr:reweighted osd.1 to 1 (10000) 2023-06-03T20:00:33.565 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:00:33.569 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:00:33.570 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:00:33.670 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:00:33.712 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:00:33.715 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:00:33.715 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:00:33.779 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:00:33 smithi089 bash[78452]: cluster 1685822432.4295352 mgr.y (mgr.34104) 727 : cluster 0 pgmap v635: 89 pgs: 89 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-06-03T20:00:33.779 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:00:33 smithi089 bash[78452]: audit 1685822433.3710427 mon.a (mon.0) 664 : audit 1 from='client.? 172.21.15.89:0/346309054' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-06-03T20:00:33.779 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:00:33 smithi089 bash[75068]: cluster 1685822432.4295352 mgr.y (mgr.34104) 727 : cluster 0 pgmap v635: 89 pgs: 89 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-06-03T20:00:33.779 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:00:33 smithi089 bash[75068]: audit 1685822433.3710427 mon.a (mon.0) 664 : audit 1 from='client.? 172.21.15.89:0/346309054' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-06-03T20:00:33.816 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:00:33.834 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:00:33 smithi164 bash[59462]: cluster 1685822432.4295352 mgr.y (mgr.34104) 727 : cluster 0 pgmap v635: 89 pgs: 89 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-06-03T20:00:33.834 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:00:33 smithi164 bash[59462]: audit 1685822433.3710427 mon.a (mon.0) 664 : audit 1 from='client.? 172.21.15.89:0/346309054' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-06-03T20:00:33.841 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:00:33.844 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:00:33.844 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:00:33.945 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:00:33.978 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:00:33.981 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:00:33.981 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:00:34.082 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:00:34.125 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:00:34.128 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:00:34.129 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:00:34.229 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:00:34.263 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:00:34.266 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:00:34.266 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:00:34.367 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:00:34.407 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:00:34.410 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:00:34.411 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:00:34.512 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:00:34.550 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:00:34.554 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:00:34.554 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:00:34.655 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:00:34.669 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:00:34 smithi164 bash[59462]: cluster 1685822433.5345871 osd.6 (osd.6) 49 : cluster 3 failed to encode map e130 with expected crc 2023-06-03T20:00:34.669 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:00:34 smithi164 bash[59462]: audit 1685822433.538944 mon.a (mon.0) 665 : audit 1 from='client.? 172.21.15.89:0/346309054' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2023-06-03T20:00:34.669 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:00:34 smithi164 bash[59462]: cluster 1685822433.5389988 mon.a (mon.0) 666 : cluster 0 osdmap e130: 8 total, 8 up, 8 in 2023-06-03T20:00:34.698 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:00:34.701 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:00:34.702 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:00:34.802 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:00:34.842 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:00:34.845 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:00:34.846 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:00:34.946 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:00:34.982 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:00:34.985 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:00:34.985 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:00:35.029 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:00:34 smithi089 bash[75068]: cluster 1685822433.5345871 osd.6 (osd.6) 49 : cluster 3 failed to encode map e130 with expected crc 2023-06-03T20:00:35.030 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:00:34 smithi089 bash[75068]: audit 1685822433.538944 mon.a (mon.0) 665 : audit 1 from='client.? 172.21.15.89:0/346309054' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2023-06-03T20:00:35.030 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:00:34 smithi089 bash[75068]: cluster 1685822433.5389988 mon.a (mon.0) 666 : cluster 0 osdmap e130: 8 total, 8 up, 8 in 2023-06-03T20:00:35.030 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:00:34 smithi089 bash[78452]: cluster 1685822433.5345871 osd.6 (osd.6) 49 : cluster 3 failed to encode map e130 with expected crc 2023-06-03T20:00:35.030 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:00:34 smithi089 bash[78452]: audit 1685822433.538944 mon.a (mon.0) 665 : audit 1 from='client.? 172.21.15.89:0/346309054' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2023-06-03T20:00:35.030 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:00:34 smithi089 bash[78452]: cluster 1685822433.5389988 mon.a (mon.0) 666 : cluster 0 osdmap e130: 8 total, 8 up, 8 in 2023-06-03T20:00:35.086 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:00:35.118 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:00:35.121 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:00:35.121 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:00:35.222 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:00:35.267 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:00:35.270 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:00:35.270 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:00:35.371 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:00:35.402 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:00:35.404 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:00:35.405 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:00:35.505 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:00:35.534 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:00:35.537 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:00:35.537 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:00:35.638 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:00:35.640 INFO:teuthology.orchestra.run.smithi089.stdout:osd.0: {} 2023-06-03T20:00:35.658 INFO:teuthology.orchestra.run.smithi089.stdout:osd.1: {} 2023-06-03T20:00:35.675 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:00:35.678 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:00:35.679 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:00:35.679 INFO:teuthology.orchestra.run.smithi089.stdout:osd.2: {} 2023-06-03T20:00:35.702 INFO:teuthology.orchestra.run.smithi089.stdout:osd.3: {} 2023-06-03T20:00:35.726 INFO:teuthology.orchestra.run.smithi089.stdout:osd.4: {} 2023-06-03T20:00:35.747 INFO:teuthology.orchestra.run.smithi089.stdout:osd.5: {} 2023-06-03T20:00:35.770 INFO:teuthology.orchestra.run.smithi089.stdout:osd.6: {} 2023-06-03T20:00:35.778 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:00:35 smithi089 bash[75068]: cluster 1685822434.430173 mgr.y (mgr.34104) 728 : cluster 0 pgmap v637: 89 pgs: 89 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-06-03T20:00:35.779 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:00:35 smithi089 bash[78452]: cluster 1685822434.430173 mgr.y (mgr.34104) 728 : cluster 0 pgmap v637: 89 pgs: 89 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-06-03T20:00:35.780 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:00:35.797 INFO:teuthology.orchestra.run.smithi089.stdout:osd.7: {} 2023-06-03T20:00:35.798 INFO:teuthology.orchestra.run.smithi089.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-06-03T20:00:35.798 INFO:teuthology.orchestra.run.smithi089.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-06-03T20:00:35.799 INFO:teuthology.orchestra.run.smithi089.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-06-03T20:00:35.799 INFO:teuthology.orchestra.run.smithi089.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-06-03T20:00:35.799 INFO:teuthology.orchestra.run.smithi089.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-06-03T20:00:35.799 INFO:teuthology.orchestra.run.smithi089.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-06-03T20:00:35.799 INFO:teuthology.orchestra.run.smithi089.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-06-03T20:00:35.799 INFO:teuthology.orchestra.run.smithi089.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-06-03T20:00:35.814 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:00:35.816 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:00:35 smithi164 bash[59462]: cluster 1685822434.430173 mgr.y (mgr.34104) 728 : cluster 0 pgmap v637: 89 pgs: 89 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-06-03T20:00:35.817 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:00:35.818 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:00:35.919 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:00:35.963 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:00:35.966 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:00:35.968 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:00:36.069 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:00:36.104 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:00:36.107 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:00:36.108 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:00:36.209 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:00:36.218 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph osd reweight 2 1 2023-06-03T20:00:36.247 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:00:36.250 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:00:36.250 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:00:36.352 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:00:36.359 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 20:00:36 smithi089 bash[71163]: ::ffff:172.21.15.164 - - [03/Jun/2023:20:00:36] "GET /metrics HTTP/1.1" 200 227688 "" "Prometheus/2.33.4" 2023-06-03T20:00:36.375 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:00:36.377 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:00:36.378 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:00:36.479 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:00:36.566 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:00:36.569 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:00:36.570 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:00:36.665 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:00:36 smithi089 bash[75068]: audit 1685822435.6142404 mon.c (mon.2) 381 : audit 0 from='client.? 172.21.15.89:0/3818824791' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-03T20:00:36.665 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:00:36 smithi089 bash[78452]: audit 1685822435.6142404 mon.c (mon.2) 381 : audit 0 from='client.? 172.21.15.89:0/3818824791' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-03T20:00:36.671 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:00:36.710 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:00:36.714 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:00:36.715 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:00:36.816 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:00:36.836 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:00:36 smithi164 bash[59462]: audit 1685822435.6142404 mon.c (mon.2) 381 : audit 0 from='client.? 172.21.15.89:0/3818824791' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-03T20:00:36.843 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:00:36.845 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:00:36.845 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:00:36.946 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:00:36.971 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:00:36.973 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:00:36.974 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:00:37.075 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:00:37.113 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:00:37.116 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:00:37.116 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:00:37.217 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:00:37.239 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:00:37.241 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:00:37.242 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:00:37.343 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:00:37.385 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:00:37.388 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:00:37.388 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:00:37.489 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:00:37.523 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:00:37.528 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:00:37.529 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:00:37.576 INFO:teuthology.orchestra.run.smithi089.stderr:nodeep-scrub is set 2023-06-03T20:00:37.578 INFO:teuthology.orchestra.run.smithi089.stderr:enabled application 'rados' on pool 'unique_pool_4' 2023-06-03T20:00:37.630 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:00:37.661 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:00:37.664 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:00:37.665 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:00:37.766 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:00:37.790 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:00:37.792 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:00:37.793 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:00:37.834 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:00:37 smithi164 bash[59462]: cluster 1685822436.4306557 mgr.y (mgr.34104) 729 : cluster 0 pgmap v638: 89 pgs: 89 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-06-03T20:00:37.834 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:00:37 smithi164 bash[59462]: audit 1685822437.0936 mon.c (mon.2) 382 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T20:00:37.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:00:37 smithi164 bash[59462]: audit 1685822437.0969803 mon.a (mon.0) 667 : audit 1 from='client.? 172.21.15.89:0/3177193558' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-06-03T20:00:37.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:00:37 smithi164 bash[59462]: audit 1685822437.2561398 mon.a (mon.0) 668 : audit 1 from='client.? 172.21.15.89:0/3912554206' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_4", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-06-03T20:00:37.894 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:00:37.929 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:00:37 smithi089 bash[78452]: cluster 1685822436.4306557 mgr.y (mgr.34104) 729 : cluster 0 pgmap v638: 89 pgs: 89 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-06-03T20:00:37.930 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:00:37 smithi089 bash[78452]: audit 1685822437.0936 mon.c (mon.2) 382 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T20:00:37.930 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:00:37 smithi089 bash[78452]: audit 1685822437.0969803 mon.a (mon.0) 667 : audit 1 from='client.? 172.21.15.89:0/3177193558' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-06-03T20:00:37.930 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:00:37 smithi089 bash[78452]: audit 1685822437.2561398 mon.a (mon.0) 668 : audit 1 from='client.? 172.21.15.89:0/3912554206' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_4", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-06-03T20:00:37.931 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:00:37 smithi089 bash[75068]: cluster 1685822436.4306557 mgr.y (mgr.34104) 729 : cluster 0 pgmap v638: 89 pgs: 89 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-06-03T20:00:37.931 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:00:37 smithi089 bash[75068]: audit 1685822437.0936 mon.c (mon.2) 382 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T20:00:37.931 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:00:37 smithi089 bash[75068]: audit 1685822437.0969803 mon.a (mon.0) 667 : audit 1 from='client.? 172.21.15.89:0/3177193558' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-06-03T20:00:37.931 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:00:37 smithi089 bash[75068]: audit 1685822437.2561398 mon.a (mon.0) 668 : audit 1 from='client.? 172.21.15.89:0/3912554206' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_4", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-06-03T20:00:37.934 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:00:37.937 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:00:37.938 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:00:38.039 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:00:38.064 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:00:38.066 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:00:38.066 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:00:38.167 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:00:38.202 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:00:38.204 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:00:38.204 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:00:38.305 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:00:38.330 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:00:38.331 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:00:38.332 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:00:38.433 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:00:38.467 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:00:38.471 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:00:38.471 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:00:38.572 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:00:38.609 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:00:38.612 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:00:38.613 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:00:38.713 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:00:38.726 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:00:38 smithi164 bash[59462]: cluster 1685822437.5611467 mon.a (mon.0) 669 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-06-03T20:00:38.726 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:00:38 smithi164 bash[59462]: cluster 1685822437.5622423 osd.6 (osd.6) 50 : cluster 3 failed to encode map e131 with expected crc 2023-06-03T20:00:38.726 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:00:38 smithi164 bash[59462]: audit 1685822437.5662987 mon.a (mon.0) 670 : audit 1 from='client.? 172.21.15.89:0/3177193558' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-06-03T20:00:38.726 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:00:38 smithi164 bash[59462]: audit 1685822437.566383 mon.a (mon.0) 671 : audit 1 from='client.? 172.21.15.89:0/3912554206' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_4", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-06-03T20:00:38.726 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:00:38 smithi164 bash[59462]: cluster 1685822437.5664446 mon.a (mon.0) 672 : cluster 0 osdmap e131: 8 total, 8 up, 8 in 2023-06-03T20:00:38.746 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:00:38.748 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:00:38.749 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:00:38.850 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:00:38.890 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:00:38.893 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:00:38.894 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:00:38.995 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:00:39.029 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:00:38 smithi089 bash[75068]: cluster 1685822437.5611467 mon.a (mon.0) 669 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-06-03T20:00:39.029 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:00:38 smithi089 bash[75068]: cluster 1685822437.5622423 osd.6 (osd.6) 50 : cluster 3 failed to encode map e131 with expected crc 2023-06-03T20:00:39.029 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:00:38 smithi089 bash[75068]: audit 1685822437.5662987 mon.a (mon.0) 670 : audit 1 from='client.? 172.21.15.89:0/3177193558' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-06-03T20:00:39.029 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:00:38 smithi089 bash[75068]: audit 1685822437.566383 mon.a (mon.0) 671 : audit 1 from='client.? 172.21.15.89:0/3912554206' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_4", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-06-03T20:00:39.029 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:00:38 smithi089 bash[75068]: cluster 1685822437.5664446 mon.a (mon.0) 672 : cluster 0 osdmap e131: 8 total, 8 up, 8 in 2023-06-03T20:00:39.030 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:00:38 smithi089 bash[78452]: cluster 1685822437.5611467 mon.a (mon.0) 669 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-06-03T20:00:39.030 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:00:38 smithi089 bash[78452]: cluster 1685822437.5622423 osd.6 (osd.6) 50 : cluster 3 failed to encode map e131 with expected crc 2023-06-03T20:00:39.030 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:00:38 smithi089 bash[78452]: audit 1685822437.5662987 mon.a (mon.0) 670 : audit 1 from='client.? 172.21.15.89:0/3177193558' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-06-03T20:00:39.030 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:00:38 smithi089 bash[78452]: audit 1685822437.566383 mon.a (mon.0) 671 : audit 1 from='client.? 172.21.15.89:0/3912554206' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_4", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-06-03T20:00:39.030 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:00:38 smithi089 bash[78452]: cluster 1685822437.5664446 mon.a (mon.0) 672 : cluster 0 osdmap e131: 8 total, 8 up, 8 in 2023-06-03T20:00:39.034 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:00:39.036 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:00:39.037 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:00:39.138 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:00:39.179 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:00:39.183 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:00:39.184 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:00:39.286 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:00:39.317 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:00:39.321 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:00:39.322 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:00:39.423 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:00:39.452 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:00:39.454 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:00:39.455 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:00:39.556 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:00:39.580 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:00:39.583 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:00:39.584 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:00:39.684 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:00:39.719 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:00:39.723 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:00:39.724 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:00:39.825 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:00:39.839 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:00:39 smithi164 bash[59462]: cluster 1685822438.2417674 osd.7 (osd.7) 43 : cluster 3 failed to encode map e131 with expected crc 2023-06-03T20:00:39.839 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:00:39 smithi164 bash[59462]: cluster 1685822438.2445693 osd.7 (osd.7) 44 : cluster 3 failed to encode map e131 with expected crc 2023-06-03T20:00:39.840 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:00:39 smithi164 bash[59462]: cluster 1685822438.431265 mgr.y (mgr.34104) 730 : cluster 0 pgmap v640: 89 pgs: 89 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-06-03T20:00:39.840 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:00:39 smithi164 bash[59462]: cluster 1685822439.176425 osd.5 (osd.5) 54 : cluster 3 failed to encode map e131 with expected crc 2023-06-03T20:00:39.862 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:00:39.865 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:00:39.866 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:00:39.967 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:00:39.978 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:00:39 smithi089 bash[78452]: cluster 1685822438.2417674 osd.7 (osd.7) 43 : cluster 3 failed to encode map e131 with expected crc 2023-06-03T20:00:39.979 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:00:39 smithi089 bash[78452]: cluster 1685822438.2445693 osd.7 (osd.7) 44 : cluster 3 failed to encode map e131 with expected crc 2023-06-03T20:00:39.979 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:00:39 smithi089 bash[78452]: cluster 1685822438.431265 mgr.y (mgr.34104) 730 : cluster 0 pgmap v640: 89 pgs: 89 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-06-03T20:00:39.979 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:00:39 smithi089 bash[78452]: cluster 1685822439.176425 osd.5 (osd.5) 54 : cluster 3 failed to encode map e131 with expected crc 2023-06-03T20:00:39.979 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:00:39 smithi089 bash[75068]: cluster 1685822438.2417674 osd.7 (osd.7) 43 : cluster 3 failed to encode map e131 with expected crc 2023-06-03T20:00:39.979 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:00:39 smithi089 bash[75068]: cluster 1685822438.2445693 osd.7 (osd.7) 44 : cluster 3 failed to encode map e131 with expected crc 2023-06-03T20:00:39.980 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:00:39 smithi089 bash[75068]: cluster 1685822438.431265 mgr.y (mgr.34104) 730 : cluster 0 pgmap v640: 89 pgs: 89 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-06-03T20:00:39.980 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:00:39 smithi089 bash[75068]: cluster 1685822439.176425 osd.5 (osd.5) 54 : cluster 3 failed to encode map e131 with expected crc 2023-06-03T20:00:39.980 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-06-03T20:00:40.003 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:00:40.006 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:00:40.007 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:00:40.107 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:00:40.145 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:00:40.148 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:00:40.148 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:00:40.249 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:00:40.288 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:00:40.292 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:00:40.293 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:00:40.394 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:00:40.423 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:00:40.427 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:00:40.427 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:00:40.528 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:00:40.565 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:00:40.567 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:00:40.568 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:00:40.669 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:00:40.703 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:00:40.706 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:00:40.707 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:00:40.807 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:00:40.843 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:00:40.845 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:00:40.846 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:00:40.947 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:00:40.987 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:00:40.989 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:00:40.990 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:00:41.092 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:00:41.123 DEBUG:teuthology.orchestra.run.smithi089:> /bin/sh -c 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage rados --no-log-to-stderr --name client.0 -b 65536 --object-size 65536 -p unique_pool_4 bench 90 write' 2023-06-03T20:00:41.134 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:00:41.136 INFO:tasks.radosbench:joining radosbench (timing out after 3000s) 2023-06-03T20:00:41.136 INFO:teuthology.orchestra.run:waiting for 3000 2023-06-03T20:00:41.137 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:00:41.138 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:00:41.175 INFO:tasks.radosbench.radosbench.0.smithi089.stdout:hints = 1 2023-06-03T20:00:41.175 INFO:tasks.radosbench.radosbench.0.smithi089.stdout:Maintaining 16 concurrent writes of 65536 bytes to objects of size 65536 for up to 90 seconds or 0 objects 2023-06-03T20:00:41.176 INFO:tasks.radosbench.radosbench.0.smithi089.stdout:Object prefix: benchmark_data_smithi089_115580 2023-06-03T20:00:41.176 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-06-03T20:00:41.176 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 0 0 0 0 0 0 - 0 2023-06-03T20:00:41.239 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:00:41.277 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:00:41.280 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:00:41.280 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:00:41.381 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:00:41.414 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:00:41.417 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:00:41.417 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:00:41.518 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:00:41.557 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:00:41.559 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:00:41.560 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:00:41.661 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:00:41.704 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:00:41.707 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:00:41.708 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:00:41.779 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:00:41 smithi089 bash[75068]: cluster 1685822440.431737 mgr.y (mgr.34104) 731 : cluster 0 pgmap v641: 89 pgs: 89 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-06-03T20:00:41.779 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:00:41 smithi089 bash[78452]: cluster 1685822440.431737 mgr.y (mgr.34104) 731 : cluster 0 pgmap v641: 89 pgs: 89 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-06-03T20:00:41.809 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:00:41.839 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:00:41.841 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:00:41.841 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:00:41.842 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:00:41 smithi164 bash[59462]: cluster 1685822440.431737 mgr.y (mgr.34104) 731 : cluster 0 pgmap v641: 89 pgs: 89 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2023-06-03T20:00:41.912 INFO:teuthology.orchestra.run.smithi089.stdout:{ 2023-06-03T20:00:41.913 INFO:teuthology.orchestra.run.smithi089.stdout: "size": 20, 2023-06-03T20:00:41.913 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 600, 2023-06-03T20:00:41.913 INFO:teuthology.orchestra.run.smithi089.stdout: "ops": [ 2023-06-03T20:00:41.913 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:00:41.914 INFO:teuthology.orchestra.run.smithi089.stdout: "description": "osd_op(client.44762.0:37593 8.1 8:8f8199ed:::benchmark_data_smithi089_91006_object37592:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e100)", 2023-06-03T20:00:41.914 INFO:teuthology.orchestra.run.smithi089.stdout: "initiated_at": "2023-06-03T19:51:33.549968+0000", 2023-06-03T20:00:41.914 INFO:teuthology.orchestra.run.smithi089.stdout: "age": 548.35310098499997, 2023-06-03T20:00:41.914 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 1.0078020860000001, 2023-06-03T20:00:41.914 INFO:teuthology.orchestra.run.smithi089.stdout: "type_data": { 2023-06-03T20:00:41.915 INFO:teuthology.orchestra.run.smithi089.stdout: "flag_point": "reached pg", 2023-06-03T20:00:41.915 INFO:teuthology.orchestra.run.smithi089.stdout: "client_info": { 2023-06-03T20:00:41.915 INFO:teuthology.orchestra.run.smithi089.stdout: "client": "client.44762", 2023-06-03T20:00:41.915 INFO:teuthology.orchestra.run.smithi089.stdout: "client_addr": "172.21.15.89:0/4205857995", 2023-06-03T20:00:41.915 INFO:teuthology.orchestra.run.smithi089.stdout: "tid": 37593 2023-06-03T20:00:41.915 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:00:41.915 INFO:teuthology.orchestra.run.smithi089.stdout: "events": [ 2023-06-03T20:00:41.916 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:00:41.916 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "initiated", 2023-06-03T20:00:41.916 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T19:51:33.549968+0000", 2023-06-03T20:00:41.916 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0 2023-06-03T20:00:41.916 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:00:41.916 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:00:41.916 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "throttled", 2023-06-03T20:00:41.917 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T19:51:33.549968+0000", 2023-06-03T20:00:41.917 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0 2023-06-03T20:00:41.917 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:00:41.917 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:00:41.917 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "header_read", 2023-06-03T20:00:41.918 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T19:51:33.549966+0000", 2023-06-03T20:00:41.918 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 4294967295.9999981 2023-06-03T20:00:41.918 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:00:41.918 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:00:41.918 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "all_read", 2023-06-03T20:00:41.919 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T19:51:33.549997+0000", 2023-06-03T20:00:41.919 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 3.0584999999999999e-05 2023-06-03T20:00:41.919 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:00:41.919 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:00:41.919 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "dispatched", 2023-06-03T20:00:41.919 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T19:51:33.550015+0000", 2023-06-03T20:00:41.920 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 1.859e-05 2023-06-03T20:00:41.920 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:00:41.920 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:00:41.920 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "queued_for_pg", 2023-06-03T20:00:41.921 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T19:51:33.550028+0000", 2023-06-03T20:00:41.921 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 1.253e-05 2023-06-03T20:00:41.921 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:00:41.921 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:00:41.921 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "reached_pg", 2023-06-03T20:00:41.922 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T19:51:33.550100+0000", 2023-06-03T20:00:41.922 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 7.2296e-05 2023-06-03T20:00:41.922 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:00:41.922 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:00:41.922 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "waiting for peered", 2023-06-03T20:00:41.923 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T19:51:33.550113+0000", 2023-06-03T20:00:41.923 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 1.2369e-05 2023-06-03T20:00:41.923 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:00:41.923 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:00:41.924 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "reached_pg", 2023-06-03T20:00:41.924 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T19:51:34.556403+0000", 2023-06-03T20:00:41.924 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 1.006290125 2023-06-03T20:00:41.924 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:00:41.924 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:00:41.924 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "done", 2023-06-03T20:00:41.925 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T19:51:34.557770+0000", 2023-06-03T20:00:41.925 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.0013677229999999999 2023-06-03T20:00:41.925 INFO:teuthology.orchestra.run.smithi089.stdout: } 2023-06-03T20:00:41.925 INFO:teuthology.orchestra.run.smithi089.stdout: ] 2023-06-03T20:00:41.925 INFO:teuthology.orchestra.run.smithi089.stdout: } 2023-06-03T20:00:41.925 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:00:41.925 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:00:41.926 INFO:teuthology.orchestra.run.smithi089.stdout: "description": "osd_op(client.44762.0:37599 8.1 8:85c7096e:::benchmark_data_smithi089_91006_object37598:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e100)", 2023-06-03T20:00:41.927 INFO:teuthology.orchestra.run.smithi089.stdout: "initiated_at": "2023-06-03T19:51:33.550048+0000", 2023-06-03T20:00:41.927 INFO:teuthology.orchestra.run.smithi089.stdout: "age": 548.35302108899998, 2023-06-03T20:00:41.927 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 1.0221049760000001, 2023-06-03T20:00:41.927 INFO:teuthology.orchestra.run.smithi089.stdout: "type_data": { 2023-06-03T20:00:41.927 INFO:teuthology.orchestra.run.smithi089.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-03T20:00:41.927 INFO:teuthology.orchestra.run.smithi089.stdout: "client_info": { 2023-06-03T20:00:41.927 INFO:teuthology.orchestra.run.smithi089.stdout: "client": "client.44762", 2023-06-03T20:00:41.928 INFO:teuthology.orchestra.run.smithi089.stdout: "client_addr": "172.21.15.89:0/4205857995", 2023-06-03T20:00:41.928 INFO:teuthology.orchestra.run.smithi089.stdout: "tid": 37599 2023-06-03T20:00:41.928 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:00:41.928 INFO:teuthology.orchestra.run.smithi089.stdout: "events": [ 2023-06-03T20:00:41.928 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:00:41.928 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "initiated", 2023-06-03T20:00:41.928 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T19:51:33.550048+0000", 2023-06-03T20:00:41.929 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0 2023-06-03T20:00:41.929 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:00:41.929 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:00:41.929 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "throttled", 2023-06-03T20:00:41.929 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T19:51:33.550048+0000", 2023-06-03T20:00:41.929 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0 2023-06-03T20:00:41.929 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:00:41.929 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:00:41.930 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "header_read", 2023-06-03T20:00:41.930 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T19:51:33.550048+0000", 2023-06-03T20:00:41.930 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 4294967295.9999995 2023-06-03T20:00:41.930 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:00:41.930 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:00:41.930 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "all_read", 2023-06-03T20:00:41.931 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T19:51:33.550072+0000", 2023-06-03T20:00:41.932 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 2.4128999999999999e-05 2023-06-03T20:00:41.932 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:00:41.932 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:00:41.932 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "dispatched", 2023-06-03T20:00:41.932 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T19:51:33.550081+0000", 2023-06-03T20:00:41.933 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 9.4350000000000003e-06 2023-06-03T20:00:41.933 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:00:41.933 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:00:41.933 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "queued_for_pg", 2023-06-03T20:00:41.933 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T19:51:33.550090+0000", 2023-06-03T20:00:41.934 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 9.1419999999999999e-06 2023-06-03T20:00:41.934 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:00:41.934 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:00:41.934 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "reached_pg", 2023-06-03T20:00:41.934 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T19:51:33.550135+0000", 2023-06-03T20:00:41.934 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 4.4480000000000001e-05 2023-06-03T20:00:41.935 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:00:41.935 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:00:41.935 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "waiting for peered", 2023-06-03T20:00:41.935 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T19:51:33.550146+0000", 2023-06-03T20:00:41.935 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 1.1423999999999999e-05 2023-06-03T20:00:41.935 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:00:41.935 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:00:41.936 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "reached_pg", 2023-06-03T20:00:41.936 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T19:51:34.558068+0000", 2023-06-03T20:00:41.936 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 1.007921568 2023-06-03T20:00:41.936 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:00:41.936 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:00:41.936 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "started", 2023-06-03T20:00:41.936 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T19:51:34.558412+0000", 2023-06-03T20:00:41.937 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.00034406900000000002 2023-06-03T20:00:41.937 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:00:41.937 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:00:41.937 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "waiting for subops from 2", 2023-06-03T20:00:41.937 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T19:51:34.558557+0000", 2023-06-03T20:00:41.937 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.00014556899999999999 2023-06-03T20:00:41.937 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:00:41.938 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:00:41.938 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:00:41.938 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T19:51:34.568968+0000", 2023-06-03T20:00:41.938 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.010410617 2023-06-03T20:00:41.938 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:00:41.938 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:00:41.939 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "op_commit", 2023-06-03T20:00:41.939 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T19:51:34.572013+0000", 2023-06-03T20:00:41.939 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.0030445609999999999 2023-06-03T20:00:41.939 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:00:41.939 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:00:41.939 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "commit_sent", 2023-06-03T20:00:41.939 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T19:51:34.572065+0000", 2023-06-03T20:00:41.940 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 5.2176000000000003e-05 2023-06-03T20:00:41.940 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:00:41.940 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:00:41.940 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "done", 2023-06-03T20:00:41.940 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T19:51:34.572153+0000", 2023-06-03T20:00:41.940 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 8.8429000000000003e-05 2023-06-03T20:00:41.940 INFO:teuthology.orchestra.run.smithi089.stdout: } 2023-06-03T20:00:41.941 INFO:teuthology.orchestra.run.smithi089.stdout: ] 2023-06-03T20:00:41.941 INFO:teuthology.orchestra.run.smithi089.stdout: } 2023-06-03T20:00:41.941 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:00:41.941 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:00:41.941 INFO:teuthology.orchestra.run.smithi089.stdout: "description": "osd_op(client.44762.0:71109 8.1 8:89085fa0:::benchmark_data_smithi089_91006_object71108:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=2 ondisk+retry+write+known_if_redirected e104)", 2023-06-03T20:00:41.941 INFO:teuthology.orchestra.run.smithi089.stdout: "initiated_at": "2023-06-03T19:51:59.350283+0000", 2023-06-03T20:00:41.941 INFO:teuthology.orchestra.run.smithi089.stdout: "age": 522.55278590199998, 2023-06-03T20:00:41.942 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 1.157047827, 2023-06-03T20:00:41.942 INFO:teuthology.orchestra.run.smithi089.stdout: "type_data": { 2023-06-03T20:00:41.942 INFO:teuthology.orchestra.run.smithi089.stdout: "flag_point": "reached pg", 2023-06-03T20:00:41.942 INFO:teuthology.orchestra.run.smithi089.stdout: "client_info": { 2023-06-03T20:00:41.942 INFO:teuthology.orchestra.run.smithi089.stdout: "client": "client.44762", 2023-06-03T20:00:41.942 INFO:teuthology.orchestra.run.smithi089.stdout: "client_addr": "172.21.15.89:0/4205857995", 2023-06-03T20:00:41.942 INFO:teuthology.orchestra.run.smithi089.stdout: "tid": 71109 2023-06-03T20:00:41.943 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:00:41.943 INFO:teuthology.orchestra.run.smithi089.stdout: "events": [ 2023-06-03T20:00:41.943 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:00:41.943 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "initiated", 2023-06-03T20:00:41.943 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T19:51:59.350283+0000", 2023-06-03T20:00:41.943 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0 2023-06-03T20:00:41.943 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:00:41.943 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:00:41.944 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "throttled", 2023-06-03T20:00:41.944 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T19:51:59.350283+0000", 2023-06-03T20:00:41.944 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0 2023-06-03T20:00:41.944 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:00:41.944 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:00:41.945 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "header_read", 2023-06-03T20:00:41.945 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T19:51:59.350281+0000", 2023-06-03T20:00:41.945 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 4294967295.9999976 2023-06-03T20:00:41.945 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:00:41.945 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:00:41.945 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "all_read", 2023-06-03T20:00:41.946 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T19:51:59.350354+0000", 2023-06-03T20:00:41.946 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 7.3090000000000007e-05 2023-06-03T20:00:41.946 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:00:41.946 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:00:41.946 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "dispatched", 2023-06-03T20:00:41.947 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T19:51:59.350385+0000", 2023-06-03T20:00:41.947 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 3.0762999999999997e-05 2023-06-03T20:00:41.947 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:00:41.947 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:00:41.948 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "queued_for_pg", 2023-06-03T20:00:41.948 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T19:51:59.350408+0000", 2023-06-03T20:00:41.948 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 2.2875999999999999e-05 2023-06-03T20:00:41.948 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:00:41.949 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:00:41.949 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "reached_pg", 2023-06-03T20:00:41.949 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T19:51:59.350626+0000", 2023-06-03T20:00:41.949 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.00021787799999999999 2023-06-03T20:00:41.949 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:00:41.950 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:00:41.950 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "op must wait for map", 2023-06-03T20:00:41.950 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T19:51:59.350639+0000", 2023-06-03T20:00:41.950 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 1.2632e-05 2023-06-03T20:00:41.950 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:00:41.950 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:00:41.950 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "reached_pg", 2023-06-03T20:00:41.951 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T19:51:59.352758+0000", 2023-06-03T20:00:41.951 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.0021196040000000002 2023-06-03T20:00:41.951 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:00:41.951 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:00:41.951 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "waiting for peered", 2023-06-03T20:00:41.951 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T19:51:59.352772+0000", 2023-06-03T20:00:41.951 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 1.3709999999999999e-05 2023-06-03T20:00:41.952 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:00:41.952 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:00:41.952 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "reached_pg", 2023-06-03T20:00:41.952 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T19:52:00.507200+0000", 2023-06-03T20:00:41.952 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 1.154427994 2023-06-03T20:00:41.952 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:00:41.953 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:00:41.953 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "done", 2023-06-03T20:00:41.953 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T19:52:00.507331+0000", 2023-06-03T20:00:41.954 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.00013149099999999999 2023-06-03T20:00:41.954 INFO:teuthology.orchestra.run.smithi089.stdout: } 2023-06-03T20:00:41.955 INFO:teuthology.orchestra.run.smithi089.stdout: ] 2023-06-03T20:00:41.955 INFO:teuthology.orchestra.run.smithi089.stdout: } 2023-06-03T20:00:41.955 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:00:41.956 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:00:41.956 INFO:teuthology.orchestra.run.smithi089.stdout: "description": "osd_op(client.44762.0:71110 8.1 8:8fc682da:::benchmark_data_smithi089_91006_object71109:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=2 ondisk+retry+write+known_if_redirected e104)", 2023-06-03T20:00:41.956 INFO:teuthology.orchestra.run.smithi089.stdout: "initiated_at": "2023-06-03T19:51:59.350434+0000", 2023-06-03T20:00:41.957 INFO:teuthology.orchestra.run.smithi089.stdout: "age": 522.55263551500002, 2023-06-03T20:00:41.957 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 1.15902609, 2023-06-03T20:00:41.957 INFO:teuthology.orchestra.run.smithi089.stdout: "type_data": { 2023-06-03T20:00:41.958 INFO:teuthology.orchestra.run.smithi089.stdout: "flag_point": "reached pg", 2023-06-03T20:00:41.958 INFO:teuthology.orchestra.run.smithi089.stdout: "client_info": { 2023-06-03T20:00:41.958 INFO:teuthology.orchestra.run.smithi089.stdout: "client": "client.44762", 2023-06-03T20:00:41.959 INFO:teuthology.orchestra.run.smithi089.stdout: "client_addr": "172.21.15.89:0/4205857995", 2023-06-03T20:00:41.959 INFO:teuthology.orchestra.run.smithi089.stdout: "tid": 71110 2023-06-03T20:00:41.959 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:00:41.960 INFO:teuthology.orchestra.run.smithi089.stdout: "events": [ 2023-06-03T20:00:41.960 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:00:41.960 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "initiated", 2023-06-03T20:00:41.961 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T19:51:59.350434+0000", 2023-06-03T20:00:41.961 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0 2023-06-03T20:00:41.961 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:00:41.962 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:00:41.962 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "throttled", 2023-06-03T20:00:41.962 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T19:51:59.350434+0000", 2023-06-03T20:00:41.962 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0 2023-06-03T20:00:41.963 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:00:41.963 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:00:41.963 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "header_read", 2023-06-03T20:00:41.963 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T19:51:59.350433+0000", 2023-06-03T20:00:41.963 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 4294967295.999999 2023-06-03T20:00:41.963 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:00:41.963 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:00:41.964 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "all_read", 2023-06-03T20:00:41.964 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T19:51:59.350469+0000", 2023-06-03T20:00:41.964 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 3.6294000000000002e-05 2023-06-03T20:00:41.964 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:00:41.965 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:00:41.965 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "dispatched", 2023-06-03T20:00:41.965 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T19:51:59.350481+0000", 2023-06-03T20:00:41.965 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 1.1817000000000001e-05 2023-06-03T20:00:41.966 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:00:41.966 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:00:41.966 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "queued_for_pg", 2023-06-03T20:00:41.966 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T19:51:59.350490+0000", 2023-06-03T20:00:41.967 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 8.4120000000000001e-06 2023-06-03T20:00:41.967 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:00:41.967 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:00:41.967 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "reached_pg", 2023-06-03T20:00:41.968 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T19:51:59.350704+0000", 2023-06-03T20:00:41.968 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.00021480800000000001 2023-06-03T20:00:41.968 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:00:41.969 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:00:41.969 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "waiting_for_map not empty", 2023-06-03T20:00:41.969 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T19:51:59.350712+0000", 2023-06-03T20:00:41.970 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 7.5909999999999998e-06 2023-06-03T20:00:41.970 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:00:41.970 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:00:41.970 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "reached_pg", 2023-06-03T20:00:41.971 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T19:51:59.352798+0000", 2023-06-03T20:00:41.971 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.0020864080000000001 2023-06-03T20:00:41.971 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:00:41.971 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:00:41.972 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "waiting for peered", 2023-06-03T20:00:41.972 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T19:51:59.352807+0000", 2023-06-03T20:00:41.972 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 8.6929999999999996e-06 2023-06-03T20:00:41.972 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:00:41.973 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:00:41.973 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "reached_pg", 2023-06-03T20:00:41.973 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T19:52:00.509358+0000", 2023-06-03T20:00:41.973 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 1.1565507509999999 2023-06-03T20:00:41.974 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:00:41.974 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:00:41.974 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "done", 2023-06-03T20:00:41.975 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T19:52:00.509460+0000", 2023-06-03T20:00:41.975 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.000102175 2023-06-03T20:00:41.975 INFO:teuthology.orchestra.run.smithi089.stdout: } 2023-06-03T20:00:41.976 INFO:teuthology.orchestra.run.smithi089.stdout: ] 2023-06-03T20:00:41.976 INFO:teuthology.orchestra.run.smithi089.stdout: } 2023-06-03T20:00:41.976 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:00:41.976 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:00:41.977 INFO:teuthology.orchestra.run.smithi089.stdout: "description": "osd_op(client.44762.0:71116 8.1 8:8d91952f:::benchmark_data_smithi089_91006_object71115:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=2 ondisk+retry+write+known_if_redirected e104)", 2023-06-03T20:00:41.977 INFO:teuthology.orchestra.run.smithi089.stdout: "initiated_at": "2023-06-03T19:51:59.350525+0000", 2023-06-03T20:00:41.977 INFO:teuthology.orchestra.run.smithi089.stdout: "age": 522.552543883, 2023-06-03T20:00:41.978 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 1.1903805789999999, 2023-06-03T20:00:41.978 INFO:teuthology.orchestra.run.smithi089.stdout: "type_data": { 2023-06-03T20:00:41.978 INFO:teuthology.orchestra.run.smithi089.stdout: "flag_point": "reached pg", 2023-06-03T20:00:41.979 INFO:teuthology.orchestra.run.smithi089.stdout: "client_info": { 2023-06-03T20:00:41.979 INFO:teuthology.orchestra.run.smithi089.stdout: "client": "client.44762", 2023-06-03T20:00:41.979 INFO:teuthology.orchestra.run.smithi089.stdout: "client_addr": "172.21.15.89:0/4205857995", 2023-06-03T20:00:41.979 INFO:teuthology.orchestra.run.smithi089.stdout: "tid": 71116 2023-06-03T20:00:41.979 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:00:41.980 INFO:teuthology.orchestra.run.smithi089.stdout: "events": [ 2023-06-03T20:00:41.980 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:00:41.980 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "initiated", 2023-06-03T20:00:41.980 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T19:51:59.350525+0000", 2023-06-03T20:00:41.980 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0 2023-06-03T20:00:41.981 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:00:41.981 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:00:41.981 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "throttled", 2023-06-03T20:00:41.981 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T19:51:59.350525+0000", 2023-06-03T20:00:41.982 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0 2023-06-03T20:00:41.982 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:00:41.982 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:00:41.982 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "header_read", 2023-06-03T20:00:41.982 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T19:51:59.350525+0000", 2023-06-03T20:00:41.982 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 4294967295.9999995 2023-06-03T20:00:41.982 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:00:41.983 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:00:41.983 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "all_read", 2023-06-03T20:00:41.983 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T19:51:59.350573+0000", 2023-06-03T20:00:41.983 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 4.8517999999999999e-05 2023-06-03T20:00:41.983 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:00:41.983 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:00:41.983 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "dispatched", 2023-06-03T20:00:41.984 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T19:51:59.350583+0000", 2023-06-03T20:00:41.984 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 1.0003999999999999e-05 2023-06-03T20:00:41.984 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:00:41.984 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:00:41.984 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "queued_for_pg", 2023-06-03T20:00:41.984 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T19:51:59.350592+0000", 2023-06-03T20:00:41.984 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 8.7420000000000003e-06 2023-06-03T20:00:41.985 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:00:41.985 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:00:41.985 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "reached_pg", 2023-06-03T20:00:41.985 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T19:51:59.350750+0000", 2023-06-03T20:00:41.985 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.00015838500000000001 2023-06-03T20:00:41.985 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:00:41.986 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:00:41.986 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "waiting_for_map not empty", 2023-06-03T20:00:41.986 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T19:51:59.350763+0000", 2023-06-03T20:00:41.986 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 1.2906e-05 2023-06-03T20:00:41.986 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:00:41.987 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:00:41.987 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "reached_pg", 2023-06-03T20:00:41.987 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T19:51:59.352831+0000", 2023-06-03T20:00:41.987 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.0020675680000000001 2023-06-03T20:00:41.988 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:00:41.988 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:00:41.988 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "waiting for peered", 2023-06-03T20:00:41.989 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T19:51:59.352839+0000", 2023-06-03T20:00:41.989 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 8.5059999999999995e-06 2023-06-03T20:00:41.989 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:00:41.989 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:00:41.990 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "reached_pg", 2023-06-03T20:00:41.990 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T19:52:00.509491+0000", 2023-06-03T20:00:41.990 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 1.156651994 2023-06-03T20:00:41.990 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:00:41.990 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:00:41.991 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "waiting for degraded object", 2023-06-03T20:00:41.991 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T19:52:00.513708+0000", 2023-06-03T20:00:41.991 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.0042168839999999997 2023-06-03T20:00:41.991 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:00:41.991 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:00:41.991 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "reached_pg", 2023-06-03T20:00:41.991 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T19:52:00.540780+0000", 2023-06-03T20:00:41.991 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.027071418 2023-06-03T20:00:41.992 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:00:41.992 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:00:41.992 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "done", 2023-06-03T20:00:41.992 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T19:52:00.540906+0000", 2023-06-03T20:00:41.992 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.00012635599999999999 2023-06-03T20:00:41.993 INFO:teuthology.orchestra.run.smithi089.stdout: } 2023-06-03T20:00:41.993 INFO:teuthology.orchestra.run.smithi089.stdout: ] 2023-06-03T20:00:41.993 INFO:teuthology.orchestra.run.smithi089.stdout: } 2023-06-03T20:00:41.993 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:00:41.993 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:00:41.993 INFO:teuthology.orchestra.run.smithi089.stdout: "description": "osd_op(client.44762.0:71142 8.1 8:87a8bac7:::benchmark_data_smithi089_91006_object71141:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e104)", 2023-06-03T20:00:41.993 INFO:teuthology.orchestra.run.smithi089.stdout: "initiated_at": "2023-06-03T19:51:59.350600+0000", 2023-06-03T20:00:41.993 INFO:teuthology.orchestra.run.smithi089.stdout: "age": 522.55246979399999, 2023-06-03T20:00:41.994 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 1.196213985, 2023-06-03T20:00:41.994 INFO:teuthology.orchestra.run.smithi089.stdout: "type_data": { 2023-06-03T20:00:41.994 INFO:teuthology.orchestra.run.smithi089.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-03T20:00:41.994 INFO:teuthology.orchestra.run.smithi089.stdout: "client_info": { 2023-06-03T20:00:41.994 INFO:teuthology.orchestra.run.smithi089.stdout: "client": "client.44762", 2023-06-03T20:00:41.994 INFO:teuthology.orchestra.run.smithi089.stdout: "client_addr": "172.21.15.89:0/4205857995", 2023-06-03T20:00:41.994 INFO:teuthology.orchestra.run.smithi089.stdout: "tid": 71142 2023-06-03T20:00:41.994 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:00:41.995 INFO:teuthology.orchestra.run.smithi089.stdout: "events": [ 2023-06-03T20:00:41.995 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:00:41.995 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "initiated", 2023-06-03T20:00:41.995 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T19:51:59.350600+0000", 2023-06-03T20:00:41.995 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0 2023-06-03T20:00:41.995 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:00:41.995 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:00:41.996 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "throttled", 2023-06-03T20:00:41.996 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T19:51:59.350600+0000", 2023-06-03T20:00:41.996 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0 2023-06-03T20:00:41.996 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:00:41.996 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:00:41.997 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "header_read", 2023-06-03T20:00:41.997 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T19:51:59.350599+0000", 2023-06-03T20:00:41.997 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 4294967295.999999 2023-06-03T20:00:41.997 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:00:41.997 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:00:41.997 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "all_read", 2023-06-03T20:00:41.997 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T19:51:59.350631+0000", 2023-06-03T20:00:41.997 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 3.2404999999999999e-05 2023-06-03T20:00:41.998 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:00:41.998 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:00:41.998 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "dispatched", 2023-06-03T20:00:41.998 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T19:51:59.350639+0000", 2023-06-03T20:00:41.998 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 7.6000000000000001e-06 2023-06-03T20:00:41.998 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:00:41.998 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:00:41.998 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "queued_for_pg", 2023-06-03T20:00:41.999 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T19:51:59.350645+0000", 2023-06-03T20:00:41.999 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 6.3040000000000001e-06 2023-06-03T20:00:41.999 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:00:41.999 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:00:41.999 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "reached_pg", 2023-06-03T20:00:41.999 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T19:51:59.350793+0000", 2023-06-03T20:00:41.999 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.00014832699999999999 2023-06-03T20:00:41.999 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:00:42.000 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:00:42.000 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "waiting_for_map not empty", 2023-06-03T20:00:42.000 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T19:51:59.350802+0000", 2023-06-03T20:00:42.000 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 8.5099999999999998e-06 2023-06-03T20:00:42.000 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:00:42.000 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:00:42.000 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "reached_pg", 2023-06-03T20:00:42.001 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T19:51:59.352863+0000", 2023-06-03T20:00:42.001 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.0020611549999999998 2023-06-03T20:00:42.001 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:00:42.001 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:00:42.001 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "waiting for peered", 2023-06-03T20:00:42.001 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T19:51:59.352872+0000", 2023-06-03T20:00:42.001 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 8.8589999999999994e-06 2023-06-03T20:00:42.001 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:00:42.002 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:00:42.002 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "reached_pg", 2023-06-03T20:00:42.002 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T19:52:00.513778+0000", 2023-06-03T20:00:42.002 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 1.16090608 2023-06-03T20:00:42.002 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:00:42.002 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:00:42.002 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "started", 2023-06-03T20:00:42.002 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T19:52:00.513943+0000", 2023-06-03T20:00:42.003 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.000164644 2023-06-03T20:00:42.003 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:00:42.003 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:00:42.003 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "waiting for subops from 1,2", 2023-06-03T20:00:42.003 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T19:52:00.514128+0000", 2023-06-03T20:00:42.003 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.00018530599999999999 2023-06-03T20:00:42.003 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:00:42.004 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:00:42.004 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:00:42.004 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T19:52:00.530194+0000", 2023-06-03T20:00:42.004 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.016065743 2023-06-03T20:00:42.004 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:00:42.004 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:00:42.005 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "op_commit", 2023-06-03T20:00:42.005 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T19:52:00.530211+0000", 2023-06-03T20:00:42.005 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 1.7241999999999999e-05 2023-06-03T20:00:42.005 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:00:42.005 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:00:42.006 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:00:42.006 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T19:52:00.544779+0000", 2023-06-03T20:00:42.006 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.01456847 2023-06-03T20:00:42.006 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:00:42.006 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:00:42.006 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "commit_sent", 2023-06-03T20:00:42.006 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T19:52:00.544844+0000", 2023-06-03T20:00:42.006 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 6.4886000000000002e-05 2023-06-03T20:00:42.007 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:00:42.007 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:00:42.007 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "done", 2023-06-03T20:00:42.007 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T19:52:00.546814+0000", 2023-06-03T20:00:42.007 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.0019692310000000001 2023-06-03T20:00:42.007 INFO:teuthology.orchestra.run.smithi089.stdout: } 2023-06-03T20:00:42.007 INFO:teuthology.orchestra.run.smithi089.stdout: ] 2023-06-03T20:00:42.008 INFO:teuthology.orchestra.run.smithi089.stdout: } 2023-06-03T20:00:42.008 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:00:42.008 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:00:42.008 INFO:teuthology.orchestra.run.smithi089.stdout: "description": "osd_op(client.45713.0:347 9.5 9:ae0481fd:::benchmark_data_smithi089_115580_object346:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e131)", 2023-06-03T20:00:42.008 INFO:teuthology.orchestra.run.smithi089.stdout: "initiated_at": "2023-06-03T20:00:41.281729+0000", 2023-06-03T20:00:42.008 INFO:teuthology.orchestra.run.smithi089.stdout: "age": 0.62134042700000003, 2023-06-03T20:00:42.008 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.013696623, 2023-06-03T20:00:42.008 INFO:teuthology.orchestra.run.smithi089.stdout: "type_data": { 2023-06-03T20:00:42.009 INFO:teuthology.orchestra.run.smithi089.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-03T20:00:42.009 INFO:teuthology.orchestra.run.smithi089.stdout: "client_info": { 2023-06-03T20:00:42.009 INFO:teuthology.orchestra.run.smithi089.stdout: "client": "client.45713", 2023-06-03T20:00:42.009 INFO:teuthology.orchestra.run.smithi089.stdout: "client_addr": "172.21.15.89:0/2400821448", 2023-06-03T20:00:42.009 INFO:teuthology.orchestra.run.smithi089.stdout: "tid": 347 2023-06-03T20:00:42.009 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:00:42.009 INFO:teuthology.orchestra.run.smithi089.stdout: "events": [ 2023-06-03T20:00:42.010 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:00:42.010 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "initiated", 2023-06-03T20:00:42.010 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:41.281729+0000", 2023-06-03T20:00:42.010 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0 2023-06-03T20:00:42.010 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:00:42.010 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:00:42.010 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "throttled", 2023-06-03T20:00:42.010 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:41.281729+0000", 2023-06-03T20:00:42.011 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0 2023-06-03T20:00:42.011 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:00:42.011 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:00:42.011 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "header_read", 2023-06-03T20:00:42.011 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:41.281727+0000", 2023-06-03T20:00:42.011 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 4294967295.9999986 2023-06-03T20:00:42.011 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:00:42.011 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:00:42.012 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "all_read", 2023-06-03T20:00:42.012 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:41.281787+0000", 2023-06-03T20:00:42.012 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 5.9503000000000003e-05 2023-06-03T20:00:42.012 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:00:42.012 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:00:42.012 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "dispatched", 2023-06-03T20:00:42.012 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:41.281801+0000", 2023-06-03T20:00:42.012 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 1.4323e-05 2023-06-03T20:00:42.013 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:00:42.013 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:00:42.013 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "queued_for_pg", 2023-06-03T20:00:42.013 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:41.281809+0000", 2023-06-03T20:00:42.013 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 8.2220000000000002e-06 2023-06-03T20:00:42.013 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:00:42.013 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:00:42.014 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "reached_pg", 2023-06-03T20:00:42.014 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:41.281874+0000", 2023-06-03T20:00:42.014 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 6.4354000000000001e-05 2023-06-03T20:00:42.014 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:00:42.014 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:00:42.014 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "started", 2023-06-03T20:00:42.014 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:41.282021+0000", 2023-06-03T20:00:42.014 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.000147787 2023-06-03T20:00:42.015 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:00:42.015 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:00:42.015 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "waiting for subops from 5,7", 2023-06-03T20:00:42.015 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:41.282191+0000", 2023-06-03T20:00:42.015 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.000169154 2023-06-03T20:00:42.015 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:00:42.015 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:00:42.015 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:00:42.016 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:41.285024+0000", 2023-06-03T20:00:42.016 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.002833056 2023-06-03T20:00:42.016 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:00:42.016 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:00:42.016 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:00:42.016 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:41.288268+0000", 2023-06-03T20:00:42.016 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.0032438900000000001 2023-06-03T20:00:42.016 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:00:42.017 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:00:42.017 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "op_commit", 2023-06-03T20:00:42.017 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:41.295331+0000", 2023-06-03T20:00:42.017 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.0070631119999999999 2023-06-03T20:00:42.017 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:00:42.017 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:00:42.018 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "commit_sent", 2023-06-03T20:00:42.018 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:41.295388+0000", 2023-06-03T20:00:42.019 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 5.6960999999999999e-05 2023-06-03T20:00:42.019 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:00:42.019 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:00:42.019 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "done", 2023-06-03T20:00:42.019 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:41.295426+0000", 2023-06-03T20:00:42.020 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 3.7908999999999999e-05 2023-06-03T20:00:42.020 INFO:teuthology.orchestra.run.smithi089.stdout: } 2023-06-03T20:00:42.020 INFO:teuthology.orchestra.run.smithi089.stdout: ] 2023-06-03T20:00:42.020 INFO:teuthology.orchestra.run.smithi089.stdout: } 2023-06-03T20:00:42.020 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:00:42.020 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:00:42.021 INFO:teuthology.orchestra.run.smithi089.stdout: "description": "osd_op(client.45713.0:353 9.9 9:9983689d:::benchmark_data_smithi089_115580_object352:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e131)", 2023-06-03T20:00:42.021 INFO:teuthology.orchestra.run.smithi089.stdout: "initiated_at": "2023-06-03T20:00:41.285071+0000", 2023-06-03T20:00:42.021 INFO:teuthology.orchestra.run.smithi089.stdout: "age": 0.61799875800000004, 2023-06-03T20:00:42.021 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.016130292000000001, 2023-06-03T20:00:42.022 INFO:teuthology.orchestra.run.smithi089.stdout: "type_data": { 2023-06-03T20:00:42.022 INFO:teuthology.orchestra.run.smithi089.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-03T20:00:42.022 INFO:teuthology.orchestra.run.smithi089.stdout: "client_info": { 2023-06-03T20:00:42.022 INFO:teuthology.orchestra.run.smithi089.stdout: "client": "client.45713", 2023-06-03T20:00:42.023 INFO:teuthology.orchestra.run.smithi089.stdout: "client_addr": "172.21.15.89:0/2400821448", 2023-06-03T20:00:42.023 INFO:teuthology.orchestra.run.smithi089.stdout: "tid": 353 2023-06-03T20:00:42.023 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:00:42.023 INFO:teuthology.orchestra.run.smithi089.stdout: "events": [ 2023-06-03T20:00:42.023 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:00:42.023 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "initiated", 2023-06-03T20:00:42.024 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:41.285071+0000", 2023-06-03T20:00:42.024 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0 2023-06-03T20:00:42.024 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:00:42.024 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:00:42.024 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "throttled", 2023-06-03T20:00:42.024 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:41.285071+0000", 2023-06-03T20:00:42.024 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0 2023-06-03T20:00:42.025 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:00:42.025 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:00:42.025 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "header_read", 2023-06-03T20:00:42.025 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:41.285069+0000", 2023-06-03T20:00:42.025 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 4294967295.9999986 2023-06-03T20:00:42.025 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:00:42.025 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:00:42.025 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "all_read", 2023-06-03T20:00:42.026 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:41.285129+0000", 2023-06-03T20:00:42.026 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 5.9694999999999997e-05 2023-06-03T20:00:42.026 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:00:42.026 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:00:42.026 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "dispatched", 2023-06-03T20:00:42.026 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:41.285142+0000", 2023-06-03T20:00:42.026 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 1.2884999999999999e-05 2023-06-03T20:00:42.026 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:00:42.027 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:00:42.027 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "queued_for_pg", 2023-06-03T20:00:42.027 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:41.285149+0000", 2023-06-03T20:00:42.027 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 7.2699999999999999e-06 2023-06-03T20:00:42.027 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:00:42.027 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:00:42.027 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "reached_pg", 2023-06-03T20:00:42.027 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:41.295105+0000", 2023-06-03T20:00:42.028 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.0099559609999999993 2023-06-03T20:00:42.028 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:00:42.028 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:00:42.029 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "started", 2023-06-03T20:00:42.029 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:41.295271+0000", 2023-06-03T20:00:42.029 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.000165818 2023-06-03T20:00:42.029 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:00:42.030 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:00:42.030 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "waiting for subops from 5,7", 2023-06-03T20:00:42.030 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:41.295469+0000", 2023-06-03T20:00:42.031 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.00019877299999999999 2023-06-03T20:00:42.031 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:00:42.031 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:00:42.032 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "op_commit", 2023-06-03T20:00:42.032 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:41.296110+0000", 2023-06-03T20:00:42.032 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.00064068800000000004 2023-06-03T20:00:42.032 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:00:42.033 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:00:42.033 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:00:42.033 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:41.298189+0000", 2023-06-03T20:00:42.034 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.00207909 2023-06-03T20:00:42.034 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:00:42.034 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:00:42.034 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:00:42.035 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:41.301095+0000", 2023-06-03T20:00:42.035 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.0029060050000000001 2023-06-03T20:00:42.035 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:00:42.035 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:00:42.036 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "commit_sent", 2023-06-03T20:00:42.036 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:41.301154+0000", 2023-06-03T20:00:42.036 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 5.8526999999999997e-05 2023-06-03T20:00:42.037 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:00:42.037 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:00:42.037 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "done", 2023-06-03T20:00:42.037 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:41.301201+0000", 2023-06-03T20:00:42.038 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 4.7172000000000002e-05 2023-06-03T20:00:42.038 INFO:teuthology.orchestra.run.smithi089.stdout: } 2023-06-03T20:00:42.038 INFO:teuthology.orchestra.run.smithi089.stdout: ] 2023-06-03T20:00:42.038 INFO:teuthology.orchestra.run.smithi089.stdout: } 2023-06-03T20:00:42.038 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:00:42.038 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:00:42.038 INFO:teuthology.orchestra.run.smithi089.stdout: "description": "osd_op(client.45713.0:354 9.5 9:ab3a72f9:::benchmark_data_smithi089_115580_object353:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e131)", 2023-06-03T20:00:42.039 INFO:teuthology.orchestra.run.smithi089.stdout: "initiated_at": "2023-06-03T20:00:41.285167+0000", 2023-06-03T20:00:42.039 INFO:teuthology.orchestra.run.smithi089.stdout: "age": 0.617902273, 2023-06-03T20:00:42.039 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.011200488, 2023-06-03T20:00:42.039 INFO:teuthology.orchestra.run.smithi089.stdout: "type_data": { 2023-06-03T20:00:42.039 INFO:teuthology.orchestra.run.smithi089.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-03T20:00:42.039 INFO:teuthology.orchestra.run.smithi089.stdout: "client_info": { 2023-06-03T20:00:42.039 INFO:teuthology.orchestra.run.smithi089.stdout: "client": "client.45713", 2023-06-03T20:00:42.040 INFO:teuthology.orchestra.run.smithi089.stdout: "client_addr": "172.21.15.89:0/2400821448", 2023-06-03T20:00:42.040 INFO:teuthology.orchestra.run.smithi089.stdout: "tid": 354 2023-06-03T20:00:42.040 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:00:42.040 INFO:teuthology.orchestra.run.smithi089.stdout: "events": [ 2023-06-03T20:00:42.040 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:00:42.040 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "initiated", 2023-06-03T20:00:42.040 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:41.285167+0000", 2023-06-03T20:00:42.040 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0 2023-06-03T20:00:42.041 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:00:42.041 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:00:42.041 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "throttled", 2023-06-03T20:00:42.041 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:41.285167+0000", 2023-06-03T20:00:42.041 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0 2023-06-03T20:00:42.041 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:00:42.041 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:00:42.041 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "header_read", 2023-06-03T20:00:42.042 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:41.285167+0000", 2023-06-03T20:00:42.042 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 4294967295.9999995 2023-06-03T20:00:42.042 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:00:42.042 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:00:42.042 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "all_read", 2023-06-03T20:00:42.042 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:41.285216+0000", 2023-06-03T20:00:42.042 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 4.9679000000000003e-05 2023-06-03T20:00:42.043 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:00:42.043 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:00:42.043 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "dispatched", 2023-06-03T20:00:42.043 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:41.285226+0000", 2023-06-03T20:00:42.043 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 9.4369999999999996e-06 2023-06-03T20:00:42.043 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:00:42.043 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:00:42.043 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "queued_for_pg", 2023-06-03T20:00:42.044 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:41.285233+0000", 2023-06-03T20:00:42.044 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 7.6529999999999994e-06 2023-06-03T20:00:42.044 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:00:42.044 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:00:42.044 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "reached_pg", 2023-06-03T20:00:42.044 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:41.285281+0000", 2023-06-03T20:00:42.044 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 4.7395999999999998e-05 2023-06-03T20:00:42.044 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:00:42.045 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:00:42.045 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "started", 2023-06-03T20:00:42.045 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:41.285413+0000", 2023-06-03T20:00:42.045 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.00013186900000000001 2023-06-03T20:00:42.045 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:00:42.045 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:00:42.045 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "waiting for subops from 5,7", 2023-06-03T20:00:42.046 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:41.285556+0000", 2023-06-03T20:00:42.046 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.000143218 2023-06-03T20:00:42.046 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:00:42.046 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:00:42.046 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:00:42.046 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:41.288136+0000", 2023-06-03T20:00:42.046 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.0025804529999999999 2023-06-03T20:00:42.046 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:00:42.047 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:00:42.047 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "op_commit", 2023-06-03T20:00:42.047 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:41.295434+0000", 2023-06-03T20:00:42.047 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.007298194 2023-06-03T20:00:42.047 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:00:42.047 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:00:42.047 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:00:42.047 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:41.296274+0000", 2023-06-03T20:00:42.048 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.00084001699999999998 2023-06-03T20:00:42.048 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:00:42.048 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:00:42.048 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "commit_sent", 2023-06-03T20:00:42.048 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:41.296331+0000", 2023-06-03T20:00:42.048 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 5.6348000000000003e-05 2023-06-03T20:00:42.048 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:00:42.048 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:00:42.049 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "done", 2023-06-03T20:00:42.049 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:41.296368+0000", 2023-06-03T20:00:42.049 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 3.6810000000000002e-05 2023-06-03T20:00:42.049 INFO:teuthology.orchestra.run.smithi089.stdout: } 2023-06-03T20:00:42.049 INFO:teuthology.orchestra.run.smithi089.stdout: ] 2023-06-03T20:00:42.049 INFO:teuthology.orchestra.run.smithi089.stdout: } 2023-06-03T20:00:42.049 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:00:42.050 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:00:42.050 INFO:teuthology.orchestra.run.smithi089.stdout: "description": "osd_op(client.45713.0:361 9.5 9:a953773f:::benchmark_data_smithi089_115580_object360:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e131)", 2023-06-03T20:00:42.050 INFO:teuthology.orchestra.run.smithi089.stdout: "initiated_at": "2023-06-03T20:00:41.285880+0000", 2023-06-03T20:00:42.050 INFO:teuthology.orchestra.run.smithi089.stdout: "age": 0.61718983400000005, 2023-06-03T20:00:42.050 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.011555395, 2023-06-03T20:00:42.050 INFO:teuthology.orchestra.run.smithi089.stdout: "type_data": { 2023-06-03T20:00:42.050 INFO:teuthology.orchestra.run.smithi089.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-03T20:00:42.050 INFO:teuthology.orchestra.run.smithi089.stdout: "client_info": { 2023-06-03T20:00:42.051 INFO:teuthology.orchestra.run.smithi089.stdout: "client": "client.45713", 2023-06-03T20:00:42.051 INFO:teuthology.orchestra.run.smithi089.stdout: "client_addr": "172.21.15.89:0/2400821448", 2023-06-03T20:00:42.051 INFO:teuthology.orchestra.run.smithi089.stdout: "tid": 361 2023-06-03T20:00:42.051 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:00:42.051 INFO:teuthology.orchestra.run.smithi089.stdout: "events": [ 2023-06-03T20:00:42.051 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:00:42.051 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "initiated", 2023-06-03T20:00:42.051 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:41.285880+0000", 2023-06-03T20:00:42.052 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0 2023-06-03T20:00:42.052 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:00:42.052 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:00:42.052 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "throttled", 2023-06-03T20:00:42.052 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:41.285880+0000", 2023-06-03T20:00:42.052 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0 2023-06-03T20:00:42.052 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:00:42.052 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:00:42.053 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "header_read", 2023-06-03T20:00:42.053 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:41.285879+0000", 2023-06-03T20:00:42.053 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 4294967295.999999 2023-06-03T20:00:42.053 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:00:42.053 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:00:42.053 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "all_read", 2023-06-03T20:00:42.053 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:41.285949+0000", 2023-06-03T20:00:42.053 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 6.9974999999999993e-05 2023-06-03T20:00:42.054 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:00:42.054 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:00:42.054 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "dispatched", 2023-06-03T20:00:42.054 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:41.285959+0000", 2023-06-03T20:00:42.054 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 1.0739e-05 2023-06-03T20:00:42.054 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:00:42.054 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:00:42.055 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "queued_for_pg", 2023-06-03T20:00:42.055 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:41.285967+0000", 2023-06-03T20:00:42.055 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 7.2259999999999999e-06 2023-06-03T20:00:42.055 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:00:42.055 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:00:42.055 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "reached_pg", 2023-06-03T20:00:42.055 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:41.286011+0000", 2023-06-03T20:00:42.055 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 4.4521999999999998e-05 2023-06-03T20:00:42.056 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:00:42.056 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:00:42.056 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "started", 2023-06-03T20:00:42.056 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:41.286127+0000", 2023-06-03T20:00:42.056 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.000115715 2023-06-03T20:00:42.056 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:00:42.056 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:00:42.056 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "waiting for subops from 5,7", 2023-06-03T20:00:42.057 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:41.286262+0000", 2023-06-03T20:00:42.057 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.00013506600000000001 2023-06-03T20:00:42.057 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:00:42.057 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:00:42.057 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:00:42.057 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:41.288573+0000", 2023-06-03T20:00:42.057 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.0023105280000000001 2023-06-03T20:00:42.057 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:00:42.058 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:00:42.058 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "op_commit", 2023-06-03T20:00:42.058 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:41.295440+0000", 2023-06-03T20:00:42.058 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.0068673320000000003 2023-06-03T20:00:42.058 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:00:42.058 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:00:42.058 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:00:42.058 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:41.297340+0000", 2023-06-03T20:00:42.059 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.001900204 2023-06-03T20:00:42.059 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:00:42.059 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:00:42.059 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "commit_sent", 2023-06-03T20:00:42.059 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:41.297389+0000", 2023-06-03T20:00:42.059 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 4.8795999999999998e-05 2023-06-03T20:00:42.059 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:00:42.059 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:00:42.060 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "done", 2023-06-03T20:00:42.060 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:41.297435+0000", 2023-06-03T20:00:42.060 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 4.6090000000000001e-05 2023-06-03T20:00:42.060 INFO:teuthology.orchestra.run.smithi089.stdout: } 2023-06-03T20:00:42.060 INFO:teuthology.orchestra.run.smithi089.stdout: ] 2023-06-03T20:00:42.060 INFO:teuthology.orchestra.run.smithi089.stdout: } 2023-06-03T20:00:42.060 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:00:42.060 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:00:42.061 INFO:teuthology.orchestra.run.smithi089.stdout: "description": "osd_op(client.45713.0:680 9.9 9:9ab5e6e1:::benchmark_data_smithi089_115580_object679:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e131)", 2023-06-03T20:00:42.061 INFO:teuthology.orchestra.run.smithi089.stdout: "initiated_at": "2023-06-03T20:00:41.384127+0000", 2023-06-03T20:00:42.061 INFO:teuthology.orchestra.run.smithi089.stdout: "age": 0.51894217200000003, 2023-06-03T20:00:42.061 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.0101247, 2023-06-03T20:00:42.061 INFO:teuthology.orchestra.run.smithi089.stdout: "type_data": { 2023-06-03T20:00:42.061 INFO:teuthology.orchestra.run.smithi089.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-03T20:00:42.061 INFO:teuthology.orchestra.run.smithi089.stdout: "client_info": { 2023-06-03T20:00:42.062 INFO:teuthology.orchestra.run.smithi089.stdout: "client": "client.45713", 2023-06-03T20:00:42.062 INFO:teuthology.orchestra.run.smithi089.stdout: "client_addr": "172.21.15.89:0/2400821448", 2023-06-03T20:00:42.062 INFO:teuthology.orchestra.run.smithi089.stdout: "tid": 680 2023-06-03T20:00:42.062 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:00:42.062 INFO:teuthology.orchestra.run.smithi089.stdout: "events": [ 2023-06-03T20:00:42.062 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:00:42.062 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "initiated", 2023-06-03T20:00:42.062 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:41.384127+0000", 2023-06-03T20:00:42.063 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0 2023-06-03T20:00:42.063 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:00:42.063 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:00:42.063 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "throttled", 2023-06-03T20:00:42.063 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:41.384127+0000", 2023-06-03T20:00:42.063 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0 2023-06-03T20:00:42.063 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:00:42.063 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:00:42.064 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "header_read", 2023-06-03T20:00:42.064 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:41.384126+0000", 2023-06-03T20:00:42.064 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 4294967295.999999 2023-06-03T20:00:42.064 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:00:42.064 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:00:42.064 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "all_read", 2023-06-03T20:00:42.064 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:41.384221+0000", 2023-06-03T20:00:42.064 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 9.4907000000000005e-05 2023-06-03T20:00:42.065 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:00:42.065 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:00:42.065 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "dispatched", 2023-06-03T20:00:42.065 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:41.384231+0000", 2023-06-03T20:00:42.065 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 9.6509999999999996e-06 2023-06-03T20:00:42.065 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:00:42.065 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:00:42.065 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "queued_for_pg", 2023-06-03T20:00:42.066 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:41.384239+0000", 2023-06-03T20:00:42.066 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 7.8990000000000001e-06 2023-06-03T20:00:42.066 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:00:42.066 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:00:42.066 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "reached_pg", 2023-06-03T20:00:42.066 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:41.384372+0000", 2023-06-03T20:00:42.066 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.00013333099999999999 2023-06-03T20:00:42.066 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:00:42.067 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:00:42.067 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "started", 2023-06-03T20:00:42.067 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:41.384509+0000", 2023-06-03T20:00:42.067 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.00013732100000000001 2023-06-03T20:00:42.067 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:00:42.067 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:00:42.067 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "waiting for subops from 5,7", 2023-06-03T20:00:42.067 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:41.384675+0000", 2023-06-03T20:00:42.068 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.000165043 2023-06-03T20:00:42.068 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:00:42.068 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:00:42.068 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "op_commit", 2023-06-03T20:00:42.068 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:41.385211+0000", 2023-06-03T20:00:42.068 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.00053660000000000003 2023-06-03T20:00:42.068 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:00:42.069 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:00:42.069 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:00:42.069 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:41.387205+0000", 2023-06-03T20:00:42.069 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.001993533 2023-06-03T20:00:42.069 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:00:42.069 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:00:42.069 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:00:42.069 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:41.394170+0000", 2023-06-03T20:00:42.070 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.0069653060000000001 2023-06-03T20:00:42.070 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:00:42.070 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:00:42.070 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "commit_sent", 2023-06-03T20:00:42.070 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:41.394214+0000", 2023-06-03T20:00:42.070 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 4.4041000000000001e-05 2023-06-03T20:00:42.070 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:00:42.070 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:00:42.071 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "done", 2023-06-03T20:00:42.071 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:41.394252+0000", 2023-06-03T20:00:42.071 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 3.7889000000000003e-05 2023-06-03T20:00:42.071 INFO:teuthology.orchestra.run.smithi089.stdout: } 2023-06-03T20:00:42.071 INFO:teuthology.orchestra.run.smithi089.stdout: ] 2023-06-03T20:00:42.071 INFO:teuthology.orchestra.run.smithi089.stdout: } 2023-06-03T20:00:42.071 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:00:42.071 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:00:42.072 INFO:teuthology.orchestra.run.smithi089.stdout: "description": "osd_op(client.45713.0:719 9.9 9:9673ed55:::benchmark_data_smithi089_115580_object718:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e131)", 2023-06-03T20:00:42.072 INFO:teuthology.orchestra.run.smithi089.stdout: "initiated_at": "2023-06-03T20:00:41.395680+0000", 2023-06-03T20:00:42.072 INFO:teuthology.orchestra.run.smithi089.stdout: "age": 0.50738971399999999, 2023-06-03T20:00:42.072 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.0082327000000000008, 2023-06-03T20:00:42.072 INFO:teuthology.orchestra.run.smithi089.stdout: "type_data": { 2023-06-03T20:00:42.072 INFO:teuthology.orchestra.run.smithi089.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-03T20:00:42.072 INFO:teuthology.orchestra.run.smithi089.stdout: "client_info": { 2023-06-03T20:00:42.073 INFO:teuthology.orchestra.run.smithi089.stdout: "client": "client.45713", 2023-06-03T20:00:42.073 INFO:teuthology.orchestra.run.smithi089.stdout: "client_addr": "172.21.15.89:0/2400821448", 2023-06-03T20:00:42.073 INFO:teuthology.orchestra.run.smithi089.stdout: "tid": 719 2023-06-03T20:00:42.073 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:00:42.073 INFO:teuthology.orchestra.run.smithi089.stdout: "events": [ 2023-06-03T20:00:42.073 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:00:42.073 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "initiated", 2023-06-03T20:00:42.073 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:41.395680+0000", 2023-06-03T20:00:42.074 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0 2023-06-03T20:00:42.074 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:00:42.074 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:00:42.074 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "throttled", 2023-06-03T20:00:42.074 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:41.395680+0000", 2023-06-03T20:00:42.074 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0 2023-06-03T20:00:42.074 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:00:42.074 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:00:42.075 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "header_read", 2023-06-03T20:00:42.075 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:41.395679+0000", 2023-06-03T20:00:42.075 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 4294967295.999999 2023-06-03T20:00:42.075 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:00:42.075 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:00:42.075 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "all_read", 2023-06-03T20:00:42.075 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:41.395741+0000", 2023-06-03T20:00:42.075 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 6.2456e-05 2023-06-03T20:00:42.076 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:00:42.076 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:00:42.076 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "dispatched", 2023-06-03T20:00:42.076 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:41.395752+0000", 2023-06-03T20:00:42.076 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 1.0186e-05 2023-06-03T20:00:42.076 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:00:42.076 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:00:42.076 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "queued_for_pg", 2023-06-03T20:00:42.077 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:41.395759+0000", 2023-06-03T20:00:42.077 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 7.0199999999999997e-06 2023-06-03T20:00:42.077 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:00:42.077 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:00:42.077 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "reached_pg", 2023-06-03T20:00:42.077 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:41.398576+0000", 2023-06-03T20:00:42.077 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.002817637 2023-06-03T20:00:42.077 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:00:42.078 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:00:42.078 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "started", 2023-06-03T20:00:42.078 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:41.398703+0000", 2023-06-03T20:00:42.078 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.00012654300000000001 2023-06-03T20:00:42.078 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:00:42.078 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:00:42.078 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "waiting for subops from 5,7", 2023-06-03T20:00:42.078 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:41.398852+0000", 2023-06-03T20:00:42.079 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.00014872600000000001 2023-06-03T20:00:42.079 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:00:42.079 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:00:42.079 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "op_commit", 2023-06-03T20:00:42.079 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:41.402337+0000", 2023-06-03T20:00:42.079 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.0034853100000000001 2023-06-03T20:00:42.079 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:00:42.079 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:00:42.080 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:00:42.080 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:41.402419+0000", 2023-06-03T20:00:42.080 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 8.2088000000000004e-05 2023-06-03T20:00:42.080 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:00:42.080 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:00:42.080 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:00:42.080 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:41.403801+0000", 2023-06-03T20:00:42.081 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.001382442 2023-06-03T20:00:42.081 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:00:42.081 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:00:42.081 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "commit_sent", 2023-06-03T20:00:42.081 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:41.403863+0000", 2023-06-03T20:00:42.081 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 6.2036e-05 2023-06-03T20:00:42.081 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:00:42.081 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:00:42.082 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "done", 2023-06-03T20:00:42.082 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:41.403912+0000", 2023-06-03T20:00:42.082 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 4.8977999999999998e-05 2023-06-03T20:00:42.082 INFO:teuthology.orchestra.run.smithi089.stdout: } 2023-06-03T20:00:42.082 INFO:teuthology.orchestra.run.smithi089.stdout: ] 2023-06-03T20:00:42.082 INFO:teuthology.orchestra.run.smithi089.stdout: } 2023-06-03T20:00:42.082 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:00:42.082 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:00:42.083 INFO:teuthology.orchestra.run.smithi089.stdout: "description": "osd_op(client.45713.0:1043 9.5 9:a9c23f3f:::benchmark_data_smithi089_115580_object1042:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e131)", 2023-06-03T20:00:42.083 INFO:teuthology.orchestra.run.smithi089.stdout: "initiated_at": "2023-06-03T20:00:41.497783+0000", 2023-06-03T20:00:42.083 INFO:teuthology.orchestra.run.smithi089.stdout: "age": 0.405286164, 2023-06-03T20:00:42.083 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.0082611009999999999, 2023-06-03T20:00:42.083 INFO:teuthology.orchestra.run.smithi089.stdout: "type_data": { 2023-06-03T20:00:42.083 INFO:teuthology.orchestra.run.smithi089.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-03T20:00:42.083 INFO:teuthology.orchestra.run.smithi089.stdout: "client_info": { 2023-06-03T20:00:42.083 INFO:teuthology.orchestra.run.smithi089.stdout: "client": "client.45713", 2023-06-03T20:00:42.084 INFO:teuthology.orchestra.run.smithi089.stdout: "client_addr": "172.21.15.89:0/2400821448", 2023-06-03T20:00:42.084 INFO:teuthology.orchestra.run.smithi089.stdout: "tid": 1043 2023-06-03T20:00:42.084 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:00:42.084 INFO:teuthology.orchestra.run.smithi089.stdout: "events": [ 2023-06-03T20:00:42.084 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:00:42.084 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "initiated", 2023-06-03T20:00:42.085 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:41.497783+0000", 2023-06-03T20:00:42.085 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0 2023-06-03T20:00:42.085 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:00:42.085 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:00:42.085 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "throttled", 2023-06-03T20:00:42.085 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:41.497783+0000", 2023-06-03T20:00:42.085 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0 2023-06-03T20:00:42.085 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:00:42.086 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:00:42.086 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "header_read", 2023-06-03T20:00:42.086 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:41.497782+0000", 2023-06-03T20:00:42.086 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 4294967295.9999986 2023-06-03T20:00:42.086 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:00:42.086 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:00:42.086 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "all_read", 2023-06-03T20:00:42.087 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:41.497811+0000", 2023-06-03T20:00:42.087 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 2.8929999999999999e-05 2023-06-03T20:00:42.087 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:00:42.087 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:00:42.087 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "dispatched", 2023-06-03T20:00:42.087 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:41.497832+0000", 2023-06-03T20:00:42.087 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 2.1192e-05 2023-06-03T20:00:42.087 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:00:42.088 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:00:42.088 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "queued_for_pg", 2023-06-03T20:00:42.088 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:41.497842+0000", 2023-06-03T20:00:42.088 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 1.0564000000000001e-05 2023-06-03T20:00:42.088 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:00:42.088 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:00:42.088 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "reached_pg", 2023-06-03T20:00:42.088 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:41.497900+0000", 2023-06-03T20:00:42.089 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 5.8072999999999998e-05 2023-06-03T20:00:42.089 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:00:42.089 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:00:42.089 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "started", 2023-06-03T20:00:42.089 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:41.498020+0000", 2023-06-03T20:00:42.089 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.000119166 2023-06-03T20:00:42.089 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:00:42.089 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:00:42.090 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "waiting for subops from 5,7", 2023-06-03T20:00:42.090 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:41.498172+0000", 2023-06-03T20:00:42.090 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.00015246199999999999 2023-06-03T20:00:42.090 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:00:42.090 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:00:42.090 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "op_commit", 2023-06-03T20:00:42.090 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:41.498995+0000", 2023-06-03T20:00:42.090 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.000823403 2023-06-03T20:00:42.091 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:00:42.091 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:00:42.091 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:00:42.091 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:41.499185+0000", 2023-06-03T20:00:42.091 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.000189283 2023-06-03T20:00:42.091 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:00:42.091 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:00:42.091 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:00:42.092 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:41.505949+0000", 2023-06-03T20:00:42.092 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.006764801 2023-06-03T20:00:42.092 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:00:42.092 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:00:42.092 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "commit_sent", 2023-06-03T20:00:42.092 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:41.506001+0000", 2023-06-03T20:00:42.092 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 5.2033999999999997e-05 2023-06-03T20:00:42.092 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:00:42.093 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:00:42.093 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "done", 2023-06-03T20:00:42.093 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:41.506044+0000", 2023-06-03T20:00:42.093 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 4.2826e-05 2023-06-03T20:00:42.093 INFO:teuthology.orchestra.run.smithi089.stdout: } 2023-06-03T20:00:42.093 INFO:teuthology.orchestra.run.smithi089.stdout: ] 2023-06-03T20:00:42.093 INFO:teuthology.orchestra.run.smithi089.stdout: } 2023-06-03T20:00:42.093 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:00:42.094 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:00:42.094 INFO:teuthology.orchestra.run.smithi089.stdout: "description": "osd_op(client.45713.0:1139 9.b 9:d7f89070:::benchmark_data_smithi089_115580_object1138:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e131)", 2023-06-03T20:00:42.094 INFO:teuthology.orchestra.run.smithi089.stdout: "initiated_at": "2023-06-03T20:00:41.529413+0000", 2023-06-03T20:00:42.094 INFO:teuthology.orchestra.run.smithi089.stdout: "age": 0.37365653900000001, 2023-06-03T20:00:42.094 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.0098144530000000008, 2023-06-03T20:00:42.094 INFO:teuthology.orchestra.run.smithi089.stdout: "type_data": { 2023-06-03T20:00:42.094 INFO:teuthology.orchestra.run.smithi089.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-03T20:00:42.095 INFO:teuthology.orchestra.run.smithi089.stdout: "client_info": { 2023-06-03T20:00:42.095 INFO:teuthology.orchestra.run.smithi089.stdout: "client": "client.45713", 2023-06-03T20:00:42.095 INFO:teuthology.orchestra.run.smithi089.stdout: "client_addr": "172.21.15.89:0/2400821448", 2023-06-03T20:00:42.095 INFO:teuthology.orchestra.run.smithi089.stdout: "tid": 1139 2023-06-03T20:00:42.095 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:00:42.095 INFO:teuthology.orchestra.run.smithi089.stdout: "events": [ 2023-06-03T20:00:42.095 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:00:42.095 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "initiated", 2023-06-03T20:00:42.096 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:41.529413+0000", 2023-06-03T20:00:42.096 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0 2023-06-03T20:00:42.096 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:00:42.096 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:00:42.096 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "throttled", 2023-06-03T20:00:42.096 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:41.529413+0000", 2023-06-03T20:00:42.096 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0 2023-06-03T20:00:42.096 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:00:42.097 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:00:42.097 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "header_read", 2023-06-03T20:00:42.097 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:41.529411+0000", 2023-06-03T20:00:42.097 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 4294967295.9999986 2023-06-03T20:00:42.097 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:00:42.097 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:00:42.098 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "all_read", 2023-06-03T20:00:42.098 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:41.529470+0000", 2023-06-03T20:00:42.098 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 5.9005000000000002e-05 2023-06-03T20:00:42.098 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:00:42.098 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:00:42.098 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "dispatched", 2023-06-03T20:00:42.098 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:41.529481+0000", 2023-06-03T20:00:42.098 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 1.0474e-05 2023-06-03T20:00:42.099 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:00:42.099 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:00:42.099 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "queued_for_pg", 2023-06-03T20:00:42.099 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:41.529489+0000", 2023-06-03T20:00:42.099 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 7.8509999999999999e-06 2023-06-03T20:00:42.099 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:00:42.099 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:00:42.100 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "reached_pg", 2023-06-03T20:00:42.100 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:41.529629+0000", 2023-06-03T20:00:42.100 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.000140431 2023-06-03T20:00:42.100 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:00:42.100 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:00:42.100 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "started", 2023-06-03T20:00:42.100 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:41.529773+0000", 2023-06-03T20:00:42.101 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.00014343099999999999 2023-06-03T20:00:42.101 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:00:42.101 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:00:42.101 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "waiting for subops from 5,6", 2023-06-03T20:00:42.101 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:41.529955+0000", 2023-06-03T20:00:42.101 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.000182509 2023-06-03T20:00:42.101 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:00:42.101 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:00:42.102 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:00:42.102 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:41.531372+0000", 2023-06-03T20:00:42.102 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.0014168119999999999 2023-06-03T20:00:42.102 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:00:42.102 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:00:42.102 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:00:42.103 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:41.533320+0000", 2023-06-03T20:00:42.103 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.0019478049999999999 2023-06-03T20:00:42.103 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:00:42.103 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:00:42.103 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "op_commit", 2023-06-03T20:00:42.103 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:41.535922+0000", 2023-06-03T20:00:42.103 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.0026017950000000001 2023-06-03T20:00:42.103 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:00:42.104 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:00:42.104 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "commit_sent", 2023-06-03T20:00:42.104 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:41.539169+0000", 2023-06-03T20:00:42.104 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.003247682 2023-06-03T20:00:42.104 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:00:42.104 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:00:42.104 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "done", 2023-06-03T20:00:42.104 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:41.539227+0000", 2023-06-03T20:00:42.105 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 5.8071e-05 2023-06-03T20:00:42.105 INFO:teuthology.orchestra.run.smithi089.stdout: } 2023-06-03T20:00:42.105 INFO:teuthology.orchestra.run.smithi089.stdout: ] 2023-06-03T20:00:42.105 INFO:teuthology.orchestra.run.smithi089.stdout: } 2023-06-03T20:00:42.105 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:00:42.105 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:00:42.105 INFO:teuthology.orchestra.run.smithi089.stdout: "description": "osd_op(client.45713.0:1236 9.5 9:ad1f1232:::benchmark_data_smithi089_115580_object1235:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e131)", 2023-06-03T20:00:42.106 INFO:teuthology.orchestra.run.smithi089.stdout: "initiated_at": "2023-06-03T20:00:41.565740+0000", 2023-06-03T20:00:42.106 INFO:teuthology.orchestra.run.smithi089.stdout: "age": 0.33732909900000002, 2023-06-03T20:00:42.106 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.0086082850000000002, 2023-06-03T20:00:42.106 INFO:teuthology.orchestra.run.smithi089.stdout: "type_data": { 2023-06-03T20:00:42.106 INFO:teuthology.orchestra.run.smithi089.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-03T20:00:42.106 INFO:teuthology.orchestra.run.smithi089.stdout: "client_info": { 2023-06-03T20:00:42.107 INFO:teuthology.orchestra.run.smithi089.stdout: "client": "client.45713", 2023-06-03T20:00:42.107 INFO:teuthology.orchestra.run.smithi089.stdout: "client_addr": "172.21.15.89:0/2400821448", 2023-06-03T20:00:42.107 INFO:teuthology.orchestra.run.smithi089.stdout: "tid": 1236 2023-06-03T20:00:42.107 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:00:42.107 INFO:teuthology.orchestra.run.smithi089.stdout: "events": [ 2023-06-03T20:00:42.107 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:00:42.107 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "initiated", 2023-06-03T20:00:42.108 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:41.565740+0000", 2023-06-03T20:00:42.108 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0 2023-06-03T20:00:42.108 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:00:42.108 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:00:42.108 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "throttled", 2023-06-03T20:00:42.108 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:41.565740+0000", 2023-06-03T20:00:42.108 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0 2023-06-03T20:00:42.108 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:00:42.109 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:00:42.109 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "header_read", 2023-06-03T20:00:42.109 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:41.565739+0000", 2023-06-03T20:00:42.109 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 4294967295.999999 2023-06-03T20:00:42.109 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:00:42.109 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:00:42.109 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "all_read", 2023-06-03T20:00:42.109 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:41.565758+0000", 2023-06-03T20:00:42.110 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 1.9210000000000001e-05 2023-06-03T20:00:42.110 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:00:42.110 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:00:42.110 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "dispatched", 2023-06-03T20:00:42.110 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:41.565771+0000", 2023-06-03T20:00:42.110 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 1.2281e-05 2023-06-03T20:00:42.110 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:00:42.110 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:00:42.111 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "queued_for_pg", 2023-06-03T20:00:42.111 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:41.565779+0000", 2023-06-03T20:00:42.111 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 8.5590000000000006e-06 2023-06-03T20:00:42.111 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:00:42.111 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:00:42.111 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "reached_pg", 2023-06-03T20:00:42.111 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:41.565903+0000", 2023-06-03T20:00:42.111 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.00012405599999999999 2023-06-03T20:00:42.112 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:00:42.112 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:00:42.112 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "started", 2023-06-03T20:00:42.112 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:41.566120+0000", 2023-06-03T20:00:42.112 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.00021694600000000001 2023-06-03T20:00:42.112 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:00:42.112 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:00:42.113 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "waiting for subops from 5,7", 2023-06-03T20:00:42.113 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:41.566272+0000", 2023-06-03T20:00:42.113 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.00015173500000000001 2023-06-03T20:00:42.113 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:00:42.113 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:00:42.113 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:00:42.113 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:41.573077+0000", 2023-06-03T20:00:42.113 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.0068053580000000001 2023-06-03T20:00:42.114 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:00:42.114 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:00:42.114 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:00:42.114 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:41.573098+0000", 2023-06-03T20:00:42.114 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 2.0973999999999999e-05 2023-06-03T20:00:42.114 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:00:42.114 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:00:42.114 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "op_commit", 2023-06-03T20:00:42.115 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:41.574237+0000", 2023-06-03T20:00:42.115 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.001139167 2023-06-03T20:00:42.115 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:00:42.115 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:00:42.115 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "commit_sent", 2023-06-03T20:00:42.115 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:41.574303+0000", 2023-06-03T20:00:42.115 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 6.5284999999999996e-05 2023-06-03T20:00:42.115 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:00:42.116 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:00:42.116 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "done", 2023-06-03T20:00:42.116 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:41.574349+0000", 2023-06-03T20:00:42.116 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 4.5881999999999999e-05 2023-06-03T20:00:42.116 INFO:teuthology.orchestra.run.smithi089.stdout: } 2023-06-03T20:00:42.116 INFO:teuthology.orchestra.run.smithi089.stdout: ] 2023-06-03T20:00:42.116 INFO:teuthology.orchestra.run.smithi089.stdout: } 2023-06-03T20:00:42.116 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:00:42.117 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:00:42.117 INFO:teuthology.orchestra.run.smithi089.stdout: "description": "osd_op(client.45713.0:1246 9.5 9:a45a3c3a:::benchmark_data_smithi089_115580_object1245:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e131)", 2023-06-03T20:00:42.117 INFO:teuthology.orchestra.run.smithi089.stdout: "initiated_at": "2023-06-03T20:00:41.569592+0000", 2023-06-03T20:00:42.117 INFO:teuthology.orchestra.run.smithi089.stdout: "age": 0.33347758500000002, 2023-06-03T20:00:42.117 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.0092182900000000005, 2023-06-03T20:00:42.117 INFO:teuthology.orchestra.run.smithi089.stdout: "type_data": { 2023-06-03T20:00:42.117 INFO:teuthology.orchestra.run.smithi089.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-03T20:00:42.117 INFO:teuthology.orchestra.run.smithi089.stdout: "client_info": { 2023-06-03T20:00:42.118 INFO:teuthology.orchestra.run.smithi089.stdout: "client": "client.45713", 2023-06-03T20:00:42.118 INFO:teuthology.orchestra.run.smithi089.stdout: "client_addr": "172.21.15.89:0/2400821448", 2023-06-03T20:00:42.118 INFO:teuthology.orchestra.run.smithi089.stdout: "tid": 1246 2023-06-03T20:00:42.118 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:00:42.118 INFO:teuthology.orchestra.run.smithi089.stdout: "events": [ 2023-06-03T20:00:42.118 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:00:42.118 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "initiated", 2023-06-03T20:00:42.118 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:41.569592+0000", 2023-06-03T20:00:42.119 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0 2023-06-03T20:00:42.119 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:00:42.119 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:00:42.119 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "throttled", 2023-06-03T20:00:42.119 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:41.569592+0000", 2023-06-03T20:00:42.119 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0 2023-06-03T20:00:42.119 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:00:42.120 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:00:42.120 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "header_read", 2023-06-03T20:00:42.120 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:41.569589+0000", 2023-06-03T20:00:42.120 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 4294967295.9999971 2023-06-03T20:00:42.120 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:00:42.120 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:00:42.120 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "all_read", 2023-06-03T20:00:42.120 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:41.569632+0000", 2023-06-03T20:00:42.121 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 4.3195000000000002e-05 2023-06-03T20:00:42.121 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:00:42.121 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:00:42.121 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "dispatched", 2023-06-03T20:00:42.121 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:41.569660+0000", 2023-06-03T20:00:42.121 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 2.7934000000000001e-05 2023-06-03T20:00:42.121 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:00:42.121 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:00:42.122 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "queued_for_pg", 2023-06-03T20:00:42.122 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:41.569675+0000", 2023-06-03T20:00:42.122 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 1.447e-05 2023-06-03T20:00:42.122 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:00:42.122 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:00:42.122 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "reached_pg", 2023-06-03T20:00:42.122 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:41.573114+0000", 2023-06-03T20:00:42.122 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.0034390890000000002 2023-06-03T20:00:42.123 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:00:42.123 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:00:42.123 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "started", 2023-06-03T20:00:42.123 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:41.573258+0000", 2023-06-03T20:00:42.123 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.00014410399999999999 2023-06-03T20:00:42.123 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:00:42.123 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:00:42.124 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "waiting for subops from 5,7", 2023-06-03T20:00:42.124 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:41.573413+0000", 2023-06-03T20:00:42.124 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.00015565800000000001 2023-06-03T20:00:42.124 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:00:42.124 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:00:42.124 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "op_commit", 2023-06-03T20:00:42.124 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:41.574360+0000", 2023-06-03T20:00:42.124 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.00094649000000000003 2023-06-03T20:00:42.125 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:00:42.125 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:00:42.125 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:00:42.125 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:41.574896+0000", 2023-06-03T20:00:42.125 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.00053573599999999998 2023-06-03T20:00:42.125 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:00:42.125 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:00:42.125 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:00:42.126 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:41.578710+0000", 2023-06-03T20:00:42.126 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.0038148430000000001 2023-06-03T20:00:42.126 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:00:42.126 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:00:42.126 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "commit_sent", 2023-06-03T20:00:42.126 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:41.578766+0000", 2023-06-03T20:00:42.126 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 5.5272000000000002e-05 2023-06-03T20:00:42.126 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:00:42.127 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:00:42.127 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "done", 2023-06-03T20:00:42.127 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:41.578810+0000", 2023-06-03T20:00:42.127 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 4.4357999999999998e-05 2023-06-03T20:00:42.127 INFO:teuthology.orchestra.run.smithi089.stdout: } 2023-06-03T20:00:42.127 INFO:teuthology.orchestra.run.smithi089.stdout: ] 2023-06-03T20:00:42.127 INFO:teuthology.orchestra.run.smithi089.stdout: } 2023-06-03T20:00:42.127 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:00:42.128 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:00:42.128 INFO:teuthology.orchestra.run.smithi089.stdout: "description": "osd_op(client.45713.0:1757 9.5 9:a9f210f2:::benchmark_data_smithi089_115580_object1756:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e131)", 2023-06-03T20:00:42.128 INFO:teuthology.orchestra.run.smithi089.stdout: "initiated_at": "2023-06-03T20:00:41.768562+0000", 2023-06-03T20:00:42.128 INFO:teuthology.orchestra.run.smithi089.stdout: "age": 0.13450725499999999, 2023-06-03T20:00:42.128 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.0079746550000000006, 2023-06-03T20:00:42.128 INFO:teuthology.orchestra.run.smithi089.stdout: "type_data": { 2023-06-03T20:00:42.128 INFO:teuthology.orchestra.run.smithi089.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-03T20:00:42.128 INFO:teuthology.orchestra.run.smithi089.stdout: "client_info": { 2023-06-03T20:00:42.129 INFO:teuthology.orchestra.run.smithi089.stdout: "client": "client.45713", 2023-06-03T20:00:42.129 INFO:teuthology.orchestra.run.smithi089.stdout: "client_addr": "172.21.15.89:0/2400821448", 2023-06-03T20:00:42.129 INFO:teuthology.orchestra.run.smithi089.stdout: "tid": 1757 2023-06-03T20:00:42.129 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:00:42.129 INFO:teuthology.orchestra.run.smithi089.stdout: "events": [ 2023-06-03T20:00:42.129 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:00:42.129 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "initiated", 2023-06-03T20:00:42.130 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:41.768562+0000", 2023-06-03T20:00:42.130 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0 2023-06-03T20:00:42.130 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:00:42.130 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:00:42.130 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "throttled", 2023-06-03T20:00:42.130 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:41.768562+0000", 2023-06-03T20:00:42.130 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0 2023-06-03T20:00:42.130 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:00:42.131 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:00:42.131 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "header_read", 2023-06-03T20:00:42.131 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:41.768561+0000", 2023-06-03T20:00:42.131 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 4294967295.9999986 2023-06-03T20:00:42.131 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:00:42.131 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:00:42.131 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "all_read", 2023-06-03T20:00:42.131 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:41.768585+0000", 2023-06-03T20:00:42.132 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 2.3828999999999998e-05 2023-06-03T20:00:42.132 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:00:42.132 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:00:42.132 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "dispatched", 2023-06-03T20:00:42.132 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:41.768598+0000", 2023-06-03T20:00:42.132 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 1.3183000000000001e-05 2023-06-03T20:00:42.132 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:00:42.132 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:00:42.133 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "queued_for_pg", 2023-06-03T20:00:42.133 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:41.768606+0000", 2023-06-03T20:00:42.133 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 8.1249999999999993e-06 2023-06-03T20:00:42.133 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:00:42.133 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:00:42.133 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "reached_pg", 2023-06-03T20:00:42.133 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:41.768648+0000", 2023-06-03T20:00:42.133 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 4.2552999999999997e-05 2023-06-03T20:00:42.134 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:00:42.134 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:00:42.134 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "started", 2023-06-03T20:00:42.134 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:41.768798+0000", 2023-06-03T20:00:42.134 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.00014918599999999999 2023-06-03T20:00:42.134 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:00:42.134 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:00:42.134 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "waiting for subops from 5,7", 2023-06-03T20:00:42.135 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:41.768895+0000", 2023-06-03T20:00:42.135 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 9.7207999999999995e-05 2023-06-03T20:00:42.135 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:00:42.135 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:00:42.135 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "op_commit", 2023-06-03T20:00:42.135 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:41.769417+0000", 2023-06-03T20:00:42.135 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.00052260900000000005 2023-06-03T20:00:42.135 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:00:42.136 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:00:42.136 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:00:42.136 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:41.769796+0000", 2023-06-03T20:00:42.136 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.00037868299999999999 2023-06-03T20:00:42.136 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:00:42.136 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:00:42.136 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:00:42.136 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:41.776418+0000", 2023-06-03T20:00:42.137 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.0066222859999999998 2023-06-03T20:00:42.137 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:00:42.137 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:00:42.137 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "commit_sent", 2023-06-03T20:00:42.137 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:41.776486+0000", 2023-06-03T20:00:42.137 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 6.7642000000000001e-05 2023-06-03T20:00:42.137 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:00:42.137 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:00:42.138 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "done", 2023-06-03T20:00:42.138 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:41.776537+0000", 2023-06-03T20:00:42.138 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 5.079e-05 2023-06-03T20:00:42.138 INFO:teuthology.orchestra.run.smithi089.stdout: } 2023-06-03T20:00:42.138 INFO:teuthology.orchestra.run.smithi089.stdout: ] 2023-06-03T20:00:42.138 INFO:teuthology.orchestra.run.smithi089.stdout: } 2023-06-03T20:00:42.138 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:00:42.138 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:00:42.139 INFO:teuthology.orchestra.run.smithi089.stdout: "description": "osd_op(client.45713.0:1980 9.9 9:906d42fd:::benchmark_data_smithi089_115580_object1979:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e131)", 2023-06-03T20:00:42.139 INFO:teuthology.orchestra.run.smithi089.stdout: "initiated_at": "2023-06-03T20:00:41.848638+0000", 2023-06-03T20:00:42.139 INFO:teuthology.orchestra.run.smithi089.stdout: "age": 0.054431269999999997, 2023-06-03T20:00:42.139 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.0092414490000000005, 2023-06-03T20:00:42.139 INFO:teuthology.orchestra.run.smithi089.stdout: "type_data": { 2023-06-03T20:00:42.139 INFO:teuthology.orchestra.run.smithi089.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-03T20:00:42.139 INFO:teuthology.orchestra.run.smithi089.stdout: "client_info": { 2023-06-03T20:00:42.139 INFO:teuthology.orchestra.run.smithi089.stdout: "client": "client.45713", 2023-06-03T20:00:42.140 INFO:teuthology.orchestra.run.smithi089.stdout: "client_addr": "172.21.15.89:0/2400821448", 2023-06-03T20:00:42.140 INFO:teuthology.orchestra.run.smithi089.stdout: "tid": 1980 2023-06-03T20:00:42.140 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:00:42.140 INFO:teuthology.orchestra.run.smithi089.stdout: "events": [ 2023-06-03T20:00:42.140 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:00:42.140 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "initiated", 2023-06-03T20:00:42.140 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:41.848638+0000", 2023-06-03T20:00:42.140 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0 2023-06-03T20:00:42.141 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:00:42.141 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:00:42.141 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "throttled", 2023-06-03T20:00:42.141 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:41.848638+0000", 2023-06-03T20:00:42.141 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0 2023-06-03T20:00:42.141 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:00:42.141 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:00:42.141 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "header_read", 2023-06-03T20:00:42.142 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:41.848637+0000", 2023-06-03T20:00:42.142 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 4294967295.9999986 2023-06-03T20:00:42.142 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:00:42.142 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:00:42.142 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "all_read", 2023-06-03T20:00:42.142 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:41.848665+0000", 2023-06-03T20:00:42.142 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 2.7684000000000001e-05 2023-06-03T20:00:42.142 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:00:42.143 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:00:42.143 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "dispatched", 2023-06-03T20:00:42.143 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:41.848681+0000", 2023-06-03T20:00:42.143 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 1.6470999999999999e-05 2023-06-03T20:00:42.143 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:00:42.143 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:00:42.143 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "queued_for_pg", 2023-06-03T20:00:42.143 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:41.848691+0000", 2023-06-03T20:00:42.144 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 1.0012e-05 2023-06-03T20:00:42.144 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:00:42.144 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:00:42.144 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "reached_pg", 2023-06-03T20:00:42.144 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:41.848780+0000", 2023-06-03T20:00:42.144 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 8.8659999999999995e-05 2023-06-03T20:00:42.144 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:00:42.145 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:00:42.145 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "started", 2023-06-03T20:00:42.145 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:41.849092+0000", 2023-06-03T20:00:42.145 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.00031275699999999999 2023-06-03T20:00:42.145 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:00:42.145 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:00:42.145 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "waiting for subops from 5,7", 2023-06-03T20:00:42.145 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:41.849261+0000", 2023-06-03T20:00:42.146 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.000168622 2023-06-03T20:00:42.146 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:00:42.146 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:00:42.146 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "op_commit", 2023-06-03T20:00:42.146 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:41.849965+0000", 2023-06-03T20:00:42.146 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.00070419600000000003 2023-06-03T20:00:42.146 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:00:42.146 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:00:42.147 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:00:42.147 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:41.850422+0000", 2023-06-03T20:00:42.147 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.00045638500000000002 2023-06-03T20:00:42.147 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:00:42.147 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:00:42.147 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:00:42.147 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:41.857781+0000", 2023-06-03T20:00:42.147 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.0073598350000000003 2023-06-03T20:00:42.148 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:00:42.148 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:00:42.148 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "commit_sent", 2023-06-03T20:00:42.148 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:41.857837+0000", 2023-06-03T20:00:42.148 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 5.5169e-05 2023-06-03T20:00:42.148 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:00:42.148 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:00:42.148 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "done", 2023-06-03T20:00:42.149 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:41.857880+0000", 2023-06-03T20:00:42.149 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 4.2923000000000002e-05 2023-06-03T20:00:42.149 INFO:teuthology.orchestra.run.smithi089.stdout: } 2023-06-03T20:00:42.149 INFO:teuthology.orchestra.run.smithi089.stdout: ] 2023-06-03T20:00:42.149 INFO:teuthology.orchestra.run.smithi089.stdout: } 2023-06-03T20:00:42.149 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:00:42.149 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:00:42.149 INFO:teuthology.orchestra.run.smithi089.stdout: "description": "osd_op(client.45713.0:2007 9.b 9:db0aeae2:::benchmark_data_smithi089_115580_object2006:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e131)", 2023-06-03T20:00:42.150 INFO:teuthology.orchestra.run.smithi089.stdout: "initiated_at": "2023-06-03T20:00:41.858847+0000", 2023-06-03T20:00:42.150 INFO:teuthology.orchestra.run.smithi089.stdout: "age": 0.044222194999999999, 2023-06-03T20:00:42.150 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.0089400499999999997, 2023-06-03T20:00:42.150 INFO:teuthology.orchestra.run.smithi089.stdout: "type_data": { 2023-06-03T20:00:42.150 INFO:teuthology.orchestra.run.smithi089.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-03T20:00:42.150 INFO:teuthology.orchestra.run.smithi089.stdout: "client_info": { 2023-06-03T20:00:42.150 INFO:teuthology.orchestra.run.smithi089.stdout: "client": "client.45713", 2023-06-03T20:00:42.150 INFO:teuthology.orchestra.run.smithi089.stdout: "client_addr": "172.21.15.89:0/2400821448", 2023-06-03T20:00:42.151 INFO:teuthology.orchestra.run.smithi089.stdout: "tid": 2007 2023-06-03T20:00:42.151 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:00:42.151 INFO:teuthology.orchestra.run.smithi089.stdout: "events": [ 2023-06-03T20:00:42.151 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:00:42.151 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "initiated", 2023-06-03T20:00:42.151 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:41.858847+0000", 2023-06-03T20:00:42.151 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0 2023-06-03T20:00:42.152 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:00:42.152 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:00:42.152 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "throttled", 2023-06-03T20:00:42.152 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:41.858847+0000", 2023-06-03T20:00:42.152 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0 2023-06-03T20:00:42.152 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:00:42.152 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:00:42.152 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "header_read", 2023-06-03T20:00:42.153 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:41.858846+0000", 2023-06-03T20:00:42.153 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 4294967295.999999 2023-06-03T20:00:42.153 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:00:42.153 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:00:42.153 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "all_read", 2023-06-03T20:00:42.153 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:41.858876+0000", 2023-06-03T20:00:42.153 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 2.9604999999999999e-05 2023-06-03T20:00:42.153 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:00:42.154 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:00:42.154 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "dispatched", 2023-06-03T20:00:42.154 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:41.858888+0000", 2023-06-03T20:00:42.154 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 1.2181000000000001e-05 2023-06-03T20:00:42.154 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:00:42.154 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:00:42.154 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "queued_for_pg", 2023-06-03T20:00:42.154 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:41.858894+0000", 2023-06-03T20:00:42.155 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 5.7729999999999996e-06 2023-06-03T20:00:42.155 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:00:42.155 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:00:42.155 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "reached_pg", 2023-06-03T20:00:42.155 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:41.863647+0000", 2023-06-03T20:00:42.155 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.0047531869999999999 2023-06-03T20:00:42.155 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:00:42.155 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:00:42.156 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "started", 2023-06-03T20:00:42.156 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:41.863747+0000", 2023-06-03T20:00:42.156 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.000100006 2023-06-03T20:00:42.156 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:00:42.156 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:00:42.156 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "waiting for subops from 5,6", 2023-06-03T20:00:42.156 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:41.863850+0000", 2023-06-03T20:00:42.156 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.000102524 2023-06-03T20:00:42.157 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:00:42.157 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:00:42.157 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:00:42.157 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:41.866067+0000", 2023-06-03T20:00:42.157 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.0022178050000000002 2023-06-03T20:00:42.157 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:00:42.157 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:00:42.157 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:00:42.158 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:41.866935+0000", 2023-06-03T20:00:42.158 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.00086737200000000004 2023-06-03T20:00:42.158 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:00:42.158 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:00:42.158 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "op_commit", 2023-06-03T20:00:42.158 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:41.867739+0000", 2023-06-03T20:00:42.158 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.00080392699999999996 2023-06-03T20:00:42.158 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:00:42.159 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:00:42.159 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "commit_sent", 2023-06-03T20:00:42.159 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:41.867765+0000", 2023-06-03T20:00:42.159 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 2.633e-05 2023-06-03T20:00:42.159 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:00:42.159 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:00:42.159 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "done", 2023-06-03T20:00:42.159 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:41.867787+0000", 2023-06-03T20:00:42.160 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 2.2191000000000001e-05 2023-06-03T20:00:42.160 INFO:teuthology.orchestra.run.smithi089.stdout: } 2023-06-03T20:00:42.160 INFO:teuthology.orchestra.run.smithi089.stdout: ] 2023-06-03T20:00:42.160 INFO:teuthology.orchestra.run.smithi089.stdout: } 2023-06-03T20:00:42.160 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:00:42.160 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:00:42.160 INFO:teuthology.orchestra.run.smithi089.stdout: "description": "osd_op(client.45713.0:2015 9.b 9:dd36373e:::benchmark_data_smithi089_115580_object2014:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e131)", 2023-06-03T20:00:42.160 INFO:teuthology.orchestra.run.smithi089.stdout: "initiated_at": "2023-06-03T20:00:41.861172+0000", 2023-06-03T20:00:42.161 INFO:teuthology.orchestra.run.smithi089.stdout: "age": 0.041897665000000001, 2023-06-03T20:00:42.161 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.0083691389999999994, 2023-06-03T20:00:42.161 INFO:teuthology.orchestra.run.smithi089.stdout: "type_data": { 2023-06-03T20:00:42.161 INFO:teuthology.orchestra.run.smithi089.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-03T20:00:42.161 INFO:teuthology.orchestra.run.smithi089.stdout: "client_info": { 2023-06-03T20:00:42.161 INFO:teuthology.orchestra.run.smithi089.stdout: "client": "client.45713", 2023-06-03T20:00:42.161 INFO:teuthology.orchestra.run.smithi089.stdout: "client_addr": "172.21.15.89:0/2400821448", 2023-06-03T20:00:42.161 INFO:teuthology.orchestra.run.smithi089.stdout: "tid": 2015 2023-06-03T20:00:42.162 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:00:42.162 INFO:teuthology.orchestra.run.smithi089.stdout: "events": [ 2023-06-03T20:00:42.162 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:00:42.162 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "initiated", 2023-06-03T20:00:42.162 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:41.861172+0000", 2023-06-03T20:00:42.162 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0 2023-06-03T20:00:42.162 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:00:42.162 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:00:42.163 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "throttled", 2023-06-03T20:00:42.163 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:41.861172+0000", 2023-06-03T20:00:42.163 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0 2023-06-03T20:00:42.163 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:00:42.163 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:00:42.163 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "header_read", 2023-06-03T20:00:42.163 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:41.861170+0000", 2023-06-03T20:00:42.163 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 4294967295.9999986 2023-06-03T20:00:42.164 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:00:42.164 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:00:42.164 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "all_read", 2023-06-03T20:00:42.164 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:41.861194+0000", 2023-06-03T20:00:42.164 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 2.3632999999999999e-05 2023-06-03T20:00:42.164 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:00:42.164 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:00:42.164 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "dispatched", 2023-06-03T20:00:42.165 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:41.861209+0000", 2023-06-03T20:00:42.165 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 1.5111e-05 2023-06-03T20:00:42.165 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:00:42.165 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:00:42.165 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "queued_for_pg", 2023-06-03T20:00:42.165 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:41.861216+0000", 2023-06-03T20:00:42.165 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 7.0260000000000002e-06 2023-06-03T20:00:42.165 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:00:42.166 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:00:42.166 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "reached_pg", 2023-06-03T20:00:42.166 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:41.864860+0000", 2023-06-03T20:00:42.166 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.003644301 2023-06-03T20:00:42.166 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:00:42.166 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:00:42.166 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "started", 2023-06-03T20:00:42.166 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:41.865011+0000", 2023-06-03T20:00:42.167 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.00015113999999999999 2023-06-03T20:00:42.167 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:00:42.167 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:00:42.167 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "waiting for subops from 5,6", 2023-06-03T20:00:42.167 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:41.865162+0000", 2023-06-03T20:00:42.167 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.00015082600000000001 2023-06-03T20:00:42.167 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:00:42.167 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:00:42.168 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:00:42.168 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:41.867264+0000", 2023-06-03T20:00:42.168 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.002101814 2023-06-03T20:00:42.168 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:00:42.168 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:00:42.168 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "op_commit", 2023-06-03T20:00:42.168 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:41.867851+0000", 2023-06-03T20:00:42.169 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.00058735000000000005 2023-06-03T20:00:42.169 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:00:42.169 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:00:42.169 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:00:42.169 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:41.869467+0000", 2023-06-03T20:00:42.169 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.001615881 2023-06-03T20:00:42.169 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:00:42.169 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:00:42.170 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "commit_sent", 2023-06-03T20:00:42.170 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:41.869507+0000", 2023-06-03T20:00:42.170 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 3.9560000000000001e-05 2023-06-03T20:00:42.170 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:00:42.170 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:00:42.170 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "done", 2023-06-03T20:00:42.170 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:41.869541+0000", 2023-06-03T20:00:42.170 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 3.4109999999999997e-05 2023-06-03T20:00:42.171 INFO:teuthology.orchestra.run.smithi089.stdout: } 2023-06-03T20:00:42.171 INFO:teuthology.orchestra.run.smithi089.stdout: ] 2023-06-03T20:00:42.171 INFO:teuthology.orchestra.run.smithi089.stdout: } 2023-06-03T20:00:42.171 INFO:teuthology.orchestra.run.smithi089.stdout: } 2023-06-03T20:00:42.171 INFO:teuthology.orchestra.run.smithi089.stdout: ] 2023-06-03T20:00:42.171 INFO:teuthology.orchestra.run.smithi089.stdout:} 2023-06-03T20:00:42.171 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:00:42.172 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph osd unset noscrub 2023-06-03T20:00:42.176 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 1 16 2939 2923 182.676 182.688 0.00495668 0.00545506 2023-06-03T20:00:42.201 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:00:42.206 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:00:42.206 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:00:42.307 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:00:42.338 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:00:42.342 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:00:42.343 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:00:42.444 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:00:42.486 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:00:42.488 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:00:42.489 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:00:42.590 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:00:42.623 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:00:42.626 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:00:42.627 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:00:42.727 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:00:42.765 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:00:42.767 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:00:42.768 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:00:42.869 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:00:42.898 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:00:42.901 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:00:42.902 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:00:43.003 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:00:43.031 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:00:43.034 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:00:43.035 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:00:43.136 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:00:43.178 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 2 16 4802 4786 149.398 116.438 0.00482152 0.00668195 2023-06-03T20:00:43.180 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:00:43.184 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:00:43.185 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:00:43.285 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:00:43.322 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:00:43.323 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:00:43.324 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:00:43.424 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:00:43.454 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:00:43.457 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:00:43.458 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:00:43.559 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:00:43.596 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:00:43.600 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:00:43.601 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:00:43.702 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:00:43.731 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:00:43 smithi089 bash[75068]: cluster 1685822442.4330025 mgr.y (mgr.34104) 732 : cluster 0 pgmap v642: 89 pgs: 89 active+clean; 5.4 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 561 KiB/s wr, 8 op/s 2023-06-03T20:00:43.732 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph daemon osd.1 dump_ops_in_flight 2023-06-03T20:00:43.733 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:00:43 smithi089 bash[78452]: cluster 1685822442.4330025 mgr.y (mgr.34104) 732 : cluster 0 pgmap v642: 89 pgs: 89 active+clean; 5.4 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 561 KiB/s wr, 8 op/s 2023-06-03T20:00:43.733 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:00:43.736 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:00:43.737 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:00:43.838 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:00:43.847 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:00:43 smithi164 bash[59462]: cluster 1685822442.4330025 mgr.y (mgr.34104) 732 : cluster 0 pgmap v642: 89 pgs: 89 active+clean; 5.4 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 561 KiB/s wr, 8 op/s 2023-06-03T20:00:43.869 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:00:43.871 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:00:43.872 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:00:43.973 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:00:44.004 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:00:44.006 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:00:44.007 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:00:44.108 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:00:44.149 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:00:44.153 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:00:44.153 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:00:44.178 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 3 16 7448 7432 154.715 165.375 0.00462507 0.00645247 2023-06-03T20:00:44.228 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/mon.a/config 2023-06-03T20:00:44.254 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:00:44.287 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:00:44.290 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:00:44.291 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:00:44.392 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:00:44.422 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:00:44.424 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:00:44.425 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:00:44.526 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:00:44.563 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:00:44.566 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:00:44.566 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:00:44.667 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:00:44.700 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:00:44.703 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:00:44.704 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:00:44.805 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:00:44.841 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:00:44.844 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:00:44.844 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:00:44.945 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:00:44.978 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:00:44.981 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:00:44.982 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:00:45.083 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:00:45.116 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:00:45.118 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:00:45.119 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:00:45.178 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 4 16 9057 9041 141.181 100.562 0.00498249 0.00707393 2023-06-03T20:00:45.220 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:00:45.254 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:00:45.257 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:00:45.258 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:00:45.359 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:00:45.396 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:00:45.398 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:00:45.399 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:00:45.500 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:00:45.533 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:00:45.537 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:00:45.538 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:00:45.639 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:00:45.672 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:00:45.675 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:00:45.676 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:00:45.777 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:00:45.779 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:00:45 smithi089 bash[75068]: cluster 1685822444.4336517 mgr.y (mgr.34104) 733 : cluster 0 pgmap v643: 89 pgs: 89 active+clean; 234 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 23 MiB/s wr, 373 op/s 2023-06-03T20:00:45.779 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:00:45 smithi089 bash[78452]: cluster 1685822444.4336517 mgr.y (mgr.34104) 733 : cluster 0 pgmap v643: 89 pgs: 89 active+clean; 234 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 23 MiB/s wr, 373 op/s 2023-06-03T20:00:45.786 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:00:45 smithi164 bash[59462]: cluster 1685822444.4336517 mgr.y (mgr.34104) 733 : cluster 0 pgmap v643: 89 pgs: 89 active+clean; 234 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 23 MiB/s wr, 373 op/s 2023-06-03T20:00:45.813 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:00:45.816 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:00:45.817 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:00:45.917 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:00:45.949 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:00:45.951 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:00:45.952 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:00:46.053 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:00:46.084 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:00:46.087 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:00:46.088 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:00:46.179 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 5 16 11559 11543 144.216 156.375 0.00264894 0.00692426 2023-06-03T20:00:46.188 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:00:46.218 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:00:46.222 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:00:46.222 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:00:46.232 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 20:00:46 smithi089 bash[71163]: ::ffff:172.21.15.164 - - [03/Jun/2023:20:00:46] "GET /metrics HTTP/1.1" 200 227680 "" "Prometheus/2.33.4" 2023-06-03T20:00:46.291 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/mon.a/config 2023-06-03T20:00:46.323 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:00:46.367 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:00:46.370 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:00:46.371 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:00:46.472 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:00:46.508 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:00:46.511 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:00:46.511 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:00:46.612 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:00:46.647 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:00:46.650 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:00:46.650 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:00:46.751 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:00:46.785 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:00:46.788 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:00:46.788 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:00:46.889 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:00:46.924 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:00:46.927 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:00:46.927 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:00:47.028 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:00:47.063 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:00:47.067 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:00:47.067 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:00:47.168 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:00:47.178 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 6 16 13613 13597 141.574 128.375 0.00169965 0.00700941 2023-06-03T20:00:47.198 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:00:47.200 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:00:47.201 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:00:47.302 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:00:47.337 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:00:47.342 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:00:47.343 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:00:47.444 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:00:47.480 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:00:47.483 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:00:47.484 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:00:47.585 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:00:47.600 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:00:47 smithi089 bash[75068]: cluster 1685822446.4341989 mgr.y (mgr.34104) 734 : cluster 0 pgmap v644: 89 pgs: 89 active+clean; 368 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 37 MiB/s wr, 587 op/s 2023-06-03T20:00:47.620 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:00:47.623 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:00:47.623 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:00:47.724 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:00:47.753 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:00:47.755 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:00:47.756 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:00:47.834 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:00:47 smithi164 bash[59462]: cluster 1685822446.4341989 mgr.y (mgr.34104) 734 : cluster 0 pgmap v644: 89 pgs: 89 active+clean; 368 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 37 MiB/s wr, 587 op/s 2023-06-03T20:00:47.856 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:00:47.865 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:00:47 smithi089 bash[78452]: cluster 1685822446.4341989 mgr.y (mgr.34104) 734 : cluster 0 pgmap v644: 89 pgs: 89 active+clean; 368 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 37 MiB/s wr, 587 op/s 2023-06-03T20:00:47.889 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:00:47.891 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:00:47.892 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:00:47.993 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:00:48.032 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:00:48.034 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:00:48.035 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:00:48.136 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:00:48.168 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:00:48.174 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:00:48.175 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:00:48.179 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 7 16 15840 15824 141.231 139.188 0.00487047 0.00707311 2023-06-03T20:00:48.276 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:00:48.313 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:00:48.318 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:00:48.319 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:00:48.346 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/mon.a/config 2023-06-03T20:00:48.346 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/mon.a/config 2023-06-03T20:00:48.420 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:00:48.452 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:00:48.454 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:00:48.455 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:00:48.556 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:00:48.585 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:00:48.587 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:00:48.587 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:00:48.688 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:00:48.725 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:00:48.727 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:00:48.728 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:00:48.829 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:00:48.859 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:00:48.861 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:00:48.862 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:00:48.963 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:00:49.000 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:00:49.006 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:00:49.007 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:00:49.108 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:00:49.143 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:00:49.146 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:00:49.147 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:00:49.179 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 8 16 17922 17906 139.842 130.125 0.0054512 0.00714459 2023-06-03T20:00:49.248 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:00:49.279 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:00:49.282 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:00:49.283 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:00:49.384 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:00:49.415 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:00:49.419 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:00:49.419 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:00:49.520 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:00:49.553 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:00:49.558 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:00:49.559 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:00:49.659 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:00:49.687 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:00:49.688 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:00:49.689 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:00:49.728 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:00:49 smithi089 bash[75068]: cluster 1685822448.4348335 mgr.y (mgr.34104) 735 : cluster 0 pgmap v645: 89 pgs: 89 active+clean; 908 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 84 MiB/s wr, 1.34k op/s 2023-06-03T20:00:49.790 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:00:49.799 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:00:49 smithi164 bash[59462]: cluster 1685822448.4348335 mgr.y (mgr.34104) 735 : cluster 0 pgmap v645: 89 pgs: 89 active+clean; 908 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 84 MiB/s wr, 1.34k op/s 2023-06-03T20:00:49.824 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:00:49.827 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:00:49.827 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:00:49.928 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:00:49.956 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:00:49.958 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:00:49.960 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:00:50.029 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:00:49 smithi089 bash[78452]: cluster 1685822448.4348335 mgr.y (mgr.34104) 735 : cluster 0 pgmap v645: 89 pgs: 89 active+clean; 908 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 84 MiB/s wr, 1.34k op/s 2023-06-03T20:00:50.061 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:00:50.102 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:00:50.106 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:00:50.107 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:00:50.179 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 9 16 20136 20120 139.678 138.375 0.0033311 0.00715291 2023-06-03T20:00:50.208 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:00:50.245 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:00:50.247 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:00:50.248 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:00:50.349 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:00:50.384 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:00:50.387 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:00:50.387 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:00:50.489 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:00:50.513 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:00:50.515 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:00:50.516 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:00:50.617 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:00:50.654 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:00:50.657 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:00:50.658 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:00:50.759 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:00:50.800 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:00:50.802 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:00:50.803 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:00:50.842 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:00:50 smithi164 bash[59462]: audit 1685822450.2833903 mon.c (mon.2) 383 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-03T20:00:50.903 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:00:50.947 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:00:50.951 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:00:50.951 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:00:51.029 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:00:50 smithi089 bash[78452]: audit 1685822450.2833903 mon.c (mon.2) 383 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-03T20:00:51.029 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:00:50 smithi089 bash[75068]: audit 1685822450.2833903 mon.c (mon.2) 383 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-03T20:00:51.052 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:00:51.090 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:00:51.093 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:00:51.093 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:00:51.179 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 10 16 21588 21572 134.784 90.75 0.00187419 0.00737498 2023-06-03T20:00:51.195 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:00:51.224 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:00:51.226 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:00:51.226 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:00:51.327 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:00:51.366 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:00:51.369 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:00:51.370 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:00:51.471 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:00:51.502 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:00:51.506 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:00:51.507 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:00:51.608 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:00:51.615 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:00:51 smithi089 bash[75068]: cluster 1685822450.4351926 mgr.y (mgr.34104) 736 : cluster 0 pgmap v646: 89 pgs: 89 active+clean; 908 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 76 MiB/s wr, 1.21k op/s 2023-06-03T20:00:51.615 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:00:51 smithi089 bash[78452]: cluster 1685822450.4351926 mgr.y (mgr.34104) 736 : cluster 0 pgmap v646: 89 pgs: 89 active+clean; 908 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 76 MiB/s wr, 1.21k op/s 2023-06-03T20:00:51.640 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:00:51.641 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:00:51.642 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:00:51.743 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:00:51.752 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:00:51 smithi164 bash[59462]: cluster 1685822450.4351926 mgr.y (mgr.34104) 736 : cluster 0 pgmap v646: 89 pgs: 89 active+clean; 908 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 76 MiB/s wr, 1.21k op/s 2023-06-03T20:00:51.776 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:00:51.779 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:00:51.779 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:00:51.880 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:00:51.915 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:00:51.916 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:00:51.917 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:00:52.018 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:00:52.054 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:00:52.056 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:00:52.057 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:00:52.158 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:00:52.179 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 11 16 23110 23094 131.179 95.125 0.00172626 0.00757629 2023-06-03T20:00:52.192 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:00:52.194 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:00:52.195 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:00:52.296 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:00:52.331 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:00:52.336 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:00:52.337 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:00:52.438 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:00:52.470 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:00:52.473 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:00:52.473 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:00:52.574 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:00:52.604 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:00:52.607 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:00:52.608 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:00:52.708 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:00:52.741 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:00:52.744 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:00:52.745 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:00:52.778 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:00:52 smithi089 bash[78452]: audit 1685822452.0944097 mon.c (mon.2) 384 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T20:00:52.779 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:00:52 smithi089 bash[75068]: audit 1685822452.0944097 mon.c (mon.2) 384 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T20:00:52.846 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:00:52.875 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:00:52.881 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:00:52 smithi164 bash[59462]: audit 1685822452.0944097 mon.c (mon.2) 384 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T20:00:52.882 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:00:52.882 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:00:52.983 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:00:53.017 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:00:53.019 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:00:53.020 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:00:53.120 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:00:53.152 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:00:53.154 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:00:53.155 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:00:53.179 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 12 16 25057 25041 130.387 121.688 0.00940161 0.00766163 2023-06-03T20:00:53.256 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:00:53.290 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:00:53.292 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:00:53.292 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:00:53.393 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:00:53.425 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:00:53.430 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:00:53.431 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:00:53.532 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:00:53.564 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:00:53.566 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:00:53.567 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:00:53.668 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:00:53.699 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:00:53.702 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:00:53.703 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:00:53.804 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:00:53.813 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:00:53 smithi164 bash[59462]: cluster 1685822452.4356892 mgr.y (mgr.34104) 737 : cluster 0 pgmap v647: 89 pgs: 89 active+clean; 1.0 GiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 85 MiB/s wr, 1.36k op/s 2023-06-03T20:00:53.837 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:00:53.841 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:00:53.842 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:00:53.943 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:00:53.978 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:00:53.981 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:00:53.982 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:00:54.028 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:00:53 smithi089 bash[78452]: cluster 1685822452.4356892 mgr.y (mgr.34104) 737 : cluster 0 pgmap v647: 89 pgs: 89 active+clean; 1.0 GiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 85 MiB/s wr, 1.36k op/s 2023-06-03T20:00:54.029 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:00:53 smithi089 bash[75068]: cluster 1685822452.4356892 mgr.y (mgr.34104) 737 : cluster 0 pgmap v647: 89 pgs: 89 active+clean; 1.0 GiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 85 MiB/s wr, 1.36k op/s 2023-06-03T20:00:54.082 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:00:54.117 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:00:54.122 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:00:54.122 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:00:54.179 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 13 16 26560 26544 127.583 93.9375 0.00460725 0.00783269 2023-06-03T20:00:54.223 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:00:54.253 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:00:54.259 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:00:54.260 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:00:54.361 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:00:54.394 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:00:54.397 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:00:54.398 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:00:54.498 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:00:54.535 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:00:54.539 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:00:54.540 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:00:54.641 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:00:54.674 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:00:54.676 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:00:54.676 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:00:54.777 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:00:54.808 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:00:54.814 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:00:54.815 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:00:54.916 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:00:54.947 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:00:54.951 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:00:54.952 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:00:55.053 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:00:55.083 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:00:55.086 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:00:55.087 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:00:55.179 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 14 16 28454 28438 126.925 118.375 0.00750568 0.00787338 2023-06-03T20:00:55.188 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:00:55.228 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:00:55.231 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:00:55.232 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:00:55.333 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:00:55.389 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:00:55.394 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:00:55.395 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:00:55.495 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:00:55.533 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:00:55.535 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:00:55.536 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:00:55.634 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:00:55 smithi164 bash[59462]: cluster 1685822454.4363334 mgr.y (mgr.34104) 738 : cluster 0 pgmap v648: 89 pgs: 89 active+clean; 1.5 GiB data, 5.5 GiB used, 710 GiB / 715 GiB avail; 124 MiB/s wr, 1.99k op/s 2023-06-03T20:00:55.637 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:00:55.666 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:00:55.669 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:00:55.670 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:00:55.771 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:00:55.778 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:00:55 smithi089 bash[75068]: cluster 1685822454.4363334 mgr.y (mgr.34104) 738 : cluster 0 pgmap v648: 89 pgs: 89 active+clean; 1.5 GiB data, 5.5 GiB used, 710 GiB / 715 GiB avail; 124 MiB/s wr, 1.99k op/s 2023-06-03T20:00:55.779 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:00:55 smithi089 bash[78452]: cluster 1685822454.4363334 mgr.y (mgr.34104) 738 : cluster 0 pgmap v648: 89 pgs: 89 active+clean; 1.5 GiB data, 5.5 GiB used, 710 GiB / 715 GiB avail; 124 MiB/s wr, 1.99k op/s 2023-06-03T20:00:55.802 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:00:55.807 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:00:55.807 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:00:55.908 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:00:55.940 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:00:55.942 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:00:55.943 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:00:56.044 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:00:56.075 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:00:56.081 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:00:56.081 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:00:56.179 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 15 16 30046 30030 125.096 99.5 0.00937121 0.00798809 2023-06-03T20:00:56.182 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:00:56.211 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:00:56.214 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:00:56.215 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:00:56.316 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:00:56.325 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 20:00:56 smithi089 bash[71163]: ::ffff:172.21.15.164 - - [03/Jun/2023:20:00:56] "GET /metrics HTTP/1.1" 200 228075 "" "Prometheus/2.33.4" 2023-06-03T20:00:56.347 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:00:56.349 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:00:56.350 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:00:56.451 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:00:56.475 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:00:56.477 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:00:56.478 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:00:56.579 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:00:56.610 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:00:56.614 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:00:56.614 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:00:56.715 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:00:56.746 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:00:56.748 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:00:56.749 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:00:56.849 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:00:56.885 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:00:56.889 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:00:56.890 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:00:56.991 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:00:57.032 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:00:57.034 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:00:57.035 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:00:57.135 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:00:57.160 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:00:57.161 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:00:57.162 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:00:57.180 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 16 16 31245 31229 121.961 74.9375 0.00256169 0.00817651 2023-06-03T20:00:57.262 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:00:57.302 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:00:57.305 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:00:57.306 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:00:57.408 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:00:57.445 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:00:57.449 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:00:57.450 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:00:57.550 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:00:57.582 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:00:57.584 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:00:57.584 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:00:57.685 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:00:57.723 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:00:57.725 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:00:57.726 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:00:57.827 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:00:57.855 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:00:57.857 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:00:57.858 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:00:57.959 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:00:57.992 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:00:57.992 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:00:57 smithi089 bash[75068]: cluster 1685822456.4369586 mgr.y (mgr.34104) 739 : cluster 0 pgmap v649: 89 pgs: 89 active+clean; 1.6 GiB data, 5.7 GiB used, 710 GiB / 715 GiB avail; 113 MiB/s wr, 1.81k op/s 2023-06-03T20:00:57.992 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:00:57 smithi089 bash[75068]: audit 1685822456.8832018 mon.a (mon.0) 673 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:00:57.993 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:00:57 smithi089 bash[75068]: audit 1685822456.8970587 mon.a (mon.0) 674 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:00:57.993 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:00:57 smithi089 bash[78452]: cluster 1685822456.4369586 mgr.y (mgr.34104) 739 : cluster 0 pgmap v649: 89 pgs: 89 active+clean; 1.6 GiB data, 5.7 GiB used, 710 GiB / 715 GiB avail; 113 MiB/s wr, 1.81k op/s 2023-06-03T20:00:57.993 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:00:57 smithi089 bash[78452]: audit 1685822456.8832018 mon.a (mon.0) 673 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:00:57.993 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:00:57 smithi089 bash[78452]: audit 1685822456.8970587 mon.a (mon.0) 674 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:00:57.995 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:00:57.996 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:00:58.097 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:00:58.122 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:00:58.124 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:00:58.125 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:00:58.183 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 17 16 32773 32757 120.384 95.5 0.0169938 0.00829955 2023-06-03T20:00:58.226 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:00:58.236 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:00:57 smithi164 bash[59462]: cluster 1685822456.4369586 mgr.y (mgr.34104) 739 : cluster 0 pgmap v649: 89 pgs: 89 active+clean; 1.6 GiB data, 5.7 GiB used, 710 GiB / 715 GiB avail; 113 MiB/s wr, 1.81k op/s 2023-06-03T20:00:58.236 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:00:57 smithi164 bash[59462]: audit 1685822456.8832018 mon.a (mon.0) 673 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:00:58.236 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:00:57 smithi164 bash[59462]: audit 1685822456.8970587 mon.a (mon.0) 674 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:00:58.261 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:00:58.264 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:00:58.265 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:00:58.366 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:00:58.396 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:00:58.398 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:00:58.399 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:00:58.500 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:00:58.535 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:00:58.538 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:00:58.538 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:00:58.639 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:00:58.661 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:00:58.662 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:00:58.663 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:00:58.764 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:00:58.795 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:00:58.796 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:00:58.797 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:00:58.898 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:00:58.924 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:00:58.927 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:00:58.928 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:00:59.028 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:00:59.064 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:00:59.067 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:00:59.067 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:00:59.168 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:00:59.183 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 18 16 34227 34211 118.744 90.875 0.01163 0.00841615 2023-06-03T20:00:59.216 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:00:59.220 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:00:59.221 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:00:59.322 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:00:59.370 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:00:59.373 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:00:59.374 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:00:59.476 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:00:59.504 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:00:59.509 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:00:59.509 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:00:59.610 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:00:59.637 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:00:59.641 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:00:59.641 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:00:59.742 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:00:59.782 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:00:59.786 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:00:59.786 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:00:59.888 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:00:59.925 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:00:59 smithi089 bash[75068]: cluster 1685822458.437996 mgr.y (mgr.34104) 740 : cluster 0 pgmap v650: 89 pgs: 89 active+clean; 1.9 GiB data, 7.0 GiB used, 708 GiB / 715 GiB avail; 136 MiB/s wr, 2.17k op/s 2023-06-03T20:00:59.925 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:00:59 smithi089 bash[78452]: cluster 1685822458.437996 mgr.y (mgr.34104) 740 : cluster 0 pgmap v650: 89 pgs: 89 active+clean; 1.9 GiB data, 7.0 GiB used, 708 GiB / 715 GiB avail; 136 MiB/s wr, 2.17k op/s 2023-06-03T20:00:59.928 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:00:59.930 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:00:59.932 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:01:00.033 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:01:00.044 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:00:59 smithi164 bash[59462]: cluster 1685822458.437996 mgr.y (mgr.34104) 740 : cluster 0 pgmap v650: 89 pgs: 89 active+clean; 1.9 GiB data, 7.0 GiB used, 708 GiB / 715 GiB avail; 136 MiB/s wr, 2.17k op/s 2023-06-03T20:01:00.069 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:01:00.074 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:01:00.074 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:01:00.175 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:01:00.182 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 19 16 35790 35774 117.636 97.6875 0.00547252 0.00849379 2023-06-03T20:01:00.208 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:01:00.210 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:01:00.211 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:01:00.312 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:01:00.347 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:01:00.357 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:01:00.358 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:01:00.459 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:01:00.500 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:01:00.505 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:01:00.505 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:01:00.606 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:01:00.639 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:01:00.641 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:01:00.641 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:01:00.742 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:01:00.775 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:01:00.781 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:01:00.782 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:01:00.883 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:01:00.910 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:01:00.912 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:01:00.913 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:01:01.014 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:01:01.104 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:01:01.106 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:01:01.106 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:01:01.183 INFO:tasks.radosbench.radosbench.0.smithi089.stdout:2023-06-03T20:01:01.179221+0000 min lat: 0.00125194 max lat: 0.378924 avg lat: 0.00858904 2023-06-03T20:01:01.183 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-06-03T20:01:01.183 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 20 15 37259 37244 116.348 91.875 0.00694566 0.00858904 2023-06-03T20:01:01.208 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:01:01.240 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:01:01.244 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:01:01.245 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:01:01.346 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:01:01.375 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:01:01.381 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:01:01.382 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:01:01.484 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:01:01.519 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:01:01.522 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:01:01.522 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:01:01.623 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:01:01.720 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:01:01.722 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:01:01.723 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:01:01.824 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:01:01.859 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:01:01.863 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:01:01.863 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:01:01.964 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:01:02.008 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:01:02.012 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:01:02.013 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:01:02.029 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:01:01 smithi089 bash[75068]: cluster 1685822460.438358 mgr.y (mgr.34104) 741 : cluster 0 pgmap v651: 89 pgs: 89 active+clean; 1.9 GiB data, 7.1 GiB used, 708 GiB / 715 GiB avail; 91 MiB/s wr, 1.45k op/s 2023-06-03T20:01:02.029 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:01:01 smithi089 bash[78452]: cluster 1685822460.438358 mgr.y (mgr.34104) 741 : cluster 0 pgmap v651: 89 pgs: 89 active+clean; 1.9 GiB data, 7.1 GiB used, 708 GiB / 715 GiB avail; 91 MiB/s wr, 1.45k op/s 2023-06-03T20:01:02.114 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:01:02.145 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:01:02.147 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:01:02.148 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:01:02.183 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 21 16 38839 38823 115.507 98.6875 0.00680764 0.00865204 2023-06-03T20:01:02.249 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:01:02.282 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:01:02.284 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:01:02.285 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:01:02.333 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:01:01 smithi164 bash[59462]: cluster 1685822460.438358 mgr.y (mgr.34104) 741 : cluster 0 pgmap v651: 89 pgs: 89 active+clean; 1.9 GiB data, 7.1 GiB used, 708 GiB / 715 GiB avail; 91 MiB/s wr, 1.45k op/s 2023-06-03T20:01:02.386 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:01:02.431 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:01:02.435 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:01:02.435 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:01:02.536 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:01:02.572 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:01:02.575 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:01:02.575 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:01:02.676 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:01:02.704 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:01:02.706 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:01:02.707 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:01:02.807 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:01:02.845 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:01:02.847 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:01:02.848 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:01:02.949 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:01:02.982 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:01:02.986 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:01:02.987 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:01:03.087 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:01:03.128 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:01:03.132 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:01:03.133 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:01:03.183 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 22 16 40394 40378 114.674 97.1875 0.0079302 0.00871436 2023-06-03T20:01:03.234 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:01:03.275 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:01:03.277 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:01:03.278 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:01:03.379 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:01:03.408 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:01:03.410 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:01:03.410 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:01:03.511 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:01:03.550 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:01:03.553 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:01:03.554 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:01:03.655 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:01:03.676 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:01:03.678 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:01:03.679 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:01:03.780 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:01:03.822 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:01:03.826 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:01:03.826 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:01:03.927 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:01:03.958 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:01:03.962 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:01:03.963 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:01:04.028 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:01:03 smithi089 bash[75068]: cluster 1685822462.438942 mgr.y (mgr.34104) 742 : cluster 0 pgmap v652: 89 pgs: 89 active+clean; 2.0 GiB data, 7.3 GiB used, 708 GiB / 715 GiB avail; 98 MiB/s wr, 1.56k op/s 2023-06-03T20:01:04.029 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:01:03 smithi089 bash[78452]: cluster 1685822462.438942 mgr.y (mgr.34104) 742 : cluster 0 pgmap v652: 89 pgs: 89 active+clean; 2.0 GiB data, 7.3 GiB used, 708 GiB / 715 GiB avail; 98 MiB/s wr, 1.56k op/s 2023-06-03T20:01:04.064 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:01:04.084 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:01:03 smithi164 bash[59462]: cluster 1685822462.438942 mgr.y (mgr.34104) 742 : cluster 0 pgmap v652: 89 pgs: 89 active+clean; 2.0 GiB data, 7.3 GiB used, 708 GiB / 715 GiB avail; 98 MiB/s wr, 1.56k op/s 2023-06-03T20:01:04.095 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:01:04.097 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:01:04.098 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:01:04.183 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 23 16 41834 41818 113.601 90 0.00408671 0.00879698 2023-06-03T20:01:04.199 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:01:04.234 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:01:04.237 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:01:04.238 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:01:04.339 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:01:04.379 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:01:04.382 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:01:04.383 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:01:04.484 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:01:04.520 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:01:04.523 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:01:04.524 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:01:04.625 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:01:04.658 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:01:04.661 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:01:04.662 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:01:04.763 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:01:04.802 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:01:04.808 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:01:04.809 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:01:04.906 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:01:04 smithi089 bash[78452]: audit 1685822464.811647 mon.a (mon.0) 675 : audit 1 from='client.? 172.21.15.89:0/2735925476' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-06-03T20:01:04.906 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:01:04 smithi089 bash[75068]: audit 1685822464.811647 mon.a (mon.0) 675 : audit 1 from='client.? 172.21.15.89:0/2735925476' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-06-03T20:01:04.910 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:01:04.932 INFO:teuthology.orchestra.run.smithi089.stderr:reweighted osd.2 to 1 (10000) 2023-06-03T20:01:04.944 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:01:04.946 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:01:04.947 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:01:05.048 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:01:05.081 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:01:05.085 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:01:05.086 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:01:05.183 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 24 16 43265 43249 112.594 89.4375 0.00595546 0.00887586 2023-06-03T20:01:05.187 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:01:05.196 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:01:04 smithi164 bash[59462]: audit 1685822464.811647 mon.a (mon.0) 675 : audit 1 from='client.? 172.21.15.89:0/2735925476' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-06-03T20:01:05.217 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:01:05.219 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:01:05.220 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:01:05.321 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:01:05.356 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:01:05.360 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:01:05.362 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:01:05.463 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:01:05.489 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:01:05.493 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:01:05.494 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:01:05.595 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:01:05.625 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:01:05.627 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:01:05.628 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:01:05.729 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:01:05.757 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:01:05.764 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:01:05.765 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:01:05.866 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:01:05.886 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:01:05.888 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:01:05.889 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:01:05.989 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:01:06.001 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:01:05 smithi089 bash[75068]: cluster 1685822464.439641 mgr.y (mgr.34104) 743 : cluster 0 pgmap v653: 89 pgs: 89 active+clean; 2.4 GiB data, 8.5 GiB used, 707 GiB / 715 GiB avail; 120 MiB/s wr, 1.92k op/s 2023-06-03T20:01:06.001 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:01:05 smithi089 bash[75068]: cluster 1685822464.9105163 osd.6 (osd.6) 51 : cluster 3 failed to encode map e132 with expected crc 2023-06-03T20:01:06.001 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:01:05 smithi089 bash[75068]: audit 1685822464.9174516 mon.a (mon.0) 676 : audit 1 from='client.? 172.21.15.89:0/2735925476' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2023-06-03T20:01:06.001 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:01:05 smithi089 bash[75068]: cluster 1685822464.917537 mon.a (mon.0) 677 : cluster 0 osdmap e132: 8 total, 8 up, 8 in 2023-06-03T20:01:06.002 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:01:05 smithi089 bash[75068]: cluster 1685822464.926801 osd.5 (osd.5) 55 : cluster 3 failed to encode map e132 with expected crc 2023-06-03T20:01:06.002 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:01:05 smithi089 bash[78452]: cluster 1685822464.439641 mgr.y (mgr.34104) 743 : cluster 0 pgmap v653: 89 pgs: 89 active+clean; 2.4 GiB data, 8.5 GiB used, 707 GiB / 715 GiB avail; 120 MiB/s wr, 1.92k op/s 2023-06-03T20:01:06.002 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:01:05 smithi089 bash[78452]: cluster 1685822464.9105163 osd.6 (osd.6) 51 : cluster 3 failed to encode map e132 with expected crc 2023-06-03T20:01:06.002 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:01:05 smithi089 bash[78452]: audit 1685822464.9174516 mon.a (mon.0) 676 : audit 1 from='client.? 172.21.15.89:0/2735925476' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2023-06-03T20:01:06.002 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:01:05 smithi089 bash[78452]: cluster 1685822464.917537 mon.a (mon.0) 677 : cluster 0 osdmap e132: 8 total, 8 up, 8 in 2023-06-03T20:01:06.002 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:01:05 smithi089 bash[78452]: cluster 1685822464.926801 osd.5 (osd.5) 55 : cluster 3 failed to encode map e132 with expected crc 2023-06-03T20:01:06.027 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:01:06.031 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:01:06.032 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:01:06.133 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:01:06.159 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:01:06.162 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:01:06.163 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:01:06.183 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 25 16 44630 44614 111.502 85.3125 0.0167329 0.00894415 2023-06-03T20:01:06.264 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:01:06.277 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:01:05 smithi164 bash[59462]: cluster 1685822464.439641 mgr.y (mgr.34104) 743 : cluster 0 pgmap v653: 89 pgs: 89 active+clean; 2.4 GiB data, 8.5 GiB used, 707 GiB / 715 GiB avail; 120 MiB/s wr, 1.92k op/s 2023-06-03T20:01:06.277 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:01:05 smithi164 bash[59462]: cluster 1685822464.9105163 osd.6 (osd.6) 51 : cluster 3 failed to encode map e132 with expected crc 2023-06-03T20:01:06.277 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:01:05 smithi164 bash[59462]: audit 1685822464.9174516 mon.a (mon.0) 676 : audit 1 from='client.? 172.21.15.89:0/2735925476' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2023-06-03T20:01:06.277 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:01:05 smithi164 bash[59462]: cluster 1685822464.917537 mon.a (mon.0) 677 : cluster 0 osdmap e132: 8 total, 8 up, 8 in 2023-06-03T20:01:06.278 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:01:05 smithi164 bash[59462]: cluster 1685822464.926801 osd.5 (osd.5) 55 : cluster 3 failed to encode map e132 with expected crc 2023-06-03T20:01:06.278 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 20:01:06 smithi089 bash[71163]: ::ffff:172.21.15.164 - - [03/Jun/2023:20:01:06] "GET /metrics HTTP/1.1" 200 228075 "" "Prometheus/2.33.4" 2023-06-03T20:01:06.297 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:01:06.304 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:01:06.305 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:01:06.406 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:01:06.422 INFO:teuthology.orchestra.run.smithi089.stdout:osd.0: {} 2023-06-03T20:01:06.435 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:01:06.437 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:01:06.438 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:01:06.459 INFO:teuthology.orchestra.run.smithi089.stdout:osd.1: {} 2023-06-03T20:01:06.496 INFO:teuthology.orchestra.run.smithi089.stdout:osd.2: {} 2023-06-03T20:01:06.534 INFO:teuthology.orchestra.run.smithi089.stdout:osd.3: {} 2023-06-03T20:01:06.540 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:01:06.571 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:01:06.574 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:01:06.575 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:01:06.588 INFO:teuthology.orchestra.run.smithi089.stdout:osd.4: {} 2023-06-03T20:01:06.630 INFO:teuthology.orchestra.run.smithi089.stdout:osd.5: {} 2023-06-03T20:01:06.664 INFO:teuthology.orchestra.run.smithi089.stdout:osd.6: {} 2023-06-03T20:01:06.676 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:01:06.697 INFO:teuthology.orchestra.run.smithi089.stdout:osd.7: {} 2023-06-03T20:01:06.711 INFO:teuthology.orchestra.run.smithi089.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-06-03T20:01:06.711 INFO:teuthology.orchestra.run.smithi089.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-06-03T20:01:06.711 INFO:teuthology.orchestra.run.smithi089.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-06-03T20:01:06.711 INFO:teuthology.orchestra.run.smithi089.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-06-03T20:01:06.711 INFO:teuthology.orchestra.run.smithi089.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-06-03T20:01:06.711 INFO:teuthology.orchestra.run.smithi089.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-06-03T20:01:06.712 INFO:teuthology.orchestra.run.smithi089.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-06-03T20:01:06.712 INFO:teuthology.orchestra.run.smithi089.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-06-03T20:01:06.712 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:01:06.713 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:01:06.714 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:01:06.815 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:01:06.843 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:01:06.845 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:01:06.846 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:01:06.947 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:01:06.957 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:01:06 smithi089 bash[75068]: audit 1685822466.3630586 mon.a (mon.0) 678 : audit 0 from='client.? 172.21.15.89:0/4206529396' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-03T20:01:06.958 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:01:06 smithi089 bash[78452]: audit 1685822466.3630586 mon.a (mon.0) 678 : audit 0 from='client.? 172.21.15.89:0/4206529396' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-03T20:01:06.982 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:01:06.983 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:01:06.984 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:01:07.084 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:01:06 smithi164 bash[59462]: audit 1685822466.3630586 mon.a (mon.0) 678 : audit 0 from='client.? 172.21.15.89:0/4206529396' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-03T20:01:07.085 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:01:07.119 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:01:07.123 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:01:07.124 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:01:07.183 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 26 16 45887 45871 110.235 78.5625 0.010579 0.00906558 2023-06-03T20:01:07.225 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:01:07.315 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:01:07.318 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:01:07.319 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:01:07.420 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:01:07.460 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:01:07.465 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:01:07.466 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:01:07.567 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:01:07.601 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:01:07.605 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:01:07.605 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:01:07.706 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:01:07.741 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:01:07.744 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:01:07.745 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:01:07.846 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:01:07.876 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:01:07.879 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:01:07.880 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:01:07.981 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:01:08.011 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:01:08.013 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:01:08.014 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:01:08.084 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:01:07 smithi164 bash[59462]: cluster 1685822466.4401174 mgr.y (mgr.34104) 744 : cluster 0 pgmap v655: 89 pgs: 89 active+clean; 2.5 GiB data, 8.7 GiB used, 707 GiB / 715 GiB avail; 96 MiB/s wr, 1.53k op/s 2023-06-03T20:01:08.085 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:01:07 smithi164 bash[59462]: audit 1685822467.0941603 mon.c (mon.2) 385 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T20:01:08.115 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:01:08.125 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:01:07 smithi089 bash[75068]: cluster 1685822466.4401174 mgr.y (mgr.34104) 744 : cluster 0 pgmap v655: 89 pgs: 89 active+clean; 2.5 GiB data, 8.7 GiB used, 707 GiB / 715 GiB avail; 96 MiB/s wr, 1.53k op/s 2023-06-03T20:01:08.125 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:01:07 smithi089 bash[75068]: audit 1685822467.0941603 mon.c (mon.2) 385 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T20:01:08.125 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:01:07 smithi089 bash[78452]: cluster 1685822466.4401174 mgr.y (mgr.34104) 744 : cluster 0 pgmap v655: 89 pgs: 89 active+clean; 2.5 GiB data, 8.7 GiB used, 707 GiB / 715 GiB avail; 96 MiB/s wr, 1.53k op/s 2023-06-03T20:01:08.126 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:01:07 smithi089 bash[78452]: audit 1685822467.0941603 mon.c (mon.2) 385 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T20:01:08.144 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:01:08.147 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:01:08.147 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:01:08.184 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 27 16 47303 47287 109.43 88.5 0.00629217 0.00912116 2023-06-03T20:01:08.248 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:01:08.279 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:01:08.282 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:01:08.282 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:01:08.384 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:01:08.415 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:01:08.419 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:01:08.419 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:01:08.522 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:01:08.556 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:01:08.560 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:01:08.561 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:01:08.662 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:01:08.698 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:01:08.703 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:01:08.704 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:01:08.805 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:01:08.925 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:01:08.930 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:01:08.930 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:01:08.946 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:01:08 smithi164 bash[59462]: cluster 1685822468.440912 mgr.y (mgr.34104) 745 : cluster 0 pgmap v656: 89 pgs: 89 active+clean; 2.8 GiB data, 9.8 GiB used, 705 GiB / 715 GiB avail; 91 MiB/s wr, 1.46k op/s 2023-06-03T20:01:09.031 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:01:09.056 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:01:09.057 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:01:09.058 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:01:09.159 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:01:09.168 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:01:08 smithi089 bash[78452]: cluster 1685822468.440912 mgr.y (mgr.34104) 745 : cluster 0 pgmap v656: 89 pgs: 89 active+clean; 2.8 GiB data, 9.8 GiB used, 705 GiB / 715 GiB avail; 91 MiB/s wr, 1.46k op/s 2023-06-03T20:01:09.169 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:01:08 smithi089 bash[75068]: cluster 1685822468.440912 mgr.y (mgr.34104) 745 : cluster 0 pgmap v656: 89 pgs: 89 active+clean; 2.8 GiB data, 9.8 GiB used, 705 GiB / 715 GiB avail; 91 MiB/s wr, 1.46k op/s 2023-06-03T20:01:09.184 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 28 16 48553 48537 108.312 78.125 0.0179501 0.00922756 2023-06-03T20:01:09.204 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:01:09.206 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:01:09.207 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:01:09.307 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:01:09.332 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:01:09.334 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:01:09.335 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:01:09.436 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:01:09.464 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:01:09.468 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:01:09.468 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:01:09.569 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:01:09.604 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:01:09.605 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:01:09.606 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:01:09.707 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:01:09.749 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:01:09.754 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:01:09.755 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:01:09.856 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:01:09.886 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:01:09.889 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:01:09.890 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:01:09.991 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:01:10.030 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:01:10.034 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:01:10.035 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:01:10.136 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:01:10.170 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:01:10.174 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:01:10.175 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:01:10.184 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 29 16 49886 49870 107.45 83.3125 0.00967447 0.00930131 2023-06-03T20:01:10.277 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:01:10.310 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:01:10.314 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:01:10.315 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:01:10.416 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:01:10.449 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:01:10.452 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:01:10.452 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:01:10.553 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:01:10.577 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:01:10.580 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:01:10.580 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:01:10.681 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:01:10.721 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:01:10.725 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:01:10.726 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:01:10.827 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:01:10.869 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:01:10.872 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:01:10.873 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:01:10.974 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:01:11.011 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:01:11.016 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:01:11.018 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:01:11.120 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:01:11.159 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:01:11.166 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:01:11.167 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:01:11.184 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 30 16 50990 50974 106.168 69 0.0158542 0.00941394 2023-06-03T20:01:11.268 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:01:11.303 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:01:11.308 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:01:11.309 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:01:11.410 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:01:11.441 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:01:11.447 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:01:11.447 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:01:11.450 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph osd reweight 3 1 2023-06-03T20:01:11.528 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:01:11 smithi089 bash[78452]: cluster 1685822470.4412854 mgr.y (mgr.34104) 746 : cluster 0 pgmap v657: 89 pgs: 89 active+clean; 2.8 GiB data, 9.9 GiB used, 705 GiB / 715 GiB avail; 91 MiB/s wr, 1.46k op/s 2023-06-03T20:01:11.529 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:01:11 smithi089 bash[75068]: cluster 1685822470.4412854 mgr.y (mgr.34104) 746 : cluster 0 pgmap v657: 89 pgs: 89 active+clean; 2.8 GiB data, 9.9 GiB used, 705 GiB / 715 GiB avail; 91 MiB/s wr, 1.46k op/s 2023-06-03T20:01:11.548 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:01:11.589 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:01:11.592 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:01:11.593 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:01:11.695 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:01:11.738 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:01:11.741 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:01:11.742 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:01:11.834 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:01:11 smithi164 bash[59462]: cluster 1685822470.4412854 mgr.y (mgr.34104) 746 : cluster 0 pgmap v657: 89 pgs: 89 active+clean; 2.8 GiB data, 9.9 GiB used, 705 GiB / 715 GiB avail; 91 MiB/s wr, 1.46k op/s 2023-06-03T20:01:11.843 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:01:11.872 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:01:11.878 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:01:11.881 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:01:11.981 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:01:12.029 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:01:12.037 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:01:12.037 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:01:12.138 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:01:12.171 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:01:12.175 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:01:12.175 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:01:12.184 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 31 16 52217 52201 105.217 76.6875 0.00687149 0.00949761 2023-06-03T20:01:12.276 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:01:12.307 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:01:12.310 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:01:12.310 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:01:12.411 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:01:12.442 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:01:12.444 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:01:12.445 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:01:12.546 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:01:12.559 INFO:teuthology.orchestra.run.smithi089.stdout:ERROR: (22) Invalid argument 2023-06-03T20:01:12.559 INFO:teuthology.orchestra.run.smithi089.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-06-03T20:01:12.592 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:01:12.595 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:01:12.596 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:01:12.697 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:01:12.735 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:01:12.740 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:01:12.741 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:01:12.842 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:01:12.884 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:01:12.886 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:01:12.887 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:01:12.988 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:01:13.024 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:01:13.028 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:01:13.029 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:01:13.130 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:01:13.172 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:01:13.176 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:01:13.177 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:01:13.186 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 32 16 53263 53247 103.972 65.375 0.0133265 0.00961148 2023-06-03T20:01:13.278 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:01:13.316 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:01:13.317 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:01:13.318 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:01:13.419 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:01:13.452 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:01:13.457 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:01:13.458 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:01:13.548 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:01:13 smithi089 bash[75068]: cluster 1685822472.4419172 mgr.y (mgr.34104) 747 : cluster 0 pgmap v658: 89 pgs: 89 active+clean; 2.9 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 90 MiB/s wr, 1.45k op/s 2023-06-03T20:01:13.549 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:01:13 smithi089 bash[75068]: audit 1685822472.9793775 mon.b (mon.1) 46 : audit 1 from='client.? 172.21.15.89:0/2191335406' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-06-03T20:01:13.549 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:01:13 smithi089 bash[75068]: audit 1685822472.9822173 mon.a (mon.0) 679 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-06-03T20:01:13.558 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:01:13.586 INFO:teuthology.orchestra.run.smithi089.stderr:noscrub is unset 2023-06-03T20:01:13.616 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:01:13.619 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:01:13.620 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:01:13.640 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-06-03T20:01:13.721 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:01:13.771 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:01:13.771 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:01:13.772 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:01:13.833 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:01:13 smithi164 bash[59462]: cluster 1685822472.4419172 mgr.y (mgr.34104) 747 : cluster 0 pgmap v658: 89 pgs: 89 active+clean; 2.9 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 90 MiB/s wr, 1.45k op/s 2023-06-03T20:01:13.834 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:01:13 smithi164 bash[59462]: audit 1685822472.9793775 mon.b (mon.1) 46 : audit 1 from='client.? 172.21.15.89:0/2191335406' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-06-03T20:01:13.834 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:01:13 smithi164 bash[59462]: audit 1685822472.9822173 mon.a (mon.0) 679 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-06-03T20:01:13.874 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:01:13.907 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:01:13.909 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:01:13.910 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:01:13.920 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:01:13 smithi089 bash[78452]: cluster 1685822472.4419172 mgr.y (mgr.34104) 747 : cluster 0 pgmap v658: 89 pgs: 89 active+clean; 2.9 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 90 MiB/s wr, 1.45k op/s 2023-06-03T20:01:13.921 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:01:13 smithi089 bash[78452]: audit 1685822472.9793775 mon.b (mon.1) 46 : audit 1 from='client.? 172.21.15.89:0/2191335406' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-06-03T20:01:13.921 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:01:13 smithi089 bash[78452]: audit 1685822472.9822173 mon.a (mon.0) 679 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-06-03T20:01:14.010 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:01:14.042 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:01:14.045 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:01:14.045 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:01:14.146 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:01:14.186 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 33 16 54475 54459 103.116 75.75 0.00996663 0.0096907 2023-06-03T20:01:14.190 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:01:14.194 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:01:14.194 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:01:14.295 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:01:14.330 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:01:14.333 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:01:14.333 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:01:14.434 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:01:14.443 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph daemon osd.1 dump_blocked_ops 2023-06-03T20:01:14.477 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:01:14.480 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:01:14.481 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:01:14.560 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:01:14 smithi089 bash[78452]: cluster 1685822473.5336742 mon.a (mon.0) 680 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-06-03T20:01:14.560 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:01:14 smithi089 bash[78452]: cluster 1685822473.5339344 osd.6 (osd.6) 52 : cluster 3 failed to encode map e133 with expected crc 2023-06-03T20:01:14.560 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:01:14 smithi089 bash[78452]: audit 1685822473.5377417 mon.a (mon.0) 681 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-06-03T20:01:14.561 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:01:14 smithi089 bash[78452]: cluster 1685822473.5378168 mon.a (mon.0) 682 : cluster 0 osdmap e133: 8 total, 8 up, 8 in 2023-06-03T20:01:14.561 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:01:14 smithi089 bash[78452]: cluster 1685822473.5386136 osd.7 (osd.7) 45 : cluster 3 failed to encode map e133 with expected crc 2023-06-03T20:01:14.561 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:01:14 smithi089 bash[78452]: cluster 1685822473.545184 osd.5 (osd.5) 56 : cluster 3 failed to encode map e133 with expected crc 2023-06-03T20:01:14.561 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:01:14 smithi089 bash[78452]: audit 1685822474.2193882 mon.a (mon.0) 683 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:01:14.561 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:01:14 smithi089 bash[78452]: audit 1685822474.237289 mon.a (mon.0) 684 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:01:14.583 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:01:14.584 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:01:14 smithi164 bash[59462]: cluster 1685822473.5336742 mon.a (mon.0) 680 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-06-03T20:01:14.584 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:01:14 smithi164 bash[59462]: cluster 1685822473.5339344 osd.6 (osd.6) 52 : cluster 3 failed to encode map e133 with expected crc 2023-06-03T20:01:14.584 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:01:14 smithi164 bash[59462]: audit 1685822473.5377417 mon.a (mon.0) 681 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-06-03T20:01:14.584 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:01:14 smithi164 bash[59462]: cluster 1685822473.5378168 mon.a (mon.0) 682 : cluster 0 osdmap e133: 8 total, 8 up, 8 in 2023-06-03T20:01:14.584 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:01:14 smithi164 bash[59462]: cluster 1685822473.5386136 osd.7 (osd.7) 45 : cluster 3 failed to encode map e133 with expected crc 2023-06-03T20:01:14.585 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:01:14 smithi164 bash[59462]: cluster 1685822473.545184 osd.5 (osd.5) 56 : cluster 3 failed to encode map e133 with expected crc 2023-06-03T20:01:14.585 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:01:14 smithi164 bash[59462]: audit 1685822474.2193882 mon.a (mon.0) 683 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:01:14.585 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:01:14 smithi164 bash[59462]: audit 1685822474.237289 mon.a (mon.0) 684 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:01:14.637 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:01:14.639 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:01:14.640 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:01:14.741 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:01:14.775 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:01:14.782 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:01:14.783 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:01:14.884 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:01:14.917 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:01:14.921 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:01:14.922 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:01:14.951 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:01:14 smithi089 bash[75068]: cluster 1685822473.5336742 mon.a (mon.0) 680 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-06-03T20:01:14.951 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:01:14 smithi089 bash[75068]: cluster 1685822473.5339344 osd.6 (osd.6) 52 : cluster 3 failed to encode map e133 with expected crc 2023-06-03T20:01:14.952 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:01:14 smithi089 bash[75068]: audit 1685822473.5377417 mon.a (mon.0) 681 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-06-03T20:01:14.952 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:01:14 smithi089 bash[75068]: cluster 1685822473.5378168 mon.a (mon.0) 682 : cluster 0 osdmap e133: 8 total, 8 up, 8 in 2023-06-03T20:01:14.952 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:01:14 smithi089 bash[75068]: cluster 1685822473.5386136 osd.7 (osd.7) 45 : cluster 3 failed to encode map e133 with expected crc 2023-06-03T20:01:14.952 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:01:14 smithi089 bash[75068]: cluster 1685822473.545184 osd.5 (osd.5) 56 : cluster 3 failed to encode map e133 with expected crc 2023-06-03T20:01:14.952 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:01:14 smithi089 bash[75068]: audit 1685822474.2193882 mon.a (mon.0) 683 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:01:14.952 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:01:14 smithi089 bash[75068]: audit 1685822474.237289 mon.a (mon.0) 684 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:01:15.023 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:01:15.056 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:01:15.060 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:01:15.061 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:01:15.162 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:01:15.184 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 34 16 55649 55633 102.241 73.375 0.00780446 0.00977567 2023-06-03T20:01:15.195 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:01:15.199 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:01:15.199 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:01:15.300 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:01:15.337 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:01:15.341 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:01:15.342 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:01:15.443 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:01:15.477 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:01:15.480 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:01:15.480 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:01:15.582 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:01:15.584 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:01:15 smithi164 bash[59462]: cluster 1685822474.4432657 mgr.y (mgr.34104) 748 : cluster 0 pgmap v660: 89 pgs: 89 active+clean; 3.2 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 86 MiB/s wr, 1.38k op/s 2023-06-03T20:01:15.584 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:01:15 smithi164 bash[59462]: audit 1685822474.926583 mon.c (mon.2) 386 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-03T20:01:15.584 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:01:15 smithi164 bash[59462]: audit 1685822474.9291623 mon.c (mon.2) 387 : audit 1 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-03T20:01:15.585 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:01:15 smithi164 bash[59462]: audit 1685822474.9423995 mon.a (mon.0) 685 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:01:15.623 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:01:15.626 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:01:15.627 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:01:15.728 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:01:15.760 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:01:15.765 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:01:15.766 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:01:15.779 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:01:15 smithi089 bash[75068]: cluster 1685822474.4432657 mgr.y (mgr.34104) 748 : cluster 0 pgmap v660: 89 pgs: 89 active+clean; 3.2 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 86 MiB/s wr, 1.38k op/s 2023-06-03T20:01:15.779 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:01:15 smithi089 bash[75068]: audit 1685822474.926583 mon.c (mon.2) 386 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-03T20:01:15.779 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:01:15 smithi089 bash[75068]: audit 1685822474.9291623 mon.c (mon.2) 387 : audit 1 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-03T20:01:15.779 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:01:15 smithi089 bash[75068]: audit 1685822474.9423995 mon.a (mon.0) 685 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:01:15.779 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:01:15 smithi089 bash[78452]: cluster 1685822474.4432657 mgr.y (mgr.34104) 748 : cluster 0 pgmap v660: 89 pgs: 89 active+clean; 3.2 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 86 MiB/s wr, 1.38k op/s 2023-06-03T20:01:15.780 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:01:15 smithi089 bash[78452]: audit 1685822474.926583 mon.c (mon.2) 386 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-03T20:01:15.780 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:01:15 smithi089 bash[78452]: audit 1685822474.9291623 mon.c (mon.2) 387 : audit 1 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-03T20:01:15.780 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:01:15 smithi089 bash[78452]: audit 1685822474.9423995 mon.a (mon.0) 685 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:01:15.866 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:01:15.894 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:01:15.896 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:01:15.897 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:01:15.998 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:01:16.039 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:01:16.043 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:01:16.044 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:01:16.145 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:01:16.183 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:01:16.185 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:01:16.185 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:01:16.186 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 35 16 56911 56895 101.572 78.875 0.0103748 0.0098407 2023-06-03T20:01:16.286 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:01:16.336 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:01:16.339 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:01:16.340 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:01:16.440 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:01:16.482 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:01:16.486 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:01:16.487 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:01:16.528 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 20:01:16 smithi089 bash[71163]: ::ffff:172.21.15.164 - - [03/Jun/2023:20:01:16] "GET /metrics HTTP/1.1" 200 228141 "" "Prometheus/2.33.4" 2023-06-03T20:01:16.588 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:01:16.618 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:01:16.619 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:01:16.620 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:01:16.722 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:01:16.755 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:01:16.759 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:01:16.760 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:01:16.861 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:01:16.892 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:01:16.894 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:01:16.895 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:01:16.996 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:01:17.031 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:01:17.036 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:01:17.037 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:01:17.138 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:01:17.177 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:01:17.180 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:01:17.181 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:01:17.185 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 36 16 58175 58159 100.945 79 0.0183473 0.0099 2023-06-03T20:01:17.281 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:01:17.325 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:01:17.330 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:01:17.331 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:01:17.431 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:01:17.466 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:01:17.470 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:01:17.470 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:01:17.554 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph osd unset nodeep-scrub 2023-06-03T20:01:17.558 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:01:17 smithi164 bash[59462]: cluster 1685822476.4437177 mgr.y (mgr.34104) 749 : cluster 0 pgmap v661: 89 pgs: 89 active+clean; 3.3 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 81 MiB/s wr, 1.29k op/s 2023-06-03T20:01:17.572 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:01:17.643 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:01:17.645 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:01:17.646 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:01:17.747 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:01:17.790 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:01:17.795 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:01:17.795 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:01:17.897 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:01:17.937 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:01:17.940 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:01:17.941 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:01:18.029 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:01:17 smithi089 bash[75068]: cluster 1685822476.4437177 mgr.y (mgr.34104) 749 : cluster 0 pgmap v661: 89 pgs: 89 active+clean; 3.3 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 81 MiB/s wr, 1.29k op/s 2023-06-03T20:01:18.029 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:01:17 smithi089 bash[78452]: cluster 1685822476.4437177 mgr.y (mgr.34104) 749 : cluster 0 pgmap v661: 89 pgs: 89 active+clean; 3.3 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 81 MiB/s wr, 1.29k op/s 2023-06-03T20:01:18.042 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:01:18.077 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:01:18.081 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:01:18.082 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:01:18.121 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/mon.a/config 2023-06-03T20:01:18.183 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:01:18.185 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 37 16 59355 59339 100.21 73.75 0.00861885 0.00997332 2023-06-03T20:01:18.220 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:01:18.226 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:01:18.227 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:01:18.327 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:01:18.357 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:01:18.359 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:01:18.360 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:01:18.461 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:01:18.492 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:01:18.496 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:01:18.497 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:01:18.598 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:01:18.629 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:01:18.631 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:01:18.632 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:01:18.733 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:01:18.762 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:01:18.767 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:01:18.768 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:01:18.869 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:01:18.904 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:01:18.908 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:01:18.909 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:01:19.009 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:01:19.037 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:01:19.039 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:01:19.040 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:01:19.140 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:01:19.172 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:01:19.174 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:01:19.175 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:01:19.185 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 38 16 60590 60574 99.6042 77.1875 0.00516069 0.0100333 2023-06-03T20:01:19.277 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:01:19.300 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:01:19.302 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:01:19.303 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:01:19.404 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:01:19.437 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:01:19.442 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:01:19.443 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:01:19.543 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:01:19.585 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:01:19.590 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:01:19.590 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:01:19.691 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:01:19.750 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:01:19.753 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:01:19.754 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:01:19.778 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:01:19 smithi089 bash[78452]: cluster 1685822478.444493 mgr.y (mgr.34104) 750 : cluster 0 pgmap v662: 89 pgs: 89 active+clean; 3.6 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 76 MiB/s wr, 1.22k op/s 2023-06-03T20:01:19.779 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:01:19 smithi089 bash[75068]: cluster 1685822478.444493 mgr.y (mgr.34104) 750 : cluster 0 pgmap v662: 89 pgs: 89 active+clean; 3.6 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 76 MiB/s wr, 1.22k op/s 2023-06-03T20:01:19.834 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:01:19 smithi164 bash[59462]: cluster 1685822478.444493 mgr.y (mgr.34104) 750 : cluster 0 pgmap v662: 89 pgs: 89 active+clean; 3.6 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 76 MiB/s wr, 1.22k op/s 2023-06-03T20:01:19.855 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:01:19.883 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:01:19.886 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:01:19.886 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:01:19.987 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:01:20.019 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:01:20.025 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:01:20.026 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:01:20.127 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:01:20.162 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:01:20.169 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:01:20.170 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:01:20.180 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/mon.a/config 2023-06-03T20:01:20.181 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/mon.a/config 2023-06-03T20:01:20.185 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 39 16 61818 61802 99.018 76.75 0.0189866 0.0100934 2023-06-03T20:01:20.271 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:01:20.311 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:01:20.314 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:01:20.314 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:01:20.415 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:01:20.451 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:01:20.455 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:01:20.455 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:01:20.556 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:01:20.600 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:01:20.601 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:01:20.601 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:01:20.702 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:01:20.734 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:01:20.736 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:01:20.737 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:01:20.838 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:01:20.873 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:01:20.876 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:01:20.876 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:01:20.977 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:01:21.019 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:01:21.022 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:01:21.023 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:01:21.124 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:01:21.173 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:01:21.176 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:01:21.176 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:01:21.185 INFO:tasks.radosbench.radosbench.0.smithi089.stdout:2023-06-03T20:01:21.181814+0000 min lat: 0.00125194 max lat: 0.378924 avg lat: 0.0101589 2023-06-03T20:01:21.186 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-06-03T20:01:21.186 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 40 16 63000 62984 98.3894 73.875 0.0055088 0.0101589 2023-06-03T20:01:21.278 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:01:21.310 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:01:21.314 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:01:21.315 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:01:21.415 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:01:21.575 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:01:21 smithi089 bash[75068]: cluster 1685822480.4450448 mgr.y (mgr.34104) 751 : cluster 0 pgmap v663: 89 pgs: 89 active+clean; 3.6 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 76 MiB/s wr, 1.22k op/s 2023-06-03T20:01:21.595 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:01:21.598 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:01:21.599 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:01:21.700 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:01:21.741 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:01:21.744 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:01:21.744 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:01:21.833 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:01:21 smithi164 bash[59462]: cluster 1685822480.4450448 mgr.y (mgr.34104) 751 : cluster 0 pgmap v663: 89 pgs: 89 active+clean; 3.6 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 76 MiB/s wr, 1.22k op/s 2023-06-03T20:01:21.845 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:01:21.854 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:01:21 smithi089 bash[78452]: cluster 1685822480.4450448 mgr.y (mgr.34104) 751 : cluster 0 pgmap v663: 89 pgs: 89 active+clean; 3.6 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 76 MiB/s wr, 1.22k op/s 2023-06-03T20:01:21.886 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:01:21.891 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:01:21.891 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:01:21.992 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:01:22.030 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:01:22.035 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:01:22.037 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:01:22.138 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:01:22.186 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 41 16 64188 64172 97.8005 74.25 0.0108429 0.0102181 2023-06-03T20:01:22.244 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/mon.a/config 2023-06-03T20:01:22.443 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:01:22.446 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:01:22.447 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:01:22.548 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:01:22.868 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:01:22 smithi164 bash[59462]: audit 1685822482.0947556 mon.c (mon.2) 388 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T20:01:22.890 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:01:22.894 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:01:22.895 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:01:22.996 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:01:23.028 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:01:22 smithi089 bash[78452]: audit 1685822482.0947556 mon.c (mon.2) 388 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T20:01:23.029 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:01:22 smithi089 bash[75068]: audit 1685822482.0947556 mon.c (mon.2) 388 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T20:01:23.029 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:01:23.033 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:01:23.033 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:01:23.134 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:01:23.170 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:01:23.172 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:01:23.173 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:01:23.186 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 42 16 65283 65267 97.1013 68.4375 0.0107034 0.0102923 2023-06-03T20:01:23.274 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:01:23.304 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:01:23.308 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:01:23.309 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:01:23.410 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:01:23.445 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:01:23.452 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:01:23.453 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:01:23.554 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:01:23.778 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:01:23 smithi089 bash[75068]: cluster 1685822482.445532 mgr.y (mgr.34104) 752 : cluster 0 pgmap v664: 89 pgs: 89 active+clean; 3.7 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 76 MiB/s wr, 1.21k op/s 2023-06-03T20:01:23.779 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:01:23 smithi089 bash[78452]: cluster 1685822482.445532 mgr.y (mgr.34104) 752 : cluster 0 pgmap v664: 89 pgs: 89 active+clean; 3.7 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 76 MiB/s wr, 1.21k op/s 2023-06-03T20:01:23.835 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:01:23.837 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:01:23.838 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:01:23.939 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:01:23.948 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:01:23 smithi164 bash[59462]: cluster 1685822482.445532 mgr.y (mgr.34104) 752 : cluster 0 pgmap v664: 89 pgs: 89 active+clean; 3.7 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 76 MiB/s wr, 1.21k op/s 2023-06-03T20:01:23.970 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:01:23.978 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:01:23.978 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:01:24.079 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:01:24.108 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:01:24.113 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:01:24.114 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:01:24.186 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 43 16 66505 66489 96.6192 76.375 0.0163237 0.0103447 2023-06-03T20:01:24.215 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:01:24.245 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:01:24.251 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:01:24.251 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:01:24.352 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:01:24.394 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:01:24.564 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:01:24.565 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:01:24.666 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:01:24.692 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:01:24.697 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:01:24.698 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:01:24.799 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:01:24.833 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:01:24.836 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:01:24.836 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:01:24.937 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:01:24.968 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:01:24.973 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:01:24.974 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:01:25.075 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:01:25.109 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:01:25.114 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:01:25.115 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:01:25.186 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 44 16 67678 67662 96.0894 73.3125 0.0106246 0.0104017 2023-06-03T20:01:25.216 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:01:25.364 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:01:25.368 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:01:25.368 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:01:25.469 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:01:25.508 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:01:25.511 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:01:25.512 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:01:25.594 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:01:25 smithi164 bash[59462]: cluster 1685822484.446035 mgr.y (mgr.34104) 753 : cluster 0 pgmap v665: 89 pgs: 89 active+clean; 3.9 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 69 MiB/s wr, 1.10k op/s 2023-06-03T20:01:25.613 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:01:25.651 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:01:25.654 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:01:25.655 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:01:25.756 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:01:25.778 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:01:25 smithi089 bash[75068]: cluster 1685822484.446035 mgr.y (mgr.34104) 753 : cluster 0 pgmap v665: 89 pgs: 89 active+clean; 3.9 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 69 MiB/s wr, 1.10k op/s 2023-06-03T20:01:25.779 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:01:25 smithi089 bash[78452]: cluster 1685822484.446035 mgr.y (mgr.34104) 753 : cluster 0 pgmap v665: 89 pgs: 89 active+clean; 3.9 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 69 MiB/s wr, 1.10k op/s 2023-06-03T20:01:25.826 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:01:25.833 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:01:25.834 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:01:25.935 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:01:25.968 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:01:25.971 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:01:25.972 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:01:26.073 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:01:26.106 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:01:26.112 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:01:26.113 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:01:26.186 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 45 16 68835 68819 95.561 72.3125 0.0161618 0.0104591 2023-06-03T20:01:26.214 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:01:26.258 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:01:26.264 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:01:26.265 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:01:26.366 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:01:26.375 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 20:01:26 smithi089 bash[71163]: ::ffff:172.21.15.164 - - [03/Jun/2023:20:01:26] "GET /metrics HTTP/1.1" 200 228233 "" "Prometheus/2.33.4" 2023-06-03T20:01:26.395 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:01:26.397 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:01:26.398 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:01:26.499 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:01:26.547 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:01:26.551 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:01:26.552 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:01:26.653 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:01:26.693 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:01:26.697 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:01:26.698 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:01:26.799 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:01:26.835 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:01:26.837 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:01:26.838 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:01:26.939 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:01:26.971 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:01:26.978 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:01:26.978 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:01:27.079 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:01:27.186 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 46 16 69848 69832 94.8599 63.3125 0.028338 0.0105352 2023-06-03T20:01:27.283 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:01:27.292 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:01:27.293 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:01:27.394 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:01:27.425 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:01:27.429 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:01:27.429 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:01:27.530 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:01:27.569 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:01:27.571 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:01:27.572 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:01:27.674 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:01:27.715 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:01:27.718 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:01:27.719 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:01:27.807 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:01:27 smithi089 bash[75068]: cluster 1685822486.4465475 mgr.y (mgr.34104) 754 : cluster 0 pgmap v666: 89 pgs: 89 active+clean; 4.0 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 68 MiB/s wr, 1.09k op/s 2023-06-03T20:01:27.808 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:01:27 smithi089 bash[78452]: cluster 1685822486.4465475 mgr.y (mgr.34104) 754 : cluster 0 pgmap v666: 89 pgs: 89 active+clean; 4.0 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 68 MiB/s wr, 1.09k op/s 2023-06-03T20:01:27.820 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:01:27.864 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:01:27.866 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:01:27.867 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:01:27.968 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:01:27.981 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:01:27 smithi164 bash[59462]: cluster 1685822486.4465475 mgr.y (mgr.34104) 754 : cluster 0 pgmap v666: 89 pgs: 89 active+clean; 4.0 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 68 MiB/s wr, 1.09k op/s 2023-06-03T20:01:28.012 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:01:28.016 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:01:28.017 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:01:28.117 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:01:28.149 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:01:28.154 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:01:28.155 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:01:28.186 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 47 16 70913 70897 94.2578 66.5625 0.0104546 0.0106033 2023-06-03T20:01:28.256 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:01:28.298 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:01:28.302 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:01:28.303 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:01:28.403 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:01:28.435 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:01:28.437 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:01:28.439 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:01:28.540 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:01:28.574 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:01:28.578 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:01:28.579 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:01:28.680 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:01:28.715 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:01:28.723 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:01:28.724 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:01:28.825 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:01:28.866 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:01:28.872 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:01:28.873 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:01:28.974 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:01:29.012 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:01:29.013 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:01:29.014 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:01:29.115 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:01:29.153 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:01:29.155 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:01:29.156 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:01:29.186 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 48 16 72047 72031 93.7705 70.875 0.00317968 0.0106564 2023-06-03T20:01:29.257 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:01:29.285 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:01:29.292 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:01:29.293 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:01:29.394 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:01:29.427 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:01:29.430 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:01:29.431 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:01:29.532 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:01:29.570 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:01:29.573 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:01:29.573 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:01:29.674 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:01:29.682 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:01:29 smithi089 bash[75068]: cluster 1685822488.4472094 mgr.y (mgr.34104) 755 : cluster 0 pgmap v667: 89 pgs: 89 active+clean; 4.3 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 86 MiB/s wr, 1.38k op/s 2023-06-03T20:01:29.683 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:01:29 smithi089 bash[78452]: cluster 1685822488.4472094 mgr.y (mgr.34104) 755 : cluster 0 pgmap v667: 89 pgs: 89 active+clean; 4.3 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 86 MiB/s wr, 1.38k op/s 2023-06-03T20:01:29.709 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:01:29.712 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:01:29.713 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:01:29.814 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:01:29.834 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:01:29 smithi164 bash[59462]: cluster 1685822488.4472094 mgr.y (mgr.34104) 755 : cluster 0 pgmap v667: 89 pgs: 89 active+clean; 4.3 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 86 MiB/s wr, 1.38k op/s 2023-06-03T20:01:29.844 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:01:29.846 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:01:29.847 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:01:29.948 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:01:29.971 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:01:29.972 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:01:29.973 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:01:30.074 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:01:30.107 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:01:30.111 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:01:30.112 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:01:30.186 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 49 16 73116 73100 93.2203 66.8125 0.0152753 0.0107219 2023-06-03T20:01:30.213 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:01:30.244 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:01:30.248 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:01:30.249 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:01:30.350 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:01:30.379 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:01:30.381 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:01:30.382 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:01:30.483 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:01:30.519 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:01:30.528 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:01:30.529 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:01:30.629 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:01:30.669 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:01:30.676 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:01:30.677 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:01:30.778 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:01:30.810 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:01:30.813 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:01:30.813 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:01:30.914 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:01:30.945 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:01:30.948 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:01:30.949 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:01:31.050 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:01:31.079 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:01:31.081 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:01:31.082 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:01:31.183 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:01:31.186 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 50 16 74240 74224 92.7608 70.25 0.00727455 0.0107741 2023-06-03T20:01:31.214 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:01:31.218 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:01:31.219 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:01:31.320 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:01:31.349 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:01:31.352 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:01:31.353 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:01:31.453 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:01:31.493 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:01:31.495 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:01:31.495 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:01:31.596 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:01:31.628 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:01:31.629 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:01:31.629 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:01:31.730 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:01:31.756 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:01:31.758 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:01:31.759 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:01:31.833 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:01:31 smithi164 bash[59462]: cluster 1685822490.4476488 mgr.y (mgr.34104) 756 : cluster 0 pgmap v668: 89 pgs: 89 active+clean; 4.3 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 60 MiB/s wr, 967 op/s 2023-06-03T20:01:31.859 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:01:31.868 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:01:31 smithi089 bash[78452]: cluster 1685822490.4476488 mgr.y (mgr.34104) 756 : cluster 0 pgmap v668: 89 pgs: 89 active+clean; 4.3 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 60 MiB/s wr, 967 op/s 2023-06-03T20:01:31.868 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:01:31 smithi089 bash[75068]: cluster 1685822490.4476488 mgr.y (mgr.34104) 756 : cluster 0 pgmap v668: 89 pgs: 89 active+clean; 4.3 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 60 MiB/s wr, 967 op/s 2023-06-03T20:01:31.893 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:01:31.897 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:01:31.898 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:01:32.000 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:01:32.031 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:01:32.034 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:01:32.035 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:01:32.136 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:01:32.168 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:01:32.170 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:01:32.171 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:01:32.186 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 51 16 75287 75271 92.225 65.4375 0.0207123 0.0108368 2023-06-03T20:01:32.272 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:01:32.303 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:01:32.305 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:01:32.306 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:01:32.407 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:01:32.446 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:01:32.448 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:01:32.449 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:01:32.550 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:01:32.589 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:01:32.593 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:01:32.594 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:01:32.695 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:01:32.726 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:01:32.728 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:01:32.729 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:01:32.830 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:01:32.863 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:01:32.865 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:01:32.866 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:01:32.967 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:01:32.996 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:01:33.003 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:01:33.004 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:01:33.105 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:01:33.187 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 52 16 76383 76367 91.7687 68.5 0.00841559 0.010892 2023-06-03T20:01:33.876 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:01:33.882 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:01:33.882 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:01:33 smithi164 bash[59462]: cluster 1685822492.4480674 mgr.y (mgr.34104) 757 : cluster 0 pgmap v669: 89 pgs: 89 active+clean; 4.4 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 66 MiB/s wr, 1.05k op/s 2023-06-03T20:01:33.883 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:01:33.984 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:01:34.029 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:01:33 smithi089 bash[78452]: cluster 1685822492.4480674 mgr.y (mgr.34104) 757 : cluster 0 pgmap v669: 89 pgs: 89 active+clean; 4.4 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 66 MiB/s wr, 1.05k op/s 2023-06-03T20:01:34.029 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:01:33 smithi089 bash[75068]: cluster 1685822492.4480674 mgr.y (mgr.34104) 757 : cluster 0 pgmap v669: 89 pgs: 89 active+clean; 4.4 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 66 MiB/s wr, 1.05k op/s 2023-06-03T20:01:34.053 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:01:34.058 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:01:34.058 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:01:34.159 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:01:34.187 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 53 16 77476 77460 91.3261 68.3125 0.014525 0.0109445 2023-06-03T20:01:34.209 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:01:34.213 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:01:34.214 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:01:34.315 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:01:34.362 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:01:34.367 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:01:34.368 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:01:34.469 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:01:34.513 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:01:34.515 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:01:34.516 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:01:34.617 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:01:34.648 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:01:34.652 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:01:34.653 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:01:34.754 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:01:34.788 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:01:34.790 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:01:34.791 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:01:34.891 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:01:34.930 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:01:34.934 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:01:34.934 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:01:35.035 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:01:35.067 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:01:35.069 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:01:35.069 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:01:35.170 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:01:35.187 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 54 16 78520 78504 90.8431 65.25 0.0163041 0.0110024 2023-06-03T20:01:35.210 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:01:35.216 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:01:35.216 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:01:35.317 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:01:35.346 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:01:35.349 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:01:35.350 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:01:35.450 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:01:35.483 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:01:35.486 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:01:35.487 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:01:35.588 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:01:35.622 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:01:35.624 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:01:35.625 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:01:35.726 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:01:35.753 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:01:35 smithi164 bash[59462]: cluster 1685822494.448601 mgr.y (mgr.34104) 758 : cluster 0 pgmap v670: 89 pgs: 89 active+clean; 4.6 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 83 MiB/s wr, 1.33k op/s 2023-06-03T20:01:35.784 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:01:35.788 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:01:35.789 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:01:35.889 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:01:35.925 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:01:35.926 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:01:35.927 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:01:36.028 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:01:36.029 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:01:35 smithi089 bash[75068]: cluster 1685822494.448601 mgr.y (mgr.34104) 758 : cluster 0 pgmap v670: 89 pgs: 89 active+clean; 4.6 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 83 MiB/s wr, 1.33k op/s 2023-06-03T20:01:36.030 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:01:35 smithi089 bash[78452]: cluster 1685822494.448601 mgr.y (mgr.34104) 758 : cluster 0 pgmap v670: 89 pgs: 89 active+clean; 4.6 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 83 MiB/s wr, 1.33k op/s 2023-06-03T20:01:36.066 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:01:36.069 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:01:36.070 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:01:36.171 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:01:36.187 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 55 16 79547 79531 90.3584 64.1875 0.0370921 0.0110613 2023-06-03T20:01:36.213 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:01:36.218 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:01:36.219 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:01:36.320 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:01:36.357 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:01:36.360 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:01:36.360 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:01:36.461 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:01:36.506 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:01:36.508 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:01:36.509 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:01:36.528 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 20:01:36 smithi089 bash[71163]: ::ffff:172.21.15.164 - - [03/Jun/2023:20:01:36] "GET /metrics HTTP/1.1" 200 228233 "" "Prometheus/2.33.4" 2023-06-03T20:01:36.610 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:01:36.646 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:01:36.651 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:01:36.652 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:01:36.753 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:01:36.783 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:01:36.785 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:01:36.787 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:01:36.888 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:01:36.923 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:01:36.925 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:01:36.926 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:01:37.027 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:01:37.061 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:01:37.063 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:01:37.064 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:01:37.165 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:01:37.187 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 56 16 80513 80497 89.823 60.375 0.0222271 0.011127 2023-06-03T20:01:37.197 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:01:37.202 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:01:37.203 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:01:37.304 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:01:37.344 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:01:37.346 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:01:37.347 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:01:37.448 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:01:37.482 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:01:37.485 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:01:37.485 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:01:37.586 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:01:37.626 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:01:37.630 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:01:37.631 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:01:37.732 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:01:37.767 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:01:37.772 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:01:37.773 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:01:37.834 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:01:37 smithi164 bash[59462]: cluster 1685822496.449215 mgr.y (mgr.34104) 759 : cluster 0 pgmap v671: 89 pgs: 89 active+clean; 4.7 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 63 MiB/s wr, 1.01k op/s 2023-06-03T20:01:37.834 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:01:37 smithi164 bash[59462]: audit 1685822497.0951734 mon.c (mon.2) 389 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T20:01:37.873 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:01:37.882 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:01:37 smithi089 bash[75068]: cluster 1685822496.449215 mgr.y (mgr.34104) 759 : cluster 0 pgmap v671: 89 pgs: 89 active+clean; 4.7 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 63 MiB/s wr, 1.01k op/s 2023-06-03T20:01:37.882 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:01:37 smithi089 bash[75068]: audit 1685822497.0951734 mon.c (mon.2) 389 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T20:01:37.883 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:01:37 smithi089 bash[78452]: cluster 1685822496.449215 mgr.y (mgr.34104) 759 : cluster 0 pgmap v671: 89 pgs: 89 active+clean; 4.7 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 63 MiB/s wr, 1.01k op/s 2023-06-03T20:01:37.883 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:01:37 smithi089 bash[78452]: audit 1685822497.0951734 mon.c (mon.2) 389 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T20:01:37.907 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:01:37.908 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:01:37.909 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:01:38.010 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:01:38.049 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:01:38.055 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:01:38.055 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:01:38.156 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:01:38.187 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 57 16 81513 81497 89.3435 62.5 0.0103851 0.0111875 2023-06-03T20:01:38.211 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:01:38.212 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:01:38.213 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:01:38.313 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:01:38.343 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:01:38.348 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:01:38.349 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:01:38.450 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:01:38.499 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:01:38.506 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:01:38.507 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:01:38.608 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:01:38.642 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:01:38.644 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:01:38.645 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:01:38.746 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:01:38.779 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:01:38.781 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:01:38.782 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:01:38.828 INFO:teuthology.orchestra.run.smithi089.stdout:ERROR: (22) Invalid argument 2023-06-03T20:01:38.828 INFO:teuthology.orchestra.run.smithi089.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-06-03T20:01:38.883 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:01:38.895 INFO:teuthology.orchestra.run.smithi089.stdout:osd.0: {} 2023-06-03T20:01:38.919 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:01:38.925 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:01:38.926 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:01:38.947 INFO:teuthology.orchestra.run.smithi089.stdout:osd.1: {} 2023-06-03T20:01:38.987 INFO:teuthology.orchestra.run.smithi089.stdout:osd.2: {} 2023-06-03T20:01:39.027 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:01:39.037 INFO:teuthology.orchestra.run.smithi089.stdout:osd.3: {} 2023-06-03T20:01:39.058 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:01:39.060 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:01:39.061 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:01:39.077 INFO:teuthology.orchestra.run.smithi089.stdout:osd.4: {} 2023-06-03T20:01:39.117 INFO:teuthology.orchestra.run.smithi089.stdout:osd.5: {} 2023-06-03T20:01:39.161 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:01:39.167 INFO:teuthology.orchestra.run.smithi089.stdout:osd.6: {} 2023-06-03T20:01:39.192 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 58 16 82333 82317 88.6787 51.25 0.0248658 0.0112672 2023-06-03T20:01:39.196 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:01:39.202 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:01:39.203 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:01:39.206 INFO:teuthology.orchestra.run.smithi089.stdout:osd.7: {} 2023-06-03T20:01:39.214 INFO:teuthology.orchestra.run.smithi089.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-06-03T20:01:39.214 INFO:teuthology.orchestra.run.smithi089.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-06-03T20:01:39.214 INFO:teuthology.orchestra.run.smithi089.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-06-03T20:01:39.215 INFO:teuthology.orchestra.run.smithi089.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-06-03T20:01:39.215 INFO:teuthology.orchestra.run.smithi089.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-06-03T20:01:39.215 INFO:teuthology.orchestra.run.smithi089.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-06-03T20:01:39.215 INFO:teuthology.orchestra.run.smithi089.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-06-03T20:01:39.215 INFO:teuthology.orchestra.run.smithi089.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-06-03T20:01:39.304 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:01:39.333 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:01:39.337 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:01:39.338 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:01:39.439 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:01:39.471 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:01:39.476 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:01:39.477 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:01:39.578 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:01:39.608 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:01:39.612 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:01:39.613 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:01:39.650 INFO:teuthology.orchestra.run.smithi089.stderr:nodeep-scrub is unset 2023-06-03T20:01:39.666 INFO:teuthology.orchestra.run.smithi089.stderr:reweighted osd.3 to 1 (10000) 2023-06-03T20:01:39.714 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:01:39.749 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:01:39.751 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:01:39.752 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:01:39.853 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:01:39.860 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:01:39 smithi164 bash[59462]: cluster 1685822498.450008 mgr.y (mgr.34104) 760 : cluster 0 pgmap v672: 89 pgs: 89 active+clean; 4.9 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 79 MiB/s wr, 1.27k op/s 2023-06-03T20:01:39.861 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:01:39 smithi164 bash[59462]: audit 1685822498.7876503 mon.c (mon.2) 390 : audit 0 from='client.? 172.21.15.89:0/2158239345' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-03T20:01:39.861 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:01:39 smithi164 bash[59462]: audit 1685822499.241052 mon.b (mon.1) 47 : audit 1 from='client.? 172.21.15.89:0/2433678387' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-06-03T20:01:39.861 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:01:39 smithi164 bash[59462]: audit 1685822499.2437758 mon.a (mon.0) 686 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-06-03T20:01:39.861 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:01:39 smithi164 bash[59462]: audit 1685822499.2644212 mon.c (mon.2) 391 : audit 1 from='client.? 172.21.15.89:0/2282483283' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-06-03T20:01:39.861 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:01:39 smithi164 bash[59462]: audit 1685822499.2650747 mon.a (mon.0) 687 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-06-03T20:01:39.885 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:01:39.887 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:01:39.888 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:01:39.989 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:01:40.026 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:01:40.028 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:01:39 smithi089 bash[78452]: cluster 1685822498.450008 mgr.y (mgr.34104) 760 : cluster 0 pgmap v672: 89 pgs: 89 active+clean; 4.9 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 79 MiB/s wr, 1.27k op/s 2023-06-03T20:01:40.029 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:01:39 smithi089 bash[78452]: audit 1685822498.7876503 mon.c (mon.2) 390 : audit 0 from='client.? 172.21.15.89:0/2158239345' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-03T20:01:40.029 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:01:39 smithi089 bash[78452]: audit 1685822499.241052 mon.b (mon.1) 47 : audit 1 from='client.? 172.21.15.89:0/2433678387' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-06-03T20:01:40.029 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:01:39 smithi089 bash[78452]: audit 1685822499.2437758 mon.a (mon.0) 686 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-06-03T20:01:40.029 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:01:39 smithi089 bash[78452]: audit 1685822499.2644212 mon.c (mon.2) 391 : audit 1 from='client.? 172.21.15.89:0/2282483283' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-06-03T20:01:40.029 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:01:39 smithi089 bash[78452]: audit 1685822499.2650747 mon.a (mon.0) 687 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-06-03T20:01:40.029 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:01:39 smithi089 bash[75068]: cluster 1685822498.450008 mgr.y (mgr.34104) 760 : cluster 0 pgmap v672: 89 pgs: 89 active+clean; 4.9 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 79 MiB/s wr, 1.27k op/s 2023-06-03T20:01:40.030 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:01:39 smithi089 bash[75068]: audit 1685822498.7876503 mon.c (mon.2) 390 : audit 0 from='client.? 172.21.15.89:0/2158239345' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-03T20:01:40.030 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:01:39 smithi089 bash[75068]: audit 1685822499.241052 mon.b (mon.1) 47 : audit 1 from='client.? 172.21.15.89:0/2433678387' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-06-03T20:01:40.030 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:01:39 smithi089 bash[75068]: audit 1685822499.2437758 mon.a (mon.0) 686 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-06-03T20:01:40.030 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:01:39 smithi089 bash[75068]: audit 1685822499.2644212 mon.c (mon.2) 391 : audit 1 from='client.? 172.21.15.89:0/2282483283' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-06-03T20:01:40.030 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:01:39 smithi089 bash[75068]: audit 1685822499.2650747 mon.a (mon.0) 687 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-06-03T20:01:40.033 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:01:40.034 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:01:40.135 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:01:40.167 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:01:40.169 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:01:40.170 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:01:40.193 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 59 16 83289 83273 88.1884 59.75 0.0183765 0.0113325 2023-06-03T20:01:40.271 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:01:40.542 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:01:40.545 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:01:40.546 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:01:40.647 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:01:40.678 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:01:40.683 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:01:40.684 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:01:40.785 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:01:40.794 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:01:40 smithi164 bash[59462]: cluster 1685822499.6251657 mon.a (mon.0) 688 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-06-03T20:01:40.794 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:01:40 smithi164 bash[59462]: cluster 1685822499.6251972 mon.a (mon.0) 689 : cluster 1 Cluster is now healthy 2023-06-03T20:01:40.794 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:01:40 smithi164 bash[59462]: cluster 1685822499.634164 osd.6 (osd.6) 53 : cluster 3 failed to encode map e134 with expected crc 2023-06-03T20:01:40.794 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:01:40 smithi164 bash[59462]: audit 1685822499.6379771 mon.a (mon.0) 690 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-06-03T20:01:40.795 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:01:40 smithi164 bash[59462]: audit 1685822499.638122 mon.a (mon.0) 691 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2023-06-03T20:01:40.795 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:01:40 smithi164 bash[59462]: cluster 1685822499.6382072 mon.a (mon.0) 692 : cluster 0 osdmap e134: 8 total, 8 up, 8 in 2023-06-03T20:01:40.795 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:01:40 smithi164 bash[59462]: cluster 1685822499.6491385 osd.5 (osd.5) 57 : cluster 3 failed to encode map e134 with expected crc 2023-06-03T20:01:40.795 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:01:40 smithi164 bash[59462]: cluster 1685822499.6501677 osd.7 (osd.7) 46 : cluster 3 failed to encode map e134 with expected crc 2023-06-03T20:01:40.795 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:01:40 smithi164 bash[59462]: cluster 1685822499.650702 osd.5 (osd.5) 58 : cluster 3 failed to encode map e134 with expected crc 2023-06-03T20:01:40.795 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:01:40 smithi164 bash[59462]: cluster 1685822499.6522665 osd.7 (osd.7) 47 : cluster 3 failed to encode map e134 with expected crc 2023-06-03T20:01:40.814 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:01:40.817 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:01:40.818 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:01:40.919 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:01:40.951 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:01:40.956 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:01:40.957 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:01:40.970 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:01:40 smithi089 bash[75068]: cluster 1685822499.6251657 mon.a (mon.0) 688 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-06-03T20:01:40.970 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:01:40 smithi089 bash[75068]: cluster 1685822499.6251972 mon.a (mon.0) 689 : cluster 1 Cluster is now healthy 2023-06-03T20:01:40.970 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:01:40 smithi089 bash[75068]: cluster 1685822499.634164 osd.6 (osd.6) 53 : cluster 3 failed to encode map e134 with expected crc 2023-06-03T20:01:40.971 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:01:40 smithi089 bash[75068]: audit 1685822499.6379771 mon.a (mon.0) 690 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-06-03T20:01:40.971 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:01:40 smithi089 bash[75068]: audit 1685822499.638122 mon.a (mon.0) 691 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2023-06-03T20:01:40.971 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:01:40 smithi089 bash[75068]: cluster 1685822499.6382072 mon.a (mon.0) 692 : cluster 0 osdmap e134: 8 total, 8 up, 8 in 2023-06-03T20:01:40.971 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:01:40 smithi089 bash[75068]: cluster 1685822499.6491385 osd.5 (osd.5) 57 : cluster 3 failed to encode map e134 with expected crc 2023-06-03T20:01:40.971 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:01:40 smithi089 bash[75068]: cluster 1685822499.6501677 osd.7 (osd.7) 46 : cluster 3 failed to encode map e134 with expected crc 2023-06-03T20:01:40.971 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:01:40 smithi089 bash[75068]: cluster 1685822499.650702 osd.5 (osd.5) 58 : cluster 3 failed to encode map e134 with expected crc 2023-06-03T20:01:40.971 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:01:40 smithi089 bash[75068]: cluster 1685822499.6522665 osd.7 (osd.7) 47 : cluster 3 failed to encode map e134 with expected crc 2023-06-03T20:01:40.972 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:01:40 smithi089 bash[78452]: cluster 1685822499.6251657 mon.a (mon.0) 688 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-06-03T20:01:40.972 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:01:40 smithi089 bash[78452]: cluster 1685822499.6251972 mon.a (mon.0) 689 : cluster 1 Cluster is now healthy 2023-06-03T20:01:40.972 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:01:40 smithi089 bash[78452]: cluster 1685822499.634164 osd.6 (osd.6) 53 : cluster 3 failed to encode map e134 with expected crc 2023-06-03T20:01:40.972 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:01:40 smithi089 bash[78452]: audit 1685822499.6379771 mon.a (mon.0) 690 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-06-03T20:01:40.972 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:01:40 smithi089 bash[78452]: audit 1685822499.638122 mon.a (mon.0) 691 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2023-06-03T20:01:40.973 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:01:40 smithi089 bash[78452]: cluster 1685822499.6382072 mon.a (mon.0) 692 : cluster 0 osdmap e134: 8 total, 8 up, 8 in 2023-06-03T20:01:40.973 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:01:40 smithi089 bash[78452]: cluster 1685822499.6491385 osd.5 (osd.5) 57 : cluster 3 failed to encode map e134 with expected crc 2023-06-03T20:01:40.973 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:01:40 smithi089 bash[78452]: cluster 1685822499.6501677 osd.7 (osd.7) 46 : cluster 3 failed to encode map e134 with expected crc 2023-06-03T20:01:40.973 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:01:40 smithi089 bash[78452]: cluster 1685822499.650702 osd.5 (osd.5) 58 : cluster 3 failed to encode map e134 with expected crc 2023-06-03T20:01:40.973 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:01:40 smithi089 bash[78452]: cluster 1685822499.6522665 osd.7 (osd.7) 47 : cluster 3 failed to encode map e134 with expected crc 2023-06-03T20:01:41.058 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:01:41.085 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:01:41.086 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:01:41.086 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:01:41.187 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:01:41.192 INFO:tasks.radosbench.radosbench.0.smithi089.stdout:2023-06-03T20:01:41.188912+0000 min lat: 0.00125194 max lat: 0.378924 avg lat: 0.0113933 2023-06-03T20:01:41.193 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-06-03T20:01:41.193 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 60 16 84253 84237 87.7227 60.25 0.0252985 0.0113933 2023-06-03T20:01:41.220 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:01:41.222 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:01:41.223 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:01:41.324 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:01:41.345 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:01:41.346 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:01:41.347 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:01:41.448 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:01:41.486 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:01:41.488 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:01:41.489 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:01:41.590 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:01:41.624 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:01:41.629 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:01:41.629 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:01:41.730 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:01:41.743 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:01:41 smithi164 bash[59462]: cluster 1685822500.4504802 mgr.y (mgr.34104) 761 : cluster 0 pgmap v674: 89 pgs: 89 active+clean; 4.9 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 67 MiB/s wr, 1.06k op/s 2023-06-03T20:01:41.765 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:01:41.773 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:01:41.774 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:01:41.778 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:01:41 smithi089 bash[75068]: cluster 1685822500.4504802 mgr.y (mgr.34104) 761 : cluster 0 pgmap v674: 89 pgs: 89 active+clean; 4.9 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 67 MiB/s wr, 1.06k op/s 2023-06-03T20:01:41.779 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:01:41 smithi089 bash[78452]: cluster 1685822500.4504802 mgr.y (mgr.34104) 761 : cluster 0 pgmap v674: 89 pgs: 89 active+clean; 4.9 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 67 MiB/s wr, 1.06k op/s 2023-06-03T20:01:41.876 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:01:41.898 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:01:41.899 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:01:41.900 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:01:41.955 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph daemon osd.1 dump_historic_ops 2023-06-03T20:01:42.001 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:01:42.041 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:01:42.047 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:01:42.048 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:01:42.148 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:01:42.186 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:01:42.193 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 61 16 84977 84961 87.0265 45.25 0.0186136 0.0114857 2023-06-03T20:01:42.194 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:01:42.195 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:01:42.296 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:01:42.336 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:01:42.341 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:01:42.342 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:01:42.443 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:01:42.486 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:01:42.490 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:01:42.491 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:01:42.592 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:01:42.631 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:01:42.634 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:01:42.635 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:01:42.736 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:01:42.783 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:01:42.786 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:01:42.786 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:01:42.887 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:01:42.920 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:01:42.926 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:01:42.926 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:01:43.027 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:01:43.062 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:01:43.065 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:01:43.066 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:01:43.166 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:01:43.193 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 62 16 85900 85884 86.5529 57.6875 0.0282012 0.0115464 2023-06-03T20:01:43.203 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:01:43.207 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:01:43.208 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:01:43.309 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:01:43.346 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:01:43.435 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:01:43.436 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:01:43.537 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:01:43.585 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph osd reweight 4 1 2023-06-03T20:01:43.590 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:01:43.590 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:01:43.591 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:01:43.693 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:01:43.726 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:01:43.730 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:01:43.731 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:01:43.832 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:01:43.873 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:01:43.876 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:01:43.877 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:01:43.978 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:01:43.989 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:01:43 smithi089 bash[78452]: cluster 1685822502.451142 mgr.y (mgr.34104) 762 : cluster 0 pgmap v675: 89 pgs: 89 active+clean; 5.0 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 66 MiB/s wr, 1.05k op/s 2023-06-03T20:01:43.989 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:01:43 smithi089 bash[75068]: cluster 1685822502.451142 mgr.y (mgr.34104) 762 : cluster 0 pgmap v675: 89 pgs: 89 active+clean; 5.0 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 66 MiB/s wr, 1.05k op/s 2023-06-03T20:01:44.012 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:01:44.017 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:01:44.018 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:01:44.083 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:01:43 smithi164 bash[59462]: cluster 1685822502.451142 mgr.y (mgr.34104) 762 : cluster 0 pgmap v675: 89 pgs: 89 active+clean; 5.0 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 66 MiB/s wr, 1.05k op/s 2023-06-03T20:01:44.119 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:01:44.152 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:01:44.155 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:01:44.155 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:01:44.193 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 63 16 86849 86833 86.1205 59.3125 0.0135404 0.0116059 2023-06-03T20:01:44.257 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:01:44.296 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:01:44.299 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:01:44.300 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:01:44.401 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:01:44.431 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:01:44.435 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:01:44.436 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:01:44.537 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:01:44.573 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:01:44.577 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:01:44.578 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:01:44.679 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:01:44.724 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:01:44.728 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:01:44.728 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:01:44.752 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-06-03T20:01:44.829 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:01:44.867 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:01:44.869 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:01:44.870 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:01:44.970 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:01:45.006 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:01:45.028 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:01:45.029 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:01:45.130 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:01:45.179 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:01:45.183 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:01:45.184 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:01:45.193 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 64 16 87796 87780 85.6996 59.1875 0.0108241 0.0116633 2023-06-03T20:01:45.285 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:01:45.321 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:01:45.326 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:01:45.326 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:01:45.427 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:01:45.463 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:01:45.469 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:01:45.469 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:01:45.536 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph osd set noscrub 2023-06-03T20:01:45.571 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:01:45.608 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:01:45.613 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:01:45.614 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:01:45.715 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:01:45.748 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:01:45.752 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:01:45.753 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:01:45.779 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:01:45 smithi089 bash[75068]: cluster 1685822504.4519246 mgr.y (mgr.34104) 763 : cluster 0 pgmap v676: 89 pgs: 89 active+clean; 5.2 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 60 MiB/s wr, 960 op/s 2023-06-03T20:01:45.779 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:01:45 smithi089 bash[78452]: cluster 1685822504.4519246 mgr.y (mgr.34104) 763 : cluster 0 pgmap v676: 89 pgs: 89 active+clean; 5.2 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 60 MiB/s wr, 960 op/s 2023-06-03T20:01:45.833 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:01:45 smithi164 bash[59462]: cluster 1685822504.4519246 mgr.y (mgr.34104) 763 : cluster 0 pgmap v676: 89 pgs: 89 active+clean; 5.2 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 60 MiB/s wr, 960 op/s 2023-06-03T20:01:45.854 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:01:45.891 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:01:45.896 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:01:45.897 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:01:45.998 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:01:46.028 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:01:46.031 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:01:46.032 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:01:46.133 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:01:46.172 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:01:46.175 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:01:46.176 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:01:46.193 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 65 16 88727 88711 85.2764 58.1875 0.0105603 0.0117206 2023-06-03T20:01:46.277 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:01:46.311 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:01:46.315 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:01:46.315 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:01:46.416 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:01:46.426 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 20:01:46 smithi089 bash[71163]: ::ffff:172.21.15.164 - - [03/Jun/2023:20:01:46] "GET /metrics HTTP/1.1" 200 228237 "" "Prometheus/2.33.4" 2023-06-03T20:01:46.457 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:01:46.460 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:01:46.461 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:01:46.561 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:01:46.609 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:01:46.612 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:01:46.613 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:01:46.714 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:01:46.755 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:01:46.757 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:01:46.758 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:01:46.859 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:01:46.889 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:01:46.892 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:01:46.892 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:01:46.993 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:01:47.033 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:01:47.039 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:01:47.039 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:01:47.130 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/mon.a/config 2023-06-03T20:01:47.140 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:01:47.195 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 66 16 89676 89660 84.8817 59.3125 0.0110293 0.0117737 2023-06-03T20:01:47.195 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:01:47.197 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:01:47.198 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:01:47.299 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:01:47.327 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:01:47.333 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:01:47.334 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:01:47.435 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:01:47.538 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:01:47.541 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:01:47.541 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:01:47.642 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:01:47.677 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:01:47.682 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:01:47.683 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:01:47.784 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:01:47.825 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:01:47.831 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:01:47.832 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:01:47.933 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:01:47.943 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:01:47 smithi164 bash[59462]: cluster 1685822506.4524565 mgr.y (mgr.34104) 764 : cluster 0 pgmap v677: 89 pgs: 89 active+clean; 5.3 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 59 MiB/s wr, 948 op/s 2023-06-03T20:01:47.976 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:01:47.982 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:01:47.983 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:01:48.028 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:01:47 smithi089 bash[75068]: cluster 1685822506.4524565 mgr.y (mgr.34104) 764 : cluster 0 pgmap v677: 89 pgs: 89 active+clean; 5.3 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 59 MiB/s wr, 948 op/s 2023-06-03T20:01:48.029 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:01:47 smithi089 bash[78452]: cluster 1685822506.4524565 mgr.y (mgr.34104) 764 : cluster 0 pgmap v677: 89 pgs: 89 active+clean; 5.3 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 59 MiB/s wr, 948 op/s 2023-06-03T20:01:48.083 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:01:48.130 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:01:48.134 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:01:48.135 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:01:48.160 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/mon.a/config 2023-06-03T20:01:48.196 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 67 16 90617 90601 84.4902 58.8125 0.0267983 0.01183 2023-06-03T20:01:48.236 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:01:48.272 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:01:48.277 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:01:48.278 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:01:48.379 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:01:48.421 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:01:48.423 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:01:48.423 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:01:48.524 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:01:48.560 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:01:48.563 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:01:48.564 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:01:48.665 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:01:48.708 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:01:48.710 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:01:48.711 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:01:48.812 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:01:48.849 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:01:48.855 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:01:48.856 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:01:48.957 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:01:49.016 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:01:49.018 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:01:49.019 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:01:49.120 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:01:49.159 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:01:49.163 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:01:49.163 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:01:49.197 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 68 16 91563 91547 84.1168 59.125 0.0100784 0.0118823 2023-06-03T20:01:49.198 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/mon.a/config 2023-06-03T20:01:49.265 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:01:49.312 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:01:49.315 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:01:49.316 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:01:49.417 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:01:49.445 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:01:49.450 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:01:49.451 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:01:49.552 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:01:49.578 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:01:49.580 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:01:49.581 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:01:49.682 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:01:49.724 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:01:49.729 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:01:49.729 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:01:49.778 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:01:49 smithi089 bash[78452]: cluster 1685822508.4533026 mgr.y (mgr.34104) 765 : cluster 0 pgmap v678: 89 pgs: 89 active+clean; 5.5 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 57 MiB/s wr, 917 op/s 2023-06-03T20:01:49.779 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:01:49 smithi089 bash[75068]: cluster 1685822508.4533026 mgr.y (mgr.34104) 765 : cluster 0 pgmap v678: 89 pgs: 89 active+clean; 5.5 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 57 MiB/s wr, 917 op/s 2023-06-03T20:01:49.831 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:01:49.864 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:01:49.868 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:01:49.869 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:01:49.970 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:01:49.997 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:01:50.002 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:01:50.002 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:01:50.083 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:01:49 smithi164 bash[59462]: cluster 1685822508.4533026 mgr.y (mgr.34104) 765 : cluster 0 pgmap v678: 89 pgs: 89 active+clean; 5.5 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 57 MiB/s wr, 917 op/s 2023-06-03T20:01:50.103 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:01:50.141 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:01:50.145 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:01:50.145 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:01:50.197 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 69 16 92379 92363 83.6369 51 0.00855777 0.0119513 2023-06-03T20:01:50.220 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/mon.a/config 2023-06-03T20:01:50.246 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:01:50.275 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:01:50.279 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:01:50.280 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:01:50.381 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:01:50.416 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:01:50.422 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:01:50.422 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:01:50.523 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:01:50.556 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:01:50.559 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:01:50.560 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:01:50.661 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:01:50.700 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:01:50.705 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:01:50.706 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:01:50.807 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:01:50.844 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:01:50.848 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:01:50.849 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:01:50.950 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:01:50.983 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:01:50.988 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:01:50.989 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:01:51.090 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:01:51.123 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:01:51.125 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:01:51.125 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:01:51.197 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 70 16 93306 93290 83.2697 57.9375 0.0138287 0.0120034 2023-06-03T20:01:51.226 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:01:51.258 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:01:51.267 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:01:51.268 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:01:51.368 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:01:51.412 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:01:51.416 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:01:51.417 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:01:51.518 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:01:51.555 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:01:51.565 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:01:51.566 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:01:51.667 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:01:51.700 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:01:51.704 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:01:51.705 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:01:51.806 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:01:51.833 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:01:51 smithi164 bash[59462]: cluster 1685822510.4538367 mgr.y (mgr.34104) 766 : cluster 0 pgmap v679: 89 pgs: 89 active+clean; 5.5 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 53 MiB/s wr, 848 op/s 2023-06-03T20:01:51.840 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:01:51.843 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:01:51.843 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:01:51.945 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:01:51.953 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:01:51 smithi089 bash[78452]: cluster 1685822510.4538367 mgr.y (mgr.34104) 766 : cluster 0 pgmap v679: 89 pgs: 89 active+clean; 5.5 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 53 MiB/s wr, 848 op/s 2023-06-03T20:01:51.954 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:01:51 smithi089 bash[75068]: cluster 1685822510.4538367 mgr.y (mgr.34104) 766 : cluster 0 pgmap v679: 89 pgs: 89 active+clean; 5.5 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 53 MiB/s wr, 848 op/s 2023-06-03T20:01:51.976 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:01:51.981 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:01:51.982 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:01:52.083 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:01:52.120 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:01:52.125 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:01:52.126 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:01:52.197 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 71 16 94175 94159 82.8619 54.3125 0.00710457 0.012062 2023-06-03T20:01:52.227 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:01:52.253 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:01:52.256 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:01:52.257 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:01:52.358 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:01:52.389 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:01:52.393 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:01:52.394 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:01:52.494 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:01:52.536 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:01:52.538 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:01:52.539 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:01:52.640 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:01:52.682 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:01:52.684 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:01:52.685 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:01:52.786 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:01:52.822 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:01:52.825 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:01:52.826 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:01:52.927 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:01:52.967 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:01:52.969 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:01:52.970 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:01:53.071 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:01:53.106 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:01:53.109 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:01:53.110 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:01:53.111 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:01:53 smithi089 bash[78452]: audit 1685822512.101859 mon.a (mon.0) 693 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:01:53.111 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:01:53 smithi089 bash[78452]: audit 1685822512.1034524 mon.c (mon.2) 392 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T20:01:53.112 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:01:53 smithi089 bash[78452]: cluster 1685822512.4545302 mgr.y (mgr.34104) 767 : cluster 0 pgmap v680: 89 pgs: 89 active+clean; 5.6 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 52 MiB/s wr, 837 op/s 2023-06-03T20:01:53.112 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:01:53 smithi089 bash[75068]: audit 1685822512.101859 mon.a (mon.0) 693 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:01:53.112 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:01:53 smithi089 bash[75068]: audit 1685822512.1034524 mon.c (mon.2) 392 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T20:01:53.112 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:01:53 smithi089 bash[75068]: cluster 1685822512.4545302 mgr.y (mgr.34104) 767 : cluster 0 pgmap v680: 89 pgs: 89 active+clean; 5.6 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 52 MiB/s wr, 837 op/s 2023-06-03T20:01:53.197 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 72 16 95096 95080 82.5105 57.5625 0.0161262 0.0121149 2023-06-03T20:01:53.211 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:01:53.247 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:01:53.249 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:01:53.249 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:01:53.334 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:01:53 smithi164 bash[59462]: audit 1685822512.101859 mon.a (mon.0) 693 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:01:53.334 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:01:53 smithi164 bash[59462]: audit 1685822512.1034524 mon.c (mon.2) 392 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T20:01:53.334 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:01:53 smithi164 bash[59462]: cluster 1685822512.4545302 mgr.y (mgr.34104) 767 : cluster 0 pgmap v680: 89 pgs: 89 active+clean; 5.6 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 52 MiB/s wr, 837 op/s 2023-06-03T20:01:53.350 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:01:53.389 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:01:53.393 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:01:53.394 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:01:53.495 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:01:53.540 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:01:53.541 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:01:53.541 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:01:53.642 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:01:53.683 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:01:53.687 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:01:53.688 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:01:53.789 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:01:53.821 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:01:53.824 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:01:53.825 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:01:53.926 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:01:53.968 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:01:53.976 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:01:53.977 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:01:54.077 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:01:54.122 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:01:54.125 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:01:54.126 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:01:54.197 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 73 16 95986 95970 82.1422 55.625 0.0240777 0.0121676 2023-06-03T20:01:54.227 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:01:54.263 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:01:54.271 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:01:54.272 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:01:54.373 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:01:54.407 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:01:54.413 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:01:54.413 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:01:54.514 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:01:54.562 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:01:54.569 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:01:54.570 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:01:54.671 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:01:54.701 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:01:54.702 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:01:54.703 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:01:54.804 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:01:54.838 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:01:54.840 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:01:54.840 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:01:54.941 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:01:54.983 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:01:54.987 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:01:54.988 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:01:55.089 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:01:55.122 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:01:55.126 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:01:55.127 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:01:55.197 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 74 16 96891 96875 81.7966 56.5625 0.015452 0.0122203 2023-06-03T20:01:55.228 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:01:55.263 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:01:55.267 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:01:55.267 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:01:55.368 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:01:55.410 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:01:55.421 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:01:55.422 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:01:55.523 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:01:55.552 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:01:55.555 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:01:55.555 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:01:55.656 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:01:55.704 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:01:55.712 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:01:55.713 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:01:55.778 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:01:55 smithi089 bash[78452]: cluster 1685822514.4551282 mgr.y (mgr.34104) 768 : cluster 0 pgmap v681: 89 pgs: 89 active+clean; 5.8 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 66 MiB/s wr, 1.06k op/s 2023-06-03T20:01:55.779 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:01:55 smithi089 bash[75068]: cluster 1685822514.4551282 mgr.y (mgr.34104) 768 : cluster 0 pgmap v681: 89 pgs: 89 active+clean; 5.8 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 66 MiB/s wr, 1.06k op/s 2023-06-03T20:01:55.813 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:01:55.823 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:01:55 smithi164 bash[59462]: cluster 1685822514.4551282 mgr.y (mgr.34104) 768 : cluster 0 pgmap v681: 89 pgs: 89 active+clean; 5.8 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 66 MiB/s wr, 1.06k op/s 2023-06-03T20:01:55.849 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:01:55.852 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:01:55.852 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:01:55.953 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:01:55.987 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:01:55.988 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:01:55.988 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:01:56.089 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:01:56.123 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:01:56.125 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:01:56.126 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:01:56.197 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 75 16 97761 97745 81.4309 54.375 0.00807216 0.012275 2023-06-03T20:01:56.227 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:01:56.260 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:01:56.263 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:01:56.263 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:01:56.364 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:01:56.405 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:01:56.412 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:01:56.413 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:01:56.513 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:01:56.523 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 20:01:56 smithi089 bash[71163]: ::ffff:172.21.15.164 - - [03/Jun/2023:20:01:56] "GET /metrics HTTP/1.1" 200 228258 "" "Prometheus/2.33.4" 2023-06-03T20:01:56.560 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:01:56.565 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:01:56.566 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:01:56.667 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:01:56.707 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:01:56.712 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:01:56.713 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:01:56.814 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:01:56.846 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:01:56.848 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:01:56.849 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:01:56.950 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:01:56.991 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:01:56.993 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:01:56.994 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:01:57.095 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:01:57.136 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:01:57.140 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:01:57.141 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:01:57.197 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 76 16 98663 98647 81.1012 56.375 0.0190256 0.0123246 2023-06-03T20:01:57.242 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:01:57.282 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:01:57.286 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:01:57.286 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:01:57.387 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:01:57.431 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:01:57.437 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:01:57.437 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:01:57.521 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:01:57 smithi089 bash[78452]: cluster 1685822516.4556475 mgr.y (mgr.34104) 769 : cluster 0 pgmap v682: 89 pgs: 89 active+clean; 5.8 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 52 MiB/s wr, 837 op/s 2023-06-03T20:01:57.528 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:01:57 smithi164 bash[59462]: cluster 1685822516.4556475 mgr.y (mgr.34104) 769 : cluster 0 pgmap v682: 89 pgs: 89 active+clean; 5.8 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 52 MiB/s wr, 837 op/s 2023-06-03T20:01:57.538 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:01:57.570 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:01:57.574 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:01:57.576 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:01:57.677 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:01:57.778 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:01:57 smithi089 bash[75068]: cluster 1685822516.4556475 mgr.y (mgr.34104) 769 : cluster 0 pgmap v682: 89 pgs: 89 active+clean; 5.8 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 52 MiB/s wr, 837 op/s 2023-06-03T20:01:57.815 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:01:57.822 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:01:57.822 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:01:57.923 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:01:57.969 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:01:57.971 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:01:57.972 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:01:58.073 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:01:58.112 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:01:58.118 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:01:58.118 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:01:58.197 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 77 16 99578 99562 80.7907 57.1875 0.0224464 0.0123722 2023-06-03T20:01:58.219 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:01:58.252 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:01:58.255 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:01:58.256 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:01:58.356 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:01:58.389 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:01:58.392 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:01:58.393 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:01:58.495 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:01:58.532 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:01:58.535 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:01:58.536 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:01:58.637 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:01:58.671 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:01:58.673 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:01:58.674 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:01:58.775 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:01:58.806 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:01:58.808 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:01:58.809 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:01:58.910 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:01:58.938 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:01:58.942 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:01:58.943 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:01:59.044 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:01:59.073 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:01:59.080 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:01:59.081 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:01:59.182 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:01:59.200 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 78 16 100453 100437 80.4549 54.6875 0.0127223 0.0124235 2023-06-03T20:01:59.220 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:01:59.226 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:01:59.227 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:01:59.328 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:01:59.359 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:01:59.361 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:01:59.362 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:01:59.463 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:01:59.484 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:01:59.487 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:01:59.488 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:01:59.589 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:01:59.643 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:01:59.645 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:01:59.645 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:01:59.746 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:01:59.756 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:01:59 smithi089 bash[78452]: cluster 1685822518.4564095 mgr.y (mgr.34104) 770 : cluster 0 pgmap v683: 89 pgs: 89 active+clean; 6.1 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 67 MiB/s wr, 1.07k op/s 2023-06-03T20:01:59.756 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:01:59 smithi089 bash[75068]: cluster 1685822518.4564095 mgr.y (mgr.34104) 770 : cluster 0 pgmap v683: 89 pgs: 89 active+clean; 6.1 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 67 MiB/s wr, 1.07k op/s 2023-06-03T20:01:59.788 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:01:59.792 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:01:59.793 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:01:59.834 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:01:59 smithi164 bash[59462]: cluster 1685822518.4564095 mgr.y (mgr.34104) 770 : cluster 0 pgmap v683: 89 pgs: 89 active+clean; 6.1 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 67 MiB/s wr, 1.07k op/s 2023-06-03T20:01:59.895 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:01:59.947 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:01:59.951 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:01:59.952 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:02:00.052 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:02:00.088 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:02:00.089 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:02:00.090 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:02:00.191 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:02:00.199 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 79 16 101266 101250 80.0797 50.8125 0.0190063 0.0124823 2023-06-03T20:02:00.221 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:02:00.229 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:02:00.230 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:02:00.331 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:02:00.359 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:02:00.363 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:02:00.364 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:02:00.465 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:02:00.504 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:02:00.513 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:02:00.514 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:02:00.615 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:02:00.653 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:02:00.661 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:02:00.662 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:02:00.763 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:02:00.798 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:02:00.800 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:02:00.801 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:02:00.901 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:02:00.944 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:02:00.945 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:02:00.946 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:02:01.047 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:02:01.080 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:02:01.087 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:02:01.087 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:02:01.188 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:02:01.199 INFO:tasks.radosbench.radosbench.0.smithi089.stdout:2023-06-03T20:02:01.195326+0000 min lat: 0.00125194 max lat: 0.378924 avg lat: 0.0125376 2023-06-03T20:02:01.199 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-06-03T20:02:01.199 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 80 16 102089 102073 79.7217 51.4375 0.0132188 0.0125376 2023-06-03T20:02:01.228 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:02:01.232 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:02:01.232 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:02:01.333 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:02:01.373 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:02:01.378 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:02:01.379 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:02:01.479 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:02:01.507 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:02:01.510 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:02:01.510 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:02:01.611 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:02:01.632 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:02:01 smithi089 bash[78452]: cluster 1685822520.4577234 mgr.y (mgr.34104) 771 : cluster 0 pgmap v684: 89 pgs: 89 active+clean; 6.1 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 47 MiB/s wr, 749 op/s 2023-06-03T20:02:01.633 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:02:01 smithi089 bash[75068]: cluster 1685822520.4577234 mgr.y (mgr.34104) 771 : cluster 0 pgmap v684: 89 pgs: 89 active+clean; 6.1 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 47 MiB/s wr, 749 op/s 2023-06-03T20:02:01.642 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:02:01.644 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:02:01.645 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:02:01.746 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:02:01.783 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:02:01.787 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:02:01.787 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:02:01.833 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:02:01 smithi164 bash[59462]: cluster 1685822520.4577234 mgr.y (mgr.34104) 771 : cluster 0 pgmap v684: 89 pgs: 89 active+clean; 6.1 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 47 MiB/s wr, 749 op/s 2023-06-03T20:02:01.888 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:02:01.931 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:02:01.933 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:02:01.934 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:02:02.035 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:02:02.068 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:02:02.072 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:02:02.073 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:02:02.174 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:02:02.199 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 81 16 102937 102921 79.3918 53 0.0164001 0.0125897 2023-06-03T20:02:02.216 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:02:02.219 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:02:02.220 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:02:02.321 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:02:02.356 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:02:02.358 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:02:02.359 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:02:02.459 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:02:02.496 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:02:02.501 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:02:02.502 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:02:02.603 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:02:02.632 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:02:02.639 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:02:02.640 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:02:02.741 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:02:02.780 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:02:02.780 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:02:02.781 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:02:02.882 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:02:02.919 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:02:02.923 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:02:02.924 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:02:03.025 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:02:03.061 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:02:03.064 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:02:03.065 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:02:03.166 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:02:03.199 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 82 16 103768 103752 79.057 51.9375 0.017204 0.0126435 2023-06-03T20:02:03.202 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:02:03.207 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:02:03.208 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:02:03.309 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:02:03.344 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:02:03.354 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:02:03.354 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:02:03.455 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:02:03.489 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:02:03.493 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:02:03.493 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:02:03.569 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:02:03 smithi164 bash[59462]: cluster 1685822522.4584715 mgr.y (mgr.34104) 772 : cluster 0 pgmap v685: 89 pgs: 89 active+clean; 6.1 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 51 MiB/s wr, 817 op/s 2023-06-03T20:02:03.595 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:02:03.622 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:02:03.625 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:02:03.626 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:02:03.727 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:02:03.734 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:02:03 smithi089 bash[75068]: cluster 1685822522.4584715 mgr.y (mgr.34104) 772 : cluster 0 pgmap v685: 89 pgs: 89 active+clean; 6.1 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 51 MiB/s wr, 817 op/s 2023-06-03T20:02:03.735 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:02:03 smithi089 bash[78452]: cluster 1685822522.4584715 mgr.y (mgr.34104) 772 : cluster 0 pgmap v685: 89 pgs: 89 active+clean; 6.1 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 51 MiB/s wr, 817 op/s 2023-06-03T20:02:03.762 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:02:03.768 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:02:03.769 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:02:03.869 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:02:03.908 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:02:03.910 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:02:03.910 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:02:04.011 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:02:04.046 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:02:04.050 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:02:04.051 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:02:04.152 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:02:04.189 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:02:04.194 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:02:04.194 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:02:04.199 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 83 16 104650 104634 78.7686 55.125 0.00865879 0.0126897 2023-06-03T20:02:04.295 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:02:04.325 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:02:04.327 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:02:04.327 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:02:04.379 INFO:teuthology.orchestra.run.smithi089.stdout:{ 2023-06-03T20:02:04.380 INFO:teuthology.orchestra.run.smithi089.stdout: "size": 20, 2023-06-03T20:02:04.380 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 600, 2023-06-03T20:02:04.380 INFO:teuthology.orchestra.run.smithi089.stdout: "ops": [ 2023-06-03T20:02:04.380 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:02:04.380 INFO:teuthology.orchestra.run.smithi089.stdout: "description": "osd_op(client.44762.0:107486 8.1 8:8c0200c8:::benchmark_data_smithi089_91006_object802:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e106)", 2023-06-03T20:02:04.381 INFO:teuthology.orchestra.run.smithi089.stdout: "initiated_at": "2023-06-03T19:52:47.357752+0000", 2023-06-03T20:02:04.381 INFO:teuthology.orchestra.run.smithi089.stdout: "age": 557.011687112, 2023-06-03T20:02:04.381 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 1.0175325369999999, 2023-06-03T20:02:04.381 INFO:teuthology.orchestra.run.smithi089.stdout: "type_data": { 2023-06-03T20:02:04.381 INFO:teuthology.orchestra.run.smithi089.stdout: "flag_point": "reached pg", 2023-06-03T20:02:04.381 INFO:teuthology.orchestra.run.smithi089.stdout: "client_info": { 2023-06-03T20:02:04.381 INFO:teuthology.orchestra.run.smithi089.stdout: "client": "client.44762", 2023-06-03T20:02:04.382 INFO:teuthology.orchestra.run.smithi089.stdout: "client_addr": "172.21.15.89:0/4205857995", 2023-06-03T20:02:04.382 INFO:teuthology.orchestra.run.smithi089.stdout: "tid": 107486 2023-06-03T20:02:04.382 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:02:04.382 INFO:teuthology.orchestra.run.smithi089.stdout: "events": [ 2023-06-03T20:02:04.382 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:02:04.382 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "initiated", 2023-06-03T20:02:04.382 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T19:52:47.357752+0000", 2023-06-03T20:02:04.382 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0 2023-06-03T20:02:04.383 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:02:04.383 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:02:04.383 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "throttled", 2023-06-03T20:02:04.383 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T19:52:47.357752+0000", 2023-06-03T20:02:04.383 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0 2023-06-03T20:02:04.383 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:02:04.383 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:02:04.383 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "header_read", 2023-06-03T20:02:04.384 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T19:52:47.357750+0000", 2023-06-03T20:02:04.384 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 4294967295.9999981 2023-06-03T20:02:04.384 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:02:04.384 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:02:04.384 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "all_read", 2023-06-03T20:02:04.384 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T19:52:47.357761+0000", 2023-06-03T20:02:04.384 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 1.1358e-05 2023-06-03T20:02:04.384 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:02:04.385 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:02:04.385 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "dispatched", 2023-06-03T20:02:04.385 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T19:52:47.357789+0000", 2023-06-03T20:02:04.385 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 2.7780000000000002e-05 2023-06-03T20:02:04.385 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:02:04.385 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:02:04.385 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "queued_for_pg", 2023-06-03T20:02:04.385 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T19:52:47.357804+0000", 2023-06-03T20:02:04.386 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 1.5121e-05 2023-06-03T20:02:04.386 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:02:04.386 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:02:04.386 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "reached_pg", 2023-06-03T20:02:04.386 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T19:52:47.361802+0000", 2023-06-03T20:02:04.386 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.0039981189999999996 2023-06-03T20:02:04.386 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:02:04.387 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:02:04.387 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "waiting for peered", 2023-06-03T20:02:04.387 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T19:52:47.361814+0000", 2023-06-03T20:02:04.387 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 1.1773999999999999e-05 2023-06-03T20:02:04.387 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:02:04.387 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:02:04.387 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "reached_pg", 2023-06-03T20:02:04.387 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T19:52:48.375197+0000", 2023-06-03T20:02:04.388 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 1.0133825860000001 2023-06-03T20:02:04.388 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:02:04.388 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:02:04.388 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "done", 2023-06-03T20:02:04.388 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T19:52:48.375285+0000", 2023-06-03T20:02:04.388 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 8.7745999999999996e-05 2023-06-03T20:02:04.388 INFO:teuthology.orchestra.run.smithi089.stdout: } 2023-06-03T20:02:04.388 INFO:teuthology.orchestra.run.smithi089.stdout: ] 2023-06-03T20:02:04.389 INFO:teuthology.orchestra.run.smithi089.stdout: } 2023-06-03T20:02:04.389 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:02:04.389 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:02:04.389 INFO:teuthology.orchestra.run.smithi089.stdout: "description": "osd_op(client.44762.0:107494 8.1 8:82151026:::benchmark_data_smithi089_91006_object810:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e106)", 2023-06-03T20:02:04.389 INFO:teuthology.orchestra.run.smithi089.stdout: "initiated_at": "2023-06-03T19:52:47.357813+0000", 2023-06-03T20:02:04.389 INFO:teuthology.orchestra.run.smithi089.stdout: "age": 557.01162597899997, 2023-06-03T20:02:04.389 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 1.0402689389999999, 2023-06-03T20:02:04.389 INFO:teuthology.orchestra.run.smithi089.stdout: "type_data": { 2023-06-03T20:02:04.390 INFO:teuthology.orchestra.run.smithi089.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-03T20:02:04.390 INFO:teuthology.orchestra.run.smithi089.stdout: "client_info": { 2023-06-03T20:02:04.390 INFO:teuthology.orchestra.run.smithi089.stdout: "client": "client.44762", 2023-06-03T20:02:04.390 INFO:teuthology.orchestra.run.smithi089.stdout: "client_addr": "172.21.15.89:0/4205857995", 2023-06-03T20:02:04.390 INFO:teuthology.orchestra.run.smithi089.stdout: "tid": 107494 2023-06-03T20:02:04.390 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:02:04.390 INFO:teuthology.orchestra.run.smithi089.stdout: "events": [ 2023-06-03T20:02:04.390 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:02:04.391 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "initiated", 2023-06-03T20:02:04.391 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T19:52:47.357813+0000", 2023-06-03T20:02:04.391 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0 2023-06-03T20:02:04.391 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:02:04.391 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:02:04.391 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "throttled", 2023-06-03T20:02:04.391 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T19:52:47.357813+0000", 2023-06-03T20:02:04.391 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0 2023-06-03T20:02:04.392 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:02:04.392 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:02:04.392 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "header_read", 2023-06-03T20:02:04.392 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T19:52:47.357812+0000", 2023-06-03T20:02:04.392 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 4294967295.9999995 2023-06-03T20:02:04.392 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:02:04.392 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:02:04.392 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "all_read", 2023-06-03T20:02:04.393 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T19:52:47.357816+0000", 2023-06-03T20:02:04.393 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 3.7069999999999998e-06 2023-06-03T20:02:04.393 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:02:04.393 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:02:04.393 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "dispatched", 2023-06-03T20:02:04.393 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T19:52:47.357824+0000", 2023-06-03T20:02:04.393 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 8.1710000000000002e-06 2023-06-03T20:02:04.393 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:02:04.394 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:02:04.394 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "queued_for_pg", 2023-06-03T20:02:04.394 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T19:52:47.357832+0000", 2023-06-03T20:02:04.394 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 7.9959999999999993e-06 2023-06-03T20:02:04.394 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:02:04.394 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:02:04.394 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "reached_pg", 2023-06-03T20:02:04.394 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T19:52:47.363013+0000", 2023-06-03T20:02:04.395 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.0051801549999999997 2023-06-03T20:02:04.395 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:02:04.395 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:02:04.395 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "waiting for peered", 2023-06-03T20:02:04.395 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T19:52:47.363023+0000", 2023-06-03T20:02:04.395 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 1.0047999999999999e-05 2023-06-03T20:02:04.395 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:02:04.395 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:02:04.396 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "reached_pg", 2023-06-03T20:02:04.396 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T19:52:48.375802+0000", 2023-06-03T20:02:04.396 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 1.0127793359999999 2023-06-03T20:02:04.396 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:02:04.396 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:02:04.396 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "started", 2023-06-03T20:02:04.396 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T19:52:48.375940+0000", 2023-06-03T20:02:04.397 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.00013856299999999999 2023-06-03T20:02:04.397 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:02:04.397 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:02:04.397 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "waiting for subops from 0,2", 2023-06-03T20:02:04.397 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T19:52:48.376071+0000", 2023-06-03T20:02:04.397 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.00013101700000000001 2023-06-03T20:02:04.397 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:02:04.397 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:02:04.398 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "op_commit", 2023-06-03T20:02:04.398 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T19:52:48.384748+0000", 2023-06-03T20:02:04.398 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.0086766369999999992 2023-06-03T20:02:04.398 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:02:04.398 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:02:04.398 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:02:04.398 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T19:52:48.397939+0000", 2023-06-03T20:02:04.398 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.013191075999999999 2023-06-03T20:02:04.399 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:02:04.399 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:02:04.399 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:02:04.399 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T19:52:48.397985+0000", 2023-06-03T20:02:04.399 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 4.5666e-05 2023-06-03T20:02:04.399 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:02:04.399 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:02:04.399 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "commit_sent", 2023-06-03T20:02:04.400 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T19:52:48.398042+0000", 2023-06-03T20:02:04.400 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 5.7352000000000002e-05 2023-06-03T20:02:04.400 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:02:04.400 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:02:04.400 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "done", 2023-06-03T20:02:04.400 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T19:52:48.398082+0000", 2023-06-03T20:02:04.400 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 3.9824999999999999e-05 2023-06-03T20:02:04.400 INFO:teuthology.orchestra.run.smithi089.stdout: } 2023-06-03T20:02:04.401 INFO:teuthology.orchestra.run.smithi089.stdout: ] 2023-06-03T20:02:04.401 INFO:teuthology.orchestra.run.smithi089.stdout: } 2023-06-03T20:02:04.401 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:02:04.401 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:02:04.401 INFO:teuthology.orchestra.run.smithi089.stdout: "description": "osd_op(client.44762.0:107504 8.1 8:8bd6f2fe:::benchmark_data_smithi089_91006_object820:head [delete] snapc 0=[] ondisk+write+known_if_redirected e106)", 2023-06-03T20:02:04.401 INFO:teuthology.orchestra.run.smithi089.stdout: "initiated_at": "2023-06-03T19:52:47.365968+0000", 2023-06-03T20:02:04.401 INFO:teuthology.orchestra.run.smithi089.stdout: "age": 557.00347061299999, 2023-06-03T20:02:04.402 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 1.034849103, 2023-06-03T20:02:04.402 INFO:teuthology.orchestra.run.smithi089.stdout: "type_data": { 2023-06-03T20:02:04.402 INFO:teuthology.orchestra.run.smithi089.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-03T20:02:04.402 INFO:teuthology.orchestra.run.smithi089.stdout: "client_info": { 2023-06-03T20:02:04.402 INFO:teuthology.orchestra.run.smithi089.stdout: "client": "client.44762", 2023-06-03T20:02:04.402 INFO:teuthology.orchestra.run.smithi089.stdout: "client_addr": "172.21.15.89:0/4205857995", 2023-06-03T20:02:04.402 INFO:teuthology.orchestra.run.smithi089.stdout: "tid": 107504 2023-06-03T20:02:04.402 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:02:04.403 INFO:teuthology.orchestra.run.smithi089.stdout: "events": [ 2023-06-03T20:02:04.403 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:02:04.403 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "initiated", 2023-06-03T20:02:04.403 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T19:52:47.365968+0000", 2023-06-03T20:02:04.403 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0 2023-06-03T20:02:04.403 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:02:04.403 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:02:04.403 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "throttled", 2023-06-03T20:02:04.404 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T19:52:47.365968+0000", 2023-06-03T20:02:04.404 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0 2023-06-03T20:02:04.404 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:02:04.404 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:02:04.404 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "header_read", 2023-06-03T20:02:04.404 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T19:52:47.365967+0000", 2023-06-03T20:02:04.404 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 4294967295.9999986 2023-06-03T20:02:04.405 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:02:04.405 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:02:04.405 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "all_read", 2023-06-03T20:02:04.405 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T19:52:47.365972+0000", 2023-06-03T20:02:04.405 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 5.2159999999999997e-06 2023-06-03T20:02:04.405 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:02:04.405 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:02:04.405 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "dispatched", 2023-06-03T20:02:04.406 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T19:52:47.365985+0000", 2023-06-03T20:02:04.406 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 1.2286999999999999e-05 2023-06-03T20:02:04.406 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:02:04.406 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:02:04.406 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "queued_for_pg", 2023-06-03T20:02:04.406 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T19:52:47.365995+0000", 2023-06-03T20:02:04.406 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 9.9529999999999994e-06 2023-06-03T20:02:04.406 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:02:04.407 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:02:04.407 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "reached_pg", 2023-06-03T20:02:04.407 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T19:52:47.366048+0000", 2023-06-03T20:02:04.407 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 5.3007999999999998e-05 2023-06-03T20:02:04.407 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:02:04.408 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:02:04.408 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "waiting for peered", 2023-06-03T20:02:04.408 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T19:52:47.366066+0000", 2023-06-03T20:02:04.408 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 1.7997999999999999e-05 2023-06-03T20:02:04.408 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:02:04.408 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:02:04.408 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "reached_pg", 2023-06-03T20:02:04.409 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T19:52:48.378374+0000", 2023-06-03T20:02:04.409 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 1.0123083660000001 2023-06-03T20:02:04.409 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:02:04.409 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:02:04.409 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "started", 2023-06-03T20:02:04.409 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T19:52:48.378489+0000", 2023-06-03T20:02:04.409 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.000114808 2023-06-03T20:02:04.409 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:02:04.410 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:02:04.410 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "waiting for subops from 0,2", 2023-06-03T20:02:04.410 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T19:52:48.378612+0000", 2023-06-03T20:02:04.410 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.000122846 2023-06-03T20:02:04.410 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:02:04.410 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:02:04.410 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "op_commit", 2023-06-03T20:02:04.410 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T19:52:48.391884+0000", 2023-06-03T20:02:04.411 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.013272393 2023-06-03T20:02:04.411 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:02:04.411 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:02:04.411 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:02:04.411 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T19:52:48.398106+0000", 2023-06-03T20:02:04.411 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.0062220390000000004 2023-06-03T20:02:04.411 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:02:04.411 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:02:04.412 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:02:04.412 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T19:52:48.400732+0000", 2023-06-03T20:02:04.412 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.0026256270000000002 2023-06-03T20:02:04.412 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:02:04.412 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:02:04.412 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "commit_sent", 2023-06-03T20:02:04.412 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T19:52:48.400776+0000", 2023-06-03T20:02:04.412 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 4.3961000000000001e-05 2023-06-03T20:02:04.413 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:02:04.413 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:02:04.413 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "done", 2023-06-03T20:02:04.413 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T19:52:48.400818+0000", 2023-06-03T20:02:04.413 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 4.1909999999999997e-05 2023-06-03T20:02:04.413 INFO:teuthology.orchestra.run.smithi089.stdout: } 2023-06-03T20:02:04.413 INFO:teuthology.orchestra.run.smithi089.stdout: ] 2023-06-03T20:02:04.413 INFO:teuthology.orchestra.run.smithi089.stdout: } 2023-06-03T20:02:04.414 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:02:04.414 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:02:04.414 INFO:teuthology.orchestra.run.smithi089.stdout: "description": "osd_op(client.44762.0:107514 8.1 8:85d3eacf:::benchmark_data_smithi089_91006_object830:head [delete] snapc 0=[] ondisk+write+known_if_redirected e106)", 2023-06-03T20:02:04.414 INFO:teuthology.orchestra.run.smithi089.stdout: "initiated_at": "2023-06-03T19:52:47.388013+0000", 2023-06-03T20:02:04.414 INFO:teuthology.orchestra.run.smithi089.stdout: "age": 556.98142604199995, 2023-06-03T20:02:04.414 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 1.019510033, 2023-06-03T20:02:04.414 INFO:teuthology.orchestra.run.smithi089.stdout: "type_data": { 2023-06-03T20:02:04.414 INFO:teuthology.orchestra.run.smithi089.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-03T20:02:04.415 INFO:teuthology.orchestra.run.smithi089.stdout: "client_info": { 2023-06-03T20:02:04.415 INFO:teuthology.orchestra.run.smithi089.stdout: "client": "client.44762", 2023-06-03T20:02:04.415 INFO:teuthology.orchestra.run.smithi089.stdout: "client_addr": "172.21.15.89:0/4205857995", 2023-06-03T20:02:04.415 INFO:teuthology.orchestra.run.smithi089.stdout: "tid": 107514 2023-06-03T20:02:04.415 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:02:04.415 INFO:teuthology.orchestra.run.smithi089.stdout: "events": [ 2023-06-03T20:02:04.415 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:02:04.415 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "initiated", 2023-06-03T20:02:04.416 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T19:52:47.388013+0000", 2023-06-03T20:02:04.416 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0 2023-06-03T20:02:04.416 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:02:04.416 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:02:04.416 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "throttled", 2023-06-03T20:02:04.416 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T19:52:47.388013+0000", 2023-06-03T20:02:04.416 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0 2023-06-03T20:02:04.417 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:02:04.417 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:02:04.417 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "header_read", 2023-06-03T20:02:04.417 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T19:52:47.388011+0000", 2023-06-03T20:02:04.417 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 4294967295.9999976 2023-06-03T20:02:04.417 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:02:04.417 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:02:04.417 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "all_read", 2023-06-03T20:02:04.418 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T19:52:47.388019+0000", 2023-06-03T20:02:04.418 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 8.1599999999999998e-06 2023-06-03T20:02:04.418 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:02:04.418 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:02:04.418 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "dispatched", 2023-06-03T20:02:04.418 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T19:52:47.388033+0000", 2023-06-03T20:02:04.418 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 1.3988999999999999e-05 2023-06-03T20:02:04.418 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:02:04.418 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:02:04.419 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "queued_for_pg", 2023-06-03T20:02:04.419 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T19:52:47.388040+0000", 2023-06-03T20:02:04.419 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 7.0899999999999999e-06 2023-06-03T20:02:04.419 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:02:04.419 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:02:04.419 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "reached_pg", 2023-06-03T20:02:04.419 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T19:52:47.388087+0000", 2023-06-03T20:02:04.420 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 4.7314999999999997e-05 2023-06-03T20:02:04.420 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:02:04.420 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:02:04.420 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "waiting for peered", 2023-06-03T20:02:04.420 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T19:52:47.388100+0000", 2023-06-03T20:02:04.420 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 1.2398000000000001e-05 2023-06-03T20:02:04.420 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:02:04.420 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:02:04.421 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "reached_pg", 2023-06-03T20:02:04.421 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T19:52:48.381610+0000", 2023-06-03T20:02:04.421 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.993510483 2023-06-03T20:02:04.421 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:02:04.421 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:02:04.421 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "started", 2023-06-03T20:02:04.421 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T19:52:48.381752+0000", 2023-06-03T20:02:04.421 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.0001417 2023-06-03T20:02:04.422 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:02:04.422 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:02:04.422 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "waiting for subops from 0,2", 2023-06-03T20:02:04.422 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T19:52:48.381887+0000", 2023-06-03T20:02:04.422 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.00013497199999999999 2023-06-03T20:02:04.422 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:02:04.422 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:02:04.422 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "op_commit", 2023-06-03T20:02:04.423 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T19:52:48.400651+0000", 2023-06-03T20:02:04.423 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.018763698999999998 2023-06-03T20:02:04.423 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:02:04.423 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:02:04.423 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:02:04.423 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T19:52:48.400865+0000", 2023-06-03T20:02:04.423 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.00021408700000000001 2023-06-03T20:02:04.423 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:02:04.424 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:02:04.424 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:02:04.424 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T19:52:48.407396+0000", 2023-06-03T20:02:04.424 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.0065308629999999996 2023-06-03T20:02:04.424 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:02:04.424 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:02:04.424 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "commit_sent", 2023-06-03T20:02:04.424 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T19:52:48.407475+0000", 2023-06-03T20:02:04.425 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 7.9138e-05 2023-06-03T20:02:04.425 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:02:04.425 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:02:04.425 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "done", 2023-06-03T20:02:04.425 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T19:52:48.407523+0000", 2023-06-03T20:02:04.425 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 4.8362000000000002e-05 2023-06-03T20:02:04.425 INFO:teuthology.orchestra.run.smithi089.stdout: } 2023-06-03T20:02:04.425 INFO:teuthology.orchestra.run.smithi089.stdout: ] 2023-06-03T20:02:04.426 INFO:teuthology.orchestra.run.smithi089.stdout: } 2023-06-03T20:02:04.426 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:02:04.426 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:02:04.426 INFO:teuthology.orchestra.run.smithi089.stdout: "description": "osd_op(client.44762.0:171252 8.3 8:c72d852f:::benchmark_data_smithi089_91006_object64568:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e108)", 2023-06-03T20:02:04.426 INFO:teuthology.orchestra.run.smithi089.stdout: "initiated_at": "2023-06-03T19:54:22.405940+0000", 2023-06-03T20:02:04.427 INFO:teuthology.orchestra.run.smithi089.stdout: "age": 461.963498716, 2023-06-03T20:02:04.427 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 1.0466815570000001, 2023-06-03T20:02:04.427 INFO:teuthology.orchestra.run.smithi089.stdout: "type_data": { 2023-06-03T20:02:04.427 INFO:teuthology.orchestra.run.smithi089.stdout: "flag_point": "reached pg", 2023-06-03T20:02:04.427 INFO:teuthology.orchestra.run.smithi089.stdout: "client_info": { 2023-06-03T20:02:04.427 INFO:teuthology.orchestra.run.smithi089.stdout: "client": "client.44762", 2023-06-03T20:02:04.427 INFO:teuthology.orchestra.run.smithi089.stdout: "client_addr": "172.21.15.89:0/4205857995", 2023-06-03T20:02:04.427 INFO:teuthology.orchestra.run.smithi089.stdout: "tid": 171252 2023-06-03T20:02:04.428 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:02:04.428 INFO:teuthology.orchestra.run.smithi089.stdout: "events": [ 2023-06-03T20:02:04.428 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:02:04.428 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "initiated", 2023-06-03T20:02:04.428 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T19:54:22.405940+0000", 2023-06-03T20:02:04.428 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0 2023-06-03T20:02:04.428 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:02:04.429 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:02:04.429 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "throttled", 2023-06-03T20:02:04.429 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T19:54:22.405940+0000", 2023-06-03T20:02:04.429 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0 2023-06-03T20:02:04.429 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:02:04.429 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:02:04.429 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "header_read", 2023-06-03T20:02:04.429 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T19:54:22.405939+0000", 2023-06-03T20:02:04.430 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 4294967295.9999986 2023-06-03T20:02:04.430 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:02:04.430 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:02:04.430 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "all_read", 2023-06-03T20:02:04.430 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T19:54:22.405944+0000", 2023-06-03T20:02:04.431 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 5.339e-06 2023-06-03T20:02:04.431 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:02:04.431 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:02:04.431 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "dispatched", 2023-06-03T20:02:04.431 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T19:54:22.405967+0000", 2023-06-03T20:02:04.431 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 2.2682000000000001e-05 2023-06-03T20:02:04.431 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:02:04.431 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:02:04.432 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "queued_for_pg", 2023-06-03T20:02:04.432 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T19:54:22.405979+0000", 2023-06-03T20:02:04.432 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 1.2558000000000001e-05 2023-06-03T20:02:04.432 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:02:04.432 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:02:04.432 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "reached_pg", 2023-06-03T20:02:04.432 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T19:54:22.407754+0000", 2023-06-03T20:02:04.432 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.0017742789999999999 2023-06-03T20:02:04.433 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:02:04.433 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:02:04.433 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "op must wait for map", 2023-06-03T20:02:04.433 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T19:54:22.407778+0000", 2023-06-03T20:02:04.433 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 2.474e-05 2023-06-03T20:02:04.433 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:02:04.433 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:02:04.434 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "reached_pg", 2023-06-03T20:02:04.434 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T19:54:22.417051+0000", 2023-06-03T20:02:04.434 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.0092725670000000007 2023-06-03T20:02:04.434 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:02:04.434 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:02:04.434 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "waiting for peered", 2023-06-03T20:02:04.434 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T19:54:22.417061+0000", 2023-06-03T20:02:04.434 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 9.9210000000000004e-06 2023-06-03T20:02:04.435 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:02:04.435 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:02:04.435 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "reached_pg", 2023-06-03T20:02:04.435 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T19:54:23.452462+0000", 2023-06-03T20:02:04.435 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 1.035401112 2023-06-03T20:02:04.435 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:02:04.435 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:02:04.435 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "done", 2023-06-03T20:02:04.436 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T19:54:23.452622+0000", 2023-06-03T20:02:04.436 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.00016001500000000001 2023-06-03T20:02:04.436 INFO:teuthology.orchestra.run.smithi089.stdout: } 2023-06-03T20:02:04.436 INFO:teuthology.orchestra.run.smithi089.stdout: ] 2023-06-03T20:02:04.436 INFO:teuthology.orchestra.run.smithi089.stdout: } 2023-06-03T20:02:04.436 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:02:04.436 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:02:04.436 INFO:teuthology.orchestra.run.smithi089.stdout: "description": "osd_op(client.44762.0:171263 8.3 8:c2fae0d0:::benchmark_data_smithi089_91006_object64579:head [delete] snapc 0=[] ondisk+write+known_if_redirected e108)", 2023-06-03T20:02:04.437 INFO:teuthology.orchestra.run.smithi089.stdout: "initiated_at": "2023-06-03T19:54:22.414692+0000", 2023-06-03T20:02:04.437 INFO:teuthology.orchestra.run.smithi089.stdout: "age": 461.95474730900003, 2023-06-03T20:02:04.437 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 1.079707494, 2023-06-03T20:02:04.437 INFO:teuthology.orchestra.run.smithi089.stdout: "type_data": { 2023-06-03T20:02:04.437 INFO:teuthology.orchestra.run.smithi089.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-03T20:02:04.437 INFO:teuthology.orchestra.run.smithi089.stdout: "client_info": { 2023-06-03T20:02:04.437 INFO:teuthology.orchestra.run.smithi089.stdout: "client": "client.44762", 2023-06-03T20:02:04.437 INFO:teuthology.orchestra.run.smithi089.stdout: "client_addr": "172.21.15.89:0/4205857995", 2023-06-03T20:02:04.438 INFO:teuthology.orchestra.run.smithi089.stdout: "tid": 171263 2023-06-03T20:02:04.438 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:02:04.438 INFO:teuthology.orchestra.run.smithi089.stdout: "events": [ 2023-06-03T20:02:04.438 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:02:04.438 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "initiated", 2023-06-03T20:02:04.438 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T19:54:22.414692+0000", 2023-06-03T20:02:04.438 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0 2023-06-03T20:02:04.438 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:02:04.439 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:02:04.439 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "throttled", 2023-06-03T20:02:04.439 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T19:54:22.414692+0000", 2023-06-03T20:02:04.439 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0 2023-06-03T20:02:04.439 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:02:04.439 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:02:04.439 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "header_read", 2023-06-03T20:02:04.439 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T19:54:22.414691+0000", 2023-06-03T20:02:04.440 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 4294967295.999999 2023-06-03T20:02:04.440 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:02:04.440 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:02:04.440 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "all_read", 2023-06-03T20:02:04.440 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T19:54:22.414696+0000", 2023-06-03T20:02:04.440 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 5.4759999999999997e-06 2023-06-03T20:02:04.440 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:02:04.440 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:02:04.441 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "dispatched", 2023-06-03T20:02:04.441 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T19:54:22.414706+0000", 2023-06-03T20:02:04.441 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 9.2939999999999994e-06 2023-06-03T20:02:04.441 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:02:04.441 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:02:04.441 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "queued_for_pg", 2023-06-03T20:02:04.441 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T19:54:22.414713+0000", 2023-06-03T20:02:04.441 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 7.7919999999999993e-06 2023-06-03T20:02:04.442 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:02:04.442 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:02:04.442 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "reached_pg", 2023-06-03T20:02:04.442 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T19:54:22.417083+0000", 2023-06-03T20:02:04.442 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.0023691630000000001 2023-06-03T20:02:04.442 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:02:04.442 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:02:04.443 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "waiting for peered", 2023-06-03T20:02:04.443 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T19:54:22.417090+0000", 2023-06-03T20:02:04.443 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 7.3309999999999998e-06 2023-06-03T20:02:04.443 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:02:04.443 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:02:04.443 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "reached_pg", 2023-06-03T20:02:04.443 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T19:54:23.452980+0000", 2023-06-03T20:02:04.443 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 1.035890135 2023-06-03T20:02:04.444 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:02:04.444 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:02:04.444 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "started", 2023-06-03T20:02:04.444 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T19:54:23.453085+0000", 2023-06-03T20:02:04.444 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.00010458699999999999 2023-06-03T20:02:04.444 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:02:04.444 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:02:04.444 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "waiting for subops from 5,6", 2023-06-03T20:02:04.445 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T19:54:23.453732+0000", 2023-06-03T20:02:04.445 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.00064748200000000001 2023-06-03T20:02:04.445 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:02:04.445 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:02:04.446 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:02:04.446 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T19:54:23.485257+0000", 2023-06-03T20:02:04.446 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.031525113 2023-06-03T20:02:04.447 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:02:04.447 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:02:04.447 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:02:04.447 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T19:54:23.490035+0000", 2023-06-03T20:02:04.447 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.0047778880000000001 2023-06-03T20:02:04.447 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:02:04.447 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:02:04.448 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "op_commit", 2023-06-03T20:02:04.448 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T19:54:23.494256+0000", 2023-06-03T20:02:04.448 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.0042207989999999999 2023-06-03T20:02:04.448 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:02:04.448 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:02:04.448 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "commit_sent", 2023-06-03T20:02:04.449 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T19:54:23.494330+0000", 2023-06-03T20:02:04.449 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 7.4146000000000005e-05 2023-06-03T20:02:04.449 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:02:04.449 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:02:04.449 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "done", 2023-06-03T20:02:04.449 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T19:54:23.494399+0000", 2023-06-03T20:02:04.449 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 6.9200000000000002e-05 2023-06-03T20:02:04.449 INFO:teuthology.orchestra.run.smithi089.stdout: } 2023-06-03T20:02:04.450 INFO:teuthology.orchestra.run.smithi089.stdout: ] 2023-06-03T20:02:04.450 INFO:teuthology.orchestra.run.smithi089.stdout: } 2023-06-03T20:02:04.450 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:02:04.450 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:02:04.450 INFO:teuthology.orchestra.run.smithi089.stdout: "description": "osd_op(client.44762.0:171275 8.3 8:c661d94f:::benchmark_data_smithi089_91006_object64591:head [delete] snapc 0=[] ondisk+write+known_if_redirected e108)", 2023-06-03T20:02:04.450 INFO:teuthology.orchestra.run.smithi089.stdout: "initiated_at": "2023-06-03T19:54:22.438931+0000", 2023-06-03T20:02:04.450 INFO:teuthology.orchestra.run.smithi089.stdout: "age": 461.93050850899999, 2023-06-03T20:02:04.450 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 1.0602507139999999, 2023-06-03T20:02:04.451 INFO:teuthology.orchestra.run.smithi089.stdout: "type_data": { 2023-06-03T20:02:04.451 INFO:teuthology.orchestra.run.smithi089.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-03T20:02:04.451 INFO:teuthology.orchestra.run.smithi089.stdout: "client_info": { 2023-06-03T20:02:04.451 INFO:teuthology.orchestra.run.smithi089.stdout: "client": "client.44762", 2023-06-03T20:02:04.451 INFO:teuthology.orchestra.run.smithi089.stdout: "client_addr": "172.21.15.89:0/4205857995", 2023-06-03T20:02:04.451 INFO:teuthology.orchestra.run.smithi089.stdout: "tid": 171275 2023-06-03T20:02:04.451 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:02:04.452 INFO:teuthology.orchestra.run.smithi089.stdout: "events": [ 2023-06-03T20:02:04.452 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:02:04.452 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "initiated", 2023-06-03T20:02:04.452 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T19:54:22.438931+0000", 2023-06-03T20:02:04.452 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0 2023-06-03T20:02:04.452 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:02:04.452 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:02:04.452 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "throttled", 2023-06-03T20:02:04.453 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T19:54:22.438931+0000", 2023-06-03T20:02:04.453 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0 2023-06-03T20:02:04.453 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:02:04.453 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:02:04.453 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "header_read", 2023-06-03T20:02:04.453 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T19:54:22.438929+0000", 2023-06-03T20:02:04.453 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 4294967295.9999986 2023-06-03T20:02:04.454 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:02:04.454 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:02:04.454 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "all_read", 2023-06-03T20:02:04.454 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T19:54:22.438936+0000", 2023-06-03T20:02:04.454 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 7.4250000000000001e-06 2023-06-03T20:02:04.454 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:02:04.454 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:02:04.455 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "dispatched", 2023-06-03T20:02:04.455 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T19:54:22.438959+0000", 2023-06-03T20:02:04.455 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 2.2339000000000001e-05 2023-06-03T20:02:04.455 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:02:04.455 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:02:04.455 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "queued_for_pg", 2023-06-03T20:02:04.456 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T19:54:22.438970+0000", 2023-06-03T20:02:04.456 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 1.1447999999999999e-05 2023-06-03T20:02:04.456 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:02:04.456 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:02:04.456 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "reached_pg", 2023-06-03T20:02:04.456 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T19:54:22.439950+0000", 2023-06-03T20:02:04.456 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.00097967999999999992 2023-06-03T20:02:04.457 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:02:04.457 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:02:04.457 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "waiting for peered", 2023-06-03T20:02:04.457 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T19:54:22.439962+0000", 2023-06-03T20:02:04.457 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 1.2496e-05 2023-06-03T20:02:04.457 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:02:04.457 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:02:04.457 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "reached_pg", 2023-06-03T20:02:04.458 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T19:54:23.475614+0000", 2023-06-03T20:02:04.458 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 1.0356515390000001 2023-06-03T20:02:04.458 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:02:04.458 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:02:04.458 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "started", 2023-06-03T20:02:04.458 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T19:54:23.475767+0000", 2023-06-03T20:02:04.458 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.00015329100000000001 2023-06-03T20:02:04.459 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:02:04.459 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:02:04.459 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "waiting for subops from 5,6", 2023-06-03T20:02:04.459 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T19:54:23.475925+0000", 2023-06-03T20:02:04.459 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.00015748699999999999 2023-06-03T20:02:04.459 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:02:04.459 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:02:04.459 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:02:04.460 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T19:54:23.485369+0000", 2023-06-03T20:02:04.460 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.0094438720000000007 2023-06-03T20:02:04.460 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:02:04.460 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:02:04.460 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "op_commit", 2023-06-03T20:02:04.460 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T19:54:23.494412+0000", 2023-06-03T20:02:04.460 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.0090433800000000002 2023-06-03T20:02:04.460 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:02:04.461 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:02:04.461 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:02:04.461 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T19:54:23.499106+0000", 2023-06-03T20:02:04.461 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.0046941730000000003 2023-06-03T20:02:04.461 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:02:04.461 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:02:04.461 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "commit_sent", 2023-06-03T20:02:04.461 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T19:54:23.499139+0000", 2023-06-03T20:02:04.462 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 3.2685000000000003e-05 2023-06-03T20:02:04.462 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:02:04.462 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:02:04.462 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "done", 2023-06-03T20:02:04.462 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T19:54:23.499181+0000", 2023-06-03T20:02:04.462 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 4.2545e-05 2023-06-03T20:02:04.462 INFO:teuthology.orchestra.run.smithi089.stdout: } 2023-06-03T20:02:04.462 INFO:teuthology.orchestra.run.smithi089.stdout: ] 2023-06-03T20:02:04.463 INFO:teuthology.orchestra.run.smithi089.stdout: } 2023-06-03T20:02:04.463 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:02:04.463 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:02:04.463 INFO:teuthology.orchestra.run.smithi089.stdout: "description": "osd_op(client.44762.0:171276 8.3 8:c8dc8e63:::benchmark_data_smithi089_91006_object64592:head [delete] snapc 0=[] ondisk+write+known_if_redirected e108)", 2023-06-03T20:02:04.463 INFO:teuthology.orchestra.run.smithi089.stdout: "initiated_at": "2023-06-03T19:54:22.439466+0000", 2023-06-03T20:02:04.463 INFO:teuthology.orchestra.run.smithi089.stdout: "age": 461.92997301000003, 2023-06-03T20:02:04.463 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 1.0637212680000001, 2023-06-03T20:02:04.464 INFO:teuthology.orchestra.run.smithi089.stdout: "type_data": { 2023-06-03T20:02:04.464 INFO:teuthology.orchestra.run.smithi089.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-03T20:02:04.464 INFO:teuthology.orchestra.run.smithi089.stdout: "client_info": { 2023-06-03T20:02:04.464 INFO:teuthology.orchestra.run.smithi089.stdout: "client": "client.44762", 2023-06-03T20:02:04.464 INFO:teuthology.orchestra.run.smithi089.stdout: "client_addr": "172.21.15.89:0/4205857995", 2023-06-03T20:02:04.464 INFO:teuthology.orchestra.run.smithi089.stdout: "tid": 171276 2023-06-03T20:02:04.464 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:02:04.464 INFO:teuthology.orchestra.run.smithi089.stdout: "events": [ 2023-06-03T20:02:04.465 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:02:04.465 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "initiated", 2023-06-03T20:02:04.465 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T19:54:22.439466+0000", 2023-06-03T20:02:04.465 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0 2023-06-03T20:02:04.465 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:02:04.465 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:02:04.465 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "throttled", 2023-06-03T20:02:04.466 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T19:54:22.439466+0000", 2023-06-03T20:02:04.466 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0 2023-06-03T20:02:04.466 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:02:04.466 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:02:04.466 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "header_read", 2023-06-03T20:02:04.466 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T19:54:22.439465+0000", 2023-06-03T20:02:04.466 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 4294967295.999999 2023-06-03T20:02:04.466 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:02:04.467 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:02:04.467 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "all_read", 2023-06-03T20:02:04.467 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T19:54:22.439472+0000", 2023-06-03T20:02:04.467 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 7.1330000000000001e-06 2023-06-03T20:02:04.467 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:02:04.467 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:02:04.467 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "dispatched", 2023-06-03T20:02:04.467 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T19:54:22.439489+0000", 2023-06-03T20:02:04.468 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 1.7348999999999999e-05 2023-06-03T20:02:04.468 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:02:04.468 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:02:04.468 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "queued_for_pg", 2023-06-03T20:02:04.468 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T19:54:22.439500+0000", 2023-06-03T20:02:04.468 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 1.1048e-05 2023-06-03T20:02:04.468 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:02:04.468 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:02:04.469 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "reached_pg", 2023-06-03T20:02:04.469 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T19:54:22.439984+0000", 2023-06-03T20:02:04.469 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.00048388899999999999 2023-06-03T20:02:04.469 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:02:04.469 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:02:04.469 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "waiting for peered", 2023-06-03T20:02:04.469 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T19:54:22.439991+0000", 2023-06-03T20:02:04.469 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 6.7909999999999999e-06 2023-06-03T20:02:04.470 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:02:04.470 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:02:04.470 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "reached_pg", 2023-06-03T20:02:04.470 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T19:54:23.478719+0000", 2023-06-03T20:02:04.470 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 1.038727932 2023-06-03T20:02:04.470 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:02:04.470 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:02:04.470 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "started", 2023-06-03T20:02:04.471 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T19:54:23.478861+0000", 2023-06-03T20:02:04.471 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.00014237399999999999 2023-06-03T20:02:04.471 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:02:04.471 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:02:04.471 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "waiting for subops from 5,6", 2023-06-03T20:02:04.471 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T19:54:23.479014+0000", 2023-06-03T20:02:04.471 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.00015296099999999999 2023-06-03T20:02:04.471 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:02:04.472 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:02:04.472 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:02:04.472 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T19:54:23.490012+0000", 2023-06-03T20:02:04.472 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.010997155999999999 2023-06-03T20:02:04.472 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:02:04.472 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:02:04.472 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "op_commit", 2023-06-03T20:02:04.473 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T19:54:23.494421+0000", 2023-06-03T20:02:04.473 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.0044099029999999997 2023-06-03T20:02:04.473 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:02:04.473 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:02:04.473 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:02:04.473 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T19:54:23.503098+0000", 2023-06-03T20:02:04.473 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.0086762439999999996 2023-06-03T20:02:04.473 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:02:04.474 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:02:04.474 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "commit_sent", 2023-06-03T20:02:04.474 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T19:54:23.503144+0000", 2023-06-03T20:02:04.474 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 4.6740000000000003e-05 2023-06-03T20:02:04.474 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:02:04.474 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:02:04.475 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "done", 2023-06-03T20:02:04.475 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T19:54:23.503187+0000", 2023-06-03T20:02:04.475 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 4.2886999999999998e-05 2023-06-03T20:02:04.475 INFO:teuthology.orchestra.run.smithi089.stdout: } 2023-06-03T20:02:04.475 INFO:teuthology.orchestra.run.smithi089.stdout: ] 2023-06-03T20:02:04.475 INFO:teuthology.orchestra.run.smithi089.stdout: } 2023-06-03T20:02:04.476 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:02:04.476 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:02:04.476 INFO:teuthology.orchestra.run.smithi089.stdout: "description": "osd_op(client.44762.0:171303 8.3 8:ce63af8b:::benchmark_data_smithi089_91006_object64619:head [delete] snapc 0=[] ondisk+write+known_if_redirected e108)", 2023-06-03T20:02:04.476 INFO:teuthology.orchestra.run.smithi089.stdout: "initiated_at": "2023-06-03T19:54:22.486419+0000", 2023-06-03T20:02:04.476 INFO:teuthology.orchestra.run.smithi089.stdout: "age": 461.883019795, 2023-06-03T20:02:04.476 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 1.016903962, 2023-06-03T20:02:04.476 INFO:teuthology.orchestra.run.smithi089.stdout: "type_data": { 2023-06-03T20:02:04.476 INFO:teuthology.orchestra.run.smithi089.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-03T20:02:04.477 INFO:teuthology.orchestra.run.smithi089.stdout: "client_info": { 2023-06-03T20:02:04.477 INFO:teuthology.orchestra.run.smithi089.stdout: "client": "client.44762", 2023-06-03T20:02:04.477 INFO:teuthology.orchestra.run.smithi089.stdout: "client_addr": "172.21.15.89:0/4205857995", 2023-06-03T20:02:04.477 INFO:teuthology.orchestra.run.smithi089.stdout: "tid": 171303 2023-06-03T20:02:04.477 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:02:04.477 INFO:teuthology.orchestra.run.smithi089.stdout: "events": [ 2023-06-03T20:02:04.477 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:02:04.478 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "initiated", 2023-06-03T20:02:04.478 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T19:54:22.486419+0000", 2023-06-03T20:02:04.478 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0 2023-06-03T20:02:04.478 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:02:04.478 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:02:04.478 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "throttled", 2023-06-03T20:02:04.478 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T19:54:22.486419+0000", 2023-06-03T20:02:04.478 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0 2023-06-03T20:02:04.479 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:02:04.479 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:02:04.479 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "header_read", 2023-06-03T20:02:04.479 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T19:54:22.486418+0000", 2023-06-03T20:02:04.479 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 4294967295.9999986 2023-06-03T20:02:04.479 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:02:04.479 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:02:04.480 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "all_read", 2023-06-03T20:02:04.481 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T19:54:22.486424+0000", 2023-06-03T20:02:04.481 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 6.2879999999999998e-06 2023-06-03T20:02:04.481 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:02:04.481 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:02:04.482 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "dispatched", 2023-06-03T20:02:04.482 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T19:54:22.486435+0000", 2023-06-03T20:02:04.482 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 1.075e-05 2023-06-03T20:02:04.482 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:02:04.482 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:02:04.483 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "queued_for_pg", 2023-06-03T20:02:04.483 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T19:54:22.486442+0000", 2023-06-03T20:02:04.483 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 7.0419999999999997e-06 2023-06-03T20:02:04.483 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:02:04.483 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:02:04.483 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "reached_pg", 2023-06-03T20:02:04.484 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T19:54:22.490462+0000", 2023-06-03T20:02:04.484 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.0040197200000000001 2023-06-03T20:02:04.484 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:02:04.484 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:02:04.484 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "waiting for peered", 2023-06-03T20:02:04.484 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T19:54:22.490474+0000", 2023-06-03T20:02:04.485 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 1.2609000000000001e-05 2023-06-03T20:02:04.485 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:02:04.485 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:02:04.485 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "reached_pg", 2023-06-03T20:02:04.485 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T19:54:23.482088+0000", 2023-06-03T20:02:04.485 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.99161392800000003 2023-06-03T20:02:04.486 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:02:04.486 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:02:04.486 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "started", 2023-06-03T20:02:04.486 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T19:54:23.482260+0000", 2023-06-03T20:02:04.486 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.000171514 2023-06-03T20:02:04.486 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:02:04.486 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:02:04.487 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "waiting for subops from 5,6", 2023-06-03T20:02:04.487 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T19:54:23.482360+0000", 2023-06-03T20:02:04.487 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.00010079 2023-06-03T20:02:04.487 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:02:04.487 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:02:04.487 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:02:04.487 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T19:54:23.490059+0000", 2023-06-03T20:02:04.487 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.0076987599999999998 2023-06-03T20:02:04.488 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:02:04.488 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:02:04.488 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "op_commit", 2023-06-03T20:02:04.488 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T19:54:23.494439+0000", 2023-06-03T20:02:04.488 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.0043796540000000002 2023-06-03T20:02:04.488 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:02:04.488 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:02:04.489 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:02:04.489 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T19:54:23.503252+0000", 2023-06-03T20:02:04.489 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.0088127740000000006 2023-06-03T20:02:04.489 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:02:04.489 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:02:04.489 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "commit_sent", 2023-06-03T20:02:04.489 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T19:54:23.503292+0000", 2023-06-03T20:02:04.489 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 4.0734e-05 2023-06-03T20:02:04.490 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:02:04.490 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:02:04.490 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "done", 2023-06-03T20:02:04.490 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T19:54:23.503323+0000", 2023-06-03T20:02:04.490 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 3.0948000000000003e-05 2023-06-03T20:02:04.490 INFO:teuthology.orchestra.run.smithi089.stdout: } 2023-06-03T20:02:04.490 INFO:teuthology.orchestra.run.smithi089.stdout: ] 2023-06-03T20:02:04.490 INFO:teuthology.orchestra.run.smithi089.stdout: } 2023-06-03T20:02:04.491 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:02:04.491 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:02:04.491 INFO:teuthology.orchestra.run.smithi089.stdout: "description": "osd_repop(client.45713.0:100639 9.3 e134/127 9:c7af3b91:::benchmark_data_smithi089_115580_object100638:head v 134'6231, mlcod=134'6229)", 2023-06-03T20:02:04.491 INFO:teuthology.orchestra.run.smithi089.stdout: "initiated_at": "2023-06-03T20:01:59.414423+0000", 2023-06-03T20:02:04.491 INFO:teuthology.orchestra.run.smithi089.stdout: "age": 4.9550160309999995, 2023-06-03T20:02:04.491 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.111434807, 2023-06-03T20:02:04.491 INFO:teuthology.orchestra.run.smithi089.stdout: "type_data": { 2023-06-03T20:02:04.491 INFO:teuthology.orchestra.run.smithi089.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-03T20:02:04.492 INFO:teuthology.orchestra.run.smithi089.stdout: "events": [ 2023-06-03T20:02:04.492 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:02:04.492 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "initiated", 2023-06-03T20:02:04.492 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:01:59.414423+0000", 2023-06-03T20:02:04.492 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0 2023-06-03T20:02:04.492 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:02:04.492 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:02:04.493 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "throttled", 2023-06-03T20:02:04.493 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:01:59.414423+0000", 2023-06-03T20:02:04.493 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0 2023-06-03T20:02:04.493 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:02:04.493 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:02:04.493 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "header_read", 2023-06-03T20:02:04.493 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:01:59.414422+0000", 2023-06-03T20:02:04.493 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 4294967295.9999995 2023-06-03T20:02:04.494 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:02:04.494 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:02:04.494 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "all_read", 2023-06-03T20:02:04.494 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:01:59.414569+0000", 2023-06-03T20:02:04.494 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.00014641599999999999 2023-06-03T20:02:04.494 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:02:04.494 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:02:04.494 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "dispatched", 2023-06-03T20:02:04.495 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:01:59.414579+0000", 2023-06-03T20:02:04.495 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 1.0033e-05 2023-06-03T20:02:04.495 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:02:04.495 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:02:04.495 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "queued_for_pg", 2023-06-03T20:02:04.495 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:01:59.414586+0000", 2023-06-03T20:02:04.495 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 6.9249999999999998e-06 2023-06-03T20:02:04.496 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:02:04.496 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:02:04.496 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "reached_pg", 2023-06-03T20:02:04.496 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:01:59.415544+0000", 2023-06-03T20:02:04.496 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.00095867399999999996 2023-06-03T20:02:04.496 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:02:04.496 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:02:04.497 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "started", 2023-06-03T20:02:04.497 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:01:59.415565+0000", 2023-06-03T20:02:04.497 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 2.0882000000000001e-05 2023-06-03T20:02:04.497 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:02:04.497 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:02:04.498 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "commit_sent", 2023-06-03T20:02:04.498 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:01:59.426827+0000", 2023-06-03T20:02:04.498 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.011261778 2023-06-03T20:02:04.498 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:02:04.498 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:02:04.498 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "done", 2023-06-03T20:02:04.498 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:01:59.525858+0000", 2023-06-03T20:02:04.499 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.099030813999999995 2023-06-03T20:02:04.499 INFO:teuthology.orchestra.run.smithi089.stdout: } 2023-06-03T20:02:04.499 INFO:teuthology.orchestra.run.smithi089.stdout: ] 2023-06-03T20:02:04.499 INFO:teuthology.orchestra.run.smithi089.stdout: } 2023-06-03T20:02:04.499 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:02:04.499 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:02:04.499 INFO:teuthology.orchestra.run.smithi089.stdout: "description": "osd_repop(client.45713.0:100648 9.2 e134/127 9:40b5de09:::benchmark_data_smithi089_115580_object100647:head v 134'6270, mlcod=134'6267)", 2023-06-03T20:02:04.500 INFO:teuthology.orchestra.run.smithi089.stdout: "initiated_at": "2023-06-03T20:01:59.425642+0000", 2023-06-03T20:02:04.500 INFO:teuthology.orchestra.run.smithi089.stdout: "age": 4.9437966160000002, 2023-06-03T20:02:04.500 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.11528683000000001, 2023-06-03T20:02:04.500 INFO:teuthology.orchestra.run.smithi089.stdout: "type_data": { 2023-06-03T20:02:04.500 INFO:teuthology.orchestra.run.smithi089.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-03T20:02:04.500 INFO:teuthology.orchestra.run.smithi089.stdout: "events": [ 2023-06-03T20:02:04.500 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:02:04.500 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "initiated", 2023-06-03T20:02:04.501 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:01:59.425642+0000", 2023-06-03T20:02:04.501 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0 2023-06-03T20:02:04.501 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:02:04.501 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:02:04.501 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "throttled", 2023-06-03T20:02:04.501 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:01:59.425642+0000", 2023-06-03T20:02:04.501 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0 2023-06-03T20:02:04.501 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:02:04.502 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:02:04.502 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "header_read", 2023-06-03T20:02:04.502 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:01:59.425641+0000", 2023-06-03T20:02:04.502 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 4294967295.999999 2023-06-03T20:02:04.502 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:02:04.502 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:02:04.502 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "all_read", 2023-06-03T20:02:04.502 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:01:59.425672+0000", 2023-06-03T20:02:04.503 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 3.0368000000000001e-05 2023-06-03T20:02:04.503 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:02:04.503 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:02:04.503 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "dispatched", 2023-06-03T20:02:04.503 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:01:59.425684+0000", 2023-06-03T20:02:04.504 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 1.2048999999999999e-05 2023-06-03T20:02:04.504 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:02:04.504 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:02:04.504 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "queued_for_pg", 2023-06-03T20:02:04.504 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:01:59.425692+0000", 2023-06-03T20:02:04.504 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 8.0469999999999994e-06 2023-06-03T20:02:04.504 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:02:04.505 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:02:04.505 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "reached_pg", 2023-06-03T20:02:04.505 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:01:59.425771+0000", 2023-06-03T20:02:04.505 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 7.9147999999999995e-05 2023-06-03T20:02:04.505 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:02:04.505 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:02:04.505 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "started", 2023-06-03T20:02:04.505 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:01:59.425794+0000", 2023-06-03T20:02:04.506 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 2.2827999999999999e-05 2023-06-03T20:02:04.506 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:02:04.506 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:02:04.506 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "commit_sent", 2023-06-03T20:02:04.506 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:01:59.540335+0000", 2023-06-03T20:02:04.506 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.11454133900000001 2023-06-03T20:02:04.507 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:02:04.507 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:02:04.507 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "done", 2023-06-03T20:02:04.507 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:01:59.540929+0000", 2023-06-03T20:02:04.507 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.00059411900000000005 2023-06-03T20:02:04.508 INFO:teuthology.orchestra.run.smithi089.stdout: } 2023-06-03T20:02:04.508 INFO:teuthology.orchestra.run.smithi089.stdout: ] 2023-06-03T20:02:04.508 INFO:teuthology.orchestra.run.smithi089.stdout: } 2023-06-03T20:02:04.508 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:02:04.508 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:02:04.508 INFO:teuthology.orchestra.run.smithi089.stdout: "description": "osd_op(client.45713.0:100650 9.0 9:09de3d26:::benchmark_data_smithi089_115580_object100649:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e134)", 2023-06-03T20:02:04.508 INFO:teuthology.orchestra.run.smithi089.stdout: "initiated_at": "2023-06-03T20:01:59.426196+0000", 2023-06-03T20:02:04.509 INFO:teuthology.orchestra.run.smithi089.stdout: "age": 4.9432429180000002, 2023-06-03T20:02:04.509 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.111284508, 2023-06-03T20:02:04.509 INFO:teuthology.orchestra.run.smithi089.stdout: "type_data": { 2023-06-03T20:02:04.509 INFO:teuthology.orchestra.run.smithi089.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-03T20:02:04.509 INFO:teuthology.orchestra.run.smithi089.stdout: "client_info": { 2023-06-03T20:02:04.509 INFO:teuthology.orchestra.run.smithi089.stdout: "client": "client.45713", 2023-06-03T20:02:04.509 INFO:teuthology.orchestra.run.smithi089.stdout: "client_addr": "172.21.15.89:0/2400821448", 2023-06-03T20:02:04.510 INFO:teuthology.orchestra.run.smithi089.stdout: "tid": 100650 2023-06-03T20:02:04.510 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:02:04.510 INFO:teuthology.orchestra.run.smithi089.stdout: "events": [ 2023-06-03T20:02:04.510 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:02:04.510 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "initiated", 2023-06-03T20:02:04.510 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:01:59.426196+0000", 2023-06-03T20:02:04.511 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0 2023-06-03T20:02:04.511 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:02:04.511 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:02:04.511 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "throttled", 2023-06-03T20:02:04.511 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:01:59.426196+0000", 2023-06-03T20:02:04.511 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0 2023-06-03T20:02:04.511 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:02:04.511 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:02:04.512 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "header_read", 2023-06-03T20:02:04.512 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:01:59.426195+0000", 2023-06-03T20:02:04.512 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 4294967295.9999986 2023-06-03T20:02:04.512 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:02:04.512 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:02:04.512 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "all_read", 2023-06-03T20:02:04.513 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:01:59.426221+0000", 2023-06-03T20:02:04.513 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 2.6681000000000001e-05 2023-06-03T20:02:04.513 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:02:04.513 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:02:04.513 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "dispatched", 2023-06-03T20:02:04.513 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:01:59.525899+0000", 2023-06-03T20:02:04.513 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.099677891000000005 2023-06-03T20:02:04.514 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:02:04.514 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:02:04.514 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "queued_for_pg", 2023-06-03T20:02:04.514 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:01:59.525931+0000", 2023-06-03T20:02:04.514 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 3.1214000000000003e-05 2023-06-03T20:02:04.514 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:02:04.515 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:02:04.515 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "reached_pg", 2023-06-03T20:02:04.515 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:01:59.526121+0000", 2023-06-03T20:02:04.515 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.00019075 2023-06-03T20:02:04.515 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:02:04.515 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:02:04.515 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "started", 2023-06-03T20:02:04.516 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:01:59.526276+0000", 2023-06-03T20:02:04.516 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.00015433800000000001 2023-06-03T20:02:04.516 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:02:04.516 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:02:04.516 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "waiting for subops from 2,6", 2023-06-03T20:02:04.516 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:01:59.526425+0000", 2023-06-03T20:02:04.517 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.00014928399999999999 2023-06-03T20:02:04.517 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:02:04.517 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:02:04.517 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:02:04.517 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:01:59.536347+0000", 2023-06-03T20:02:04.518 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.0099220109999999997 2023-06-03T20:02:04.518 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:02:04.518 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:02:04.518 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "op_commit", 2023-06-03T20:02:04.518 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:01:59.536365+0000", 2023-06-03T20:02:04.518 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 1.7617000000000001e-05 2023-06-03T20:02:04.518 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:02:04.518 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:02:04.519 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:02:04.519 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:01:59.536850+0000", 2023-06-03T20:02:04.519 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.000485036 2023-06-03T20:02:04.519 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:02:04.519 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:02:04.519 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "commit_sent", 2023-06-03T20:02:04.519 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:01:59.537436+0000", 2023-06-03T20:02:04.520 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.00058672499999999996 2023-06-03T20:02:04.520 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:02:04.520 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:02:04.520 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "done", 2023-06-03T20:02:04.520 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:01:59.537481+0000", 2023-06-03T20:02:04.520 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 4.4369000000000002e-05 2023-06-03T20:02:04.521 INFO:teuthology.orchestra.run.smithi089.stdout: } 2023-06-03T20:02:04.521 INFO:teuthology.orchestra.run.smithi089.stdout: ] 2023-06-03T20:02:04.521 INFO:teuthology.orchestra.run.smithi089.stdout: } 2023-06-03T20:02:04.521 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:02:04.521 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:02:04.521 INFO:teuthology.orchestra.run.smithi089.stdout: "description": "osd_repop(client.45713.0:100651 9.d e134/127 9:b6c93fda:::benchmark_data_smithi089_115580_object100650:head v 134'6425, mlcod=134'6423)", 2023-06-03T20:02:04.521 INFO:teuthology.orchestra.run.smithi089.stdout: "initiated_at": "2023-06-03T20:01:59.426752+0000", 2023-06-03T20:02:04.522 INFO:teuthology.orchestra.run.smithi089.stdout: "age": 4.9426871749999997, 2023-06-03T20:02:04.522 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.110605706, 2023-06-03T20:02:04.522 INFO:teuthology.orchestra.run.smithi089.stdout: "type_data": { 2023-06-03T20:02:04.522 INFO:teuthology.orchestra.run.smithi089.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-03T20:02:04.522 INFO:teuthology.orchestra.run.smithi089.stdout: "events": [ 2023-06-03T20:02:04.522 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:02:04.522 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "initiated", 2023-06-03T20:02:04.522 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:01:59.426752+0000", 2023-06-03T20:02:04.523 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0 2023-06-03T20:02:04.523 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:02:04.523 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:02:04.523 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "throttled", 2023-06-03T20:02:04.523 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:01:59.426752+0000", 2023-06-03T20:02:04.523 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0 2023-06-03T20:02:04.524 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:02:04.524 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:02:04.524 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "header_read", 2023-06-03T20:02:04.524 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:01:59.426751+0000", 2023-06-03T20:02:04.524 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 4294967295.999999 2023-06-03T20:02:04.524 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:02:04.525 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:02:04.525 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "all_read", 2023-06-03T20:02:04.525 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:01:59.426783+0000", 2023-06-03T20:02:04.525 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 3.1968999999999998e-05 2023-06-03T20:02:04.525 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:02:04.525 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:02:04.525 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "dispatched", 2023-06-03T20:02:04.526 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:01:59.525851+0000", 2023-06-03T20:02:04.526 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.099067890000000006 2023-06-03T20:02:04.526 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:02:04.526 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:02:04.526 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "queued_for_pg", 2023-06-03T20:02:04.526 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:01:59.525875+0000", 2023-06-03T20:02:04.526 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 2.3828e-05 2023-06-03T20:02:04.526 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:02:04.527 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:02:04.527 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "reached_pg", 2023-06-03T20:02:04.527 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:01:59.525981+0000", 2023-06-03T20:02:04.527 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.000106159 2023-06-03T20:02:04.527 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:02:04.527 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:02:04.527 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "started", 2023-06-03T20:02:04.528 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:01:59.526008+0000", 2023-06-03T20:02:04.528 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 2.7531000000000001e-05 2023-06-03T20:02:04.528 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:02:04.528 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:02:04.528 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "commit_sent", 2023-06-03T20:02:04.528 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:01:59.537313+0000", 2023-06-03T20:02:04.528 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.011304822000000001 2023-06-03T20:02:04.529 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:02:04.529 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:02:04.529 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "done", 2023-06-03T20:02:04.529 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:01:59.537358+0000", 2023-06-03T20:02:04.529 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 4.4295000000000002e-05 2023-06-03T20:02:04.529 INFO:teuthology.orchestra.run.smithi089.stdout: } 2023-06-03T20:02:04.529 INFO:teuthology.orchestra.run.smithi089.stdout: ] 2023-06-03T20:02:04.529 INFO:teuthology.orchestra.run.smithi089.stdout: } 2023-06-03T20:02:04.530 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:02:04.530 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:02:04.530 INFO:teuthology.orchestra.run.smithi089.stdout: "description": "osd_repop(client.45713.0:100663 9.2 e134/127 9:4aded491:::benchmark_data_smithi089_115580_object100662:head v 134'6271, mlcod=134'6268)", 2023-06-03T20:02:04.530 INFO:teuthology.orchestra.run.smithi089.stdout: "initiated_at": "2023-06-03T20:01:59.438393+0000", 2023-06-03T20:02:04.530 INFO:teuthology.orchestra.run.smithi089.stdout: "age": 4.931045686, 2023-06-03T20:02:04.530 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.10268160799999999, 2023-06-03T20:02:04.530 INFO:teuthology.orchestra.run.smithi089.stdout: "type_data": { 2023-06-03T20:02:04.531 INFO:teuthology.orchestra.run.smithi089.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-03T20:02:04.531 INFO:teuthology.orchestra.run.smithi089.stdout: "events": [ 2023-06-03T20:02:04.531 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:02:04.531 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "initiated", 2023-06-03T20:02:04.531 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:01:59.438393+0000", 2023-06-03T20:02:04.531 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0 2023-06-03T20:02:04.531 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:02:04.531 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:02:04.532 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "throttled", 2023-06-03T20:02:04.532 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:01:59.438393+0000", 2023-06-03T20:02:04.532 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0 2023-06-03T20:02:04.532 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:02:04.532 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:02:04.532 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "header_read", 2023-06-03T20:02:04.532 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:01:59.438392+0000", 2023-06-03T20:02:04.533 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 4294967295.999999 2023-06-03T20:02:04.533 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:02:04.533 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:02:04.533 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "all_read", 2023-06-03T20:02:04.533 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:01:59.438418+0000", 2023-06-03T20:02:04.533 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 2.5358e-05 2023-06-03T20:02:04.533 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:02:04.533 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:02:04.534 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "dispatched", 2023-06-03T20:02:04.534 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:01:59.525822+0000", 2023-06-03T20:02:04.534 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.087404467999999999 2023-06-03T20:02:04.534 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:02:04.534 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:02:04.534 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "queued_for_pg", 2023-06-03T20:02:04.535 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:01:59.525839+0000", 2023-06-03T20:02:04.535 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 1.6684000000000002e-05 2023-06-03T20:02:04.535 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:02:04.535 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:02:04.535 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "reached_pg", 2023-06-03T20:02:04.535 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:01:59.536134+0000", 2023-06-03T20:02:04.536 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.010295083 2023-06-03T20:02:04.536 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:02:04.536 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:02:04.536 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "started", 2023-06-03T20:02:04.536 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:01:59.536186+0000", 2023-06-03T20:02:04.536 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 5.2562999999999998e-05 2023-06-03T20:02:04.536 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:02:04.536 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:02:04.537 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "commit_sent", 2023-06-03T20:02:04.537 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:01:59.540970+0000", 2023-06-03T20:02:04.537 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.0047832229999999996 2023-06-03T20:02:04.537 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:02:04.537 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:02:04.537 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "done", 2023-06-03T20:02:04.537 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:01:59.541075+0000", 2023-06-03T20:02:04.537 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.00010535900000000001 2023-06-03T20:02:04.538 INFO:teuthology.orchestra.run.smithi089.stdout: } 2023-06-03T20:02:04.538 INFO:teuthology.orchestra.run.smithi089.stdout: ] 2023-06-03T20:02:04.538 INFO:teuthology.orchestra.run.smithi089.stdout: } 2023-06-03T20:02:04.538 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:02:04.538 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:02:04.538 INFO:teuthology.orchestra.run.smithi089.stdout: "description": "osd_op(client.45713.0:101736 9.4 9:2fd29524:::benchmark_data_smithi089_115580_object101735:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e134)", 2023-06-03T20:02:04.539 INFO:teuthology.orchestra.run.smithi089.stdout: "initiated_at": "2023-06-03T20:02:00.712208+0000", 2023-06-03T20:02:04.539 INFO:teuthology.orchestra.run.smithi089.stdout: "age": 3.657231076, 2023-06-03T20:02:04.539 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.103551955, 2023-06-03T20:02:04.539 INFO:teuthology.orchestra.run.smithi089.stdout: "type_data": { 2023-06-03T20:02:04.539 INFO:teuthology.orchestra.run.smithi089.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-03T20:02:04.539 INFO:teuthology.orchestra.run.smithi089.stdout: "client_info": { 2023-06-03T20:02:04.539 INFO:teuthology.orchestra.run.smithi089.stdout: "client": "client.45713", 2023-06-03T20:02:04.540 INFO:teuthology.orchestra.run.smithi089.stdout: "client_addr": "172.21.15.89:0/2400821448", 2023-06-03T20:02:04.540 INFO:teuthology.orchestra.run.smithi089.stdout: "tid": 101736 2023-06-03T20:02:04.540 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:02:04.540 INFO:teuthology.orchestra.run.smithi089.stdout: "events": [ 2023-06-03T20:02:04.540 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:02:04.540 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "initiated", 2023-06-03T20:02:04.540 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:02:00.712208+0000", 2023-06-03T20:02:04.540 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0 2023-06-03T20:02:04.541 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:02:04.541 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:02:04.541 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "throttled", 2023-06-03T20:02:04.541 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:02:00.712208+0000", 2023-06-03T20:02:04.541 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0 2023-06-03T20:02:04.541 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:02:04.541 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:02:04.541 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "header_read", 2023-06-03T20:02:04.542 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:02:00.712206+0000", 2023-06-03T20:02:04.542 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 4294967295.9999986 2023-06-03T20:02:04.542 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:02:04.542 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:02:04.543 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "all_read", 2023-06-03T20:02:04.543 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:02:00.712245+0000", 2023-06-03T20:02:04.543 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 3.8974000000000003e-05 2023-06-03T20:02:04.543 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:02:04.543 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:02:04.543 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "dispatched", 2023-06-03T20:02:04.543 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:02:00.712261+0000", 2023-06-03T20:02:04.544 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 1.5826999999999999e-05 2023-06-03T20:02:04.544 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:02:04.544 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:02:04.544 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "queued_for_pg", 2023-06-03T20:02:04.544 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:02:00.712270+0000", 2023-06-03T20:02:04.545 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 8.9800000000000004e-06 2023-06-03T20:02:04.545 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:02:04.545 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:02:04.545 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "reached_pg", 2023-06-03T20:02:04.546 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:02:00.712325+0000", 2023-06-03T20:02:04.546 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 5.5352999999999997e-05 2023-06-03T20:02:04.546 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:02:04.546 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:02:04.547 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "started", 2023-06-03T20:02:04.547 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:02:00.712469+0000", 2023-06-03T20:02:04.547 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.00014343599999999999 2023-06-03T20:02:04.547 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:02:04.547 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:02:04.548 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "waiting for subops from 3,4", 2023-06-03T20:02:04.548 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:02:00.712605+0000", 2023-06-03T20:02:04.548 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.000136079 2023-06-03T20:02:04.548 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:02:04.548 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:02:04.548 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "op_commit", 2023-06-03T20:02:04.548 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:02:00.717593+0000", 2023-06-03T20:02:04.549 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.0049881500000000002 2023-06-03T20:02:04.549 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:02:04.549 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:02:04.549 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:02:04.549 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:02:00.718732+0000", 2023-06-03T20:02:04.549 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.001138565 2023-06-03T20:02:04.549 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:02:04.549 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:02:04.550 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:02:04.550 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:02:00.815407+0000", 2023-06-03T20:02:04.550 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.096675175000000002 2023-06-03T20:02:04.550 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:02:04.550 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:02:04.550 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "commit_sent", 2023-06-03T20:02:04.550 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:02:00.815463+0000", 2023-06-03T20:02:04.550 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 5.6541e-05 2023-06-03T20:02:04.551 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:02:04.551 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:02:04.551 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "done", 2023-06-03T20:02:04.551 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:02:00.815760+0000", 2023-06-03T20:02:04.551 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.00029652600000000003 2023-06-03T20:02:04.551 INFO:teuthology.orchestra.run.smithi089.stdout: } 2023-06-03T20:02:04.551 INFO:teuthology.orchestra.run.smithi089.stdout: ] 2023-06-03T20:02:04.551 INFO:teuthology.orchestra.run.smithi089.stdout: } 2023-06-03T20:02:04.552 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:02:04.552 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:02:04.552 INFO:teuthology.orchestra.run.smithi089.stdout: "description": "osd_op(client.45713.0:101737 9.4 9:291fb32f:::benchmark_data_smithi089_115580_object101736:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e134)", 2023-06-03T20:02:04.552 INFO:teuthology.orchestra.run.smithi089.stdout: "initiated_at": "2023-06-03T20:02:00.712285+0000", 2023-06-03T20:02:04.552 INFO:teuthology.orchestra.run.smithi089.stdout: "age": 3.6571542959999999, 2023-06-03T20:02:04.552 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.1104612, 2023-06-03T20:02:04.553 INFO:teuthology.orchestra.run.smithi089.stdout: "type_data": { 2023-06-03T20:02:04.553 INFO:teuthology.orchestra.run.smithi089.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-03T20:02:04.553 INFO:teuthology.orchestra.run.smithi089.stdout: "client_info": { 2023-06-03T20:02:04.554 INFO:teuthology.orchestra.run.smithi089.stdout: "client": "client.45713", 2023-06-03T20:02:04.554 INFO:teuthology.orchestra.run.smithi089.stdout: "client_addr": "172.21.15.89:0/2400821448", 2023-06-03T20:02:04.554 INFO:teuthology.orchestra.run.smithi089.stdout: "tid": 101737 2023-06-03T20:02:04.554 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:02:04.555 INFO:teuthology.orchestra.run.smithi089.stdout: "events": [ 2023-06-03T20:02:04.555 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:02:04.555 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "initiated", 2023-06-03T20:02:04.555 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:02:00.712285+0000", 2023-06-03T20:02:04.556 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0 2023-06-03T20:02:04.556 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:02:04.556 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:02:04.556 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "throttled", 2023-06-03T20:02:04.557 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:02:00.712285+0000", 2023-06-03T20:02:04.557 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0 2023-06-03T20:02:04.557 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:02:04.557 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:02:04.558 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "header_read", 2023-06-03T20:02:04.558 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:02:00.712284+0000", 2023-06-03T20:02:04.558 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 4294967295.9999995 2023-06-03T20:02:04.558 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:02:04.559 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:02:04.559 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "all_read", 2023-06-03T20:02:04.559 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:02:00.712308+0000", 2023-06-03T20:02:04.560 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 2.3909000000000001e-05 2023-06-03T20:02:04.560 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:02:04.560 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:02:04.560 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "dispatched", 2023-06-03T20:02:04.560 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:02:00.712319+0000", 2023-06-03T20:02:04.560 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 1.0937e-05 2023-06-03T20:02:04.561 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:02:04.561 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:02:04.561 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "queued_for_pg", 2023-06-03T20:02:04.561 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:02:00.712326+0000", 2023-06-03T20:02:04.561 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 7.0809999999999996e-06 2023-06-03T20:02:04.561 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:02:04.561 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:02:04.561 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "reached_pg", 2023-06-03T20:02:04.562 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:02:00.715279+0000", 2023-06-03T20:02:04.562 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.0029532320000000001 2023-06-03T20:02:04.562 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:02:04.562 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:02:04.562 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "started", 2023-06-03T20:02:04.562 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:02:00.715390+0000", 2023-06-03T20:02:04.562 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.000111152 2023-06-03T20:02:04.563 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:02:04.563 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:02:04.563 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "waiting for subops from 3,4", 2023-06-03T20:02:04.563 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:02:00.715489+0000", 2023-06-03T20:02:04.563 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 9.8088e-05 2023-06-03T20:02:04.563 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:02:04.564 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:02:04.564 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "op_commit", 2023-06-03T20:02:04.564 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:02:00.717787+0000", 2023-06-03T20:02:04.564 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.0022982409999999999 2023-06-03T20:02:04.565 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:02:04.565 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:02:04.565 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:02:04.565 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:02:00.718842+0000", 2023-06-03T20:02:04.565 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.0010550679999999999 2023-06-03T20:02:04.565 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:02:04.565 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:02:04.566 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:02:04.566 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:02:00.822582+0000", 2023-06-03T20:02:04.566 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.10373977400000001 2023-06-03T20:02:04.566 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:02:04.566 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:02:04.567 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "commit_sent", 2023-06-03T20:02:04.567 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:02:00.822695+0000", 2023-06-03T20:02:04.567 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.000113836 2023-06-03T20:02:04.567 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:02:04.567 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:02:04.567 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "done", 2023-06-03T20:02:04.567 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:02:00.822746+0000", 2023-06-03T20:02:04.568 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 5.0523999999999999e-05 2023-06-03T20:02:04.568 INFO:teuthology.orchestra.run.smithi089.stdout: } 2023-06-03T20:02:04.568 INFO:teuthology.orchestra.run.smithi089.stdout: ] 2023-06-03T20:02:04.568 INFO:teuthology.orchestra.run.smithi089.stdout: } 2023-06-03T20:02:04.568 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:02:04.568 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:02:04.568 INFO:teuthology.orchestra.run.smithi089.stdout: "description": "osd_op(client.45713.0:101739 9.4 9:277eb965:::benchmark_data_smithi089_115580_object101738:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e134)", 2023-06-03T20:02:04.568 INFO:teuthology.orchestra.run.smithi089.stdout: "initiated_at": "2023-06-03T20:02:00.718902+0000", 2023-06-03T20:02:04.569 INFO:teuthology.orchestra.run.smithi089.stdout: "age": 3.650537226, 2023-06-03T20:02:04.569 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.104231374, 2023-06-03T20:02:04.569 INFO:teuthology.orchestra.run.smithi089.stdout: "type_data": { 2023-06-03T20:02:04.569 INFO:teuthology.orchestra.run.smithi089.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-03T20:02:04.569 INFO:teuthology.orchestra.run.smithi089.stdout: "client_info": { 2023-06-03T20:02:04.569 INFO:teuthology.orchestra.run.smithi089.stdout: "client": "client.45713", 2023-06-03T20:02:04.569 INFO:teuthology.orchestra.run.smithi089.stdout: "client_addr": "172.21.15.89:0/2400821448", 2023-06-03T20:02:04.569 INFO:teuthology.orchestra.run.smithi089.stdout: "tid": 101739 2023-06-03T20:02:04.570 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:02:04.570 INFO:teuthology.orchestra.run.smithi089.stdout: "events": [ 2023-06-03T20:02:04.570 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:02:04.570 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "initiated", 2023-06-03T20:02:04.570 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:02:00.718902+0000", 2023-06-03T20:02:04.570 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0 2023-06-03T20:02:04.570 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:02:04.571 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:02:04.571 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "throttled", 2023-06-03T20:02:04.571 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:02:00.718902+0000", 2023-06-03T20:02:04.571 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0 2023-06-03T20:02:04.571 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:02:04.571 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:02:04.571 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "header_read", 2023-06-03T20:02:04.571 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:02:00.718901+0000", 2023-06-03T20:02:04.572 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 4294967295.9999986 2023-06-03T20:02:04.572 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:02:04.572 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:02:04.572 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "all_read", 2023-06-03T20:02:04.572 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:02:00.718927+0000", 2023-06-03T20:02:04.572 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 2.6942999999999999e-05 2023-06-03T20:02:04.572 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:02:04.573 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:02:04.573 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "dispatched", 2023-06-03T20:02:04.573 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:02:00.718941+0000", 2023-06-03T20:02:04.573 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 1.3633e-05 2023-06-03T20:02:04.573 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:02:04.573 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:02:04.573 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "queued_for_pg", 2023-06-03T20:02:04.573 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:02:00.718950+0000", 2023-06-03T20:02:04.574 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 8.6929999999999996e-06 2023-06-03T20:02:04.574 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:02:04.574 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:02:04.574 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "reached_pg", 2023-06-03T20:02:04.574 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:02:00.718991+0000", 2023-06-03T20:02:04.574 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 4.1313999999999998e-05 2023-06-03T20:02:04.574 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:02:04.574 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:02:04.575 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "started", 2023-06-03T20:02:04.575 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:02:00.719079+0000", 2023-06-03T20:02:04.575 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 8.8257000000000006e-05 2023-06-03T20:02:04.575 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:02:04.575 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:02:04.575 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "waiting for subops from 3,4", 2023-06-03T20:02:04.575 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:02:00.719171+0000", 2023-06-03T20:02:04.576 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 9.1867000000000007e-05 2023-06-03T20:02:04.576 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:02:04.576 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:02:04.576 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "op_commit", 2023-06-03T20:02:04.576 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:02:00.720916+0000", 2023-06-03T20:02:04.576 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.001744301 2023-06-03T20:02:04.577 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:02:04.577 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:02:04.577 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:02:04.577 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:02:00.721856+0000", 2023-06-03T20:02:04.577 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.00094068199999999998 2023-06-03T20:02:04.577 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:02:04.578 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:02:04.578 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:02:04.578 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:02:00.823014+0000", 2023-06-03T20:02:04.578 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.101158205 2023-06-03T20:02:04.578 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:02:04.578 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:02:04.578 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "commit_sent", 2023-06-03T20:02:04.578 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:02:00.823094+0000", 2023-06-03T20:02:04.579 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 7.9417000000000002e-05 2023-06-03T20:02:04.579 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:02:04.579 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:02:04.579 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "done", 2023-06-03T20:02:04.579 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:02:00.823133+0000", 2023-06-03T20:02:04.579 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 3.9396e-05 2023-06-03T20:02:04.579 INFO:teuthology.orchestra.run.smithi089.stdout: } 2023-06-03T20:02:04.579 INFO:teuthology.orchestra.run.smithi089.stdout: ] 2023-06-03T20:02:04.580 INFO:teuthology.orchestra.run.smithi089.stdout: } 2023-06-03T20:02:04.580 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:02:04.580 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:02:04.580 INFO:teuthology.orchestra.run.smithi089.stdout: "description": "osd_op(client.45713.0:102655 9.4 9:2edef89e:::benchmark_data_smithi089_115580_object102654:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e134)", 2023-06-03T20:02:04.580 INFO:teuthology.orchestra.run.smithi089.stdout: "initiated_at": "2023-06-03T20:02:01.824845+0000", 2023-06-03T20:02:04.580 INFO:teuthology.orchestra.run.smithi089.stdout: "age": 2.544594214, 2023-06-03T20:02:04.580 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.091472803000000005, 2023-06-03T20:02:04.581 INFO:teuthology.orchestra.run.smithi089.stdout: "type_data": { 2023-06-03T20:02:04.581 INFO:teuthology.orchestra.run.smithi089.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-03T20:02:04.581 INFO:teuthology.orchestra.run.smithi089.stdout: "client_info": { 2023-06-03T20:02:04.581 INFO:teuthology.orchestra.run.smithi089.stdout: "client": "client.45713", 2023-06-03T20:02:04.581 INFO:teuthology.orchestra.run.smithi089.stdout: "client_addr": "172.21.15.89:0/2400821448", 2023-06-03T20:02:04.581 INFO:teuthology.orchestra.run.smithi089.stdout: "tid": 102655 2023-06-03T20:02:04.582 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:02:04.582 INFO:teuthology.orchestra.run.smithi089.stdout: "events": [ 2023-06-03T20:02:04.582 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:02:04.582 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "initiated", 2023-06-03T20:02:04.582 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:02:01.824845+0000", 2023-06-03T20:02:04.582 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0 2023-06-03T20:02:04.582 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:02:04.582 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:02:04.583 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "throttled", 2023-06-03T20:02:04.583 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:02:01.824845+0000", 2023-06-03T20:02:04.583 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0 2023-06-03T20:02:04.583 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:02:04.583 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:02:04.583 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "header_read", 2023-06-03T20:02:04.583 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:02:01.824844+0000", 2023-06-03T20:02:04.584 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 4294967295.9999986 2023-06-03T20:02:04.584 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:02:04.584 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:02:04.584 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "all_read", 2023-06-03T20:02:04.584 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:02:01.824866+0000", 2023-06-03T20:02:04.584 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 2.2291999999999999e-05 2023-06-03T20:02:04.584 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:02:04.584 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:02:04.585 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "dispatched", 2023-06-03T20:02:04.585 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:02:01.824876+0000", 2023-06-03T20:02:04.585 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 1.039e-05 2023-06-03T20:02:04.585 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:02:04.585 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:02:04.585 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "queued_for_pg", 2023-06-03T20:02:04.585 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:02:01.824883+0000", 2023-06-03T20:02:04.586 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 6.9110000000000004e-06 2023-06-03T20:02:04.586 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:02:04.586 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:02:04.586 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "reached_pg", 2023-06-03T20:02:04.586 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:02:01.824946+0000", 2023-06-03T20:02:04.587 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 6.2782999999999998e-05 2023-06-03T20:02:04.587 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:02:04.587 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:02:04.587 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "started", 2023-06-03T20:02:04.588 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:02:01.825150+0000", 2023-06-03T20:02:04.588 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.00020382500000000001 2023-06-03T20:02:04.588 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:02:04.588 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:02:04.588 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "waiting for subops from 3,4", 2023-06-03T20:02:04.588 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:02:01.825443+0000", 2023-06-03T20:02:04.588 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.00029287300000000003 2023-06-03T20:02:04.589 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:02:04.589 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:02:04.589 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:02:04.589 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:02:01.833100+0000", 2023-06-03T20:02:04.589 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.0076576129999999997 2023-06-03T20:02:04.589 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:02:04.589 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:02:04.590 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "op_commit", 2023-06-03T20:02:04.590 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:02:01.833211+0000", 2023-06-03T20:02:04.590 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.000110213 2023-06-03T20:02:04.590 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:02:04.590 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:02:04.590 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:02:04.590 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:02:01.916226+0000", 2023-06-03T20:02:04.590 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.083015736000000007 2023-06-03T20:02:04.591 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:02:04.591 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:02:04.591 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "commit_sent", 2023-06-03T20:02:04.591 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:02:01.916279+0000", 2023-06-03T20:02:04.591 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 5.2401000000000002e-05 2023-06-03T20:02:04.591 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:02:04.591 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:02:04.591 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "done", 2023-06-03T20:02:04.592 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:02:01.916318+0000", 2023-06-03T20:02:04.592 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 3.8967999999999997e-05 2023-06-03T20:02:04.592 INFO:teuthology.orchestra.run.smithi089.stdout: } 2023-06-03T20:02:04.592 INFO:teuthology.orchestra.run.smithi089.stdout: ] 2023-06-03T20:02:04.592 INFO:teuthology.orchestra.run.smithi089.stdout: } 2023-06-03T20:02:04.592 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:02:04.592 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:02:04.593 INFO:teuthology.orchestra.run.smithi089.stdout: "description": "osd_repop(client.45713.0:102666 9.a e134/127 9:5e6364c7:::benchmark_data_smithi089_115580_object102665:head v 134'6395, mlcod=134'6392)", 2023-06-03T20:02:04.593 INFO:teuthology.orchestra.run.smithi089.stdout: "initiated_at": "2023-06-03T20:02:01.839143+0000", 2023-06-03T20:02:04.593 INFO:teuthology.orchestra.run.smithi089.stdout: "age": 2.5302958860000002, 2023-06-03T20:02:04.593 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.080424763999999996, 2023-06-03T20:02:04.593 INFO:teuthology.orchestra.run.smithi089.stdout: "type_data": { 2023-06-03T20:02:04.593 INFO:teuthology.orchestra.run.smithi089.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-03T20:02:04.593 INFO:teuthology.orchestra.run.smithi089.stdout: "events": [ 2023-06-03T20:02:04.593 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:02:04.594 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "initiated", 2023-06-03T20:02:04.594 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:02:01.839143+0000", 2023-06-03T20:02:04.594 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0 2023-06-03T20:02:04.594 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:02:04.594 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:02:04.594 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "throttled", 2023-06-03T20:02:04.594 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:02:01.839143+0000", 2023-06-03T20:02:04.594 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0 2023-06-03T20:02:04.595 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:02:04.595 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:02:04.595 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "header_read", 2023-06-03T20:02:04.595 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:02:01.839142+0000", 2023-06-03T20:02:04.595 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 4294967295.999999 2023-06-03T20:02:04.595 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:02:04.595 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:02:04.596 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "all_read", 2023-06-03T20:02:04.596 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:02:01.839204+0000", 2023-06-03T20:02:04.596 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 6.1597e-05 2023-06-03T20:02:04.596 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:02:04.596 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:02:04.596 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "dispatched", 2023-06-03T20:02:04.597 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:02:01.908776+0000", 2023-06-03T20:02:04.597 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.069572330000000002 2023-06-03T20:02:04.597 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:02:04.597 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:02:04.597 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "queued_for_pg", 2023-06-03T20:02:04.597 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:02:01.908789+0000", 2023-06-03T20:02:04.597 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 1.3033e-05 2023-06-03T20:02:04.598 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:02:04.598 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:02:04.598 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "reached_pg", 2023-06-03T20:02:04.598 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:02:01.909769+0000", 2023-06-03T20:02:04.599 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.00097988700000000003 2023-06-03T20:02:04.599 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:02:04.599 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:02:04.599 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "started", 2023-06-03T20:02:04.600 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:02:01.909794+0000", 2023-06-03T20:02:04.600 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 2.5011000000000002e-05 2023-06-03T20:02:04.600 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:02:04.600 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:02:04.600 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "commit_sent", 2023-06-03T20:02:04.600 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:02:01.919526+0000", 2023-06-03T20:02:04.600 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.0097313150000000008 2023-06-03T20:02:04.601 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:02:04.601 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:02:04.601 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "done", 2023-06-03T20:02:04.601 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:02:01.919568+0000", 2023-06-03T20:02:04.601 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 4.2352999999999999e-05 2023-06-03T20:02:04.601 INFO:teuthology.orchestra.run.smithi089.stdout: } 2023-06-03T20:02:04.601 INFO:teuthology.orchestra.run.smithi089.stdout: ] 2023-06-03T20:02:04.601 INFO:teuthology.orchestra.run.smithi089.stdout: } 2023-06-03T20:02:04.602 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:02:04.602 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:02:04.602 INFO:teuthology.orchestra.run.smithi089.stdout: "description": "osd_op(client.45713.0:103429 9.4 9:208f7018:::benchmark_data_smithi089_115580_object103428:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e134)", 2023-06-03T20:02:04.602 INFO:teuthology.orchestra.run.smithi089.stdout: "initiated_at": "2023-06-03T20:02:02.765863+0000", 2023-06-03T20:02:04.602 INFO:teuthology.orchestra.run.smithi089.stdout: "age": 1.603576165, 2023-06-03T20:02:04.602 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.079362187000000001, 2023-06-03T20:02:04.602 INFO:teuthology.orchestra.run.smithi089.stdout: "type_data": { 2023-06-03T20:02:04.602 INFO:teuthology.orchestra.run.smithi089.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-03T20:02:04.603 INFO:teuthology.orchestra.run.smithi089.stdout: "client_info": { 2023-06-03T20:02:04.603 INFO:teuthology.orchestra.run.smithi089.stdout: "client": "client.45713", 2023-06-03T20:02:04.603 INFO:teuthology.orchestra.run.smithi089.stdout: "client_addr": "172.21.15.89:0/2400821448", 2023-06-03T20:02:04.603 INFO:teuthology.orchestra.run.smithi089.stdout: "tid": 103429 2023-06-03T20:02:04.603 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:02:04.603 INFO:teuthology.orchestra.run.smithi089.stdout: "events": [ 2023-06-03T20:02:04.603 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:02:04.603 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "initiated", 2023-06-03T20:02:04.604 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:02:02.765863+0000", 2023-06-03T20:02:04.604 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0 2023-06-03T20:02:04.604 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:02:04.604 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:02:04.604 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "throttled", 2023-06-03T20:02:04.604 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:02:02.765863+0000", 2023-06-03T20:02:04.604 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0 2023-06-03T20:02:04.605 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:02:04.605 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:02:04.605 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "header_read", 2023-06-03T20:02:04.605 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:02:02.765862+0000", 2023-06-03T20:02:04.605 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 4294967295.9999986 2023-06-03T20:02:04.605 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:02:04.605 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:02:04.605 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "all_read", 2023-06-03T20:02:04.606 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:02:02.765884+0000", 2023-06-03T20:02:04.606 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 2.1895999999999998e-05 2023-06-03T20:02:04.606 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:02:04.606 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:02:04.606 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "dispatched", 2023-06-03T20:02:04.606 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:02:02.765894+0000", 2023-06-03T20:02:04.606 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 1.0514e-05 2023-06-03T20:02:04.607 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:02:04.607 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:02:04.607 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "queued_for_pg", 2023-06-03T20:02:04.607 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:02:02.765900+0000", 2023-06-03T20:02:04.607 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 6.3570000000000003e-06 2023-06-03T20:02:04.607 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:02:04.607 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:02:04.608 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "reached_pg", 2023-06-03T20:02:04.608 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:02:02.771849+0000", 2023-06-03T20:02:04.608 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.0059481480000000003 2023-06-03T20:02:04.608 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:02:04.608 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:02:04.608 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "started", 2023-06-03T20:02:04.608 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:02:02.771986+0000", 2023-06-03T20:02:04.608 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.000137331 2023-06-03T20:02:04.609 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:02:04.609 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:02:04.609 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "waiting for subops from 3,4", 2023-06-03T20:02:04.609 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:02:02.772133+0000", 2023-06-03T20:02:04.609 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.00014699400000000001 2023-06-03T20:02:04.609 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:02:04.610 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:02:04.610 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "op_commit", 2023-06-03T20:02:04.610 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:02:02.782633+0000", 2023-06-03T20:02:04.610 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.010499893 2023-06-03T20:02:04.610 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:02:04.610 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:02:04.611 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:02:04.611 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:02:02.783506+0000", 2023-06-03T20:02:04.611 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.00087351699999999998 2023-06-03T20:02:04.611 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:02:04.611 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:02:04.611 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:02:04.611 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:02:02.787118+0000", 2023-06-03T20:02:04.611 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.0036118489999999999 2023-06-03T20:02:04.612 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:02:04.612 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:02:04.612 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "commit_sent", 2023-06-03T20:02:04.612 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:02:02.787158+0000", 2023-06-03T20:02:04.612 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 3.9449999999999997e-05 2023-06-03T20:02:04.612 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:02:04.612 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:02:04.613 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "done", 2023-06-03T20:02:04.613 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:02:02.845225+0000", 2023-06-03T20:02:04.613 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.058067465999999998 2023-06-03T20:02:04.613 INFO:teuthology.orchestra.run.smithi089.stdout: } 2023-06-03T20:02:04.613 INFO:teuthology.orchestra.run.smithi089.stdout: ] 2023-06-03T20:02:04.613 INFO:teuthology.orchestra.run.smithi089.stdout: } 2023-06-03T20:02:04.613 INFO:teuthology.orchestra.run.smithi089.stdout: } 2023-06-03T20:02:04.614 INFO:teuthology.orchestra.run.smithi089.stdout: ] 2023-06-03T20:02:04.614 INFO:teuthology.orchestra.run.smithi089.stdout:} 2023-06-03T20:02:04.614 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:02:04.656 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:02:04.659 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:02:04.660 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:02:04.761 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:02:04.802 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:02:04.806 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:02:04.807 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:02:04.907 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:02:04.937 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:02:04.939 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:02:04.940 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:02:04.995 INFO:teuthology.orchestra.run.smithi089.stdout:osd.0: {} 2023-06-03T20:02:05.035 INFO:teuthology.orchestra.run.smithi089.stdout:osd.1: {} 2023-06-03T20:02:05.041 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:02:05.069 INFO:teuthology.orchestra.run.smithi089.stdout:osd.2: {} 2023-06-03T20:02:05.091 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:02:05.093 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:02:05.094 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:02:05.112 INFO:teuthology.orchestra.run.smithi089.stdout:osd.3: {} 2023-06-03T20:02:05.150 INFO:teuthology.orchestra.run.smithi089.stdout:osd.4: {} 2023-06-03T20:02:05.184 INFO:teuthology.orchestra.run.smithi089.stdout:osd.5: {} 2023-06-03T20:02:05.195 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:02:05.199 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 84 16 105397 105381 78.3867 46.6875 0.0129244 0.012751 2023-06-03T20:02:05.224 INFO:teuthology.orchestra.run.smithi089.stdout:osd.6: {} 2023-06-03T20:02:05.230 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:02:05.234 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:02:05.234 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:02:05.255 INFO:teuthology.orchestra.run.smithi089.stdout:osd.7: {} 2023-06-03T20:02:05.267 INFO:teuthology.orchestra.run.smithi089.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-06-03T20:02:05.267 INFO:teuthology.orchestra.run.smithi089.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-06-03T20:02:05.267 INFO:teuthology.orchestra.run.smithi089.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-06-03T20:02:05.267 INFO:teuthology.orchestra.run.smithi089.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-06-03T20:02:05.267 INFO:teuthology.orchestra.run.smithi089.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-06-03T20:02:05.268 INFO:teuthology.orchestra.run.smithi089.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-06-03T20:02:05.268 INFO:teuthology.orchestra.run.smithi089.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-06-03T20:02:05.268 INFO:teuthology.orchestra.run.smithi089.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-06-03T20:02:05.336 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:02:05.381 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:02:05.387 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:02:05.388 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:02:05.489 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:02:05.520 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:02:05.525 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:02:05.526 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:02:05.588 INFO:teuthology.orchestra.run.smithi089.stderr:noscrub is set 2023-06-03T20:02:05.627 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:02:05.655 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:02:05.660 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:02:05.661 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:02:05.762 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:02:05.775 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:02:05 smithi164 bash[59462]: cluster 1685822524.4590316 mgr.y (mgr.34104) 773 : cluster 0 pgmap v686: 89 pgs: 89 active+clean; 6.3 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 64 MiB/s wr, 1.03k op/s 2023-06-03T20:02:05.775 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:02:05 smithi164 bash[59462]: audit 1685822524.939449 mon.c (mon.2) 393 : audit 0 from='client.? 172.21.15.89:0/1296185341' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-03T20:02:05.775 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:02:05 smithi164 bash[59462]: audit 1685822525.0352442 mon.a (mon.0) 694 : audit 1 from='client.? 172.21.15.89:0/3409743049' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-06-03T20:02:05.813 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:02:05.821 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:02:05.822 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:02:05.924 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:02:05.966 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:02:05.970 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:02:05.970 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:02:06.071 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:02:06.102 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:02:06.104 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:02:06.105 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:02:06.174 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:02:05 smithi089 bash[75068]: cluster 1685822524.4590316 mgr.y (mgr.34104) 773 : cluster 0 pgmap v686: 89 pgs: 89 active+clean; 6.3 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 64 MiB/s wr, 1.03k op/s 2023-06-03T20:02:06.174 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:02:05 smithi089 bash[75068]: audit 1685822524.939449 mon.c (mon.2) 393 : audit 0 from='client.? 172.21.15.89:0/1296185341' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-03T20:02:06.175 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:02:05 smithi089 bash[75068]: audit 1685822525.0352442 mon.a (mon.0) 694 : audit 1 from='client.? 172.21.15.89:0/3409743049' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-06-03T20:02:06.175 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:02:05 smithi089 bash[78452]: cluster 1685822524.4590316 mgr.y (mgr.34104) 773 : cluster 0 pgmap v686: 89 pgs: 89 active+clean; 6.3 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 64 MiB/s wr, 1.03k op/s 2023-06-03T20:02:06.175 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:02:05 smithi089 bash[78452]: audit 1685822524.939449 mon.c (mon.2) 393 : audit 0 from='client.? 172.21.15.89:0/1296185341' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-03T20:02:06.176 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:02:05 smithi089 bash[78452]: audit 1685822525.0352442 mon.a (mon.0) 694 : audit 1 from='client.? 172.21.15.89:0/3409743049' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-06-03T20:02:06.199 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 85 16 106081 106065 77.9675 42.75 0.0160137 0.01282 2023-06-03T20:02:06.206 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:02:06.242 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:02:06.245 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:02:06.246 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:02:06.347 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:02:06.384 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:02:06.390 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:02:06.391 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:02:06.492 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:02:06.528 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 20:02:06 smithi089 bash[71163]: ::ffff:172.21.15.164 - - [03/Jun/2023:20:02:06] "GET /metrics HTTP/1.1" 200 228258 "" "Prometheus/2.33.4" 2023-06-03T20:02:06.534 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:02:06.538 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:02:06.539 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:02:06.604 INFO:teuthology.orchestra.run.smithi089.stderr:reweighted osd.4 to 1 (10000) 2023-06-03T20:02:06.639 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:02:06.683 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:02:06.688 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:02:06 smithi089 bash[75068]: cluster 1685822525.5692122 mon.a (mon.0) 695 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-06-03T20:02:06.689 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:02:06 smithi089 bash[75068]: cluster 1685822525.5704246 osd.6 (osd.6) 54 : cluster 3 failed to encode map e135 with expected crc 2023-06-03T20:02:06.689 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:02:06 smithi089 bash[75068]: audit 1685822525.5751188 mon.a (mon.0) 696 : audit 1 from='client.? 172.21.15.89:0/3409743049' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-06-03T20:02:06.689 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:02:06 smithi089 bash[75068]: cluster 1685822525.575202 mon.a (mon.0) 697 : cluster 0 osdmap e135: 8 total, 8 up, 8 in 2023-06-03T20:02:06.689 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:02:06 smithi089 bash[75068]: cluster 1685822525.5778918 osd.7 (osd.7) 48 : cluster 3 failed to encode map e135 with expected crc 2023-06-03T20:02:06.689 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:02:06 smithi089 bash[75068]: cluster 1685822525.5796738 osd.7 (osd.7) 49 : cluster 3 failed to encode map e135 with expected crc 2023-06-03T20:02:06.690 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:02:06 smithi089 bash[75068]: cluster 1685822525.5834262 osd.5 (osd.5) 59 : cluster 3 failed to encode map e135 with expected crc 2023-06-03T20:02:06.690 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:02:06 smithi089 bash[75068]: cluster 1685822525.58468 osd.5 (osd.5) 60 : cluster 3 failed to encode map e135 with expected crc 2023-06-03T20:02:06.690 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:02:06 smithi089 bash[75068]: audit 1685822525.7262003 mon.c (mon.2) 394 : audit 1 from='client.? 172.21.15.89:0/2252556803' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-06-03T20:02:06.690 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:02:06 smithi089 bash[75068]: audit 1685822525.7266245 mon.a (mon.0) 698 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-06-03T20:02:06.690 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:02:06 smithi089 bash[78452]: cluster 1685822525.5692122 mon.a (mon.0) 695 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-06-03T20:02:06.691 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:02:06 smithi089 bash[78452]: cluster 1685822525.5704246 osd.6 (osd.6) 54 : cluster 3 failed to encode map e135 with expected crc 2023-06-03T20:02:06.691 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:02:06 smithi089 bash[78452]: audit 1685822525.5751188 mon.a (mon.0) 696 : audit 1 from='client.? 172.21.15.89:0/3409743049' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-06-03T20:02:06.691 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:02:06 smithi089 bash[78452]: cluster 1685822525.575202 mon.a (mon.0) 697 : cluster 0 osdmap e135: 8 total, 8 up, 8 in 2023-06-03T20:02:06.691 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:02:06 smithi089 bash[78452]: cluster 1685822525.5778918 osd.7 (osd.7) 48 : cluster 3 failed to encode map e135 with expected crc 2023-06-03T20:02:06.691 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:02:06 smithi089 bash[78452]: cluster 1685822525.5796738 osd.7 (osd.7) 49 : cluster 3 failed to encode map e135 with expected crc 2023-06-03T20:02:06.691 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:02:06 smithi089 bash[78452]: cluster 1685822525.5834262 osd.5 (osd.5) 59 : cluster 3 failed to encode map e135 with expected crc 2023-06-03T20:02:06.692 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:02:06 smithi089 bash[78452]: cluster 1685822525.58468 osd.5 (osd.5) 60 : cluster 3 failed to encode map e135 with expected crc 2023-06-03T20:02:06.692 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:02:06 smithi089 bash[78452]: audit 1685822525.7262003 mon.c (mon.2) 394 : audit 1 from='client.? 172.21.15.89:0/2252556803' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-06-03T20:02:06.692 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:02:06 smithi089 bash[78452]: audit 1685822525.7266245 mon.a (mon.0) 698 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-06-03T20:02:06.692 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:02:06.693 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:02:06.794 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:02:06.803 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:02:06 smithi164 bash[59462]: cluster 1685822525.5692122 mon.a (mon.0) 695 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-06-03T20:02:06.804 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:02:06 smithi164 bash[59462]: cluster 1685822525.5704246 osd.6 (osd.6) 54 : cluster 3 failed to encode map e135 with expected crc 2023-06-03T20:02:06.804 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:02:06 smithi164 bash[59462]: audit 1685822525.5751188 mon.a (mon.0) 696 : audit 1 from='client.? 172.21.15.89:0/3409743049' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-06-03T20:02:06.804 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:02:06 smithi164 bash[59462]: cluster 1685822525.575202 mon.a (mon.0) 697 : cluster 0 osdmap e135: 8 total, 8 up, 8 in 2023-06-03T20:02:06.804 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:02:06 smithi164 bash[59462]: cluster 1685822525.5778918 osd.7 (osd.7) 48 : cluster 3 failed to encode map e135 with expected crc 2023-06-03T20:02:06.805 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:02:06 smithi164 bash[59462]: cluster 1685822525.5796738 osd.7 (osd.7) 49 : cluster 3 failed to encode map e135 with expected crc 2023-06-03T20:02:06.806 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:02:06 smithi164 bash[59462]: cluster 1685822525.5834262 osd.5 (osd.5) 59 : cluster 3 failed to encode map e135 with expected crc 2023-06-03T20:02:06.806 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:02:06 smithi164 bash[59462]: cluster 1685822525.58468 osd.5 (osd.5) 60 : cluster 3 failed to encode map e135 with expected crc 2023-06-03T20:02:06.806 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:02:06 smithi164 bash[59462]: audit 1685822525.7262003 mon.c (mon.2) 394 : audit 1 from='client.? 172.21.15.89:0/2252556803' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-06-03T20:02:06.806 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:02:06 smithi164 bash[59462]: audit 1685822525.7266245 mon.a (mon.0) 698 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-06-03T20:02:06.833 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:02:06.838 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:02:06.839 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:02:06.940 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:02:06.971 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:02:06.973 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:02:06.973 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:02:07.051 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph daemon osd.2 dump_ops_in_flight 2023-06-03T20:02:07.074 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:02:07.140 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:02:07.148 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:02:07.149 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:02:07.200 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 86 16 106813 106797 77.5928 45.75 0.00854402 0.0128778 2023-06-03T20:02:07.249 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:02:07.278 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:02:07.283 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:02:07.283 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:02:07.384 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:02:07.419 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:02:07.422 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:02:07.422 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:02:07.523 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:02:07.560 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:02:07.564 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:02:07.565 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:02:07.666 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:02:07.697 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:02:07.700 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:02:07.700 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:02:07.801 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:02:07.841 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:02:07.846 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:02:07.848 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:02:07.948 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:02:07.957 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:02:07 smithi164 bash[59462]: cluster 1685822526.4593797 mgr.y (mgr.34104) 774 : cluster 0 pgmap v688: 89 pgs: 89 active+clean; 6.3 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 52 MiB/s wr, 838 op/s 2023-06-03T20:02:07.957 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:02:07 smithi164 bash[59462]: cluster 1685822526.5903716 osd.6 (osd.6) 55 : cluster 3 failed to encode map e136 with expected crc 2023-06-03T20:02:07.957 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:02:07 smithi164 bash[59462]: cluster 1685822526.5906644 osd.5 (osd.5) 61 : cluster 3 failed to encode map e136 with expected crc 2023-06-03T20:02:07.957 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:02:07 smithi164 bash[59462]: audit 1685822526.5951598 mon.a (mon.0) 699 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2023-06-03T20:02:07.958 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:02:07 smithi164 bash[59462]: cluster 1685822526.595233 mon.a (mon.0) 700 : cluster 0 osdmap e136: 8 total, 8 up, 8 in 2023-06-03T20:02:07.958 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:02:07 smithi164 bash[59462]: audit 1685822527.1027148 mon.a (mon.0) 701 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:02:07.958 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:02:07 smithi164 bash[59462]: audit 1685822527.110928 mon.c (mon.2) 395 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T20:02:07.996 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:02:08.000 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:02:08.001 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:02:08.028 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:02:07 smithi089 bash[75068]: cluster 1685822526.4593797 mgr.y (mgr.34104) 774 : cluster 0 pgmap v688: 89 pgs: 89 active+clean; 6.3 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 52 MiB/s wr, 838 op/s 2023-06-03T20:02:08.029 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:02:07 smithi089 bash[75068]: cluster 1685822526.5903716 osd.6 (osd.6) 55 : cluster 3 failed to encode map e136 with expected crc 2023-06-03T20:02:08.029 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:02:07 smithi089 bash[75068]: cluster 1685822526.5906644 osd.5 (osd.5) 61 : cluster 3 failed to encode map e136 with expected crc 2023-06-03T20:02:08.029 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:02:07 smithi089 bash[75068]: audit 1685822526.5951598 mon.a (mon.0) 699 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2023-06-03T20:02:08.029 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:02:07 smithi089 bash[75068]: cluster 1685822526.595233 mon.a (mon.0) 700 : cluster 0 osdmap e136: 8 total, 8 up, 8 in 2023-06-03T20:02:08.029 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:02:07 smithi089 bash[75068]: audit 1685822527.1027148 mon.a (mon.0) 701 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:02:08.029 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:02:07 smithi089 bash[75068]: audit 1685822527.110928 mon.c (mon.2) 395 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T20:02:08.030 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:02:07 smithi089 bash[78452]: cluster 1685822526.4593797 mgr.y (mgr.34104) 774 : cluster 0 pgmap v688: 89 pgs: 89 active+clean; 6.3 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 52 MiB/s wr, 838 op/s 2023-06-03T20:02:08.030 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:02:07 smithi089 bash[78452]: cluster 1685822526.5903716 osd.6 (osd.6) 55 : cluster 3 failed to encode map e136 with expected crc 2023-06-03T20:02:08.030 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:02:07 smithi089 bash[78452]: cluster 1685822526.5906644 osd.5 (osd.5) 61 : cluster 3 failed to encode map e136 with expected crc 2023-06-03T20:02:08.030 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:02:07 smithi089 bash[78452]: audit 1685822526.5951598 mon.a (mon.0) 699 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2023-06-03T20:02:08.030 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:02:07 smithi089 bash[78452]: cluster 1685822526.595233 mon.a (mon.0) 700 : cluster 0 osdmap e136: 8 total, 8 up, 8 in 2023-06-03T20:02:08.030 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:02:07 smithi089 bash[78452]: audit 1685822527.1027148 mon.a (mon.0) 701 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:02:08.031 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:02:07 smithi089 bash[78452]: audit 1685822527.110928 mon.c (mon.2) 395 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T20:02:08.102 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:02:08.142 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:02:08.147 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:02:08.148 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:02:08.200 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 87 16 107662 107646 77.3109 53.0625 0.0134528 0.0129291 2023-06-03T20:02:08.248 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:02:08.280 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:02:08.284 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:02:08.285 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:02:08.386 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:02:08.413 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:02:08.418 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:02:08.419 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:02:08.519 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:02:08.571 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:02:08.575 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:02:08.575 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:02:08.676 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:02:08.722 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:02:08.725 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:02:08.725 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:02:08.826 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:02:08.881 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:02:08.886 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:02:08.887 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:02:08.903 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph osd reweight 5 1 2023-06-03T20:02:08.988 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:02:09.033 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:02:09.039 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:02:09.040 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:02:09.141 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:02:09.176 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:02:09.184 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:02:09.184 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:02:09.200 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 88 16 108512 108496 77.036 53.125 0.0234081 0.0129752 2023-06-03T20:02:09.286 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:02:09.320 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:02:09.322 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:02:09.323 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:02:09.424 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:02:09.519 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:02:09.521 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:02:09.522 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:02:09.623 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:02:09.657 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:02:09.660 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:02:09.661 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:02:09.761 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:02:09.770 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:02:09 smithi089 bash[75068]: cluster 1685822528.4599533 mgr.y (mgr.34104) 775 : cluster 0 pgmap v690: 89 pgs: 89 active+clean; 6.5 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 61 MiB/s wr, 982 op/s 2023-06-03T20:02:09.771 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:02:09 smithi089 bash[78452]: cluster 1685822528.4599533 mgr.y (mgr.34104) 775 : cluster 0 pgmap v690: 89 pgs: 89 active+clean; 6.5 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 61 MiB/s wr, 982 op/s 2023-06-03T20:02:09.798 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:02:09.801 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:02:09.802 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:02:09.903 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:02:09.944 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:02:09.947 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:02:09.948 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:02:10.048 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:02:10.057 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:02:09 smithi164 bash[59462]: cluster 1685822528.4599533 mgr.y (mgr.34104) 775 : cluster 0 pgmap v690: 89 pgs: 89 active+clean; 6.5 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 61 MiB/s wr, 982 op/s 2023-06-03T20:02:10.090 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:02:10.094 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:02:10.095 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:02:10.196 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:02:10.200 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 89 16 109361 109345 76.7666 53.0625 0.0107207 0.0130208 2023-06-03T20:02:10.231 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:02:10.235 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:02:10.236 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:02:10.337 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:02:10.388 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:02:10.391 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:02:10.391 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:02:10.493 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:02:10.524 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:02:10.526 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:02:10.527 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:02:10.627 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:02:10.659 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:02:10.666 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:02:10.666 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:02:10.767 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:02:10.809 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:02:10.813 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:02:10.814 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:02:10.814 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-06-03T20:02:10.886 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph osd set nodeep-scrub 2023-06-03T20:02:10.914 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:02:10.950 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:02:10.954 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:02:10.955 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:02:11.057 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:02:11.105 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:02:11.109 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:02:11.110 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:02:11.201 INFO:tasks.radosbench.radosbench.0.smithi089.stdout:Total time run: 90.013 2023-06-03T20:02:11.201 INFO:tasks.radosbench.radosbench.0.smithi089.stdout:Total writes made: 110131 2023-06-03T20:02:11.201 INFO:tasks.radosbench.radosbench.0.smithi089.stdout:Write size: 65536 2023-06-03T20:02:11.202 INFO:tasks.radosbench.radosbench.0.smithi089.stdout:Object size: 65536 2023-06-03T20:02:11.202 INFO:tasks.radosbench.radosbench.0.smithi089.stdout:Bandwidth (MB/sec): 76.4689 2023-06-03T20:02:11.202 INFO:tasks.radosbench.radosbench.0.smithi089.stdout:Stddev Bandwidth: 27.605 2023-06-03T20:02:11.202 INFO:tasks.radosbench.radosbench.0.smithi089.stdout:Max bandwidth (MB/sec): 182.688 2023-06-03T20:02:11.202 INFO:tasks.radosbench.radosbench.0.smithi089.stdout:Min bandwidth (MB/sec): 42.75 2023-06-03T20:02:11.202 INFO:tasks.radosbench.radosbench.0.smithi089.stdout:Average IOPS: 1223 2023-06-03T20:02:11.202 INFO:tasks.radosbench.radosbench.0.smithi089.stdout:Stddev IOPS: 441.68 2023-06-03T20:02:11.203 INFO:tasks.radosbench.radosbench.0.smithi089.stdout:Max IOPS: 2923 2023-06-03T20:02:11.203 INFO:tasks.radosbench.radosbench.0.smithi089.stdout:Min IOPS: 684 2023-06-03T20:02:11.203 INFO:tasks.radosbench.radosbench.0.smithi089.stdout:Average Latency(s): 0.0130726 2023-06-03T20:02:11.203 INFO:tasks.radosbench.radosbench.0.smithi089.stdout:Stddev Latency(s): 0.0123915 2023-06-03T20:02:11.203 INFO:tasks.radosbench.radosbench.0.smithi089.stdout:Max latency(s): 0.378924 2023-06-03T20:02:11.203 INFO:tasks.radosbench.radosbench.0.smithi089.stdout:Min latency(s): 0.00125194 2023-06-03T20:02:11.211 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:02:11.211 INFO:tasks.radosbench.radosbench.0.smithi089.stdout:Cleaning up (deleting benchmark objects) 2023-06-03T20:02:11.247 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:02:11.253 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:02:11.254 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:02:11.355 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:02:11.384 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:02:11.391 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:02:11.392 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:02:11.493 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:02:11.525 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:02:11.528 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:02:11.529 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:02:11.630 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:02:11.664 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:02:11.666 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:02:11.667 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:02:11.768 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:02:11.776 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:02:11 smithi089 bash[75068]: cluster 1685822530.46043 mgr.y (mgr.34104) 776 : cluster 0 pgmap v691: 89 pgs: 89 active+clean; 6.6 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 61 MiB/s wr, 976 op/s 2023-06-03T20:02:11.777 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:02:11 smithi089 bash[78452]: cluster 1685822530.46043 mgr.y (mgr.34104) 776 : cluster 0 pgmap v691: 89 pgs: 89 active+clean; 6.6 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 61 MiB/s wr, 976 op/s 2023-06-03T20:02:11.803 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:02:11.807 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:02:11.807 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:02:11.834 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:02:11 smithi164 bash[59462]: cluster 1685822530.46043 mgr.y (mgr.34104) 776 : cluster 0 pgmap v691: 89 pgs: 89 active+clean; 6.6 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 61 MiB/s wr, 976 op/s 2023-06-03T20:02:11.904 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/mon.a/config 2023-06-03T20:02:11.909 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:02:11.953 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:02:11.956 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:02:11.957 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:02:12.058 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:02:12.087 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:02:12.090 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:02:12.091 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:02:12.191 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:02:12.225 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:02:12.229 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:02:12.230 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:02:12.331 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:02:12.379 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:02:12.382 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:02:12.383 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:02:12.484 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:02:12.514 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:02:12.516 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:02:12.517 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:02:12.618 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:02:12.652 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:02:12.655 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:02:12.656 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:02:12.757 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:02:12.792 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:02:12.795 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:02:12.795 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:02:12.896 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:02:12.926 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:02:12.929 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:02:12.930 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:02:12.941 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/mon.a/config 2023-06-03T20:02:13.031 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:02:13.070 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:02:13.072 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:02:13.073 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:02:13.173 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:02:13.214 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:02:13.215 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:02:13.215 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:02:13.316 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:02:13.349 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:02:13.356 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:02:13.357 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:02:13.458 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:02:13.503 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:02:13.507 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:02:13.507 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:02:13.608 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:02:13.656 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:02:13.658 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:02:13.658 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:02:13.759 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:02:13.769 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:02:13 smithi164 bash[59462]: cluster 1685822532.4610307 mgr.y (mgr.34104) 777 : cluster 0 pgmap v692: 89 pgs: 89 active+clean; 6.6 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 34 MiB/s wr, 549 op/s 2023-06-03T20:02:13.808 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:02:13.811 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:02:13.812 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:02:13.913 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:02:13.947 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:02:13.954 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:02:13.955 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:02:14.028 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:02:13 smithi089 bash[75068]: cluster 1685822532.4610307 mgr.y (mgr.34104) 777 : cluster 0 pgmap v692: 89 pgs: 89 active+clean; 6.6 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 34 MiB/s wr, 549 op/s 2023-06-03T20:02:14.029 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:02:13 smithi089 bash[78452]: cluster 1685822532.4610307 mgr.y (mgr.34104) 777 : cluster 0 pgmap v692: 89 pgs: 89 active+clean; 6.6 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 34 MiB/s wr, 549 op/s 2023-06-03T20:02:14.055 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:02:14.090 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:02:14.094 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:02:14.095 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:02:14.196 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:02:14.224 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:02:14.227 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:02:14.228 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:02:14.329 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:02:14.366 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:02:14.368 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:02:14.369 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:02:14.470 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:02:14.502 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:02:14.508 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:02:14.509 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:02:14.610 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:02:14.659 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:02:14.665 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:02:14.666 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:02:14.766 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:02:14.805 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:02:14.811 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:02:14.811 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:02:14.912 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:02:14.949 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:02:14.956 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:02:14.957 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:02:14.997 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/mon.a/config 2023-06-03T20:02:15.002 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/mon.a/config 2023-06-03T20:02:15.059 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:02:15.097 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:02:15.108 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:02:15.109 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:02:15.210 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:02:15.288 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:02:15.292 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:02:15.293 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:02:15.394 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:02:15.429 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:02:15.434 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:02:15.434 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:02:15.535 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:02:15.577 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:02:15.580 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:02:15.581 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:02:15.682 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:02:15.729 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:02:15.734 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:02:15.735 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:02:15.778 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:02:15 smithi089 bash[75068]: cluster 1685822534.4624724 mgr.y (mgr.34104) 778 : cluster 0 pgmap v693: 89 pgs: 89 active+clean; 6.6 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 115 B/s rd, 43 MiB/s wr, 829 op/s 2023-06-03T20:02:15.779 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:02:15 smithi089 bash[75068]: audit 1685822535.0266345 mon.c (mon.2) 396 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-03T20:02:15.779 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:02:15 smithi089 bash[78452]: cluster 1685822534.4624724 mgr.y (mgr.34104) 778 : cluster 0 pgmap v693: 89 pgs: 89 active+clean; 6.6 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 115 B/s rd, 43 MiB/s wr, 829 op/s 2023-06-03T20:02:15.779 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:02:15 smithi089 bash[78452]: audit 1685822535.0266345 mon.c (mon.2) 396 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-03T20:02:15.836 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:02:15.868 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:02:15.874 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:02:15.875 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:02:15.976 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:02:15.984 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:02:15 smithi164 bash[59462]: cluster 1685822534.4624724 mgr.y (mgr.34104) 778 : cluster 0 pgmap v693: 89 pgs: 89 active+clean; 6.6 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 115 B/s rd, 43 MiB/s wr, 829 op/s 2023-06-03T20:02:15.985 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:02:15 smithi164 bash[59462]: audit 1685822535.0266345 mon.c (mon.2) 396 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-03T20:02:16.016 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:02:16.019 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:02:16.019 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:02:16.120 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:02:16.142 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:02:16.143 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:02:16.144 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:02:16.245 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:02:16.261 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 20:02:16 smithi089 bash[71163]: ::ffff:172.21.15.164 - - [03/Jun/2023:20:02:16] "GET /metrics HTTP/1.1" 200 228283 "" "Prometheus/2.33.4" 2023-06-03T20:02:16.305 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:02:16.312 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:02:16.313 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:02:16.413 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:02:16.450 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:02:16.452 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:02:16.453 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:02:16.554 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:02:16.598 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:02:16.604 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:02:16.605 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:02:16.706 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:02:16.821 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:02:16.825 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:02:16.826 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:02:16.927 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:02:16.959 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:02:16.966 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:02:16.967 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:02:17.069 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:02:17.100 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:02:17.102 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:02:17.102 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:02:17.203 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:02:17.240 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:02:17.245 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:02:17.246 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:02:17.347 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:02:17.396 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:02:17.397 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:02:17.398 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:02:17.499 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:02:17.535 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:02:17.538 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:02:17.539 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:02:17.640 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:02:17.677 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:02:17.683 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:02:17.684 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:02:17.785 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:02:17.789 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:02:17 smithi089 bash[78452]: cluster 1685822536.462983 mgr.y (mgr.34104) 779 : cluster 0 pgmap v694: 89 pgs: 89 active+clean; 6.6 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 102 B/s rd, 39 MiB/s wr, 813 op/s 2023-06-03T20:02:17.789 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:02:17 smithi089 bash[75068]: cluster 1685822536.462983 mgr.y (mgr.34104) 779 : cluster 0 pgmap v694: 89 pgs: 89 active+clean; 6.6 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 102 B/s rd, 39 MiB/s wr, 813 op/s 2023-06-03T20:02:17.813 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:02:17.818 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:02:17.819 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:02:17.919 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:02:17.929 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:02:17 smithi164 bash[59462]: cluster 1685822536.462983 mgr.y (mgr.34104) 779 : cluster 0 pgmap v694: 89 pgs: 89 active+clean; 6.6 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 102 B/s rd, 39 MiB/s wr, 813 op/s 2023-06-03T20:02:17.961 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:02:17.967 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:02:17.968 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:02:18.069 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:02:18.114 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:02:18.118 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:02:18.118 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:02:18.219 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:02:18.261 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:02:18.266 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:02:18.267 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:02:18.368 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:02:18.399 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:02:18.401 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:02:18.402 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:02:18.502 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:02:18.536 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:02:18.543 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:02:18.544 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:02:18.645 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:02:18.675 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:02:18.680 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:02:18.681 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:02:18.782 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:02:18.809 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:02:18.816 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:02:18.817 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:02:18.918 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:02:18.956 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:02:18.958 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:02:18.959 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:02:19.060 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:02:19.089 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:02:19.091 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:02:19.091 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:02:19.192 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:02:19.214 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:02:19.216 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:02:19.217 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:02:19.318 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:02:19.347 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:02:19.352 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:02:19.353 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:02:19.454 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:02:19.482 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:02:19.485 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:02:19.485 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:02:19.586 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:02:19.624 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:02:19.628 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:02:19.629 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:02:19.730 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:02:19.755 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:02:19 smithi164 bash[59462]: cluster 1685822538.4635754 mgr.y (mgr.34104) 780 : cluster 0 pgmap v695: 89 pgs: 89 active+clean; 6.4 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 86 B/s rd, 33 MiB/s wr, 962 op/s 2023-06-03T20:02:19.777 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:02:19.782 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:02:19.783 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:02:19.884 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:02:19.925 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:02:19.931 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:02:19.932 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:02:20.029 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:02:19 smithi089 bash[78452]: cluster 1685822538.4635754 mgr.y (mgr.34104) 780 : cluster 0 pgmap v695: 89 pgs: 89 active+clean; 6.4 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 86 B/s rd, 33 MiB/s wr, 962 op/s 2023-06-03T20:02:20.029 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:02:19 smithi089 bash[75068]: cluster 1685822538.4635754 mgr.y (mgr.34104) 780 : cluster 0 pgmap v695: 89 pgs: 89 active+clean; 6.4 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 86 B/s rd, 33 MiB/s wr, 962 op/s 2023-06-03T20:02:20.033 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:02:20.060 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:02:20.062 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:02:20.063 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:02:20.163 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:02:20.192 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:02:20.194 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:02:20.195 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:02:20.296 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:02:20.334 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:02:20.336 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:02:20.336 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:02:20.437 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:02:20.478 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:02:20.482 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:02:20.482 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:02:20.583 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:02:20.610 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:02:20.613 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:02:20.614 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:02:20.715 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:02:20.764 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:02:20.770 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:02:20.771 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:02:20.871 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:02:20.900 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:02:20.903 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:02:20.903 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:02:21.004 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:02:21.064 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:02:21.067 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:02:21.068 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:02:21.169 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:02:21.221 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:02:21.224 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:02:21.225 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:02:21.326 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:02:21.367 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:02:21.369 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:02:21.369 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:02:21.470 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:02:21.506 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:02:21.510 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:02:21.510 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:02:21.611 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:02:21.652 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:02:21.656 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:02:21.657 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:02:21.758 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:02:21.786 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:02:21.787 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:02:21.788 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:02:21.889 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:02:21.898 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:02:21 smithi089 bash[75068]: cluster 1685822540.4640894 mgr.y (mgr.34104) 781 : cluster 0 pgmap v696: 89 pgs: 89 active+clean; 6.4 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 85 B/s rd, 16 MiB/s wr, 755 op/s 2023-06-03T20:02:21.899 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:02:21 smithi089 bash[78452]: cluster 1685822540.4640894 mgr.y (mgr.34104) 781 : cluster 0 pgmap v696: 89 pgs: 89 active+clean; 6.4 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 85 B/s rd, 16 MiB/s wr, 755 op/s 2023-06-03T20:02:21.919 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:02:21.924 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:02:21.925 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:02:22.026 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:02:22.035 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:02:21 smithi164 bash[59462]: cluster 1685822540.4640894 mgr.y (mgr.34104) 781 : cluster 0 pgmap v696: 89 pgs: 89 active+clean; 6.4 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 85 B/s rd, 16 MiB/s wr, 755 op/s 2023-06-03T20:02:22.060 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:02:22.069 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:02:22.070 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:02:22.171 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:02:22.221 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:02:22.225 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:02:22.226 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:02:22.327 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:02:22.402 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:02:22.409 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:02:22.409 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:02:22.510 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:02:22.543 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:02:22.553 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:02:22.554 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:02:22.655 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:02:22.679 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:02:22.683 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:02:22.683 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:02:22.778 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:02:22 smithi089 bash[75068]: audit 1685822542.0975227 mon.c (mon.2) 397 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T20:02:22.779 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:02:22 smithi089 bash[75068]: audit 1685822542.4564967 mon.a (mon.0) 702 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:02:22.779 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:02:22 smithi089 bash[75068]: audit 1685822542.4757617 mon.a (mon.0) 703 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:02:22.779 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:02:22 smithi089 bash[78452]: audit 1685822542.0975227 mon.c (mon.2) 397 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T20:02:22.779 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:02:22 smithi089 bash[78452]: audit 1685822542.4564967 mon.a (mon.0) 702 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:02:22.780 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:02:22 smithi089 bash[78452]: audit 1685822542.4757617 mon.a (mon.0) 703 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:02:22.784 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:02:22.818 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:02:22.822 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:02:22.823 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:02:22.924 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:02:22.934 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:02:22 smithi164 bash[59462]: audit 1685822542.0975227 mon.c (mon.2) 397 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T20:02:22.935 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:02:22 smithi164 bash[59462]: audit 1685822542.4564967 mon.a (mon.0) 702 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:02:22.935 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:02:22 smithi164 bash[59462]: audit 1685822542.4757617 mon.a (mon.0) 703 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:02:22.968 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:02:22.972 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:02:22.972 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:02:23.073 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:02:23.108 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:02:23.115 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:02:23.115 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:02:23.216 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:02:23.242 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:02:23.243 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:02:23.244 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:02:23.345 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:02:23.377 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:02:23.379 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:02:23.380 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:02:23.481 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:02:23.508 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:02:23.510 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:02:23.511 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:02:23.612 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:02:23.642 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:02:23.647 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:02:23.648 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:02:23.749 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:02:23.756 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:02:23 smithi089 bash[75068]: cluster 1685822542.4653618 mgr.y (mgr.34104) 782 : cluster 0 pgmap v697: 89 pgs: 89 active+clean; 6.4 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 85 B/s rd, 12 MiB/s wr, 691 op/s 2023-06-03T20:02:23.757 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:02:23 smithi089 bash[78452]: cluster 1685822542.4653618 mgr.y (mgr.34104) 782 : cluster 0 pgmap v697: 89 pgs: 89 active+clean; 6.4 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 85 B/s rd, 12 MiB/s wr, 691 op/s 2023-06-03T20:02:23.787 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:02:23.790 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:02:23.791 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:02:23.892 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:02:23.930 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:02:23.933 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:02:23.934 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:02:24.035 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:02:24.063 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:02:24.066 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:02:24.067 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:02:24.083 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:02:23 smithi164 bash[59462]: cluster 1685822542.4653618 mgr.y (mgr.34104) 782 : cluster 0 pgmap v697: 89 pgs: 89 active+clean; 6.4 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 85 B/s rd, 12 MiB/s wr, 691 op/s 2023-06-03T20:02:24.168 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:02:24.193 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:02:24.194 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:02:24.195 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:02:24.296 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:02:24.324 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:02:24.326 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:02:24.326 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:02:24.427 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:02:24.460 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:02:24.463 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:02:24.464 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:02:24.565 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:02:24.595 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:02:24.598 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:02:24.599 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:02:24.700 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:02:24.731 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:02:24.733 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:02:24.734 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:02:24.834 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:02:24.870 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:02:24.874 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:02:24.875 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:02:24.976 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:02:25.007 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:02:25.012 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:02:25.012 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:02:25.113 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:02:25.142 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:02:25.146 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:02:25.147 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:02:25.249 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:02:25.280 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:02:25.282 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:02:25.282 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:02:25.383 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:02:25.434 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:02:25.440 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:02:25.441 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:02:25.542 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:02:25.587 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:02:25.593 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:02:25.594 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:02:25.695 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:02:25.736 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:02:25.739 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:02:25.740 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:02:25.841 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:02:25.889 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:02:25.892 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:02:25.893 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:02:25.974 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:02:25 smithi089 bash[78452]: cluster 1685822544.4659898 mgr.y (mgr.34104) 783 : cluster 0 pgmap v698: 89 pgs: 89 active+clean; 6.2 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 85 B/s rd, 12 MiB/s wr, 941 op/s 2023-06-03T20:02:25.974 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:02:25 smithi089 bash[78452]: audit 1685822545.4337668 mon.a (mon.0) 704 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:02:25.974 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:02:25 smithi089 bash[78452]: audit 1685822545.446575 mon.a (mon.0) 705 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:02:25.974 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:02:25 smithi089 bash[75068]: cluster 1685822544.4659898 mgr.y (mgr.34104) 783 : cluster 0 pgmap v698: 89 pgs: 89 active+clean; 6.2 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 85 B/s rd, 12 MiB/s wr, 941 op/s 2023-06-03T20:02:25.975 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:02:25 smithi089 bash[75068]: audit 1685822545.4337668 mon.a (mon.0) 704 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:02:25.975 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:02:25 smithi089 bash[75068]: audit 1685822545.446575 mon.a (mon.0) 705 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:02:25.994 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:02:26.036 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:02:26.038 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:02:26.038 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:02:26.084 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:02:25 smithi164 bash[59462]: cluster 1685822544.4659898 mgr.y (mgr.34104) 783 : cluster 0 pgmap v698: 89 pgs: 89 active+clean; 6.2 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 85 B/s rd, 12 MiB/s wr, 941 op/s 2023-06-03T20:02:26.084 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:02:25 smithi164 bash[59462]: audit 1685822545.4337668 mon.a (mon.0) 704 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:02:26.084 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:02:25 smithi164 bash[59462]: audit 1685822545.446575 mon.a (mon.0) 705 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:02:26.140 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:02:26.171 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:02:26.175 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:02:26.176 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:02:26.234 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 20:02:26 smithi089 bash[71163]: ::ffff:172.21.15.164 - - [03/Jun/2023:20:02:26] "GET /metrics HTTP/1.1" 200 228295 "" "Prometheus/2.33.4" 2023-06-03T20:02:26.277 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:02:26.308 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:02:26.315 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:02:26.315 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:02:26.416 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:02:26.460 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:02:26.463 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:02:26.463 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:02:26.564 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:02:26.593 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:02:26.598 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:02:26.599 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:02:26.700 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:02:26.737 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:02:26.740 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:02:26.741 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:02:26.842 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:02:26.897 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:02:26.899 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:02:26.899 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:02:27.000 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:02:27.011 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:02:26 smithi089 bash[75068]: audit 1685822546.0471487 mon.c (mon.2) 398 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-03T20:02:27.011 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:02:26 smithi089 bash[75068]: audit 1685822546.04832 mon.c (mon.2) 399 : audit 1 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-03T20:02:27.012 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:02:26 smithi089 bash[75068]: audit 1685822546.054397 mon.a (mon.0) 706 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:02:27.012 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:02:26 smithi089 bash[78452]: audit 1685822546.0471487 mon.c (mon.2) 398 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-03T20:02:27.012 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:02:26 smithi089 bash[78452]: audit 1685822546.04832 mon.c (mon.2) 399 : audit 1 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-03T20:02:27.012 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:02:26 smithi089 bash[78452]: audit 1685822546.054397 mon.a (mon.0) 706 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:02:27.033 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:02:27.038 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:02:27.039 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:02:27.083 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:02:26 smithi164 bash[59462]: audit 1685822546.0471487 mon.c (mon.2) 398 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-03T20:02:27.083 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:02:26 smithi164 bash[59462]: audit 1685822546.04832 mon.c (mon.2) 399 : audit 1 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-03T20:02:27.084 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:02:26 smithi164 bash[59462]: audit 1685822546.054397 mon.a (mon.0) 706 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:02:27.140 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:02:27.180 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:02:27.183 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:02:27.184 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:02:27.284 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:02:27.319 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:02:27.326 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:02:27.327 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:02:27.428 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:02:27.460 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:02:27.468 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:02:27.469 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:02:27.570 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:02:27.599 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:02:27.604 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:02:27.605 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:02:27.706 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:02:27.734 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:02:27.735 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:02:27.736 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:02:27.833 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:02:27 smithi164 bash[59462]: cluster 1685822546.4663951 mgr.y (mgr.34104) 784 : cluster 0 pgmap v699: 89 pgs: 89 active+clean; 6.1 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 1.1 MiB/s wr, 709 op/s 2023-06-03T20:02:27.837 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:02:27.856 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:02:27 smithi089 bash[78452]: cluster 1685822546.4663951 mgr.y (mgr.34104) 784 : cluster 0 pgmap v699: 89 pgs: 89 active+clean; 6.1 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 1.1 MiB/s wr, 709 op/s 2023-06-03T20:02:27.857 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:02:27 smithi089 bash[75068]: cluster 1685822546.4663951 mgr.y (mgr.34104) 784 : cluster 0 pgmap v699: 89 pgs: 89 active+clean; 6.1 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 1.1 MiB/s wr, 709 op/s 2023-06-03T20:02:27.880 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:02:27.882 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:02:27.883 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:02:27.984 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:02:28.019 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:02:28.024 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:02:28.024 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:02:28.125 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:02:28.155 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:02:28.161 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:02:28.161 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:02:28.262 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:02:28.296 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:02:28.303 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:02:28.303 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:02:28.405 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:02:28.446 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:02:28.450 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:02:28.451 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:02:28.552 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:02:28.587 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:02:28.588 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:02:28.589 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:02:28.637 INFO:teuthology.orchestra.run.smithi089.stdout:ERROR: (22) Invalid argument 2023-06-03T20:02:28.638 INFO:teuthology.orchestra.run.smithi089.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-06-03T20:02:28.690 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:02:28.729 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:02:28.733 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:02:28.734 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:02:28.835 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:02:28.876 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:02:28.885 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:02:28.886 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:02:28.987 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:02:29.025 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:02:29.032 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:02:29.033 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:02:29.134 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:02:29.167 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:02:29.172 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:02:29.172 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:02:29.273 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:02:29.310 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:02:29.312 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:02:29.313 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:02:29.414 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:02:29.448 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:02:29.451 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:02:29.451 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:02:29.552 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:02:29.588 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:02:29.592 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:02:29.594 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:02:29.695 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:02:29.730 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:02:29.738 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:02:29.739 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:02:29.833 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:02:29 smithi164 bash[59462]: cluster 1685822548.467109 mgr.y (mgr.34104) 785 : cluster 0 pgmap v700: 89 pgs: 89 active+clean; 6.0 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 900 op/s 2023-06-03T20:02:29.840 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:02:29.880 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:02:29.886 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:02:29.887 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:02:29.988 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:02:30.031 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:02:30.037 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:02:30.038 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:02:30.055 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:02:29 smithi089 bash[78452]: cluster 1685822548.467109 mgr.y (mgr.34104) 785 : cluster 0 pgmap v700: 89 pgs: 89 active+clean; 6.0 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 900 op/s 2023-06-03T20:02:30.056 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:02:29 smithi089 bash[75068]: cluster 1685822548.467109 mgr.y (mgr.34104) 785 : cluster 0 pgmap v700: 89 pgs: 89 active+clean; 6.0 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 900 op/s 2023-06-03T20:02:30.139 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:02:30.178 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:02:30.186 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:02:30.187 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:02:30.288 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:02:30.320 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:02:30.323 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:02:30.324 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:02:30.424 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:02:30.464 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:02:30.466 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:02:30.466 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:02:30.567 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:02:30.608 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:02:30.613 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:02:30.614 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:02:30.689 INFO:teuthology.orchestra.run.smithi089.stderr:reweighted osd.5 to 1 (10000) 2023-06-03T20:02:30.715 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:02:30.750 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:02:30.756 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:02:30.757 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:02:30.834 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:02:30 smithi164 bash[59462]: audit 1685822549.703604 mon.a (mon.0) 707 : audit 1 from='client.? 172.21.15.89:0/201496782' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-06-03T20:02:30.835 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:02:30 smithi089 bash[75068]: audit 1685822549.703604 mon.a (mon.0) 707 : audit 1 from='client.? 172.21.15.89:0/201496782' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-06-03T20:02:30.835 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:02:30 smithi089 bash[78452]: audit 1685822549.703604 mon.a (mon.0) 707 : audit 1 from='client.? 172.21.15.89:0/201496782' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-06-03T20:02:30.858 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:02:30.885 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:02:30.889 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:02:30.890 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:02:30.991 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:02:31.023 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:02:31.025 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:02:31.026 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:02:31.128 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:02:31.161 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:02:31.164 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:02:31.164 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:02:31.265 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:02:31.303 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:02:31.307 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:02:31.308 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:02:31.409 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:02:31.444 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:02:31.446 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:02:31.447 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:02:31.548 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:02:31.581 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:02:31.583 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:02:31.584 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:02:31.685 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:02:31.716 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:02:31.719 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:02:31.719 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:02:31.720 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:02:31 smithi089 bash[75068]: cluster 1685822550.467557 mgr.y (mgr.34104) 786 : cluster 0 pgmap v701: 89 pgs: 89 active+clean; 5.9 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 680 op/s 2023-06-03T20:02:31.720 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:02:31 smithi089 bash[75068]: cluster 1685822550.6658394 osd.6 (osd.6) 56 : cluster 3 failed to encode map e137 with expected crc 2023-06-03T20:02:31.720 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:02:31 smithi089 bash[75068]: cluster 1685822550.6718652 osd.7 (osd.7) 50 : cluster 3 failed to encode map e137 with expected crc 2023-06-03T20:02:31.721 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:02:31 smithi089 bash[75068]: audit 1685822550.675347 mon.a (mon.0) 708 : audit 1 from='client.? 172.21.15.89:0/201496782' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2023-06-03T20:02:31.721 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:02:31 smithi089 bash[75068]: cluster 1685822550.675623 mon.a (mon.0) 709 : cluster 0 osdmap e137: 8 total, 8 up, 8 in 2023-06-03T20:02:31.820 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:02:31.852 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:02:31.854 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:02:31.855 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:02:31.956 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:02:31.963 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph daemon osd.2 dump_blocked_ops 2023-06-03T20:02:31.965 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:02:31 smithi164 bash[59462]: cluster 1685822550.467557 mgr.y (mgr.34104) 786 : cluster 0 pgmap v701: 89 pgs: 89 active+clean; 5.9 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 680 op/s 2023-06-03T20:02:31.965 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:02:31 smithi164 bash[59462]: cluster 1685822550.6658394 osd.6 (osd.6) 56 : cluster 3 failed to encode map e137 with expected crc 2023-06-03T20:02:31.965 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:02:31 smithi164 bash[59462]: cluster 1685822550.6718652 osd.7 (osd.7) 50 : cluster 3 failed to encode map e137 with expected crc 2023-06-03T20:02:31.965 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:02:31 smithi164 bash[59462]: audit 1685822550.675347 mon.a (mon.0) 708 : audit 1 from='client.? 172.21.15.89:0/201496782' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2023-06-03T20:02:31.965 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:02:31 smithi164 bash[59462]: cluster 1685822550.675623 mon.a (mon.0) 709 : cluster 0 osdmap e137: 8 total, 8 up, 8 in 2023-06-03T20:02:31.966 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:02:31 smithi164 bash[59462]: cluster 1685822550.678219 osd.5 (osd.5) 62 : cluster 3 failed to encode map e137 with expected crc 2023-06-03T20:02:31.972 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:02:31 smithi089 bash[78452]: cluster 1685822550.467557 mgr.y (mgr.34104) 786 : cluster 0 pgmap v701: 89 pgs: 89 active+clean; 5.9 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 680 op/s 2023-06-03T20:02:31.972 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:02:31 smithi089 bash[78452]: cluster 1685822550.6658394 osd.6 (osd.6) 56 : cluster 3 failed to encode map e137 with expected crc 2023-06-03T20:02:31.972 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:02:31 smithi089 bash[78452]: cluster 1685822550.6718652 osd.7 (osd.7) 50 : cluster 3 failed to encode map e137 with expected crc 2023-06-03T20:02:31.973 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:02:31 smithi089 bash[78452]: audit 1685822550.675347 mon.a (mon.0) 708 : audit 1 from='client.? 172.21.15.89:0/201496782' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2023-06-03T20:02:31.973 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:02:31 smithi089 bash[78452]: cluster 1685822550.675623 mon.a (mon.0) 709 : cluster 0 osdmap e137: 8 total, 8 up, 8 in 2023-06-03T20:02:31.973 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:02:31 smithi089 bash[78452]: cluster 1685822550.678219 osd.5 (osd.5) 62 : cluster 3 failed to encode map e137 with expected crc 2023-06-03T20:02:31.974 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:02:31 smithi089 bash[75068]: cluster 1685822550.678219 osd.5 (osd.5) 62 : cluster 3 failed to encode map e137 with expected crc 2023-06-03T20:02:31.991 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:02:31.998 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:02:31.999 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:02:32.100 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:02:32.131 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:02:32.136 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:02:32.137 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:02:32.238 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:02:32.270 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:02:32.278 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:02:32.279 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:02:32.380 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:02:32.415 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:02:32.416 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:02:32.416 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:02:32.517 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:02:32.544 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:02:32.549 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:02:32.550 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:02:32.650 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:02:32.679 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:02:32.682 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:02:32.683 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:02:32.783 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:02:32.817 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:02:32.822 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:02:32.822 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:02:32.923 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:02:32.955 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:02:32.960 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:02:32.961 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:02:33.061 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:02:33.096 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:02:33.099 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:02:33.100 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:02:33.201 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:02:33.245 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:02:33.247 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:02:33.247 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:02:33.348 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:02:33.386 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:02:33.389 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:02:33.390 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:02:33.398 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph osd reweight 6 1 2023-06-03T20:02:33.491 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:02:33.542 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:02:33.545 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:02:33.546 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:02:33.647 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:02:33.658 INFO:teuthology.orchestra.run.smithi089.stdout:osd.0: {} 2023-06-03T20:02:33.697 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:02:33.701 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:02:33.702 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:02:33.706 INFO:teuthology.orchestra.run.smithi089.stdout:osd.1: {} 2023-06-03T20:02:33.748 INFO:teuthology.orchestra.run.smithi089.stdout:osd.2: {} 2023-06-03T20:02:33.778 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:02:33 smithi089 bash[78452]: cluster 1685822552.468011 mgr.y (mgr.34104) 787 : cluster 0 pgmap v703: 89 pgs: 89 active+clean; 5.9 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 738 op/s 2023-06-03T20:02:33.779 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:02:33 smithi089 bash[78452]: audit 1685822553.5957386 mon.a (mon.0) 710 : audit 0 from='client.? 172.21.15.89:0/3636343887' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-03T20:02:33.779 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:02:33 smithi089 bash[75068]: cluster 1685822552.468011 mgr.y (mgr.34104) 787 : cluster 0 pgmap v703: 89 pgs: 89 active+clean; 5.9 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 738 op/s 2023-06-03T20:02:33.780 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:02:33 smithi089 bash[75068]: audit 1685822553.5957386 mon.a (mon.0) 710 : audit 0 from='client.? 172.21.15.89:0/3636343887' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-03T20:02:33.790 INFO:teuthology.orchestra.run.smithi089.stdout:osd.3: {} 2023-06-03T20:02:33.803 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:02:33.835 INFO:teuthology.orchestra.run.smithi089.stdout:osd.4: {} 2023-06-03T20:02:33.861 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:02:33.866 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:02:33.866 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:02:33.874 INFO:teuthology.orchestra.run.smithi089.stdout:osd.5: {} 2023-06-03T20:02:33.912 INFO:teuthology.orchestra.run.smithi089.stdout:osd.6: {} 2023-06-03T20:02:33.946 INFO:teuthology.orchestra.run.smithi089.stdout:osd.7: {} 2023-06-03T20:02:33.951 INFO:teuthology.orchestra.run.smithi089.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-06-03T20:02:33.951 INFO:teuthology.orchestra.run.smithi089.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-06-03T20:02:33.951 INFO:teuthology.orchestra.run.smithi089.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-06-03T20:02:33.951 INFO:teuthology.orchestra.run.smithi089.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-06-03T20:02:33.951 INFO:teuthology.orchestra.run.smithi089.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-06-03T20:02:33.951 INFO:teuthology.orchestra.run.smithi089.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-06-03T20:02:33.952 INFO:teuthology.orchestra.run.smithi089.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-06-03T20:02:33.952 INFO:teuthology.orchestra.run.smithi089.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-06-03T20:02:33.967 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:02:34.006 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:02:34.010 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:02:34.011 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:02:34.083 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:02:33 smithi164 bash[59462]: cluster 1685822552.468011 mgr.y (mgr.34104) 787 : cluster 0 pgmap v703: 89 pgs: 89 active+clean; 5.9 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 738 op/s 2023-06-03T20:02:34.084 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:02:33 smithi164 bash[59462]: audit 1685822553.5957386 mon.a (mon.0) 710 : audit 0 from='client.? 172.21.15.89:0/3636343887' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-03T20:02:34.112 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:02:34.149 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:02:34.153 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:02:34.154 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:02:34.255 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:02:34.297 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:02:34.300 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:02:34.301 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:02:34.402 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:02:34.433 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:02:34.436 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:02:34.436 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:02:34.537 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:02:34.585 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:02:34.591 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:02:34.592 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:02:34.693 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:02:34.698 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:02:34 smithi164 bash[59462]: audit 1685822554.0584433 mon.b (mon.1) 48 : audit 1 from='client.? 172.21.15.89:0/1876441903' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-06-03T20:02:34.698 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:02:34 smithi164 bash[59462]: audit 1685822554.060943 mon.a (mon.0) 711 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-06-03T20:02:34.710 INFO:teuthology.orchestra.run.smithi089.stderr:nodeep-scrub is set 2023-06-03T20:02:34.731 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:02:34.734 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:02:34.735 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:02:34.836 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:02:34.878 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:02:34.882 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:02:34.882 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:02:34.983 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:02:35.022 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:02:35.024 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:02:35.025 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:02:35.029 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:02:34 smithi089 bash[78452]: audit 1685822554.0584433 mon.b (mon.1) 48 : audit 1 from='client.? 172.21.15.89:0/1876441903' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-06-03T20:02:35.029 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:02:34 smithi089 bash[78452]: audit 1685822554.060943 mon.a (mon.0) 711 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-06-03T20:02:35.029 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:02:34 smithi089 bash[75068]: audit 1685822554.0584433 mon.b (mon.1) 48 : audit 1 from='client.? 172.21.15.89:0/1876441903' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-06-03T20:02:35.029 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:02:34 smithi089 bash[75068]: audit 1685822554.060943 mon.a (mon.0) 711 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-06-03T20:02:35.126 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:02:35.155 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:02:35.160 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:02:35.161 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:02:35.263 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:02:35.294 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:02:35.298 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:02:35.299 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:02:35.401 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:02:35.441 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:02:35.446 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:02:35.447 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:02:35.548 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:02:35.608 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:02:35.610 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:02:35.610 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:02:35.696 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:02:35 smithi089 bash[75068]: cluster 1685822554.4685657 mgr.y (mgr.34104) 788 : cluster 0 pgmap v704: 89 pgs: 89 active+clean; 5.7 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 755 op/s 2023-06-03T20:02:35.696 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:02:35 smithi089 bash[75068]: cluster 1685822554.6788728 mon.a (mon.0) 712 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-06-03T20:02:35.712 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:02:35.754 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:02:35.756 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:02:35.757 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:02:35.857 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:02:35.892 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:02:35.894 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:02:35.895 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:02:35.996 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:02:36.005 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:02:35 smithi164 bash[59462]: cluster 1685822554.4685657 mgr.y (mgr.34104) 788 : cluster 0 pgmap v704: 89 pgs: 89 active+clean; 5.7 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 755 op/s 2023-06-03T20:02:36.006 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:02:35 smithi164 bash[59462]: cluster 1685822554.6788728 mon.a (mon.0) 712 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-06-03T20:02:36.006 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:02:35 smithi164 bash[59462]: cluster 1685822554.69191 osd.6 (osd.6) 57 : cluster 3 failed to encode map e138 with expected crc 2023-06-03T20:02:36.006 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:02:35 smithi164 bash[59462]: audit 1685822554.6964605 mon.a (mon.0) 713 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-06-03T20:02:36.006 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:02:35 smithi164 bash[59462]: cluster 1685822554.6965399 mon.a (mon.0) 714 : cluster 0 osdmap e138: 8 total, 8 up, 8 in 2023-06-03T20:02:36.029 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:02:35 smithi089 bash[75068]: cluster 1685822554.69191 osd.6 (osd.6) 57 : cluster 3 failed to encode map e138 with expected crc 2023-06-03T20:02:36.029 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:02:35 smithi089 bash[75068]: audit 1685822554.6964605 mon.a (mon.0) 713 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-06-03T20:02:36.029 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:02:35 smithi089 bash[75068]: cluster 1685822554.6965399 mon.a (mon.0) 714 : cluster 0 osdmap e138: 8 total, 8 up, 8 in 2023-06-03T20:02:36.029 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:02:35 smithi089 bash[78452]: cluster 1685822554.4685657 mgr.y (mgr.34104) 788 : cluster 0 pgmap v704: 89 pgs: 89 active+clean; 5.7 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 755 op/s 2023-06-03T20:02:36.030 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:02:35 smithi089 bash[78452]: cluster 1685822554.6788728 mon.a (mon.0) 712 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-06-03T20:02:36.030 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:02:35 smithi089 bash[78452]: cluster 1685822554.69191 osd.6 (osd.6) 57 : cluster 3 failed to encode map e138 with expected crc 2023-06-03T20:02:36.030 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:02:35 smithi089 bash[78452]: audit 1685822554.6964605 mon.a (mon.0) 713 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-06-03T20:02:36.030 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:02:35 smithi089 bash[78452]: cluster 1685822554.6965399 mon.a (mon.0) 714 : cluster 0 osdmap e138: 8 total, 8 up, 8 in 2023-06-03T20:02:36.033 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:02:36.042 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:02:36.042 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:02:36.143 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:02:36.173 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:02:36.178 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:02:36.178 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:02:36.279 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:02:36.311 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:02:36.314 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:02:36.315 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:02:36.416 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:02:36.464 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:02:36.470 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:02:36.471 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:02:36.529 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 20:02:36 smithi089 bash[71163]: ::ffff:172.21.15.164 - - [03/Jun/2023:20:02:36] "GET /metrics HTTP/1.1" 200 228295 "" "Prometheus/2.33.4" 2023-06-03T20:02:36.571 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:02:36.609 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:02:36.615 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:02:36.616 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:02:36.709 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:02:36 smithi164 bash[59462]: cluster 1685822554.6970747 osd.7 (osd.7) 51 : cluster 3 failed to encode map e138 with expected crc 2023-06-03T20:02:36.717 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:02:36.749 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:02:36.754 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:02:36.755 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:02:36.855 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:02:36.874 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:02:36 smithi089 bash[75068]: cluster 1685822554.6970747 osd.7 (osd.7) 51 : cluster 3 failed to encode map e138 with expected crc 2023-06-03T20:02:36.874 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:02:36 smithi089 bash[78452]: cluster 1685822554.6970747 osd.7 (osd.7) 51 : cluster 3 failed to encode map e138 with expected crc 2023-06-03T20:02:36.904 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:02:36.912 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:02:36.913 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:02:37.014 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:02:37.049 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:02:37.054 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:02:37.054 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:02:37.156 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:02:37.200 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:02:37.207 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:02:37.207 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:02:37.308 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:02:37.344 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:02:37.347 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:02:37.348 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:02:37.449 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:02:37.479 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:02:37.485 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:02:37.486 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:02:37.586 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:02:37.624 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:02:37.629 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:02:37.630 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:02:37.711 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:02:37 smithi089 bash[75068]: cluster 1685822556.4691577 mgr.y (mgr.34104) 789 : cluster 0 pgmap v706: 89 pgs: 89 active+clean; 5.7 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 572 op/s 2023-06-03T20:02:37.712 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:02:37 smithi089 bash[75068]: audit 1685822557.098643 mon.c (mon.2) 400 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T20:02:37.717 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-06-03T20:02:37.731 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:02:37.761 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:02:37.766 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:02:37.767 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:02:37.867 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:02:37.905 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:02:37.910 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:02:37.911 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:02:38.012 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:02:38.020 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:02:37 smithi164 bash[59462]: cluster 1685822556.4691577 mgr.y (mgr.34104) 789 : cluster 0 pgmap v706: 89 pgs: 89 active+clean; 5.7 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 572 op/s 2023-06-03T20:02:38.021 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:02:37 smithi164 bash[59462]: audit 1685822557.098643 mon.c (mon.2) 400 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T20:02:38.028 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:02:37 smithi089 bash[78452]: cluster 1685822556.4691577 mgr.y (mgr.34104) 789 : cluster 0 pgmap v706: 89 pgs: 89 active+clean; 5.7 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 572 op/s 2023-06-03T20:02:38.029 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:02:37 smithi089 bash[78452]: audit 1685822557.098643 mon.c (mon.2) 400 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T20:02:38.048 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:02:38.050 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:02:38.051 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:02:38.151 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:02:38.193 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:02:38.200 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:02:38.201 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:02:38.302 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:02:38.330 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:02:38.335 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:02:38.336 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:02:38.360 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph osd unset noscrub 2023-06-03T20:02:38.437 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:02:38.474 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:02:38.480 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:02:38.481 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:02:38.582 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:02:38.612 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:02:38.618 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:02:38.619 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:02:38.720 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:02:38.750 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:02:38.754 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:02:38.755 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:02:38.856 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:02:38.891 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:02:38.893 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/mon.a/config 2023-06-03T20:02:38.894 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:02:38.895 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:02:38.996 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:02:39.030 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:02:39.034 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:02:39.035 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:02:39.135 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:02:39.177 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:02:39.183 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:02:39.184 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:02:39.285 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:02:39.334 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:02:39.339 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:02:39.340 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:02:39.441 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:02:39.474 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:02:39.477 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:02:39.478 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:02:39.579 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:02:39.622 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:02:39.629 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:02:39.630 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:02:39.731 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:02:39.767 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:02:39.770 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:02:39.771 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:02:39.873 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:02:39.910 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:02:39.912 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:02:39.913 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:02:39.951 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/mon.a/config 2023-06-03T20:02:39.989 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:02:39 smithi089 bash[75068]: cluster 1685822558.469952 mgr.y (mgr.34104) 790 : cluster 0 pgmap v707: 89 pgs: 89 active+clean; 5.5 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 876 op/s 2023-06-03T20:02:39.990 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:02:39 smithi089 bash[78452]: cluster 1685822558.469952 mgr.y (mgr.34104) 790 : cluster 0 pgmap v707: 89 pgs: 89 active+clean; 5.5 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 876 op/s 2023-06-03T20:02:40.014 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:02:40.045 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:02:40.048 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:02:40.049 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:02:40.083 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:02:39 smithi164 bash[59462]: cluster 1685822558.469952 mgr.y (mgr.34104) 790 : cluster 0 pgmap v707: 89 pgs: 89 active+clean; 5.5 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 876 op/s 2023-06-03T20:02:40.150 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:02:40.188 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:02:40.193 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:02:40.194 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:02:40.295 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:02:40.335 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:02:40.338 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:02:40.338 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:02:40.439 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:02:40.479 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:02:40.481 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:02:40.482 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:02:40.583 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:02:40.627 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:02:40.631 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:02:40.632 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:02:40.733 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:02:40.782 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:02:40.786 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:02:40.786 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:02:40.887 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:02:40.922 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:02:40.923 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:02:40.924 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:02:41.025 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:02:41.066 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:02:41.070 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:02:41.071 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:02:41.172 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:02:41.207 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:02:41.210 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:02:41.211 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:02:41.312 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:02:41.359 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:02:41.364 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:02:41.365 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:02:41.466 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:02:41.500 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:02:41.502 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:02:41.503 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:02:41.604 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:02:41.649 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:02:41.652 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:02:41.653 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:02:41.753 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:02:41.795 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:02:41.798 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:02:41.799 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:02:41.900 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:02:41.954 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:02:41.956 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:02:41.957 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:02:41.999 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/mon.a/config 2023-06-03T20:02:42.032 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:02:41 smithi089 bash[75068]: cluster 1685822560.4705932 mgr.y (mgr.34104) 791 : cluster 0 pgmap v708: 89 pgs: 89 active+clean; 5.4 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 790 op/s 2023-06-03T20:02:42.033 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:02:41 smithi089 bash[78452]: cluster 1685822560.4705932 mgr.y (mgr.34104) 791 : cluster 0 pgmap v708: 89 pgs: 89 active+clean; 5.4 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 790 op/s 2023-06-03T20:02:42.059 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:02:42.083 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:02:41 smithi164 bash[59462]: cluster 1685822560.4705932 mgr.y (mgr.34104) 791 : cluster 0 pgmap v708: 89 pgs: 89 active+clean; 5.4 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 790 op/s 2023-06-03T20:02:42.097 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:02:42.098 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:02:42.100 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:02:42.201 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:02:42.250 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:02:42.254 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:02:42.255 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:02:42.356 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:02:42.394 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:02:42.396 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:02:42.397 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:02:42.498 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:02:42.530 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:02:42.538 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:02:42.539 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:02:42.640 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:02:42.672 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:02:42.674 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:02:42.674 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:02:42.775 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:02:42.823 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:02:42.828 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:02:42.829 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:02:42.930 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:02:42.968 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:02:42.970 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:02:42.970 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:02:43.034 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/mon.a/config 2023-06-03T20:02:43.071 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:02:43.117 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:02:43.120 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:02:43.121 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:02:43.222 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:02:43.250 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:02:43.256 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:02:43.257 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:02:43.358 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:02:43.388 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:02:43.393 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:02:43.393 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:02:43.495 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:02:43.529 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:02:43.532 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:02:43.533 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:02:43.634 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:02:43.668 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:02:43.672 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:02:43.673 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:02:43.719 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:02:43 smithi164 bash[59462]: cluster 1685822562.4713123 mgr.y (mgr.34104) 792 : cluster 0 pgmap v709: 89 pgs: 89 active+clean; 5.4 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 774 op/s 2023-06-03T20:02:43.773 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:02:43.778 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:02:43 smithi089 bash[75068]: cluster 1685822562.4713123 mgr.y (mgr.34104) 792 : cluster 0 pgmap v709: 89 pgs: 89 active+clean; 5.4 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 774 op/s 2023-06-03T20:02:43.778 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:02:43 smithi089 bash[78452]: cluster 1685822562.4713123 mgr.y (mgr.34104) 792 : cluster 0 pgmap v709: 89 pgs: 89 active+clean; 5.4 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 774 op/s 2023-06-03T20:02:43.813 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:02:43.817 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:02:43.817 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:02:43.918 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:02:43.955 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:02:43.958 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:02:43.959 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:02:44.060 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:02:44.101 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:02:44.106 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:02:44.107 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:02:44.208 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:02:44.262 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:02:44.267 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:02:44.268 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:02:44.369 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:02:44.409 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:02:44.414 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:02:44.415 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:02:44.515 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:02:44.563 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:02:44.564 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:02:44.565 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:02:44.665 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:02:44.692 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:02:44.695 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:02:44.695 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:02:44.797 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:02:44.832 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:02:44.835 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:02:44.836 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:02:44.941 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:02:44.971 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:02:44.974 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:02:44.975 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:02:45.076 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:02:45.104 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:02:45.106 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:02:45.107 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:02:45.207 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:02:45.248 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:02:45.256 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:02:45.257 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:02:45.358 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:02:45.401 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:02:45.404 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:02:45.405 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:02:45.505 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:02:45.526 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:02:45.527 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:02:45.527 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:02:45.628 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:02:45.668 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:02:45.670 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:02:45.671 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:02:45.772 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:02:45.813 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:02:45.815 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:02:45.816 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:02:45.917 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:02:45.929 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:02:45 smithi164 bash[59462]: cluster 1685822564.4742985 mgr.y (mgr.34104) 793 : cluster 0 pgmap v710: 89 pgs: 89 active+clean; 5.2 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 766 op/s 2023-06-03T20:02:45.959 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:02:45.964 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:02:45.965 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:02:46.028 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:02:45 smithi089 bash[75068]: cluster 1685822564.4742985 mgr.y (mgr.34104) 793 : cluster 0 pgmap v710: 89 pgs: 89 active+clean; 5.2 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 766 op/s 2023-06-03T20:02:46.029 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:02:45 smithi089 bash[78452]: cluster 1685822564.4742985 mgr.y (mgr.34104) 793 : cluster 0 pgmap v710: 89 pgs: 89 active+clean; 5.2 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 766 op/s 2023-06-03T20:02:46.066 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:02:46.097 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:02:46.102 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:02:46.103 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:02:46.204 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:02:46.232 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:02:46.234 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:02:46.234 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:02:46.335 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:02:46.344 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 20:02:46 smithi089 bash[71163]: ::ffff:172.21.15.164 - - [03/Jun/2023:20:02:46] "GET /metrics HTTP/1.1" 200 228283 "" "Prometheus/2.33.4" 2023-06-03T20:02:46.368 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:02:46.375 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:02:46.375 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:02:46.476 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:02:46.517 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:02:46.519 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:02:46.519 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:02:46.620 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:02:46.660 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:02:46.666 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:02:46.666 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:02:46.767 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:02:46.805 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:02:46.806 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:02:46.806 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:02:46.907 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:02:46.946 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:02:46.947 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:02:46.948 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:02:47.049 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:02:47.092 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:02:47.093 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:02:47.093 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:02:47.194 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:02:47.220 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:02:47.222 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:02:47.223 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:02:47.324 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:02:47.355 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:02:47.358 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:02:47.359 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:02:47.461 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:02:47.481 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:02:47.482 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:02:47.483 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:02:47.583 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:02:47.614 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:02:47.624 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:02:47.625 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:02:47.723 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:02:47 smithi089 bash[75068]: cluster 1685822566.4747393 mgr.y (mgr.34104) 794 : cluster 0 pgmap v711: 89 pgs: 89 active+clean; 5.2 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 707 op/s 2023-06-03T20:02:47.726 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:02:47.764 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:02:47.771 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:02:47.772 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:02:47.873 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:02:47.920 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:02:47.922 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:02:47.923 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:02:48.024 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:02:48.028 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:02:47 smithi089 bash[78452]: cluster 1685822566.4747393 mgr.y (mgr.34104) 794 : cluster 0 pgmap v711: 89 pgs: 89 active+clean; 5.2 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 707 op/s 2023-06-03T20:02:48.033 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:02:47 smithi164 bash[59462]: cluster 1685822566.4747393 mgr.y (mgr.34104) 794 : cluster 0 pgmap v711: 89 pgs: 89 active+clean; 5.2 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 707 op/s 2023-06-03T20:02:48.076 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:02:48.078 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:02:48.079 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:02:48.180 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:02:48.216 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:02:48.220 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:02:48.221 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:02:48.322 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:02:48.371 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:02:48.378 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:02:48.378 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:02:48.480 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:02:48.515 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:02:48.519 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:02:48.519 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:02:48.620 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:02:48.653 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:02:48.656 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:02:48.657 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:02:48.758 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:02:48.794 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:02:48.797 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:02:48.798 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:02:48.899 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:02:48.925 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:02:48.929 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:02:48.929 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:02:49.030 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:02:49.069 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:02:49.074 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:02:49.075 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:02:49.176 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:02:49.197 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:02:49.199 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:02:49.199 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:02:49.301 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:02:49.332 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:02:49.337 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:02:49.338 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:02:49.439 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:02:49.472 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:02:49.476 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:02:49.477 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:02:49.578 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:02:49.607 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:02:49.614 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:02:49.614 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:02:49.715 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:02:49.764 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:02:49.766 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:02:49.768 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:02:49.869 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:02:49.916 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:02:49.920 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:02:49.920 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:02:49.991 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:02:49 smithi089 bash[75068]: cluster 1685822568.4754353 mgr.y (mgr.34104) 795 : cluster 0 pgmap v712: 89 pgs: 89 active+clean; 5.0 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 876 op/s 2023-06-03T20:02:49.992 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:02:49 smithi089 bash[78452]: cluster 1685822568.4754353 mgr.y (mgr.34104) 795 : cluster 0 pgmap v712: 89 pgs: 89 active+clean; 5.0 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 876 op/s 2023-06-03T20:02:50.021 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:02:50.057 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:02:50.061 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:02:50.062 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:02:50.083 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:02:49 smithi164 bash[59462]: cluster 1685822568.4754353 mgr.y (mgr.34104) 795 : cluster 0 pgmap v712: 89 pgs: 89 active+clean; 5.0 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 876 op/s 2023-06-03T20:02:50.163 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:02:50.196 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:02:50.202 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:02:50.203 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:02:50.304 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:02:50.340 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:02:50.345 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:02:50.346 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:02:50.447 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:02:50.481 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:02:50.483 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:02:50.484 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:02:50.585 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:02:50.614 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:02:50.616 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:02:50.616 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:02:50.717 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:02:50.751 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:02:50.753 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:02:50.753 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:02:50.854 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:02:50.896 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:02:50.903 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:02:50.904 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:02:51.005 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:02:51.030 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:02:51.033 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:02:51.033 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:02:51.134 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:02:51.167 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:02:51.172 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:02:51.173 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:02:51.274 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:02:51.296 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:02:51.301 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:02:51.302 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:02:51.403 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:02:51.432 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:02:51.436 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:02:51.436 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:02:51.537 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:02:51.576 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:02:51.579 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:02:51.580 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:02:51.681 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:02:51.713 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:02:51.717 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:02:51.718 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:02:51.778 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:02:51 smithi089 bash[75068]: cluster 1685822570.475949 mgr.y (mgr.34104) 796 : cluster 0 pgmap v713: 89 pgs: 89 active+clean; 5.0 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 672 op/s 2023-06-03T20:02:51.779 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:02:51 smithi089 bash[78452]: cluster 1685822570.475949 mgr.y (mgr.34104) 796 : cluster 0 pgmap v713: 89 pgs: 89 active+clean; 5.0 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 672 op/s 2023-06-03T20:02:51.819 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:02:51.833 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:02:51 smithi164 bash[59462]: cluster 1685822570.475949 mgr.y (mgr.34104) 796 : cluster 0 pgmap v713: 89 pgs: 89 active+clean; 5.0 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 672 op/s 2023-06-03T20:02:51.872 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:02:51.877 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:02:51.878 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:02:51.979 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:02:52.011 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:02:52.015 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:02:52.016 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:02:52.117 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:02:52.155 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:02:52.160 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:02:52.161 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:02:52.261 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:02:52.299 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:02:52.304 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:02:52.304 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:02:52.405 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:02:52.434 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:02:52.440 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:02:52.441 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:02:52.542 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:02:52.572 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:02:52.577 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:02:52.578 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:02:52.679 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:02:52.710 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:02:52.718 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:02:52.719 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:02:52.819 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:02:52.850 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:02:52.856 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:02:52 smithi089 bash[75068]: audit 1685822572.0985377 mon.c (mon.2) 401 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T20:02:52.857 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:02:52.857 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:02:52.958 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:02:52.988 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:02:52.995 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:02:52.996 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:02:53.084 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:02:52 smithi164 bash[59462]: audit 1685822572.0985377 mon.c (mon.2) 401 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T20:02:53.098 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:02:53.106 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:02:52 smithi089 bash[78452]: audit 1685822572.0985377 mon.c (mon.2) 401 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T20:02:53.130 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:02:53.134 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:02:53.135 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:02:53.236 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:02:53.274 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:02:53.280 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:02:53.280 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:02:53.381 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:02:53.426 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:02:53.429 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:02:53.430 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:02:53.530 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:02:53.561 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:02:53.567 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:02:53.568 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:02:53.669 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:02:53.713 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:02:53.717 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:02:53.718 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:02:53.743 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:02:53 smithi089 bash[75068]: cluster 1685822572.4764569 mgr.y (mgr.34104) 797 : cluster 0 pgmap v714: 89 pgs: 89 active+clean; 5.0 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 611 op/s 2023-06-03T20:02:53.819 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:02:53.828 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:02:53 smithi164 bash[59462]: cluster 1685822572.4764569 mgr.y (mgr.34104) 797 : cluster 0 pgmap v714: 89 pgs: 89 active+clean; 5.0 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 611 op/s 2023-06-03T20:02:53.856 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:02:53.863 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:02:53.863 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:02:53.964 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:02:54.000 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:02:54.002 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:02:54.003 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:02:54.032 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:02:53 smithi089 bash[78452]: cluster 1685822572.4764569 mgr.y (mgr.34104) 797 : cluster 0 pgmap v714: 89 pgs: 89 active+clean; 5.0 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 611 op/s 2023-06-03T20:02:54.103 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:02:54.135 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:02:54.140 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:02:54.141 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:02:54.242 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:02:54.284 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:02:54.290 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:02:54.290 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:02:54.392 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:02:54.426 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:02:54.431 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:02:54.432 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:02:54.533 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:02:54.575 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:02:54.580 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:02:54.581 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:02:54.682 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:02:54.715 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:02:54.721 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:02:54.722 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:02:54.823 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:02:54.861 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:02:54.866 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:02:54.867 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:02:54.968 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:02:55.011 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:02:55.013 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:02:55.014 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:02:55.115 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:02:55.151 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:02:55.156 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:02:55.157 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:02:55.258 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:02:55.278 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:02:55.279 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:02:55.280 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:02:55.381 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:02:55.424 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:02:55.430 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:02:55.431 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:02:55.532 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:02:55.564 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:02:55.571 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:02:55.572 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:02:55.673 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:02:55.704 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:02:55.705 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:02:55.705 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:02:55.806 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:02:55.844 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:02:55.852 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:02:55.853 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:02:55.954 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:02:55.990 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:02:55.993 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:02:55.993 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:02:56.028 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:02:55 smithi089 bash[75068]: cluster 1685822574.4771352 mgr.y (mgr.34104) 798 : cluster 0 pgmap v715: 89 pgs: 89 active+clean; 4.8 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 857 op/s 2023-06-03T20:02:56.029 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:02:55 smithi089 bash[78452]: cluster 1685822574.4771352 mgr.y (mgr.34104) 798 : cluster 0 pgmap v715: 89 pgs: 89 active+clean; 4.8 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 857 op/s 2023-06-03T20:02:56.083 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:02:55 smithi164 bash[59462]: cluster 1685822574.4771352 mgr.y (mgr.34104) 798 : cluster 0 pgmap v715: 89 pgs: 89 active+clean; 4.8 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 857 op/s 2023-06-03T20:02:56.094 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:02:56.134 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:02:56.142 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:02:56.142 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:02:56.243 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:02:56.285 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:02:56.288 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:02:56.289 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:02:56.390 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:02:56.432 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:02:56.438 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:02:56.439 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:02:56.528 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 20:02:56 smithi089 bash[71163]: ::ffff:172.21.15.164 - - [03/Jun/2023:20:02:56] "GET /metrics HTTP/1.1" 200 228290 "" "Prometheus/2.33.4" 2023-06-03T20:02:56.540 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:02:56.569 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:02:56.573 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:02:56.573 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:02:56.674 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:02:56.707 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:02:56.711 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:02:56.712 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:02:56.813 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:02:56.846 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:02:56.849 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:02:56.850 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:02:56.951 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:02:56.984 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:02:56.987 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:02:56.988 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:02:57.006 INFO:teuthology.orchestra.run.smithi089.stdout:{ 2023-06-03T20:02:57.006 INFO:teuthology.orchestra.run.smithi089.stdout: "ops": [], 2023-06-03T20:02:57.006 INFO:teuthology.orchestra.run.smithi089.stdout: "complaint_time": 30, 2023-06-03T20:02:57.006 INFO:teuthology.orchestra.run.smithi089.stdout: "num_blocked_ops": 0 2023-06-03T20:02:57.006 INFO:teuthology.orchestra.run.smithi089.stdout:} 2023-06-03T20:02:57.089 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:02:57.126 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:02:57.132 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:02:57.133 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:02:57.234 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:02:57.276 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:02:57.278 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:02:57.279 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:02:57.380 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:02:57.419 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:02:57.425 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:02:57.426 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:02:57.527 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:02:57.560 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:02:57.565 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:02:57.566 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:02:57.667 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:02:57.707 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:02:57.711 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:02:57.712 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:02:57.758 INFO:teuthology.orchestra.run.smithi089.stderr:reweighted osd.6 to 1 (10000) 2023-06-03T20:02:57.816 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:02:57.836 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:02:57.838 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:02:57.839 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:02:57.939 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:02:57.973 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:02:57.976 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:02:57.977 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:02:58.028 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:02:57 smithi089 bash[78452]: cluster 1685822576.4776075 mgr.y (mgr.34104) 799 : cluster 0 pgmap v716: 89 pgs: 89 active+clean; 4.8 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 658 op/s 2023-06-03T20:02:58.029 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:02:57 smithi089 bash[78452]: audit 1685822577.4211175 mon.a (mon.0) 715 : audit 1 from='client.? 172.21.15.89:0/2473583108' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-06-03T20:02:58.029 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:02:57 smithi089 bash[75068]: cluster 1685822576.4776075 mgr.y (mgr.34104) 799 : cluster 0 pgmap v716: 89 pgs: 89 active+clean; 4.8 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 658 op/s 2023-06-03T20:02:58.029 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:02:57 smithi089 bash[75068]: audit 1685822577.4211175 mon.a (mon.0) 715 : audit 1 from='client.? 172.21.15.89:0/2473583108' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-06-03T20:02:58.079 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:02:58.083 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:02:57 smithi164 bash[59462]: cluster 1685822576.4776075 mgr.y (mgr.34104) 799 : cluster 0 pgmap v716: 89 pgs: 89 active+clean; 4.8 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 658 op/s 2023-06-03T20:02:58.083 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:02:57 smithi164 bash[59462]: audit 1685822577.4211175 mon.a (mon.0) 715 : audit 1 from='client.? 172.21.15.89:0/2473583108' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-06-03T20:02:58.113 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:02:58.114 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:02:58.115 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:02:58.216 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:02:58.248 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:02:58.252 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:02:58.253 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:02:58.354 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:02:58.385 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:02:58.390 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:02:58.391 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:02:58.492 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:02:58.518 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:02:58.522 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:02:58.523 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:02:58.624 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:02:58.660 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:02:58.663 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:02:58.663 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:02:58.764 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:02:58.789 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:02:58.794 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:02:58.795 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:02:58.895 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:02:58.905 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:02:58 smithi089 bash[78452]: cluster 1685822577.7453241 osd.6 (osd.6) 58 : cluster 3 failed to encode map e139 with expected crc 2023-06-03T20:02:58.905 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:02:58 smithi089 bash[78452]: audit 1685822577.7501953 mon.a (mon.0) 716 : audit 1 from='client.? 172.21.15.89:0/2473583108' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2023-06-03T20:02:58.905 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:02:58 smithi089 bash[78452]: cluster 1685822577.7502692 mon.a (mon.0) 717 : cluster 0 osdmap e139: 8 total, 8 up, 8 in 2023-06-03T20:02:58.905 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:02:58 smithi089 bash[78452]: cluster 1685822577.8167589 osd.5 (osd.5) 63 : cluster 3 failed to encode map e139 with expected crc 2023-06-03T20:02:58.906 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:02:58 smithi089 bash[75068]: cluster 1685822577.7453241 osd.6 (osd.6) 58 : cluster 3 failed to encode map e139 with expected crc 2023-06-03T20:02:58.906 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:02:58 smithi089 bash[75068]: audit 1685822577.7501953 mon.a (mon.0) 716 : audit 1 from='client.? 172.21.15.89:0/2473583108' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2023-06-03T20:02:58.906 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:02:58 smithi089 bash[75068]: cluster 1685822577.7502692 mon.a (mon.0) 717 : cluster 0 osdmap e139: 8 total, 8 up, 8 in 2023-06-03T20:02:58.906 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:02:58 smithi089 bash[75068]: cluster 1685822577.8167589 osd.5 (osd.5) 63 : cluster 3 failed to encode map e139 with expected crc 2023-06-03T20:02:58.933 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:02:58.936 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:02:58.937 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:02:59.038 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:02:59.072 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:02:59.079 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:02:59.080 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:02:59.083 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:02:58 smithi164 bash[59462]: cluster 1685822577.7453241 osd.6 (osd.6) 58 : cluster 3 failed to encode map e139 with expected crc 2023-06-03T20:02:59.083 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:02:58 smithi164 bash[59462]: audit 1685822577.7501953 mon.a (mon.0) 716 : audit 1 from='client.? 172.21.15.89:0/2473583108' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2023-06-03T20:02:59.084 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:02:58 smithi164 bash[59462]: cluster 1685822577.7502692 mon.a (mon.0) 717 : cluster 0 osdmap e139: 8 total, 8 up, 8 in 2023-06-03T20:02:59.084 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:02:58 smithi164 bash[59462]: cluster 1685822577.8167589 osd.5 (osd.5) 63 : cluster 3 failed to encode map e139 with expected crc 2023-06-03T20:02:59.180 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:02:59.207 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:02:59.208 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:02:59.209 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:02:59.311 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:02:59.350 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:02:59.354 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:02:59.355 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:02:59.456 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:02:59.490 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:02:59.492 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:02:59.493 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:02:59.594 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:02:59.621 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:02:59.625 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:02:59.626 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:02:59.727 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:02:59.764 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:02:59 smithi089 bash[75068]: cluster 1685822578.478171 mgr.y (mgr.34104) 800 : cluster 0 pgmap v718: 89 pgs: 89 active+clean; 4.6 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 711 op/s 2023-06-03T20:02:59.765 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:02:59.771 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:02:59.771 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:02:59.872 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:02:59.896 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:02:59.898 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:02:59.898 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:02:59.999 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:03:00.014 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:02:59 smithi164 bash[59462]: cluster 1685822578.478171 mgr.y (mgr.34104) 800 : cluster 0 pgmap v718: 89 pgs: 89 active+clean; 4.6 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 711 op/s 2023-06-03T20:03:00.037 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:03:00.040 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:03:00.041 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:03:00.125 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:02:59 smithi089 bash[78452]: cluster 1685822578.478171 mgr.y (mgr.34104) 800 : cluster 0 pgmap v718: 89 pgs: 89 active+clean; 4.6 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 711 op/s 2023-06-03T20:03:00.141 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:03:00.176 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:03:00.178 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:03:00.179 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:03:00.280 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:03:00.313 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:03:00.318 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:03:00.320 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:03:00.421 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:03:00.447 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:03:00.452 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:03:00.453 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:03:00.554 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:03:00.578 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:03:00.580 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:03:00.581 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:03:00.682 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:03:00.712 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:03:00.714 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:03:00.715 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:03:00.816 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:03:00.848 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:03:00.853 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:03:00.854 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:03:00.955 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:03:00.966 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph daemon osd.2 dump_historic_ops 2023-06-03T20:03:01.010 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:03:01.012 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:03:01.012 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:03:01.041 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph osd reweight 7 1 2023-06-03T20:03:01.114 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:03:01.188 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:03:01.191 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:03:01.191 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:03:01.292 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:03:01.304 INFO:teuthology.orchestra.run.smithi089.stdout:osd.0: {} 2023-06-03T20:03:01.323 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:03:01.326 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:03:01.326 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:03:01.344 INFO:teuthology.orchestra.run.smithi089.stdout:osd.1: {} 2023-06-03T20:03:01.390 INFO:teuthology.orchestra.run.smithi089.stdout:osd.2: {} 2023-06-03T20:03:01.427 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:03:01.434 INFO:teuthology.orchestra.run.smithi089.stdout:osd.3: {} 2023-06-03T20:03:01.456 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:03:01.458 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:03:01.459 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:03:01.473 INFO:teuthology.orchestra.run.smithi089.stdout:osd.4: {} 2023-06-03T20:03:01.508 INFO:teuthology.orchestra.run.smithi089.stdout:osd.5: {} 2023-06-03T20:03:01.547 INFO:teuthology.orchestra.run.smithi089.stdout:osd.6: {} 2023-06-03T20:03:01.559 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:03:01.585 INFO:teuthology.orchestra.run.smithi089.stdout:osd.7: {} 2023-06-03T20:03:01.589 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:03:01.593 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:03:01.594 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:03:01.594 INFO:teuthology.orchestra.run.smithi089.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-06-03T20:03:01.595 INFO:teuthology.orchestra.run.smithi089.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-06-03T20:03:01.595 INFO:teuthology.orchestra.run.smithi089.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-06-03T20:03:01.595 INFO:teuthology.orchestra.run.smithi089.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-06-03T20:03:01.595 INFO:teuthology.orchestra.run.smithi089.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-06-03T20:03:01.595 INFO:teuthology.orchestra.run.smithi089.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-06-03T20:03:01.595 INFO:teuthology.orchestra.run.smithi089.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-06-03T20:03:01.596 INFO:teuthology.orchestra.run.smithi089.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-06-03T20:03:01.695 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:03:01.736 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:03:01.738 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:03:01.739 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:03:01.797 INFO:teuthology.orchestra.run.smithi089.stderr:noscrub is unset 2023-06-03T20:03:01.833 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:03:01 smithi164 bash[59462]: cluster 1685822580.4787152 mgr.y (mgr.34104) 801 : cluster 0 pgmap v719: 89 pgs: 89 active+clean; 4.6 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 709 op/s 2023-06-03T20:03:01.834 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:03:01 smithi164 bash[59462]: audit 1685822581.2602682 mon.c (mon.2) 402 : audit 0 from='client.? 172.21.15.89:0/1372645817' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-03T20:03:01.834 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:03:01 smithi164 bash[59462]: audit 1685822581.6939592 mon.a (mon.0) 718 : audit 1 from='client.? 172.21.15.89:0/3552786607' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-06-03T20:03:01.840 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:03:01.882 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:03:01.884 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:03:01.885 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:03:01.986 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:03:01.995 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:03:01 smithi089 bash[75068]: cluster 1685822580.4787152 mgr.y (mgr.34104) 801 : cluster 0 pgmap v719: 89 pgs: 89 active+clean; 4.6 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 709 op/s 2023-06-03T20:03:01.995 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:03:01 smithi089 bash[75068]: audit 1685822581.2602682 mon.c (mon.2) 402 : audit 0 from='client.? 172.21.15.89:0/1372645817' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-03T20:03:01.996 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:03:01 smithi089 bash[75068]: audit 1685822581.6939592 mon.a (mon.0) 718 : audit 1 from='client.? 172.21.15.89:0/3552786607' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-06-03T20:03:01.996 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:03:01 smithi089 bash[78452]: cluster 1685822580.4787152 mgr.y (mgr.34104) 801 : cluster 0 pgmap v719: 89 pgs: 89 active+clean; 4.6 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 709 op/s 2023-06-03T20:03:01.996 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:03:01 smithi089 bash[78452]: audit 1685822581.2602682 mon.c (mon.2) 402 : audit 0 from='client.? 172.21.15.89:0/1372645817' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-03T20:03:01.997 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:03:01 smithi089 bash[78452]: audit 1685822581.6939592 mon.a (mon.0) 718 : audit 1 from='client.? 172.21.15.89:0/3552786607' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-06-03T20:03:02.022 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:03:02.027 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:03:02.028 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:03:02.129 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:03:02.168 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:03:02.177 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:03:02.178 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:03:02.279 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:03:02.324 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:03:02.330 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:03:02.331 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:03:02.432 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:03:02.461 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:03:02.463 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:03:02.464 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:03:02.565 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:03:02.595 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:03:02.598 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:03:02.598 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:03:02.699 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:03:02.744 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:03:02.749 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:03:02.750 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:03:02.771 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:03:02 smithi089 bash[78452]: cluster 1685822581.7616098 mon.a (mon.0) 719 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-06-03T20:03:02.772 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:03:02 smithi089 bash[78452]: cluster 1685822581.771245 osd.6 (osd.6) 59 : cluster 3 failed to encode map e140 with expected crc 2023-06-03T20:03:02.772 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:03:02 smithi089 bash[78452]: audit 1685822581.7759378 mon.a (mon.0) 720 : audit 1 from='client.? 172.21.15.89:0/3552786607' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-06-03T20:03:02.772 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:03:02 smithi089 bash[78452]: cluster 1685822581.776032 mon.a (mon.0) 721 : cluster 0 osdmap e140: 8 total, 8 up, 8 in 2023-06-03T20:03:02.772 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:03:02 smithi089 bash[78452]: cluster 1685822581.7793055 osd.5 (osd.5) 64 : cluster 3 failed to encode map e140 with expected crc 2023-06-03T20:03:02.772 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:03:02 smithi089 bash[78452]: cluster 1685822581.7793381 osd.7 (osd.7) 52 : cluster 3 failed to encode map e140 with expected crc 2023-06-03T20:03:02.772 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:03:02 smithi089 bash[78452]: cluster 1685822581.7857056 osd.5 (osd.5) 65 : cluster 3 failed to encode map e140 with expected crc 2023-06-03T20:03:02.773 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:03:02 smithi089 bash[75068]: cluster 1685822581.7616098 mon.a (mon.0) 719 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-06-03T20:03:02.773 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:03:02 smithi089 bash[75068]: cluster 1685822581.771245 osd.6 (osd.6) 59 : cluster 3 failed to encode map e140 with expected crc 2023-06-03T20:03:02.773 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:03:02 smithi089 bash[75068]: audit 1685822581.7759378 mon.a (mon.0) 720 : audit 1 from='client.? 172.21.15.89:0/3552786607' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-06-03T20:03:02.773 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:03:02 smithi089 bash[75068]: cluster 1685822581.776032 mon.a (mon.0) 721 : cluster 0 osdmap e140: 8 total, 8 up, 8 in 2023-06-03T20:03:02.773 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:03:02 smithi089 bash[75068]: cluster 1685822581.7793055 osd.5 (osd.5) 64 : cluster 3 failed to encode map e140 with expected crc 2023-06-03T20:03:02.773 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:03:02 smithi089 bash[75068]: cluster 1685822581.7793381 osd.7 (osd.7) 52 : cluster 3 failed to encode map e140 with expected crc 2023-06-03T20:03:02.773 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:03:02 smithi089 bash[75068]: cluster 1685822581.7857056 osd.5 (osd.5) 65 : cluster 3 failed to encode map e140 with expected crc 2023-06-03T20:03:02.851 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:03:02.893 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:03:02.899 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:03:02.899 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:03:03.000 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:03:03.010 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:03:02 smithi164 bash[59462]: cluster 1685822581.7616098 mon.a (mon.0) 719 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-06-03T20:03:03.010 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:03:02 smithi164 bash[59462]: cluster 1685822581.771245 osd.6 (osd.6) 59 : cluster 3 failed to encode map e140 with expected crc 2023-06-03T20:03:03.011 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:03:02 smithi164 bash[59462]: audit 1685822581.7759378 mon.a (mon.0) 720 : audit 1 from='client.? 172.21.15.89:0/3552786607' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-06-03T20:03:03.011 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:03:02 smithi164 bash[59462]: cluster 1685822581.776032 mon.a (mon.0) 721 : cluster 0 osdmap e140: 8 total, 8 up, 8 in 2023-06-03T20:03:03.011 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:03:02 smithi164 bash[59462]: cluster 1685822581.7793055 osd.5 (osd.5) 64 : cluster 3 failed to encode map e140 with expected crc 2023-06-03T20:03:03.011 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:03:02 smithi164 bash[59462]: cluster 1685822581.7793381 osd.7 (osd.7) 52 : cluster 3 failed to encode map e140 with expected crc 2023-06-03T20:03:03.011 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:03:02 smithi164 bash[59462]: cluster 1685822581.7857056 osd.5 (osd.5) 65 : cluster 3 failed to encode map e140 with expected crc 2023-06-03T20:03:03.035 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:03:03.039 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:03:03.040 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:03:03.141 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:03:03.183 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:03:03.187 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:03:03.187 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:03:03.288 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:03:03.324 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:03:03.326 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:03:03.327 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:03:03.428 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:03:03.460 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:03:03.464 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:03:03.464 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:03:03.565 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:03:03.600 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:03:03.606 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:03:03.606 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:03:03.707 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:03:03.731 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:03:03.732 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:03:03.733 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:03:03.833 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:03:03 smithi164 bash[59462]: cluster 1685822582.4791613 mgr.y (mgr.34104) 802 : cluster 0 pgmap v721: 89 pgs: 89 active+clean; 4.6 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 517 op/s 2023-06-03T20:03:03.834 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:03:03.862 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:03:03.865 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:03:03.866 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:03:03.966 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:03:04.005 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:03:04.010 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:03:04.011 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:03:04.028 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:03:03 smithi089 bash[75068]: cluster 1685822582.4791613 mgr.y (mgr.34104) 802 : cluster 0 pgmap v721: 89 pgs: 89 active+clean; 4.6 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 517 op/s 2023-06-03T20:03:04.029 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:03:03 smithi089 bash[78452]: cluster 1685822582.4791613 mgr.y (mgr.34104) 802 : cluster 0 pgmap v721: 89 pgs: 89 active+clean; 4.6 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 517 op/s 2023-06-03T20:03:04.112 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:03:04.137 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:03:04.140 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:03:04.141 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:03:04.242 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:03:04.270 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:03:04.283 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:03:04.284 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:03:04.385 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:03:04.417 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:03:04.423 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:03:04.425 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:03:04.526 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:03:04.557 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:03:04.559 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:03:04.559 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:03:04.660 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:03:04.691 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:03:04.695 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:03:04.695 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:03:04.796 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:03:04.844 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:03:04.850 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:03:04.851 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:03:04.951 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:03:04.981 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:03:04.984 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:03:04.984 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:03:05.085 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:03:05.123 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:03:05.125 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:03:05.126 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:03:05.227 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:03:05.268 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:03:05.271 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:03:05.272 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:03:05.372 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:03:05.413 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:03:05.417 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:03:05.418 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:03:05.519 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:03:05.562 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:03:05.565 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:03:05.566 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:03:05.667 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:03:05.704 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:03:05.707 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:03:05.708 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:03:05.787 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:03:05 smithi164 bash[59462]: cluster 1685822584.4797637 mgr.y (mgr.34104) 803 : cluster 0 pgmap v722: 89 pgs: 89 active+clean; 4.4 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 782 op/s 2023-06-03T20:03:05.809 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:03:05.857 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:03:05.859 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:03:05.860 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:03:05.961 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:03:05.970 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:03:05 smithi089 bash[75068]: cluster 1685822584.4797637 mgr.y (mgr.34104) 803 : cluster 0 pgmap v722: 89 pgs: 89 active+clean; 4.4 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 782 op/s 2023-06-03T20:03:05.970 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:03:05 smithi089 bash[78452]: cluster 1685822584.4797637 mgr.y (mgr.34104) 803 : cluster 0 pgmap v722: 89 pgs: 89 active+clean; 4.4 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 782 op/s 2023-06-03T20:03:05.993 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:03:05.996 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:03:05.996 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:03:06.097 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:03:06.124 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-06-03T20:03:06.134 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:03:06.138 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:03:06.139 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:03:06.232 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 20:03:06 smithi089 bash[71163]: ::ffff:172.21.15.164 - - [03/Jun/2023:20:03:06] "GET /metrics HTTP/1.1" 200 228290 "" "Prometheus/2.33.4" 2023-06-03T20:03:06.240 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:03:06.299 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph osd unset nodeep-scrub 2023-06-03T20:03:06.300 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:03:06.306 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:03:06.307 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:03:06.408 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:03:06.439 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:03:06.443 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:03:06.444 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:03:06.545 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:03:06.583 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:03:06.589 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:03:06.589 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:03:06.690 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:03:06.732 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:03:06.738 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:03:06.739 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:03:06.840 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:03:06.877 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:03:06.883 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:03:06.883 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:03:06.984 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:03:07.025 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:03:07.031 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:03:07.031 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:03:07.132 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:03:07.167 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:03:07.174 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:03:07.175 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:03:07.276 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:03:07.311 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:03:07.314 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:03:07.315 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:03:07.416 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:03:07.449 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:03:07.453 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:03:07.454 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:03:07.555 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:03:07.590 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:03:07.592 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:03:07.593 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:03:07.694 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:03:07.726 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:03:07.729 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:03:07.730 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:03:07.831 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:03:07.840 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:03:07 smithi089 bash[75068]: cluster 1685822586.4803438 mgr.y (mgr.34104) 804 : cluster 0 pgmap v723: 89 pgs: 89 active+clean; 4.3 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 789 op/s 2023-06-03T20:03:07.840 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:03:07 smithi089 bash[75068]: audit 1685822587.1028147 mon.c (mon.2) 403 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T20:03:07.841 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:03:07 smithi089 bash[78452]: cluster 1685822586.4803438 mgr.y (mgr.34104) 804 : cluster 0 pgmap v723: 89 pgs: 89 active+clean; 4.3 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 789 op/s 2023-06-03T20:03:07.841 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:03:07 smithi089 bash[78452]: audit 1685822587.1028147 mon.c (mon.2) 403 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T20:03:07.872 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:03:07.880 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:03:07.881 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/mon.a/config 2023-06-03T20:03:07.881 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:03:07.884 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/mon.a/config 2023-06-03T20:03:07.982 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:03:08.032 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:03:08.034 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:03:08.035 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:03:08.083 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:03:07 smithi164 bash[59462]: cluster 1685822586.4803438 mgr.y (mgr.34104) 804 : cluster 0 pgmap v723: 89 pgs: 89 active+clean; 4.3 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 789 op/s 2023-06-03T20:03:08.084 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:03:07 smithi164 bash[59462]: audit 1685822587.1028147 mon.c (mon.2) 403 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T20:03:08.135 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:03:08.174 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:03:08.180 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:03:08.181 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:03:08.282 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:03:08.330 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:03:08.336 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:03:08.337 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:03:08.437 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:03:08.468 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:03:08.473 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:03:08.473 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:03:08.574 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:03:08.613 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:03:08.618 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:03:08.619 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:03:08.719 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:03:08.752 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:03:08.754 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:03:08.754 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:03:08.855 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:03:08.894 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:03:08.898 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:03:08.899 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:03:09.000 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:03:09.035 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:03:09.038 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:03:09.039 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:03:09.140 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:03:09.177 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:03:09.182 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:03:09.182 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:03:09.283 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:03:09.317 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:03:09.321 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:03:09.321 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:03:09.423 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:03:09.455 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:03:09.464 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:03:09.465 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:03:09.566 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:03:09.602 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:03:09.605 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:03:09.605 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:03:09.706 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:03:09.745 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:03:09.747 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:03:09.748 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:03:09.849 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:03:09.909 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:03:09.911 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:03:09.912 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:03:10.013 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:03:10.022 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:03:09 smithi164 bash[59462]: cluster 1685822588.4810705 mgr.y (mgr.34104) 805 : cluster 0 pgmap v724: 89 pgs: 89 active+clean; 4.2 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 711 op/s 2023-06-03T20:03:10.058 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:03:10.063 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:03:10.064 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:03:10.165 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:03:10.200 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:03:10.205 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:03:10.206 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:03:10.278 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:03:09 smithi089 bash[75068]: cluster 1685822588.4810705 mgr.y (mgr.34104) 805 : cluster 0 pgmap v724: 89 pgs: 89 active+clean; 4.2 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 711 op/s 2023-06-03T20:03:10.279 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:03:09 smithi089 bash[78452]: cluster 1685822588.4810705 mgr.y (mgr.34104) 805 : cluster 0 pgmap v724: 89 pgs: 89 active+clean; 4.2 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 711 op/s 2023-06-03T20:03:10.307 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:03:10.348 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:03:10.354 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:03:10.354 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:03:10.455 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:03:10.486 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:03:10.489 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:03:10.490 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:03:10.591 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:03:10.626 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:03:10.631 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:03:10.632 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:03:10.733 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:03:10.767 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:03:10.769 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:03:10.769 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:03:10.870 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:03:10.903 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:03:10.911 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:03:10.912 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:03:10.971 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/mon.a/config 2023-06-03T20:03:10.971 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/mon.a/config 2023-06-03T20:03:11.013 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:03:11.059 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:03:11.064 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:03:11.065 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:03:11.166 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:03:11.203 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:03:11.208 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:03:11.209 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:03:11.310 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:03:11.351 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:03:11.411 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:03:11.412 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:03:11.513 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:03:11.548 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:03:11.550 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:03:11.551 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:03:11.652 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:03:11.688 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:03:11.689 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:03:11.689 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:03:11.790 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:03:11.841 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:03:11.843 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:03:11.844 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:03:11.945 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:03:11.987 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:03:11.990 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:03:11.990 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:03:12.029 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:03:11 smithi089 bash[75068]: cluster 1685822590.481677 mgr.y (mgr.34104) 806 : cluster 0 pgmap v725: 89 pgs: 89 active+clean; 4.1 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 715 op/s 2023-06-03T20:03:12.029 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:03:11 smithi089 bash[78452]: cluster 1685822590.481677 mgr.y (mgr.34104) 806 : cluster 0 pgmap v725: 89 pgs: 89 active+clean; 4.1 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 715 op/s 2023-06-03T20:03:12.083 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:03:11 smithi164 bash[59462]: cluster 1685822590.481677 mgr.y (mgr.34104) 806 : cluster 0 pgmap v725: 89 pgs: 89 active+clean; 4.1 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 715 op/s 2023-06-03T20:03:12.092 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:03:12.143 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:03:12.145 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:03:12.146 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:03:12.247 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:03:12.289 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:03:12.293 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:03:12.293 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:03:12.394 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:03:12.429 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:03:12.433 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:03:12.434 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:03:12.535 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:03:12.565 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:03:12.578 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:03:12.579 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:03:12.679 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:03:12.703 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:03:12.708 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:03:12.709 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:03:12.810 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:03:12.879 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:03:12.884 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:03:12.885 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:03:12.986 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:03:13.031 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:03:13.035 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:03:13.036 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:03:13.137 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:03:13.166 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:03:13.168 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:03:13.169 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:03:13.270 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:03:13.309 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:03:13.313 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:03:13.314 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:03:13.415 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:03:13.457 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:03:13.460 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:03:13.461 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:03:13.562 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:03:13.596 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:03:13.600 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:03:13.601 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:03:13.702 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:03:13.737 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:03:13.742 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:03:13.743 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:03:13.844 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:03:13.896 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:03:13.902 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:03:13.903 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:03:14.004 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:03:14.012 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:03:13 smithi164 bash[59462]: cluster 1685822592.482214 mgr.y (mgr.34104) 807 : cluster 0 pgmap v726: 89 pgs: 89 active+clean; 4.1 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 668 op/s 2023-06-03T20:03:14.042 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:03:14.046 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:03:14.047 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:03:14.097 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:03:13 smithi089 bash[75068]: cluster 1685822592.482214 mgr.y (mgr.34104) 807 : cluster 0 pgmap v726: 89 pgs: 89 active+clean; 4.1 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 668 op/s 2023-06-03T20:03:14.098 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:03:13 smithi089 bash[78452]: cluster 1685822592.482214 mgr.y (mgr.34104) 807 : cluster 0 pgmap v726: 89 pgs: 89 active+clean; 4.1 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 668 op/s 2023-06-03T20:03:14.147 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:03:14.180 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:03:14.184 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:03:14.185 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:03:14.286 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:03:14.319 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:03:14.326 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:03:14.328 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:03:14.429 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:03:14.468 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:03:14.473 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:03:14.474 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:03:14.575 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:03:14.620 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:03:14.626 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:03:14.626 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:03:14.727 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:03:14.762 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:03:14.763 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:03:14.763 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:03:14.864 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:03:14.900 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:03:14.902 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:03:14.903 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:03:15.003 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:03:15.042 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:03:15.045 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:03:15.045 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:03:15.146 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:03:15.177 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:03:15.183 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:03:15.185 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:03:15.286 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:03:15.324 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:03:15.327 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:03:15.327 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:03:15.428 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:03:15.481 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:03:15.487 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:03:15.488 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:03:15.589 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:03:15.627 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:03:15.633 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:03:15.634 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:03:15.734 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:03:15.772 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:03:15.776 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:03:15.777 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:03:15.878 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:03:15.907 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:03:15.911 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:03:15.912 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:03:16.013 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:03:16.051 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:03:16.053 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:03:16.054 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:03:16.083 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:03:15 smithi164 bash[59462]: cluster 1685822594.4828553 mgr.y (mgr.34104) 808 : cluster 0 pgmap v727: 89 pgs: 89 active+clean; 3.9 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 850 op/s 2023-06-03T20:03:16.155 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:03:16.192 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:03:16.194 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:03:16.195 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:03:16.231 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:03:15 smithi089 bash[78452]: cluster 1685822594.4828553 mgr.y (mgr.34104) 808 : cluster 0 pgmap v727: 89 pgs: 89 active+clean; 3.9 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 850 op/s 2023-06-03T20:03:16.231 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:03:15 smithi089 bash[75068]: cluster 1685822594.4828553 mgr.y (mgr.34104) 808 : cluster 0 pgmap v727: 89 pgs: 89 active+clean; 3.9 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 850 op/s 2023-06-03T20:03:16.296 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:03:16.325 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:03:16.329 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:03:16.330 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:03:16.430 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:03:16.458 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:03:16.462 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:03:16.463 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:03:16.528 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 20:03:16 smithi089 bash[71163]: ::ffff:172.21.15.164 - - [03/Jun/2023:20:03:16] "GET /metrics HTTP/1.1" 200 228305 "" "Prometheus/2.33.4" 2023-06-03T20:03:16.564 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:03:16.600 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:03:16.607 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:03:16.608 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:03:16.708 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:03:16.741 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:03:16.745 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:03:16.746 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:03:16.847 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:03:16.882 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:03:16.887 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:03:16.887 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:03:16.988 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:03:17.029 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:03:17.032 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:03:17.033 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:03:17.134 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:03:17.169 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:03:17.170 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:03:17.171 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:03:17.272 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:03:17.298 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:03:17.300 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:03:17.301 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:03:17.402 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:03:17.440 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:03:17.447 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:03:17.447 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:03:17.548 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:03:17.583 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:03:17.590 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:03:17.590 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:03:17.691 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:03:17.713 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:03:17.717 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:03:17.717 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:03:17.810 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:03:17 smithi164 bash[59462]: cluster 1685822596.4833512 mgr.y (mgr.34104) 809 : cluster 0 pgmap v728: 89 pgs: 89 active+clean; 3.9 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 674 op/s 2023-06-03T20:03:17.818 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:03:17.852 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:03:17.854 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:03:17.855 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:03:17.956 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:03:17.992 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:03:17.998 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:03:17.999 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:03:18.028 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:03:17 smithi089 bash[78452]: cluster 1685822596.4833512 mgr.y (mgr.34104) 809 : cluster 0 pgmap v728: 89 pgs: 89 active+clean; 3.9 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 674 op/s 2023-06-03T20:03:18.029 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:03:17 smithi089 bash[75068]: cluster 1685822596.4833512 mgr.y (mgr.34104) 809 : cluster 0 pgmap v728: 89 pgs: 89 active+clean; 3.9 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 674 op/s 2023-06-03T20:03:18.101 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:03:18.126 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:03:18.132 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:03:18.133 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:03:18.233 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:03:18.262 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:03:18.263 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:03:18.264 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:03:18.365 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:03:18.400 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:03:18.406 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:03:18.407 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:03:18.507 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:03:18.553 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:03:18.556 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:03:18.557 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:03:18.658 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:03:18.685 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:03:18.686 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:03:18.687 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:03:18.788 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:03:18.831 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:03:18.833 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:03:18.834 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:03:18.935 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:03:18.963 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:03:18.966 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:03:18.967 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:03:19.068 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:03:19.118 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:03:19.120 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:03:19.122 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:03:19.223 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:03:19.254 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:03:19.258 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:03:19.258 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:03:19.359 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:03:19.394 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:03:19.398 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:03:19.399 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:03:19.500 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:03:19.541 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:03:19.545 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:03:19.545 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:03:19.646 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:03:19.676 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:03:19.684 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:03:19.686 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:03:19.787 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:03:19.819 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:03:19.824 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:03:19.825 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:03:19.926 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:03:19.961 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:03:19.967 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:03:19.967 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:03:20.068 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:03:20.076 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:03:19 smithi089 bash[75068]: cluster 1685822598.484001 mgr.y (mgr.34104) 810 : cluster 0 pgmap v729: 89 pgs: 89 active+clean; 3.7 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 866 op/s 2023-06-03T20:03:20.077 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:03:19 smithi089 bash[78452]: cluster 1685822598.484001 mgr.y (mgr.34104) 810 : cluster 0 pgmap v729: 89 pgs: 89 active+clean; 3.7 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 866 op/s 2023-06-03T20:03:20.083 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:03:19 smithi164 bash[59462]: cluster 1685822598.484001 mgr.y (mgr.34104) 810 : cluster 0 pgmap v729: 89 pgs: 89 active+clean; 3.7 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 866 op/s 2023-06-03T20:03:20.107 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:03:20.110 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:03:20.111 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:03:20.212 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:03:20.248 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:03:20.253 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:03:20.254 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:03:20.354 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:03:20.401 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:03:20.404 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:03:20.404 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:03:20.505 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:03:20.549 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:03:20.550 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:03:20.552 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:03:20.653 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:03:20.688 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:03:20.691 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:03:20.691 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:03:20.792 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:03:20.849 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:03:20.854 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:03:20.854 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:03:20.955 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:03:20.991 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:03:20.995 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:03:20.996 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:03:21.097 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:03:21.127 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:03:21.132 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:03:21.133 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:03:21.234 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:03:21.272 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:03:21.273 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:03:21.273 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:03:21.374 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:03:21.418 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:03:21.423 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:03:21.424 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:03:21.525 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:03:21.574 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:03:21.576 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:03:21.577 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:03:21.679 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:03:21.717 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:03:21.722 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:03:21.723 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:03:21.824 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:03:21.832 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:03:21 smithi164 bash[59462]: cluster 1685822600.4845428 mgr.y (mgr.34104) 811 : cluster 0 pgmap v730: 89 pgs: 89 active+clean; 3.7 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 672 op/s 2023-06-03T20:03:21.856 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:03:21.859 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:03:21.859 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:03:21.960 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:03:21.990 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:03:21.994 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:03:21.995 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:03:22.096 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:03:22.134 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:03:22.139 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:03:22.140 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:03:22.241 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:03:22.278 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:03:21 smithi089 bash[78452]: cluster 1685822600.4845428 mgr.y (mgr.34104) 811 : cluster 0 pgmap v730: 89 pgs: 89 active+clean; 3.7 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 672 op/s 2023-06-03T20:03:22.279 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:03:21 smithi089 bash[75068]: cluster 1685822600.4845428 mgr.y (mgr.34104) 811 : cluster 0 pgmap v730: 89 pgs: 89 active+clean; 3.7 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 672 op/s 2023-06-03T20:03:22.279 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:03:22.286 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:03:22.286 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:03:22.387 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:03:22.417 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:03:22.422 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:03:22.423 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:03:22.524 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:03:22.559 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:03:22.566 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:03:22.566 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:03:22.667 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:03:22.710 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:03:22.714 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:03:22.714 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:03:22.815 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:03:22.858 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:03:22.867 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:03:22.867 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:03:22.968 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:03:23.007 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:03:23.010 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:03:23.010 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:03:23.085 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:03:22 smithi164 bash[59462]: audit 1685822602.1023414 mon.c (mon.2) 404 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T20:03:23.111 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:03:23.119 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:03:22 smithi089 bash[78452]: audit 1685822602.1023414 mon.c (mon.2) 404 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T20:03:23.120 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:03:22 smithi089 bash[75068]: audit 1685822602.1023414 mon.c (mon.2) 404 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T20:03:23.151 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:03:23.156 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:03:23.157 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:03:23.258 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:03:23.286 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:03:23.290 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:03:23.290 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:03:23.391 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:03:23.423 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:03:23.431 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:03:23.433 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:03:23.533 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:03:23.571 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:03:23.572 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:03:23.572 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:03:23.673 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:03:23.707 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:03:23.709 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:03:23.710 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:03:23.811 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:03:23.846 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:03:23.849 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:03:23.850 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:03:23.856 INFO:teuthology.orchestra.run.smithi089.stderr:reweighted osd.7 to 1 (10000) 2023-06-03T20:03:23.936 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:03:23 smithi089 bash[75068]: cluster 1685822602.4850366 mgr.y (mgr.34104) 812 : cluster 0 pgmap v731: 89 pgs: 89 active+clean; 3.7 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 617 op/s 2023-06-03T20:03:23.937 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:03:23 smithi089 bash[75068]: audit 1685822603.4811306 mon.a (mon.0) 722 : audit 1 from='client.? 172.21.15.89:0/3515449398' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-06-03T20:03:23.937 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:03:23 smithi089 bash[78452]: cluster 1685822602.4850366 mgr.y (mgr.34104) 812 : cluster 0 pgmap v731: 89 pgs: 89 active+clean; 3.7 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 617 op/s 2023-06-03T20:03:23.937 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:03:23 smithi089 bash[78452]: audit 1685822603.4811306 mon.a (mon.0) 722 : audit 1 from='client.? 172.21.15.89:0/3515449398' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-06-03T20:03:23.952 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:03:23.995 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:03:24.001 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:03:24.003 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:03:24.083 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:03:23 smithi164 bash[59462]: cluster 1685822602.4850366 mgr.y (mgr.34104) 812 : cluster 0 pgmap v731: 89 pgs: 89 active+clean; 3.7 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 617 op/s 2023-06-03T20:03:24.084 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:03:23 smithi164 bash[59462]: audit 1685822603.4811306 mon.a (mon.0) 722 : audit 1 from='client.? 172.21.15.89:0/3515449398' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-06-03T20:03:24.104 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:03:24.132 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:03:24.136 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:03:24.137 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:03:24.237 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:03:24.260 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:03:24.264 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:03:24.265 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:03:24.366 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:03:24.408 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:03:24.416 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:03:24.417 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:03:24.518 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:03:24.554 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:03:24.558 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:03:24.559 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:03:24.660 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:03:24.699 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:03:24.706 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:03:24.707 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:03:24.808 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:03:24.840 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:03:24.847 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:03:24.847 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:03:24.948 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:03:24.957 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:03:24 smithi089 bash[75068]: cluster 1685822603.8416414 osd.7 (osd.7) 53 : cluster 3 failed to encode map e141 with expected crc 2023-06-03T20:03:24.957 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:03:24 smithi089 bash[75068]: audit 1685822603.8465402 mon.a (mon.0) 723 : audit 1 from='client.? 172.21.15.89:0/3515449398' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2023-06-03T20:03:24.957 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:03:24 smithi089 bash[75068]: cluster 1685822603.8466156 mon.a (mon.0) 724 : cluster 0 osdmap e141: 8 total, 8 up, 8 in 2023-06-03T20:03:24.957 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:03:24 smithi089 bash[75068]: cluster 1685822603.8660657 osd.6 (osd.6) 60 : cluster 3 failed to encode map e141 with expected crc 2023-06-03T20:03:24.958 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:03:24 smithi089 bash[78452]: cluster 1685822603.8416414 osd.7 (osd.7) 53 : cluster 3 failed to encode map e141 with expected crc 2023-06-03T20:03:24.958 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:03:24 smithi089 bash[78452]: audit 1685822603.8465402 mon.a (mon.0) 723 : audit 1 from='client.? 172.21.15.89:0/3515449398' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2023-06-03T20:03:24.958 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:03:24 smithi089 bash[78452]: cluster 1685822603.8466156 mon.a (mon.0) 724 : cluster 0 osdmap e141: 8 total, 8 up, 8 in 2023-06-03T20:03:24.958 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:03:24 smithi089 bash[78452]: cluster 1685822603.8660657 osd.6 (osd.6) 60 : cluster 3 failed to encode map e141 with expected crc 2023-06-03T20:03:24.981 INFO:teuthology.orchestra.run.smithi089.stdout:ERROR: (22) Invalid argument 2023-06-03T20:03:24.981 INFO:teuthology.orchestra.run.smithi089.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-06-03T20:03:24.983 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:03:24.986 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:03:24.986 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:03:25.083 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:03:24 smithi164 bash[59462]: cluster 1685822603.8416414 osd.7 (osd.7) 53 : cluster 3 failed to encode map e141 with expected crc 2023-06-03T20:03:25.083 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:03:24 smithi164 bash[59462]: audit 1685822603.8465402 mon.a (mon.0) 723 : audit 1 from='client.? 172.21.15.89:0/3515449398' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2023-06-03T20:03:25.084 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:03:24 smithi164 bash[59462]: cluster 1685822603.8466156 mon.a (mon.0) 724 : cluster 0 osdmap e141: 8 total, 8 up, 8 in 2023-06-03T20:03:25.084 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:03:24 smithi164 bash[59462]: cluster 1685822603.8660657 osd.6 (osd.6) 60 : cluster 3 failed to encode map e141 with expected crc 2023-06-03T20:03:25.087 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:03:25.121 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:03:25.125 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:03:25.126 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:03:25.227 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:03:25.273 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:03:25.277 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:03:25.278 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:03:25.378 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:03:25.418 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:03:25.420 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:03:25.420 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:03:25.521 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:03:25.556 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:03:25.559 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:03:25.561 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:03:25.661 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:03:25.702 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:03:25.706 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:03:25.707 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:03:25.808 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:03:25.848 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:03:25.850 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:03:25.851 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:03:25.953 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:03:25.979 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:03:25.986 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:03:25.987 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:03:26.088 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:03:26.096 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:03:25 smithi089 bash[78452]: cluster 1685822604.4854298 mgr.y (mgr.34104) 813 : cluster 0 pgmap v733: 89 pgs: 89 active+clean; 3.5 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 717 op/s 2023-06-03T20:03:26.096 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:03:25 smithi089 bash[75068]: cluster 1685822604.4854298 mgr.y (mgr.34104) 813 : cluster 0 pgmap v733: 89 pgs: 89 active+clean; 3.5 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 717 op/s 2023-06-03T20:03:26.122 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:03:26.128 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:03:26.129 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:03:26.229 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:03:26.239 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:03:25 smithi164 bash[59462]: cluster 1685822604.4854298 mgr.y (mgr.34104) 813 : cluster 0 pgmap v733: 89 pgs: 89 active+clean; 3.5 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 717 op/s 2023-06-03T20:03:26.283 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:03:26.284 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:03:26.285 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:03:26.386 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:03:26.427 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 20:03:26 smithi089 bash[71163]: ::ffff:172.21.15.164 - - [03/Jun/2023:20:03:26] "GET /metrics HTTP/1.1" 200 228297 "" "Prometheus/2.33.4" 2023-06-03T20:03:26.432 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:03:26.435 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:03:26.435 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:03:26.536 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:03:26.577 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:03:26.583 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:03:26.584 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:03:26.685 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:03:26.720 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:03:26.723 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:03:26.724 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:03:26.825 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:03:26.857 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:03:26.861 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:03:26.862 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:03:26.963 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:03:26.997 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:03:27.000 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:03:27.002 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:03:27.103 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:03:27.142 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:03:27.146 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:03:27.147 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:03:27.147 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:03:26 smithi164 bash[59462]: audit 1685822606.1344104 mon.c (mon.2) 405 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-03T20:03:27.248 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:03:27.279 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:03:26 smithi089 bash[78452]: audit 1685822606.1344104 mon.c (mon.2) 405 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-03T20:03:27.279 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:03:26 smithi089 bash[75068]: audit 1685822606.1344104 mon.c (mon.2) 405 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-03T20:03:27.284 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:03:27.287 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:03:27.287 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:03:27.388 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:03:27.424 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:03:27.428 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:03:27.429 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:03:27.530 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:03:27.567 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:03:27.568 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:03:27.568 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:03:27.642 INFO:tasks.cephadm.ceph_manager.ceph:waiting for recovery to complete 2023-06-03T20:03:27.642 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph pg dump --format=json 2023-06-03T20:03:27.670 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:03:27.689 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph daemon osd.3 dump_ops_in_flight 2023-06-03T20:03:27.718 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:03:27.720 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:03:27.721 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:03:27.822 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:03:27.856 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:03:27.858 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:03:27.859 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:03:27.960 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:03:27.968 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:03:27 smithi164 bash[59462]: cluster 1685822606.487443 mgr.y (mgr.34104) 814 : cluster 0 pgmap v734: 89 pgs: 89 active+clean; 3.5 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 712 op/s 2023-06-03T20:03:27.975 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:03:27 smithi089 bash[75068]: cluster 1685822606.487443 mgr.y (mgr.34104) 814 : cluster 0 pgmap v734: 89 pgs: 89 active+clean; 3.5 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 712 op/s 2023-06-03T20:03:27.980 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:03:27 smithi089 bash[78452]: cluster 1685822606.487443 mgr.y (mgr.34104) 814 : cluster 0 pgmap v734: 89 pgs: 89 active+clean; 3.5 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 712 op/s 2023-06-03T20:03:27.995 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:03:27.998 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:03:27.998 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:03:28.099 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:03:28.135 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:03:28.140 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:03:28.141 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:03:28.242 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:03:28.286 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:03:28.290 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:03:28.291 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:03:28.355 INFO:teuthology.orchestra.run.smithi089.stdout:osd.0: {} 2023-06-03T20:03:28.392 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:03:28.404 INFO:teuthology.orchestra.run.smithi089.stdout:osd.1: {} 2023-06-03T20:03:28.437 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:03:28.443 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:03:28.445 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:03:28.448 INFO:teuthology.orchestra.run.smithi089.stdout:osd.2: {} 2023-06-03T20:03:28.490 INFO:teuthology.orchestra.run.smithi089.stdout:osd.3: {} 2023-06-03T20:03:28.532 INFO:teuthology.orchestra.run.smithi089.stdout:osd.4: {} 2023-06-03T20:03:28.546 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:03:28.573 INFO:teuthology.orchestra.run.smithi089.stdout:osd.5: {} 2023-06-03T20:03:28.606 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:03:28.608 INFO:teuthology.orchestra.run.smithi089.stdout:osd.6: {} 2023-06-03T20:03:28.609 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:03:28.610 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:03:28.644 INFO:teuthology.orchestra.run.smithi089.stdout:osd.7: {} 2023-06-03T20:03:28.650 INFO:teuthology.orchestra.run.smithi089.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-06-03T20:03:28.651 INFO:teuthology.orchestra.run.smithi089.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-06-03T20:03:28.651 INFO:teuthology.orchestra.run.smithi089.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-06-03T20:03:28.651 INFO:teuthology.orchestra.run.smithi089.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-06-03T20:03:28.651 INFO:teuthology.orchestra.run.smithi089.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-06-03T20:03:28.651 INFO:teuthology.orchestra.run.smithi089.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-06-03T20:03:28.651 INFO:teuthology.orchestra.run.smithi089.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-06-03T20:03:28.652 INFO:teuthology.orchestra.run.smithi089.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-06-03T20:03:28.711 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:03:28.743 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:03:28.746 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:03:28.747 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:03:28.847 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:03:28.903 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:03:28.909 INFO:teuthology.orchestra.run.smithi089.stderr:nodeep-scrub is unset 2023-06-03T20:03:28.909 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:03:28.910 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:03:29.011 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:03:29.042 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:03:29.044 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:03:29.045 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:03:29.146 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:03:29.154 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:03:28 smithi089 bash[75068]: audit 1685822608.3148816 mon.a (mon.0) 725 : audit 0 from='client.? 172.21.15.89:0/2086449163' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-03T20:03:29.154 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:03:28 smithi089 bash[75068]: audit 1685822608.7179637 mon.b (mon.1) 49 : audit 1 from='client.? 172.21.15.89:0/3030548479' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-06-03T20:03:29.155 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:03:28 smithi089 bash[75068]: audit 1685822608.720659 mon.a (mon.0) 726 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-06-03T20:03:29.155 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:03:28 smithi089 bash[78452]: audit 1685822608.3148816 mon.a (mon.0) 725 : audit 0 from='client.? 172.21.15.89:0/2086449163' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-03T20:03:29.155 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:03:28 smithi089 bash[78452]: audit 1685822608.7179637 mon.b (mon.1) 49 : audit 1 from='client.? 172.21.15.89:0/3030548479' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-06-03T20:03:29.155 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:03:28 smithi089 bash[78452]: audit 1685822608.720659 mon.a (mon.0) 726 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-06-03T20:03:29.175 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:03:29.177 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:03:29.179 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:03:29.280 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:03:29.315 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:03:29.319 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:03:29.320 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:03:29.333 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:03:28 smithi164 bash[59462]: audit 1685822608.3148816 mon.a (mon.0) 725 : audit 0 from='client.? 172.21.15.89:0/2086449163' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-03T20:03:29.334 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:03:28 smithi164 bash[59462]: audit 1685822608.7179637 mon.b (mon.1) 49 : audit 1 from='client.? 172.21.15.89:0/3030548479' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-06-03T20:03:29.334 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:03:28 smithi164 bash[59462]: audit 1685822608.720659 mon.a (mon.0) 726 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-06-03T20:03:29.421 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:03:29.455 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:03:29.460 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:03:29.460 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:03:29.561 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:03:29.591 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:03:29.597 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:03:29.598 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:03:29.699 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:03:29.728 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:03:29.736 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:03:29.737 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:03:29.838 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:03:29.873 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:03:29.875 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:03:29.875 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:03:29.976 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:03:29.985 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:03:29 smithi089 bash[75068]: cluster 1685822608.4881053 mgr.y (mgr.34104) 815 : cluster 0 pgmap v735: 89 pgs: 89 active+clean; 3.3 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 656 op/s 2023-06-03T20:03:29.986 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:03:29 smithi089 bash[75068]: cluster 1685822608.8768327 mon.a (mon.0) 727 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-06-03T20:03:29.986 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:03:29 smithi089 bash[75068]: cluster 1685822608.876873 mon.a (mon.0) 728 : cluster 1 Cluster is now healthy 2023-06-03T20:03:29.986 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:03:29 smithi089 bash[75068]: cluster 1685822608.8926723 osd.5 (osd.5) 66 : cluster 3 failed to encode map e142 with expected crc 2023-06-03T20:03:29.986 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:03:29 smithi089 bash[75068]: cluster 1685822608.8947635 osd.7 (osd.7) 54 : cluster 3 failed to encode map e142 with expected crc 2023-06-03T20:03:29.987 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:03:29 smithi089 bash[75068]: audit 1685822608.896343 mon.a (mon.0) 729 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-06-03T20:03:29.987 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:03:29 smithi089 bash[75068]: cluster 1685822608.8964283 mon.a (mon.0) 730 : cluster 0 osdmap e142: 8 total, 8 up, 8 in 2023-06-03T20:03:29.987 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:03:29 smithi089 bash[75068]: cluster 1685822608.8983057 osd.6 (osd.6) 61 : cluster 3 failed to encode map e142 with expected crc 2023-06-03T20:03:29.988 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:03:29 smithi089 bash[78452]: cluster 1685822608.4881053 mgr.y (mgr.34104) 815 : cluster 0 pgmap v735: 89 pgs: 89 active+clean; 3.3 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 656 op/s 2023-06-03T20:03:29.988 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:03:29 smithi089 bash[78452]: cluster 1685822608.8768327 mon.a (mon.0) 727 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-06-03T20:03:29.988 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:03:29 smithi089 bash[78452]: cluster 1685822608.876873 mon.a (mon.0) 728 : cluster 1 Cluster is now healthy 2023-06-03T20:03:29.989 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:03:29 smithi089 bash[78452]: cluster 1685822608.8926723 osd.5 (osd.5) 66 : cluster 3 failed to encode map e142 with expected crc 2023-06-03T20:03:29.989 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:03:29 smithi089 bash[78452]: cluster 1685822608.8947635 osd.7 (osd.7) 54 : cluster 3 failed to encode map e142 with expected crc 2023-06-03T20:03:29.989 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:03:29 smithi089 bash[78452]: audit 1685822608.896343 mon.a (mon.0) 729 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-06-03T20:03:29.989 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:03:29 smithi089 bash[78452]: cluster 1685822608.8964283 mon.a (mon.0) 730 : cluster 0 osdmap e142: 8 total, 8 up, 8 in 2023-06-03T20:03:29.990 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:03:29 smithi089 bash[78452]: cluster 1685822608.8983057 osd.6 (osd.6) 61 : cluster 3 failed to encode map e142 with expected crc 2023-06-03T20:03:30.006 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:03:30.013 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:03:30.014 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:03:30.115 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:03:30.149 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:03:30.155 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:03:30.156 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:03:30.257 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:03:30.289 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:03:30.292 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:03:30.292 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:03:30.333 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:03:29 smithi164 bash[59462]: cluster 1685822608.4881053 mgr.y (mgr.34104) 815 : cluster 0 pgmap v735: 89 pgs: 89 active+clean; 3.3 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 656 op/s 2023-06-03T20:03:30.334 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:03:29 smithi164 bash[59462]: cluster 1685822608.8768327 mon.a (mon.0) 727 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-06-03T20:03:30.334 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:03:29 smithi164 bash[59462]: cluster 1685822608.876873 mon.a (mon.0) 728 : cluster 1 Cluster is now healthy 2023-06-03T20:03:30.334 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:03:29 smithi164 bash[59462]: cluster 1685822608.8926723 osd.5 (osd.5) 66 : cluster 3 failed to encode map e142 with expected crc 2023-06-03T20:03:30.334 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:03:29 smithi164 bash[59462]: cluster 1685822608.8947635 osd.7 (osd.7) 54 : cluster 3 failed to encode map e142 with expected crc 2023-06-03T20:03:30.335 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:03:29 smithi164 bash[59462]: audit 1685822608.896343 mon.a (mon.0) 729 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-06-03T20:03:30.335 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:03:29 smithi164 bash[59462]: cluster 1685822608.8964283 mon.a (mon.0) 730 : cluster 0 osdmap e142: 8 total, 8 up, 8 in 2023-06-03T20:03:30.335 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:03:29 smithi164 bash[59462]: cluster 1685822608.8983057 osd.6 (osd.6) 61 : cluster 3 failed to encode map e142 with expected crc 2023-06-03T20:03:30.393 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:03:30.445 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:03:30.449 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:03:30.452 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:03:30.553 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:03:30.589 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:03:30.596 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:03:30.597 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:03:30.698 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:03:30.731 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:03:30.735 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:03:30.735 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:03:30.836 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:03:30.871 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:03:30.873 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:03:30.873 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:03:30.975 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:03:31.028 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:03:31.031 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:03:31.032 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:03:31.133 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:03:31.163 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:03:31.171 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:03:31.172 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:03:31.273 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:03:31.314 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:03:31.318 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:03:31.319 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:03:31.420 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:03:31.460 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:03:31.466 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:03:31.467 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:03:31.568 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:03:31.607 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:03:31.612 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:03:31.613 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:03:31.714 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:03:31.753 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:03:31.755 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:03:31.756 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:03:31.857 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:03:31.930 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:03:31.933 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:03:31.934 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:03:32.035 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:03:32.045 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:03:31 smithi164 bash[59462]: cluster 1685822610.4886153 mgr.y (mgr.34104) 816 : cluster 0 pgmap v737: 89 pgs: 89 active+clean; 3.2 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 872 op/s 2023-06-03T20:03:32.078 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:03:32.085 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:03:32.086 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:03:32.187 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:03:32.227 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:03:32.230 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:03:32.232 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:03:32.278 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:03:31 smithi089 bash[75068]: cluster 1685822610.4886153 mgr.y (mgr.34104) 816 : cluster 0 pgmap v737: 89 pgs: 89 active+clean; 3.2 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 872 op/s 2023-06-03T20:03:32.279 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:03:31 smithi089 bash[78452]: cluster 1685822610.4886153 mgr.y (mgr.34104) 816 : cluster 0 pgmap v737: 89 pgs: 89 active+clean; 3.2 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 872 op/s 2023-06-03T20:03:32.333 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:03:32.368 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:03:32.370 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:03:32.371 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:03:32.471 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:03:32.504 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:03:32.511 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:03:32.511 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:03:32.613 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:03:32.632 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-06-03T20:03:32.655 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:03:32.658 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:03:32.658 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:03:32.678 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph osd set noscrub 2023-06-03T20:03:32.759 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:03:32.798 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:03:32.801 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:03:32.802 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:03:32.903 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:03:32.939 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:03:32.941 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:03:32.942 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:03:33.043 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:03:33.087 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:03:33.091 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:03:33.092 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:03:33.193 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:03:33.213 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/mon.a/config 2023-06-03T20:03:33.214 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/mon.a/config 2023-06-03T20:03:33.234 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:03:33.239 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:03:33.240 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:03:33.341 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:03:33.388 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:03:33.396 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:03:33.397 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:03:33.497 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:03:33.544 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:03:33.549 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:03:33.550 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:03:33.651 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:03:33.692 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:03:33.696 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:03:33.697 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:03:33.798 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:03:33.832 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:03:33.837 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:03:33.838 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:03:33.939 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:03:33.970 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:03:33.978 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:03:33.979 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:03:34.080 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:03:34.083 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:03:33 smithi164 bash[59462]: cluster 1685822612.4890199 mgr.y (mgr.34104) 817 : cluster 0 pgmap v738: 89 pgs: 89 active+clean; 3.2 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 482 op/s 2023-06-03T20:03:34.083 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:03:33 smithi164 bash[59462]: audit 1685822613.0544631 mon.a (mon.0) 731 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:03:34.084 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:03:33 smithi164 bash[59462]: audit 1685822613.0673633 mon.a (mon.0) 732 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:03:34.109 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:03:34.116 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:03:34.117 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:03:34.218 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:03:34.226 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:03:33 smithi089 bash[75068]: cluster 1685822612.4890199 mgr.y (mgr.34104) 817 : cluster 0 pgmap v738: 89 pgs: 89 active+clean; 3.2 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 482 op/s 2023-06-03T20:03:34.227 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:03:33 smithi089 bash[75068]: audit 1685822613.0544631 mon.a (mon.0) 731 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:03:34.227 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:03:33 smithi089 bash[75068]: audit 1685822613.0673633 mon.a (mon.0) 732 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:03:34.227 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:03:33 smithi089 bash[78452]: cluster 1685822612.4890199 mgr.y (mgr.34104) 817 : cluster 0 pgmap v738: 89 pgs: 89 active+clean; 3.2 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 482 op/s 2023-06-03T20:03:34.227 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:03:33 smithi089 bash[78452]: audit 1685822613.0544631 mon.a (mon.0) 731 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:03:34.228 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:03:33 smithi089 bash[78452]: audit 1685822613.0673633 mon.a (mon.0) 732 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:03:34.269 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:03:34.275 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:03:34.277 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:03:34.378 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:03:34.406 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:03:34.409 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:03:34.410 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:03:34.511 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:03:34.549 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:03:34.554 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:03:34.555 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:03:34.656 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:03:34.688 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:03:34.697 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:03:34.698 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:03:34.798 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:03:34.831 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:03:34.835 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:03:34.835 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:03:34.936 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:03:34.980 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:03:34.984 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:03:34.984 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:03:35.085 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:03:35.125 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:03:35.128 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:03:35.129 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:03:35.230 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:03:35.266 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:03:35.273 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:03:35.273 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:03:35.374 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:03:35.414 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:03:35.417 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:03:35.418 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:03:35.519 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:03:35.547 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:03:35.554 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:03:35.555 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:03:35.656 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:03:35.684 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:03:35.691 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:03:35.691 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:03:35.792 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:03:35.826 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:03:35.834 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:03:35.834 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:03:35.935 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:03:35.945 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:03:35 smithi164 bash[59462]: cluster 1685822614.4899366 mgr.y (mgr.34104) 818 : cluster 0 pgmap v739: 89 pgs: 89 active+clean; 3.1 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 690 op/s 2023-06-03T20:03:35.945 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:03:35 smithi164 bash[59462]: audit 1685822614.7860937 mon.a (mon.0) 733 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:03:35.945 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:03:35 smithi164 bash[59462]: audit 1685822614.8001723 mon.a (mon.0) 734 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:03:35.945 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:03:35 smithi164 bash[59462]: audit 1685822615.3771164 mon.c (mon.2) 406 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-03T20:03:35.945 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:03:35 smithi164 bash[59462]: audit 1685822615.3785863 mon.c (mon.2) 407 : audit 1 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-03T20:03:35.946 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:03:35 smithi164 bash[59462]: audit 1685822615.3857298 mon.a (mon.0) 735 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:03:35.968 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:03:35.970 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:03:35.971 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:03:36.073 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:03:36.078 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:03:35 smithi089 bash[75068]: cluster 1685822614.4899366 mgr.y (mgr.34104) 818 : cluster 0 pgmap v739: 89 pgs: 89 active+clean; 3.1 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 690 op/s 2023-06-03T20:03:36.078 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:03:35 smithi089 bash[75068]: audit 1685822614.7860937 mon.a (mon.0) 733 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:03:36.078 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:03:35 smithi089 bash[75068]: audit 1685822614.8001723 mon.a (mon.0) 734 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:03:36.079 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:03:35 smithi089 bash[75068]: audit 1685822615.3771164 mon.c (mon.2) 406 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-03T20:03:36.079 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:03:35 smithi089 bash[75068]: audit 1685822615.3785863 mon.c (mon.2) 407 : audit 1 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-03T20:03:36.079 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:03:35 smithi089 bash[75068]: audit 1685822615.3857298 mon.a (mon.0) 735 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:03:36.079 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:03:35 smithi089 bash[78452]: cluster 1685822614.4899366 mgr.y (mgr.34104) 818 : cluster 0 pgmap v739: 89 pgs: 89 active+clean; 3.1 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 690 op/s 2023-06-03T20:03:36.079 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:03:35 smithi089 bash[78452]: audit 1685822614.7860937 mon.a (mon.0) 733 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:03:36.079 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:03:35 smithi089 bash[78452]: audit 1685822614.8001723 mon.a (mon.0) 734 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:03:36.080 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:03:35 smithi089 bash[78452]: audit 1685822615.3771164 mon.c (mon.2) 406 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-03T20:03:36.080 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:03:35 smithi089 bash[78452]: audit 1685822615.3785863 mon.c (mon.2) 407 : audit 1 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-03T20:03:36.080 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:03:35 smithi089 bash[78452]: audit 1685822615.3857298 mon.a (mon.0) 735 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:03:36.115 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:03:36.119 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:03:36.119 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:03:36.220 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:03:36.231 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 20:03:36 smithi089 bash[71163]: ::ffff:172.21.15.164 - - [03/Jun/2023:20:03:36] "GET /metrics HTTP/1.1" 200 228297 "" "Prometheus/2.33.4" 2023-06-03T20:03:36.261 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:03:36.266 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:03:36.266 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:03:36.367 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:03:36.397 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:03:36.398 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:03:36.399 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:03:36.500 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:03:36.540 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:03:36.542 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:03:36.543 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:03:36.644 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:03:36.684 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:03:36.685 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:03:36.686 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:03:36.787 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:03:36.827 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:03:36.829 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:03:36.830 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:03:36.931 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:03:36.960 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:03:36.964 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:03:36.965 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:03:37.066 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:03:37.094 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:03:37.098 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:03:37.099 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:03:37.200 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:03:37.227 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:03:37.232 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:03:37.233 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:03:37.334 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:03:37.369 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:03:37.372 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:03:37.373 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:03:37.474 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:03:37.504 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:03:37.506 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:03:37.506 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:03:37.607 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:03:37.637 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:03:37.640 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:03:37.641 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:03:37.742 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:03:37.784 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:03:37.785 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:03:37.785 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:03:37.886 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:03:37.927 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:03:37.928 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:03:37.928 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:03:38.029 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:03:37 smithi089 bash[75068]: cluster 1685822616.4904797 mgr.y (mgr.34104) 819 : cluster 0 pgmap v740: 89 pgs: 89 active+clean; 3.0 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 686 op/s 2023-06-03T20:03:38.029 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:03:37 smithi089 bash[75068]: audit 1685822617.1081436 mon.a (mon.0) 736 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:03:38.029 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:03:37 smithi089 bash[75068]: audit 1685822617.1139693 mon.c (mon.2) 408 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T20:03:38.030 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:03:37 smithi089 bash[78452]: cluster 1685822616.4904797 mgr.y (mgr.34104) 819 : cluster 0 pgmap v740: 89 pgs: 89 active+clean; 3.0 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 686 op/s 2023-06-03T20:03:38.030 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:03:37 smithi089 bash[78452]: audit 1685822617.1081436 mon.a (mon.0) 736 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:03:38.030 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:03:37 smithi089 bash[78452]: audit 1685822617.1139693 mon.c (mon.2) 408 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T20:03:38.030 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:03:38.055 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:03:38.057 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:03:38.058 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:03:38.083 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:03:37 smithi164 bash[59462]: cluster 1685822616.4904797 mgr.y (mgr.34104) 819 : cluster 0 pgmap v740: 89 pgs: 89 active+clean; 3.0 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 686 op/s 2023-06-03T20:03:38.084 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:03:37 smithi164 bash[59462]: audit 1685822617.1081436 mon.a (mon.0) 736 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:03:38.084 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:03:37 smithi164 bash[59462]: audit 1685822617.1139693 mon.c (mon.2) 408 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T20:03:38.158 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:03:38.189 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:03:38.193 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:03:38.194 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:03:38.295 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:03:38.338 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:03:38.339 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:03:38.339 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:03:38.440 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:03:38.475 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:03:38.480 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:03:38.481 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:03:38.581 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:03:38.618 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/mon.a/config 2023-06-03T20:03:38.619 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:03:38.620 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/mon.a/config 2023-06-03T20:03:38.625 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:03:38.626 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:03:38.726 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:03:38.758 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:03:38.761 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:03:38.762 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:03:38.864 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:03:38.890 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:03:38.893 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:03:38.894 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:03:38.995 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:03:39.032 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:03:39.034 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:03:39.034 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:03:39.135 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:03:39.169 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:03:39.171 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:03:39.172 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:03:39.273 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:03:39.301 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:03:39.305 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:03:39.305 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:03:39.406 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:03:39.437 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:03:39.444 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:03:39.445 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:03:39.546 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:03:39.587 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:03:39.588 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:03:39.589 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:03:39.690 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:03:39.715 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:03:39.721 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:03:39.722 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:03:39.795 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:03:39 smithi164 bash[59462]: cluster 1685822618.4910507 mgr.y (mgr.34104) 820 : cluster 0 pgmap v741: 89 pgs: 89 active+clean; 2.9 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 688 op/s 2023-06-03T20:03:39.824 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:03:39.860 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:03:39.862 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:03:39.863 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:03:39.964 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:03:39.973 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:03:39 smithi089 bash[78452]: cluster 1685822618.4910507 mgr.y (mgr.34104) 820 : cluster 0 pgmap v741: 89 pgs: 89 active+clean; 2.9 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 688 op/s 2023-06-03T20:03:39.973 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:03:39 smithi089 bash[75068]: cluster 1685822618.4910507 mgr.y (mgr.34104) 820 : cluster 0 pgmap v741: 89 pgs: 89 active+clean; 2.9 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 688 op/s 2023-06-03T20:03:39.996 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:03:39.998 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:03:39.999 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:03:40.100 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:03:40.134 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:03:40.136 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:03:40.137 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:03:40.238 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:03:40.274 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:03:40.279 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:03:40.280 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:03:40.381 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:03:40.410 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:03:40.414 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:03:40.414 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:03:40.515 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:03:40.557 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:03:40.560 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:03:40.561 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:03:40.662 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:03:40.702 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:03:40.708 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:03:40.709 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:03:40.810 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:03:40.848 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:03:40.851 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:03:40.852 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:03:40.953 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:03:40.990 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:03:40.996 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:03:40.996 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:03:41.097 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:03:41.138 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:03:41.146 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:03:41.147 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:03:41.248 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:03:41.285 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:03:41.290 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:03:41.291 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:03:41.392 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:03:41.431 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:03:41.437 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:03:41.438 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:03:41.539 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:03:41.583 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:03:41.585 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:03:41.586 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:03:41.687 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:03:41.726 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:03:41.731 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:03:41.732 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:03:41.802 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:03:41 smithi164 bash[59462]: cluster 1685822620.4916375 mgr.y (mgr.34104) 821 : cluster 0 pgmap v742: 89 pgs: 89 active+clean; 2.8 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 650 op/s 2023-06-03T20:03:41.833 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:03:41.868 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:03:41.871 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:03:41.872 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:03:41.963 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:03:41 smithi089 bash[75068]: cluster 1685822620.4916375 mgr.y (mgr.34104) 821 : cluster 0 pgmap v742: 89 pgs: 89 active+clean; 2.8 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 650 op/s 2023-06-03T20:03:41.964 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:03:41 smithi089 bash[78452]: cluster 1685822620.4916375 mgr.y (mgr.34104) 821 : cluster 0 pgmap v742: 89 pgs: 89 active+clean; 2.8 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 650 op/s 2023-06-03T20:03:41.973 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:03:42.010 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:03:42.015 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:03:42.016 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:03:42.117 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:03:42.161 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:03:42.165 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:03:42.166 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:03:42.267 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:03:42.304 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:03:42.310 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:03:42.310 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:03:42.411 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:03:42.450 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:03:42.453 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:03:42.454 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:03:42.555 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:03:42.593 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:03:42.596 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:03:42.597 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:03:42.698 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:03:42.730 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:03:42.732 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:03:42.733 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:03:42.834 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:03:42.861 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:03:42.866 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:03:42.867 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:03:42.968 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:03:43.022 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:03:43.025 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:03:43.026 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:03:43.128 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:03:43.157 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:03:43.159 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:03:43.160 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:03:43.261 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:03:43.284 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:03:43.285 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:03:43.286 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:03:43.387 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:03:43.415 INFO:teuthology.orchestra.run.smithi089.stdout:{ 2023-06-03T20:03:43.415 INFO:teuthology.orchestra.run.smithi089.stdout: "ops": [ 2023-06-03T20:03:43.416 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:03:43.416 INFO:teuthology.orchestra.run.smithi089.stdout: "description": "osd_op(client.45713.0:177517 9.2 9:4e3c05ef:::benchmark_data_smithi089_115580_object67383:head [delete] snapc 0=[] ondisk+write+known_if_redirected e142)", 2023-06-03T20:03:43.416 INFO:teuthology.orchestra.run.smithi089.stdout: "initiated_at": "2023-06-03T20:03:43.375529+0000", 2023-06-03T20:03:43.416 INFO:teuthology.orchestra.run.smithi089.stdout: "age": 0.034058908999999998, 2023-06-03T20:03:43.416 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.034094221000000001, 2023-06-03T20:03:43.416 INFO:teuthology.orchestra.run.smithi089.stdout: "type_data": { 2023-06-03T20:03:43.416 INFO:teuthology.orchestra.run.smithi089.stdout: "flag_point": "waiting for sub ops", 2023-06-03T20:03:43.417 INFO:teuthology.orchestra.run.smithi089.stdout: "client_info": { 2023-06-03T20:03:43.417 INFO:teuthology.orchestra.run.smithi089.stdout: "client": "client.45713", 2023-06-03T20:03:43.417 INFO:teuthology.orchestra.run.smithi089.stdout: "client_addr": "172.21.15.89:0/2400821448", 2023-06-03T20:03:43.417 INFO:teuthology.orchestra.run.smithi089.stdout: "tid": 177517 2023-06-03T20:03:43.417 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:03:43.417 INFO:teuthology.orchestra.run.smithi089.stdout: "events": [ 2023-06-03T20:03:43.417 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:03:43.418 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "initiated", 2023-06-03T20:03:43.418 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:03:43.375529+0000", 2023-06-03T20:03:43.418 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0 2023-06-03T20:03:43.418 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:03:43.418 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:03:43.418 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "throttled", 2023-06-03T20:03:43.418 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:03:43.375529+0000", 2023-06-03T20:03:43.418 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0 2023-06-03T20:03:43.419 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:03:43.419 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:03:43.419 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "header_read", 2023-06-03T20:03:43.419 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:03:43.375527+0000", 2023-06-03T20:03:43.419 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 4294967295.9999981 2023-06-03T20:03:43.419 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:03:43.419 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:03:43.419 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "all_read", 2023-06-03T20:03:43.420 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:03:43.375534+0000", 2023-06-03T20:03:43.420 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 7.2470000000000002e-06 2023-06-03T20:03:43.420 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:03:43.420 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:03:43.420 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "dispatched", 2023-06-03T20:03:43.420 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:03:43.375547+0000", 2023-06-03T20:03:43.420 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 1.3114e-05 2023-06-03T20:03:43.421 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:03:43.421 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:03:43.421 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "queued_for_pg", 2023-06-03T20:03:43.421 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:03:43.375568+0000", 2023-06-03T20:03:43.421 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 2.0356999999999998e-05 2023-06-03T20:03:43.421 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:03:43.421 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:03:43.421 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "reached_pg", 2023-06-03T20:03:43.422 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:03:43.375767+0000", 2023-06-03T20:03:43.422 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.000199438 2023-06-03T20:03:43.422 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:03:43.422 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:03:43.422 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "started", 2023-06-03T20:03:43.422 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:03:43.375894+0000", 2023-06-03T20:03:43.422 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.000126496 2023-06-03T20:03:43.422 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:03:43.423 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:03:43.423 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "waiting for subops from 1,2", 2023-06-03T20:03:43.423 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:03:43.376009+0000", 2023-06-03T20:03:43.423 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.00011502699999999999 2023-06-03T20:03:43.423 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:03:43.423 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:03:43.423 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:03:43.423 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:03:43.386736+0000", 2023-06-03T20:03:43.424 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.010726744 2023-06-03T20:03:43.424 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:03:43.424 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:03:43.424 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "op_commit", 2023-06-03T20:03:43.424 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:03:43.395906+0000", 2023-06-03T20:03:43.424 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.0091707609999999995 2023-06-03T20:03:43.424 INFO:teuthology.orchestra.run.smithi089.stdout: } 2023-06-03T20:03:43.424 INFO:teuthology.orchestra.run.smithi089.stdout: ] 2023-06-03T20:03:43.425 INFO:teuthology.orchestra.run.smithi089.stdout: } 2023-06-03T20:03:43.425 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:03:43.425 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:03:43.425 INFO:teuthology.orchestra.run.smithi089.stdout: "description": "osd_op(client.45713.0:177519 9.2 9:4d475b68:::benchmark_data_smithi089_115580_object67385:head [delete] snapc 0=[] ondisk+write+known_if_redirected e142)", 2023-06-03T20:03:43.425 INFO:teuthology.orchestra.run.smithi089.stdout: "initiated_at": "2023-06-03T20:03:43.375737+0000", 2023-06-03T20:03:43.425 INFO:teuthology.orchestra.run.smithi089.stdout: "age": 0.033850959, 2023-06-03T20:03:43.425 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.033996105999999998, 2023-06-03T20:03:43.425 INFO:teuthology.orchestra.run.smithi089.stdout: "type_data": { 2023-06-03T20:03:43.426 INFO:teuthology.orchestra.run.smithi089.stdout: "flag_point": "waiting for sub ops", 2023-06-03T20:03:43.426 INFO:teuthology.orchestra.run.smithi089.stdout: "client_info": { 2023-06-03T20:03:43.426 INFO:teuthology.orchestra.run.smithi089.stdout: "client": "client.45713", 2023-06-03T20:03:43.426 INFO:teuthology.orchestra.run.smithi089.stdout: "client_addr": "172.21.15.89:0/2400821448", 2023-06-03T20:03:43.426 INFO:teuthology.orchestra.run.smithi089.stdout: "tid": 177519 2023-06-03T20:03:43.426 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:03:43.426 INFO:teuthology.orchestra.run.smithi089.stdout: "events": [ 2023-06-03T20:03:43.426 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:03:43.427 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "initiated", 2023-06-03T20:03:43.427 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:03:43.375737+0000", 2023-06-03T20:03:43.427 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0 2023-06-03T20:03:43.427 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:03:43.427 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:03:43.427 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "throttled", 2023-06-03T20:03:43.427 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:03:43.375737+0000", 2023-06-03T20:03:43.428 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0 2023-06-03T20:03:43.428 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:03:43.428 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:03:43.428 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "header_read", 2023-06-03T20:03:43.429 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:03:43.375736+0000", 2023-06-03T20:03:43.429 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 4294967295.9999995 2023-06-03T20:03:43.429 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:03:43.429 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:03:43.429 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "all_read", 2023-06-03T20:03:43.429 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:03:43.375740+0000", 2023-06-03T20:03:43.430 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 3.4010000000000001e-06 2023-06-03T20:03:43.430 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:03:43.430 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:03:43.430 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "dispatched", 2023-06-03T20:03:43.430 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:03:43.375750+0000", 2023-06-03T20:03:43.430 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 1.0866000000000001e-05 2023-06-03T20:03:43.430 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:03:43.430 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:03:43.431 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "queued_for_pg", 2023-06-03T20:03:43.431 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:03:43.375757+0000", 2023-06-03T20:03:43.431 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 6.2580000000000001e-06 2023-06-03T20:03:43.431 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:03:43.431 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:03:43.431 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "reached_pg", 2023-06-03T20:03:43.431 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:03:43.382213+0000", 2023-06-03T20:03:43.431 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.0064566240000000002 2023-06-03T20:03:43.432 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:03:43.432 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:03:43.432 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "started", 2023-06-03T20:03:43.432 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:03:43.382336+0000", 2023-06-03T20:03:43.432 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.00012264599999999999 2023-06-03T20:03:43.432 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:03:43.432 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:03:43.433 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "waiting for subops from 1,2", 2023-06-03T20:03:43.433 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:03:43.382452+0000", 2023-06-03T20:03:43.433 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.000115857 2023-06-03T20:03:43.433 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:03:43.433 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:03:43.433 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "op_commit", 2023-06-03T20:03:43.433 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:03:43.395912+0000", 2023-06-03T20:03:43.433 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.013459871999999999 2023-06-03T20:03:43.434 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:03:43.434 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:03:43.434 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:03:43.434 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:03:43.400639+0000", 2023-06-03T20:03:43.434 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.0047275900000000003 2023-06-03T20:03:43.434 INFO:teuthology.orchestra.run.smithi089.stdout: } 2023-06-03T20:03:43.434 INFO:teuthology.orchestra.run.smithi089.stdout: ] 2023-06-03T20:03:43.434 INFO:teuthology.orchestra.run.smithi089.stdout: } 2023-06-03T20:03:43.435 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:03:43.435 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:03:43.435 INFO:teuthology.orchestra.run.smithi089.stdout: "description": "osd_op(client.45713.0:177520 9.2 9:4389352e:::benchmark_data_smithi089_115580_object67386:head [delete] snapc 0=[] ondisk+write+known_if_redirected e142)", 2023-06-03T20:03:43.435 INFO:teuthology.orchestra.run.smithi089.stdout: "initiated_at": "2023-06-03T20:03:43.375762+0000", 2023-06-03T20:03:43.435 INFO:teuthology.orchestra.run.smithi089.stdout: "age": 0.033825969999999997, 2023-06-03T20:03:43.435 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.034070700000000002, 2023-06-03T20:03:43.435 INFO:teuthology.orchestra.run.smithi089.stdout: "type_data": { 2023-06-03T20:03:43.435 INFO:teuthology.orchestra.run.smithi089.stdout: "flag_point": "waiting for sub ops", 2023-06-03T20:03:43.436 INFO:teuthology.orchestra.run.smithi089.stdout: "client_info": { 2023-06-03T20:03:43.436 INFO:teuthology.orchestra.run.smithi089.stdout: "client": "client.45713", 2023-06-03T20:03:43.436 INFO:teuthology.orchestra.run.smithi089.stdout: "client_addr": "172.21.15.89:0/2400821448", 2023-06-03T20:03:43.436 INFO:teuthology.orchestra.run.smithi089.stdout: "tid": 177520 2023-06-03T20:03:43.436 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:03:43.436 INFO:teuthology.orchestra.run.smithi089.stdout: "events": [ 2023-06-03T20:03:43.436 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:03:43.436 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "initiated", 2023-06-03T20:03:43.437 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:03:43.375762+0000", 2023-06-03T20:03:43.437 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0 2023-06-03T20:03:43.437 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:03:43.437 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:03:43.437 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "throttled", 2023-06-03T20:03:43.437 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:03:43.375762+0000", 2023-06-03T20:03:43.437 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0 2023-06-03T20:03:43.437 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:03:43.438 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:03:43.438 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "header_read", 2023-06-03T20:03:43.438 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:03:43.375761+0000", 2023-06-03T20:03:43.438 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 4294967295.9999995 2023-06-03T20:03:43.438 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:03:43.438 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:03:43.438 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "all_read", 2023-06-03T20:03:43.438 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:03:43.375764+0000", 2023-06-03T20:03:43.439 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 2.8480000000000001e-06 2023-06-03T20:03:43.439 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:03:43.439 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:03:43.439 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "dispatched", 2023-06-03T20:03:43.439 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:03:43.375770+0000", 2023-06-03T20:03:43.439 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 5.4160000000000003e-06 2023-06-03T20:03:43.439 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:03:43.440 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:03:43.440 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "queued_for_pg", 2023-06-03T20:03:43.440 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:03:43.375774+0000", 2023-06-03T20:03:43.440 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 4.515e-06 2023-06-03T20:03:43.440 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:03:43.440 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:03:43.440 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "reached_pg", 2023-06-03T20:03:43.440 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:03:43.386910+0000", 2023-06-03T20:03:43.441 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.011136276000000001 2023-06-03T20:03:43.441 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:03:43.441 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:03:43.441 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "started", 2023-06-03T20:03:43.441 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:03:43.387033+0000", 2023-06-03T20:03:43.441 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.00012225199999999999 2023-06-03T20:03:43.441 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:03:43.441 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:03:43.442 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "waiting for subops from 1,2", 2023-06-03T20:03:43.442 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:03:43.387154+0000", 2023-06-03T20:03:43.442 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.000121693 2023-06-03T20:03:43.442 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:03:43.442 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:03:43.442 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "op_commit", 2023-06-03T20:03:43.442 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:03:43.408968+0000", 2023-06-03T20:03:43.442 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.021813813000000001 2023-06-03T20:03:43.443 INFO:teuthology.orchestra.run.smithi089.stdout: } 2023-06-03T20:03:43.443 INFO:teuthology.orchestra.run.smithi089.stdout: ] 2023-06-03T20:03:43.443 INFO:teuthology.orchestra.run.smithi089.stdout: } 2023-06-03T20:03:43.443 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:03:43.443 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:03:43.443 INFO:teuthology.orchestra.run.smithi089.stdout: "description": "osd_op(client.45713.0:177523 9.2 9:4eb5cc4d:::benchmark_data_smithi089_115580_object67389:head [delete] snapc 0=[] ondisk+write+known_if_redirected e142)", 2023-06-03T20:03:43.443 INFO:teuthology.orchestra.run.smithi089.stdout: "initiated_at": "2023-06-03T20:03:43.379727+0000", 2023-06-03T20:03:43.444 INFO:teuthology.orchestra.run.smithi089.stdout: "age": 0.029861104999999999, 2023-06-03T20:03:43.444 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.030205717999999999, 2023-06-03T20:03:43.444 INFO:teuthology.orchestra.run.smithi089.stdout: "type_data": { 2023-06-03T20:03:43.444 INFO:teuthology.orchestra.run.smithi089.stdout: "flag_point": "waiting for sub ops", 2023-06-03T20:03:43.444 INFO:teuthology.orchestra.run.smithi089.stdout: "client_info": { 2023-06-03T20:03:43.444 INFO:teuthology.orchestra.run.smithi089.stdout: "client": "client.45713", 2023-06-03T20:03:43.444 INFO:teuthology.orchestra.run.smithi089.stdout: "client_addr": "172.21.15.89:0/2400821448", 2023-06-03T20:03:43.444 INFO:teuthology.orchestra.run.smithi089.stdout: "tid": 177523 2023-06-03T20:03:43.445 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:03:43.445 INFO:teuthology.orchestra.run.smithi089.stdout: "events": [ 2023-06-03T20:03:43.445 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:03:43.445 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "initiated", 2023-06-03T20:03:43.445 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:03:43.379727+0000", 2023-06-03T20:03:43.445 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0 2023-06-03T20:03:43.445 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:03:43.445 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:03:43.446 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "throttled", 2023-06-03T20:03:43.446 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:03:43.379727+0000", 2023-06-03T20:03:43.446 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0 2023-06-03T20:03:43.446 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:03:43.446 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:03:43.446 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "header_read", 2023-06-03T20:03:43.446 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:03:43.379726+0000", 2023-06-03T20:03:43.446 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 4294967295.999999 2023-06-03T20:03:43.447 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:03:43.447 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:03:43.447 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "all_read", 2023-06-03T20:03:43.447 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:03:43.379732+0000", 2023-06-03T20:03:43.447 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 6.2469999999999997e-06 2023-06-03T20:03:43.447 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:03:43.447 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:03:43.447 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "dispatched", 2023-06-03T20:03:43.448 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:03:43.379744+0000", 2023-06-03T20:03:43.448 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 1.2159e-05 2023-06-03T20:03:43.448 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:03:43.448 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:03:43.448 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "queued_for_pg", 2023-06-03T20:03:43.448 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:03:43.379752+0000", 2023-06-03T20:03:43.448 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 7.8210000000000002e-06 2023-06-03T20:03:43.449 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:03:43.449 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:03:43.449 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "reached_pg", 2023-06-03T20:03:43.449 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:03:43.392058+0000", 2023-06-03T20:03:43.449 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.012306335999999999 2023-06-03T20:03:43.449 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:03:43.449 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:03:43.449 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "started", 2023-06-03T20:03:43.450 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:03:43.392208+0000", 2023-06-03T20:03:43.450 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.00014949700000000001 2023-06-03T20:03:43.450 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:03:43.450 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:03:43.450 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "waiting for subops from 1,2", 2023-06-03T20:03:43.450 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:03:43.392354+0000", 2023-06-03T20:03:43.450 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.000146448 2023-06-03T20:03:43.450 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:03:43.451 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:03:43.451 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "op_commit", 2023-06-03T20:03:43.451 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:03:43.408975+0000", 2023-06-03T20:03:43.451 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.016620537000000001 2023-06-03T20:03:43.451 INFO:teuthology.orchestra.run.smithi089.stdout: } 2023-06-03T20:03:43.451 INFO:teuthology.orchestra.run.smithi089.stdout: ] 2023-06-03T20:03:43.451 INFO:teuthology.orchestra.run.smithi089.stdout: } 2023-06-03T20:03:43.451 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:03:43.452 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:03:43.452 INFO:teuthology.orchestra.run.smithi089.stdout: "description": "osd_repop(client.45713.0:177530 9.7 e142/127 9:e7320a6f:::benchmark_data_smithi089_115580_object67396:head v 142'11328, mlcod=142'11326)", 2023-06-03T20:03:43.452 INFO:teuthology.orchestra.run.smithi089.stdout: "initiated_at": "2023-06-03T20:03:43.389354+0000", 2023-06-03T20:03:43.452 INFO:teuthology.orchestra.run.smithi089.stdout: "age": 0.020233522, 2023-06-03T20:03:43.452 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.020669831, 2023-06-03T20:03:43.452 INFO:teuthology.orchestra.run.smithi089.stdout: "type_data": { 2023-06-03T20:03:43.452 INFO:teuthology.orchestra.run.smithi089.stdout: "flag_point": "started", 2023-06-03T20:03:43.453 INFO:teuthology.orchestra.run.smithi089.stdout: "events": [ 2023-06-03T20:03:43.453 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:03:43.453 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "initiated", 2023-06-03T20:03:43.453 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:03:43.389354+0000", 2023-06-03T20:03:43.453 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0 2023-06-03T20:03:43.453 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:03:43.453 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:03:43.453 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "throttled", 2023-06-03T20:03:43.454 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:03:43.389354+0000", 2023-06-03T20:03:43.454 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0 2023-06-03T20:03:43.454 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:03:43.454 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:03:43.454 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "header_read", 2023-06-03T20:03:43.454 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:03:43.389353+0000", 2023-06-03T20:03:43.454 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 4294967295.999999 2023-06-03T20:03:43.454 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:03:43.455 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:03:43.455 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "all_read", 2023-06-03T20:03:43.455 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:03:43.389364+0000", 2023-06-03T20:03:43.455 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 1.0474e-05 2023-06-03T20:03:43.455 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:03:43.455 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:03:43.455 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "dispatched", 2023-06-03T20:03:43.455 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:03:43.389375+0000", 2023-06-03T20:03:43.456 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 1.1246e-05 2023-06-03T20:03:43.456 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:03:43.456 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:03:43.456 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "queued_for_pg", 2023-06-03T20:03:43.456 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:03:43.389383+0000", 2023-06-03T20:03:43.456 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 7.5360000000000004e-06 2023-06-03T20:03:43.456 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:03:43.457 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:03:43.457 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "reached_pg", 2023-06-03T20:03:43.457 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:03:43.389507+0000", 2023-06-03T20:03:43.457 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.00012424199999999999 2023-06-03T20:03:43.457 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:03:43.457 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:03:43.457 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "started", 2023-06-03T20:03:43.457 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:03:43.389525+0000", 2023-06-03T20:03:43.458 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 1.8518000000000001e-05 2023-06-03T20:03:43.458 INFO:teuthology.orchestra.run.smithi089.stdout: } 2023-06-03T20:03:43.458 INFO:teuthology.orchestra.run.smithi089.stdout: ] 2023-06-03T20:03:43.458 INFO:teuthology.orchestra.run.smithi089.stdout: } 2023-06-03T20:03:43.458 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:03:43.458 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:03:43.458 INFO:teuthology.orchestra.run.smithi089.stdout: "description": "osd_op(client.45713.0:177532 9.2 9:4c671e00:::benchmark_data_smithi089_115580_object67398:head [delete] snapc 0=[] ondisk+write+known_if_redirected e142)", 2023-06-03T20:03:43.458 INFO:teuthology.orchestra.run.smithi089.stdout: "initiated_at": "2023-06-03T20:03:43.393908+0000", 2023-06-03T20:03:43.459 INFO:teuthology.orchestra.run.smithi089.stdout: "age": 0.015679992, 2023-06-03T20:03:43.459 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.016186704, 2023-06-03T20:03:43.459 INFO:teuthology.orchestra.run.smithi089.stdout: "type_data": { 2023-06-03T20:03:43.459 INFO:teuthology.orchestra.run.smithi089.stdout: "flag_point": "waiting for sub ops", 2023-06-03T20:03:43.459 INFO:teuthology.orchestra.run.smithi089.stdout: "client_info": { 2023-06-03T20:03:43.459 INFO:teuthology.orchestra.run.smithi089.stdout: "client": "client.45713", 2023-06-03T20:03:43.459 INFO:teuthology.orchestra.run.smithi089.stdout: "client_addr": "172.21.15.89:0/2400821448", 2023-06-03T20:03:43.459 INFO:teuthology.orchestra.run.smithi089.stdout: "tid": 177532 2023-06-03T20:03:43.460 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:03:43.460 INFO:teuthology.orchestra.run.smithi089.stdout: "events": [ 2023-06-03T20:03:43.460 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:03:43.460 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "initiated", 2023-06-03T20:03:43.460 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:03:43.393908+0000", 2023-06-03T20:03:43.460 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0 2023-06-03T20:03:43.460 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:03:43.461 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:03:43.461 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "throttled", 2023-06-03T20:03:43.461 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:03:43.393908+0000", 2023-06-03T20:03:43.461 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0 2023-06-03T20:03:43.461 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:03:43.461 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:03:43.461 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "header_read", 2023-06-03T20:03:43.461 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:03:43.393906+0000", 2023-06-03T20:03:43.462 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 4294967295.9999981 2023-06-03T20:03:43.462 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:03:43.462 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:03:43.462 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "all_read", 2023-06-03T20:03:43.462 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:03:43.393914+0000", 2023-06-03T20:03:43.462 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 7.7279999999999995e-06 2023-06-03T20:03:43.462 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:03:43.462 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:03:43.463 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "dispatched", 2023-06-03T20:03:43.463 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:03:43.393928+0000", 2023-06-03T20:03:43.463 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 1.4044e-05 2023-06-03T20:03:43.463 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:03:43.463 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:03:43.463 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "queued_for_pg", 2023-06-03T20:03:43.463 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:03:43.393937+0000", 2023-06-03T20:03:43.463 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 9.4409999999999999e-06 2023-06-03T20:03:43.464 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:03:43.464 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:03:43.464 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "reached_pg", 2023-06-03T20:03:43.464 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:03:43.395930+0000", 2023-06-03T20:03:43.464 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.0019924439999999999 2023-06-03T20:03:43.464 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:03:43.464 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:03:43.464 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "started", 2023-06-03T20:03:43.465 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:03:43.396015+0000", 2023-06-03T20:03:43.465 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 8.4897999999999999e-05 2023-06-03T20:03:43.465 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:03:43.465 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:03:43.465 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "waiting for subops from 1,2", 2023-06-03T20:03:43.465 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:03:43.396099+0000", 2023-06-03T20:03:43.465 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 8.4043999999999997e-05 2023-06-03T20:03:43.465 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:03:43.466 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:03:43.466 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "op_commit", 2023-06-03T20:03:43.466 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:03:43.408980+0000", 2023-06-03T20:03:43.466 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.01288158 2023-06-03T20:03:43.466 INFO:teuthology.orchestra.run.smithi089.stdout: } 2023-06-03T20:03:43.466 INFO:teuthology.orchestra.run.smithi089.stdout: ] 2023-06-03T20:03:43.466 INFO:teuthology.orchestra.run.smithi089.stdout: } 2023-06-03T20:03:43.466 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:03:43.467 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:03:43.467 INFO:teuthology.orchestra.run.smithi089.stdout: "description": "osd_repop(client.45713.0:177535 9.f e142/127 9:f35c81d3:::benchmark_data_smithi089_115580_object67401:head v 142'10878, mlcod=142'10876)", 2023-06-03T20:03:43.467 INFO:teuthology.orchestra.run.smithi089.stdout: "initiated_at": "2023-06-03T20:03:43.398778+0000", 2023-06-03T20:03:43.467 INFO:teuthology.orchestra.run.smithi089.stdout: "age": 0.010809622, 2023-06-03T20:03:43.467 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.011401982, 2023-06-03T20:03:43.467 INFO:teuthology.orchestra.run.smithi089.stdout: "type_data": { 2023-06-03T20:03:43.467 INFO:teuthology.orchestra.run.smithi089.stdout: "flag_point": "started", 2023-06-03T20:03:43.467 INFO:teuthology.orchestra.run.smithi089.stdout: "events": [ 2023-06-03T20:03:43.468 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:03:43.468 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "initiated", 2023-06-03T20:03:43.468 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:03:43.398778+0000", 2023-06-03T20:03:43.468 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0 2023-06-03T20:03:43.468 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:03:43.468 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:03:43.468 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "throttled", 2023-06-03T20:03:43.469 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:03:43.398778+0000", 2023-06-03T20:03:43.469 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0 2023-06-03T20:03:43.469 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:03:43.469 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:03:43.469 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "header_read", 2023-06-03T20:03:43.469 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:03:43.398777+0000", 2023-06-03T20:03:43.469 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 4294967295.999999 2023-06-03T20:03:43.469 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:03:43.470 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:03:43.470 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "all_read", 2023-06-03T20:03:43.470 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:03:43.398786+0000", 2023-06-03T20:03:43.470 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 9.2229999999999996e-06 2023-06-03T20:03:43.470 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:03:43.470 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:03:43.470 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "dispatched", 2023-06-03T20:03:43.470 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:03:43.398805+0000", 2023-06-03T20:03:43.471 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 1.8335999999999999e-05 2023-06-03T20:03:43.471 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:03:43.471 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:03:43.471 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "queued_for_pg", 2023-06-03T20:03:43.471 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:03:43.398813+0000", 2023-06-03T20:03:43.471 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 8.5629999999999992e-06 2023-06-03T20:03:43.471 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:03:43.471 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:03:43.472 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "reached_pg", 2023-06-03T20:03:43.472 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:03:43.400379+0000", 2023-06-03T20:03:43.472 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.001566209 2023-06-03T20:03:43.472 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:03:43.472 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:03:43.472 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "started", 2023-06-03T20:03:43.472 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:03:43.400402+0000", 2023-06-03T20:03:43.472 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 2.2889999999999999e-05 2023-06-03T20:03:43.473 INFO:teuthology.orchestra.run.smithi089.stdout: } 2023-06-03T20:03:43.473 INFO:teuthology.orchestra.run.smithi089.stdout: ] 2023-06-03T20:03:43.473 INFO:teuthology.orchestra.run.smithi089.stdout: } 2023-06-03T20:03:43.473 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:03:43.473 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:03:43.473 INFO:teuthology.orchestra.run.smithi089.stdout: "description": "osd_repop(client.45713.0:177536 9.7 e142/127 9:eff52999:::benchmark_data_smithi089_115580_object67402:head v 142'11329, mlcod=142'11326)", 2023-06-03T20:03:43.473 INFO:teuthology.orchestra.run.smithi089.stdout: "initiated_at": "2023-06-03T20:03:43.407006+0000", 2023-06-03T20:03:43.473 INFO:teuthology.orchestra.run.smithi089.stdout: "age": 0.002581809, 2023-06-03T20:03:43.474 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.003235492, 2023-06-03T20:03:43.474 INFO:teuthology.orchestra.run.smithi089.stdout: "type_data": { 2023-06-03T20:03:43.474 INFO:teuthology.orchestra.run.smithi089.stdout: "flag_point": "started", 2023-06-03T20:03:43.474 INFO:teuthology.orchestra.run.smithi089.stdout: "events": [ 2023-06-03T20:03:43.474 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:03:43.474 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "initiated", 2023-06-03T20:03:43.474 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:03:43.407006+0000", 2023-06-03T20:03:43.475 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0 2023-06-03T20:03:43.475 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:03:43.475 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:03:43.475 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "throttled", 2023-06-03T20:03:43.475 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:03:43.407006+0000", 2023-06-03T20:03:43.475 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0 2023-06-03T20:03:43.475 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:03:43.475 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:03:43.476 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "header_read", 2023-06-03T20:03:43.476 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:03:43.407005+0000", 2023-06-03T20:03:43.476 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 4294967295.999999 2023-06-03T20:03:43.476 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:03:43.476 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:03:43.476 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "all_read", 2023-06-03T20:03:43.476 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:03:43.407015+0000", 2023-06-03T20:03:43.476 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 1.0251000000000001e-05 2023-06-03T20:03:43.477 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:03:43.477 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:03:43.477 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "dispatched", 2023-06-03T20:03:43.477 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:03:43.407029+0000", 2023-06-03T20:03:43.477 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 1.344e-05 2023-06-03T20:03:43.477 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:03:43.477 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:03:43.477 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "queued_for_pg", 2023-06-03T20:03:43.478 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:03:43.407037+0000", 2023-06-03T20:03:43.478 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 8.3690000000000007e-06 2023-06-03T20:03:43.478 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:03:43.478 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:03:43.478 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "reached_pg", 2023-06-03T20:03:43.478 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:03:43.407107+0000", 2023-06-03T20:03:43.478 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 6.9367000000000002e-05 2023-06-03T20:03:43.479 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:03:43.479 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:03:43.479 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "started", 2023-06-03T20:03:43.479 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:03:43.407128+0000", 2023-06-03T20:03:43.479 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 2.1651e-05 2023-06-03T20:03:43.479 INFO:teuthology.orchestra.run.smithi089.stdout: } 2023-06-03T20:03:43.479 INFO:teuthology.orchestra.run.smithi089.stdout: ] 2023-06-03T20:03:43.479 INFO:teuthology.orchestra.run.smithi089.stdout: } 2023-06-03T20:03:43.480 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:03:43.480 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:03:43.480 INFO:teuthology.orchestra.run.smithi089.stdout: "description": "osd_op(client.45713.0:177538 9.1 9:8370bff9:::benchmark_data_smithi089_115580_object67404:head [delete] snapc 0=[] ondisk+write+known_if_redirected e142)", 2023-06-03T20:03:43.480 INFO:teuthology.orchestra.run.smithi089.stdout: "initiated_at": "2023-06-03T20:03:43.407313+0000", 2023-06-03T20:03:43.480 INFO:teuthology.orchestra.run.smithi089.stdout: "age": 0.0022752200000000001, 2023-06-03T20:03:43.480 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.002991554, 2023-06-03T20:03:43.480 INFO:teuthology.orchestra.run.smithi089.stdout: "type_data": { 2023-06-03T20:03:43.480 INFO:teuthology.orchestra.run.smithi089.stdout: "flag_point": "waiting for sub ops", 2023-06-03T20:03:43.481 INFO:teuthology.orchestra.run.smithi089.stdout: "client_info": { 2023-06-03T20:03:43.481 INFO:teuthology.orchestra.run.smithi089.stdout: "client": "client.45713", 2023-06-03T20:03:43.481 INFO:teuthology.orchestra.run.smithi089.stdout: "client_addr": "172.21.15.89:0/2400821448", 2023-06-03T20:03:43.481 INFO:teuthology.orchestra.run.smithi089.stdout: "tid": 177538 2023-06-03T20:03:43.481 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:03:43.481 INFO:teuthology.orchestra.run.smithi089.stdout: "events": [ 2023-06-03T20:03:43.481 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:03:43.481 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "initiated", 2023-06-03T20:03:43.482 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:03:43.407313+0000", 2023-06-03T20:03:43.482 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0 2023-06-03T20:03:43.482 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:03:43.482 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:03:43.482 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "throttled", 2023-06-03T20:03:43.482 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:03:43.407313+0000", 2023-06-03T20:03:43.482 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0 2023-06-03T20:03:43.482 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:03:43.483 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:03:43.483 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "header_read", 2023-06-03T20:03:43.483 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:03:43.407311+0000", 2023-06-03T20:03:43.483 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 4294967295.9999981 2023-06-03T20:03:43.483 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:03:43.483 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:03:43.483 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "all_read", 2023-06-03T20:03:43.483 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:03:43.407318+0000", 2023-06-03T20:03:43.484 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 7.0720000000000002e-06 2023-06-03T20:03:43.484 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:03:43.484 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:03:43.484 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "dispatched", 2023-06-03T20:03:43.484 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:03:43.407330+0000", 2023-06-03T20:03:43.484 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 1.2264e-05 2023-06-03T20:03:43.484 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:03:43.484 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:03:43.485 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "queued_for_pg", 2023-06-03T20:03:43.485 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:03:43.407338+0000", 2023-06-03T20:03:43.485 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 8.0569999999999993e-06 2023-06-03T20:03:43.485 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:03:43.485 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:03:43.485 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "reached_pg", 2023-06-03T20:03:43.485 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:03:43.407394+0000", 2023-06-03T20:03:43.486 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 5.6159000000000002e-05 2023-06-03T20:03:43.486 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:03:43.486 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:03:43.486 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "started", 2023-06-03T20:03:43.486 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:03:43.407544+0000", 2023-06-03T20:03:43.486 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.00015005500000000001 2023-06-03T20:03:43.486 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:03:43.486 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:03:43.487 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "waiting for subops from 6,7", 2023-06-03T20:03:43.487 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:03:43.407682+0000", 2023-06-03T20:03:43.487 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.000138152 2023-06-03T20:03:43.487 INFO:teuthology.orchestra.run.smithi089.stdout: } 2023-06-03T20:03:43.487 INFO:teuthology.orchestra.run.smithi089.stdout: ] 2023-06-03T20:03:43.487 INFO:teuthology.orchestra.run.smithi089.stdout: } 2023-06-03T20:03:43.487 INFO:teuthology.orchestra.run.smithi089.stdout: } 2023-06-03T20:03:43.487 INFO:teuthology.orchestra.run.smithi089.stdout: ], 2023-06-03T20:03:43.488 INFO:teuthology.orchestra.run.smithi089.stdout: "num_ops": 9 2023-06-03T20:03:43.488 INFO:teuthology.orchestra.run.smithi089.stdout:} 2023-06-03T20:03:43.488 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:03:43.489 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:03:43.489 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:03:43.590 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:03:43.630 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:03:43.635 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:03:43.636 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:03:43.737 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:03:43.779 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:03:43.780 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:03:43.780 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:03:43.881 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:03:43.911 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:03:43.914 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:03:43.915 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:03:44.001 INFO:teuthology.orchestra.run.smithi089.stdout: 2023-06-03T20:03:44.002 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:03:43 smithi089 bash[78452]: cluster 1685822622.4921174 mgr.y (mgr.34104) 822 : cluster 0 pgmap v743: 89 pgs: 89 active+clean; 2.8 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 571 op/s 2023-06-03T20:03:44.003 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:03:43 smithi089 bash[75068]: cluster 1685822622.4921174 mgr.y (mgr.34104) 822 : cluster 0 pgmap v743: 89 pgs: 89 active+clean; 2.8 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 571 op/s 2023-06-03T20:03:44.004 INFO:teuthology.orchestra.run.smithi089.stderr:dumped all 2023-06-03T20:03:44.016 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:03:44.026 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:03:43 smithi164 bash[59462]: cluster 1685822622.4921174 mgr.y (mgr.34104) 822 : cluster 0 pgmap v743: 89 pgs: 89 active+clean; 2.8 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 571 op/s 2023-06-03T20:03:44.049 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:03:44.053 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:03:44.053 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:03:44.154 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:03:44.189 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:03:44.192 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:03:44.192 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:03:44.294 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:03:44.328 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:03:44.330 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:03:44.331 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:03:44.432 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:03:44.472 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:03:44.477 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:03:44.478 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:03:44.579 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:03:44.609 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:03:44.613 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:03:44.614 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:03:44.715 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:03:44.741 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:03:44.743 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:03:44.744 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:03:44.845 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:03:44.882 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:03:44.884 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:03:44.885 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:03:44.985 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:03:45.009 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:03:45.017 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:03:45.018 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:03:45.119 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:03:45.158 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:03:45.164 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:03:45.165 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:03:45.265 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:03:45.302 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:03:45.306 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:03:45.307 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:03:45.408 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:03:45.439 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:03:45.445 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:03:45.445 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:03:45.546 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:03:45.584 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:03:45.590 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:03:45.590 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:03:45.691 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:03:45.747 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:03:45.752 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:03:45.753 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:03:45.810 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:03:45 smithi164 bash[59462]: audit 1685822623.976198 mgr.y (mgr.34104) 823 : audit 0 from='client.35871 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-06-03T20:03:45.811 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:03:45 smithi164 bash[59462]: cluster 1685822624.492768 mgr.y (mgr.34104) 824 : cluster 0 pgmap v744: 89 pgs: 89 active+clean; 2.6 GiB data, 9.7 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 818 op/s 2023-06-03T20:03:45.853 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:03:45.893 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:03:45.899 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:03:45.900 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:03:46.001 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:03:46.032 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:03:46.039 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:03:46.040 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:03:46.141 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:03:46.150 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:03:45 smithi089 bash[75068]: audit 1685822623.976198 mgr.y (mgr.34104) 823 : audit 0 from='client.35871 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-06-03T20:03:46.150 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:03:45 smithi089 bash[75068]: cluster 1685822624.492768 mgr.y (mgr.34104) 824 : cluster 0 pgmap v744: 89 pgs: 89 active+clean; 2.6 GiB data, 9.7 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 818 op/s 2023-06-03T20:03:46.150 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:03:45 smithi089 bash[78452]: audit 1685822623.976198 mgr.y (mgr.34104) 823 : audit 0 from='client.35871 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-06-03T20:03:46.150 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:03:45 smithi089 bash[78452]: cluster 1685822624.492768 mgr.y (mgr.34104) 824 : cluster 0 pgmap v744: 89 pgs: 89 active+clean; 2.6 GiB data, 9.7 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 818 op/s 2023-06-03T20:03:46.178 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:03:46.181 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:03:46.181 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:03:46.282 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:03:46.321 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:03:46.326 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:03:46.327 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:03:46.428 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:03:46.437 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 20:03:46 smithi089 bash[71163]: ::ffff:172.21.15.164 - - [03/Jun/2023:20:03:46] "GET /metrics HTTP/1.1" 200 228313 "" "Prometheus/2.33.4" 2023-06-03T20:03:46.460 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:03:46.470 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:03:46.471 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:03:46.572 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:03:46.610 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:03:46.614 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:03:46.614 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:03:46.715 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:03:46.755 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:03:46.756 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:03:46.757 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:03:46.858 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:03:46.899 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:03:46.901 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:03:46.902 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:03:47.003 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:03:47.040 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:03:47.045 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:03:47.046 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:03:47.072 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph daemon osd.3 dump_blocked_ops 2023-06-03T20:03:47.147 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:03:47.197 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:03:47.201 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:03:47.202 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:03:47.235 INFO:teuthology.orchestra.run.smithi089.stdout:{"pg_ready":true,"pg_map":{"version":743,"stamp":"2023-06-03T20:03:42.491832+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":3030390694,"num_objects":46256,"num_object_clones":0,"num_object_copies":138768,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":46256,"num_whiteouts":0,"num_read":309,"num_read_kb":513,"num_write":174293,"num_write_kb":7050831,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139427,"ondisk_log_size":139427,"up":267,"acting":267,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":267,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":10727480,"kb_used_data":8936824,"kb_used_omap":30,"kb_used_meta":1790113,"kb_avail":739233736,"statfs":{"total":767960285184,"available":756975345664,"internally_reserved":0,"allocated":9151307776,"data_stored":9147149684,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":31502,"internal_metadata":1833075954},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[5,0,3,2,8,3,4],"upper_bound":128},"perf_stat":{"commit_latency_ms":4,"apply_latency_ms":4,"commit_latency_ns":4000000,"apply_latency_ns":4000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-449576960,"num_objects":-6860,"num_object_clones":0,"num_object_copies":-20580,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":-6860,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6860,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.003537"},"pg_stats":[{"pgid":"3.1a","version":"60'1","reported_seq":129,"reported_epoch":141,"state":"active+clean","last_fresh":"2023-06-03T20:03:23.845153+0000","last_change":"2023-06-03T19:57:28.467559+0000","last_active":"2023-06-03T20:03:23.845153+0000","last_peered":"2023-06-03T20:03:23.845153+0000","last_clean":"2023-06-03T20:03:23.845153+0000","last_became_active":"2023-06-03T19:57:28.467077+0000","last_became_peered":"2023-06-03T19:57:28.467077+0000","last_unstale":"2023-06-03T20:03:23.845153+0000","last_undegraded":"2023-06-03T20:03:23.845153+0000","last_fullsized":"2023-06-03T20:03:23.845153+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T21:46:02.795576+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":122,"reported_epoch":141,"state":"active+clean","last_fresh":"2023-06-03T20:03:23.849826+0000","last_change":"2023-06-03T19:57:28.470287+0000","last_active":"2023-06-03T20:03:23.849826+0000","last_peered":"2023-06-03T20:03:23.849826+0000","last_clean":"2023-06-03T20:03:23.849826+0000","last_became_active":"2023-06-03T19:57:28.469477+0000","last_became_peered":"2023-06-03T19:57:28.469477+0000","last_unstale":"2023-06-03T20:03:23.849826+0000","last_undegraded":"2023-06-03T20:03:23.849826+0000","last_fullsized":"2023-06-03T20:03:23.849826+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T06:48:19.229190+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":135,"reported_epoch":141,"state":"active+clean","last_fresh":"2023-06-03T20:03:23.851027+0000","last_change":"2023-06-03T19:57:28.473350+0000","last_active":"2023-06-03T20:03:23.851027+0000","last_peered":"2023-06-03T20:03:23.851027+0000","last_clean":"2023-06-03T20:03:23.851027+0000","last_became_active":"2023-06-03T19:57:28.472757+0000","last_became_peered":"2023-06-03T19:57:28.472757+0000","last_unstale":"2023-06-03T20:03:23.851027+0000","last_undegraded":"2023-06-03T20:03:23.851027+0000","last_fullsized":"2023-06-03T20:03:23.851027+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T19:58:10.432188+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":100,"reported_epoch":141,"state":"active+clean","last_fresh":"2023-06-03T20:03:23.860756+0000","last_change":"2023-06-03T19:56:54.507653+0000","last_active":"2023-06-03T20:03:23.860756+0000","last_peered":"2023-06-03T20:03:23.860756+0000","last_clean":"2023-06-03T20:03:23.860756+0000","last_became_active":"2023-06-03T19:56:54.507350+0000","last_became_peered":"2023-06-03T19:56:54.507350+0000","last_unstale":"2023-06-03T20:03:23.860756+0000","last_undegraded":"2023-06-03T20:03:23.860756+0000","last_fullsized":"2023-06-03T20:03:23.860756+0000","mapping_epoch":119,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T22:33:20.065844+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":123,"reported_epoch":141,"state":"active+clean","last_fresh":"2023-06-03T20:03:23.849392+0000","last_change":"2023-06-03T19:56:54.512932+0000","last_active":"2023-06-03T20:03:23.849392+0000","last_peered":"2023-06-03T20:03:23.849392+0000","last_clean":"2023-06-03T20:03:23.849392+0000","last_became_active":"2023-06-03T19:56:54.511992+0000","last_became_peered":"2023-06-03T19:56:54.511992+0000","last_unstale":"2023-06-03T20:03:23.849392+0000","last_undegraded":"2023-06-03T20:03:23.849392+0000","last_fullsized":"2023-06-03T20:03:23.849392+0000","mapping_epoch":119,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T21:01:35.069911+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":108,"reported_epoch":141,"state":"active+clean","last_fresh":"2023-06-03T20:03:23.870784+0000","last_change":"2023-06-03T19:51:59.359522+0000","last_active":"2023-06-03T20:03:23.870784+0000","last_peered":"2023-06-03T20:03:23.870784+0000","last_clean":"2023-06-03T20:03:23.870784+0000","last_became_active":"2023-06-03T19:51:59.359202+0000","last_became_peered":"2023-06-03T19:51:59.359202+0000","last_unstale":"2023-06-03T20:03:23.870784+0000","last_undegraded":"2023-06-03T20:03:23.870784+0000","last_fullsized":"2023-06-03T20:03:23.870784+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":97,"reported_epoch":141,"state":"active+clean","last_fresh":"2023-06-03T20:03:23.861067+0000","last_change":"2023-06-03T19:56:54.509490+0000","last_active":"2023-06-03T20:03:23.861067+0000","last_peered":"2023-06-03T20:03:23.861067+0000","last_clean":"2023-06-03T20:03:23.861067+0000","last_became_active":"2023-06-03T19:56:54.509145+0000","last_became_peered":"2023-06-03T19:56:54.509145+0000","last_unstale":"2023-06-03T20:03:23.861067+0000","last_undegraded":"2023-06-03T20:03:23.861067+0000","last_fullsized":"2023-06-03T20:03:23.861067+0000","mapping_epoch":119,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T07:12:03.964110+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":107,"reported_epoch":141,"state":"active+clean","last_fresh":"2023-06-03T20:03:23.849304+0000","last_change":"2023-06-03T19:56:54.511325+0000","last_active":"2023-06-03T20:03:23.849304+0000","last_peered":"2023-06-03T20:03:23.849304+0000","last_clean":"2023-06-03T20:03:23.849304+0000","last_became_active":"2023-06-03T19:56:54.511033+0000","last_became_peered":"2023-06-03T19:56:54.511033+0000","last_unstale":"2023-06-03T20:03:23.849304+0000","last_undegraded":"2023-06-03T20:03:23.849304+0000","last_fullsized":"2023-06-03T20:03:23.849304+0000","mapping_epoch":119,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T23:50:08.711478+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":109,"reported_epoch":141,"state":"active+clean","last_fresh":"2023-06-03T20:03:23.856929+0000","last_change":"2023-06-03T19:51:59.374143+0000","last_active":"2023-06-03T20:03:23.856929+0000","last_peered":"2023-06-03T20:03:23.856929+0000","last_clean":"2023-06-03T20:03:23.856929+0000","last_became_active":"2023-06-03T19:51:59.373146+0000","last_became_peered":"2023-06-03T19:51:59.373146+0000","last_unstale":"2023-06-03T20:03:23.856929+0000","last_undegraded":"2023-06-03T20:03:23.856929+0000","last_fullsized":"2023-06-03T20:03:23.856929+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":88,"reported_epoch":141,"state":"active+clean","last_fresh":"2023-06-03T20:03:23.849330+0000","last_change":"2023-06-03T19:56:54.510462+0000","last_active":"2023-06-03T20:03:23.849330+0000","last_peered":"2023-06-03T20:03:23.849330+0000","last_clean":"2023-06-03T20:03:23.849330+0000","last_became_active":"2023-06-03T19:56:54.509783+0000","last_became_peered":"2023-06-03T19:56:54.509783+0000","last_unstale":"2023-06-03T20:03:23.849330+0000","last_undegraded":"2023-06-03T20:03:23.849330+0000","last_fullsized":"2023-06-03T20:03:23.849330+0000","mapping_epoch":119,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T00:16:09.624276+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"60'3","reported_seq":127,"reported_epoch":141,"state":"active+clean","last_fresh":"2023-06-03T20:03:23.856518+0000","last_change":"2023-06-03T19:57:28.466401+0000","last_active":"2023-06-03T20:03:23.856518+0000","last_peered":"2023-06-03T20:03:23.856518+0000","last_clean":"2023-06-03T20:03:23.856518+0000","last_became_active":"2023-06-03T19:57:28.465452+0000","last_became_peered":"2023-06-03T19:57:28.465452+0000","last_unstale":"2023-06-03T20:03:23.856518+0000","last_undegraded":"2023-06-03T20:03:23.856518+0000","last_fullsized":"2023-06-03T20:03:23.856518+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":80,"reported_epoch":142,"state":"active+clean","last_fresh":"2023-06-03T20:03:28.908217+0000","last_change":"2023-06-03T19:56:11.753995+0000","last_active":"2023-06-03T20:03:28.908217+0000","last_peered":"2023-06-03T20:03:28.908217+0000","last_clean":"2023-06-03T20:03:28.908217+0000","last_became_active":"2023-06-03T19:56:11.753676+0000","last_became_peered":"2023-06-03T19:56:11.753676+0000","last_unstale":"2023-06-03T20:03:28.908217+0000","last_undegraded":"2023-06-03T20:03:28.908217+0000","last_fullsized":"2023-06-03T20:03:28.908217+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":115,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T01:00:40.343647+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"77'23","reported_seq":125,"reported_epoch":141,"state":"active+clean","last_fresh":"2023-06-03T20:03:23.845733+0000","last_change":"2023-06-03T19:57:28.462522+0000","last_active":"2023-06-03T20:03:23.845733+0000","last_peered":"2023-06-03T20:03:23.845733+0000","last_clean":"2023-06-03T20:03:23.845733+0000","last_became_active":"2023-06-03T19:57:28.462048+0000","last_became_peered":"2023-06-03T19:57:28.462048+0000","last_unstale":"2023-06-03T20:03:23.845733+0000","last_undegraded":"2023-06-03T20:03:23.845733+0000","last_fullsized":"2023-06-03T20:03:23.845733+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":23,"log_dups_size":0,"ondisk_log_size":23,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T23:23:56.697534+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4716,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":23,"num_write_kb":20,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":102,"reported_epoch":141,"state":"active+clean","last_fresh":"2023-06-03T20:03:23.844808+0000","last_change":"2023-06-03T19:57:28.469676+0000","last_active":"2023-06-03T20:03:23.844808+0000","last_peered":"2023-06-03T20:03:23.844808+0000","last_clean":"2023-06-03T20:03:23.844808+0000","last_became_active":"2023-06-03T19:57:28.469328+0000","last_became_peered":"2023-06-03T19:57:28.469328+0000","last_unstale":"2023-06-03T20:03:23.844808+0000","last_undegraded":"2023-06-03T20:03:23.844808+0000","last_fullsized":"2023-06-03T20:03:23.844808+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T23:25:26.041930+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"60'2","reported_seq":126,"reported_epoch":141,"state":"active+clean","last_fresh":"2023-06-03T20:03:23.854980+0000","last_change":"2023-06-03T19:57:28.467711+0000","last_active":"2023-06-03T20:03:23.854980+0000","last_peered":"2023-06-03T20:03:23.854980+0000","last_clean":"2023-06-03T20:03:23.854980+0000","last_became_active":"2023-06-03T19:57:28.467234+0000","last_became_peered":"2023-06-03T19:57:28.467234+0000","last_unstale":"2023-06-03T20:03:23.854980+0000","last_undegraded":"2023-06-03T20:03:23.854980+0000","last_fullsized":"2023-06-03T20:03:23.854980+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":107,"reported_epoch":141,"state":"active+clean","last_fresh":"2023-06-03T20:03:23.849906+0000","last_change":"2023-06-03T19:56:54.514616+0000","last_active":"2023-06-03T20:03:23.849906+0000","last_peered":"2023-06-03T20:03:23.849906+0000","last_clean":"2023-06-03T20:03:23.849906+0000","last_became_active":"2023-06-03T19:56:54.513918+0000","last_became_peered":"2023-06-03T19:56:54.513918+0000","last_unstale":"2023-06-03T20:03:23.849906+0000","last_undegraded":"2023-06-03T20:03:23.849906+0000","last_fullsized":"2023-06-03T20:03:23.849906+0000","mapping_epoch":119,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T19:36:10.967386+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":98,"reported_epoch":141,"state":"active+clean","last_fresh":"2023-06-03T20:03:24.788553+0000","last_change":"2023-06-03T19:56:54.507479+0000","last_active":"2023-06-03T20:03:24.788553+0000","last_peered":"2023-06-03T20:03:24.788553+0000","last_clean":"2023-06-03T20:03:24.788553+0000","last_became_active":"2023-06-03T19:56:54.507149+0000","last_became_peered":"2023-06-03T19:56:54.507149+0000","last_unstale":"2023-06-03T20:03:24.788553+0000","last_undegraded":"2023-06-03T20:03:24.788553+0000","last_fullsized":"2023-06-03T20:03:24.788553+0000","mapping_epoch":119,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T20:22:36.676317+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":108,"reported_epoch":141,"state":"active+clean","last_fresh":"2023-06-03T20:03:23.859393+0000","last_change":"2023-06-03T19:56:54.512472+0000","last_active":"2023-06-03T20:03:23.859393+0000","last_peered":"2023-06-03T20:03:23.859393+0000","last_clean":"2023-06-03T20:03:23.859393+0000","last_became_active":"2023-06-03T19:56:54.512051+0000","last_became_peered":"2023-06-03T19:56:54.512051+0000","last_unstale":"2023-06-03T20:03:23.859393+0000","last_undegraded":"2023-06-03T20:03:23.859393+0000","last_fullsized":"2023-06-03T20:03:23.859393+0000","mapping_epoch":119,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":109,"reported_epoch":141,"state":"active+clean","last_fresh":"2023-06-03T20:03:23.850582+0000","last_change":"2023-06-03T19:57:28.462780+0000","last_active":"2023-06-03T20:03:23.850582+0000","last_peered":"2023-06-03T20:03:23.850582+0000","last_clean":"2023-06-03T20:03:23.850582+0000","last_became_active":"2023-06-03T19:57:28.462298+0000","last_became_peered":"2023-06-03T19:57:28.462298+0000","last_unstale":"2023-06-03T20:03:23.850582+0000","last_undegraded":"2023-06-03T20:03:23.850582+0000","last_fullsized":"2023-06-03T20:03:23.850582+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":98,"reported_epoch":141,"state":"active+clean","last_fresh":"2023-06-03T20:03:23.860840+0000","last_change":"2023-06-03T19:56:54.510289+0000","last_active":"2023-06-03T20:03:23.860840+0000","last_peered":"2023-06-03T20:03:23.860840+0000","last_clean":"2023-06-03T20:03:23.860840+0000","last_became_active":"2023-06-03T19:56:54.509905+0000","last_became_peered":"2023-06-03T19:56:54.509905+0000","last_unstale":"2023-06-03T20:03:23.860840+0000","last_undegraded":"2023-06-03T20:03:23.860840+0000","last_fullsized":"2023-06-03T20:03:23.860840+0000","mapping_epoch":119,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T02:54:26.792401+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"60'8","reported_seq":117,"reported_epoch":141,"state":"active+clean","last_fresh":"2023-06-03T20:03:23.870375+0000","last_change":"2023-06-03T19:47:06.727778+0000","last_active":"2023-06-03T20:03:23.870375+0000","last_peered":"2023-06-03T20:03:23.870375+0000","last_clean":"2023-06-03T20:03:23.870375+0000","last_became_active":"2023-06-03T19:47:06.727305+0000","last_became_peered":"2023-06-03T19:47:06.727305+0000","last_unstale":"2023-06-03T20:03:23.870375+0000","last_undegraded":"2023-06-03T20:03:23.870375+0000","last_fullsized":"2023-06-03T20:03:23.870375+0000","mapping_epoch":86,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":87,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":107,"reported_epoch":141,"state":"active+clean","last_fresh":"2023-06-03T20:03:23.850081+0000","last_change":"2023-06-03T19:56:54.512994+0000","last_active":"2023-06-03T20:03:23.850081+0000","last_peered":"2023-06-03T20:03:23.850081+0000","last_clean":"2023-06-03T20:03:23.850081+0000","last_became_active":"2023-06-03T19:56:54.512579+0000","last_became_peered":"2023-06-03T19:56:54.512579+0000","last_unstale":"2023-06-03T20:03:23.850081+0000","last_undegraded":"2023-06-03T20:03:23.850081+0000","last_fullsized":"2023-06-03T20:03:23.850081+0000","mapping_epoch":119,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T07:02:45.187961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"9.5","version":"142'10750","reported_seq":10775,"reported_epoch":142,"state":"active+clean","last_fresh":"2023-06-03T20:03:39.820466+0000","last_change":"2023-06-03T20:00:12.908073+0000","last_active":"2023-06-03T20:03:39.820466+0000","last_peered":"2023-06-03T20:03:39.820466+0000","last_clean":"2023-06-03T20:03:39.820466+0000","last_became_active":"2023-06-03T20:00:12.907738+0000","last_became_peered":"2023-06-03T20:00:12.907738+0000","last_unstale":"2023-06-03T20:03:39.820466+0000","last_undegraded":"2023-06-03T20:03:39.820466+0000","last_fullsized":"2023-06-03T20:03:39.820466+0000","mapping_epoch":127,"log_start":"131'700","ondisk_log_start":"131'700","created":127,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T20:00:11.873149+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T20:00:11.873149+0000","last_clean_scrub_stamp":"2023-06-03T20:00:11.873149+0000","objects_scrubbed":0,"log_size":10050,"log_dups_size":0,"ondisk_log_size":10050,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T02:31:40.900210+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":185335808,"num_objects":2828,"num_object_clones":0,"num_object_copies":8484,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2828,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10750,"num_write_kb":434496,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.f","version":"60'1","reported_seq":125,"reported_epoch":141,"state":"active+clean","last_fresh":"2023-06-03T20:03:23.851651+0000","last_change":"2023-06-03T19:57:28.464482+0000","last_active":"2023-06-03T20:03:23.851651+0000","last_peered":"2023-06-03T20:03:23.851651+0000","last_clean":"2023-06-03T20:03:23.851651+0000","last_became_active":"2023-06-03T19:57:28.464081+0000","last_became_peered":"2023-06-03T19:57:28.464081+0000","last_unstale":"2023-06-03T20:03:23.851651+0000","last_undegraded":"2023-06-03T20:03:23.851651+0000","last_fullsized":"2023-06-03T20:03:23.851651+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":93,"reported_epoch":141,"state":"active+clean","last_fresh":"2023-06-03T20:03:23.855935+0000","last_change":"2023-06-03T19:34:17.848358+0000","last_active":"2023-06-03T20:03:23.855935+0000","last_peered":"2023-06-03T20:03:23.855935+0000","last_clean":"2023-06-03T20:03:23.855935+0000","last_became_active":"2023-06-03T19:34:17.847811+0000","last_became_peered":"2023-06-03T19:34:17.847811+0000","last_unstale":"2023-06-03T20:03:23.855935+0000","last_undegraded":"2023-06-03T20:03:23.855935+0000","last_fullsized":"2023-06-03T20:03:23.855935+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"9.4","version":"142'10970","reported_seq":10994,"reported_epoch":142,"state":"active+clean","last_fresh":"2023-06-03T20:03:38.556073+0000","last_change":"2023-06-03T20:00:12.905941+0000","last_active":"2023-06-03T20:03:38.556073+0000","last_peered":"2023-06-03T20:03:38.556073+0000","last_clean":"2023-06-03T20:03:38.556073+0000","last_became_active":"2023-06-03T20:00:12.905661+0000","last_became_peered":"2023-06-03T20:00:12.905661+0000","last_unstale":"2023-06-03T20:03:38.556073+0000","last_undegraded":"2023-06-03T20:03:38.556073+0000","last_fullsized":"2023-06-03T20:03:38.556073+0000","mapping_epoch":127,"log_start":"133'3600","ondisk_log_start":"133'3600","created":127,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T20:00:11.873149+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T20:00:11.873149+0000","last_clean_scrub_stamp":"2023-06-03T20:00:11.873149+0000","objects_scrubbed":0,"log_size":7370,"log_dups_size":0,"ondisk_log_size":7370,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T06:19:01.942119+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":192282624,"num_objects":2934,"num_object_clones":0,"num_object_copies":8802,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2934,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10970,"num_write_kb":444928,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.e","version":"60'2","reported_seq":126,"reported_epoch":141,"state":"active+clean","last_fresh":"2023-06-03T20:03:23.851322+0000","last_change":"2023-06-03T19:57:28.468330+0000","last_active":"2023-06-03T20:03:23.851322+0000","last_peered":"2023-06-03T20:03:23.851322+0000","last_clean":"2023-06-03T20:03:23.851322+0000","last_became_active":"2023-06-03T19:57:28.467823+0000","last_became_peered":"2023-06-03T19:57:28.467823+0000","last_unstale":"2023-06-03T20:03:23.851322+0000","last_undegraded":"2023-06-03T20:03:23.851322+0000","last_fullsized":"2023-06-03T20:03:23.851322+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":140,"reported_epoch":141,"state":"active+clean","last_fresh":"2023-06-03T20:03:23.844995+0000","last_change":"2023-06-03T19:57:28.461449+0000","last_active":"2023-06-03T20:03:23.844995+0000","last_peered":"2023-06-03T20:03:23.844995+0000","last_clean":"2023-06-03T20:03:23.844995+0000","last_became_active":"2023-06-03T19:57:28.460745+0000","last_became_peered":"2023-06-03T19:57:28.460745+0000","last_unstale":"2023-06-03T20:03:23.844995+0000","last_undegraded":"2023-06-03T20:03:23.844995+0000","last_fullsized":"2023-06-03T20:03:23.844995+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T20:41:19.361638+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,3],"acting":[4,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.7","version":"142'11123","reported_seq":11146,"reported_epoch":142,"state":"active+clean","last_fresh":"2023-06-03T20:03:38.220008+0000","last_change":"2023-06-03T20:00:12.909354+0000","last_active":"2023-06-03T20:03:38.220008+0000","last_peered":"2023-06-03T20:03:38.220008+0000","last_clean":"2023-06-03T20:03:38.220008+0000","last_became_active":"2023-06-03T20:00:12.908971+0000","last_became_peered":"2023-06-03T20:00:12.908971+0000","last_unstale":"2023-06-03T20:03:38.220008+0000","last_undegraded":"2023-06-03T20:03:38.220008+0000","last_fullsized":"2023-06-03T20:03:38.220008+0000","mapping_epoch":127,"log_start":"131'1100","ondisk_log_start":"131'1100","created":127,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T20:00:11.873149+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T20:00:11.873149+0000","last_clean_scrub_stamp":"2023-06-03T20:00:11.873149+0000","objects_scrubbed":0,"log_size":10023,"log_dups_size":0,"ondisk_log_size":10023,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":192348160,"num_objects":2935,"num_object_clones":0,"num_object_copies":8805,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2935,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11123,"num_write_kb":449856,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.d","version":"60'1","reported_seq":95,"reported_epoch":141,"state":"active+clean","last_fresh":"2023-06-03T20:03:23.856363+0000","last_change":"2023-06-03T19:34:16.806559+0000","last_active":"2023-06-03T20:03:23.856363+0000","last_peered":"2023-06-03T20:03:23.856363+0000","last_clean":"2023-06-03T20:03:23.856363+0000","last_became_active":"2023-06-03T19:34:16.806149+0000","last_became_peered":"2023-06-03T19:34:16.806149+0000","last_unstale":"2023-06-03T20:03:23.856363+0000","last_undegraded":"2023-06-03T20:03:23.856363+0000","last_fullsized":"2023-06-03T20:03:23.856363+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":108,"reported_epoch":141,"state":"active+clean","last_fresh":"2023-06-03T20:03:23.870636+0000","last_change":"2023-06-03T19:51:59.360650+0000","last_active":"2023-06-03T20:03:23.870636+0000","last_peered":"2023-06-03T20:03:23.870636+0000","last_clean":"2023-06-03T20:03:23.870636+0000","last_became_active":"2023-06-03T19:51:59.360237+0000","last_became_peered":"2023-06-03T19:51:59.360237+0000","last_unstale":"2023-06-03T20:03:23.870636+0000","last_undegraded":"2023-06-03T20:03:23.870636+0000","last_fullsized":"2023-06-03T20:03:23.870636+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"9.6","version":"142'10691","reported_seq":10713,"reported_epoch":142,"state":"active+clean","last_fresh":"2023-06-03T20:03:38.244079+0000","last_change":"2023-06-03T20:00:13.172391+0000","last_active":"2023-06-03T20:03:38.244079+0000","last_peered":"2023-06-03T20:03:38.244079+0000","last_clean":"2023-06-03T20:03:38.244079+0000","last_became_active":"2023-06-03T20:00:13.171568+0000","last_became_peered":"2023-06-03T20:00:13.171568+0000","last_unstale":"2023-06-03T20:03:38.244079+0000","last_undegraded":"2023-06-03T20:03:38.244079+0000","last_fullsized":"2023-06-03T20:03:38.244079+0000","mapping_epoch":127,"log_start":"131'600","ondisk_log_start":"131'600","created":127,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T20:00:11.873149+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T20:00:11.873149+0000","last_clean_scrub_stamp":"2023-06-03T20:00:11.873149+0000","objects_scrubbed":0,"log_size":10091,"log_dups_size":0,"ondisk_log_size":10091,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":188940288,"num_objects":2883,"num_object_clones":0,"num_object_copies":8649,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2883,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10691,"num_write_kb":434368,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,1],"acting":[5,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":109,"reported_epoch":141,"state":"active+clean","last_fresh":"2023-06-03T20:03:23.857109+0000","last_change":"2023-06-03T19:56:54.507432+0000","last_active":"2023-06-03T20:03:23.857109+0000","last_peered":"2023-06-03T20:03:23.857109+0000","last_clean":"2023-06-03T20:03:23.857109+0000","last_became_active":"2023-06-03T19:56:54.507012+0000","last_became_peered":"2023-06-03T19:56:54.507012+0000","last_unstale":"2023-06-03T20:03:23.857109+0000","last_undegraded":"2023-06-03T20:03:23.857109+0000","last_fullsized":"2023-06-03T20:03:23.857109+0000","mapping_epoch":119,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":126,"reported_epoch":141,"state":"active+clean","last_fresh":"2023-06-03T20:03:23.860663+0000","last_change":"2023-06-03T19:57:28.464153+0000","last_active":"2023-06-03T20:03:23.860663+0000","last_peered":"2023-06-03T20:03:23.860663+0000","last_clean":"2023-06-03T20:03:23.860663+0000","last_became_active":"2023-06-03T19:57:28.463502+0000","last_became_peered":"2023-06-03T19:57:28.463502+0000","last_unstale":"2023-06-03T20:03:23.860663+0000","last_undegraded":"2023-06-03T20:03:23.860663+0000","last_fullsized":"2023-06-03T20:03:23.860663+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T06:11:00.999526+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,4],"acting":[0,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.1","version":"142'10918","reported_seq":10943,"reported_epoch":142,"state":"active+clean","last_fresh":"2023-06-03T20:03:38.036183+0000","last_change":"2023-06-03T20:00:12.905059+0000","last_active":"2023-06-03T20:03:38.036183+0000","last_peered":"2023-06-03T20:03:38.036183+0000","last_clean":"2023-06-03T20:03:38.036183+0000","last_became_active":"2023-06-03T20:00:12.904765+0000","last_became_peered":"2023-06-03T20:00:12.904765+0000","last_unstale":"2023-06-03T20:03:38.036183+0000","last_undegraded":"2023-06-03T20:03:38.036183+0000","last_fullsized":"2023-06-03T20:03:38.036183+0000","mapping_epoch":127,"log_start":"132'3200","ondisk_log_start":"132'3200","created":127,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T20:00:11.873149+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T20:00:11.873149+0000","last_clean_scrub_stamp":"2023-06-03T20:00:11.873149+0000","objects_scrubbed":0,"log_size":7718,"log_dups_size":0,"ondisk_log_size":7718,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T07:39:39.205403+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":189399040,"num_objects":2890,"num_object_clones":0,"num_object_copies":8670,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2890,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10918,"num_write_kb":441856,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,7],"acting":[3,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.b","version":"60'2","reported_seq":109,"reported_epoch":142,"state":"active+clean","last_fresh":"2023-06-03T20:03:28.906220+0000","last_change":"2023-06-03T19:57:28.470310+0000","last_active":"2023-06-03T20:03:28.906220+0000","last_peered":"2023-06-03T20:03:28.906220+0000","last_clean":"2023-06-03T20:03:28.906220+0000","last_became_active":"2023-06-03T19:57:28.469887+0000","last_became_peered":"2023-06-03T19:57:28.469887+0000","last_unstale":"2023-06-03T20:03:28.906220+0000","last_undegraded":"2023-06-03T20:03:28.906220+0000","last_fullsized":"2023-06-03T20:03:28.906220+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T21:58:53.387131+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":116,"reported_epoch":141,"state":"active+clean","last_fresh":"2023-06-03T20:03:23.845813+0000","last_change":"2023-06-03T19:57:28.465140+0000","last_active":"2023-06-03T20:03:23.845813+0000","last_peered":"2023-06-03T20:03:23.845813+0000","last_clean":"2023-06-03T20:03:23.845813+0000","last_became_active":"2023-06-03T19:57:28.464629+0000","last_became_peered":"2023-06-03T19:57:28.464629+0000","last_unstale":"2023-06-03T20:03:23.845813+0000","last_undegraded":"2023-06-03T20:03:23.845813+0000","last_fullsized":"2023-06-03T20:03:23.845813+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T21:47:11.619583+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.0","version":"142'10727","reported_seq":10752,"reported_epoch":142,"state":"active+clean","last_fresh":"2023-06-03T20:03:38.526771+0000","last_change":"2023-06-03T20:00:13.174012+0000","last_active":"2023-06-03T20:03:38.526771+0000","last_peered":"2023-06-03T20:03:38.526771+0000","last_clean":"2023-06-03T20:03:38.526771+0000","last_became_active":"2023-06-03T20:00:13.173630+0000","last_became_peered":"2023-06-03T20:00:13.173630+0000","last_unstale":"2023-06-03T20:03:38.526771+0000","last_undegraded":"2023-06-03T20:03:38.526771+0000","last_fullsized":"2023-06-03T20:03:38.526771+0000","mapping_epoch":127,"log_start":"133'3400","ondisk_log_start":"133'3400","created":127,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T20:00:11.873149+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T20:00:11.873149+0000","last_clean_scrub_stamp":"2023-06-03T20:00:11.873149+0000","objects_scrubbed":0,"log_size":7327,"log_dups_size":0,"ondisk_log_size":7327,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T23:14:07.402519+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":191954944,"num_objects":2929,"num_object_clones":0,"num_object_copies":8787,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2929,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10727,"num_write_kb":436992,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":124,"reported_epoch":141,"state":"active+clean","last_fresh":"2023-06-03T20:03:23.870544+0000","last_change":"2023-06-03T19:57:28.461401+0000","last_active":"2023-06-03T20:03:23.870544+0000","last_peered":"2023-06-03T20:03:23.870544+0000","last_clean":"2023-06-03T20:03:23.870544+0000","last_became_active":"2023-06-03T19:57:28.461057+0000","last_became_peered":"2023-06-03T19:57:28.461057+0000","last_unstale":"2023-06-03T20:03:23.870544+0000","last_undegraded":"2023-06-03T20:03:23.870544+0000","last_fullsized":"2023-06-03T20:03:23.870544+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":121,"reported_epoch":141,"state":"active+clean","last_fresh":"2023-06-03T20:03:23.845670+0000","last_change":"2023-06-03T19:57:28.456750+0000","last_active":"2023-06-03T20:03:23.845670+0000","last_peered":"2023-06-03T20:03:23.845670+0000","last_clean":"2023-06-03T20:03:23.845670+0000","last_became_active":"2023-06-03T19:57:28.456436+0000","last_became_peered":"2023-06-03T19:57:28.456436+0000","last_unstale":"2023-06-03T20:03:23.845670+0000","last_undegraded":"2023-06-03T20:03:23.845670+0000","last_fullsized":"2023-06-03T20:03:23.845670+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T01:49:25.481554+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,5],"acting":[4,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.3","version":"142'10717","reported_seq":10740,"reported_epoch":142,"state":"active+clean","last_fresh":"2023-06-03T20:03:38.244366+0000","last_change":"2023-06-03T20:00:12.903729+0000","last_active":"2023-06-03T20:03:38.244366+0000","last_peered":"2023-06-03T20:03:38.244366+0000","last_clean":"2023-06-03T20:03:38.244366+0000","last_became_active":"2023-06-03T20:00:12.903378+0000","last_became_peered":"2023-06-03T20:00:12.903378+0000","last_unstale":"2023-06-03T20:03:38.244366+0000","last_undegraded":"2023-06-03T20:03:38.244366+0000","last_fullsized":"2023-06-03T20:03:38.244366+0000","mapping_epoch":127,"log_start":"132'3000","ondisk_log_start":"132'3000","created":127,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T20:00:11.873149+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T20:00:11.873149+0000","last_clean_scrub_stamp":"2023-06-03T20:00:11.873149+0000","objects_scrubbed":0,"log_size":7717,"log_dups_size":0,"ondisk_log_size":7717,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":191168512,"num_objects":2917,"num_object_clones":0,"num_object_copies":8751,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2917,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10717,"num_write_kb":436288,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,1],"acting":[7,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":113,"reported_epoch":141,"state":"active+clean","last_fresh":"2023-06-03T20:03:23.844733+0000","last_change":"2023-06-03T19:57:28.462344+0000","last_active":"2023-06-03T20:03:23.844733+0000","last_peered":"2023-06-03T20:03:23.844733+0000","last_clean":"2023-06-03T20:03:23.844733+0000","last_became_active":"2023-06-03T19:57:28.462001+0000","last_became_peered":"2023-06-03T19:57:28.462001+0000","last_unstale":"2023-06-03T20:03:23.844733+0000","last_undegraded":"2023-06-03T20:03:23.844733+0000","last_fullsized":"2023-06-03T20:03:23.844733+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T00:45:13.789650+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":102,"reported_epoch":141,"state":"active+clean","last_fresh":"2023-06-03T20:03:23.846164+0000","last_change":"2023-06-03T19:57:28.467917+0000","last_active":"2023-06-03T20:03:23.846164+0000","last_peered":"2023-06-03T20:03:23.846164+0000","last_clean":"2023-06-03T20:03:23.846164+0000","last_became_active":"2023-06-03T19:57:28.467617+0000","last_became_peered":"2023-06-03T19:57:28.467617+0000","last_unstale":"2023-06-03T20:03:23.846164+0000","last_undegraded":"2023-06-03T20:03:23.846164+0000","last_fullsized":"2023-06-03T20:03:23.846164+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T23:34:23.840582+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.2","version":"142'10928","reported_seq":10953,"reported_epoch":142,"state":"active+clean","last_fresh":"2023-06-03T20:03:37.978734+0000","last_change":"2023-06-03T20:00:13.173625+0000","last_active":"2023-06-03T20:03:37.978734+0000","last_peered":"2023-06-03T20:03:37.978734+0000","last_clean":"2023-06-03T20:03:37.978734+0000","last_became_active":"2023-06-03T20:00:13.173348+0000","last_became_peered":"2023-06-03T20:00:13.173348+0000","last_unstale":"2023-06-03T20:03:37.978734+0000","last_undegraded":"2023-06-03T20:03:37.978734+0000","last_fullsized":"2023-06-03T20:03:37.978734+0000","mapping_epoch":127,"log_start":"132'3200","ondisk_log_start":"132'3200","created":127,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T20:00:11.873149+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T20:00:11.873149+0000","last_clean_scrub_stamp":"2023-06-03T20:00:11.873149+0000","objects_scrubbed":0,"log_size":7728,"log_dups_size":0,"ondisk_log_size":7728,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T03:41:17.261687+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":185860096,"num_objects":2836,"num_object_clones":0,"num_object_copies":8508,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2836,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10928,"num_write_kb":440448,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":109,"reported_epoch":141,"state":"active+clean","last_fresh":"2023-06-03T20:03:23.849548+0000","last_change":"2023-06-03T19:56:54.512219+0000","last_active":"2023-06-03T20:03:23.849548+0000","last_peered":"2023-06-03T20:03:23.849548+0000","last_clean":"2023-06-03T20:03:23.849548+0000","last_became_active":"2023-06-03T19:56:54.511771+0000","last_became_peered":"2023-06-03T19:56:54.511771+0000","last_unstale":"2023-06-03T20:03:23.849548+0000","last_undegraded":"2023-06-03T20:03:23.849548+0000","last_fullsized":"2023-06-03T20:03:23.849548+0000","mapping_epoch":119,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T19:35:47.796520+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":95,"reported_epoch":141,"state":"active+clean","last_fresh":"2023-06-03T20:03:23.851330+0000","last_change":"2023-06-03T19:57:28.471023+0000","last_active":"2023-06-03T20:03:23.851330+0000","last_peered":"2023-06-03T20:03:23.851330+0000","last_clean":"2023-06-03T20:03:23.851330+0000","last_became_active":"2023-06-03T19:57:28.470452+0000","last_became_peered":"2023-06-03T19:57:28.470452+0000","last_unstale":"2023-06-03T20:03:23.851330+0000","last_undegraded":"2023-06-03T20:03:23.851330+0000","last_fullsized":"2023-06-03T20:03:23.851330+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T05:58:33.305030+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.f","version":"142'10639","reported_seq":10662,"reported_epoch":142,"state":"active+clean","last_fresh":"2023-06-03T20:03:38.208801+0000","last_change":"2023-06-03T20:00:12.909234+0000","last_active":"2023-06-03T20:03:38.208801+0000","last_peered":"2023-06-03T20:03:38.208801+0000","last_clean":"2023-06-03T20:03:38.208801+0000","last_became_active":"2023-06-03T20:00:12.908871+0000","last_became_peered":"2023-06-03T20:00:12.908871+0000","last_unstale":"2023-06-03T20:03:38.208801+0000","last_undegraded":"2023-06-03T20:03:38.208801+0000","last_fullsized":"2023-06-03T20:03:38.208801+0000","mapping_epoch":127,"log_start":"131'600","ondisk_log_start":"131'600","created":127,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T20:00:11.873149+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T20:00:11.873149+0000","last_clean_scrub_stamp":"2023-06-03T20:00:11.873149+0000","objects_scrubbed":0,"log_size":10039,"log_dups_size":0,"ondisk_log_size":10039,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":189726720,"num_objects":2895,"num_object_clones":0,"num_object_copies":8685,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2895,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10639,"num_write_kb":433088,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":113,"reported_epoch":141,"state":"active+clean","last_fresh":"2023-06-03T20:03:23.857380+0000","last_change":"2023-06-03T19:51:59.392796+0000","last_active":"2023-06-03T20:03:23.857380+0000","last_peered":"2023-06-03T20:03:23.857380+0000","last_clean":"2023-06-03T20:03:23.857380+0000","last_became_active":"2023-06-03T19:51:59.380207+0000","last_became_peered":"2023-06-03T19:51:59.380207+0000","last_unstale":"2023-06-03T20:03:23.857380+0000","last_undegraded":"2023-06-03T20:03:23.857380+0000","last_fullsized":"2023-06-03T20:03:23.857380+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:32:20.692043+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:32:20.692043+0000","last_clean_scrub_stamp":"2023-06-03T19:32:20.692043+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T00:28:05.866873+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":124,"reported_epoch":141,"state":"active+clean","last_fresh":"2023-06-03T20:03:23.859271+0000","last_change":"2023-06-03T19:56:54.512607+0000","last_active":"2023-06-03T20:03:23.859271+0000","last_peered":"2023-06-03T20:03:23.859271+0000","last_clean":"2023-06-03T20:03:23.859271+0000","last_became_active":"2023-06-03T19:56:54.512282+0000","last_became_peered":"2023-06-03T19:56:54.512282+0000","last_unstale":"2023-06-03T20:03:23.859271+0000","last_undegraded":"2023-06-03T20:03:23.859271+0000","last_fullsized":"2023-06-03T20:03:23.859271+0000","mapping_epoch":119,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":80,"reported_epoch":141,"state":"active+clean","last_fresh":"2023-06-03T20:03:23.861298+0000","last_change":"2023-06-03T19:57:28.470295+0000","last_active":"2023-06-03T20:03:23.861298+0000","last_peered":"2023-06-03T20:03:23.861298+0000","last_clean":"2023-06-03T20:03:23.861298+0000","last_became_active":"2023-06-03T19:57:28.469383+0000","last_became_peered":"2023-06-03T19:57:28.469383+0000","last_unstale":"2023-06-03T20:03:23.861298+0000","last_undegraded":"2023-06-03T20:03:23.861298+0000","last_fullsized":"2023-06-03T20:03:23.861298+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T05:35:19.667447+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.9","version":"142'10945","reported_seq":10970,"reported_epoch":142,"state":"active+clean","last_fresh":"2023-06-03T20:03:39.851366+0000","last_change":"2023-06-03T20:00:12.910626+0000","last_active":"2023-06-03T20:03:39.851366+0000","last_peered":"2023-06-03T20:03:39.851366+0000","last_clean":"2023-06-03T20:03:39.851366+0000","last_became_active":"2023-06-03T20:00:12.910285+0000","last_became_peered":"2023-06-03T20:00:12.910285+0000","last_unstale":"2023-06-03T20:03:39.851366+0000","last_undegraded":"2023-06-03T20:03:39.851366+0000","last_fullsized":"2023-06-03T20:03:39.851366+0000","mapping_epoch":127,"log_start":"131'900","ondisk_log_start":"131'900","created":127,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T20:00:11.873149+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T20:00:11.873149+0000","last_clean_scrub_stamp":"2023-06-03T20:00:11.873149+0000","objects_scrubbed":0,"log_size":10045,"log_dups_size":0,"ondisk_log_size":10045,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T04:42:34.630759+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":188809216,"num_objects":2881,"num_object_clones":0,"num_object_copies":8643,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2881,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10945,"num_write_kb":442432,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,5],"acting":[0,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":113,"reported_epoch":141,"state":"active+clean","last_fresh":"2023-06-03T20:03:23.856317+0000","last_change":"2023-06-03T19:51:59.373777+0000","last_active":"2023-06-03T20:03:23.856317+0000","last_peered":"2023-06-03T20:03:23.856317+0000","last_clean":"2023-06-03T20:03:23.856317+0000","last_became_active":"2023-06-03T19:51:59.372988+0000","last_became_peered":"2023-06-03T19:51:59.372988+0000","last_unstale":"2023-06-03T20:03:23.856317+0000","last_undegraded":"2023-06-03T20:03:23.856317+0000","last_fullsized":"2023-06-03T20:03:23.856317+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:32:20.692043+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:32:20.692043+0000","last_clean_scrub_stamp":"2023-06-03T19:32:20.692043+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":87,"reported_epoch":142,"state":"active+clean","last_fresh":"2023-06-03T20:03:28.895734+0000","last_change":"2023-06-03T19:57:28.460195+0000","last_active":"2023-06-03T20:03:28.895734+0000","last_peered":"2023-06-03T20:03:28.895734+0000","last_clean":"2023-06-03T20:03:28.895734+0000","last_became_active":"2023-06-03T19:57:28.459823+0000","last_became_peered":"2023-06-03T19:57:28.459823+0000","last_unstale":"2023-06-03T20:03:28.895734+0000","last_undegraded":"2023-06-03T20:03:28.895734+0000","last_fullsized":"2023-06-03T20:03:28.895734+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T20:08:48.686380+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":95,"reported_epoch":141,"state":"active+clean","last_fresh":"2023-06-03T20:03:23.857341+0000","last_change":"2023-06-03T19:56:54.516859+0000","last_active":"2023-06-03T20:03:23.857341+0000","last_peered":"2023-06-03T20:03:23.857341+0000","last_clean":"2023-06-03T20:03:23.857341+0000","last_became_active":"2023-06-03T19:56:54.516521+0000","last_became_peered":"2023-06-03T19:56:54.516521+0000","last_unstale":"2023-06-03T20:03:23.857341+0000","last_undegraded":"2023-06-03T20:03:23.857341+0000","last_fullsized":"2023-06-03T20:03:23.857341+0000","mapping_epoch":119,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T20:03:25.227382+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.a","version":"142'10874","reported_seq":10897,"reported_epoch":142,"state":"active+clean","last_fresh":"2023-06-03T20:03:38.235972+0000","last_change":"2023-06-03T20:00:12.909720+0000","last_active":"2023-06-03T20:03:38.235972+0000","last_peered":"2023-06-03T20:03:38.235972+0000","last_clean":"2023-06-03T20:03:38.235972+0000","last_became_active":"2023-06-03T20:00:12.909428+0000","last_became_peered":"2023-06-03T20:00:12.909428+0000","last_unstale":"2023-06-03T20:03:38.235972+0000","last_undegraded":"2023-06-03T20:03:38.235972+0000","last_fullsized":"2023-06-03T20:03:38.235972+0000","mapping_epoch":127,"log_start":"131'800","ondisk_log_start":"131'800","created":127,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T20:00:11.873149+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T20:00:11.873149+0000","last_clean_scrub_stamp":"2023-06-03T20:00:11.873149+0000","objects_scrubbed":0,"log_size":10074,"log_dups_size":0,"ondisk_log_size":10074,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":189923328,"num_objects":2898,"num_object_clones":0,"num_object_copies":8694,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2898,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10874,"num_write_kb":440704,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,7],"acting":[5,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":116,"reported_epoch":141,"state":"active+clean","last_fresh":"2023-06-03T20:03:23.866239+0000","last_change":"2023-06-03T19:56:54.504859+0000","last_active":"2023-06-03T20:03:23.866239+0000","last_peered":"2023-06-03T20:03:23.866239+0000","last_clean":"2023-06-03T20:03:23.866239+0000","last_became_active":"2023-06-03T19:56:54.504455+0000","last_became_peered":"2023-06-03T19:56:54.504455+0000","last_unstale":"2023-06-03T20:03:23.866239+0000","last_undegraded":"2023-06-03T20:03:23.866239+0000","last_fullsized":"2023-06-03T20:03:23.866239+0000","mapping_epoch":119,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:32:20.692043+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:32:20.692043+0000","last_clean_scrub_stamp":"2023-06-03T19:32:20.692043+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T06:25:45.522100+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":121,"reported_epoch":141,"state":"active+clean","last_fresh":"2023-06-03T20:03:23.851135+0000","last_change":"2023-06-03T19:56:11.754048+0000","last_active":"2023-06-03T20:03:23.851135+0000","last_peered":"2023-06-03T20:03:23.851135+0000","last_clean":"2023-06-03T20:03:23.851135+0000","last_became_active":"2023-06-03T19:56:11.753687+0000","last_became_peered":"2023-06-03T19:56:11.753687+0000","last_unstale":"2023-06-03T20:03:23.851135+0000","last_undegraded":"2023-06-03T20:03:23.851135+0000","last_fullsized":"2023-06-03T20:03:23.851135+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":115,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T23:29:41.511496+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":108,"reported_epoch":141,"state":"active+clean","last_fresh":"2023-06-03T20:03:23.851434+0000","last_change":"2023-06-03T19:51:59.396573+0000","last_active":"2023-06-03T20:03:23.851434+0000","last_peered":"2023-06-03T20:03:23.851434+0000","last_clean":"2023-06-03T20:03:23.851434+0000","last_became_active":"2023-06-03T19:51:59.381593+0000","last_became_peered":"2023-06-03T19:51:59.381593+0000","last_unstale":"2023-06-03T20:03:23.851434+0000","last_undegraded":"2023-06-03T20:03:23.851434+0000","last_fullsized":"2023-06-03T20:03:23.851434+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T21:33:23.043772+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.8","version":"142'10809","reported_seq":10834,"reported_epoch":142,"state":"active+clean","last_fresh":"2023-06-03T20:03:38.061562+0000","last_change":"2023-06-03T20:00:12.908817+0000","last_active":"2023-06-03T20:03:38.061562+0000","last_peered":"2023-06-03T20:03:38.061562+0000","last_clean":"2023-06-03T20:03:38.061562+0000","last_became_active":"2023-06-03T20:00:12.908431+0000","last_became_peered":"2023-06-03T20:00:12.908431+0000","last_unstale":"2023-06-03T20:03:38.061562+0000","last_undegraded":"2023-06-03T20:03:38.061562+0000","last_fullsized":"2023-06-03T20:03:38.061562+0000","mapping_epoch":127,"log_start":"132'3100","ondisk_log_start":"132'3100","created":127,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T20:00:11.873149+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T20:00:11.873149+0000","last_clean_scrub_stamp":"2023-06-03T20:00:11.873149+0000","objects_scrubbed":0,"log_size":7709,"log_dups_size":0,"ondisk_log_size":7709,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T21:11:15.734132+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":190513152,"num_objects":2907,"num_object_clones":0,"num_object_copies":8721,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2907,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10809,"num_write_kb":438912,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":113,"reported_epoch":141,"state":"active+clean","last_fresh":"2023-06-03T20:03:23.857259+0000","last_change":"2023-06-03T19:56:11.752151+0000","last_active":"2023-06-03T20:03:23.857259+0000","last_peered":"2023-06-03T20:03:23.857259+0000","last_clean":"2023-06-03T20:03:23.857259+0000","last_became_active":"2023-06-03T19:56:11.751767+0000","last_became_peered":"2023-06-03T19:56:11.751767+0000","last_unstale":"2023-06-03T20:03:23.857259+0000","last_undegraded":"2023-06-03T20:03:23.857259+0000","last_fullsized":"2023-06-03T20:03:23.857259+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":115,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:32:20.692043+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:32:20.692043+0000","last_clean_scrub_stamp":"2023-06-03T19:32:20.692043+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"72'139","reported_seq":345,"reported_epoch":141,"state":"active+clean","last_fresh":"2023-06-03T20:03:23.850047+0000","last_change":"2023-06-03T19:47:06.731872+0000","last_active":"2023-06-03T20:03:23.850047+0000","last_peered":"2023-06-03T20:03:23.850047+0000","last_clean":"2023-06-03T20:03:23.850047+0000","last_became_active":"2023-06-03T19:47:06.731175+0000","last_became_peered":"2023-06-03T19:47:06.731175+0000","last_unstale":"2023-06-03T20:03:23.850047+0000","last_undegraded":"2023-06-03T20:03:23.850047+0000","last_fullsized":"2023-06-03T20:03:23.850047+0000","mapping_epoch":86,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":87,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:27:28.100808+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:27:28.100808+0000","last_clean_scrub_stamp":"2023-06-03T19:27:28.100808+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":89,"reported_epoch":141,"state":"active+clean","last_fresh":"2023-06-03T20:03:23.849673+0000","last_change":"2023-06-03T19:56:54.509571+0000","last_active":"2023-06-03T20:03:23.849673+0000","last_peered":"2023-06-03T20:03:23.849673+0000","last_clean":"2023-06-03T20:03:23.849673+0000","last_became_active":"2023-06-03T19:56:54.509213+0000","last_became_peered":"2023-06-03T19:56:54.509213+0000","last_unstale":"2023-06-03T20:03:23.849673+0000","last_undegraded":"2023-06-03T20:03:23.849673+0000","last_fullsized":"2023-06-03T20:03:23.849673+0000","mapping_epoch":119,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T21:26:56.003203+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":108,"reported_epoch":141,"state":"active+clean","last_fresh":"2023-06-03T20:03:23.870980+0000","last_change":"2023-06-03T19:57:28.459664+0000","last_active":"2023-06-03T20:03:23.870980+0000","last_peered":"2023-06-03T20:03:23.870980+0000","last_clean":"2023-06-03T20:03:23.870980+0000","last_became_active":"2023-06-03T19:57:28.459289+0000","last_became_peered":"2023-06-03T19:57:28.459289+0000","last_unstale":"2023-06-03T20:03:23.870980+0000","last_undegraded":"2023-06-03T20:03:23.870980+0000","last_fullsized":"2023-06-03T20:03:23.870980+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"9.b","version":"142'11100","reported_seq":11125,"reported_epoch":142,"state":"active+clean","last_fresh":"2023-06-03T20:03:39.872104+0000","last_change":"2023-06-03T20:00:12.908303+0000","last_active":"2023-06-03T20:03:39.872104+0000","last_peered":"2023-06-03T20:03:39.872104+0000","last_clean":"2023-06-03T20:03:39.872104+0000","last_became_active":"2023-06-03T20:00:12.907799+0000","last_became_peered":"2023-06-03T20:00:12.907799+0000","last_unstale":"2023-06-03T20:03:39.872104+0000","last_undegraded":"2023-06-03T20:03:39.872104+0000","last_fullsized":"2023-06-03T20:03:39.872104+0000","mapping_epoch":127,"log_start":"131'1000","ondisk_log_start":"131'1000","created":127,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T20:00:11.873149+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T20:00:11.873149+0000","last_clean_scrub_stamp":"2023-06-03T20:00:11.873149+0000","objects_scrubbed":0,"log_size":10100,"log_dups_size":0,"ondisk_log_size":10100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T23:11:45.966674+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":190054400,"num_objects":2900,"num_object_clones":0,"num_object_copies":8700,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2900,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11100,"num_write_kb":448000,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,5],"acting":[0,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":128,"reported_epoch":141,"state":"active+clean","last_fresh":"2023-06-03T20:03:23.850018+0000","last_change":"2023-06-03T19:51:59.358157+0000","last_active":"2023-06-03T20:03:23.850018+0000","last_peered":"2023-06-03T20:03:23.850018+0000","last_clean":"2023-06-03T20:03:23.850018+0000","last_became_active":"2023-06-03T19:51:59.357363+0000","last_became_peered":"2023-06-03T19:51:59.357363+0000","last_unstale":"2023-06-03T20:03:23.850018+0000","last_undegraded":"2023-06-03T20:03:23.850018+0000","last_fullsized":"2023-06-03T20:03:23.850018+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:32:20.692043+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:32:20.692043+0000","last_clean_scrub_stamp":"2023-06-03T19:32:20.692043+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":113,"reported_epoch":141,"state":"active+clean","last_fresh":"2023-06-03T20:03:23.862969+0000","last_change":"2023-06-03T19:57:28.459611+0000","last_active":"2023-06-03T20:03:23.862969+0000","last_peered":"2023-06-03T20:03:23.862969+0000","last_clean":"2023-06-03T20:03:23.862969+0000","last_became_active":"2023-06-03T19:57:28.459264+0000","last_became_peered":"2023-06-03T19:57:28.459264+0000","last_unstale":"2023-06-03T20:03:23.862969+0000","last_undegraded":"2023-06-03T20:03:23.862969+0000","last_fullsized":"2023-06-03T20:03:23.862969+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T22:15:06.557231+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":114,"reported_epoch":141,"state":"active+clean","last_fresh":"2023-06-03T20:03:23.860819+0000","last_change":"2023-06-03T19:56:11.745678+0000","last_active":"2023-06-03T20:03:23.860819+0000","last_peered":"2023-06-03T20:03:23.860819+0000","last_clean":"2023-06-03T20:03:23.860819+0000","last_became_active":"2023-06-03T19:56:11.744966+0000","last_became_peered":"2023-06-03T19:56:11.744966+0000","last_unstale":"2023-06-03T20:03:23.860819+0000","last_undegraded":"2023-06-03T20:03:23.860819+0000","last_fullsized":"2023-06-03T20:03:23.860819+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":115,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T21:12:06.830295+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.e","version":"142'10820","reported_seq":10843,"reported_epoch":142,"state":"active+clean","last_fresh":"2023-06-03T20:03:38.249610+0000","last_change":"2023-06-03T20:00:12.908248+0000","last_active":"2023-06-03T20:03:38.249610+0000","last_peered":"2023-06-03T20:03:38.249610+0000","last_clean":"2023-06-03T20:03:38.249610+0000","last_became_active":"2023-06-03T20:00:12.907900+0000","last_became_peered":"2023-06-03T20:00:12.907900+0000","last_unstale":"2023-06-03T20:03:38.249610+0000","last_undegraded":"2023-06-03T20:03:38.249610+0000","last_fullsized":"2023-06-03T20:03:38.249610+0000","mapping_epoch":127,"log_start":"132'3100","ondisk_log_start":"132'3100","created":127,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T20:00:11.873149+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T20:00:11.873149+0000","last_clean_scrub_stamp":"2023-06-03T20:00:11.873149+0000","objects_scrubbed":0,"log_size":7720,"log_dups_size":0,"ondisk_log_size":7720,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":187432960,"num_objects":2860,"num_object_clones":0,"num_object_copies":8580,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2860,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10820,"num_write_kb":437760,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":128,"reported_epoch":141,"state":"active+clean","last_fresh":"2023-06-03T20:03:23.856436+0000","last_change":"2023-06-03T19:57:28.466096+0000","last_active":"2023-06-03T20:03:23.856436+0000","last_peered":"2023-06-03T20:03:23.856436+0000","last_clean":"2023-06-03T20:03:23.856436+0000","last_became_active":"2023-06-03T19:57:28.464954+0000","last_became_peered":"2023-06-03T19:57:28.464954+0000","last_unstale":"2023-06-03T20:03:23.856436+0000","last_undegraded":"2023-06-03T20:03:23.856436+0000","last_fullsized":"2023-06-03T20:03:23.856436+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:32:20.692043+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:32:20.692043+0000","last_clean_scrub_stamp":"2023-06-03T19:32:20.692043+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":121,"reported_epoch":141,"state":"active+clean","last_fresh":"2023-06-03T20:03:23.857294+0000","last_change":"2023-06-03T19:56:11.750241+0000","last_active":"2023-06-03T20:03:23.857294+0000","last_peered":"2023-06-03T20:03:23.857294+0000","last_clean":"2023-06-03T20:03:23.857294+0000","last_became_active":"2023-06-03T19:56:11.749687+0000","last_became_peered":"2023-06-03T19:56:11.749687+0000","last_unstale":"2023-06-03T20:03:23.857294+0000","last_undegraded":"2023-06-03T20:03:23.857294+0000","last_fullsized":"2023-06-03T20:03:23.857294+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":115,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T22:38:25.819929+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":84,"reported_epoch":141,"state":"active+clean","last_fresh":"2023-06-03T20:03:23.860524+0000","last_change":"2023-06-03T19:47:06.733532+0000","last_active":"2023-06-03T20:03:23.860524+0000","last_peered":"2023-06-03T20:03:23.860524+0000","last_clean":"2023-06-03T20:03:23.860524+0000","last_became_active":"2023-06-03T19:47:06.733131+0000","last_became_peered":"2023-06-03T19:47:06.733131+0000","last_unstale":"2023-06-03T20:03:23.860524+0000","last_undegraded":"2023-06-03T20:03:23.860524+0000","last_fullsized":"2023-06-03T20:03:23.860524+0000","mapping_epoch":86,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":87,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T23:46:03.788646+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.d","version":"142'11164","reported_seq":11188,"reported_epoch":142,"state":"active+clean","last_fresh":"2023-06-03T20:03:38.230409+0000","last_change":"2023-06-03T20:00:12.905723+0000","last_active":"2023-06-03T20:03:38.230409+0000","last_peered":"2023-06-03T20:03:38.230409+0000","last_clean":"2023-06-03T20:03:38.230409+0000","last_became_active":"2023-06-03T20:00:12.905437+0000","last_became_peered":"2023-06-03T20:00:12.905437+0000","last_unstale":"2023-06-03T20:03:38.230409+0000","last_undegraded":"2023-06-03T20:03:38.230409+0000","last_fullsized":"2023-06-03T20:03:38.230409+0000","mapping_epoch":127,"log_start":"132'3400","ondisk_log_start":"132'3400","created":127,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T20:00:11.873149+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T20:00:11.873149+0000","last_clean_scrub_stamp":"2023-06-03T20:00:11.873149+0000","objects_scrubbed":0,"log_size":7764,"log_dups_size":0,"ondisk_log_size":7764,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":189857816,"num_objects":2898,"num_object_clones":0,"num_object_copies":8694,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2898,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":11164,"num_write_kb":449921,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,5],"acting":[7,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.7","version":"60'2","reported_seq":94,"reported_epoch":141,"state":"active+clean","last_fresh":"2023-06-03T20:03:23.850169+0000","last_change":"2023-06-03T19:56:54.510148+0000","last_active":"2023-06-03T20:03:23.850169+0000","last_peered":"2023-06-03T20:03:23.850169+0000","last_clean":"2023-06-03T20:03:23.850169+0000","last_became_active":"2023-06-03T19:56:54.509771+0000","last_became_peered":"2023-06-03T19:56:54.509771+0000","last_unstale":"2023-06-03T20:03:23.850169+0000","last_undegraded":"2023-06-03T20:03:23.850169+0000","last_fullsized":"2023-06-03T20:03:23.850169+0000","mapping_epoch":119,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T00:52:42.807432+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":121,"reported_epoch":141,"state":"active+clean","last_fresh":"2023-06-03T20:03:23.852774+0000","last_change":"2023-06-03T19:57:28.470954+0000","last_active":"2023-06-03T20:03:23.852774+0000","last_peered":"2023-06-03T20:03:23.852774+0000","last_clean":"2023-06-03T20:03:23.852774+0000","last_became_active":"2023-06-03T19:57:28.470524+0000","last_became_peered":"2023-06-03T19:57:28.470524+0000","last_unstale":"2023-06-03T20:03:23.852774+0000","last_undegraded":"2023-06-03T20:03:23.852774+0000","last_fullsized":"2023-06-03T20:03:23.852774+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:32:20.692043+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:32:20.692043+0000","last_clean_scrub_stamp":"2023-06-03T19:32:20.692043+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T19:39:19.983271+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":91,"reported_epoch":141,"state":"active+clean","last_fresh":"2023-06-03T20:03:23.849471+0000","last_change":"2023-06-03T19:56:54.507670+0000","last_active":"2023-06-03T20:03:23.849471+0000","last_peered":"2023-06-03T20:03:23.849471+0000","last_clean":"2023-06-03T20:03:23.849471+0000","last_became_active":"2023-06-03T19:56:54.507293+0000","last_became_peered":"2023-06-03T19:56:54.507293+0000","last_unstale":"2023-06-03T20:03:23.849471+0000","last_undegraded":"2023-06-03T20:03:23.849471+0000","last_fullsized":"2023-06-03T20:03:23.849471+0000","mapping_epoch":119,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T01:49:19.966054+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"9.c","version":"142'10857","reported_seq":10882,"reported_epoch":142,"state":"active+clean","last_fresh":"2023-06-03T20:03:38.004855+0000","last_change":"2023-06-03T20:00:12.904758+0000","last_active":"2023-06-03T20:03:38.004855+0000","last_peered":"2023-06-03T20:03:38.004855+0000","last_clean":"2023-06-03T20:03:38.004855+0000","last_became_active":"2023-06-03T20:00:12.904435+0000","last_became_peered":"2023-06-03T20:00:12.904435+0000","last_unstale":"2023-06-03T20:03:38.004855+0000","last_undegraded":"2023-06-03T20:03:38.004855+0000","last_fullsized":"2023-06-03T20:03:38.004855+0000","mapping_epoch":127,"log_start":"132'3100","ondisk_log_start":"132'3100","created":127,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T20:00:11.873149+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T20:00:11.873149+0000","last_clean_scrub_stamp":"2023-06-03T20:00:11.873149+0000","objects_scrubbed":0,"log_size":7757,"log_dups_size":0,"ondisk_log_size":7757,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T04:00:13.693202+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":186187776,"num_objects":2841,"num_object_clones":0,"num_object_copies":8523,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2841,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10857,"num_write_kb":438336,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,7],"acting":[3,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":113,"reported_epoch":141,"state":"active+clean","last_fresh":"2023-06-03T20:03:23.870676+0000","last_change":"2023-06-03T19:56:11.752730+0000","last_active":"2023-06-03T20:03:23.870676+0000","last_peered":"2023-06-03T20:03:23.870676+0000","last_clean":"2023-06-03T20:03:23.870676+0000","last_became_active":"2023-06-03T19:56:11.752441+0000","last_became_peered":"2023-06-03T19:56:11.752441+0000","last_unstale":"2023-06-03T20:03:23.870676+0000","last_undegraded":"2023-06-03T20:03:23.870676+0000","last_fullsized":"2023-06-03T20:03:23.870676+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":115,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:32:20.692043+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:32:20.692043+0000","last_clean_scrub_stamp":"2023-06-03T19:32:20.692043+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"60'1","reported_seq":117,"reported_epoch":141,"state":"active+clean","last_fresh":"2023-06-03T20:03:23.860668+0000","last_change":"2023-06-03T19:57:28.463999+0000","last_active":"2023-06-03T20:03:23.860668+0000","last_peered":"2023-06-03T20:03:23.860668+0000","last_clean":"2023-06-03T20:03:23.860668+0000","last_became_active":"2023-06-03T19:57:28.463588+0000","last_became_peered":"2023-06-03T19:57:28.463588+0000","last_unstale":"2023-06-03T20:03:23.860668+0000","last_undegraded":"2023-06-03T20:03:23.860668+0000","last_fullsized":"2023-06-03T20:03:23.860668+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T21:44:11.969059+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":85,"reported_epoch":142,"state":"active+clean","last_fresh":"2023-06-03T20:03:28.894810+0000","last_change":"2023-06-03T19:57:28.456467+0000","last_active":"2023-06-03T20:03:28.894810+0000","last_peered":"2023-06-03T20:03:28.894810+0000","last_clean":"2023-06-03T20:03:28.894810+0000","last_became_active":"2023-06-03T19:57:28.456120+0000","last_became_peered":"2023-06-03T19:57:28.456120+0000","last_unstale":"2023-06-03T20:03:28.894810+0000","last_undegraded":"2023-06-03T20:03:28.894810+0000","last_fullsized":"2023-06-03T20:03:28.894810+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T07:28:14.306624+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"60'1","reported_seq":101,"reported_epoch":141,"state":"active+clean","last_fresh":"2023-06-03T20:03:23.860776+0000","last_change":"2023-06-03T19:57:28.465124+0000","last_active":"2023-06-03T20:03:23.860776+0000","last_peered":"2023-06-03T20:03:23.860776+0000","last_clean":"2023-06-03T20:03:23.860776+0000","last_became_active":"2023-06-03T19:57:28.464654+0000","last_became_peered":"2023-06-03T19:57:28.464654+0000","last_unstale":"2023-06-03T20:03:23.860776+0000","last_undegraded":"2023-06-03T20:03:23.860776+0000","last_fullsized":"2023-06-03T20:03:23.860776+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T01:12:32.305507+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":128,"reported_epoch":142,"state":"active+clean","last_fresh":"2023-06-03T20:03:28.895450+0000","last_change":"2023-06-03T19:56:54.511760+0000","last_active":"2023-06-03T20:03:28.895450+0000","last_peered":"2023-06-03T20:03:28.895450+0000","last_clean":"2023-06-03T20:03:28.895450+0000","last_became_active":"2023-06-03T19:56:54.511429+0000","last_became_peered":"2023-06-03T19:56:54.511429+0000","last_unstale":"2023-06-03T20:03:28.895450+0000","last_undegraded":"2023-06-03T20:03:28.895450+0000","last_fullsized":"2023-06-03T20:03:28.895450+0000","mapping_epoch":119,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T05:42:46.254343+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"60'2","reported_seq":126,"reported_epoch":141,"state":"active+clean","last_fresh":"2023-06-03T20:03:23.857129+0000","last_change":"2023-06-03T19:57:28.462252+0000","last_active":"2023-06-03T20:03:23.857129+0000","last_peered":"2023-06-03T20:03:23.857129+0000","last_clean":"2023-06-03T20:03:23.857129+0000","last_became_active":"2023-06-03T19:57:28.461844+0000","last_became_peered":"2023-06-03T19:57:28.461844+0000","last_unstale":"2023-06-03T20:03:23.857129+0000","last_undegraded":"2023-06-03T20:03:23.857129+0000","last_fullsized":"2023-06-03T20:03:23.857129+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":131,"reported_epoch":141,"state":"active+clean","last_fresh":"2023-06-03T20:03:23.845536+0000","last_change":"2023-06-03T19:57:28.469248+0000","last_active":"2023-06-03T20:03:23.845536+0000","last_peered":"2023-06-03T20:03:23.845536+0000","last_clean":"2023-06-03T20:03:23.845536+0000","last_became_active":"2023-06-03T19:57:28.468903+0000","last_became_peered":"2023-06-03T19:57:28.468903+0000","last_unstale":"2023-06-03T20:03:23.845536+0000","last_undegraded":"2023-06-03T20:03:23.845536+0000","last_fullsized":"2023-06-03T20:03:23.845536+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T06:08:20.956098+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"60'2","reported_seq":111,"reported_epoch":141,"state":"active+clean","last_fresh":"2023-06-03T20:03:23.859136+0000","last_change":"2023-06-03T19:57:28.462610+0000","last_active":"2023-06-03T20:03:23.859136+0000","last_peered":"2023-06-03T20:03:23.859136+0000","last_clean":"2023-06-03T20:03:23.859136+0000","last_became_active":"2023-06-03T19:57:28.460655+0000","last_became_peered":"2023-06-03T19:57:28.460655+0000","last_unstale":"2023-06-03T20:03:23.859136+0000","last_undegraded":"2023-06-03T20:03:23.859136+0000","last_fullsized":"2023-06-03T20:03:23.859136+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":131,"reported_epoch":141,"state":"active+clean","last_fresh":"2023-06-03T20:03:23.845339+0000","last_change":"2023-06-03T19:57:28.464690+0000","last_active":"2023-06-03T20:03:23.845339+0000","last_peered":"2023-06-03T20:03:23.845339+0000","last_clean":"2023-06-03T20:03:23.845339+0000","last_became_active":"2023-06-03T19:57:28.463856+0000","last_became_peered":"2023-06-03T19:57:28.463856+0000","last_unstale":"2023-06-03T20:03:23.845339+0000","last_undegraded":"2023-06-03T20:03:23.845339+0000","last_fullsized":"2023-06-03T20:03:23.845339+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T00:11:00.991088+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"60'1","reported_seq":120,"reported_epoch":141,"state":"active+clean","last_fresh":"2023-06-03T20:03:23.849796+0000","last_change":"2023-06-03T19:56:54.505096+0000","last_active":"2023-06-03T20:03:23.849796+0000","last_peered":"2023-06-03T20:03:23.849796+0000","last_clean":"2023-06-03T20:03:23.849796+0000","last_became_active":"2023-06-03T19:56:54.504255+0000","last_became_peered":"2023-06-03T19:56:54.504255+0000","last_unstale":"2023-06-03T20:03:23.849796+0000","last_undegraded":"2023-06-03T20:03:23.849796+0000","last_fullsized":"2023-06-03T20:03:23.849796+0000","mapping_epoch":119,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T19:52:25.922308+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":132,"reported_epoch":141,"state":"active+clean","last_fresh":"2023-06-03T20:03:23.849283+0000","last_change":"2023-06-03T19:56:54.503416+0000","last_active":"2023-06-03T20:03:23.849283+0000","last_peered":"2023-06-03T20:03:23.849283+0000","last_clean":"2023-06-03T20:03:23.849283+0000","last_became_active":"2023-06-03T19:56:54.503018+0000","last_became_peered":"2023-06-03T19:56:54.503018+0000","last_unstale":"2023-06-03T20:03:23.849283+0000","last_undegraded":"2023-06-03T20:03:23.849283+0000","last_fullsized":"2023-06-03T20:03:23.849283+0000","mapping_epoch":119,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T20:14:00.330616+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"60'4","reported_seq":102,"reported_epoch":141,"state":"active+clean","last_fresh":"2023-06-03T20:03:23.860952+0000","last_change":"2023-06-03T19:56:11.748116+0000","last_active":"2023-06-03T20:03:23.860952+0000","last_peered":"2023-06-03T20:03:23.860952+0000","last_clean":"2023-06-03T20:03:23.860952+0000","last_became_active":"2023-06-03T19:56:11.747636+0000","last_became_peered":"2023-06-03T19:56:11.747636+0000","last_unstale":"2023-06-03T20:03:23.860952+0000","last_undegraded":"2023-06-03T20:03:23.860952+0000","last_fullsized":"2023-06-03T20:03:23.860952+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":115,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T06:57:48.052055+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":102,"reported_epoch":141,"state":"active+clean","last_fresh":"2023-06-03T20:03:23.844663+0000","last_change":"2023-06-03T19:57:28.459234+0000","last_active":"2023-06-03T20:03:23.844663+0000","last_peered":"2023-06-03T20:03:23.844663+0000","last_clean":"2023-06-03T20:03:23.844663+0000","last_became_active":"2023-06-03T19:57:28.458896+0000","last_became_peered":"2023-06-03T19:57:28.458896+0000","last_unstale":"2023-06-03T20:03:23.844663+0000","last_undegraded":"2023-06-03T20:03:23.844663+0000","last_fullsized":"2023-06-03T20:03:23.844663+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T21:18:37.142280+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5486,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":62,"num_write_kb":30,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16458,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":30319,"internal_metadata":0},"log_size":56,"ondisk_log_size":56,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":3,"num_store_stats":3},{"poolid":9,"num_pg":16,"stat_sum":{"num_bytes":3029794840,"num_objects":46232,"num_object_clones":0,"num_object_copies":138696,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":46232,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":174032,"num_write_kb":7048385,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":9062133760,"data_stored":9062121544,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139232,"ondisk_log_size":139232,"up":48,"acting":48,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":48,"seq":206158430599,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2057412,"kb_used_data":1676540,"kb_used_omap":0,"kb_used_meta":380864,"kb_avail":91687740,"statfs":{"total":95995035648,"available":93888245760,"internally_reserved":0,"allocated":1716776960,"data_stored":1716295028,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":390004736},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1],"upper_bound":8},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 3 20:02:40 2023","interfaces":[{"interface":"back","average":{"1min":1.1120000000000001,"5min":1.0089999999999999,"15min":1.2629999999999999},"min":{"1min":0.36699999999999999,"5min":0.36599999999999999,"15min":0.253},"max":{"1min":3.7189999999999999,"5min":3.7189999999999999,"15min":8.6099999999999994},"last":2.9540000000000002},{"interface":"front","average":{"1min":2.1629999999999998,"5min":1.3169999999999999,"15min":1.4850000000000001},"min":{"1min":0.318,"5min":0.318,"15min":0.28499999999999998},"max":{"1min":11.877000000000001,"5min":11.877000000000001,"15min":15.220000000000001},"last":3.0880000000000001}]},{"osd":1,"last update":"Sat Jun 3 20:03:22 2023","interfaces":[{"interface":"back","average":{"1min":2.3039999999999998,"5min":2.4510000000000001,"15min":1.964},"min":{"1min":0.29699999999999999,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":22.533999999999999,"5min":91.811000000000007,"15min":91.811000000000007},"last":0.85899999999999999},{"interface":"front","average":{"1min":3.6070000000000002,"5min":2.7080000000000002,"15min":1.95},"min":{"1min":0.32200000000000001,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":22.675000000000001,"5min":91.695999999999998,"15min":91.695999999999998},"last":0.97499999999999998}]},{"osd":2,"last update":"Sat Jun 3 20:03:22 2023","interfaces":[{"interface":"back","average":{"1min":1.3029999999999999,"5min":1.157,"15min":1.0669999999999999},"min":{"1min":0.32800000000000001,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":7.71,"5min":7.71,"15min":7.71},"last":1.3600000000000001},{"interface":"front","average":{"1min":2.3199999999999998,"5min":1.3029999999999999,"15min":1.093},"min":{"1min":0.38500000000000001,"5min":0.38500000000000001,"15min":0.38500000000000001},"max":{"1min":8.3059999999999992,"5min":8.3059999999999992,"15min":8.3059999999999992},"last":1.448}]},{"osd":3,"last update":"Sat Jun 3 20:03:10 2023","interfaces":[{"interface":"back","average":{"1min":1.3169999999999999,"5min":2.411,"15min":1.5740000000000001},"min":{"1min":0.34699999999999998,"5min":0.34699999999999998,"15min":0.34699999999999998},"max":{"1min":4.8010000000000002,"5min":95.962000000000003,"15min":95.962000000000003},"last":11.904},{"interface":"front","average":{"1min":2.214,"5min":2.5659999999999998,"15min":1.655},"min":{"1min":0.36399999999999999,"5min":0.36399999999999999,"15min":0.36399999999999999},"max":{"1min":7.9630000000000001,"5min":94.174999999999997,"15min":94.174999999999997},"last":12.084}]},{"osd":4,"last update":"Sat Jun 3 20:03:33 2023","interfaces":[{"interface":"back","average":{"1min":1.1919999999999999,"5min":1.056,"15min":1.052},"min":{"1min":0.38200000000000001,"5min":0.35999999999999999,"15min":0.35999999999999999},"max":{"1min":7.7350000000000003,"5min":7.7350000000000003,"15min":7.7350000000000003},"last":0.81799999999999995},{"interface":"front","average":{"1min":2.6640000000000001,"5min":1.5309999999999999,"15min":1.256},"min":{"1min":0.39600000000000002,"5min":0.39600000000000002,"15min":0.39600000000000002},"max":{"1min":12.917,"5min":12.917,"15min":12.917},"last":0.90400000000000003}]},{"osd":5,"last update":"Sat Jun 3 20:03:19 2023","interfaces":[{"interface":"back","average":{"1min":1.383,"5min":1.1679999999999999,"15min":1.1719999999999999},"min":{"1min":0.443,"5min":0.443,"15min":0.30199999999999999},"max":{"1min":7.6829999999999998,"5min":7.6829999999999998,"15min":12.778},"last":1.2809999999999999},{"interface":"front","average":{"1min":2.2650000000000001,"5min":1.5269999999999999,"15min":1.387},"min":{"1min":0.33100000000000002,"5min":0.33100000000000002,"15min":0.30599999999999999},"max":{"1min":8.1340000000000003,"5min":8.1340000000000003,"15min":8.1340000000000003},"last":0.94399999999999995}]},{"osd":6,"last update":"Sat Jun 3 20:03:19 2023","interfaces":[{"interface":"back","average":{"1min":2.3929999999999998,"5min":1.431,"15min":1.363},"min":{"1min":0.42199999999999999,"5min":0.42199999999999999,"15min":0.36099999999999999},"max":{"1min":8.3219999999999992,"5min":8.3219999999999992,"15min":8.3219999999999992},"last":1.8959999999999999},{"interface":"front","average":{"1min":1.6240000000000001,"5min":1.4490000000000001,"15min":1.2509999999999999},"min":{"1min":0.36899999999999999,"5min":0.36899999999999999,"15min":0.32900000000000001},"max":{"1min":4.6210000000000004,"5min":6.6840000000000002,"15min":9.0269999999999992},"last":1.9330000000000001}]}]},{"osd":6,"up_from":42,"seq":180388626832,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":870384,"kb_used_data":575336,"kb_used_omap":0,"kb_used_meta":295040,"kb_avail":92874768,"statfs":{"total":95995035648,"available":95103762432,"internally_reserved":0,"allocated":589144064,"data_stored":588682127,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":302120960},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 3 20:02:50 2023","interfaces":[{"interface":"back","average":{"1min":3.9409999999999998,"5min":1.522,"15min":1.6379999999999999},"min":{"1min":0.26700000000000002,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":54.786000000000001,"5min":54.786000000000001,"15min":54.786000000000001},"last":0.66000000000000003},{"interface":"front","average":{"1min":4.2309999999999999,"5min":1.534,"15min":1.7130000000000001},"min":{"1min":0.29399999999999998,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":54.936,"5min":54.936,"15min":54.936},"last":0.45900000000000002}]},{"osd":1,"last update":"Sat Jun 3 20:03:24 2023","interfaces":[{"interface":"back","average":{"1min":5.8339999999999996,"5min":4.1920000000000002,"15min":2.4239999999999999},"min":{"1min":0.28000000000000003,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":55.064999999999998,"5min":160.38499999999999,"15min":160.38499999999999},"last":0.78300000000000003},{"interface":"front","average":{"1min":6.1440000000000001,"5min":3.4830000000000001,"15min":2.032},"min":{"1min":0.29599999999999999,"5min":0.29599999999999999,"15min":0.29599999999999999},"max":{"1min":55.091000000000001,"5min":160.434,"15min":160.434},"last":0.41999999999999998}]},{"osd":2,"last update":"Sat Jun 3 20:03:24 2023","interfaces":[{"interface":"back","average":{"1min":0.97899999999999998,"5min":0.88300000000000001,"15min":0.85699999999999998},"min":{"1min":0.33300000000000002,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":3.4260000000000002,"5min":3.4260000000000002,"15min":3.4260000000000002},"last":0.44},{"interface":"front","average":{"1min":0.78500000000000003,"5min":0.872,"15min":0.92100000000000004},"min":{"1min":0.34799999999999998,"5min":0.34799999999999998,"15min":0.34799999999999998},"max":{"1min":3.1779999999999999,"5min":3.1779999999999999,"15min":3.1779999999999999},"last":0.755}]},{"osd":3,"last update":"Sat Jun 3 20:03:06 2023","interfaces":[{"interface":"back","average":{"1min":4.4710000000000001,"5min":4.1440000000000001,"15min":1.95},"min":{"1min":0.32400000000000001,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":55.633000000000003,"5min":164.99199999999999,"15min":164.99199999999999},"last":0.50600000000000001},{"interface":"front","average":{"1min":4.952,"5min":4.2160000000000002,"15min":2.0680000000000001},"min":{"1min":0.48199999999999998,"5min":0.36799999999999999,"15min":0.36799999999999999},"max":{"1min":61.143000000000001,"5min":163.459,"15min":163.459},"last":0.83899999999999997}]},{"osd":4,"last update":"Sat Jun 3 20:02:40 2023","interfaces":[{"interface":"back","average":{"1min":0.65800000000000003,"5min":0.88300000000000001,"15min":0.95999999999999996},"min":{"1min":0.33300000000000002,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":1.6839999999999999,"5min":1.6839999999999999,"15min":1.6839999999999999},"last":0.376},{"interface":"front","average":{"1min":0.54000000000000004,"5min":0.90500000000000003,"15min":1.089},"min":{"1min":0.39100000000000001,"5min":0.38,"15min":0.38},"max":{"1min":0.72699999999999998,"5min":1.5009999999999999,"15min":1.5009999999999999},"last":0.70299999999999996}]},{"osd":5,"last update":"Sat Jun 3 20:03:37 2023","interfaces":[{"interface":"back","average":{"1min":0.68600000000000005,"5min":0.93200000000000005,"15min":1.159},"min":{"1min":0.34999999999999998,"5min":0.34999999999999998,"15min":0.317},"max":{"1min":1.1719999999999999,"5min":3.0750000000000002,"15min":11.083},"last":0.81000000000000005},{"interface":"front","average":{"1min":2.214,"5min":1.399,"15min":1.389},"min":{"1min":0.32900000000000001,"5min":0.32900000000000001,"15min":0.32700000000000001},"max":{"1min":8.4160000000000004,"5min":8.4160000000000004,"15min":10.154},"last":4.0460000000000003}]},{"osd":7,"last update":"Sat Jun 3 20:03:09 2023","interfaces":[{"interface":"back","average":{"1min":2.0609999999999999,"5min":1.329,"15min":1.304},"min":{"1min":0.39400000000000002,"5min":0.39400000000000002,"15min":0.313},"max":{"1min":8.7409999999999997,"5min":8.7409999999999997,"15min":8.7409999999999997},"last":0.73099999999999998},{"interface":"front","average":{"1min":2.0470000000000002,"5min":1.2929999999999999,"15min":1.395},"min":{"1min":0.433,"5min":0.33900000000000002,"15min":0.33000000000000002},"max":{"1min":6.5359999999999996,"5min":6.5359999999999996,"15min":12.407999999999999},"last":0.95899999999999996}]}]},{"osd":1,"up_from":103,"seq":442381631630,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1904604,"kb_used_data":1675412,"kb_used_omap":4,"kb_used_meta":229179,"kb_avail":91840548,"statfs":{"total":95995035648,"available":94044721152,"internally_reserved":0,"allocated":1715621888,"data_stored":1715062511,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":234680206},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1],"upper_bound":8},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 3 20:03:25 2023","interfaces":[{"interface":"back","average":{"1min":7.4909999999999997,"5min":2.3479999999999999,"15min":1.921},"min":{"1min":0.217,"5min":0.185,"15min":0.185},"max":{"1min":119.902,"5min":119.902,"15min":119.902},"last":0.26500000000000001},{"interface":"front","average":{"1min":7.5640000000000001,"5min":2.2360000000000002,"15min":1.978},"min":{"1min":0.22500000000000001,"5min":0.22500000000000001,"15min":0.192},"max":{"1min":119.863,"5min":119.863,"15min":119.863},"last":0.32100000000000001}]},{"osd":2,"last update":"Sat Jun 3 20:03:34 2023","interfaces":[{"interface":"back","average":{"1min":0.97299999999999998,"5min":1.9550000000000001,"15min":1.1279999999999999},"min":{"1min":0.24099999999999999,"5min":0.23599999999999999,"15min":0.23599999999999999},"max":{"1min":4.6280000000000001,"5min":119.946,"15min":119.946},"last":0.36899999999999999},{"interface":"front","average":{"1min":1.6859999999999999,"5min":1.0680000000000001,"15min":0.84999999999999998},"min":{"1min":0.26600000000000001,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":9.8160000000000007,"5min":9.8160000000000007,"15min":9.8160000000000007},"last":0.35599999999999998}]},{"osd":3,"last update":"Sat Jun 3 20:03:05 2023","interfaces":[{"interface":"back","average":{"1min":1.6399999999999999,"5min":1.3140000000000001,"15min":0.93000000000000005},"min":{"1min":0.34999999999999998,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":7.1289999999999996,"5min":9.4079999999999995,"15min":9.4079999999999995},"last":0.39100000000000001},{"interface":"front","average":{"1min":1.5760000000000001,"5min":1.304,"15min":1.014},"min":{"1min":0.32900000000000001,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":7.0990000000000002,"5min":9.3200000000000003,"15min":9.3200000000000003},"last":0.41599999999999998}]},{"osd":4,"last update":"Sat Jun 3 20:03:36 2023","interfaces":[{"interface":"back","average":{"1min":2.6720000000000002,"5min":1.5149999999999999,"15min":1.167},"min":{"1min":0.38300000000000001,"5min":0.35999999999999999,"15min":0.35999999999999999},"max":{"1min":9.9589999999999996,"5min":9.9589999999999996,"15min":9.9589999999999996},"last":4.1100000000000003},{"interface":"front","average":{"1min":1.0680000000000001,"5min":2.52,"15min":1.478},"min":{"1min":0.32700000000000001,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":4.0670000000000002,"5min":120.084,"15min":120.084},"last":0.54400000000000004}]},{"osd":5,"last update":"Sat Jun 3 20:03:25 2023","interfaces":[{"interface":"back","average":{"1min":7.6779999999999999,"5min":2.8860000000000001,"15min":2.3690000000000002},"min":{"1min":0.36799999999999999,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":120.39700000000001,"5min":120.39700000000001,"15min":120.39700000000001},"last":0.72999999999999998},{"interface":"front","average":{"1min":7.75,"5min":2.7229999999999999,"15min":2.5800000000000001},"min":{"1min":0.48299999999999998,"5min":0.48299999999999998,"15min":0.40000000000000002},"max":{"1min":120.34399999999999,"5min":120.34399999999999,"15min":120.34399999999999},"last":0.63100000000000001}]},{"osd":6,"last update":"Sat Jun 3 20:03:25 2023","interfaces":[{"interface":"back","average":{"1min":7.1900000000000004,"5min":2.3290000000000002,"15min":2.2730000000000001},"min":{"1min":0.432,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":120.386,"5min":120.386,"15min":120.386},"last":0.44700000000000001},{"interface":"front","average":{"1min":7.3460000000000001,"5min":2.4500000000000002,"15min":2.5590000000000002},"min":{"1min":0.39600000000000002,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":119.96299999999999,"5min":119.96299999999999,"15min":119.96299999999999},"last":0.45800000000000002}]},{"osd":7,"last update":"Sat Jun 3 20:03:25 2023","interfaces":[{"interface":"back","average":{"1min":7.8109999999999999,"5min":2.7269999999999999,"15min":2.4430000000000001},"min":{"1min":0.40999999999999998,"5min":0.38200000000000001,"15min":0.38200000000000001},"max":{"1min":119.996,"5min":119.996,"15min":119.996},"last":0.48399999999999999},{"interface":"front","average":{"1min":7.758,"5min":2.653,"15min":2.0190000000000001},"min":{"1min":0.44800000000000001,"5min":0.40400000000000003,"15min":0.371},"max":{"1min":120.17100000000001,"5min":120.17100000000001,"15min":120.17100000000001},"last":0.57799999999999996}]}]},{"osd":0,"up_from":86,"seq":369367187657,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":972084,"kb_used_data":733420,"kb_used_omap":5,"kb_used_meta":238586,"kb_avail":92773068,"statfs":{"total":95995035648,"available":94999621632,"internally_reserved":0,"allocated":751022080,"data_stored":750441517,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":244312698},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,1,0,2],"upper_bound":32},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Jun 3 20:03:15 2023","interfaces":[{"interface":"back","average":{"1min":6.5179999999999998,"5min":2.4199999999999999,"15min":2.1400000000000001},"min":{"1min":0.192,"5min":0.192,"15min":0.192},"max":{"1min":81.310000000000002,"5min":81.310000000000002,"15min":81.310000000000002},"last":0.41099999999999998},{"interface":"front","average":{"1min":5.3650000000000002,"5min":2.306,"15min":1.881},"min":{"1min":0.223,"5min":0.187,"15min":0.187},"max":{"1min":76.971999999999994,"5min":76.971999999999994,"15min":76.971999999999994},"last":0.81299999999999994}]},{"osd":2,"last update":"Sat Jun 3 20:03:15 2023","interfaces":[{"interface":"back","average":{"1min":1.589,"5min":0.95799999999999996,"15min":0.76100000000000001},"min":{"1min":0.22900000000000001,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":6.0339999999999998,"5min":8.923,"15min":8.923},"last":1.978},{"interface":"front","average":{"1min":1.653,"5min":1.0589999999999999,"15min":0.85399999999999998},"min":{"1min":0.29399999999999998,"5min":0.218,"15min":0.218},"max":{"1min":4.4139999999999997,"5min":11.194000000000001,"15min":11.194000000000001},"last":0.372}]},{"osd":3,"last update":"Sat Jun 3 20:03:05 2023","interfaces":[{"interface":"back","average":{"1min":2.2130000000000001,"5min":1.0940000000000001,"15min":0.90800000000000003},"min":{"1min":0.38700000000000001,"5min":0.216,"15min":0.216},"max":{"1min":7.5990000000000002,"5min":7.5990000000000002,"15min":7.5990000000000002},"last":1.9510000000000001},{"interface":"front","average":{"1min":1.917,"5min":1.167,"15min":0.89600000000000002},"min":{"1min":0.36099999999999999,"5min":0.35399999999999998,"15min":0.35399999999999998},"max":{"1min":4.6340000000000003,"5min":5.3380000000000001,"15min":5.3380000000000001},"last":1.103}]},{"osd":4,"last update":"Sat Jun 3 20:02:43 2023","interfaces":[{"interface":"back","average":{"1min":1.1919999999999999,"5min":0.98799999999999999,"15min":1.0600000000000001},"min":{"1min":0.39800000000000002,"5min":0.33800000000000002,"15min":0.33800000000000002},"max":{"1min":3.6459999999999999,"5min":3.6459999999999999,"15min":3.6459999999999999},"last":2.0169999999999999},{"interface":"front","average":{"1min":1.782,"5min":1.1819999999999999,"15min":1.139},"min":{"1min":0.38800000000000001,"5min":0.38800000000000001,"15min":0.38800000000000001},"max":{"1min":4.8220000000000001,"5min":4.8220000000000001,"15min":4.8220000000000001},"last":1.514}]},{"osd":5,"last update":"Sat Jun 3 20:02:43 2023","interfaces":[{"interface":"back","average":{"1min":1.4550000000000001,"5min":1.081,"15min":1.669},"min":{"1min":0.439,"5min":0.36799999999999999,"15min":0.33900000000000002},"max":{"1min":4.1070000000000002,"5min":4.9359999999999999,"15min":16.312000000000001},"last":3.532},{"interface":"front","average":{"1min":1.516,"5min":1.149,"15min":1.665},"min":{"1min":0.40000000000000002,"5min":0.40000000000000002,"15min":0.32500000000000001},"max":{"1min":3.931,"5min":3.931,"15min":24.661000000000001},"last":7.9779999999999998}]},{"osd":6,"last update":"Sat Jun 3 20:02:43 2023","interfaces":[{"interface":"back","average":{"1min":1.4159999999999999,"5min":1.0920000000000001,"15min":1.621},"min":{"1min":0.38,"5min":0.38,"15min":0.33500000000000002},"max":{"1min":4.0359999999999996,"5min":4.0359999999999996,"15min":24.599},"last":1.998},{"interface":"front","average":{"1min":2.125,"5min":1.3149999999999999,"15min":1.8300000000000001},"min":{"1min":0.49099999999999999,"5min":0.49099999999999999,"15min":0.42999999999999999},"max":{"1min":4.7930000000000001,"5min":4.7930000000000001,"15min":13.601000000000001},"last":1.4650000000000001}]},{"osd":7,"last update":"Sat Jun 3 20:02:43 2023","interfaces":[{"interface":"back","average":{"1min":1.5620000000000001,"5min":1.2230000000000001,"15min":1.639},"min":{"1min":0.35099999999999998,"5min":0.35099999999999998,"15min":0.35099999999999998},"max":{"1min":5.3490000000000002,"5min":5.3490000000000002,"15min":24.634},"last":2.0339999999999998},{"interface":"front","average":{"1min":2.2589999999999999,"5min":1.4330000000000001,"15min":1.758},"min":{"1min":0.53700000000000003,"5min":0.51400000000000001,"15min":0.40699999999999997},"max":{"1min":4.883,"5min":4.883,"15min":12.621},"last":1.9670000000000001}]}]},{"osd":2,"up_from":114,"seq":489626271836,"num_pgs":22,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":588996,"kb_used_data":546748,"kb_used_omap":7,"kb_used_meta":42232,"kb_avail":93156156,"statfs":{"total":95995035648,"available":95391903744,"internally_reserved":0,"allocated":559869952,"data_stored":559332093,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8111,"internal_metadata":43245649},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 3 20:03:22 2023","interfaces":[{"interface":"back","average":{"1min":0.76100000000000001,"5min":2.2989999999999999,"15min":1.181},"min":{"1min":0.20899999999999999,"5min":0.20899999999999999,"15min":0.20899999999999999},"max":{"1min":3.4769999999999999,"5min":126.416,"15min":126.416},"last":0.23300000000000001},{"interface":"front","average":{"1min":0.67300000000000004,"5min":2.3860000000000001,"15min":1.268},"min":{"1min":0.22700000000000001,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":3.2290000000000001,"5min":126.976,"15min":126.976},"last":0.78900000000000003}]},{"osd":1,"last update":"Sat Jun 3 20:03:22 2023","interfaces":[{"interface":"back","average":{"1min":1.0449999999999999,"5min":1.296,"15min":0.91000000000000003},"min":{"1min":0.22900000000000001,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":6.5060000000000002,"5min":28.484999999999999,"15min":28.484999999999999},"last":0.52600000000000002},{"interface":"front","average":{"1min":1.0569999999999999,"5min":1.1870000000000001,"15min":0.84499999999999997},"min":{"1min":0.255,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":4.0259999999999998,"5min":28.588999999999999,"15min":28.588999999999999},"last":0.57099999999999995}]},{"osd":3,"last update":"Sat Jun 3 20:03:08 2023","interfaces":[{"interface":"back","average":{"1min":1.653,"5min":1.196,"15min":0.89900000000000002},"min":{"1min":0.23699999999999999,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":13.016999999999999,"5min":29.251999999999999,"15min":29.251999999999999},"last":0.43099999999999999},{"interface":"front","average":{"1min":5.0410000000000004,"5min":2.1179999999999999,"15min":1.2110000000000001},"min":{"1min":0.29099999999999998,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":54.549999999999997,"5min":54.549999999999997,"15min":54.549999999999997},"last":0.53800000000000003}]},{"osd":4,"last update":"Sat Jun 3 20:02:38 2023","interfaces":[{"interface":"back","average":{"1min":0.77700000000000002,"5min":0.91300000000000003,"15min":0.91800000000000004},"min":{"1min":0.33900000000000002,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":3.8530000000000002,"5min":7.9089999999999998,"15min":7.9089999999999998},"last":0.499},{"interface":"front","average":{"1min":0.94699999999999995,"5min":0.92200000000000004,"15min":0.95699999999999996},"min":{"1min":0.377,"5min":0.372,"15min":0.372},"max":{"1min":3.4540000000000002,"5min":3.4540000000000002,"15min":3.4540000000000002},"last":0.871}]},{"osd":5,"last update":"Sat Jun 3 20:03:22 2023","interfaces":[{"interface":"back","average":{"1min":1.0049999999999999,"5min":1,"15min":1.0229999999999999},"min":{"1min":0.36599999999999999,"5min":0.36099999999999999,"15min":0.36099999999999999},"max":{"1min":3.5339999999999998,"5min":3.5339999999999998,"15min":3.5339999999999998},"last":0.39700000000000002},{"interface":"front","average":{"1min":1.583,"5min":1.327,"15min":1.1399999999999999},"min":{"1min":0.39400000000000002,"5min":0.308,"15min":0.308},"max":{"1min":4.9370000000000003,"5min":7.8689999999999998,"15min":7.8689999999999998},"last":0.995}]},{"osd":6,"last update":"Sat Jun 3 20:03:22 2023","interfaces":[{"interface":"back","average":{"1min":1.085,"5min":1.032,"15min":1.034},"min":{"1min":0.38600000000000001,"5min":0.38500000000000001,"15min":0.38500000000000001},"max":{"1min":4.9950000000000001,"5min":6.7759999999999998,"15min":6.7759999999999998},"last":0.46000000000000002},{"interface":"front","average":{"1min":1.0649999999999999,"5min":1.012,"15min":1.036},"min":{"1min":0.41199999999999998,"5min":0.36299999999999999,"15min":0.36299999999999999},"max":{"1min":4.3559999999999999,"5min":7.9550000000000001,"15min":7.9550000000000001},"last":0.55300000000000005}]},{"osd":7,"last update":"Sat Jun 3 20:03:22 2023","interfaces":[{"interface":"back","average":{"1min":2.0899999999999999,"5min":1.4990000000000001,"15min":1.282},"min":{"1min":0.44700000000000001,"5min":0.44700000000000001,"15min":0.44700000000000001},"max":{"1min":6.0800000000000001,"5min":6.0800000000000001,"15min":6.0800000000000001},"last":7.3920000000000003},{"interface":"front","average":{"1min":1.992,"5min":1.472,"15min":1.2609999999999999},"min":{"1min":0.47899999999999998,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":6.0529999999999999,"5min":7.9320000000000004,"15min":7.9320000000000004},"last":4.5350000000000001}]}]},{"osd":3,"up_from":119,"seq":511101108307,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1597368,"kb_used_data":1502320,"kb_used_omap":2,"kb_used_meta":94909,"kb_avail":92147784,"statfs":{"total":95995035648,"available":94359330816,"internally_reserved":0,"allocated":1538375680,"data_stored":1537846796,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":97186869},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[2,0,0,0,3,2,1],"upper_bound":128},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 3 20:03:07 2023","interfaces":[{"interface":"back","average":{"1min":1.752,"5min":1.8129999999999999,"15min":1.0249999999999999},"min":{"1min":0.20799999999999999,"5min":0.20799999999999999,"15min":0.20799999999999999},"max":{"1min":7.1859999999999999,"5min":68.647999999999996,"15min":68.647999999999996},"last":1.002},{"interface":"front","average":{"1min":1.091,"5min":1.702,"15min":1.054},"min":{"1min":0.28199999999999997,"5min":0.224,"15min":0.224},"max":{"1min":4.032,"5min":69.691000000000003,"15min":69.691000000000003},"last":0.41799999999999998}]},{"osd":1,"last update":"Sat Jun 3 20:03:07 2023","interfaces":[{"interface":"back","average":{"1min":2.0019999999999998,"5min":1.171,"15min":0.86399999999999999},"min":{"1min":0.32300000000000001,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":10.065,"5min":10.065,"15min":10.065},"last":0.95699999999999996},{"interface":"front","average":{"1min":1.8839999999999999,"5min":1.206,"15min":0.90900000000000003},"min":{"1min":0.26100000000000001,"5min":0.23499999999999999,"15min":0.23499999999999999},"max":{"1min":10.112,"5min":10.792999999999999,"15min":10.792999999999999},"last":0.86799999999999999}]},{"osd":2,"last update":"Sat Jun 3 20:03:07 2023","interfaces":[{"interface":"back","average":{"1min":1.28,"5min":1.0089999999999999,"15min":0.76500000000000001},"min":{"1min":0.28399999999999997,"5min":0.245,"15min":0.245},"max":{"1min":3.8410000000000002,"5min":8.3149999999999995,"15min":8.3149999999999995},"last":0.44500000000000001},{"interface":"front","average":{"1min":1.3540000000000001,"5min":1.083,"15min":0.86899999999999999},"min":{"1min":0.34799999999999998,"5min":0.24199999999999999,"15min":0.24199999999999999},"max":{"1min":7.7290000000000001,"5min":7.7290000000000001,"15min":7.7290000000000001},"last":0.40000000000000002}]},{"osd":4,"last update":"Sat Jun 3 20:02:36 2023","interfaces":[{"interface":"back","average":{"1min":1.2629999999999999,"5min":1.1850000000000001,"15min":1.0609999999999999},"min":{"1min":0.46500000000000002,"5min":0.46500000000000002,"15min":0.46500000000000002},"max":{"1min":4.774,"5min":4.774,"15min":4.774},"last":1.2110000000000001},{"interface":"front","average":{"1min":1.0009999999999999,"5min":1.105,"15min":1.042},"min":{"1min":0.33000000000000002,"5min":0.33000000000000002,"15min":0.33000000000000002},"max":{"1min":5.2329999999999997,"5min":5.2329999999999997,"15min":5.2329999999999997},"last":1.1739999999999999}]},{"osd":5,"last update":"Sat Jun 3 20:03:07 2023","interfaces":[{"interface":"back","average":{"1min":2.1560000000000001,"5min":1.3169999999999999,"15min":1.097},"min":{"1min":0.53500000000000003,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":7.1230000000000002,"5min":7.1230000000000002,"15min":7.1230000000000002},"last":2.2200000000000002},{"interface":"front","average":{"1min":2.3490000000000002,"5min":1.508,"15min":1.1859999999999999},"min":{"1min":0.35599999999999998,"5min":0.35599999999999998,"15min":0.35599999999999998},"max":{"1min":7.2560000000000002,"5min":8.2289999999999992,"15min":8.2289999999999992},"last":5.4770000000000003}]},{"osd":6,"last update":"Sat Jun 3 20:03:07 2023","interfaces":[{"interface":"back","average":{"1min":1.7010000000000001,"5min":1.3859999999999999,"15min":1.1499999999999999},"min":{"1min":0.38600000000000001,"5min":0.38600000000000001,"15min":0.38600000000000001},"max":{"1min":7.2080000000000002,"5min":8.282,"15min":8.282},"last":0.69799999999999995},{"interface":"front","average":{"1min":1.1359999999999999,"5min":1.2709999999999999,"15min":1.129},"min":{"1min":0.53900000000000003,"5min":0.499,"15min":0.499},"max":{"1min":3.3799999999999999,"5min":4.9480000000000004,"15min":4.9480000000000004},"last":0.61399999999999999}]},{"osd":7,"last update":"Sat Jun 3 20:03:07 2023","interfaces":[{"interface":"back","average":{"1min":1.446,"5min":1.341,"15min":1.143},"min":{"1min":0.56399999999999995,"5min":0.495,"15min":0.495},"max":{"1min":4.6859999999999999,"5min":5.04,"15min":5.04},"last":8.6910000000000007},{"interface":"front","average":{"1min":1.7869999999999999,"5min":1.677,"15min":1.244},"min":{"1min":0.47799999999999998,"5min":0.435,"15min":0.435},"max":{"1min":7.2439999999999998,"5min":12.715999999999999,"15min":12.715999999999999},"last":0.90300000000000002}]}]},{"osd":4,"up_from":124,"seq":532575944781,"num_pgs":36,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":575248,"kb_used_data":532552,"kb_used_omap":10,"kb_used_meta":42677,"kb_avail":93169904,"statfs":{"total":95995035648,"available":95405981696,"internally_reserved":0,"allocated":545333248,"data_stored":544784135,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":10652,"internal_metadata":43701860},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 3 20:02:37 2023","interfaces":[{"interface":"back","average":{"1min":1.266,"5min":0.94599999999999995,"15min":0.96499999999999997},"min":{"1min":0.27300000000000002,"5min":0.254,"15min":0.254},"max":{"1min":3.9039999999999999,"5min":3.9039999999999999,"15min":3.9039999999999999},"last":0.42199999999999999},{"interface":"front","average":{"1min":1.2490000000000001,"5min":1.01,"15min":1.024},"min":{"1min":0.32400000000000001,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":6.1299999999999999,"5min":6.1299999999999999,"15min":6.1299999999999999},"last":0.47199999999999998}]},{"osd":1,"last update":"Sat Jun 3 20:02:37 2023","interfaces":[{"interface":"back","average":{"1min":1.8029999999999999,"5min":1.097,"15min":0.94599999999999995},"min":{"1min":0.31900000000000001,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":10.18,"5min":10.18,"15min":10.18},"last":2.0750000000000002},{"interface":"front","average":{"1min":1.754,"5min":1.04,"15min":0.92200000000000004},"min":{"1min":0.34899999999999998,"5min":0.311,"15min":0.311},"max":{"1min":9.6449999999999996,"5min":9.6449999999999996,"15min":9.6449999999999996},"last":0.35499999999999998}]},{"osd":2,"last update":"Sat Jun 3 20:02:37 2023","interfaces":[{"interface":"back","average":{"1min":0.76700000000000002,"5min":0.81999999999999995,"15min":0.85399999999999998},"min":{"1min":0.312,"5min":0.312,"15min":0.312},"max":{"1min":2.6419999999999999,"5min":2.6419999999999999,"15min":2.6419999999999999},"last":0.39600000000000002},{"interface":"front","average":{"1min":0.86799999999999999,"5min":0.88400000000000001,"15min":0.92000000000000004},"min":{"1min":0.36899999999999999,"5min":0.34499999999999997,"15min":0.34499999999999997},"max":{"1min":2.6989999999999998,"5min":2.6989999999999998,"15min":2.6989999999999998},"last":0.38100000000000001}]},{"osd":3,"last update":"Sat Jun 3 20:02:37 2023","interfaces":[{"interface":"back","average":{"1min":1.3440000000000001,"5min":1.0309999999999999,"15min":0.98499999999999999},"min":{"1min":0.32500000000000001,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":7.3300000000000001,"5min":7.3300000000000001,"15min":7.3300000000000001},"last":2.0270000000000001},{"interface":"front","average":{"1min":1.4550000000000001,"5min":1.0800000000000001,"15min":1.0640000000000001},"min":{"1min":0.39900000000000002,"5min":0.39900000000000002,"15min":0.39900000000000002},"max":{"1min":7.4020000000000001,"5min":7.4020000000000001,"15min":7.4020000000000001},"last":0.505}]},{"osd":5,"last update":"Sat Jun 3 20:02:37 2023","interfaces":[{"interface":"back","average":{"1min":1.4990000000000001,"5min":1.052,"15min":0.98599999999999999},"min":{"1min":0.379,"5min":0.36099999999999999,"15min":0.36099999999999999},"max":{"1min":7.335,"5min":7.335,"15min":7.335},"last":0.63},{"interface":"front","average":{"1min":1.302,"5min":1.113,"15min":1.0289999999999999},"min":{"1min":0.33600000000000002,"5min":0.32600000000000001,"15min":0.32600000000000001},"max":{"1min":4.819,"5min":4.819,"15min":4.819},"last":1.9870000000000001}]},{"osd":6,"last update":"Sat Jun 3 20:02:37 2023","interfaces":[{"interface":"back","average":{"1min":0.81100000000000005,"5min":0.92000000000000004,"15min":1.0029999999999999},"min":{"1min":0.36899999999999999,"5min":0.36899999999999999,"15min":0.36899999999999999},"max":{"1min":2.3889999999999998,"5min":2.3889999999999998,"15min":2.3889999999999998},"last":0.434},{"interface":"front","average":{"1min":0.74399999999999999,"5min":0.93600000000000005,"15min":1.0309999999999999},"min":{"1min":0.36499999999999999,"5min":0.36499999999999999,"15min":0.36499999999999999},"max":{"1min":3.2490000000000001,"5min":3.2490000000000001,"15min":3.2490000000000001},"last":2.0579999999999998}]},{"osd":7,"last update":"Sat Jun 3 20:02:37 2023","interfaces":[{"interface":"back","average":{"1min":1.7949999999999999,"5min":1.2090000000000001,"15min":1.161},"min":{"1min":0.44,"5min":0.41799999999999998,"15min":0.41799999999999998},"max":{"1min":10.048999999999999,"5min":10.048999999999999,"15min":10.048999999999999},"last":2.2229999999999999},{"interface":"front","average":{"1min":2.0659999999999998,"5min":1.2669999999999999,"15min":1.2030000000000001},"min":{"1min":0.47199999999999998,"5min":0.46600000000000003,"15min":0.46600000000000003},"max":{"1min":8.8450000000000006,"5min":8.8450000000000006,"15min":8.8450000000000006},"last":2.2050000000000001}]}]},{"osd":5,"up_from":36,"seq":154618823066,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2161384,"kb_used_data":1694496,"kb_used_omap":0,"kb_used_meta":466624,"kb_avail":91583768,"statfs":{"total":95995035648,"available":93781778432,"internally_reserved":0,"allocated":1735163904,"data_stored":1734705477,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":477822976},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[2,0,0,2,3,1,3],"upper_bound":128},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 3 20:03:27 2023","interfaces":[{"interface":"back","average":{"1min":1.2589999999999999,"5min":1.369,"15min":1.472},"min":{"1min":0.26900000000000002,"5min":0.22800000000000001,"15min":0.22800000000000001},"max":{"1min":4.2199999999999998,"5min":37.689999999999998,"15min":37.689999999999998},"last":4.7519999999999998},{"interface":"front","average":{"1min":2.0920000000000001,"5min":1.8140000000000001,"15min":1.8480000000000001},"min":{"1min":0.26300000000000001,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":8.266,"5min":37.783999999999999,"15min":37.783999999999999},"last":4.766}]},{"osd":1,"last update":"Sat Jun 3 20:03:27 2023","interfaces":[{"interface":"back","average":{"1min":6.7569999999999997,"5min":3.7749999999999999,"15min":2.4620000000000002},"min":{"1min":0.47399999999999998,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":47.747999999999998,"5min":114.681,"15min":114.681},"last":0.59299999999999997},{"interface":"front","average":{"1min":6.1749999999999998,"5min":3.726,"15min":2.5059999999999998},"min":{"1min":0.312,"5min":0.307,"15min":0.29999999999999999},"max":{"1min":48.015000000000001,"5min":115.366,"15min":115.366},"last":0.33100000000000002}]},{"osd":2,"last update":"Sat Jun 3 20:03:27 2023","interfaces":[{"interface":"back","average":{"1min":1.177,"5min":0.97299999999999998,"15min":0.90500000000000003},"min":{"1min":0.35399999999999998,"5min":0.35099999999999998,"15min":0.34999999999999998},"max":{"1min":4.0549999999999997,"5min":4.3529999999999998,"15min":4.3529999999999998},"last":0.58599999999999997},{"interface":"front","average":{"1min":1.784,"5min":1.2929999999999999,"15min":1.0800000000000001},"min":{"1min":0.34100000000000003,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":8.2119999999999997,"5min":8.2119999999999997,"15min":8.2119999999999997},"last":1.329}]},{"osd":3,"last update":"Sat Jun 3 20:03:10 2023","interfaces":[{"interface":"back","average":{"1min":4.6360000000000001,"5min":4.7800000000000002,"15min":2.2719999999999998},"min":{"1min":0.35299999999999998,"5min":0.34599999999999997,"15min":0.34599999999999997},"max":{"1min":48.308,"5min":261.14100000000002,"15min":261.14100000000002},"last":0.52700000000000002},{"interface":"front","average":{"1min":3.597,"5min":4.4240000000000004,"15min":2.1349999999999998},"min":{"1min":0.35399999999999998,"5min":0.35399999999999998,"15min":0.35399999999999998},"max":{"1min":48.124000000000002,"5min":261.27100000000002,"15min":261.27100000000002},"last":0.61199999999999999}]},{"osd":4,"last update":"Sat Jun 3 20:02:41 2023","interfaces":[{"interface":"back","average":{"1min":2.089,"5min":1.3260000000000001,"15min":1.1100000000000001},"min":{"1min":0.378,"5min":0.378,"15min":0.378},"max":{"1min":10.162000000000001,"5min":10.162000000000001,"15min":10.162000000000001},"last":0.46100000000000002},{"interface":"front","average":{"1min":0.94299999999999995,"5min":1.014,"15min":1.1299999999999999},"min":{"1min":0.39400000000000002,"5min":0.39400000000000002,"15min":0.39400000000000002},"max":{"1min":4.2480000000000002,"5min":4.2480000000000002,"15min":4.2480000000000002},"last":0.377}]},{"osd":6,"last update":"Sat Jun 3 20:03:27 2023","interfaces":[{"interface":"back","average":{"1min":2.5419999999999998,"5min":1.6439999999999999,"15min":1.4450000000000001},"min":{"1min":0.56100000000000005,"5min":0.41199999999999998,"15min":0.33000000000000002},"max":{"1min":10.278,"5min":10.278,"15min":12.731999999999999},"last":0.48999999999999999},{"interface":"front","average":{"1min":2.4060000000000001,"5min":1.712,"15min":1.532},"min":{"1min":0.438,"5min":0.438,"15min":0.32800000000000001},"max":{"1min":13.714,"5min":13.714,"15min":13.714},"last":0.54400000000000004}]},{"osd":7,"last update":"Sat Jun 3 20:03:14 2023","interfaces":[{"interface":"back","average":{"1min":3.46,"5min":1.9430000000000001,"15min":1.655},"min":{"1min":0.63600000000000001,"5min":0.503,"15min":0.379},"max":{"1min":10.192,"5min":10.192,"15min":11.297000000000001},"last":0.50800000000000001},{"interface":"front","average":{"1min":1.4379999999999999,"5min":1.2709999999999999,"15min":1.234},"min":{"1min":0.50600000000000001,"5min":0.43099999999999999,"15min":0.32100000000000001},"max":{"1min":4.2839999999999998,"5min":5.8719999999999999,"15min":7.4720000000000004},"last":0.40300000000000002}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6928,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4862,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":10652,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4786,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4932,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":748290048,"data_stored":748290048,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":1698631680,"data_stored":1698627608,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":552665088,"data_stored":552665088,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":1520173056,"data_stored":1520173056,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":537133056,"data_stored":537133056,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":1718620160,"data_stored":1718616088,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":578617344,"data_stored":578617344,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":1708003328,"data_stored":1707999256,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-06-03T20:03:47.243 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph pg dump --format=json 2023-06-03T20:03:47.303 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:03:47.337 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:03:47.341 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:03:47.341 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:03:47.442 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:03:47.479 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:03:47.481 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:03:47.482 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:03:47.583 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:03:47.610 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:03:47.613 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:03:47.614 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:03:47.714 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:03:47.754 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:03:47.757 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:03:47.758 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:03:47.859 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:03:47.898 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:03:47.903 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:03:47.904 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:03:48.005 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:03:48.012 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:03:47 smithi089 bash[78452]: cluster 1685822626.4963806 mgr.y (mgr.34104) 825 : cluster 0 pgmap v745: 89 pgs: 89 active+clean; 2.6 GiB data, 9.6 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 645 op/s 2023-06-03T20:03:48.013 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:03:47 smithi089 bash[75068]: cluster 1685822626.4963806 mgr.y (mgr.34104) 825 : cluster 0 pgmap v745: 89 pgs: 89 active+clean; 2.6 GiB data, 9.6 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 645 op/s 2023-06-03T20:03:48.044 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:03:48.046 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:03:48.047 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:03:48.083 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:03:47 smithi164 bash[59462]: cluster 1685822626.4963806 mgr.y (mgr.34104) 825 : cluster 0 pgmap v745: 89 pgs: 89 active+clean; 2.6 GiB data, 9.6 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 645 op/s 2023-06-03T20:03:48.148 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:03:48.179 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:03:48.182 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:03:48.183 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:03:48.283 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:03:48.323 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:03:48.329 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:03:48.330 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:03:48.431 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:03:48.466 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:03:48.470 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:03:48.471 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:03:48.572 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:03:48.616 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:03:48.618 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:03:48.619 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:03:48.720 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:03:48.758 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:03:48.764 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:03:48.765 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:03:48.866 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:03:48.902 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:03:48.911 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:03:48.911 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:03:49.012 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:03:49.049 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:03:49.054 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:03:49.055 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:03:49.155 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:03:49.204 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:03:49.206 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:03:49.207 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:03:49.308 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:03:49.348 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:03:49.354 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:03:49.355 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:03:49.456 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:03:49.491 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:03:49.498 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:03:49.498 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:03:49.599 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:03:49.638 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:03:49.645 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:03:49.647 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:03:49.748 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:03:49.794 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:03:49.796 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:03:49.797 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:03:49.898 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:03:49.956 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:03:49.960 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:03:49.961 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:03:50.062 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:03:50.083 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:03:49 smithi164 bash[59462]: cluster 1685822628.4970853 mgr.y (mgr.34104) 826 : cluster 0 pgmap v746: 89 pgs: 89 active+clean; 2.5 GiB data, 9.2 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 792 op/s 2023-06-03T20:03:50.097 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:03:50.100 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:03:49 smithi089 bash[78452]: cluster 1685822628.4970853 mgr.y (mgr.34104) 826 : cluster 0 pgmap v746: 89 pgs: 89 active+clean; 2.5 GiB data, 9.2 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 792 op/s 2023-06-03T20:03:50.100 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:03:49 smithi089 bash[75068]: cluster 1685822628.4970853 mgr.y (mgr.34104) 826 : cluster 0 pgmap v746: 89 pgs: 89 active+clean; 2.5 GiB data, 9.2 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 792 op/s 2023-06-03T20:03:50.100 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:03:50.101 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:03:50.116 INFO:teuthology.orchestra.run.smithi089.stdout:osd.0: {} 2023-06-03T20:03:50.161 INFO:teuthology.orchestra.run.smithi089.stdout:osd.1: {} 2023-06-03T20:03:50.196 INFO:teuthology.orchestra.run.smithi089.stdout:osd.2: {} 2023-06-03T20:03:50.202 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:03:50.234 INFO:teuthology.orchestra.run.smithi089.stdout:osd.3: {} 2023-06-03T20:03:50.237 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:03:50.242 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:03:50.243 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:03:50.270 INFO:teuthology.orchestra.run.smithi089.stdout:osd.4: {} 2023-06-03T20:03:50.300 INFO:teuthology.orchestra.run.smithi089.stdout:osd.5: {} 2023-06-03T20:03:50.341 INFO:teuthology.orchestra.run.smithi089.stdout:osd.6: {} 2023-06-03T20:03:50.344 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:03:50.371 INFO:teuthology.orchestra.run.smithi089.stdout:osd.7: {} 2023-06-03T20:03:50.373 INFO:teuthology.orchestra.run.smithi089.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-06-03T20:03:50.374 INFO:teuthology.orchestra.run.smithi089.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-06-03T20:03:50.374 INFO:teuthology.orchestra.run.smithi089.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-06-03T20:03:50.374 INFO:teuthology.orchestra.run.smithi089.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-06-03T20:03:50.375 INFO:teuthology.orchestra.run.smithi089.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-06-03T20:03:50.375 INFO:teuthology.orchestra.run.smithi089.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-06-03T20:03:50.375 INFO:teuthology.orchestra.run.smithi089.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-06-03T20:03:50.376 INFO:teuthology.orchestra.run.smithi089.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-06-03T20:03:50.376 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:03:50.380 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:03:50.381 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:03:50.483 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:03:50.523 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:03:50.526 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:03:50.527 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:03:50.628 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:03:50.667 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:03:50.673 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:03:50.674 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:03:50.775 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:03:50.812 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:03:50.816 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:03:50.817 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:03:50.834 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:03:50 smithi089 bash[75068]: audit 1685822629.9966962 mon.a (mon.0) 737 : audit 1 from='client.? 172.21.15.89:0/4192029689' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-06-03T20:03:50.834 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:03:50 smithi089 bash[75068]: audit 1685822630.0622647 mon.c (mon.2) 409 : audit 0 from='client.? 172.21.15.89:0/3578378412' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-03T20:03:50.842 INFO:teuthology.orchestra.run.smithi089.stderr:noscrub is set 2023-06-03T20:03:50.918 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:03:50.927 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:03:50 smithi164 bash[59462]: audit 1685822629.9966962 mon.a (mon.0) 737 : audit 1 from='client.? 172.21.15.89:0/4192029689' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-06-03T20:03:50.927 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:03:50 smithi164 bash[59462]: audit 1685822630.0622647 mon.c (mon.2) 409 : audit 0 from='client.? 172.21.15.89:0/3578378412' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-03T20:03:50.960 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:03:50.966 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:03:50.966 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:03:51.067 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:03:51.109 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:03:51.114 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:03:50 smithi089 bash[78452]: audit 1685822629.9966962 mon.a (mon.0) 737 : audit 1 from='client.? 172.21.15.89:0/4192029689' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-06-03T20:03:51.115 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:03:50 smithi089 bash[78452]: audit 1685822630.0622647 mon.c (mon.2) 409 : audit 0 from='client.? 172.21.15.89:0/3578378412' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-03T20:03:51.115 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:03:51.116 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:03:51.217 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:03:51.250 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:03:51.257 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:03:51.257 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:03:51.358 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:03:51.405 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:03:51.407 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:03:51.408 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:03:51.508 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:03:51.548 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:03:51.551 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:03:51.552 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:03:51.653 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:03:51.685 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:03:51.688 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:03:51.688 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:03:51.789 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:03:51.818 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:03:51.821 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:03:51.821 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:03:51.923 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:03:51.932 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:03:51 smithi164 bash[59462]: cluster 1685822630.4977078 mgr.y (mgr.34104) 827 : cluster 0 pgmap v747: 89 pgs: 89 active+clean; 2.4 GiB data, 9.0 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 682 op/s 2023-06-03T20:03:51.932 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:03:51 smithi164 bash[59462]: cluster 1685822630.817999 mon.a (mon.0) 738 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-06-03T20:03:51.932 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:03:51 smithi164 bash[59462]: cluster 1685822630.8268003 osd.6 (osd.6) 62 : cluster 3 failed to encode map e143 with expected crc 2023-06-03T20:03:51.933 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:03:51 smithi164 bash[59462]: cluster 1685822630.8310106 osd.5 (osd.5) 67 : cluster 3 failed to encode map e143 with expected crc 2023-06-03T20:03:51.933 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:03:51 smithi164 bash[59462]: audit 1685822630.8326414 mon.a (mon.0) 739 : audit 1 from='client.? 172.21.15.89:0/4192029689' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-06-03T20:03:51.933 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:03:51 smithi164 bash[59462]: cluster 1685822630.832694 mon.a (mon.0) 740 : cluster 0 osdmap e143: 8 total, 8 up, 8 in 2023-06-03T20:03:51.933 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:03:51 smithi164 bash[59462]: cluster 1685822630.8420882 osd.7 (osd.7) 55 : cluster 3 failed to encode map e143 with expected crc 2023-06-03T20:03:51.963 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:03:51.966 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:03:51.966 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:03:52.067 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:03:52.075 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:03:51 smithi089 bash[78452]: cluster 1685822630.4977078 mgr.y (mgr.34104) 827 : cluster 0 pgmap v747: 89 pgs: 89 active+clean; 2.4 GiB data, 9.0 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 682 op/s 2023-06-03T20:03:52.075 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:03:51 smithi089 bash[78452]: cluster 1685822630.817999 mon.a (mon.0) 738 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-06-03T20:03:52.075 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:03:51 smithi089 bash[78452]: cluster 1685822630.8268003 osd.6 (osd.6) 62 : cluster 3 failed to encode map e143 with expected crc 2023-06-03T20:03:52.075 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:03:51 smithi089 bash[78452]: cluster 1685822630.8310106 osd.5 (osd.5) 67 : cluster 3 failed to encode map e143 with expected crc 2023-06-03T20:03:52.075 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:03:51 smithi089 bash[78452]: audit 1685822630.8326414 mon.a (mon.0) 739 : audit 1 from='client.? 172.21.15.89:0/4192029689' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-06-03T20:03:52.076 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:03:51 smithi089 bash[78452]: cluster 1685822630.832694 mon.a (mon.0) 740 : cluster 0 osdmap e143: 8 total, 8 up, 8 in 2023-06-03T20:03:52.076 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:03:51 smithi089 bash[78452]: cluster 1685822630.8420882 osd.7 (osd.7) 55 : cluster 3 failed to encode map e143 with expected crc 2023-06-03T20:03:52.076 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:03:51 smithi089 bash[75068]: cluster 1685822630.4977078 mgr.y (mgr.34104) 827 : cluster 0 pgmap v747: 89 pgs: 89 active+clean; 2.4 GiB data, 9.0 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 682 op/s 2023-06-03T20:03:52.076 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:03:51 smithi089 bash[75068]: cluster 1685822630.817999 mon.a (mon.0) 738 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-06-03T20:03:52.076 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:03:51 smithi089 bash[75068]: cluster 1685822630.8268003 osd.6 (osd.6) 62 : cluster 3 failed to encode map e143 with expected crc 2023-06-03T20:03:52.076 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:03:51 smithi089 bash[75068]: cluster 1685822630.8310106 osd.5 (osd.5) 67 : cluster 3 failed to encode map e143 with expected crc 2023-06-03T20:03:52.077 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:03:51 smithi089 bash[75068]: audit 1685822630.8326414 mon.a (mon.0) 739 : audit 1 from='client.? 172.21.15.89:0/4192029689' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-06-03T20:03:52.077 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:03:51 smithi089 bash[75068]: cluster 1685822630.832694 mon.a (mon.0) 740 : cluster 0 osdmap e143: 8 total, 8 up, 8 in 2023-06-03T20:03:52.077 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:03:51 smithi089 bash[75068]: cluster 1685822630.8420882 osd.7 (osd.7) 55 : cluster 3 failed to encode map e143 with expected crc 2023-06-03T20:03:52.116 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:03:52.122 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:03:52.123 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:03:52.224 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:03:52.256 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:03:52.259 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:03:52.259 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:03:52.361 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:03:52.392 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:03:52.397 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:03:52.397 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:03:52.498 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:03:52.543 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:03:52.545 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:03:52.546 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:03:52.646 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:03:52.676 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:03:52.681 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:03:52.682 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:03:52.783 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:03:52.812 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:03:52.815 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:03:52.816 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:03:52.917 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:03:52.952 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:03:52.957 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:03:52.958 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:03:53.059 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:03:53.092 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:03:53.093 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:03:53.093 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:03:53.115 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:03:53 smithi089 bash[75068]: audit 1685822632.1166596 mon.a (mon.0) 741 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:03:53.115 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:03:53 smithi089 bash[75068]: audit 1685822632.1198583 mon.c (mon.2) 410 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T20:03:53.115 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:03:53 smithi089 bash[75068]: cluster 1685822632.4982197 mgr.y (mgr.34104) 828 : cluster 0 pgmap v749: 89 pgs: 89 active+clean; 2.4 GiB data, 9.0 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 710 op/s 2023-06-03T20:03:53.194 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:03:53.202 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:03:53 smithi164 bash[59462]: audit 1685822632.1166596 mon.a (mon.0) 741 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:03:53.202 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:03:53 smithi164 bash[59462]: audit 1685822632.1198583 mon.c (mon.2) 410 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T20:03:53.202 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:03:53 smithi164 bash[59462]: cluster 1685822632.4982197 mgr.y (mgr.34104) 828 : cluster 0 pgmap v749: 89 pgs: 89 active+clean; 2.4 GiB data, 9.0 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 710 op/s 2023-06-03T20:03:53.232 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:03:53.235 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:03:53.236 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:03:53.337 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:03:53.369 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:03:53.373 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:03:53 smithi089 bash[78452]: audit 1685822632.1166596 mon.a (mon.0) 741 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:03:53.373 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:03:53 smithi089 bash[78452]: audit 1685822632.1198583 mon.c (mon.2) 410 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T20:03:53.374 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:03:53 smithi089 bash[78452]: cluster 1685822632.4982197 mgr.y (mgr.34104) 828 : cluster 0 pgmap v749: 89 pgs: 89 active+clean; 2.4 GiB data, 9.0 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 710 op/s 2023-06-03T20:03:53.374 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:03:53.375 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:03:53.448 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/mon.a/config 2023-06-03T20:03:53.456 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/mon.a/config 2023-06-03T20:03:53.476 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:03:53.529 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:03:53.532 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:03:53.532 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:03:53.633 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:03:53.673 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:03:53.675 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:03:53.676 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:03:53.777 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:03:53.812 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:03:53.814 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:03:53.815 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:03:53.916 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:03:53.937 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-06-03T20:03:53.962 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:03:53.966 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:03:53.966 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:03:54.067 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:03:54.109 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:03:54.115 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:03:54.115 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:03:54.216 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:03:54.248 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:03:54.253 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:03:54.254 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:03:54.355 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:03:54.399 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:03:54.406 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:03:54.407 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:03:54.508 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:03:54.546 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:03:54.552 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:03:54.553 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:03:54.654 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:03:54.686 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:03:54.692 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:03:54.693 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:03:54.785 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph osd set nodeep-scrub 2023-06-03T20:03:54.794 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:03:54.826 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:03:54.829 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:03:54.830 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:03:54.931 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:03:54.985 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:03:54.991 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:03:54.992 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:03:55.093 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:03:55.123 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:03:55.126 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:03:55.127 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:03:55.228 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:03:55.269 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:03:55.274 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:03:55.274 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:03:55.375 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:03:55.410 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:03:55.418 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:03:55.419 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:03:55.520 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:03:55.561 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:03:55 smithi164 bash[59462]: cluster 1685822634.4989028 mgr.y (mgr.34104) 829 : cluster 0 pgmap v750: 89 pgs: 89 active+clean; 2.2 GiB data, 8.4 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 694 op/s 2023-06-03T20:03:55.566 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:03:55.574 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:03:55.575 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:03:55.676 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:03:55.706 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:03:55.708 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:03:55.709 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:03:55.778 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:03:55 smithi089 bash[78452]: cluster 1685822634.4989028 mgr.y (mgr.34104) 829 : cluster 0 pgmap v750: 89 pgs: 89 active+clean; 2.2 GiB data, 8.4 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 694 op/s 2023-06-03T20:03:55.779 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:03:55 smithi089 bash[75068]: cluster 1685822634.4989028 mgr.y (mgr.34104) 829 : cluster 0 pgmap v750: 89 pgs: 89 active+clean; 2.2 GiB data, 8.4 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 694 op/s 2023-06-03T20:03:55.810 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:03:55.861 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:03:55.863 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:03:55.864 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:03:55.965 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:03:55.994 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:03:56.000 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:03:56.001 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:03:56.102 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:03:56.139 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:03:56.142 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:03:56.143 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:03:56.244 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:03:56.301 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:03:56.303 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:03:56.304 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:03:56.404 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:03:56.444 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:03:56.450 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:03:56.451 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:03:56.528 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 20:03:56 smithi089 bash[71163]: ::ffff:172.21.15.164 - - [03/Jun/2023:20:03:56] "GET /metrics HTTP/1.1" 200 228310 "" "Prometheus/2.33.4" 2023-06-03T20:03:56.552 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:03:56.600 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:03:56.603 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:03:56.603 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:03:56.704 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:03:56.750 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:03:56.754 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:03:56.755 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:03:56.856 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:03:56.891 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:03:56.894 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:03:56.895 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:03:56.996 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:03:57.043 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:03:57.045 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:03:57.046 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:03:57.147 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:03:57.186 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:03:57.190 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:03:57.191 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:03:57.292 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:03:57.328 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:03:57.336 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:03:57.337 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:03:57.438 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:03:57.484 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:03:57.491 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:03:57.492 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:03:57.593 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:03:57.624 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:03:57.628 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:03:57.629 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:03:57.729 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:03:57.779 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:03:57.782 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:03:57.783 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:03:57.833 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:03:57 smithi164 bash[59462]: cluster 1685822636.4994934 mgr.y (mgr.34104) 830 : cluster 0 pgmap v751: 89 pgs: 89 active+clean; 2.2 GiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 697 op/s 2023-06-03T20:03:57.884 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:03:57.928 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:03:57.932 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:03:57.932 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:03:58.028 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:03:57 smithi089 bash[78452]: cluster 1685822636.4994934 mgr.y (mgr.34104) 830 : cluster 0 pgmap v751: 89 pgs: 89 active+clean; 2.2 GiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 697 op/s 2023-06-03T20:03:58.029 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:03:57 smithi089 bash[75068]: cluster 1685822636.4994934 mgr.y (mgr.34104) 830 : cluster 0 pgmap v751: 89 pgs: 89 active+clean; 2.2 GiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 697 op/s 2023-06-03T20:03:58.033 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:03:58.082 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:03:58.085 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:03:58.085 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:03:58.186 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:03:58.223 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:03:58.226 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:03:58.226 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:03:58.327 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:03:58.382 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:03:58.388 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:03:58.389 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:03:58.490 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:03:58.538 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:03:58.541 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:03:58.542 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:03:58.643 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:03:58.673 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:03:58.678 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:03:58.679 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:03:58.780 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:03:58.814 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:03:58.816 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:03:58.817 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:03:58.918 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:03:58.959 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:03:58.967 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:03:58.968 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:03:59.069 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:03:59.096 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:03:59.101 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:03:59.102 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:03:59.203 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:03:59.250 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:03:59.253 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:03:59.253 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:03:59.354 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:03:59.388 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:03:59.396 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:03:59.397 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:03:59.498 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:03:59.544 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:03:59.547 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:03:59.548 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:03:59.648 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:03:59.700 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:03:59.702 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:03:59.703 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:03:59.804 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:03:59.842 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:03:59.845 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:03:59.845 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:03:59.911 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:03:59 smithi089 bash[75068]: cluster 1685822638.500039 mgr.y (mgr.34104) 831 : cluster 0 pgmap v752: 89 pgs: 89 active+clean; 2.0 GiB data, 7.9 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 698 op/s 2023-06-03T20:03:59.912 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:03:59 smithi089 bash[78452]: cluster 1685822638.500039 mgr.y (mgr.34104) 831 : cluster 0 pgmap v752: 89 pgs: 89 active+clean; 2.0 GiB data, 7.9 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 698 op/s 2023-06-03T20:03:59.946 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:03:59.954 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:03:59 smithi164 bash[59462]: cluster 1685822638.500039 mgr.y (mgr.34104) 831 : cluster 0 pgmap v752: 89 pgs: 89 active+clean; 2.0 GiB data, 7.9 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 698 op/s 2023-06-03T20:03:59.992 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:03:59.996 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:03:59.997 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:04:00.098 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:04:00.132 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:04:00.134 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:04:00.135 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:04:00.236 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:04:00.279 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:04:00.285 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:04:00.285 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:04:00.386 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:04:00.428 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:04:00.434 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:04:00.434 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:04:00.536 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:04:00.567 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:04:00.573 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:04:00.573 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:04:00.674 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:04:00.707 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:04:00.709 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:04:00.710 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:04:00.811 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:04:00.857 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:04:00.862 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:04:00.863 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:04:00.964 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:04:00.994 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:04:01.001 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:04:01.002 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:04:01.103 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:04:01.141 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:04:01.148 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:04:01.149 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:04:01.250 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:04:01.278 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:04:01.280 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:04:01.280 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:04:01.381 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:04:01.411 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:04:01.414 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:04:01.415 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:04:01.431 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/mon.a/config 2023-06-03T20:04:01.431 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/mon.a/config 2023-06-03T20:04:01.490 INFO:teuthology.orchestra.run.smithi089.stdout:ERROR: (22) Invalid argument 2023-06-03T20:04:01.490 INFO:teuthology.orchestra.run.smithi089.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-06-03T20:04:01.516 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:04:01.545 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:04:01.553 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:04:01.553 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:04:01.654 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:04:01.699 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:04:01.701 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:04:01.702 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:04:01.778 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:04:01 smithi089 bash[75068]: cluster 1685822640.5005457 mgr.y (mgr.34104) 832 : cluster 0 pgmap v753: 89 pgs: 89 active+clean; 2.0 GiB data, 7.7 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 700 op/s 2023-06-03T20:04:01.779 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:04:01 smithi089 bash[78452]: cluster 1685822640.5005457 mgr.y (mgr.34104) 832 : cluster 0 pgmap v753: 89 pgs: 89 active+clean; 2.0 GiB data, 7.7 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 700 op/s 2023-06-03T20:04:01.803 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:04:01.831 INFO:teuthology.orchestra.run.smithi089.stdout: 2023-06-03T20:04:01.832 INFO:teuthology.orchestra.run.smithi089.stderr:dumped all 2023-06-03T20:04:01.833 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:04:01 smithi164 bash[59462]: cluster 1685822640.5005457 mgr.y (mgr.34104) 832 : cluster 0 pgmap v753: 89 pgs: 89 active+clean; 2.0 GiB data, 7.7 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 700 op/s 2023-06-03T20:04:01.844 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:04:01.849 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:04:01.850 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:04:01.951 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:04:01.981 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:04:01.983 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:04:01.984 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:04:02.085 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:04:02.116 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:04:02.119 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:04:02.120 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:04:02.221 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:04:02.255 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:04:02.257 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:04:02.257 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:04:02.358 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:04:02.392 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:04:02.394 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:04:02.395 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:04:02.496 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:04:02.532 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:04:02.535 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:04:02.536 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:04:02.638 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:04:02.665 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:04:02.667 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:04:02.668 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:04:02.739 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:04:02 smithi089 bash[75068]: audit 1685822641.8019168 mgr.y (mgr.34104) 833 : audit 0 from='client.35886 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-06-03T20:04:02.739 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:04:02 smithi089 bash[78452]: audit 1685822641.8019168 mgr.y (mgr.34104) 833 : audit 0 from='client.35886 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-06-03T20:04:02.770 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:04:02.823 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:04:02.826 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:04:02.826 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:04:02.927 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:04:02.934 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:04:02 smithi164 bash[59462]: audit 1685822641.8019168 mgr.y (mgr.34104) 833 : audit 0 from='client.35886 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-06-03T20:04:02.953 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:04:02.955 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:04:02.956 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:04:03.057 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:04:03.095 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:04:03.100 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:04:03.101 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:04:03.202 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:04:03.229 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:04:03.234 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:04:03.235 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:04:03.336 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:04:03.357 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:04:03.358 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:04:03.359 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:04:03.460 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:04:03.499 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:04:03.502 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:04:03.502 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:04:03.603 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:04:03.644 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:04:03.650 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:04:03.650 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:04:03.751 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:04:03.760 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:04:03 smithi089 bash[75068]: cluster 1685822642.5010474 mgr.y (mgr.34104) 834 : cluster 0 pgmap v754: 89 pgs: 89 active+clean; 2.0 GiB data, 7.7 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 600 op/s 2023-06-03T20:04:03.760 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:04:03 smithi089 bash[78452]: cluster 1685822642.5010474 mgr.y (mgr.34104) 834 : cluster 0 pgmap v754: 89 pgs: 89 active+clean; 2.0 GiB data, 7.7 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 600 op/s 2023-06-03T20:04:03.796 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:04:03.802 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:04:03.803 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:04:03.904 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:04:03.936 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:04:03.939 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:04:03.940 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:04:04.041 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:04:04.083 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:04:03 smithi164 bash[59462]: cluster 1685822642.5010474 mgr.y (mgr.34104) 834 : cluster 0 pgmap v754: 89 pgs: 89 active+clean; 2.0 GiB data, 7.7 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 600 op/s 2023-06-03T20:04:04.084 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:04:04.086 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:04:04.087 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:04:04.188 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:04:04.228 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:04:04.230 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:04:04.230 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:04:04.331 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:04:04.376 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:04:04.380 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:04:04.381 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:04:04.481 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:04:04.520 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:04:04.529 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:04:04.529 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:04:04.630 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:04:04.660 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:04:04.665 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:04:04.666 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:04:04.767 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:04:04.795 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:04:04.800 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:04:04.801 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:04:04.901 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:04:04.933 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:04:04.942 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:04:04.943 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:04:05.043 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:04:05.091 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:04:05.094 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:04:05.094 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:04:05.195 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:04:05.242 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:04:05.247 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:04:05.248 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:04:05.349 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:04:05.377 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:04:05.379 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:04:05.380 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:04:05.481 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:04:05.522 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:04:05.535 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:04:05.535 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:04:05.536 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph daemon osd.3 dump_historic_ops 2023-06-03T20:04:05.550 INFO:teuthology.orchestra.run.smithi089.stdout:{"pg_ready":true,"pg_map":{"version":753,"stamp":"2023-06-03T20:04:00.500203+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":2105284518,"num_objects":32140,"num_object_clones":0,"num_object_copies":96420,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":32140,"num_whiteouts":0,"num_read":309,"num_read_kb":513,"num_write":188409,"num_write_kb":7050831,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139643,"ondisk_log_size":139643,"up":267,"acting":267,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":267,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":8083464,"kb_used_data":6273096,"kb_used_omap":30,"kb_used_meta":1810017,"kb_avail":741877752,"statfs":{"total":767960285184,"available":759682818048,"internally_reserved":0,"allocated":6423650304,"data_stored":6419473616,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":31502,"internal_metadata":1853457650},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":5,"apply_latency_ms":5,"commit_latency_ns":5000000,"apply_latency_ns":5000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-459472896,"num_objects":-7011,"num_object_clones":0,"num_object_copies":-21033,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":-7011,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7011,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.002930"},"pg_stats":[{"pgid":"3.1a","version":"60'1","reported_seq":130,"reported_epoch":142,"state":"active+clean","last_fresh":"2023-06-03T20:03:28.894591+0000","last_change":"2023-06-03T19:57:28.467559+0000","last_active":"2023-06-03T20:03:28.894591+0000","last_peered":"2023-06-03T20:03:28.894591+0000","last_clean":"2023-06-03T20:03:28.894591+0000","last_became_active":"2023-06-03T19:57:28.467077+0000","last_became_peered":"2023-06-03T19:57:28.467077+0000","last_unstale":"2023-06-03T20:03:28.894591+0000","last_undegraded":"2023-06-03T20:03:28.894591+0000","last_fullsized":"2023-06-03T20:03:28.894591+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T21:46:02.795576+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":123,"reported_epoch":142,"state":"active+clean","last_fresh":"2023-06-03T20:03:28.906396+0000","last_change":"2023-06-03T19:57:28.470287+0000","last_active":"2023-06-03T20:03:28.906396+0000","last_peered":"2023-06-03T20:03:28.906396+0000","last_clean":"2023-06-03T20:03:28.906396+0000","last_became_active":"2023-06-03T19:57:28.469477+0000","last_became_peered":"2023-06-03T19:57:28.469477+0000","last_unstale":"2023-06-03T20:03:28.906396+0000","last_undegraded":"2023-06-03T20:03:28.906396+0000","last_fullsized":"2023-06-03T20:03:28.906396+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T06:48:19.229190+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":136,"reported_epoch":142,"state":"active+clean","last_fresh":"2023-06-03T20:03:28.907806+0000","last_change":"2023-06-03T19:57:28.473350+0000","last_active":"2023-06-03T20:03:28.907806+0000","last_peered":"2023-06-03T20:03:28.907806+0000","last_clean":"2023-06-03T20:03:28.907806+0000","last_became_active":"2023-06-03T19:57:28.472757+0000","last_became_peered":"2023-06-03T19:57:28.472757+0000","last_unstale":"2023-06-03T20:03:28.907806+0000","last_undegraded":"2023-06-03T20:03:28.907806+0000","last_fullsized":"2023-06-03T20:03:28.907806+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T19:58:10.432188+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":101,"reported_epoch":142,"state":"active+clean","last_fresh":"2023-06-03T20:03:28.893473+0000","last_change":"2023-06-03T19:56:54.507653+0000","last_active":"2023-06-03T20:03:28.893473+0000","last_peered":"2023-06-03T20:03:28.893473+0000","last_clean":"2023-06-03T20:03:28.893473+0000","last_became_active":"2023-06-03T19:56:54.507350+0000","last_became_peered":"2023-06-03T19:56:54.507350+0000","last_unstale":"2023-06-03T20:03:28.893473+0000","last_undegraded":"2023-06-03T20:03:28.893473+0000","last_fullsized":"2023-06-03T20:03:28.893473+0000","mapping_epoch":119,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T22:33:20.065844+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":124,"reported_epoch":142,"state":"active+clean","last_fresh":"2023-06-03T20:03:28.905936+0000","last_change":"2023-06-03T19:56:54.512932+0000","last_active":"2023-06-03T20:03:28.905936+0000","last_peered":"2023-06-03T20:03:28.905936+0000","last_clean":"2023-06-03T20:03:28.905936+0000","last_became_active":"2023-06-03T19:56:54.511992+0000","last_became_peered":"2023-06-03T19:56:54.511992+0000","last_unstale":"2023-06-03T20:03:28.905936+0000","last_undegraded":"2023-06-03T20:03:28.905936+0000","last_fullsized":"2023-06-03T20:03:28.905936+0000","mapping_epoch":119,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T21:01:35.069911+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":109,"reported_epoch":142,"state":"active+clean","last_fresh":"2023-06-03T20:03:28.901661+0000","last_change":"2023-06-03T19:51:59.359522+0000","last_active":"2023-06-03T20:03:28.901661+0000","last_peered":"2023-06-03T20:03:28.901661+0000","last_clean":"2023-06-03T20:03:28.901661+0000","last_became_active":"2023-06-03T19:51:59.359202+0000","last_became_peered":"2023-06-03T19:51:59.359202+0000","last_unstale":"2023-06-03T20:03:28.901661+0000","last_undegraded":"2023-06-03T20:03:28.901661+0000","last_fullsized":"2023-06-03T20:03:28.901661+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":99,"reported_epoch":143,"state":"active+clean","last_fresh":"2023-06-03T20:03:50.857399+0000","last_change":"2023-06-03T19:56:54.509490+0000","last_active":"2023-06-03T20:03:50.857399+0000","last_peered":"2023-06-03T20:03:50.857399+0000","last_clean":"2023-06-03T20:03:50.857399+0000","last_became_active":"2023-06-03T19:56:54.509145+0000","last_became_peered":"2023-06-03T19:56:54.509145+0000","last_unstale":"2023-06-03T20:03:50.857399+0000","last_undegraded":"2023-06-03T20:03:50.857399+0000","last_fullsized":"2023-06-03T20:03:50.857399+0000","mapping_epoch":119,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T07:12:03.964110+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":109,"reported_epoch":143,"state":"active+clean","last_fresh":"2023-06-03T20:03:50.848860+0000","last_change":"2023-06-03T19:56:54.511325+0000","last_active":"2023-06-03T20:03:50.848860+0000","last_peered":"2023-06-03T20:03:50.848860+0000","last_clean":"2023-06-03T20:03:50.848860+0000","last_became_active":"2023-06-03T19:56:54.511033+0000","last_became_peered":"2023-06-03T19:56:54.511033+0000","last_unstale":"2023-06-03T20:03:50.848860+0000","last_undegraded":"2023-06-03T20:03:50.848860+0000","last_fullsized":"2023-06-03T20:03:50.848860+0000","mapping_epoch":119,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T23:50:08.711478+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":110,"reported_epoch":142,"state":"active+clean","last_fresh":"2023-06-03T20:03:28.915202+0000","last_change":"2023-06-03T19:51:59.374143+0000","last_active":"2023-06-03T20:03:28.915202+0000","last_peered":"2023-06-03T20:03:28.915202+0000","last_clean":"2023-06-03T20:03:28.915202+0000","last_became_active":"2023-06-03T19:51:59.373146+0000","last_became_peered":"2023-06-03T19:51:59.373146+0000","last_unstale":"2023-06-03T20:03:28.915202+0000","last_undegraded":"2023-06-03T20:03:28.915202+0000","last_fullsized":"2023-06-03T20:03:28.915202+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":90,"reported_epoch":143,"state":"active+clean","last_fresh":"2023-06-03T20:03:50.849308+0000","last_change":"2023-06-03T19:56:54.510462+0000","last_active":"2023-06-03T20:03:50.849308+0000","last_peered":"2023-06-03T20:03:50.849308+0000","last_clean":"2023-06-03T20:03:50.849308+0000","last_became_active":"2023-06-03T19:56:54.509783+0000","last_became_peered":"2023-06-03T19:56:54.509783+0000","last_unstale":"2023-06-03T20:03:50.849308+0000","last_undegraded":"2023-06-03T20:03:50.849308+0000","last_fullsized":"2023-06-03T20:03:50.849308+0000","mapping_epoch":119,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T00:16:09.624276+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"60'3","reported_seq":128,"reported_epoch":142,"state":"active+clean","last_fresh":"2023-06-03T20:03:28.915098+0000","last_change":"2023-06-03T19:57:28.466401+0000","last_active":"2023-06-03T20:03:28.915098+0000","last_peered":"2023-06-03T20:03:28.915098+0000","last_clean":"2023-06-03T20:03:28.915098+0000","last_became_active":"2023-06-03T19:57:28.465452+0000","last_became_peered":"2023-06-03T19:57:28.465452+0000","last_unstale":"2023-06-03T20:03:28.915098+0000","last_undegraded":"2023-06-03T20:03:28.915098+0000","last_fullsized":"2023-06-03T20:03:28.915098+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":81,"reported_epoch":142,"state":"active+clean","last_fresh":"2023-06-03T20:03:38.569995+0000","last_change":"2023-06-03T19:56:11.753995+0000","last_active":"2023-06-03T20:03:38.569995+0000","last_peered":"2023-06-03T20:03:38.569995+0000","last_clean":"2023-06-03T20:03:38.569995+0000","last_became_active":"2023-06-03T19:56:11.753676+0000","last_became_peered":"2023-06-03T19:56:11.753676+0000","last_unstale":"2023-06-03T20:03:38.569995+0000","last_undegraded":"2023-06-03T20:03:38.569995+0000","last_fullsized":"2023-06-03T20:03:38.569995+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":115,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T01:00:40.343647+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"77'23","reported_seq":126,"reported_epoch":142,"state":"active+clean","last_fresh":"2023-06-03T20:03:28.900942+0000","last_change":"2023-06-03T19:57:28.462522+0000","last_active":"2023-06-03T20:03:28.900942+0000","last_peered":"2023-06-03T20:03:28.900942+0000","last_clean":"2023-06-03T20:03:28.900942+0000","last_became_active":"2023-06-03T19:57:28.462048+0000","last_became_peered":"2023-06-03T19:57:28.462048+0000","last_unstale":"2023-06-03T20:03:28.900942+0000","last_undegraded":"2023-06-03T20:03:28.900942+0000","last_fullsized":"2023-06-03T20:03:28.900942+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":23,"log_dups_size":0,"ondisk_log_size":23,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T23:23:56.697534+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4716,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":23,"num_write_kb":20,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":103,"reported_epoch":142,"state":"active+clean","last_fresh":"2023-06-03T20:03:28.895217+0000","last_change":"2023-06-03T19:57:28.469676+0000","last_active":"2023-06-03T20:03:28.895217+0000","last_peered":"2023-06-03T20:03:28.895217+0000","last_clean":"2023-06-03T20:03:28.895217+0000","last_became_active":"2023-06-03T19:57:28.469328+0000","last_became_peered":"2023-06-03T19:57:28.469328+0000","last_unstale":"2023-06-03T20:03:28.895217+0000","last_undegraded":"2023-06-03T20:03:28.895217+0000","last_fullsized":"2023-06-03T20:03:28.895217+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T23:25:26.041930+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"60'2","reported_seq":127,"reported_epoch":142,"state":"active+clean","last_fresh":"2023-06-03T20:03:28.906312+0000","last_change":"2023-06-03T19:57:28.467711+0000","last_active":"2023-06-03T20:03:28.906312+0000","last_peered":"2023-06-03T20:03:28.906312+0000","last_clean":"2023-06-03T20:03:28.906312+0000","last_became_active":"2023-06-03T19:57:28.467234+0000","last_became_peered":"2023-06-03T19:57:28.467234+0000","last_unstale":"2023-06-03T20:03:28.906312+0000","last_undegraded":"2023-06-03T20:03:28.906312+0000","last_fullsized":"2023-06-03T20:03:28.906312+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":109,"reported_epoch":143,"state":"active+clean","last_fresh":"2023-06-03T20:03:50.849013+0000","last_change":"2023-06-03T19:56:54.514616+0000","last_active":"2023-06-03T20:03:50.849013+0000","last_peered":"2023-06-03T20:03:50.849013+0000","last_clean":"2023-06-03T20:03:50.849013+0000","last_became_active":"2023-06-03T19:56:54.513918+0000","last_became_peered":"2023-06-03T19:56:54.513918+0000","last_unstale":"2023-06-03T20:03:50.849013+0000","last_undegraded":"2023-06-03T20:03:50.849013+0000","last_fullsized":"2023-06-03T20:03:50.849013+0000","mapping_epoch":119,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T19:36:10.967386+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":99,"reported_epoch":142,"state":"active+clean","last_fresh":"2023-06-03T20:03:28.893336+0000","last_change":"2023-06-03T19:56:54.507479+0000","last_active":"2023-06-03T20:03:28.893336+0000","last_peered":"2023-06-03T20:03:28.893336+0000","last_clean":"2023-06-03T20:03:28.893336+0000","last_became_active":"2023-06-03T19:56:54.507149+0000","last_became_peered":"2023-06-03T19:56:54.507149+0000","last_unstale":"2023-06-03T20:03:28.893336+0000","last_undegraded":"2023-06-03T20:03:28.893336+0000","last_fullsized":"2023-06-03T20:03:28.893336+0000","mapping_epoch":119,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T20:22:36.676317+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":109,"reported_epoch":142,"state":"active+clean","last_fresh":"2023-06-03T20:03:28.918026+0000","last_change":"2023-06-03T19:56:54.512472+0000","last_active":"2023-06-03T20:03:28.918026+0000","last_peered":"2023-06-03T20:03:28.918026+0000","last_clean":"2023-06-03T20:03:28.918026+0000","last_became_active":"2023-06-03T19:56:54.512051+0000","last_became_peered":"2023-06-03T19:56:54.512051+0000","last_unstale":"2023-06-03T20:03:28.918026+0000","last_undegraded":"2023-06-03T20:03:28.918026+0000","last_fullsized":"2023-06-03T20:03:28.918026+0000","mapping_epoch":119,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":110,"reported_epoch":142,"state":"active+clean","last_fresh":"2023-06-03T20:03:28.900089+0000","last_change":"2023-06-03T19:57:28.462780+0000","last_active":"2023-06-03T20:03:28.900089+0000","last_peered":"2023-06-03T20:03:28.900089+0000","last_clean":"2023-06-03T20:03:28.900089+0000","last_became_active":"2023-06-03T19:57:28.462298+0000","last_became_peered":"2023-06-03T19:57:28.462298+0000","last_unstale":"2023-06-03T20:03:28.900089+0000","last_undegraded":"2023-06-03T20:03:28.900089+0000","last_fullsized":"2023-06-03T20:03:28.900089+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":100,"reported_epoch":143,"state":"active+clean","last_fresh":"2023-06-03T20:03:50.857297+0000","last_change":"2023-06-03T19:56:54.510289+0000","last_active":"2023-06-03T20:03:50.857297+0000","last_peered":"2023-06-03T20:03:50.857297+0000","last_clean":"2023-06-03T20:03:50.857297+0000","last_became_active":"2023-06-03T19:56:54.509905+0000","last_became_peered":"2023-06-03T19:56:54.509905+0000","last_unstale":"2023-06-03T20:03:50.857297+0000","last_undegraded":"2023-06-03T20:03:50.857297+0000","last_fullsized":"2023-06-03T20:03:50.857297+0000","mapping_epoch":119,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T02:54:26.792401+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"60'8","reported_seq":118,"reported_epoch":142,"state":"active+clean","last_fresh":"2023-06-03T20:03:28.902107+0000","last_change":"2023-06-03T19:47:06.727778+0000","last_active":"2023-06-03T20:03:28.902107+0000","last_peered":"2023-06-03T20:03:28.902107+0000","last_clean":"2023-06-03T20:03:28.902107+0000","last_became_active":"2023-06-03T19:47:06.727305+0000","last_became_peered":"2023-06-03T19:47:06.727305+0000","last_unstale":"2023-06-03T20:03:28.902107+0000","last_undegraded":"2023-06-03T20:03:28.902107+0000","last_fullsized":"2023-06-03T20:03:28.902107+0000","mapping_epoch":86,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":87,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":108,"reported_epoch":142,"state":"active+clean","last_fresh":"2023-06-03T20:03:28.907227+0000","last_change":"2023-06-03T19:56:54.512994+0000","last_active":"2023-06-03T20:03:28.907227+0000","last_peered":"2023-06-03T20:03:28.907227+0000","last_clean":"2023-06-03T20:03:28.907227+0000","last_became_active":"2023-06-03T19:56:54.512579+0000","last_became_peered":"2023-06-03T19:56:54.512579+0000","last_unstale":"2023-06-03T20:03:28.907227+0000","last_undegraded":"2023-06-03T20:03:28.907227+0000","last_fullsized":"2023-06-03T20:03:28.907227+0000","mapping_epoch":119,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T07:02:45.187961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"9.5","version":"143'11599","reported_seq":11624,"reported_epoch":143,"state":"active+clean","last_fresh":"2023-06-03T20:03:59.942288+0000","last_change":"2023-06-03T20:00:12.908073+0000","last_active":"2023-06-03T20:03:59.942288+0000","last_peered":"2023-06-03T20:03:59.942288+0000","last_clean":"2023-06-03T20:03:59.942288+0000","last_became_active":"2023-06-03T20:00:12.907738+0000","last_became_peered":"2023-06-03T20:00:12.907738+0000","last_unstale":"2023-06-03T20:03:59.942288+0000","last_undegraded":"2023-06-03T20:03:59.942288+0000","last_fullsized":"2023-06-03T20:03:59.942288+0000","mapping_epoch":127,"log_start":"131'1500","ondisk_log_start":"131'1500","created":127,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T20:00:11.873149+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T20:00:11.873149+0000","last_clean_scrub_stamp":"2023-06-03T20:00:11.873149+0000","objects_scrubbed":0,"log_size":10099,"log_dups_size":0,"ondisk_log_size":10099,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T02:31:40.900210+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":129695744,"num_objects":1979,"num_object_clones":0,"num_object_copies":5937,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1979,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11599,"num_write_kb":434496,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.f","version":"60'1","reported_seq":126,"reported_epoch":142,"state":"active+clean","last_fresh":"2023-06-03T20:03:28.906721+0000","last_change":"2023-06-03T19:57:28.464482+0000","last_active":"2023-06-03T20:03:28.906721+0000","last_peered":"2023-06-03T20:03:28.906721+0000","last_clean":"2023-06-03T20:03:28.906721+0000","last_became_active":"2023-06-03T19:57:28.464081+0000","last_became_peered":"2023-06-03T19:57:28.464081+0000","last_unstale":"2023-06-03T20:03:28.906721+0000","last_undegraded":"2023-06-03T20:03:28.906721+0000","last_fullsized":"2023-06-03T20:03:28.906721+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":94,"reported_epoch":142,"state":"active+clean","last_fresh":"2023-06-03T20:03:28.917616+0000","last_change":"2023-06-03T19:34:17.848358+0000","last_active":"2023-06-03T20:03:28.917616+0000","last_peered":"2023-06-03T20:03:28.917616+0000","last_clean":"2023-06-03T20:03:28.917616+0000","last_became_active":"2023-06-03T19:34:17.847811+0000","last_became_peered":"2023-06-03T19:34:17.847811+0000","last_unstale":"2023-06-03T20:03:28.917616+0000","last_undegraded":"2023-06-03T20:03:28.917616+0000","last_fullsized":"2023-06-03T20:03:28.917616+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"9.4","version":"143'11843","reported_seq":11869,"reported_epoch":143,"state":"active+clean","last_fresh":"2023-06-03T20:03:58.596255+0000","last_change":"2023-06-03T20:00:12.905941+0000","last_active":"2023-06-03T20:03:58.596255+0000","last_peered":"2023-06-03T20:03:58.596255+0000","last_clean":"2023-06-03T20:03:58.596255+0000","last_became_active":"2023-06-03T20:00:12.905661+0000","last_became_peered":"2023-06-03T20:00:12.905661+0000","last_unstale":"2023-06-03T20:03:58.596255+0000","last_undegraded":"2023-06-03T20:03:58.596255+0000","last_fullsized":"2023-06-03T20:03:58.596255+0000","mapping_epoch":127,"log_start":"133'4500","ondisk_log_start":"133'4500","created":127,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T20:00:11.873149+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T20:00:11.873149+0000","last_clean_scrub_stamp":"2023-06-03T20:00:11.873149+0000","objects_scrubbed":0,"log_size":7343,"log_dups_size":0,"ondisk_log_size":7343,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T06:19:01.942119+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":135069696,"num_objects":2061,"num_object_clones":0,"num_object_copies":6183,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2061,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11843,"num_write_kb":444928,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.e","version":"60'2","reported_seq":127,"reported_epoch":142,"state":"active+clean","last_fresh":"2023-06-03T20:03:28.908568+0000","last_change":"2023-06-03T19:57:28.468330+0000","last_active":"2023-06-03T20:03:28.908568+0000","last_peered":"2023-06-03T20:03:28.908568+0000","last_clean":"2023-06-03T20:03:28.908568+0000","last_became_active":"2023-06-03T19:57:28.467823+0000","last_became_peered":"2023-06-03T19:57:28.467823+0000","last_unstale":"2023-06-03T20:03:28.908568+0000","last_undegraded":"2023-06-03T20:03:28.908568+0000","last_fullsized":"2023-06-03T20:03:28.908568+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":142,"reported_epoch":143,"state":"active+clean","last_fresh":"2023-06-03T20:03:50.828944+0000","last_change":"2023-06-03T19:57:28.461449+0000","last_active":"2023-06-03T20:03:50.828944+0000","last_peered":"2023-06-03T20:03:50.828944+0000","last_clean":"2023-06-03T20:03:50.828944+0000","last_became_active":"2023-06-03T19:57:28.460745+0000","last_became_peered":"2023-06-03T19:57:28.460745+0000","last_unstale":"2023-06-03T20:03:50.828944+0000","last_undegraded":"2023-06-03T20:03:50.828944+0000","last_fullsized":"2023-06-03T20:03:50.828944+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T20:41:19.361638+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,3],"acting":[4,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.7","version":"143'11988","reported_seq":12012,"reported_epoch":143,"state":"active+clean","last_fresh":"2023-06-03T20:03:58.258398+0000","last_change":"2023-06-03T20:00:12.909354+0000","last_active":"2023-06-03T20:03:58.258398+0000","last_peered":"2023-06-03T20:03:58.258398+0000","last_clean":"2023-06-03T20:03:58.258398+0000","last_became_active":"2023-06-03T20:00:12.908971+0000","last_became_peered":"2023-06-03T20:00:12.908971+0000","last_unstale":"2023-06-03T20:03:58.258398+0000","last_undegraded":"2023-06-03T20:03:58.258398+0000","last_fullsized":"2023-06-03T20:03:58.258398+0000","mapping_epoch":127,"log_start":"131'1900","ondisk_log_start":"131'1900","created":127,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T20:00:11.873149+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T20:00:11.873149+0000","last_clean_scrub_stamp":"2023-06-03T20:00:11.873149+0000","objects_scrubbed":0,"log_size":10088,"log_dups_size":0,"ondisk_log_size":10088,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":135659520,"num_objects":2070,"num_object_clones":0,"num_object_copies":6210,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2070,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11988,"num_write_kb":449856,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.d","version":"60'1","reported_seq":96,"reported_epoch":142,"state":"active+clean","last_fresh":"2023-06-03T20:03:28.914737+0000","last_change":"2023-06-03T19:34:16.806559+0000","last_active":"2023-06-03T20:03:28.914737+0000","last_peered":"2023-06-03T20:03:28.914737+0000","last_clean":"2023-06-03T20:03:28.914737+0000","last_became_active":"2023-06-03T19:34:16.806149+0000","last_became_peered":"2023-06-03T19:34:16.806149+0000","last_unstale":"2023-06-03T20:03:28.914737+0000","last_undegraded":"2023-06-03T20:03:28.914737+0000","last_fullsized":"2023-06-03T20:03:28.914737+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":109,"reported_epoch":142,"state":"active+clean","last_fresh":"2023-06-03T20:03:28.901251+0000","last_change":"2023-06-03T19:51:59.360650+0000","last_active":"2023-06-03T20:03:28.901251+0000","last_peered":"2023-06-03T20:03:28.901251+0000","last_clean":"2023-06-03T20:03:28.901251+0000","last_became_active":"2023-06-03T19:51:59.360237+0000","last_became_peered":"2023-06-03T19:51:59.360237+0000","last_unstale":"2023-06-03T20:03:28.901251+0000","last_undegraded":"2023-06-03T20:03:28.901251+0000","last_fullsized":"2023-06-03T20:03:28.901251+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"9.6","version":"143'11572","reported_seq":11596,"reported_epoch":143,"state":"active+clean","last_fresh":"2023-06-03T20:03:58.301022+0000","last_change":"2023-06-03T20:00:13.172391+0000","last_active":"2023-06-03T20:03:58.301022+0000","last_peered":"2023-06-03T20:03:58.301022+0000","last_clean":"2023-06-03T20:03:58.301022+0000","last_became_active":"2023-06-03T20:00:13.171568+0000","last_became_peered":"2023-06-03T20:00:13.171568+0000","last_unstale":"2023-06-03T20:03:58.301022+0000","last_undegraded":"2023-06-03T20:03:58.301022+0000","last_fullsized":"2023-06-03T20:03:58.301022+0000","mapping_epoch":127,"log_start":"131'1500","ondisk_log_start":"131'1500","created":127,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T20:00:11.873149+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T20:00:11.873149+0000","last_clean_scrub_stamp":"2023-06-03T20:00:11.873149+0000","objects_scrubbed":0,"log_size":10072,"log_dups_size":0,"ondisk_log_size":10072,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":131203072,"num_objects":2002,"num_object_clones":0,"num_object_copies":6006,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2002,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11572,"num_write_kb":434368,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,1],"acting":[5,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":110,"reported_epoch":142,"state":"active+clean","last_fresh":"2023-06-03T20:03:28.914722+0000","last_change":"2023-06-03T19:56:54.507432+0000","last_active":"2023-06-03T20:03:28.914722+0000","last_peered":"2023-06-03T20:03:28.914722+0000","last_clean":"2023-06-03T20:03:28.914722+0000","last_became_active":"2023-06-03T19:56:54.507012+0000","last_became_peered":"2023-06-03T19:56:54.507012+0000","last_unstale":"2023-06-03T20:03:28.914722+0000","last_undegraded":"2023-06-03T20:03:28.914722+0000","last_fullsized":"2023-06-03T20:03:28.914722+0000","mapping_epoch":119,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":127,"reported_epoch":142,"state":"active+clean","last_fresh":"2023-06-03T20:03:28.898625+0000","last_change":"2023-06-03T19:57:28.464153+0000","last_active":"2023-06-03T20:03:28.898625+0000","last_peered":"2023-06-03T20:03:28.898625+0000","last_clean":"2023-06-03T20:03:28.898625+0000","last_became_active":"2023-06-03T19:57:28.463502+0000","last_became_peered":"2023-06-03T19:57:28.463502+0000","last_unstale":"2023-06-03T20:03:28.898625+0000","last_undegraded":"2023-06-03T20:03:28.898625+0000","last_fullsized":"2023-06-03T20:03:28.898625+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T06:11:00.999526+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,4],"acting":[0,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.1","version":"143'11842","reported_seq":11868,"reported_epoch":143,"state":"active+clean","last_fresh":"2023-06-03T20:03:58.052463+0000","last_change":"2023-06-03T20:00:12.905059+0000","last_active":"2023-06-03T20:03:58.052463+0000","last_peered":"2023-06-03T20:03:58.052463+0000","last_clean":"2023-06-03T20:03:58.052463+0000","last_became_active":"2023-06-03T20:00:12.904765+0000","last_became_peered":"2023-06-03T20:00:12.904765+0000","last_unstale":"2023-06-03T20:03:58.052463+0000","last_undegraded":"2023-06-03T20:03:58.052463+0000","last_fullsized":"2023-06-03T20:03:58.052463+0000","mapping_epoch":127,"log_start":"133'4100","ondisk_log_start":"133'4100","created":127,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T20:00:11.873149+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T20:00:11.873149+0000","last_clean_scrub_stamp":"2023-06-03T20:00:11.873149+0000","objects_scrubbed":0,"log_size":7742,"log_dups_size":0,"ondisk_log_size":7742,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T07:39:39.205403+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":128843776,"num_objects":1966,"num_object_clones":0,"num_object_copies":5898,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1966,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11842,"num_write_kb":441856,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,7],"acting":[3,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.b","version":"60'2","reported_seq":110,"reported_epoch":142,"state":"active+clean","last_fresh":"2023-06-03T20:03:38.072467+0000","last_change":"2023-06-03T19:57:28.470310+0000","last_active":"2023-06-03T20:03:38.072467+0000","last_peered":"2023-06-03T20:03:38.072467+0000","last_clean":"2023-06-03T20:03:38.072467+0000","last_became_active":"2023-06-03T19:57:28.469887+0000","last_became_peered":"2023-06-03T19:57:28.469887+0000","last_unstale":"2023-06-03T20:03:38.072467+0000","last_undegraded":"2023-06-03T20:03:38.072467+0000","last_fullsized":"2023-06-03T20:03:38.072467+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T21:58:53.387131+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":117,"reported_epoch":142,"state":"active+clean","last_fresh":"2023-06-03T20:03:28.900885+0000","last_change":"2023-06-03T19:57:28.465140+0000","last_active":"2023-06-03T20:03:28.900885+0000","last_peered":"2023-06-03T20:03:28.900885+0000","last_clean":"2023-06-03T20:03:28.900885+0000","last_became_active":"2023-06-03T19:57:28.464629+0000","last_became_peered":"2023-06-03T19:57:28.464629+0000","last_unstale":"2023-06-03T20:03:28.900885+0000","last_undegraded":"2023-06-03T20:03:28.900885+0000","last_fullsized":"2023-06-03T20:03:28.900885+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T21:47:11.619583+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.0","version":"143'11617","reported_seq":11643,"reported_epoch":143,"state":"active+clean","last_fresh":"2023-06-03T20:03:58.580871+0000","last_change":"2023-06-03T20:00:13.174012+0000","last_active":"2023-06-03T20:03:58.580871+0000","last_peered":"2023-06-03T20:03:58.580871+0000","last_clean":"2023-06-03T20:03:58.580871+0000","last_became_active":"2023-06-03T20:00:13.173630+0000","last_became_peered":"2023-06-03T20:00:13.173630+0000","last_unstale":"2023-06-03T20:03:58.580871+0000","last_undegraded":"2023-06-03T20:03:58.580871+0000","last_fullsized":"2023-06-03T20:03:58.580871+0000","mapping_epoch":127,"log_start":"133'4200","ondisk_log_start":"133'4200","created":127,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T20:00:11.873149+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T20:00:11.873149+0000","last_clean_scrub_stamp":"2023-06-03T20:00:11.873149+0000","objects_scrubbed":0,"log_size":7417,"log_dups_size":0,"ondisk_log_size":7417,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T23:14:07.402519+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":133627904,"num_objects":2039,"num_object_clones":0,"num_object_copies":6117,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2039,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11617,"num_write_kb":436992,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":125,"reported_epoch":142,"state":"active+clean","last_fresh":"2023-06-03T20:03:28.901063+0000","last_change":"2023-06-03T19:57:28.461401+0000","last_active":"2023-06-03T20:03:28.901063+0000","last_peered":"2023-06-03T20:03:28.901063+0000","last_clean":"2023-06-03T20:03:28.901063+0000","last_became_active":"2023-06-03T19:57:28.461057+0000","last_became_peered":"2023-06-03T19:57:28.461057+0000","last_unstale":"2023-06-03T20:03:28.901063+0000","last_undegraded":"2023-06-03T20:03:28.901063+0000","last_fullsized":"2023-06-03T20:03:28.901063+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":122,"reported_epoch":142,"state":"active+clean","last_fresh":"2023-06-03T20:03:28.895393+0000","last_change":"2023-06-03T19:57:28.456750+0000","last_active":"2023-06-03T20:03:28.895393+0000","last_peered":"2023-06-03T20:03:28.895393+0000","last_clean":"2023-06-03T20:03:28.895393+0000","last_became_active":"2023-06-03T19:57:28.456436+0000","last_became_peered":"2023-06-03T19:57:28.456436+0000","last_unstale":"2023-06-03T20:03:28.895393+0000","last_undegraded":"2023-06-03T20:03:28.895393+0000","last_fullsized":"2023-06-03T20:03:28.895393+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T01:49:25.481554+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,5],"acting":[4,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.3","version":"143'11588","reported_seq":11612,"reported_epoch":143,"state":"active+clean","last_fresh":"2023-06-03T20:03:58.233750+0000","last_change":"2023-06-03T20:00:12.903729+0000","last_active":"2023-06-03T20:03:58.233750+0000","last_peered":"2023-06-03T20:03:58.233750+0000","last_clean":"2023-06-03T20:03:58.233750+0000","last_became_active":"2023-06-03T20:00:12.903378+0000","last_became_peered":"2023-06-03T20:00:12.903378+0000","last_unstale":"2023-06-03T20:03:58.233750+0000","last_undegraded":"2023-06-03T20:03:58.233750+0000","last_fullsized":"2023-06-03T20:03:58.233750+0000","mapping_epoch":127,"log_start":"133'3800","ondisk_log_start":"133'3800","created":127,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T20:00:11.873149+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T20:00:11.873149+0000","last_clean_scrub_stamp":"2023-06-03T20:00:11.873149+0000","objects_scrubbed":0,"log_size":7788,"log_dups_size":0,"ondisk_log_size":7788,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":134086656,"num_objects":2046,"num_object_clones":0,"num_object_copies":6138,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2046,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11588,"num_write_kb":436288,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,1],"acting":[7,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":115,"reported_epoch":143,"state":"active+clean","last_fresh":"2023-06-03T20:03:50.828544+0000","last_change":"2023-06-03T19:57:28.462344+0000","last_active":"2023-06-03T20:03:50.828544+0000","last_peered":"2023-06-03T20:03:50.828544+0000","last_clean":"2023-06-03T20:03:50.828544+0000","last_became_active":"2023-06-03T19:57:28.462001+0000","last_became_peered":"2023-06-03T19:57:28.462001+0000","last_unstale":"2023-06-03T20:03:50.828544+0000","last_undegraded":"2023-06-03T20:03:50.828544+0000","last_fullsized":"2023-06-03T20:03:50.828544+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T00:45:13.789650+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":103,"reported_epoch":142,"state":"active+clean","last_fresh":"2023-06-03T20:03:28.897637+0000","last_change":"2023-06-03T19:57:28.467917+0000","last_active":"2023-06-03T20:03:28.897637+0000","last_peered":"2023-06-03T20:03:28.897637+0000","last_clean":"2023-06-03T20:03:28.897637+0000","last_became_active":"2023-06-03T19:57:28.467617+0000","last_became_peered":"2023-06-03T19:57:28.467617+0000","last_unstale":"2023-06-03T20:03:28.897637+0000","last_undegraded":"2023-06-03T20:03:28.897637+0000","last_fullsized":"2023-06-03T20:03:28.897637+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T23:34:23.840582+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.2","version":"143'11774","reported_seq":11800,"reported_epoch":143,"state":"active+clean","last_fresh":"2023-06-03T20:03:58.074019+0000","last_change":"2023-06-03T20:00:13.173625+0000","last_active":"2023-06-03T20:03:58.074019+0000","last_peered":"2023-06-03T20:03:58.074019+0000","last_clean":"2023-06-03T20:03:58.074019+0000","last_became_active":"2023-06-03T20:00:13.173348+0000","last_became_peered":"2023-06-03T20:00:13.173348+0000","last_unstale":"2023-06-03T20:03:58.074019+0000","last_undegraded":"2023-06-03T20:03:58.074019+0000","last_fullsized":"2023-06-03T20:03:58.074019+0000","mapping_epoch":127,"log_start":"133'4000","ondisk_log_start":"133'4000","created":127,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T20:00:11.873149+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T20:00:11.873149+0000","last_clean_scrub_stamp":"2023-06-03T20:00:11.873149+0000","objects_scrubbed":0,"log_size":7774,"log_dups_size":0,"ondisk_log_size":7774,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T03:41:17.261687+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":130416640,"num_objects":1990,"num_object_clones":0,"num_object_copies":5970,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1990,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11774,"num_write_kb":440448,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":110,"reported_epoch":142,"state":"active+clean","last_fresh":"2023-06-03T20:03:28.906107+0000","last_change":"2023-06-03T19:56:54.512219+0000","last_active":"2023-06-03T20:03:28.906107+0000","last_peered":"2023-06-03T20:03:28.906107+0000","last_clean":"2023-06-03T20:03:28.906107+0000","last_became_active":"2023-06-03T19:56:54.511771+0000","last_became_peered":"2023-06-03T19:56:54.511771+0000","last_unstale":"2023-06-03T20:03:28.906107+0000","last_undegraded":"2023-06-03T20:03:28.906107+0000","last_fullsized":"2023-06-03T20:03:28.906107+0000","mapping_epoch":119,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T19:35:47.796520+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":96,"reported_epoch":142,"state":"active+clean","last_fresh":"2023-06-03T20:03:28.908014+0000","last_change":"2023-06-03T19:57:28.471023+0000","last_active":"2023-06-03T20:03:28.908014+0000","last_peered":"2023-06-03T20:03:28.908014+0000","last_clean":"2023-06-03T20:03:28.908014+0000","last_became_active":"2023-06-03T19:57:28.470452+0000","last_became_peered":"2023-06-03T19:57:28.470452+0000","last_unstale":"2023-06-03T20:03:28.908014+0000","last_undegraded":"2023-06-03T20:03:28.908014+0000","last_fullsized":"2023-06-03T20:03:28.908014+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T05:58:33.305030+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.f","version":"143'11521","reported_seq":11545,"reported_epoch":143,"state":"active+clean","last_fresh":"2023-06-03T20:03:58.254385+0000","last_change":"2023-06-03T20:00:12.909234+0000","last_active":"2023-06-03T20:03:58.254385+0000","last_peered":"2023-06-03T20:03:58.254385+0000","last_clean":"2023-06-03T20:03:58.254385+0000","last_became_active":"2023-06-03T20:00:12.908871+0000","last_became_peered":"2023-06-03T20:00:12.908871+0000","last_unstale":"2023-06-03T20:03:58.254385+0000","last_undegraded":"2023-06-03T20:03:58.254385+0000","last_fullsized":"2023-06-03T20:03:58.254385+0000","mapping_epoch":127,"log_start":"131'1500","ondisk_log_start":"131'1500","created":127,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T20:00:11.873149+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T20:00:11.873149+0000","last_clean_scrub_stamp":"2023-06-03T20:00:11.873149+0000","objects_scrubbed":0,"log_size":10021,"log_dups_size":0,"ondisk_log_size":10021,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":131923968,"num_objects":2013,"num_object_clones":0,"num_object_copies":6039,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2013,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11521,"num_write_kb":433088,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":114,"reported_epoch":142,"state":"active+clean","last_fresh":"2023-06-03T20:03:28.908650+0000","last_change":"2023-06-03T19:51:59.392796+0000","last_active":"2023-06-03T20:03:28.908650+0000","last_peered":"2023-06-03T20:03:28.908650+0000","last_clean":"2023-06-03T20:03:28.908650+0000","last_became_active":"2023-06-03T19:51:59.380207+0000","last_became_peered":"2023-06-03T19:51:59.380207+0000","last_unstale":"2023-06-03T20:03:28.908650+0000","last_undegraded":"2023-06-03T20:03:28.908650+0000","last_fullsized":"2023-06-03T20:03:28.908650+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:32:20.692043+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:32:20.692043+0000","last_clean_scrub_stamp":"2023-06-03T19:32:20.692043+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T00:28:05.866873+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":125,"reported_epoch":142,"state":"active+clean","last_fresh":"2023-06-03T20:03:28.917960+0000","last_change":"2023-06-03T19:56:54.512607+0000","last_active":"2023-06-03T20:03:28.917960+0000","last_peered":"2023-06-03T20:03:28.917960+0000","last_clean":"2023-06-03T20:03:28.917960+0000","last_became_active":"2023-06-03T19:56:54.512282+0000","last_became_peered":"2023-06-03T19:56:54.512282+0000","last_unstale":"2023-06-03T20:03:28.917960+0000","last_undegraded":"2023-06-03T20:03:28.917960+0000","last_fullsized":"2023-06-03T20:03:28.917960+0000","mapping_epoch":119,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":81,"reported_epoch":142,"state":"active+clean","last_fresh":"2023-06-03T20:03:28.908260+0000","last_change":"2023-06-03T19:57:28.470295+0000","last_active":"2023-06-03T20:03:28.908260+0000","last_peered":"2023-06-03T20:03:28.908260+0000","last_clean":"2023-06-03T20:03:28.908260+0000","last_became_active":"2023-06-03T19:57:28.469383+0000","last_became_peered":"2023-06-03T19:57:28.469383+0000","last_unstale":"2023-06-03T20:03:28.908260+0000","last_undegraded":"2023-06-03T20:03:28.908260+0000","last_fullsized":"2023-06-03T20:03:28.908260+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T05:35:19.667447+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.9","version":"143'11844","reported_seq":11870,"reported_epoch":143,"state":"active+clean","last_fresh":"2023-06-03T20:03:59.936640+0000","last_change":"2023-06-03T20:00:12.910626+0000","last_active":"2023-06-03T20:03:59.936640+0000","last_peered":"2023-06-03T20:03:59.936640+0000","last_clean":"2023-06-03T20:03:59.936640+0000","last_became_active":"2023-06-03T20:00:12.910285+0000","last_became_peered":"2023-06-03T20:00:12.910285+0000","last_unstale":"2023-06-03T20:03:59.936640+0000","last_undegraded":"2023-06-03T20:03:59.936640+0000","last_fullsized":"2023-06-03T20:03:59.936640+0000","mapping_epoch":127,"log_start":"131'1800","ondisk_log_start":"131'1800","created":127,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T20:00:11.873149+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T20:00:11.873149+0000","last_clean_scrub_stamp":"2023-06-03T20:00:11.873149+0000","objects_scrubbed":0,"log_size":10044,"log_dups_size":0,"ondisk_log_size":10044,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T04:42:34.630759+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":129892352,"num_objects":1982,"num_object_clones":0,"num_object_copies":5946,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1982,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11844,"num_write_kb":442432,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,5],"acting":[0,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":114,"reported_epoch":142,"state":"active+clean","last_fresh":"2023-06-03T20:03:28.914838+0000","last_change":"2023-06-03T19:51:59.373777+0000","last_active":"2023-06-03T20:03:28.914838+0000","last_peered":"2023-06-03T20:03:28.914838+0000","last_clean":"2023-06-03T20:03:28.914838+0000","last_became_active":"2023-06-03T19:51:59.372988+0000","last_became_peered":"2023-06-03T19:51:59.372988+0000","last_unstale":"2023-06-03T20:03:28.914838+0000","last_undegraded":"2023-06-03T20:03:28.914838+0000","last_fullsized":"2023-06-03T20:03:28.914838+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:32:20.692043+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:32:20.692043+0000","last_clean_scrub_stamp":"2023-06-03T19:32:20.692043+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":88,"reported_epoch":142,"state":"active+clean","last_fresh":"2023-06-03T20:03:41.521951+0000","last_change":"2023-06-03T19:57:28.460195+0000","last_active":"2023-06-03T20:03:41.521951+0000","last_peered":"2023-06-03T20:03:41.521951+0000","last_clean":"2023-06-03T20:03:41.521951+0000","last_became_active":"2023-06-03T19:57:28.459823+0000","last_became_peered":"2023-06-03T19:57:28.459823+0000","last_unstale":"2023-06-03T20:03:41.521951+0000","last_undegraded":"2023-06-03T20:03:41.521951+0000","last_fullsized":"2023-06-03T20:03:41.521951+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T20:08:48.686380+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":96,"reported_epoch":142,"state":"active+clean","last_fresh":"2023-06-03T20:03:28.908613+0000","last_change":"2023-06-03T19:56:54.516859+0000","last_active":"2023-06-03T20:03:28.908613+0000","last_peered":"2023-06-03T20:03:28.908613+0000","last_clean":"2023-06-03T20:03:28.908613+0000","last_became_active":"2023-06-03T19:56:54.516521+0000","last_became_peered":"2023-06-03T19:56:54.516521+0000","last_unstale":"2023-06-03T20:03:28.908613+0000","last_undegraded":"2023-06-03T20:03:28.908613+0000","last_fullsized":"2023-06-03T20:03:28.908613+0000","mapping_epoch":119,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T20:03:25.227382+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.a","version":"143'11793","reported_seq":11817,"reported_epoch":143,"state":"active+clean","last_fresh":"2023-06-03T20:03:58.271261+0000","last_change":"2023-06-03T20:00:12.909720+0000","last_active":"2023-06-03T20:03:58.271261+0000","last_peered":"2023-06-03T20:03:58.271261+0000","last_clean":"2023-06-03T20:03:58.271261+0000","last_became_active":"2023-06-03T20:00:12.909428+0000","last_became_peered":"2023-06-03T20:00:12.909428+0000","last_unstale":"2023-06-03T20:03:58.271261+0000","last_undegraded":"2023-06-03T20:03:58.271261+0000","last_fullsized":"2023-06-03T20:03:58.271261+0000","mapping_epoch":127,"log_start":"131'1700","ondisk_log_start":"131'1700","created":127,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T20:00:11.873149+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T20:00:11.873149+0000","last_clean_scrub_stamp":"2023-06-03T20:00:11.873149+0000","objects_scrubbed":0,"log_size":10093,"log_dups_size":0,"ondisk_log_size":10093,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":129695744,"num_objects":1979,"num_object_clones":0,"num_object_copies":5937,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1979,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11793,"num_write_kb":440704,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,7],"acting":[5,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":117,"reported_epoch":142,"state":"active+clean","last_fresh":"2023-06-03T20:03:28.895354+0000","last_change":"2023-06-03T19:56:54.504859+0000","last_active":"2023-06-03T20:03:28.895354+0000","last_peered":"2023-06-03T20:03:28.895354+0000","last_clean":"2023-06-03T20:03:28.895354+0000","last_became_active":"2023-06-03T19:56:54.504455+0000","last_became_peered":"2023-06-03T19:56:54.504455+0000","last_unstale":"2023-06-03T20:03:28.895354+0000","last_undegraded":"2023-06-03T20:03:28.895354+0000","last_fullsized":"2023-06-03T20:03:28.895354+0000","mapping_epoch":119,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:32:20.692043+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:32:20.692043+0000","last_clean_scrub_stamp":"2023-06-03T19:32:20.692043+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T06:25:45.522100+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":123,"reported_epoch":143,"state":"active+clean","last_fresh":"2023-06-03T20:03:50.843432+0000","last_change":"2023-06-03T19:56:11.754048+0000","last_active":"2023-06-03T20:03:50.843432+0000","last_peered":"2023-06-03T20:03:50.843432+0000","last_clean":"2023-06-03T20:03:50.843432+0000","last_became_active":"2023-06-03T19:56:11.753687+0000","last_became_peered":"2023-06-03T19:56:11.753687+0000","last_unstale":"2023-06-03T20:03:50.843432+0000","last_undegraded":"2023-06-03T20:03:50.843432+0000","last_fullsized":"2023-06-03T20:03:50.843432+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":115,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T23:29:41.511496+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":109,"reported_epoch":142,"state":"active+clean","last_fresh":"2023-06-03T20:03:28.908250+0000","last_change":"2023-06-03T19:51:59.396573+0000","last_active":"2023-06-03T20:03:28.908250+0000","last_peered":"2023-06-03T20:03:28.908250+0000","last_clean":"2023-06-03T20:03:28.908250+0000","last_became_active":"2023-06-03T19:51:59.381593+0000","last_became_peered":"2023-06-03T19:51:59.381593+0000","last_unstale":"2023-06-03T20:03:28.908250+0000","last_undegraded":"2023-06-03T20:03:28.908250+0000","last_fullsized":"2023-06-03T20:03:28.908250+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T21:33:23.043772+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.8","version":"143'11673","reported_seq":11699,"reported_epoch":143,"state":"active+clean","last_fresh":"2023-06-03T20:03:58.052552+0000","last_change":"2023-06-03T20:00:12.908817+0000","last_active":"2023-06-03T20:03:58.052552+0000","last_peered":"2023-06-03T20:03:58.052552+0000","last_clean":"2023-06-03T20:03:58.052552+0000","last_became_active":"2023-06-03T20:00:12.908431+0000","last_became_peered":"2023-06-03T20:00:12.908431+0000","last_unstale":"2023-06-03T20:03:58.052552+0000","last_undegraded":"2023-06-03T20:03:58.052552+0000","last_fullsized":"2023-06-03T20:03:58.052552+0000","mapping_epoch":127,"log_start":"133'3900","ondisk_log_start":"133'3900","created":127,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T20:00:11.873149+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T20:00:11.873149+0000","last_clean_scrub_stamp":"2023-06-03T20:00:11.873149+0000","objects_scrubbed":0,"log_size":7773,"log_dups_size":0,"ondisk_log_size":7773,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T21:11:15.734132+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":133890048,"num_objects":2043,"num_object_clones":0,"num_object_copies":6129,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2043,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11673,"num_write_kb":438912,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":114,"reported_epoch":142,"state":"active+clean","last_fresh":"2023-06-03T20:03:28.914993+0000","last_change":"2023-06-03T19:56:11.752151+0000","last_active":"2023-06-03T20:03:28.914993+0000","last_peered":"2023-06-03T20:03:28.914993+0000","last_clean":"2023-06-03T20:03:28.914993+0000","last_became_active":"2023-06-03T19:56:11.751767+0000","last_became_peered":"2023-06-03T19:56:11.751767+0000","last_unstale":"2023-06-03T20:03:28.914993+0000","last_undegraded":"2023-06-03T20:03:28.914993+0000","last_fullsized":"2023-06-03T20:03:28.914993+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":115,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:32:20.692043+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:32:20.692043+0000","last_clean_scrub_stamp":"2023-06-03T19:32:20.692043+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"72'139","reported_seq":346,"reported_epoch":142,"state":"active+clean","last_fresh":"2023-06-03T20:03:28.899214+0000","last_change":"2023-06-03T19:47:06.731872+0000","last_active":"2023-06-03T20:03:28.899214+0000","last_peered":"2023-06-03T20:03:28.899214+0000","last_clean":"2023-06-03T20:03:28.899214+0000","last_became_active":"2023-06-03T19:47:06.731175+0000","last_became_peered":"2023-06-03T19:47:06.731175+0000","last_unstale":"2023-06-03T20:03:28.899214+0000","last_undegraded":"2023-06-03T20:03:28.899214+0000","last_fullsized":"2023-06-03T20:03:28.899214+0000","mapping_epoch":86,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":87,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:27:28.100808+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:27:28.100808+0000","last_clean_scrub_stamp":"2023-06-03T19:27:28.100808+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":90,"reported_epoch":142,"state":"active+clean","last_fresh":"2023-06-03T20:03:28.905915+0000","last_change":"2023-06-03T19:56:54.509571+0000","last_active":"2023-06-03T20:03:28.905915+0000","last_peered":"2023-06-03T20:03:28.905915+0000","last_clean":"2023-06-03T20:03:28.905915+0000","last_became_active":"2023-06-03T19:56:54.509213+0000","last_became_peered":"2023-06-03T19:56:54.509213+0000","last_unstale":"2023-06-03T20:03:28.905915+0000","last_undegraded":"2023-06-03T20:03:28.905915+0000","last_fullsized":"2023-06-03T20:03:28.905915+0000","mapping_epoch":119,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T21:26:56.003203+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":109,"reported_epoch":142,"state":"active+clean","last_fresh":"2023-06-03T20:03:28.901527+0000","last_change":"2023-06-03T19:57:28.459664+0000","last_active":"2023-06-03T20:03:28.901527+0000","last_peered":"2023-06-03T20:03:28.901527+0000","last_clean":"2023-06-03T20:03:28.901527+0000","last_became_active":"2023-06-03T19:57:28.459289+0000","last_became_peered":"2023-06-03T19:57:28.459289+0000","last_unstale":"2023-06-03T20:03:28.901527+0000","last_undegraded":"2023-06-03T20:03:28.901527+0000","last_fullsized":"2023-06-03T20:03:28.901527+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"9.b","version":"143'12045","reported_seq":12071,"reported_epoch":143,"state":"active+clean","last_fresh":"2023-06-03T20:03:59.855831+0000","last_change":"2023-06-03T20:00:12.908303+0000","last_active":"2023-06-03T20:03:59.855831+0000","last_peered":"2023-06-03T20:03:59.855831+0000","last_clean":"2023-06-03T20:03:59.855831+0000","last_became_active":"2023-06-03T20:00:12.907799+0000","last_became_peered":"2023-06-03T20:00:12.907799+0000","last_unstale":"2023-06-03T20:03:59.855831+0000","last_undegraded":"2023-06-03T20:03:59.855831+0000","last_fullsized":"2023-06-03T20:03:59.855831+0000","mapping_epoch":127,"log_start":"131'2000","ondisk_log_start":"131'2000","created":127,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T20:00:11.873149+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T20:00:11.873149+0000","last_clean_scrub_stamp":"2023-06-03T20:00:11.873149+0000","objects_scrubbed":0,"log_size":10045,"log_dups_size":0,"ondisk_log_size":10045,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T23:11:45.966674+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":128122880,"num_objects":1955,"num_object_clones":0,"num_object_copies":5865,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1955,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12045,"num_write_kb":448000,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,5],"acting":[0,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":129,"reported_epoch":142,"state":"active+clean","last_fresh":"2023-06-03T20:03:28.899088+0000","last_change":"2023-06-03T19:51:59.358157+0000","last_active":"2023-06-03T20:03:28.899088+0000","last_peered":"2023-06-03T20:03:28.899088+0000","last_clean":"2023-06-03T20:03:28.899088+0000","last_became_active":"2023-06-03T19:51:59.357363+0000","last_became_peered":"2023-06-03T19:51:59.357363+0000","last_unstale":"2023-06-03T20:03:28.899088+0000","last_undegraded":"2023-06-03T20:03:28.899088+0000","last_fullsized":"2023-06-03T20:03:28.899088+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:32:20.692043+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:32:20.692043+0000","last_clean_scrub_stamp":"2023-06-03T19:32:20.692043+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":115,"reported_epoch":142,"state":"active+clean","last_fresh":"2023-06-03T20:03:44.930428+0000","last_change":"2023-06-03T19:57:28.459611+0000","last_active":"2023-06-03T20:03:44.930428+0000","last_peered":"2023-06-03T20:03:44.930428+0000","last_clean":"2023-06-03T20:03:44.930428+0000","last_became_active":"2023-06-03T19:57:28.459264+0000","last_became_peered":"2023-06-03T19:57:28.459264+0000","last_unstale":"2023-06-03T20:03:44.930428+0000","last_undegraded":"2023-06-03T20:03:44.930428+0000","last_fullsized":"2023-06-03T20:03:44.930428+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T22:15:06.557231+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":115,"reported_epoch":142,"state":"active+clean","last_fresh":"2023-06-03T20:03:28.893524+0000","last_change":"2023-06-03T19:56:11.745678+0000","last_active":"2023-06-03T20:03:28.893524+0000","last_peered":"2023-06-03T20:03:28.893524+0000","last_clean":"2023-06-03T20:03:28.893524+0000","last_became_active":"2023-06-03T19:56:11.744966+0000","last_became_peered":"2023-06-03T19:56:11.744966+0000","last_unstale":"2023-06-03T20:03:28.893524+0000","last_undegraded":"2023-06-03T20:03:28.893524+0000","last_fullsized":"2023-06-03T20:03:28.893524+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":115,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T21:12:06.830295+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.e","version":"143'11717","reported_seq":11741,"reported_epoch":143,"state":"active+clean","last_fresh":"2023-06-03T20:03:58.257918+0000","last_change":"2023-06-03T20:00:12.908248+0000","last_active":"2023-06-03T20:03:58.257918+0000","last_peered":"2023-06-03T20:03:58.257918+0000","last_clean":"2023-06-03T20:03:58.257918+0000","last_became_active":"2023-06-03T20:00:12.907900+0000","last_became_peered":"2023-06-03T20:00:12.907900+0000","last_unstale":"2023-06-03T20:03:58.257918+0000","last_undegraded":"2023-06-03T20:03:58.257918+0000","last_fullsized":"2023-06-03T20:03:58.257918+0000","mapping_epoch":127,"log_start":"133'4000","ondisk_log_start":"133'4000","created":127,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T20:00:11.873149+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T20:00:11.873149+0000","last_clean_scrub_stamp":"2023-06-03T20:00:11.873149+0000","objects_scrubbed":0,"log_size":7717,"log_dups_size":0,"ondisk_log_size":7717,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":128647168,"num_objects":1963,"num_object_clones":0,"num_object_copies":5889,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1963,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11717,"num_write_kb":437760,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":129,"reported_epoch":142,"state":"active+clean","last_fresh":"2023-06-03T20:03:28.914993+0000","last_change":"2023-06-03T19:57:28.466096+0000","last_active":"2023-06-03T20:03:28.914993+0000","last_peered":"2023-06-03T20:03:28.914993+0000","last_clean":"2023-06-03T20:03:28.914993+0000","last_became_active":"2023-06-03T19:57:28.464954+0000","last_became_peered":"2023-06-03T19:57:28.464954+0000","last_unstale":"2023-06-03T20:03:28.914993+0000","last_undegraded":"2023-06-03T20:03:28.914993+0000","last_fullsized":"2023-06-03T20:03:28.914993+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:32:20.692043+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:32:20.692043+0000","last_clean_scrub_stamp":"2023-06-03T19:32:20.692043+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":122,"reported_epoch":142,"state":"active+clean","last_fresh":"2023-06-03T20:03:28.908568+0000","last_change":"2023-06-03T19:56:11.750241+0000","last_active":"2023-06-03T20:03:28.908568+0000","last_peered":"2023-06-03T20:03:28.908568+0000","last_clean":"2023-06-03T20:03:28.908568+0000","last_became_active":"2023-06-03T19:56:11.749687+0000","last_became_peered":"2023-06-03T19:56:11.749687+0000","last_unstale":"2023-06-03T20:03:28.908568+0000","last_undegraded":"2023-06-03T20:03:28.908568+0000","last_fullsized":"2023-06-03T20:03:28.908568+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":115,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T22:38:25.819929+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":86,"reported_epoch":142,"state":"active+clean","last_fresh":"2023-06-03T20:03:44.930471+0000","last_change":"2023-06-03T19:47:06.733532+0000","last_active":"2023-06-03T20:03:44.930471+0000","last_peered":"2023-06-03T20:03:44.930471+0000","last_clean":"2023-06-03T20:03:44.930471+0000","last_became_active":"2023-06-03T19:47:06.733131+0000","last_became_peered":"2023-06-03T19:47:06.733131+0000","last_unstale":"2023-06-03T20:03:44.930471+0000","last_undegraded":"2023-06-03T20:03:44.930471+0000","last_fullsized":"2023-06-03T20:03:44.930471+0000","mapping_epoch":86,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":87,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T23:46:03.788646+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.d","version":"143'12015","reported_seq":12040,"reported_epoch":143,"state":"active+clean","last_fresh":"2023-06-03T20:03:58.197991+0000","last_change":"2023-06-03T20:00:12.905723+0000","last_active":"2023-06-03T20:03:58.197991+0000","last_peered":"2023-06-03T20:03:58.197991+0000","last_clean":"2023-06-03T20:03:58.197991+0000","last_became_active":"2023-06-03T20:00:12.905437+0000","last_became_peered":"2023-06-03T20:00:12.905437+0000","last_unstale":"2023-06-03T20:03:58.197991+0000","last_undegraded":"2023-06-03T20:03:58.197991+0000","last_fullsized":"2023-06-03T20:03:58.197991+0000","mapping_epoch":127,"log_start":"133'4300","ondisk_log_start":"133'4300","created":127,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T20:00:11.873149+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T20:00:11.873149+0000","last_clean_scrub_stamp":"2023-06-03T20:00:11.873149+0000","objects_scrubbed":0,"log_size":7715,"log_dups_size":0,"ondisk_log_size":7715,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":134086680,"num_objects":2047,"num_object_clones":0,"num_object_copies":6141,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2047,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":12015,"num_write_kb":449921,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,5],"acting":[7,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.7","version":"60'2","reported_seq":95,"reported_epoch":142,"state":"active+clean","last_fresh":"2023-06-03T20:03:28.907325+0000","last_change":"2023-06-03T19:56:54.510148+0000","last_active":"2023-06-03T20:03:28.907325+0000","last_peered":"2023-06-03T20:03:28.907325+0000","last_clean":"2023-06-03T20:03:28.907325+0000","last_became_active":"2023-06-03T19:56:54.509771+0000","last_became_peered":"2023-06-03T19:56:54.509771+0000","last_unstale":"2023-06-03T20:03:28.907325+0000","last_undegraded":"2023-06-03T20:03:28.907325+0000","last_fullsized":"2023-06-03T20:03:28.907325+0000","mapping_epoch":119,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T00:52:42.807432+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":122,"reported_epoch":142,"state":"active+clean","last_fresh":"2023-06-03T20:03:28.908313+0000","last_change":"2023-06-03T19:57:28.470954+0000","last_active":"2023-06-03T20:03:28.908313+0000","last_peered":"2023-06-03T20:03:28.908313+0000","last_clean":"2023-06-03T20:03:28.908313+0000","last_became_active":"2023-06-03T19:57:28.470524+0000","last_became_peered":"2023-06-03T19:57:28.470524+0000","last_unstale":"2023-06-03T20:03:28.908313+0000","last_undegraded":"2023-06-03T20:03:28.908313+0000","last_fullsized":"2023-06-03T20:03:28.908313+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:32:20.692043+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:32:20.692043+0000","last_clean_scrub_stamp":"2023-06-03T19:32:20.692043+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T19:39:19.983271+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":92,"reported_epoch":142,"state":"active+clean","last_fresh":"2023-06-03T20:03:28.906020+0000","last_change":"2023-06-03T19:56:54.507670+0000","last_active":"2023-06-03T20:03:28.906020+0000","last_peered":"2023-06-03T20:03:28.906020+0000","last_clean":"2023-06-03T20:03:28.906020+0000","last_became_active":"2023-06-03T19:56:54.507293+0000","last_became_peered":"2023-06-03T19:56:54.507293+0000","last_unstale":"2023-06-03T20:03:28.906020+0000","last_undegraded":"2023-06-03T20:03:28.906020+0000","last_fullsized":"2023-06-03T20:03:28.906020+0000","mapping_epoch":119,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T01:49:19.966054+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"9.c","version":"143'11717","reported_seq":11743,"reported_epoch":143,"state":"active+clean","last_fresh":"2023-06-03T20:03:58.031981+0000","last_change":"2023-06-03T20:00:12.904758+0000","last_active":"2023-06-03T20:03:58.031981+0000","last_peered":"2023-06-03T20:03:58.031981+0000","last_clean":"2023-06-03T20:03:58.031981+0000","last_became_active":"2023-06-03T20:00:12.904435+0000","last_became_peered":"2023-06-03T20:00:12.904435+0000","last_unstale":"2023-06-03T20:03:58.031981+0000","last_undegraded":"2023-06-03T20:03:58.031981+0000","last_fullsized":"2023-06-03T20:03:58.031981+0000","mapping_epoch":127,"log_start":"133'4000","ondisk_log_start":"133'4000","created":127,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T20:00:11.873149+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T20:00:11.873149+0000","last_clean_scrub_stamp":"2023-06-03T20:00:11.873149+0000","objects_scrubbed":0,"log_size":7717,"log_dups_size":0,"ondisk_log_size":7717,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T04:00:13.693202+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":129826816,"num_objects":1981,"num_object_clones":0,"num_object_copies":5943,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1981,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11717,"num_write_kb":438336,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,7],"acting":[3,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":114,"reported_epoch":142,"state":"active+clean","last_fresh":"2023-06-03T20:03:28.901594+0000","last_change":"2023-06-03T19:56:11.752730+0000","last_active":"2023-06-03T20:03:28.901594+0000","last_peered":"2023-06-03T20:03:28.901594+0000","last_clean":"2023-06-03T20:03:28.901594+0000","last_became_active":"2023-06-03T19:56:11.752441+0000","last_became_peered":"2023-06-03T19:56:11.752441+0000","last_unstale":"2023-06-03T20:03:28.901594+0000","last_undegraded":"2023-06-03T20:03:28.901594+0000","last_fullsized":"2023-06-03T20:03:28.901594+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":115,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:32:20.692043+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:32:20.692043+0000","last_clean_scrub_stamp":"2023-06-03T19:32:20.692043+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"60'1","reported_seq":118,"reported_epoch":142,"state":"active+clean","last_fresh":"2023-06-03T20:03:28.893427+0000","last_change":"2023-06-03T19:57:28.463999+0000","last_active":"2023-06-03T20:03:28.893427+0000","last_peered":"2023-06-03T20:03:28.893427+0000","last_clean":"2023-06-03T20:03:28.893427+0000","last_became_active":"2023-06-03T19:57:28.463588+0000","last_became_peered":"2023-06-03T19:57:28.463588+0000","last_unstale":"2023-06-03T20:03:28.893427+0000","last_undegraded":"2023-06-03T20:03:28.893427+0000","last_fullsized":"2023-06-03T20:03:28.893427+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T21:44:11.969059+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":86,"reported_epoch":142,"state":"active+clean","last_fresh":"2023-06-03T20:03:41.521877+0000","last_change":"2023-06-03T19:57:28.456467+0000","last_active":"2023-06-03T20:03:41.521877+0000","last_peered":"2023-06-03T20:03:41.521877+0000","last_clean":"2023-06-03T20:03:41.521877+0000","last_became_active":"2023-06-03T19:57:28.456120+0000","last_became_peered":"2023-06-03T19:57:28.456120+0000","last_unstale":"2023-06-03T20:03:41.521877+0000","last_undegraded":"2023-06-03T20:03:41.521877+0000","last_fullsized":"2023-06-03T20:03:41.521877+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T07:28:14.306624+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"60'1","reported_seq":102,"reported_epoch":142,"state":"active+clean","last_fresh":"2023-06-03T20:03:28.898718+0000","last_change":"2023-06-03T19:57:28.465124+0000","last_active":"2023-06-03T20:03:28.898718+0000","last_peered":"2023-06-03T20:03:28.898718+0000","last_clean":"2023-06-03T20:03:28.898718+0000","last_became_active":"2023-06-03T19:57:28.464654+0000","last_became_peered":"2023-06-03T19:57:28.464654+0000","last_unstale":"2023-06-03T20:03:28.898718+0000","last_undegraded":"2023-06-03T20:03:28.898718+0000","last_fullsized":"2023-06-03T20:03:28.898718+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T01:12:32.305507+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":129,"reported_epoch":142,"state":"active+clean","last_fresh":"2023-06-03T20:03:39.814687+0000","last_change":"2023-06-03T19:56:54.511760+0000","last_active":"2023-06-03T20:03:39.814687+0000","last_peered":"2023-06-03T20:03:39.814687+0000","last_clean":"2023-06-03T20:03:39.814687+0000","last_became_active":"2023-06-03T19:56:54.511429+0000","last_became_peered":"2023-06-03T19:56:54.511429+0000","last_unstale":"2023-06-03T20:03:39.814687+0000","last_undegraded":"2023-06-03T20:03:39.814687+0000","last_fullsized":"2023-06-03T20:03:39.814687+0000","mapping_epoch":119,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T05:42:46.254343+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"60'2","reported_seq":127,"reported_epoch":142,"state":"active+clean","last_fresh":"2023-06-03T20:03:28.914818+0000","last_change":"2023-06-03T19:57:28.462252+0000","last_active":"2023-06-03T20:03:28.914818+0000","last_peered":"2023-06-03T20:03:28.914818+0000","last_clean":"2023-06-03T20:03:28.914818+0000","last_became_active":"2023-06-03T19:57:28.461844+0000","last_became_peered":"2023-06-03T19:57:28.461844+0000","last_unstale":"2023-06-03T20:03:28.914818+0000","last_undegraded":"2023-06-03T20:03:28.914818+0000","last_fullsized":"2023-06-03T20:03:28.914818+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":132,"reported_epoch":142,"state":"active+clean","last_fresh":"2023-06-03T20:03:28.895596+0000","last_change":"2023-06-03T19:57:28.469248+0000","last_active":"2023-06-03T20:03:28.895596+0000","last_peered":"2023-06-03T20:03:28.895596+0000","last_clean":"2023-06-03T20:03:28.895596+0000","last_became_active":"2023-06-03T19:57:28.468903+0000","last_became_peered":"2023-06-03T19:57:28.468903+0000","last_unstale":"2023-06-03T20:03:28.895596+0000","last_undegraded":"2023-06-03T20:03:28.895596+0000","last_fullsized":"2023-06-03T20:03:28.895596+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T06:08:20.956098+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"60'2","reported_seq":112,"reported_epoch":142,"state":"active+clean","last_fresh":"2023-06-03T20:03:28.917842+0000","last_change":"2023-06-03T19:57:28.462610+0000","last_active":"2023-06-03T20:03:28.917842+0000","last_peered":"2023-06-03T20:03:28.917842+0000","last_clean":"2023-06-03T20:03:28.917842+0000","last_became_active":"2023-06-03T19:57:28.460655+0000","last_became_peered":"2023-06-03T19:57:28.460655+0000","last_unstale":"2023-06-03T20:03:28.917842+0000","last_undegraded":"2023-06-03T20:03:28.917842+0000","last_fullsized":"2023-06-03T20:03:28.917842+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":132,"reported_epoch":142,"state":"active+clean","last_fresh":"2023-06-03T20:03:28.895138+0000","last_change":"2023-06-03T19:57:28.464690+0000","last_active":"2023-06-03T20:03:28.895138+0000","last_peered":"2023-06-03T20:03:28.895138+0000","last_clean":"2023-06-03T20:03:28.895138+0000","last_became_active":"2023-06-03T19:57:28.463856+0000","last_became_peered":"2023-06-03T19:57:28.463856+0000","last_unstale":"2023-06-03T20:03:28.895138+0000","last_undegraded":"2023-06-03T20:03:28.895138+0000","last_fullsized":"2023-06-03T20:03:28.895138+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T00:11:00.991088+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"60'1","reported_seq":121,"reported_epoch":142,"state":"active+clean","last_fresh":"2023-06-03T20:03:28.906273+0000","last_change":"2023-06-03T19:56:54.505096+0000","last_active":"2023-06-03T20:03:28.906273+0000","last_peered":"2023-06-03T20:03:28.906273+0000","last_clean":"2023-06-03T20:03:28.906273+0000","last_became_active":"2023-06-03T19:56:54.504255+0000","last_became_peered":"2023-06-03T19:56:54.504255+0000","last_unstale":"2023-06-03T20:03:28.906273+0000","last_undegraded":"2023-06-03T20:03:28.906273+0000","last_fullsized":"2023-06-03T20:03:28.906273+0000","mapping_epoch":119,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T19:52:25.922308+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":133,"reported_epoch":142,"state":"active+clean","last_fresh":"2023-06-03T20:03:28.906515+0000","last_change":"2023-06-03T19:56:54.503416+0000","last_active":"2023-06-03T20:03:28.906515+0000","last_peered":"2023-06-03T20:03:28.906515+0000","last_clean":"2023-06-03T20:03:28.906515+0000","last_became_active":"2023-06-03T19:56:54.503018+0000","last_became_peered":"2023-06-03T19:56:54.503018+0000","last_unstale":"2023-06-03T20:03:28.906515+0000","last_undegraded":"2023-06-03T20:03:28.906515+0000","last_fullsized":"2023-06-03T20:03:28.906515+0000","mapping_epoch":119,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T20:14:00.330616+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"60'4","reported_seq":104,"reported_epoch":142,"state":"active+clean","last_fresh":"2023-06-03T20:03:44.930551+0000","last_change":"2023-06-03T19:56:11.748116+0000","last_active":"2023-06-03T20:03:44.930551+0000","last_peered":"2023-06-03T20:03:44.930551+0000","last_clean":"2023-06-03T20:03:44.930551+0000","last_became_active":"2023-06-03T19:56:11.747636+0000","last_became_peered":"2023-06-03T19:56:11.747636+0000","last_unstale":"2023-06-03T20:03:44.930551+0000","last_undegraded":"2023-06-03T20:03:44.930551+0000","last_fullsized":"2023-06-03T20:03:44.930551+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":115,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T06:57:48.052055+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":103,"reported_epoch":142,"state":"active+clean","last_fresh":"2023-06-03T20:03:28.894542+0000","last_change":"2023-06-03T19:57:28.459234+0000","last_active":"2023-06-03T20:03:28.894542+0000","last_peered":"2023-06-03T20:03:28.894542+0000","last_clean":"2023-06-03T20:03:28.894542+0000","last_became_active":"2023-06-03T19:57:28.458896+0000","last_became_peered":"2023-06-03T19:57:28.458896+0000","last_unstale":"2023-06-03T20:03:28.894542+0000","last_undegraded":"2023-06-03T20:03:28.894542+0000","last_fullsized":"2023-06-03T20:03:28.894542+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T21:18:37.142280+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5486,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":62,"num_write_kb":30,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16458,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":30319,"internal_metadata":0},"log_size":56,"ondisk_log_size":56,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":3,"num_store_stats":3},{"poolid":9,"num_pg":16,"stat_sum":{"num_bytes":2104688664,"num_objects":32116,"num_object_clones":0,"num_object_copies":96348,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":32116,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":188148,"num_write_kb":7048385,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":6334132224,"data_stored":6334120008,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139448,"ondisk_log_size":139448,"up":48,"acting":48,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":48,"seq":206158430603,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1549068,"kb_used_data":1166404,"kb_used_omap":0,"kb_used_meta":382464,"kb_avail":92196084,"statfs":{"total":95995035648,"available":94408790016,"internally_reserved":0,"allocated":1194397696,"data_stored":1193915322,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":391643136},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 3 20:03:42 2023","interfaces":[{"interface":"back","average":{"1min":1.966,"5min":1.2290000000000001,"15min":1.3220000000000001},"min":{"1min":0.25900000000000001,"5min":0.25900000000000001,"15min":0.253},"max":{"1min":7.7839999999999998,"5min":7.7839999999999998,"15min":8.6099999999999994},"last":0.72099999999999997},{"interface":"front","average":{"1min":2.0379999999999998,"5min":1.5429999999999999,"15min":1.5209999999999999},"min":{"1min":0.34100000000000003,"5min":0.318,"15min":0.28499999999999998},"max":{"1min":8.2460000000000004,"5min":11.877000000000001,"15min":15.220000000000001},"last":2.3879999999999999}]},{"osd":1,"last update":"Sat Jun 3 20:03:22 2023","interfaces":[{"interface":"back","average":{"1min":2.3039999999999998,"5min":2.4510000000000001,"15min":1.964},"min":{"1min":0.29699999999999999,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":22.533999999999999,"5min":91.811000000000007,"15min":91.811000000000007},"last":0.874},{"interface":"front","average":{"1min":3.6070000000000002,"5min":2.7080000000000002,"15min":1.95},"min":{"1min":0.32200000000000001,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":22.675000000000001,"5min":91.695999999999998,"15min":91.695999999999998},"last":2.274}]},{"osd":2,"last update":"Sat Jun 3 20:03:22 2023","interfaces":[{"interface":"back","average":{"1min":1.3029999999999999,"5min":1.157,"15min":1.0669999999999999},"min":{"1min":0.32800000000000001,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":7.71,"5min":7.71,"15min":7.71},"last":0.79200000000000004},{"interface":"front","average":{"1min":2.3199999999999998,"5min":1.3029999999999999,"15min":1.093},"min":{"1min":0.38500000000000001,"5min":0.38500000000000001,"15min":0.38500000000000001},"max":{"1min":8.3059999999999992,"5min":8.3059999999999992,"15min":8.3059999999999992},"last":2.3100000000000001}]},{"osd":3,"last update":"Sat Jun 3 20:03:10 2023","interfaces":[{"interface":"back","average":{"1min":1.3169999999999999,"5min":2.411,"15min":1.5740000000000001},"min":{"1min":0.34699999999999998,"5min":0.34699999999999998,"15min":0.34699999999999998},"max":{"1min":4.8010000000000002,"5min":95.962000000000003,"15min":95.962000000000003},"last":0.81899999999999995},{"interface":"front","average":{"1min":2.214,"5min":2.5659999999999998,"15min":1.655},"min":{"1min":0.36399999999999999,"5min":0.36399999999999999,"15min":0.36399999999999999},"max":{"1min":7.9630000000000001,"5min":94.174999999999997,"15min":94.174999999999997},"last":2.3340000000000001}]},{"osd":4,"last update":"Sat Jun 3 20:03:33 2023","interfaces":[{"interface":"back","average":{"1min":1.1919999999999999,"5min":1.056,"15min":1.052},"min":{"1min":0.38200000000000001,"5min":0.35999999999999999,"15min":0.35999999999999999},"max":{"1min":7.7350000000000003,"5min":7.7350000000000003,"15min":7.7350000000000003},"last":0.76500000000000001},{"interface":"front","average":{"1min":2.6640000000000001,"5min":1.5309999999999999,"15min":1.256},"min":{"1min":0.39600000000000002,"5min":0.39600000000000002,"15min":0.39600000000000002},"max":{"1min":12.917,"5min":12.917,"15min":12.917},"last":2.3599999999999999}]},{"osd":5,"last update":"Sat Jun 3 20:03:19 2023","interfaces":[{"interface":"back","average":{"1min":1.383,"5min":1.1679999999999999,"15min":1.1719999999999999},"min":{"1min":0.443,"5min":0.443,"15min":0.30199999999999999},"max":{"1min":7.6829999999999998,"5min":7.6829999999999998,"15min":12.778},"last":0.84799999999999998},{"interface":"front","average":{"1min":2.2650000000000001,"5min":1.5269999999999999,"15min":1.387},"min":{"1min":0.33100000000000002,"5min":0.33100000000000002,"15min":0.30599999999999999},"max":{"1min":8.1340000000000003,"5min":8.1340000000000003,"15min":8.1340000000000003},"last":2.6190000000000002}]},{"osd":6,"last update":"Sat Jun 3 20:03:19 2023","interfaces":[{"interface":"back","average":{"1min":2.3929999999999998,"5min":1.431,"15min":1.363},"min":{"1min":0.42199999999999999,"5min":0.42199999999999999,"15min":0.36099999999999999},"max":{"1min":8.3219999999999992,"5min":8.3219999999999992,"15min":8.3219999999999992},"last":2.4409999999999998},{"interface":"front","average":{"1min":1.6240000000000001,"5min":1.4490000000000001,"15min":1.2509999999999999},"min":{"1min":0.36899999999999999,"5min":0.36899999999999999,"15min":0.32900000000000001},"max":{"1min":4.6210000000000004,"5min":6.6840000000000002,"15min":9.0269999999999992},"last":1.2889999999999999}]}]},{"osd":6,"up_from":42,"seq":180388626836,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":691448,"kb_used_data":398448,"kb_used_omap":0,"kb_used_meta":292928,"kb_avail":93053704,"statfs":{"total":95995035648,"available":95286992896,"internally_reserved":0,"allocated":408010752,"data_stored":407548373,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":299958272},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 3 20:03:51 2023","interfaces":[{"interface":"back","average":{"1min":0.60899999999999999,"5min":1.4470000000000001,"15min":1.587},"min":{"1min":0.26400000000000001,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":3.3239999999999998,"5min":54.786000000000001,"15min":54.786000000000001},"last":0.47899999999999998},{"interface":"front","average":{"1min":0.872,"5min":1.5189999999999999,"15min":1.6719999999999999},"min":{"1min":0.317,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":3.335,"5min":54.936,"15min":54.936},"last":0.34999999999999998}]},{"osd":1,"last update":"Sat Jun 3 20:03:24 2023","interfaces":[{"interface":"back","average":{"1min":5.8339999999999996,"5min":4.1920000000000002,"15min":2.4239999999999999},"min":{"1min":0.28000000000000003,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":55.064999999999998,"5min":160.38499999999999,"15min":160.38499999999999},"last":1.2170000000000001},{"interface":"front","average":{"1min":6.1440000000000001,"5min":3.4830000000000001,"15min":2.032},"min":{"1min":0.29599999999999999,"5min":0.29599999999999999,"15min":0.29599999999999999},"max":{"1min":55.091000000000001,"5min":160.434,"15min":160.434},"last":1.228}]},{"osd":2,"last update":"Sat Jun 3 20:03:24 2023","interfaces":[{"interface":"back","average":{"1min":0.97899999999999998,"5min":0.88300000000000001,"15min":0.85699999999999998},"min":{"1min":0.33300000000000002,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":3.4260000000000002,"5min":3.4260000000000002,"15min":3.4260000000000002},"last":1.9059999999999999},{"interface":"front","average":{"1min":0.78500000000000003,"5min":0.872,"15min":0.92100000000000004},"min":{"1min":0.34799999999999998,"5min":0.34799999999999998,"15min":0.34799999999999998},"max":{"1min":3.1779999999999999,"5min":3.1779999999999999,"15min":3.1779999999999999},"last":1.9450000000000001}]},{"osd":3,"last update":"Sat Jun 3 20:03:06 2023","interfaces":[{"interface":"back","average":{"1min":4.4710000000000001,"5min":4.1440000000000001,"15min":1.95},"min":{"1min":0.32400000000000001,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":55.633000000000003,"5min":164.99199999999999,"15min":164.99199999999999},"last":5.484},{"interface":"front","average":{"1min":4.952,"5min":4.2160000000000002,"15min":2.0680000000000001},"min":{"1min":0.48199999999999998,"5min":0.36799999999999999,"15min":0.36799999999999999},"max":{"1min":61.143000000000001,"5min":163.459,"15min":163.459},"last":0.55200000000000005}]},{"osd":4,"last update":"Sat Jun 3 20:03:42 2023","interfaces":[{"interface":"back","average":{"1min":0.84999999999999998,"5min":0.85299999999999998,"15min":0.94999999999999996},"min":{"1min":0.35399999999999998,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":3.6040000000000001,"5min":3.6040000000000001,"15min":3.6040000000000001},"last":1.8340000000000001},{"interface":"front","average":{"1min":1.014,"5min":0.871,"15min":1.077},"min":{"1min":0.377,"5min":0.377,"15min":0.377},"max":{"1min":5.343,"5min":5.343,"15min":5.343},"last":1.0429999999999999}]},{"osd":5,"last update":"Sat Jun 3 20:03:37 2023","interfaces":[{"interface":"back","average":{"1min":0.68600000000000005,"5min":0.93200000000000005,"15min":1.159},"min":{"1min":0.34999999999999998,"5min":0.34999999999999998,"15min":0.317},"max":{"1min":1.1719999999999999,"5min":3.0750000000000002,"15min":11.083},"last":0.84999999999999998},{"interface":"front","average":{"1min":2.214,"5min":1.399,"15min":1.389},"min":{"1min":0.32900000000000001,"5min":0.32900000000000001,"15min":0.32700000000000001},"max":{"1min":8.4160000000000004,"5min":8.4160000000000004,"15min":10.154},"last":0.82899999999999996}]},{"osd":7,"last update":"Sat Jun 3 20:03:09 2023","interfaces":[{"interface":"back","average":{"1min":2.0609999999999999,"5min":1.329,"15min":1.304},"min":{"1min":0.39400000000000002,"5min":0.39400000000000002,"15min":0.313},"max":{"1min":8.7409999999999997,"5min":8.7409999999999997,"15min":8.7409999999999997},"last":0.52100000000000002},{"interface":"front","average":{"1min":2.0470000000000002,"5min":1.2929999999999999,"15min":1.395},"min":{"1min":0.433,"5min":0.33900000000000002,"15min":0.33000000000000002},"max":{"1min":6.5359999999999996,"5min":6.5359999999999996,"15min":12.407999999999999},"last":1.871}]}]},{"osd":1,"up_from":103,"seq":442381631634,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1400936,"kb_used_data":1170848,"kb_used_omap":4,"kb_used_meta":230075,"kb_avail":92344216,"statfs":{"total":95995035648,"available":94560477184,"internally_reserved":0,"allocated":1198948352,"data_stored":1198384768,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":235597710},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 3 20:03:25 2023","interfaces":[{"interface":"back","average":{"1min":7.4909999999999997,"5min":2.3479999999999999,"15min":1.921},"min":{"1min":0.217,"5min":0.185,"15min":0.185},"max":{"1min":119.902,"5min":119.902,"15min":119.902},"last":0.29299999999999998},{"interface":"front","average":{"1min":7.5640000000000001,"5min":2.2360000000000002,"15min":1.978},"min":{"1min":0.22500000000000001,"5min":0.22500000000000001,"15min":0.192},"max":{"1min":119.863,"5min":119.863,"15min":119.863},"last":0.68700000000000006}]},{"osd":2,"last update":"Sat Jun 3 20:03:34 2023","interfaces":[{"interface":"back","average":{"1min":0.97299999999999998,"5min":1.9550000000000001,"15min":1.1279999999999999},"min":{"1min":0.24099999999999999,"5min":0.23599999999999999,"15min":0.23599999999999999},"max":{"1min":4.6280000000000001,"5min":119.946,"15min":119.946},"last":1.212},{"interface":"front","average":{"1min":1.6859999999999999,"5min":1.0680000000000001,"15min":0.84999999999999998},"min":{"1min":0.26600000000000001,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":9.8160000000000007,"5min":9.8160000000000007,"15min":9.8160000000000007},"last":1.3759999999999999}]},{"osd":3,"last update":"Sat Jun 3 20:03:05 2023","interfaces":[{"interface":"back","average":{"1min":1.6399999999999999,"5min":1.3140000000000001,"15min":0.93000000000000005},"min":{"1min":0.34999999999999998,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":7.1289999999999996,"5min":9.4079999999999995,"15min":9.4079999999999995},"last":2.661},{"interface":"front","average":{"1min":1.5760000000000001,"5min":1.304,"15min":1.014},"min":{"1min":0.32900000000000001,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":7.0990000000000002,"5min":9.3200000000000003,"15min":9.3200000000000003},"last":1.4219999999999999}]},{"osd":4,"last update":"Sat Jun 3 20:03:36 2023","interfaces":[{"interface":"back","average":{"1min":2.6720000000000002,"5min":1.5149999999999999,"15min":1.167},"min":{"1min":0.38300000000000001,"5min":0.35999999999999999,"15min":0.35999999999999999},"max":{"1min":9.9589999999999996,"5min":9.9589999999999996,"15min":9.9589999999999996},"last":2.6960000000000002},{"interface":"front","average":{"1min":1.0680000000000001,"5min":2.52,"15min":1.478},"min":{"1min":0.32700000000000001,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":4.0670000000000002,"5min":120.084,"15min":120.084},"last":0.38100000000000001}]},{"osd":5,"last update":"Sat Jun 3 20:03:25 2023","interfaces":[{"interface":"back","average":{"1min":7.6779999999999999,"5min":2.8860000000000001,"15min":2.3690000000000002},"min":{"1min":0.36799999999999999,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":120.39700000000001,"5min":120.39700000000001,"15min":120.39700000000001},"last":0.56100000000000005},{"interface":"front","average":{"1min":7.75,"5min":2.7229999999999999,"15min":2.5800000000000001},"min":{"1min":0.48299999999999998,"5min":0.48299999999999998,"15min":0.40000000000000002},"max":{"1min":120.34399999999999,"5min":120.34399999999999,"15min":120.34399999999999},"last":0.73399999999999999}]},{"osd":6,"last update":"Sat Jun 3 20:03:25 2023","interfaces":[{"interface":"back","average":{"1min":7.1900000000000004,"5min":2.3290000000000002,"15min":2.2730000000000001},"min":{"1min":0.432,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":120.386,"5min":120.386,"15min":120.386},"last":0.76100000000000001},{"interface":"front","average":{"1min":7.3460000000000001,"5min":2.4500000000000002,"15min":2.5590000000000002},"min":{"1min":0.39600000000000002,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":119.96299999999999,"5min":119.96299999999999,"15min":119.96299999999999},"last":2.7210000000000001}]},{"osd":7,"last update":"Sat Jun 3 20:03:25 2023","interfaces":[{"interface":"back","average":{"1min":7.8109999999999999,"5min":2.7269999999999999,"15min":2.4430000000000001},"min":{"1min":0.40999999999999998,"5min":0.38200000000000001,"15min":0.38200000000000001},"max":{"1min":119.996,"5min":119.996,"15min":119.996},"last":7.3529999999999998},{"interface":"front","average":{"1min":7.758,"5min":2.653,"15min":2.0190000000000001},"min":{"1min":0.44800000000000001,"5min":0.40400000000000003,"15min":0.371},"max":{"1min":120.17100000000001,"5min":120.17100000000001,"15min":120.17100000000001},"last":0.59399999999999997}]}]},{"osd":0,"up_from":86,"seq":369367187661,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":764796,"kb_used_data":507572,"kb_used_omap":5,"kb_used_meta":257210,"kb_avail":92980356,"statfs":{"total":95995035648,"available":95211884544,"internally_reserved":0,"allocated":519753728,"data_stored":519172723,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":263383674},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Jun 3 20:03:15 2023","interfaces":[{"interface":"back","average":{"1min":6.5179999999999998,"5min":2.4199999999999999,"15min":2.1400000000000001},"min":{"1min":0.192,"5min":0.192,"15min":0.192},"max":{"1min":81.310000000000002,"5min":81.310000000000002,"15min":81.310000000000002},"last":0.78700000000000003},{"interface":"front","average":{"1min":5.3650000000000002,"5min":2.306,"15min":1.881},"min":{"1min":0.223,"5min":0.187,"15min":0.187},"max":{"1min":76.971999999999994,"5min":76.971999999999994,"15min":76.971999999999994},"last":0.753}]},{"osd":2,"last update":"Sat Jun 3 20:03:15 2023","interfaces":[{"interface":"back","average":{"1min":1.589,"5min":0.95799999999999996,"15min":0.76100000000000001},"min":{"1min":0.22900000000000001,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":6.0339999999999998,"5min":8.923,"15min":8.923},"last":0.42999999999999999},{"interface":"front","average":{"1min":1.653,"5min":1.0589999999999999,"15min":0.85399999999999998},"min":{"1min":0.29399999999999998,"5min":0.218,"15min":0.218},"max":{"1min":4.4139999999999997,"5min":11.194000000000001,"15min":11.194000000000001},"last":0.85599999999999998}]},{"osd":3,"last update":"Sat Jun 3 20:03:05 2023","interfaces":[{"interface":"back","average":{"1min":2.2130000000000001,"5min":1.0940000000000001,"15min":0.90800000000000003},"min":{"1min":0.38700000000000001,"5min":0.216,"15min":0.216},"max":{"1min":7.5990000000000002,"5min":7.5990000000000002,"15min":7.5990000000000002},"last":2.7370000000000001},{"interface":"front","average":{"1min":1.917,"5min":1.167,"15min":0.89600000000000002},"min":{"1min":0.36099999999999999,"5min":0.35399999999999998,"15min":0.35399999999999998},"max":{"1min":4.6340000000000003,"5min":5.3380000000000001,"15min":5.3380000000000001},"last":1.1639999999999999}]},{"osd":4,"last update":"Sat Jun 3 20:03:43 2023","interfaces":[{"interface":"back","average":{"1min":1.3700000000000001,"5min":1.0429999999999999,"15min":1.0780000000000001},"min":{"1min":0.41299999999999998,"5min":0.33800000000000002,"15min":0.33800000000000002},"max":{"1min":5.0099999999999998,"5min":5.0099999999999998,"15min":5.0099999999999998},"last":0.58999999999999997},{"interface":"front","average":{"1min":1.341,"5min":1.226,"15min":1.1539999999999999},"min":{"1min":0.43099999999999999,"5min":0.38800000000000001,"15min":0.38800000000000001},"max":{"1min":4.8140000000000001,"5min":4.8220000000000001,"15min":4.8220000000000001},"last":0.81200000000000006}]},{"osd":5,"last update":"Sat Jun 3 20:03:43 2023","interfaces":[{"interface":"back","average":{"1min":2.4980000000000002,"5min":1.3819999999999999,"15min":1.6479999999999999},"min":{"1min":0.38900000000000001,"5min":0.36799999999999999,"15min":0.33900000000000002},"max":{"1min":8.3100000000000005,"5min":8.3100000000000005,"15min":16.312000000000001},"last":0.71899999999999997},{"interface":"front","average":{"1min":2.5,"5min":1.4239999999999999,"15min":1.7190000000000001},"min":{"1min":0.36899999999999999,"5min":0.36899999999999999,"15min":0.32500000000000001},"max":{"1min":7.9779999999999998,"5min":7.9779999999999998,"15min":24.661000000000001},"last":0.64000000000000001}]},{"osd":6,"last update":"Sat Jun 3 20:03:43 2023","interfaces":[{"interface":"back","average":{"1min":1.7130000000000001,"5min":1.202,"15min":1.6240000000000001},"min":{"1min":0.46600000000000003,"5min":0.38,"15min":0.33500000000000002},"max":{"1min":6.3899999999999997,"5min":6.3899999999999997,"15min":24.599},"last":2.7970000000000002},{"interface":"front","average":{"1min":1.536,"5min":1.3859999999999999,"15min":1.7529999999999999},"min":{"1min":0.46999999999999997,"5min":0.46999999999999997,"15min":0.42999999999999999},"max":{"1min":4.2919999999999998,"5min":4.7930000000000001,"15min":12.474},"last":2.851}]},{"osd":7,"last update":"Sat Jun 3 20:03:43 2023","interfaces":[{"interface":"back","average":{"1min":2.6669999999999998,"5min":1.524,"15min":1.7190000000000001},"min":{"1min":0.52300000000000002,"5min":0.35099999999999998,"15min":0.35099999999999998},"max":{"1min":11.196,"5min":11.196,"15min":24.634},"last":0.66900000000000004},{"interface":"front","average":{"1min":2.2320000000000002,"5min":1.6459999999999999,"15min":1.7470000000000001},"min":{"1min":0.52300000000000002,"5min":0.51400000000000001,"15min":0.40699999999999997},"max":{"1min":7.327,"5min":7.327,"15min":12.621},"last":0.83399999999999996}]}]},{"osd":2,"up_from":114,"seq":489626271840,"num_pgs":22,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":417296,"kb_used_data":381576,"kb_used_omap":7,"kb_used_meta":35704,"kb_avail":93327856,"statfs":{"total":95995035648,"available":95567724544,"internally_reserved":0,"allocated":390733824,"data_stored":390191758,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8111,"internal_metadata":36560977},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 3 20:03:22 2023","interfaces":[{"interface":"back","average":{"1min":0.76100000000000001,"5min":2.2989999999999999,"15min":1.181},"min":{"1min":0.20899999999999999,"5min":0.20899999999999999,"15min":0.20899999999999999},"max":{"1min":3.4769999999999999,"5min":126.416,"15min":126.416},"last":0.46400000000000002},{"interface":"front","average":{"1min":0.67300000000000004,"5min":2.3860000000000001,"15min":1.268},"min":{"1min":0.22700000000000001,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":3.2290000000000001,"5min":126.976,"15min":126.976},"last":2.5299999999999998}]},{"osd":1,"last update":"Sat Jun 3 20:03:22 2023","interfaces":[{"interface":"back","average":{"1min":1.0449999999999999,"5min":1.296,"15min":0.91000000000000003},"min":{"1min":0.22900000000000001,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":6.5060000000000002,"5min":28.484999999999999,"15min":28.484999999999999},"last":2.3809999999999998},{"interface":"front","average":{"1min":1.0569999999999999,"5min":1.1870000000000001,"15min":0.84499999999999997},"min":{"1min":0.255,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":4.0259999999999998,"5min":28.588999999999999,"15min":28.588999999999999},"last":0.35299999999999998}]},{"osd":3,"last update":"Sat Jun 3 20:03:08 2023","interfaces":[{"interface":"back","average":{"1min":1.653,"5min":1.196,"15min":0.89900000000000002},"min":{"1min":0.23699999999999999,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":13.016999999999999,"5min":29.251999999999999,"15min":29.251999999999999},"last":0.371},{"interface":"front","average":{"1min":5.0410000000000004,"5min":2.1179999999999999,"15min":1.2110000000000001},"min":{"1min":0.29099999999999998,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":54.549999999999997,"5min":54.549999999999997,"15min":54.549999999999997},"last":0.47299999999999998}]},{"osd":4,"last update":"Sat Jun 3 20:03:42 2023","interfaces":[{"interface":"back","average":{"1min":0.747,"5min":0.879,"15min":0.90700000000000003},"min":{"1min":0.36899999999999999,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":3.359,"5min":7.9089999999999998,"15min":7.9089999999999998},"last":2.4729999999999999},{"interface":"front","average":{"1min":0.90800000000000003,"5min":0.90900000000000003,"15min":0.95299999999999996},"min":{"1min":0.32600000000000001,"5min":0.32600000000000001,"15min":0.32600000000000001},"max":{"1min":3.7749999999999999,"5min":3.7749999999999999,"15min":3.7749999999999999},"last":0.51300000000000001}]},{"osd":5,"last update":"Sat Jun 3 20:03:22 2023","interfaces":[{"interface":"back","average":{"1min":1.0049999999999999,"5min":1,"15min":1.0229999999999999},"min":{"1min":0.36599999999999999,"5min":0.36099999999999999,"15min":0.36099999999999999},"max":{"1min":3.5339999999999998,"5min":3.5339999999999998,"15min":3.5339999999999998},"last":0.42799999999999999},{"interface":"front","average":{"1min":1.583,"5min":1.327,"15min":1.1399999999999999},"min":{"1min":0.39400000000000002,"5min":0.308,"15min":0.308},"max":{"1min":4.9370000000000003,"5min":7.8689999999999998,"15min":7.8689999999999998},"last":2.4399999999999999}]},{"osd":6,"last update":"Sat Jun 3 20:03:22 2023","interfaces":[{"interface":"back","average":{"1min":1.085,"5min":1.032,"15min":1.034},"min":{"1min":0.38600000000000001,"5min":0.38500000000000001,"15min":0.38500000000000001},"max":{"1min":4.9950000000000001,"5min":6.7759999999999998,"15min":6.7759999999999998},"last":0.496},{"interface":"front","average":{"1min":1.0649999999999999,"5min":1.012,"15min":1.036},"min":{"1min":0.41199999999999998,"5min":0.36299999999999999,"15min":0.36299999999999999},"max":{"1min":4.3559999999999999,"5min":7.9550000000000001,"15min":7.9550000000000001},"last":2.5030000000000001}]},{"osd":7,"last update":"Sat Jun 3 20:03:22 2023","interfaces":[{"interface":"back","average":{"1min":2.0899999999999999,"5min":1.4990000000000001,"15min":1.282},"min":{"1min":0.44700000000000001,"5min":0.44700000000000001,"15min":0.44700000000000001},"max":{"1min":6.0800000000000001,"5min":6.0800000000000001,"15min":6.0800000000000001},"last":0.52300000000000002},{"interface":"front","average":{"1min":1.992,"5min":1.472,"15min":1.2609999999999999},"min":{"1min":0.47899999999999998,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":6.0529999999999999,"5min":7.9320000000000004,"15min":7.9320000000000004},"last":2.5659999999999998}]}]},{"osd":3,"up_from":119,"seq":511101108311,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1142916,"kb_used_data":1051644,"kb_used_omap":2,"kb_used_meta":91261,"kb_avail":92602236,"statfs":{"total":95995035648,"available":94824689664,"internally_reserved":0,"allocated":1076883456,"data_stored":1076350365,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":93451317},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 3 20:03:07 2023","interfaces":[{"interface":"back","average":{"1min":1.752,"5min":1.8129999999999999,"15min":1.0249999999999999},"min":{"1min":0.20799999999999999,"5min":0.20799999999999999,"15min":0.20799999999999999},"max":{"1min":7.1859999999999999,"5min":68.647999999999996,"15min":68.647999999999996},"last":1.7569999999999999},{"interface":"front","average":{"1min":1.091,"5min":1.702,"15min":1.054},"min":{"1min":0.28199999999999997,"5min":0.224,"15min":0.224},"max":{"1min":4.032,"5min":69.691000000000003,"15min":69.691000000000003},"last":0.34100000000000003}]},{"osd":1,"last update":"Sat Jun 3 20:03:07 2023","interfaces":[{"interface":"back","average":{"1min":2.0019999999999998,"5min":1.171,"15min":0.86399999999999999},"min":{"1min":0.32300000000000001,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":10.065,"5min":10.065,"15min":10.065},"last":1.619},{"interface":"front","average":{"1min":1.8839999999999999,"5min":1.206,"15min":0.90900000000000003},"min":{"1min":0.26100000000000001,"5min":0.23499999999999999,"15min":0.23499999999999999},"max":{"1min":10.112,"5min":10.792999999999999,"15min":10.792999999999999},"last":2.9700000000000002}]},{"osd":2,"last update":"Sat Jun 3 20:03:07 2023","interfaces":[{"interface":"back","average":{"1min":1.28,"5min":1.0089999999999999,"15min":0.76500000000000001},"min":{"1min":0.28399999999999997,"5min":0.245,"15min":0.245},"max":{"1min":3.8410000000000002,"5min":8.3149999999999995,"15min":8.3149999999999995},"last":0.373},{"interface":"front","average":{"1min":1.3540000000000001,"5min":1.083,"15min":0.86899999999999999},"min":{"1min":0.34799999999999998,"5min":0.24199999999999999,"15min":0.24199999999999999},"max":{"1min":7.7290000000000001,"5min":7.7290000000000001,"15min":7.7290000000000001},"last":2.839}]},{"osd":4,"last update":"Sat Jun 3 20:03:38 2023","interfaces":[{"interface":"back","average":{"1min":0.98099999999999998,"5min":1.1819999999999999,"15min":1.0600000000000001},"min":{"1min":0.38300000000000001,"5min":0.38300000000000001,"15min":0.38300000000000001},"max":{"1min":2.5190000000000001,"5min":4.774,"15min":4.774},"last":1.6539999999999999},{"interface":"front","average":{"1min":1.258,"5min":1.155,"15min":1.0589999999999999},"min":{"1min":0.47199999999999998,"5min":0.33000000000000002,"15min":0.33000000000000002},"max":{"1min":7.0720000000000001,"5min":7.0720000000000001,"15min":7.0720000000000001},"last":1.823}]},{"osd":5,"last update":"Sat Jun 3 20:03:07 2023","interfaces":[{"interface":"back","average":{"1min":2.1560000000000001,"5min":1.3169999999999999,"15min":1.097},"min":{"1min":0.53500000000000003,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":7.1230000000000002,"5min":7.1230000000000002,"15min":7.1230000000000002},"last":1.7929999999999999},{"interface":"front","average":{"1min":2.3490000000000002,"5min":1.508,"15min":1.1859999999999999},"min":{"1min":0.35599999999999998,"5min":0.35599999999999998,"15min":0.35599999999999998},"max":{"1min":7.2560000000000002,"5min":8.2289999999999992,"15min":8.2289999999999992},"last":0.41699999999999998}]},{"osd":6,"last update":"Sat Jun 3 20:03:07 2023","interfaces":[{"interface":"back","average":{"1min":1.7010000000000001,"5min":1.3859999999999999,"15min":1.1499999999999999},"min":{"1min":0.38600000000000001,"5min":0.38600000000000001,"15min":0.38600000000000001},"max":{"1min":7.2080000000000002,"5min":8.282,"15min":8.282},"last":0.63},{"interface":"front","average":{"1min":1.1359999999999999,"5min":1.2709999999999999,"15min":1.129},"min":{"1min":0.53900000000000003,"5min":0.499,"15min":0.499},"max":{"1min":3.3799999999999999,"5min":4.9480000000000004,"15min":4.9480000000000004},"last":1.7130000000000001}]},{"osd":7,"last update":"Sat Jun 3 20:03:07 2023","interfaces":[{"interface":"back","average":{"1min":1.446,"5min":1.341,"15min":1.143},"min":{"1min":0.56399999999999995,"5min":0.495,"15min":0.495},"max":{"1min":4.6859999999999999,"5min":5.04,"15min":5.04},"last":1.6799999999999999},{"interface":"front","average":{"1min":1.7869999999999999,"5min":1.677,"15min":1.244},"min":{"1min":0.47799999999999998,"5min":0.435,"15min":0.435},"max":{"1min":7.2439999999999998,"5min":12.715999999999999,"15min":12.715999999999999},"last":1.8500000000000001}]}]},{"osd":4,"up_from":124,"seq":532575944784,"num_pgs":36,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":460060,"kb_used_data":407892,"kb_used_omap":10,"kb_used_meta":52149,"kb_avail":93285092,"statfs":{"total":95995035648,"available":95523934208,"internally_reserved":0,"allocated":417681408,"data_stored":417128088,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":10652,"internal_metadata":53401188},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 3 20:03:42 2023","interfaces":[{"interface":"back","average":{"1min":3.7679999999999998,"5min":1.504,"15min":1.151},"min":{"1min":0.255,"5min":0.254,"15min":0.254},"max":{"1min":57.786000000000001,"5min":57.786000000000001,"15min":57.786000000000001},"last":2.1789999999999998},{"interface":"front","average":{"1min":3.4409999999999998,"5min":1.492,"15min":1.1839999999999999},"min":{"1min":0.372,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":57.820999999999998,"5min":57.820999999999998,"15min":57.820999999999998},"last":1.8959999999999999}]},{"osd":1,"last update":"Sat Jun 3 20:03:42 2023","interfaces":[{"interface":"back","average":{"1min":5.1710000000000003,"5min":1.9570000000000001,"15min":1.2330000000000001},"min":{"1min":0.28799999999999998,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":58.32,"5min":58.32,"15min":58.32},"last":0.316},{"interface":"front","average":{"1min":5.2439999999999998,"5min":1.9159999999999999,"15min":1.214},"min":{"1min":0.307,"5min":0.307,"15min":0.307},"max":{"1min":58.286000000000001,"5min":58.286000000000001,"15min":58.286000000000001},"last":2.5550000000000002}]},{"osd":2,"last update":"Sat Jun 3 20:03:42 2023","interfaces":[{"interface":"back","average":{"1min":1.1899999999999999,"5min":0.88400000000000001,"15min":0.875},"min":{"1min":0.30299999999999999,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":4.3940000000000001,"5min":4.3940000000000001,"15min":4.3940000000000001},"last":2.2799999999999998},{"interface":"front","average":{"1min":0.85399999999999998,"5min":0.86699999999999999,"15min":0.91400000000000003},"min":{"1min":0.38100000000000001,"5min":0.34499999999999997,"15min":0.34499999999999997},"max":{"1min":5.069,"5min":5.069,"15min":5.069},"last":2.4950000000000001}]},{"osd":3,"last update":"Sat Jun 3 20:03:42 2023","interfaces":[{"interface":"back","average":{"1min":6.032,"5min":2.0449999999999999,"15min":1.323},"min":{"1min":0.35299999999999998,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":57.954999999999998,"5min":57.954999999999998,"15min":57.954999999999998},"last":0.373},{"interface":"front","average":{"1min":6.0030000000000001,"5min":2.069,"15min":1.393},"min":{"1min":0.439,"5min":0.39900000000000002,"15min":0.39900000000000002},"max":{"1min":57.889000000000003,"5min":57.889000000000003,"15min":57.889000000000003},"last":2.2229999999999999}]},{"osd":5,"last update":"Sat Jun 3 20:03:42 2023","interfaces":[{"interface":"back","average":{"1min":4.6159999999999997,"5min":1.7849999999999999,"15min":1.23},"min":{"1min":0.38,"5min":0.36099999999999999,"15min":0.36099999999999999},"max":{"1min":86.576999999999998,"5min":86.576999999999998,"15min":86.576999999999998},"last":0.82299999999999995},{"interface":"front","average":{"1min":5.0209999999999999,"5min":1.919,"15min":1.298},"min":{"1min":0.38100000000000001,"5min":0.32600000000000001,"15min":0.32600000000000001},"max":{"1min":88.013000000000005,"5min":88.013000000000005,"15min":88.013000000000005},"last":0.45400000000000001}]},{"osd":6,"last update":"Sat Jun 3 20:03:42 2023","interfaces":[{"interface":"back","average":{"1min":0.95199999999999996,"5min":0.90100000000000002,"15min":0.997},"min":{"1min":0.36599999999999999,"5min":0.36599999999999999,"15min":0.36599999999999999},"max":{"1min":5.1420000000000003,"5min":5.1420000000000003,"15min":5.1420000000000003},"last":0.60599999999999998},{"interface":"front","average":{"1min":1.1240000000000001,"5min":0.94499999999999995,"15min":1.034},"min":{"1min":0.39700000000000002,"5min":0.36499999999999999,"15min":0.36499999999999999},"max":{"1min":7.2220000000000004,"5min":7.2220000000000004,"15min":7.2220000000000004},"last":0.56899999999999995}]},{"osd":7,"last update":"Sat Jun 3 20:03:42 2023","interfaces":[{"interface":"back","average":{"1min":1.8049999999999999,"5min":1.3420000000000001,"15min":1.206},"min":{"1min":0.497,"5min":0.41799999999999998,"15min":0.41799999999999998},"max":{"1min":5.3179999999999996,"5min":10.048999999999999,"15min":10.048999999999999},"last":6.2050000000000001},{"interface":"front","average":{"1min":2.1869999999999998,"5min":1.4710000000000001,"15min":1.2709999999999999},"min":{"1min":0.31900000000000001,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":9.3309999999999995,"5min":9.3309999999999995,"15min":9.3309999999999995},"last":2.9510000000000001}]}]},{"osd":5,"up_from":36,"seq":154618823070,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1656944,"kb_used_data":1188712,"kb_used_omap":0,"kb_used_meta":468224,"kb_avail":92088208,"statfs":{"total":95995035648,"available":94298324992,"internally_reserved":0,"allocated":1217241088,"data_stored":1216782219,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":479461376},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 3 20:03:27 2023","interfaces":[{"interface":"back","average":{"1min":1.2589999999999999,"5min":1.369,"15min":1.472},"min":{"1min":0.26900000000000002,"5min":0.22800000000000001,"15min":0.22800000000000001},"max":{"1min":4.2199999999999998,"5min":37.689999999999998,"15min":37.689999999999998},"last":0.41499999999999998},{"interface":"front","average":{"1min":2.0920000000000001,"5min":1.8140000000000001,"15min":1.8480000000000001},"min":{"1min":0.26300000000000001,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":8.266,"5min":37.783999999999999,"15min":37.783999999999999},"last":0.373}]},{"osd":1,"last update":"Sat Jun 3 20:03:27 2023","interfaces":[{"interface":"back","average":{"1min":6.7569999999999997,"5min":3.7749999999999999,"15min":2.4620000000000002},"min":{"1min":0.47399999999999998,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":47.747999999999998,"5min":114.681,"15min":114.681},"last":3.5670000000000002},{"interface":"front","average":{"1min":6.1749999999999998,"5min":3.726,"15min":2.5059999999999998},"min":{"1min":0.312,"5min":0.307,"15min":0.29999999999999999},"max":{"1min":48.015000000000001,"5min":115.366,"15min":115.366},"last":0.94199999999999995}]},{"osd":2,"last update":"Sat Jun 3 20:03:27 2023","interfaces":[{"interface":"back","average":{"1min":1.177,"5min":0.97299999999999998,"15min":0.90500000000000003},"min":{"1min":0.35399999999999998,"5min":0.35099999999999998,"15min":0.34999999999999998},"max":{"1min":4.0549999999999997,"5min":4.3529999999999998,"15min":4.3529999999999998},"last":0.45600000000000002},{"interface":"front","average":{"1min":1.784,"5min":1.2929999999999999,"15min":1.0800000000000001},"min":{"1min":0.34100000000000003,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":8.2119999999999997,"5min":8.2119999999999997,"15min":8.2119999999999997},"last":0.40400000000000003}]},{"osd":3,"last update":"Sat Jun 3 20:03:10 2023","interfaces":[{"interface":"back","average":{"1min":4.6360000000000001,"5min":4.7800000000000002,"15min":2.2719999999999998},"min":{"1min":0.35299999999999998,"5min":0.34599999999999997,"15min":0.34599999999999997},"max":{"1min":48.308,"5min":261.14100000000002,"15min":261.14100000000002},"last":4.726},{"interface":"front","average":{"1min":3.597,"5min":4.4240000000000004,"15min":2.1349999999999998},"min":{"1min":0.35399999999999998,"5min":0.35399999999999998,"15min":0.35399999999999998},"max":{"1min":48.124000000000002,"5min":261.27100000000002,"15min":261.27100000000002},"last":0.50700000000000001}]},{"osd":4,"last update":"Sat Jun 3 20:03:43 2023","interfaces":[{"interface":"back","average":{"1min":1.9550000000000001,"5min":1.5169999999999999,"15min":1.173},"min":{"1min":0.40400000000000003,"5min":0.378,"15min":0.378},"max":{"1min":4.8479999999999999,"5min":10.162000000000001,"15min":10.162000000000001},"last":3.6360000000000001},{"interface":"front","average":{"1min":1.1819999999999999,"5min":1.0129999999999999,"15min":1.129},"min":{"1min":0.377,"5min":0.377,"15min":0.377},"max":{"1min":4.9980000000000002,"5min":4.9980000000000002,"15min":4.9980000000000002},"last":0.52900000000000003}]},{"osd":6,"last update":"Sat Jun 3 20:03:27 2023","interfaces":[{"interface":"back","average":{"1min":2.5419999999999998,"5min":1.6439999999999999,"15min":1.4450000000000001},"min":{"1min":0.56100000000000005,"5min":0.41199999999999998,"15min":0.33000000000000002},"max":{"1min":10.278,"5min":10.278,"15min":12.731999999999999},"last":3.6709999999999998},{"interface":"front","average":{"1min":2.4060000000000001,"5min":1.712,"15min":1.532},"min":{"1min":0.438,"5min":0.438,"15min":0.32800000000000001},"max":{"1min":13.714,"5min":13.714,"15min":13.714},"last":3.7010000000000001}]},{"osd":7,"last update":"Sat Jun 3 20:03:14 2023","interfaces":[{"interface":"back","average":{"1min":3.46,"5min":1.9430000000000001,"15min":1.655},"min":{"1min":0.63600000000000001,"5min":0.503,"15min":0.379},"max":{"1min":10.192,"5min":10.192,"15min":11.297000000000001},"last":3.7309999999999999},{"interface":"front","average":{"1min":1.4379999999999999,"5min":1.2709999999999999,"15min":1.234},"min":{"1min":0.50600000000000001,"5min":0.43099999999999999,"15min":0.32100000000000001},"max":{"1min":4.2839999999999998,"5min":5.8719999999999999,"15min":7.4720000000000004},"last":0.48299999999999998}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6928,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4862,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":10652,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4786,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4932,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":515047424,"data_stored":515047424,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":1181487104,"data_stored":1181483032,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":383123456,"data_stored":383123456,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":1062666240,"data_stored":1062666240,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":408223744,"data_stored":408223744,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":1197150208,"data_stored":1197146136,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":398589952,"data_stored":398589952,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":1187844096,"data_stored":1187840024,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-06-03T20:04:05.557 INFO:tasks.cephadm.ceph_manager.ceph:recovered! 2023-06-03T20:04:05.558 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-06-03T20:04:05.558 INFO:tasks.thrashosds.thrasher:fixing pg num pool unique_pool_4 2023-06-03T20:04:05.558 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph pg dump --format=json 2023-06-03T20:04:05.611 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:04:05 smithi164 bash[59462]: cluster 1685822644.5017126 mgr.y (mgr.34104) 835 : cluster 0 pgmap v755: 89 pgs: 89 active+clean; 1.8 GiB data, 7.2 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 810 op/s 2023-06-03T20:04:05.637 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:04:05.697 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:04:05.700 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:04:05.701 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:04:05.778 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:04:05 smithi089 bash[78452]: cluster 1685822644.5017126 mgr.y (mgr.34104) 835 : cluster 0 pgmap v755: 89 pgs: 89 active+clean; 1.8 GiB data, 7.2 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 810 op/s 2023-06-03T20:04:05.779 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:04:05 smithi089 bash[75068]: cluster 1685822644.5017126 mgr.y (mgr.34104) 835 : cluster 0 pgmap v755: 89 pgs: 89 active+clean; 1.8 GiB data, 7.2 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 810 op/s 2023-06-03T20:04:05.802 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:04:05.843 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:04:05.843 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:04:05.845 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:04:05.945 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:04:05.990 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:04:05.993 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:04:05.994 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:04:06.095 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:04:06.127 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:04:06.134 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:04:06.135 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:04:06.236 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:04:06.287 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:04:06.289 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:04:06.289 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:04:06.390 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:04:06.429 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:04:06.434 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:04:06.434 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:04:06.528 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 20:04:06 smithi089 bash[71163]: ::ffff:172.21.15.164 - - [03/Jun/2023:20:04:06] "GET /metrics HTTP/1.1" 200 228310 "" "Prometheus/2.33.4" 2023-06-03T20:04:06.536 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:04:06.569 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:04:06.573 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:04:06.574 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:04:06.674 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:04:06.704 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:04:06.706 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:04:06.707 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:04:06.808 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:04:06.846 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:04:06.848 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:04:06.849 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:04:06.950 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:04:06.981 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:04:06.985 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:04:06.986 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:04:07.086 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:04:07.119 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:04:07.121 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:04:07.121 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:04:07.222 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:04:07.252 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:04:07.258 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:04:07.259 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:04:07.360 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:04:07.392 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:04:07.396 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:04:07.396 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:04:07.497 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:04:07.534 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:04:07.539 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:04:07.540 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:04:07.641 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:04:07.680 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:04:07.686 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:04:07.686 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:04:07.787 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:04:07.821 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:04:07.826 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:04:07.827 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:04:07.833 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:04:07 smithi164 bash[59462]: cluster 1685822646.5022354 mgr.y (mgr.34104) 836 : cluster 0 pgmap v756: 89 pgs: 89 active+clean; 1.8 GiB data, 7.1 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 629 op/s 2023-06-03T20:04:07.833 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:04:07 smithi164 bash[59462]: audit 1685822647.1049583 mon.c (mon.2) 411 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T20:04:07.901 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:04:07 smithi089 bash[75068]: cluster 1685822646.5022354 mgr.y (mgr.34104) 836 : cluster 0 pgmap v756: 89 pgs: 89 active+clean; 1.8 GiB data, 7.1 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 629 op/s 2023-06-03T20:04:07.902 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:04:07 smithi089 bash[75068]: audit 1685822647.1049583 mon.c (mon.2) 411 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T20:04:07.902 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:04:07 smithi089 bash[78452]: cluster 1685822646.5022354 mgr.y (mgr.34104) 836 : cluster 0 pgmap v756: 89 pgs: 89 active+clean; 1.8 GiB data, 7.1 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 629 op/s 2023-06-03T20:04:07.902 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:04:07 smithi089 bash[78452]: audit 1685822647.1049583 mon.c (mon.2) 411 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T20:04:07.928 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:04:07.967 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:04:07.974 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:04:07.974 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:04:08.075 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:04:08.115 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:04:08.117 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:04:08.118 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:04:08.219 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:04:08.285 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:04:08.292 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:04:08.292 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:04:08.393 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:04:08.424 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:04:08.430 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:04:08.431 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:04:08.532 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:04:08.565 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:04:08.570 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:04:08.570 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:04:08.672 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:04:08.718 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:04:08.722 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:04:08.723 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:04:08.823 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:04:08.857 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:04:08.864 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:04:08.865 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:04:08.966 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:04:09.001 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:04:09.005 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:04:09.006 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:04:09.107 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:04:09.136 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:04:09.138 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:04:09.138 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:04:09.239 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:04:09.275 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:04:09.277 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:04:09.278 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:04:09.380 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:04:09.435 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:04:09.437 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:04:09.438 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:04:09.539 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:04:09.581 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:04:09.590 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:04:09.591 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:04:09.692 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:04:09.734 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:04:09.743 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:04:09.744 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:04:09.833 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:04:09 smithi164 bash[59462]: cluster 1685822648.5029025 mgr.y (mgr.34104) 837 : cluster 0 pgmap v757: 89 pgs: 89 active+clean; 1.6 GiB data, 6.7 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 765 op/s 2023-06-03T20:04:09.845 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:04:09.881 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:04:09.885 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:04:09.885 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:04:09.886 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:04:09 smithi089 bash[78452]: cluster 1685822648.5029025 mgr.y (mgr.34104) 837 : cluster 0 pgmap v757: 89 pgs: 89 active+clean; 1.6 GiB data, 6.7 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 765 op/s 2023-06-03T20:04:09.886 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:04:09 smithi089 bash[75068]: cluster 1685822648.5029025 mgr.y (mgr.34104) 837 : cluster 0 pgmap v757: 89 pgs: 89 active+clean; 1.6 GiB data, 6.7 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 765 op/s 2023-06-03T20:04:09.986 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:04:10.014 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:04:10.020 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:04:10.021 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:04:10.122 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:04:10.160 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:04:10.164 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:04:10.165 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:04:10.244 INFO:teuthology.orchestra.run.smithi089.stdout:osd.0: {} 2023-06-03T20:04:10.266 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:04:10.292 INFO:teuthology.orchestra.run.smithi089.stdout:osd.1: {} 2023-06-03T20:04:10.295 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:04:10.301 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:04:10.301 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:04:10.331 INFO:teuthology.orchestra.run.smithi089.stdout:osd.2: {} 2023-06-03T20:04:10.376 INFO:teuthology.orchestra.run.smithi089.stdout:osd.3: {} 2023-06-03T20:04:10.403 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:04:10.419 INFO:teuthology.orchestra.run.smithi089.stdout:osd.4: {} 2023-06-03T20:04:10.443 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:04:10.450 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:04:10.451 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:04:10.465 INFO:teuthology.orchestra.run.smithi089.stdout:osd.5: {} 2023-06-03T20:04:10.501 INFO:teuthology.orchestra.run.smithi089.stdout:osd.6: {} 2023-06-03T20:04:10.521 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/mon.a/config 2023-06-03T20:04:10.529 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/mon.a/config 2023-06-03T20:04:10.552 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:04:10.576 INFO:teuthology.orchestra.run.smithi089.stdout:osd.7: {} 2023-06-03T20:04:10.577 INFO:teuthology.orchestra.run.smithi089.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-06-03T20:04:10.577 INFO:teuthology.orchestra.run.smithi089.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-06-03T20:04:10.577 INFO:teuthology.orchestra.run.smithi089.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-06-03T20:04:10.577 INFO:teuthology.orchestra.run.smithi089.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-06-03T20:04:10.578 INFO:teuthology.orchestra.run.smithi089.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-06-03T20:04:10.578 INFO:teuthology.orchestra.run.smithi089.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-06-03T20:04:10.578 INFO:teuthology.orchestra.run.smithi089.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-06-03T20:04:10.578 INFO:teuthology.orchestra.run.smithi089.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-06-03T20:04:10.578 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:04:10.581 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:04:10.582 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:04:10.654 INFO:teuthology.orchestra.run.smithi089.stderr:nodeep-scrub is set 2023-06-03T20:04:10.683 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:04:10.713 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:04:10.718 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:04:10.718 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:04:10.819 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:04:10.828 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:04:10 smithi089 bash[75068]: audit 1685822650.1748083 mon.c (mon.2) 412 : audit 0 from='client.? 172.21.15.89:0/1638015690' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-03T20:04:10.829 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:04:10 smithi089 bash[75068]: audit 1685822650.5533638 mon.c (mon.2) 413 : audit 1 from='client.? 172.21.15.89:0/2891987602' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-06-03T20:04:10.829 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:04:10 smithi089 bash[75068]: audit 1685822650.5543013 mon.a (mon.0) 742 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-06-03T20:04:10.829 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:04:10 smithi089 bash[78452]: audit 1685822650.1748083 mon.c (mon.2) 412 : audit 0 from='client.? 172.21.15.89:0/1638015690' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-03T20:04:10.829 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:04:10 smithi089 bash[78452]: audit 1685822650.5533638 mon.c (mon.2) 413 : audit 1 from='client.? 172.21.15.89:0/2891987602' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-06-03T20:04:10.829 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:04:10 smithi089 bash[78452]: audit 1685822650.5543013 mon.a (mon.0) 742 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-06-03T20:04:10.833 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:04:10 smithi164 bash[59462]: audit 1685822650.1748083 mon.c (mon.2) 412 : audit 0 from='client.? 172.21.15.89:0/1638015690' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-03T20:04:10.833 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:04:10 smithi164 bash[59462]: audit 1685822650.5533638 mon.c (mon.2) 413 : audit 1 from='client.? 172.21.15.89:0/2891987602' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-06-03T20:04:10.833 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:04:10 smithi164 bash[59462]: audit 1685822650.5543013 mon.a (mon.0) 742 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-06-03T20:04:10.859 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:04:10.864 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:04:10.865 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:04:10.966 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:04:11.006 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:04:11.015 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:04:11.016 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:04:11.117 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:04:11.152 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:04:11.158 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:04:11.158 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:04:11.259 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:04:11.291 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:04:11.293 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:04:11.294 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:04:11.395 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:04:11.422 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:04:11.423 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:04:11.424 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:04:11.525 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:04:11.566 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:04:11.570 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:04:11.571 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:04:11.671 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:04:11.688 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:04:11 smithi089 bash[75068]: cluster 1685822650.5035002 mgr.y (mgr.34104) 838 : cluster 0 pgmap v758: 89 pgs: 89 active+clean; 1.5 GiB data, 6.5 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 663 op/s 2023-06-03T20:04:11.689 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:04:11 smithi089 bash[75068]: cluster 1685822650.615798 mon.a (mon.0) 743 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-06-03T20:04:11.689 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:04:11 smithi089 bash[75068]: cluster 1685822650.627716 osd.6 (osd.6) 63 : cluster 3 failed to encode map e144 with expected crc 2023-06-03T20:04:11.689 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:04:11 smithi089 bash[75068]: cluster 1685822650.6304896 osd.5 (osd.5) 68 : cluster 3 failed to encode map e144 with expected crc 2023-06-03T20:04:11.689 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:04:11 smithi089 bash[75068]: audit 1685822650.6350636 mon.a (mon.0) 744 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-06-03T20:04:11.690 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:04:11 smithi089 bash[75068]: cluster 1685822650.6352942 mon.a (mon.0) 745 : cluster 0 osdmap e144: 8 total, 8 up, 8 in 2023-06-03T20:04:11.690 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:04:11 smithi089 bash[75068]: cluster 1685822650.639336 osd.7 (osd.7) 56 : cluster 3 failed to encode map e144 with expected crc 2023-06-03T20:04:11.690 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:04:11 smithi089 bash[78452]: cluster 1685822650.5035002 mgr.y (mgr.34104) 838 : cluster 0 pgmap v758: 89 pgs: 89 active+clean; 1.5 GiB data, 6.5 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 663 op/s 2023-06-03T20:04:11.691 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:04:11 smithi089 bash[78452]: cluster 1685822650.615798 mon.a (mon.0) 743 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-06-03T20:04:11.691 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:04:11 smithi089 bash[78452]: cluster 1685822650.627716 osd.6 (osd.6) 63 : cluster 3 failed to encode map e144 with expected crc 2023-06-03T20:04:11.691 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:04:11 smithi089 bash[78452]: cluster 1685822650.6304896 osd.5 (osd.5) 68 : cluster 3 failed to encode map e144 with expected crc 2023-06-03T20:04:11.691 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:04:11 smithi089 bash[78452]: audit 1685822650.6350636 mon.a (mon.0) 744 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-06-03T20:04:11.691 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:04:11 smithi089 bash[78452]: cluster 1685822650.6352942 mon.a (mon.0) 745 : cluster 0 osdmap e144: 8 total, 8 up, 8 in 2023-06-03T20:04:11.692 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:04:11 smithi089 bash[78452]: cluster 1685822650.639336 osd.7 (osd.7) 56 : cluster 3 failed to encode map e144 with expected crc 2023-06-03T20:04:11.709 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:04:11.715 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:04:11.715 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:04:11.817 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:04:11.826 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:04:11 smithi164 bash[59462]: cluster 1685822650.5035002 mgr.y (mgr.34104) 838 : cluster 0 pgmap v758: 89 pgs: 89 active+clean; 1.5 GiB data, 6.5 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 663 op/s 2023-06-03T20:04:11.826 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:04:11 smithi164 bash[59462]: cluster 1685822650.615798 mon.a (mon.0) 743 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-06-03T20:04:11.826 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:04:11 smithi164 bash[59462]: cluster 1685822650.627716 osd.6 (osd.6) 63 : cluster 3 failed to encode map e144 with expected crc 2023-06-03T20:04:11.826 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:04:11 smithi164 bash[59462]: cluster 1685822650.6304896 osd.5 (osd.5) 68 : cluster 3 failed to encode map e144 with expected crc 2023-06-03T20:04:11.827 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:04:11 smithi164 bash[59462]: audit 1685822650.6350636 mon.a (mon.0) 744 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-06-03T20:04:11.827 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:04:11 smithi164 bash[59462]: cluster 1685822650.6352942 mon.a (mon.0) 745 : cluster 0 osdmap e144: 8 total, 8 up, 8 in 2023-06-03T20:04:11.827 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:04:11 smithi164 bash[59462]: cluster 1685822650.639336 osd.7 (osd.7) 56 : cluster 3 failed to encode map e144 with expected crc 2023-06-03T20:04:11.860 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:04:11.864 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:04:11.865 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:04:11.966 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:04:11.998 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:04:12.002 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:04:12.003 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:04:12.104 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:04:12.132 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:04:12.138 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:04:12.140 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:04:12.241 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:04:12.277 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:04:12.282 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:04:12.282 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:04:12.383 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:04:12.411 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:04:12.418 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:04:12.419 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:04:12.520 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:04:12.556 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:04:12.558 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:04:12.559 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:04:12.660 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:04:12.687 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:04:12.689 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:04:12.689 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:04:12.790 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:04:12.818 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:04:12.823 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:04:12.824 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:04:12.925 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:04:12.959 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:04:12.960 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:04:12.960 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:04:13.061 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:04:13.089 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:04:13.097 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:04:13.097 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:04:13.198 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:04:13.239 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:04:13.244 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:04:13.245 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:04:13.346 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:04:13.372 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:04:13.374 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:04:13.375 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:04:13.476 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:04:13.508 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:04:13.511 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:04:13.512 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:04:13.613 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:04:13.645 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:04:13.651 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:04:13.652 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:04:13.753 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:04:13.806 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:04:13.809 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:04:13.810 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:04:13.911 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:04:13.921 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:04:13 smithi164 bash[59462]: cluster 1685822652.5039742 mgr.y (mgr.34104) 839 : cluster 0 pgmap v760: 89 pgs: 89 active+clean; 1.5 GiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 685 op/s 2023-06-03T20:04:13.959 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:04:13.965 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:04:13.965 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:04:14.028 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:04:13 smithi089 bash[75068]: cluster 1685822652.5039742 mgr.y (mgr.34104) 839 : cluster 0 pgmap v760: 89 pgs: 89 active+clean; 1.5 GiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 685 op/s 2023-06-03T20:04:14.029 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:04:13 smithi089 bash[78452]: cluster 1685822652.5039742 mgr.y (mgr.34104) 839 : cluster 0 pgmap v760: 89 pgs: 89 active+clean; 1.5 GiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 685 op/s 2023-06-03T20:04:14.066 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:04:14.102 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:04:14.105 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:04:14.105 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:04:14.206 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:04:14.245 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:04:14.251 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:04:14.252 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:04:14.353 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:04:14.388 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:04:14.391 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:04:14.391 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:04:14.492 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:04:14.526 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:04:14.528 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:04:14.529 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:04:14.630 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:04:14.679 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:04:14.682 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:04:14.683 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:04:14.784 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:04:14.842 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:04:14.849 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:04:14.849 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:04:14.950 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:04:14.990 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:04:14.994 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:04:14.995 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:04:15.096 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:04:15.131 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:04:15.134 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:04:15.135 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:04:15.236 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:04:15.266 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:04:15.271 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:04:15.272 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:04:15.373 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:04:15.408 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:04:15.414 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:04:15.415 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:04:15.516 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:04:15.555 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:04:15.561 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:04:15.562 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:04:15.663 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:04:15.689 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:04:15.691 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:04:15.692 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:04:15.778 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:04:15 smithi089 bash[75068]: cluster 1685822654.5045218 mgr.y (mgr.34104) 840 : cluster 0 pgmap v761: 89 pgs: 89 active+clean; 1.4 GiB data, 5.9 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 696 op/s 2023-06-03T20:04:15.779 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:04:15 smithi089 bash[78452]: cluster 1685822654.5045218 mgr.y (mgr.34104) 840 : cluster 0 pgmap v761: 89 pgs: 89 active+clean; 1.4 GiB data, 5.9 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 696 op/s 2023-06-03T20:04:15.793 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:04:15.839 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:04:15.842 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:04:15.843 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:04:15.944 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:04:15.982 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:04:15.987 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:04:15.987 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:04:16.083 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:04:15 smithi164 bash[59462]: cluster 1685822654.5045218 mgr.y (mgr.34104) 840 : cluster 0 pgmap v761: 89 pgs: 89 active+clean; 1.4 GiB data, 5.9 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 696 op/s 2023-06-03T20:04:16.089 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:04:16.136 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:04:16.145 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:04:16.145 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:04:16.246 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:04:16.277 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:04:16.281 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:04:16.282 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:04:16.382 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:04:16.417 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:04:16.420 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:04:16.421 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:04:16.521 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:04:16.528 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 20:04:16 smithi089 bash[71163]: ::ffff:172.21.15.164 - - [03/Jun/2023:20:04:16] "GET /metrics HTTP/1.1" 200 228309 "" "Prometheus/2.33.4" 2023-06-03T20:04:16.556 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:04:16.561 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:04:16.562 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:04:16.663 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:04:16.709 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:04:16.710 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:04:16.711 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:04:16.812 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:04:16.847 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-06-03T20:04:16.868 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:04:16.869 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:04:16.869 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:04:16.970 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:04:16.989 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph osd unset noscrub 2023-06-03T20:04:17.006 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:04:17.012 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:04:17.013 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:04:17.114 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:04:17.147 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:04:17.153 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:04:17.153 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:04:17.254 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:04:17.301 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:04:17.305 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:04:17.305 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:04:17.406 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:04:17.437 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:04:17.443 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:04:17.443 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:04:17.544 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:04:17.575 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:04:17.578 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:04:17.579 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:04:17.680 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:04:17.733 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:04:17.737 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:04:17.738 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:04:17.839 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:04:17.848 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:04:17 smithi089 bash[75068]: cluster 1685822656.5050256 mgr.y (mgr.34104) 841 : cluster 0 pgmap v762: 89 pgs: 89 active+clean; 1.3 GiB data, 5.8 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 695 op/s 2023-06-03T20:04:17.848 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:04:17 smithi089 bash[78452]: cluster 1685822656.5050256 mgr.y (mgr.34104) 841 : cluster 0 pgmap v762: 89 pgs: 89 active+clean; 1.3 GiB data, 5.8 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 695 op/s 2023-06-03T20:04:17.865 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:04:17.871 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:04:17.872 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:04:17.973 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:04:17.982 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:04:17 smithi164 bash[59462]: cluster 1685822656.5050256 mgr.y (mgr.34104) 841 : cluster 0 pgmap v762: 89 pgs: 89 active+clean; 1.3 GiB data, 5.8 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 695 op/s 2023-06-03T20:04:18.012 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:04:18.017 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:04:18.018 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:04:18.119 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:04:18.153 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:04:18.158 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:04:18.158 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:04:18.259 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:04:18.293 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:04:18.294 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:04:18.295 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:04:18.396 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:04:18.764 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:04:18.771 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:04:18.772 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:04:18.873 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:04:18.917 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:04:18.919 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:04:18.920 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:04:19.021 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:04:19.061 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:04:19.068 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:04:19.069 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:04:19.170 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:04:19.208 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:04:19.211 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:04:19.211 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:04:19.312 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:04:19.345 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:04:19.349 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:04:19.350 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:04:19.451 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:04:19.491 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:04:19.497 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:04:19.498 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:04:19.598 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:04:19.636 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:04:19.643 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:04:19.644 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:04:19.745 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:04:19.754 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:04:19 smithi164 bash[59462]: cluster 1685822658.505643 mgr.y (mgr.34104) 842 : cluster 0 pgmap v763: 89 pgs: 89 active+clean; 1.2 GiB data, 5.4 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 709 op/s 2023-06-03T20:04:19.781 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:04:19.786 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:04:19.787 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:04:19.888 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:04:19.936 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:04:19.944 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:04:19.945 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:04:20.029 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:04:19 smithi089 bash[75068]: cluster 1685822658.505643 mgr.y (mgr.34104) 842 : cluster 0 pgmap v763: 89 pgs: 89 active+clean; 1.2 GiB data, 5.4 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 709 op/s 2023-06-03T20:04:20.029 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:04:19 smithi089 bash[78452]: cluster 1685822658.505643 mgr.y (mgr.34104) 842 : cluster 0 pgmap v763: 89 pgs: 89 active+clean; 1.2 GiB data, 5.4 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 709 op/s 2023-06-03T20:04:20.046 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:04:20.101 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:04:20.102 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:04:20.102 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:04:20.203 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:04:20.235 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:04:20.242 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:04:20.243 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:04:20.344 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:04:20.378 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:04:20.381 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:04:20.382 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:04:20.483 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:04:20.534 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:04:20.537 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:04:20.537 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:04:20.617 INFO:teuthology.orchestra.run.smithi089.stdout:{ 2023-06-03T20:04:20.618 INFO:teuthology.orchestra.run.smithi089.stdout: "size": 20, 2023-06-03T20:04:20.618 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 600, 2023-06-03T20:04:20.618 INFO:teuthology.orchestra.run.smithi089.stdout: "ops": [ 2023-06-03T20:04:20.619 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:04:20.619 INFO:teuthology.orchestra.run.smithi089.stdout: "description": "osd_op(client.45713.0:4728 9.2 9:4f42c2fa:::benchmark_data_smithi089_115580_object4727:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e131)", 2023-06-03T20:04:20.619 INFO:teuthology.orchestra.run.smithi089.stdout: "initiated_at": "2023-06-03T20:00:42.831135+0000", 2023-06-03T20:04:20.619 INFO:teuthology.orchestra.run.smithi089.stdout: "age": 217.77752377499999, 2023-06-03T20:04:20.620 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.32749713899999999, 2023-06-03T20:04:20.620 INFO:teuthology.orchestra.run.smithi089.stdout: "type_data": { 2023-06-03T20:04:20.620 INFO:teuthology.orchestra.run.smithi089.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-03T20:04:20.620 INFO:teuthology.orchestra.run.smithi089.stdout: "client_info": { 2023-06-03T20:04:20.620 INFO:teuthology.orchestra.run.smithi089.stdout: "client": "client.45713", 2023-06-03T20:04:20.621 INFO:teuthology.orchestra.run.smithi089.stdout: "client_addr": "172.21.15.89:0/2400821448", 2023-06-03T20:04:20.621 INFO:teuthology.orchestra.run.smithi089.stdout: "tid": 4728 2023-06-03T20:04:20.621 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:04:20.621 INFO:teuthology.orchestra.run.smithi089.stdout: "events": [ 2023-06-03T20:04:20.622 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:04:20.622 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "initiated", 2023-06-03T20:04:20.622 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:42.831135+0000", 2023-06-03T20:04:20.622 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0 2023-06-03T20:04:20.622 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:04:20.623 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:04:20.623 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "throttled", 2023-06-03T20:04:20.623 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:42.831135+0000", 2023-06-03T20:04:20.623 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0 2023-06-03T20:04:20.623 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:04:20.624 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:04:20.624 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "header_read", 2023-06-03T20:04:20.624 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:42.831134+0000", 2023-06-03T20:04:20.624 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 4294967295.9999986 2023-06-03T20:04:20.625 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:04:20.625 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:04:20.625 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "all_read", 2023-06-03T20:04:20.625 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:42.831209+0000", 2023-06-03T20:04:20.625 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 7.4967000000000002e-05 2023-06-03T20:04:20.626 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:04:20.626 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:04:20.626 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "dispatched", 2023-06-03T20:04:20.626 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:42.831221+0000", 2023-06-03T20:04:20.627 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 1.1541e-05 2023-06-03T20:04:20.627 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:04:20.627 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:04:20.627 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "queued_for_pg", 2023-06-03T20:04:20.628 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:42.831227+0000", 2023-06-03T20:04:20.628 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 6.3810000000000004e-06 2023-06-03T20:04:20.628 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:04:20.628 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:04:20.628 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "reached_pg", 2023-06-03T20:04:20.629 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:42.831270+0000", 2023-06-03T20:04:20.629 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 4.2802000000000001e-05 2023-06-03T20:04:20.629 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:04:20.629 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:04:20.629 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "started", 2023-06-03T20:04:20.630 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:42.831399+0000", 2023-06-03T20:04:20.630 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.00012926699999999999 2023-06-03T20:04:20.630 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:04:20.630 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:04:20.631 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "waiting for subops from 1,2", 2023-06-03T20:04:20.631 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:42.831560+0000", 2023-06-03T20:04:20.631 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.000160786 2023-06-03T20:04:20.631 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:04:20.631 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:04:20.632 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "op_commit", 2023-06-03T20:04:20.632 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:42.833412+0000", 2023-06-03T20:04:20.632 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.001852018 2023-06-03T20:04:20.632 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:04:20.633 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:04:20.633 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:04:20.633 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:42.833750+0000", 2023-06-03T20:04:20.633 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.00033845099999999998 2023-06-03T20:04:20.633 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:04:20.633 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:04:20.634 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:04:20.634 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:43.158538+0000", 2023-06-03T20:04:20.634 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.32478789600000002 2023-06-03T20:04:20.634 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:04:20.634 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:04:20.634 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "commit_sent", 2023-06-03T20:04:20.635 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:43.158592+0000", 2023-06-03T20:04:20.635 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 5.3693000000000002e-05 2023-06-03T20:04:20.635 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:04:20.635 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:04:20.636 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "done", 2023-06-03T20:04:20.636 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:43.158632+0000", 2023-06-03T20:04:20.636 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 4.0540999999999997e-05 2023-06-03T20:04:20.636 INFO:teuthology.orchestra.run.smithi089.stdout: } 2023-06-03T20:04:20.636 INFO:teuthology.orchestra.run.smithi089.stdout: ] 2023-06-03T20:04:20.636 INFO:teuthology.orchestra.run.smithi089.stdout: } 2023-06-03T20:04:20.636 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:04:20.637 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:04:20.637 INFO:teuthology.orchestra.run.smithi089.stdout: "description": "osd_op(client.45713.0:4736 9.2 9:41711c33:::benchmark_data_smithi089_115580_object4735:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e131)", 2023-06-03T20:04:20.637 INFO:teuthology.orchestra.run.smithi089.stdout: "initiated_at": "2023-06-03T20:00:42.833775+0000", 2023-06-03T20:04:20.637 INFO:teuthology.orchestra.run.smithi089.stdout: "age": 217.77488419299999, 2023-06-03T20:04:20.637 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.32498873099999998, 2023-06-03T20:04:20.637 INFO:teuthology.orchestra.run.smithi089.stdout: "type_data": { 2023-06-03T20:04:20.637 INFO:teuthology.orchestra.run.smithi089.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-03T20:04:20.638 INFO:teuthology.orchestra.run.smithi089.stdout: "client_info": { 2023-06-03T20:04:20.638 INFO:teuthology.orchestra.run.smithi089.stdout: "client": "client.45713", 2023-06-03T20:04:20.638 INFO:teuthology.orchestra.run.smithi089.stdout: "client_addr": "172.21.15.89:0/2400821448", 2023-06-03T20:04:20.638 INFO:teuthology.orchestra.run.smithi089.stdout: "tid": 4736 2023-06-03T20:04:20.638 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:04:20.638 INFO:teuthology.orchestra.run.smithi089.stdout: "events": [ 2023-06-03T20:04:20.638 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:04:20.638 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "initiated", 2023-06-03T20:04:20.639 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:42.833775+0000", 2023-06-03T20:04:20.639 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0 2023-06-03T20:04:20.639 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:04:20.639 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:04:20.639 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "throttled", 2023-06-03T20:04:20.639 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:42.833775+0000", 2023-06-03T20:04:20.639 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0 2023-06-03T20:04:20.640 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:04:20.640 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:04:20.640 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "header_read", 2023-06-03T20:04:20.640 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:42.833774+0000", 2023-06-03T20:04:20.640 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 4294967295.9999995 2023-06-03T20:04:20.640 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:04:20.640 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:04:20.641 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "all_read", 2023-06-03T20:04:20.641 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:42.833792+0000", 2023-06-03T20:04:20.641 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 1.7955e-05 2023-06-03T20:04:20.641 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:04:20.641 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:04:20.641 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "dispatched", 2023-06-03T20:04:20.641 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:42.833798+0000", 2023-06-03T20:04:20.641 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 5.4129999999999997e-06 2023-06-03T20:04:20.642 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:04:20.642 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:04:20.642 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "queued_for_pg", 2023-06-03T20:04:20.642 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:42.833802+0000", 2023-06-03T20:04:20.642 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 4.4170000000000002e-06 2023-06-03T20:04:20.642 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:04:20.643 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:04:20.643 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "reached_pg", 2023-06-03T20:04:20.643 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:42.833836+0000", 2023-06-03T20:04:20.643 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 3.3760000000000002e-05 2023-06-03T20:04:20.643 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:04:20.643 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:04:20.643 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "started", 2023-06-03T20:04:20.644 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:42.833981+0000", 2023-06-03T20:04:20.644 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.00014480600000000001 2023-06-03T20:04:20.644 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:04:20.644 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:04:20.644 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "waiting for subops from 1,2", 2023-06-03T20:04:20.644 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:42.834562+0000", 2023-06-03T20:04:20.645 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.00058108200000000002 2023-06-03T20:04:20.645 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:04:20.645 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:04:20.645 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:04:20.645 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:42.836024+0000", 2023-06-03T20:04:20.645 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.001462631 2023-06-03T20:04:20.646 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:04:20.646 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:04:20.646 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "op_commit", 2023-06-03T20:04:20.646 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:42.836036+0000", 2023-06-03T20:04:20.646 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 1.1574e-05 2023-06-03T20:04:20.647 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:04:20.647 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:04:20.647 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:04:20.647 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:43.158679+0000", 2023-06-03T20:04:20.648 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.32264322000000001 2023-06-03T20:04:20.648 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:04:20.648 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:04:20.648 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "commit_sent", 2023-06-03T20:04:20.648 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:43.158727+0000", 2023-06-03T20:04:20.649 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 4.7304e-05 2023-06-03T20:04:20.649 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:04:20.649 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:04:20.649 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "done", 2023-06-03T20:04:20.649 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:43.158764+0000", 2023-06-03T20:04:20.649 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 3.7041e-05 2023-06-03T20:04:20.649 INFO:teuthology.orchestra.run.smithi089.stdout: } 2023-06-03T20:04:20.650 INFO:teuthology.orchestra.run.smithi089.stdout: ] 2023-06-03T20:04:20.650 INFO:teuthology.orchestra.run.smithi089.stdout: } 2023-06-03T20:04:20.650 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:04:20.650 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:04:20.650 INFO:teuthology.orchestra.run.smithi089.stdout: "description": "osd_op(client.45713.0:4737 9.2 9:481aa351:::benchmark_data_smithi089_115580_object4736:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e131)", 2023-06-03T20:04:20.650 INFO:teuthology.orchestra.run.smithi089.stdout: "initiated_at": "2023-06-03T20:00:42.833809+0000", 2023-06-03T20:04:20.650 INFO:teuthology.orchestra.run.smithi089.stdout: "age": 217.77485025999999, 2023-06-03T20:04:20.651 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.32516771100000003, 2023-06-03T20:04:20.651 INFO:teuthology.orchestra.run.smithi089.stdout: "type_data": { 2023-06-03T20:04:20.651 INFO:teuthology.orchestra.run.smithi089.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-03T20:04:20.651 INFO:teuthology.orchestra.run.smithi089.stdout: "client_info": { 2023-06-03T20:04:20.651 INFO:teuthology.orchestra.run.smithi089.stdout: "client": "client.45713", 2023-06-03T20:04:20.651 INFO:teuthology.orchestra.run.smithi089.stdout: "client_addr": "172.21.15.89:0/2400821448", 2023-06-03T20:04:20.651 INFO:teuthology.orchestra.run.smithi089.stdout: "tid": 4737 2023-06-03T20:04:20.652 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:04:20.652 INFO:teuthology.orchestra.run.smithi089.stdout: "events": [ 2023-06-03T20:04:20.652 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:04:20.652 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "initiated", 2023-06-03T20:04:20.652 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:42.833809+0000", 2023-06-03T20:04:20.653 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0 2023-06-03T20:04:20.653 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:04:20.653 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:04:20.653 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "throttled", 2023-06-03T20:04:20.653 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:42.833809+0000", 2023-06-03T20:04:20.653 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0 2023-06-03T20:04:20.653 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:04:20.653 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:04:20.654 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "header_read", 2023-06-03T20:04:20.654 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:42.833808+0000", 2023-06-03T20:04:20.654 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 4294967295.9999995 2023-06-03T20:04:20.654 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:04:20.654 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:04:20.654 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "all_read", 2023-06-03T20:04:20.655 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:42.833831+0000", 2023-06-03T20:04:20.655 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 2.2577000000000001e-05 2023-06-03T20:04:20.655 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:04:20.655 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:04:20.655 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "dispatched", 2023-06-03T20:04:20.655 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:42.833838+0000", 2023-06-03T20:04:20.655 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 7.0010000000000004e-06 2023-06-03T20:04:20.655 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:04:20.656 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:04:20.656 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "queued_for_pg", 2023-06-03T20:04:20.656 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:42.833845+0000", 2023-06-03T20:04:20.656 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 6.7530000000000004e-06 2023-06-03T20:04:20.656 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:04:20.656 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:04:20.656 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "reached_pg", 2023-06-03T20:04:20.657 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:42.834949+0000", 2023-06-03T20:04:20.657 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.0011039769999999999 2023-06-03T20:04:20.657 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:04:20.657 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:04:20.657 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "started", 2023-06-03T20:04:20.657 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:42.835070+0000", 2023-06-03T20:04:20.657 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.00012098499999999999 2023-06-03T20:04:20.658 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:04:20.658 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:04:20.658 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "waiting for subops from 1,2", 2023-06-03T20:04:20.658 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:42.835225+0000", 2023-06-03T20:04:20.658 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.00015558999999999999 2023-06-03T20:04:20.658 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:04:20.658 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:04:20.658 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "op_commit", 2023-06-03T20:04:20.659 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:42.836041+0000", 2023-06-03T20:04:20.659 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.00081591900000000002 2023-06-03T20:04:20.659 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:04:20.659 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:04:20.659 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:04:20.659 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:42.836068+0000", 2023-06-03T20:04:20.660 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 2.6404999999999999e-05 2023-06-03T20:04:20.660 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:04:20.660 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:04:20.660 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:04:20.661 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:43.158900+0000", 2023-06-03T20:04:20.661 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.32283200699999998 2023-06-03T20:04:20.661 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:04:20.661 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:04:20.661 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "commit_sent", 2023-06-03T20:04:20.661 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:43.158941+0000", 2023-06-03T20:04:20.662 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 4.1610999999999999e-05 2023-06-03T20:04:20.662 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:04:20.662 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:04:20.662 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "done", 2023-06-03T20:04:20.662 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:43.158977+0000", 2023-06-03T20:04:20.662 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 3.5339999999999997e-05 2023-06-03T20:04:20.663 INFO:teuthology.orchestra.run.smithi089.stdout: } 2023-06-03T20:04:20.663 INFO:teuthology.orchestra.run.smithi089.stdout: ] 2023-06-03T20:04:20.663 INFO:teuthology.orchestra.run.smithi089.stdout: } 2023-06-03T20:04:20.663 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:04:20.663 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:04:20.663 INFO:teuthology.orchestra.run.smithi089.stdout: "description": "osd_op(client.45713.0:4746 9.2 9:452cbafe:::benchmark_data_smithi089_115580_object4745:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e131)", 2023-06-03T20:04:20.663 INFO:teuthology.orchestra.run.smithi089.stdout: "initiated_at": "2023-06-03T20:00:42.837817+0000", 2023-06-03T20:04:20.664 INFO:teuthology.orchestra.run.smithi089.stdout: "age": 217.77084210500001, 2023-06-03T20:04:20.664 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.32157364399999999, 2023-06-03T20:04:20.664 INFO:teuthology.orchestra.run.smithi089.stdout: "type_data": { 2023-06-03T20:04:20.664 INFO:teuthology.orchestra.run.smithi089.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-03T20:04:20.664 INFO:teuthology.orchestra.run.smithi089.stdout: "client_info": { 2023-06-03T20:04:20.664 INFO:teuthology.orchestra.run.smithi089.stdout: "client": "client.45713", 2023-06-03T20:04:20.665 INFO:teuthology.orchestra.run.smithi089.stdout: "client_addr": "172.21.15.89:0/2400821448", 2023-06-03T20:04:20.665 INFO:teuthology.orchestra.run.smithi089.stdout: "tid": 4746 2023-06-03T20:04:20.665 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:04:20.665 INFO:teuthology.orchestra.run.smithi089.stdout: "events": [ 2023-06-03T20:04:20.665 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:04:20.665 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "initiated", 2023-06-03T20:04:20.665 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:42.837817+0000", 2023-06-03T20:04:20.665 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0 2023-06-03T20:04:20.666 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:04:20.666 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:04:20.666 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "throttled", 2023-06-03T20:04:20.666 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:42.837817+0000", 2023-06-03T20:04:20.666 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0 2023-06-03T20:04:20.666 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:04:20.666 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:04:20.667 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "header_read", 2023-06-03T20:04:20.667 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:42.837816+0000", 2023-06-03T20:04:20.667 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 4294967295.999999 2023-06-03T20:04:20.667 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:04:20.667 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:04:20.667 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "all_read", 2023-06-03T20:04:20.667 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:42.837835+0000", 2023-06-03T20:04:20.668 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 1.9031000000000001e-05 2023-06-03T20:04:20.668 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:04:20.668 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:04:20.668 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "dispatched", 2023-06-03T20:04:20.668 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:42.837845+0000", 2023-06-03T20:04:20.668 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 1.0467e-05 2023-06-03T20:04:20.668 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:04:20.669 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:04:20.669 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "queued_for_pg", 2023-06-03T20:04:20.669 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:42.837851+0000", 2023-06-03T20:04:20.669 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 5.3669999999999996e-06 2023-06-03T20:04:20.669 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:04:20.669 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:04:20.669 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "reached_pg", 2023-06-03T20:04:20.669 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:42.837886+0000", 2023-06-03T20:04:20.670 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 3.5247999999999999e-05 2023-06-03T20:04:20.670 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:04:20.670 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:04:20.670 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "started", 2023-06-03T20:04:20.670 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:42.838014+0000", 2023-06-03T20:04:20.670 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.000127496 2023-06-03T20:04:20.670 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:04:20.670 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:04:20.671 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "waiting for subops from 1,2", 2023-06-03T20:04:20.671 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:42.838169+0000", 2023-06-03T20:04:20.671 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.00015557099999999999 2023-06-03T20:04:20.671 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:04:20.671 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:04:20.671 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "op_commit", 2023-06-03T20:04:20.671 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:42.838735+0000", 2023-06-03T20:04:20.672 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.00056540500000000005 2023-06-03T20:04:20.672 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:04:20.672 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:04:20.672 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:04:20.672 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:42.838991+0000", 2023-06-03T20:04:20.672 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.00025634699999999998 2023-06-03T20:04:20.672 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:04:20.673 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:04:20.673 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:04:20.673 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:43.159186+0000", 2023-06-03T20:04:20.673 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.32019499800000001 2023-06-03T20:04:20.673 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:04:20.673 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:04:20.673 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "commit_sent", 2023-06-03T20:04:20.674 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:43.159348+0000", 2023-06-03T20:04:20.674 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.00016195000000000001 2023-06-03T20:04:20.674 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:04:20.674 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:04:20.674 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "done", 2023-06-03T20:04:20.674 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:43.159391+0000", 2023-06-03T20:04:20.674 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 4.2763999999999999e-05 2023-06-03T20:04:20.674 INFO:teuthology.orchestra.run.smithi089.stdout: } 2023-06-03T20:04:20.675 INFO:teuthology.orchestra.run.smithi089.stdout: ] 2023-06-03T20:04:20.675 INFO:teuthology.orchestra.run.smithi089.stdout: } 2023-06-03T20:04:20.675 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:04:20.675 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:04:20.675 INFO:teuthology.orchestra.run.smithi089.stdout: "description": "osd_op(client.45713.0:8936 9.8 9:15467103:::benchmark_data_smithi089_115580_object8935:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e131)", 2023-06-03T20:04:20.675 INFO:teuthology.orchestra.run.smithi089.stdout: "initiated_at": "2023-06-03T20:00:44.768160+0000", 2023-06-03T20:04:20.676 INFO:teuthology.orchestra.run.smithi089.stdout: "age": 215.840498841, 2023-06-03T20:04:20.676 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.37263233099999998, 2023-06-03T20:04:20.676 INFO:teuthology.orchestra.run.smithi089.stdout: "type_data": { 2023-06-03T20:04:20.676 INFO:teuthology.orchestra.run.smithi089.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-03T20:04:20.676 INFO:teuthology.orchestra.run.smithi089.stdout: "client_info": { 2023-06-03T20:04:20.676 INFO:teuthology.orchestra.run.smithi089.stdout: "client": "client.45713", 2023-06-03T20:04:20.677 INFO:teuthology.orchestra.run.smithi089.stdout: "client_addr": "172.21.15.89:0/2400821448", 2023-06-03T20:04:20.677 INFO:teuthology.orchestra.run.smithi089.stdout: "tid": 8936 2023-06-03T20:04:20.677 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:04:20.677 INFO:teuthology.orchestra.run.smithi089.stdout: "events": [ 2023-06-03T20:04:20.677 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:04:20.677 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "initiated", 2023-06-03T20:04:20.677 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:44.768160+0000", 2023-06-03T20:04:20.677 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0 2023-06-03T20:04:20.678 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:04:20.678 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:04:20.678 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "throttled", 2023-06-03T20:04:20.678 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:44.768160+0000", 2023-06-03T20:04:20.678 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0 2023-06-03T20:04:20.678 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:04:20.678 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:04:20.679 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "header_read", 2023-06-03T20:04:20.679 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:44.768159+0000", 2023-06-03T20:04:20.679 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 4294967295.999999 2023-06-03T20:04:20.679 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:04:20.679 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:04:20.679 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "all_read", 2023-06-03T20:04:20.679 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:44.768180+0000", 2023-06-03T20:04:20.680 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 2.0375000000000001e-05 2023-06-03T20:04:20.680 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:04:20.680 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:04:20.680 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "dispatched", 2023-06-03T20:04:20.680 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:44.768191+0000", 2023-06-03T20:04:20.680 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 1.1868999999999999e-05 2023-06-03T20:04:20.680 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:04:20.681 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:04:20.681 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "queued_for_pg", 2023-06-03T20:04:20.681 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:44.768210+0000", 2023-06-03T20:04:20.681 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 1.8408000000000001e-05 2023-06-03T20:04:20.681 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:04:20.681 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:04:20.681 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "reached_pg", 2023-06-03T20:04:20.682 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:44.768241+0000", 2023-06-03T20:04:20.682 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 3.1378999999999999e-05 2023-06-03T20:04:20.682 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:04:20.682 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:04:20.682 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "started", 2023-06-03T20:04:20.682 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:44.768356+0000", 2023-06-03T20:04:20.683 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.000114801 2023-06-03T20:04:20.683 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:04:20.683 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:04:20.683 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "waiting for subops from 0,5", 2023-06-03T20:04:20.683 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:44.768537+0000", 2023-06-03T20:04:20.683 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.00018138099999999999 2023-06-03T20:04:20.684 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:04:20.684 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:04:20.684 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "op_commit", 2023-06-03T20:04:20.684 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:44.769659+0000", 2023-06-03T20:04:20.684 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.0011211070000000001 2023-06-03T20:04:20.684 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:04:20.685 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:04:20.685 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:04:20.685 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:44.771252+0000", 2023-06-03T20:04:20.685 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.0015936819999999999 2023-06-03T20:04:20.686 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:04:20.686 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:04:20.686 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:04:20.686 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:45.140672+0000", 2023-06-03T20:04:20.686 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.36942024499999998 2023-06-03T20:04:20.686 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:04:20.686 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:04:20.687 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "commit_sent", 2023-06-03T20:04:20.687 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:45.140747+0000", 2023-06-03T20:04:20.687 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 7.4022000000000004e-05 2023-06-03T20:04:20.687 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:04:20.687 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:04:20.687 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "done", 2023-06-03T20:04:20.687 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:45.140793+0000", 2023-06-03T20:04:20.688 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 4.6053000000000001e-05 2023-06-03T20:04:20.688 INFO:teuthology.orchestra.run.smithi089.stdout: } 2023-06-03T20:04:20.688 INFO:teuthology.orchestra.run.smithi089.stdout: ] 2023-06-03T20:04:20.688 INFO:teuthology.orchestra.run.smithi089.stdout: } 2023-06-03T20:04:20.688 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:04:20.688 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:04:20.689 INFO:teuthology.orchestra.run.smithi089.stdout: "description": "osd_op(client.45713.0:8938 9.8 9:1485cac5:::benchmark_data_smithi089_115580_object8937:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e131)", 2023-06-03T20:04:20.689 INFO:teuthology.orchestra.run.smithi089.stdout: "initiated_at": "2023-06-03T20:00:44.769091+0000", 2023-06-03T20:04:20.689 INFO:teuthology.orchestra.run.smithi089.stdout: "age": 215.839567832, 2023-06-03T20:04:20.689 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.37191968199999997, 2023-06-03T20:04:20.689 INFO:teuthology.orchestra.run.smithi089.stdout: "type_data": { 2023-06-03T20:04:20.690 INFO:teuthology.orchestra.run.smithi089.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-03T20:04:20.690 INFO:teuthology.orchestra.run.smithi089.stdout: "client_info": { 2023-06-03T20:04:20.690 INFO:teuthology.orchestra.run.smithi089.stdout: "client": "client.45713", 2023-06-03T20:04:20.690 INFO:teuthology.orchestra.run.smithi089.stdout: "client_addr": "172.21.15.89:0/2400821448", 2023-06-03T20:04:20.690 INFO:teuthology.orchestra.run.smithi089.stdout: "tid": 8938 2023-06-03T20:04:20.690 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:04:20.691 INFO:teuthology.orchestra.run.smithi089.stdout: "events": [ 2023-06-03T20:04:20.691 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:04:20.691 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "initiated", 2023-06-03T20:04:20.691 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:44.769091+0000", 2023-06-03T20:04:20.691 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0 2023-06-03T20:04:20.691 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:04:20.692 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:04:20.692 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "throttled", 2023-06-03T20:04:20.692 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:44.769091+0000", 2023-06-03T20:04:20.692 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0 2023-06-03T20:04:20.692 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:04:20.692 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:04:20.692 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "header_read", 2023-06-03T20:04:20.693 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:44.769090+0000", 2023-06-03T20:04:20.693 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 4294967295.999999 2023-06-03T20:04:20.693 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:04:20.693 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:04:20.693 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "all_read", 2023-06-03T20:04:20.693 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:44.769115+0000", 2023-06-03T20:04:20.693 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 2.4532000000000002e-05 2023-06-03T20:04:20.693 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:04:20.694 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:04:20.694 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "dispatched", 2023-06-03T20:04:20.694 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:44.769123+0000", 2023-06-03T20:04:20.694 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 8.5080000000000005e-06 2023-06-03T20:04:20.694 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:04:20.694 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:04:20.694 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "queued_for_pg", 2023-06-03T20:04:20.695 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:44.769129+0000", 2023-06-03T20:04:20.695 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 5.9950000000000002e-06 2023-06-03T20:04:20.695 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:04:20.695 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:04:20.695 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "reached_pg", 2023-06-03T20:04:20.695 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:44.769175+0000", 2023-06-03T20:04:20.695 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 4.6183999999999997e-05 2023-06-03T20:04:20.696 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:04:20.696 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:04:20.696 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "started", 2023-06-03T20:04:20.696 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:44.769284+0000", 2023-06-03T20:04:20.696 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.00010824599999999999 2023-06-03T20:04:20.696 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:04:20.697 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:04:20.697 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "waiting for subops from 0,5", 2023-06-03T20:04:20.697 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:44.769366+0000", 2023-06-03T20:04:20.697 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 8.2338999999999999e-05 2023-06-03T20:04:20.697 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:04:20.698 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:04:20.698 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "op_commit", 2023-06-03T20:04:20.698 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:44.769962+0000", 2023-06-03T20:04:20.698 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.00059546800000000004 2023-06-03T20:04:20.698 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:04:20.698 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:04:20.698 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:04:20.699 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:44.771566+0000", 2023-06-03T20:04:20.699 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.0016043310000000001 2023-06-03T20:04:20.699 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:04:20.699 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:04:20.699 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:04:20.699 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:45.140935+0000", 2023-06-03T20:04:20.699 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.369369004 2023-06-03T20:04:20.700 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:04:20.700 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:04:20.700 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "commit_sent", 2023-06-03T20:04:20.700 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:45.140985+0000", 2023-06-03T20:04:20.700 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 4.9858999999999998e-05 2023-06-03T20:04:20.700 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:04:20.701 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:04:20.701 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "done", 2023-06-03T20:04:20.701 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:45.141011+0000", 2023-06-03T20:04:20.701 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 2.6214e-05 2023-06-03T20:04:20.702 INFO:teuthology.orchestra.run.smithi089.stdout: } 2023-06-03T20:04:20.703 INFO:teuthology.orchestra.run.smithi089.stdout: ] 2023-06-03T20:04:20.704 INFO:teuthology.orchestra.run.smithi089.stdout: } 2023-06-03T20:04:20.705 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:04:20.705 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:04:20.706 INFO:teuthology.orchestra.run.smithi089.stdout: "description": "osd_op(client.45713.0:8957 9.8 9:1fbe28dc:::benchmark_data_smithi089_115580_object8956:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e131)", 2023-06-03T20:04:20.706 INFO:teuthology.orchestra.run.smithi089.stdout: "initiated_at": "2023-06-03T20:00:44.777664+0000", 2023-06-03T20:04:20.707 INFO:teuthology.orchestra.run.smithi089.stdout: "age": 215.830995229, 2023-06-03T20:04:20.707 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.36382674500000001, 2023-06-03T20:04:20.707 INFO:teuthology.orchestra.run.smithi089.stdout: "type_data": { 2023-06-03T20:04:20.708 INFO:teuthology.orchestra.run.smithi089.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-03T20:04:20.708 INFO:teuthology.orchestra.run.smithi089.stdout: "client_info": { 2023-06-03T20:04:20.708 INFO:teuthology.orchestra.run.smithi089.stdout: "client": "client.45713", 2023-06-03T20:04:20.709 INFO:teuthology.orchestra.run.smithi089.stdout: "client_addr": "172.21.15.89:0/2400821448", 2023-06-03T20:04:20.709 INFO:teuthology.orchestra.run.smithi089.stdout: "tid": 8957 2023-06-03T20:04:20.709 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:04:20.709 INFO:teuthology.orchestra.run.smithi089.stdout: "events": [ 2023-06-03T20:04:20.710 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:04:20.710 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "initiated", 2023-06-03T20:04:20.710 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:44.777664+0000", 2023-06-03T20:04:20.711 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0 2023-06-03T20:04:20.711 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:04:20.711 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:04:20.711 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "throttled", 2023-06-03T20:04:20.711 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:44.777664+0000", 2023-06-03T20:04:20.711 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0 2023-06-03T20:04:20.712 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:04:20.712 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:04:20.712 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "header_read", 2023-06-03T20:04:20.712 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:44.777663+0000", 2023-06-03T20:04:20.712 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 4294967295.999999 2023-06-03T20:04:20.712 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:04:20.713 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:04:20.713 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "all_read", 2023-06-03T20:04:20.713 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:44.777683+0000", 2023-06-03T20:04:20.713 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 1.9607999999999999e-05 2023-06-03T20:04:20.714 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:04:20.714 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:04:20.714 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "dispatched", 2023-06-03T20:04:20.714 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:44.777694+0000", 2023-06-03T20:04:20.715 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 1.1564e-05 2023-06-03T20:04:20.715 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:04:20.715 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:04:20.715 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "queued_for_pg", 2023-06-03T20:04:20.715 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:44.777700+0000", 2023-06-03T20:04:20.715 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 6.0599999999999996e-06 2023-06-03T20:04:20.716 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:04:20.716 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:04:20.716 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "reached_pg", 2023-06-03T20:04:20.716 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:44.777878+0000", 2023-06-03T20:04:20.716 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.00017752600000000001 2023-06-03T20:04:20.717 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:04:20.717 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:04:20.717 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "started", 2023-06-03T20:04:20.717 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:44.777957+0000", 2023-06-03T20:04:20.718 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 7.9240999999999996e-05 2023-06-03T20:04:20.718 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:04:20.718 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:04:20.718 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "waiting for subops from 0,5", 2023-06-03T20:04:20.719 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:44.778044+0000", 2023-06-03T20:04:20.719 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 8.6841000000000004e-05 2023-06-03T20:04:20.719 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:04:20.719 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:04:20.719 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "op_commit", 2023-06-03T20:04:20.719 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:44.779154+0000", 2023-06-03T20:04:20.720 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.001109928 2023-06-03T20:04:20.720 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:04:20.720 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:04:20.721 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:04:20.721 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:44.779524+0000", 2023-06-03T20:04:20.721 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.000370761 2023-06-03T20:04:20.721 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:04:20.721 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:04:20.722 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:04:20.722 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:45.141314+0000", 2023-06-03T20:04:20.722 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.36179000300000003 2023-06-03T20:04:20.722 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:04:20.722 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:04:20.722 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "commit_sent", 2023-06-03T20:04:20.722 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:45.141434+0000", 2023-06-03T20:04:20.723 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.000119932 2023-06-03T20:04:20.723 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:04:20.723 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:04:20.723 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "done", 2023-06-03T20:04:20.723 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:45.141491+0000", 2023-06-03T20:04:20.723 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 5.6214000000000001e-05 2023-06-03T20:04:20.723 INFO:teuthology.orchestra.run.smithi089.stdout: } 2023-06-03T20:04:20.724 INFO:teuthology.orchestra.run.smithi089.stdout: ] 2023-06-03T20:04:20.724 INFO:teuthology.orchestra.run.smithi089.stdout: } 2023-06-03T20:04:20.724 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:04:20.724 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:04:20.724 INFO:teuthology.orchestra.run.smithi089.stdout: "description": "osd_op(client.45713.0:8961 9.8 9:177bfc5e:::benchmark_data_smithi089_115580_object8960:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e131)", 2023-06-03T20:04:20.725 INFO:teuthology.orchestra.run.smithi089.stdout: "initiated_at": "2023-06-03T20:00:44.780291+0000", 2023-06-03T20:04:20.725 INFO:teuthology.orchestra.run.smithi089.stdout: "age": 215.82836854799999, 2023-06-03T20:04:20.725 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.36140980299999997, 2023-06-03T20:04:20.725 INFO:teuthology.orchestra.run.smithi089.stdout: "type_data": { 2023-06-03T20:04:20.726 INFO:teuthology.orchestra.run.smithi089.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-03T20:04:20.726 INFO:teuthology.orchestra.run.smithi089.stdout: "client_info": { 2023-06-03T20:04:20.726 INFO:teuthology.orchestra.run.smithi089.stdout: "client": "client.45713", 2023-06-03T20:04:20.726 INFO:teuthology.orchestra.run.smithi089.stdout: "client_addr": "172.21.15.89:0/2400821448", 2023-06-03T20:04:20.726 INFO:teuthology.orchestra.run.smithi089.stdout: "tid": 8961 2023-06-03T20:04:20.726 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:04:20.727 INFO:teuthology.orchestra.run.smithi089.stdout: "events": [ 2023-06-03T20:04:20.727 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:04:20.727 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "initiated", 2023-06-03T20:04:20.727 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:44.780291+0000", 2023-06-03T20:04:20.727 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0 2023-06-03T20:04:20.727 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:04:20.728 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:04:20.728 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "throttled", 2023-06-03T20:04:20.728 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:44.780291+0000", 2023-06-03T20:04:20.728 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0 2023-06-03T20:04:20.728 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:04:20.728 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:04:20.728 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "header_read", 2023-06-03T20:04:20.729 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:44.780290+0000", 2023-06-03T20:04:20.729 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 4294967295.999999 2023-06-03T20:04:20.729 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:04:20.729 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:04:20.729 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "all_read", 2023-06-03T20:04:20.729 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:44.780309+0000", 2023-06-03T20:04:20.730 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 1.8822999999999999e-05 2023-06-03T20:04:20.730 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:04:20.730 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:04:20.730 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "dispatched", 2023-06-03T20:04:20.730 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:44.780319+0000", 2023-06-03T20:04:20.730 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 1.0681e-05 2023-06-03T20:04:20.730 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:04:20.731 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:04:20.731 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "queued_for_pg", 2023-06-03T20:04:20.731 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:44.780324+0000", 2023-06-03T20:04:20.731 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 5.2599999999999996e-06 2023-06-03T20:04:20.731 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:04:20.731 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:04:20.732 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "reached_pg", 2023-06-03T20:04:20.732 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:44.780353+0000", 2023-06-03T20:04:20.732 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 2.8943999999999999e-05 2023-06-03T20:04:20.732 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:04:20.732 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:04:20.732 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "started", 2023-06-03T20:04:20.732 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:44.780429+0000", 2023-06-03T20:04:20.733 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 7.5527999999999999e-05 2023-06-03T20:04:20.733 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:04:20.733 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:04:20.733 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "waiting for subops from 0,5", 2023-06-03T20:04:20.733 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:44.780517+0000", 2023-06-03T20:04:20.733 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 8.7878000000000001e-05 2023-06-03T20:04:20.734 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:04:20.734 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:04:20.734 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "op_commit", 2023-06-03T20:04:20.734 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:44.781292+0000", 2023-06-03T20:04:20.734 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.00077565299999999995 2023-06-03T20:04:20.735 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:04:20.735 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:04:20.735 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:04:20.735 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:44.781578+0000", 2023-06-03T20:04:20.735 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.00028544200000000001 2023-06-03T20:04:20.736 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:04:20.736 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:04:20.736 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:04:20.736 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:45.141608+0000", 2023-06-03T20:04:20.736 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.36003042400000002 2023-06-03T20:04:20.737 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:04:20.737 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:04:20.737 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "commit_sent", 2023-06-03T20:04:20.737 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:45.141660+0000", 2023-06-03T20:04:20.737 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 5.1765999999999999e-05 2023-06-03T20:04:20.737 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:04:20.737 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:04:20.738 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "done", 2023-06-03T20:04:20.738 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:45.141700+0000", 2023-06-03T20:04:20.738 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 4.0207000000000003e-05 2023-06-03T20:04:20.738 INFO:teuthology.orchestra.run.smithi089.stdout: } 2023-06-03T20:04:20.738 INFO:teuthology.orchestra.run.smithi089.stdout: ] 2023-06-03T20:04:20.738 INFO:teuthology.orchestra.run.smithi089.stdout: } 2023-06-03T20:04:20.739 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:04:20.739 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:04:20.739 INFO:teuthology.orchestra.run.smithi089.stdout: "description": "osd_op(client.45713.0:8962 9.8 9:1310ba96:::benchmark_data_smithi089_115580_object8961:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e131)", 2023-06-03T20:04:20.739 INFO:teuthology.orchestra.run.smithi089.stdout: "initiated_at": "2023-06-03T20:00:44.781724+0000", 2023-06-03T20:04:20.739 INFO:teuthology.orchestra.run.smithi089.stdout: "age": 215.82693533700001, 2023-06-03T20:04:20.740 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.362647473, 2023-06-03T20:04:20.740 INFO:teuthology.orchestra.run.smithi089.stdout: "type_data": { 2023-06-03T20:04:20.740 INFO:teuthology.orchestra.run.smithi089.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-03T20:04:20.740 INFO:teuthology.orchestra.run.smithi089.stdout: "client_info": { 2023-06-03T20:04:20.740 INFO:teuthology.orchestra.run.smithi089.stdout: "client": "client.45713", 2023-06-03T20:04:20.740 INFO:teuthology.orchestra.run.smithi089.stdout: "client_addr": "172.21.15.89:0/2400821448", 2023-06-03T20:04:20.741 INFO:teuthology.orchestra.run.smithi089.stdout: "tid": 8962 2023-06-03T20:04:20.741 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:04:20.741 INFO:teuthology.orchestra.run.smithi089.stdout: "events": [ 2023-06-03T20:04:20.741 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:04:20.741 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "initiated", 2023-06-03T20:04:20.741 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:44.781724+0000", 2023-06-03T20:04:20.741 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0 2023-06-03T20:04:20.742 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:04:20.742 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:04:20.742 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "throttled", 2023-06-03T20:04:20.742 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:44.781724+0000", 2023-06-03T20:04:20.742 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0 2023-06-03T20:04:20.742 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:04:20.742 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:04:20.742 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "header_read", 2023-06-03T20:04:20.743 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:44.781723+0000", 2023-06-03T20:04:20.743 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 4294967295.999999 2023-06-03T20:04:20.743 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:04:20.743 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:04:20.743 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "all_read", 2023-06-03T20:04:20.744 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:44.781744+0000", 2023-06-03T20:04:20.744 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 2.1271e-05 2023-06-03T20:04:20.744 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:04:20.745 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:04:20.745 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "dispatched", 2023-06-03T20:04:20.745 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:44.781753+0000", 2023-06-03T20:04:20.746 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 9.2159999999999995e-06 2023-06-03T20:04:20.746 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:04:20.746 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:04:20.746 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "queued_for_pg", 2023-06-03T20:04:20.747 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:44.781758+0000", 2023-06-03T20:04:20.747 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 4.9239999999999998e-06 2023-06-03T20:04:20.747 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:04:20.747 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:04:20.748 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "reached_pg", 2023-06-03T20:04:20.748 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:44.781807+0000", 2023-06-03T20:04:20.748 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 4.9252999999999998e-05 2023-06-03T20:04:20.748 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:04:20.748 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:04:20.749 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "started", 2023-06-03T20:04:20.749 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:44.781957+0000", 2023-06-03T20:04:20.749 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.000149591 2023-06-03T20:04:20.749 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:04:20.749 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:04:20.749 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "waiting for subops from 0,5", 2023-06-03T20:04:20.749 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:44.782151+0000", 2023-06-03T20:04:20.750 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.00019411499999999999 2023-06-03T20:04:20.750 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:04:20.750 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:04:20.750 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "op_commit", 2023-06-03T20:04:20.750 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:44.782731+0000", 2023-06-03T20:04:20.750 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.00058003199999999997 2023-06-03T20:04:20.751 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:04:20.751 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:04:20.751 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:04:20.751 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:44.783096+0000", 2023-06-03T20:04:20.751 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.00036488999999999998 2023-06-03T20:04:20.751 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:04:20.752 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:04:20.752 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:04:20.752 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:45.144219+0000", 2023-06-03T20:04:20.752 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.36112258899999999 2023-06-03T20:04:20.753 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:04:20.753 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:04:20.753 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "commit_sent", 2023-06-03T20:04:20.753 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:45.144292+0000", 2023-06-03T20:04:20.753 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 7.2867000000000006e-05 2023-06-03T20:04:20.754 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:04:20.754 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:04:20.754 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "done", 2023-06-03T20:04:20.754 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:45.144371+0000", 2023-06-03T20:04:20.754 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 7.9641999999999994e-05 2023-06-03T20:04:20.754 INFO:teuthology.orchestra.run.smithi089.stdout: } 2023-06-03T20:04:20.755 INFO:teuthology.orchestra.run.smithi089.stdout: ] 2023-06-03T20:04:20.755 INFO:teuthology.orchestra.run.smithi089.stdout: } 2023-06-03T20:04:20.755 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:04:20.755 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:04:20.756 INFO:teuthology.orchestra.run.smithi089.stdout: "description": "osd_op(client.45713.0:20537 9.2 9:499ab7e3:::benchmark_data_smithi089_115580_object20536:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e131)", 2023-06-03T20:04:20.756 INFO:teuthology.orchestra.run.smithi089.stdout: "initiated_at": "2023-06-03T20:00:50.377357+0000", 2023-06-03T20:04:20.756 INFO:teuthology.orchestra.run.smithi089.stdout: "age": 210.231302223, 2023-06-03T20:04:20.756 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.26465608400000001, 2023-06-03T20:04:20.757 INFO:teuthology.orchestra.run.smithi089.stdout: "type_data": { 2023-06-03T20:04:20.757 INFO:teuthology.orchestra.run.smithi089.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-03T20:04:20.757 INFO:teuthology.orchestra.run.smithi089.stdout: "client_info": { 2023-06-03T20:04:20.757 INFO:teuthology.orchestra.run.smithi089.stdout: "client": "client.45713", 2023-06-03T20:04:20.758 INFO:teuthology.orchestra.run.smithi089.stdout: "client_addr": "172.21.15.89:0/2400821448", 2023-06-03T20:04:20.758 INFO:teuthology.orchestra.run.smithi089.stdout: "tid": 20537 2023-06-03T20:04:20.758 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:04:20.758 INFO:teuthology.orchestra.run.smithi089.stdout: "events": [ 2023-06-03T20:04:20.759 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:04:20.759 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "initiated", 2023-06-03T20:04:20.760 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:50.377357+0000", 2023-06-03T20:04:20.760 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0 2023-06-03T20:04:20.760 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:04:20.760 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:04:20.761 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "throttled", 2023-06-03T20:04:20.761 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:50.377357+0000", 2023-06-03T20:04:20.761 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0 2023-06-03T20:04:20.762 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:04:20.762 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:04:20.762 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "header_read", 2023-06-03T20:04:20.762 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:50.377355+0000", 2023-06-03T20:04:20.762 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 4294967295.9999981 2023-06-03T20:04:20.763 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:04:20.763 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:04:20.763 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "all_read", 2023-06-03T20:04:20.763 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:50.377385+0000", 2023-06-03T20:04:20.763 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 2.9918000000000001e-05 2023-06-03T20:04:20.763 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:04:20.763 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:04:20.764 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "dispatched", 2023-06-03T20:04:20.764 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:50.377396+0000", 2023-06-03T20:04:20.764 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 1.152e-05 2023-06-03T20:04:20.764 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:04:20.764 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:04:20.764 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "queued_for_pg", 2023-06-03T20:04:20.765 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:50.377404+0000", 2023-06-03T20:04:20.765 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 8.1480000000000006e-06 2023-06-03T20:04:20.765 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:04:20.765 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:04:20.765 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "reached_pg", 2023-06-03T20:04:20.765 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:50.377577+0000", 2023-06-03T20:04:20.766 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.000172456 2023-06-03T20:04:20.766 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:04:20.766 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:04:20.766 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "started", 2023-06-03T20:04:20.766 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:50.377714+0000", 2023-06-03T20:04:20.767 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.00013675199999999999 2023-06-03T20:04:20.767 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:04:20.767 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:04:20.767 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "waiting for subops from 1,2", 2023-06-03T20:04:20.768 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:50.377863+0000", 2023-06-03T20:04:20.768 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.00014962600000000001 2023-06-03T20:04:20.768 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:04:20.768 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:04:20.768 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:04:20.768 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:50.640451+0000", 2023-06-03T20:04:20.768 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.26258738300000001 2023-06-03T20:04:20.769 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:04:20.769 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:04:20.769 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "op_commit", 2023-06-03T20:04:20.769 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:50.641000+0000", 2023-06-03T20:04:20.769 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.000549674 2023-06-03T20:04:20.770 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:04:20.770 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:04:20.770 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:04:20.770 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:50.641934+0000", 2023-06-03T20:04:20.770 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.00093376899999999998 2023-06-03T20:04:20.770 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:04:20.770 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:04:20.771 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "commit_sent", 2023-06-03T20:04:20.771 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:50.641979+0000", 2023-06-03T20:04:20.771 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 4.5237000000000001e-05 2023-06-03T20:04:20.771 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:04:20.771 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:04:20.771 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "done", 2023-06-03T20:04:20.771 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:50.642013+0000", 2023-06-03T20:04:20.772 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 3.3621000000000002e-05 2023-06-03T20:04:20.772 INFO:teuthology.orchestra.run.smithi089.stdout: } 2023-06-03T20:04:20.772 INFO:teuthology.orchestra.run.smithi089.stdout: ] 2023-06-03T20:04:20.772 INFO:teuthology.orchestra.run.smithi089.stdout: } 2023-06-03T20:04:20.772 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:04:20.772 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:04:20.772 INFO:teuthology.orchestra.run.smithi089.stdout: "description": "osd_op(client.45713.0:20539 9.8 9:1b026201:::benchmark_data_smithi089_115580_object20538:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e131)", 2023-06-03T20:04:20.773 INFO:teuthology.orchestra.run.smithi089.stdout: "initiated_at": "2023-06-03T20:00:50.377424+0000", 2023-06-03T20:04:20.773 INFO:teuthology.orchestra.run.smithi089.stdout: "age": 210.23123534300001, 2023-06-03T20:04:20.773 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.264260104, 2023-06-03T20:04:20.773 INFO:teuthology.orchestra.run.smithi089.stdout: "type_data": { 2023-06-03T20:04:20.773 INFO:teuthology.orchestra.run.smithi089.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-03T20:04:20.773 INFO:teuthology.orchestra.run.smithi089.stdout: "client_info": { 2023-06-03T20:04:20.773 INFO:teuthology.orchestra.run.smithi089.stdout: "client": "client.45713", 2023-06-03T20:04:20.774 INFO:teuthology.orchestra.run.smithi089.stdout: "client_addr": "172.21.15.89:0/2400821448", 2023-06-03T20:04:20.774 INFO:teuthology.orchestra.run.smithi089.stdout: "tid": 20539 2023-06-03T20:04:20.774 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:04:20.774 INFO:teuthology.orchestra.run.smithi089.stdout: "events": [ 2023-06-03T20:04:20.774 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:04:20.774 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "initiated", 2023-06-03T20:04:20.774 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:50.377424+0000", 2023-06-03T20:04:20.775 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0 2023-06-03T20:04:20.775 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:04:20.775 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:04:20.775 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "throttled", 2023-06-03T20:04:20.775 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:50.377424+0000", 2023-06-03T20:04:20.775 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0 2023-06-03T20:04:20.776 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:04:20.776 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:04:20.776 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "header_read", 2023-06-03T20:04:20.776 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:50.377423+0000", 2023-06-03T20:04:20.777 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 4294967295.999999 2023-06-03T20:04:20.777 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:04:20.777 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:04:20.777 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "all_read", 2023-06-03T20:04:20.777 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:50.377447+0000", 2023-06-03T20:04:20.778 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 2.3668000000000001e-05 2023-06-03T20:04:20.778 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:04:20.778 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:04:20.779 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "dispatched", 2023-06-03T20:04:20.779 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:50.377455+0000", 2023-06-03T20:04:20.779 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 8.8149999999999995e-06 2023-06-03T20:04:20.779 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:04:20.780 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:04:20.780 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "queued_for_pg", 2023-06-03T20:04:20.780 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:50.377462+0000", 2023-06-03T20:04:20.780 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 6.6590000000000001e-06 2023-06-03T20:04:20.780 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:04:20.781 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:04:20.781 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "reached_pg", 2023-06-03T20:04:20.781 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:50.377609+0000", 2023-06-03T20:04:20.781 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.00014704299999999999 2023-06-03T20:04:20.781 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:04:20.781 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:04:20.781 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "started", 2023-06-03T20:04:20.782 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:50.377714+0000", 2023-06-03T20:04:20.782 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.00010467399999999999 2023-06-03T20:04:20.782 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:04:20.782 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:04:20.782 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "waiting for subops from 0,5", 2023-06-03T20:04:20.782 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:50.377886+0000", 2023-06-03T20:04:20.782 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.00017243599999999999 2023-06-03T20:04:20.783 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:04:20.783 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:04:20.783 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "op_commit", 2023-06-03T20:04:20.783 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:50.636597+0000", 2023-06-03T20:04:20.783 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.258710726 2023-06-03T20:04:20.783 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:04:20.783 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:04:20.784 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:04:20.784 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:50.636634+0000", 2023-06-03T20:04:20.784 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 3.6999000000000003e-05 2023-06-03T20:04:20.784 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:04:20.784 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:04:20.784 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:04:20.784 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:50.641590+0000", 2023-06-03T20:04:20.785 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.0049562180000000001 2023-06-03T20:04:20.785 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:04:20.785 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:04:20.785 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "commit_sent", 2023-06-03T20:04:20.785 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:50.641646+0000", 2023-06-03T20:04:20.785 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 5.5708999999999998e-05 2023-06-03T20:04:20.786 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:04:20.786 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:04:20.786 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "done", 2023-06-03T20:04:20.786 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:50.641684+0000", 2023-06-03T20:04:20.786 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 3.7978999999999997e-05 2023-06-03T20:04:20.786 INFO:teuthology.orchestra.run.smithi089.stdout: } 2023-06-03T20:04:20.787 INFO:teuthology.orchestra.run.smithi089.stdout: ] 2023-06-03T20:04:20.787 INFO:teuthology.orchestra.run.smithi089.stdout: } 2023-06-03T20:04:20.787 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:04:20.787 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:04:20.787 INFO:teuthology.orchestra.run.smithi089.stdout: "description": "osd_op(client.45713.0:20542 9.c 9:3f98f011:::benchmark_data_smithi089_115580_object20541:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e131)", 2023-06-03T20:04:20.788 INFO:teuthology.orchestra.run.smithi089.stdout: "initiated_at": "2023-06-03T20:00:50.377477+0000", 2023-06-03T20:04:20.788 INFO:teuthology.orchestra.run.smithi089.stdout: "age": 210.231182351, 2023-06-03T20:04:20.788 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.267194496, 2023-06-03T20:04:20.788 INFO:teuthology.orchestra.run.smithi089.stdout: "type_data": { 2023-06-03T20:04:20.789 INFO:teuthology.orchestra.run.smithi089.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-03T20:04:20.789 INFO:teuthology.orchestra.run.smithi089.stdout: "client_info": { 2023-06-03T20:04:20.789 INFO:teuthology.orchestra.run.smithi089.stdout: "client": "client.45713", 2023-06-03T20:04:20.789 INFO:teuthology.orchestra.run.smithi089.stdout: "client_addr": "172.21.15.89:0/2400821448", 2023-06-03T20:04:20.789 INFO:teuthology.orchestra.run.smithi089.stdout: "tid": 20542 2023-06-03T20:04:20.789 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:04:20.790 INFO:teuthology.orchestra.run.smithi089.stdout: "events": [ 2023-06-03T20:04:20.790 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:04:20.790 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "initiated", 2023-06-03T20:04:20.790 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:50.377477+0000", 2023-06-03T20:04:20.790 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0 2023-06-03T20:04:20.791 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:04:20.791 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:04:20.791 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "throttled", 2023-06-03T20:04:20.791 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:50.377477+0000", 2023-06-03T20:04:20.791 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0 2023-06-03T20:04:20.791 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:04:20.791 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:04:20.792 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "header_read", 2023-06-03T20:04:20.792 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:50.377476+0000", 2023-06-03T20:04:20.792 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 4294967295.9999995 2023-06-03T20:04:20.792 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:04:20.792 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:04:20.792 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "all_read", 2023-06-03T20:04:20.792 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:50.377510+0000", 2023-06-03T20:04:20.793 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 3.4143000000000002e-05 2023-06-03T20:04:20.793 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:04:20.793 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:04:20.793 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "dispatched", 2023-06-03T20:04:20.794 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:50.377519+0000", 2023-06-03T20:04:20.794 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 8.4919999999999993e-06 2023-06-03T20:04:20.794 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:04:20.794 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:04:20.794 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "queued_for_pg", 2023-06-03T20:04:20.794 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:50.377525+0000", 2023-06-03T20:04:20.794 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 6.1029999999999998e-06 2023-06-03T20:04:20.795 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:04:20.795 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:04:20.795 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "reached_pg", 2023-06-03T20:04:20.795 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:50.377933+0000", 2023-06-03T20:04:20.795 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.00040767000000000001 2023-06-03T20:04:20.795 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:04:20.795 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:04:20.796 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "started", 2023-06-03T20:04:20.796 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:50.378037+0000", 2023-06-03T20:04:20.796 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.00010406100000000001 2023-06-03T20:04:20.796 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:04:20.796 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:04:20.796 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "waiting for subops from 4,7", 2023-06-03T20:04:20.797 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:50.636555+0000", 2023-06-03T20:04:20.797 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.25851858900000002 2023-06-03T20:04:20.797 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:04:20.797 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:04:20.797 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "op_commit", 2023-06-03T20:04:20.797 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:50.642399+0000", 2023-06-03T20:04:20.798 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.0058435939999999997 2023-06-03T20:04:20.798 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:04:20.798 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:04:20.798 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:04:20.798 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:50.642443+0000", 2023-06-03T20:04:20.798 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 4.4122000000000002e-05 2023-06-03T20:04:20.799 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:04:20.799 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:04:20.799 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:04:20.799 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:50.644589+0000", 2023-06-03T20:04:20.799 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.0021459510000000001 2023-06-03T20:04:20.800 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:04:20.800 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:04:20.800 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "commit_sent", 2023-06-03T20:04:20.800 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:50.644634+0000", 2023-06-03T20:04:20.800 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 4.4569e-05 2023-06-03T20:04:20.801 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:04:20.801 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:04:20.801 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "done", 2023-06-03T20:04:20.801 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:50.644671+0000", 2023-06-03T20:04:20.801 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 3.7659e-05 2023-06-03T20:04:20.801 INFO:teuthology.orchestra.run.smithi089.stdout: } 2023-06-03T20:04:20.802 INFO:teuthology.orchestra.run.smithi089.stdout: ] 2023-06-03T20:04:20.802 INFO:teuthology.orchestra.run.smithi089.stdout: } 2023-06-03T20:04:20.802 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:04:20.802 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:04:20.802 INFO:teuthology.orchestra.run.smithi089.stdout: "description": "osd_op(client.45713.0:20543 9.2 9:4e00e85f:::benchmark_data_smithi089_115580_object20542:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e131)", 2023-06-03T20:04:20.802 INFO:teuthology.orchestra.run.smithi089.stdout: "initiated_at": "2023-06-03T20:00:50.377537+0000", 2023-06-03T20:04:20.802 INFO:teuthology.orchestra.run.smithi089.stdout: "age": 210.23112235799999, 2023-06-03T20:04:20.802 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.26690673199999998, 2023-06-03T20:04:20.803 INFO:teuthology.orchestra.run.smithi089.stdout: "type_data": { 2023-06-03T20:04:20.803 INFO:teuthology.orchestra.run.smithi089.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-03T20:04:20.803 INFO:teuthology.orchestra.run.smithi089.stdout: "client_info": { 2023-06-03T20:04:20.803 INFO:teuthology.orchestra.run.smithi089.stdout: "client": "client.45713", 2023-06-03T20:04:20.803 INFO:teuthology.orchestra.run.smithi089.stdout: "client_addr": "172.21.15.89:0/2400821448", 2023-06-03T20:04:20.804 INFO:teuthology.orchestra.run.smithi089.stdout: "tid": 20543 2023-06-03T20:04:20.804 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:04:20.804 INFO:teuthology.orchestra.run.smithi089.stdout: "events": [ 2023-06-03T20:04:20.804 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:04:20.804 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "initiated", 2023-06-03T20:04:20.805 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:50.377537+0000", 2023-06-03T20:04:20.805 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0 2023-06-03T20:04:20.805 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:04:20.805 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:04:20.805 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "throttled", 2023-06-03T20:04:20.805 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:50.377537+0000", 2023-06-03T20:04:20.806 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0 2023-06-03T20:04:20.806 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:04:20.806 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:04:20.806 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "header_read", 2023-06-03T20:04:20.806 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:50.377536+0000", 2023-06-03T20:04:20.806 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 4294967295.9999995 2023-06-03T20:04:20.806 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:04:20.806 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:04:20.807 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "all_read", 2023-06-03T20:04:20.807 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:50.377558+0000", 2023-06-03T20:04:20.807 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 2.2107e-05 2023-06-03T20:04:20.807 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:04:20.807 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:04:20.807 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "dispatched", 2023-06-03T20:04:20.808 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:50.377565+0000", 2023-06-03T20:04:20.808 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 6.8820000000000003e-06 2023-06-03T20:04:20.808 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:04:20.808 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:04:20.808 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "queued_for_pg", 2023-06-03T20:04:20.809 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:50.377571+0000", 2023-06-03T20:04:20.809 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 5.8409999999999996e-06 2023-06-03T20:04:20.809 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:04:20.809 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:04:20.809 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "reached_pg", 2023-06-03T20:04:20.810 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:50.638223+0000", 2023-06-03T20:04:20.810 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.26065168900000002 2023-06-03T20:04:20.810 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:04:20.810 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:04:20.810 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "started", 2023-06-03T20:04:20.810 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:50.638629+0000", 2023-06-03T20:04:20.811 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.000406609 2023-06-03T20:04:20.811 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:04:20.811 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:04:20.811 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "waiting for subops from 1,2", 2023-06-03T20:04:20.811 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:50.639041+0000", 2023-06-03T20:04:20.812 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.00041143899999999999 2023-06-03T20:04:20.812 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:04:20.812 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:04:20.812 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:04:20.812 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:50.642052+0000", 2023-06-03T20:04:20.812 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.0030110419999999998 2023-06-03T20:04:20.813 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:04:20.813 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:04:20.813 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "op_commit", 2023-06-03T20:04:20.813 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:50.642635+0000", 2023-06-03T20:04:20.813 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.00058312100000000003 2023-06-03T20:04:20.813 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:04:20.814 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:04:20.814 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:04:20.814 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:50.644351+0000", 2023-06-03T20:04:20.814 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.0017159779999999999 2023-06-03T20:04:20.814 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:04:20.814 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:04:20.814 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "commit_sent", 2023-06-03T20:04:20.815 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:50.644404+0000", 2023-06-03T20:04:20.815 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 5.257e-05 2023-06-03T20:04:20.815 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:04:20.815 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:04:20.815 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "done", 2023-06-03T20:04:20.815 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:50.644443+0000", 2023-06-03T20:04:20.815 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 3.9872e-05 2023-06-03T20:04:20.816 INFO:teuthology.orchestra.run.smithi089.stdout: } 2023-06-03T20:04:20.816 INFO:teuthology.orchestra.run.smithi089.stdout: ] 2023-06-03T20:04:20.816 INFO:teuthology.orchestra.run.smithi089.stdout: } 2023-06-03T20:04:20.816 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:04:20.816 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:04:20.816 INFO:teuthology.orchestra.run.smithi089.stdout: "description": "osd_repop(client.45713.0:20535 9.3 e131/127 9:c75f0c6c:::benchmark_data_smithi089_115580_object20534:head v 131'1286, mlcod=131'1284)", 2023-06-03T20:04:20.816 INFO:teuthology.orchestra.run.smithi089.stdout: "initiated_at": "2023-06-03T20:00:50.379414+0000", 2023-06-03T20:04:20.817 INFO:teuthology.orchestra.run.smithi089.stdout: "age": 210.22924486100001, 2023-06-03T20:04:20.817 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.25809855300000001, 2023-06-03T20:04:20.817 INFO:teuthology.orchestra.run.smithi089.stdout: "type_data": { 2023-06-03T20:04:20.817 INFO:teuthology.orchestra.run.smithi089.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-03T20:04:20.817 INFO:teuthology.orchestra.run.smithi089.stdout: "events": [ 2023-06-03T20:04:20.817 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:04:20.817 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "initiated", 2023-06-03T20:04:20.818 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:50.379414+0000", 2023-06-03T20:04:20.818 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0 2023-06-03T20:04:20.818 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:04:20.818 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:04:20.818 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "throttled", 2023-06-03T20:04:20.818 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:50.379414+0000", 2023-06-03T20:04:20.819 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0 2023-06-03T20:04:20.819 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:04:20.819 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:04:20.819 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "header_read", 2023-06-03T20:04:20.819 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:50.379414+0000", 2023-06-03T20:04:20.820 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 4294967295.9999995 2023-06-03T20:04:20.820 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:04:20.820 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:04:20.820 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "all_read", 2023-06-03T20:04:20.820 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:50.379443+0000", 2023-06-03T20:04:20.821 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 2.9734e-05 2023-06-03T20:04:20.821 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:04:20.821 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:04:20.821 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "dispatched", 2023-06-03T20:04:20.821 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:50.636239+0000", 2023-06-03T20:04:20.822 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.25679589400000002 2023-06-03T20:04:20.822 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:04:20.822 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:04:20.822 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "queued_for_pg", 2023-06-03T20:04:20.822 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:50.636248+0000", 2023-06-03T20:04:20.823 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 8.9500000000000007e-06 2023-06-03T20:04:20.823 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:04:20.823 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:04:20.823 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "reached_pg", 2023-06-03T20:04:20.823 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:50.636287+0000", 2023-06-03T20:04:20.823 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 3.8809000000000001e-05 2023-06-03T20:04:20.824 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:04:20.824 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:04:20.824 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "started", 2023-06-03T20:04:20.824 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:50.636313+0000", 2023-06-03T20:04:20.824 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 2.6296999999999999e-05 2023-06-03T20:04:20.824 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:04:20.824 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:04:20.825 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "commit_sent", 2023-06-03T20:04:20.825 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:50.637486+0000", 2023-06-03T20:04:20.825 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.0011725889999999999 2023-06-03T20:04:20.825 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:04:20.825 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:04:20.825 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "done", 2023-06-03T20:04:20.825 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:50.637513+0000", 2023-06-03T20:04:20.826 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 2.6976999999999999e-05 2023-06-03T20:04:20.826 INFO:teuthology.orchestra.run.smithi089.stdout: } 2023-06-03T20:04:20.826 INFO:teuthology.orchestra.run.smithi089.stdout: ] 2023-06-03T20:04:20.826 INFO:teuthology.orchestra.run.smithi089.stdout: } 2023-06-03T20:04:20.826 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:04:20.826 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:04:20.827 INFO:teuthology.orchestra.run.smithi089.stdout: "description": "osd_repop_reply(client.45713.0:20539 9.8 e131/127 ondisk, result = 0)", 2023-06-03T20:04:20.827 INFO:teuthology.orchestra.run.smithi089.stdout: "initiated_at": "2023-06-03T20:00:50.380738+0000", 2023-06-03T20:04:20.827 INFO:teuthology.orchestra.run.smithi089.stdout: "age": 210.22792085200001, 2023-06-03T20:04:20.827 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.25589922100000001, 2023-06-03T20:04:20.827 INFO:teuthology.orchestra.run.smithi089.stdout: "type_data": { 2023-06-03T20:04:20.827 INFO:teuthology.orchestra.run.smithi089.stdout: "flag_point": "started", 2023-06-03T20:04:20.828 INFO:teuthology.orchestra.run.smithi089.stdout: "events": [ 2023-06-03T20:04:20.828 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:04:20.828 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "initiated", 2023-06-03T20:04:20.828 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:50.380738+0000", 2023-06-03T20:04:20.828 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0 2023-06-03T20:04:20.828 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:04:20.828 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:04:20.829 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "throttled", 2023-06-03T20:04:20.829 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:50.380738+0000", 2023-06-03T20:04:20.829 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0 2023-06-03T20:04:20.829 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:04:20.829 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:04:20.829 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "header_read", 2023-06-03T20:04:20.829 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:50.380697+0000", 2023-06-03T20:04:20.830 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 4294967295.999959 2023-06-03T20:04:20.830 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:04:20.830 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:04:20.830 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "all_read", 2023-06-03T20:04:20.830 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:50.380745+0000", 2023-06-03T20:04:20.830 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 4.7159e-05 2023-06-03T20:04:20.830 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:04:20.831 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:04:20.831 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "dispatched", 2023-06-03T20:04:20.831 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:50.636236+0000", 2023-06-03T20:04:20.831 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.25549137500000002 2023-06-03T20:04:20.831 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:04:20.831 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:04:20.831 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "queued_for_pg", 2023-06-03T20:04:20.832 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:50.636249+0000", 2023-06-03T20:04:20.832 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 1.3380000000000001e-05 2023-06-03T20:04:20.832 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:04:20.832 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:04:20.832 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "reached_pg", 2023-06-03T20:04:20.832 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:50.636619+0000", 2023-06-03T20:04:20.832 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.00036976199999999999 2023-06-03T20:04:20.833 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:04:20.833 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:04:20.833 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "started", 2023-06-03T20:04:20.833 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:50.636628+0000", 2023-06-03T20:04:20.833 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 8.9539999999999993e-06 2023-06-03T20:04:20.833 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:04:20.833 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:04:20.834 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "done", 2023-06-03T20:04:20.834 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:50.636637+0000", 2023-06-03T20:04:20.834 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 9.4669999999999993e-06 2023-06-03T20:04:20.834 INFO:teuthology.orchestra.run.smithi089.stdout: } 2023-06-03T20:04:20.834 INFO:teuthology.orchestra.run.smithi089.stdout: ] 2023-06-03T20:04:20.834 INFO:teuthology.orchestra.run.smithi089.stdout: } 2023-06-03T20:04:20.834 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:04:20.835 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:04:20.835 INFO:teuthology.orchestra.run.smithi089.stdout: "description": "osd_op(client.45713.0:44601 9.c 9:305a32fa:::benchmark_data_smithi089_115580_object44600:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e132)", 2023-06-03T20:04:20.835 INFO:teuthology.orchestra.run.smithi089.stdout: "initiated_at": "2023-06-03T20:01:06.112554+0000", 2023-06-03T20:04:20.835 INFO:teuthology.orchestra.run.smithi089.stdout: "age": 194.49610498600001, 2023-06-03T20:04:20.835 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.17998325100000001, 2023-06-03T20:04:20.835 INFO:teuthology.orchestra.run.smithi089.stdout: "type_data": { 2023-06-03T20:04:20.836 INFO:teuthology.orchestra.run.smithi089.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-03T20:04:20.836 INFO:teuthology.orchestra.run.smithi089.stdout: "client_info": { 2023-06-03T20:04:20.836 INFO:teuthology.orchestra.run.smithi089.stdout: "client": "client.45713", 2023-06-03T20:04:20.836 INFO:teuthology.orchestra.run.smithi089.stdout: "client_addr": "172.21.15.89:0/2400821448", 2023-06-03T20:04:20.836 INFO:teuthology.orchestra.run.smithi089.stdout: "tid": 44601 2023-06-03T20:04:20.836 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:04:20.836 INFO:teuthology.orchestra.run.smithi089.stdout: "events": [ 2023-06-03T20:04:20.836 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:04:20.837 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "initiated", 2023-06-03T20:04:20.837 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:01:06.112554+0000", 2023-06-03T20:04:20.837 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0 2023-06-03T20:04:20.837 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:04:20.837 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:04:20.837 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "throttled", 2023-06-03T20:04:20.837 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:01:06.112554+0000", 2023-06-03T20:04:20.838 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0 2023-06-03T20:04:20.838 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:04:20.838 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:04:20.838 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "header_read", 2023-06-03T20:04:20.838 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:01:06.112553+0000", 2023-06-03T20:04:20.838 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 4294967295.9999986 2023-06-03T20:04:20.838 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:04:20.839 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:04:20.839 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "all_read", 2023-06-03T20:04:20.839 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:01:06.112600+0000", 2023-06-03T20:04:20.839 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 4.7367000000000002e-05 2023-06-03T20:04:20.839 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:04:20.839 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:04:20.839 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "dispatched", 2023-06-03T20:04:20.840 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:01:06.112613+0000", 2023-06-03T20:04:20.840 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 1.2595000000000001e-05 2023-06-03T20:04:20.840 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:04:20.840 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:04:20.840 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "queued_for_pg", 2023-06-03T20:04:20.840 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:01:06.112623+0000", 2023-06-03T20:04:20.840 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 1.0271e-05 2023-06-03T20:04:20.841 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:04:20.841 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:04:20.841 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "reached_pg", 2023-06-03T20:04:20.841 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:01:06.112755+0000", 2023-06-03T20:04:20.841 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.00013233799999999999 2023-06-03T20:04:20.841 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:04:20.841 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:04:20.842 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "started", 2023-06-03T20:04:20.842 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:01:06.115014+0000", 2023-06-03T20:04:20.842 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.0022589749999999999 2023-06-03T20:04:20.842 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:04:20.842 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:04:20.842 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "waiting for subops from 4,7", 2023-06-03T20:04:20.842 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:01:06.115167+0000", 2023-06-03T20:04:20.843 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.00015251300000000001 2023-06-03T20:04:20.843 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:04:20.843 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:04:20.843 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:04:20.843 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:01:06.291925+0000", 2023-06-03T20:04:20.843 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.176758624 2023-06-03T20:04:20.843 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:04:20.844 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:04:20.844 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:04:20.844 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:01:06.291955+0000", 2023-06-03T20:04:20.844 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 2.9396000000000001e-05 2023-06-03T20:04:20.844 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:04:20.844 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:04:20.844 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "op_commit", 2023-06-03T20:04:20.845 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:01:06.292448+0000", 2023-06-03T20:04:20.845 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.000493678 2023-06-03T20:04:20.845 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:04:20.845 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:04:20.845 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "commit_sent", 2023-06-03T20:04:20.845 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:01:06.292499+0000", 2023-06-03T20:04:20.845 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 5.0377000000000002e-05 2023-06-03T20:04:20.846 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:04:20.846 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:04:20.846 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "done", 2023-06-03T20:04:20.846 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:01:06.292537+0000", 2023-06-03T20:04:20.846 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 3.8571000000000001e-05 2023-06-03T20:04:20.846 INFO:teuthology.orchestra.run.smithi089.stdout: } 2023-06-03T20:04:20.846 INFO:teuthology.orchestra.run.smithi089.stdout: ] 2023-06-03T20:04:20.846 INFO:teuthology.orchestra.run.smithi089.stdout: } 2023-06-03T20:04:20.847 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:04:20.847 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:04:20.847 INFO:teuthology.orchestra.run.smithi089.stdout: "description": "osd_op(client.45713.0:44604 9.c 9:3e7fc66f:::benchmark_data_smithi089_115580_object44603:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e132)", 2023-06-03T20:04:20.847 INFO:teuthology.orchestra.run.smithi089.stdout: "initiated_at": "2023-06-03T20:01:06.112902+0000", 2023-06-03T20:04:20.847 INFO:teuthology.orchestra.run.smithi089.stdout: "age": 194.49575675899999, 2023-06-03T20:04:20.847 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.18271771000000001, 2023-06-03T20:04:20.847 INFO:teuthology.orchestra.run.smithi089.stdout: "type_data": { 2023-06-03T20:04:20.848 INFO:teuthology.orchestra.run.smithi089.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-03T20:04:20.848 INFO:teuthology.orchestra.run.smithi089.stdout: "client_info": { 2023-06-03T20:04:20.848 INFO:teuthology.orchestra.run.smithi089.stdout: "client": "client.45713", 2023-06-03T20:04:20.848 INFO:teuthology.orchestra.run.smithi089.stdout: "client_addr": "172.21.15.89:0/2400821448", 2023-06-03T20:04:20.848 INFO:teuthology.orchestra.run.smithi089.stdout: "tid": 44604 2023-06-03T20:04:20.848 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:04:20.848 INFO:teuthology.orchestra.run.smithi089.stdout: "events": [ 2023-06-03T20:04:20.849 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:04:20.849 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "initiated", 2023-06-03T20:04:20.849 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:01:06.112902+0000", 2023-06-03T20:04:20.849 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0 2023-06-03T20:04:20.849 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:04:20.849 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:04:20.849 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "throttled", 2023-06-03T20:04:20.850 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:01:06.112902+0000", 2023-06-03T20:04:20.850 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0 2023-06-03T20:04:20.850 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:04:20.850 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:04:20.850 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "header_read", 2023-06-03T20:04:20.851 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:01:06.112901+0000", 2023-06-03T20:04:20.851 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 4294967295.999999 2023-06-03T20:04:20.851 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:04:20.851 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:04:20.851 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "all_read", 2023-06-03T20:04:20.851 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:01:06.112925+0000", 2023-06-03T20:04:20.851 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 2.3680999999999999e-05 2023-06-03T20:04:20.852 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:04:20.852 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:04:20.852 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "dispatched", 2023-06-03T20:04:20.852 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:01:06.112938+0000", 2023-06-03T20:04:20.852 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 1.3076e-05 2023-06-03T20:04:20.852 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:04:20.852 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:04:20.853 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "queued_for_pg", 2023-06-03T20:04:20.853 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:01:06.112947+0000", 2023-06-03T20:04:20.853 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 9.1500000000000005e-06 2023-06-03T20:04:20.853 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:04:20.853 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:04:20.853 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "reached_pg", 2023-06-03T20:04:20.854 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:01:06.117031+0000", 2023-06-03T20:04:20.854 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.0040836370000000002 2023-06-03T20:04:20.854 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:04:20.854 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:04:20.854 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "started", 2023-06-03T20:04:20.854 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:01:06.117215+0000", 2023-06-03T20:04:20.854 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.00018372700000000001 2023-06-03T20:04:20.855 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:04:20.855 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:04:20.855 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "waiting for subops from 4,7", 2023-06-03T20:04:20.855 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:01:06.117370+0000", 2023-06-03T20:04:20.855 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.000155682 2023-06-03T20:04:20.855 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:04:20.855 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:04:20.856 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:04:20.856 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:01:06.291986+0000", 2023-06-03T20:04:20.856 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.174615933 2023-06-03T20:04:20.856 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:04:20.856 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:04:20.856 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:04:20.857 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:01:06.292439+0000", 2023-06-03T20:04:20.857 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.00045322299999999999 2023-06-03T20:04:20.857 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:04:20.857 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:04:20.857 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "op_commit", 2023-06-03T20:04:20.857 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:01:06.295562+0000", 2023-06-03T20:04:20.857 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.0031223259999999999 2023-06-03T20:04:20.858 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:04:20.858 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:04:20.858 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "commit_sent", 2023-06-03T20:04:20.858 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:01:06.295597+0000", 2023-06-03T20:04:20.858 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 3.5669e-05 2023-06-03T20:04:20.858 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:04:20.858 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:04:20.859 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "done", 2023-06-03T20:04:20.859 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:01:06.295620+0000", 2023-06-03T20:04:20.859 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 2.2685999999999999e-05 2023-06-03T20:04:20.859 INFO:teuthology.orchestra.run.smithi089.stdout: } 2023-06-03T20:04:20.859 INFO:teuthology.orchestra.run.smithi089.stdout: ] 2023-06-03T20:04:20.859 INFO:teuthology.orchestra.run.smithi089.stdout: } 2023-06-03T20:04:20.859 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:04:20.860 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:04:20.860 INFO:teuthology.orchestra.run.smithi089.stdout: "description": "osd_op(client.45713.0:44609 9.c 9:3e296c41:::benchmark_data_smithi089_115580_object44608:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e132)", 2023-06-03T20:04:20.860 INFO:teuthology.orchestra.run.smithi089.stdout: "initiated_at": "2023-06-03T20:01:06.117659+0000", 2023-06-03T20:04:20.860 INFO:teuthology.orchestra.run.smithi089.stdout: "age": 194.49100032000001, 2023-06-03T20:04:20.860 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.182704428, 2023-06-03T20:04:20.860 INFO:teuthology.orchestra.run.smithi089.stdout: "type_data": { 2023-06-03T20:04:20.861 INFO:teuthology.orchestra.run.smithi089.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-03T20:04:20.861 INFO:teuthology.orchestra.run.smithi089.stdout: "client_info": { 2023-06-03T20:04:20.861 INFO:teuthology.orchestra.run.smithi089.stdout: "client": "client.45713", 2023-06-03T20:04:20.861 INFO:teuthology.orchestra.run.smithi089.stdout: "client_addr": "172.21.15.89:0/2400821448", 2023-06-03T20:04:20.861 INFO:teuthology.orchestra.run.smithi089.stdout: "tid": 44609 2023-06-03T20:04:20.861 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:04:20.861 INFO:teuthology.orchestra.run.smithi089.stdout: "events": [ 2023-06-03T20:04:20.862 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:04:20.862 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "initiated", 2023-06-03T20:04:20.862 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:01:06.117659+0000", 2023-06-03T20:04:20.862 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0 2023-06-03T20:04:20.862 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:04:20.862 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:04:20.862 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "throttled", 2023-06-03T20:04:20.863 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:01:06.117659+0000", 2023-06-03T20:04:20.863 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0 2023-06-03T20:04:20.863 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:04:20.863 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:04:20.863 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "header_read", 2023-06-03T20:04:20.863 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:01:06.117658+0000", 2023-06-03T20:04:20.863 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 4294967295.999999 2023-06-03T20:04:20.863 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:04:20.864 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:04:20.864 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "all_read", 2023-06-03T20:04:20.864 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:01:06.117680+0000", 2023-06-03T20:04:20.864 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 2.2787999999999999e-05 2023-06-03T20:04:20.864 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:04:20.864 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:04:20.864 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "dispatched", 2023-06-03T20:04:20.865 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:01:06.117690+0000", 2023-06-03T20:04:20.865 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 9.3160000000000003e-06 2023-06-03T20:04:20.865 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:04:20.865 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:04:20.865 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "queued_for_pg", 2023-06-03T20:04:20.865 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:01:06.117697+0000", 2023-06-03T20:04:20.865 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 7.3089999999999998e-06 2023-06-03T20:04:20.866 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:04:20.866 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:04:20.866 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "reached_pg", 2023-06-03T20:04:20.866 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:01:06.289910+0000", 2023-06-03T20:04:20.866 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.17221251300000001 2023-06-03T20:04:20.866 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:04:20.867 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:04:20.867 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "started", 2023-06-03T20:04:20.867 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:01:06.290075+0000", 2023-06-03T20:04:20.867 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.000165141 2023-06-03T20:04:20.867 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:04:20.867 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:04:20.867 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "waiting for subops from 4,7", 2023-06-03T20:04:20.868 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:01:06.290251+0000", 2023-06-03T20:04:20.868 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.00017654300000000001 2023-06-03T20:04:20.868 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:04:20.868 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:04:20.868 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:04:20.868 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:01:06.293599+0000", 2023-06-03T20:04:20.868 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.003347877 2023-06-03T20:04:20.869 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:04:20.869 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:04:20.869 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "op_commit", 2023-06-03T20:04:20.869 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:01:06.295628+0000", 2023-06-03T20:04:20.869 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.002028827 2023-06-03T20:04:20.869 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:04:20.869 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:04:20.870 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:04:20.870 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:01:06.300208+0000", 2023-06-03T20:04:20.870 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.0045802799999999999 2023-06-03T20:04:20.870 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:04:20.870 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:04:20.870 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "commit_sent", 2023-06-03T20:04:20.871 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:01:06.300298+0000", 2023-06-03T20:04:20.871 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 8.9876999999999993e-05 2023-06-03T20:04:20.871 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:04:20.871 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:04:20.871 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "done", 2023-06-03T20:04:20.871 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:01:06.300363+0000", 2023-06-03T20:04:20.871 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 6.512e-05 2023-06-03T20:04:20.872 INFO:teuthology.orchestra.run.smithi089.stdout: } 2023-06-03T20:04:20.872 INFO:teuthology.orchestra.run.smithi089.stdout: ] 2023-06-03T20:04:20.872 INFO:teuthology.orchestra.run.smithi089.stdout: } 2023-06-03T20:04:20.872 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:04:20.872 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:04:20.872 INFO:teuthology.orchestra.run.smithi089.stdout: "description": "osd_op(client.45713.0:136510 9.c 9:3046c07e:::benchmark_data_smithi089_115580_object26376:head [delete] snapc 0=[] ondisk+write+known_if_redirected e138)", 2023-06-03T20:04:20.872 INFO:teuthology.orchestra.run.smithi089.stdout: "initiated_at": "2023-06-03T20:02:45.473436+0000", 2023-06-03T20:04:20.873 INFO:teuthology.orchestra.run.smithi089.stdout: "age": 95.135223191999998, 2023-06-03T20:04:20.873 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.192932662, 2023-06-03T20:04:20.873 INFO:teuthology.orchestra.run.smithi089.stdout: "type_data": { 2023-06-03T20:04:20.873 INFO:teuthology.orchestra.run.smithi089.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-03T20:04:20.873 INFO:teuthology.orchestra.run.smithi089.stdout: "client_info": { 2023-06-03T20:04:20.873 INFO:teuthology.orchestra.run.smithi089.stdout: "client": "client.45713", 2023-06-03T20:04:20.873 INFO:teuthology.orchestra.run.smithi089.stdout: "client_addr": "172.21.15.89:0/2400821448", 2023-06-03T20:04:20.874 INFO:teuthology.orchestra.run.smithi089.stdout: "tid": 136510 2023-06-03T20:04:20.874 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:04:20.874 INFO:teuthology.orchestra.run.smithi089.stdout: "events": [ 2023-06-03T20:04:20.874 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:04:20.874 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "initiated", 2023-06-03T20:04:20.874 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:02:45.473436+0000", 2023-06-03T20:04:20.874 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0 2023-06-03T20:04:20.875 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:04:20.875 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:04:20.875 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "throttled", 2023-06-03T20:04:20.875 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:02:45.473436+0000", 2023-06-03T20:04:20.875 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0 2023-06-03T20:04:20.875 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:04:20.875 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:04:20.876 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "header_read", 2023-06-03T20:04:20.876 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:02:45.473435+0000", 2023-06-03T20:04:20.876 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 4294967295.999999 2023-06-03T20:04:20.876 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:04:20.876 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:04:20.876 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "all_read", 2023-06-03T20:04:20.876 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:02:45.473440+0000", 2023-06-03T20:04:20.876 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 4.9679999999999997e-06 2023-06-03T20:04:20.877 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:04:20.877 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:04:20.877 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "dispatched", 2023-06-03T20:04:20.877 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:02:45.636494+0000", 2023-06-03T20:04:20.877 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.16305381199999999 2023-06-03T20:04:20.877 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:04:20.877 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:04:20.878 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "queued_for_pg", 2023-06-03T20:04:20.878 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:02:45.636502+0000", 2023-06-03T20:04:20.878 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 8.2989999999999997e-06 2023-06-03T20:04:20.878 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:04:20.878 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:04:20.878 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "reached_pg", 2023-06-03T20:04:20.878 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:02:45.636538+0000", 2023-06-03T20:04:20.879 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 3.6202999999999999e-05 2023-06-03T20:04:20.879 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:04:20.879 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:04:20.879 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "started", 2023-06-03T20:04:20.879 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:02:45.636723+0000", 2023-06-03T20:04:20.879 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.000184222 2023-06-03T20:04:20.879 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:04:20.880 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:04:20.880 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "waiting for subops from 4,7", 2023-06-03T20:04:20.880 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:02:45.636884+0000", 2023-06-03T20:04:20.880 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.00016191900000000001 2023-06-03T20:04:20.880 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:04:20.880 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:04:20.880 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "op_commit", 2023-06-03T20:04:20.881 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:02:45.653591+0000", 2023-06-03T20:04:20.881 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.016706978000000001 2023-06-03T20:04:20.881 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:04:20.881 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:04:20.881 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:04:20.881 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:02:45.656133+0000", 2023-06-03T20:04:20.881 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.0025413570000000002 2023-06-03T20:04:20.882 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:04:20.882 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:04:20.882 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:04:20.882 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:02:45.666274+0000", 2023-06-03T20:04:20.882 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.010141537000000001 2023-06-03T20:04:20.882 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:04:20.883 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:04:20.883 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "commit_sent", 2023-06-03T20:04:20.883 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:02:45.666326+0000", 2023-06-03T20:04:20.883 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 5.1672000000000003e-05 2023-06-03T20:04:20.883 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:04:20.883 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:04:20.883 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "done", 2023-06-03T20:04:20.883 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:02:45.666369+0000", 2023-06-03T20:04:20.884 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 4.2512999999999998e-05 2023-06-03T20:04:20.884 INFO:teuthology.orchestra.run.smithi089.stdout: } 2023-06-03T20:04:20.884 INFO:teuthology.orchestra.run.smithi089.stdout: ] 2023-06-03T20:04:20.884 INFO:teuthology.orchestra.run.smithi089.stdout: } 2023-06-03T20:04:20.884 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:04:20.884 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:04:20.885 INFO:teuthology.orchestra.run.smithi089.stdout: "description": "osd_op(client.45713.0:146658 9.8 9:1a2d8745:::benchmark_data_smithi089_115580_object36524:head [delete] snapc 0=[] ondisk+write+known_if_redirected e139)", 2023-06-03T20:04:20.885 INFO:teuthology.orchestra.run.smithi089.stdout: "initiated_at": "2023-06-03T20:02:59.940640+0000", 2023-06-03T20:04:20.885 INFO:teuthology.orchestra.run.smithi089.stdout: "age": 80.668018958000005, 2023-06-03T20:04:20.885 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.180612262, 2023-06-03T20:04:20.885 INFO:teuthology.orchestra.run.smithi089.stdout: "type_data": { 2023-06-03T20:04:20.885 INFO:teuthology.orchestra.run.smithi089.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-03T20:04:20.885 INFO:teuthology.orchestra.run.smithi089.stdout: "client_info": { 2023-06-03T20:04:20.885 INFO:teuthology.orchestra.run.smithi089.stdout: "client": "client.45713", 2023-06-03T20:04:20.886 INFO:teuthology.orchestra.run.smithi089.stdout: "client_addr": "172.21.15.89:0/2400821448", 2023-06-03T20:04:20.886 INFO:teuthology.orchestra.run.smithi089.stdout: "tid": 146658 2023-06-03T20:04:20.886 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:04:20.886 INFO:teuthology.orchestra.run.smithi089.stdout: "events": [ 2023-06-03T20:04:20.886 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:04:20.886 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "initiated", 2023-06-03T20:04:20.886 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:02:59.940640+0000", 2023-06-03T20:04:20.887 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0 2023-06-03T20:04:20.887 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:04:20.887 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:04:20.887 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "throttled", 2023-06-03T20:04:20.887 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:02:59.940640+0000", 2023-06-03T20:04:20.887 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0 2023-06-03T20:04:20.887 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:04:20.888 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:04:20.888 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "header_read", 2023-06-03T20:04:20.888 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:02:59.940638+0000", 2023-06-03T20:04:20.888 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 4294967295.9999981 2023-06-03T20:04:20.888 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:04:20.888 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:04:20.888 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "all_read", 2023-06-03T20:04:20.889 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:02:59.940645+0000", 2023-06-03T20:04:20.889 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 6.9229999999999996e-06 2023-06-03T20:04:20.889 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:04:20.889 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:04:20.889 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "dispatched", 2023-06-03T20:04:20.889 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:02:59.940657+0000", 2023-06-03T20:04:20.890 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 1.1610999999999999e-05 2023-06-03T20:04:20.890 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:04:20.890 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:04:20.890 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "queued_for_pg", 2023-06-03T20:04:20.890 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:02:59.940665+0000", 2023-06-03T20:04:20.890 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 8.3620000000000006e-06 2023-06-03T20:04:20.890 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:04:20.890 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:04:20.891 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "reached_pg", 2023-06-03T20:04:20.891 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:02:59.942002+0000", 2023-06-03T20:04:20.891 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.001336372 2023-06-03T20:04:20.891 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:04:20.891 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:04:20.891 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "started", 2023-06-03T20:04:20.891 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:02:59.942073+0000", 2023-06-03T20:04:20.892 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 7.1060000000000001e-05 2023-06-03T20:04:20.892 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:04:20.892 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:04:20.892 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "waiting for subops from 0,5", 2023-06-03T20:04:20.892 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:02:59.942157+0000", 2023-06-03T20:04:20.892 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 8.4030999999999996e-05 2023-06-03T20:04:20.893 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:04:20.893 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:04:20.893 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "op_commit", 2023-06-03T20:04:20.893 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:02:59.946057+0000", 2023-06-03T20:04:20.893 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.0039008200000000002 2023-06-03T20:04:20.893 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:04:20.893 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:04:20.893 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:04:20.894 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:02:59.946507+0000", 2023-06-03T20:04:20.894 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.00044961600000000001 2023-06-03T20:04:20.894 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:04:20.894 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:04:20.894 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:04:20.894 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:03:00.121156+0000", 2023-06-03T20:04:20.894 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.17464881199999999 2023-06-03T20:04:20.895 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:04:20.895 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:04:20.895 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "commit_sent", 2023-06-03T20:04:20.895 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:03:00.121205+0000", 2023-06-03T20:04:20.895 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 4.9425000000000002e-05 2023-06-03T20:04:20.895 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:04:20.895 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:04:20.896 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "done", 2023-06-03T20:04:20.896 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:03:00.121252+0000", 2023-06-03T20:04:20.896 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 4.7040999999999999e-05 2023-06-03T20:04:20.896 INFO:teuthology.orchestra.run.smithi089.stdout: } 2023-06-03T20:04:20.896 INFO:teuthology.orchestra.run.smithi089.stdout: ] 2023-06-03T20:04:20.896 INFO:teuthology.orchestra.run.smithi089.stdout: } 2023-06-03T20:04:20.896 INFO:teuthology.orchestra.run.smithi089.stdout: } 2023-06-03T20:04:20.897 INFO:teuthology.orchestra.run.smithi089.stdout: ] 2023-06-03T20:04:20.897 INFO:teuthology.orchestra.run.smithi089.stdout:} 2023-06-03T20:04:20.897 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:04:20.934 INFO:teuthology.orchestra.run.smithi089.stdout: 2023-06-03T20:04:20.939 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:04:20.941 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:04:20.941 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:04:20.948 INFO:teuthology.orchestra.run.smithi089.stderr:dumped all 2023-06-03T20:04:21.042 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:04:21.073 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:04:21.080 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:04:21.081 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:04:21.182 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:04:21.220 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:04:21.223 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:04:21.224 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:04:21.324 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:04:21.366 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:04:21.368 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:04:21.368 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:04:21.469 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:04:21.511 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:04:21.517 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:04:21.518 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:04:21.619 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:04:21.649 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:04:21.654 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:04:21.656 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:04:21.757 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:04:21.767 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:04:21 smithi164 bash[59462]: cluster 1685822660.5061982 mgr.y (mgr.34104) 843 : cluster 0 pgmap v764: 89 pgs: 89 active+clean; 1.1 GiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 701 op/s 2023-06-03T20:04:21.767 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:04:21 smithi164 bash[59462]: audit 1685822660.9141088 mgr.y (mgr.34104) 844 : audit 0 from='client.45839 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-06-03T20:04:21.807 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:04:21.810 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:04:21.811 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:04:21.893 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:04:21 smithi089 bash[75068]: cluster 1685822660.5061982 mgr.y (mgr.34104) 843 : cluster 0 pgmap v764: 89 pgs: 89 active+clean; 1.1 GiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 701 op/s 2023-06-03T20:04:21.894 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:04:21 smithi089 bash[75068]: audit 1685822660.9141088 mgr.y (mgr.34104) 844 : audit 0 from='client.45839 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-06-03T20:04:21.894 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:04:21 smithi089 bash[78452]: cluster 1685822660.5061982 mgr.y (mgr.34104) 843 : cluster 0 pgmap v764: 89 pgs: 89 active+clean; 1.1 GiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 701 op/s 2023-06-03T20:04:21.894 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:04:21 smithi089 bash[78452]: audit 1685822660.9141088 mgr.y (mgr.34104) 844 : audit 0 from='client.45839 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-06-03T20:04:21.912 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:04:21.945 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:04:21.949 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:04:21.950 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:04:22.051 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:04:22.088 DEBUG:teuthology.orchestra.run.smithi164:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph daemon osd.4 dump_ops_in_flight 2023-06-03T20:04:22.099 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:04:22.106 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:04:22.107 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:04:22.208 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:04:22.245 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:04:22.249 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:04:22.250 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:04:22.351 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:04:22.390 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:04:22.391 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:04:22.391 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:04:22.492 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:04:22.527 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:04:22.531 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:04:22.532 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:04:22.633 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:04:22.666 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:04:22 smithi164 bash[59462]: audit 1685822662.1048894 mon.c (mon.2) 414 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T20:04:22.673 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:04:22.677 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:04:22.678 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:04:22.778 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:04:22 smithi089 bash[78452]: audit 1685822662.1048894 mon.c (mon.2) 414 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T20:04:22.779 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:04:22 smithi089 bash[75068]: audit 1685822662.1048894 mon.c (mon.2) 414 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T20:04:22.779 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:04:22.810 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:04:22.819 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:04:22.820 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:04:22.857 INFO:teuthology.orchestra.run.smithi089.stdout:{"pg_ready":true,"pg_map":{"version":764,"stamp":"2023-06-03T20:04:20.505834+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":1195972518,"num_objects":18265,"num_object_clones":0,"num_object_copies":54795,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":18265,"num_whiteouts":0,"num_read":309,"num_read_kb":513,"num_write":202284,"num_write_kb":7050831,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139518,"ondisk_log_size":139518,"up":267,"acting":267,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":267,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":5471180,"kb_used_data":3608076,"kb_used_omap":30,"kb_used_meta":1862817,"kb_avail":744490036,"statfs":{"total":767960285184,"available":762357796864,"internally_reserved":0,"allocated":3694669824,"data_stored":3690485835,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":31502,"internal_metadata":1907524850},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[3,0,5,9,8,7,3],"upper_bound":128},"perf_stat":{"commit_latency_ms":4,"apply_latency_ms":4,"commit_latency_ns":4000000,"apply_latency_ns":4000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-459931648,"num_objects":-7018,"num_object_clones":0,"num_object_copies":-21054,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":-7018,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7018,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.002723"},"pg_stats":[{"pgid":"3.1a","version":"60'1","reported_seq":131,"reported_epoch":143,"state":"active+clean","last_fresh":"2023-06-03T20:03:50.828846+0000","last_change":"2023-06-03T19:57:28.467559+0000","last_active":"2023-06-03T20:03:50.828846+0000","last_peered":"2023-06-03T20:03:50.828846+0000","last_clean":"2023-06-03T20:03:50.828846+0000","last_became_active":"2023-06-03T19:57:28.467077+0000","last_became_peered":"2023-06-03T19:57:28.467077+0000","last_unstale":"2023-06-03T20:03:50.828846+0000","last_undegraded":"2023-06-03T20:03:50.828846+0000","last_fullsized":"2023-06-03T20:03:50.828846+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T21:46:02.795576+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":124,"reported_epoch":143,"state":"active+clean","last_fresh":"2023-06-03T20:03:50.849175+0000","last_change":"2023-06-03T19:57:28.470287+0000","last_active":"2023-06-03T20:03:50.849175+0000","last_peered":"2023-06-03T20:03:50.849175+0000","last_clean":"2023-06-03T20:03:50.849175+0000","last_became_active":"2023-06-03T19:57:28.469477+0000","last_became_peered":"2023-06-03T19:57:28.469477+0000","last_unstale":"2023-06-03T20:03:50.849175+0000","last_undegraded":"2023-06-03T20:03:50.849175+0000","last_fullsized":"2023-06-03T20:03:50.849175+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T06:48:19.229190+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":138,"reported_epoch":143,"state":"active+clean","last_fresh":"2023-06-03T20:04:03.617670+0000","last_change":"2023-06-03T19:57:28.473350+0000","last_active":"2023-06-03T20:04:03.617670+0000","last_peered":"2023-06-03T20:04:03.617670+0000","last_clean":"2023-06-03T20:04:03.617670+0000","last_became_active":"2023-06-03T19:57:28.472757+0000","last_became_peered":"2023-06-03T19:57:28.472757+0000","last_unstale":"2023-06-03T20:04:03.617670+0000","last_undegraded":"2023-06-03T20:04:03.617670+0000","last_fullsized":"2023-06-03T20:04:03.617670+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T19:58:10.432188+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":102,"reported_epoch":143,"state":"active+clean","last_fresh":"2023-06-03T20:03:50.857122+0000","last_change":"2023-06-03T19:56:54.507653+0000","last_active":"2023-06-03T20:03:50.857122+0000","last_peered":"2023-06-03T20:03:50.857122+0000","last_clean":"2023-06-03T20:03:50.857122+0000","last_became_active":"2023-06-03T19:56:54.507350+0000","last_became_peered":"2023-06-03T19:56:54.507350+0000","last_unstale":"2023-06-03T20:03:50.857122+0000","last_undegraded":"2023-06-03T20:03:50.857122+0000","last_fullsized":"2023-06-03T20:03:50.857122+0000","mapping_epoch":119,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T22:33:20.065844+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":126,"reported_epoch":143,"state":"active+clean","last_fresh":"2023-06-03T20:04:08.100990+0000","last_change":"2023-06-03T19:56:54.512932+0000","last_active":"2023-06-03T20:04:08.100990+0000","last_peered":"2023-06-03T20:04:08.100990+0000","last_clean":"2023-06-03T20:04:08.100990+0000","last_became_active":"2023-06-03T19:56:54.511992+0000","last_became_peered":"2023-06-03T19:56:54.511992+0000","last_unstale":"2023-06-03T20:04:08.100990+0000","last_undegraded":"2023-06-03T20:04:08.100990+0000","last_fullsized":"2023-06-03T20:04:08.100990+0000","mapping_epoch":119,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T21:01:35.069911+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":110,"reported_epoch":143,"state":"active+clean","last_fresh":"2023-06-03T20:03:50.833368+0000","last_change":"2023-06-03T19:51:59.359522+0000","last_active":"2023-06-03T20:03:50.833368+0000","last_peered":"2023-06-03T20:03:50.833368+0000","last_clean":"2023-06-03T20:03:50.833368+0000","last_became_active":"2023-06-03T19:51:59.359202+0000","last_became_peered":"2023-06-03T19:51:59.359202+0000","last_unstale":"2023-06-03T20:03:50.833368+0000","last_undegraded":"2023-06-03T20:03:50.833368+0000","last_fullsized":"2023-06-03T20:03:50.833368+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":100,"reported_epoch":143,"state":"active+clean","last_fresh":"2023-06-03T20:03:59.965810+0000","last_change":"2023-06-03T19:56:54.509490+0000","last_active":"2023-06-03T20:03:59.965810+0000","last_peered":"2023-06-03T20:03:59.965810+0000","last_clean":"2023-06-03T20:03:59.965810+0000","last_became_active":"2023-06-03T19:56:54.509145+0000","last_became_peered":"2023-06-03T19:56:54.509145+0000","last_unstale":"2023-06-03T20:03:59.965810+0000","last_undegraded":"2023-06-03T20:03:59.965810+0000","last_fullsized":"2023-06-03T20:03:59.965810+0000","mapping_epoch":119,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T07:12:03.964110+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":110,"reported_epoch":143,"state":"active+clean","last_fresh":"2023-06-03T20:03:53.088816+0000","last_change":"2023-06-03T19:56:54.511325+0000","last_active":"2023-06-03T20:03:53.088816+0000","last_peered":"2023-06-03T20:03:53.088816+0000","last_clean":"2023-06-03T20:03:53.088816+0000","last_became_active":"2023-06-03T19:56:54.511033+0000","last_became_peered":"2023-06-03T19:56:54.511033+0000","last_unstale":"2023-06-03T20:03:53.088816+0000","last_undegraded":"2023-06-03T20:03:53.088816+0000","last_fullsized":"2023-06-03T20:03:53.088816+0000","mapping_epoch":119,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T23:50:08.711478+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":111,"reported_epoch":143,"state":"active+clean","last_fresh":"2023-06-03T20:03:50.843420+0000","last_change":"2023-06-03T19:51:59.374143+0000","last_active":"2023-06-03T20:03:50.843420+0000","last_peered":"2023-06-03T20:03:50.843420+0000","last_clean":"2023-06-03T20:03:50.843420+0000","last_became_active":"2023-06-03T19:51:59.373146+0000","last_became_peered":"2023-06-03T19:51:59.373146+0000","last_unstale":"2023-06-03T20:03:50.843420+0000","last_undegraded":"2023-06-03T20:03:50.843420+0000","last_fullsized":"2023-06-03T20:03:50.843420+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":91,"reported_epoch":143,"state":"active+clean","last_fresh":"2023-06-03T20:03:53.088892+0000","last_change":"2023-06-03T19:56:54.510462+0000","last_active":"2023-06-03T20:03:53.088892+0000","last_peered":"2023-06-03T20:03:53.088892+0000","last_clean":"2023-06-03T20:03:53.088892+0000","last_became_active":"2023-06-03T19:56:54.509783+0000","last_became_peered":"2023-06-03T19:56:54.509783+0000","last_unstale":"2023-06-03T20:03:53.088892+0000","last_undegraded":"2023-06-03T20:03:53.088892+0000","last_fullsized":"2023-06-03T20:03:53.088892+0000","mapping_epoch":119,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T00:16:09.624276+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"60'3","reported_seq":129,"reported_epoch":143,"state":"active+clean","last_fresh":"2023-06-03T20:03:50.851283+0000","last_change":"2023-06-03T19:57:28.466401+0000","last_active":"2023-06-03T20:03:50.851283+0000","last_peered":"2023-06-03T20:03:50.851283+0000","last_clean":"2023-06-03T20:03:50.851283+0000","last_became_active":"2023-06-03T19:57:28.465452+0000","last_became_peered":"2023-06-03T19:57:28.465452+0000","last_unstale":"2023-06-03T20:03:50.851283+0000","last_undegraded":"2023-06-03T20:03:50.851283+0000","last_fullsized":"2023-06-03T20:03:50.851283+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":82,"reported_epoch":143,"state":"active+clean","last_fresh":"2023-06-03T20:03:50.843525+0000","last_change":"2023-06-03T19:56:11.753995+0000","last_active":"2023-06-03T20:03:50.843525+0000","last_peered":"2023-06-03T20:03:50.843525+0000","last_clean":"2023-06-03T20:03:50.843525+0000","last_became_active":"2023-06-03T19:56:11.753676+0000","last_became_peered":"2023-06-03T19:56:11.753676+0000","last_unstale":"2023-06-03T20:03:50.843525+0000","last_undegraded":"2023-06-03T20:03:50.843525+0000","last_fullsized":"2023-06-03T20:03:50.843525+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":115,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T01:00:40.343647+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"77'23","reported_seq":127,"reported_epoch":143,"state":"active+clean","last_fresh":"2023-06-03T20:03:50.835121+0000","last_change":"2023-06-03T19:57:28.462522+0000","last_active":"2023-06-03T20:03:50.835121+0000","last_peered":"2023-06-03T20:03:50.835121+0000","last_clean":"2023-06-03T20:03:50.835121+0000","last_became_active":"2023-06-03T19:57:28.462048+0000","last_became_peered":"2023-06-03T19:57:28.462048+0000","last_unstale":"2023-06-03T20:03:50.835121+0000","last_undegraded":"2023-06-03T20:03:50.835121+0000","last_fullsized":"2023-06-03T20:03:50.835121+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":23,"log_dups_size":0,"ondisk_log_size":23,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T23:23:56.697534+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4716,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":23,"num_write_kb":20,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":104,"reported_epoch":143,"state":"active+clean","last_fresh":"2023-06-03T20:03:50.829436+0000","last_change":"2023-06-03T19:57:28.469676+0000","last_active":"2023-06-03T20:03:50.829436+0000","last_peered":"2023-06-03T20:03:50.829436+0000","last_clean":"2023-06-03T20:03:50.829436+0000","last_became_active":"2023-06-03T19:57:28.469328+0000","last_became_peered":"2023-06-03T19:57:28.469328+0000","last_unstale":"2023-06-03T20:03:50.829436+0000","last_undegraded":"2023-06-03T20:03:50.829436+0000","last_fullsized":"2023-06-03T20:03:50.829436+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T23:25:26.041930+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"60'2","reported_seq":128,"reported_epoch":143,"state":"active+clean","last_fresh":"2023-06-03T20:03:50.850103+0000","last_change":"2023-06-03T19:57:28.467711+0000","last_active":"2023-06-03T20:03:50.850103+0000","last_peered":"2023-06-03T20:03:50.850103+0000","last_clean":"2023-06-03T20:03:50.850103+0000","last_became_active":"2023-06-03T19:57:28.467234+0000","last_became_peered":"2023-06-03T19:57:28.467234+0000","last_unstale":"2023-06-03T20:03:50.850103+0000","last_undegraded":"2023-06-03T20:03:50.850103+0000","last_fullsized":"2023-06-03T20:03:50.850103+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":110,"reported_epoch":143,"state":"active+clean","last_fresh":"2023-06-03T20:03:53.088943+0000","last_change":"2023-06-03T19:56:54.514616+0000","last_active":"2023-06-03T20:03:53.088943+0000","last_peered":"2023-06-03T20:03:53.088943+0000","last_clean":"2023-06-03T20:03:53.088943+0000","last_became_active":"2023-06-03T19:56:54.513918+0000","last_became_peered":"2023-06-03T19:56:54.513918+0000","last_unstale":"2023-06-03T20:03:53.088943+0000","last_undegraded":"2023-06-03T20:03:53.088943+0000","last_fullsized":"2023-06-03T20:03:53.088943+0000","mapping_epoch":119,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T19:36:10.967386+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":100,"reported_epoch":143,"state":"active+clean","last_fresh":"2023-06-03T20:03:50.857009+0000","last_change":"2023-06-03T19:56:54.507479+0000","last_active":"2023-06-03T20:03:50.857009+0000","last_peered":"2023-06-03T20:03:50.857009+0000","last_clean":"2023-06-03T20:03:50.857009+0000","last_became_active":"2023-06-03T19:56:54.507149+0000","last_became_peered":"2023-06-03T19:56:54.507149+0000","last_unstale":"2023-06-03T20:03:50.857009+0000","last_undegraded":"2023-06-03T20:03:50.857009+0000","last_fullsized":"2023-06-03T20:03:50.857009+0000","mapping_epoch":119,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T20:22:36.676317+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":110,"reported_epoch":143,"state":"active+clean","last_fresh":"2023-06-03T20:03:50.849536+0000","last_change":"2023-06-03T19:56:54.512472+0000","last_active":"2023-06-03T20:03:50.849536+0000","last_peered":"2023-06-03T20:03:50.849536+0000","last_clean":"2023-06-03T20:03:50.849536+0000","last_became_active":"2023-06-03T19:56:54.512051+0000","last_became_peered":"2023-06-03T19:56:54.512051+0000","last_unstale":"2023-06-03T20:03:50.849536+0000","last_undegraded":"2023-06-03T20:03:50.849536+0000","last_fullsized":"2023-06-03T20:03:50.849536+0000","mapping_epoch":119,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":111,"reported_epoch":143,"state":"active+clean","last_fresh":"2023-06-03T20:03:50.849666+0000","last_change":"2023-06-03T19:57:28.462780+0000","last_active":"2023-06-03T20:03:50.849666+0000","last_peered":"2023-06-03T20:03:50.849666+0000","last_clean":"2023-06-03T20:03:50.849666+0000","last_became_active":"2023-06-03T19:57:28.462298+0000","last_became_peered":"2023-06-03T19:57:28.462298+0000","last_unstale":"2023-06-03T20:03:50.849666+0000","last_undegraded":"2023-06-03T20:03:50.849666+0000","last_fullsized":"2023-06-03T20:03:50.849666+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":101,"reported_epoch":143,"state":"active+clean","last_fresh":"2023-06-03T20:03:59.965779+0000","last_change":"2023-06-03T19:56:54.510289+0000","last_active":"2023-06-03T20:03:59.965779+0000","last_peered":"2023-06-03T20:03:59.965779+0000","last_clean":"2023-06-03T20:03:59.965779+0000","last_became_active":"2023-06-03T19:56:54.509905+0000","last_became_peered":"2023-06-03T19:56:54.509905+0000","last_unstale":"2023-06-03T20:03:59.965779+0000","last_undegraded":"2023-06-03T20:03:59.965779+0000","last_fullsized":"2023-06-03T20:03:59.965779+0000","mapping_epoch":119,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T02:54:26.792401+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"60'8","reported_seq":119,"reported_epoch":143,"state":"active+clean","last_fresh":"2023-06-03T20:03:50.835347+0000","last_change":"2023-06-03T19:47:06.727778+0000","last_active":"2023-06-03T20:03:50.835347+0000","last_peered":"2023-06-03T20:03:50.835347+0000","last_clean":"2023-06-03T20:03:50.835347+0000","last_became_active":"2023-06-03T19:47:06.727305+0000","last_became_peered":"2023-06-03T19:47:06.727305+0000","last_unstale":"2023-06-03T20:03:50.835347+0000","last_undegraded":"2023-06-03T20:03:50.835347+0000","last_fullsized":"2023-06-03T20:03:50.835347+0000","mapping_epoch":86,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":87,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":110,"reported_epoch":143,"state":"active+clean","last_fresh":"2023-06-03T20:04:08.101125+0000","last_change":"2023-06-03T19:56:54.512994+0000","last_active":"2023-06-03T20:04:08.101125+0000","last_peered":"2023-06-03T20:04:08.101125+0000","last_clean":"2023-06-03T20:04:08.101125+0000","last_became_active":"2023-06-03T19:56:54.512579+0000","last_became_peered":"2023-06-03T19:56:54.512579+0000","last_unstale":"2023-06-03T20:04:08.101125+0000","last_undegraded":"2023-06-03T20:04:08.101125+0000","last_fullsized":"2023-06-03T20:04:08.101125+0000","mapping_epoch":119,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T07:02:45.187961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"9.5","version":"144'12483","reported_seq":12510,"reported_epoch":144,"state":"active+clean","last_fresh":"2023-06-03T20:04:19.956058+0000","last_change":"2023-06-03T20:00:12.908073+0000","last_active":"2023-06-03T20:04:19.956058+0000","last_peered":"2023-06-03T20:04:19.956058+0000","last_clean":"2023-06-03T20:04:19.956058+0000","last_became_active":"2023-06-03T20:00:12.907738+0000","last_became_peered":"2023-06-03T20:00:12.907738+0000","last_unstale":"2023-06-03T20:04:19.956058+0000","last_undegraded":"2023-06-03T20:04:19.956058+0000","last_fullsized":"2023-06-03T20:04:19.956058+0000","mapping_epoch":127,"log_start":"131'2400","ondisk_log_start":"131'2400","created":127,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T20:00:11.873149+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T20:00:11.873149+0000","last_clean_scrub_stamp":"2023-06-03T20:00:11.873149+0000","objects_scrubbed":0,"log_size":10083,"log_dups_size":0,"ondisk_log_size":10083,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T02:31:40.900210+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":71761920,"num_objects":1095,"num_object_clones":0,"num_object_copies":3285,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1095,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12483,"num_write_kb":434496,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.f","version":"60'1","reported_seq":127,"reported_epoch":143,"state":"active+clean","last_fresh":"2023-06-03T20:03:50.857140+0000","last_change":"2023-06-03T19:57:28.464482+0000","last_active":"2023-06-03T20:03:50.857140+0000","last_peered":"2023-06-03T20:03:50.857140+0000","last_clean":"2023-06-03T20:03:50.857140+0000","last_became_active":"2023-06-03T19:57:28.464081+0000","last_became_peered":"2023-06-03T19:57:28.464081+0000","last_unstale":"2023-06-03T20:03:50.857140+0000","last_undegraded":"2023-06-03T20:03:50.857140+0000","last_fullsized":"2023-06-03T20:03:50.857140+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":95,"reported_epoch":143,"state":"active+clean","last_fresh":"2023-06-03T20:03:50.842772+0000","last_change":"2023-06-03T19:34:17.848358+0000","last_active":"2023-06-03T20:03:50.842772+0000","last_peered":"2023-06-03T20:03:50.842772+0000","last_clean":"2023-06-03T20:03:50.842772+0000","last_became_active":"2023-06-03T19:34:17.847811+0000","last_became_peered":"2023-06-03T19:34:17.847811+0000","last_unstale":"2023-06-03T20:03:50.842772+0000","last_undegraded":"2023-06-03T20:03:50.842772+0000","last_fullsized":"2023-06-03T20:03:50.842772+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"9.4","version":"144'12727","reported_seq":12754,"reported_epoch":144,"state":"active+clean","last_fresh":"2023-06-03T20:04:18.601552+0000","last_change":"2023-06-03T20:00:12.905941+0000","last_active":"2023-06-03T20:04:18.601552+0000","last_peered":"2023-06-03T20:04:18.601552+0000","last_clean":"2023-06-03T20:04:18.601552+0000","last_became_active":"2023-06-03T20:00:12.905661+0000","last_became_peered":"2023-06-03T20:00:12.905661+0000","last_unstale":"2023-06-03T20:04:18.601552+0000","last_undegraded":"2023-06-03T20:04:18.601552+0000","last_fullsized":"2023-06-03T20:04:18.601552+0000","mapping_epoch":127,"log_start":"134'5400","ondisk_log_start":"134'5400","created":127,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T20:00:11.873149+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T20:00:11.873149+0000","last_clean_scrub_stamp":"2023-06-03T20:00:11.873149+0000","objects_scrubbed":0,"log_size":7327,"log_dups_size":0,"ondisk_log_size":7327,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T06:19:01.942119+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":77135872,"num_objects":1177,"num_object_clones":0,"num_object_copies":3531,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1177,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12727,"num_write_kb":444928,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.e","version":"60'2","reported_seq":128,"reported_epoch":143,"state":"active+clean","last_fresh":"2023-06-03T20:03:50.850755+0000","last_change":"2023-06-03T19:57:28.468330+0000","last_active":"2023-06-03T20:03:50.850755+0000","last_peered":"2023-06-03T20:03:50.850755+0000","last_clean":"2023-06-03T20:03:50.850755+0000","last_became_active":"2023-06-03T19:57:28.467823+0000","last_became_peered":"2023-06-03T19:57:28.467823+0000","last_unstale":"2023-06-03T20:03:50.850755+0000","last_undegraded":"2023-06-03T20:03:50.850755+0000","last_fullsized":"2023-06-03T20:03:50.850755+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":143,"reported_epoch":143,"state":"active+clean","last_fresh":"2023-06-03T20:03:56.525761+0000","last_change":"2023-06-03T19:57:28.461449+0000","last_active":"2023-06-03T20:03:56.525761+0000","last_peered":"2023-06-03T20:03:56.525761+0000","last_clean":"2023-06-03T20:03:56.525761+0000","last_became_active":"2023-06-03T19:57:28.460745+0000","last_became_peered":"2023-06-03T19:57:28.460745+0000","last_unstale":"2023-06-03T20:03:56.525761+0000","last_undegraded":"2023-06-03T20:03:56.525761+0000","last_fullsized":"2023-06-03T20:03:56.525761+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T20:41:19.361638+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,3],"acting":[4,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.7","version":"144'12890","reported_seq":12915,"reported_epoch":144,"state":"active+clean","last_fresh":"2023-06-03T20:04:18.302429+0000","last_change":"2023-06-03T20:00:12.909354+0000","last_active":"2023-06-03T20:04:18.302429+0000","last_peered":"2023-06-03T20:04:18.302429+0000","last_clean":"2023-06-03T20:04:18.302429+0000","last_became_active":"2023-06-03T20:00:12.908971+0000","last_became_peered":"2023-06-03T20:00:12.908971+0000","last_unstale":"2023-06-03T20:04:18.302429+0000","last_undegraded":"2023-06-03T20:04:18.302429+0000","last_fullsized":"2023-06-03T20:04:18.302429+0000","mapping_epoch":127,"log_start":"132'2800","ondisk_log_start":"132'2800","created":127,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T20:00:11.873149+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T20:00:11.873149+0000","last_clean_scrub_stamp":"2023-06-03T20:00:11.873149+0000","objects_scrubbed":0,"log_size":10090,"log_dups_size":0,"ondisk_log_size":10090,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":76546048,"num_objects":1168,"num_object_clones":0,"num_object_copies":3504,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1168,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12890,"num_write_kb":449856,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.d","version":"60'1","reported_seq":97,"reported_epoch":143,"state":"active+clean","last_fresh":"2023-06-03T20:03:50.851132+0000","last_change":"2023-06-03T19:34:16.806559+0000","last_active":"2023-06-03T20:03:50.851132+0000","last_peered":"2023-06-03T20:03:50.851132+0000","last_clean":"2023-06-03T20:03:50.851132+0000","last_became_active":"2023-06-03T19:34:16.806149+0000","last_became_peered":"2023-06-03T19:34:16.806149+0000","last_unstale":"2023-06-03T20:03:50.851132+0000","last_undegraded":"2023-06-03T20:03:50.851132+0000","last_fullsized":"2023-06-03T20:03:50.851132+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":110,"reported_epoch":143,"state":"active+clean","last_fresh":"2023-06-03T20:03:50.833147+0000","last_change":"2023-06-03T19:51:59.360650+0000","last_active":"2023-06-03T20:03:50.833147+0000","last_peered":"2023-06-03T20:03:50.833147+0000","last_clean":"2023-06-03T20:03:50.833147+0000","last_became_active":"2023-06-03T19:51:59.360237+0000","last_became_peered":"2023-06-03T19:51:59.360237+0000","last_unstale":"2023-06-03T20:03:50.833147+0000","last_undegraded":"2023-06-03T20:03:50.833147+0000","last_fullsized":"2023-06-03T20:03:50.833147+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"9.6","version":"144'12417","reported_seq":12441,"reported_epoch":144,"state":"active+clean","last_fresh":"2023-06-03T20:04:18.335682+0000","last_change":"2023-06-03T20:00:13.172391+0000","last_active":"2023-06-03T20:04:18.335682+0000","last_peered":"2023-06-03T20:04:18.335682+0000","last_clean":"2023-06-03T20:04:18.335682+0000","last_became_active":"2023-06-03T20:00:13.171568+0000","last_became_peered":"2023-06-03T20:00:13.171568+0000","last_unstale":"2023-06-03T20:04:18.335682+0000","last_undegraded":"2023-06-03T20:04:18.335682+0000","last_fullsized":"2023-06-03T20:04:18.335682+0000","mapping_epoch":127,"log_start":"131'2400","ondisk_log_start":"131'2400","created":127,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T20:00:11.873149+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T20:00:11.873149+0000","last_clean_scrub_stamp":"2023-06-03T20:00:11.873149+0000","objects_scrubbed":0,"log_size":10017,"log_dups_size":0,"ondisk_log_size":10017,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":75825152,"num_objects":1157,"num_object_clones":0,"num_object_copies":3471,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1157,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12417,"num_write_kb":434368,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,1],"acting":[5,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":111,"reported_epoch":143,"state":"active+clean","last_fresh":"2023-06-03T20:03:50.842917+0000","last_change":"2023-06-03T19:56:54.507432+0000","last_active":"2023-06-03T20:03:50.842917+0000","last_peered":"2023-06-03T20:03:50.842917+0000","last_clean":"2023-06-03T20:03:50.842917+0000","last_became_active":"2023-06-03T19:56:54.507012+0000","last_became_peered":"2023-06-03T19:56:54.507012+0000","last_unstale":"2023-06-03T20:03:50.842917+0000","last_undegraded":"2023-06-03T20:03:50.842917+0000","last_fullsized":"2023-06-03T20:03:50.842917+0000","mapping_epoch":119,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":128,"reported_epoch":143,"state":"active+clean","last_fresh":"2023-06-03T20:03:50.857440+0000","last_change":"2023-06-03T19:57:28.464153+0000","last_active":"2023-06-03T20:03:50.857440+0000","last_peered":"2023-06-03T20:03:50.857440+0000","last_clean":"2023-06-03T20:03:50.857440+0000","last_became_active":"2023-06-03T19:57:28.463502+0000","last_became_peered":"2023-06-03T19:57:28.463502+0000","last_unstale":"2023-06-03T20:03:50.857440+0000","last_undegraded":"2023-06-03T20:03:50.857440+0000","last_fullsized":"2023-06-03T20:03:50.857440+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T06:11:00.999526+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,4],"acting":[0,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.1","version":"144'12659","reported_seq":12686,"reported_epoch":144,"state":"active+clean","last_fresh":"2023-06-03T20:04:18.092993+0000","last_change":"2023-06-03T20:00:12.905059+0000","last_active":"2023-06-03T20:04:18.092993+0000","last_peered":"2023-06-03T20:04:18.092993+0000","last_clean":"2023-06-03T20:04:18.092993+0000","last_became_active":"2023-06-03T20:00:12.904765+0000","last_became_peered":"2023-06-03T20:00:12.904765+0000","last_unstale":"2023-06-03T20:04:18.092993+0000","last_undegraded":"2023-06-03T20:04:18.092993+0000","last_fullsized":"2023-06-03T20:04:18.092993+0000","mapping_epoch":127,"log_start":"133'4900","ondisk_log_start":"133'4900","created":127,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T20:00:11.873149+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T20:00:11.873149+0000","last_clean_scrub_stamp":"2023-06-03T20:00:11.873149+0000","objects_scrubbed":0,"log_size":7759,"log_dups_size":0,"ondisk_log_size":7759,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T07:39:39.205403+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":75300864,"num_objects":1149,"num_object_clones":0,"num_object_copies":3447,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1149,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12659,"num_write_kb":441856,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,7],"acting":[3,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.b","version":"60'2","reported_seq":111,"reported_epoch":143,"state":"active+clean","last_fresh":"2023-06-03T20:03:50.849116+0000","last_change":"2023-06-03T19:57:28.470310+0000","last_active":"2023-06-03T20:03:50.849116+0000","last_peered":"2023-06-03T20:03:50.849116+0000","last_clean":"2023-06-03T20:03:50.849116+0000","last_became_active":"2023-06-03T19:57:28.469887+0000","last_became_peered":"2023-06-03T19:57:28.469887+0000","last_unstale":"2023-06-03T20:03:50.849116+0000","last_undegraded":"2023-06-03T20:03:50.849116+0000","last_fullsized":"2023-06-03T20:03:50.849116+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T21:58:53.387131+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":119,"reported_epoch":144,"state":"active+clean","last_fresh":"2023-06-03T20:04:10.628678+0000","last_change":"2023-06-03T19:57:28.465140+0000","last_active":"2023-06-03T20:04:10.628678+0000","last_peered":"2023-06-03T20:04:10.628678+0000","last_clean":"2023-06-03T20:04:10.628678+0000","last_became_active":"2023-06-03T19:57:28.464629+0000","last_became_peered":"2023-06-03T19:57:28.464629+0000","last_unstale":"2023-06-03T20:04:10.628678+0000","last_undegraded":"2023-06-03T20:04:10.628678+0000","last_fullsized":"2023-06-03T20:04:10.628678+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T21:47:11.619583+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.0","version":"144'12501","reported_seq":12529,"reported_epoch":144,"state":"active+clean","last_fresh":"2023-06-03T20:04:18.639538+0000","last_change":"2023-06-03T20:00:13.174012+0000","last_active":"2023-06-03T20:04:18.639538+0000","last_peered":"2023-06-03T20:04:18.639538+0000","last_clean":"2023-06-03T20:04:18.639538+0000","last_became_active":"2023-06-03T20:00:13.173630+0000","last_became_peered":"2023-06-03T20:00:13.173630+0000","last_unstale":"2023-06-03T20:04:18.639538+0000","last_undegraded":"2023-06-03T20:04:18.639538+0000","last_fullsized":"2023-06-03T20:04:18.639538+0000","mapping_epoch":127,"log_start":"134'5100","ondisk_log_start":"134'5100","created":127,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T20:00:11.873149+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T20:00:11.873149+0000","last_clean_scrub_stamp":"2023-06-03T20:00:11.873149+0000","objects_scrubbed":0,"log_size":7401,"log_dups_size":0,"ondisk_log_size":7401,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T23:14:07.402519+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":75694080,"num_objects":1155,"num_object_clones":0,"num_object_copies":3465,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1155,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12501,"num_write_kb":436992,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":126,"reported_epoch":143,"state":"active+clean","last_fresh":"2023-06-03T20:03:50.832966+0000","last_change":"2023-06-03T19:57:28.461401+0000","last_active":"2023-06-03T20:03:50.832966+0000","last_peered":"2023-06-03T20:03:50.832966+0000","last_clean":"2023-06-03T20:03:50.832966+0000","last_became_active":"2023-06-03T19:57:28.461057+0000","last_became_peered":"2023-06-03T19:57:28.461057+0000","last_unstale":"2023-06-03T20:03:50.832966+0000","last_undegraded":"2023-06-03T20:03:50.832966+0000","last_fullsized":"2023-06-03T20:03:50.832966+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":124,"reported_epoch":144,"state":"active+clean","last_fresh":"2023-06-03T20:04:10.628350+0000","last_change":"2023-06-03T19:57:28.456750+0000","last_active":"2023-06-03T20:04:10.628350+0000","last_peered":"2023-06-03T20:04:10.628350+0000","last_clean":"2023-06-03T20:04:10.628350+0000","last_became_active":"2023-06-03T19:57:28.456436+0000","last_became_peered":"2023-06-03T19:57:28.456436+0000","last_unstale":"2023-06-03T20:04:10.628350+0000","last_undegraded":"2023-06-03T20:04:10.628350+0000","last_fullsized":"2023-06-03T20:04:10.628350+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T01:49:25.481554+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,5],"acting":[4,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.3","version":"144'12435","reported_seq":12460,"reported_epoch":144,"state":"active+clean","last_fresh":"2023-06-03T20:04:18.297617+0000","last_change":"2023-06-03T20:00:12.903729+0000","last_active":"2023-06-03T20:04:18.297617+0000","last_peered":"2023-06-03T20:04:18.297617+0000","last_clean":"2023-06-03T20:04:18.297617+0000","last_became_active":"2023-06-03T20:00:12.903378+0000","last_became_peered":"2023-06-03T20:00:12.903378+0000","last_unstale":"2023-06-03T20:04:18.297617+0000","last_undegraded":"2023-06-03T20:04:18.297617+0000","last_fullsized":"2023-06-03T20:04:18.297617+0000","mapping_epoch":127,"log_start":"133'4700","ondisk_log_start":"133'4700","created":127,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T20:00:11.873149+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T20:00:11.873149+0000","last_clean_scrub_stamp":"2023-06-03T20:00:11.873149+0000","objects_scrubbed":0,"log_size":7735,"log_dups_size":0,"ondisk_log_size":7735,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":78577664,"num_objects":1199,"num_object_clones":0,"num_object_copies":3597,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1199,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12435,"num_write_kb":436288,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,1],"acting":[7,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":116,"reported_epoch":143,"state":"active+clean","last_fresh":"2023-06-03T20:03:56.525718+0000","last_change":"2023-06-03T19:57:28.462344+0000","last_active":"2023-06-03T20:03:56.525718+0000","last_peered":"2023-06-03T20:03:56.525718+0000","last_clean":"2023-06-03T20:03:56.525718+0000","last_became_active":"2023-06-03T19:57:28.462001+0000","last_became_peered":"2023-06-03T19:57:28.462001+0000","last_unstale":"2023-06-03T20:03:56.525718+0000","last_undegraded":"2023-06-03T20:03:56.525718+0000","last_fullsized":"2023-06-03T20:03:56.525718+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T00:45:13.789650+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":105,"reported_epoch":144,"state":"active+clean","last_fresh":"2023-06-03T20:04:10.629598+0000","last_change":"2023-06-03T19:57:28.467917+0000","last_active":"2023-06-03T20:04:10.629598+0000","last_peered":"2023-06-03T20:04:10.629598+0000","last_clean":"2023-06-03T20:04:10.629598+0000","last_became_active":"2023-06-03T19:57:28.467617+0000","last_became_peered":"2023-06-03T19:57:28.467617+0000","last_unstale":"2023-06-03T20:04:10.629598+0000","last_undegraded":"2023-06-03T20:04:10.629598+0000","last_fullsized":"2023-06-03T20:04:10.629598+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T23:34:23.840582+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.2","version":"144'12594","reported_seq":12621,"reported_epoch":144,"state":"active+clean","last_fresh":"2023-06-03T20:04:18.109707+0000","last_change":"2023-06-03T20:00:13.173625+0000","last_active":"2023-06-03T20:04:18.109707+0000","last_peered":"2023-06-03T20:04:18.109707+0000","last_clean":"2023-06-03T20:04:18.109707+0000","last_became_active":"2023-06-03T20:00:13.173348+0000","last_became_peered":"2023-06-03T20:00:13.173348+0000","last_unstale":"2023-06-03T20:04:18.109707+0000","last_undegraded":"2023-06-03T20:04:18.109707+0000","last_fullsized":"2023-06-03T20:04:18.109707+0000","mapping_epoch":127,"log_start":"133'4900","ondisk_log_start":"133'4900","created":127,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T20:00:11.873149+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T20:00:11.873149+0000","last_clean_scrub_stamp":"2023-06-03T20:00:11.873149+0000","objects_scrubbed":0,"log_size":7694,"log_dups_size":0,"ondisk_log_size":7694,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T03:41:17.261687+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":76677120,"num_objects":1170,"num_object_clones":0,"num_object_copies":3510,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1170,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12594,"num_write_kb":440448,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":111,"reported_epoch":143,"state":"active+clean","last_fresh":"2023-06-03T20:03:50.849841+0000","last_change":"2023-06-03T19:56:54.512219+0000","last_active":"2023-06-03T20:03:50.849841+0000","last_peered":"2023-06-03T20:03:50.849841+0000","last_clean":"2023-06-03T20:03:50.849841+0000","last_became_active":"2023-06-03T19:56:54.511771+0000","last_became_peered":"2023-06-03T19:56:54.511771+0000","last_unstale":"2023-06-03T20:03:50.849841+0000","last_undegraded":"2023-06-03T20:03:50.849841+0000","last_fullsized":"2023-06-03T20:03:50.849841+0000","mapping_epoch":119,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T19:35:47.796520+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":97,"reported_epoch":143,"state":"active+clean","last_fresh":"2023-06-03T20:03:50.845001+0000","last_change":"2023-06-03T19:57:28.471023+0000","last_active":"2023-06-03T20:03:50.845001+0000","last_peered":"2023-06-03T20:03:50.845001+0000","last_clean":"2023-06-03T20:03:50.845001+0000","last_became_active":"2023-06-03T19:57:28.470452+0000","last_became_peered":"2023-06-03T19:57:28.470452+0000","last_unstale":"2023-06-03T20:03:50.845001+0000","last_undegraded":"2023-06-03T20:03:50.845001+0000","last_fullsized":"2023-06-03T20:03:50.845001+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T05:58:33.305030+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.f","version":"144'12386","reported_seq":12411,"reported_epoch":144,"state":"active+clean","last_fresh":"2023-06-03T20:04:18.358312+0000","last_change":"2023-06-03T20:00:12.909234+0000","last_active":"2023-06-03T20:04:18.358312+0000","last_peered":"2023-06-03T20:04:18.358312+0000","last_clean":"2023-06-03T20:04:18.358312+0000","last_became_active":"2023-06-03T20:00:12.908871+0000","last_became_peered":"2023-06-03T20:00:12.908871+0000","last_unstale":"2023-06-03T20:04:18.358312+0000","last_undegraded":"2023-06-03T20:04:18.358312+0000","last_fullsized":"2023-06-03T20:04:18.358312+0000","mapping_epoch":127,"log_start":"131'2300","ondisk_log_start":"131'2300","created":127,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T20:00:11.873149+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T20:00:11.873149+0000","last_clean_scrub_stamp":"2023-06-03T20:00:11.873149+0000","objects_scrubbed":0,"log_size":10086,"log_dups_size":0,"ondisk_log_size":10086,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":75235328,"num_objects":1148,"num_object_clones":0,"num_object_copies":3444,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1148,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12386,"num_write_kb":433088,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":115,"reported_epoch":143,"state":"active+clean","last_fresh":"2023-06-03T20:03:50.846056+0000","last_change":"2023-06-03T19:51:59.392796+0000","last_active":"2023-06-03T20:03:50.846056+0000","last_peered":"2023-06-03T20:03:50.846056+0000","last_clean":"2023-06-03T20:03:50.846056+0000","last_became_active":"2023-06-03T19:51:59.380207+0000","last_became_peered":"2023-06-03T19:51:59.380207+0000","last_unstale":"2023-06-03T20:03:50.846056+0000","last_undegraded":"2023-06-03T20:03:50.846056+0000","last_fullsized":"2023-06-03T20:03:50.846056+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:32:20.692043+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:32:20.692043+0000","last_clean_scrub_stamp":"2023-06-03T19:32:20.692043+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T00:28:05.866873+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":126,"reported_epoch":143,"state":"active+clean","last_fresh":"2023-06-03T20:03:50.849481+0000","last_change":"2023-06-03T19:56:54.512607+0000","last_active":"2023-06-03T20:03:50.849481+0000","last_peered":"2023-06-03T20:03:50.849481+0000","last_clean":"2023-06-03T20:03:50.849481+0000","last_became_active":"2023-06-03T19:56:54.512282+0000","last_became_peered":"2023-06-03T19:56:54.512282+0000","last_unstale":"2023-06-03T20:03:50.849481+0000","last_undegraded":"2023-06-03T20:03:50.849481+0000","last_fullsized":"2023-06-03T20:03:50.849481+0000","mapping_epoch":119,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":82,"reported_epoch":143,"state":"active+clean","last_fresh":"2023-06-03T20:03:50.844870+0000","last_change":"2023-06-03T19:57:28.470295+0000","last_active":"2023-06-03T20:03:50.844870+0000","last_peered":"2023-06-03T20:03:50.844870+0000","last_clean":"2023-06-03T20:03:50.844870+0000","last_became_active":"2023-06-03T19:57:28.469383+0000","last_became_peered":"2023-06-03T19:57:28.469383+0000","last_unstale":"2023-06-03T20:03:50.844870+0000","last_undegraded":"2023-06-03T20:03:50.844870+0000","last_fullsized":"2023-06-03T20:03:50.844870+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T05:35:19.667447+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.9","version":"144'12730","reported_seq":12757,"reported_epoch":144,"state":"active+clean","last_fresh":"2023-06-03T20:04:19.963497+0000","last_change":"2023-06-03T20:00:12.910626+0000","last_active":"2023-06-03T20:04:19.963497+0000","last_peered":"2023-06-03T20:04:19.963497+0000","last_clean":"2023-06-03T20:04:19.963497+0000","last_became_active":"2023-06-03T20:00:12.910285+0000","last_became_peered":"2023-06-03T20:00:12.910285+0000","last_unstale":"2023-06-03T20:04:19.963497+0000","last_undegraded":"2023-06-03T20:04:19.963497+0000","last_fullsized":"2023-06-03T20:04:19.963497+0000","mapping_epoch":127,"log_start":"132'2700","ondisk_log_start":"132'2700","created":127,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T20:00:11.873149+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T20:00:11.873149+0000","last_clean_scrub_stamp":"2023-06-03T20:00:11.873149+0000","objects_scrubbed":0,"log_size":10030,"log_dups_size":0,"ondisk_log_size":10030,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T04:42:34.630759+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":71827456,"num_objects":1096,"num_object_clones":0,"num_object_copies":3288,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1096,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12730,"num_write_kb":442432,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,5],"acting":[0,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":115,"reported_epoch":143,"state":"active+clean","last_fresh":"2023-06-03T20:03:50.842736+0000","last_change":"2023-06-03T19:51:59.373777+0000","last_active":"2023-06-03T20:03:50.842736+0000","last_peered":"2023-06-03T20:03:50.842736+0000","last_clean":"2023-06-03T20:03:50.842736+0000","last_became_active":"2023-06-03T19:51:59.372988+0000","last_became_peered":"2023-06-03T19:51:59.372988+0000","last_unstale":"2023-06-03T20:03:50.842736+0000","last_undegraded":"2023-06-03T20:03:50.842736+0000","last_fullsized":"2023-06-03T20:03:50.842736+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:32:20.692043+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:32:20.692043+0000","last_clean_scrub_stamp":"2023-06-03T19:32:20.692043+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":89,"reported_epoch":143,"state":"active+clean","last_fresh":"2023-06-03T20:03:50.829791+0000","last_change":"2023-06-03T19:57:28.460195+0000","last_active":"2023-06-03T20:03:50.829791+0000","last_peered":"2023-06-03T20:03:50.829791+0000","last_clean":"2023-06-03T20:03:50.829791+0000","last_became_active":"2023-06-03T19:57:28.459823+0000","last_became_peered":"2023-06-03T19:57:28.459823+0000","last_unstale":"2023-06-03T20:03:50.829791+0000","last_undegraded":"2023-06-03T20:03:50.829791+0000","last_fullsized":"2023-06-03T20:03:50.829791+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T20:08:48.686380+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":97,"reported_epoch":143,"state":"active+clean","last_fresh":"2023-06-03T20:03:50.845964+0000","last_change":"2023-06-03T19:56:54.516859+0000","last_active":"2023-06-03T20:03:50.845964+0000","last_peered":"2023-06-03T20:03:50.845964+0000","last_clean":"2023-06-03T20:03:50.845964+0000","last_became_active":"2023-06-03T19:56:54.516521+0000","last_became_peered":"2023-06-03T19:56:54.516521+0000","last_unstale":"2023-06-03T20:03:50.845964+0000","last_undegraded":"2023-06-03T20:03:50.845964+0000","last_fullsized":"2023-06-03T20:03:50.845964+0000","mapping_epoch":119,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T20:03:25.227382+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.a","version":"144'12640","reported_seq":12665,"reported_epoch":144,"state":"active+clean","last_fresh":"2023-06-03T20:04:18.325654+0000","last_change":"2023-06-03T20:00:12.909720+0000","last_active":"2023-06-03T20:04:18.325654+0000","last_peered":"2023-06-03T20:04:18.325654+0000","last_clean":"2023-06-03T20:04:18.325654+0000","last_became_active":"2023-06-03T20:00:12.909428+0000","last_became_peered":"2023-06-03T20:00:12.909428+0000","last_unstale":"2023-06-03T20:04:18.325654+0000","last_undegraded":"2023-06-03T20:04:18.325654+0000","last_fullsized":"2023-06-03T20:04:18.325654+0000","mapping_epoch":127,"log_start":"131'2600","ondisk_log_start":"131'2600","created":127,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T20:00:11.873149+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T20:00:11.873149+0000","last_clean_scrub_stamp":"2023-06-03T20:00:11.873149+0000","objects_scrubbed":0,"log_size":10040,"log_dups_size":0,"ondisk_log_size":10040,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":74186752,"num_objects":1132,"num_object_clones":0,"num_object_copies":3396,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1132,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12640,"num_write_kb":440704,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,7],"acting":[5,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":118,"reported_epoch":143,"state":"active+clean","last_fresh":"2023-06-03T20:03:50.857087+0000","last_change":"2023-06-03T19:56:54.504859+0000","last_active":"2023-06-03T20:03:50.857087+0000","last_peered":"2023-06-03T20:03:50.857087+0000","last_clean":"2023-06-03T20:03:50.857087+0000","last_became_active":"2023-06-03T19:56:54.504455+0000","last_became_peered":"2023-06-03T19:56:54.504455+0000","last_unstale":"2023-06-03T20:03:50.857087+0000","last_undegraded":"2023-06-03T20:03:50.857087+0000","last_fullsized":"2023-06-03T20:03:50.857087+0000","mapping_epoch":119,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:32:20.692043+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:32:20.692043+0000","last_clean_scrub_stamp":"2023-06-03T19:32:20.692043+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T06:25:45.522100+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":124,"reported_epoch":143,"state":"active+clean","last_fresh":"2023-06-03T20:03:58.607548+0000","last_change":"2023-06-03T19:56:11.754048+0000","last_active":"2023-06-03T20:03:58.607548+0000","last_peered":"2023-06-03T20:03:58.607548+0000","last_clean":"2023-06-03T20:03:58.607548+0000","last_became_active":"2023-06-03T19:56:11.753687+0000","last_became_peered":"2023-06-03T19:56:11.753687+0000","last_unstale":"2023-06-03T20:03:58.607548+0000","last_undegraded":"2023-06-03T20:03:58.607548+0000","last_fullsized":"2023-06-03T20:03:58.607548+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":115,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T23:29:41.511496+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":110,"reported_epoch":143,"state":"active+clean","last_fresh":"2023-06-03T20:03:50.845194+0000","last_change":"2023-06-03T19:51:59.396573+0000","last_active":"2023-06-03T20:03:50.845194+0000","last_peered":"2023-06-03T20:03:50.845194+0000","last_clean":"2023-06-03T20:03:50.845194+0000","last_became_active":"2023-06-03T19:51:59.381593+0000","last_became_peered":"2023-06-03T19:51:59.381593+0000","last_unstale":"2023-06-03T20:03:50.845194+0000","last_undegraded":"2023-06-03T20:03:50.845194+0000","last_fullsized":"2023-06-03T20:03:50.845194+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T21:33:23.043772+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.8","version":"144'12560","reported_seq":12587,"reported_epoch":144,"state":"active+clean","last_fresh":"2023-06-03T20:04:18.118138+0000","last_change":"2023-06-03T20:00:12.908817+0000","last_active":"2023-06-03T20:04:18.118138+0000","last_peered":"2023-06-03T20:04:18.118138+0000","last_clean":"2023-06-03T20:04:18.118138+0000","last_became_active":"2023-06-03T20:00:12.908431+0000","last_became_peered":"2023-06-03T20:00:12.908431+0000","last_unstale":"2023-06-03T20:04:18.118138+0000","last_undegraded":"2023-06-03T20:04:18.118138+0000","last_fullsized":"2023-06-03T20:04:18.118138+0000","mapping_epoch":127,"log_start":"133'4800","ondisk_log_start":"133'4800","created":127,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T20:00:11.873149+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T20:00:11.873149+0000","last_clean_scrub_stamp":"2023-06-03T20:00:11.873149+0000","objects_scrubbed":0,"log_size":7760,"log_dups_size":0,"ondisk_log_size":7760,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T21:11:15.734132+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":75759616,"num_objects":1156,"num_object_clones":0,"num_object_copies":3468,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1156,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12560,"num_write_kb":438912,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":115,"reported_epoch":143,"state":"active+clean","last_fresh":"2023-06-03T20:03:50.843094+0000","last_change":"2023-06-03T19:56:11.752151+0000","last_active":"2023-06-03T20:03:50.843094+0000","last_peered":"2023-06-03T20:03:50.843094+0000","last_clean":"2023-06-03T20:03:50.843094+0000","last_became_active":"2023-06-03T19:56:11.751767+0000","last_became_peered":"2023-06-03T19:56:11.751767+0000","last_unstale":"2023-06-03T20:03:50.843094+0000","last_undegraded":"2023-06-03T20:03:50.843094+0000","last_fullsized":"2023-06-03T20:03:50.843094+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":115,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:32:20.692043+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:32:20.692043+0000","last_clean_scrub_stamp":"2023-06-03T19:32:20.692043+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"72'139","reported_seq":347,"reported_epoch":143,"state":"active+clean","last_fresh":"2023-06-03T20:03:50.849310+0000","last_change":"2023-06-03T19:47:06.731872+0000","last_active":"2023-06-03T20:03:50.849310+0000","last_peered":"2023-06-03T20:03:50.849310+0000","last_clean":"2023-06-03T20:03:50.849310+0000","last_became_active":"2023-06-03T19:47:06.731175+0000","last_became_peered":"2023-06-03T19:47:06.731175+0000","last_unstale":"2023-06-03T20:03:50.849310+0000","last_undegraded":"2023-06-03T20:03:50.849310+0000","last_fullsized":"2023-06-03T20:03:50.849310+0000","mapping_epoch":86,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":87,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:27:28.100808+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:27:28.100808+0000","last_clean_scrub_stamp":"2023-06-03T19:27:28.100808+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":91,"reported_epoch":143,"state":"active+clean","last_fresh":"2023-06-03T20:03:50.848951+0000","last_change":"2023-06-03T19:56:54.509571+0000","last_active":"2023-06-03T20:03:50.848951+0000","last_peered":"2023-06-03T20:03:50.848951+0000","last_clean":"2023-06-03T20:03:50.848951+0000","last_became_active":"2023-06-03T19:56:54.509213+0000","last_became_peered":"2023-06-03T19:56:54.509213+0000","last_unstale":"2023-06-03T20:03:50.848951+0000","last_undegraded":"2023-06-03T20:03:50.848951+0000","last_fullsized":"2023-06-03T20:03:50.848951+0000","mapping_epoch":119,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T21:26:56.003203+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":110,"reported_epoch":143,"state":"active+clean","last_fresh":"2023-06-03T20:03:50.833649+0000","last_change":"2023-06-03T19:57:28.459664+0000","last_active":"2023-06-03T20:03:50.833649+0000","last_peered":"2023-06-03T20:03:50.833649+0000","last_clean":"2023-06-03T20:03:50.833649+0000","last_became_active":"2023-06-03T19:57:28.459289+0000","last_became_peered":"2023-06-03T19:57:28.459289+0000","last_unstale":"2023-06-03T20:03:50.833649+0000","last_undegraded":"2023-06-03T20:03:50.833649+0000","last_fullsized":"2023-06-03T20:03:50.833649+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"9.b","version":"144'12940","reported_seq":12967,"reported_epoch":144,"state":"active+clean","last_fresh":"2023-06-03T20:04:19.958817+0000","last_change":"2023-06-03T20:00:12.908303+0000","last_active":"2023-06-03T20:04:19.958817+0000","last_peered":"2023-06-03T20:04:19.958817+0000","last_clean":"2023-06-03T20:04:19.958817+0000","last_became_active":"2023-06-03T20:00:12.907799+0000","last_became_peered":"2023-06-03T20:00:12.907799+0000","last_unstale":"2023-06-03T20:04:19.958817+0000","last_undegraded":"2023-06-03T20:04:19.958817+0000","last_fullsized":"2023-06-03T20:04:19.958817+0000","mapping_epoch":127,"log_start":"132'2900","ondisk_log_start":"132'2900","created":127,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T20:00:11.873149+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T20:00:11.873149+0000","last_clean_scrub_stamp":"2023-06-03T20:00:11.873149+0000","objects_scrubbed":0,"log_size":10040,"log_dups_size":0,"ondisk_log_size":10040,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T23:11:45.966674+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":69468160,"num_objects":1060,"num_object_clones":0,"num_object_copies":3180,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1060,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12940,"num_write_kb":448000,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,5],"acting":[0,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":130,"reported_epoch":143,"state":"active+clean","last_fresh":"2023-06-03T20:03:50.849206+0000","last_change":"2023-06-03T19:51:59.358157+0000","last_active":"2023-06-03T20:03:50.849206+0000","last_peered":"2023-06-03T20:03:50.849206+0000","last_clean":"2023-06-03T20:03:50.849206+0000","last_became_active":"2023-06-03T19:51:59.357363+0000","last_became_peered":"2023-06-03T19:51:59.357363+0000","last_unstale":"2023-06-03T20:03:50.849206+0000","last_undegraded":"2023-06-03T20:03:50.849206+0000","last_fullsized":"2023-06-03T20:03:50.849206+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:32:20.692043+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:32:20.692043+0000","last_clean_scrub_stamp":"2023-06-03T19:32:20.692043+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":116,"reported_epoch":143,"state":"active+clean","last_fresh":"2023-06-03T20:03:50.857185+0000","last_change":"2023-06-03T19:57:28.459611+0000","last_active":"2023-06-03T20:03:50.857185+0000","last_peered":"2023-06-03T20:03:50.857185+0000","last_clean":"2023-06-03T20:03:50.857185+0000","last_became_active":"2023-06-03T19:57:28.459264+0000","last_became_peered":"2023-06-03T19:57:28.459264+0000","last_unstale":"2023-06-03T20:03:50.857185+0000","last_undegraded":"2023-06-03T20:03:50.857185+0000","last_fullsized":"2023-06-03T20:03:50.857185+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T22:15:06.557231+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":116,"reported_epoch":143,"state":"active+clean","last_fresh":"2023-06-03T20:03:50.857214+0000","last_change":"2023-06-03T19:56:11.745678+0000","last_active":"2023-06-03T20:03:50.857214+0000","last_peered":"2023-06-03T20:03:50.857214+0000","last_clean":"2023-06-03T20:03:50.857214+0000","last_became_active":"2023-06-03T19:56:11.744966+0000","last_became_peered":"2023-06-03T19:56:11.744966+0000","last_unstale":"2023-06-03T20:03:50.857214+0000","last_undegraded":"2023-06-03T20:03:50.857214+0000","last_fullsized":"2023-06-03T20:03:50.857214+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":115,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T21:12:06.830295+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.e","version":"144'12582","reported_seq":12607,"reported_epoch":144,"state":"active+clean","last_fresh":"2023-06-03T20:04:18.330516+0000","last_change":"2023-06-03T20:00:12.908248+0000","last_active":"2023-06-03T20:04:18.330516+0000","last_peered":"2023-06-03T20:04:18.330516+0000","last_clean":"2023-06-03T20:04:18.330516+0000","last_became_active":"2023-06-03T20:00:12.907900+0000","last_became_peered":"2023-06-03T20:00:12.907900+0000","last_unstale":"2023-06-03T20:04:18.330516+0000","last_undegraded":"2023-06-03T20:04:18.330516+0000","last_fullsized":"2023-06-03T20:04:18.330516+0000","mapping_epoch":127,"log_start":"133'4800","ondisk_log_start":"133'4800","created":127,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T20:00:11.873149+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T20:00:11.873149+0000","last_clean_scrub_stamp":"2023-06-03T20:00:11.873149+0000","objects_scrubbed":0,"log_size":7782,"log_dups_size":0,"ondisk_log_size":7782,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":71958528,"num_objects":1098,"num_object_clones":0,"num_object_copies":3294,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1098,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12582,"num_write_kb":437760,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":130,"reported_epoch":143,"state":"active+clean","last_fresh":"2023-06-03T20:03:50.851203+0000","last_change":"2023-06-03T19:57:28.466096+0000","last_active":"2023-06-03T20:03:50.851203+0000","last_peered":"2023-06-03T20:03:50.851203+0000","last_clean":"2023-06-03T20:03:50.851203+0000","last_became_active":"2023-06-03T19:57:28.464954+0000","last_became_peered":"2023-06-03T19:57:28.464954+0000","last_unstale":"2023-06-03T20:03:50.851203+0000","last_undegraded":"2023-06-03T20:03:50.851203+0000","last_fullsized":"2023-06-03T20:03:50.851203+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:32:20.692043+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:32:20.692043+0000","last_clean_scrub_stamp":"2023-06-03T19:32:20.692043+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":123,"reported_epoch":143,"state":"active+clean","last_fresh":"2023-06-03T20:03:50.845872+0000","last_change":"2023-06-03T19:56:11.750241+0000","last_active":"2023-06-03T20:03:50.845872+0000","last_peered":"2023-06-03T20:03:50.845872+0000","last_clean":"2023-06-03T20:03:50.845872+0000","last_became_active":"2023-06-03T19:56:11.749687+0000","last_became_peered":"2023-06-03T19:56:11.749687+0000","last_unstale":"2023-06-03T20:03:50.845872+0000","last_undegraded":"2023-06-03T20:03:50.845872+0000","last_fullsized":"2023-06-03T20:03:50.845872+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":115,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T22:38:25.819929+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":87,"reported_epoch":143,"state":"active+clean","last_fresh":"2023-06-03T20:03:50.857305+0000","last_change":"2023-06-03T19:47:06.733532+0000","last_active":"2023-06-03T20:03:50.857305+0000","last_peered":"2023-06-03T20:03:50.857305+0000","last_clean":"2023-06-03T20:03:50.857305+0000","last_became_active":"2023-06-03T19:47:06.733131+0000","last_became_peered":"2023-06-03T19:47:06.733131+0000","last_unstale":"2023-06-03T20:03:50.857305+0000","last_undegraded":"2023-06-03T20:03:50.857305+0000","last_fullsized":"2023-06-03T20:03:50.857305+0000","mapping_epoch":86,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":87,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T23:46:03.788646+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.d","version":"144'12904","reported_seq":12930,"reported_epoch":144,"state":"active+clean","last_fresh":"2023-06-03T20:04:18.301960+0000","last_change":"2023-06-03T20:00:12.905723+0000","last_active":"2023-06-03T20:04:18.301960+0000","last_peered":"2023-06-03T20:04:18.301960+0000","last_clean":"2023-06-03T20:04:18.301960+0000","last_became_active":"2023-06-03T20:00:12.905437+0000","last_became_peered":"2023-06-03T20:00:12.905437+0000","last_unstale":"2023-06-03T20:04:18.301960+0000","last_undegraded":"2023-06-03T20:04:18.301960+0000","last_fullsized":"2023-06-03T20:04:18.301960+0000","mapping_epoch":127,"log_start":"133'5200","ondisk_log_start":"133'5200","created":127,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T20:00:11.873149+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T20:00:11.873149+0000","last_clean_scrub_stamp":"2023-06-03T20:00:11.873149+0000","objects_scrubbed":0,"log_size":7704,"log_dups_size":0,"ondisk_log_size":7704,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":75825176,"num_objects":1158,"num_object_clones":0,"num_object_copies":3474,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1158,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":12904,"num_write_kb":449921,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,5],"acting":[7,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.7","version":"60'2","reported_seq":96,"reported_epoch":143,"state":"active+clean","last_fresh":"2023-06-03T20:03:50.858001+0000","last_change":"2023-06-03T19:56:54.510148+0000","last_active":"2023-06-03T20:03:50.858001+0000","last_peered":"2023-06-03T20:03:50.858001+0000","last_clean":"2023-06-03T20:03:50.858001+0000","last_became_active":"2023-06-03T19:56:54.509771+0000","last_became_peered":"2023-06-03T19:56:54.509771+0000","last_unstale":"2023-06-03T20:03:50.858001+0000","last_undegraded":"2023-06-03T20:03:50.858001+0000","last_fullsized":"2023-06-03T20:03:50.858001+0000","mapping_epoch":119,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T00:52:42.807432+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":123,"reported_epoch":143,"state":"active+clean","last_fresh":"2023-06-03T20:03:50.845641+0000","last_change":"2023-06-03T19:57:28.470954+0000","last_active":"2023-06-03T20:03:50.845641+0000","last_peered":"2023-06-03T20:03:50.845641+0000","last_clean":"2023-06-03T20:03:50.845641+0000","last_became_active":"2023-06-03T19:57:28.470524+0000","last_became_peered":"2023-06-03T19:57:28.470524+0000","last_unstale":"2023-06-03T20:03:50.845641+0000","last_undegraded":"2023-06-03T20:03:50.845641+0000","last_fullsized":"2023-06-03T20:03:50.845641+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:32:20.692043+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:32:20.692043+0000","last_clean_scrub_stamp":"2023-06-03T19:32:20.692043+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T19:39:19.983271+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":93,"reported_epoch":143,"state":"active+clean","last_fresh":"2023-06-03T20:03:50.849817+0000","last_change":"2023-06-03T19:56:54.507670+0000","last_active":"2023-06-03T20:03:50.849817+0000","last_peered":"2023-06-03T20:03:50.849817+0000","last_clean":"2023-06-03T20:03:50.849817+0000","last_became_active":"2023-06-03T19:56:54.507293+0000","last_became_peered":"2023-06-03T19:56:54.507293+0000","last_unstale":"2023-06-03T20:03:50.849817+0000","last_undegraded":"2023-06-03T20:03:50.849817+0000","last_fullsized":"2023-06-03T20:03:50.849817+0000","mapping_epoch":119,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T01:49:19.966054+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"9.c","version":"144'12575","reported_seq":12602,"reported_epoch":144,"state":"active+clean","last_fresh":"2023-06-03T20:04:18.105043+0000","last_change":"2023-06-03T20:00:12.904758+0000","last_active":"2023-06-03T20:04:18.105043+0000","last_peered":"2023-06-03T20:04:18.105043+0000","last_clean":"2023-06-03T20:04:18.105043+0000","last_became_active":"2023-06-03T20:00:12.904435+0000","last_became_peered":"2023-06-03T20:00:12.904435+0000","last_unstale":"2023-06-03T20:04:18.105043+0000","last_undegraded":"2023-06-03T20:04:18.105043+0000","last_fullsized":"2023-06-03T20:04:18.105043+0000","mapping_epoch":127,"log_start":"133'4800","ondisk_log_start":"133'4800","created":127,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T20:00:11.873149+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T20:00:11.873149+0000","last_clean_scrub_stamp":"2023-06-03T20:00:11.873149+0000","objects_scrubbed":0,"log_size":7775,"log_dups_size":0,"ondisk_log_size":7775,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T04:00:13.693202+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":73596928,"num_objects":1123,"num_object_clones":0,"num_object_copies":3369,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1123,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12575,"num_write_kb":438336,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,7],"acting":[3,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":115,"reported_epoch":143,"state":"active+clean","last_fresh":"2023-06-03T20:03:50.833282+0000","last_change":"2023-06-03T19:56:11.752730+0000","last_active":"2023-06-03T20:03:50.833282+0000","last_peered":"2023-06-03T20:03:50.833282+0000","last_clean":"2023-06-03T20:03:50.833282+0000","last_became_active":"2023-06-03T19:56:11.752441+0000","last_became_peered":"2023-06-03T19:56:11.752441+0000","last_unstale":"2023-06-03T20:03:50.833282+0000","last_undegraded":"2023-06-03T20:03:50.833282+0000","last_fullsized":"2023-06-03T20:03:50.833282+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":115,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:32:20.692043+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:32:20.692043+0000","last_clean_scrub_stamp":"2023-06-03T19:32:20.692043+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"60'1","reported_seq":119,"reported_epoch":143,"state":"active+clean","last_fresh":"2023-06-03T20:03:50.857240+0000","last_change":"2023-06-03T19:57:28.463999+0000","last_active":"2023-06-03T20:03:50.857240+0000","last_peered":"2023-06-03T20:03:50.857240+0000","last_clean":"2023-06-03T20:03:50.857240+0000","last_became_active":"2023-06-03T19:57:28.463588+0000","last_became_peered":"2023-06-03T19:57:28.463588+0000","last_unstale":"2023-06-03T20:03:50.857240+0000","last_undegraded":"2023-06-03T20:03:50.857240+0000","last_fullsized":"2023-06-03T20:03:50.857240+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T21:44:11.969059+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":87,"reported_epoch":143,"state":"active+clean","last_fresh":"2023-06-03T20:03:50.828875+0000","last_change":"2023-06-03T19:57:28.456467+0000","last_active":"2023-06-03T20:03:50.828875+0000","last_peered":"2023-06-03T20:03:50.828875+0000","last_clean":"2023-06-03T20:03:50.828875+0000","last_became_active":"2023-06-03T19:57:28.456120+0000","last_became_peered":"2023-06-03T19:57:28.456120+0000","last_unstale":"2023-06-03T20:03:50.828875+0000","last_undegraded":"2023-06-03T20:03:50.828875+0000","last_fullsized":"2023-06-03T20:03:50.828875+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T07:28:14.306624+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"60'1","reported_seq":103,"reported_epoch":143,"state":"active+clean","last_fresh":"2023-06-03T20:03:50.857525+0000","last_change":"2023-06-03T19:57:28.465124+0000","last_active":"2023-06-03T20:03:50.857525+0000","last_peered":"2023-06-03T20:03:50.857525+0000","last_clean":"2023-06-03T20:03:50.857525+0000","last_became_active":"2023-06-03T19:57:28.464654+0000","last_became_peered":"2023-06-03T19:57:28.464654+0000","last_unstale":"2023-06-03T20:03:50.857525+0000","last_undegraded":"2023-06-03T20:03:50.857525+0000","last_fullsized":"2023-06-03T20:03:50.857525+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T01:12:32.305507+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":130,"reported_epoch":143,"state":"active+clean","last_fresh":"2023-06-03T20:03:50.858027+0000","last_change":"2023-06-03T19:56:54.511760+0000","last_active":"2023-06-03T20:03:50.858027+0000","last_peered":"2023-06-03T20:03:50.858027+0000","last_clean":"2023-06-03T20:03:50.858027+0000","last_became_active":"2023-06-03T19:56:54.511429+0000","last_became_peered":"2023-06-03T19:56:54.511429+0000","last_unstale":"2023-06-03T20:03:50.858027+0000","last_undegraded":"2023-06-03T20:03:50.858027+0000","last_fullsized":"2023-06-03T20:03:50.858027+0000","mapping_epoch":119,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T05:42:46.254343+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"60'2","reported_seq":128,"reported_epoch":143,"state":"active+clean","last_fresh":"2023-06-03T20:03:50.842873+0000","last_change":"2023-06-03T19:57:28.462252+0000","last_active":"2023-06-03T20:03:50.842873+0000","last_peered":"2023-06-03T20:03:50.842873+0000","last_clean":"2023-06-03T20:03:50.842873+0000","last_became_active":"2023-06-03T19:57:28.461844+0000","last_became_peered":"2023-06-03T19:57:28.461844+0000","last_unstale":"2023-06-03T20:03:50.842873+0000","last_undegraded":"2023-06-03T20:03:50.842873+0000","last_fullsized":"2023-06-03T20:03:50.842873+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":133,"reported_epoch":143,"state":"active+clean","last_fresh":"2023-06-03T20:03:50.829624+0000","last_change":"2023-06-03T19:57:28.469248+0000","last_active":"2023-06-03T20:03:50.829624+0000","last_peered":"2023-06-03T20:03:50.829624+0000","last_clean":"2023-06-03T20:03:50.829624+0000","last_became_active":"2023-06-03T19:57:28.468903+0000","last_became_peered":"2023-06-03T19:57:28.468903+0000","last_unstale":"2023-06-03T20:03:50.829624+0000","last_undegraded":"2023-06-03T20:03:50.829624+0000","last_fullsized":"2023-06-03T20:03:50.829624+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T06:08:20.956098+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"60'2","reported_seq":113,"reported_epoch":143,"state":"active+clean","last_fresh":"2023-06-03T20:03:50.849289+0000","last_change":"2023-06-03T19:57:28.462610+0000","last_active":"2023-06-03T20:03:50.849289+0000","last_peered":"2023-06-03T20:03:50.849289+0000","last_clean":"2023-06-03T20:03:50.849289+0000","last_became_active":"2023-06-03T19:57:28.460655+0000","last_became_peered":"2023-06-03T19:57:28.460655+0000","last_unstale":"2023-06-03T20:03:50.849289+0000","last_undegraded":"2023-06-03T20:03:50.849289+0000","last_fullsized":"2023-06-03T20:03:50.849289+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":133,"reported_epoch":143,"state":"active+clean","last_fresh":"2023-06-03T20:03:50.829882+0000","last_change":"2023-06-03T19:57:28.464690+0000","last_active":"2023-06-03T20:03:50.829882+0000","last_peered":"2023-06-03T20:03:50.829882+0000","last_clean":"2023-06-03T20:03:50.829882+0000","last_became_active":"2023-06-03T19:57:28.463856+0000","last_became_peered":"2023-06-03T19:57:28.463856+0000","last_unstale":"2023-06-03T20:03:50.829882+0000","last_undegraded":"2023-06-03T20:03:50.829882+0000","last_fullsized":"2023-06-03T20:03:50.829882+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T00:11:00.991088+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"60'1","reported_seq":122,"reported_epoch":143,"state":"active+clean","last_fresh":"2023-06-03T20:03:50.849135+0000","last_change":"2023-06-03T19:56:54.505096+0000","last_active":"2023-06-03T20:03:50.849135+0000","last_peered":"2023-06-03T20:03:50.849135+0000","last_clean":"2023-06-03T20:03:50.849135+0000","last_became_active":"2023-06-03T19:56:54.504255+0000","last_became_peered":"2023-06-03T19:56:54.504255+0000","last_unstale":"2023-06-03T20:03:50.849135+0000","last_undegraded":"2023-06-03T20:03:50.849135+0000","last_fullsized":"2023-06-03T20:03:50.849135+0000","mapping_epoch":119,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T19:52:25.922308+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":135,"reported_epoch":143,"state":"active+clean","last_fresh":"2023-06-03T20:04:08.101039+0000","last_change":"2023-06-03T19:56:54.503416+0000","last_active":"2023-06-03T20:04:08.101039+0000","last_peered":"2023-06-03T20:04:08.101039+0000","last_clean":"2023-06-03T20:04:08.101039+0000","last_became_active":"2023-06-03T19:56:54.503018+0000","last_became_peered":"2023-06-03T19:56:54.503018+0000","last_unstale":"2023-06-03T20:04:08.101039+0000","last_undegraded":"2023-06-03T20:04:08.101039+0000","last_fullsized":"2023-06-03T20:04:08.101039+0000","mapping_epoch":119,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T20:14:00.330616+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"60'4","reported_seq":105,"reported_epoch":143,"state":"active+clean","last_fresh":"2023-06-03T20:03:50.857222+0000","last_change":"2023-06-03T19:56:11.748116+0000","last_active":"2023-06-03T20:03:50.857222+0000","last_peered":"2023-06-03T20:03:50.857222+0000","last_clean":"2023-06-03T20:03:50.857222+0000","last_became_active":"2023-06-03T19:56:11.747636+0000","last_became_peered":"2023-06-03T19:56:11.747636+0000","last_unstale":"2023-06-03T20:03:50.857222+0000","last_undegraded":"2023-06-03T20:03:50.857222+0000","last_fullsized":"2023-06-03T20:03:50.857222+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":115,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T06:57:48.052055+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":104,"reported_epoch":143,"state":"active+clean","last_fresh":"2023-06-03T20:03:50.828529+0000","last_change":"2023-06-03T19:57:28.459234+0000","last_active":"2023-06-03T20:03:50.828529+0000","last_peered":"2023-06-03T20:03:50.828529+0000","last_clean":"2023-06-03T20:03:50.828529+0000","last_became_active":"2023-06-03T19:57:28.458896+0000","last_became_peered":"2023-06-03T19:57:28.458896+0000","last_unstale":"2023-06-03T20:03:50.828529+0000","last_undegraded":"2023-06-03T20:03:50.828529+0000","last_fullsized":"2023-06-03T20:03:50.828529+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T21:18:37.142280+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5486,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":62,"num_write_kb":30,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16458,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":30319,"internal_metadata":0},"log_size":56,"ondisk_log_size":56,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":3,"num_store_stats":3},{"poolid":9,"num_pg":16,"stat_sum":{"num_bytes":1195376664,"num_objects":18241,"num_object_clones":0,"num_object_copies":54723,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":18241,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":202023,"num_write_kb":7048385,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":3608948736,"data_stored":3608936520,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139323,"ondisk_log_size":139323,"up":48,"acting":48,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":48,"seq":206158430607,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1056276,"kb_used_data":672204,"kb_used_omap":0,"kb_used_meta":383936,"kb_avail":92688876,"statfs":{"total":95995035648,"available":94913409024,"internally_reserved":0,"allocated":688336896,"data_stored":687854080,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":393150464},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,5,4,3,1],"upper_bound":128},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 3 20:03:42 2023","interfaces":[{"interface":"back","average":{"1min":1.966,"5min":1.2290000000000001,"15min":1.3220000000000001},"min":{"1min":0.25900000000000001,"5min":0.25900000000000001,"15min":0.253},"max":{"1min":7.7839999999999998,"5min":7.7839999999999998,"15min":8.6099999999999994},"last":0.40000000000000002},{"interface":"front","average":{"1min":2.0379999999999998,"5min":1.5429999999999999,"15min":1.5209999999999999},"min":{"1min":0.34100000000000003,"5min":0.318,"15min":0.28499999999999998},"max":{"1min":8.2460000000000004,"5min":11.877000000000001,"15min":15.220000000000001},"last":0.82899999999999996}]},{"osd":1,"last update":"Sat Jun 3 20:03:22 2023","interfaces":[{"interface":"back","average":{"1min":2.3039999999999998,"5min":2.4510000000000001,"15min":1.964},"min":{"1min":0.29699999999999999,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":22.533999999999999,"5min":91.811000000000007,"15min":91.811000000000007},"last":0.45800000000000002},{"interface":"front","average":{"1min":3.6070000000000002,"5min":2.7080000000000002,"15min":1.95},"min":{"1min":0.32200000000000001,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":22.675000000000001,"5min":91.695999999999998,"15min":91.695999999999998},"last":1.536}]},{"osd":2,"last update":"Sat Jun 3 20:03:22 2023","interfaces":[{"interface":"back","average":{"1min":1.3029999999999999,"5min":1.157,"15min":1.0669999999999999},"min":{"1min":0.32800000000000001,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":7.71,"5min":7.71,"15min":7.71},"last":0.48699999999999999},{"interface":"front","average":{"1min":2.3199999999999998,"5min":1.3029999999999999,"15min":1.093},"min":{"1min":0.38500000000000001,"5min":0.38500000000000001,"15min":0.38500000000000001},"max":{"1min":8.3059999999999992,"5min":8.3059999999999992,"15min":8.3059999999999992},"last":0.86099999999999999}]},{"osd":3,"last update":"Sat Jun 3 20:04:11 2023","interfaces":[{"interface":"back","average":{"1min":2.1349999999999998,"5min":2.6419999999999999,"15min":1.639},"min":{"1min":0.35299999999999998,"5min":0.34699999999999998,"15min":0.34699999999999998},"max":{"1min":11.904,"5min":95.962000000000003,"15min":95.962000000000003},"last":0.625},{"interface":"front","average":{"1min":2.4849999999999999,"5min":2.8519999999999999,"15min":1.7410000000000001},"min":{"1min":0.503,"5min":0.36399999999999999,"15min":0.36399999999999999},"max":{"1min":12.084,"5min":94.174999999999997,"15min":94.174999999999997},"last":0.876}]},{"osd":4,"last update":"Sat Jun 3 20:03:33 2023","interfaces":[{"interface":"back","average":{"1min":1.1919999999999999,"5min":1.056,"15min":1.052},"min":{"1min":0.38200000000000001,"5min":0.35999999999999999,"15min":0.35999999999999999},"max":{"1min":7.7350000000000003,"5min":7.7350000000000003,"15min":7.7350000000000003},"last":0.68899999999999995},{"interface":"front","average":{"1min":2.6640000000000001,"5min":1.5309999999999999,"15min":1.256},"min":{"1min":0.39600000000000002,"5min":0.39600000000000002,"15min":0.39600000000000002},"max":{"1min":12.917,"5min":12.917,"15min":12.917},"last":0.79800000000000004}]},{"osd":5,"last update":"Sat Jun 3 20:03:19 2023","interfaces":[{"interface":"back","average":{"1min":1.383,"5min":1.1679999999999999,"15min":1.1719999999999999},"min":{"1min":0.443,"5min":0.443,"15min":0.30199999999999999},"max":{"1min":7.6829999999999998,"5min":7.6829999999999998,"15min":12.778},"last":0.52400000000000002},{"interface":"front","average":{"1min":2.2650000000000001,"5min":1.5269999999999999,"15min":1.387},"min":{"1min":0.33100000000000002,"5min":0.33100000000000002,"15min":0.30599999999999999},"max":{"1min":8.1340000000000003,"5min":8.1340000000000003,"15min":8.1340000000000003},"last":0.84499999999999997}]},{"osd":6,"last update":"Sat Jun 3 20:03:19 2023","interfaces":[{"interface":"back","average":{"1min":2.3929999999999998,"5min":1.431,"15min":1.363},"min":{"1min":0.42199999999999999,"5min":0.42199999999999999,"15min":0.36099999999999999},"max":{"1min":8.3219999999999992,"5min":8.3219999999999992,"15min":8.3219999999999992},"last":0.90900000000000003},{"interface":"front","average":{"1min":1.6240000000000001,"5min":1.4490000000000001,"15min":1.2509999999999999},"min":{"1min":0.36899999999999999,"5min":0.36899999999999999,"15min":0.32900000000000001},"max":{"1min":4.6210000000000004,"5min":6.6840000000000002,"15min":9.0269999999999992},"last":3.9100000000000001}]}]},{"osd":6,"up_from":42,"seq":180388626840,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":537472,"kb_used_data":230328,"kb_used_omap":0,"kb_used_meta":307136,"kb_avail":93207680,"statfs":{"total":95995035648,"available":95444664320,"internally_reserved":0,"allocated":235855872,"data_stored":235393051,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":314507264},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,3],"upper_bound":8},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 3 20:03:51 2023","interfaces":[{"interface":"back","average":{"1min":0.60899999999999999,"5min":1.4470000000000001,"15min":1.587},"min":{"1min":0.26400000000000001,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":3.3239999999999998,"5min":54.786000000000001,"15min":54.786000000000001},"last":0.53500000000000003},{"interface":"front","average":{"1min":0.872,"5min":1.5189999999999999,"15min":1.6719999999999999},"min":{"1min":0.317,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":3.335,"5min":54.936,"15min":54.936},"last":3.5129999999999999}]},{"osd":1,"last update":"Sat Jun 3 20:03:24 2023","interfaces":[{"interface":"back","average":{"1min":5.8339999999999996,"5min":4.1920000000000002,"15min":2.4239999999999999},"min":{"1min":0.28000000000000003,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":55.064999999999998,"5min":160.38499999999999,"15min":160.38499999999999},"last":0.56499999999999995},{"interface":"front","average":{"1min":6.1440000000000001,"5min":3.4830000000000001,"15min":2.032},"min":{"1min":0.29599999999999999,"5min":0.29599999999999999,"15min":0.29599999999999999},"max":{"1min":55.091000000000001,"5min":160.434,"15min":160.434},"last":4.0570000000000004}]},{"osd":2,"last update":"Sat Jun 3 20:03:24 2023","interfaces":[{"interface":"back","average":{"1min":0.97899999999999998,"5min":0.88300000000000001,"15min":0.85699999999999998},"min":{"1min":0.33300000000000002,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":3.4260000000000002,"5min":3.4260000000000002,"15min":3.4260000000000002},"last":0.41699999999999998},{"interface":"front","average":{"1min":0.78500000000000003,"5min":0.872,"15min":0.92100000000000004},"min":{"1min":0.34799999999999998,"5min":0.34799999999999998,"15min":0.34799999999999998},"max":{"1min":3.1779999999999999,"5min":3.1779999999999999,"15min":3.1779999999999999},"last":0.59199999999999997}]},{"osd":3,"last update":"Sat Jun 3 20:04:06 2023","interfaces":[{"interface":"back","average":{"1min":1.5620000000000001,"5min":4.2770000000000001,"15min":1.998},"min":{"1min":0.35299999999999998,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":5.7549999999999999,"5min":164.99199999999999,"15min":164.99199999999999},"last":0.46300000000000002},{"interface":"front","average":{"1min":1.3180000000000001,"5min":4.2569999999999997,"15min":2.089},"min":{"1min":0.316,"5min":0.316,"15min":0.316},"max":{"1min":4.5300000000000002,"5min":163.459,"15min":163.459},"last":0.64800000000000002}]},{"osd":4,"last update":"Sat Jun 3 20:03:42 2023","interfaces":[{"interface":"back","average":{"1min":0.84999999999999998,"5min":0.85299999999999998,"15min":0.94999999999999996},"min":{"1min":0.35399999999999998,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":3.6040000000000001,"5min":3.6040000000000001,"15min":3.6040000000000001},"last":0.371},{"interface":"front","average":{"1min":1.014,"5min":0.871,"15min":1.077},"min":{"1min":0.377,"5min":0.377,"15min":0.377},"max":{"1min":5.343,"5min":5.343,"15min":5.343},"last":0.48899999999999999}]},{"osd":5,"last update":"Sat Jun 3 20:03:37 2023","interfaces":[{"interface":"back","average":{"1min":0.68600000000000005,"5min":0.93200000000000005,"15min":1.159},"min":{"1min":0.34999999999999998,"5min":0.34999999999999998,"15min":0.317},"max":{"1min":1.1719999999999999,"5min":3.0750000000000002,"15min":11.083},"last":3.452},{"interface":"front","average":{"1min":2.214,"5min":1.399,"15min":1.389},"min":{"1min":0.32900000000000001,"5min":0.32900000000000001,"15min":0.32700000000000001},"max":{"1min":8.4160000000000004,"5min":8.4160000000000004,"15min":10.154},"last":3.9129999999999998}]},{"osd":7,"last update":"Sat Jun 3 20:04:12 2023","interfaces":[{"interface":"back","average":{"1min":1.3580000000000001,"5min":1.3500000000000001,"15min":1.3340000000000001},"min":{"1min":0.40000000000000002,"5min":0.39400000000000002,"15min":0.313},"max":{"1min":6.944,"5min":8.7409999999999997,"15min":8.7409999999999997},"last":0.621},{"interface":"front","average":{"1min":1.6319999999999999,"5min":1.395,"15min":1.4139999999999999},"min":{"1min":0.48699999999999999,"5min":0.33900000000000002,"15min":0.33000000000000002},"max":{"1min":7.3079999999999998,"5min":7.3079999999999998,"15min":12.407999999999999},"last":3.1120000000000001}]}]},{"osd":1,"up_from":103,"seq":442381631638,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":921136,"kb_used_data":673768,"kb_used_omap":4,"kb_used_meta":247355,"kb_avail":92824016,"statfs":{"total":95995035648,"available":95051792384,"internally_reserved":0,"allocated":689938432,"data_stored":689374406,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":253292430},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1,3],"upper_bound":16},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 3 20:03:25 2023","interfaces":[{"interface":"back","average":{"1min":7.4909999999999997,"5min":2.3479999999999999,"15min":1.921},"min":{"1min":0.217,"5min":0.185,"15min":0.185},"max":{"1min":119.902,"5min":119.902,"15min":119.902},"last":0.437},{"interface":"front","average":{"1min":7.5640000000000001,"5min":2.2360000000000002,"15min":1.978},"min":{"1min":0.22500000000000001,"5min":0.22500000000000001,"15min":0.192},"max":{"1min":119.863,"5min":119.863,"15min":119.863},"last":0.254}]},{"osd":2,"last update":"Sat Jun 3 20:03:34 2023","interfaces":[{"interface":"back","average":{"1min":0.97299999999999998,"5min":1.9550000000000001,"15min":1.1279999999999999},"min":{"1min":0.24099999999999999,"5min":0.23599999999999999,"15min":0.23599999999999999},"max":{"1min":4.6280000000000001,"5min":119.946,"15min":119.946},"last":0.223},{"interface":"front","average":{"1min":1.6859999999999999,"5min":1.0680000000000001,"15min":0.84999999999999998},"min":{"1min":0.26600000000000001,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":9.8160000000000007,"5min":9.8160000000000007,"15min":9.8160000000000007},"last":0.41099999999999998}]},{"osd":3,"last update":"Sat Jun 3 20:04:05 2023","interfaces":[{"interface":"back","average":{"1min":2.593,"5min":1.6990000000000001,"15min":1.0529999999999999},"min":{"1min":0.29899999999999999,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":9.657,"5min":9.657,"15min":9.657},"last":0.373},{"interface":"front","average":{"1min":2.456,"5min":1.6399999999999999,"15min":1.119},"min":{"1min":0.28999999999999998,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":9.7509999999999994,"5min":9.7509999999999994,"15min":9.7509999999999994},"last":0.69499999999999995}]},{"osd":4,"last update":"Sat Jun 3 20:03:36 2023","interfaces":[{"interface":"back","average":{"1min":2.6720000000000002,"5min":1.5149999999999999,"15min":1.167},"min":{"1min":0.38300000000000001,"5min":0.35999999999999999,"15min":0.35999999999999999},"max":{"1min":9.9589999999999996,"5min":9.9589999999999996,"15min":9.9589999999999996},"last":0.55700000000000005},{"interface":"front","average":{"1min":1.0680000000000001,"5min":2.52,"15min":1.478},"min":{"1min":0.32700000000000001,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":4.0670000000000002,"5min":120.084,"15min":120.084},"last":0.71699999999999997}]},{"osd":5,"last update":"Sat Jun 3 20:03:25 2023","interfaces":[{"interface":"back","average":{"1min":7.6779999999999999,"5min":2.8860000000000001,"15min":2.3690000000000002},"min":{"1min":0.36799999999999999,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":120.39700000000001,"5min":120.39700000000001,"15min":120.39700000000001},"last":0.67000000000000004},{"interface":"front","average":{"1min":7.75,"5min":2.7229999999999999,"15min":2.5800000000000001},"min":{"1min":0.48299999999999998,"5min":0.48299999999999998,"15min":0.40000000000000002},"max":{"1min":120.34399999999999,"5min":120.34399999999999,"15min":120.34399999999999},"last":0.60199999999999998}]},{"osd":6,"last update":"Sat Jun 3 20:03:25 2023","interfaces":[{"interface":"back","average":{"1min":7.1900000000000004,"5min":2.3290000000000002,"15min":2.2730000000000001},"min":{"1min":0.432,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":120.386,"5min":120.386,"15min":120.386},"last":0.42499999999999999},{"interface":"front","average":{"1min":7.3460000000000001,"5min":2.4500000000000002,"15min":2.5590000000000002},"min":{"1min":0.39600000000000002,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":119.96299999999999,"5min":119.96299999999999,"15min":119.96299999999999},"last":0.51500000000000001}]},{"osd":7,"last update":"Sat Jun 3 20:03:25 2023","interfaces":[{"interface":"back","average":{"1min":7.8109999999999999,"5min":2.7269999999999999,"15min":2.4430000000000001},"min":{"1min":0.40999999999999998,"5min":0.38200000000000001,"15min":0.38200000000000001},"max":{"1min":119.996,"5min":119.996,"15min":119.996},"last":0.83099999999999996},{"interface":"front","average":{"1min":7.758,"5min":2.653,"15min":2.0190000000000001},"min":{"1min":0.44800000000000001,"5min":0.40400000000000003,"15min":0.371},"max":{"1min":120.17100000000001,"5min":120.17100000000001,"15min":120.17100000000001},"last":3.8540000000000001}]}]},{"osd":0,"up_from":86,"seq":369367187665,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":537220,"kb_used_data":281788,"kb_used_omap":5,"kb_used_meta":255418,"kb_avail":93207932,"statfs":{"total":95995035648,"available":95444922368,"internally_reserved":0,"allocated":288550912,"data_stored":287969465,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":261548666},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,0,0,0,1],"upper_bound":64},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Jun 3 20:04:18 2023","interfaces":[{"interface":"back","average":{"1min":1.53,"5min":2.597,"15min":2.0699999999999998},"min":{"1min":0.23300000000000001,"5min":0.192,"15min":0.192},"max":{"1min":5.9020000000000001,"5min":81.310000000000002,"15min":81.310000000000002},"last":1.694},{"interface":"front","average":{"1min":2.3460000000000001,"5min":2.6379999999999999,"15min":1.8979999999999999},"min":{"1min":0.23300000000000001,"5min":0.187,"15min":0.187},"max":{"1min":8.2189999999999994,"5min":76.971999999999994,"15min":76.971999999999994},"last":0.38200000000000001}]},{"osd":2,"last update":"Sat Jun 3 20:04:18 2023","interfaces":[{"interface":"back","average":{"1min":2.0230000000000001,"5min":1.2430000000000001,"15min":0.85199999999999998},"min":{"1min":0.27500000000000002,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":6.2549999999999999,"5min":8.923,"15min":8.923},"last":2.827},{"interface":"front","average":{"1min":1.3939999999999999,"5min":1.1830000000000001,"15min":0.89700000000000002},"min":{"1min":0.28199999999999997,"5min":0.218,"15min":0.218},"max":{"1min":4.0419999999999998,"5min":11.194000000000001,"15min":11.194000000000001},"last":1.603}]},{"osd":3,"last update":"Sat Jun 3 20:04:10 2023","interfaces":[{"interface":"back","average":{"1min":2.278,"5min":1.409,"15min":1.006},"min":{"1min":0.252,"5min":0.216,"15min":0.216},"max":{"1min":6.343,"5min":7.5990000000000002,"15min":7.5990000000000002},"last":4.2720000000000002},{"interface":"front","average":{"1min":2.1779999999999999,"5min":1.448,"15min":0.98999999999999999},"min":{"1min":0.39800000000000002,"5min":0.35799999999999998,"15min":0.35399999999999998},"max":{"1min":8.4459999999999997,"5min":8.4459999999999997,"15min":8.4459999999999997},"last":1.5700000000000001}]},{"osd":4,"last update":"Sat Jun 3 20:03:43 2023","interfaces":[{"interface":"back","average":{"1min":1.3700000000000001,"5min":1.0429999999999999,"15min":1.0780000000000001},"min":{"1min":0.41299999999999998,"5min":0.33800000000000002,"15min":0.33800000000000002},"max":{"1min":5.0099999999999998,"5min":5.0099999999999998,"15min":5.0099999999999998},"last":2.9529999999999998},{"interface":"front","average":{"1min":1.341,"5min":1.226,"15min":1.1539999999999999},"min":{"1min":0.43099999999999999,"5min":0.38800000000000001,"15min":0.38800000000000001},"max":{"1min":4.8140000000000001,"5min":4.8220000000000001,"15min":4.8220000000000001},"last":1.671}]},{"osd":5,"last update":"Sat Jun 3 20:03:43 2023","interfaces":[{"interface":"back","average":{"1min":2.4980000000000002,"5min":1.3819999999999999,"15min":1.6479999999999999},"min":{"1min":0.38900000000000001,"5min":0.36799999999999999,"15min":0.33900000000000002},"max":{"1min":8.3100000000000005,"5min":8.3100000000000005,"15min":16.312000000000001},"last":2.9159999999999999},{"interface":"front","average":{"1min":2.5,"5min":1.4239999999999999,"15min":1.7190000000000001},"min":{"1min":0.36899999999999999,"5min":0.36899999999999999,"15min":0.32500000000000001},"max":{"1min":7.9779999999999998,"5min":7.9779999999999998,"15min":24.661000000000001},"last":2.867}]},{"osd":6,"last update":"Sat Jun 3 20:03:43 2023","interfaces":[{"interface":"back","average":{"1min":1.7130000000000001,"5min":1.202,"15min":1.6240000000000001},"min":{"1min":0.46600000000000003,"5min":0.38,"15min":0.33500000000000002},"max":{"1min":6.3899999999999997,"5min":6.3899999999999997,"15min":24.599},"last":2.891},{"interface":"front","average":{"1min":1.536,"5min":1.3859999999999999,"15min":1.7529999999999999},"min":{"1min":0.46999999999999997,"5min":0.46999999999999997,"15min":0.42999999999999999},"max":{"1min":4.2919999999999998,"5min":4.7930000000000001,"15min":12.474},"last":1.649}]},{"osd":7,"last update":"Sat Jun 3 20:03:43 2023","interfaces":[{"interface":"back","average":{"1min":2.6669999999999998,"5min":1.524,"15min":1.7190000000000001},"min":{"1min":0.52300000000000002,"5min":0.35099999999999998,"15min":0.35099999999999998},"max":{"1min":11.196,"5min":11.196,"15min":24.634},"last":2.9769999999999999},{"interface":"front","average":{"1min":2.2320000000000002,"5min":1.6459999999999999,"15min":1.7470000000000001},"min":{"1min":0.52300000000000002,"5min":0.51400000000000001,"15min":0.40699999999999997},"max":{"1min":7.327,"5min":7.327,"15min":12.621},"last":1.627}]}]},{"osd":2,"up_from":114,"seq":489626271844,"num_pgs":22,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":268248,"kb_used_data":218576,"kb_used_omap":7,"kb_used_meta":49656,"kb_avail":93476904,"statfs":{"total":95995035648,"available":95720349696,"internally_reserved":0,"allocated":223821824,"data_stored":223279316,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8111,"internal_metadata":50847825},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 3 20:03:22 2023","interfaces":[{"interface":"back","average":{"1min":0.76100000000000001,"5min":2.2989999999999999,"15min":1.181},"min":{"1min":0.20899999999999999,"5min":0.20899999999999999,"15min":0.20899999999999999},"max":{"1min":3.4769999999999999,"5min":126.416,"15min":126.416},"last":0.224},{"interface":"front","average":{"1min":0.67300000000000004,"5min":2.3860000000000001,"15min":1.268},"min":{"1min":0.22700000000000001,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":3.2290000000000001,"5min":126.976,"15min":126.976},"last":0.26600000000000001}]},{"osd":1,"last update":"Sat Jun 3 20:03:22 2023","interfaces":[{"interface":"back","average":{"1min":1.0449999999999999,"5min":1.296,"15min":0.91000000000000003},"min":{"1min":0.22900000000000001,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":6.5060000000000002,"5min":28.484999999999999,"15min":28.484999999999999},"last":0.315},{"interface":"front","average":{"1min":1.0569999999999999,"5min":1.1870000000000001,"15min":0.84499999999999997},"min":{"1min":0.255,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":4.0259999999999998,"5min":28.588999999999999,"15min":28.588999999999999},"last":0.38900000000000001}]},{"osd":3,"last update":"Sat Jun 3 20:04:08 2023","interfaces":[{"interface":"back","average":{"1min":0.94799999999999995,"5min":1.2549999999999999,"15min":0.91200000000000003},"min":{"1min":0.307,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":2.6230000000000002,"5min":29.251999999999999,"15min":29.251999999999999},"last":0.36399999999999999},{"interface":"front","average":{"1min":1.599,"5min":2.2949999999999999,"15min":1.2669999999999999},"min":{"1min":0.32100000000000001,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":6.0389999999999997,"5min":54.549999999999997,"15min":54.549999999999997},"last":0.29099999999999998}]},{"osd":4,"last update":"Sat Jun 3 20:03:42 2023","interfaces":[{"interface":"back","average":{"1min":0.747,"5min":0.879,"15min":0.90700000000000003},"min":{"1min":0.36899999999999999,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":3.359,"5min":7.9089999999999998,"15min":7.9089999999999998},"last":0.40100000000000002},{"interface":"front","average":{"1min":0.90800000000000003,"5min":0.90900000000000003,"15min":0.95299999999999996},"min":{"1min":0.32600000000000001,"5min":0.32600000000000001,"15min":0.32600000000000001},"max":{"1min":3.7749999999999999,"5min":3.7749999999999999,"15min":3.7749999999999999},"last":0.96199999999999997}]},{"osd":5,"last update":"Sat Jun 3 20:03:22 2023","interfaces":[{"interface":"back","average":{"1min":1.0049999999999999,"5min":1,"15min":1.0229999999999999},"min":{"1min":0.36599999999999999,"5min":0.36099999999999999,"15min":0.36099999999999999},"max":{"1min":3.5339999999999998,"5min":3.5339999999999998,"15min":3.5339999999999998},"last":0.54300000000000004},{"interface":"front","average":{"1min":1.583,"5min":1.327,"15min":1.1399999999999999},"min":{"1min":0.39400000000000002,"5min":0.308,"15min":0.308},"max":{"1min":4.9370000000000003,"5min":7.8689999999999998,"15min":7.8689999999999998},"last":4.8659999999999997}]},{"osd":6,"last update":"Sat Jun 3 20:03:22 2023","interfaces":[{"interface":"back","average":{"1min":1.085,"5min":1.032,"15min":1.034},"min":{"1min":0.38600000000000001,"5min":0.38500000000000001,"15min":0.38500000000000001},"max":{"1min":4.9950000000000001,"5min":6.7759999999999998,"15min":6.7759999999999998},"last":0.98799999999999999},{"interface":"front","average":{"1min":1.0649999999999999,"5min":1.012,"15min":1.036},"min":{"1min":0.41199999999999998,"5min":0.36299999999999999,"15min":0.36299999999999999},"max":{"1min":4.3559999999999999,"5min":7.9550000000000001,"15min":7.9550000000000001},"last":2.2400000000000002}]},{"osd":7,"last update":"Sat Jun 3 20:03:22 2023","interfaces":[{"interface":"back","average":{"1min":2.0899999999999999,"5min":1.4990000000000001,"15min":1.282},"min":{"1min":0.44700000000000001,"5min":0.44700000000000001,"15min":0.44700000000000001},"max":{"1min":6.0800000000000001,"5min":6.0800000000000001,"15min":6.0800000000000001},"last":1.4239999999999999},{"interface":"front","average":{"1min":1.992,"5min":1.472,"15min":1.2609999999999999},"min":{"1min":0.47899999999999998,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":6.0529999999999999,"5min":7.9320000000000004,"15min":7.9320000000000004},"last":1.413}]}]},{"osd":3,"up_from":119,"seq":511101108315,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":710540,"kb_used_data":606596,"kb_used_omap":2,"kb_used_meta":103933,"kb_avail":93034612,"statfs":{"total":95995035648,"available":95267442688,"internally_reserved":0,"allocated":621154304,"data_stored":620620771,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":106427445},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1,0,1,2],"upper_bound":64},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 3 20:04:12 2023","interfaces":[{"interface":"back","average":{"1min":1.5760000000000001,"5min":2.012,"15min":1.0880000000000001},"min":{"1min":0.19,"5min":0.19,"15min":0.19},"max":{"1min":6.5940000000000003,"5min":68.647999999999996,"15min":68.647999999999996},"last":0.58599999999999997},{"interface":"front","average":{"1min":1.9079999999999999,"5min":1.966,"15min":1.133},"min":{"1min":0.27600000000000002,"5min":0.224,"15min":0.224},"max":{"1min":8.3179999999999996,"5min":69.691000000000003,"15min":69.691000000000003},"last":0.45000000000000001}]},{"osd":1,"last update":"Sat Jun 3 20:04:12 2023","interfaces":[{"interface":"back","average":{"1min":2.1070000000000002,"5min":1.4730000000000001,"15min":0.95699999999999996},"min":{"1min":0.23000000000000001,"5min":0.23000000000000001,"15min":0.23000000000000001},"max":{"1min":4.077,"5min":10.065,"15min":10.065},"last":0.77600000000000002},{"interface":"front","average":{"1min":1.708,"5min":1.4119999999999999,"15min":0.97199999999999998},"min":{"1min":0.30099999999999999,"5min":0.23499999999999999,"15min":0.23499999999999999},"max":{"1min":6.6219999999999999,"5min":10.792999999999999,"15min":10.792999999999999},"last":0.66700000000000004}]},{"osd":2,"last update":"Sat Jun 3 20:04:12 2023","interfaces":[{"interface":"back","average":{"1min":1.833,"5min":1.246,"15min":0.84499999999999997},"min":{"1min":0.29399999999999998,"5min":0.245,"15min":0.245},"max":{"1min":8.3640000000000008,"5min":8.3640000000000008,"15min":8.3640000000000008},"last":0.51000000000000001},{"interface":"front","average":{"1min":1.752,"5min":1.304,"15min":0.93500000000000005},"min":{"1min":0.27800000000000002,"5min":0.24199999999999999,"15min":0.24199999999999999},"max":{"1min":4.1470000000000002,"5min":7.7290000000000001,"15min":7.7290000000000001},"last":0.432}]},{"osd":4,"last update":"Sat Jun 3 20:03:38 2023","interfaces":[{"interface":"back","average":{"1min":0.98099999999999998,"5min":1.1819999999999999,"15min":1.0600000000000001},"min":{"1min":0.38300000000000001,"5min":0.38300000000000001,"15min":0.38300000000000001},"max":{"1min":2.5190000000000001,"5min":4.774,"15min":4.774},"last":0.67800000000000005},{"interface":"front","average":{"1min":1.258,"5min":1.155,"15min":1.0589999999999999},"min":{"1min":0.47199999999999998,"5min":0.33000000000000002,"15min":0.33000000000000002},"max":{"1min":7.0720000000000001,"5min":7.0720000000000001,"15min":7.0720000000000001},"last":0.75}]},{"osd":5,"last update":"Sat Jun 3 20:04:12 2023","interfaces":[{"interface":"back","average":{"1min":1.508,"5min":1.423,"15min":1.131},"min":{"1min":0.46000000000000002,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":3.8450000000000002,"5min":7.1230000000000002,"15min":7.1230000000000002},"last":0.71499999999999997},{"interface":"front","average":{"1min":2.8300000000000001,"5min":1.883,"15min":1.306},"min":{"1min":0.39100000000000001,"5min":0.35599999999999998,"15min":0.35599999999999998},"max":{"1min":8.3949999999999996,"5min":8.3949999999999996,"15min":8.3949999999999996},"last":0.61899999999999999}]},{"osd":6,"last update":"Sat Jun 3 20:04:12 2023","interfaces":[{"interface":"back","average":{"1min":2.258,"5min":1.6200000000000001,"15min":1.232},"min":{"1min":0.44700000000000001,"5min":0.38600000000000001,"15min":0.38600000000000001},"max":{"1min":8.4169999999999998,"5min":8.4169999999999998,"15min":8.4169999999999998},"last":0.64900000000000002},{"interface":"front","average":{"1min":1.526,"5min":1.3680000000000001,"15min":1.1599999999999999},"min":{"1min":0.39000000000000001,"5min":0.39000000000000001,"15min":0.39000000000000001},"max":{"1min":4.1719999999999997,"5min":4.9480000000000004,"15min":4.9480000000000004},"last":0.92200000000000004}]},{"osd":7,"last update":"Sat Jun 3 20:04:12 2023","interfaces":[{"interface":"back","average":{"1min":2.1400000000000001,"5min":1.571,"15min":1.216},"min":{"1min":0.496,"5min":0.495,"15min":0.495},"max":{"1min":8.6910000000000007,"5min":8.6910000000000007,"15min":8.6910000000000007},"last":1.0629999999999999},{"interface":"front","average":{"1min":1.8859999999999999,"5min":1.859,"15min":1.3009999999999999},"min":{"1min":0.622,"5min":0.435,"15min":0.435},"max":{"1min":6.5609999999999999,"5min":12.715999999999999,"15min":12.715999999999999},"last":2.9079999999999999}]}]},{"osd":4,"up_from":124,"seq":532575944788,"num_pgs":36,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":287976,"kb_used_data":242336,"kb_used_omap":10,"kb_used_meta":45621,"kb_avail":93457176,"statfs":{"total":95995035648,"available":95700148224,"internally_reserved":0,"allocated":248152064,"data_stored":247594537,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":10652,"internal_metadata":46716516},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 3 20:03:42 2023","interfaces":[{"interface":"back","average":{"1min":3.7679999999999998,"5min":1.504,"15min":1.151},"min":{"1min":0.255,"5min":0.254,"15min":0.254},"max":{"1min":57.786000000000001,"5min":57.786000000000001,"15min":57.786000000000001},"last":2.0310000000000001},{"interface":"front","average":{"1min":3.4409999999999998,"5min":1.492,"15min":1.1839999999999999},"min":{"1min":0.372,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":57.820999999999998,"5min":57.820999999999998,"15min":57.820999999999998},"last":2.0409999999999999}]},{"osd":1,"last update":"Sat Jun 3 20:03:42 2023","interfaces":[{"interface":"back","average":{"1min":5.1710000000000003,"5min":1.9570000000000001,"15min":1.2330000000000001},"min":{"1min":0.28799999999999998,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":58.32,"5min":58.32,"15min":58.32},"last":0.34100000000000003},{"interface":"front","average":{"1min":5.2439999999999998,"5min":1.9159999999999999,"15min":1.214},"min":{"1min":0.307,"5min":0.307,"15min":0.307},"max":{"1min":58.286000000000001,"5min":58.286000000000001,"15min":58.286000000000001},"last":0.47899999999999998}]},{"osd":2,"last update":"Sat Jun 3 20:03:42 2023","interfaces":[{"interface":"back","average":{"1min":1.1899999999999999,"5min":0.88400000000000001,"15min":0.875},"min":{"1min":0.30299999999999999,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":4.3940000000000001,"5min":4.3940000000000001,"15min":4.3940000000000001},"last":0.41499999999999998},{"interface":"front","average":{"1min":0.85399999999999998,"5min":0.86699999999999999,"15min":0.91400000000000003},"min":{"1min":0.38100000000000001,"5min":0.34499999999999997,"15min":0.34499999999999997},"max":{"1min":5.069,"5min":5.069,"15min":5.069},"last":0.92600000000000005}]},{"osd":3,"last update":"Sat Jun 3 20:03:42 2023","interfaces":[{"interface":"back","average":{"1min":6.032,"5min":2.0449999999999999,"15min":1.323},"min":{"1min":0.35299999999999998,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":57.954999999999998,"5min":57.954999999999998,"15min":57.954999999999998},"last":2.718},{"interface":"front","average":{"1min":6.0030000000000001,"5min":2.069,"15min":1.393},"min":{"1min":0.439,"5min":0.39900000000000002,"15min":0.39900000000000002},"max":{"1min":57.889000000000003,"5min":57.889000000000003,"15min":57.889000000000003},"last":2.7709999999999999}]},{"osd":5,"last update":"Sat Jun 3 20:03:42 2023","interfaces":[{"interface":"back","average":{"1min":4.6159999999999997,"5min":1.7849999999999999,"15min":1.23},"min":{"1min":0.38,"5min":0.36099999999999999,"15min":0.36099999999999999},"max":{"1min":86.576999999999998,"5min":86.576999999999998,"15min":86.576999999999998},"last":0.379},{"interface":"front","average":{"1min":5.0209999999999999,"5min":1.919,"15min":1.298},"min":{"1min":0.38100000000000001,"5min":0.32600000000000001,"15min":0.32600000000000001},"max":{"1min":88.013000000000005,"5min":88.013000000000005,"15min":88.013000000000005},"last":0.46899999999999997}]},{"osd":6,"last update":"Sat Jun 3 20:03:42 2023","interfaces":[{"interface":"back","average":{"1min":0.95199999999999996,"5min":0.90100000000000002,"15min":0.997},"min":{"1min":0.36599999999999999,"5min":0.36599999999999999,"15min":0.36599999999999999},"max":{"1min":5.1420000000000003,"5min":5.1420000000000003,"15min":5.1420000000000003},"last":0.497},{"interface":"front","average":{"1min":1.1240000000000001,"5min":0.94499999999999995,"15min":1.034},"min":{"1min":0.39700000000000002,"5min":0.36499999999999999,"15min":0.36499999999999999},"max":{"1min":7.2220000000000004,"5min":7.2220000000000004,"15min":7.2220000000000004},"last":0.439}]},{"osd":7,"last update":"Sat Jun 3 20:03:42 2023","interfaces":[{"interface":"back","average":{"1min":1.8049999999999999,"5min":1.3420000000000001,"15min":1.206},"min":{"1min":0.497,"5min":0.41799999999999998,"15min":0.41799999999999998},"max":{"1min":5.3179999999999996,"5min":10.048999999999999,"15min":10.048999999999999},"last":0.39700000000000002},{"interface":"front","average":{"1min":2.1869999999999998,"5min":1.4710000000000001,"15min":1.2709999999999999},"min":{"1min":0.31900000000000001,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":9.3309999999999995,"5min":9.3309999999999995,"15min":9.3309999999999995},"last":0.51800000000000002}]}]},{"osd":5,"up_from":36,"seq":154618823074,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1152312,"kb_used_data":682480,"kb_used_omap":0,"kb_used_meta":469760,"kb_avail":92592840,"statfs":{"total":95995035648,"available":94815068160,"internally_reserved":0,"allocated":698859520,"data_stored":698400209,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":481034240},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,0,1,3,1,2],"upper_bound":128},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 3 20:03:27 2023","interfaces":[{"interface":"back","average":{"1min":1.2589999999999999,"5min":1.369,"15min":1.472},"min":{"1min":0.26900000000000002,"5min":0.22800000000000001,"15min":0.22800000000000001},"max":{"1min":4.2199999999999998,"5min":37.689999999999998,"15min":37.689999999999998},"last":0.71299999999999997},{"interface":"front","average":{"1min":2.0920000000000001,"5min":1.8140000000000001,"15min":1.8480000000000001},"min":{"1min":0.26300000000000001,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":8.266,"5min":37.783999999999999,"15min":37.783999999999999},"last":5.0540000000000003}]},{"osd":1,"last update":"Sat Jun 3 20:03:27 2023","interfaces":[{"interface":"back","average":{"1min":6.7569999999999997,"5min":3.7749999999999999,"15min":2.4620000000000002},"min":{"1min":0.47399999999999998,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":47.747999999999998,"5min":114.681,"15min":114.681},"last":0.30199999999999999},{"interface":"front","average":{"1min":6.1749999999999998,"5min":3.726,"15min":2.5059999999999998},"min":{"1min":0.312,"5min":0.307,"15min":0.29999999999999999},"max":{"1min":48.015000000000001,"5min":115.366,"15min":115.366},"last":4.9779999999999998}]},{"osd":2,"last update":"Sat Jun 3 20:03:27 2023","interfaces":[{"interface":"back","average":{"1min":1.177,"5min":0.97299999999999998,"15min":0.90500000000000003},"min":{"1min":0.35399999999999998,"5min":0.35099999999999998,"15min":0.34999999999999998},"max":{"1min":4.0549999999999997,"5min":4.3529999999999998,"15min":4.3529999999999998},"last":0.73999999999999999},{"interface":"front","average":{"1min":1.784,"5min":1.2929999999999999,"15min":1.0800000000000001},"min":{"1min":0.34100000000000003,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":8.2119999999999997,"5min":8.2119999999999997,"15min":8.2119999999999997},"last":5.016}]},{"osd":3,"last update":"Sat Jun 3 20:04:10 2023","interfaces":[{"interface":"back","average":{"1min":3.6429999999999998,"5min":5.3079999999999998,"15min":2.4470000000000001},"min":{"1min":0.38700000000000001,"5min":0.34599999999999997,"15min":0.34599999999999997},"max":{"1min":29.134,"5min":261.14100000000002,"15min":261.14100000000002},"last":0.33700000000000002},{"interface":"front","average":{"1min":2.5699999999999998,"5min":4.7199999999999998,"15min":2.2400000000000002},"min":{"1min":0.44500000000000001,"5min":0.35399999999999998,"15min":0.35399999999999998},"max":{"1min":29.024000000000001,"5min":261.27100000000002,"15min":261.27100000000002},"last":0.48199999999999998}]},{"osd":4,"last update":"Sat Jun 3 20:03:43 2023","interfaces":[{"interface":"back","average":{"1min":1.9550000000000001,"5min":1.5169999999999999,"15min":1.173},"min":{"1min":0.40400000000000003,"5min":0.378,"15min":0.378},"max":{"1min":4.8479999999999999,"5min":10.162000000000001,"15min":10.162000000000001},"last":0.42899999999999999},{"interface":"front","average":{"1min":1.1819999999999999,"5min":1.0129999999999999,"15min":1.129},"min":{"1min":0.377,"5min":0.377,"15min":0.377},"max":{"1min":4.9980000000000002,"5min":4.9980000000000002,"15min":4.9980000000000002},"last":0.51000000000000001}]},{"osd":6,"last update":"Sat Jun 3 20:03:27 2023","interfaces":[{"interface":"back","average":{"1min":2.5419999999999998,"5min":1.6439999999999999,"15min":1.4450000000000001},"min":{"1min":0.56100000000000005,"5min":0.41199999999999998,"15min":0.33000000000000002},"max":{"1min":10.278,"5min":10.278,"15min":12.731999999999999},"last":4.383},{"interface":"front","average":{"1min":2.4060000000000001,"5min":1.712,"15min":1.532},"min":{"1min":0.438,"5min":0.438,"15min":0.32800000000000001},"max":{"1min":13.714,"5min":13.714,"15min":13.714},"last":0.39700000000000002}]},{"osd":7,"last update":"Sat Jun 3 20:04:15 2023","interfaces":[{"interface":"back","average":{"1min":2.5270000000000001,"5min":2.2080000000000002,"15min":1.742},"min":{"1min":0.50800000000000001,"5min":0.503,"15min":0.379},"max":{"1min":9.6050000000000004,"5min":10.192,"15min":11.297000000000001},"last":0.66100000000000003},{"interface":"front","average":{"1min":1.0609999999999999,"5min":1.244,"15min":1.246},"min":{"1min":0.38400000000000001,"5min":0.38400000000000001,"15min":0.32100000000000001},"max":{"1min":3.8719999999999999,"5min":5.8719999999999999,"15min":7.4720000000000004},"last":0.45200000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6928,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4862,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":10652,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4786,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4932,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":283705344,"data_stored":283705344,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":674041856,"data_stored":674037784,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":214630400,"data_stored":214630400,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":611713024,"data_stored":611713024,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":239927296,"data_stored":239927296,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":679481344,"data_stored":679477272,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":227606528,"data_stored":227606528,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":677842944,"data_stored":677838872,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-06-03T20:04:22.868 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph osd pool set unique_pool_4 pgp_num 16 2023-06-03T20:04:22.921 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:04:22.964 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:04:22.969 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:04:22.970 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:04:23.070 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:04:23.104 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:04:23.110 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:04:23.110 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:04:23.211 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:04:23.242 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:04:23.245 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:04:23.246 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:04:23.296 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/mon.a/config 2023-06-03T20:04:23.297 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/mon.a/config 2023-06-03T20:04:23.347 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:04:23.381 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:04:23.386 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:04:23.387 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:04:23.487 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:04:23.526 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:04:23.529 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:04:23.529 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:04:23.630 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:04:23.667 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:04:23.670 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:04:23.670 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:04:23.671 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:04:23 smithi164 bash[59462]: cluster 1685822662.5066125 mgr.y (mgr.34104) 845 : cluster 0 pgmap v765: 89 pgs: 89 active+clean; 1.1 GiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 591 op/s 2023-06-03T20:04:23.772 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:04:23.812 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:04:23.816 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:04:23.816 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:04:23.846 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:04:23 smithi089 bash[75068]: cluster 1685822662.5066125 mgr.y (mgr.34104) 845 : cluster 0 pgmap v765: 89 pgs: 89 active+clean; 1.1 GiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 591 op/s 2023-06-03T20:04:23.917 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:04:23.954 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:04:23.959 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:04:23.960 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:04:24.061 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:04:24.092 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:04:24.095 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:04:24.096 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:04:24.197 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:04:24.205 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:04:23 smithi089 bash[78452]: cluster 1685822662.5066125 mgr.y (mgr.34104) 845 : cluster 0 pgmap v765: 89 pgs: 89 active+clean; 1.1 GiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 591 op/s 2023-06-03T20:04:24.227 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:04:24.230 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:04:24.231 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:04:24.332 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:04:24.371 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:04:24.372 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:04:24.373 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:04:24.474 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:04:24.509 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:04:24.514 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:04:24.515 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:04:24.616 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:04:24.649 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:04:24.655 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:04:24.656 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:04:24.756 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:04:24.795 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:04:24.801 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:04:24.802 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:04:24.902 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:04:24.943 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:04:24.945 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:04:24.946 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:04:25.047 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:04:25.083 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:04:25.086 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:04:25.087 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:04:25.188 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:04:25.222 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:04:25.228 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:04:25.229 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:04:25.330 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:04:25.369 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:04:25.371 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:04:25.372 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:04:25.473 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:04:25.509 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:04:25.511 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:04:25.512 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:04:25.613 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:04:25.652 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:04:25.655 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:04:25.656 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:04:25.757 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:04:25.767 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:04:25 smithi089 bash[75068]: cluster 1685822664.5071535 mgr.y (mgr.34104) 846 : cluster 0 pgmap v766: 89 pgs: 89 active+clean; 961 MiB data, 4.7 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 823 op/s 2023-06-03T20:04:25.767 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:04:25 smithi089 bash[78452]: cluster 1685822664.5071535 mgr.y (mgr.34104) 846 : cluster 0 pgmap v766: 89 pgs: 89 active+clean; 961 MiB data, 4.7 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 823 op/s 2023-06-03T20:04:25.808 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:04:25.812 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:04:25.813 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:04:25.913 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:04:25.921 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:04:25 smithi164 bash[59462]: cluster 1685822664.5071535 mgr.y (mgr.34104) 846 : cluster 0 pgmap v766: 89 pgs: 89 active+clean; 961 MiB data, 4.7 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 823 op/s 2023-06-03T20:04:25.952 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:04:25.958 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:04:25.958 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:04:26.059 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:04:26.092 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:04:26.093 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:04:26.093 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:04:26.194 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:04:26.225 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:04:26.229 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:04:26.229 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:04:26.330 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:04:26.370 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:04:26.372 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:04:26.372 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:04:26.473 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:04:26.482 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 20:04:26 smithi089 bash[71163]: ::ffff:172.21.15.164 - - [03/Jun/2023:20:04:26] "GET /metrics HTTP/1.1" 200 228314 "" "Prometheus/2.33.4" 2023-06-03T20:04:26.502 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:04:26.506 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:04:26.507 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:04:26.608 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:04:26.642 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:04:26.647 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:04:26.647 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:04:26.748 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:04:26.782 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:04:26.785 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:04:26.785 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:04:26.886 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:04:26.923 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:04:26.925 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:04:26.926 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:04:27.027 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:04:27.082 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:04:27.086 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:04:27.087 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:04:27.147 INFO:teuthology.orchestra.run.smithi164.stderr:Inferring config /var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/mon.b/config 2023-06-03T20:04:27.188 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:04:27.235 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:04:27.238 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:04:27.239 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:04:27.340 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:04:27.368 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:04:27.373 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:04:27.374 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:04:27.475 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:04:27.525 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:04:27.526 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:04:27.527 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:04:27.628 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:04:27.670 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:04:27.673 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:04:27.674 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:04:27.775 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:04:27.803 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:04:27.811 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:04:27.812 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:04:27.913 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:04:27.922 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:04:27 smithi089 bash[75068]: cluster 1685822666.5076706 mgr.y (mgr.34104) 847 : cluster 0 pgmap v767: 89 pgs: 89 active+clean; 918 MiB data, 4.6 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 645 op/s 2023-06-03T20:04:27.922 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:04:27 smithi089 bash[78452]: cluster 1685822666.5076706 mgr.y (mgr.34104) 847 : cluster 0 pgmap v767: 89 pgs: 89 active+clean; 918 MiB data, 4.6 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 645 op/s 2023-06-03T20:04:27.947 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:04:27.950 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:04:27.951 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:04:28.052 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:04:28.083 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:04:27 smithi164 bash[59462]: cluster 1685822666.5076706 mgr.y (mgr.34104) 847 : cluster 0 pgmap v767: 89 pgs: 89 active+clean; 918 MiB data, 4.6 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 645 op/s 2023-06-03T20:04:28.097 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:04:28.099 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:04:28.100 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:04:28.201 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:04:28.240 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:04:28.245 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:04:28.246 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:04:28.347 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:04:28.385 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:04:28.393 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:04:28.394 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:04:28.431 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/mon.a/config 2023-06-03T20:04:28.495 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:04:28.549 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:04:28.554 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:04:28.554 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:04:28.655 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:04:28.692 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:04:28.694 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:04:28.694 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:04:28.795 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:04:28.828 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:04:28.834 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:04:28.835 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:04:28.936 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:04:28.974 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:04:28.980 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:04:28.981 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:04:29.082 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:04:29.117 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:04:29.123 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:04:29.124 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:04:29.225 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:04:29.281 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:04:29.286 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:04:29.287 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:04:29.388 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:04:29.422 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:04:29.428 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:04:29.429 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:04:29.530 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:04:29.572 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:04:29.577 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:04:29.578 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:04:29.679 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:04:29.687 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:04:29 smithi089 bash[75068]: cluster 1685822668.5083594 mgr.y (mgr.34104) 848 : cluster 0 pgmap v768: 89 pgs: 89 active+clean; 765 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 797 op/s 2023-06-03T20:04:29.688 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:04:29 smithi089 bash[78452]: cluster 1685822668.5083594 mgr.y (mgr.34104) 848 : cluster 0 pgmap v768: 89 pgs: 89 active+clean; 765 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 797 op/s 2023-06-03T20:04:29.710 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:04:29.712 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:04:29.713 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:04:29.814 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:04:29.833 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:04:29 smithi164 bash[59462]: cluster 1685822668.5083594 mgr.y (mgr.34104) 848 : cluster 0 pgmap v768: 89 pgs: 89 active+clean; 765 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 797 op/s 2023-06-03T20:04:29.844 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:04:29.852 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:04:29.854 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:04:29.955 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:04:29.991 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:04:29.995 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:04:29.995 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:04:30.096 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:04:30.147 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:04:30.154 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:04:30.155 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:04:30.256 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:04:30.294 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:04:30.298 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:04:30.299 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:04:30.400 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:04:30.437 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:04:30.442 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:04:30.442 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:04:30.544 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:04:30.582 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:04:30.589 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:04:30.590 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:04:30.691 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:04:30.725 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:04:30.732 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:04:30.733 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:04:30.834 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:04:30.867 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:04:30.875 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:04:30.876 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:04:30.976 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:04:31.009 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:04:31.013 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:04:31.014 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:04:31.115 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:04:31.167 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:04:31.173 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:04:31.174 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:04:31.275 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:04:31.318 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:04:31.322 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:04:31.322 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:04:31.423 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:04:31.457 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:04:31.465 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:04:31.466 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:04:31.567 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:04:31.605 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:04:31.607 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:04:31.608 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:04:31.709 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:04:31.751 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:04:31.754 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:04:31.754 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:04:31.833 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:04:31 smithi164 bash[59462]: cluster 1685822670.5090275 mgr.y (mgr.34104) 849 : cluster 0 pgmap v769: 89 pgs: 89 active+clean; 694 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 687 op/s 2023-06-03T20:04:31.856 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:04:31.892 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:04:31.899 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:04:31.900 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:04:31.951 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:04:31 smithi089 bash[75068]: cluster 1685822670.5090275 mgr.y (mgr.34104) 849 : cluster 0 pgmap v769: 89 pgs: 89 active+clean; 694 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 687 op/s 2023-06-03T20:04:31.952 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:04:31 smithi089 bash[78452]: cluster 1685822670.5090275 mgr.y (mgr.34104) 849 : cluster 0 pgmap v769: 89 pgs: 89 active+clean; 694 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 687 op/s 2023-06-03T20:04:32.001 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:04:32.045 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:04:32.050 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:04:32.050 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:04:32.075 INFO:teuthology.orchestra.run.smithi164.stdout:{ 2023-06-03T20:04:32.076 INFO:teuthology.orchestra.run.smithi164.stdout: "ops": [ 2023-06-03T20:04:32.076 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:04:32.076 INFO:teuthology.orchestra.run.smithi164.stdout: "description": "osd_repop(client.45713.0:211587 9.e e144/127 9:72256cfd:::benchmark_data_smithi089_115580_object101453:head v 144'13184, mlcod=144'13182)", 2023-06-03T20:04:32.076 INFO:teuthology.orchestra.run.smithi164.stdout: "initiated_at": "2023-06-03T20:04:32.059342+0000", 2023-06-03T20:04:32.076 INFO:teuthology.orchestra.run.smithi164.stdout: "age": 0.0043136149999999998, 2023-06-03T20:04:32.077 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.0043584319999999998, 2023-06-03T20:04:32.077 INFO:teuthology.orchestra.run.smithi164.stdout: "type_data": { 2023-06-03T20:04:32.077 INFO:teuthology.orchestra.run.smithi164.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-03T20:04:32.077 INFO:teuthology.orchestra.run.smithi164.stdout: "events": [ 2023-06-03T20:04:32.077 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:04:32.077 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "initiated", 2023-06-03T20:04:32.077 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:04:32.059342+0000", 2023-06-03T20:04:32.077 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0 2023-06-03T20:04:32.078 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:04:32.078 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:04:32.078 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "throttled", 2023-06-03T20:04:32.078 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:04:32.059342+0000", 2023-06-03T20:04:32.078 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0 2023-06-03T20:04:32.078 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:04:32.078 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:04:32.078 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "header_read", 2023-06-03T20:04:32.079 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:04:32.059340+0000", 2023-06-03T20:04:32.079 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 4294967295.9999981 2023-06-03T20:04:32.079 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:04:32.079 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:04:32.079 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "all_read", 2023-06-03T20:04:32.079 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:04:32.059358+0000", 2023-06-03T20:04:32.080 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 1.7577000000000001e-05 2023-06-03T20:04:32.080 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:04:32.080 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:04:32.080 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "dispatched", 2023-06-03T20:04:32.080 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:04:32.059384+0000", 2023-06-03T20:04:32.080 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 2.5829e-05 2023-06-03T20:04:32.080 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:04:32.081 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:04:32.081 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "queued_for_pg", 2023-06-03T20:04:32.081 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:04:32.059395+0000", 2023-06-03T20:04:32.081 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 1.1e-05 2023-06-03T20:04:32.081 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:04:32.081 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:04:32.081 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "reached_pg", 2023-06-03T20:04:32.081 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:04:32.059455+0000", 2023-06-03T20:04:32.082 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 6.0282000000000003e-05 2023-06-03T20:04:32.082 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:04:32.082 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:04:32.082 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "started", 2023-06-03T20:04:32.082 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:04:32.059485+0000", 2023-06-03T20:04:32.082 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 2.9852000000000001e-05 2023-06-03T20:04:32.082 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:04:32.082 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:04:32.083 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "commit_sent", 2023-06-03T20:04:32.083 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:04:32.063648+0000", 2023-06-03T20:04:32.083 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.0041631860000000001 2023-06-03T20:04:32.083 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:04:32.083 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:04:32.083 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "done", 2023-06-03T20:04:32.083 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:04:32.063701+0000", 2023-06-03T20:04:32.083 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 5.2713000000000002e-05 2023-06-03T20:04:32.084 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:04:32.084 INFO:teuthology.orchestra.run.smithi164.stdout: ] 2023-06-03T20:04:32.084 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:04:32.084 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:04:32.084 INFO:teuthology.orchestra.run.smithi164.stdout: ], 2023-06-03T20:04:32.084 INFO:teuthology.orchestra.run.smithi164.stdout: "num_ops": 1 2023-06-03T20:04:32.084 INFO:teuthology.orchestra.run.smithi164.stdout:} 2023-06-03T20:04:32.152 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:04:32.187 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:04:32.189 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:04:32.190 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:04:32.217 INFO:teuthology.orchestra.run.smithi089.stdout:osd.0: {} 2023-06-03T20:04:32.252 INFO:teuthology.orchestra.run.smithi089.stdout:osd.1: {} 2023-06-03T20:04:32.283 INFO:teuthology.orchestra.run.smithi089.stdout:osd.2: {} 2023-06-03T20:04:32.291 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:04:32.325 INFO:teuthology.orchestra.run.smithi089.stdout:osd.3: {} 2023-06-03T20:04:32.327 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:04:32.332 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:04:32.333 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:04:32.365 INFO:teuthology.orchestra.run.smithi089.stdout:osd.4: {} 2023-06-03T20:04:32.400 INFO:teuthology.orchestra.run.smithi089.stdout:osd.5: {} 2023-06-03T20:04:32.428 INFO:teuthology.orchestra.run.smithi089.stdout:osd.6: {} 2023-06-03T20:04:32.435 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:04:32.450 INFO:teuthology.orchestra.run.smithi089.stdout:osd.7: {} 2023-06-03T20:04:32.451 INFO:teuthology.orchestra.run.smithi089.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-06-03T20:04:32.452 INFO:teuthology.orchestra.run.smithi089.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-06-03T20:04:32.452 INFO:teuthology.orchestra.run.smithi089.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-06-03T20:04:32.452 INFO:teuthology.orchestra.run.smithi089.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-06-03T20:04:32.452 INFO:teuthology.orchestra.run.smithi089.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-06-03T20:04:32.452 INFO:teuthology.orchestra.run.smithi089.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-06-03T20:04:32.453 INFO:teuthology.orchestra.run.smithi089.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-06-03T20:04:32.453 INFO:teuthology.orchestra.run.smithi089.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-06-03T20:04:32.460 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:04:32.462 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:04:32.462 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:04:32.563 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:04:32.603 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:04:32.609 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:04:32.610 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:04:32.705 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:04:32 smithi164 bash[59462]: audit 1685822672.1544008 mon.a (mon.0) 746 : audit 0 from='client.? 172.21.15.89:0/1017700586' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-03T20:04:32.705 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:04:32 smithi164 bash[59462]: audit 1685822672.2152321 mon.a (mon.0) 747 : audit 1 from='client.? 172.21.15.89:0/507798448' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-06-03T20:04:32.712 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:04:32.717 INFO:teuthology.orchestra.run.smithi089.stderr:noscrub is unset 2023-06-03T20:04:32.753 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:04:32.759 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:04:32.761 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:04:32.862 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:04:32.903 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:04:32.909 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:04:32.909 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:04:33.010 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:04:33.028 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:04:32 smithi089 bash[78452]: audit 1685822672.1544008 mon.a (mon.0) 746 : audit 0 from='client.? 172.21.15.89:0/1017700586' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-03T20:04:33.029 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:04:32 smithi089 bash[78452]: audit 1685822672.2152321 mon.a (mon.0) 747 : audit 1 from='client.? 172.21.15.89:0/507798448' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-06-03T20:04:33.029 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:04:32 smithi089 bash[75068]: audit 1685822672.1544008 mon.a (mon.0) 746 : audit 0 from='client.? 172.21.15.89:0/1017700586' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-03T20:04:33.029 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:04:32 smithi089 bash[75068]: audit 1685822672.2152321 mon.a (mon.0) 747 : audit 1 from='client.? 172.21.15.89:0/507798448' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-06-03T20:04:33.057 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:04:33.063 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:04:33.064 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:04:33.096 DEBUG:teuthology.orchestra.run.smithi164:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph daemon osd.4 dump_blocked_ops 2023-06-03T20:04:33.165 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:04:33.238 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:04:33.242 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:04:33.243 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:04:33.344 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:04:33.388 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:04:33.392 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:04:33.392 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:04:33.493 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:04:33.526 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:04:33.529 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:04:33.529 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:04:33.630 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:04:33.668 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:04:33.673 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:04:33.674 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:04:33.775 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:04:33.807 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:04:33.810 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:04:33.811 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:04:33.912 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:04:33.938 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:04:33.948 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:04:33.949 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:04:33.949 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:04:33 smithi089 bash[78452]: cluster 1685822672.5095303 mgr.y (mgr.34104) 850 : cluster 0 pgmap v770: 89 pgs: 89 active+clean; 694 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 594 op/s 2023-06-03T20:04:33.950 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:04:33 smithi089 bash[78452]: cluster 1685822672.6911573 mon.a (mon.0) 748 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-06-03T20:04:33.950 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:04:33 smithi089 bash[78452]: cluster 1685822672.7030776 osd.6 (osd.6) 64 : cluster 3 failed to encode map e145 with expected crc 2023-06-03T20:04:33.950 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:04:33 smithi089 bash[78452]: cluster 1685822672.705488 osd.7 (osd.7) 57 : cluster 3 failed to encode map e145 with expected crc 2023-06-03T20:04:33.950 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:04:33 smithi089 bash[78452]: cluster 1685822672.7068055 osd.5 (osd.5) 69 : cluster 3 failed to encode map e145 with expected crc 2023-06-03T20:04:33.950 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:04:33 smithi089 bash[78452]: audit 1685822672.708074 mon.a (mon.0) 749 : audit 1 from='client.? 172.21.15.89:0/507798448' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-06-03T20:04:33.950 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:04:33 smithi089 bash[78452]: cluster 1685822672.7082794 mon.a (mon.0) 750 : cluster 0 osdmap e145: 8 total, 8 up, 8 in 2023-06-03T20:04:34.050 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:04:34.081 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:04:34.083 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:04:33 smithi164 bash[59462]: cluster 1685822672.5095303 mgr.y (mgr.34104) 850 : cluster 0 pgmap v770: 89 pgs: 89 active+clean; 694 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 594 op/s 2023-06-03T20:04:34.083 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:04:33 smithi164 bash[59462]: cluster 1685822672.6911573 mon.a (mon.0) 748 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-06-03T20:04:34.084 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:04:33 smithi164 bash[59462]: cluster 1685822672.7030776 osd.6 (osd.6) 64 : cluster 3 failed to encode map e145 with expected crc 2023-06-03T20:04:34.084 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:04:33 smithi164 bash[59462]: cluster 1685822672.705488 osd.7 (osd.7) 57 : cluster 3 failed to encode map e145 with expected crc 2023-06-03T20:04:34.084 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:04:33 smithi164 bash[59462]: cluster 1685822672.7068055 osd.5 (osd.5) 69 : cluster 3 failed to encode map e145 with expected crc 2023-06-03T20:04:34.084 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:04:33 smithi164 bash[59462]: audit 1685822672.708074 mon.a (mon.0) 749 : audit 1 from='client.? 172.21.15.89:0/507798448' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-06-03T20:04:34.084 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:04:33 smithi164 bash[59462]: cluster 1685822672.7082794 mon.a (mon.0) 750 : cluster 0 osdmap e145: 8 total, 8 up, 8 in 2023-06-03T20:04:34.085 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:04:34.085 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:04:34.186 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:04:34.219 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:04:34.232 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:04:34.232 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:04:34.278 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:04:33 smithi089 bash[75068]: cluster 1685822672.5095303 mgr.y (mgr.34104) 850 : cluster 0 pgmap v770: 89 pgs: 89 active+clean; 694 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 594 op/s 2023-06-03T20:04:34.279 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:04:33 smithi089 bash[75068]: cluster 1685822672.6911573 mon.a (mon.0) 748 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-06-03T20:04:34.279 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:04:33 smithi089 bash[75068]: cluster 1685822672.7030776 osd.6 (osd.6) 64 : cluster 3 failed to encode map e145 with expected crc 2023-06-03T20:04:34.279 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:04:33 smithi089 bash[75068]: cluster 1685822672.705488 osd.7 (osd.7) 57 : cluster 3 failed to encode map e145 with expected crc 2023-06-03T20:04:34.279 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:04:33 smithi089 bash[75068]: cluster 1685822672.7068055 osd.5 (osd.5) 69 : cluster 3 failed to encode map e145 with expected crc 2023-06-03T20:04:34.279 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:04:33 smithi089 bash[75068]: audit 1685822672.708074 mon.a (mon.0) 749 : audit 1 from='client.? 172.21.15.89:0/507798448' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-06-03T20:04:34.279 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:04:33 smithi089 bash[75068]: cluster 1685822672.7082794 mon.a (mon.0) 750 : cluster 0 osdmap e145: 8 total, 8 up, 8 in 2023-06-03T20:04:34.333 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:04:34.375 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:04:34.379 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:04:34.379 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:04:34.480 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:04:34.514 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:04:34.518 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:04:34.519 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:04:34.621 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:04:34.662 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:04:34.668 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:04:34.669 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:04:34.769 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:04:34.804 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:04:34.807 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:04:34.808 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:04:34.909 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:04:34.945 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:04:34.954 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:04:34.955 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:04:35.055 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:04:35.096 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:04:35.099 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:04:35.100 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:04:35.201 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:04:35.233 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:04:35.237 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:04:35.238 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:04:35.338 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:04:35.375 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:04:35.378 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:04:35.379 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:04:35.480 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:04:35.529 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:04:35.532 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:04:35.533 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:04:35.634 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:04:35.663 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:04:35.665 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:04:35.665 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:04:35.766 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:04:35.778 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:04:35 smithi089 bash[78452]: cluster 1685822674.5103912 mgr.y (mgr.34104) 851 : cluster 0 pgmap v772: 89 pgs: 89 active+clean; 514 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 715 op/s 2023-06-03T20:04:35.779 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:04:35 smithi089 bash[78452]: audit 1685822675.46739 mon.c (mon.2) 415 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-03T20:04:35.783 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:04:35 smithi089 bash[75068]: cluster 1685822674.5103912 mgr.y (mgr.34104) 851 : cluster 0 pgmap v772: 89 pgs: 89 active+clean; 514 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 715 op/s 2023-06-03T20:04:35.783 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:04:35 smithi089 bash[75068]: audit 1685822675.46739 mon.c (mon.2) 415 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-03T20:04:35.802 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:04:35.805 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:04:35.806 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:04:35.906 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:04:35.939 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:04:35.947 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:04:35.948 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:04:36.045 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:04:35 smithi164 bash[59462]: cluster 1685822674.5103912 mgr.y (mgr.34104) 851 : cluster 0 pgmap v772: 89 pgs: 89 active+clean; 514 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 715 op/s 2023-06-03T20:04:36.045 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:04:35 smithi164 bash[59462]: audit 1685822675.46739 mon.c (mon.2) 415 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-03T20:04:36.049 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:04:36.079 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:04:36.085 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:04:36.086 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:04:36.187 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:04:36.225 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:04:36.232 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:04:36.233 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:04:36.334 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:04:36.379 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:04:36.382 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:04:36.383 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:04:36.484 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:04:36.516 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:04:36.518 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:04:36.519 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:04:36.528 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 20:04:36 smithi089 bash[71163]: ::ffff:172.21.15.164 - - [03/Jun/2023:20:04:36] "GET /metrics HTTP/1.1" 200 228314 "" "Prometheus/2.33.4" 2023-06-03T20:04:36.620 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:04:36.671 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:04:36.671 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:04:36.673 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:04:36.774 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:04:36.802 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:04:36.804 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:04:36.805 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:04:36.906 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:04:36.939 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:04:36.942 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:04:36.943 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:04:37.044 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:04:37.076 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:04:37.082 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:04:37.082 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:04:37.183 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:04:37.226 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:04:37.230 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:04:37.230 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:04:37.331 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:04:37.365 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:04:37.370 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:04:37.371 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:04:37.472 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:04:37.505 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:04:37.511 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:04:37.512 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:04:37.603 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph osd unset nodeep-scrub 2023-06-03T20:04:37.612 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:04:37.660 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:04:37.661 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:04:37.662 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:04:37.729 INFO:teuthology.orchestra.run.smithi089.stderr:set pool 9 pgp_num to 16 2023-06-03T20:04:37.763 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:04:37.803 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:04:37.807 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:04:37.807 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:04:37.818 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-06-03T20:04:37.861 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:04:37 smithi089 bash[78452]: cluster 1685822676.510905 mgr.y (mgr.34104) 852 : cluster 0 pgmap v773: 89 pgs: 89 active+clean; 472 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 714 op/s 2023-06-03T20:04:37.862 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:04:37 smithi089 bash[78452]: audit 1685822677.10489 mon.c (mon.2) 416 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T20:04:37.862 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:04:37 smithi089 bash[78452]: audit 1685822677.2473888 mon.c (mon.2) 417 : audit 1 from='client.? 172.21.15.89:0/336389242' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num", "val": "16"}]: dispatch 2023-06-03T20:04:37.862 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:04:37 smithi089 bash[78452]: audit 1685822677.2477357 mon.a (mon.0) 751 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num", "val": "16"}]: dispatch 2023-06-03T20:04:37.864 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:04:37 smithi089 bash[75068]: cluster 1685822676.510905 mgr.y (mgr.34104) 852 : cluster 0 pgmap v773: 89 pgs: 89 active+clean; 472 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 714 op/s 2023-06-03T20:04:37.864 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:04:37 smithi089 bash[75068]: audit 1685822677.10489 mon.c (mon.2) 416 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T20:04:37.864 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:04:37 smithi089 bash[75068]: audit 1685822677.2473888 mon.c (mon.2) 417 : audit 1 from='client.? 172.21.15.89:0/336389242' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num", "val": "16"}]: dispatch 2023-06-03T20:04:37.865 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:04:37 smithi089 bash[75068]: audit 1685822677.2477357 mon.a (mon.0) 751 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num", "val": "16"}]: dispatch 2023-06-03T20:04:37.908 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:04:37.948 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:04:37.953 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:04:37.954 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:04:38.055 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:04:38.067 INFO:teuthology.orchestra.run.smithi164.stderr:Inferring config /var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/mon.b/config 2023-06-03T20:04:38.083 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:04:37 smithi164 bash[59462]: cluster 1685822676.510905 mgr.y (mgr.34104) 852 : cluster 0 pgmap v773: 89 pgs: 89 active+clean; 472 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 714 op/s 2023-06-03T20:04:38.083 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:04:37 smithi164 bash[59462]: audit 1685822677.10489 mon.c (mon.2) 416 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T20:04:38.084 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:04:37 smithi164 bash[59462]: audit 1685822677.2473888 mon.c (mon.2) 417 : audit 1 from='client.? 172.21.15.89:0/336389242' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num", "val": "16"}]: dispatch 2023-06-03T20:04:38.084 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:04:37 smithi164 bash[59462]: audit 1685822677.2477357 mon.a (mon.0) 751 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num", "val": "16"}]: dispatch 2023-06-03T20:04:38.088 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:04:38.091 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:04:38.092 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:04:38.193 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:04:38.237 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:04:38.240 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:04:38.241 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:04:38.342 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:04:38.390 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:04:38.396 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:04:38.397 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:04:38.498 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:04:38.538 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:04:38.540 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:04:38.540 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:04:38.641 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:04:38.683 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:04:38.691 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:04:38.692 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:04:38.745 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:04:38 smithi164 bash[59462]: cluster 1685822677.7066395 osd.6 (osd.6) 65 : cluster 3 failed to encode map e146 with expected crc 2023-06-03T20:04:38.745 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:04:38 smithi164 bash[59462]: audit 1685822677.7110903 mon.a (mon.0) 752 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num", "val": "16"}]': finished 2023-06-03T20:04:38.746 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:04:38 smithi164 bash[59462]: cluster 1685822677.7111773 mon.a (mon.0) 753 : cluster 0 osdmap e146: 8 total, 8 up, 8 in 2023-06-03T20:04:38.746 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:04:38 smithi164 bash[59462]: cluster 1685822677.7263281 osd.7 (osd.7) 58 : cluster 3 failed to encode map e146 with expected crc 2023-06-03T20:04:38.746 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:04:38 smithi164 bash[59462]: cluster 1685822677.7299938 osd.5 (osd.5) 70 : cluster 3 failed to encode map e146 with expected crc 2023-06-03T20:04:38.746 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:04:38 smithi164 bash[59462]: cluster 1685822677.7309766 osd.7 (osd.7) 59 : cluster 3 failed to encode map e146 with expected crc 2023-06-03T20:04:38.792 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:04:38.837 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:04:38.840 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:04:38.841 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:04:38.943 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:04:38.950 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:04:38 smithi089 bash[78452]: cluster 1685822677.7066395 osd.6 (osd.6) 65 : cluster 3 failed to encode map e146 with expected crc 2023-06-03T20:04:38.950 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:04:38 smithi089 bash[78452]: audit 1685822677.7110903 mon.a (mon.0) 752 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num", "val": "16"}]': finished 2023-06-03T20:04:38.951 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:04:38 smithi089 bash[78452]: cluster 1685822677.7111773 mon.a (mon.0) 753 : cluster 0 osdmap e146: 8 total, 8 up, 8 in 2023-06-03T20:04:38.951 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:04:38 smithi089 bash[78452]: cluster 1685822677.7263281 osd.7 (osd.7) 58 : cluster 3 failed to encode map e146 with expected crc 2023-06-03T20:04:38.951 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:04:38 smithi089 bash[78452]: cluster 1685822677.7299938 osd.5 (osd.5) 70 : cluster 3 failed to encode map e146 with expected crc 2023-06-03T20:04:38.952 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:04:38 smithi089 bash[78452]: cluster 1685822677.7309766 osd.7 (osd.7) 59 : cluster 3 failed to encode map e146 with expected crc 2023-06-03T20:04:38.952 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:04:38 smithi089 bash[75068]: cluster 1685822677.7066395 osd.6 (osd.6) 65 : cluster 3 failed to encode map e146 with expected crc 2023-06-03T20:04:38.953 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:04:38 smithi089 bash[75068]: audit 1685822677.7110903 mon.a (mon.0) 752 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num", "val": "16"}]': finished 2023-06-03T20:04:38.953 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:04:38 smithi089 bash[75068]: cluster 1685822677.7111773 mon.a (mon.0) 753 : cluster 0 osdmap e146: 8 total, 8 up, 8 in 2023-06-03T20:04:38.953 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:04:38 smithi089 bash[75068]: cluster 1685822677.7263281 osd.7 (osd.7) 58 : cluster 3 failed to encode map e146 with expected crc 2023-06-03T20:04:38.953 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:04:38 smithi089 bash[75068]: cluster 1685822677.7299938 osd.5 (osd.5) 70 : cluster 3 failed to encode map e146 with expected crc 2023-06-03T20:04:38.954 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:04:38 smithi089 bash[75068]: cluster 1685822677.7309766 osd.7 (osd.7) 59 : cluster 3 failed to encode map e146 with expected crc 2023-06-03T20:04:38.971 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:04:38.978 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:04:38.979 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:04:39.080 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:04:39.113 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:04:39.119 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:04:39.120 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:04:39.222 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:04:39.259 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:04:39.269 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:04:39.269 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:04:39.370 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:04:39.405 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:04:39.407 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:04:39.408 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:04:39.509 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:04:39.552 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:04:39.555 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:04:39.556 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:04:39.656 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:04:39.682 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:04:39.686 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:04:39.687 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:04:39.778 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:04:39 smithi089 bash[75068]: cluster 1685822678.5115333 mgr.y (mgr.34104) 853 : cluster 0 pgmap v775: 89 pgs: 89 active+clean; 323 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 742 op/s 2023-06-03T20:04:39.779 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:04:39 smithi089 bash[78452]: cluster 1685822678.5115333 mgr.y (mgr.34104) 853 : cluster 0 pgmap v775: 89 pgs: 89 active+clean; 323 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 742 op/s 2023-06-03T20:04:39.788 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:04:39.827 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:04:39.836 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:04:39.837 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:04:39.938 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:04:39.946 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:04:39 smithi164 bash[59462]: cluster 1685822678.5115333 mgr.y (mgr.34104) 853 : cluster 0 pgmap v775: 89 pgs: 89 active+clean; 323 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 742 op/s 2023-06-03T20:04:39.971 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:04:39.973 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:04:39.974 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:04:40.075 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:04:40.110 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:04:40.119 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:04:40.120 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:04:40.221 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:04:40.253 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:04:40.256 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:04:40.257 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:04:40.358 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:04:40.397 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:04:40.405 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:04:40.405 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:04:40.506 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:04:40.542 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:04:40.548 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:04:40.549 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:04:40.650 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:04:40.693 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:04:40.699 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:04:40.699 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:04:40.800 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:04:40.863 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:04:40.865 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:04:40.866 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:04:40.967 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:04:40.998 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:04:41.000 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:04:41.001 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:04:41.102 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:04:41.132 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:04:41.138 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:04:41.139 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:04:41.240 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:04:41.293 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:04:41.298 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:04:41.298 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:04:41.399 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:04:41.428 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:04:41.433 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:04:41.434 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:04:41.535 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:04:41.584 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:04:41.592 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:04:41.593 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:04:41.694 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:04:41.757 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:04:41.763 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:04:41.764 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:04:41.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:04:41 smithi164 bash[59462]: cluster 1685822680.5121043 mgr.y (mgr.34104) 854 : cluster 0 pgmap v776: 89 pgs: 89 active+clean; 254 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 881 op/s 2023-06-03T20:04:41.864 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:04:41.912 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:04:41.919 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:04:41.920 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:04:42.021 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:04:42.034 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:04:41 smithi089 bash[75068]: cluster 1685822680.5121043 mgr.y (mgr.34104) 854 : cluster 0 pgmap v776: 89 pgs: 89 active+clean; 254 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 881 op/s 2023-06-03T20:04:42.034 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:04:41 smithi089 bash[78452]: cluster 1685822680.5121043 mgr.y (mgr.34104) 854 : cluster 0 pgmap v776: 89 pgs: 89 active+clean; 254 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 881 op/s 2023-06-03T20:04:42.057 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:04:42.061 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:04:42.062 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:04:42.163 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:04:42.209 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:04:42.217 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:04:42.218 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:04:42.319 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:04:42.350 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:04:42.352 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:04:42.353 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:04:42.454 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:04:42.487 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:04:42.489 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:04:42.490 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:04:42.591 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:04:42.612 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/mon.a/config 2023-06-03T20:04:42.619 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:04:42.622 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:04:42.623 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:04:42.724 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:04:42.779 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:04:42.789 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:04:42.790 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:04:42.892 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:04:42.949 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:04:42.953 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:04:42.954 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:04:43.027 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:04:43 smithi089 bash[75068]: audit 1685822682.0251167 mon.a (mon.0) 754 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:04:43.028 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:04:43 smithi089 bash[75068]: audit 1685822682.0457294 mon.a (mon.0) 755 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:04:43.028 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:04:43 smithi089 bash[75068]: cluster 1685822682.512548 mgr.y (mgr.34104) 855 : cluster 0 pgmap v777: 89 pgs: 89 active+clean; 254 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 718 op/s 2023-06-03T20:04:43.056 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:04:43.090 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:04:43.093 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:04:43.093 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:04:43.180 INFO:teuthology.orchestra.run.smithi164.stdout:ERROR: (22) Invalid argument 2023-06-03T20:04:43.180 INFO:teuthology.orchestra.run.smithi164.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-06-03T20:04:43.195 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:04:43.215 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:04:43 smithi164 bash[59462]: audit 1685822682.0251167 mon.a (mon.0) 754 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:04:43.216 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:04:43 smithi164 bash[59462]: audit 1685822682.0457294 mon.a (mon.0) 755 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:04:43.216 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:04:43 smithi164 bash[59462]: cluster 1685822682.512548 mgr.y (mgr.34104) 855 : cluster 0 pgmap v777: 89 pgs: 89 active+clean; 254 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 718 op/s 2023-06-03T20:04:43.241 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:04:43.246 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:04:43.247 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:04:43.278 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:04:43 smithi089 bash[78452]: audit 1685822682.0251167 mon.a (mon.0) 754 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:04:43.279 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:04:43 smithi089 bash[78452]: audit 1685822682.0457294 mon.a (mon.0) 755 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:04:43.279 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:04:43 smithi089 bash[78452]: cluster 1685822682.512548 mgr.y (mgr.34104) 855 : cluster 0 pgmap v777: 89 pgs: 89 active+clean; 254 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 718 op/s 2023-06-03T20:04:43.348 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:04:43.391 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:04:43.393 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:04:43.393 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:04:43.494 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:04:43.533 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:04:43.535 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:04:43.536 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:04:43.637 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:04:43.638 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/mon.a/config 2023-06-03T20:04:43.668 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:04:43.671 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:04:43.671 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:04:43.772 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:04:43.814 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:04:43.818 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:04:43.818 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:04:43.919 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:04:43.953 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:04:43.960 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:04:43.961 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:04:44.022 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 5, 6, 7] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-06-03T20:04:44.022 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-06-03T20:04:44.022 INFO:tasks.thrashosds.thrasher:Growing pool unique_pool_4 2023-06-03T20:04:44.023 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph pg dump --format=json 2023-06-03T20:04:44.062 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:04:44.111 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:04:44.118 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:04:44.118 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:04:44.219 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:04:44.234 DEBUG:teuthology.orchestra.run.smithi164:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph daemon osd.4 dump_historic_ops 2023-06-03T20:04:44.252 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:04:44.259 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:04:44.259 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:04:44.360 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:04:44.408 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:04:44.416 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:04:44.417 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:04:44.518 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:04:44.527 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:04:44 smithi089 bash[78452]: audit 1685822683.255794 mon.a (mon.0) 756 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:04:44.528 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:04:44 smithi089 bash[78452]: audit 1685822683.2677844 mon.a (mon.0) 757 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:04:44.528 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:04:44 smithi089 bash[78452]: audit 1685822683.8951147 mon.c (mon.2) 418 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-03T20:04:44.529 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:04:44 smithi089 bash[78452]: audit 1685822683.8965926 mon.c (mon.2) 419 : audit 1 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-03T20:04:44.529 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:04:44 smithi089 bash[78452]: audit 1685822683.9038403 mon.a (mon.0) 758 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:04:44.529 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:04:44 smithi089 bash[75068]: audit 1685822683.255794 mon.a (mon.0) 756 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:04:44.529 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:04:44 smithi089 bash[75068]: audit 1685822683.2677844 mon.a (mon.0) 757 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:04:44.529 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:04:44 smithi089 bash[75068]: audit 1685822683.8951147 mon.c (mon.2) 418 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-03T20:04:44.529 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:04:44 smithi089 bash[75068]: audit 1685822683.8965926 mon.c (mon.2) 419 : audit 1 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-03T20:04:44.531 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:04:44 smithi089 bash[75068]: audit 1685822683.9038403 mon.a (mon.0) 758 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:04:44.553 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:04:44.555 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:04:44.556 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:04:44.583 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:04:44 smithi164 bash[59462]: audit 1685822683.255794 mon.a (mon.0) 756 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:04:44.584 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:04:44 smithi164 bash[59462]: audit 1685822683.2677844 mon.a (mon.0) 757 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:04:44.584 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:04:44 smithi164 bash[59462]: audit 1685822683.8951147 mon.c (mon.2) 418 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-03T20:04:44.584 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:04:44 smithi164 bash[59462]: audit 1685822683.8965926 mon.c (mon.2) 419 : audit 1 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-03T20:04:44.584 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:04:44 smithi164 bash[59462]: audit 1685822683.9038403 mon.a (mon.0) 758 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:04:44.657 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:04:44.694 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:04:44.704 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:04:44.705 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:04:44.778 INFO:tasks.radosbench.radosbench.0.smithi089.stdout:Removed 110131 objects 2023-06-03T20:04:44.778 INFO:tasks.radosbench.radosbench.0.smithi089.stdout:Clean up completed and total clean up time :153.567 2023-06-03T20:04:44.806 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:04:44.840 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:04:44.843 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:04:44.844 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:04:44.945 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:04:44.984 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:04:44.986 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:04:44.987 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:04:45.088 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:04:45.123 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:04:45.125 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:04:45.126 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:04:45.227 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:04:45.255 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:04:45.257 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:04:45.258 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:04:45.359 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:04:45.398 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:04:45.400 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:04:45.401 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:04:45.502 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:04:45.528 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:04:45 smithi089 bash[75068]: cluster 1685822684.5132225 mgr.y (mgr.34104) 856 : cluster 0 pgmap v778: 89 pgs: 89 active+clean; 85 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 686 op/s 2023-06-03T20:04:45.529 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:04:45 smithi089 bash[78452]: cluster 1685822684.5132225 mgr.y (mgr.34104) 856 : cluster 0 pgmap v778: 89 pgs: 89 active+clean; 85 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 686 op/s 2023-06-03T20:04:45.536 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:04:45.538 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:04:45 smithi164 bash[59462]: cluster 1685822684.5132225 mgr.y (mgr.34104) 856 : cluster 0 pgmap v778: 89 pgs: 89 active+clean; 85 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 686 op/s 2023-06-03T20:04:45.538 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:04:45.539 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:04:45.640 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:04:45.676 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:04:45.678 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:04:45.679 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:04:45.780 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:04:45.817 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:04:45.820 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:04:45.820 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:04:45.921 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:04:45.958 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:04:45.960 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:04:45.960 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:04:46.061 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:04:46.098 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:04:46.101 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:04:46.101 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:04:46.202 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:04:46.238 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:04:46.240 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:04:46.241 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:04:46.342 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:04:46.376 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:04:46.379 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:04:46.379 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:04:46.480 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:04:46.493 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 20:04:46 smithi089 bash[71163]: ::ffff:172.21.15.164 - - [03/Jun/2023:20:04:46] "GET /metrics HTTP/1.1" 200 228294 "" "Prometheus/2.33.4" 2023-06-03T20:04:46.515 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:04:46.517 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:04:46.518 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:04:46.619 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:04:46.643 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:04:46.644 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:04:46.645 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:04:46.746 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:04:46.775 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:04:46.777 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:04:46.777 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:04:46.878 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:04:46.911 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:04:46.914 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:04:46.915 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:04:47.016 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:04:47.053 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:04:47.057 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:04:47.058 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:04:47.159 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:04:47.183 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:04:47.185 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:04:47.185 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:04:47.286 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:04:47.311 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:04:47.313 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:04:47.314 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:04:47.415 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:04:47.450 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:04:47.453 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:04:47.454 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:04:47.555 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:04:47.586 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:04:47.588 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:04:47.588 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:04:47.689 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:04:47.724 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:04:47.727 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:04:47.728 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:04:47.829 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:04:47.833 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:04:47 smithi164 bash[59462]: cluster 1685822686.513715 mgr.y (mgr.34104) 857 : cluster 0 pgmap v779: 89 pgs: 89 active+clean; 49 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 676 op/s 2023-06-03T20:04:47.840 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:04:47 smithi089 bash[78452]: cluster 1685822686.513715 mgr.y (mgr.34104) 857 : cluster 0 pgmap v779: 89 pgs: 89 active+clean; 49 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 676 op/s 2023-06-03T20:04:47.841 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:04:47 smithi089 bash[75068]: cluster 1685822686.513715 mgr.y (mgr.34104) 857 : cluster 0 pgmap v779: 89 pgs: 89 active+clean; 49 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 676 op/s 2023-06-03T20:04:47.858 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:04:47.861 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:04:47.862 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:04:47.963 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:04:47.998 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:04:48.001 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:04:48.001 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:04:48.102 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:04:48.144 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:04:48.146 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:04:48.147 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:04:48.248 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:04:48.280 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:04:48.283 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:04:48.284 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:04:48.385 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:04:48.421 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:04:48.424 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:04:48.425 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:04:48.525 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:04:48.562 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:04:48.565 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:04:48.566 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:04:48.667 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:04:48.709 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:04:48.711 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:04:48.712 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:04:48.813 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:04:48.845 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:04:48.847 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:04:48.848 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:04:48.949 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:04:48.978 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:04:48.981 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:04:48.981 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:04:49.082 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:04:49.098 INFO:teuthology.orchestra.run.smithi164.stderr:Inferring config /var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/mon.b/config 2023-06-03T20:04:49.111 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:04:49.113 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:04:49.114 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:04:49.215 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:04:49.248 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:04:49.250 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:04:49.250 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:04:49.351 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:04:49.380 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:04:49.382 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:04:49.382 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:04:49.483 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:04:49.521 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:04:49.524 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:04:49.525 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:04:49.626 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:04:49.661 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:04:49.663 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:04:49.664 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:04:49.739 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:04:49 smithi089 bash[75068]: cluster 1685822688.514631 mgr.y (mgr.34104) 858 : cluster 0 pgmap v780: 89 pgs: 89 active+clean; 6.8 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 688 op/s 2023-06-03T20:04:49.765 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:04:49.775 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:04:49 smithi164 bash[59462]: cluster 1685822688.514631 mgr.y (mgr.34104) 858 : cluster 0 pgmap v780: 89 pgs: 89 active+clean; 6.8 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 688 op/s 2023-06-03T20:04:49.796 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:04:49.798 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:04:49.799 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:04:49.900 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:04:49.929 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:04:49.932 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:04:49.933 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:04:50.028 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:04:49 smithi089 bash[78452]: cluster 1685822688.514631 mgr.y (mgr.34104) 858 : cluster 0 pgmap v780: 89 pgs: 89 active+clean; 6.8 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 688 op/s 2023-06-03T20:04:50.034 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:04:50.065 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:04:50.067 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:04:50.067 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:04:50.168 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:04:50.203 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:04:50.205 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:04:50.206 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:04:50.307 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:04:50.336 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:04:50.339 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:04:50.339 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:04:50.440 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:04:50.468 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:04:50.470 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:04:50.471 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:04:50.572 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:04:50.613 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:04:50.614 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:04:50.614 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:04:50.715 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:04:50.749 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:04:50.752 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:04:50.753 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:04:50.853 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:04:50.882 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:04:50.884 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:04:50.885 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:04:50.886 INFO:teuthology.orchestra.run.smithi089.stdout:osd.0: {} 2023-06-03T20:04:50.900 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/mon.a/config 2023-06-03T20:04:50.912 INFO:teuthology.orchestra.run.smithi089.stdout:osd.1: {} 2023-06-03T20:04:50.943 INFO:teuthology.orchestra.run.smithi089.stdout:osd.2: {} 2023-06-03T20:04:50.965 INFO:teuthology.orchestra.run.smithi089.stdout:osd.3: {} 2023-06-03T20:04:50.986 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:04:50.995 INFO:teuthology.orchestra.run.smithi089.stdout:osd.4: {} 2023-06-03T20:04:51.017 INFO:teuthology.orchestra.run.smithi089.stdout:osd.5: {} 2023-06-03T20:04:51.021 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:04:51.024 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:04:51.024 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:04:51.046 INFO:teuthology.orchestra.run.smithi089.stdout:osd.6: {} 2023-06-03T20:04:51.072 INFO:teuthology.orchestra.run.smithi089.stdout:osd.7: {} 2023-06-03T20:04:51.074 INFO:teuthology.orchestra.run.smithi089.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-06-03T20:04:51.074 INFO:teuthology.orchestra.run.smithi089.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-06-03T20:04:51.075 INFO:teuthology.orchestra.run.smithi089.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-06-03T20:04:51.075 INFO:teuthology.orchestra.run.smithi089.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-06-03T20:04:51.075 INFO:teuthology.orchestra.run.smithi089.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-06-03T20:04:51.075 INFO:teuthology.orchestra.run.smithi089.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-06-03T20:04:51.075 INFO:teuthology.orchestra.run.smithi089.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-06-03T20:04:51.075 INFO:teuthology.orchestra.run.smithi089.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-06-03T20:04:51.125 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:04:51.157 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:04:51.160 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:04:51.161 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:04:51.261 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:04:51.297 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:04:51.300 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:04:51.301 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:04:51.402 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:04:51.433 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:04:51.437 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:04:51.438 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:04:51.538 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:04:51.573 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:04:51.576 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:04:51.576 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:04:51.621 INFO:teuthology.orchestra.run.smithi089.stderr:nodeep-scrub is unset 2023-06-03T20:04:51.677 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:04:51.712 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:04:51.715 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:04:51.716 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:04:51.817 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:04:51.833 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:04:51 smithi164 bash[59462]: cluster 1685822690.5153856 mgr.y (mgr.34104) 859 : cluster 0 pgmap v781: 89 pgs: 89 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 430 op/s 2023-06-03T20:04:51.834 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:04:51 smithi164 bash[59462]: audit 1685822690.85373 mon.a (mon.0) 759 : audit 0 from='client.? 172.21.15.89:0/2271866598' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-03T20:04:51.834 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:04:51 smithi164 bash[59462]: audit 1685822691.0976033 mon.a (mon.0) 760 : audit 1 from='client.? 172.21.15.89:0/3125691012' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-06-03T20:04:51.843 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:04:51.845 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:04:51 smithi089 bash[75068]: cluster 1685822690.5153856 mgr.y (mgr.34104) 859 : cluster 0 pgmap v781: 89 pgs: 89 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 430 op/s 2023-06-03T20:04:51.846 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:04:51 smithi089 bash[75068]: audit 1685822690.85373 mon.a (mon.0) 759 : audit 0 from='client.? 172.21.15.89:0/2271866598' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-03T20:04:51.846 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:04:51 smithi089 bash[75068]: audit 1685822691.0976033 mon.a (mon.0) 760 : audit 1 from='client.? 172.21.15.89:0/3125691012' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-06-03T20:04:51.846 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:04:51 smithi089 bash[78452]: cluster 1685822690.5153856 mgr.y (mgr.34104) 859 : cluster 0 pgmap v781: 89 pgs: 89 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 430 op/s 2023-06-03T20:04:51.846 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:04:51 smithi089 bash[78452]: audit 1685822690.85373 mon.a (mon.0) 759 : audit 0 from='client.? 172.21.15.89:0/2271866598' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-03T20:04:51.847 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:04:51 smithi089 bash[78452]: audit 1685822691.0976033 mon.a (mon.0) 760 : audit 1 from='client.? 172.21.15.89:0/3125691012' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-06-03T20:04:51.847 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:04:51.848 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:04:51.949 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:04:51.979 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:04:51.981 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:04:51.982 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:04:52.083 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:04:52.114 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:04:52.117 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:04:52.118 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:04:52.219 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:04:52.254 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:04:52.256 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:04:52.257 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:04:52.358 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:04:52.398 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:04:52.400 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:04:52.401 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:04:52.501 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:04:52.538 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:04:52.541 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:04:52.542 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:04:52.643 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:04:52.678 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:04:52.681 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:04:52.681 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:04:52.782 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:04:52.794 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:04:52 smithi089 bash[75068]: cluster 1685822691.6028855 mon.a (mon.0) 761 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-06-03T20:04:52.794 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:04:52 smithi089 bash[75068]: cluster 1685822691.6029415 mon.a (mon.0) 762 : cluster 1 Cluster is now healthy 2023-06-03T20:04:52.794 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:04:52 smithi089 bash[75068]: cluster 1685822691.6043396 osd.6 (osd.6) 66 : cluster 3 failed to encode map e147 with expected crc 2023-06-03T20:04:52.794 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:04:52 smithi089 bash[75068]: cluster 1685822691.6051497 osd.5 (osd.5) 71 : cluster 3 failed to encode map e147 with expected crc 2023-06-03T20:04:52.794 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:04:52 smithi089 bash[75068]: audit 1685822691.6097655 mon.a (mon.0) 763 : audit 1 from='client.? 172.21.15.89:0/3125691012' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-06-03T20:04:52.795 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:04:52 smithi089 bash[75068]: cluster 1685822691.6098905 mon.a (mon.0) 764 : cluster 0 osdmap e147: 8 total, 8 up, 8 in 2023-06-03T20:04:52.795 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:04:52 smithi089 bash[75068]: audit 1685822692.1090522 mon.a (mon.0) 765 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:04:52.795 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:04:52 smithi089 bash[75068]: audit 1685822692.1123424 mon.c (mon.2) 420 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T20:04:52.795 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:04:52 smithi089 bash[78452]: cluster 1685822691.6028855 mon.a (mon.0) 761 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-06-03T20:04:52.795 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:04:52 smithi089 bash[78452]: cluster 1685822691.6029415 mon.a (mon.0) 762 : cluster 1 Cluster is now healthy 2023-06-03T20:04:52.796 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:04:52 smithi089 bash[78452]: cluster 1685822691.6043396 osd.6 (osd.6) 66 : cluster 3 failed to encode map e147 with expected crc 2023-06-03T20:04:52.796 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:04:52 smithi089 bash[78452]: cluster 1685822691.6051497 osd.5 (osd.5) 71 : cluster 3 failed to encode map e147 with expected crc 2023-06-03T20:04:52.796 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:04:52 smithi089 bash[78452]: audit 1685822691.6097655 mon.a (mon.0) 763 : audit 1 from='client.? 172.21.15.89:0/3125691012' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-06-03T20:04:52.796 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:04:52 smithi089 bash[78452]: cluster 1685822691.6098905 mon.a (mon.0) 764 : cluster 0 osdmap e147: 8 total, 8 up, 8 in 2023-06-03T20:04:52.796 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:04:52 smithi089 bash[78452]: audit 1685822692.1090522 mon.a (mon.0) 765 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:04:52.796 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:04:52 smithi089 bash[78452]: audit 1685822692.1123424 mon.c (mon.2) 420 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T20:04:52.821 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:04:52.824 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:04:52.825 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:04:52.925 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:04:52.939 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:04:52 smithi164 bash[59462]: cluster 1685822691.6028855 mon.a (mon.0) 761 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-06-03T20:04:52.940 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:04:52 smithi164 bash[59462]: cluster 1685822691.6029415 mon.a (mon.0) 762 : cluster 1 Cluster is now healthy 2023-06-03T20:04:52.940 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:04:52 smithi164 bash[59462]: cluster 1685822691.6043396 osd.6 (osd.6) 66 : cluster 3 failed to encode map e147 with expected crc 2023-06-03T20:04:52.940 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:04:52 smithi164 bash[59462]: cluster 1685822691.6051497 osd.5 (osd.5) 71 : cluster 3 failed to encode map e147 with expected crc 2023-06-03T20:04:52.940 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:04:52 smithi164 bash[59462]: audit 1685822691.6097655 mon.a (mon.0) 763 : audit 1 from='client.? 172.21.15.89:0/3125691012' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-06-03T20:04:52.941 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:04:52 smithi164 bash[59462]: cluster 1685822691.6098905 mon.a (mon.0) 764 : cluster 0 osdmap e147: 8 total, 8 up, 8 in 2023-06-03T20:04:52.941 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:04:52 smithi164 bash[59462]: audit 1685822692.1090522 mon.a (mon.0) 765 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:04:52.941 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:04:52 smithi164 bash[59462]: audit 1685822692.1123424 mon.c (mon.2) 420 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T20:04:52.963 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:04:52.965 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:04:52.965 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:04:53.066 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:04:53.101 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:04:53.103 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:04:53.104 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:04:53.205 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:04:53.230 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:04:53.232 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:04:53.233 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:04:53.334 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:04:53.346 INFO:teuthology.orchestra.run.smithi164.stdout:{ 2023-06-03T20:04:53.346 INFO:teuthology.orchestra.run.smithi164.stdout: "size": 20, 2023-06-03T20:04:53.347 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 600, 2023-06-03T20:04:53.347 INFO:teuthology.orchestra.run.smithi164.stdout: "ops": [ 2023-06-03T20:04:53.347 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:04:53.348 INFO:teuthology.orchestra.run.smithi164.stdout: "description": "osd_repop(client.45713.0:103888 9.4 e134/127 9:2aad781c:::benchmark_data_smithi089_115580_object103887:head v 134'6548, mlcod=134'6545)", 2023-06-03T20:04:53.348 INFO:teuthology.orchestra.run.smithi164.stdout: "initiated_at": "2023-06-03T20:02:03.329084+0000", 2023-06-03T20:04:53.348 INFO:teuthology.orchestra.run.smithi164.stdout: "age": 170.00708976999999, 2023-06-03T20:04:53.348 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.028251533999999998, 2023-06-03T20:04:53.349 INFO:teuthology.orchestra.run.smithi164.stdout: "type_data": { 2023-06-03T20:04:53.349 INFO:teuthology.orchestra.run.smithi164.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-03T20:04:53.349 INFO:teuthology.orchestra.run.smithi164.stdout: "events": [ 2023-06-03T20:04:53.349 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:04:53.350 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "initiated", 2023-06-03T20:04:53.350 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:02:03.329084+0000", 2023-06-03T20:04:53.350 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0 2023-06-03T20:04:53.350 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:04:53.351 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:04:53.351 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "throttled", 2023-06-03T20:04:53.351 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:02:03.329084+0000", 2023-06-03T20:04:53.351 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0 2023-06-03T20:04:53.352 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:04:53.352 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:04:53.352 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "header_read", 2023-06-03T20:04:53.353 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:02:03.329082+0000", 2023-06-03T20:04:53.353 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 4294967295.9999986 2023-06-03T20:04:53.353 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:04:53.353 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:04:53.354 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "all_read", 2023-06-03T20:04:53.354 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:02:03.329156+0000", 2023-06-03T20:04:53.354 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 7.3687000000000001e-05 2023-06-03T20:04:53.354 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:04:53.355 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:04:53.355 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "dispatched", 2023-06-03T20:04:53.355 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:02:03.329174+0000", 2023-06-03T20:04:53.355 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 1.8176999999999999e-05 2023-06-03T20:04:53.356 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:04:53.356 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:04:53.356 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "queued_for_pg", 2023-06-03T20:04:53.356 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:02:03.329184+0000", 2023-06-03T20:04:53.357 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 9.5019999999999998e-06 2023-06-03T20:04:53.357 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:04:53.357 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:04:53.357 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "reached_pg", 2023-06-03T20:04:53.358 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:02:03.335649+0000", 2023-06-03T20:04:53.358 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.0064649549999999997 2023-06-03T20:04:53.358 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:04:53.358 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:04:53.359 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "started", 2023-06-03T20:04:53.359 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:02:03.335726+0000", 2023-06-03T20:04:53.359 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 7.7750999999999995e-05 2023-06-03T20:04:53.359 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:04:53.360 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:04:53.360 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "commit_sent", 2023-06-03T20:04:53.360 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:02:03.357196+0000", 2023-06-03T20:04:53.361 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.021469414999999999 2023-06-03T20:04:53.361 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:04:53.361 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:04:53.361 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "done", 2023-06-03T20:04:53.362 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:02:03.357335+0000", 2023-06-03T20:04:53.362 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.00013925 2023-06-03T20:04:53.362 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:04:53.362 INFO:teuthology.orchestra.run.smithi164.stdout: ] 2023-06-03T20:04:53.363 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:04:53.363 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:04:53.363 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:04:53.363 INFO:teuthology.orchestra.run.smithi164.stdout: "description": "osd_repop(client.45713.0:103890 9.4 e134/127 9:2ccbc956:::benchmark_data_smithi089_115580_object103889:head v 134'6549, mlcod=134'6545)", 2023-06-03T20:04:53.364 INFO:teuthology.orchestra.run.smithi164.stdout: "initiated_at": "2023-06-03T20:02:03.332769+0000", 2023-06-03T20:04:53.364 INFO:teuthology.orchestra.run.smithi164.stdout: "age": 170.00340456500001, 2023-06-03T20:04:53.364 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.027522614000000001, 2023-06-03T20:04:53.364 INFO:teuthology.orchestra.run.smithi164.stdout: "type_data": { 2023-06-03T20:04:53.365 INFO:teuthology.orchestra.run.smithi164.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-03T20:04:53.365 INFO:teuthology.orchestra.run.smithi164.stdout: "events": [ 2023-06-03T20:04:53.365 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:04:53.365 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "initiated", 2023-06-03T20:04:53.366 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:02:03.332769+0000", 2023-06-03T20:04:53.366 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0 2023-06-03T20:04:53.366 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:04:53.366 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:04:53.367 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "throttled", 2023-06-03T20:04:53.367 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:02:03.332769+0000", 2023-06-03T20:04:53.367 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0 2023-06-03T20:04:53.367 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:04:53.368 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:04:53.368 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "header_read", 2023-06-03T20:04:53.368 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:02:03.332768+0000", 2023-06-03T20:04:53.368 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 4294967295.999999 2023-06-03T20:04:53.369 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:04:53.369 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:04:53.369 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "all_read", 2023-06-03T20:04:53.369 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:02:03.332901+0000", 2023-06-03T20:04:53.370 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.00013306700000000001 2023-06-03T20:04:53.370 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:04:53.370 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:04:53.370 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "dispatched", 2023-06-03T20:04:53.371 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:02:03.332909+0000", 2023-06-03T20:04:53.371 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 8.0499999999999992e-06 2023-06-03T20:04:53.371 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:04:53.371 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:04:53.372 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "queued_for_pg", 2023-06-03T20:04:53.372 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:02:03.332914+0000", 2023-06-03T20:04:53.372 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 4.8509999999999998e-06 2023-06-03T20:04:53.373 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:04:53.373 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:04:53.373 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "reached_pg", 2023-06-03T20:04:53.373 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:02:03.339319+0000", 2023-06-03T20:04:53.374 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.0064049220000000004 2023-06-03T20:04:53.374 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:04:53.374 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:04:53.374 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "started", 2023-06-03T20:04:53.375 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:02:03.339341+0000", 2023-06-03T20:04:53.375 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 2.1891000000000001e-05 2023-06-03T20:04:53.375 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:04:53.375 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:04:53.376 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "commit_sent", 2023-06-03T20:04:53.376 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:02:03.360119+0000", 2023-06-03T20:04:53.376 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.020778665000000002 2023-06-03T20:04:53.376 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:04:53.377 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:04:53.377 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "done", 2023-06-03T20:04:53.377 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:02:03.360291+0000", 2023-06-03T20:04:53.377 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.00017191400000000001 2023-06-03T20:04:53.378 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:04:53.378 INFO:teuthology.orchestra.run.smithi164.stdout: ] 2023-06-03T20:04:53.378 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:04:53.378 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:04:53.379 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:04:53.379 INFO:teuthology.orchestra.run.smithi164.stdout: "description": "osd_repop(client.45713.0:135984 9.4 e138/127 9:2837551c:::benchmark_data_smithi089_115580_object25850:head v 138'8555, mlcod=138'8553)", 2023-06-03T20:04:53.379 INFO:teuthology.orchestra.run.smithi164.stdout: "initiated_at": "2023-06-03T20:02:44.827017+0000", 2023-06-03T20:04:53.379 INFO:teuthology.orchestra.run.smithi164.stdout: "age": 128.50915636900001, 2023-06-03T20:04:53.380 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.024843897, 2023-06-03T20:04:53.380 INFO:teuthology.orchestra.run.smithi164.stdout: "type_data": { 2023-06-03T20:04:53.380 INFO:teuthology.orchestra.run.smithi164.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-03T20:04:53.381 INFO:teuthology.orchestra.run.smithi164.stdout: "events": [ 2023-06-03T20:04:53.381 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:04:53.381 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "initiated", 2023-06-03T20:04:53.381 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:02:44.827017+0000", 2023-06-03T20:04:53.382 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0 2023-06-03T20:04:53.382 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:04:53.382 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:04:53.382 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "throttled", 2023-06-03T20:04:53.383 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:02:44.827017+0000", 2023-06-03T20:04:53.383 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0 2023-06-03T20:04:53.383 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:04:53.383 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:04:53.384 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "header_read", 2023-06-03T20:04:53.384 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:02:44.827016+0000", 2023-06-03T20:04:53.384 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 4294967295.999999 2023-06-03T20:04:53.384 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:04:53.385 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:04:53.385 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "all_read", 2023-06-03T20:04:53.385 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:02:44.827027+0000", 2023-06-03T20:04:53.385 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 1.1647999999999999e-05 2023-06-03T20:04:53.386 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:04:53.386 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:04:53.386 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "dispatched", 2023-06-03T20:04:53.386 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:02:44.827051+0000", 2023-06-03T20:04:53.387 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 2.3249999999999999e-05 2023-06-03T20:04:53.387 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:04:53.387 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:04:53.387 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "queued_for_pg", 2023-06-03T20:04:53.388 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:02:44.827061+0000", 2023-06-03T20:04:53.388 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 1.0584000000000001e-05 2023-06-03T20:04:53.388 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:04:53.388 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:04:53.389 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "reached_pg", 2023-06-03T20:04:53.389 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:02:44.827169+0000", 2023-06-03T20:04:53.389 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.000107876 2023-06-03T20:04:53.389 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:04:53.389 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:04:53.389 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "started", 2023-06-03T20:04:53.390 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:02:44.827225+0000", 2023-06-03T20:04:53.390 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 5.5711000000000002e-05 2023-06-03T20:04:53.390 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:04:53.390 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:04:53.390 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "commit_sent", 2023-06-03T20:04:53.390 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:02:44.851795+0000", 2023-06-03T20:04:53.390 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.024570609 2023-06-03T20:04:53.390 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:04:53.391 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:04:53.391 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "done", 2023-06-03T20:04:53.391 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:02:44.851861+0000", 2023-06-03T20:04:53.391 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 6.5388999999999993e-05 2023-06-03T20:04:53.391 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:04:53.391 INFO:teuthology.orchestra.run.smithi164.stdout: ] 2023-06-03T20:04:53.391 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:04:53.392 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:04:53.392 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:04:53.392 INFO:teuthology.orchestra.run.smithi164.stdout: "description": "osd_repop(client.45713.0:136510 9.c e138/127 9:3046c07e:::benchmark_data_smithi089_115580_object26376:head v 138'8552, mlcod=138'8549)", 2023-06-03T20:04:53.392 INFO:teuthology.orchestra.run.smithi164.stdout: "initiated_at": "2023-06-03T20:02:45.636650+0000", 2023-06-03T20:04:53.392 INFO:teuthology.orchestra.run.smithi164.stdout: "age": 127.699523564, 2023-06-03T20:04:53.392 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.025948787000000001, 2023-06-03T20:04:53.392 INFO:teuthology.orchestra.run.smithi164.stdout: "type_data": { 2023-06-03T20:04:53.393 INFO:teuthology.orchestra.run.smithi164.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-03T20:04:53.393 INFO:teuthology.orchestra.run.smithi164.stdout: "events": [ 2023-06-03T20:04:53.393 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:04:53.393 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "initiated", 2023-06-03T20:04:53.393 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:02:45.636650+0000", 2023-06-03T20:04:53.393 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0 2023-06-03T20:04:53.393 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:04:53.393 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:04:53.394 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "throttled", 2023-06-03T20:04:53.394 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:02:45.636650+0000", 2023-06-03T20:04:53.394 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0 2023-06-03T20:04:53.394 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:04:53.394 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:04:53.394 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "header_read", 2023-06-03T20:04:53.394 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:02:45.636648+0000", 2023-06-03T20:04:53.394 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 4294967295.9999976 2023-06-03T20:04:53.395 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:04:53.395 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:04:53.395 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "all_read", 2023-06-03T20:04:53.395 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:02:45.636666+0000", 2023-06-03T20:04:53.395 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 1.8646000000000001e-05 2023-06-03T20:04:53.395 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:04:53.395 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:04:53.396 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "dispatched", 2023-06-03T20:04:53.396 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:02:45.636698+0000", 2023-06-03T20:04:53.396 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 3.1492999999999998e-05 2023-06-03T20:04:53.396 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:04:53.396 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:04:53.396 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "queued_for_pg", 2023-06-03T20:04:53.396 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:02:45.636719+0000", 2023-06-03T20:04:53.396 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 2.1653000000000001e-05 2023-06-03T20:04:53.397 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:04:53.397 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:04:53.397 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "reached_pg", 2023-06-03T20:04:53.397 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:02:45.636813+0000", 2023-06-03T20:04:53.397 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 9.3857000000000006e-05 2023-06-03T20:04:53.397 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:04:53.397 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:04:53.397 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "started", 2023-06-03T20:04:53.398 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:02:45.636880+0000", 2023-06-03T20:04:53.398 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 6.7200999999999996e-05 2023-06-03T20:04:53.398 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:04:53.398 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:04:53.398 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "commit_sent", 2023-06-03T20:04:53.398 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:02:45.662546+0000", 2023-06-03T20:04:53.398 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.025665804 2023-06-03T20:04:53.398 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:04:53.399 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:04:53.399 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "done", 2023-06-03T20:04:53.399 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:02:45.662599+0000", 2023-06-03T20:04:53.399 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 5.2286e-05 2023-06-03T20:04:53.399 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:04:53.399 INFO:teuthology.orchestra.run.smithi164.stdout: ] 2023-06-03T20:04:53.399 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:04:53.400 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:04:53.400 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:04:53.400 INFO:teuthology.orchestra.run.smithi164.stdout: "description": "osd_repop(client.45713.0:136516 9.c e138/127 9:3cf60302:::benchmark_data_smithi089_115580_object26382:head v 138'8553, mlcod=138'8550)", 2023-06-03T20:04:53.400 INFO:teuthology.orchestra.run.smithi164.stdout: "initiated_at": "2023-06-03T20:02:45.641612+0000", 2023-06-03T20:04:53.400 INFO:teuthology.orchestra.run.smithi164.stdout: "age": 127.69456174600001, 2023-06-03T20:04:53.400 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.028536664999999999, 2023-06-03T20:04:53.400 INFO:teuthology.orchestra.run.smithi164.stdout: "type_data": { 2023-06-03T20:04:53.400 INFO:teuthology.orchestra.run.smithi164.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-03T20:04:53.401 INFO:teuthology.orchestra.run.smithi164.stdout: "events": [ 2023-06-03T20:04:53.401 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:04:53.401 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "initiated", 2023-06-03T20:04:53.401 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:02:45.641612+0000", 2023-06-03T20:04:53.401 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0 2023-06-03T20:04:53.401 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:04:53.401 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:04:53.401 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "throttled", 2023-06-03T20:04:53.402 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:02:45.641612+0000", 2023-06-03T20:04:53.402 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0 2023-06-03T20:04:53.402 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:04:53.402 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:04:53.402 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "header_read", 2023-06-03T20:04:53.402 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:02:45.641609+0000", 2023-06-03T20:04:53.402 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 4294967295.9999976 2023-06-03T20:04:53.402 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:04:53.403 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:04:53.403 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "all_read", 2023-06-03T20:04:53.403 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:02:45.641634+0000", 2023-06-03T20:04:53.403 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 2.4669999999999999e-05 2023-06-03T20:04:53.403 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:04:53.403 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:04:53.403 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "dispatched", 2023-06-03T20:04:53.403 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:02:45.641682+0000", 2023-06-03T20:04:53.404 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 4.8588999999999998e-05 2023-06-03T20:04:53.404 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:04:53.404 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:04:53.404 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "queued_for_pg", 2023-06-03T20:04:53.404 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:02:45.641705+0000", 2023-06-03T20:04:53.404 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 2.2586e-05 2023-06-03T20:04:53.404 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:04:53.405 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:04:53.405 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "reached_pg", 2023-06-03T20:04:53.405 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:02:45.642281+0000", 2023-06-03T20:04:53.405 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.00057584200000000002 2023-06-03T20:04:53.405 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:04:53.405 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:04:53.405 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "started", 2023-06-03T20:04:53.405 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:02:45.642429+0000", 2023-06-03T20:04:53.406 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.00014821800000000001 2023-06-03T20:04:53.406 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:04:53.406 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:04:53.406 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "commit_sent", 2023-06-03T20:04:53.406 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:02:45.670039+0000", 2023-06-03T20:04:53.406 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.027610279000000001 2023-06-03T20:04:53.406 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:04:53.406 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:04:53.407 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "done", 2023-06-03T20:04:53.407 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:02:45.670148+0000", 2023-06-03T20:04:53.407 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.000108854 2023-06-03T20:04:53.407 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:04:53.407 INFO:teuthology.orchestra.run.smithi164.stdout: ] 2023-06-03T20:04:53.407 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:04:53.407 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:04:53.407 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:04:53.408 INFO:teuthology.orchestra.run.smithi164.stdout: "description": "osd_repop(client.45713.0:136526 9.c e138/127 9:38758465:::benchmark_data_smithi089_115580_object26392:head v 138'8554, mlcod=138'8550)", 2023-06-03T20:04:53.408 INFO:teuthology.orchestra.run.smithi164.stdout: "initiated_at": "2023-06-03T20:02:45.644667+0000", 2023-06-03T20:04:53.408 INFO:teuthology.orchestra.run.smithi164.stdout: "age": 127.691506342, 2023-06-03T20:04:53.408 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.025564017000000001, 2023-06-03T20:04:53.408 INFO:teuthology.orchestra.run.smithi164.stdout: "type_data": { 2023-06-03T20:04:53.408 INFO:teuthology.orchestra.run.smithi164.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-03T20:04:53.408 INFO:teuthology.orchestra.run.smithi164.stdout: "events": [ 2023-06-03T20:04:53.409 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:04:53.409 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "initiated", 2023-06-03T20:04:53.409 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:02:45.644667+0000", 2023-06-03T20:04:53.409 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0 2023-06-03T20:04:53.409 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:04:53.409 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:04:53.409 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "throttled", 2023-06-03T20:04:53.409 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:02:45.644667+0000", 2023-06-03T20:04:53.410 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0 2023-06-03T20:04:53.410 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:04:53.410 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:04:53.410 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "header_read", 2023-06-03T20:04:53.410 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:02:45.644662+0000", 2023-06-03T20:04:53.410 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 4294967295.9999952 2023-06-03T20:04:53.410 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:04:53.410 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:04:53.411 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "all_read", 2023-06-03T20:04:53.411 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:02:45.644680+0000", 2023-06-03T20:04:53.411 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 1.7362000000000001e-05 2023-06-03T20:04:53.411 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:04:53.411 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:04:53.411 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "dispatched", 2023-06-03T20:04:53.411 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:02:45.644702+0000", 2023-06-03T20:04:53.411 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 2.2246e-05 2023-06-03T20:04:53.412 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:04:53.412 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:04:53.412 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "queued_for_pg", 2023-06-03T20:04:53.412 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:02:45.644714+0000", 2023-06-03T20:04:53.412 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 1.2347999999999999e-05 2023-06-03T20:04:53.412 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:04:53.412 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:04:53.413 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "reached_pg", 2023-06-03T20:04:53.413 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:02:45.647992+0000", 2023-06-03T20:04:53.413 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.0032773440000000002 2023-06-03T20:04:53.413 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:04:53.413 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:04:53.413 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "started", 2023-06-03T20:04:53.413 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:02:45.648074+0000", 2023-06-03T20:04:53.413 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 8.2559999999999996e-05 2023-06-03T20:04:53.414 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:04:53.414 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:04:53.414 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "commit_sent", 2023-06-03T20:04:53.414 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:02:45.670155+0000", 2023-06-03T20:04:53.414 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.022080854 2023-06-03T20:04:53.414 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:04:53.414 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:04:53.414 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "done", 2023-06-03T20:04:53.415 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:02:45.670231+0000", 2023-06-03T20:04:53.415 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 7.6077999999999999e-05 2023-06-03T20:04:53.415 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:04:53.415 INFO:teuthology.orchestra.run.smithi164.stdout: ] 2023-06-03T20:04:53.415 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:04:53.415 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:04:53.415 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:04:53.415 INFO:teuthology.orchestra.run.smithi164.stdout: "description": "osd_repop(client.45713.0:136539 9.4 e138/127 9:2acfb180:::benchmark_data_smithi089_115580_object26405:head v 138'8583, mlcod=138'8580)", 2023-06-03T20:04:53.416 INFO:teuthology.orchestra.run.smithi164.stdout: "initiated_at": "2023-06-03T20:02:45.651852+0000", 2023-06-03T20:04:53.416 INFO:teuthology.orchestra.run.smithi164.stdout: "age": 127.684321193, 2023-06-03T20:04:53.416 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.026773228, 2023-06-03T20:04:53.416 INFO:teuthology.orchestra.run.smithi164.stdout: "type_data": { 2023-06-03T20:04:53.416 INFO:teuthology.orchestra.run.smithi164.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-03T20:04:53.416 INFO:teuthology.orchestra.run.smithi164.stdout: "events": [ 2023-06-03T20:04:53.416 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:04:53.416 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "initiated", 2023-06-03T20:04:53.417 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:02:45.651852+0000", 2023-06-03T20:04:53.417 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0 2023-06-03T20:04:53.417 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:04:53.417 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:04:53.417 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "throttled", 2023-06-03T20:04:53.417 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:02:45.651852+0000", 2023-06-03T20:04:53.417 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0 2023-06-03T20:04:53.418 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:04:53.418 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:04:53.418 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "header_read", 2023-06-03T20:04:53.418 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:02:45.651851+0000", 2023-06-03T20:04:53.418 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 4294967295.9999986 2023-06-03T20:04:53.418 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:04:53.418 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:04:53.418 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "all_read", 2023-06-03T20:04:53.419 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:02:45.651868+0000", 2023-06-03T20:04:53.419 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 1.6897000000000001e-05 2023-06-03T20:04:53.419 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:04:53.419 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:04:53.419 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "dispatched", 2023-06-03T20:04:53.419 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:02:45.651889+0000", 2023-06-03T20:04:53.419 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 2.1889999999999999e-05 2023-06-03T20:04:53.419 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:04:53.420 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:04:53.420 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "queued_for_pg", 2023-06-03T20:04:53.420 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:02:45.651900+0000", 2023-06-03T20:04:53.420 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 1.0968000000000001e-05 2023-06-03T20:04:53.420 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:04:53.420 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:04:53.420 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "reached_pg", 2023-06-03T20:04:53.420 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:02:45.653890+0000", 2023-06-03T20:04:53.421 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.0019899890000000002 2023-06-03T20:04:53.421 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:04:53.421 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:04:53.421 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "started", 2023-06-03T20:04:53.421 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:02:45.654052+0000", 2023-06-03T20:04:53.421 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.00016205200000000001 2023-06-03T20:04:53.421 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:04:53.422 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:04:53.422 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "commit_sent", 2023-06-03T20:04:53.422 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:02:45.674646+0000", 2023-06-03T20:04:53.422 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.020594000000000001 2023-06-03T20:04:53.422 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:04:53.422 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:04:53.422 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "done", 2023-06-03T20:04:53.422 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:02:45.678625+0000", 2023-06-03T20:04:53.423 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.0039789650000000001 2023-06-03T20:04:53.423 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:04:53.423 INFO:teuthology.orchestra.run.smithi164.stdout: ] 2023-06-03T20:04:53.423 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:04:53.423 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:04:53.423 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:04:53.423 INFO:teuthology.orchestra.run.smithi164.stdout: "description": "osd_repop(client.45713.0:136528 9.c e138/127 9:30d8430f:::benchmark_data_smithi089_115580_object26394:head v 138'8556, mlcod=138'8550)", 2023-06-03T20:04:53.424 INFO:teuthology.orchestra.run.smithi164.stdout: "initiated_at": "2023-06-03T20:02:45.651967+0000", 2023-06-03T20:04:53.424 INFO:teuthology.orchestra.run.smithi164.stdout: "age": 127.68420587999999, 2023-06-03T20:04:53.424 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.037599243999999997, 2023-06-03T20:04:53.424 INFO:teuthology.orchestra.run.smithi164.stdout: "type_data": { 2023-06-03T20:04:53.424 INFO:teuthology.orchestra.run.smithi164.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-03T20:04:53.424 INFO:teuthology.orchestra.run.smithi164.stdout: "events": [ 2023-06-03T20:04:53.424 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:04:53.425 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "initiated", 2023-06-03T20:04:53.425 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:02:45.651967+0000", 2023-06-03T20:04:53.425 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0 2023-06-03T20:04:53.425 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:04:53.425 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:04:53.425 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "throttled", 2023-06-03T20:04:53.425 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:02:45.651967+0000", 2023-06-03T20:04:53.425 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0 2023-06-03T20:04:53.426 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:04:53.426 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:04:53.426 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "header_read", 2023-06-03T20:04:53.426 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:02:45.651967+0000", 2023-06-03T20:04:53.426 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 4294967295.999999 2023-06-03T20:04:53.426 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:04:53.426 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:04:53.426 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "all_read", 2023-06-03T20:04:53.427 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:02:45.651976+0000", 2023-06-03T20:04:53.427 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 9.4630000000000007e-06 2023-06-03T20:04:53.427 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:04:53.427 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:04:53.427 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "dispatched", 2023-06-03T20:04:53.427 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:02:45.651987+0000", 2023-06-03T20:04:53.427 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 1.0832999999999999e-05 2023-06-03T20:04:53.427 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:04:53.428 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:04:53.428 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "queued_for_pg", 2023-06-03T20:04:53.428 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:02:45.651995+0000", 2023-06-03T20:04:53.428 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 8.3729999999999993e-06 2023-06-03T20:04:53.428 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:04:53.428 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:04:53.428 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "reached_pg", 2023-06-03T20:04:53.429 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:02:45.662630+0000", 2023-06-03T20:04:53.429 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.010634703000000001 2023-06-03T20:04:53.429 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:04:53.429 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:04:53.429 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "started", 2023-06-03T20:04:53.429 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:02:45.662655+0000", 2023-06-03T20:04:53.429 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 2.4601999999999999e-05 2023-06-03T20:04:53.429 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:04:53.430 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:04:53.430 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "commit_sent", 2023-06-03T20:04:53.430 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:02:45.689513+0000", 2023-06-03T20:04:53.430 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.026858546000000001 2023-06-03T20:04:53.430 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:04:53.430 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:04:53.430 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "done", 2023-06-03T20:04:53.430 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:02:45.689567+0000", 2023-06-03T20:04:53.431 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 5.3505999999999999e-05 2023-06-03T20:04:53.431 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:04:53.431 INFO:teuthology.orchestra.run.smithi164.stdout: ] 2023-06-03T20:04:53.431 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:04:53.431 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:04:53.431 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:04:53.431 INFO:teuthology.orchestra.run.smithi164.stdout: "description": "osd_repop(client.45713.0:136529 9.c e138/127 9:3e65d9ce:::benchmark_data_smithi089_115580_object26395:head v 138'8557, mlcod=138'8550)", 2023-06-03T20:04:53.432 INFO:teuthology.orchestra.run.smithi164.stdout: "initiated_at": "2023-06-03T20:02:45.654620+0000", 2023-06-03T20:04:53.432 INFO:teuthology.orchestra.run.smithi164.stdout: "age": 127.681553458, 2023-06-03T20:04:53.432 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.034973492000000002, 2023-06-03T20:04:53.432 INFO:teuthology.orchestra.run.smithi164.stdout: "type_data": { 2023-06-03T20:04:53.432 INFO:teuthology.orchestra.run.smithi164.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-03T20:04:53.432 INFO:teuthology.orchestra.run.smithi164.stdout: "events": [ 2023-06-03T20:04:53.432 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:04:53.433 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "initiated", 2023-06-03T20:04:53.433 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:02:45.654620+0000", 2023-06-03T20:04:53.433 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0 2023-06-03T20:04:53.433 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:04:53.433 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:04:53.433 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "throttled", 2023-06-03T20:04:53.433 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:02:45.654620+0000", 2023-06-03T20:04:53.433 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0 2023-06-03T20:04:53.434 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:04:53.434 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:04:53.434 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "header_read", 2023-06-03T20:04:53.434 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:02:45.654619+0000", 2023-06-03T20:04:53.434 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 4294967295.999999 2023-06-03T20:04:53.434 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:04:53.434 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:04:53.435 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "all_read", 2023-06-03T20:04:53.435 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:02:45.654633+0000", 2023-06-03T20:04:53.435 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 1.4266e-05 2023-06-03T20:04:53.435 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:04:53.435 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:04:53.435 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "dispatched", 2023-06-03T20:04:53.435 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:02:45.654650+0000", 2023-06-03T20:04:53.436 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 1.7476999999999999e-05 2023-06-03T20:04:53.436 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:04:53.436 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:04:53.436 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "queued_for_pg", 2023-06-03T20:04:53.436 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:02:45.654659+0000", 2023-06-03T20:04:53.436 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 8.4689999999999997e-06 2023-06-03T20:04:53.436 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:04:53.436 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:04:53.437 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "reached_pg", 2023-06-03T20:04:53.437 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:02:45.670408+0000", 2023-06-03T20:04:53.437 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.01574921 2023-06-03T20:04:53.437 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:04:53.437 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:04:53.437 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "started", 2023-06-03T20:04:53.437 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:02:45.670465+0000", 2023-06-03T20:04:53.437 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 5.6634e-05 2023-06-03T20:04:53.438 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:04:53.438 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:04:53.438 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "commit_sent", 2023-06-03T20:04:53.438 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:02:45.689571+0000", 2023-06-03T20:04:53.438 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.019105798 2023-06-03T20:04:53.438 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:04:53.438 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:04:53.438 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "done", 2023-06-03T20:04:53.439 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:02:45.689593+0000", 2023-06-03T20:04:53.439 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 2.2782999999999999e-05 2023-06-03T20:04:53.439 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:04:53.439 INFO:teuthology.orchestra.run.smithi164.stdout: ] 2023-06-03T20:04:53.439 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:04:53.439 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:04:53.439 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:04:53.439 INFO:teuthology.orchestra.run.smithi164.stdout: "description": "osd_repop(client.45713.0:136543 9.c e138/127 9:3792b6a8:::benchmark_data_smithi089_115580_object26409:head v 138'8558, mlcod=138'8550)", 2023-06-03T20:04:53.440 INFO:teuthology.orchestra.run.smithi164.stdout: "initiated_at": "2023-06-03T20:02:45.661142+0000", 2023-06-03T20:04:53.440 INFO:teuthology.orchestra.run.smithi164.stdout: "age": 127.675030979, 2023-06-03T20:04:53.440 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.028475015999999999, 2023-06-03T20:04:53.440 INFO:teuthology.orchestra.run.smithi164.stdout: "type_data": { 2023-06-03T20:04:53.440 INFO:teuthology.orchestra.run.smithi164.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-03T20:04:53.440 INFO:teuthology.orchestra.run.smithi164.stdout: "events": [ 2023-06-03T20:04:53.440 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:04:53.441 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "initiated", 2023-06-03T20:04:53.441 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:02:45.661142+0000", 2023-06-03T20:04:53.441 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0 2023-06-03T20:04:53.441 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:04:53.441 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:04:53.441 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "throttled", 2023-06-03T20:04:53.441 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:02:45.661142+0000", 2023-06-03T20:04:53.441 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0 2023-06-03T20:04:53.442 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:04:53.442 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:04:53.442 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "header_read", 2023-06-03T20:04:53.442 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:02:45.661141+0000", 2023-06-03T20:04:53.442 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 4294967295.9999986 2023-06-03T20:04:53.442 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:04:53.442 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:04:53.442 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "all_read", 2023-06-03T20:04:53.443 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:02:45.661159+0000", 2023-06-03T20:04:53.443 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 1.8351000000000001e-05 2023-06-03T20:04:53.443 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:04:53.443 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:04:53.443 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "dispatched", 2023-06-03T20:04:53.443 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:02:45.663096+0000", 2023-06-03T20:04:53.443 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.0019366240000000001 2023-06-03T20:04:53.443 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:04:53.444 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:04:53.444 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "queued_for_pg", 2023-06-03T20:04:53.444 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:02:45.663110+0000", 2023-06-03T20:04:53.444 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 1.4667999999999999e-05 2023-06-03T20:04:53.444 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:04:53.444 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:04:53.444 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "reached_pg", 2023-06-03T20:04:53.445 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:02:45.674584+0000", 2023-06-03T20:04:53.445 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.011473542 2023-06-03T20:04:53.445 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:04:53.445 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:04:53.445 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "started", 2023-06-03T20:04:53.445 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:02:45.674610+0000", 2023-06-03T20:04:53.445 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 2.6432999999999999e-05 2023-06-03T20:04:53.445 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:04:53.446 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:04:53.446 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "commit_sent", 2023-06-03T20:04:53.446 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:02:45.689599+0000", 2023-06-03T20:04:53.446 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.014988207 2023-06-03T20:04:53.446 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:04:53.446 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:04:53.446 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "done", 2023-06-03T20:04:53.446 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:02:45.689617+0000", 2023-06-03T20:04:53.447 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 1.8776000000000001e-05 2023-06-03T20:04:53.447 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:04:53.447 INFO:teuthology.orchestra.run.smithi164.stdout: ] 2023-06-03T20:04:53.447 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:04:53.447 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:04:53.447 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:04:53.447 INFO:teuthology.orchestra.run.smithi164.stdout: "description": "osd_repop(client.45713.0:136549 9.c e138/127 9:3c74e9b0:::benchmark_data_smithi089_115580_object26415:head v 138'8559, mlcod=138'8553)", 2023-06-03T20:04:53.447 INFO:teuthology.orchestra.run.smithi164.stdout: "initiated_at": "2023-06-03T20:02:45.671125+0000", 2023-06-03T20:04:53.448 INFO:teuthology.orchestra.run.smithi164.stdout: "age": 127.665048137, 2023-06-03T20:04:53.448 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.027142988999999999, 2023-06-03T20:04:53.448 INFO:teuthology.orchestra.run.smithi164.stdout: "type_data": { 2023-06-03T20:04:53.448 INFO:teuthology.orchestra.run.smithi164.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-03T20:04:53.448 INFO:teuthology.orchestra.run.smithi164.stdout: "events": [ 2023-06-03T20:04:53.448 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:04:53.448 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "initiated", 2023-06-03T20:04:53.448 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:02:45.671125+0000", 2023-06-03T20:04:53.449 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0 2023-06-03T20:04:53.449 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:04:53.449 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:04:53.449 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "throttled", 2023-06-03T20:04:53.449 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:02:45.671125+0000", 2023-06-03T20:04:53.449 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0 2023-06-03T20:04:53.449 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:04:53.450 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:04:53.450 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "header_read", 2023-06-03T20:04:53.450 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:02:45.671124+0000", 2023-06-03T20:04:53.450 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 4294967295.9999986 2023-06-03T20:04:53.450 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:04:53.450 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:04:53.450 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "all_read", 2023-06-03T20:04:53.450 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:02:45.671139+0000", 2023-06-03T20:04:53.451 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 1.5242000000000001e-05 2023-06-03T20:04:53.451 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:04:53.451 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:04:53.451 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "dispatched", 2023-06-03T20:04:53.451 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:02:45.671158+0000", 2023-06-03T20:04:53.451 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 1.8737e-05 2023-06-03T20:04:53.451 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:04:53.451 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:04:53.452 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "queued_for_pg", 2023-06-03T20:04:53.452 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:02:45.671169+0000", 2023-06-03T20:04:53.452 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 1.1306e-05 2023-06-03T20:04:53.452 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:04:53.452 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:04:53.452 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "reached_pg", 2023-06-03T20:04:53.452 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:02:45.686316+0000", 2023-06-03T20:04:53.452 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.015147045 2023-06-03T20:04:53.453 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:04:53.453 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:04:53.453 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "started", 2023-06-03T20:04:53.453 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:02:45.686352+0000", 2023-06-03T20:04:53.453 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 3.6026999999999999e-05 2023-06-03T20:04:53.453 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:04:53.453 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:04:53.454 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "commit_sent", 2023-06-03T20:04:53.454 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:02:45.698231+0000", 2023-06-03T20:04:53.454 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.011878718 2023-06-03T20:04:53.454 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:04:53.454 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:04:53.454 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "done", 2023-06-03T20:04:53.454 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:02:45.698268+0000", 2023-06-03T20:04:53.454 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 3.7465999999999997e-05 2023-06-03T20:04:53.455 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:04:53.455 INFO:teuthology.orchestra.run.smithi164.stdout: ] 2023-06-03T20:04:53.455 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:04:53.455 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:04:53.455 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:04:53.455 INFO:teuthology.orchestra.run.smithi164.stdout: "description": "osd_repop(client.45713.0:167607 9.c e142/127 9:3c0e2e87:::benchmark_data_smithi089_115580_object57473:head v 142'10491, mlcod=142'10489)", 2023-06-03T20:04:53.455 INFO:teuthology.orchestra.run.smithi164.stdout: "initiated_at": "2023-06-03T20:03:29.178842+0000", 2023-06-03T20:04:53.455 INFO:teuthology.orchestra.run.smithi164.stdout: "age": 84.157331510000006, 2023-06-03T20:04:53.456 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.025541082, 2023-06-03T20:04:53.456 INFO:teuthology.orchestra.run.smithi164.stdout: "type_data": { 2023-06-03T20:04:53.456 INFO:teuthology.orchestra.run.smithi164.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-03T20:04:53.456 INFO:teuthology.orchestra.run.smithi164.stdout: "events": [ 2023-06-03T20:04:53.456 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:04:53.456 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "initiated", 2023-06-03T20:04:53.456 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:03:29.178842+0000", 2023-06-03T20:04:53.456 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0 2023-06-03T20:04:53.457 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:04:53.457 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:04:53.457 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "throttled", 2023-06-03T20:04:53.457 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:03:29.178842+0000", 2023-06-03T20:04:53.457 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0 2023-06-03T20:04:53.457 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:04:53.457 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:04:53.457 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "header_read", 2023-06-03T20:04:53.458 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:03:29.178840+0000", 2023-06-03T20:04:53.458 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 4294967295.9999981 2023-06-03T20:04:53.458 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:04:53.458 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:04:53.458 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "all_read", 2023-06-03T20:04:53.458 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:03:29.178855+0000", 2023-06-03T20:04:53.458 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 1.4938e-05 2023-06-03T20:04:53.458 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:04:53.459 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:04:53.459 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "dispatched", 2023-06-03T20:04:53.459 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:03:29.178873+0000", 2023-06-03T20:04:53.459 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 1.7912999999999999e-05 2023-06-03T20:04:53.459 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:04:53.459 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:04:53.459 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "queued_for_pg", 2023-06-03T20:04:53.459 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:03:29.178881+0000", 2023-06-03T20:04:53.460 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 8.1300000000000001e-06 2023-06-03T20:04:53.460 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:04:53.460 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:04:53.460 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "reached_pg", 2023-06-03T20:04:53.460 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:03:29.180746+0000", 2023-06-03T20:04:53.460 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.0018653999999999999 2023-06-03T20:04:53.460 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:04:53.461 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:04:53.461 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "started", 2023-06-03T20:04:53.461 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:03:29.180768+0000", 2023-06-03T20:04:53.461 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 2.1976000000000001e-05 2023-06-03T20:04:53.461 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:04:53.461 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:04:53.461 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "commit_sent", 2023-06-03T20:04:53.461 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:03:29.204322+0000", 2023-06-03T20:04:53.462 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.023553725000000001 2023-06-03T20:04:53.462 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:04:53.462 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:04:53.462 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "done", 2023-06-03T20:04:53.462 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:03:29.204383+0000", 2023-06-03T20:04:53.462 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 6.0741e-05 2023-06-03T20:04:53.462 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:04:53.462 INFO:teuthology.orchestra.run.smithi164.stdout: ] 2023-06-03T20:04:53.463 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:04:53.463 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:04:53.463 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:04:53.463 INFO:teuthology.orchestra.run.smithi164.stdout: "description": "osd_repop(client.45713.0:167922 9.e e142/127 9:7ecbcb69:::benchmark_data_smithi089_115580_object57788:head v 142'10493, mlcod=142'10491)", 2023-06-03T20:04:53.463 INFO:teuthology.orchestra.run.smithi164.stdout: "initiated_at": "2023-06-03T20:03:29.601657+0000", 2023-06-03T20:04:53.463 INFO:teuthology.orchestra.run.smithi164.stdout: "age": 83.734516830000004, 2023-06-03T20:04:53.463 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.024924741, 2023-06-03T20:04:53.463 INFO:teuthology.orchestra.run.smithi164.stdout: "type_data": { 2023-06-03T20:04:53.464 INFO:teuthology.orchestra.run.smithi164.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-03T20:04:53.464 INFO:teuthology.orchestra.run.smithi164.stdout: "events": [ 2023-06-03T20:04:53.464 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:04:53.464 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "initiated", 2023-06-03T20:04:53.464 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:03:29.601657+0000", 2023-06-03T20:04:53.464 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0 2023-06-03T20:04:53.464 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:04:53.465 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:04:53.465 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "throttled", 2023-06-03T20:04:53.465 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:03:29.601657+0000", 2023-06-03T20:04:53.465 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0 2023-06-03T20:04:53.465 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:04:53.465 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:04:53.465 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "header_read", 2023-06-03T20:04:53.465 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:03:29.601655+0000", 2023-06-03T20:04:53.466 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 4294967295.9999981 2023-06-03T20:04:53.466 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:04:53.466 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:04:53.466 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "all_read", 2023-06-03T20:04:53.466 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:03:29.601672+0000", 2023-06-03T20:04:53.466 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 1.6806000000000001e-05 2023-06-03T20:04:53.466 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:04:53.466 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:04:53.467 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "dispatched", 2023-06-03T20:04:53.467 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:03:29.601700+0000", 2023-06-03T20:04:53.467 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 2.8648000000000001e-05 2023-06-03T20:04:53.467 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:04:53.467 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:04:53.467 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "queued_for_pg", 2023-06-03T20:04:53.467 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:03:29.601714+0000", 2023-06-03T20:04:53.467 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 1.4174e-05 2023-06-03T20:04:53.468 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:04:53.468 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:04:53.468 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "reached_pg", 2023-06-03T20:04:53.468 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:03:29.601809+0000", 2023-06-03T20:04:53.468 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 9.4975000000000005e-05 2023-06-03T20:04:53.468 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:04:53.468 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:04:53.469 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "started", 2023-06-03T20:04:53.469 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:03:29.601838+0000", 2023-06-03T20:04:53.469 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 2.8493e-05 2023-06-03T20:04:53.469 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:04:53.469 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:04:53.469 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "commit_sent", 2023-06-03T20:04:53.469 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:03:29.626397+0000", 2023-06-03T20:04:53.469 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.024558895000000001 2023-06-03T20:04:53.470 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:04:53.470 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:04:53.470 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "done", 2023-06-03T20:04:53.470 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:03:29.626581+0000", 2023-06-03T20:04:53.470 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.00018446000000000001 2023-06-03T20:04:53.470 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:04:53.470 INFO:teuthology.orchestra.run.smithi164.stdout: ] 2023-06-03T20:04:53.470 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:04:53.471 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:04:53.471 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:04:53.471 INFO:teuthology.orchestra.run.smithi164.stdout: "description": "osd_repop(client.45713.0:167926 9.e e142/127 9:7ba27725:::benchmark_data_smithi089_115580_object57792:head v 142'10494, mlcod=142'10491)", 2023-06-03T20:04:53.471 INFO:teuthology.orchestra.run.smithi164.stdout: "initiated_at": "2023-06-03T20:03:29.606770+0000", 2023-06-03T20:04:53.471 INFO:teuthology.orchestra.run.smithi164.stdout: "age": 83.729403050000002, 2023-06-03T20:04:53.471 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.028929577000000001, 2023-06-03T20:04:53.471 INFO:teuthology.orchestra.run.smithi164.stdout: "type_data": { 2023-06-03T20:04:53.471 INFO:teuthology.orchestra.run.smithi164.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-03T20:04:53.472 INFO:teuthology.orchestra.run.smithi164.stdout: "events": [ 2023-06-03T20:04:53.472 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:04:53.472 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "initiated", 2023-06-03T20:04:53.472 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:03:29.606770+0000", 2023-06-03T20:04:53.472 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0 2023-06-03T20:04:53.472 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:04:53.472 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:04:53.473 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "throttled", 2023-06-03T20:04:53.473 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:03:29.606770+0000", 2023-06-03T20:04:53.473 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0 2023-06-03T20:04:53.473 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:04:53.473 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:04:53.473 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "header_read", 2023-06-03T20:04:53.473 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:03:29.606769+0000", 2023-06-03T20:04:53.473 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 4294967295.999999 2023-06-03T20:04:53.474 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:04:53.474 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:04:53.474 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "all_read", 2023-06-03T20:04:53.474 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:03:29.606778+0000", 2023-06-03T20:04:53.474 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 8.6070000000000008e-06 2023-06-03T20:04:53.474 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:04:53.474 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:04:53.474 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "dispatched", 2023-06-03T20:04:53.475 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:03:29.606794+0000", 2023-06-03T20:04:53.475 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 1.6086000000000002e-05 2023-06-03T20:04:53.475 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:04:53.475 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:04:53.475 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "queued_for_pg", 2023-06-03T20:04:53.475 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:03:29.606802+0000", 2023-06-03T20:04:53.475 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 7.79e-06 2023-06-03T20:04:53.475 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:04:53.476 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:04:53.476 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "reached_pg", 2023-06-03T20:04:53.476 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:03:29.618297+0000", 2023-06-03T20:04:53.476 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.011495399 2023-06-03T20:04:53.476 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:04:53.476 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:04:53.476 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "started", 2023-06-03T20:04:53.476 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:03:29.618321+0000", 2023-06-03T20:04:53.477 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 2.4170999999999999e-05 2023-06-03T20:04:53.477 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:04:53.477 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:04:53.477 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "commit_sent", 2023-06-03T20:04:53.477 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:03:29.635642+0000", 2023-06-03T20:04:53.477 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.017320390000000001 2023-06-03T20:04:53.477 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:04:53.478 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:04:53.478 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "done", 2023-06-03T20:04:53.478 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:03:29.635700+0000", 2023-06-03T20:04:53.478 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 5.8038999999999998e-05 2023-06-03T20:04:53.478 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:04:53.478 INFO:teuthology.orchestra.run.smithi164.stdout: ] 2023-06-03T20:04:53.478 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:04:53.478 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:04:53.479 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:04:53.479 INFO:teuthology.orchestra.run.smithi164.stdout: "description": "osd_repop(client.45713.0:167927 9.e e142/127 9:72e04942:::benchmark_data_smithi089_115580_object57793:head v 142'10495, mlcod=142'10491)", 2023-06-03T20:04:53.479 INFO:teuthology.orchestra.run.smithi164.stdout: "initiated_at": "2023-06-03T20:03:29.611193+0000", 2023-06-03T20:04:53.479 INFO:teuthology.orchestra.run.smithi164.stdout: "age": 83.724980504000001, 2023-06-03T20:04:53.479 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.024541436, 2023-06-03T20:04:53.479 INFO:teuthology.orchestra.run.smithi164.stdout: "type_data": { 2023-06-03T20:04:53.479 INFO:teuthology.orchestra.run.smithi164.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-03T20:04:53.479 INFO:teuthology.orchestra.run.smithi164.stdout: "events": [ 2023-06-03T20:04:53.480 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:04:53.480 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "initiated", 2023-06-03T20:04:53.480 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:03:29.611193+0000", 2023-06-03T20:04:53.480 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0 2023-06-03T20:04:53.480 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:04:53.480 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:04:53.480 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "throttled", 2023-06-03T20:04:53.480 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:03:29.611193+0000", 2023-06-03T20:04:53.481 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0 2023-06-03T20:04:53.481 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:04:53.481 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:04:53.481 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "header_read", 2023-06-03T20:04:53.481 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:03:29.611192+0000", 2023-06-03T20:04:53.481 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 4294967295.999999 2023-06-03T20:04:53.481 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:04:53.482 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:04:53.482 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "all_read", 2023-06-03T20:04:53.482 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:03:29.611201+0000", 2023-06-03T20:04:53.482 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 8.5439999999999999e-06 2023-06-03T20:04:53.482 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:04:53.482 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:04:53.482 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "dispatched", 2023-06-03T20:04:53.482 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:03:29.611214+0000", 2023-06-03T20:04:53.483 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 1.3234999999999999e-05 2023-06-03T20:04:53.483 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:04:53.483 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:04:53.483 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "queued_for_pg", 2023-06-03T20:04:53.483 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:03:29.611233+0000", 2023-06-03T20:04:53.483 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 1.8780999999999999e-05 2023-06-03T20:04:53.483 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:04:53.483 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:04:53.484 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "reached_pg", 2023-06-03T20:04:53.484 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:03:29.622058+0000", 2023-06-03T20:04:53.484 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.010824902000000001 2023-06-03T20:04:53.484 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:04:53.484 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:04:53.484 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "started", 2023-06-03T20:04:53.484 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:03:29.622123+0000", 2023-06-03T20:04:53.484 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 6.5203999999999994e-05 2023-06-03T20:04:53.485 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:04:53.485 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:04:53.485 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "commit_sent", 2023-06-03T20:04:53.485 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:03:29.635706+0000", 2023-06-03T20:04:53.485 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.013583247999999999 2023-06-03T20:04:53.485 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:04:53.485 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:04:53.485 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "done", 2023-06-03T20:04:53.486 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:03:29.635734+0000", 2023-06-03T20:04:53.486 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 2.8320999999999999e-05 2023-06-03T20:04:53.486 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:04:53.486 INFO:teuthology.orchestra.run.smithi164.stdout: ] 2023-06-03T20:04:53.486 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:04:53.486 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:04:53.486 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:04:53.486 INFO:teuthology.orchestra.run.smithi164.stdout: "description": "osd_repop(client.45713.0:169183 9.c e142/127 9:3d6cbce2:::benchmark_data_smithi089_115580_object59049:head v 142'10583, mlcod=142'10581)", 2023-06-03T20:04:53.487 INFO:teuthology.orchestra.run.smithi164.stdout: "initiated_at": "2023-06-03T20:03:31.394511+0000", 2023-06-03T20:04:53.487 INFO:teuthology.orchestra.run.smithi164.stdout: "age": 81.941662441000005, 2023-06-03T20:04:53.487 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.025761038, 2023-06-03T20:04:53.487 INFO:teuthology.orchestra.run.smithi164.stdout: "type_data": { 2023-06-03T20:04:53.487 INFO:teuthology.orchestra.run.smithi164.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-03T20:04:53.487 INFO:teuthology.orchestra.run.smithi164.stdout: "events": [ 2023-06-03T20:04:53.487 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:04:53.488 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "initiated", 2023-06-03T20:04:53.488 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:03:31.394511+0000", 2023-06-03T20:04:53.488 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0 2023-06-03T20:04:53.488 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:04:53.488 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:04:53.488 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "throttled", 2023-06-03T20:04:53.488 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:03:31.394511+0000", 2023-06-03T20:04:53.488 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0 2023-06-03T20:04:53.489 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:04:53.489 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:04:53.489 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "header_read", 2023-06-03T20:04:53.489 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:03:31.394506+0000", 2023-06-03T20:04:53.489 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 4294967295.9999948 2023-06-03T20:04:53.489 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:04:53.489 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:04:53.489 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "all_read", 2023-06-03T20:04:53.490 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:03:31.394524+0000", 2023-06-03T20:04:53.490 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 1.7686e-05 2023-06-03T20:04:53.490 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:04:53.490 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:04:53.490 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "dispatched", 2023-06-03T20:04:53.490 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:03:31.394549+0000", 2023-06-03T20:04:53.490 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 2.5735999999999999e-05 2023-06-03T20:04:53.490 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:04:53.491 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:04:53.491 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "queued_for_pg", 2023-06-03T20:04:53.491 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:03:31.394560+0000", 2023-06-03T20:04:53.491 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 1.1003e-05 2023-06-03T20:04:53.491 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:04:53.491 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:04:53.491 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "reached_pg", 2023-06-03T20:04:53.491 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:03:31.394619+0000", 2023-06-03T20:04:53.492 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 5.8576999999999998e-05 2023-06-03T20:04:53.492 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:04:53.492 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:04:53.492 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "started", 2023-06-03T20:04:53.492 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:03:31.394657+0000", 2023-06-03T20:04:53.492 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 3.8127999999999998e-05 2023-06-03T20:04:53.492 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:04:53.492 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:04:53.493 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "commit_sent", 2023-06-03T20:04:53.493 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:03:31.420165+0000", 2023-06-03T20:04:53.493 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.025507951000000001 2023-06-03T20:04:53.493 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:04:53.493 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:04:53.493 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "done", 2023-06-03T20:04:53.493 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:03:31.420272+0000", 2023-06-03T20:04:53.494 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.00010704099999999999 2023-06-03T20:04:53.494 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:04:53.494 INFO:teuthology.orchestra.run.smithi164.stdout: ] 2023-06-03T20:04:53.494 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:04:53.494 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:04:53.494 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:04:53.494 INFO:teuthology.orchestra.run.smithi164.stdout: "description": "osd_repop(client.45713.0:169191 9.c e142/127 9:324089a3:::benchmark_data_smithi089_115580_object59057:head v 142'10584, mlcod=142'10581)", 2023-06-03T20:04:53.494 INFO:teuthology.orchestra.run.smithi164.stdout: "initiated_at": "2023-06-03T20:03:31.397498+0000", 2023-06-03T20:04:53.495 INFO:teuthology.orchestra.run.smithi164.stdout: "age": 81.938675474999997, 2023-06-03T20:04:53.495 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.028750468000000001, 2023-06-03T20:04:53.495 INFO:teuthology.orchestra.run.smithi164.stdout: "type_data": { 2023-06-03T20:04:53.495 INFO:teuthology.orchestra.run.smithi164.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-03T20:04:53.495 INFO:teuthology.orchestra.run.smithi164.stdout: "events": [ 2023-06-03T20:04:53.495 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:04:53.495 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "initiated", 2023-06-03T20:04:53.495 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:03:31.397498+0000", 2023-06-03T20:04:53.496 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0 2023-06-03T20:04:53.496 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:04:53.496 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:04:53.496 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "throttled", 2023-06-03T20:04:53.496 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:03:31.397498+0000", 2023-06-03T20:04:53.496 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0 2023-06-03T20:04:53.496 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:04:53.497 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:04:53.497 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "header_read", 2023-06-03T20:04:53.497 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:03:31.397497+0000", 2023-06-03T20:04:53.497 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 4294967295.999999 2023-06-03T20:04:53.497 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:04:53.497 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:04:53.497 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "all_read", 2023-06-03T20:04:53.497 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:03:31.397514+0000", 2023-06-03T20:04:53.498 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 1.7150999999999999e-05 2023-06-03T20:04:53.498 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:04:53.498 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:04:53.498 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "dispatched", 2023-06-03T20:04:53.498 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:03:31.397532+0000", 2023-06-03T20:04:53.498 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 1.8442000000000001e-05 2023-06-03T20:04:53.498 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:04:53.498 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:04:53.499 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "queued_for_pg", 2023-06-03T20:04:53.499 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:03:31.397543+0000", 2023-06-03T20:04:53.499 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 1.0679999999999999e-05 2023-06-03T20:04:53.499 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:04:53.499 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:04:53.499 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "reached_pg", 2023-06-03T20:04:53.499 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:03:31.406581+0000", 2023-06-03T20:04:53.499 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.0090382489999999999 2023-06-03T20:04:53.500 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:04:53.500 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:04:53.500 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "started", 2023-06-03T20:04:53.500 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:03:31.406609+0000", 2023-06-03T20:04:53.500 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 2.7319000000000001e-05 2023-06-03T20:04:53.500 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:04:53.500 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:04:53.500 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "commit_sent", 2023-06-03T20:04:53.501 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:03:31.426202+0000", 2023-06-03T20:04:53.501 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.019593421 2023-06-03T20:04:53.501 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:04:53.501 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:04:53.501 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "done", 2023-06-03T20:04:53.501 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:03:31.426248+0000", 2023-06-03T20:04:53.501 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 4.6387000000000002e-05 2023-06-03T20:04:53.501 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:04:53.502 INFO:teuthology.orchestra.run.smithi164.stdout: ] 2023-06-03T20:04:53.502 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:04:53.502 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:04:53.502 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:04:53.502 INFO:teuthology.orchestra.run.smithi164.stdout: "description": "osd_repop(client.45713.0:206255 9.c e144/127 9:322b9376:::benchmark_data_smithi089_115580_object96121:head v 144'12857, mlcod=144'12854)", 2023-06-03T20:04:53.502 INFO:teuthology.orchestra.run.smithi164.stdout: "initiated_at": "2023-06-03T20:04:24.584792+0000", 2023-06-03T20:04:53.502 INFO:teuthology.orchestra.run.smithi164.stdout: "age": 28.7513814, 2023-06-03T20:04:53.503 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.024195463, 2023-06-03T20:04:53.503 INFO:teuthology.orchestra.run.smithi164.stdout: "type_data": { 2023-06-03T20:04:53.503 INFO:teuthology.orchestra.run.smithi164.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-03T20:04:53.503 INFO:teuthology.orchestra.run.smithi164.stdout: "events": [ 2023-06-03T20:04:53.503 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:04:53.503 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "initiated", 2023-06-03T20:04:53.503 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:04:24.584792+0000", 2023-06-03T20:04:53.503 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0 2023-06-03T20:04:53.504 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:04:53.504 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:04:53.504 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "throttled", 2023-06-03T20:04:53.504 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:04:24.584792+0000", 2023-06-03T20:04:53.504 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0 2023-06-03T20:04:53.504 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:04:53.504 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:04:53.504 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "header_read", 2023-06-03T20:04:53.505 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:04:24.584791+0000", 2023-06-03T20:04:53.505 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 4294967295.999999 2023-06-03T20:04:53.505 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:04:53.505 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:04:53.505 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "all_read", 2023-06-03T20:04:53.505 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:04:24.584802+0000", 2023-06-03T20:04:53.505 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 1.1060000000000001e-05 2023-06-03T20:04:53.505 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:04:53.506 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:04:53.506 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "dispatched", 2023-06-03T20:04:53.506 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:04:24.584823+0000", 2023-06-03T20:04:53.506 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 2.1291999999999999e-05 2023-06-03T20:04:53.506 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:04:53.506 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:04:53.506 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "queued_for_pg", 2023-06-03T20:04:53.506 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:04:24.584832+0000", 2023-06-03T20:04:53.507 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 9.0080000000000008e-06 2023-06-03T20:04:53.507 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:04:53.507 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:04:53.507 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "reached_pg", 2023-06-03T20:04:53.507 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:04:24.591795+0000", 2023-06-03T20:04:53.507 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.0069630509999999996 2023-06-03T20:04:53.507 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:04:53.507 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:04:53.508 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "started", 2023-06-03T20:04:53.508 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:04:24.591900+0000", 2023-06-03T20:04:53.508 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.000104323 2023-06-03T20:04:53.508 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:04:53.508 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:04:53.508 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "commit_sent", 2023-06-03T20:04:53.508 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:04:24.608950+0000", 2023-06-03T20:04:53.509 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.017050247000000001 2023-06-03T20:04:53.509 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:04:53.509 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:04:53.509 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "done", 2023-06-03T20:04:53.509 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:04:24.608987+0000", 2023-06-03T20:04:53.509 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 3.7574000000000003e-05 2023-06-03T20:04:53.509 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:04:53.509 INFO:teuthology.orchestra.run.smithi164.stdout: ] 2023-06-03T20:04:53.510 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:04:53.510 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:04:53.510 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:04:53.510 INFO:teuthology.orchestra.run.smithi164.stdout: "description": "osd_repop(client.45713.0:206263 9.4 e144/127 9:214bcf38:::benchmark_data_smithi089_115580_object96129:head v 144'12989, mlcod=144'12987)", 2023-06-03T20:04:53.510 INFO:teuthology.orchestra.run.smithi164.stdout: "initiated_at": "2023-06-03T20:04:24.593962+0000", 2023-06-03T20:04:53.510 INFO:teuthology.orchestra.run.smithi164.stdout: "age": 28.742211537999999, 2023-06-03T20:04:53.510 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.026045759000000002, 2023-06-03T20:04:53.510 INFO:teuthology.orchestra.run.smithi164.stdout: "type_data": { 2023-06-03T20:04:53.511 INFO:teuthology.orchestra.run.smithi164.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-03T20:04:53.511 INFO:teuthology.orchestra.run.smithi164.stdout: "events": [ 2023-06-03T20:04:53.511 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:04:53.511 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "initiated", 2023-06-03T20:04:53.511 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:04:24.593962+0000", 2023-06-03T20:04:53.511 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0 2023-06-03T20:04:53.511 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:04:53.512 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:04:53.512 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "throttled", 2023-06-03T20:04:53.512 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:04:24.593962+0000", 2023-06-03T20:04:53.512 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0 2023-06-03T20:04:53.512 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:04:53.512 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:04:53.512 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "header_read", 2023-06-03T20:04:53.512 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:04:24.593960+0000", 2023-06-03T20:04:53.513 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 4294967295.9999986 2023-06-03T20:04:53.513 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:04:53.513 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:04:53.513 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "all_read", 2023-06-03T20:04:53.513 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:04:24.593972+0000", 2023-06-03T20:04:53.513 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 1.1080999999999999e-05 2023-06-03T20:04:53.513 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:04:53.513 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:04:53.514 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "dispatched", 2023-06-03T20:04:53.514 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:04:24.593992+0000", 2023-06-03T20:04:53.514 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 2.0174e-05 2023-06-03T20:04:53.514 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:04:53.514 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:04:53.514 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "queued_for_pg", 2023-06-03T20:04:53.514 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:04:24.594000+0000", 2023-06-03T20:04:53.514 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 8.3529999999999995e-06 2023-06-03T20:04:53.515 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:04:53.515 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:04:53.515 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "reached_pg", 2023-06-03T20:04:53.515 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:04:24.600996+0000", 2023-06-03T20:04:53.515 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.0069961509999999999 2023-06-03T20:04:53.515 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:04:53.515 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:04:53.516 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "started", 2023-06-03T20:04:53.516 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:04:24.601054+0000", 2023-06-03T20:04:53.516 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 5.791e-05 2023-06-03T20:04:53.516 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:04:53.516 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:04:53.516 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "commit_sent", 2023-06-03T20:04:53.516 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:04:24.619852+0000", 2023-06-03T20:04:53.516 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.018798272000000001 2023-06-03T20:04:53.517 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:04:53.517 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:04:53.517 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "done", 2023-06-03T20:04:53.517 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:04:24.620008+0000", 2023-06-03T20:04:53.517 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.00015518300000000001 2023-06-03T20:04:53.517 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:04:53.517 INFO:teuthology.orchestra.run.smithi164.stdout: ] 2023-06-03T20:04:53.517 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:04:53.518 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:04:53.518 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:04:53.518 INFO:teuthology.orchestra.run.smithi164.stdout: "description": "osd_repop(client.45713.0:209834 9.e e144/127 9:73f2f02e:::benchmark_data_smithi089_115580_object99700:head v 144'13072, mlcod=144'13070)", 2023-06-03T20:04:53.518 INFO:teuthology.orchestra.run.smithi164.stdout: "initiated_at": "2023-06-03T20:04:29.596363+0000", 2023-06-03T20:04:53.518 INFO:teuthology.orchestra.run.smithi164.stdout: "age": 23.739810587000001, 2023-06-03T20:04:53.518 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.028329264, 2023-06-03T20:04:53.518 INFO:teuthology.orchestra.run.smithi164.stdout: "type_data": { 2023-06-03T20:04:53.519 INFO:teuthology.orchestra.run.smithi164.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-03T20:04:53.519 INFO:teuthology.orchestra.run.smithi164.stdout: "events": [ 2023-06-03T20:04:53.519 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:04:53.519 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "initiated", 2023-06-03T20:04:53.519 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:04:29.596363+0000", 2023-06-03T20:04:53.519 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0 2023-06-03T20:04:53.519 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:04:53.519 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:04:53.520 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "throttled", 2023-06-03T20:04:53.520 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:04:29.596363+0000", 2023-06-03T20:04:53.520 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0 2023-06-03T20:04:53.520 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:04:53.520 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:04:53.520 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "header_read", 2023-06-03T20:04:53.520 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:04:29.596361+0000", 2023-06-03T20:04:53.520 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 4294967295.9999986 2023-06-03T20:04:53.521 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:04:53.521 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:04:53.521 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "all_read", 2023-06-03T20:04:53.521 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:04:29.596374+0000", 2023-06-03T20:04:53.521 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 1.3004e-05 2023-06-03T20:04:53.521 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:04:53.521 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:04:53.521 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "dispatched", 2023-06-03T20:04:53.522 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:04:29.596390+0000", 2023-06-03T20:04:53.522 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 1.5455e-05 2023-06-03T20:04:53.522 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:04:53.522 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:04:53.522 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "queued_for_pg", 2023-06-03T20:04:53.522 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:04:29.596400+0000", 2023-06-03T20:04:53.522 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 1.008e-05 2023-06-03T20:04:53.522 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:04:53.523 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:04:53.523 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "reached_pg", 2023-06-03T20:04:53.523 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:04:29.596445+0000", 2023-06-03T20:04:53.523 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 4.5253000000000002e-05 2023-06-03T20:04:53.523 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:04:53.523 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:04:53.523 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "started", 2023-06-03T20:04:53.523 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:04:29.596472+0000", 2023-06-03T20:04:53.524 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 2.6659e-05 2023-06-03T20:04:53.524 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:04:53.524 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:04:53.524 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "commit_sent", 2023-06-03T20:04:53.524 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:04:29.624443+0000", 2023-06-03T20:04:53.524 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.027971685 2023-06-03T20:04:53.524 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:04:53.525 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:04:53.525 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "done", 2023-06-03T20:04:53.525 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:04:29.624692+0000", 2023-06-03T20:04:53.525 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.000248543 2023-06-03T20:04:53.525 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:04:53.525 INFO:teuthology.orchestra.run.smithi164.stdout: ] 2023-06-03T20:04:53.525 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:04:53.525 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:04:53.526 INFO:teuthology.orchestra.run.smithi164.stdout: ] 2023-06-03T20:04:53.526 INFO:teuthology.orchestra.run.smithi164.stdout:} 2023-06-03T20:04:53.527 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:04:53.528 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:04:53.528 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:04:53.629 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:04:53.662 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:04:53.665 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:04:53.665 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:04:53.766 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:04:53.800 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:04:53.803 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:04:53.804 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:04:53.833 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:04:53 smithi164 bash[59462]: cluster 1685822692.5160732 mgr.y (mgr.34104) 860 : cluster 0 pgmap v783: 89 pgs: 89 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 405 op/s 2023-06-03T20:04:53.896 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:04:53 smithi089 bash[78452]: cluster 1685822692.5160732 mgr.y (mgr.34104) 860 : cluster 0 pgmap v783: 89 pgs: 89 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 405 op/s 2023-06-03T20:04:53.896 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:04:53 smithi089 bash[75068]: cluster 1685822692.5160732 mgr.y (mgr.34104) 860 : cluster 0 pgmap v783: 89 pgs: 89 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 405 op/s 2023-06-03T20:04:53.905 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:04:53.942 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:04:53.944 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:04:53.945 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:04:54.046 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:04:54.078 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:04:54.080 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:04:54.081 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:04:54.182 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:04:54.222 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:04:54.225 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:04:54.225 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:04:54.228 DEBUG:teuthology.orchestra.run.smithi164:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph daemon osd.5 dump_ops_in_flight 2023-06-03T20:04:54.327 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:04:54.353 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:04:54.356 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:04:54.357 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:04:54.458 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:04:54.496 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:04:54.498 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:04:54.499 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:04:54.600 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:04:54.645 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:04:54.646 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:04:54.647 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:04:54.748 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:04:54.773 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:04:54.776 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:04:54.777 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:04:54.878 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:04:54.904 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:04:54.906 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:04:54.907 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:04:55.008 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:04:55.042 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:04:55.045 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:04:55.045 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:04:55.146 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:04:55.184 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:04:55.187 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:04:55.188 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:04:55.289 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:04:55.318 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:04:55.320 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:04:55.321 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:04:55.337 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-06-03T20:04:55.422 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:04:55.458 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:04:55.461 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:04:55.462 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:04:55.527 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph osd set noscrub 2023-06-03T20:04:55.563 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:04:55.599 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:04:55.602 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:04:55.602 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:04:55.703 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:04:55.732 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:04:55.734 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:04:55.735 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:04:55.833 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:04:55 smithi164 bash[59462]: cluster 1685822694.5172603 mgr.y (mgr.34104) 861 : cluster 0 pgmap v784: 89 pgs: 89 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 135 op/s 2023-06-03T20:04:55.838 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:04:55.878 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:04:55.881 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:04:55.882 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:04:55.978 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:04:55 smithi089 bash[75068]: cluster 1685822694.5172603 mgr.y (mgr.34104) 861 : cluster 0 pgmap v784: 89 pgs: 89 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 135 op/s 2023-06-03T20:04:55.979 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:04:55 smithi089 bash[78452]: cluster 1685822694.5172603 mgr.y (mgr.34104) 861 : cluster 0 pgmap v784: 89 pgs: 89 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 135 op/s 2023-06-03T20:04:55.982 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:04:56.016 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:04:56.019 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:04:56.020 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:04:56.121 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:04:56.158 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:04:56.160 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:04:56.161 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:04:56.262 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:04:56.306 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:04:56.309 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:04:56.310 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:04:56.410 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:04:56.436 INFO:teuthology.orchestra.run.smithi089.stdout: 2023-06-03T20:04:56.439 INFO:teuthology.orchestra.run.smithi089.stderr:dumped all 2023-06-03T20:04:56.443 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:04:56.447 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:04:56.447 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:04:56.495 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 20:04:56 smithi089 bash[71163]: ::ffff:172.21.15.164 - - [03/Jun/2023:20:04:56] "GET /metrics HTTP/1.1" 200 228226 "" "Prometheus/2.33.4" 2023-06-03T20:04:56.549 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:04:56.581 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:04:56.584 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:04:56.585 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:04:56.685 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:04:56.720 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:04:56.723 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:04:56.723 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:04:56.824 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:04:56.864 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:04:56.867 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:04:56.868 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:04:56.968 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:04:57.003 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:04:57.005 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:04:57.006 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:04:57.107 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:04:57.146 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:04:57.148 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:04:57.149 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:04:57.250 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:04:57.285 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:04:57.287 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:04:57.288 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:04:57.389 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:04:57.424 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:04:57.427 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:04:57.429 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:04:57.437 INFO:teuthology.orchestra.run.smithi089.stdout:{"pg_ready":true,"pg_map":{"version":784,"stamp":"2023-06-03T20:04:54.516401+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":595854,"num_objects":24,"num_object_clones":0,"num_object_copies":72,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":24,"num_whiteouts":0,"num_read":309,"num_read_kb":513,"num_write":220525,"num_write_kb":7050831,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139559,"ondisk_log_size":139559,"up":267,"acting":267,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":267,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":1901832,"kb_used_data":13128,"kb_used_omap":30,"kb_used_meta":1888609,"kb_avail":748059384,"statfs":{"total":767960285184,"available":766012809216,"internally_reserved":0,"allocated":13443072,"data_stored":9233257,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":31502,"internal_metadata":1933935858},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-88539160,"num_objects":-1352,"num_object_clones":0,"num_object_copies":-4056,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":-1352,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1352,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.003656"},"pg_stats":[{"pgid":"3.1a","version":"60'1","reported_seq":133,"reported_epoch":145,"state":"active+clean","last_fresh":"2023-06-03T20:04:32.705838+0000","last_change":"2023-06-03T19:57:28.467559+0000","last_active":"2023-06-03T20:04:32.705838+0000","last_peered":"2023-06-03T20:04:32.705838+0000","last_clean":"2023-06-03T20:04:32.705838+0000","last_became_active":"2023-06-03T19:57:28.467077+0000","last_became_peered":"2023-06-03T19:57:28.467077+0000","last_unstale":"2023-06-03T20:04:32.705838+0000","last_undegraded":"2023-06-03T20:04:32.705838+0000","last_fullsized":"2023-06-03T20:04:32.705838+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T21:46:02.795576+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":128,"reported_epoch":146,"state":"active+clean","last_fresh":"2023-06-03T20:04:37.736426+0000","last_change":"2023-06-03T19:57:28.470287+0000","last_active":"2023-06-03T20:04:37.736426+0000","last_peered":"2023-06-03T20:04:37.736426+0000","last_clean":"2023-06-03T20:04:37.736426+0000","last_became_active":"2023-06-03T19:57:28.469477+0000","last_became_peered":"2023-06-03T19:57:28.469477+0000","last_unstale":"2023-06-03T20:04:37.736426+0000","last_undegraded":"2023-06-03T20:04:37.736426+0000","last_fullsized":"2023-06-03T20:04:37.736426+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T06:48:19.229190+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":141,"reported_epoch":146,"state":"active+clean","last_fresh":"2023-06-03T20:04:37.723099+0000","last_change":"2023-06-03T19:57:28.473350+0000","last_active":"2023-06-03T20:04:37.723099+0000","last_peered":"2023-06-03T20:04:37.723099+0000","last_clean":"2023-06-03T20:04:37.723099+0000","last_became_active":"2023-06-03T19:57:28.472757+0000","last_became_peered":"2023-06-03T19:57:28.472757+0000","last_unstale":"2023-06-03T20:04:37.723099+0000","last_undegraded":"2023-06-03T20:04:37.723099+0000","last_fullsized":"2023-06-03T20:04:37.723099+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T19:58:10.432188+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":104,"reported_epoch":145,"state":"active+clean","last_fresh":"2023-06-03T20:04:32.725426+0000","last_change":"2023-06-03T19:56:54.507653+0000","last_active":"2023-06-03T20:04:32.725426+0000","last_peered":"2023-06-03T20:04:32.725426+0000","last_clean":"2023-06-03T20:04:32.725426+0000","last_became_active":"2023-06-03T19:56:54.507350+0000","last_became_peered":"2023-06-03T19:56:54.507350+0000","last_unstale":"2023-06-03T20:04:32.725426+0000","last_undegraded":"2023-06-03T20:04:32.725426+0000","last_fullsized":"2023-06-03T20:04:32.725426+0000","mapping_epoch":119,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T22:33:20.065844+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":129,"reported_epoch":146,"state":"active+clean","last_fresh":"2023-06-03T20:04:37.729272+0000","last_change":"2023-06-03T19:56:54.512932+0000","last_active":"2023-06-03T20:04:37.729272+0000","last_peered":"2023-06-03T20:04:37.729272+0000","last_clean":"2023-06-03T20:04:37.729272+0000","last_became_active":"2023-06-03T19:56:54.511992+0000","last_became_peered":"2023-06-03T19:56:54.511992+0000","last_unstale":"2023-06-03T20:04:37.729272+0000","last_undegraded":"2023-06-03T20:04:37.729272+0000","last_fullsized":"2023-06-03T20:04:37.729272+0000","mapping_epoch":119,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T21:01:35.069911+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":113,"reported_epoch":146,"state":"active+clean","last_fresh":"2023-06-03T20:04:37.716696+0000","last_change":"2023-06-03T19:51:59.359522+0000","last_active":"2023-06-03T20:04:37.716696+0000","last_peered":"2023-06-03T20:04:37.716696+0000","last_clean":"2023-06-03T20:04:37.716696+0000","last_became_active":"2023-06-03T19:51:59.359202+0000","last_became_peered":"2023-06-03T19:51:59.359202+0000","last_unstale":"2023-06-03T20:04:37.716696+0000","last_undegraded":"2023-06-03T20:04:37.716696+0000","last_fullsized":"2023-06-03T20:04:37.716696+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":102,"reported_epoch":145,"state":"active+clean","last_fresh":"2023-06-03T20:04:32.726019+0000","last_change":"2023-06-03T19:56:54.509490+0000","last_active":"2023-06-03T20:04:32.726019+0000","last_peered":"2023-06-03T20:04:32.726019+0000","last_clean":"2023-06-03T20:04:32.726019+0000","last_became_active":"2023-06-03T19:56:54.509145+0000","last_became_peered":"2023-06-03T19:56:54.509145+0000","last_unstale":"2023-06-03T20:04:32.726019+0000","last_undegraded":"2023-06-03T20:04:32.726019+0000","last_fullsized":"2023-06-03T20:04:32.726019+0000","mapping_epoch":119,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T07:12:03.964110+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":113,"reported_epoch":146,"state":"active+clean","last_fresh":"2023-06-03T20:04:37.725704+0000","last_change":"2023-06-03T19:56:54.511325+0000","last_active":"2023-06-03T20:04:37.725704+0000","last_peered":"2023-06-03T20:04:37.725704+0000","last_clean":"2023-06-03T20:04:37.725704+0000","last_became_active":"2023-06-03T19:56:54.511033+0000","last_became_peered":"2023-06-03T19:56:54.511033+0000","last_unstale":"2023-06-03T20:04:37.725704+0000","last_undegraded":"2023-06-03T20:04:37.725704+0000","last_fullsized":"2023-06-03T20:04:37.725704+0000","mapping_epoch":119,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T23:50:08.711478+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":114,"reported_epoch":146,"state":"active+clean","last_fresh":"2023-06-03T20:04:37.740167+0000","last_change":"2023-06-03T19:51:59.374143+0000","last_active":"2023-06-03T20:04:37.740167+0000","last_peered":"2023-06-03T20:04:37.740167+0000","last_clean":"2023-06-03T20:04:37.740167+0000","last_became_active":"2023-06-03T19:51:59.373146+0000","last_became_peered":"2023-06-03T19:51:59.373146+0000","last_unstale":"2023-06-03T20:04:37.740167+0000","last_undegraded":"2023-06-03T20:04:37.740167+0000","last_fullsized":"2023-06-03T20:04:37.740167+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":94,"reported_epoch":146,"state":"active+clean","last_fresh":"2023-06-03T20:04:37.736324+0000","last_change":"2023-06-03T19:56:54.510462+0000","last_active":"2023-06-03T20:04:37.736324+0000","last_peered":"2023-06-03T20:04:37.736324+0000","last_clean":"2023-06-03T20:04:37.736324+0000","last_became_active":"2023-06-03T19:56:54.509783+0000","last_became_peered":"2023-06-03T19:56:54.509783+0000","last_unstale":"2023-06-03T20:04:37.736324+0000","last_undegraded":"2023-06-03T20:04:37.736324+0000","last_fullsized":"2023-06-03T20:04:37.736324+0000","mapping_epoch":119,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T00:16:09.624276+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"60'3","reported_seq":132,"reported_epoch":146,"state":"active+clean","last_fresh":"2023-06-03T20:04:37.742214+0000","last_change":"2023-06-03T19:57:28.466401+0000","last_active":"2023-06-03T20:04:37.742214+0000","last_peered":"2023-06-03T20:04:37.742214+0000","last_clean":"2023-06-03T20:04:37.742214+0000","last_became_active":"2023-06-03T19:57:28.465452+0000","last_became_peered":"2023-06-03T19:57:28.465452+0000","last_unstale":"2023-06-03T20:04:37.742214+0000","last_undegraded":"2023-06-03T20:04:37.742214+0000","last_fullsized":"2023-06-03T20:04:37.742214+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":85,"reported_epoch":146,"state":"active+clean","last_fresh":"2023-06-03T20:04:37.727444+0000","last_change":"2023-06-03T19:56:11.753995+0000","last_active":"2023-06-03T20:04:37.727444+0000","last_peered":"2023-06-03T20:04:37.727444+0000","last_clean":"2023-06-03T20:04:37.727444+0000","last_became_active":"2023-06-03T19:56:11.753676+0000","last_became_peered":"2023-06-03T19:56:11.753676+0000","last_unstale":"2023-06-03T20:04:37.727444+0000","last_undegraded":"2023-06-03T20:04:37.727444+0000","last_fullsized":"2023-06-03T20:04:37.727444+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":115,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T01:00:40.343647+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"77'23","reported_seq":130,"reported_epoch":145,"state":"active+clean","last_fresh":"2023-06-03T20:04:32.707277+0000","last_change":"2023-06-03T19:57:28.462522+0000","last_active":"2023-06-03T20:04:32.707277+0000","last_peered":"2023-06-03T20:04:32.707277+0000","last_clean":"2023-06-03T20:04:32.707277+0000","last_became_active":"2023-06-03T19:57:28.462048+0000","last_became_peered":"2023-06-03T19:57:28.462048+0000","last_unstale":"2023-06-03T20:04:32.707277+0000","last_undegraded":"2023-06-03T20:04:32.707277+0000","last_fullsized":"2023-06-03T20:04:32.707277+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":23,"log_dups_size":0,"ondisk_log_size":23,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T23:23:56.697534+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4716,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":23,"num_write_kb":20,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":107,"reported_epoch":145,"state":"active+clean","last_fresh":"2023-06-03T20:04:32.706015+0000","last_change":"2023-06-03T19:57:28.469676+0000","last_active":"2023-06-03T20:04:32.706015+0000","last_peered":"2023-06-03T20:04:32.706015+0000","last_clean":"2023-06-03T20:04:32.706015+0000","last_became_active":"2023-06-03T19:57:28.469328+0000","last_became_peered":"2023-06-03T19:57:28.469328+0000","last_unstale":"2023-06-03T20:04:32.706015+0000","last_undegraded":"2023-06-03T20:04:32.706015+0000","last_fullsized":"2023-06-03T20:04:32.706015+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T23:25:26.041930+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"60'2","reported_seq":131,"reported_epoch":146,"state":"active+clean","last_fresh":"2023-06-03T20:04:37.747122+0000","last_change":"2023-06-03T19:57:28.467711+0000","last_active":"2023-06-03T20:04:37.747122+0000","last_peered":"2023-06-03T20:04:37.747122+0000","last_clean":"2023-06-03T20:04:37.747122+0000","last_became_active":"2023-06-03T19:57:28.467234+0000","last_became_peered":"2023-06-03T19:57:28.467234+0000","last_unstale":"2023-06-03T20:04:37.747122+0000","last_undegraded":"2023-06-03T20:04:37.747122+0000","last_fullsized":"2023-06-03T20:04:37.747122+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":113,"reported_epoch":146,"state":"active+clean","last_fresh":"2023-06-03T20:04:37.737014+0000","last_change":"2023-06-03T19:56:54.514616+0000","last_active":"2023-06-03T20:04:37.737014+0000","last_peered":"2023-06-03T20:04:37.737014+0000","last_clean":"2023-06-03T20:04:37.737014+0000","last_became_active":"2023-06-03T19:56:54.513918+0000","last_became_peered":"2023-06-03T19:56:54.513918+0000","last_unstale":"2023-06-03T20:04:37.737014+0000","last_undegraded":"2023-06-03T20:04:37.737014+0000","last_fullsized":"2023-06-03T20:04:37.737014+0000","mapping_epoch":119,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T19:36:10.967386+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":102,"reported_epoch":145,"state":"active+clean","last_fresh":"2023-06-03T20:04:32.725095+0000","last_change":"2023-06-03T19:56:54.507479+0000","last_active":"2023-06-03T20:04:32.725095+0000","last_peered":"2023-06-03T20:04:32.725095+0000","last_clean":"2023-06-03T20:04:32.725095+0000","last_became_active":"2023-06-03T19:56:54.507149+0000","last_became_peered":"2023-06-03T19:56:54.507149+0000","last_unstale":"2023-06-03T20:04:32.725095+0000","last_undegraded":"2023-06-03T20:04:32.725095+0000","last_fullsized":"2023-06-03T20:04:32.725095+0000","mapping_epoch":119,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T20:22:36.676317+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":113,"reported_epoch":146,"state":"active+clean","last_fresh":"2023-06-03T20:04:37.745903+0000","last_change":"2023-06-03T19:56:54.512472+0000","last_active":"2023-06-03T20:04:37.745903+0000","last_peered":"2023-06-03T20:04:37.745903+0000","last_clean":"2023-06-03T20:04:37.745903+0000","last_became_active":"2023-06-03T19:56:54.512051+0000","last_became_peered":"2023-06-03T19:56:54.512051+0000","last_unstale":"2023-06-03T20:04:37.745903+0000","last_undegraded":"2023-06-03T20:04:37.745903+0000","last_fullsized":"2023-06-03T20:04:37.745903+0000","mapping_epoch":119,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":114,"reported_epoch":146,"state":"active+clean","last_fresh":"2023-06-03T20:04:37.738613+0000","last_change":"2023-06-03T19:57:28.462780+0000","last_active":"2023-06-03T20:04:37.738613+0000","last_peered":"2023-06-03T20:04:37.738613+0000","last_clean":"2023-06-03T20:04:37.738613+0000","last_became_active":"2023-06-03T19:57:28.462298+0000","last_became_peered":"2023-06-03T19:57:28.462298+0000","last_unstale":"2023-06-03T20:04:37.738613+0000","last_undegraded":"2023-06-03T20:04:37.738613+0000","last_fullsized":"2023-06-03T20:04:37.738613+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":103,"reported_epoch":145,"state":"active+clean","last_fresh":"2023-06-03T20:04:32.725580+0000","last_change":"2023-06-03T19:56:54.510289+0000","last_active":"2023-06-03T20:04:32.725580+0000","last_peered":"2023-06-03T20:04:32.725580+0000","last_clean":"2023-06-03T20:04:32.725580+0000","last_became_active":"2023-06-03T19:56:54.509905+0000","last_became_peered":"2023-06-03T19:56:54.509905+0000","last_unstale":"2023-06-03T20:04:32.725580+0000","last_undegraded":"2023-06-03T20:04:32.725580+0000","last_fullsized":"2023-06-03T20:04:32.725580+0000","mapping_epoch":119,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T02:54:26.792401+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"60'8","reported_seq":122,"reported_epoch":146,"state":"active+clean","last_fresh":"2023-06-03T20:04:37.716021+0000","last_change":"2023-06-03T19:47:06.727778+0000","last_active":"2023-06-03T20:04:37.716021+0000","last_peered":"2023-06-03T20:04:37.716021+0000","last_clean":"2023-06-03T20:04:37.716021+0000","last_became_active":"2023-06-03T19:47:06.727305+0000","last_became_peered":"2023-06-03T19:47:06.727305+0000","last_unstale":"2023-06-03T20:04:37.716021+0000","last_undegraded":"2023-06-03T20:04:37.716021+0000","last_fullsized":"2023-06-03T20:04:37.716021+0000","mapping_epoch":86,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":87,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":113,"reported_epoch":146,"state":"active+clean","last_fresh":"2023-06-03T20:04:37.736615+0000","last_change":"2023-06-03T19:56:54.512994+0000","last_active":"2023-06-03T20:04:37.736615+0000","last_peered":"2023-06-03T20:04:37.736615+0000","last_clean":"2023-06-03T20:04:37.736615+0000","last_became_active":"2023-06-03T19:56:54.512579+0000","last_became_peered":"2023-06-03T19:56:54.512579+0000","last_unstale":"2023-06-03T20:04:37.736615+0000","last_undegraded":"2023-06-03T20:04:37.736615+0000","last_fullsized":"2023-06-03T20:04:37.736615+0000","mapping_epoch":119,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T07:02:45.187961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"9.5","version":"146'13578","reported_seq":13608,"reported_epoch":146,"state":"active+clean","last_fresh":"2023-06-03T20:04:44.748682+0000","last_change":"2023-06-03T20:00:12.908073+0000","last_active":"2023-06-03T20:04:44.748682+0000","last_peered":"2023-06-03T20:04:44.748682+0000","last_clean":"2023-06-03T20:04:44.748682+0000","last_became_active":"2023-06-03T20:00:12.907738+0000","last_became_peered":"2023-06-03T20:00:12.907738+0000","last_unstale":"2023-06-03T20:04:44.748682+0000","last_undegraded":"2023-06-03T20:04:44.748682+0000","last_fullsized":"2023-06-03T20:04:44.748682+0000","mapping_epoch":127,"log_start":"133'3500","ondisk_log_start":"133'3500","created":127,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T20:00:11.873149+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T20:00:11.873149+0000","last_clean_scrub_stamp":"2023-06-03T20:00:11.873149+0000","objects_scrubbed":0,"log_size":10078,"log_dups_size":0,"ondisk_log_size":10078,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T02:31:40.900210+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13578,"num_write_kb":434496,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.f","version":"60'1","reported_seq":130,"reported_epoch":146,"state":"active+clean","last_fresh":"2023-06-03T20:04:37.742365+0000","last_change":"2023-06-03T19:57:28.464482+0000","last_active":"2023-06-03T20:04:37.742365+0000","last_peered":"2023-06-03T20:04:37.742365+0000","last_clean":"2023-06-03T20:04:37.742365+0000","last_became_active":"2023-06-03T19:57:28.464081+0000","last_became_peered":"2023-06-03T19:57:28.464081+0000","last_unstale":"2023-06-03T20:04:37.742365+0000","last_undegraded":"2023-06-03T20:04:37.742365+0000","last_fullsized":"2023-06-03T20:04:37.742365+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":98,"reported_epoch":146,"state":"active+clean","last_fresh":"2023-06-03T20:04:37.734116+0000","last_change":"2023-06-03T19:34:17.848358+0000","last_active":"2023-06-03T20:04:37.734116+0000","last_peered":"2023-06-03T20:04:37.734116+0000","last_clean":"2023-06-03T20:04:37.734116+0000","last_became_active":"2023-06-03T19:34:17.847811+0000","last_became_peered":"2023-06-03T19:34:17.847811+0000","last_unstale":"2023-06-03T20:04:37.734116+0000","last_undegraded":"2023-06-03T20:04:37.734116+0000","last_fullsized":"2023-06-03T20:04:37.734116+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"9.4","version":"146'13904","reported_seq":13935,"reported_epoch":146,"state":"active+clean","last_fresh":"2023-06-03T20:04:44.772801+0000","last_change":"2023-06-03T20:00:12.905941+0000","last_active":"2023-06-03T20:04:44.772801+0000","last_peered":"2023-06-03T20:04:44.772801+0000","last_clean":"2023-06-03T20:04:44.772801+0000","last_became_active":"2023-06-03T20:00:12.905661+0000","last_became_peered":"2023-06-03T20:00:12.905661+0000","last_unstale":"2023-06-03T20:04:44.772801+0000","last_undegraded":"2023-06-03T20:04:44.772801+0000","last_fullsized":"2023-06-03T20:04:44.772801+0000","mapping_epoch":127,"log_start":"134'6500","ondisk_log_start":"134'6500","created":127,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T20:00:11.873149+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T20:00:11.873149+0000","last_clean_scrub_stamp":"2023-06-03T20:00:11.873149+0000","objects_scrubbed":0,"log_size":7404,"log_dups_size":0,"ondisk_log_size":7404,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T06:19:01.942119+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13904,"num_write_kb":444928,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.e","version":"60'2","reported_seq":131,"reported_epoch":146,"state":"active+clean","last_fresh":"2023-06-03T20:04:37.742021+0000","last_change":"2023-06-03T19:57:28.468330+0000","last_active":"2023-06-03T20:04:37.742021+0000","last_peered":"2023-06-03T20:04:37.742021+0000","last_clean":"2023-06-03T20:04:37.742021+0000","last_became_active":"2023-06-03T19:57:28.467823+0000","last_became_peered":"2023-06-03T19:57:28.467823+0000","last_unstale":"2023-06-03T20:04:37.742021+0000","last_undegraded":"2023-06-03T20:04:37.742021+0000","last_fullsized":"2023-06-03T20:04:37.742021+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":145,"reported_epoch":145,"state":"active+clean","last_fresh":"2023-06-03T20:04:32.705872+0000","last_change":"2023-06-03T19:57:28.461449+0000","last_active":"2023-06-03T20:04:32.705872+0000","last_peered":"2023-06-03T20:04:32.705872+0000","last_clean":"2023-06-03T20:04:32.705872+0000","last_became_active":"2023-06-03T19:57:28.460745+0000","last_became_peered":"2023-06-03T19:57:28.460745+0000","last_unstale":"2023-06-03T20:04:32.705872+0000","last_undegraded":"2023-06-03T20:04:32.705872+0000","last_fullsized":"2023-06-03T20:04:32.705872+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T20:41:19.361638+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,3],"acting":[4,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.7","version":"146'14058","reported_seq":14086,"reported_epoch":146,"state":"active+clean","last_fresh":"2023-06-03T20:04:44.772589+0000","last_change":"2023-06-03T20:00:12.909354+0000","last_active":"2023-06-03T20:04:44.772589+0000","last_peered":"2023-06-03T20:04:44.772589+0000","last_clean":"2023-06-03T20:04:44.772589+0000","last_became_active":"2023-06-03T20:00:12.908971+0000","last_became_peered":"2023-06-03T20:00:12.908971+0000","last_unstale":"2023-06-03T20:04:44.772589+0000","last_undegraded":"2023-06-03T20:04:44.772589+0000","last_fullsized":"2023-06-03T20:04:44.772589+0000","mapping_epoch":127,"log_start":"133'4000","ondisk_log_start":"133'4000","created":127,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T20:00:11.873149+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T20:00:11.873149+0000","last_clean_scrub_stamp":"2023-06-03T20:00:11.873149+0000","objects_scrubbed":0,"log_size":10058,"log_dups_size":0,"ondisk_log_size":10058,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":14058,"num_write_kb":449856,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.d","version":"60'1","reported_seq":100,"reported_epoch":146,"state":"active+clean","last_fresh":"2023-06-03T20:04:37.741973+0000","last_change":"2023-06-03T19:34:16.806559+0000","last_active":"2023-06-03T20:04:37.741973+0000","last_peered":"2023-06-03T20:04:37.741973+0000","last_clean":"2023-06-03T20:04:37.741973+0000","last_became_active":"2023-06-03T19:34:16.806149+0000","last_became_peered":"2023-06-03T19:34:16.806149+0000","last_unstale":"2023-06-03T20:04:37.741973+0000","last_undegraded":"2023-06-03T20:04:37.741973+0000","last_fullsized":"2023-06-03T20:04:37.741973+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":113,"reported_epoch":146,"state":"active+clean","last_fresh":"2023-06-03T20:04:37.716316+0000","last_change":"2023-06-03T19:51:59.360650+0000","last_active":"2023-06-03T20:04:37.716316+0000","last_peered":"2023-06-03T20:04:37.716316+0000","last_clean":"2023-06-03T20:04:37.716316+0000","last_became_active":"2023-06-03T19:51:59.360237+0000","last_became_peered":"2023-06-03T19:51:59.360237+0000","last_unstale":"2023-06-03T20:04:37.716316+0000","last_undegraded":"2023-06-03T20:04:37.716316+0000","last_fullsized":"2023-06-03T20:04:37.716316+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"9.6","version":"146'13574","reported_seq":13602,"reported_epoch":146,"state":"active+clean","last_fresh":"2023-06-03T20:04:44.736389+0000","last_change":"2023-06-03T20:00:13.172391+0000","last_active":"2023-06-03T20:04:44.736389+0000","last_peered":"2023-06-03T20:04:44.736389+0000","last_clean":"2023-06-03T20:04:44.736389+0000","last_became_active":"2023-06-03T20:00:13.171568+0000","last_became_peered":"2023-06-03T20:00:13.171568+0000","last_unstale":"2023-06-03T20:04:44.736389+0000","last_undegraded":"2023-06-03T20:04:44.736389+0000","last_fullsized":"2023-06-03T20:04:44.736389+0000","mapping_epoch":127,"log_start":"133'3500","ondisk_log_start":"133'3500","created":127,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T20:00:11.873149+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T20:00:11.873149+0000","last_clean_scrub_stamp":"2023-06-03T20:00:11.873149+0000","objects_scrubbed":0,"log_size":10074,"log_dups_size":0,"ondisk_log_size":10074,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13574,"num_write_kb":434368,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,1],"acting":[5,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":114,"reported_epoch":146,"state":"active+clean","last_fresh":"2023-06-03T20:04:37.734340+0000","last_change":"2023-06-03T19:56:54.507432+0000","last_active":"2023-06-03T20:04:37.734340+0000","last_peered":"2023-06-03T20:04:37.734340+0000","last_clean":"2023-06-03T20:04:37.734340+0000","last_became_active":"2023-06-03T19:56:54.507012+0000","last_became_peered":"2023-06-03T19:56:54.507012+0000","last_unstale":"2023-06-03T20:04:37.734340+0000","last_undegraded":"2023-06-03T20:04:37.734340+0000","last_fullsized":"2023-06-03T20:04:37.734340+0000","mapping_epoch":119,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":130,"reported_epoch":145,"state":"active+clean","last_fresh":"2023-06-03T20:04:32.725428+0000","last_change":"2023-06-03T19:57:28.464153+0000","last_active":"2023-06-03T20:04:32.725428+0000","last_peered":"2023-06-03T20:04:32.725428+0000","last_clean":"2023-06-03T20:04:32.725428+0000","last_became_active":"2023-06-03T19:57:28.463502+0000","last_became_peered":"2023-06-03T19:57:28.463502+0000","last_unstale":"2023-06-03T20:04:32.725428+0000","last_undegraded":"2023-06-03T20:04:32.725428+0000","last_fullsized":"2023-06-03T20:04:32.725428+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T06:11:00.999526+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,4],"acting":[0,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.1","version":"146'13808","reported_seq":13839,"reported_epoch":146,"state":"active+clean","last_fresh":"2023-06-03T20:04:44.722007+0000","last_change":"2023-06-03T20:00:12.905059+0000","last_active":"2023-06-03T20:04:44.722007+0000","last_peered":"2023-06-03T20:04:44.722007+0000","last_clean":"2023-06-03T20:04:44.722007+0000","last_became_active":"2023-06-03T20:00:12.904765+0000","last_became_peered":"2023-06-03T20:00:12.904765+0000","last_unstale":"2023-06-03T20:04:44.722007+0000","last_undegraded":"2023-06-03T20:04:44.722007+0000","last_fullsized":"2023-06-03T20:04:44.722007+0000","mapping_epoch":127,"log_start":"134'6100","ondisk_log_start":"134'6100","created":127,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T20:00:11.873149+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T20:00:11.873149+0000","last_clean_scrub_stamp":"2023-06-03T20:00:11.873149+0000","objects_scrubbed":0,"log_size":7708,"log_dups_size":0,"ondisk_log_size":7708,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T07:39:39.205403+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13808,"num_write_kb":441856,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,7],"acting":[3,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.b","version":"60'2","reported_seq":114,"reported_epoch":146,"state":"active+clean","last_fresh":"2023-06-03T20:04:37.736377+0000","last_change":"2023-06-03T19:57:28.470310+0000","last_active":"2023-06-03T20:04:37.736377+0000","last_peered":"2023-06-03T20:04:37.736377+0000","last_clean":"2023-06-03T20:04:37.736377+0000","last_became_active":"2023-06-03T19:57:28.469887+0000","last_became_peered":"2023-06-03T19:57:28.469887+0000","last_unstale":"2023-06-03T20:04:37.736377+0000","last_undegraded":"2023-06-03T20:04:37.736377+0000","last_fullsized":"2023-06-03T20:04:37.736377+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T21:58:53.387131+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":121,"reported_epoch":145,"state":"active+clean","last_fresh":"2023-06-03T20:04:32.707353+0000","last_change":"2023-06-03T19:57:28.465140+0000","last_active":"2023-06-03T20:04:32.707353+0000","last_peered":"2023-06-03T20:04:32.707353+0000","last_clean":"2023-06-03T20:04:32.707353+0000","last_became_active":"2023-06-03T19:57:28.464629+0000","last_became_peered":"2023-06-03T19:57:28.464629+0000","last_unstale":"2023-06-03T20:04:32.707353+0000","last_undegraded":"2023-06-03T20:04:32.707353+0000","last_fullsized":"2023-06-03T20:04:32.707353+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T21:47:11.619583+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.0","version":"146'13656","reported_seq":13687,"reported_epoch":146,"state":"active+clean","last_fresh":"2023-06-03T20:04:44.765432+0000","last_change":"2023-06-03T20:00:13.174012+0000","last_active":"2023-06-03T20:04:44.765432+0000","last_peered":"2023-06-03T20:04:44.765432+0000","last_clean":"2023-06-03T20:04:44.765432+0000","last_became_active":"2023-06-03T20:00:13.173630+0000","last_became_peered":"2023-06-03T20:00:13.173630+0000","last_unstale":"2023-06-03T20:04:44.765432+0000","last_undegraded":"2023-06-03T20:04:44.765432+0000","last_fullsized":"2023-06-03T20:04:44.765432+0000","mapping_epoch":127,"log_start":"134'6300","ondisk_log_start":"134'6300","created":127,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T20:00:11.873149+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T20:00:11.873149+0000","last_clean_scrub_stamp":"2023-06-03T20:00:11.873149+0000","objects_scrubbed":0,"log_size":7356,"log_dups_size":0,"ondisk_log_size":7356,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T23:14:07.402519+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13656,"num_write_kb":436992,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":129,"reported_epoch":146,"state":"active+clean","last_fresh":"2023-06-03T20:04:37.716169+0000","last_change":"2023-06-03T19:57:28.461401+0000","last_active":"2023-06-03T20:04:37.716169+0000","last_peered":"2023-06-03T20:04:37.716169+0000","last_clean":"2023-06-03T20:04:37.716169+0000","last_became_active":"2023-06-03T19:57:28.461057+0000","last_became_peered":"2023-06-03T19:57:28.461057+0000","last_unstale":"2023-06-03T20:04:37.716169+0000","last_undegraded":"2023-06-03T20:04:37.716169+0000","last_fullsized":"2023-06-03T20:04:37.716169+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":126,"reported_epoch":145,"state":"active+clean","last_fresh":"2023-06-03T20:04:32.707378+0000","last_change":"2023-06-03T19:57:28.456750+0000","last_active":"2023-06-03T20:04:32.707378+0000","last_peered":"2023-06-03T20:04:32.707378+0000","last_clean":"2023-06-03T20:04:32.707378+0000","last_became_active":"2023-06-03T19:57:28.456436+0000","last_became_peered":"2023-06-03T19:57:28.456436+0000","last_unstale":"2023-06-03T20:04:32.707378+0000","last_undegraded":"2023-06-03T20:04:32.707378+0000","last_fullsized":"2023-06-03T20:04:32.707378+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T01:49:25.481554+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,5],"acting":[4,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.3","version":"146'13634","reported_seq":13662,"reported_epoch":146,"state":"active+clean","last_fresh":"2023-06-03T20:04:44.770929+0000","last_change":"2023-06-03T20:00:12.903729+0000","last_active":"2023-06-03T20:04:44.770929+0000","last_peered":"2023-06-03T20:04:44.770929+0000","last_clean":"2023-06-03T20:04:44.770929+0000","last_became_active":"2023-06-03T20:00:12.903378+0000","last_became_peered":"2023-06-03T20:00:12.903378+0000","last_unstale":"2023-06-03T20:04:44.770929+0000","last_undegraded":"2023-06-03T20:04:44.770929+0000","last_fullsized":"2023-06-03T20:04:44.770929+0000","mapping_epoch":127,"log_start":"134'5900","ondisk_log_start":"134'5900","created":127,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T20:00:11.873149+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T20:00:11.873149+0000","last_clean_scrub_stamp":"2023-06-03T20:00:11.873149+0000","objects_scrubbed":0,"log_size":7734,"log_dups_size":0,"ondisk_log_size":7734,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13634,"num_write_kb":436288,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,1],"acting":[7,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":118,"reported_epoch":145,"state":"active+clean","last_fresh":"2023-06-03T20:04:32.705582+0000","last_change":"2023-06-03T19:57:28.462344+0000","last_active":"2023-06-03T20:04:32.705582+0000","last_peered":"2023-06-03T20:04:32.705582+0000","last_clean":"2023-06-03T20:04:32.705582+0000","last_became_active":"2023-06-03T19:57:28.462001+0000","last_became_peered":"2023-06-03T19:57:28.462001+0000","last_unstale":"2023-06-03T20:04:32.705582+0000","last_undegraded":"2023-06-03T20:04:32.705582+0000","last_fullsized":"2023-06-03T20:04:32.705582+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T00:45:13.789650+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":107,"reported_epoch":145,"state":"active+clean","last_fresh":"2023-06-03T20:04:32.707985+0000","last_change":"2023-06-03T19:57:28.467917+0000","last_active":"2023-06-03T20:04:32.707985+0000","last_peered":"2023-06-03T20:04:32.707985+0000","last_clean":"2023-06-03T20:04:32.707985+0000","last_became_active":"2023-06-03T19:57:28.467617+0000","last_became_peered":"2023-06-03T19:57:28.467617+0000","last_unstale":"2023-06-03T20:04:32.707985+0000","last_undegraded":"2023-06-03T20:04:32.707985+0000","last_fullsized":"2023-06-03T20:04:32.707985+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T23:34:23.840582+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.2","version":"146'13764","reported_seq":13795,"reported_epoch":146,"state":"active+clean","last_fresh":"2023-06-03T20:04:44.765279+0000","last_change":"2023-06-03T20:00:13.173625+0000","last_active":"2023-06-03T20:04:44.765279+0000","last_peered":"2023-06-03T20:04:44.765279+0000","last_clean":"2023-06-03T20:04:44.765279+0000","last_became_active":"2023-06-03T20:00:13.173348+0000","last_became_peered":"2023-06-03T20:00:13.173348+0000","last_unstale":"2023-06-03T20:04:44.765279+0000","last_undegraded":"2023-06-03T20:04:44.765279+0000","last_fullsized":"2023-06-03T20:04:44.765279+0000","mapping_epoch":127,"log_start":"134'6000","ondisk_log_start":"134'6000","created":127,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T20:00:11.873149+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T20:00:11.873149+0000","last_clean_scrub_stamp":"2023-06-03T20:00:11.873149+0000","objects_scrubbed":0,"log_size":7764,"log_dups_size":0,"ondisk_log_size":7764,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T03:41:17.261687+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13764,"num_write_kb":440448,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":114,"reported_epoch":146,"state":"active+clean","last_fresh":"2023-06-03T20:04:37.736083+0000","last_change":"2023-06-03T19:56:54.512219+0000","last_active":"2023-06-03T20:04:37.736083+0000","last_peered":"2023-06-03T20:04:37.736083+0000","last_clean":"2023-06-03T20:04:37.736083+0000","last_became_active":"2023-06-03T19:56:54.511771+0000","last_became_peered":"2023-06-03T19:56:54.511771+0000","last_unstale":"2023-06-03T20:04:37.736083+0000","last_undegraded":"2023-06-03T20:04:37.736083+0000","last_fullsized":"2023-06-03T20:04:37.736083+0000","mapping_epoch":119,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T19:35:47.796520+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":100,"reported_epoch":146,"state":"active+clean","last_fresh":"2023-06-03T20:04:37.734448+0000","last_change":"2023-06-03T19:57:28.471023+0000","last_active":"2023-06-03T20:04:37.734448+0000","last_peered":"2023-06-03T20:04:37.734448+0000","last_clean":"2023-06-03T20:04:37.734448+0000","last_became_active":"2023-06-03T19:57:28.470452+0000","last_became_peered":"2023-06-03T19:57:28.470452+0000","last_unstale":"2023-06-03T20:04:37.734448+0000","last_undegraded":"2023-06-03T20:04:37.734448+0000","last_fullsized":"2023-06-03T20:04:37.734448+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T05:58:33.305030+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.f","version":"146'13534","reported_seq":13562,"reported_epoch":146,"state":"active+clean","last_fresh":"2023-06-03T20:04:44.772563+0000","last_change":"2023-06-03T20:00:12.909234+0000","last_active":"2023-06-03T20:04:44.772563+0000","last_peered":"2023-06-03T20:04:44.772563+0000","last_clean":"2023-06-03T20:04:44.772563+0000","last_became_active":"2023-06-03T20:00:12.908871+0000","last_became_peered":"2023-06-03T20:00:12.908871+0000","last_unstale":"2023-06-03T20:04:44.772563+0000","last_undegraded":"2023-06-03T20:04:44.772563+0000","last_fullsized":"2023-06-03T20:04:44.772563+0000","mapping_epoch":127,"log_start":"133'3500","ondisk_log_start":"133'3500","created":127,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T20:00:11.873149+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T20:00:11.873149+0000","last_clean_scrub_stamp":"2023-06-03T20:00:11.873149+0000","objects_scrubbed":0,"log_size":10034,"log_dups_size":0,"ondisk_log_size":10034,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13534,"num_write_kb":433088,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":119,"reported_epoch":146,"state":"active+clean","last_fresh":"2023-06-03T20:04:37.728674+0000","last_change":"2023-06-03T19:51:59.392796+0000","last_active":"2023-06-03T20:04:37.728674+0000","last_peered":"2023-06-03T20:04:37.728674+0000","last_clean":"2023-06-03T20:04:37.728674+0000","last_became_active":"2023-06-03T19:51:59.380207+0000","last_became_peered":"2023-06-03T19:51:59.380207+0000","last_unstale":"2023-06-03T20:04:37.728674+0000","last_undegraded":"2023-06-03T20:04:37.728674+0000","last_fullsized":"2023-06-03T20:04:37.728674+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:32:20.692043+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:32:20.692043+0000","last_clean_scrub_stamp":"2023-06-03T19:32:20.692043+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T00:28:05.866873+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":129,"reported_epoch":146,"state":"active+clean","last_fresh":"2023-06-03T20:04:37.745817+0000","last_change":"2023-06-03T19:56:54.512607+0000","last_active":"2023-06-03T20:04:37.745817+0000","last_peered":"2023-06-03T20:04:37.745817+0000","last_clean":"2023-06-03T20:04:37.745817+0000","last_became_active":"2023-06-03T19:56:54.512282+0000","last_became_peered":"2023-06-03T19:56:54.512282+0000","last_unstale":"2023-06-03T20:04:37.745817+0000","last_undegraded":"2023-06-03T20:04:37.745817+0000","last_fullsized":"2023-06-03T20:04:37.745817+0000","mapping_epoch":119,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":86,"reported_epoch":146,"state":"active+clean","last_fresh":"2023-06-03T20:04:37.727888+0000","last_change":"2023-06-03T19:57:28.470295+0000","last_active":"2023-06-03T20:04:37.727888+0000","last_peered":"2023-06-03T20:04:37.727888+0000","last_clean":"2023-06-03T20:04:37.727888+0000","last_became_active":"2023-06-03T19:57:28.469383+0000","last_became_peered":"2023-06-03T19:57:28.469383+0000","last_unstale":"2023-06-03T20:04:37.727888+0000","last_undegraded":"2023-06-03T20:04:37.727888+0000","last_fullsized":"2023-06-03T20:04:37.727888+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T05:35:19.667447+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.9","version":"146'13826","reported_seq":13856,"reported_epoch":146,"state":"active+clean","last_fresh":"2023-06-03T20:04:44.745172+0000","last_change":"2023-06-03T20:00:12.910626+0000","last_active":"2023-06-03T20:04:44.745172+0000","last_peered":"2023-06-03T20:04:44.745172+0000","last_clean":"2023-06-03T20:04:44.745172+0000","last_became_active":"2023-06-03T20:00:12.910285+0000","last_became_peered":"2023-06-03T20:00:12.910285+0000","last_unstale":"2023-06-03T20:04:44.745172+0000","last_undegraded":"2023-06-03T20:04:44.745172+0000","last_fullsized":"2023-06-03T20:04:44.745172+0000","mapping_epoch":127,"log_start":"133'3800","ondisk_log_start":"133'3800","created":127,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T20:00:11.873149+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T20:00:11.873149+0000","last_clean_scrub_stamp":"2023-06-03T20:00:11.873149+0000","objects_scrubbed":0,"log_size":10026,"log_dups_size":0,"ondisk_log_size":10026,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T04:42:34.630759+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13826,"num_write_kb":442432,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,5],"acting":[0,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":118,"reported_epoch":146,"state":"active+clean","last_fresh":"2023-06-03T20:04:37.734269+0000","last_change":"2023-06-03T19:51:59.373777+0000","last_active":"2023-06-03T20:04:37.734269+0000","last_peered":"2023-06-03T20:04:37.734269+0000","last_clean":"2023-06-03T20:04:37.734269+0000","last_became_active":"2023-06-03T19:51:59.372988+0000","last_became_peered":"2023-06-03T19:51:59.372988+0000","last_unstale":"2023-06-03T20:04:37.734269+0000","last_undegraded":"2023-06-03T20:04:37.734269+0000","last_fullsized":"2023-06-03T20:04:37.734269+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:32:20.692043+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:32:20.692043+0000","last_clean_scrub_stamp":"2023-06-03T19:32:20.692043+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":91,"reported_epoch":145,"state":"active+clean","last_fresh":"2023-06-03T20:04:32.706630+0000","last_change":"2023-06-03T19:57:28.460195+0000","last_active":"2023-06-03T20:04:32.706630+0000","last_peered":"2023-06-03T20:04:32.706630+0000","last_clean":"2023-06-03T20:04:32.706630+0000","last_became_active":"2023-06-03T19:57:28.459823+0000","last_became_peered":"2023-06-03T19:57:28.459823+0000","last_unstale":"2023-06-03T20:04:32.706630+0000","last_undegraded":"2023-06-03T20:04:32.706630+0000","last_fullsized":"2023-06-03T20:04:32.706630+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T20:08:48.686380+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":101,"reported_epoch":146,"state":"active+clean","last_fresh":"2023-06-03T20:04:37.728607+0000","last_change":"2023-06-03T19:56:54.516859+0000","last_active":"2023-06-03T20:04:37.728607+0000","last_peered":"2023-06-03T20:04:37.728607+0000","last_clean":"2023-06-03T20:04:37.728607+0000","last_became_active":"2023-06-03T19:56:54.516521+0000","last_became_peered":"2023-06-03T19:56:54.516521+0000","last_unstale":"2023-06-03T20:04:37.728607+0000","last_undegraded":"2023-06-03T20:04:37.728607+0000","last_fullsized":"2023-06-03T20:04:37.728607+0000","mapping_epoch":119,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T20:03:25.227382+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.a","version":"146'13772","reported_seq":13800,"reported_epoch":146,"state":"active+clean","last_fresh":"2023-06-03T20:04:44.770905+0000","last_change":"2023-06-03T20:00:12.909720+0000","last_active":"2023-06-03T20:04:44.770905+0000","last_peered":"2023-06-03T20:04:44.770905+0000","last_clean":"2023-06-03T20:04:44.770905+0000","last_became_active":"2023-06-03T20:00:12.909428+0000","last_became_peered":"2023-06-03T20:00:12.909428+0000","last_unstale":"2023-06-03T20:04:44.770905+0000","last_undegraded":"2023-06-03T20:04:44.770905+0000","last_fullsized":"2023-06-03T20:04:44.770905+0000","mapping_epoch":127,"log_start":"133'3700","ondisk_log_start":"133'3700","created":127,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T20:00:11.873149+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T20:00:11.873149+0000","last_clean_scrub_stamp":"2023-06-03T20:00:11.873149+0000","objects_scrubbed":0,"log_size":10072,"log_dups_size":0,"ondisk_log_size":10072,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13772,"num_write_kb":440704,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,7],"acting":[5,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":121,"reported_epoch":145,"state":"active+clean","last_fresh":"2023-06-03T20:04:32.727042+0000","last_change":"2023-06-03T19:56:54.504859+0000","last_active":"2023-06-03T20:04:32.727042+0000","last_peered":"2023-06-03T20:04:32.727042+0000","last_clean":"2023-06-03T20:04:32.727042+0000","last_became_active":"2023-06-03T19:56:54.504455+0000","last_became_peered":"2023-06-03T19:56:54.504455+0000","last_unstale":"2023-06-03T20:04:32.727042+0000","last_undegraded":"2023-06-03T20:04:32.727042+0000","last_fullsized":"2023-06-03T20:04:32.727042+0000","mapping_epoch":119,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:32:20.692043+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:32:20.692043+0000","last_clean_scrub_stamp":"2023-06-03T19:32:20.692043+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T06:25:45.522100+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":127,"reported_epoch":146,"state":"active+clean","last_fresh":"2023-06-03T20:04:37.725175+0000","last_change":"2023-06-03T19:56:11.754048+0000","last_active":"2023-06-03T20:04:37.725175+0000","last_peered":"2023-06-03T20:04:37.725175+0000","last_clean":"2023-06-03T20:04:37.725175+0000","last_became_active":"2023-06-03T19:56:11.753687+0000","last_became_peered":"2023-06-03T19:56:11.753687+0000","last_unstale":"2023-06-03T20:04:37.725175+0000","last_undegraded":"2023-06-03T20:04:37.725175+0000","last_fullsized":"2023-06-03T20:04:37.725175+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":115,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T23:29:41.511496+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":113,"reported_epoch":146,"state":"active+clean","last_fresh":"2023-06-03T20:04:37.734659+0000","last_change":"2023-06-03T19:51:59.396573+0000","last_active":"2023-06-03T20:04:37.734659+0000","last_peered":"2023-06-03T20:04:37.734659+0000","last_clean":"2023-06-03T20:04:37.734659+0000","last_became_active":"2023-06-03T19:51:59.381593+0000","last_became_peered":"2023-06-03T19:51:59.381593+0000","last_unstale":"2023-06-03T20:04:37.734659+0000","last_undegraded":"2023-06-03T20:04:37.734659+0000","last_fullsized":"2023-06-03T20:04:37.734659+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T21:33:23.043772+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.8","version":"146'13716","reported_seq":13747,"reported_epoch":146,"state":"active+clean","last_fresh":"2023-06-03T20:04:44.772350+0000","last_change":"2023-06-03T20:00:12.908817+0000","last_active":"2023-06-03T20:04:44.772350+0000","last_peered":"2023-06-03T20:04:44.772350+0000","last_clean":"2023-06-03T20:04:44.772350+0000","last_became_active":"2023-06-03T20:00:12.908431+0000","last_became_peered":"2023-06-03T20:00:12.908431+0000","last_unstale":"2023-06-03T20:04:44.772350+0000","last_undegraded":"2023-06-03T20:04:44.772350+0000","last_fullsized":"2023-06-03T20:04:44.772350+0000","mapping_epoch":127,"log_start":"134'6000","ondisk_log_start":"134'6000","created":127,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T20:00:11.873149+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T20:00:11.873149+0000","last_clean_scrub_stamp":"2023-06-03T20:00:11.873149+0000","objects_scrubbed":0,"log_size":7716,"log_dups_size":0,"ondisk_log_size":7716,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T21:11:15.734132+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13716,"num_write_kb":438912,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":118,"reported_epoch":146,"state":"active+clean","last_fresh":"2023-06-03T20:04:37.739960+0000","last_change":"2023-06-03T19:56:11.752151+0000","last_active":"2023-06-03T20:04:37.739960+0000","last_peered":"2023-06-03T20:04:37.739960+0000","last_clean":"2023-06-03T20:04:37.739960+0000","last_became_active":"2023-06-03T19:56:11.751767+0000","last_became_peered":"2023-06-03T19:56:11.751767+0000","last_unstale":"2023-06-03T20:04:37.739960+0000","last_undegraded":"2023-06-03T20:04:37.739960+0000","last_fullsized":"2023-06-03T20:04:37.739960+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":115,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:32:20.692043+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:32:20.692043+0000","last_clean_scrub_stamp":"2023-06-03T19:32:20.692043+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"72'139","reported_seq":350,"reported_epoch":146,"state":"active+clean","last_fresh":"2023-06-03T20:04:37.738468+0000","last_change":"2023-06-03T19:47:06.731872+0000","last_active":"2023-06-03T20:04:37.738468+0000","last_peered":"2023-06-03T20:04:37.738468+0000","last_clean":"2023-06-03T20:04:37.738468+0000","last_became_active":"2023-06-03T19:47:06.731175+0000","last_became_peered":"2023-06-03T19:47:06.731175+0000","last_unstale":"2023-06-03T20:04:37.738468+0000","last_undegraded":"2023-06-03T20:04:37.738468+0000","last_fullsized":"2023-06-03T20:04:37.738468+0000","mapping_epoch":86,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":87,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:27:28.100808+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:27:28.100808+0000","last_clean_scrub_stamp":"2023-06-03T19:27:28.100808+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":94,"reported_epoch":146,"state":"active+clean","last_fresh":"2023-06-03T20:04:37.739984+0000","last_change":"2023-06-03T19:56:54.509571+0000","last_active":"2023-06-03T20:04:37.739984+0000","last_peered":"2023-06-03T20:04:37.739984+0000","last_clean":"2023-06-03T20:04:37.739984+0000","last_became_active":"2023-06-03T19:56:54.509213+0000","last_became_peered":"2023-06-03T19:56:54.509213+0000","last_unstale":"2023-06-03T20:04:37.739984+0000","last_undegraded":"2023-06-03T20:04:37.739984+0000","last_fullsized":"2023-06-03T20:04:37.739984+0000","mapping_epoch":119,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T21:26:56.003203+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":113,"reported_epoch":146,"state":"active+clean","last_fresh":"2023-06-03T20:04:37.716487+0000","last_change":"2023-06-03T19:57:28.459664+0000","last_active":"2023-06-03T20:04:37.716487+0000","last_peered":"2023-06-03T20:04:37.716487+0000","last_clean":"2023-06-03T20:04:37.716487+0000","last_became_active":"2023-06-03T19:57:28.459289+0000","last_became_peered":"2023-06-03T19:57:28.459289+0000","last_unstale":"2023-06-03T20:04:37.716487+0000","last_undegraded":"2023-06-03T20:04:37.716487+0000","last_fullsized":"2023-06-03T20:04:37.716487+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"9.b","version":"146'14000","reported_seq":14030,"reported_epoch":146,"state":"active+clean","last_fresh":"2023-06-03T20:04:44.721477+0000","last_change":"2023-06-03T20:00:12.908303+0000","last_active":"2023-06-03T20:04:44.721477+0000","last_peered":"2023-06-03T20:04:44.721477+0000","last_clean":"2023-06-03T20:04:44.721477+0000","last_became_active":"2023-06-03T20:00:12.907799+0000","last_became_peered":"2023-06-03T20:00:12.907799+0000","last_unstale":"2023-06-03T20:04:44.721477+0000","last_undegraded":"2023-06-03T20:04:44.721477+0000","last_fullsized":"2023-06-03T20:04:44.721477+0000","mapping_epoch":127,"log_start":"133'3900","ondisk_log_start":"133'3900","created":127,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T20:00:11.873149+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T20:00:11.873149+0000","last_clean_scrub_stamp":"2023-06-03T20:00:11.873149+0000","objects_scrubbed":0,"log_size":10100,"log_dups_size":0,"ondisk_log_size":10100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T23:11:45.966674+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":14000,"num_write_kb":448000,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,5],"acting":[0,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":133,"reported_epoch":146,"state":"active+clean","last_fresh":"2023-06-03T20:04:37.738779+0000","last_change":"2023-06-03T19:51:59.358157+0000","last_active":"2023-06-03T20:04:37.738779+0000","last_peered":"2023-06-03T20:04:37.738779+0000","last_clean":"2023-06-03T20:04:37.738779+0000","last_became_active":"2023-06-03T19:51:59.357363+0000","last_became_peered":"2023-06-03T19:51:59.357363+0000","last_unstale":"2023-06-03T20:04:37.738779+0000","last_undegraded":"2023-06-03T20:04:37.738779+0000","last_fullsized":"2023-06-03T20:04:37.738779+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:32:20.692043+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:32:20.692043+0000","last_clean_scrub_stamp":"2023-06-03T19:32:20.692043+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":118,"reported_epoch":145,"state":"active+clean","last_fresh":"2023-06-03T20:04:32.725106+0000","last_change":"2023-06-03T19:57:28.459611+0000","last_active":"2023-06-03T20:04:32.725106+0000","last_peered":"2023-06-03T20:04:32.725106+0000","last_clean":"2023-06-03T20:04:32.725106+0000","last_became_active":"2023-06-03T19:57:28.459264+0000","last_became_peered":"2023-06-03T19:57:28.459264+0000","last_unstale":"2023-06-03T20:04:32.725106+0000","last_undegraded":"2023-06-03T20:04:32.725106+0000","last_fullsized":"2023-06-03T20:04:32.725106+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T22:15:06.557231+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":119,"reported_epoch":145,"state":"active+clean","last_fresh":"2023-06-03T20:04:32.725504+0000","last_change":"2023-06-03T19:56:11.745678+0000","last_active":"2023-06-03T20:04:32.725504+0000","last_peered":"2023-06-03T20:04:32.725504+0000","last_clean":"2023-06-03T20:04:32.725504+0000","last_became_active":"2023-06-03T19:56:11.744966+0000","last_became_peered":"2023-06-03T19:56:11.744966+0000","last_unstale":"2023-06-03T20:04:32.725504+0000","last_undegraded":"2023-06-03T20:04:32.725504+0000","last_fullsized":"2023-06-03T20:04:32.725504+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":115,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T21:12:06.830295+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.e","version":"146'13680","reported_seq":13708,"reported_epoch":146,"state":"active+clean","last_fresh":"2023-06-03T20:04:44.724971+0000","last_change":"2023-06-03T20:00:12.908248+0000","last_active":"2023-06-03T20:04:44.724971+0000","last_peered":"2023-06-03T20:04:44.724971+0000","last_clean":"2023-06-03T20:04:44.724971+0000","last_became_active":"2023-06-03T20:00:12.907900+0000","last_became_peered":"2023-06-03T20:00:12.907900+0000","last_unstale":"2023-06-03T20:04:44.724971+0000","last_undegraded":"2023-06-03T20:04:44.724971+0000","last_fullsized":"2023-06-03T20:04:44.724971+0000","mapping_epoch":127,"log_start":"134'5900","ondisk_log_start":"134'5900","created":127,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T20:00:11.873149+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T20:00:11.873149+0000","last_clean_scrub_stamp":"2023-06-03T20:00:11.873149+0000","objects_scrubbed":0,"log_size":7780,"log_dups_size":0,"ondisk_log_size":7780,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13680,"num_write_kb":437760,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":133,"reported_epoch":146,"state":"active+clean","last_fresh":"2023-06-03T20:04:37.742103+0000","last_change":"2023-06-03T19:57:28.466096+0000","last_active":"2023-06-03T20:04:37.742103+0000","last_peered":"2023-06-03T20:04:37.742103+0000","last_clean":"2023-06-03T20:04:37.742103+0000","last_became_active":"2023-06-03T19:57:28.464954+0000","last_became_peered":"2023-06-03T19:57:28.464954+0000","last_unstale":"2023-06-03T20:04:37.742103+0000","last_undegraded":"2023-06-03T20:04:37.742103+0000","last_fullsized":"2023-06-03T20:04:37.742103+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:32:20.692043+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:32:20.692043+0000","last_clean_scrub_stamp":"2023-06-03T19:32:20.692043+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":127,"reported_epoch":146,"state":"active+clean","last_fresh":"2023-06-03T20:04:37.727840+0000","last_change":"2023-06-03T19:56:11.750241+0000","last_active":"2023-06-03T20:04:37.727840+0000","last_peered":"2023-06-03T20:04:37.727840+0000","last_clean":"2023-06-03T20:04:37.727840+0000","last_became_active":"2023-06-03T19:56:11.749687+0000","last_became_peered":"2023-06-03T19:56:11.749687+0000","last_unstale":"2023-06-03T20:04:37.727840+0000","last_undegraded":"2023-06-03T20:04:37.727840+0000","last_fullsized":"2023-06-03T20:04:37.727840+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":115,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T22:38:25.819929+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":89,"reported_epoch":145,"state":"active+clean","last_fresh":"2023-06-03T20:04:32.725267+0000","last_change":"2023-06-03T19:47:06.733532+0000","last_active":"2023-06-03T20:04:32.725267+0000","last_peered":"2023-06-03T20:04:32.725267+0000","last_clean":"2023-06-03T20:04:32.725267+0000","last_became_active":"2023-06-03T19:47:06.733131+0000","last_became_peered":"2023-06-03T19:47:06.733131+0000","last_unstale":"2023-06-03T20:04:32.725267+0000","last_undegraded":"2023-06-03T20:04:32.725267+0000","last_fullsized":"2023-06-03T20:04:32.725267+0000","mapping_epoch":86,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":87,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T23:46:03.788646+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.d","version":"146'14062","reported_seq":14091,"reported_epoch":146,"state":"active+clean","last_fresh":"2023-06-03T20:04:44.777277+0000","last_change":"2023-06-03T20:00:12.905723+0000","last_active":"2023-06-03T20:04:44.777277+0000","last_peered":"2023-06-03T20:04:44.777277+0000","last_clean":"2023-06-03T20:04:44.777277+0000","last_became_active":"2023-06-03T20:00:12.905437+0000","last_became_peered":"2023-06-03T20:00:12.905437+0000","last_unstale":"2023-06-03T20:04:44.777277+0000","last_undegraded":"2023-06-03T20:04:44.777277+0000","last_fullsized":"2023-06-03T20:04:44.777277+0000","mapping_epoch":127,"log_start":"134'6300","ondisk_log_start":"134'6300","created":127,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T20:00:11.873149+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T20:00:11.873149+0000","last_clean_scrub_stamp":"2023-06-03T20:00:11.873149+0000","objects_scrubbed":0,"log_size":7762,"log_dups_size":0,"ondisk_log_size":7762,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":14062,"num_write_kb":449921,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,5],"acting":[7,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.7","version":"60'2","reported_seq":99,"reported_epoch":146,"state":"active+clean","last_fresh":"2023-06-03T20:04:37.736718+0000","last_change":"2023-06-03T19:56:54.510148+0000","last_active":"2023-06-03T20:04:37.736718+0000","last_peered":"2023-06-03T20:04:37.736718+0000","last_clean":"2023-06-03T20:04:37.736718+0000","last_became_active":"2023-06-03T19:56:54.509771+0000","last_became_peered":"2023-06-03T19:56:54.509771+0000","last_unstale":"2023-06-03T20:04:37.736718+0000","last_undegraded":"2023-06-03T20:04:37.736718+0000","last_fullsized":"2023-06-03T20:04:37.736718+0000","mapping_epoch":119,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T00:52:42.807432+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":126,"reported_epoch":146,"state":"active+clean","last_fresh":"2023-06-03T20:04:37.727529+0000","last_change":"2023-06-03T19:57:28.470954+0000","last_active":"2023-06-03T20:04:37.727529+0000","last_peered":"2023-06-03T20:04:37.727529+0000","last_clean":"2023-06-03T20:04:37.727529+0000","last_became_active":"2023-06-03T19:57:28.470524+0000","last_became_peered":"2023-06-03T19:57:28.470524+0000","last_unstale":"2023-06-03T20:04:37.727529+0000","last_undegraded":"2023-06-03T20:04:37.727529+0000","last_fullsized":"2023-06-03T20:04:37.727529+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:32:20.692043+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:32:20.692043+0000","last_clean_scrub_stamp":"2023-06-03T19:32:20.692043+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T19:39:19.983271+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":97,"reported_epoch":147,"state":"active+clean","last_fresh":"2023-06-03T20:04:51.959005+0000","last_change":"2023-06-03T19:56:54.507670+0000","last_active":"2023-06-03T20:04:51.959005+0000","last_peered":"2023-06-03T20:04:51.959005+0000","last_clean":"2023-06-03T20:04:51.959005+0000","last_became_active":"2023-06-03T19:56:54.507293+0000","last_became_peered":"2023-06-03T19:56:54.507293+0000","last_unstale":"2023-06-03T20:04:51.959005+0000","last_undegraded":"2023-06-03T20:04:51.959005+0000","last_fullsized":"2023-06-03T20:04:51.959005+0000","mapping_epoch":119,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T01:49:19.966054+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"9.c","version":"146'13698","reported_seq":13729,"reported_epoch":146,"state":"active+clean","last_fresh":"2023-06-03T20:04:44.766852+0000","last_change":"2023-06-03T20:00:12.904758+0000","last_active":"2023-06-03T20:04:44.766852+0000","last_peered":"2023-06-03T20:04:44.766852+0000","last_clean":"2023-06-03T20:04:44.766852+0000","last_became_active":"2023-06-03T20:00:12.904435+0000","last_became_peered":"2023-06-03T20:00:12.904435+0000","last_unstale":"2023-06-03T20:04:44.766852+0000","last_undegraded":"2023-06-03T20:04:44.766852+0000","last_fullsized":"2023-06-03T20:04:44.766852+0000","mapping_epoch":127,"log_start":"134'6000","ondisk_log_start":"134'6000","created":127,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T20:00:11.873149+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T20:00:11.873149+0000","last_clean_scrub_stamp":"2023-06-03T20:00:11.873149+0000","objects_scrubbed":0,"log_size":7698,"log_dups_size":0,"ondisk_log_size":7698,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T04:00:13.693202+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13698,"num_write_kb":438336,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,7],"acting":[3,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":118,"reported_epoch":146,"state":"active+clean","last_fresh":"2023-06-03T20:04:37.716655+0000","last_change":"2023-06-03T19:56:11.752730+0000","last_active":"2023-06-03T20:04:37.716655+0000","last_peered":"2023-06-03T20:04:37.716655+0000","last_clean":"2023-06-03T20:04:37.716655+0000","last_became_active":"2023-06-03T19:56:11.752441+0000","last_became_peered":"2023-06-03T19:56:11.752441+0000","last_unstale":"2023-06-03T20:04:37.716655+0000","last_undegraded":"2023-06-03T20:04:37.716655+0000","last_fullsized":"2023-06-03T20:04:37.716655+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":115,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:32:20.692043+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:32:20.692043+0000","last_clean_scrub_stamp":"2023-06-03T19:32:20.692043+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"60'1","reported_seq":122,"reported_epoch":145,"state":"active+clean","last_fresh":"2023-06-03T20:04:32.725338+0000","last_change":"2023-06-03T19:57:28.463999+0000","last_active":"2023-06-03T20:04:32.725338+0000","last_peered":"2023-06-03T20:04:32.725338+0000","last_clean":"2023-06-03T20:04:32.725338+0000","last_became_active":"2023-06-03T19:57:28.463588+0000","last_became_peered":"2023-06-03T19:57:28.463588+0000","last_unstale":"2023-06-03T20:04:32.725338+0000","last_undegraded":"2023-06-03T20:04:32.725338+0000","last_fullsized":"2023-06-03T20:04:32.725338+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T21:44:11.969059+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":89,"reported_epoch":145,"state":"active+clean","last_fresh":"2023-06-03T20:04:32.705797+0000","last_change":"2023-06-03T19:57:28.456467+0000","last_active":"2023-06-03T20:04:32.705797+0000","last_peered":"2023-06-03T20:04:32.705797+0000","last_clean":"2023-06-03T20:04:32.705797+0000","last_became_active":"2023-06-03T19:57:28.456120+0000","last_became_peered":"2023-06-03T19:57:28.456120+0000","last_unstale":"2023-06-03T20:04:32.705797+0000","last_undegraded":"2023-06-03T20:04:32.705797+0000","last_fullsized":"2023-06-03T20:04:32.705797+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T07:28:14.306624+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"60'1","reported_seq":106,"reported_epoch":145,"state":"active+clean","last_fresh":"2023-06-03T20:04:34.995776+0000","last_change":"2023-06-03T19:57:28.465124+0000","last_active":"2023-06-03T20:04:34.995776+0000","last_peered":"2023-06-03T20:04:34.995776+0000","last_clean":"2023-06-03T20:04:34.995776+0000","last_became_active":"2023-06-03T19:57:28.464654+0000","last_became_peered":"2023-06-03T19:57:28.464654+0000","last_unstale":"2023-06-03T20:04:34.995776+0000","last_undegraded":"2023-06-03T20:04:34.995776+0000","last_fullsized":"2023-06-03T20:04:34.995776+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T01:12:32.305507+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":132,"reported_epoch":145,"state":"active+clean","last_fresh":"2023-06-03T20:04:32.726060+0000","last_change":"2023-06-03T19:56:54.511760+0000","last_active":"2023-06-03T20:04:32.726060+0000","last_peered":"2023-06-03T20:04:32.726060+0000","last_clean":"2023-06-03T20:04:32.726060+0000","last_became_active":"2023-06-03T19:56:54.511429+0000","last_became_peered":"2023-06-03T19:56:54.511429+0000","last_unstale":"2023-06-03T20:04:32.726060+0000","last_undegraded":"2023-06-03T20:04:32.726060+0000","last_fullsized":"2023-06-03T20:04:32.726060+0000","mapping_epoch":119,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T05:42:46.254343+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"60'2","reported_seq":131,"reported_epoch":146,"state":"active+clean","last_fresh":"2023-06-03T20:04:37.734462+0000","last_change":"2023-06-03T19:57:28.462252+0000","last_active":"2023-06-03T20:04:37.734462+0000","last_peered":"2023-06-03T20:04:37.734462+0000","last_clean":"2023-06-03T20:04:37.734462+0000","last_became_active":"2023-06-03T19:57:28.461844+0000","last_became_peered":"2023-06-03T19:57:28.461844+0000","last_unstale":"2023-06-03T20:04:37.734462+0000","last_undegraded":"2023-06-03T20:04:37.734462+0000","last_fullsized":"2023-06-03T20:04:37.734462+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":135,"reported_epoch":145,"state":"active+clean","last_fresh":"2023-06-03T20:04:32.706507+0000","last_change":"2023-06-03T19:57:28.469248+0000","last_active":"2023-06-03T20:04:32.706507+0000","last_peered":"2023-06-03T20:04:32.706507+0000","last_clean":"2023-06-03T20:04:32.706507+0000","last_became_active":"2023-06-03T19:57:28.468903+0000","last_became_peered":"2023-06-03T19:57:28.468903+0000","last_unstale":"2023-06-03T20:04:32.706507+0000","last_undegraded":"2023-06-03T20:04:32.706507+0000","last_fullsized":"2023-06-03T20:04:32.706507+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T06:08:20.956098+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"60'2","reported_seq":116,"reported_epoch":146,"state":"active+clean","last_fresh":"2023-06-03T20:04:37.745676+0000","last_change":"2023-06-03T19:57:28.462610+0000","last_active":"2023-06-03T20:04:37.745676+0000","last_peered":"2023-06-03T20:04:37.745676+0000","last_clean":"2023-06-03T20:04:37.745676+0000","last_became_active":"2023-06-03T19:57:28.460655+0000","last_became_peered":"2023-06-03T19:57:28.460655+0000","last_unstale":"2023-06-03T20:04:37.745676+0000","last_undegraded":"2023-06-03T20:04:37.745676+0000","last_fullsized":"2023-06-03T20:04:37.745676+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":135,"reported_epoch":145,"state":"active+clean","last_fresh":"2023-06-03T20:04:32.706340+0000","last_change":"2023-06-03T19:57:28.464690+0000","last_active":"2023-06-03T20:04:32.706340+0000","last_peered":"2023-06-03T20:04:32.706340+0000","last_clean":"2023-06-03T20:04:32.706340+0000","last_became_active":"2023-06-03T19:57:28.463856+0000","last_became_peered":"2023-06-03T19:57:28.463856+0000","last_unstale":"2023-06-03T20:04:32.706340+0000","last_undegraded":"2023-06-03T20:04:32.706340+0000","last_fullsized":"2023-06-03T20:04:32.706340+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T00:11:00.991088+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"60'1","reported_seq":126,"reported_epoch":147,"state":"active+clean","last_fresh":"2023-06-03T20:04:51.959711+0000","last_change":"2023-06-03T19:56:54.505096+0000","last_active":"2023-06-03T20:04:51.959711+0000","last_peered":"2023-06-03T20:04:51.959711+0000","last_clean":"2023-06-03T20:04:51.959711+0000","last_became_active":"2023-06-03T19:56:54.504255+0000","last_became_peered":"2023-06-03T19:56:54.504255+0000","last_unstale":"2023-06-03T20:04:51.959711+0000","last_undegraded":"2023-06-03T20:04:51.959711+0000","last_fullsized":"2023-06-03T20:04:51.959711+0000","mapping_epoch":119,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T19:52:25.922308+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":138,"reported_epoch":146,"state":"active+clean","last_fresh":"2023-06-03T20:04:37.736205+0000","last_change":"2023-06-03T19:56:54.503416+0000","last_active":"2023-06-03T20:04:37.736205+0000","last_peered":"2023-06-03T20:04:37.736205+0000","last_clean":"2023-06-03T20:04:37.736205+0000","last_became_active":"2023-06-03T19:56:54.503018+0000","last_became_peered":"2023-06-03T19:56:54.503018+0000","last_unstale":"2023-06-03T20:04:37.736205+0000","last_undegraded":"2023-06-03T20:04:37.736205+0000","last_fullsized":"2023-06-03T20:04:37.736205+0000","mapping_epoch":119,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T20:14:00.330616+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"60'4","reported_seq":107,"reported_epoch":145,"state":"active+clean","last_fresh":"2023-06-03T20:04:32.725836+0000","last_change":"2023-06-03T19:56:11.748116+0000","last_active":"2023-06-03T20:04:32.725836+0000","last_peered":"2023-06-03T20:04:32.725836+0000","last_clean":"2023-06-03T20:04:32.725836+0000","last_became_active":"2023-06-03T19:56:11.747636+0000","last_became_peered":"2023-06-03T19:56:11.747636+0000","last_unstale":"2023-06-03T20:04:32.725836+0000","last_undegraded":"2023-06-03T20:04:32.725836+0000","last_fullsized":"2023-06-03T20:04:32.725836+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":115,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T06:57:48.052055+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":107,"reported_epoch":146,"state":"active+clean","last_fresh":"2023-06-03T20:04:37.707837+0000","last_change":"2023-06-03T19:57:28.459234+0000","last_active":"2023-06-03T20:04:37.707837+0000","last_peered":"2023-06-03T20:04:37.707837+0000","last_clean":"2023-06-03T20:04:37.707837+0000","last_became_active":"2023-06-03T19:57:28.458896+0000","last_became_peered":"2023-06-03T19:57:28.458896+0000","last_unstale":"2023-06-03T20:04:37.707837+0000","last_undegraded":"2023-06-03T20:04:37.707837+0000","last_fullsized":"2023-06-03T20:04:37.707837+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T21:18:37.142280+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5486,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":62,"num_write_kb":30,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16458,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":30319,"internal_metadata":0},"log_size":56,"ondisk_log_size":56,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":3,"num_store_stats":3},{"poolid":9,"num_pg":16,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":220264,"num_write_kb":7048385,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139364,"ondisk_log_size":139364,"up":48,"acting":48,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":48,"seq":206158430614,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":397544,"kb_used_data":1952,"kb_used_omap":0,"kb_used_meta":395584,"kb_avail":93347608,"statfs":{"total":95995035648,"available":95587950592,"internally_reserved":0,"allocated":1998848,"data_stored":1518778,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":405078016},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 3 20:04:43 2023","interfaces":[{"interface":"back","average":{"1min":0.78400000000000003,"5min":1.1970000000000001,"15min":1.272},"min":{"1min":0.29699999999999999,"5min":0.25900000000000001,"15min":0.253},"max":{"1min":2.274,"5min":7.7839999999999998,"15min":8.6099999999999994},"last":1.046},{"interface":"front","average":{"1min":1.5820000000000001,"5min":1.655,"15min":1.4870000000000001},"min":{"1min":0.309,"5min":0.309,"15min":0.28499999999999998},"max":{"1min":9.2309999999999999,"5min":11.877000000000001,"15min":15.220000000000001},"last":0.93400000000000005}]},{"osd":1,"last update":"Sat Jun 3 20:04:23 2023","interfaces":[{"interface":"back","average":{"1min":1.0820000000000001,"5min":2.4700000000000002,"15min":1.899},"min":{"1min":0.373,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":5.2709999999999999,"5min":91.811000000000007,"15min":91.811000000000007},"last":1.117},{"interface":"front","average":{"1min":1.631,"5min":2.8399999999999999,"15min":1.9430000000000001},"min":{"1min":0.33000000000000002,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":5.5330000000000004,"5min":91.695999999999998,"15min":91.695999999999998},"last":1.2609999999999999}]},{"osd":2,"last update":"Sat Jun 3 20:04:23 2023","interfaces":[{"interface":"back","average":{"1min":0.83499999999999996,"5min":1.127,"15min":1.0549999999999999},"min":{"1min":0.34300000000000003,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":2.3069999999999999,"5min":7.71,"15min":7.71},"last":0.79600000000000004},{"interface":"front","average":{"1min":1.395,"5min":1.3799999999999999,"15min":1.121},"min":{"1min":0.51100000000000001,"5min":0.38500000000000001,"15min":0.38500000000000001},"max":{"1min":3.3479999999999999,"5min":8.3059999999999992,"15min":8.3059999999999992},"last":1.079}]},{"osd":3,"last update":"Sat Jun 3 20:04:11 2023","interfaces":[{"interface":"back","average":{"1min":2.1349999999999998,"5min":2.6419999999999999,"15min":1.639},"min":{"1min":0.35299999999999998,"5min":0.34699999999999998,"15min":0.34699999999999998},"max":{"1min":11.904,"5min":95.962000000000003,"15min":95.962000000000003},"last":0.96899999999999997},{"interface":"front","average":{"1min":2.4849999999999999,"5min":2.8519999999999999,"15min":1.7410000000000001},"min":{"1min":0.503,"5min":0.36399999999999999,"15min":0.36399999999999999},"max":{"1min":12.084,"5min":94.174999999999997,"15min":94.174999999999997},"last":1.1850000000000001}]},{"osd":4,"last update":"Sat Jun 3 20:04:38 2023","interfaces":[{"interface":"back","average":{"1min":0.90800000000000003,"5min":1.05,"15min":1.0429999999999999},"min":{"1min":0.501,"5min":0.35999999999999999,"15min":0.35999999999999999},"max":{"1min":2.4430000000000001,"5min":7.7350000000000003,"15min":7.7350000000000003},"last":1.3340000000000001},{"interface":"front","average":{"1min":2.048,"5min":1.7190000000000001,"15min":1.3180000000000001},"min":{"1min":0.38800000000000001,"5min":0.38800000000000001,"15min":0.38800000000000001},"max":{"1min":9.0370000000000008,"5min":12.917,"15min":12.917},"last":1.4279999999999999}]},{"osd":5,"last update":"Sat Jun 3 20:04:19 2023","interfaces":[{"interface":"back","average":{"1min":1.3819999999999999,"5min":1.2330000000000001,"15min":1.202},"min":{"1min":0.52400000000000002,"5min":0.443,"15min":0.30199999999999999},"max":{"1min":4.2519999999999998,"5min":7.6829999999999998,"15min":12.778},"last":1.161},{"interface":"front","average":{"1min":3.2349999999999999,"5min":1.9430000000000001,"15min":1.5309999999999999},"min":{"1min":0.55500000000000005,"5min":0.33100000000000002,"15min":0.30599999999999999},"max":{"1min":20.306999999999999,"5min":20.306999999999999,"15min":20.306999999999999},"last":1.3640000000000001}]},{"osd":6,"last update":"Sat Jun 3 20:04:19 2023","interfaces":[{"interface":"back","average":{"1min":1.6140000000000001,"5min":1.5109999999999999,"15min":1.373},"min":{"1min":0.53500000000000003,"5min":0.42199999999999999,"15min":0.36099999999999999},"max":{"1min":3.3999999999999999,"5min":8.3219999999999992,"15min":8.3219999999999992},"last":1.49},{"interface":"front","average":{"1min":2.8620000000000001,"5min":1.8109999999999999,"15min":1.3999999999999999},"min":{"1min":0.47499999999999998,"5min":0.36899999999999999,"15min":0.32900000000000001},"max":{"1min":9.532,"5min":9.532,"15min":9.532},"last":1.282}]}]},{"osd":6,"up_from":42,"seq":180388626847,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":307544,"kb_used_data":1936,"kb_used_omap":0,"kb_used_meta":305600,"kb_avail":93437608,"statfs":{"total":95995035648,"available":95680110592,"internally_reserved":0,"allocated":1982464,"data_stored":1518317,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":312934400},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 3 20:03:51 2023","interfaces":[{"interface":"back","average":{"1min":0.60899999999999999,"5min":1.4470000000000001,"15min":1.587},"min":{"1min":0.26400000000000001,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":3.3239999999999998,"5min":54.786000000000001,"15min":54.786000000000001},"last":1.0049999999999999},{"interface":"front","average":{"1min":0.872,"5min":1.5189999999999999,"15min":1.6719999999999999},"min":{"1min":0.317,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":3.335,"5min":54.936,"15min":54.936},"last":0.873}]},{"osd":1,"last update":"Sat Jun 3 20:04:26 2023","interfaces":[{"interface":"back","average":{"1min":0.83199999999999996,"5min":4.1619999999999999,"15min":2.367},"min":{"1min":0.32100000000000001,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":3.4580000000000002,"5min":160.38499999999999,"15min":160.38499999999999},"last":0.90300000000000002},{"interface":"front","average":{"1min":1.1799999999999999,"5min":3.532,"15min":2.0409999999999999},"min":{"1min":0.35199999999999998,"5min":0.29599999999999999,"15min":0.29599999999999999},"max":{"1min":4.0570000000000004,"5min":160.434,"15min":160.434},"last":0.83199999999999996}]},{"osd":2,"last update":"Sat Jun 3 20:04:26 2023","interfaces":[{"interface":"back","average":{"1min":0.876,"5min":0.875,"15min":0.85999999999999999},"min":{"1min":0.33600000000000002,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":2.879,"5min":3.4260000000000002,"15min":3.4260000000000002},"last":0.73499999999999999},{"interface":"front","average":{"1min":1.2689999999999999,"5min":0.93300000000000005,"15min":0.94199999999999995},"min":{"1min":0.36799999999999999,"5min":0.34799999999999998,"15min":0.34799999999999998},"max":{"1min":4.2389999999999999,"5min":4.2389999999999999,"15min":4.2389999999999999},"last":1.0780000000000001}]},{"osd":3,"last update":"Sat Jun 3 20:04:06 2023","interfaces":[{"interface":"back","average":{"1min":1.5620000000000001,"5min":4.2770000000000001,"15min":1.998},"min":{"1min":0.35299999999999998,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":5.7549999999999999,"5min":164.99199999999999,"15min":164.99199999999999},"last":0.80200000000000005},{"interface":"front","average":{"1min":1.3180000000000001,"5min":4.2569999999999997,"15min":2.089},"min":{"1min":0.316,"5min":0.316,"15min":0.316},"max":{"1min":4.5300000000000002,"5min":163.459,"15min":163.459},"last":0.95999999999999996}]},{"osd":4,"last update":"Sat Jun 3 20:04:44 2023","interfaces":[{"interface":"back","average":{"1min":1.1020000000000001,"5min":0.874,"15min":0.95699999999999996},"min":{"1min":0.33800000000000002,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":3.2959999999999998,"5min":3.6040000000000001,"15min":3.6040000000000001},"last":0.88300000000000001},{"interface":"front","average":{"1min":1.2509999999999999,"5min":0.90100000000000002,"15min":1.0820000000000001},"min":{"1min":0.40200000000000002,"5min":0.377,"15min":0.377},"max":{"1min":6.1150000000000002,"5min":6.1150000000000002,"15min":6.1150000000000002},"last":0.75600000000000001}]},{"osd":5,"last update":"Sat Jun 3 20:04:38 2023","interfaces":[{"interface":"back","average":{"1min":1.6830000000000001,"5min":1.0329999999999999,"15min":1.208},"min":{"1min":0.32200000000000001,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":6.1580000000000004,"5min":6.1580000000000004,"15min":11.083},"last":1.0429999999999999},{"interface":"front","average":{"1min":1.5,"5min":1.4990000000000001,"15min":1.411},"min":{"1min":0.29499999999999998,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":4.327,"5min":8.4160000000000004,"15min":10.154},"last":0.63700000000000001}]},{"osd":7,"last update":"Sat Jun 3 20:04:12 2023","interfaces":[{"interface":"back","average":{"1min":1.3580000000000001,"5min":1.3500000000000001,"15min":1.3340000000000001},"min":{"1min":0.40000000000000002,"5min":0.39400000000000002,"15min":0.313},"max":{"1min":6.944,"5min":8.7409999999999997,"15min":8.7409999999999997},"last":0.81499999999999995},{"interface":"front","average":{"1min":1.6319999999999999,"5min":1.395,"15min":1.4139999999999999},"min":{"1min":0.48699999999999999,"5min":0.33900000000000002,"15min":0.33000000000000002},"max":{"1min":7.3079999999999998,"5min":7.3079999999999998,"15min":12.407999999999999},"last":0.84799999999999998}]}]},{"osd":1,"up_from":103,"seq":442381631645,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":243340,"kb_used_data":1476,"kb_used_omap":4,"kb_used_meta":241851,"kb_avail":93501812,"statfs":{"total":95995035648,"available":95745855488,"internally_reserved":0,"allocated":1511424,"data_stored":942954,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":247656334},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 3 20:04:30 2023","interfaces":[{"interface":"back","average":{"1min":2.4689999999999999,"5min":2.7160000000000002,"15min":1.9410000000000001},"min":{"1min":0.245,"5min":0.185,"15min":0.185},"max":{"1min":11.050000000000001,"5min":119.902,"15min":119.902},"last":0.55400000000000005},{"interface":"front","average":{"1min":1.9650000000000001,"5min":2.492,"15min":1.97},"min":{"1min":0.254,"5min":0.22500000000000001,"15min":0.192},"max":{"1min":11.374000000000001,"5min":119.863,"15min":119.863},"last":0.61799999999999999}]},{"osd":2,"last update":"Sat Jun 3 20:04:34 2023","interfaces":[{"interface":"back","average":{"1min":1.3440000000000001,"5min":2.0939999999999999,"15min":1.1699999999999999},"min":{"1min":0.223,"5min":0.223,"15min":0.223},"max":{"1min":7.6760000000000002,"5min":119.946,"15min":119.946},"last":0.33600000000000002},{"interface":"front","average":{"1min":1.335,"5min":1.204,"15min":0.89000000000000001},"min":{"1min":0.33600000000000002,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":6.5449999999999999,"5min":9.8160000000000007,"15min":9.8160000000000007},"last":0.442}]},{"osd":3,"last update":"Sat Jun 3 20:04:05 2023","interfaces":[{"interface":"back","average":{"1min":2.593,"5min":1.6990000000000001,"15min":1.0529999999999999},"min":{"1min":0.29899999999999999,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":9.657,"5min":9.657,"15min":9.657},"last":0.59499999999999997},{"interface":"front","average":{"1min":2.456,"5min":1.6399999999999999,"15min":1.119},"min":{"1min":0.28999999999999998,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":9.7509999999999994,"5min":9.7509999999999994,"15min":9.7509999999999994},"last":0.53000000000000003}]},{"osd":4,"last update":"Sat Jun 3 20:04:40 2023","interfaces":[{"interface":"back","average":{"1min":2.0249999999999999,"5min":1.7190000000000001,"15min":1.2350000000000001},"min":{"1min":0.433,"5min":0.35999999999999999,"15min":0.35999999999999999},"max":{"1min":6.2640000000000002,"5min":9.9589999999999996,"15min":9.9589999999999996},"last":0.74199999999999999},{"interface":"front","average":{"1min":1.6240000000000001,"5min":2.6469999999999998,"15min":1.522},"min":{"1min":0.38100000000000001,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":6.1520000000000001,"5min":120.084,"15min":120.084},"last":0.80700000000000005}]},{"osd":5,"last update":"Sat Jun 3 20:04:30 2023","interfaces":[{"interface":"back","average":{"1min":2.1600000000000001,"5min":3.1230000000000002,"15min":2.3479999999999999},"min":{"1min":0.36099999999999999,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":7.0490000000000004,"5min":120.39700000000001,"15min":120.39700000000001},"last":0.76600000000000001},{"interface":"front","average":{"1min":1.996,"5min":2.9260000000000002,"15min":2.5070000000000001},"min":{"1min":0.56299999999999994,"5min":0.48299999999999998,"15min":0.40000000000000002},"max":{"1min":7.3700000000000001,"5min":120.34399999999999,"15min":120.34399999999999},"last":0.83599999999999997}]},{"osd":6,"last update":"Sat Jun 3 20:04:30 2023","interfaces":[{"interface":"back","average":{"1min":1.677,"5min":2.4660000000000002,"15min":2.2149999999999999},"min":{"1min":0.40600000000000003,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":7.4039999999999999,"5min":120.386,"15min":120.386},"last":0.65100000000000002},{"interface":"front","average":{"1min":2.214,"5min":2.6829999999999998,"15min":2.484},"min":{"1min":0.41399999999999998,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":9.7889999999999997,"5min":119.96299999999999,"15min":119.96299999999999},"last":0.86199999999999999}]},{"osd":7,"last update":"Sat Jun 3 20:04:30 2023","interfaces":[{"interface":"back","average":{"1min":3.27,"5min":3.1640000000000001,"15min":2.4670000000000001},"min":{"1min":0.40899999999999997,"5min":0.38200000000000001,"15min":0.38200000000000001},"max":{"1min":10.516999999999999,"5min":119.996,"15min":119.996},"last":0.93700000000000006},{"interface":"front","average":{"1min":3.327,"5min":3.105,"15min":2.113},"min":{"1min":0.44900000000000001,"5min":0.40400000000000003,"15min":0.371},"max":{"1min":8.2370000000000001,"5min":120.17100000000001,"15min":120.17100000000001},"last":0.72799999999999998}]}]},{"osd":0,"up_from":86,"seq":369367187671,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":260312,"kb_used_data":2064,"kb_used_omap":5,"kb_used_meta":258234,"kb_avail":93484840,"statfs":{"total":95995035648,"available":95728476160,"internally_reserved":0,"allocated":2113536,"data_stored":1527440,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":264432250},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Jun 3 20:04:18 2023","interfaces":[{"interface":"back","average":{"1min":1.53,"5min":2.597,"15min":2.0699999999999998},"min":{"1min":0.23300000000000001,"5min":0.192,"15min":0.192},"max":{"1min":5.9020000000000001,"5min":81.310000000000002,"15min":81.310000000000002},"last":0.73699999999999999},{"interface":"front","average":{"1min":2.3460000000000001,"5min":2.6379999999999999,"15min":1.8979999999999999},"min":{"1min":0.23300000000000001,"5min":0.187,"15min":0.187},"max":{"1min":8.2189999999999994,"5min":76.971999999999994,"15min":76.971999999999994},"last":0.60899999999999999}]},{"osd":2,"last update":"Sat Jun 3 20:04:18 2023","interfaces":[{"interface":"back","average":{"1min":2.0230000000000001,"5min":1.2430000000000001,"15min":0.85199999999999998},"min":{"1min":0.27500000000000002,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":6.2549999999999999,"5min":8.923,"15min":8.923},"last":0.83699999999999997},{"interface":"front","average":{"1min":1.3939999999999999,"5min":1.1830000000000001,"15min":0.89700000000000002},"min":{"1min":0.28199999999999997,"5min":0.218,"15min":0.218},"max":{"1min":4.0419999999999998,"5min":11.194000000000001,"15min":11.194000000000001},"last":0.64800000000000002}]},{"osd":3,"last update":"Sat Jun 3 20:04:10 2023","interfaces":[{"interface":"back","average":{"1min":2.278,"5min":1.409,"15min":1.006},"min":{"1min":0.252,"5min":0.216,"15min":0.216},"max":{"1min":6.343,"5min":7.5990000000000002,"15min":7.5990000000000002},"last":0.78600000000000003},{"interface":"front","average":{"1min":2.1779999999999999,"5min":1.448,"15min":0.98999999999999999},"min":{"1min":0.39800000000000002,"5min":0.35799999999999998,"15min":0.35399999999999998},"max":{"1min":8.4459999999999997,"5min":8.4459999999999997,"15min":8.4459999999999997},"last":0.55500000000000005}]},{"osd":4,"last update":"Sat Jun 3 20:04:44 2023","interfaces":[{"interface":"back","average":{"1min":1.9950000000000001,"5min":1.2410000000000001,"15min":1.1379999999999999},"min":{"1min":0.45400000000000001,"5min":0.33800000000000002,"15min":0.33800000000000002},"max":{"1min":6.319,"5min":6.319,"15min":6.319},"last":1.0980000000000001},{"interface":"front","average":{"1min":1.837,"5min":1.379,"15min":1.202},"min":{"1min":0.48099999999999998,"5min":0.38800000000000001,"15min":0.38800000000000001},"max":{"1min":7.3200000000000003,"5min":7.3200000000000003,"15min":7.3200000000000003},"last":0.90200000000000002}]},{"osd":5,"last update":"Sat Jun 3 20:04:44 2023","interfaces":[{"interface":"back","average":{"1min":2.0539999999999998,"5min":1.6020000000000001,"15min":1.607},"min":{"1min":0.34899999999999998,"5min":0.34899999999999998,"15min":0.33900000000000002},"max":{"1min":6.1029999999999998,"5min":8.3100000000000005,"15min":9.2059999999999995},"last":0.93999999999999995},{"interface":"front","average":{"1min":2.302,"5min":1.681,"15min":1.758},"min":{"1min":0.35099999999999998,"5min":0.35099999999999998,"15min":0.34899999999999998},"max":{"1min":8.2189999999999994,"5min":8.2189999999999994,"15min":24.661000000000001},"last":1.1379999999999999}]},{"osd":6,"last update":"Sat Jun 3 20:04:44 2023","interfaces":[{"interface":"back","average":{"1min":2.0800000000000001,"5min":1.4059999999999999,"15min":1.633},"min":{"1min":0.40999999999999998,"5min":0.38,"15min":0.33500000000000002},"max":{"1min":6.3739999999999997,"5min":6.3899999999999997,"15min":24.599},"last":1.121},{"interface":"front","average":{"1min":1.917,"5min":1.542,"15min":1.704},"min":{"1min":0.42799999999999999,"5min":0.42799999999999999,"15min":0.42799999999999999},"max":{"1min":7.2779999999999996,"5min":7.2779999999999996,"15min":12.474},"last":1.2270000000000001}]},{"osd":7,"last update":"Sat Jun 3 20:04:44 2023","interfaces":[{"interface":"back","average":{"1min":2.6349999999999998,"5min":1.831,"15min":1.786},"min":{"1min":0.54100000000000004,"5min":0.35099999999999998,"15min":0.35099999999999998},"max":{"1min":8.9779999999999998,"5min":11.196,"15min":24.634},"last":1.0700000000000001},{"interface":"front","average":{"1min":2.823,"5min":1.9750000000000001,"15min":1.772},"min":{"1min":0.48199999999999998,"5min":0.48199999999999998,"15min":0.40699999999999997},"max":{"1min":8.891,"5min":8.891,"15min":12.621},"last":1.159}]}]},{"osd":2,"up_from":114,"seq":489626271850,"num_pgs":22,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":49196,"kb_used_data":1444,"kb_used_omap":7,"kb_used_meta":47736,"kb_avail":93695956,"statfs":{"total":95995035648,"available":95944658944,"internally_reserved":0,"allocated":1478656,"data_stored":931839,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8111,"internal_metadata":48881745},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 3 20:04:26 2023","interfaces":[{"interface":"back","average":{"1min":1.1040000000000001,"5min":2.3879999999999999,"15min":1.2130000000000001},"min":{"1min":0.224,"5min":0.20899999999999999,"15min":0.20899999999999999},"max":{"1min":8.3200000000000003,"5min":126.416,"15min":126.416},"last":0.51800000000000002},{"interface":"front","average":{"1min":1.125,"5min":2.468,"15min":1.296},"min":{"1min":0.23899999999999999,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":4.4770000000000003,"5min":126.976,"15min":126.976},"last":0.54500000000000004}]},{"osd":1,"last update":"Sat Jun 3 20:04:26 2023","interfaces":[{"interface":"back","average":{"1min":1.363,"5min":1.4390000000000001,"15min":0.95299999999999996},"min":{"1min":0.26500000000000001,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":10.077,"5min":28.484999999999999,"15min":28.484999999999999},"last":0.63700000000000001},{"interface":"front","average":{"1min":1.4490000000000001,"5min":1.3400000000000001,"15min":0.89600000000000002},"min":{"1min":0.308,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":7.9509999999999996,"5min":28.588999999999999,"15min":28.588999999999999},"last":0.69199999999999995}]},{"osd":3,"last update":"Sat Jun 3 20:04:08 2023","interfaces":[{"interface":"back","average":{"1min":0.94799999999999995,"5min":1.2549999999999999,"15min":0.91200000000000003},"min":{"1min":0.307,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":2.6230000000000002,"5min":29.251999999999999,"15min":29.251999999999999},"last":0.57599999999999996},{"interface":"front","average":{"1min":1.599,"5min":2.2949999999999999,"15min":1.2669999999999999},"min":{"1min":0.32100000000000001,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":6.0389999999999997,"5min":54.549999999999997,"15min":54.549999999999997},"last":0.59099999999999997}]},{"osd":4,"last update":"Sat Jun 3 20:04:42 2023","interfaces":[{"interface":"back","average":{"1min":0.78200000000000003,"5min":0.84099999999999997,"15min":0.89700000000000002},"min":{"1min":0.32700000000000001,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":2.4729999999999999,"5min":7.9089999999999998,"15min":7.9089999999999998},"last":0.67100000000000004},{"interface":"front","average":{"1min":0.68200000000000005,"5min":0.83699999999999997,"15min":0.93300000000000005},"min":{"1min":0.38300000000000001,"5min":0.32600000000000001,"15min":0.32600000000000001},"max":{"1min":3.3300000000000001,"5min":3.7749999999999999,"15min":3.7749999999999999},"last":0.83899999999999997}]},{"osd":5,"last update":"Sat Jun 3 20:04:26 2023","interfaces":[{"interface":"back","average":{"1min":0.93799999999999994,"5min":0.99399999999999999,"15min":1.016},"min":{"1min":0.39700000000000002,"5min":0.36099999999999999,"15min":0.36099999999999999},"max":{"1min":3.2149999999999999,"5min":3.5339999999999998,"15min":3.5339999999999998},"last":1.1419999999999999},{"interface":"front","average":{"1min":1.2529999999999999,"5min":1.373,"15min":1.1539999999999999},"min":{"1min":0.41999999999999998,"5min":0.308,"15min":0.308},"max":{"1min":4.8659999999999997,"5min":7.8689999999999998,"15min":7.8689999999999998},"last":0.96199999999999997}]},{"osd":6,"last update":"Sat Jun 3 20:04:26 2023","interfaces":[{"interface":"back","average":{"1min":0.97499999999999998,"5min":1.028,"15min":1.03},"min":{"1min":0.46000000000000002,"5min":0.38500000000000001,"15min":0.38500000000000001},"max":{"1min":2.9460000000000002,"5min":6.7759999999999998,"15min":6.7759999999999998},"last":1.081},{"interface":"front","average":{"1min":1.0820000000000001,"5min":1.0309999999999999,"15min":1.0389999999999999},"min":{"1min":0.40400000000000003,"5min":0.36299999999999999,"15min":0.36299999999999999},"max":{"1min":4.4000000000000004,"5min":7.9550000000000001,"15min":7.9550000000000001},"last":0.98899999999999999}]},{"osd":7,"last update":"Sat Jun 3 20:04:26 2023","interfaces":[{"interface":"back","average":{"1min":2.48,"5min":1.7749999999999999,"15min":1.369},"min":{"1min":0.52300000000000002,"5min":0.44700000000000001,"15min":0.44700000000000001},"max":{"1min":7.3920000000000003,"5min":7.3920000000000003,"15min":7.3920000000000003},"last":1.1180000000000001},{"interface":"front","average":{"1min":2.3290000000000002,"5min":1.714,"15min":1.339},"min":{"1min":0.51500000000000001,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":6.1440000000000001,"5min":7.9320000000000004,"15min":7.9320000000000004},"last":1.0640000000000001}]}]},{"osd":3,"up_from":119,"seq":511101108322,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":111468,"kb_used_data":1444,"kb_used_omap":2,"kb_used_meta":110013,"kb_avail":93633684,"statfs":{"total":95995035648,"available":95880892416,"internally_reserved":0,"allocated":1478656,"data_stored":936607,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":112653365},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 3 20:04:12 2023","interfaces":[{"interface":"back","average":{"1min":1.5760000000000001,"5min":2.012,"15min":1.0880000000000001},"min":{"1min":0.19,"5min":0.19,"15min":0.19},"max":{"1min":6.5940000000000003,"5min":68.647999999999996,"15min":68.647999999999996},"last":0.81299999999999994},{"interface":"front","average":{"1min":1.9079999999999999,"5min":1.966,"15min":1.133},"min":{"1min":0.27600000000000002,"5min":0.224,"15min":0.224},"max":{"1min":8.3179999999999996,"5min":69.691000000000003,"15min":69.691000000000003},"last":0.53700000000000003}]},{"osd":1,"last update":"Sat Jun 3 20:04:12 2023","interfaces":[{"interface":"back","average":{"1min":2.1070000000000002,"5min":1.4730000000000001,"15min":0.95699999999999996},"min":{"1min":0.23000000000000001,"5min":0.23000000000000001,"15min":0.23000000000000001},"max":{"1min":4.077,"5min":10.065,"15min":10.065},"last":0.67500000000000004},{"interface":"front","average":{"1min":1.708,"5min":1.4119999999999999,"15min":0.97199999999999998},"min":{"1min":0.30099999999999999,"5min":0.23499999999999999,"15min":0.23499999999999999},"max":{"1min":6.6219999999999999,"5min":10.792999999999999,"15min":10.792999999999999},"last":0.61199999999999999}]},{"osd":2,"last update":"Sat Jun 3 20:04:12 2023","interfaces":[{"interface":"back","average":{"1min":1.833,"5min":1.246,"15min":0.84499999999999997},"min":{"1min":0.29399999999999998,"5min":0.245,"15min":0.245},"max":{"1min":8.3640000000000008,"5min":8.3640000000000008,"15min":8.3640000000000008},"last":0.84599999999999997},{"interface":"front","average":{"1min":1.752,"5min":1.304,"15min":0.93500000000000005},"min":{"1min":0.27800000000000002,"5min":0.24199999999999999,"15min":0.24199999999999999},"max":{"1min":4.1470000000000002,"5min":7.7290000000000001,"15min":7.7290000000000001},"last":0.745}]},{"osd":4,"last update":"Sat Jun 3 20:04:39 2023","interfaces":[{"interface":"back","average":{"1min":1.425,"5min":1.2609999999999999,"15min":1.089},"min":{"1min":0.40200000000000002,"5min":0.38300000000000001,"15min":0.38300000000000001},"max":{"1min":4.1429999999999998,"5min":4.774,"15min":4.774},"last":0.83499999999999996},{"interface":"front","average":{"1min":1.488,"5min":1.234,"15min":1.0900000000000001},"min":{"1min":0.42199999999999999,"5min":0.33000000000000002,"15min":0.33000000000000002},"max":{"1min":6.5229999999999997,"5min":7.0720000000000001,"15min":7.0720000000000001},"last":0.98599999999999999}]},{"osd":5,"last update":"Sat Jun 3 20:04:12 2023","interfaces":[{"interface":"back","average":{"1min":1.508,"5min":1.423,"15min":1.131},"min":{"1min":0.46000000000000002,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":3.8450000000000002,"5min":7.1230000000000002,"15min":7.1230000000000002},"last":1.056},{"interface":"front","average":{"1min":2.8300000000000001,"5min":1.883,"15min":1.306},"min":{"1min":0.39100000000000001,"5min":0.35599999999999998,"15min":0.35599999999999998},"max":{"1min":8.3949999999999996,"5min":8.3949999999999996,"15min":8.3949999999999996},"last":1.1910000000000001}]},{"osd":6,"last update":"Sat Jun 3 20:04:12 2023","interfaces":[{"interface":"back","average":{"1min":2.258,"5min":1.6200000000000001,"15min":1.232},"min":{"1min":0.44700000000000001,"5min":0.38600000000000001,"15min":0.38600000000000001},"max":{"1min":8.4169999999999998,"5min":8.4169999999999998,"15min":8.4169999999999998},"last":1.1080000000000001},{"interface":"front","average":{"1min":1.526,"5min":1.3680000000000001,"15min":1.1599999999999999},"min":{"1min":0.39000000000000001,"5min":0.39000000000000001,"15min":0.39000000000000001},"max":{"1min":4.1719999999999997,"5min":4.9480000000000004,"15min":4.9480000000000004},"last":0.89600000000000002}]},{"osd":7,"last update":"Sat Jun 3 20:04:12 2023","interfaces":[{"interface":"back","average":{"1min":2.1400000000000001,"5min":1.571,"15min":1.216},"min":{"1min":0.496,"5min":0.495,"15min":0.495},"max":{"1min":8.6910000000000007,"5min":8.6910000000000007,"15min":8.6910000000000007},"last":1.036},{"interface":"front","average":{"1min":1.8859999999999999,"5min":1.859,"15min":1.3009999999999999},"min":{"1min":0.622,"5min":0.435,"15min":0.435},"max":{"1min":6.5609999999999999,"5min":12.715999999999999,"15min":12.715999999999999},"last":1.2569999999999999}]}]},{"osd":4,"up_from":124,"seq":532575944795,"num_pgs":36,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":49536,"kb_used_data":1464,"kb_used_omap":10,"kb_used_meta":48053,"kb_avail":93695616,"statfs":{"total":95995035648,"available":95944310784,"internally_reserved":0,"allocated":1499136,"data_stored":933535,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":10652,"internal_metadata":49206884},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 3 20:04:47 2023","interfaces":[{"interface":"back","average":{"1min":1.573,"5min":1.6279999999999999,"15min":1.1910000000000001},"min":{"1min":0.25600000000000001,"5min":0.254,"15min":0.254},"max":{"1min":9.0449999999999999,"5min":57.786000000000001,"15min":57.786000000000001},"last":0.81699999999999995},{"interface":"front","average":{"1min":1.6080000000000001,"5min":1.605,"15min":1.2230000000000001},"min":{"1min":0.38700000000000001,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":9.0809999999999995,"5min":57.820999999999998,"15min":57.820999999999998},"last":1.085}]},{"osd":1,"last update":"Sat Jun 3 20:04:47 2023","interfaces":[{"interface":"back","average":{"1min":1.5069999999999999,"5min":2.0819999999999999,"15min":1.2749999999999999},"min":{"1min":0.316,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":4.2910000000000004,"5min":58.32,"15min":58.32},"last":0.70299999999999996},{"interface":"front","average":{"1min":1.3799999999999999,"5min":2.0089999999999999,"15min":1.248},"min":{"1min":0.32200000000000001,"5min":0.307,"15min":0.307},"max":{"1min":6.9509999999999996,"5min":58.286000000000001,"15min":58.286000000000001},"last":1.1870000000000001}]},{"osd":2,"last update":"Sat Jun 3 20:04:47 2023","interfaces":[{"interface":"back","average":{"1min":1.1870000000000001,"5min":0.94299999999999995,"15min":0.89600000000000002},"min":{"1min":0.33800000000000002,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":7.3109999999999999,"5min":7.3109999999999999,"15min":7.3109999999999999},"last":0.93500000000000005},{"interface":"front","average":{"1min":0.89600000000000002,"5min":0.86799999999999999,"15min":0.91100000000000003},"min":{"1min":0.373,"5min":0.34499999999999997,"15min":0.34499999999999997},"max":{"1min":2.4950000000000001,"5min":5.069,"15min":5.069},"last":1.421}]},{"osd":3,"last update":"Sat Jun 3 20:04:47 2023","interfaces":[{"interface":"back","average":{"1min":1.391,"5min":2.129,"15min":1.351},"min":{"1min":0.373,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":4.149,"5min":57.954999999999998,"15min":57.954999999999998},"last":1.149},{"interface":"front","average":{"1min":1.4610000000000001,"5min":2.169,"15min":1.4199999999999999},"min":{"1min":0.33700000000000002,"5min":0.33700000000000002,"15min":0.33700000000000002},"max":{"1min":4.1630000000000003,"5min":57.889000000000003,"15min":57.889000000000003},"last":1.3200000000000001}]},{"osd":5,"last update":"Sat Jun 3 20:04:47 2023","interfaces":[{"interface":"back","average":{"1min":1.2549999999999999,"5min":1.8320000000000001,"15min":1.25},"min":{"1min":0.313,"5min":0.313,"15min":0.313},"max":{"1min":5.9530000000000003,"5min":86.576999999999998,"15min":86.576999999999998},"last":1.2749999999999999},{"interface":"front","average":{"1min":1.742,"5min":2.0550000000000002,"15min":1.3480000000000001},"min":{"1min":0.33100000000000002,"5min":0.32600000000000001,"15min":0.32600000000000001},"max":{"1min":6.992,"5min":88.013000000000005,"15min":88.013000000000005},"last":0.96899999999999997}]},{"osd":6,"last update":"Sat Jun 3 20:04:47 2023","interfaces":[{"interface":"back","average":{"1min":0.70099999999999996,"5min":0.84599999999999997,"15min":0.97399999999999998},"min":{"1min":0.35399999999999998,"5min":0.35399999999999998,"15min":0.35399999999999998},"max":{"1min":2.214,"5min":5.1420000000000003,"15min":5.1420000000000003},"last":1.3480000000000001},{"interface":"front","average":{"1min":1.194,"5min":0.95899999999999996,"15min":1.042},"min":{"1min":0.41599999999999998,"5min":0.36499999999999999,"15min":0.36499999999999999},"max":{"1min":4.5759999999999996,"5min":7.2220000000000004,"15min":7.2220000000000004},"last":1.2470000000000001}]},{"osd":7,"last update":"Sat Jun 3 20:04:47 2023","interfaces":[{"interface":"back","average":{"1min":1.8460000000000001,"5min":1.4830000000000001,"15min":1.2529999999999999},"min":{"1min":0.39700000000000002,"5min":0.39700000000000002,"15min":0.39700000000000002},"max":{"1min":6.444,"5min":10.048999999999999,"15min":10.048999999999999},"last":1.373},{"interface":"front","average":{"1min":1.6080000000000001,"5min":1.5649999999999999,"15min":1.3},"min":{"1min":0.45600000000000002,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":4.899,"5min":9.3309999999999995,"15min":9.3309999999999995},"last":1.3979999999999999}]}]},{"osd":5,"up_from":36,"seq":154618823081,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":482892,"kb_used_data":1348,"kb_used_omap":0,"kb_used_meta":481536,"kb_avail":93262260,"statfs":{"total":95995035648,"available":95500554240,"internally_reserved":0,"allocated":1380352,"data_stored":923787,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":493092864},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 3 20:04:30 2023","interfaces":[{"interface":"back","average":{"1min":1.1240000000000001,"5min":1.4179999999999999,"15min":1.419},"min":{"1min":0.308,"5min":0.22800000000000001,"15min":0.22800000000000001},"max":{"1min":4.7519999999999998,"5min":37.689999999999998,"15min":37.689999999999998},"last":0.69599999999999995},{"interface":"front","average":{"1min":2.5,"5min":2.117,"15min":1.8600000000000001},"min":{"1min":0.373,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":7.6180000000000003,"5min":37.783999999999999,"15min":37.783999999999999},"last":0.86599999999999999}]},{"osd":1,"last update":"Sat Jun 3 20:04:30 2023","interfaces":[{"interface":"back","average":{"1min":1.972,"5min":3.988,"15min":2.464},"min":{"1min":0.30199999999999999,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":9.2390000000000008,"5min":114.681,"15min":114.681},"last":1.0700000000000001},{"interface":"front","average":{"1min":2.3769999999999998,"5min":4.0179999999999998,"15min":2.5299999999999998},"min":{"1min":0.33100000000000002,"5min":0.307,"15min":0.29999999999999999},"max":{"1min":7.6559999999999997,"5min":115.366,"15min":115.366},"last":0.73299999999999998}]},{"osd":2,"last update":"Sat Jun 3 20:04:30 2023","interfaces":[{"interface":"back","average":{"1min":1.1879999999999999,"5min":1.01,"15min":0.92700000000000005},"min":{"1min":0.36499999999999999,"5min":0.35099999999999998,"15min":0.34999999999999998},"max":{"1min":4.274,"5min":4.3529999999999998,"15min":4.3529999999999998},"last":0.78500000000000003},{"interface":"front","average":{"1min":2.3530000000000002,"5min":1.5700000000000001,"15min":1.1719999999999999},"min":{"1min":0.33600000000000002,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":7.6769999999999996,"5min":8.2119999999999997,"15min":8.2119999999999997},"last":0.97199999999999998}]},{"osd":3,"last update":"Sat Jun 3 20:04:10 2023","interfaces":[{"interface":"back","average":{"1min":3.6429999999999998,"5min":5.3079999999999998,"15min":2.4470000000000001},"min":{"1min":0.38700000000000001,"5min":0.34599999999999997,"15min":0.34599999999999997},"max":{"1min":29.134,"5min":261.14100000000002,"15min":261.14100000000002},"last":0.99199999999999999},{"interface":"front","average":{"1min":2.5699999999999998,"5min":4.7199999999999998,"15min":2.2400000000000002},"min":{"1min":0.44500000000000001,"5min":0.35399999999999998,"15min":0.35399999999999998},"max":{"1min":29.024000000000001,"5min":261.27100000000002,"15min":261.27100000000002},"last":1.04}]},{"osd":4,"last update":"Sat Jun 3 20:04:44 2023","interfaces":[{"interface":"back","average":{"1min":1.391,"5min":1.5800000000000001,"15min":1.1990000000000001},"min":{"1min":0.42099999999999999,"5min":0.378,"15min":0.378},"max":{"1min":5.8259999999999996,"5min":10.162000000000001,"15min":10.162000000000001},"last":0.90400000000000003},{"interface":"front","average":{"1min":0.97399999999999998,"5min":0.98699999999999999,"15min":1.115},"min":{"1min":0.39300000000000002,"5min":0.377,"15min":0.377},"max":{"1min":4.242,"5min":4.9980000000000002,"15min":4.9980000000000002},"last":0.58599999999999997}]},{"osd":6,"last update":"Sat Jun 3 20:04:30 2023","interfaces":[{"interface":"back","average":{"1min":1.607,"5min":1.7549999999999999,"15min":1.492},"min":{"1min":0.40799999999999997,"5min":0.40799999999999997,"15min":0.33000000000000002},"max":{"1min":5.8630000000000004,"5min":10.278,"15min":12.731999999999999},"last":1.0229999999999999},{"interface":"front","average":{"1min":1.464,"5min":1.7689999999999999,"15min":1.5680000000000001},"min":{"1min":0.39700000000000002,"5min":0.39700000000000002,"15min":0.32800000000000001},"max":{"1min":5.9109999999999996,"5min":13.714,"15min":13.714},"last":0.77100000000000002}]},{"osd":7,"last update":"Sat Jun 3 20:04:15 2023","interfaces":[{"interface":"back","average":{"1min":2.5270000000000001,"5min":2.2080000000000002,"15min":1.742},"min":{"1min":0.50800000000000001,"5min":0.503,"15min":0.379},"max":{"1min":9.6050000000000004,"5min":10.192,"15min":11.297000000000001},"last":1.0980000000000001},{"interface":"front","average":{"1min":1.0609999999999999,"5min":1.244,"15min":1.246},"min":{"1min":0.38400000000000001,"5min":0.38400000000000001,"15min":0.32100000000000001},"max":{"1min":3.8719999999999999,"5min":5.8719999999999999,"15min":7.4720000000000004},"last":0.94499999999999995}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6928,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4862,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":10652,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4786,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4932,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-06-03T20:04:57.448 INFO:tasks.cephadm.ceph_manager.ceph:increase pool size by 10 2023-06-03T20:04:57.449 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph osd pool set unique_pool_4 pg_num 26 2023-06-03T20:04:57.530 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:04:57.564 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:04:57.567 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:04:57.568 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:04:57.669 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:04:57.676 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:04:57 smithi089 bash[75068]: audit 1685822696.4153266 mgr.y (mgr.34104) 862 : audit 0 from='client.35937 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-06-03T20:04:57.677 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:04:57 smithi089 bash[75068]: cluster 1685822696.517992 mgr.y (mgr.34104) 863 : cluster 0 pgmap v785: 89 pgs: 89 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 77 op/s 2023-06-03T20:04:57.677 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:04:57 smithi089 bash[78452]: audit 1685822696.4153266 mgr.y (mgr.34104) 862 : audit 0 from='client.35937 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-06-03T20:04:57.677 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:04:57 smithi089 bash[78452]: cluster 1685822696.517992 mgr.y (mgr.34104) 863 : cluster 0 pgmap v785: 89 pgs: 89 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 77 op/s 2023-06-03T20:04:57.701 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:04:57.703 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:04:57.704 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:04:57.805 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:04:57.817 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:04:57 smithi164 bash[59462]: audit 1685822696.4153266 mgr.y (mgr.34104) 862 : audit 0 from='client.35937 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-06-03T20:04:57.817 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:04:57 smithi164 bash[59462]: cluster 1685822696.517992 mgr.y (mgr.34104) 863 : cluster 0 pgmap v785: 89 pgs: 89 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 77 op/s 2023-06-03T20:04:57.840 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:04:57.842 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:04:57.843 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:04:57.944 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:04:57.981 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:04:57.984 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:04:57.985 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:04:58.087 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:04:58.122 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:04:58.125 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:04:58.126 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:04:58.227 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:04:58.262 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:04:58.266 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:04:58.266 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:04:58.367 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:04:58.402 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:04:58.406 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:04:58.407 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:04:58.508 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:04:58.546 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:04:58.548 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:04:58.549 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:04:58.650 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:04:58.690 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:04:58.692 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:04:58.693 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:04:58.794 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:04:58.830 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:04:58.833 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:04:58.834 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:04:58.935 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:04:58.969 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:04:58.972 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:04:58.973 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:04:59.074 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:04:59.097 INFO:teuthology.orchestra.run.smithi164.stderr:Inferring config /var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/mon.b/config 2023-06-03T20:04:59.115 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:04:59.117 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:04:59.118 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:04:59.219 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:04:59.255 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:04:59.258 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:04:59.258 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:04:59.359 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:04:59.388 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:04:59.390 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:04:59.391 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:04:59.492 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:04:59.525 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:04:59.528 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:04:59.529 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:04:59.630 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:04:59.661 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:04:59.663 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:04:59.664 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:04:59.764 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:04:59.778 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:04:59 smithi089 bash[75068]: cluster 1685822698.5191064 mgr.y (mgr.34104) 864 : cluster 0 pgmap v786: 89 pgs: 89 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 9 op/s 2023-06-03T20:04:59.779 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:04:59 smithi089 bash[78452]: cluster 1685822698.5191064 mgr.y (mgr.34104) 864 : cluster 0 pgmap v786: 89 pgs: 89 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 9 op/s 2023-06-03T20:04:59.797 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:04:59.801 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:04:59.802 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:04:59.903 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:04:59.941 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:04:59.944 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:04:59.945 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:05:00.045 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:05:00.083 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:05:00.083 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:04:59 smithi164 bash[59462]: cluster 1685822698.5191064 mgr.y (mgr.34104) 864 : cluster 0 pgmap v786: 89 pgs: 89 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 9 op/s 2023-06-03T20:05:00.085 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:05:00.086 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:05:00.187 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:05:00.218 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:05:00.220 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:05:00.221 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:05:00.322 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:05:00.364 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:05:00.366 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:05:00.367 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:05:00.468 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:05:00.501 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:05:00.503 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:05:00.504 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:05:00.606 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:05:00.642 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:05:00.645 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:05:00.645 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:05:00.746 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:05:00.785 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:05:00.787 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:05:00.788 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:05:00.889 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:05:00.925 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:05:00.928 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:05:00.929 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:05:00.946 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/mon.a/config 2023-06-03T20:05:00.947 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/mon.a/config 2023-06-03T20:05:01.030 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:05:01.062 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:05:01.064 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:05:01.064 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:05:01.165 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:05:01.194 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:05:01.196 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:05:01.197 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:05:01.298 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:05:01.329 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:05:01.333 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:05:01.334 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:05:01.436 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:05:01.468 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:05:01.470 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:05:01.471 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:05:01.571 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:05:01.606 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:05:01.608 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:05:01.609 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:05:01.710 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:05:01.747 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:05:01 smithi089 bash[75068]: cluster 1685822700.519975 mgr.y (mgr.34104) 865 : cluster 0 pgmap v787: 89 pgs: 89 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2023-06-03T20:05:01.748 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:05:01 smithi089 bash[78452]: cluster 1685822700.519975 mgr.y (mgr.34104) 865 : cluster 0 pgmap v787: 89 pgs: 89 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2023-06-03T20:05:01.750 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:05:01.753 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:05:01.754 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:05:01.854 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:05:01.890 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:05:01.893 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:05:01.894 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:05:01.928 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:05:01 smithi164 bash[59462]: cluster 1685822700.519975 mgr.y (mgr.34104) 865 : cluster 0 pgmap v787: 89 pgs: 89 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2023-06-03T20:05:01.974 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/mon.a/config 2023-06-03T20:05:01.996 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:05:02.029 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:05:02.031 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:05:02.033 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:05:02.134 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:05:02.169 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:05:02.171 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:05:02.172 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:05:02.273 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:05:02.311 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:05:02.314 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:05:02.314 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:05:02.415 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:05:02.449 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:05:02.453 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:05:02.454 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:05:02.555 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:05:02.588 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:05:02.591 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:05:02.592 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:05:02.693 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:05:02.727 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:05:02.730 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:05:02.731 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:05:02.832 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:05:02.863 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:05:02.865 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:05:02.866 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:05:02.967 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:05:02.974 INFO:teuthology.orchestra.run.smithi164.stdout:{ 2023-06-03T20:05:02.975 INFO:teuthology.orchestra.run.smithi164.stdout: "ops": [], 2023-06-03T20:05:02.975 INFO:teuthology.orchestra.run.smithi164.stdout: "num_ops": 0 2023-06-03T20:05:02.975 INFO:teuthology.orchestra.run.smithi164.stdout:} 2023-06-03T20:05:02.997 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:05:03.000 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:05:03.000 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:05:03.101 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:05:03.133 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:05:03.136 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:05:03.136 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:05:03.237 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:05:03.266 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:05:03.268 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:05:03.269 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:05:03.370 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:05:03.399 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:05:03.402 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:05:03.403 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:05:03.504 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:05:03.535 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:05:03.538 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:05:03.538 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:05:03.639 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:05:03.672 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:05:03.674 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:05:03.675 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:05:03.775 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:05:03.808 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:05:03.811 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:05:03.812 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:05:03.823 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:05:03 smithi164 bash[59462]: cluster 1685822702.5206141 mgr.y (mgr.34104) 866 : cluster 0 pgmap v788: 89 pgs: 89 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2023-06-03T20:05:03.824 DEBUG:teuthology.orchestra.run.smithi164:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph daemon osd.5 dump_blocked_ops 2023-06-03T20:05:03.912 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:05:03.950 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:05:03.952 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:05:03.954 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:05:04.028 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:05:03 smithi089 bash[75068]: cluster 1685822702.5206141 mgr.y (mgr.34104) 866 : cluster 0 pgmap v788: 89 pgs: 89 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2023-06-03T20:05:04.029 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:05:03 smithi089 bash[78452]: cluster 1685822702.5206141 mgr.y (mgr.34104) 866 : cluster 0 pgmap v788: 89 pgs: 89 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2023-06-03T20:05:04.054 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:05:04.093 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:05:04.096 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:05:04.097 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:05:04.199 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:05:04.236 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:05:04.239 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:05:04.240 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:05:04.341 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:05:04.372 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:05:04.375 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:05:04.376 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:05:04.478 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:05:04.505 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:05:04.508 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:05:04.508 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:05:04.609 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:05:04.646 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:05:04.648 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:05:04.649 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:05:04.750 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:05:04.783 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:05:04.786 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:05:04.787 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:05:04.888 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:05:04.921 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:05:04.924 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:05:04.924 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:05:05.025 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:05:05.062 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:05:05.064 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:05:05.065 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:05:05.166 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:05:05.199 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:05:05.201 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:05:05.202 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:05:05.303 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:05:05.342 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:05:05.344 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:05:05.345 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:05:05.445 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:05:05.484 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:05:05.486 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:05:05.487 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:05:05.588 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:05:05.631 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:05:05.633 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:05:05.634 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:05:05.735 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:05:05.744 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:05:05 smithi089 bash[78452]: cluster 1685822704.521412 mgr.y (mgr.34104) 867 : cluster 0 pgmap v789: 89 pgs: 89 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2023-06-03T20:05:05.744 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:05:05 smithi089 bash[75068]: cluster 1685822704.521412 mgr.y (mgr.34104) 867 : cluster 0 pgmap v789: 89 pgs: 89 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2023-06-03T20:05:05.764 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:05:05.766 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:05:05.767 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:05:05.868 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:05:05.894 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:05:05.896 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:05:05.897 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:05:05.998 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:05:06.034 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:05:06.036 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:05:06.037 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:05:06.083 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:05:05 smithi164 bash[59462]: cluster 1685822704.521412 mgr.y (mgr.34104) 867 : cluster 0 pgmap v789: 89 pgs: 89 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2023-06-03T20:05:06.138 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:05:06.177 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:05:06.180 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:05:06.180 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:05:06.278 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 20:05:06 smithi089 bash[71163]: ::ffff:172.21.15.164 - - [03/Jun/2023:20:05:06] "GET /metrics HTTP/1.1" 200 228226 "" "Prometheus/2.33.4" 2023-06-03T20:05:06.281 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:05:06.311 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:05:06.313 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:05:06.314 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:05:06.415 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:05:06.450 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:05:06.452 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:05:06.453 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:05:06.554 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:05:06.590 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:05:06.593 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:05:06.594 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:05:06.695 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:05:06.725 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:05:06.728 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:05:06.728 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:05:06.829 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:05:06.866 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:05:06.869 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:05:06.869 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:05:06.970 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:05:07.004 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:05:07.008 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:05:07.009 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:05:07.109 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:05:07.143 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:05:07.146 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:05:07.147 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:05:07.248 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:05:07.281 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:05:07.284 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:05:07.284 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:05:07.385 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:05:07.422 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:05:07.424 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:05:07.425 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:05:07.526 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:05:07.562 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:05:07.565 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:05:07.565 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:05:07.666 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:05:07.692 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:05:07.694 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:05:07.695 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:05:07.795 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:05:07.826 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:05:07.828 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:05:07.829 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:05:07.930 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:05:07.942 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:05:07 smithi164 bash[59462]: cluster 1685822706.5222104 mgr.y (mgr.34104) 868 : cluster 0 pgmap v790: 89 pgs: 89 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2023-06-03T20:05:07.942 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:05:07 smithi164 bash[59462]: audit 1685822707.1047611 mon.c (mon.2) 421 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T20:05:07.958 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:05:07 smithi089 bash[78452]: cluster 1685822706.5222104 mgr.y (mgr.34104) 868 : cluster 0 pgmap v790: 89 pgs: 89 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2023-06-03T20:05:07.958 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:05:07 smithi089 bash[78452]: audit 1685822707.1047611 mon.c (mon.2) 421 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T20:05:07.958 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:05:07 smithi089 bash[75068]: cluster 1685822706.5222104 mgr.y (mgr.34104) 868 : cluster 0 pgmap v790: 89 pgs: 89 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2023-06-03T20:05:07.958 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:05:07 smithi089 bash[75068]: audit 1685822707.1047611 mon.c (mon.2) 421 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T20:05:07.963 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:05:07.966 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:05:07.967 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:05:08.068 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:05:08.103 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:05:08.107 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:05:08.107 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:05:08.208 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:05:08.237 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:05:08.239 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:05:08.240 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:05:08.341 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:05:08.373 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:05:08.376 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:05:08.376 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:05:08.478 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:05:08.518 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:05:08.521 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:05:08.522 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:05:08.622 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:05:08.629 INFO:teuthology.orchestra.run.smithi164.stderr:Inferring config /var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/mon.b/config 2023-06-03T20:05:08.659 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:05:08.662 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:05:08.663 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:05:08.764 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:05:08.798 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:05:08.801 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:05:08.801 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:05:08.902 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:05:08.928 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:05:08.932 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:05:08.933 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:05:09.034 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:05:09.072 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:05:09.075 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:05:09.076 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:05:09.177 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:05:09.216 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:05:09.219 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:05:09.219 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:05:09.320 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:05:09.352 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:05:09.354 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:05:09.355 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:05:09.456 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:05:09.491 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:05:09.495 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:05:09.496 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:05:09.597 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:05:09.627 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:05:09.630 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:05:09.630 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:05:09.731 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:05:09.771 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:05:09.773 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:05:09.774 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:05:09.874 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:05:09.886 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:05:09 smithi164 bash[59462]: cluster 1685822708.523295 mgr.y (mgr.34104) 869 : cluster 0 pgmap v791: 89 pgs: 89 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2023-06-03T20:05:09.907 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:05:09.909 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:05:09.910 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:05:10.011 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:05:10.047 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:05:10.049 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:05:10.049 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:05:10.108 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:05:09 smithi089 bash[78452]: cluster 1685822708.523295 mgr.y (mgr.34104) 869 : cluster 0 pgmap v791: 89 pgs: 89 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2023-06-03T20:05:10.108 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:05:09 smithi089 bash[75068]: cluster 1685822708.523295 mgr.y (mgr.34104) 869 : cluster 0 pgmap v791: 89 pgs: 89 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2023-06-03T20:05:10.150 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:05:10.183 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:05:10.186 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:05:10.186 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:05:10.287 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:05:10.322 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:05:10.323 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:05:10.324 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:05:10.427 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:05:10.458 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:05:10.461 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:05:10.462 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:05:10.563 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:05:10.593 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:05:10.597 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:05:10.597 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:05:10.698 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:05:10.725 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:05:10.728 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:05:10.728 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:05:10.830 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:05:10.861 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:05:10.865 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:05:10.865 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:05:10.966 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:05:11.001 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:05:11.003 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:05:11.004 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:05:11.039 INFO:teuthology.orchestra.run.smithi089.stdout:osd.0: {} 2023-06-03T20:05:11.069 INFO:teuthology.orchestra.run.smithi089.stdout:osd.1: {} 2023-06-03T20:05:11.095 INFO:teuthology.orchestra.run.smithi089.stdout:osd.2: {} 2023-06-03T20:05:11.105 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:05:11.121 INFO:teuthology.orchestra.run.smithi089.stdout:osd.3: {} 2023-06-03T20:05:11.130 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:05:11.132 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:05:11.133 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:05:11.151 INFO:teuthology.orchestra.run.smithi089.stdout:osd.4: {} 2023-06-03T20:05:11.177 INFO:teuthology.orchestra.run.smithi089.stdout:osd.5: {} 2023-06-03T20:05:11.204 INFO:teuthology.orchestra.run.smithi089.stdout:osd.6: {} 2023-06-03T20:05:11.229 INFO:teuthology.orchestra.run.smithi089.stdout:osd.7: {} 2023-06-03T20:05:11.230 INFO:teuthology.orchestra.run.smithi089.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-06-03T20:05:11.230 INFO:teuthology.orchestra.run.smithi089.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-06-03T20:05:11.230 INFO:teuthology.orchestra.run.smithi089.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-06-03T20:05:11.231 INFO:teuthology.orchestra.run.smithi089.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-06-03T20:05:11.231 INFO:teuthology.orchestra.run.smithi089.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-06-03T20:05:11.231 INFO:teuthology.orchestra.run.smithi089.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-06-03T20:05:11.231 INFO:teuthology.orchestra.run.smithi089.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-06-03T20:05:11.231 INFO:teuthology.orchestra.run.smithi089.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-06-03T20:05:11.234 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:05:11.267 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:05:11.269 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:05:11.270 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:05:11.371 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:05:11.395 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:05:11.397 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:05:11.398 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:05:11.499 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:05:11.529 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:05:11.532 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:05:11.532 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:05:11.633 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:05:11.670 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:05:11.672 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:05:11.673 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:05:11.774 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:05:11.778 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:05:11 smithi089 bash[78452]: cluster 1685822710.5242066 mgr.y (mgr.34104) 870 : cluster 0 pgmap v792: 89 pgs: 89 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2023-06-03T20:05:11.778 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:05:11 smithi089 bash[78452]: audit 1685822711.0087411 mon.c (mon.2) 422 : audit 0 from='client.? 172.21.15.89:0/393000661' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-03T20:05:11.779 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:05:11 smithi089 bash[75068]: cluster 1685822710.5242066 mgr.y (mgr.34104) 870 : cluster 0 pgmap v792: 89 pgs: 89 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2023-06-03T20:05:11.779 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:05:11 smithi089 bash[75068]: audit 1685822711.0087411 mon.c (mon.2) 422 : audit 0 from='client.? 172.21.15.89:0/393000661' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-03T20:05:11.798 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:05:11.800 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:05:11.801 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:05:11.833 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:05:11 smithi164 bash[59462]: cluster 1685822710.5242066 mgr.y (mgr.34104) 870 : cluster 0 pgmap v792: 89 pgs: 89 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2023-06-03T20:05:11.833 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:05:11 smithi164 bash[59462]: audit 1685822711.0087411 mon.c (mon.2) 422 : audit 0 from='client.? 172.21.15.89:0/393000661' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-03T20:05:11.902 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:05:11.940 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:05:11.943 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:05:11.943 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:05:12.044 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:05:12.083 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:05:12.086 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:05:12.087 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:05:12.187 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:05:12.220 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:05:12.223 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:05:12.223 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:05:12.324 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:05:12.357 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:05:12.360 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:05:12.360 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:05:12.462 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:05:12.479 INFO:teuthology.orchestra.run.smithi164.stdout:ERROR: (22) Invalid argument 2023-06-03T20:05:12.479 INFO:teuthology.orchestra.run.smithi164.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-06-03T20:05:12.503 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:05:12.507 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:05:12.507 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:05:12.608 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:05:12.645 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:05:12.648 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:05:12.648 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:05:12.679 INFO:teuthology.orchestra.run.smithi089.stderr:set pool 9 pg_num to 26 2023-06-03T20:05:12.679 INFO:teuthology.orchestra.run.smithi089.stderr:noscrub is set 2023-06-03T20:05:12.739 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:05:12 smithi089 bash[78452]: audit 1685822711.6889062 mon.a (mon.0) 766 : audit 1 from='client.? 172.21.15.89:0/4285693182' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pg_num", "val": "26"}]: dispatch 2023-06-03T20:05:12.740 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:05:12 smithi089 bash[78452]: audit 1685822711.72845 mon.c (mon.2) 423 : audit 1 from='client.? 172.21.15.89:0/2467873598' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-06-03T20:05:12.740 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:05:12 smithi089 bash[78452]: audit 1685822711.729173 mon.a (mon.0) 767 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-06-03T20:05:12.740 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:05:12 smithi089 bash[75068]: audit 1685822711.6889062 mon.a (mon.0) 766 : audit 1 from='client.? 172.21.15.89:0/4285693182' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pg_num", "val": "26"}]: dispatch 2023-06-03T20:05:12.740 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:05:12 smithi089 bash[75068]: audit 1685822711.72845 mon.c (mon.2) 423 : audit 1 from='client.? 172.21.15.89:0/2467873598' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-06-03T20:05:12.740 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:05:12 smithi089 bash[75068]: audit 1685822711.729173 mon.a (mon.0) 767 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-06-03T20:05:12.749 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:05:12.792 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:05:12.793 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:05:12.793 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:05:12.894 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:05:12.906 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:05:12 smithi164 bash[59462]: audit 1685822711.6889062 mon.a (mon.0) 766 : audit 1 from='client.? 172.21.15.89:0/4285693182' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pg_num", "val": "26"}]: dispatch 2023-06-03T20:05:12.906 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:05:12 smithi164 bash[59462]: audit 1685822711.72845 mon.c (mon.2) 423 : audit 1 from='client.? 172.21.15.89:0/2467873598' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-06-03T20:05:12.906 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:05:12 smithi164 bash[59462]: audit 1685822711.729173 mon.a (mon.0) 767 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-06-03T20:05:12.923 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:05:12.924 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:05:12.925 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:05:13.026 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:05:13.060 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:05:13.063 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:05:13.064 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:05:13.165 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:05:13.198 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:05:13.202 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:05:13.203 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:05:13.304 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:05:13.336 DEBUG:teuthology.orchestra.run.smithi164:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph daemon osd.5 dump_historic_ops 2023-06-03T20:05:13.339 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:05:13.341 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:05:13.341 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:05:13.442 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:05:13.472 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:05:13.475 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:05:13.476 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:05:13.577 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:05:13.614 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:05:13.617 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:05:13.618 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:05:13.719 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:05:13.760 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:05:13.763 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:05:13.763 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:05:13.865 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:05:13.879 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:05:13 smithi164 bash[59462]: cluster 1685822712.5249856 mgr.y (mgr.34104) 871 : cluster 0 pgmap v793: 89 pgs: 89 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2023-06-03T20:05:13.879 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:05:13 smithi164 bash[59462]: cluster 1685822712.656777 mon.a (mon.0) 768 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-06-03T20:05:13.879 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:05:13 smithi164 bash[59462]: cluster 1685822712.662829 osd.6 (osd.6) 67 : cluster 3 failed to encode map e148 with expected crc 2023-06-03T20:05:13.880 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:05:13 smithi164 bash[59462]: audit 1685822712.6703753 mon.a (mon.0) 769 : audit 1 from='client.? 172.21.15.89:0/4285693182' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pg_num", "val": "26"}]': finished 2023-06-03T20:05:13.880 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:05:13 smithi164 bash[59462]: audit 1685822712.6704817 mon.a (mon.0) 770 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-06-03T20:05:13.880 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:05:13 smithi164 bash[59462]: cluster 1685822712.6705842 mon.a (mon.0) 771 : cluster 0 osdmap e148: 8 total, 8 up, 8 in 2023-06-03T20:05:13.904 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:05:13.907 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:05:13.908 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:05:14.009 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:05:14.028 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:05:13 smithi089 bash[75068]: cluster 1685822712.5249856 mgr.y (mgr.34104) 871 : cluster 0 pgmap v793: 89 pgs: 89 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2023-06-03T20:05:14.029 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:05:13 smithi089 bash[75068]: cluster 1685822712.656777 mon.a (mon.0) 768 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-06-03T20:05:14.029 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:05:13 smithi089 bash[75068]: cluster 1685822712.662829 osd.6 (osd.6) 67 : cluster 3 failed to encode map e148 with expected crc 2023-06-03T20:05:14.029 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:05:13 smithi089 bash[75068]: audit 1685822712.6703753 mon.a (mon.0) 769 : audit 1 from='client.? 172.21.15.89:0/4285693182' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pg_num", "val": "26"}]': finished 2023-06-03T20:05:14.029 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:05:13 smithi089 bash[75068]: audit 1685822712.6704817 mon.a (mon.0) 770 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-06-03T20:05:14.030 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:05:13 smithi089 bash[75068]: cluster 1685822712.6705842 mon.a (mon.0) 771 : cluster 0 osdmap e148: 8 total, 8 up, 8 in 2023-06-03T20:05:14.030 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:05:13 smithi089 bash[78452]: cluster 1685822712.5249856 mgr.y (mgr.34104) 871 : cluster 0 pgmap v793: 89 pgs: 89 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2023-06-03T20:05:14.030 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:05:13 smithi089 bash[78452]: cluster 1685822712.656777 mon.a (mon.0) 768 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-06-03T20:05:14.030 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:05:13 smithi089 bash[78452]: cluster 1685822712.662829 osd.6 (osd.6) 67 : cluster 3 failed to encode map e148 with expected crc 2023-06-03T20:05:14.030 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:05:13 smithi089 bash[78452]: audit 1685822712.6703753 mon.a (mon.0) 769 : audit 1 from='client.? 172.21.15.89:0/4285693182' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pg_num", "val": "26"}]': finished 2023-06-03T20:05:14.030 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:05:13 smithi089 bash[78452]: audit 1685822712.6704817 mon.a (mon.0) 770 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-06-03T20:05:14.031 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:05:13 smithi089 bash[78452]: cluster 1685822712.6705842 mon.a (mon.0) 771 : cluster 0 osdmap e148: 8 total, 8 up, 8 in 2023-06-03T20:05:14.048 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:05:14.050 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:05:14.051 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:05:14.078 INFO:tasks.cephadm.ceph_manager.ceph:removing pool_name unique_pool_4 2023-06-03T20:05:14.079 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph osd pool rm unique_pool_4 unique_pool_4 --yes-i-really-really-mean-it 2023-06-03T20:05:14.153 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:05:14.188 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:05:14.191 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:05:14.192 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:05:14.293 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:05:14.329 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:05:14.332 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:05:14.332 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:05:14.375 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-06-03T20:05:14.433 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:05:14.469 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:05:14.472 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:05:14.473 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:05:14.573 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:05:14.605 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:05:14.608 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:05:14.608 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:05:14.710 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:05:14.745 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:05:14.752 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:05:14.752 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:05:14.853 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:05:14.931 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:05:14.939 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:05:14.940 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:05:14 smithi089 bash[75068]: cluster 1685822712.8378224 osd.7 (osd.7) 60 : cluster 3 failed to encode map e148 with expected crc 2023-06-03T20:05:14.940 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:05:14 smithi089 bash[75068]: cluster 1685822712.840638 osd.7 (osd.7) 61 : cluster 3 failed to encode map e148 with expected crc 2023-06-03T20:05:14.940 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:05:14 smithi089 bash[75068]: cluster 1685822713.7115002 osd.5 (osd.5) 72 : cluster 3 failed to encode map e148 with expected crc 2023-06-03T20:05:14.941 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:05:14 smithi089 bash[75068]: audit 1685822714.526507 mon.c (mon.2) 424 : audit 1 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pg_num_actual", "val": "26"}]: dispatch 2023-06-03T20:05:14.941 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:05:14 smithi089 bash[75068]: audit 1685822714.5282586 mon.a (mon.0) 772 : audit 1 from='mgr.34104 ' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pg_num_actual", "val": "26"}]: dispatch 2023-06-03T20:05:14.941 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:05:14 smithi089 bash[78452]: cluster 1685822712.8378224 osd.7 (osd.7) 60 : cluster 3 failed to encode map e148 with expected crc 2023-06-03T20:05:14.942 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:05:14 smithi089 bash[78452]: cluster 1685822712.840638 osd.7 (osd.7) 61 : cluster 3 failed to encode map e148 with expected crc 2023-06-03T20:05:14.942 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:05:14 smithi089 bash[78452]: cluster 1685822713.7115002 osd.5 (osd.5) 72 : cluster 3 failed to encode map e148 with expected crc 2023-06-03T20:05:14.942 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:05:14 smithi089 bash[78452]: audit 1685822714.526507 mon.c (mon.2) 424 : audit 1 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pg_num_actual", "val": "26"}]: dispatch 2023-06-03T20:05:14.943 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:05:14 smithi089 bash[78452]: audit 1685822714.5282586 mon.a (mon.0) 772 : audit 1 from='mgr.34104 ' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pg_num_actual", "val": "26"}]: dispatch 2023-06-03T20:05:14.943 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:05:15.045 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:05:15.062 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:05:14 smithi164 bash[59462]: cluster 1685822712.8378224 osd.7 (osd.7) 60 : cluster 3 failed to encode map e148 with expected crc 2023-06-03T20:05:15.062 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:05:14 smithi164 bash[59462]: cluster 1685822712.840638 osd.7 (osd.7) 61 : cluster 3 failed to encode map e148 with expected crc 2023-06-03T20:05:15.063 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:05:14 smithi164 bash[59462]: cluster 1685822713.7115002 osd.5 (osd.5) 72 : cluster 3 failed to encode map e148 with expected crc 2023-06-03T20:05:15.063 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:05:14 smithi164 bash[59462]: audit 1685822714.526507 mon.c (mon.2) 424 : audit 1 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pg_num_actual", "val": "26"}]: dispatch 2023-06-03T20:05:15.063 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:05:14 smithi164 bash[59462]: audit 1685822714.5282586 mon.a (mon.0) 772 : audit 1 from='mgr.34104 ' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pg_num_actual", "val": "26"}]: dispatch 2023-06-03T20:05:15.093 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:05:15.100 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:05:15.101 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:05:15.202 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:05:15.233 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:05:15.235 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:05:15.235 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:05:15.336 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:05:15.371 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:05:15.373 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:05:15.373 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:05:15.474 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:05:15.513 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:05:15.516 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:05:15.516 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:05:15.617 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:05:15.653 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:05:15.656 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:05:15.656 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:05:15.757 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:05:15.786 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:05:15.791 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:05:15.792 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:05:15.893 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:05:15.927 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:05:15.929 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:05:15.930 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:05:16.028 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:05:15 smithi089 bash[78452]: cluster 1685822714.5259273 mgr.y (mgr.34104) 872 : cluster 0 pgmap v795: 89 pgs: 89 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2023-06-03T20:05:16.029 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:05:15 smithi089 bash[78452]: cluster 1685822714.6727219 osd.7 (osd.7) 62 : cluster 3 failed to encode map e149 with expected crc 2023-06-03T20:05:16.029 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:05:15 smithi089 bash[78452]: audit 1685822714.6789289 mon.a (mon.0) 773 : audit 1 from='mgr.34104 ' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pg_num_actual", "val": "26"}]': finished 2023-06-03T20:05:16.029 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:05:15 smithi089 bash[78452]: cluster 1685822714.6790695 mon.a (mon.0) 774 : cluster 0 osdmap e149: 8 total, 8 up, 8 in 2023-06-03T20:05:16.029 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:05:15 smithi089 bash[78452]: cluster 1685822714.8634064 osd.6 (osd.6) 68 : cluster 3 failed to encode map e149 with expected crc 2023-06-03T20:05:16.029 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:05:15 smithi089 bash[75068]: cluster 1685822714.5259273 mgr.y (mgr.34104) 872 : cluster 0 pgmap v795: 89 pgs: 89 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2023-06-03T20:05:16.030 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:05:15 smithi089 bash[75068]: cluster 1685822714.6727219 osd.7 (osd.7) 62 : cluster 3 failed to encode map e149 with expected crc 2023-06-03T20:05:16.030 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:05:15 smithi089 bash[75068]: audit 1685822714.6789289 mon.a (mon.0) 773 : audit 1 from='mgr.34104 ' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pg_num_actual", "val": "26"}]': finished 2023-06-03T20:05:16.030 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:05:15 smithi089 bash[75068]: cluster 1685822714.6790695 mon.a (mon.0) 774 : cluster 0 osdmap e149: 8 total, 8 up, 8 in 2023-06-03T20:05:16.030 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:05:15 smithi089 bash[75068]: cluster 1685822714.8634064 osd.6 (osd.6) 68 : cluster 3 failed to encode map e149 with expected crc 2023-06-03T20:05:16.030 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:05:16.042 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:05:15 smithi164 bash[59462]: cluster 1685822714.5259273 mgr.y (mgr.34104) 872 : cluster 0 pgmap v795: 89 pgs: 89 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2023-06-03T20:05:16.043 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:05:15 smithi164 bash[59462]: cluster 1685822714.6727219 osd.7 (osd.7) 62 : cluster 3 failed to encode map e149 with expected crc 2023-06-03T20:05:16.043 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:05:15 smithi164 bash[59462]: audit 1685822714.6789289 mon.a (mon.0) 773 : audit 1 from='mgr.34104 ' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pg_num_actual", "val": "26"}]': finished 2023-06-03T20:05:16.043 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:05:15 smithi164 bash[59462]: cluster 1685822714.6790695 mon.a (mon.0) 774 : cluster 0 osdmap e149: 8 total, 8 up, 8 in 2023-06-03T20:05:16.043 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:05:15 smithi164 bash[59462]: cluster 1685822714.8634064 osd.6 (osd.6) 68 : cluster 3 failed to encode map e149 with expected crc 2023-06-03T20:05:16.061 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph osd set nodeep-scrub 2023-06-03T20:05:16.063 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:05:16.066 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:05:16.066 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:05:16.167 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:05:16.206 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:05:16.209 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:05:16.210 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:05:16.311 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:05:16.348 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:05:16.350 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:05:16.351 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:05:16.452 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:05:16.465 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 20:05:16 smithi089 bash[71163]: ::ffff:172.21.15.164 - - [03/Jun/2023:20:05:16] "GET /metrics HTTP/1.1" 200 228185 "" "Prometheus/2.33.4" 2023-06-03T20:05:16.491 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:05:16.494 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:05:16.494 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:05:16.595 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:05:16.634 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:05:16.637 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:05:16.638 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:05:16.739 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:05:16.779 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:05:16 smithi089 bash[78452]: cluster 1685822715.683856 osd.6 (osd.6) 69 : cluster 3 failed to encode map e150 with expected crc 2023-06-03T20:05:16.779 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:05:16 smithi089 bash[78452]: cluster 1685822715.6870582 osd.7 (osd.7) 63 : cluster 3 failed to encode map e150 with expected crc 2023-06-03T20:05:16.779 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:05:16 smithi089 bash[78452]: cluster 1685822715.687995 osd.5 (osd.5) 73 : cluster 3 failed to encode map e150 with expected crc 2023-06-03T20:05:16.779 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:05:16 smithi089 bash[78452]: cluster 1685822715.6888096 mon.a (mon.0) 775 : cluster 0 osdmap e150: 8 total, 8 up, 8 in 2023-06-03T20:05:16.779 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:05:16 smithi089 bash[78452]: cluster 1685822715.6889315 osd.6 (osd.6) 70 : cluster 3 failed to encode map e150 with expected crc 2023-06-03T20:05:16.779 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:05:16 smithi089 bash[78452]: cluster 1685822715.6925244 osd.5 (osd.5) 74 : cluster 3 failed to encode map e150 with expected crc 2023-06-03T20:05:16.780 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:05:16 smithi089 bash[75068]: cluster 1685822715.683856 osd.6 (osd.6) 69 : cluster 3 failed to encode map e150 with expected crc 2023-06-03T20:05:16.780 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:05:16 smithi089 bash[75068]: cluster 1685822715.6870582 osd.7 (osd.7) 63 : cluster 3 failed to encode map e150 with expected crc 2023-06-03T20:05:16.780 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:05:16 smithi089 bash[75068]: cluster 1685822715.687995 osd.5 (osd.5) 73 : cluster 3 failed to encode map e150 with expected crc 2023-06-03T20:05:16.780 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:05:16 smithi089 bash[75068]: cluster 1685822715.6888096 mon.a (mon.0) 775 : cluster 0 osdmap e150: 8 total, 8 up, 8 in 2023-06-03T20:05:16.780 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:05:16 smithi089 bash[75068]: cluster 1685822715.6889315 osd.6 (osd.6) 70 : cluster 3 failed to encode map e150 with expected crc 2023-06-03T20:05:16.780 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:05:16 smithi089 bash[75068]: cluster 1685822715.6925244 osd.5 (osd.5) 74 : cluster 3 failed to encode map e150 with expected crc 2023-06-03T20:05:16.781 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:05:16.781 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:05:16.782 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:05:16.883 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:05:16.912 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:05:16.915 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:05:16.916 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:05:17.016 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:05:17.052 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:05:17.055 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:05:17.055 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:05:17.083 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:05:16 smithi164 bash[59462]: cluster 1685822715.683856 osd.6 (osd.6) 69 : cluster 3 failed to encode map e150 with expected crc 2023-06-03T20:05:17.084 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:05:16 smithi164 bash[59462]: cluster 1685822715.6870582 osd.7 (osd.7) 63 : cluster 3 failed to encode map e150 with expected crc 2023-06-03T20:05:17.084 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:05:16 smithi164 bash[59462]: cluster 1685822715.687995 osd.5 (osd.5) 73 : cluster 3 failed to encode map e150 with expected crc 2023-06-03T20:05:17.084 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:05:16 smithi164 bash[59462]: cluster 1685822715.6888096 mon.a (mon.0) 775 : cluster 0 osdmap e150: 8 total, 8 up, 8 in 2023-06-03T20:05:17.084 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:05:16 smithi164 bash[59462]: cluster 1685822715.6889315 osd.6 (osd.6) 70 : cluster 3 failed to encode map e150 with expected crc 2023-06-03T20:05:17.084 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:05:16 smithi164 bash[59462]: cluster 1685822715.6925244 osd.5 (osd.5) 74 : cluster 3 failed to encode map e150 with expected crc 2023-06-03T20:05:17.156 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:05:17.194 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:05:17.197 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:05:17.198 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:05:17.300 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:05:17.340 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:05:17.342 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:05:17.343 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:05:17.444 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:05:17.482 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:05:17.485 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:05:17.485 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:05:17.586 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:05:17.624 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:05:17.627 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:05:17.627 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:05:17.728 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:05:17.739 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:05:17 smithi164 bash[59462]: cluster 1685822716.526623 mgr.y (mgr.34104) 873 : cluster 0 pgmap v798: 99 pgs: 2 peering, 10 unknown, 87 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2023-06-03T20:05:17.761 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:05:17.763 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:05:17.764 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:05:17.864 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:05:17.904 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:05:17.908 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:05:17.908 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:05:18.009 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:05:18.022 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:05:17 smithi089 bash[75068]: cluster 1685822716.526623 mgr.y (mgr.34104) 873 : cluster 0 pgmap v798: 99 pgs: 2 peering, 10 unknown, 87 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2023-06-03T20:05:18.022 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:05:17 smithi089 bash[78452]: cluster 1685822716.526623 mgr.y (mgr.34104) 873 : cluster 0 pgmap v798: 99 pgs: 2 peering, 10 unknown, 87 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2023-06-03T20:05:18.047 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:05:18.050 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:05:18.051 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:05:18.150 INFO:teuthology.orchestra.run.smithi164.stderr:Inferring config /var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/mon.b/config 2023-06-03T20:05:18.151 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:05:18.194 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:05:18.197 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:05:18.198 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:05:18.299 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:05:18.335 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:05:18.338 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:05:18.338 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:05:18.439 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:05:18.468 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:05:18.471 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:05:18.472 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:05:18.572 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:05:18.604 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:05:18.606 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:05:18.607 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:05:18.708 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:05:18.744 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:05:18.747 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:05:18.747 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:05:18.848 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:05:18.878 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:05:18.880 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:05:18.881 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:05:18.940 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/mon.a/config 2023-06-03T20:05:18.940 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/mon.a/config 2023-06-03T20:05:18.983 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:05:19.013 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:05:19.016 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:05:19.017 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:05:19.079 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 5, 6, 7] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-06-03T20:05:19.079 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-06-03T20:05:19.080 INFO:tasks.thrashosds.thrasher:inject_pause on osd.1 2023-06-03T20:05:19.080 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2023-06-03T20:05:19.080 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2023-06-03T20:05:19.080 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph daemon osd.1 config set filestore_inject_stall 3 2023-06-03T20:05:19.118 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:05:19.154 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:05:19.156 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:05:19.157 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:05:19.258 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:05:19.287 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:05:19.289 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:05:19.290 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:05:19.390 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:05:19.422 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:05:19.424 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:05:19.425 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:05:19.526 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:05:19.555 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:05:19.556 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:05:19.557 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:05:19.658 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:05:19.695 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:05:19.698 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:05:19.699 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:05:19.800 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:05:19.814 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:05:19 smithi089 bash[78452]: cluster 1685822718.5275888 mgr.y (mgr.34104) 874 : cluster 0 pgmap v799: 99 pgs: 2 peering, 4 unknown, 93 active+clean; 582 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail 2023-06-03T20:05:19.814 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:05:19 smithi089 bash[75068]: cluster 1685822718.5275888 mgr.y (mgr.34104) 874 : cluster 0 pgmap v799: 99 pgs: 2 peering, 4 unknown, 93 active+clean; 582 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail 2023-06-03T20:05:19.834 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:05:19.836 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:05:19.837 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:05:19.938 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:05:19.958 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/mon.a/config 2023-06-03T20:05:19.967 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:05:19.969 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:05:19.970 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:05:20.071 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:05:20.079 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:05:19 smithi164 bash[59462]: cluster 1685822718.5275888 mgr.y (mgr.34104) 874 : cluster 0 pgmap v799: 99 pgs: 2 peering, 4 unknown, 93 active+clean; 582 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail 2023-06-03T20:05:20.101 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:05:20.103 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:05:20.103 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:05:20.205 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:05:20.236 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:05:20.239 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:05:20.239 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:05:20.340 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:05:20.382 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:05:20.385 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:05:20.385 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:05:20.486 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:05:20.517 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:05:20.520 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:05:20.521 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:05:20.622 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:05:20.651 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:05:20.653 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:05:20.653 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:05:20.754 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:05:20.770 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:05:20 smithi089 bash[75068]: audit 1685822720.528728 mon.c (mon.2) 425 : audit 1 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num_actual", "val": "26"}]: dispatch 2023-06-03T20:05:20.770 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:05:20 smithi089 bash[75068]: audit 1685822720.5304036 mon.a (mon.0) 776 : audit 1 from='mgr.34104 ' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num_actual", "val": "26"}]: dispatch 2023-06-03T20:05:20.780 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:05:20.782 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:05:20.783 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:05:20.833 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:05:20 smithi164 bash[59462]: audit 1685822720.528728 mon.c (mon.2) 425 : audit 1 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num_actual", "val": "26"}]: dispatch 2023-06-03T20:05:20.834 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:05:20 smithi164 bash[59462]: audit 1685822720.5304036 mon.a (mon.0) 776 : audit 1 from='mgr.34104 ' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num_actual", "val": "26"}]: dispatch 2023-06-03T20:05:20.883 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:05:20.908 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:05:20.911 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:05:20.911 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:05:21.012 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:05:21.028 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:05:20 smithi089 bash[78452]: audit 1685822720.528728 mon.c (mon.2) 425 : audit 1 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num_actual", "val": "26"}]: dispatch 2023-06-03T20:05:21.029 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:05:20 smithi089 bash[78452]: audit 1685822720.5304036 mon.a (mon.0) 776 : audit 1 from='mgr.34104 ' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num_actual", "val": "26"}]: dispatch 2023-06-03T20:05:21.041 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:05:21.045 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:05:21.045 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:05:21.146 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:05:21.182 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:05:21.184 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:05:21.185 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:05:21.286 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:05:21.311 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:05:21.315 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:05:21.315 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:05:21.416 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:05:21.444 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:05:21.447 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:05:21.447 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:05:21.548 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:05:21.585 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:05:21.587 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:05:21.588 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:05:21.689 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:05:21.727 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:05:21.729 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:05:21.730 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:05:21.830 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:05:21.833 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:05:21 smithi164 bash[59462]: cluster 1685822720.528223 mgr.y (mgr.34104) 875 : cluster 0 pgmap v800: 99 pgs: 99 active+clean; 582 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail 2023-06-03T20:05:21.834 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:05:21 smithi164 bash[59462]: cluster 1685822720.7079735 osd.6 (osd.6) 71 : cluster 3 failed to encode map e151 with expected crc 2023-06-03T20:05:21.834 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:05:21 smithi164 bash[59462]: audit 1685822720.7144158 mon.a (mon.0) 777 : audit 1 from='mgr.34104 ' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num_actual", "val": "26"}]': finished 2023-06-03T20:05:21.834 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:05:21 smithi164 bash[59462]: cluster 1685822720.7145035 mon.a (mon.0) 778 : cluster 0 osdmap e151: 8 total, 8 up, 8 in 2023-06-03T20:05:21.834 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:05:21 smithi164 bash[59462]: cluster 1685822720.715902 osd.5 (osd.5) 75 : cluster 3 failed to encode map e151 with expected crc 2023-06-03T20:05:21.866 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:05:21.868 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:05:21.869 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:05:21.970 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:05:21.984 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:05:21 smithi089 bash[75068]: cluster 1685822720.528223 mgr.y (mgr.34104) 875 : cluster 0 pgmap v800: 99 pgs: 99 active+clean; 582 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail 2023-06-03T20:05:21.984 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:05:21 smithi089 bash[75068]: cluster 1685822720.7079735 osd.6 (osd.6) 71 : cluster 3 failed to encode map e151 with expected crc 2023-06-03T20:05:21.984 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:05:21 smithi089 bash[75068]: audit 1685822720.7144158 mon.a (mon.0) 777 : audit 1 from='mgr.34104 ' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num_actual", "val": "26"}]': finished 2023-06-03T20:05:21.984 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:05:21 smithi089 bash[75068]: cluster 1685822720.7145035 mon.a (mon.0) 778 : cluster 0 osdmap e151: 8 total, 8 up, 8 in 2023-06-03T20:05:21.985 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:05:21 smithi089 bash[75068]: cluster 1685822720.715902 osd.5 (osd.5) 75 : cluster 3 failed to encode map e151 with expected crc 2023-06-03T20:05:21.985 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:05:21 smithi089 bash[78452]: cluster 1685822720.528223 mgr.y (mgr.34104) 875 : cluster 0 pgmap v800: 99 pgs: 99 active+clean; 582 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail 2023-06-03T20:05:21.985 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:05:21 smithi089 bash[78452]: cluster 1685822720.7079735 osd.6 (osd.6) 71 : cluster 3 failed to encode map e151 with expected crc 2023-06-03T20:05:21.985 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:05:21 smithi089 bash[78452]: audit 1685822720.7144158 mon.a (mon.0) 777 : audit 1 from='mgr.34104 ' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num_actual", "val": "26"}]': finished 2023-06-03T20:05:21.985 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:05:21 smithi089 bash[78452]: cluster 1685822720.7145035 mon.a (mon.0) 778 : cluster 0 osdmap e151: 8 total, 8 up, 8 in 2023-06-03T20:05:21.985 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:05:21 smithi089 bash[78452]: cluster 1685822720.715902 osd.5 (osd.5) 75 : cluster 3 failed to encode map e151 with expected crc 2023-06-03T20:05:22.013 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:05:22.014 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:05:22.014 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:05:22.115 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:05:22.148 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:05:22.151 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:05:22.151 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:05:22.229 INFO:teuthology.orchestra.run.smithi164.stdout:ERROR: (22) Invalid argument 2023-06-03T20:05:22.229 INFO:teuthology.orchestra.run.smithi164.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-06-03T20:05:22.252 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:05:22.281 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:05:22.283 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:05:22.284 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:05:22.385 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:05:22.426 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:05:22.428 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:05:22.429 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:05:22.530 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:05:22.566 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:05:22.568 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:05:22.569 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:05:22.669 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:05:22.709 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:05:22.711 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:05:22.712 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:05:22.813 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:05:22.834 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:05:22 smithi164 bash[59462]: cluster 1685822721.7099226 osd.6 (osd.6) 72 : cluster 3 failed to encode map e152 with expected crc 2023-06-03T20:05:22.834 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:05:22 smithi164 bash[59462]: cluster 1685822721.7129178 osd.7 (osd.7) 64 : cluster 3 failed to encode map e152 with expected crc 2023-06-03T20:05:22.834 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:05:22 smithi164 bash[59462]: cluster 1685822721.7142794 mon.a (mon.0) 779 : cluster 0 osdmap e152: 8 total, 8 up, 8 in 2023-06-03T20:05:22.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:05:22 smithi164 bash[59462]: cluster 1685822721.714425 osd.6 (osd.6) 73 : cluster 3 failed to encode map e152 with expected crc 2023-06-03T20:05:22.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:05:22 smithi164 bash[59462]: cluster 1685822721.715082 osd.5 (osd.5) 76 : cluster 3 failed to encode map e152 with expected crc 2023-06-03T20:05:22.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:05:22 smithi164 bash[59462]: cluster 1685822721.7170136 osd.7 (osd.7) 65 : cluster 3 failed to encode map e152 with expected crc 2023-06-03T20:05:22.836 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:05:22 smithi164 bash[59462]: cluster 1685822721.7203932 osd.7 (osd.7) 66 : cluster 3 failed to encode map e152 with expected crc 2023-06-03T20:05:22.836 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:05:22 smithi164 bash[59462]: cluster 1685822721.7226336 osd.5 (osd.5) 77 : cluster 3 failed to encode map e152 with expected crc 2023-06-03T20:05:22.836 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:05:22 smithi164 bash[59462]: cluster 1685822721.7253227 osd.5 (osd.5) 78 : cluster 3 failed to encode map e152 with expected crc 2023-06-03T20:05:22.836 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:05:22 smithi164 bash[59462]: audit 1685822722.1119647 mon.a (mon.0) 780 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:05:22.837 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:05:22 smithi164 bash[59462]: audit 1685822722.1138067 mon.c (mon.2) 426 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T20:05:22.843 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:05:22.847 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:05:22.848 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:05:22.949 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:05:22.982 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:05:22.983 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:05:22.984 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:05:22.985 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:05:22 smithi089 bash[78452]: cluster 1685822721.7099226 osd.6 (osd.6) 72 : cluster 3 failed to encode map e152 with expected crc 2023-06-03T20:05:22.985 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:05:22 smithi089 bash[78452]: cluster 1685822721.7129178 osd.7 (osd.7) 64 : cluster 3 failed to encode map e152 with expected crc 2023-06-03T20:05:22.985 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:05:22 smithi089 bash[78452]: cluster 1685822721.7142794 mon.a (mon.0) 779 : cluster 0 osdmap e152: 8 total, 8 up, 8 in 2023-06-03T20:05:22.986 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:05:22 smithi089 bash[78452]: cluster 1685822721.714425 osd.6 (osd.6) 73 : cluster 3 failed to encode map e152 with expected crc 2023-06-03T20:05:22.986 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:05:22 smithi089 bash[78452]: cluster 1685822721.715082 osd.5 (osd.5) 76 : cluster 3 failed to encode map e152 with expected crc 2023-06-03T20:05:22.986 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:05:22 smithi089 bash[78452]: cluster 1685822721.7170136 osd.7 (osd.7) 65 : cluster 3 failed to encode map e152 with expected crc 2023-06-03T20:05:22.986 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:05:22 smithi089 bash[78452]: cluster 1685822721.7203932 osd.7 (osd.7) 66 : cluster 3 failed to encode map e152 with expected crc 2023-06-03T20:05:22.986 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:05:22 smithi089 bash[78452]: cluster 1685822721.7226336 osd.5 (osd.5) 77 : cluster 3 failed to encode map e152 with expected crc 2023-06-03T20:05:22.986 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:05:22 smithi089 bash[78452]: cluster 1685822721.7253227 osd.5 (osd.5) 78 : cluster 3 failed to encode map e152 with expected crc 2023-06-03T20:05:22.986 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:05:22 smithi089 bash[78452]: audit 1685822722.1119647 mon.a (mon.0) 780 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:05:22.987 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:05:22 smithi089 bash[78452]: audit 1685822722.1138067 mon.c (mon.2) 426 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T20:05:22.987 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:05:22 smithi089 bash[75068]: cluster 1685822721.7099226 osd.6 (osd.6) 72 : cluster 3 failed to encode map e152 with expected crc 2023-06-03T20:05:22.987 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:05:22 smithi089 bash[75068]: cluster 1685822721.7129178 osd.7 (osd.7) 64 : cluster 3 failed to encode map e152 with expected crc 2023-06-03T20:05:22.987 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:05:22 smithi089 bash[75068]: cluster 1685822721.7142794 mon.a (mon.0) 779 : cluster 0 osdmap e152: 8 total, 8 up, 8 in 2023-06-03T20:05:22.987 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:05:22 smithi089 bash[75068]: cluster 1685822721.714425 osd.6 (osd.6) 73 : cluster 3 failed to encode map e152 with expected crc 2023-06-03T20:05:22.987 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:05:22 smithi089 bash[75068]: cluster 1685822721.715082 osd.5 (osd.5) 76 : cluster 3 failed to encode map e152 with expected crc 2023-06-03T20:05:22.988 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:05:22 smithi089 bash[75068]: cluster 1685822721.7170136 osd.7 (osd.7) 65 : cluster 3 failed to encode map e152 with expected crc 2023-06-03T20:05:22.988 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:05:22 smithi089 bash[75068]: cluster 1685822721.7203932 osd.7 (osd.7) 66 : cluster 3 failed to encode map e152 with expected crc 2023-06-03T20:05:22.988 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:05:22 smithi089 bash[75068]: cluster 1685822721.7226336 osd.5 (osd.5) 77 : cluster 3 failed to encode map e152 with expected crc 2023-06-03T20:05:22.988 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:05:22 smithi089 bash[75068]: cluster 1685822721.7253227 osd.5 (osd.5) 78 : cluster 3 failed to encode map e152 with expected crc 2023-06-03T20:05:22.989 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:05:22 smithi089 bash[75068]: audit 1685822722.1119647 mon.a (mon.0) 780 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:05:22.989 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:05:22 smithi089 bash[75068]: audit 1685822722.1138067 mon.c (mon.2) 426 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T20:05:23.018 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/mon.a/config 2023-06-03T20:05:23.085 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:05:23.112 DEBUG:teuthology.orchestra.run.smithi164:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph daemon osd.6 dump_ops_in_flight 2023-06-03T20:05:23.113 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:05:23.116 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:05:23.116 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:05:23.217 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:05:23.254 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:05:23.259 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:05:23.259 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:05:23.360 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:05:23.396 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:05:23.398 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:05:23.398 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:05:23.500 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:05:23.536 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:05:23.539 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:05:23.540 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:05:23.640 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:05:23.675 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:05:23.678 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:05:23.678 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:05:23.779 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:05:23.814 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:05:23.817 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:05:23.818 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:05:23.918 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:05:23.932 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:05:23 smithi164 bash[59462]: cluster 1685822722.5287964 mgr.y (mgr.34104) 876 : cluster 0 pgmap v803: 99 pgs: 2 unknown, 97 active+clean; 582 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail 2023-06-03T20:05:23.933 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:05:23 smithi164 bash[59462]: cluster 1685822722.7194638 osd.7 (osd.7) 67 : cluster 3 failed to encode map e153 with expected crc 2023-06-03T20:05:23.933 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:05:23 smithi164 bash[59462]: cluster 1685822722.7242737 mon.a (mon.0) 781 : cluster 0 osdmap e153: 8 total, 8 up, 8 in 2023-06-03T20:05:23.933 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:05:23 smithi164 bash[59462]: cluster 1685822722.7245352 osd.7 (osd.7) 68 : cluster 3 failed to encode map e153 with expected crc 2023-06-03T20:05:23.933 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:05:23 smithi164 bash[59462]: cluster 1685822722.7254179 osd.0 (osd.0) 1 : cluster 0 9.19 starting backfill to osd.3 from (0'0,0'0] MAX to 146'13826 2023-06-03T20:05:23.933 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:05:23 smithi164 bash[59462]: cluster 1685822722.7254496 osd.0 (osd.0) 2 : cluster 0 9.18 starting backfill to osd.6 from (0'0,0'0] MAX to 146'13716 2023-06-03T20:05:23.933 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:05:23 smithi164 bash[59462]: cluster 1685822722.7257245 osd.7 (osd.7) 69 : cluster 3 failed to encode map e153 with expected crc 2023-06-03T20:05:23.934 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:05:23 smithi164 bash[59462]: cluster 1685822722.7269492 osd.1 (osd.1) 1 : cluster 0 9.10 starting backfill to osd.0 from (0'0,0'0] MAX to 146'13656 2023-06-03T20:05:23.934 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:05:23 smithi164 bash[59462]: cluster 1685822722.7274182 osd.1 (osd.1) 2 : cluster 0 9.12 starting backfill to osd.0 from (0'0,0'0] MAX to 146'13764 2023-06-03T20:05:23.934 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:05:23 smithi164 bash[59462]: cluster 1685822722.727435 osd.7 (osd.7) 70 : cluster 3 failed to encode map e153 with expected crc 2023-06-03T20:05:23.934 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:05:23 smithi164 bash[59462]: cluster 1685822722.72746 osd.1 (osd.1) 3 : cluster 0 9.16 starting backfill to osd.4 from (0'0,0'0] MAX to 146'13574 2023-06-03T20:05:23.934 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:05:23 smithi164 bash[59462]: cluster 1685822722.7296233 osd.7 (osd.7) 71 : cluster 0 9.13 starting backfill to osd.5 from (0'0,0'0] MAX to 146'13634 2023-06-03T20:05:23.934 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:05:23 smithi164 bash[59462]: cluster 1685822722.729645 osd.7 (osd.7) 72 : cluster 0 9.17 starting backfill to osd.0 from (0'0,0'0] MAX to 146'14058 2023-06-03T20:05:23.934 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:05:23 smithi164 bash[59462]: cluster 1685822722.730449 osd.7 (osd.7) 73 : cluster 0 9.15 starting backfill to osd.1 from (0'0,0'0] MAX to 146'13578 2023-06-03T20:05:23.935 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:05:23 smithi164 bash[59462]: cluster 1685822722.742503 osd.0 (osd.0) 3 : cluster 0 9.18 starting backfill to osd.7 from (0'0,0'0] MAX to 146'13716 2023-06-03T20:05:23.935 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:05:23 smithi164 bash[59462]: cluster 1685822722.7432249 osd.0 (osd.0) 4 : cluster 0 9.19 starting backfill to osd.6 from (0'0,0'0] MAX to 146'13826 2023-06-03T20:05:23.935 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:05:23 smithi164 bash[59462]: cluster 1685822722.7460334 osd.1 (osd.1) 4 : cluster 0 9.10 starting backfill to osd.4 from (0'0,0'0] MAX to 146'13656 2023-06-03T20:05:23.935 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:05:23 smithi164 bash[59462]: cluster 1685822722.7471583 osd.1 (osd.1) 5 : cluster 0 9.12 starting backfill to osd.5 from (0'0,0'0] MAX to 146'13764 2023-06-03T20:05:23.935 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:05:23 smithi164 bash[59462]: cluster 1685822722.7496583 osd.7 (osd.7) 74 : cluster 0 9.13 starting backfill to osd.6 from (0'0,0'0] MAX to 146'13634 2023-06-03T20:05:23.935 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:05:23 smithi164 bash[59462]: cluster 1685822722.7530923 osd.7 (osd.7) 75 : cluster 0 9.15 starting backfill to osd.3 from (0'0,0'0] MAX to 146'13578 2023-06-03T20:05:23.935 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:05:23 smithi164 bash[59462]: cluster 1685822722.7548308 osd.7 (osd.7) 76 : cluster 0 9.17 starting backfill to osd.1 from (0'0,0'0] MAX to 146'14058 2023-06-03T20:05:23.935 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:05:23 smithi164 bash[59462]: cluster 1685822722.756838 osd.5 (osd.5) 79 : cluster 3 failed to encode map e153 with expected crc 2023-06-03T20:05:23.936 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:05:23 smithi164 bash[59462]: cluster 1685822722.7594793 osd.5 (osd.5) 80 : cluster 3 failed to encode map e153 with expected crc 2023-06-03T20:05:23.936 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:05:23 smithi164 bash[59462]: cluster 1685822722.7671902 osd.1 (osd.1) 6 : cluster 0 9.10 starting backfill to osd.7 from (0'0,0'0] MAX to 146'13656 2023-06-03T20:05:23.936 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:05:23 smithi164 bash[59462]: cluster 1685822722.7806547 osd.6 (osd.6) 74 : cluster 3 failed to encode map e153 with expected crc 2023-06-03T20:05:23.961 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:05:23.963 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:05:23.964 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:05:24.029 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:05:23 smithi089 bash[75068]: cluster 1685822722.5287964 mgr.y (mgr.34104) 876 : cluster 0 pgmap v803: 99 pgs: 2 unknown, 97 active+clean; 582 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail 2023-06-03T20:05:24.030 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:05:23 smithi089 bash[75068]: cluster 1685822722.7194638 osd.7 (osd.7) 67 : cluster 3 failed to encode map e153 with expected crc 2023-06-03T20:05:24.030 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:05:23 smithi089 bash[75068]: cluster 1685822722.7242737 mon.a (mon.0) 781 : cluster 0 osdmap e153: 8 total, 8 up, 8 in 2023-06-03T20:05:24.030 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:05:23 smithi089 bash[75068]: cluster 1685822722.7245352 osd.7 (osd.7) 68 : cluster 3 failed to encode map e153 with expected crc 2023-06-03T20:05:24.030 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:05:23 smithi089 bash[75068]: cluster 1685822722.7254179 osd.0 (osd.0) 1 : cluster 0 9.19 starting backfill to osd.3 from (0'0,0'0] MAX to 146'13826 2023-06-03T20:05:24.031 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:05:23 smithi089 bash[75068]: cluster 1685822722.7254496 osd.0 (osd.0) 2 : cluster 0 9.18 starting backfill to osd.6 from (0'0,0'0] MAX to 146'13716 2023-06-03T20:05:24.031 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:05:23 smithi089 bash[75068]: cluster 1685822722.7257245 osd.7 (osd.7) 69 : cluster 3 failed to encode map e153 with expected crc 2023-06-03T20:05:24.031 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:05:23 smithi089 bash[75068]: cluster 1685822722.7269492 osd.1 (osd.1) 1 : cluster 0 9.10 starting backfill to osd.0 from (0'0,0'0] MAX to 146'13656 2023-06-03T20:05:24.032 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:05:23 smithi089 bash[75068]: cluster 1685822722.7274182 osd.1 (osd.1) 2 : cluster 0 9.12 starting backfill to osd.0 from (0'0,0'0] MAX to 146'13764 2023-06-03T20:05:24.032 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:05:23 smithi089 bash[75068]: cluster 1685822722.727435 osd.7 (osd.7) 70 : cluster 3 failed to encode map e153 with expected crc 2023-06-03T20:05:24.032 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:05:23 smithi089 bash[75068]: cluster 1685822722.72746 osd.1 (osd.1) 3 : cluster 0 9.16 starting backfill to osd.4 from (0'0,0'0] MAX to 146'13574 2023-06-03T20:05:24.033 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:05:23 smithi089 bash[75068]: cluster 1685822722.7296233 osd.7 (osd.7) 71 : cluster 0 9.13 starting backfill to osd.5 from (0'0,0'0] MAX to 146'13634 2023-06-03T20:05:24.033 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:05:23 smithi089 bash[75068]: cluster 1685822722.729645 osd.7 (osd.7) 72 : cluster 0 9.17 starting backfill to osd.0 from (0'0,0'0] MAX to 146'14058 2023-06-03T20:05:24.033 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:05:23 smithi089 bash[75068]: cluster 1685822722.730449 osd.7 (osd.7) 73 : cluster 0 9.15 starting backfill to osd.1 from (0'0,0'0] MAX to 146'13578 2023-06-03T20:05:24.034 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:05:23 smithi089 bash[75068]: cluster 1685822722.742503 osd.0 (osd.0) 3 : cluster 0 9.18 starting backfill to osd.7 from (0'0,0'0] MAX to 146'13716 2023-06-03T20:05:24.034 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:05:23 smithi089 bash[75068]: cluster 1685822722.7432249 osd.0 (osd.0) 4 : cluster 0 9.19 starting backfill to osd.6 from (0'0,0'0] MAX to 146'13826 2023-06-03T20:05:24.034 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:05:23 smithi089 bash[75068]: cluster 1685822722.7460334 osd.1 (osd.1) 4 : cluster 0 9.10 starting backfill to osd.4 from (0'0,0'0] MAX to 146'13656 2023-06-03T20:05:24.035 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:05:23 smithi089 bash[75068]: cluster 1685822722.7471583 osd.1 (osd.1) 5 : cluster 0 9.12 starting backfill to osd.5 from (0'0,0'0] MAX to 146'13764 2023-06-03T20:05:24.035 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:05:23 smithi089 bash[75068]: cluster 1685822722.7496583 osd.7 (osd.7) 74 : cluster 0 9.13 starting backfill to osd.6 from (0'0,0'0] MAX to 146'13634 2023-06-03T20:05:24.035 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:05:23 smithi089 bash[75068]: cluster 1685822722.7530923 osd.7 (osd.7) 75 : cluster 0 9.15 starting backfill to osd.3 from (0'0,0'0] MAX to 146'13578 2023-06-03T20:05:24.035 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:05:23 smithi089 bash[75068]: cluster 1685822722.7548308 osd.7 (osd.7) 76 : cluster 0 9.17 starting backfill to osd.1 from (0'0,0'0] MAX to 146'14058 2023-06-03T20:05:24.036 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:05:23 smithi089 bash[75068]: cluster 1685822722.756838 osd.5 (osd.5) 79 : cluster 3 failed to encode map e153 with expected crc 2023-06-03T20:05:24.036 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:05:23 smithi089 bash[75068]: cluster 1685822722.7594793 osd.5 (osd.5) 80 : cluster 3 failed to encode map e153 with expected crc 2023-06-03T20:05:24.036 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:05:23 smithi089 bash[75068]: cluster 1685822722.7671902 osd.1 (osd.1) 6 : cluster 0 9.10 starting backfill to osd.7 from (0'0,0'0] MAX to 146'13656 2023-06-03T20:05:24.036 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:05:23 smithi089 bash[75068]: cluster 1685822722.7806547 osd.6 (osd.6) 74 : cluster 3 failed to encode map e153 with expected crc 2023-06-03T20:05:24.037 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:05:23 smithi089 bash[78452]: cluster 1685822722.5287964 mgr.y (mgr.34104) 876 : cluster 0 pgmap v803: 99 pgs: 2 unknown, 97 active+clean; 582 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail 2023-06-03T20:05:24.037 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:05:23 smithi089 bash[78452]: cluster 1685822722.7194638 osd.7 (osd.7) 67 : cluster 3 failed to encode map e153 with expected crc 2023-06-03T20:05:24.037 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:05:23 smithi089 bash[78452]: cluster 1685822722.7242737 mon.a (mon.0) 781 : cluster 0 osdmap e153: 8 total, 8 up, 8 in 2023-06-03T20:05:24.038 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:05:23 smithi089 bash[78452]: cluster 1685822722.7245352 osd.7 (osd.7) 68 : cluster 3 failed to encode map e153 with expected crc 2023-06-03T20:05:24.038 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:05:23 smithi089 bash[78452]: cluster 1685822722.7254179 osd.0 (osd.0) 1 : cluster 0 9.19 starting backfill to osd.3 from (0'0,0'0] MAX to 146'13826 2023-06-03T20:05:24.038 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:05:23 smithi089 bash[78452]: cluster 1685822722.7254496 osd.0 (osd.0) 2 : cluster 0 9.18 starting backfill to osd.6 from (0'0,0'0] MAX to 146'13716 2023-06-03T20:05:24.039 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:05:23 smithi089 bash[78452]: cluster 1685822722.7257245 osd.7 (osd.7) 69 : cluster 3 failed to encode map e153 with expected crc 2023-06-03T20:05:24.039 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:05:23 smithi089 bash[78452]: cluster 1685822722.7269492 osd.1 (osd.1) 1 : cluster 0 9.10 starting backfill to osd.0 from (0'0,0'0] MAX to 146'13656 2023-06-03T20:05:24.039 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:05:23 smithi089 bash[78452]: cluster 1685822722.7274182 osd.1 (osd.1) 2 : cluster 0 9.12 starting backfill to osd.0 from (0'0,0'0] MAX to 146'13764 2023-06-03T20:05:24.039 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:05:23 smithi089 bash[78452]: cluster 1685822722.727435 osd.7 (osd.7) 70 : cluster 3 failed to encode map e153 with expected crc 2023-06-03T20:05:24.039 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:05:23 smithi089 bash[78452]: cluster 1685822722.72746 osd.1 (osd.1) 3 : cluster 0 9.16 starting backfill to osd.4 from (0'0,0'0] MAX to 146'13574 2023-06-03T20:05:24.039 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:05:23 smithi089 bash[78452]: cluster 1685822722.7296233 osd.7 (osd.7) 71 : cluster 0 9.13 starting backfill to osd.5 from (0'0,0'0] MAX to 146'13634 2023-06-03T20:05:24.040 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:05:23 smithi089 bash[78452]: cluster 1685822722.729645 osd.7 (osd.7) 72 : cluster 0 9.17 starting backfill to osd.0 from (0'0,0'0] MAX to 146'14058 2023-06-03T20:05:24.040 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:05:23 smithi089 bash[78452]: cluster 1685822722.730449 osd.7 (osd.7) 73 : cluster 0 9.15 starting backfill to osd.1 from (0'0,0'0] MAX to 146'13578 2023-06-03T20:05:24.040 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:05:23 smithi089 bash[78452]: cluster 1685822722.742503 osd.0 (osd.0) 3 : cluster 0 9.18 starting backfill to osd.7 from (0'0,0'0] MAX to 146'13716 2023-06-03T20:05:24.040 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:05:23 smithi089 bash[78452]: cluster 1685822722.7432249 osd.0 (osd.0) 4 : cluster 0 9.19 starting backfill to osd.6 from (0'0,0'0] MAX to 146'13826 2023-06-03T20:05:24.040 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:05:23 smithi089 bash[78452]: cluster 1685822722.7460334 osd.1 (osd.1) 4 : cluster 0 9.10 starting backfill to osd.4 from (0'0,0'0] MAX to 146'13656 2023-06-03T20:05:24.040 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:05:23 smithi089 bash[78452]: cluster 1685822722.7471583 osd.1 (osd.1) 5 : cluster 0 9.12 starting backfill to osd.5 from (0'0,0'0] MAX to 146'13764 2023-06-03T20:05:24.040 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:05:23 smithi089 bash[78452]: cluster 1685822722.7496583 osd.7 (osd.7) 74 : cluster 0 9.13 starting backfill to osd.6 from (0'0,0'0] MAX to 146'13634 2023-06-03T20:05:24.041 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:05:23 smithi089 bash[78452]: cluster 1685822722.7530923 osd.7 (osd.7) 75 : cluster 0 9.15 starting backfill to osd.3 from (0'0,0'0] MAX to 146'13578 2023-06-03T20:05:24.041 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:05:23 smithi089 bash[78452]: cluster 1685822722.7548308 osd.7 (osd.7) 76 : cluster 0 9.17 starting backfill to osd.1 from (0'0,0'0] MAX to 146'14058 2023-06-03T20:05:24.041 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:05:23 smithi089 bash[78452]: cluster 1685822722.756838 osd.5 (osd.5) 79 : cluster 3 failed to encode map e153 with expected crc 2023-06-03T20:05:24.041 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:05:23 smithi089 bash[78452]: cluster 1685822722.7594793 osd.5 (osd.5) 80 : cluster 3 failed to encode map e153 with expected crc 2023-06-03T20:05:24.041 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:05:23 smithi089 bash[78452]: cluster 1685822722.7671902 osd.1 (osd.1) 6 : cluster 0 9.10 starting backfill to osd.7 from (0'0,0'0] MAX to 146'13656 2023-06-03T20:05:24.041 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:05:23 smithi089 bash[78452]: cluster 1685822722.7806547 osd.6 (osd.6) 74 : cluster 3 failed to encode map e153 with expected crc 2023-06-03T20:05:24.065 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:05:24.100 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:05:24.103 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:05:24.103 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:05:24.204 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:05:24.242 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:05:24.244 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:05:24.244 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:05:24.345 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:05:24.377 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:05:24.380 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:05:24.381 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:05:24.481 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:05:24.508 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:05:24.510 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:05:24.511 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:05:24.612 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:05:24.643 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:05:24.645 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:05:24.646 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:05:24.747 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:05:24.754 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:05:24 smithi164 bash[59462]: cluster 1685822723.737264 osd.6 (osd.6) 75 : cluster 3 failed to encode map e154 with expected crc 2023-06-03T20:05:24.755 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:05:24 smithi164 bash[59462]: cluster 1685822723.7399125 osd.5 (osd.5) 81 : cluster 3 failed to encode map e154 with expected crc 2023-06-03T20:05:24.755 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:05:24 smithi164 bash[59462]: cluster 1685822723.7416131 mon.a (mon.0) 782 : cluster 0 osdmap e154: 8 total, 8 up, 8 in 2023-06-03T20:05:24.755 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:05:24 smithi164 bash[59462]: cluster 1685822723.7418096 osd.7 (osd.7) 77 : cluster 3 failed to encode map e154 with expected crc 2023-06-03T20:05:24.755 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:05:24 smithi164 bash[59462]: cluster 1685822723.7430432 osd.5 (osd.5) 82 : cluster 3 failed to encode map e154 with expected crc 2023-06-03T20:05:24.755 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:05:24 smithi164 bash[59462]: cluster 1685822723.747435 osd.7 (osd.7) 78 : cluster 3 failed to encode map e154 with expected crc 2023-06-03T20:05:24.757 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:05:24 smithi089 bash[78452]: cluster 1685822723.737264 osd.6 (osd.6) 75 : cluster 3 failed to encode map e154 with expected crc 2023-06-03T20:05:24.758 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:05:24 smithi089 bash[78452]: cluster 1685822723.7399125 osd.5 (osd.5) 81 : cluster 3 failed to encode map e154 with expected crc 2023-06-03T20:05:24.758 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:05:24 smithi089 bash[78452]: cluster 1685822723.7416131 mon.a (mon.0) 782 : cluster 0 osdmap e154: 8 total, 8 up, 8 in 2023-06-03T20:05:24.758 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:05:24 smithi089 bash[78452]: cluster 1685822723.7418096 osd.7 (osd.7) 77 : cluster 3 failed to encode map e154 with expected crc 2023-06-03T20:05:24.758 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:05:24 smithi089 bash[78452]: cluster 1685822723.7430432 osd.5 (osd.5) 82 : cluster 3 failed to encode map e154 with expected crc 2023-06-03T20:05:24.758 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:05:24 smithi089 bash[78452]: cluster 1685822723.747435 osd.7 (osd.7) 78 : cluster 3 failed to encode map e154 with expected crc 2023-06-03T20:05:24.786 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:05:24.790 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:05:24.791 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:05:24.892 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:05:24.934 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:05:24.937 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:05:24.938 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:05:25.028 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:05:24 smithi089 bash[75068]: cluster 1685822723.737264 osd.6 (osd.6) 75 : cluster 3 failed to encode map e154 with expected crc 2023-06-03T20:05:25.028 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:05:24 smithi089 bash[75068]: cluster 1685822723.7399125 osd.5 (osd.5) 81 : cluster 3 failed to encode map e154 with expected crc 2023-06-03T20:05:25.029 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:05:24 smithi089 bash[75068]: cluster 1685822723.7416131 mon.a (mon.0) 782 : cluster 0 osdmap e154: 8 total, 8 up, 8 in 2023-06-03T20:05:25.029 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:05:24 smithi089 bash[75068]: cluster 1685822723.7418096 osd.7 (osd.7) 77 : cluster 3 failed to encode map e154 with expected crc 2023-06-03T20:05:25.029 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:05:24 smithi089 bash[75068]: cluster 1685822723.7430432 osd.5 (osd.5) 82 : cluster 3 failed to encode map e154 with expected crc 2023-06-03T20:05:25.029 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:05:24 smithi089 bash[75068]: cluster 1685822723.747435 osd.7 (osd.7) 78 : cluster 3 failed to encode map e154 with expected crc 2023-06-03T20:05:25.038 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:05:25.068 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:05:25.070 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:05:25.071 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:05:25.172 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:05:25.203 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:05:25.206 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:05:25.206 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:05:25.307 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:05:25.346 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:05:25.350 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:05:25.351 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:05:25.452 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:05:25.481 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:05:25.484 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:05:25.485 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:05:25.586 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:05:25.618 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:05:25.620 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:05:25.620 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:05:25.721 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:05:25.758 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:05:25.761 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:05:25.761 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:05:25.862 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:05:25.867 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:05:25 smithi089 bash[78452]: cluster 1685822724.5294368 mgr.y (mgr.34104) 877 : cluster 0 pgmap v806: 99 pgs: 3 active+clean+remapped, 2 unknown, 94 active+clean; 582 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail 2023-06-03T20:05:25.868 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:05:25 smithi089 bash[78452]: cluster 1685822724.7390645 osd.6 (osd.6) 76 : cluster 3 failed to encode map e155 with expected crc 2023-06-03T20:05:25.868 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:05:25 smithi089 bash[78452]: cluster 1685822724.7422292 osd.6 (osd.6) 77 : cluster 3 failed to encode map e155 with expected crc 2023-06-03T20:05:25.868 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:05:25 smithi089 bash[78452]: cluster 1685822724.7425978 osd.5 (osd.5) 83 : cluster 3 failed to encode map e155 with expected crc 2023-06-03T20:05:25.868 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:05:25 smithi089 bash[78452]: cluster 1685822724.7432501 mon.a (mon.0) 783 : cluster 0 osdmap e155: 8 total, 8 up, 8 in 2023-06-03T20:05:25.868 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:05:25 smithi089 bash[78452]: cluster 1685822724.7434406 osd.7 (osd.7) 79 : cluster 3 failed to encode map e155 with expected crc 2023-06-03T20:05:25.869 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:05:25 smithi089 bash[78452]: cluster 1685822724.7471747 osd.6 (osd.6) 78 : cluster 3 failed to encode map e155 with expected crc 2023-06-03T20:05:25.869 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:05:25 smithi089 bash[78452]: cluster 1685822724.7474039 osd.5 (osd.5) 84 : cluster 3 failed to encode map e155 with expected crc 2023-06-03T20:05:25.869 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:05:25 smithi089 bash[78452]: cluster 1685822724.7490587 osd.7 (osd.7) 80 : cluster 3 failed to encode map e155 with expected crc 2023-06-03T20:05:25.869 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:05:25 smithi089 bash[78452]: cluster 1685822724.749927 osd.5 (osd.5) 85 : cluster 3 failed to encode map e155 with expected crc 2023-06-03T20:05:25.869 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:05:25 smithi089 bash[78452]: cluster 1685822724.75114 osd.7 (osd.7) 81 : cluster 3 failed to encode map e155 with expected crc 2023-06-03T20:05:25.869 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:05:25 smithi089 bash[78452]: cluster 1685822724.7532694 osd.7 (osd.7) 82 : cluster 3 failed to encode map e155 with expected crc 2023-06-03T20:05:25.870 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:05:25 smithi089 bash[75068]: cluster 1685822724.5294368 mgr.y (mgr.34104) 877 : cluster 0 pgmap v806: 99 pgs: 3 active+clean+remapped, 2 unknown, 94 active+clean; 582 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail 2023-06-03T20:05:25.870 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:05:25 smithi089 bash[75068]: cluster 1685822724.7390645 osd.6 (osd.6) 76 : cluster 3 failed to encode map e155 with expected crc 2023-06-03T20:05:25.870 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:05:25 smithi089 bash[75068]: cluster 1685822724.7422292 osd.6 (osd.6) 77 : cluster 3 failed to encode map e155 with expected crc 2023-06-03T20:05:25.870 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:05:25 smithi089 bash[75068]: cluster 1685822724.7425978 osd.5 (osd.5) 83 : cluster 3 failed to encode map e155 with expected crc 2023-06-03T20:05:25.870 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:05:25 smithi089 bash[75068]: cluster 1685822724.7432501 mon.a (mon.0) 783 : cluster 0 osdmap e155: 8 total, 8 up, 8 in 2023-06-03T20:05:25.870 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:05:25 smithi089 bash[75068]: cluster 1685822724.7434406 osd.7 (osd.7) 79 : cluster 3 failed to encode map e155 with expected crc 2023-06-03T20:05:25.871 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:05:25 smithi089 bash[75068]: cluster 1685822724.7471747 osd.6 (osd.6) 78 : cluster 3 failed to encode map e155 with expected crc 2023-06-03T20:05:25.871 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:05:25 smithi089 bash[75068]: cluster 1685822724.7474039 osd.5 (osd.5) 84 : cluster 3 failed to encode map e155 with expected crc 2023-06-03T20:05:25.871 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:05:25 smithi089 bash[75068]: cluster 1685822724.7490587 osd.7 (osd.7) 80 : cluster 3 failed to encode map e155 with expected crc 2023-06-03T20:05:25.871 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:05:25 smithi089 bash[75068]: cluster 1685822724.749927 osd.5 (osd.5) 85 : cluster 3 failed to encode map e155 with expected crc 2023-06-03T20:05:25.871 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:05:25 smithi089 bash[75068]: cluster 1685822724.75114 osd.7 (osd.7) 81 : cluster 3 failed to encode map e155 with expected crc 2023-06-03T20:05:25.871 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:05:25 smithi089 bash[75068]: cluster 1685822724.7532694 osd.7 (osd.7) 82 : cluster 3 failed to encode map e155 with expected crc 2023-06-03T20:05:25.893 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:05:25.896 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:05:25.897 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:05:25.998 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:05:26.029 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:05:26.031 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:05:26.032 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:05:26.084 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:05:25 smithi164 bash[59462]: cluster 1685822724.5294368 mgr.y (mgr.34104) 877 : cluster 0 pgmap v806: 99 pgs: 3 active+clean+remapped, 2 unknown, 94 active+clean; 582 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail 2023-06-03T20:05:26.084 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:05:25 smithi164 bash[59462]: cluster 1685822724.7390645 osd.6 (osd.6) 76 : cluster 3 failed to encode map e155 with expected crc 2023-06-03T20:05:26.085 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:05:25 smithi164 bash[59462]: cluster 1685822724.7422292 osd.6 (osd.6) 77 : cluster 3 failed to encode map e155 with expected crc 2023-06-03T20:05:26.085 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:05:25 smithi164 bash[59462]: cluster 1685822724.7425978 osd.5 (osd.5) 83 : cluster 3 failed to encode map e155 with expected crc 2023-06-03T20:05:26.085 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:05:25 smithi164 bash[59462]: cluster 1685822724.7432501 mon.a (mon.0) 783 : cluster 0 osdmap e155: 8 total, 8 up, 8 in 2023-06-03T20:05:26.085 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:05:25 smithi164 bash[59462]: cluster 1685822724.7434406 osd.7 (osd.7) 79 : cluster 3 failed to encode map e155 with expected crc 2023-06-03T20:05:26.086 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:05:25 smithi164 bash[59462]: cluster 1685822724.7471747 osd.6 (osd.6) 78 : cluster 3 failed to encode map e155 with expected crc 2023-06-03T20:05:26.086 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:05:25 smithi164 bash[59462]: cluster 1685822724.7474039 osd.5 (osd.5) 84 : cluster 3 failed to encode map e155 with expected crc 2023-06-03T20:05:26.086 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:05:25 smithi164 bash[59462]: cluster 1685822724.7490587 osd.7 (osd.7) 80 : cluster 3 failed to encode map e155 with expected crc 2023-06-03T20:05:26.086 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:05:25 smithi164 bash[59462]: cluster 1685822724.749927 osd.5 (osd.5) 85 : cluster 3 failed to encode map e155 with expected crc 2023-06-03T20:05:26.087 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:05:25 smithi164 bash[59462]: cluster 1685822724.75114 osd.7 (osd.7) 81 : cluster 3 failed to encode map e155 with expected crc 2023-06-03T20:05:26.087 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:05:25 smithi164 bash[59462]: cluster 1685822724.7532694 osd.7 (osd.7) 82 : cluster 3 failed to encode map e155 with expected crc 2023-06-03T20:05:26.133 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:05:26.163 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:05:26.166 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:05:26.167 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:05:26.268 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:05:26.309 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:05:26.312 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:05:26.312 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:05:26.413 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:05:26.456 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:05:26.459 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:05:26.460 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:05:26.528 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 20:05:26 smithi089 bash[71163]: ::ffff:172.21.15.164 - - [03/Jun/2023:20:05:26] "GET /metrics HTTP/1.1" 200 228209 "" "Prometheus/2.33.4" 2023-06-03T20:05:26.561 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:05:26.596 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:05:26.599 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:05:26.600 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:05:26.700 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:05:26.730 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:05:26.732 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:05:26.732 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:05:26.833 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:05:26.871 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:05:26.873 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:05:26.874 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:05:26.975 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:05:27.005 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:05:27.006 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:05:27.007 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:05:27.108 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:05:27.149 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:05:27.152 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:05:27.152 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:05:27.254 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:05:27.294 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:05:27.296 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:05:27.297 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:05:27.398 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:05:27.431 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:05:27.434 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:05:27.435 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:05:27.536 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:05:27.572 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:05:27.576 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:05:27.577 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:05:27.678 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:05:27.715 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:05:27.718 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:05:27.719 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:05:27.820 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:05:27.857 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:05:27.859 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:05:27.860 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:05:27.916 INFO:teuthology.orchestra.run.smithi164.stderr:Inferring config /var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/mon.b/config 2023-06-03T20:05:27.950 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:05:27 smithi164 bash[59462]: cluster 1685822726.5302417 mgr.y (mgr.34104) 878 : cluster 0 pgmap v808: 99 pgs: 3 active+clean+remapped, 2 unknown, 94 active+clean; 582 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail 2023-06-03T20:05:27.950 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:05:27 smithi164 bash[59462]: audit 1685822727.6282487 mon.a (mon.0) 784 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:05:27.961 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:05:27.972 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:05:27 smithi089 bash[75068]: cluster 1685822726.5302417 mgr.y (mgr.34104) 878 : cluster 0 pgmap v808: 99 pgs: 3 active+clean+remapped, 2 unknown, 94 active+clean; 582 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail 2023-06-03T20:05:27.972 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:05:27 smithi089 bash[75068]: audit 1685822727.6282487 mon.a (mon.0) 784 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:05:27.972 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:05:27 smithi089 bash[78452]: cluster 1685822726.5302417 mgr.y (mgr.34104) 878 : cluster 0 pgmap v808: 99 pgs: 3 active+clean+remapped, 2 unknown, 94 active+clean; 582 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail 2023-06-03T20:05:27.972 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:05:27 smithi089 bash[78452]: audit 1685822727.6282487 mon.a (mon.0) 784 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:05:27.989 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:05:27.991 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:05:27.993 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:05:28.094 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:05:28.119 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:05:28.121 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:05:28.122 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:05:28.223 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:05:28.249 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:05:28.251 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:05:28.252 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:05:28.353 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:05:28.385 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:05:28.387 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:05:28.388 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:05:28.488 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:05:28.515 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:05:28.518 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:05:28.519 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:05:28.620 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:05:28.655 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:05:28.657 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:05:28.658 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:05:28.759 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:05:28.792 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:05:28.796 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:05:28.797 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:05:28.898 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:05:28.929 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:05:28.932 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:05:28.933 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:05:29.034 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:05:29.063 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:05:29.065 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:05:29.066 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:05:29.167 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:05:29.196 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:05:29.199 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:05:29.200 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:05:29.301 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:05:29.331 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:05:29.334 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:05:29.335 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:05:29.436 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:05:29.464 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:05:29.467 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:05:29.467 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:05:29.568 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:05:29.603 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:05:29.606 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:05:29.607 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:05:29.708 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:05:29.747 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:05:29.749 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:05:29.750 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:05:29.831 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:05:29 smithi164 bash[59462]: cluster 1685822728.5313754 mgr.y (mgr.34104) 879 : cluster 0 pgmap v809: 99 pgs: 99 active+clean; 582 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 71 MiB/s wr, 2.27k op/s 2023-06-03T20:05:29.851 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:05:29.883 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:05:29.885 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:05:29.886 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:05:29.987 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:05:30.018 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:05:30.020 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:05:30.021 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:05:30.028 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:05:29 smithi089 bash[75068]: cluster 1685822728.5313754 mgr.y (mgr.34104) 879 : cluster 0 pgmap v809: 99 pgs: 99 active+clean; 582 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 71 MiB/s wr, 2.27k op/s 2023-06-03T20:05:30.029 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:05:29 smithi089 bash[78452]: cluster 1685822728.5313754 mgr.y (mgr.34104) 879 : cluster 0 pgmap v809: 99 pgs: 99 active+clean; 582 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 71 MiB/s wr, 2.27k op/s 2023-06-03T20:05:30.122 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:05:30.149 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:05:30.151 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:05:30.152 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:05:30.253 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:05:30.285 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:05:30.287 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:05:30.288 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:05:30.389 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:05:30.415 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:05:30.419 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:05:30.419 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:05:30.520 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:05:30.557 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:05:30.560 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:05:30.561 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:05:30.662 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:05:30.704 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:05:30.706 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:05:30.707 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:05:30.808 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:05:30.835 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:05:30.838 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:05:30.838 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:05:30.939 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:05:30.969 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:05:30.971 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:05:30.973 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:05:31.074 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:05:31.113 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:05:31.115 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:05:31.116 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:05:31.217 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:05:31.249 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:05:31.253 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:05:31.254 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:05:31.355 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:05:31.386 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:05:31.388 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:05:31.388 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:05:31.489 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:05:31.520 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:05:31.523 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:05:31.523 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:05:31.624 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:05:31.655 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:05:31.657 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:05:31.658 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:05:31.759 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:05:31.792 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:05:31.795 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:05:31.796 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:05:31.897 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:05:31.910 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:05:31 smithi164 bash[59462]: cluster 1685822730.5321572 mgr.y (mgr.34104) 880 : cluster 0 pgmap v810: 99 pgs: 99 active+clean; 582 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 54 MiB/s wr, 1.74k op/s 2023-06-03T20:05:31.930 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:05:31.933 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:05:31.933 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:05:32.028 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:05:31 smithi089 bash[75068]: cluster 1685822730.5321572 mgr.y (mgr.34104) 880 : cluster 0 pgmap v810: 99 pgs: 99 active+clean; 582 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 54 MiB/s wr, 1.74k op/s 2023-06-03T20:05:32.029 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:05:31 smithi089 bash[78452]: cluster 1685822730.5321572 mgr.y (mgr.34104) 880 : cluster 0 pgmap v810: 99 pgs: 99 active+clean; 582 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 54 MiB/s wr, 1.74k op/s 2023-06-03T20:05:32.034 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:05:32.068 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:05:32.070 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:05:32.071 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:05:32.172 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:05:32.202 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:05:32.205 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:05:32.205 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:05:32.288 INFO:teuthology.orchestra.run.smithi164.stdout:ERROR: (22) Invalid argument 2023-06-03T20:05:32.288 INFO:teuthology.orchestra.run.smithi164.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-06-03T20:05:32.306 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:05:32.343 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:05:32.345 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:05:32.346 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:05:32.447 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:05:32.479 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:05:32.481 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:05:32.481 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:05:32.582 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:05:32.617 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:05:32.620 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:05:32.620 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:05:32.721 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:05:32.753 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:05:32.756 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:05:32.757 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:05:32.858 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:05:32.890 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:05:32.892 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:05:32.893 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:05:32.994 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:05:33.023 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:05:33.025 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:05:33.026 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:05:33.127 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:05:33.152 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:05:33.155 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:05:33.155 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:05:33.209 DEBUG:teuthology.orchestra.run.smithi164:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph daemon osd.6 dump_blocked_ops 2023-06-03T20:05:33.257 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:05:33.290 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:05:33.293 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:05:33.294 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:05:33.395 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:05:33.426 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:05:33.428 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:05:33.428 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:05:33.529 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:05:33.570 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:05:33.574 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:05:33.574 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:05:33.675 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:05:33.700 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:05:33.702 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:05:33.703 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:05:33.793 INFO:teuthology.orchestra.run.smithi089.stdout:{ 2023-06-03T20:05:33.793 INFO:teuthology.orchestra.run.smithi089.stdout: "success": "filestore_inject_stall = '3' (not observed, change may require restart) " 2023-06-03T20:05:33.793 INFO:teuthology.orchestra.run.smithi089.stdout:} 2023-06-03T20:05:33.804 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:05:33.817 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:05:33 smithi164 bash[59462]: cluster 1685822732.5329094 mgr.y (mgr.34104) 881 : cluster 0 pgmap v811: 99 pgs: 99 active+clean; 582 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 48 MiB/s wr, 1.55k op/s 2023-06-03T20:05:33.839 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:05:33.842 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:05:33.842 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:05:33.849 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:05:33 smithi089 bash[75068]: cluster 1685822732.5329094 mgr.y (mgr.34104) 881 : cluster 0 pgmap v811: 99 pgs: 99 active+clean; 582 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 48 MiB/s wr, 1.55k op/s 2023-06-03T20:05:33.849 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:05:33 smithi089 bash[78452]: cluster 1685822732.5329094 mgr.y (mgr.34104) 881 : cluster 0 pgmap v811: 99 pgs: 99 active+clean; 582 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 48 MiB/s wr, 1.55k op/s 2023-06-03T20:05:33.943 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:05:33.947 INFO:teuthology.orchestra.run.smithi089.stdout:osd.0: {} 2023-06-03T20:05:33.978 INFO:teuthology.orchestra.run.smithi089.stdout:osd.1: {} 2023-06-03T20:05:33.983 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:05:33.986 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:05:33.986 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:05:33.997 INFO:teuthology.orchestra.run.smithi089.stdout:osd.2: {} 2023-06-03T20:05:34.019 INFO:teuthology.orchestra.run.smithi089.stdout:osd.3: {} 2023-06-03T20:05:34.042 INFO:teuthology.orchestra.run.smithi089.stdout:osd.4: {} 2023-06-03T20:05:34.066 INFO:teuthology.orchestra.run.smithi089.stdout:osd.5: {} 2023-06-03T20:05:34.087 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:05:34.088 INFO:teuthology.orchestra.run.smithi089.stdout:osd.6: {} 2023-06-03T20:05:34.110 INFO:teuthology.orchestra.run.smithi089.stdout:osd.7: {} 2023-06-03T20:05:34.112 INFO:teuthology.orchestra.run.smithi089.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-06-03T20:05:34.113 INFO:teuthology.orchestra.run.smithi089.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-06-03T20:05:34.113 INFO:teuthology.orchestra.run.smithi089.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-06-03T20:05:34.113 INFO:teuthology.orchestra.run.smithi089.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-06-03T20:05:34.113 INFO:teuthology.orchestra.run.smithi089.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-06-03T20:05:34.113 INFO:teuthology.orchestra.run.smithi089.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-06-03T20:05:34.113 INFO:teuthology.orchestra.run.smithi089.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-06-03T20:05:34.113 INFO:teuthology.orchestra.run.smithi089.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-06-03T20:05:34.123 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:05:34.126 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:05:34.127 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:05:34.228 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:05:34.257 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:05:34.259 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:05:34.260 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:05:34.361 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:05:34.391 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:05:34.394 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:05:34.395 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:05:34.495 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:05:34.538 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:05:34.542 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:05:34.543 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:05:34.644 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:05:34.681 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:05:34.684 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:05:34.685 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:05:34.786 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:05:34.802 INFO:teuthology.orchestra.run.smithi089.stderr:nodeep-scrub is set 2023-06-03T20:05:34.812 INFO:teuthology.orchestra.run.smithi089.stderr:pool 'unique_pool_4' removed 2023-06-03T20:05:34.820 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:05:34.827 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:05:34.829 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:05:34.930 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:05:34.941 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:05:34 smithi164 bash[59462]: audit 1685822733.9208913 mon.c (mon.2) 427 : audit 0 from='client.? 172.21.15.89:0/1061368140' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-03T20:05:34.941 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:05:34 smithi164 bash[59462]: audit 1685822733.9279065 mon.a (mon.0) 785 : audit 1 from='client.? 172.21.15.89:0/3461399123' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-06-03T20:05:34.941 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:05:34 smithi164 bash[59462]: audit 1685822734.06017 mon.a (mon.0) 786 : audit 1 from='client.? 172.21.15.89:0/3498916832' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_4", "pool2": "unique_pool_4", "yes_i_really_really_mean_it": true}]: dispatch 2023-06-03T20:05:34.969 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:05:34.970 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:05:34.971 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:05:35.072 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:05:35.080 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:05:34 smithi089 bash[75068]: audit 1685822733.9208913 mon.c (mon.2) 427 : audit 0 from='client.? 172.21.15.89:0/1061368140' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-03T20:05:35.080 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:05:34 smithi089 bash[75068]: audit 1685822733.9279065 mon.a (mon.0) 785 : audit 1 from='client.? 172.21.15.89:0/3461399123' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-06-03T20:05:35.080 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:05:34 smithi089 bash[75068]: audit 1685822734.06017 mon.a (mon.0) 786 : audit 1 from='client.? 172.21.15.89:0/3498916832' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_4", "pool2": "unique_pool_4", "yes_i_really_really_mean_it": true}]: dispatch 2023-06-03T20:05:35.081 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:05:34 smithi089 bash[78452]: audit 1685822733.9208913 mon.c (mon.2) 427 : audit 0 from='client.? 172.21.15.89:0/1061368140' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-03T20:05:35.081 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:05:34 smithi089 bash[78452]: audit 1685822733.9279065 mon.a (mon.0) 785 : audit 1 from='client.? 172.21.15.89:0/3461399123' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-06-03T20:05:35.081 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:05:34 smithi089 bash[78452]: audit 1685822734.06017 mon.a (mon.0) 786 : audit 1 from='client.? 172.21.15.89:0/3498916832' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_4", "pool2": "unique_pool_4", "yes_i_really_really_mean_it": true}]: dispatch 2023-06-03T20:05:35.103 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:05:35.105 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:05:35.106 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:05:35.207 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:05:35.245 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:05:35.249 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:05:35.250 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:05:35.351 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:05:35.393 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:05:35.397 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:05:35.397 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:05:35.498 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:05:35.530 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:05:35.533 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:05:35.534 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:05:35.635 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:05:35.675 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:05:35.680 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:05:35.681 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:05:35.782 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:05:35.829 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:05:35.834 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:05:35.835 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:05:35.936 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:05:35.959 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:05:35.961 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:05:35.962 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:05:36.062 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:05:36.070 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:05:35 smithi089 bash[78452]: cluster 1685822734.5335796 mgr.y (mgr.34104) 882 : cluster 0 pgmap v812: 99 pgs: 99 active+clean; 582 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 43 MiB/s wr, 1.36k op/s 2023-06-03T20:05:36.070 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:05:35 smithi089 bash[78452]: cluster 1685822734.7735205 mon.a (mon.0) 787 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-06-03T20:05:36.070 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:05:35 smithi089 bash[78452]: cluster 1685822734.7827191 osd.6 (osd.6) 79 : cluster 3 failed to encode map e156 with expected crc 2023-06-03T20:05:36.070 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:05:35 smithi089 bash[78452]: audit 1685822734.7889345 mon.a (mon.0) 788 : audit 1 from='client.? 172.21.15.89:0/3461399123' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-06-03T20:05:36.070 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:05:35 smithi089 bash[78452]: audit 1685822734.789065 mon.a (mon.0) 789 : audit 1 from='client.? 172.21.15.89:0/3498916832' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_4", "pool2": "unique_pool_4", "yes_i_really_really_mean_it": true}]': finished 2023-06-03T20:05:36.071 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:05:35 smithi089 bash[78452]: cluster 1685822734.7891548 mon.a (mon.0) 790 : cluster 0 osdmap e156: 8 total, 8 up, 8 in 2023-06-03T20:05:36.071 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:05:35 smithi089 bash[75068]: cluster 1685822734.5335796 mgr.y (mgr.34104) 882 : cluster 0 pgmap v812: 99 pgs: 99 active+clean; 582 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 43 MiB/s wr, 1.36k op/s 2023-06-03T20:05:36.071 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:05:35 smithi089 bash[75068]: cluster 1685822734.7735205 mon.a (mon.0) 787 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-06-03T20:05:36.071 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:05:35 smithi089 bash[75068]: cluster 1685822734.7827191 osd.6 (osd.6) 79 : cluster 3 failed to encode map e156 with expected crc 2023-06-03T20:05:36.071 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:05:35 smithi089 bash[75068]: audit 1685822734.7889345 mon.a (mon.0) 788 : audit 1 from='client.? 172.21.15.89:0/3461399123' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-06-03T20:05:36.071 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:05:35 smithi089 bash[75068]: audit 1685822734.789065 mon.a (mon.0) 789 : audit 1 from='client.? 172.21.15.89:0/3498916832' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_4", "pool2": "unique_pool_4", "yes_i_really_really_mean_it": true}]': finished 2023-06-03T20:05:36.072 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:05:35 smithi089 bash[75068]: cluster 1685822734.7891548 mon.a (mon.0) 790 : cluster 0 osdmap e156: 8 total, 8 up, 8 in 2023-06-03T20:05:36.083 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:05:35 smithi164 bash[59462]: cluster 1685822734.5335796 mgr.y (mgr.34104) 882 : cluster 0 pgmap v812: 99 pgs: 99 active+clean; 582 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 43 MiB/s wr, 1.36k op/s 2023-06-03T20:05:36.084 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:05:35 smithi164 bash[59462]: cluster 1685822734.7735205 mon.a (mon.0) 787 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-06-03T20:05:36.084 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:05:35 smithi164 bash[59462]: cluster 1685822734.7827191 osd.6 (osd.6) 79 : cluster 3 failed to encode map e156 with expected crc 2023-06-03T20:05:36.084 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:05:35 smithi164 bash[59462]: audit 1685822734.7889345 mon.a (mon.0) 788 : audit 1 from='client.? 172.21.15.89:0/3461399123' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-06-03T20:05:36.084 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:05:35 smithi164 bash[59462]: audit 1685822734.789065 mon.a (mon.0) 789 : audit 1 from='client.? 172.21.15.89:0/3498916832' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_4", "pool2": "unique_pool_4", "yes_i_really_really_mean_it": true}]': finished 2023-06-03T20:05:36.084 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:05:35 smithi164 bash[59462]: cluster 1685822734.7891548 mon.a (mon.0) 790 : cluster 0 osdmap e156: 8 total, 8 up, 8 in 2023-06-03T20:05:36.090 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:05:36.093 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:05:36.094 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:05:36.195 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:05:36.229 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:05:36.232 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:05:36.232 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:05:36.333 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:05:36.344 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 20:05:36 smithi089 bash[71163]: ::ffff:172.21.15.164 - - [03/Jun/2023:20:05:36] "GET /metrics HTTP/1.1" 200 228209 "" "Prometheus/2.33.4" 2023-06-03T20:05:36.365 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:05:36.368 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:05:36.369 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:05:36.470 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:05:36.504 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:05:36.507 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:05:36.508 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:05:36.609 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:05:36.642 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:05:36.645 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:05:36.645 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:05:36.746 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:05:36.786 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:05:36.790 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:05:36.790 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:05:36.891 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:05:36.918 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:05:36.920 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:05:36.920 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:05:37.021 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:05:37.056 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:05:37.058 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:05:37.059 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:05:37.083 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:05:36 smithi164 bash[59462]: cluster 1685822735.638692 osd.7 (osd.7) 83 : cluster 3 failed to encode map e156 with expected crc 2023-06-03T20:05:37.159 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:05:37.171 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:05:36 smithi089 bash[75068]: cluster 1685822735.638692 osd.7 (osd.7) 83 : cluster 3 failed to encode map e156 with expected crc 2023-06-03T20:05:37.171 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:05:36 smithi089 bash[78452]: cluster 1685822735.638692 osd.7 (osd.7) 83 : cluster 3 failed to encode map e156 with expected crc 2023-06-03T20:05:37.191 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:05:37.193 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:05:37.194 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:05:37.295 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:05:37.331 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:05:37.334 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:05:37.335 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:05:37.436 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:05:37.474 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:05:37.477 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:05:37.478 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:05:37.579 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:05:37.607 INFO:teuthology.task.full_sequential:In full_sequential, running task radosbench... 2023-06-03T20:05:37.608 INFO:tasks.radosbench:Beginning radosbench... 2023-06-03T20:05:37.608 INFO:tasks.cephadm.ceph_manager.ceph:creating pool_name unique_pool_5 2023-06-03T20:05:37.609 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph osd pool create unique_pool_5 16 2023-06-03T20:05:37.618 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:05:37.621 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:05:37.622 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:05:37.723 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:05:37.762 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:05:37.765 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:05:37.766 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:05:37.867 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:05:37.878 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:05:37 smithi089 bash[78452]: cluster 1685822736.5345428 mgr.y (mgr.34104) 883 : cluster 0 pgmap v814: 73 pgs: 73 active+clean; 582 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail 2023-06-03T20:05:37.878 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:05:37 smithi089 bash[78452]: audit 1685822737.1054676 mon.c (mon.2) 428 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T20:05:37.878 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:05:37 smithi089 bash[78452]: audit 1685822737.652967 mon.a (mon.0) 791 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:05:37.879 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:05:37 smithi089 bash[75068]: cluster 1685822736.5345428 mgr.y (mgr.34104) 883 : cluster 0 pgmap v814: 73 pgs: 73 active+clean; 582 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail 2023-06-03T20:05:37.879 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:05:37 smithi089 bash[75068]: audit 1685822737.1054676 mon.c (mon.2) 428 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T20:05:37.879 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:05:37 smithi089 bash[75068]: audit 1685822737.652967 mon.a (mon.0) 791 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:05:37.898 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:05:37.900 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:05:37.901 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:05:38.002 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:05:38.024 INFO:teuthology.orchestra.run.smithi164.stderr:Inferring config /var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/mon.b/config 2023-06-03T20:05:38.040 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:05:38.043 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:05:38.044 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:05:38.061 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:05:37 smithi164 bash[59462]: cluster 1685822736.5345428 mgr.y (mgr.34104) 883 : cluster 0 pgmap v814: 73 pgs: 73 active+clean; 582 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail 2023-06-03T20:05:38.062 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:05:37 smithi164 bash[59462]: audit 1685822737.1054676 mon.c (mon.2) 428 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T20:05:38.062 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:05:37 smithi164 bash[59462]: audit 1685822737.652967 mon.a (mon.0) 791 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:05:38.145 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:05:38.182 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:05:38.185 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:05:38.186 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:05:38.287 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:05:38.326 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:05:38.329 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:05:38.330 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:05:38.430 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:05:38.462 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:05:38.465 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:05:38.466 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:05:38.567 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:05:38.606 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:05:38.608 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:05:38.609 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:05:38.710 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:05:38.724 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-06-03T20:05:38.743 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:05:38.745 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:05:38.746 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:05:38.847 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:05:38.881 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:05:38.884 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:05:38.884 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:05:38.985 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:05:39.022 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:05:39.024 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:05:39.025 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:05:39.126 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:05:39.163 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:05:39.167 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:05:39.167 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:05:39.268 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:05:39.308 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:05:39.311 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:05:39.312 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:05:39.413 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:05:39.448 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:05:39.453 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:05:39.454 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:05:39.555 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:05:39.591 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:05:39.593 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:05:39.594 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:05:39.619 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph osd unset noscrub 2023-06-03T20:05:39.694 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:05:39.731 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:05:39.734 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:05:39.734 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:05:39.835 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:05:39.913 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:05:39 smithi089 bash[78452]: cluster 1685822738.5355108 mgr.y (mgr.34104) 884 : cluster 0 pgmap v815: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-06-03T20:05:39.913 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:05:39 smithi089 bash[75068]: cluster 1685822738.5355108 mgr.y (mgr.34104) 884 : cluster 0 pgmap v815: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-06-03T20:05:39.938 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:05:39.940 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:05:39.941 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:05:40.042 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:05:40.082 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:05:40.084 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:05:39 smithi164 bash[59462]: cluster 1685822738.5355108 mgr.y (mgr.34104) 884 : cluster 0 pgmap v815: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-06-03T20:05:40.085 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:05:40.085 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:05:40.186 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:05:40.230 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:05:40.233 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:05:40.234 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:05:40.335 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:05:40.363 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:05:40.366 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:05:40.367 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:05:40.468 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:05:40.502 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 5, 6, 7] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-06-03T20:05:40.502 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-06-03T20:05:40.502 INFO:tasks.thrashosds.thrasher:primary_affinity 2023-06-03T20:05:40.503 INFO:tasks.thrashosds.thrasher:Setting osd 7 primary_affinity to 0.531414 2023-06-03T20:05:40.503 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph osd primary-affinity 7 0.5314142386440327 2023-06-03T20:05:40.504 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:05:40.505 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:05:40.506 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:05:40.607 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:05:40.635 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:05:40.637 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:05:40.637 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:05:40.738 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:05:40.769 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:05:40.771 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:05:40.771 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:05:40.872 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:05:40.909 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:05:40.911 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:05:40.912 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:05:41.013 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:05:41.052 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:05:41.055 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:05:41.055 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:05:41.156 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:05:41.198 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:05:41.200 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:05:41.200 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:05:41.301 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:05:41.338 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:05:41.342 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:05:41.342 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:05:41.443 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:05:41.480 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:05:41.483 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:05:41.484 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:05:41.586 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:05:41.626 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:05:41.628 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:05:41.629 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:05:41.730 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:05:41.768 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:05:41.771 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:05:41.772 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:05:41.873 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:05:41.908 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:05:41.911 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:05:41.912 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:05:41.937 INFO:teuthology.orchestra.run.smithi164.stdout:{ 2023-06-03T20:05:41.937 INFO:teuthology.orchestra.run.smithi164.stdout: "ops": [], 2023-06-03T20:05:41.937 INFO:teuthology.orchestra.run.smithi164.stdout: "complaint_time": 30, 2023-06-03T20:05:41.937 INFO:teuthology.orchestra.run.smithi164.stdout: "num_blocked_ops": 0 2023-06-03T20:05:41.937 INFO:teuthology.orchestra.run.smithi164.stdout:} 2023-06-03T20:05:41.981 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:05:41 smithi164 bash[59462]: cluster 1685822740.5364933 mgr.y (mgr.34104) 885 : cluster 0 pgmap v816: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-06-03T20:05:42.013 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:05:42.026 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:05:41 smithi089 bash[78452]: cluster 1685822740.5364933 mgr.y (mgr.34104) 885 : cluster 0 pgmap v816: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-06-03T20:05:42.026 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:05:41 smithi089 bash[75068]: cluster 1685822740.5364933 mgr.y (mgr.34104) 885 : cluster 0 pgmap v816: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-06-03T20:05:42.051 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:05:42.054 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:05:42.054 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:05:42.155 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:05:42.193 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:05:42.196 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:05:42.197 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:05:42.298 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:05:42.335 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:05:42.338 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:05:42.338 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:05:42.407 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/mon.a/config 2023-06-03T20:05:42.439 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:05:42.469 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:05:42.470 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:05:42.471 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:05:42.572 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:05:42.607 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:05:42.610 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:05:42.611 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:05:42.712 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:05:42.749 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:05:42.751 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:05:42.752 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:05:42.802 DEBUG:teuthology.orchestra.run.smithi164:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph daemon osd.6 dump_historic_ops 2023-06-03T20:05:42.853 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:05:42.890 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:05:42.893 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:05:42.894 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:05:42.995 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:05:43.035 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:05:43.037 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:05:43.038 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:05:43.139 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:05:43.176 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:05:43.179 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:05:43.180 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:05:43.281 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:05:43.319 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:05:43.323 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:05:43.323 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:05:43.424 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:05:43.431 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/mon.a/config 2023-06-03T20:05:43.468 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:05:43.471 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:05:43.471 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:05:43.572 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:05:43.610 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:05:43.613 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:05:43.613 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:05:43.714 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:05:43.752 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:05:43.754 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:05:43.754 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:05:43.855 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:05:43.896 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:05:43.899 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:05:43.900 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:05:44.001 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:05:44.011 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:05:43 smithi089 bash[75068]: cluster 1685822742.537235 mgr.y (mgr.34104) 886 : cluster 0 pgmap v817: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-06-03T20:05:44.011 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:05:43 smithi089 bash[78452]: cluster 1685822742.537235 mgr.y (mgr.34104) 886 : cluster 0 pgmap v817: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-06-03T20:05:44.031 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:05:44.034 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:05:44.034 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:05:44.083 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:05:43 smithi164 bash[59462]: cluster 1685822742.537235 mgr.y (mgr.34104) 886 : cluster 0 pgmap v817: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-06-03T20:05:44.135 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:05:44.168 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:05:44.171 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:05:44.171 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:05:44.272 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:05:44.302 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:05:44.305 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:05:44.305 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:05:44.406 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:05:44.432 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:05:44.435 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:05:44.436 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:05:44.452 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/mon.a/config 2023-06-03T20:05:44.453 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/mon.a/config 2023-06-03T20:05:44.537 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:05:44.572 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:05:44.575 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:05:44.576 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:05:44.677 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:05:44.704 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:05:44.706 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:05:44.707 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:05:44.808 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:05:44.835 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:05:44.837 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:05:44 smithi089 bash[78452]: audit 1685822743.9862742 mon.c (mon.2) 429 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-03T20:05:44.837 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:05:44 smithi089 bash[78452]: audit 1685822744.3598673 mon.a (mon.0) 792 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:05:44.837 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:05:44 smithi089 bash[78452]: audit 1685822744.3733776 mon.a (mon.0) 793 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:05:44.837 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:05:44 smithi089 bash[78452]: audit 1685822744.4121897 mon.a (mon.0) 794 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:05:44.837 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:05:44 smithi089 bash[78452]: audit 1685822744.4227817 mon.a (mon.0) 795 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:05:44.838 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:05:44 smithi089 bash[75068]: audit 1685822743.9862742 mon.c (mon.2) 429 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-03T20:05:44.838 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:05:44 smithi089 bash[75068]: audit 1685822744.3598673 mon.a (mon.0) 792 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:05:44.838 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:05:44 smithi089 bash[75068]: audit 1685822744.3733776 mon.a (mon.0) 793 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:05:44.839 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:05:44 smithi089 bash[75068]: audit 1685822744.4121897 mon.a (mon.0) 794 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:05:44.839 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:05:44 smithi089 bash[75068]: audit 1685822744.4227817 mon.a (mon.0) 795 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:05:44.839 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:05:44.840 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:05:44.940 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:05:44.969 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:05:44.971 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:05:44.972 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:05:45.072 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:05:45.083 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:05:44 smithi164 bash[59462]: audit 1685822743.9862742 mon.c (mon.2) 429 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-03T20:05:45.084 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:05:44 smithi164 bash[59462]: audit 1685822744.3598673 mon.a (mon.0) 792 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:05:45.084 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:05:44 smithi164 bash[59462]: audit 1685822744.3733776 mon.a (mon.0) 793 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:05:45.084 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:05:44 smithi164 bash[59462]: audit 1685822744.4121897 mon.a (mon.0) 794 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:05:45.084 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:05:44 smithi164 bash[59462]: audit 1685822744.4227817 mon.a (mon.0) 795 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:05:45.101 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:05:45.104 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:05:45.105 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:05:45.206 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:05:45.246 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:05:45.249 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:05:45.250 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:05:45.350 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:05:45.389 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:05:45.392 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:05:45.393 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:05:45.494 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:05:45.532 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:05:45.535 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:05:45.535 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:05:45.636 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:05:45.670 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:05:45.674 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:05:45.674 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:05:45.775 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:05:45.809 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:05:45.812 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:05:45.813 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:05:45.914 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:05:45.957 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:05:45.960 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:05:45.961 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:05:46.062 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:05:46.073 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:05:45 smithi164 bash[59462]: cluster 1685822744.5378761 mgr.y (mgr.34104) 887 : cluster 0 pgmap v818: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-06-03T20:05:46.094 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:05:46.097 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:05:46.098 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:05:46.199 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:05:46.210 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:05:45 smithi089 bash[78452]: cluster 1685822744.5378761 mgr.y (mgr.34104) 887 : cluster 0 pgmap v818: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-06-03T20:05:46.211 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:05:45 smithi089 bash[75068]: cluster 1685822744.5378761 mgr.y (mgr.34104) 887 : cluster 0 pgmap v818: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-06-03T20:05:46.230 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:05:46.234 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:05:46.235 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:05:46.335 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:05:46.360 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:05:46.362 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:05:46.363 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:05:46.464 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:05:46.478 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 20:05:46 smithi089 bash[71163]: ::ffff:172.21.15.164 - - [03/Jun/2023:20:05:46] "GET /metrics HTTP/1.1" 200 225926 "" "Prometheus/2.33.4" 2023-06-03T20:05:46.501 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:05:46.505 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:05:46.505 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:05:46.606 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:05:46.638 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:05:46.641 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:05:46.642 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:05:46.743 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:05:46.782 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:05:46.786 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:05:46.787 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:05:46.887 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:05:46.912 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:05:46.914 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:05:46.915 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:05:47.016 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:05:47.062 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:05:47.062 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:05:47.062 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:05:47.163 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:05:47.197 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:05:47.200 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:05:47.201 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:05:47.302 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:05:47.339 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:05:47.342 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:05:47.343 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:05:47.444 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:05:47.481 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:05:47.484 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:05:47.485 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:05:47.586 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:05:47.619 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:05:47.622 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:05:47.623 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:05:47.666 INFO:teuthology.orchestra.run.smithi164.stderr:Inferring config /var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/mon.b/config 2023-06-03T20:05:47.724 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:05:47.758 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:05:47.761 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:05:47.762 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:05:47.863 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:05:47.893 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:05:47.895 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:05:47.895 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:05:47.996 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:05:48.007 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:05:47 smithi164 bash[59462]: cluster 1685822746.5383751 mgr.y (mgr.34104) 888 : cluster 0 pgmap v819: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-06-03T20:05:48.027 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:05:48.030 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:05:48.031 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:05:48.101 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:05:47 smithi089 bash[75068]: cluster 1685822746.5383751 mgr.y (mgr.34104) 888 : cluster 0 pgmap v819: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-06-03T20:05:48.101 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:05:47 smithi089 bash[78452]: cluster 1685822746.5383751 mgr.y (mgr.34104) 888 : cluster 0 pgmap v819: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-06-03T20:05:48.132 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:05:48.173 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:05:48.175 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:05:48.176 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:05:48.277 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:05:48.321 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:05:48.323 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:05:48.324 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:05:48.425 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:05:48.458 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:05:48.462 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:05:48.463 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:05:48.564 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:05:48.593 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:05:48.596 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:05:48.596 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:05:48.697 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:05:48.729 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:05:48.731 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:05:48.732 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:05:48.833 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:05:48.878 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:05:48.881 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:05:48.882 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:05:48.983 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:05:49.009 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:05:49.012 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:05:49.013 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:05:49.113 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:05:49.145 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:05:49.148 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:05:49.149 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:05:49.250 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:05:49.280 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:05:49.282 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:05:49.283 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:05:49.384 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:05:49.412 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:05:49.414 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:05:49.414 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:05:49.515 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:05:49.544 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:05:49.547 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:05:49.547 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:05:49.648 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:05:49.675 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:05:49.678 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:05:49.678 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:05:49.779 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:05:49.812 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:05:49.814 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:05:49.815 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:05:49.916 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:05:49.948 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:05:49.951 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:05:49.952 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:05:50.028 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:05:49 smithi089 bash[75068]: cluster 1685822748.539227 mgr.y (mgr.34104) 889 : cluster 0 pgmap v820: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-06-03T20:05:50.029 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:05:49 smithi089 bash[78452]: cluster 1685822748.539227 mgr.y (mgr.34104) 889 : cluster 0 pgmap v820: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-06-03T20:05:50.052 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:05:50.079 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:05:50.081 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:05:50.082 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:05:50.083 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:05:49 smithi164 bash[59462]: cluster 1685822748.539227 mgr.y (mgr.34104) 889 : cluster 0 pgmap v820: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-06-03T20:05:50.183 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:05:50.213 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:05:50.216 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:05:50.217 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:05:50.317 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:05:50.340 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:05:50.341 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:05:50.342 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:05:50.443 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:05:50.478 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:05:50.480 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:05:50.480 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:05:50.581 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:05:50.608 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:05:50.609 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:05:50.609 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:05:50.710 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:05:50.751 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:05:50.754 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:05:50.755 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:05:50.856 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:05:50.894 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:05:50.896 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:05:50.897 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:05:50.998 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:05:51.030 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:05:51.033 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:05:51.033 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:05:51.134 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:05:51.164 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:05:51.166 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:05:51.167 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:05:51.267 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:05:51.278 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:05:51 smithi089 bash[78452]: audit 1685822750.0627482 mon.a (mon.0) 796 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:05:51.278 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:05:51 smithi089 bash[78452]: audit 1685822750.072902 mon.a (mon.0) 797 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:05:51.278 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:05:51 smithi089 bash[78452]: cluster 1685822750.5397205 mgr.y (mgr.34104) 890 : cluster 0 pgmap v821: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-06-03T20:05:51.279 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:05:51 smithi089 bash[75068]: audit 1685822750.0627482 mon.a (mon.0) 796 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:05:51.279 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:05:51 smithi089 bash[75068]: audit 1685822750.072902 mon.a (mon.0) 797 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:05:51.279 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:05:51 smithi089 bash[75068]: cluster 1685822750.5397205 mgr.y (mgr.34104) 890 : cluster 0 pgmap v821: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-06-03T20:05:51.299 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:05:51.301 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:05:51.302 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:05:51.333 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:05:51 smithi164 bash[59462]: audit 1685822750.0627482 mon.a (mon.0) 796 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:05:51.333 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:05:51 smithi164 bash[59462]: audit 1685822750.072902 mon.a (mon.0) 797 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:05:51.334 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:05:51 smithi164 bash[59462]: cluster 1685822750.5397205 mgr.y (mgr.34104) 890 : cluster 0 pgmap v821: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-06-03T20:05:51.403 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:05:51.438 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:05:51.441 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:05:51.442 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:05:51.543 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:05:51.578 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:05:51.581 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:05:51.581 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:05:51.682 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:05:51.714 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:05:51.715 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:05:51.716 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:05:51.744 INFO:teuthology.orchestra.run.smithi164.stdout:{ 2023-06-03T20:05:51.745 INFO:teuthology.orchestra.run.smithi164.stdout: "size": 20, 2023-06-03T20:05:51.745 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 600, 2023-06-03T20:05:51.745 INFO:teuthology.orchestra.run.smithi164.stdout: "ops": [ 2023-06-03T20:05:51.745 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:05:51.745 INFO:teuthology.orchestra.run.smithi164.stdout: "description": "osd_repop(client.45713.0:90151 9.b e134/127 9:dcb8b4aa:::benchmark_data_smithi089_115580_object90150:head v 134'5780, mlcod=134'5777)", 2023-06-03T20:05:51.745 INFO:teuthology.orchestra.run.smithi164.stdout: "initiated_at": "2023-06-03T20:01:47.695641+0000", 2023-06-03T20:05:51.746 INFO:teuthology.orchestra.run.smithi164.stdout: "age": 244.039522705, 2023-06-03T20:05:51.746 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.031851722999999998, 2023-06-03T20:05:51.746 INFO:teuthology.orchestra.run.smithi164.stdout: "type_data": { 2023-06-03T20:05:51.746 INFO:teuthology.orchestra.run.smithi164.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-03T20:05:51.746 INFO:teuthology.orchestra.run.smithi164.stdout: "events": [ 2023-06-03T20:05:51.746 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:05:51.746 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "initiated", 2023-06-03T20:05:51.746 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:01:47.695641+0000", 2023-06-03T20:05:51.747 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0 2023-06-03T20:05:51.747 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:05:51.747 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:05:51.747 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "throttled", 2023-06-03T20:05:51.747 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:01:47.695641+0000", 2023-06-03T20:05:51.747 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 4294967295.999999 2023-06-03T20:05:51.747 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:05:51.748 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:05:51.748 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "header_read", 2023-06-03T20:05:51.748 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:01:47.695639+0000", 2023-06-03T20:05:51.748 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 7.0538999999999997e-05 2023-06-03T20:05:51.748 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:05:51.748 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:05:51.748 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "all_read", 2023-06-03T20:05:51.749 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:01:47.695710+0000", 2023-06-03T20:05:51.749 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 1.3927999999999999e-05 2023-06-03T20:05:51.749 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:05:51.749 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:05:51.749 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "dispatched", 2023-06-03T20:05:51.749 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:01:47.695724+0000", 2023-06-03T20:05:51.749 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 7.1829999999999997e-06 2023-06-03T20:05:51.750 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:05:51.750 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:05:51.750 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "queued_for_pg", 2023-06-03T20:05:51.750 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:01:47.695731+0000", 2023-06-03T20:05:51.750 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.0069512819999999996 2023-06-03T20:05:51.751 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:05:51.751 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:05:51.751 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "reached_pg", 2023-06-03T20:05:51.751 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:01:47.702682+0000", 2023-06-03T20:05:51.752 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 2.8849000000000001e-05 2023-06-03T20:05:51.752 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:05:51.752 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:05:51.753 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "started", 2023-06-03T20:05:51.753 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:01:47.702711+0000", 2023-06-03T20:05:51.753 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.024723762 2023-06-03T20:05:51.753 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:05:51.753 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:05:51.754 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "commit_sent", 2023-06-03T20:05:51.754 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:01:47.727435+0000", 2023-06-03T20:05:51.754 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 5.7309999999999998e-05 2023-06-03T20:05:51.755 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:05:51.755 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:05:51.755 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "done", 2023-06-03T20:05:51.755 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:01:47.727492+0000", 2023-06-03T20:05:51.756 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.031851722999999998 2023-06-03T20:05:51.756 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:05:51.756 INFO:teuthology.orchestra.run.smithi164.stdout: ] 2023-06-03T20:05:51.757 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:05:51.757 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:05:51.757 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:05:51.758 INFO:teuthology.orchestra.run.smithi164.stdout: "description": "osd_repop(client.45713.0:90158 9.b e134/127 9:d4e73270:::benchmark_data_smithi089_115580_object90157:head v 134'5781, mlcod=134'5777)", 2023-06-03T20:05:51.758 INFO:teuthology.orchestra.run.smithi164.stdout: "initiated_at": "2023-06-03T20:01:47.699999+0000", 2023-06-03T20:05:51.758 INFO:teuthology.orchestra.run.smithi164.stdout: "age": 244.035163958, 2023-06-03T20:05:51.758 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.027518658000000001, 2023-06-03T20:05:51.759 INFO:teuthology.orchestra.run.smithi164.stdout: "type_data": { 2023-06-03T20:05:51.759 INFO:teuthology.orchestra.run.smithi164.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-03T20:05:51.759 INFO:teuthology.orchestra.run.smithi164.stdout: "events": [ 2023-06-03T20:05:51.759 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:05:51.759 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "initiated", 2023-06-03T20:05:51.760 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:01:47.699999+0000", 2023-06-03T20:05:51.760 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0 2023-06-03T20:05:51.760 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:05:51.761 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:05:51.761 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "throttled", 2023-06-03T20:05:51.761 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:01:47.699999+0000", 2023-06-03T20:05:51.761 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 4294967295.999999 2023-06-03T20:05:51.761 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:05:51.762 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:05:51.762 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "header_read", 2023-06-03T20:05:51.762 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:01:47.699998+0000", 2023-06-03T20:05:51.762 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.000117843 2023-06-03T20:05:51.762 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:05:51.762 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:05:51.762 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "all_read", 2023-06-03T20:05:51.763 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:01:47.700116+0000", 2023-06-03T20:05:51.763 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 1.7635e-05 2023-06-03T20:05:51.763 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:05:51.763 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:05:51.763 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "dispatched", 2023-06-03T20:05:51.763 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:01:47.700134+0000", 2023-06-03T20:05:51.763 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 6.781e-06 2023-06-03T20:05:51.764 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:05:51.764 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:05:51.764 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "queued_for_pg", 2023-06-03T20:05:51.764 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:01:47.700141+0000", 2023-06-03T20:05:51.764 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.015066772000000001 2023-06-03T20:05:51.764 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:05:51.764 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:05:51.765 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "reached_pg", 2023-06-03T20:05:51.765 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:01:47.715208+0000", 2023-06-03T20:05:51.765 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 2.9560000000000002e-05 2023-06-03T20:05:51.765 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:05:51.765 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:05:51.765 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "started", 2023-06-03T20:05:51.766 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:01:47.715237+0000", 2023-06-03T20:05:51.766 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.012260607999999999 2023-06-03T20:05:51.766 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:05:51.766 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:05:51.766 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "commit_sent", 2023-06-03T20:05:51.766 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:01:47.727498+0000", 2023-06-03T20:05:51.766 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 2.0251999999999999e-05 2023-06-03T20:05:51.767 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:05:51.767 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:05:51.767 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "done", 2023-06-03T20:05:51.767 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:01:47.727518+0000", 2023-06-03T20:05:51.767 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.027518658000000001 2023-06-03T20:05:51.768 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:05:51.768 INFO:teuthology.orchestra.run.smithi164.stdout: ] 2023-06-03T20:05:51.768 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:05:51.768 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:05:51.768 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:05:51.768 INFO:teuthology.orchestra.run.smithi164.stdout: "description": "osd_repop(client.45713.0:135052 9.0 e138/127 9:0caa0d93:::benchmark_data_smithi089_115580_object24918:head v 138'8390, mlcod=138'8388)", 2023-06-03T20:05:51.769 INFO:teuthology.orchestra.run.smithi164.stdout: "initiated_at": "2023-06-03T20:02:43.528929+0000", 2023-06-03T20:05:51.769 INFO:teuthology.orchestra.run.smithi164.stdout: "age": 188.206234352, 2023-06-03T20:05:51.769 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.028574972000000001, 2023-06-03T20:05:51.769 INFO:teuthology.orchestra.run.smithi164.stdout: "type_data": { 2023-06-03T20:05:51.769 INFO:teuthology.orchestra.run.smithi164.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-03T20:05:51.769 INFO:teuthology.orchestra.run.smithi164.stdout: "events": [ 2023-06-03T20:05:51.769 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:05:51.769 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "initiated", 2023-06-03T20:05:51.770 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:02:43.528929+0000", 2023-06-03T20:05:51.770 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0 2023-06-03T20:05:51.770 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:05:51.770 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:05:51.770 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "throttled", 2023-06-03T20:05:51.770 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:02:43.528929+0000", 2023-06-03T20:05:51.770 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 4294967295.9999995 2023-06-03T20:05:51.771 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:05:51.771 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:05:51.771 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "header_read", 2023-06-03T20:05:51.771 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:02:43.528928+0000", 2023-06-03T20:05:51.771 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 4.9919999999999998e-06 2023-06-03T20:05:51.771 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:05:51.771 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:05:51.771 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "all_read", 2023-06-03T20:05:51.772 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:02:43.528933+0000", 2023-06-03T20:05:51.772 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 8.9420000000000001e-06 2023-06-03T20:05:51.772 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:05:51.772 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:05:51.772 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "dispatched", 2023-06-03T20:05:51.772 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:02:43.528942+0000", 2023-06-03T20:05:51.772 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 3.625e-06 2023-06-03T20:05:51.773 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:05:51.773 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:05:51.773 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "queued_for_pg", 2023-06-03T20:05:51.773 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:02:43.528946+0000", 2023-06-03T20:05:51.773 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 4.5841999999999999e-05 2023-06-03T20:05:51.773 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:05:51.774 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:05:51.774 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "reached_pg", 2023-06-03T20:05:51.774 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:02:43.528992+0000", 2023-06-03T20:05:51.774 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 2.0891e-05 2023-06-03T20:05:51.774 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:05:51.774 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:05:51.774 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "started", 2023-06-03T20:05:51.774 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:02:43.529013+0000", 2023-06-03T20:05:51.775 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.028454963 2023-06-03T20:05:51.775 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:05:51.775 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:05:51.775 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "commit_sent", 2023-06-03T20:05:51.775 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:02:43.557468+0000", 2023-06-03T20:05:51.775 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 3.6316999999999998e-05 2023-06-03T20:05:51.775 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:05:51.775 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:05:51.776 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "done", 2023-06-03T20:05:51.776 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:02:43.557504+0000", 2023-06-03T20:05:51.777 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.028574972000000001 2023-06-03T20:05:51.777 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:05:51.777 INFO:teuthology.orchestra.run.smithi164.stdout: ] 2023-06-03T20:05:51.777 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:05:51.777 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:05:51.778 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:05:51.778 INFO:teuthology.orchestra.run.smithi164.stdout: "description": "osd_repop(client.45713.0:141619 9.1 e138/127 9:87951da7:::benchmark_data_smithi089_115580_object31485:head v 138'8901, mlcod=138'8899)", 2023-06-03T20:05:51.778 INFO:teuthology.orchestra.run.smithi164.stdout: "initiated_at": "2023-06-03T20:02:52.679334+0000", 2023-06-03T20:05:51.778 INFO:teuthology.orchestra.run.smithi164.stdout: "age": 179.055829692, 2023-06-03T20:05:51.779 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.028497740000000001, 2023-06-03T20:05:51.779 INFO:teuthology.orchestra.run.smithi164.stdout: "type_data": { 2023-06-03T20:05:51.779 INFO:teuthology.orchestra.run.smithi164.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-03T20:05:51.779 INFO:teuthology.orchestra.run.smithi164.stdout: "events": [ 2023-06-03T20:05:51.780 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:05:51.780 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "initiated", 2023-06-03T20:05:51.780 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:02:52.679334+0000", 2023-06-03T20:05:51.780 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0 2023-06-03T20:05:51.781 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:05:51.781 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:05:51.781 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "throttled", 2023-06-03T20:05:51.781 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:02:52.679334+0000", 2023-06-03T20:05:51.782 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 4294967295.9999986 2023-06-03T20:05:51.782 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:05:51.782 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:05:51.783 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "header_read", 2023-06-03T20:05:51.783 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:02:52.679332+0000", 2023-06-03T20:05:51.783 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 1.6130999999999999e-05 2023-06-03T20:05:51.783 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:05:51.784 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:05:51.784 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "all_read", 2023-06-03T20:05:51.784 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:02:52.679348+0000", 2023-06-03T20:05:51.785 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 1.645e-05 2023-06-03T20:05:51.785 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:05:51.785 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:05:51.786 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "dispatched", 2023-06-03T20:05:51.786 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:02:52.679365+0000", 2023-06-03T20:05:51.786 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 7.6210000000000004e-06 2023-06-03T20:05:51.787 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:05:51.787 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:05:51.787 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "queued_for_pg", 2023-06-03T20:05:51.787 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:02:52.679372+0000", 2023-06-03T20:05:51.787 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 4.7559999999999999e-05 2023-06-03T20:05:51.787 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:05:51.788 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:05:51.788 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "reached_pg", 2023-06-03T20:05:51.788 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:02:52.679420+0000", 2023-06-03T20:05:51.788 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 2.4079000000000001e-05 2023-06-03T20:05:51.788 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:05:51.788 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:05:51.788 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "started", 2023-06-03T20:05:51.789 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:02:52.679444+0000", 2023-06-03T20:05:51.789 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.028260859999999999 2023-06-03T20:05:51.789 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:05:51.789 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:05:51.789 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "commit_sent", 2023-06-03T20:05:51.789 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:02:52.707705+0000", 2023-06-03T20:05:51.790 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.000126308 2023-06-03T20:05:51.790 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:05:51.790 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:05:51.790 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "done", 2023-06-03T20:05:51.790 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:02:52.707831+0000", 2023-06-03T20:05:51.790 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.028497740000000001 2023-06-03T20:05:51.791 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:05:51.791 INFO:teuthology.orchestra.run.smithi164.stdout: ] 2023-06-03T20:05:51.791 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:05:51.791 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:05:51.791 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:05:51.791 INFO:teuthology.orchestra.run.smithi164.stdout: "description": "osd_repop(client.45713.0:170423 9.b e142/127 9:daaeea1b:::benchmark_data_smithi089_115580_object60289:head v 142'10794, mlcod=142'10790)", 2023-06-03T20:05:51.791 INFO:teuthology.orchestra.run.smithi164.stdout: "initiated_at": "2023-06-03T20:03:33.352837+0000", 2023-06-03T20:05:51.792 INFO:teuthology.orchestra.run.smithi164.stdout: "age": 138.38232590300001, 2023-06-03T20:05:51.792 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.035350801000000001, 2023-06-03T20:05:51.792 INFO:teuthology.orchestra.run.smithi164.stdout: "type_data": { 2023-06-03T20:05:51.792 INFO:teuthology.orchestra.run.smithi164.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-03T20:05:51.792 INFO:teuthology.orchestra.run.smithi164.stdout: "events": [ 2023-06-03T20:05:51.792 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:05:51.793 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "initiated", 2023-06-03T20:05:51.793 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:03:33.352837+0000", 2023-06-03T20:05:51.793 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0 2023-06-03T20:05:51.793 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:05:51.793 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:05:51.793 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "throttled", 2023-06-03T20:05:51.794 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:03:33.352837+0000", 2023-06-03T20:05:51.794 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 4294967295.999999 2023-06-03T20:05:51.794 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:05:51.794 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:05:51.794 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "header_read", 2023-06-03T20:05:51.794 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:03:33.352836+0000", 2023-06-03T20:05:51.794 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 1.324e-05 2023-06-03T20:05:51.795 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:05:51.795 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:05:51.795 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "all_read", 2023-06-03T20:05:51.795 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:03:33.352849+0000", 2023-06-03T20:05:51.795 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 2.2286e-05 2023-06-03T20:05:51.795 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:05:51.795 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:05:51.796 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "dispatched", 2023-06-03T20:05:51.796 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:03:33.352872+0000", 2023-06-03T20:05:51.796 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 7.6009999999999997e-06 2023-06-03T20:05:51.796 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:05:51.796 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:05:51.796 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "queued_for_pg", 2023-06-03T20:05:51.797 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:03:33.352879+0000", 2023-06-03T20:05:51.797 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.0073963850000000001 2023-06-03T20:05:51.797 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:05:51.797 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:05:51.797 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "reached_pg", 2023-06-03T20:05:51.797 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:03:33.360276+0000", 2023-06-03T20:05:51.797 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 6.5716999999999994e-05 2023-06-03T20:05:51.798 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:05:51.798 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:05:51.798 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "started", 2023-06-03T20:05:51.798 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:03:33.360341+0000", 2023-06-03T20:05:51.798 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.027755025999999999 2023-06-03T20:05:51.798 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:05:51.798 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:05:51.798 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "commit_sent", 2023-06-03T20:05:51.799 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:03:33.388096+0000", 2023-06-03T20:05:51.799 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 9.1650999999999994e-05 2023-06-03T20:05:51.799 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:05:51.799 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:05:51.799 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "done", 2023-06-03T20:05:51.799 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:03:33.388188+0000", 2023-06-03T20:05:51.799 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.035350801000000001 2023-06-03T20:05:51.800 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:05:51.800 INFO:teuthology.orchestra.run.smithi164.stdout: ] 2023-06-03T20:05:51.800 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:05:51.800 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:05:51.800 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:05:51.800 INFO:teuthology.orchestra.run.smithi164.stdout: "description": "osd_repop(client.45713.0:176105 9.b e142/127 9:dd44b9f2:::benchmark_data_smithi089_115580_object65971:head v 142'11185, mlcod=142'11183)", 2023-06-03T20:05:51.801 INFO:teuthology.orchestra.run.smithi164.stdout: "initiated_at": "2023-06-03T20:03:41.305302+0000", 2023-06-03T20:05:51.801 INFO:teuthology.orchestra.run.smithi164.stdout: "age": 130.429861047, 2023-06-03T20:05:51.801 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.032868961000000002, 2023-06-03T20:05:51.802 INFO:teuthology.orchestra.run.smithi164.stdout: "type_data": { 2023-06-03T20:05:51.802 INFO:teuthology.orchestra.run.smithi164.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-03T20:05:51.802 INFO:teuthology.orchestra.run.smithi164.stdout: "events": [ 2023-06-03T20:05:51.802 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:05:51.803 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "initiated", 2023-06-03T20:05:51.803 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:03:41.305302+0000", 2023-06-03T20:05:51.803 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0 2023-06-03T20:05:51.804 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:05:51.804 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:05:51.804 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "throttled", 2023-06-03T20:05:51.805 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:03:41.305302+0000", 2023-06-03T20:05:51.805 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 4294967295.9999952 2023-06-03T20:05:51.805 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:05:51.806 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:05:51.806 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "header_read", 2023-06-03T20:05:51.806 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:03:41.305297+0000", 2023-06-03T20:05:51.807 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 1.8002000000000001e-05 2023-06-03T20:05:51.807 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:05:51.807 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:05:51.808 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "all_read", 2023-06-03T20:05:51.808 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:03:41.305315+0000", 2023-06-03T20:05:51.808 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 2.1542000000000001e-05 2023-06-03T20:05:51.809 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:05:51.809 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:05:51.809 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "dispatched", 2023-06-03T20:05:51.809 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:03:41.305337+0000", 2023-06-03T20:05:51.810 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 9.2089999999999994e-06 2023-06-03T20:05:51.810 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:05:51.810 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:05:51.811 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "queued_for_pg", 2023-06-03T20:05:51.811 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:03:41.305346+0000", 2023-06-03T20:05:51.811 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 4.8807000000000002e-05 2023-06-03T20:05:51.811 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:05:51.812 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:05:51.812 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "reached_pg", 2023-06-03T20:05:51.812 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:03:41.305395+0000", 2023-06-03T20:05:51.812 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 2.3558999999999999e-05 2023-06-03T20:05:51.813 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:05:51.813 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:05:51.813 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "started", 2023-06-03T20:05:51.813 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:03:41.305419+0000", 2023-06-03T20:05:51.813 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.032671009000000001 2023-06-03T20:05:51.813 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:05:51.813 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:05:51.814 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "commit_sent", 2023-06-03T20:05:51.814 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:03:41.338090+0000", 2023-06-03T20:05:51.814 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 8.1559999999999998e-05 2023-06-03T20:05:51.814 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:05:51.814 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:05:51.815 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "done", 2023-06-03T20:05:51.815 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:03:41.338171+0000", 2023-06-03T20:05:51.815 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.032868961000000002 2023-06-03T20:05:51.815 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:05:51.815 INFO:teuthology.orchestra.run.smithi164.stdout: ] 2023-06-03T20:05:51.815 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:05:51.816 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:05:51.816 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:05:51.816 INFO:teuthology.orchestra.run.smithi164.stdout: "description": "osd_repop(client.45713.0:176108 9.b e142/127 9:d0f72982:::benchmark_data_smithi089_115580_object65974:head v 142'11186, mlcod=142'11183)", 2023-06-03T20:05:51.816 INFO:teuthology.orchestra.run.smithi164.stdout: "initiated_at": "2023-06-03T20:03:41.310920+0000", 2023-06-03T20:05:51.816 INFO:teuthology.orchestra.run.smithi164.stdout: "age": 130.424243628, 2023-06-03T20:05:51.816 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.027315369999999999, 2023-06-03T20:05:51.816 INFO:teuthology.orchestra.run.smithi164.stdout: "type_data": { 2023-06-03T20:05:51.817 INFO:teuthology.orchestra.run.smithi164.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-03T20:05:51.817 INFO:teuthology.orchestra.run.smithi164.stdout: "events": [ 2023-06-03T20:05:51.817 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:05:51.817 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "initiated", 2023-06-03T20:05:51.817 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:03:41.310920+0000", 2023-06-03T20:05:51.817 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0 2023-06-03T20:05:51.817 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:05:51.818 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:05:51.818 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "throttled", 2023-06-03T20:05:51.818 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:03:41.310920+0000", 2023-06-03T20:05:51.818 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 4294967295.999999 2023-06-03T20:05:51.818 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:05:51.818 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:05:51.818 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "header_read", 2023-06-03T20:05:51.819 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:03:41.310919+0000", 2023-06-03T20:05:51.819 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 1.1574e-05 2023-06-03T20:05:51.819 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:05:51.819 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:05:51.819 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "all_read", 2023-06-03T20:05:51.819 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:03:41.310930+0000", 2023-06-03T20:05:51.819 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 1.3112000000000001e-05 2023-06-03T20:05:51.820 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:05:51.820 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:05:51.820 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "dispatched", 2023-06-03T20:05:51.820 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:03:41.310943+0000", 2023-06-03T20:05:51.820 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 6.4609999999999996e-06 2023-06-03T20:05:51.820 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:05:51.821 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:05:51.821 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "queued_for_pg", 2023-06-03T20:05:51.821 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:03:41.310950+0000", 2023-06-03T20:05:51.821 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.0068865439999999997 2023-06-03T20:05:51.821 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:05:51.821 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:05:51.821 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "reached_pg", 2023-06-03T20:05:51.822 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:03:41.317836+0000", 2023-06-03T20:05:51.822 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 5.1409999999999997e-05 2023-06-03T20:05:51.822 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:05:51.822 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:05:51.822 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "started", 2023-06-03T20:05:51.822 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:03:41.317888+0000", 2023-06-03T20:05:51.822 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.020289633000000001 2023-06-03T20:05:51.823 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:05:51.823 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:05:51.823 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "commit_sent", 2023-06-03T20:05:51.823 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:03:41.338177+0000", 2023-06-03T20:05:51.824 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 5.7504999999999999e-05 2023-06-03T20:05:51.824 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:05:51.824 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:05:51.825 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "done", 2023-06-03T20:05:51.825 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:03:41.338235+0000", 2023-06-03T20:05:51.825 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.027315369999999999 2023-06-03T20:05:51.825 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:05:51.826 INFO:teuthology.orchestra.run.smithi164.stdout: ] 2023-06-03T20:05:51.826 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:05:51.826 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:05:51.827 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:05:51.827 INFO:teuthology.orchestra.run.smithi164.stdout: "description": "osd_repop(client.45713.0:176795 9.b e142/127 9:d9a86080:::benchmark_data_smithi089_115580_object66661:head v 142'11233, mlcod=142'11230)", 2023-06-03T20:05:51.827 INFO:teuthology.orchestra.run.smithi164.stdout: "initiated_at": "2023-06-03T20:03:42.270884+0000", 2023-06-03T20:05:51.827 INFO:teuthology.orchestra.run.smithi164.stdout: "age": 129.464279688, 2023-06-03T20:05:51.828 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.035368230000000001, 2023-06-03T20:05:51.828 INFO:teuthology.orchestra.run.smithi164.stdout: "type_data": { 2023-06-03T20:05:51.828 INFO:teuthology.orchestra.run.smithi164.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-03T20:05:51.828 INFO:teuthology.orchestra.run.smithi164.stdout: "events": [ 2023-06-03T20:05:51.828 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:05:51.828 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "initiated", 2023-06-03T20:05:51.829 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:03:42.270884+0000", 2023-06-03T20:05:51.829 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0 2023-06-03T20:05:51.829 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:05:51.829 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:05:51.829 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "throttled", 2023-06-03T20:05:51.829 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:03:42.270884+0000", 2023-06-03T20:05:51.829 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 4294967295.9999986 2023-06-03T20:05:51.830 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:05:51.830 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:05:51.830 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "header_read", 2023-06-03T20:05:51.830 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:03:42.270882+0000", 2023-06-03T20:05:51.830 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 1.5690999999999999e-05 2023-06-03T20:05:51.830 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:05:51.830 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:05:51.831 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "all_read", 2023-06-03T20:05:51.831 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:03:42.270898+0000", 2023-06-03T20:05:51.831 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 2.2949999999999999e-05 2023-06-03T20:05:51.831 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:05:51.831 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:05:51.831 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "dispatched", 2023-06-03T20:05:51.831 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:03:42.270921+0000", 2023-06-03T20:05:51.832 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 9.4250000000000004e-06 2023-06-03T20:05:51.832 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:05:51.832 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:05:51.832 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "queued_for_pg", 2023-06-03T20:05:51.832 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:03:42.270930+0000", 2023-06-03T20:05:51.832 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.00059237000000000005 2023-06-03T20:05:51.832 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:05:51.833 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:05:51.833 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "reached_pg", 2023-06-03T20:05:51.833 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:03:42.271523+0000", 2023-06-03T20:05:51.833 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 6.6642000000000003e-05 2023-06-03T20:05:51.833 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:05:51.833 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:05:51.833 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "started", 2023-06-03T20:05:51.834 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:03:42.271589+0000", 2023-06-03T20:05:51.834 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.034605266000000003 2023-06-03T20:05:51.834 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:05:51.834 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:05:51.834 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "commit_sent", 2023-06-03T20:05:51.834 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:03:42.306194+0000", 2023-06-03T20:05:51.834 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 5.7352000000000002e-05 2023-06-03T20:05:51.835 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:05:51.835 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:05:51.835 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "done", 2023-06-03T20:05:51.835 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:03:42.306252+0000", 2023-06-03T20:05:51.835 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.035368230000000001 2023-06-03T20:05:51.835 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:05:51.835 INFO:teuthology.orchestra.run.smithi164.stdout: ] 2023-06-03T20:05:51.836 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:05:51.836 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:05:51.836 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:05:51.836 INFO:teuthology.orchestra.run.smithi164.stdout: "description": "osd_repop(client.45713.0:178671 9.b e142/127 9:db7fba2e:::benchmark_data_smithi089_115580_object68537:head v 142'11358, mlcod=142'11354)", 2023-06-03T20:05:51.836 INFO:teuthology.orchestra.run.smithi164.stdout: "initiated_at": "2023-06-03T20:03:45.100937+0000", 2023-06-03T20:05:51.836 INFO:teuthology.orchestra.run.smithi164.stdout: "age": 126.634225976, 2023-06-03T20:05:51.836 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.027545686, 2023-06-03T20:05:51.837 INFO:teuthology.orchestra.run.smithi164.stdout: "type_data": { 2023-06-03T20:05:51.837 INFO:teuthology.orchestra.run.smithi164.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-03T20:05:51.837 INFO:teuthology.orchestra.run.smithi164.stdout: "events": [ 2023-06-03T20:05:51.837 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:05:51.837 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "initiated", 2023-06-03T20:05:51.837 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:03:45.100937+0000", 2023-06-03T20:05:51.837 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0 2023-06-03T20:05:51.838 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:05:51.838 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:05:51.839 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "throttled", 2023-06-03T20:05:51.839 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:03:45.100937+0000", 2023-06-03T20:05:51.839 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 4294967295.9999986 2023-06-03T20:05:51.839 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:05:51.839 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:05:51.839 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "header_read", 2023-06-03T20:05:51.840 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:03:45.100936+0000", 2023-06-03T20:05:51.840 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 1.6103000000000002e-05 2023-06-03T20:05:51.840 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:05:51.840 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:05:51.840 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "all_read", 2023-06-03T20:05:51.840 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:03:45.100952+0000", 2023-06-03T20:05:51.841 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 3.7067999999999998e-05 2023-06-03T20:05:51.841 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:05:51.841 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:05:51.841 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "dispatched", 2023-06-03T20:05:51.841 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:03:45.100989+0000", 2023-06-03T20:05:51.841 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 8.0930000000000003e-06 2023-06-03T20:05:51.842 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:05:51.842 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:05:51.842 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "queued_for_pg", 2023-06-03T20:05:51.842 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:03:45.100997+0000", 2023-06-03T20:05:51.842 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.000100714 2023-06-03T20:05:51.842 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:05:51.843 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:05:51.843 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "reached_pg", 2023-06-03T20:05:51.843 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:03:45.101098+0000", 2023-06-03T20:05:51.843 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 2.5425000000000001e-05 2023-06-03T20:05:51.843 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:05:51.843 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:05:51.843 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "started", 2023-06-03T20:05:51.844 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:03:45.101123+0000", 2023-06-03T20:05:51.844 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.027319030000000001 2023-06-03T20:05:51.844 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:05:51.844 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:05:51.844 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "commit_sent", 2023-06-03T20:05:51.844 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:03:45.128442+0000", 2023-06-03T20:05:51.844 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 4.0663e-05 2023-06-03T20:05:51.845 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:05:51.845 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:05:51.845 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "done", 2023-06-03T20:05:51.845 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:03:45.128483+0000", 2023-06-03T20:05:51.845 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.027545686 2023-06-03T20:05:51.845 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:05:51.845 INFO:teuthology.orchestra.run.smithi164.stdout: ] 2023-06-03T20:05:51.845 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:05:51.846 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:05:51.846 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:05:51.846 INFO:teuthology.orchestra.run.smithi164.stdout: "description": "osd_repop(client.45713.0:182071 9.0 e142/127 9:0222719f:::benchmark_data_smithi089_115580_object71937:head v 142'11229, mlcod=142'11226)", 2023-06-03T20:05:51.846 INFO:teuthology.orchestra.run.smithi164.stdout: "initiated_at": "2023-06-03T20:03:50.033869+0000", 2023-06-03T20:05:51.846 INFO:teuthology.orchestra.run.smithi164.stdout: "age": 121.701294619, 2023-06-03T20:05:51.846 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.027561446999999999, 2023-06-03T20:05:51.847 INFO:teuthology.orchestra.run.smithi164.stdout: "type_data": { 2023-06-03T20:05:51.847 INFO:teuthology.orchestra.run.smithi164.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-03T20:05:51.847 INFO:teuthology.orchestra.run.smithi164.stdout: "events": [ 2023-06-03T20:05:51.847 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:05:51.847 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "initiated", 2023-06-03T20:05:51.847 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:03:50.033869+0000", 2023-06-03T20:05:51.847 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0 2023-06-03T20:05:51.847 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:05:51.848 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:05:51.848 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "throttled", 2023-06-03T20:05:51.848 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:03:50.033869+0000", 2023-06-03T20:05:51.848 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 4294967295.999999 2023-06-03T20:05:51.848 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:05:51.848 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:05:51.848 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "header_read", 2023-06-03T20:05:51.849 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:03:50.033868+0000", 2023-06-03T20:05:51.849 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 1.5639999999999999e-05 2023-06-03T20:05:51.849 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:05:51.849 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:05:51.849 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "all_read", 2023-06-03T20:05:51.849 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:03:50.033883+0000", 2023-06-03T20:05:51.849 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 1.9531e-05 2023-06-03T20:05:51.850 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:05:51.850 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:05:51.850 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "dispatched", 2023-06-03T20:05:51.850 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:03:50.033903+0000", 2023-06-03T20:05:51.850 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 2.4204000000000001e-05 2023-06-03T20:05:51.850 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:05:51.850 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:05:51.851 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "queued_for_pg", 2023-06-03T20:05:51.851 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:03:50.033927+0000", 2023-06-03T20:05:51.851 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 7.0038999999999998e-05 2023-06-03T20:05:51.851 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:05:51.851 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:05:51.851 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "reached_pg", 2023-06-03T20:05:51.851 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:03:50.033997+0000", 2023-06-03T20:05:51.852 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 2.9283999999999999e-05 2023-06-03T20:05:51.852 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:05:51.852 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:05:51.852 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "started", 2023-06-03T20:05:51.852 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:03:50.034026+0000", 2023-06-03T20:05:51.852 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.027338289000000002 2023-06-03T20:05:51.852 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:05:51.853 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:05:51.853 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "commit_sent", 2023-06-03T20:05:51.853 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:03:50.061365+0000", 2023-06-03T20:05:51.853 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 6.5536000000000004e-05 2023-06-03T20:05:51.853 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:05:51.853 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:05:51.853 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "done", 2023-06-03T20:05:51.853 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:03:50.061430+0000", 2023-06-03T20:05:51.854 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.027561446999999999 2023-06-03T20:05:51.854 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:05:51.854 INFO:teuthology.orchestra.run.smithi164.stdout: ] 2023-06-03T20:05:51.854 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:05:51.854 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:05:51.854 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:05:51.854 INFO:teuthology.orchestra.run.smithi164.stdout: "description": "osd_repop(client.45713.0:196609 9.1 e144/127 9:80203227:::benchmark_data_smithi089_115580_object86475:head v 144'12348, mlcod=144'12342)", 2023-06-03T20:05:51.855 INFO:teuthology.orchestra.run.smithi164.stdout: "initiated_at": "2023-06-03T20:04:10.949041+0000", 2023-06-03T20:05:51.855 INFO:teuthology.orchestra.run.smithi164.stdout: "age": 100.786122318, 2023-06-03T20:05:51.855 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.026235827, 2023-06-03T20:05:51.855 INFO:teuthology.orchestra.run.smithi164.stdout: "type_data": { 2023-06-03T20:05:51.855 INFO:teuthology.orchestra.run.smithi164.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-03T20:05:51.855 INFO:teuthology.orchestra.run.smithi164.stdout: "events": [ 2023-06-03T20:05:51.855 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:05:51.856 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "initiated", 2023-06-03T20:05:51.856 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:04:10.949041+0000", 2023-06-03T20:05:51.856 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0 2023-06-03T20:05:51.856 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:05:51.856 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:05:51.856 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "throttled", 2023-06-03T20:05:51.856 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:04:10.949041+0000", 2023-06-03T20:05:51.857 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 4294967295.9999995 2023-06-03T20:05:51.857 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:05:51.857 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:05:51.857 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "header_read", 2023-06-03T20:05:51.857 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:04:10.949040+0000", 2023-06-03T20:05:51.857 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 8.0329999999999992e-06 2023-06-03T20:05:51.857 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:05:51.858 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:05:51.858 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "all_read", 2023-06-03T20:05:51.858 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:04:10.949048+0000", 2023-06-03T20:05:51.858 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 1.0903999999999999e-05 2023-06-03T20:05:51.858 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:05:51.858 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:05:51.858 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "dispatched", 2023-06-03T20:05:51.859 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:04:10.949059+0000", 2023-06-03T20:05:51.859 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 5.8089999999999998e-06 2023-06-03T20:05:51.859 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:05:51.859 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:05:51.859 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "queued_for_pg", 2023-06-03T20:05:51.859 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:04:10.949065+0000", 2023-06-03T20:05:51.859 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.0088653470000000009 2023-06-03T20:05:51.860 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:05:51.860 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:05:51.860 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "reached_pg", 2023-06-03T20:05:51.860 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:04:10.957930+0000", 2023-06-03T20:05:51.860 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 1.891e-05 2023-06-03T20:05:51.860 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:05:51.860 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:05:51.861 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "started", 2023-06-03T20:05:51.861 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:04:10.957949+0000", 2023-06-03T20:05:51.861 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.017204283000000001 2023-06-03T20:05:51.861 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:05:51.861 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:05:51.861 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "commit_sent", 2023-06-03T20:05:51.861 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:04:10.975154+0000", 2023-06-03T20:05:51.862 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.00012320000000000001 2023-06-03T20:05:51.862 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:05:51.862 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:05:51.862 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "done", 2023-06-03T20:05:51.862 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:04:10.975277+0000", 2023-06-03T20:05:51.862 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.026235827 2023-06-03T20:05:51.862 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:05:51.863 INFO:teuthology.orchestra.run.smithi164.stdout: ] 2023-06-03T20:05:51.863 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:05:51.863 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:05:51.863 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:05:51.863 INFO:teuthology.orchestra.run.smithi164.stdout: "description": "osd_repop(client.45713.0:198318 9.b e144/127 9:de6ab50a:::benchmark_data_smithi089_115580_object88184:head v 144'12645, mlcod=144'12640)", 2023-06-03T20:05:51.863 INFO:teuthology.orchestra.run.smithi164.stdout: "initiated_at": "2023-06-03T20:04:13.401217+0000", 2023-06-03T20:05:51.863 INFO:teuthology.orchestra.run.smithi164.stdout: "age": 98.333945799000006, 2023-06-03T20:05:51.864 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.028036043, 2023-06-03T20:05:51.864 INFO:teuthology.orchestra.run.smithi164.stdout: "type_data": { 2023-06-03T20:05:51.864 INFO:teuthology.orchestra.run.smithi164.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-03T20:05:51.864 INFO:teuthology.orchestra.run.smithi164.stdout: "events": [ 2023-06-03T20:05:51.864 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:05:51.864 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "initiated", 2023-06-03T20:05:51.864 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:04:13.401217+0000", 2023-06-03T20:05:51.865 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0 2023-06-03T20:05:51.865 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:05:51.865 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:05:51.865 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "throttled", 2023-06-03T20:05:51.865 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:04:13.401217+0000", 2023-06-03T20:05:51.865 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 4294967295.999999 2023-06-03T20:05:51.865 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:05:51.866 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:05:51.866 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "header_read", 2023-06-03T20:05:51.866 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:04:13.401216+0000", 2023-06-03T20:05:51.866 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 1.3495e-05 2023-06-03T20:05:51.866 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:05:51.866 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:05:51.866 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "all_read", 2023-06-03T20:05:51.866 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:04:13.401230+0000", 2023-06-03T20:05:51.867 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 1.9792999999999998e-05 2023-06-03T20:05:51.867 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:05:51.867 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:05:51.867 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "dispatched", 2023-06-03T20:05:51.867 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:04:13.401250+0000", 2023-06-03T20:05:51.867 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 6.6939999999999998e-06 2023-06-03T20:05:51.867 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:05:51.868 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:05:51.868 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "queued_for_pg", 2023-06-03T20:05:51.868 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:04:13.401256+0000", 2023-06-03T20:05:51.868 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.0041537910000000004 2023-06-03T20:05:51.868 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:05:51.868 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:05:51.868 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "reached_pg", 2023-06-03T20:05:51.869 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:04:13.405410+0000", 2023-06-03T20:05:51.869 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 2.0432e-05 2023-06-03T20:05:51.869 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:05:51.869 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:05:51.869 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "started", 2023-06-03T20:05:51.869 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:04:13.405430+0000", 2023-06-03T20:05:51.869 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.023779531 2023-06-03T20:05:51.870 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:05:51.870 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:05:51.870 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "commit_sent", 2023-06-03T20:05:51.870 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:04:13.429210+0000", 2023-06-03T20:05:51.870 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 4.3454000000000001e-05 2023-06-03T20:05:51.870 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:05:51.870 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:05:51.871 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "done", 2023-06-03T20:05:51.871 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:04:13.429253+0000", 2023-06-03T20:05:51.871 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.028036043 2023-06-03T20:05:51.871 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:05:51.871 INFO:teuthology.orchestra.run.smithi164.stdout: ] 2023-06-03T20:05:51.871 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:05:51.871 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:05:51.872 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:05:51.872 INFO:teuthology.orchestra.run.smithi164.stdout: "description": "osd_repop(client.45713.0:198810 9.1 e144/127 9:845077b2:::benchmark_data_smithi089_115580_object88676:head v 144'12492, mlcod=144'12488)", 2023-06-03T20:05:51.872 INFO:teuthology.orchestra.run.smithi164.stdout: "initiated_at": "2023-06-03T20:04:14.064514+0000", 2023-06-03T20:05:51.872 INFO:teuthology.orchestra.run.smithi164.stdout: "age": 97.670648927000002, 2023-06-03T20:05:51.872 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.029753119000000001, 2023-06-03T20:05:51.872 INFO:teuthology.orchestra.run.smithi164.stdout: "type_data": { 2023-06-03T20:05:51.872 INFO:teuthology.orchestra.run.smithi164.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-03T20:05:51.873 INFO:teuthology.orchestra.run.smithi164.stdout: "events": [ 2023-06-03T20:05:51.873 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:05:51.873 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "initiated", 2023-06-03T20:05:51.873 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:04:14.064514+0000", 2023-06-03T20:05:51.873 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0 2023-06-03T20:05:51.873 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:05:51.873 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:05:51.874 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "throttled", 2023-06-03T20:05:51.874 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:04:14.064514+0000", 2023-06-03T20:05:51.874 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 4294967295.999999 2023-06-03T20:05:51.874 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:05:51.874 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:05:51.874 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "header_read", 2023-06-03T20:05:51.874 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:04:14.064513+0000", 2023-06-03T20:05:51.875 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 1.2911999999999999e-05 2023-06-03T20:05:51.875 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:05:51.875 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:05:51.875 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "all_read", 2023-06-03T20:05:51.875 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:04:14.064526+0000", 2023-06-03T20:05:51.875 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 1.432e-05 2023-06-03T20:05:51.875 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:05:51.876 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:05:51.876 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "dispatched", 2023-06-03T20:05:51.876 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:04:14.064540+0000", 2023-06-03T20:05:51.876 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 6.1310000000000003e-06 2023-06-03T20:05:51.876 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:05:51.876 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:05:51.877 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "queued_for_pg", 2023-06-03T20:05:51.877 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:04:14.064547+0000", 2023-06-03T20:05:51.877 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.017403812000000001 2023-06-03T20:05:51.877 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:05:51.877 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:05:51.877 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "reached_pg", 2023-06-03T20:05:51.877 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:04:14.081950+0000", 2023-06-03T20:05:51.877 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 6.8826999999999997e-05 2023-06-03T20:05:51.878 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:05:51.878 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:05:51.878 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "started", 2023-06-03T20:05:51.878 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:04:14.082019+0000", 2023-06-03T20:05:51.878 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.011045601 2023-06-03T20:05:51.878 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:05:51.878 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:05:51.879 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "commit_sent", 2023-06-03T20:05:51.879 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:04:14.093065+0000", 2023-06-03T20:05:51.879 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.001202655 2023-06-03T20:05:51.879 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:05:51.879 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:05:51.879 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "done", 2023-06-03T20:05:51.879 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:04:14.094267+0000", 2023-06-03T20:05:51.880 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.029753119000000001 2023-06-03T20:05:51.880 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:05:51.880 INFO:teuthology.orchestra.run.smithi164.stdout: ] 2023-06-03T20:05:51.880 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:05:51.880 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:05:51.880 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:05:51.880 INFO:teuthology.orchestra.run.smithi164.stdout: "description": "osd_repop(client.45713.0:199270 9.b e144/127 9:dfd7aa73:::benchmark_data_smithi089_115580_object89136:head v 144'12714, mlcod=144'12711)", 2023-06-03T20:05:51.881 INFO:teuthology.orchestra.run.smithi164.stdout: "initiated_at": "2023-06-03T20:04:14.717783+0000", 2023-06-03T20:05:51.881 INFO:teuthology.orchestra.run.smithi164.stdout: "age": 97.017380059999994, 2023-06-03T20:05:51.881 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.026165590999999998, 2023-06-03T20:05:51.881 INFO:teuthology.orchestra.run.smithi164.stdout: "type_data": { 2023-06-03T20:05:51.881 INFO:teuthology.orchestra.run.smithi164.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-03T20:05:51.881 INFO:teuthology.orchestra.run.smithi164.stdout: "events": [ 2023-06-03T20:05:51.881 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:05:51.882 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "initiated", 2023-06-03T20:05:51.882 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:04:14.717783+0000", 2023-06-03T20:05:51.882 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0 2023-06-03T20:05:51.882 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:05:51.882 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:05:51.882 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "throttled", 2023-06-03T20:05:51.882 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:04:14.717783+0000", 2023-06-03T20:05:51.883 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 4294967295.999999 2023-06-03T20:05:51.883 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:05:51.883 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:05:51.883 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "header_read", 2023-06-03T20:05:51.883 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:04:14.717782+0000", 2023-06-03T20:05:51.883 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 1.4711e-05 2023-06-03T20:05:51.883 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:05:51.884 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:05:51.884 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "all_read", 2023-06-03T20:05:51.884 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:04:14.717797+0000", 2023-06-03T20:05:51.884 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 1.6504e-05 2023-06-03T20:05:51.884 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:05:51.884 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:05:51.884 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "dispatched", 2023-06-03T20:05:51.885 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:04:14.717813+0000", 2023-06-03T20:05:51.885 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 7.4270000000000002e-06 2023-06-03T20:05:51.885 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:05:51.885 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:05:51.885 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "queued_for_pg", 2023-06-03T20:05:51.885 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:04:14.717821+0000", 2023-06-03T20:05:51.885 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 8.3258999999999997e-05 2023-06-03T20:05:51.886 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:05:51.886 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:05:51.886 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "reached_pg", 2023-06-03T20:05:51.886 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:04:14.717904+0000", 2023-06-03T20:05:51.886 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 2.6821e-05 2023-06-03T20:05:51.886 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:05:51.886 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:05:51.887 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "started", 2023-06-03T20:05:51.887 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:04:14.717931+0000", 2023-06-03T20:05:51.887 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.025967884 2023-06-03T20:05:51.887 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:05:51.887 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:05:51.887 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "commit_sent", 2023-06-03T20:05:51.887 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:04:14.743899+0000", 2023-06-03T20:05:51.888 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 5.0053999999999999e-05 2023-06-03T20:05:51.888 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:05:51.888 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:05:51.888 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "done", 2023-06-03T20:05:51.888 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:04:14.743949+0000", 2023-06-03T20:05:51.888 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.026165590999999998 2023-06-03T20:05:51.888 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:05:51.888 INFO:teuthology.orchestra.run.smithi164.stdout: ] 2023-06-03T20:05:51.889 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:05:51.889 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:05:51.889 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:05:51.889 INFO:teuthology.orchestra.run.smithi164.stdout: "description": "osd_repop(client.45713.0:205604 9.b e144/127 9:d166677e:::benchmark_data_smithi089_115580_object95470:head v 144'13103, mlcod=144'13100)", 2023-06-03T20:05:51.889 INFO:teuthology.orchestra.run.smithi164.stdout: "initiated_at": "2023-06-03T20:04:23.685535+0000", 2023-06-03T20:05:51.889 INFO:teuthology.orchestra.run.smithi164.stdout: "age": 88.049628372000001, 2023-06-03T20:05:51.889 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.026576459, 2023-06-03T20:05:51.890 INFO:teuthology.orchestra.run.smithi164.stdout: "type_data": { 2023-06-03T20:05:51.890 INFO:teuthology.orchestra.run.smithi164.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-03T20:05:51.890 INFO:teuthology.orchestra.run.smithi164.stdout: "events": [ 2023-06-03T20:05:51.890 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:05:51.890 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "initiated", 2023-06-03T20:05:51.890 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:04:23.685535+0000", 2023-06-03T20:05:51.890 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0 2023-06-03T20:05:51.891 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:05:51.891 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:05:51.891 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "throttled", 2023-06-03T20:05:51.891 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:04:23.685535+0000", 2023-06-03T20:05:51.891 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 4294967295.999999 2023-06-03T20:05:51.891 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:05:51.891 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:05:51.892 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "header_read", 2023-06-03T20:05:51.892 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:04:23.685534+0000", 2023-06-03T20:05:51.892 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 1.3494e-05 2023-06-03T20:05:51.892 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:05:51.892 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:05:51.892 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "all_read", 2023-06-03T20:05:51.893 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:04:23.685547+0000", 2023-06-03T20:05:51.893 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 1.8604999999999999e-05 2023-06-03T20:05:51.893 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:05:51.893 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:05:51.893 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "dispatched", 2023-06-03T20:05:51.893 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:04:23.685566+0000", 2023-06-03T20:05:51.893 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 8.687e-06 2023-06-03T20:05:51.894 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:05:51.894 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:05:51.894 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "queued_for_pg", 2023-06-03T20:05:51.894 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:04:23.685575+0000", 2023-06-03T20:05:51.894 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 4.9061999999999999e-05 2023-06-03T20:05:51.894 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:05:51.894 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:05:51.894 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "reached_pg", 2023-06-03T20:05:51.895 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:04:23.685624+0000", 2023-06-03T20:05:51.895 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 2.0914e-05 2023-06-03T20:05:51.895 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:05:51.895 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:05:51.895 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "started", 2023-06-03T20:05:51.895 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:04:23.685645+0000", 2023-06-03T20:05:51.895 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.026422982000000001 2023-06-03T20:05:51.896 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:05:51.896 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:05:51.896 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "commit_sent", 2023-06-03T20:05:51.896 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:04:23.712067+0000", 2023-06-03T20:05:51.896 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 4.3825000000000001e-05 2023-06-03T20:05:51.896 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:05:51.896 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:05:51.897 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "done", 2023-06-03T20:05:51.897 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:04:23.712111+0000", 2023-06-03T20:05:51.897 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.026576459 2023-06-03T20:05:51.897 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:05:51.897 INFO:teuthology.orchestra.run.smithi164.stdout: ] 2023-06-03T20:05:51.897 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:05:51.897 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:05:51.898 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:05:51.898 INFO:teuthology.orchestra.run.smithi164.stdout: "description": "osd_repop(client.45713.0:208095 9.0 e144/127 9:03b2b719:::benchmark_data_smithi089_115580_object97961:head v 144'12899, mlcod=144'12896)", 2023-06-03T20:05:51.898 INFO:teuthology.orchestra.run.smithi164.stdout: "initiated_at": "2023-06-03T20:04:27.161703+0000", 2023-06-03T20:05:51.898 INFO:teuthology.orchestra.run.smithi164.stdout: "age": 84.573459876000001, 2023-06-03T20:05:51.898 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.028249982, 2023-06-03T20:05:51.898 INFO:teuthology.orchestra.run.smithi164.stdout: "type_data": { 2023-06-03T20:05:51.898 INFO:teuthology.orchestra.run.smithi164.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-03T20:05:51.899 INFO:teuthology.orchestra.run.smithi164.stdout: "events": [ 2023-06-03T20:05:51.899 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:05:51.899 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "initiated", 2023-06-03T20:05:51.899 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:04:27.161703+0000", 2023-06-03T20:05:51.899 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0 2023-06-03T20:05:51.899 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:05:51.899 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:05:51.900 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "throttled", 2023-06-03T20:05:51.900 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:04:27.161703+0000", 2023-06-03T20:05:51.900 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 4294967295.999999 2023-06-03T20:05:51.900 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:05:51.900 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:05:51.900 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "header_read", 2023-06-03T20:05:51.900 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:04:27.161702+0000", 2023-06-03T20:05:51.901 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 1.2189e-05 2023-06-03T20:05:51.901 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:05:51.901 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:05:51.901 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "all_read", 2023-06-03T20:05:51.901 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:04:27.161715+0000", 2023-06-03T20:05:51.901 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 3.2530000000000002e-05 2023-06-03T20:05:51.901 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:05:51.902 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:05:51.902 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "dispatched", 2023-06-03T20:05:51.902 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:04:27.161747+0000", 2023-06-03T20:05:51.902 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 6.7479999999999996e-06 2023-06-03T20:05:51.902 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:05:51.902 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:05:51.902 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "queued_for_pg", 2023-06-03T20:05:51.903 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:04:27.161754+0000", 2023-06-03T20:05:51.903 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.000119467 2023-06-03T20:05:51.903 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:05:51.903 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:05:51.903 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "reached_pg", 2023-06-03T20:05:51.903 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:04:27.161873+0000", 2023-06-03T20:05:51.903 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 2.0696e-05 2023-06-03T20:05:51.904 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:05:51.904 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:05:51.904 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "started", 2023-06-03T20:05:51.904 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:04:27.161894+0000", 2023-06-03T20:05:51.904 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.028013461999999999 2023-06-03T20:05:51.904 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:05:51.905 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:05:51.905 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "commit_sent", 2023-06-03T20:05:51.905 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:04:27.189907+0000", 2023-06-03T20:05:51.905 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 4.5904e-05 2023-06-03T20:05:51.905 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:05:51.905 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:05:51.905 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "done", 2023-06-03T20:05:51.905 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:04:27.189953+0000", 2023-06-03T20:05:51.906 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.028249982 2023-06-03T20:05:51.906 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:05:51.906 INFO:teuthology.orchestra.run.smithi164.stdout: ] 2023-06-03T20:05:51.906 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:05:51.906 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:05:51.906 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:05:51.906 INFO:teuthology.orchestra.run.smithi164.stdout: "description": "osd_repop(client.45713.0:211298 9.b e144/127 9:de87380d:::benchmark_data_smithi089_115580_object101164:head v 144'13452, mlcod=144'13450)", 2023-06-03T20:05:51.907 INFO:teuthology.orchestra.run.smithi164.stdout: "initiated_at": "2023-06-03T20:04:31.602490+0000", 2023-06-03T20:05:51.907 INFO:teuthology.orchestra.run.smithi164.stdout: "age": 80.132673304999997, 2023-06-03T20:05:51.907 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.038480831, 2023-06-03T20:05:51.907 INFO:teuthology.orchestra.run.smithi164.stdout: "type_data": { 2023-06-03T20:05:51.907 INFO:teuthology.orchestra.run.smithi164.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-03T20:05:51.907 INFO:teuthology.orchestra.run.smithi164.stdout: "events": [ 2023-06-03T20:05:51.907 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:05:51.908 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "initiated", 2023-06-03T20:05:51.908 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:04:31.602490+0000", 2023-06-03T20:05:51.908 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0 2023-06-03T20:05:51.908 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:05:51.908 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:05:51.908 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "throttled", 2023-06-03T20:05:51.908 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:04:31.602490+0000", 2023-06-03T20:05:51.909 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 4294967295.999999 2023-06-03T20:05:51.909 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:05:51.909 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:05:51.909 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "header_read", 2023-06-03T20:05:51.909 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:04:31.602489+0000", 2023-06-03T20:05:51.909 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 9.9969999999999993e-06 2023-06-03T20:05:51.909 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:05:51.909 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:05:51.910 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "all_read", 2023-06-03T20:05:51.910 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:04:31.602499+0000", 2023-06-03T20:05:51.910 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 2.0874999999999999e-05 2023-06-03T20:05:51.910 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:05:51.910 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:05:51.910 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "dispatched", 2023-06-03T20:05:51.911 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:04:31.602520+0000", 2023-06-03T20:05:51.911 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 8.3890000000000005e-06 2023-06-03T20:05:51.911 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:05:51.911 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:05:51.911 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "queued_for_pg", 2023-06-03T20:05:51.911 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:04:31.602528+0000", 2023-06-03T20:05:51.911 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.000118274 2023-06-03T20:05:51.911 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:05:51.912 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:05:51.912 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "reached_pg", 2023-06-03T20:05:51.912 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:04:31.602646+0000", 2023-06-03T20:05:51.912 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 2.2311999999999999e-05 2023-06-03T20:05:51.912 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:05:51.912 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:05:51.912 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "started", 2023-06-03T20:05:51.913 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:04:31.602669+0000", 2023-06-03T20:05:51.913 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.038261337999999999 2023-06-03T20:05:51.913 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:05:51.913 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:05:51.913 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "commit_sent", 2023-06-03T20:05:51.913 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:04:31.640930+0000", 2023-06-03T20:05:51.913 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 4.0608999999999997e-05 2023-06-03T20:05:51.914 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:05:51.914 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:05:51.914 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "done", 2023-06-03T20:05:51.914 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:04:31.640971+0000", 2023-06-03T20:05:51.914 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.038480831 2023-06-03T20:05:51.914 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:05:51.914 INFO:teuthology.orchestra.run.smithi164.stdout: ] 2023-06-03T20:05:51.915 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:05:51.915 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:05:51.915 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:05:51.915 INFO:teuthology.orchestra.run.smithi164.stdout: "description": "osd_repop(client.45713.0:211301 9.b e144/127 9:dc24a46a:::benchmark_data_smithi089_115580_object101167:head v 144'13453, mlcod=144'13450)", 2023-06-03T20:05:51.915 INFO:teuthology.orchestra.run.smithi164.stdout: "initiated_at": "2023-06-03T20:04:31.607967+0000", 2023-06-03T20:05:51.915 INFO:teuthology.orchestra.run.smithi164.stdout: "age": 80.127196174000005, 2023-06-03T20:05:51.915 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.033025387000000003, 2023-06-03T20:05:51.916 INFO:teuthology.orchestra.run.smithi164.stdout: "type_data": { 2023-06-03T20:05:51.916 INFO:teuthology.orchestra.run.smithi164.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-03T20:05:51.916 INFO:teuthology.orchestra.run.smithi164.stdout: "events": [ 2023-06-03T20:05:51.916 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:05:51.916 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "initiated", 2023-06-03T20:05:51.916 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:04:31.607967+0000", 2023-06-03T20:05:51.916 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0 2023-06-03T20:05:51.917 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:05:51.917 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:05:51.917 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "throttled", 2023-06-03T20:05:51.917 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:04:31.607967+0000", 2023-06-03T20:05:51.917 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 4294967295.999999 2023-06-03T20:05:51.917 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:05:51.917 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:05:51.918 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "header_read", 2023-06-03T20:05:51.918 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:04:31.607966+0000", 2023-06-03T20:05:51.918 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 1.1785e-05 2023-06-03T20:05:51.918 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:05:51.918 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:05:51.918 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "all_read", 2023-06-03T20:05:51.918 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:04:31.607978+0000", 2023-06-03T20:05:51.918 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 1.3956e-05 2023-06-03T20:05:51.919 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:05:51.919 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:05:51.919 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "dispatched", 2023-06-03T20:05:51.919 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:04:31.607992+0000", 2023-06-03T20:05:51.919 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 6.7850000000000003e-06 2023-06-03T20:05:51.919 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:05:51.919 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:05:51.920 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "queued_for_pg", 2023-06-03T20:05:51.920 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:04:31.607999+0000", 2023-06-03T20:05:51.920 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 5.1140999999999997e-05 2023-06-03T20:05:51.920 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:05:51.920 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:05:51.920 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "reached_pg", 2023-06-03T20:05:51.920 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:04:31.608050+0000", 2023-06-03T20:05:51.921 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 3.1835999999999998e-05 2023-06-03T20:05:51.921 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:05:51.921 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:05:51.921 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "started", 2023-06-03T20:05:51.921 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:04:31.608082+0000", 2023-06-03T20:05:51.921 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.032893605999999999 2023-06-03T20:05:51.922 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:05:51.922 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:05:51.922 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "commit_sent", 2023-06-03T20:05:51.922 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:04:31.640975+0000", 2023-06-03T20:05:51.922 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 1.7098000000000001e-05 2023-06-03T20:05:51.922 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:05:51.922 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:05:51.922 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "done", 2023-06-03T20:05:51.923 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:04:31.640992+0000", 2023-06-03T20:05:51.923 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.033025387000000003 2023-06-03T20:05:51.923 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:05:51.923 INFO:teuthology.orchestra.run.smithi164.stdout: ] 2023-06-03T20:05:51.923 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:05:51.923 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:05:51.923 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:05:51.924 INFO:teuthology.orchestra.run.smithi164.stdout: "description": "osd_repop(client.45713.0:211303 9.b e144/127 9:d9612a3f:::benchmark_data_smithi089_115580_object101169:head v 144'13454, mlcod=144'13450)", 2023-06-03T20:05:51.924 INFO:teuthology.orchestra.run.smithi164.stdout: "initiated_at": "2023-06-03T20:04:31.613270+0000", 2023-06-03T20:05:51.924 INFO:teuthology.orchestra.run.smithi164.stdout: "age": 80.121892825000003, 2023-06-03T20:05:51.924 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.033111440999999998, 2023-06-03T20:05:51.924 INFO:teuthology.orchestra.run.smithi164.stdout: "type_data": { 2023-06-03T20:05:51.924 INFO:teuthology.orchestra.run.smithi164.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-03T20:05:51.924 INFO:teuthology.orchestra.run.smithi164.stdout: "events": [ 2023-06-03T20:05:51.925 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:05:51.925 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "initiated", 2023-06-03T20:05:51.925 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:04:31.613270+0000", 2023-06-03T20:05:51.925 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0 2023-06-03T20:05:51.925 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:05:51.925 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:05:51.925 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "throttled", 2023-06-03T20:05:51.926 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:04:31.613270+0000", 2023-06-03T20:05:51.926 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 4294967295.999999 2023-06-03T20:05:51.926 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:05:51.926 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:05:51.926 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "header_read", 2023-06-03T20:05:51.926 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:04:31.613269+0000", 2023-06-03T20:05:51.926 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 1.3961000000000001e-05 2023-06-03T20:05:51.927 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:05:51.927 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:05:51.927 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "all_read", 2023-06-03T20:05:51.927 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:04:31.613283+0000", 2023-06-03T20:05:51.927 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 1.2268e-05 2023-06-03T20:05:51.927 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:05:51.927 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:05:51.928 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "dispatched", 2023-06-03T20:05:51.928 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:04:31.613296+0000", 2023-06-03T20:05:51.928 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 6.7839999999999998e-06 2023-06-03T20:05:51.928 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:05:51.928 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:05:51.928 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "queued_for_pg", 2023-06-03T20:05:51.928 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:04:31.613302+0000", 2023-06-03T20:05:51.929 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.00084380200000000005 2023-06-03T20:05:51.929 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:05:51.929 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:05:51.929 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "reached_pg", 2023-06-03T20:05:51.929 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:04:31.614146+0000", 2023-06-03T20:05:51.929 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 4.2327000000000003e-05 2023-06-03T20:05:51.929 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:05:51.930 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:05:51.930 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "started", 2023-06-03T20:05:51.930 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:04:31.614189+0000", 2023-06-03T20:05:51.930 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.032151297000000002 2023-06-03T20:05:51.930 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:05:51.930 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:05:51.931 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "commit_sent", 2023-06-03T20:05:51.931 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:04:31.646340+0000", 2023-06-03T20:05:51.931 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 4.2036000000000002e-05 2023-06-03T20:05:51.931 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:05:51.931 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:05:51.931 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "done", 2023-06-03T20:05:51.932 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:04:31.646382+0000", 2023-06-03T20:05:51.932 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.033111440999999998 2023-06-03T20:05:51.932 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:05:51.932 INFO:teuthology.orchestra.run.smithi164.stdout: ] 2023-06-03T20:05:51.932 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:05:51.932 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:05:51.932 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:05:51.933 INFO:teuthology.orchestra.run.smithi164.stdout: "description": "osd_repop(client.45713.0:211315 9.b e144/127 9:d7ff0340:::benchmark_data_smithi089_115580_object101181:head v 144'13455, mlcod=144'13450)", 2023-06-03T20:05:51.933 INFO:teuthology.orchestra.run.smithi164.stdout: "initiated_at": "2023-06-03T20:04:31.618849+0000", 2023-06-03T20:05:51.933 INFO:teuthology.orchestra.run.smithi164.stdout: "age": 80.116314102999993, 2023-06-03T20:05:51.933 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.027554359, 2023-06-03T20:05:51.933 INFO:teuthology.orchestra.run.smithi164.stdout: "type_data": { 2023-06-03T20:05:51.933 INFO:teuthology.orchestra.run.smithi164.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-03T20:05:51.933 INFO:teuthology.orchestra.run.smithi164.stdout: "events": [ 2023-06-03T20:05:51.934 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:05:51.934 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "initiated", 2023-06-03T20:05:51.934 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:04:31.618849+0000", 2023-06-03T20:05:51.934 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0 2023-06-03T20:05:51.934 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:05:51.934 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:05:51.934 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "throttled", 2023-06-03T20:05:51.935 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:04:31.618849+0000", 2023-06-03T20:05:51.935 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 4294967295.999999 2023-06-03T20:05:51.935 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:05:51.935 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:05:51.935 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "header_read", 2023-06-03T20:05:51.935 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:04:31.618848+0000", 2023-06-03T20:05:51.935 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 1.2632999999999999e-05 2023-06-03T20:05:51.936 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:05:51.936 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:05:51.936 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "all_read", 2023-06-03T20:05:51.936 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:04:31.618861+0000", 2023-06-03T20:05:51.936 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 1.4579000000000001e-05 2023-06-03T20:05:51.936 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:05:51.936 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:05:51.937 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "dispatched", 2023-06-03T20:05:51.937 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:04:31.618875+0000", 2023-06-03T20:05:51.937 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 6.489e-06 2023-06-03T20:05:51.937 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:05:51.937 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:05:51.937 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "queued_for_pg", 2023-06-03T20:05:51.938 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:04:31.618882+0000", 2023-06-03T20:05:51.938 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.011554368000000001 2023-06-03T20:05:51.938 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:05:51.938 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:05:51.938 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "reached_pg", 2023-06-03T20:05:51.938 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:04:31.630436+0000", 2023-06-03T20:05:51.939 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 2.4635999999999999e-05 2023-06-03T20:05:51.939 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:05:51.939 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:05:51.939 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "started", 2023-06-03T20:05:51.939 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:04:31.630461+0000", 2023-06-03T20:05:51.939 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.015925182 2023-06-03T20:05:51.939 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:05:51.940 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:05:51.940 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "commit_sent", 2023-06-03T20:05:51.940 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:04:31.646386+0000", 2023-06-03T20:05:51.940 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 1.7638999999999998e-05 2023-06-03T20:05:51.940 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:05:51.940 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:05:51.941 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "done", 2023-06-03T20:05:51.941 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:04:31.646403+0000", 2023-06-03T20:05:51.941 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.027554359 2023-06-03T20:05:51.941 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:05:51.941 INFO:teuthology.orchestra.run.smithi164.stdout: ] 2023-06-03T20:05:51.941 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:05:51.941 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:05:51.941 INFO:teuthology.orchestra.run.smithi164.stdout: ] 2023-06-03T20:05:51.942 INFO:teuthology.orchestra.run.smithi164.stdout:} 2023-06-03T20:05:51.942 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:05:51.971 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:05:51.974 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:05:51.974 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:05:52.075 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:05:52.109 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:05:52.112 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:05:52.112 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:05:52.213 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:05:52.222 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:05:52 smithi089 bash[78452]: audit 1685822752.105894 mon.c (mon.2) 430 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T20:05:52.222 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:05:52 smithi089 bash[75068]: audit 1685822752.105894 mon.c (mon.2) 430 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T20:05:52.242 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:05:52.243 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:05:52.244 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:05:52.333 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:05:52 smithi164 bash[59462]: audit 1685822752.105894 mon.c (mon.2) 430 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T20:05:52.345 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:05:52.379 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:05:52.382 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:05:52.382 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:05:52.483 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:05:52.518 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:05:52.520 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:05:52.521 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:05:52.594 DEBUG:teuthology.orchestra.run.smithi164:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph daemon osd.7 dump_ops_in_flight 2023-06-03T20:05:52.623 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:05:52.671 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:05:52.673 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:05:52.674 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:05:52.775 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:05:52.802 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:05:52.804 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:05:52.805 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:05:52.905 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:05:52.933 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:05:52.936 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:05:52.936 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:05:53.038 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:05:53.069 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:05:53.071 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:05:53.072 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:05:53.173 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:05:53.203 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:05:53.205 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:05:53.206 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:05:53.307 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:05:53.338 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:05:53.342 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:05:53.342 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:05:53.443 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:05:53.476 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:05:53.478 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:05:53.479 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:05:53.580 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:05:53.612 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:05:53.614 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:05:53.614 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:05:53.715 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:05:53.727 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:05:53 smithi089 bash[78452]: cluster 1685822752.5403526 mgr.y (mgr.34104) 891 : cluster 0 pgmap v822: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-06-03T20:05:53.728 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:05:53 smithi089 bash[75068]: cluster 1685822752.5403526 mgr.y (mgr.34104) 891 : cluster 0 pgmap v822: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-06-03T20:05:53.747 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:05:53.749 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:05:53.750 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:05:53.851 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:05:53.882 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:05:53.884 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:05:53.884 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:05:53.985 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:05:54.011 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:05:54.013 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:05:54.013 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:05:54.083 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:05:53 smithi164 bash[59462]: cluster 1685822752.5403526 mgr.y (mgr.34104) 891 : cluster 0 pgmap v822: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-06-03T20:05:54.114 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:05:54.149 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:05:54.151 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:05:54.152 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:05:54.253 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:05:54.288 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:05:54.291 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:05:54.292 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:05:54.393 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:05:54.423 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:05:54.426 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:05:54.426 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:05:54.527 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:05:54.553 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:05:54.555 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:05:54.555 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:05:54.656 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:05:54.694 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:05:54.698 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:05:54.698 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:05:54.733 INFO:teuthology.orchestra.run.smithi089.stderr:pool 'unique_pool_5' created 2023-06-03T20:05:54.800 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:05:54.829 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:05:54.832 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:05:54.833 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:05:54.935 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:05:54.947 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:05:54 smithi164 bash[59462]: audit 1685822754.2141466 mon.c (mon.2) 431 : audit 1 from='client.? 172.21.15.89:0/1033431660' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_5", "pg_num": 16}]: dispatch 2023-06-03T20:05:54.947 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:05:54 smithi164 bash[59462]: audit 1685822754.214754 mon.a (mon.0) 798 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_5", "pg_num": 16}]: dispatch 2023-06-03T20:05:54.968 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:05:54.971 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:05:54.972 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:05:55.028 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:05:54 smithi089 bash[78452]: audit 1685822754.2141466 mon.c (mon.2) 431 : audit 1 from='client.? 172.21.15.89:0/1033431660' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_5", "pg_num": 16}]: dispatch 2023-06-03T20:05:55.029 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:05:54 smithi089 bash[78452]: audit 1685822754.214754 mon.a (mon.0) 798 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_5", "pg_num": 16}]: dispatch 2023-06-03T20:05:55.029 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:05:54 smithi089 bash[75068]: audit 1685822754.2141466 mon.c (mon.2) 431 : audit 1 from='client.? 172.21.15.89:0/1033431660' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_5", "pg_num": 16}]: dispatch 2023-06-03T20:05:55.029 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:05:54 smithi089 bash[75068]: audit 1685822754.214754 mon.a (mon.0) 798 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_5", "pg_num": 16}]: dispatch 2023-06-03T20:05:55.073 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:05:55.115 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:05:55.117 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:05:55.118 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:05:55.219 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:05:55.255 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:05:55.258 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:05:55.258 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:05:55.359 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:05:55.392 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:05:55.396 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:05:55.397 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:05:55.498 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:05:55.537 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:05:55.541 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:05:55.541 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:05:55.642 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:05:55.677 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:05:55.680 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:05:55.681 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:05:55.782 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:05:55.819 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:05:55.822 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:05:55.822 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:05:55.923 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:05:55.935 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:05:55 smithi164 bash[59462]: cluster 1685822754.5409276 mgr.y (mgr.34104) 892 : cluster 0 pgmap v823: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-06-03T20:05:55.935 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:05:55 smithi164 bash[59462]: cluster 1685822754.7133076 osd.6 (osd.6) 80 : cluster 3 failed to encode map e157 with expected crc 2023-06-03T20:05:55.936 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:05:55 smithi164 bash[59462]: cluster 1685822754.714906 osd.5 (osd.5) 86 : cluster 3 failed to encode map e157 with expected crc 2023-06-03T20:05:55.936 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:05:55 smithi164 bash[59462]: cluster 1685822754.716659 osd.7 (osd.7) 84 : cluster 3 failed to encode map e157 with expected crc 2023-06-03T20:05:55.936 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:05:55 smithi164 bash[59462]: cluster 1685822754.716748 osd.6 (osd.6) 81 : cluster 3 failed to encode map e157 with expected crc 2023-06-03T20:05:55.936 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:05:55 smithi164 bash[59462]: audit 1685822754.7215807 mon.a (mon.0) 799 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_5", "pg_num": 16}]': finished 2023-06-03T20:05:55.937 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:05:55 smithi164 bash[59462]: cluster 1685822754.721722 mon.a (mon.0) 800 : cluster 0 osdmap e157: 8 total, 8 up, 8 in 2023-06-03T20:05:55.937 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:05:55 smithi164 bash[59462]: cluster 1685822755.7187972 mon.a (mon.0) 801 : cluster 0 osdmap e158: 8 total, 8 up, 8 in 2023-06-03T20:05:55.956 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:05:55.959 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:05:55.959 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:05:56.052 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:05:55 smithi089 bash[75068]: cluster 1685822754.5409276 mgr.y (mgr.34104) 892 : cluster 0 pgmap v823: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-06-03T20:05:56.052 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:05:55 smithi089 bash[75068]: cluster 1685822754.7133076 osd.6 (osd.6) 80 : cluster 3 failed to encode map e157 with expected crc 2023-06-03T20:05:56.052 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:05:55 smithi089 bash[75068]: cluster 1685822754.714906 osd.5 (osd.5) 86 : cluster 3 failed to encode map e157 with expected crc 2023-06-03T20:05:56.053 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:05:55 smithi089 bash[75068]: cluster 1685822754.716659 osd.7 (osd.7) 84 : cluster 3 failed to encode map e157 with expected crc 2023-06-03T20:05:56.053 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:05:55 smithi089 bash[75068]: cluster 1685822754.716748 osd.6 (osd.6) 81 : cluster 3 failed to encode map e157 with expected crc 2023-06-03T20:05:56.053 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:05:55 smithi089 bash[75068]: audit 1685822754.7215807 mon.a (mon.0) 799 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_5", "pg_num": 16}]': finished 2023-06-03T20:05:56.053 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:05:55 smithi089 bash[75068]: cluster 1685822754.721722 mon.a (mon.0) 800 : cluster 0 osdmap e157: 8 total, 8 up, 8 in 2023-06-03T20:05:56.053 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:05:55 smithi089 bash[75068]: cluster 1685822755.7187972 mon.a (mon.0) 801 : cluster 0 osdmap e158: 8 total, 8 up, 8 in 2023-06-03T20:05:56.053 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:05:55 smithi089 bash[78452]: cluster 1685822754.5409276 mgr.y (mgr.34104) 892 : cluster 0 pgmap v823: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-06-03T20:05:56.054 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:05:55 smithi089 bash[78452]: cluster 1685822754.7133076 osd.6 (osd.6) 80 : cluster 3 failed to encode map e157 with expected crc 2023-06-03T20:05:56.054 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:05:55 smithi089 bash[78452]: cluster 1685822754.714906 osd.5 (osd.5) 86 : cluster 3 failed to encode map e157 with expected crc 2023-06-03T20:05:56.054 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:05:55 smithi089 bash[78452]: cluster 1685822754.716659 osd.7 (osd.7) 84 : cluster 3 failed to encode map e157 with expected crc 2023-06-03T20:05:56.054 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:05:55 smithi089 bash[78452]: cluster 1685822754.716748 osd.6 (osd.6) 81 : cluster 3 failed to encode map e157 with expected crc 2023-06-03T20:05:56.054 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:05:55 smithi089 bash[78452]: audit 1685822754.7215807 mon.a (mon.0) 799 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_5", "pg_num": 16}]': finished 2023-06-03T20:05:56.054 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:05:55 smithi089 bash[78452]: cluster 1685822754.721722 mon.a (mon.0) 800 : cluster 0 osdmap e157: 8 total, 8 up, 8 in 2023-06-03T20:05:56.054 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:05:55 smithi089 bash[78452]: cluster 1685822755.7187972 mon.a (mon.0) 801 : cluster 0 osdmap e158: 8 total, 8 up, 8 in 2023-06-03T20:05:56.060 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:05:56.090 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:05:56.092 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:05:56.093 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:05:56.195 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:05:56.223 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:05:56.226 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:05:56.227 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:05:56.328 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:05:56.359 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:05:56.362 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:05:56.363 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:05:56.464 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:05:56.501 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:05:56.504 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:05:56.505 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:05:56.529 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 20:05:56 smithi089 bash[71163]: ::ffff:172.21.15.164 - - [03/Jun/2023:20:05:56] "GET /metrics HTTP/1.1" 200 225911 "" "Prometheus/2.33.4" 2023-06-03T20:05:56.605 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:05:56.645 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:05:56.648 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:05:56.649 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:05:56.750 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:05:56.779 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:05:56.781 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:05:56.782 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:05:56.883 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:05:56.893 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:05:56 smithi164 bash[59462]: cluster 1685822755.7147377 osd.7 (osd.7) 85 : cluster 3 failed to encode map e158 with expected crc 2023-06-03T20:05:56.893 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:05:56 smithi164 bash[59462]: cluster 1685822755.7175336 osd.6 (osd.6) 82 : cluster 3 failed to encode map e158 with expected crc 2023-06-03T20:05:56.893 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:05:56 smithi164 bash[59462]: cluster 1685822755.7190065 osd.7 (osd.7) 86 : cluster 3 failed to encode map e158 with expected crc 2023-06-03T20:05:56.894 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:05:56 smithi164 bash[59462]: cluster 1685822755.7213895 osd.5 (osd.5) 87 : cluster 3 failed to encode map e158 with expected crc 2023-06-03T20:05:56.894 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:05:56 smithi164 bash[59462]: cluster 1685822756.7221222 mon.a (mon.0) 802 : cluster 0 osdmap e159: 8 total, 8 up, 8 in 2023-06-03T20:05:56.915 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:05:56.918 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:05:56.918 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:05:57.019 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:05:57.028 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:05:56 smithi089 bash[75068]: cluster 1685822755.7147377 osd.7 (osd.7) 85 : cluster 3 failed to encode map e158 with expected crc 2023-06-03T20:05:57.028 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:05:56 smithi089 bash[75068]: cluster 1685822755.7175336 osd.6 (osd.6) 82 : cluster 3 failed to encode map e158 with expected crc 2023-06-03T20:05:57.028 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:05:56 smithi089 bash[75068]: cluster 1685822755.7190065 osd.7 (osd.7) 86 : cluster 3 failed to encode map e158 with expected crc 2023-06-03T20:05:57.029 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:05:56 smithi089 bash[75068]: cluster 1685822755.7213895 osd.5 (osd.5) 87 : cluster 3 failed to encode map e158 with expected crc 2023-06-03T20:05:57.029 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:05:56 smithi089 bash[75068]: cluster 1685822756.7221222 mon.a (mon.0) 802 : cluster 0 osdmap e159: 8 total, 8 up, 8 in 2023-06-03T20:05:57.029 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:05:56 smithi089 bash[78452]: cluster 1685822755.7147377 osd.7 (osd.7) 85 : cluster 3 failed to encode map e158 with expected crc 2023-06-03T20:05:57.029 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:05:56 smithi089 bash[78452]: cluster 1685822755.7175336 osd.6 (osd.6) 82 : cluster 3 failed to encode map e158 with expected crc 2023-06-03T20:05:57.029 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:05:56 smithi089 bash[78452]: cluster 1685822755.7190065 osd.7 (osd.7) 86 : cluster 3 failed to encode map e158 with expected crc 2023-06-03T20:05:57.029 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:05:56 smithi089 bash[78452]: cluster 1685822755.7213895 osd.5 (osd.5) 87 : cluster 3 failed to encode map e158 with expected crc 2023-06-03T20:05:57.030 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:05:56 smithi089 bash[78452]: cluster 1685822756.7221222 mon.a (mon.0) 802 : cluster 0 osdmap e159: 8 total, 8 up, 8 in 2023-06-03T20:05:57.048 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:05:57.050 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:05:57.050 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:05:57.151 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:05:57.193 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:05:57.196 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:05:57.196 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:05:57.220 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph osd pool application enable unique_pool_5 rados --yes-i-really-mean-it || true 2023-06-03T20:05:57.297 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:05:57.336 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:05:57.339 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:05:57.340 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:05:57.406 INFO:teuthology.orchestra.run.smithi164.stderr:Inferring config /var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/mon.b/config 2023-06-03T20:05:57.441 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:05:57.478 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:05:57.481 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:05:57.482 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:05:57.583 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:05:57.620 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:05:57.625 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:05:57.626 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:05:57.727 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:05:57.757 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:05:57.760 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:05:57.761 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:05:57.778 INFO:teuthology.orchestra.run.smithi089.stdout:osd.0: {} 2023-06-03T20:05:57.830 INFO:teuthology.orchestra.run.smithi089.stdout:osd.1: {} 2023-06-03T20:05:57.857 INFO:teuthology.orchestra.run.smithi089.stdout:osd.2: {} 2023-06-03T20:05:57.862 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:05:57.869 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:05:57 smithi089 bash[78452]: cluster 1685822756.541558 mgr.y (mgr.34104) 893 : cluster 0 pgmap v826: 89 pgs: 4 creating+peering, 12 unknown, 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-06-03T20:05:57.869 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:05:57 smithi089 bash[78452]: cluster 1685822757.2398982 osd.6 (osd.6) 83 : cluster 3 failed to encode map e159 with expected crc 2023-06-03T20:05:57.869 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:05:57 smithi089 bash[78452]: cluster 1685822757.2398982 osd.5 (osd.5) 88 : cluster 3 failed to encode map e159 with expected crc 2023-06-03T20:05:57.869 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:05:57 smithi089 bash[75068]: cluster 1685822756.541558 mgr.y (mgr.34104) 893 : cluster 0 pgmap v826: 89 pgs: 4 creating+peering, 12 unknown, 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-06-03T20:05:57.870 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:05:57 smithi089 bash[75068]: cluster 1685822757.2398982 osd.6 (osd.6) 83 : cluster 3 failed to encode map e159 with expected crc 2023-06-03T20:05:57.870 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:05:57 smithi089 bash[75068]: cluster 1685822757.2398982 osd.5 (osd.5) 88 : cluster 3 failed to encode map e159 with expected crc 2023-06-03T20:05:57.880 INFO:teuthology.orchestra.run.smithi089.stdout:osd.3: {} 2023-06-03T20:05:57.885 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:05:57.887 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:05:57.888 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:05:57.908 INFO:teuthology.orchestra.run.smithi089.stdout:osd.4: {} 2023-06-03T20:05:57.929 INFO:teuthology.orchestra.run.smithi089.stdout:osd.5: {} 2023-06-03T20:05:57.954 INFO:teuthology.orchestra.run.smithi089.stdout:osd.6: {} 2023-06-03T20:05:57.973 INFO:teuthology.orchestra.run.smithi089.stdout:osd.7: {} 2023-06-03T20:05:57.975 INFO:teuthology.orchestra.run.smithi089.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-06-03T20:05:57.976 INFO:teuthology.orchestra.run.smithi089.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-06-03T20:05:57.976 INFO:teuthology.orchestra.run.smithi089.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-06-03T20:05:57.976 INFO:teuthology.orchestra.run.smithi089.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-06-03T20:05:57.976 INFO:teuthology.orchestra.run.smithi089.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-06-03T20:05:57.976 INFO:teuthology.orchestra.run.smithi089.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-06-03T20:05:57.976 INFO:teuthology.orchestra.run.smithi089.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-06-03T20:05:57.976 INFO:teuthology.orchestra.run.smithi089.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-06-03T20:05:57.989 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:05:58.016 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:05:58.018 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:05:58.019 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:05:58.083 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:05:57 smithi164 bash[59462]: cluster 1685822756.541558 mgr.y (mgr.34104) 893 : cluster 0 pgmap v826: 89 pgs: 4 creating+peering, 12 unknown, 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-06-03T20:05:58.084 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:05:57 smithi164 bash[59462]: cluster 1685822757.2398982 osd.6 (osd.6) 83 : cluster 3 failed to encode map e159 with expected crc 2023-06-03T20:05:58.084 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:05:57 smithi164 bash[59462]: cluster 1685822757.2398982 osd.5 (osd.5) 88 : cluster 3 failed to encode map e159 with expected crc 2023-06-03T20:05:58.119 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:05:58.157 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:05:58.160 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:05:58.161 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:05:58.262 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:05:58.297 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:05:58.300 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:05:58.300 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:05:58.401 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:05:58.430 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:05:58.432 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:05:58.433 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:05:58.534 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:05:58.560 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:05:58.562 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:05:58.563 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:05:58.664 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:05:58.697 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:05:58.700 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:05:58.700 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:05:58.801 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:05:58.808 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:05:58 smithi089 bash[75068]: cluster 1685822757.0484164 osd.7 (osd.7) 87 : cluster 3 failed to encode map e159 with expected crc 2023-06-03T20:05:58.809 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:05:58 smithi089 bash[75068]: cluster 1685822757.0513537 osd.7 (osd.7) 88 : cluster 3 failed to encode map e159 with expected crc 2023-06-03T20:05:58.809 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:05:58 smithi089 bash[75068]: audit 1685822757.7486691 mon.a (mon.0) 803 : audit 0 from='client.? 172.21.15.89:0/949010887' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-03T20:05:58.809 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:05:58 smithi089 bash[78452]: cluster 1685822757.0484164 osd.7 (osd.7) 87 : cluster 3 failed to encode map e159 with expected crc 2023-06-03T20:05:58.809 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:05:58 smithi089 bash[78452]: cluster 1685822757.0513537 osd.7 (osd.7) 88 : cluster 3 failed to encode map e159 with expected crc 2023-06-03T20:05:58.809 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:05:58 smithi089 bash[78452]: audit 1685822757.7486691 mon.a (mon.0) 803 : audit 0 from='client.? 172.21.15.89:0/949010887' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-03T20:05:58.828 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:05:58.830 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:05:58.830 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:05:58.931 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:05:58.935 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:05:58 smithi164 bash[59462]: cluster 1685822757.0484164 osd.7 (osd.7) 87 : cluster 3 failed to encode map e159 with expected crc 2023-06-03T20:05:58.936 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:05:58 smithi164 bash[59462]: cluster 1685822757.0513537 osd.7 (osd.7) 88 : cluster 3 failed to encode map e159 with expected crc 2023-06-03T20:05:58.936 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:05:58 smithi164 bash[59462]: audit 1685822757.7486691 mon.a (mon.0) 803 : audit 0 from='client.? 172.21.15.89:0/949010887' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-03T20:05:58.971 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:05:58.973 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:05:58.973 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:05:59.074 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:05:59.101 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:05:59.103 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:05:59.104 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:05:59.205 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:05:59.236 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:05:59.238 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:05:59.238 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:05:59.339 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:05:59.363 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:05:59.367 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:05:59.367 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:05:59.468 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:05:59.504 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:05:59.507 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:05:59.508 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:05:59.610 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:05:59.638 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:05:59.640 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:05:59.641 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:05:59.742 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:05:59.767 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:05:59.770 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:05:59.771 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:05:59.872 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:05:59.909 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:05:59.913 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:05:59.914 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:06:00.015 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:06:00.028 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:05:59 smithi089 bash[78452]: cluster 1685822758.5421565 mgr.y (mgr.34104) 894 : cluster 0 pgmap v828: 89 pgs: 4 creating+peering, 1 unknown, 84 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-06-03T20:06:00.029 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:05:59 smithi089 bash[78452]: audit 1685822758.8305655 mon.a (mon.0) 804 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:06:00.029 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:05:59 smithi089 bash[78452]: audit 1685822758.8387468 mon.a (mon.0) 805 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:06:00.029 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:05:59 smithi089 bash[78452]: audit 1685822759.2583628 mon.c (mon.2) 432 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-03T20:06:00.029 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:05:59 smithi089 bash[78452]: audit 1685822759.2594693 mon.c (mon.2) 433 : audit 1 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-03T20:06:00.029 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:05:59 smithi089 bash[78452]: audit 1685822759.266372 mon.a (mon.0) 806 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:06:00.029 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:05:59 smithi089 bash[78452]: audit 1685822759.7859268 mon.b (mon.1) 50 : audit 1 from='client.? 172.21.15.89:0/3722569462' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-06-03T20:06:00.029 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:05:59 smithi089 bash[78452]: audit 1685822759.7886746 mon.a (mon.0) 807 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-06-03T20:06:00.030 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:05:59 smithi089 bash[75068]: cluster 1685822758.5421565 mgr.y (mgr.34104) 894 : cluster 0 pgmap v828: 89 pgs: 4 creating+peering, 1 unknown, 84 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-06-03T20:06:00.030 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:05:59 smithi089 bash[75068]: audit 1685822758.8305655 mon.a (mon.0) 804 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:06:00.030 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:05:59 smithi089 bash[75068]: audit 1685822758.8387468 mon.a (mon.0) 805 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:06:00.030 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:05:59 smithi089 bash[75068]: audit 1685822759.2583628 mon.c (mon.2) 432 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-03T20:06:00.030 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:05:59 smithi089 bash[75068]: audit 1685822759.2594693 mon.c (mon.2) 433 : audit 1 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-03T20:06:00.030 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:05:59 smithi089 bash[75068]: audit 1685822759.266372 mon.a (mon.0) 806 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:06:00.030 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:05:59 smithi089 bash[75068]: audit 1685822759.7859268 mon.b (mon.1) 50 : audit 1 from='client.? 172.21.15.89:0/3722569462' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-06-03T20:06:00.031 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:05:59 smithi089 bash[75068]: audit 1685822759.7886746 mon.a (mon.0) 807 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-06-03T20:06:00.052 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:06:00.055 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:06:00.056 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:06:00.084 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:05:59 smithi164 bash[59462]: cluster 1685822758.5421565 mgr.y (mgr.34104) 894 : cluster 0 pgmap v828: 89 pgs: 4 creating+peering, 1 unknown, 84 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-06-03T20:06:00.084 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:05:59 smithi164 bash[59462]: audit 1685822758.8305655 mon.a (mon.0) 804 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:06:00.084 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:05:59 smithi164 bash[59462]: audit 1685822758.8387468 mon.a (mon.0) 805 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:06:00.084 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:05:59 smithi164 bash[59462]: audit 1685822759.2583628 mon.c (mon.2) 432 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-03T20:06:00.084 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:05:59 smithi164 bash[59462]: audit 1685822759.2594693 mon.c (mon.2) 433 : audit 1 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-03T20:06:00.084 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:05:59 smithi164 bash[59462]: audit 1685822759.266372 mon.a (mon.0) 806 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:06:00.085 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:05:59 smithi164 bash[59462]: audit 1685822759.7859268 mon.b (mon.1) 50 : audit 1 from='client.? 172.21.15.89:0/3722569462' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-06-03T20:06:00.085 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:05:59 smithi164 bash[59462]: audit 1685822759.7886746 mon.a (mon.0) 807 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-06-03T20:06:00.157 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:06:00.192 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:06:00.193 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:06:00.194 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:06:00.287 INFO:teuthology.orchestra.run.smithi089.stderr:noscrub is unset 2023-06-03T20:06:00.288 INFO:teuthology.orchestra.run.smithi089.stderr:set osd.7 primary-affinity to 0.531414 (8348262) 2023-06-03T20:06:00.295 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:06:00.325 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:06:00.327 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:06:00.328 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:06:00.429 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:06:00.463 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:06:00.465 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:06:00.465 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:06:00.566 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:06:00.595 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:06:00.597 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:06:00.598 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:06:00.700 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:06:00.731 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:06:00.733 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:06:00.734 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:06:00.835 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:06:00.860 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:06:00.863 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:06:00.863 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:06:00.964 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:06:00.997 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:06:01.000 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:06:01.001 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:06:01.083 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:06:00 smithi164 bash[59462]: audit 1685822759.966524 mon.c (mon.2) 434 : audit 1 from='client.? 172.21.15.89:0/3790659683' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 7, "weight": 0.5314142386440327}]: dispatch 2023-06-03T20:06:01.084 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:06:00 smithi164 bash[59462]: audit 1685822759.9672217 mon.a (mon.0) 808 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 7, "weight": 0.5314142386440327}]: dispatch 2023-06-03T20:06:01.084 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:06:00 smithi164 bash[59462]: cluster 1685822760.2633681 mon.a (mon.0) 809 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-06-03T20:06:01.084 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:06:00 smithi164 bash[59462]: cluster 1685822760.2703059 osd.7 (osd.7) 89 : cluster 3 failed to encode map e160 with expected crc 2023-06-03T20:06:01.084 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:06:00 smithi164 bash[59462]: audit 1685822760.272808 mon.a (mon.0) 810 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-06-03T20:06:01.084 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:06:00 smithi164 bash[59462]: audit 1685822760.2729714 mon.a (mon.0) 811 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 7, "weight": 0.5314142386440327}]': finished 2023-06-03T20:06:01.084 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:06:00 smithi164 bash[59462]: cluster 1685822760.2730994 mon.a (mon.0) 812 : cluster 0 osdmap e160: 8 total, 8 up, 8 in 2023-06-03T20:06:01.102 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:06:01.136 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:06:01.138 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:06:01.139 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:06:01.240 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:06:01.253 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:06:00 smithi089 bash[78452]: audit 1685822759.966524 mon.c (mon.2) 434 : audit 1 from='client.? 172.21.15.89:0/3790659683' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 7, "weight": 0.5314142386440327}]: dispatch 2023-06-03T20:06:01.253 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:06:00 smithi089 bash[78452]: audit 1685822759.9672217 mon.a (mon.0) 808 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 7, "weight": 0.5314142386440327}]: dispatch 2023-06-03T20:06:01.254 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:06:00 smithi089 bash[78452]: cluster 1685822760.2633681 mon.a (mon.0) 809 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-06-03T20:06:01.254 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:06:00 smithi089 bash[78452]: cluster 1685822760.2703059 osd.7 (osd.7) 89 : cluster 3 failed to encode map e160 with expected crc 2023-06-03T20:06:01.254 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:06:00 smithi089 bash[78452]: audit 1685822760.272808 mon.a (mon.0) 810 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-06-03T20:06:01.254 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:06:00 smithi089 bash[78452]: audit 1685822760.2729714 mon.a (mon.0) 811 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 7, "weight": 0.5314142386440327}]': finished 2023-06-03T20:06:01.254 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:06:00 smithi089 bash[78452]: cluster 1685822760.2730994 mon.a (mon.0) 812 : cluster 0 osdmap e160: 8 total, 8 up, 8 in 2023-06-03T20:06:01.254 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:06:00 smithi089 bash[75068]: audit 1685822759.966524 mon.c (mon.2) 434 : audit 1 from='client.? 172.21.15.89:0/3790659683' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 7, "weight": 0.5314142386440327}]: dispatch 2023-06-03T20:06:01.255 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:06:00 smithi089 bash[75068]: audit 1685822759.9672217 mon.a (mon.0) 808 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 7, "weight": 0.5314142386440327}]: dispatch 2023-06-03T20:06:01.255 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:06:00 smithi089 bash[75068]: cluster 1685822760.2633681 mon.a (mon.0) 809 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-06-03T20:06:01.255 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:06:00 smithi089 bash[75068]: cluster 1685822760.2703059 osd.7 (osd.7) 89 : cluster 3 failed to encode map e160 with expected crc 2023-06-03T20:06:01.255 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:06:00 smithi089 bash[75068]: audit 1685822760.272808 mon.a (mon.0) 810 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-06-03T20:06:01.255 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:06:00 smithi089 bash[75068]: audit 1685822760.2729714 mon.a (mon.0) 811 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 7, "weight": 0.5314142386440327}]': finished 2023-06-03T20:06:01.255 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:06:00 smithi089 bash[75068]: cluster 1685822760.2730994 mon.a (mon.0) 812 : cluster 0 osdmap e160: 8 total, 8 up, 8 in 2023-06-03T20:06:01.274 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:06:01.277 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:06:01.278 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:06:01.379 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:06:01.416 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:06:01.418 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:06:01.419 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:06:01.461 INFO:teuthology.orchestra.run.smithi164.stdout:ERROR: (22) Invalid argument 2023-06-03T20:06:01.462 INFO:teuthology.orchestra.run.smithi164.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-06-03T20:06:01.520 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:06:01.552 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:06:01.555 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:06:01.556 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:06:01.568 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-06-03T20:06:01.657 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:06:01.689 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:06:01.692 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:06:01.693 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:06:01.794 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:06:01.824 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:06:01.826 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:06:01.827 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:06:01.928 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:06:01.956 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:06:01.958 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:06:01.959 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:06:02.060 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:06:02.072 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:06:01 smithi164 bash[59462]: cluster 1685822760.26696 osd.6 (osd.6) 84 : cluster 3 failed to encode map e160 with expected crc 2023-06-03T20:06:02.073 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:06:01 smithi164 bash[59462]: cluster 1685822760.2704182 osd.6 (osd.6) 85 : cluster 3 failed to encode map e160 with expected crc 2023-06-03T20:06:02.073 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:06:01 smithi164 bash[59462]: cluster 1685822760.5428045 mgr.y (mgr.34104) 895 : cluster 0 pgmap v830: 89 pgs: 89 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-06-03T20:06:02.073 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:06:01 smithi164 bash[59462]: cluster 1685822761.279415 mon.a (mon.0) 813 : cluster 0 osdmap e161: 8 total, 8 up, 8 in 2023-06-03T20:06:02.074 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:06:01 smithi164 bash[59462]: cluster 1685822761.281477 osd.7 (osd.7) 90 : cluster 3 failed to encode map e161 with expected crc 2023-06-03T20:06:02.074 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:06:01 smithi164 bash[59462]: cluster 1685822761.2821982 osd.5 (osd.5) 89 : cluster 3 failed to encode map e161 with expected crc 2023-06-03T20:06:02.091 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:06:02.092 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:06:02.093 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:06:02.104 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:06:01 smithi089 bash[75068]: cluster 1685822760.26696 osd.6 (osd.6) 84 : cluster 3 failed to encode map e160 with expected crc 2023-06-03T20:06:02.104 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:06:01 smithi089 bash[75068]: cluster 1685822760.2704182 osd.6 (osd.6) 85 : cluster 3 failed to encode map e160 with expected crc 2023-06-03T20:06:02.105 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:06:01 smithi089 bash[75068]: cluster 1685822760.5428045 mgr.y (mgr.34104) 895 : cluster 0 pgmap v830: 89 pgs: 89 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-06-03T20:06:02.105 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:06:01 smithi089 bash[75068]: cluster 1685822761.279415 mon.a (mon.0) 813 : cluster 0 osdmap e161: 8 total, 8 up, 8 in 2023-06-03T20:06:02.105 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:06:01 smithi089 bash[75068]: cluster 1685822761.281477 osd.7 (osd.7) 90 : cluster 3 failed to encode map e161 with expected crc 2023-06-03T20:06:02.105 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:06:01 smithi089 bash[75068]: cluster 1685822761.2821982 osd.5 (osd.5) 89 : cluster 3 failed to encode map e161 with expected crc 2023-06-03T20:06:02.105 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:06:01 smithi089 bash[78452]: cluster 1685822760.26696 osd.6 (osd.6) 84 : cluster 3 failed to encode map e160 with expected crc 2023-06-03T20:06:02.105 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:06:01 smithi089 bash[78452]: cluster 1685822760.2704182 osd.6 (osd.6) 85 : cluster 3 failed to encode map e160 with expected crc 2023-06-03T20:06:02.106 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:06:01 smithi089 bash[78452]: cluster 1685822760.5428045 mgr.y (mgr.34104) 895 : cluster 0 pgmap v830: 89 pgs: 89 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-06-03T20:06:02.106 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:06:01 smithi089 bash[78452]: cluster 1685822761.279415 mon.a (mon.0) 813 : cluster 0 osdmap e161: 8 total, 8 up, 8 in 2023-06-03T20:06:02.106 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:06:01 smithi089 bash[78452]: cluster 1685822761.281477 osd.7 (osd.7) 90 : cluster 3 failed to encode map e161 with expected crc 2023-06-03T20:06:02.106 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:06:01 smithi089 bash[78452]: cluster 1685822761.2821982 osd.5 (osd.5) 89 : cluster 3 failed to encode map e161 with expected crc 2023-06-03T20:06:02.195 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:06:02.234 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:06:02.237 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:06:02.238 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:06:02.339 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:06:02.371 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:06:02.372 DEBUG:teuthology.orchestra.run.smithi164:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph daemon osd.7 dump_blocked_ops 2023-06-03T20:06:02.372 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:06:02.373 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:06:02.474 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:06:02.512 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:06:02.515 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:06:02.516 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:06:02.617 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:06:02.654 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:06:02.656 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:06:02.657 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:06:02.758 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:06:02.796 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:06:02.799 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:06:02.800 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:06:02.901 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:06:02.940 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:06:02.943 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:06:02.944 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:06:03.045 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:06:03.083 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:06:02 smithi164 bash[59462]: cluster 1685822761.2757578 osd.6 (osd.6) 86 : cluster 3 failed to encode map e161 with expected crc 2023-06-03T20:06:03.084 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:06:03.087 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:06:03.087 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:06:03.188 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:06:03.227 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:06:03.229 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:06:03.230 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:06:03.279 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:06:02 smithi089 bash[78452]: cluster 1685822761.2757578 osd.6 (osd.6) 86 : cluster 3 failed to encode map e161 with expected crc 2023-06-03T20:06:03.279 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:06:02 smithi089 bash[75068]: cluster 1685822761.2757578 osd.6 (osd.6) 86 : cluster 3 failed to encode map e161 with expected crc 2023-06-03T20:06:03.331 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:06:03.371 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:06:03.374 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:06:03.375 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:06:03.475 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:06:03.511 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:06:03.515 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:06:03.515 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:06:03.604 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/mon.a/config 2023-06-03T20:06:03.616 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:06:03.654 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:06:03.657 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:06:03.658 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:06:03.758 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:06:03.796 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:06:03.798 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:06:03.799 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:06:03.900 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:06:03.938 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:06:03.941 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:06:03.942 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:06:04.028 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:06:03 smithi089 bash[78452]: cluster 1685822762.5436218 mgr.y (mgr.34104) 896 : cluster 0 pgmap v832: 89 pgs: 89 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-06-03T20:06:04.029 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:06:03 smithi089 bash[75068]: cluster 1685822762.5436218 mgr.y (mgr.34104) 896 : cluster 0 pgmap v832: 89 pgs: 89 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-06-03T20:06:04.043 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:06:04.078 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:06:04.080 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:06:04.081 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:06:04.083 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:06:03 smithi164 bash[59462]: cluster 1685822762.5436218 mgr.y (mgr.34104) 896 : cluster 0 pgmap v832: 89 pgs: 89 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-06-03T20:06:04.108 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph osd unset nodeep-scrub 2023-06-03T20:06:04.182 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:06:04.217 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:06:04.219 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:06:04.219 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:06:04.320 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:06:04.355 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:06:04.359 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:06:04.360 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:06:04.461 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:06:04.494 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:06:04.498 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:06:04.499 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:06:04.600 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:06:04.639 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:06:04.642 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:06:04.643 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:06:04.744 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:06:04.781 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:06:04.784 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:06:04.784 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:06:04.885 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:06:04.924 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:06:04.926 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:06:04.927 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:06:05.028 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:06:05.066 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:06:05.068 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:06:05.068 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:06:05.169 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:06:05.204 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:06:05.207 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:06:05.208 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:06:05.308 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:06:05.346 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:06:05.348 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:06:05.349 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:06:05.450 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:06:05.484 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:06:05.487 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:06:05.487 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:06:05.588 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:06:05.627 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:06:05.630 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:06:05.630 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:06:05.731 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:06:05.757 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:06:05.760 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:06:05.760 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:06:05.861 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:06:05.894 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:06:05.898 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:06:05.899 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:06:06.000 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:06:06.013 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:06:05 smithi089 bash[75068]: cluster 1685822764.5446823 mgr.y (mgr.34104) 897 : cluster 0 pgmap v833: 89 pgs: 89 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-06-03T20:06:06.013 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:06:05 smithi089 bash[78452]: cluster 1685822764.5446823 mgr.y (mgr.34104) 897 : cluster 0 pgmap v833: 89 pgs: 89 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-06-03T20:06:06.035 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:06:06.038 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:06:06.039 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:06:06.140 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:06:06.153 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:06:05 smithi164 bash[59462]: cluster 1685822764.5446823 mgr.y (mgr.34104) 897 : cluster 0 pgmap v833: 89 pgs: 89 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-06-03T20:06:06.180 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:06:06.182 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:06:06.183 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:06:06.278 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 20:06:06 smithi089 bash[71163]: ::ffff:172.21.15.164 - - [03/Jun/2023:20:06:06] "GET /metrics HTTP/1.1" 200 225911 "" "Prometheus/2.33.4" 2023-06-03T20:06:06.284 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:06:06.312 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:06:06.315 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:06:06.315 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:06:06.416 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:06:06.454 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:06:06.456 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:06:06.457 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:06:06.558 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:06:06.594 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:06:06.597 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:06:06.598 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:06:06.672 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/mon.a/config 2023-06-03T20:06:06.698 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:06:06.734 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:06:06.737 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:06:06.738 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:06:06.838 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:06:06.874 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:06:06.876 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:06:06.877 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:06:06.978 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:06:07.021 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:06:07.023 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:06:07.024 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:06:07.070 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 5, 6, 7] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-06-03T20:06:07.070 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-06-03T20:06:07.125 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:06:07.165 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:06:07.167 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:06:07.168 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:06:07.230 INFO:teuthology.orchestra.run.smithi164.stderr:Inferring config /var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/mon.b/config 2023-06-03T20:06:07.269 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:06:07.297 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:06:07.300 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:06:07.301 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:06:07.402 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:06:07.437 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:06:07.440 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:06:07.441 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:06:07.542 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:06:07.573 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:06:07.576 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:06:07.576 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:06:07.678 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:06:07.711 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:06:07.713 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:06:07.714 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:06:07.815 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:06:07.847 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:06:07.850 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:06:07.850 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:06:07.951 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:06:07.962 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:06:07 smithi089 bash[75068]: cluster 1685822766.5454063 mgr.y (mgr.34104) 898 : cluster 0 pgmap v834: 89 pgs: 89 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-06-03T20:06:07.963 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:06:07 smithi089 bash[75068]: audit 1685822767.1058638 mon.c (mon.2) 435 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T20:06:07.963 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:06:07 smithi089 bash[75068]: audit 1685822767.6778698 mon.a (mon.0) 814 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:06:07.963 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:06:07 smithi089 bash[78452]: cluster 1685822766.5454063 mgr.y (mgr.34104) 898 : cluster 0 pgmap v834: 89 pgs: 89 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-06-03T20:06:07.963 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:06:07 smithi089 bash[78452]: audit 1685822767.1058638 mon.c (mon.2) 435 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T20:06:07.963 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:06:07 smithi089 bash[78452]: audit 1685822767.6778698 mon.a (mon.0) 814 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:06:07.979 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:06:07.981 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:06:07.982 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:06:08.083 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:06:08.118 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:06:08.121 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:06:07 smithi164 bash[59462]: cluster 1685822766.5454063 mgr.y (mgr.34104) 898 : cluster 0 pgmap v834: 89 pgs: 89 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-06-03T20:06:08.121 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:06:07 smithi164 bash[59462]: audit 1685822767.1058638 mon.c (mon.2) 435 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T20:06:08.121 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:06:07 smithi164 bash[59462]: audit 1685822767.6778698 mon.a (mon.0) 814 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:06:08.122 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:06:08.122 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:06:08.224 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:06:08.262 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:06:08.266 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:06:08.267 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:06:08.368 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:06:08.403 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:06:08.405 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:06:08.406 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:06:08.507 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:06:08.536 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:06:08.540 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:06:08.541 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:06:08.642 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:06:08.682 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:06:08.684 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:06:08.685 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:06:08.718 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/mon.a/config 2023-06-03T20:06:08.787 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:06:08.813 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:06:08.816 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:06:08.816 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:06:08.918 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:06:08.959 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:06:08.962 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:06:08.963 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:06:09.064 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:06:09.099 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:06:09.101 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:06:09.102 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:06:09.203 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:06:09.236 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:06:09.237 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:06:09.238 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:06:09.339 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:06:09.374 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:06:09.376 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:06:09.377 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:06:09.478 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:06:09.505 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:06:09.507 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:06:09.508 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:06:09.609 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:06:09.640 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:06:09.642 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:06:09.644 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:06:09.746 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:06:09.779 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:06:09.782 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:06:09.783 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:06:09.874 INFO:teuthology.orchestra.run.smithi089.stderr:enabled application 'rados' on pool 'unique_pool_5' 2023-06-03T20:06:09.883 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:06:09.918 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:06:09.921 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:06:09 smithi164 bash[59462]: cluster 1685822768.5464554 mgr.y (mgr.34104) 899 : cluster 0 pgmap v835: 89 pgs: 89 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-06-03T20:06:09.922 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:06:09 smithi164 bash[59462]: audit 1685822769.0830786 mon.c (mon.2) 436 : audit 1 from='client.? 172.21.15.89:0/3935675830' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_5", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-06-03T20:06:09.922 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:06:09 smithi164 bash[59462]: audit 1685822769.0835576 mon.a (mon.0) 815 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_5", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-06-03T20:06:09.923 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:06:09.923 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:06:10.024 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:06:10.028 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:06:09 smithi089 bash[75068]: cluster 1685822768.5464554 mgr.y (mgr.34104) 899 : cluster 0 pgmap v835: 89 pgs: 89 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-06-03T20:06:10.028 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:06:09 smithi089 bash[75068]: audit 1685822769.0830786 mon.c (mon.2) 436 : audit 1 from='client.? 172.21.15.89:0/3935675830' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_5", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-06-03T20:06:10.029 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:06:09 smithi089 bash[75068]: audit 1685822769.0835576 mon.a (mon.0) 815 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_5", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-06-03T20:06:10.029 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:06:09 smithi089 bash[78452]: cluster 1685822768.5464554 mgr.y (mgr.34104) 899 : cluster 0 pgmap v835: 89 pgs: 89 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-06-03T20:06:10.029 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:06:09 smithi089 bash[78452]: audit 1685822769.0830786 mon.c (mon.2) 436 : audit 1 from='client.? 172.21.15.89:0/3935675830' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_5", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-06-03T20:06:10.029 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:06:09 smithi089 bash[78452]: audit 1685822769.0835576 mon.a (mon.0) 815 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_5", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-06-03T20:06:10.054 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:06:10.056 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:06:10.056 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:06:10.157 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:06:10.193 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:06:10.195 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:06:10.196 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:06:10.297 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:06:10.323 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:06:10.326 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:06:10.326 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:06:10.427 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:06:10.457 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:06:10.459 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:06:10.460 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:06:10.561 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:06:10.633 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:06:10.637 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:06:10.638 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:06:10.739 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:06:10.769 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:06:10.771 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:06:10.772 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:06:10.873 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:06:10.905 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:06:10.907 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:06:10.908 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:06:11.009 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:06:11.048 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:06:11.050 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:06:11.051 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:06:11.152 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:06:11.163 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:06:10 smithi164 bash[59462]: cluster 1685822769.8567846 osd.6 (osd.6) 87 : cluster 3 failed to encode map e162 with expected crc 2023-06-03T20:06:11.163 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:06:10 smithi164 bash[59462]: audit 1685822769.8611858 mon.a (mon.0) 816 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_5", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-06-03T20:06:11.163 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:06:10 smithi164 bash[59462]: cluster 1685822769.8612761 mon.a (mon.0) 817 : cluster 0 osdmap e162: 8 total, 8 up, 8 in 2023-06-03T20:06:11.163 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:06:10 smithi164 bash[59462]: cluster 1685822770.6792583 osd.7 (osd.7) 91 : cluster 3 failed to encode map e162 with expected crc 2023-06-03T20:06:11.187 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:06:11.190 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:06:11.190 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:06:11.238 INFO:teuthology.orchestra.run.smithi164.stdout:ERROR: (22) Invalid argument 2023-06-03T20:06:11.238 INFO:teuthology.orchestra.run.smithi164.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-06-03T20:06:11.278 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:06:10 smithi089 bash[75068]: cluster 1685822769.8567846 osd.6 (osd.6) 87 : cluster 3 failed to encode map e162 with expected crc 2023-06-03T20:06:11.279 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:06:10 smithi089 bash[75068]: audit 1685822769.8611858 mon.a (mon.0) 816 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_5", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-06-03T20:06:11.279 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:06:10 smithi089 bash[75068]: cluster 1685822769.8612761 mon.a (mon.0) 817 : cluster 0 osdmap e162: 8 total, 8 up, 8 in 2023-06-03T20:06:11.279 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:06:10 smithi089 bash[75068]: cluster 1685822770.6792583 osd.7 (osd.7) 91 : cluster 3 failed to encode map e162 with expected crc 2023-06-03T20:06:11.279 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:06:10 smithi089 bash[78452]: cluster 1685822769.8567846 osd.6 (osd.6) 87 : cluster 3 failed to encode map e162 with expected crc 2023-06-03T20:06:11.279 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:06:10 smithi089 bash[78452]: audit 1685822769.8611858 mon.a (mon.0) 816 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_5", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-06-03T20:06:11.280 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:06:10 smithi089 bash[78452]: cluster 1685822769.8612761 mon.a (mon.0) 817 : cluster 0 osdmap e162: 8 total, 8 up, 8 in 2023-06-03T20:06:11.280 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:06:10 smithi089 bash[78452]: cluster 1685822770.6792583 osd.7 (osd.7) 91 : cluster 3 failed to encode map e162 with expected crc 2023-06-03T20:06:11.291 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:06:11.321 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:06:11.323 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:06:11.324 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:06:11.425 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:06:11.457 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:06:11.460 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:06:11.460 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:06:11.561 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:06:11.590 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:06:11.592 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:06:11.593 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:06:11.694 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:06:11.723 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:06:11.725 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:06:11.726 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:06:11.827 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:06:11.860 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:06:11.862 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:06:11.863 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:06:11.964 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:06:11.990 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:06:11.992 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:06:11.993 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:06:12.093 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:06:12.101 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:06:11 smithi089 bash[75068]: cluster 1685822770.5472806 mgr.y (mgr.34104) 900 : cluster 0 pgmap v837: 89 pgs: 89 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-06-03T20:06:12.101 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:06:11 smithi089 bash[75068]: cluster 1685822770.6791513 osd.5 (osd.5) 90 : cluster 3 failed to encode map e162 with expected crc 2023-06-03T20:06:12.102 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:06:11 smithi089 bash[78452]: cluster 1685822770.5472806 mgr.y (mgr.34104) 900 : cluster 0 pgmap v837: 89 pgs: 89 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-06-03T20:06:12.102 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:06:11 smithi089 bash[78452]: cluster 1685822770.6791513 osd.5 (osd.5) 90 : cluster 3 failed to encode map e162 with expected crc 2023-06-03T20:06:12.119 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:06:12.121 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:06:12.122 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:06:12.136 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:06:11 smithi164 bash[59462]: cluster 1685822770.5472806 mgr.y (mgr.34104) 900 : cluster 0 pgmap v837: 89 pgs: 89 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-06-03T20:06:12.136 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:06:11 smithi164 bash[59462]: cluster 1685822770.6791513 osd.5 (osd.5) 90 : cluster 3 failed to encode map e162 with expected crc 2023-06-03T20:06:12.137 DEBUG:teuthology.orchestra.run.smithi164:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph daemon osd.7 dump_historic_ops 2023-06-03T20:06:12.223 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:06:12.255 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:06:12.258 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:06:12.259 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:06:12.360 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:06:12.399 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:06:12.402 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:06:12.403 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:06:12.503 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:06:12.543 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:06:12.546 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:06:12.547 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:06:12.647 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:06:12.684 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:06:12.688 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:06:12.688 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:06:12.789 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:06:12.825 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:06:12.828 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:06:12.829 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:06:12.929 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:06:12.959 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:06:12.961 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:06:12.962 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:06:13.063 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:06:13.101 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:06:13.104 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:06:13.104 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:06:13.205 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:06:13.243 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:06:13.246 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:06:13.247 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:06:13.347 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:06:13.375 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:06:13.377 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:06:13.377 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:06:13.478 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:06:13.515 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:06:13.518 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:06:13.519 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:06:13.619 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:06:13.654 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:06:13.657 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:06:13.657 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:06:13.758 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:06:13.783 DEBUG:teuthology.orchestra.run.smithi089:> /bin/sh -c 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage rados --no-log-to-stderr --name client.0 -b 65536 --object-size 65536 -p unique_pool_5 bench 90 write' 2023-06-03T20:06:13.784 INFO:tasks.thrashosds.thrasher:fixing pg num pool unique_pool_5 2023-06-03T20:06:13.784 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph pg dump --format=json 2023-06-03T20:06:13.785 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:06:13.787 INFO:tasks.radosbench:joining radosbench (timing out after 3000s) 2023-06-03T20:06:13.787 INFO:teuthology.orchestra.run:waiting for 3000 2023-06-03T20:06:13.788 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:06:13.788 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:06:13.884 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:06:13 smithi089 bash[75068]: cluster 1685822772.5479915 mgr.y (mgr.34104) 901 : cluster 0 pgmap v838: 89 pgs: 89 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-06-03T20:06:13.889 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:06:13.914 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:06:13.917 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:06:13.918 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:06:14.019 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:06:14.048 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:06:14.051 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:06:14.051 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:06:14.152 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:06:14.163 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:06:13 smithi164 bash[59462]: cluster 1685822772.5479915 mgr.y (mgr.34104) 901 : cluster 0 pgmap v838: 89 pgs: 89 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-06-03T20:06:14.189 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:06:14.191 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:06:14.191 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:06:14.229 INFO:tasks.radosbench.radosbench.0.smithi089.stdout:hints = 1 2023-06-03T20:06:14.229 INFO:tasks.radosbench.radosbench.0.smithi089.stdout:Maintaining 16 concurrent writes of 65536 bytes to objects of size 65536 for up to 90 seconds or 0 objects 2023-06-03T20:06:14.229 INFO:tasks.radosbench.radosbench.0.smithi089.stdout:Object prefix: benchmark_data_smithi089_140876 2023-06-03T20:06:14.230 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-06-03T20:06:14.230 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 0 0 0 0 0 0 - 0 2023-06-03T20:06:14.278 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:06:13 smithi089 bash[78452]: cluster 1685822772.5479915 mgr.y (mgr.34104) 901 : cluster 0 pgmap v838: 89 pgs: 89 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-06-03T20:06:14.293 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:06:14.326 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:06:14.328 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:06:14.329 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:06:14.430 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:06:14.463 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:06:14.467 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:06:14.468 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:06:14.569 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:06:14.617 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:06:14.622 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:06:14.623 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:06:14.724 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:06:14.758 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:06:14.762 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:06:14.763 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:06:14.865 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:06:14.897 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:06:14.899 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:06:14.900 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:06:15.001 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:06:15.035 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:06:15.038 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:06:15.038 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:06:15.139 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:06:15.177 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:06:15.181 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:06:15.181 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:06:15.229 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 1 16 2996 2980 186.239 186.25 0.00480539 0.00535033 2023-06-03T20:06:15.282 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:06:15.319 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:06:15.322 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:06:15.323 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:06:15.423 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:06:15.455 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:06:15.458 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:06:15.458 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:06:15.559 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:06:15.591 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:06:15.593 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:06:15.594 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:06:15.695 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:06:15.727 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:06:15.729 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:06:15.730 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:06:15.831 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:06:15.859 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:06:15.862 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:06:15.863 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:06:15.964 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:06:15.998 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:06:16.001 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:06:16.002 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:06:16.028 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:06:15 smithi089 bash[75068]: cluster 1685822774.5491552 mgr.y (mgr.34104) 902 : cluster 0 pgmap v839: 89 pgs: 89 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-06-03T20:06:16.029 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:06:15 smithi089 bash[78452]: cluster 1685822774.5491552 mgr.y (mgr.34104) 902 : cluster 0 pgmap v839: 89 pgs: 89 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-06-03T20:06:16.103 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:06:16.139 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:06:16.143 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:06:16.144 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:06:16.229 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 2 16 5752 5736 179.235 172.25 0.00878248 0.00556071 2023-06-03T20:06:16.245 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:06:16.255 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:06:15 smithi164 bash[59462]: cluster 1685822774.5491552 mgr.y (mgr.34104) 902 : cluster 0 pgmap v839: 89 pgs: 89 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-06-03T20:06:16.283 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:06:16.286 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:06:16.286 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:06:16.363 INFO:teuthology.orchestra.run.smithi089.stdout:osd.0: {} 2023-06-03T20:06:16.387 INFO:teuthology.orchestra.run.smithi089.stdout:osd.1: {} 2023-06-03T20:06:16.387 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:06:16.425 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:06:16.428 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:06:16.428 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:06:16.503 INFO:teuthology.orchestra.run.smithi089.stdout:osd.2: {} 2023-06-03T20:06:16.528 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 20:06:16 smithi089 bash[71163]: ::ffff:172.21.15.164 - - [03/Jun/2023:20:06:16] "GET /metrics HTTP/1.1" 200 228287 "" "Prometheus/2.33.4" 2023-06-03T20:06:16.529 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:06:16.547 INFO:teuthology.orchestra.run.smithi089.stdout:osd.3: {} 2023-06-03T20:06:16.560 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:06:16.564 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:06:16.565 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:06:16.581 INFO:teuthology.orchestra.run.smithi089.stdout:osd.4: {} 2023-06-03T20:06:16.619 INFO:teuthology.orchestra.run.smithi089.stdout:osd.5: {} 2023-06-03T20:06:16.663 INFO:teuthology.orchestra.run.smithi089.stdout:osd.6: {} 2023-06-03T20:06:16.665 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:06:16.697 INFO:teuthology.orchestra.run.smithi089.stdout:osd.7: {} 2023-06-03T20:06:16.699 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:06:16.700 INFO:teuthology.orchestra.run.smithi089.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-06-03T20:06:16.700 INFO:teuthology.orchestra.run.smithi089.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-06-03T20:06:16.701 INFO:teuthology.orchestra.run.smithi089.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-06-03T20:06:16.701 INFO:teuthology.orchestra.run.smithi089.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-06-03T20:06:16.701 INFO:teuthology.orchestra.run.smithi089.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-06-03T20:06:16.701 INFO:teuthology.orchestra.run.smithi089.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-06-03T20:06:16.701 INFO:teuthology.orchestra.run.smithi089.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-06-03T20:06:16.701 INFO:teuthology.orchestra.run.smithi089.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-06-03T20:06:16.702 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:06:16.702 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:06:16.804 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:06:16.844 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:06:16.848 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:06:16.849 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:06:16.950 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:06:16.959 INFO:teuthology.orchestra.run.smithi164.stderr:Inferring config /var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/mon.b/config 2023-06-03T20:06:16.961 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:06:16 smithi164 bash[59462]: audit 1685822776.3131373 mon.a (mon.0) 818 : audit 0 from='client.? 172.21.15.89:0/2873062358' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-03T20:06:16.991 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:06:16.993 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:06:16.993 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:06:17.094 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:06:17.121 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:06:17.123 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:06:17.124 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:06:17.225 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:06:17.229 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 3 16 7858 7842 163.36 131.625 0.00897824 0.0061118 2023-06-03T20:06:17.234 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:06:16 smithi089 bash[75068]: audit 1685822776.3131373 mon.a (mon.0) 818 : audit 0 from='client.? 172.21.15.89:0/2873062358' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-03T20:06:17.234 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:06:16 smithi089 bash[78452]: audit 1685822776.3131373 mon.a (mon.0) 818 : audit 0 from='client.? 172.21.15.89:0/2873062358' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-03T20:06:17.260 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:06:17.265 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:06:17.265 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:06:17.367 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:06:17.398 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:06:17.403 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:06:17.404 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:06:17.505 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:06:17.532 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:06:17.535 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:06:17.535 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:06:17.636 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:06:17.660 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:06:17.662 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:06:17.663 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:06:17.764 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:06:17.798 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:06:17.800 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:06:17.801 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:06:17.902 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:06:17.940 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:06:17.943 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:06:17.944 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:06:18.045 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:06:18.083 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:06:17 smithi164 bash[59462]: cluster 1685822776.5516226 mgr.y (mgr.34104) 903 : cluster 0 pgmap v840: 89 pgs: 89 active+clean; 43 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 4.2 MiB/s wr, 67 op/s 2023-06-03T20:06:18.084 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:06:18.086 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:06:18.087 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:06:18.188 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:06:18.222 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:06:18.226 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:06:18.226 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:06:18.229 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 4 16 10314 10298 160.892 153.5 0.0040387 0.00619312 2023-06-03T20:06:18.278 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:06:17 smithi089 bash[78452]: cluster 1685822776.5516226 mgr.y (mgr.34104) 903 : cluster 0 pgmap v840: 89 pgs: 89 active+clean; 43 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 4.2 MiB/s wr, 67 op/s 2023-06-03T20:06:18.279 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:06:17 smithi089 bash[75068]: cluster 1685822776.5516226 mgr.y (mgr.34104) 903 : cluster 0 pgmap v840: 89 pgs: 89 active+clean; 43 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 4.2 MiB/s wr, 67 op/s 2023-06-03T20:06:18.327 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:06:18.366 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:06:18.368 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:06:18.369 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:06:18.470 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:06:18.506 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:06:18.509 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:06:18.510 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:06:18.611 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:06:18.643 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:06:18.647 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:06:18.648 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:06:18.749 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:06:18.782 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:06:18.786 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:06:18.786 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:06:18.887 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:06:18.922 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:06:18.925 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:06:18.927 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:06:19.028 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:06:19.061 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:06:19.064 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:06:19.065 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:06:19.165 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:06:19.197 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:06:19.202 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:06:19.203 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:06:19.230 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 5 16 12303 12287 153.574 124.312 0.010243 0.00650399 2023-06-03T20:06:19.304 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:06:19.337 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:06:19.339 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:06:19.339 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:06:19.440 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:06:19.478 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:06:19.482 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:06:19.483 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:06:19.583 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:06:19.618 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:06:19.622 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:06:19.623 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:06:19.687 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/mon.a/config 2023-06-03T20:06:19.724 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:06:19.748 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:06:19.751 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:06:19.751 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:06:19.852 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:06:19.884 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:06:19.886 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:06:19.887 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:06:19.988 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:06:20.027 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:06:20.029 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:06:20.030 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:06:20.131 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:06:20.157 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:06:20.161 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:06:19 smithi089 bash[75068]: cluster 1685822778.5526407 mgr.y (mgr.34104) 904 : cluster 0 pgmap v841: 89 pgs: 89 active+clean; 449 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 45 MiB/s wr, 717 op/s 2023-06-03T20:06:20.161 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:06:19 smithi089 bash[78452]: cluster 1685822778.5526407 mgr.y (mgr.34104) 904 : cluster 0 pgmap v841: 89 pgs: 89 active+clean; 449 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 45 MiB/s wr, 717 op/s 2023-06-03T20:06:20.162 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:06:20.162 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:06:20.230 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 6 16 14445 14429 150.289 133.875 0.00602483 0.00664496 2023-06-03T20:06:20.263 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:06:20.297 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:06:20.300 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:06:20.301 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:06:20.333 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:06:19 smithi164 bash[59462]: cluster 1685822778.5526407 mgr.y (mgr.34104) 904 : cluster 0 pgmap v841: 89 pgs: 89 active+clean; 449 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 45 MiB/s wr, 717 op/s 2023-06-03T20:06:20.401 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:06:20.438 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:06:20.442 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:06:20.443 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:06:20.545 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:06:20.581 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:06:20.584 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:06:20.585 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:06:20.686 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:06:20.717 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:06:20.719 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:06:20.720 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:06:20.821 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:06:20.855 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:06:20.857 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:06:20.858 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:06:20.915 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:06:20 smithi089 bash[75068]: audit 1685822780.019317 mon.c (mon.2) 437 : audit 1 from='client.? 172.21.15.89:0/1463612581' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-06-03T20:06:20.915 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:06:20 smithi089 bash[75068]: audit 1685822780.0197387 mon.a (mon.0) 819 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-06-03T20:06:20.931 INFO:teuthology.orchestra.run.smithi089.stderr:nodeep-scrub is unset 2023-06-03T20:06:20.959 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:06:20.993 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:06:20.995 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:06:20.996 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:06:21.046 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:06:20 smithi164 bash[59462]: audit 1685822780.019317 mon.c (mon.2) 437 : audit 1 from='client.? 172.21.15.89:0/1463612581' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-06-03T20:06:21.046 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:06:20 smithi164 bash[59462]: audit 1685822780.0197387 mon.a (mon.0) 819 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-06-03T20:06:21.097 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:06:21.134 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:06:21.137 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:06:21.138 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:06:21.231 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 7 16 16609 16593 148.138 135.25 0.00456922 0.00674304 2023-06-03T20:06:21.238 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:06:21.246 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:06:20 smithi089 bash[78452]: audit 1685822780.019317 mon.c (mon.2) 437 : audit 1 from='client.? 172.21.15.89:0/1463612581' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-06-03T20:06:21.247 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:06:20 smithi089 bash[78452]: audit 1685822780.0197387 mon.a (mon.0) 819 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-06-03T20:06:21.271 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:06:21.274 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:06:21.275 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:06:21.309 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-06-03T20:06:21.376 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:06:21.429 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:06:21.432 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:06:21.433 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:06:21.533 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:06:21.567 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:06:21.569 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:06:21.571 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:06:21.672 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:06:21.702 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:06:21.706 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:06:21.707 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:06:21.808 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:06:21.842 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:06:21.846 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:06:21.846 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:06:21.948 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:06:21.981 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:06:21.983 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:06:21.984 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:06:22.085 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:06:22.121 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:06:22.123 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:06:22.124 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:06:22.225 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:06:22.230 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 8 16 18628 18612 145.393 126.188 0.0123435 0.00687244 2023-06-03T20:06:22.236 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:06:21 smithi089 bash[75068]: cluster 1685822780.5533454 mgr.y (mgr.34104) 905 : cluster 0 pgmap v842: 89 pgs: 89 active+clean; 675 MiB data, 3.7 GiB used, 711 GiB / 715 GiB avail; 63 MiB/s wr, 1.01k op/s 2023-06-03T20:06:22.236 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:06:21 smithi089 bash[75068]: cluster 1685822780.902499 mon.a (mon.0) 820 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-06-03T20:06:22.236 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:06:21 smithi089 bash[75068]: cluster 1685822780.902531 mon.a (mon.0) 821 : cluster 1 Cluster is now healthy 2023-06-03T20:06:22.236 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:06:21 smithi089 bash[75068]: cluster 1685822780.9155474 osd.5 (osd.5) 91 : cluster 3 failed to encode map e163 with expected crc 2023-06-03T20:06:22.236 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:06:21 smithi089 bash[75068]: cluster 1685822780.9156168 osd.7 (osd.7) 92 : cluster 3 failed to encode map e163 with expected crc 2023-06-03T20:06:22.237 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:06:21 smithi089 bash[75068]: cluster 1685822780.9167287 osd.6 (osd.6) 88 : cluster 3 failed to encode map e163 with expected crc 2023-06-03T20:06:22.237 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:06:21 smithi089 bash[75068]: cluster 1685822780.918452 osd.7 (osd.7) 93 : cluster 3 failed to encode map e163 with expected crc 2023-06-03T20:06:22.237 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:06:21 smithi089 bash[75068]: cluster 1685822780.9194508 osd.5 (osd.5) 92 : cluster 3 failed to encode map e163 with expected crc 2023-06-03T20:06:22.237 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:06:21 smithi089 bash[75068]: audit 1685822780.9204473 mon.a (mon.0) 822 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-06-03T20:06:22.237 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:06:21 smithi089 bash[75068]: cluster 1685822780.9207654 mon.a (mon.0) 823 : cluster 0 osdmap e163: 8 total, 8 up, 8 in 2023-06-03T20:06:22.237 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:06:21 smithi089 bash[75068]: cluster 1685822780.9215138 osd.6 (osd.6) 89 : cluster 3 failed to encode map e163 with expected crc 2023-06-03T20:06:22.237 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:06:21 smithi089 bash[75068]: cluster 1685822780.9237719 osd.6 (osd.6) 90 : cluster 3 failed to encode map e163 with expected crc 2023-06-03T20:06:22.237 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:06:21 smithi089 bash[75068]: cluster 1685822780.9250882 osd.6 (osd.6) 91 : cluster 3 failed to encode map e163 with expected crc 2023-06-03T20:06:22.238 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:06:21 smithi089 bash[75068]: cluster 1685822780.9274902 osd.6 (osd.6) 92 : cluster 3 failed to encode map e163 with expected crc 2023-06-03T20:06:22.238 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:06:21 smithi089 bash[78452]: cluster 1685822780.5533454 mgr.y (mgr.34104) 905 : cluster 0 pgmap v842: 89 pgs: 89 active+clean; 675 MiB data, 3.7 GiB used, 711 GiB / 715 GiB avail; 63 MiB/s wr, 1.01k op/s 2023-06-03T20:06:22.238 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:06:21 smithi089 bash[78452]: cluster 1685822780.902499 mon.a (mon.0) 820 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-06-03T20:06:22.238 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:06:21 smithi089 bash[78452]: cluster 1685822780.902531 mon.a (mon.0) 821 : cluster 1 Cluster is now healthy 2023-06-03T20:06:22.238 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:06:21 smithi089 bash[78452]: cluster 1685822780.9155474 osd.5 (osd.5) 91 : cluster 3 failed to encode map e163 with expected crc 2023-06-03T20:06:22.238 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:06:21 smithi089 bash[78452]: cluster 1685822780.9156168 osd.7 (osd.7) 92 : cluster 3 failed to encode map e163 with expected crc 2023-06-03T20:06:22.239 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:06:21 smithi089 bash[78452]: cluster 1685822780.9167287 osd.6 (osd.6) 88 : cluster 3 failed to encode map e163 with expected crc 2023-06-03T20:06:22.239 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:06:21 smithi089 bash[78452]: cluster 1685822780.918452 osd.7 (osd.7) 93 : cluster 3 failed to encode map e163 with expected crc 2023-06-03T20:06:22.239 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:06:21 smithi089 bash[78452]: cluster 1685822780.9194508 osd.5 (osd.5) 92 : cluster 3 failed to encode map e163 with expected crc 2023-06-03T20:06:22.239 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:06:21 smithi089 bash[78452]: audit 1685822780.9204473 mon.a (mon.0) 822 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-06-03T20:06:22.239 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:06:21 smithi089 bash[78452]: cluster 1685822780.9207654 mon.a (mon.0) 823 : cluster 0 osdmap e163: 8 total, 8 up, 8 in 2023-06-03T20:06:22.240 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:06:21 smithi089 bash[78452]: cluster 1685822780.9215138 osd.6 (osd.6) 89 : cluster 3 failed to encode map e163 with expected crc 2023-06-03T20:06:22.240 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:06:21 smithi089 bash[78452]: cluster 1685822780.9237719 osd.6 (osd.6) 90 : cluster 3 failed to encode map e163 with expected crc 2023-06-03T20:06:22.240 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:06:21 smithi089 bash[78452]: cluster 1685822780.9250882 osd.6 (osd.6) 91 : cluster 3 failed to encode map e163 with expected crc 2023-06-03T20:06:22.240 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:06:21 smithi089 bash[78452]: cluster 1685822780.9274902 osd.6 (osd.6) 92 : cluster 3 failed to encode map e163 with expected crc 2023-06-03T20:06:22.257 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:06:22.260 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:06:22.261 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:06:22.334 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:06:21 smithi164 bash[59462]: cluster 1685822780.5533454 mgr.y (mgr.34104) 905 : cluster 0 pgmap v842: 89 pgs: 89 active+clean; 675 MiB data, 3.7 GiB used, 711 GiB / 715 GiB avail; 63 MiB/s wr, 1.01k op/s 2023-06-03T20:06:22.334 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:06:21 smithi164 bash[59462]: cluster 1685822780.902499 mon.a (mon.0) 820 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-06-03T20:06:22.334 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:06:21 smithi164 bash[59462]: cluster 1685822780.902531 mon.a (mon.0) 821 : cluster 1 Cluster is now healthy 2023-06-03T20:06:22.334 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:06:21 smithi164 bash[59462]: cluster 1685822780.9155474 osd.5 (osd.5) 91 : cluster 3 failed to encode map e163 with expected crc 2023-06-03T20:06:22.335 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:06:21 smithi164 bash[59462]: cluster 1685822780.9156168 osd.7 (osd.7) 92 : cluster 3 failed to encode map e163 with expected crc 2023-06-03T20:06:22.335 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:06:21 smithi164 bash[59462]: cluster 1685822780.9167287 osd.6 (osd.6) 88 : cluster 3 failed to encode map e163 with expected crc 2023-06-03T20:06:22.335 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:06:21 smithi164 bash[59462]: cluster 1685822780.918452 osd.7 (osd.7) 93 : cluster 3 failed to encode map e163 with expected crc 2023-06-03T20:06:22.335 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:06:21 smithi164 bash[59462]: cluster 1685822780.9194508 osd.5 (osd.5) 92 : cluster 3 failed to encode map e163 with expected crc 2023-06-03T20:06:22.335 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:06:21 smithi164 bash[59462]: audit 1685822780.9204473 mon.a (mon.0) 822 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-06-03T20:06:22.335 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:06:21 smithi164 bash[59462]: cluster 1685822780.9207654 mon.a (mon.0) 823 : cluster 0 osdmap e163: 8 total, 8 up, 8 in 2023-06-03T20:06:22.335 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:06:21 smithi164 bash[59462]: cluster 1685822780.9215138 osd.6 (osd.6) 89 : cluster 3 failed to encode map e163 with expected crc 2023-06-03T20:06:22.335 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:06:21 smithi164 bash[59462]: cluster 1685822780.9237719 osd.6 (osd.6) 90 : cluster 3 failed to encode map e163 with expected crc 2023-06-03T20:06:22.336 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:06:21 smithi164 bash[59462]: cluster 1685822780.9250882 osd.6 (osd.6) 91 : cluster 3 failed to encode map e163 with expected crc 2023-06-03T20:06:22.336 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:06:21 smithi164 bash[59462]: cluster 1685822780.9274902 osd.6 (osd.6) 92 : cluster 3 failed to encode map e163 with expected crc 2023-06-03T20:06:22.362 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:06:22.398 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:06:22.402 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:06:22.403 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:06:22.504 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:06:22.539 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:06:22.542 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:06:22.542 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:06:22.644 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:06:22.686 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:06:22.689 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:06:22.689 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:06:22.790 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:06:22.824 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:06:22.827 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:06:22.828 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:06:22.929 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:06:22.964 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:06:22.967 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:06:22.968 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:06:23.068 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:06:23.108 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:06:23.110 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:06:23.110 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:06:23.127 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:06:23 smithi089 bash[75068]: audit 1685822782.1153755 mon.a (mon.0) 824 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:06:23.127 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:06:23 smithi089 bash[75068]: audit 1685822782.1170955 mon.c (mon.2) 438 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T20:06:23.128 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:06:23 smithi089 bash[75068]: cluster 1685822782.5539944 mgr.y (mgr.34104) 906 : cluster 0 pgmap v844: 89 pgs: 89 active+clean; 762 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 76 MiB/s wr, 1.22k op/s 2023-06-03T20:06:23.128 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:06:23 smithi089 bash[78452]: audit 1685822782.1153755 mon.a (mon.0) 824 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:06:23.128 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:06:23 smithi089 bash[78452]: audit 1685822782.1170955 mon.c (mon.2) 438 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T20:06:23.128 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:06:23 smithi089 bash[78452]: cluster 1685822782.5539944 mgr.y (mgr.34104) 906 : cluster 0 pgmap v844: 89 pgs: 89 active+clean; 762 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 76 MiB/s wr, 1.22k op/s 2023-06-03T20:06:23.211 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:06:23.231 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 9 16 20700 20684 143.612 129.5 0.0103601 0.00695551 2023-06-03T20:06:23.244 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:06:23.248 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:06:23.249 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:06:23.333 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:06:23 smithi164 bash[59462]: audit 1685822782.1153755 mon.a (mon.0) 824 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:06:23.334 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:06:23 smithi164 bash[59462]: audit 1685822782.1170955 mon.c (mon.2) 438 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T20:06:23.334 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:06:23 smithi164 bash[59462]: cluster 1685822782.5539944 mgr.y (mgr.34104) 906 : cluster 0 pgmap v844: 89 pgs: 89 active+clean; 762 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 76 MiB/s wr, 1.22k op/s 2023-06-03T20:06:23.350 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:06:23.383 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:06:23.385 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:06:23.386 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:06:23.486 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:06:23.517 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:06:23.522 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:06:23.523 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:06:23.623 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:06:23.661 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:06:23.668 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:06:23.669 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:06:23.770 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:06:23.804 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:06:23.810 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:06:23.810 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:06:23.911 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:06:23.946 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:06:23.950 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:06:23.951 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:06:24.053 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:06:24.085 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:06:24.086 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:06:24.087 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:06:24.188 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:06:24.226 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:06:24.229 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:06:24.229 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:06:24.231 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 10 16 22660 22644 141.5 122.5 0.00871151 0.00705947 2023-06-03T20:06:24.330 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:06:24.359 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:06:24.361 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:06:24.362 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:06:24.463 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:06:24.494 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:06:24.497 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:06:24.498 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:06:24.599 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:06:24.633 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:06:24.636 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:06:24.637 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:06:24.737 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:06:24.743 INFO:teuthology.orchestra.run.smithi164.stdout:{ 2023-06-03T20:06:24.743 INFO:teuthology.orchestra.run.smithi164.stdout: "size": 20, 2023-06-03T20:06:24.743 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 600, 2023-06-03T20:06:24.743 INFO:teuthology.orchestra.run.smithi164.stdout: "ops": [ 2023-06-03T20:06:24.744 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:06:24.744 INFO:teuthology.orchestra.run.smithi164.stdout: "description": "osd_op(client.45713.0:4722 9.d 9:b43df3da:::benchmark_data_smithi089_115580_object4721:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e131)", 2023-06-03T20:06:24.744 INFO:teuthology.orchestra.run.smithi164.stdout: "initiated_at": "2023-06-03T20:00:42.825628+0000", 2023-06-03T20:06:24.744 INFO:teuthology.orchestra.run.smithi164.stdout: "age": 341.900506759, 2023-06-03T20:06:24.744 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.32671114699999998, 2023-06-03T20:06:24.744 INFO:teuthology.orchestra.run.smithi164.stdout: "type_data": { 2023-06-03T20:06:24.744 INFO:teuthology.orchestra.run.smithi164.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-03T20:06:24.745 INFO:teuthology.orchestra.run.smithi164.stdout: "client_info": { 2023-06-03T20:06:24.745 INFO:teuthology.orchestra.run.smithi164.stdout: "client": "client.45713", 2023-06-03T20:06:24.745 INFO:teuthology.orchestra.run.smithi164.stdout: "client_addr": "172.21.15.89:0/2400821448", 2023-06-03T20:06:24.745 INFO:teuthology.orchestra.run.smithi164.stdout: "tid": 4722 2023-06-03T20:06:24.745 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:06:24.745 INFO:teuthology.orchestra.run.smithi164.stdout: "events": [ 2023-06-03T20:06:24.745 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:06:24.746 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "initiated", 2023-06-03T20:06:24.746 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:00:42.825628+0000", 2023-06-03T20:06:24.746 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0 2023-06-03T20:06:24.746 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:06:24.746 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:06:24.746 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "throttled", 2023-06-03T20:06:24.747 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:00:42.825628+0000", 2023-06-03T20:06:24.747 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 4294967295.9999986 2023-06-03T20:06:24.747 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:06:24.747 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:06:24.747 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "header_read", 2023-06-03T20:06:24.747 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:00:42.825627+0000", 2023-06-03T20:06:24.747 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 8.5945999999999993e-05 2023-06-03T20:06:24.747 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:06:24.748 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:06:24.748 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "all_read", 2023-06-03T20:06:24.748 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:00:42.825713+0000", 2023-06-03T20:06:24.748 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 3.6770999999999997e-05 2023-06-03T20:06:24.748 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:06:24.748 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:06:24.749 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "dispatched", 2023-06-03T20:06:24.749 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:00:42.825749+0000", 2023-06-03T20:06:24.749 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 8.5290000000000008e-06 2023-06-03T20:06:24.749 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:06:24.749 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:06:24.749 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "queued_for_pg", 2023-06-03T20:06:24.749 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:00:42.825758+0000", 2023-06-03T20:06:24.749 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.001807842 2023-06-03T20:06:24.750 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:06:24.750 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:06:24.750 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "reached_pg", 2023-06-03T20:06:24.750 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:00:42.827566+0000", 2023-06-03T20:06:24.750 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.00023524599999999999 2023-06-03T20:06:24.750 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:06:24.750 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:06:24.751 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "started", 2023-06-03T20:06:24.751 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:00:42.827801+0000", 2023-06-03T20:06:24.751 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.000193294 2023-06-03T20:06:24.751 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:06:24.751 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:06:24.751 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "waiting for subops from 1,5", 2023-06-03T20:06:24.751 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:00:42.827994+0000", 2023-06-03T20:06:24.752 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.00083248999999999997 2023-06-03T20:06:24.752 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:06:24.752 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:06:24.752 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "op_commit", 2023-06-03T20:06:24.752 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:00:42.828827+0000", 2023-06-03T20:06:24.752 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.00091845100000000003 2023-06-03T20:06:24.752 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:06:24.753 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:06:24.753 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:06:24.753 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:00:42.829745+0000", 2023-06-03T20:06:24.753 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.32217826399999999 2023-06-03T20:06:24.753 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:06:24.754 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:06:24.754 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:06:24.754 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:00:43.151924+0000", 2023-06-03T20:06:24.754 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.00018278700000000001 2023-06-03T20:06:24.755 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:06:24.755 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:06:24.755 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "commit_sent", 2023-06-03T20:06:24.756 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:00:43.152106+0000", 2023-06-03T20:06:24.756 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.000232737 2023-06-03T20:06:24.756 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:06:24.756 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:06:24.757 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "done", 2023-06-03T20:06:24.757 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:00:43.152339+0000", 2023-06-03T20:06:24.757 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.32671114699999998 2023-06-03T20:06:24.757 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:06:24.758 INFO:teuthology.orchestra.run.smithi164.stdout: ] 2023-06-03T20:06:24.758 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:06:24.758 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:06:24.758 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:06:24.759 INFO:teuthology.orchestra.run.smithi164.stdout: "description": "osd_op(client.45713.0:4726 9.3 9:c54f261f:::benchmark_data_smithi089_115580_object4725:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e131)", 2023-06-03T20:06:24.759 INFO:teuthology.orchestra.run.smithi164.stdout: "initiated_at": "2023-06-03T20:00:42.828289+0000", 2023-06-03T20:06:24.759 INFO:teuthology.orchestra.run.smithi164.stdout: "age": 341.8978457, 2023-06-03T20:06:24.760 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.32384883399999997, 2023-06-03T20:06:24.760 INFO:teuthology.orchestra.run.smithi164.stdout: "type_data": { 2023-06-03T20:06:24.760 INFO:teuthology.orchestra.run.smithi164.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-03T20:06:24.760 INFO:teuthology.orchestra.run.smithi164.stdout: "client_info": { 2023-06-03T20:06:24.760 INFO:teuthology.orchestra.run.smithi164.stdout: "client": "client.45713", 2023-06-03T20:06:24.761 INFO:teuthology.orchestra.run.smithi164.stdout: "client_addr": "172.21.15.89:0/2400821448", 2023-06-03T20:06:24.761 INFO:teuthology.orchestra.run.smithi164.stdout: "tid": 4726 2023-06-03T20:06:24.761 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:06:24.761 INFO:teuthology.orchestra.run.smithi164.stdout: "events": [ 2023-06-03T20:06:24.761 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:06:24.761 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "initiated", 2023-06-03T20:06:24.762 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:00:42.828289+0000", 2023-06-03T20:06:24.762 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0 2023-06-03T20:06:24.762 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:06:24.762 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:06:24.762 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "throttled", 2023-06-03T20:06:24.762 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:00:42.828289+0000", 2023-06-03T20:06:24.762 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 4294967295.9999986 2023-06-03T20:06:24.763 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:06:24.763 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:06:24.763 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "header_read", 2023-06-03T20:06:24.763 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:00:42.828287+0000", 2023-06-03T20:06:24.763 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 4.2537999999999998e-05 2023-06-03T20:06:24.763 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:06:24.763 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:06:24.763 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "all_read", 2023-06-03T20:06:24.764 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:00:42.828330+0000", 2023-06-03T20:06:24.764 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 1.3164e-05 2023-06-03T20:06:24.764 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:06:24.764 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:06:24.764 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "dispatched", 2023-06-03T20:06:24.764 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:00:42.828343+0000", 2023-06-03T20:06:24.764 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 7.6590000000000007e-06 2023-06-03T20:06:24.765 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:06:24.765 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:06:24.765 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "queued_for_pg", 2023-06-03T20:06:24.765 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:00:42.828351+0000", 2023-06-03T20:06:24.765 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 6.8156000000000003e-05 2023-06-03T20:06:24.765 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:06:24.765 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:06:24.765 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "reached_pg", 2023-06-03T20:06:24.766 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:00:42.828419+0000", 2023-06-03T20:06:24.766 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.000115407 2023-06-03T20:06:24.766 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:06:24.766 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:06:24.766 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "started", 2023-06-03T20:06:24.766 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:00:42.828534+0000", 2023-06-03T20:06:24.766 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.00021946300000000001 2023-06-03T20:06:24.766 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:06:24.767 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:06:24.767 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "waiting for subops from 1,3", 2023-06-03T20:06:24.767 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:00:42.828754+0000", 2023-06-03T20:06:24.767 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.00070476499999999999 2023-06-03T20:06:24.767 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:06:24.767 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:06:24.767 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "op_commit", 2023-06-03T20:06:24.767 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:00:42.829459+0000", 2023-06-03T20:06:24.768 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.002220863 2023-06-03T20:06:24.768 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:06:24.768 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:06:24.768 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:06:24.768 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:00:42.831679+0000", 2023-06-03T20:06:24.768 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.320294525 2023-06-03T20:06:24.768 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:06:24.768 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:06:24.769 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:06:24.769 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:00:43.151974+0000", 2023-06-03T20:06:24.769 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 9.5532000000000006e-05 2023-06-03T20:06:24.769 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:06:24.769 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:06:24.769 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "commit_sent", 2023-06-03T20:06:24.769 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:00:43.152069+0000", 2023-06-03T20:06:24.769 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 6.8398999999999993e-05 2023-06-03T20:06:24.770 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:06:24.770 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:06:24.770 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "done", 2023-06-03T20:06:24.770 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:00:43.152138+0000", 2023-06-03T20:06:24.770 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.32384883399999997 2023-06-03T20:06:24.770 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:06:24.770 INFO:teuthology.orchestra.run.smithi164.stdout: ] 2023-06-03T20:06:24.771 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:06:24.771 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:06:24.771 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:06:24.771 INFO:teuthology.orchestra.run.smithi164.stdout: "description": "osd_op(client.45713.0:4734 9.e 9:787a03b2:::benchmark_data_smithi089_115580_object4733:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e131)", 2023-06-03T20:06:24.771 INFO:teuthology.orchestra.run.smithi164.stdout: "initiated_at": "2023-06-03T20:00:42.831396+0000", 2023-06-03T20:06:24.771 INFO:teuthology.orchestra.run.smithi164.stdout: "age": 341.894738749, 2023-06-03T20:06:24.771 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.32256747499999999, 2023-06-03T20:06:24.771 INFO:teuthology.orchestra.run.smithi164.stdout: "type_data": { 2023-06-03T20:06:24.772 INFO:teuthology.orchestra.run.smithi164.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-03T20:06:24.772 INFO:teuthology.orchestra.run.smithi164.stdout: "client_info": { 2023-06-03T20:06:24.772 INFO:teuthology.orchestra.run.smithi164.stdout: "client": "client.45713", 2023-06-03T20:06:24.772 INFO:teuthology.orchestra.run.smithi164.stdout: "client_addr": "172.21.15.89:0/2400821448", 2023-06-03T20:06:24.772 INFO:teuthology.orchestra.run.smithi164.stdout: "tid": 4734 2023-06-03T20:06:24.772 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:06:24.772 INFO:teuthology.orchestra.run.smithi164.stdout: "events": [ 2023-06-03T20:06:24.773 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:06:24.773 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "initiated", 2023-06-03T20:06:24.773 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:00:42.831396+0000", 2023-06-03T20:06:24.773 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0 2023-06-03T20:06:24.773 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:06:24.773 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:06:24.773 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "throttled", 2023-06-03T20:06:24.773 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:00:42.831396+0000", 2023-06-03T20:06:24.774 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 4294967295.999999 2023-06-03T20:06:24.774 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:06:24.774 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:06:24.774 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "header_read", 2023-06-03T20:06:24.774 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:00:42.831395+0000", 2023-06-03T20:06:24.774 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 2.7807e-05 2023-06-03T20:06:24.774 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:06:24.774 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:06:24.775 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "all_read", 2023-06-03T20:06:24.775 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:00:42.831423+0000", 2023-06-03T20:06:24.775 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 9.3340000000000008e-06 2023-06-03T20:06:24.775 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:06:24.775 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:06:24.775 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "dispatched", 2023-06-03T20:06:24.775 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:00:42.831432+0000", 2023-06-03T20:06:24.775 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 5.0479999999999998e-06 2023-06-03T20:06:24.776 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:06:24.776 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:06:24.776 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "queued_for_pg", 2023-06-03T20:06:24.776 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:00:42.831437+0000", 2023-06-03T20:06:24.776 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 6.2229000000000003e-05 2023-06-03T20:06:24.776 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:06:24.776 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:06:24.776 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "reached_pg", 2023-06-03T20:06:24.777 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:00:42.831499+0000", 2023-06-03T20:06:24.777 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.00012256699999999999 2023-06-03T20:06:24.777 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:06:24.777 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:06:24.777 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "started", 2023-06-03T20:06:24.777 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:00:42.831622+0000", 2023-06-03T20:06:24.777 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.000179111 2023-06-03T20:06:24.778 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:06:24.778 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:06:24.778 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "waiting for subops from 1,4", 2023-06-03T20:06:24.778 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:00:42.831801+0000", 2023-06-03T20:06:24.778 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.00063648199999999995 2023-06-03T20:06:24.778 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:06:24.778 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:06:24.778 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "op_commit", 2023-06-03T20:06:24.779 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:00:42.832437+0000", 2023-06-03T20:06:24.779 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.001141747 2023-06-03T20:06:24.779 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:06:24.779 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:06:24.779 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:06:24.779 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:00:42.833579+0000", 2023-06-03T20:06:24.779 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.32012126600000002 2023-06-03T20:06:24.779 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:06:24.780 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:06:24.780 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:06:24.780 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:00:43.153700+0000", 2023-06-03T20:06:24.780 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.00014213499999999999 2023-06-03T20:06:24.780 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:06:24.780 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:06:24.780 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "commit_sent", 2023-06-03T20:06:24.781 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:00:43.153843+0000", 2023-06-03T20:06:24.781 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.00012088500000000001 2023-06-03T20:06:24.781 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:06:24.781 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:06:24.781 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "done", 2023-06-03T20:06:24.781 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:00:43.153963+0000", 2023-06-03T20:06:24.781 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.32256747499999999 2023-06-03T20:06:24.781 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:06:24.782 INFO:teuthology.orchestra.run.smithi164.stdout: ] 2023-06-03T20:06:24.782 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:06:24.782 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:06:24.782 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:06:24.782 INFO:teuthology.orchestra.run.smithi164.stdout: "description": "osd_op(client.45713.0:4740 9.d 9:b44318a5:::benchmark_data_smithi089_115580_object4739:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e131)", 2023-06-03T20:06:24.782 INFO:teuthology.orchestra.run.smithi164.stdout: "initiated_at": "2023-06-03T20:00:42.831742+0000", 2023-06-03T20:06:24.782 INFO:teuthology.orchestra.run.smithi164.stdout: "age": 341.894392987, 2023-06-03T20:06:24.782 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.32620720199999997, 2023-06-03T20:06:24.783 INFO:teuthology.orchestra.run.smithi164.stdout: "type_data": { 2023-06-03T20:06:24.783 INFO:teuthology.orchestra.run.smithi164.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-03T20:06:24.783 INFO:teuthology.orchestra.run.smithi164.stdout: "client_info": { 2023-06-03T20:06:24.783 INFO:teuthology.orchestra.run.smithi164.stdout: "client": "client.45713", 2023-06-03T20:06:24.783 INFO:teuthology.orchestra.run.smithi164.stdout: "client_addr": "172.21.15.89:0/2400821448", 2023-06-03T20:06:24.783 INFO:teuthology.orchestra.run.smithi164.stdout: "tid": 4740 2023-06-03T20:06:24.783 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:06:24.783 INFO:teuthology.orchestra.run.smithi164.stdout: "events": [ 2023-06-03T20:06:24.784 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:06:24.784 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "initiated", 2023-06-03T20:06:24.784 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:00:42.831742+0000", 2023-06-03T20:06:24.784 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0 2023-06-03T20:06:24.784 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:06:24.784 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:06:24.784 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "throttled", 2023-06-03T20:06:24.784 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:00:42.831742+0000", 2023-06-03T20:06:24.785 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 4294967295.9999862 2023-06-03T20:06:24.785 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:06:24.785 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:06:24.785 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "header_read", 2023-06-03T20:06:24.785 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:00:42.831728+0000", 2023-06-03T20:06:24.785 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 5.0816000000000003e-05 2023-06-03T20:06:24.785 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:06:24.785 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:06:24.786 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "all_read", 2023-06-03T20:06:24.786 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:00:42.831779+0000", 2023-06-03T20:06:24.786 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 1.1968e-05 2023-06-03T20:06:24.786 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:06:24.786 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:06:24.786 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "dispatched", 2023-06-03T20:06:24.786 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:00:42.831791+0000", 2023-06-03T20:06:24.786 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 7.1960000000000002e-06 2023-06-03T20:06:24.787 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:06:24.787 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:06:24.787 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "queued_for_pg", 2023-06-03T20:06:24.787 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:00:42.831798+0000", 2023-06-03T20:06:24.787 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 5.7410000000000001e-05 2023-06-03T20:06:24.788 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:06:24.788 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:06:24.788 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "reached_pg", 2023-06-03T20:06:24.788 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:00:42.831855+0000", 2023-06-03T20:06:24.788 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.000152058 2023-06-03T20:06:24.788 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:06:24.788 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:06:24.788 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "started", 2023-06-03T20:06:24.789 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:00:42.832007+0000", 2023-06-03T20:06:24.789 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.000195231 2023-06-03T20:06:24.789 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:06:24.789 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:06:24.789 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "waiting for subops from 1,5", 2023-06-03T20:06:24.789 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:00:42.832203+0000", 2023-06-03T20:06:24.789 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.00066036800000000004 2023-06-03T20:06:24.790 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:06:24.790 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:06:24.790 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "op_commit", 2023-06-03T20:06:24.790 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:00:42.832863+0000", 2023-06-03T20:06:24.790 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.000575159 2023-06-03T20:06:24.790 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:06:24.790 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:06:24.790 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:06:24.791 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:00:42.833438+0000", 2023-06-03T20:06:24.791 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.32334931500000003 2023-06-03T20:06:24.791 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:06:24.791 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:06:24.791 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:06:24.791 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:00:43.156787+0000", 2023-06-03T20:06:24.791 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.001101362 2023-06-03T20:06:24.791 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:06:24.792 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:06:24.792 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "commit_sent", 2023-06-03T20:06:24.792 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:00:43.157889+0000", 2023-06-03T20:06:24.792 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 6.0114000000000001e-05 2023-06-03T20:06:24.792 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:06:24.792 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:06:24.792 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "done", 2023-06-03T20:06:24.792 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:00:43.157949+0000", 2023-06-03T20:06:24.793 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.32620720199999997 2023-06-03T20:06:24.793 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:06:24.793 INFO:teuthology.orchestra.run.smithi164.stdout: ] 2023-06-03T20:06:24.793 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:06:24.793 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:06:24.793 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:06:24.793 INFO:teuthology.orchestra.run.smithi164.stdout: "description": "osd_op(client.45713.0:4743 9.d 9:b85321a2:::benchmark_data_smithi089_115580_object4742:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e131)", 2023-06-03T20:06:24.793 INFO:teuthology.orchestra.run.smithi164.stdout: "initiated_at": "2023-06-03T20:00:42.833720+0000", 2023-06-03T20:06:24.794 INFO:teuthology.orchestra.run.smithi164.stdout: "age": 341.89241507100002, 2023-06-03T20:06:24.794 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.32457217799999999, 2023-06-03T20:06:24.794 INFO:teuthology.orchestra.run.smithi164.stdout: "type_data": { 2023-06-03T20:06:24.794 INFO:teuthology.orchestra.run.smithi164.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-03T20:06:24.794 INFO:teuthology.orchestra.run.smithi164.stdout: "client_info": { 2023-06-03T20:06:24.794 INFO:teuthology.orchestra.run.smithi164.stdout: "client": "client.45713", 2023-06-03T20:06:24.794 INFO:teuthology.orchestra.run.smithi164.stdout: "client_addr": "172.21.15.89:0/2400821448", 2023-06-03T20:06:24.795 INFO:teuthology.orchestra.run.smithi164.stdout: "tid": 4743 2023-06-03T20:06:24.795 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:06:24.795 INFO:teuthology.orchestra.run.smithi164.stdout: "events": [ 2023-06-03T20:06:24.795 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:06:24.795 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "initiated", 2023-06-03T20:06:24.795 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:00:42.833720+0000", 2023-06-03T20:06:24.795 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0 2023-06-03T20:06:24.795 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:06:24.796 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:06:24.796 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "throttled", 2023-06-03T20:06:24.796 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:00:42.833720+0000", 2023-06-03T20:06:24.796 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 4294967295.999999 2023-06-03T20:06:24.796 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:06:24.796 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:06:24.796 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "header_read", 2023-06-03T20:06:24.796 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:00:42.833719+0000", 2023-06-03T20:06:24.797 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 5.9607e-05 2023-06-03T20:06:24.797 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:06:24.797 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:06:24.797 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "all_read", 2023-06-03T20:06:24.797 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:00:42.833778+0000", 2023-06-03T20:06:24.797 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 9.3170000000000008e-06 2023-06-03T20:06:24.797 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:06:24.798 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:06:24.798 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "dispatched", 2023-06-03T20:06:24.798 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:00:42.833788+0000", 2023-06-03T20:06:24.798 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 4.6280000000000004e-06 2023-06-03T20:06:24.798 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:06:24.798 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:06:24.798 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "queued_for_pg", 2023-06-03T20:06:24.798 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:00:42.833792+0000", 2023-06-03T20:06:24.799 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 4.3098e-05 2023-06-03T20:06:24.799 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:06:24.799 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:06:24.799 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "reached_pg", 2023-06-03T20:06:24.799 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:00:42.833835+0000", 2023-06-03T20:06:24.799 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.00015837300000000001 2023-06-03T20:06:24.799 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:06:24.799 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:06:24.800 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "started", 2023-06-03T20:06:24.800 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:00:42.833994+0000", 2023-06-03T20:06:24.800 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.00025586899999999999 2023-06-03T20:06:24.800 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:06:24.800 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:06:24.800 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "waiting for subops from 1,5", 2023-06-03T20:06:24.800 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:00:42.834250+0000", 2023-06-03T20:06:24.800 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.00099802300000000001 2023-06-03T20:06:24.801 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:06:24.801 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:06:24.801 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:06:24.801 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:00:42.835248+0000", 2023-06-03T20:06:24.801 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.000212167 2023-06-03T20:06:24.801 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:06:24.801 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:06:24.801 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "op_commit", 2023-06-03T20:06:24.802 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:00:42.835460+0000", 2023-06-03T20:06:24.802 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.322598475 2023-06-03T20:06:24.802 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:06:24.802 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:06:24.802 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:06:24.802 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:00:43.158058+0000", 2023-06-03T20:06:24.802 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.000124535 2023-06-03T20:06:24.803 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:06:24.803 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:06:24.803 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "commit_sent", 2023-06-03T20:06:24.803 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:00:43.158183+0000", 2023-06-03T20:06:24.803 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.000109067 2023-06-03T20:06:24.803 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:06:24.803 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:06:24.803 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "done", 2023-06-03T20:06:24.804 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:00:43.158292+0000", 2023-06-03T20:06:24.804 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.32457217799999999 2023-06-03T20:06:24.804 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:06:24.804 INFO:teuthology.orchestra.run.smithi164.stdout: ] 2023-06-03T20:06:24.804 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:06:24.804 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:06:24.804 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:06:24.804 INFO:teuthology.orchestra.run.smithi164.stdout: "description": "osd_op(client.45713.0:4745 9.e 9:72464cee:::benchmark_data_smithi089_115580_object4744:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e131)", 2023-06-03T20:06:24.805 INFO:teuthology.orchestra.run.smithi164.stdout: "initiated_at": "2023-06-03T20:00:42.835097+0000", 2023-06-03T20:06:24.805 INFO:teuthology.orchestra.run.smithi164.stdout: "age": 341.89103767300003, 2023-06-03T20:06:24.805 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.32042158900000001, 2023-06-03T20:06:24.805 INFO:teuthology.orchestra.run.smithi164.stdout: "type_data": { 2023-06-03T20:06:24.805 INFO:teuthology.orchestra.run.smithi164.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-03T20:06:24.805 INFO:teuthology.orchestra.run.smithi164.stdout: "client_info": { 2023-06-03T20:06:24.805 INFO:teuthology.orchestra.run.smithi164.stdout: "client": "client.45713", 2023-06-03T20:06:24.805 INFO:teuthology.orchestra.run.smithi164.stdout: "client_addr": "172.21.15.89:0/2400821448", 2023-06-03T20:06:24.806 INFO:teuthology.orchestra.run.smithi164.stdout: "tid": 4745 2023-06-03T20:06:24.806 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:06:24.806 INFO:teuthology.orchestra.run.smithi164.stdout: "events": [ 2023-06-03T20:06:24.806 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:06:24.806 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "initiated", 2023-06-03T20:06:24.806 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:00:42.835097+0000", 2023-06-03T20:06:24.806 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0 2023-06-03T20:06:24.806 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:06:24.807 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:06:24.807 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "throttled", 2023-06-03T20:06:24.807 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:00:42.835097+0000", 2023-06-03T20:06:24.807 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 4294967295.9999986 2023-06-03T20:06:24.807 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:06:24.807 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:06:24.807 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "header_read", 2023-06-03T20:06:24.807 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:00:42.835096+0000", 2023-06-03T20:06:24.808 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 6.9690000000000005e-05 2023-06-03T20:06:24.808 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:06:24.808 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:06:24.808 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "all_read", 2023-06-03T20:06:24.808 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:00:42.835165+0000", 2023-06-03T20:06:24.808 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 1.3548e-05 2023-06-03T20:06:24.808 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:06:24.809 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:06:24.809 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "dispatched", 2023-06-03T20:06:24.809 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:00:42.835179+0000", 2023-06-03T20:06:24.809 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 7.4599999999999997e-06 2023-06-03T20:06:24.809 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:06:24.809 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:06:24.809 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "queued_for_pg", 2023-06-03T20:06:24.809 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:00:42.835186+0000", 2023-06-03T20:06:24.810 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 6.6892999999999998e-05 2023-06-03T20:06:24.810 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:06:24.810 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:06:24.810 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "reached_pg", 2023-06-03T20:06:24.810 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:00:42.835253+0000", 2023-06-03T20:06:24.810 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.00014683 2023-06-03T20:06:24.810 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:06:24.810 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:06:24.811 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "started", 2023-06-03T20:06:24.811 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:00:42.835400+0000", 2023-06-03T20:06:24.811 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.000127312 2023-06-03T20:06:24.811 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:06:24.811 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:06:24.811 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "waiting for subops from 1,4", 2023-06-03T20:06:24.811 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:00:42.835527+0000", 2023-06-03T20:06:24.811 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.00045894000000000002 2023-06-03T20:06:24.812 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:06:24.812 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:06:24.812 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "op_commit", 2023-06-03T20:06:24.812 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:00:42.835986+0000", 2023-06-03T20:06:24.812 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.00035262299999999999 2023-06-03T20:06:24.812 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:06:24.812 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:06:24.813 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:06:24.813 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:00:42.836339+0000", 2023-06-03T20:06:24.813 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.318950503 2023-06-03T20:06:24.813 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:06:24.813 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:06:24.813 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:06:24.813 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:00:43.155290+0000", 2023-06-03T20:06:24.813 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.00013186099999999999 2023-06-03T20:06:24.814 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:06:24.814 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:06:24.814 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "commit_sent", 2023-06-03T20:06:24.814 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:00:43.155421+0000", 2023-06-03T20:06:24.814 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 9.7242000000000002e-05 2023-06-03T20:06:24.815 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:06:24.815 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:06:24.815 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "done", 2023-06-03T20:06:24.815 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:00:43.155519+0000", 2023-06-03T20:06:24.815 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.32042158900000001 2023-06-03T20:06:24.815 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:06:24.815 INFO:teuthology.orchestra.run.smithi164.stdout: ] 2023-06-03T20:06:24.815 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:06:24.816 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:06:24.816 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:06:24.816 INFO:teuthology.orchestra.run.smithi164.stdout: "description": "osd_op(client.45713.0:20535 9.3 9:c75f0c6c:::benchmark_data_smithi089_115580_object20534:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e131)", 2023-06-03T20:06:24.816 INFO:teuthology.orchestra.run.smithi164.stdout: "initiated_at": "2023-06-03T20:00:50.375556+0000", 2023-06-03T20:06:24.816 INFO:teuthology.orchestra.run.smithi164.stdout: "age": 334.35057858800002, 2023-06-03T20:06:24.816 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.26295085499999998, 2023-06-03T20:06:24.816 INFO:teuthology.orchestra.run.smithi164.stdout: "type_data": { 2023-06-03T20:06:24.817 INFO:teuthology.orchestra.run.smithi164.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-03T20:06:24.817 INFO:teuthology.orchestra.run.smithi164.stdout: "client_info": { 2023-06-03T20:06:24.817 INFO:teuthology.orchestra.run.smithi164.stdout: "client": "client.45713", 2023-06-03T20:06:24.817 INFO:teuthology.orchestra.run.smithi164.stdout: "client_addr": "172.21.15.89:0/2400821448", 2023-06-03T20:06:24.817 INFO:teuthology.orchestra.run.smithi164.stdout: "tid": 20535 2023-06-03T20:06:24.817 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:06:24.817 INFO:teuthology.orchestra.run.smithi164.stdout: "events": [ 2023-06-03T20:06:24.817 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:06:24.818 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "initiated", 2023-06-03T20:06:24.818 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:00:50.375556+0000", 2023-06-03T20:06:24.818 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0 2023-06-03T20:06:24.818 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:06:24.818 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:06:24.818 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "throttled", 2023-06-03T20:06:24.818 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:00:50.375556+0000", 2023-06-03T20:06:24.818 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 4294967295.9999981 2023-06-03T20:06:24.819 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:06:24.819 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:06:24.819 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "header_read", 2023-06-03T20:06:24.819 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:00:50.375554+0000", 2023-06-03T20:06:24.819 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 8.8891e-05 2023-06-03T20:06:24.819 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:06:24.819 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:06:24.820 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "all_read", 2023-06-03T20:06:24.820 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:00:50.375643+0000", 2023-06-03T20:06:24.820 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 1.4572999999999999e-05 2023-06-03T20:06:24.820 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:06:24.820 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:06:24.820 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "dispatched", 2023-06-03T20:06:24.820 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:00:50.375658+0000", 2023-06-03T20:06:24.820 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 5.9340000000000003e-06 2023-06-03T20:06:24.821 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:06:24.821 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:06:24.821 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "queued_for_pg", 2023-06-03T20:06:24.821 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:00:50.375664+0000", 2023-06-03T20:06:24.821 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 4.2064000000000002e-05 2023-06-03T20:06:24.821 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:06:24.821 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:06:24.821 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "reached_pg", 2023-06-03T20:06:24.822 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:00:50.375706+0000", 2023-06-03T20:06:24.822 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.000102228 2023-06-03T20:06:24.822 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:06:24.822 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:06:24.822 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "started", 2023-06-03T20:06:24.822 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:00:50.375808+0000", 2023-06-03T20:06:24.822 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.000248953 2023-06-03T20:06:24.823 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:06:24.823 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:06:24.823 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "waiting for subops from 1,3", 2023-06-03T20:06:24.823 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:00:50.376057+0000", 2023-06-03T20:06:24.823 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.001330936 2023-06-03T20:06:24.823 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:06:24.823 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:06:24.823 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "op_commit", 2023-06-03T20:06:24.824 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:00:50.377388+0000", 2023-06-03T20:06:24.824 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.26057239700000001 2023-06-03T20:06:24.824 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:06:24.824 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:06:24.824 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:06:24.824 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:00:50.637960+0000", 2023-06-03T20:06:24.824 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.00030453799999999998 2023-06-03T20:06:24.824 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:06:24.825 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:06:24.825 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:06:24.825 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:00:50.638265+0000", 2023-06-03T20:06:24.825 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.00012940599999999999 2023-06-03T20:06:24.825 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:06:24.825 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:06:24.825 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "commit_sent", 2023-06-03T20:06:24.825 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:00:50.638394+0000", 2023-06-03T20:06:24.826 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.00011282599999999999 2023-06-03T20:06:24.826 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:06:24.826 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:06:24.826 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "done", 2023-06-03T20:06:24.826 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:00:50.638507+0000", 2023-06-03T20:06:24.826 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.26295085499999998 2023-06-03T20:06:24.826 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:06:24.826 INFO:teuthology.orchestra.run.smithi164.stdout: ] 2023-06-03T20:06:24.827 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:06:24.827 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:06:24.827 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:06:24.827 INFO:teuthology.orchestra.run.smithi164.stdout: "description": "osd_op(client.45713.0:21563 9.e 9:7ea57ec1:::benchmark_data_smithi089_115580_object21562:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e131)", 2023-06-03T20:06:24.827 INFO:teuthology.orchestra.run.smithi164.stdout: "initiated_at": "2023-06-03T20:00:51.114616+0000", 2023-06-03T20:06:24.827 INFO:teuthology.orchestra.run.smithi164.stdout: "age": 333.61151829599999, 2023-06-03T20:06:24.827 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.172942812, 2023-06-03T20:06:24.828 INFO:teuthology.orchestra.run.smithi164.stdout: "type_data": { 2023-06-03T20:06:24.828 INFO:teuthology.orchestra.run.smithi164.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-03T20:06:24.828 INFO:teuthology.orchestra.run.smithi164.stdout: "client_info": { 2023-06-03T20:06:24.828 INFO:teuthology.orchestra.run.smithi164.stdout: "client": "client.45713", 2023-06-03T20:06:24.828 INFO:teuthology.orchestra.run.smithi164.stdout: "client_addr": "172.21.15.89:0/2400821448", 2023-06-03T20:06:24.828 INFO:teuthology.orchestra.run.smithi164.stdout: "tid": 21563 2023-06-03T20:06:24.828 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:06:24.828 INFO:teuthology.orchestra.run.smithi164.stdout: "events": [ 2023-06-03T20:06:24.829 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:06:24.829 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "initiated", 2023-06-03T20:06:24.829 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:00:51.114616+0000", 2023-06-03T20:06:24.829 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0 2023-06-03T20:06:24.829 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:06:24.829 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:06:24.830 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "throttled", 2023-06-03T20:06:24.830 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:00:51.114616+0000", 2023-06-03T20:06:24.830 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 4294967295.9999967 2023-06-03T20:06:24.830 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:06:24.830 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:06:24.830 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "header_read", 2023-06-03T20:06:24.830 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:00:51.114613+0000", 2023-06-03T20:06:24.831 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.00016197100000000001 2023-06-03T20:06:24.831 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:06:24.831 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:06:24.831 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "all_read", 2023-06-03T20:06:24.831 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:00:51.114775+0000", 2023-06-03T20:06:24.831 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.00025914699999999999 2023-06-03T20:06:24.831 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:06:24.831 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:06:24.832 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "dispatched", 2023-06-03T20:06:24.832 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:00:51.115034+0000", 2023-06-03T20:06:24.832 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 5.0270000000000003e-06 2023-06-03T20:06:24.832 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:06:24.832 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:06:24.832 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "queued_for_pg", 2023-06-03T20:06:24.832 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:00:51.115039+0000", 2023-06-03T20:06:24.832 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 4.3334000000000002e-05 2023-06-03T20:06:24.833 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:06:24.833 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:06:24.833 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "reached_pg", 2023-06-03T20:06:24.833 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:00:51.115083+0000", 2023-06-03T20:06:24.833 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.00016154200000000001 2023-06-03T20:06:24.833 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:06:24.833 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:06:24.834 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "started", 2023-06-03T20:06:24.834 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:00:51.115244+0000", 2023-06-03T20:06:24.834 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.000169254 2023-06-03T20:06:24.834 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:06:24.834 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:06:24.834 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "waiting for subops from 1,4", 2023-06-03T20:06:24.834 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:00:51.115413+0000", 2023-06-03T20:06:24.834 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.001141862 2023-06-03T20:06:24.835 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:06:24.835 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:06:24.835 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "op_commit", 2023-06-03T20:06:24.835 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:00:51.116555+0000", 2023-06-03T20:06:24.835 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.0012849630000000001 2023-06-03T20:06:24.835 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:06:24.835 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:06:24.835 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:06:24.836 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:00:51.117840+0000", 2023-06-03T20:06:24.836 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.169459212 2023-06-03T20:06:24.836 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:06:24.836 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:06:24.836 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:06:24.836 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:00:51.287299+0000", 2023-06-03T20:06:24.836 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.000145568 2023-06-03T20:06:24.836 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:06:24.837 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:06:24.837 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "commit_sent", 2023-06-03T20:06:24.837 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:00:51.287445+0000", 2023-06-03T20:06:24.837 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.000114162 2023-06-03T20:06:24.837 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:06:24.837 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:06:24.837 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "done", 2023-06-03T20:06:24.838 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:00:51.287559+0000", 2023-06-03T20:06:24.838 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.172942812 2023-06-03T20:06:24.838 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:06:24.838 INFO:teuthology.orchestra.run.smithi164.stdout: ] 2023-06-03T20:06:24.838 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:06:24.838 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:06:24.838 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:06:24.838 INFO:teuthology.orchestra.run.smithi164.stdout: "description": "osd_op(client.45713.0:21576 9.3 9:c4dfdef9:::benchmark_data_smithi089_115580_object21575:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e131)", 2023-06-03T20:06:24.839 INFO:teuthology.orchestra.run.smithi164.stdout: "initiated_at": "2023-06-03T20:00:51.118241+0000", 2023-06-03T20:06:24.839 INFO:teuthology.orchestra.run.smithi164.stdout: "age": 333.60789416300003, 2023-06-03T20:06:24.839 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.16893076200000001, 2023-06-03T20:06:24.839 INFO:teuthology.orchestra.run.smithi164.stdout: "type_data": { 2023-06-03T20:06:24.839 INFO:teuthology.orchestra.run.smithi164.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-03T20:06:24.839 INFO:teuthology.orchestra.run.smithi164.stdout: "client_info": { 2023-06-03T20:06:24.839 INFO:teuthology.orchestra.run.smithi164.stdout: "client": "client.45713", 2023-06-03T20:06:24.839 INFO:teuthology.orchestra.run.smithi164.stdout: "client_addr": "172.21.15.89:0/2400821448", 2023-06-03T20:06:24.840 INFO:teuthology.orchestra.run.smithi164.stdout: "tid": 21576 2023-06-03T20:06:24.840 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:06:24.840 INFO:teuthology.orchestra.run.smithi164.stdout: "events": [ 2023-06-03T20:06:24.840 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:06:24.840 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "initiated", 2023-06-03T20:06:24.840 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:00:51.118241+0000", 2023-06-03T20:06:24.840 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0 2023-06-03T20:06:24.840 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:06:24.841 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:06:24.841 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "throttled", 2023-06-03T20:06:24.841 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:00:51.118241+0000", 2023-06-03T20:06:24.841 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 4294967295.9999986 2023-06-03T20:06:24.841 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:06:24.841 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:06:24.841 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "header_read", 2023-06-03T20:06:24.841 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:00:51.118239+0000", 2023-06-03T20:06:24.842 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 3.7568999999999999e-05 2023-06-03T20:06:24.842 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:06:24.842 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:06:24.842 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "all_read", 2023-06-03T20:06:24.842 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:00:51.118277+0000", 2023-06-03T20:06:24.842 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 1.4195e-05 2023-06-03T20:06:24.842 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:06:24.842 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:06:24.843 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "dispatched", 2023-06-03T20:06:24.843 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:00:51.118291+0000", 2023-06-03T20:06:24.843 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 5.789e-06 2023-06-03T20:06:24.843 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:06:24.843 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:06:24.843 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "queued_for_pg", 2023-06-03T20:06:24.843 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:00:51.118297+0000", 2023-06-03T20:06:24.843 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 5.4463999999999999e-05 2023-06-03T20:06:24.844 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:06:24.844 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:06:24.844 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "reached_pg", 2023-06-03T20:06:24.844 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:00:51.118351+0000", 2023-06-03T20:06:24.844 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.00013280400000000001 2023-06-03T20:06:24.844 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:06:24.844 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:06:24.845 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "started", 2023-06-03T20:06:24.845 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:00:51.118484+0000", 2023-06-03T20:06:24.845 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.00017989199999999999 2023-06-03T20:06:24.845 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:06:24.845 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:06:24.845 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "waiting for subops from 1,3", 2023-06-03T20:06:24.845 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:00:51.118664+0000", 2023-06-03T20:06:24.845 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.001074282 2023-06-03T20:06:24.846 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:06:24.846 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:06:24.846 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "op_commit", 2023-06-03T20:06:24.846 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:00:51.119738+0000", 2023-06-03T20:06:24.846 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.00028460699999999998 2023-06-03T20:06:24.846 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:06:24.846 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:06:24.846 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:06:24.847 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:00:51.120023+0000", 2023-06-03T20:06:24.847 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.16702969500000001 2023-06-03T20:06:24.847 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:06:24.847 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:06:24.847 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:06:24.847 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:00:51.287052+0000", 2023-06-03T20:06:24.847 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 6.3142e-05 2023-06-03T20:06:24.847 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:06:24.848 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:06:24.848 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "commit_sent", 2023-06-03T20:06:24.848 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:00:51.287116+0000", 2023-06-03T20:06:24.848 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 5.5784e-05 2023-06-03T20:06:24.848 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:06:24.848 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:06:24.848 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "done", 2023-06-03T20:06:24.848 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:00:51.287171+0000", 2023-06-03T20:06:24.849 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.16893076200000001 2023-06-03T20:06:24.849 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:06:24.849 INFO:teuthology.orchestra.run.smithi164.stdout: ] 2023-06-03T20:06:24.849 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:06:24.849 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:06:24.849 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:06:24.849 INFO:teuthology.orchestra.run.smithi164.stdout: "description": "osd_op(client.45713.0:21588 9.d 9:b722800d:::benchmark_data_smithi089_115580_object21587:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e131)", 2023-06-03T20:06:24.849 INFO:teuthology.orchestra.run.smithi164.stdout: "initiated_at": "2023-06-03T20:00:51.135996+0000", 2023-06-03T20:06:24.850 INFO:teuthology.orchestra.run.smithi164.stdout: "age": 333.59013906299998, 2023-06-03T20:06:24.850 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.15122136, 2023-06-03T20:06:24.850 INFO:teuthology.orchestra.run.smithi164.stdout: "type_data": { 2023-06-03T20:06:24.850 INFO:teuthology.orchestra.run.smithi164.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-03T20:06:24.850 INFO:teuthology.orchestra.run.smithi164.stdout: "client_info": { 2023-06-03T20:06:24.850 INFO:teuthology.orchestra.run.smithi164.stdout: "client": "client.45713", 2023-06-03T20:06:24.850 INFO:teuthology.orchestra.run.smithi164.stdout: "client_addr": "172.21.15.89:0/2400821448", 2023-06-03T20:06:24.851 INFO:teuthology.orchestra.run.smithi164.stdout: "tid": 21588 2023-06-03T20:06:24.851 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:06:24.851 INFO:teuthology.orchestra.run.smithi164.stdout: "events": [ 2023-06-03T20:06:24.851 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:06:24.851 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "initiated", 2023-06-03T20:06:24.851 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:00:51.135996+0000", 2023-06-03T20:06:24.851 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0 2023-06-03T20:06:24.851 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:06:24.852 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:06:24.852 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "throttled", 2023-06-03T20:06:24.852 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:00:51.135996+0000", 2023-06-03T20:06:24.852 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 4294967295.9999981 2023-06-03T20:06:24.852 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:06:24.852 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:06:24.852 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "header_read", 2023-06-03T20:06:24.852 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:00:51.135993+0000", 2023-06-03T20:06:24.853 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 5.9590999999999999e-05 2023-06-03T20:06:24.853 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:06:24.853 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:06:24.853 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "all_read", 2023-06-03T20:06:24.853 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:00:51.136053+0000", 2023-06-03T20:06:24.853 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 2.0475999999999998e-05 2023-06-03T20:06:24.853 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:06:24.853 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:06:24.854 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "dispatched", 2023-06-03T20:06:24.854 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:00:51.136074+0000", 2023-06-03T20:06:24.854 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 9.4220000000000006e-06 2023-06-03T20:06:24.854 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:06:24.854 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:06:24.854 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "queued_for_pg", 2023-06-03T20:06:24.854 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:00:51.136083+0000", 2023-06-03T20:06:24.854 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 8.9332000000000005e-05 2023-06-03T20:06:24.855 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:06:24.855 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:06:24.855 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "reached_pg", 2023-06-03T20:06:24.855 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:00:51.136172+0000", 2023-06-03T20:06:24.855 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.00019292699999999999 2023-06-03T20:06:24.855 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:06:24.855 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:06:24.855 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "started", 2023-06-03T20:06:24.856 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:00:51.136365+0000", 2023-06-03T20:06:24.856 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.00023216900000000001 2023-06-03T20:06:24.856 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:06:24.856 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:06:24.856 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "waiting for subops from 1,5", 2023-06-03T20:06:24.856 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:00:51.136597+0000", 2023-06-03T20:06:24.856 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.001325285 2023-06-03T20:06:24.857 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:06:24.857 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:06:24.857 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:06:24.857 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:00:51.137923+0000", 2023-06-03T20:06:24.857 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 7.0567000000000004e-05 2023-06-03T20:06:24.857 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:06:24.857 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:06:24.857 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "op_commit", 2023-06-03T20:06:24.858 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:00:51.137993+0000", 2023-06-03T20:06:24.858 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.14909755699999999 2023-06-03T20:06:24.858 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:06:24.858 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:06:24.858 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:06:24.858 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:00:51.287091+0000", 2023-06-03T20:06:24.858 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 5.6832000000000001e-05 2023-06-03T20:06:24.858 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:06:24.859 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:06:24.859 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "commit_sent", 2023-06-03T20:06:24.859 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:00:51.287148+0000", 2023-06-03T20:06:24.859 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 6.9333999999999997e-05 2023-06-03T20:06:24.859 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:06:24.859 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:06:24.859 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "done", 2023-06-03T20:06:24.859 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:00:51.287217+0000", 2023-06-03T20:06:24.860 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.15122136 2023-06-03T20:06:24.860 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:06:24.860 INFO:teuthology.orchestra.run.smithi164.stdout: ] 2023-06-03T20:06:24.860 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:06:24.860 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:06:24.860 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:06:24.860 INFO:teuthology.orchestra.run.smithi164.stdout: "description": "osd_op(client.45713.0:44618 9.3 9:cbd5d8c6:::benchmark_data_smithi089_115580_object44617:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e132)", 2023-06-03T20:06:24.860 INFO:teuthology.orchestra.run.smithi164.stdout: "initiated_at": "2023-06-03T20:01:06.122166+0000", 2023-06-03T20:06:24.861 INFO:teuthology.orchestra.run.smithi164.stdout: "age": 318.60396848200003, 2023-06-03T20:06:24.861 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.17021108700000001, 2023-06-03T20:06:24.861 INFO:teuthology.orchestra.run.smithi164.stdout: "type_data": { 2023-06-03T20:06:24.861 INFO:teuthology.orchestra.run.smithi164.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-03T20:06:24.861 INFO:teuthology.orchestra.run.smithi164.stdout: "client_info": { 2023-06-03T20:06:24.861 INFO:teuthology.orchestra.run.smithi164.stdout: "client": "client.45713", 2023-06-03T20:06:24.861 INFO:teuthology.orchestra.run.smithi164.stdout: "client_addr": "172.21.15.89:0/2400821448", 2023-06-03T20:06:24.861 INFO:teuthology.orchestra.run.smithi164.stdout: "tid": 44618 2023-06-03T20:06:24.862 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:06:24.862 INFO:teuthology.orchestra.run.smithi164.stdout: "events": [ 2023-06-03T20:06:24.862 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:06:24.862 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "initiated", 2023-06-03T20:06:24.862 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:01:06.122166+0000", 2023-06-03T20:06:24.862 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0 2023-06-03T20:06:24.862 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:06:24.863 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:06:24.863 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "throttled", 2023-06-03T20:06:24.863 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:01:06.122166+0000", 2023-06-03T20:06:24.863 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 4294967295.9999981 2023-06-03T20:06:24.863 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:06:24.863 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:06:24.863 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "header_read", 2023-06-03T20:06:24.863 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:01:06.122164+0000", 2023-06-03T20:06:24.864 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 8.6347000000000005e-05 2023-06-03T20:06:24.864 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:06:24.864 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:06:24.864 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "all_read", 2023-06-03T20:06:24.864 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:01:06.122251+0000", 2023-06-03T20:06:24.864 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 1.7439e-05 2023-06-03T20:06:24.864 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:06:24.864 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:06:24.865 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "dispatched", 2023-06-03T20:06:24.865 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:01:06.122268+0000", 2023-06-03T20:06:24.865 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 8.6389999999999998e-06 2023-06-03T20:06:24.865 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:06:24.865 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:06:24.865 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "queued_for_pg", 2023-06-03T20:06:24.865 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:01:06.122277+0000", 2023-06-03T20:06:24.865 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 6.1155999999999995e-05 2023-06-03T20:06:24.866 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:06:24.866 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:06:24.866 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "reached_pg", 2023-06-03T20:06:24.866 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:01:06.122338+0000", 2023-06-03T20:06:24.866 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.000146584 2023-06-03T20:06:24.866 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:06:24.866 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:06:24.866 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "started", 2023-06-03T20:06:24.867 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:01:06.122485+0000", 2023-06-03T20:06:24.867 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.000186007 2023-06-03T20:06:24.867 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:06:24.867 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:06:24.867 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "waiting for subops from 1,3", 2023-06-03T20:06:24.867 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:01:06.122671+0000", 2023-06-03T20:06:24.867 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.0020083229999999998 2023-06-03T20:06:24.867 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:06:24.868 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:06:24.868 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "op_commit", 2023-06-03T20:06:24.868 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:01:06.124679+0000", 2023-06-03T20:06:24.868 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.00233986 2023-06-03T20:06:24.868 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:06:24.868 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:06:24.868 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:06:24.869 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:01:06.127019+0000", 2023-06-03T20:06:24.869 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.16519372299999999 2023-06-03T20:06:24.869 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:06:24.869 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:06:24.869 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:06:24.869 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:01:06.292213+0000", 2023-06-03T20:06:24.869 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 9.8429000000000002e-05 2023-06-03T20:06:24.869 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:06:24.870 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:06:24.870 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "commit_sent", 2023-06-03T20:06:24.870 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:01:06.292311+0000", 2023-06-03T20:06:24.870 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 6.6317999999999998e-05 2023-06-03T20:06:24.870 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:06:24.870 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:06:24.870 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "done", 2023-06-03T20:06:24.871 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:01:06.292377+0000", 2023-06-03T20:06:24.871 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.17021108700000001 2023-06-03T20:06:24.871 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:06:24.871 INFO:teuthology.orchestra.run.smithi164.stdout: ] 2023-06-03T20:06:24.871 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:06:24.871 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:06:24.871 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:06:24.871 INFO:teuthology.orchestra.run.smithi164.stdout: "description": "osd_op(client.45713.0:139103 9.3 9:c7e51c1d:::benchmark_data_smithi089_115580_object28969:head [delete] snapc 0=[] ondisk+write+known_if_redirected e138)", 2023-06-03T20:06:24.872 INFO:teuthology.orchestra.run.smithi164.stdout: "initiated_at": "2023-06-03T20:02:49.137720+0000", 2023-06-03T20:06:24.872 INFO:teuthology.orchestra.run.smithi164.stdout: "age": 215.58841454500001, 2023-06-03T20:06:24.872 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.15283118400000001, 2023-06-03T20:06:24.872 INFO:teuthology.orchestra.run.smithi164.stdout: "type_data": { 2023-06-03T20:06:24.872 INFO:teuthology.orchestra.run.smithi164.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-03T20:06:24.872 INFO:teuthology.orchestra.run.smithi164.stdout: "client_info": { 2023-06-03T20:06:24.872 INFO:teuthology.orchestra.run.smithi164.stdout: "client": "client.45713", 2023-06-03T20:06:24.872 INFO:teuthology.orchestra.run.smithi164.stdout: "client_addr": "172.21.15.89:0/2400821448", 2023-06-03T20:06:24.873 INFO:teuthology.orchestra.run.smithi164.stdout: "tid": 139103 2023-06-03T20:06:24.873 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:06:24.873 INFO:teuthology.orchestra.run.smithi164.stdout: "events": [ 2023-06-03T20:06:24.873 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:06:24.873 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "initiated", 2023-06-03T20:06:24.873 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:02:49.137720+0000", 2023-06-03T20:06:24.873 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0 2023-06-03T20:06:24.874 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:06:24.874 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:06:24.874 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "throttled", 2023-06-03T20:06:24.874 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:02:49.137720+0000", 2023-06-03T20:06:24.874 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 4294967295.9999986 2023-06-03T20:06:24.874 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:06:24.874 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:06:24.874 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "header_read", 2023-06-03T20:06:24.875 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:02:49.137719+0000", 2023-06-03T20:06:24.875 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 1.7237000000000001e-05 2023-06-03T20:06:24.875 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:06:24.875 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:06:24.875 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "all_read", 2023-06-03T20:06:24.875 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:02:49.137736+0000", 2023-06-03T20:06:24.875 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 1.1996e-05 2023-06-03T20:06:24.875 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:06:24.876 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:06:24.876 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "dispatched", 2023-06-03T20:06:24.876 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:02:49.137748+0000", 2023-06-03T20:06:24.876 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 6.0820000000000004e-06 2023-06-03T20:06:24.876 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:06:24.876 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:06:24.876 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "queued_for_pg", 2023-06-03T20:06:24.876 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:02:49.137754+0000", 2023-06-03T20:06:24.877 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 4.0318000000000003e-05 2023-06-03T20:06:24.877 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:06:24.877 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:06:24.877 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "reached_pg", 2023-06-03T20:06:24.877 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:02:49.137794+0000", 2023-06-03T20:06:24.877 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.00013068199999999999 2023-06-03T20:06:24.877 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:06:24.877 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:06:24.878 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "started", 2023-06-03T20:06:24.878 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:02:49.137925+0000", 2023-06-03T20:06:24.878 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.000147126 2023-06-03T20:06:24.878 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:06:24.878 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:06:24.878 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "waiting for subops from 1,3", 2023-06-03T20:06:24.878 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:02:49.138072+0000", 2023-06-03T20:06:24.878 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.0081767469999999998 2023-06-03T20:06:24.879 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:06:24.879 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:06:24.879 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:06:24.879 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:02:49.146249+0000", 2023-06-03T20:06:24.879 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.000118551 2023-06-03T20:06:24.879 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:06:24.879 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:06:24.879 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "op_commit", 2023-06-03T20:06:24.880 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:02:49.146367+0000", 2023-06-03T20:06:24.880 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.14403329500000001 2023-06-03T20:06:24.880 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:06:24.880 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:06:24.880 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:06:24.880 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:02:49.290401+0000", 2023-06-03T20:06:24.880 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.00010476699999999999 2023-06-03T20:06:24.881 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:06:24.881 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:06:24.881 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "commit_sent", 2023-06-03T20:06:24.881 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:02:49.290505+0000", 2023-06-03T20:06:24.881 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 4.5881000000000003e-05 2023-06-03T20:06:24.881 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:06:24.881 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:06:24.881 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "done", 2023-06-03T20:06:24.882 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:02:49.290551+0000", 2023-06-03T20:06:24.882 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.15283118400000001 2023-06-03T20:06:24.882 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:06:24.882 INFO:teuthology.orchestra.run.smithi164.stdout: ] 2023-06-03T20:06:24.882 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:06:24.882 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:06:24.882 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:06:24.882 INFO:teuthology.orchestra.run.smithi164.stdout: "description": "osd_op(client.45713.0:143792 9.e 9:71c0967b:::benchmark_data_smithi089_115580_object33658:head [delete] snapc 0=[] ondisk+write+known_if_redirected e138)", 2023-06-03T20:06:24.883 INFO:teuthology.orchestra.run.smithi164.stdout: "initiated_at": "2023-06-03T20:02:55.615479+0000", 2023-06-03T20:06:24.883 INFO:teuthology.orchestra.run.smithi164.stdout: "age": 209.110655417, 2023-06-03T20:06:24.883 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.140760207, 2023-06-03T20:06:24.883 INFO:teuthology.orchestra.run.smithi164.stdout: "type_data": { 2023-06-03T20:06:24.883 INFO:teuthology.orchestra.run.smithi164.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-03T20:06:24.883 INFO:teuthology.orchestra.run.smithi164.stdout: "client_info": { 2023-06-03T20:06:24.883 INFO:teuthology.orchestra.run.smithi164.stdout: "client": "client.45713", 2023-06-03T20:06:24.883 INFO:teuthology.orchestra.run.smithi164.stdout: "client_addr": "172.21.15.89:0/2400821448", 2023-06-03T20:06:24.884 INFO:teuthology.orchestra.run.smithi164.stdout: "tid": 143792 2023-06-03T20:06:24.884 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:06:24.884 INFO:teuthology.orchestra.run.smithi164.stdout: "events": [ 2023-06-03T20:06:24.884 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:06:24.884 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "initiated", 2023-06-03T20:06:24.884 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:02:55.615479+0000", 2023-06-03T20:06:24.884 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0 2023-06-03T20:06:24.885 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:06:24.885 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:06:24.885 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "throttled", 2023-06-03T20:06:24.885 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:02:55.615479+0000", 2023-06-03T20:06:24.885 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 4294967295.9999986 2023-06-03T20:06:24.885 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:06:24.885 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:06:24.885 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "header_read", 2023-06-03T20:06:24.886 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:02:55.615478+0000", 2023-06-03T20:06:24.886 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 5.5979999999999996e-06 2023-06-03T20:06:24.886 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:06:24.886 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:06:24.886 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "all_read", 2023-06-03T20:06:24.886 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:02:55.615484+0000", 2023-06-03T20:06:24.886 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 8.8270000000000004e-06 2023-06-03T20:06:24.886 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:06:24.887 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:06:24.887 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "dispatched", 2023-06-03T20:06:24.887 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:02:55.615492+0000", 2023-06-03T20:06:24.887 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 5.3290000000000001e-06 2023-06-03T20:06:24.887 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:06:24.887 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:06:24.887 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "queued_for_pg", 2023-06-03T20:06:24.887 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:02:55.615498+0000", 2023-06-03T20:06:24.888 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.002897377 2023-06-03T20:06:24.888 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:06:24.888 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:06:24.888 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "reached_pg", 2023-06-03T20:06:24.888 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:02:55.618395+0000", 2023-06-03T20:06:24.888 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.000129245 2023-06-03T20:06:24.888 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:06:24.889 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:06:24.889 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "started", 2023-06-03T20:06:24.889 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:02:55.618524+0000", 2023-06-03T20:06:24.889 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.000145215 2023-06-03T20:06:24.889 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:06:24.889 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:06:24.889 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "waiting for subops from 1,4", 2023-06-03T20:06:24.889 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:02:55.618670+0000", 2023-06-03T20:06:24.890 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.0042883499999999998 2023-06-03T20:06:24.890 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:06:24.890 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:06:24.890 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:06:24.890 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:02:55.622958+0000", 2023-06-03T20:06:24.890 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.00061331500000000002 2023-06-03T20:06:24.890 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:06:24.891 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:06:24.891 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "op_commit", 2023-06-03T20:06:24.891 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:02:55.623571+0000", 2023-06-03T20:06:24.891 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.13245637800000001 2023-06-03T20:06:24.891 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:06:24.891 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:06:24.891 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:06:24.891 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:02:55.756028+0000", 2023-06-03T20:06:24.892 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.000107636 2023-06-03T20:06:24.892 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:06:24.892 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:06:24.892 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "commit_sent", 2023-06-03T20:06:24.892 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:02:55.756135+0000", 2023-06-03T20:06:24.892 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.000104238 2023-06-03T20:06:24.892 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:06:24.892 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:06:24.893 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "done", 2023-06-03T20:06:24.893 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:02:55.756239+0000", 2023-06-03T20:06:24.893 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.140760207 2023-06-03T20:06:24.893 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:06:24.893 INFO:teuthology.orchestra.run.smithi164.stdout: ] 2023-06-03T20:06:24.893 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:06:24.893 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:06:24.893 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:06:24.894 INFO:teuthology.orchestra.run.smithi164.stdout: "description": "osd_op(client.45713.0:145263 9.d 9:baf6d9f0:::benchmark_data_smithi089_115580_object35129:head [delete] snapc 0=[] ondisk+write+known_if_redirected e138)", 2023-06-03T20:06:24.894 INFO:teuthology.orchestra.run.smithi164.stdout: "initiated_at": "2023-06-03T20:02:57.724121+0000", 2023-06-03T20:06:24.894 INFO:teuthology.orchestra.run.smithi164.stdout: "age": 207.00201413600001, 2023-06-03T20:06:24.894 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.14300256, 2023-06-03T20:06:24.894 INFO:teuthology.orchestra.run.smithi164.stdout: "type_data": { 2023-06-03T20:06:24.894 INFO:teuthology.orchestra.run.smithi164.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-03T20:06:24.894 INFO:teuthology.orchestra.run.smithi164.stdout: "client_info": { 2023-06-03T20:06:24.894 INFO:teuthology.orchestra.run.smithi164.stdout: "client": "client.45713", 2023-06-03T20:06:24.895 INFO:teuthology.orchestra.run.smithi164.stdout: "client_addr": "172.21.15.89:0/2400821448", 2023-06-03T20:06:24.895 INFO:teuthology.orchestra.run.smithi164.stdout: "tid": 145263 2023-06-03T20:06:24.895 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:06:24.895 INFO:teuthology.orchestra.run.smithi164.stdout: "events": [ 2023-06-03T20:06:24.895 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:06:24.895 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "initiated", 2023-06-03T20:06:24.895 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:02:57.724121+0000", 2023-06-03T20:06:24.896 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0 2023-06-03T20:06:24.896 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:06:24.896 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:06:24.896 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "throttled", 2023-06-03T20:06:24.896 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:02:57.724121+0000", 2023-06-03T20:06:24.896 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 4294967295.9999986 2023-06-03T20:06:24.896 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:06:24.896 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:06:24.897 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "header_read", 2023-06-03T20:06:24.897 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:02:57.724119+0000", 2023-06-03T20:06:24.897 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 7.554e-06 2023-06-03T20:06:24.897 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:06:24.897 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:06:24.897 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "all_read", 2023-06-03T20:06:24.897 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:02:57.724127+0000", 2023-06-03T20:06:24.897 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 1.6463000000000001e-05 2023-06-03T20:06:24.898 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:06:24.898 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:06:24.898 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "dispatched", 2023-06-03T20:06:24.898 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:02:57.724143+0000", 2023-06-03T20:06:24.898 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 6.4980000000000003e-06 2023-06-03T20:06:24.898 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:06:24.898 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:06:24.898 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "queued_for_pg", 2023-06-03T20:06:24.899 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:02:57.724150+0000", 2023-06-03T20:06:24.899 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.013684029 2023-06-03T20:06:24.899 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:06:24.899 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:06:24.899 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "reached_pg", 2023-06-03T20:06:24.899 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:02:57.737834+0000", 2023-06-03T20:06:24.899 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.001017312 2023-06-03T20:06:24.900 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:06:24.900 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:06:24.900 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "started", 2023-06-03T20:06:24.900 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:02:57.738851+0000", 2023-06-03T20:06:24.900 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.00015122200000000001 2023-06-03T20:06:24.900 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:06:24.900 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:06:24.900 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "waiting for subops from 1,5", 2023-06-03T20:06:24.901 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:02:57.739002+0000", 2023-06-03T20:06:24.901 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.0049338979999999999 2023-06-03T20:06:24.901 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:06:24.901 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:06:24.901 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "op_commit", 2023-06-03T20:06:24.901 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:02:57.743936+0000", 2023-06-03T20:06:24.901 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.00081422799999999996 2023-06-03T20:06:24.901 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:06:24.902 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:06:24.902 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:06:24.902 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:02:57.744750+0000", 2023-06-03T20:06:24.902 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.122289067 2023-06-03T20:06:24.902 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:06:24.902 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:06:24.903 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:06:24.903 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:02:57.867039+0000", 2023-06-03T20:06:24.903 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 4.5250000000000002e-05 2023-06-03T20:06:24.903 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:06:24.903 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:06:24.903 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "commit_sent", 2023-06-03T20:06:24.903 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:02:57.867085+0000", 2023-06-03T20:06:24.904 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 3.8507000000000002e-05 2023-06-03T20:06:24.904 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:06:24.904 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:06:24.904 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "done", 2023-06-03T20:06:24.904 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:02:57.867123+0000", 2023-06-03T20:06:24.904 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.14300256 2023-06-03T20:06:24.904 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:06:24.904 INFO:teuthology.orchestra.run.smithi164.stdout: ] 2023-06-03T20:06:24.905 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:06:24.905 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:06:24.905 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:06:24.905 INFO:teuthology.orchestra.run.smithi164.stdout: "description": "osd_op(client.45713.0:145267 9.e 9:72d46eb7:::benchmark_data_smithi089_115580_object35133:head [delete] snapc 0=[] ondisk+write+known_if_redirected e138)", 2023-06-03T20:06:24.905 INFO:teuthology.orchestra.run.smithi164.stdout: "initiated_at": "2023-06-03T20:02:57.726542+0000", 2023-06-03T20:06:24.905 INFO:teuthology.orchestra.run.smithi164.stdout: "age": 206.99959260599999, 2023-06-03T20:06:24.905 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.22929925800000001, 2023-06-03T20:06:24.905 INFO:teuthology.orchestra.run.smithi164.stdout: "type_data": { 2023-06-03T20:06:24.906 INFO:teuthology.orchestra.run.smithi164.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-03T20:06:24.906 INFO:teuthology.orchestra.run.smithi164.stdout: "client_info": { 2023-06-03T20:06:24.906 INFO:teuthology.orchestra.run.smithi164.stdout: "client": "client.45713", 2023-06-03T20:06:24.906 INFO:teuthology.orchestra.run.smithi164.stdout: "client_addr": "172.21.15.89:0/2400821448", 2023-06-03T20:06:24.906 INFO:teuthology.orchestra.run.smithi164.stdout: "tid": 145267 2023-06-03T20:06:24.906 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:06:24.906 INFO:teuthology.orchestra.run.smithi164.stdout: "events": [ 2023-06-03T20:06:24.907 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:06:24.907 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "initiated", 2023-06-03T20:06:24.907 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:02:57.726542+0000", 2023-06-03T20:06:24.907 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0 2023-06-03T20:06:24.907 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:06:24.907 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:06:24.907 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "throttled", 2023-06-03T20:06:24.907 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:02:57.726542+0000", 2023-06-03T20:06:24.908 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 4294967295.999999 2023-06-03T20:06:24.908 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:06:24.908 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:06:24.908 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "header_read", 2023-06-03T20:06:24.908 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:02:57.726541+0000", 2023-06-03T20:06:24.908 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 6.3709999999999996e-06 2023-06-03T20:06:24.908 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:06:24.909 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:06:24.909 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "all_read", 2023-06-03T20:06:24.909 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:02:57.726547+0000", 2023-06-03T20:06:24.909 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 1.1785e-05 2023-06-03T20:06:24.909 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:06:24.909 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:06:24.909 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "dispatched", 2023-06-03T20:06:24.909 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:02:57.726559+0000", 2023-06-03T20:06:24.910 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 6.5479999999999998e-06 2023-06-03T20:06:24.910 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:06:24.910 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:06:24.910 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "queued_for_pg", 2023-06-03T20:06:24.910 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:02:57.726566+0000", 2023-06-03T20:06:24.910 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.0069441829999999996 2023-06-03T20:06:24.910 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:06:24.910 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:06:24.911 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "reached_pg", 2023-06-03T20:06:24.911 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:02:57.733510+0000", 2023-06-03T20:06:24.911 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.000118329 2023-06-03T20:06:24.911 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:06:24.911 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:06:24.911 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "started", 2023-06-03T20:06:24.911 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:02:57.733628+0000", 2023-06-03T20:06:24.911 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.000146229 2023-06-03T20:06:24.912 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:06:24.912 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:06:24.912 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "waiting for subops from 1,4", 2023-06-03T20:06:24.912 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:02:57.733775+0000", 2023-06-03T20:06:24.912 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.0081442289999999994 2023-06-03T20:06:24.912 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:06:24.912 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:06:24.912 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:06:24.913 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:02:57.741919+0000", 2023-06-03T20:06:24.913 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.00049648300000000004 2023-06-03T20:06:24.913 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:06:24.913 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:06:24.913 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "op_commit", 2023-06-03T20:06:24.913 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:02:57.742415+0000", 2023-06-03T20:06:24.913 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.21315916500000001 2023-06-03T20:06:24.914 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:06:24.914 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:06:24.914 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:06:24.914 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:02:57.955574+0000", 2023-06-03T20:06:24.914 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.00014740099999999999 2023-06-03T20:06:24.914 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:06:24.914 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:06:24.914 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "commit_sent", 2023-06-03T20:06:24.915 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:02:57.955722+0000", 2023-06-03T20:06:24.915 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.000119548 2023-06-03T20:06:24.915 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:06:24.915 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:06:24.915 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "done", 2023-06-03T20:06:24.915 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:02:57.955841+0000", 2023-06-03T20:06:24.915 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.22929925800000001 2023-06-03T20:06:24.915 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:06:24.916 INFO:teuthology.orchestra.run.smithi164.stdout: ] 2023-06-03T20:06:24.916 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:06:24.916 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:06:24.916 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:06:24.916 INFO:teuthology.orchestra.run.smithi164.stdout: "description": "osd_op(client.45713.0:146192 9.d 9:b2d0c0c3:::benchmark_data_smithi089_115580_object36058:head [delete] snapc 0=[] ondisk+write+known_if_redirected e139)", 2023-06-03T20:06:24.916 INFO:teuthology.orchestra.run.smithi164.stdout: "initiated_at": "2023-06-03T20:02:59.141701+0000", 2023-06-03T20:06:24.916 INFO:teuthology.orchestra.run.smithi164.stdout: "age": 205.58443352099999, 2023-06-03T20:06:24.916 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.14884718, 2023-06-03T20:06:24.917 INFO:teuthology.orchestra.run.smithi164.stdout: "type_data": { 2023-06-03T20:06:24.917 INFO:teuthology.orchestra.run.smithi164.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-03T20:06:24.917 INFO:teuthology.orchestra.run.smithi164.stdout: "client_info": { 2023-06-03T20:06:24.917 INFO:teuthology.orchestra.run.smithi164.stdout: "client": "client.45713", 2023-06-03T20:06:24.917 INFO:teuthology.orchestra.run.smithi164.stdout: "client_addr": "172.21.15.89:0/2400821448", 2023-06-03T20:06:24.917 INFO:teuthology.orchestra.run.smithi164.stdout: "tid": 146192 2023-06-03T20:06:24.917 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:06:24.917 INFO:teuthology.orchestra.run.smithi164.stdout: "events": [ 2023-06-03T20:06:24.918 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:06:24.918 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "initiated", 2023-06-03T20:06:24.918 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:02:59.141701+0000", 2023-06-03T20:06:24.918 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0 2023-06-03T20:06:24.918 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:06:24.918 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:06:24.918 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "throttled", 2023-06-03T20:06:24.919 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:02:59.141701+0000", 2023-06-03T20:06:24.919 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 4294967295.9999981 2023-06-03T20:06:24.919 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:06:24.919 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:06:24.919 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "header_read", 2023-06-03T20:06:24.919 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:02:59.141699+0000", 2023-06-03T20:06:24.919 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 7.5519999999999999e-06 2023-06-03T20:06:24.919 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:06:24.920 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:06:24.920 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "all_read", 2023-06-03T20:06:24.920 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:02:59.141707+0000", 2023-06-03T20:06:24.920 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 1.2899e-05 2023-06-03T20:06:24.920 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:06:24.920 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:06:24.920 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "dispatched", 2023-06-03T20:06:24.921 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:02:59.141720+0000", 2023-06-03T20:06:24.921 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 2.1157000000000001e-05 2023-06-03T20:06:24.921 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:06:24.921 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:06:24.921 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "queued_for_pg", 2023-06-03T20:06:24.921 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:02:59.141741+0000", 2023-06-03T20:06:24.921 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 5.1996999999999997e-05 2023-06-03T20:06:24.921 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:06:24.922 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:06:24.922 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "reached_pg", 2023-06-03T20:06:24.922 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:02:59.141793+0000", 2023-06-03T20:06:24.922 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.00011925399999999999 2023-06-03T20:06:24.922 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:06:24.922 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:06:24.922 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "started", 2023-06-03T20:06:24.922 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:02:59.141912+0000", 2023-06-03T20:06:24.923 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.00014944900000000001 2023-06-03T20:06:24.923 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:06:24.923 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:06:24.923 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "waiting for subops from 1,5", 2023-06-03T20:06:24.923 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:02:59.142061+0000", 2023-06-03T20:06:24.923 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.013283121 2023-06-03T20:06:24.923 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:06:24.923 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:06:24.924 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:06:24.924 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:02:59.155345+0000", 2023-06-03T20:06:24.924 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.0050267319999999999 2023-06-03T20:06:24.924 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:06:24.924 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:06:24.924 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "op_commit", 2023-06-03T20:06:24.924 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:02:59.160371+0000", 2023-06-03T20:06:24.924 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.130112002 2023-06-03T20:06:24.925 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:06:24.925 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:06:24.925 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:06:24.925 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:02:59.290483+0000", 2023-06-03T20:06:24.925 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 3.5626000000000001e-05 2023-06-03T20:06:24.925 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:06:24.925 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:06:24.926 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "commit_sent", 2023-06-03T20:06:24.926 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:02:59.290519+0000", 2023-06-03T20:06:24.926 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 2.9434999999999999e-05 2023-06-03T20:06:24.926 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:06:24.926 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:06:24.926 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "done", 2023-06-03T20:06:24.926 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:02:59.290548+0000", 2023-06-03T20:06:24.926 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.14884718 2023-06-03T20:06:24.927 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:06:24.927 INFO:teuthology.orchestra.run.smithi164.stdout: ] 2023-06-03T20:06:24.927 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:06:24.927 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:06:24.927 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:06:24.927 INFO:teuthology.orchestra.run.smithi164.stdout: "description": "osd_op(client.45713.0:146196 9.d 9:bf4b5fce:::benchmark_data_smithi089_115580_object36062:head [delete] snapc 0=[] ondisk+write+known_if_redirected e139)", 2023-06-03T20:06:24.927 INFO:teuthology.orchestra.run.smithi164.stdout: "initiated_at": "2023-06-03T20:02:59.143155+0000", 2023-06-03T20:06:24.927 INFO:teuthology.orchestra.run.smithi164.stdout: "age": 205.58297932100001, 2023-06-03T20:06:24.928 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.14751489700000001, 2023-06-03T20:06:24.928 INFO:teuthology.orchestra.run.smithi164.stdout: "type_data": { 2023-06-03T20:06:24.928 INFO:teuthology.orchestra.run.smithi164.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-03T20:06:24.928 INFO:teuthology.orchestra.run.smithi164.stdout: "client_info": { 2023-06-03T20:06:24.928 INFO:teuthology.orchestra.run.smithi164.stdout: "client": "client.45713", 2023-06-03T20:06:24.928 INFO:teuthology.orchestra.run.smithi164.stdout: "client_addr": "172.21.15.89:0/2400821448", 2023-06-03T20:06:24.928 INFO:teuthology.orchestra.run.smithi164.stdout: "tid": 146196 2023-06-03T20:06:24.928 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:06:24.929 INFO:teuthology.orchestra.run.smithi164.stdout: "events": [ 2023-06-03T20:06:24.929 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:06:24.929 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "initiated", 2023-06-03T20:06:24.929 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:02:59.143155+0000", 2023-06-03T20:06:24.929 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0 2023-06-03T20:06:24.929 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:06:24.929 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:06:24.929 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "throttled", 2023-06-03T20:06:24.930 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:02:59.143155+0000", 2023-06-03T20:06:24.930 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 4294967295.9999986 2023-06-03T20:06:24.930 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:06:24.930 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:06:24.930 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "header_read", 2023-06-03T20:06:24.930 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:02:59.143154+0000", 2023-06-03T20:06:24.930 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 9.7659999999999993e-06 2023-06-03T20:06:24.930 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:06:24.931 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:06:24.931 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "all_read", 2023-06-03T20:06:24.931 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:02:59.143164+0000", 2023-06-03T20:06:24.931 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 1.1885999999999999e-05 2023-06-03T20:06:24.931 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:06:24.931 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:06:24.931 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "dispatched", 2023-06-03T20:06:24.932 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:02:59.143176+0000", 2023-06-03T20:06:24.932 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 5.6380000000000001e-06 2023-06-03T20:06:24.932 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:06:24.932 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:06:24.932 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "queued_for_pg", 2023-06-03T20:06:24.932 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:02:59.143181+0000", 2023-06-03T20:06:24.932 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.0071659310000000004 2023-06-03T20:06:24.932 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:06:24.933 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:06:24.933 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "reached_pg", 2023-06-03T20:06:24.933 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:02:59.150347+0000", 2023-06-03T20:06:24.933 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.00012711299999999999 2023-06-03T20:06:24.933 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:06:24.933 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:06:24.933 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "started", 2023-06-03T20:06:24.933 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:02:59.150474+0000", 2023-06-03T20:06:24.934 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.00015004600000000001 2023-06-03T20:06:24.934 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:06:24.934 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:06:24.934 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "waiting for subops from 1,5", 2023-06-03T20:06:24.934 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:02:59.150624+0000", 2023-06-03T20:06:24.934 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.0097521150000000004 2023-06-03T20:06:24.934 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:06:24.934 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:06:24.935 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "op_commit", 2023-06-03T20:06:24.935 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:02:59.160376+0000", 2023-06-03T20:06:24.935 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.0039605930000000001 2023-06-03T20:06:24.935 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:06:24.935 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:06:24.935 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:06:24.935 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:02:59.164337+0000", 2023-06-03T20:06:24.936 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.12625507 2023-06-03T20:06:24.936 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:06:24.936 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:06:24.936 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:06:24.936 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:02:59.290592+0000", 2023-06-03T20:06:24.936 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 4.0363e-05 2023-06-03T20:06:24.936 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:06:24.936 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:06:24.937 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "commit_sent", 2023-06-03T20:06:24.937 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:02:59.290632+0000", 2023-06-03T20:06:24.937 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 3.7864999999999997e-05 2023-06-03T20:06:24.937 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:06:24.937 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:06:24.937 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "done", 2023-06-03T20:06:24.937 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:02:59.290670+0000", 2023-06-03T20:06:24.937 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.14751489700000001 2023-06-03T20:06:24.938 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:06:24.938 INFO:teuthology.orchestra.run.smithi164.stdout: ] 2023-06-03T20:06:24.938 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:06:24.938 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:06:24.938 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:06:24.938 INFO:teuthology.orchestra.run.smithi164.stdout: "description": "osd_op(client.45713.0:146210 9.d 9:b4c464d8:::benchmark_data_smithi089_115580_object36076:head [delete] snapc 0=[] ondisk+write+known_if_redirected e139)", 2023-06-03T20:06:24.938 INFO:teuthology.orchestra.run.smithi164.stdout: "initiated_at": "2023-06-03T20:02:59.160575+0000", 2023-06-03T20:06:24.938 INFO:teuthology.orchestra.run.smithi164.stdout: "age": 205.56555965199999, 2023-06-03T20:06:24.939 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.143396832, 2023-06-03T20:06:24.939 INFO:teuthology.orchestra.run.smithi164.stdout: "type_data": { 2023-06-03T20:06:24.939 INFO:teuthology.orchestra.run.smithi164.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-03T20:06:24.939 INFO:teuthology.orchestra.run.smithi164.stdout: "client_info": { 2023-06-03T20:06:24.939 INFO:teuthology.orchestra.run.smithi164.stdout: "client": "client.45713", 2023-06-03T20:06:24.939 INFO:teuthology.orchestra.run.smithi164.stdout: "client_addr": "172.21.15.89:0/2400821448", 2023-06-03T20:06:24.939 INFO:teuthology.orchestra.run.smithi164.stdout: "tid": 146210 2023-06-03T20:06:24.939 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:06:24.940 INFO:teuthology.orchestra.run.smithi164.stdout: "events": [ 2023-06-03T20:06:24.940 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:06:24.940 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "initiated", 2023-06-03T20:06:24.940 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:02:59.160575+0000", 2023-06-03T20:06:24.940 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0 2023-06-03T20:06:24.940 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:06:24.940 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:06:24.941 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "throttled", 2023-06-03T20:06:24.941 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:02:59.160575+0000", 2023-06-03T20:06:24.941 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 4294967295.9999986 2023-06-03T20:06:24.941 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:06:24.941 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:06:24.941 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "header_read", 2023-06-03T20:06:24.941 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:02:59.160574+0000", 2023-06-03T20:06:24.941 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 6.5849999999999996e-06 2023-06-03T20:06:24.942 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:06:24.942 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:06:24.942 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "all_read", 2023-06-03T20:06:24.942 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:02:59.160580+0000", 2023-06-03T20:06:24.942 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 1.2160999999999999e-05 2023-06-03T20:06:24.942 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:06:24.942 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:06:24.942 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "dispatched", 2023-06-03T20:06:24.943 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:02:59.160592+0000", 2023-06-03T20:06:24.943 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 5.2120000000000002e-06 2023-06-03T20:06:24.943 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:06:24.943 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:06:24.943 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "queued_for_pg", 2023-06-03T20:06:24.943 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:02:59.160598+0000", 2023-06-03T20:06:24.943 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 4.2485999999999999e-05 2023-06-03T20:06:24.943 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:06:24.944 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:06:24.944 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "reached_pg", 2023-06-03T20:06:24.944 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:02:59.160640+0000", 2023-06-03T20:06:24.944 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.00014865399999999999 2023-06-03T20:06:24.944 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:06:24.944 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:06:24.944 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "started", 2023-06-03T20:06:24.944 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:02:59.160789+0000", 2023-06-03T20:06:24.945 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.000184124 2023-06-03T20:06:24.945 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:06:24.945 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:06:24.945 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "waiting for subops from 1,5", 2023-06-03T20:06:24.945 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:02:59.160973+0000", 2023-06-03T20:06:24.945 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.0034544749999999998 2023-06-03T20:06:24.945 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:06:24.946 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:06:24.946 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "op_commit", 2023-06-03T20:06:24.946 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:02:59.164427+0000", 2023-06-03T20:06:24.946 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.0028824860000000001 2023-06-03T20:06:24.946 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:06:24.946 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:06:24.946 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:06:24.946 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:02:59.167310+0000", 2023-06-03T20:06:24.947 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.13657192100000001 2023-06-03T20:06:24.947 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:06:24.947 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:06:24.947 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:06:24.947 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:02:59.303882+0000", 2023-06-03T20:06:24.947 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 4.8637999999999997e-05 2023-06-03T20:06:24.947 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:06:24.947 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:06:24.948 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "commit_sent", 2023-06-03T20:06:24.948 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:02:59.303930+0000", 2023-06-03T20:06:24.948 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 4.1553e-05 2023-06-03T20:06:24.948 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:06:24.948 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:06:24.948 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "done", 2023-06-03T20:06:24.948 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:02:59.303972+0000", 2023-06-03T20:06:24.949 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.143396832 2023-06-03T20:06:24.949 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:06:24.949 INFO:teuthology.orchestra.run.smithi164.stdout: ] 2023-06-03T20:06:24.949 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:06:24.949 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:06:24.949 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:06:24.949 INFO:teuthology.orchestra.run.smithi164.stdout: "description": "osd_op(client.45713.0:146675 9.3 9:cfeab434:::benchmark_data_smithi089_115580_object36541:head [delete] snapc 0=[] ondisk+write+known_if_redirected e139)", 2023-06-03T20:06:24.949 INFO:teuthology.orchestra.run.smithi164.stdout: "initiated_at": "2023-06-03T20:02:59.954659+0000", 2023-06-03T20:06:24.950 INFO:teuthology.orchestra.run.smithi164.stdout: "age": 204.77147614099999, 2023-06-03T20:06:24.950 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.16524977599999999, 2023-06-03T20:06:24.950 INFO:teuthology.orchestra.run.smithi164.stdout: "type_data": { 2023-06-03T20:06:24.950 INFO:teuthology.orchestra.run.smithi164.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-03T20:06:24.950 INFO:teuthology.orchestra.run.smithi164.stdout: "client_info": { 2023-06-03T20:06:24.950 INFO:teuthology.orchestra.run.smithi164.stdout: "client": "client.45713", 2023-06-03T20:06:24.950 INFO:teuthology.orchestra.run.smithi164.stdout: "client_addr": "172.21.15.89:0/2400821448", 2023-06-03T20:06:24.950 INFO:teuthology.orchestra.run.smithi164.stdout: "tid": 146675 2023-06-03T20:06:24.951 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:06:24.951 INFO:teuthology.orchestra.run.smithi164.stdout: "events": [ 2023-06-03T20:06:24.951 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:06:24.951 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "initiated", 2023-06-03T20:06:24.951 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:02:59.954659+0000", 2023-06-03T20:06:24.951 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0 2023-06-03T20:06:24.951 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:06:24.951 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:06:24.952 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "throttled", 2023-06-03T20:06:24.952 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:02:59.954659+0000", 2023-06-03T20:06:24.952 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 4294967295.9999981 2023-06-03T20:06:24.952 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:06:24.952 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:06:24.952 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "header_read", 2023-06-03T20:06:24.952 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:02:59.954657+0000", 2023-06-03T20:06:24.952 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 1.1245e-05 2023-06-03T20:06:24.953 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:06:24.953 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:06:24.953 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "all_read", 2023-06-03T20:06:24.953 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:02:59.954668+0000", 2023-06-03T20:06:24.953 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 1.2048999999999999e-05 2023-06-03T20:06:24.953 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:06:24.953 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:06:24.954 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "dispatched", 2023-06-03T20:06:24.954 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:02:59.954680+0000", 2023-06-03T20:06:24.954 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 6.5400000000000001e-06 2023-06-03T20:06:24.954 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:06:24.954 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:06:24.954 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "queued_for_pg", 2023-06-03T20:06:24.954 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:02:59.954686+0000", 2023-06-03T20:06:24.954 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.0028255049999999999 2023-06-03T20:06:24.955 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:06:24.955 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:06:24.955 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "reached_pg", 2023-06-03T20:06:24.955 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:02:59.957512+0000", 2023-06-03T20:06:24.955 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.000137298 2023-06-03T20:06:24.955 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:06:24.955 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:06:24.955 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "started", 2023-06-03T20:06:24.956 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:02:59.957649+0000", 2023-06-03T20:06:24.956 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.00016586199999999999 2023-06-03T20:06:24.956 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:06:24.956 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:06:24.956 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "waiting for subops from 1,3", 2023-06-03T20:06:24.956 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:02:59.957815+0000", 2023-06-03T20:06:24.956 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.009189727 2023-06-03T20:06:24.957 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:06:24.957 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:06:24.957 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:06:24.957 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:02:59.967005+0000", 2023-06-03T20:06:24.957 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 7.8569999999999995e-06 2023-06-03T20:06:24.957 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:06:24.957 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:06:24.957 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "op_commit", 2023-06-03T20:06:24.958 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:02:59.967013+0000", 2023-06-03T20:06:24.958 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.15270435299999999 2023-06-03T20:06:24.958 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:06:24.958 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:06:24.958 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:06:24.958 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:03:00.119717+0000", 2023-06-03T20:06:24.958 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.000116592 2023-06-03T20:06:24.958 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:06:24.959 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:06:24.959 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "commit_sent", 2023-06-03T20:06:24.959 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:03:00.119834+0000", 2023-06-03T20:06:24.959 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 7.4758000000000005e-05 2023-06-03T20:06:24.959 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:06:24.959 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:06:24.959 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "done", 2023-06-03T20:06:24.959 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:03:00.119908+0000", 2023-06-03T20:06:24.960 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.16524977599999999 2023-06-03T20:06:24.960 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:06:24.960 INFO:teuthology.orchestra.run.smithi164.stdout: ] 2023-06-03T20:06:24.960 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:06:24.960 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:06:24.960 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:06:24.960 INFO:teuthology.orchestra.run.smithi164.stdout: "description": "osd_op(client.45713.0:146681 9.3 9:c185dbf4:::benchmark_data_smithi089_115580_object36547:head [delete] snapc 0=[] ondisk+write+known_if_redirected e139)", 2023-06-03T20:06:24.960 INFO:teuthology.orchestra.run.smithi164.stdout: "initiated_at": "2023-06-03T20:02:59.970649+0000", 2023-06-03T20:06:24.961 INFO:teuthology.orchestra.run.smithi164.stdout: "age": 204.755485644, 2023-06-03T20:06:24.961 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.15465549100000001, 2023-06-03T20:06:24.961 INFO:teuthology.orchestra.run.smithi164.stdout: "type_data": { 2023-06-03T20:06:24.961 INFO:teuthology.orchestra.run.smithi164.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-03T20:06:24.961 INFO:teuthology.orchestra.run.smithi164.stdout: "client_info": { 2023-06-03T20:06:24.961 INFO:teuthology.orchestra.run.smithi164.stdout: "client": "client.45713", 2023-06-03T20:06:24.961 INFO:teuthology.orchestra.run.smithi164.stdout: "client_addr": "172.21.15.89:0/2400821448", 2023-06-03T20:06:24.962 INFO:teuthology.orchestra.run.smithi164.stdout: "tid": 146681 2023-06-03T20:06:24.962 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:06:24.962 INFO:teuthology.orchestra.run.smithi164.stdout: "events": [ 2023-06-03T20:06:24.962 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:06:24.962 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "initiated", 2023-06-03T20:06:24.962 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:02:59.970649+0000", 2023-06-03T20:06:24.962 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0 2023-06-03T20:06:24.962 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:06:24.963 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:06:24.963 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "throttled", 2023-06-03T20:06:24.963 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:02:59.970649+0000", 2023-06-03T20:06:24.963 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 4294967295.9999981 2023-06-03T20:06:24.963 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:06:24.963 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:06:24.963 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "header_read", 2023-06-03T20:06:24.963 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:02:59.970647+0000", 2023-06-03T20:06:24.964 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 5.7799999999999997e-06 2023-06-03T20:06:24.964 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:06:24.964 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:06:24.964 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "all_read", 2023-06-03T20:06:24.964 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:02:59.970653+0000", 2023-06-03T20:06:24.964 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 8.8440000000000004e-06 2023-06-03T20:06:24.964 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:06:24.964 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:06:24.965 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "dispatched", 2023-06-03T20:06:24.965 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:02:59.970662+0000", 2023-06-03T20:06:24.965 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 4.9980000000000003e-06 2023-06-03T20:06:24.965 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:06:24.965 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:06:24.965 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "queued_for_pg", 2023-06-03T20:06:24.965 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:02:59.970667+0000", 2023-06-03T20:06:24.965 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 4.9075e-05 2023-06-03T20:06:24.966 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:06:24.966 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:06:24.966 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "reached_pg", 2023-06-03T20:06:24.966 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:02:59.970716+0000", 2023-06-03T20:06:24.966 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.000119283 2023-06-03T20:06:24.966 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:06:24.966 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:06:24.967 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "started", 2023-06-03T20:06:24.967 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:02:59.970835+0000", 2023-06-03T20:06:24.967 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.00013477199999999999 2023-06-03T20:06:24.967 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:06:24.967 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:06:24.967 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "waiting for subops from 1,3", 2023-06-03T20:06:24.967 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:02:59.970970+0000", 2023-06-03T20:06:24.967 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.0039862389999999999 2023-06-03T20:06:24.968 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:06:24.968 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:06:24.968 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "op_commit", 2023-06-03T20:06:24.968 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:02:59.974956+0000", 2023-06-03T20:06:24.968 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.0033183599999999998 2023-06-03T20:06:24.968 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:06:24.968 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:06:24.968 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:06:24.969 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:02:59.978275+0000", 2023-06-03T20:06:24.969 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.146773127 2023-06-03T20:06:24.969 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:06:24.969 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:06:24.969 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:06:24.969 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:03:00.125048+0000", 2023-06-03T20:06:24.969 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.00013961099999999999 2023-06-03T20:06:24.970 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:06:24.970 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:06:24.970 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "commit_sent", 2023-06-03T20:06:24.970 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:03:00.125187+0000", 2023-06-03T20:06:24.970 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.000117111 2023-06-03T20:06:24.970 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:06:24.970 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:06:24.970 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "done", 2023-06-03T20:06:24.971 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:03:00.125305+0000", 2023-06-03T20:06:24.971 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.15465549100000001 2023-06-03T20:06:24.971 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:06:24.971 INFO:teuthology.orchestra.run.smithi164.stdout: ] 2023-06-03T20:06:24.971 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:06:24.971 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:06:24.971 INFO:teuthology.orchestra.run.smithi164.stdout: ] 2023-06-03T20:06:24.971 INFO:teuthology.orchestra.run.smithi164.stdout:} 2023-06-03T20:06:24.972 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:06:24.973 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:06:24.973 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:06:25.074 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:06:25.109 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:06:25.112 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:06:25.112 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:06:25.213 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:06:25.231 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 11 16 24514 24498 139.168 115.875 0.00958369 0.0071787 2023-06-03T20:06:25.248 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:06:25.252 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:06:25.253 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:06:25.268 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph osd set noscrub 2023-06-03T20:06:25.354 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:06:25.404 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:06:25.408 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:06:25.409 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:06:25.510 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:06:25.539 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:06:25.544 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:06:25.545 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:06:25.646 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:06:25.687 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:06:25.689 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:06:25.690 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:06:25.791 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:06:25.801 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:06:25 smithi164 bash[59462]: cluster 1685822784.5546567 mgr.y (mgr.34104) 907 : cluster 0 pgmap v845: 89 pgs: 89 active+clean; 1.1 GiB data, 5.1 GiB used, 710 GiB / 715 GiB avail; 117 MiB/s wr, 1.87k op/s 2023-06-03T20:06:25.828 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:06:25.830 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:06:25.830 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:06:25.931 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:06:25.972 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:06:25.976 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:06:25.976 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:06:26.028 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:06:25 smithi089 bash[75068]: cluster 1685822784.5546567 mgr.y (mgr.34104) 907 : cluster 0 pgmap v845: 89 pgs: 89 active+clean; 1.1 GiB data, 5.1 GiB used, 710 GiB / 715 GiB avail; 117 MiB/s wr, 1.87k op/s 2023-06-03T20:06:26.029 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:06:25 smithi089 bash[78452]: cluster 1685822784.5546567 mgr.y (mgr.34104) 907 : cluster 0 pgmap v845: 89 pgs: 89 active+clean; 1.1 GiB data, 5.1 GiB used, 710 GiB / 715 GiB avail; 117 MiB/s wr, 1.87k op/s 2023-06-03T20:06:26.077 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:06:26.112 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:06:26.116 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:06:26.117 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:06:26.218 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:06:26.231 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 12 16 26336 26320 137.06 113.875 0.00373787 0.007289 2023-06-03T20:06:26.248 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:06:26.250 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:06:26.251 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:06:26.352 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:06:26.372 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph daemon osd.0 dump_ops_in_flight 2023-06-03T20:06:26.384 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:06:26.387 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:06:26.387 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:06:26.488 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:06:26.525 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:06:26.528 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 20:06:26 smithi089 bash[71163]: ::ffff:172.21.15.164 - - [03/Jun/2023:20:06:26] "GET /metrics HTTP/1.1" 200 228585 "" "Prometheus/2.33.4" 2023-06-03T20:06:26.531 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:06:26.532 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:06:26.633 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:06:26.668 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:06:26.670 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:06:26.670 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:06:26.771 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:06:26.801 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:06:26.803 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:06:26.804 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:06:26.905 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:06:26.939 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:06:26.945 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:06:26.946 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:06:27.047 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:06:27.076 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:06:27.078 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:06:27.079 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:06:27.180 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:06:27.212 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:06:27.218 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:06:27.219 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:06:27.231 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 13 16 28097 28081 134.983 110.062 0.00669244 0.00740213 2023-06-03T20:06:27.320 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:06:27.348 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:06:27.354 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:06:27.355 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:06:27.455 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:06:27.490 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:06:27.493 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:06:27.493 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:06:27.594 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:06:27.627 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:06:27.632 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:06:27.633 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:06:27.734 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:06:27.768 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:06:27.774 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:06:27.775 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:06:27.876 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:06:27.885 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:06:27 smithi089 bash[78452]: cluster 1685822786.5554814 mgr.y (mgr.34104) 908 : cluster 0 pgmap v846: 89 pgs: 89 active+clean; 1.3 GiB data, 5.7 GiB used, 710 GiB / 715 GiB avail; 129 MiB/s wr, 2.06k op/s 2023-06-03T20:06:27.886 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:06:27 smithi089 bash[75068]: cluster 1685822786.5554814 mgr.y (mgr.34104) 908 : cluster 0 pgmap v846: 89 pgs: 89 active+clean; 1.3 GiB data, 5.7 GiB used, 710 GiB / 715 GiB avail; 129 MiB/s wr, 2.06k op/s 2023-06-03T20:06:27.932 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:06:27.935 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:06:27.935 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:06:27.958 INFO:teuthology.orchestra.run.smithi089.stdout: 2023-06-03T20:06:27.964 INFO:teuthology.orchestra.run.smithi089.stderr:dumped all 2023-06-03T20:06:28.037 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:06:28.072 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:06:28.075 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:06:28.075 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:06:28.083 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:06:27 smithi164 bash[59462]: cluster 1685822786.5554814 mgr.y (mgr.34104) 908 : cluster 0 pgmap v846: 89 pgs: 89 active+clean; 1.3 GiB data, 5.7 GiB used, 710 GiB / 715 GiB avail; 129 MiB/s wr, 2.06k op/s 2023-06-03T20:06:28.176 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:06:28.214 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:06:28.217 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:06:28.218 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:06:28.231 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 14 16 29739 29723 132.671 102.625 0.00289859 0.00753103 2023-06-03T20:06:28.319 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:06:28.349 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:06:28.352 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:06:28.353 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:06:28.454 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:06:28.489 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:06:28.492 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:06:28.493 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:06:28.594 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:06:28.617 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:06:28.618 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:06:28.619 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:06:28.720 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:06:28.753 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:06:28.755 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:06:28.755 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:06:28.833 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:06:28 smithi164 bash[59462]: audit 1685822787.9383876 mgr.y (mgr.34104) 909 : audit 0 from='client.45962 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-06-03T20:06:28.856 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:06:28.888 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:06:28.891 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:06:28.893 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:06:28.994 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:06:29.025 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:06:29.028 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:06:28 smithi089 bash[75068]: audit 1685822787.9383876 mgr.y (mgr.34104) 909 : audit 0 from='client.45962 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-06-03T20:06:29.028 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:06:28 smithi089 bash[78452]: audit 1685822787.9383876 mgr.y (mgr.34104) 909 : audit 0 from='client.45962 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-06-03T20:06:29.029 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:06:29.030 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:06:29.131 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:06:29.162 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:06:29.165 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:06:29.166 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:06:29.232 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 15 16 31310 31294 130.371 98.1875 0.00905162 0.00766347 2023-06-03T20:06:29.267 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:06:29.298 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:06:29.300 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:06:29.300 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:06:29.401 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:06:29.430 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:06:29.433 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:06:29.434 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:06:29.535 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:06:29.575 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:06:29.579 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:06:29.579 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:06:29.586 INFO:teuthology.orchestra.run.smithi089.stdout:{"pg_ready":true,"pg_map":{"version":846,"stamp":"2023-06-03T20:06:26.554934+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":1393170318,"num_objects":21273,"num_object_clones":0,"num_object_copies":63819,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":21273,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":21510,"num_write_kb":1362382,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":21444,"ondisk_log_size":21444,"up":267,"acting":267,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":267,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":5977332,"kb_used_data":3891380,"kb_used_omap":30,"kb_used_meta":2085857,"kb_avail":743983884,"statfs":{"total":767960285184,"available":761839497216,"internally_reserved":0,"allocated":3984773120,"data_stored":3980200729,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":31502,"internal_metadata":2135917810},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[6,2,4,3,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":1348534272,"num_objects":20577,"num_object_clones":0,"num_object_copies":61731,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":20577,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":20577,"num_write_kb":1316928,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.003640"},"pg_stats":[{"pgid":"3.1a","version":"60'1","reported_seq":151,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-06-03T20:06:09.861193+0000","last_change":"2023-06-03T19:57:28.467559+0000","last_active":"2023-06-03T20:06:09.861193+0000","last_peered":"2023-06-03T20:06:09.861193+0000","last_clean":"2023-06-03T20:06:09.861193+0000","last_became_active":"2023-06-03T19:57:28.467077+0000","last_became_peered":"2023-06-03T19:57:28.467077+0000","last_unstale":"2023-06-03T20:06:09.861193+0000","last_undegraded":"2023-06-03T20:06:09.861193+0000","last_fullsized":"2023-06-03T20:06:09.861193+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T21:46:02.795576+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":144,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-06-03T20:06:10.308279+0000","last_change":"2023-06-03T19:57:28.470287+0000","last_active":"2023-06-03T20:06:10.308279+0000","last_peered":"2023-06-03T20:06:10.308279+0000","last_clean":"2023-06-03T20:06:10.308279+0000","last_became_active":"2023-06-03T19:57:28.469477+0000","last_became_peered":"2023-06-03T19:57:28.469477+0000","last_unstale":"2023-06-03T20:06:10.308279+0000","last_undegraded":"2023-06-03T20:06:10.308279+0000","last_fullsized":"2023-06-03T20:06:10.308279+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T06:48:19.229190+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":157,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-06-03T20:06:10.681975+0000","last_change":"2023-06-03T19:57:28.473350+0000","last_active":"2023-06-03T20:06:10.681975+0000","last_peered":"2023-06-03T20:06:10.681975+0000","last_clean":"2023-06-03T20:06:10.681975+0000","last_became_active":"2023-06-03T19:57:28.472757+0000","last_became_peered":"2023-06-03T19:57:28.472757+0000","last_unstale":"2023-06-03T20:06:10.681975+0000","last_undegraded":"2023-06-03T20:06:10.681975+0000","last_fullsized":"2023-06-03T20:06:10.681975+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T19:58:10.432188+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":122,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-06-03T20:06:10.644950+0000","last_change":"2023-06-03T19:56:54.507653+0000","last_active":"2023-06-03T20:06:10.644950+0000","last_peered":"2023-06-03T20:06:10.644950+0000","last_clean":"2023-06-03T20:06:10.644950+0000","last_became_active":"2023-06-03T19:56:54.507350+0000","last_became_peered":"2023-06-03T19:56:54.507350+0000","last_unstale":"2023-06-03T20:06:10.644950+0000","last_undegraded":"2023-06-03T20:06:10.644950+0000","last_fullsized":"2023-06-03T20:06:10.644950+0000","mapping_epoch":119,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T22:33:20.065844+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":145,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-06-03T20:06:10.307748+0000","last_change":"2023-06-03T19:56:54.512932+0000","last_active":"2023-06-03T20:06:10.307748+0000","last_peered":"2023-06-03T20:06:10.307748+0000","last_clean":"2023-06-03T20:06:10.307748+0000","last_became_active":"2023-06-03T19:56:54.511992+0000","last_became_peered":"2023-06-03T19:56:54.511992+0000","last_unstale":"2023-06-03T20:06:10.307748+0000","last_undegraded":"2023-06-03T20:06:10.307748+0000","last_fullsized":"2023-06-03T20:06:10.307748+0000","mapping_epoch":119,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T21:01:35.069911+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":129,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-06-03T20:06:09.863043+0000","last_change":"2023-06-03T19:51:59.359522+0000","last_active":"2023-06-03T20:06:09.863043+0000","last_peered":"2023-06-03T20:06:09.863043+0000","last_clean":"2023-06-03T20:06:09.863043+0000","last_became_active":"2023-06-03T19:51:59.359202+0000","last_became_peered":"2023-06-03T19:51:59.359202+0000","last_unstale":"2023-06-03T20:06:09.863043+0000","last_undegraded":"2023-06-03T20:06:09.863043+0000","last_fullsized":"2023-06-03T20:06:09.863043+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":120,"reported_epoch":163,"state":"active+clean","last_fresh":"2023-06-03T20:06:20.920151+0000","last_change":"2023-06-03T19:56:54.509490+0000","last_active":"2023-06-03T20:06:20.920151+0000","last_peered":"2023-06-03T20:06:20.920151+0000","last_clean":"2023-06-03T20:06:20.920151+0000","last_became_active":"2023-06-03T19:56:54.509145+0000","last_became_peered":"2023-06-03T19:56:54.509145+0000","last_unstale":"2023-06-03T20:06:20.920151+0000","last_undegraded":"2023-06-03T20:06:20.920151+0000","last_fullsized":"2023-06-03T20:06:20.920151+0000","mapping_epoch":119,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T07:12:03.964110+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":130,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-06-03T20:06:13.207343+0000","last_change":"2023-06-03T19:56:54.511325+0000","last_active":"2023-06-03T20:06:13.207343+0000","last_peered":"2023-06-03T20:06:13.207343+0000","last_clean":"2023-06-03T20:06:13.207343+0000","last_became_active":"2023-06-03T19:56:54.511033+0000","last_became_peered":"2023-06-03T19:56:54.511033+0000","last_unstale":"2023-06-03T20:06:13.207343+0000","last_undegraded":"2023-06-03T20:06:13.207343+0000","last_fullsized":"2023-06-03T20:06:13.207343+0000","mapping_epoch":119,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T23:50:08.711478+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":130,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-06-03T20:06:10.684258+0000","last_change":"2023-06-03T19:51:59.374143+0000","last_active":"2023-06-03T20:06:10.684258+0000","last_peered":"2023-06-03T20:06:10.684258+0000","last_clean":"2023-06-03T20:06:10.684258+0000","last_became_active":"2023-06-03T19:51:59.373146+0000","last_became_peered":"2023-06-03T19:51:59.373146+0000","last_unstale":"2023-06-03T20:06:10.684258+0000","last_undegraded":"2023-06-03T20:06:10.684258+0000","last_fullsized":"2023-06-03T20:06:10.684258+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":111,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-06-03T20:06:13.207456+0000","last_change":"2023-06-03T19:56:54.510462+0000","last_active":"2023-06-03T20:06:13.207456+0000","last_peered":"2023-06-03T20:06:13.207456+0000","last_clean":"2023-06-03T20:06:13.207456+0000","last_became_active":"2023-06-03T19:56:54.509783+0000","last_became_peered":"2023-06-03T19:56:54.509783+0000","last_unstale":"2023-06-03T20:06:13.207456+0000","last_undegraded":"2023-06-03T20:06:13.207456+0000","last_fullsized":"2023-06-03T20:06:13.207456+0000","mapping_epoch":119,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T00:16:09.624276+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"60'3","reported_seq":26,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-06-03T20:06:10.308319+0000","last_change":"2023-06-03T20:06:01.290013+0000","last_active":"2023-06-03T20:06:10.308319+0000","last_peered":"2023-06-03T20:06:10.308319+0000","last_clean":"2023-06-03T20:06:10.308319+0000","last_became_active":"2023-06-03T20:06:01.289622+0000","last_became_peered":"2023-06-03T20:06:01.289622+0000","last_unstale":"2023-06-03T20:06:10.308319+0000","last_undegraded":"2023-06-03T20:06:10.308319+0000","last_fullsized":"2023-06-03T20:06:10.308319+0000","mapping_epoch":160,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":161,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T07:15:37.027388+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,4],"acting":[3,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":102,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-06-03T20:06:10.682267+0000","last_change":"2023-06-03T19:56:11.753995+0000","last_active":"2023-06-03T20:06:10.682267+0000","last_peered":"2023-06-03T20:06:10.682267+0000","last_clean":"2023-06-03T20:06:10.682267+0000","last_became_active":"2023-06-03T19:56:11.753676+0000","last_became_peered":"2023-06-03T19:56:11.753676+0000","last_unstale":"2023-06-03T20:06:10.682267+0000","last_undegraded":"2023-06-03T20:06:10.682267+0000","last_fullsized":"2023-06-03T20:06:10.682267+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":115,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T01:00:40.343647+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"77'23","reported_seq":147,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-06-03T20:06:09.861586+0000","last_change":"2023-06-03T19:57:28.462522+0000","last_active":"2023-06-03T20:06:09.861586+0000","last_peered":"2023-06-03T20:06:09.861586+0000","last_clean":"2023-06-03T20:06:09.861586+0000","last_became_active":"2023-06-03T19:57:28.462048+0000","last_became_peered":"2023-06-03T19:57:28.462048+0000","last_unstale":"2023-06-03T20:06:09.861586+0000","last_undegraded":"2023-06-03T20:06:09.861586+0000","last_fullsized":"2023-06-03T20:06:09.861586+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":23,"log_dups_size":0,"ondisk_log_size":23,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T23:23:56.697534+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4716,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":23,"num_write_kb":20,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":124,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-06-03T20:06:09.861222+0000","last_change":"2023-06-03T19:57:28.469676+0000","last_active":"2023-06-03T20:06:09.861222+0000","last_peered":"2023-06-03T20:06:09.861222+0000","last_clean":"2023-06-03T20:06:09.861222+0000","last_became_active":"2023-06-03T19:57:28.469328+0000","last_became_peered":"2023-06-03T19:57:28.469328+0000","last_unstale":"2023-06-03T20:06:09.861222+0000","last_undegraded":"2023-06-03T20:06:09.861222+0000","last_fullsized":"2023-06-03T20:06:09.861222+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T23:25:26.041930+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"60'2","reported_seq":147,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-06-03T20:06:10.684129+0000","last_change":"2023-06-03T19:57:28.467711+0000","last_active":"2023-06-03T20:06:10.684129+0000","last_peered":"2023-06-03T20:06:10.684129+0000","last_clean":"2023-06-03T20:06:10.684129+0000","last_became_active":"2023-06-03T19:57:28.467234+0000","last_became_peered":"2023-06-03T19:57:28.467234+0000","last_unstale":"2023-06-03T20:06:10.684129+0000","last_undegraded":"2023-06-03T20:06:10.684129+0000","last_fullsized":"2023-06-03T20:06:10.684129+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":130,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-06-03T20:06:13.207518+0000","last_change":"2023-06-03T19:56:54.514616+0000","last_active":"2023-06-03T20:06:13.207518+0000","last_peered":"2023-06-03T20:06:13.207518+0000","last_clean":"2023-06-03T20:06:13.207518+0000","last_became_active":"2023-06-03T19:56:54.513918+0000","last_became_peered":"2023-06-03T19:56:54.513918+0000","last_unstale":"2023-06-03T20:06:13.207518+0000","last_undegraded":"2023-06-03T20:06:13.207518+0000","last_fullsized":"2023-06-03T20:06:13.207518+0000","mapping_epoch":119,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T19:36:10.967386+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":120,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-06-03T20:06:10.644940+0000","last_change":"2023-06-03T19:56:54.507479+0000","last_active":"2023-06-03T20:06:10.644940+0000","last_peered":"2023-06-03T20:06:10.644940+0000","last_clean":"2023-06-03T20:06:10.644940+0000","last_became_active":"2023-06-03T19:56:54.507149+0000","last_became_peered":"2023-06-03T19:56:54.507149+0000","last_unstale":"2023-06-03T20:06:10.644940+0000","last_undegraded":"2023-06-03T20:06:10.644940+0000","last_fullsized":"2023-06-03T20:06:10.644940+0000","mapping_epoch":119,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T20:22:36.676317+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":129,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-06-03T20:06:10.684136+0000","last_change":"2023-06-03T19:56:54.512472+0000","last_active":"2023-06-03T20:06:10.684136+0000","last_peered":"2023-06-03T20:06:10.684136+0000","last_clean":"2023-06-03T20:06:10.684136+0000","last_became_active":"2023-06-03T19:56:54.512051+0000","last_became_peered":"2023-06-03T19:56:54.512051+0000","last_unstale":"2023-06-03T20:06:10.684136+0000","last_undegraded":"2023-06-03T20:06:10.684136+0000","last_fullsized":"2023-06-03T20:06:10.684136+0000","mapping_epoch":119,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":97,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-06-03T20:06:09.860875+0000","last_change":"2023-06-03T20:06:01.288024+0000","last_active":"2023-06-03T20:06:09.860875+0000","last_peered":"2023-06-03T20:06:09.860875+0000","last_clean":"2023-06-03T20:06:09.860875+0000","last_became_active":"2023-06-03T20:06:01.287567+0000","last_became_peered":"2023-06-03T20:06:01.287567+0000","last_unstale":"2023-06-03T20:06:09.860875+0000","last_undegraded":"2023-06-03T20:06:09.860875+0000","last_fullsized":"2023-06-03T20:06:09.860875+0000","mapping_epoch":160,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":161,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T00:58:03.850069+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":121,"reported_epoch":163,"state":"active+clean","last_fresh":"2023-06-03T20:06:20.920635+0000","last_change":"2023-06-03T19:56:54.510289+0000","last_active":"2023-06-03T20:06:20.920635+0000","last_peered":"2023-06-03T20:06:20.920635+0000","last_clean":"2023-06-03T20:06:20.920635+0000","last_became_active":"2023-06-03T19:56:54.509905+0000","last_became_peered":"2023-06-03T19:56:54.509905+0000","last_unstale":"2023-06-03T20:06:20.920635+0000","last_undegraded":"2023-06-03T20:06:20.920635+0000","last_fullsized":"2023-06-03T20:06:20.920635+0000","mapping_epoch":119,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T02:54:26.792401+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"60'8","reported_seq":138,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-06-03T20:06:09.862488+0000","last_change":"2023-06-03T19:47:06.727778+0000","last_active":"2023-06-03T20:06:09.862488+0000","last_peered":"2023-06-03T20:06:09.862488+0000","last_clean":"2023-06-03T20:06:09.862488+0000","last_became_active":"2023-06-03T19:47:06.727305+0000","last_became_peered":"2023-06-03T19:47:06.727305+0000","last_unstale":"2023-06-03T20:06:09.862488+0000","last_undegraded":"2023-06-03T20:06:09.862488+0000","last_fullsized":"2023-06-03T20:06:09.862488+0000","mapping_epoch":86,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":87,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":129,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-06-03T20:06:10.308516+0000","last_change":"2023-06-03T19:56:54.512994+0000","last_active":"2023-06-03T20:06:10.308516+0000","last_peered":"2023-06-03T20:06:10.308516+0000","last_clean":"2023-06-03T20:06:10.308516+0000","last_became_active":"2023-06-03T19:56:54.512579+0000","last_became_peered":"2023-06-03T19:56:54.512579+0000","last_unstale":"2023-06-03T20:06:10.308516+0000","last_undegraded":"2023-06-03T20:06:10.308516+0000","last_fullsized":"2023-06-03T20:06:10.308516+0000","mapping_epoch":119,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T07:02:45.187961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"10.6","version":"163'1297","reported_seq":1310,"reported_epoch":163,"state":"active+clean","last_fresh":"2023-06-03T20:06:23.481838+0000","last_change":"2023-06-03T20:05:55.735684+0000","last_active":"2023-06-03T20:06:23.481838+0000","last_peered":"2023-06-03T20:06:23.481838+0000","last_clean":"2023-06-03T20:06:23.481838+0000","last_became_active":"2023-06-03T20:05:55.735414+0000","last_became_peered":"2023-06-03T20:05:55.735414+0000","last_unstale":"2023-06-03T20:06:23.481838+0000","last_undegraded":"2023-06-03T20:06:23.481838+0000","last_fullsized":"2023-06-03T20:06:23.481838+0000","mapping_epoch":157,"log_start":"0'0","ondisk_log_start":"0'0","created":157,"last_epoch_clean":158,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T20:05:54.710708+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T20:05:54.710708+0000","last_clean_scrub_stamp":"2023-06-03T20:05:54.710708+0000","objects_scrubbed":0,"log_size":1297,"log_dups_size":0,"ondisk_log_size":1297,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":85000192,"num_objects":1297,"num_object_clones":0,"num_object_copies":3891,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1297,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1297,"num_write_kb":83008,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,4],"acting":[5,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"60'1","reported_seq":146,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-06-03T20:06:10.684738+0000","last_change":"2023-06-03T19:57:28.464482+0000","last_active":"2023-06-03T20:06:10.684738+0000","last_peered":"2023-06-03T20:06:10.684738+0000","last_clean":"2023-06-03T20:06:10.684738+0000","last_became_active":"2023-06-03T19:57:28.464081+0000","last_became_peered":"2023-06-03T19:57:28.464081+0000","last_unstale":"2023-06-03T20:06:10.684738+0000","last_undegraded":"2023-06-03T20:06:10.684738+0000","last_fullsized":"2023-06-03T20:06:10.684738+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":114,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-06-03T20:06:10.683432+0000","last_change":"2023-06-03T19:34:17.848358+0000","last_active":"2023-06-03T20:06:10.683432+0000","last_peered":"2023-06-03T20:06:10.683432+0000","last_clean":"2023-06-03T20:06:10.683432+0000","last_became_active":"2023-06-03T19:34:17.847811+0000","last_became_peered":"2023-06-03T19:34:17.847811+0000","last_unstale":"2023-06-03T20:06:10.683432+0000","last_undegraded":"2023-06-03T20:06:10.683432+0000","last_fullsized":"2023-06-03T20:06:10.683432+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.7","version":"163'1314","reported_seq":1328,"reported_epoch":163,"state":"active+clean","last_fresh":"2023-06-03T20:06:23.084608+0000","last_change":"2023-06-03T20:05:56.145092+0000","last_active":"2023-06-03T20:06:23.084608+0000","last_peered":"2023-06-03T20:06:23.084608+0000","last_clean":"2023-06-03T20:06:23.084608+0000","last_became_active":"2023-06-03T20:05:56.144340+0000","last_became_peered":"2023-06-03T20:05:56.144340+0000","last_unstale":"2023-06-03T20:06:23.084608+0000","last_undegraded":"2023-06-03T20:06:23.084608+0000","last_fullsized":"2023-06-03T20:06:23.084608+0000","mapping_epoch":157,"log_start":"0'0","ondisk_log_start":"0'0","created":157,"last_epoch_clean":158,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T20:05:54.710708+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T20:05:54.710708+0000","last_clean_scrub_stamp":"2023-06-03T20:05:54.710708+0000","objects_scrubbed":0,"log_size":1314,"log_dups_size":0,"ondisk_log_size":1314,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":86114304,"num_objects":1314,"num_object_clones":0,"num_object_copies":3942,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1314,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1314,"num_write_kb":84096,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,5],"acting":[6,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.e","version":"60'2","reported_seq":147,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-06-03T20:06:10.684611+0000","last_change":"2023-06-03T19:57:28.468330+0000","last_active":"2023-06-03T20:06:10.684611+0000","last_peered":"2023-06-03T20:06:10.684611+0000","last_clean":"2023-06-03T20:06:10.684611+0000","last_became_active":"2023-06-03T19:57:28.467823+0000","last_became_peered":"2023-06-03T19:57:28.467823+0000","last_unstale":"2023-06-03T20:06:10.684611+0000","last_undegraded":"2023-06-03T20:06:10.684611+0000","last_fullsized":"2023-06-03T20:06:10.684611+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":163,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-06-03T20:06:09.860952+0000","last_change":"2023-06-03T19:57:28.461449+0000","last_active":"2023-06-03T20:06:09.860952+0000","last_peered":"2023-06-03T20:06:09.860952+0000","last_clean":"2023-06-03T20:06:09.860952+0000","last_became_active":"2023-06-03T19:57:28.460745+0000","last_became_peered":"2023-06-03T19:57:28.460745+0000","last_unstale":"2023-06-03T20:06:09.860952+0000","last_undegraded":"2023-06-03T20:06:09.860952+0000","last_fullsized":"2023-06-03T20:06:09.860952+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T20:41:19.361638+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,3],"acting":[4,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.4","version":"163'1532","reported_seq":1547,"reported_epoch":163,"state":"active+clean","last_fresh":"2023-06-03T20:06:25.199363+0000","last_change":"2023-06-03T20:05:56.144843+0000","last_active":"2023-06-03T20:06:25.199363+0000","last_peered":"2023-06-03T20:06:25.199363+0000","last_clean":"2023-06-03T20:06:25.199363+0000","last_became_active":"2023-06-03T20:05:56.144451+0000","last_became_peered":"2023-06-03T20:05:56.144451+0000","last_unstale":"2023-06-03T20:06:25.199363+0000","last_undegraded":"2023-06-03T20:06:25.199363+0000","last_fullsized":"2023-06-03T20:06:25.199363+0000","mapping_epoch":157,"log_start":"0'0","ondisk_log_start":"0'0","created":157,"last_epoch_clean":158,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T20:05:54.710708+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T20:05:54.710708+0000","last_clean_scrub_stamp":"2023-06-03T20:05:54.710708+0000","objects_scrubbed":0,"log_size":1532,"log_dups_size":0,"ondisk_log_size":1532,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T08:03:06.595205+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":100401152,"num_objects":1532,"num_object_clones":0,"num_object_copies":4596,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1532,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1532,"num_write_kb":98048,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.d","version":"60'1","reported_seq":25,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-06-03T20:06:10.684254+0000","last_change":"2023-06-03T20:06:01.288495+0000","last_active":"2023-06-03T20:06:10.684254+0000","last_peered":"2023-06-03T20:06:10.684254+0000","last_clean":"2023-06-03T20:06:10.684254+0000","last_became_active":"2023-06-03T20:06:01.288018+0000","last_became_peered":"2023-06-03T20:06:01.288018+0000","last_unstale":"2023-06-03T20:06:10.684254+0000","last_undegraded":"2023-06-03T20:06:10.684254+0000","last_fullsized":"2023-06-03T20:06:10.684254+0000","mapping_epoch":160,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":161,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":129,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-06-03T20:06:09.862080+0000","last_change":"2023-06-03T19:51:59.360650+0000","last_active":"2023-06-03T20:06:09.862080+0000","last_peered":"2023-06-03T20:06:09.862080+0000","last_clean":"2023-06-03T20:06:09.862080+0000","last_became_active":"2023-06-03T19:51:59.360237+0000","last_became_peered":"2023-06-03T19:51:59.360237+0000","last_unstale":"2023-06-03T20:06:09.862080+0000","last_undegraded":"2023-06-03T20:06:09.862080+0000","last_fullsized":"2023-06-03T20:06:09.862080+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"10.5","version":"163'1492","reported_seq":1507,"reported_epoch":163,"state":"active+clean","last_fresh":"2023-06-03T20:06:25.075648+0000","last_change":"2023-06-03T20:05:56.146177+0000","last_active":"2023-06-03T20:06:25.075648+0000","last_peered":"2023-06-03T20:06:25.075648+0000","last_clean":"2023-06-03T20:06:25.075648+0000","last_became_active":"2023-06-03T20:05:56.145605+0000","last_became_peered":"2023-06-03T20:05:56.145605+0000","last_unstale":"2023-06-03T20:06:25.075648+0000","last_undegraded":"2023-06-03T20:06:25.075648+0000","last_fullsized":"2023-06-03T20:06:25.075648+0000","mapping_epoch":157,"log_start":"0'0","ondisk_log_start":"0'0","created":157,"last_epoch_clean":158,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T20:05:54.710708+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T20:05:54.710708+0000","last_clean_scrub_stamp":"2023-06-03T20:05:54.710708+0000","objects_scrubbed":0,"log_size":1492,"log_dups_size":0,"ondisk_log_size":1492,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T00:13:09.237373+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":97779712,"num_objects":1492,"num_object_clones":0,"num_object_copies":4476,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1492,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1492,"num_write_kb":95488,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,3],"acting":[2,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":130,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-06-03T20:06:10.683722+0000","last_change":"2023-06-03T19:56:54.507432+0000","last_active":"2023-06-03T20:06:10.683722+0000","last_peered":"2023-06-03T20:06:10.683722+0000","last_clean":"2023-06-03T20:06:10.683722+0000","last_became_active":"2023-06-03T19:56:54.507012+0000","last_became_peered":"2023-06-03T19:56:54.507012+0000","last_unstale":"2023-06-03T20:06:10.683722+0000","last_undegraded":"2023-06-03T20:06:10.683722+0000","last_fullsized":"2023-06-03T20:06:10.683722+0000","mapping_epoch":119,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":148,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-06-03T20:06:10.645163+0000","last_change":"2023-06-03T19:57:28.464153+0000","last_active":"2023-06-03T20:06:10.645163+0000","last_peered":"2023-06-03T20:06:10.645163+0000","last_clean":"2023-06-03T20:06:10.645163+0000","last_became_active":"2023-06-03T19:57:28.463502+0000","last_became_peered":"2023-06-03T19:57:28.463502+0000","last_unstale":"2023-06-03T20:06:10.645163+0000","last_undegraded":"2023-06-03T20:06:10.645163+0000","last_fullsized":"2023-06-03T20:06:10.645163+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T06:11:00.999526+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,4],"acting":[0,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.2","version":"163'1405","reported_seq":1418,"reported_epoch":163,"state":"active+clean","last_fresh":"2023-06-03T20:06:23.748116+0000","last_change":"2023-06-03T20:05:55.725220+0000","last_active":"2023-06-03T20:06:23.748116+0000","last_peered":"2023-06-03T20:06:23.748116+0000","last_clean":"2023-06-03T20:06:23.748116+0000","last_became_active":"2023-06-03T20:05:55.724084+0000","last_became_peered":"2023-06-03T20:05:55.724084+0000","last_unstale":"2023-06-03T20:06:23.748116+0000","last_undegraded":"2023-06-03T20:06:23.748116+0000","last_fullsized":"2023-06-03T20:06:23.748116+0000","mapping_epoch":157,"log_start":"0'0","ondisk_log_start":"0'0","created":157,"last_epoch_clean":158,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T20:05:54.710708+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T20:05:54.710708+0000","last_clean_scrub_stamp":"2023-06-03T20:05:54.710708+0000","objects_scrubbed":0,"log_size":1405,"log_dups_size":0,"ondisk_log_size":1405,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T07:46:12.486865+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":92078080,"num_objects":1405,"num_object_clones":0,"num_object_copies":4215,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1405,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1405,"num_write_kb":89920,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,0],"acting":[1,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.b","version":"60'2","reported_seq":131,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-06-03T20:06:10.307996+0000","last_change":"2023-06-03T19:57:28.470310+0000","last_active":"2023-06-03T20:06:10.307996+0000","last_peered":"2023-06-03T20:06:10.307996+0000","last_clean":"2023-06-03T20:06:10.307996+0000","last_became_active":"2023-06-03T19:57:28.469887+0000","last_became_peered":"2023-06-03T19:57:28.469887+0000","last_unstale":"2023-06-03T20:06:10.307996+0000","last_undegraded":"2023-06-03T20:06:10.307996+0000","last_fullsized":"2023-06-03T20:06:10.307996+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T21:58:53.387131+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":138,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-06-03T20:06:09.861677+0000","last_change":"2023-06-03T19:57:28.465140+0000","last_active":"2023-06-03T20:06:09.861677+0000","last_peered":"2023-06-03T20:06:09.861677+0000","last_clean":"2023-06-03T20:06:09.861677+0000","last_became_active":"2023-06-03T19:57:28.464629+0000","last_became_peered":"2023-06-03T19:57:28.464629+0000","last_unstale":"2023-06-03T20:06:09.861677+0000","last_undegraded":"2023-06-03T20:06:09.861677+0000","last_fullsized":"2023-06-03T20:06:09.861677+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T21:47:11.619583+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.3","version":"163'1456","reported_seq":1471,"reported_epoch":163,"state":"active+clean","last_fresh":"2023-06-03T20:06:25.061642+0000","last_change":"2023-06-03T20:05:56.146193+0000","last_active":"2023-06-03T20:06:25.061642+0000","last_peered":"2023-06-03T20:06:25.061642+0000","last_clean":"2023-06-03T20:06:25.061642+0000","last_became_active":"2023-06-03T20:05:56.145532+0000","last_became_peered":"2023-06-03T20:05:56.145532+0000","last_unstale":"2023-06-03T20:06:25.061642+0000","last_undegraded":"2023-06-03T20:06:25.061642+0000","last_fullsized":"2023-06-03T20:06:25.061642+0000","mapping_epoch":157,"log_start":"0'0","ondisk_log_start":"0'0","created":157,"last_epoch_clean":158,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T20:05:54.710708+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T20:05:54.710708+0000","last_clean_scrub_stamp":"2023-06-03T20:05:54.710708+0000","objects_scrubbed":0,"log_size":1456,"log_dups_size":0,"ondisk_log_size":1456,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T22:05:40.865679+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":95420416,"num_objects":1456,"num_object_clones":0,"num_object_copies":4368,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1456,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1456,"num_write_kb":93184,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":145,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-06-03T20:06:09.861909+0000","last_change":"2023-06-03T19:57:28.461401+0000","last_active":"2023-06-03T20:06:09.861909+0000","last_peered":"2023-06-03T20:06:09.861909+0000","last_clean":"2023-06-03T20:06:09.861909+0000","last_became_active":"2023-06-03T19:57:28.461057+0000","last_became_peered":"2023-06-03T19:57:28.461057+0000","last_unstale":"2023-06-03T20:06:09.861909+0000","last_undegraded":"2023-06-03T20:06:09.861909+0000","last_fullsized":"2023-06-03T20:06:09.861909+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":143,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-06-03T20:06:09.862003+0000","last_change":"2023-06-03T19:57:28.456750+0000","last_active":"2023-06-03T20:06:09.862003+0000","last_peered":"2023-06-03T20:06:09.862003+0000","last_clean":"2023-06-03T20:06:09.862003+0000","last_became_active":"2023-06-03T19:57:28.456436+0000","last_became_peered":"2023-06-03T19:57:28.456436+0000","last_unstale":"2023-06-03T20:06:09.862003+0000","last_undegraded":"2023-06-03T20:06:09.862003+0000","last_fullsized":"2023-06-03T20:06:09.862003+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T01:49:25.481554+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,5],"acting":[4,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.0","version":"163'1243","reported_seq":1257,"reported_epoch":163,"state":"active+clean","last_fresh":"2023-06-03T20:06:23.063094+0000","last_change":"2023-06-03T20:05:55.729742+0000","last_active":"2023-06-03T20:06:23.063094+0000","last_peered":"2023-06-03T20:06:23.063094+0000","last_clean":"2023-06-03T20:06:23.063094+0000","last_became_active":"2023-06-03T20:05:55.729339+0000","last_became_peered":"2023-06-03T20:05:55.729339+0000","last_unstale":"2023-06-03T20:06:23.063094+0000","last_undegraded":"2023-06-03T20:06:23.063094+0000","last_fullsized":"2023-06-03T20:06:23.063094+0000","mapping_epoch":157,"log_start":"0'0","ondisk_log_start":"0'0","created":157,"last_epoch_clean":158,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T20:05:54.710708+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T20:05:54.710708+0000","last_clean_scrub_stamp":"2023-06-03T20:05:54.710708+0000","objects_scrubbed":0,"log_size":1243,"log_dups_size":0,"ondisk_log_size":1243,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":81461248,"num_objects":1243,"num_object_clones":0,"num_object_copies":3729,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1243,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1243,"num_write_kb":79552,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,4],"acting":[6,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":136,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-06-03T20:06:09.861133+0000","last_change":"2023-06-03T19:57:28.462344+0000","last_active":"2023-06-03T20:06:09.861133+0000","last_peered":"2023-06-03T20:06:09.861133+0000","last_clean":"2023-06-03T20:06:09.861133+0000","last_became_active":"2023-06-03T19:57:28.462001+0000","last_became_peered":"2023-06-03T19:57:28.462001+0000","last_unstale":"2023-06-03T20:06:09.861133+0000","last_undegraded":"2023-06-03T20:06:09.861133+0000","last_fullsized":"2023-06-03T20:06:09.861133+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T00:45:13.789650+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":124,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-06-03T20:06:09.861971+0000","last_change":"2023-06-03T19:57:28.467917+0000","last_active":"2023-06-03T20:06:09.861971+0000","last_peered":"2023-06-03T20:06:09.861971+0000","last_clean":"2023-06-03T20:06:09.861971+0000","last_became_active":"2023-06-03T19:57:28.467617+0000","last_became_peered":"2023-06-03T19:57:28.467617+0000","last_unstale":"2023-06-03T20:06:09.861971+0000","last_undegraded":"2023-06-03T20:06:09.861971+0000","last_fullsized":"2023-06-03T20:06:09.861971+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T23:34:23.840582+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.1","version":"163'1063","reported_seq":1078,"reported_epoch":163,"state":"active+clean","last_fresh":"2023-06-03T20:06:21.552940+0000","last_change":"2023-06-03T20:05:55.725679+0000","last_active":"2023-06-03T20:06:21.552940+0000","last_peered":"2023-06-03T20:06:21.552940+0000","last_clean":"2023-06-03T20:06:21.552940+0000","last_became_active":"2023-06-03T20:05:55.725336+0000","last_became_peered":"2023-06-03T20:05:55.725336+0000","last_unstale":"2023-06-03T20:06:21.552940+0000","last_undegraded":"2023-06-03T20:06:21.552940+0000","last_fullsized":"2023-06-03T20:06:21.552940+0000","mapping_epoch":157,"log_start":"0'0","ondisk_log_start":"0'0","created":157,"last_epoch_clean":158,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T20:05:54.710708+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T20:05:54.710708+0000","last_clean_scrub_stamp":"2023-06-03T20:05:54.710708+0000","objects_scrubbed":0,"log_size":1063,"log_dups_size":0,"ondisk_log_size":1063,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T03:22:27.476918+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":69664768,"num_objects":1063,"num_object_clones":0,"num_object_copies":3189,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1063,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1063,"num_write_kb":68032,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":131,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-06-03T20:06:10.307934+0000","last_change":"2023-06-03T19:56:54.512219+0000","last_active":"2023-06-03T20:06:10.307934+0000","last_peered":"2023-06-03T20:06:10.307934+0000","last_clean":"2023-06-03T20:06:10.307934+0000","last_became_active":"2023-06-03T19:56:54.511771+0000","last_became_peered":"2023-06-03T19:56:54.511771+0000","last_unstale":"2023-06-03T20:06:10.307934+0000","last_undegraded":"2023-06-03T20:06:10.307934+0000","last_fullsized":"2023-06-03T20:06:10.307934+0000","mapping_epoch":119,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T19:35:47.796520+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":117,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-06-03T20:06:10.682634+0000","last_change":"2023-06-03T19:57:28.471023+0000","last_active":"2023-06-03T20:06:10.682634+0000","last_peered":"2023-06-03T20:06:10.682634+0000","last_clean":"2023-06-03T20:06:10.682634+0000","last_became_active":"2023-06-03T19:57:28.470452+0000","last_became_peered":"2023-06-03T19:57:28.470452+0000","last_unstale":"2023-06-03T20:06:10.682634+0000","last_undegraded":"2023-06-03T20:06:10.682634+0000","last_fullsized":"2023-06-03T20:06:10.682634+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T05:58:33.305030+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.c","version":"163'1325","reported_seq":1339,"reported_epoch":163,"state":"active+clean","last_fresh":"2023-06-03T20:06:23.390643+0000","last_change":"2023-06-03T20:05:55.731273+0000","last_active":"2023-06-03T20:06:23.390643+0000","last_peered":"2023-06-03T20:06:23.390643+0000","last_clean":"2023-06-03T20:06:23.390643+0000","last_became_active":"2023-06-03T20:05:55.730890+0000","last_became_peered":"2023-06-03T20:05:55.730890+0000","last_unstale":"2023-06-03T20:06:23.390643+0000","last_undegraded":"2023-06-03T20:06:23.390643+0000","last_fullsized":"2023-06-03T20:06:23.390643+0000","mapping_epoch":157,"log_start":"0'0","ondisk_log_start":"0'0","created":157,"last_epoch_clean":158,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T20:05:54.710708+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T20:05:54.710708+0000","last_clean_scrub_stamp":"2023-06-03T20:05:54.710708+0000","objects_scrubbed":0,"log_size":1325,"log_dups_size":0,"ondisk_log_size":1325,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":86835200,"num_objects":1325,"num_object_clones":0,"num_object_copies":3975,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1325,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1325,"num_write_kb":84800,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":135,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-06-03T20:06:10.682956+0000","last_change":"2023-06-03T19:51:59.392796+0000","last_active":"2023-06-03T20:06:10.682956+0000","last_peered":"2023-06-03T20:06:10.682956+0000","last_clean":"2023-06-03T20:06:10.682956+0000","last_became_active":"2023-06-03T19:51:59.380207+0000","last_became_peered":"2023-06-03T19:51:59.380207+0000","last_unstale":"2023-06-03T20:06:10.682956+0000","last_undegraded":"2023-06-03T20:06:10.682956+0000","last_fullsized":"2023-06-03T20:06:10.682956+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:32:20.692043+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:32:20.692043+0000","last_clean_scrub_stamp":"2023-06-03T19:32:20.692043+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T00:28:05.866873+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":145,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-06-03T20:06:10.684027+0000","last_change":"2023-06-03T19:56:54.512607+0000","last_active":"2023-06-03T20:06:10.684027+0000","last_peered":"2023-06-03T20:06:10.684027+0000","last_clean":"2023-06-03T20:06:10.684027+0000","last_became_active":"2023-06-03T19:56:54.512282+0000","last_became_peered":"2023-06-03T19:56:54.512282+0000","last_unstale":"2023-06-03T20:06:10.684027+0000","last_undegraded":"2023-06-03T20:06:10.684027+0000","last_fullsized":"2023-06-03T20:06:10.684027+0000","mapping_epoch":119,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":102,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-06-03T20:06:10.682333+0000","last_change":"2023-06-03T19:57:28.470295+0000","last_active":"2023-06-03T20:06:10.682333+0000","last_peered":"2023-06-03T20:06:10.682333+0000","last_clean":"2023-06-03T20:06:10.682333+0000","last_became_active":"2023-06-03T19:57:28.469383+0000","last_became_peered":"2023-06-03T19:57:28.469383+0000","last_unstale":"2023-06-03T20:06:10.682333+0000","last_undegraded":"2023-06-03T20:06:10.682333+0000","last_fullsized":"2023-06-03T20:06:10.682333+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T05:35:19.667447+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.a","version":"163'1218","reported_seq":1232,"reported_epoch":163,"state":"active+clean","last_fresh":"2023-06-03T20:06:23.087427+0000","last_change":"2023-06-03T20:05:55.734776+0000","last_active":"2023-06-03T20:06:23.087427+0000","last_peered":"2023-06-03T20:06:23.087427+0000","last_clean":"2023-06-03T20:06:23.087427+0000","last_became_active":"2023-06-03T20:05:55.734421+0000","last_became_peered":"2023-06-03T20:05:55.734421+0000","last_unstale":"2023-06-03T20:06:23.087427+0000","last_undegraded":"2023-06-03T20:06:23.087427+0000","last_fullsized":"2023-06-03T20:06:23.087427+0000","mapping_epoch":157,"log_start":"0'0","ondisk_log_start":"0'0","created":157,"last_epoch_clean":158,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T20:05:54.710708+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T20:05:54.710708+0000","last_clean_scrub_stamp":"2023-06-03T20:05:54.710708+0000","objects_scrubbed":0,"log_size":1218,"log_dups_size":0,"ondisk_log_size":1218,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":79822848,"num_objects":1218,"num_object_clones":0,"num_object_copies":3654,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1218,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1218,"num_write_kb":77952,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":134,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-06-03T20:06:10.683742+0000","last_change":"2023-06-03T19:51:59.373777+0000","last_active":"2023-06-03T20:06:10.683742+0000","last_peered":"2023-06-03T20:06:10.683742+0000","last_clean":"2023-06-03T20:06:10.683742+0000","last_became_active":"2023-06-03T19:51:59.372988+0000","last_became_peered":"2023-06-03T19:51:59.372988+0000","last_unstale":"2023-06-03T20:06:10.683742+0000","last_undegraded":"2023-06-03T20:06:10.683742+0000","last_fullsized":"2023-06-03T20:06:10.683742+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:32:20.692043+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:32:20.692043+0000","last_clean_scrub_stamp":"2023-06-03T19:32:20.692043+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":109,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-06-03T20:06:09.862290+0000","last_change":"2023-06-03T19:57:28.460195+0000","last_active":"2023-06-03T20:06:09.862290+0000","last_peered":"2023-06-03T20:06:09.862290+0000","last_clean":"2023-06-03T20:06:09.862290+0000","last_became_active":"2023-06-03T19:57:28.459823+0000","last_became_peered":"2023-06-03T19:57:28.459823+0000","last_unstale":"2023-06-03T20:06:09.862290+0000","last_undegraded":"2023-06-03T20:06:09.862290+0000","last_fullsized":"2023-06-03T20:06:09.862290+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T20:08:48.686380+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":117,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-06-03T20:06:10.682739+0000","last_change":"2023-06-03T19:56:54.516859+0000","last_active":"2023-06-03T20:06:10.682739+0000","last_peered":"2023-06-03T20:06:10.682739+0000","last_clean":"2023-06-03T20:06:10.682739+0000","last_became_active":"2023-06-03T19:56:54.516521+0000","last_became_peered":"2023-06-03T19:56:54.516521+0000","last_unstale":"2023-06-03T20:06:10.682739+0000","last_undegraded":"2023-06-03T20:06:10.682739+0000","last_fullsized":"2023-06-03T20:06:10.682739+0000","mapping_epoch":119,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T20:03:25.227382+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.9","version":"163'1336","reported_seq":1350,"reported_epoch":163,"state":"active+clean","last_fresh":"2023-06-03T20:06:23.465961+0000","last_change":"2023-06-03T20:05:55.734829+0000","last_active":"2023-06-03T20:06:23.465961+0000","last_peered":"2023-06-03T20:06:23.465961+0000","last_clean":"2023-06-03T20:06:23.465961+0000","last_became_active":"2023-06-03T20:05:55.734289+0000","last_became_peered":"2023-06-03T20:05:55.734289+0000","last_unstale":"2023-06-03T20:06:23.465961+0000","last_undegraded":"2023-06-03T20:06:23.465961+0000","last_fullsized":"2023-06-03T20:06:23.465961+0000","mapping_epoch":157,"log_start":"0'0","ondisk_log_start":"0'0","created":157,"last_epoch_clean":158,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T20:05:54.710708+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T20:05:54.710708+0000","last_clean_scrub_stamp":"2023-06-03T20:05:54.710708+0000","objects_scrubbed":0,"log_size":1336,"log_dups_size":0,"ondisk_log_size":1336,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":87556096,"num_objects":1336,"num_object_clones":0,"num_object_copies":4008,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1336,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1336,"num_write_kb":85504,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":138,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-06-03T20:06:09.860884+0000","last_change":"2023-06-03T19:56:54.504859+0000","last_active":"2023-06-03T20:06:09.860884+0000","last_peered":"2023-06-03T20:06:09.860884+0000","last_clean":"2023-06-03T20:06:09.860884+0000","last_became_active":"2023-06-03T19:56:54.504455+0000","last_became_peered":"2023-06-03T19:56:54.504455+0000","last_unstale":"2023-06-03T20:06:09.860884+0000","last_undegraded":"2023-06-03T20:06:09.860884+0000","last_fullsized":"2023-06-03T20:06:09.860884+0000","mapping_epoch":119,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:32:20.692043+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:32:20.692043+0000","last_clean_scrub_stamp":"2023-06-03T19:32:20.692043+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T06:25:45.522100+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":144,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-06-03T20:06:10.682294+0000","last_change":"2023-06-03T19:56:11.754048+0000","last_active":"2023-06-03T20:06:10.682294+0000","last_peered":"2023-06-03T20:06:10.682294+0000","last_clean":"2023-06-03T20:06:10.682294+0000","last_became_active":"2023-06-03T19:56:11.753687+0000","last_became_peered":"2023-06-03T19:56:11.753687+0000","last_unstale":"2023-06-03T20:06:10.682294+0000","last_undegraded":"2023-06-03T20:06:10.682294+0000","last_fullsized":"2023-06-03T20:06:10.682294+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":115,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T23:29:41.511496+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":130,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-06-03T20:06:10.682745+0000","last_change":"2023-06-03T19:51:59.396573+0000","last_active":"2023-06-03T20:06:10.682745+0000","last_peered":"2023-06-03T20:06:10.682745+0000","last_clean":"2023-06-03T20:06:10.682745+0000","last_became_active":"2023-06-03T19:51:59.381593+0000","last_became_peered":"2023-06-03T19:51:59.381593+0000","last_unstale":"2023-06-03T20:06:10.682745+0000","last_undegraded":"2023-06-03T20:06:10.682745+0000","last_fullsized":"2023-06-03T20:06:10.682745+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T21:33:23.043772+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.b","version":"163'1206","reported_seq":1218,"reported_epoch":163,"state":"active+clean","last_fresh":"2023-06-03T20:06:23.092084+0000","last_change":"2023-06-03T20:05:55.729711+0000","last_active":"2023-06-03T20:06:23.092084+0000","last_peered":"2023-06-03T20:06:23.092084+0000","last_clean":"2023-06-03T20:06:23.092084+0000","last_became_active":"2023-06-03T20:05:55.729321+0000","last_became_peered":"2023-06-03T20:05:55.729321+0000","last_unstale":"2023-06-03T20:06:23.092084+0000","last_undegraded":"2023-06-03T20:06:23.092084+0000","last_fullsized":"2023-06-03T20:06:23.092084+0000","mapping_epoch":157,"log_start":"0'0","ondisk_log_start":"0'0","created":157,"last_epoch_clean":158,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T20:05:54.710708+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T20:05:54.710708+0000","last_clean_scrub_stamp":"2023-06-03T20:05:54.710708+0000","objects_scrubbed":0,"log_size":1206,"log_dups_size":0,"ondisk_log_size":1206,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":79036416,"num_objects":1206,"num_object_clones":0,"num_object_copies":3618,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1206,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1206,"num_write_kb":77184,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,0],"acting":[6,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":134,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-06-03T20:06:10.683960+0000","last_change":"2023-06-03T19:56:11.752151+0000","last_active":"2023-06-03T20:06:10.683960+0000","last_peered":"2023-06-03T20:06:10.683960+0000","last_clean":"2023-06-03T20:06:10.683960+0000","last_became_active":"2023-06-03T19:56:11.751767+0000","last_became_peered":"2023-06-03T19:56:11.751767+0000","last_unstale":"2023-06-03T20:06:10.683960+0000","last_undegraded":"2023-06-03T20:06:10.683960+0000","last_fullsized":"2023-06-03T20:06:10.683960+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":115,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:32:20.692043+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:32:20.692043+0000","last_clean_scrub_stamp":"2023-06-03T19:32:20.692043+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"72'139","reported_seq":272,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-06-03T20:06:10.644431+0000","last_change":"2023-06-03T20:06:01.290016+0000","last_active":"2023-06-03T20:06:10.644431+0000","last_peered":"2023-06-03T20:06:10.644431+0000","last_clean":"2023-06-03T20:06:10.644431+0000","last_became_active":"2023-06-03T20:06:01.289525+0000","last_became_peered":"2023-06-03T20:06:01.289525+0000","last_unstale":"2023-06-03T20:06:10.644431+0000","last_undegraded":"2023-06-03T20:06:10.644431+0000","last_fullsized":"2023-06-03T20:06:10.644431+0000","mapping_epoch":160,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":161,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:27:28.100808+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:27:28.100808+0000","last_clean_scrub_stamp":"2023-06-03T19:27:28.100808+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T23:38:26.201980+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6],"acting":[0,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":111,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-06-03T20:06:10.307745+0000","last_change":"2023-06-03T19:56:54.509571+0000","last_active":"2023-06-03T20:06:10.307745+0000","last_peered":"2023-06-03T20:06:10.307745+0000","last_clean":"2023-06-03T20:06:10.307745+0000","last_became_active":"2023-06-03T19:56:54.509213+0000","last_became_peered":"2023-06-03T19:56:54.509213+0000","last_unstale":"2023-06-03T20:06:10.307745+0000","last_undegraded":"2023-06-03T20:06:10.307745+0000","last_fullsized":"2023-06-03T20:06:10.307745+0000","mapping_epoch":119,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T21:26:56.003203+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":129,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-06-03T20:06:09.862778+0000","last_change":"2023-06-03T19:57:28.459664+0000","last_active":"2023-06-03T20:06:09.862778+0000","last_peered":"2023-06-03T20:06:09.862778+0000","last_clean":"2023-06-03T20:06:09.862778+0000","last_became_active":"2023-06-03T19:57:28.459289+0000","last_became_peered":"2023-06-03T19:57:28.459289+0000","last_unstale":"2023-06-03T20:06:09.862778+0000","last_undegraded":"2023-06-03T20:06:09.862778+0000","last_fullsized":"2023-06-03T20:06:09.862778+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"10.8","version":"163'1576","reported_seq":1591,"reported_epoch":163,"state":"active+clean","last_fresh":"2023-06-03T20:06:25.219047+0000","last_change":"2023-06-03T20:05:55.727736+0000","last_active":"2023-06-03T20:06:25.219047+0000","last_peered":"2023-06-03T20:06:25.219047+0000","last_clean":"2023-06-03T20:06:25.219047+0000","last_became_active":"2023-06-03T20:05:55.727414+0000","last_became_peered":"2023-06-03T20:05:55.727414+0000","last_unstale":"2023-06-03T20:06:25.219047+0000","last_undegraded":"2023-06-03T20:06:25.219047+0000","last_fullsized":"2023-06-03T20:06:25.219047+0000","mapping_epoch":157,"log_start":"0'0","ondisk_log_start":"0'0","created":157,"last_epoch_clean":158,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T20:05:54.710708+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T20:05:54.710708+0000","last_clean_scrub_stamp":"2023-06-03T20:05:54.710708+0000","objects_scrubbed":0,"log_size":1576,"log_dups_size":0,"ondisk_log_size":1576,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T04:09:06.487024+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":103284736,"num_objects":1576,"num_object_clones":0,"num_object_copies":4728,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1576,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1576,"num_write_kb":100864,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,7],"acting":[0,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":95,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-06-03T20:06:10.681897+0000","last_change":"2023-06-03T20:06:01.289986+0000","last_active":"2023-06-03T20:06:10.681897+0000","last_peered":"2023-06-03T20:06:10.681897+0000","last_clean":"2023-06-03T20:06:10.681897+0000","last_became_active":"2023-06-03T20:06:01.289624+0000","last_became_peered":"2023-06-03T20:06:01.289624+0000","last_unstale":"2023-06-03T20:06:10.681897+0000","last_undegraded":"2023-06-03T20:06:10.681897+0000","last_fullsized":"2023-06-03T20:06:10.681897+0000","mapping_epoch":160,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":161,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:32:20.692043+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:32:20.692043+0000","last_clean_scrub_stamp":"2023-06-03T19:32:20.692043+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T02:29:30.043580+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,0],"acting":[1,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":136,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-06-03T20:06:10.644606+0000","last_change":"2023-06-03T19:57:28.459611+0000","last_active":"2023-06-03T20:06:10.644606+0000","last_peered":"2023-06-03T20:06:10.644606+0000","last_clean":"2023-06-03T20:06:10.644606+0000","last_became_active":"2023-06-03T19:57:28.459264+0000","last_became_peered":"2023-06-03T19:57:28.459264+0000","last_unstale":"2023-06-03T20:06:10.644606+0000","last_undegraded":"2023-06-03T20:06:10.644606+0000","last_fullsized":"2023-06-03T20:06:10.644606+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T22:15:06.557231+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":136,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-06-03T20:06:10.645079+0000","last_change":"2023-06-03T19:56:11.745678+0000","last_active":"2023-06-03T20:06:10.645079+0000","last_peered":"2023-06-03T20:06:10.645079+0000","last_clean":"2023-06-03T20:06:10.645079+0000","last_became_active":"2023-06-03T19:56:11.744966+0000","last_became_peered":"2023-06-03T19:56:11.744966+0000","last_unstale":"2023-06-03T20:06:10.645079+0000","last_undegraded":"2023-06-03T20:06:10.645079+0000","last_fullsized":"2023-06-03T20:06:10.645079+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":115,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T21:12:06.830295+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.d","version":"163'1357","reported_seq":1371,"reported_epoch":163,"state":"active+clean","last_fresh":"2023-06-03T20:06:23.095553+0000","last_change":"2023-06-03T20:05:55.729883+0000","last_active":"2023-06-03T20:06:23.095553+0000","last_peered":"2023-06-03T20:06:23.095553+0000","last_clean":"2023-06-03T20:06:23.095553+0000","last_became_active":"2023-06-03T20:05:55.729472+0000","last_became_peered":"2023-06-03T20:05:55.729472+0000","last_unstale":"2023-06-03T20:06:23.095553+0000","last_undegraded":"2023-06-03T20:06:23.095553+0000","last_fullsized":"2023-06-03T20:06:23.095553+0000","mapping_epoch":157,"log_start":"0'0","ondisk_log_start":"0'0","created":157,"last_epoch_clean":158,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T20:05:54.710708+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T20:05:54.710708+0000","last_clean_scrub_stamp":"2023-06-03T20:05:54.710708+0000","objects_scrubbed":0,"log_size":1357,"log_dups_size":0,"ondisk_log_size":1357,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":88932352,"num_objects":1357,"num_object_clones":0,"num_object_copies":4071,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1357,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1357,"num_write_kb":86848,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":149,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-06-03T20:06:10.685074+0000","last_change":"2023-06-03T19:57:28.466096+0000","last_active":"2023-06-03T20:06:10.685074+0000","last_peered":"2023-06-03T20:06:10.685074+0000","last_clean":"2023-06-03T20:06:10.685074+0000","last_became_active":"2023-06-03T19:57:28.464954+0000","last_became_peered":"2023-06-03T19:57:28.464954+0000","last_unstale":"2023-06-03T20:06:10.685074+0000","last_undegraded":"2023-06-03T20:06:10.685074+0000","last_fullsized":"2023-06-03T20:06:10.685074+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:32:20.692043+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:32:20.692043+0000","last_clean_scrub_stamp":"2023-06-03T19:32:20.692043+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":143,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-06-03T20:06:10.682784+0000","last_change":"2023-06-03T19:56:11.750241+0000","last_active":"2023-06-03T20:06:10.682784+0000","last_peered":"2023-06-03T20:06:10.682784+0000","last_clean":"2023-06-03T20:06:10.682784+0000","last_became_active":"2023-06-03T19:56:11.749687+0000","last_became_peered":"2023-06-03T19:56:11.749687+0000","last_unstale":"2023-06-03T20:06:10.682784+0000","last_undegraded":"2023-06-03T20:06:10.682784+0000","last_fullsized":"2023-06-03T20:06:10.682784+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":115,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T22:38:25.819929+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":107,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-06-03T20:06:10.644909+0000","last_change":"2023-06-03T19:47:06.733532+0000","last_active":"2023-06-03T20:06:10.644909+0000","last_peered":"2023-06-03T20:06:10.644909+0000","last_clean":"2023-06-03T20:06:10.644909+0000","last_became_active":"2023-06-03T19:47:06.733131+0000","last_became_peered":"2023-06-03T19:47:06.733131+0000","last_unstale":"2023-06-03T20:06:10.644909+0000","last_undegraded":"2023-06-03T20:06:10.644909+0000","last_fullsized":"2023-06-03T20:06:10.644909+0000","mapping_epoch":86,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":87,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T23:46:03.788646+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.e","version":"163'1107","reported_seq":1122,"reported_epoch":163,"state":"active+clean","last_fresh":"2023-06-03T20:06:21.539638+0000","last_change":"2023-06-03T20:05:55.729725+0000","last_active":"2023-06-03T20:06:21.539638+0000","last_peered":"2023-06-03T20:06:21.539638+0000","last_clean":"2023-06-03T20:06:21.539638+0000","last_became_active":"2023-06-03T20:05:55.729373+0000","last_became_peered":"2023-06-03T20:05:55.729373+0000","last_unstale":"2023-06-03T20:06:21.539638+0000","last_undegraded":"2023-06-03T20:06:21.539638+0000","last_fullsized":"2023-06-03T20:06:21.539638+0000","mapping_epoch":157,"log_start":"0'0","ondisk_log_start":"0'0","created":157,"last_epoch_clean":158,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T20:05:54.710708+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T20:05:54.710708+0000","last_clean_scrub_stamp":"2023-06-03T20:05:54.710708+0000","objects_scrubbed":0,"log_size":1107,"log_dups_size":0,"ondisk_log_size":1107,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T21:15:44.533891+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":72548352,"num_objects":1107,"num_object_clones":0,"num_object_copies":3321,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1107,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1107,"num_write_kb":70848,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"60'2","reported_seq":116,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-06-03T20:06:10.308491+0000","last_change":"2023-06-03T19:56:54.510148+0000","last_active":"2023-06-03T20:06:10.308491+0000","last_peered":"2023-06-03T20:06:10.308491+0000","last_clean":"2023-06-03T20:06:10.308491+0000","last_became_active":"2023-06-03T19:56:54.509771+0000","last_became_peered":"2023-06-03T19:56:54.509771+0000","last_unstale":"2023-06-03T20:06:10.308491+0000","last_undegraded":"2023-06-03T20:06:10.308491+0000","last_fullsized":"2023-06-03T20:06:10.308491+0000","mapping_epoch":119,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T00:52:42.807432+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":143,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-06-03T20:06:10.682666+0000","last_change":"2023-06-03T19:57:28.470954+0000","last_active":"2023-06-03T20:06:10.682666+0000","last_peered":"2023-06-03T20:06:10.682666+0000","last_clean":"2023-06-03T20:06:10.682666+0000","last_became_active":"2023-06-03T19:57:28.470524+0000","last_became_peered":"2023-06-03T19:57:28.470524+0000","last_unstale":"2023-06-03T20:06:10.682666+0000","last_undegraded":"2023-06-03T20:06:10.682666+0000","last_fullsized":"2023-06-03T20:06:10.682666+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:32:20.692043+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:32:20.692043+0000","last_clean_scrub_stamp":"2023-06-03T19:32:20.692043+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T19:39:19.983271+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":113,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-06-03T20:06:10.307891+0000","last_change":"2023-06-03T19:56:54.507670+0000","last_active":"2023-06-03T20:06:10.307891+0000","last_peered":"2023-06-03T20:06:10.307891+0000","last_clean":"2023-06-03T20:06:10.307891+0000","last_became_active":"2023-06-03T19:56:54.507293+0000","last_became_peered":"2023-06-03T19:56:54.507293+0000","last_unstale":"2023-06-03T20:06:10.307891+0000","last_undegraded":"2023-06-03T20:06:10.307891+0000","last_fullsized":"2023-06-03T20:06:10.307891+0000","mapping_epoch":119,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T01:49:19.966054+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"10.f","version":"163'1322","reported_seq":1336,"reported_epoch":163,"state":"active+clean","last_fresh":"2023-06-03T20:06:23.093960+0000","last_change":"2023-06-03T20:05:56.145329+0000","last_active":"2023-06-03T20:06:23.093960+0000","last_peered":"2023-06-03T20:06:23.093960+0000","last_clean":"2023-06-03T20:06:23.093960+0000","last_became_active":"2023-06-03T20:05:56.144587+0000","last_became_peered":"2023-06-03T20:05:56.144587+0000","last_unstale":"2023-06-03T20:06:23.093960+0000","last_undegraded":"2023-06-03T20:06:23.093960+0000","last_fullsized":"2023-06-03T20:06:23.093960+0000","mapping_epoch":157,"log_start":"0'0","ondisk_log_start":"0'0","created":157,"last_epoch_clean":158,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T20:05:54.710708+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T20:05:54.710708+0000","last_clean_scrub_stamp":"2023-06-03T20:05:54.710708+0000","objects_scrubbed":0,"log_size":1322,"log_dups_size":0,"ondisk_log_size":1322,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":86638592,"num_objects":1322,"num_object_clones":0,"num_object_copies":3966,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1322,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1322,"num_write_kb":84608,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,2],"acting":[6,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":134,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-06-03T20:06:09.862934+0000","last_change":"2023-06-03T19:56:11.752730+0000","last_active":"2023-06-03T20:06:09.862934+0000","last_peered":"2023-06-03T20:06:09.862934+0000","last_clean":"2023-06-03T20:06:09.862934+0000","last_became_active":"2023-06-03T19:56:11.752441+0000","last_became_peered":"2023-06-03T19:56:11.752441+0000","last_unstale":"2023-06-03T20:06:09.862934+0000","last_undegraded":"2023-06-03T20:06:09.862934+0000","last_fullsized":"2023-06-03T20:06:09.862934+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":115,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:32:20.692043+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:32:20.692043+0000","last_clean_scrub_stamp":"2023-06-03T19:32:20.692043+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"60'1","reported_seq":139,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-06-03T20:06:10.645042+0000","last_change":"2023-06-03T19:57:28.463999+0000","last_active":"2023-06-03T20:06:10.645042+0000","last_peered":"2023-06-03T20:06:10.645042+0000","last_clean":"2023-06-03T20:06:10.645042+0000","last_became_active":"2023-06-03T19:57:28.463588+0000","last_became_peered":"2023-06-03T19:57:28.463588+0000","last_unstale":"2023-06-03T20:06:10.645042+0000","last_undegraded":"2023-06-03T20:06:10.645042+0000","last_fullsized":"2023-06-03T20:06:10.645042+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T21:44:11.969059+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":107,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-06-03T20:06:09.860974+0000","last_change":"2023-06-03T19:57:28.456467+0000","last_active":"2023-06-03T20:06:09.860974+0000","last_peered":"2023-06-03T20:06:09.860974+0000","last_clean":"2023-06-03T20:06:09.860974+0000","last_became_active":"2023-06-03T19:57:28.456120+0000","last_became_peered":"2023-06-03T19:57:28.456120+0000","last_unstale":"2023-06-03T20:06:09.860974+0000","last_undegraded":"2023-06-03T20:06:09.860974+0000","last_fullsized":"2023-06-03T20:06:09.860974+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T07:28:14.306624+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"60'1","reported_seq":123,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-06-03T20:06:10.645170+0000","last_change":"2023-06-03T19:57:28.465124+0000","last_active":"2023-06-03T20:06:10.645170+0000","last_peered":"2023-06-03T20:06:10.645170+0000","last_clean":"2023-06-03T20:06:10.645170+0000","last_became_active":"2023-06-03T19:57:28.464654+0000","last_became_peered":"2023-06-03T19:57:28.464654+0000","last_unstale":"2023-06-03T20:06:10.645170+0000","last_undegraded":"2023-06-03T20:06:10.645170+0000","last_fullsized":"2023-06-03T20:06:10.645170+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T01:12:32.305507+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":150,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-06-03T20:06:10.083439+0000","last_change":"2023-06-03T19:56:54.511760+0000","last_active":"2023-06-03T20:06:10.083439+0000","last_peered":"2023-06-03T20:06:10.083439+0000","last_clean":"2023-06-03T20:06:10.083439+0000","last_became_active":"2023-06-03T19:56:54.511429+0000","last_became_peered":"2023-06-03T19:56:54.511429+0000","last_unstale":"2023-06-03T20:06:10.083439+0000","last_undegraded":"2023-06-03T20:06:10.083439+0000","last_fullsized":"2023-06-03T20:06:10.083439+0000","mapping_epoch":119,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T05:42:46.254343+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"60'2","reported_seq":147,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-06-03T20:06:10.683839+0000","last_change":"2023-06-03T19:57:28.462252+0000","last_active":"2023-06-03T20:06:10.683839+0000","last_peered":"2023-06-03T20:06:10.683839+0000","last_clean":"2023-06-03T20:06:10.683839+0000","last_became_active":"2023-06-03T19:57:28.461844+0000","last_became_peered":"2023-06-03T19:57:28.461844+0000","last_unstale":"2023-06-03T20:06:10.683839+0000","last_undegraded":"2023-06-03T20:06:10.683839+0000","last_fullsized":"2023-06-03T20:06:10.683839+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":153,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-06-03T20:06:09.861387+0000","last_change":"2023-06-03T19:57:28.469248+0000","last_active":"2023-06-03T20:06:09.861387+0000","last_peered":"2023-06-03T20:06:09.861387+0000","last_clean":"2023-06-03T20:06:09.861387+0000","last_became_active":"2023-06-03T19:57:28.468903+0000","last_became_peered":"2023-06-03T19:57:28.468903+0000","last_unstale":"2023-06-03T20:06:09.861387+0000","last_undegraded":"2023-06-03T20:06:09.861387+0000","last_fullsized":"2023-06-03T20:06:09.861387+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T06:08:20.956098+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"60'2","reported_seq":132,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-06-03T20:06:10.683759+0000","last_change":"2023-06-03T19:57:28.462610+0000","last_active":"2023-06-03T20:06:10.683759+0000","last_peered":"2023-06-03T20:06:10.683759+0000","last_clean":"2023-06-03T20:06:10.683759+0000","last_became_active":"2023-06-03T19:57:28.460655+0000","last_became_peered":"2023-06-03T19:57:28.460655+0000","last_unstale":"2023-06-03T20:06:10.683759+0000","last_undegraded":"2023-06-03T20:06:10.683759+0000","last_fullsized":"2023-06-03T20:06:10.683759+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":153,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-06-03T20:06:09.861323+0000","last_change":"2023-06-03T19:57:28.464690+0000","last_active":"2023-06-03T20:06:09.861323+0000","last_peered":"2023-06-03T20:06:09.861323+0000","last_clean":"2023-06-03T20:06:09.861323+0000","last_became_active":"2023-06-03T19:57:28.463856+0000","last_became_peered":"2023-06-03T19:57:28.463856+0000","last_unstale":"2023-06-03T20:06:09.861323+0000","last_undegraded":"2023-06-03T20:06:09.861323+0000","last_fullsized":"2023-06-03T20:06:09.861323+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T00:11:00.991088+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"60'1","reported_seq":142,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-06-03T20:06:10.308315+0000","last_change":"2023-06-03T19:56:54.505096+0000","last_active":"2023-06-03T20:06:10.308315+0000","last_peered":"2023-06-03T20:06:10.308315+0000","last_clean":"2023-06-03T20:06:10.308315+0000","last_became_active":"2023-06-03T19:56:54.504255+0000","last_became_peered":"2023-06-03T19:56:54.504255+0000","last_unstale":"2023-06-03T20:06:10.308315+0000","last_undegraded":"2023-06-03T20:06:10.308315+0000","last_fullsized":"2023-06-03T20:06:10.308315+0000","mapping_epoch":119,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T19:52:25.922308+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":154,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-06-03T20:06:10.307673+0000","last_change":"2023-06-03T19:56:54.503416+0000","last_active":"2023-06-03T20:06:10.307673+0000","last_peered":"2023-06-03T20:06:10.307673+0000","last_clean":"2023-06-03T20:06:10.307673+0000","last_became_active":"2023-06-03T19:56:54.503018+0000","last_became_peered":"2023-06-03T19:56:54.503018+0000","last_unstale":"2023-06-03T20:06:10.307673+0000","last_undegraded":"2023-06-03T20:06:10.307673+0000","last_fullsized":"2023-06-03T20:06:10.307673+0000","mapping_epoch":119,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T20:14:00.330616+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"60'4","reported_seq":125,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-06-03T20:06:10.645245+0000","last_change":"2023-06-03T19:56:11.748116+0000","last_active":"2023-06-03T20:06:10.645245+0000","last_peered":"2023-06-03T20:06:10.645245+0000","last_clean":"2023-06-03T20:06:10.645245+0000","last_became_active":"2023-06-03T19:56:11.747636+0000","last_became_peered":"2023-06-03T19:56:11.747636+0000","last_unstale":"2023-06-03T20:06:10.645245+0000","last_undegraded":"2023-06-03T20:06:10.645245+0000","last_fullsized":"2023-06-03T20:06:10.645245+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":115,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T06:57:48.052055+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":124,"reported_epoch":162,"state":"active+clean","last_fresh":"2023-06-03T20:06:09.860896+0000","last_change":"2023-06-03T19:57:28.459234+0000","last_active":"2023-06-03T20:06:09.860896+0000","last_peered":"2023-06-03T20:06:09.860896+0000","last_clean":"2023-06-03T20:06:09.860896+0000","last_became_active":"2023-06-03T19:57:28.458896+0000","last_became_peered":"2023-06-03T19:57:28.458896+0000","last_unstale":"2023-06-03T20:06:09.860896+0000","last_undegraded":"2023-06-03T20:06:09.860896+0000","last_fullsized":"2023-06-03T20:06:09.860896+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T21:18:37.142280+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":10,"num_pg":16,"stat_sum":{"num_bytes":1392574464,"num_objects":21249,"num_object_clones":0,"num_object_copies":63747,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":21249,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":21249,"num_write_kb":1359936,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":4221370368,"data_stored":4221370368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":21249,"ondisk_log_size":21249,"up":48,"acting":48,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5486,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":62,"num_write_kb":30,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16458,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":30319,"internal_metadata":0},"log_size":56,"ondisk_log_size":56,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":3,"num_store_stats":3},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":48,"seq":206158430632,"num_pgs":34,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":779372,"kb_used_data":329572,"kb_used_omap":0,"kb_used_meta":449792,"kb_avail":92965780,"statfs":{"total":95995035648,"available":95196958720,"internally_reserved":0,"allocated":337481728,"data_stored":336991109,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":460587008},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 3 20:05:43 2023","interfaces":[{"interface":"back","average":{"1min":0.98699999999999999,"5min":1.218,"15min":1.21},"min":{"1min":0.65100000000000002,"5min":0.25900000000000001,"15min":0.253},"max":{"1min":1.3720000000000001,"5min":7.7839999999999998,"15min":7.7839999999999998},"last":2.2240000000000002},{"interface":"front","average":{"1min":0.96799999999999997,"5min":1.649,"15min":1.3859999999999999},"min":{"1min":0.755,"5min":0.309,"15min":0.29799999999999999},"max":{"1min":1.288,"5min":11.877000000000001,"15min":11.877000000000001},"last":0.502}]},{"osd":1,"last update":"Sat Jun 3 20:05:26 2023","interfaces":[{"interface":"back","average":{"1min":1.782,"5min":2.6150000000000002,"15min":1.881},"min":{"1min":0.498,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":9.4629999999999992,"5min":91.811000000000007,"15min":91.811000000000007},"last":0.59399999999999997},{"interface":"front","average":{"1min":1.851,"5min":3.0179999999999998,"15min":1.9510000000000001},"min":{"1min":0.49099999999999999,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":9.2080000000000002,"5min":91.695999999999998,"15min":91.695999999999998},"last":0.56299999999999994}]},{"osd":2,"last update":"Sat Jun 3 20:05:26 2023","interfaces":[{"interface":"back","average":{"1min":0.96699999999999997,"5min":1.1180000000000001,"15min":1.052},"min":{"1min":0.63900000000000001,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":1.361,"5min":7.71,"15min":7.71},"last":0.47699999999999998},{"interface":"front","average":{"1min":1.411,"5min":1.456,"15min":1.1499999999999999},"min":{"1min":0.44700000000000001,"5min":0.38500000000000001,"15min":0.38500000000000001},"max":{"1min":9.2579999999999991,"5min":9.2579999999999991,"15min":9.2579999999999991},"last":0.61299999999999999}]},{"osd":3,"last update":"Sat Jun 3 20:06:16 2023","interfaces":[{"interface":"back","average":{"1min":1.03,"5min":1.446,"15min":1.6399999999999999},"min":{"1min":0.67900000000000005,"5min":0.34699999999999998,"15min":0.34699999999999998},"max":{"1min":1.425,"5min":11.904,"15min":95.962000000000003},"last":0.72599999999999998},{"interface":"front","average":{"1min":1.0600000000000001,"5min":1.923,"15min":1.8029999999999999},"min":{"1min":0.67100000000000004,"5min":0.36399999999999999,"15min":0.36399999999999999},"max":{"1min":1.681,"5min":12.084,"15min":94.174999999999997},"last":0.65500000000000003}]},{"osd":4,"last update":"Sat Jun 3 20:05:43 2023","interfaces":[{"interface":"back","average":{"1min":1.113,"5min":1.054,"15min":1.0469999999999999},"min":{"1min":0.80400000000000005,"5min":0.35999999999999999,"15min":0.35999999999999999},"max":{"1min":1.3340000000000001,"5min":7.7350000000000003,"15min":7.7350000000000003},"last":0.54600000000000004},{"interface":"front","average":{"1min":1.1359999999999999,"5min":1.728,"15min":1.319},"min":{"1min":0.78600000000000003,"5min":0.38800000000000001,"15min":0.38800000000000001},"max":{"1min":1.4450000000000001,"5min":12.917,"15min":12.917},"last":0.53200000000000003}]},{"osd":5,"last update":"Sat Jun 3 20:06:22 2023","interfaces":[{"interface":"back","average":{"1min":1.04,"5min":1.2150000000000001,"15min":1.204},"min":{"1min":0.434,"5min":0.434,"15min":0.32900000000000001},"max":{"1min":1.458,"5min":7.6829999999999998,"15min":12.778},"last":0.434},{"interface":"front","average":{"1min":1.1060000000000001,"5min":1.9990000000000001,"15min":1.575},"min":{"1min":0.45400000000000001,"5min":0.33100000000000002,"15min":0.30599999999999999},"max":{"1min":1.8220000000000001,"5min":20.306999999999999,"15min":20.306999999999999},"last":0.45400000000000001}]},{"osd":6,"last update":"Sat Jun 3 20:06:22 2023","interfaces":[{"interface":"back","average":{"1min":1.258,"5min":1.5860000000000001,"15min":1.3560000000000001},"min":{"1min":0.73199999999999998,"5min":0.42199999999999999,"15min":0.36499999999999999},"max":{"1min":2.0489999999999999,"5min":9.1180000000000003,"15min":9.1180000000000003},"last":2.0489999999999999},{"interface":"front","average":{"1min":1.1950000000000001,"5min":1.8100000000000001,"15min":1.377},"min":{"1min":0.38100000000000001,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":5.6260000000000003,"5min":9.532,"15min":9.532},"last":0.84799999999999998}]}]},{"osd":6,"up_from":42,"seq":180388626865,"num_pgs":34,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":846108,"kb_used_data":530260,"kb_used_omap":0,"kb_used_meta":315840,"kb_avail":92899044,"statfs":{"total":95995035648,"available":95128621056,"internally_reserved":0,"allocated":542986240,"data_stored":542511544,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":323420160},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,1],"upper_bound":8},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 3 20:05:55 2023","interfaces":[{"interface":"back","average":{"1min":0.97099999999999997,"5min":1.748,"15min":1.556},"min":{"1min":0.61399999999999999,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":1.385,"5min":54.786000000000001,"15min":54.786000000000001},"last":1.0369999999999999},{"interface":"front","average":{"1min":0.94099999999999995,"5min":1.794,"15min":1.5980000000000001},"min":{"1min":0.74099999999999999,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":1.1379999999999999,"5min":54.936,"15min":54.936},"last":6.1479999999999997}]},{"osd":1,"last update":"Sat Jun 3 20:05:27 2023","interfaces":[{"interface":"back","average":{"1min":1.389,"5min":4.2530000000000001,"15min":2.347},"min":{"1min":0.63300000000000001,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":6.0540000000000003,"5min":160.38499999999999,"15min":160.38499999999999},"last":0.97699999999999998},{"interface":"front","average":{"1min":0.97999999999999998,"5min":3.5449999999999999,"15min":2.036},"min":{"1min":0.66100000000000003,"5min":0.29599999999999999,"15min":0.29599999999999999},"max":{"1min":1.5760000000000001,"5min":160.434,"15min":160.434},"last":6.101}]},{"osd":2,"last update":"Sat Jun 3 20:05:27 2023","interfaces":[{"interface":"back","average":{"1min":0.80200000000000005,"5min":0.84599999999999997,"15min":0.85699999999999998},"min":{"1min":0.35599999999999998,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":1.165,"5min":3.4260000000000002,"15min":3.4260000000000002},"last":0.35299999999999998},{"interface":"front","average":{"1min":1.2949999999999999,"5min":1.004,"15min":0.96599999999999997},"min":{"1min":0.38800000000000001,"5min":0.34799999999999998,"15min":0.34799999999999998},"max":{"1min":6.1989999999999998,"5min":6.1989999999999998,"15min":6.1989999999999998},"last":0.91000000000000003}]},{"osd":3,"last update":"Sat Jun 3 20:06:11 2023","interfaces":[{"interface":"back","average":{"1min":0.876,"5min":2.746,"15min":2.0819999999999999},"min":{"1min":0.60899999999999999,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":1.1830000000000001,"5min":71.700000000000003,"15min":164.99199999999999},"last":0.40600000000000003},{"interface":"front","average":{"1min":0.99399999999999999,"5min":2.6709999999999998,"15min":2.1059999999999999},"min":{"1min":0.71399999999999997,"5min":0.316,"15min":0.316},"max":{"1min":1.3100000000000001,"5min":71.890000000000001,"15min":163.459},"last":1.0029999999999999}]},{"osd":4,"last update":"Sat Jun 3 20:05:46 2023","interfaces":[{"interface":"back","average":{"1min":1.0109999999999999,"5min":0.86699999999999999,"15min":0.95799999999999996},"min":{"1min":0.67800000000000005,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":1.3400000000000001,"5min":3.6040000000000001,"15min":3.6040000000000001},"last":0.53200000000000003},{"interface":"front","average":{"1min":1.018,"5min":0.89000000000000001,"15min":1.071},"min":{"1min":0.66900000000000004,"5min":0.377,"15min":0.377},"max":{"1min":1.3799999999999999,"5min":6.1150000000000002,"15min":6.1150000000000002},"last":0.95199999999999996}]},{"osd":5,"last update":"Sat Jun 3 20:05:41 2023","interfaces":[{"interface":"back","average":{"1min":1.1060000000000001,"5min":1.034,"15min":1.196},"min":{"1min":0.61499999999999999,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":1.3979999999999999,"5min":6.1580000000000004,"15min":11.083},"last":2.0139999999999998},{"interface":"front","average":{"1min":0.96499999999999997,"5min":1.4930000000000001,"15min":1.365},"min":{"1min":0.63700000000000001,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":1.3999999999999999,"5min":8.4160000000000004,"15min":10.154},"last":6.641}]},{"osd":7,"last update":"Sat Jun 3 20:06:20 2023","interfaces":[{"interface":"back","average":{"1min":1.2949999999999999,"5min":1.5,"15min":1.347},"min":{"1min":0.65100000000000002,"5min":0.39400000000000002,"15min":0.313},"max":{"1min":3.6680000000000001,"5min":11.362,"15min":11.362},"last":1.0620000000000001},{"interface":"front","average":{"1min":1.028,"5min":1.4510000000000001,"15min":1.375},"min":{"1min":0.57699999999999996,"5min":0.33900000000000002,"15min":0.33000000000000002},"max":{"1min":1.333,"5min":7.3079999999999998,"15min":12.407999999999999},"last":0.57699999999999996}]}]},{"osd":1,"up_from":103,"seq":442381631663,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":515272,"kb_used_data":238592,"kb_used_omap":4,"kb_used_meta":276667,"kb_avail":93229880,"statfs":{"total":95995035648,"available":95467397120,"internally_reserved":0,"allocated":244318208,"data_stored":243688870,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":283307918},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1],"upper_bound":2},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 3 20:05:32 2023","interfaces":[{"interface":"back","average":{"1min":0.66600000000000004,"5min":2.7130000000000001,"15min":1.841},"min":{"1min":0.19600000000000001,"5min":0.185,"15min":0.185},"max":{"1min":1.8149999999999999,"5min":119.902,"15min":119.902},"last":0.28899999999999998},{"interface":"front","average":{"1min":0.66400000000000003,"5min":2.4790000000000001,"15min":1.8759999999999999},"min":{"1min":0.42899999999999999,"5min":0.22500000000000001,"15min":0.192},"max":{"1min":1.423,"5min":119.863,"15min":119.863},"last":0.73699999999999999}]},{"osd":2,"last update":"Sat Jun 3 20:05:35 2023","interfaces":[{"interface":"back","average":{"1min":0.66000000000000003,"5min":2.0990000000000002,"15min":1.1659999999999999},"min":{"1min":0.318,"5min":0.223,"15min":0.223},"max":{"1min":1.145,"5min":119.946,"15min":119.946},"last":0.83599999999999997},{"interface":"front","average":{"1min":0.86199999999999999,"5min":1.236,"15min":0.89700000000000002},"min":{"1min":0.32900000000000001,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":5.0090000000000003,"5min":9.8160000000000007,"15min":9.8160000000000007},"last":4.5149999999999997}]},{"osd":3,"last update":"Sat Jun 3 20:06:09 2023","interfaces":[{"interface":"back","average":{"1min":0.65500000000000003,"5min":1.798,"15min":1.091},"min":{"1min":0.36799999999999999,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":1.0089999999999999,"5min":9.657,"15min":9.657},"last":4.3369999999999997},{"interface":"front","average":{"1min":0.72799999999999998,"5min":1.645,"15min":1.129},"min":{"1min":0.33400000000000002,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":1.21,"5min":9.7509999999999994,"15min":9.7509999999999994},"last":4.7000000000000002}]},{"osd":4,"last update":"Sat Jun 3 20:05:41 2023","interfaces":[{"interface":"back","average":{"1min":0.86699999999999999,"5min":1.7050000000000001,"15min":1.2270000000000001},"min":{"1min":0.46200000000000002,"5min":0.35999999999999999,"15min":0.35999999999999999},"max":{"1min":1.292,"5min":9.9589999999999996,"15min":9.9589999999999996},"last":4.4779999999999998},{"interface":"front","average":{"1min":0.89700000000000002,"5min":2.6459999999999999,"15min":1.518},"min":{"1min":0.52700000000000002,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":1.486,"5min":120.084,"15min":120.084},"last":0.41999999999999998}]},{"osd":5,"last update":"Sat Jun 3 20:05:32 2023","interfaces":[{"interface":"back","average":{"1min":1.198,"5min":3.1699999999999999,"15min":2.262},"min":{"1min":0.375,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":6.6399999999999997,"5min":120.39700000000001,"15min":120.39700000000001},"last":0.372},{"interface":"front","average":{"1min":1.452,"5min":3.0230000000000001,"15min":2.3980000000000001},"min":{"1min":0.46000000000000002,"5min":0.46000000000000002,"15min":0.40000000000000002},"max":{"1min":11.9,"5min":120.34399999999999,"15min":120.34399999999999},"last":0.78100000000000003}]},{"osd":6,"last update":"Sat Jun 3 20:05:32 2023","interfaces":[{"interface":"back","average":{"1min":0.88600000000000001,"5min":2.452,"15min":2.1040000000000001},"min":{"1min":0.50600000000000001,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":1.4870000000000001,"5min":120.386,"15min":120.386},"last":0.88},{"interface":"front","average":{"1min":1.143,"5min":2.6869999999999998,"15min":2.3370000000000002},"min":{"1min":0.55300000000000005,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":5.1790000000000003,"5min":119.96299999999999,"15min":119.96299999999999},"last":4.4489999999999998}]},{"osd":7,"last update":"Sat Jun 3 20:05:32 2023","interfaces":[{"interface":"back","average":{"1min":1.133,"5min":3.1800000000000002,"15min":2.3479999999999999},"min":{"1min":0.47499999999999998,"5min":0.38200000000000001,"15min":0.38200000000000001},"max":{"1min":5.0860000000000003,"5min":119.996,"15min":119.996},"last":4.3929999999999998},{"interface":"front","average":{"1min":1.0249999999999999,"5min":3.1099999999999999,"15min":2.0550000000000002},"min":{"1min":0.44700000000000001,"5min":0.40400000000000003,"15min":0.371},"max":{"1min":3.3650000000000002,"5min":120.17100000000001,"15min":120.17100000000001},"last":0.54700000000000004}]}]},{"osd":0,"up_from":86,"seq":369367187690,"num_pgs":36,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1202080,"kb_used_data":919640,"kb_used_omap":5,"kb_used_meta":282426,"kb_avail":92543072,"statfs":{"total":95995035648,"available":94764105728,"internally_reserved":0,"allocated":941711360,"data_stored":941060209,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":289204858},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,1,2,3,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Jun 3 20:06:19 2023","interfaces":[{"interface":"back","average":{"1min":0.68500000000000005,"5min":2.5760000000000001,"15min":1.8340000000000001},"min":{"1min":0.38600000000000001,"5min":0.192,"15min":0.192},"max":{"1min":2.085,"5min":81.310000000000002,"15min":81.310000000000002},"last":0.29899999999999999},{"interface":"front","average":{"1min":1.0449999999999999,"5min":2.2799999999999998,"15min":1.7490000000000001},"min":{"1min":0.34399999999999997,"5min":0.223,"15min":0.187},"max":{"1min":8.2509999999999994,"5min":76.971999999999994,"15min":76.971999999999994},"last":0.99199999999999999}]},{"osd":2,"last update":"Sat Jun 3 20:06:19 2023","interfaces":[{"interface":"back","average":{"1min":0.60699999999999998,"5min":1.3080000000000001,"15min":0.876},"min":{"1min":0.30299999999999999,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":1.2949999999999999,"5min":8.923,"15min":8.923},"last":0.5},{"interface":"front","average":{"1min":0.95299999999999996,"5min":1.254,"15min":0.92800000000000005},"min":{"1min":0.376,"5min":0.218,"15min":0.218},"max":{"1min":4.6799999999999997,"5min":11.194000000000001,"15min":11.194000000000001},"last":0.32600000000000001}]},{"osd":3,"last update":"Sat Jun 3 20:06:10 2023","interfaces":[{"interface":"back","average":{"1min":0.63400000000000001,"5min":1.5,"15min":1.032},"min":{"1min":0.36599999999999999,"5min":0.216,"15min":0.216},"max":{"1min":1.083,"5min":7.5990000000000002,"15min":7.5990000000000002},"last":0.95799999999999996},{"interface":"front","average":{"1min":0.67800000000000005,"5min":1.522,"15min":1.0329999999999999},"min":{"1min":0.32900000000000001,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":0.94899999999999995,"5min":8.4459999999999997,"15min":8.4459999999999997},"last":0.34399999999999997}]},{"osd":4,"last update":"Sat Jun 3 20:05:49 2023","interfaces":[{"interface":"back","average":{"1min":1.054,"5min":1.264,"15min":1.135},"min":{"1min":0.46300000000000002,"5min":0.33800000000000002,"15min":0.33800000000000002},"max":{"1min":2.238,"5min":6.319,"15min":6.319},"last":0.70099999999999996},{"interface":"front","average":{"1min":1.0329999999999999,"5min":1.3979999999999999,"15min":1.196},"min":{"1min":0.47799999999999998,"5min":0.38800000000000001,"15min":0.38800000000000001},"max":{"1min":2.1640000000000001,"5min":7.3200000000000003,"15min":7.3200000000000003},"last":0.76500000000000001}]},{"osd":5,"last update":"Sat Jun 3 20:05:49 2023","interfaces":[{"interface":"back","average":{"1min":1.0780000000000001,"5min":1.6319999999999999,"15min":1.524},"min":{"1min":0.60599999999999998,"5min":0.34899999999999998,"15min":0.33900000000000002},"max":{"1min":1.9990000000000001,"5min":8.3100000000000005,"15min":9.2059999999999995},"last":1.1180000000000001},{"interface":"front","average":{"1min":1.0329999999999999,"5min":1.6859999999999999,"15min":1.6859999999999999},"min":{"1min":0.42499999999999999,"5min":0.35099999999999998,"15min":0.35099999999999998},"max":{"1min":1.276,"5min":8.2189999999999994,"15min":24.661000000000001},"last":0.66000000000000003}]},{"osd":6,"last update":"Sat Jun 3 20:05:49 2023","interfaces":[{"interface":"back","average":{"1min":1.1459999999999999,"5min":1.421,"15min":1.5620000000000001},"min":{"1min":0.443,"5min":0.38,"15min":0.33500000000000002},"max":{"1min":2.3599999999999999,"5min":6.3899999999999997,"15min":24.599},"last":0.56999999999999995},{"interface":"front","average":{"1min":1.1779999999999999,"5min":1.571,"15min":1.629},"min":{"1min":0.627,"5min":0.42799999999999999,"15min":0.42799999999999999},"max":{"1min":2.1030000000000002,"5min":7.2779999999999996,"15min":12.474},"last":0.72199999999999998}]},{"osd":7,"last update":"Sat Jun 3 20:05:49 2023","interfaces":[{"interface":"back","average":{"1min":1.1220000000000001,"5min":1.837,"15min":1.7190000000000001},"min":{"1min":0.53200000000000003,"5min":0.35099999999999998,"15min":0.35099999999999998},"max":{"1min":1.54,"5min":11.196,"15min":24.634},"last":0.90400000000000003},{"interface":"front","average":{"1min":1.1930000000000001,"5min":2,"15min":1.7010000000000001},"min":{"1min":0.59299999999999997,"5min":0.48199999999999998,"15min":0.40699999999999997},"max":{"1min":2.202,"5min":8.891,"15min":12.621},"last":0.88900000000000001}]}]},{"osd":2,"up_from":114,"seq":489626271869,"num_pgs":25,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":608116,"kb_used_data":547116,"kb_used_omap":7,"kb_used_meta":60984,"kb_avail":93137036,"statfs":{"total":95995035648,"available":95372324864,"internally_reserved":0,"allocated":560246784,"data_stored":559635244,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8111,"internal_metadata":62447697},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1],"upper_bound":2},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 3 20:05:27 2023","interfaces":[{"interface":"back","average":{"1min":0.77200000000000002,"5min":2.3940000000000001,"15min":1.2230000000000001},"min":{"1min":0.23499999999999999,"5min":0.20899999999999999,"15min":0.20899999999999999},"max":{"1min":4.2430000000000003,"5min":126.416,"15min":126.416},"last":0.66400000000000003},{"interface":"front","average":{"1min":1.4850000000000001,"5min":2.633,"15min":1.347},"min":{"1min":0.221,"5min":0.221,"15min":0.221},"max":{"1min":16.093,"5min":126.976,"15min":126.976},"last":0.42399999999999999}]},{"osd":1,"last update":"Sat Jun 3 20:05:27 2023","interfaces":[{"interface":"back","average":{"1min":0.61599999999999999,"5min":1.4179999999999999,"15min":0.94599999999999995},"min":{"1min":0.28699999999999998,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":0.92700000000000005,"5min":28.484999999999999,"15min":28.484999999999999},"last":0.51900000000000002},{"interface":"front","average":{"1min":1.2929999999999999,"5min":1.452,"15min":0.93700000000000006},"min":{"1min":0.26900000000000002,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":5.1269999999999998,"5min":28.588999999999999,"15min":28.588999999999999},"last":3.5859999999999999}]},{"osd":3,"last update":"Sat Jun 3 20:06:10 2023","interfaces":[{"interface":"back","average":{"1min":1.798,"5min":1.431,"15min":0.96999999999999997},"min":{"1min":0.44,"5min":0.23699999999999999,"15min":0.22700000000000001},"max":{"1min":24.739999999999998,"5min":29.251999999999999,"15min":29.251999999999999},"last":2.6800000000000002},{"interface":"front","average":{"1min":0.66700000000000004,"5min":2.2170000000000001,"15min":1.286},"min":{"1min":0.40600000000000003,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":0.86799999999999999,"5min":54.549999999999997,"15min":54.549999999999997},"last":0.86699999999999999}]},{"osd":4,"last update":"Sat Jun 3 20:05:47 2023","interfaces":[{"interface":"back","average":{"1min":0.97599999999999998,"5min":0.85599999999999998,"15min":0.90100000000000002},"min":{"1min":0.59399999999999997,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":1.5840000000000001,"5min":7.9089999999999998,"15min":7.9089999999999998},"last":0.55000000000000004},{"interface":"front","average":{"1min":0.88700000000000001,"5min":0.82499999999999996,"15min":0.92700000000000005},"min":{"1min":0.621,"5min":0.32600000000000001,"15min":0.32600000000000001},"max":{"1min":1.276,"5min":3.7749999999999999,"15min":3.7749999999999999},"last":2.3420000000000001}]},{"osd":5,"last update":"Sat Jun 3 20:05:27 2023","interfaces":[{"interface":"back","average":{"1min":0.97799999999999998,"5min":0.97999999999999998,"15min":1.012},"min":{"1min":0.47999999999999998,"5min":0.36099999999999999,"15min":0.36099999999999999},"max":{"1min":2.7069999999999999,"5min":3.5339999999999998,"15min":3.5339999999999998},"last":2.5720000000000001},{"interface":"front","average":{"1min":2.0569999999999999,"5min":1.5780000000000001,"15min":1.2210000000000001},"min":{"1min":0.376,"5min":0.308,"15min":0.308},"max":{"1min":24.164000000000001,"5min":24.164000000000001,"15min":24.164000000000001},"last":2.1379999999999999}]},{"osd":6,"last update":"Sat Jun 3 20:05:27 2023","interfaces":[{"interface":"back","average":{"1min":1.0720000000000001,"5min":1.026,"15min":1.032},"min":{"1min":0.53800000000000003,"5min":0.38500000000000001,"15min":0.38500000000000001},"max":{"1min":3.1309999999999998,"5min":6.7759999999999998,"15min":6.7759999999999998},"last":0.63},{"interface":"front","average":{"1min":1.0029999999999999,"5min":1.018,"15min":1.036},"min":{"1min":0.34899999999999998,"5min":0.34899999999999998,"15min":0.34899999999999998},"max":{"1min":2.843,"5min":7.9550000000000001,"15min":7.9550000000000001},"last":2.1760000000000002}]},{"osd":7,"last update":"Sat Jun 3 20:05:27 2023","interfaces":[{"interface":"back","average":{"1min":1.1799999999999999,"5min":1.782,"15min":1.369},"min":{"1min":0.57199999999999995,"5min":0.44700000000000001,"15min":0.44700000000000001},"max":{"1min":4.2380000000000004,"5min":7.3920000000000003,"15min":7.3920000000000003},"last":2.6299999999999999},{"interface":"front","average":{"1min":1.1919999999999999,"5min":1.7250000000000001,"15min":1.341},"min":{"1min":0.55000000000000004,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":4.3230000000000004,"5min":7.9320000000000004,"15min":7.9320000000000004},"last":1.3480000000000001}]}]},{"osd":3,"up_from":119,"seq":511101108340,"num_pgs":36,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":518064,"kb_used_data":389288,"kb_used_omap":2,"kb_used_meta":128765,"kb_avail":93227088,"statfs":{"total":95995035648,"available":95464538112,"internally_reserved":0,"allocated":398630912,"data_stored":398020825,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":131855413},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 3 20:06:15 2023","interfaces":[{"interface":"back","average":{"1min":0.67700000000000005,"5min":2.0430000000000001,"15min":1.1060000000000001},"min":{"1min":0.496,"5min":0.19,"15min":0.19},"max":{"1min":1.0980000000000001,"5min":68.647999999999996,"15min":68.647999999999996},"last":0.37},{"interface":"front","average":{"1min":0.64700000000000002,"5min":1.9350000000000001,"15min":1.1279999999999999},"min":{"1min":0.45700000000000002,"5min":0.214,"15min":0.214},"max":{"1min":1.5589999999999999,"5min":69.691000000000003,"15min":69.691000000000003},"last":1.7050000000000001}]},{"osd":1,"last update":"Sat Jun 3 20:06:15 2023","interfaces":[{"interface":"back","average":{"1min":0.67500000000000004,"5min":1.3899999999999999,"15min":0.95899999999999996},"min":{"1min":0.41999999999999998,"5min":0.218,"15min":0.218},"max":{"1min":1.3839999999999999,"5min":10.065,"15min":10.065},"last":0.27300000000000002},{"interface":"front","average":{"1min":0.71899999999999997,"5min":1.4379999999999999,"15min":0.97999999999999998},"min":{"1min":0.433,"5min":0.26100000000000001,"15min":0.23499999999999999},"max":{"1min":1.1990000000000001,"5min":10.792999999999999,"15min":10.792999999999999},"last":0.24199999999999999}]},{"osd":2,"last update":"Sat Jun 3 20:06:15 2023","interfaces":[{"interface":"back","average":{"1min":0.69199999999999995,"5min":1.2110000000000001,"15min":0.85199999999999998},"min":{"1min":0.46600000000000003,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":1.413,"5min":8.3640000000000008,"15min":8.3640000000000008},"last":1.3109999999999999},{"interface":"front","average":{"1min":0.65600000000000003,"5min":1.196,"15min":0.92400000000000004},"min":{"1min":0.38200000000000001,"5min":0.24199999999999999,"15min":0.24199999999999999},"max":{"1min":0.93799999999999994,"5min":7.7290000000000001,"15min":7.7290000000000001},"last":1.2130000000000001}]},{"osd":4,"last update":"Sat Jun 3 20:05:43 2023","interfaces":[{"interface":"back","average":{"1min":0.96699999999999997,"5min":1.258,"15min":1.0860000000000001},"min":{"1min":0.60399999999999998,"5min":0.38300000000000001,"15min":0.38300000000000001},"max":{"1min":1.2669999999999999,"5min":4.774,"15min":4.774},"last":0.76200000000000001},{"interface":"front","average":{"1min":0.98599999999999999,"5min":1.218,"15min":1.089},"min":{"1min":0.68000000000000005,"5min":0.33000000000000002,"15min":0.33000000000000002},"max":{"1min":1.3129999999999999,"5min":7.0720000000000001,"15min":7.0720000000000001},"last":0.79900000000000004}]},{"osd":5,"last update":"Sat Jun 3 20:06:15 2023","interfaces":[{"interface":"back","average":{"1min":0.995,"5min":1.444,"15min":1.149},"min":{"1min":0.71299999999999997,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":1.321,"5min":7.1230000000000002,"15min":7.1230000000000002},"last":0.92000000000000004},{"interface":"front","average":{"1min":0.99199999999999999,"5min":1.946,"15min":1.3500000000000001},"min":{"1min":0.69499999999999995,"5min":0.35599999999999998,"15min":0.35599999999999998},"max":{"1min":1.331,"5min":8.3949999999999996,"15min":8.3949999999999996},"last":0.50700000000000001}]},{"osd":6,"last update":"Sat Jun 3 20:06:15 2023","interfaces":[{"interface":"back","average":{"1min":1.024,"5min":1.611,"15min":1.2350000000000001},"min":{"1min":0.71599999999999997,"5min":0.38600000000000001,"15min":0.38600000000000001},"max":{"1min":1.347,"5min":8.4169999999999998,"15min":8.4169999999999998},"last":3.4980000000000002},{"interface":"front","average":{"1min":1.0780000000000001,"5min":1.2490000000000001,"15min":1.153},"min":{"1min":0.70299999999999996,"5min":0.38,"15min":0.38},"max":{"1min":1.4470000000000001,"5min":4.9480000000000004,"15min":4.9480000000000004},"last":3.5150000000000001}]},{"osd":7,"last update":"Sat Jun 3 20:06:15 2023","interfaces":[{"interface":"back","average":{"1min":0.95199999999999996,"5min":1.4219999999999999,"15min":1.204},"min":{"1min":0.66900000000000004,"5min":0.48299999999999998,"15min":0.48299999999999998},"max":{"1min":1.2370000000000001,"5min":8.6910000000000007,"15min":8.6910000000000007},"last":0.39100000000000001},{"interface":"front","average":{"1min":0.98799999999999999,"5min":2.0129999999999999,"15min":1.393},"min":{"1min":0.67700000000000005,"5min":0.435,"15min":0.435},"max":{"1min":1.23,"5min":12.715999999999999,"15min":12.715999999999999},"last":0.46300000000000002}]}]},{"osd":4,"up_from":124,"seq":532575944813,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":593744,"kb_used_data":539464,"kb_used_omap":10,"kb_used_meta":54261,"kb_avail":93151408,"statfs":{"total":95995035648,"available":95387041792,"internally_reserved":0,"allocated":552411136,"data_stored":551773290,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":10652,"internal_metadata":55563876},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,1],"upper_bound":8},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 3 20:05:49 2023","interfaces":[{"interface":"back","average":{"1min":0.79900000000000004,"5min":1.603,"15min":1.179},"min":{"1min":0.36499999999999999,"5min":0.254,"15min":0.254},"max":{"1min":1.1519999999999999,"5min":57.786000000000001,"15min":57.786000000000001},"last":2.2040000000000002},{"interface":"front","average":{"1min":0.89500000000000002,"5min":1.579,"15min":1.214},"min":{"1min":0.66100000000000003,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":1.48,"5min":57.820999999999998,"15min":57.820999999999998},"last":1.603}]},{"osd":1,"last update":"Sat Jun 3 20:05:49 2023","interfaces":[{"interface":"back","average":{"1min":1.1459999999999999,"5min":2.1240000000000001,"15min":1.2929999999999999},"min":{"1min":0.56299999999999994,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":4.7969999999999997,"5min":58.32,"15min":58.32},"last":2.4870000000000001},{"interface":"front","average":{"1min":0.86799999999999999,"5min":1.9930000000000001,"15min":1.2490000000000001},"min":{"1min":0.45200000000000001,"5min":0.307,"15min":0.307},"max":{"1min":1.5269999999999999,"5min":58.286000000000001,"15min":58.286000000000001},"last":1.8100000000000001}]},{"osd":2,"last update":"Sat Jun 3 20:05:49 2023","interfaces":[{"interface":"back","average":{"1min":0.871,"5min":0.92100000000000004,"15min":0.89600000000000002},"min":{"1min":0.42099999999999999,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":1.3009999999999999,"5min":7.3109999999999999,"15min":7.3109999999999999},"last":1.8759999999999999},{"interface":"front","average":{"1min":0.93700000000000006,"5min":0.874,"15min":0.91100000000000003},"min":{"1min":0.54300000000000004,"5min":0.34499999999999997,"15min":0.34499999999999997},"max":{"1min":1.421,"5min":5.069,"15min":5.069},"last":1.218}]},{"osd":3,"last update":"Sat Jun 3 20:05:49 2023","interfaces":[{"interface":"back","average":{"1min":0.97199999999999998,"5min":2.1200000000000001,"15min":1.3520000000000001},"min":{"1min":0.625,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":1.357,"5min":57.954999999999998,"15min":57.954999999999998},"last":2.5539999999999998},{"interface":"front","average":{"1min":0.98499999999999999,"5min":2.161,"15min":1.4159999999999999},"min":{"1min":0.51500000000000001,"5min":0.33700000000000002,"15min":0.33700000000000002},"max":{"1min":1.4450000000000001,"5min":57.889000000000003,"15min":57.889000000000003},"last":1.843}]},{"osd":5,"last update":"Sat Jun 3 20:05:49 2023","interfaces":[{"interface":"back","average":{"1min":0.90000000000000002,"5min":1.8029999999999999,"15min":1.246},"min":{"1min":0.52900000000000003,"5min":0.313,"15min":0.313},"max":{"1min":1.3879999999999999,"5min":86.576999999999998,"15min":86.576999999999998},"last":1.177},{"interface":"front","average":{"1min":0.91000000000000003,"5min":2.0299999999999998,"15min":1.343},"min":{"1min":0.55200000000000005,"5min":0.32600000000000001,"15min":0.32600000000000001},"max":{"1min":1.2509999999999999,"5min":88.013000000000005,"15min":88.013000000000005},"last":2.4369999999999998}]},{"osd":6,"last update":"Sat Jun 3 20:05:49 2023","interfaces":[{"interface":"back","average":{"1min":0.90600000000000003,"5min":0.82599999999999996,"15min":0.96499999999999997},"min":{"1min":0.58699999999999997,"5min":0.35399999999999998,"15min":0.35399999999999998},"max":{"1min":1.3480000000000001,"5min":5.1420000000000003,"15min":5.1420000000000003},"last":1.359},{"interface":"front","average":{"1min":0.93799999999999994,"5min":0.93600000000000005,"15min":1.032},"min":{"1min":0.61899999999999999,"5min":0.36499999999999999,"15min":0.36499999999999999},"max":{"1min":1.349,"5min":7.2220000000000004,"15min":7.2220000000000004},"last":5.2080000000000002}]},{"osd":7,"last update":"Sat Jun 3 20:05:49 2023","interfaces":[{"interface":"back","average":{"1min":1.02,"5min":1.4550000000000001,"15min":1.2450000000000001},"min":{"1min":0.56899999999999995,"5min":0.39700000000000002,"15min":0.39700000000000002},"max":{"1min":1.423,"5min":10.048999999999999,"15min":10.048999999999999},"last":2.5219999999999998},{"interface":"front","average":{"1min":0.95099999999999996,"5min":1.5309999999999999,"15min":1.2849999999999999},"min":{"1min":0.48399999999999999,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":1.581,"5min":9.3309999999999995,"15min":9.3309999999999995},"last":1.9490000000000001}]}]},{"osd":5,"up_from":36,"seq":154618823099,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":914576,"kb_used_data":397448,"kb_used_omap":0,"kb_used_meta":517120,"kb_avail":92830576,"statfs":{"total":95995035648,"available":95058509824,"internally_reserved":0,"allocated":406986752,"data_stored":406519638,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":529530880},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,1],"upper_bound":4},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 3 20:05:34 2023","interfaces":[{"interface":"back","average":{"1min":0.85199999999999998,"5min":1.409,"15min":1.379},"min":{"1min":0.43099999999999999,"5min":0.22800000000000001,"15min":0.22800000000000001},"max":{"1min":1.502,"5min":37.689999999999998,"15min":37.689999999999998},"last":0.84599999999999997},{"interface":"front","average":{"1min":0.97499999999999998,"5min":2.1150000000000002,"15min":1.798},"min":{"1min":0.57599999999999996,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":3.0539999999999998,"5min":37.783999999999999,"15min":37.783999999999999},"last":0.872}]},{"osd":1,"last update":"Sat Jun 3 20:05:34 2023","interfaces":[{"interface":"back","average":{"1min":0.92200000000000004,"5min":3.98,"15min":2.3959999999999999},"min":{"1min":0.38300000000000001,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":2.7240000000000002,"5min":114.681,"15min":114.681},"last":0.48599999999999999},{"interface":"front","average":{"1min":0.89200000000000002,"5min":4.0170000000000003,"15min":2.456},"min":{"1min":0.432,"5min":0.307,"15min":0.29999999999999999},"max":{"1min":3.2200000000000002,"5min":115.366,"15min":115.366},"last":0.307}]},{"osd":2,"last update":"Sat Jun 3 20:05:34 2023","interfaces":[{"interface":"back","average":{"1min":0.85599999999999998,"5min":1.0049999999999999,"15min":0.92600000000000005},"min":{"1min":0.47499999999999998,"5min":0.35099999999999998,"15min":0.34999999999999998},"max":{"1min":1.3500000000000001,"5min":4.3529999999999998,"15min":4.3529999999999998},"last":1.22},{"interface":"front","average":{"1min":1.0129999999999999,"5min":1.589,"15min":1.1739999999999999},"min":{"1min":0.46500000000000002,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":3.105,"5min":8.2119999999999997,"15min":8.2119999999999997},"last":1.1579999999999999}]},{"osd":3,"last update":"Sat Jun 3 20:06:13 2023","interfaces":[{"interface":"back","average":{"1min":1.0580000000000001,"5min":3.1840000000000002,"15min":2.4710000000000001},"min":{"1min":0.73099999999999998,"5min":0.33700000000000002,"15min":0.33700000000000002},"max":{"1min":1.4810000000000001,"5min":48.308,"15min":261.14100000000002},"last":0.51400000000000001},{"interface":"front","average":{"1min":0.96299999999999997,"5min":2.637,"15min":2.2749999999999999},"min":{"1min":0.69899999999999995,"5min":0.35399999999999998,"15min":0.35399999999999998},"max":{"1min":1.2589999999999999,"5min":48.124000000000002,"15min":261.27100000000002},"last":0.39300000000000002}]},{"osd":4,"last update":"Sat Jun 3 20:05:45 2023","interfaces":[{"interface":"back","average":{"1min":1.0349999999999999,"5min":1.569,"15min":1.2010000000000001},"min":{"1min":0.64800000000000002,"5min":0.378,"15min":0.378},"max":{"1min":1.5029999999999999,"5min":10.162000000000001,"15min":10.162000000000001},"last":0.53700000000000003},{"interface":"front","average":{"1min":1.0409999999999999,"5min":0.96599999999999997,"15min":1.105},"min":{"1min":0.58199999999999996,"5min":0.377,"15min":0.377},"max":{"1min":1.321,"5min":4.9980000000000002,"15min":4.9980000000000002},"last":0.44500000000000001}]},{"osd":6,"last update":"Sat Jun 3 20:05:34 2023","interfaces":[{"interface":"back","average":{"1min":0.96199999999999997,"5min":1.7230000000000001,"15min":1.498},"min":{"1min":0.41699999999999998,"5min":0.40799999999999997,"15min":0.33000000000000002},"max":{"1min":1.4319999999999999,"5min":10.278,"15min":12.731999999999999},"last":0.99399999999999999},{"interface":"front","average":{"1min":1.093,"5min":1.7470000000000001,"15min":1.5800000000000001},"min":{"1min":0.46899999999999997,"5min":0.39700000000000002,"15min":0.32800000000000001},"max":{"1min":1.514,"5min":13.714,"15min":13.714},"last":1.899}]},{"osd":7,"last update":"Sat Jun 3 20:06:17 2023","interfaces":[{"interface":"back","average":{"1min":1.143,"5min":2.3809999999999998,"15min":1.8129999999999999},"min":{"1min":0.46100000000000002,"5min":0.46100000000000002,"15min":0.379},"max":{"1min":1.419,"5min":10.192,"15min":11.297000000000001},"last":1.2390000000000001},{"interface":"front","average":{"1min":1.0589999999999999,"5min":1.2929999999999999,"15min":1.2549999999999999},"min":{"1min":0.505,"5min":0.38400000000000001,"15min":0.32100000000000001},"max":{"1min":1.3999999999999999,"5min":5.8719999999999999,"15min":7.4720000000000004},"last":0.50700000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6928,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4862,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":10652,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4786,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4932,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":1002373120,"data_stored":1002373120,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":262930432,"data_stored":262930432,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":596901888,"data_stored":596901888,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":426377216,"data_stored":426377216,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":571342848,"data_stored":571342848,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":429064192,"data_stored":429064192,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":585498624,"data_stored":585498624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":346882048,"data_stored":346882048,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-06-03T20:06:29.593 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph osd pool set unique_pool_5 pgp_num 16 2023-06-03T20:06:29.680 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:06:29.713 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:06:29.716 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:06:29.716 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:06:29.818 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:06:29.856 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:06:29.859 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:06:29.860 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:06:29.960 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:06:29.970 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:06:29 smithi089 bash[75068]: cluster 1685822788.5560205 mgr.y (mgr.34104) 910 : cluster 0 pgmap v847: 89 pgs: 89 active+clean; 1.7 GiB data, 6.8 GiB used, 708 GiB / 715 GiB avail; 127 MiB/s wr, 2.03k op/s 2023-06-03T20:06:29.971 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:06:29 smithi089 bash[78452]: cluster 1685822788.5560205 mgr.y (mgr.34104) 910 : cluster 0 pgmap v847: 89 pgs: 89 active+clean; 1.7 GiB data, 6.8 GiB used, 708 GiB / 715 GiB avail; 127 MiB/s wr, 2.03k op/s 2023-06-03T20:06:29.995 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/mon.a/config 2023-06-03T20:06:30.009 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:06:30.013 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:06:30.014 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:06:30.083 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:06:29 smithi164 bash[59462]: cluster 1685822788.5560205 mgr.y (mgr.34104) 910 : cluster 0 pgmap v847: 89 pgs: 89 active+clean; 1.7 GiB data, 6.8 GiB used, 708 GiB / 715 GiB avail; 127 MiB/s wr, 2.03k op/s 2023-06-03T20:06:30.115 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:06:30.148 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:06:30.153 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:06:30.154 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:06:30.232 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 16 16 32960 32944 128.668 103.125 0.00897042 0.00776671 2023-06-03T20:06:30.254 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:06:30.290 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:06:30.295 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:06:30.296 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:06:30.397 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:06:30.431 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:06:30.437 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:06:30.438 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:06:30.539 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:06:30.572 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:06:30.577 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:06:30.577 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:06:30.678 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:06:30.720 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:06:30.723 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:06:30.724 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:06:30.824 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:06:30.854 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:06:30.859 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:06:30.860 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:06:30.961 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:06:30.993 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:06:30.996 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:06:30.996 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:06:31.097 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:06:31.132 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:06:31.138 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:06:31.138 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:06:31.232 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 17 16 34658 34642 127.342 106.125 0.00768521 0.00784669 2023-06-03T20:06:31.239 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:06:31.273 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:06:31.277 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:06:31.277 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:06:31.379 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:06:31.407 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:06:31.412 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:06:31.413 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:06:31.514 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:06:31.555 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:06:31.560 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:06:31.560 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:06:31.661 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:06:31.695 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:06:31.699 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:06:31.700 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:06:31.801 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:06:31.833 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:06:31 smithi164 bash[59462]: cluster 1685822790.5569024 mgr.y (mgr.34104) 911 : cluster 0 pgmap v848: 89 pgs: 89 active+clean; 1.8 GiB data, 7.4 GiB used, 708 GiB / 715 GiB avail; 121 MiB/s wr, 1.94k op/s 2023-06-03T20:06:31.834 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:06:31.836 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:06:31.837 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:06:31.938 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:06:31.962 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:06:31.963 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:06:31.964 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:06:32.017 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:06:31 smithi089 bash[75068]: cluster 1685822790.5569024 mgr.y (mgr.34104) 911 : cluster 0 pgmap v848: 89 pgs: 89 active+clean; 1.8 GiB data, 7.4 GiB used, 708 GiB / 715 GiB avail; 121 MiB/s wr, 1.94k op/s 2023-06-03T20:06:32.018 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:06:31 smithi089 bash[78452]: cluster 1685822790.5569024 mgr.y (mgr.34104) 911 : cluster 0 pgmap v848: 89 pgs: 89 active+clean; 1.8 GiB data, 7.4 GiB used, 708 GiB / 715 GiB avail; 121 MiB/s wr, 1.94k op/s 2023-06-03T20:06:32.065 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:06:32.105 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:06:32.109 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:06:32.110 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:06:32.117 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/mon.a/config 2023-06-03T20:06:32.121 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/mon.a/config 2023-06-03T20:06:32.211 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:06:32.234 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 18 16 36161 36145 125.483 93.9375 0.0060705 0.00796296 2023-06-03T20:06:32.250 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:06:32.255 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:06:32.256 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:06:32.357 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:06:32.395 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:06:32.398 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:06:32.398 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:06:32.499 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:06:32.537 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:06:32.541 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:06:32.542 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:06:32.643 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:06:32.679 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:06:32.682 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:06:32.683 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:06:32.784 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:06:32.806 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:06:32.807 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:06:32.808 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:06:32.909 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:06:32.934 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:06:32.937 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:06:32.938 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:06:33.039 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:06:33.073 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:06:33.076 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:06:33.077 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:06:33.178 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:06:33.207 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:06:33.212 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:06:33.213 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:06:33.232 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 19 16 37583 37567 123.557 88.875 0.00684253 0.00808717 2023-06-03T20:06:33.314 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:06:33.338 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:06:33.340 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:06:33.340 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:06:33.441 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:06:33.475 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:06:33.479 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:06:33.479 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:06:33.580 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:06:33.610 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:06:33.612 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:06:33.613 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:06:33.714 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:06:33.745 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:06:33.749 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:06:33.749 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:06:33.850 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:06:33.884 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:06:33.887 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:06:33.888 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:06:33.989 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:06:33.998 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:06:33 smithi164 bash[59462]: cluster 1685822792.5585573 mgr.y (mgr.34104) 912 : cluster 0 pgmap v849: 89 pgs: 89 active+clean; 1.9 GiB data, 7.7 GiB used, 708 GiB / 715 GiB avail; 110 MiB/s wr, 1.75k op/s 2023-06-03T20:06:34.028 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:06:33 smithi089 bash[75068]: cluster 1685822792.5585573 mgr.y (mgr.34104) 912 : cluster 0 pgmap v849: 89 pgs: 89 active+clean; 1.9 GiB data, 7.7 GiB used, 708 GiB / 715 GiB avail; 110 MiB/s wr, 1.75k op/s 2023-06-03T20:06:34.029 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:06:33 smithi089 bash[78452]: cluster 1685822792.5585573 mgr.y (mgr.34104) 912 : cluster 0 pgmap v849: 89 pgs: 89 active+clean; 1.9 GiB data, 7.7 GiB used, 708 GiB / 715 GiB avail; 110 MiB/s wr, 1.75k op/s 2023-06-03T20:06:34.031 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:06:34.034 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:06:34.035 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:06:34.136 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:06:34.169 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:06:34.172 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:06:34.174 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:06:34.185 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/mon.a/config 2023-06-03T20:06:34.235 INFO:tasks.radosbench.radosbench.0.smithi089.stdout:2023-06-03T20:06:34.231664+0000 min lat: 0.00128391 max lat: 0.197653 avg lat: 0.00822257 2023-06-03T20:06:34.235 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-06-03T20:06:34.235 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 20 16 38919 38903 121.537 83.5 0.010877 0.00822257 2023-06-03T20:06:34.276 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:06:34.312 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:06:34.317 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:06:34.318 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:06:34.419 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:06:34.456 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:06:34.460 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:06:34.460 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:06:34.561 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:06:34.595 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:06:34.598 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:06:34.598 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:06:34.699 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:06:34.731 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:06:34.733 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:06:34.734 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:06:34.834 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:06:34.871 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:06:34.874 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:06:34.875 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:06:34.975 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:06:35.005 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:06:35.007 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:06:35.007 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:06:35.108 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:06:35.151 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:06:35.155 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:06:35.156 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:06:35.235 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 21 16 40419 40403 120.213 93.75 0.00702866 0.0083131 2023-06-03T20:06:35.257 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:06:35.292 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:06:35.298 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:06:35.298 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:06:35.399 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:06:35.433 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:06:35.434 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:06:35.435 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:06:35.536 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:06:35.571 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:06:35.574 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:06:35.575 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:06:35.676 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:06:35.707 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:06:35.713 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:06:35.714 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:06:35.815 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:06:35.846 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:06:35.851 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:06:35.852 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:06:35.953 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:06:35.961 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:06:35 smithi164 bash[59462]: cluster 1685822794.5606866 mgr.y (mgr.34104) 913 : cluster 0 pgmap v850: 89 pgs: 89 active+clean; 2.2 GiB data, 8.4 GiB used, 707 GiB / 715 GiB avail; 125 MiB/s wr, 1.99k op/s 2023-06-03T20:06:35.984 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:06:35.986 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:06:35.986 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:06:36.028 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:06:35 smithi089 bash[75068]: cluster 1685822794.5606866 mgr.y (mgr.34104) 913 : cluster 0 pgmap v850: 89 pgs: 89 active+clean; 2.2 GiB data, 8.4 GiB used, 707 GiB / 715 GiB avail; 125 MiB/s wr, 1.99k op/s 2023-06-03T20:06:36.029 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:06:35 smithi089 bash[78452]: cluster 1685822794.5606866 mgr.y (mgr.34104) 913 : cluster 0 pgmap v850: 89 pgs: 89 active+clean; 2.2 GiB data, 8.4 GiB used, 707 GiB / 715 GiB avail; 125 MiB/s wr, 1.99k op/s 2023-06-03T20:06:36.087 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:06:36.121 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:06:36.124 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:06:36.125 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:06:36.225 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:06:36.235 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 22 16 41946 41930 119.087 95.4375 0.00831709 0.00839012 2023-06-03T20:06:36.268 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:06:36.271 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:06:36.271 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:06:36.372 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:06:36.395 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:06:36.397 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:06:36.398 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:06:36.499 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:06:36.527 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:06:36.528 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 20:06:36 smithi089 bash[71163]: ::ffff:172.21.15.164 - - [03/Jun/2023:20:06:36] "GET /metrics HTTP/1.1" 200 228585 "" "Prometheus/2.33.4" 2023-06-03T20:06:36.528 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:06:36.529 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:06:36.630 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:06:36.666 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:06:36.668 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:06:36.669 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:06:36.770 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:06:36.810 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:06:36.816 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:06:36.817 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:06:36.918 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:06:36.955 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:06:36.959 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:06:36.960 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:06:37.061 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:06:37.103 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:06:37.107 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:06:37.108 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:06:37.209 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:06:37.235 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 23 16 43281 43265 117.537 83.4375 0.0260602 0.00850154 2023-06-03T20:06:37.238 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:06:37.240 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:06:37.241 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:06:37.342 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:06:37.376 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:06:37.380 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:06:37.380 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:06:37.481 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:06:37.519 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:06:37.526 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:06:37.526 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:06:37.627 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:06:37.673 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:06:37.679 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:06:37.680 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:06:37.780 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:06:37.808 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:06:37.810 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:06:37.811 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:06:37.912 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:06:37.923 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:06:37 smithi164 bash[59462]: cluster 1685822796.5612378 mgr.y (mgr.34104) 914 : cluster 0 pgmap v851: 89 pgs: 89 active+clean; 2.3 GiB data, 8.9 GiB used, 706 GiB / 715 GiB avail; 100 MiB/s wr, 1.60k op/s 2023-06-03T20:06:37.923 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:06:37 smithi164 bash[59462]: audit 1685822797.108741 mon.c (mon.2) 439 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T20:06:37.951 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:06:37.955 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:06:37.956 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:06:38.028 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:06:37 smithi089 bash[75068]: cluster 1685822796.5612378 mgr.y (mgr.34104) 914 : cluster 0 pgmap v851: 89 pgs: 89 active+clean; 2.3 GiB data, 8.9 GiB used, 706 GiB / 715 GiB avail; 100 MiB/s wr, 1.60k op/s 2023-06-03T20:06:38.029 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:06:37 smithi089 bash[75068]: audit 1685822797.108741 mon.c (mon.2) 439 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T20:06:38.029 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:06:37 smithi089 bash[78452]: cluster 1685822796.5612378 mgr.y (mgr.34104) 914 : cluster 0 pgmap v851: 89 pgs: 89 active+clean; 2.3 GiB data, 8.9 GiB used, 706 GiB / 715 GiB avail; 100 MiB/s wr, 1.60k op/s 2023-06-03T20:06:38.029 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:06:37 smithi089 bash[78452]: audit 1685822797.108741 mon.c (mon.2) 439 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T20:06:38.058 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:06:38.086 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:06:38.089 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:06:38.091 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:06:38.192 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:06:38.225 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:06:38.227 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:06:38.228 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:06:38.235 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 24 16 44665 44649 116.243 86.5 0.0158185 0.00859529 2023-06-03T20:06:38.328 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:06:38.365 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:06:38.367 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:06:38.368 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:06:38.471 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:06:38.499 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:06:38.501 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:06:38.501 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:06:38.602 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:06:38.635 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:06:38.637 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:06:38.638 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:06:38.739 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:06:38.768 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:06:38.770 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:06:38.770 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:06:38.871 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:06:38.906 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:06:38.909 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:06:38.910 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:06:39.011 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:06:39.047 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:06:39.050 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:06:39.050 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:06:39.151 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:06:39.186 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:06:39.189 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:06:39.190 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:06:39.236 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 25 16 46075 46059 115.118 88.125 0.00729641 0.00867997 2023-06-03T20:06:39.292 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:06:39.339 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:06:39.340 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:06:39.340 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:06:39.441 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:06:39.471 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:06:39.475 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:06:39.477 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:06:39.578 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:06:39.618 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:06:39.621 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:06:39.621 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:06:39.722 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:06:39.763 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:06:39.766 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:06:39.767 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:06:39.868 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:06:39.906 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:06:39.907 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:06:39.907 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:06:40.008 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:06:40.017 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:06:39 smithi089 bash[78452]: cluster 1685822798.561965 mgr.y (mgr.34104) 915 : cluster 0 pgmap v852: 89 pgs: 89 active+clean; 2.6 GiB data, 9.7 GiB used, 706 GiB / 715 GiB avail; 113 MiB/s wr, 1.80k op/s 2023-06-03T20:06:40.018 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:06:39 smithi089 bash[75068]: cluster 1685822798.561965 mgr.y (mgr.34104) 915 : cluster 0 pgmap v852: 89 pgs: 89 active+clean; 2.6 GiB data, 9.7 GiB used, 706 GiB / 715 GiB avail; 113 MiB/s wr, 1.80k op/s 2023-06-03T20:06:40.042 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:06:40.045 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:06:40.046 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:06:40.083 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:06:39 smithi164 bash[59462]: cluster 1685822798.561965 mgr.y (mgr.34104) 915 : cluster 0 pgmap v852: 89 pgs: 89 active+clean; 2.6 GiB data, 9.7 GiB used, 706 GiB / 715 GiB avail; 113 MiB/s wr, 1.80k op/s 2023-06-03T20:06:40.147 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:06:40.188 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:06:40.191 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:06:40.192 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:06:40.236 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 26 16 47432 47416 113.953 84.8125 0.00265327 0.00876265 2023-06-03T20:06:40.292 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:06:40.325 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:06:40.329 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:06:40.329 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:06:40.430 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:06:40.467 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:06:40.470 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:06:40.470 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:06:40.571 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:06:40.602 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:06:40.605 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:06:40.605 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:06:40.706 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:06:40.760 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:06:40.765 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:06:40.766 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:06:40.867 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:06:40.901 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:06:40.905 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:06:40.906 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:06:41.006 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:06:41.041 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:06:41.043 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:06:41.044 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:06:41.145 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:06:41.183 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:06:41.186 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:06:41.186 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:06:41.236 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 27 16 48589 48573 112.41 72.3125 0.00723521 0.00889063 2023-06-03T20:06:41.287 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:06:41.419 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:06:41.422 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:06:41.423 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:06:41.524 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:06:41.551 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:06:41.555 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:06:41.556 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:06:41.657 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:06:41.694 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:06:41.698 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:06:41.699 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:06:41.701 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:06:41 smithi089 bash[75068]: cluster 1685822800.562603 mgr.y (mgr.34104) 916 : cluster 0 pgmap v853: 89 pgs: 89 active+clean; 2.8 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 92 MiB/s wr, 1.47k op/s 2023-06-03T20:06:41.799 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:06:41.835 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:06:41.837 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:06:41.837 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:06:41.939 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:06:41.948 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:06:41 smithi089 bash[78452]: cluster 1685822800.562603 mgr.y (mgr.34104) 916 : cluster 0 pgmap v853: 89 pgs: 89 active+clean; 2.8 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 92 MiB/s wr, 1.47k op/s 2023-06-03T20:06:41.979 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:06:41.982 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:06:41.984 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:06:42.083 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:06:41 smithi164 bash[59462]: cluster 1685822800.562603 mgr.y (mgr.34104) 916 : cluster 0 pgmap v853: 89 pgs: 89 active+clean; 2.8 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 92 MiB/s wr, 1.47k op/s 2023-06-03T20:06:42.085 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:06:42.124 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:06:42.125 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:06:42.125 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:06:42.226 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:06:42.238 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 28 16 49841 49825 111.182 78.25 0.017512 0.00898753 2023-06-03T20:06:42.254 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:06:42.256 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:06:42.257 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:06:42.357 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:06:42.387 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:06:42.391 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:06:42.392 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:06:42.493 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:06:42.525 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:06:42.529 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:06:42.530 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:06:42.630 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:06:42.660 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:06:42.664 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:06:42.665 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:06:42.766 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:06:42.801 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:06:42.805 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:06:42.806 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:06:42.907 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:06:42.938 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:06:42.941 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:06:42.942 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:06:43.043 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:06:43.077 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:06:43.080 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:06:43.080 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:06:43.181 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:06:43.211 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:06:43.216 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:06:43.216 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:06:43.238 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 29 16 51252 51236 110.389 88.1875 0.00749154 0.00905342 2023-06-03T20:06:43.318 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:06:43.356 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:06:43.359 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:06:43.360 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:06:43.461 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:06:43.508 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:06:43.511 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:06:43.511 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:06:43.612 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:06:43.651 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:06:43.653 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:06:43.654 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:06:43.754 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:06:43.775 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:06:43.776 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:06:43.777 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:06:43.878 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:06:43.887 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:06:43 smithi164 bash[59462]: cluster 1685822802.5631125 mgr.y (mgr.34104) 917 : cluster 0 pgmap v854: 89 pgs: 89 active+clean; 2.8 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 82 MiB/s wr, 1.31k op/s 2023-06-03T20:06:43.914 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:06:43.920 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:06:43.921 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:06:44.022 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:06:44.028 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:06:43 smithi089 bash[75068]: cluster 1685822802.5631125 mgr.y (mgr.34104) 917 : cluster 0 pgmap v854: 89 pgs: 89 active+clean; 2.8 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 82 MiB/s wr, 1.31k op/s 2023-06-03T20:06:44.029 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:06:43 smithi089 bash[78452]: cluster 1685822802.5631125 mgr.y (mgr.34104) 917 : cluster 0 pgmap v854: 89 pgs: 89 active+clean; 2.8 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 82 MiB/s wr, 1.31k op/s 2023-06-03T20:06:44.061 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:06:44.064 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:06:44.065 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:06:44.166 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:06:44.197 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:06:44.200 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:06:44.201 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:06:44.238 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 30 16 52243 52227 108.774 61.9375 0.00888803 0.00918789 2023-06-03T20:06:44.302 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:06:44.336 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:06:44.339 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:06:44.340 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:06:44.441 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:06:44.473 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:06:44.476 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:06:44.477 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:06:44.578 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:06:44.613 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:06:44.619 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:06:44.619 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:06:44.720 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:06:44.758 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:06:44.765 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:06:44.766 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:06:44.867 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:06:44.905 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:06:44.910 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:06:44.911 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:06:44.980 INFO:teuthology.orchestra.run.smithi089.stdout:osd.0: {} 2023-06-03T20:06:45.012 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:06:45.023 INFO:teuthology.orchestra.run.smithi089.stdout:osd.1: {} 2023-06-03T20:06:45.053 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:06:45.055 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:06:45.056 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:06:45.066 INFO:teuthology.orchestra.run.smithi089.stdout:osd.2: {} 2023-06-03T20:06:45.111 INFO:teuthology.orchestra.run.smithi089.stdout:osd.3: {} 2023-06-03T20:06:45.150 INFO:teuthology.orchestra.run.smithi089.stdout:osd.4: {} 2023-06-03T20:06:45.157 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:06:45.184 INFO:teuthology.orchestra.run.smithi089.stdout:osd.5: {} 2023-06-03T20:06:45.190 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:06:45.192 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:06:45.193 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:06:45.221 INFO:teuthology.orchestra.run.smithi089.stdout:osd.6: {} 2023-06-03T20:06:45.238 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 31 16 53476 53460 107.751 77.0625 0.0109052 0.00927496 2023-06-03T20:06:45.264 INFO:teuthology.orchestra.run.smithi089.stdout:osd.7: {} 2023-06-03T20:06:45.270 INFO:teuthology.orchestra.run.smithi089.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-06-03T20:06:45.270 INFO:teuthology.orchestra.run.smithi089.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-06-03T20:06:45.270 INFO:teuthology.orchestra.run.smithi089.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-06-03T20:06:45.270 INFO:teuthology.orchestra.run.smithi089.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-06-03T20:06:45.270 INFO:teuthology.orchestra.run.smithi089.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-06-03T20:06:45.271 INFO:teuthology.orchestra.run.smithi089.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-06-03T20:06:45.271 INFO:teuthology.orchestra.run.smithi089.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-06-03T20:06:45.271 INFO:teuthology.orchestra.run.smithi089.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-06-03T20:06:45.293 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:06:45.324 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:06:45.330 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:06:45.330 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:06:45.431 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:06:45.459 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:06:45.464 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:06:45.464 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:06:45.565 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:06:45.599 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:06:45.602 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:06:45.602 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:06:45.703 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:06:45.737 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:06:45.743 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:06:45.744 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:06:45.845 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:06:45.853 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:06:45 smithi089 bash[78452]: cluster 1685822804.565201 mgr.y (mgr.34104) 918 : cluster 0 pgmap v855: 89 pgs: 89 active+clean; 3.1 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 98 MiB/s wr, 1.56k op/s 2023-06-03T20:06:45.853 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:06:45 smithi089 bash[78452]: audit 1685822804.923642 mon.c (mon.2) 440 : audit 0 from='client.? 172.21.15.89:0/2283618568' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-03T20:06:45.854 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:06:45 smithi089 bash[75068]: cluster 1685822804.565201 mgr.y (mgr.34104) 918 : cluster 0 pgmap v855: 89 pgs: 89 active+clean; 3.1 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 98 MiB/s wr, 1.56k op/s 2023-06-03T20:06:45.854 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:06:45 smithi089 bash[75068]: audit 1685822804.923642 mon.c (mon.2) 440 : audit 0 from='client.? 172.21.15.89:0/2283618568' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-03T20:06:45.879 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:06:45.884 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:06:45.884 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:06:45.985 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:06:46.022 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:06:46.026 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:06:46.026 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:06:46.083 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:06:45 smithi164 bash[59462]: cluster 1685822804.565201 mgr.y (mgr.34104) 918 : cluster 0 pgmap v855: 89 pgs: 89 active+clean; 3.1 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 98 MiB/s wr, 1.56k op/s 2023-06-03T20:06:46.084 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:06:45 smithi164 bash[59462]: audit 1685822804.923642 mon.c (mon.2) 440 : audit 0 from='client.? 172.21.15.89:0/2283618568' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-03T20:06:46.127 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:06:46.164 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:06:46.168 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:06:46.168 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:06:46.231 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 20:06:46 smithi089 bash[71163]: ::ffff:172.21.15.164 - - [03/Jun/2023:20:06:46] "GET /metrics HTTP/1.1" 200 228663 "" "Prometheus/2.33.4" 2023-06-03T20:06:46.238 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 32 16 54798 54782 106.966 82.625 0.013503 0.00934265 2023-06-03T20:06:46.270 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:06:46.303 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:06:46.306 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:06:46.307 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:06:46.408 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:06:46.435 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:06:46.437 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:06:46.438 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:06:46.539 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:06:46.573 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:06:46.578 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:06:46.578 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:06:46.679 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:06:46.715 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:06:46.717 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:06:46.718 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:06:46.819 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:06:46.854 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:06:46.857 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:06:46.858 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:06:46.959 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:06:46.989 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:06:46.993 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:06:46.994 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:06:47.095 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:06:47.137 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:06:47.140 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:06:47.141 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:06:47.238 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 33 16 56039 56023 106.074 77.5625 0.00654416 0.00942016 2023-06-03T20:06:47.242 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:06:47.274 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:06:47.278 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:06:47.278 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:06:47.379 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:06:47.428 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:06:47.430 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:06:47.431 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:06:47.531 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:06:47.564 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:06:47.568 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:06:47.569 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:06:47.670 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:06:47.703 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:06:47.710 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:06:47.711 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:06:47.812 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:06:47.852 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:06:47.857 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:06:47.858 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:06:47.959 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:06:47.968 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:06:47 smithi164 bash[59462]: cluster 1685822806.5657012 mgr.y (mgr.34104) 919 : cluster 0 pgmap v856: 89 pgs: 89 active+clean; 3.1 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 80 MiB/s wr, 1.28k op/s 2023-06-03T20:06:47.994 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:06:47.998 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:06:47.999 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:06:48.028 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:06:47 smithi089 bash[75068]: cluster 1685822806.5657012 mgr.y (mgr.34104) 919 : cluster 0 pgmap v856: 89 pgs: 89 active+clean; 3.1 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 80 MiB/s wr, 1.28k op/s 2023-06-03T20:06:48.029 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:06:47 smithi089 bash[78452]: cluster 1685822806.5657012 mgr.y (mgr.34104) 919 : cluster 0 pgmap v856: 89 pgs: 89 active+clean; 3.1 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 80 MiB/s wr, 1.28k op/s 2023-06-03T20:06:48.100 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:06:48.129 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:06:48.131 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:06:48.132 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:06:48.233 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:06:48.238 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 34 16 57347 57331 105.359 81.75 0.0149147 0.00948536 2023-06-03T20:06:48.267 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:06:48.269 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:06:48.270 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:06:48.371 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:06:48.407 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:06:48.409 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:06:48.410 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:06:48.511 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:06:48.541 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:06:48.544 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:06:48.544 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:06:48.645 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:06:48.686 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:06:48.689 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:06:48.690 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:06:48.790 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:06:48.821 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:06:48.824 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:06:48.825 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:06:48.925 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:06:48.954 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:06:48.957 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:06:48.958 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:06:49.060 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:06:49.091 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:06:49.094 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:06:49.094 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:06:49.195 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:06:49.227 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:06:49.229 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:06:49.231 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:06:49.238 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 35 16 58571 58555 104.534 76.5 0.0124156 0.00956063 2023-06-03T20:06:49.331 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:06:49.353 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:06:49.356 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:06:49.356 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:06:49.457 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:06:49.493 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:06:49.499 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:06:49.500 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:06:49.601 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:06:49.646 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:06:49.651 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:06:49.651 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:06:49.752 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:06:49.789 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:06:49.794 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:06:49.794 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:06:49.895 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:06:49.932 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:06:49.935 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:06:49.936 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:06:50.028 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:06:49 smithi089 bash[75068]: cluster 1685822808.5664496 mgr.y (mgr.34104) 920 : cluster 0 pgmap v857: 89 pgs: 89 active+clean; 3.4 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 92 MiB/s wr, 1.48k op/s 2023-06-03T20:06:50.029 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:06:49 smithi089 bash[78452]: cluster 1685822808.5664496 mgr.y (mgr.34104) 920 : cluster 0 pgmap v857: 89 pgs: 89 active+clean; 3.4 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 92 MiB/s wr, 1.48k op/s 2023-06-03T20:06:50.037 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:06:50.047 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:06:49 smithi164 bash[59462]: cluster 1685822808.5664496 mgr.y (mgr.34104) 920 : cluster 0 pgmap v857: 89 pgs: 89 active+clean; 3.4 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 92 MiB/s wr, 1.48k op/s 2023-06-03T20:06:50.071 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:06:50.075 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:06:50.075 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:06:50.176 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:06:50.210 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:06:50.215 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:06:50.215 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:06:50.239 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 36 16 59629 59613 103.467 66.125 0.0156215 0.00965987 2023-06-03T20:06:50.317 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:06:50.348 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:06:50.352 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:06:50.353 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:06:50.454 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:06:50.486 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:06:50.492 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:06:50.493 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:06:50.594 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:06:50.627 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:06:50.630 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:06:50.630 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:06:50.731 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:06:50.773 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:06:50.775 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:06:50.776 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:06:50.877 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:06:50.931 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:06:50.934 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:06:50.935 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:06:51.036 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:06:51.075 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:06:51.076 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:06:51.077 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:06:51.099 INFO:teuthology.orchestra.run.smithi089.stdout:ERROR: (22) Invalid argument 2023-06-03T20:06:51.099 INFO:teuthology.orchestra.run.smithi089.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-06-03T20:06:51.178 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:06:51.219 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:06:51.221 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:06:51.222 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:06:51.239 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 37 16 60739 60723 102.546 69.375 0.00805974 0.00974538 2023-06-03T20:06:51.322 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:06:51.362 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:06:51.365 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:06:51.366 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:06:51.467 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:06:51.500 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:06:51.503 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:06:51.504 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:06:51.605 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:06:51.644 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:06:51.645 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:06:51.646 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:06:51.747 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:06:51.753 INFO:teuthology.orchestra.run.smithi089.stderr:noscrub is set 2023-06-03T20:06:51.756 INFO:teuthology.orchestra.run.smithi089.stderr:set pool 10 pgp_num to 16 2023-06-03T20:06:51.782 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:06:51.785 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:06:51.786 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:06:51.887 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:06:51.923 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:06:51.926 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:06:51.927 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:06:51.941 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-06-03T20:06:51.953 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:06:51 smithi089 bash[78452]: cluster 1685822810.5673926 mgr.y (mgr.34104) 921 : cluster 0 pgmap v858: 89 pgs: 89 active+clean; 3.5 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 77 MiB/s wr, 1.23k op/s 2023-06-03T20:06:51.953 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:06:51 smithi089 bash[78452]: audit 1685822811.3496375 mon.b (mon.1) 51 : audit 1 from='client.? 172.21.15.89:0/3250197428' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num", "val": "16"}]: dispatch 2023-06-03T20:06:51.953 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:06:51 smithi089 bash[78452]: audit 1685822811.3528311 mon.a (mon.0) 825 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num", "val": "16"}]: dispatch 2023-06-03T20:06:51.953 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:06:51 smithi089 bash[78452]: audit 1685822811.6165733 mon.c (mon.2) 441 : audit 1 from='client.? 172.21.15.89:0/2705091666' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-06-03T20:06:51.954 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:06:51 smithi089 bash[78452]: audit 1685822811.619901 mon.a (mon.0) 826 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-06-03T20:06:51.954 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:06:51 smithi089 bash[75068]: cluster 1685822810.5673926 mgr.y (mgr.34104) 921 : cluster 0 pgmap v858: 89 pgs: 89 active+clean; 3.5 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 77 MiB/s wr, 1.23k op/s 2023-06-03T20:06:51.954 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:06:51 smithi089 bash[75068]: audit 1685822811.3496375 mon.b (mon.1) 51 : audit 1 from='client.? 172.21.15.89:0/3250197428' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num", "val": "16"}]: dispatch 2023-06-03T20:06:51.954 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:06:51 smithi089 bash[75068]: audit 1685822811.3528311 mon.a (mon.0) 825 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num", "val": "16"}]: dispatch 2023-06-03T20:06:51.954 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:06:51 smithi089 bash[75068]: audit 1685822811.6165733 mon.c (mon.2) 441 : audit 1 from='client.? 172.21.15.89:0/2705091666' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-06-03T20:06:51.955 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:06:51 smithi089 bash[75068]: audit 1685822811.619901 mon.a (mon.0) 826 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-06-03T20:06:52.028 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:06:52.077 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:06:52.080 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:06:52.080 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:06:52.083 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:06:51 smithi164 bash[59462]: cluster 1685822810.5673926 mgr.y (mgr.34104) 921 : cluster 0 pgmap v858: 89 pgs: 89 active+clean; 3.5 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 77 MiB/s wr, 1.23k op/s 2023-06-03T20:06:52.083 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:06:51 smithi164 bash[59462]: audit 1685822811.3496375 mon.b (mon.1) 51 : audit 1 from='client.? 172.21.15.89:0/3250197428' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num", "val": "16"}]: dispatch 2023-06-03T20:06:52.083 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:06:51 smithi164 bash[59462]: audit 1685822811.3528311 mon.a (mon.0) 825 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num", "val": "16"}]: dispatch 2023-06-03T20:06:52.084 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:06:51 smithi164 bash[59462]: audit 1685822811.6165733 mon.c (mon.2) 441 : audit 1 from='client.? 172.21.15.89:0/2705091666' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-06-03T20:06:52.084 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:06:51 smithi164 bash[59462]: audit 1685822811.619901 mon.a (mon.0) 826 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-06-03T20:06:52.181 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:06:52.223 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:06:52.226 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:06:52.227 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:06:52.239 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 38 16 61842 61826 101.662 68.9375 0.0180167 0.0098307 2023-06-03T20:06:52.328 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:06:52.363 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:06:52.365 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:06:52.366 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:06:52.467 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:06:52.500 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:06:52.507 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:06:52.508 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:06:52.608 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:06:52.643 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:06:52.646 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:06:52.646 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:06:52.747 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:06:52.757 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:06:52 smithi164 bash[59462]: cluster 1685822811.7320337 mon.a (mon.0) 827 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-06-03T20:06:52.757 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:06:52 smithi164 bash[59462]: cluster 1685822811.7327569 osd.6 (osd.6) 93 : cluster 3 failed to encode map e164 with expected crc 2023-06-03T20:06:52.757 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:06:52 smithi164 bash[59462]: audit 1685822811.7390616 mon.a (mon.0) 828 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num", "val": "16"}]': finished 2023-06-03T20:06:52.757 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:06:52 smithi164 bash[59462]: audit 1685822811.7393332 mon.a (mon.0) 829 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-06-03T20:06:52.758 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:06:52 smithi164 bash[59462]: cluster 1685822811.7398434 mon.a (mon.0) 830 : cluster 0 osdmap e164: 8 total, 8 up, 8 in 2023-06-03T20:06:52.758 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:06:52 smithi164 bash[59462]: cluster 1685822811.7438457 osd.7 (osd.7) 94 : cluster 3 failed to encode map e164 with expected crc 2023-06-03T20:06:52.758 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:06:52 smithi164 bash[59462]: audit 1685822812.1169536 mon.a (mon.0) 831 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:06:52.758 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:06:52 smithi164 bash[59462]: audit 1685822812.1194174 mon.c (mon.2) 442 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T20:06:52.786 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:06:52.791 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:06:52.793 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:06:52.894 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph daemon osd.0 dump_blocked_ops 2023-06-03T20:06:52.894 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:06:52 smithi089 bash[78452]: cluster 1685822811.7320337 mon.a (mon.0) 827 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-06-03T20:06:52.894 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:06:52 smithi089 bash[78452]: cluster 1685822811.7327569 osd.6 (osd.6) 93 : cluster 3 failed to encode map e164 with expected crc 2023-06-03T20:06:52.895 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:06:52 smithi089 bash[78452]: audit 1685822811.7390616 mon.a (mon.0) 828 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num", "val": "16"}]': finished 2023-06-03T20:06:52.895 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:06:52 smithi089 bash[78452]: audit 1685822811.7393332 mon.a (mon.0) 829 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-06-03T20:06:52.895 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:06:52 smithi089 bash[78452]: cluster 1685822811.7398434 mon.a (mon.0) 830 : cluster 0 osdmap e164: 8 total, 8 up, 8 in 2023-06-03T20:06:52.895 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:06:52 smithi089 bash[78452]: cluster 1685822811.7438457 osd.7 (osd.7) 94 : cluster 3 failed to encode map e164 with expected crc 2023-06-03T20:06:52.895 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:06:52 smithi089 bash[78452]: audit 1685822812.1169536 mon.a (mon.0) 831 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:06:52.895 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:06:52 smithi089 bash[78452]: audit 1685822812.1194174 mon.c (mon.2) 442 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T20:06:52.896 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:06:52 smithi089 bash[75068]: cluster 1685822811.7320337 mon.a (mon.0) 827 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-06-03T20:06:52.896 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:06:52 smithi089 bash[75068]: cluster 1685822811.7327569 osd.6 (osd.6) 93 : cluster 3 failed to encode map e164 with expected crc 2023-06-03T20:06:52.896 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:06:52 smithi089 bash[75068]: audit 1685822811.7390616 mon.a (mon.0) 828 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num", "val": "16"}]': finished 2023-06-03T20:06:52.896 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:06:52 smithi089 bash[75068]: audit 1685822811.7393332 mon.a (mon.0) 829 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-06-03T20:06:52.896 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:06:52 smithi089 bash[75068]: cluster 1685822811.7398434 mon.a (mon.0) 830 : cluster 0 osdmap e164: 8 total, 8 up, 8 in 2023-06-03T20:06:52.897 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:06:52 smithi089 bash[75068]: cluster 1685822811.7438457 osd.7 (osd.7) 94 : cluster 3 failed to encode map e164 with expected crc 2023-06-03T20:06:52.897 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:06:52 smithi089 bash[75068]: audit 1685822812.1169536 mon.a (mon.0) 831 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:06:52.897 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:06:52 smithi089 bash[75068]: audit 1685822812.1194174 mon.c (mon.2) 442 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T20:06:52.897 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:06:52.938 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:06:52.942 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:06:52.943 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:06:53.043 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:06:53.078 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:06:53.083 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:06:53.084 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:06:53.185 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:06:53.234 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:06:53.237 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:06:53.238 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:06:53.240 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 39 16 63020 63004 100.94 73.625 0.0141913 0.00990099 2023-06-03T20:06:53.340 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:06:53.372 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:06:53.376 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:06:53.377 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:06:53.478 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:06:53.511 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:06:53.516 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:06:53.517 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:06:53.619 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:06:53.653 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:06:53.654 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:06:53.655 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:06:53.755 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:06:53.792 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:06:53.797 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:06:53.798 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:06:53.898 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:06:53.908 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:06:53 smithi164 bash[59462]: cluster 1685822812.5716867 mgr.y (mgr.34104) 922 : cluster 0 pgmap v860: 89 pgs: 89 active+clean; 3.6 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 78 MiB/s wr, 1.24k op/s 2023-06-03T20:06:53.947 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:06:53.952 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:06:53.952 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:06:54.028 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:06:53 smithi089 bash[75068]: cluster 1685822812.5716867 mgr.y (mgr.34104) 922 : cluster 0 pgmap v860: 89 pgs: 89 active+clean; 3.6 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 78 MiB/s wr, 1.24k op/s 2023-06-03T20:06:54.029 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:06:53 smithi089 bash[78452]: cluster 1685822812.5716867 mgr.y (mgr.34104) 922 : cluster 0 pgmap v860: 89 pgs: 89 active+clean; 3.6 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 78 MiB/s wr, 1.24k op/s 2023-06-03T20:06:54.053 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:06:54.090 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:06:54.092 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:06:54.093 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:06:54.195 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:06:54.223 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:06:54.226 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:06:54.227 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:06:54.240 INFO:tasks.radosbench.radosbench.0.smithi089.stdout:2023-06-03T20:06:54.236651+0000 min lat: 0.00128391 max lat: 0.19851 avg lat: 0.00996758 2023-06-03T20:06:54.240 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-06-03T20:06:54.241 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 40 16 64207 64191 100.271 74.1875 0.0196743 0.00996758 2023-06-03T20:06:54.328 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:06:54.363 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:06:54.366 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:06:54.366 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:06:54.467 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:06:54.506 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:06:54.509 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:06:54.510 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:06:54.611 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:06:54.648 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:06:54.654 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:06:54.656 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:06:54.757 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:06:54.795 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:06:54.796 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:06:54.797 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:06:54.897 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:06:54.932 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:06:54.936 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:06:54.937 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:06:55.038 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:06:55.079 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:06:55.082 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:06:55.083 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:06:55.184 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:06:55.223 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:06:55.226 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:06:55.227 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:06:55.240 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 41 16 65354 65338 99.5742 71.6875 0.00986467 0.0100371 2023-06-03T20:06:55.328 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:06:55.361 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:06:55.365 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:06:55.366 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:06:55.467 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:06:55.511 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:06:55.514 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:06:55.514 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:06:55.615 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:06:55.656 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:06:55.662 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:06:55.663 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:06:55.762 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:06:55 smithi089 bash[75068]: cluster 1685822814.5724347 mgr.y (mgr.34104) 923 : cluster 0 pgmap v861: 89 pgs: 89 active+clean; 3.8 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 75 MiB/s wr, 1.20k op/s 2023-06-03T20:06:55.764 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:06:55.799 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:06:55.803 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:06:55.804 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:06:55.904 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:06:55.945 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:06:55.951 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:06:55.952 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:06:56.028 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:06:55 smithi089 bash[78452]: cluster 1685822814.5724347 mgr.y (mgr.34104) 923 : cluster 0 pgmap v861: 89 pgs: 89 active+clean; 3.8 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 75 MiB/s wr, 1.20k op/s 2023-06-03T20:06:56.052 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:06:56.061 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:06:55 smithi164 bash[59462]: cluster 1685822814.5724347 mgr.y (mgr.34104) 923 : cluster 0 pgmap v861: 89 pgs: 89 active+clean; 3.8 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 75 MiB/s wr, 1.20k op/s 2023-06-03T20:06:56.092 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:06:56.097 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:06:56.098 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:06:56.199 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:06:56.217 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph osd set nodeep-scrub 2023-06-03T20:06:56.238 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:06:56.240 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 42 16 66533 66517 98.9578 73.6875 0.0162082 0.0100981 2023-06-03T20:06:56.245 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:06:56.246 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:06:56.347 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:06:56.385 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:06:56.390 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:06:56.391 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:06:56.492 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:06:56.503 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 20:06:56 smithi089 bash[71163]: ::ffff:172.21.15.164 - - [03/Jun/2023:20:06:56] "GET /metrics HTTP/1.1" 200 228685 "" "Prometheus/2.33.4" 2023-06-03T20:06:56.533 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:06:56.536 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:06:56.537 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:06:56.638 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:06:56.677 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:06:56.680 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:06:56.681 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:06:56.783 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:06:56.818 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:06:56.822 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:06:56.824 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:06:56.841 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/mon.a/config 2023-06-03T20:06:56.841 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/mon.a/config 2023-06-03T20:06:56.926 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:06:56.956 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:06:56.962 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:06:56.963 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:06:57.064 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:06:57.100 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:06:57.105 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:06:57.106 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:06:57.207 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:06:57.236 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:06:57.240 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 43 16 67620 67604 98.2363 67.9375 0.00957261 0.010173 2023-06-03T20:06:57.242 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:06:57.243 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:06:57.343 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:06:57.373 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:06:57.377 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:06:57.378 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:06:57.480 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:06:57.515 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:06:57.518 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:06:57.519 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:06:57.620 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:06:57.655 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:06:57.660 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:06:57.660 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:06:57.761 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:06:57.799 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:06:57.802 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:06:57.803 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:06:57.904 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:06:57.930 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:06:57.934 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:06:57.936 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:06:58.036 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:06:58.046 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:06:57 smithi164 bash[59462]: cluster 1685822816.5731084 mgr.y (mgr.34104) 924 : cluster 0 pgmap v862: 89 pgs: 89 active+clean; 3.9 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 74 MiB/s wr, 1.18k op/s 2023-06-03T20:06:58.068 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:06:57 smithi089 bash[75068]: cluster 1685822816.5731084 mgr.y (mgr.34104) 924 : cluster 0 pgmap v862: 89 pgs: 89 active+clean; 3.9 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 74 MiB/s wr, 1.18k op/s 2023-06-03T20:06:58.069 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:06:57 smithi089 bash[78452]: cluster 1685822816.5731084 mgr.y (mgr.34104) 924 : cluster 0 pgmap v862: 89 pgs: 89 active+clean; 3.9 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 74 MiB/s wr, 1.18k op/s 2023-06-03T20:06:58.070 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:06:58.075 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:06:58.076 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:06:58.177 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:06:58.214 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:06:58.217 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:06:58.218 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:06:58.240 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 44 16 68708 68692 97.5491 68 0.0146535 0.010246 2023-06-03T20:06:58.318 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:06:58.350 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:06:58.353 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:06:58.354 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:06:58.457 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:06:58.493 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:06:58.494 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:06:58.495 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:06:58.597 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:06:58.626 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:06:58.630 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:06:58.631 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:06:58.731 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:06:58.770 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:06:58.774 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:06:58.775 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:06:58.876 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:06:58.925 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:06:58.926 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:06:58.927 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:06:59.028 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:06:59.067 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:06:59.072 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:06:59.073 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:06:59.174 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:06:59.204 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:06:59.206 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:06:59.207 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:06:59.240 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 45 16 69839 69823 96.9521 70.6875 0.0164558 0.0103081 2023-06-03T20:06:59.264 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 5, 6, 7] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-06-03T20:06:59.264 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-06-03T20:06:59.265 INFO:tasks.thrashosds.thrasher:Removing osd 1, in_osds are: [0, 1, 2, 3, 4, 5, 6, 7] 2023-06-03T20:06:59.265 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph osd out 1 2023-06-03T20:06:59.308 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:06:59.354 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:06:59.358 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:06:59.359 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:06:59.460 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:06:59.496 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:06:59.502 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:06:59.503 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:06:59.604 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:06:59.649 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:06:59.656 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:06:59.657 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:06:59.758 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:06:59.790 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:06:59.793 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:06:59.794 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:06:59.895 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:06:59.940 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:06:59.945 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:06:59.946 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:07:00.047 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:07:00.056 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:06:59 smithi089 bash[78452]: cluster 1685822818.5738761 mgr.y (mgr.34104) 925 : cluster 0 pgmap v863: 89 pgs: 89 active+clean; 4.1 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 74 MiB/s wr, 1.19k op/s 2023-06-03T20:07:00.057 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:06:59 smithi089 bash[78452]: audit 1685822819.3259823 mon.c (mon.2) 443 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-03T20:07:00.057 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:06:59 smithi089 bash[75068]: cluster 1685822818.5738761 mgr.y (mgr.34104) 925 : cluster 0 pgmap v863: 89 pgs: 89 active+clean; 4.1 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 74 MiB/s wr, 1.19k op/s 2023-06-03T20:07:00.057 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:06:59 smithi089 bash[75068]: audit 1685822819.3259823 mon.c (mon.2) 443 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-03T20:07:00.083 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:06:59 smithi164 bash[59462]: cluster 1685822818.5738761 mgr.y (mgr.34104) 925 : cluster 0 pgmap v863: 89 pgs: 89 active+clean; 4.1 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 74 MiB/s wr, 1.19k op/s 2023-06-03T20:07:00.084 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:06:59 smithi164 bash[59462]: audit 1685822819.3259823 mon.c (mon.2) 443 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-03T20:07:00.084 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:07:00.089 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:07:00.090 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:07:00.191 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:07:00.226 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:07:00.230 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:07:00.231 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:07:00.240 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 46 16 70840 70824 96.2045 62.5625 0.00722109 0.0103884 2023-06-03T20:07:00.332 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:07:00.379 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:07:00.384 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:07:00.389 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:07:00.489 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:07:00.522 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:07:00.525 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:07:00.526 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:07:00.627 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:07:00.665 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:07:00.669 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:07:00.670 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:07:00.771 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:07:00.801 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:07:00.804 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:07:00.805 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:07:00.906 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:07:00.944 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/mon.a/config 2023-06-03T20:07:00.950 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:07:00.955 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:07:00.956 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:07:01.057 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:07:01.091 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:07:01.099 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:07:01.099 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:07:01.200 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:07:01.230 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:07:01.233 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:07:01.234 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:07:01.241 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 47 16 71903 71887 95.5712 66.4375 0.0220101 0.0104568 2023-06-03T20:07:01.334 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:07:01.358 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:07:01.359 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:07:01.360 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:07:01.461 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:07:01.505 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:07:01.508 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:07:01.508 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:07:01.609 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:07:01.652 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:07:01.654 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:07:01.655 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:07:01.756 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:07:01.794 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:07:01.799 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:07:01.799 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:07:01.900 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:07:01.938 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:07:01.941 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:07:01.941 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:07:02.028 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:07:01 smithi089 bash[78452]: cluster 1685822820.5758617 mgr.y (mgr.34104) 926 : cluster 0 pgmap v864: 89 pgs: 89 active+clean; 4.2 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 70 MiB/s wr, 1.12k op/s 2023-06-03T20:07:02.029 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:07:01 smithi089 bash[75068]: cluster 1685822820.5758617 mgr.y (mgr.34104) 926 : cluster 0 pgmap v864: 89 pgs: 89 active+clean; 4.2 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 70 MiB/s wr, 1.12k op/s 2023-06-03T20:07:02.043 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:07:02.052 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:07:01 smithi164 bash[59462]: cluster 1685822820.5758617 mgr.y (mgr.34104) 926 : cluster 0 pgmap v864: 89 pgs: 89 active+clean; 4.2 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 70 MiB/s wr, 1.12k op/s 2023-06-03T20:07:02.074 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:07:02.077 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:07:02.078 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:07:02.179 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:07:02.218 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:07:02.221 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:07:02.222 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:07:02.241 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 48 16 72987 72971 94.9915 67.75 0.0083763 0.0105213 2023-06-03T20:07:02.323 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:07:02.357 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:07:02.362 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:07:02.363 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:07:02.464 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:07:02.506 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:07:02.510 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:07:02.510 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:07:02.612 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:07:02.651 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:07:02.657 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:07:02.658 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:07:02.759 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:07:02.787 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:07:02.789 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:07:02.789 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:07:02.890 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:07:02.923 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:07:02.924 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:07:02.925 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:07:03.026 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:07:03.057 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:07:03.062 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:07:03.063 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:07:03.164 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:07:03.200 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:07:03.205 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:07:03.206 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:07:03.241 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 49 16 74093 74077 94.4636 69.125 0.0114968 0.0105805 2023-06-03T20:07:03.306 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:07:03.343 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:07:03.348 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:07:03.349 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:07:03.450 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:07:03.493 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:07:03.496 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:07:03.497 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:07:03.598 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:07:03.627 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:07:03.633 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:07:03.634 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:07:03.735 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:07:03.792 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:07:03.797 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:07:03.798 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:07:03.891 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:07:03 smithi089 bash[78452]: cluster 1685822822.5765831 mgr.y (mgr.34104) 927 : cluster 0 pgmap v865: 89 pgs: 89 active+clean; 4.3 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 67 MiB/s wr, 1.07k op/s 2023-06-03T20:07:03.891 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:07:03 smithi089 bash[75068]: cluster 1685822822.5765831 mgr.y (mgr.34104) 927 : cluster 0 pgmap v865: 89 pgs: 89 active+clean; 4.3 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 67 MiB/s wr, 1.07k op/s 2023-06-03T20:07:03.899 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:07:03.942 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:07:03.945 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:07:03.946 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:07:04.047 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:07:04.083 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:07:03 smithi164 bash[59462]: cluster 1685822822.5765831 mgr.y (mgr.34104) 927 : cluster 0 pgmap v865: 89 pgs: 89 active+clean; 4.3 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 67 MiB/s wr, 1.07k op/s 2023-06-03T20:07:04.089 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:07:04.095 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:07:04.096 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:07:04.196 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:07:04.241 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 50 16 75143 75127 93.8867 65.625 0.0113995 0.0106451 2023-06-03T20:07:04.363 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:07:04.369 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:07:04.369 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:07:04.470 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:07:04.515 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:07:04.518 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:07:04.518 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:07:04.619 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:07:04.647 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:07:04.650 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:07:04.651 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:07:04.751 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:07:04.782 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:07:04.783 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:07:04.784 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:07:04.885 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:07:04.923 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:07:04.927 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:07:04.927 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:07:05.028 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:07:05.060 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:07:05.062 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:07:05.063 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:07:05.164 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:07:05.198 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:07:05.205 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:07:05.207 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:07:05.241 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 51 16 76001 75985 93.0973 53.625 0.020114 0.0107357 2023-06-03T20:07:05.308 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:07:05.336 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:07:05.344 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:07:05.345 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:07:05.446 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:07:05.476 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:07:05.482 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:07:05.482 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:07:05.583 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:07:05.614 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:07:05.618 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:07:05.619 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:07:05.720 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:07:05.752 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:07:05.755 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:07:05.759 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:07:05.860 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:07:05.895 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:07:05.901 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:07:05.902 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:07:06.004 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:07:06.043 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:07:06.048 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:07:06.049 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:07:06.083 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:07:05 smithi164 bash[59462]: cluster 1685822824.5771768 mgr.y (mgr.34104) 928 : cluster 0 pgmap v866: 89 pgs: 89 active+clean; 4.5 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 76 MiB/s wr, 1.21k op/s 2023-06-03T20:07:06.150 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:07:06.191 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:07:06.194 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:07:06.194 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:07:06.231 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 20:07:06 smithi089 bash[71163]: ::ffff:172.21.15.164 - - [03/Jun/2023:20:07:06] "GET /metrics HTTP/1.1" 200 228685 "" "Prometheus/2.33.4" 2023-06-03T20:07:06.232 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:07:05 smithi089 bash[78452]: cluster 1685822824.5771768 mgr.y (mgr.34104) 928 : cluster 0 pgmap v866: 89 pgs: 89 active+clean; 4.5 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 76 MiB/s wr, 1.21k op/s 2023-06-03T20:07:06.232 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:07:05 smithi089 bash[75068]: cluster 1685822824.5771768 mgr.y (mgr.34104) 928 : cluster 0 pgmap v866: 89 pgs: 89 active+clean; 4.5 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 76 MiB/s wr, 1.21k op/s 2023-06-03T20:07:06.241 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 52 16 77028 77012 92.5413 64.1875 0.00567319 0.0107984 2023-06-03T20:07:06.296 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:07:06.328 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:07:06.331 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:07:06.331 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:07:06.432 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:07:06.475 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:07:06.479 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:07:06.480 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:07:06.581 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:07:06.621 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:07:06.626 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:07:06.626 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:07:06.727 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:07:06.759 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:07:06.762 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:07:06.763 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:07:06.864 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:07:06.896 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:07:06.901 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:07:06.902 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:07:07.003 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:07:07.039 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:07:07.041 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:07:07.042 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:07:07.143 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:07:07.190 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:07:07.194 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:07:07.195 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:07:07.241 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 53 16 78057 78041 92.0086 64.3125 0.00882748 0.0108627 2023-06-03T20:07:07.296 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:07:07.328 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:07:07.335 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:07:07.336 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:07:07.437 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:07:07.476 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:07:07.482 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:07:07.483 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:07:07.584 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:07:07.621 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:07:07.627 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:07:07.628 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:07:07.729 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:07:07.761 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:07:07.764 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:07:07.765 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:07:07.812 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:07:07 smithi164 bash[59462]: cluster 1685822826.5776894 mgr.y (mgr.34104) 929 : cluster 0 pgmap v867: 89 pgs: 89 active+clean; 4.5 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 64 MiB/s wr, 1.02k op/s 2023-06-03T20:07:07.812 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:07:07 smithi164 bash[59462]: audit 1685822827.1092944 mon.c (mon.2) 444 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T20:07:07.812 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:07:07 smithi164 bash[59462]: audit 1685822827.7503726 mon.a (mon.0) 832 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:07:07.812 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:07:07 smithi164 bash[59462]: audit 1685822827.7656028 mon.a (mon.0) 833 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:07:07.866 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:07:07.900 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:07:07.902 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:07:07.903 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:07:08.004 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:07:08.013 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:07:07 smithi089 bash[75068]: cluster 1685822826.5776894 mgr.y (mgr.34104) 929 : cluster 0 pgmap v867: 89 pgs: 89 active+clean; 4.5 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 64 MiB/s wr, 1.02k op/s 2023-06-03T20:07:08.013 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:07:07 smithi089 bash[75068]: audit 1685822827.1092944 mon.c (mon.2) 444 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T20:07:08.013 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:07:07 smithi089 bash[75068]: audit 1685822827.7503726 mon.a (mon.0) 832 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:07:08.014 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:07:07 smithi089 bash[75068]: audit 1685822827.7656028 mon.a (mon.0) 833 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:07:08.014 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:07:07 smithi089 bash[78452]: cluster 1685822826.5776894 mgr.y (mgr.34104) 929 : cluster 0 pgmap v867: 89 pgs: 89 active+clean; 4.5 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 64 MiB/s wr, 1.02k op/s 2023-06-03T20:07:08.014 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:07:07 smithi089 bash[78452]: audit 1685822827.1092944 mon.c (mon.2) 444 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T20:07:08.014 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:07:07 smithi089 bash[78452]: audit 1685822827.7503726 mon.a (mon.0) 832 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:07:08.014 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:07:07 smithi089 bash[78452]: audit 1685822827.7656028 mon.a (mon.0) 833 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:07:08.035 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:07:08.037 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:07:08.038 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:07:08.139 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:07:08.196 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:07:08.197 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:07:08.198 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:07:08.241 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 54 16 79015 78999 91.4136 59.875 0.0154288 0.0109324 2023-06-03T20:07:08.299 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:07:08.328 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:07:08.335 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:07:08.335 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:07:08.436 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:07:08.480 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:07:08.485 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:07:08.486 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:07:08.587 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:07:08.628 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:07:08.633 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:07:08.634 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:07:08.735 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:07:08.766 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:07:08.768 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:07:08.769 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:07:08.870 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:07:08.905 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:07:08.915 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:07:08.916 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:07:09.017 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:07:09.048 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:07:09.050 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:07:09.051 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:07:09.152 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:07:09.183 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:07:09.189 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:07:09.190 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:07:09.241 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 55 16 80022 80006 90.8958 62.9375 0.0235583 0.0109966 2023-06-03T20:07:09.291 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:07:09.328 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:07:09.330 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:07:09.331 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:07:09.364 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/mon.a/config 2023-06-03T20:07:09.432 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:07:09.505 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:07:09.513 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:07:09.514 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:07:09.520 INFO:teuthology.orchestra.run.smithi089.stdout:osd.0: {} 2023-06-03T20:07:09.605 INFO:teuthology.orchestra.run.smithi089.stdout:osd.1: {} 2023-06-03T20:07:09.615 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:07:09.643 INFO:teuthology.orchestra.run.smithi089.stdout:osd.2: {} 2023-06-03T20:07:09.654 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:07:09.655 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:07:09.655 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:07:09.701 INFO:teuthology.orchestra.run.smithi089.stdout:osd.3: {} 2023-06-03T20:07:09.712 INFO:teuthology.orchestra.run.smithi089.stdout:{ 2023-06-03T20:07:09.712 INFO:teuthology.orchestra.run.smithi089.stdout: "ops": [], 2023-06-03T20:07:09.712 INFO:teuthology.orchestra.run.smithi089.stdout: "complaint_time": 30, 2023-06-03T20:07:09.712 INFO:teuthology.orchestra.run.smithi089.stdout: "num_blocked_ops": 0 2023-06-03T20:07:09.715 INFO:teuthology.orchestra.run.smithi089.stdout:} 2023-06-03T20:07:09.752 INFO:teuthology.orchestra.run.smithi089.stdout:osd.4: {} 2023-06-03T20:07:09.758 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:07:09.794 INFO:teuthology.orchestra.run.smithi089.stdout:osd.5: {} 2023-06-03T20:07:09.812 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:07:09.815 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:07:09.815 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:07:09.830 INFO:teuthology.orchestra.run.smithi089.stdout:osd.6: {} 2023-06-03T20:07:09.866 INFO:teuthology.orchestra.run.smithi089.stdout:osd.7: {} 2023-06-03T20:07:09.871 INFO:teuthology.orchestra.run.smithi089.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-06-03T20:07:09.871 INFO:teuthology.orchestra.run.smithi089.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-06-03T20:07:09.871 INFO:teuthology.orchestra.run.smithi089.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-06-03T20:07:09.872 INFO:teuthology.orchestra.run.smithi089.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-06-03T20:07:09.872 INFO:teuthology.orchestra.run.smithi089.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-06-03T20:07:09.872 INFO:teuthology.orchestra.run.smithi089.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-06-03T20:07:09.872 INFO:teuthology.orchestra.run.smithi089.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-06-03T20:07:09.872 INFO:teuthology.orchestra.run.smithi089.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-06-03T20:07:09.904 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:07:09 smithi089 bash[75068]: cluster 1685822828.578268 mgr.y (mgr.34104) 930 : cluster 0 pgmap v868: 89 pgs: 89 active+clean; 4.8 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 76 MiB/s wr, 1.22k op/s 2023-06-03T20:07:09.904 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:07:09 smithi089 bash[75068]: audit 1685822829.4769099 mon.b (mon.1) 52 : audit 0 from='client.? 172.21.15.89:0/648995991' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-03T20:07:09.904 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:07:09 smithi089 bash[78452]: cluster 1685822828.578268 mgr.y (mgr.34104) 930 : cluster 0 pgmap v868: 89 pgs: 89 active+clean; 4.8 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 76 MiB/s wr, 1.22k op/s 2023-06-03T20:07:09.904 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:07:09 smithi089 bash[78452]: audit 1685822829.4769099 mon.b (mon.1) 52 : audit 0 from='client.? 172.21.15.89:0/648995991' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-03T20:07:09.917 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:07:09.948 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:07:09.951 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:07:09.951 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:07:10.052 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:07:10.084 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:07:09 smithi164 bash[59462]: cluster 1685822828.578268 mgr.y (mgr.34104) 930 : cluster 0 pgmap v868: 89 pgs: 89 active+clean; 4.8 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 76 MiB/s wr, 1.22k op/s 2023-06-03T20:07:10.084 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:07:09 smithi164 bash[59462]: audit 1685822829.4769099 mon.b (mon.1) 52 : audit 0 from='client.? 172.21.15.89:0/648995991' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-03T20:07:10.092 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:07:10.098 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:07:10.098 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:07:10.199 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:07:10.242 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 56 16 80778 80762 90.1164 47.25 0.0078352 0.0110899 2023-06-03T20:07:10.243 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:07:10.245 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:07:10.245 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:07:10.347 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:07:10.376 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:07:10.378 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:07:10.379 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:07:10.480 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:07:10.532 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:07:10.567 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:07:10.567 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:07:10.668 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:07:10.699 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:07:10.705 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:07:10.706 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:07:10.806 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:07:10.848 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:07:10.853 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:07:10.854 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:07:10.955 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:07:11.005 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:07:11.009 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:07:11.009 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:07:11.110 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:07:11.153 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:07:11.157 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:07:11.157 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:07:11.241 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 57 16 81723 81707 89.5716 59.0625 0.017526 0.0111588 2023-06-03T20:07:11.258 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:07:11.295 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:07:11.297 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:07:11.298 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:07:11.399 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:07:11.439 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:07:11.443 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:07:11.443 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:07:11.544 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:07:11.574 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:07:11.577 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:07:11.578 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:07:11.679 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:07:11.708 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:07:11.713 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:07:11.714 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:07:11.815 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:07:11.851 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:07:11.855 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:07:11.855 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:07:11.956 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:07:11.994 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:07:12.012 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:07:12.013 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:07:12.115 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:07:12.123 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:07:11 smithi164 bash[59462]: cluster 1685822830.5787587 mgr.y (mgr.34104) 931 : cluster 0 pgmap v869: 89 pgs: 89 active+clean; 4.8 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 61 MiB/s wr, 968 op/s 2023-06-03T20:07:12.123 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:07:11 smithi164 bash[59462]: audit 1685822830.8366992 mon.a (mon.0) 834 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:07:12.124 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:07:11 smithi164 bash[59462]: audit 1685822830.8498483 mon.a (mon.0) 835 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:07:12.124 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:07:11 smithi164 bash[59462]: audit 1685822831.4256546 mon.c (mon.2) 445 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-03T20:07:12.124 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:07:11 smithi164 bash[59462]: audit 1685822831.4267855 mon.c (mon.2) 446 : audit 1 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-03T20:07:12.124 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:07:11 smithi164 bash[59462]: audit 1685822831.4340255 mon.a (mon.0) 836 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:07:12.144 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:07:12.151 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:07:12.152 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:07:12.188 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:07:12 smithi089 bash[75068]: cluster 1685822830.5787587 mgr.y (mgr.34104) 931 : cluster 0 pgmap v869: 89 pgs: 89 active+clean; 4.8 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 61 MiB/s wr, 968 op/s 2023-06-03T20:07:12.188 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:07:12 smithi089 bash[75068]: audit 1685822830.8366992 mon.a (mon.0) 834 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:07:12.188 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:07:12 smithi089 bash[75068]: audit 1685822830.8498483 mon.a (mon.0) 835 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:07:12.188 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:07:12 smithi089 bash[75068]: audit 1685822831.4256546 mon.c (mon.2) 445 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-03T20:07:12.188 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:07:12 smithi089 bash[75068]: audit 1685822831.4267855 mon.c (mon.2) 446 : audit 1 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-03T20:07:12.189 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:07:12 smithi089 bash[75068]: audit 1685822831.4340255 mon.a (mon.0) 836 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:07:12.189 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:07:12 smithi089 bash[78452]: cluster 1685822830.5787587 mgr.y (mgr.34104) 931 : cluster 0 pgmap v869: 89 pgs: 89 active+clean; 4.8 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 61 MiB/s wr, 968 op/s 2023-06-03T20:07:12.189 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:07:12 smithi089 bash[78452]: audit 1685822830.8366992 mon.a (mon.0) 834 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:07:12.189 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:07:12 smithi089 bash[78452]: audit 1685822830.8498483 mon.a (mon.0) 835 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:07:12.189 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:07:12 smithi089 bash[78452]: audit 1685822831.4256546 mon.c (mon.2) 445 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-03T20:07:12.189 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:07:12 smithi089 bash[78452]: audit 1685822831.4267855 mon.c (mon.2) 446 : audit 1 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-03T20:07:12.190 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:07:12 smithi089 bash[78452]: audit 1685822831.4340255 mon.a (mon.0) 836 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:07:12.242 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 58 16 82748 82732 89.1317 64.0625 0.0137609 0.011214 2023-06-03T20:07:12.253 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:07:12.288 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:07:12.293 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:07:12.294 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:07:12.395 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:07:12.431 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:07:12.433 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:07:12.433 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:07:12.534 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:07:12.579 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:07:12.582 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:07:12.583 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:07:12.684 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:07:12.721 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:07:12.727 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:07:12.727 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:07:12.828 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:07:12.870 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:07:12.875 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:07:12.875 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:07:12.976 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:07:13.020 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:07:13.030 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:07:13.030 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:07:13.131 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:07:13.168 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:07:13.172 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:07:13.173 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:07:13.242 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 59 16 83776 83760 88.7099 64.25 0.0124706 0.0112666 2023-06-03T20:07:13.274 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:07:13.308 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:07:13.311 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:07:13.312 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:07:13.412 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:07:13.451 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:07:13.455 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:07:13.456 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:07:13.477 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph daemon osd.0 dump_historic_ops 2023-06-03T20:07:13.557 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:07:13.603 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:07:13.605 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:07:13.606 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:07:13.707 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:07:13.751 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:07:13.754 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:07:13.755 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:07:13.855 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:07:13.890 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:07:13.893 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:07:13.894 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:07:13.995 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:07:14.029 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:07:14.039 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:07:14.039 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:07:14.095 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:07:13 smithi089 bash[78452]: cluster 1685822832.5792851 mgr.y (mgr.34104) 932 : cluster 0 pgmap v870: 89 pgs: 89 active+clean; 4.9 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 58 MiB/s wr, 931 op/s 2023-06-03T20:07:14.096 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:07:13 smithi089 bash[75068]: cluster 1685822832.5792851 mgr.y (mgr.34104) 932 : cluster 0 pgmap v870: 89 pgs: 89 active+clean; 4.9 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 58 MiB/s wr, 931 op/s 2023-06-03T20:07:14.140 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:07:14.150 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:07:13 smithi164 bash[59462]: cluster 1685822832.5792851 mgr.y (mgr.34104) 932 : cluster 0 pgmap v870: 89 pgs: 89 active+clean; 4.9 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 58 MiB/s wr, 931 op/s 2023-06-03T20:07:14.185 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:07:14.188 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:07:14.188 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:07:14.242 INFO:tasks.radosbench.radosbench.0.smithi089.stdout:2023-06-03T20:07:14.238488+0000 min lat: 0.00128391 max lat: 0.19851 avg lat: 0.0113323 2023-06-03T20:07:14.242 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-06-03T20:07:14.242 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 60 16 84697 84681 88.1908 57.5625 0.00431845 0.0113323 2023-06-03T20:07:14.289 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:07:14.325 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:07:14.334 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:07:14.334 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:07:14.436 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:07:14.475 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:07:14.483 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:07:14.484 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:07:14.585 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:07:14.631 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:07:14.633 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:07:14.634 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:07:14.735 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:07:14.770 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:07:14.776 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:07:14.777 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:07:14.847 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:07:14 smithi164 bash[59462]: audit 1685822834.5655758 mon.c (mon.2) 447 : audit 1 from='client.? 172.21.15.89:0/1860791833' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-06-03T20:07:14.847 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:07:14 smithi164 bash[59462]: audit 1685822834.5664606 mon.a (mon.0) 837 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-06-03T20:07:14.878 INFO:teuthology.orchestra.run.smithi089.stderr:nodeep-scrub is set 2023-06-03T20:07:14.878 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:07:14.909 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:07:14.913 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:07:14.914 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:07:15.015 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:07:15.024 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:07:14 smithi089 bash[75068]: audit 1685822834.5655758 mon.c (mon.2) 447 : audit 1 from='client.? 172.21.15.89:0/1860791833' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-06-03T20:07:15.024 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:07:14 smithi089 bash[75068]: audit 1685822834.5664606 mon.a (mon.0) 837 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-06-03T20:07:15.025 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:07:14 smithi089 bash[78452]: audit 1685822834.5655758 mon.c (mon.2) 447 : audit 1 from='client.? 172.21.15.89:0/1860791833' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-06-03T20:07:15.025 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:07:14 smithi089 bash[78452]: audit 1685822834.5664606 mon.a (mon.0) 837 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-06-03T20:07:15.042 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:07:15.045 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:07:15.045 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:07:15.146 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:07:15.200 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:07:15.203 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:07:15.204 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:07:15.242 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 61 16 85641 85625 87.7122 59 0.0174456 0.0113955 2023-06-03T20:07:15.305 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:07:15.343 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:07:15.345 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:07:15.346 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:07:15.446 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:07:15.483 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:07:15.488 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:07:15.489 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:07:15.557 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-06-03T20:07:15.590 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:07:15.632 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:07:15.637 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:07:15.637 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:07:15.738 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:07:15.783 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:07:15.787 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:07:15.787 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:07:15.888 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:07:15.928 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:07:15.933 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:07:15.934 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:07:16.028 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:07:15 smithi089 bash[75068]: cluster 1685822834.5799663 mgr.y (mgr.34104) 933 : cluster 0 pgmap v871: 89 pgs: 89 active+clean; 5.1 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 67 MiB/s wr, 1.07k op/s 2023-06-03T20:07:16.029 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:07:15 smithi089 bash[75068]: cluster 1685822834.8410943 mon.a (mon.0) 838 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-06-03T20:07:16.029 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:07:15 smithi089 bash[75068]: cluster 1685822834.8622818 osd.6 (osd.6) 94 : cluster 3 failed to encode map e165 with expected crc 2023-06-03T20:07:16.030 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:07:15 smithi089 bash[75068]: cluster 1685822834.8626757 osd.5 (osd.5) 93 : cluster 3 failed to encode map e165 with expected crc 2023-06-03T20:07:16.030 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:07:15 smithi089 bash[75068]: audit 1685822834.8660061 mon.a (mon.0) 839 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-06-03T20:07:16.030 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:07:15 smithi089 bash[75068]: cluster 1685822834.8661013 mon.a (mon.0) 840 : cluster 0 osdmap e165: 8 total, 8 up, 8 in 2023-06-03T20:07:16.030 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:07:15 smithi089 bash[75068]: cluster 1685822834.8720608 osd.7 (osd.7) 95 : cluster 3 failed to encode map e165 with expected crc 2023-06-03T20:07:16.031 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:07:15 smithi089 bash[78452]: cluster 1685822834.5799663 mgr.y (mgr.34104) 933 : cluster 0 pgmap v871: 89 pgs: 89 active+clean; 5.1 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 67 MiB/s wr, 1.07k op/s 2023-06-03T20:07:16.031 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:07:15 smithi089 bash[78452]: cluster 1685822834.8410943 mon.a (mon.0) 838 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-06-03T20:07:16.031 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:07:15 smithi089 bash[78452]: cluster 1685822834.8622818 osd.6 (osd.6) 94 : cluster 3 failed to encode map e165 with expected crc 2023-06-03T20:07:16.032 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:07:15 smithi089 bash[78452]: cluster 1685822834.8626757 osd.5 (osd.5) 93 : cluster 3 failed to encode map e165 with expected crc 2023-06-03T20:07:16.032 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:07:15 smithi089 bash[78452]: audit 1685822834.8660061 mon.a (mon.0) 839 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-06-03T20:07:16.032 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:07:15 smithi089 bash[78452]: cluster 1685822834.8661013 mon.a (mon.0) 840 : cluster 0 osdmap e165: 8 total, 8 up, 8 in 2023-06-03T20:07:16.033 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:07:15 smithi089 bash[78452]: cluster 1685822834.8720608 osd.7 (osd.7) 95 : cluster 3 failed to encode map e165 with expected crc 2023-06-03T20:07:16.035 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:07:16.065 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:07:16.072 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:07:16.073 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:07:16.174 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:07:16.187 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:07:15 smithi164 bash[59462]: cluster 1685822834.5799663 mgr.y (mgr.34104) 933 : cluster 0 pgmap v871: 89 pgs: 89 active+clean; 5.1 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 67 MiB/s wr, 1.07k op/s 2023-06-03T20:07:16.187 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:07:15 smithi164 bash[59462]: cluster 1685822834.8410943 mon.a (mon.0) 838 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-06-03T20:07:16.188 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:07:15 smithi164 bash[59462]: cluster 1685822834.8622818 osd.6 (osd.6) 94 : cluster 3 failed to encode map e165 with expected crc 2023-06-03T20:07:16.188 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:07:15 smithi164 bash[59462]: cluster 1685822834.8626757 osd.5 (osd.5) 93 : cluster 3 failed to encode map e165 with expected crc 2023-06-03T20:07:16.188 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:07:15 smithi164 bash[59462]: audit 1685822834.8660061 mon.a (mon.0) 839 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-06-03T20:07:16.188 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:07:15 smithi164 bash[59462]: cluster 1685822834.8661013 mon.a (mon.0) 840 : cluster 0 osdmap e165: 8 total, 8 up, 8 in 2023-06-03T20:07:16.188 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:07:15 smithi164 bash[59462]: cluster 1685822834.8720608 osd.7 (osd.7) 95 : cluster 3 failed to encode map e165 with expected crc 2023-06-03T20:07:16.212 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:07:16.219 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:07:16.219 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:07:16.242 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 62 16 86605 86589 87.2692 60.25 0.0238708 0.0114538 2023-06-03T20:07:16.321 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:07:16.369 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:07:16.371 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:07:16.372 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:07:16.473 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:07:16.481 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 20:07:16 smithi089 bash[71163]: ::ffff:172.21.15.164 - - [03/Jun/2023:20:07:16] "GET /metrics HTTP/1.1" 200 228720 "" "Prometheus/2.33.4" 2023-06-03T20:07:16.508 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:07:16.511 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:07:16.512 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:07:16.613 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:07:16.652 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:07:16.658 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:07:16.659 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:07:16.761 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:07:16.793 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:07:16.798 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:07:16.799 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:07:16.900 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:07:16.933 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:07:16.938 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:07:16.939 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:07:17.039 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:07:17.079 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:07:17.084 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:07:17.085 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:07:17.187 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:07:17.220 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:07:17.222 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:07:17.223 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:07:17.243 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 63 16 87575 87559 86.8462 60.625 0.0195407 0.0115089 2023-06-03T20:07:17.325 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:07:17.368 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:07:17.370 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:07:17.371 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:07:17.472 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:07:17.505 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:07:17.507 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:07:17.508 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:07:17.609 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:07:17.659 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:07:17.661 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:07:17.662 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:07:17.763 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:07:17.801 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:07:17.803 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:07:17.804 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:07:17.905 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:07:17.953 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:07:17.957 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:07:17.958 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:07:18.029 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:07:17 smithi089 bash[75068]: cluster 1685822836.5805185 mgr.y (mgr.34104) 934 : cluster 0 pgmap v873: 89 pgs: 89 active+clean; 5.1 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 61 MiB/s wr, 979 op/s 2023-06-03T20:07:18.029 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:07:17 smithi089 bash[78452]: cluster 1685822836.5805185 mgr.y (mgr.34104) 934 : cluster 0 pgmap v873: 89 pgs: 89 active+clean; 5.1 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 61 MiB/s wr, 979 op/s 2023-06-03T20:07:18.059 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:07:18.101 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:07:18.105 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:07:18.106 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:07:18.106 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:07:17 smithi164 bash[59462]: cluster 1685822836.5805185 mgr.y (mgr.34104) 934 : cluster 0 pgmap v873: 89 pgs: 89 active+clean; 5.1 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 61 MiB/s wr, 979 op/s 2023-06-03T20:07:18.207 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:07:18.242 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:07:18.243 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 64 16 88465 88449 86.3584 55.625 0.0252899 0.011574 2023-06-03T20:07:18.245 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:07:18.245 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:07:18.346 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:07:18.376 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:07:18.378 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:07:18.379 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:07:18.480 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:07:18.511 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:07:18.517 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:07:18.518 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:07:18.619 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:07:18.628 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:07:18 smithi089 bash[75068]: debug 2023-06-03T20:07:18.293+0000 7f0fee4d2700 -1 mon.a@0(leader).osd e165 definitely_dead 0 2023-06-03T20:07:18.652 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:07:18.658 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:07:18.659 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:07:18.760 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:07:18.800 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:07:18.804 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:07:18.805 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:07:18.836 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph osd unset noscrub 2023-06-03T20:07:18.902 INFO:teuthology.orchestra.run.smithi089.stderr:marked out osd.1. 2023-06-03T20:07:18.906 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:07:18.915 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:07:18 smithi089 bash[75068]: audit 1685822838.2941859 mon.b (mon.1) 53 : audit 1 from='client.? 172.21.15.89:0/2453224596' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["1"]}]: dispatch 2023-06-03T20:07:18.915 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:07:18 smithi089 bash[75068]: audit 1685822838.2971127 mon.a (mon.0) 841 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["1"]}]: dispatch 2023-06-03T20:07:18.915 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:07:18 smithi089 bash[75068]: cluster 1685822838.29721 mon.a (mon.0) 842 : cluster 1 Client client.admin marked osd.1 out, while it was still marked up 2023-06-03T20:07:18.916 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:07:18 smithi089 bash[78452]: audit 1685822838.2941859 mon.b (mon.1) 53 : audit 1 from='client.? 172.21.15.89:0/2453224596' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["1"]}]: dispatch 2023-06-03T20:07:18.916 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:07:18 smithi089 bash[78452]: audit 1685822838.2971127 mon.a (mon.0) 841 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["1"]}]: dispatch 2023-06-03T20:07:18.916 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:07:18 smithi089 bash[78452]: cluster 1685822838.29721 mon.a (mon.0) 842 : cluster 1 Client client.admin marked osd.1 out, while it was still marked up 2023-06-03T20:07:18.942 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:07:18.944 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:07:18.945 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:07:19.046 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:07:19.073 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:07:19.076 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:07:19.077 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:07:19.177 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:07:19.191 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:07:18 smithi164 bash[59462]: audit 1685822838.2941859 mon.b (mon.1) 53 : audit 1 from='client.? 172.21.15.89:0/2453224596' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["1"]}]: dispatch 2023-06-03T20:07:19.191 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:07:18 smithi164 bash[59462]: audit 1685822838.2971127 mon.a (mon.0) 841 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["1"]}]: dispatch 2023-06-03T20:07:19.191 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:07:18 smithi164 bash[59462]: cluster 1685822838.29721 mon.a (mon.0) 842 : cluster 1 Client client.admin marked osd.1 out, while it was still marked up 2023-06-03T20:07:19.214 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:07:19.217 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:07:19.218 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:07:19.242 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 65 16 89184 89168 85.7211 44.9375 0.00459548 0.0116133 2023-06-03T20:07:19.319 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:07:19.353 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:07:19.354 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:07:19.355 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:07:19.378 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/mon.a/config 2023-06-03T20:07:19.456 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:07:19.491 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:07:19.494 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:07:19.494 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:07:19.595 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:07:19.633 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:07:19.636 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:07:19.637 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:07:19.738 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:07:19.770 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:07:19.773 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:07:19.774 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:07:19.875 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:07:19.917 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:07:19.921 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:07:19.922 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:07:20.022 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:07:20.028 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:07:19 smithi089 bash[75068]: cluster 1685822838.5812564 mgr.y (mgr.34104) 935 : cluster 0 pgmap v874: 89 pgs: 89 active+clean; 5.3 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 60 MiB/s wr, 953 op/s 2023-06-03T20:07:20.028 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:07:19 smithi089 bash[75068]: cluster 1685822838.8687544 osd.6 (osd.6) 95 : cluster 3 failed to encode map e166 with expected crc 2023-06-03T20:07:20.028 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:07:19 smithi089 bash[75068]: cluster 1685822838.872617 osd.5 (osd.5) 94 : cluster 3 failed to encode map e166 with expected crc 2023-06-03T20:07:20.029 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:07:19 smithi089 bash[75068]: cluster 1685822838.8756752 osd.6 (osd.6) 96 : cluster 3 failed to encode map e166 with expected crc 2023-06-03T20:07:20.029 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:07:19 smithi089 bash[75068]: cluster 1685822838.8759997 osd.7 (osd.7) 96 : cluster 3 failed to encode map e166 with expected crc 2023-06-03T20:07:20.029 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:07:19 smithi089 bash[75068]: audit 1685822838.8789887 mon.a (mon.0) 843 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["1"]}]': finished 2023-06-03T20:07:20.029 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:07:19 smithi089 bash[75068]: cluster 1685822838.8792439 mon.a (mon.0) 844 : cluster 0 osdmap e166: 8 total, 8 up, 7 in 2023-06-03T20:07:20.029 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:07:19 smithi089 bash[75068]: cluster 1685822838.8798826 osd.7 (osd.7) 97 : cluster 3 failed to encode map e166 with expected crc 2023-06-03T20:07:20.029 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:07:19 smithi089 bash[75068]: cluster 1685822838.880022 osd.5 (osd.5) 95 : cluster 3 failed to encode map e166 with expected crc 2023-06-03T20:07:20.030 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:07:19 smithi089 bash[75068]: cluster 1685822838.8853834 osd.7 (osd.7) 98 : cluster 3 failed to encode map e166 with expected crc 2023-06-03T20:07:20.030 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:07:19 smithi089 bash[75068]: cluster 1685822838.8875778 osd.5 (osd.5) 96 : cluster 3 failed to encode map e166 with expected crc 2023-06-03T20:07:20.030 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:07:19 smithi089 bash[78452]: cluster 1685822838.5812564 mgr.y (mgr.34104) 935 : cluster 0 pgmap v874: 89 pgs: 89 active+clean; 5.3 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 60 MiB/s wr, 953 op/s 2023-06-03T20:07:20.030 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:07:19 smithi089 bash[78452]: cluster 1685822838.8687544 osd.6 (osd.6) 95 : cluster 3 failed to encode map e166 with expected crc 2023-06-03T20:07:20.030 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:07:19 smithi089 bash[78452]: cluster 1685822838.872617 osd.5 (osd.5) 94 : cluster 3 failed to encode map e166 with expected crc 2023-06-03T20:07:20.030 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:07:19 smithi089 bash[78452]: cluster 1685822838.8756752 osd.6 (osd.6) 96 : cluster 3 failed to encode map e166 with expected crc 2023-06-03T20:07:20.031 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:07:19 smithi089 bash[78452]: cluster 1685822838.8759997 osd.7 (osd.7) 96 : cluster 3 failed to encode map e166 with expected crc 2023-06-03T20:07:20.031 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:07:19 smithi089 bash[78452]: audit 1685822838.8789887 mon.a (mon.0) 843 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["1"]}]': finished 2023-06-03T20:07:20.031 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:07:19 smithi089 bash[78452]: cluster 1685822838.8792439 mon.a (mon.0) 844 : cluster 0 osdmap e166: 8 total, 8 up, 7 in 2023-06-03T20:07:20.031 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:07:19 smithi089 bash[78452]: cluster 1685822838.8798826 osd.7 (osd.7) 97 : cluster 3 failed to encode map e166 with expected crc 2023-06-03T20:07:20.031 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:07:19 smithi089 bash[78452]: cluster 1685822838.880022 osd.5 (osd.5) 95 : cluster 3 failed to encode map e166 with expected crc 2023-06-03T20:07:20.031 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:07:19 smithi089 bash[78452]: cluster 1685822838.8853834 osd.7 (osd.7) 98 : cluster 3 failed to encode map e166 with expected crc 2023-06-03T20:07:20.031 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:07:19 smithi089 bash[78452]: cluster 1685822838.8875778 osd.5 (osd.5) 96 : cluster 3 failed to encode map e166 with expected crc 2023-06-03T20:07:20.058 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:07:20.060 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:07:20.061 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:07:20.162 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:07:20.175 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:07:19 smithi164 bash[59462]: cluster 1685822838.5812564 mgr.y (mgr.34104) 935 : cluster 0 pgmap v874: 89 pgs: 89 active+clean; 5.3 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 60 MiB/s wr, 953 op/s 2023-06-03T20:07:20.176 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:07:19 smithi164 bash[59462]: cluster 1685822838.8687544 osd.6 (osd.6) 95 : cluster 3 failed to encode map e166 with expected crc 2023-06-03T20:07:20.176 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:07:19 smithi164 bash[59462]: cluster 1685822838.872617 osd.5 (osd.5) 94 : cluster 3 failed to encode map e166 with expected crc 2023-06-03T20:07:20.176 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:07:19 smithi164 bash[59462]: cluster 1685822838.8756752 osd.6 (osd.6) 96 : cluster 3 failed to encode map e166 with expected crc 2023-06-03T20:07:20.176 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:07:19 smithi164 bash[59462]: cluster 1685822838.8759997 osd.7 (osd.7) 96 : cluster 3 failed to encode map e166 with expected crc 2023-06-03T20:07:20.176 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:07:19 smithi164 bash[59462]: audit 1685822838.8789887 mon.a (mon.0) 843 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["1"]}]': finished 2023-06-03T20:07:20.176 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:07:19 smithi164 bash[59462]: cluster 1685822838.8792439 mon.a (mon.0) 844 : cluster 0 osdmap e166: 8 total, 8 up, 7 in 2023-06-03T20:07:20.177 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:07:19 smithi164 bash[59462]: cluster 1685822838.8798826 osd.7 (osd.7) 97 : cluster 3 failed to encode map e166 with expected crc 2023-06-03T20:07:20.177 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:07:19 smithi164 bash[59462]: cluster 1685822838.880022 osd.5 (osd.5) 95 : cluster 3 failed to encode map e166 with expected crc 2023-06-03T20:07:20.177 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:07:19 smithi164 bash[59462]: cluster 1685822838.8853834 osd.7 (osd.7) 98 : cluster 3 failed to encode map e166 with expected crc 2023-06-03T20:07:20.177 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:07:19 smithi164 bash[59462]: cluster 1685822838.8875778 osd.5 (osd.5) 96 : cluster 3 failed to encode map e166 with expected crc 2023-06-03T20:07:20.197 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:07:20.199 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:07:20.200 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:07:20.242 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 66 16 89184 89168 84.4224 0 - 0.0116133 2023-06-03T20:07:20.301 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:07:20.327 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:07:20.330 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:07:20.330 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:07:20.431 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:07:20.463 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:07:20.464 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:07:20.465 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:07:20.566 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:07:20.592 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:07:20.594 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:07:20.595 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:07:20.696 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:07:20.727 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:07:20.729 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:07:20.730 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:07:20.831 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:07:20.868 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:07:20.872 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:07:20.873 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:07:20.974 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:07:21.009 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:07:21.012 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:07:21.012 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:07:21.113 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:07:21.122 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:07:20 smithi164 bash[59462]: cluster 1685822839.884907 osd.7 (osd.7) 99 : cluster 3 failed to encode map e167 with expected crc 2023-06-03T20:07:21.122 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:07:20 smithi164 bash[59462]: cluster 1685822839.8861933 osd.5 (osd.5) 97 : cluster 3 failed to encode map e167 with expected crc 2023-06-03T20:07:21.122 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:07:20 smithi164 bash[59462]: cluster 1685822839.8869653 osd.6 (osd.6) 97 : cluster 3 failed to encode map e167 with expected crc 2023-06-03T20:07:21.123 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:07:20 smithi164 bash[59462]: cluster 1685822839.8872638 osd.7 (osd.7) 100 : cluster 3 failed to encode map e167 with expected crc 2023-06-03T20:07:21.123 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:07:20 smithi164 bash[59462]: cluster 1685822839.888163 osd.5 (osd.5) 98 : cluster 3 failed to encode map e167 with expected crc 2023-06-03T20:07:21.123 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:07:20 smithi164 bash[59462]: cluster 1685822839.8888624 osd.6 (osd.6) 98 : cluster 3 failed to encode map e167 with expected crc 2023-06-03T20:07:21.123 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:07:20 smithi164 bash[59462]: cluster 1685822839.8907695 osd.6 (osd.6) 99 : cluster 3 failed to encode map e167 with expected crc 2023-06-03T20:07:21.124 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:07:20 smithi164 bash[59462]: cluster 1685822839.8917792 osd.7 (osd.7) 101 : cluster 3 failed to encode map e167 with expected crc 2023-06-03T20:07:21.124 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:07:20 smithi164 bash[59462]: cluster 1685822839.8918421 mon.a (mon.0) 845 : cluster 0 osdmap e167: 8 total, 8 up, 7 in 2023-06-03T20:07:21.125 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:07:20 smithi164 bash[59462]: cluster 1685822839.89202 osd.5 (osd.5) 99 : cluster 3 failed to encode map e167 with expected crc 2023-06-03T20:07:21.125 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:07:20 smithi164 bash[59462]: cluster 1685822839.8922906 osd.6 (osd.6) 100 : cluster 3 failed to encode map e167 with expected crc 2023-06-03T20:07:21.125 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:07:20 smithi164 bash[59462]: cluster 1685822839.8941364 osd.7 (osd.7) 102 : cluster 3 failed to encode map e167 with expected crc 2023-06-03T20:07:21.143 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:07:21.147 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:07:21.148 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:07:21.242 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 67 16 89240 89224 83.2147 1.75 0.00655295 0.0118552 2023-06-03T20:07:21.248 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:07:21.255 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:07:20 smithi089 bash[75068]: cluster 1685822839.884907 osd.7 (osd.7) 99 : cluster 3 failed to encode map e167 with expected crc 2023-06-03T20:07:21.255 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:07:20 smithi089 bash[75068]: cluster 1685822839.8861933 osd.5 (osd.5) 97 : cluster 3 failed to encode map e167 with expected crc 2023-06-03T20:07:21.256 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:07:20 smithi089 bash[75068]: cluster 1685822839.8869653 osd.6 (osd.6) 97 : cluster 3 failed to encode map e167 with expected crc 2023-06-03T20:07:21.256 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:07:20 smithi089 bash[75068]: cluster 1685822839.8872638 osd.7 (osd.7) 100 : cluster 3 failed to encode map e167 with expected crc 2023-06-03T20:07:21.256 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:07:20 smithi089 bash[75068]: cluster 1685822839.888163 osd.5 (osd.5) 98 : cluster 3 failed to encode map e167 with expected crc 2023-06-03T20:07:21.256 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:07:20 smithi089 bash[75068]: cluster 1685822839.8888624 osd.6 (osd.6) 98 : cluster 3 failed to encode map e167 with expected crc 2023-06-03T20:07:21.256 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:07:20 smithi089 bash[75068]: cluster 1685822839.8907695 osd.6 (osd.6) 99 : cluster 3 failed to encode map e167 with expected crc 2023-06-03T20:07:21.256 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:07:20 smithi089 bash[75068]: cluster 1685822839.8917792 osd.7 (osd.7) 101 : cluster 3 failed to encode map e167 with expected crc 2023-06-03T20:07:21.256 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:07:20 smithi089 bash[75068]: cluster 1685822839.8918421 mon.a (mon.0) 845 : cluster 0 osdmap e167: 8 total, 8 up, 7 in 2023-06-03T20:07:21.257 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:07:20 smithi089 bash[75068]: cluster 1685822839.89202 osd.5 (osd.5) 99 : cluster 3 failed to encode map e167 with expected crc 2023-06-03T20:07:21.257 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:07:20 smithi089 bash[75068]: cluster 1685822839.8922906 osd.6 (osd.6) 100 : cluster 3 failed to encode map e167 with expected crc 2023-06-03T20:07:21.257 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:07:20 smithi089 bash[75068]: cluster 1685822839.8941364 osd.7 (osd.7) 102 : cluster 3 failed to encode map e167 with expected crc 2023-06-03T20:07:21.257 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:07:20 smithi089 bash[78452]: cluster 1685822839.884907 osd.7 (osd.7) 99 : cluster 3 failed to encode map e167 with expected crc 2023-06-03T20:07:21.257 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:07:20 smithi089 bash[78452]: cluster 1685822839.8861933 osd.5 (osd.5) 97 : cluster 3 failed to encode map e167 with expected crc 2023-06-03T20:07:21.258 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:07:20 smithi089 bash[78452]: cluster 1685822839.8869653 osd.6 (osd.6) 97 : cluster 3 failed to encode map e167 with expected crc 2023-06-03T20:07:21.258 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:07:20 smithi089 bash[78452]: cluster 1685822839.8872638 osd.7 (osd.7) 100 : cluster 3 failed to encode map e167 with expected crc 2023-06-03T20:07:21.258 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:07:20 smithi089 bash[78452]: cluster 1685822839.888163 osd.5 (osd.5) 98 : cluster 3 failed to encode map e167 with expected crc 2023-06-03T20:07:21.258 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:07:20 smithi089 bash[78452]: cluster 1685822839.8888624 osd.6 (osd.6) 98 : cluster 3 failed to encode map e167 with expected crc 2023-06-03T20:07:21.258 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:07:20 smithi089 bash[78452]: cluster 1685822839.8907695 osd.6 (osd.6) 99 : cluster 3 failed to encode map e167 with expected crc 2023-06-03T20:07:21.258 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:07:20 smithi089 bash[78452]: cluster 1685822839.8917792 osd.7 (osd.7) 101 : cluster 3 failed to encode map e167 with expected crc 2023-06-03T20:07:21.258 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:07:20 smithi089 bash[78452]: cluster 1685822839.8918421 mon.a (mon.0) 845 : cluster 0 osdmap e167: 8 total, 8 up, 7 in 2023-06-03T20:07:21.258 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:07:20 smithi089 bash[78452]: cluster 1685822839.89202 osd.5 (osd.5) 99 : cluster 3 failed to encode map e167 with expected crc 2023-06-03T20:07:21.259 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:07:20 smithi089 bash[78452]: cluster 1685822839.8922906 osd.6 (osd.6) 100 : cluster 3 failed to encode map e167 with expected crc 2023-06-03T20:07:21.259 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:07:20 smithi089 bash[78452]: cluster 1685822839.8941364 osd.7 (osd.7) 102 : cluster 3 failed to encode map e167 with expected crc 2023-06-03T20:07:21.272 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:07:21.274 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:07:21.275 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:07:21.376 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:07:21.405 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:07:21.408 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:07:21.409 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:07:21.434 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/mon.a/config 2023-06-03T20:07:21.510 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:07:21.540 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:07:21.542 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:07:21.543 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:07:21.644 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:07:21.684 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:07:21.689 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:07:21.690 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:07:21.791 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:07:21.812 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:07:21.813 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:07:21.813 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:07:21.914 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:07:21.946 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:07:21.951 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:07:21.951 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:07:22.052 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:07:22.083 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:07:21 smithi164 bash[59462]: cluster 1685822840.581923 mgr.y (mgr.34104) 936 : cluster 0 pgmap v877: 89 pgs: 89 active+clean; 5.4 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 58 MiB/s wr, 934 op/s; 0 B/s, 0 objects/s recovering 2023-06-03T20:07:22.084 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:07:21 smithi164 bash[59462]: cluster 1685822840.88977 osd.6 (osd.6) 101 : cluster 3 failed to encode map e168 with expected crc 2023-06-03T20:07:22.084 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:07:21 smithi164 bash[59462]: cluster 1685822840.8953135 osd.6 (osd.6) 102 : cluster 3 failed to encode map e168 with expected crc 2023-06-03T20:07:22.084 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:07:21 smithi164 bash[59462]: cluster 1685822840.8955426 mon.a (mon.0) 846 : cluster 0 osdmap e168: 8 total, 8 up, 7 in 2023-06-03T20:07:22.084 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:07:21 smithi164 bash[59462]: cluster 1685822840.89659 osd.6 (osd.6) 103 : cluster 3 failed to encode map e168 with expected crc 2023-06-03T20:07:22.085 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:07:22.087 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:07:21 smithi089 bash[78452]: cluster 1685822840.581923 mgr.y (mgr.34104) 936 : cluster 0 pgmap v877: 89 pgs: 89 active+clean; 5.4 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 58 MiB/s wr, 934 op/s; 0 B/s, 0 objects/s recovering 2023-06-03T20:07:22.087 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:07:21 smithi089 bash[78452]: cluster 1685822840.88977 osd.6 (osd.6) 101 : cluster 3 failed to encode map e168 with expected crc 2023-06-03T20:07:22.087 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:07:21 smithi089 bash[78452]: cluster 1685822840.8953135 osd.6 (osd.6) 102 : cluster 3 failed to encode map e168 with expected crc 2023-06-03T20:07:22.087 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:07:21 smithi089 bash[78452]: cluster 1685822840.8955426 mon.a (mon.0) 846 : cluster 0 osdmap e168: 8 total, 8 up, 7 in 2023-06-03T20:07:22.088 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:07:21 smithi089 bash[78452]: cluster 1685822840.89659 osd.6 (osd.6) 103 : cluster 3 failed to encode map e168 with expected crc 2023-06-03T20:07:22.088 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:07:21 smithi089 bash[75068]: cluster 1685822840.581923 mgr.y (mgr.34104) 936 : cluster 0 pgmap v877: 89 pgs: 89 active+clean; 5.4 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 58 MiB/s wr, 934 op/s; 0 B/s, 0 objects/s recovering 2023-06-03T20:07:22.088 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:07:21 smithi089 bash[75068]: cluster 1685822840.88977 osd.6 (osd.6) 101 : cluster 3 failed to encode map e168 with expected crc 2023-06-03T20:07:22.088 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:07:21 smithi089 bash[75068]: cluster 1685822840.8953135 osd.6 (osd.6) 102 : cluster 3 failed to encode map e168 with expected crc 2023-06-03T20:07:22.088 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:07:21 smithi089 bash[75068]: cluster 1685822840.8955426 mon.a (mon.0) 846 : cluster 0 osdmap e168: 8 total, 8 up, 7 in 2023-06-03T20:07:22.089 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:07:21 smithi089 bash[75068]: cluster 1685822840.89659 osd.6 (osd.6) 103 : cluster 3 failed to encode map e168 with expected crc 2023-06-03T20:07:22.089 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:07:22.089 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:07:22.190 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:07:22.226 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:07:22.230 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:07:22.230 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:07:22.243 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 68 16 89872 89856 82.5719 39.5 0.0166855 0.0121051 2023-06-03T20:07:22.331 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:07:22.363 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:07:22.365 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:07:22.366 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:07:22.467 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:07:22.507 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:07:22.513 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:07:22.513 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:07:22.614 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:07:22.644 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:07:22.648 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:07:22.649 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:07:22.750 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:07:22.780 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:07:22.782 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:07:22.783 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:07:22.883 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:07:22.909 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:07:22 smithi089 bash[75068]: audit 1685822842.0887604 mon.c (mon.2) 448 : audit 1 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.18", "id": [3, 2]}]: dispatch 2023-06-03T20:07:22.909 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:07:22 smithi089 bash[75068]: audit 1685822842.0893471 mon.c (mon.2) 449 : audit 1 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1d", "id": [3, 5]}]: dispatch 2023-06-03T20:07:22.909 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:07:22 smithi089 bash[75068]: audit 1685822842.0896637 mon.a (mon.0) 847 : audit 1 from='mgr.34104 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.18", "id": [3, 2]}]: dispatch 2023-06-03T20:07:22.910 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:07:22 smithi089 bash[75068]: audit 1685822842.0902803 mon.a (mon.0) 848 : audit 1 from='mgr.34104 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1d", "id": [3, 5]}]: dispatch 2023-06-03T20:07:22.910 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:07:22 smithi089 bash[75068]: audit 1685822842.1096156 mon.c (mon.2) 450 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T20:07:22.949 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:07:22.953 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:07:22.953 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:07:23.054 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:07:23.083 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:07:23.086 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:07:23.086 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:07:23.187 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:07:23.198 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:07:22 smithi164 bash[59462]: audit 1685822842.0887604 mon.c (mon.2) 448 : audit 1 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.18", "id": [3, 2]}]: dispatch 2023-06-03T20:07:23.198 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:07:22 smithi164 bash[59462]: audit 1685822842.0893471 mon.c (mon.2) 449 : audit 1 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1d", "id": [3, 5]}]: dispatch 2023-06-03T20:07:23.199 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:07:22 smithi164 bash[59462]: audit 1685822842.0896637 mon.a (mon.0) 847 : audit 1 from='mgr.34104 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.18", "id": [3, 2]}]: dispatch 2023-06-03T20:07:23.199 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:07:22 smithi164 bash[59462]: audit 1685822842.0902803 mon.a (mon.0) 848 : audit 1 from='mgr.34104 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1d", "id": [3, 5]}]: dispatch 2023-06-03T20:07:23.199 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:07:22 smithi164 bash[59462]: audit 1685822842.1096156 mon.c (mon.2) 450 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T20:07:23.223 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:07:23.230 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:07:23.231 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:07:23.243 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 69 16 90594 90578 82.0292 45.125 0.0413217 0.0121802 2023-06-03T20:07:23.278 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:07:22 smithi089 bash[78452]: audit 1685822842.0887604 mon.c (mon.2) 448 : audit 1 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.18", "id": [3, 2]}]: dispatch 2023-06-03T20:07:23.279 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:07:22 smithi089 bash[78452]: audit 1685822842.0893471 mon.c (mon.2) 449 : audit 1 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1d", "id": [3, 5]}]: dispatch 2023-06-03T20:07:23.279 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:07:22 smithi089 bash[78452]: audit 1685822842.0896637 mon.a (mon.0) 847 : audit 1 from='mgr.34104 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.18", "id": [3, 2]}]: dispatch 2023-06-03T20:07:23.279 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:07:22 smithi089 bash[78452]: audit 1685822842.0902803 mon.a (mon.0) 848 : audit 1 from='mgr.34104 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1d", "id": [3, 5]}]: dispatch 2023-06-03T20:07:23.279 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:07:22 smithi089 bash[78452]: audit 1685822842.1096156 mon.c (mon.2) 450 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T20:07:23.332 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:07:23.370 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:07:23.376 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:07:23.377 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:07:23.478 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:07:23.492 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/mon.a/config 2023-06-03T20:07:23.513 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:07:23.517 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:07:23.518 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:07:23.619 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:07:23.657 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:07:23.662 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:07:23.662 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:07:23.763 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:07:23.799 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:07:23.808 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:07:23.809 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:07:23.910 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:07:23.912 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:07:23 smithi164 bash[59462]: cluster 1685822842.5825207 mgr.y (mgr.34104) 937 : cluster 0 pgmap v879: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 88 active+clean; 5.4 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 50 MiB/s wr, 801 op/s; 5703/266607 objects degraded (2.139%); 0 B/s, 0 objects/s recovering 2023-06-03T20:07:23.913 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:07:23 smithi164 bash[59462]: cluster 1685822842.8913121 mon.a (mon.0) 849 : cluster 3 Health check failed: Degraded data redundancy: 5703/266607 objects degraded (2.139%), 1 pg degraded (PG_DEGRADED) 2023-06-03T20:07:23.913 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:07:23 smithi164 bash[59462]: cluster 1685822842.89394 osd.6 (osd.6) 104 : cluster 3 failed to encode map e169 with expected crc 2023-06-03T20:07:23.913 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:07:23 smithi164 bash[59462]: audit 1685822842.8953862 mon.a (mon.0) 850 : audit 1 from='mgr.34104 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.18", "id": [3, 2]}]': finished 2023-06-03T20:07:23.914 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:07:23 smithi164 bash[59462]: audit 1685822842.895444 mon.a (mon.0) 851 : audit 1 from='mgr.34104 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1d", "id": [3, 5]}]': finished 2023-06-03T20:07:23.914 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:07:23 smithi164 bash[59462]: cluster 1685822842.8955114 mon.a (mon.0) 852 : cluster 0 osdmap e169: 8 total, 8 up, 7 in 2023-06-03T20:07:23.958 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:07:23.961 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:07:23.962 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:07:24.063 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:07:24.103 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:07:24.105 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:07:24.106 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:07:24.207 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:07:24.216 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:07:23 smithi089 bash[78452]: cluster 1685822842.5825207 mgr.y (mgr.34104) 937 : cluster 0 pgmap v879: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 88 active+clean; 5.4 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 50 MiB/s wr, 801 op/s; 5703/266607 objects degraded (2.139%); 0 B/s, 0 objects/s recovering 2023-06-03T20:07:24.216 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:07:23 smithi089 bash[78452]: cluster 1685822842.8913121 mon.a (mon.0) 849 : cluster 3 Health check failed: Degraded data redundancy: 5703/266607 objects degraded (2.139%), 1 pg degraded (PG_DEGRADED) 2023-06-03T20:07:24.216 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:07:23 smithi089 bash[78452]: cluster 1685822842.89394 osd.6 (osd.6) 104 : cluster 3 failed to encode map e169 with expected crc 2023-06-03T20:07:24.216 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:07:23 smithi089 bash[78452]: audit 1685822842.8953862 mon.a (mon.0) 850 : audit 1 from='mgr.34104 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.18", "id": [3, 2]}]': finished 2023-06-03T20:07:24.217 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:07:23 smithi089 bash[78452]: audit 1685822842.895444 mon.a (mon.0) 851 : audit 1 from='mgr.34104 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1d", "id": [3, 5]}]': finished 2023-06-03T20:07:24.217 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:07:23 smithi089 bash[78452]: cluster 1685822842.8955114 mon.a (mon.0) 852 : cluster 0 osdmap e169: 8 total, 8 up, 7 in 2023-06-03T20:07:24.217 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:07:23 smithi089 bash[75068]: cluster 1685822842.5825207 mgr.y (mgr.34104) 937 : cluster 0 pgmap v879: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 88 active+clean; 5.4 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 50 MiB/s wr, 801 op/s; 5703/266607 objects degraded (2.139%); 0 B/s, 0 objects/s recovering 2023-06-03T20:07:24.217 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:07:23 smithi089 bash[75068]: cluster 1685822842.8913121 mon.a (mon.0) 849 : cluster 3 Health check failed: Degraded data redundancy: 5703/266607 objects degraded (2.139%), 1 pg degraded (PG_DEGRADED) 2023-06-03T20:07:24.217 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:07:23 smithi089 bash[75068]: cluster 1685822842.89394 osd.6 (osd.6) 104 : cluster 3 failed to encode map e169 with expected crc 2023-06-03T20:07:24.218 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:07:23 smithi089 bash[75068]: audit 1685822842.8953862 mon.a (mon.0) 850 : audit 1 from='mgr.34104 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.18", "id": [3, 2]}]': finished 2023-06-03T20:07:24.218 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:07:23 smithi089 bash[75068]: audit 1685822842.895444 mon.a (mon.0) 851 : audit 1 from='mgr.34104 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1d", "id": [3, 5]}]': finished 2023-06-03T20:07:24.218 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:07:23 smithi089 bash[75068]: cluster 1685822842.8955114 mon.a (mon.0) 852 : cluster 0 osdmap e169: 8 total, 8 up, 7 in 2023-06-03T20:07:24.243 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 70 16 91293 91277 81.4814 43.6875 0.0438603 0.0122646 2023-06-03T20:07:24.249 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:07:24.250 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:07:24.251 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:07:24.352 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:07:24.381 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:07:24.383 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:07:24.384 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:07:24.485 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:07:24.515 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:07:24.517 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:07:24.518 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:07:24.619 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:07:24.653 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:07:24.655 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:07:24.655 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:07:24.757 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:07:24.798 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:07:24.802 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:07:24.803 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:07:24.904 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:07:24.943 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:07:24.946 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:07:24.947 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:07:25.028 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:07:24 smithi089 bash[75068]: cluster 1685822843.9132476 osd.6 (osd.6) 105 : cluster 3 failed to encode map e170 with expected crc 2023-06-03T20:07:25.029 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:07:24 smithi089 bash[75068]: cluster 1685822843.916418 osd.5 (osd.5) 100 : cluster 3 failed to encode map e170 with expected crc 2023-06-03T20:07:25.029 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:07:24 smithi089 bash[75068]: cluster 1685822843.917886 mon.a (mon.0) 853 : cluster 0 osdmap e170: 8 total, 8 up, 7 in 2023-06-03T20:07:25.029 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:07:24 smithi089 bash[75068]: cluster 1685822843.9188192 osd.5 (osd.5) 101 : cluster 3 failed to encode map e170 with expected crc 2023-06-03T20:07:25.029 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:07:24 smithi089 bash[75068]: cluster 1685822843.9250202 osd.7 (osd.7) 103 : cluster 3 failed to encode map e170 with expected crc 2023-06-03T20:07:25.029 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:07:24 smithi089 bash[75068]: cluster 1685822843.9310374 osd.7 (osd.7) 104 : cluster 3 failed to encode map e170 with expected crc 2023-06-03T20:07:25.029 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:07:24 smithi089 bash[78452]: cluster 1685822843.9132476 osd.6 (osd.6) 105 : cluster 3 failed to encode map e170 with expected crc 2023-06-03T20:07:25.030 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:07:24 smithi089 bash[78452]: cluster 1685822843.916418 osd.5 (osd.5) 100 : cluster 3 failed to encode map e170 with expected crc 2023-06-03T20:07:25.030 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:07:24 smithi089 bash[78452]: cluster 1685822843.917886 mon.a (mon.0) 853 : cluster 0 osdmap e170: 8 total, 8 up, 7 in 2023-06-03T20:07:25.030 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:07:24 smithi089 bash[78452]: cluster 1685822843.9188192 osd.5 (osd.5) 101 : cluster 3 failed to encode map e170 with expected crc 2023-06-03T20:07:25.030 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:07:24 smithi089 bash[78452]: cluster 1685822843.9250202 osd.7 (osd.7) 103 : cluster 3 failed to encode map e170 with expected crc 2023-06-03T20:07:25.030 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:07:24 smithi089 bash[78452]: cluster 1685822843.9310374 osd.7 (osd.7) 104 : cluster 3 failed to encode map e170 with expected crc 2023-06-03T20:07:25.047 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:07:25.059 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:07:24 smithi164 bash[59462]: cluster 1685822843.9132476 osd.6 (osd.6) 105 : cluster 3 failed to encode map e170 with expected crc 2023-06-03T20:07:25.059 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:07:24 smithi164 bash[59462]: cluster 1685822843.916418 osd.5 (osd.5) 100 : cluster 3 failed to encode map e170 with expected crc 2023-06-03T20:07:25.060 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:07:24 smithi164 bash[59462]: cluster 1685822843.917886 mon.a (mon.0) 853 : cluster 0 osdmap e170: 8 total, 8 up, 7 in 2023-06-03T20:07:25.060 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:07:24 smithi164 bash[59462]: cluster 1685822843.9188192 osd.5 (osd.5) 101 : cluster 3 failed to encode map e170 with expected crc 2023-06-03T20:07:25.060 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:07:24 smithi164 bash[59462]: cluster 1685822843.9250202 osd.7 (osd.7) 103 : cluster 3 failed to encode map e170 with expected crc 2023-06-03T20:07:25.060 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:07:24 smithi164 bash[59462]: cluster 1685822843.9310374 osd.7 (osd.7) 104 : cluster 3 failed to encode map e170 with expected crc 2023-06-03T20:07:25.087 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:07:25.091 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:07:25.092 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:07:25.193 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:07:25.228 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:07:25.230 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:07:25.230 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:07:25.243 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 71 16 92017 92001 80.9711 45.25 0.0166879 0.0123437 2023-06-03T20:07:25.331 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:07:25.368 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:07:25.371 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:07:25.372 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:07:25.473 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:07:25.511 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:07:25.514 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:07:25.515 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:07:25.616 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:07:25.661 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:07:25.664 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:07:25.665 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:07:25.765 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:07:25.794 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:07:25.796 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:07:25.797 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:07:25.899 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:07:25.926 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:07:25.929 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:07:25.930 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:07:25.931 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:07:25 smithi164 bash[59462]: cluster 1685822844.5829902 mgr.y (mgr.34104) 938 : cluster 0 pgmap v882: 89 pgs: 1 active+recovering+undersized+remapped, 1 peering, 2 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 5.5 GiB data, 17 GiB used, 608 GiB / 626 GiB avail; 34 MiB/s wr, 537 op/s; 11312/270144 objects degraded (4.187%); 5408/270144 objects misplaced (2.002%); 491 KiB/s, 8 objects/s recovering 2023-06-03T20:07:25.940 INFO:tasks.thrashosds.thrasher:in_osds: [0, 2, 3, 4, 5, 6, 7] out_osds: [1] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-06-03T20:07:25.940 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-06-03T20:07:25.941 INFO:tasks.thrashosds.thrasher:primary_affinity 2023-06-03T20:07:25.941 INFO:tasks.thrashosds.thrasher:Setting osd 7 primary_affinity to 1.000000 2023-06-03T20:07:25.941 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph osd primary-affinity 7 1 2023-06-03T20:07:26.028 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:07:25 smithi089 bash[78452]: cluster 1685822844.5829902 mgr.y (mgr.34104) 938 : cluster 0 pgmap v882: 89 pgs: 1 active+recovering+undersized+remapped, 1 peering, 2 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 5.5 GiB data, 17 GiB used, 608 GiB / 626 GiB avail; 34 MiB/s wr, 537 op/s; 11312/270144 objects degraded (4.187%); 5408/270144 objects misplaced (2.002%); 491 KiB/s, 8 objects/s recovering 2023-06-03T20:07:26.029 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:07:25 smithi089 bash[75068]: cluster 1685822844.5829902 mgr.y (mgr.34104) 938 : cluster 0 pgmap v882: 89 pgs: 1 active+recovering+undersized+remapped, 1 peering, 2 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 5.5 GiB data, 17 GiB used, 608 GiB / 626 GiB avail; 34 MiB/s wr, 537 op/s; 11312/270144 objects degraded (4.187%); 5408/270144 objects misplaced (2.002%); 491 KiB/s, 8 objects/s recovering 2023-06-03T20:07:26.031 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:07:26.062 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:07:26.065 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:07:26.065 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:07:26.166 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:07:26.196 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:07:26.200 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:07:26.201 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:07:26.244 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 72 16 92734 92718 80.4689 44.8125 0.00720887 0.0124201 2023-06-03T20:07:26.302 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:07:26.334 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:07:26.336 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:07:26.336 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:07:26.437 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:07:26.467 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:07:26.469 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:07:26.469 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:07:26.528 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 20:07:26 smithi089 bash[71163]: ::ffff:172.21.15.164 - - [03/Jun/2023:20:07:26] "GET /metrics HTTP/1.1" 200 228742 "" "Prometheus/2.33.4" 2023-06-03T20:07:26.571 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:07:26.611 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:07:26.615 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:07:26.616 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:07:26.717 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:07:26.759 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:07:26.764 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:07:26.765 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:07:26.866 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:07:26.912 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:07:26.918 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:07:26.919 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:07:27.020 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:07:27.050 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:07:27.053 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:07:27.054 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:07:27.155 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:07:27.191 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:07:27.193 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:07:27.194 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:07:27.243 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 73 16 93424 93408 79.9573 43.125 0.0175311 0.0125 2023-06-03T20:07:27.294 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:07:27.327 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:07:27.331 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:07:27.332 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:07:27.433 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:07:27.470 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:07:27.474 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:07:27.475 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:07:27.576 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:07:27.618 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:07:27.624 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:07:27.625 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:07:27.726 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:07:27.766 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:07:27.769 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:07:27.770 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:07:27.871 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:07:27.894 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:07:27.897 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:07:27.897 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:07:27.998 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:07:28.045 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:07:28.055 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:07:28.056 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:07:28.121 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:07:27 smithi089 bash[75068]: cluster 1685822846.5836594 mgr.y (mgr.34104) 939 : cluster 0 pgmap v883: 89 pgs: 1 active+recovering+undersized+remapped, 1 peering, 2 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 5.5 GiB data, 17 GiB used, 608 GiB / 626 GiB avail; 25 MiB/s wr, 394 op/s; 11312/272343 objects degraded (4.154%); 5408/272343 objects misplaced (1.986%); 384 KiB/s, 6 objects/s recovering 2023-06-03T20:07:28.122 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:07:27 smithi089 bash[78452]: cluster 1685822846.5836594 mgr.y (mgr.34104) 939 : cluster 0 pgmap v883: 89 pgs: 1 active+recovering+undersized+remapped, 1 peering, 2 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 5.5 GiB data, 17 GiB used, 608 GiB / 626 GiB avail; 25 MiB/s wr, 394 op/s; 11312/272343 objects degraded (4.154%); 5408/272343 objects misplaced (1.986%); 384 KiB/s, 6 objects/s recovering 2023-06-03T20:07:28.157 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:07:28.192 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:07:28.195 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:07:28.196 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:07:28.243 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 74 16 93980 93964 79.3464 34.75 0.0185599 0.0125963 2023-06-03T20:07:28.297 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:07:28.326 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:07:28.328 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:07:28.329 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:07:28.333 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:07:27 smithi164 bash[59462]: cluster 1685822846.5836594 mgr.y (mgr.34104) 939 : cluster 0 pgmap v883: 89 pgs: 1 active+recovering+undersized+remapped, 1 peering, 2 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 5.5 GiB data, 17 GiB used, 608 GiB / 626 GiB avail; 25 MiB/s wr, 394 op/s; 11312/272343 objects degraded (4.154%); 5408/272343 objects misplaced (1.986%); 384 KiB/s, 6 objects/s recovering 2023-06-03T20:07:28.430 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:07:28.473 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:07:28.478 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:07:28.479 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:07:28.580 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:07:28.611 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:07:28.613 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:07:28.614 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:07:28.715 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:07:28.749 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:07:28.752 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:07:28.753 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:07:28.855 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:07:28.894 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:07:28.897 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:07:28.898 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:07:28.988 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:07:28 smithi089 bash[75068]: cluster 1685822848.9295824 mon.a (mon.0) 854 : cluster 3 Health check update: Degraded data redundancy: 11312/279927 objects degraded (4.041%), 2 pgs degraded (PG_DEGRADED) 2023-06-03T20:07:28.999 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:07:29.031 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:07:29.035 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:07:29.035 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:07:29.136 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:07:29.145 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:07:28 smithi164 bash[59462]: cluster 1685822848.9295824 mon.a (mon.0) 854 : cluster 3 Health check update: Degraded data redundancy: 11312/279927 objects degraded (4.041%), 2 pgs degraded (PG_DEGRADED) 2023-06-03T20:07:29.164 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:07:29.166 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:07:29.167 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:07:29.243 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 75 16 94641 94625 78.8393 41.3125 0.00501342 0.0126761 2023-06-03T20:07:29.268 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:07:29.282 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:07:28 smithi089 bash[78452]: cluster 1685822848.9295824 mon.a (mon.0) 854 : cluster 3 Health check update: Degraded data redundancy: 11312/279927 objects degraded (4.041%), 2 pgs degraded (PG_DEGRADED) 2023-06-03T20:07:29.304 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:07:29.306 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:07:29.307 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:07:29.407 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:07:29.440 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:07:29.448 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:07:29.449 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:07:29.550 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:07:29.581 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:07:29.587 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:07:29.588 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:07:29.689 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:07:29.721 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:07:29.725 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:07:29.726 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:07:29.827 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:07:29.858 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:07:29.863 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:07:29.863 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:07:29.964 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:07:30.013 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:07:30.017 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:07:30.018 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:07:30.119 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:07:30.193 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:07:30.198 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:07:30.198 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:07:30.244 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 76 16 95388 95372 78.4163 46.6875 0.0646338 0.0127457 2023-06-03T20:07:30.278 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:07:30 smithi089 bash[78452]: cluster 1685822848.5844202 mgr.y (mgr.34104) 940 : cluster 0 pgmap v884: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 5.7 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 40 MiB/s wr, 636 op/s; 11312/279927 objects degraded (4.041%); 5329/279927 objects misplaced (1.904%); 957 KiB/s, 15 objects/s recovering 2023-06-03T20:07:30.279 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:07:29 smithi089 bash[75068]: cluster 1685822848.5844202 mgr.y (mgr.34104) 940 : cluster 0 pgmap v884: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 5.7 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 40 MiB/s wr, 636 op/s; 11312/279927 objects degraded (4.041%); 5329/279927 objects misplaced (1.904%); 957 KiB/s, 15 objects/s recovering 2023-06-03T20:07:30.299 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:07:30.333 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:07:29 smithi164 bash[59462]: cluster 1685822848.5844202 mgr.y (mgr.34104) 940 : cluster 0 pgmap v884: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 5.7 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 40 MiB/s wr, 636 op/s; 11312/279927 objects degraded (4.041%); 5329/279927 objects misplaced (1.904%); 957 KiB/s, 15 objects/s recovering 2023-06-03T20:07:30.334 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:07:30.338 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:07:30.339 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:07:30.440 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:07:30.470 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:07:30.472 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:07:30.473 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:07:30.574 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:07:30.604 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:07:30.607 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:07:30.607 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:07:30.708 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:07:30.762 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:07:30.765 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:07:30.766 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:07:30.867 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:07:30.903 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:07:30.905 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:07:30.906 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:07:31.007 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:07:31.037 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:07:31.042 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:07:31.043 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:07:31.144 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:07:31.169 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:07:31.171 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:07:31.172 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:07:31.244 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 77 16 96126 96110 77.9969 46.125 0.0107317 0.0128137 2023-06-03T20:07:31.272 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:07:31.304 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:07:31.305 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:07:31.306 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:07:31.407 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:07:31.438 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:07:31.445 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:07:31.446 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:07:31.547 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:07:31.573 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:07:31.576 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:07:31.576 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:07:31.677 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:07:31.728 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:07:31.733 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:07:31.734 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:07:31.835 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:07:31.884 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:07:31.887 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:07:31.888 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:07:31.989 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:07:32.046 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:07:32.050 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:07:32.050 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:07:32.151 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:07:32.191 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:07:32.194 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:07:32.195 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:07:32.244 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 78 16 96862 96846 77.5866 46 0.0531694 0.0128826 2023-06-03T20:07:32.278 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:07:31 smithi089 bash[78452]: cluster 1685822850.5850549 mgr.y (mgr.34104) 941 : cluster 0 pgmap v885: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 5.7 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 41 MiB/s wr, 662 op/s; 11312/282516 objects degraded (4.004%); 5329/282516 objects misplaced (1.886%); 920 KiB/s, 14 objects/s recovering 2023-06-03T20:07:32.279 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:07:31 smithi089 bash[75068]: cluster 1685822850.5850549 mgr.y (mgr.34104) 941 : cluster 0 pgmap v885: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 5.7 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 41 MiB/s wr, 662 op/s; 11312/282516 objects degraded (4.004%); 5329/282516 objects misplaced (1.886%); 920 KiB/s, 14 objects/s recovering 2023-06-03T20:07:32.296 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:07:32.305 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:07:31 smithi164 bash[59462]: cluster 1685822850.5850549 mgr.y (mgr.34104) 941 : cluster 0 pgmap v885: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 5.7 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 41 MiB/s wr, 662 op/s; 11312/282516 objects degraded (4.004%); 5329/282516 objects misplaced (1.886%); 920 KiB/s, 14 objects/s recovering 2023-06-03T20:07:32.338 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:07:32.341 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:07:32.342 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:07:32.443 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:07:32.471 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:07:32.474 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:07:32.475 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:07:32.576 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:07:32.615 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:07:32.618 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:07:32.619 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:07:32.720 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:07:32.747 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:07:32.751 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:07:32.752 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:07:32.853 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:07:32.890 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:07:32.893 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:07:32.893 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:07:32.994 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:07:33.022 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:07:33.030 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:07:33.031 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:07:33.132 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:07:33.180 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:07:33.183 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:07:33.184 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:07:33.244 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 79 16 97556 97540 77.1536 43.375 0.0100036 0.0129548 2023-06-03T20:07:33.284 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:07:33.323 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:07:33.330 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:07:33.330 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:07:33.431 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:07:33.470 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:07:33.473 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:07:33.474 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:07:33.492 INFO:teuthology.orchestra.run.smithi089.stdout:{ 2023-06-03T20:07:33.493 INFO:teuthology.orchestra.run.smithi089.stdout: "size": 20, 2023-06-03T20:07:33.493 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 600, 2023-06-03T20:07:33.493 INFO:teuthology.orchestra.run.smithi089.stdout: "ops": [ 2023-06-03T20:07:33.493 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:07:33.493 INFO:teuthology.orchestra.run.smithi089.stdout: "description": "osd_op(client.45713.0:8914 9.9 9:9de55760:::benchmark_data_smithi089_115580_object8913:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e131)", 2023-06-03T20:07:33.493 INFO:teuthology.orchestra.run.smithi089.stdout: "initiated_at": "2023-06-03T20:00:44.760429+0000", 2023-06-03T20:07:33.494 INFO:teuthology.orchestra.run.smithi089.stdout: "age": 408.72305771399999, 2023-06-03T20:07:33.494 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.37586828, 2023-06-03T20:07:33.494 INFO:teuthology.orchestra.run.smithi089.stdout: "type_data": { 2023-06-03T20:07:33.494 INFO:teuthology.orchestra.run.smithi089.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-03T20:07:33.494 INFO:teuthology.orchestra.run.smithi089.stdout: "client_info": { 2023-06-03T20:07:33.494 INFO:teuthology.orchestra.run.smithi089.stdout: "client": "client.45713", 2023-06-03T20:07:33.495 INFO:teuthology.orchestra.run.smithi089.stdout: "client_addr": "172.21.15.89:0/2400821448", 2023-06-03T20:07:33.495 INFO:teuthology.orchestra.run.smithi089.stdout: "tid": 8914 2023-06-03T20:07:33.495 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:07:33.495 INFO:teuthology.orchestra.run.smithi089.stdout: "events": [ 2023-06-03T20:07:33.495 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:07:33.495 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "initiated", 2023-06-03T20:07:33.495 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:44.760429+0000", 2023-06-03T20:07:33.496 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0 2023-06-03T20:07:33.496 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:07:33.496 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:07:33.496 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "throttled", 2023-06-03T20:07:33.496 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:44.760429+0000", 2023-06-03T20:07:33.496 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0 2023-06-03T20:07:33.496 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:07:33.497 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:07:33.497 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "header_read", 2023-06-03T20:07:33.497 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:44.760427+0000", 2023-06-03T20:07:33.497 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 4294967295.9999981 2023-06-03T20:07:33.497 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:07:33.497 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:07:33.497 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "all_read", 2023-06-03T20:07:33.497 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:44.760451+0000", 2023-06-03T20:07:33.498 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 2.3876999999999999e-05 2023-06-03T20:07:33.498 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:07:33.498 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:07:33.498 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "dispatched", 2023-06-03T20:07:33.498 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:44.760462+0000", 2023-06-03T20:07:33.498 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 1.1755e-05 2023-06-03T20:07:33.498 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:07:33.498 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:07:33.499 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "queued_for_pg", 2023-06-03T20:07:33.499 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:44.760469+0000", 2023-06-03T20:07:33.499 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 6.5429999999999999e-06 2023-06-03T20:07:33.499 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:07:33.499 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:07:33.499 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "reached_pg", 2023-06-03T20:07:33.499 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:44.760512+0000", 2023-06-03T20:07:33.500 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 4.3179000000000001e-05 2023-06-03T20:07:33.500 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:07:33.500 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:07:33.500 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "started", 2023-06-03T20:07:33.500 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:44.760608+0000", 2023-06-03T20:07:33.501 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 9.6262999999999996e-05 2023-06-03T20:07:33.501 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:07:33.501 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:07:33.501 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "waiting for subops from 5,7", 2023-06-03T20:07:33.501 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:44.760700+0000", 2023-06-03T20:07:33.501 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 9.1521999999999996e-05 2023-06-03T20:07:33.501 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:07:33.502 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:07:33.502 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "op_commit", 2023-06-03T20:07:33.502 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:44.763865+0000", 2023-06-03T20:07:33.502 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.0031654489999999999 2023-06-03T20:07:33.502 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:07:33.502 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:07:33.502 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:07:33.502 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:44.763897+0000", 2023-06-03T20:07:33.503 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 3.1371000000000002e-05 2023-06-03T20:07:33.503 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:07:33.503 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:07:33.503 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:07:33.503 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:45.136129+0000", 2023-06-03T20:07:33.503 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.37223246100000001 2023-06-03T20:07:33.503 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:07:33.503 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:07:33.504 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "commit_sent", 2023-06-03T20:07:33.504 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:45.136235+0000", 2023-06-03T20:07:33.504 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.000105318 2023-06-03T20:07:33.504 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:07:33.504 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:07:33.504 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "done", 2023-06-03T20:07:33.504 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:45.136297+0000", 2023-06-03T20:07:33.505 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 6.2312999999999998e-05 2023-06-03T20:07:33.505 INFO:teuthology.orchestra.run.smithi089.stdout: } 2023-06-03T20:07:33.505 INFO:teuthology.orchestra.run.smithi089.stdout: ] 2023-06-03T20:07:33.505 INFO:teuthology.orchestra.run.smithi089.stdout: } 2023-06-03T20:07:33.505 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:07:33.505 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:07:33.505 INFO:teuthology.orchestra.run.smithi089.stdout: "description": "osd_op(client.45713.0:8917 9.5 9:aac1ab35:::benchmark_data_smithi089_115580_object8916:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e131)", 2023-06-03T20:07:33.505 INFO:teuthology.orchestra.run.smithi089.stdout: "initiated_at": "2023-06-03T20:00:44.760998+0000", 2023-06-03T20:07:33.506 INFO:teuthology.orchestra.run.smithi089.stdout: "age": 408.72248848499999, 2023-06-03T20:07:33.506 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.37616303899999998, 2023-06-03T20:07:33.506 INFO:teuthology.orchestra.run.smithi089.stdout: "type_data": { 2023-06-03T20:07:33.506 INFO:teuthology.orchestra.run.smithi089.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-03T20:07:33.506 INFO:teuthology.orchestra.run.smithi089.stdout: "client_info": { 2023-06-03T20:07:33.506 INFO:teuthology.orchestra.run.smithi089.stdout: "client": "client.45713", 2023-06-03T20:07:33.506 INFO:teuthology.orchestra.run.smithi089.stdout: "client_addr": "172.21.15.89:0/2400821448", 2023-06-03T20:07:33.507 INFO:teuthology.orchestra.run.smithi089.stdout: "tid": 8917 2023-06-03T20:07:33.507 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:07:33.507 INFO:teuthology.orchestra.run.smithi089.stdout: "events": [ 2023-06-03T20:07:33.507 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:07:33.507 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "initiated", 2023-06-03T20:07:33.508 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:44.760998+0000", 2023-06-03T20:07:33.508 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0 2023-06-03T20:07:33.508 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:07:33.508 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:07:33.508 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "throttled", 2023-06-03T20:07:33.508 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:44.760998+0000", 2023-06-03T20:07:33.508 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0 2023-06-03T20:07:33.509 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:07:33.509 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:07:33.509 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "header_read", 2023-06-03T20:07:33.509 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:44.760996+0000", 2023-06-03T20:07:33.509 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 4294967295.9999986 2023-06-03T20:07:33.509 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:07:33.509 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:07:33.509 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "all_read", 2023-06-03T20:07:33.510 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:44.761055+0000", 2023-06-03T20:07:33.510 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 5.8335000000000003e-05 2023-06-03T20:07:33.510 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:07:33.510 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:07:33.510 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "dispatched", 2023-06-03T20:07:33.510 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:44.761065+0000", 2023-06-03T20:07:33.511 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 9.9550000000000004e-06 2023-06-03T20:07:33.511 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:07:33.511 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:07:33.511 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "queued_for_pg", 2023-06-03T20:07:33.512 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:44.761071+0000", 2023-06-03T20:07:33.512 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 6.2090000000000002e-06 2023-06-03T20:07:33.512 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:07:33.513 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:07:33.513 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "reached_pg", 2023-06-03T20:07:33.513 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:44.761112+0000", 2023-06-03T20:07:33.513 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 4.0796e-05 2023-06-03T20:07:33.513 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:07:33.513 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:07:33.513 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "started", 2023-06-03T20:07:33.513 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:44.761200+0000", 2023-06-03T20:07:33.514 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 8.8283999999999997e-05 2023-06-03T20:07:33.514 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:07:33.514 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:07:33.514 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "waiting for subops from 5,7", 2023-06-03T20:07:33.514 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:44.761287+0000", 2023-06-03T20:07:33.514 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 8.6688999999999996e-05 2023-06-03T20:07:33.515 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:07:33.515 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:07:33.515 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "op_commit", 2023-06-03T20:07:33.515 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:44.762714+0000", 2023-06-03T20:07:33.515 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.001427097 2023-06-03T20:07:33.515 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:07:33.515 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:07:33.516 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:07:33.516 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:44.763651+0000", 2023-06-03T20:07:33.516 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.000937213 2023-06-03T20:07:33.516 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:07:33.516 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:07:33.516 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:07:33.516 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:45.137073+0000", 2023-06-03T20:07:33.517 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.37342209700000001 2023-06-03T20:07:33.517 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:07:33.517 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:07:33.517 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "commit_sent", 2023-06-03T20:07:33.517 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:45.137121+0000", 2023-06-03T20:07:33.517 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 4.7462000000000001e-05 2023-06-03T20:07:33.517 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:07:33.518 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:07:33.518 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "done", 2023-06-03T20:07:33.518 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:45.137161+0000", 2023-06-03T20:07:33.518 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 4.0244999999999998e-05 2023-06-03T20:07:33.518 INFO:teuthology.orchestra.run.smithi089.stdout: } 2023-06-03T20:07:33.518 INFO:teuthology.orchestra.run.smithi089.stdout: ] 2023-06-03T20:07:33.518 INFO:teuthology.orchestra.run.smithi089.stdout: } 2023-06-03T20:07:33.518 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:07:33.519 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:07:33.519 INFO:teuthology.orchestra.run.smithi089.stdout: "description": "osd_op(client.45713.0:8920 9.9 9:92e9a3b7:::benchmark_data_smithi089_115580_object8919:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e131)", 2023-06-03T20:07:33.519 INFO:teuthology.orchestra.run.smithi089.stdout: "initiated_at": "2023-06-03T20:00:44.762045+0000", 2023-06-03T20:07:33.519 INFO:teuthology.orchestra.run.smithi089.stdout: "age": 408.72144120000002, 2023-06-03T20:07:33.519 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.37482944499999998, 2023-06-03T20:07:33.519 INFO:teuthology.orchestra.run.smithi089.stdout: "type_data": { 2023-06-03T20:07:33.519 INFO:teuthology.orchestra.run.smithi089.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-03T20:07:33.519 INFO:teuthology.orchestra.run.smithi089.stdout: "client_info": { 2023-06-03T20:07:33.520 INFO:teuthology.orchestra.run.smithi089.stdout: "client": "client.45713", 2023-06-03T20:07:33.520 INFO:teuthology.orchestra.run.smithi089.stdout: "client_addr": "172.21.15.89:0/2400821448", 2023-06-03T20:07:33.520 INFO:teuthology.orchestra.run.smithi089.stdout: "tid": 8920 2023-06-03T20:07:33.520 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:07:33.520 INFO:teuthology.orchestra.run.smithi089.stdout: "events": [ 2023-06-03T20:07:33.520 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:07:33.521 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "initiated", 2023-06-03T20:07:33.521 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:44.762045+0000", 2023-06-03T20:07:33.521 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0 2023-06-03T20:07:33.521 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:07:33.521 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:07:33.521 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "throttled", 2023-06-03T20:07:33.521 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:44.762045+0000", 2023-06-03T20:07:33.522 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0 2023-06-03T20:07:33.522 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:07:33.522 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:07:33.522 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "header_read", 2023-06-03T20:07:33.522 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:44.762044+0000", 2023-06-03T20:07:33.522 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 4294967295.999999 2023-06-03T20:07:33.522 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:07:33.523 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:07:33.523 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "all_read", 2023-06-03T20:07:33.523 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:44.762070+0000", 2023-06-03T20:07:33.523 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 2.6410999999999998e-05 2023-06-03T20:07:33.523 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:07:33.523 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:07:33.523 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "dispatched", 2023-06-03T20:07:33.523 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:44.762082+0000", 2023-06-03T20:07:33.524 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 1.1782e-05 2023-06-03T20:07:33.524 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:07:33.524 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:07:33.524 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "queued_for_pg", 2023-06-03T20:07:33.524 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:44.762090+0000", 2023-06-03T20:07:33.524 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 8.1329999999999999e-06 2023-06-03T20:07:33.524 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:07:33.525 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:07:33.525 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "reached_pg", 2023-06-03T20:07:33.525 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:44.762863+0000", 2023-06-03T20:07:33.525 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.00077303599999999995 2023-06-03T20:07:33.525 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:07:33.526 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:07:33.526 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "started", 2023-06-03T20:07:33.526 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:44.762996+0000", 2023-06-03T20:07:33.526 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.00013276299999999999 2023-06-03T20:07:33.526 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:07:33.526 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:07:33.526 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "waiting for subops from 5,7", 2023-06-03T20:07:33.527 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:44.763142+0000", 2023-06-03T20:07:33.527 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.00014613000000000001 2023-06-03T20:07:33.527 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:07:33.527 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:07:33.527 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "op_commit", 2023-06-03T20:07:33.527 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:44.763871+0000", 2023-06-03T20:07:33.527 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.000728873 2023-06-03T20:07:33.527 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:07:33.528 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:07:33.528 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:07:33.528 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:45.136032+0000", 2023-06-03T20:07:33.528 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.37216058000000002 2023-06-03T20:07:33.528 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:07:33.528 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:07:33.528 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:07:33.529 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:45.136628+0000", 2023-06-03T20:07:33.529 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.00059603 2023-06-03T20:07:33.529 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:07:33.529 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:07:33.529 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "commit_sent", 2023-06-03T20:07:33.529 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:45.136789+0000", 2023-06-03T20:07:33.529 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.00016094199999999999 2023-06-03T20:07:33.529 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:07:33.530 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:07:33.530 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "done", 2023-06-03T20:07:33.530 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:45.136875+0000", 2023-06-03T20:07:33.530 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 8.5907000000000003e-05 2023-06-03T20:07:33.531 INFO:teuthology.orchestra.run.smithi089.stdout: } 2023-06-03T20:07:33.531 INFO:teuthology.orchestra.run.smithi089.stdout: ] 2023-06-03T20:07:33.531 INFO:teuthology.orchestra.run.smithi089.stdout: } 2023-06-03T20:07:33.531 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:07:33.532 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:07:33.532 INFO:teuthology.orchestra.run.smithi089.stdout: "description": "osd_op(client.45713.0:8924 9.5 9:a76f7127:::benchmark_data_smithi089_115580_object8923:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e131)", 2023-06-03T20:07:33.532 INFO:teuthology.orchestra.run.smithi089.stdout: "initiated_at": "2023-06-03T20:00:44.764489+0000", 2023-06-03T20:07:33.533 INFO:teuthology.orchestra.run.smithi089.stdout: "age": 408.718997672, 2023-06-03T20:07:33.533 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.374591703, 2023-06-03T20:07:33.533 INFO:teuthology.orchestra.run.smithi089.stdout: "type_data": { 2023-06-03T20:07:33.533 INFO:teuthology.orchestra.run.smithi089.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-03T20:07:33.534 INFO:teuthology.orchestra.run.smithi089.stdout: "client_info": { 2023-06-03T20:07:33.534 INFO:teuthology.orchestra.run.smithi089.stdout: "client": "client.45713", 2023-06-03T20:07:33.534 INFO:teuthology.orchestra.run.smithi089.stdout: "client_addr": "172.21.15.89:0/2400821448", 2023-06-03T20:07:33.535 INFO:teuthology.orchestra.run.smithi089.stdout: "tid": 8924 2023-06-03T20:07:33.535 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:07:33.535 INFO:teuthology.orchestra.run.smithi089.stdout: "events": [ 2023-06-03T20:07:33.536 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:07:33.536 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "initiated", 2023-06-03T20:07:33.536 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:44.764489+0000", 2023-06-03T20:07:33.537 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0 2023-06-03T20:07:33.537 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:07:33.537 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:07:33.537 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "throttled", 2023-06-03T20:07:33.537 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:44.764489+0000", 2023-06-03T20:07:33.538 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0 2023-06-03T20:07:33.538 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:07:33.538 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:07:33.538 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "header_read", 2023-06-03T20:07:33.538 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:44.764487+0000", 2023-06-03T20:07:33.539 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 4294967295.9999986 2023-06-03T20:07:33.539 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:07:33.539 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:07:33.540 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "all_read", 2023-06-03T20:07:33.540 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:44.764509+0000", 2023-06-03T20:07:33.540 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 2.1444999999999999e-05 2023-06-03T20:07:33.540 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:07:33.540 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:07:33.541 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "dispatched", 2023-06-03T20:07:33.541 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:44.764520+0000", 2023-06-03T20:07:33.541 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 1.0791999999999999e-05 2023-06-03T20:07:33.541 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:07:33.541 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:07:33.541 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "queued_for_pg", 2023-06-03T20:07:33.541 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:44.764526+0000", 2023-06-03T20:07:33.542 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 5.9569999999999999e-06 2023-06-03T20:07:33.542 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:07:33.542 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:07:33.542 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "reached_pg", 2023-06-03T20:07:33.542 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:44.764574+0000", 2023-06-03T20:07:33.542 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 4.8520999999999998e-05 2023-06-03T20:07:33.542 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:07:33.542 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:07:33.543 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "started", 2023-06-03T20:07:33.543 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:44.764677+0000", 2023-06-03T20:07:33.543 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.00010307299999999999 2023-06-03T20:07:33.543 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:07:33.543 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:07:33.543 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "waiting for subops from 5,7", 2023-06-03T20:07:33.544 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:44.764931+0000", 2023-06-03T20:07:33.544 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.00025413600000000002 2023-06-03T20:07:33.544 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:07:33.544 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:07:33.544 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:07:33.544 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:45.137206+0000", 2023-06-03T20:07:33.544 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.37227488800000003 2023-06-03T20:07:33.545 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:07:33.545 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:07:33.545 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:07:33.545 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:45.137325+0000", 2023-06-03T20:07:33.545 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.000118918 2023-06-03T20:07:33.545 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:07:33.545 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:07:33.546 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "op_commit", 2023-06-03T20:07:33.546 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:45.138972+0000", 2023-06-03T20:07:33.546 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.0016469259999999999 2023-06-03T20:07:33.546 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:07:33.546 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:07:33.546 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "commit_sent", 2023-06-03T20:07:33.546 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:45.139036+0000", 2023-06-03T20:07:33.546 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 6.4166999999999997e-05 2023-06-03T20:07:33.547 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:07:33.547 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:07:33.547 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "done", 2023-06-03T20:07:33.547 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:45.139080+0000", 2023-06-03T20:07:33.547 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 4.4146000000000001e-05 2023-06-03T20:07:33.547 INFO:teuthology.orchestra.run.smithi089.stdout: } 2023-06-03T20:07:33.547 INFO:teuthology.orchestra.run.smithi089.stdout: ] 2023-06-03T20:07:33.548 INFO:teuthology.orchestra.run.smithi089.stdout: } 2023-06-03T20:07:33.548 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:07:33.548 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:07:33.548 INFO:teuthology.orchestra.run.smithi089.stdout: "description": "osd_repop_reply(client.45713.0:8920 9.9 e131/127 ondisk, result = 0)", 2023-06-03T20:07:33.548 INFO:teuthology.orchestra.run.smithi089.stdout: "initiated_at": "2023-06-03T20:00:44.764802+0000", 2023-06-03T20:07:33.548 INFO:teuthology.orchestra.run.smithi089.stdout: "age": 408.71868473299998, 2023-06-03T20:07:33.548 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.37123982500000002, 2023-06-03T20:07:33.549 INFO:teuthology.orchestra.run.smithi089.stdout: "type_data": { 2023-06-03T20:07:33.549 INFO:teuthology.orchestra.run.smithi089.stdout: "flag_point": "started", 2023-06-03T20:07:33.549 INFO:teuthology.orchestra.run.smithi089.stdout: "events": [ 2023-06-03T20:07:33.549 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:07:33.549 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "initiated", 2023-06-03T20:07:33.549 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:44.764802+0000", 2023-06-03T20:07:33.549 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0 2023-06-03T20:07:33.549 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:07:33.550 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:07:33.550 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "throttled", 2023-06-03T20:07:33.550 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:44.764802+0000", 2023-06-03T20:07:33.550 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0 2023-06-03T20:07:33.550 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:07:33.550 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:07:33.550 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "header_read", 2023-06-03T20:07:33.551 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:44.764801+0000", 2023-06-03T20:07:33.551 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 4294967295.999999 2023-06-03T20:07:33.551 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:07:33.551 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:07:33.551 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "all_read", 2023-06-03T20:07:33.551 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:44.764807+0000", 2023-06-03T20:07:33.551 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 5.8250000000000001e-06 2023-06-03T20:07:33.551 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:07:33.552 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:07:33.552 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "dispatched", 2023-06-03T20:07:33.552 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:44.764816+0000", 2023-06-03T20:07:33.552 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 9.1540000000000008e-06 2023-06-03T20:07:33.552 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:07:33.552 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:07:33.552 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "queued_for_pg", 2023-06-03T20:07:33.553 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:44.764821+0000", 2023-06-03T20:07:33.553 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 4.9180000000000002e-06 2023-06-03T20:07:33.553 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:07:33.553 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:07:33.553 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "reached_pg", 2023-06-03T20:07:33.553 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:45.135983+0000", 2023-06-03T20:07:33.553 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.37116188999999999 2023-06-03T20:07:33.553 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:07:33.554 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:07:33.554 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "started", 2023-06-03T20:07:33.554 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:45.136003+0000", 2023-06-03T20:07:33.554 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 2.0516000000000001e-05 2023-06-03T20:07:33.554 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:07:33.554 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:07:33.554 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "done", 2023-06-03T20:07:33.554 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:45.136041+0000", 2023-06-03T20:07:33.555 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 3.8279999999999999e-05 2023-06-03T20:07:33.555 INFO:teuthology.orchestra.run.smithi089.stdout: } 2023-06-03T20:07:33.555 INFO:teuthology.orchestra.run.smithi089.stdout: ] 2023-06-03T20:07:33.555 INFO:teuthology.orchestra.run.smithi089.stdout: } 2023-06-03T20:07:33.555 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:07:33.555 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:07:33.555 INFO:teuthology.orchestra.run.smithi089.stdout: "description": "osd_op(client.45713.0:8926 9.5 9:a1989046:::benchmark_data_smithi089_115580_object8925:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e131)", 2023-06-03T20:07:33.555 INFO:teuthology.orchestra.run.smithi089.stdout: "initiated_at": "2023-06-03T20:00:44.765451+0000", 2023-06-03T20:07:33.556 INFO:teuthology.orchestra.run.smithi089.stdout: "age": 408.71803513899999, 2023-06-03T20:07:33.556 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.37387337300000001, 2023-06-03T20:07:33.556 INFO:teuthology.orchestra.run.smithi089.stdout: "type_data": { 2023-06-03T20:07:33.556 INFO:teuthology.orchestra.run.smithi089.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-03T20:07:33.556 INFO:teuthology.orchestra.run.smithi089.stdout: "client_info": { 2023-06-03T20:07:33.556 INFO:teuthology.orchestra.run.smithi089.stdout: "client": "client.45713", 2023-06-03T20:07:33.556 INFO:teuthology.orchestra.run.smithi089.stdout: "client_addr": "172.21.15.89:0/2400821448", 2023-06-03T20:07:33.557 INFO:teuthology.orchestra.run.smithi089.stdout: "tid": 8926 2023-06-03T20:07:33.557 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:07:33.557 INFO:teuthology.orchestra.run.smithi089.stdout: "events": [ 2023-06-03T20:07:33.557 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:07:33.557 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "initiated", 2023-06-03T20:07:33.557 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:44.765451+0000", 2023-06-03T20:07:33.557 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0 2023-06-03T20:07:33.557 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:07:33.558 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:07:33.558 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "throttled", 2023-06-03T20:07:33.558 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:44.765451+0000", 2023-06-03T20:07:33.558 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0 2023-06-03T20:07:33.558 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:07:33.558 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:07:33.558 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "header_read", 2023-06-03T20:07:33.558 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:44.765450+0000", 2023-06-03T20:07:33.559 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 4294967295.999999 2023-06-03T20:07:33.559 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:07:33.559 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:07:33.559 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "all_read", 2023-06-03T20:07:33.559 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:44.765471+0000", 2023-06-03T20:07:33.559 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 2.1005999999999998e-05 2023-06-03T20:07:33.559 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:07:33.559 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:07:33.560 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "dispatched", 2023-06-03T20:07:33.560 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:44.765481+0000", 2023-06-03T20:07:33.560 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 1.0020999999999999e-05 2023-06-03T20:07:33.560 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:07:33.560 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:07:33.560 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "queued_for_pg", 2023-06-03T20:07:33.560 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:44.765487+0000", 2023-06-03T20:07:33.561 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 5.8619999999999999e-06 2023-06-03T20:07:33.561 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:07:33.561 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:07:33.561 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "reached_pg", 2023-06-03T20:07:33.561 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:45.135891+0000", 2023-06-03T20:07:33.561 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.37040393100000002 2023-06-03T20:07:33.561 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:07:33.561 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:07:33.562 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "started", 2023-06-03T20:07:33.562 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:45.136084+0000", 2023-06-03T20:07:33.562 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.00019294000000000001 2023-06-03T20:07:33.562 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:07:33.562 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:07:33.562 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "waiting for subops from 5,7", 2023-06-03T20:07:33.562 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:45.136277+0000", 2023-06-03T20:07:33.563 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.00019294099999999999 2023-06-03T20:07:33.563 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:07:33.563 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:07:33.563 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:07:33.563 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:45.139125+0000", 2023-06-03T20:07:33.563 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.0028480020000000001 2023-06-03T20:07:33.563 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:07:33.563 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:07:33.564 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "op_commit", 2023-06-03T20:07:33.564 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:45.139145+0000", 2023-06-03T20:07:33.564 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 1.9703000000000001e-05 2023-06-03T20:07:33.564 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:07:33.564 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:07:33.564 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:07:33.564 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:45.139231+0000", 2023-06-03T20:07:33.564 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 8.6260000000000004e-05 2023-06-03T20:07:33.565 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:07:33.565 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:07:33.565 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "commit_sent", 2023-06-03T20:07:33.565 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:45.139285+0000", 2023-06-03T20:07:33.565 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 5.4042999999999997e-05 2023-06-03T20:07:33.565 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:07:33.565 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:07:33.566 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "done", 2023-06-03T20:07:33.566 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:45.139325+0000", 2023-06-03T20:07:33.566 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 3.9561000000000003e-05 2023-06-03T20:07:33.566 INFO:teuthology.orchestra.run.smithi089.stdout: } 2023-06-03T20:07:33.567 INFO:teuthology.orchestra.run.smithi089.stdout: ] 2023-06-03T20:07:33.567 INFO:teuthology.orchestra.run.smithi089.stdout: } 2023-06-03T20:07:33.567 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:07:33.568 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:07:33.568 INFO:teuthology.orchestra.run.smithi089.stdout: "description": "osd_repop_reply(client.45713.0:8914 9.9 e131/127 ondisk, result = 0)", 2023-06-03T20:07:33.568 INFO:teuthology.orchestra.run.smithi089.stdout: "initiated_at": "2023-06-03T20:00:44.765691+0000", 2023-06-03T20:07:33.568 INFO:teuthology.orchestra.run.smithi089.stdout: "age": 408.71779485299999, 2023-06-03T20:07:33.569 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.37061955299999999, 2023-06-03T20:07:33.569 INFO:teuthology.orchestra.run.smithi089.stdout: "type_data": { 2023-06-03T20:07:33.569 INFO:teuthology.orchestra.run.smithi089.stdout: "flag_point": "started", 2023-06-03T20:07:33.569 INFO:teuthology.orchestra.run.smithi089.stdout: "events": [ 2023-06-03T20:07:33.570 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:07:33.570 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "initiated", 2023-06-03T20:07:33.570 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:44.765691+0000", 2023-06-03T20:07:33.571 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0 2023-06-03T20:07:33.571 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:07:33.571 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:07:33.571 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "throttled", 2023-06-03T20:07:33.572 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:44.765691+0000", 2023-06-03T20:07:33.572 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0 2023-06-03T20:07:33.572 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:07:33.573 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:07:33.573 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "header_read", 2023-06-03T20:07:33.573 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:44.765691+0000", 2023-06-03T20:07:33.573 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 4294967295.9999995 2023-06-03T20:07:33.574 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:07:33.574 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:07:33.574 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "all_read", 2023-06-03T20:07:33.574 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:44.765696+0000", 2023-06-03T20:07:33.574 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 5.0309999999999998e-06 2023-06-03T20:07:33.575 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:07:33.575 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:07:33.575 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "dispatched", 2023-06-03T20:07:33.575 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:44.765706+0000", 2023-06-03T20:07:33.576 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 1.0387999999999999e-05 2023-06-03T20:07:33.576 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:07:33.576 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:07:33.577 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "queued_for_pg", 2023-06-03T20:07:33.577 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:44.765713+0000", 2023-06-03T20:07:33.577 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 7.0140000000000001e-06 2023-06-03T20:07:33.577 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:07:33.577 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:07:33.577 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "reached_pg", 2023-06-03T20:07:33.578 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:45.136098+0000", 2023-06-03T20:07:33.578 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.37038465199999998 2023-06-03T20:07:33.578 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:07:33.578 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:07:33.578 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "started", 2023-06-03T20:07:33.578 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:45.136115+0000", 2023-06-03T20:07:33.578 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 1.6702e-05 2023-06-03T20:07:33.578 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:07:33.579 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:07:33.579 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "done", 2023-06-03T20:07:33.579 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:45.136311+0000", 2023-06-03T20:07:33.579 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.00019639599999999999 2023-06-03T20:07:33.579 INFO:teuthology.orchestra.run.smithi089.stdout: } 2023-06-03T20:07:33.579 INFO:teuthology.orchestra.run.smithi089.stdout: ] 2023-06-03T20:07:33.579 INFO:teuthology.orchestra.run.smithi089.stdout: } 2023-06-03T20:07:33.579 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:07:33.580 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:07:33.580 INFO:teuthology.orchestra.run.smithi089.stdout: "description": "osd_repop_reply(client.45713.0:8917 9.5 e131/127 ondisk, result = 0)", 2023-06-03T20:07:33.580 INFO:teuthology.orchestra.run.smithi089.stdout: "initiated_at": "2023-06-03T20:00:44.765720+0000", 2023-06-03T20:07:33.580 INFO:teuthology.orchestra.run.smithi089.stdout: "age": 408.71776663999998, 2023-06-03T20:07:33.580 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.37145095099999997, 2023-06-03T20:07:33.580 INFO:teuthology.orchestra.run.smithi089.stdout: "type_data": { 2023-06-03T20:07:33.581 INFO:teuthology.orchestra.run.smithi089.stdout: "flag_point": "started", 2023-06-03T20:07:33.581 INFO:teuthology.orchestra.run.smithi089.stdout: "events": [ 2023-06-03T20:07:33.581 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:07:33.581 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "initiated", 2023-06-03T20:07:33.582 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:44.765720+0000", 2023-06-03T20:07:33.582 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0 2023-06-03T20:07:33.582 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:07:33.582 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:07:33.583 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "throttled", 2023-06-03T20:07:33.583 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:44.765720+0000", 2023-06-03T20:07:33.584 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0 2023-06-03T20:07:33.584 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:07:33.584 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:07:33.584 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "header_read", 2023-06-03T20:07:33.585 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:44.765719+0000", 2023-06-03T20:07:33.585 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 4294967295.9999995 2023-06-03T20:07:33.585 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:07:33.586 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:07:33.586 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "all_read", 2023-06-03T20:07:33.586 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:44.765722+0000", 2023-06-03T20:07:33.587 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 2.8619999999999999e-06 2023-06-03T20:07:33.587 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:07:33.587 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:07:33.588 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:07:33.590 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "dispatched", 2023-06-03T20:07:33.590 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:44.765728+0000", 2023-06-03T20:07:33.590 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 5.3240000000000002e-06 2023-06-03T20:07:33.590 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:07:33.590 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:07:33.590 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "queued_for_pg", 2023-06-03T20:07:33.590 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:44.765732+0000", 2023-06-03T20:07:33.591 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 4.0620000000000002e-06 2023-06-03T20:07:33.591 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:07:33.591 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:07:33.591 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "reached_pg", 2023-06-03T20:07:33.591 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:45.137058+0000", 2023-06-03T20:07:33.591 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.37132620700000002 2023-06-03T20:07:33.591 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:07:33.592 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:07:33.592 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "started", 2023-06-03T20:07:33.592 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:45.137067+0000", 2023-06-03T20:07:33.592 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 9.0599999999999997e-06 2023-06-03T20:07:33.592 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:07:33.592 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:07:33.592 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "done", 2023-06-03T20:07:33.592 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:45.137171+0000", 2023-06-03T20:07:33.593 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.00010378399999999999 2023-06-03T20:07:33.593 INFO:teuthology.orchestra.run.smithi089.stdout: } 2023-06-03T20:07:33.593 INFO:teuthology.orchestra.run.smithi089.stdout: ] 2023-06-03T20:07:33.593 INFO:teuthology.orchestra.run.smithi089.stdout: } 2023-06-03T20:07:33.593 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:07:33.593 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:07:33.593 INFO:teuthology.orchestra.run.smithi089.stdout: "description": "osd_repop_reply(client.45713.0:8924 9.5 e131/127 ondisk, result = 0)", 2023-06-03T20:07:33.594 INFO:teuthology.orchestra.run.smithi089.stdout: "initiated_at": "2023-06-03T20:00:44.766015+0000", 2023-06-03T20:07:33.594 INFO:teuthology.orchestra.run.smithi089.stdout: "age": 408.71747093599998, 2023-06-03T20:07:33.594 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.371193312, 2023-06-03T20:07:33.594 INFO:teuthology.orchestra.run.smithi089.stdout: "type_data": { 2023-06-03T20:07:33.594 INFO:teuthology.orchestra.run.smithi089.stdout: "flag_point": "started", 2023-06-03T20:07:33.594 INFO:teuthology.orchestra.run.smithi089.stdout: "events": [ 2023-06-03T20:07:33.594 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:07:33.594 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "initiated", 2023-06-03T20:07:33.595 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:44.766015+0000", 2023-06-03T20:07:33.595 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0 2023-06-03T20:07:33.595 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:07:33.595 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:07:33.595 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "throttled", 2023-06-03T20:07:33.595 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:44.766015+0000", 2023-06-03T20:07:33.595 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0 2023-06-03T20:07:33.595 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:07:33.596 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:07:33.596 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "header_read", 2023-06-03T20:07:33.596 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:44.766015+0000", 2023-06-03T20:07:33.596 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 4294967295.9999995 2023-06-03T20:07:33.596 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:07:33.596 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:07:33.596 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "all_read", 2023-06-03T20:07:33.597 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:44.766020+0000", 2023-06-03T20:07:33.597 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 4.8629999999999999e-06 2023-06-03T20:07:33.597 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:07:33.597 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:07:33.597 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "dispatched", 2023-06-03T20:07:33.597 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:44.766028+0000", 2023-06-03T20:07:33.597 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 8.1669999999999999e-06 2023-06-03T20:07:33.597 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:07:33.598 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:07:33.598 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "queued_for_pg", 2023-06-03T20:07:33.598 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:44.766035+0000", 2023-06-03T20:07:33.598 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 7.1450000000000002e-06 2023-06-03T20:07:33.598 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:07:33.598 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:07:33.598 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "reached_pg", 2023-06-03T20:07:33.598 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:45.137193+0000", 2023-06-03T20:07:33.599 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.371157713 2023-06-03T20:07:33.599 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:07:33.599 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:07:33.599 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "started", 2023-06-03T20:07:33.599 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:45.137200+0000", 2023-06-03T20:07:33.599 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 7.3690000000000001e-06 2023-06-03T20:07:33.600 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:07:33.600 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:07:33.600 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "done", 2023-06-03T20:07:33.600 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:45.137209+0000", 2023-06-03T20:07:33.600 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 8.6629999999999999e-06 2023-06-03T20:07:33.600 INFO:teuthology.orchestra.run.smithi089.stdout: } 2023-06-03T20:07:33.600 INFO:teuthology.orchestra.run.smithi089.stdout: ] 2023-06-03T20:07:33.600 INFO:teuthology.orchestra.run.smithi089.stdout: } 2023-06-03T20:07:33.601 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:07:33.601 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:07:33.601 INFO:teuthology.orchestra.run.smithi089.stdout: "description": "osd_op(client.45713.0:8931 9.9 9:91181acb:::benchmark_data_smithi089_115580_object8930:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e131)", 2023-06-03T20:07:33.601 INFO:teuthology.orchestra.run.smithi089.stdout: "initiated_at": "2023-06-03T20:00:44.766784+0000", 2023-06-03T20:07:33.601 INFO:teuthology.orchestra.run.smithi089.stdout: "age": 408.71670273400002, 2023-06-03T20:07:33.601 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.37793961399999998, 2023-06-03T20:07:33.601 INFO:teuthology.orchestra.run.smithi089.stdout: "type_data": { 2023-06-03T20:07:33.602 INFO:teuthology.orchestra.run.smithi089.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-03T20:07:33.602 INFO:teuthology.orchestra.run.smithi089.stdout: "client_info": { 2023-06-03T20:07:33.602 INFO:teuthology.orchestra.run.smithi089.stdout: "client": "client.45713", 2023-06-03T20:07:33.602 INFO:teuthology.orchestra.run.smithi089.stdout: "client_addr": "172.21.15.89:0/2400821448", 2023-06-03T20:07:33.602 INFO:teuthology.orchestra.run.smithi089.stdout: "tid": 8931 2023-06-03T20:07:33.602 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:07:33.602 INFO:teuthology.orchestra.run.smithi089.stdout: "events": [ 2023-06-03T20:07:33.602 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:07:33.603 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "initiated", 2023-06-03T20:07:33.603 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:44.766784+0000", 2023-06-03T20:07:33.603 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0 2023-06-03T20:07:33.603 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:07:33.603 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:07:33.603 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "throttled", 2023-06-03T20:07:33.603 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:44.766784+0000", 2023-06-03T20:07:33.603 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0 2023-06-03T20:07:33.604 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:07:33.604 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:07:33.604 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "header_read", 2023-06-03T20:07:33.604 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:44.766782+0000", 2023-06-03T20:07:33.604 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 4294967295.9999986 2023-06-03T20:07:33.604 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:07:33.604 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:07:33.604 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "all_read", 2023-06-03T20:07:33.605 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:44.766807+0000", 2023-06-03T20:07:33.605 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 2.461e-05 2023-06-03T20:07:33.605 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:07:33.605 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:07:33.605 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "dispatched", 2023-06-03T20:07:33.605 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:45.135902+0000", 2023-06-03T20:07:33.605 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.36909568700000001 2023-06-03T20:07:33.606 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:07:33.606 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:07:33.606 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "queued_for_pg", 2023-06-03T20:07:33.606 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:45.136048+0000", 2023-06-03T20:07:33.606 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.00014613300000000001 2023-06-03T20:07:33.606 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:07:33.606 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:07:33.606 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "reached_pg", 2023-06-03T20:07:33.607 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:45.136968+0000", 2023-06-03T20:07:33.607 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.000919599 2023-06-03T20:07:33.607 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:07:33.607 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:07:33.607 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "started", 2023-06-03T20:07:33.607 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:45.137246+0000", 2023-06-03T20:07:33.607 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.000278394 2023-06-03T20:07:33.607 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:07:33.608 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:07:33.608 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "waiting for subops from 5,7", 2023-06-03T20:07:33.608 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:45.137695+0000", 2023-06-03T20:07:33.608 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.000448073 2023-06-03T20:07:33.608 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:07:33.608 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:07:33.608 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "op_commit", 2023-06-03T20:07:33.609 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:45.141610+0000", 2023-06-03T20:07:33.609 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.0039156750000000004 2023-06-03T20:07:33.609 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:07:33.609 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:07:33.609 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:07:33.609 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:45.143947+0000", 2023-06-03T20:07:33.609 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.00233644 2023-06-03T20:07:33.609 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:07:33.610 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:07:33.610 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:07:33.610 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:45.144593+0000", 2023-06-03T20:07:33.610 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.00064617299999999999 2023-06-03T20:07:33.610 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:07:33.610 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:07:33.610 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "commit_sent", 2023-06-03T20:07:33.610 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:45.144665+0000", 2023-06-03T20:07:33.611 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 7.2498999999999998e-05 2023-06-03T20:07:33.611 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:07:33.611 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:07:33.611 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "done", 2023-06-03T20:07:33.611 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:45.144723+0000", 2023-06-03T20:07:33.611 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 5.7850999999999999e-05 2023-06-03T20:07:33.611 INFO:teuthology.orchestra.run.smithi089.stdout: } 2023-06-03T20:07:33.611 INFO:teuthology.orchestra.run.smithi089.stdout: ] 2023-06-03T20:07:33.612 INFO:teuthology.orchestra.run.smithi089.stdout: } 2023-06-03T20:07:33.612 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:07:33.612 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:07:33.612 INFO:teuthology.orchestra.run.smithi089.stdout: "description": "osd_repop_reply(client.45713.0:8924 9.5 e131/127 ondisk, result = 0)", 2023-06-03T20:07:33.612 INFO:teuthology.orchestra.run.smithi089.stdout: "initiated_at": "2023-06-03T20:00:44.766798+0000", 2023-06-03T20:07:33.612 INFO:teuthology.orchestra.run.smithi089.stdout: "age": 408.71668795300002, 2023-06-03T20:07:33.612 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.37052961200000001, 2023-06-03T20:07:33.612 INFO:teuthology.orchestra.run.smithi089.stdout: "type_data": { 2023-06-03T20:07:33.613 INFO:teuthology.orchestra.run.smithi089.stdout: "flag_point": "started", 2023-06-03T20:07:33.613 INFO:teuthology.orchestra.run.smithi089.stdout: "events": [ 2023-06-03T20:07:33.613 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:07:33.613 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "initiated", 2023-06-03T20:07:33.613 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:44.766798+0000", 2023-06-03T20:07:33.613 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0 2023-06-03T20:07:33.613 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:07:33.614 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:07:33.614 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "throttled", 2023-06-03T20:07:33.614 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:44.766798+0000", 2023-06-03T20:07:33.614 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0 2023-06-03T20:07:33.614 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:07:33.614 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:07:33.614 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "header_read", 2023-06-03T20:07:33.614 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:44.766798+0000", 2023-06-03T20:07:33.615 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 4294967295.9999995 2023-06-03T20:07:33.615 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:07:33.615 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:07:33.615 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "all_read", 2023-06-03T20:07:33.615 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:44.766802+0000", 2023-06-03T20:07:33.615 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 4.5419999999999999e-06 2023-06-03T20:07:33.615 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:07:33.615 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:07:33.616 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "dispatched", 2023-06-03T20:07:33.616 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:45.135774+0000", 2023-06-03T20:07:33.616 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.36897194100000003 2023-06-03T20:07:33.616 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:07:33.616 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:07:33.616 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "queued_for_pg", 2023-06-03T20:07:33.616 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:45.135841+0000", 2023-06-03T20:07:33.617 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 6.6319e-05 2023-06-03T20:07:33.617 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:07:33.617 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:07:33.617 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "reached_pg", 2023-06-03T20:07:33.617 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:45.137312+0000", 2023-06-03T20:07:33.617 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.001471577 2023-06-03T20:07:33.617 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:07:33.617 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:07:33.618 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "started", 2023-06-03T20:07:33.618 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:45.137320+0000", 2023-06-03T20:07:33.618 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 7.6960000000000005e-06 2023-06-03T20:07:33.618 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:07:33.618 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:07:33.618 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "done", 2023-06-03T20:07:33.618 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:00:45.137328+0000", 2023-06-03T20:07:33.618 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 8.1939999999999998e-06 2023-06-03T20:07:33.619 INFO:teuthology.orchestra.run.smithi089.stdout: } 2023-06-03T20:07:33.619 INFO:teuthology.orchestra.run.smithi089.stdout: ] 2023-06-03T20:07:33.619 INFO:teuthology.orchestra.run.smithi089.stdout: } 2023-06-03T20:07:33.619 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:07:33.619 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:07:33.619 INFO:teuthology.orchestra.run.smithi089.stdout: "description": "osd_op(client.36009.0:51297 10.4 10:269d043d:::benchmark_data_smithi089_140876_object51296:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e163)", 2023-06-03T20:07:33.619 INFO:teuthology.orchestra.run.smithi089.stdout: "initiated_at": "2023-06-03T20:06:43.267927+0000", 2023-06-03T20:07:33.620 INFO:teuthology.orchestra.run.smithi089.stdout: "age": 50.215559476000003, 2023-06-03T20:07:33.620 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.19792483599999999, 2023-06-03T20:07:33.620 INFO:teuthology.orchestra.run.smithi089.stdout: "type_data": { 2023-06-03T20:07:33.620 INFO:teuthology.orchestra.run.smithi089.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-03T20:07:33.620 INFO:teuthology.orchestra.run.smithi089.stdout: "client_info": { 2023-06-03T20:07:33.620 INFO:teuthology.orchestra.run.smithi089.stdout: "client": "client.36009", 2023-06-03T20:07:33.620 INFO:teuthology.orchestra.run.smithi089.stdout: "client_addr": "172.21.15.89:0/2550116169", 2023-06-03T20:07:33.620 INFO:teuthology.orchestra.run.smithi089.stdout: "tid": 51297 2023-06-03T20:07:33.621 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:07:33.621 INFO:teuthology.orchestra.run.smithi089.stdout: "events": [ 2023-06-03T20:07:33.621 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:07:33.621 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "initiated", 2023-06-03T20:07:33.621 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:06:43.267927+0000", 2023-06-03T20:07:33.621 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0 2023-06-03T20:07:33.621 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:07:33.621 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:07:33.622 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "throttled", 2023-06-03T20:07:33.622 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:06:43.267927+0000", 2023-06-03T20:07:33.622 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0 2023-06-03T20:07:33.622 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:07:33.622 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:07:33.622 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "header_read", 2023-06-03T20:07:33.622 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:06:43.267925+0000", 2023-06-03T20:07:33.622 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 4294967295.9999981 2023-06-03T20:07:33.623 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:07:33.623 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:07:33.623 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "all_read", 2023-06-03T20:07:33.623 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:06:43.267968+0000", 2023-06-03T20:07:33.623 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 4.2518000000000002e-05 2023-06-03T20:07:33.623 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:07:33.623 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:07:33.623 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "dispatched", 2023-06-03T20:07:33.624 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:06:43.267982+0000", 2023-06-03T20:07:33.624 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 1.4783000000000001e-05 2023-06-03T20:07:33.624 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:07:33.624 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:07:33.624 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "queued_for_pg", 2023-06-03T20:07:33.624 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:06:43.267992+0000", 2023-06-03T20:07:33.624 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 9.6069999999999997e-06 2023-06-03T20:07:33.625 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:07:33.625 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:07:33.625 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "reached_pg", 2023-06-03T20:07:33.625 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:06:43.268107+0000", 2023-06-03T20:07:33.625 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.000115124 2023-06-03T20:07:33.625 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:07:33.625 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:07:33.625 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "started", 2023-06-03T20:07:33.626 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:06:43.268222+0000", 2023-06-03T20:07:33.626 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.000115238 2023-06-03T20:07:33.626 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:07:33.626 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:07:33.626 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "waiting for subops from 3,4", 2023-06-03T20:07:33.626 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:06:43.268354+0000", 2023-06-03T20:07:33.626 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.000131982 2023-06-03T20:07:33.626 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:07:33.627 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:07:33.627 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "op_commit", 2023-06-03T20:07:33.627 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:06:43.273768+0000", 2023-06-03T20:07:33.627 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.0054134150000000004 2023-06-03T20:07:33.627 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:07:33.627 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:07:33.627 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:07:33.627 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:06:43.273799+0000", 2023-06-03T20:07:33.628 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 3.1034999999999997e-05 2023-06-03T20:07:33.628 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:07:33.628 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:07:33.628 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:07:33.628 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:06:43.465770+0000", 2023-06-03T20:07:33.628 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.19197162400000001 2023-06-03T20:07:33.628 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:07:33.628 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:07:33.629 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "commit_sent", 2023-06-03T20:07:33.629 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:06:43.465811+0000", 2023-06-03T20:07:33.629 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 4.0969e-05 2023-06-03T20:07:33.629 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:07:33.629 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:07:33.629 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "done", 2023-06-03T20:07:33.629 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:06:43.465852+0000", 2023-06-03T20:07:33.630 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 4.0297999999999999e-05 2023-06-03T20:07:33.630 INFO:teuthology.orchestra.run.smithi089.stdout: } 2023-06-03T20:07:33.630 INFO:teuthology.orchestra.run.smithi089.stdout: ] 2023-06-03T20:07:33.630 INFO:teuthology.orchestra.run.smithi089.stdout: } 2023-06-03T20:07:33.630 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:07:33.630 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:07:33.630 INFO:teuthology.orchestra.run.smithi089.stdout: "description": "osd_op(client.36009.0:51298 10.4 10:2e0070db:::benchmark_data_smithi089_140876_object51297:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e163)", 2023-06-03T20:07:33.630 INFO:teuthology.orchestra.run.smithi089.stdout: "initiated_at": "2023-06-03T20:06:43.268011+0000", 2023-06-03T20:07:33.631 INFO:teuthology.orchestra.run.smithi089.stdout: "age": 50.215475402000003, 2023-06-03T20:07:33.631 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.19810921100000001, 2023-06-03T20:07:33.631 INFO:teuthology.orchestra.run.smithi089.stdout: "type_data": { 2023-06-03T20:07:33.631 INFO:teuthology.orchestra.run.smithi089.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-03T20:07:33.631 INFO:teuthology.orchestra.run.smithi089.stdout: "client_info": { 2023-06-03T20:07:33.631 INFO:teuthology.orchestra.run.smithi089.stdout: "client": "client.36009", 2023-06-03T20:07:33.631 INFO:teuthology.orchestra.run.smithi089.stdout: "client_addr": "172.21.15.89:0/2550116169", 2023-06-03T20:07:33.631 INFO:teuthology.orchestra.run.smithi089.stdout: "tid": 51298 2023-06-03T20:07:33.632 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:07:33.632 INFO:teuthology.orchestra.run.smithi089.stdout: "events": [ 2023-06-03T20:07:33.632 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:07:33.632 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "initiated", 2023-06-03T20:07:33.632 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:06:43.268011+0000", 2023-06-03T20:07:33.632 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0 2023-06-03T20:07:33.632 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:07:33.632 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:07:33.633 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "throttled", 2023-06-03T20:07:33.633 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:06:43.268011+0000", 2023-06-03T20:07:33.633 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0 2023-06-03T20:07:33.633 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:07:33.633 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:07:33.633 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "header_read", 2023-06-03T20:07:33.633 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:06:43.268010+0000", 2023-06-03T20:07:33.634 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 4294967295.999999 2023-06-03T20:07:33.634 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:07:33.634 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:07:33.634 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "all_read", 2023-06-03T20:07:33.634 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:06:43.268033+0000", 2023-06-03T20:07:33.634 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 2.3215999999999999e-05 2023-06-03T20:07:33.634 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:07:33.634 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:07:33.635 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "dispatched", 2023-06-03T20:07:33.635 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:06:43.268042+0000", 2023-06-03T20:07:33.635 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 8.9579999999999996e-06 2023-06-03T20:07:33.635 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:07:33.635 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:07:33.635 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "queued_for_pg", 2023-06-03T20:07:33.635 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:06:43.268049+0000", 2023-06-03T20:07:33.635 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 6.8630000000000002e-06 2023-06-03T20:07:33.636 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:07:33.636 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:07:33.636 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "reached_pg", 2023-06-03T20:07:33.636 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:06:43.270164+0000", 2023-06-03T20:07:33.636 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.0021152570000000002 2023-06-03T20:07:33.636 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:07:33.636 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:07:33.636 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "started", 2023-06-03T20:07:33.637 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:06:43.270256+0000", 2023-06-03T20:07:33.637 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 9.1512000000000001e-05 2023-06-03T20:07:33.637 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:07:33.637 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:07:33.637 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "waiting for subops from 3,4", 2023-06-03T20:07:33.637 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:06:43.270336+0000", 2023-06-03T20:07:33.637 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 8.0556000000000006e-05 2023-06-03T20:07:33.637 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:07:33.638 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:07:33.638 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "op_commit", 2023-06-03T20:07:33.638 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:06:43.276118+0000", 2023-06-03T20:07:33.638 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.0057811800000000003 2023-06-03T20:07:33.638 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:07:33.638 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:07:33.638 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:07:33.638 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:06:43.465885+0000", 2023-06-03T20:07:33.639 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.18976767999999999 2023-06-03T20:07:33.639 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:07:33.639 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:07:33.639 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:07:33.639 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:06:43.466046+0000", 2023-06-03T20:07:33.639 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.00016026899999999999 2023-06-03T20:07:33.639 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:07:33.640 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:07:33.640 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "commit_sent", 2023-06-03T20:07:33.640 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:06:43.466086+0000", 2023-06-03T20:07:33.640 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 4.0865000000000003e-05 2023-06-03T20:07:33.640 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:07:33.640 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:07:33.640 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "done", 2023-06-03T20:07:33.640 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:06:43.466120+0000", 2023-06-03T20:07:33.641 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 3.3634999999999999e-05 2023-06-03T20:07:33.641 INFO:teuthology.orchestra.run.smithi089.stdout: } 2023-06-03T20:07:33.641 INFO:teuthology.orchestra.run.smithi089.stdout: ] 2023-06-03T20:07:33.641 INFO:teuthology.orchestra.run.smithi089.stdout: } 2023-06-03T20:07:33.641 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:07:33.641 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:07:33.641 INFO:teuthology.orchestra.run.smithi089.stdout: "description": "osd_op(client.36009.0:51299 10.4 10:2da28083:::benchmark_data_smithi089_140876_object51298:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e163)", 2023-06-03T20:07:33.641 INFO:teuthology.orchestra.run.smithi089.stdout: "initiated_at": "2023-06-03T20:06:43.268187+0000", 2023-06-03T20:07:33.642 INFO:teuthology.orchestra.run.smithi089.stdout: "age": 50.215299383999998, 2023-06-03T20:07:33.642 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.19812323100000001, 2023-06-03T20:07:33.642 INFO:teuthology.orchestra.run.smithi089.stdout: "type_data": { 2023-06-03T20:07:33.642 INFO:teuthology.orchestra.run.smithi089.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-03T20:07:33.642 INFO:teuthology.orchestra.run.smithi089.stdout: "client_info": { 2023-06-03T20:07:33.642 INFO:teuthology.orchestra.run.smithi089.stdout: "client": "client.36009", 2023-06-03T20:07:33.642 INFO:teuthology.orchestra.run.smithi089.stdout: "client_addr": "172.21.15.89:0/2550116169", 2023-06-03T20:07:33.643 INFO:teuthology.orchestra.run.smithi089.stdout: "tid": 51299 2023-06-03T20:07:33.643 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:07:33.643 INFO:teuthology.orchestra.run.smithi089.stdout: "events": [ 2023-06-03T20:07:33.643 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:07:33.643 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "initiated", 2023-06-03T20:07:33.643 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:06:43.268187+0000", 2023-06-03T20:07:33.643 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0 2023-06-03T20:07:33.643 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:07:33.644 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:07:33.644 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "throttled", 2023-06-03T20:07:33.644 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:06:43.268187+0000", 2023-06-03T20:07:33.644 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0 2023-06-03T20:07:33.644 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:07:33.644 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:07:33.644 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "header_read", 2023-06-03T20:07:33.644 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:06:43.268186+0000", 2023-06-03T20:07:33.645 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 4294967295.999999 2023-06-03T20:07:33.645 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:07:33.645 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:07:33.645 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "all_read", 2023-06-03T20:07:33.645 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:06:43.268232+0000", 2023-06-03T20:07:33.645 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 4.5856000000000003e-05 2023-06-03T20:07:33.645 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:07:33.645 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:07:33.646 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "dispatched", 2023-06-03T20:07:33.646 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:06:43.268244+0000", 2023-06-03T20:07:33.646 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 1.2199e-05 2023-06-03T20:07:33.646 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:07:33.646 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:07:33.646 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "queued_for_pg", 2023-06-03T20:07:33.646 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:06:43.268258+0000", 2023-06-03T20:07:33.646 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 1.4183e-05 2023-06-03T20:07:33.647 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:07:33.647 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:07:33.647 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "reached_pg", 2023-06-03T20:07:33.647 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:06:43.271888+0000", 2023-06-03T20:07:33.647 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.0036301020000000001 2023-06-03T20:07:33.647 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:07:33.647 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:07:33.647 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "started", 2023-06-03T20:07:33.648 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:06:43.272008+0000", 2023-06-03T20:07:33.648 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.000119814 2023-06-03T20:07:33.648 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:07:33.648 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:07:33.648 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "waiting for subops from 3,4", 2023-06-03T20:07:33.648 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:06:43.272137+0000", 2023-06-03T20:07:33.648 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.000128971 2023-06-03T20:07:33.649 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:07:33.649 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:07:33.649 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "op_commit", 2023-06-03T20:07:33.649 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:06:43.276124+0000", 2023-06-03T20:07:33.649 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.0039871189999999999 2023-06-03T20:07:33.649 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:07:33.649 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:07:33.649 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:07:33.650 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:06:43.465913+0000", 2023-06-03T20:07:33.650 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.189788188 2023-06-03T20:07:33.650 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:07:33.650 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:07:33.650 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:07:33.650 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:06:43.466193+0000", 2023-06-03T20:07:33.650 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.00027996599999999998 2023-06-03T20:07:33.650 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:07:33.651 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:07:33.651 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "commit_sent", 2023-06-03T20:07:33.651 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:06:43.466268+0000", 2023-06-03T20:07:33.651 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 7.5395999999999994e-05 2023-06-03T20:07:33.651 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:07:33.651 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:07:33.651 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "done", 2023-06-03T20:07:33.651 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:06:43.466310+0000", 2023-06-03T20:07:33.652 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 4.2231999999999998e-05 2023-06-03T20:07:33.652 INFO:teuthology.orchestra.run.smithi089.stdout: } 2023-06-03T20:07:33.652 INFO:teuthology.orchestra.run.smithi089.stdout: ] 2023-06-03T20:07:33.652 INFO:teuthology.orchestra.run.smithi089.stdout: } 2023-06-03T20:07:33.652 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:07:33.652 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:07:33.652 INFO:teuthology.orchestra.run.smithi089.stdout: "description": "osd_op(client.36009.0:51300 10.4 10:2719176a:::benchmark_data_smithi089_140876_object51299:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e163)", 2023-06-03T20:07:33.652 INFO:teuthology.orchestra.run.smithi089.stdout: "initiated_at": "2023-06-03T20:06:43.268269+0000", 2023-06-03T20:07:33.653 INFO:teuthology.orchestra.run.smithi089.stdout: "age": 50.215217367999998, 2023-06-03T20:07:33.653 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.19818335100000001, 2023-06-03T20:07:33.653 INFO:teuthology.orchestra.run.smithi089.stdout: "type_data": { 2023-06-03T20:07:33.653 INFO:teuthology.orchestra.run.smithi089.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-03T20:07:33.653 INFO:teuthology.orchestra.run.smithi089.stdout: "client_info": { 2023-06-03T20:07:33.653 INFO:teuthology.orchestra.run.smithi089.stdout: "client": "client.36009", 2023-06-03T20:07:33.653 INFO:teuthology.orchestra.run.smithi089.stdout: "client_addr": "172.21.15.89:0/2550116169", 2023-06-03T20:07:33.654 INFO:teuthology.orchestra.run.smithi089.stdout: "tid": 51300 2023-06-03T20:07:33.654 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:07:33.654 INFO:teuthology.orchestra.run.smithi089.stdout: "events": [ 2023-06-03T20:07:33.654 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:07:33.654 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "initiated", 2023-06-03T20:07:33.654 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:06:43.268269+0000", 2023-06-03T20:07:33.654 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0 2023-06-03T20:07:33.654 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:07:33.655 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:07:33.655 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "throttled", 2023-06-03T20:07:33.655 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:06:43.268269+0000", 2023-06-03T20:07:33.655 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0 2023-06-03T20:07:33.655 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:07:33.655 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:07:33.655 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "header_read", 2023-06-03T20:07:33.655 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:06:43.268268+0000", 2023-06-03T20:07:33.656 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 4294967295.999999 2023-06-03T20:07:33.656 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:07:33.656 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:07:33.656 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "all_read", 2023-06-03T20:07:33.656 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:06:43.270599+0000", 2023-06-03T20:07:33.656 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.0023313330000000001 2023-06-03T20:07:33.656 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:07:33.656 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:07:33.657 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "dispatched", 2023-06-03T20:07:33.657 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:06:43.270611+0000", 2023-06-03T20:07:33.657 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 1.1484000000000001e-05 2023-06-03T20:07:33.657 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:07:33.657 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:07:33.657 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "queued_for_pg", 2023-06-03T20:07:33.657 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:06:43.270618+0000", 2023-06-03T20:07:33.657 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 6.7490000000000001e-06 2023-06-03T20:07:33.658 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:07:33.658 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:07:33.658 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "reached_pg", 2023-06-03T20:07:33.658 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:06:43.274257+0000", 2023-06-03T20:07:33.658 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.0036394660000000001 2023-06-03T20:07:33.658 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:07:33.658 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:07:33.659 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "started", 2023-06-03T20:07:33.659 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:06:43.274391+0000", 2023-06-03T20:07:33.659 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.00013391 2023-06-03T20:07:33.659 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:07:33.659 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:07:33.659 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "waiting for subops from 3,4", 2023-06-03T20:07:33.659 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:06:43.274525+0000", 2023-06-03T20:07:33.659 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.00013420700000000001 2023-06-03T20:07:33.660 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:07:33.660 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:07:33.660 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "op_commit", 2023-06-03T20:07:33.660 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:06:43.276268+0000", 2023-06-03T20:07:33.660 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.0017431429999999999 2023-06-03T20:07:33.660 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:07:33.660 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:07:33.660 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:07:33.661 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:06:43.465985+0000", 2023-06-03T20:07:33.661 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.18971680799999999 2023-06-03T20:07:33.661 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:07:33.661 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:07:33.661 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:07:33.661 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:06:43.466357+0000", 2023-06-03T20:07:33.661 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.00037210299999999998 2023-06-03T20:07:33.661 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:07:33.662 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:07:33.662 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "commit_sent", 2023-06-03T20:07:33.662 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:06:43.466418+0000", 2023-06-03T20:07:33.662 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 6.0415999999999999e-05 2023-06-03T20:07:33.662 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:07:33.662 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:07:33.662 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "done", 2023-06-03T20:07:33.662 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:06:43.466452+0000", 2023-06-03T20:07:33.663 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 3.4527999999999999e-05 2023-06-03T20:07:33.663 INFO:teuthology.orchestra.run.smithi089.stdout: } 2023-06-03T20:07:33.663 INFO:teuthology.orchestra.run.smithi089.stdout: ] 2023-06-03T20:07:33.663 INFO:teuthology.orchestra.run.smithi089.stdout: } 2023-06-03T20:07:33.663 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:07:33.663 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:07:33.663 INFO:teuthology.orchestra.run.smithi089.stdout: "description": "osd_op(client.36009.0:93787 10.8 10:1ed0e12f:::benchmark_data_smithi089_140876_object93786:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e170)", 2023-06-03T20:07:33.664 INFO:teuthology.orchestra.run.smithi089.stdout: "initiated_at": "2023-06-03T20:07:27.763800+0000", 2023-06-03T20:07:33.664 INFO:teuthology.orchestra.run.smithi089.stdout: "age": 5.7196862350000002, 2023-06-03T20:07:33.664 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.24366188, 2023-06-03T20:07:33.664 INFO:teuthology.orchestra.run.smithi089.stdout: "type_data": { 2023-06-03T20:07:33.664 INFO:teuthology.orchestra.run.smithi089.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-03T20:07:33.664 INFO:teuthology.orchestra.run.smithi089.stdout: "client_info": { 2023-06-03T20:07:33.664 INFO:teuthology.orchestra.run.smithi089.stdout: "client": "client.36009", 2023-06-03T20:07:33.664 INFO:teuthology.orchestra.run.smithi089.stdout: "client_addr": "172.21.15.89:0/2550116169", 2023-06-03T20:07:33.665 INFO:teuthology.orchestra.run.smithi089.stdout: "tid": 93787 2023-06-03T20:07:33.665 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:07:33.665 INFO:teuthology.orchestra.run.smithi089.stdout: "events": [ 2023-06-03T20:07:33.665 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:07:33.665 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "initiated", 2023-06-03T20:07:33.665 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:07:27.763800+0000", 2023-06-03T20:07:33.665 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0 2023-06-03T20:07:33.666 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:07:33.666 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:07:33.666 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "throttled", 2023-06-03T20:07:33.666 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:07:27.763800+0000", 2023-06-03T20:07:33.666 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0 2023-06-03T20:07:33.666 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:07:33.666 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:07:33.666 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "header_read", 2023-06-03T20:07:33.667 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:07:27.763798+0000", 2023-06-03T20:07:33.667 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 4294967295.9999981 2023-06-03T20:07:33.667 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:07:33.667 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:07:33.667 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "all_read", 2023-06-03T20:07:33.667 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:07:27.763823+0000", 2023-06-03T20:07:33.667 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 2.4756000000000002e-05 2023-06-03T20:07:33.667 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:07:33.668 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:07:33.668 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "dispatched", 2023-06-03T20:07:33.668 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:07:27.763834+0000", 2023-06-03T20:07:33.668 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 1.0482999999999999e-05 2023-06-03T20:07:33.668 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:07:33.668 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:07:33.668 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "queued_for_pg", 2023-06-03T20:07:33.668 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:07:27.763840+0000", 2023-06-03T20:07:33.669 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 6.2790000000000004e-06 2023-06-03T20:07:33.669 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:07:33.669 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:07:33.669 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "reached_pg", 2023-06-03T20:07:33.669 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:07:27.763890+0000", 2023-06-03T20:07:33.669 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 5.0318000000000002e-05 2023-06-03T20:07:33.669 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:07:33.669 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:07:33.670 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "started", 2023-06-03T20:07:33.670 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:07:27.764008+0000", 2023-06-03T20:07:33.670 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.000118196 2023-06-03T20:07:33.670 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:07:33.670 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:07:33.670 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "waiting for subops from 2,7", 2023-06-03T20:07:33.670 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:07:27.764108+0000", 2023-06-03T20:07:33.670 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 9.9907000000000004e-05 2023-06-03T20:07:33.671 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:07:33.671 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:07:33.671 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:07:33.671 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:07:27.767246+0000", 2023-06-03T20:07:33.671 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.0031374509999999999 2023-06-03T20:07:33.671 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:07:33.671 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:07:33.672 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "op_commit", 2023-06-03T20:07:33.672 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:07:27.767285+0000", 2023-06-03T20:07:33.672 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 3.9320000000000003e-05 2023-06-03T20:07:33.672 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:07:33.672 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:07:33.672 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:07:33.672 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:07:28.007200+0000", 2023-06-03T20:07:33.672 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.23991483299999999 2023-06-03T20:07:33.673 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:07:33.673 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:07:33.673 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "commit_sent", 2023-06-03T20:07:33.673 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:07:28.007254+0000", 2023-06-03T20:07:33.673 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 5.4221999999999997e-05 2023-06-03T20:07:33.673 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:07:33.673 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:07:33.673 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "done", 2023-06-03T20:07:33.674 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:07:28.007462+0000", 2023-06-03T20:07:33.674 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.000207911 2023-06-03T20:07:33.674 INFO:teuthology.orchestra.run.smithi089.stdout: } 2023-06-03T20:07:33.674 INFO:teuthology.orchestra.run.smithi089.stdout: ] 2023-06-03T20:07:33.674 INFO:teuthology.orchestra.run.smithi089.stdout: } 2023-06-03T20:07:33.674 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:07:33.674 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:07:33.674 INFO:teuthology.orchestra.run.smithi089.stdout: "description": "osd_repop(client.36009.0:93788 10.d e170/157 10:bfe7c435:::benchmark_data_smithi089_140876_object93787:head v 170'5945, mlcod=170'5942)", 2023-06-03T20:07:33.675 INFO:teuthology.orchestra.run.smithi089.stdout: "initiated_at": "2023-06-03T20:07:27.769502+0000", 2023-06-03T20:07:33.675 INFO:teuthology.orchestra.run.smithi089.stdout: "age": 5.7139840800000004, 2023-06-03T20:07:33.675 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.23818402999999999, 2023-06-03T20:07:33.675 INFO:teuthology.orchestra.run.smithi089.stdout: "type_data": { 2023-06-03T20:07:33.675 INFO:teuthology.orchestra.run.smithi089.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-03T20:07:33.675 INFO:teuthology.orchestra.run.smithi089.stdout: "events": [ 2023-06-03T20:07:33.675 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:07:33.676 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "initiated", 2023-06-03T20:07:33.676 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:07:27.769502+0000", 2023-06-03T20:07:33.676 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0 2023-06-03T20:07:33.676 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:07:33.676 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:07:33.676 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "throttled", 2023-06-03T20:07:33.676 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:07:27.769502+0000", 2023-06-03T20:07:33.676 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0 2023-06-03T20:07:33.677 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:07:33.677 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:07:33.677 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "header_read", 2023-06-03T20:07:33.677 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:07:27.769502+0000", 2023-06-03T20:07:33.677 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 4294967295.9999995 2023-06-03T20:07:33.677 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:07:33.677 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:07:33.677 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "all_read", 2023-06-03T20:07:33.678 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:07:27.769566+0000", 2023-06-03T20:07:33.678 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 6.4240999999999997e-05 2023-06-03T20:07:33.678 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:07:33.678 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:07:33.678 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "dispatched", 2023-06-03T20:07:33.678 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:07:27.769575+0000", 2023-06-03T20:07:33.678 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 8.7579999999999998e-06 2023-06-03T20:07:33.678 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:07:33.679 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:07:33.679 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "queued_for_pg", 2023-06-03T20:07:33.679 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:07:27.769580+0000", 2023-06-03T20:07:33.679 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 5.6200000000000004e-06 2023-06-03T20:07:33.679 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:07:33.679 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:07:33.679 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "reached_pg", 2023-06-03T20:07:33.680 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:07:27.769626+0000", 2023-06-03T20:07:33.680 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 4.5466000000000001e-05 2023-06-03T20:07:33.680 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:07:33.680 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:07:33.680 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "started", 2023-06-03T20:07:33.680 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:07:27.769644+0000", 2023-06-03T20:07:33.680 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 1.8671000000000001e-05 2023-06-03T20:07:33.680 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:07:33.681 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:07:33.681 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "commit_sent", 2023-06-03T20:07:33.681 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:07:28.007643+0000", 2023-06-03T20:07:33.681 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.23799832800000001 2023-06-03T20:07:33.681 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:07:33.681 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:07:33.681 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "done", 2023-06-03T20:07:33.681 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:07:28.007686+0000", 2023-06-03T20:07:33.682 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 4.3526999999999998e-05 2023-06-03T20:07:33.682 INFO:teuthology.orchestra.run.smithi089.stdout: } 2023-06-03T20:07:33.682 INFO:teuthology.orchestra.run.smithi089.stdout: ] 2023-06-03T20:07:33.682 INFO:teuthology.orchestra.run.smithi089.stdout: } 2023-06-03T20:07:33.682 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:07:33.682 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:07:33.682 INFO:teuthology.orchestra.run.smithi089.stdout: "description": "osd_op(client.36009.0:93794 10.4 10:2bc5e295:::benchmark_data_smithi089_140876_object93793:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e170)", 2023-06-03T20:07:33.682 INFO:teuthology.orchestra.run.smithi089.stdout: "initiated_at": "2023-06-03T20:07:27.771618+0000", 2023-06-03T20:07:33.683 INFO:teuthology.orchestra.run.smithi089.stdout: "age": 5.711868043, 2023-06-03T20:07:33.683 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.24791359499999999, 2023-06-03T20:07:33.683 INFO:teuthology.orchestra.run.smithi089.stdout: "type_data": { 2023-06-03T20:07:33.683 INFO:teuthology.orchestra.run.smithi089.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-03T20:07:33.683 INFO:teuthology.orchestra.run.smithi089.stdout: "client_info": { 2023-06-03T20:07:33.683 INFO:teuthology.orchestra.run.smithi089.stdout: "client": "client.36009", 2023-06-03T20:07:33.683 INFO:teuthology.orchestra.run.smithi089.stdout: "client_addr": "172.21.15.89:0/2550116169", 2023-06-03T20:07:33.683 INFO:teuthology.orchestra.run.smithi089.stdout: "tid": 93794 2023-06-03T20:07:33.684 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:07:33.684 INFO:teuthology.orchestra.run.smithi089.stdout: "events": [ 2023-06-03T20:07:33.684 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:07:33.684 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "initiated", 2023-06-03T20:07:33.684 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:07:27.771618+0000", 2023-06-03T20:07:33.684 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0 2023-06-03T20:07:33.684 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:07:33.685 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:07:33.685 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "throttled", 2023-06-03T20:07:33.685 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:07:27.771618+0000", 2023-06-03T20:07:33.685 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0 2023-06-03T20:07:33.685 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:07:33.685 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:07:33.685 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "header_read", 2023-06-03T20:07:33.685 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:07:27.771616+0000", 2023-06-03T20:07:33.686 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 4294967295.9999976 2023-06-03T20:07:33.686 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:07:33.686 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:07:33.686 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "all_read", 2023-06-03T20:07:33.686 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:07:27.771653+0000", 2023-06-03T20:07:33.686 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 3.7571000000000003e-05 2023-06-03T20:07:33.686 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:07:33.686 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:07:33.687 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "dispatched", 2023-06-03T20:07:33.687 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:07:27.771667+0000", 2023-06-03T20:07:33.687 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 1.3981999999999999e-05 2023-06-03T20:07:33.687 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:07:33.687 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:07:33.687 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "queued_for_pg", 2023-06-03T20:07:33.687 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:07:27.771674+0000", 2023-06-03T20:07:33.687 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 6.7129999999999999e-06 2023-06-03T20:07:33.688 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:07:33.688 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:07:33.688 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "reached_pg", 2023-06-03T20:07:33.688 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:07:27.771717+0000", 2023-06-03T20:07:33.688 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 4.3219000000000001e-05 2023-06-03T20:07:33.688 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:07:33.688 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:07:33.689 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "started", 2023-06-03T20:07:33.689 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:07:27.771848+0000", 2023-06-03T20:07:33.689 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.00013075399999999999 2023-06-03T20:07:33.689 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:07:33.689 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:07:33.689 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "waiting for subops from 3,4", 2023-06-03T20:07:33.689 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:07:27.771986+0000", 2023-06-03T20:07:33.689 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.000138145 2023-06-03T20:07:33.690 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:07:33.690 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:07:33.690 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:07:33.690 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:07:28.011224+0000", 2023-06-03T20:07:33.690 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.23923821100000001 2023-06-03T20:07:33.690 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:07:33.690 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:07:33.690 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:07:33.691 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:07:28.011294+0000", 2023-06-03T20:07:33.691 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 7.0158000000000002e-05 2023-06-03T20:07:33.691 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:07:33.691 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:07:33.691 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "op_commit", 2023-06-03T20:07:33.691 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:07:28.019454+0000", 2023-06-03T20:07:33.691 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.0081596599999999991 2023-06-03T20:07:33.692 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:07:33.692 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:07:33.692 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "commit_sent", 2023-06-03T20:07:33.692 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:07:28.019491+0000", 2023-06-03T20:07:33.692 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 3.6511000000000003e-05 2023-06-03T20:07:33.692 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:07:33.692 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:07:33.692 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "done", 2023-06-03T20:07:33.693 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:07:28.019532+0000", 2023-06-03T20:07:33.693 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 4.1196999999999999e-05 2023-06-03T20:07:33.693 INFO:teuthology.orchestra.run.smithi089.stdout: } 2023-06-03T20:07:33.693 INFO:teuthology.orchestra.run.smithi089.stdout: ] 2023-06-03T20:07:33.693 INFO:teuthology.orchestra.run.smithi089.stdout: } 2023-06-03T20:07:33.693 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:07:33.693 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:07:33.693 INFO:teuthology.orchestra.run.smithi089.stdout: "description": "osd_repop_reply(client.36009.0:93787 10.8 e170/157 ondisk, result = 0)", 2023-06-03T20:07:33.694 INFO:teuthology.orchestra.run.smithi089.stdout: "initiated_at": "2023-06-03T20:07:27.773023+0000", 2023-06-03T20:07:33.694 INFO:teuthology.orchestra.run.smithi089.stdout: "age": 5.7104635679999998, 2023-06-03T20:07:33.694 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.234450609, 2023-06-03T20:07:33.694 INFO:teuthology.orchestra.run.smithi089.stdout: "type_data": { 2023-06-03T20:07:33.694 INFO:teuthology.orchestra.run.smithi089.stdout: "flag_point": "started", 2023-06-03T20:07:33.694 INFO:teuthology.orchestra.run.smithi089.stdout: "events": [ 2023-06-03T20:07:33.694 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:07:33.694 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "initiated", 2023-06-03T20:07:33.695 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:07:27.773023+0000", 2023-06-03T20:07:33.695 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0 2023-06-03T20:07:33.695 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:07:33.695 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:07:33.695 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "throttled", 2023-06-03T20:07:33.695 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:07:27.773023+0000", 2023-06-03T20:07:33.695 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0 2023-06-03T20:07:33.695 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:07:33.696 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:07:33.696 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "header_read", 2023-06-03T20:07:33.696 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:07:27.773022+0000", 2023-06-03T20:07:33.696 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 4294967295.999999 2023-06-03T20:07:33.696 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:07:33.696 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:07:33.696 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "all_read", 2023-06-03T20:07:33.697 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:07:27.773030+0000", 2023-06-03T20:07:33.697 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 7.7319999999999998e-06 2023-06-03T20:07:33.697 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:07:33.697 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:07:33.697 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "dispatched", 2023-06-03T20:07:33.697 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:07:28.006956+0000", 2023-06-03T20:07:33.697 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.233926412 2023-06-03T20:07:33.697 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:07:33.698 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:07:33.698 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "queued_for_pg", 2023-06-03T20:07:33.698 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:07:28.006980+0000", 2023-06-03T20:07:33.698 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 2.4019000000000002e-05 2023-06-03T20:07:33.698 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:07:33.698 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:07:33.698 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "reached_pg", 2023-06-03T20:07:33.698 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:07:28.007174+0000", 2023-06-03T20:07:33.699 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.000194413 2023-06-03T20:07:33.699 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:07:33.699 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:07:33.699 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "started", 2023-06-03T20:07:33.699 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:07:28.007189+0000", 2023-06-03T20:07:33.699 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 1.4314000000000001e-05 2023-06-03T20:07:33.699 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:07:33.699 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:07:33.700 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "done", 2023-06-03T20:07:33.700 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:07:28.007473+0000", 2023-06-03T20:07:33.700 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.00028461900000000001 2023-06-03T20:07:33.700 INFO:teuthology.orchestra.run.smithi089.stdout: } 2023-06-03T20:07:33.700 INFO:teuthology.orchestra.run.smithi089.stdout: ] 2023-06-03T20:07:33.700 INFO:teuthology.orchestra.run.smithi089.stdout: } 2023-06-03T20:07:33.700 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:07:33.700 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:07:33.701 INFO:teuthology.orchestra.run.smithi089.stdout: "description": "osd_op(client.36009.0:93797 10.4 10:2ab04804:::benchmark_data_smithi089_140876_object93796:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e170)", 2023-06-03T20:07:33.701 INFO:teuthology.orchestra.run.smithi089.stdout: "initiated_at": "2023-06-03T20:07:27.774857+0000", 2023-06-03T20:07:33.701 INFO:teuthology.orchestra.run.smithi089.stdout: "age": 5.7086292670000001, 2023-06-03T20:07:33.701 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.248770939, 2023-06-03T20:07:33.701 INFO:teuthology.orchestra.run.smithi089.stdout: "type_data": { 2023-06-03T20:07:33.701 INFO:teuthology.orchestra.run.smithi089.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-03T20:07:33.701 INFO:teuthology.orchestra.run.smithi089.stdout: "client_info": { 2023-06-03T20:07:33.701 INFO:teuthology.orchestra.run.smithi089.stdout: "client": "client.36009", 2023-06-03T20:07:33.702 INFO:teuthology.orchestra.run.smithi089.stdout: "client_addr": "172.21.15.89:0/2550116169", 2023-06-03T20:07:33.702 INFO:teuthology.orchestra.run.smithi089.stdout: "tid": 93797 2023-06-03T20:07:33.702 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:07:33.702 INFO:teuthology.orchestra.run.smithi089.stdout: "events": [ 2023-06-03T20:07:33.702 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:07:33.702 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "initiated", 2023-06-03T20:07:33.702 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:07:27.774857+0000", 2023-06-03T20:07:33.703 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0 2023-06-03T20:07:33.703 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:07:33.703 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:07:33.703 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "throttled", 2023-06-03T20:07:33.703 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:07:27.774857+0000", 2023-06-03T20:07:33.703 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0 2023-06-03T20:07:33.703 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:07:33.703 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:07:33.704 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "header_read", 2023-06-03T20:07:33.704 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:07:27.774855+0000", 2023-06-03T20:07:33.704 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 4294967295.9999986 2023-06-03T20:07:33.704 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:07:33.704 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:07:33.704 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "all_read", 2023-06-03T20:07:33.704 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:07:27.774880+0000", 2023-06-03T20:07:33.704 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 2.4372e-05 2023-06-03T20:07:33.705 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:07:33.705 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:07:33.705 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "dispatched", 2023-06-03T20:07:33.705 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:07:28.006970+0000", 2023-06-03T20:07:33.705 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.23209026099999999 2023-06-03T20:07:33.705 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:07:33.705 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:07:33.705 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "queued_for_pg", 2023-06-03T20:07:33.706 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:07:28.006990+0000", 2023-06-03T20:07:33.706 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 2.0344999999999999e-05 2023-06-03T20:07:33.706 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:07:33.706 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:07:33.706 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "reached_pg", 2023-06-03T20:07:33.706 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:07:28.011461+0000", 2023-06-03T20:07:33.706 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.0044709290000000002 2023-06-03T20:07:33.706 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:07:33.707 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:07:33.707 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "started", 2023-06-03T20:07:33.707 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:07:28.011655+0000", 2023-06-03T20:07:33.707 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.000193969 2023-06-03T20:07:33.707 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:07:33.707 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:07:33.707 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "waiting for subops from 3,4", 2023-06-03T20:07:33.707 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:07:28.011814+0000", 2023-06-03T20:07:33.708 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.00015902200000000001 2023-06-03T20:07:33.708 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:07:33.708 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:07:33.708 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:07:33.708 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:07:28.023473+0000", 2023-06-03T20:07:33.708 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.011658236000000001 2023-06-03T20:07:33.708 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:07:33.709 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:07:33.709 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "op_commit", 2023-06-03T20:07:33.709 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:07:28.023484+0000", 2023-06-03T20:07:33.709 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 1.1401e-05 2023-06-03T20:07:33.709 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:07:33.709 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:07:33.709 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:07:33.709 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:07:28.023531+0000", 2023-06-03T20:07:33.710 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 4.7058000000000002e-05 2023-06-03T20:07:33.710 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:07:33.710 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:07:33.710 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "commit_sent", 2023-06-03T20:07:33.710 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:07:28.023581+0000", 2023-06-03T20:07:33.710 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 5.0064e-05 2023-06-03T20:07:33.710 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:07:33.710 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:07:33.711 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "done", 2023-06-03T20:07:33.711 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:07:28.023628+0000", 2023-06-03T20:07:33.711 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 4.6897999999999997e-05 2023-06-03T20:07:33.711 INFO:teuthology.orchestra.run.smithi089.stdout: } 2023-06-03T20:07:33.711 INFO:teuthology.orchestra.run.smithi089.stdout: ] 2023-06-03T20:07:33.711 INFO:teuthology.orchestra.run.smithi089.stdout: } 2023-06-03T20:07:33.711 INFO:teuthology.orchestra.run.smithi089.stdout: } 2023-06-03T20:07:33.711 INFO:teuthology.orchestra.run.smithi089.stdout: ] 2023-06-03T20:07:33.712 INFO:teuthology.orchestra.run.smithi089.stdout:} 2023-06-03T20:07:33.712 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:07:33 smithi164 bash[59462]: cluster 1685822852.5856042 mgr.y (mgr.34104) 942 : cluster 0 pgmap v886: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 5.8 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 38 MiB/s wr, 612 op/s; 11312/284415 objects degraded (3.977%); 5329/284415 objects misplaced (1.874%); 759 KiB/s, 12 objects/s recovering 2023-06-03T20:07:33.712 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:07:33 smithi164 bash[59462]: audit 1685822852.70247 mon.a (mon.0) 855 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:07:33.741 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:07:33.745 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:07:33.745 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:07:33.846 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:07:33.884 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:07:33.887 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:07:33.887 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:07:33.983 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:07:33 smithi089 bash[78452]: cluster 1685822852.5856042 mgr.y (mgr.34104) 942 : cluster 0 pgmap v886: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 5.8 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 38 MiB/s wr, 612 op/s; 11312/284415 objects degraded (3.977%); 5329/284415 objects misplaced (1.874%); 759 KiB/s, 12 objects/s recovering 2023-06-03T20:07:33.983 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:07:33 smithi089 bash[78452]: audit 1685822852.70247 mon.a (mon.0) 855 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:07:33.983 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:07:33 smithi089 bash[75068]: cluster 1685822852.5856042 mgr.y (mgr.34104) 942 : cluster 0 pgmap v886: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 5.8 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 38 MiB/s wr, 612 op/s; 11312/284415 objects degraded (3.977%); 5329/284415 objects misplaced (1.874%); 759 KiB/s, 12 objects/s recovering 2023-06-03T20:07:33.983 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:07:33 smithi089 bash[75068]: audit 1685822852.70247 mon.a (mon.0) 855 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:07:33.989 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:07:34.017 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:07:34.019 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:07:34.020 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:07:34.121 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:07:34.161 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:07:34.164 INFO:teuthology.orchestra.run.smithi089.stdout:osd.0: {} 2023-06-03T20:07:34.165 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:07:34.166 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:07:34.198 INFO:teuthology.orchestra.run.smithi089.stdout:osd.1: {} 2023-06-03T20:07:34.224 INFO:teuthology.orchestra.run.smithi089.stdout:osd.2: {} 2023-06-03T20:07:34.244 INFO:tasks.radosbench.radosbench.0.smithi089.stdout:2023-06-03T20:07:34.240425+0000 min lat: 0.00128391 max lat: 2.52292 avg lat: 0.0130242 2023-06-03T20:07:34.244 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-06-03T20:07:34.244 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 80 16 98250 98234 76.7313 43.375 0.0128699 0.0130242 2023-06-03T20:07:34.254 INFO:teuthology.orchestra.run.smithi089.stdout:osd.3: {} 2023-06-03T20:07:34.267 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:07:34.290 INFO:teuthology.orchestra.run.smithi089.stdout:osd.4: {} 2023-06-03T20:07:34.305 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:07:34.309 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:07:34.310 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:07:34.325 INFO:teuthology.orchestra.run.smithi089.stdout:osd.5: {} 2023-06-03T20:07:34.361 INFO:teuthology.orchestra.run.smithi089.stdout:osd.6: {} 2023-06-03T20:07:34.388 INFO:teuthology.orchestra.run.smithi089.stdout:osd.7: {} 2023-06-03T20:07:34.389 INFO:teuthology.orchestra.run.smithi089.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-06-03T20:07:34.389 INFO:teuthology.orchestra.run.smithi089.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-06-03T20:07:34.390 INFO:teuthology.orchestra.run.smithi089.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-06-03T20:07:34.390 INFO:teuthology.orchestra.run.smithi089.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-06-03T20:07:34.390 INFO:teuthology.orchestra.run.smithi089.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-06-03T20:07:34.390 INFO:teuthology.orchestra.run.smithi089.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-06-03T20:07:34.390 INFO:teuthology.orchestra.run.smithi089.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-06-03T20:07:34.390 INFO:teuthology.orchestra.run.smithi089.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-06-03T20:07:34.411 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:07:34.457 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:07:34.463 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:07:34.464 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:07:34.565 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:07:34.599 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:07:34.603 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:07:34.604 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:07:34.705 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:07:34.749 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:07:34.753 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:07:34.754 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:07:34.778 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:07:34 smithi089 bash[78452]: audit 1685822854.1092923 mon.c (mon.2) 451 : audit 0 from='client.? 172.21.15.89:0/3740150616' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-03T20:07:34.778 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:07:34 smithi089 bash[75068]: audit 1685822854.1092923 mon.c (mon.2) 451 : audit 0 from='client.? 172.21.15.89:0/3740150616' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-03T20:07:34.855 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:07:34.898 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:07:34.900 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:07:34.902 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:07:35.004 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:07:35.013 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:07:34 smithi164 bash[59462]: audit 1685822854.1092923 mon.c (mon.2) 451 : audit 0 from='client.? 172.21.15.89:0/3740150616' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-03T20:07:35.055 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:07:35.058 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:07:35.058 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:07:35.159 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:07:35.204 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:07:35.209 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:07:35.210 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:07:35.244 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 81 16 98928 98912 76.3072 42.375 0.0462578 0.0130994 2023-06-03T20:07:35.312 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:07:35.349 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:07:35.353 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:07:35.353 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:07:35.454 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:07:35.500 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:07:35.502 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:07:35.503 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:07:35.604 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:07:35.644 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:07:35.649 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:07:35.650 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:07:35.741 INFO:teuthology.orchestra.run.smithi089.stderr:noscrub is unset 2023-06-03T20:07:35.752 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:07:35.794 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:07:35.797 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:07:35.798 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:07:35.899 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:07:35.933 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:07:35.936 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:07:35.936 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:07:36.028 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:07:35 smithi089 bash[78452]: cluster 1685822854.586189 mgr.y (mgr.34104) 943 : cluster 0 pgmap v887: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 5.9 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 40 MiB/s wr, 636 op/s; 11312/290532 objects degraded (3.894%); 5254/290532 objects misplaced (1.808%); 924 KiB/s, 14 objects/s recovering 2023-06-03T20:07:36.029 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:07:35 smithi089 bash[78452]: cluster 1685822854.7021468 mon.a (mon.0) 856 : cluster 3 Health check update: Degraded data redundancy: 11312/290532 objects degraded (3.894%), 2 pgs degraded (PG_DEGRADED) 2023-06-03T20:07:36.029 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:07:35 smithi089 bash[78452]: audit 1685822855.101218 mon.a (mon.0) 857 : audit 1 from='client.? 172.21.15.89:0/506321390' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-06-03T20:07:36.029 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:07:35 smithi089 bash[75068]: cluster 1685822854.586189 mgr.y (mgr.34104) 943 : cluster 0 pgmap v887: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 5.9 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 40 MiB/s wr, 636 op/s; 11312/290532 objects degraded (3.894%); 5254/290532 objects misplaced (1.808%); 924 KiB/s, 14 objects/s recovering 2023-06-03T20:07:36.030 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:07:35 smithi089 bash[75068]: cluster 1685822854.7021468 mon.a (mon.0) 856 : cluster 3 Health check update: Degraded data redundancy: 11312/290532 objects degraded (3.894%), 2 pgs degraded (PG_DEGRADED) 2023-06-03T20:07:36.030 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:07:35 smithi089 bash[75068]: audit 1685822855.101218 mon.a (mon.0) 857 : audit 1 from='client.? 172.21.15.89:0/506321390' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-06-03T20:07:36.037 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:07:36.046 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:07:35 smithi164 bash[59462]: cluster 1685822854.586189 mgr.y (mgr.34104) 943 : cluster 0 pgmap v887: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 5.9 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 40 MiB/s wr, 636 op/s; 11312/290532 objects degraded (3.894%); 5254/290532 objects misplaced (1.808%); 924 KiB/s, 14 objects/s recovering 2023-06-03T20:07:36.046 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:07:35 smithi164 bash[59462]: cluster 1685822854.7021468 mon.a (mon.0) 856 : cluster 3 Health check update: Degraded data redundancy: 11312/290532 objects degraded (3.894%), 2 pgs degraded (PG_DEGRADED) 2023-06-03T20:07:36.046 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:07:35 smithi164 bash[59462]: audit 1685822855.101218 mon.a (mon.0) 857 : audit 1 from='client.? 172.21.15.89:0/506321390' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-06-03T20:07:36.066 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:07:36.067 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:07:36.068 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:07:36.090 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph daemon osd.1 dump_ops_in_flight 2023-06-03T20:07:36.169 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:07:36.209 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:07:36.213 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:07:36.213 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:07:36.244 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 82 16 99628 99612 75.9101 43.75 0.0111621 0.0131667 2023-06-03T20:07:36.315 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:07:36.324 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 20:07:36 smithi089 bash[71163]: ::ffff:172.21.15.164 - - [03/Jun/2023:20:07:36] "GET /metrics HTTP/1.1" 200 228742 "" "Prometheus/2.33.4" 2023-06-03T20:07:36.364 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:07:36.370 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:07:36.371 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:07:36.472 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:07:36.508 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:07:36.514 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:07:36.514 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:07:36.615 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:07:36.649 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:07:36.651 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:07:36.651 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:07:36.752 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:07:36.763 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:07:36 smithi164 bash[59462]: cluster 1685822855.7147882 mon.a (mon.0) 858 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-06-03T20:07:36.763 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:07:36 smithi164 bash[59462]: audit 1685822855.7293398 mon.a (mon.0) 859 : audit 1 from='client.? 172.21.15.89:0/506321390' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-06-03T20:07:36.763 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:07:36 smithi164 bash[59462]: cluster 1685822855.7293959 mon.a (mon.0) 860 : cluster 0 osdmap e171: 8 total, 8 up, 7 in 2023-06-03T20:07:36.764 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:07:36 smithi164 bash[59462]: cluster 1685822855.732098 osd.6 (osd.6) 106 : cluster 3 failed to encode map e171 with expected crc 2023-06-03T20:07:36.764 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:07:36 smithi164 bash[59462]: cluster 1685822855.7339213 osd.6 (osd.6) 107 : cluster 3 failed to encode map e171 with expected crc 2023-06-03T20:07:36.764 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:07:36 smithi164 bash[59462]: cluster 1685822855.7341163 osd.5 (osd.5) 102 : cluster 3 failed to encode map e171 with expected crc 2023-06-03T20:07:36.765 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:07:36 smithi164 bash[59462]: cluster 1685822855.734515 osd.7 (osd.7) 105 : cluster 3 failed to encode map e171 with expected crc 2023-06-03T20:07:36.786 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:07:36.793 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:07:36.794 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:07:36.895 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:07:36.949 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:07:36.954 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:07:36.954 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:07:37.028 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:07:36 smithi089 bash[75068]: cluster 1685822855.7147882 mon.a (mon.0) 858 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-06-03T20:07:37.029 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:07:36 smithi089 bash[75068]: audit 1685822855.7293398 mon.a (mon.0) 859 : audit 1 from='client.? 172.21.15.89:0/506321390' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-06-03T20:07:37.029 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:07:36 smithi089 bash[75068]: cluster 1685822855.7293959 mon.a (mon.0) 860 : cluster 0 osdmap e171: 8 total, 8 up, 7 in 2023-06-03T20:07:37.029 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:07:36 smithi089 bash[75068]: cluster 1685822855.732098 osd.6 (osd.6) 106 : cluster 3 failed to encode map e171 with expected crc 2023-06-03T20:07:37.029 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:07:36 smithi089 bash[75068]: cluster 1685822855.7339213 osd.6 (osd.6) 107 : cluster 3 failed to encode map e171 with expected crc 2023-06-03T20:07:37.029 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:07:36 smithi089 bash[75068]: cluster 1685822855.7341163 osd.5 (osd.5) 102 : cluster 3 failed to encode map e171 with expected crc 2023-06-03T20:07:37.029 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:07:36 smithi089 bash[75068]: cluster 1685822855.734515 osd.7 (osd.7) 105 : cluster 3 failed to encode map e171 with expected crc 2023-06-03T20:07:37.030 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:07:36 smithi089 bash[78452]: cluster 1685822855.7147882 mon.a (mon.0) 858 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-06-03T20:07:37.030 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:07:36 smithi089 bash[78452]: audit 1685822855.7293398 mon.a (mon.0) 859 : audit 1 from='client.? 172.21.15.89:0/506321390' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-06-03T20:07:37.030 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:07:36 smithi089 bash[78452]: cluster 1685822855.7293959 mon.a (mon.0) 860 : cluster 0 osdmap e171: 8 total, 8 up, 7 in 2023-06-03T20:07:37.030 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:07:36 smithi089 bash[78452]: cluster 1685822855.732098 osd.6 (osd.6) 106 : cluster 3 failed to encode map e171 with expected crc 2023-06-03T20:07:37.030 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:07:36 smithi089 bash[78452]: cluster 1685822855.7339213 osd.6 (osd.6) 107 : cluster 3 failed to encode map e171 with expected crc 2023-06-03T20:07:37.030 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:07:36 smithi089 bash[78452]: cluster 1685822855.7341163 osd.5 (osd.5) 102 : cluster 3 failed to encode map e171 with expected crc 2023-06-03T20:07:37.030 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:07:36 smithi089 bash[78452]: cluster 1685822855.734515 osd.7 (osd.7) 105 : cluster 3 failed to encode map e171 with expected crc 2023-06-03T20:07:37.055 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:07:37.105 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:07:37.108 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:07:37.108 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:07:37.210 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:07:37.244 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 83 16 100341 100325 75.5324 44.5625 0.035571 0.013233 2023-06-03T20:07:37.252 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:07:37.260 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:07:37.261 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:07:37.362 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:07:37.387 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:07:37.391 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:07:37.391 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:07:37.492 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:07:37.503 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/mon.a/config 2023-06-03T20:07:37.535 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:07:37.538 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:07:37.539 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:07:37.640 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:07:37.689 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:07:37.696 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:07:37.696 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:07:37.797 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:07:37.836 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:07:37.841 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:07:37.842 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:07:37.944 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:07:37.954 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:07:37 smithi164 bash[59462]: cluster 1685822856.5868235 mgr.y (mgr.34104) 944 : cluster 0 pgmap v889: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 6.0 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 43 MiB/s wr, 693 op/s; 11312/293148 objects degraded (3.859%); 5254/293148 objects misplaced (1.792%); 985 KiB/s, 15 objects/s recovering 2023-06-03T20:07:37.954 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:07:37 smithi164 bash[59462]: audit 1685822857.116532 mon.a (mon.0) 861 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:07:37.954 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:07:37 smithi164 bash[59462]: audit 1685822857.1211016 mon.c (mon.2) 452 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T20:07:38.003 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:07:38.006 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:07:38.006 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:07:38.030 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:07:37 smithi089 bash[78452]: cluster 1685822856.5868235 mgr.y (mgr.34104) 944 : cluster 0 pgmap v889: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 6.0 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 43 MiB/s wr, 693 op/s; 11312/293148 objects degraded (3.859%); 5254/293148 objects misplaced (1.792%); 985 KiB/s, 15 objects/s recovering 2023-06-03T20:07:38.030 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:07:37 smithi089 bash[78452]: audit 1685822857.116532 mon.a (mon.0) 861 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:07:38.030 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:07:37 smithi089 bash[78452]: audit 1685822857.1211016 mon.c (mon.2) 452 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T20:07:38.031 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:07:37 smithi089 bash[75068]: cluster 1685822856.5868235 mgr.y (mgr.34104) 944 : cluster 0 pgmap v889: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 6.0 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 43 MiB/s wr, 693 op/s; 11312/293148 objects degraded (3.859%); 5254/293148 objects misplaced (1.792%); 985 KiB/s, 15 objects/s recovering 2023-06-03T20:07:38.031 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:07:37 smithi089 bash[75068]: audit 1685822857.116532 mon.a (mon.0) 861 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:07:38.031 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:07:37 smithi089 bash[75068]: audit 1685822857.1211016 mon.c (mon.2) 452 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T20:07:38.107 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:07:38.142 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:07:38.146 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:07:38.147 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:07:38.201 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-06-03T20:07:38.244 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 84 16 101029 101013 75.1451 43 0.0166016 0.0133006 2023-06-03T20:07:38.249 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:07:38.289 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:07:38.292 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:07:38.293 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:07:38.393 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:07:38.430 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:07:38.433 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:07:38.434 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:07:38.535 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:07:38.567 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:07:38.573 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:07:38.574 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:07:38.675 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:07:38.704 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:07:38.706 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:07:38.707 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:07:38.808 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:07:38.854 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:07:38.860 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:07:38.861 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:07:38.961 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:07:39.009 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:07:39.013 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:07:39.014 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:07:39.101 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph osd unset nodeep-scrub 2023-06-03T20:07:39.115 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:07:39.188 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:07:39.191 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:07:39.193 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:07:39.245 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 85 16 101736 101720 74.7809 44.1875 0.00955903 0.0133669 2023-06-03T20:07:39.293 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:07:39.340 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:07:39.346 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:07:39.347 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:07:39.448 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:07:39.489 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:07:39.493 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:07:39.494 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:07:39.594 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:07:39.647 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:07:39.649 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:07:39.649 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:07:39.741 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:07:39 smithi089 bash[75068]: cluster 1685822858.5875542 mgr.y (mgr.34104) 945 : cluster 0 pgmap v890: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 6.1 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 44 MiB/s wr, 702 op/s; 11312/300996 objects degraded (3.758%); 5181/300996 objects misplaced (1.721%); 947 KiB/s, 14 objects/s recovering 2023-06-03T20:07:39.750 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:07:39.786 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:07:39.792 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:07:39.793 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:07:39.894 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:07:39.928 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:07:39.930 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:07:39.930 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:07:40.013 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:07:39 smithi089 bash[78452]: cluster 1685822858.5875542 mgr.y (mgr.34104) 945 : cluster 0 pgmap v890: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 6.1 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 44 MiB/s wr, 702 op/s; 11312/300996 objects degraded (3.758%); 5181/300996 objects misplaced (1.721%); 947 KiB/s, 14 objects/s recovering 2023-06-03T20:07:40.032 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:07:40.042 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:07:39 smithi164 bash[59462]: cluster 1685822858.5875542 mgr.y (mgr.34104) 945 : cluster 0 pgmap v890: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 6.1 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 44 MiB/s wr, 702 op/s; 11312/300996 objects degraded (3.758%); 5181/300996 objects misplaced (1.721%); 947 KiB/s, 14 objects/s recovering 2023-06-03T20:07:40.077 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:07:40.080 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:07:40.081 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:07:40.182 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:07:40.212 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:07:40.218 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:07:40.219 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:07:40.244 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 86 16 102416 102400 74.4055 42.5 0.0294473 0.0134332 2023-06-03T20:07:40.320 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:07:40.363 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:07:40.366 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:07:40.366 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:07:40.467 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:07:40.495 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:07:40.497 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:07:40.498 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:07:40.599 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:07:40.628 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:07:40.630 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:07:40.631 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:07:40.732 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:07:40.784 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:07:40.790 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:07:40.791 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:07:40.891 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:07:40.901 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:07:40 smithi089 bash[78452]: cluster 1685822860.7371905 mon.a (mon.0) 862 : cluster 3 Health check update: Degraded data redundancy: 11312/303759 objects degraded (3.724%), 2 pgs degraded (PG_DEGRADED) 2023-06-03T20:07:40.902 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:07:40 smithi089 bash[75068]: cluster 1685822860.7371905 mon.a (mon.0) 862 : cluster 3 Health check update: Degraded data redundancy: 11312/303759 objects degraded (3.724%), 2 pgs degraded (PG_DEGRADED) 2023-06-03T20:07:40.928 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:07:40.934 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:07:40.934 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:07:41.036 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:07:41.066 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:07:41.068 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:07:41.069 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:07:41.083 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:07:40 smithi164 bash[59462]: cluster 1685822860.7371905 mon.a (mon.0) 862 : cluster 3 Health check update: Degraded data redundancy: 11312/303759 objects degraded (3.724%), 2 pgs degraded (PG_DEGRADED) 2023-06-03T20:07:41.170 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:07:41.196 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:07:41.198 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:07:41.198 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:07:41.244 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 87 16 103083 103067 74.0294 41.6875 0.0295939 0.0135008 2023-06-03T20:07:41.299 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:07:41.328 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:07:41.334 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:07:41.335 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:07:41.341 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/mon.a/config 2023-06-03T20:07:41.436 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:07:41.468 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:07:41.470 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:07:41.470 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:07:41.571 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:07:41.600 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:07:41.605 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:07:41.606 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:07:41.707 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:07:41.739 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:07:41.743 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:07:41.743 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:07:41.845 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:07:41.888 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:07:41.893 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:07:41.894 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:07:41.995 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:07:42.028 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:07:41 smithi089 bash[75068]: cluster 1685822860.588196 mgr.y (mgr.34104) 946 : cluster 0 pgmap v891: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 6.2 GiB data, 19 GiB used, 606 GiB / 626 GiB avail; 44 MiB/s wr, 707 op/s; 11312/303759 objects degraded (3.724%); 5181/303759 objects misplaced (1.706%); 947 KiB/s, 14 objects/s recovering 2023-06-03T20:07:42.029 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:07:41 smithi089 bash[78452]: cluster 1685822860.588196 mgr.y (mgr.34104) 946 : cluster 0 pgmap v891: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 6.2 GiB data, 19 GiB used, 606 GiB / 626 GiB avail; 44 MiB/s wr, 707 op/s; 11312/303759 objects degraded (3.724%); 5181/303759 objects misplaced (1.706%); 947 KiB/s, 14 objects/s recovering 2023-06-03T20:07:42.046 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:07:42.052 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:07:42.053 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:07:42.054 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:07:41 smithi164 bash[59462]: cluster 1685822860.588196 mgr.y (mgr.34104) 946 : cluster 0 pgmap v891: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 6.2 GiB data, 19 GiB used, 606 GiB / 626 GiB avail; 44 MiB/s wr, 707 op/s; 11312/303759 objects degraded (3.724%); 5181/303759 objects misplaced (1.706%); 947 KiB/s, 14 objects/s recovering 2023-06-03T20:07:42.155 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:07:42.209 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:07:42.214 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:07:42.215 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:07:42.245 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 88 16 103770 103754 73.6761 42.9375 0.00937215 0.013567 2023-06-03T20:07:42.317 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:07:42.358 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:07:42.361 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:07:42.362 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:07:42.462 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:07:42.504 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:07:42.507 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:07:42.507 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:07:42.608 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:07:42.638 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:07:42.643 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:07:42.645 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:07:42.746 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:07:42.819 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:07:42.822 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:07:42.823 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:07:42.924 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:07:42.970 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:07:42.972 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:07:42.973 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:07:43.074 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:07:43.112 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:07:43.114 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:07:43.115 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:07:43.216 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:07:43.245 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 89 16 104462 104446 73.3342 43.25 0.0592503 0.0136301 2023-06-03T20:07:43.254 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:07:43.258 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:07:43.259 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:07:43.360 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:07:43.391 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/mon.a/config 2023-06-03T20:07:43.393 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:07:43.400 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:07:43.401 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:07:43.501 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:07:43.531 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:07:43.538 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:07:43.539 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:07:43.640 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:07:43.676 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:07:43.678 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:07:43.679 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:07:43.780 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:07:43.817 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:07:43.824 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:07:43.825 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:07:43.925 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:07:43.933 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:07:43 smithi089 bash[78452]: cluster 1685822862.5889156 mgr.y (mgr.34104) 947 : cluster 0 pgmap v892: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 6.2 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 44 MiB/s wr, 703 op/s; 11312/305538 objects degraded (3.702%); 5181/305538 objects misplaced (1.696%); 947 KiB/s, 14 objects/s recovering 2023-06-03T20:07:43.933 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:07:43 smithi089 bash[75068]: cluster 1685822862.5889156 mgr.y (mgr.34104) 947 : cluster 0 pgmap v892: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 6.2 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 44 MiB/s wr, 703 op/s; 11312/305538 objects degraded (3.702%); 5181/305538 objects misplaced (1.696%); 947 KiB/s, 14 objects/s recovering 2023-06-03T20:07:43.957 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:07:43.961 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:07:43.961 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:07:44.062 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:07:44.083 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:07:43 smithi164 bash[59462]: cluster 1685822862.5889156 mgr.y (mgr.34104) 947 : cluster 0 pgmap v892: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 6.2 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 44 MiB/s wr, 703 op/s; 11312/305538 objects degraded (3.702%); 5181/305538 objects misplaced (1.696%); 947 KiB/s, 14 objects/s recovering 2023-06-03T20:07:44.105 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:07:44.106 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:07:44.107 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:07:44.208 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:07:44.236 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:07:44.238 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:07:44.239 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:07:44.245 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 90 3 105132 105129 72.9936 42.6875 0.0814188 0.0136933 2023-06-03T20:07:44.339 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:07:44.374 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:07:44.377 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:07:44.377 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:07:44.414 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/mon.a/config 2023-06-03T20:07:44.479 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:07:44.514 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:07:44.517 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:07:44.518 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:07:44.619 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:07:44.653 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:07:44.654 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:07:44.655 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:07:44.756 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:07:44.787 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:07:44.789 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:07:44.790 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:07:44.891 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:07:44.923 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:07:44.926 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:07:44.926 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:07:45.027 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:07:45.056 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:07:45.059 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:07:45.060 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:07:45.161 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:07:45.205 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:07:45.207 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:07:45.208 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:07:45.245 INFO:tasks.radosbench.radosbench.0.smithi089.stdout:Total time run: 90.023 2023-06-03T20:07:45.246 INFO:tasks.radosbench.radosbench.0.smithi089.stdout:Total writes made: 105132 2023-06-03T20:07:45.246 INFO:tasks.radosbench.radosbench.0.smithi089.stdout:Write size: 65536 2023-06-03T20:07:45.246 INFO:tasks.radosbench.radosbench.0.smithi089.stdout:Object size: 65536 2023-06-03T20:07:45.246 INFO:tasks.radosbench.radosbench.0.smithi089.stdout:Bandwidth (MB/sec): 72.9897 2023-06-03T20:07:45.246 INFO:tasks.radosbench.radosbench.0.smithi089.stdout:Stddev Bandwidth: 33.124 2023-06-03T20:07:45.247 INFO:tasks.radosbench.radosbench.0.smithi089.stdout:Max bandwidth (MB/sec): 186.25 2023-06-03T20:07:45.247 INFO:tasks.radosbench.radosbench.0.smithi089.stdout:Min bandwidth (MB/sec): 0 2023-06-03T20:07:45.247 INFO:tasks.radosbench.radosbench.0.smithi089.stdout:Average IOPS: 1167 2023-06-03T20:07:45.247 INFO:tasks.radosbench.radosbench.0.smithi089.stdout:Stddev IOPS: 529.984 2023-06-03T20:07:45.247 INFO:tasks.radosbench.radosbench.0.smithi089.stdout:Max IOPS: 2980 2023-06-03T20:07:45.247 INFO:tasks.radosbench.radosbench.0.smithi089.stdout:Min IOPS: 0 2023-06-03T20:07:45.248 INFO:tasks.radosbench.radosbench.0.smithi089.stdout:Average Latency(s): 0.0136943 2023-06-03T20:07:45.248 INFO:tasks.radosbench.radosbench.0.smithi089.stdout:Stddev Latency(s): 0.0304027 2023-06-03T20:07:45.248 INFO:tasks.radosbench.radosbench.0.smithi089.stdout:Max latency(s): 2.52292 2023-06-03T20:07:45.248 INFO:tasks.radosbench.radosbench.0.smithi089.stdout:Min latency(s): 0.00128391 2023-06-03T20:07:45.253 INFO:tasks.radosbench.radosbench.0.smithi089.stdout:Cleaning up (deleting benchmark objects) 2023-06-03T20:07:45.309 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:07:45.346 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:07:45.349 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:07:45.349 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:07:45.450 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:07:45.492 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:07:45.497 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:07:45.498 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:07:45.599 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:07:45.630 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:07:45.634 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:07:45.635 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:07:45.736 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:07:45.773 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:07:45.775 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:07:45.776 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:07:45.811 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:07:45 smithi164 bash[59462]: cluster 1685822864.5897484 mgr.y (mgr.34104) 948 : cluster 0 pgmap v893: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 6.3 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 44 MiB/s wr, 696 op/s; 11312/311442 objects degraded (3.632%); 5113/311442 objects misplaced (1.642%); 902 KiB/s, 14 objects/s recovering 2023-06-03T20:07:45.877 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:07:45.912 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:07:45.914 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:07:45.915 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:07:46.016 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:07:46.028 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:07:45 smithi089 bash[75068]: cluster 1685822864.5897484 mgr.y (mgr.34104) 948 : cluster 0 pgmap v893: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 6.3 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 44 MiB/s wr, 696 op/s; 11312/311442 objects degraded (3.632%); 5113/311442 objects misplaced (1.642%); 902 KiB/s, 14 objects/s recovering 2023-06-03T20:07:46.029 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:07:45 smithi089 bash[78452]: cluster 1685822864.5897484 mgr.y (mgr.34104) 948 : cluster 0 pgmap v893: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 6.3 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 44 MiB/s wr, 696 op/s; 11312/311442 objects degraded (3.632%); 5113/311442 objects misplaced (1.642%); 902 KiB/s, 14 objects/s recovering 2023-06-03T20:07:46.057 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:07:46.062 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:07:46.063 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:07:46.164 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:07:46.193 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:07:46.195 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:07:46.195 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:07:46.296 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:07:46.346 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:07:46.353 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:07:46.353 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:07:46.454 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:07:46.482 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:07:46.484 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:07:46.484 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:07:46.585 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:07:46.587 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 20:07:46 smithi089 bash[71163]: ::ffff:172.21.15.164 - - [03/Jun/2023:20:07:46] "GET /metrics HTTP/1.1" 200 228768 "" "Prometheus/2.33.4" 2023-06-03T20:07:46.635 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:07:46.637 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:07:46.637 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:07:46.738 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:07:46.771 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:07:46.774 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:07:46.775 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:07:46.823 INFO:teuthology.orchestra.run.smithi089.stderr:set osd.7 primary-affinity to 1 (8655362) 2023-06-03T20:07:46.877 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:07:46.904 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:07:46.909 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:07:46.910 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:07:47.011 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:07:47.021 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:07:46 smithi164 bash[59462]: audit 1685822866.6323662 mon.c (mon.2) 453 : audit 1 from='client.? 172.21.15.89:0/1029384237' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 7, "weight": 1.0}]: dispatch 2023-06-03T20:07:47.021 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:07:46 smithi164 bash[59462]: audit 1685822866.6328099 mon.a (mon.0) 863 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 7, "weight": 1.0}]: dispatch 2023-06-03T20:07:47.059 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:07:47.060 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:07:47.061 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:07:47.162 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:07:47.211 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:07:47.212 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:07:47.213 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:07:47.278 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:07:46 smithi089 bash[78452]: audit 1685822866.6323662 mon.c (mon.2) 453 : audit 1 from='client.? 172.21.15.89:0/1029384237' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 7, "weight": 1.0}]: dispatch 2023-06-03T20:07:47.278 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:07:46 smithi089 bash[78452]: audit 1685822866.6328099 mon.a (mon.0) 863 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 7, "weight": 1.0}]: dispatch 2023-06-03T20:07:47.279 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:07:46 smithi089 bash[75068]: audit 1685822866.6323662 mon.c (mon.2) 453 : audit 1 from='client.? 172.21.15.89:0/1029384237' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 7, "weight": 1.0}]: dispatch 2023-06-03T20:07:47.279 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:07:46 smithi089 bash[75068]: audit 1685822866.6328099 mon.a (mon.0) 863 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 7, "weight": 1.0}]: dispatch 2023-06-03T20:07:47.314 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:07:47.344 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:07:47.346 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:07:47.347 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:07:47.447 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:07:47.483 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:07:47.485 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:07:47.486 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:07:47.587 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:07:47.616 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:07:47.622 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:07:47.623 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:07:47.724 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:07:47.755 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:07:47.758 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:07:47.759 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:07:47.825 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:07:47 smithi164 bash[59462]: cluster 1685822866.590415 mgr.y (mgr.34104) 949 : cluster 0 pgmap v894: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 6.4 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 39 MiB/s wr, 623 op/s; 11312/313458 objects degraded (3.609%); 5113/313458 objects misplaced (1.631%); 831 KiB/s, 12 objects/s recovering 2023-06-03T20:07:47.826 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:07:47 smithi164 bash[59462]: cluster 1685822866.80615 mon.a (mon.0) 864 : cluster 3 Health check update: Degraded data redundancy: 11312/313458 objects degraded (3.609%), 2 pgs degraded (PG_DEGRADED) 2023-06-03T20:07:47.826 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:07:47 smithi164 bash[59462]: cluster 1685822866.8096771 osd.6 (osd.6) 108 : cluster 3 failed to encode map e172 with expected crc 2023-06-03T20:07:47.826 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:07:47 smithi164 bash[59462]: audit 1685822866.8119545 mon.a (mon.0) 865 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 7, "weight": 1.0}]': finished 2023-06-03T20:07:47.826 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:07:47 smithi164 bash[59462]: cluster 1685822866.8120165 mon.a (mon.0) 866 : cluster 0 osdmap e172: 8 total, 8 up, 7 in 2023-06-03T20:07:47.826 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:07:47 smithi164 bash[59462]: cluster 1685822866.8321552 osd.7 (osd.7) 106 : cluster 3 failed to encode map e172 with expected crc 2023-06-03T20:07:47.826 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:07:47 smithi164 bash[59462]: cluster 1685822866.8334951 osd.7 (osd.7) 107 : cluster 3 failed to encode map e172 with expected crc 2023-06-03T20:07:47.860 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:07:47.870 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:07:47 smithi089 bash[78452]: cluster 1685822866.590415 mgr.y (mgr.34104) 949 : cluster 0 pgmap v894: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 6.4 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 39 MiB/s wr, 623 op/s; 11312/313458 objects degraded (3.609%); 5113/313458 objects misplaced (1.631%); 831 KiB/s, 12 objects/s recovering 2023-06-03T20:07:47.870 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:07:47 smithi089 bash[78452]: cluster 1685822866.80615 mon.a (mon.0) 864 : cluster 3 Health check update: Degraded data redundancy: 11312/313458 objects degraded (3.609%), 2 pgs degraded (PG_DEGRADED) 2023-06-03T20:07:47.871 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:07:47 smithi089 bash[78452]: cluster 1685822866.8096771 osd.6 (osd.6) 108 : cluster 3 failed to encode map e172 with expected crc 2023-06-03T20:07:47.871 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:07:47 smithi089 bash[78452]: audit 1685822866.8119545 mon.a (mon.0) 865 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 7, "weight": 1.0}]': finished 2023-06-03T20:07:47.871 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:07:47 smithi089 bash[78452]: cluster 1685822866.8120165 mon.a (mon.0) 866 : cluster 0 osdmap e172: 8 total, 8 up, 7 in 2023-06-03T20:07:47.871 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:07:47 smithi089 bash[78452]: cluster 1685822866.8321552 osd.7 (osd.7) 106 : cluster 3 failed to encode map e172 with expected crc 2023-06-03T20:07:47.871 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:07:47 smithi089 bash[78452]: cluster 1685822866.8334951 osd.7 (osd.7) 107 : cluster 3 failed to encode map e172 with expected crc 2023-06-03T20:07:47.871 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:07:47 smithi089 bash[75068]: cluster 1685822866.590415 mgr.y (mgr.34104) 949 : cluster 0 pgmap v894: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 6.4 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 39 MiB/s wr, 623 op/s; 11312/313458 objects degraded (3.609%); 5113/313458 objects misplaced (1.631%); 831 KiB/s, 12 objects/s recovering 2023-06-03T20:07:47.872 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:07:47 smithi089 bash[75068]: cluster 1685822866.80615 mon.a (mon.0) 864 : cluster 3 Health check update: Degraded data redundancy: 11312/313458 objects degraded (3.609%), 2 pgs degraded (PG_DEGRADED) 2023-06-03T20:07:47.872 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:07:47 smithi089 bash[75068]: cluster 1685822866.8096771 osd.6 (osd.6) 108 : cluster 3 failed to encode map e172 with expected crc 2023-06-03T20:07:47.872 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:07:47 smithi089 bash[75068]: audit 1685822866.8119545 mon.a (mon.0) 865 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 7, "weight": 1.0}]': finished 2023-06-03T20:07:47.872 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:07:47 smithi089 bash[75068]: cluster 1685822866.8120165 mon.a (mon.0) 866 : cluster 0 osdmap e172: 8 total, 8 up, 7 in 2023-06-03T20:07:47.872 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:07:47 smithi089 bash[75068]: cluster 1685822866.8321552 osd.7 (osd.7) 106 : cluster 3 failed to encode map e172 with expected crc 2023-06-03T20:07:47.873 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:07:47 smithi089 bash[75068]: cluster 1685822866.8334951 osd.7 (osd.7) 107 : cluster 3 failed to encode map e172 with expected crc 2023-06-03T20:07:47.895 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:07:47.900 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:07:47.901 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:07:48.002 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:07:48.056 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:07:48.060 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:07:48.060 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:07:48.161 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:07:48.198 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:07:48.201 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:07:48.202 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:07:48.302 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:07:48.339 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:07:48.347 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:07:48.348 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:07:48.448 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:07:48.487 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:07:48.493 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:07:48.494 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:07:48.595 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:07:48.628 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:07:48.633 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:07:48.634 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:07:48.735 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:07:48.774 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:07:48.777 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:07:48.777 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:07:48.878 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:07:48.932 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:07:48.937 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:07:48.937 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:07:49.028 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:07:48 smithi089 bash[75068]: cluster 1685822867.8286304 osd.6 (osd.6) 109 : cluster 3 failed to encode map e173 with expected crc 2023-06-03T20:07:49.029 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:07:48 smithi089 bash[75068]: cluster 1685822867.8296895 osd.5 (osd.5) 103 : cluster 3 failed to encode map e173 with expected crc 2023-06-03T20:07:49.029 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:07:48 smithi089 bash[75068]: cluster 1685822867.8320484 mon.a (mon.0) 867 : cluster 0 osdmap e173: 8 total, 8 up, 7 in 2023-06-03T20:07:49.029 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:07:48 smithi089 bash[75068]: cluster 1685822867.832276 osd.7 (osd.7) 108 : cluster 3 failed to encode map e173 with expected crc 2023-06-03T20:07:49.029 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:07:48 smithi089 bash[78452]: cluster 1685822867.8286304 osd.6 (osd.6) 109 : cluster 3 failed to encode map e173 with expected crc 2023-06-03T20:07:49.029 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:07:48 smithi089 bash[78452]: cluster 1685822867.8296895 osd.5 (osd.5) 103 : cluster 3 failed to encode map e173 with expected crc 2023-06-03T20:07:49.030 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:07:48 smithi089 bash[78452]: cluster 1685822867.8320484 mon.a (mon.0) 867 : cluster 0 osdmap e173: 8 total, 8 up, 7 in 2023-06-03T20:07:49.030 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:07:48 smithi089 bash[78452]: cluster 1685822867.832276 osd.7 (osd.7) 108 : cluster 3 failed to encode map e173 with expected crc 2023-06-03T20:07:49.040 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:07:49.091 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:07:49.093 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:07:49.094 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:07:49.195 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:07:49.222 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:07:49.224 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:07:49.225 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:07:49.325 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:07:49.333 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:07:48 smithi164 bash[59462]: cluster 1685822867.8286304 osd.6 (osd.6) 109 : cluster 3 failed to encode map e173 with expected crc 2023-06-03T20:07:49.333 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:07:48 smithi164 bash[59462]: cluster 1685822867.8296895 osd.5 (osd.5) 103 : cluster 3 failed to encode map e173 with expected crc 2023-06-03T20:07:49.334 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:07:48 smithi164 bash[59462]: cluster 1685822867.8320484 mon.a (mon.0) 867 : cluster 0 osdmap e173: 8 total, 8 up, 7 in 2023-06-03T20:07:49.334 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:07:48 smithi164 bash[59462]: cluster 1685822867.832276 osd.7 (osd.7) 108 : cluster 3 failed to encode map e173 with expected crc 2023-06-03T20:07:49.363 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:07:49.369 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:07:49.370 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:07:49.471 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:07:49.500 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:07:49.509 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:07:49.510 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:07:49.611 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:07:49.642 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:07:49.647 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:07:49.648 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:07:49.749 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:07:49.779 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:07:49.781 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:07:49.781 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:07:49.882 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:07:49.928 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:07:49.932 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:07:49.933 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:07:50.028 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:07:49 smithi089 bash[75068]: cluster 1685822868.591145 mgr.y (mgr.34104) 950 : cluster 0 pgmap v897: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 6.3 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 1.6 KiB/s rd, 30 MiB/s wr, 657 op/s; 11312/311439 objects degraded (3.632%); 5027/311439 objects misplaced (1.614%); 1.2 MiB/s, 19 objects/s recovering 2023-06-03T20:07:50.028 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:07:49 smithi089 bash[78452]: cluster 1685822868.591145 mgr.y (mgr.34104) 950 : cluster 0 pgmap v897: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 6.3 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 1.6 KiB/s rd, 30 MiB/s wr, 657 op/s; 11312/311439 objects degraded (3.632%); 5027/311439 objects misplaced (1.614%); 1.2 MiB/s, 19 objects/s recovering 2023-06-03T20:07:50.035 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:07:50.046 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:07:49 smithi164 bash[59462]: cluster 1685822868.591145 mgr.y (mgr.34104) 950 : cluster 0 pgmap v897: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 6.3 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 1.6 KiB/s rd, 30 MiB/s wr, 657 op/s; 11312/311439 objects degraded (3.632%); 5027/311439 objects misplaced (1.614%); 1.2 MiB/s, 19 objects/s recovering 2023-06-03T20:07:50.075 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:07:50.078 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:07:50.078 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:07:50.179 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:07:50.207 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:07:50.208 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:07:50.209 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:07:50.310 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:07:50.359 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:07:50.363 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:07:50.363 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:07:50.464 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:07:50.492 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:07:50.498 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:07:50.499 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:07:50.600 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:07:50.635 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:07:50.637 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:07:50.638 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:07:50.739 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:07:50.791 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:07:50.793 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:07:50.794 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:07:50.895 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:07:50.930 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:07:50.938 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:07:50.939 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:07:51.040 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:07:51.070 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:07:51.072 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:07:51.072 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:07:51.173 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:07:51.204 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:07:51.208 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:07:51.209 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:07:51.310 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:07:51.365 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:07:51.368 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:07:51.368 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:07:51.470 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:07:51.510 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:07:51.514 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:07:51.514 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:07:51.615 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:07:51.647 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:07:51.651 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:07:51.652 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:07:51.753 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:07:51.788 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:07:51.791 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:07:51.791 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:07:51.892 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:07:51.932 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:07:51.940 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:07:51.941 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:07:52.041 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:07:52.056 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:07:51 smithi089 bash[75068]: cluster 1685822870.5917256 mgr.y (mgr.34104) 951 : cluster 0 pgmap v898: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 6.3 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 1.6 KiB/s rd, 26 MiB/s wr, 682 op/s; 11312/309069 objects degraded (3.660%); 5027/309069 objects misplaced (1.626%); 1.2 MiB/s, 19 objects/s recovering 2023-06-03T20:07:52.057 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:07:51 smithi089 bash[78452]: cluster 1685822870.5917256 mgr.y (mgr.34104) 951 : cluster 0 pgmap v898: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 6.3 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 1.6 KiB/s rd, 26 MiB/s wr, 682 op/s; 11312/309069 objects degraded (3.660%); 5027/309069 objects misplaced (1.626%); 1.2 MiB/s, 19 objects/s recovering 2023-06-03T20:07:52.071 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:07:52.073 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:07:52.074 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:07:52.175 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:07:52.206 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:07:52.209 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:07:52.210 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:07:52.311 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:07:52.333 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:07:51 smithi164 bash[59462]: cluster 1685822870.5917256 mgr.y (mgr.34104) 951 : cluster 0 pgmap v898: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 6.3 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 1.6 KiB/s rd, 26 MiB/s wr, 682 op/s; 11312/309069 objects degraded (3.660%); 5027/309069 objects misplaced (1.626%); 1.2 MiB/s, 19 objects/s recovering 2023-06-03T20:07:52.335 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:07:52.337 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:07:52.338 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:07:52.439 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:07:52.465 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:07:52.474 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:07:52.475 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:07:52.576 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:07:52.602 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:07:52.610 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:07:52.611 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:07:52.712 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:07:52.756 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:07:52.760 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:07:52.760 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:07:52.861 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:07:52.896 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:07:52.902 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:07:52.903 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:07:53.003 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:07:53.030 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:07:53.033 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:07:53.034 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:07:53.083 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:07:52 smithi164 bash[59462]: audit 1685822872.110395 mon.c (mon.2) 454 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T20:07:53.084 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:07:52 smithi164 bash[59462]: cluster 1685822872.6554642 mon.a (mon.0) 868 : cluster 3 Health check update: Degraded data redundancy: 11312/309069 objects degraded (3.660%), 2 pgs degraded (PG_DEGRADED) 2023-06-03T20:07:53.135 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:07:53.143 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:07:52 smithi089 bash[75068]: audit 1685822872.110395 mon.c (mon.2) 454 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T20:07:53.144 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:07:52 smithi089 bash[75068]: cluster 1685822872.6554642 mon.a (mon.0) 868 : cluster 3 Health check update: Degraded data redundancy: 11312/309069 objects degraded (3.660%), 2 pgs degraded (PG_DEGRADED) 2023-06-03T20:07:53.144 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:07:52 smithi089 bash[78452]: audit 1685822872.110395 mon.c (mon.2) 454 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T20:07:53.144 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:07:52 smithi089 bash[78452]: cluster 1685822872.6554642 mon.a (mon.0) 868 : cluster 3 Health check update: Degraded data redundancy: 11312/309069 objects degraded (3.660%), 2 pgs degraded (PG_DEGRADED) 2023-06-03T20:07:53.171 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:07:53.178 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:07:53.179 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:07:53.280 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:07:53.322 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:07:53.323 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:07:53.325 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:07:53.426 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:07:53.462 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:07:53.465 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:07:53.465 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:07:53.566 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:07:53.591 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:07:53.596 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:07:53.596 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:07:53.697 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:07:53.735 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:07:53.735 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:07:53.736 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:07:53.837 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:07:53.878 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:07:53.881 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:07:53.882 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:07:53.983 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:07:54.015 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:07:54.018 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:07:54.019 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:07:54.083 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:07:53 smithi164 bash[59462]: cluster 1685822872.5922036 mgr.y (mgr.34104) 952 : cluster 0 pgmap v899: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 6.2 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 1.6 KiB/s rd, 10 MiB/s wr, 514 op/s; 11312/307197 objects degraded (3.682%); 5027/307197 objects misplaced (1.636%); 688 KiB/s, 10 objects/s recovering 2023-06-03T20:07:54.121 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:07:54.154 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:07:54.159 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:07:54.160 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:07:54.261 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:07:54.286 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:07:54.289 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:07:54.290 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:07:54.379 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:07:53 smithi089 bash[75068]: cluster 1685822872.5922036 mgr.y (mgr.34104) 952 : cluster 0 pgmap v899: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 6.2 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 1.6 KiB/s rd, 10 MiB/s wr, 514 op/s; 11312/307197 objects degraded (3.682%); 5027/307197 objects misplaced (1.636%); 688 KiB/s, 10 objects/s recovering 2023-06-03T20:07:54.380 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:07:53 smithi089 bash[78452]: cluster 1685822872.5922036 mgr.y (mgr.34104) 952 : cluster 0 pgmap v899: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 6.2 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 1.6 KiB/s rd, 10 MiB/s wr, 514 op/s; 11312/307197 objects degraded (3.682%); 5027/307197 objects misplaced (1.636%); 688 KiB/s, 10 objects/s recovering 2023-06-03T20:07:54.391 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:07:54.448 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:07:54.453 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:07:54.454 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:07:54.555 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:07:54.587 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:07:54.590 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:07:54.590 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:07:54.691 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:07:54.720 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:07:54.722 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:07:54.723 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:07:54.824 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:07:54.858 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:07:54.861 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:07:54.861 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:07:54.962 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:07:54.993 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:07:54.995 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:07:54.996 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:07:55.098 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:07:55.128 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:07:55.135 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:07:55.136 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:07:55.237 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:07:55.287 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:07:55.290 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:07:55.291 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:07:55.392 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:07:55.427 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:07:55.429 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:07:55.430 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:07:55.531 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:07:55.563 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:07:55.566 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:07:55.567 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:07:55.668 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:07:55.703 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:07:55.708 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:07:55.709 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:07:55.810 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:07:55.851 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:07:55.856 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:07:55.857 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:07:55.957 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:07:55.967 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:07:55 smithi164 bash[59462]: cluster 1685822874.592936 mgr.y (mgr.34104) 953 : cluster 0 pgmap v900: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 6.1 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 1.6 KiB/s rd, 5.2 MiB/s wr, 676 op/s; 11312/301260 objects degraded (3.755%); 4949/301260 objects misplaced (1.643%); 1.3 MiB/s, 20 objects/s recovering 2023-06-03T20:07:55.995 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:07:55.998 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:07:55.998 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:07:56.100 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:07:56.133 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:07:56.135 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:07:56.136 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:07:56.136 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:07:55 smithi089 bash[75068]: cluster 1685822874.592936 mgr.y (mgr.34104) 953 : cluster 0 pgmap v900: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 6.1 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 1.6 KiB/s rd, 5.2 MiB/s wr, 676 op/s; 11312/301260 objects degraded (3.755%); 4949/301260 objects misplaced (1.643%); 1.3 MiB/s, 20 objects/s recovering 2023-06-03T20:07:56.137 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:07:55 smithi089 bash[78452]: cluster 1685822874.592936 mgr.y (mgr.34104) 953 : cluster 0 pgmap v900: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 6.1 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 1.6 KiB/s rd, 5.2 MiB/s wr, 676 op/s; 11312/301260 objects degraded (3.755%); 4949/301260 objects misplaced (1.643%); 1.3 MiB/s, 20 objects/s recovering 2023-06-03T20:07:56.238 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:07:56.272 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:07:56.274 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:07:56.275 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:07:56.376 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:07:56.418 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:07:56.425 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:07:56.426 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:07:56.468 INFO:tasks.thrashosds.thrasher:in_osds: [0, 2, 3, 4, 5, 6, 7] out_osds: [1] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-06-03T20:07:56.468 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-06-03T20:07:56.469 INFO:tasks.thrashosds.thrasher:inject_pause on osd.0 2023-06-03T20:07:56.469 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2023-06-03T20:07:56.469 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2023-06-03T20:07:56.469 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph daemon osd.0 config set filestore_inject_stall 3 2023-06-03T20:07:56.479 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 20:07:56 smithi089 bash[71163]: ::ffff:172.21.15.164 - - [03/Jun/2023:20:07:56] "GET /metrics HTTP/1.1" 200 228778 "" "Prometheus/2.33.4" 2023-06-03T20:07:56.517 INFO:teuthology.orchestra.run.smithi089.stdout:ERROR: (22) Invalid argument 2023-06-03T20:07:56.517 INFO:teuthology.orchestra.run.smithi089.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-06-03T20:07:56.527 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:07:56.573 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:07:56.575 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:07:56.576 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:07:56.677 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:07:56.714 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:07:56.717 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:07:56.717 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:07:56.818 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:07:56.849 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:07:56.854 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:07:56.855 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:07:56.956 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:07:56.993 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:07:57.000 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:07:57.001 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:07:57.102 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:07:57.131 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:07:57.134 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:07:57.135 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:07:57.236 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:07:57.292 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:07:57.296 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:07:57.297 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:07:57.398 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:07:57.399 INFO:teuthology.orchestra.run.smithi089.stdout:osd.0: {} 2023-06-03T20:07:57.424 INFO:teuthology.orchestra.run.smithi089.stdout:osd.1: {} 2023-06-03T20:07:57.436 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:07:57.444 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:07:57.445 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:07:57.450 INFO:teuthology.orchestra.run.smithi089.stdout:osd.2: {} 2023-06-03T20:07:57.480 INFO:teuthology.orchestra.run.smithi089.stdout:osd.3: {} 2023-06-03T20:07:57.510 INFO:teuthology.orchestra.run.smithi089.stdout:osd.4: {} 2023-06-03T20:07:57.536 INFO:teuthology.orchestra.run.smithi089.stdout:osd.5: {} 2023-06-03T20:07:57.546 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:07:57.581 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:07:57.586 INFO:teuthology.orchestra.run.smithi089.stdout:osd.6: {} 2023-06-03T20:07:57.586 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:07:57.587 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:07:57.627 INFO:teuthology.orchestra.run.smithi089.stdout:osd.7: {} 2023-06-03T20:07:57.633 INFO:teuthology.orchestra.run.smithi089.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-06-03T20:07:57.634 INFO:teuthology.orchestra.run.smithi089.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-06-03T20:07:57.634 INFO:teuthology.orchestra.run.smithi089.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-06-03T20:07:57.634 INFO:teuthology.orchestra.run.smithi089.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-06-03T20:07:57.634 INFO:teuthology.orchestra.run.smithi089.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-06-03T20:07:57.634 INFO:teuthology.orchestra.run.smithi089.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-06-03T20:07:57.635 INFO:teuthology.orchestra.run.smithi089.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-06-03T20:07:57.635 INFO:teuthology.orchestra.run.smithi089.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-06-03T20:07:57.689 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:07:57.723 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:07:57.727 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:07:57.729 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:07:57.829 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:07:57.875 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:07:57.878 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:07:57.879 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:07:57.980 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:07:58.016 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:07:58.022 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:07:58.022 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:07:58.028 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:07:57 smithi089 bash[75068]: cluster 1685822876.5935667 mgr.y (mgr.34104) 954 : cluster 0 pgmap v901: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 6.1 GiB data, 19 GiB used, 606 GiB / 626 GiB avail; 1.3 KiB/s rd, 2.3 MiB/s wr, 593 op/s; 11312/298641 objects degraded (3.788%); 4949/298641 objects misplaced (1.657%); 1.0 MiB/s, 16 objects/s recovering 2023-06-03T20:07:58.028 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:07:57 smithi089 bash[75068]: audit 1685822877.3560724 mon.c (mon.2) 455 : audit 0 from='client.? 172.21.15.89:0/1004342045' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-03T20:07:58.029 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:07:57 smithi089 bash[75068]: cluster 1685822877.656476 mon.a (mon.0) 869 : cluster 3 Health check update: Degraded data redundancy: 11312/298641 objects degraded (3.788%), 2 pgs degraded (PG_DEGRADED) 2023-06-03T20:07:58.029 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:07:57 smithi089 bash[78452]: cluster 1685822876.5935667 mgr.y (mgr.34104) 954 : cluster 0 pgmap v901: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 6.1 GiB data, 19 GiB used, 606 GiB / 626 GiB avail; 1.3 KiB/s rd, 2.3 MiB/s wr, 593 op/s; 11312/298641 objects degraded (3.788%); 4949/298641 objects misplaced (1.657%); 1.0 MiB/s, 16 objects/s recovering 2023-06-03T20:07:58.029 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:07:57 smithi089 bash[78452]: audit 1685822877.3560724 mon.c (mon.2) 455 : audit 0 from='client.? 172.21.15.89:0/1004342045' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-03T20:07:58.029 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:07:57 smithi089 bash[78452]: cluster 1685822877.656476 mon.a (mon.0) 869 : cluster 3 Health check update: Degraded data redundancy: 11312/298641 objects degraded (3.788%), 2 pgs degraded (PG_DEGRADED) 2023-06-03T20:07:58.124 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:07:58.132 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:07:57 smithi164 bash[59462]: cluster 1685822876.5935667 mgr.y (mgr.34104) 954 : cluster 0 pgmap v901: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 6.1 GiB data, 19 GiB used, 606 GiB / 626 GiB avail; 1.3 KiB/s rd, 2.3 MiB/s wr, 593 op/s; 11312/298641 objects degraded (3.788%); 4949/298641 objects misplaced (1.657%); 1.0 MiB/s, 16 objects/s recovering 2023-06-03T20:07:58.133 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:07:57 smithi164 bash[59462]: audit 1685822877.3560724 mon.c (mon.2) 455 : audit 0 from='client.? 172.21.15.89:0/1004342045' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-03T20:07:58.133 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:07:57 smithi164 bash[59462]: cluster 1685822877.656476 mon.a (mon.0) 869 : cluster 3 Health check update: Degraded data redundancy: 11312/298641 objects degraded (3.788%), 2 pgs degraded (PG_DEGRADED) 2023-06-03T20:07:58.150 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:07:58.156 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:07:58.157 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:07:58.258 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:07:58.309 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:07:58.315 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:07:58.316 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:07:58.417 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:07:58.457 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:07:58.465 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:07:58.466 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:07:58.534 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph daemon osd.1 dump_blocked_ops 2023-06-03T20:07:58.567 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:07:58.624 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:07:58.628 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:07:58.630 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:07:58.731 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:07:58.775 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:07:58.781 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:07:58.782 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:07:58.883 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:07:58.929 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:07:58.937 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:07:58.938 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:07:59.039 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:07:59.070 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:07:59.072 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:07:59.073 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:07:59.174 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:07:59.215 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:07:59.217 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:07:59.218 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:07:59.319 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:07:59.352 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:07:59.359 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:07:59.360 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:07:59.461 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:07:59.515 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:07:59.522 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:07:59.522 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:07:59.623 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:07:59.660 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:07:59.665 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:07:59.665 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:07:59.766 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:07:59.797 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:07:59.802 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:07:59.802 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:07:59.903 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:07:59.913 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:07:59 smithi089 bash[75068]: cluster 1685822878.5944078 mgr.y (mgr.34104) 955 : cluster 0 pgmap v902: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 5.9 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 1.2 KiB/s rd, 2.1 MiB/s wr, 777 op/s; 11312/290949 objects degraded (3.888%); 4867/290949 objects misplaced (1.673%); 1.4 MiB/s, 22 objects/s recovering 2023-06-03T20:07:59.913 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:07:59 smithi089 bash[78452]: cluster 1685822878.5944078 mgr.y (mgr.34104) 955 : cluster 0 pgmap v902: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 5.9 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 1.2 KiB/s rd, 2.1 MiB/s wr, 777 op/s; 11312/290949 objects degraded (3.888%); 4867/290949 objects misplaced (1.673%); 1.4 MiB/s, 22 objects/s recovering 2023-06-03T20:07:59.947 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:07:59.952 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:07:59.953 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:08:00.054 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:08:00.100 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:08:00.103 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:08:00.104 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:08:00.205 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:08:00.212 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:07:59 smithi164 bash[59462]: cluster 1685822878.5944078 mgr.y (mgr.34104) 955 : cluster 0 pgmap v902: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 5.9 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 1.2 KiB/s rd, 2.1 MiB/s wr, 777 op/s; 11312/290949 objects degraded (3.888%); 4867/290949 objects misplaced (1.673%); 1.4 MiB/s, 22 objects/s recovering 2023-06-03T20:08:00.252 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:08:00.256 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:08:00.256 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:08:00.357 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:08:00.392 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:08:00.396 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:08:00.397 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:08:00.497 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:08:00.529 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:08:00.534 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:08:00.535 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:08:00.636 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:08:00.668 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:08:00.674 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:08:00.675 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:08:00.776 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:08:00.824 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:08:00.829 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:08:00.830 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:08:00.888 INFO:teuthology.orchestra.run.smithi089.stderr:nodeep-scrub is unset 2023-06-03T20:08:00.931 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:08:00.966 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:08:00.971 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:08:00.972 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:08:01.073 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:08:01.083 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:08:00 smithi164 bash[59462]: audit 1685822880.2490506 mon.a (mon.0) 870 : audit 1 from='client.? 172.21.15.89:0/3445366279' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-06-03T20:08:01.111 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:08:01.114 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:08:01.115 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:08:01.216 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:08:01.254 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:08:01.257 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:08:01.258 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:08:01.278 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:08:00 smithi089 bash[78452]: audit 1685822880.2490506 mon.a (mon.0) 870 : audit 1 from='client.? 172.21.15.89:0/3445366279' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-06-03T20:08:01.278 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:08:00 smithi089 bash[75068]: audit 1685822880.2490506 mon.a (mon.0) 870 : audit 1 from='client.? 172.21.15.89:0/3445366279' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-06-03T20:08:01.359 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:08:01.392 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:08:01.396 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:08:01.397 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:08:01.498 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:08:01.539 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:08:01.542 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:08:01.543 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:08:01.644 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:08:01.647 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-06-03T20:08:01.681 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:08:01.686 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:08:01.687 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:08:01.788 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:08:01.843 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:08:01.849 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:08:01.850 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:08:01.951 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:08:01.991 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:08:01.994 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:08:01.998 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:08:02.029 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:08:01 smithi089 bash[75068]: cluster 1685822880.5951247 mgr.y (mgr.34104) 956 : cluster 0 pgmap v903: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 5.9 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 639 op/s; 11312/288396 objects degraded (3.922%); 4867/288396 objects misplaced (1.688%); 853 KiB/s, 13 objects/s recovering 2023-06-03T20:08:02.030 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:08:01 smithi089 bash[75068]: cluster 1685822880.8676238 mon.a (mon.0) 871 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-06-03T20:08:02.031 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:08:01 smithi089 bash[75068]: cluster 1685822880.8758247 osd.5 (osd.5) 104 : cluster 3 failed to encode map e174 with expected crc 2023-06-03T20:08:02.031 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:08:01 smithi089 bash[75068]: cluster 1685822880.8773828 osd.6 (osd.6) 110 : cluster 3 failed to encode map e174 with expected crc 2023-06-03T20:08:02.032 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:08:01 smithi089 bash[75068]: audit 1685822880.8804948 mon.a (mon.0) 872 : audit 1 from='client.? 172.21.15.89:0/3445366279' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-06-03T20:08:02.032 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:08:01 smithi089 bash[75068]: cluster 1685822880.8805678 mon.a (mon.0) 873 : cluster 0 osdmap e174: 8 total, 8 up, 7 in 2023-06-03T20:08:02.033 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:08:01 smithi089 bash[78452]: cluster 1685822880.5951247 mgr.y (mgr.34104) 956 : cluster 0 pgmap v903: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 5.9 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 639 op/s; 11312/288396 objects degraded (3.922%); 4867/288396 objects misplaced (1.688%); 853 KiB/s, 13 objects/s recovering 2023-06-03T20:08:02.033 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:08:01 smithi089 bash[78452]: cluster 1685822880.8676238 mon.a (mon.0) 871 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-06-03T20:08:02.033 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:08:01 smithi089 bash[78452]: cluster 1685822880.8758247 osd.5 (osd.5) 104 : cluster 3 failed to encode map e174 with expected crc 2023-06-03T20:08:02.034 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:08:01 smithi089 bash[78452]: cluster 1685822880.8773828 osd.6 (osd.6) 110 : cluster 3 failed to encode map e174 with expected crc 2023-06-03T20:08:02.034 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:08:01 smithi089 bash[78452]: audit 1685822880.8804948 mon.a (mon.0) 872 : audit 1 from='client.? 172.21.15.89:0/3445366279' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-06-03T20:08:02.034 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:08:01 smithi089 bash[78452]: cluster 1685822880.8805678 mon.a (mon.0) 873 : cluster 0 osdmap e174: 8 total, 8 up, 7 in 2023-06-03T20:08:02.099 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:08:02.189 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:08:02.198 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:08:02.200 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:08:02.300 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:08:02.333 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:08:01 smithi164 bash[59462]: cluster 1685822880.5951247 mgr.y (mgr.34104) 956 : cluster 0 pgmap v903: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 5.9 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 639 op/s; 11312/288396 objects degraded (3.922%); 4867/288396 objects misplaced (1.688%); 853 KiB/s, 13 objects/s recovering 2023-06-03T20:08:02.334 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:08:01 smithi164 bash[59462]: cluster 1685822880.8676238 mon.a (mon.0) 871 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-06-03T20:08:02.334 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:08:01 smithi164 bash[59462]: cluster 1685822880.8758247 osd.5 (osd.5) 104 : cluster 3 failed to encode map e174 with expected crc 2023-06-03T20:08:02.334 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:08:01 smithi164 bash[59462]: cluster 1685822880.8773828 osd.6 (osd.6) 110 : cluster 3 failed to encode map e174 with expected crc 2023-06-03T20:08:02.334 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:08:01 smithi164 bash[59462]: audit 1685822880.8804948 mon.a (mon.0) 872 : audit 1 from='client.? 172.21.15.89:0/3445366279' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-06-03T20:08:02.334 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:08:01 smithi164 bash[59462]: cluster 1685822880.8805678 mon.a (mon.0) 873 : cluster 0 osdmap e174: 8 total, 8 up, 7 in 2023-06-03T20:08:02.337 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:08:02.341 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:08:02.342 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:08:02.443 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:08:02.475 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:08:02.477 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:08:02.478 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:08:02.579 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:08:02.615 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:08:02.618 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:08:02.619 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:08:02.720 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:08:02.769 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:08:02.773 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:08:02.774 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:08:02.875 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:08:02.883 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:08:02 smithi089 bash[75068]: cluster 1685822880.9100063 osd.7 (osd.7) 109 : cluster 3 failed to encode map e174 with expected crc 2023-06-03T20:08:02.883 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:08:02 smithi089 bash[75068]: cluster 1685822880.911989 osd.7 (osd.7) 110 : cluster 3 failed to encode map e174 with expected crc 2023-06-03T20:08:02.883 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:08:02 smithi089 bash[75068]: cluster 1685822882.6575038 mon.a (mon.0) 874 : cluster 3 Health check update: Degraded data redundancy: 11312/288396 objects degraded (3.922%), 2 pgs degraded (PG_DEGRADED) 2023-06-03T20:08:02.909 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:08:02.913 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:08:02.914 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:08:03.015 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:08:03.023 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:08:02 smithi164 bash[59462]: cluster 1685822880.9100063 osd.7 (osd.7) 109 : cluster 3 failed to encode map e174 with expected crc 2023-06-03T20:08:03.023 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:08:02 smithi164 bash[59462]: cluster 1685822880.911989 osd.7 (osd.7) 110 : cluster 3 failed to encode map e174 with expected crc 2023-06-03T20:08:03.024 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:08:02 smithi164 bash[59462]: cluster 1685822882.6575038 mon.a (mon.0) 874 : cluster 3 Health check update: Degraded data redundancy: 11312/288396 objects degraded (3.922%), 2 pgs degraded (PG_DEGRADED) 2023-06-03T20:08:03.047 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:08:03.049 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:08:03.050 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:08:03.151 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:08:03.160 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:08:02 smithi089 bash[78452]: cluster 1685822880.9100063 osd.7 (osd.7) 109 : cluster 3 failed to encode map e174 with expected crc 2023-06-03T20:08:03.161 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:08:02 smithi089 bash[78452]: cluster 1685822880.911989 osd.7 (osd.7) 110 : cluster 3 failed to encode map e174 with expected crc 2023-06-03T20:08:03.161 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:08:02 smithi089 bash[78452]: cluster 1685822882.6575038 mon.a (mon.0) 874 : cluster 3 Health check update: Degraded data redundancy: 11312/288396 objects degraded (3.922%), 2 pgs degraded (PG_DEGRADED) 2023-06-03T20:08:03.181 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:08:03.185 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:08:03.186 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:08:03.288 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:08:03.325 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:08:03.332 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:08:03.333 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:08:03.433 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:08:03.466 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:08:03.472 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:08:03.473 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:08:03.573 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:08:03.602 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:08:03.609 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:08:03.610 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:08:03.711 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:08:03.742 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:08:03.746 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:08:03.747 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:08:03.848 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:08:03.886 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:08:03.889 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:08:03.890 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:08:03.991 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:08:04.001 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:08:03 smithi164 bash[59462]: cluster 1685822882.595799 mgr.y (mgr.34104) 957 : cluster 0 pgmap v905: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 5.8 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 694 op/s; 11312/286341 objects degraded (3.951%); 4867/286341 objects misplaced (1.700%); 1024 KiB/s, 15 objects/s recovering 2023-06-03T20:08:04.021 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:08:04.025 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:08:04.026 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:08:04.102 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph osd set noscrub 2023-06-03T20:08:04.111 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:08:03 smithi089 bash[78452]: cluster 1685822882.595799 mgr.y (mgr.34104) 957 : cluster 0 pgmap v905: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 5.8 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 694 op/s; 11312/286341 objects degraded (3.951%); 4867/286341 objects misplaced (1.700%); 1024 KiB/s, 15 objects/s recovering 2023-06-03T20:08:04.113 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:08:03 smithi089 bash[75068]: cluster 1685822882.595799 mgr.y (mgr.34104) 957 : cluster 0 pgmap v905: 89 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 5.8 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 694 op/s; 11312/286341 objects degraded (3.951%); 4867/286341 objects misplaced (1.700%); 1024 KiB/s, 15 objects/s recovering 2023-06-03T20:08:04.127 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:08:04.179 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:08:04.186 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:08:04.186 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:08:04.287 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:08:04.322 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:08:04.323 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:08:04.323 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:08:04.424 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:08:04.452 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:08:04.454 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:08:04.455 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:08:04.556 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:08:04.601 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:08:04.603 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:08:04.604 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:08:04.626 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/mon.a/config 2023-06-03T20:08:04.626 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/mon.a/config 2023-06-03T20:08:04.705 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:08:04.755 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:08:04.764 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:08:04.764 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:08:04.865 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:08:04.905 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:08:04.909 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:08:04.909 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:08:05.010 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:08:05.045 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:08:05.047 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:08:05.048 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:08:05.149 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:08:05.186 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:08:05.193 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:08:05.194 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:08:05.295 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:08:05.335 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:08:05.339 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:08:05.340 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:08:05.441 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:08:05.473 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:08:05.476 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:08:05.477 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:08:05.577 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:08:05.614 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:08:05.618 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:08:05.619 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:08:05.720 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:08:05.752 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:08:05.754 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:08:05.754 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:08:05.856 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:08:05.899 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:08:05.904 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:08:05.904 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:08:06.005 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:08:06.028 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:08:05 smithi089 bash[78452]: cluster 1685822884.5965922 mgr.y (mgr.34104) 958 : cluster 0 pgmap v906: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 5.7 GiB data, 18 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 694 op/s; 11377/280419 objects degraded (4.057%); 4710/280419 objects misplaced (1.680%); 1.1 MiB/s, 17 objects/s recovering 2023-06-03T20:08:06.029 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:08:05 smithi089 bash[75068]: cluster 1685822884.5965922 mgr.y (mgr.34104) 958 : cluster 0 pgmap v906: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 5.7 GiB data, 18 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 694 op/s; 11377/280419 objects degraded (4.057%); 4710/280419 objects misplaced (1.680%); 1.1 MiB/s, 17 objects/s recovering 2023-06-03T20:08:06.052 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:08:06.059 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:08:06.060 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:08:06.161 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:08:06.174 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:08:05 smithi164 bash[59462]: cluster 1685822884.5965922 mgr.y (mgr.34104) 958 : cluster 0 pgmap v906: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 5.7 GiB data, 18 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 694 op/s; 11377/280419 objects degraded (4.057%); 4710/280419 objects misplaced (1.680%); 1.1 MiB/s, 17 objects/s recovering 2023-06-03T20:08:06.202 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:08:06.206 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:08:06.207 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:08:06.308 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:08:06.338 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:08:06.341 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:08:06.342 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:08:06.443 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:08:06.479 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:08:06.486 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:08:06.487 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:08:06.528 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 20:08:06 smithi089 bash[71163]: ::ffff:172.21.15.164 - - [03/Jun/2023:20:08:06] "GET /metrics HTTP/1.1" 200 228778 "" "Prometheus/2.33.4" 2023-06-03T20:08:06.588 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:08:06.636 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:08:06.641 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:08:06.642 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:08:06.675 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/mon.a/config 2023-06-03T20:08:06.743 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:08:06.771 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:08:06.773 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:08:06.774 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:08:06.875 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:08:06.905 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:08:06.908 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:08:06.909 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:08:07.010 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:08:07.078 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:08:07.084 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:08:07.085 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:08:07.185 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:08:07.247 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:08:07.250 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:08:07.250 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:08:07.351 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:08:07.389 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:08:07.391 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:08:07.392 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:08:07.493 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:08:07.524 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:08:07.527 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:08:07.527 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:08:07.628 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:08:07.673 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:08:07.682 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:08:07.682 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:08:07.783 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:08:07.834 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:08:07.837 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:08:07.838 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:08:07.939 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:08:07.973 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:08:07.980 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:08:07.981 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:08:08.082 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:08:08.083 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:08:07 smithi164 bash[59462]: cluster 1685822886.5971901 mgr.y (mgr.34104) 959 : cluster 0 pgmap v907: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 5.6 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 697 op/s; 11377/277713 objects degraded (4.097%); 4710/277713 objects misplaced (1.696%); 1.1 MiB/s, 17 objects/s recovering 2023-06-03T20:08:08.083 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:08:07 smithi164 bash[59462]: audit 1685822887.1199167 mon.a (mon.0) 875 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:08:08.083 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:08:07 smithi164 bash[59462]: audit 1685822887.1246507 mon.c (mon.2) 456 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T20:08:08.084 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:08:07 smithi164 bash[59462]: cluster 1685822887.658921 mon.a (mon.0) 876 : cluster 3 Health check update: Degraded data redundancy: 11377/277713 objects degraded (4.097%), 3 pgs degraded (PG_DEGRADED) 2023-06-03T20:08:08.113 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:08:08.115 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:08:08.116 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:08:08.217 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:08:08.224 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:08:07 smithi089 bash[78452]: cluster 1685822886.5971901 mgr.y (mgr.34104) 959 : cluster 0 pgmap v907: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 5.6 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 697 op/s; 11377/277713 objects degraded (4.097%); 4710/277713 objects misplaced (1.696%); 1.1 MiB/s, 17 objects/s recovering 2023-06-03T20:08:08.225 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:08:07 smithi089 bash[78452]: audit 1685822887.1199167 mon.a (mon.0) 875 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:08:08.225 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:08:07 smithi089 bash[78452]: audit 1685822887.1246507 mon.c (mon.2) 456 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T20:08:08.225 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:08:07 smithi089 bash[78452]: cluster 1685822887.658921 mon.a (mon.0) 876 : cluster 3 Health check update: Degraded data redundancy: 11377/277713 objects degraded (4.097%), 3 pgs degraded (PG_DEGRADED) 2023-06-03T20:08:08.225 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:08:07 smithi089 bash[75068]: cluster 1685822886.5971901 mgr.y (mgr.34104) 959 : cluster 0 pgmap v907: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 5.6 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 697 op/s; 11377/277713 objects degraded (4.097%); 4710/277713 objects misplaced (1.696%); 1.1 MiB/s, 17 objects/s recovering 2023-06-03T20:08:08.225 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:08:07 smithi089 bash[75068]: audit 1685822887.1199167 mon.a (mon.0) 875 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:08:08.226 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:08:07 smithi089 bash[75068]: audit 1685822887.1246507 mon.c (mon.2) 456 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T20:08:08.226 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:08:07 smithi089 bash[75068]: cluster 1685822887.658921 mon.a (mon.0) 876 : cluster 3 Health check update: Degraded data redundancy: 11377/277713 objects degraded (4.097%), 3 pgs degraded (PG_DEGRADED) 2023-06-03T20:08:08.251 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:08:08.254 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:08:08.255 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:08:08.355 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:08:08.384 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:08:08.386 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:08:08.387 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:08:08.487 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:08:08.517 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:08:08.521 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:08:08.522 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:08:08.623 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:08:08.675 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:08:08.678 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:08:08.678 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:08:08.720 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/mon.a/config 2023-06-03T20:08:08.779 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:08:08.809 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:08:08.816 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:08:08.817 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:08:08.917 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:08:08.944 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:08:08.947 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:08:08.948 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:08:09.049 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:08:09.084 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:08:09.085 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:08:09.086 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:08:09.187 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:08:09.245 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:08:09.247 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:08:09.248 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:08:09.349 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:08:09.383 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:08:09.386 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:08:09.387 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:08:09.488 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:08:09.530 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:08:09.533 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:08:09.533 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:08:09.634 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:08:09.675 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:08:09.678 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:08:09.679 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:08:09.780 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:08:09.816 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:08:09.820 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:08:09.821 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:08:09.900 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:08:09 smithi164 bash[59462]: cluster 1685822888.597938 mgr.y (mgr.34104) 960 : cluster 0 pgmap v908: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 5.5 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 697 op/s; 11465/270027 objects degraded (4.246%); 4535/270027 objects misplaced (1.679%); 1.1 MiB/s, 17 objects/s recovering 2023-06-03T20:08:09.922 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:08:09.957 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:08:09.959 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:08:09.960 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:08:10.061 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:08:10.100 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:08:10.102 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:08:10.103 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:08:10.204 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:08:10.214 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:08:09 smithi089 bash[78452]: cluster 1685822888.597938 mgr.y (mgr.34104) 960 : cluster 0 pgmap v908: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 5.5 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 697 op/s; 11465/270027 objects degraded (4.246%); 4535/270027 objects misplaced (1.679%); 1.1 MiB/s, 17 objects/s recovering 2023-06-03T20:08:10.216 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:08:09 smithi089 bash[75068]: cluster 1685822888.597938 mgr.y (mgr.34104) 960 : cluster 0 pgmap v908: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 5.5 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 697 op/s; 11465/270027 objects degraded (4.246%); 4535/270027 objects misplaced (1.679%); 1.1 MiB/s, 17 objects/s recovering 2023-06-03T20:08:10.236 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:08:10.238 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:08:10.239 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:08:10.340 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:08:10.370 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:08:10.374 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:08:10.375 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:08:10.476 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:08:10.517 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:08:10.521 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:08:10.522 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:08:10.623 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:08:10.651 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:08:10.654 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:08:10.654 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:08:10.755 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:08:10.787 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:08:10.793 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:08:10.794 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:08:10.895 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:08:10.929 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:08:10.934 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:08:10.934 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:08:11.035 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:08:11.080 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:08:11.082 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:08:11.083 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:08:11.184 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:08:11.223 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:08:11.228 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:08:11.229 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:08:11.329 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:08:11.362 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:08:11.363 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:08:11.364 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:08:11.465 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:08:11.513 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:08:11.518 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:08:11.519 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:08:11.620 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:08:11.673 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:08:11.682 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:08:11.682 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:08:11.783 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:08:11.818 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:08:11.825 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:08:11.826 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:08:11.927 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:08:11.957 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:08:11.962 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:08:11.962 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:08:12.063 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:08:12.086 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:08:12 smithi089 bash[78452]: cluster 1685822890.59855 mgr.y (mgr.34104) 961 : cluster 0 pgmap v909: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 5.4 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 696 op/s; 11465/267507 objects degraded (4.286%); 4535/267507 objects misplaced (1.695%); 1.1 MiB/s, 17 objects/s recovering 2023-06-03T20:08:12.086 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:08:12 smithi089 bash[78452]: audit 1685822891.522596 mon.c (mon.2) 457 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-03T20:08:12.090 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:08:12.094 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:08:12.095 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:08:12.196 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:08:12.206 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:08:11 smithi164 bash[59462]: cluster 1685822890.59855 mgr.y (mgr.34104) 961 : cluster 0 pgmap v909: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 5.4 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 696 op/s; 11465/267507 objects degraded (4.286%); 4535/267507 objects misplaced (1.695%); 1.1 MiB/s, 17 objects/s recovering 2023-06-03T20:08:12.206 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:08:11 smithi164 bash[59462]: audit 1685822891.522596 mon.c (mon.2) 457 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-03T20:08:12.236 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:08:12.240 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:08:12.240 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:08:12.341 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:08:12.350 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:08:12 smithi089 bash[75068]: cluster 1685822890.59855 mgr.y (mgr.34104) 961 : cluster 0 pgmap v909: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 5.4 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 696 op/s; 11465/267507 objects degraded (4.286%); 4535/267507 objects misplaced (1.695%); 1.1 MiB/s, 17 objects/s recovering 2023-06-03T20:08:12.350 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:08:12 smithi089 bash[75068]: audit 1685822891.522596 mon.c (mon.2) 457 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-03T20:08:12.386 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:08:12.390 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:08:12.391 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:08:12.492 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:08:12.529 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:08:12.531 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:08:12.532 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:08:12.633 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:08:12.675 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:08:12.683 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:08:12.683 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:08:12.784 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:08:12.825 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:08:12.828 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:08:12.829 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:08:12.915 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:08:12 smithi164 bash[59462]: cluster 1685822892.6608634 mon.a (mon.0) 877 : cluster 3 Health check update: Degraded data redundancy: 11465/267507 objects degraded (4.286%), 3 pgs degraded (PG_DEGRADED) 2023-06-03T20:08:12.930 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:08:12.960 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:08:12.964 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:08:12.965 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:08:13.066 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:08:13.095 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:08:13.097 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:08:13.098 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:08:13.199 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:08:13.252 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:08:13.256 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:08:13.257 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:08:13.278 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:08:12 smithi089 bash[78452]: cluster 1685822892.6608634 mon.a (mon.0) 877 : cluster 3 Health check update: Degraded data redundancy: 11465/267507 objects degraded (4.286%), 3 pgs degraded (PG_DEGRADED) 2023-06-03T20:08:13.278 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:08:12 smithi089 bash[75068]: cluster 1685822892.6608634 mon.a (mon.0) 877 : cluster 3 Health check update: Degraded data redundancy: 11465/267507 objects degraded (4.286%), 3 pgs degraded (PG_DEGRADED) 2023-06-03T20:08:13.358 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:08:13.394 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:08:13.400 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:08:13.400 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:08:13.501 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:08:13.533 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:08:13.536 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:08:13.537 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:08:13.638 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:08:13.670 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:08:13.673 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:08:13.674 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:08:13.774 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:08:13.814 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:08:13.818 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:08:13.819 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:08:13.920 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:08:13.955 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:08:13.961 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:08:13.962 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:08:14.063 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:08:14.072 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:08:13 smithi089 bash[78452]: cluster 1685822892.59922 mgr.y (mgr.34104) 962 : cluster 0 pgmap v910: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 5.4 GiB data, 17 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 646 op/s; 11465/265677 objects degraded (4.315%); 4535/265677 objects misplaced (1.707%); 977 KiB/s, 15 objects/s recovering 2023-06-03T20:08:14.073 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:08:13 smithi089 bash[75068]: cluster 1685822892.59922 mgr.y (mgr.34104) 962 : cluster 0 pgmap v910: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 5.4 GiB data, 17 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 646 op/s; 11465/265677 objects degraded (4.315%); 4535/265677 objects misplaced (1.707%); 977 KiB/s, 15 objects/s recovering 2023-06-03T20:08:14.083 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:08:13 smithi164 bash[59462]: cluster 1685822892.59922 mgr.y (mgr.34104) 962 : cluster 0 pgmap v910: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 5.4 GiB data, 17 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 646 op/s; 11465/265677 objects degraded (4.315%); 4535/265677 objects misplaced (1.707%); 977 KiB/s, 15 objects/s recovering 2023-06-03T20:08:14.097 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:08:14.102 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:08:14.104 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:08:14.205 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:08:14.252 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:08:14.255 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:08:14.256 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:08:14.356 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:08:14.385 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:08:14.388 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:08:14.388 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:08:14.489 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:08:14.531 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:08:14.537 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:08:14.537 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:08:14.638 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:08:14.669 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:08:14.671 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:08:14.672 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:08:14.773 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:08:14.808 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:08:14.816 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:08:14.817 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:08:14.917 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:08:14.956 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:08:14.956 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:08:14.957 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:08:15.058 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:08:15.094 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:08:15.097 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:08:15.097 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:08:15.198 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:08:15.231 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:08:15.236 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:08:15.237 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:08:15.337 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:08:15.377 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:08:15.380 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:08:15.381 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:08:15.482 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:08:15.531 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:08:15.539 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:08:15.540 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:08:15.642 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:08:15.671 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:08:15.672 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:08:15.673 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:08:15.774 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:08:15.811 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:08:15.820 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:08:15.821 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:08:15.922 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:08:15.923 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:08:15 smithi089 bash[78452]: cluster 1685822894.5998743 mgr.y (mgr.34104) 963 : cluster 0 pgmap v911: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 5.3 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 721 op/s; 11585/260346 objects degraded (4.450%); 4333/260346 objects misplaced (1.664%); 1.4 MiB/s, 21 objects/s recovering 2023-06-03T20:08:15.923 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:08:15 smithi089 bash[75068]: cluster 1685822894.5998743 mgr.y (mgr.34104) 963 : cluster 0 pgmap v911: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 5.3 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 721 op/s; 11585/260346 objects degraded (4.450%); 4333/260346 objects misplaced (1.664%); 1.4 MiB/s, 21 objects/s recovering 2023-06-03T20:08:15.953 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:08:15.955 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:08:15.956 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:08:16.057 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:08:16.067 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:08:15 smithi164 bash[59462]: cluster 1685822894.5998743 mgr.y (mgr.34104) 963 : cluster 0 pgmap v911: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 5.3 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 721 op/s; 11585/260346 objects degraded (4.450%); 4333/260346 objects misplaced (1.664%); 1.4 MiB/s, 21 objects/s recovering 2023-06-03T20:08:16.091 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:08:16.098 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:08:16.098 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:08:16.199 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:08:16.230 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:08:16.235 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:08:16.236 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:08:16.337 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:08:16.384 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:08:16.389 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:08:16.390 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:08:16.491 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:08:16.527 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:08:16.532 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:08:16.533 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:08:16.633 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:08:16.664 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:08:16.667 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:08:16.668 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:08:16.696 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 20:08:16 smithi089 bash[71163]: ::ffff:172.21.15.164 - - [03/Jun/2023:20:08:16] "GET /metrics HTTP/1.1" 200 228760 "" "Prometheus/2.33.4" 2023-06-03T20:08:16.768 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:08:16.805 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:08:16.810 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:08:16.811 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:08:16.911 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:08:16.972 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:08:16.980 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:08:16.980 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:08:17.081 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:08:17.127 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:08:17.141 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:08:17.142 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:08:17.242 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:08:17.309 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:08:17.312 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:08:17.313 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:08:17.414 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:08:17.453 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:08:17.455 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:08:17.455 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:08:17.556 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:08:17.588 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:08:17.593 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:08:17.593 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:08:17.694 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:08:17.747 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:08:17.752 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:08:17.753 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:08:17.853 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:08:17.896 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:08:17.900 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:08:17.901 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:08:18.002 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:08:18.040 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:08:18.043 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:08:18.043 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:08:18.083 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:08:17 smithi164 bash[59462]: cluster 1685822896.6005356 mgr.y (mgr.34104) 964 : cluster 0 pgmap v912: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 5.2 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 624 op/s; 11585/257931 objects degraded (4.492%); 4333/257931 objects misplaced (1.680%); 901 KiB/s, 14 objects/s recovering 2023-06-03T20:08:18.083 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:08:17 smithi164 bash[59462]: cluster 1685822897.6618943 mon.a (mon.0) 878 : cluster 3 Health check update: Degraded data redundancy: 11585/257931 objects degraded (4.492%), 3 pgs degraded (PG_DEGRADED) 2023-06-03T20:08:18.144 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:08:18.175 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:08:18.178 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:08:18.179 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:08:18.179 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:08:17 smithi089 bash[78452]: cluster 1685822896.6005356 mgr.y (mgr.34104) 964 : cluster 0 pgmap v912: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 5.2 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 624 op/s; 11585/257931 objects degraded (4.492%); 4333/257931 objects misplaced (1.680%); 901 KiB/s, 14 objects/s recovering 2023-06-03T20:08:18.180 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:08:17 smithi089 bash[78452]: cluster 1685822897.6618943 mon.a (mon.0) 878 : cluster 3 Health check update: Degraded data redundancy: 11585/257931 objects degraded (4.492%), 3 pgs degraded (PG_DEGRADED) 2023-06-03T20:08:18.180 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:08:17 smithi089 bash[75068]: cluster 1685822896.6005356 mgr.y (mgr.34104) 964 : cluster 0 pgmap v912: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 5.2 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 624 op/s; 11585/257931 objects degraded (4.492%); 4333/257931 objects misplaced (1.680%); 901 KiB/s, 14 objects/s recovering 2023-06-03T20:08:18.180 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:08:17 smithi089 bash[75068]: cluster 1685822897.6618943 mon.a (mon.0) 878 : cluster 3 Health check update: Degraded data redundancy: 11585/257931 objects degraded (4.492%), 3 pgs degraded (PG_DEGRADED) 2023-06-03T20:08:18.280 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:08:18.306 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:08:18.310 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:08:18.311 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:08:18.412 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:08:18.440 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:08:18.442 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:08:18.443 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:08:18.543 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:08:18.581 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:08:18.586 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:08:18.586 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:08:18.687 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:08:18.732 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:08:18.735 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:08:18.735 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:08:18.836 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:08:18.867 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:08:18.870 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:08:18.871 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:08:18.972 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:08:19.024 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:08:19.030 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:08:19.030 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:08:19.131 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:08:19.165 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:08:19.169 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:08:19.170 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:08:19.271 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:08:19.273 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:08:19 smithi164 bash[59462]: audit 1685822898.273953 mon.a (mon.0) 879 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:08:19.273 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:08:19 smithi164 bash[59462]: audit 1685822898.2839751 mon.a (mon.0) 880 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:08:19.273 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:08:19 smithi164 bash[59462]: audit 1685822898.4717886 mon.a (mon.0) 881 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:08:19.308 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:08:19.315 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:08:19.316 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:08:19.417 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:08:19.450 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:08:19.454 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:08:19.454 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:08:19.531 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:08:19 smithi089 bash[75068]: audit 1685822898.273953 mon.a (mon.0) 879 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:08:19.532 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:08:19 smithi089 bash[75068]: audit 1685822898.2839751 mon.a (mon.0) 880 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:08:19.532 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:08:19 smithi089 bash[75068]: audit 1685822898.4717886 mon.a (mon.0) 881 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:08:19.532 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:08:19 smithi089 bash[75068]: audit 1685822898.481652 mon.a (mon.0) 882 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:08:19.532 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:08:19 smithi089 bash[75068]: cluster 1685822898.6013925 mgr.y (mgr.34104) 965 : cluster 0 pgmap v913: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 5.1 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 734 op/s; 11690/251274 objects degraded (4.652%); 4145/251274 objects misplaced (1.650%); 1.3 MiB/s, 20 objects/s recovering 2023-06-03T20:08:19.532 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:08:19 smithi089 bash[75068]: audit 1685822899.117392 mon.c (mon.2) 458 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-03T20:08:19.533 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:08:19 smithi089 bash[75068]: audit 1685822899.1249764 mon.c (mon.2) 459 : audit 1 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-03T20:08:19.533 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:08:19 smithi089 bash[75068]: audit 1685822899.1372948 mon.a (mon.0) 883 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:08:19.533 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:08:19 smithi089 bash[78452]: audit 1685822898.273953 mon.a (mon.0) 879 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:08:19.533 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:08:19 smithi089 bash[78452]: audit 1685822898.2839751 mon.a (mon.0) 880 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:08:19.533 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:08:19 smithi089 bash[78452]: audit 1685822898.4717886 mon.a (mon.0) 881 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:08:19.533 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:08:19 smithi089 bash[78452]: audit 1685822898.481652 mon.a (mon.0) 882 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:08:19.534 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:08:19 smithi089 bash[78452]: cluster 1685822898.6013925 mgr.y (mgr.34104) 965 : cluster 0 pgmap v913: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 5.1 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 734 op/s; 11690/251274 objects degraded (4.652%); 4145/251274 objects misplaced (1.650%); 1.3 MiB/s, 20 objects/s recovering 2023-06-03T20:08:19.534 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:08:19 smithi089 bash[78452]: audit 1685822899.117392 mon.c (mon.2) 458 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-03T20:08:19.534 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:08:19 smithi089 bash[78452]: audit 1685822899.1249764 mon.c (mon.2) 459 : audit 1 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-03T20:08:19.534 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:08:19 smithi089 bash[78452]: audit 1685822899.1372948 mon.a (mon.0) 883 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:08:19.555 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:08:19.583 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:08:19 smithi164 bash[59462]: audit 1685822898.481652 mon.a (mon.0) 882 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:08:19.583 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:08:19 smithi164 bash[59462]: cluster 1685822898.6013925 mgr.y (mgr.34104) 965 : cluster 0 pgmap v913: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 5.1 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 734 op/s; 11690/251274 objects degraded (4.652%); 4145/251274 objects misplaced (1.650%); 1.3 MiB/s, 20 objects/s recovering 2023-06-03T20:08:19.584 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:08:19 smithi164 bash[59462]: audit 1685822899.117392 mon.c (mon.2) 458 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-03T20:08:19.584 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:08:19 smithi164 bash[59462]: audit 1685822899.1249764 mon.c (mon.2) 459 : audit 1 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-03T20:08:19.584 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:08:19 smithi164 bash[59462]: audit 1685822899.1372948 mon.a (mon.0) 883 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:08:19.592 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:08:19.594 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:08:19.595 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:08:19.696 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:08:19.723 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:08:19.725 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:08:19.726 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:08:19.827 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:08:19.864 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:08:19.868 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:08:19.869 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:08:19.970 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:08:20.003 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:08:20.007 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:08:20.008 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:08:20.109 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:08:20.141 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:08:20.144 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:08:20.145 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:08:20.198 INFO:teuthology.orchestra.run.smithi089.stdout:{ 2023-06-03T20:08:20.199 INFO:teuthology.orchestra.run.smithi089.stdout: "success": "filestore_inject_stall = '3' (not observed, change may require restart) " 2023-06-03T20:08:20.199 INFO:teuthology.orchestra.run.smithi089.stdout:} 2023-06-03T20:08:20.246 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:08:20.282 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:08:20.284 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:08:20.284 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:08:20.385 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:08:20.430 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:08:20.432 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:08:20.432 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:08:20.533 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:08:20.561 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:08:20.563 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:08:20.563 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:08:20.664 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:08:20.692 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:08:20.693 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:08:20.694 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:08:20.795 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:08:20.832 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:08:20.833 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:08:20.834 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:08:20.935 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:08:20.961 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:08:20.966 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:08:20.967 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:08:21.068 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:08:21.100 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:08:21.102 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:08:21.103 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:08:21.204 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:08:21.239 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:08:21.241 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:08:21.241 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:08:21.342 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:08:21.373 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:08:21.377 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:08:21.378 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:08:21.479 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:08:21.511 INFO:teuthology.orchestra.run.smithi089.stdout:{ 2023-06-03T20:08:21.511 INFO:teuthology.orchestra.run.smithi089.stdout: "ops": [], 2023-06-03T20:08:21.512 INFO:teuthology.orchestra.run.smithi089.stdout: "complaint_time": 30, 2023-06-03T20:08:21.512 INFO:teuthology.orchestra.run.smithi089.stdout: "num_blocked_ops": 0 2023-06-03T20:08:21.512 INFO:teuthology.orchestra.run.smithi089.stdout:} 2023-06-03T20:08:21.523 INFO:teuthology.orchestra.run.smithi089.stdout:osd.0: {} 2023-06-03T20:08:21.528 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:08:21.534 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:08:21.535 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:08:21.555 INFO:teuthology.orchestra.run.smithi089.stdout:osd.1: {} 2023-06-03T20:08:21.605 INFO:teuthology.orchestra.run.smithi089.stdout:osd.2: {} 2023-06-03T20:08:21.636 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:08:21.637 INFO:teuthology.orchestra.run.smithi089.stdout:osd.3: {} 2023-06-03T20:08:21.669 INFO:teuthology.orchestra.run.smithi089.stdout:osd.4: {} 2023-06-03T20:08:21.680 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:08:21.685 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:08:21.686 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:08:21.702 INFO:teuthology.orchestra.run.smithi089.stdout:osd.5: {} 2023-06-03T20:08:21.740 INFO:teuthology.orchestra.run.smithi089.stdout:osd.6: {} 2023-06-03T20:08:21.766 INFO:teuthology.orchestra.run.smithi089.stdout:osd.7: {} 2023-06-03T20:08:21.774 INFO:teuthology.orchestra.run.smithi089.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-06-03T20:08:21.774 INFO:teuthology.orchestra.run.smithi089.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-06-03T20:08:21.775 INFO:teuthology.orchestra.run.smithi089.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-06-03T20:08:21.775 INFO:teuthology.orchestra.run.smithi089.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-06-03T20:08:21.776 INFO:teuthology.orchestra.run.smithi089.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-06-03T20:08:21.776 INFO:teuthology.orchestra.run.smithi089.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-06-03T20:08:21.776 INFO:teuthology.orchestra.run.smithi089.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-06-03T20:08:21.776 INFO:teuthology.orchestra.run.smithi089.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-06-03T20:08:21.787 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:08:21.812 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:08:21.814 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:08:21.815 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:08:21.833 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:08:21 smithi164 bash[59462]: cluster 1685822900.602063 mgr.y (mgr.34104) 966 : cluster 0 pgmap v914: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 5.1 GiB data, 16 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 577 op/s; 11690/249231 objects degraded (4.690%); 4145/249231 objects misplaced (1.663%); 880 KiB/s, 13 objects/s recovering 2023-06-03T20:08:21.833 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:08:21 smithi164 bash[59462]: audit 1685822901.483584 mon.c (mon.2) 460 : audit 0 from='client.? 172.21.15.89:0/2149276579' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-03T20:08:21.916 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:08:21.962 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:08:21.964 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:08:21.965 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:08:22.028 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:08:21 smithi089 bash[78452]: cluster 1685822900.602063 mgr.y (mgr.34104) 966 : cluster 0 pgmap v914: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 5.1 GiB data, 16 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 577 op/s; 11690/249231 objects degraded (4.690%); 4145/249231 objects misplaced (1.663%); 880 KiB/s, 13 objects/s recovering 2023-06-03T20:08:22.029 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:08:21 smithi089 bash[78452]: audit 1685822901.483584 mon.c (mon.2) 460 : audit 0 from='client.? 172.21.15.89:0/2149276579' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-03T20:08:22.029 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:08:21 smithi089 bash[75068]: cluster 1685822900.602063 mgr.y (mgr.34104) 966 : cluster 0 pgmap v914: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 5.1 GiB data, 16 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 577 op/s; 11690/249231 objects degraded (4.690%); 4145/249231 objects misplaced (1.663%); 880 KiB/s, 13 objects/s recovering 2023-06-03T20:08:22.029 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:08:21 smithi089 bash[75068]: audit 1685822901.483584 mon.c (mon.2) 460 : audit 0 from='client.? 172.21.15.89:0/2149276579' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-03T20:08:22.066 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:08:22.105 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:08:22.112 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:08:22.113 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:08:22.214 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:08:22.260 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:08:22.266 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:08:22.267 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:08:22.369 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:08:22.393 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:08:22.395 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:08:22.395 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:08:22.496 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:08:22.555 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:08:22.559 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:08:22.559 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:08:22.660 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:08:22.700 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:08:22.708 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:08:22.709 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:08:22.810 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:08:22.833 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:08:22 smithi164 bash[59462]: audit 1685822902.1112764 mon.c (mon.2) 461 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T20:08:22.834 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:08:22 smithi164 bash[59462]: cluster 1685822902.662839 mon.a (mon.0) 884 : cluster 3 Health check update: Degraded data redundancy: 11690/249231 objects degraded (4.690%), 3 pgs degraded (PG_DEGRADED) 2023-06-03T20:08:22.844 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:08:22.848 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:08:22.849 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:08:22.950 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:08:22.958 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:08:22 smithi089 bash[75068]: audit 1685822902.1112764 mon.c (mon.2) 461 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T20:08:22.958 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:08:22 smithi089 bash[75068]: cluster 1685822902.662839 mon.a (mon.0) 884 : cluster 3 Health check update: Degraded data redundancy: 11690/249231 objects degraded (4.690%), 3 pgs degraded (PG_DEGRADED) 2023-06-03T20:08:22.958 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:08:22 smithi089 bash[78452]: audit 1685822902.1112764 mon.c (mon.2) 461 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T20:08:22.959 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:08:22 smithi089 bash[78452]: cluster 1685822902.662839 mon.a (mon.0) 884 : cluster 3 Health check update: Degraded data redundancy: 11690/249231 objects degraded (4.690%), 3 pgs degraded (PG_DEGRADED) 2023-06-03T20:08:22.990 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:08:22.993 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:08:22.994 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:08:23.095 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:08:23.126 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:08:23.129 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:08:23.130 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:08:23.231 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:08:23.278 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:08:23.283 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:08:23.284 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:08:23.385 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:08:23.423 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:08:23.426 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:08:23.427 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:08:23.528 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:08:23.565 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:08:23.570 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:08:23.571 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:08:23.672 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:08:23.677 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:08:23 smithi164 bash[59462]: cluster 1685822902.6027286 mgr.y (mgr.34104) 967 : cluster 0 pgmap v915: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 5.0 GiB data, 16 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 555 op/s; 11690/247506 objects degraded (4.723%); 4145/247506 objects misplaced (1.675%); 880 KiB/s, 13 objects/s recovering 2023-06-03T20:08:23.708 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:08:23.710 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:08:23.711 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:08:23.778 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:08:23 smithi089 bash[75068]: cluster 1685822902.6027286 mgr.y (mgr.34104) 967 : cluster 0 pgmap v915: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 5.0 GiB data, 16 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 555 op/s; 11690/247506 objects degraded (4.723%); 4145/247506 objects misplaced (1.675%); 880 KiB/s, 13 objects/s recovering 2023-06-03T20:08:23.779 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:08:23 smithi089 bash[78452]: cluster 1685822902.6027286 mgr.y (mgr.34104) 967 : cluster 0 pgmap v915: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 5.0 GiB data, 16 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 555 op/s; 11690/247506 objects degraded (4.723%); 4145/247506 objects misplaced (1.675%); 880 KiB/s, 13 objects/s recovering 2023-06-03T20:08:23.812 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:08:23.851 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:08:23.853 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:08:23.854 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:08:23.955 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:08:23.994 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:08:23.998 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:08:23.999 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:08:24.100 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:08:24.128 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:08:24.130 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:08:24.130 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:08:24.231 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:08:24.263 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:08:24.264 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:08:24.264 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:08:24.365 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:08:24.415 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:08:24.416 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:08:24.417 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:08:24.518 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:08:24.552 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:08:24.554 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:08:24.554 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:08:24.655 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:08:24.701 INFO:teuthology.orchestra.run.smithi089.stderr:noscrub is set 2023-06-03T20:08:24.704 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:08:24.708 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:08:24.709 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:08:24.778 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:08:24 smithi089 bash[75068]: audit 1685822904.4688778 mon.a (mon.0) 885 : audit 1 from='client.? 172.21.15.89:0/3062771267' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-06-03T20:08:24.778 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:08:24 smithi089 bash[78452]: audit 1685822904.4688778 mon.a (mon.0) 885 : audit 1 from='client.? 172.21.15.89:0/3062771267' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-06-03T20:08:24.810 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:08:24.847 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:08:24.849 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:08:24.849 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:08:24.950 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:08:24.960 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:08:24 smithi164 bash[59462]: audit 1685822904.4688778 mon.a (mon.0) 885 : audit 1 from='client.? 172.21.15.89:0/3062771267' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-06-03T20:08:24.984 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:08:24.990 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:08:24.990 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:08:25.091 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:08:25.137 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:08:25.142 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:08:25.143 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:08:25.244 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:08:25.279 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:08:25.288 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:08:25.289 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:08:25.389 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:08:25.423 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:08:25.425 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:08:25.426 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:08:25.518 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph daemon osd.1 dump_historic_ops 2023-06-03T20:08:25.527 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:08:25.597 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:08:25.599 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:08:25.599 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:08:25.700 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:08:25.736 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:08:25.742 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:08:25.742 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:08:25.778 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:08:25 smithi089 bash[78452]: cluster 1685822904.6034076 mgr.y (mgr.34104) 968 : cluster 0 pgmap v916: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 4.9 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 651 op/s; 11792/242205 objects degraded (4.869%); 3955/242205 objects misplaced (1.633%); 1.3 MiB/s, 21 objects/s recovering 2023-06-03T20:08:25.779 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:08:25 smithi089 bash[78452]: cluster 1685822904.6721964 mon.a (mon.0) 886 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-06-03T20:08:25.779 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:08:25 smithi089 bash[78452]: cluster 1685822904.6862805 osd.6 (osd.6) 111 : cluster 3 failed to encode map e175 with expected crc 2023-06-03T20:08:25.779 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:08:25 smithi089 bash[78452]: audit 1685822904.6869292 mon.a (mon.0) 887 : audit 1 from='client.? 172.21.15.89:0/3062771267' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-06-03T20:08:25.779 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:08:25 smithi089 bash[78452]: cluster 1685822904.6869752 mon.a (mon.0) 888 : cluster 0 osdmap e175: 8 total, 8 up, 7 in 2023-06-03T20:08:25.779 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:08:25 smithi089 bash[78452]: cluster 1685822904.6901972 osd.6 (osd.6) 112 : cluster 3 failed to encode map e175 with expected crc 2023-06-03T20:08:25.779 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:08:25 smithi089 bash[78452]: cluster 1685822904.6921425 osd.5 (osd.5) 105 : cluster 3 failed to encode map e175 with expected crc 2023-06-03T20:08:25.779 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:08:25 smithi089 bash[78452]: cluster 1685822904.69893 osd.7 (osd.7) 111 : cluster 3 failed to encode map e175 with expected crc 2023-06-03T20:08:25.780 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:08:25 smithi089 bash[75068]: cluster 1685822904.6034076 mgr.y (mgr.34104) 968 : cluster 0 pgmap v916: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 4.9 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 651 op/s; 11792/242205 objects degraded (4.869%); 3955/242205 objects misplaced (1.633%); 1.3 MiB/s, 21 objects/s recovering 2023-06-03T20:08:25.780 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:08:25 smithi089 bash[75068]: cluster 1685822904.6721964 mon.a (mon.0) 886 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-06-03T20:08:25.780 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:08:25 smithi089 bash[75068]: cluster 1685822904.6862805 osd.6 (osd.6) 111 : cluster 3 failed to encode map e175 with expected crc 2023-06-03T20:08:25.780 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:08:25 smithi089 bash[75068]: audit 1685822904.6869292 mon.a (mon.0) 887 : audit 1 from='client.? 172.21.15.89:0/3062771267' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-06-03T20:08:25.780 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:08:25 smithi089 bash[75068]: cluster 1685822904.6869752 mon.a (mon.0) 888 : cluster 0 osdmap e175: 8 total, 8 up, 7 in 2023-06-03T20:08:25.781 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:08:25 smithi089 bash[75068]: cluster 1685822904.6901972 osd.6 (osd.6) 112 : cluster 3 failed to encode map e175 with expected crc 2023-06-03T20:08:25.781 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:08:25 smithi089 bash[75068]: cluster 1685822904.6921425 osd.5 (osd.5) 105 : cluster 3 failed to encode map e175 with expected crc 2023-06-03T20:08:25.781 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:08:25 smithi089 bash[75068]: cluster 1685822904.69893 osd.7 (osd.7) 111 : cluster 3 failed to encode map e175 with expected crc 2023-06-03T20:08:25.843 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:08:25.872 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:08:25.877 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:08:25.878 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:08:25.979 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:08:26.008 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:08:26.012 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:08:26.013 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:08:26.083 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:08:25 smithi164 bash[59462]: cluster 1685822904.6034076 mgr.y (mgr.34104) 968 : cluster 0 pgmap v916: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 4.9 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 651 op/s; 11792/242205 objects degraded (4.869%); 3955/242205 objects misplaced (1.633%); 1.3 MiB/s, 21 objects/s recovering 2023-06-03T20:08:26.084 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:08:25 smithi164 bash[59462]: cluster 1685822904.6721964 mon.a (mon.0) 886 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-06-03T20:08:26.084 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:08:25 smithi164 bash[59462]: cluster 1685822904.6862805 osd.6 (osd.6) 111 : cluster 3 failed to encode map e175 with expected crc 2023-06-03T20:08:26.084 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:08:25 smithi164 bash[59462]: audit 1685822904.6869292 mon.a (mon.0) 887 : audit 1 from='client.? 172.21.15.89:0/3062771267' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-06-03T20:08:26.084 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:08:25 smithi164 bash[59462]: cluster 1685822904.6869752 mon.a (mon.0) 888 : cluster 0 osdmap e175: 8 total, 8 up, 7 in 2023-06-03T20:08:26.084 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:08:25 smithi164 bash[59462]: cluster 1685822904.6901972 osd.6 (osd.6) 112 : cluster 3 failed to encode map e175 with expected crc 2023-06-03T20:08:26.084 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:08:25 smithi164 bash[59462]: cluster 1685822904.6921425 osd.5 (osd.5) 105 : cluster 3 failed to encode map e175 with expected crc 2023-06-03T20:08:26.085 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:08:25 smithi164 bash[59462]: cluster 1685822904.69893 osd.7 (osd.7) 111 : cluster 3 failed to encode map e175 with expected crc 2023-06-03T20:08:26.113 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:08:26.152 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:08:26.158 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:08:26.159 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:08:26.231 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 20:08:26 smithi089 bash[71163]: ::ffff:172.21.15.164 - - [03/Jun/2023:20:08:26] "GET /metrics HTTP/1.1" 200 228730 "" "Prometheus/2.33.4" 2023-06-03T20:08:26.260 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:08:26.302 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:08:26.305 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:08:26.305 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:08:26.406 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:08:26.449 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:08:26.452 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:08:26.453 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:08:26.554 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:08:26.590 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:08:26.596 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:08:26.597 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:08:26.698 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:08:26.734 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:08:26.741 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:08:26.742 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:08:26.843 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:08:26.881 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:08:26.885 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:08:26.886 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:08:26.987 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:08:27.034 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:08:27.036 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:08:27.037 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:08:27.138 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:08:27.170 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:08:27.172 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:08:27.173 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:08:27.275 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:08:27.316 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:08:27.319 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:08:27.320 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:08:27.421 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:08:27.471 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:08:27.473 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:08:27.474 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:08:27.575 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:08:27.607 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:08:27.610 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:08:27.610 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:08:27.686 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:08:27 smithi164 bash[59462]: cluster 1685822906.6041224 mgr.y (mgr.34104) 969 : cluster 0 pgmap v918: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 4.9 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 601 op/s; 11792/239883 objects degraded (4.916%); 3955/239883 objects misplaced (1.649%); 1.1 MiB/s, 17 objects/s recovering 2023-06-03T20:08:27.686 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:08:27 smithi164 bash[59462]: cluster 1685822907.6639392 mon.a (mon.0) 889 : cluster 3 Health check update: Degraded data redundancy: 11792/239883 objects degraded (4.916%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-06-03T20:08:27.692 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-06-03T20:08:27.711 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:08:27.765 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:08:27.768 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:08:27.769 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:08:27.870 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:08:27.915 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:08:27.918 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:08:27.919 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:08:28.020 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:08:28.028 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:08:27 smithi089 bash[75068]: cluster 1685822906.6041224 mgr.y (mgr.34104) 969 : cluster 0 pgmap v918: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 4.9 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 601 op/s; 11792/239883 objects degraded (4.916%); 3955/239883 objects misplaced (1.649%); 1.1 MiB/s, 17 objects/s recovering 2023-06-03T20:08:28.028 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:08:27 smithi089 bash[75068]: cluster 1685822907.6639392 mon.a (mon.0) 889 : cluster 3 Health check update: Degraded data redundancy: 11792/239883 objects degraded (4.916%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-06-03T20:08:28.029 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:08:27 smithi089 bash[78452]: cluster 1685822906.6041224 mgr.y (mgr.34104) 969 : cluster 0 pgmap v918: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 4.9 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 601 op/s; 11792/239883 objects degraded (4.916%); 3955/239883 objects misplaced (1.649%); 1.1 MiB/s, 17 objects/s recovering 2023-06-03T20:08:28.029 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:08:27 smithi089 bash[78452]: cluster 1685822907.6639392 mon.a (mon.0) 889 : cluster 3 Health check update: Degraded data redundancy: 11792/239883 objects degraded (4.916%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-06-03T20:08:28.052 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:08:28.058 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:08:28.059 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:08:28.161 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:08:28.199 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:08:28.200 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:08:28.201 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:08:28.268 INFO:tasks.thrashosds.thrasher:in_osds: [0, 2, 3, 4, 5, 6, 7] out_osds: [1] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-06-03T20:08:28.268 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph osd reweight 0 1 2023-06-03T20:08:28.302 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:08:28.328 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:08:28.334 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:08:28.334 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:08:28.409 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph osd set nodeep-scrub 2023-06-03T20:08:28.435 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:08:28.467 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:08:28.473 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:08:28.473 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:08:28.574 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:08:28.612 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:08:28.617 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:08:28.618 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:08:28.719 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:08:28.764 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:08:28.769 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:08:28.770 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:08:28.870 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:08:28.908 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:08:28.911 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:08:28.912 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:08:29.013 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:08:29.047 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:08:29.051 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:08:29.052 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:08:29.153 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:08:29.191 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:08:29.199 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:08:29.199 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:08:29.300 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:08:29.340 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:08:29.344 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:08:29.344 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:08:29.445 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:08:29.478 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:08:29.482 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:08:29.482 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:08:29.583 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:08:29.627 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:08:29.628 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:08:29.629 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:08:29.730 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:08:29.762 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:08:29.765 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:08:29.766 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:08:29.778 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:08:29 smithi089 bash[78452]: cluster 1685822908.6049201 mgr.y (mgr.34104) 970 : cluster 0 pgmap v919: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 4.7 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 627 op/s; 11889/232431 objects degraded (5.115%); 3767/232431 objects misplaced (1.621%); 1.1 MiB/s, 17 objects/s recovering 2023-06-03T20:08:29.778 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:08:29 smithi089 bash[75068]: cluster 1685822908.6049201 mgr.y (mgr.34104) 970 : cluster 0 pgmap v919: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 4.7 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 627 op/s; 11889/232431 objects degraded (5.115%); 3767/232431 objects misplaced (1.621%); 1.1 MiB/s, 17 objects/s recovering 2023-06-03T20:08:29.867 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:08:29.900 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:08:29.908 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:08:29.909 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:08:30.010 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:08:30.018 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:08:29 smithi164 bash[59462]: cluster 1685822908.6049201 mgr.y (mgr.34104) 970 : cluster 0 pgmap v919: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 4.7 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 627 op/s; 11889/232431 objects degraded (5.115%); 3767/232431 objects misplaced (1.621%); 1.1 MiB/s, 17 objects/s recovering 2023-06-03T20:08:30.047 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:08:30.054 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:08:30.055 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:08:30.156 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:08:30.196 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:08:30.202 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:08:30.202 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:08:30.303 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:08:30.342 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:08:30.343 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:08:30.343 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:08:30.383 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/mon.a/config 2023-06-03T20:08:30.444 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:08:30.483 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:08:30.487 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:08:30.487 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:08:30.588 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:08:30.623 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:08:30.625 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:08:30.626 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:08:30.726 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:08:30.768 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:08:30.770 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:08:30.770 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:08:30.871 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:08:30.902 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:08:30.909 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:08:30.910 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:08:31.011 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:08:31.052 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:08:31.054 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:08:31.055 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:08:31.156 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:08:31.189 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:08:31.191 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:08:31.192 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:08:31.293 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:08:31.325 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:08:31.327 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:08:31.328 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:08:31.429 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:08:31.462 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:08:31.465 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:08:31.465 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:08:31.566 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:08:31.608 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:08:31.613 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:08:31.614 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:08:31.715 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:08:31.749 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:08:31.752 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:08:31.752 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:08:31.833 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:08:31 smithi164 bash[59462]: cluster 1685822910.6056087 mgr.y (mgr.34104) 971 : cluster 0 pgmap v920: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 4.7 GiB data, 15 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 643 op/s; 11889/229905 objects degraded (5.171%); 3767/229905 objects misplaced (1.639%); 1.1 MiB/s, 17 objects/s recovering 2023-06-03T20:08:31.854 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:08:31.891 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:08:31.892 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:08:31.892 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:08:31.993 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:08:32.002 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:08:31 smithi089 bash[78452]: cluster 1685822910.6056087 mgr.y (mgr.34104) 971 : cluster 0 pgmap v920: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 4.7 GiB data, 15 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 643 op/s; 11889/229905 objects degraded (5.171%); 3767/229905 objects misplaced (1.639%); 1.1 MiB/s, 17 objects/s recovering 2023-06-03T20:08:32.002 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:08:31 smithi089 bash[75068]: cluster 1685822910.6056087 mgr.y (mgr.34104) 971 : cluster 0 pgmap v920: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 4.7 GiB data, 15 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 643 op/s; 11889/229905 objects degraded (5.171%); 3767/229905 objects misplaced (1.639%); 1.1 MiB/s, 17 objects/s recovering 2023-06-03T20:08:32.029 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:08:32.033 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:08:32.034 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:08:32.135 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:08:32.170 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:08:32.174 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:08:32.175 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:08:32.276 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:08:32.313 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:08:32.316 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:08:32.317 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:08:32.418 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:08:32.430 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/mon.a/config 2023-06-03T20:08:32.431 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/mon.a/config 2023-06-03T20:08:32.435 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/mon.a/config 2023-06-03T20:08:32.464 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:08:32.468 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:08:32.468 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:08:32.569 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:08:32.606 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:08:32.612 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:08:32.613 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:08:32.713 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:08:32.722 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:08:32 smithi089 bash[78452]: cluster 1685822912.6649826 mon.a (mon.0) 890 : cluster 3 Health check update: Degraded data redundancy: 11889/229905 objects degraded (5.171%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-06-03T20:08:32.723 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:08:32 smithi089 bash[75068]: cluster 1685822912.6649826 mon.a (mon.0) 890 : cluster 3 Health check update: Degraded data redundancy: 11889/229905 objects degraded (5.171%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-06-03T20:08:32.742 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:08:32.744 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:08:32.745 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:08:32.846 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:08:32.886 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:08:32.889 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:08:32.890 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:08:32.991 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:08:33.000 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:08:32 smithi164 bash[59462]: cluster 1685822912.6649826 mon.a (mon.0) 890 : cluster 3 Health check update: Degraded data redundancy: 11889/229905 objects degraded (5.171%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-06-03T20:08:33.026 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:08:33.030 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:08:33.031 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:08:33.132 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:08:33.167 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:08:33.170 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:08:33.170 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:08:33.271 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:08:33.299 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:08:33.302 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:08:33.303 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:08:33.403 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:08:33.442 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:08:33.446 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:08:33.447 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:08:33.549 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:08:33.577 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:08:33.583 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:08:33.584 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:08:33.685 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:08:33.719 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:08:33.722 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:08:33.723 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:08:33.824 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:08:33.831 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:08:33 smithi089 bash[75068]: cluster 1685822912.606249 mgr.y (mgr.34104) 972 : cluster 0 pgmap v921: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 4.6 GiB data, 15 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 645 op/s; 11889/228147 objects degraded (5.211%); 3767/228147 objects misplaced (1.651%); 1.1 MiB/s, 17 objects/s recovering 2023-06-03T20:08:33.832 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:08:33 smithi089 bash[78452]: cluster 1685822912.606249 mgr.y (mgr.34104) 972 : cluster 0 pgmap v921: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 4.6 GiB data, 15 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 645 op/s; 11889/228147 objects degraded (5.211%); 3767/228147 objects misplaced (1.651%); 1.1 MiB/s, 17 objects/s recovering 2023-06-03T20:08:33.861 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:08:33.863 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:08:33.864 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:08:33.965 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:08:33.998 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:08:34.000 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:08:34.000 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:08:34.083 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:08:33 smithi164 bash[59462]: cluster 1685822912.606249 mgr.y (mgr.34104) 972 : cluster 0 pgmap v921: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 4.6 GiB data, 15 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 645 op/s; 11889/228147 objects degraded (5.211%); 3767/228147 objects misplaced (1.651%); 1.1 MiB/s, 17 objects/s recovering 2023-06-03T20:08:34.102 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:08:34.134 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:08:34.138 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:08:34.138 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:08:34.239 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:08:34.278 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:08:34.285 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:08:34.286 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:08:34.386 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:08:34.415 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:08:34.418 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:08:34.418 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:08:34.519 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:08:34.560 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:08:34.565 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:08:34.566 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:08:34.666 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:08:34.702 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:08:34.705 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:08:34.706 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:08:34.807 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:08:34.836 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:08:34.842 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:08:34.843 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:08:34.944 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:08:34.988 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:08:34.993 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:08:34.994 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:08:35.095 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:08:35.120 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:08:35.122 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:08:35.123 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:08:35.223 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:08:35.260 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:08:35.261 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:08:35.262 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:08:35.363 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:08:35.398 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:08:35.399 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:08:35.399 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:08:35.500 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:08:35.539 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:08:35.542 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:08:35.542 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:08:35.643 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:08:35.674 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:08:35.678 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:08:35.679 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:08:35.780 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:08:35.815 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:08:35.818 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:08:35.818 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:08:35.919 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:08:35.927 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:08:35 smithi089 bash[78452]: cluster 1685822914.6069775 mgr.y (mgr.34104) 973 : cluster 0 pgmap v922: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 4.5 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 651 op/s; 12003/222663 objects degraded (5.391%); 3562/222663 objects misplaced (1.600%); 1.1 MiB/s, 18 objects/s recovering 2023-06-03T20:08:35.927 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:08:35 smithi089 bash[75068]: cluster 1685822914.6069775 mgr.y (mgr.34104) 973 : cluster 0 pgmap v922: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 4.5 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 651 op/s; 12003/222663 objects degraded (5.391%); 3562/222663 objects misplaced (1.600%); 1.1 MiB/s, 18 objects/s recovering 2023-06-03T20:08:35.957 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:08:35.961 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:08:35.961 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:08:36.062 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:08:36.071 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:08:35 smithi164 bash[59462]: cluster 1685822914.6069775 mgr.y (mgr.34104) 973 : cluster 0 pgmap v922: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 4.5 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 651 op/s; 12003/222663 objects degraded (5.391%); 3562/222663 objects misplaced (1.600%); 1.1 MiB/s, 18 objects/s recovering 2023-06-03T20:08:36.104 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:08:36.113 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:08:36.114 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:08:36.215 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:08:36.254 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:08:36.258 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:08:36.259 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:08:36.359 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:08:36.388 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:08:36.392 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:08:36.393 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:08:36.494 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:08:36.528 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 20:08:36 smithi089 bash[71163]: ::ffff:172.21.15.164 - - [03/Jun/2023:20:08:36] "GET /metrics HTTP/1.1" 200 228730 "" "Prometheus/2.33.4" 2023-06-03T20:08:36.538 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:08:36.545 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:08:36.546 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:08:36.648 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:08:36.675 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:08:36.678 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:08:36.679 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:08:36.780 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:08:36.832 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:08:36.834 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:08:36.835 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:08:36.936 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:08:36.965 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:08:36.967 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:08:36.967 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:08:37.068 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:08:37.102 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:08:37.106 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:08:37.106 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:08:37.207 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:08:37.241 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:08:37.246 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:08:37.246 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:08:37.347 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:08:37.377 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:08:37.386 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:08:37.387 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:08:37.488 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:08:37.519 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:08:37.521 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:08:37.522 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:08:37.623 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:08:37.652 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:08:37.654 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:08:37.655 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:08:37.756 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:08:37.765 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:08:37 smithi164 bash[59462]: cluster 1685822916.6074765 mgr.y (mgr.34104) 974 : cluster 0 pgmap v923: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 4.5 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 615 op/s; 12003/220197 objects degraded (5.451%); 3562/220197 objects misplaced (1.618%); 977 KiB/s, 15 objects/s recovering 2023-06-03T20:08:37.766 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:08:37 smithi164 bash[59462]: audit 1685822917.1163144 mon.a (mon.0) 891 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:08:37.766 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:08:37 smithi164 bash[59462]: audit 1685822917.1206217 mon.c (mon.2) 462 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T20:08:37.766 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:08:37 smithi164 bash[59462]: cluster 1685822917.6660511 mon.a (mon.0) 892 : cluster 3 Health check update: Degraded data redundancy: 12003/220197 objects degraded (5.451%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-06-03T20:08:37.811 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:08:37.817 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:08:37.818 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:08:37.905 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:08:37 smithi089 bash[75068]: cluster 1685822916.6074765 mgr.y (mgr.34104) 974 : cluster 0 pgmap v923: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 4.5 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 615 op/s; 12003/220197 objects degraded (5.451%); 3562/220197 objects misplaced (1.618%); 977 KiB/s, 15 objects/s recovering 2023-06-03T20:08:37.905 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:08:37 smithi089 bash[75068]: audit 1685822917.1163144 mon.a (mon.0) 891 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:08:37.905 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:08:37 smithi089 bash[75068]: audit 1685822917.1206217 mon.c (mon.2) 462 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T20:08:37.905 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:08:37 smithi089 bash[75068]: cluster 1685822917.6660511 mon.a (mon.0) 892 : cluster 3 Health check update: Degraded data redundancy: 12003/220197 objects degraded (5.451%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-06-03T20:08:37.906 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:08:37 smithi089 bash[78452]: cluster 1685822916.6074765 mgr.y (mgr.34104) 974 : cluster 0 pgmap v923: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 4.5 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 615 op/s; 12003/220197 objects degraded (5.451%); 3562/220197 objects misplaced (1.618%); 977 KiB/s, 15 objects/s recovering 2023-06-03T20:08:37.906 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:08:37 smithi089 bash[78452]: audit 1685822917.1163144 mon.a (mon.0) 891 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:08:37.906 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:08:37 smithi089 bash[78452]: audit 1685822917.1206217 mon.c (mon.2) 462 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T20:08:37.906 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:08:37 smithi089 bash[78452]: cluster 1685822917.6660511 mon.a (mon.0) 892 : cluster 3 Health check update: Degraded data redundancy: 12003/220197 objects degraded (5.451%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-06-03T20:08:37.918 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:08:37.966 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:08:37.969 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:08:37.970 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:08:38.071 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:08:38.103 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:08:38.111 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:08:38.113 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:08:38.214 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:08:38.250 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:08:38.253 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:08:38.254 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:08:38.355 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:08:38.404 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:08:38.410 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:08:38.410 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:08:38.511 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:08:38.550 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:08:38.551 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:08:38.552 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:08:38.653 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:08:38.684 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:08:38.690 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:08:38.692 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:08:38.793 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:08:38.833 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:08:38.840 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:08:38.841 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:08:38.942 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:08:38.980 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:08:38.986 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:08:38.987 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:08:39.087 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:08:39.131 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:08:39.137 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:08:39.137 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:08:39.238 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:08:39.269 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:08:39.274 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:08:39.275 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:08:39.376 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:08:39.423 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:08:39.425 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:08:39.426 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:08:39.527 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:08:39.550 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:08:39.553 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:08:39.554 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:08:39.655 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:08:39.692 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:08:39.694 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:08:39.695 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:08:39.727 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:08:39 smithi164 bash[59462]: cluster 1685822918.608129 mgr.y (mgr.34104) 975 : cluster 0 pgmap v924: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 4.3 GiB data, 14 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 737 op/s; 12086/213324 objects degraded (5.666%); 3385/213324 objects misplaced (1.587%); 1.4 MiB/s, 22 objects/s recovering 2023-06-03T20:08:39.796 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:08:39.826 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:08:39.830 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:08:39.830 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:08:39.931 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:08:39.932 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:08:39 smithi089 bash[75068]: cluster 1685822918.608129 mgr.y (mgr.34104) 975 : cluster 0 pgmap v924: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 4.3 GiB data, 14 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 737 op/s; 12086/213324 objects degraded (5.666%); 3385/213324 objects misplaced (1.587%); 1.4 MiB/s, 22 objects/s recovering 2023-06-03T20:08:39.933 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:08:39 smithi089 bash[78452]: cluster 1685822918.608129 mgr.y (mgr.34104) 975 : cluster 0 pgmap v924: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 4.3 GiB data, 14 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 737 op/s; 12086/213324 objects degraded (5.666%); 3385/213324 objects misplaced (1.587%); 1.4 MiB/s, 22 objects/s recovering 2023-06-03T20:08:39.968 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:08:39.972 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:08:39.973 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:08:40.074 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:08:40.106 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:08:40.112 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:08:40.113 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:08:40.214 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:08:40.247 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:08:40.252 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:08:40.253 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:08:40.354 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:08:40.388 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:08:40.393 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:08:40.394 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:08:40.495 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:08:40.523 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:08:40.525 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:08:40.526 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:08:40.627 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:08:40.664 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:08:40.666 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:08:40.667 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:08:40.767 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:08:40.805 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:08:40.809 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:08:40.810 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:08:40.911 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:08:40.948 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:08:40.950 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:08:40.951 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:08:41.052 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:08:41.084 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:08:41.086 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:08:41.087 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:08:41.188 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:08:41.232 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:08:41.235 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:08:41.236 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:08:41.337 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:08:41.390 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:08:41.393 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:08:41.394 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:08:41.495 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:08:41.525 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:08:41.527 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:08:41.527 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:08:41.629 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:08:41.676 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:08:41.682 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:08:41.683 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:08:41.783 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:08:41.793 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:08:41 smithi164 bash[59462]: cluster 1685822920.608816 mgr.y (mgr.34104) 976 : cluster 0 pgmap v925: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 4.3 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 595 op/s; 12086/210975 objects degraded (5.729%); 3385/210975 objects misplaced (1.604%); 986 KiB/s, 15 objects/s recovering 2023-06-03T20:08:41.824 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:08:41.829 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:08:41.830 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:08:41.931 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:08:41.966 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:08:41.973 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:08:41.973 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:08:42.028 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:08:41 smithi089 bash[75068]: cluster 1685822920.608816 mgr.y (mgr.34104) 976 : cluster 0 pgmap v925: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 4.3 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 595 op/s; 12086/210975 objects degraded (5.729%); 3385/210975 objects misplaced (1.604%); 986 KiB/s, 15 objects/s recovering 2023-06-03T20:08:42.029 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:08:41 smithi089 bash[78452]: cluster 1685822920.608816 mgr.y (mgr.34104) 976 : cluster 0 pgmap v925: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 4.3 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 595 op/s; 12086/210975 objects degraded (5.729%); 3385/210975 objects misplaced (1.604%); 986 KiB/s, 15 objects/s recovering 2023-06-03T20:08:42.074 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:08:42.128 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:08:42.133 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:08:42.134 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:08:42.235 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:08:42.269 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:08:42.274 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:08:42.275 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:08:42.376 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:08:42.427 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:08:42.428 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:08:42.428 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:08:42.529 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:08:42.555 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:08:42.558 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:08:42.559 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:08:42.659 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:08:42.703 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:08:42.711 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:08:42.712 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:08:42.738 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:08:42 smithi164 bash[59462]: cluster 1685822922.6670368 mon.a (mon.0) 893 : cluster 3 Health check update: Degraded data redundancy: 12086/210975 objects degraded (5.729%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-06-03T20:08:42.813 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:08:42.822 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:08:42 smithi089 bash[75068]: cluster 1685822922.6670368 mon.a (mon.0) 893 : cluster 3 Health check update: Degraded data redundancy: 12086/210975 objects degraded (5.729%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-06-03T20:08:42.822 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:08:42 smithi089 bash[78452]: cluster 1685822922.6670368 mon.a (mon.0) 893 : cluster 3 Health check update: Degraded data redundancy: 12086/210975 objects degraded (5.729%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-06-03T20:08:42.844 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:08:42.847 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:08:42.848 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:08:42.949 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:08:42.985 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:08:42.990 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:08:42.991 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:08:43.091 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:08:43.116 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:08:43.118 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:08:43.119 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:08:43.219 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:08:43.259 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:08:43.264 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:08:43.265 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:08:43.366 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:08:43.397 INFO:teuthology.orchestra.run.smithi089.stdout:ERROR: (22) Invalid argument 2023-06-03T20:08:43.397 INFO:teuthology.orchestra.run.smithi089.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-06-03T20:08:43.423 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:08:43.429 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:08:43.430 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:08:43.531 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:08:43.567 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:08:43.573 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:08:43.573 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:08:43.674 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:08:43.716 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:08:43.718 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:08:43.719 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:08:43.820 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:08:43.851 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:08:43.853 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:08:43.854 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:08:43.955 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:08:43.964 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:08:43 smithi089 bash[75068]: cluster 1685822922.6098244 mgr.y (mgr.34104) 977 : cluster 0 pgmap v926: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 4.3 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 575 op/s; 12086/209166 objects degraded (5.778%); 3385/209166 objects misplaced (1.618%); 986 KiB/s, 15 objects/s recovering 2023-06-03T20:08:43.964 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:08:43 smithi089 bash[78452]: cluster 1685822922.6098244 mgr.y (mgr.34104) 977 : cluster 0 pgmap v926: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 4.3 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 575 op/s; 12086/209166 objects degraded (5.778%); 3385/209166 objects misplaced (1.618%); 986 KiB/s, 15 objects/s recovering 2023-06-03T20:08:43.987 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:08:43.989 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:08:43.989 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:08:44.083 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:08:43 smithi164 bash[59462]: cluster 1685822922.6098244 mgr.y (mgr.34104) 977 : cluster 0 pgmap v926: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 4.3 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 575 op/s; 12086/209166 objects degraded (5.778%); 3385/209166 objects misplaced (1.618%); 986 KiB/s, 15 objects/s recovering 2023-06-03T20:08:44.090 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:08:44.127 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:08:44.129 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:08:44.129 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:08:44.230 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:08:44.272 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:08:44.278 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:08:44.279 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:08:44.380 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:08:44.416 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:08:44.421 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:08:44.421 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:08:44.522 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:08:44.554 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:08:44.557 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:08:44.557 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:08:44.658 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:08:44.698 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:08:44.701 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:08:44.702 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:08:44.803 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:08:44.851 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:08:44.854 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:08:44.854 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:08:44.955 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:08:44.989 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:08:44.992 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:08:44.993 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:08:45.094 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:08:45.130 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:08:45.138 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:08:45.139 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:08:45.240 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:08:45.275 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:08:45.278 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:08:45.278 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:08:45.379 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:08:45.414 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:08:45.419 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:08:45.419 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:08:45.520 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:08:45.559 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:08:45.568 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:08:45.569 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:08:45.670 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:08:45.704 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:08:45.706 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:08:45.707 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:08:45.808 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:08:45.818 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:08:45 smithi164 bash[59462]: cluster 1685822924.6108317 mgr.y (mgr.34104) 978 : cluster 0 pgmap v927: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 4.1 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 680 op/s; 12192/203652 objects degraded (5.987%); 3188/203652 objects misplaced (1.565%); 1.4 MiB/s, 22 objects/s recovering 2023-06-03T20:08:45.846 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:08:45.853 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:08:45.854 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:08:45.955 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:08:45.964 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:08:45 smithi089 bash[78452]: cluster 1685822924.6108317 mgr.y (mgr.34104) 978 : cluster 0 pgmap v927: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 4.1 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 680 op/s; 12192/203652 objects degraded (5.987%); 3188/203652 objects misplaced (1.565%); 1.4 MiB/s, 22 objects/s recovering 2023-06-03T20:08:45.965 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:08:45 smithi089 bash[75068]: cluster 1685822924.6108317 mgr.y (mgr.34104) 978 : cluster 0 pgmap v927: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 4.1 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 680 op/s; 12192/203652 objects degraded (5.987%); 3188/203652 objects misplaced (1.565%); 1.4 MiB/s, 22 objects/s recovering 2023-06-03T20:08:45.981 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:08:45.983 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:08:45.983 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:08:46.084 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:08:46.122 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:08:46.124 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:08:46.125 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:08:46.226 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:08:46.263 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:08:46.265 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:08:46.266 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:08:46.367 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:08:46.416 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:08:46.421 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:08:46.422 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:08:46.523 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:08:46.528 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 20:08:46 smithi089 bash[71163]: ::ffff:172.21.15.164 - - [03/Jun/2023:20:08:46] "GET /metrics HTTP/1.1" 200 228760 "" "Prometheus/2.33.4" 2023-06-03T20:08:46.565 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:08:46.574 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:08:46.575 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:08:46.676 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:08:46.727 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:08:46.734 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:08:46.734 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:08:46.835 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:08:46.871 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:08:46.876 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:08:46.877 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:08:46.978 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:08:47.008 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:08:47.012 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:08:47.013 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:08:47.114 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:08:47.149 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:08:47.157 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:08:47.158 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:08:47.259 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:08:47.307 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:08:47.309 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:08:47.310 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:08:47.411 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:08:47.441 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:08:47.445 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:08:47.446 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:08:47.508 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph daemon osd.2 dump_ops_in_flight 2023-06-03T20:08:47.547 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:08:47.590 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:08:47.592 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:08:47.593 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:08:47.694 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:08:47.743 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:08:47.751 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:08:47.752 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:08:47.779 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:08:47 smithi089 bash[75068]: cluster 1685822926.6114416 mgr.y (mgr.34104) 979 : cluster 0 pgmap v928: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 4.1 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 590 op/s; 12192/201411 objects degraded (6.053%); 3188/201411 objects misplaced (1.583%); 986 KiB/s, 15 objects/s recovering 2023-06-03T20:08:47.780 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:08:47 smithi089 bash[75068]: cluster 1685822927.6682892 mon.a (mon.0) 894 : cluster 3 Health check update: Degraded data redundancy: 12192/201411 objects degraded (6.053%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-06-03T20:08:47.780 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:08:47 smithi089 bash[78452]: cluster 1685822926.6114416 mgr.y (mgr.34104) 979 : cluster 0 pgmap v928: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 4.1 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 590 op/s; 12192/201411 objects degraded (6.053%); 3188/201411 objects misplaced (1.583%); 986 KiB/s, 15 objects/s recovering 2023-06-03T20:08:47.780 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:08:47 smithi089 bash[78452]: cluster 1685822927.6682892 mon.a (mon.0) 894 : cluster 3 Health check update: Degraded data redundancy: 12192/201411 objects degraded (6.053%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-06-03T20:08:47.853 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:08:47.886 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:08:47.889 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:08:47.889 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:08:47.990 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:08:48.018 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:08:48.019 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:08:48.020 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:08:48.083 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:08:47 smithi164 bash[59462]: cluster 1685822926.6114416 mgr.y (mgr.34104) 979 : cluster 0 pgmap v928: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 4.1 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 590 op/s; 12192/201411 objects degraded (6.053%); 3188/201411 objects misplaced (1.583%); 986 KiB/s, 15 objects/s recovering 2023-06-03T20:08:48.084 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:08:47 smithi164 bash[59462]: cluster 1685822927.6682892 mon.a (mon.0) 894 : cluster 3 Health check update: Degraded data redundancy: 12192/201411 objects degraded (6.053%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-06-03T20:08:48.121 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:08:48.170 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:08:48.176 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:08:48.177 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:08:48.278 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:08:48.312 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:08:48.322 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:08:48.322 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:08:48.423 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:08:48.456 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:08:48.457 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:08:48.458 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:08:48.559 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:08:48.603 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:08:48.606 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:08:48.606 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:08:48.707 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:08:48.749 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:08:48.751 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:08:48.751 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:08:48.852 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:08:48.885 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:08:48.891 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:08:48.892 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:08:48.896 INFO:teuthology.orchestra.run.smithi089.stdout:osd.0: {} 2023-06-03T20:08:48.929 INFO:teuthology.orchestra.run.smithi089.stdout:osd.1: {} 2023-06-03T20:08:48.981 INFO:teuthology.orchestra.run.smithi089.stdout:osd.2: {} 2023-06-03T20:08:48.993 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:08:49.022 INFO:teuthology.orchestra.run.smithi089.stdout:osd.3: {} 2023-06-03T20:08:49.034 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:08:49.036 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:08:49.037 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:08:49.057 INFO:teuthology.orchestra.run.smithi089.stdout:osd.4: {} 2023-06-03T20:08:49.091 INFO:teuthology.orchestra.run.smithi089.stdout:osd.5: {} 2023-06-03T20:08:49.126 INFO:teuthology.orchestra.run.smithi089.stdout:osd.6: {} 2023-06-03T20:08:49.138 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:08:49.164 INFO:teuthology.orchestra.run.smithi089.stdout:osd.7: {} 2023-06-03T20:08:49.168 INFO:teuthology.orchestra.run.smithi089.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-06-03T20:08:49.168 INFO:teuthology.orchestra.run.smithi089.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-06-03T20:08:49.169 INFO:teuthology.orchestra.run.smithi089.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-06-03T20:08:49.169 INFO:teuthology.orchestra.run.smithi089.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-06-03T20:08:49.169 INFO:teuthology.orchestra.run.smithi089.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-06-03T20:08:49.169 INFO:teuthology.orchestra.run.smithi089.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-06-03T20:08:49.169 INFO:teuthology.orchestra.run.smithi089.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-06-03T20:08:49.169 INFO:teuthology.orchestra.run.smithi089.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-06-03T20:08:49.176 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:08:49.185 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:08:49.186 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:08:49.287 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:08:49.330 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:08:49.336 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:08:49.337 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:08:49.438 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:08:49.468 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:08:49.473 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:08:49.474 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:08:49.574 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:08:49.616 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:08:49.621 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:08:49.622 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:08:49.723 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:08:49.754 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:08:49.758 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:08:49 smithi164 bash[59462]: cluster 1685822928.6121395 mgr.y (mgr.34104) 980 : cluster 0 pgmap v929: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 4.0 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 617 op/s; 12192/197943 objects degraded (6.159%); 3188/197943 objects misplaced (1.611%); 986 KiB/s, 15 objects/s recovering 2023-06-03T20:08:49.758 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:08:49 smithi164 bash[59462]: audit 1685822928.8539581 mon.c (mon.2) 463 : audit 0 from='client.? 172.21.15.89:0/83339456' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-03T20:08:49.758 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:08:49 smithi164 bash[59462]: audit 1685822928.8829665 mon.c (mon.2) 464 : audit 1 from='client.? 172.21.15.89:0/4220191753' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-06-03T20:08:49.759 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:08:49 smithi164 bash[59462]: audit 1685822928.8839917 mon.a (mon.0) 895 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-06-03T20:08:49.759 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:08:49 smithi164 bash[59462]: audit 1685822929.4866216 mon.c (mon.2) 465 : audit 1 from='client.? 172.21.15.89:0/2190729926' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-06-03T20:08:49.759 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:08:49 smithi164 bash[59462]: audit 1685822929.4872205 mon.a (mon.0) 896 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-06-03T20:08:49.759 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:08:49.760 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:08:49.771 INFO:teuthology.orchestra.run.smithi089.stderr:reweighted osd.0 to 1 (10000) 2023-06-03T20:08:49.773 INFO:teuthology.orchestra.run.smithi089.stderr:nodeep-scrub is set 2023-06-03T20:08:49.806 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:08:49 smithi089 bash[78452]: cluster 1685822928.6121395 mgr.y (mgr.34104) 980 : cluster 0 pgmap v929: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 4.0 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 617 op/s; 12192/197943 objects degraded (6.159%); 3188/197943 objects misplaced (1.611%); 986 KiB/s, 15 objects/s recovering 2023-06-03T20:08:49.807 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:08:49 smithi089 bash[78452]: audit 1685822928.8539581 mon.c (mon.2) 463 : audit 0 from='client.? 172.21.15.89:0/83339456' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-03T20:08:49.807 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:08:49 smithi089 bash[78452]: audit 1685822928.8829665 mon.c (mon.2) 464 : audit 1 from='client.? 172.21.15.89:0/4220191753' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-06-03T20:08:49.807 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:08:49 smithi089 bash[78452]: audit 1685822928.8839917 mon.a (mon.0) 895 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-06-03T20:08:49.807 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:08:49 smithi089 bash[78452]: audit 1685822929.4866216 mon.c (mon.2) 465 : audit 1 from='client.? 172.21.15.89:0/2190729926' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-06-03T20:08:49.807 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:08:49 smithi089 bash[78452]: audit 1685822929.4872205 mon.a (mon.0) 896 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-06-03T20:08:49.808 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:08:49 smithi089 bash[75068]: cluster 1685822928.6121395 mgr.y (mgr.34104) 980 : cluster 0 pgmap v929: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 4.0 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 617 op/s; 12192/197943 objects degraded (6.159%); 3188/197943 objects misplaced (1.611%); 986 KiB/s, 15 objects/s recovering 2023-06-03T20:08:49.808 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:08:49 smithi089 bash[75068]: audit 1685822928.8539581 mon.c (mon.2) 463 : audit 0 from='client.? 172.21.15.89:0/83339456' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-03T20:08:49.808 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:08:49 smithi089 bash[75068]: audit 1685822928.8829665 mon.c (mon.2) 464 : audit 1 from='client.? 172.21.15.89:0/4220191753' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-06-03T20:08:49.808 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:08:49 smithi089 bash[75068]: audit 1685822928.8839917 mon.a (mon.0) 895 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-06-03T20:08:49.808 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:08:49 smithi089 bash[75068]: audit 1685822929.4866216 mon.c (mon.2) 465 : audit 1 from='client.? 172.21.15.89:0/2190729926' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-06-03T20:08:49.808 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:08:49 smithi089 bash[75068]: audit 1685822929.4872205 mon.a (mon.0) 896 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-06-03T20:08:49.861 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:08:49.903 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:08:49.905 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:08:49.906 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:08:50.007 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:08:50.032 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:08:50.038 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:08:50.039 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:08:50.140 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:08:50.180 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:08:50.182 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:08:50.183 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:08:50.283 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:08:50.342 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:08:50.344 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:08:50.344 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:08:50.445 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:08:50.511 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:08:50.516 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:08:50.517 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:08:50.618 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:08:50.662 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:08:50.669 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:08:50.670 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:08:50.764 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:08:50 smithi164 bash[59462]: cluster 1685822929.7458014 mon.a (mon.0) 897 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-06-03T20:08:50.764 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:08:50 smithi164 bash[59462]: cluster 1685822929.7569845 osd.6 (osd.6) 113 : cluster 3 failed to encode map e176 with expected crc 2023-06-03T20:08:50.764 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:08:50 smithi164 bash[59462]: audit 1685822929.7620904 mon.a (mon.0) 898 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-06-03T20:08:50.764 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:08:50 smithi164 bash[59462]: audit 1685822929.7621763 mon.a (mon.0) 899 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2023-06-03T20:08:50.764 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:08:50 smithi164 bash[59462]: cluster 1685822929.7622344 mon.a (mon.0) 900 : cluster 0 osdmap e176: 8 total, 8 up, 7 in 2023-06-03T20:08:50.764 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:08:50 smithi164 bash[59462]: cluster 1685822929.7686331 osd.7 (osd.7) 112 : cluster 3 failed to encode map e176 with expected crc 2023-06-03T20:08:50.771 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:08:50.824 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:08:50.827 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:08:50.828 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:08:50.929 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:08:50.979 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:08:50.984 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:08:50.985 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:08:51.086 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:08:51.087 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:08:50 smithi089 bash[78452]: cluster 1685822929.7458014 mon.a (mon.0) 897 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-06-03T20:08:51.088 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:08:50 smithi089 bash[78452]: cluster 1685822929.7569845 osd.6 (osd.6) 113 : cluster 3 failed to encode map e176 with expected crc 2023-06-03T20:08:51.088 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:08:50 smithi089 bash[78452]: audit 1685822929.7620904 mon.a (mon.0) 898 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-06-03T20:08:51.088 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:08:50 smithi089 bash[78452]: audit 1685822929.7621763 mon.a (mon.0) 899 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2023-06-03T20:08:51.088 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:08:50 smithi089 bash[78452]: cluster 1685822929.7622344 mon.a (mon.0) 900 : cluster 0 osdmap e176: 8 total, 8 up, 7 in 2023-06-03T20:08:51.088 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:08:50 smithi089 bash[78452]: cluster 1685822929.7686331 osd.7 (osd.7) 112 : cluster 3 failed to encode map e176 with expected crc 2023-06-03T20:08:51.089 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:08:50 smithi089 bash[75068]: cluster 1685822929.7458014 mon.a (mon.0) 897 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-06-03T20:08:51.089 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:08:50 smithi089 bash[75068]: cluster 1685822929.7569845 osd.6 (osd.6) 113 : cluster 3 failed to encode map e176 with expected crc 2023-06-03T20:08:51.089 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:08:50 smithi089 bash[75068]: audit 1685822929.7620904 mon.a (mon.0) 898 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-06-03T20:08:51.089 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:08:50 smithi089 bash[75068]: audit 1685822929.7621763 mon.a (mon.0) 899 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2023-06-03T20:08:51.089 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:08:50 smithi089 bash[75068]: cluster 1685822929.7622344 mon.a (mon.0) 900 : cluster 0 osdmap e176: 8 total, 8 up, 7 in 2023-06-03T20:08:51.089 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:08:50 smithi089 bash[75068]: cluster 1685822929.7686331 osd.7 (osd.7) 112 : cluster 3 failed to encode map e176 with expected crc 2023-06-03T20:08:51.135 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:08:51.139 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:08:51.140 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:08:51.240 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:08:51.280 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:08:51.284 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:08:51.285 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:08:51.386 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:08:51.419 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:08:51.427 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:08:51.428 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:08:51.529 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:08:51.565 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:08:51.568 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:08:51.569 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:08:51.670 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:08:51.710 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:08:51.712 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:08:51.713 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:08:51.802 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph osd reweight 2 1 2023-06-03T20:08:51.803 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:08:51 smithi089 bash[78452]: cluster 1685822930.6127934 mgr.y (mgr.34104) 981 : cluster 0 pgmap v931: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 3.9 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 636 op/s; 12302/191865 objects degraded (6.412%); 2986/191865 objects misplaced (1.556%); 1.1 MiB/s, 18 objects/s recovering 2023-06-03T20:08:51.803 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:08:51 smithi089 bash[75068]: cluster 1685822930.6127934 mgr.y (mgr.34104) 981 : cluster 0 pgmap v931: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 3.9 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 636 op/s; 12302/191865 objects degraded (6.412%); 2986/191865 objects misplaced (1.556%); 1.1 MiB/s, 18 objects/s recovering 2023-06-03T20:08:51.814 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:08:51.871 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:08:51.876 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:08:51.876 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:08:51.977 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:08:51.986 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:08:51 smithi164 bash[59462]: cluster 1685822930.6127934 mgr.y (mgr.34104) 981 : cluster 0 pgmap v931: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 3.9 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 636 op/s; 12302/191865 objects degraded (6.412%); 2986/191865 objects misplaced (1.556%); 1.1 MiB/s, 18 objects/s recovering 2023-06-03T20:08:52.016 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:08:52.024 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:08:52.024 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:08:52.125 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:08:52.167 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:08:52.170 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:08:52.170 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:08:52.271 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:08:52.307 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:08:52.311 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:08:52.312 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:08:52.413 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:08:52.447 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-06-03T20:08:52.448 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:08:52.457 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:08:52.457 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:08:52.559 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:08:52.592 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:08:52.599 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:08:52.599 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:08:52.700 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:08:52.732 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:08:52.737 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:08:52.738 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:08:52.838 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:08:52.878 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:08:52.882 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:08:52.883 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:08:52.984 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:08:53.019 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:08:53.021 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:08:53.022 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:08:53.028 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:08:52 smithi089 bash[78452]: audit 1685822932.1117373 mon.c (mon.2) 466 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T20:08:53.028 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:08:52 smithi089 bash[78452]: cluster 1685822932.6695929 mon.a (mon.0) 901 : cluster 3 Health check update: Degraded data redundancy: 12302/191865 objects degraded (6.412%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-06-03T20:08:53.028 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:08:52 smithi089 bash[75068]: audit 1685822932.1117373 mon.c (mon.2) 466 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T20:08:53.029 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:08:52 smithi089 bash[75068]: cluster 1685822932.6695929 mon.a (mon.0) 901 : cluster 3 Health check update: Degraded data redundancy: 12302/191865 objects degraded (6.412%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-06-03T20:08:53.083 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:08:52 smithi164 bash[59462]: audit 1685822932.1117373 mon.c (mon.2) 466 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T20:08:53.084 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:08:52 smithi164 bash[59462]: cluster 1685822932.6695929 mon.a (mon.0) 901 : cluster 3 Health check update: Degraded data redundancy: 12302/191865 objects degraded (6.412%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-06-03T20:08:53.123 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:08:53.154 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:08:53.155 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:08:53.156 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:08:53.257 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:08:53.294 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:08:53.296 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:08:53.296 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:08:53.397 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:08:53.444 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:08:53.449 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:08:53.450 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:08:53.551 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:08:53.592 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:08:53.600 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:08:53.601 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:08:53.702 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:08:53.731 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:08:53.734 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:08:53.735 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:08:53.762 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph osd unset noscrub 2023-06-03T20:08:53.833 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:08:53 smithi164 bash[59462]: cluster 1685822932.6134012 mgr.y (mgr.34104) 982 : cluster 0 pgmap v932: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 3.9 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 635 op/s; 12302/190101 objects degraded (6.471%); 2986/190101 objects misplaced (1.571%); 1.1 MiB/s, 18 objects/s recovering 2023-06-03T20:08:53.836 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:08:53.891 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:08:53.897 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:08:53.898 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:08:53.999 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:08:54.015 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:08:53 smithi089 bash[78452]: cluster 1685822932.6134012 mgr.y (mgr.34104) 982 : cluster 0 pgmap v932: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 3.9 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 635 op/s; 12302/190101 objects degraded (6.471%); 2986/190101 objects misplaced (1.571%); 1.1 MiB/s, 18 objects/s recovering 2023-06-03T20:08:54.015 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:08:53 smithi089 bash[75068]: cluster 1685822932.6134012 mgr.y (mgr.34104) 982 : cluster 0 pgmap v932: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 3.9 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 635 op/s; 12302/190101 objects degraded (6.471%); 2986/190101 objects misplaced (1.571%); 1.1 MiB/s, 18 objects/s recovering 2023-06-03T20:08:54.037 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:08:54.041 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:08:54.041 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:08:54.142 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:08:54.174 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:08:54.177 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:08:54.177 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:08:54.278 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:08:54.308 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:08:54.311 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:08:54.312 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:08:54.340 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/mon.a/config 2023-06-03T20:08:54.414 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:08:54.450 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:08:54.457 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:08:54.459 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:08:54.560 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:08:54.595 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:08:54.600 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:08:54.601 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:08:54.702 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:08:54.731 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:08:54.734 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:08:54.735 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:08:54.836 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:08:54.868 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:08:54.872 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:08:54.873 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:08:54.974 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:08:55.008 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:08:55.012 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:08:55.012 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:08:55.113 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:08:55.151 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:08:55.156 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:08:55.156 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:08:55.257 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:08:55.293 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:08:55.301 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:08:55.302 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:08:55.403 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:08:55.451 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:08:55.457 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:08:55.457 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:08:55.558 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:08:55.590 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:08:55.592 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:08:55.593 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:08:55.694 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:08:55.729 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:08:55.731 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:08:55.732 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:08:55.832 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:08:55.834 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:08:55 smithi089 bash[75068]: cluster 1685822934.6149063 mgr.y (mgr.34104) 983 : cluster 0 pgmap v933: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 3.8 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 635 op/s; 12387/184587 objects degraded (6.711%); 2800/184587 objects misplaced (1.517%); 1.2 MiB/s, 19 objects/s recovering 2023-06-03T20:08:55.864 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:08:55.872 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:08:55.872 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:08:55.973 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:08:56.009 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:08:56.013 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:08:56.013 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:08:56.083 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:08:55 smithi164 bash[59462]: cluster 1685822934.6149063 mgr.y (mgr.34104) 983 : cluster 0 pgmap v933: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 3.8 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 635 op/s; 12387/184587 objects degraded (6.711%); 2800/184587 objects misplaced (1.517%); 1.2 MiB/s, 19 objects/s recovering 2023-06-03T20:08:56.115 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:08:56.168 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:08:56.174 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:08:56.175 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:08:56.230 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:08:55 smithi089 bash[78452]: cluster 1685822934.6149063 mgr.y (mgr.34104) 983 : cluster 0 pgmap v933: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 3.8 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 635 op/s; 12387/184587 objects degraded (6.711%); 2800/184587 objects misplaced (1.517%); 1.2 MiB/s, 19 objects/s recovering 2023-06-03T20:08:56.276 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:08:56.309 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:08:56.312 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:08:56.313 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:08:56.397 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/mon.a/config 2023-06-03T20:08:56.402 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/mon.a/config 2023-06-03T20:08:56.414 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:08:56.464 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:08:56.467 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:08:56.468 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:08:56.528 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 20:08:56 smithi089 bash[71163]: ::ffff:172.21.15.164 - - [03/Jun/2023:20:08:56] "GET /metrics HTTP/1.1" 200 228756 "" "Prometheus/2.33.4" 2023-06-03T20:08:56.570 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:08:56.601 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:08:56.606 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:08:56.607 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:08:56.708 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:08:56.735 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:08:56.741 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:08:56.742 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:08:56.842 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:08:56.880 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:08:56.886 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:08:56.887 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:08:56.988 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:08:57.019 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:08:57.022 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:08:57.022 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:08:57.123 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:08:57.157 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:08:57.161 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:08:57.162 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:08:57.263 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:08:57.293 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:08:57.298 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:08:57.299 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:08:57.400 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:08:57.429 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:08:57.434 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:08:57.435 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:08:57.536 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:08:57.568 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:08:57.570 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:08:57.570 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:08:57.671 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:08:57.729 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:08:57.731 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:08:57.732 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:08:57.833 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:08:57.880 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:08:57.885 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:08:57.886 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:08:57.987 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:08:57.997 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:08:57 smithi164 bash[59462]: cluster 1685822936.6155233 mgr.y (mgr.34104) 984 : cluster 0 pgmap v934: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 3.7 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 639 op/s; 12387/182205 objects degraded (6.798%); 2800/182205 objects misplaced (1.537%); 1.2 MiB/s, 19 objects/s recovering 2023-06-03T20:08:57.998 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:08:57 smithi164 bash[59462]: cluster 1685822937.6709223 mon.a (mon.0) 902 : cluster 3 Health check update: Degraded data redundancy: 12387/182205 objects degraded (6.798%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-06-03T20:08:58.023 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:08:58.025 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:08:58.025 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:08:58.126 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:08:58.155 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:08:58.161 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:08:58.162 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:08:58.175 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:08:58 smithi089 bash[78452]: cluster 1685822936.6155233 mgr.y (mgr.34104) 984 : cluster 0 pgmap v934: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 3.7 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 639 op/s; 12387/182205 objects degraded (6.798%); 2800/182205 objects misplaced (1.537%); 1.2 MiB/s, 19 objects/s recovering 2023-06-03T20:08:58.175 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:08:58 smithi089 bash[78452]: cluster 1685822937.6709223 mon.a (mon.0) 902 : cluster 3 Health check update: Degraded data redundancy: 12387/182205 objects degraded (6.798%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-06-03T20:08:58.176 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:08:58 smithi089 bash[75068]: cluster 1685822936.6155233 mgr.y (mgr.34104) 984 : cluster 0 pgmap v934: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 3.7 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 639 op/s; 12387/182205 objects degraded (6.798%); 2800/182205 objects misplaced (1.537%); 1.2 MiB/s, 19 objects/s recovering 2023-06-03T20:08:58.176 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:08:58 smithi089 bash[75068]: cluster 1685822937.6709223 mon.a (mon.0) 902 : cluster 3 Health check update: Degraded data redundancy: 12387/182205 objects degraded (6.798%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-06-03T20:08:58.264 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:08:58.306 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:08:58.309 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:08:58.309 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:08:58.410 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:08:58.448 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:08:58.454 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:08:58.454 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:08:58.555 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:08:58.594 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:08:58.598 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:08:58.599 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:08:58.700 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:08:58.732 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:08:58.740 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:08:58.741 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:08:58.842 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:08:58.889 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:08:58.891 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:08:58.892 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:08:58.993 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:08:59.019 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:08:59.029 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:08:59.030 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:08:59.131 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:08:59.159 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:08:59.162 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:08:59.162 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:08:59.263 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:08:59.288 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:08:59.290 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:08:59.291 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:08:59.391 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:08:59.420 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:08:59.425 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:08:59.425 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:08:59.526 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:08:59.585 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:08:59.590 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:08:59.591 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:08:59.692 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:08:59.725 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:08:59.729 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:08:59.730 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:08:59.831 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:08:59.873 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:08:59.876 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:08:59.877 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:08:59.977 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:08:59.988 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:08:59 smithi164 bash[59462]: cluster 1685822938.616199 mgr.y (mgr.34104) 985 : cluster 0 pgmap v935: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 3.6 GiB data, 12 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 648 op/s; 12387/178485 objects degraded (6.940%); 2800/178485 objects misplaced (1.569%); 1.2 MiB/s, 19 objects/s recovering 2023-06-03T20:09:00.028 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:08:59 smithi089 bash[75068]: cluster 1685822938.616199 mgr.y (mgr.34104) 985 : cluster 0 pgmap v935: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 3.6 GiB data, 12 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 648 op/s; 12387/178485 objects degraded (6.940%); 2800/178485 objects misplaced (1.569%); 1.2 MiB/s, 19 objects/s recovering 2023-06-03T20:09:00.029 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:08:59 smithi089 bash[78452]: cluster 1685822938.616199 mgr.y (mgr.34104) 985 : cluster 0 pgmap v935: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 3.6 GiB data, 12 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 648 op/s; 12387/178485 objects degraded (6.940%); 2800/178485 objects misplaced (1.569%); 1.2 MiB/s, 19 objects/s recovering 2023-06-03T20:09:00.032 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:09:00.037 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:09:00.038 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:09:00.139 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:09:00.183 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:09:00.188 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:09:00.188 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:09:00.289 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:09:00.327 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:09:00.330 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:09:00.331 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:09:00.432 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:09:00.472 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:09:00.478 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:09:00.478 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:09:00.579 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:09:00.618 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:09:00.625 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:09:00.626 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:09:00.727 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:09:00.759 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:09:00.761 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:09:00.762 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:09:00.863 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:09:00.897 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:09:00.901 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:09:00.902 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:09:01.003 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:09:01.033 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:09:01.039 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:09:01.040 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:09:01.111 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/mon.a/config 2023-06-03T20:09:01.140 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:09:01.179 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:09:01.188 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:09:01.188 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:09:01.289 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:09:01.331 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:09:01.336 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:09:01.336 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:09:01.437 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:09:01.465 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:09:01.470 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:09:01.471 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:09:01.571 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:09:01.610 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:09:01.617 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:09:01.618 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:09:01.718 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:09:01.757 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:09:01.761 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:09:01.762 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:09:01.833 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:09:01 smithi164 bash[59462]: cluster 1685822940.6169622 mgr.y (mgr.34104) 986 : cluster 0 pgmap v936: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 3.5 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 673 op/s; 12468/172440 objects degraded (7.230%); 2620/172440 objects misplaced (1.519%); 1.2 MiB/s, 18 objects/s recovering 2023-06-03T20:09:01.863 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:09:01.896 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:09:01.898 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:09:01.899 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:09:02.000 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:09:02.040 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:09:02.048 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:09:02.049 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:09:02.149 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:09:02.198 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:09:02.201 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:09:02.202 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:09:02.278 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:09:01 smithi089 bash[78452]: cluster 1685822940.6169622 mgr.y (mgr.34104) 986 : cluster 0 pgmap v936: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 3.5 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 673 op/s; 12468/172440 objects degraded (7.230%); 2620/172440 objects misplaced (1.519%); 1.2 MiB/s, 18 objects/s recovering 2023-06-03T20:09:02.278 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:09:01 smithi089 bash[75068]: cluster 1685822940.6169622 mgr.y (mgr.34104) 986 : cluster 0 pgmap v936: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 3.5 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 673 op/s; 12468/172440 objects degraded (7.230%); 2620/172440 objects misplaced (1.519%); 1.2 MiB/s, 18 objects/s recovering 2023-06-03T20:09:02.303 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:09:02.343 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:09:02.346 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:09:02.347 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:09:02.447 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:09:02.479 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:09:02.484 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:09:02.485 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:09:02.586 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:09:02.623 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:09:02.625 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:09:02.626 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:09:02.727 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:09:02.756 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:09:02.761 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:09:02.762 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:09:02.862 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:09:02.872 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:09:02 smithi164 bash[59462]: cluster 1685822942.6718733 mon.a (mon.0) 903 : cluster 3 Health check update: Degraded data redundancy: 12468/172440 objects degraded (7.230%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-06-03T20:09:02.911 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:09:02.915 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:09:02.916 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:09:02.999 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:09:02 smithi089 bash[75068]: cluster 1685822942.6718733 mon.a (mon.0) 903 : cluster 3 Health check update: Degraded data redundancy: 12468/172440 objects degraded (7.230%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-06-03T20:09:02.999 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:09:02 smithi089 bash[78452]: cluster 1685822942.6718733 mon.a (mon.0) 903 : cluster 3 Health check update: Degraded data redundancy: 12468/172440 objects degraded (7.230%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-06-03T20:09:03.016 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:09:03.052 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:09:03.056 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:09:03.057 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:09:03.158 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:09:03.186 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:09:03.190 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:09:03.191 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:09:03.292 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:09:03.320 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:09:03.322 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:09:03.323 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:09:03.424 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:09:03.457 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:09:03.466 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:09:03.467 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:09:03.568 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:09:03.599 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:09:03.602 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:09:03.603 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:09:03.703 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:09:03.734 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:09:03.738 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:09:03.739 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:09:03.839 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:09:03.895 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:09:03.897 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:09:03.898 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:09:03.930 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:09:03 smithi089 bash[78452]: cluster 1685822942.6175754 mgr.y (mgr.34104) 987 : cluster 0 pgmap v937: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 3.5 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 588 op/s; 12468/170655 objects degraded (7.306%); 2620/170655 objects misplaced (1.535%); 1.0 MiB/s, 16 objects/s recovering 2023-06-03T20:09:03.930 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:09:03 smithi089 bash[75068]: cluster 1685822942.6175754 mgr.y (mgr.34104) 987 : cluster 0 pgmap v937: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 3.5 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 588 op/s; 12468/170655 objects degraded (7.306%); 2620/170655 objects misplaced (1.535%); 1.0 MiB/s, 16 objects/s recovering 2023-06-03T20:09:03.998 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:09:04.030 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:09:04.035 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:09:04.035 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:09:04.083 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:09:03 smithi164 bash[59462]: cluster 1685822942.6175754 mgr.y (mgr.34104) 987 : cluster 0 pgmap v937: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 3.5 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 588 op/s; 12468/170655 objects degraded (7.306%); 2620/170655 objects misplaced (1.535%); 1.0 MiB/s, 16 objects/s recovering 2023-06-03T20:09:04.137 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:09:04.170 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:09:04.173 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:09:04.174 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:09:04.274 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:09:04.308 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:09:04.313 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:09:04.314 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:09:04.416 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:09:04.445 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:09:04.449 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:09:04.450 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:09:04.551 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:09:04.603 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:09:04.608 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:09:04.609 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:09:04.710 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:09:04.743 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:09:04.746 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:09:04.746 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:09:04.847 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:09:04.881 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:09:04.884 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:09:04.885 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:09:04.986 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:09:05.014 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:09:05.018 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:09:05.019 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:09:05.120 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:09:05.169 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:09:05.174 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:09:05.175 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:09:05.276 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:09:05.319 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:09:05.320 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:09:05.321 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:09:05.422 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:09:05.456 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:09:05.467 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:09:05.468 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:09:05.569 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:09:05.602 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:09:05.608 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:09:05.608 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:09:05.709 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:09:05.748 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:09:05.751 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:09:05.752 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:09:05.853 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:09:05.887 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:09:05.889 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:09:05.890 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:09:05.991 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:09:05.999 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:09:05 smithi164 bash[59462]: cluster 1685822944.6183438 mgr.y (mgr.34104) 988 : cluster 0 pgmap v938: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 3.4 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 690 op/s; 12590/165222 objects degraded (7.620%); 2401/165222 objects misplaced (1.453%); 1.5 MiB/s, 24 objects/s recovering 2023-06-03T20:09:06.025 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:09:06.028 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:09:06.028 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:09:06.129 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:09:06.160 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:09:05 smithi089 bash[75068]: cluster 1685822944.6183438 mgr.y (mgr.34104) 988 : cluster 0 pgmap v938: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 3.4 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 690 op/s; 12590/165222 objects degraded (7.620%); 2401/165222 objects misplaced (1.453%); 1.5 MiB/s, 24 objects/s recovering 2023-06-03T20:09:06.160 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:09:05 smithi089 bash[78452]: cluster 1685822944.6183438 mgr.y (mgr.34104) 988 : cluster 0 pgmap v938: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 3.4 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 690 op/s; 12590/165222 objects degraded (7.620%); 2401/165222 objects misplaced (1.453%); 1.5 MiB/s, 24 objects/s recovering 2023-06-03T20:09:06.166 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:09:06.170 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:09:06.171 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:09:06.272 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:09:06.307 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:09:06.314 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:09:06.314 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:09:06.415 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:09:06.424 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 20:09:06 smithi089 bash[71163]: ::ffff:172.21.15.164 - - [03/Jun/2023:20:09:06] "GET /metrics HTTP/1.1" 200 228756 "" "Prometheus/2.33.4" 2023-06-03T20:09:06.451 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:09:06.458 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:09:06.459 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:09:06.560 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:09:06.599 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:09:06.602 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:09:06.603 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:09:06.704 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:09:06.739 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:09:06.747 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:09:06.748 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:09:06.849 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:09:06.879 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:09:06.885 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:09:06.886 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:09:06.989 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:09:07.036 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:09:07.040 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:09:07.041 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:09:07.142 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:09:07.176 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:09:07.180 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:09:07.181 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:09:07.282 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:09:07.323 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:09:07.329 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:09:07.330 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:09:07.431 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:09:07.438 INFO:teuthology.orchestra.run.smithi089.stdout:{ 2023-06-03T20:09:07.439 INFO:teuthology.orchestra.run.smithi089.stdout: "ops": [ 2023-06-03T20:09:07.439 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:09:07.439 INFO:teuthology.orchestra.run.smithi089.stdout: "description": "osd_repop(client.36009.0:158341 10.e e176/157 10:7f370840:::benchmark_data_smithi089_140876_object53206:head v 176'9827, mlcod=176'9825)", 2023-06-03T20:09:07.439 INFO:teuthology.orchestra.run.smithi089.stdout: "initiated_at": "2023-06-03T20:09:07.429665+0000", 2023-06-03T20:09:07.439 INFO:teuthology.orchestra.run.smithi089.stdout: "age": 0.0047894519999999996, 2023-06-03T20:09:07.440 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.0048179130000000001, 2023-06-03T20:09:07.440 INFO:teuthology.orchestra.run.smithi089.stdout: "type_data": { 2023-06-03T20:09:07.440 INFO:teuthology.orchestra.run.smithi089.stdout: "flag_point": "started", 2023-06-03T20:09:07.441 INFO:teuthology.orchestra.run.smithi089.stdout: "events": [ 2023-06-03T20:09:07.441 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:09:07.441 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "initiated", 2023-06-03T20:09:07.441 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:09:07.429665+0000", 2023-06-03T20:09:07.442 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0 2023-06-03T20:09:07.442 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:09:07.442 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:09:07.442 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "throttled", 2023-06-03T20:09:07.443 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:09:07.429665+0000", 2023-06-03T20:09:07.443 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0 2023-06-03T20:09:07.443 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:09:07.443 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:09:07.443 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "header_read", 2023-06-03T20:09:07.443 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:09:07.429664+0000", 2023-06-03T20:09:07.443 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 4294967295.9999986 2023-06-03T20:09:07.444 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:09:07.444 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:09:07.444 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "all_read", 2023-06-03T20:09:07.444 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:09:07.429674+0000", 2023-06-03T20:09:07.444 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 9.9299999999999998e-06 2023-06-03T20:09:07.444 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:09:07.444 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:09:07.445 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "dispatched", 2023-06-03T20:09:07.445 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:09:07.429684+0000", 2023-06-03T20:09:07.445 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 9.4970000000000007e-06 2023-06-03T20:09:07.445 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:09:07.445 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:09:07.445 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "queued_for_pg", 2023-06-03T20:09:07.446 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:09:07.429690+0000", 2023-06-03T20:09:07.446 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 6.4609999999999996e-06 2023-06-03T20:09:07.446 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:09:07.446 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:09:07.447 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "reached_pg", 2023-06-03T20:09:07.447 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:09:07.429754+0000", 2023-06-03T20:09:07.447 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 6.3915e-05 2023-06-03T20:09:07.448 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:09:07.448 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:09:07.448 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "started", 2023-06-03T20:09:07.448 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:09:07.429781+0000", 2023-06-03T20:09:07.449 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 2.6803000000000001e-05 2023-06-03T20:09:07.449 INFO:teuthology.orchestra.run.smithi089.stdout: } 2023-06-03T20:09:07.449 INFO:teuthology.orchestra.run.smithi089.stdout: ] 2023-06-03T20:09:07.449 INFO:teuthology.orchestra.run.smithi089.stdout: } 2023-06-03T20:09:07.450 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:09:07.450 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:09:07.450 INFO:teuthology.orchestra.run.smithi089.stdout: "description": "osd_op(client.36009.0:158345 10.5 10:a6b09632:::benchmark_data_smithi089_140876_object53210:head [delete] snapc 0=[] ondisk+write+known_if_redirected e176)", 2023-06-03T20:09:07.450 INFO:teuthology.orchestra.run.smithi089.stdout: "initiated_at": "2023-06-03T20:09:07.429996+0000", 2023-06-03T20:09:07.451 INFO:teuthology.orchestra.run.smithi089.stdout: "age": 0.0044586779999999998, 2023-06-03T20:09:07.451 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.0045328749999999996, 2023-06-03T20:09:07.451 INFO:teuthology.orchestra.run.smithi089.stdout: "type_data": { 2023-06-03T20:09:07.451 INFO:teuthology.orchestra.run.smithi089.stdout: "flag_point": "waiting for sub ops", 2023-06-03T20:09:07.452 INFO:teuthology.orchestra.run.smithi089.stdout: "client_info": { 2023-06-03T20:09:07.452 INFO:teuthology.orchestra.run.smithi089.stdout: "client": "client.36009", 2023-06-03T20:09:07.452 INFO:teuthology.orchestra.run.smithi089.stdout: "client_addr": "172.21.15.89:0/2550116169", 2023-06-03T20:09:07.452 INFO:teuthology.orchestra.run.smithi089.stdout: "tid": 158345 2023-06-03T20:09:07.452 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:09:07.452 INFO:teuthology.orchestra.run.smithi089.stdout: "events": [ 2023-06-03T20:09:07.452 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:09:07.453 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "initiated", 2023-06-03T20:09:07.453 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:09:07.429996+0000", 2023-06-03T20:09:07.453 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0 2023-06-03T20:09:07.453 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:09:07.453 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:09:07.453 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "throttled", 2023-06-03T20:09:07.454 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:09:07.429996+0000", 2023-06-03T20:09:07.454 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0 2023-06-03T20:09:07.454 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:09:07.454 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:09:07.454 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "header_read", 2023-06-03T20:09:07.454 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:09:07.429995+0000", 2023-06-03T20:09:07.454 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 4294967295.9999986 2023-06-03T20:09:07.454 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:09:07.455 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:09:07.455 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "all_read", 2023-06-03T20:09:07.455 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:09:07.430002+0000", 2023-06-03T20:09:07.455 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 7.17e-06 2023-06-03T20:09:07.455 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:09:07.455 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:09:07.455 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "dispatched", 2023-06-03T20:09:07.455 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:09:07.430013+0000", 2023-06-03T20:09:07.456 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 1.13e-05 2023-06-03T20:09:07.456 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:09:07.456 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:09:07.456 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "queued_for_pg", 2023-06-03T20:09:07.456 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:09:07.430021+0000", 2023-06-03T20:09:07.456 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 7.6769999999999995e-06 2023-06-03T20:09:07.457 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:09:07.457 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:09:07.457 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "reached_pg", 2023-06-03T20:09:07.457 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:09:07.432831+0000", 2023-06-03T20:09:07.457 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.0028101549999999999 2023-06-03T20:09:07.457 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:09:07.457 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:09:07.457 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "started", 2023-06-03T20:09:07.458 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:09:07.433008+0000", 2023-06-03T20:09:07.458 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.00017751099999999999 2023-06-03T20:09:07.458 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:09:07.458 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:09:07.458 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "waiting for subops from 0,3", 2023-06-03T20:09:07.458 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:09:07.433196+0000", 2023-06-03T20:09:07.459 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.00018792900000000001 2023-06-03T20:09:07.459 INFO:teuthology.orchestra.run.smithi089.stdout: } 2023-06-03T20:09:07.459 INFO:teuthology.orchestra.run.smithi089.stdout: ] 2023-06-03T20:09:07.459 INFO:teuthology.orchestra.run.smithi089.stdout: } 2023-06-03T20:09:07.459 INFO:teuthology.orchestra.run.smithi089.stdout: } 2023-06-03T20:09:07.459 INFO:teuthology.orchestra.run.smithi089.stdout: ], 2023-06-03T20:09:07.459 INFO:teuthology.orchestra.run.smithi089.stdout: "num_ops": 2 2023-06-03T20:09:07.459 INFO:teuthology.orchestra.run.smithi089.stdout:} 2023-06-03T20:09:07.464 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:09:07.470 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:09:07.471 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:09:07.573 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:09:07.608 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:09:07.611 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:09:07.611 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:09:07.712 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:09:07.768 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:09:07.770 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:09:07.771 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:09:07.872 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:09:07.882 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:09:07 smithi089 bash[78452]: cluster 1685822946.6189196 mgr.y (mgr.34104) 989 : cluster 0 pgmap v939: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 3.3 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 606 op/s; 12590/162744 objects degraded (7.736%); 2401/162744 objects misplaced (1.475%); 1.0 MiB/s, 16 objects/s recovering 2023-06-03T20:09:07.882 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:09:07 smithi089 bash[78452]: audit 1685822947.1118681 mon.c (mon.2) 467 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T20:09:07.882 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:09:07 smithi089 bash[78452]: cluster 1685822947.672899 mon.a (mon.0) 904 : cluster 3 Health check update: Degraded data redundancy: 12590/162744 objects degraded (7.736%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-06-03T20:09:07.883 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:09:07 smithi089 bash[75068]: cluster 1685822946.6189196 mgr.y (mgr.34104) 989 : cluster 0 pgmap v939: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 3.3 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 606 op/s; 12590/162744 objects degraded (7.736%); 2401/162744 objects misplaced (1.475%); 1.0 MiB/s, 16 objects/s recovering 2023-06-03T20:09:07.883 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:09:07 smithi089 bash[75068]: audit 1685822947.1118681 mon.c (mon.2) 467 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T20:09:07.883 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:09:07 smithi089 bash[75068]: cluster 1685822947.672899 mon.a (mon.0) 904 : cluster 3 Health check update: Degraded data redundancy: 12590/162744 objects degraded (7.736%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-06-03T20:09:07.901 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:09:07.904 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:09:07.904 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:09:08.005 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:09:08.014 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:09:07 smithi164 bash[59462]: cluster 1685822946.6189196 mgr.y (mgr.34104) 989 : cluster 0 pgmap v939: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 3.3 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 606 op/s; 12590/162744 objects degraded (7.736%); 2401/162744 objects misplaced (1.475%); 1.0 MiB/s, 16 objects/s recovering 2023-06-03T20:09:08.014 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:09:07 smithi164 bash[59462]: audit 1685822947.1118681 mon.c (mon.2) 467 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T20:09:08.015 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:09:07 smithi164 bash[59462]: cluster 1685822947.672899 mon.a (mon.0) 904 : cluster 3 Health check update: Degraded data redundancy: 12590/162744 objects degraded (7.736%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-06-03T20:09:08.043 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:09:08.045 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:09:08.046 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:09:08.147 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:09:08.177 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:09:08.180 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:09:08.182 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:09:08.282 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:09:08.309 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:09:08.313 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:09:08.314 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:09:08.415 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:09:08.460 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:09:08.463 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:09:08.464 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:09:08.565 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:09:08.603 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:09:08.605 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:09:08.605 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:09:08.706 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:09:08.746 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:09:08.753 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:09:08.754 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:09:08.854 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:09:08.894 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:09:08.897 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:09:08.897 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:09:08.998 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:09:09.035 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:09:09.037 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:09:09.038 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:09:09.139 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:09:09.171 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:09:09.179 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:09:09.179 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:09:09.280 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:09:09.320 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:09:09.325 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:09:09.326 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:09:09.427 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:09:09.472 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:09:09.474 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:09:09.475 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:09:09.576 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:09:09.626 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:09:09.634 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:09:09.634 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:09:09.735 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:09:09.767 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:09:09.771 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:09:09.772 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:09:09.873 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:09:09.908 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:09:09.912 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:09:09.913 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:09:10.014 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:09:10.022 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:09:09 smithi089 bash[75068]: cluster 1685822948.619638 mgr.y (mgr.34104) 990 : cluster 0 pgmap v940: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 3.2 GiB data, 11 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 638 op/s; 12590/159222 objects degraded (7.907%); 2401/159222 objects misplaced (1.508%); 1.0 MiB/s, 16 objects/s recovering 2023-06-03T20:09:10.022 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:09:09 smithi089 bash[78452]: cluster 1685822948.619638 mgr.y (mgr.34104) 990 : cluster 0 pgmap v940: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 3.2 GiB data, 11 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 638 op/s; 12590/159222 objects degraded (7.907%); 2401/159222 objects misplaced (1.508%); 1.0 MiB/s, 16 objects/s recovering 2023-06-03T20:09:10.054 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:09:10.058 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:09:10.059 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:09:10.083 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:09:09 smithi164 bash[59462]: cluster 1685822948.619638 mgr.y (mgr.34104) 990 : cluster 0 pgmap v940: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 3.2 GiB data, 11 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 638 op/s; 12590/159222 objects degraded (7.907%); 2401/159222 objects misplaced (1.508%); 1.0 MiB/s, 16 objects/s recovering 2023-06-03T20:09:10.160 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:09:10.193 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:09:10.196 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:09:10.197 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:09:10.298 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:09:10.352 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:09:10.354 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:09:10.355 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:09:10.456 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:09:10.482 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:09:10.484 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:09:10.485 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:09:10.586 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:09:10.618 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:09:10.625 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:09:10.626 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:09:10.726 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:09:10.768 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:09:10.772 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:09:10.773 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:09:10.874 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:09:10.910 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:09:10.913 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:09:10.913 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:09:11.014 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:09:11.049 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:09:11.056 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:09:11.057 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:09:11.111 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph daemon osd.2 dump_blocked_ops 2023-06-03T20:09:11.158 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:09:11.193 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:09:11.200 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:09:11.201 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:09:11.302 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:09:11.332 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:09:11.339 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:09:11.340 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:09:11.441 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:09:11.489 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:09:11.491 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:09:11.492 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:09:11.593 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:09:11.632 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:09:11.637 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:09:11.638 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:09:11.739 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:09:11.775 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:09:11.777 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:09:11.778 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:09:11.879 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:09:11.906 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:09:11.919 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:09:11.920 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:09:12.021 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:09:12.032 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:09:11 smithi089 bash[78452]: cluster 1685822950.620327 mgr.y (mgr.34104) 991 : cluster 0 pgmap v941: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 3.1 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 699 op/s; 12665/153312 objects degraded (8.261%); 2226/153312 objects misplaced (1.452%); 1.5 MiB/s, 24 objects/s recovering 2023-06-03T20:09:12.033 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:09:11 smithi089 bash[75068]: cluster 1685822950.620327 mgr.y (mgr.34104) 991 : cluster 0 pgmap v941: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 3.1 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 699 op/s; 12665/153312 objects degraded (8.261%); 2226/153312 objects misplaced (1.452%); 1.5 MiB/s, 24 objects/s recovering 2023-06-03T20:09:12.061 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:09:12.067 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:09:12.068 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:09:12.147 INFO:teuthology.orchestra.run.smithi089.stdout:osd.0: {} 2023-06-03T20:09:12.168 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:09:12.172 INFO:teuthology.orchestra.run.smithi089.stdout:osd.1: {} 2023-06-03T20:09:12.195 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:09:12.199 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:09:12.199 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:09:12.204 INFO:teuthology.orchestra.run.smithi089.stdout:osd.2: {} 2023-06-03T20:09:12.226 INFO:teuthology.orchestra.run.smithi089.stdout:osd.3: {} 2023-06-03T20:09:12.269 INFO:teuthology.orchestra.run.smithi089.stdout:osd.4: {} 2023-06-03T20:09:12.297 INFO:teuthology.orchestra.run.smithi089.stdout:osd.5: {} 2023-06-03T20:09:12.300 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:09:12.326 INFO:teuthology.orchestra.run.smithi089.stdout:osd.6: {} 2023-06-03T20:09:12.327 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:09:12.332 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:09:12.334 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:09:12.335 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:09:11 smithi164 bash[59462]: cluster 1685822950.620327 mgr.y (mgr.34104) 991 : cluster 0 pgmap v941: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 3.1 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 699 op/s; 12665/153312 objects degraded (8.261%); 2226/153312 objects misplaced (1.452%); 1.5 MiB/s, 24 objects/s recovering 2023-06-03T20:09:12.363 INFO:teuthology.orchestra.run.smithi089.stdout:osd.7: {} 2023-06-03T20:09:12.365 INFO:teuthology.orchestra.run.smithi089.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-06-03T20:09:12.365 INFO:teuthology.orchestra.run.smithi089.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-06-03T20:09:12.365 INFO:teuthology.orchestra.run.smithi089.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-06-03T20:09:12.366 INFO:teuthology.orchestra.run.smithi089.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-06-03T20:09:12.366 INFO:teuthology.orchestra.run.smithi089.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-06-03T20:09:12.366 INFO:teuthology.orchestra.run.smithi089.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-06-03T20:09:12.366 INFO:teuthology.orchestra.run.smithi089.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-06-03T20:09:12.366 INFO:teuthology.orchestra.run.smithi089.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-06-03T20:09:12.436 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:09:12.468 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:09:12.472 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:09:12.473 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:09:12.574 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:09:12.606 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:09:12.609 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:09:12.610 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:09:12.711 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:09:12.751 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:09:12.755 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:09:12.757 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:09:12.845 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:09:12 smithi089 bash[75068]: audit 1685822952.1129878 mon.c (mon.2) 468 : audit 0 from='client.? 172.21.15.89:0/1487566414' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-03T20:09:12.845 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:09:12 smithi089 bash[75068]: cluster 1685822952.6740677 mon.a (mon.0) 905 : cluster 3 Health check update: Degraded data redundancy: 12665/153312 objects degraded (8.261%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-06-03T20:09:12.858 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:09:12.908 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:09:12.910 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:09:12.911 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:09:13.012 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:09:13.057 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:09:13.062 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:09:13.063 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:09:13.083 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:09:12 smithi164 bash[59462]: audit 1685822952.1129878 mon.c (mon.2) 468 : audit 0 from='client.? 172.21.15.89:0/1487566414' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-03T20:09:13.083 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:09:12 smithi164 bash[59462]: cluster 1685822952.6740677 mon.a (mon.0) 905 : cluster 3 Health check update: Degraded data redundancy: 12665/153312 objects degraded (8.261%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-06-03T20:09:13.164 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:09:13.207 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:09:13.214 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:09:13.215 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:09:13.278 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:09:12 smithi089 bash[78452]: audit 1685822952.1129878 mon.c (mon.2) 468 : audit 0 from='client.? 172.21.15.89:0/1487566414' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-03T20:09:13.278 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:09:12 smithi089 bash[78452]: cluster 1685822952.6740677 mon.a (mon.0) 905 : cluster 3 Health check update: Degraded data redundancy: 12665/153312 objects degraded (8.261%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-06-03T20:09:13.316 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:09:13.342 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:09:13.344 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:09:13.344 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:09:13.445 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:09:13.488 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:09:13.492 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:09:13.492 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:09:13.594 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:09:13.619 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:09:13.622 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:09:13.623 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:09:13.723 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:09:13.766 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:09:13.769 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:09:13.770 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:09:13.861 INFO:teuthology.orchestra.run.smithi089.stderr:reweighted osd.2 to 1 (10000) 2023-06-03T20:09:13.871 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:09:13.898 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:09:13.900 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:09:13.901 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:09:14.003 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:09:14.030 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:09:14.033 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:09:14.034 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:09:14.083 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:09:13 smithi164 bash[59462]: cluster 1685822952.6209764 mgr.y (mgr.34104) 992 : cluster 0 pgmap v942: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 3.1 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 577 op/s; 12665/151647 objects degraded (8.352%); 2226/151647 objects misplaced (1.468%); 1.0 MiB/s, 16 objects/s recovering 2023-06-03T20:09:14.084 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:09:13 smithi164 bash[59462]: audit 1685822952.9896288 mon.b (mon.1) 54 : audit 1 from='client.? 172.21.15.89:0/283331470' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-06-03T20:09:14.084 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:09:13 smithi164 bash[59462]: audit 1685822952.9962175 mon.a (mon.0) 906 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-06-03T20:09:14.134 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:09:14.142 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:09:13 smithi089 bash[78452]: cluster 1685822952.6209764 mgr.y (mgr.34104) 992 : cluster 0 pgmap v942: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 3.1 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 577 op/s; 12665/151647 objects degraded (8.352%); 2226/151647 objects misplaced (1.468%); 1.0 MiB/s, 16 objects/s recovering 2023-06-03T20:09:14.142 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:09:13 smithi089 bash[78452]: audit 1685822952.9896288 mon.b (mon.1) 54 : audit 1 from='client.? 172.21.15.89:0/283331470' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-06-03T20:09:14.142 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:09:13 smithi089 bash[78452]: audit 1685822952.9962175 mon.a (mon.0) 906 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-06-03T20:09:14.143 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:09:13 smithi089 bash[75068]: cluster 1685822952.6209764 mgr.y (mgr.34104) 992 : cluster 0 pgmap v942: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 3.1 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 577 op/s; 12665/151647 objects degraded (8.352%); 2226/151647 objects misplaced (1.468%); 1.0 MiB/s, 16 objects/s recovering 2023-06-03T20:09:14.143 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:09:13 smithi089 bash[75068]: audit 1685822952.9896288 mon.b (mon.1) 54 : audit 1 from='client.? 172.21.15.89:0/283331470' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-06-03T20:09:14.143 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:09:13 smithi089 bash[75068]: audit 1685822952.9962175 mon.a (mon.0) 906 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-06-03T20:09:14.168 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:09:14.170 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:09:14.171 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:09:14.272 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:09:14.314 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:09:14.320 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:09:14.321 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:09:14.421 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:09:14.469 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:09:14.472 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:09:14.472 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:09:14.573 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:09:14.602 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:09:14.609 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:09:14.610 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:09:14.711 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:09:14.740 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:09:14.741 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:09:14.742 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:09:14.843 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:09:14.879 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:09:14.884 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:09:14.885 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:09:14.986 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:09:15.029 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:09:14 smithi089 bash[75068]: cluster 1685822953.8431928 osd.5 (osd.5) 106 : cluster 3 failed to encode map e177 with expected crc 2023-06-03T20:09:15.029 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:09:14 smithi089 bash[75068]: cluster 1685822953.8446684 osd.6 (osd.6) 114 : cluster 3 failed to encode map e177 with expected crc 2023-06-03T20:09:15.029 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:09:14 smithi089 bash[75068]: audit 1685822953.8460631 mon.a (mon.0) 907 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2023-06-03T20:09:15.030 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:09:14 smithi089 bash[75068]: cluster 1685822953.8461423 mon.a (mon.0) 908 : cluster 0 osdmap e177: 8 total, 8 up, 7 in 2023-06-03T20:09:15.030 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:09:14 smithi089 bash[75068]: cluster 1685822953.8569794 osd.7 (osd.7) 113 : cluster 3 failed to encode map e177 with expected crc 2023-06-03T20:09:15.030 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:09:14 smithi089 bash[78452]: cluster 1685822953.8431928 osd.5 (osd.5) 106 : cluster 3 failed to encode map e177 with expected crc 2023-06-03T20:09:15.031 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:09:14 smithi089 bash[78452]: cluster 1685822953.8446684 osd.6 (osd.6) 114 : cluster 3 failed to encode map e177 with expected crc 2023-06-03T20:09:15.031 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:09:14 smithi089 bash[78452]: audit 1685822953.8460631 mon.a (mon.0) 907 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2023-06-03T20:09:15.031 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:09:14 smithi089 bash[78452]: cluster 1685822953.8461423 mon.a (mon.0) 908 : cluster 0 osdmap e177: 8 total, 8 up, 7 in 2023-06-03T20:09:15.032 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:09:14 smithi089 bash[78452]: cluster 1685822953.8569794 osd.7 (osd.7) 113 : cluster 3 failed to encode map e177 with expected crc 2023-06-03T20:09:15.032 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:09:15.034 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:09:15.035 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:09:15.136 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:09:15.166 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:09:15.170 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:09:15.170 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:09:15.271 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:09:15.311 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:09:15.314 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:09:15.315 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:09:15.333 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:09:14 smithi164 bash[59462]: cluster 1685822953.8431928 osd.5 (osd.5) 106 : cluster 3 failed to encode map e177 with expected crc 2023-06-03T20:09:15.333 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:09:14 smithi164 bash[59462]: cluster 1685822953.8446684 osd.6 (osd.6) 114 : cluster 3 failed to encode map e177 with expected crc 2023-06-03T20:09:15.334 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:09:14 smithi164 bash[59462]: audit 1685822953.8460631 mon.a (mon.0) 907 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2023-06-03T20:09:15.334 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:09:14 smithi164 bash[59462]: cluster 1685822953.8461423 mon.a (mon.0) 908 : cluster 0 osdmap e177: 8 total, 8 up, 7 in 2023-06-03T20:09:15.334 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:09:14 smithi164 bash[59462]: cluster 1685822953.8569794 osd.7 (osd.7) 113 : cluster 3 failed to encode map e177 with expected crc 2023-06-03T20:09:15.415 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:09:15.462 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:09:15.469 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:09:15.469 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:09:15.570 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:09:15.625 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:09:15.629 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:09:15.630 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:09:15.718 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph osd reweight 3 1 2023-06-03T20:09:15.731 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:09:15.775 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:09:15.780 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:09:15.781 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:09:15.868 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:09:15 smithi089 bash[75068]: cluster 1685822954.6217709 mgr.y (mgr.34104) 993 : cluster 0 pgmap v944: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 3.0 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 618 op/s; 12765/146664 objects degraded (8.704%); 2028/146664 objects misplaced (1.383%); 1.2 MiB/s, 19 objects/s recovering 2023-06-03T20:09:15.868 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:09:15 smithi089 bash[78452]: cluster 1685822954.6217709 mgr.y (mgr.34104) 993 : cluster 0 pgmap v944: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 3.0 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 618 op/s; 12765/146664 objects degraded (8.704%); 2028/146664 objects misplaced (1.383%); 1.2 MiB/s, 19 objects/s recovering 2023-06-03T20:09:15.882 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:09:15.917 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:09:15.929 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:09:15.930 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:09:16.030 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:09:16.077 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:09:16.083 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:09:15 smithi164 bash[59462]: cluster 1685822954.6217709 mgr.y (mgr.34104) 993 : cluster 0 pgmap v944: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 3.0 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 618 op/s; 12765/146664 objects degraded (8.704%); 2028/146664 objects misplaced (1.383%); 1.2 MiB/s, 19 objects/s recovering 2023-06-03T20:09:16.087 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:09:16.088 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:09:16.189 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:09:16.221 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:09:16.225 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:09:16.226 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:09:16.328 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:09:16.365 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:09:16.369 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:09:16.370 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:09:16.470 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:09:16.472 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-06-03T20:09:16.523 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:09:16.528 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 20:09:16 smithi089 bash[71163]: ::ffff:172.21.15.164 - - [03/Jun/2023:20:09:16] "GET /metrics HTTP/1.1" 200 228747 "" "Prometheus/2.33.4" 2023-06-03T20:09:16.529 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:09:16.530 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:09:16.632 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:09:16.679 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:09:16.690 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:09:16.691 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:09:16.792 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:09:16.825 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:09:16.832 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:09:16.833 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:09:16.886 INFO:teuthology.orchestra.run.smithi089.stderr:noscrub is unset 2023-06-03T20:09:16.934 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:09:16.968 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:09:16.969 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:09:16.970 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:09:17.071 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:09:17.080 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:09:16 smithi089 bash[75068]: audit 1685822956.227872 mon.c (mon.2) 469 : audit 1 from='client.? 172.21.15.89:0/3462730164' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-06-03T20:09:17.081 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:09:16 smithi089 bash[75068]: audit 1685822956.2284417 mon.a (mon.0) 909 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-06-03T20:09:17.081 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:09:16 smithi089 bash[78452]: audit 1685822956.227872 mon.c (mon.2) 469 : audit 1 from='client.? 172.21.15.89:0/3462730164' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-06-03T20:09:17.081 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:09:16 smithi089 bash[78452]: audit 1685822956.2284417 mon.a (mon.0) 909 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-06-03T20:09:17.101 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:09:17.103 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:09:17.103 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:09:17.204 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:09:17.214 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:09:16 smithi164 bash[59462]: audit 1685822956.227872 mon.c (mon.2) 469 : audit 1 from='client.? 172.21.15.89:0/3462730164' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-06-03T20:09:17.214 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:09:16 smithi164 bash[59462]: audit 1685822956.2284417 mon.a (mon.0) 909 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-06-03T20:09:17.246 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:09:17.251 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:09:17.252 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:09:17.352 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:09:17.391 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:09:17.393 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:09:17.393 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:09:17.494 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:09:17.531 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:09:17.533 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:09:17.534 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:09:17.635 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:09:17.676 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:09:17.682 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:09:17.683 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:09:17.784 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:09:17.817 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:09:17.822 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:09:17.822 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:09:17.924 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:09:17.954 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:09:17.957 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:09:17.958 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:09:18.059 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:09:18.083 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:09:17 smithi164 bash[59462]: cluster 1685822956.6225488 mgr.y (mgr.34104) 994 : cluster 0 pgmap v945: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 2.9 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 612 op/s; 12765/144354 objects degraded (8.843%); 2028/144354 objects misplaced (1.405%); 1.2 MiB/s, 19 objects/s recovering 2023-06-03T20:09:18.084 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:09:17 smithi164 bash[59462]: cluster 1685822956.863248 mon.a (mon.0) 910 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-06-03T20:09:18.084 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:09:17 smithi164 bash[59462]: cluster 1685822956.8695173 osd.6 (osd.6) 115 : cluster 3 failed to encode map e178 with expected crc 2023-06-03T20:09:18.084 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:09:17 smithi164 bash[59462]: audit 1685822956.8724992 mon.a (mon.0) 911 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-06-03T20:09:18.084 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:09:17 smithi164 bash[59462]: cluster 1685822956.872617 mon.a (mon.0) 912 : cluster 0 osdmap e178: 8 total, 8 up, 7 in 2023-06-03T20:09:18.084 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:09:17 smithi164 bash[59462]: cluster 1685822956.8801064 osd.5 (osd.5) 107 : cluster 3 failed to encode map e178 with expected crc 2023-06-03T20:09:18.085 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:09:17 smithi164 bash[59462]: cluster 1685822956.8912084 osd.7 (osd.7) 114 : cluster 3 failed to encode map e178 with expected crc 2023-06-03T20:09:18.085 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:09:17 smithi164 bash[59462]: cluster 1685822956.8931124 osd.7 (osd.7) 115 : cluster 3 failed to encode map e178 with expected crc 2023-06-03T20:09:18.085 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:09:17 smithi164 bash[59462]: cluster 1685822957.6751215 mon.a (mon.0) 913 : cluster 3 Health check update: Degraded data redundancy: 12765/144354 objects degraded (8.843%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-06-03T20:09:18.090 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:09:18.096 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:09:18.097 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:09:18.199 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:09:18.206 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:09:17 smithi089 bash[78452]: cluster 1685822956.6225488 mgr.y (mgr.34104) 994 : cluster 0 pgmap v945: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 2.9 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 612 op/s; 12765/144354 objects degraded (8.843%); 2028/144354 objects misplaced (1.405%); 1.2 MiB/s, 19 objects/s recovering 2023-06-03T20:09:18.207 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:09:17 smithi089 bash[78452]: cluster 1685822956.863248 mon.a (mon.0) 910 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-06-03T20:09:18.207 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:09:17 smithi089 bash[78452]: cluster 1685822956.8695173 osd.6 (osd.6) 115 : cluster 3 failed to encode map e178 with expected crc 2023-06-03T20:09:18.207 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:09:17 smithi089 bash[78452]: audit 1685822956.8724992 mon.a (mon.0) 911 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-06-03T20:09:18.208 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:09:17 smithi089 bash[78452]: cluster 1685822956.872617 mon.a (mon.0) 912 : cluster 0 osdmap e178: 8 total, 8 up, 7 in 2023-06-03T20:09:18.208 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:09:17 smithi089 bash[78452]: cluster 1685822956.8801064 osd.5 (osd.5) 107 : cluster 3 failed to encode map e178 with expected crc 2023-06-03T20:09:18.208 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:09:17 smithi089 bash[78452]: cluster 1685822956.8912084 osd.7 (osd.7) 114 : cluster 3 failed to encode map e178 with expected crc 2023-06-03T20:09:18.208 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:09:17 smithi089 bash[78452]: cluster 1685822956.8931124 osd.7 (osd.7) 115 : cluster 3 failed to encode map e178 with expected crc 2023-06-03T20:09:18.208 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:09:17 smithi089 bash[78452]: cluster 1685822957.6751215 mon.a (mon.0) 913 : cluster 3 Health check update: Degraded data redundancy: 12765/144354 objects degraded (8.843%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-06-03T20:09:18.209 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:09:17 smithi089 bash[75068]: cluster 1685822956.6225488 mgr.y (mgr.34104) 994 : cluster 0 pgmap v945: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 2.9 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 612 op/s; 12765/144354 objects degraded (8.843%); 2028/144354 objects misplaced (1.405%); 1.2 MiB/s, 19 objects/s recovering 2023-06-03T20:09:18.209 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:09:17 smithi089 bash[75068]: cluster 1685822956.863248 mon.a (mon.0) 910 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-06-03T20:09:18.209 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:09:17 smithi089 bash[75068]: cluster 1685822956.8695173 osd.6 (osd.6) 115 : cluster 3 failed to encode map e178 with expected crc 2023-06-03T20:09:18.209 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:09:17 smithi089 bash[75068]: audit 1685822956.8724992 mon.a (mon.0) 911 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-06-03T20:09:18.209 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:09:17 smithi089 bash[75068]: cluster 1685822956.872617 mon.a (mon.0) 912 : cluster 0 osdmap e178: 8 total, 8 up, 7 in 2023-06-03T20:09:18.209 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:09:17 smithi089 bash[75068]: cluster 1685822956.8801064 osd.5 (osd.5) 107 : cluster 3 failed to encode map e178 with expected crc 2023-06-03T20:09:18.210 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:09:17 smithi089 bash[75068]: cluster 1685822956.8912084 osd.7 (osd.7) 114 : cluster 3 failed to encode map e178 with expected crc 2023-06-03T20:09:18.210 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:09:17 smithi089 bash[75068]: cluster 1685822956.8931124 osd.7 (osd.7) 115 : cluster 3 failed to encode map e178 with expected crc 2023-06-03T20:09:18.210 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:09:17 smithi089 bash[75068]: cluster 1685822957.6751215 mon.a (mon.0) 913 : cluster 3 Health check update: Degraded data redundancy: 12765/144354 objects degraded (8.843%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-06-03T20:09:18.237 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:09:18.247 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:09:18.248 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:09:18.349 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:09:18.385 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:09:18.386 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:09:18.386 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:09:18.487 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:09:18.524 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:09:18.529 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:09:18.530 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:09:18.630 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:09:18.670 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:09:18.676 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:09:18.677 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:09:18.681 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/mon.a/config 2023-06-03T20:09:18.778 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:09:18.815 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:09:18.818 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:09:18.819 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:09:18.920 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:09:18.953 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:09:18.959 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:09:18.959 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:09:19.060 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:09:19.109 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:09:19.118 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:09:19.118 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:09:19.219 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:09:19.256 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:09:19.259 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:09:19.260 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:09:19.361 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:09:19.408 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:09:19.412 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:09:19.413 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:09:19.514 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:09:19.569 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:09:19.574 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:09:19.575 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:09:19.676 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:09:19.713 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:09:19.720 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:09:19.721 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:09:19.822 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:09:19.869 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:09:19.870 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:09:19.870 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:09:19.972 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:09:20.004 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:09:20.006 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:09:20.007 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:09:20.083 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:09:19 smithi164 bash[59462]: cluster 1685822958.6234155 mgr.y (mgr.34104) 995 : cluster 0 pgmap v947: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 2.9 GiB data, 10 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 468 op/s; 12765/142074 objects degraded (8.985%); 2028/142074 objects misplaced (1.427%); 784 KiB/s, 12 objects/s recovering 2023-06-03T20:09:20.084 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:09:19 smithi164 bash[59462]: audit 1685822959.2169762 mon.c (mon.2) 470 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-03T20:09:20.108 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:09:20.149 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:09:20.155 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:09:20.156 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:09:20.207 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph osd unset nodeep-scrub 2023-06-03T20:09:20.216 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:09:19 smithi089 bash[78452]: cluster 1685822958.6234155 mgr.y (mgr.34104) 995 : cluster 0 pgmap v947: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 2.9 GiB data, 10 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 468 op/s; 12765/142074 objects degraded (8.985%); 2028/142074 objects misplaced (1.427%); 784 KiB/s, 12 objects/s recovering 2023-06-03T20:09:20.216 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:09:19 smithi089 bash[78452]: audit 1685822959.2169762 mon.c (mon.2) 470 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-03T20:09:20.217 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:09:19 smithi089 bash[75068]: cluster 1685822958.6234155 mgr.y (mgr.34104) 995 : cluster 0 pgmap v947: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 2.9 GiB data, 10 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 468 op/s; 12765/142074 objects degraded (8.985%); 2028/142074 objects misplaced (1.427%); 784 KiB/s, 12 objects/s recovering 2023-06-03T20:09:20.217 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:09:19 smithi089 bash[75068]: audit 1685822959.2169762 mon.c (mon.2) 470 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-03T20:09:20.258 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:09:20.306 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:09:20.312 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:09:20.313 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:09:20.414 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:09:20.453 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:09:20.458 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:09:20.459 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:09:20.560 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:09:20.590 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:09:20.592 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:09:20.593 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:09:20.694 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:09:20.731 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:09:20.735 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:09:20.736 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:09:20.757 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/mon.a/config 2023-06-03T20:09:20.836 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:09:20.866 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:09:20.872 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:09:20.873 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:09:20.974 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:09:21.001 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:09:21.006 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:09:21.007 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:09:21.108 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:09:21.133 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:09:21.141 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:09:21.141 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:09:21.242 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:09:21.277 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:09:21.285 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:09:21.287 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:09:21.388 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:09:21.413 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:09:21.415 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:09:21.416 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:09:21.518 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:09:21.548 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:09:21.559 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:09:21.560 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:09:21.661 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:09:21.687 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:09:21.689 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:09:21.689 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:09:21.790 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:09:21.820 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:09:21.822 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:09:21.822 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:09:21.924 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:09:21.976 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:09:21.979 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:09:21.979 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:09:22.080 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:09:22.083 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:09:21 smithi164 bash[59462]: cluster 1685822960.624182 mgr.y (mgr.34104) 996 : cluster 0 pgmap v948: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 2.7 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 698 op/s; 12849/134880 objects degraded (9.526%); 1841/134880 objects misplaced (1.365%); 1.6 MiB/s, 25 objects/s recovering 2023-06-03T20:09:22.125 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:09:22.130 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:09:22.130 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:09:22.231 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:09:22.278 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:09:21 smithi089 bash[75068]: cluster 1685822960.624182 mgr.y (mgr.34104) 996 : cluster 0 pgmap v948: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 2.7 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 698 op/s; 12849/134880 objects degraded (9.526%); 1841/134880 objects misplaced (1.365%); 1.6 MiB/s, 25 objects/s recovering 2023-06-03T20:09:22.279 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:09:21 smithi089 bash[78452]: cluster 1685822960.624182 mgr.y (mgr.34104) 996 : cluster 0 pgmap v948: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 2.7 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 698 op/s; 12849/134880 objects degraded (9.526%); 1841/134880 objects misplaced (1.365%); 1.6 MiB/s, 25 objects/s recovering 2023-06-03T20:09:22.283 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:09:22.292 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:09:22.293 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:09:22.394 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:09:22.433 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:09:22.437 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:09:22.438 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:09:22.539 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:09:22.580 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:09:22.585 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:09:22.586 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:09:22.687 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:09:22.728 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:09:22.732 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:09:22.732 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:09:22.833 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:09:22.879 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:09:22.886 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:09:22.890 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:09:22.991 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:09:23.027 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:09:23.028 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:09:22 smithi089 bash[78452]: audit 1685822962.1123598 mon.c (mon.2) 471 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T20:09:23.028 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:09:22 smithi089 bash[78452]: cluster 1685822962.676433 mon.a (mon.0) 914 : cluster 3 Health check update: Degraded data redundancy: 12849/134880 objects degraded (9.526%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-06-03T20:09:23.028 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:09:22 smithi089 bash[75068]: audit 1685822962.1123598 mon.c (mon.2) 471 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T20:09:23.029 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:09:22 smithi089 bash[75068]: cluster 1685822962.676433 mon.a (mon.0) 914 : cluster 3 Health check update: Degraded data redundancy: 12849/134880 objects degraded (9.526%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-06-03T20:09:23.032 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:09:23.033 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:09:23.134 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:09:23.165 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:09:23.170 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:09:23.171 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:09:23.190 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/mon.a/config 2023-06-03T20:09:23.272 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:09:23.306 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:09:23.309 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:09:23.310 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:09:23.334 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:09:22 smithi164 bash[59462]: audit 1685822962.1123598 mon.c (mon.2) 471 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T20:09:23.334 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:09:22 smithi164 bash[59462]: cluster 1685822962.676433 mon.a (mon.0) 914 : cluster 3 Health check update: Degraded data redundancy: 12849/134880 objects degraded (9.526%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-06-03T20:09:23.411 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:09:23.448 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:09:23.449 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:09:23.450 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:09:23.551 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:09:23.592 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:09:23.598 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:09:23.598 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:09:23.699 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:09:23.733 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:09:23.739 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:09:23.740 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:09:23.841 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:09:23.885 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:09:23.890 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:09:23.891 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:09:23.992 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:09:24.000 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:09:23 smithi089 bash[78452]: cluster 1685822962.6248553 mgr.y (mgr.34104) 997 : cluster 0 pgmap v949: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 2.7 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 514 op/s; 12849/133122 objects degraded (9.652%); 1841/133122 objects misplaced (1.383%); 751 KiB/s, 11 objects/s recovering 2023-06-03T20:09:24.001 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:09:23 smithi089 bash[75068]: cluster 1685822962.6248553 mgr.y (mgr.34104) 997 : cluster 0 pgmap v949: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 2.7 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 514 op/s; 12849/133122 objects degraded (9.652%); 1841/133122 objects misplaced (1.383%); 751 KiB/s, 11 objects/s recovering 2023-06-03T20:09:24.025 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:09:24.026 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:09:24.027 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:09:24.128 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:09:24.136 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:09:23 smithi164 bash[59462]: cluster 1685822962.6248553 mgr.y (mgr.34104) 997 : cluster 0 pgmap v949: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 2.7 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 514 op/s; 12849/133122 objects degraded (9.652%); 1841/133122 objects misplaced (1.383%); 751 KiB/s, 11 objects/s recovering 2023-06-03T20:09:24.158 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:09:24.161 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:09:24.162 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:09:24.263 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:09:24.292 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:09:24.303 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:09:24.304 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:09:24.405 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:09:24.455 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:09:24.479 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:09:24.480 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:09:24.581 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:09:24.610 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:09:24.614 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:09:24.615 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:09:24.716 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:09:24.749 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:09:24.754 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:09:24.755 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:09:24.856 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:09:24.866 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/mon.a/config 2023-06-03T20:09:24.913 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:09:24.915 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:09:24.915 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:09:25.016 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:09:25.067 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:09:25.071 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:09:25.072 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:09:25.173 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:09:25.214 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:09:25.221 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:09:25.222 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:09:25.322 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:09:25.365 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:09:25.368 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:09:25.369 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:09:25.470 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:09:25.526 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:09:25.535 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:09:25.536 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:09:25.637 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:09:25.679 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:09:25.680 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:09:25.680 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:09:25.781 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:09:25.820 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:09:25.823 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:09:25.824 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:09:25.925 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:09:25.946 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:09:25 smithi089 bash[75068]: cluster 1685822964.6255805 mgr.y (mgr.34104) 998 : cluster 0 pgmap v950: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 2.6 GiB data, 9.7 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 625 op/s; 12922/127896 objects degraded (10.104%); 1664/127896 objects misplaced (1.301%); 1.3 MiB/s, 20 objects/s recovering 2023-06-03T20:09:25.966 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:09:25.969 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:09:25.970 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:09:26.071 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:09:26.096 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:09:26.098 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:09:26.098 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:09:26.199 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:09:26.210 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:09:25 smithi164 bash[59462]: cluster 1685822964.6255805 mgr.y (mgr.34104) 998 : cluster 0 pgmap v950: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 2.6 GiB data, 9.7 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 625 op/s; 12922/127896 objects degraded (10.104%); 1664/127896 objects misplaced (1.301%); 1.3 MiB/s, 20 objects/s recovering 2023-06-03T20:09:26.231 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 20:09:26 smithi089 bash[71163]: ::ffff:172.21.15.164 - - [03/Jun/2023:20:09:26] "GET /metrics HTTP/1.1" 200 228768 "" "Prometheus/2.33.4" 2023-06-03T20:09:26.231 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:09:25 smithi089 bash[78452]: cluster 1685822964.6255805 mgr.y (mgr.34104) 998 : cluster 0 pgmap v950: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 2.6 GiB data, 9.7 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 625 op/s; 12922/127896 objects degraded (10.104%); 1664/127896 objects misplaced (1.301%); 1.3 MiB/s, 20 objects/s recovering 2023-06-03T20:09:26.241 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:09:26.246 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:09:26.247 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:09:26.348 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:09:26.388 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:09:26.395 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:09:26.395 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:09:26.496 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:09:26.540 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:09:26.547 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:09:26.548 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:09:26.648 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:09:26.687 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:09:26.690 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:09:26.690 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:09:26.791 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:09:26.859 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:09:26.863 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:09:26.864 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:09:26.965 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:09:27.004 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:09:27.007 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:09:27.008 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:09:27.109 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:09:27.140 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:09:27.144 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:09:27.145 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:09:27.246 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:09:27.279 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:09:27.281 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:09:27.282 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:09:27.383 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:09:27.416 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:09:27.418 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:09:27.419 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:09:27.520 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:09:27.528 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:09:27 smithi089 bash[75068]: audit 1685822966.3252234 mon.a (mon.0) 915 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:09:27.529 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:09:27 smithi089 bash[75068]: audit 1685822966.3380282 mon.a (mon.0) 916 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:09:27.529 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:09:27 smithi089 bash[75068]: audit 1685822966.3773959 mon.a (mon.0) 917 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:09:27.529 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:09:27 smithi089 bash[75068]: audit 1685822966.3908374 mon.a (mon.0) 918 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:09:27.529 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:09:27 smithi089 bash[75068]: cluster 1685822966.626048 mgr.y (mgr.34104) 999 : cluster 0 pgmap v951: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 2.6 GiB data, 9.5 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 619 op/s; 12922/125754 objects degraded (10.276%); 1664/125754 objects misplaced (1.323%); 1.3 MiB/s, 20 objects/s recovering 2023-06-03T20:09:27.529 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:09:27 smithi089 bash[75068]: audit 1685822966.9654932 mon.c (mon.2) 472 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-03T20:09:27.530 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:09:27 smithi089 bash[75068]: audit 1685822966.966728 mon.c (mon.2) 473 : audit 1 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-03T20:09:27.530 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:09:27 smithi089 bash[75068]: audit 1685822966.9733012 mon.a (mon.0) 919 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:09:27.530 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:09:27 smithi089 bash[78452]: audit 1685822966.3252234 mon.a (mon.0) 915 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:09:27.530 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:09:27 smithi089 bash[78452]: audit 1685822966.3380282 mon.a (mon.0) 916 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:09:27.530 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:09:27 smithi089 bash[78452]: audit 1685822966.3773959 mon.a (mon.0) 917 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:09:27.531 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:09:27 smithi089 bash[78452]: audit 1685822966.3908374 mon.a (mon.0) 918 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:09:27.531 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:09:27 smithi089 bash[78452]: cluster 1685822966.626048 mgr.y (mgr.34104) 999 : cluster 0 pgmap v951: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 2.6 GiB data, 9.5 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 619 op/s; 12922/125754 objects degraded (10.276%); 1664/125754 objects misplaced (1.323%); 1.3 MiB/s, 20 objects/s recovering 2023-06-03T20:09:27.531 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:09:27 smithi089 bash[78452]: audit 1685822966.9654932 mon.c (mon.2) 472 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-03T20:09:27.531 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:09:27 smithi089 bash[78452]: audit 1685822966.966728 mon.c (mon.2) 473 : audit 1 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-03T20:09:27.531 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:09:27 smithi089 bash[78452]: audit 1685822966.9733012 mon.a (mon.0) 919 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:09:27.549 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:09:27.551 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:09:27.552 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:09:27.583 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:09:27 smithi164 bash[59462]: audit 1685822966.3252234 mon.a (mon.0) 915 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:09:27.584 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:09:27 smithi164 bash[59462]: audit 1685822966.3380282 mon.a (mon.0) 916 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:09:27.584 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:09:27 smithi164 bash[59462]: audit 1685822966.3773959 mon.a (mon.0) 917 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:09:27.584 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:09:27 smithi164 bash[59462]: audit 1685822966.3908374 mon.a (mon.0) 918 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:09:27.585 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:09:27 smithi164 bash[59462]: cluster 1685822966.626048 mgr.y (mgr.34104) 999 : cluster 0 pgmap v951: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 2.6 GiB data, 9.5 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 619 op/s; 12922/125754 objects degraded (10.276%); 1664/125754 objects misplaced (1.323%); 1.3 MiB/s, 20 objects/s recovering 2023-06-03T20:09:27.585 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:09:27 smithi164 bash[59462]: audit 1685822966.9654932 mon.c (mon.2) 472 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-03T20:09:27.585 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:09:27 smithi164 bash[59462]: audit 1685822966.966728 mon.c (mon.2) 473 : audit 1 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-03T20:09:27.585 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:09:27 smithi164 bash[59462]: audit 1685822966.9733012 mon.a (mon.0) 919 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:09:27.653 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:09:27.694 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:09:27.697 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:09:27.698 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:09:27.799 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:09:27.835 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:09:27.841 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:09:27.842 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:09:27.943 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:09:27.986 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:09:27.990 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:09:27.991 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:09:28.092 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:09:28.127 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:09:28.129 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:09:28.130 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:09:28.231 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:09:28.254 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:09:28.255 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:09:28.256 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:09:28.357 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:09:28.383 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:09:28.386 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:09:28.387 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:09:28.397 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:09:28 smithi164 bash[59462]: cluster 1685822967.6775336 mon.a (mon.0) 920 : cluster 3 Health check update: Degraded data redundancy: 12922/125754 objects degraded (10.276%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-06-03T20:09:28.488 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:09:28.522 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:09:28.526 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:09:28.526 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:09:28 smithi089 bash[75068]: cluster 1685822967.6775336 mon.a (mon.0) 920 : cluster 3 Health check update: Degraded data redundancy: 12922/125754 objects degraded (10.276%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-06-03T20:09:28.527 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:09:28 smithi089 bash[78452]: cluster 1685822967.6775336 mon.a (mon.0) 920 : cluster 3 Health check update: Degraded data redundancy: 12922/125754 objects degraded (10.276%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-06-03T20:09:28.527 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:09:28.628 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:09:28.657 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:09:28.660 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:09:28.660 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:09:28.761 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:09:28.797 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:09:28.802 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:09:28.803 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:09:28.904 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:09:28.949 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:09:28.951 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:09:28.952 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:09:29.054 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:09:29.103 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:09:29.105 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:09:29.106 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:09:29.207 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:09:29.243 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:09:29.247 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:09:29.248 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:09:29.349 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:09:29.389 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:09:29.393 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:09:29.393 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:09:29.494 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:09:29.528 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:09:29.535 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:09:29.536 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:09:29.584 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:09:29 smithi164 bash[59462]: cluster 1685822968.6269288 mgr.y (mgr.34104) 1000 : cluster 0 pgmap v952: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 2.5 GiB data, 9.4 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 574 op/s; 12922/124101 objects degraded (10.412%); 1664/124101 objects misplaced (1.341%); 1.1 MiB/s, 17 objects/s recovering 2023-06-03T20:09:29.636 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:09:29.671 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:09:29.676 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:09:29.677 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:09:29.731 INFO:teuthology.orchestra.run.smithi089.stdout:ERROR: (22) Invalid argument 2023-06-03T20:09:29.731 INFO:teuthology.orchestra.run.smithi089.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-06-03T20:09:29.777 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:09:29.779 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:09:29 smithi089 bash[78452]: cluster 1685822968.6269288 mgr.y (mgr.34104) 1000 : cluster 0 pgmap v952: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 2.5 GiB data, 9.4 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 574 op/s; 12922/124101 objects degraded (10.412%); 1664/124101 objects misplaced (1.341%); 1.1 MiB/s, 17 objects/s recovering 2023-06-03T20:09:29.779 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:09:29 smithi089 bash[75068]: cluster 1685822968.6269288 mgr.y (mgr.34104) 1000 : cluster 0 pgmap v952: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 2.5 GiB data, 9.4 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 574 op/s; 12922/124101 objects degraded (10.412%); 1664/124101 objects misplaced (1.341%); 1.1 MiB/s, 17 objects/s recovering 2023-06-03T20:09:29.808 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:09:29.815 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:09:29.816 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:09:29.917 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:09:29.959 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:09:29.961 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:09:29.962 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:09:30.062 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:09:30.102 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:09:30.108 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:09:30.109 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:09:30.210 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:09:30.249 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:09:30.302 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:09:30.303 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:09:30.404 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:09:30.431 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:09:30.434 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:09:30.436 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:09:30.537 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:09:30.571 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:09:30.573 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:09:30.574 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:09:30.675 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:09:30.740 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:09:30.745 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:09:30.747 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:09:30.848 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:09:30.889 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:09:30.891 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:09:30.892 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:09:30.993 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:09:31.020 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:09:31.025 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:09:31.026 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:09:31.127 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:09:31.160 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:09:31.162 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:09:31.163 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:09:31.264 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:09:31.300 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:09:31.302 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:09:31.303 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:09:31.405 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:09:31.445 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:09:31.448 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:09:31.449 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:09:31.550 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:09:31.592 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:09:31.597 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:09:31.598 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:09:31.690 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:09:31 smithi164 bash[59462]: cluster 1685822970.627624 mgr.y (mgr.34104) 1001 : cluster 0 pgmap v953: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 2.4 GiB data, 9.0 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 702 op/s; 13011/116757 objects degraded (11.144%); 1476/116757 objects misplaced (1.264%); 1.6 MiB/s, 25 objects/s recovering 2023-06-03T20:09:31.699 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:09:31.733 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:09:31.738 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:09:31.739 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:09:31.840 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:09:31.877 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:09:31.882 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:09:31.883 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:09:31.984 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:09:32.017 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:09:32.018 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:09:32.019 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:09:32.028 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:09:31 smithi089 bash[75068]: cluster 1685822970.627624 mgr.y (mgr.34104) 1001 : cluster 0 pgmap v953: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 2.4 GiB data, 9.0 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 702 op/s; 13011/116757 objects degraded (11.144%); 1476/116757 objects misplaced (1.264%); 1.6 MiB/s, 25 objects/s recovering 2023-06-03T20:09:32.029 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:09:31 smithi089 bash[78452]: cluster 1685822970.627624 mgr.y (mgr.34104) 1001 : cluster 0 pgmap v953: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 2.4 GiB data, 9.0 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 702 op/s; 13011/116757 objects degraded (11.144%); 1476/116757 objects misplaced (1.264%); 1.6 MiB/s, 25 objects/s recovering 2023-06-03T20:09:32.121 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:09:32.144 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:09:32.145 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:09:32.146 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:09:32.247 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:09:32.291 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:09:32.297 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:09:32.298 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:09:32.399 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:09:32.442 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:09:32.445 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:09:32.446 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:09:32.547 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:09:32.599 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:09:32.603 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:09:32.604 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:09:32.705 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:09:32.735 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:09:32.738 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:09:32.738 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:09:32.839 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:09:32.879 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:09:32.880 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:09:32.880 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:09:32.982 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:09:32.990 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:09:32 smithi089 bash[78452]: cluster 1685822972.678662 mon.a (mon.0) 921 : cluster 3 Health check update: Degraded data redundancy: 13011/116757 objects degraded (11.144%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-06-03T20:09:32.991 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:09:32 smithi089 bash[75068]: cluster 1685822972.678662 mon.a (mon.0) 921 : cluster 3 Health check update: Degraded data redundancy: 13011/116757 objects degraded (11.144%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-06-03T20:09:33.019 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:09:33.025 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:09:33.025 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:09:33.084 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:09:32 smithi164 bash[59462]: cluster 1685822972.678662 mon.a (mon.0) 921 : cluster 3 Health check update: Degraded data redundancy: 13011/116757 objects degraded (11.144%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-06-03T20:09:33.126 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:09:33.168 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:09:33.172 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:09:33.173 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:09:33.274 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:09:33.334 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:09:33.337 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:09:33.339 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:09:33.439 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:09:33.464 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:09:33.469 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:09:33.470 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:09:33.571 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:09:33.603 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:09:33.606 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:09:33.607 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:09:33.703 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:09:33 smithi089 bash[75068]: cluster 1685822972.6282103 mgr.y (mgr.34104) 1002 : cluster 0 pgmap v954: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 2.3 GiB data, 8.9 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 555 op/s; 13011/114891 objects degraded (11.325%); 1476/114891 objects misplaced (1.285%); 1.1 MiB/s, 16 objects/s recovering 2023-06-03T20:09:33.708 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:09:33.760 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:09:33.768 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:09:33.769 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:09:33.870 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:09:33.893 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph daemon osd.2 dump_historic_ops 2023-06-03T20:09:33.918 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:09:33.920 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:09:33.920 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:09:34.021 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:09:34.030 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:09:33 smithi089 bash[78452]: cluster 1685822972.6282103 mgr.y (mgr.34104) 1002 : cluster 0 pgmap v954: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 2.3 GiB data, 8.9 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 555 op/s; 13011/114891 objects degraded (11.325%); 1476/114891 objects misplaced (1.285%); 1.1 MiB/s, 16 objects/s recovering 2023-06-03T20:09:34.072 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:09:34.076 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:09:34.078 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:09:34.083 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:09:33 smithi164 bash[59462]: cluster 1685822972.6282103 mgr.y (mgr.34104) 1002 : cluster 0 pgmap v954: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 2.3 GiB data, 8.9 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 555 op/s; 13011/114891 objects degraded (11.325%); 1476/114891 objects misplaced (1.285%); 1.1 MiB/s, 16 objects/s recovering 2023-06-03T20:09:34.178 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:09:34.219 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:09:34.223 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:09:34.224 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:09:34.325 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:09:34.372 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:09:34.382 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:09:34.383 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:09:34.484 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:09:34.519 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:09:34.522 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:09:34.523 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:09:34.624 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:09:34.647 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:09:34.649 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:09:34.650 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:09:34.751 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:09:34.783 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:09:34.788 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:09:34.789 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:09:34.890 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:09:34.921 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:09:34.927 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:09:34.928 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:09:35.029 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:09:35.064 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:09:35.067 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:09:35.068 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:09:35.169 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:09:35.205 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:09:35.212 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:09:35.213 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:09:35.315 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:09:35.368 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:09:35.373 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:09:35.374 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:09:35.475 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:09:35.513 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:09:35.519 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:09:35.520 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:09:35.621 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:09:35.650 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:09:35.652 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:09:35.653 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:09:35.754 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:09:35.800 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:09:35.804 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:09:35.805 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:09:35.906 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:09:35.941 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:09:35.945 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:09:35.946 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:09:36.028 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:09:35 smithi089 bash[78452]: cluster 1685822974.6292512 mgr.y (mgr.34104) 1003 : cluster 0 pgmap v955: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 2.2 GiB data, 8.7 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 660 op/s; 13126/109329 objects degraded (12.006%); 1253/109329 objects misplaced (1.146%); 1.6 MiB/s, 25 objects/s recovering 2023-06-03T20:09:36.028 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:09:35 smithi089 bash[75068]: cluster 1685822974.6292512 mgr.y (mgr.34104) 1003 : cluster 0 pgmap v955: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 2.2 GiB data, 8.7 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 660 op/s; 13126/109329 objects degraded (12.006%); 1253/109329 objects misplaced (1.146%); 1.6 MiB/s, 25 objects/s recovering 2023-06-03T20:09:36.047 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:09:36.057 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:09:35 smithi164 bash[59462]: cluster 1685822974.6292512 mgr.y (mgr.34104) 1003 : cluster 0 pgmap v955: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 2.2 GiB data, 8.7 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 660 op/s; 13126/109329 objects degraded (12.006%); 1253/109329 objects misplaced (1.146%); 1.6 MiB/s, 25 objects/s recovering 2023-06-03T20:09:36.084 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:09:36.089 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:09:36.090 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:09:36.190 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:09:36.242 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:09:36.246 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:09:36.247 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:09:36.349 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:09:36.359 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 20:09:36 smithi089 bash[71163]: ::ffff:172.21.15.164 - - [03/Jun/2023:20:09:36] "GET /metrics HTTP/1.1" 200 228768 "" "Prometheus/2.33.4" 2023-06-03T20:09:36.387 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:09:36.390 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:09:36.391 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:09:36.492 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:09:36.532 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:09:36.535 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:09:36.536 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:09:36.637 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:09:36.681 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:09:36.684 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:09:36.685 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:09:36.742 INFO:teuthology.orchestra.run.smithi089.stderr:reweighted osd.3 to 1 (10000) 2023-06-03T20:09:36.786 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:09:36.796 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:09:36 smithi164 bash[59462]: audit 1685822975.9507132 mon.c (mon.2) 474 : audit 1 from='client.? 172.21.15.89:0/3139882055' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-06-03T20:09:36.796 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:09:36 smithi164 bash[59462]: audit 1685822975.9513133 mon.a (mon.0) 922 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-06-03T20:09:36.820 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:09:36.824 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:09:36.825 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:09:36.926 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:09:36.958 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:09:36.961 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:09:36.962 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:09:36.999 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:09:36 smithi089 bash[75068]: audit 1685822975.9507132 mon.c (mon.2) 474 : audit 1 from='client.? 172.21.15.89:0/3139882055' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-06-03T20:09:36.999 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:09:36 smithi089 bash[75068]: audit 1685822975.9513133 mon.a (mon.0) 922 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-06-03T20:09:37.000 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:09:36 smithi089 bash[78452]: audit 1685822975.9507132 mon.c (mon.2) 474 : audit 1 from='client.? 172.21.15.89:0/3139882055' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-06-03T20:09:37.000 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:09:36 smithi089 bash[78452]: audit 1685822975.9513133 mon.a (mon.0) 922 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-06-03T20:09:37.064 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:09:37.096 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:09:37.103 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:09:37.104 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:09:37.206 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:09:37.247 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:09:37.249 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:09:37.250 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:09:37.350 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:09:37.383 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:09:37.389 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:09:37.390 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:09:37.490 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:09:37.538 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:09:37.540 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:09:37.540 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:09:37.641 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:09:37.677 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:09:37.681 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:09:37.682 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:09:37.735 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:09:37 smithi089 bash[75068]: cluster 1685822976.6299133 mgr.y (mgr.34104) 1004 : cluster 0 pgmap v956: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 2.2 GiB data, 8.5 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 582 op/s; 13126/106908 objects degraded (12.278%); 1253/106908 objects misplaced (1.172%); 1.1 MiB/s, 17 objects/s recovering 2023-06-03T20:09:37.735 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:09:37 smithi089 bash[75068]: cluster 1685822976.7191565 osd.6 (osd.6) 116 : cluster 3 failed to encode map e179 with expected crc 2023-06-03T20:09:37.735 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:09:37 smithi089 bash[75068]: cluster 1685822976.7205112 osd.5 (osd.5) 108 : cluster 3 failed to encode map e179 with expected crc 2023-06-03T20:09:37.735 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:09:37 smithi089 bash[75068]: audit 1685822976.7241812 mon.a (mon.0) 923 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2023-06-03T20:09:37.735 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:09:37 smithi089 bash[75068]: cluster 1685822976.7242453 mon.a (mon.0) 924 : cluster 0 osdmap e179: 8 total, 8 up, 7 in 2023-06-03T20:09:37.736 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:09:37 smithi089 bash[75068]: audit 1685822977.1124177 mon.c (mon.2) 475 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T20:09:37.783 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:09:37.827 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:09:37.828 INFO:teuthology.orchestra.run.smithi089.stdout:osd.0: {} 2023-06-03T20:09:37.829 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:09:37.830 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:09:37.865 INFO:teuthology.orchestra.run.smithi089.stdout:osd.1: {} 2023-06-03T20:09:37.891 INFO:teuthology.orchestra.run.smithi089.stdout:osd.2: {} 2023-06-03T20:09:37.931 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:09:37.933 INFO:teuthology.orchestra.run.smithi089.stdout:osd.3: {} 2023-06-03T20:09:37.962 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:09:37.964 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:09:37.965 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:09:37.974 INFO:teuthology.orchestra.run.smithi089.stdout:osd.4: {} 2023-06-03T20:09:38.003 INFO:teuthology.orchestra.run.smithi089.stdout:osd.5: {} 2023-06-03T20:09:38.037 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:09:37 smithi089 bash[78452]: cluster 1685822976.6299133 mgr.y (mgr.34104) 1004 : cluster 0 pgmap v956: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 2.2 GiB data, 8.5 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 582 op/s; 13126/106908 objects degraded (12.278%); 1253/106908 objects misplaced (1.172%); 1.1 MiB/s, 17 objects/s recovering 2023-06-03T20:09:38.038 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:09:37 smithi089 bash[78452]: cluster 1685822976.7191565 osd.6 (osd.6) 116 : cluster 3 failed to encode map e179 with expected crc 2023-06-03T20:09:38.038 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:09:37 smithi089 bash[78452]: cluster 1685822976.7205112 osd.5 (osd.5) 108 : cluster 3 failed to encode map e179 with expected crc 2023-06-03T20:09:38.038 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:09:37 smithi089 bash[78452]: audit 1685822976.7241812 mon.a (mon.0) 923 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2023-06-03T20:09:38.038 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:09:37 smithi089 bash[78452]: cluster 1685822976.7242453 mon.a (mon.0) 924 : cluster 0 osdmap e179: 8 total, 8 up, 7 in 2023-06-03T20:09:38.039 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:09:37 smithi089 bash[78452]: audit 1685822977.1124177 mon.c (mon.2) 475 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T20:09:38.039 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:09:37 smithi089 bash[78452]: cluster 1685822977.6800718 mon.a (mon.0) 925 : cluster 3 Health check update: Degraded data redundancy: 13126/106908 objects degraded (12.278%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-06-03T20:09:38.039 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:09:37 smithi089 bash[75068]: cluster 1685822977.6800718 mon.a (mon.0) 925 : cluster 3 Health check update: Degraded data redundancy: 13126/106908 objects degraded (12.278%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-06-03T20:09:38.048 INFO:teuthology.orchestra.run.smithi089.stdout:osd.6: {} 2023-06-03T20:09:38.066 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:09:38.076 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:09:37 smithi164 bash[59462]: cluster 1685822976.6299133 mgr.y (mgr.34104) 1004 : cluster 0 pgmap v956: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 2.2 GiB data, 8.5 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 582 op/s; 13126/106908 objects degraded (12.278%); 1253/106908 objects misplaced (1.172%); 1.1 MiB/s, 17 objects/s recovering 2023-06-03T20:09:38.077 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:09:37 smithi164 bash[59462]: cluster 1685822976.7191565 osd.6 (osd.6) 116 : cluster 3 failed to encode map e179 with expected crc 2023-06-03T20:09:38.077 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:09:37 smithi164 bash[59462]: cluster 1685822976.7205112 osd.5 (osd.5) 108 : cluster 3 failed to encode map e179 with expected crc 2023-06-03T20:09:38.077 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:09:37 smithi164 bash[59462]: audit 1685822976.7241812 mon.a (mon.0) 923 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2023-06-03T20:09:38.077 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:09:37 smithi164 bash[59462]: cluster 1685822976.7242453 mon.a (mon.0) 924 : cluster 0 osdmap e179: 8 total, 8 up, 7 in 2023-06-03T20:09:38.077 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:09:37 smithi164 bash[59462]: audit 1685822977.1124177 mon.c (mon.2) 475 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T20:09:38.078 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:09:37 smithi164 bash[59462]: cluster 1685822977.6800718 mon.a (mon.0) 925 : cluster 3 Health check update: Degraded data redundancy: 13126/106908 objects degraded (12.278%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-06-03T20:09:38.083 INFO:teuthology.orchestra.run.smithi089.stdout:osd.7: {} 2023-06-03T20:09:38.095 INFO:teuthology.orchestra.run.smithi089.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-06-03T20:09:38.096 INFO:teuthology.orchestra.run.smithi089.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-06-03T20:09:38.096 INFO:teuthology.orchestra.run.smithi089.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-06-03T20:09:38.096 INFO:teuthology.orchestra.run.smithi089.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-06-03T20:09:38.097 INFO:teuthology.orchestra.run.smithi089.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-06-03T20:09:38.098 INFO:teuthology.orchestra.run.smithi089.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-06-03T20:09:38.098 INFO:teuthology.orchestra.run.smithi089.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-06-03T20:09:38.098 INFO:teuthology.orchestra.run.smithi089.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-06-03T20:09:38.118 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:09:38.122 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:09:38.123 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:09:38.224 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:09:38.268 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:09:38.273 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:09:38.274 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:09:38.375 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:09:38.415 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:09:38.420 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:09:38.420 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:09:38.521 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:09:38.557 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:09:38.561 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:09:38.561 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:09:38.662 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:09:38.692 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:09:38.694 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:09:38.694 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:09:38.757 INFO:teuthology.orchestra.run.smithi089.stderr:nodeep-scrub is unset 2023-06-03T20:09:38.763 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:09:38 smithi089 bash[75068]: audit 1685822977.7704175 mon.c (mon.2) 476 : audit 0 from='client.? 172.21.15.89:0/3786930929' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-03T20:09:38.763 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:09:38 smithi089 bash[75068]: audit 1685822977.822573 mon.c (mon.2) 477 : audit 1 from='client.? 172.21.15.89:0/3455901005' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-06-03T20:09:38.763 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:09:38 smithi089 bash[75068]: audit 1685822977.8238952 mon.a (mon.0) 926 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-06-03T20:09:38.763 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:09:38 smithi089 bash[78452]: audit 1685822977.7704175 mon.c (mon.2) 476 : audit 0 from='client.? 172.21.15.89:0/3786930929' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-03T20:09:38.764 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:09:38 smithi089 bash[78452]: audit 1685822977.822573 mon.c (mon.2) 477 : audit 1 from='client.? 172.21.15.89:0/3455901005' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-06-03T20:09:38.764 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:09:38 smithi089 bash[78452]: audit 1685822977.8238952 mon.a (mon.0) 926 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-06-03T20:09:38.795 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:09:38.827 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:09:38.830 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:09:38.831 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:09:38.834 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:09:38 smithi164 bash[59462]: audit 1685822977.7704175 mon.c (mon.2) 476 : audit 0 from='client.? 172.21.15.89:0/3786930929' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-03T20:09:38.834 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:09:38 smithi164 bash[59462]: audit 1685822977.822573 mon.c (mon.2) 477 : audit 1 from='client.? 172.21.15.89:0/3455901005' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-06-03T20:09:38.834 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:09:38 smithi164 bash[59462]: audit 1685822977.8238952 mon.a (mon.0) 926 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-06-03T20:09:38.843 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph osd reweight 4 1 2023-06-03T20:09:38.933 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:09:38.972 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:09:38.974 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:09:38.975 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:09:39.076 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:09:39.114 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:09:39.117 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:09:39.118 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:09:39.219 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:09:39.255 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:09:39.258 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:09:39.259 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:09:39.360 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:09:39.386 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:09:39.393 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:09:39.393 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:09:39.494 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:09:39.540 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:09:39.542 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:09:39.543 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:09:39.644 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:09:39.685 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:09:39.685 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:09:39.686 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:09:39.786 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:09:39.795 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:09:39 smithi164 bash[59462]: cluster 1685822978.630586 mgr.y (mgr.34104) 1005 : cluster 0 pgmap v958: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 2.1 GiB data, 8.4 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 635 op/s; 13126/105015 objects degraded (12.499%); 1253/105015 objects misplaced (1.193%); 1.3 MiB/s, 20 objects/s recovering 2023-06-03T20:09:39.795 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:09:39 smithi164 bash[59462]: cluster 1685822978.730512 mon.a (mon.0) 927 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-06-03T20:09:39.796 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:09:39 smithi164 bash[59462]: cluster 1685822978.7382228 osd.6 (osd.6) 117 : cluster 3 failed to encode map e180 with expected crc 2023-06-03T20:09:39.796 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:09:39 smithi164 bash[59462]: cluster 1685822978.7420573 osd.6 (osd.6) 118 : cluster 3 failed to encode map e180 with expected crc 2023-06-03T20:09:39.796 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:09:39 smithi164 bash[59462]: audit 1685822978.74343 mon.a (mon.0) 928 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-06-03T20:09:39.796 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:09:39 smithi164 bash[59462]: cluster 1685822978.7435336 mon.a (mon.0) 929 : cluster 0 osdmap e180: 8 total, 8 up, 7 in 2023-06-03T20:09:39.796 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:09:39 smithi164 bash[59462]: cluster 1685822978.7474856 osd.7 (osd.7) 116 : cluster 3 failed to encode map e180 with expected crc 2023-06-03T20:09:39.796 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:09:39 smithi164 bash[59462]: cluster 1685822978.7513473 osd.5 (osd.5) 109 : cluster 3 failed to encode map e180 with expected crc 2023-06-03T20:09:39.796 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:09:39 smithi164 bash[59462]: cluster 1685822978.75319 osd.5 (osd.5) 110 : cluster 3 failed to encode map e180 with expected crc 2023-06-03T20:09:39.797 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:09:39 smithi164 bash[59462]: cluster 1685822978.7561347 osd.5 (osd.5) 111 : cluster 3 failed to encode map e180 with expected crc 2023-06-03T20:09:39.797 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:09:39 smithi164 bash[59462]: cluster 1685822978.7579713 osd.5 (osd.5) 112 : cluster 3 failed to encode map e180 with expected crc 2023-06-03T20:09:39.836 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:09:39.841 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:09:39.842 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:09:39.935 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:09:39 smithi089 bash[78452]: cluster 1685822978.630586 mgr.y (mgr.34104) 1005 : cluster 0 pgmap v958: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 2.1 GiB data, 8.4 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 635 op/s; 13126/105015 objects degraded (12.499%); 1253/105015 objects misplaced (1.193%); 1.3 MiB/s, 20 objects/s recovering 2023-06-03T20:09:39.936 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:09:39 smithi089 bash[78452]: cluster 1685822978.730512 mon.a (mon.0) 927 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-06-03T20:09:39.936 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:09:39 smithi089 bash[78452]: cluster 1685822978.7382228 osd.6 (osd.6) 117 : cluster 3 failed to encode map e180 with expected crc 2023-06-03T20:09:39.936 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:09:39 smithi089 bash[78452]: cluster 1685822978.7420573 osd.6 (osd.6) 118 : cluster 3 failed to encode map e180 with expected crc 2023-06-03T20:09:39.936 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:09:39 smithi089 bash[78452]: audit 1685822978.74343 mon.a (mon.0) 928 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-06-03T20:09:39.936 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:09:39 smithi089 bash[78452]: cluster 1685822978.7435336 mon.a (mon.0) 929 : cluster 0 osdmap e180: 8 total, 8 up, 7 in 2023-06-03T20:09:39.937 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:09:39 smithi089 bash[78452]: cluster 1685822978.7474856 osd.7 (osd.7) 116 : cluster 3 failed to encode map e180 with expected crc 2023-06-03T20:09:39.937 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:09:39 smithi089 bash[78452]: cluster 1685822978.7513473 osd.5 (osd.5) 109 : cluster 3 failed to encode map e180 with expected crc 2023-06-03T20:09:39.937 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:09:39 smithi089 bash[78452]: cluster 1685822978.75319 osd.5 (osd.5) 110 : cluster 3 failed to encode map e180 with expected crc 2023-06-03T20:09:39.937 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:09:39 smithi089 bash[78452]: cluster 1685822978.7561347 osd.5 (osd.5) 111 : cluster 3 failed to encode map e180 with expected crc 2023-06-03T20:09:39.944 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:09:39.985 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:09:39.990 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:09:39.990 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:09:40.091 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:09:40.124 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:09:40.129 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:09:40.130 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:09:40.231 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:09:40.240 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:09:39 smithi089 bash[75068]: cluster 1685822978.630586 mgr.y (mgr.34104) 1005 : cluster 0 pgmap v958: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 2.1 GiB data, 8.4 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 635 op/s; 13126/105015 objects degraded (12.499%); 1253/105015 objects misplaced (1.193%); 1.3 MiB/s, 20 objects/s recovering 2023-06-03T20:09:40.240 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:09:39 smithi089 bash[75068]: cluster 1685822978.730512 mon.a (mon.0) 927 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-06-03T20:09:40.241 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:09:39 smithi089 bash[75068]: cluster 1685822978.7382228 osd.6 (osd.6) 117 : cluster 3 failed to encode map e180 with expected crc 2023-06-03T20:09:40.241 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:09:39 smithi089 bash[75068]: cluster 1685822978.7420573 osd.6 (osd.6) 118 : cluster 3 failed to encode map e180 with expected crc 2023-06-03T20:09:40.241 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:09:39 smithi089 bash[75068]: audit 1685822978.74343 mon.a (mon.0) 928 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-06-03T20:09:40.241 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:09:39 smithi089 bash[75068]: cluster 1685822978.7435336 mon.a (mon.0) 929 : cluster 0 osdmap e180: 8 total, 8 up, 7 in 2023-06-03T20:09:40.241 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:09:39 smithi089 bash[75068]: cluster 1685822978.7474856 osd.7 (osd.7) 116 : cluster 3 failed to encode map e180 with expected crc 2023-06-03T20:09:40.241 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:09:39 smithi089 bash[75068]: cluster 1685822978.7513473 osd.5 (osd.5) 109 : cluster 3 failed to encode map e180 with expected crc 2023-06-03T20:09:40.241 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:09:39 smithi089 bash[75068]: cluster 1685822978.75319 osd.5 (osd.5) 110 : cluster 3 failed to encode map e180 with expected crc 2023-06-03T20:09:40.242 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:09:39 smithi089 bash[75068]: cluster 1685822978.7561347 osd.5 (osd.5) 111 : cluster 3 failed to encode map e180 with expected crc 2023-06-03T20:09:40.242 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:09:39 smithi089 bash[75068]: cluster 1685822978.7579713 osd.5 (osd.5) 112 : cluster 3 failed to encode map e180 with expected crc 2023-06-03T20:09:40.242 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:09:39 smithi089 bash[78452]: cluster 1685822978.7579713 osd.5 (osd.5) 112 : cluster 3 failed to encode map e180 with expected crc 2023-06-03T20:09:40.267 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:09:40.270 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:09:40.271 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:09:40.372 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:09:40.405 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:09:40.409 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:09:40.410 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:09:40.511 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:09:40.540 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:09:40.545 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:09:40.546 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:09:40.647 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:09:40.705 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:09:40.711 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:09:40.712 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:09:40.813 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:09:40.857 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:09:40.861 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:09:40.862 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:09:40.964 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:09:41.007 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:09:41.010 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:09:41.011 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:09:41.112 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:09:41.147 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:09:41.148 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:09:41.148 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:09:41.250 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:09:41.287 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:09:41.292 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:09:41.293 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:09:41.395 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:09:41.437 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:09:41.441 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:09:41.443 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:09:41.544 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:09:41.577 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:09:41.581 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:09:41.582 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:09:41.683 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:09:41.712 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:09:41.716 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:09:41.717 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:09:41.818 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:09:41.876 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:09:41.881 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:09:41.881 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:09:41.982 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:09:42.015 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:09:42.020 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:09:42.021 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:09:42.021 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:09:41 smithi164 bash[59462]: cluster 1685822980.6313932 mgr.y (mgr.34104) 1006 : cluster 0 pgmap v960: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 2.0 GiB data, 8.0 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 734 op/s; 13232/97257 objects degraded (13.605%); 1042/97257 objects misplaced (1.071%); 1.7 MiB/s, 26 objects/s recovering 2023-06-03T20:09:42.028 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:09:41 smithi089 bash[75068]: cluster 1685822980.6313932 mgr.y (mgr.34104) 1006 : cluster 0 pgmap v960: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 2.0 GiB data, 8.0 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 734 op/s; 13232/97257 objects degraded (13.605%); 1042/97257 objects misplaced (1.071%); 1.7 MiB/s, 26 objects/s recovering 2023-06-03T20:09:42.028 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:09:41 smithi089 bash[78452]: cluster 1685822980.6313932 mgr.y (mgr.34104) 1006 : cluster 0 pgmap v960: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 2.0 GiB data, 8.0 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 734 op/s; 13232/97257 objects degraded (13.605%); 1042/97257 objects misplaced (1.071%); 1.7 MiB/s, 26 objects/s recovering 2023-06-03T20:09:42.122 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:09:42.153 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:09:42.163 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:09:42.164 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:09:42.265 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:09:42.295 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:09:42.298 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:09:42.299 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:09:42.400 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:09:42.437 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:09:42.441 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:09:42.442 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:09:42.543 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:09:42.590 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:09:42.593 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:09:42.594 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:09:42.695 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:09:42.735 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:09:42.737 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:09:42.738 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:09:42.772 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-06-03T20:09:42.839 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:09:42.849 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:09:42 smithi089 bash[78452]: cluster 1685822982.6809185 mon.a (mon.0) 930 : cluster 3 Health check update: Degraded data redundancy: 13232/97257 objects degraded (13.605%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-06-03T20:09:42.850 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:09:42 smithi089 bash[75068]: cluster 1685822982.6809185 mon.a (mon.0) 930 : cluster 3 Health check update: Degraded data redundancy: 13232/97257 objects degraded (13.605%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-06-03T20:09:42.873 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:09:42.876 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:09:42.877 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:09:42.901 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph osd set noscrub 2023-06-03T20:09:42.979 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:09:42.988 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:09:42 smithi164 bash[59462]: cluster 1685822982.6809185 mon.a (mon.0) 930 : cluster 3 Health check update: Degraded data redundancy: 13232/97257 objects degraded (13.605%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-06-03T20:09:43.022 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:09:43.025 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:09:43.027 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:09:43.128 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:09:43.188 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:09:43.195 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:09:43.196 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:09:43.298 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:09:43.328 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:09:43.332 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:09:43.333 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:09:43.434 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:09:43.443 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/mon.a/config 2023-06-03T20:09:43.464 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:09:43.470 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:09:43.471 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:09:43.572 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:09:43.607 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:09:43.609 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:09:43.610 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:09:43.711 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:09:43.758 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:09:43.759 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:09:43.760 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:09:43.860 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:09:43.897 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:09:43.901 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:09:43.902 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:09:44.003 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:09:44.011 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:09:43 smithi089 bash[78452]: cluster 1685822982.6321259 mgr.y (mgr.34104) 1007 : cluster 0 pgmap v961: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 1.9 GiB data, 7.9 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 576 op/s; 13232/95487 objects degraded (13.857%); 1042/95487 objects misplaced (1.091%); 840 KiB/s, 13 objects/s recovering 2023-06-03T20:09:44.011 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:09:43 smithi089 bash[75068]: cluster 1685822982.6321259 mgr.y (mgr.34104) 1007 : cluster 0 pgmap v961: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 1.9 GiB data, 7.9 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 576 op/s; 13232/95487 objects degraded (13.857%); 1042/95487 objects misplaced (1.091%); 840 KiB/s, 13 objects/s recovering 2023-06-03T20:09:44.034 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:09:44.036 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:09:44.037 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:09:44.083 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:09:43 smithi164 bash[59462]: cluster 1685822982.6321259 mgr.y (mgr.34104) 1007 : cluster 0 pgmap v961: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 1.9 GiB data, 7.9 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 576 op/s; 13232/95487 objects degraded (13.857%); 1042/95487 objects misplaced (1.091%); 840 KiB/s, 13 objects/s recovering 2023-06-03T20:09:44.138 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:09:44.177 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:09:44.183 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:09:44.184 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:09:44.285 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:09:44.325 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:09:44.329 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:09:44.330 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:09:44.431 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:09:44.463 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:09:44.468 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:09:44.469 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:09:44.472 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/mon.a/config 2023-06-03T20:09:44.570 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:09:44.600 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:09:44.608 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:09:44.609 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:09:44.710 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:09:44.764 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:09:44.771 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:09:44.772 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:09:44.873 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:09:44.908 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:09:44.916 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:09:44.917 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:09:45.018 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:09:45.051 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:09:45.054 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:09:45.055 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:09:45.156 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:09:45.183 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:09:45.190 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:09:45.190 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:09:45.291 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:09:45.324 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:09:45.326 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:09:45.327 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:09:45.428 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:09:45.465 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:09:45.468 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:09:45.469 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:09:45.570 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:09:45.603 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:09:45.606 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:09:45.606 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:09:45.707 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:09:45.744 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:09:45.746 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:09:45.747 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:09:45.833 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:09:45 smithi164 bash[59462]: cluster 1685822984.6330912 mgr.y (mgr.34104) 1008 : cluster 0 pgmap v962: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 1.8 GiB data, 7.6 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 703 op/s; 13335/90024 objects degraded (14.813%); 832/90024 objects misplaced (0.924%); 1.7 MiB/s, 26 objects/s recovering 2023-06-03T20:09:45.848 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:09:45.873 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:09:45.877 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:09:45.877 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:09:45.978 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:09:45.985 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:09:45 smithi089 bash[78452]: cluster 1685822984.6330912 mgr.y (mgr.34104) 1008 : cluster 0 pgmap v962: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 1.8 GiB data, 7.6 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 703 op/s; 13335/90024 objects degraded (14.813%); 832/90024 objects misplaced (0.924%); 1.7 MiB/s, 26 objects/s recovering 2023-06-03T20:09:45.986 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:09:45 smithi089 bash[75068]: cluster 1685822984.6330912 mgr.y (mgr.34104) 1008 : cluster 0 pgmap v962: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 1.8 GiB data, 7.6 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 703 op/s; 13335/90024 objects degraded (14.813%); 832/90024 objects misplaced (0.924%); 1.7 MiB/s, 26 objects/s recovering 2023-06-03T20:09:46.010 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:09:46.014 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:09:46.015 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:09:46.116 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:09:46.151 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:09:46.154 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:09:46.154 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:09:46.255 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:09:46.283 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:09:46.286 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:09:46.286 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:09:46.388 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:09:46.405 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 20:09:46 smithi089 bash[71163]: ::ffff:172.21.15.164 - - [03/Jun/2023:20:09:46] "GET /metrics HTTP/1.1" 200 228759 "" "Prometheus/2.33.4" 2023-06-03T20:09:46.433 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:09:46.437 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:09:46.438 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:09:46.540 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:09:46.578 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:09:46.581 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:09:46.582 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:09:46.683 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:09:46.715 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:09:46.717 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:09:46.718 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:09:46.819 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:09:46.845 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:09:46.852 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:09:46.853 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:09:46.954 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:09:46.981 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:09:46.985 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:09:46.986 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:09:47.087 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:09:47.118 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:09:47.124 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:09:47.125 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:09:47.226 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:09:47.257 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:09:47.263 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:09:47.264 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:09:47.365 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:09:47.393 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:09:47.395 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:09:47.396 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:09:47.497 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:09:47.528 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:09:47.533 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:09:47.533 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:09:47.558 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/mon.a/config 2023-06-03T20:09:47.559 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/mon.a/config 2023-06-03T20:09:47.635 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:09:47.669 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:09:47.672 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:09:47.674 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:09:47.775 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:09:47.783 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:09:47 smithi089 bash[75068]: cluster 1685822986.6336334 mgr.y (mgr.34104) 1009 : cluster 0 pgmap v963: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 1.8 GiB data, 7.4 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 648 op/s; 13335/87645 objects degraded (15.215%); 832/87645 objects misplaced (0.949%); 1.3 MiB/s, 21 objects/s recovering 2023-06-03T20:09:47.784 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:09:47 smithi089 bash[75068]: cluster 1685822987.682136 mon.a (mon.0) 931 : cluster 3 Health check update: Degraded data redundancy: 13335/87645 objects degraded (15.215%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-06-03T20:09:47.813 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:09:47.823 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:09:47.824 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:09:47.925 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:09:47.971 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:09:47.980 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:09:47.981 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:09:48.082 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:09:48.083 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:09:47 smithi164 bash[59462]: cluster 1685822986.6336334 mgr.y (mgr.34104) 1009 : cluster 0 pgmap v963: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 1.8 GiB data, 7.4 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 648 op/s; 13335/87645 objects degraded (15.215%); 832/87645 objects misplaced (0.949%); 1.3 MiB/s, 21 objects/s recovering 2023-06-03T20:09:48.083 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:09:47 smithi164 bash[59462]: cluster 1685822987.682136 mon.a (mon.0) 931 : cluster 3 Health check update: Degraded data redundancy: 13335/87645 objects degraded (15.215%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-06-03T20:09:48.131 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:09:48.134 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:09:48.135 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:09:48.236 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:09:48.273 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:09:48.278 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:09:47 smithi089 bash[78452]: cluster 1685822986.6336334 mgr.y (mgr.34104) 1009 : cluster 0 pgmap v963: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 1.8 GiB data, 7.4 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 648 op/s; 13335/87645 objects degraded (15.215%); 832/87645 objects misplaced (0.949%); 1.3 MiB/s, 21 objects/s recovering 2023-06-03T20:09:48.278 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:09:47 smithi089 bash[78452]: cluster 1685822987.682136 mon.a (mon.0) 931 : cluster 3 Health check update: Degraded data redundancy: 13335/87645 objects degraded (15.215%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-06-03T20:09:48.279 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:09:48.280 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:09:48.381 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:09:48.423 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:09:48.425 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:09:48.426 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:09:48.527 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:09:48.556 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:09:48.565 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:09:48.566 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:09:48.666 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:09:48.702 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:09:48.707 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:09:48.708 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:09:48.809 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:09:48.869 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:09:48.871 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:09:48.872 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:09:48.973 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:09:49.013 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:09:49.018 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:09:49.019 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:09:49.120 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:09:49.164 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:09:49.170 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:09:49.171 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:09:49.272 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:09:49.314 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:09:49.322 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:09:49.322 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:09:49.423 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:09:49.464 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:09:49.469 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:09:49.470 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:09:49.571 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:09:49.607 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:09:49.610 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:09:49.611 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:09:49.712 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:09:49.748 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:09:49.754 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:09:49.755 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:09:49.769 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:09:49 smithi164 bash[59462]: cluster 1685822988.6342154 mgr.y (mgr.34104) 1010 : cluster 0 pgmap v964: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 1.7 GiB data, 7.2 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 640 op/s; 13335/85785 objects degraded (15.545%); 832/85785 objects misplaced (0.970%); 1.3 MiB/s, 21 objects/s recovering 2023-06-03T20:09:49.855 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:09:49.887 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:09:49.891 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:09:49.891 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:09:49.993 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:09:50.030 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:09:49 smithi089 bash[78452]: cluster 1685822988.6342154 mgr.y (mgr.34104) 1010 : cluster 0 pgmap v964: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 1.7 GiB data, 7.2 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 640 op/s; 13335/85785 objects degraded (15.545%); 832/85785 objects misplaced (0.970%); 1.3 MiB/s, 21 objects/s recovering 2023-06-03T20:09:50.031 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:09:49 smithi089 bash[75068]: cluster 1685822988.6342154 mgr.y (mgr.34104) 1010 : cluster 0 pgmap v964: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 1.7 GiB data, 7.2 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 640 op/s; 13335/85785 objects degraded (15.545%); 832/85785 objects misplaced (0.970%); 1.3 MiB/s, 21 objects/s recovering 2023-06-03T20:09:50.052 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:09:50.060 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:09:50.061 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:09:50.162 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:09:50.194 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:09:50.197 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:09:50.197 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:09:50.298 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:09:50.336 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:09:50.342 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:09:50.343 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:09:50.444 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:09:50.481 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:09:50.486 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:09:50.487 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:09:50.588 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:09:50.634 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:09:50.637 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:09:50.638 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:09:50.739 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:09:50.773 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:09:50.777 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:09:50.778 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:09:50.880 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:09:50.914 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:09:50.918 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:09:50.919 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:09:51.020 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:09:51.052 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:09:51.057 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:09:51.058 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:09:51.159 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:09:51.208 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:09:51.212 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:09:51.213 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:09:51.313 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:09:51.363 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:09:51.365 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:09:51.366 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:09:51.466 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:09:51.504 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:09:51.510 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:09:51.511 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:09:51.612 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:09:51.644 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:09:51.649 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:09:51.649 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:09:51.750 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:09:51.778 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:09:51.779 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:09:51.780 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:09:51.882 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:09:51.889 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:09:51 smithi089 bash[75068]: cluster 1685822990.6350288 mgr.y (mgr.34104) 1011 : cluster 0 pgmap v965: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 1.6 GiB data, 6.9 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 714 op/s; 13439/77754 objects degraded (17.284%); 622/77754 objects misplaced (0.800%); 1.7 MiB/s, 26 objects/s recovering 2023-06-03T20:09:51.889 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:09:51 smithi089 bash[78452]: cluster 1685822990.6350288 mgr.y (mgr.34104) 1011 : cluster 0 pgmap v965: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 1.6 GiB data, 6.9 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 714 op/s; 13439/77754 objects degraded (17.284%); 622/77754 objects misplaced (0.800%); 1.7 MiB/s, 26 objects/s recovering 2023-06-03T20:09:51.915 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:09:51.917 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:09:51.918 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:09:52.019 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:09:52.052 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:09:52.057 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:09:52.058 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:09:52.083 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:09:51 smithi164 bash[59462]: cluster 1685822990.6350288 mgr.y (mgr.34104) 1011 : cluster 0 pgmap v965: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 1.6 GiB data, 6.9 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 714 op/s; 13439/77754 objects degraded (17.284%); 622/77754 objects misplaced (0.800%); 1.7 MiB/s, 26 objects/s recovering 2023-06-03T20:09:52.161 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:09:52.197 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:09:52.203 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:09:52.204 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:09:52.305 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:09:52.336 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:09:52.338 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:09:52.339 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:09:52.440 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:09:52.479 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:09:52.481 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:09:52.482 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:09:52.583 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:09:52.622 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:09:52.626 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:09:52.626 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:09:52.727 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:09:52.769 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:09:52.777 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:09:52.778 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:09:52.878 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:09:52.904 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:09:52.906 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:09:52.907 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:09:53.008 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:09:53.039 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:09:53.042 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:09:53.043 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:09:53.144 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:09:53.175 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:09:53.182 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:09:53.183 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:09:53.284 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:09:53.324 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:09:53.326 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:09:53.327 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:09:53.428 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:09:53.440 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:09:53 smithi164 bash[59462]: audit 1685822992.1176183 mon.a (mon.0) 932 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:09:53.440 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:09:53 smithi164 bash[59462]: audit 1685822992.1215913 mon.c (mon.2) 478 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T20:09:53.440 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:09:53 smithi164 bash[59462]: cluster 1685822992.635668 mgr.y (mgr.34104) 1012 : cluster 0 pgmap v966: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 1.5 GiB data, 6.8 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 592 op/s; 13439/75909 objects degraded (17.704%); 622/75909 objects misplaced (0.819%); 1.1 MiB/s, 17 objects/s recovering 2023-06-03T20:09:53.441 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:09:53 smithi164 bash[59462]: cluster 1685822992.68333 mon.a (mon.0) 933 : cluster 3 Health check update: Degraded data redundancy: 13439/77754 objects degraded (17.284%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-06-03T20:09:53.473 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:09:53.478 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:09:53.479 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:09:53.580 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:09:53.588 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:09:53 smithi089 bash[75068]: audit 1685822992.1176183 mon.a (mon.0) 932 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:09:53.588 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:09:53 smithi089 bash[75068]: audit 1685822992.1215913 mon.c (mon.2) 478 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T20:09:53.589 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:09:53 smithi089 bash[75068]: cluster 1685822992.635668 mgr.y (mgr.34104) 1012 : cluster 0 pgmap v966: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 1.5 GiB data, 6.8 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 592 op/s; 13439/75909 objects degraded (17.704%); 622/75909 objects misplaced (0.819%); 1.1 MiB/s, 17 objects/s recovering 2023-06-03T20:09:53.589 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:09:53 smithi089 bash[75068]: cluster 1685822992.68333 mon.a (mon.0) 933 : cluster 3 Health check update: Degraded data redundancy: 13439/77754 objects degraded (17.284%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-06-03T20:09:53.589 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:09:53 smithi089 bash[78452]: audit 1685822992.1176183 mon.a (mon.0) 932 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:09:53.589 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:09:53 smithi089 bash[78452]: audit 1685822992.1215913 mon.c (mon.2) 478 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T20:09:53.590 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:09:53 smithi089 bash[78452]: cluster 1685822992.635668 mgr.y (mgr.34104) 1012 : cluster 0 pgmap v966: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 1.5 GiB data, 6.8 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 592 op/s; 13439/75909 objects degraded (17.704%); 622/75909 objects misplaced (0.819%); 1.1 MiB/s, 17 objects/s recovering 2023-06-03T20:09:53.590 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:09:53 smithi089 bash[78452]: cluster 1685822992.68333 mon.a (mon.0) 933 : cluster 3 Health check update: Degraded data redundancy: 13439/77754 objects degraded (17.284%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-06-03T20:09:53.612 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:09:53.613 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:09:53.614 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:09:53.715 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:09:53.745 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:09:53.747 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:09:53.747 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:09:53.848 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:09:53.884 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:09:53.888 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:09:53.889 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:09:53.989 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:09:54.029 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:09:54.034 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:09:54.035 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:09:54.136 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:09:54.176 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:09:54.181 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:09:54.182 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:09:54.283 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:09:54.322 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:09:54.324 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:09:54.325 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:09:54.427 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:09:54.467 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:09:54.471 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:09:54.472 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:09:54.573 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:09:54.600 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:09:54.603 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:09:54.604 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:09:54.705 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:09:54.743 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:09:54.750 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:09:54.750 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:09:54.851 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:09:54.879 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:09:54.881 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:09:54.882 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:09:54.983 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:09:55.022 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:09:55.028 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:09:55.028 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:09:55.126 INFO:teuthology.orchestra.run.smithi089.stdout:{ 2023-06-03T20:09:55.126 INFO:teuthology.orchestra.run.smithi089.stdout: "size": 20, 2023-06-03T20:09:55.126 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 600, 2023-06-03T20:09:55.127 INFO:teuthology.orchestra.run.smithi089.stdout: "ops": [ 2023-06-03T20:09:55.127 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:09:55.127 INFO:teuthology.orchestra.run.smithi089.stdout: "description": "osd_op(client.36009.0:30294 10.5 10:abafa394:::benchmark_data_smithi089_140876_object30293:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e163)", 2023-06-03T20:09:55.127 INFO:teuthology.orchestra.run.smithi089.stdout: "initiated_at": "2023-06-03T20:06:28.539210+0000", 2023-06-03T20:09:55.127 INFO:teuthology.orchestra.run.smithi089.stdout: "age": 206.58046320599999, 2023-06-03T20:09:55.127 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.145338092, 2023-06-03T20:09:55.127 INFO:teuthology.orchestra.run.smithi089.stdout: "type_data": { 2023-06-03T20:09:55.127 INFO:teuthology.orchestra.run.smithi089.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-03T20:09:55.128 INFO:teuthology.orchestra.run.smithi089.stdout: "client_info": { 2023-06-03T20:09:55.128 INFO:teuthology.orchestra.run.smithi089.stdout: "client": "client.36009", 2023-06-03T20:09:55.128 INFO:teuthology.orchestra.run.smithi089.stdout: "client_addr": "172.21.15.89:0/2550116169", 2023-06-03T20:09:55.128 INFO:teuthology.orchestra.run.smithi089.stdout: "tid": 30294 2023-06-03T20:09:55.128 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:09:55.129 INFO:teuthology.orchestra.run.smithi089.stdout: "events": [ 2023-06-03T20:09:55.129 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:09:55.130 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "initiated", 2023-06-03T20:09:55.130 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:06:28.539210+0000", 2023-06-03T20:09:55.130 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0 2023-06-03T20:09:55.130 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:09:55.130 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:09:55.130 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "throttled", 2023-06-03T20:09:55.130 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:06:28.539210+0000", 2023-06-03T20:09:55.130 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0 2023-06-03T20:09:55.131 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:09:55.131 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:09:55.131 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "header_read", 2023-06-03T20:09:55.131 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:06:28.539209+0000", 2023-06-03T20:09:55.131 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 4294967295.999999 2023-06-03T20:09:55.132 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:09:55.132 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:09:55.132 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "all_read", 2023-06-03T20:09:55.132 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:06:28.539236+0000", 2023-06-03T20:09:55.132 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 2.6644000000000001e-05 2023-06-03T20:09:55.132 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:09:55.133 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:09:55.133 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "dispatched", 2023-06-03T20:09:55.133 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:06:28.539247+0000", 2023-06-03T20:09:55.133 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 1.0788000000000001e-05 2023-06-03T20:09:55.133 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:09:55.133 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:09:55.133 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "queued_for_pg", 2023-06-03T20:09:55.133 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:06:28.539253+0000", 2023-06-03T20:09:55.134 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 6.1090000000000003e-06 2023-06-03T20:09:55.134 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:09:55.134 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:09:55.134 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "reached_pg", 2023-06-03T20:09:55.134 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:06:28.539287+0000", 2023-06-03T20:09:55.134 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 3.4073999999999999e-05 2023-06-03T20:09:55.134 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:09:55.135 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:09:55.135 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "started", 2023-06-03T20:09:55.135 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:06:28.539375+0000", 2023-06-03T20:09:55.135 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 8.8499999999999996e-05 2023-06-03T20:09:55.135 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:09:55.136 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:09:55.136 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "waiting for subops from 0,3", 2023-06-03T20:09:55.136 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:06:28.539476+0000", 2023-06-03T20:09:55.138 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.00010022399999999999 2023-06-03T20:09:55.138 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:09:55.138 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:09:55.139 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "op_commit", 2023-06-03T20:09:55.140 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:06:28.540560+0000", 2023-06-03T20:09:55.140 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.0010840680000000001 2023-06-03T20:09:55.140 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:09:55.141 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:09:55.141 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:09:55.141 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:06:28.671942+0000", 2023-06-03T20:09:55.142 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.131381998 2023-06-03T20:09:55.142 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:09:55.142 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:09:55.143 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:09:55.143 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:06:28.684421+0000", 2023-06-03T20:09:55.143 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.012478835000000001 2023-06-03T20:09:55.144 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:09:55.144 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:09:55.144 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "commit_sent", 2023-06-03T20:09:55.145 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:06:28.684491+0000", 2023-06-03T20:09:55.145 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 7.0677000000000001e-05 2023-06-03T20:09:55.145 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:09:55.146 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:09:55.146 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "done", 2023-06-03T20:09:55.146 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:06:28.684549+0000", 2023-06-03T20:09:55.146 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 5.7355999999999998e-05 2023-06-03T20:09:55.147 INFO:teuthology.orchestra.run.smithi089.stdout: } 2023-06-03T20:09:55.147 INFO:teuthology.orchestra.run.smithi089.stdout: ] 2023-06-03T20:09:55.148 INFO:teuthology.orchestra.run.smithi089.stdout: } 2023-06-03T20:09:55.148 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:09:55.148 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:09:55.149 INFO:teuthology.orchestra.run.smithi089.stdout: "description": "osd_op(client.36009.0:37599 10.5 10:a374dead:::benchmark_data_smithi089_140876_object37598:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e163)", 2023-06-03T20:09:55.149 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:09:55.152 INFO:teuthology.orchestra.run.smithi089.stdout: "initiated_at": "2023-06-03T20:06:33.238661+0000", 2023-06-03T20:09:55.152 INFO:teuthology.orchestra.run.smithi089.stdout: "age": 201.881012793, 2023-06-03T20:09:55.152 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.15934906900000001, 2023-06-03T20:09:55.153 INFO:teuthology.orchestra.run.smithi089.stdout: "type_data": { 2023-06-03T20:09:55.153 INFO:teuthology.orchestra.run.smithi089.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-03T20:09:55.153 INFO:teuthology.orchestra.run.smithi089.stdout: "client_info": { 2023-06-03T20:09:55.153 INFO:teuthology.orchestra.run.smithi089.stdout: "client": "client.36009", 2023-06-03T20:09:55.153 INFO:teuthology.orchestra.run.smithi089.stdout: "client_addr": "172.21.15.89:0/2550116169", 2023-06-03T20:09:55.153 INFO:teuthology.orchestra.run.smithi089.stdout: "tid": 37599 2023-06-03T20:09:55.154 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:09:55.154 INFO:teuthology.orchestra.run.smithi089.stdout: "events": [ 2023-06-03T20:09:55.154 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:09:55.154 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "initiated", 2023-06-03T20:09:55.154 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:06:33.238661+0000", 2023-06-03T20:09:55.154 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0 2023-06-03T20:09:55.154 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:09:55.155 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:09:55.155 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "throttled", 2023-06-03T20:09:55.155 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:06:33.238661+0000", 2023-06-03T20:09:55.155 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0 2023-06-03T20:09:55.155 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:09:55.155 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:09:55.155 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "header_read", 2023-06-03T20:09:55.155 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:06:33.238659+0000", 2023-06-03T20:09:55.156 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 4294967295.9999981 2023-06-03T20:09:55.156 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:09:55.156 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:09:55.156 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "all_read", 2023-06-03T20:09:55.156 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:06:33.238688+0000", 2023-06-03T20:09:55.156 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 2.9261e-05 2023-06-03T20:09:55.157 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:09:55.157 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:09:55.157 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "dispatched", 2023-06-03T20:09:55.157 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:06:33.238701+0000", 2023-06-03T20:09:55.157 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 1.2899e-05 2023-06-03T20:09:55.157 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:09:55.157 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:09:55.157 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "queued_for_pg", 2023-06-03T20:09:55.158 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:06:33.238710+0000", 2023-06-03T20:09:55.158 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 8.6789999999999994e-06 2023-06-03T20:09:55.158 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:09:55.158 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:09:55.158 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "reached_pg", 2023-06-03T20:09:55.158 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:06:33.238759+0000", 2023-06-03T20:09:55.158 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 4.8764999999999998e-05 2023-06-03T20:09:55.158 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:09:55.159 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:09:55.159 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "started", 2023-06-03T20:09:55.159 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:06:33.238922+0000", 2023-06-03T20:09:55.159 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.00016286 2023-06-03T20:09:55.159 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:09:55.159 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:09:55.159 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "waiting for subops from 0,3", 2023-06-03T20:09:55.160 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:06:33.239099+0000", 2023-06-03T20:09:55.160 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.00017710600000000001 2023-06-03T20:09:55.160 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:09:55.160 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:09:55.160 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "op_commit", 2023-06-03T20:09:55.160 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:06:33.240460+0000", 2023-06-03T20:09:55.160 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.001360977 2023-06-03T20:09:55.161 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:09:55.161 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:09:55.161 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:09:55.161 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:06:33.241776+0000", 2023-06-03T20:09:55.161 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.001316259 2023-06-03T20:09:55.161 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:09:55.161 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:09:55.161 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:09:55.162 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:06:33.397828+0000", 2023-06-03T20:09:55.162 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.15605253299999999 2023-06-03T20:09:55.162 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:09:55.162 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:09:55.162 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "commit_sent", 2023-06-03T20:09:55.162 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:06:33.397897+0000", 2023-06-03T20:09:55.162 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 6.8993999999999997e-05 2023-06-03T20:09:55.162 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:09:55.163 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:09:55.163 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "done", 2023-06-03T20:09:55.163 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:06:33.398010+0000", 2023-06-03T20:09:55.163 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.000112549 2023-06-03T20:09:55.164 INFO:teuthology.orchestra.run.smithi089.stdout: } 2023-06-03T20:09:55.164 INFO:teuthology.orchestra.run.smithi089.stdout: ] 2023-06-03T20:09:55.164 INFO:teuthology.orchestra.run.smithi089.stdout: } 2023-06-03T20:09:55.164 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:09:55.165 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:09:55.165 INFO:teuthology.orchestra.run.smithi089.stdout: "description": "osd_op(client.36009.0:37605 10.5 10:a7c45cde:::benchmark_data_smithi089_140876_object37604:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e163)", 2023-06-03T20:09:55.165 INFO:teuthology.orchestra.run.smithi089.stdout: "initiated_at": "2023-06-03T20:06:33.239478+0000", 2023-06-03T20:09:55.165 INFO:teuthology.orchestra.run.smithi089.stdout: "age": 201.88019565799999, 2023-06-03T20:09:55.166 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.15863480799999999, 2023-06-03T20:09:55.166 INFO:teuthology.orchestra.run.smithi089.stdout: "type_data": { 2023-06-03T20:09:55.166 INFO:teuthology.orchestra.run.smithi089.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-03T20:09:55.166 INFO:teuthology.orchestra.run.smithi089.stdout: "client_info": { 2023-06-03T20:09:55.167 INFO:teuthology.orchestra.run.smithi089.stdout: "client": "client.36009", 2023-06-03T20:09:55.167 INFO:teuthology.orchestra.run.smithi089.stdout: "client_addr": "172.21.15.89:0/2550116169", 2023-06-03T20:09:55.168 INFO:teuthology.orchestra.run.smithi089.stdout: "tid": 37605 2023-06-03T20:09:55.168 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:09:55.168 INFO:teuthology.orchestra.run.smithi089.stdout: "events": [ 2023-06-03T20:09:55.168 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:09:55.168 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "initiated", 2023-06-03T20:09:55.169 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:06:33.239478+0000", 2023-06-03T20:09:55.169 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0 2023-06-03T20:09:55.169 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:09:55.169 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:09:55.170 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "throttled", 2023-06-03T20:09:55.170 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:06:33.239478+0000", 2023-06-03T20:09:55.170 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0 2023-06-03T20:09:55.170 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:09:55.170 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:09:55.170 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "header_read", 2023-06-03T20:09:55.171 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:06:33.239477+0000", 2023-06-03T20:09:55.171 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 4294967295.999999 2023-06-03T20:09:55.171 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:09:55.171 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:09:55.171 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "all_read", 2023-06-03T20:09:55.171 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:06:33.239501+0000", 2023-06-03T20:09:55.171 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 2.4015e-05 2023-06-03T20:09:55.172 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:09:55.172 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:09:55.172 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "dispatched", 2023-06-03T20:09:55.172 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:06:33.239513+0000", 2023-06-03T20:09:55.173 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 1.1949e-05 2023-06-03T20:09:55.173 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:09:55.173 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:09:55.174 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "queued_for_pg", 2023-06-03T20:09:55.174 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:06:33.239521+0000", 2023-06-03T20:09:55.175 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 7.6580000000000002e-06 2023-06-03T20:09:55.175 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:09:55.175 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:09:55.176 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "reached_pg", 2023-06-03T20:09:55.176 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:06:33.240478+0000", 2023-06-03T20:09:55.176 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.00095758200000000001 2023-06-03T20:09:55.176 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:09:55.176 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:09:55.176 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "started", 2023-06-03T20:09:55.177 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:06:33.240572+0000", 2023-06-03T20:09:55.177 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 9.3407999999999997e-05 2023-06-03T20:09:55.177 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:09:55.177 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:09:55.177 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "waiting for subops from 0,3", 2023-06-03T20:09:55.177 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:06:33.240672+0000", 2023-06-03T20:09:55.177 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 9.9994000000000005e-05 2023-06-03T20:09:55.178 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:09:55.178 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:09:55.178 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "op_commit", 2023-06-03T20:09:55.178 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:06:33.241918+0000", 2023-06-03T20:09:55.178 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.001246661 2023-06-03T20:09:55.178 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:09:55.178 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:09:55.178 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:09:55.179 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:06:33.242040+0000", 2023-06-03T20:09:55.179 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.000122067 2023-06-03T20:09:55.179 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:09:55.179 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:09:55.179 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:09:55.179 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:06:33.398053+0000", 2023-06-03T20:09:55.179 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.15601306700000001 2023-06-03T20:09:55.179 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:09:55.180 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:09:55.180 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "commit_sent", 2023-06-03T20:09:55.180 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:06:33.398084+0000", 2023-06-03T20:09:55.180 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 3.0862999999999999e-05 2023-06-03T20:09:55.180 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:09:55.180 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:09:55.180 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "done", 2023-06-03T20:09:55.181 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:06:33.398113+0000", 2023-06-03T20:09:55.181 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 2.8623999999999999e-05 2023-06-03T20:09:55.181 INFO:teuthology.orchestra.run.smithi089.stdout: } 2023-06-03T20:09:55.181 INFO:teuthology.orchestra.run.smithi089.stdout: ] 2023-06-03T20:09:55.181 INFO:teuthology.orchestra.run.smithi089.stdout: } 2023-06-03T20:09:55.181 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:09:55.181 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:09:55.181 INFO:teuthology.orchestra.run.smithi089.stdout: "description": "osd_repop(client.36009.0:37609 10.e e163/157 10:7c3604f0:::benchmark_data_smithi089_140876_object37608:head v 163'2317, mlcod=163'2311)", 2023-06-03T20:09:55.182 INFO:teuthology.orchestra.run.smithi089.stdout: "initiated_at": "2023-06-03T20:06:33.245062+0000", 2023-06-03T20:09:55.182 INFO:teuthology.orchestra.run.smithi089.stdout: "age": 201.874611563, 2023-06-03T20:09:55.182 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.155029166, 2023-06-03T20:09:55.182 INFO:teuthology.orchestra.run.smithi089.stdout: "type_data": { 2023-06-03T20:09:55.182 INFO:teuthology.orchestra.run.smithi089.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-03T20:09:55.182 INFO:teuthology.orchestra.run.smithi089.stdout: "events": [ 2023-06-03T20:09:55.182 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:09:55.182 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "initiated", 2023-06-03T20:09:55.183 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:06:33.245062+0000", 2023-06-03T20:09:55.183 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0 2023-06-03T20:09:55.183 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:09:55.183 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:09:55.183 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "throttled", 2023-06-03T20:09:55.183 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:06:33.245062+0000", 2023-06-03T20:09:55.183 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0 2023-06-03T20:09:55.184 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:09:55.184 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:09:55.184 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "header_read", 2023-06-03T20:09:55.184 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:06:33.245061+0000", 2023-06-03T20:09:55.184 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 4294967295.999999 2023-06-03T20:09:55.184 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:09:55.184 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:09:55.185 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "all_read", 2023-06-03T20:09:55.185 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:06:33.245112+0000", 2023-06-03T20:09:55.185 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 5.0955999999999998e-05 2023-06-03T20:09:55.185 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:09:55.185 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:09:55.185 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "dispatched", 2023-06-03T20:09:55.185 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:06:33.245123+0000", 2023-06-03T20:09:55.185 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 1.1014e-05 2023-06-03T20:09:55.186 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:09:55.186 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:09:55.186 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "queued_for_pg", 2023-06-03T20:09:55.186 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:06:33.245130+0000", 2023-06-03T20:09:55.186 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 6.6780000000000003e-06 2023-06-03T20:09:55.186 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:09:55.186 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:09:55.187 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "reached_pg", 2023-06-03T20:09:55.187 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:06:33.245168+0000", 2023-06-03T20:09:55.187 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 3.8458999999999999e-05 2023-06-03T20:09:55.187 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:09:55.187 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:09:55.187 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "started", 2023-06-03T20:09:55.187 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:06:33.397050+0000", 2023-06-03T20:09:55.187 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.151881918 2023-06-03T20:09:55.188 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:09:55.188 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:09:55.188 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "commit_sent", 2023-06-03T20:09:55.188 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:06:33.400031+0000", 2023-06-03T20:09:55.188 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.0029806379999999999 2023-06-03T20:09:55.188 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:09:55.188 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:09:55.189 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "done", 2023-06-03T20:09:55.189 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:06:33.400091+0000", 2023-06-03T20:09:55.189 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 6.0350000000000003e-05 2023-06-03T20:09:55.189 INFO:teuthology.orchestra.run.smithi089.stdout: } 2023-06-03T20:09:55.189 INFO:teuthology.orchestra.run.smithi089.stdout: ] 2023-06-03T20:09:55.189 INFO:teuthology.orchestra.run.smithi089.stdout: } 2023-06-03T20:09:55.189 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:09:55.189 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:09:55.190 INFO:teuthology.orchestra.run.smithi089.stdout: "description": "osd_repop(client.36009.0:37614 10.e e163/157 10:73103e12:::benchmark_data_smithi089_140876_object37613:head v 163'2318, mlcod=163'2313)", 2023-06-03T20:09:55.190 INFO:teuthology.orchestra.run.smithi089.stdout: "initiated_at": "2023-06-03T20:06:33.247611+0000", 2023-06-03T20:09:55.190 INFO:teuthology.orchestra.run.smithi089.stdout: "age": 201.87206233800001, 2023-06-03T20:09:55.190 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.15389926100000001, 2023-06-03T20:09:55.190 INFO:teuthology.orchestra.run.smithi089.stdout: "type_data": { 2023-06-03T20:09:55.190 INFO:teuthology.orchestra.run.smithi089.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-03T20:09:55.190 INFO:teuthology.orchestra.run.smithi089.stdout: "events": [ 2023-06-03T20:09:55.190 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:09:55.191 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "initiated", 2023-06-03T20:09:55.191 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:06:33.247611+0000", 2023-06-03T20:09:55.191 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0 2023-06-03T20:09:55.191 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:09:55.191 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:09:55.191 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "throttled", 2023-06-03T20:09:55.191 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:06:33.247611+0000", 2023-06-03T20:09:55.192 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0 2023-06-03T20:09:55.192 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:09:55.192 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:09:55.192 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "header_read", 2023-06-03T20:09:55.192 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:06:33.247611+0000", 2023-06-03T20:09:55.192 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 4294967295.999999 2023-06-03T20:09:55.192 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:09:55.193 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:09:55.193 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "all_read", 2023-06-03T20:09:55.193 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:06:33.247710+0000", 2023-06-03T20:09:55.193 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 9.9576999999999999e-05 2023-06-03T20:09:55.193 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:09:55.193 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:09:55.194 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "dispatched", 2023-06-03T20:09:55.194 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:06:33.397020+0000", 2023-06-03T20:09:55.194 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.149309946 2023-06-03T20:09:55.194 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:09:55.194 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:09:55.194 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "queued_for_pg", 2023-06-03T20:09:55.194 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:06:33.397045+0000", 2023-06-03T20:09:55.195 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 2.4998e-05 2023-06-03T20:09:55.195 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:09:55.195 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:09:55.195 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "reached_pg", 2023-06-03T20:09:55.195 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:06:33.398672+0000", 2023-06-03T20:09:55.195 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.0016268949999999999 2023-06-03T20:09:55.195 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:09:55.196 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:09:55.196 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "started", 2023-06-03T20:09:55.196 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:06:33.398691+0000", 2023-06-03T20:09:55.196 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 1.9281000000000001e-05 2023-06-03T20:09:55.196 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:09:55.196 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:09:55.197 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "commit_sent", 2023-06-03T20:09:55.197 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:06:33.401470+0000", 2023-06-03T20:09:55.197 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.0027786909999999998 2023-06-03T20:09:55.197 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:09:55.197 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:09:55.197 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "done", 2023-06-03T20:09:55.197 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:06:33.401511+0000", 2023-06-03T20:09:55.198 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 4.0648000000000001e-05 2023-06-03T20:09:55.198 INFO:teuthology.orchestra.run.smithi089.stdout: } 2023-06-03T20:09:55.198 INFO:teuthology.orchestra.run.smithi089.stdout: ] 2023-06-03T20:09:55.198 INFO:teuthology.orchestra.run.smithi089.stdout: } 2023-06-03T20:09:55.198 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:09:55.198 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:09:55.199 INFO:teuthology.orchestra.run.smithi089.stdout: "description": "osd_repop(client.36009.0:37613 10.9 e163/157 10:96181204:::benchmark_data_smithi089_140876_object37612:head v 163'2380, mlcod=163'2378)", 2023-06-03T20:09:55.199 INFO:teuthology.orchestra.run.smithi089.stdout: "initiated_at": "2023-06-03T20:06:33.247689+0000", 2023-06-03T20:09:55.199 INFO:teuthology.orchestra.run.smithi089.stdout: "age": 201.871985099, 2023-06-03T20:09:55.199 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.155347022, 2023-06-03T20:09:55.199 INFO:teuthology.orchestra.run.smithi089.stdout: "type_data": { 2023-06-03T20:09:55.199 INFO:teuthology.orchestra.run.smithi089.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-03T20:09:55.200 INFO:teuthology.orchestra.run.smithi089.stdout: "events": [ 2023-06-03T20:09:55.200 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:09:55.200 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "initiated", 2023-06-03T20:09:55.200 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:06:33.247689+0000", 2023-06-03T20:09:55.200 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0 2023-06-03T20:09:55.200 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:09:55.201 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:09:55.201 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "throttled", 2023-06-03T20:09:55.201 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:06:33.247689+0000", 2023-06-03T20:09:55.201 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0 2023-06-03T20:09:55.201 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:09:55.202 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:09:55.202 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "header_read", 2023-06-03T20:09:55.202 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:06:33.247688+0000", 2023-06-03T20:09:55.202 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 4294967295.999999 2023-06-03T20:09:55.202 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:09:55.202 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:09:55.202 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "all_read", 2023-06-03T20:09:55.202 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:06:33.247739+0000", 2023-06-03T20:09:55.203 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 5.0713e-05 2023-06-03T20:09:55.203 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:09:55.203 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:09:55.203 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "dispatched", 2023-06-03T20:09:55.203 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:06:33.397020+0000", 2023-06-03T20:09:55.203 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.14928155600000001 2023-06-03T20:09:55.203 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:09:55.203 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:09:55.204 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "queued_for_pg", 2023-06-03T20:09:55.204 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:06:33.397045+0000", 2023-06-03T20:09:55.204 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 2.5004999999999999e-05 2023-06-03T20:09:55.204 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:09:55.204 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:09:55.204 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "reached_pg", 2023-06-03T20:09:55.204 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:06:33.397132+0000", 2023-06-03T20:09:55.204 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 8.6605000000000002e-05 2023-06-03T20:09:55.205 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:09:55.205 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:09:55.205 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "started", 2023-06-03T20:09:55.205 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:06:33.397165+0000", 2023-06-03T20:09:55.205 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 3.3040000000000002e-05 2023-06-03T20:09:55.205 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:09:55.206 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:09:55.206 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "commit_sent", 2023-06-03T20:09:55.206 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:06:33.402992+0000", 2023-06-03T20:09:55.206 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.0058277349999999997 2023-06-03T20:09:55.206 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:09:55.207 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:09:55.207 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "done", 2023-06-03T20:09:55.207 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:06:33.403036+0000", 2023-06-03T20:09:55.207 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 4.3130000000000002e-05 2023-06-03T20:09:55.207 INFO:teuthology.orchestra.run.smithi089.stdout: } 2023-06-03T20:09:55.207 INFO:teuthology.orchestra.run.smithi089.stdout: ] 2023-06-03T20:09:55.207 INFO:teuthology.orchestra.run.smithi089.stdout: } 2023-06-03T20:09:55.207 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:09:55.208 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:09:55.208 INFO:teuthology.orchestra.run.smithi089.stdout: "description": "osd_op(client.36009.0:51311 10.5 10:a3d238a1:::benchmark_data_smithi089_140876_object51310:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e163)", 2023-06-03T20:09:55.208 INFO:teuthology.orchestra.run.smithi089.stdout: "initiated_at": "2023-06-03T20:06:43.277285+0000", 2023-06-03T20:09:55.208 INFO:teuthology.orchestra.run.smithi089.stdout: "age": 191.842388599, 2023-06-03T20:09:55.208 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.188609796, 2023-06-03T20:09:55.208 INFO:teuthology.orchestra.run.smithi089.stdout: "type_data": { 2023-06-03T20:09:55.208 INFO:teuthology.orchestra.run.smithi089.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-03T20:09:55.208 INFO:teuthology.orchestra.run.smithi089.stdout: "client_info": { 2023-06-03T20:09:55.209 INFO:teuthology.orchestra.run.smithi089.stdout: "client": "client.36009", 2023-06-03T20:09:55.209 INFO:teuthology.orchestra.run.smithi089.stdout: "client_addr": "172.21.15.89:0/2550116169", 2023-06-03T20:09:55.209 INFO:teuthology.orchestra.run.smithi089.stdout: "tid": 51311 2023-06-03T20:09:55.209 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:09:55.209 INFO:teuthology.orchestra.run.smithi089.stdout: "events": [ 2023-06-03T20:09:55.209 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:09:55.209 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "initiated", 2023-06-03T20:09:55.209 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:06:43.277285+0000", 2023-06-03T20:09:55.210 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0 2023-06-03T20:09:55.210 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:09:55.210 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:09:55.210 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "throttled", 2023-06-03T20:09:55.210 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:06:43.277285+0000", 2023-06-03T20:09:55.210 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0 2023-06-03T20:09:55.210 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:09:55.211 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:09:55.211 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "header_read", 2023-06-03T20:09:55.211 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:06:43.277284+0000", 2023-06-03T20:09:55.211 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 4294967295.999999 2023-06-03T20:09:55.211 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:09:55.211 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:09:55.211 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "all_read", 2023-06-03T20:09:55.211 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:06:43.277305+0000", 2023-06-03T20:09:55.212 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 2.0883e-05 2023-06-03T20:09:55.212 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:09:55.212 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:09:55.212 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "dispatched", 2023-06-03T20:09:55.212 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:06:43.277313+0000", 2023-06-03T20:09:55.212 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 8.3229999999999998e-06 2023-06-03T20:09:55.212 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:09:55.213 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:09:55.213 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "queued_for_pg", 2023-06-03T20:09:55.213 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:06:43.277318+0000", 2023-06-03T20:09:55.213 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 5.3460000000000001e-06 2023-06-03T20:09:55.213 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:09:55.213 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:09:55.213 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "reached_pg", 2023-06-03T20:09:55.213 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:06:43.277370+0000", 2023-06-03T20:09:55.214 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 5.1805000000000003e-05 2023-06-03T20:09:55.214 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:09:55.214 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:09:55.214 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "started", 2023-06-03T20:09:55.214 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:06:43.277489+0000", 2023-06-03T20:09:55.214 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.00011848199999999999 2023-06-03T20:09:55.214 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:09:55.215 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:09:55.215 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "waiting for subops from 0,3", 2023-06-03T20:09:55.215 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:06:43.277626+0000", 2023-06-03T20:09:55.215 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.00013736799999999999 2023-06-03T20:09:55.215 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:09:55.215 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:09:55.215 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:09:55.215 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:06:43.282987+0000", 2023-06-03T20:09:55.216 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.0053610810000000002 2023-06-03T20:09:55.216 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:09:55.216 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:09:55.216 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "op_commit", 2023-06-03T20:09:55.216 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:06:43.282998+0000", 2023-06-03T20:09:55.216 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 1.0753e-05 2023-06-03T20:09:55.216 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:09:55.216 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:09:55.217 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:09:55.217 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:06:43.465776+0000", 2023-06-03T20:09:55.217 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.18277768 2023-06-03T20:09:55.217 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:09:55.217 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:09:55.217 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "commit_sent", 2023-06-03T20:09:55.217 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:06:43.465836+0000", 2023-06-03T20:09:55.217 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 6.0711999999999997e-05 2023-06-03T20:09:55.218 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:09:55.218 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:09:55.218 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "done", 2023-06-03T20:09:55.218 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:06:43.465895+0000", 2023-06-03T20:09:55.218 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 5.8553e-05 2023-06-03T20:09:55.218 INFO:teuthology.orchestra.run.smithi089.stdout: } 2023-06-03T20:09:55.218 INFO:teuthology.orchestra.run.smithi089.stdout: ] 2023-06-03T20:09:55.218 INFO:teuthology.orchestra.run.smithi089.stdout: } 2023-06-03T20:09:55.219 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:09:55.219 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:09:55.219 INFO:teuthology.orchestra.run.smithi089.stdout: "description": "osd_op(client.36009.0:51312 10.5 10:a50aeb8e:::benchmark_data_smithi089_140876_object51311:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e163)", 2023-06-03T20:09:55.219 INFO:teuthology.orchestra.run.smithi089.stdout: "initiated_at": "2023-06-03T20:06:43.277329+0000", 2023-06-03T20:09:55.219 INFO:teuthology.orchestra.run.smithi089.stdout: "age": 191.84234504099999, 2023-06-03T20:09:55.219 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.19546854499999999, 2023-06-03T20:09:55.219 INFO:teuthology.orchestra.run.smithi089.stdout: "type_data": { 2023-06-03T20:09:55.219 INFO:teuthology.orchestra.run.smithi089.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-03T20:09:55.220 INFO:teuthology.orchestra.run.smithi089.stdout: "client_info": { 2023-06-03T20:09:55.220 INFO:teuthology.orchestra.run.smithi089.stdout: "client": "client.36009", 2023-06-03T20:09:55.220 INFO:teuthology.orchestra.run.smithi089.stdout: "client_addr": "172.21.15.89:0/2550116169", 2023-06-03T20:09:55.220 INFO:teuthology.orchestra.run.smithi089.stdout: "tid": 51312 2023-06-03T20:09:55.220 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:09:55.220 INFO:teuthology.orchestra.run.smithi089.stdout: "events": [ 2023-06-03T20:09:55.220 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:09:55.221 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "initiated", 2023-06-03T20:09:55.221 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:06:43.277329+0000", 2023-06-03T20:09:55.221 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0 2023-06-03T20:09:55.221 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:09:55.221 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:09:55.221 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "throttled", 2023-06-03T20:09:55.221 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:06:43.277329+0000", 2023-06-03T20:09:55.221 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0 2023-06-03T20:09:55.222 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:09:55.222 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:09:55.222 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "header_read", 2023-06-03T20:09:55.222 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:06:43.277328+0000", 2023-06-03T20:09:55.222 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 4294967295.9999995 2023-06-03T20:09:55.222 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:09:55.222 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:09:55.223 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "all_read", 2023-06-03T20:09:55.223 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:06:43.277344+0000", 2023-06-03T20:09:55.223 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 1.5495999999999999e-05 2023-06-03T20:09:55.223 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:09:55.223 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:09:55.223 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "dispatched", 2023-06-03T20:09:55.224 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:06:43.277348+0000", 2023-06-03T20:09:55.224 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 4.5600000000000004e-06 2023-06-03T20:09:55.224 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:09:55.224 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:09:55.224 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "queued_for_pg", 2023-06-03T20:09:55.224 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:06:43.277352+0000", 2023-06-03T20:09:55.224 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 3.3960000000000002e-06 2023-06-03T20:09:55.225 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:09:55.225 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:09:55.225 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "reached_pg", 2023-06-03T20:09:55.225 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:06:43.281231+0000", 2023-06-03T20:09:55.225 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.0038794530000000002 2023-06-03T20:09:55.225 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:09:55.226 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:09:55.226 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "started", 2023-06-03T20:09:55.226 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:06:43.281359+0000", 2023-06-03T20:09:55.226 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.000127806 2023-06-03T20:09:55.226 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:09:55.226 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:09:55.226 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "waiting for subops from 0,3", 2023-06-03T20:09:55.226 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:06:43.281514+0000", 2023-06-03T20:09:55.227 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.00015483900000000001 2023-06-03T20:09:55.227 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:09:55.227 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:09:55.227 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:09:55.227 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:06:43.283314+0000", 2023-06-03T20:09:55.227 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.0017998249999999999 2023-06-03T20:09:55.228 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:09:55.228 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:09:55.228 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "op_commit", 2023-06-03T20:09:55.228 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:06:43.284647+0000", 2023-06-03T20:09:55.228 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.0013335809999999999 2023-06-03T20:09:55.228 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:09:55.229 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:09:55.229 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:09:55.229 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:06:43.472695+0000", 2023-06-03T20:09:55.229 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.188047511 2023-06-03T20:09:55.229 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:09:55.229 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:09:55.230 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "commit_sent", 2023-06-03T20:09:55.230 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:06:43.472759+0000", 2023-06-03T20:09:55.230 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 6.4442000000000004e-05 2023-06-03T20:09:55.230 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:09:55.230 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:09:55.231 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "done", 2023-06-03T20:09:55.231 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:06:43.472797+0000", 2023-06-03T20:09:55.231 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 3.8000999999999998e-05 2023-06-03T20:09:55.231 INFO:teuthology.orchestra.run.smithi089.stdout: } 2023-06-03T20:09:55.231 INFO:teuthology.orchestra.run.smithi089.stdout: ] 2023-06-03T20:09:55.231 INFO:teuthology.orchestra.run.smithi089.stdout: } 2023-06-03T20:09:55.232 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:09:55.232 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:09:55.232 INFO:teuthology.orchestra.run.smithi089.stdout: "description": "osd_op(client.36009.0:126096 10.5 10:a130c36d:::benchmark_data_smithi089_140876_object20961:head [delete] snapc 0=[] ondisk+write+known_if_redirected e174)", 2023-06-03T20:09:55.232 INFO:teuthology.orchestra.run.smithi089.stdout: "initiated_at": "2023-06-03T20:08:16.530709+0000", 2023-06-03T20:09:55.233 INFO:teuthology.orchestra.run.smithi089.stdout: "age": 98.588964621000002, 2023-06-03T20:09:55.233 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.17283011100000001, 2023-06-03T20:09:55.233 INFO:teuthology.orchestra.run.smithi089.stdout: "type_data": { 2023-06-03T20:09:55.233 INFO:teuthology.orchestra.run.smithi089.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-03T20:09:55.233 INFO:teuthology.orchestra.run.smithi089.stdout: "client_info": { 2023-06-03T20:09:55.234 INFO:teuthology.orchestra.run.smithi089.stdout: "client": "client.36009", 2023-06-03T20:09:55.234 INFO:teuthology.orchestra.run.smithi089.stdout: "client_addr": "172.21.15.89:0/2550116169", 2023-06-03T20:09:55.234 INFO:teuthology.orchestra.run.smithi089.stdout: "tid": 126096 2023-06-03T20:09:55.234 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:09:55.234 INFO:teuthology.orchestra.run.smithi089.stdout: "events": [ 2023-06-03T20:09:55.234 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:09:55.234 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "initiated", 2023-06-03T20:09:55.234 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:08:16.530709+0000", 2023-06-03T20:09:55.235 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0 2023-06-03T20:09:55.235 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:09:55.235 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:09:55.235 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "throttled", 2023-06-03T20:09:55.235 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:08:16.530709+0000", 2023-06-03T20:09:55.235 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0 2023-06-03T20:09:55.235 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:09:55.236 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:09:55.236 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "header_read", 2023-06-03T20:09:55.236 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:08:16.530707+0000", 2023-06-03T20:09:55.236 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 4294967295.9999981 2023-06-03T20:09:55.236 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:09:55.236 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:09:55.236 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "all_read", 2023-06-03T20:09:55.237 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:08:16.530719+0000", 2023-06-03T20:09:55.237 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 1.155e-05 2023-06-03T20:09:55.237 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:09:55.237 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:09:55.237 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "dispatched", 2023-06-03T20:09:55.237 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:08:16.530737+0000", 2023-06-03T20:09:55.237 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 1.8442000000000001e-05 2023-06-03T20:09:55.238 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:09:55.238 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:09:55.238 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "queued_for_pg", 2023-06-03T20:09:55.238 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:08:16.530748+0000", 2023-06-03T20:09:55.238 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 1.1429e-05 2023-06-03T20:09:55.238 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:09:55.238 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:09:55.238 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "reached_pg", 2023-06-03T20:09:55.239 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:08:16.530807+0000", 2023-06-03T20:09:55.239 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 5.8533999999999999e-05 2023-06-03T20:09:55.239 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:09:55.239 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:09:55.239 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "started", 2023-06-03T20:09:55.240 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:08:16.530971+0000", 2023-06-03T20:09:55.240 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.000163752 2023-06-03T20:09:55.240 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:09:55.240 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:09:55.240 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "waiting for subops from 0,3", 2023-06-03T20:09:55.240 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:08:16.531113+0000", 2023-06-03T20:09:55.240 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.000142235 2023-06-03T20:09:55.240 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:09:55.241 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:09:55.241 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "op_commit", 2023-06-03T20:09:55.241 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:08:16.534880+0000", 2023-06-03T20:09:55.241 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.0037669349999999999 2023-06-03T20:09:55.241 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:09:55.241 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:09:55.241 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:09:55.241 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:08:16.536193+0000", 2023-06-03T20:09:55.242 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.00131303 2023-06-03T20:09:55.242 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:09:55.242 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:09:55.242 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:09:55.242 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:08:16.703387+0000", 2023-06-03T20:09:55.242 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.16719386999999999 2023-06-03T20:09:55.242 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:09:55.243 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:09:55.243 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "commit_sent", 2023-06-03T20:09:55.243 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:08:16.703493+0000", 2023-06-03T20:09:55.243 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.000106346 2023-06-03T20:09:55.243 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:09:55.243 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:09:55.243 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "done", 2023-06-03T20:09:55.243 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:08:16.703539+0000", 2023-06-03T20:09:55.244 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 4.6054000000000003e-05 2023-06-03T20:09:55.244 INFO:teuthology.orchestra.run.smithi089.stdout: } 2023-06-03T20:09:55.244 INFO:teuthology.orchestra.run.smithi089.stdout: ] 2023-06-03T20:09:55.244 INFO:teuthology.orchestra.run.smithi089.stdout: } 2023-06-03T20:09:55.244 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:09:55.244 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:09:55.244 INFO:teuthology.orchestra.run.smithi089.stdout: "description": "osd_op(client.36009.0:147778 10.5 10:a1842ad4:::benchmark_data_smithi089_140876_object42643:head [delete] snapc 0=[] ondisk+write+known_if_redirected e176)", 2023-06-03T20:09:55.244 INFO:teuthology.orchestra.run.smithi089.stdout: "initiated_at": "2023-06-03T20:08:50.960592+0000", 2023-06-03T20:09:55.245 INFO:teuthology.orchestra.run.smithi089.stdout: "age": 64.159081318999995, 2023-06-03T20:09:55.245 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.143679483, 2023-06-03T20:09:55.245 INFO:teuthology.orchestra.run.smithi089.stdout: "type_data": { 2023-06-03T20:09:55.245 INFO:teuthology.orchestra.run.smithi089.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-03T20:09:55.245 INFO:teuthology.orchestra.run.smithi089.stdout: "client_info": { 2023-06-03T20:09:55.245 INFO:teuthology.orchestra.run.smithi089.stdout: "client": "client.36009", 2023-06-03T20:09:55.245 INFO:teuthology.orchestra.run.smithi089.stdout: "client_addr": "172.21.15.89:0/2550116169", 2023-06-03T20:09:55.245 INFO:teuthology.orchestra.run.smithi089.stdout: "tid": 147778 2023-06-03T20:09:55.246 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:09:55.246 INFO:teuthology.orchestra.run.smithi089.stdout: "events": [ 2023-06-03T20:09:55.246 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:09:55.246 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "initiated", 2023-06-03T20:09:55.246 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:08:50.960592+0000", 2023-06-03T20:09:55.246 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0 2023-06-03T20:09:55.246 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:09:55.246 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:09:55.247 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "throttled", 2023-06-03T20:09:55.247 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:08:50.960592+0000", 2023-06-03T20:09:55.247 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0 2023-06-03T20:09:55.247 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:09:55.247 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:09:55.247 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "header_read", 2023-06-03T20:09:55.247 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:08:50.960591+0000", 2023-06-03T20:09:55.247 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 4294967295.999999 2023-06-03T20:09:55.248 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:09:55.248 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:09:55.248 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "all_read", 2023-06-03T20:09:55.248 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:08:50.960597+0000", 2023-06-03T20:09:55.248 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 6.0379999999999996e-06 2023-06-03T20:09:55.248 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:09:55.248 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:09:55.249 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "dispatched", 2023-06-03T20:09:55.249 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:08:50.960610+0000", 2023-06-03T20:09:55.249 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 1.2181000000000001e-05 2023-06-03T20:09:55.249 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:09:55.249 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:09:55.249 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "queued_for_pg", 2023-06-03T20:09:55.250 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:08:50.960617+0000", 2023-06-03T20:09:55.250 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 7.7300000000000005e-06 2023-06-03T20:09:55.250 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:09:55.250 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:09:55.250 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "reached_pg", 2023-06-03T20:09:55.250 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:08:50.960878+0000", 2023-06-03T20:09:55.250 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.00026108499999999998 2023-06-03T20:09:55.251 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:09:55.251 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:09:55.251 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "started", 2023-06-03T20:09:55.251 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:08:51.084317+0000", 2023-06-03T20:09:55.251 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.123438408 2023-06-03T20:09:55.251 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:09:55.251 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:09:55.252 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "waiting for subops from 0,3", 2023-06-03T20:09:55.252 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:08:51.084443+0000", 2023-06-03T20:09:55.252 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.000125776 2023-06-03T20:09:55.252 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:09:55.252 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:09:55.252 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "op_commit", 2023-06-03T20:09:55.253 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:08:51.097484+0000", 2023-06-03T20:09:55.253 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.013041211 2023-06-03T20:09:55.253 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:09:55.253 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:09:55.253 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:09:55.253 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:08:51.104116+0000", 2023-06-03T20:09:55.253 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.006632214 2023-06-03T20:09:55.254 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:09:55.254 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:09:55.254 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:09:55.254 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:08:51.104173+0000", 2023-06-03T20:09:55.254 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 5.7068999999999999e-05 2023-06-03T20:09:55.254 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:09:55.255 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:09:55.255 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "commit_sent", 2023-06-03T20:09:55.255 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:08:51.104223+0000", 2023-06-03T20:09:55.255 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 4.9941000000000002e-05 2023-06-03T20:09:55.255 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:09:55.255 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:09:55.256 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "done", 2023-06-03T20:09:55.256 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:08:51.104272+0000", 2023-06-03T20:09:55.256 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 4.8779999999999997e-05 2023-06-03T20:09:55.256 INFO:teuthology.orchestra.run.smithi089.stdout: } 2023-06-03T20:09:55.256 INFO:teuthology.orchestra.run.smithi089.stdout: ] 2023-06-03T20:09:55.256 INFO:teuthology.orchestra.run.smithi089.stdout: } 2023-06-03T20:09:55.257 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:09:55.257 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:09:55.257 INFO:teuthology.orchestra.run.smithi089.stdout: "description": "osd_op(client.36009.0:147784 10.5 10:a029e97f:::benchmark_data_smithi089_140876_object42649:head [delete] snapc 0=[] ondisk+write+known_if_redirected e176)", 2023-06-03T20:09:55.257 INFO:teuthology.orchestra.run.smithi089.stdout: "initiated_at": "2023-06-03T20:08:50.971727+0000", 2023-06-03T20:09:55.257 INFO:teuthology.orchestra.run.smithi089.stdout: "age": 64.147947110999993, 2023-06-03T20:09:55.258 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.14850369999999999, 2023-06-03T20:09:55.258 INFO:teuthology.orchestra.run.smithi089.stdout: "type_data": { 2023-06-03T20:09:55.258 INFO:teuthology.orchestra.run.smithi089.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-03T20:09:55.258 INFO:teuthology.orchestra.run.smithi089.stdout: "client_info": { 2023-06-03T20:09:55.258 INFO:teuthology.orchestra.run.smithi089.stdout: "client": "client.36009", 2023-06-03T20:09:55.258 INFO:teuthology.orchestra.run.smithi089.stdout: "client_addr": "172.21.15.89:0/2550116169", 2023-06-03T20:09:55.258 INFO:teuthology.orchestra.run.smithi089.stdout: "tid": 147784 2023-06-03T20:09:55.259 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:09:55.259 INFO:teuthology.orchestra.run.smithi089.stdout: "events": [ 2023-06-03T20:09:55.259 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:09:55.259 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "initiated", 2023-06-03T20:09:55.259 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:08:50.971727+0000", 2023-06-03T20:09:55.259 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0 2023-06-03T20:09:55.259 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:09:55.259 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:09:55.260 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "throttled", 2023-06-03T20:09:55.260 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:08:50.971727+0000", 2023-06-03T20:09:55.260 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0 2023-06-03T20:09:55.260 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:09:55.260 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:09:55.260 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "header_read", 2023-06-03T20:09:55.260 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:08:50.971724+0000", 2023-06-03T20:09:55.260 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 4294967295.9999976 2023-06-03T20:09:55.261 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:09:55.261 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:09:55.261 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "all_read", 2023-06-03T20:09:55.261 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:08:50.971736+0000", 2023-06-03T20:09:55.261 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 1.1187e-05 2023-06-03T20:09:55.261 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:09:55.261 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:09:55.262 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "dispatched", 2023-06-03T20:09:55.262 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:08:51.083981+0000", 2023-06-03T20:09:55.262 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.112245923 2023-06-03T20:09:55.262 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:09:55.262 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:09:55.262 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "queued_for_pg", 2023-06-03T20:09:55.263 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:08:51.084005+0000", 2023-06-03T20:09:55.263 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 2.3373000000000001e-05 2023-06-03T20:09:55.263 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:09:55.263 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:09:55.263 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "reached_pg", 2023-06-03T20:09:55.263 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:08:51.091450+0000", 2023-06-03T20:09:55.263 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.0074451880000000002 2023-06-03T20:09:55.263 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:09:55.264 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:09:55.264 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "started", 2023-06-03T20:09:55.264 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:08:51.091606+0000", 2023-06-03T20:09:55.264 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.00015563900000000001 2023-06-03T20:09:55.264 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:09:55.264 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:09:55.264 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "waiting for subops from 0,3", 2023-06-03T20:09:55.265 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:08:51.091790+0000", 2023-06-03T20:09:55.265 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.00018458999999999999 2023-06-03T20:09:55.265 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:09:55.265 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:09:55.265 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "op_commit", 2023-06-03T20:09:55.265 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:08:51.104057+0000", 2023-06-03T20:09:55.265 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.012266536 2023-06-03T20:09:55.265 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:09:55.266 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:09:55.266 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:09:55.266 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:08:51.114302+0000", 2023-06-03T20:09:55.266 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.010245651999999999 2023-06-03T20:09:55.266 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:09:55.266 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:09:55.266 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:09:55.266 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:08:51.120126+0000", 2023-06-03T20:09:55.267 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.005823819 2023-06-03T20:09:55.267 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:09:55.267 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:09:55.267 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "commit_sent", 2023-06-03T20:09:55.267 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:08:51.120184+0000", 2023-06-03T20:09:55.267 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 5.7871000000000002e-05 2023-06-03T20:09:55.267 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:09:55.268 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:09:55.268 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "done", 2023-06-03T20:09:55.268 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:08:51.120230+0000", 2023-06-03T20:09:55.268 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 4.6087000000000001e-05 2023-06-03T20:09:55.268 INFO:teuthology.orchestra.run.smithi089.stdout: } 2023-06-03T20:09:55.268 INFO:teuthology.orchestra.run.smithi089.stdout: ] 2023-06-03T20:09:55.268 INFO:teuthology.orchestra.run.smithi089.stdout: } 2023-06-03T20:09:55.268 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:09:55.269 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:09:55.269 INFO:teuthology.orchestra.run.smithi089.stdout: "description": "osd_op(client.36009.0:157377 10.3 10:c455c9c7:::benchmark_data_smithi089_140876_object52242:head [delete] snapc 0=[] ondisk+write+known_if_redirected e176)", 2023-06-03T20:09:55.269 INFO:teuthology.orchestra.run.smithi089.stdout: "initiated_at": "2023-06-03T20:09:05.795774+0000", 2023-06-03T20:09:55.269 INFO:teuthology.orchestra.run.smithi089.stdout: "age": 49.323899431000001, 2023-06-03T20:09:55.269 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.143902, 2023-06-03T20:09:55.269 INFO:teuthology.orchestra.run.smithi089.stdout: "type_data": { 2023-06-03T20:09:55.269 INFO:teuthology.orchestra.run.smithi089.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-03T20:09:55.270 INFO:teuthology.orchestra.run.smithi089.stdout: "client_info": { 2023-06-03T20:09:55.270 INFO:teuthology.orchestra.run.smithi089.stdout: "client": "client.36009", 2023-06-03T20:09:55.270 INFO:teuthology.orchestra.run.smithi089.stdout: "client_addr": "172.21.15.89:0/2550116169", 2023-06-03T20:09:55.270 INFO:teuthology.orchestra.run.smithi089.stdout: "tid": 157377 2023-06-03T20:09:55.270 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:09:55.270 INFO:teuthology.orchestra.run.smithi089.stdout: "events": [ 2023-06-03T20:09:55.270 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:09:55.270 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "initiated", 2023-06-03T20:09:55.271 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:09:05.795774+0000", 2023-06-03T20:09:55.271 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0 2023-06-03T20:09:55.271 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:09:55.271 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:09:55.271 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "throttled", 2023-06-03T20:09:55.271 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:09:05.795774+0000", 2023-06-03T20:09:55.271 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0 2023-06-03T20:09:55.271 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:09:55.272 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:09:55.272 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "header_read", 2023-06-03T20:09:55.272 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:09:05.795773+0000", 2023-06-03T20:09:55.272 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 4294967295.9999986 2023-06-03T20:09:55.272 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:09:55.272 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:09:55.272 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "all_read", 2023-06-03T20:09:55.273 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:09:05.795780+0000", 2023-06-03T20:09:55.273 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 6.737e-06 2023-06-03T20:09:55.273 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:09:55.273 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:09:55.273 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "dispatched", 2023-06-03T20:09:55.273 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:09:05.795790+0000", 2023-06-03T20:09:55.273 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 9.9450000000000005e-06 2023-06-03T20:09:55.273 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:09:55.274 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:09:55.274 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "queued_for_pg", 2023-06-03T20:09:55.274 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:09:05.795795+0000", 2023-06-03T20:09:55.274 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 5.8889999999999999e-06 2023-06-03T20:09:55.274 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:09:55.274 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:09:55.274 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "reached_pg", 2023-06-03T20:09:55.274 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:09:05.795841+0000", 2023-06-03T20:09:55.275 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 4.5355000000000002e-05 2023-06-03T20:09:55.275 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:09:55.275 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:09:55.275 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "started", 2023-06-03T20:09:55.275 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:09:05.795931+0000", 2023-06-03T20:09:55.275 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 8.9827000000000006e-05 2023-06-03T20:09:55.275 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:09:55.275 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:09:55.276 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "waiting for subops from 3,5", 2023-06-03T20:09:55.276 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:09:05.796011+0000", 2023-06-03T20:09:55.276 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 8.0111000000000006e-05 2023-06-03T20:09:55.276 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:09:55.276 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:09:55.276 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "op_commit", 2023-06-03T20:09:55.276 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:09:05.805055+0000", 2023-06-03T20:09:55.277 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.0090444449999999999 2023-06-03T20:09:55.277 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:09:55.277 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:09:55.277 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:09:55.277 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:09:05.809408+0000", 2023-06-03T20:09:55.277 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.0043527230000000002 2023-06-03T20:09:55.277 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:09:55.277 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:09:55.278 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:09:55.278 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:09:05.938311+0000", 2023-06-03T20:09:55.278 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.128902866 2023-06-03T20:09:55.278 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:09:55.278 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:09:55.278 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "commit_sent", 2023-06-03T20:09:55.278 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:09:05.938366+0000", 2023-06-03T20:09:55.279 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 5.4994999999999998e-05 2023-06-03T20:09:55.279 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:09:55.279 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:09:55.279 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "done", 2023-06-03T20:09:55.280 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:09:05.939676+0000", 2023-06-03T20:09:55.280 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.0013104670000000001 2023-06-03T20:09:55.280 INFO:teuthology.orchestra.run.smithi089.stdout: } 2023-06-03T20:09:55.280 INFO:teuthology.orchestra.run.smithi089.stdout: ] 2023-06-03T20:09:55.280 INFO:teuthology.orchestra.run.smithi089.stdout: } 2023-06-03T20:09:55.280 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:09:55.281 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:09:55.281 INFO:teuthology.orchestra.run.smithi089.stdout: "description": "osd_op(client.36009.0:157381 10.3 10:c8949368:::benchmark_data_smithi089_140876_object52246:head [delete] snapc 0=[] ondisk+write+known_if_redirected e176)", 2023-06-03T20:09:55.281 INFO:teuthology.orchestra.run.smithi089.stdout: "initiated_at": "2023-06-03T20:09:05.796339+0000", 2023-06-03T20:09:55.281 INFO:teuthology.orchestra.run.smithi089.stdout: "age": 49.323335057999998, 2023-06-03T20:09:55.281 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.14347036299999999, 2023-06-03T20:09:55.281 INFO:teuthology.orchestra.run.smithi089.stdout: "type_data": { 2023-06-03T20:09:55.282 INFO:teuthology.orchestra.run.smithi089.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-03T20:09:55.282 INFO:teuthology.orchestra.run.smithi089.stdout: "client_info": { 2023-06-03T20:09:55.282 INFO:teuthology.orchestra.run.smithi089.stdout: "client": "client.36009", 2023-06-03T20:09:55.282 INFO:teuthology.orchestra.run.smithi089.stdout: "client_addr": "172.21.15.89:0/2550116169", 2023-06-03T20:09:55.282 INFO:teuthology.orchestra.run.smithi089.stdout: "tid": 157381 2023-06-03T20:09:55.282 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:09:55.283 INFO:teuthology.orchestra.run.smithi089.stdout: "events": [ 2023-06-03T20:09:55.283 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:09:55.283 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "initiated", 2023-06-03T20:09:55.283 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:09:05.796339+0000", 2023-06-03T20:09:55.283 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0 2023-06-03T20:09:55.283 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:09:55.283 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:09:55.284 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "throttled", 2023-06-03T20:09:55.284 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:09:05.796339+0000", 2023-06-03T20:09:55.284 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0 2023-06-03T20:09:55.284 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:09:55.284 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:09:55.284 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "header_read", 2023-06-03T20:09:55.285 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:09:05.796338+0000", 2023-06-03T20:09:55.285 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 4294967295.9999995 2023-06-03T20:09:55.285 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:09:55.285 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:09:55.285 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "all_read", 2023-06-03T20:09:55.285 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:09:05.796342+0000", 2023-06-03T20:09:55.286 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 3.8319999999999999e-06 2023-06-03T20:09:55.286 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:09:55.286 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:09:55.286 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "dispatched", 2023-06-03T20:09:55.286 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:09:05.796349+0000", 2023-06-03T20:09:55.286 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 7.678e-06 2023-06-03T20:09:55.287 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:09:55.287 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:09:55.287 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "queued_for_pg", 2023-06-03T20:09:55.287 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:09:05.796356+0000", 2023-06-03T20:09:55.287 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 6.5259999999999999e-06 2023-06-03T20:09:55.287 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:09:55.288 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:09:55.288 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "reached_pg", 2023-06-03T20:09:55.288 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:09:05.800545+0000", 2023-06-03T20:09:55.288 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.00418915 2023-06-03T20:09:55.288 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:09:55.288 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:09:55.289 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "started", 2023-06-03T20:09:55.289 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:09:05.800669+0000", 2023-06-03T20:09:55.289 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.000124146 2023-06-03T20:09:55.289 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:09:55.289 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:09:55.289 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "waiting for subops from 3,5", 2023-06-03T20:09:55.289 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:09:05.800813+0000", 2023-06-03T20:09:55.290 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.000143447 2023-06-03T20:09:55.290 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:09:55.290 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:09:55.290 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "op_commit", 2023-06-03T20:09:55.290 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:09:05.812713+0000", 2023-06-03T20:09:55.290 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.011900551000000001 2023-06-03T20:09:55.290 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:09:55.290 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:09:55.291 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:09:55.291 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:09:05.813369+0000", 2023-06-03T20:09:55.291 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.00065542699999999998 2023-06-03T20:09:55.291 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:09:55.291 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:09:55.291 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:09:55.291 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:09:05.939725+0000", 2023-06-03T20:09:55.291 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.12635622299999999 2023-06-03T20:09:55.292 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:09:55.292 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:09:55.292 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "commit_sent", 2023-06-03T20:09:55.292 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:09:05.939773+0000", 2023-06-03T20:09:55.292 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 4.8049e-05 2023-06-03T20:09:55.292 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:09:55.292 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:09:55.293 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "done", 2023-06-03T20:09:55.293 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:09:05.939809+0000", 2023-06-03T20:09:55.293 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 3.6007000000000003e-05 2023-06-03T20:09:55.293 INFO:teuthology.orchestra.run.smithi089.stdout: } 2023-06-03T20:09:55.293 INFO:teuthology.orchestra.run.smithi089.stdout: ] 2023-06-03T20:09:55.293 INFO:teuthology.orchestra.run.smithi089.stdout: } 2023-06-03T20:09:55.293 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:09:55.293 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:09:55.294 INFO:teuthology.orchestra.run.smithi089.stdout: "description": "osd_repop(client.36009.0:159787 10.e e176/157 10:7475d4a8:::benchmark_data_smithi089_140876_object54652:head v 176'9918, mlcod=176'9915)", 2023-06-03T20:09:55.294 INFO:teuthology.orchestra.run.smithi089.stdout: "initiated_at": "2023-06-03T20:09:09.713911+0000", 2023-06-03T20:09:55.294 INFO:teuthology.orchestra.run.smithi089.stdout: "age": 45.405762780000003, 2023-06-03T20:09:55.294 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.14303506799999999, 2023-06-03T20:09:55.294 INFO:teuthology.orchestra.run.smithi089.stdout: "type_data": { 2023-06-03T20:09:55.294 INFO:teuthology.orchestra.run.smithi089.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-03T20:09:55.295 INFO:teuthology.orchestra.run.smithi089.stdout: "events": [ 2023-06-03T20:09:55.295 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:09:55.295 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "initiated", 2023-06-03T20:09:55.295 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:09:09.713911+0000", 2023-06-03T20:09:55.295 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0 2023-06-03T20:09:55.295 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:09:55.295 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:09:55.296 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "throttled", 2023-06-03T20:09:55.296 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:09:09.713911+0000", 2023-06-03T20:09:55.296 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0 2023-06-03T20:09:55.296 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:09:55.296 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:09:55.296 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "header_read", 2023-06-03T20:09:55.296 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:09:09.713910+0000", 2023-06-03T20:09:55.297 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 4294967295.999999 2023-06-03T20:09:55.297 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:09:55.297 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:09:55.297 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "all_read", 2023-06-03T20:09:55.297 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:09:09.713927+0000", 2023-06-03T20:09:55.297 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 1.6871000000000002e-05 2023-06-03T20:09:55.297 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:09:55.297 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:09:55.298 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "dispatched", 2023-06-03T20:09:55.298 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:09:09.713943+0000", 2023-06-03T20:09:55.298 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 1.6444e-05 2023-06-03T20:09:55.298 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:09:55.298 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:09:55.298 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "queued_for_pg", 2023-06-03T20:09:55.298 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:09:09.713954+0000", 2023-06-03T20:09:55.298 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 1.0431999999999999e-05 2023-06-03T20:09:55.299 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:09:55.299 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:09:55.299 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "reached_pg", 2023-06-03T20:09:55.299 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:09:09.714008+0000", 2023-06-03T20:09:55.299 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 5.4308999999999998e-05 2023-06-03T20:09:55.299 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:09:55.299 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:09:55.299 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "started", 2023-06-03T20:09:55.300 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:09:09.714037+0000", 2023-06-03T20:09:55.300 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 2.9394999999999999e-05 2023-06-03T20:09:55.300 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:09:55.300 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:09:55.300 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "commit_sent", 2023-06-03T20:09:55.300 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:09:09.856862+0000", 2023-06-03T20:09:55.300 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.14282460299999999 2023-06-03T20:09:55.301 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:09:55.301 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:09:55.301 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "done", 2023-06-03T20:09:55.301 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:09:09.856946+0000", 2023-06-03T20:09:55.301 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 8.4141000000000006e-05 2023-06-03T20:09:55.301 INFO:teuthology.orchestra.run.smithi089.stdout: } 2023-06-03T20:09:55.301 INFO:teuthology.orchestra.run.smithi089.stdout: ] 2023-06-03T20:09:55.301 INFO:teuthology.orchestra.run.smithi089.stdout: } 2023-06-03T20:09:55.302 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:09:55.302 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:09:55.302 INFO:teuthology.orchestra.run.smithi089.stdout: "description": "osd_repop(client.36009.0:159791 10.e e176/157 10:78c60ebd:::benchmark_data_smithi089_140876_object54656:head v 176'9919, mlcod=176'9915)", 2023-06-03T20:09:55.302 INFO:teuthology.orchestra.run.smithi089.stdout: "initiated_at": "2023-06-03T20:09:09.713971+0000", 2023-06-03T20:09:55.302 INFO:teuthology.orchestra.run.smithi089.stdout: "age": 45.405703150999997, 2023-06-03T20:09:55.302 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.14303713200000001, 2023-06-03T20:09:55.302 INFO:teuthology.orchestra.run.smithi089.stdout: "type_data": { 2023-06-03T20:09:55.303 INFO:teuthology.orchestra.run.smithi089.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-03T20:09:55.303 INFO:teuthology.orchestra.run.smithi089.stdout: "events": [ 2023-06-03T20:09:55.303 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:09:55.303 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "initiated", 2023-06-03T20:09:55.303 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:09:09.713971+0000", 2023-06-03T20:09:55.303 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0 2023-06-03T20:09:55.303 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:09:55.303 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:09:55.304 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "throttled", 2023-06-03T20:09:55.304 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:09:09.713971+0000", 2023-06-03T20:09:55.304 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0 2023-06-03T20:09:55.304 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:09:55.304 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:09:55.304 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "header_read", 2023-06-03T20:09:55.305 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:09:09.713970+0000", 2023-06-03T20:09:55.305 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 4294967295.9999995 2023-06-03T20:09:55.305 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:09:55.305 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:09:55.305 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "all_read", 2023-06-03T20:09:55.305 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:09:09.713975+0000", 2023-06-03T20:09:55.305 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 4.7530000000000001e-06 2023-06-03T20:09:55.306 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:09:55.306 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:09:55.306 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "dispatched", 2023-06-03T20:09:55.306 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:09:09.713981+0000", 2023-06-03T20:09:55.306 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 6.5119999999999997e-06 2023-06-03T20:09:55.306 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:09:55.307 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:09:55.307 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "queued_for_pg", 2023-06-03T20:09:55.307 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:09:09.713987+0000", 2023-06-03T20:09:55.307 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 5.2000000000000002e-06 2023-06-03T20:09:55.307 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:09:55.307 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:09:55.307 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "reached_pg", 2023-06-03T20:09:55.308 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:09:09.717284+0000", 2023-06-03T20:09:55.308 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.0032970619999999999 2023-06-03T20:09:55.308 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:09:55.308 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:09:55.308 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "started", 2023-06-03T20:09:55.308 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:09:09.717307+0000", 2023-06-03T20:09:55.308 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 2.2920000000000001e-05 2023-06-03T20:09:55.309 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:09:55.309 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:09:55.309 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "commit_sent", 2023-06-03T20:09:55.309 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:09:09.856952+0000", 2023-06-03T20:09:55.309 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.139645399 2023-06-03T20:09:55.309 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:09:55.310 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:09:55.310 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "done", 2023-06-03T20:09:55.310 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:09:09.857008+0000", 2023-06-03T20:09:55.310 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 5.5639e-05 2023-06-03T20:09:55.310 INFO:teuthology.orchestra.run.smithi089.stdout: } 2023-06-03T20:09:55.310 INFO:teuthology.orchestra.run.smithi089.stdout: ] 2023-06-03T20:09:55.311 INFO:teuthology.orchestra.run.smithi089.stdout: } 2023-06-03T20:09:55.311 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:09:55.311 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:09:55.311 INFO:teuthology.orchestra.run.smithi089.stdout: "description": "osd_repop(client.36009.0:159796 10.e e176/157 10:7577d358:::benchmark_data_smithi089_140876_object54661:head v 176'9920, mlcod=176'9915)", 2023-06-03T20:09:55.311 INFO:teuthology.orchestra.run.smithi089.stdout: "initiated_at": "2023-06-03T20:09:09.714003+0000", 2023-06-03T20:09:55.311 INFO:teuthology.orchestra.run.smithi089.stdout: "age": 45.405670538999999, 2023-06-03T20:09:55.312 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.143066165, 2023-06-03T20:09:55.312 INFO:teuthology.orchestra.run.smithi089.stdout: "type_data": { 2023-06-03T20:09:55.312 INFO:teuthology.orchestra.run.smithi089.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-03T20:09:55.312 INFO:teuthology.orchestra.run.smithi089.stdout: "events": [ 2023-06-03T20:09:55.312 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:09:55.312 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "initiated", 2023-06-03T20:09:55.313 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:09:09.714003+0000", 2023-06-03T20:09:55.313 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0 2023-06-03T20:09:55.313 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:09:55.313 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:09:55.313 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "throttled", 2023-06-03T20:09:55.313 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:09:09.714003+0000", 2023-06-03T20:09:55.314 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0 2023-06-03T20:09:55.314 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:09:55.314 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:09:55.314 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "header_read", 2023-06-03T20:09:55.314 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:09:09.714003+0000", 2023-06-03T20:09:55.314 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 4294967295.9999995 2023-06-03T20:09:55.314 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:09:55.315 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:09:55.315 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "all_read", 2023-06-03T20:09:55.315 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:09:09.714012+0000", 2023-06-03T20:09:55.315 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 8.7010000000000002e-06 2023-06-03T20:09:55.315 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:09:55.315 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:09:55.315 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "dispatched", 2023-06-03T20:09:55.316 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:09:09.714018+0000", 2023-06-03T20:09:55.316 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 6.7340000000000002e-06 2023-06-03T20:09:55.316 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:09:55.316 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:09:55.316 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "queued_for_pg", 2023-06-03T20:09:55.316 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:09:09.714023+0000", 2023-06-03T20:09:55.316 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 4.8119999999999998e-06 2023-06-03T20:09:55.316 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:09:55.317 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:09:55.317 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "reached_pg", 2023-06-03T20:09:55.317 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:09:09.721037+0000", 2023-06-03T20:09:55.317 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.0070137480000000002 2023-06-03T20:09:55.317 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:09:55.317 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:09:55.317 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "started", 2023-06-03T20:09:55.318 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:09:09.721058+0000", 2023-06-03T20:09:55.318 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 2.1452000000000001e-05 2023-06-03T20:09:55.318 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:09:55.318 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:09:55.318 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "commit_sent", 2023-06-03T20:09:55.318 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:09:09.857018+0000", 2023-06-03T20:09:55.318 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.135959467 2023-06-03T20:09:55.318 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:09:55.319 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:09:55.319 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "done", 2023-06-03T20:09:55.319 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:09:09.857069+0000", 2023-06-03T20:09:55.319 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 5.1496999999999998e-05 2023-06-03T20:09:55.319 INFO:teuthology.orchestra.run.smithi089.stdout: } 2023-06-03T20:09:55.319 INFO:teuthology.orchestra.run.smithi089.stdout: ] 2023-06-03T20:09:55.319 INFO:teuthology.orchestra.run.smithi089.stdout: } 2023-06-03T20:09:55.319 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:09:55.320 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:09:55.320 INFO:teuthology.orchestra.run.smithi089.stdout: "description": "osd_repop(client.36009.0:159797 10.e e176/157 10:7732dec5:::benchmark_data_smithi089_140876_object54662:head v 176'9921, mlcod=176'9915)", 2023-06-03T20:09:55.320 INFO:teuthology.orchestra.run.smithi089.stdout: "initiated_at": "2023-06-03T20:09:09.719378+0000", 2023-06-03T20:09:55.320 INFO:teuthology.orchestra.run.smithi089.stdout: "age": 45.400295685000003, 2023-06-03T20:09:55.320 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.14414485099999999, 2023-06-03T20:09:55.320 INFO:teuthology.orchestra.run.smithi089.stdout: "type_data": { 2023-06-03T20:09:55.320 INFO:teuthology.orchestra.run.smithi089.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-03T20:09:55.320 INFO:teuthology.orchestra.run.smithi089.stdout: "events": [ 2023-06-03T20:09:55.321 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:09:55.321 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "initiated", 2023-06-03T20:09:55.321 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:09:09.719378+0000", 2023-06-03T20:09:55.321 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0 2023-06-03T20:09:55.321 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:09:55.321 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:09:55.321 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "throttled", 2023-06-03T20:09:55.322 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:09:09.719378+0000", 2023-06-03T20:09:55.322 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0 2023-06-03T20:09:55.322 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:09:55.322 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:09:55.322 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "header_read", 2023-06-03T20:09:55.322 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:09:09.719377+0000", 2023-06-03T20:09:55.322 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 4294967295.999999 2023-06-03T20:09:55.322 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:09:55.323 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:09:55.323 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "all_read", 2023-06-03T20:09:55.323 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:09:09.719386+0000", 2023-06-03T20:09:55.323 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 9.2590000000000006e-06 2023-06-03T20:09:55.323 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:09:55.323 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:09:55.323 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "dispatched", 2023-06-03T20:09:55.323 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:09:09.719398+0000", 2023-06-03T20:09:55.324 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 1.1323e-05 2023-06-03T20:09:55.324 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:09:55.324 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:09:55.324 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "queued_for_pg", 2023-06-03T20:09:55.324 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:09:09.719405+0000", 2023-06-03T20:09:55.324 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 7.43e-06 2023-06-03T20:09:55.324 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:09:55.325 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:09:55.325 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "reached_pg", 2023-06-03T20:09:55.325 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:09:09.724351+0000", 2023-06-03T20:09:55.325 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.0049461100000000001 2023-06-03T20:09:55.325 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:09:55.325 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:09:55.325 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "started", 2023-06-03T20:09:55.325 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:09:09.724364+0000", 2023-06-03T20:09:55.326 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 1.3084999999999999e-05 2023-06-03T20:09:55.326 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:09:55.326 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:09:55.326 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "commit_sent", 2023-06-03T20:09:55.326 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:09:09.863424+0000", 2023-06-03T20:09:55.326 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.13906004999999999 2023-06-03T20:09:55.326 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:09:55.326 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:09:55.327 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "done", 2023-06-03T20:09:55.327 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:09:09.863523+0000", 2023-06-03T20:09:55.327 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 9.8634999999999993e-05 2023-06-03T20:09:55.327 INFO:teuthology.orchestra.run.smithi089.stdout: } 2023-06-03T20:09:55.327 INFO:teuthology.orchestra.run.smithi089.stdout: ] 2023-06-03T20:09:55.327 INFO:teuthology.orchestra.run.smithi089.stdout: } 2023-06-03T20:09:55.327 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:09:55.328 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:09:55.328 INFO:teuthology.orchestra.run.smithi089.stdout: "description": "osd_op(client.36009.0:159978 10.5 10:acac2327:::benchmark_data_smithi089_140876_object54843:head [delete] snapc 0=[] ondisk+write+known_if_redirected e176)", 2023-06-03T20:09:55.328 INFO:teuthology.orchestra.run.smithi089.stdout: "initiated_at": "2023-06-03T20:09:10.070577+0000", 2023-06-03T20:09:55.328 INFO:teuthology.orchestra.run.smithi089.stdout: "age": 45.049096702999996, 2023-06-03T20:09:55.328 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.23775432099999999, 2023-06-03T20:09:55.328 INFO:teuthology.orchestra.run.smithi089.stdout: "type_data": { 2023-06-03T20:09:55.328 INFO:teuthology.orchestra.run.smithi089.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-03T20:09:55.328 INFO:teuthology.orchestra.run.smithi089.stdout: "client_info": { 2023-06-03T20:09:55.329 INFO:teuthology.orchestra.run.smithi089.stdout: "client": "client.36009", 2023-06-03T20:09:55.329 INFO:teuthology.orchestra.run.smithi089.stdout: "client_addr": "172.21.15.89:0/2550116169", 2023-06-03T20:09:55.329 INFO:teuthology.orchestra.run.smithi089.stdout: "tid": 159978 2023-06-03T20:09:55.329 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:09:55.329 INFO:teuthology.orchestra.run.smithi089.stdout: "events": [ 2023-06-03T20:09:55.329 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:09:55.329 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "initiated", 2023-06-03T20:09:55.330 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:09:10.070577+0000", 2023-06-03T20:09:55.330 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0 2023-06-03T20:09:55.330 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:09:55.330 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:09:55.330 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "throttled", 2023-06-03T20:09:55.330 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:09:10.070577+0000", 2023-06-03T20:09:55.331 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0 2023-06-03T20:09:55.331 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:09:55.331 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:09:55.331 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "header_read", 2023-06-03T20:09:55.331 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:09:10.070575+0000", 2023-06-03T20:09:55.331 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 4294967295.9999981 2023-06-03T20:09:55.331 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:09:55.331 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:09:55.332 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "all_read", 2023-06-03T20:09:55.332 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:09:10.070584+0000", 2023-06-03T20:09:55.332 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 8.5159999999999994e-06 2023-06-03T20:09:55.332 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:09:55.332 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:09:55.332 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "dispatched", 2023-06-03T20:09:55.332 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:09:10.070599+0000", 2023-06-03T20:09:55.332 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 1.5274e-05 2023-06-03T20:09:55.333 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:09:55.333 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:09:55.333 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "queued_for_pg", 2023-06-03T20:09:55.333 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:09:10.070607+0000", 2023-06-03T20:09:55.333 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 8.5110000000000003e-06 2023-06-03T20:09:55.333 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:09:55.333 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:09:55.334 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "reached_pg", 2023-06-03T20:09:55.335 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:09:10.070663+0000", 2023-06-03T20:09:55.336 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 5.6022e-05 2023-06-03T20:09:55.336 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:09:55.336 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:09:55.336 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "started", 2023-06-03T20:09:55.336 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:09:10.070821+0000", 2023-06-03T20:09:55.336 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.000157734 2023-06-03T20:09:55.337 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:09:55.337 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:09:55.337 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "waiting for subops from 0,3", 2023-06-03T20:09:55.337 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:09:10.070959+0000", 2023-06-03T20:09:55.337 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.00013812199999999999 2023-06-03T20:09:55.337 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:09:55.337 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:09:55.338 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "op_commit", 2023-06-03T20:09:55.338 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:09:10.075818+0000", 2023-06-03T20:09:55.338 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.0048591160000000001 2023-06-03T20:09:55.338 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:09:55.338 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:09:55.338 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:09:55.338 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:09:10.076056+0000", 2023-06-03T20:09:55.339 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.00023725100000000001 2023-06-03T20:09:55.339 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:09:55.339 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:09:55.339 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:09:55.339 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:09:10.307991+0000", 2023-06-03T20:09:55.340 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.231935263 2023-06-03T20:09:55.340 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:09:55.340 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:09:55.340 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "commit_sent", 2023-06-03T20:09:55.340 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:09:10.308282+0000", 2023-06-03T20:09:55.340 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.00029144499999999999 2023-06-03T20:09:55.341 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:09:55.341 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:09:55.341 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "done", 2023-06-03T20:09:55.341 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:09:10.308331+0000", 2023-06-03T20:09:55.341 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 4.9033000000000003e-05 2023-06-03T20:09:55.341 INFO:teuthology.orchestra.run.smithi089.stdout: } 2023-06-03T20:09:55.342 INFO:teuthology.orchestra.run.smithi089.stdout: ] 2023-06-03T20:09:55.342 INFO:teuthology.orchestra.run.smithi089.stdout: } 2023-06-03T20:09:55.342 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:09:55.342 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:09:55.342 INFO:teuthology.orchestra.run.smithi089.stdout: "description": "osd_op(client.36009.0:178054 10.5 10:a220f104:::benchmark_data_smithi089_140876_object72919:head [delete] snapc 0=[] ondisk+write+known_if_redirected e180)", 2023-06-03T20:09:55.342 INFO:teuthology.orchestra.run.smithi089.stdout: "initiated_at": "2023-06-03T20:09:39.075995+0000", 2023-06-03T20:09:55.343 INFO:teuthology.orchestra.run.smithi089.stdout: "age": 16.043678843999999, 2023-06-03T20:09:55.343 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.164846348, 2023-06-03T20:09:55.343 INFO:teuthology.orchestra.run.smithi089.stdout: "type_data": { 2023-06-03T20:09:55.343 INFO:teuthology.orchestra.run.smithi089.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-03T20:09:55.343 INFO:teuthology.orchestra.run.smithi089.stdout: "client_info": { 2023-06-03T20:09:55.344 INFO:teuthology.orchestra.run.smithi089.stdout: "client": "client.36009", 2023-06-03T20:09:55.344 INFO:teuthology.orchestra.run.smithi089.stdout: "client_addr": "172.21.15.89:0/2550116169", 2023-06-03T20:09:55.344 INFO:teuthology.orchestra.run.smithi089.stdout: "tid": 178054 2023-06-03T20:09:55.344 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:09:55.344 INFO:teuthology.orchestra.run.smithi089.stdout: "events": [ 2023-06-03T20:09:55.344 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:09:55.344 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "initiated", 2023-06-03T20:09:55.344 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:09:39.075995+0000", 2023-06-03T20:09:55.345 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0 2023-06-03T20:09:55.345 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:09:55.345 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:09:55.345 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "throttled", 2023-06-03T20:09:55.345 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:09:39.075995+0000", 2023-06-03T20:09:55.345 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0 2023-06-03T20:09:55.346 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:09:55.346 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:09:55.346 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "header_read", 2023-06-03T20:09:55.346 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:09:39.075993+0000", 2023-06-03T20:09:55.346 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 4294967295.9999986 2023-06-03T20:09:55.346 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:09:55.346 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:09:55.346 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "all_read", 2023-06-03T20:09:55.347 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:09:39.076000+0000", 2023-06-03T20:09:55.347 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 6.6390000000000003e-06 2023-06-03T20:09:55.347 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:09:55.347 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:09:55.347 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "dispatched", 2023-06-03T20:09:55.347 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:09:39.076010+0000", 2023-06-03T20:09:55.347 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 1.0319000000000001e-05 2023-06-03T20:09:55.348 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:09:55.348 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:09:55.348 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "queued_for_pg", 2023-06-03T20:09:55.348 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:09:39.076017+0000", 2023-06-03T20:09:55.348 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 7.0759999999999997e-06 2023-06-03T20:09:55.348 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:09:55.348 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:09:55.348 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "reached_pg", 2023-06-03T20:09:55.349 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:09:39.076065+0000", 2023-06-03T20:09:55.349 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 4.7307e-05 2023-06-03T20:09:55.349 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:09:55.349 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:09:55.349 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "started", 2023-06-03T20:09:55.349 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:09:39.076184+0000", 2023-06-03T20:09:55.349 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.000118921 2023-06-03T20:09:55.349 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:09:55.350 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:09:55.350 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "waiting for subops from 0,3", 2023-06-03T20:09:55.350 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:09:39.076296+0000", 2023-06-03T20:09:55.350 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.00011206500000000001 2023-06-03T20:09:55.350 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:09:55.350 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:09:55.350 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "op_commit", 2023-06-03T20:09:55.351 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:09:39.081570+0000", 2023-06-03T20:09:55.351 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.0052740369999999996 2023-06-03T20:09:55.351 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:09:55.351 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:09:55.351 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:09:55.351 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:09:39.082047+0000", 2023-06-03T20:09:55.351 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.00047747600000000001 2023-06-03T20:09:55.351 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:09:55.352 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:09:55.352 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:09:55.352 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:09:39.240733+0000", 2023-06-03T20:09:55.352 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.15868605799999999 2023-06-03T20:09:55.352 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:09:55.352 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:09:55.352 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "commit_sent", 2023-06-03T20:09:55.352 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:09:39.240783+0000", 2023-06-03T20:09:55.353 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 4.9965e-05 2023-06-03T20:09:55.353 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:09:55.353 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:09:55.353 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "done", 2023-06-03T20:09:55.353 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:09:39.240841+0000", 2023-06-03T20:09:55.353 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 5.7855000000000001e-05 2023-06-03T20:09:55.353 INFO:teuthology.orchestra.run.smithi089.stdout: } 2023-06-03T20:09:55.353 INFO:teuthology.orchestra.run.smithi089.stdout: ] 2023-06-03T20:09:55.354 INFO:teuthology.orchestra.run.smithi089.stdout: } 2023-06-03T20:09:55.354 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:09:55.354 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:09:55.354 INFO:teuthology.orchestra.run.smithi089.stdout: "description": "osd_op(client.36009.0:178059 10.5 10:a836dc8b:::benchmark_data_smithi089_140876_object72924:head [delete] snapc 0=[] ondisk+write+known_if_redirected e180)", 2023-06-03T20:09:55.354 INFO:teuthology.orchestra.run.smithi089.stdout: "initiated_at": "2023-06-03T20:09:39.088224+0000", 2023-06-03T20:09:55.354 INFO:teuthology.orchestra.run.smithi089.stdout: "age": 16.031450113999998, 2023-06-03T20:09:55.354 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.152719935, 2023-06-03T20:09:55.354 INFO:teuthology.orchestra.run.smithi089.stdout: "type_data": { 2023-06-03T20:09:55.355 INFO:teuthology.orchestra.run.smithi089.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-03T20:09:55.355 INFO:teuthology.orchestra.run.smithi089.stdout: "client_info": { 2023-06-03T20:09:55.355 INFO:teuthology.orchestra.run.smithi089.stdout: "client": "client.36009", 2023-06-03T20:09:55.355 INFO:teuthology.orchestra.run.smithi089.stdout: "client_addr": "172.21.15.89:0/2550116169", 2023-06-03T20:09:55.355 INFO:teuthology.orchestra.run.smithi089.stdout: "tid": 178059 2023-06-03T20:09:55.355 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:09:55.355 INFO:teuthology.orchestra.run.smithi089.stdout: "events": [ 2023-06-03T20:09:55.355 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:09:55.356 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "initiated", 2023-06-03T20:09:55.356 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:09:39.088224+0000", 2023-06-03T20:09:55.356 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0 2023-06-03T20:09:55.356 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:09:55.356 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:09:55.356 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "throttled", 2023-06-03T20:09:55.356 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:09:39.088224+0000", 2023-06-03T20:09:55.357 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0 2023-06-03T20:09:55.357 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:09:55.357 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:09:55.357 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "header_read", 2023-06-03T20:09:55.357 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:09:39.088222+0000", 2023-06-03T20:09:55.357 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 4294967295.9999986 2023-06-03T20:09:55.357 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:09:55.357 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:09:55.358 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "all_read", 2023-06-03T20:09:55.358 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:09:39.088228+0000", 2023-06-03T20:09:55.358 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 5.7350000000000001e-06 2023-06-03T20:09:55.358 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:09:55.358 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:09:55.358 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "dispatched", 2023-06-03T20:09:55.358 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:09:39.088237+0000", 2023-06-03T20:09:55.358 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 8.9809999999999992e-06 2023-06-03T20:09:55.359 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:09:55.359 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:09:55.359 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "queued_for_pg", 2023-06-03T20:09:55.359 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:09:39.088242+0000", 2023-06-03T20:09:55.359 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 5.3759999999999999e-06 2023-06-03T20:09:55.359 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:09:55.359 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:09:55.359 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "reached_pg", 2023-06-03T20:09:55.360 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:09:39.088274+0000", 2023-06-03T20:09:55.360 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 3.154e-05 2023-06-03T20:09:55.360 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:09:55.360 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:09:55.360 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "started", 2023-06-03T20:09:55.361 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:09:39.088368+0000", 2023-06-03T20:09:55.361 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 9.3948000000000002e-05 2023-06-03T20:09:55.361 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:09:55.361 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:09:55.361 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "waiting for subops from 0,3", 2023-06-03T20:09:55.361 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:09:39.195384+0000", 2023-06-03T20:09:55.361 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.107016052 2023-06-03T20:09:55.361 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:09:55.362 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:09:55.362 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:09:55.362 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:09:39.203078+0000", 2023-06-03T20:09:55.362 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.0076941989999999997 2023-06-03T20:09:55.362 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:09:55.362 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:09:55.363 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "op_commit", 2023-06-03T20:09:55.363 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:09:39.203237+0000", 2023-06-03T20:09:55.363 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.00015916199999999999 2023-06-03T20:09:55.363 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:09:55.363 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:09:55.363 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:09:55.363 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:09:39.240892+0000", 2023-06-03T20:09:55.364 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.037654595999999999 2023-06-03T20:09:55.364 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:09:55.364 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:09:55.364 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "commit_sent", 2023-06-03T20:09:55.364 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:09:39.240920+0000", 2023-06-03T20:09:55.364 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 2.8438000000000001e-05 2023-06-03T20:09:55.364 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:09:55.365 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:09:55.365 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "done", 2023-06-03T20:09:55.365 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:09:39.240943+0000", 2023-06-03T20:09:55.365 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 2.3246000000000001e-05 2023-06-03T20:09:55.365 INFO:teuthology.orchestra.run.smithi089.stdout: } 2023-06-03T20:09:55.365 INFO:teuthology.orchestra.run.smithi089.stdout: ] 2023-06-03T20:09:55.366 INFO:teuthology.orchestra.run.smithi089.stdout: } 2023-06-03T20:09:55.366 INFO:teuthology.orchestra.run.smithi089.stdout: } 2023-06-03T20:09:55.366 INFO:teuthology.orchestra.run.smithi089.stdout: ] 2023-06-03T20:09:55.366 INFO:teuthology.orchestra.run.smithi089.stdout:} 2023-06-03T20:09:55.403 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:09:55.406 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:09:55.407 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:09:55.508 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:09:55.551 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:09:55.553 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:09:55.553 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:09:55.654 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:09:55.689 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:09:55.697 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:09:55.697 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:09:55.798 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:09:55.814 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:09:55 smithi164 bash[59462]: cluster 1685822994.636396 mgr.y (mgr.34104) 1013 : cluster 0 pgmap v967: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 1.4 GiB data, 6.5 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 689 op/s; 13555/70647 objects degraded (19.187%); 398/70647 objects misplaced (0.563%); 1.7 MiB/s, 26 objects/s recovering 2023-06-03T20:09:55.841 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:09:55.844 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:09:55.845 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:09:55.946 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:09:55.959 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:09:55 smithi089 bash[75068]: cluster 1685822994.636396 mgr.y (mgr.34104) 1013 : cluster 0 pgmap v967: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 1.4 GiB data, 6.5 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 689 op/s; 13555/70647 objects degraded (19.187%); 398/70647 objects misplaced (0.563%); 1.7 MiB/s, 26 objects/s recovering 2023-06-03T20:09:55.959 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:09:55 smithi089 bash[78452]: cluster 1685822994.636396 mgr.y (mgr.34104) 1013 : cluster 0 pgmap v967: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 1.4 GiB data, 6.5 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 689 op/s; 13555/70647 objects degraded (19.187%); 398/70647 objects misplaced (0.563%); 1.7 MiB/s, 26 objects/s recovering 2023-06-03T20:09:55.975 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:09:55.978 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:09:55.979 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:09:56.080 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:09:56.122 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:09:56.129 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:09:56.130 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:09:56.230 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:09:56.231 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 20:09:56 smithi089 bash[71163]: ::ffff:172.21.15.164 - - [03/Jun/2023:20:09:56] "GET /metrics HTTP/1.1" 200 228738 "" "Prometheus/2.33.4" 2023-06-03T20:09:56.273 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:09:56.279 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:09:56.279 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:09:56.380 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:09:56.435 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:09:56.440 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:09:56.441 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:09:56.542 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:09:56.573 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:09:56.578 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:09:56.578 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:09:56.679 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:09:56.714 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:09:56.719 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:09:56.721 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:09:56.822 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:09:56.855 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:09:56.856 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:09:56.856 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:09:56.957 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:09:56.992 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:09:56.994 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:09:56.994 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:09:57.095 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:09:57.132 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:09:57.138 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:09:57.139 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:09:57.239 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:09:57.293 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:09:57.299 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:09:57.299 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:09:57.400 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:09:57.430 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:09:57.433 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:09:57.433 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:09:57.534 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:09:57.573 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:09:57.574 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:09:57.574 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:09:57.675 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:09:57.712 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:09:57.714 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:09:57.715 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:09:57.715 INFO:teuthology.orchestra.run.smithi089.stderr:reweighted osd.4 to 1 (10000) 2023-06-03T20:09:57.816 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:09:57.833 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:09:57 smithi164 bash[59462]: cluster 1685822996.6370387 mgr.y (mgr.34104) 1014 : cluster 0 pgmap v968: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 1.4 GiB data, 6.3 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 606 op/s; 13555/68187 objects degraded (19.879%); 398/68187 objects misplaced (0.584%); 1.1 MiB/s, 17 objects/s recovering 2023-06-03T20:09:57.834 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:09:57 smithi164 bash[59462]: audit 1685822997.1170795 mon.c (mon.2) 479 : audit 1 from='client.? 172.21.15.89:0/3902531068' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-06-03T20:09:57.834 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:09:57 smithi164 bash[59462]: audit 1685822997.117673 mon.a (mon.0) 934 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-06-03T20:09:57.834 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:09:57 smithi164 bash[59462]: cluster 1685822997.6842022 mon.a (mon.0) 935 : cluster 3 Health check update: Degraded data redundancy: 13555/68187 objects degraded (19.879%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-06-03T20:09:57.846 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:09:57.851 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:09:57.852 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:09:57.953 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:09:57.961 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:09:57 smithi089 bash[75068]: cluster 1685822996.6370387 mgr.y (mgr.34104) 1014 : cluster 0 pgmap v968: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 1.4 GiB data, 6.3 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 606 op/s; 13555/68187 objects degraded (19.879%); 398/68187 objects misplaced (0.584%); 1.1 MiB/s, 17 objects/s recovering 2023-06-03T20:09:57.961 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:09:57 smithi089 bash[75068]: audit 1685822997.1170795 mon.c (mon.2) 479 : audit 1 from='client.? 172.21.15.89:0/3902531068' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-06-03T20:09:57.962 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:09:57 smithi089 bash[75068]: audit 1685822997.117673 mon.a (mon.0) 934 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-06-03T20:09:57.962 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:09:57 smithi089 bash[75068]: cluster 1685822997.6842022 mon.a (mon.0) 935 : cluster 3 Health check update: Degraded data redundancy: 13555/68187 objects degraded (19.879%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-06-03T20:09:57.962 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:09:57 smithi089 bash[78452]: cluster 1685822996.6370387 mgr.y (mgr.34104) 1014 : cluster 0 pgmap v968: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 1.4 GiB data, 6.3 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 606 op/s; 13555/68187 objects degraded (19.879%); 398/68187 objects misplaced (0.584%); 1.1 MiB/s, 17 objects/s recovering 2023-06-03T20:09:57.962 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:09:57 smithi089 bash[78452]: audit 1685822997.1170795 mon.c (mon.2) 479 : audit 1 from='client.? 172.21.15.89:0/3902531068' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-06-03T20:09:57.962 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:09:57 smithi089 bash[78452]: audit 1685822997.117673 mon.a (mon.0) 934 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-06-03T20:09:57.963 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:09:57 smithi089 bash[78452]: cluster 1685822997.6842022 mon.a (mon.0) 935 : cluster 3 Health check update: Degraded data redundancy: 13555/68187 objects degraded (19.879%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-06-03T20:09:57.979 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:09:57.982 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:09:57.982 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:09:58.083 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:09:58.120 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:09:58.124 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:09:58.125 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:09:58.225 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:09:58.266 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:09:58.272 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:09:58.272 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:09:58.373 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:09:58.422 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:09:58.425 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:09:58.425 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:09:58.526 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:09:58.559 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:09:58.567 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:09:58.568 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:09:58.669 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:09:58.704 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:09:58.706 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:09:58.706 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:09:58.808 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:09:58.833 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:09:58 smithi164 bash[59462]: cluster 1685822997.7034013 osd.5 (osd.5) 113 : cluster 3 failed to encode map e181 with expected crc 2023-06-03T20:09:58.834 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:09:58 smithi164 bash[59462]: cluster 1685822997.703683 osd.6 (osd.6) 119 : cluster 3 failed to encode map e181 with expected crc 2023-06-03T20:09:58.834 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:09:58 smithi164 bash[59462]: audit 1685822997.7050579 mon.a (mon.0) 936 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2023-06-03T20:09:58.834 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:09:58 smithi164 bash[59462]: cluster 1685822997.70542 mon.a (mon.0) 937 : cluster 0 osdmap e181: 8 total, 8 up, 7 in 2023-06-03T20:09:58.834 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:09:58 smithi164 bash[59462]: cluster 1685822997.7072177 osd.6 (osd.6) 120 : cluster 3 failed to encode map e181 with expected crc 2023-06-03T20:09:58.834 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:09:58 smithi164 bash[59462]: cluster 1685822997.709207 osd.7 (osd.7) 117 : cluster 3 failed to encode map e181 with expected crc 2023-06-03T20:09:58.838 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:09:58.841 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:09:58.842 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:09:58.943 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:09:58.989 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:09:58.991 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:09:58.992 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:09:59.031 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:09:58 smithi089 bash[75068]: cluster 1685822997.7034013 osd.5 (osd.5) 113 : cluster 3 failed to encode map e181 with expected crc 2023-06-03T20:09:59.032 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:09:58 smithi089 bash[75068]: cluster 1685822997.703683 osd.6 (osd.6) 119 : cluster 3 failed to encode map e181 with expected crc 2023-06-03T20:09:59.032 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:09:58 smithi089 bash[75068]: audit 1685822997.7050579 mon.a (mon.0) 936 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2023-06-03T20:09:59.032 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:09:58 smithi089 bash[75068]: cluster 1685822997.70542 mon.a (mon.0) 937 : cluster 0 osdmap e181: 8 total, 8 up, 7 in 2023-06-03T20:09:59.032 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:09:58 smithi089 bash[75068]: cluster 1685822997.7072177 osd.6 (osd.6) 120 : cluster 3 failed to encode map e181 with expected crc 2023-06-03T20:09:59.032 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:09:58 smithi089 bash[75068]: cluster 1685822997.709207 osd.7 (osd.7) 117 : cluster 3 failed to encode map e181 with expected crc 2023-06-03T20:09:59.032 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:09:58 smithi089 bash[78452]: cluster 1685822997.7034013 osd.5 (osd.5) 113 : cluster 3 failed to encode map e181 with expected crc 2023-06-03T20:09:59.033 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:09:58 smithi089 bash[78452]: cluster 1685822997.703683 osd.6 (osd.6) 119 : cluster 3 failed to encode map e181 with expected crc 2023-06-03T20:09:59.033 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:09:58 smithi089 bash[78452]: audit 1685822997.7050579 mon.a (mon.0) 936 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2023-06-03T20:09:59.033 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:09:58 smithi089 bash[78452]: cluster 1685822997.70542 mon.a (mon.0) 937 : cluster 0 osdmap e181: 8 total, 8 up, 7 in 2023-06-03T20:09:59.033 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:09:58 smithi089 bash[78452]: cluster 1685822997.7072177 osd.6 (osd.6) 120 : cluster 3 failed to encode map e181 with expected crc 2023-06-03T20:09:59.033 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:09:58 smithi089 bash[78452]: cluster 1685822997.709207 osd.7 (osd.7) 117 : cluster 3 failed to encode map e181 with expected crc 2023-06-03T20:09:59.090 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph daemon osd.3 dump_ops_in_flight 2023-06-03T20:09:59.093 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:09:59.149 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:09:59.154 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:09:59.155 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:09:59.256 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:09:59.296 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:09:59.302 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:09:59.303 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:09:59.404 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:09:59.447 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:09:59.454 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:09:59.455 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:09:59.556 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:09:59.597 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:09:59.601 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:09:59.602 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:09:59.703 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:09:59.731 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:09:59 smithi089 bash[78452]: cluster 1685822998.637637 mgr.y (mgr.34104) 1015 : cluster 0 pgmap v970: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 1.4 GiB data, 6.2 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 642 op/s; 13555/66492 objects degraded (20.386%); 398/66492 objects misplaced (0.599%); 1.3 MiB/s, 21 objects/s recovering 2023-06-03T20:09:59.732 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:09:59 smithi089 bash[75068]: cluster 1685822998.637637 mgr.y (mgr.34104) 1015 : cluster 0 pgmap v970: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 1.4 GiB data, 6.2 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 642 op/s; 13555/66492 objects degraded (20.386%); 398/66492 objects misplaced (0.599%); 1.3 MiB/s, 21 objects/s recovering 2023-06-03T20:09:59.740 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:09:59.747 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:09:59.748 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:09:59.833 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:09:59 smithi164 bash[59462]: cluster 1685822998.637637 mgr.y (mgr.34104) 1015 : cluster 0 pgmap v970: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 1.4 GiB data, 6.2 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 642 op/s; 13555/66492 objects degraded (20.386%); 398/66492 objects misplaced (0.599%); 1.3 MiB/s, 21 objects/s recovering 2023-06-03T20:09:59.849 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:09:59.899 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:09:59.904 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:09:59.905 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:10:00.006 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:10:00.066 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:10:00.073 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:10:00.074 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:10:00.180 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:10:00.213 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:10:00.218 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:10:00.219 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:10:00.320 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:10:00.363 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:10:00.368 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:10:00.369 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:10:00.470 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:10:00.493 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph osd reweight 5 1 2023-06-03T20:10:00.512 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:10:00.516 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:10:00.517 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:10:00.618 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:10:00.656 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:10:00.661 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:10:00.662 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:10:00.745 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:10:00 smithi089 bash[75068]: cluster 1685823000.0001163 mon.a (mon.0) 938 : cluster 3 Health detail: HEALTH_WARN Degraded data redundancy: 13555/66492 objects degraded (20.386%), 3 pgs degraded, 3 pgs undersized 2023-06-03T20:10:00.746 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:10:00 smithi089 bash[75068]: cluster 1685823000.000165 mon.a (mon.0) 939 : cluster 3 [WRN] PG_DEGRADED: Degraded data redundancy: 13555/66492 objects degraded (20.386%), 3 pgs degraded, 3 pgs undersized 2023-06-03T20:10:00.746 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:10:00 smithi089 bash[75068]: cluster 1685823000.0001774 mon.a (mon.0) 940 : cluster 3 pg 10.0 is stuck undersized for 2m, current state active+recovery_wait+undersized+degraded+remapped, last acting [6,4] 2023-06-03T20:10:00.746 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:10:00 smithi089 bash[75068]: cluster 1685823000.00019 mon.a (mon.0) 941 : cluster 3 pg 10.2 is stuck undersized for 2m, current state active+recovery_wait+undersized+degraded+remapped, last acting [4,0] 2023-06-03T20:10:00.746 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:10:00 smithi089 bash[75068]: cluster 1685823000.000201 mon.a (mon.0) 942 : cluster 3 pg 10.b is stuck undersized for 2m, current state active+recovering+undersized+degraded+remapped, last acting [6,0] 2023-06-03T20:10:00.764 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:10:00.807 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:10:00.808 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:10:00.809 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:10:00.910 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:10:00.919 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:10:00 smithi164 bash[59462]: cluster 1685823000.0001163 mon.a (mon.0) 938 : cluster 3 Health detail: HEALTH_WARN Degraded data redundancy: 13555/66492 objects degraded (20.386%), 3 pgs degraded, 3 pgs undersized 2023-06-03T20:10:00.920 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:10:00 smithi164 bash[59462]: cluster 1685823000.000165 mon.a (mon.0) 939 : cluster 3 [WRN] PG_DEGRADED: Degraded data redundancy: 13555/66492 objects degraded (20.386%), 3 pgs degraded, 3 pgs undersized 2023-06-03T20:10:00.920 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:10:00 smithi164 bash[59462]: cluster 1685823000.0001774 mon.a (mon.0) 940 : cluster 3 pg 10.0 is stuck undersized for 2m, current state active+recovery_wait+undersized+degraded+remapped, last acting [6,4] 2023-06-03T20:10:00.921 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:10:00 smithi164 bash[59462]: cluster 1685823000.00019 mon.a (mon.0) 941 : cluster 3 pg 10.2 is stuck undersized for 2m, current state active+recovery_wait+undersized+degraded+remapped, last acting [4,0] 2023-06-03T20:10:00.921 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:10:00 smithi164 bash[59462]: cluster 1685823000.000201 mon.a (mon.0) 942 : cluster 3 pg 10.b is stuck undersized for 2m, current state active+recovering+undersized+degraded+remapped, last acting [6,0] 2023-06-03T20:10:00.948 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:10:00.950 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:10:00.951 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:10:01.028 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:10:00 smithi089 bash[78452]: cluster 1685823000.0001163 mon.a (mon.0) 938 : cluster 3 Health detail: HEALTH_WARN Degraded data redundancy: 13555/66492 objects degraded (20.386%), 3 pgs degraded, 3 pgs undersized 2023-06-03T20:10:01.028 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:10:00 smithi089 bash[78452]: cluster 1685823000.000165 mon.a (mon.0) 939 : cluster 3 [WRN] PG_DEGRADED: Degraded data redundancy: 13555/66492 objects degraded (20.386%), 3 pgs degraded, 3 pgs undersized 2023-06-03T20:10:01.029 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:10:00 smithi089 bash[78452]: cluster 1685823000.0001774 mon.a (mon.0) 940 : cluster 3 pg 10.0 is stuck undersized for 2m, current state active+recovery_wait+undersized+degraded+remapped, last acting [6,4] 2023-06-03T20:10:01.029 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:10:00 smithi089 bash[78452]: cluster 1685823000.00019 mon.a (mon.0) 941 : cluster 3 pg 10.2 is stuck undersized for 2m, current state active+recovery_wait+undersized+degraded+remapped, last acting [4,0] 2023-06-03T20:10:01.029 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:10:00 smithi089 bash[78452]: cluster 1685823000.000201 mon.a (mon.0) 942 : cluster 3 pg 10.b is stuck undersized for 2m, current state active+recovering+undersized+degraded+remapped, last acting [6,0] 2023-06-03T20:10:01.051 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:10:01.082 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:10:01.092 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:10:01.093 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:10:01.194 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:10:01.236 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:10:01.274 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:10:01.276 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:10:01.378 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:10:01.416 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:10:01.421 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:10:01.421 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:10:01.522 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:10:01.553 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:10:01.559 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:10:01.559 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:10:01.660 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:10:01.707 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:10:01.713 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:10:01.714 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:10:01.815 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:10:01.824 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:10:01 smithi089 bash[78452]: cluster 1685823000.6383855 mgr.y (mgr.34104) 1016 : cluster 0 pgmap v971: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 1.2 GiB data, 5.8 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 642 op/s; 13631/58470 objects degraded (23.313%); 213/58470 objects misplaced (0.364%); 1.4 MiB/s, 21 objects/s recovering 2023-06-03T20:10:01.824 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:10:01 smithi089 bash[75068]: cluster 1685823000.6383855 mgr.y (mgr.34104) 1016 : cluster 0 pgmap v971: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 1.2 GiB data, 5.8 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 642 op/s; 13631/58470 objects degraded (23.313%); 213/58470 objects misplaced (0.364%); 1.4 MiB/s, 21 objects/s recovering 2023-06-03T20:10:01.858 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:10:01.862 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:10:01.862 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:10:01.964 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:10:01.994 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:10:02.003 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:10:02.004 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:10:02.083 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:10:01 smithi164 bash[59462]: cluster 1685823000.6383855 mgr.y (mgr.34104) 1016 : cluster 0 pgmap v971: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 1.2 GiB data, 5.8 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 642 op/s; 13631/58470 objects degraded (23.313%); 213/58470 objects misplaced (0.364%); 1.4 MiB/s, 21 objects/s recovering 2023-06-03T20:10:02.106 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:10:02.135 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:10:02.139 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:10:02.140 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:10:02.241 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:10:02.279 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:10:02.282 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:10:02.283 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:10:02.384 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:10:02.413 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:10:02.417 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:10:02.417 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:10:02.465 INFO:teuthology.orchestra.run.smithi089.stdout:osd.0: {} 2023-06-03T20:10:02.498 INFO:teuthology.orchestra.run.smithi089.stdout:osd.1: {} 2023-06-03T20:10:02.518 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:10:02.542 INFO:teuthology.orchestra.run.smithi089.stdout:osd.2: {} 2023-06-03T20:10:02.551 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:10:02.553 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:10:02.553 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:10:02.588 INFO:teuthology.orchestra.run.smithi089.stdout:osd.3: {} 2023-06-03T20:10:02.632 INFO:teuthology.orchestra.run.smithi089.stdout:osd.4: {} 2023-06-03T20:10:02.654 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:10:02.672 INFO:teuthology.orchestra.run.smithi089.stdout:osd.5: {} 2023-06-03T20:10:02.705 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:10:02.711 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:10:02.711 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:10:02.721 INFO:teuthology.orchestra.run.smithi089.stdout:osd.6: {} 2023-06-03T20:10:02.758 INFO:teuthology.orchestra.run.smithi089.stdout:osd.7: {} 2023-06-03T20:10:02.765 INFO:teuthology.orchestra.run.smithi089.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-06-03T20:10:02.765 INFO:teuthology.orchestra.run.smithi089.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-06-03T20:10:02.766 INFO:teuthology.orchestra.run.smithi089.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-06-03T20:10:02.766 INFO:teuthology.orchestra.run.smithi089.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-06-03T20:10:02.766 INFO:teuthology.orchestra.run.smithi089.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-06-03T20:10:02.766 INFO:teuthology.orchestra.run.smithi089.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-06-03T20:10:02.766 INFO:teuthology.orchestra.run.smithi089.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-06-03T20:10:02.766 INFO:teuthology.orchestra.run.smithi089.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-06-03T20:10:02.778 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:10:02 smithi089 bash[75068]: audit 1685823002.420766 mon.c (mon.2) 480 : audit 0 from='client.? 172.21.15.89:0/573334678' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-03T20:10:02.778 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:10:02 smithi089 bash[75068]: cluster 1685823002.6853185 mon.a (mon.0) 943 : cluster 3 Health check update: Degraded data redundancy: 13631/58470 objects degraded (23.313%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-06-03T20:10:02.778 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:10:02 smithi089 bash[78452]: audit 1685823002.420766 mon.c (mon.2) 480 : audit 0 from='client.? 172.21.15.89:0/573334678' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-03T20:10:02.779 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:10:02 smithi089 bash[78452]: cluster 1685823002.6853185 mon.a (mon.0) 943 : cluster 3 Health check update: Degraded data redundancy: 13631/58470 objects degraded (23.313%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-06-03T20:10:02.812 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:10:02.861 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:10:02.862 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:10:02.862 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:10:02.963 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:10:02.997 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:10:03.001 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:10:03.001 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:10:03.084 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:10:02 smithi164 bash[59462]: audit 1685823002.420766 mon.c (mon.2) 480 : audit 0 from='client.? 172.21.15.89:0/573334678' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-03T20:10:03.084 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:10:02 smithi164 bash[59462]: cluster 1685823002.6853185 mon.a (mon.0) 943 : cluster 3 Health check update: Degraded data redundancy: 13631/58470 objects degraded (23.313%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-06-03T20:10:03.103 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:10:03.138 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:10:03.141 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:10:03.142 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:10:03.243 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:10:03.286 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:10:03.289 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:10:03.290 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:10:03.391 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:10:03.425 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:10:03.428 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:10:03.429 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:10:03.529 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:10:03.560 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:10:03.565 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:10:03.566 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:10:03.668 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:10:03.703 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:10:03.710 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:10:03.710 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:10:03.786 INFO:teuthology.orchestra.run.smithi089.stderr:noscrub is set 2023-06-03T20:10:03.812 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:10:03.842 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:10:03.847 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:10:03.848 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:10:03.949 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:10:03.959 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:10:03 smithi089 bash[75068]: cluster 1685823002.6390445 mgr.y (mgr.34104) 1017 : cluster 0 pgmap v972: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 1.2 GiB data, 5.7 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 643 op/s; 13631/56595 objects degraded (24.085%); 213/56595 objects misplaced (0.376%); 1.4 MiB/s, 21 objects/s recovering 2023-06-03T20:10:03.959 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:10:03 smithi089 bash[75068]: audit 1685823002.765837 mon.c (mon.2) 481 : audit 1 from='client.? 172.21.15.89:0/1165272144' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-06-03T20:10:03.959 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:10:03 smithi089 bash[75068]: audit 1685823002.7663147 mon.a (mon.0) 944 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-06-03T20:10:03.959 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:10:03 smithi089 bash[78452]: cluster 1685823002.6390445 mgr.y (mgr.34104) 1017 : cluster 0 pgmap v972: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 1.2 GiB data, 5.7 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 643 op/s; 13631/56595 objects degraded (24.085%); 213/56595 objects misplaced (0.376%); 1.4 MiB/s, 21 objects/s recovering 2023-06-03T20:10:03.959 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:10:03 smithi089 bash[78452]: audit 1685823002.765837 mon.c (mon.2) 481 : audit 1 from='client.? 172.21.15.89:0/1165272144' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-06-03T20:10:03.960 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:10:03 smithi089 bash[78452]: audit 1685823002.7663147 mon.a (mon.0) 944 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-06-03T20:10:03.988 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:10:03.992 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:10:03.993 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:10:04.083 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:10:03 smithi164 bash[59462]: cluster 1685823002.6390445 mgr.y (mgr.34104) 1017 : cluster 0 pgmap v972: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 1.2 GiB data, 5.7 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 643 op/s; 13631/56595 objects degraded (24.085%); 213/56595 objects misplaced (0.376%); 1.4 MiB/s, 21 objects/s recovering 2023-06-03T20:10:04.083 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:10:03 smithi164 bash[59462]: audit 1685823002.765837 mon.c (mon.2) 481 : audit 1 from='client.? 172.21.15.89:0/1165272144' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-06-03T20:10:04.083 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:10:03 smithi164 bash[59462]: audit 1685823002.7663147 mon.a (mon.0) 944 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-06-03T20:10:04.094 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:10:04.124 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:10:04.127 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:10:04.128 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:10:04.228 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:10:04.261 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:10:04.265 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:10:04.266 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:10:04.367 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:10:04.396 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:10:04.399 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:10:04.400 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:10:04.501 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:10:04.539 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:10:04.545 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:10:04.546 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:10:04.647 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:10:04.677 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:10:04.682 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:10:04.684 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:10:04.785 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:10:04.824 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:10:04.830 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:10:04.830 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:10:04.931 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:10:04.977 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:10:04.984 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:10:04.985 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:10:05.028 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:10:04 smithi089 bash[75068]: cluster 1685823003.7559938 mon.a (mon.0) 945 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-06-03T20:10:05.029 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:10:04 smithi089 bash[75068]: cluster 1685823003.7678025 osd.5 (osd.5) 114 : cluster 3 failed to encode map e182 with expected crc 2023-06-03T20:10:05.029 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:10:04 smithi089 bash[75068]: cluster 1685823003.7714481 osd.6 (osd.6) 121 : cluster 3 failed to encode map e182 with expected crc 2023-06-03T20:10:05.029 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:10:04 smithi089 bash[75068]: audit 1685823003.7719786 mon.a (mon.0) 946 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-06-03T20:10:05.029 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:10:04 smithi089 bash[75068]: cluster 1685823003.772058 mon.a (mon.0) 947 : cluster 0 osdmap e182: 8 total, 8 up, 7 in 2023-06-03T20:10:05.029 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:10:04 smithi089 bash[75068]: cluster 1685823003.778788 osd.7 (osd.7) 118 : cluster 3 failed to encode map e182 with expected crc 2023-06-03T20:10:05.030 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:10:04 smithi089 bash[78452]: cluster 1685823003.7559938 mon.a (mon.0) 945 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-06-03T20:10:05.031 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:10:04 smithi089 bash[78452]: cluster 1685823003.7678025 osd.5 (osd.5) 114 : cluster 3 failed to encode map e182 with expected crc 2023-06-03T20:10:05.031 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:10:04 smithi089 bash[78452]: cluster 1685823003.7714481 osd.6 (osd.6) 121 : cluster 3 failed to encode map e182 with expected crc 2023-06-03T20:10:05.031 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:10:04 smithi089 bash[78452]: audit 1685823003.7719786 mon.a (mon.0) 946 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-06-03T20:10:05.031 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:10:04 smithi089 bash[78452]: cluster 1685823003.772058 mon.a (mon.0) 947 : cluster 0 osdmap e182: 8 total, 8 up, 7 in 2023-06-03T20:10:05.031 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:10:04 smithi089 bash[78452]: cluster 1685823003.778788 osd.7 (osd.7) 118 : cluster 3 failed to encode map e182 with expected crc 2023-06-03T20:10:05.083 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:10:04 smithi164 bash[59462]: cluster 1685823003.7559938 mon.a (mon.0) 945 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-06-03T20:10:05.084 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:10:04 smithi164 bash[59462]: cluster 1685823003.7678025 osd.5 (osd.5) 114 : cluster 3 failed to encode map e182 with expected crc 2023-06-03T20:10:05.084 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:10:04 smithi164 bash[59462]: cluster 1685823003.7714481 osd.6 (osd.6) 121 : cluster 3 failed to encode map e182 with expected crc 2023-06-03T20:10:05.085 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:10:04 smithi164 bash[59462]: audit 1685823003.7719786 mon.a (mon.0) 946 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-06-03T20:10:05.085 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:10:04 smithi164 bash[59462]: cluster 1685823003.772058 mon.a (mon.0) 947 : cluster 0 osdmap e182: 8 total, 8 up, 7 in 2023-06-03T20:10:05.086 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:10:04 smithi164 bash[59462]: cluster 1685823003.778788 osd.7 (osd.7) 118 : cluster 3 failed to encode map e182 with expected crc 2023-06-03T20:10:05.086 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:10:05.133 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:10:05.140 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:10:05.141 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:10:05.242 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:10:05.284 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:10:05.290 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:10:05.290 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:10:05.391 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:10:05.428 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:10:05.431 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:10:05.432 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:10:05.533 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:10:05.572 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:10:05.578 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:10:05.579 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:10:05.631 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/mon.a/config 2023-06-03T20:10:05.680 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:10:05.710 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:10:05.716 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:10:05.717 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:10:05.780 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:10:05 smithi164 bash[59462]: cluster 1685823004.6395533 mgr.y (mgr.34104) 1018 : cluster 0 pgmap v974: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 1.0 GiB data, 5.4 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 704 op/s; 13720/51279 objects degraded (26.756%); 12/51279 objects misplaced (0.023%); 1.7 MiB/s, 27 objects/s recovering 2023-06-03T20:10:05.818 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:10:05.857 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:10:05.860 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:10:05.861 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:10:05.961 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:10:06.007 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:10:06.012 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:10:06.012 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:10:06.028 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:10:05 smithi089 bash[75068]: cluster 1685823004.6395533 mgr.y (mgr.34104) 1018 : cluster 0 pgmap v974: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 1.0 GiB data, 5.4 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 704 op/s; 13720/51279 objects degraded (26.756%); 12/51279 objects misplaced (0.023%); 1.7 MiB/s, 27 objects/s recovering 2023-06-03T20:10:06.028 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:10:05 smithi089 bash[78452]: cluster 1685823004.6395533 mgr.y (mgr.34104) 1018 : cluster 0 pgmap v974: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 1.0 GiB data, 5.4 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 704 op/s; 13720/51279 objects degraded (26.756%); 12/51279 objects misplaced (0.023%); 1.7 MiB/s, 27 objects/s recovering 2023-06-03T20:10:06.113 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:10:06.149 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:10:06.154 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:10:06.155 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:10:06.256 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:10:06.290 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:10:06.295 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:10:06.296 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:10:06.352 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-06-03T20:10:06.397 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:10:06.435 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:10:06.438 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:10:06.439 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:10:06.528 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 20:10:06 smithi089 bash[71163]: ::ffff:172.21.15.164 - - [03/Jun/2023:20:10:06] "GET /metrics HTTP/1.1" 200 228738 "" "Prometheus/2.33.4" 2023-06-03T20:10:06.540 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:10:06.586 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:10:06.589 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:10:06.589 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:10:06.691 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:10:06.738 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:10:06.741 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:10:06.741 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:10:06.842 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:10:06.874 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:10:06.877 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:10:06.878 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:10:06.980 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:10:07.016 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:10:07.018 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:10:07.019 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:10:07.120 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:10:07.166 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:10:07.170 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:10:07.171 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:10:07.230 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph osd set nodeep-scrub 2023-06-03T20:10:07.271 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:10:07.317 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:10:07.321 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:10:07.322 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:10:07.423 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:10:07.457 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:10:07.459 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:10:07.460 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:10:07.561 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:10:07.612 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:10:07.618 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:10:07.618 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:10:07.681 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/mon.a/config 2023-06-03T20:10:07.719 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:10:07.755 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:10:07.761 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:10:07.762 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:10:07.833 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:10:07 smithi164 bash[59462]: cluster 1685823006.640137 mgr.y (mgr.34104) 1019 : cluster 0 pgmap v975: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 1017 MiB data, 5.2 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 657 op/s; 13720/48858 objects degraded (28.081%); 12/48858 objects misplaced (0.025%); 1.5 MiB/s, 24 objects/s recovering 2023-06-03T20:10:07.833 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:10:07 smithi164 bash[59462]: audit 1685823007.121108 mon.a (mon.0) 948 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:10:07.834 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:10:07 smithi164 bash[59462]: audit 1685823007.1260028 mon.c (mon.2) 482 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T20:10:07.834 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:10:07 smithi164 bash[59462]: cluster 1685823007.6864579 mon.a (mon.0) 949 : cluster 3 Health check update: Degraded data redundancy: 13720/48858 objects degraded (28.081%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-06-03T20:10:07.863 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:10:07.874 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:10:07 smithi089 bash[75068]: cluster 1685823006.640137 mgr.y (mgr.34104) 1019 : cluster 0 pgmap v975: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 1017 MiB data, 5.2 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 657 op/s; 13720/48858 objects degraded (28.081%); 12/48858 objects misplaced (0.025%); 1.5 MiB/s, 24 objects/s recovering 2023-06-03T20:10:07.875 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:10:07 smithi089 bash[75068]: audit 1685823007.121108 mon.a (mon.0) 948 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:10:07.875 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:10:07 smithi089 bash[75068]: audit 1685823007.1260028 mon.c (mon.2) 482 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T20:10:07.875 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:10:07 smithi089 bash[75068]: cluster 1685823007.6864579 mon.a (mon.0) 949 : cluster 3 Health check update: Degraded data redundancy: 13720/48858 objects degraded (28.081%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-06-03T20:10:07.876 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:10:07 smithi089 bash[78452]: cluster 1685823006.640137 mgr.y (mgr.34104) 1019 : cluster 0 pgmap v975: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 1017 MiB data, 5.2 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 657 op/s; 13720/48858 objects degraded (28.081%); 12/48858 objects misplaced (0.025%); 1.5 MiB/s, 24 objects/s recovering 2023-06-03T20:10:07.876 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:10:07 smithi089 bash[78452]: audit 1685823007.121108 mon.a (mon.0) 948 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:10:07.876 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:10:07 smithi089 bash[78452]: audit 1685823007.1260028 mon.c (mon.2) 482 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T20:10:07.877 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:10:07 smithi089 bash[78452]: cluster 1685823007.6864579 mon.a (mon.0) 949 : cluster 3 Health check update: Degraded data redundancy: 13720/48858 objects degraded (28.081%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-06-03T20:10:07.900 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:10:07.902 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:10:07.903 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:10:08.004 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:10:08.035 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:10:08.038 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:10:08.039 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:10:08.139 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:10:08.179 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:10:08.182 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:10:08.183 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:10:08.283 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:10:08.342 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:10:08.348 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:10:08.348 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:10:08.449 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:10:08.481 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:10:08.486 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:10:08.487 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:10:08.588 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:10:08.621 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:10:08.626 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:10:08.626 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:10:08.727 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:10:08.764 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:10:08.768 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:10:08.769 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:10:08.869 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:10:08.903 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:10:08.905 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:10:08.906 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:10:09.006 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:10:09.044 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:10:09.046 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:10:09.047 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:10:09.148 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:10:09.193 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:10:09.199 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:10:09.200 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:10:09.302 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:10:09.344 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:10:09.346 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:10:09.347 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:10:09.448 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:10:09.483 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:10:09.485 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:10:09.486 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:10:09.587 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:10:09.618 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:10:09.626 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:10:09.627 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:10:09.728 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:10:09.761 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:10:09.766 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:10:09.767 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:10:09.792 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:10:09 smithi164 bash[59462]: cluster 1685823008.640841 mgr.y (mgr.34104) 1020 : cluster 0 pgmap v976: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 977 MiB data, 5.1 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 651 op/s; 13720/46950 objects degraded (29.223%); 12/46950 objects misplaced (0.026%); 1.4 MiB/s, 22 objects/s recovering 2023-06-03T20:10:09.868 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:10:09.877 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:10:09 smithi089 bash[75068]: cluster 1685823008.640841 mgr.y (mgr.34104) 1020 : cluster 0 pgmap v976: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 977 MiB data, 5.1 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 651 op/s; 13720/46950 objects degraded (29.223%); 12/46950 objects misplaced (0.026%); 1.4 MiB/s, 22 objects/s recovering 2023-06-03T20:10:09.877 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:10:09 smithi089 bash[78452]: cluster 1685823008.640841 mgr.y (mgr.34104) 1020 : cluster 0 pgmap v976: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 977 MiB data, 5.1 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 651 op/s; 13720/46950 objects degraded (29.223%); 12/46950 objects misplaced (0.026%); 1.4 MiB/s, 22 objects/s recovering 2023-06-03T20:10:09.902 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:10:09.904 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:10:09.905 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:10:10.006 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:10:10.049 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:10:10.053 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:10:10.054 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:10:10.154 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:10:10.188 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:10:10.190 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:10:10.191 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:10:10.291 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:10:10.338 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:10:10.341 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:10:10.341 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:10:10.442 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:10:10.485 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:10:10.490 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:10:10.491 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:10:10.591 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:10:10.635 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:10:10.636 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:10:10.636 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:10:10.737 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:10:10.787 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:10:10.789 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:10:10.790 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:10:10.891 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:10:10.921 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:10:10.926 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:10:10.927 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:10:11.028 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:10:11.066 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:10:11.069 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:10:11.070 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:10:11.170 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:10:11.219 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:10:11.222 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:10:11.223 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:10:11.323 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:10:11.359 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:10:11.364 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:10:11.365 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:10:11.466 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:10:11.506 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:10:11.512 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:10:11.513 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:10:11.614 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:10:11.665 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:10:11.668 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:10:11.668 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:10:11.769 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:10:11.797 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/mon.a/config 2023-06-03T20:10:11.802 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/mon.a/config 2023-06-03T20:10:11.807 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:10:11.809 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:10:11.809 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:10:11.910 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:10:11.917 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:10:11 smithi164 bash[59462]: cluster 1685823010.6414647 mgr.y (mgr.34104) 1021 : cluster 0 pgmap v977: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 808 MiB data, 4.7 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 654 op/s; 13624/38820 objects degraded (35.095%); 755 KiB/s, 21 objects/s recovering 2023-06-03T20:10:11.939 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:10:11.941 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:10:11.942 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:10:12.028 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:10:11 smithi089 bash[78452]: cluster 1685823010.6414647 mgr.y (mgr.34104) 1021 : cluster 0 pgmap v977: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 808 MiB data, 4.7 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 654 op/s; 13624/38820 objects degraded (35.095%); 755 KiB/s, 21 objects/s recovering 2023-06-03T20:10:12.029 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:10:11 smithi089 bash[75068]: cluster 1685823010.6414647 mgr.y (mgr.34104) 1021 : cluster 0 pgmap v977: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 808 MiB data, 4.7 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 654 op/s; 13624/38820 objects degraded (35.095%); 755 KiB/s, 21 objects/s recovering 2023-06-03T20:10:12.043 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:10:12.100 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:10:12.105 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:10:12.106 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:10:12.207 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:10:12.245 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:10:12.247 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:10:12.247 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:10:12.348 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:10:12.376 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:10:12.382 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:10:12.383 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:10:12.484 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:10:12.485 INFO:teuthology.orchestra.run.smithi089.stdout:ERROR: (22) Invalid argument 2023-06-03T20:10:12.485 INFO:teuthology.orchestra.run.smithi089.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-06-03T20:10:12.520 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:10:12.525 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:10:12.526 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:10:12.627 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:10:12.657 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:10:12.663 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:10:12.664 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:10:12.765 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:10:12.798 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:10:12.802 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:10:12.803 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:10:12.904 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:10:12.913 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:10:12 smithi164 bash[59462]: cluster 1685823012.6875637 mon.a (mon.0) 950 : cluster 3 Health check update: Degraded data redundancy: 13624/38820 objects degraded (35.095%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-06-03T20:10:12.935 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:10:12.940 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:10:12.941 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:10:13.028 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:10:12 smithi089 bash[78452]: cluster 1685823012.6875637 mon.a (mon.0) 950 : cluster 3 Health check update: Degraded data redundancy: 13624/38820 objects degraded (35.095%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-06-03T20:10:13.028 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:10:12 smithi089 bash[75068]: cluster 1685823012.6875637 mon.a (mon.0) 950 : cluster 3 Health check update: Degraded data redundancy: 13624/38820 objects degraded (35.095%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-06-03T20:10:13.042 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:10:13.070 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:10:13.072 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:10:13.073 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:10:13.174 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:10:13.203 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:10:13.206 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:10:13.207 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:10:13.307 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:10:13.346 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:10:13.349 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:10:13.350 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:10:13.451 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:10:13.484 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:10:13.487 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:10:13.488 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:10:13.589 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:10:13.627 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:10:13.629 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:10:13.630 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:10:13.731 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:10:13.775 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:10:13.778 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:10:13.780 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:10:13.881 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:10:13.919 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:10:13.921 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:10:13.921 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:10:14.022 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:10:14.031 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:10:13 smithi164 bash[59462]: cluster 1685823012.6422913 mgr.y (mgr.34104) 1022 : cluster 0 pgmap v978: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 768 MiB data, 4.5 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 655 op/s; 13624/36912 objects degraded (36.909%); 755 KiB/s, 21 objects/s recovering 2023-06-03T20:10:14.078 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:10:14.085 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:10:14.086 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:10:14.186 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:10:14.218 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:10:13 smithi089 bash[78452]: cluster 1685823012.6422913 mgr.y (mgr.34104) 1022 : cluster 0 pgmap v978: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 768 MiB data, 4.5 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 655 op/s; 13624/36912 objects degraded (36.909%); 755 KiB/s, 21 objects/s recovering 2023-06-03T20:10:14.218 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:10:13 smithi089 bash[75068]: cluster 1685823012.6422913 mgr.y (mgr.34104) 1022 : cluster 0 pgmap v978: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 768 MiB data, 4.5 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 655 op/s; 13624/36912 objects degraded (36.909%); 755 KiB/s, 21 objects/s recovering 2023-06-03T20:10:14.231 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:10:14.233 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:10:14.234 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:10:14.335 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:10:14.376 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:10:14.387 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:10:14.388 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:10:14.489 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:10:14.522 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:10:14.528 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:10:14.529 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:10:14.630 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:10:14.676 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:10:14.679 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:10:14.679 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:10:14.780 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:10:14.826 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:10:14.829 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:10:14.830 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:10:14.931 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:10:14.968 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:10:14.975 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:10:14.976 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:10:15.076 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:10:15.108 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:10:15.110 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:10:15.111 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:10:15.212 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:10:15.253 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:10:15.258 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:10:15.258 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:10:15.359 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:10:15.397 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:10:15.401 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:10:15.402 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:10:15.502 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:10:15.534 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:10:15.537 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:10:15.538 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:10:15.639 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:10:15.668 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:10:15.670 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:10:15.671 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:10:15.772 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:10:15.813 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:10:15.817 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:10:15.818 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:10:15.919 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:10:15.953 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:10:15.961 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:10:15.961 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:10:16.062 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:10:16.072 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:10:15 smithi089 bash[75068]: cluster 1685823014.6428938 mgr.y (mgr.34104) 1023 : cluster 0 pgmap v979: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 652 MiB data, 4.3 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 719 op/s; 13514/31341 objects degraded (43.119%); 695 KiB/s, 30 objects/s recovering 2023-06-03T20:10:16.072 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:10:15 smithi089 bash[78452]: cluster 1685823014.6428938 mgr.y (mgr.34104) 1023 : cluster 0 pgmap v979: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 652 MiB data, 4.3 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 719 op/s; 13514/31341 objects degraded (43.119%); 695 KiB/s, 30 objects/s recovering 2023-06-03T20:10:16.083 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:10:15 smithi164 bash[59462]: cluster 1685823014.6428938 mgr.y (mgr.34104) 1023 : cluster 0 pgmap v979: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 652 MiB data, 4.3 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 719 op/s; 13514/31341 objects degraded (43.119%); 695 KiB/s, 30 objects/s recovering 2023-06-03T20:10:16.096 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:10:16.101 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:10:16.102 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:10:16.203 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:10:16.235 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:10:16.237 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:10:16.239 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:10:16.340 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:10:16.373 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:10:16.378 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:10:16.378 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:10:16.480 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:10:16.492 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 20:10:16 smithi089 bash[71163]: ::ffff:172.21.15.164 - - [03/Jun/2023:20:10:16] "GET /metrics HTTP/1.1" 200 228765 "" "Prometheus/2.33.4" 2023-06-03T20:10:16.522 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:10:16.526 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:10:16.528 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:10:16.628 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:10:16.670 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:10:16.674 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:10:16.675 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:10:16.776 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:10:16.808 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:10:16.814 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:10:16.814 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:10:16.915 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:10:16.952 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:10:16.957 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:10:16.958 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:10:17.059 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:10:17.097 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:10:17.101 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:10:17.101 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:10:17.204 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:10:17.252 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:10:17.255 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:10:17.256 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:10:17.357 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:10:17.393 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:10:17.398 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:10:17.399 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:10:17.500 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:10:17.535 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:10:17.537 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:10:17.538 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:10:17.608 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph daemon osd.3 dump_blocked_ops 2023-06-03T20:10:17.639 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:10:17.681 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:10:17.683 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:10:17.684 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:10:17.785 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:10:17.834 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:10:17.836 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:10:17.837 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:10:17.938 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:10:17.972 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:10:17.978 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:10:17.979 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:10:18.028 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:10:17 smithi089 bash[75068]: cluster 1685823016.6434946 mgr.y (mgr.34104) 1024 : cluster 0 pgmap v980: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 602 MiB data, 4.1 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 620 op/s; 13514/28935 objects degraded (46.705%); 32 KiB/s, 18 objects/s recovering 2023-06-03T20:10:18.028 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:10:17 smithi089 bash[75068]: cluster 1685823017.6886787 mon.a (mon.0) 951 : cluster 3 Health check update: Degraded data redundancy: 13514/28935 objects degraded (46.705%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-06-03T20:10:18.029 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:10:17 smithi089 bash[78452]: cluster 1685823016.6434946 mgr.y (mgr.34104) 1024 : cluster 0 pgmap v980: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 602 MiB data, 4.1 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 620 op/s; 13514/28935 objects degraded (46.705%); 32 KiB/s, 18 objects/s recovering 2023-06-03T20:10:18.029 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:10:17 smithi089 bash[78452]: cluster 1685823017.6886787 mon.a (mon.0) 951 : cluster 3 Health check update: Degraded data redundancy: 13514/28935 objects degraded (46.705%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-06-03T20:10:18.080 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:10:18.083 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:10:17 smithi164 bash[59462]: cluster 1685823016.6434946 mgr.y (mgr.34104) 1024 : cluster 0 pgmap v980: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 602 MiB data, 4.1 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 620 op/s; 13514/28935 objects degraded (46.705%); 32 KiB/s, 18 objects/s recovering 2023-06-03T20:10:18.083 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:10:17 smithi164 bash[59462]: cluster 1685823017.6886787 mon.a (mon.0) 951 : cluster 3 Health check update: Degraded data redundancy: 13514/28935 objects degraded (46.705%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-06-03T20:10:18.119 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:10:18.128 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:10:18.129 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:10:18.230 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:10:18.262 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:10:18.268 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:10:18.269 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:10:18.370 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:10:18.405 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:10:18.414 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:10:18.415 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:10:18.516 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:10:18.547 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:10:18.550 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:10:18.551 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:10:18.652 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:10:18.692 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:10:18.695 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:10:18.696 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:10:18.797 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:10:18.827 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:10:18.828 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:10:18.829 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:10:18.930 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:10:18.966 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:10:18.973 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:10:18.973 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:10:19.074 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:10:19.104 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:10:19.109 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:10:19.110 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:10:19.211 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:10:19.245 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:10:19.251 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:10:19.252 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:10:19.353 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:10:19.389 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:10:19.394 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:10:19.395 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:10:19.496 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:10:19.534 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:10:19.536 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:10:19.537 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:10:19.638 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:10:19.666 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:10:19.675 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:10:19.676 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:10:19.776 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:10:19.824 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:10:19.830 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:10:19.831 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:10:19.849 INFO:teuthology.orchestra.run.smithi089.stderr:reweighted osd.5 to 1 (10000) 2023-06-03T20:10:19.899 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:10:19 smithi089 bash[78452]: cluster 1685823018.6440358 mgr.y (mgr.34104) 1025 : cluster 0 pgmap v981: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 562 MiB data, 3.9 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 605 op/s; 13514/27042 objects degraded (49.974%); 32 KiB/s, 18 objects/s recovering 2023-06-03T20:10:19.900 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:10:19 smithi089 bash[78452]: audit 1685823019.6379974 mon.c (mon.2) 483 : audit 1 from='client.? 172.21.15.89:0/1899006225' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-06-03T20:10:19.900 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:10:19 smithi089 bash[78452]: audit 1685823019.6384242 mon.a (mon.0) 952 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-06-03T20:10:19.900 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:10:19 smithi089 bash[75068]: cluster 1685823018.6440358 mgr.y (mgr.34104) 1025 : cluster 0 pgmap v981: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 562 MiB data, 3.9 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 605 op/s; 13514/27042 objects degraded (49.974%); 32 KiB/s, 18 objects/s recovering 2023-06-03T20:10:19.900 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:10:19 smithi089 bash[75068]: audit 1685823019.6379974 mon.c (mon.2) 483 : audit 1 from='client.? 172.21.15.89:0/1899006225' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-06-03T20:10:19.900 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:10:19 smithi089 bash[75068]: audit 1685823019.6384242 mon.a (mon.0) 952 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-06-03T20:10:19.932 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:10:19.964 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:10:19.966 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:10:19.967 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:10:20.068 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:10:20.077 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:10:19 smithi164 bash[59462]: cluster 1685823018.6440358 mgr.y (mgr.34104) 1025 : cluster 0 pgmap v981: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 562 MiB data, 3.9 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 605 op/s; 13514/27042 objects degraded (49.974%); 32 KiB/s, 18 objects/s recovering 2023-06-03T20:10:20.078 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:10:19 smithi164 bash[59462]: audit 1685823019.6379974 mon.c (mon.2) 483 : audit 1 from='client.? 172.21.15.89:0/1899006225' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-06-03T20:10:20.078 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:10:19 smithi164 bash[59462]: audit 1685823019.6384242 mon.a (mon.0) 952 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-06-03T20:10:20.111 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:10:20.115 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:10:20.116 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:10:20.217 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:10:20.251 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:10:20.254 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:10:20.255 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:10:20.355 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:10:20.389 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:10:20.394 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:10:20.395 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:10:20.496 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:10:20.524 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:10:20.526 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:10:20.527 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:10:20.628 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:10:20.662 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:10:20.669 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:10:20.670 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:10:20.771 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:10:20.805 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:10:20.808 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:10:20.809 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:10:20.911 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:10:20.942 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:10:20.945 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:10:20.945 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:10:21.046 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:10:21.056 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:10:20 smithi089 bash[75068]: cluster 1685823019.8304956 osd.6 (osd.6) 122 : cluster 3 failed to encode map e183 with expected crc 2023-06-03T20:10:21.057 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:10:20 smithi089 bash[75068]: audit 1685823019.8339674 mon.a (mon.0) 953 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2023-06-03T20:10:21.057 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:10:20 smithi089 bash[75068]: cluster 1685823019.8340542 mon.a (mon.0) 954 : cluster 0 osdmap e183: 8 total, 8 up, 7 in 2023-06-03T20:10:21.057 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:10:20 smithi089 bash[75068]: cluster 1685823019.8394346 osd.6 (osd.6) 123 : cluster 3 failed to encode map e183 with expected crc 2023-06-03T20:10:21.057 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:10:20 smithi089 bash[75068]: cluster 1685823019.8410563 osd.6 (osd.6) 124 : cluster 3 failed to encode map e183 with expected crc 2023-06-03T20:10:21.057 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:10:20 smithi089 bash[78452]: cluster 1685823019.8304956 osd.6 (osd.6) 122 : cluster 3 failed to encode map e183 with expected crc 2023-06-03T20:10:21.058 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:10:20 smithi089 bash[78452]: audit 1685823019.8339674 mon.a (mon.0) 953 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2023-06-03T20:10:21.058 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:10:20 smithi089 bash[78452]: cluster 1685823019.8340542 mon.a (mon.0) 954 : cluster 0 osdmap e183: 8 total, 8 up, 7 in 2023-06-03T20:10:21.058 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:10:20 smithi089 bash[78452]: cluster 1685823019.8394346 osd.6 (osd.6) 123 : cluster 3 failed to encode map e183 with expected crc 2023-06-03T20:10:21.058 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:10:20 smithi089 bash[78452]: cluster 1685823019.8410563 osd.6 (osd.6) 124 : cluster 3 failed to encode map e183 with expected crc 2023-06-03T20:10:21.076 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:10:21.082 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:10:21.083 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:10:21.083 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:10:20 smithi164 bash[59462]: cluster 1685823019.8304956 osd.6 (osd.6) 122 : cluster 3 failed to encode map e183 with expected crc 2023-06-03T20:10:21.083 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:10:20 smithi164 bash[59462]: audit 1685823019.8339674 mon.a (mon.0) 953 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2023-06-03T20:10:21.083 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:10:20 smithi164 bash[59462]: cluster 1685823019.8340542 mon.a (mon.0) 954 : cluster 0 osdmap e183: 8 total, 8 up, 7 in 2023-06-03T20:10:21.084 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:10:20 smithi164 bash[59462]: cluster 1685823019.8394346 osd.6 (osd.6) 123 : cluster 3 failed to encode map e183 with expected crc 2023-06-03T20:10:21.084 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:10:20 smithi164 bash[59462]: cluster 1685823019.8410563 osd.6 (osd.6) 124 : cluster 3 failed to encode map e183 with expected crc 2023-06-03T20:10:21.183 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:10:21.205 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:10:21.206 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:10:21.207 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:10:21.308 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:10:21.335 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:10:21.338 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:10:21.339 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:10:21.440 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:10:21.481 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:10:21.486 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:10:21.487 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:10:21.588 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:10:21.624 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:10:21.630 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:10:21.631 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:10:21.732 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:10:21.763 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:10:21.767 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:10:21.768 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:10:21.869 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:10:21.902 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:10:21.904 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:10:21.905 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:10:22.005 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:10:22.029 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:10:21 smithi089 bash[75068]: cluster 1685823020.6446743 mgr.y (mgr.34104) 1026 : cluster 0 pgmap v983: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 395 MiB data, 3.5 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 660 op/s; 13409/18993 objects degraded (70.600%); 0 B/s, 21 objects/s recovering 2023-06-03T20:10:22.030 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:10:21 smithi089 bash[78452]: cluster 1685823020.6446743 mgr.y (mgr.34104) 1026 : cluster 0 pgmap v983: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 395 MiB data, 3.5 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 660 op/s; 13409/18993 objects degraded (70.600%); 0 B/s, 21 objects/s recovering 2023-06-03T20:10:22.060 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:10:22.063 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:10:22.064 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:10:22.165 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:10:22.176 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:10:21 smithi164 bash[59462]: cluster 1685823020.6446743 mgr.y (mgr.34104) 1026 : cluster 0 pgmap v983: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 395 MiB data, 3.5 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 660 op/s; 13409/18993 objects degraded (70.600%); 0 B/s, 21 objects/s recovering 2023-06-03T20:10:22.206 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:10:22.209 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:10:22.210 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:10:22.311 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:10:22.346 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:10:22.350 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:10:22.351 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:10:22.452 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:10:22.483 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:10:22.486 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:10:22.487 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:10:22.588 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:10:22.639 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:10:22.641 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:10:22.642 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:10:22.743 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:10:22.783 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:10:22.787 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:10:22.788 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:10:22.889 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:10:22.921 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:10:22.923 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:10:22.924 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:10:23.025 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:10:23.072 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:10:23.074 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:10:23.075 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:10:23.154 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:10:22 smithi089 bash[78452]: audit 1685823022.1139278 mon.c (mon.2) 484 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T20:10:23.154 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:10:22 smithi089 bash[78452]: cluster 1685823022.6900656 mon.a (mon.0) 955 : cluster 3 Health check update: Degraded data redundancy: 13409/18993 objects degraded (70.600%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-06-03T20:10:23.155 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:10:22 smithi089 bash[75068]: audit 1685823022.1139278 mon.c (mon.2) 484 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T20:10:23.155 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:10:22 smithi089 bash[75068]: cluster 1685823022.6900656 mon.a (mon.0) 955 : cluster 3 Health check update: Degraded data redundancy: 13409/18993 objects degraded (70.600%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-06-03T20:10:23.156 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph osd reweight 6 1 2023-06-03T20:10:23.176 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:10:23.186 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:10:22 smithi164 bash[59462]: audit 1685823022.1139278 mon.c (mon.2) 484 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T20:10:23.186 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:10:22 smithi164 bash[59462]: cluster 1685823022.6900656 mon.a (mon.0) 955 : cluster 3 Health check update: Degraded data redundancy: 13409/18993 objects degraded (70.600%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-06-03T20:10:23.206 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:10:23.212 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:10:23.213 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:10:23.314 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:10:23.353 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:10:23.359 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:10:23.360 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:10:23.461 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:10:23.492 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:10:23.503 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:10:23.504 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:10:23.605 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:10:23.640 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:10:23.645 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:10:23.646 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:10:23.747 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:10:23.800 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:10:23.807 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:10:23.808 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:10:23.908 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:10:23.942 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:10:23.947 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:10:23.948 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:10:24.049 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:10:24.059 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:10:23 smithi164 bash[59462]: cluster 1685823022.6490576 mgr.y (mgr.34104) 1027 : cluster 0 pgmap v984: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 360 MiB data, 3.4 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 652 op/s; 13409/17310 objects degraded (77.464%); 0 B/s, 21 objects/s recovering 2023-06-03T20:10:24.093 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:10:24.100 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:10:24.100 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:10:24.201 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:10:24.254 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:10:24.259 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:10:24.260 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:10:24.278 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:10:23 smithi089 bash[78452]: cluster 1685823022.6490576 mgr.y (mgr.34104) 1027 : cluster 0 pgmap v984: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 360 MiB data, 3.4 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 652 op/s; 13409/17310 objects degraded (77.464%); 0 B/s, 21 objects/s recovering 2023-06-03T20:10:24.278 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:10:23 smithi089 bash[75068]: cluster 1685823022.6490576 mgr.y (mgr.34104) 1027 : cluster 0 pgmap v984: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 360 MiB data, 3.4 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 652 op/s; 13409/17310 objects degraded (77.464%); 0 B/s, 21 objects/s recovering 2023-06-03T20:10:24.360 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:10:24.389 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:10:24.393 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:10:24.394 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:10:24.495 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:10:24.524 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:10:24.532 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:10:24.533 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:10:24.633 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:10:24.667 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:10:24.672 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:10:24.672 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:10:24.773 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:10:24.806 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:10:24.808 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:10:24.809 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:10:24.911 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:10:24.967 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:10:24.972 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:10:24.973 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:10:25.074 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:10:25.138 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:10:25.139 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:10:25.139 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:10:25.240 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:10:25.288 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:10:25.290 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:10:25.290 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:10:25.391 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:10:25.420 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:10:25.422 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:10:25.423 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:10:25.523 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:10:25.565 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:10:25.569 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:10:25.569 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:10:25.670 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:10:25.671 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/mon.a/config 2023-06-03T20:10:25.709 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:10:25.716 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:10:25.717 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:10:25.776 INFO:teuthology.orchestra.run.smithi089.stdout:osd.0: {} 2023-06-03T20:10:25.818 INFO:teuthology.orchestra.run.smithi089.stdout:osd.1: {} 2023-06-03T20:10:25.819 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:10:25.849 INFO:teuthology.orchestra.run.smithi089.stdout:osd.2: {} 2023-06-03T20:10:25.874 INFO:teuthology.orchestra.run.smithi089.stdout:osd.3: {} 2023-06-03T20:10:25.875 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:10:25.880 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:10:25.881 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:10:25.904 INFO:teuthology.orchestra.run.smithi089.stdout:osd.4: {} 2023-06-03T20:10:25.935 INFO:teuthology.orchestra.run.smithi089.stdout:osd.5: {} 2023-06-03T20:10:25.981 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:10:25.988 INFO:teuthology.orchestra.run.smithi089.stdout:osd.6: {} 2023-06-03T20:10:26.022 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:10:26.024 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:10:26.025 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:10:26.028 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:10:25 smithi089 bash[78452]: cluster 1685823024.649963 mgr.y (mgr.34104) 1028 : cluster 0 pgmap v985: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 248 MiB data, 3.1 GiB used, 623 GiB / 626 GiB avail; 0 B/s wr, 646 op/s; 13297/11943 objects degraded (111.337%); 0 B/s, 21 objects/s recovering 2023-06-03T20:10:26.028 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:10:25 smithi089 bash[78452]: audit 1685823025.7310288 mon.c (mon.2) 485 : audit 0 from='client.? 172.21.15.89:0/1604138322' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-03T20:10:26.028 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:10:25 smithi089 bash[75068]: cluster 1685823024.649963 mgr.y (mgr.34104) 1028 : cluster 0 pgmap v985: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 248 MiB data, 3.1 GiB used, 623 GiB / 626 GiB avail; 0 B/s wr, 646 op/s; 13297/11943 objects degraded (111.337%); 0 B/s, 21 objects/s recovering 2023-06-03T20:10:26.028 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:10:25 smithi089 bash[75068]: audit 1685823025.7310288 mon.c (mon.2) 485 : audit 0 from='client.? 172.21.15.89:0/1604138322' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-03T20:10:26.031 INFO:teuthology.orchestra.run.smithi089.stdout:osd.7: {} 2023-06-03T20:10:26.033 INFO:teuthology.orchestra.run.smithi089.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-06-03T20:10:26.033 INFO:teuthology.orchestra.run.smithi089.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-06-03T20:10:26.033 INFO:teuthology.orchestra.run.smithi089.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-06-03T20:10:26.034 INFO:teuthology.orchestra.run.smithi089.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-06-03T20:10:26.034 INFO:teuthology.orchestra.run.smithi089.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-06-03T20:10:26.034 INFO:teuthology.orchestra.run.smithi089.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-06-03T20:10:26.034 INFO:teuthology.orchestra.run.smithi089.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-06-03T20:10:26.034 INFO:teuthology.orchestra.run.smithi089.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-06-03T20:10:26.083 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:10:25 smithi164 bash[59462]: cluster 1685823024.649963 mgr.y (mgr.34104) 1028 : cluster 0 pgmap v985: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 248 MiB data, 3.1 GiB used, 623 GiB / 626 GiB avail; 0 B/s wr, 646 op/s; 13297/11943 objects degraded (111.337%); 0 B/s, 21 objects/s recovering 2023-06-03T20:10:26.083 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:10:25 smithi164 bash[59462]: audit 1685823025.7310288 mon.c (mon.2) 485 : audit 0 from='client.? 172.21.15.89:0/1604138322' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-03T20:10:26.126 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:10:26.154 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:10:26.156 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:10:26.157 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:10:26.259 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:10:26.303 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:10:26.310 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:10:26.311 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:10:26.412 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:10:26.458 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:10:26.463 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:10:26.464 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:10:26.512 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 20:10:26 smithi089 bash[71163]: ::ffff:172.21.15.164 - - [03/Jun/2023:20:10:26] "GET /metrics HTTP/1.1" 200 228752 "" "Prometheus/2.33.4" 2023-06-03T20:10:26.565 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:10:26.594 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:10:26.598 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:10:26.598 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:10:26.699 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:10:26.730 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:10:26.733 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:10:26.734 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:10:26.835 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:10:26.876 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:10:26.882 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:10:26.883 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:10:26.896 INFO:teuthology.orchestra.run.smithi089.stderr:nodeep-scrub is set 2023-06-03T20:10:26.984 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:10:27.019 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:10:27.027 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:10:27.028 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:10:27.129 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:10:27.138 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:10:26 smithi164 bash[59462]: audit 1685823026.0747464 mon.a (mon.0) 956 : audit 1 from='client.? 172.21.15.89:0/1454291074' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-06-03T20:10:27.198 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:10:27.202 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:10:27.203 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:10:27.278 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:10:26 smithi089 bash[78452]: audit 1685823026.0747464 mon.a (mon.0) 956 : audit 1 from='client.? 172.21.15.89:0/1454291074' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-06-03T20:10:27.279 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:10:26 smithi089 bash[75068]: audit 1685823026.0747464 mon.a (mon.0) 956 : audit 1 from='client.? 172.21.15.89:0/1454291074' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-06-03T20:10:27.305 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:10:27.360 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:10:27.363 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:10:27.364 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:10:27.465 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:10:27.519 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:10:27.521 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:10:27.522 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:10:27.622 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:10:27.663 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:10:27.672 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:10:27.674 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:10:27.775 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:10:27.835 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:10:27.841 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:10:27.842 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:10:27.943 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:10:27.973 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:10:27.975 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:10:27.976 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:10:28.042 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:10:27 smithi089 bash[75068]: cluster 1685823026.6506195 mgr.y (mgr.34104) 1029 : cluster 0 pgmap v986: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 197 MiB data, 2.9 GiB used, 623 GiB / 626 GiB avail; 0 B/s wr, 646 op/s; 13297/9513 objects degraded (139.777%); 0 B/s, 21 objects/s recovering 2023-06-03T20:10:28.043 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:10:27 smithi089 bash[75068]: cluster 1685823026.860416 mon.a (mon.0) 957 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-06-03T20:10:28.043 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:10:27 smithi089 bash[75068]: cluster 1685823026.878489 osd.5 (osd.5) 115 : cluster 3 failed to encode map e184 with expected crc 2023-06-03T20:10:28.043 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:10:27 smithi089 bash[75068]: audit 1685823026.8850427 mon.a (mon.0) 958 : audit 1 from='client.? 172.21.15.89:0/1454291074' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-06-03T20:10:28.043 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:10:27 smithi089 bash[75068]: cluster 1685823026.8852794 mon.a (mon.0) 959 : cluster 0 osdmap e184: 8 total, 8 up, 7 in 2023-06-03T20:10:28.043 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:10:27 smithi089 bash[75068]: cluster 1685823026.885352 osd.6 (osd.6) 125 : cluster 3 failed to encode map e184 with expected crc 2023-06-03T20:10:28.043 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:10:27 smithi089 bash[75068]: cluster 1685823026.8939908 osd.6 (osd.6) 126 : cluster 3 failed to encode map e184 with expected crc 2023-06-03T20:10:28.043 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:10:27 smithi089 bash[75068]: audit 1685823027.0552998 mon.c (mon.2) 486 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-03T20:10:28.043 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:10:27 smithi089 bash[75068]: cluster 1685823027.6913903 mon.a (mon.0) 960 : cluster 3 Health check update: Degraded data redundancy: 13297/9513 objects degraded (139.777%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-06-03T20:10:28.077 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:10:28.083 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:10:27 smithi164 bash[59462]: cluster 1685823026.6506195 mgr.y (mgr.34104) 1029 : cluster 0 pgmap v986: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 197 MiB data, 2.9 GiB used, 623 GiB / 626 GiB avail; 0 B/s wr, 646 op/s; 13297/9513 objects degraded (139.777%); 0 B/s, 21 objects/s recovering 2023-06-03T20:10:28.083 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:10:27 smithi164 bash[59462]: cluster 1685823026.860416 mon.a (mon.0) 957 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-06-03T20:10:28.084 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:10:27 smithi164 bash[59462]: cluster 1685823026.878489 osd.5 (osd.5) 115 : cluster 3 failed to encode map e184 with expected crc 2023-06-03T20:10:28.084 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:10:27 smithi164 bash[59462]: audit 1685823026.8850427 mon.a (mon.0) 958 : audit 1 from='client.? 172.21.15.89:0/1454291074' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-06-03T20:10:28.084 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:10:27 smithi164 bash[59462]: cluster 1685823026.8852794 mon.a (mon.0) 959 : cluster 0 osdmap e184: 8 total, 8 up, 7 in 2023-06-03T20:10:28.084 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:10:27 smithi164 bash[59462]: cluster 1685823026.885352 osd.6 (osd.6) 125 : cluster 3 failed to encode map e184 with expected crc 2023-06-03T20:10:28.084 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:10:27 smithi164 bash[59462]: cluster 1685823026.8939908 osd.6 (osd.6) 126 : cluster 3 failed to encode map e184 with expected crc 2023-06-03T20:10:28.084 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:10:27 smithi164 bash[59462]: audit 1685823027.0552998 mon.c (mon.2) 486 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-03T20:10:28.085 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:10:27 smithi164 bash[59462]: cluster 1685823027.6913903 mon.a (mon.0) 960 : cluster 3 Health check update: Degraded data redundancy: 13297/9513 objects degraded (139.777%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-06-03T20:10:28.100 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:10:28.102 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:10:28.102 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:10:28.203 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:10:28.243 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:10:28.246 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:10:28.246 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:10:28.347 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:10:28.355 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:10:28 smithi089 bash[78452]: cluster 1685823026.6506195 mgr.y (mgr.34104) 1029 : cluster 0 pgmap v986: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 197 MiB data, 2.9 GiB used, 623 GiB / 626 GiB avail; 0 B/s wr, 646 op/s; 13297/9513 objects degraded (139.777%); 0 B/s, 21 objects/s recovering 2023-06-03T20:10:28.356 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:10:28 smithi089 bash[78452]: cluster 1685823026.860416 mon.a (mon.0) 957 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-06-03T20:10:28.356 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:10:28 smithi089 bash[78452]: cluster 1685823026.878489 osd.5 (osd.5) 115 : cluster 3 failed to encode map e184 with expected crc 2023-06-03T20:10:28.356 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:10:28 smithi089 bash[78452]: audit 1685823026.8850427 mon.a (mon.0) 958 : audit 1 from='client.? 172.21.15.89:0/1454291074' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-06-03T20:10:28.356 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:10:28 smithi089 bash[78452]: cluster 1685823026.8852794 mon.a (mon.0) 959 : cluster 0 osdmap e184: 8 total, 8 up, 7 in 2023-06-03T20:10:28.356 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:10:28 smithi089 bash[78452]: cluster 1685823026.885352 osd.6 (osd.6) 125 : cluster 3 failed to encode map e184 with expected crc 2023-06-03T20:10:28.356 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:10:28 smithi089 bash[78452]: cluster 1685823026.8939908 osd.6 (osd.6) 126 : cluster 3 failed to encode map e184 with expected crc 2023-06-03T20:10:28.356 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:10:28 smithi089 bash[78452]: audit 1685823027.0552998 mon.c (mon.2) 486 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-03T20:10:28.357 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:10:28 smithi089 bash[78452]: cluster 1685823027.6913903 mon.a (mon.0) 960 : cluster 3 Health check update: Degraded data redundancy: 13297/9513 objects degraded (139.777%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-06-03T20:10:28.381 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:10:28.385 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:10:28.385 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:10:28.486 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:10:28.515 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:10:28.518 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:10:28.518 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:10:28.619 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:10:28.656 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:10:28.661 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:10:28.662 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:10:28.763 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:10:28.791 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:10:28.794 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:10:28.794 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:10:28.895 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:10:28.932 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:10:28.937 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:10:28.938 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:10:28.940 INFO:tasks.radosbench.radosbench.0.smithi089.stdout:Removed 105132 objects 2023-06-03T20:10:28.940 INFO:tasks.radosbench.radosbench.0.smithi089.stdout:Clean up completed and total clean up time :163.687 2023-06-03T20:10:29.039 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:10:29.075 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:10:29.081 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:10:29.082 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:10:29.183 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:10:29.213 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:10:29.215 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:10:29.216 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:10:29.317 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:10:29.344 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:10:29.347 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:10:29.348 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:10:29.448 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:10:29.479 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:10:29.481 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:10:29.482 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:10:29.583 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:10:29.623 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:10:29.625 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:10:29.626 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:10:29.727 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:10:29.756 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:10:29.758 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:10:29.759 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:10:29.860 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:10:29.894 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:10:29.896 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:10:29.896 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:10:29.997 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:10:30.010 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:10:29 smithi089 bash[75068]: cluster 1685823028.6512702 mgr.y (mgr.34104) 1030 : cluster 0 pgmap v988: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 162 MiB data, 2.7 GiB used, 623 GiB / 626 GiB avail; 0 B/s wr, 518 op/s; 13297/7824 objects degraded (169.951%); 0 B/s, 12 objects/s recovering 2023-06-03T20:10:30.011 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:10:29 smithi089 bash[78452]: cluster 1685823028.6512702 mgr.y (mgr.34104) 1030 : cluster 0 pgmap v988: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 162 MiB data, 2.7 GiB used, 623 GiB / 626 GiB avail; 0 B/s wr, 518 op/s; 13297/7824 objects degraded (169.951%); 0 B/s, 12 objects/s recovering 2023-06-03T20:10:30.031 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:10:30.033 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:10:30.034 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:10:30.048 INFO:tasks.cephadm.ceph_manager.ceph:removing pool_name unique_pool_5 2023-06-03T20:10:30.049 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph osd pool rm unique_pool_5 unique_pool_5 --yes-i-really-really-mean-it 2023-06-03T20:10:30.135 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:10:30.150 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:10:29 smithi164 bash[59462]: cluster 1685823028.6512702 mgr.y (mgr.34104) 1030 : cluster 0 pgmap v988: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 162 MiB data, 2.7 GiB used, 623 GiB / 626 GiB avail; 0 B/s wr, 518 op/s; 13297/7824 objects degraded (169.951%); 0 B/s, 12 objects/s recovering 2023-06-03T20:10:30.172 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:10:30.175 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:10:30.176 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:10:30.277 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:10:30.308 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:10:30.310 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:10:30.311 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:10:30.412 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:10:30.443 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:10:30.446 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:10:30.447 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:10:30.548 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:10:30.577 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:10:30.579 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:10:30.580 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:10:30.681 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:10:30.715 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:10:30.718 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:10:30.718 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:10:30.819 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:10:30.847 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:10:30.850 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:10:30.851 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:10:30.952 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:10:30.986 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:10:30.988 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:10:30.989 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:10:31.090 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:10:31.127 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:10:31.129 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:10:31.129 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:10:31.230 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:10:31.262 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:10:31.265 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:10:31.265 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:10:31.326 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-06-03T20:10:31.366 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:10:31.404 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:10:31.407 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:10:31.407 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:10:31.508 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:10:31.541 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:10:31.544 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:10:31.544 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:10:31.645 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:10:31.678 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:10:31.681 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:10:31.681 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:10:31.782 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:10:31.819 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:10:31.821 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:10:31.822 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:10:31.923 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:10:31.934 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/mon.a/config 2023-06-03T20:10:31.958 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:10:31.960 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:10:31.961 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:10:31.967 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:10:31 smithi089 bash[75068]: cluster 1685823030.6523986 mgr.y (mgr.34104) 1031 : cluster 0 pgmap v989: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 15 MiB data, 2.3 GiB used, 623 GiB / 626 GiB avail; 0 B/s wr, 607 op/s; 13189/750 objects degraded (1758.533%); 0 B/s, 21 objects/s recovering 2023-06-03T20:10:31.968 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:10:31 smithi089 bash[78452]: cluster 1685823030.6523986 mgr.y (mgr.34104) 1031 : cluster 0 pgmap v989: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 15 MiB data, 2.3 GiB used, 623 GiB / 626 GiB avail; 0 B/s wr, 607 op/s; 13189/750 objects degraded (1758.533%); 0 B/s, 21 objects/s recovering 2023-06-03T20:10:32.062 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:10:32.091 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:10:32.093 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:10:32.094 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:10:32.114 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph osd unset noscrub 2023-06-03T20:10:32.195 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:10:32.224 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:10:31 smithi164 bash[59462]: cluster 1685823030.6523986 mgr.y (mgr.34104) 1031 : cluster 0 pgmap v989: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 15 MiB data, 2.3 GiB used, 623 GiB / 626 GiB avail; 0 B/s wr, 607 op/s; 13189/750 objects degraded (1758.533%); 0 B/s, 21 objects/s recovering 2023-06-03T20:10:32.247 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:10:32.249 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:10:32.250 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:10:32.351 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:10:32.380 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:10:32.382 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:10:32.383 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:10:32.484 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:10:32.517 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:10:32.521 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:10:32.522 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:10:32.622 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:10:32.654 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:10:32.656 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:10:32.656 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:10:32.757 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:10:32.793 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:10:32.796 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:10:32.797 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:10:32.898 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:10:32.930 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:10:32.933 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:10:32.933 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:10:33.028 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:10:32 smithi089 bash[75068]: cluster 1685823032.6923535 mon.a (mon.0) 961 : cluster 3 Health check update: Degraded data redundancy: 13189/750 objects degraded (1758.533%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-06-03T20:10:33.029 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:10:32 smithi089 bash[78452]: cluster 1685823032.6923535 mon.a (mon.0) 961 : cluster 3 Health check update: Degraded data redundancy: 13189/750 objects degraded (1758.533%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-06-03T20:10:33.034 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:10:33.064 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:10:33.066 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:10:33.067 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:10:33.168 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:10:33.198 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:10:33.200 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:10:33.201 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:10:33.302 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:10:33.333 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:10:32 smithi164 bash[59462]: cluster 1685823032.6923535 mon.a (mon.0) 961 : cluster 3 Health check update: Degraded data redundancy: 13189/750 objects degraded (1758.533%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-06-03T20:10:33.334 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:10:33.336 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:10:33.337 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:10:33.438 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:10:33.467 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:10:33.470 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:10:33.471 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:10:33.572 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:10:33.602 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:10:33.604 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:10:33.605 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:10:33.706 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:10:33.733 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:10:33.736 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:10:33.736 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:10:33.837 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:10:33.871 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:10:33.874 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:10:33.875 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:10:33.905 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/mon.a/config 2023-06-03T20:10:33.975 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:10:34.008 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:10:34.010 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:10:34.010 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:10:34.111 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:10:34.122 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:10:33 smithi164 bash[59462]: cluster 1685823032.6533368 mgr.y (mgr.34104) 1032 : cluster 0 pgmap v990: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 2.4 MiB data, 2.3 GiB used, 624 GiB / 626 GiB avail; 0 B/s wr, 571 op/s; 13189/159 objects degraded (8294.969%); 0 B/s, 21 objects/s recovering 2023-06-03T20:10:34.122 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:10:33 smithi164 bash[59462]: audit 1685823033.559103 mon.a (mon.0) 962 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:10:34.123 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:10:33 smithi164 bash[59462]: audit 1685823033.5698967 mon.a (mon.0) 963 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:10:34.143 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:10:34.146 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:10:34.146 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:10:34.247 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:10:34.254 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:10:33 smithi089 bash[78452]: cluster 1685823032.6533368 mgr.y (mgr.34104) 1032 : cluster 0 pgmap v990: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 2.4 MiB data, 2.3 GiB used, 624 GiB / 626 GiB avail; 0 B/s wr, 571 op/s; 13189/159 objects degraded (8294.969%); 0 B/s, 21 objects/s recovering 2023-06-03T20:10:34.254 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:10:33 smithi089 bash[78452]: audit 1685823033.559103 mon.a (mon.0) 962 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:10:34.254 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:10:33 smithi089 bash[78452]: audit 1685823033.5698967 mon.a (mon.0) 963 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:10:34.254 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:10:33 smithi089 bash[75068]: cluster 1685823032.6533368 mgr.y (mgr.34104) 1032 : cluster 0 pgmap v990: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 2.4 MiB data, 2.3 GiB used, 624 GiB / 626 GiB avail; 0 B/s wr, 571 op/s; 13189/159 objects degraded (8294.969%); 0 B/s, 21 objects/s recovering 2023-06-03T20:10:34.254 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:10:33 smithi089 bash[75068]: audit 1685823033.559103 mon.a (mon.0) 962 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:10:34.255 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:10:33 smithi089 bash[75068]: audit 1685823033.5698967 mon.a (mon.0) 963 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:10:34.269 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:10:34.271 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:10:34.271 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:10:34.373 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:10:34.406 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:10:34.408 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:10:34.409 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:10:34.510 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:10:34.533 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:10:34.535 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:10:34.536 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:10:34.637 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:10:34.670 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:10:34.673 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:10:34.674 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:10:34.775 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:10:34.797 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:10:34.798 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:10:34.799 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:10:34.900 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:10:34.934 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:10:34.937 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:10:34.938 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:10:35.038 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:10:35.060 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:10:35.062 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:10:35.062 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:10:35.163 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:10:35.196 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:10:35.199 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:10:35.199 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:10:35.234 INFO:teuthology.orchestra.run.smithi089.stdout:{ 2023-06-03T20:10:35.234 INFO:teuthology.orchestra.run.smithi089.stdout: "ops": [], 2023-06-03T20:10:35.234 INFO:teuthology.orchestra.run.smithi089.stdout: "complaint_time": 30, 2023-06-03T20:10:35.235 INFO:teuthology.orchestra.run.smithi089.stdout: "num_blocked_ops": 0 2023-06-03T20:10:35.235 INFO:teuthology.orchestra.run.smithi089.stdout:} 2023-06-03T20:10:35.300 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:10:35.325 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:10:35.328 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:10:35.329 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:10:35.430 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:10:35.456 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:10:35.459 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:10:35.459 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:10:35.560 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:10:35.595 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:10:35.598 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:10:35.599 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:10:35.700 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:10:35.730 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:10:35.732 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:10:35.732 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:10:35.833 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:10:35.866 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:10:35.869 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:10:35.870 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:10:35.971 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:10:36.005 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:10:36.007 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:10:36.008 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:10:36.083 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:10:35 smithi164 bash[59462]: cluster 1685823034.6539803 mgr.y (mgr.34104) 1033 : cluster 0 pgmap v991: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 582 KiB data, 2.2 GiB used, 624 GiB / 626 GiB avail; 0 B/s wr, 395 op/s; 13073/72 objects degraded (18156.944%); 0 B/s, 22 objects/s recovering 2023-06-03T20:10:36.084 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:10:35 smithi164 bash[59462]: audit 1685823035.2529578 mon.a (mon.0) 964 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:10:36.084 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:10:35 smithi164 bash[59462]: audit 1685823035.2634287 mon.a (mon.0) 965 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:10:36.084 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:10:35 smithi164 bash[59462]: audit 1685823035.6828685 mon.c (mon.2) 487 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-03T20:10:36.084 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:10:35 smithi164 bash[59462]: audit 1685823035.6837695 mon.c (mon.2) 488 : audit 1 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-03T20:10:36.084 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:10:35 smithi164 bash[59462]: audit 1685823035.6895506 mon.a (mon.0) 966 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:10:36.109 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:10:36.137 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:10:36.139 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:10:36.140 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:10:36.241 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:10:36.267 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:10:36.269 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:10:36.269 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:10:36.281 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:10:35 smithi089 bash[75068]: cluster 1685823034.6539803 mgr.y (mgr.34104) 1033 : cluster 0 pgmap v991: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 582 KiB data, 2.2 GiB used, 624 GiB / 626 GiB avail; 0 B/s wr, 395 op/s; 13073/72 objects degraded (18156.944%); 0 B/s, 22 objects/s recovering 2023-06-03T20:10:36.281 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:10:35 smithi089 bash[75068]: audit 1685823035.2529578 mon.a (mon.0) 964 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:10:36.282 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:10:35 smithi089 bash[75068]: audit 1685823035.2634287 mon.a (mon.0) 965 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:10:36.282 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:10:35 smithi089 bash[75068]: audit 1685823035.6828685 mon.c (mon.2) 487 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-03T20:10:36.284 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:10:35 smithi089 bash[75068]: audit 1685823035.6837695 mon.c (mon.2) 488 : audit 1 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-03T20:10:36.284 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:10:35 smithi089 bash[75068]: audit 1685823035.6895506 mon.a (mon.0) 966 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:10:36.285 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:10:35 smithi089 bash[78452]: cluster 1685823034.6539803 mgr.y (mgr.34104) 1033 : cluster 0 pgmap v991: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 582 KiB data, 2.2 GiB used, 624 GiB / 626 GiB avail; 0 B/s wr, 395 op/s; 13073/72 objects degraded (18156.944%); 0 B/s, 22 objects/s recovering 2023-06-03T20:10:36.285 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:10:35 smithi089 bash[78452]: audit 1685823035.2529578 mon.a (mon.0) 964 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:10:36.286 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:10:35 smithi089 bash[78452]: audit 1685823035.2634287 mon.a (mon.0) 965 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:10:36.286 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:10:35 smithi089 bash[78452]: audit 1685823035.6828685 mon.c (mon.2) 487 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-03T20:10:36.286 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:10:35 smithi089 bash[78452]: audit 1685823035.6837695 mon.c (mon.2) 488 : audit 1 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-03T20:10:36.286 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:10:35 smithi089 bash[78452]: audit 1685823035.6895506 mon.a (mon.0) 966 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:10:36.370 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:10:36.401 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:10:36.404 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:10:36.405 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:10:36.430 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 20:10:36 smithi089 bash[71163]: ::ffff:172.21.15.164 - - [03/Jun/2023:20:10:36] "GET /metrics HTTP/1.1" 200 228752 "" "Prometheus/2.33.4" 2023-06-03T20:10:36.506 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:10:36.531 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:10:36.534 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:10:36.534 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:10:36.635 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:10:36.672 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:10:36.674 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:10:36.675 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:10:36.775 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:10:36.810 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:10:36.812 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:10:36.813 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:10:36.913 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:10:36.950 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:10:36.953 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:10:36.954 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:10:37.055 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:10:37.088 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:10:37.090 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:10:37.091 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:10:37.192 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:10:37.225 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:10:37.228 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:10:37.228 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:10:37.329 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:10:37.364 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:10:37.365 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:10:37.366 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:10:37.467 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:10:37.501 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:10:37.504 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:10:37.504 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:10:37.606 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph daemon osd.3 dump_historic_ops 2023-06-03T20:10:37.607 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:10:37.640 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:10:37.643 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:10:37.644 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:10:37.745 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:10:37.784 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:10:37.786 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:10:37.787 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:10:37.888 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:10:37.924 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:10:37.928 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:10:37.928 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:10:38.029 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:10:38.062 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:10:38.065 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:10:38.066 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:10:38.167 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:10:38.180 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:10:37 smithi164 bash[59462]: cluster 1685823036.65454 mgr.y (mgr.34104) 1034 : cluster 0 pgmap v992: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 582 KiB data, 2.2 GiB used, 624 GiB / 626 GiB avail; 0 B/s wr, 314 op/s; 13073/72 objects degraded (18156.944%); 0 B/s, 22 objects/s recovering 2023-06-03T20:10:38.180 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:10:37 smithi164 bash[59462]: audit 1685823037.1143422 mon.c (mon.2) 489 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T20:10:38.180 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:10:37 smithi164 bash[59462]: cluster 1685823037.694152 mon.a (mon.0) 967 : cluster 3 Health check update: Degraded data redundancy: 13073/72 objects degraded (18156.944%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-06-03T20:10:38.202 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:10:38.204 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:10:38.205 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:10:38.233 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:10:37 smithi089 bash[75068]: cluster 1685823036.65454 mgr.y (mgr.34104) 1034 : cluster 0 pgmap v992: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 582 KiB data, 2.2 GiB used, 624 GiB / 626 GiB avail; 0 B/s wr, 314 op/s; 13073/72 objects degraded (18156.944%); 0 B/s, 22 objects/s recovering 2023-06-03T20:10:38.234 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:10:37 smithi089 bash[75068]: audit 1685823037.1143422 mon.c (mon.2) 489 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T20:10:38.234 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:10:37 smithi089 bash[75068]: cluster 1685823037.694152 mon.a (mon.0) 967 : cluster 3 Health check update: Degraded data redundancy: 13073/72 objects degraded (18156.944%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-06-03T20:10:38.234 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:10:37 smithi089 bash[78452]: cluster 1685823036.65454 mgr.y (mgr.34104) 1034 : cluster 0 pgmap v992: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 582 KiB data, 2.2 GiB used, 624 GiB / 626 GiB avail; 0 B/s wr, 314 op/s; 13073/72 objects degraded (18156.944%); 0 B/s, 22 objects/s recovering 2023-06-03T20:10:38.235 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:10:37 smithi089 bash[78452]: audit 1685823037.1143422 mon.c (mon.2) 489 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T20:10:38.235 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:10:37 smithi089 bash[78452]: cluster 1685823037.694152 mon.a (mon.0) 967 : cluster 3 Health check update: Degraded data redundancy: 13073/72 objects degraded (18156.944%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-06-03T20:10:38.307 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:10:38.343 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:10:38.345 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:10:38.346 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:10:38.447 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:10:38.480 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:10:38.484 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:10:38.484 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:10:38.585 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:10:38.619 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:10:38.621 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:10:38.622 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:10:38.723 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:10:38.752 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:10:38.753 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:10:38.754 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:10:38.855 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:10:38.886 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:10:38.887 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:10:38.888 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:10:38.988 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:10:39.023 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:10:39.026 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:10:39.027 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:10:39.127 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:10:39.159 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:10:39.162 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:10:39.163 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:10:39.264 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:10:39.297 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:10:39.300 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:10:39.301 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:10:39.402 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:10:39.437 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:10:39.440 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:10:39.440 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:10:39.541 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:10:39.575 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:10:39.577 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:10:39.578 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:10:39.679 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:10:39.716 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:10:39.717 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:10:39.718 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:10:39.819 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:10:39.854 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:10:39.856 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:10:39.857 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:10:39.958 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:10:39.969 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:10:39 smithi164 bash[59462]: cluster 1685823038.6556218 mgr.y (mgr.34104) 1035 : cluster 0 pgmap v993: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 582 KiB data, 2.2 GiB used, 624 GiB / 626 GiB avail; 0 B/s wr, 267 op/s; 13073/72 objects degraded (18156.944%); 0 B/s, 19 objects/s recovering 2023-06-03T20:10:39.988 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:10:39.990 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:10:39.991 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:10:40.092 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:10:40.120 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:10:40.122 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:10:40.122 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:10:40.223 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:10:40.253 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:10:40.255 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:10:40.256 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:10:40.278 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:10:39 smithi089 bash[75068]: cluster 1685823038.6556218 mgr.y (mgr.34104) 1035 : cluster 0 pgmap v993: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 582 KiB data, 2.2 GiB used, 624 GiB / 626 GiB avail; 0 B/s wr, 267 op/s; 13073/72 objects degraded (18156.944%); 0 B/s, 19 objects/s recovering 2023-06-03T20:10:40.278 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:10:39 smithi089 bash[78452]: cluster 1685823038.6556218 mgr.y (mgr.34104) 1035 : cluster 0 pgmap v993: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 582 KiB data, 2.2 GiB used, 624 GiB / 626 GiB avail; 0 B/s wr, 267 op/s; 13073/72 objects degraded (18156.944%); 0 B/s, 19 objects/s recovering 2023-06-03T20:10:40.357 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:10:40.395 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:10:40.397 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:10:40.397 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:10:40.498 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:10:40.530 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:10:40.532 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:10:40.532 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:10:40.633 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:10:40.664 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:10:40.667 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:10:40.668 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:10:40.769 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:10:40.800 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:10:40.803 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:10:40.804 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:10:40.826 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/mon.a/config 2023-06-03T20:10:40.827 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/mon.a/config 2023-06-03T20:10:40.906 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:10:40.932 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:10:40.934 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:10:40.935 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:10:41.036 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:10:41.071 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:10:41.074 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:10:41.074 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:10:41.176 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:10:41.207 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:10:41.210 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:10:41.211 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:10:41.312 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:10:41.349 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:10:41.352 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:10:41.354 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:10:41.456 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:10:41.491 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:10:41.494 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:10:41.494 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:10:41.595 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:10:41.628 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:10:41.630 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:10:41.631 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:10:41.731 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:10:41.769 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:10:41.771 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:10:41.772 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:10:41.873 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:10:41.902 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:10:41.905 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:10:41.905 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:10:42.006 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:10:42.040 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:10:42.042 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:10:42.043 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:10:42.144 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:10:42.177 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:10:42.180 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:10:42.181 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:10:42.181 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:10:41 smithi164 bash[59462]: cluster 1685823040.6570184 mgr.y (mgr.34104) 1036 : cluster 0 pgmap v994: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 582 KiB data, 2.2 GiB used, 624 GiB / 626 GiB avail; 0 B/s wr, 215 op/s; 12956/72 objects degraded (17994.444%); 0 B/s, 28 objects/s recovering 2023-06-03T20:10:42.282 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:10:42.300 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:10:41 smithi089 bash[78452]: cluster 1685823040.6570184 mgr.y (mgr.34104) 1036 : cluster 0 pgmap v994: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 582 KiB data, 2.2 GiB used, 624 GiB / 626 GiB avail; 0 B/s wr, 215 op/s; 12956/72 objects degraded (17994.444%); 0 B/s, 28 objects/s recovering 2023-06-03T20:10:42.300 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:10:41 smithi089 bash[75068]: cluster 1685823040.6570184 mgr.y (mgr.34104) 1036 : cluster 0 pgmap v994: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 582 KiB data, 2.2 GiB used, 624 GiB / 626 GiB avail; 0 B/s wr, 215 op/s; 12956/72 objects degraded (17994.444%); 0 B/s, 28 objects/s recovering 2023-06-03T20:10:42.314 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:10:42.316 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:10:42.317 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:10:42.417 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:10:42.443 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:10:42.445 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:10:42.446 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:10:42.546 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:10:42.579 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:10:42.581 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:10:42.582 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:10:42.683 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:10:42.718 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:10:42.720 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:10:42.721 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:10:42.822 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:10:42.853 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:10:42.856 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:10:42.857 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:10:42.921 INFO:teuthology.orchestra.run.smithi089.stderr:reweighted osd.6 to 1 (10000) 2023-06-03T20:10:42.957 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:10:42.994 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:10:42.995 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:10:42.996 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:10:43.097 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:10:43.105 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:10:42 smithi164 bash[59462]: audit 1685823042.337722 mon.c (mon.2) 490 : audit 1 from='client.? 172.21.15.89:0/223586872' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-06-03T20:10:43.105 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:10:42 smithi164 bash[59462]: audit 1685823042.338096 mon.a (mon.0) 968 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-06-03T20:10:43.106 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:10:42 smithi164 bash[59462]: cluster 1685823042.695171 mon.a (mon.0) 969 : cluster 3 Health check update: Degraded data redundancy: 12956/72 objects degraded (17994.444%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-06-03T20:10:43.127 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:10:43.130 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:10:43.131 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:10:43.232 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:10:43.240 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:10:42 smithi089 bash[75068]: audit 1685823042.337722 mon.c (mon.2) 490 : audit 1 from='client.? 172.21.15.89:0/223586872' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-06-03T20:10:43.241 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:10:42 smithi089 bash[75068]: audit 1685823042.338096 mon.a (mon.0) 968 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-06-03T20:10:43.241 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:10:42 smithi089 bash[75068]: cluster 1685823042.695171 mon.a (mon.0) 969 : cluster 3 Health check update: Degraded data redundancy: 12956/72 objects degraded (17994.444%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-06-03T20:10:43.241 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:10:42 smithi089 bash[78452]: audit 1685823042.337722 mon.c (mon.2) 490 : audit 1 from='client.? 172.21.15.89:0/223586872' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-06-03T20:10:43.241 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:10:42 smithi089 bash[78452]: audit 1685823042.338096 mon.a (mon.0) 968 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-06-03T20:10:43.241 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:10:42 smithi089 bash[78452]: cluster 1685823042.695171 mon.a (mon.0) 969 : cluster 3 Health check update: Degraded data redundancy: 12956/72 objects degraded (17994.444%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-06-03T20:10:43.264 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:10:43.267 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:10:43.267 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:10:43.369 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:10:43.402 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:10:43.404 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:10:43.404 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:10:43.505 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:10:43.535 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:10:43.537 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:10:43.538 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:10:43.639 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:10:43.668 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:10:43.670 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:10:43.670 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:10:43.771 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:10:43.799 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:10:43.801 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:10:43.802 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:10:43.903 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:10:43.926 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:10:43.927 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:10:43.928 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:10:44.030 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:10:44.058 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:10:44.061 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:10:44.062 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:10:44.162 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:10:44.172 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:10:43 smithi164 bash[59462]: cluster 1685823042.657956 mgr.y (mgr.34104) 1037 : cluster 0 pgmap v995: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 582 KiB data, 2.2 GiB used, 624 GiB / 626 GiB avail; 0 B/s wr, 18 op/s; 12956/72 objects degraded (17994.444%); 0 B/s, 19 objects/s recovering 2023-06-03T20:10:44.172 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:10:43 smithi164 bash[59462]: cluster 1685823042.9087312 osd.6 (osd.6) 127 : cluster 3 failed to encode map e185 with expected crc 2023-06-03T20:10:44.172 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:10:43 smithi164 bash[59462]: audit 1685823042.9132757 mon.a (mon.0) 970 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2023-06-03T20:10:44.173 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:10:43 smithi164 bash[59462]: cluster 1685823042.913357 mon.a (mon.0) 971 : cluster 0 osdmap e185: 8 total, 8 up, 7 in 2023-06-03T20:10:44.173 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:10:43 smithi164 bash[59462]: cluster 1685823043.3814304 osd.5 (osd.5) 116 : cluster 3 failed to encode map e185 with expected crc 2023-06-03T20:10:44.195 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:10:44.197 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:10:44.198 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:10:44.278 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:10:43 smithi089 bash[75068]: cluster 1685823042.657956 mgr.y (mgr.34104) 1037 : cluster 0 pgmap v995: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 582 KiB data, 2.2 GiB used, 624 GiB / 626 GiB avail; 0 B/s wr, 18 op/s; 12956/72 objects degraded (17994.444%); 0 B/s, 19 objects/s recovering 2023-06-03T20:10:44.278 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:10:43 smithi089 bash[75068]: cluster 1685823042.9087312 osd.6 (osd.6) 127 : cluster 3 failed to encode map e185 with expected crc 2023-06-03T20:10:44.279 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:10:43 smithi089 bash[75068]: audit 1685823042.9132757 mon.a (mon.0) 970 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2023-06-03T20:10:44.279 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:10:43 smithi089 bash[75068]: cluster 1685823042.913357 mon.a (mon.0) 971 : cluster 0 osdmap e185: 8 total, 8 up, 7 in 2023-06-03T20:10:44.279 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:10:43 smithi089 bash[75068]: cluster 1685823043.3814304 osd.5 (osd.5) 116 : cluster 3 failed to encode map e185 with expected crc 2023-06-03T20:10:44.279 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:10:43 smithi089 bash[78452]: cluster 1685823042.657956 mgr.y (mgr.34104) 1037 : cluster 0 pgmap v995: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 582 KiB data, 2.2 GiB used, 624 GiB / 626 GiB avail; 0 B/s wr, 18 op/s; 12956/72 objects degraded (17994.444%); 0 B/s, 19 objects/s recovering 2023-06-03T20:10:44.279 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:10:43 smithi089 bash[78452]: cluster 1685823042.9087312 osd.6 (osd.6) 127 : cluster 3 failed to encode map e185 with expected crc 2023-06-03T20:10:44.279 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:10:43 smithi089 bash[78452]: audit 1685823042.9132757 mon.a (mon.0) 970 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2023-06-03T20:10:44.280 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:10:43 smithi089 bash[78452]: cluster 1685823042.913357 mon.a (mon.0) 971 : cluster 0 osdmap e185: 8 total, 8 up, 7 in 2023-06-03T20:10:44.280 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:10:43 smithi089 bash[78452]: cluster 1685823043.3814304 osd.5 (osd.5) 116 : cluster 3 failed to encode map e185 with expected crc 2023-06-03T20:10:44.298 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:10:44.328 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:10:44.331 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:10:44.331 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:10:44.432 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:10:44.461 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:10:44.465 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:10:44.466 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:10:44.567 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:10:44.600 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:10:44.602 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:10:44.603 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:10:44.704 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:10:44.738 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:10:44.740 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:10:44.740 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:10:44.841 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:10:44.879 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:10:44.881 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:10:44.882 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:10:44.931 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph osd reweight 7 1 2023-06-03T20:10:44.952 INFO:teuthology.orchestra.run.smithi089.stderr:pool 'unique_pool_5' removed 2023-06-03T20:10:44.983 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:10:45.020 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:10:45.024 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:10:45.025 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:10:45.126 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:10:45.133 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:10:44 smithi089 bash[78452]: cluster 1685823043.3480186 osd.7 (osd.7) 119 : cluster 3 failed to encode map e185 with expected crc 2023-06-03T20:10:45.133 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:10:44 smithi089 bash[78452]: cluster 1685823043.3490107 osd.7 (osd.7) 120 : cluster 3 failed to encode map e185 with expected crc 2023-06-03T20:10:45.133 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:10:44 smithi089 bash[78452]: audit 1685823043.978999 mon.a (mon.0) 972 : audit 1 from='client.? 172.21.15.89:0/2480622978' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_5", "pool2": "unique_pool_5", "yes_i_really_really_mean_it": true}]: dispatch 2023-06-03T20:10:45.134 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:10:44 smithi089 bash[75068]: cluster 1685823043.3480186 osd.7 (osd.7) 119 : cluster 3 failed to encode map e185 with expected crc 2023-06-03T20:10:45.134 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:10:44 smithi089 bash[75068]: cluster 1685823043.3490107 osd.7 (osd.7) 120 : cluster 3 failed to encode map e185 with expected crc 2023-06-03T20:10:45.134 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:10:44 smithi089 bash[75068]: audit 1685823043.978999 mon.a (mon.0) 972 : audit 1 from='client.? 172.21.15.89:0/2480622978' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_5", "pool2": "unique_pool_5", "yes_i_really_really_mean_it": true}]: dispatch 2023-06-03T20:10:45.151 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:10:45.153 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:10:45.153 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:10:45.254 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:10:45.268 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:10:44 smithi164 bash[59462]: cluster 1685823043.3480186 osd.7 (osd.7) 119 : cluster 3 failed to encode map e185 with expected crc 2023-06-03T20:10:45.268 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:10:44 smithi164 bash[59462]: cluster 1685823043.3490107 osd.7 (osd.7) 120 : cluster 3 failed to encode map e185 with expected crc 2023-06-03T20:10:45.268 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:10:44 smithi164 bash[59462]: audit 1685823043.978999 mon.a (mon.0) 972 : audit 1 from='client.? 172.21.15.89:0/2480622978' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_5", "pool2": "unique_pool_5", "yes_i_really_really_mean_it": true}]: dispatch 2023-06-03T20:10:45.286 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:10:45.289 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:10:45.290 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:10:45.391 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:10:45.421 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:10:45.424 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:10:45.424 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:10:45.525 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:10:45.566 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:10:45.568 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:10:45.569 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:10:45.670 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:10:45.697 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:10:45.699 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:10:45.699 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:10:45.800 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:10:45.841 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:10:45.843 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:10:45.844 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:10:45.945 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:10:45.979 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:10:45.982 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:10:45.983 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:10:46.083 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:10:45 smithi164 bash[59462]: cluster 1685823044.6588116 mgr.y (mgr.34104) 1038 : cluster 0 pgmap v997: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 582 KiB data, 2.2 GiB used, 624 GiB / 626 GiB avail; 12840/72 objects degraded (17833.333%); 0 B/s, 23 objects/s recovering 2023-06-03T20:10:46.084 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:10:45 smithi164 bash[59462]: cluster 1685823044.9249816 osd.6 (osd.6) 128 : cluster 3 failed to encode map e186 with expected crc 2023-06-03T20:10:46.084 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:10:45 smithi164 bash[59462]: audit 1685823044.927884 mon.a (mon.0) 973 : audit 1 from='client.? 172.21.15.89:0/2480622978' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_5", "pool2": "unique_pool_5", "yes_i_really_really_mean_it": true}]': finished 2023-06-03T20:10:46.084 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:10:45 smithi164 bash[59462]: cluster 1685823044.9279568 mon.a (mon.0) 974 : cluster 0 osdmap e186: 8 total, 8 up, 7 in 2023-06-03T20:10:46.084 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:10:45 smithi164 bash[59462]: cluster 1685823045.572381 osd.5 (osd.5) 117 : cluster 3 failed to encode map e186 with expected crc 2023-06-03T20:10:46.085 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:10:46.120 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:10:46.123 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:10:46.124 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:10:46.225 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:10:46.232 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:10:45 smithi089 bash[75068]: cluster 1685823044.6588116 mgr.y (mgr.34104) 1038 : cluster 0 pgmap v997: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 582 KiB data, 2.2 GiB used, 624 GiB / 626 GiB avail; 12840/72 objects degraded (17833.333%); 0 B/s, 23 objects/s recovering 2023-06-03T20:10:46.232 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:10:45 smithi089 bash[75068]: cluster 1685823044.9249816 osd.6 (osd.6) 128 : cluster 3 failed to encode map e186 with expected crc 2023-06-03T20:10:46.232 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:10:45 smithi089 bash[75068]: audit 1685823044.927884 mon.a (mon.0) 973 : audit 1 from='client.? 172.21.15.89:0/2480622978' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_5", "pool2": "unique_pool_5", "yes_i_really_really_mean_it": true}]': finished 2023-06-03T20:10:46.233 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:10:45 smithi089 bash[75068]: cluster 1685823044.9279568 mon.a (mon.0) 974 : cluster 0 osdmap e186: 8 total, 8 up, 7 in 2023-06-03T20:10:46.233 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:10:45 smithi089 bash[75068]: cluster 1685823045.572381 osd.5 (osd.5) 117 : cluster 3 failed to encode map e186 with expected crc 2023-06-03T20:10:46.233 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:10:45 smithi089 bash[78452]: cluster 1685823044.6588116 mgr.y (mgr.34104) 1038 : cluster 0 pgmap v997: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 582 KiB data, 2.2 GiB used, 624 GiB / 626 GiB avail; 12840/72 objects degraded (17833.333%); 0 B/s, 23 objects/s recovering 2023-06-03T20:10:46.233 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:10:45 smithi089 bash[78452]: cluster 1685823044.9249816 osd.6 (osd.6) 128 : cluster 3 failed to encode map e186 with expected crc 2023-06-03T20:10:46.233 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:10:45 smithi089 bash[78452]: audit 1685823044.927884 mon.a (mon.0) 973 : audit 1 from='client.? 172.21.15.89:0/2480622978' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_5", "pool2": "unique_pool_5", "yes_i_really_really_mean_it": true}]': finished 2023-06-03T20:10:46.233 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:10:45 smithi089 bash[78452]: cluster 1685823044.9279568 mon.a (mon.0) 974 : cluster 0 osdmap e186: 8 total, 8 up, 7 in 2023-06-03T20:10:46.234 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:10:45 smithi089 bash[78452]: cluster 1685823045.572381 osd.5 (osd.5) 117 : cluster 3 failed to encode map e186 with expected crc 2023-06-03T20:10:46.254 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:10:46.256 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:10:46.257 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:10:46.358 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:10:46.390 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:10:46.393 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:10:46.393 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:10:46.494 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:10:46.528 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 20:10:46 smithi089 bash[71163]: ::ffff:172.21.15.164 - - [03/Jun/2023:20:10:46] "GET /metrics HTTP/1.1" 200 228724 "" "Prometheus/2.33.4" 2023-06-03T20:10:46.537 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:10:46.540 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:10:46.541 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:10:46.642 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:10:46.677 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:10:46.680 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:10:46.681 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:10:46.782 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:10:46.818 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:10:46.820 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:10:46.821 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:10:46.922 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:10:46.959 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:10:46.963 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:10:46.964 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:10:47.064 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:10:47.076 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:10:46 smithi164 bash[59462]: cluster 1685823045.5726178 osd.7 (osd.7) 121 : cluster 3 failed to encode map e186 with expected crc 2023-06-03T20:10:47.104 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:10:47.106 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:10:47.107 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:10:47.207 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:10:47.245 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:10:47.247 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:10:47.248 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:10:47.278 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:10:46 smithi089 bash[75068]: cluster 1685823045.5726178 osd.7 (osd.7) 121 : cluster 3 failed to encode map e186 with expected crc 2023-06-03T20:10:47.278 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:10:46 smithi089 bash[78452]: cluster 1685823045.5726178 osd.7 (osd.7) 121 : cluster 3 failed to encode map e186 with expected crc 2023-06-03T20:10:47.349 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:10:47.386 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:10:47.389 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:10:47.390 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:10:47.491 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:10:47.519 INFO:teuthology.task.full_sequential:In full_sequential, running task radosbench... 2023-06-03T20:10:47.520 INFO:tasks.radosbench:Beginning radosbench... 2023-06-03T20:10:47.521 INFO:tasks.cephadm.ceph_manager.ceph:creating pool_name unique_pool_6 2023-06-03T20:10:47.521 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph osd pool create unique_pool_6 16 2023-06-03T20:10:47.522 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:10:47.525 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:10:47.525 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:10:47.626 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:10:47.657 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:10:47.659 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:10:47.660 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:10:47.761 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:10:47.799 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:10:47.802 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:10:47.803 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:10:47.904 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:10:47.936 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:10:47.940 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:10:47.941 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:10:48.042 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:10:48.053 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:10:47 smithi164 bash[59462]: cluster 1685823046.659797 mgr.y (mgr.34104) 1039 : cluster 0 pgmap v999: 73 pgs: 73 active+clean; 582 KiB data, 2.2 GiB used, 624 GiB / 626 GiB avail 2023-06-03T20:10:48.054 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:10:47 smithi164 bash[59462]: cluster 1685823046.9272366 mon.a (mon.0) 975 : cluster 1 Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 12840/72 objects degraded (17833.333%), 3 pgs degraded, 3 pgs undersized) 2023-06-03T20:10:48.079 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:10:48.083 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:10:48.083 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:10:48.184 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:10:48.221 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:10:48.223 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:10:48.224 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:10:48.278 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:10:47 smithi089 bash[78452]: cluster 1685823046.659797 mgr.y (mgr.34104) 1039 : cluster 0 pgmap v999: 73 pgs: 73 active+clean; 582 KiB data, 2.2 GiB used, 624 GiB / 626 GiB avail 2023-06-03T20:10:48.279 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:10:47 smithi089 bash[78452]: cluster 1685823046.9272366 mon.a (mon.0) 975 : cluster 1 Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 12840/72 objects degraded (17833.333%), 3 pgs degraded, 3 pgs undersized) 2023-06-03T20:10:48.279 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:10:47 smithi089 bash[75068]: cluster 1685823046.659797 mgr.y (mgr.34104) 1039 : cluster 0 pgmap v999: 73 pgs: 73 active+clean; 582 KiB data, 2.2 GiB used, 624 GiB / 626 GiB avail 2023-06-03T20:10:48.279 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:10:47 smithi089 bash[75068]: cluster 1685823046.9272366 mon.a (mon.0) 975 : cluster 1 Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 12840/72 objects degraded (17833.333%), 3 pgs degraded, 3 pgs undersized) 2023-06-03T20:10:48.325 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:10:48.359 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:10:48.361 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:10:48.361 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:10:48.380 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/mon.a/config 2023-06-03T20:10:48.463 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:10:48.489 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:10:48.490 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:10:48.491 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:10:48.592 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:10:48.632 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:10:48.635 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:10:48.636 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:10:48.737 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:10:48.770 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:10:48.773 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:10:48.774 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:10:48.875 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:10:48.910 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:10:48.913 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:10:48.914 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:10:49.015 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:10:49.051 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:10:49.054 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:10:49.055 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:10:49.156 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:10:49.191 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:10:49.195 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:10:49.195 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:10:49.296 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:10:49.332 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:10:49.334 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:10:49.336 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:10:49.437 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:10:49.471 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:10:49.473 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:10:49.474 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:10:49.574 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:10:49.600 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:10:49.602 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:10:49.603 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:10:49.704 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:10:49.733 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:10:49.735 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:10:49.735 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:10:49.836 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:10:49.861 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:10:49.864 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:10:49.865 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:10:49.966 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:10:50.000 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:10:50.002 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:10:50.002 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:10:50.103 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:10:50.143 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:10:50.146 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:10:50.146 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:10:50.248 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:10:50.260 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:10:49 smithi164 bash[59462]: cluster 1685823048.6608229 mgr.y (mgr.34104) 1040 : cluster 0 pgmap v1000: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 624 GiB / 626 GiB avail 2023-06-03T20:10:50.278 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:10:49 smithi089 bash[75068]: cluster 1685823048.6608229 mgr.y (mgr.34104) 1040 : cluster 0 pgmap v1000: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 624 GiB / 626 GiB avail 2023-06-03T20:10:50.279 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:10:49 smithi089 bash[78452]: cluster 1685823048.6608229 mgr.y (mgr.34104) 1040 : cluster 0 pgmap v1000: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 624 GiB / 626 GiB avail 2023-06-03T20:10:50.286 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:10:50.288 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:10:50.289 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:10:50.390 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:10:50.418 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:10:50.421 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:10:50.422 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:10:50.522 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:10:50.563 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:10:50.565 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:10:50.566 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:10:50.667 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:10:50.706 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:10:50.709 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:10:50.709 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:10:50.810 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:10:50.845 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:10:50.849 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:10:50.849 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:10:50.950 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:10:50.984 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:10:50.986 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:10:50.987 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:10:51.058 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/mon.a/config 2023-06-03T20:10:51.088 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:10:51.115 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:10:51.117 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:10:51.118 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:10:51.207 INFO:teuthology.orchestra.run.smithi089.stdout:osd.0: {} 2023-06-03T20:10:51.219 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:10:51.228 INFO:teuthology.orchestra.run.smithi089.stdout:osd.1: {} 2023-06-03T20:10:51.248 INFO:teuthology.orchestra.run.smithi089.stdout:osd.2: {} 2023-06-03T20:10:51.253 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:10:51.256 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:10:51.256 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:10:51.272 INFO:teuthology.orchestra.run.smithi089.stdout:osd.3: {} 2023-06-03T20:10:51.300 INFO:teuthology.orchestra.run.smithi089.stdout:osd.4: {} 2023-06-03T20:10:51.330 INFO:teuthology.orchestra.run.smithi089.stdout:osd.5: {} 2023-06-03T20:10:51.356 INFO:teuthology.orchestra.run.smithi089.stdout:osd.6: {} 2023-06-03T20:10:51.357 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:10:51.387 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:10:51.388 INFO:teuthology.orchestra.run.smithi089.stdout:osd.7: {} 2023-06-03T20:10:51.390 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:10:51.390 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:10:51.391 INFO:teuthology.orchestra.run.smithi089.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-06-03T20:10:51.391 INFO:teuthology.orchestra.run.smithi089.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-06-03T20:10:51.391 INFO:teuthology.orchestra.run.smithi089.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-06-03T20:10:51.391 INFO:teuthology.orchestra.run.smithi089.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-06-03T20:10:51.391 INFO:teuthology.orchestra.run.smithi089.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-06-03T20:10:51.391 INFO:teuthology.orchestra.run.smithi089.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-06-03T20:10:51.391 INFO:teuthology.orchestra.run.smithi089.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-06-03T20:10:51.392 INFO:teuthology.orchestra.run.smithi089.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-06-03T20:10:51.491 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:10:51.530 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:10:51.532 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:10:51.533 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:10:51.634 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:10:51.668 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:10:51.671 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:10:51.672 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:10:51.773 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:10:51.799 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:10:51.801 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:10:51.801 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:10:51.902 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:10:51.932 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:10:51.934 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:10:51.935 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:10:52.036 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:10:52.072 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:10:52.075 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:10:52.076 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:10:52.177 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:10:52.184 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:10:51 smithi089 bash[75068]: cluster 1685823050.661722 mgr.y (mgr.34104) 1041 : cluster 0 pgmap v1001: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 624 GiB / 626 GiB avail 2023-06-03T20:10:52.185 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:10:51 smithi089 bash[75068]: audit 1685823051.176092 mon.c (mon.2) 491 : audit 0 from='client.? 172.21.15.89:0/118162211' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-03T20:10:52.185 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:10:52 smithi089 bash[78452]: cluster 1685823050.661722 mgr.y (mgr.34104) 1041 : cluster 0 pgmap v1001: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 624 GiB / 626 GiB avail 2023-06-03T20:10:52.185 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:10:52 smithi089 bash[78452]: audit 1685823051.176092 mon.c (mon.2) 491 : audit 0 from='client.? 172.21.15.89:0/118162211' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-03T20:10:52.204 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:10:52.206 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:10:52.207 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:10:52.308 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:10:52.321 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:10:51 smithi164 bash[59462]: cluster 1685823050.661722 mgr.y (mgr.34104) 1041 : cluster 0 pgmap v1001: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 624 GiB / 626 GiB avail 2023-06-03T20:10:52.321 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:10:51 smithi164 bash[59462]: audit 1685823051.176092 mon.c (mon.2) 491 : audit 0 from='client.? 172.21.15.89:0/118162211' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-03T20:10:52.345 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:10:52.348 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:10:52.349 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:10:52.450 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:10:52.475 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:10:52.477 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:10:52.478 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:10:52.579 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:10:52.616 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:10:52.620 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:10:52.620 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:10:52.721 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:10:52.744 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:10:52.745 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:10:52.746 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:10:52.847 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:10:52.873 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:10:52.874 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:10:52.875 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:10:52.976 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:10:53.003 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:10:53.005 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:10:53.006 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:10:53.107 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:10:53.148 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:10:53.150 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:10:53.151 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:10:53.252 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:10:53.278 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:10:53 smithi089 bash[78452]: audit 1685823052.1232674 mon.a (mon.0) 976 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:10:53.279 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:10:53 smithi089 bash[78452]: audit 1685823052.1247683 mon.c (mon.2) 492 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T20:10:53.279 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:10:53 smithi089 bash[78452]: cluster 1685823052.6621854 mgr.y (mgr.34104) 1042 : cluster 0 pgmap v1002: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 624 GiB / 626 GiB avail 2023-06-03T20:10:53.279 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:10:53 smithi089 bash[78452]: audit 1685823052.7531917 mon.a (mon.0) 977 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:10:53.279 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:10:53 smithi089 bash[78452]: audit 1685823052.95321 mon.a (mon.0) 978 : audit 1 from='client.? 172.21.15.89:0/4047100895' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-06-03T20:10:53.280 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:10:53 smithi089 bash[75068]: audit 1685823052.1232674 mon.a (mon.0) 976 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:10:53.280 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:10:53 smithi089 bash[75068]: audit 1685823052.1247683 mon.c (mon.2) 492 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T20:10:53.280 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:10:53 smithi089 bash[75068]: cluster 1685823052.6621854 mgr.y (mgr.34104) 1042 : cluster 0 pgmap v1002: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 624 GiB / 626 GiB avail 2023-06-03T20:10:53.280 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:10:53 smithi089 bash[75068]: audit 1685823052.7531917 mon.a (mon.0) 977 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:10:53.280 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:10:53 smithi089 bash[75068]: audit 1685823052.95321 mon.a (mon.0) 978 : audit 1 from='client.? 172.21.15.89:0/4047100895' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-06-03T20:10:53.291 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:10:53.294 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:10:53.295 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:10:53.396 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:10:53.431 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:10:53.434 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:10:53.434 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:10:53.535 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:10:53.563 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:10:53.565 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:10:53.565 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:10:53.583 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:10:53 smithi164 bash[59462]: audit 1685823052.1232674 mon.a (mon.0) 976 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:10:53.584 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:10:53 smithi164 bash[59462]: audit 1685823052.1247683 mon.c (mon.2) 492 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T20:10:53.584 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:10:53 smithi164 bash[59462]: cluster 1685823052.6621854 mgr.y (mgr.34104) 1042 : cluster 0 pgmap v1002: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 624 GiB / 626 GiB avail 2023-06-03T20:10:53.584 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:10:53 smithi164 bash[59462]: audit 1685823052.7531917 mon.a (mon.0) 977 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:10:53.585 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:10:53 smithi164 bash[59462]: audit 1685823052.95321 mon.a (mon.0) 978 : audit 1 from='client.? 172.21.15.89:0/4047100895' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-06-03T20:10:53.666 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:10:53.699 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:10:53.701 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:10:53.702 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:10:53.765 INFO:teuthology.orchestra.run.smithi089.stderr:noscrub is unset 2023-06-03T20:10:53.803 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:10:53.848 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:10:53.851 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:10:53.851 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:10:53.952 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:10:53.993 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:10:53.995 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:10:53.996 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:10:54.097 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:10:54.122 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:10:54.125 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:10:54.126 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:10:54.227 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:10:54.260 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:10:54.262 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:10:54.263 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:10:54.364 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:10:54.375 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:10:54 smithi089 bash[75068]: cluster 1685823053.7504578 mon.a (mon.0) 979 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-06-03T20:10:54.375 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:10:54 smithi089 bash[75068]: audit 1685823053.755597 mon.a (mon.0) 980 : audit 1 from='client.? 172.21.15.89:0/4047100895' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-06-03T20:10:54.375 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:10:54 smithi089 bash[75068]: cluster 1685823053.755698 mon.a (mon.0) 981 : cluster 0 osdmap e187: 8 total, 8 up, 7 in 2023-06-03T20:10:54.376 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:10:54 smithi089 bash[78452]: cluster 1685823053.7504578 mon.a (mon.0) 979 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-06-03T20:10:54.376 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:10:54 smithi089 bash[78452]: audit 1685823053.755597 mon.a (mon.0) 980 : audit 1 from='client.? 172.21.15.89:0/4047100895' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-06-03T20:10:54.376 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:10:54 smithi089 bash[78452]: cluster 1685823053.755698 mon.a (mon.0) 981 : cluster 0 osdmap e187: 8 total, 8 up, 7 in 2023-06-03T20:10:54.394 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:10:54.397 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:10:54.398 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:10:54.499 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:10:54.529 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:10:54.531 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:10:54.532 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:10:54.583 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:10:54 smithi164 bash[59462]: cluster 1685823053.7504578 mon.a (mon.0) 979 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-06-03T20:10:54.584 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:10:54 smithi164 bash[59462]: audit 1685823053.755597 mon.a (mon.0) 980 : audit 1 from='client.? 172.21.15.89:0/4047100895' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-06-03T20:10:54.584 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:10:54 smithi164 bash[59462]: cluster 1685823053.755698 mon.a (mon.0) 981 : cluster 0 osdmap e187: 8 total, 8 up, 7 in 2023-06-03T20:10:54.633 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:10:54.670 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:10:54.673 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:10:54.674 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:10:54.775 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:10:54.806 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:10:54.808 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:10:54.809 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:10:54.910 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:10:54.937 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:10:54.939 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:10:54.940 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:10:55.041 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:10:55.069 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:10:55.071 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:10:55.072 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:10:55.174 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:10:55.213 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:10:55.217 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:10:55.218 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:10:55.318 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:10:55.353 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:10:55.356 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:10:55.357 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:10:55.458 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:10:55.472 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:10:55 smithi164 bash[59462]: cluster 1685823053.751947 osd.6 (osd.6) 129 : cluster 3 failed to encode map e187 with expected crc 2023-06-03T20:10:55.472 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:10:55 smithi164 bash[59462]: cluster 1685823054.438912 osd.7 (osd.7) 122 : cluster 3 failed to encode map e187 with expected crc 2023-06-03T20:10:55.472 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:10:55 smithi164 bash[59462]: cluster 1685823054.439072 osd.5 (osd.5) 118 : cluster 3 failed to encode map e187 with expected crc 2023-06-03T20:10:55.472 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:10:55 smithi164 bash[59462]: cluster 1685823054.662723 mgr.y (mgr.34104) 1043 : cluster 0 pgmap v1004: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 624 GiB / 626 GiB avail 2023-06-03T20:10:55.498 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:10:55.500 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:10:55.501 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:10:55.528 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:10:55 smithi089 bash[75068]: cluster 1685823053.751947 osd.6 (osd.6) 129 : cluster 3 failed to encode map e187 with expected crc 2023-06-03T20:10:55.528 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:10:55 smithi089 bash[75068]: cluster 1685823054.438912 osd.7 (osd.7) 122 : cluster 3 failed to encode map e187 with expected crc 2023-06-03T20:10:55.529 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:10:55 smithi089 bash[75068]: cluster 1685823054.439072 osd.5 (osd.5) 118 : cluster 3 failed to encode map e187 with expected crc 2023-06-03T20:10:55.529 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:10:55 smithi089 bash[75068]: cluster 1685823054.662723 mgr.y (mgr.34104) 1043 : cluster 0 pgmap v1004: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 624 GiB / 626 GiB avail 2023-06-03T20:10:55.529 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:10:55 smithi089 bash[78452]: cluster 1685823053.751947 osd.6 (osd.6) 129 : cluster 3 failed to encode map e187 with expected crc 2023-06-03T20:10:55.529 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:10:55 smithi089 bash[78452]: cluster 1685823054.438912 osd.7 (osd.7) 122 : cluster 3 failed to encode map e187 with expected crc 2023-06-03T20:10:55.529 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:10:55 smithi089 bash[78452]: cluster 1685823054.439072 osd.5 (osd.5) 118 : cluster 3 failed to encode map e187 with expected crc 2023-06-03T20:10:55.529 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:10:55 smithi089 bash[78452]: cluster 1685823054.662723 mgr.y (mgr.34104) 1043 : cluster 0 pgmap v1004: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 624 GiB / 626 GiB avail 2023-06-03T20:10:55.601 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:10:55.639 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:10:55.640 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:10:55.641 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:10:55.742 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:10:55.769 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:10:55.772 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:10:55.772 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:10:55.873 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:10:55.910 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:10:55.912 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:10:55.913 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:10:56.014 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:10:56.049 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:10:56.052 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:10:56.053 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:10:56.154 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:10:56.191 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:10:56.194 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:10:56.194 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:10:56.295 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:10:56.333 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:10:56.336 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:10:56.336 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:10:56.438 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:10:56.475 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:10:56.478 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:10:56.478 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:10:56.488 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 20:10:56 smithi089 bash[71163]: ::ffff:172.21.15.164 - - [03/Jun/2023:20:10:56] "GET /metrics HTTP/1.1" 200 226303 "" "Prometheus/2.33.4" 2023-06-03T20:10:56.580 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:10:56.614 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:10:56.617 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:10:56.618 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:10:56.719 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:10:56.748 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:10:56.750 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:10:56.751 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:10:56.852 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:10:56.890 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:10:56.892 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:10:56.893 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:10:56.994 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:10:57.028 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:10:57.031 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:10:57.032 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:10:57.058 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/mon.a/config 2023-06-03T20:10:57.133 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:10:57.169 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:10:57.171 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:10:57.171 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:10:57.253 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-06-03T20:10:57.273 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:10:57.333 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:10:57.336 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:10:57.336 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:10:57.437 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:10:57.460 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:10:57.462 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:10:57.463 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:10:57.563 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:10:57.598 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:10:57.601 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:10:57.601 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:10:57.702 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:10:57.727 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:10:57.730 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:10:57.731 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:10:57.778 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:10:57 smithi089 bash[75068]: cluster 1685823056.6635027 mgr.y (mgr.34104) 1044 : cluster 0 pgmap v1005: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 624 GiB / 626 GiB avail 2023-06-03T20:10:57.778 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:10:57 smithi089 bash[78452]: cluster 1685823056.6635027 mgr.y (mgr.34104) 1044 : cluster 0 pgmap v1005: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 624 GiB / 626 GiB avail 2023-06-03T20:10:57.832 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:10:57.833 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:10:57 smithi164 bash[59462]: cluster 1685823056.6635027 mgr.y (mgr.34104) 1044 : cluster 0 pgmap v1005: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 624 GiB / 626 GiB avail 2023-06-03T20:10:57.860 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:10:57.862 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:10:57.862 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:10:57.963 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:10:57.994 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:10:57.996 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:10:57.997 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:10:58.098 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:10:58.137 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:10:58.140 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:10:58.141 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:10:58.242 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:10:58.282 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:10:58.285 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:10:58.287 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:10:58.388 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:10:58.421 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:10:58.423 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:10:58.424 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:10:58.525 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:10:58.566 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:10:58.570 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:10:58.571 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:10:58.610 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph osd unset nodeep-scrub 2023-06-03T20:10:58.673 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:10:58.697 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:10:58.700 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:10:58.701 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:10:58.802 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:10:58.828 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:10:58.832 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:10:58.833 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:10:58.934 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:10:58.969 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:10:58.972 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:10:58.973 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:10:59.074 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:10:59.111 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:10:59.113 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:10:59.114 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:10:59.215 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:10:59.254 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:10:59.257 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:10:59.258 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:10:59.359 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:10:59.396 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:10:59.399 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:10:59.400 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:10:59.501 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:10:59.542 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:10:59.545 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:10:59.546 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:10:59.647 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:10:59.682 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:10:59.686 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:10:59.687 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:10:59.788 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:10:59.801 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:10:59 smithi164 bash[59462]: cluster 1685823058.6640103 mgr.y (mgr.34104) 1045 : cluster 0 pgmap v1006: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 624 GiB / 626 GiB avail 2023-06-03T20:10:59.825 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:10:59.829 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:10:59.830 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:10:59.931 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:10:59.964 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:10:59.967 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:10:59.967 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:11:00.028 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:10:59 smithi089 bash[78452]: cluster 1685823058.6640103 mgr.y (mgr.34104) 1045 : cluster 0 pgmap v1006: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 624 GiB / 626 GiB avail 2023-06-03T20:11:00.028 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:10:59 smithi089 bash[75068]: cluster 1685823058.6640103 mgr.y (mgr.34104) 1045 : cluster 0 pgmap v1006: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 624 GiB / 626 GiB avail 2023-06-03T20:11:00.068 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:11:00.107 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:11:00.110 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:11:00.110 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:11:00.211 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:11:00.250 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:11:00.253 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:11:00.254 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:11:00.355 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:11:00.389 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:11:00.391 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:11:00.392 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:11:00.493 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:11:00.516 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:11:00.519 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:11:00.521 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:11:00.622 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:11:00.662 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:11:00.664 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:11:00.665 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:11:00.766 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:11:00.807 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:11:00.810 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:11:00.811 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:11:00.912 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:11:00.940 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:11:00.942 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:11:00.942 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:11:01.043 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:11:01.071 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:11:01.073 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:11:01.074 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:11:01.175 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:11:01.212 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:11:01.215 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:11:01.215 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:11:01.316 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:11:01.350 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:11:01.352 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:11:01.353 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:11:01.454 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:11:01.489 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:11:01.492 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:11:01.493 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:11:01.594 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:11:01.622 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:11:01.624 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:11:01.625 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:11:01.726 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:11:01.760 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:11:01.763 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:11:01.764 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:11:01.831 INFO:teuthology.orchestra.run.smithi089.stdout:ERROR: (22) Invalid argument 2023-06-03T20:11:01.832 INFO:teuthology.orchestra.run.smithi089.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-06-03T20:11:01.864 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:11:01.907 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:11:01.910 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:11:01.911 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:11:02.012 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:11:02.024 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:11:01 smithi164 bash[59462]: cluster 1685823060.6644888 mgr.y (mgr.34104) 1046 : cluster 0 pgmap v1007: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 624 GiB / 626 GiB avail 2023-06-03T20:11:02.050 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:11:02.053 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:11:02.053 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:11:02.060 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:11:01 smithi089 bash[78452]: cluster 1685823060.6644888 mgr.y (mgr.34104) 1046 : cluster 0 pgmap v1007: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 624 GiB / 626 GiB avail 2023-06-03T20:11:02.060 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:11:01 smithi089 bash[75068]: cluster 1685823060.6644888 mgr.y (mgr.34104) 1046 : cluster 0 pgmap v1007: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 624 GiB / 626 GiB avail 2023-06-03T20:11:02.154 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:11:02.188 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:11:02.191 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:11:02.191 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:11:02.292 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:11:02.323 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:11:02.325 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:11:02.326 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:11:02.427 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:11:02.466 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:11:02.470 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:11:02.471 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:11:02.572 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:11:02.608 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:11:02.611 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:11:02.611 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:11:02.712 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:11:02.746 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:11:02.749 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:11:02.749 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:11:02.850 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:11:02.890 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:11:02.893 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:11:02.894 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:11:02.995 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:11:03.022 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:11:03.024 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:11:03.024 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:11:03.125 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:11:03.151 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:11:03.153 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:11:03.154 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:11:03.255 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:11:03.287 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:11:03.290 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:11:03.290 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:11:03.391 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:11:03.418 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:11:03.419 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:11:03.420 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:11:03.521 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:11:03.554 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:11:03.557 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:11:03.557 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:11:03.658 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:11:03.690 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:11:03.693 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:11:03.693 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:11:03.794 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:11:03.825 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:11:03.827 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:11:03.828 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:11:03.929 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:11:03.936 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:11:03 smithi089 bash[78452]: cluster 1685823062.6651978 mgr.y (mgr.34104) 1047 : cluster 0 pgmap v1008: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 624 GiB / 626 GiB avail 2023-06-03T20:11:03.937 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:11:03 smithi089 bash[75068]: cluster 1685823062.6651978 mgr.y (mgr.34104) 1047 : cluster 0 pgmap v1008: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 624 GiB / 626 GiB avail 2023-06-03T20:11:03.954 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:11:03.956 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:11:03.956 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:11:04.057 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:11:04.083 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:11:03 smithi164 bash[59462]: cluster 1685823062.6651978 mgr.y (mgr.34104) 1047 : cluster 0 pgmap v1008: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 624 GiB / 626 GiB avail 2023-06-03T20:11:04.090 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:11:04.092 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:11:04.093 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:11:04.194 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:11:04.227 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:11:04.230 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:11:04.231 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:11:04.332 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:11:04.367 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:11:04.369 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:11:04.370 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:11:04.471 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:11:04.497 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:11:04.500 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:11:04.501 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:11:04.569 DEBUG:teuthology.orchestra.run.smithi164:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph daemon osd.4 dump_ops_in_flight 2023-06-03T20:11:04.602 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:11:04.634 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:11:04.637 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:11:04.638 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:11:04.739 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:11:04.767 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:11:04.769 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:11:04.770 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:11:04.871 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:11:04.906 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:11:04.909 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:11:04.910 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:11:05.011 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:11:05.040 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:11:05.042 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:11:05.042 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:11:05.143 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:11:05.182 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:11:05.185 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:11:05.186 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:11:05.224 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/mon.a/config 2023-06-03T20:11:05.224 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/mon.a/config 2023-06-03T20:11:05.286 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:11:05.317 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:11:05.319 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:11:05.319 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:11:05.420 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:11:05.451 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:11:05.453 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:11:05.454 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:11:05.555 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:11:05.585 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:11:05.589 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:11:05.589 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:11:05.690 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:11:05.720 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:11:05.722 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:11:05.722 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:11:05.747 INFO:teuthology.orchestra.run.smithi089.stderr:reweighted osd.7 to 1 (10000) 2023-06-03T20:11:05.823 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:11:05.860 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:11:05.864 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:11:05.865 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:11:05.966 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:11:05.990 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:11:05 smithi089 bash[75068]: cluster 1685823064.6659174 mgr.y (mgr.34104) 1048 : cluster 0 pgmap v1009: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 624 GiB / 626 GiB avail 2023-06-03T20:11:05.990 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:11:05 smithi089 bash[75068]: audit 1685823065.1315944 mon.a (mon.0) 982 : audit 1 from='client.? 172.21.15.89:0/293717979' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-06-03T20:11:05.991 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:11:05 smithi089 bash[78452]: cluster 1685823064.6659174 mgr.y (mgr.34104) 1048 : cluster 0 pgmap v1009: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 624 GiB / 626 GiB avail 2023-06-03T20:11:05.991 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:11:05 smithi089 bash[78452]: audit 1685823065.1315944 mon.a (mon.0) 982 : audit 1 from='client.? 172.21.15.89:0/293717979' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-06-03T20:11:05.997 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:11:05.999 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:11:06.000 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:11:06.083 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:11:05 smithi164 bash[59462]: cluster 1685823064.6659174 mgr.y (mgr.34104) 1048 : cluster 0 pgmap v1009: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 624 GiB / 626 GiB avail 2023-06-03T20:11:06.084 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:11:05 smithi164 bash[59462]: audit 1685823065.1315944 mon.a (mon.0) 982 : audit 1 from='client.? 172.21.15.89:0/293717979' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-06-03T20:11:06.101 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:11:06.128 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:11:06.130 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:11:06.131 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:11:06.232 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:11:06.266 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:11:06.270 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:11:06.271 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:11:06.278 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 20:11:06 smithi089 bash[71163]: ::ffff:172.21.15.164 - - [03/Jun/2023:20:11:06] "GET /metrics HTTP/1.1" 200 226303 "" "Prometheus/2.33.4" 2023-06-03T20:11:06.372 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:11:06.406 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:11:06.409 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:11:06.409 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:11:06.510 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:11:06.546 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:11:06.549 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:11:06.550 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:11:06.651 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:11:06.687 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:11:06.690 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:11:06.691 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:11:06.791 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:11:06.817 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:11:06.819 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:11:06.820 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:11:06.833 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:11:06 smithi164 bash[59462]: cluster 1685823065.7353332 osd.6 (osd.6) 130 : cluster 3 failed to encode map e188 with expected crc 2023-06-03T20:11:06.833 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:11:06 smithi164 bash[59462]: cluster 1685823065.7358806 osd.5 (osd.5) 119 : cluster 3 failed to encode map e188 with expected crc 2023-06-03T20:11:06.833 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:11:06 smithi164 bash[59462]: audit 1685823065.739431 mon.a (mon.0) 983 : audit 1 from='client.? 172.21.15.89:0/293717979' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2023-06-03T20:11:06.834 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:11:06 smithi164 bash[59462]: cluster 1685823065.7395127 mon.a (mon.0) 984 : cluster 0 osdmap e188: 8 total, 8 up, 7 in 2023-06-03T20:11:06.920 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:11:06.954 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:11:06.958 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:11:06.958 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:11:07.028 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:11:06 smithi089 bash[75068]: cluster 1685823065.7353332 osd.6 (osd.6) 130 : cluster 3 failed to encode map e188 with expected crc 2023-06-03T20:11:07.028 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:11:06 smithi089 bash[75068]: cluster 1685823065.7358806 osd.5 (osd.5) 119 : cluster 3 failed to encode map e188 with expected crc 2023-06-03T20:11:07.028 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:11:06 smithi089 bash[75068]: audit 1685823065.739431 mon.a (mon.0) 983 : audit 1 from='client.? 172.21.15.89:0/293717979' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2023-06-03T20:11:07.029 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:11:06 smithi089 bash[75068]: cluster 1685823065.7395127 mon.a (mon.0) 984 : cluster 0 osdmap e188: 8 total, 8 up, 7 in 2023-06-03T20:11:07.029 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:11:06 smithi089 bash[78452]: cluster 1685823065.7353332 osd.6 (osd.6) 130 : cluster 3 failed to encode map e188 with expected crc 2023-06-03T20:11:07.029 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:11:06 smithi089 bash[78452]: cluster 1685823065.7358806 osd.5 (osd.5) 119 : cluster 3 failed to encode map e188 with expected crc 2023-06-03T20:11:07.029 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:11:06 smithi089 bash[78452]: audit 1685823065.739431 mon.a (mon.0) 983 : audit 1 from='client.? 172.21.15.89:0/293717979' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2023-06-03T20:11:07.029 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:11:06 smithi089 bash[78452]: cluster 1685823065.7395127 mon.a (mon.0) 984 : cluster 0 osdmap e188: 8 total, 8 up, 7 in 2023-06-03T20:11:07.059 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:11:07.084 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:11:07.086 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:11:07.087 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:11:07.188 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:11:07.222 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:11:07.226 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:11:07.226 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:11:07.328 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:11:07.362 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:11:07.364 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:11:07.365 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:11:07.466 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:11:07.494 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:11:07.496 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:11:07.497 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:11:07.598 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:11:07.624 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:11:07.627 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:11:07.628 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:11:07.729 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:11:07.759 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:11:07.763 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:11:07.764 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:11:07.778 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:11:07 smithi089 bash[75068]: cluster 1685823066.666729 mgr.y (mgr.34104) 1049 : cluster 0 pgmap v1011: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 624 GiB / 626 GiB avail 2023-06-03T20:11:07.778 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:11:07 smithi089 bash[75068]: audit 1685823067.1159403 mon.c (mon.2) 493 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T20:11:07.778 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:11:07 smithi089 bash[78452]: cluster 1685823066.666729 mgr.y (mgr.34104) 1049 : cluster 0 pgmap v1011: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 624 GiB / 626 GiB avail 2023-06-03T20:11:07.779 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:11:07 smithi089 bash[78452]: audit 1685823067.1159403 mon.c (mon.2) 493 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T20:11:07.865 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:11:07.889 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:11:07.891 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:11:07.892 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:11:07.993 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:11:08.033 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:11:08.036 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:11:07 smithi164 bash[59462]: cluster 1685823066.666729 mgr.y (mgr.34104) 1049 : cluster 0 pgmap v1011: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 624 GiB / 626 GiB avail 2023-06-03T20:11:08.036 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:11:07 smithi164 bash[59462]: audit 1685823067.1159403 mon.c (mon.2) 493 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T20:11:08.036 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:11:08.037 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:11:08.138 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:11:08.169 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:11:08.170 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:11:08.171 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:11:08.272 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:11:08.304 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:11:08.306 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:11:08.307 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:11:08.408 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:11:08.443 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:11:08.447 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:11:08.448 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:11:08.549 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:11:08.586 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:11:08.589 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:11:08.590 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:11:08.691 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:11:08.728 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:11:08.730 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:11:08.731 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:11:08.831 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:11:08.859 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:11:08.861 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:11:08.862 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:11:08.963 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:11:09.004 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:11:09.007 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:11:09.007 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:11:09.108 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:11:09.150 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:11:09.153 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:11:09.154 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:11:09.255 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:11:09.282 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:11:09.284 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:11:09.284 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:11:09.380 INFO:teuthology.orchestra.run.smithi164.stderr:Inferring config /var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/mon.b/config 2023-06-03T20:11:09.385 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:11:09.421 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:11:09.423 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:11:09.424 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:11:09.525 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:11:09.556 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:11:09.558 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:11:09.559 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:11:09.660 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:11:09.697 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:11:09.700 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:11:09.701 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:11:09.766 INFO:tasks.cephadm.ceph_manager.ceph:waiting for recovery to complete 2023-06-03T20:11:09.766 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph pg dump --format=json 2023-06-03T20:11:09.799 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:11:09 smithi089 bash[75068]: cluster 1685823068.6672878 mgr.y (mgr.34104) 1050 : cluster 0 pgmap v1012: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 624 GiB / 626 GiB avail 2023-06-03T20:11:09.799 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:11:09 smithi089 bash[78452]: cluster 1685823068.6672878 mgr.y (mgr.34104) 1050 : cluster 0 pgmap v1012: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 624 GiB / 626 GiB avail 2023-06-03T20:11:09.802 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:11:09.833 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:11:09.838 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:11:09.839 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:11:09.940 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:11:09.967 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:11:09.971 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:11:09.972 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:11:10.073 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:11:10.082 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:11:09 smithi164 bash[59462]: cluster 1685823068.6672878 mgr.y (mgr.34104) 1050 : cluster 0 pgmap v1012: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 624 GiB / 626 GiB avail 2023-06-03T20:11:10.104 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:11:10.107 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:11:10.108 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:11:10.209 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:11:10.247 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:11:10.250 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:11:10.251 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:11:10.352 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:11:10.385 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:11:10.387 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:11:10.388 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:11:10.489 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:11:10.518 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:11:10.521 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:11:10.521 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:11:10.622 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:11:10.658 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:11:10.661 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:11:10.662 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:11:10.763 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:11:10.775 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:11:10 smithi089 bash[75068]: audit 1685823070.2050347 mon.c (mon.2) 494 : audit 1 from='client.? 172.21.15.89:0/4050920637' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_6", "pg_num": 16}]: dispatch 2023-06-03T20:11:10.775 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:11:10 smithi089 bash[75068]: audit 1685823070.2056959 mon.a (mon.0) 985 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_6", "pg_num": 16}]: dispatch 2023-06-03T20:11:10.784 INFO:teuthology.orchestra.run.smithi089.stderr:pool 'unique_pool_6' created 2023-06-03T20:11:10.796 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:11:10.798 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:11:10.799 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:11:10.900 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:11:10.914 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:11:10 smithi164 bash[59462]: audit 1685823070.2050347 mon.c (mon.2) 494 : audit 1 from='client.? 172.21.15.89:0/4050920637' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_6", "pg_num": 16}]: dispatch 2023-06-03T20:11:10.914 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:11:10 smithi164 bash[59462]: audit 1685823070.2056959 mon.a (mon.0) 985 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_6", "pg_num": 16}]: dispatch 2023-06-03T20:11:10.935 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:11:10.938 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:11:10.939 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:11:11.039 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:11:11.073 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:11:11.075 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:11:11.076 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:11:11.076 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:11:10 smithi089 bash[78452]: audit 1685823070.2050347 mon.c (mon.2) 494 : audit 1 from='client.? 172.21.15.89:0/4050920637' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_6", "pg_num": 16}]: dispatch 2023-06-03T20:11:11.076 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:11:10 smithi089 bash[78452]: audit 1685823070.2056959 mon.a (mon.0) 985 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_6", "pg_num": 16}]: dispatch 2023-06-03T20:11:11.177 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:11:11.207 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:11:11.211 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:11:11.212 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:11:11.313 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:11:11.346 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:11:11.348 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:11:11.349 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:11:11.451 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:11:11.483 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:11:11.485 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:11:11.485 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:11:11.586 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:11:11.617 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:11:11.620 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:11:11.620 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:11:11.721 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:11:11.757 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:11:11.759 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:11:11.760 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:11:11.861 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:11:11.952 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:11:11 smithi089 bash[78452]: cluster 1685823070.6677415 mgr.y (mgr.34104) 1051 : cluster 0 pgmap v1013: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 624 GiB / 626 GiB avail 2023-06-03T20:11:11.952 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:11:11 smithi089 bash[78452]: cluster 1685823070.7667022 osd.6 (osd.6) 131 : cluster 3 failed to encode map e189 with expected crc 2023-06-03T20:11:11.952 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:11:11 smithi089 bash[78452]: audit 1685823070.772098 mon.a (mon.0) 986 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_6", "pg_num": 16}]': finished 2023-06-03T20:11:11.952 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:11:11 smithi089 bash[78452]: cluster 1685823070.772176 mon.a (mon.0) 987 : cluster 0 osdmap e189: 8 total, 8 up, 7 in 2023-06-03T20:11:11.953 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:11:11 smithi089 bash[78452]: cluster 1685823070.772828 osd.7 (osd.7) 123 : cluster 3 failed to encode map e189 with expected crc 2023-06-03T20:11:11.953 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:11:11 smithi089 bash[75068]: cluster 1685823070.6677415 mgr.y (mgr.34104) 1051 : cluster 0 pgmap v1013: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 624 GiB / 626 GiB avail 2023-06-03T20:11:11.953 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:11:11 smithi089 bash[75068]: cluster 1685823070.7667022 osd.6 (osd.6) 131 : cluster 3 failed to encode map e189 with expected crc 2023-06-03T20:11:11.953 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:11:11 smithi089 bash[75068]: audit 1685823070.772098 mon.a (mon.0) 986 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_6", "pg_num": 16}]': finished 2023-06-03T20:11:11.953 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:11:11 smithi089 bash[75068]: cluster 1685823070.772176 mon.a (mon.0) 987 : cluster 0 osdmap e189: 8 total, 8 up, 7 in 2023-06-03T20:11:11.953 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:11:11 smithi089 bash[75068]: cluster 1685823070.772828 osd.7 (osd.7) 123 : cluster 3 failed to encode map e189 with expected crc 2023-06-03T20:11:11.972 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:11:11.975 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:11:11.975 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:11:12.076 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:11:12.083 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:11:11 smithi164 bash[59462]: cluster 1685823070.6677415 mgr.y (mgr.34104) 1051 : cluster 0 pgmap v1013: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 624 GiB / 626 GiB avail 2023-06-03T20:11:12.083 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:11:11 smithi164 bash[59462]: cluster 1685823070.7667022 osd.6 (osd.6) 131 : cluster 3 failed to encode map e189 with expected crc 2023-06-03T20:11:12.083 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:11:11 smithi164 bash[59462]: audit 1685823070.772098 mon.a (mon.0) 986 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_6", "pg_num": 16}]': finished 2023-06-03T20:11:12.084 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:11:11 smithi164 bash[59462]: cluster 1685823070.772176 mon.a (mon.0) 987 : cluster 0 osdmap e189: 8 total, 8 up, 7 in 2023-06-03T20:11:12.084 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:11:11 smithi164 bash[59462]: cluster 1685823070.772828 osd.7 (osd.7) 123 : cluster 3 failed to encode map e189 with expected crc 2023-06-03T20:11:12.114 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:11:12.116 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:11:12.118 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:11:12.219 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:11:12.250 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:11:12.252 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:11:12.253 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:11:12.354 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:11:12.385 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:11:12.387 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:11:12.387 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:11:12.488 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:11:12.522 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:11:12.524 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:11:12.525 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:11:12.625 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:11:12.659 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:11:12.662 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:11:12.662 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:11:12.763 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:11:12.803 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:11:12.805 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:11:12.806 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:11:12.907 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:11:12.918 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:11:12 smithi089 bash[75068]: cluster 1685823071.775276 osd.6 (osd.6) 132 : cluster 3 failed to encode map e190 with expected crc 2023-06-03T20:11:12.919 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:11:12 smithi089 bash[75068]: cluster 1685823071.7794373 osd.7 (osd.7) 124 : cluster 3 failed to encode map e190 with expected crc 2023-06-03T20:11:12.919 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:11:12 smithi089 bash[75068]: cluster 1685823071.779537 osd.5 (osd.5) 120 : cluster 3 failed to encode map e190 with expected crc 2023-06-03T20:11:12.919 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:11:12 smithi089 bash[75068]: cluster 1685823071.7796319 osd.6 (osd.6) 133 : cluster 3 failed to encode map e190 with expected crc 2023-06-03T20:11:12.919 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:11:12 smithi089 bash[75068]: cluster 1685823071.7809236 mon.a (mon.0) 988 : cluster 0 osdmap e190: 8 total, 8 up, 7 in 2023-06-03T20:11:12.919 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:11:12 smithi089 bash[78452]: cluster 1685823071.775276 osd.6 (osd.6) 132 : cluster 3 failed to encode map e190 with expected crc 2023-06-03T20:11:12.920 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:11:12 smithi089 bash[78452]: cluster 1685823071.7794373 osd.7 (osd.7) 124 : cluster 3 failed to encode map e190 with expected crc 2023-06-03T20:11:12.920 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:11:12 smithi089 bash[78452]: cluster 1685823071.779537 osd.5 (osd.5) 120 : cluster 3 failed to encode map e190 with expected crc 2023-06-03T20:11:12.920 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:11:12 smithi089 bash[78452]: cluster 1685823071.7796319 osd.6 (osd.6) 133 : cluster 3 failed to encode map e190 with expected crc 2023-06-03T20:11:12.920 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:11:12 smithi089 bash[78452]: cluster 1685823071.7809236 mon.a (mon.0) 988 : cluster 0 osdmap e190: 8 total, 8 up, 7 in 2023-06-03T20:11:12.937 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:11:12.939 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:11:12.939 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:11:13.040 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:11:13.053 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:11:12 smithi164 bash[59462]: cluster 1685823071.775276 osd.6 (osd.6) 132 : cluster 3 failed to encode map e190 with expected crc 2023-06-03T20:11:13.053 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:11:12 smithi164 bash[59462]: cluster 1685823071.7794373 osd.7 (osd.7) 124 : cluster 3 failed to encode map e190 with expected crc 2023-06-03T20:11:13.053 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:11:12 smithi164 bash[59462]: cluster 1685823071.779537 osd.5 (osd.5) 120 : cluster 3 failed to encode map e190 with expected crc 2023-06-03T20:11:13.053 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:11:12 smithi164 bash[59462]: cluster 1685823071.7796319 osd.6 (osd.6) 133 : cluster 3 failed to encode map e190 with expected crc 2023-06-03T20:11:13.054 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:11:12 smithi164 bash[59462]: cluster 1685823071.7809236 mon.a (mon.0) 988 : cluster 0 osdmap e190: 8 total, 8 up, 7 in 2023-06-03T20:11:13.079 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:11:13.082 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:11:13.083 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:11:13.183 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:11:13.210 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:11:13.211 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:11:13.212 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:11:13.313 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:11:13.350 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:11:13.352 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:11:13.353 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:11:13.454 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:11:13.480 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:11:13.482 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:11:13.483 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:11:13.584 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:11:13.616 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:11:13.620 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:11:13.620 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:11:13.721 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:11:13.752 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:11:13.755 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:11:13.755 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:11:13.772 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph osd pool application enable unique_pool_6 rados --yes-i-really-mean-it || true 2023-06-03T20:11:13.856 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:11:13.891 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:11:13.894 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:11:13.895 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:11:13.940 INFO:teuthology.orchestra.run.smithi164.stdout:ERROR: (22) Invalid argument 2023-06-03T20:11:13.941 INFO:teuthology.orchestra.run.smithi164.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-06-03T20:11:13.990 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:11:13 smithi164 bash[59462]: cluster 1685823072.6683192 mgr.y (mgr.34104) 1052 : cluster 0 pgmap v1016: 89 pgs: 1 creating+peering, 14 unknown, 74 active+clean; 582 KiB data, 2.3 GiB used, 624 GiB / 626 GiB avail 2023-06-03T20:11:13.991 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:11:13 smithi164 bash[59462]: cluster 1685823072.792834 mon.a (mon.0) 989 : cluster 0 osdmap e191: 8 total, 8 up, 7 in 2023-06-03T20:11:13.996 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:11:14.026 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:11:14.028 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:11:13 smithi089 bash[75068]: cluster 1685823072.6683192 mgr.y (mgr.34104) 1052 : cluster 0 pgmap v1016: 89 pgs: 1 creating+peering, 14 unknown, 74 active+clean; 582 KiB data, 2.3 GiB used, 624 GiB / 626 GiB avail 2023-06-03T20:11:14.028 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:11:13 smithi089 bash[75068]: cluster 1685823072.792834 mon.a (mon.0) 989 : cluster 0 osdmap e191: 8 total, 8 up, 7 in 2023-06-03T20:11:14.028 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:11:13 smithi089 bash[78452]: cluster 1685823072.6683192 mgr.y (mgr.34104) 1052 : cluster 0 pgmap v1016: 89 pgs: 1 creating+peering, 14 unknown, 74 active+clean; 582 KiB data, 2.3 GiB used, 624 GiB / 626 GiB avail 2023-06-03T20:11:14.029 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:11:13 smithi089 bash[78452]: cluster 1685823072.792834 mon.a (mon.0) 989 : cluster 0 osdmap e191: 8 total, 8 up, 7 in 2023-06-03T20:11:14.029 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:11:14.030 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:11:14.130 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:11:14.159 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:11:14.163 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:11:14.164 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:11:14.265 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:11:14.295 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:11:14.298 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:11:14.298 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:11:14.399 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:11:14.434 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:11:14.437 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:11:14.437 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:11:14.538 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:11:14.573 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:11:14.576 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:11:14.577 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:11:14.677 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:11:14.718 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:11:14.721 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:11:14.722 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:11:14.823 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:11:14.836 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:11:14 smithi089 bash[75068]: cluster 1685823073.56135 osd.6 (osd.6) 134 : cluster 3 failed to encode map e191 with expected crc 2023-06-03T20:11:14.837 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:11:14 smithi089 bash[75068]: cluster 1685823073.5642123 osd.6 (osd.6) 135 : cluster 3 failed to encode map e191 with expected crc 2023-06-03T20:11:14.837 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:11:14 smithi089 bash[75068]: cluster 1685823073.5671008 osd.6 (osd.6) 136 : cluster 3 failed to encode map e191 with expected crc 2023-06-03T20:11:14.837 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:11:14 smithi089 bash[75068]: cluster 1685823073.569554 osd.6 (osd.6) 137 : cluster 3 failed to encode map e191 with expected crc 2023-06-03T20:11:14.837 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:11:14 smithi089 bash[75068]: cluster 1685823073.7793934 osd.5 (osd.5) 121 : cluster 3 failed to encode map e191 with expected crc 2023-06-03T20:11:14.837 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:11:14 smithi089 bash[75068]: cluster 1685823073.7819636 osd.5 (osd.5) 122 : cluster 3 failed to encode map e191 with expected crc 2023-06-03T20:11:14.838 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:11:14 smithi089 bash[78452]: cluster 1685823073.56135 osd.6 (osd.6) 134 : cluster 3 failed to encode map e191 with expected crc 2023-06-03T20:11:14.838 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:11:14 smithi089 bash[78452]: cluster 1685823073.5642123 osd.6 (osd.6) 135 : cluster 3 failed to encode map e191 with expected crc 2023-06-03T20:11:14.838 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:11:14 smithi089 bash[78452]: cluster 1685823073.5671008 osd.6 (osd.6) 136 : cluster 3 failed to encode map e191 with expected crc 2023-06-03T20:11:14.838 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:11:14 smithi089 bash[78452]: cluster 1685823073.569554 osd.6 (osd.6) 137 : cluster 3 failed to encode map e191 with expected crc 2023-06-03T20:11:14.838 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:11:14 smithi089 bash[78452]: cluster 1685823073.7793934 osd.5 (osd.5) 121 : cluster 3 failed to encode map e191 with expected crc 2023-06-03T20:11:14.838 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:11:14 smithi089 bash[78452]: cluster 1685823073.7819636 osd.5 (osd.5) 122 : cluster 3 failed to encode map e191 with expected crc 2023-06-03T20:11:14.857 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:11:14.859 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:11:14.860 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:11:14.881 DEBUG:teuthology.orchestra.run.smithi164:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph daemon osd.4 dump_blocked_ops 2023-06-03T20:11:14.960 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:11:14.993 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:11:14.995 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:11:14.996 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:11:15.083 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:11:14 smithi164 bash[59462]: cluster 1685823073.56135 osd.6 (osd.6) 134 : cluster 3 failed to encode map e191 with expected crc 2023-06-03T20:11:15.083 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:11:14 smithi164 bash[59462]: cluster 1685823073.5642123 osd.6 (osd.6) 135 : cluster 3 failed to encode map e191 with expected crc 2023-06-03T20:11:15.083 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:11:14 smithi164 bash[59462]: cluster 1685823073.5671008 osd.6 (osd.6) 136 : cluster 3 failed to encode map e191 with expected crc 2023-06-03T20:11:15.084 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:11:14 smithi164 bash[59462]: cluster 1685823073.569554 osd.6 (osd.6) 137 : cluster 3 failed to encode map e191 with expected crc 2023-06-03T20:11:15.084 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:11:14 smithi164 bash[59462]: cluster 1685823073.7793934 osd.5 (osd.5) 121 : cluster 3 failed to encode map e191 with expected crc 2023-06-03T20:11:15.084 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:11:14 smithi164 bash[59462]: cluster 1685823073.7819636 osd.5 (osd.5) 122 : cluster 3 failed to encode map e191 with expected crc 2023-06-03T20:11:15.096 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:11:15.129 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:11:15.131 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:11:15.132 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:11:15.233 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:11:15.275 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:11:15.278 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:11:15.278 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:11:15.379 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:11:15.408 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:11:15.410 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:11:15.411 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:11:15.512 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:11:15.547 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:11:15.551 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:11:15.551 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:11:15.652 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:11:15.688 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:11:15.691 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:11:15.692 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:11:15.793 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:11:15.824 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:11:15.826 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:11:15.828 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:11:15.929 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:11:15.970 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:11:15.973 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:11:15.974 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:11:16.075 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:11:16.083 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:11:15 smithi164 bash[59462]: cluster 1685823074.327797 osd.7 (osd.7) 125 : cluster 3 failed to encode map e191 with expected crc 2023-06-03T20:11:16.083 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:11:15 smithi164 bash[59462]: cluster 1685823074.6690922 mgr.y (mgr.34104) 1053 : cluster 0 pgmap v1018: 89 pgs: 1 creating+peering, 3 unknown, 85 active+clean; 582 KiB data, 2.3 GiB used, 624 GiB / 626 GiB avail 2023-06-03T20:11:16.115 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:11:16.117 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:11:16.118 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:11:16.219 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:11:16.231 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:11:15 smithi089 bash[78452]: cluster 1685823074.327797 osd.7 (osd.7) 125 : cluster 3 failed to encode map e191 with expected crc 2023-06-03T20:11:16.231 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:11:15 smithi089 bash[78452]: cluster 1685823074.6690922 mgr.y (mgr.34104) 1053 : cluster 0 pgmap v1018: 89 pgs: 1 creating+peering, 3 unknown, 85 active+clean; 582 KiB data, 2.3 GiB used, 624 GiB / 626 GiB avail 2023-06-03T20:11:16.232 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:11:15 smithi089 bash[75068]: cluster 1685823074.327797 osd.7 (osd.7) 125 : cluster 3 failed to encode map e191 with expected crc 2023-06-03T20:11:16.232 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:11:15 smithi089 bash[75068]: cluster 1685823074.6690922 mgr.y (mgr.34104) 1053 : cluster 0 pgmap v1018: 89 pgs: 1 creating+peering, 3 unknown, 85 active+clean; 582 KiB data, 2.3 GiB used, 624 GiB / 626 GiB avail 2023-06-03T20:11:16.258 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:11:16.260 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:11:16.261 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:11:16.362 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:11:16.398 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:11:16.401 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:11:16.402 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:11:16.502 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:11:16.528 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 20:11:16 smithi089 bash[71163]: ::ffff:172.21.15.164 - - [03/Jun/2023:20:11:16] "GET /metrics HTTP/1.1" 200 226270 "" "Prometheus/2.33.4" 2023-06-03T20:11:16.535 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:11:16.539 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:11:16.539 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:11:16.640 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:11:16.678 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:11:16.680 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:11:16.681 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:11:16.781 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:11:16.814 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:11:16.817 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:11:16.819 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:11:16.920 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:11:16.955 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:11:16.958 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:11:16.959 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:11:17.060 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:11:17.086 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:11:17.089 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:11:17.089 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:11:17.136 INFO:teuthology.orchestra.run.smithi089.stdout:osd.0: {} 2023-06-03T20:11:17.160 INFO:teuthology.orchestra.run.smithi089.stdout:osd.1: {} 2023-06-03T20:11:17.188 INFO:teuthology.orchestra.run.smithi089.stdout:osd.2: {} 2023-06-03T20:11:17.190 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:11:17.215 INFO:teuthology.orchestra.run.smithi089.stdout:osd.3: {} 2023-06-03T20:11:17.224 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:11:17.227 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:11:17.228 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:11:17.242 INFO:teuthology.orchestra.run.smithi089.stdout:osd.4: {} 2023-06-03T20:11:17.265 INFO:teuthology.orchestra.run.smithi089.stdout:osd.5: {} 2023-06-03T20:11:17.288 INFO:teuthology.orchestra.run.smithi089.stdout:osd.6: {} 2023-06-03T20:11:17.317 INFO:teuthology.orchestra.run.smithi089.stdout:osd.7: {} 2023-06-03T20:11:17.319 INFO:teuthology.orchestra.run.smithi089.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-06-03T20:11:17.319 INFO:teuthology.orchestra.run.smithi089.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-06-03T20:11:17.319 INFO:teuthology.orchestra.run.smithi089.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-06-03T20:11:17.320 INFO:teuthology.orchestra.run.smithi089.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-06-03T20:11:17.320 INFO:teuthology.orchestra.run.smithi089.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-06-03T20:11:17.320 INFO:teuthology.orchestra.run.smithi089.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-06-03T20:11:17.320 INFO:teuthology.orchestra.run.smithi089.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-06-03T20:11:17.320 INFO:teuthology.orchestra.run.smithi089.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-06-03T20:11:17.323 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/mon.a/config 2023-06-03T20:11:17.330 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:11:17.365 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:11:17.368 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:11:17.368 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:11:17.469 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:11:17.496 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:11:17.498 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:11:17.498 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:11:17.599 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:11:17.622 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:11:17.624 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:11:17.625 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:11:17.726 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:11:17.756 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:11:17.760 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:11:17.761 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:11:17.835 INFO:teuthology.orchestra.run.smithi089.stderr:nodeep-scrub is unset 2023-06-03T20:11:17.860 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:11:17 smithi089 bash[78452]: cluster 1685823076.6695814 mgr.y (mgr.34104) 1054 : cluster 0 pgmap v1019: 89 pgs: 89 active+clean; 582 KiB data, 2.3 GiB used, 624 GiB / 626 GiB avail 2023-06-03T20:11:17.861 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:11:17 smithi089 bash[78452]: audit 1685823077.106691 mon.c (mon.2) 495 : audit 0 from='client.? 172.21.15.89:0/3945369761' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-03T20:11:17.861 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:11:17 smithi089 bash[78452]: audit 1685823077.5707266 mon.c (mon.2) 496 : audit 1 from='client.? 172.21.15.89:0/2982224590' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-06-03T20:11:17.861 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:11:17 smithi089 bash[78452]: audit 1685823077.5711823 mon.a (mon.0) 990 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-06-03T20:11:17.861 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:11:17 smithi089 bash[75068]: cluster 1685823076.6695814 mgr.y (mgr.34104) 1054 : cluster 0 pgmap v1019: 89 pgs: 89 active+clean; 582 KiB data, 2.3 GiB used, 624 GiB / 626 GiB avail 2023-06-03T20:11:17.862 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:11:17 smithi089 bash[75068]: audit 1685823077.106691 mon.c (mon.2) 495 : audit 0 from='client.? 172.21.15.89:0/3945369761' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-03T20:11:17.862 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:11:17 smithi089 bash[75068]: audit 1685823077.5707266 mon.c (mon.2) 496 : audit 1 from='client.? 172.21.15.89:0/2982224590' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-06-03T20:11:17.862 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:11:17 smithi089 bash[75068]: audit 1685823077.5711823 mon.a (mon.0) 990 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-06-03T20:11:17.862 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:11:17.891 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:11:17.893 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:11:17.893 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:11:17.994 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:11:18.025 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:11:18.027 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:11:18.028 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:11:18.083 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:11:17 smithi164 bash[59462]: cluster 1685823076.6695814 mgr.y (mgr.34104) 1054 : cluster 0 pgmap v1019: 89 pgs: 89 active+clean; 582 KiB data, 2.3 GiB used, 624 GiB / 626 GiB avail 2023-06-03T20:11:18.083 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:11:17 smithi164 bash[59462]: audit 1685823077.106691 mon.c (mon.2) 495 : audit 0 from='client.? 172.21.15.89:0/3945369761' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-03T20:11:18.084 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:11:17 smithi164 bash[59462]: audit 1685823077.5707266 mon.c (mon.2) 496 : audit 1 from='client.? 172.21.15.89:0/2982224590' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-06-03T20:11:18.084 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:11:17 smithi164 bash[59462]: audit 1685823077.5711823 mon.a (mon.0) 990 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-06-03T20:11:18.128 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:11:18.169 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:11:18.172 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:11:18.173 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:11:18.273 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:11:18.305 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:11:18.309 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:11:18.309 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:11:18.410 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:11:18.447 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:11:18.450 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:11:18.451 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:11:18.552 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:11:18.581 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:11:18.584 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:11:18.585 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:11:18.685 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:11:18.711 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:11:18.713 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:11:18.714 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:11:18.815 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:11:18.848 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:11:18.851 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:11:18.852 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:11:18.895 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:11:18 smithi089 bash[75068]: cluster 1685823077.8021095 mon.a (mon.0) 991 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-06-03T20:11:18.896 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:11:18 smithi089 bash[75068]: cluster 1685823077.8021574 mon.a (mon.0) 992 : cluster 1 Cluster is now healthy 2023-06-03T20:11:18.896 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:11:18 smithi089 bash[75068]: cluster 1685823077.8171246 osd.6 (osd.6) 138 : cluster 3 failed to encode map e192 with expected crc 2023-06-03T20:11:18.896 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:11:18 smithi089 bash[75068]: audit 1685823077.8217754 mon.a (mon.0) 993 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-06-03T20:11:18.896 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:11:18 smithi089 bash[75068]: cluster 1685823077.821859 mon.a (mon.0) 994 : cluster 0 osdmap e192: 8 total, 8 up, 7 in 2023-06-03T20:11:18.896 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:11:18 smithi089 bash[75068]: cluster 1685823077.9808836 osd.5 (osd.5) 123 : cluster 3 failed to encode map e192 with expected crc 2023-06-03T20:11:18.896 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:11:18 smithi089 bash[75068]: cluster 1685823077.9811578 osd.7 (osd.7) 126 : cluster 3 failed to encode map e192 with expected crc 2023-06-03T20:11:18.897 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:11:18 smithi089 bash[78452]: cluster 1685823077.8021095 mon.a (mon.0) 991 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-06-03T20:11:18.897 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:11:18 smithi089 bash[78452]: cluster 1685823077.8021574 mon.a (mon.0) 992 : cluster 1 Cluster is now healthy 2023-06-03T20:11:18.897 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:11:18 smithi089 bash[78452]: cluster 1685823077.8171246 osd.6 (osd.6) 138 : cluster 3 failed to encode map e192 with expected crc 2023-06-03T20:11:18.897 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:11:18 smithi089 bash[78452]: audit 1685823077.8217754 mon.a (mon.0) 993 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-06-03T20:11:18.897 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:11:18 smithi089 bash[78452]: cluster 1685823077.821859 mon.a (mon.0) 994 : cluster 0 osdmap e192: 8 total, 8 up, 7 in 2023-06-03T20:11:18.897 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:11:18 smithi089 bash[78452]: cluster 1685823077.9808836 osd.5 (osd.5) 123 : cluster 3 failed to encode map e192 with expected crc 2023-06-03T20:11:18.898 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:11:18 smithi089 bash[78452]: cluster 1685823077.9811578 osd.7 (osd.7) 126 : cluster 3 failed to encode map e192 with expected crc 2023-06-03T20:11:18.953 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:11:18.994 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:11:18.997 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:11:18.997 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:11:19.083 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:11:18 smithi164 bash[59462]: cluster 1685823077.8021095 mon.a (mon.0) 991 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-06-03T20:11:19.084 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:11:18 smithi164 bash[59462]: cluster 1685823077.8021574 mon.a (mon.0) 992 : cluster 1 Cluster is now healthy 2023-06-03T20:11:19.084 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:11:18 smithi164 bash[59462]: cluster 1685823077.8171246 osd.6 (osd.6) 138 : cluster 3 failed to encode map e192 with expected crc 2023-06-03T20:11:19.084 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:11:18 smithi164 bash[59462]: audit 1685823077.8217754 mon.a (mon.0) 993 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-06-03T20:11:19.084 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:11:18 smithi164 bash[59462]: cluster 1685823077.821859 mon.a (mon.0) 994 : cluster 0 osdmap e192: 8 total, 8 up, 7 in 2023-06-03T20:11:19.085 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:11:18 smithi164 bash[59462]: cluster 1685823077.9808836 osd.5 (osd.5) 123 : cluster 3 failed to encode map e192 with expected crc 2023-06-03T20:11:19.085 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:11:18 smithi164 bash[59462]: cluster 1685823077.9811578 osd.7 (osd.7) 126 : cluster 3 failed to encode map e192 with expected crc 2023-06-03T20:11:19.099 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:11:19.134 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:11:19.136 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:11:19.137 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:11:19.238 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:11:19.270 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:11:19.272 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:11:19.273 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:11:19.374 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:11:19.413 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:11:19.415 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:11:19.416 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:11:19.517 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:11:19.549 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:11:19.552 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:11:19.553 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:11:19.654 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:11:19.694 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:11:19.701 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:11:19.702 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:11:19.702 INFO:teuthology.orchestra.run.smithi164.stderr:Inferring config /var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/mon.b/config 2023-06-03T20:11:19.803 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:11:19.832 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:11:19.835 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:11:19.836 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:11:19.937 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:11:19.946 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:11:19 smithi164 bash[59462]: cluster 1685823078.6702323 mgr.y (mgr.34104) 1055 : cluster 0 pgmap v1021: 89 pgs: 89 active+clean; 582 KiB data, 2.3 GiB used, 624 GiB / 626 GiB avail 2023-06-03T20:11:19.968 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:11:19.970 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:11:19.970 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:11:20.071 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:11:20.100 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:11:20.102 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:11:20.103 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:11:20.204 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:11:20.237 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:11:20.239 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:11:20.240 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:11:20.255 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:11:19 smithi089 bash[78452]: cluster 1685823078.6702323 mgr.y (mgr.34104) 1055 : cluster 0 pgmap v1021: 89 pgs: 89 active+clean; 582 KiB data, 2.3 GiB used, 624 GiB / 626 GiB avail 2023-06-03T20:11:20.255 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:11:19 smithi089 bash[75068]: cluster 1685823078.6702323 mgr.y (mgr.34104) 1055 : cluster 0 pgmap v1021: 89 pgs: 89 active+clean; 582 KiB data, 2.3 GiB used, 624 GiB / 626 GiB avail 2023-06-03T20:11:20.341 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:11:20.373 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:11:20.377 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:11:20.377 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:11:20.478 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:11:20.509 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:11:20.512 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:11:20.513 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:11:20.614 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:11:20.651 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:11:20.653 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:11:20.654 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:11:20.755 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:11:20.788 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:11:20.792 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:11:20.793 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:11:20.822 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/mon.a/config 2023-06-03T20:11:20.894 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:11:20.923 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:11:20.924 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:11:20.925 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:11:21.026 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:11:21.067 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:11:21.069 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:11:21.069 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:11:21.170 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:11:21.200 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:11:21.202 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:11:21.203 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:11:21.303 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:11:21.335 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:11:21.338 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:11:21.339 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:11:21.440 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:11:21.477 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:11:21.479 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:11:21.480 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:11:21.581 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:11:21.613 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:11:21.616 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:11:21.617 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:11:21.718 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:11:21.757 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:11:21.760 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:11:21.761 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:11:21.862 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:11:21.891 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:11:21.893 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:11:21.893 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:11:21.994 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:11:22.008 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:11:21 smithi164 bash[59462]: cluster 1685823080.67113 mgr.y (mgr.34104) 1056 : cluster 0 pgmap v1022: 89 pgs: 89 active+clean; 582 KiB data, 2.3 GiB used, 624 GiB / 626 GiB avail 2023-06-03T20:11:22.031 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:11:22.034 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:11:22.035 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:11:22.136 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:11:22.148 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:11:21 smithi089 bash[78452]: cluster 1685823080.67113 mgr.y (mgr.34104) 1056 : cluster 0 pgmap v1022: 89 pgs: 89 active+clean; 582 KiB data, 2.3 GiB used, 624 GiB / 626 GiB avail 2023-06-03T20:11:22.149 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:11:21 smithi089 bash[75068]: cluster 1685823080.67113 mgr.y (mgr.34104) 1056 : cluster 0 pgmap v1022: 89 pgs: 89 active+clean; 582 KiB data, 2.3 GiB used, 624 GiB / 626 GiB avail 2023-06-03T20:11:22.168 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:11:22.171 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:11:22.172 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:11:22.257 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph osd set noscrub 2023-06-03T20:11:22.274 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:11:22.310 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:11:22.312 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:11:22.313 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:11:22.346 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-06-03T20:11:22.415 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:11:22.446 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:11:22.450 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:11:22.451 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:11:22.552 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:11:22.587 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:11:22.591 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:11:22.592 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:11:22.693 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:11:22.732 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:11:22.734 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:11:22.735 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:11:22.836 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:11:22.865 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:11:22.866 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:11:22.867 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:11:22.968 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:11:22.992 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:11:22.996 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:11:22.997 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:11:23.098 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:11:23.131 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:11:23.133 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:11:23.134 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:11:23.236 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:11:23.266 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:11:23.269 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:11:23.269 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:11:23.370 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:11:23.411 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:11:23.413 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:11:23 smithi164 bash[59462]: audit 1685823082.1222253 mon.a (mon.0) 995 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:11:23.413 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:11:23 smithi164 bash[59462]: audit 1685823082.123881 mon.c (mon.2) 497 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T20:11:23.413 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:11:23 smithi164 bash[59462]: cluster 1685823082.6715796 mgr.y (mgr.34104) 1057 : cluster 0 pgmap v1023: 89 pgs: 89 active+clean; 582 KiB data, 2.3 GiB used, 624 GiB / 626 GiB avail 2023-06-03T20:11:23.414 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:11:23 smithi164 bash[59462]: audit 1685823082.7925777 mon.a (mon.0) 996 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:11:23.414 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:11:23.415 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:11:23.516 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:11:23.527 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:11:23 smithi089 bash[78452]: audit 1685823082.1222253 mon.a (mon.0) 995 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:11:23.528 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:11:23 smithi089 bash[78452]: audit 1685823082.123881 mon.c (mon.2) 497 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T20:11:23.528 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:11:23 smithi089 bash[78452]: cluster 1685823082.6715796 mgr.y (mgr.34104) 1057 : cluster 0 pgmap v1023: 89 pgs: 89 active+clean; 582 KiB data, 2.3 GiB used, 624 GiB / 626 GiB avail 2023-06-03T20:11:23.528 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:11:23 smithi089 bash[78452]: audit 1685823082.7925777 mon.a (mon.0) 996 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:11:23.528 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:11:23 smithi089 bash[75068]: audit 1685823082.1222253 mon.a (mon.0) 995 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:11:23.529 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:11:23 smithi089 bash[75068]: audit 1685823082.123881 mon.c (mon.2) 497 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T20:11:23.529 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:11:23 smithi089 bash[75068]: cluster 1685823082.6715796 mgr.y (mgr.34104) 1057 : cluster 0 pgmap v1023: 89 pgs: 89 active+clean; 582 KiB data, 2.3 GiB used, 624 GiB / 626 GiB avail 2023-06-03T20:11:23.529 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:11:23 smithi089 bash[75068]: audit 1685823082.7925777 mon.a (mon.0) 996 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:11:23.545 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:11:23.549 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:11:23.550 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:11:23.651 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:11:23.687 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:11:23.690 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:11:23.691 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:11:23.792 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:11:23.829 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:11:23.832 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:11:23.832 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:11:23.934 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:11:23.967 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:11:23.969 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:11:23.970 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:11:23.973 INFO:teuthology.orchestra.run.smithi164.stdout:{ 2023-06-03T20:11:23.973 INFO:teuthology.orchestra.run.smithi164.stdout: "ops": [], 2023-06-03T20:11:23.973 INFO:teuthology.orchestra.run.smithi164.stdout: "complaint_time": 30, 2023-06-03T20:11:23.973 INFO:teuthology.orchestra.run.smithi164.stdout: "num_blocked_ops": 0 2023-06-03T20:11:23.974 INFO:teuthology.orchestra.run.smithi164.stdout:} 2023-06-03T20:11:24.071 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:11:24.102 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:11:24.105 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:11:24.106 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:11:24.207 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:11:24.240 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:11:24.242 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:11:24.243 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:11:24.344 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:11:24.381 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:11:24.383 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:11:24.384 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:11:24.485 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:11:24.516 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:11:24.518 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:11:24.519 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:11:24.620 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:11:24.656 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:11:24.659 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:11:24.660 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:11:24.761 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:11:24.792 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:11:24.796 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:11:24.797 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:11:24.898 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:11:24.937 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:11:24.939 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:11:24.940 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:11:24.948 DEBUG:teuthology.orchestra.run.smithi164:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph daemon osd.4 dump_historic_ops 2023-06-03T20:11:25.042 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:11:25.063 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:11:25.065 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:11:25.066 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:11:25.167 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:11:25.188 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:11:25.190 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:11:25.191 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:11:25.253 INFO:teuthology.orchestra.run.smithi089.stdout: 2023-06-03T20:11:25.254 INFO:teuthology.orchestra.run.smithi089.stderr:dumped all 2023-06-03T20:11:25.292 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:11:25.324 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:11:25.326 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:11:25.327 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:11:25.428 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:11:25.457 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:11:25.460 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:11:25.461 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:11:25.563 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:11:25.597 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:11:25.601 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:11:25.601 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:11:25.702 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:11:25.741 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:11:25.743 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:11:25.744 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:11:25.833 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:11:25 smithi164 bash[59462]: cluster 1685823084.672591 mgr.y (mgr.34104) 1058 : cluster 0 pgmap v1024: 89 pgs: 89 active+clean; 582 KiB data, 2.3 GiB used, 624 GiB / 626 GiB avail 2023-06-03T20:11:25.846 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:11:25.885 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:11:25.888 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:11:25.889 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:11:25.990 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:11:26.005 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:11:25 smithi089 bash[75068]: cluster 1685823084.672591 mgr.y (mgr.34104) 1058 : cluster 0 pgmap v1024: 89 pgs: 89 active+clean; 582 KiB data, 2.3 GiB used, 624 GiB / 626 GiB avail 2023-06-03T20:11:26.005 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:11:25 smithi089 bash[78452]: cluster 1685823084.672591 mgr.y (mgr.34104) 1058 : cluster 0 pgmap v1024: 89 pgs: 89 active+clean; 582 KiB data, 2.3 GiB used, 624 GiB / 626 GiB avail 2023-06-03T20:11:26.029 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:11:26.031 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:11:26.032 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:11:26.133 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:11:26.169 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:11:26.172 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:11:26.173 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:11:26.274 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:11:26.300 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:11:26.302 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:11:26.303 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:11:26.404 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:11:26.442 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:11:26.444 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:11:26.445 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:11:26.528 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 20:11:26 smithi089 bash[71163]: ::ffff:172.21.15.164 - - [03/Jun/2023:20:11:26] "GET /metrics HTTP/1.1" 200 228632 "" "Prometheus/2.33.4" 2023-06-03T20:11:26.546 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:11:26.575 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:11:26.578 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:11:26.578 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:11:26.679 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:11:26.717 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:11:26.720 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:11:26.720 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:11:26.821 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:11:26.861 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:11:26.863 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:11:26.864 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:11:26.965 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:11:26.979 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:11:26 smithi164 bash[59462]: audit 1685823085.2301228 mgr.y (mgr.34104) 1059 : audit 0 from='client.36219 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-06-03T20:11:27.002 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:11:27.004 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:11:27.005 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:11:27.028 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:11:26 smithi089 bash[78452]: audit 1685823085.2301228 mgr.y (mgr.34104) 1059 : audit 0 from='client.36219 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-06-03T20:11:27.028 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:11:26 smithi089 bash[75068]: audit 1685823085.2301228 mgr.y (mgr.34104) 1059 : audit 0 from='client.36219 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-06-03T20:11:27.106 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:11:27.144 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:11:27.147 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:11:27.148 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:11:27.249 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:11:27.282 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:11:27.286 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:11:27.286 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:11:27.387 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:11:27.422 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:11:27.425 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:11:27.425 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:11:27.526 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:11:27.528 INFO:teuthology.orchestra.run.smithi089.stdout:{"pg_ready":true,"pg_map":{"version":1024,"stamp":"2023-06-03T20:11:24.671861+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":595854,"num_objects":24,"num_object_clones":0,"num_object_copies":72,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":24,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":261,"num_write_kb":2446,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":195,"ondisk_log_size":195,"up":267,"acting":267,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":267,"num_osds":7,"num_per_pool_osds":7,"num_per_pool_omap_osds":7,"kb":656216064,"kb_used":2377116,"kb_used_data":14884,"kb_used_omap":26,"kb_used_meta":2362149,"kb_avail":653838948,"statfs":{"total":671965249536,"available":669531082752,"internally_reserved":0,"allocated":15241216,"data_stored":10833247,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":27292,"internal_metadata":2418840932},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.003335"},"pg_stats":[{"pgid":"3.1a","version":"60'1","reported_seq":191,"reported_epoch":191,"state":"active+clean","last_fresh":"2023-06-03T20:11:12.787056+0000","last_change":"2023-06-03T20:07:19.904870+0000","last_active":"2023-06-03T20:11:12.787056+0000","last_peered":"2023-06-03T20:11:12.787056+0000","last_clean":"2023-06-03T20:11:12.787056+0000","last_became_active":"2023-06-03T20:07:19.904541+0000","last_became_peered":"2023-06-03T20:07:19.904541+0000","last_unstale":"2023-06-03T20:11:12.787056+0000","last_undegraded":"2023-06-03T20:11:12.787056+0000","last_fullsized":"2023-06-03T20:11:12.787056+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T01:48:10.778198+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,0],"acting":[4,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":32,"reported_epoch":191,"state":"active+clean","last_fresh":"2023-06-03T20:11:13.786202+0000","last_change":"2023-06-03T20:07:23.934703+0000","last_active":"2023-06-03T20:11:13.786202+0000","last_peered":"2023-06-03T20:11:13.786202+0000","last_clean":"2023-06-03T20:11:13.786202+0000","last_became_active":"2023-06-03T20:07:23.934266+0000","last_became_peered":"2023-06-03T20:07:23.934266+0000","last_unstale":"2023-06-03T20:11:13.786202+0000","last_undegraded":"2023-06-03T20:11:13.786202+0000","last_fullsized":"2023-06-03T20:11:13.786202+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":145,"reported_epoch":191,"state":"active+clean","last_fresh":"2023-06-03T20:11:13.084752+0000","last_change":"2023-06-03T20:07:19.906895+0000","last_active":"2023-06-03T20:11:13.084752+0000","last_peered":"2023-06-03T20:11:13.084752+0000","last_clean":"2023-06-03T20:11:13.084752+0000","last_became_active":"2023-06-03T20:07:19.906512+0000","last_became_peered":"2023-06-03T20:07:19.906512+0000","last_unstale":"2023-06-03T20:11:13.084752+0000","last_undegraded":"2023-06-03T20:11:13.084752+0000","last_fullsized":"2023-06-03T20:11:13.084752+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T05:12:17.526160+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,6],"acting":[3,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":154,"reported_epoch":191,"state":"active+clean","last_fresh":"2023-06-03T20:11:12.792081+0000","last_change":"2023-06-03T19:56:54.507653+0000","last_active":"2023-06-03T20:11:12.792081+0000","last_peered":"2023-06-03T20:11:12.792081+0000","last_clean":"2023-06-03T20:11:12.792081+0000","last_became_active":"2023-06-03T19:56:54.507350+0000","last_became_peered":"2023-06-03T19:56:54.507350+0000","last_unstale":"2023-06-03T20:11:12.792081+0000","last_undegraded":"2023-06-03T20:11:12.792081+0000","last_fullsized":"2023-06-03T20:11:12.792081+0000","mapping_epoch":119,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T22:33:20.065844+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":186,"reported_epoch":192,"state":"active+clean","last_fresh":"2023-06-03T20:11:17.983943+0000","last_change":"2023-06-03T20:07:19.912004+0000","last_active":"2023-06-03T20:11:17.983943+0000","last_peered":"2023-06-03T20:11:17.983943+0000","last_clean":"2023-06-03T20:11:17.983943+0000","last_became_active":"2023-06-03T20:07:19.910895+0000","last_became_peered":"2023-06-03T20:07:19.910895+0000","last_unstale":"2023-06-03T20:11:17.983943+0000","last_undegraded":"2023-06-03T20:11:17.983943+0000","last_fullsized":"2023-06-03T20:11:17.983943+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T06:46:44.564002+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,2],"acting":[3,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":167,"reported_epoch":191,"state":"active+clean","last_fresh":"2023-06-03T20:11:13.572943+0000","last_change":"2023-06-03T20:07:19.918531+0000","last_active":"2023-06-03T20:11:13.572943+0000","last_peered":"2023-06-03T20:11:13.572943+0000","last_clean":"2023-06-03T20:11:13.572943+0000","last_became_active":"2023-06-03T20:07:19.918173+0000","last_became_peered":"2023-06-03T20:07:19.918173+0000","last_unstale":"2023-06-03T20:11:13.572943+0000","last_undegraded":"2023-06-03T20:11:13.572943+0000","last_fullsized":"2023-06-03T20:11:13.572943+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,0],"acting":[6,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":151,"reported_epoch":191,"state":"active+clean","last_fresh":"2023-06-03T20:11:15.788644+0000","last_change":"2023-06-03T19:56:54.509490+0000","last_active":"2023-06-03T20:11:15.788644+0000","last_peered":"2023-06-03T20:11:15.788644+0000","last_clean":"2023-06-03T20:11:15.788644+0000","last_became_active":"2023-06-03T19:56:54.509145+0000","last_became_peered":"2023-06-03T19:56:54.509145+0000","last_unstale":"2023-06-03T20:11:15.788644+0000","last_undegraded":"2023-06-03T20:11:15.788644+0000","last_fullsized":"2023-06-03T20:11:15.788644+0000","mapping_epoch":119,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T07:12:03.964110+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":170,"reported_epoch":191,"state":"active+clean","last_fresh":"2023-06-03T20:11:13.084907+0000","last_change":"2023-06-03T20:07:19.907524+0000","last_active":"2023-06-03T20:11:13.084907+0000","last_peered":"2023-06-03T20:11:13.084907+0000","last_clean":"2023-06-03T20:11:13.084907+0000","last_became_active":"2023-06-03T20:07:19.907098+0000","last_became_peered":"2023-06-03T20:07:19.907098+0000","last_unstale":"2023-06-03T20:11:13.084907+0000","last_undegraded":"2023-06-03T20:11:13.084907+0000","last_fullsized":"2023-06-03T20:11:13.084907+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T05:30:24.161646+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,4],"acting":[3,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":168,"reported_epoch":191,"state":"active+clean","last_fresh":"2023-06-03T20:11:13.786306+0000","last_change":"2023-06-03T20:07:19.912362+0000","last_active":"2023-06-03T20:11:13.786306+0000","last_peered":"2023-06-03T20:11:13.786306+0000","last_clean":"2023-06-03T20:11:13.786306+0000","last_became_active":"2023-06-03T20:07:19.907385+0000","last_became_peered":"2023-06-03T20:07:19.907385+0000","last_unstale":"2023-06-03T20:11:13.786306+0000","last_undegraded":"2023-06-03T20:11:13.786306+0000","last_fullsized":"2023-06-03T20:11:13.786306+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":142,"reported_epoch":191,"state":"active+clean","last_fresh":"2023-06-03T20:11:13.084497+0000","last_change":"2023-06-03T19:56:54.510462+0000","last_active":"2023-06-03T20:11:13.084497+0000","last_peered":"2023-06-03T20:11:13.084497+0000","last_clean":"2023-06-03T20:11:13.084497+0000","last_became_active":"2023-06-03T19:56:54.509783+0000","last_became_peered":"2023-06-03T19:56:54.509783+0000","last_unstale":"2023-06-03T20:11:13.084497+0000","last_undegraded":"2023-06-03T20:11:13.084497+0000","last_fullsized":"2023-06-03T20:11:13.084497+0000","mapping_epoch":119,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T00:16:09.624276+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"60'3","reported_seq":174,"reported_epoch":191,"state":"active+clean","last_fresh":"2023-06-03T20:11:14.331548+0000","last_change":"2023-06-03T20:07:47.854292+0000","last_active":"2023-06-03T20:11:14.331548+0000","last_peered":"2023-06-03T20:11:14.331548+0000","last_clean":"2023-06-03T20:11:14.331548+0000","last_became_active":"2023-06-03T20:07:47.853941+0000","last_became_peered":"2023-06-03T20:07:47.853941+0000","last_unstale":"2023-06-03T20:11:14.331548+0000","last_undegraded":"2023-06-03T20:11:14.331548+0000","last_fullsized":"2023-06-03T20:11:14.331548+0000","mapping_epoch":172,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":173,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":77,"reported_epoch":191,"state":"active+clean","last_fresh":"2023-06-03T20:11:13.573188+0000","last_change":"2023-06-03T20:07:19.913393+0000","last_active":"2023-06-03T20:11:13.573188+0000","last_peered":"2023-06-03T20:11:13.573188+0000","last_clean":"2023-06-03T20:11:13.573188+0000","last_became_active":"2023-06-03T20:07:19.913074+0000","last_became_peered":"2023-06-03T20:07:19.913074+0000","last_unstale":"2023-06-03T20:11:13.573188+0000","last_undegraded":"2023-06-03T20:11:13.573188+0000","last_fullsized":"2023-06-03T20:11:13.573188+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,0],"acting":[6,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.14","version":"77'23","reported_seq":178,"reported_epoch":191,"state":"active+clean","last_fresh":"2023-06-03T20:11:12.787548+0000","last_change":"2023-06-03T19:57:28.462522+0000","last_active":"2023-06-03T20:11:12.787548+0000","last_peered":"2023-06-03T20:11:12.787548+0000","last_clean":"2023-06-03T20:11:12.787548+0000","last_became_active":"2023-06-03T19:57:28.462048+0000","last_became_peered":"2023-06-03T19:57:28.462048+0000","last_unstale":"2023-06-03T20:11:12.787548+0000","last_undegraded":"2023-06-03T20:11:12.787548+0000","last_fullsized":"2023-06-03T20:11:12.787548+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":23,"log_dups_size":0,"ondisk_log_size":23,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T23:23:56.697534+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4716,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":23,"num_write_kb":20,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":164,"reported_epoch":191,"state":"active+clean","last_fresh":"2023-06-03T20:11:12.787068+0000","last_change":"2023-06-03T20:07:19.902581+0000","last_active":"2023-06-03T20:11:12.787068+0000","last_peered":"2023-06-03T20:11:12.787068+0000","last_clean":"2023-06-03T20:11:12.787068+0000","last_became_active":"2023-06-03T20:07:19.902153+0000","last_became_peered":"2023-06-03T20:07:19.902153+0000","last_unstale":"2023-06-03T20:11:12.787068+0000","last_undegraded":"2023-06-03T20:11:12.787068+0000","last_fullsized":"2023-06-03T20:11:12.787068+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T02:58:27.079772+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,5],"acting":[4,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"60'2","reported_seq":176,"reported_epoch":191,"state":"active+clean","last_fresh":"2023-06-03T20:11:14.331385+0000","last_change":"2023-06-03T19:57:28.467711+0000","last_active":"2023-06-03T20:11:14.331385+0000","last_peered":"2023-06-03T20:11:14.331385+0000","last_clean":"2023-06-03T20:11:14.331385+0000","last_became_active":"2023-06-03T19:57:28.467234+0000","last_became_peered":"2023-06-03T19:57:28.467234+0000","last_unstale":"2023-06-03T20:11:14.331385+0000","last_undegraded":"2023-06-03T20:11:14.331385+0000","last_fullsized":"2023-06-03T20:11:14.331385+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":170,"reported_epoch":191,"state":"active+clean","last_fresh":"2023-06-03T20:11:13.084550+0000","last_change":"2023-06-03T20:07:19.907827+0000","last_active":"2023-06-03T20:11:13.084550+0000","last_peered":"2023-06-03T20:11:13.084550+0000","last_clean":"2023-06-03T20:11:13.084550+0000","last_became_active":"2023-06-03T20:07:19.907296+0000","last_became_peered":"2023-06-03T20:07:19.907296+0000","last_unstale":"2023-06-03T20:11:13.084550+0000","last_undegraded":"2023-06-03T20:11:13.084550+0000","last_fullsized":"2023-06-03T20:11:13.084550+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T00:34:13.230147+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":151,"reported_epoch":191,"state":"active+clean","last_fresh":"2023-06-03T20:11:12.791507+0000","last_change":"2023-06-03T19:56:54.507479+0000","last_active":"2023-06-03T20:11:12.791507+0000","last_peered":"2023-06-03T20:11:12.791507+0000","last_clean":"2023-06-03T20:11:12.791507+0000","last_became_active":"2023-06-03T19:56:54.507149+0000","last_became_peered":"2023-06-03T19:56:54.507149+0000","last_unstale":"2023-06-03T20:11:12.791507+0000","last_undegraded":"2023-06-03T20:11:12.791507+0000","last_fullsized":"2023-06-03T20:11:12.791507+0000","mapping_epoch":119,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T20:22:36.676317+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":158,"reported_epoch":191,"state":"active+clean","last_fresh":"2023-06-03T20:11:13.786437+0000","last_change":"2023-06-03T19:56:54.512472+0000","last_active":"2023-06-03T20:11:13.786437+0000","last_peered":"2023-06-03T20:11:13.786437+0000","last_clean":"2023-06-03T20:11:13.786437+0000","last_became_active":"2023-06-03T19:56:54.512051+0000","last_became_peered":"2023-06-03T19:56:54.512051+0000","last_unstale":"2023-06-03T20:11:13.786437+0000","last_undegraded":"2023-06-03T20:11:13.786437+0000","last_fullsized":"2023-06-03T20:11:13.786437+0000","mapping_epoch":119,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":156,"reported_epoch":191,"state":"active+clean","last_fresh":"2023-06-03T20:11:14.330810+0000","last_change":"2023-06-03T20:07:47.855912+0000","last_active":"2023-06-03T20:11:14.330810+0000","last_peered":"2023-06-03T20:11:14.330810+0000","last_clean":"2023-06-03T20:11:14.330810+0000","last_became_active":"2023-06-03T20:07:47.855608+0000","last_became_peered":"2023-06-03T20:07:47.855608+0000","last_unstale":"2023-06-03T20:11:14.330810+0000","last_undegraded":"2023-06-03T20:11:14.330810+0000","last_fullsized":"2023-06-03T20:11:14.330810+0000","mapping_epoch":172,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":173,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":152,"reported_epoch":191,"state":"active+clean","last_fresh":"2023-06-03T20:11:15.788605+0000","last_change":"2023-06-03T19:56:54.510289+0000","last_active":"2023-06-03T20:11:15.788605+0000","last_peered":"2023-06-03T20:11:15.788605+0000","last_clean":"2023-06-03T20:11:15.788605+0000","last_became_active":"2023-06-03T19:56:54.509905+0000","last_became_peered":"2023-06-03T19:56:54.509905+0000","last_unstale":"2023-06-03T20:11:15.788605+0000","last_undegraded":"2023-06-03T20:11:15.788605+0000","last_fullsized":"2023-06-03T20:11:15.788605+0000","mapping_epoch":119,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T02:54:26.792401+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"60'8","reported_seq":167,"reported_epoch":191,"state":"active+clean","last_fresh":"2023-06-03T20:11:13.572061+0000","last_change":"2023-06-03T19:47:06.727778+0000","last_active":"2023-06-03T20:11:13.572061+0000","last_peered":"2023-06-03T20:11:13.572061+0000","last_clean":"2023-06-03T20:11:13.572061+0000","last_became_active":"2023-06-03T19:47:06.727305+0000","last_became_peered":"2023-06-03T19:47:06.727305+0000","last_unstale":"2023-06-03T20:11:13.572061+0000","last_undegraded":"2023-06-03T20:11:13.572061+0000","last_fullsized":"2023-06-03T20:11:13.572061+0000","mapping_epoch":86,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":87,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":170,"reported_epoch":192,"state":"active+clean","last_fresh":"2023-06-03T20:11:17.985111+0000","last_change":"2023-06-03T20:07:19.907151+0000","last_active":"2023-06-03T20:11:17.985111+0000","last_peered":"2023-06-03T20:11:17.985111+0000","last_clean":"2023-06-03T20:11:17.985111+0000","last_became_active":"2023-06-03T20:07:19.906479+0000","last_became_peered":"2023-06-03T20:07:19.906479+0000","last_unstale":"2023-06-03T20:11:17.985111+0000","last_undegraded":"2023-06-03T20:11:17.985111+0000","last_fullsized":"2023-06-03T20:11:17.985111+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T03:18:22.525692+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"11.7","version":"0'0","reported_seq":11,"reported_epoch":191,"state":"active+clean","last_fresh":"2023-06-03T20:11:13.786338+0000","last_change":"2023-06-03T20:11:11.792506+0000","last_active":"2023-06-03T20:11:13.786338+0000","last_peered":"2023-06-03T20:11:13.786338+0000","last_clean":"2023-06-03T20:11:13.786338+0000","last_became_active":"2023-06-03T20:11:11.792115+0000","last_became_peered":"2023-06-03T20:11:11.792115+0000","last_unstale":"2023-06-03T20:11:13.786338+0000","last_undegraded":"2023-06-03T20:11:13.786338+0000","last_fullsized":"2023-06-03T20:11:13.786338+0000","mapping_epoch":189,"log_start":"0'0","ondisk_log_start":"0'0","created":189,"last_epoch_clean":190,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T20:11:10.759955+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T20:11:10.759955+0000","last_clean_scrub_stamp":"2023-06-03T20:11:10.759955+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,0],"acting":[5,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"60'1","reported_seq":175,"reported_epoch":191,"state":"active+clean","last_fresh":"2023-06-03T20:11:14.331849+0000","last_change":"2023-06-03T19:57:28.464482+0000","last_active":"2023-06-03T20:11:14.331849+0000","last_peered":"2023-06-03T20:11:14.331849+0000","last_clean":"2023-06-03T20:11:14.331849+0000","last_became_active":"2023-06-03T19:57:28.464081+0000","last_became_peered":"2023-06-03T19:57:28.464081+0000","last_unstale":"2023-06-03T20:11:14.331849+0000","last_undegraded":"2023-06-03T20:11:14.331849+0000","last_fullsized":"2023-06-03T20:11:14.331849+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":143,"reported_epoch":191,"state":"active+clean","last_fresh":"2023-06-03T20:11:13.785651+0000","last_change":"2023-06-03T19:34:17.848358+0000","last_active":"2023-06-03T20:11:13.785651+0000","last_peered":"2023-06-03T20:11:13.785651+0000","last_clean":"2023-06-03T20:11:13.785651+0000","last_became_active":"2023-06-03T19:34:17.847811+0000","last_became_peered":"2023-06-03T19:34:17.847811+0000","last_unstale":"2023-06-03T20:11:13.785651+0000","last_undegraded":"2023-06-03T20:11:13.785651+0000","last_fullsized":"2023-06-03T20:11:13.785651+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"11.6","version":"0'0","reported_seq":12,"reported_epoch":191,"state":"active+clean","last_fresh":"2023-06-03T20:11:12.792064+0000","last_change":"2023-06-03T20:11:11.790320+0000","last_active":"2023-06-03T20:11:12.792064+0000","last_peered":"2023-06-03T20:11:12.792064+0000","last_clean":"2023-06-03T20:11:12.792064+0000","last_became_active":"2023-06-03T20:11:11.789918+0000","last_became_peered":"2023-06-03T20:11:11.789918+0000","last_unstale":"2023-06-03T20:11:12.792064+0000","last_undegraded":"2023-06-03T20:11:12.792064+0000","last_fullsized":"2023-06-03T20:11:12.792064+0000","mapping_epoch":189,"log_start":"0'0","ondisk_log_start":"0'0","created":189,"last_epoch_clean":190,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T20:11:10.759955+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T20:11:10.759955+0000","last_clean_scrub_stamp":"2023-06-03T20:11:10.759955+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T02:41:09.861533+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,4],"acting":[0,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.e","version":"60'2","reported_seq":189,"reported_epoch":191,"state":"active+clean","last_fresh":"2023-06-03T20:11:14.331805+0000","last_change":"2023-06-03T20:07:19.965764+0000","last_active":"2023-06-03T20:11:14.331805+0000","last_peered":"2023-06-03T20:11:14.331805+0000","last_clean":"2023-06-03T20:11:14.331805+0000","last_became_active":"2023-06-03T20:07:19.910260+0000","last_became_peered":"2023-06-03T20:07:19.910260+0000","last_unstale":"2023-06-03T20:11:14.331805+0000","last_undegraded":"2023-06-03T20:11:14.331805+0000","last_fullsized":"2023-06-03T20:11:14.331805+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":203,"reported_epoch":191,"state":"active+clean","last_fresh":"2023-06-03T20:11:12.786974+0000","last_change":"2023-06-03T20:07:19.907926+0000","last_active":"2023-06-03T20:11:12.786974+0000","last_peered":"2023-06-03T20:11:12.786974+0000","last_clean":"2023-06-03T20:11:12.786974+0000","last_became_active":"2023-06-03T20:07:19.907441+0000","last_became_peered":"2023-06-03T20:07:19.907441+0000","last_unstale":"2023-06-03T20:11:12.786974+0000","last_undegraded":"2023-06-03T20:11:12.786974+0000","last_fullsized":"2023-06-03T20:11:12.786974+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T06:45:37.611197+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.5","version":"0'0","reported_seq":11,"reported_epoch":191,"state":"active+clean","last_fresh":"2023-06-03T20:11:14.331290+0000","last_change":"2023-06-03T20:11:11.796552+0000","last_active":"2023-06-03T20:11:14.331290+0000","last_peered":"2023-06-03T20:11:14.331290+0000","last_clean":"2023-06-03T20:11:14.331290+0000","last_became_active":"2023-06-03T20:11:11.796144+0000","last_became_peered":"2023-06-03T20:11:11.796144+0000","last_unstale":"2023-06-03T20:11:14.331290+0000","last_undegraded":"2023-06-03T20:11:14.331290+0000","last_fullsized":"2023-06-03T20:11:14.331290+0000","mapping_epoch":189,"log_start":"0'0","ondisk_log_start":"0'0","created":189,"last_epoch_clean":190,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T20:11:10.759955+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T20:11:10.759955+0000","last_clean_scrub_stamp":"2023-06-03T20:11:10.759955+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.d","version":"60'1","reported_seq":142,"reported_epoch":191,"state":"active+clean","last_fresh":"2023-06-03T20:11:14.331468+0000","last_change":"2023-06-03T20:07:47.857710+0000","last_active":"2023-06-03T20:11:14.331468+0000","last_peered":"2023-06-03T20:11:14.331468+0000","last_clean":"2023-06-03T20:11:14.331468+0000","last_became_active":"2023-06-03T20:07:47.857320+0000","last_became_peered":"2023-06-03T20:07:47.857320+0000","last_unstale":"2023-06-03T20:11:14.331468+0000","last_undegraded":"2023-06-03T20:11:14.331468+0000","last_fullsized":"2023-06-03T20:11:14.331468+0000","mapping_epoch":172,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":173,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":167,"reported_epoch":191,"state":"active+clean","last_fresh":"2023-06-03T20:11:13.572740+0000","last_change":"2023-06-03T20:07:19.907984+0000","last_active":"2023-06-03T20:11:13.572740+0000","last_peered":"2023-06-03T20:11:13.572740+0000","last_clean":"2023-06-03T20:11:13.572740+0000","last_became_active":"2023-06-03T20:07:19.907176+0000","last_became_peered":"2023-06-03T20:07:19.907176+0000","last_unstale":"2023-06-03T20:11:13.572740+0000","last_undegraded":"2023-06-03T20:11:13.572740+0000","last_fullsized":"2023-06-03T20:11:13.572740+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,5],"acting":[6,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"11.4","version":"0'0","reported_seq":11,"reported_epoch":191,"state":"active+clean","last_fresh":"2023-06-03T20:11:14.331781+0000","last_change":"2023-06-03T20:11:11.796227+0000","last_active":"2023-06-03T20:11:14.331781+0000","last_peered":"2023-06-03T20:11:14.331781+0000","last_clean":"2023-06-03T20:11:14.331781+0000","last_became_active":"2023-06-03T20:11:11.795900+0000","last_became_peered":"2023-06-03T20:11:11.795900+0000","last_unstale":"2023-06-03T20:11:14.331781+0000","last_undegraded":"2023-06-03T20:11:14.331781+0000","last_fullsized":"2023-06-03T20:11:14.331781+0000","mapping_epoch":189,"log_start":"0'0","ondisk_log_start":"0'0","created":189,"last_epoch_clean":190,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T20:11:10.759955+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T20:11:10.759955+0000","last_clean_scrub_stamp":"2023-06-03T20:11:10.759955+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,2],"acting":[7,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":159,"reported_epoch":191,"state":"active+clean","last_fresh":"2023-06-03T20:11:13.786205+0000","last_change":"2023-06-03T19:56:54.507432+0000","last_active":"2023-06-03T20:11:13.786205+0000","last_peered":"2023-06-03T20:11:13.786205+0000","last_clean":"2023-06-03T20:11:13.786205+0000","last_became_active":"2023-06-03T19:56:54.507012+0000","last_became_peered":"2023-06-03T19:56:54.507012+0000","last_unstale":"2023-06-03T20:11:13.786205+0000","last_undegraded":"2023-06-03T20:11:13.786205+0000","last_fullsized":"2023-06-03T20:11:13.786205+0000","mapping_epoch":119,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":188,"reported_epoch":191,"state":"active+clean","last_fresh":"2023-06-03T20:11:12.791895+0000","last_change":"2023-06-03T20:07:19.914491+0000","last_active":"2023-06-03T20:11:12.791895+0000","last_peered":"2023-06-03T20:11:12.791895+0000","last_clean":"2023-06-03T20:11:12.791895+0000","last_became_active":"2023-06-03T20:07:19.913953+0000","last_became_peered":"2023-06-03T20:07:19.913953+0000","last_unstale":"2023-06-03T20:11:12.791895+0000","last_undegraded":"2023-06-03T20:11:12.791895+0000","last_fullsized":"2023-06-03T20:11:12.791895+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T00:21:40.046785+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"11.3","version":"0'0","reported_seq":11,"reported_epoch":191,"state":"active+clean","last_fresh":"2023-06-03T20:11:13.572612+0000","last_change":"2023-06-03T20:11:11.794495+0000","last_active":"2023-06-03T20:11:13.572612+0000","last_peered":"2023-06-03T20:11:13.572612+0000","last_clean":"2023-06-03T20:11:13.572612+0000","last_became_active":"2023-06-03T20:11:11.794124+0000","last_became_peered":"2023-06-03T20:11:11.794124+0000","last_unstale":"2023-06-03T20:11:13.572612+0000","last_undegraded":"2023-06-03T20:11:13.572612+0000","last_fullsized":"2023-06-03T20:11:13.572612+0000","mapping_epoch":189,"log_start":"0'0","ondisk_log_start":"0'0","created":189,"last_epoch_clean":190,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T20:11:10.759955+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T20:11:10.759955+0000","last_clean_scrub_stamp":"2023-06-03T20:11:10.759955+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,5],"acting":[6,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.b","version":"60'2","reported_seq":162,"reported_epoch":191,"state":"active+clean","last_fresh":"2023-06-03T20:11:13.085347+0000","last_change":"2023-06-03T19:57:28.470310+0000","last_active":"2023-06-03T20:11:13.085347+0000","last_peered":"2023-06-03T20:11:13.085347+0000","last_clean":"2023-06-03T20:11:13.085347+0000","last_became_active":"2023-06-03T19:57:28.469887+0000","last_became_peered":"2023-06-03T19:57:28.469887+0000","last_unstale":"2023-06-03T20:11:13.085347+0000","last_undegraded":"2023-06-03T20:11:13.085347+0000","last_fullsized":"2023-06-03T20:11:13.085347+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T21:58:53.387131+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":170,"reported_epoch":191,"state":"active+clean","last_fresh":"2023-06-03T20:11:12.787601+0000","last_change":"2023-06-03T19:57:28.465140+0000","last_active":"2023-06-03T20:11:12.787601+0000","last_peered":"2023-06-03T20:11:12.787601+0000","last_clean":"2023-06-03T20:11:12.787601+0000","last_became_active":"2023-06-03T19:57:28.464629+0000","last_became_peered":"2023-06-03T19:57:28.464629+0000","last_unstale":"2023-06-03T20:11:12.787601+0000","last_undegraded":"2023-06-03T20:11:12.787601+0000","last_fullsized":"2023-06-03T20:11:12.787601+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T21:47:11.619583+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.2","version":"0'0","reported_seq":12,"reported_epoch":191,"state":"active+clean","last_fresh":"2023-06-03T20:11:14.734267+0000","last_change":"2023-06-03T20:11:11.788495+0000","last_active":"2023-06-03T20:11:14.734267+0000","last_peered":"2023-06-03T20:11:14.734267+0000","last_clean":"2023-06-03T20:11:14.734267+0000","last_became_active":"2023-06-03T20:11:11.788122+0000","last_became_peered":"2023-06-03T20:11:11.788122+0000","last_unstale":"2023-06-03T20:11:14.734267+0000","last_undegraded":"2023-06-03T20:11:14.734267+0000","last_fullsized":"2023-06-03T20:11:14.734267+0000","mapping_epoch":189,"log_start":"0'0","ondisk_log_start":"0'0","created":189,"last_epoch_clean":190,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T20:11:10.759955+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T20:11:10.759955+0000","last_clean_scrub_stamp":"2023-06-03T20:11:10.759955+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T05:43:08.101794+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":183,"reported_epoch":191,"state":"active+clean","last_fresh":"2023-06-03T20:11:13.572534+0000","last_change":"2023-06-03T20:07:19.911766+0000","last_active":"2023-06-03T20:11:13.572534+0000","last_peered":"2023-06-03T20:11:13.572534+0000","last_clean":"2023-06-03T20:11:13.572534+0000","last_became_active":"2023-06-03T20:07:19.911353+0000","last_became_peered":"2023-06-03T20:07:19.911353+0000","last_unstale":"2023-06-03T20:11:13.572534+0000","last_undegraded":"2023-06-03T20:11:13.572534+0000","last_fullsized":"2023-06-03T20:11:13.572534+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,3],"acting":[6,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":184,"reported_epoch":191,"state":"active+clean","last_fresh":"2023-06-03T20:11:12.787726+0000","last_change":"2023-06-03T20:07:19.907673+0000","last_active":"2023-06-03T20:11:12.787726+0000","last_peered":"2023-06-03T20:11:12.787726+0000","last_clean":"2023-06-03T20:11:12.787726+0000","last_became_active":"2023-06-03T20:07:19.907186+0000","last_became_peered":"2023-06-03T20:07:19.907186+0000","last_unstale":"2023-06-03T20:11:12.787726+0000","last_undegraded":"2023-06-03T20:11:12.787726+0000","last_fullsized":"2023-06-03T20:11:12.787726+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T00:08:24.015355+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.1","version":"0'0","reported_seq":11,"reported_epoch":191,"state":"active+clean","last_fresh":"2023-06-03T20:11:13.573004+0000","last_change":"2023-06-03T20:11:11.787443+0000","last_active":"2023-06-03T20:11:13.573004+0000","last_peered":"2023-06-03T20:11:13.573004+0000","last_clean":"2023-06-03T20:11:13.573004+0000","last_became_active":"2023-06-03T20:11:11.787117+0000","last_became_peered":"2023-06-03T20:11:11.787117+0000","last_unstale":"2023-06-03T20:11:13.573004+0000","last_undegraded":"2023-06-03T20:11:13.573004+0000","last_fullsized":"2023-06-03T20:11:13.573004+0000","mapping_epoch":189,"log_start":"0'0","ondisk_log_start":"0'0","created":189,"last_epoch_clean":190,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T20:11:10.759955+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T20:11:10.759955+0000","last_clean_scrub_stamp":"2023-06-03T20:11:10.759955+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,0],"acting":[6,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":167,"reported_epoch":191,"state":"active+clean","last_fresh":"2023-06-03T20:11:12.787760+0000","last_change":"2023-06-03T19:57:28.462344+0000","last_active":"2023-06-03T20:11:12.787760+0000","last_peered":"2023-06-03T20:11:12.787760+0000","last_clean":"2023-06-03T20:11:12.787760+0000","last_became_active":"2023-06-03T19:57:28.462001+0000","last_became_peered":"2023-06-03T19:57:28.462001+0000","last_unstale":"2023-06-03T20:11:12.787760+0000","last_undegraded":"2023-06-03T20:11:12.787760+0000","last_fullsized":"2023-06-03T20:11:12.787760+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T00:45:13.789650+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":165,"reported_epoch":191,"state":"active+clean","last_fresh":"2023-06-03T20:11:12.788043+0000","last_change":"2023-06-03T20:07:19.909344+0000","last_active":"2023-06-03T20:11:12.788043+0000","last_peered":"2023-06-03T20:11:12.788043+0000","last_clean":"2023-06-03T20:11:12.788043+0000","last_became_active":"2023-06-03T20:07:19.908845+0000","last_became_peered":"2023-06-03T20:07:19.908845+0000","last_unstale":"2023-06-03T20:11:12.788043+0000","last_undegraded":"2023-06-03T20:11:12.788043+0000","last_fullsized":"2023-06-03T20:11:12.788043+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T02:42:22.071829+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,3],"acting":[4,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.0","version":"0'0","reported_seq":12,"reported_epoch":191,"state":"active+clean","last_fresh":"2023-06-03T20:11:13.368402+0000","last_change":"2023-06-03T20:11:11.793945+0000","last_active":"2023-06-03T20:11:13.368402+0000","last_peered":"2023-06-03T20:11:13.368402+0000","last_clean":"2023-06-03T20:11:13.368402+0000","last_became_active":"2023-06-03T20:11:11.793560+0000","last_became_peered":"2023-06-03T20:11:11.793560+0000","last_unstale":"2023-06-03T20:11:13.368402+0000","last_undegraded":"2023-06-03T20:11:13.368402+0000","last_fullsized":"2023-06-03T20:11:13.368402+0000","mapping_epoch":189,"log_start":"0'0","ondisk_log_start":"0'0","created":189,"last_epoch_clean":190,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T20:11:10.759955+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T20:11:10.759955+0000","last_clean_scrub_stamp":"2023-06-03T20:11:10.759955+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T03:53:55.401325+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":171,"reported_epoch":191,"state":"active+clean","last_fresh":"2023-06-03T20:11:13.084178+0000","last_change":"2023-06-03T20:07:19.909283+0000","last_active":"2023-06-03T20:11:13.084178+0000","last_peered":"2023-06-03T20:11:13.084178+0000","last_clean":"2023-06-03T20:11:13.084178+0000","last_became_active":"2023-06-03T20:07:19.908902+0000","last_became_peered":"2023-06-03T20:07:19.908902+0000","last_unstale":"2023-06-03T20:11:13.084178+0000","last_undegraded":"2023-06-03T20:11:13.084178+0000","last_fullsized":"2023-06-03T20:11:13.084178+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T23:08:01.870466+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,4],"acting":[3,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":104,"reported_epoch":191,"state":"active+clean","last_fresh":"2023-06-03T20:11:13.085347+0000","last_change":"2023-06-03T20:07:19.908660+0000","last_active":"2023-06-03T20:11:13.085347+0000","last_peered":"2023-06-03T20:11:13.085347+0000","last_clean":"2023-06-03T20:11:13.085347+0000","last_became_active":"2023-06-03T20:07:19.908313+0000","last_became_peered":"2023-06-03T20:07:19.908313+0000","last_unstale":"2023-06-03T20:11:13.085347+0000","last_undegraded":"2023-06-03T20:11:13.085347+0000","last_fullsized":"2023-06-03T20:11:13.085347+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T22:08:11.385111+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,7],"acting":[3,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"11.d","version":"0'0","reported_seq":12,"reported_epoch":191,"state":"active+clean","last_fresh":"2023-06-03T20:11:14.734455+0000","last_change":"2023-06-03T20:11:11.790003+0000","last_active":"2023-06-03T20:11:14.734455+0000","last_peered":"2023-06-03T20:11:14.734455+0000","last_clean":"2023-06-03T20:11:14.734455+0000","last_became_active":"2023-06-03T20:11:11.789662+0000","last_became_peered":"2023-06-03T20:11:11.789662+0000","last_unstale":"2023-06-03T20:11:14.734455+0000","last_undegraded":"2023-06-03T20:11:14.734455+0000","last_fullsized":"2023-06-03T20:11:14.734455+0000","mapping_epoch":189,"log_start":"0'0","ondisk_log_start":"0'0","created":189,"last_epoch_clean":190,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T20:11:10.759955+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T20:11:10.759955+0000","last_clean_scrub_stamp":"2023-06-03T20:11:10.759955+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T22:42:13.193745+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,6],"acting":[2,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":100,"reported_epoch":191,"state":"active+clean","last_fresh":"2023-06-03T20:11:12.791779+0000","last_change":"2023-06-03T20:07:19.926958+0000","last_active":"2023-06-03T20:11:12.791779+0000","last_peered":"2023-06-03T20:11:12.791779+0000","last_clean":"2023-06-03T20:11:12.791779+0000","last_became_active":"2023-06-03T20:07:19.926456+0000","last_became_peered":"2023-06-03T20:07:19.926456+0000","last_unstale":"2023-06-03T20:11:12.791779+0000","last_undegraded":"2023-06-03T20:11:12.791779+0000","last_fullsized":"2023-06-03T20:11:12.791779+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:32:20.692043+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:32:20.692043+0000","last_clean_scrub_stamp":"2023-06-03T19:32:20.692043+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T20:47:07.666782+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,2],"acting":[0,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":174,"reported_epoch":191,"state":"active+clean","last_fresh":"2023-06-03T20:11:13.786344+0000","last_change":"2023-06-03T19:56:54.512607+0000","last_active":"2023-06-03T20:11:13.786344+0000","last_peered":"2023-06-03T20:11:13.786344+0000","last_clean":"2023-06-03T20:11:13.786344+0000","last_became_active":"2023-06-03T19:56:54.512282+0000","last_became_peered":"2023-06-03T19:56:54.512282+0000","last_unstale":"2023-06-03T20:11:13.786344+0000","last_undegraded":"2023-06-03T20:11:13.786344+0000","last_fullsized":"2023-06-03T20:11:13.786344+0000","mapping_epoch":119,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":88,"reported_epoch":191,"state":"active+clean","last_fresh":"2023-06-03T20:11:13.785649+0000","last_change":"2023-06-03T20:07:19.912018+0000","last_active":"2023-06-03T20:11:13.785649+0000","last_peered":"2023-06-03T20:11:13.785649+0000","last_clean":"2023-06-03T20:11:13.785649+0000","last_became_active":"2023-06-03T20:07:19.911682+0000","last_became_peered":"2023-06-03T20:07:19.911682+0000","last_unstale":"2023-06-03T20:11:13.785649+0000","last_undegraded":"2023-06-03T20:11:13.785649+0000","last_fullsized":"2023-06-03T20:11:13.785649+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,0],"acting":[5,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"11.b","version":"0'0","reported_seq":12,"reported_epoch":191,"state":"active+clean","last_fresh":"2023-06-03T20:11:13.368624+0000","last_change":"2023-06-03T20:11:11.790129+0000","last_active":"2023-06-03T20:11:13.368624+0000","last_peered":"2023-06-03T20:11:13.368624+0000","last_clean":"2023-06-03T20:11:13.368624+0000","last_became_active":"2023-06-03T20:11:11.789729+0000","last_became_peered":"2023-06-03T20:11:11.789729+0000","last_unstale":"2023-06-03T20:11:13.368624+0000","last_undegraded":"2023-06-03T20:11:13.368624+0000","last_fullsized":"2023-06-03T20:11:13.368624+0000","mapping_epoch":189,"log_start":"0'0","ondisk_log_start":"0'0","created":189,"last_epoch_clean":190,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T20:11:10.759955+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T20:11:10.759955+0000","last_clean_scrub_stamp":"2023-06-03T20:11:10.759955+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T04:59:39.200434+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":172,"reported_epoch":191,"state":"active+clean","last_fresh":"2023-06-03T20:11:13.785764+0000","last_change":"2023-06-03T20:07:19.906785+0000","last_active":"2023-06-03T20:11:13.785764+0000","last_peered":"2023-06-03T20:11:13.785764+0000","last_clean":"2023-06-03T20:11:13.785764+0000","last_became_active":"2023-06-03T20:07:19.906412+0000","last_became_peered":"2023-06-03T20:07:19.906412+0000","last_unstale":"2023-06-03T20:11:13.785764+0000","last_undegraded":"2023-06-03T20:11:13.785764+0000","last_fullsized":"2023-06-03T20:11:13.785764+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:32:20.692043+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:32:20.692043+0000","last_clean_scrub_stamp":"2023-06-03T19:32:20.692043+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,0],"acting":[5,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":140,"reported_epoch":191,"state":"active+clean","last_fresh":"2023-06-03T20:11:12.787930+0000","last_change":"2023-06-03T19:57:28.460195+0000","last_active":"2023-06-03T20:11:12.787930+0000","last_peered":"2023-06-03T20:11:12.787930+0000","last_clean":"2023-06-03T20:11:12.787930+0000","last_became_active":"2023-06-03T19:57:28.459823+0000","last_became_peered":"2023-06-03T19:57:28.459823+0000","last_unstale":"2023-06-03T20:11:12.787930+0000","last_undegraded":"2023-06-03T20:11:12.787930+0000","last_fullsized":"2023-06-03T20:11:12.787930+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T20:08:48.686380+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":99,"reported_epoch":191,"state":"active+clean","last_fresh":"2023-06-03T20:11:12.790540+0000","last_change":"2023-06-03T20:07:19.900602+0000","last_active":"2023-06-03T20:11:12.790540+0000","last_peered":"2023-06-03T20:11:12.790540+0000","last_clean":"2023-06-03T20:11:12.790540+0000","last_became_active":"2023-06-03T20:07:19.900046+0000","last_became_peered":"2023-06-03T20:07:19.900046+0000","last_unstale":"2023-06-03T20:11:12.790540+0000","last_undegraded":"2023-06-03T20:11:12.790540+0000","last_fullsized":"2023-06-03T20:11:12.790540+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T00:52:57.353642+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,6],"acting":[2,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"11.8","version":"0'0","reported_seq":12,"reported_epoch":191,"state":"active+clean","last_fresh":"2023-06-03T20:11:13.368458+0000","last_change":"2023-06-03T20:11:11.793649+0000","last_active":"2023-06-03T20:11:13.368458+0000","last_peered":"2023-06-03T20:11:13.368458+0000","last_clean":"2023-06-03T20:11:13.368458+0000","last_became_active":"2023-06-03T20:11:11.793292+0000","last_became_peered":"2023-06-03T20:11:11.793292+0000","last_unstale":"2023-06-03T20:11:13.368458+0000","last_undegraded":"2023-06-03T20:11:13.368458+0000","last_fullsized":"2023-06-03T20:11:13.368458+0000","mapping_epoch":189,"log_start":"0'0","ondisk_log_start":"0'0","created":189,"last_epoch_clean":190,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T20:11:10.759955+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T20:11:10.759955+0000","last_clean_scrub_stamp":"2023-06-03T20:11:10.759955+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T07:27:16.604437+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,5],"acting":[3,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":170,"reported_epoch":191,"state":"active+clean","last_fresh":"2023-06-03T20:11:12.790580+0000","last_change":"2023-06-03T19:56:54.504859+0000","last_active":"2023-06-03T20:11:12.790580+0000","last_peered":"2023-06-03T20:11:12.790580+0000","last_clean":"2023-06-03T20:11:12.790580+0000","last_became_active":"2023-06-03T19:56:54.504455+0000","last_became_peered":"2023-06-03T19:56:54.504455+0000","last_unstale":"2023-06-03T20:11:12.790580+0000","last_undegraded":"2023-06-03T20:11:12.790580+0000","last_fullsized":"2023-06-03T20:11:12.790580+0000","mapping_epoch":119,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:32:20.692043+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:32:20.692043+0000","last_clean_scrub_stamp":"2023-06-03T19:32:20.692043+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T06:25:45.522100+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":120,"reported_epoch":191,"state":"active+clean","last_fresh":"2023-06-03T20:11:12.790217+0000","last_change":"2023-06-03T20:07:19.906703+0000","last_active":"2023-06-03T20:11:12.790217+0000","last_peered":"2023-06-03T20:11:12.790217+0000","last_clean":"2023-06-03T20:11:12.790217+0000","last_became_active":"2023-06-03T20:07:19.906373+0000","last_became_peered":"2023-06-03T20:07:19.906373+0000","last_unstale":"2023-06-03T20:11:12.790217+0000","last_undegraded":"2023-06-03T20:11:12.790217+0000","last_fullsized":"2023-06-03T20:11:12.790217+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T21:06:05.945328+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6,0],"acting":[2,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":94,"reported_epoch":191,"state":"active+clean","last_fresh":"2023-06-03T20:11:13.786524+0000","last_change":"2023-06-03T20:07:19.906757+0000","last_active":"2023-06-03T20:11:13.786524+0000","last_peered":"2023-06-03T20:11:13.786524+0000","last_clean":"2023-06-03T20:11:13.786524+0000","last_became_active":"2023-06-03T20:07:19.906380+0000","last_became_peered":"2023-06-03T20:07:19.906380+0000","last_unstale":"2023-06-03T20:11:13.786524+0000","last_undegraded":"2023-06-03T20:11:13.786524+0000","last_fullsized":"2023-06-03T20:11:13.786524+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,7],"acting":[5,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"11.a","version":"0'0","reported_seq":12,"reported_epoch":191,"state":"active+clean","last_fresh":"2023-06-03T20:11:13.368579+0000","last_change":"2023-06-03T20:11:11.793791+0000","last_active":"2023-06-03T20:11:13.368579+0000","last_peered":"2023-06-03T20:11:13.368579+0000","last_clean":"2023-06-03T20:11:13.368579+0000","last_became_active":"2023-06-03T20:11:11.793127+0000","last_became_peered":"2023-06-03T20:11:11.793127+0000","last_unstale":"2023-06-03T20:11:13.368579+0000","last_undegraded":"2023-06-03T20:11:13.368579+0000","last_fullsized":"2023-06-03T20:11:13.368579+0000","mapping_epoch":189,"log_start":"0'0","ondisk_log_start":"0'0","created":189,"last_epoch_clean":190,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T20:11:10.759955+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T20:11:10.759955+0000","last_clean_scrub_stamp":"2023-06-03T20:11:10.759955+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T22:21:19.854335+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,5],"acting":[3,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":163,"reported_epoch":191,"state":"active+clean","last_fresh":"2023-06-03T20:11:13.785927+0000","last_change":"2023-06-03T19:56:11.752151+0000","last_active":"2023-06-03T20:11:13.785927+0000","last_peered":"2023-06-03T20:11:13.785927+0000","last_clean":"2023-06-03T20:11:13.785927+0000","last_became_active":"2023-06-03T19:56:11.751767+0000","last_became_peered":"2023-06-03T19:56:11.751767+0000","last_unstale":"2023-06-03T20:11:13.785927+0000","last_undegraded":"2023-06-03T20:11:13.785927+0000","last_fullsized":"2023-06-03T20:11:13.785927+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":115,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:32:20.692043+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:32:20.692043+0000","last_clean_scrub_stamp":"2023-06-03T19:32:20.692043+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"72'139","reported_seq":392,"reported_epoch":191,"state":"active+clean","last_fresh":"2023-06-03T20:11:14.331142+0000","last_change":"2023-06-03T20:07:47.872942+0000","last_active":"2023-06-03T20:11:14.331142+0000","last_peered":"2023-06-03T20:11:14.331142+0000","last_clean":"2023-06-03T20:11:14.331142+0000","last_became_active":"2023-06-03T20:07:47.872426+0000","last_became_peered":"2023-06-03T20:07:47.872426+0000","last_unstale":"2023-06-03T20:11:14.331142+0000","last_undegraded":"2023-06-03T20:11:14.331142+0000","last_fullsized":"2023-06-03T20:11:14.331142+0000","mapping_epoch":172,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":173,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:27:28.100808+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:27:28.100808+0000","last_clean_scrub_stamp":"2023-06-03T19:27:28.100808+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":142,"reported_epoch":191,"state":"active+clean","last_fresh":"2023-06-03T20:11:13.084545+0000","last_change":"2023-06-03T19:56:54.509571+0000","last_active":"2023-06-03T20:11:13.084545+0000","last_peered":"2023-06-03T20:11:13.084545+0000","last_clean":"2023-06-03T20:11:13.084545+0000","last_became_active":"2023-06-03T19:56:54.509213+0000","last_became_peered":"2023-06-03T19:56:54.509213+0000","last_unstale":"2023-06-03T20:11:13.084545+0000","last_undegraded":"2023-06-03T20:11:13.084545+0000","last_fullsized":"2023-06-03T20:11:13.084545+0000","mapping_epoch":119,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T21:26:56.003203+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":158,"reported_epoch":191,"state":"active+clean","last_fresh":"2023-06-03T20:11:13.572983+0000","last_change":"2023-06-03T19:57:28.459664+0000","last_active":"2023-06-03T20:11:13.572983+0000","last_peered":"2023-06-03T20:11:13.572983+0000","last_clean":"2023-06-03T20:11:13.572983+0000","last_became_active":"2023-06-03T19:57:28.459289+0000","last_became_peered":"2023-06-03T19:57:28.459289+0000","last_unstale":"2023-06-03T20:11:13.572983+0000","last_undegraded":"2023-06-03T20:11:13.572983+0000","last_fullsized":"2023-06-03T20:11:13.572983+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"11.9","version":"0'0","reported_seq":12,"reported_epoch":191,"state":"active+clean","last_fresh":"2023-06-03T20:11:13.368549+0000","last_change":"2023-06-03T20:11:11.792411+0000","last_active":"2023-06-03T20:11:13.368549+0000","last_peered":"2023-06-03T20:11:13.368549+0000","last_clean":"2023-06-03T20:11:13.368549+0000","last_became_active":"2023-06-03T20:11:11.792109+0000","last_became_peered":"2023-06-03T20:11:11.792109+0000","last_unstale":"2023-06-03T20:11:13.368549+0000","last_undegraded":"2023-06-03T20:11:13.368549+0000","last_fullsized":"2023-06-03T20:11:13.368549+0000","mapping_epoch":189,"log_start":"0'0","ondisk_log_start":"0'0","created":189,"last_epoch_clean":190,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T20:11:10.759955+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T20:11:10.759955+0000","last_clean_scrub_stamp":"2023-06-03T20:11:10.759955+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T02:10:39.470012+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,0],"acting":[3,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":175,"reported_epoch":191,"state":"active+clean","last_fresh":"2023-06-03T20:11:14.330812+0000","last_change":"2023-06-03T20:07:47.855767+0000","last_active":"2023-06-03T20:11:14.330812+0000","last_peered":"2023-06-03T20:11:14.330812+0000","last_clean":"2023-06-03T20:11:14.330812+0000","last_became_active":"2023-06-03T20:07:47.855444+0000","last_became_peered":"2023-06-03T20:07:47.855444+0000","last_unstale":"2023-06-03T20:11:14.330812+0000","last_undegraded":"2023-06-03T20:11:14.330812+0000","last_fullsized":"2023-06-03T20:11:14.330812+0000","mapping_epoch":172,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":173,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:32:20.692043+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:32:20.692043+0000","last_clean_scrub_stamp":"2023-06-03T19:32:20.692043+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,3],"acting":[7,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":167,"reported_epoch":191,"state":"active+clean","last_fresh":"2023-06-03T20:11:12.789368+0000","last_change":"2023-06-03T19:57:28.459611+0000","last_active":"2023-06-03T20:11:12.789368+0000","last_peered":"2023-06-03T20:11:12.789368+0000","last_clean":"2023-06-03T20:11:12.789368+0000","last_became_active":"2023-06-03T19:57:28.459264+0000","last_became_peered":"2023-06-03T19:57:28.459264+0000","last_unstale":"2023-06-03T20:11:12.789368+0000","last_undegraded":"2023-06-03T20:11:12.789368+0000","last_fullsized":"2023-06-03T20:11:12.789368+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T22:15:06.557231+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":177,"reported_epoch":191,"state":"active+clean","last_fresh":"2023-06-03T20:11:12.792177+0000","last_change":"2023-06-03T20:07:19.925864+0000","last_active":"2023-06-03T20:11:12.792177+0000","last_peered":"2023-06-03T20:11:12.792177+0000","last_clean":"2023-06-03T20:11:12.792177+0000","last_became_active":"2023-06-03T20:07:19.925541+0000","last_became_peered":"2023-06-03T20:07:19.925541+0000","last_unstale":"2023-06-03T20:11:12.792177+0000","last_undegraded":"2023-06-03T20:11:12.792177+0000","last_fullsized":"2023-06-03T20:11:12.792177+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T00:10:11.790831+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,7],"acting":[0,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"11.c","version":"0'0","reported_seq":12,"reported_epoch":191,"state":"active+clean","last_fresh":"2023-06-03T20:11:14.734375+0000","last_change":"2023-06-03T20:11:11.792633+0000","last_active":"2023-06-03T20:11:14.734375+0000","last_peered":"2023-06-03T20:11:14.734375+0000","last_clean":"2023-06-03T20:11:14.734375+0000","last_became_active":"2023-06-03T20:11:11.792266+0000","last_became_peered":"2023-06-03T20:11:11.792266+0000","last_unstale":"2023-06-03T20:11:14.734375+0000","last_undegraded":"2023-06-03T20:11:14.734375+0000","last_fullsized":"2023-06-03T20:11:14.734375+0000","mapping_epoch":189,"log_start":"0'0","ondisk_log_start":"0'0","created":189,"last_epoch_clean":190,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T20:11:10.759955+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T20:11:10.759955+0000","last_clean_scrub_stamp":"2023-06-03T20:11:10.759955+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T07:35:53.770713+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,7],"acting":[2,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":178,"reported_epoch":191,"state":"active+clean","last_fresh":"2023-06-03T20:11:14.331400+0000","last_change":"2023-06-03T19:57:28.466096+0000","last_active":"2023-06-03T20:11:14.331400+0000","last_peered":"2023-06-03T20:11:14.331400+0000","last_clean":"2023-06-03T20:11:14.331400+0000","last_became_active":"2023-06-03T19:57:28.464954+0000","last_became_peered":"2023-06-03T19:57:28.464954+0000","last_unstale":"2023-06-03T20:11:14.331400+0000","last_undegraded":"2023-06-03T20:11:14.331400+0000","last_fullsized":"2023-06-03T20:11:14.331400+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:32:20.692043+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:32:20.692043+0000","last_clean_scrub_stamp":"2023-06-03T19:32:20.692043+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":120,"reported_epoch":191,"state":"active+clean","last_fresh":"2023-06-03T20:11:12.790672+0000","last_change":"2023-06-03T20:07:19.915065+0000","last_active":"2023-06-03T20:11:12.790672+0000","last_peered":"2023-06-03T20:11:12.790672+0000","last_clean":"2023-06-03T20:11:12.790672+0000","last_became_active":"2023-06-03T20:07:19.914699+0000","last_became_peered":"2023-06-03T20:07:19.914699+0000","last_unstale":"2023-06-03T20:11:12.790672+0000","last_undegraded":"2023-06-03T20:11:12.790672+0000","last_fullsized":"2023-06-03T20:11:12.790672+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T07:16:09.077184+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,0],"acting":[2,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":138,"reported_epoch":191,"state":"active+clean","last_fresh":"2023-06-03T20:11:12.791718+0000","last_change":"2023-06-03T19:47:06.733532+0000","last_active":"2023-06-03T20:11:12.791718+0000","last_peered":"2023-06-03T20:11:12.791718+0000","last_clean":"2023-06-03T20:11:12.791718+0000","last_became_active":"2023-06-03T19:47:06.733131+0000","last_became_peered":"2023-06-03T19:47:06.733131+0000","last_unstale":"2023-06-03T20:11:12.791718+0000","last_undegraded":"2023-06-03T20:11:12.791718+0000","last_fullsized":"2023-06-03T20:11:12.791718+0000","mapping_epoch":86,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":87,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T23:46:03.788646+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"11.f","version":"0'0","reported_seq":11,"reported_epoch":191,"state":"active+clean","last_fresh":"2023-06-03T20:11:14.331915+0000","last_change":"2023-06-03T20:11:11.794557+0000","last_active":"2023-06-03T20:11:14.331915+0000","last_peered":"2023-06-03T20:11:14.331915+0000","last_clean":"2023-06-03T20:11:14.331915+0000","last_became_active":"2023-06-03T20:11:11.794234+0000","last_became_peered":"2023-06-03T20:11:11.794234+0000","last_unstale":"2023-06-03T20:11:14.331915+0000","last_undegraded":"2023-06-03T20:11:14.331915+0000","last_fullsized":"2023-06-03T20:11:14.331915+0000","mapping_epoch":189,"log_start":"0'0","ondisk_log_start":"0'0","created":189,"last_epoch_clean":190,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T20:11:10.759955+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T20:11:10.759955+0000","last_clean_scrub_stamp":"2023-06-03T20:11:10.759955+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,0],"acting":[7,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.7","version":"60'2","reported_seq":147,"reported_epoch":191,"state":"active+clean","last_fresh":"2023-06-03T20:11:13.085486+0000","last_change":"2023-06-03T19:56:54.510148+0000","last_active":"2023-06-03T20:11:13.085486+0000","last_peered":"2023-06-03T20:11:13.085486+0000","last_clean":"2023-06-03T20:11:13.085486+0000","last_became_active":"2023-06-03T19:56:54.509771+0000","last_became_peered":"2023-06-03T19:56:54.509771+0000","last_unstale":"2023-06-03T20:11:13.085486+0000","last_undegraded":"2023-06-03T20:11:13.085486+0000","last_fullsized":"2023-06-03T20:11:13.085486+0000","mapping_epoch":119,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T00:52:42.807432+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":129,"reported_epoch":191,"state":"active+clean","last_fresh":"2023-06-03T20:11:13.573005+0000","last_change":"2023-06-03T20:07:19.911591+0000","last_active":"2023-06-03T20:11:13.573005+0000","last_peered":"2023-06-03T20:11:13.573005+0000","last_clean":"2023-06-03T20:11:13.573005+0000","last_became_active":"2023-06-03T20:07:19.911172+0000","last_became_peered":"2023-06-03T20:07:19.911172+0000","last_unstale":"2023-06-03T20:11:13.573005+0000","last_undegraded":"2023-06-03T20:11:13.573005+0000","last_fullsized":"2023-06-03T20:11:13.573005+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:32:20.692043+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:32:20.692043+0000","last_clean_scrub_stamp":"2023-06-03T19:32:20.692043+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,3],"acting":[6,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":145,"reported_epoch":191,"state":"active+clean","last_fresh":"2023-06-03T20:11:13.084157+0000","last_change":"2023-06-03T19:56:54.507670+0000","last_active":"2023-06-03T20:11:13.084157+0000","last_peered":"2023-06-03T20:11:13.084157+0000","last_clean":"2023-06-03T20:11:13.084157+0000","last_became_active":"2023-06-03T19:56:54.507293+0000","last_became_peered":"2023-06-03T19:56:54.507293+0000","last_unstale":"2023-06-03T20:11:13.084157+0000","last_undegraded":"2023-06-03T20:11:13.084157+0000","last_fullsized":"2023-06-03T20:11:13.084157+0000","mapping_epoch":119,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T01:49:19.966054+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"11.e","version":"0'0","reported_seq":12,"reported_epoch":191,"state":"active+clean","last_fresh":"2023-06-03T20:11:12.788152+0000","last_change":"2023-06-03T20:11:11.790808+0000","last_active":"2023-06-03T20:11:12.788152+0000","last_peered":"2023-06-03T20:11:12.788152+0000","last_clean":"2023-06-03T20:11:12.788152+0000","last_became_active":"2023-06-03T20:11:11.790533+0000","last_became_peered":"2023-06-03T20:11:11.790533+0000","last_unstale":"2023-06-03T20:11:12.788152+0000","last_undegraded":"2023-06-03T20:11:12.788152+0000","last_fullsized":"2023-06-03T20:11:12.788152+0000","mapping_epoch":189,"log_start":"0'0","ondisk_log_start":"0'0","created":189,"last_epoch_clean":190,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T20:11:10.759955+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T20:11:10.759955+0000","last_clean_scrub_stamp":"2023-06-03T20:11:10.759955+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T20:55:44.304940+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":163,"reported_epoch":191,"state":"active+clean","last_fresh":"2023-06-03T20:11:13.572921+0000","last_change":"2023-06-03T19:56:11.752730+0000","last_active":"2023-06-03T20:11:13.572921+0000","last_peered":"2023-06-03T20:11:13.572921+0000","last_clean":"2023-06-03T20:11:13.572921+0000","last_became_active":"2023-06-03T19:56:11.752441+0000","last_became_peered":"2023-06-03T19:56:11.752441+0000","last_unstale":"2023-06-03T20:11:13.572921+0000","last_undegraded":"2023-06-03T20:11:13.572921+0000","last_fullsized":"2023-06-03T20:11:13.572921+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":115,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:32:20.692043+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:32:20.692043+0000","last_clean_scrub_stamp":"2023-06-03T19:32:20.692043+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"60'1","reported_seq":184,"reported_epoch":191,"state":"active+clean","last_fresh":"2023-06-03T20:11:12.791960+0000","last_change":"2023-06-03T20:07:19.965503+0000","last_active":"2023-06-03T20:11:12.791960+0000","last_peered":"2023-06-03T20:11:12.791960+0000","last_clean":"2023-06-03T20:11:12.791960+0000","last_became_active":"2023-06-03T20:07:19.913879+0000","last_became_peered":"2023-06-03T20:07:19.913879+0000","last_unstale":"2023-06-03T20:11:12.791960+0000","last_undegraded":"2023-06-03T20:11:12.791960+0000","last_fullsized":"2023-06-03T20:11:12.791960+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T20:37:07.402876+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":138,"reported_epoch":191,"state":"active+clean","last_fresh":"2023-06-03T20:11:12.787129+0000","last_change":"2023-06-03T19:57:28.456467+0000","last_active":"2023-06-03T20:11:12.787129+0000","last_peered":"2023-06-03T20:11:12.787129+0000","last_clean":"2023-06-03T20:11:12.787129+0000","last_became_active":"2023-06-03T19:57:28.456120+0000","last_became_peered":"2023-06-03T19:57:28.456120+0000","last_unstale":"2023-06-03T20:11:12.787129+0000","last_undegraded":"2023-06-03T20:11:12.787129+0000","last_fullsized":"2023-06-03T20:11:12.787129+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T07:28:14.306624+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"60'1","reported_seq":154,"reported_epoch":191,"state":"active+clean","last_fresh":"2023-06-03T20:11:12.792025+0000","last_change":"2023-06-03T19:57:28.465124+0000","last_active":"2023-06-03T20:11:12.792025+0000","last_peered":"2023-06-03T20:11:12.792025+0000","last_clean":"2023-06-03T20:11:12.792025+0000","last_became_active":"2023-06-03T19:57:28.464654+0000","last_became_peered":"2023-06-03T19:57:28.464654+0000","last_unstale":"2023-06-03T20:11:12.792025+0000","last_undegraded":"2023-06-03T20:11:12.792025+0000","last_fullsized":"2023-06-03T20:11:12.792025+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T01:12:32.305507+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":190,"reported_epoch":191,"state":"active+clean","last_fresh":"2023-06-03T20:11:12.790768+0000","last_change":"2023-06-03T20:07:19.914985+0000","last_active":"2023-06-03T20:11:12.790768+0000","last_peered":"2023-06-03T20:11:12.790768+0000","last_clean":"2023-06-03T20:11:12.790768+0000","last_became_active":"2023-06-03T20:07:19.914624+0000","last_became_peered":"2023-06-03T20:07:19.914624+0000","last_unstale":"2023-06-03T20:11:12.790768+0000","last_undegraded":"2023-06-03T20:11:12.790768+0000","last_fullsized":"2023-06-03T20:11:12.790768+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T07:02:50.515170+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"60'2","reported_seq":189,"reported_epoch":191,"state":"active+clean","last_fresh":"2023-06-03T20:11:13.786228+0000","last_change":"2023-06-03T20:07:19.960260+0000","last_active":"2023-06-03T20:11:13.786228+0000","last_peered":"2023-06-03T20:11:13.786228+0000","last_clean":"2023-06-03T20:11:13.786228+0000","last_became_active":"2023-06-03T20:07:19.911480+0000","last_became_peered":"2023-06-03T20:07:19.911480+0000","last_unstale":"2023-06-03T20:11:13.786228+0000","last_undegraded":"2023-06-03T20:11:13.786228+0000","last_fullsized":"2023-06-03T20:11:13.786228+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":193,"reported_epoch":191,"state":"active+clean","last_fresh":"2023-06-03T20:11:12.787619+0000","last_change":"2023-06-03T20:07:19.902344+0000","last_active":"2023-06-03T20:11:12.787619+0000","last_peered":"2023-06-03T20:11:12.787619+0000","last_clean":"2023-06-03T20:11:12.787619+0000","last_became_active":"2023-06-03T20:07:19.901903+0000","last_became_peered":"2023-06-03T20:07:19.901903+0000","last_unstale":"2023-06-03T20:11:12.787619+0000","last_undegraded":"2023-06-03T20:11:12.787619+0000","last_fullsized":"2023-06-03T20:11:12.787619+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T23:17:54.574533+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[4,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"60'2","reported_seq":161,"reported_epoch":191,"state":"active+clean","last_fresh":"2023-06-03T20:11:13.786277+0000","last_change":"2023-06-03T19:57:28.462610+0000","last_active":"2023-06-03T20:11:13.786277+0000","last_peered":"2023-06-03T20:11:13.786277+0000","last_clean":"2023-06-03T20:11:13.786277+0000","last_became_active":"2023-06-03T19:57:28.460655+0000","last_became_peered":"2023-06-03T19:57:28.460655+0000","last_unstale":"2023-06-03T20:11:13.786277+0000","last_undegraded":"2023-06-03T20:11:13.786277+0000","last_fullsized":"2023-06-03T20:11:13.786277+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":184,"reported_epoch":191,"state":"active+clean","last_fresh":"2023-06-03T20:11:12.787654+0000","last_change":"2023-06-03T19:57:28.464690+0000","last_active":"2023-06-03T20:11:12.787654+0000","last_peered":"2023-06-03T20:11:12.787654+0000","last_clean":"2023-06-03T20:11:12.787654+0000","last_became_active":"2023-06-03T19:57:28.463856+0000","last_became_peered":"2023-06-03T19:57:28.463856+0000","last_unstale":"2023-06-03T20:11:12.787654+0000","last_undegraded":"2023-06-03T20:11:12.787654+0000","last_fullsized":"2023-06-03T20:11:12.787654+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T00:11:00.991088+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"60'1","reported_seq":174,"reported_epoch":191,"state":"active+clean","last_fresh":"2023-06-03T20:11:13.085210+0000","last_change":"2023-06-03T19:56:54.505096+0000","last_active":"2023-06-03T20:11:13.085210+0000","last_peered":"2023-06-03T20:11:13.085210+0000","last_clean":"2023-06-03T20:11:13.085210+0000","last_became_active":"2023-06-03T19:56:54.504255+0000","last_became_peered":"2023-06-03T19:56:54.504255+0000","last_unstale":"2023-06-03T20:11:13.085210+0000","last_undegraded":"2023-06-03T20:11:13.085210+0000","last_fullsized":"2023-06-03T20:11:13.085210+0000","mapping_epoch":119,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T19:52:25.922308+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":186,"reported_epoch":192,"state":"active+clean","last_fresh":"2023-06-03T20:11:17.984264+0000","last_change":"2023-06-03T19:56:54.503416+0000","last_active":"2023-06-03T20:11:17.984264+0000","last_peered":"2023-06-03T20:11:17.984264+0000","last_clean":"2023-06-03T20:11:17.984264+0000","last_became_active":"2023-06-03T19:56:54.503018+0000","last_became_peered":"2023-06-03T19:56:54.503018+0000","last_unstale":"2023-06-03T20:11:17.984264+0000","last_undegraded":"2023-06-03T20:11:17.984264+0000","last_fullsized":"2023-06-03T20:11:17.984264+0000","mapping_epoch":119,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T20:14:00.330616+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"60'4","reported_seq":156,"reported_epoch":191,"state":"active+clean","last_fresh":"2023-06-03T20:11:12.792275+0000","last_change":"2023-06-03T19:56:11.748116+0000","last_active":"2023-06-03T20:11:12.792275+0000","last_peered":"2023-06-03T20:11:12.792275+0000","last_clean":"2023-06-03T20:11:12.792275+0000","last_became_active":"2023-06-03T19:56:11.747636+0000","last_became_peered":"2023-06-03T19:56:11.747636+0000","last_unstale":"2023-06-03T20:11:12.792275+0000","last_undegraded":"2023-06-03T20:11:12.792275+0000","last_fullsized":"2023-06-03T20:11:12.792275+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":115,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T06:57:48.052055+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":173,"reported_epoch":191,"state":"active+clean","last_fresh":"2023-06-03T20:11:12.787015+0000","last_change":"2023-06-03T20:07:23.956460+0000","last_active":"2023-06-03T20:11:12.787015+0000","last_peered":"2023-06-03T20:11:12.787015+0000","last_clean":"2023-06-03T20:11:12.787015+0000","last_became_active":"2023-06-03T20:07:23.956110+0000","last_became_peered":"2023-06-03T20:07:23.956110+0000","last_unstale":"2023-06-03T20:11:12.787015+0000","last_undegraded":"2023-06-03T20:11:12.787015+0000","last_fullsized":"2023-06-03T20:11:12.787015+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T19:35:07.921742+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":11,"num_pg":16,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":48,"acting":48,"num_store_stats":7},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5486,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":62,"num_write_kb":30,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16458,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":30319,"internal_metadata":0},"log_size":56,"ondisk_log_size":56,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":3,"num_store_stats":3},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":48,"seq":206158430692,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":499608,"kb_used_data":2320,"kb_used_omap":0,"kb_used_meta":497280,"kb_avail":93245544,"statfs":{"total":95995035648,"available":95483437056,"internally_reserved":0,"allocated":2375680,"data_stored":1869009,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":509214720},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 3 20:10:54 2023","interfaces":[{"interface":"back","average":{"1min":1.411,"5min":2.5750000000000002,"15min":1.5609999999999999},"min":{"1min":0.42299999999999999,"5min":0.26200000000000001,"15min":0.25900000000000001},"max":{"1min":6.5289999999999999,"5min":48.338999999999999,"15min":48.338999999999999},"last":0.88700000000000001},{"interface":"front","average":{"1min":1.371,"5min":1.8939999999999999,"15min":1.494},"min":{"1min":0.33600000000000002,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":4.9630000000000001,"5min":46.412999999999997,"15min":46.412999999999997},"last":1.056}]},{"osd":1,"last update":"Sat Jun 3 20:10:39 2023","interfaces":[{"interface":"back","average":{"1min":1.071,"5min":1.3160000000000001,"15min":1.635},"min":{"1min":0.42599999999999999,"5min":0.30099999999999999,"15min":0.29699999999999999},"max":{"1min":3.798,"5min":11.4,"15min":91.811000000000007},"last":1.159},{"interface":"front","average":{"1min":1.4490000000000001,"5min":1.444,"15min":1.847},"min":{"1min":0.375,"5min":0.34000000000000002,"15min":0.32200000000000001},"max":{"1min":4.9370000000000003,"5min":4.9850000000000003,"15min":91.695999999999998},"last":1.006}]},{"osd":2,"last update":"Sat Jun 3 20:10:39 2023","interfaces":[{"interface":"back","average":{"1min":1.2150000000000001,"5min":2.0059999999999998,"15min":1.383},"min":{"1min":0.372,"5min":0.34799999999999998,"15min":0.32800000000000001},"max":{"1min":3.7709999999999999,"5min":46.331000000000003,"15min":46.331000000000003},"last":1.083},{"interface":"front","average":{"1min":1.605,"5min":2.0350000000000001,"15min":1.5029999999999999},"min":{"1min":0.40500000000000003,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":5.0359999999999996,"5min":48.216000000000001,"15min":48.216000000000001},"last":0.94499999999999995}]},{"osd":3,"last update":"Sat Jun 3 20:10:30 2023","interfaces":[{"interface":"back","average":{"1min":1.3140000000000001,"5min":1.7130000000000001,"15min":1.8340000000000001},"min":{"1min":0.44600000000000001,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":4.141,"5min":11.483000000000001,"15min":95.962000000000003},"last":1.014},{"interface":"front","average":{"1min":2.363,"5min":1.71,"15min":1.982},"min":{"1min":0.34399999999999997,"5min":0.34399999999999997,"15min":0.34399999999999997},"max":{"1min":5.4560000000000004,"5min":8.0310000000000006,"15min":94.174999999999997},"last":1.111}]},{"osd":4,"last update":"Sat Jun 3 20:10:54 2023","interfaces":[{"interface":"back","average":{"1min":1.417,"5min":1.8879999999999999,"15min":1.3260000000000001},"min":{"1min":0.33900000000000002,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":7.1189999999999998,"5min":21.888999999999999,"15min":21.888999999999999},"last":0.79200000000000004},{"interface":"front","average":{"1min":1.484,"5min":2.1739999999999999,"15min":1.671},"min":{"1min":0.60799999999999998,"5min":0.443,"15min":0.38800000000000001},"max":{"1min":4.8600000000000003,"5min":10.507999999999999,"15min":12.917},"last":1.2190000000000001}]},{"osd":5,"last update":"Sat Jun 3 20:10:30 2023","interfaces":[{"interface":"back","average":{"1min":1.6439999999999999,"5min":1.925,"15min":1.3979999999999999},"min":{"1min":0.46500000000000002,"5min":0.36399999999999999,"15min":0.36399999999999999},"max":{"1min":6.2560000000000002,"5min":11.443,"15min":11.443},"last":0.71099999999999997},{"interface":"front","average":{"1min":3.3519999999999999,"5min":2.1200000000000001,"15min":1.7729999999999999},"min":{"1min":0.48599999999999999,"5min":0.35799999999999998,"15min":0.30599999999999999},"max":{"1min":8.6300000000000008,"5min":8.6300000000000008,"15min":20.306999999999999},"last":1.179}]},{"osd":6,"last update":"Sat Jun 3 20:10:30 2023","interfaces":[{"interface":"back","average":{"1min":4.1479999999999997,"5min":2.4239999999999999,"15min":1.7110000000000001},"min":{"1min":0.56799999999999995,"5min":0.41299999999999998,"15min":0.36499999999999999},"max":{"1min":11.256,"5min":11.256,"15min":11.256},"last":1.2569999999999999},{"interface":"front","average":{"1min":3.3450000000000002,"5min":2.5169999999999999,"15min":1.776},"min":{"1min":0.38100000000000001,"5min":0.34899999999999998,"15min":0.32500000000000001},"max":{"1min":8.8650000000000002,"5min":10.894,"15min":10.894},"last":0.83799999999999997}]}]},{"osd":6,"up_from":42,"seq":180388626925,"num_pgs":42,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":427336,"kb_used_data":2304,"kb_used_omap":0,"kb_used_meta":425024,"kb_avail":93317816,"statfs":{"total":95995035648,"available":95557443584,"internally_reserved":0,"allocated":2359296,"data_stored":1868548,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":435224576},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 3 20:11:01 2023","interfaces":[{"interface":"back","average":{"1min":2.754,"5min":3.7519999999999998,"15min":2.157},"min":{"1min":0.39500000000000002,"5min":0.26600000000000001,"15min":0.26400000000000001},"max":{"1min":28.117000000000001,"5min":54.283999999999999,"15min":54.786000000000001},"last":0.78200000000000003},{"interface":"front","average":{"1min":3.2730000000000001,"5min":4.2510000000000003,"15min":2.3290000000000002},"min":{"1min":0.38800000000000001,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":28.138000000000002,"5min":54.314999999999998,"15min":54.936},"last":1.444}]},{"osd":1,"last update":"Sat Jun 3 20:10:38 2023","interfaces":[{"interface":"back","average":{"1min":1.7130000000000001,"5min":1.405,"15min":2.2530000000000001},"min":{"1min":0.39000000000000001,"5min":0.34300000000000003,"15min":0.28000000000000003},"max":{"1min":6.7510000000000003,"5min":6.7510000000000003,"15min":160.38499999999999},"last":1.415},{"interface":"front","average":{"1min":2.4390000000000001,"5min":2.02,"15min":2.2120000000000002},"min":{"1min":0.38200000000000001,"5min":0.32400000000000001,"15min":0.29599999999999999},"max":{"1min":10.077,"5min":12.132999999999999,"15min":160.434},"last":1.621}]},{"osd":2,"last update":"Sat Jun 3 20:10:38 2023","interfaces":[{"interface":"back","average":{"1min":4.2089999999999996,"5min":2.319,"15min":1.351},"min":{"1min":0.32500000000000001,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":42.091999999999999,"5min":42.091999999999999,"15min":42.091999999999999},"last":0.90400000000000003},{"interface":"front","average":{"1min":4.4219999999999997,"5min":2.6440000000000001,"15min":1.5329999999999999},"min":{"1min":0.33700000000000002,"5min":0.33700000000000002,"15min":0.33700000000000002},"max":{"1min":43.448,"5min":43.448,"15min":43.448},"last":1.4870000000000001}]},{"osd":3,"last update":"Sat Jun 3 20:11:17 2023","interfaces":[{"interface":"back","average":{"1min":1.0680000000000001,"5min":1.458,"15min":2.2829999999999999},"min":{"1min":0.49299999999999999,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":3.7080000000000002,"5min":9.5950000000000006,"15min":164.99199999999999},"last":0.81899999999999995},{"interface":"front","average":{"1min":0.94399999999999995,"5min":1.5620000000000001,"15min":2.2949999999999999},"min":{"1min":0.44,"5min":0.38400000000000001,"15min":0.316},"max":{"1min":1.2769999999999999,"5min":6.7770000000000001,"15min":163.459},"last":1.655}]},{"osd":4,"last update":"Sat Jun 3 20:10:57 2023","interfaces":[{"interface":"back","average":{"1min":2.0510000000000002,"5min":2.0350000000000001,"15min":1.3029999999999999},"min":{"1min":0.35999999999999999,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":8.2230000000000008,"5min":14.098000000000001,"15min":14.098000000000001},"last":0.71099999999999997},{"interface":"front","average":{"1min":1.929,"5min":1.738,"15min":1.2569999999999999},"min":{"1min":0.52900000000000003,"5min":0.39600000000000002,"15min":0.377},"max":{"1min":8.452,"5min":8.452,"15min":8.452},"last":1.835}]},{"osd":5,"last update":"Sat Jun 3 20:10:56 2023","interfaces":[{"interface":"back","average":{"1min":1.4850000000000001,"5min":1.736,"15min":1.284},"min":{"1min":0.57999999999999996,"5min":0.36099999999999999,"15min":0.32200000000000001},"max":{"1min":3.302,"5min":14.731999999999999,"15min":14.731999999999999},"last":1.7150000000000001},{"interface":"front","average":{"1min":2.4039999999999999,"5min":2.504,"15min":1.6779999999999999},"min":{"1min":0.41799999999999998,"5min":0.35899999999999999,"15min":0.29499999999999998},"max":{"1min":10.786,"5min":15.295,"15min":15.295},"last":1.7350000000000001}]},{"osd":7,"last update":"Sat Jun 3 20:10:25 2023","interfaces":[{"interface":"back","average":{"1min":2.6459999999999999,"5min":1.9239999999999999,"15min":1.524},"min":{"1min":0.48099999999999998,"5min":0.42799999999999999,"15min":0.313},"max":{"1min":7.5979999999999999,"5min":8.4269999999999996,"15min":11.362},"last":1.881},{"interface":"front","average":{"1min":3.835,"5min":2.2549999999999999,"15min":1.601},"min":{"1min":0.43099999999999999,"5min":0.38900000000000001,"15min":0.33900000000000002},"max":{"1min":21.202000000000002,"5min":21.202000000000002,"15min":21.202000000000002},"last":1.2390000000000001}]}]},{"osd":1,"up_from":0,"seq":442381631723,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":0,"up_from":86,"seq":369367187749,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":389548,"kb_used_data":2596,"kb_used_omap":5,"kb_used_meta":386938,"kb_avail":93355604,"statfs":{"total":95995035648,"available":95596138496,"internally_reserved":0,"allocated":2658304,"data_stored":1901853,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":396225146},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Jun 3 20:10:25 2023","interfaces":[{"interface":"back","average":{"1min":0.68000000000000005,"5min":0.93300000000000005,"15min":1.427},"min":{"1min":0.26000000000000001,"5min":0.249,"15min":0.192},"max":{"1min":2.8559999999999999,"5min":9.3040000000000003,"15min":81.310000000000002},"last":0.41999999999999998},{"interface":"front","average":{"1min":1.2849999999999999,"5min":1.4059999999999999,"15min":1.6020000000000001},"min":{"1min":0.245,"5min":0.22600000000000001,"15min":0.187},"max":{"1min":3.9529999999999998,"5min":11.736000000000001,"15min":76.971999999999994},"last":0.46100000000000002}]},{"osd":2,"last update":"Sat Jun 3 20:10:25 2023","interfaces":[{"interface":"back","average":{"1min":1.0780000000000001,"5min":1.0489999999999999,"15min":1.0089999999999999},"min":{"1min":0.30499999999999999,"5min":0.193,"15min":0.193},"max":{"1min":4.0810000000000004,"5min":11.901,"15min":11.901},"last":0.623},{"interface":"front","average":{"1min":0.93300000000000005,"5min":1.2749999999999999,"15min":1.0900000000000001},"min":{"1min":0.27800000000000002,"5min":0.255,"15min":0.218},"max":{"1min":2.9329999999999998,"5min":17.315999999999999,"15min":17.315999999999999},"last":0.66000000000000003}]},{"osd":3,"last update":"Sat Jun 3 20:11:14 2023","interfaces":[{"interface":"back","average":{"1min":0.68000000000000005,"5min":1.04,"15min":1.107},"min":{"1min":0.36399999999999999,"5min":0.26800000000000002,"15min":0.216},"max":{"1min":1.843,"5min":4.8570000000000002,"15min":7.5990000000000002},"last":0.57999999999999996},{"interface":"front","average":{"1min":0.78600000000000003,"5min":1.157,"15min":1.165},"min":{"1min":0.32000000000000001,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":1.978,"5min":6.3769999999999998,"15min":8.4459999999999997},"last":0.55200000000000005}]},{"osd":4,"last update":"Sat Jun 3 20:10:59 2023","interfaces":[{"interface":"back","average":{"1min":1.9370000000000001,"5min":1.7110000000000001,"15min":1.3400000000000001},"min":{"1min":0.45200000000000001,"5min":0.318,"15min":0.318},"max":{"1min":10.025,"5min":10.025,"15min":10.025},"last":0.75900000000000001},{"interface":"front","average":{"1min":1.673,"5min":1.8160000000000001,"15min":1.429},"min":{"1min":0.56699999999999995,"5min":0.375,"15min":0.375},"max":{"1min":4.6059999999999999,"5min":13.058999999999999,"15min":13.058999999999999},"last":0.95499999999999996}]},{"osd":5,"last update":"Sat Jun 3 20:10:59 2023","interfaces":[{"interface":"back","average":{"1min":1.516,"5min":2.3239999999999998,"15min":1.6439999999999999},"min":{"1min":0.442,"5min":0.42099999999999999,"15min":0.34899999999999998},"max":{"1min":4.633,"5min":8.3249999999999993,"15min":8.3249999999999993},"last":1.117},{"interface":"front","average":{"1min":1.4670000000000001,"5min":1.542,"15min":1.4319999999999999},"min":{"1min":0.51200000000000001,"5min":0.34000000000000002,"15min":0.34000000000000002},"max":{"1min":7.0529999999999999,"5min":7.0529999999999999,"15min":8.2189999999999994},"last":0.92300000000000004}]},{"osd":6,"last update":"Sat Jun 3 20:10:59 2023","interfaces":[{"interface":"back","average":{"1min":1.3460000000000001,"5min":1.5449999999999999,"15min":1.3480000000000001},"min":{"1min":0.439,"5min":0.38,"15min":0.38},"max":{"1min":7.3220000000000001,"5min":10.140000000000001,"15min":10.140000000000001},"last":0.86899999999999999},{"interface":"front","average":{"1min":1.2190000000000001,"5min":1.7949999999999999,"15min":1.4830000000000001},"min":{"1min":0.48199999999999998,"5min":0.432,"15min":0.42799999999999999},"max":{"1min":2.968,"5min":13.117000000000001,"15min":13.117000000000001},"last":1.077}]},{"osd":7,"last update":"Sat Jun 3 20:10:59 2023","interfaces":[{"interface":"back","average":{"1min":1.5029999999999999,"5min":1.546,"15min":1.4990000000000001},"min":{"1min":0.60499999999999998,"5min":0.39800000000000002,"15min":0.35099999999999998},"max":{"1min":5.609,"5min":8.4329999999999998,"15min":11.196},"last":0.90000000000000002},{"interface":"front","average":{"1min":1.5529999999999999,"5min":1.79,"15min":1.6359999999999999},"min":{"1min":0.64700000000000002,"5min":0.47899999999999998,"15min":0.47899999999999998},"max":{"1min":5.6289999999999996,"5min":8.5960000000000001,"15min":8.891},"last":1.0469999999999999}]}]},{"osd":2,"up_from":114,"seq":489626271928,"num_pgs":29,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":120496,"kb_used_data":1960,"kb_used_omap":7,"kb_used_meta":118520,"kb_avail":93624656,"statfs":{"total":95995035648,"available":95871647744,"internally_reserved":0,"allocated":2007040,"data_stored":1305683,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8111,"internal_metadata":121364561},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 3 20:10:34 2023","interfaces":[{"interface":"back","average":{"1min":14.058999999999999,"5min":4.641,"15min":2.5609999999999999},"min":{"1min":0.22700000000000001,"5min":0.222,"15min":0.20899999999999999},"max":{"1min":273.51100000000002,"5min":273.51100000000002,"15min":273.51100000000002},"last":0.628},{"interface":"front","average":{"1min":14.625999999999999,"5min":4.8109999999999999,"15min":2.7149999999999999},"min":{"1min":0.28299999999999997,"5min":0.247,"15min":0.221},"max":{"1min":273.67399999999998,"5min":273.67399999999998,"15min":273.67399999999998},"last":1.1579999999999999}]},{"osd":1,"last update":"Sat Jun 3 20:10:34 2023","interfaces":[{"interface":"back","average":{"1min":14.125999999999999,"5min":4.5220000000000002,"15min":2.2130000000000001},"min":{"1min":0.25800000000000001,"5min":0.254,"15min":0.22900000000000001},"max":{"1min":273.46499999999997,"5min":273.46499999999997,"15min":273.46499999999997},"last":1.0109999999999999},{"interface":"front","average":{"1min":14.214,"5min":4.7510000000000003,"15min":2.2959999999999998},"min":{"1min":0.26300000000000001,"5min":0.246,"15min":0.23699999999999999},"max":{"1min":274.38400000000001,"5min":274.38400000000001,"15min":274.38400000000001},"last":0.79200000000000004}]},{"osd":3,"last update":"Sat Jun 3 20:10:20 2023","interfaces":[{"interface":"back","average":{"1min":1.1399999999999999,"5min":1.4770000000000001,"15min":1.1419999999999999},"min":{"1min":0.33400000000000002,"5min":0.28799999999999998,"15min":0.22700000000000001},"max":{"1min":5.4720000000000004,"5min":24.739999999999998,"15min":29.251999999999999},"last":0.83299999999999996},{"interface":"front","average":{"1min":1.764,"5min":1.377,"15min":1.498},"min":{"1min":0.30199999999999999,"5min":0.24199999999999999,"15min":0.24199999999999999},"max":{"1min":7.8369999999999997,"5min":10.503,"15min":54.549999999999997},"last":0.877}]},{"osd":4,"last update":"Sat Jun 3 20:10:59 2023","interfaces":[{"interface":"back","average":{"1min":16.218,"5min":4.9749999999999996,"15min":2.2519999999999998},"min":{"1min":0.48699999999999999,"5min":0.32800000000000001,"15min":0.32700000000000001},"max":{"1min":273.75700000000001,"5min":273.75700000000001,"15min":273.75700000000001},"last":1.2190000000000001},{"interface":"front","average":{"1min":15.512,"5min":4.7480000000000002,"15min":2.1850000000000001},"min":{"1min":0.36199999999999999,"5min":0.35699999999999998,"15min":0.32600000000000001},"max":{"1min":274.27300000000002,"5min":274.27300000000002,"15min":274.27300000000002},"last":1.0840000000000001}]},{"osd":5,"last update":"Sat Jun 3 20:10:34 2023","interfaces":[{"interface":"back","average":{"1min":14.409000000000001,"5min":4.0970000000000004,"15min":2.032},"min":{"1min":0.442,"5min":0.36099999999999999,"15min":0.36099999999999999},"max":{"1min":273.78800000000001,"5min":273.78800000000001,"15min":273.78800000000001},"last":1.236},{"interface":"front","average":{"1min":15.069000000000001,"5min":4.2430000000000003,"15min":2.286},"min":{"1min":0.377,"5min":0.377,"15min":0.308},"max":{"1min":274.15800000000002,"5min":274.15800000000002,"15min":274.15800000000002},"last":1.335}]},{"osd":6,"last update":"Sat Jun 3 20:10:34 2023","interfaces":[{"interface":"back","average":{"1min":15.287000000000001,"5min":4.5650000000000004,"15min":2.2090000000000001},"min":{"1min":0.44700000000000001,"5min":0.376,"15min":0.376},"max":{"1min":274.03899999999999,"5min":274.03899999999999,"15min":274.03899999999999},"last":1.1220000000000001},{"interface":"front","average":{"1min":14.928000000000001,"5min":4.7350000000000003,"15min":2.266},"min":{"1min":0.42699999999999999,"5min":0.39100000000000001,"15min":0.34899999999999998},"max":{"1min":274.06299999999999,"5min":274.06299999999999,"15min":274.06299999999999},"last":1.6499999999999999}]},{"osd":7,"last update":"Sat Jun 3 20:10:34 2023","interfaces":[{"interface":"back","average":{"1min":14.941000000000001,"5min":4.335,"15min":2.4199999999999999},"min":{"1min":0.50600000000000001,"5min":0.39600000000000002,"15min":0.39600000000000002},"max":{"1min":274.54000000000002,"5min":274.54000000000002,"15min":274.54000000000002},"last":1.1899999999999999},{"interface":"front","average":{"1min":15.121,"5min":4.2110000000000003,"15min":2.3580000000000001},"min":{"1min":0.51000000000000001,"5min":0.40200000000000002,"15min":0.35699999999999998},"max":{"1min":274.12700000000001,"5min":274.12700000000001,"15min":274.12700000000001},"last":1.7130000000000001}]}]},{"osd":3,"up_from":119,"seq":511101108400,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":212916,"kb_used_data":1964,"kb_used_omap":2,"kb_used_meta":210941,"kb_avail":93532236,"statfs":{"total":95995035648,"available":95777009664,"internally_reserved":0,"allocated":2011136,"data_stored":1303300,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":216003637},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 3 20:10:22 2023","interfaces":[{"interface":"back","average":{"1min":1.0049999999999999,"5min":2.3460000000000001,"15min":1.675},"min":{"1min":0.32900000000000001,"5min":0.223,"15min":0.19},"max":{"1min":4.1500000000000004,"5min":58.106999999999999,"15min":68.647999999999996},"last":0.92900000000000005},{"interface":"front","average":{"1min":1.9830000000000001,"5min":2.9169999999999998,"15min":1.8620000000000001},"min":{"1min":0.254,"5min":0.23400000000000001,"15min":0.214},"max":{"1min":8.0800000000000001,"5min":62.585000000000001,"15min":69.691000000000003},"last":0.97399999999999998}]},{"osd":1,"last update":"Sat Jun 3 20:10:22 2023","interfaces":[{"interface":"back","average":{"1min":1.5129999999999999,"5min":0.96299999999999997,"15min":1.0449999999999999},"min":{"1min":0.27800000000000002,"5min":0.214,"15min":0.214},"max":{"1min":10.749000000000001,"5min":10.749000000000001,"15min":10.749000000000001},"last":0.76000000000000001},{"interface":"front","average":{"1min":0.752,"5min":0.72099999999999997,"15min":0.96999999999999997},"min":{"1min":0.245,"5min":0.24199999999999999,"15min":0.23499999999999999},"max":{"1min":4.077,"5min":5.4960000000000004,"15min":10.792999999999999},"last":0.80900000000000005}]},{"osd":2,"last update":"Sat Jun 3 20:10:22 2023","interfaces":[{"interface":"back","average":{"1min":1.1879999999999999,"5min":1.7809999999999999,"15min":1.228},"min":{"1min":0.217,"5min":0.217,"15min":0.217},"max":{"1min":5.2640000000000002,"5min":62.527999999999999,"15min":62.527999999999999},"last":0.63800000000000001},{"interface":"front","average":{"1min":1.877,"5min":1.7889999999999999,"15min":1.274},"min":{"1min":0.28499999999999998,"5min":0.214,"15min":0.214},"max":{"1min":10.704000000000001,"5min":57.935000000000002,"15min":57.935000000000002},"last":0.59899999999999998}]},{"osd":4,"last update":"Sat Jun 3 20:10:53 2023","interfaces":[{"interface":"back","average":{"1min":2.0379999999999998,"5min":1.98,"15min":1.413},"min":{"1min":0.42399999999999999,"5min":0.38200000000000001,"15min":0.38200000000000001},"max":{"1min":9.8360000000000003,"5min":12.757,"15min":12.757},"last":1.0469999999999999},{"interface":"front","average":{"1min":2.1749999999999998,"5min":1.744,"15min":1.333},"min":{"1min":0.39000000000000001,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":10.122,"5min":10.122,"15min":10.122},"last":1.0620000000000001}]},{"osd":5,"last update":"Sat Jun 3 20:10:22 2023","interfaces":[{"interface":"back","average":{"1min":1.1859999999999999,"5min":1.135,"15min":1.198},"min":{"1min":0.40600000000000003,"5min":0.36299999999999999,"15min":0.32900000000000001},"max":{"1min":3.8439999999999999,"5min":5.4269999999999996,"15min":7.1230000000000002},"last":0.90300000000000002},{"interface":"front","average":{"1min":1.609,"5min":1.4770000000000001,"15min":1.5029999999999999},"min":{"1min":0.35099999999999998,"5min":0.314,"15min":0.314},"max":{"1min":9.5359999999999996,"5min":12.56,"15min":12.56},"last":1.2350000000000001}]},{"osd":6,"last update":"Sat Jun 3 20:10:22 2023","interfaces":[{"interface":"back","average":{"1min":1.571,"5min":1.901,"15min":1.5249999999999999},"min":{"1min":0.39000000000000001,"5min":0.379,"15min":0.379},"max":{"1min":7.4740000000000002,"5min":11.093999999999999,"15min":11.093999999999999},"last":1.298},{"interface":"front","average":{"1min":2.4710000000000001,"5min":2.1320000000000001,"15min":1.51},"min":{"1min":0.42199999999999999,"5min":0.35599999999999998,"15min":0.35599999999999998},"max":{"1min":10.779999999999999,"5min":11.116,"15min":11.116},"last":1.1279999999999999}]},{"osd":7,"last update":"Sat Jun 3 20:10:22 2023","interfaces":[{"interface":"back","average":{"1min":2.911,"5min":1.587,"15min":1.3919999999999999},"min":{"1min":0.51000000000000001,"5min":0.39100000000000001,"15min":0.39100000000000001},"max":{"1min":10.858000000000001,"5min":10.858000000000001,"15min":10.858000000000001},"last":1.089},{"interface":"front","average":{"1min":2.5310000000000001,"5min":1.6779999999999999,"15min":1.6120000000000001},"min":{"1min":0.45900000000000002,"5min":0.39700000000000002,"15min":0.39700000000000002},"max":{"1min":11.084,"5min":11.084,"15min":12.715999999999999},"last":1.1060000000000001}]}]},{"osd":4,"up_from":124,"seq":532575944873,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":167472,"kb_used_data":2024,"kb_used_omap":10,"kb_used_meta":165429,"kb_avail":93577680,"statfs":{"total":95995035648,"available":95823544320,"internally_reserved":0,"allocated":2072576,"data_stored":1310836,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":10652,"internal_metadata":169399908},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 3 20:11:03 2023","interfaces":[{"interface":"back","average":{"1min":1.002,"5min":4.8399999999999999,"15min":2.468},"min":{"1min":0.58599999999999997,"5min":0.29499999999999998,"15min":0.254},"max":{"1min":3.0139999999999998,"5min":214.72999999999999,"15min":214.72999999999999},"last":0.72799999999999998},{"interface":"front","average":{"1min":2.04,"5min":5.327,"15min":2.6459999999999999},"min":{"1min":0.41399999999999998,"5min":0.40200000000000002,"15min":0.32400000000000001},"max":{"1min":15,"5min":214.76499999999999,"15min":214.76499999999999},"last":0.87}]},{"osd":1,"last update":"Sat Jun 3 20:11:03 2023","interfaces":[{"interface":"back","average":{"1min":1.1439999999999999,"5min":1.639,"15min":1.55},"min":{"1min":0.371,"5min":0.35199999999999998,"15min":0.28100000000000003},"max":{"1min":3.8210000000000002,"5min":10.446,"15min":58.32},"last":0.89000000000000001},{"interface":"front","average":{"1min":0.878,"5min":1.4990000000000001,"15min":1.4610000000000001},"min":{"1min":0.498,"5min":0.34999999999999998,"15min":0.307},"max":{"1min":1.8,"5min":12.919,"15min":58.286000000000001},"last":0.94699999999999995}]},{"osd":2,"last update":"Sat Jun 3 20:11:03 2023","interfaces":[{"interface":"back","average":{"1min":13.406000000000001,"5min":4.6070000000000002,"15min":2.1419999999999999},"min":{"1min":0.55000000000000004,"5min":0.317,"15min":0.30299999999999999},"max":{"1min":248.65799999999999,"5min":248.65799999999999,"15min":248.65799999999999},"last":0.84299999999999997},{"interface":"front","average":{"1min":14.144,"5min":4.7720000000000002,"15min":2.1890000000000001},"min":{"1min":0.46700000000000003,"5min":0.373,"15min":0.34499999999999997},"max":{"1min":249.077,"5min":249.077,"15min":249.077},"last":0.91900000000000004}]},{"osd":3,"last update":"Sat Jun 3 20:11:03 2023","interfaces":[{"interface":"back","average":{"1min":1.2330000000000001,"5min":1.778,"15min":1.6240000000000001},"min":{"1min":0.40600000000000003,"5min":0.35199999999999998,"15min":0.32500000000000001},"max":{"1min":3.7759999999999998,"5min":10.301,"15min":57.954999999999998},"last":1.0329999999999999},{"interface":"front","average":{"1min":1.226,"5min":1.6479999999999999,"15min":1.613},"min":{"1min":0.50700000000000001,"5min":0.40300000000000002,"15min":0.33700000000000002},"max":{"1min":4.9210000000000003,"5min":12.890000000000001,"15min":57.889000000000003},"last":0.99299999999999999}]},{"osd":5,"last update":"Sat Jun 3 20:11:03 2023","interfaces":[{"interface":"back","average":{"1min":1.4590000000000001,"5min":2.0819999999999999,"15min":1.623},"min":{"1min":0.35899999999999999,"5min":0.32000000000000001,"15min":0.313},"max":{"1min":6.6859999999999999,"5min":11.311,"15min":86.576999999999998},"last":1.101},{"interface":"front","average":{"1min":1.5269999999999999,"5min":2.1760000000000002,"15min":1.7390000000000001},"min":{"1min":0.32900000000000001,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":7.9269999999999996,"5min":10.997999999999999,"15min":88.013000000000005},"last":0.80600000000000005}]},{"osd":6,"last update":"Sat Jun 3 20:11:03 2023","interfaces":[{"interface":"back","average":{"1min":1.8140000000000001,"5min":2.7429999999999999,"15min":1.5309999999999999},"min":{"1min":0.36099999999999999,"5min":0.34599999999999997,"15min":0.34599999999999997},"max":{"1min":6.6520000000000001,"5min":14.244999999999999,"15min":14.244999999999999},"last":0.77600000000000002},{"interface":"front","average":{"1min":2.4329999999999998,"5min":2.8679999999999999,"15min":1.629},"min":{"1min":0.42599999999999999,"5min":0.40100000000000002,"15min":0.36499999999999999},"max":{"1min":10.359,"5min":12.145,"15min":12.145},"last":1.1220000000000001}]},{"osd":7,"last update":"Sat Jun 3 20:11:03 2023","interfaces":[{"interface":"back","average":{"1min":1.7270000000000001,"5min":2.052,"15min":1.55},"min":{"1min":0.44,"5min":0.41199999999999998,"15min":0.39700000000000002},"max":{"1min":9.6210000000000004,"5min":10.388,"15min":10.388},"last":0.96999999999999997},{"interface":"front","average":{"1min":1.0629999999999999,"5min":1.7729999999999999,"15min":1.486},"min":{"1min":0.44800000000000001,"5min":0.34200000000000003,"15min":0.31900000000000001},"max":{"1min":1.8260000000000001,"5min":12.951000000000001,"15min":12.951000000000001},"last":1.018}]}]},{"osd":5,"up_from":36,"seq":154618823159,"num_pgs":28,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":559740,"kb_used_data":1716,"kb_used_omap":0,"kb_used_meta":558016,"kb_avail":93185412,"statfs":{"total":95995035648,"available":95421861888,"internally_reserved":0,"allocated":1757184,"data_stored":1274018,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":571408384},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 3 20:10:46 2023","interfaces":[{"interface":"back","average":{"1min":6.1239999999999997,"5min":4.0549999999999997,"15min":2.1429999999999998},"min":{"1min":0.28100000000000003,"5min":0.27000000000000002,"15min":0.22800000000000001},"max":{"1min":92.706999999999994,"5min":185.29900000000001,"15min":185.29900000000001},"last":0.89500000000000002},{"interface":"front","average":{"1min":6.2569999999999997,"5min":4.3380000000000001,"15min":2.504},"min":{"1min":0.307,"5min":0.26400000000000001,"15min":0.26300000000000001},"max":{"1min":92.858999999999995,"5min":185.44200000000001,"15min":185.44200000000001},"last":0.88400000000000001}]},{"osd":1,"last update":"Sat Jun 3 20:10:46 2023","interfaces":[{"interface":"back","average":{"1min":1.6539999999999999,"5min":1.4359999999999999,"15min":2.181},"min":{"1min":0.41199999999999998,"5min":0.32900000000000001,"15min":0.30199999999999999},"max":{"1min":6.5300000000000002,"5min":8.5050000000000008,"15min":114.681},"last":0.82099999999999995},{"interface":"front","average":{"1min":1.2290000000000001,"5min":1.488,"15min":2.2029999999999998},"min":{"1min":0.41099999999999998,"5min":0.307,"15min":0.307},"max":{"1min":6.3239999999999998,"5min":9.2739999999999991,"15min":115.366},"last":0.66700000000000004}]},{"osd":2,"last update":"Sat Jun 3 20:10:46 2023","interfaces":[{"interface":"back","average":{"1min":6.5,"5min":2.798,"15min":1.5700000000000001},"min":{"1min":0.48099999999999998,"5min":0.318,"15min":0.318},"max":{"1min":92.412000000000006,"5min":92.412000000000006,"15min":92.412000000000006},"last":0.623},{"interface":"front","average":{"1min":6.4409999999999998,"5min":3.0230000000000001,"15min":1.8560000000000001},"min":{"1min":0.39300000000000002,"5min":0.32100000000000001,"15min":0.31900000000000001},"max":{"1min":94.552000000000007,"5min":94.552000000000007,"15min":94.552000000000007},"last":0.76000000000000001}]},{"osd":3,"last update":"Sat Jun 3 20:11:23 2023","interfaces":[{"interface":"back","average":{"1min":1.4530000000000001,"5min":1.8100000000000001,"15min":2.7360000000000002},"min":{"1min":0.57599999999999996,"5min":0.317,"15min":0.317},"max":{"1min":6.8440000000000003,"5min":8.4179999999999993,"15min":261.14100000000002},"last":0.871},{"interface":"front","average":{"1min":0.88100000000000001,"5min":1.4199999999999999,"15min":2.4180000000000001},"min":{"1min":0.39500000000000002,"5min":0.35799999999999998,"15min":0.35399999999999998},"max":{"1min":1.2509999999999999,"5min":9.0990000000000002,"15min":261.27100000000002},"last":0.80300000000000005}]},{"osd":4,"last update":"Sat Jun 3 20:11:00 2023","interfaces":[{"interface":"back","average":{"1min":2.2930000000000001,"5min":2.4510000000000001,"15min":1.6839999999999999},"min":{"1min":0.44,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":7.5910000000000002,"5min":14.951000000000001,"15min":14.951000000000001},"last":0.745},{"interface":"front","average":{"1min":1.619,"5min":2.0129999999999999,"15min":1.3799999999999999},"min":{"1min":0.42299999999999999,"5min":0.376,"15min":0.376},"max":{"1min":7.2000000000000002,"5min":12.109,"15min":12.109},"last":0.84999999999999998}]},{"osd":6,"last update":"Sat Jun 3 20:10:46 2023","interfaces":[{"interface":"back","average":{"1min":2.0049999999999999,"5min":2.4980000000000002,"15min":1.7649999999999999},"min":{"1min":0.47499999999999998,"5min":0.35799999999999998,"15min":0.33000000000000002},"max":{"1min":6.8849999999999998,"5min":22.945,"15min":22.945},"last":0.70299999999999996},{"interface":"front","average":{"1min":2.9079999999999999,"5min":2.895,"15min":1.9359999999999999},"min":{"1min":0.56699999999999995,"5min":0.38300000000000001,"15min":0.38300000000000001},"max":{"1min":10.708,"5min":22.428000000000001,"15min":22.428000000000001},"last":0.78700000000000003}]},{"osd":7,"last update":"Sat Jun 3 20:10:28 2023","interfaces":[{"interface":"back","average":{"1min":3.3290000000000002,"5min":2.0659999999999998,"15min":1.9419999999999999},"min":{"1min":0.379,"5min":0.379,"15min":0.379},"max":{"1min":14.223000000000001,"5min":14.223000000000001,"15min":14.223000000000001},"last":0.90800000000000003},{"interface":"front","average":{"1min":2.6669999999999998,"5min":1.7909999999999999,"15min":1.452},"min":{"1min":0.434,"5min":0.38100000000000001,"15min":0.38100000000000001},"max":{"1min":9.0039999999999996,"5min":9.0719999999999992,"15min":9.0719999999999992},"last":0.71599999999999997}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6928,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4862,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":10652,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4786,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4932,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-06-03T20:11:27.535 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph pg dump --format=json 2023-06-03T20:11:27.572 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:11:27.575 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:11:27.576 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:11:27.677 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:11:27.714 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:11:27.717 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:11:27.717 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:11:27.818 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:11:27.833 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:11:27 smithi164 bash[59462]: cluster 1685823086.673275 mgr.y (mgr.34104) 1060 : cluster 0 pgmap v1025: 89 pgs: 89 active+clean; 582 KiB data, 2.3 GiB used, 624 GiB / 626 GiB avail 2023-06-03T20:11:27.852 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:11:27.855 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:11:27.855 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:11:27.885 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/mon.a/config 2023-06-03T20:11:27.956 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:11:27.993 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:11:27.996 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:11:27.997 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:11:27.998 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/mon.a/config 2023-06-03T20:11:28.028 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:11:27 smithi089 bash[75068]: cluster 1685823086.673275 mgr.y (mgr.34104) 1060 : cluster 0 pgmap v1025: 89 pgs: 89 active+clean; 582 KiB data, 2.3 GiB used, 624 GiB / 626 GiB avail 2023-06-03T20:11:28.028 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:11:27 smithi089 bash[78452]: cluster 1685823086.673275 mgr.y (mgr.34104) 1060 : cluster 0 pgmap v1025: 89 pgs: 89 active+clean; 582 KiB data, 2.3 GiB used, 624 GiB / 626 GiB avail 2023-06-03T20:11:28.098 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:11:28.124 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:11:28.126 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:11:28.126 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:11:28.227 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:11:28.269 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:11:28.272 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:11:28.273 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:11:28.373 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:11:28.403 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:11:28.405 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:11:28.406 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:11:28.507 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:11:28.540 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:11:28.542 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:11:28.542 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:11:28.643 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:11:28.675 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:11:28.678 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:11:28.679 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:11:28.780 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:11:28.808 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:11:28.811 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:11:28.812 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:11:28.913 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:11:28.953 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:11:28.955 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:11:28.956 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:11:29.057 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:11:29.099 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:11:29.101 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:11:29.101 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:11:29.202 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:11:29.234 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:11:29.236 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:11:29.237 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:11:29.338 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:11:29.375 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:11:29.377 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:11:29.378 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:11:29.479 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:11:29.516 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:11:29.519 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:11:29.520 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:11:29.621 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:11:29.651 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:11:29.656 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:11:29.656 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:11:29.757 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:11:29.794 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:11:29.798 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:11:29.799 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:11:29.807 INFO:teuthology.orchestra.run.smithi164.stderr:Inferring config /var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/mon.b/config 2023-06-03T20:11:29.900 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:11:29.915 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:11:29 smithi089 bash[75068]: cluster 1685823088.6738548 mgr.y (mgr.34104) 1061 : cluster 0 pgmap v1026: 89 pgs: 89 active+clean; 582 KiB data, 2.3 GiB used, 624 GiB / 626 GiB avail 2023-06-03T20:11:29.915 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:11:29 smithi089 bash[78452]: cluster 1685823088.6738548 mgr.y (mgr.34104) 1061 : cluster 0 pgmap v1026: 89 pgs: 89 active+clean; 582 KiB data, 2.3 GiB used, 624 GiB / 626 GiB avail 2023-06-03T20:11:29.936 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:11:29.937 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:11:29.938 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:11:30.038 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:11:30.060 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:11:30.061 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:11:30.062 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:11:30.083 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:11:29 smithi164 bash[59462]: cluster 1685823088.6738548 mgr.y (mgr.34104) 1061 : cluster 0 pgmap v1026: 89 pgs: 89 active+clean; 582 KiB data, 2.3 GiB used, 624 GiB / 626 GiB avail 2023-06-03T20:11:30.163 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:11:30.197 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:11:30.200 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:11:30.201 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:11:30.301 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:11:30.337 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:11:30.341 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:11:30.342 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:11:30.442 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:11:30.475 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:11:30.478 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:11:30.479 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:11:30.580 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:11:30.616 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:11:30.618 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:11:30.619 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:11:30.720 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:11:30.752 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:11:30.756 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:11:30.756 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:11:30.766 INFO:teuthology.orchestra.run.smithi089.stderr:enabled application 'rados' on pool 'unique_pool_6' 2023-06-03T20:11:30.857 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:11:30.893 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:11:30.895 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:11:30.896 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:11:30.997 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:11:31.009 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:11:30 smithi164 bash[59462]: audit 1685823090.099689 mon.a (mon.0) 997 : audit 1 from='client.? 172.21.15.89:0/2060854800' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_6", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-06-03T20:11:31.027 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:11:30 smithi089 bash[75068]: audit 1685823090.099689 mon.a (mon.0) 997 : audit 1 from='client.? 172.21.15.89:0/2060854800' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_6", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-06-03T20:11:31.028 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:11:30 smithi089 bash[78452]: audit 1685823090.099689 mon.a (mon.0) 997 : audit 1 from='client.? 172.21.15.89:0/2060854800' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_6", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-06-03T20:11:31.029 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:11:31.032 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:11:31.033 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:11:31.134 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:11:31.164 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:11:31.166 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:11:31.167 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:11:31.268 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:11:31.298 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:11:31.300 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:11:31.300 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:11:31.401 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:11:31.428 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:11:31.431 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:11:31.433 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:11:31.535 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:11:31.564 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:11:31.567 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:11:31.567 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:11:31.668 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:11:31.700 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:11:31.702 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:11:31.702 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:11:31.804 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:11:31.812 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:11:31 smithi164 bash[59462]: cluster 1685823090.6747012 mgr.y (mgr.34104) 1062 : cluster 0 pgmap v1027: 89 pgs: 89 active+clean; 582 KiB data, 2.3 GiB used, 624 GiB / 626 GiB avail 2023-06-03T20:11:31.812 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:11:31 smithi164 bash[59462]: cluster 1685823090.7527442 osd.6 (osd.6) 139 : cluster 3 failed to encode map e193 with expected crc 2023-06-03T20:11:31.812 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:11:31 smithi164 bash[59462]: audit 1685823090.7581298 mon.a (mon.0) 998 : audit 1 from='client.? 172.21.15.89:0/2060854800' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_6", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-06-03T20:11:31.812 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:11:31 smithi164 bash[59462]: cluster 1685823090.7582045 mon.a (mon.0) 999 : cluster 0 osdmap e193: 8 total, 8 up, 7 in 2023-06-03T20:11:31.813 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:11:31 smithi164 bash[59462]: cluster 1685823091.3902063 osd.5 (osd.5) 124 : cluster 3 failed to encode map e193 with expected crc 2023-06-03T20:11:31.829 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:11:31.832 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:11:31.832 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:11:31.933 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:11:31.945 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:11:31 smithi089 bash[78452]: cluster 1685823090.6747012 mgr.y (mgr.34104) 1062 : cluster 0 pgmap v1027: 89 pgs: 89 active+clean; 582 KiB data, 2.3 GiB used, 624 GiB / 626 GiB avail 2023-06-03T20:11:31.946 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:11:31 smithi089 bash[78452]: cluster 1685823090.7527442 osd.6 (osd.6) 139 : cluster 3 failed to encode map e193 with expected crc 2023-06-03T20:11:31.946 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:11:31 smithi089 bash[78452]: audit 1685823090.7581298 mon.a (mon.0) 998 : audit 1 from='client.? 172.21.15.89:0/2060854800' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_6", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-06-03T20:11:31.946 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:11:31 smithi089 bash[78452]: cluster 1685823090.7582045 mon.a (mon.0) 999 : cluster 0 osdmap e193: 8 total, 8 up, 7 in 2023-06-03T20:11:31.946 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:11:31 smithi089 bash[78452]: cluster 1685823091.3902063 osd.5 (osd.5) 124 : cluster 3 failed to encode map e193 with expected crc 2023-06-03T20:11:31.947 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:11:31 smithi089 bash[75068]: cluster 1685823090.6747012 mgr.y (mgr.34104) 1062 : cluster 0 pgmap v1027: 89 pgs: 89 active+clean; 582 KiB data, 2.3 GiB used, 624 GiB / 626 GiB avail 2023-06-03T20:11:31.947 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:11:31 smithi089 bash[75068]: cluster 1685823090.7527442 osd.6 (osd.6) 139 : cluster 3 failed to encode map e193 with expected crc 2023-06-03T20:11:31.947 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:11:31 smithi089 bash[75068]: audit 1685823090.7581298 mon.a (mon.0) 998 : audit 1 from='client.? 172.21.15.89:0/2060854800' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_6", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-06-03T20:11:31.947 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:11:31 smithi089 bash[75068]: cluster 1685823090.7582045 mon.a (mon.0) 999 : cluster 0 osdmap e193: 8 total, 8 up, 7 in 2023-06-03T20:11:31.947 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:11:31 smithi089 bash[75068]: cluster 1685823091.3902063 osd.5 (osd.5) 124 : cluster 3 failed to encode map e193 with expected crc 2023-06-03T20:11:31.964 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:11:31.967 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:11:31.968 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:11:32.068 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:11:32.100 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:11:32.103 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:11:32.104 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:11:32.205 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:11:32.240 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:11:32.243 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:11:32.243 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:11:32.344 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:11:32.378 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:11:32.380 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:11:32.381 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:11:32.483 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:11:32.512 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:11:32.515 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:11:32.515 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:11:32.616 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:11:32.651 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:11:32.654 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:11:32.655 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:11:32.756 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:11:32.797 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:11:32.800 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:11:32.801 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:11:32.902 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:11:32.943 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:11:32.945 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:11:32.946 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:11:33.047 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:11:33.080 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:11:33.082 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:11:33.083 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:11:33.184 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:11:33.231 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:11:33.233 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:11:33.233 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:11:33.334 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:11:33.374 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:11:33.377 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:11:33.378 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:11:33.480 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:11:33.505 DEBUG:teuthology.orchestra.run.smithi089:> /bin/sh -c 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage rados --no-log-to-stderr --name client.0 -b 65536 --object-size 65536 -p unique_pool_6 bench 90 write' 2023-06-03T20:11:33.508 INFO:tasks.radosbench:joining radosbench (timing out after 3000s) 2023-06-03T20:11:33.508 INFO:teuthology.orchestra.run:waiting for 3000 2023-06-03T20:11:33.515 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:11:33.518 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:11:33.518 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:11:33.549 INFO:tasks.radosbench.radosbench.0.smithi089.stdout:hints = 1 2023-06-03T20:11:33.549 INFO:tasks.radosbench.radosbench.0.smithi089.stdout:Maintaining 16 concurrent writes of 65536 bytes to objects of size 65536 for up to 90 seconds or 0 objects 2023-06-03T20:11:33.549 INFO:tasks.radosbench.radosbench.0.smithi089.stdout:Object prefix: benchmark_data_smithi089_165412 2023-06-03T20:11:33.550 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-06-03T20:11:33.550 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 0 0 0 0 0 0 - 0 2023-06-03T20:11:33.619 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:11:33.650 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:11:33.653 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:11:33.653 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:11:33.755 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:11:33.763 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:11:33 smithi164 bash[59462]: cluster 1685823092.6756427 mgr.y (mgr.34104) 1063 : cluster 0 pgmap v1029: 89 pgs: 89 active+clean; 582 KiB data, 2.3 GiB used, 624 GiB / 626 GiB avail 2023-06-03T20:11:33.778 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:11:33 smithi089 bash[75068]: cluster 1685823092.6756427 mgr.y (mgr.34104) 1063 : cluster 0 pgmap v1029: 89 pgs: 89 active+clean; 582 KiB data, 2.3 GiB used, 624 GiB / 626 GiB avail 2023-06-03T20:11:33.779 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:11:33 smithi089 bash[78452]: cluster 1685823092.6756427 mgr.y (mgr.34104) 1063 : cluster 0 pgmap v1029: 89 pgs: 89 active+clean; 582 KiB data, 2.3 GiB used, 624 GiB / 626 GiB avail 2023-06-03T20:11:33.789 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:11:33.791 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:11:33.792 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:11:33.893 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:11:33.924 INFO:teuthology.orchestra.run.smithi164.stdout:{ 2023-06-03T20:11:33.925 INFO:teuthology.orchestra.run.smithi164.stdout: "size": 20, 2023-06-03T20:11:33.925 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 600, 2023-06-03T20:11:33.925 INFO:teuthology.orchestra.run.smithi164.stdout: "ops": [ 2023-06-03T20:11:33.925 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:11:33.925 INFO:teuthology.orchestra.run.smithi164.stdout: "description": "osd_op(client.36009.0:37609 10.e 10:7c3604f0:::benchmark_data_smithi089_140876_object37608:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e163)", 2023-06-03T20:11:33.925 INFO:teuthology.orchestra.run.smithi164.stdout: "initiated_at": "2023-06-03T20:06:33.242467+0000", 2023-06-03T20:11:33.925 INFO:teuthology.orchestra.run.smithi164.stdout: "age": 300.66654534700001, 2023-06-03T20:11:33.926 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.15637182899999999, 2023-06-03T20:11:33.926 INFO:teuthology.orchestra.run.smithi164.stdout: "type_data": { 2023-06-03T20:11:33.926 INFO:teuthology.orchestra.run.smithi164.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-03T20:11:33.926 INFO:teuthology.orchestra.run.smithi164.stdout: "client_info": { 2023-06-03T20:11:33.926 INFO:teuthology.orchestra.run.smithi164.stdout: "client": "client.36009", 2023-06-03T20:11:33.926 INFO:teuthology.orchestra.run.smithi164.stdout: "client_addr": "172.21.15.89:0/2550116169", 2023-06-03T20:11:33.926 INFO:teuthology.orchestra.run.smithi164.stdout: "tid": 37609 2023-06-03T20:11:33.927 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:11:33.927 INFO:teuthology.orchestra.run.smithi164.stdout: "events": [ 2023-06-03T20:11:33.927 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:11:33.927 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "initiated", 2023-06-03T20:11:33.927 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:06:33.242467+0000", 2023-06-03T20:11:33.927 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0 2023-06-03T20:11:33.927 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:11:33.927 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:11:33.928 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "throttled", 2023-06-03T20:11:33.928 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:06:33.242467+0000", 2023-06-03T20:11:33.928 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0 2023-06-03T20:11:33.928 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:11:33.928 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:11:33.928 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "header_read", 2023-06-03T20:11:33.928 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:06:33.242465+0000", 2023-06-03T20:11:33.929 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 4294967295.9999986 2023-06-03T20:11:33.929 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:11:33.929 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:11:33.929 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "all_read", 2023-06-03T20:11:33.929 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:06:33.242517+0000", 2023-06-03T20:11:33.929 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 5.1659999999999997e-05 2023-06-03T20:11:33.929 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:11:33.929 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:11:33.930 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "dispatched", 2023-06-03T20:11:33.930 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:06:33.242532+0000", 2023-06-03T20:11:33.930 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 1.5172e-05 2023-06-03T20:11:33.930 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:11:33.930 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:11:33.930 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "queued_for_pg", 2023-06-03T20:11:33.930 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:06:33.242541+0000", 2023-06-03T20:11:33.931 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 8.5969999999999992e-06 2023-06-03T20:11:33.931 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:11:33.931 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:11:33.931 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "reached_pg", 2023-06-03T20:11:33.931 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:06:33.242593+0000", 2023-06-03T20:11:33.931 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 5.2193e-05 2023-06-03T20:11:33.931 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:11:33.931 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:11:33.932 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "started", 2023-06-03T20:11:33.932 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:06:33.242731+0000", 2023-06-03T20:11:33.932 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.000137727 2023-06-03T20:11:33.932 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:11:33.932 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:11:33.932 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "waiting for subops from 2,6", 2023-06-03T20:11:33.932 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:06:33.242857+0000", 2023-06-03T20:11:33.933 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.00012576699999999999 2023-06-03T20:11:33.933 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:11:33.933 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:11:33.933 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "op_commit", 2023-06-03T20:11:33.933 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:06:33.245502+0000", 2023-06-03T20:11:33.933 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.0026450129999999999 2023-06-03T20:11:33.933 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:11:33.933 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:11:33.934 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:11:33.934 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:06:33.249673+0000", 2023-06-03T20:11:33.934 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.0041713269999999998 2023-06-03T20:11:33.934 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:11:33.934 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:11:33.934 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:11:33.934 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:06:33.398597+0000", 2023-06-03T20:11:33.934 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.148924261 2023-06-03T20:11:33.935 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:11:33.935 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:11:33.935 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "commit_sent", 2023-06-03T20:11:33.935 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:06:33.398730+0000", 2023-06-03T20:11:33.935 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.000133318 2023-06-03T20:11:33.935 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:11:33.935 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:11:33.935 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "done", 2023-06-03T20:11:33.936 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:06:33.398839+0000", 2023-06-03T20:11:33.936 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.000108427 2023-06-03T20:11:33.936 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:11:33.936 INFO:teuthology.orchestra.run.smithi164.stdout: ] 2023-06-03T20:11:33.936 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:11:33.936 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:11:33.936 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:11:33.936 INFO:teuthology.orchestra.run.smithi164.stdout: "description": "osd_op(client.36009.0:37614 10.e 10:73103e12:::benchmark_data_smithi089_140876_object37613:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e163)", 2023-06-03T20:11:33.937 INFO:teuthology.orchestra.run.smithi164.stdout: "initiated_at": "2023-06-03T20:06:33.244165+0000", 2023-06-03T20:11:33.937 INFO:teuthology.orchestra.run.smithi164.stdout: "age": 300.66484690499999, 2023-06-03T20:11:33.937 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.15614383800000001, 2023-06-03T20:11:33.937 INFO:teuthology.orchestra.run.smithi164.stdout: "type_data": { 2023-06-03T20:11:33.937 INFO:teuthology.orchestra.run.smithi164.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-03T20:11:33.937 INFO:teuthology.orchestra.run.smithi164.stdout: "client_info": { 2023-06-03T20:11:33.937 INFO:teuthology.orchestra.run.smithi164.stdout: "client": "client.36009", 2023-06-03T20:11:33.938 INFO:teuthology.orchestra.run.smithi164.stdout: "client_addr": "172.21.15.89:0/2550116169", 2023-06-03T20:11:33.938 INFO:teuthology.orchestra.run.smithi164.stdout: "tid": 37614 2023-06-03T20:11:33.938 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:11:33.938 INFO:teuthology.orchestra.run.smithi164.stdout: "events": [ 2023-06-03T20:11:33.938 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:11:33.938 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "initiated", 2023-06-03T20:11:33.938 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:06:33.244165+0000", 2023-06-03T20:11:33.938 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0 2023-06-03T20:11:33.939 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:11:33.939 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:11:33.939 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "throttled", 2023-06-03T20:11:33.939 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:06:33.244165+0000", 2023-06-03T20:11:33.939 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0 2023-06-03T20:11:33.939 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:11:33.939 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:11:33.939 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "header_read", 2023-06-03T20:11:33.940 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:06:33.244164+0000", 2023-06-03T20:11:33.940 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 4294967295.9999986 2023-06-03T20:11:33.940 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:11:33.940 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:11:33.940 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "all_read", 2023-06-03T20:11:33.940 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:06:33.244232+0000", 2023-06-03T20:11:33.940 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 6.8139000000000006e-05 2023-06-03T20:11:33.940 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:11:33.941 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:11:33.941 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "dispatched", 2023-06-03T20:11:33.941 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:06:33.244249+0000", 2023-06-03T20:11:33.941 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 1.6385e-05 2023-06-03T20:11:33.941 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:11:33.941 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:11:33.941 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "queued_for_pg", 2023-06-03T20:11:33.942 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:06:33.244257+0000", 2023-06-03T20:11:33.942 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 8.4460000000000001e-06 2023-06-03T20:11:33.942 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:11:33.942 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:11:33.942 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "reached_pg", 2023-06-03T20:11:33.942 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:06:33.244281+0000", 2023-06-03T20:11:33.942 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 2.4093000000000001e-05 2023-06-03T20:11:33.943 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:11:33.943 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:11:33.943 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "started", 2023-06-03T20:11:33.943 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:06:33.244364+0000", 2023-06-03T20:11:33.943 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 8.2731000000000005e-05 2023-06-03T20:11:33.943 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:11:33.943 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:11:33.943 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "waiting for subops from 2,6", 2023-06-03T20:11:33.944 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:06:33.244448+0000", 2023-06-03T20:11:33.944 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 8.3937000000000006e-05 2023-06-03T20:11:33.944 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:11:33.944 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:11:33.944 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "op_commit", 2023-06-03T20:11:33.944 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:06:33.245572+0000", 2023-06-03T20:11:33.944 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.0011238330000000001 2023-06-03T20:11:33.945 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:11:33.945 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:11:33.945 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:11:33.945 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:06:33.249691+0000", 2023-06-03T20:11:33.945 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.004119602 2023-06-03T20:11:33.945 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:11:33.945 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:11:33.945 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:11:33.946 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:06:33.399774+0000", 2023-06-03T20:11:33.946 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.150082245 2023-06-03T20:11:33.946 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:11:33.946 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:11:33.946 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "commit_sent", 2023-06-03T20:11:33.946 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:06:33.400202+0000", 2023-06-03T20:11:33.946 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.00042805000000000002 2023-06-03T20:11:33.946 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:11:33.947 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:11:33.947 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "done", 2023-06-03T20:11:33.947 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:06:33.400309+0000", 2023-06-03T20:11:33.947 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.000107607 2023-06-03T20:11:33.947 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:11:33.947 INFO:teuthology.orchestra.run.smithi164.stdout: ] 2023-06-03T20:11:33.947 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:11:33.947 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:11:33.948 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:11:33.948 INFO:teuthology.orchestra.run.smithi164.stdout: "description": "osd_op(client.36009.0:89062 10.2 10:483a2364:::benchmark_data_smithi089_140876_object89061:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e166)", 2023-06-03T20:11:33.948 INFO:teuthology.orchestra.run.smithi164.stdout: "initiated_at": "2023-06-03T20:07:18.897001+0000", 2023-06-03T20:11:33.948 INFO:teuthology.orchestra.run.smithi164.stdout: "age": 255.01201176999999, 2023-06-03T20:11:33.948 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 2.4270343580000002, 2023-06-03T20:11:33.948 INFO:teuthology.orchestra.run.smithi164.stdout: "type_data": { 2023-06-03T20:11:33.948 INFO:teuthology.orchestra.run.smithi164.stdout: "flag_point": "reached pg", 2023-06-03T20:11:33.949 INFO:teuthology.orchestra.run.smithi164.stdout: "client_info": { 2023-06-03T20:11:33.949 INFO:teuthology.orchestra.run.smithi164.stdout: "client": "client.36009", 2023-06-03T20:11:33.949 INFO:teuthology.orchestra.run.smithi164.stdout: "client_addr": "172.21.15.89:0/2550116169", 2023-06-03T20:11:33.949 INFO:teuthology.orchestra.run.smithi164.stdout: "tid": 89062 2023-06-03T20:11:33.949 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:11:33.949 INFO:teuthology.orchestra.run.smithi164.stdout: "events": [ 2023-06-03T20:11:33.949 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:11:33.949 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "initiated", 2023-06-03T20:11:33.950 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:07:18.897001+0000", 2023-06-03T20:11:33.950 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0 2023-06-03T20:11:33.950 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:11:33.950 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:11:33.950 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "throttled", 2023-06-03T20:11:33.950 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:07:18.897001+0000", 2023-06-03T20:11:33.950 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0 2023-06-03T20:11:33.950 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:11:33.951 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:11:33.951 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "header_read", 2023-06-03T20:11:33.951 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:07:18.896999+0000", 2023-06-03T20:11:33.951 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 4294967295.9999981 2023-06-03T20:11:33.951 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:11:33.951 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:11:33.951 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "all_read", 2023-06-03T20:11:33.951 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:07:18.897040+0000", 2023-06-03T20:11:33.952 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 4.1307999999999998e-05 2023-06-03T20:11:33.952 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:11:33.952 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:11:33.952 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "dispatched", 2023-06-03T20:11:33.952 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:07:18.897051+0000", 2023-06-03T20:11:33.952 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 1.1304999999999999e-05 2023-06-03T20:11:33.952 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:11:33.953 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:11:33.953 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "queued_for_pg", 2023-06-03T20:11:33.953 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:07:18.897061+0000", 2023-06-03T20:11:33.953 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 9.4800000000000007e-06 2023-06-03T20:11:33.953 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:11:33.953 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:11:33.953 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "reached_pg", 2023-06-03T20:11:33.953 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:07:18.897192+0000", 2023-06-03T20:11:33.954 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.000131027 2023-06-03T20:11:33.954 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:11:33.954 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:11:33.954 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "waiting for peered", 2023-06-03T20:11:33.954 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:07:18.897205+0000", 2023-06-03T20:11:33.954 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 1.3057999999999999e-05 2023-06-03T20:11:33.954 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:11:33.955 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:11:33.955 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "reached_pg", 2023-06-03T20:11:33.955 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:07:19.886396+0000", 2023-06-03T20:11:33.955 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.98919061399999997 2023-06-03T20:11:33.955 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:11:33.955 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:11:33.955 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "waiting for peered", 2023-06-03T20:11:33.955 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:07:19.886419+0000", 2023-06-03T20:11:33.956 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 2.2897000000000001e-05 2023-06-03T20:11:33.956 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:11:33.956 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:11:33.956 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "reached_pg", 2023-06-03T20:11:33.956 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:07:21.322368+0000", 2023-06-03T20:11:33.956 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 1.4359493139999999 2023-06-03T20:11:33.956 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:11:33.956 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:11:33.957 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "done", 2023-06-03T20:11:33.957 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:07:21.324035+0000", 2023-06-03T20:11:33.957 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.001667148 2023-06-03T20:11:33.957 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:11:33.957 INFO:teuthology.orchestra.run.smithi164.stdout: ] 2023-06-03T20:11:33.957 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:11:33.957 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:11:33.958 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:11:33.958 INFO:teuthology.orchestra.run.smithi164.stdout: "description": "osd_op(client.36009.0:89078 10.2 10:436022d9:::benchmark_data_smithi089_140876_object89077:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e166)", 2023-06-03T20:11:33.958 INFO:teuthology.orchestra.run.smithi164.stdout: "initiated_at": "2023-06-03T20:07:18.898041+0000", 2023-06-03T20:11:33.958 INFO:teuthology.orchestra.run.smithi164.stdout: "age": 255.010971066, 2023-06-03T20:11:33.958 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 2.447556729, 2023-06-03T20:11:33.958 INFO:teuthology.orchestra.run.smithi164.stdout: "type_data": { 2023-06-03T20:11:33.958 INFO:teuthology.orchestra.run.smithi164.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-03T20:11:33.958 INFO:teuthology.orchestra.run.smithi164.stdout: "client_info": { 2023-06-03T20:11:33.959 INFO:teuthology.orchestra.run.smithi164.stdout: "client": "client.36009", 2023-06-03T20:11:33.959 INFO:teuthology.orchestra.run.smithi164.stdout: "client_addr": "172.21.15.89:0/2550116169", 2023-06-03T20:11:33.959 INFO:teuthology.orchestra.run.smithi164.stdout: "tid": 89078 2023-06-03T20:11:33.959 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:11:33.959 INFO:teuthology.orchestra.run.smithi164.stdout: "events": [ 2023-06-03T20:11:33.959 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:11:33.959 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "initiated", 2023-06-03T20:11:33.959 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:07:18.898041+0000", 2023-06-03T20:11:33.960 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0 2023-06-03T20:11:33.960 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:11:33.960 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:11:33.960 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "throttled", 2023-06-03T20:11:33.960 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:07:18.898041+0000", 2023-06-03T20:11:33.960 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0 2023-06-03T20:11:33.960 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:11:33.960 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:11:33.961 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "header_read", 2023-06-03T20:11:33.961 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:07:18.898035+0000", 2023-06-03T20:11:33.961 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 4294967295.9999938 2023-06-03T20:11:33.961 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:11:33.961 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:11:33.961 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "all_read", 2023-06-03T20:11:33.961 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:07:18.898175+0000", 2023-06-03T20:11:33.961 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.00014004299999999999 2023-06-03T20:11:33.962 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:11:33.962 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:11:33.962 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "dispatched", 2023-06-03T20:11:33.962 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:07:18.898189+0000", 2023-06-03T20:11:33.962 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 1.3721e-05 2023-06-03T20:11:33.962 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:11:33.962 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:11:33.962 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "queued_for_pg", 2023-06-03T20:11:33.963 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:07:18.898198+0000", 2023-06-03T20:11:33.963 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 9.3729999999999999e-06 2023-06-03T20:11:33.963 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:11:33.963 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:11:33.963 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "reached_pg", 2023-06-03T20:11:33.963 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:07:18.898257+0000", 2023-06-03T20:11:33.963 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 5.8925000000000002e-05 2023-06-03T20:11:33.964 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:11:33.964 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:11:33.964 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "waiting for peered", 2023-06-03T20:11:33.964 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:07:18.898272+0000", 2023-06-03T20:11:33.964 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 1.4814000000000001e-05 2023-06-03T20:11:33.964 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:11:33.964 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:11:33.964 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "reached_pg", 2023-06-03T20:11:33.965 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:07:19.886522+0000", 2023-06-03T20:11:33.965 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.98825009200000002 2023-06-03T20:11:33.965 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:11:33.965 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:11:33.965 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "waiting for peered", 2023-06-03T20:11:33.965 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:07:19.886537+0000", 2023-06-03T20:11:33.965 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 1.5031000000000001e-05 2023-06-03T20:11:33.965 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:11:33.966 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:11:33.966 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "reached_pg", 2023-06-03T20:11:33.966 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:07:21.326297+0000", 2023-06-03T20:11:33.966 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 1.4397595540000001 2023-06-03T20:11:33.966 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:11:33.966 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:11:33.966 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "started", 2023-06-03T20:11:33.966 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:07:21.327161+0000", 2023-06-03T20:11:33.967 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.00086428299999999998 2023-06-03T20:11:33.967 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:11:33.967 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:11:33.967 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "waiting for subops from 0,3", 2023-06-03T20:11:33.967 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:07:21.327485+0000", 2023-06-03T20:11:33.967 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.00032367600000000001 2023-06-03T20:11:33.967 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:11:33.967 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:11:33.968 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:11:33.968 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:07:21.339015+0000", 2023-06-03T20:11:33.968 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.011530436 2023-06-03T20:11:33.968 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:11:33.968 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:11:33.968 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "op_commit", 2023-06-03T20:11:33.968 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:07:21.342997+0000", 2023-06-03T20:11:33.969 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.0039819950000000003 2023-06-03T20:11:33.969 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:11:33.969 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:11:33.969 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:11:33.969 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:07:21.343139+0000", 2023-06-03T20:11:33.969 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.000142027 2023-06-03T20:11:33.969 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:11:33.969 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:11:33.970 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "commit_sent", 2023-06-03T20:11:33.970 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:07:21.343210+0000", 2023-06-03T20:11:33.970 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 7.0951000000000006e-05 2023-06-03T20:11:33.970 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:11:33.970 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:11:33.970 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "done", 2023-06-03T20:11:33.970 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:07:21.345598+0000", 2023-06-03T20:11:33.970 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.0023879249999999999 2023-06-03T20:11:33.971 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:11:33.971 INFO:teuthology.orchestra.run.smithi164.stdout: ] 2023-06-03T20:11:33.971 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:11:33.971 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:11:33.971 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:11:33.971 INFO:teuthology.orchestra.run.smithi164.stdout: "description": "osd_op(client.36009.0:93796 10.1 10:883c2d85:::benchmark_data_smithi089_140876_object93795:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e170)", 2023-06-03T20:11:33.971 INFO:teuthology.orchestra.run.smithi164.stdout: "initiated_at": "2023-06-03T20:07:27.772745+0000", 2023-06-03T20:11:33.971 INFO:teuthology.orchestra.run.smithi164.stdout: "age": 246.13626724, 2023-06-03T20:11:33.972 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.23955933099999999, 2023-06-03T20:11:33.972 INFO:teuthology.orchestra.run.smithi164.stdout: "type_data": { 2023-06-03T20:11:33.972 INFO:teuthology.orchestra.run.smithi164.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-03T20:11:33.972 INFO:teuthology.orchestra.run.smithi164.stdout: "client_info": { 2023-06-03T20:11:33.972 INFO:teuthology.orchestra.run.smithi164.stdout: "client": "client.36009", 2023-06-03T20:11:33.972 INFO:teuthology.orchestra.run.smithi164.stdout: "client_addr": "172.21.15.89:0/2550116169", 2023-06-03T20:11:33.972 INFO:teuthology.orchestra.run.smithi164.stdout: "tid": 93796 2023-06-03T20:11:33.972 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:11:33.973 INFO:teuthology.orchestra.run.smithi164.stdout: "events": [ 2023-06-03T20:11:33.973 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:11:33.973 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "initiated", 2023-06-03T20:11:33.973 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:07:27.772745+0000", 2023-06-03T20:11:33.973 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0 2023-06-03T20:11:33.973 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:11:33.973 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:11:33.974 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "throttled", 2023-06-03T20:11:33.974 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:07:27.772745+0000", 2023-06-03T20:11:33.974 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0 2023-06-03T20:11:33.974 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:11:33.974 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:11:33.974 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "header_read", 2023-06-03T20:11:33.974 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:07:27.772744+0000", 2023-06-03T20:11:33.974 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 4294967295.9999986 2023-06-03T20:11:33.975 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:11:33.975 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:11:33.975 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "all_read", 2023-06-03T20:11:33.975 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:07:27.772776+0000", 2023-06-03T20:11:33.975 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 3.2227999999999997e-05 2023-06-03T20:11:33.975 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:11:33.975 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:11:33.976 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "dispatched", 2023-06-03T20:11:33.976 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:07:27.772786+0000", 2023-06-03T20:11:33.976 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 1.0536999999999999e-05 2023-06-03T20:11:33.976 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:11:33.976 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:11:33.976 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "queued_for_pg", 2023-06-03T20:11:33.976 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:07:27.772794+0000", 2023-06-03T20:11:33.976 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 8.0439999999999996e-06 2023-06-03T20:11:33.977 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:11:33.977 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:11:33.977 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "reached_pg", 2023-06-03T20:11:33.977 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:07:27.772886+0000", 2023-06-03T20:11:33.977 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 9.1458000000000004e-05 2023-06-03T20:11:33.977 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:11:33.977 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:11:33.977 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "started", 2023-06-03T20:11:33.978 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:07:27.773032+0000", 2023-06-03T20:11:33.978 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.00014569299999999999 2023-06-03T20:11:33.978 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:11:33.978 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:11:33.978 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "waiting for subops from 0,7", 2023-06-03T20:11:33.978 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:07:27.773139+0000", 2023-06-03T20:11:33.978 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.000107193 2023-06-03T20:11:33.978 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:11:33.979 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:11:33.979 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "op_commit", 2023-06-03T20:11:33.979 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:07:27.777093+0000", 2023-06-03T20:11:33.979 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.0039544949999999997 2023-06-03T20:11:33.979 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:11:33.979 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:11:33.980 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:11:33.980 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:07:27.785474+0000", 2023-06-03T20:11:33.980 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.0083809590000000003 2023-06-03T20:11:33.980 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:11:33.980 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:11:33.980 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:11:33.980 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:07:28.012198+0000", 2023-06-03T20:11:33.980 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.22672367700000001 2023-06-03T20:11:33.981 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:11:33.981 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:11:33.981 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "commit_sent", 2023-06-03T20:11:33.981 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:07:28.012258+0000", 2023-06-03T20:11:33.981 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 5.9756000000000002e-05 2023-06-03T20:11:33.981 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:11:33.981 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:11:33.981 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "done", 2023-06-03T20:11:33.982 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:07:28.012304+0000", 2023-06-03T20:11:33.982 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 4.6773000000000001e-05 2023-06-03T20:11:33.982 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:11:33.982 INFO:teuthology.orchestra.run.smithi164.stdout: ] 2023-06-03T20:11:33.982 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:11:33.982 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:11:33.982 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:11:33.982 INFO:teuthology.orchestra.run.smithi164.stdout: "description": "osd_op(client.36009.0:93804 10.1 10:8a179b3d:::benchmark_data_smithi089_140876_object93803:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e170)", 2023-06-03T20:11:33.983 INFO:teuthology.orchestra.run.smithi164.stdout: "initiated_at": "2023-06-03T20:07:27.780609+0000", 2023-06-03T20:11:33.983 INFO:teuthology.orchestra.run.smithi164.stdout: "age": 246.128403577, 2023-06-03T20:11:33.983 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.236769536, 2023-06-03T20:11:33.983 INFO:teuthology.orchestra.run.smithi164.stdout: "type_data": { 2023-06-03T20:11:33.983 INFO:teuthology.orchestra.run.smithi164.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-03T20:11:33.983 INFO:teuthology.orchestra.run.smithi164.stdout: "client_info": { 2023-06-03T20:11:33.983 INFO:teuthology.orchestra.run.smithi164.stdout: "client": "client.36009", 2023-06-03T20:11:33.983 INFO:teuthology.orchestra.run.smithi164.stdout: "client_addr": "172.21.15.89:0/2550116169", 2023-06-03T20:11:33.984 INFO:teuthology.orchestra.run.smithi164.stdout: "tid": 93804 2023-06-03T20:11:33.984 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:11:33.984 INFO:teuthology.orchestra.run.smithi164.stdout: "events": [ 2023-06-03T20:11:33.984 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:11:33.984 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "initiated", 2023-06-03T20:11:33.984 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:07:27.780609+0000", 2023-06-03T20:11:33.984 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0 2023-06-03T20:11:33.985 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:11:33.985 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:11:33.985 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "throttled", 2023-06-03T20:11:33.985 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:07:27.780609+0000", 2023-06-03T20:11:33.985 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0 2023-06-03T20:11:33.985 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:11:33.985 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:11:33.985 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "header_read", 2023-06-03T20:11:33.986 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:07:27.780607+0000", 2023-06-03T20:11:33.986 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 4294967295.9999986 2023-06-03T20:11:33.986 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:11:33.986 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:11:33.986 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "all_read", 2023-06-03T20:11:33.986 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:07:27.780693+0000", 2023-06-03T20:11:33.986 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 8.5871999999999994e-05 2023-06-03T20:11:33.987 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:11:33.987 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:11:33.987 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "dispatched", 2023-06-03T20:11:33.987 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:07:27.780707+0000", 2023-06-03T20:11:33.987 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 1.3427999999999999e-05 2023-06-03T20:11:33.987 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:11:33.987 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:11:33.988 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "queued_for_pg", 2023-06-03T20:11:33.988 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:07:27.780716+0000", 2023-06-03T20:11:33.988 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 9.1589999999999999e-06 2023-06-03T20:11:33.988 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:11:33.988 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:11:33.988 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "reached_pg", 2023-06-03T20:11:33.988 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:07:27.780770+0000", 2023-06-03T20:11:33.988 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 5.3859000000000001e-05 2023-06-03T20:11:33.989 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:11:33.989 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:11:33.989 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "started", 2023-06-03T20:11:33.989 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:07:27.780884+0000", 2023-06-03T20:11:33.989 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.000114861 2023-06-03T20:11:33.989 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:11:33.989 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:11:33.989 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "waiting for subops from 0,7", 2023-06-03T20:11:33.990 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:07:27.780998+0000", 2023-06-03T20:11:33.990 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.000113936 2023-06-03T20:11:33.990 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:11:33.990 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:11:33.990 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "op_commit", 2023-06-03T20:11:33.990 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:07:27.789950+0000", 2023-06-03T20:11:33.990 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.008951779 2023-06-03T20:11:33.990 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:11:33.991 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:11:33.991 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:11:33.991 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:07:27.794044+0000", 2023-06-03T20:11:33.991 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.0040941980000000003 2023-06-03T20:11:33.991 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:11:33.991 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:11:33.991 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:11:33.992 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:07:28.017203+0000", 2023-06-03T20:11:33.992 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.223158208 2023-06-03T20:11:33.992 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:11:33.992 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:11:33.992 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "commit_sent", 2023-06-03T20:11:33.992 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:07:28.017324+0000", 2023-06-03T20:11:33.992 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.000121604 2023-06-03T20:11:33.992 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:11:33.993 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:11:33.993 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "done", 2023-06-03T20:11:33.993 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:07:28.017378+0000", 2023-06-03T20:11:33.993 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 5.4240999999999998e-05 2023-06-03T20:11:33.993 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:11:33.993 INFO:teuthology.orchestra.run.smithi164.stdout: ] 2023-06-03T20:11:33.993 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:11:33.993 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:11:33.994 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:11:33.994 INFO:teuthology.orchestra.run.smithi164.stdout: "description": "osd_op(client.36009.0:93814 10.2 10:47d562fd:::benchmark_data_smithi089_140876_object93813:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e170)", 2023-06-03T20:11:33.994 INFO:teuthology.orchestra.run.smithi164.stdout: "initiated_at": "2023-06-03T20:07:27.802317+0000", 2023-06-03T20:11:33.994 INFO:teuthology.orchestra.run.smithi164.stdout: "age": 246.10669544199999, 2023-06-03T20:11:33.994 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.21702903700000001, 2023-06-03T20:11:33.994 INFO:teuthology.orchestra.run.smithi164.stdout: "type_data": { 2023-06-03T20:11:33.994 INFO:teuthology.orchestra.run.smithi164.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-03T20:11:33.994 INFO:teuthology.orchestra.run.smithi164.stdout: "client_info": { 2023-06-03T20:11:33.995 INFO:teuthology.orchestra.run.smithi164.stdout: "client": "client.36009", 2023-06-03T20:11:33.995 INFO:teuthology.orchestra.run.smithi164.stdout: "client_addr": "172.21.15.89:0/2550116169", 2023-06-03T20:11:33.995 INFO:teuthology.orchestra.run.smithi164.stdout: "tid": 93814 2023-06-03T20:11:33.995 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:11:33.995 INFO:teuthology.orchestra.run.smithi164.stdout: "events": [ 2023-06-03T20:11:33.995 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:11:33.995 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "initiated", 2023-06-03T20:11:33.995 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:07:27.802317+0000", 2023-06-03T20:11:33.996 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0 2023-06-03T20:11:33.996 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:11:33.996 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:11:33.996 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "throttled", 2023-06-03T20:11:33.996 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:07:27.802317+0000", 2023-06-03T20:11:33.996 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0 2023-06-03T20:11:33.996 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:11:33.997 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:11:33.997 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "header_read", 2023-06-03T20:11:33.997 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:07:27.802311+0000", 2023-06-03T20:11:33.997 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 4294967295.9999943 2023-06-03T20:11:33.997 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:11:33.997 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:11:33.997 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "all_read", 2023-06-03T20:11:33.997 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:07:27.802369+0000", 2023-06-03T20:11:33.998 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 5.7698000000000002e-05 2023-06-03T20:11:33.998 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:11:33.998 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:11:33.998 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "dispatched", 2023-06-03T20:11:33.998 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:07:27.802395+0000", 2023-06-03T20:11:33.998 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 2.5996999999999999e-05 2023-06-03T20:11:33.998 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:11:33.999 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:11:33.999 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "queued_for_pg", 2023-06-03T20:11:33.999 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:07:27.802409+0000", 2023-06-03T20:11:33.999 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 1.3886999999999999e-05 2023-06-03T20:11:33.999 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:11:33.999 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:11:33.999 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "reached_pg", 2023-06-03T20:11:33.999 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:07:27.802491+0000", 2023-06-03T20:11:34.000 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 8.2257999999999997e-05 2023-06-03T20:11:34.000 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:11:34.000 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:11:34.000 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "started", 2023-06-03T20:11:34.000 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:07:27.802649+0000", 2023-06-03T20:11:34.000 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.00015801000000000001 2023-06-03T20:11:34.000 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:11:34.001 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:11:34.001 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "waiting for subops from 0,3", 2023-06-03T20:11:34.001 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:07:27.802813+0000", 2023-06-03T20:11:34.001 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.00016403899999999999 2023-06-03T20:11:34.001 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:11:34.001 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:11:34.001 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "op_commit", 2023-06-03T20:11:34.001 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:07:27.807373+0000", 2023-06-03T20:11:34.002 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.0045597629999999997 2023-06-03T20:11:34.002 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:11:34.002 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:11:34.002 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:11:34.002 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:07:27.808999+0000", 2023-06-03T20:11:34.002 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.0016263619999999999 2023-06-03T20:11:34.002 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:11:34.002 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:11:34.003 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:11:34.003 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:07:28.017319+0000", 2023-06-03T20:11:34.003 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.20831960499999999 2023-06-03T20:11:34.003 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:11:34.003 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:11:34.003 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "commit_sent", 2023-06-03T20:11:34.003 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:07:28.017375+0000", 2023-06-03T20:11:34.003 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 5.6412000000000001e-05 2023-06-03T20:11:34.004 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:11:34.004 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:11:34.004 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "done", 2023-06-03T20:11:34.004 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:07:28.019346+0000", 2023-06-03T20:11:34.004 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.0019707040000000002 2023-06-03T20:11:34.004 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:11:34.004 INFO:teuthology.orchestra.run.smithi164.stdout: ] 2023-06-03T20:11:34.005 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:11:34.005 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:11:34.005 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:11:34.005 INFO:teuthology.orchestra.run.smithi164.stdout: "description": "osd_op(client.36009.0:126094 10.1 10:8c9981bb:::benchmark_data_smithi089_140876_object20959:head [delete] snapc 0=[] ondisk+write+known_if_redirected e174)", 2023-06-03T20:11:34.005 INFO:teuthology.orchestra.run.smithi164.stdout: "initiated_at": "2023-06-03T20:08:16.526856+0000", 2023-06-03T20:11:34.005 INFO:teuthology.orchestra.run.smithi164.stdout: "age": 197.38215603200001, 2023-06-03T20:11:34.005 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.17235614099999999, 2023-06-03T20:11:34.005 INFO:teuthology.orchestra.run.smithi164.stdout: "type_data": { 2023-06-03T20:11:34.006 INFO:teuthology.orchestra.run.smithi164.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-03T20:11:34.006 INFO:teuthology.orchestra.run.smithi164.stdout: "client_info": { 2023-06-03T20:11:34.006 INFO:teuthology.orchestra.run.smithi164.stdout: "client": "client.36009", 2023-06-03T20:11:34.006 INFO:teuthology.orchestra.run.smithi164.stdout: "client_addr": "172.21.15.89:0/2550116169", 2023-06-03T20:11:34.006 INFO:teuthology.orchestra.run.smithi164.stdout: "tid": 126094 2023-06-03T20:11:34.006 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:11:34.006 INFO:teuthology.orchestra.run.smithi164.stdout: "events": [ 2023-06-03T20:11:34.006 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:11:34.007 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "initiated", 2023-06-03T20:11:34.007 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:08:16.526856+0000", 2023-06-03T20:11:34.007 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0 2023-06-03T20:11:34.007 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:11:34.007 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:11:34.007 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "throttled", 2023-06-03T20:11:34.007 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:08:16.526856+0000", 2023-06-03T20:11:34.007 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0 2023-06-03T20:11:34.008 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:11:34.008 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:11:34.008 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "header_read", 2023-06-03T20:11:34.008 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:08:16.526854+0000", 2023-06-03T20:11:34.008 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 4294967295.9999981 2023-06-03T20:11:34.008 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:11:34.008 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:11:34.008 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "all_read", 2023-06-03T20:11:34.009 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:08:16.526864+0000", 2023-06-03T20:11:34.009 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 9.5559999999999996e-06 2023-06-03T20:11:34.009 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:11:34.009 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:11:34.009 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "dispatched", 2023-06-03T20:11:34.009 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:08:16.526882+0000", 2023-06-03T20:11:34.009 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 1.7779999999999999e-05 2023-06-03T20:11:34.009 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:11:34.010 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:11:34.010 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "queued_for_pg", 2023-06-03T20:11:34.010 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:08:16.526890+0000", 2023-06-03T20:11:34.010 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 8.6009999999999995e-06 2023-06-03T20:11:34.010 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:11:34.010 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:11:34.010 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "reached_pg", 2023-06-03T20:11:34.010 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:08:16.526980+0000", 2023-06-03T20:11:34.011 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 8.9604999999999993e-05 2023-06-03T20:11:34.011 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:11:34.011 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:11:34.011 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "started", 2023-06-03T20:11:34.011 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:08:16.527219+0000", 2023-06-03T20:11:34.011 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.00023889500000000001 2023-06-03T20:11:34.011 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:11:34.011 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:11:34.012 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "waiting for subops from 0,7", 2023-06-03T20:11:34.012 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:08:16.527437+0000", 2023-06-03T20:11:34.012 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.000218206 2023-06-03T20:11:34.012 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:11:34.012 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:11:34.012 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "op_commit", 2023-06-03T20:11:34.012 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:08:16.531625+0000", 2023-06-03T20:11:34.013 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.0041879969999999997 2023-06-03T20:11:34.013 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:11:34.013 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:11:34.013 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:11:34.013 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:08:16.533414+0000", 2023-06-03T20:11:34.013 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.0017893760000000001 2023-06-03T20:11:34.013 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:11:34.013 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:11:34.014 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:11:34.014 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:08:16.699068+0000", 2023-06-03T20:11:34.014 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.16565333099999999 2023-06-03T20:11:34.014 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:11:34.014 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:11:34.014 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "commit_sent", 2023-06-03T20:11:34.014 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:08:16.699146+0000", 2023-06-03T20:11:34.015 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 7.7922000000000004e-05 2023-06-03T20:11:34.015 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:11:34.015 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:11:34.015 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "done", 2023-06-03T20:11:34.015 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:08:16.699213+0000", 2023-06-03T20:11:34.015 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 6.6983999999999994e-05 2023-06-03T20:11:34.015 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:11:34.015 INFO:teuthology.orchestra.run.smithi164.stdout: ] 2023-06-03T20:11:34.016 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:11:34.016 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:11:34.016 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:11:34.016 INFO:teuthology.orchestra.run.smithi164.stdout: "description": "osd_op(client.36009.0:126097 10.2 10:453b66da:::benchmark_data_smithi089_140876_object20962:head [delete] snapc 0=[] ondisk+write+known_if_redirected e174)", 2023-06-03T20:11:34.016 INFO:teuthology.orchestra.run.smithi164.stdout: "initiated_at": "2023-06-03T20:08:16.531205+0000", 2023-06-03T20:11:34.016 INFO:teuthology.orchestra.run.smithi164.stdout: "age": 197.377807093, 2023-06-03T20:11:34.016 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.17006711599999999, 2023-06-03T20:11:34.016 INFO:teuthology.orchestra.run.smithi164.stdout: "type_data": { 2023-06-03T20:11:34.017 INFO:teuthology.orchestra.run.smithi164.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-03T20:11:34.017 INFO:teuthology.orchestra.run.smithi164.stdout: "client_info": { 2023-06-03T20:11:34.017 INFO:teuthology.orchestra.run.smithi164.stdout: "client": "client.36009", 2023-06-03T20:11:34.017 INFO:teuthology.orchestra.run.smithi164.stdout: "client_addr": "172.21.15.89:0/2550116169", 2023-06-03T20:11:34.017 INFO:teuthology.orchestra.run.smithi164.stdout: "tid": 126097 2023-06-03T20:11:34.017 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:11:34.017 INFO:teuthology.orchestra.run.smithi164.stdout: "events": [ 2023-06-03T20:11:34.017 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:11:34.018 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "initiated", 2023-06-03T20:11:34.018 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:08:16.531205+0000", 2023-06-03T20:11:34.018 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0 2023-06-03T20:11:34.018 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:11:34.018 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:11:34.018 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "throttled", 2023-06-03T20:11:34.018 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:08:16.531205+0000", 2023-06-03T20:11:34.018 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0 2023-06-03T20:11:34.019 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:11:34.019 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:11:34.019 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "header_read", 2023-06-03T20:11:34.019 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:08:16.531204+0000", 2023-06-03T20:11:34.019 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 4294967295.9999986 2023-06-03T20:11:34.019 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:11:34.019 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:11:34.019 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "all_read", 2023-06-03T20:11:34.020 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:08:16.531212+0000", 2023-06-03T20:11:34.020 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 8.3310000000000004e-06 2023-06-03T20:11:34.020 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:11:34.020 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:11:34.020 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "dispatched", 2023-06-03T20:11:34.021 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:08:16.531226+0000", 2023-06-03T20:11:34.021 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 1.3534e-05 2023-06-03T20:11:34.021 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:11:34.021 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:11:34.021 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "queued_for_pg", 2023-06-03T20:11:34.021 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:08:16.531234+0000", 2023-06-03T20:11:34.021 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 8.8300000000000002e-06 2023-06-03T20:11:34.021 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:11:34.022 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:11:34.022 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "reached_pg", 2023-06-03T20:11:34.022 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:08:16.533291+0000", 2023-06-03T20:11:34.022 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.002056988 2023-06-03T20:11:34.022 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:11:34.022 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:11:34.022 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "started", 2023-06-03T20:11:34.022 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:08:16.533606+0000", 2023-06-03T20:11:34.023 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.00031490000000000001 2023-06-03T20:11:34.023 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:11:34.023 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:11:34.023 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "waiting for subops from 0,3", 2023-06-03T20:11:34.023 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:08:16.533723+0000", 2023-06-03T20:11:34.023 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.00011662099999999999 2023-06-03T20:11:34.023 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:11:34.024 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:11:34.024 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "op_commit", 2023-06-03T20:11:34.024 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:08:16.540778+0000", 2023-06-03T20:11:34.024 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.0070550509999999997 2023-06-03T20:11:34.024 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:11:34.024 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:11:34.024 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:11:34.024 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:08:16.541169+0000", 2023-06-03T20:11:34.025 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.00039093899999999998 2023-06-03T20:11:34.025 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:11:34.025 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:11:34.025 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:11:34.025 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:08:16.698883+0000", 2023-06-03T20:11:34.025 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.157714194 2023-06-03T20:11:34.025 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:11:34.026 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:11:34.026 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "commit_sent", 2023-06-03T20:11:34.026 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:08:16.698960+0000", 2023-06-03T20:11:34.026 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 7.6462000000000001e-05 2023-06-03T20:11:34.026 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:11:34.026 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:11:34.026 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "done", 2023-06-03T20:11:34.026 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:08:16.701272+0000", 2023-06-03T20:11:34.027 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.0023128379999999998 2023-06-03T20:11:34.027 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:11:34.027 INFO:teuthology.orchestra.run.smithi164.stdout: ] 2023-06-03T20:11:34.027 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:11:34.027 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:11:34.027 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:11:34.027 INFO:teuthology.orchestra.run.smithi164.stdout: "description": "osd_op(client.36009.0:147781 10.2 10:44807418:::benchmark_data_smithi089_140876_object42646:head [delete] snapc 0=[] ondisk+write+known_if_redirected e176)", 2023-06-03T20:11:34.027 INFO:teuthology.orchestra.run.smithi164.stdout: "initiated_at": "2023-06-03T20:08:50.958695+0000", 2023-06-03T20:11:34.028 INFO:teuthology.orchestra.run.smithi164.stdout: "age": 162.95031722799999, 2023-06-03T20:11:34.028 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.148194032, 2023-06-03T20:11:34.028 INFO:teuthology.orchestra.run.smithi164.stdout: "type_data": { 2023-06-03T20:11:34.028 INFO:teuthology.orchestra.run.smithi164.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-03T20:11:34.028 INFO:teuthology.orchestra.run.smithi164.stdout: "client_info": { 2023-06-03T20:11:34.028 INFO:teuthology.orchestra.run.smithi164.stdout: "client": "client.36009", 2023-06-03T20:11:34.028 INFO:teuthology.orchestra.run.smithi164.stdout: "client_addr": "172.21.15.89:0/2550116169", 2023-06-03T20:11:34.029 INFO:teuthology.orchestra.run.smithi164.stdout: "tid": 147781 2023-06-03T20:11:34.029 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:11:34.029 INFO:teuthology.orchestra.run.smithi164.stdout: "events": [ 2023-06-03T20:11:34.029 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:11:34.029 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "initiated", 2023-06-03T20:11:34.029 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:08:50.958695+0000", 2023-06-03T20:11:34.029 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0 2023-06-03T20:11:34.029 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:11:34.030 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:11:34.030 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "throttled", 2023-06-03T20:11:34.030 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:08:50.958695+0000", 2023-06-03T20:11:34.030 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0 2023-06-03T20:11:34.030 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:11:34.030 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:11:34.030 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "header_read", 2023-06-03T20:11:34.030 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:08:50.958695+0000", 2023-06-03T20:11:34.031 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 4294967295.9999995 2023-06-03T20:11:34.031 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:11:34.031 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:11:34.031 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "all_read", 2023-06-03T20:11:34.031 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:08:50.958697+0000", 2023-06-03T20:11:34.031 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 2.3760000000000002e-06 2023-06-03T20:11:34.031 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:11:34.031 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:11:34.032 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "dispatched", 2023-06-03T20:11:34.032 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:08:50.958703+0000", 2023-06-03T20:11:34.032 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 5.4029999999999998e-06 2023-06-03T20:11:34.032 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:11:34.032 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:11:34.032 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "queued_for_pg", 2023-06-03T20:11:34.032 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:08:50.958708+0000", 2023-06-03T20:11:34.032 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 4.9060000000000001e-06 2023-06-03T20:11:34.033 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:11:34.033 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:11:34.033 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "reached_pg", 2023-06-03T20:11:34.033 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:08:50.963827+0000", 2023-06-03T20:11:34.033 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.0051199460000000002 2023-06-03T20:11:34.033 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:11:34.033 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:11:34.033 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "started", 2023-06-03T20:11:34.034 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:08:50.963997+0000", 2023-06-03T20:11:34.034 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.00016925299999999999 2023-06-03T20:11:34.034 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:11:34.034 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:11:34.034 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "waiting for subops from 0,3", 2023-06-03T20:11:34.034 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:08:50.964163+0000", 2023-06-03T20:11:34.034 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.00016600799999999999 2023-06-03T20:11:34.034 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:11:34.035 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:11:34.035 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "op_commit", 2023-06-03T20:11:34.035 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:08:50.967858+0000", 2023-06-03T20:11:34.035 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.003694864 2023-06-03T20:11:34.035 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:11:34.035 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:11:34.035 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:11:34.035 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:08:50.976752+0000", 2023-06-03T20:11:34.036 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.0088948450000000002 2023-06-03T20:11:34.036 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:11:34.036 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:11:34.036 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:11:34.036 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:08:51.095700+0000", 2023-06-03T20:11:34.036 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.118947144 2023-06-03T20:11:34.036 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:11:34.036 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:11:34.037 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "commit_sent", 2023-06-03T20:11:34.037 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:08:51.095760+0000", 2023-06-03T20:11:34.037 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 6.0191e-05 2023-06-03T20:11:34.037 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:11:34.037 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:11:34.037 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "done", 2023-06-03T20:11:34.037 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:08:51.106889+0000", 2023-06-03T20:11:34.037 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.011129445 2023-06-03T20:11:34.038 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:11:34.038 INFO:teuthology.orchestra.run.smithi164.stdout: ] 2023-06-03T20:11:34.038 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:11:34.038 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:11:34.038 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:11:34.038 INFO:teuthology.orchestra.run.smithi164.stdout: "description": "osd_op(client.36009.0:157386 10.2 10:460d08b4:::benchmark_data_smithi089_140876_object52251:head [delete] snapc 0=[] ondisk+write+known_if_redirected e176)", 2023-06-03T20:11:34.038 INFO:teuthology.orchestra.run.smithi164.stdout: "initiated_at": "2023-06-03T20:09:05.800266+0000", 2023-06-03T20:11:34.039 INFO:teuthology.orchestra.run.smithi164.stdout: "age": 148.108746489, 2023-06-03T20:11:34.039 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.148802764, 2023-06-03T20:11:34.039 INFO:teuthology.orchestra.run.smithi164.stdout: "type_data": { 2023-06-03T20:11:34.039 INFO:teuthology.orchestra.run.smithi164.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-03T20:11:34.039 INFO:teuthology.orchestra.run.smithi164.stdout: "client_info": { 2023-06-03T20:11:34.039 INFO:teuthology.orchestra.run.smithi164.stdout: "client": "client.36009", 2023-06-03T20:11:34.039 INFO:teuthology.orchestra.run.smithi164.stdout: "client_addr": "172.21.15.89:0/2550116169", 2023-06-03T20:11:34.039 INFO:teuthology.orchestra.run.smithi164.stdout: "tid": 157386 2023-06-03T20:11:34.040 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:11:34.040 INFO:teuthology.orchestra.run.smithi164.stdout: "events": [ 2023-06-03T20:11:34.040 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:11:34.040 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "initiated", 2023-06-03T20:11:34.040 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:09:05.800266+0000", 2023-06-03T20:11:34.040 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0 2023-06-03T20:11:34.040 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:11:34.040 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:11:34.041 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "throttled", 2023-06-03T20:11:34.041 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:09:05.800266+0000", 2023-06-03T20:11:34.041 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0 2023-06-03T20:11:34.041 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:11:34.041 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:11:34.041 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "header_read", 2023-06-03T20:11:34.041 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:09:05.800264+0000", 2023-06-03T20:11:34.042 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 4294967295.9999976 2023-06-03T20:11:34.042 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:11:34.042 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:11:34.042 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "all_read", 2023-06-03T20:11:34.042 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:09:05.800274+0000", 2023-06-03T20:11:34.042 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 9.9420000000000007e-06 2023-06-03T20:11:34.042 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:11:34.042 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:11:34.043 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "dispatched", 2023-06-03T20:11:34.043 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:09:05.800306+0000", 2023-06-03T20:11:34.043 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 3.2339999999999999e-05 2023-06-03T20:11:34.043 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:11:34.043 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:11:34.043 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "queued_for_pg", 2023-06-03T20:11:34.043 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:09:05.800316+0000", 2023-06-03T20:11:34.044 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 9.978e-06 2023-06-03T20:11:34.044 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:11:34.044 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:11:34.044 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "reached_pg", 2023-06-03T20:11:34.044 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:09:05.800374+0000", 2023-06-03T20:11:34.044 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 5.8035000000000003e-05 2023-06-03T20:11:34.044 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:11:34.044 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:11:34.045 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "started", 2023-06-03T20:11:34.045 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:09:05.800592+0000", 2023-06-03T20:11:34.045 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.00021808400000000001 2023-06-03T20:11:34.045 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:11:34.045 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:11:34.045 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "waiting for subops from 0,3", 2023-06-03T20:11:34.045 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:09:05.807312+0000", 2023-06-03T20:11:34.045 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.0067203530000000001 2023-06-03T20:11:34.046 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:11:34.046 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:11:34.046 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "op_commit", 2023-06-03T20:11:34.046 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:09:05.811774+0000", 2023-06-03T20:11:34.046 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.0044619619999999999 2023-06-03T20:11:34.046 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:11:34.046 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:11:34.046 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:11:34.047 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:09:05.818240+0000", 2023-06-03T20:11:34.047 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.0064660070000000002 2023-06-03T20:11:34.047 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:11:34.047 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:11:34.047 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:11:34.047 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:09:05.946431+0000", 2023-06-03T20:11:34.047 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.12819077600000001 2023-06-03T20:11:34.047 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:11:34.048 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:11:34.048 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "commit_sent", 2023-06-03T20:11:34.048 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:09:05.946692+0000", 2023-06-03T20:11:34.048 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.00026062999999999998 2023-06-03T20:11:34.048 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:11:34.048 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:11:34.048 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "done", 2023-06-03T20:11:34.049 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:09:05.949069+0000", 2023-06-03T20:11:34.049 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.0023768690000000002 2023-06-03T20:11:34.049 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:11:34.049 INFO:teuthology.orchestra.run.smithi164.stdout: ] 2023-06-03T20:11:34.049 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:11:34.049 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:11:34.049 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:11:34.049 INFO:teuthology.orchestra.run.smithi164.stdout: "description": "osd_op(client.36009.0:159787 10.e 10:7475d4a8:::benchmark_data_smithi089_140876_object54652:head [delete] snapc 0=[] ondisk+write+known_if_redirected e176)", 2023-06-03T20:11:34.050 INFO:teuthology.orchestra.run.smithi164.stdout: "initiated_at": "2023-06-03T20:09:09.694550+0000", 2023-06-03T20:11:34.050 INFO:teuthology.orchestra.run.smithi164.stdout: "age": 144.21446193700001, 2023-06-03T20:11:34.050 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.16122798299999999, 2023-06-03T20:11:34.050 INFO:teuthology.orchestra.run.smithi164.stdout: "type_data": { 2023-06-03T20:11:34.050 INFO:teuthology.orchestra.run.smithi164.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-03T20:11:34.050 INFO:teuthology.orchestra.run.smithi164.stdout: "client_info": { 2023-06-03T20:11:34.050 INFO:teuthology.orchestra.run.smithi164.stdout: "client": "client.36009", 2023-06-03T20:11:34.050 INFO:teuthology.orchestra.run.smithi164.stdout: "client_addr": "172.21.15.89:0/2550116169", 2023-06-03T20:11:34.051 INFO:teuthology.orchestra.run.smithi164.stdout: "tid": 159787 2023-06-03T20:11:34.051 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:11:34.051 INFO:teuthology.orchestra.run.smithi164.stdout: "events": [ 2023-06-03T20:11:34.051 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:11:34.051 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "initiated", 2023-06-03T20:11:34.051 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:09:09.694550+0000", 2023-06-03T20:11:34.051 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0 2023-06-03T20:11:34.051 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:11:34.052 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:11:34.052 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "throttled", 2023-06-03T20:11:34.052 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:09:09.694550+0000", 2023-06-03T20:11:34.052 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0 2023-06-03T20:11:34.052 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:11:34.052 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:11:34.052 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "header_read", 2023-06-03T20:11:34.052 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:09:09.694549+0000", 2023-06-03T20:11:34.053 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 4294967295.9999981 2023-06-03T20:11:34.053 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:11:34.053 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:11:34.053 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "all_read", 2023-06-03T20:11:34.053 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:09:09.694557+0000", 2023-06-03T20:11:34.053 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 7.9659999999999996e-06 2023-06-03T20:11:34.053 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:11:34.053 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:11:34.054 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "dispatched", 2023-06-03T20:11:34.054 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:09:09.694569+0000", 2023-06-03T20:11:34.054 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 1.2593e-05 2023-06-03T20:11:34.054 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:11:34.054 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:11:34.054 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "queued_for_pg", 2023-06-03T20:11:34.054 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:09:09.694579+0000", 2023-06-03T20:11:34.054 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 9.9010000000000006e-06 2023-06-03T20:11:34.055 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:11:34.055 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:11:34.055 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "reached_pg", 2023-06-03T20:11:34.055 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:09:09.700634+0000", 2023-06-03T20:11:34.055 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.0060549209999999996 2023-06-03T20:11:34.055 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:11:34.055 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:11:34.055 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "started", 2023-06-03T20:11:34.056 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:09:09.700754+0000", 2023-06-03T20:11:34.056 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.000119876 2023-06-03T20:11:34.056 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:11:34.056 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:11:34.056 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "waiting for subops from 2,6", 2023-06-03T20:11:34.056 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:09:09.700866+0000", 2023-06-03T20:11:34.056 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.000112404 2023-06-03T20:11:34.056 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:11:34.057 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:11:34.057 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:11:34.057 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:09:09.715866+0000", 2023-06-03T20:11:34.057 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.014999357 2023-06-03T20:11:34.057 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:11:34.057 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:11:34.057 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "op_commit", 2023-06-03T20:11:34.057 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:09:09.720088+0000", 2023-06-03T20:11:34.058 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.0042226570000000003 2023-06-03T20:11:34.058 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:11:34.058 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:11:34.058 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:11:34.058 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:09:09.855481+0000", 2023-06-03T20:11:34.058 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.13539256299999999 2023-06-03T20:11:34.058 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:11:34.059 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:11:34.059 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "commit_sent", 2023-06-03T20:11:34.059 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:09:09.855628+0000", 2023-06-03T20:11:34.059 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.00014698599999999999 2023-06-03T20:11:34.059 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:11:34.059 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:11:34.059 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "done", 2023-06-03T20:11:34.059 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:09:09.855778+0000", 2023-06-03T20:11:34.060 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.000150521 2023-06-03T20:11:34.060 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:11:34.060 INFO:teuthology.orchestra.run.smithi164.stdout: ] 2023-06-03T20:11:34.060 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:11:34.060 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:11:34.060 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:11:34.060 INFO:teuthology.orchestra.run.smithi164.stdout: "description": "osd_op(client.36009.0:159791 10.e 10:78c60ebd:::benchmark_data_smithi089_140876_object54656:head [delete] snapc 0=[] ondisk+write+known_if_redirected e176)", 2023-06-03T20:11:34.060 INFO:teuthology.orchestra.run.smithi164.stdout: "initiated_at": "2023-06-03T20:09:09.696051+0000", 2023-06-03T20:11:34.061 INFO:teuthology.orchestra.run.smithi164.stdout: "age": 144.212961385, 2023-06-03T20:11:34.061 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.159919477, 2023-06-03T20:11:34.061 INFO:teuthology.orchestra.run.smithi164.stdout: "type_data": { 2023-06-03T20:11:34.061 INFO:teuthology.orchestra.run.smithi164.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-03T20:11:34.061 INFO:teuthology.orchestra.run.smithi164.stdout: "client_info": { 2023-06-03T20:11:34.061 INFO:teuthology.orchestra.run.smithi164.stdout: "client": "client.36009", 2023-06-03T20:11:34.061 INFO:teuthology.orchestra.run.smithi164.stdout: "client_addr": "172.21.15.89:0/2550116169", 2023-06-03T20:11:34.061 INFO:teuthology.orchestra.run.smithi164.stdout: "tid": 159791 2023-06-03T20:11:34.062 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:11:34.062 INFO:teuthology.orchestra.run.smithi164.stdout: "events": [ 2023-06-03T20:11:34.062 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:11:34.062 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "initiated", 2023-06-03T20:11:34.062 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:09:09.696051+0000", 2023-06-03T20:11:34.062 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0 2023-06-03T20:11:34.062 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:11:34.063 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:11:34.063 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "throttled", 2023-06-03T20:11:34.063 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:09:09.696051+0000", 2023-06-03T20:11:34.063 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0 2023-06-03T20:11:34.063 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:11:34.063 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:11:34.063 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "header_read", 2023-06-03T20:11:34.063 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:09:09.696050+0000", 2023-06-03T20:11:34.064 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 4294967295.999999 2023-06-03T20:11:34.064 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:11:34.064 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:11:34.064 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "all_read", 2023-06-03T20:11:34.064 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:09:09.696055+0000", 2023-06-03T20:11:34.064 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 4.8930000000000004e-06 2023-06-03T20:11:34.064 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:11:34.065 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:11:34.065 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "dispatched", 2023-06-03T20:11:34.065 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:09:09.696063+0000", 2023-06-03T20:11:34.065 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 8.3329999999999997e-06 2023-06-03T20:11:34.065 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:11:34.065 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:11:34.065 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "queued_for_pg", 2023-06-03T20:11:34.065 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:09:09.696068+0000", 2023-06-03T20:11:34.066 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 4.9060000000000001e-06 2023-06-03T20:11:34.066 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:11:34.066 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:11:34.066 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "reached_pg", 2023-06-03T20:11:34.066 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:09:09.705952+0000", 2023-06-03T20:11:34.066 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.0098832580000000007 2023-06-03T20:11:34.066 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:11:34.066 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:11:34.067 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "started", 2023-06-03T20:11:34.067 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:09:09.706139+0000", 2023-06-03T20:11:34.067 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.00018730600000000001 2023-06-03T20:11:34.067 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:11:34.067 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:11:34.067 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "waiting for subops from 2,6", 2023-06-03T20:11:34.067 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:09:09.706799+0000", 2023-06-03T20:11:34.067 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.00066024599999999997 2023-06-03T20:11:34.068 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:11:34.068 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:11:34.068 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "op_commit", 2023-06-03T20:11:34.068 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:09:09.720093+0000", 2023-06-03T20:11:34.068 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.013293581 2023-06-03T20:11:34.068 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:11:34.068 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:11:34.068 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:11:34.069 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:09:09.738471+0000", 2023-06-03T20:11:34.069 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.018378657 2023-06-03T20:11:34.069 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:11:34.069 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:11:34.069 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:11:34.069 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:09:09.855875+0000", 2023-06-03T20:11:34.069 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.11740366000000001 2023-06-03T20:11:34.069 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:11:34.070 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:11:34.070 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "commit_sent", 2023-06-03T20:11:34.070 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:09:09.855927+0000", 2023-06-03T20:11:34.070 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 5.1727000000000001e-05 2023-06-03T20:11:34.070 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:11:34.070 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:11:34.070 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "done", 2023-06-03T20:11:34.070 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:09:09.855970+0000", 2023-06-03T20:11:34.071 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 4.3673e-05 2023-06-03T20:11:34.071 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:11:34.071 INFO:teuthology.orchestra.run.smithi164.stdout: ] 2023-06-03T20:11:34.071 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:11:34.071 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:11:34.071 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:11:34.071 INFO:teuthology.orchestra.run.smithi164.stdout: "description": "osd_op(client.36009.0:159796 10.e 10:7577d358:::benchmark_data_smithi089_140876_object54661:head [delete] snapc 0=[] ondisk+write+known_if_redirected e176)", 2023-06-03T20:11:34.072 INFO:teuthology.orchestra.run.smithi164.stdout: "initiated_at": "2023-06-03T20:09:09.708329+0000", 2023-06-03T20:11:34.072 INFO:teuthology.orchestra.run.smithi164.stdout: "age": 144.20068317299999, 2023-06-03T20:11:34.072 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.14796763499999999, 2023-06-03T20:11:34.072 INFO:teuthology.orchestra.run.smithi164.stdout: "type_data": { 2023-06-03T20:11:34.072 INFO:teuthology.orchestra.run.smithi164.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-03T20:11:34.072 INFO:teuthology.orchestra.run.smithi164.stdout: "client_info": { 2023-06-03T20:11:34.072 INFO:teuthology.orchestra.run.smithi164.stdout: "client": "client.36009", 2023-06-03T20:11:34.072 INFO:teuthology.orchestra.run.smithi164.stdout: "client_addr": "172.21.15.89:0/2550116169", 2023-06-03T20:11:34.073 INFO:teuthology.orchestra.run.smithi164.stdout: "tid": 159796 2023-06-03T20:11:34.073 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:11:34.073 INFO:teuthology.orchestra.run.smithi164.stdout: "events": [ 2023-06-03T20:11:34.073 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:11:34.073 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "initiated", 2023-06-03T20:11:34.073 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:09:09.708329+0000", 2023-06-03T20:11:34.073 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0 2023-06-03T20:11:34.074 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:11:34.074 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:11:34.074 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "throttled", 2023-06-03T20:11:34.074 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:09:09.708329+0000", 2023-06-03T20:11:34.074 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0 2023-06-03T20:11:34.074 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:11:34.074 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:11:34.074 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "header_read", 2023-06-03T20:11:34.075 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:09:09.708328+0000", 2023-06-03T20:11:34.075 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 4294967295.9999986 2023-06-03T20:11:34.075 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:11:34.075 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:11:34.075 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "all_read", 2023-06-03T20:11:34.075 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:09:09.708333+0000", 2023-06-03T20:11:34.075 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 5.2900000000000002e-06 2023-06-03T20:11:34.075 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:11:34.076 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:11:34.076 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "dispatched", 2023-06-03T20:11:34.076 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:09:09.708342+0000", 2023-06-03T20:11:34.076 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 8.7329999999999992e-06 2023-06-03T20:11:34.076 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:11:34.076 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:11:34.076 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "queued_for_pg", 2023-06-03T20:11:34.076 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:09:09.708347+0000", 2023-06-03T20:11:34.077 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 5.6679999999999998e-06 2023-06-03T20:11:34.077 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:11:34.077 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:11:34.077 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "reached_pg", 2023-06-03T20:11:34.077 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:09:09.711382+0000", 2023-06-03T20:11:34.077 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.0030344899999999999 2023-06-03T20:11:34.077 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:11:34.078 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:11:34.078 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "started", 2023-06-03T20:11:34.078 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:09:09.711512+0000", 2023-06-03T20:11:34.078 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.00013017000000000001 2023-06-03T20:11:34.078 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:11:34.078 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:11:34.078 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "waiting for subops from 2,6", 2023-06-03T20:11:34.078 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:09:09.711634+0000", 2023-06-03T20:11:34.079 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.00012225199999999999 2023-06-03T20:11:34.079 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:11:34.079 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:11:34.079 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "op_commit", 2023-06-03T20:11:34.079 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:09:09.724302+0000", 2023-06-03T20:11:34.079 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.012667629 2023-06-03T20:11:34.079 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:11:34.079 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:11:34.080 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:11:34.080 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:09:09.738513+0000", 2023-06-03T20:11:34.080 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.014211082 2023-06-03T20:11:34.080 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:11:34.080 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:11:34.080 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:11:34.080 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:09:09.856089+0000", 2023-06-03T20:11:34.080 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.117576427 2023-06-03T20:11:34.081 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:11:34.081 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:11:34.081 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "commit_sent", 2023-06-03T20:11:34.081 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:09:09.856199+0000", 2023-06-03T20:11:34.081 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.000109036 2023-06-03T20:11:34.081 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:11:34.081 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:11:34.081 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "done", 2023-06-03T20:11:34.082 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:09:09.856297+0000", 2023-06-03T20:11:34.082 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 9.8358999999999998e-05 2023-06-03T20:11:34.082 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:11:34.082 INFO:teuthology.orchestra.run.smithi164.stdout: ] 2023-06-03T20:11:34.082 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:11:34.082 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:11:34.082 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:11:34.082 INFO:teuthology.orchestra.run.smithi164.stdout: "description": "osd_op(client.36009.0:159797 10.e 10:7732dec5:::benchmark_data_smithi089_140876_object54662:head [delete] snapc 0=[] ondisk+write+known_if_redirected e176)", 2023-06-03T20:11:34.083 INFO:teuthology.orchestra.run.smithi164.stdout: "initiated_at": "2023-06-03T20:09:09.712584+0000", 2023-06-03T20:11:34.083 INFO:teuthology.orchestra.run.smithi164.stdout: "age": 144.19642817100001, 2023-06-03T20:11:34.083 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.150313367, 2023-06-03T20:11:34.083 INFO:teuthology.orchestra.run.smithi164.stdout: "type_data": { 2023-06-03T20:11:34.083 INFO:teuthology.orchestra.run.smithi164.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-03T20:11:34.083 INFO:teuthology.orchestra.run.smithi164.stdout: "client_info": { 2023-06-03T20:11:34.083 INFO:teuthology.orchestra.run.smithi164.stdout: "client": "client.36009", 2023-06-03T20:11:34.084 INFO:teuthology.orchestra.run.smithi164.stdout: "client_addr": "172.21.15.89:0/2550116169", 2023-06-03T20:11:34.084 INFO:teuthology.orchestra.run.smithi164.stdout: "tid": 159797 2023-06-03T20:11:34.084 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:11:34.084 INFO:teuthology.orchestra.run.smithi164.stdout: "events": [ 2023-06-03T20:11:34.084 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:11:34.084 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "initiated", 2023-06-03T20:11:34.084 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:09:09.712584+0000", 2023-06-03T20:11:34.084 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0 2023-06-03T20:11:34.085 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:11:34.085 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:11:34.085 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "throttled", 2023-06-03T20:11:34.085 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:09:09.712584+0000", 2023-06-03T20:11:34.085 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0 2023-06-03T20:11:34.085 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:11:34.085 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:11:34.085 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "header_read", 2023-06-03T20:11:34.086 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:09:09.712583+0000", 2023-06-03T20:11:34.086 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 4294967295.9999986 2023-06-03T20:11:34.086 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:11:34.086 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:11:34.086 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "all_read", 2023-06-03T20:11:34.086 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:09:09.712590+0000", 2023-06-03T20:11:34.086 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 7.3499999999999999e-06 2023-06-03T20:11:34.087 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:11:34.087 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:11:34.087 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "dispatched", 2023-06-03T20:11:34.087 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:09:09.712604+0000", 2023-06-03T20:11:34.087 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 1.4242e-05 2023-06-03T20:11:34.087 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:11:34.087 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:11:34.087 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "queued_for_pg", 2023-06-03T20:11:34.088 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:09:09.712613+0000", 2023-06-03T20:11:34.088 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 8.8950000000000004e-06 2023-06-03T20:11:34.088 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:11:34.088 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:11:34.088 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "reached_pg", 2023-06-03T20:11:34.088 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:09:09.715892+0000", 2023-06-03T20:11:34.088 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.0032783859999999999 2023-06-03T20:11:34.088 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:11:34.089 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:11:34.089 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "started", 2023-06-03T20:11:34.089 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:09:09.716032+0000", 2023-06-03T20:11:34.089 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.00014034299999999999 2023-06-03T20:11:34.089 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:11:34.089 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:11:34.089 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "waiting for subops from 2,6", 2023-06-03T20:11:34.090 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:09:09.716150+0000", 2023-06-03T20:11:34.090 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.000117831 2023-06-03T20:11:34.090 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:11:34.090 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:11:34.090 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "op_commit", 2023-06-03T20:11:34.090 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:09:09.728002+0000", 2023-06-03T20:11:34.090 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.011851893 2023-06-03T20:11:34.090 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:11:34.091 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:11:34.091 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:11:34.091 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:09:09.738563+0000", 2023-06-03T20:11:34.091 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.010560936 2023-06-03T20:11:34.091 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:11:34.091 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:11:34.091 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:11:34.091 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:09:09.862773+0000", 2023-06-03T20:11:34.092 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.124210537 2023-06-03T20:11:34.092 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:11:34.092 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:11:34.092 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "commit_sent", 2023-06-03T20:11:34.092 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:09:09.862832+0000", 2023-06-03T20:11:34.092 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 5.9097e-05 2023-06-03T20:11:34.092 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:11:34.093 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:11:34.093 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "done", 2023-06-03T20:11:34.093 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:09:09.862898+0000", 2023-06-03T20:11:34.093 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 6.5230999999999999e-05 2023-06-03T20:11:34.093 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:11:34.093 INFO:teuthology.orchestra.run.smithi164.stdout: ] 2023-06-03T20:11:34.093 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:11:34.093 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:11:34.094 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:11:34.094 INFO:teuthology.orchestra.run.smithi164.stdout: "description": "osd_op(client.36009.0:159804 10.e 10:7d98edac:::benchmark_data_smithi089_140876_object54669:head [delete] snapc 0=[] ondisk+write+known_if_redirected e176)", 2023-06-03T20:11:34.094 INFO:teuthology.orchestra.run.smithi164.stdout: "initiated_at": "2023-06-03T20:09:09.718516+0000", 2023-06-03T20:11:34.094 INFO:teuthology.orchestra.run.smithi164.stdout: "age": 144.19049629099999, 2023-06-03T20:11:34.094 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.14450540100000001, 2023-06-03T20:11:34.094 INFO:teuthology.orchestra.run.smithi164.stdout: "type_data": { 2023-06-03T20:11:34.095 INFO:teuthology.orchestra.run.smithi164.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-03T20:11:34.095 INFO:teuthology.orchestra.run.smithi164.stdout: "client_info": { 2023-06-03T20:11:34.095 INFO:teuthology.orchestra.run.smithi164.stdout: "client": "client.36009", 2023-06-03T20:11:34.095 INFO:teuthology.orchestra.run.smithi164.stdout: "client_addr": "172.21.15.89:0/2550116169", 2023-06-03T20:11:34.095 INFO:teuthology.orchestra.run.smithi164.stdout: "tid": 159804 2023-06-03T20:11:34.095 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:11:34.095 INFO:teuthology.orchestra.run.smithi164.stdout: "events": [ 2023-06-03T20:11:34.095 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:11:34.096 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "initiated", 2023-06-03T20:11:34.096 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:09:09.718516+0000", 2023-06-03T20:11:34.096 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0 2023-06-03T20:11:34.096 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:11:34.096 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:11:34.096 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "throttled", 2023-06-03T20:11:34.096 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:09:09.718516+0000", 2023-06-03T20:11:34.096 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0 2023-06-03T20:11:34.097 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:11:34.097 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:11:34.097 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "header_read", 2023-06-03T20:11:34.097 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:09:09.718515+0000", 2023-06-03T20:11:34.097 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 4294967295.999999 2023-06-03T20:11:34.097 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:11:34.097 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:11:34.097 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "all_read", 2023-06-03T20:11:34.098 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:09:09.718520+0000", 2023-06-03T20:11:34.098 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 4.6990000000000002e-06 2023-06-03T20:11:34.098 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:11:34.098 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:11:34.098 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "dispatched", 2023-06-03T20:11:34.098 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:09:09.718529+0000", 2023-06-03T20:11:34.098 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 9.0960000000000007e-06 2023-06-03T20:11:34.099 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:11:34.099 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:11:34.099 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "queued_for_pg", 2023-06-03T20:11:34.099 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:09:09.718536+0000", 2023-06-03T20:11:34.099 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 6.6939999999999998e-06 2023-06-03T20:11:34.099 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:11:34.099 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:11:34.099 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "reached_pg", 2023-06-03T20:11:34.100 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:09:09.720119+0000", 2023-06-03T20:11:34.100 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.001582918 2023-06-03T20:11:34.100 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:11:34.100 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:11:34.100 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "started", 2023-06-03T20:11:34.100 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:09:09.720251+0000", 2023-06-03T20:11:34.100 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.00013233400000000001 2023-06-03T20:11:34.100 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:11:34.101 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:11:34.101 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "waiting for subops from 2,6", 2023-06-03T20:11:34.101 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:09:09.720373+0000", 2023-06-03T20:11:34.101 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.000122024 2023-06-03T20:11:34.101 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:11:34.101 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:11:34.101 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "op_commit", 2023-06-03T20:11:34.102 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:09:09.728006+0000", 2023-06-03T20:11:34.102 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.0076330310000000002 2023-06-03T20:11:34.102 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:11:34.102 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:11:34.102 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:11:34.102 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:09:09.738588+0000", 2023-06-03T20:11:34.102 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.010582322999999999 2023-06-03T20:11:34.102 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:11:34.103 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:11:34.103 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:11:34.103 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:09:09.862946+0000", 2023-06-03T20:11:34.103 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.12435774199999999 2023-06-03T20:11:34.103 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:11:34.103 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:11:34.103 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "commit_sent", 2023-06-03T20:11:34.103 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:09:09.862986+0000", 2023-06-03T20:11:34.104 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 3.9728999999999999e-05 2023-06-03T20:11:34.104 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:11:34.104 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:11:34.104 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "done", 2023-06-03T20:11:34.104 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:09:09.863022+0000", 2023-06-03T20:11:34.104 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 3.5543999999999997e-05 2023-06-03T20:11:34.104 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:11:34.104 INFO:teuthology.orchestra.run.smithi164.stdout: ] 2023-06-03T20:11:34.105 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:11:34.105 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:11:34.105 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:11:34.105 INFO:teuthology.orchestra.run.smithi164.stdout: "description": "osd_op(client.36009.0:159965 10.2 10:4c8f1693:::benchmark_data_smithi089_140876_object54830:head [delete] snapc 0=[] ondisk+write+known_if_redirected e176)", 2023-06-03T20:11:34.105 INFO:teuthology.orchestra.run.smithi164.stdout: "initiated_at": "2023-06-03T20:09:10.052419+0000", 2023-06-03T20:11:34.105 INFO:teuthology.orchestra.run.smithi164.stdout: "age": 143.85659389, 2023-06-03T20:11:34.105 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.27038748200000001, 2023-06-03T20:11:34.105 INFO:teuthology.orchestra.run.smithi164.stdout: "type_data": { 2023-06-03T20:11:34.106 INFO:teuthology.orchestra.run.smithi164.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-03T20:11:34.106 INFO:teuthology.orchestra.run.smithi164.stdout: "client_info": { 2023-06-03T20:11:34.106 INFO:teuthology.orchestra.run.smithi164.stdout: "client": "client.36009", 2023-06-03T20:11:34.106 INFO:teuthology.orchestra.run.smithi164.stdout: "client_addr": "172.21.15.89:0/2550116169", 2023-06-03T20:11:34.106 INFO:teuthology.orchestra.run.smithi164.stdout: "tid": 159965 2023-06-03T20:11:34.106 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:11:34.106 INFO:teuthology.orchestra.run.smithi164.stdout: "events": [ 2023-06-03T20:11:34.106 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:11:34.107 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "initiated", 2023-06-03T20:11:34.107 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:09:10.052419+0000", 2023-06-03T20:11:34.107 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0 2023-06-03T20:11:34.107 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:11:34.107 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:11:34.107 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "throttled", 2023-06-03T20:11:34.107 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:09:10.052419+0000", 2023-06-03T20:11:34.107 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0 2023-06-03T20:11:34.108 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:11:34.108 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:11:34.108 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "header_read", 2023-06-03T20:11:34.108 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:09:10.052417+0000", 2023-06-03T20:11:34.108 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 4294967295.9999986 2023-06-03T20:11:34.108 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:11:34.108 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:11:34.108 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "all_read", 2023-06-03T20:11:34.109 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:09:10.052429+0000", 2023-06-03T20:11:34.109 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 1.2173e-05 2023-06-03T20:11:34.109 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:11:34.109 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:11:34.109 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "dispatched", 2023-06-03T20:11:34.109 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:09:10.052447+0000", 2023-06-03T20:11:34.109 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 1.7615999999999999e-05 2023-06-03T20:11:34.109 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:11:34.110 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:11:34.110 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "queued_for_pg", 2023-06-03T20:11:34.110 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:09:10.052457+0000", 2023-06-03T20:11:34.110 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 9.9529999999999994e-06 2023-06-03T20:11:34.110 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:11:34.110 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:11:34.110 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "reached_pg", 2023-06-03T20:11:34.110 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:09:10.052890+0000", 2023-06-03T20:11:34.111 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.000433588 2023-06-03T20:11:34.111 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:11:34.111 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:11:34.111 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "started", 2023-06-03T20:11:34.111 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:09:10.053063+0000", 2023-06-03T20:11:34.111 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.00017225000000000001 2023-06-03T20:11:34.111 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:11:34.111 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:11:34.112 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "waiting for subops from 0,3", 2023-06-03T20:11:34.112 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:09:10.053199+0000", 2023-06-03T20:11:34.112 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.000136501 2023-06-03T20:11:34.112 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:11:34.112 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:11:34.112 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "op_commit", 2023-06-03T20:11:34.112 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:09:10.061787+0000", 2023-06-03T20:11:34.113 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.0085878840000000005 2023-06-03T20:11:34.113 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:11:34.113 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:11:34.113 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:11:34.113 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:09:10.070150+0000", 2023-06-03T20:11:34.113 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.0083634869999999993 2023-06-03T20:11:34.113 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:11:34.113 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:11:34.114 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:11:34.114 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:09:10.320979+0000", 2023-06-03T20:11:34.114 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.25082871699999998 2023-06-03T20:11:34.114 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:11:34.114 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:11:34.114 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "commit_sent", 2023-06-03T20:11:34.114 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:09:10.321038+0000", 2023-06-03T20:11:34.114 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 5.8777999999999999e-05 2023-06-03T20:11:34.115 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:11:34.115 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:11:34.115 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "done", 2023-06-03T20:11:34.115 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:09:10.322806+0000", 2023-06-03T20:11:34.115 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.001768052 2023-06-03T20:11:34.115 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:11:34.115 INFO:teuthology.orchestra.run.smithi164.stdout: ] 2023-06-03T20:11:34.115 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:11:34.116 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:11:34.116 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:11:34.116 INFO:teuthology.orchestra.run.smithi164.stdout: "description": "osd_op(client.36009.0:159972 10.1 10:89d299ee:::benchmark_data_smithi089_140876_object54837:head [delete] snapc 0=[] ondisk+write+known_if_redirected e176)", 2023-06-03T20:11:34.116 INFO:teuthology.orchestra.run.smithi164.stdout: "initiated_at": "2023-06-03T20:09:10.057579+0000", 2023-06-03T20:11:34.116 INFO:teuthology.orchestra.run.smithi164.stdout: "age": 143.85143381500001, 2023-06-03T20:11:34.116 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.24133273299999999, 2023-06-03T20:11:34.116 INFO:teuthology.orchestra.run.smithi164.stdout: "type_data": { 2023-06-03T20:11:34.116 INFO:teuthology.orchestra.run.smithi164.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-03T20:11:34.117 INFO:teuthology.orchestra.run.smithi164.stdout: "client_info": { 2023-06-03T20:11:34.117 INFO:teuthology.orchestra.run.smithi164.stdout: "client": "client.36009", 2023-06-03T20:11:34.117 INFO:teuthology.orchestra.run.smithi164.stdout: "client_addr": "172.21.15.89:0/2550116169", 2023-06-03T20:11:34.117 INFO:teuthology.orchestra.run.smithi164.stdout: "tid": 159972 2023-06-03T20:11:34.117 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:11:34.117 INFO:teuthology.orchestra.run.smithi164.stdout: "events": [ 2023-06-03T20:11:34.117 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:11:34.117 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "initiated", 2023-06-03T20:11:34.118 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:09:10.057579+0000", 2023-06-03T20:11:34.118 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0 2023-06-03T20:11:34.118 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:11:34.118 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:11:34.118 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "throttled", 2023-06-03T20:11:34.118 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:09:10.057579+0000", 2023-06-03T20:11:34.118 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0 2023-06-03T20:11:34.118 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:11:34.119 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:11:34.119 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "header_read", 2023-06-03T20:11:34.119 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:09:10.057577+0000", 2023-06-03T20:11:34.119 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 4294967295.9999981 2023-06-03T20:11:34.119 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:11:34.119 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:11:34.119 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "all_read", 2023-06-03T20:11:34.119 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:09:10.057589+0000", 2023-06-03T20:11:34.120 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 1.226e-05 2023-06-03T20:11:34.120 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:11:34.120 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:11:34.120 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "dispatched", 2023-06-03T20:11:34.120 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:09:10.057609+0000", 2023-06-03T20:11:34.120 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 1.9735999999999999e-05 2023-06-03T20:11:34.120 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:11:34.120 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:11:34.121 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "queued_for_pg", 2023-06-03T20:11:34.121 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:09:10.057622+0000", 2023-06-03T20:11:34.121 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 1.3064999999999999e-05 2023-06-03T20:11:34.121 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:11:34.121 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:11:34.121 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "reached_pg", 2023-06-03T20:11:34.121 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:09:10.057984+0000", 2023-06-03T20:11:34.121 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.00036193199999999998 2023-06-03T20:11:34.122 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:11:34.122 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:11:34.122 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "started", 2023-06-03T20:11:34.122 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:09:10.058149+0000", 2023-06-03T20:11:34.122 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.00016499799999999999 2023-06-03T20:11:34.122 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:11:34.122 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:11:34.122 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "waiting for subops from 0,7", 2023-06-03T20:11:34.123 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:09:10.058300+0000", 2023-06-03T20:11:34.123 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.00015131399999999999 2023-06-03T20:11:34.123 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:11:34.123 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:11:34.123 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "op_commit", 2023-06-03T20:11:34.123 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:09:10.072664+0000", 2023-06-03T20:11:34.123 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.014364043999999999 2023-06-03T20:11:34.123 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:11:34.124 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:11:34.124 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:11:34.124 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:09:10.078750+0000", 2023-06-03T20:11:34.124 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.0060864580000000003 2023-06-03T20:11:34.124 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:11:34.124 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:11:34.124 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:11:34.125 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:09:10.298839+0000", 2023-06-03T20:11:34.125 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.22008897799999999 2023-06-03T20:11:34.125 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:11:34.125 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:11:34.125 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "commit_sent", 2023-06-03T20:11:34.125 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:09:10.298879+0000", 2023-06-03T20:11:34.125 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 3.9304999999999997e-05 2023-06-03T20:11:34.125 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:11:34.126 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:11:34.126 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "done", 2023-06-03T20:11:34.126 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:09:10.298911+0000", 2023-06-03T20:11:34.126 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 3.2712000000000001e-05 2023-06-03T20:11:34.126 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:11:34.126 INFO:teuthology.orchestra.run.smithi164.stdout: ] 2023-06-03T20:11:34.126 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:11:34.126 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:11:34.127 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:11:34.127 INFO:teuthology.orchestra.run.smithi164.stdout: "description": "osd_op(client.36009.0:159975 10.1 10:851a60fd:::benchmark_data_smithi089_140876_object54840:head [delete] snapc 0=[] ondisk+write+known_if_redirected e176)", 2023-06-03T20:11:34.127 INFO:teuthology.orchestra.run.smithi164.stdout: "initiated_at": "2023-06-03T20:09:10.067850+0000", 2023-06-03T20:11:34.127 INFO:teuthology.orchestra.run.smithi164.stdout: "age": 143.84116271900001, 2023-06-03T20:11:34.127 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.232033449, 2023-06-03T20:11:34.127 INFO:teuthology.orchestra.run.smithi164.stdout: "type_data": { 2023-06-03T20:11:34.127 INFO:teuthology.orchestra.run.smithi164.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-03T20:11:34.128 INFO:teuthology.orchestra.run.smithi164.stdout: "client_info": { 2023-06-03T20:11:34.128 INFO:teuthology.orchestra.run.smithi164.stdout: "client": "client.36009", 2023-06-03T20:11:34.128 INFO:teuthology.orchestra.run.smithi164.stdout: "client_addr": "172.21.15.89:0/2550116169", 2023-06-03T20:11:34.128 INFO:teuthology.orchestra.run.smithi164.stdout: "tid": 159975 2023-06-03T20:11:34.128 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:11:34.128 INFO:teuthology.orchestra.run.smithi164.stdout: "events": [ 2023-06-03T20:11:34.128 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:11:34.129 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "initiated", 2023-06-03T20:11:34.129 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:09:10.067850+0000", 2023-06-03T20:11:34.129 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0 2023-06-03T20:11:34.129 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:11:34.129 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:11:34.129 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "throttled", 2023-06-03T20:11:34.129 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:09:10.067850+0000", 2023-06-03T20:11:34.129 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0 2023-06-03T20:11:34.130 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:11:34.130 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:11:34.130 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "header_read", 2023-06-03T20:11:34.130 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:09:10.067846+0000", 2023-06-03T20:11:34.130 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 4294967295.9999967 2023-06-03T20:11:34.130 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:11:34.130 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:11:34.130 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "all_read", 2023-06-03T20:11:34.131 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:09:10.067864+0000", 2023-06-03T20:11:34.131 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 1.8002000000000001e-05 2023-06-03T20:11:34.131 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:11:34.131 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:11:34.131 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "dispatched", 2023-06-03T20:11:34.131 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:09:10.067900+0000", 2023-06-03T20:11:34.131 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 3.5429999999999998e-05 2023-06-03T20:11:34.131 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:11:34.132 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:11:34.132 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "queued_for_pg", 2023-06-03T20:11:34.132 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:09:10.067917+0000", 2023-06-03T20:11:34.132 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 1.6733000000000001e-05 2023-06-03T20:11:34.132 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:11:34.132 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:11:34.132 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "reached_pg", 2023-06-03T20:11:34.132 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:09:10.068392+0000", 2023-06-03T20:11:34.133 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.00047551199999999998 2023-06-03T20:11:34.133 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:11:34.133 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:11:34.133 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "started", 2023-06-03T20:11:34.133 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:09:10.068526+0000", 2023-06-03T20:11:34.133 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.000133497 2023-06-03T20:11:34.133 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:11:34.133 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:11:34.134 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "waiting for subops from 0,7", 2023-06-03T20:11:34.134 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:09:10.068674+0000", 2023-06-03T20:11:34.134 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.00014797700000000001 2023-06-03T20:11:34.134 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:11:34.134 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:11:34.134 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "op_commit", 2023-06-03T20:11:34.134 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:09:10.083568+0000", 2023-06-03T20:11:34.135 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.014894579 2023-06-03T20:11:34.135 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:11:34.135 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:11:34.135 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:11:34.135 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:09:10.085847+0000", 2023-06-03T20:11:34.135 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.002279152 2023-06-03T20:11:34.135 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:11:34.135 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:11:34.136 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:11:34.136 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:09:10.299806+0000", 2023-06-03T20:11:34.136 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.21395848100000001 2023-06-03T20:11:34.136 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:11:34.136 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:11:34.136 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "commit_sent", 2023-06-03T20:11:34.136 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:09:10.299846+0000", 2023-06-03T20:11:34.136 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 3.9944000000000002e-05 2023-06-03T20:11:34.137 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:11:34.137 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:11:34.137 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "done", 2023-06-03T20:11:34.137 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:09:10.299883+0000", 2023-06-03T20:11:34.137 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 3.7435999999999999e-05 2023-06-03T20:11:34.137 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:11:34.137 INFO:teuthology.orchestra.run.smithi164.stdout: ] 2023-06-03T20:11:34.137 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:11:34.138 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:11:34.138 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:11:34.138 INFO:teuthology.orchestra.run.smithi164.stdout: "description": "osd_op(client.36009.0:159984 10.2 10:4b99f02c:::benchmark_data_smithi089_140876_object54849:head [delete] snapc 0=[] ondisk+write+known_if_redirected e176)", 2023-06-03T20:11:34.138 INFO:teuthology.orchestra.run.smithi164.stdout: "initiated_at": "2023-06-03T20:09:10.086974+0000", 2023-06-03T20:11:34.138 INFO:teuthology.orchestra.run.smithi164.stdout: "age": 143.82203837899999, 2023-06-03T20:11:34.138 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.237743435, 2023-06-03T20:11:34.138 INFO:teuthology.orchestra.run.smithi164.stdout: "type_data": { 2023-06-03T20:11:34.138 INFO:teuthology.orchestra.run.smithi164.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-03T20:11:34.139 INFO:teuthology.orchestra.run.smithi164.stdout: "client_info": { 2023-06-03T20:11:34.139 INFO:teuthology.orchestra.run.smithi164.stdout: "client": "client.36009", 2023-06-03T20:11:34.139 INFO:teuthology.orchestra.run.smithi164.stdout: "client_addr": "172.21.15.89:0/2550116169", 2023-06-03T20:11:34.139 INFO:teuthology.orchestra.run.smithi164.stdout: "tid": 159984 2023-06-03T20:11:34.139 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:11:34.139 INFO:teuthology.orchestra.run.smithi164.stdout: "events": [ 2023-06-03T20:11:34.139 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:11:34.139 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "initiated", 2023-06-03T20:11:34.140 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:09:10.086974+0000", 2023-06-03T20:11:34.140 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0 2023-06-03T20:11:34.140 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:11:34.140 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:11:34.140 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "throttled", 2023-06-03T20:11:34.140 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:09:10.086974+0000", 2023-06-03T20:11:34.140 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0 2023-06-03T20:11:34.140 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:11:34.141 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:11:34.141 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "header_read", 2023-06-03T20:11:34.141 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:09:10.086972+0000", 2023-06-03T20:11:34.141 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 4294967295.9999981 2023-06-03T20:11:34.141 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:11:34.141 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:11:34.141 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "all_read", 2023-06-03T20:11:34.141 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:09:10.086981+0000", 2023-06-03T20:11:34.142 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 8.8869999999999998e-06 2023-06-03T20:11:34.142 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:11:34.142 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:11:34.142 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "dispatched", 2023-06-03T20:11:34.142 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:09:10.087001+0000", 2023-06-03T20:11:34.142 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 2.0244000000000001e-05 2023-06-03T20:11:34.142 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:11:34.142 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:11:34.143 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "queued_for_pg", 2023-06-03T20:11:34.143 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:09:10.087012+0000", 2023-06-03T20:11:34.143 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 1.0795000000000001e-05 2023-06-03T20:11:34.143 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:11:34.143 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:11:34.143 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "reached_pg", 2023-06-03T20:11:34.143 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:09:10.087080+0000", 2023-06-03T20:11:34.143 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 6.7892999999999995e-05 2023-06-03T20:11:34.144 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:11:34.144 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:11:34.144 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "started", 2023-06-03T20:11:34.144 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:09:10.087282+0000", 2023-06-03T20:11:34.144 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.000201456 2023-06-03T20:11:34.144 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:11:34.144 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:11:34.144 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "waiting for subops from 0,3", 2023-06-03T20:11:34.145 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:09:10.087476+0000", 2023-06-03T20:11:34.145 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.00019437499999999999 2023-06-03T20:11:34.145 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:11:34.145 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:11:34.145 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:11:34.145 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:09:10.096908+0000", 2023-06-03T20:11:34.145 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.009432569 2023-06-03T20:11:34.146 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:11:34.146 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:11:34.146 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "op_commit", 2023-06-03T20:11:34.146 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:09:10.097212+0000", 2023-06-03T20:11:34.146 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.000303944 2023-06-03T20:11:34.146 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:11:34.146 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:11:34.146 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:11:34.147 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:09:10.323199+0000", 2023-06-03T20:11:34.147 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.22598694999999999 2023-06-03T20:11:34.147 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:11:34.147 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:11:34.147 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "commit_sent", 2023-06-03T20:11:34.147 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:09:10.323257+0000", 2023-06-03T20:11:34.147 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 5.8023999999999999e-05 2023-06-03T20:11:34.147 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:11:34.148 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:11:34.148 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "done", 2023-06-03T20:11:34.148 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:09:10.324717+0000", 2023-06-03T20:11:34.148 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.001460046 2023-06-03T20:11:34.148 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:11:34.148 INFO:teuthology.orchestra.run.smithi164.stdout: ] 2023-06-03T20:11:34.148 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:11:34.149 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:11:34.149 INFO:teuthology.orchestra.run.smithi164.stdout: ] 2023-06-03T20:11:34.149 INFO:teuthology.orchestra.run.smithi164.stdout:} 2023-06-03T20:11:34.149 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:11:34.150 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:11:34.150 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:11:34.251 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:11:34.291 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:11:34.294 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:11:34.295 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:11:34.395 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:11:34.432 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:11:34.434 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:11:34.435 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:11:34.536 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:11:34.551 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 1 16 2921 2905 181.394 181.562 0.00518722 0.00549559 2023-06-03T20:11:34.572 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:11:34.577 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:11:34.577 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:11:34.678 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:11:34.711 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:11:34.714 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:11:34.715 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:11:34.816 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:11:34.856 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:11:34.858 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:11:34.859 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:11:34.959 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:11:34.996 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:11:34.998 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:11:34.999 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:11:35.100 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:11:35.133 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:11:35.137 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:11:35.137 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:11:35.238 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:11:35.271 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:11:35.275 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:11:35.276 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:11:35.367 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/mon.a/config 2023-06-03T20:11:35.371 DEBUG:teuthology.orchestra.run.smithi164:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph daemon osd.5 dump_ops_in_flight 2023-06-03T20:11:35.377 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:11:35.408 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:11:35.411 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:11:35.412 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:11:35.513 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:11:35.551 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 2 16 5746 5730 178.969 176.562 0.00670365 0.00556745 2023-06-03T20:11:35.553 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:11:35.556 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:11:35.557 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:11:35.658 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:11:35.688 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:11:35.693 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:11:35.694 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:11:35.795 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:11:35.806 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:11:35 smithi164 bash[59462]: cluster 1685823094.6762252 mgr.y (mgr.34104) 1064 : cluster 0 pgmap v1030: 89 pgs: 89 active+clean; 2.1 MiB data, 2.3 GiB used, 624 GiB / 626 GiB avail; 160 KiB/s wr, 2 op/s 2023-06-03T20:11:35.807 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:11:35 smithi164 bash[59462]: audit 1685823095.74815 mon.c (mon.2) 498 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-03T20:11:35.835 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:11:35.838 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:11:35.839 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:11:35.940 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:11:35.980 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:11:35.984 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:11:35.984 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:11:36.028 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:11:35 smithi089 bash[78452]: cluster 1685823094.6762252 mgr.y (mgr.34104) 1064 : cluster 0 pgmap v1030: 89 pgs: 89 active+clean; 2.1 MiB data, 2.3 GiB used, 624 GiB / 626 GiB avail; 160 KiB/s wr, 2 op/s 2023-06-03T20:11:36.028 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:11:35 smithi089 bash[78452]: audit 1685823095.74815 mon.c (mon.2) 498 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-03T20:11:36.029 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:11:35 smithi089 bash[75068]: cluster 1685823094.6762252 mgr.y (mgr.34104) 1064 : cluster 0 pgmap v1030: 89 pgs: 89 active+clean; 2.1 MiB data, 2.3 GiB used, 624 GiB / 626 GiB avail; 160 KiB/s wr, 2 op/s 2023-06-03T20:11:36.029 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:11:35 smithi089 bash[75068]: audit 1685823095.74815 mon.c (mon.2) 498 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-03T20:11:36.086 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:11:36.118 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:11:36.123 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:11:36.124 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:11:36.225 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:11:36.258 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:11:36.261 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:11:36.262 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:11:36.363 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:11:36.374 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 20:11:36 smithi089 bash[71163]: ::ffff:172.21.15.164 - - [03/Jun/2023:20:11:36] "GET /metrics HTTP/1.1" 200 228632 "" "Prometheus/2.33.4" 2023-06-03T20:11:36.394 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:11:36.397 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:11:36.398 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:11:36.499 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:11:36.534 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:11:36.538 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:11:36.538 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:11:36.551 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 3 16 8259 8243 171.664 157.062 0.00514125 0.00581578 2023-06-03T20:11:36.639 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:11:36.676 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:11:36.686 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:11:36.686 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:11:36.787 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:11:36.823 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:11:36.826 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:11:36.827 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:11:36.928 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:11:36.963 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:11:36.965 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:11:36.966 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:11:37.067 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:11:37.090 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:11:37.092 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:11:37.093 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:11:37.194 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:11:37.222 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:11:37.224 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:11:37.225 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:11:37.326 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:11:37.354 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:11:37.357 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:11:37.358 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:11:37.459 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:11:37.492 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:11:37.494 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:11:37.495 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:11:37.551 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 4 16 10397 10381 162.153 133.625 0.00417608 0.00615842 2023-06-03T20:11:37.596 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:11:37.625 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:11:37.627 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:11:37.628 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:11:37.728 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:11:37.756 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:11:37.758 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:11:37.758 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:11:37.775 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:11:37 smithi089 bash[75068]: cluster 1685823096.676683 mgr.y (mgr.34104) 1065 : cluster 0 pgmap v1031: 89 pgs: 89 active+clean; 65 MiB data, 2.5 GiB used, 623 GiB / 626 GiB avail; 6.4 MiB/s wr, 102 op/s 2023-06-03T20:11:37.775 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:11:37 smithi089 bash[75068]: audit 1685823097.1165123 mon.c (mon.2) 499 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T20:11:37.859 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:11:37.892 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:11:37.894 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:11:37.895 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:11:37.996 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:11:38.028 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:11:37 smithi089 bash[78452]: cluster 1685823096.676683 mgr.y (mgr.34104) 1065 : cluster 0 pgmap v1031: 89 pgs: 89 active+clean; 65 MiB data, 2.5 GiB used, 623 GiB / 626 GiB avail; 6.4 MiB/s wr, 102 op/s 2023-06-03T20:11:38.028 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:11:37 smithi089 bash[78452]: audit 1685823097.1165123 mon.c (mon.2) 499 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T20:11:38.030 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:11:38.032 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:11:38.033 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:11:38.083 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:11:37 smithi164 bash[59462]: cluster 1685823096.676683 mgr.y (mgr.34104) 1065 : cluster 0 pgmap v1031: 89 pgs: 89 active+clean; 65 MiB data, 2.5 GiB used, 623 GiB / 626 GiB avail; 6.4 MiB/s wr, 102 op/s 2023-06-03T20:11:38.083 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:11:37 smithi164 bash[59462]: audit 1685823097.1165123 mon.c (mon.2) 499 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T20:11:38.134 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:11:38.164 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:11:38.166 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:11:38.166 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:11:38.267 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:11:38.298 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:11:38.301 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:11:38.301 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:11:38.402 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:11:38.437 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:11:38.440 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:11:38.441 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:11:38.542 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:11:38.551 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 5 16 12485 12469 155.821 130.5 0.00643183 0.00640924 2023-06-03T20:11:38.573 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:11:38.575 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:11:38.576 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:11:38.677 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:11:38.710 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:11:38.712 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:11:38.713 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:11:38.814 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:11:38.851 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:11:38.854 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:11:38.854 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:11:38.955 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:11:38.987 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:11:38.993 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:11:38.994 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:11:39.095 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:11:39.126 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:11:39.130 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:11:39.131 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:11:39.232 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:11:39.265 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:11:39.267 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:11:39.267 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:11:39.368 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:11:39.402 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:11:39.405 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:11:39.405 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:11:39.506 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:11:39.528 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:11:39.529 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:11:39.530 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:11:39.551 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 6 16 14512 14496 150.962 126.688 0.00303838 0.00655948 2023-06-03T20:11:39.631 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:11:39.666 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:11:39.668 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:11:39.669 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:11:39.770 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:11:39.805 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:11:39.810 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:11:39.810 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:11:39.911 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:11:39.953 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:11:39.958 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:11:39.959 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:11:40.028 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:11:39 smithi089 bash[78452]: cluster 1685823098.67743 mgr.y (mgr.34104) 1066 : cluster 0 pgmap v1032: 89 pgs: 89 active+clean; 558 MiB data, 3.5 GiB used, 622 GiB / 626 GiB avail; 56 MiB/s wr, 891 op/s 2023-06-03T20:11:40.029 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:11:39 smithi089 bash[75068]: cluster 1685823098.67743 mgr.y (mgr.34104) 1066 : cluster 0 pgmap v1032: 89 pgs: 89 active+clean; 558 MiB data, 3.5 GiB used, 622 GiB / 626 GiB avail; 56 MiB/s wr, 891 op/s 2023-06-03T20:11:40.060 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:11:40.071 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:11:39 smithi164 bash[59462]: cluster 1685823098.67743 mgr.y (mgr.34104) 1066 : cluster 0 pgmap v1032: 89 pgs: 89 active+clean; 558 MiB data, 3.5 GiB used, 622 GiB / 626 GiB avail; 56 MiB/s wr, 891 op/s 2023-06-03T20:11:40.093 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:11:40.096 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:11:40.097 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:11:40.198 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:11:40.228 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:11:40.231 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:11:40.232 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:11:40.287 INFO:teuthology.orchestra.run.smithi164.stderr:Inferring config /var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/mon.b/config 2023-06-03T20:11:40.333 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:11:40.366 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:11:40.368 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:11:40.369 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:11:40.470 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:11:40.501 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:11:40.504 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:11:40.505 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:11:40.551 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 7 16 16524 16508 147.359 125.75 0.00394574 0.00677632 2023-06-03T20:11:40.606 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:11:40.647 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:11:40.653 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:11:40.654 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:11:40.755 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:11:40.791 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:11:40.797 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:11:40.797 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:11:40.898 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:11:40.959 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:11:40.962 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:11:40.963 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:11:41.065 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:11:41.100 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:11:41.103 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:11:41.103 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:11:41.204 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:11:41.229 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:11:41.232 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:11:41.233 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:11:41.334 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:11:41.379 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:11:41.383 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:11:41.384 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:11:41.485 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:11:41.537 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:11:41.540 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:11:41.540 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:11:41.553 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 8 16 18099 18083 141.215 98.4375 0.00476749 0.00707472 2023-06-03T20:11:41.641 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:11:41.674 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:11:41.682 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:11:41.683 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:11:41.784 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:11:41.821 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:11:41.824 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:11:41.825 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:11:41.833 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:11:41 smithi164 bash[59462]: cluster 1685823100.678048 mgr.y (mgr.34104) 1067 : cluster 0 pgmap v1033: 89 pgs: 89 active+clean; 741 MiB data, 4.1 GiB used, 622 GiB / 626 GiB avail; 74 MiB/s wr, 1.18k op/s 2023-06-03T20:11:41.926 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:11:41.934 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:11:41 smithi089 bash[75068]: cluster 1685823100.678048 mgr.y (mgr.34104) 1067 : cluster 0 pgmap v1033: 89 pgs: 89 active+clean; 741 MiB data, 4.1 GiB used, 622 GiB / 626 GiB avail; 74 MiB/s wr, 1.18k op/s 2023-06-03T20:11:41.935 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:11:41 smithi089 bash[78452]: cluster 1685823100.678048 mgr.y (mgr.34104) 1067 : cluster 0 pgmap v1033: 89 pgs: 89 active+clean; 741 MiB data, 4.1 GiB used, 622 GiB / 626 GiB avail; 74 MiB/s wr, 1.18k op/s 2023-06-03T20:11:41.961 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:11:41.965 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:11:41.965 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:11:42.066 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:11:42.120 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:11:42.124 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:11:42.125 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:11:42.226 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:11:42.261 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:11:42.267 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:11:42.267 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:11:42.368 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:11:42.401 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:11:42.406 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:11:42.407 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:11:42.451 INFO:teuthology.orchestra.run.smithi089.stdout:osd.0: {} 2023-06-03T20:11:42.486 INFO:teuthology.orchestra.run.smithi089.stdout:osd.1: {} 2023-06-03T20:11:42.508 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:11:42.528 INFO:teuthology.orchestra.run.smithi089.stdout:osd.2: {} 2023-06-03T20:11:42.537 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:11:42.538 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:11:42.539 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:11:42.553 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 9 15 19689 19674 136.571 99.4375 0.016007 0.00730767 2023-06-03T20:11:42.586 INFO:teuthology.orchestra.run.smithi089.stdout:osd.3: {} 2023-06-03T20:11:42.632 INFO:teuthology.orchestra.run.smithi089.stdout:osd.4: {} 2023-06-03T20:11:42.641 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:11:42.676 INFO:teuthology.orchestra.run.smithi089.stdout:osd.5: {} 2023-06-03T20:11:42.677 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:11:42.680 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:11:42.680 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:11:42.723 INFO:teuthology.orchestra.run.smithi089.stdout:osd.6: {} 2023-06-03T20:11:42.754 INFO:teuthology.orchestra.run.smithi089.stdout:osd.7: {} 2023-06-03T20:11:42.758 INFO:teuthology.orchestra.run.smithi089.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-06-03T20:11:42.758 INFO:teuthology.orchestra.run.smithi089.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-06-03T20:11:42.758 INFO:teuthology.orchestra.run.smithi089.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-06-03T20:11:42.759 INFO:teuthology.orchestra.run.smithi089.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-06-03T20:11:42.759 INFO:teuthology.orchestra.run.smithi089.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-06-03T20:11:42.759 INFO:teuthology.orchestra.run.smithi089.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-06-03T20:11:42.759 INFO:teuthology.orchestra.run.smithi089.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-06-03T20:11:42.759 INFO:teuthology.orchestra.run.smithi089.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-06-03T20:11:42.782 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:11:42.821 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:11:42.824 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:11:42.825 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:11:42.926 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:11:42.962 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:11:42.965 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:11:42.965 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:11:43.066 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:11:43.097 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:11:43.154 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:11:43.154 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:11:43.255 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:11:43.265 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:11:42 smithi164 bash[59462]: audit 1685823101.9565177 mon.a (mon.0) 1000 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:11:43.266 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:11:42 smithi164 bash[59462]: audit 1685823101.978762 mon.a (mon.0) 1001 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:11:43.266 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:11:42 smithi164 bash[59462]: audit 1685823102.1602387 mon.a (mon.0) 1002 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:11:43.266 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:11:42 smithi164 bash[59462]: audit 1685823102.1761754 mon.a (mon.0) 1003 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:11:43.266 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:11:42 smithi164 bash[59462]: audit 1685823102.4012423 mon.a (mon.0) 1004 : audit 0 from='client.? 172.21.15.89:0/4262834500' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-03T20:11:43.266 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:11:42 smithi164 bash[59462]: audit 1685823102.8081765 mon.c (mon.2) 500 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-03T20:11:43.266 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:11:42 smithi164 bash[59462]: audit 1685823102.8107371 mon.c (mon.2) 501 : audit 1 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-03T20:11:43.266 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:11:42 smithi164 bash[59462]: audit 1685823102.822442 mon.a (mon.0) 1005 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:11:43.278 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:11:42 smithi089 bash[75068]: audit 1685823101.9565177 mon.a (mon.0) 1000 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:11:43.279 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:11:42 smithi089 bash[75068]: audit 1685823101.978762 mon.a (mon.0) 1001 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:11:43.279 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:11:42 smithi089 bash[75068]: audit 1685823102.1602387 mon.a (mon.0) 1002 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:11:43.279 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:11:42 smithi089 bash[75068]: audit 1685823102.1761754 mon.a (mon.0) 1003 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:11:43.279 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:11:42 smithi089 bash[75068]: audit 1685823102.4012423 mon.a (mon.0) 1004 : audit 0 from='client.? 172.21.15.89:0/4262834500' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-03T20:11:43.279 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:11:42 smithi089 bash[75068]: audit 1685823102.8081765 mon.c (mon.2) 500 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-03T20:11:43.279 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:11:42 smithi089 bash[75068]: audit 1685823102.8107371 mon.c (mon.2) 501 : audit 1 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-03T20:11:43.280 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:11:42 smithi089 bash[75068]: audit 1685823102.822442 mon.a (mon.0) 1005 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:11:43.280 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:11:42 smithi089 bash[78452]: audit 1685823101.9565177 mon.a (mon.0) 1000 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:11:43.280 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:11:42 smithi089 bash[78452]: audit 1685823101.978762 mon.a (mon.0) 1001 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:11:43.280 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:11:42 smithi089 bash[78452]: audit 1685823102.1602387 mon.a (mon.0) 1002 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:11:43.280 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:11:42 smithi089 bash[78452]: audit 1685823102.1761754 mon.a (mon.0) 1003 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:11:43.281 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:11:42 smithi089 bash[78452]: audit 1685823102.4012423 mon.a (mon.0) 1004 : audit 0 from='client.? 172.21.15.89:0/4262834500' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-03T20:11:43.281 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:11:42 smithi089 bash[78452]: audit 1685823102.8081765 mon.c (mon.2) 500 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-03T20:11:43.281 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:11:42 smithi089 bash[78452]: audit 1685823102.8107371 mon.c (mon.2) 501 : audit 1 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-03T20:11:43.281 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:11:42 smithi089 bash[78452]: audit 1685823102.822442 mon.a (mon.0) 1005 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:11:43.292 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:11:43.294 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:11:43.295 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:11:43.395 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:11:43.429 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:11:43.436 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:11:43.437 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:11:43.537 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:11:43.553 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 10 15 21488 21473 134.157 112.438 0.00912081 0.00744735 2023-06-03T20:11:43.574 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:11:43.577 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:11:43.578 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:11:43.679 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:11:43.711 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:11:43.714 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:11:43.715 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:11:43.816 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:11:43.845 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:11:43.847 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:11:43.848 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:11:43.950 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:11:43.960 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:11:43 smithi089 bash[75068]: cluster 1685823102.678625 mgr.y (mgr.34104) 1068 : cluster 0 pgmap v1034: 89 pgs: 89 active+clean; 822 MiB data, 4.6 GiB used, 621 GiB / 626 GiB avail; 69 MiB/s wr, 1.10k op/s 2023-06-03T20:11:43.960 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:11:43 smithi089 bash[75068]: audit 1685823103.1535172 mon.b (mon.1) 55 : audit 1 from='client.? 172.21.15.89:0/4134145771' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-06-03T20:11:43.961 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:11:43 smithi089 bash[75068]: audit 1685823103.155891 mon.a (mon.0) 1006 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-06-03T20:11:43.961 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:11:43 smithi089 bash[78452]: cluster 1685823102.678625 mgr.y (mgr.34104) 1068 : cluster 0 pgmap v1034: 89 pgs: 89 active+clean; 822 MiB data, 4.6 GiB used, 621 GiB / 626 GiB avail; 69 MiB/s wr, 1.10k op/s 2023-06-03T20:11:43.961 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:11:43 smithi089 bash[78452]: audit 1685823103.1535172 mon.b (mon.1) 55 : audit 1 from='client.? 172.21.15.89:0/4134145771' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-06-03T20:11:43.961 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:11:43 smithi089 bash[78452]: audit 1685823103.155891 mon.a (mon.0) 1006 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-06-03T20:11:43.979 INFO:teuthology.orchestra.run.smithi089.stderr:noscrub is set 2023-06-03T20:11:43.983 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:11:43.987 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:11:43.987 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:11:44.088 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:11:44.122 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:11:44.124 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:11:44.125 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:11:44.226 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:11:44.236 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:11:43 smithi164 bash[59462]: cluster 1685823102.678625 mgr.y (mgr.34104) 1068 : cluster 0 pgmap v1034: 89 pgs: 89 active+clean; 822 MiB data, 4.6 GiB used, 621 GiB / 626 GiB avail; 69 MiB/s wr, 1.10k op/s 2023-06-03T20:11:44.236 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:11:43 smithi164 bash[59462]: audit 1685823103.1535172 mon.b (mon.1) 55 : audit 1 from='client.? 172.21.15.89:0/4134145771' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-06-03T20:11:44.236 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:11:43 smithi164 bash[59462]: audit 1685823103.155891 mon.a (mon.0) 1006 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-06-03T20:11:44.268 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:11:44.272 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:11:44.273 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:11:44.373 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:11:44.407 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:11:44.410 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:11:44.411 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:11:44.512 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:11:44.543 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:11:44.545 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:11:44.546 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:11:44.553 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 11 16 23361 23345 132.597 117 0.0238904 0.00753445 2023-06-03T20:11:44.647 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:11:44.683 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:11:44.688 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:11:44.689 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:11:44.790 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:11:44.826 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:11:44.830 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:11:44.831 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:11:44.932 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:11:44.968 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:11:44.972 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:11:44.973 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:11:45.073 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:11:45.083 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:11:44 smithi164 bash[59462]: cluster 1685823103.95356 mon.a (mon.0) 1007 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-06-03T20:11:45.084 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:11:44 smithi164 bash[59462]: cluster 1685823103.9641535 osd.6 (osd.6) 140 : cluster 3 failed to encode map e194 with expected crc 2023-06-03T20:11:45.084 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:11:44 smithi164 bash[59462]: audit 1685823103.9683006 mon.a (mon.0) 1008 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-06-03T20:11:45.084 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:11:44 smithi164 bash[59462]: cluster 1685823103.968491 mon.a (mon.0) 1009 : cluster 0 osdmap e194: 8 total, 8 up, 7 in 2023-06-03T20:11:45.084 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:11:44 smithi164 bash[59462]: cluster 1685823103.9738262 osd.5 (osd.5) 125 : cluster 3 failed to encode map e194 with expected crc 2023-06-03T20:11:45.084 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:11:44 smithi164 bash[59462]: cluster 1685823103.9776506 osd.5 (osd.5) 126 : cluster 3 failed to encode map e194 with expected crc 2023-06-03T20:11:45.102 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:11:45.106 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:11:45.106 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:11:45.207 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:11:45.238 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:11:45.242 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:11:45.243 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:11:45.344 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:11:45.376 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:11:45.379 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:11:45.380 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:11:45.480 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:11:45.517 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:11:45.520 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:11:45.521 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:11:45.528 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:11:45 smithi089 bash[78452]: cluster 1685823103.95356 mon.a (mon.0) 1007 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-06-03T20:11:45.528 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:11:45 smithi089 bash[78452]: cluster 1685823103.9641535 osd.6 (osd.6) 140 : cluster 3 failed to encode map e194 with expected crc 2023-06-03T20:11:45.528 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:11:45 smithi089 bash[78452]: audit 1685823103.9683006 mon.a (mon.0) 1008 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-06-03T20:11:45.529 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:11:45 smithi089 bash[78452]: cluster 1685823103.968491 mon.a (mon.0) 1009 : cluster 0 osdmap e194: 8 total, 8 up, 7 in 2023-06-03T20:11:45.529 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:11:45 smithi089 bash[78452]: cluster 1685823103.9738262 osd.5 (osd.5) 125 : cluster 3 failed to encode map e194 with expected crc 2023-06-03T20:11:45.529 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:11:45 smithi089 bash[78452]: cluster 1685823103.9776506 osd.5 (osd.5) 126 : cluster 3 failed to encode map e194 with expected crc 2023-06-03T20:11:45.529 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:11:45 smithi089 bash[75068]: cluster 1685823103.95356 mon.a (mon.0) 1007 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-06-03T20:11:45.529 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:11:45 smithi089 bash[75068]: cluster 1685823103.9641535 osd.6 (osd.6) 140 : cluster 3 failed to encode map e194 with expected crc 2023-06-03T20:11:45.529 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:11:45 smithi089 bash[75068]: audit 1685823103.9683006 mon.a (mon.0) 1008 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-06-03T20:11:45.530 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:11:45 smithi089 bash[75068]: cluster 1685823103.968491 mon.a (mon.0) 1009 : cluster 0 osdmap e194: 8 total, 8 up, 7 in 2023-06-03T20:11:45.530 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:11:45 smithi089 bash[75068]: cluster 1685823103.9738262 osd.5 (osd.5) 125 : cluster 3 failed to encode map e194 with expected crc 2023-06-03T20:11:45.530 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:11:45 smithi089 bash[75068]: cluster 1685823103.9776506 osd.5 (osd.5) 126 : cluster 3 failed to encode map e194 with expected crc 2023-06-03T20:11:45.553 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 12 16 25261 25245 131.442 118.75 0.00321569 0.00760144 2023-06-03T20:11:45.621 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:11:45.654 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:11:45.658 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:11:45.659 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:11:45.760 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:11:45.798 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:11:45.801 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:11:45.802 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:11:45.903 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:11:45.931 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:11:45.935 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:11:45.935 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:11:45.971 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:11:45 smithi089 bash[75068]: cluster 1685823104.6791153 mgr.y (mgr.34104) 1069 : cluster 0 pgmap v1036: 89 pgs: 89 active+clean; 1.2 GiB data, 5.7 GiB used, 620 GiB / 626 GiB avail; 121 MiB/s wr, 1.94k op/s 2023-06-03T20:11:46.036 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:11:46.046 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:11:45 smithi164 bash[59462]: cluster 1685823104.6791153 mgr.y (mgr.34104) 1069 : cluster 0 pgmap v1036: 89 pgs: 89 active+clean; 1.2 GiB data, 5.7 GiB used, 620 GiB / 626 GiB avail; 121 MiB/s wr, 1.94k op/s 2023-06-03T20:11:46.076 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:11:46.080 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:11:46.081 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:11:46.181 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:11:46.212 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:11:46.214 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:11:46.214 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:11:46.231 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:11:45 smithi089 bash[78452]: cluster 1685823104.6791153 mgr.y (mgr.34104) 1069 : cluster 0 pgmap v1036: 89 pgs: 89 active+clean; 1.2 GiB data, 5.7 GiB used, 620 GiB / 626 GiB avail; 121 MiB/s wr, 1.94k op/s 2023-06-03T20:11:46.315 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:11:46.349 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:11:46.353 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:11:46.354 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:11:46.455 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:11:46.495 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:11:46.497 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:11:46.498 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:11:46.554 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 13 16 27138 27122 130.355 117.312 0.00987894 0.00766443 2023-06-03T20:11:46.598 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:11:46.624 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:11:46.626 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:11:46.627 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:11:46.728 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:11:46.753 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:11:46.755 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:11:46.755 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:11:46.856 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:11:46.892 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:11:46.894 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:11:46.894 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:11:46.995 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:11:47.005 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 20:11:46 smithi089 bash[71163]: ::ffff:172.21.15.164 - - [03/Jun/2023:20:11:46] "GET /metrics HTTP/1.1" 200 228683 "" "Prometheus/2.33.4" 2023-06-03T20:11:47.031 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:11:47.034 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:11:47.035 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:11:47.136 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:11:47.173 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:11:47.177 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:11:47.177 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:11:47.278 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:11:47.312 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:11:47.314 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:11:47.315 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:11:47.416 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:11:47.448 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:11:47.453 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:11:47.454 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:11:47.554 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 14 16 28451 28435 126.905 82.0625 0.00518007 0.00787337 2023-06-03T20:11:47.555 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:11:47.590 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:11:47.594 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:11:47.595 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:11:47.695 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:11:47.730 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:11:47.734 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:11:47.735 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:11:47.835 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:11:47.879 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:11:47.883 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:11:47.884 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:11:47.978 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:11:47 smithi089 bash[75068]: cluster 1685823106.679655 mgr.y (mgr.34104) 1070 : cluster 0 pgmap v1037: 89 pgs: 89 active+clean; 1.3 GiB data, 6.1 GiB used, 620 GiB / 626 GiB avail; 129 MiB/s wr, 2.06k op/s 2023-06-03T20:11:47.985 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:11:48.020 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:11:48.023 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:11:48.024 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:11:48.124 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:11:48.133 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:11:47 smithi164 bash[59462]: cluster 1685823106.679655 mgr.y (mgr.34104) 1070 : cluster 0 pgmap v1037: 89 pgs: 89 active+clean; 1.3 GiB data, 6.1 GiB used, 620 GiB / 626 GiB avail; 129 MiB/s wr, 2.06k op/s 2023-06-03T20:11:48.156 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:11:48.160 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:11:48.161 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:11:48.262 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:11:48.279 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:11:47 smithi089 bash[78452]: cluster 1685823106.679655 mgr.y (mgr.34104) 1070 : cluster 0 pgmap v1037: 89 pgs: 89 active+clean; 1.3 GiB data, 6.1 GiB used, 620 GiB / 626 GiB avail; 129 MiB/s wr, 2.06k op/s 2023-06-03T20:11:48.305 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:11:48.307 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:11:48.308 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:11:48.409 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:11:48.440 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:11:48.443 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:11:48.443 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:11:48.523 INFO:teuthology.orchestra.run.smithi164.stdout:ERROR: (22) Invalid argument 2023-06-03T20:11:48.523 INFO:teuthology.orchestra.run.smithi164.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-06-03T20:11:48.545 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:11:48.554 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 15 16 30093 30077 125.286 102.625 0.0101007 0.00797451 2023-06-03T20:11:48.578 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:11:48.582 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:11:48.582 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:11:48.683 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:11:48.709 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:11:48.711 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:11:48.712 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:11:48.813 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:11:48.843 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:11:48.846 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:11:48.847 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:11:48.948 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:11:48.979 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:11:48.982 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:11:48.982 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:11:49.083 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:11:49.118 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:11:49.120 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:11:49.121 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:11:49.222 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:11:49.269 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:11:49.275 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:11:49.276 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:11:49.377 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:11:49.407 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:11:49.411 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:11:49.412 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:11:49.432 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-06-03T20:11:49.512 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:11:49.522 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph osd set nodeep-scrub 2023-06-03T20:11:49.542 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:11:49.549 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:11:49.549 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:11:49.554 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 16 16 31222 31206 121.866 70.5625 0.010112 0.00819883 2023-06-03T20:11:49.650 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:11:49.689 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:11:49.690 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:11:49.691 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:11:49.718 DEBUG:teuthology.orchestra.run.smithi164:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph daemon osd.5 dump_blocked_ops 2023-06-03T20:11:49.792 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:11:49.831 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:11:49.833 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:11:49.834 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:11:49.935 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:11:49.974 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:11:49.978 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:11:49.979 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:11:50.080 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:11:50.121 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:11:50.125 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:11:50.126 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:11:50.207 INFO:teuthology.orchestra.run.smithi089.stdout: 2023-06-03T20:11:50.213 INFO:teuthology.orchestra.run.smithi089.stderr:dumped all 2023-06-03T20:11:50.227 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:11:50.237 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:11:49 smithi089 bash[75068]: cluster 1685823108.6801984 mgr.y (mgr.34104) 1071 : cluster 0 pgmap v1038: 89 pgs: 89 active+clean; 1.7 GiB data, 7.1 GiB used, 619 GiB / 626 GiB avail; 117 MiB/s wr, 1.87k op/s 2023-06-03T20:11:50.237 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:11:49 smithi089 bash[78452]: cluster 1685823108.6801984 mgr.y (mgr.34104) 1071 : cluster 0 pgmap v1038: 89 pgs: 89 active+clean; 1.7 GiB data, 7.1 GiB used, 619 GiB / 626 GiB avail; 117 MiB/s wr, 1.87k op/s 2023-06-03T20:11:50.265 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:11:50.265 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:11:50.266 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:11:50.333 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:11:49 smithi164 bash[59462]: cluster 1685823108.6801984 mgr.y (mgr.34104) 1071 : cluster 0 pgmap v1038: 89 pgs: 89 active+clean; 1.7 GiB data, 7.1 GiB used, 619 GiB / 626 GiB avail; 117 MiB/s wr, 1.87k op/s 2023-06-03T20:11:50.366 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:11:50.404 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:11:50.406 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:11:50.407 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:11:50.508 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:11:50.541 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:11:50.546 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:11:50.546 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:11:50.554 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 17 16 32682 32666 120.065 91.25 0.00631189 0.00832153 2023-06-03T20:11:50.648 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:11:50.677 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:11:50.680 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:11:50.681 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:11:50.782 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:11:50.815 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:11:50.818 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:11:50.818 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:11:50.919 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:11:50.955 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:11:50.956 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:11:50.956 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:11:51.057 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:11:51.086 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:11:51.089 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:11:51.089 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:11:51.190 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:11:51.222 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:11:51.226 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:11:51.227 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:11:51.328 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:11:51.362 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:11:51.366 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:11:51.367 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:11:51.468 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:11:51.500 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:11:51.502 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:11:51.503 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:11:51.554 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 18 16 34337 34321 119.141 103.438 0.00633717 0.00838757 2023-06-03T20:11:51.604 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:11:51.643 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:11:51.646 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:11:51.647 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:11:51.748 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:11:51.779 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:11:51.782 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:11:51.783 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:11:51.846 INFO:teuthology.orchestra.run.smithi089.stdout:{"pg_ready":true,"pg_map":{"version":1038,"stamp":"2023-06-03T20:11:48.679834+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":1808930702,"num_objects":27617,"num_object_clones":0,"num_object_copies":82851,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":27617,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":27854,"num_write_kb":1768398,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":27788,"ondisk_log_size":27788,"up":267,"acting":267,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":267,"num_osds":7,"num_per_pool_osds":7,"num_per_pool_omap_osds":7,"kb":656216064,"kb_used":7456784,"kb_used_data":5010968,"kb_used_omap":26,"kb_used_meta":2445733,"kb_avail":648759280,"statfs":{"total":671965249536,"available":664329502720,"internally_reserved":0,"allocated":5131231232,"data_stored":5126799705,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":27292,"internal_metadata":2504430948},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":1223819264,"num_objects":18674,"num_object_clones":0,"num_object_copies":56022,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":18674,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":18674,"num_write_kb":1195136,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.002949"},"pg_stats":[{"pgid":"3.1a","version":"60'1","reported_seq":193,"reported_epoch":193,"state":"active+clean","last_fresh":"2023-06-03T20:11:30.754444+0000","last_change":"2023-06-03T20:07:19.904870+0000","last_active":"2023-06-03T20:11:30.754444+0000","last_peered":"2023-06-03T20:11:30.754444+0000","last_clean":"2023-06-03T20:11:30.754444+0000","last_became_active":"2023-06-03T20:07:19.904541+0000","last_became_peered":"2023-06-03T20:07:19.904541+0000","last_unstale":"2023-06-03T20:11:30.754444+0000","last_undegraded":"2023-06-03T20:11:30.754444+0000","last_fullsized":"2023-06-03T20:11:30.754444+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T01:48:10.778198+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,0],"acting":[4,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":33,"reported_epoch":192,"state":"active+clean","last_fresh":"2023-06-03T20:11:17.988110+0000","last_change":"2023-06-03T20:07:23.934703+0000","last_active":"2023-06-03T20:11:17.988110+0000","last_peered":"2023-06-03T20:11:17.988110+0000","last_clean":"2023-06-03T20:11:17.988110+0000","last_became_active":"2023-06-03T20:07:23.934266+0000","last_became_peered":"2023-06-03T20:07:23.934266+0000","last_unstale":"2023-06-03T20:11:17.988110+0000","last_undegraded":"2023-06-03T20:11:17.988110+0000","last_fullsized":"2023-06-03T20:11:17.988110+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":147,"reported_epoch":193,"state":"active+clean","last_fresh":"2023-06-03T20:11:31.887205+0000","last_change":"2023-06-03T20:07:19.906895+0000","last_active":"2023-06-03T20:11:31.887205+0000","last_peered":"2023-06-03T20:11:31.887205+0000","last_clean":"2023-06-03T20:11:31.887205+0000","last_became_active":"2023-06-03T20:07:19.906512+0000","last_became_peered":"2023-06-03T20:07:19.906512+0000","last_unstale":"2023-06-03T20:11:31.887205+0000","last_undegraded":"2023-06-03T20:11:31.887205+0000","last_fullsized":"2023-06-03T20:11:31.887205+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T05:12:17.526160+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,6],"acting":[3,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":156,"reported_epoch":193,"state":"active+clean","last_fresh":"2023-06-03T20:11:31.275800+0000","last_change":"2023-06-03T19:56:54.507653+0000","last_active":"2023-06-03T20:11:31.275800+0000","last_peered":"2023-06-03T20:11:31.275800+0000","last_clean":"2023-06-03T20:11:31.275800+0000","last_became_active":"2023-06-03T19:56:54.507350+0000","last_became_peered":"2023-06-03T19:56:54.507350+0000","last_unstale":"2023-06-03T20:11:31.275800+0000","last_undegraded":"2023-06-03T20:11:31.275800+0000","last_fullsized":"2023-06-03T20:11:31.275800+0000","mapping_epoch":119,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T22:33:20.065844+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":188,"reported_epoch":193,"state":"active+clean","last_fresh":"2023-06-03T20:11:31.886803+0000","last_change":"2023-06-03T20:07:19.912004+0000","last_active":"2023-06-03T20:11:31.886803+0000","last_peered":"2023-06-03T20:11:31.886803+0000","last_clean":"2023-06-03T20:11:31.886803+0000","last_became_active":"2023-06-03T20:07:19.910895+0000","last_became_peered":"2023-06-03T20:07:19.910895+0000","last_unstale":"2023-06-03T20:11:31.886803+0000","last_undegraded":"2023-06-03T20:11:31.886803+0000","last_fullsized":"2023-06-03T20:11:31.886803+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T06:46:44.564002+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,2],"acting":[3,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":169,"reported_epoch":193,"state":"active+clean","last_fresh":"2023-06-03T20:11:30.758717+0000","last_change":"2023-06-03T20:07:19.918531+0000","last_active":"2023-06-03T20:11:30.758717+0000","last_peered":"2023-06-03T20:11:30.758717+0000","last_clean":"2023-06-03T20:11:30.758717+0000","last_became_active":"2023-06-03T20:07:19.918173+0000","last_became_peered":"2023-06-03T20:07:19.918173+0000","last_unstale":"2023-06-03T20:11:30.758717+0000","last_undegraded":"2023-06-03T20:11:30.758717+0000","last_fullsized":"2023-06-03T20:11:30.758717+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,0],"acting":[6,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":153,"reported_epoch":193,"state":"active+clean","last_fresh":"2023-06-03T20:11:31.276152+0000","last_change":"2023-06-03T19:56:54.509490+0000","last_active":"2023-06-03T20:11:31.276152+0000","last_peered":"2023-06-03T20:11:31.276152+0000","last_clean":"2023-06-03T20:11:31.276152+0000","last_became_active":"2023-06-03T19:56:54.509145+0000","last_became_peered":"2023-06-03T19:56:54.509145+0000","last_unstale":"2023-06-03T20:11:31.276152+0000","last_undegraded":"2023-06-03T20:11:31.276152+0000","last_fullsized":"2023-06-03T20:11:31.276152+0000","mapping_epoch":119,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T07:12:03.964110+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":172,"reported_epoch":193,"state":"active+clean","last_fresh":"2023-06-03T20:11:31.886944+0000","last_change":"2023-06-03T20:07:19.907524+0000","last_active":"2023-06-03T20:11:31.886944+0000","last_peered":"2023-06-03T20:11:31.886944+0000","last_clean":"2023-06-03T20:11:31.886944+0000","last_became_active":"2023-06-03T20:07:19.907098+0000","last_became_peered":"2023-06-03T20:07:19.907098+0000","last_unstale":"2023-06-03T20:11:31.886944+0000","last_undegraded":"2023-06-03T20:11:31.886944+0000","last_fullsized":"2023-06-03T20:11:31.886944+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T05:30:24.161646+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,4],"acting":[3,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":169,"reported_epoch":192,"state":"active+clean","last_fresh":"2023-06-03T20:11:17.986377+0000","last_change":"2023-06-03T20:07:19.912362+0000","last_active":"2023-06-03T20:11:17.986377+0000","last_peered":"2023-06-03T20:11:17.986377+0000","last_clean":"2023-06-03T20:11:17.986377+0000","last_became_active":"2023-06-03T20:07:19.907385+0000","last_became_peered":"2023-06-03T20:07:19.907385+0000","last_unstale":"2023-06-03T20:11:17.986377+0000","last_undegraded":"2023-06-03T20:11:17.986377+0000","last_fullsized":"2023-06-03T20:11:17.986377+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":144,"reported_epoch":193,"state":"active+clean","last_fresh":"2023-06-03T20:11:31.887038+0000","last_change":"2023-06-03T19:56:54.510462+0000","last_active":"2023-06-03T20:11:31.887038+0000","last_peered":"2023-06-03T20:11:31.887038+0000","last_clean":"2023-06-03T20:11:31.887038+0000","last_became_active":"2023-06-03T19:56:54.509783+0000","last_became_peered":"2023-06-03T19:56:54.509783+0000","last_unstale":"2023-06-03T20:11:31.887038+0000","last_undegraded":"2023-06-03T20:11:31.887038+0000","last_fullsized":"2023-06-03T20:11:31.887038+0000","mapping_epoch":119,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T00:16:09.624276+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"60'3","reported_seq":176,"reported_epoch":193,"state":"active+clean","last_fresh":"2023-06-03T20:11:31.883734+0000","last_change":"2023-06-03T20:07:47.854292+0000","last_active":"2023-06-03T20:11:31.883734+0000","last_peered":"2023-06-03T20:11:31.883734+0000","last_clean":"2023-06-03T20:11:31.883734+0000","last_became_active":"2023-06-03T20:07:47.853941+0000","last_became_peered":"2023-06-03T20:07:47.853941+0000","last_unstale":"2023-06-03T20:11:31.883734+0000","last_undegraded":"2023-06-03T20:11:31.883734+0000","last_fullsized":"2023-06-03T20:11:31.883734+0000","mapping_epoch":172,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":173,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":79,"reported_epoch":193,"state":"active+clean","last_fresh":"2023-06-03T20:11:30.758310+0000","last_change":"2023-06-03T20:07:19.913393+0000","last_active":"2023-06-03T20:11:30.758310+0000","last_peered":"2023-06-03T20:11:30.758310+0000","last_clean":"2023-06-03T20:11:30.758310+0000","last_became_active":"2023-06-03T20:07:19.913074+0000","last_became_peered":"2023-06-03T20:07:19.913074+0000","last_unstale":"2023-06-03T20:11:30.758310+0000","last_undegraded":"2023-06-03T20:11:30.758310+0000","last_fullsized":"2023-06-03T20:11:30.758310+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,0],"acting":[6,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.14","version":"77'23","reported_seq":181,"reported_epoch":193,"state":"active+clean","last_fresh":"2023-06-03T20:11:30.755040+0000","last_change":"2023-06-03T19:57:28.462522+0000","last_active":"2023-06-03T20:11:30.755040+0000","last_peered":"2023-06-03T20:11:30.755040+0000","last_clean":"2023-06-03T20:11:30.755040+0000","last_became_active":"2023-06-03T19:57:28.462048+0000","last_became_peered":"2023-06-03T19:57:28.462048+0000","last_unstale":"2023-06-03T20:11:30.755040+0000","last_undegraded":"2023-06-03T20:11:30.755040+0000","last_fullsized":"2023-06-03T20:11:30.755040+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":23,"log_dups_size":0,"ondisk_log_size":23,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T23:23:56.697534+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4716,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":23,"num_write_kb":20,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":167,"reported_epoch":193,"state":"active+clean","last_fresh":"2023-06-03T20:11:30.754816+0000","last_change":"2023-06-03T20:07:19.902581+0000","last_active":"2023-06-03T20:11:30.754816+0000","last_peered":"2023-06-03T20:11:30.754816+0000","last_clean":"2023-06-03T20:11:30.754816+0000","last_became_active":"2023-06-03T20:07:19.902153+0000","last_became_peered":"2023-06-03T20:07:19.902153+0000","last_unstale":"2023-06-03T20:11:30.754816+0000","last_undegraded":"2023-06-03T20:11:30.754816+0000","last_fullsized":"2023-06-03T20:11:30.754816+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T02:58:27.079772+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,5],"acting":[4,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"60'2","reported_seq":178,"reported_epoch":193,"state":"active+clean","last_fresh":"2023-06-03T20:11:31.884897+0000","last_change":"2023-06-03T19:57:28.467711+0000","last_active":"2023-06-03T20:11:31.884897+0000","last_peered":"2023-06-03T20:11:31.884897+0000","last_clean":"2023-06-03T20:11:31.884897+0000","last_became_active":"2023-06-03T19:57:28.467234+0000","last_became_peered":"2023-06-03T19:57:28.467234+0000","last_unstale":"2023-06-03T20:11:31.884897+0000","last_undegraded":"2023-06-03T20:11:31.884897+0000","last_fullsized":"2023-06-03T20:11:31.884897+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":172,"reported_epoch":193,"state":"active+clean","last_fresh":"2023-06-03T20:11:31.887493+0000","last_change":"2023-06-03T20:07:19.907827+0000","last_active":"2023-06-03T20:11:31.887493+0000","last_peered":"2023-06-03T20:11:31.887493+0000","last_clean":"2023-06-03T20:11:31.887493+0000","last_became_active":"2023-06-03T20:07:19.907296+0000","last_became_peered":"2023-06-03T20:07:19.907296+0000","last_unstale":"2023-06-03T20:11:31.887493+0000","last_undegraded":"2023-06-03T20:11:31.887493+0000","last_fullsized":"2023-06-03T20:11:31.887493+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T00:34:13.230147+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":153,"reported_epoch":193,"state":"active+clean","last_fresh":"2023-06-03T20:11:31.275493+0000","last_change":"2023-06-03T19:56:54.507479+0000","last_active":"2023-06-03T20:11:31.275493+0000","last_peered":"2023-06-03T20:11:31.275493+0000","last_clean":"2023-06-03T20:11:31.275493+0000","last_became_active":"2023-06-03T19:56:54.507149+0000","last_became_peered":"2023-06-03T19:56:54.507149+0000","last_unstale":"2023-06-03T20:11:31.275493+0000","last_undegraded":"2023-06-03T20:11:31.275493+0000","last_fullsized":"2023-06-03T20:11:31.275493+0000","mapping_epoch":119,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T20:22:36.676317+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":159,"reported_epoch":192,"state":"active+clean","last_fresh":"2023-06-03T20:11:17.988289+0000","last_change":"2023-06-03T19:56:54.512472+0000","last_active":"2023-06-03T20:11:17.988289+0000","last_peered":"2023-06-03T20:11:17.988289+0000","last_clean":"2023-06-03T20:11:17.988289+0000","last_became_active":"2023-06-03T19:56:54.512051+0000","last_became_peered":"2023-06-03T19:56:54.512051+0000","last_unstale":"2023-06-03T20:11:17.988289+0000","last_undegraded":"2023-06-03T20:11:17.988289+0000","last_fullsized":"2023-06-03T20:11:17.988289+0000","mapping_epoch":119,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":158,"reported_epoch":193,"state":"active+clean","last_fresh":"2023-06-03T20:11:31.884270+0000","last_change":"2023-06-03T20:07:47.855912+0000","last_active":"2023-06-03T20:11:31.884270+0000","last_peered":"2023-06-03T20:11:31.884270+0000","last_clean":"2023-06-03T20:11:31.884270+0000","last_became_active":"2023-06-03T20:07:47.855608+0000","last_became_peered":"2023-06-03T20:07:47.855608+0000","last_unstale":"2023-06-03T20:11:31.884270+0000","last_undegraded":"2023-06-03T20:11:31.884270+0000","last_fullsized":"2023-06-03T20:11:31.884270+0000","mapping_epoch":172,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":173,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":154,"reported_epoch":193,"state":"active+clean","last_fresh":"2023-06-03T20:11:31.275916+0000","last_change":"2023-06-03T19:56:54.510289+0000","last_active":"2023-06-03T20:11:31.275916+0000","last_peered":"2023-06-03T20:11:31.275916+0000","last_clean":"2023-06-03T20:11:31.275916+0000","last_became_active":"2023-06-03T19:56:54.509905+0000","last_became_peered":"2023-06-03T19:56:54.509905+0000","last_unstale":"2023-06-03T20:11:31.275916+0000","last_undegraded":"2023-06-03T20:11:31.275916+0000","last_fullsized":"2023-06-03T20:11:31.275916+0000","mapping_epoch":119,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T02:54:26.792401+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"60'8","reported_seq":169,"reported_epoch":193,"state":"active+clean","last_fresh":"2023-06-03T20:11:30.757789+0000","last_change":"2023-06-03T19:47:06.727778+0000","last_active":"2023-06-03T20:11:30.757789+0000","last_peered":"2023-06-03T20:11:30.757789+0000","last_clean":"2023-06-03T20:11:30.757789+0000","last_became_active":"2023-06-03T19:47:06.727305+0000","last_became_peered":"2023-06-03T19:47:06.727305+0000","last_unstale":"2023-06-03T20:11:30.757789+0000","last_undegraded":"2023-06-03T20:11:30.757789+0000","last_fullsized":"2023-06-03T20:11:30.757789+0000","mapping_epoch":86,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":87,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":172,"reported_epoch":193,"state":"active+clean","last_fresh":"2023-06-03T20:11:31.888081+0000","last_change":"2023-06-03T20:07:19.907151+0000","last_active":"2023-06-03T20:11:31.888081+0000","last_peered":"2023-06-03T20:11:31.888081+0000","last_clean":"2023-06-03T20:11:31.888081+0000","last_became_active":"2023-06-03T20:07:19.906479+0000","last_became_peered":"2023-06-03T20:07:19.906479+0000","last_unstale":"2023-06-03T20:11:31.888081+0000","last_undegraded":"2023-06-03T20:11:31.888081+0000","last_fullsized":"2023-06-03T20:11:31.888081+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T03:18:22.525692+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"11.7","version":"193'1375","reported_seq":1386,"reported_epoch":193,"state":"active+clean","last_fresh":"2023-06-03T20:11:43.697303+0000","last_change":"2023-06-03T20:11:11.792506+0000","last_active":"2023-06-03T20:11:43.697303+0000","last_peered":"2023-06-03T20:11:43.697303+0000","last_clean":"2023-06-03T20:11:43.697303+0000","last_became_active":"2023-06-03T20:11:11.792115+0000","last_became_peered":"2023-06-03T20:11:11.792115+0000","last_unstale":"2023-06-03T20:11:43.697303+0000","last_undegraded":"2023-06-03T20:11:43.697303+0000","last_fullsized":"2023-06-03T20:11:43.697303+0000","mapping_epoch":189,"log_start":"0'0","ondisk_log_start":"0'0","created":189,"last_epoch_clean":190,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T20:11:10.759955+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T20:11:10.759955+0000","last_clean_scrub_stamp":"2023-06-03T20:11:10.759955+0000","objects_scrubbed":0,"log_size":1375,"log_dups_size":0,"ondisk_log_size":1375,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90112000,"num_objects":1375,"num_object_clones":0,"num_object_copies":4125,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1375,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1375,"num_write_kb":88000,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,0],"acting":[5,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"60'1","reported_seq":177,"reported_epoch":193,"state":"active+clean","last_fresh":"2023-06-03T20:11:31.885101+0000","last_change":"2023-06-03T19:57:28.464482+0000","last_active":"2023-06-03T20:11:31.885101+0000","last_peered":"2023-06-03T20:11:31.885101+0000","last_clean":"2023-06-03T20:11:31.885101+0000","last_became_active":"2023-06-03T19:57:28.464081+0000","last_became_peered":"2023-06-03T19:57:28.464081+0000","last_unstale":"2023-06-03T20:11:31.885101+0000","last_undegraded":"2023-06-03T20:11:31.885101+0000","last_fullsized":"2023-06-03T20:11:31.885101+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":144,"reported_epoch":192,"state":"active+clean","last_fresh":"2023-06-03T20:11:17.986121+0000","last_change":"2023-06-03T19:34:17.848358+0000","last_active":"2023-06-03T20:11:17.986121+0000","last_peered":"2023-06-03T20:11:17.986121+0000","last_clean":"2023-06-03T20:11:17.986121+0000","last_became_active":"2023-06-03T19:34:17.847811+0000","last_became_peered":"2023-06-03T19:34:17.847811+0000","last_unstale":"2023-06-03T20:11:17.986121+0000","last_undegraded":"2023-06-03T20:11:17.986121+0000","last_fullsized":"2023-06-03T20:11:17.986121+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"11.6","version":"194'1597","reported_seq":1611,"reported_epoch":194,"state":"active+clean","last_fresh":"2023-06-03T20:11:45.783914+0000","last_change":"2023-06-03T20:11:11.790320+0000","last_active":"2023-06-03T20:11:45.783914+0000","last_peered":"2023-06-03T20:11:45.783914+0000","last_clean":"2023-06-03T20:11:45.783914+0000","last_became_active":"2023-06-03T20:11:11.789918+0000","last_became_peered":"2023-06-03T20:11:11.789918+0000","last_unstale":"2023-06-03T20:11:45.783914+0000","last_undegraded":"2023-06-03T20:11:45.783914+0000","last_fullsized":"2023-06-03T20:11:45.783914+0000","mapping_epoch":189,"log_start":"0'0","ondisk_log_start":"0'0","created":189,"last_epoch_clean":190,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T20:11:10.759955+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T20:11:10.759955+0000","last_clean_scrub_stamp":"2023-06-03T20:11:10.759955+0000","objects_scrubbed":0,"log_size":1597,"log_dups_size":0,"ondisk_log_size":1597,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T02:41:09.861533+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":104660992,"num_objects":1597,"num_object_clones":0,"num_object_copies":4791,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1597,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1597,"num_write_kb":102208,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,4],"acting":[0,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.e","version":"60'2","reported_seq":191,"reported_epoch":193,"state":"active+clean","last_fresh":"2023-06-03T20:11:31.885038+0000","last_change":"2023-06-03T20:07:19.965764+0000","last_active":"2023-06-03T20:11:31.885038+0000","last_peered":"2023-06-03T20:11:31.885038+0000","last_clean":"2023-06-03T20:11:31.885038+0000","last_became_active":"2023-06-03T20:07:19.910260+0000","last_became_peered":"2023-06-03T20:07:19.910260+0000","last_unstale":"2023-06-03T20:11:31.885038+0000","last_undegraded":"2023-06-03T20:11:31.885038+0000","last_fullsized":"2023-06-03T20:11:31.885038+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":205,"reported_epoch":193,"state":"active+clean","last_fresh":"2023-06-03T20:11:30.754266+0000","last_change":"2023-06-03T20:07:19.907926+0000","last_active":"2023-06-03T20:11:30.754266+0000","last_peered":"2023-06-03T20:11:30.754266+0000","last_clean":"2023-06-03T20:11:30.754266+0000","last_became_active":"2023-06-03T20:07:19.907441+0000","last_became_peered":"2023-06-03T20:07:19.907441+0000","last_unstale":"2023-06-03T20:11:30.754266+0000","last_undegraded":"2023-06-03T20:11:30.754266+0000","last_fullsized":"2023-06-03T20:11:30.754266+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T06:45:37.611197+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.5","version":"194'1834","reported_seq":1846,"reported_epoch":194,"state":"active+clean","last_fresh":"2023-06-03T20:11:47.962033+0000","last_change":"2023-06-03T20:11:11.796552+0000","last_active":"2023-06-03T20:11:47.962033+0000","last_peered":"2023-06-03T20:11:47.962033+0000","last_clean":"2023-06-03T20:11:47.962033+0000","last_became_active":"2023-06-03T20:11:11.796144+0000","last_became_peered":"2023-06-03T20:11:11.796144+0000","last_unstale":"2023-06-03T20:11:47.962033+0000","last_undegraded":"2023-06-03T20:11:47.962033+0000","last_fullsized":"2023-06-03T20:11:47.962033+0000","mapping_epoch":189,"log_start":"0'0","ondisk_log_start":"0'0","created":189,"last_epoch_clean":190,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T20:11:10.759955+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T20:11:10.759955+0000","last_clean_scrub_stamp":"2023-06-03T20:11:10.759955+0000","objects_scrubbed":0,"log_size":1834,"log_dups_size":0,"ondisk_log_size":1834,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":120193024,"num_objects":1834,"num_object_clones":0,"num_object_copies":5502,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1834,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1834,"num_write_kb":117376,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.d","version":"60'1","reported_seq":144,"reported_epoch":193,"state":"active+clean","last_fresh":"2023-06-03T20:11:31.884285+0000","last_change":"2023-06-03T20:07:47.857710+0000","last_active":"2023-06-03T20:11:31.884285+0000","last_peered":"2023-06-03T20:11:31.884285+0000","last_clean":"2023-06-03T20:11:31.884285+0000","last_became_active":"2023-06-03T20:07:47.857320+0000","last_became_peered":"2023-06-03T20:07:47.857320+0000","last_unstale":"2023-06-03T20:11:31.884285+0000","last_undegraded":"2023-06-03T20:11:31.884285+0000","last_fullsized":"2023-06-03T20:11:31.884285+0000","mapping_epoch":172,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":173,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":169,"reported_epoch":193,"state":"active+clean","last_fresh":"2023-06-03T20:11:30.758245+0000","last_change":"2023-06-03T20:07:19.907984+0000","last_active":"2023-06-03T20:11:30.758245+0000","last_peered":"2023-06-03T20:11:30.758245+0000","last_clean":"2023-06-03T20:11:30.758245+0000","last_became_active":"2023-06-03T20:07:19.907176+0000","last_became_peered":"2023-06-03T20:07:19.907176+0000","last_unstale":"2023-06-03T20:11:30.758245+0000","last_undegraded":"2023-06-03T20:11:30.758245+0000","last_fullsized":"2023-06-03T20:11:30.758245+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,5],"acting":[6,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"11.4","version":"194'1799","reported_seq":1812,"reported_epoch":194,"state":"active+clean","last_fresh":"2023-06-03T20:11:47.958056+0000","last_change":"2023-06-03T20:11:11.796227+0000","last_active":"2023-06-03T20:11:47.958056+0000","last_peered":"2023-06-03T20:11:47.958056+0000","last_clean":"2023-06-03T20:11:47.958056+0000","last_became_active":"2023-06-03T20:11:11.795900+0000","last_became_peered":"2023-06-03T20:11:11.795900+0000","last_unstale":"2023-06-03T20:11:47.958056+0000","last_undegraded":"2023-06-03T20:11:47.958056+0000","last_fullsized":"2023-06-03T20:11:47.958056+0000","mapping_epoch":189,"log_start":"0'0","ondisk_log_start":"0'0","created":189,"last_epoch_clean":190,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T20:11:10.759955+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T20:11:10.759955+0000","last_clean_scrub_stamp":"2023-06-03T20:11:10.759955+0000","objects_scrubbed":0,"log_size":1799,"log_dups_size":0,"ondisk_log_size":1799,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":117899264,"num_objects":1799,"num_object_clones":0,"num_object_copies":5397,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1799,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1799,"num_write_kb":115136,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,2],"acting":[7,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":160,"reported_epoch":192,"state":"active+clean","last_fresh":"2023-06-03T20:11:17.987874+0000","last_change":"2023-06-03T19:56:54.507432+0000","last_active":"2023-06-03T20:11:17.987874+0000","last_peered":"2023-06-03T20:11:17.987874+0000","last_clean":"2023-06-03T20:11:17.987874+0000","last_became_active":"2023-06-03T19:56:54.507012+0000","last_became_peered":"2023-06-03T19:56:54.507012+0000","last_unstale":"2023-06-03T20:11:17.987874+0000","last_undegraded":"2023-06-03T20:11:17.987874+0000","last_fullsized":"2023-06-03T20:11:17.987874+0000","mapping_epoch":119,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":190,"reported_epoch":193,"state":"active+clean","last_fresh":"2023-06-03T20:11:31.275655+0000","last_change":"2023-06-03T20:07:19.914491+0000","last_active":"2023-06-03T20:11:31.275655+0000","last_peered":"2023-06-03T20:11:31.275655+0000","last_clean":"2023-06-03T20:11:31.275655+0000","last_became_active":"2023-06-03T20:07:19.913953+0000","last_became_peered":"2023-06-03T20:07:19.913953+0000","last_unstale":"2023-06-03T20:11:31.275655+0000","last_undegraded":"2023-06-03T20:11:31.275655+0000","last_fullsized":"2023-06-03T20:11:31.275655+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T00:21:40.046785+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"11.3","version":"194'1860","reported_seq":1872,"reported_epoch":194,"state":"active+clean","last_fresh":"2023-06-03T20:11:47.923628+0000","last_change":"2023-06-03T20:11:11.794495+0000","last_active":"2023-06-03T20:11:47.923628+0000","last_peered":"2023-06-03T20:11:47.923628+0000","last_clean":"2023-06-03T20:11:47.923628+0000","last_became_active":"2023-06-03T20:11:11.794124+0000","last_became_peered":"2023-06-03T20:11:11.794124+0000","last_unstale":"2023-06-03T20:11:47.923628+0000","last_undegraded":"2023-06-03T20:11:47.923628+0000","last_fullsized":"2023-06-03T20:11:47.923628+0000","mapping_epoch":189,"log_start":"0'0","ondisk_log_start":"0'0","created":189,"last_epoch_clean":190,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T20:11:10.759955+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T20:11:10.759955+0000","last_clean_scrub_stamp":"2023-06-03T20:11:10.759955+0000","objects_scrubbed":0,"log_size":1860,"log_dups_size":0,"ondisk_log_size":1860,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":121896960,"num_objects":1860,"num_object_clones":0,"num_object_copies":5580,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1860,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1860,"num_write_kb":119040,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,5],"acting":[6,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.b","version":"60'2","reported_seq":164,"reported_epoch":193,"state":"active+clean","last_fresh":"2023-06-03T20:11:31.887550+0000","last_change":"2023-06-03T19:57:28.470310+0000","last_active":"2023-06-03T20:11:31.887550+0000","last_peered":"2023-06-03T20:11:31.887550+0000","last_clean":"2023-06-03T20:11:31.887550+0000","last_became_active":"2023-06-03T19:57:28.469887+0000","last_became_peered":"2023-06-03T19:57:28.469887+0000","last_unstale":"2023-06-03T20:11:31.887550+0000","last_undegraded":"2023-06-03T20:11:31.887550+0000","last_fullsized":"2023-06-03T20:11:31.887550+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T21:58:53.387131+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":172,"reported_epoch":193,"state":"active+clean","last_fresh":"2023-06-03T20:11:30.755063+0000","last_change":"2023-06-03T19:57:28.465140+0000","last_active":"2023-06-03T20:11:30.755063+0000","last_peered":"2023-06-03T20:11:30.755063+0000","last_clean":"2023-06-03T20:11:30.755063+0000","last_became_active":"2023-06-03T19:57:28.464629+0000","last_became_peered":"2023-06-03T19:57:28.464629+0000","last_unstale":"2023-06-03T20:11:30.755063+0000","last_undegraded":"2023-06-03T20:11:30.755063+0000","last_fullsized":"2023-06-03T20:11:30.755063+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T21:47:11.619583+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.2","version":"194'1447","reported_seq":1461,"reported_epoch":194,"state":"active+clean","last_fresh":"2023-06-03T20:11:44.719111+0000","last_change":"2023-06-03T20:11:11.788495+0000","last_active":"2023-06-03T20:11:44.719111+0000","last_peered":"2023-06-03T20:11:44.719111+0000","last_clean":"2023-06-03T20:11:44.719111+0000","last_became_active":"2023-06-03T20:11:11.788122+0000","last_became_peered":"2023-06-03T20:11:11.788122+0000","last_unstale":"2023-06-03T20:11:44.719111+0000","last_undegraded":"2023-06-03T20:11:44.719111+0000","last_fullsized":"2023-06-03T20:11:44.719111+0000","mapping_epoch":189,"log_start":"0'0","ondisk_log_start":"0'0","created":189,"last_epoch_clean":190,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T20:11:10.759955+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T20:11:10.759955+0000","last_clean_scrub_stamp":"2023-06-03T20:11:10.759955+0000","objects_scrubbed":0,"log_size":1447,"log_dups_size":0,"ondisk_log_size":1447,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T05:43:08.101794+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":94830592,"num_objects":1447,"num_object_clones":0,"num_object_copies":4341,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1447,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1447,"num_write_kb":92608,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":185,"reported_epoch":193,"state":"active+clean","last_fresh":"2023-06-03T20:11:30.757983+0000","last_change":"2023-06-03T20:07:19.911766+0000","last_active":"2023-06-03T20:11:30.757983+0000","last_peered":"2023-06-03T20:11:30.757983+0000","last_clean":"2023-06-03T20:11:30.757983+0000","last_became_active":"2023-06-03T20:07:19.911353+0000","last_became_peered":"2023-06-03T20:07:19.911353+0000","last_unstale":"2023-06-03T20:11:30.757983+0000","last_undegraded":"2023-06-03T20:11:30.757983+0000","last_fullsized":"2023-06-03T20:11:30.757983+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,3],"acting":[6,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":186,"reported_epoch":193,"state":"active+clean","last_fresh":"2023-06-03T20:11:30.755506+0000","last_change":"2023-06-03T20:07:19.907673+0000","last_active":"2023-06-03T20:11:30.755506+0000","last_peered":"2023-06-03T20:11:30.755506+0000","last_clean":"2023-06-03T20:11:30.755506+0000","last_became_active":"2023-06-03T20:07:19.907186+0000","last_became_peered":"2023-06-03T20:07:19.907186+0000","last_unstale":"2023-06-03T20:11:30.755506+0000","last_undegraded":"2023-06-03T20:11:30.755506+0000","last_fullsized":"2023-06-03T20:11:30.755506+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T00:08:24.015355+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.1","version":"194'1775","reported_seq":1788,"reported_epoch":194,"state":"active+clean","last_fresh":"2023-06-03T20:11:47.957988+0000","last_change":"2023-06-03T20:11:11.787443+0000","last_active":"2023-06-03T20:11:47.957988+0000","last_peered":"2023-06-03T20:11:47.957988+0000","last_clean":"2023-06-03T20:11:47.957988+0000","last_became_active":"2023-06-03T20:11:11.787117+0000","last_became_peered":"2023-06-03T20:11:11.787117+0000","last_unstale":"2023-06-03T20:11:47.957988+0000","last_undegraded":"2023-06-03T20:11:47.957988+0000","last_fullsized":"2023-06-03T20:11:47.957988+0000","mapping_epoch":189,"log_start":"0'0","ondisk_log_start":"0'0","created":189,"last_epoch_clean":190,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T20:11:10.759955+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T20:11:10.759955+0000","last_clean_scrub_stamp":"2023-06-03T20:11:10.759955+0000","objects_scrubbed":0,"log_size":1775,"log_dups_size":0,"ondisk_log_size":1775,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":116326400,"num_objects":1775,"num_object_clones":0,"num_object_copies":5325,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1775,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1775,"num_write_kb":113600,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,0],"acting":[6,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":169,"reported_epoch":193,"state":"active+clean","last_fresh":"2023-06-03T20:11:30.754568+0000","last_change":"2023-06-03T19:57:28.462344+0000","last_active":"2023-06-03T20:11:30.754568+0000","last_peered":"2023-06-03T20:11:30.754568+0000","last_clean":"2023-06-03T20:11:30.754568+0000","last_became_active":"2023-06-03T19:57:28.462001+0000","last_became_peered":"2023-06-03T19:57:28.462001+0000","last_unstale":"2023-06-03T20:11:30.754568+0000","last_undegraded":"2023-06-03T20:11:30.754568+0000","last_fullsized":"2023-06-03T20:11:30.754568+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T00:45:13.789650+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":167,"reported_epoch":193,"state":"active+clean","last_fresh":"2023-06-03T20:11:30.755811+0000","last_change":"2023-06-03T20:07:19.909344+0000","last_active":"2023-06-03T20:11:30.755811+0000","last_peered":"2023-06-03T20:11:30.755811+0000","last_clean":"2023-06-03T20:11:30.755811+0000","last_became_active":"2023-06-03T20:07:19.908845+0000","last_became_peered":"2023-06-03T20:07:19.908845+0000","last_unstale":"2023-06-03T20:11:30.755811+0000","last_undegraded":"2023-06-03T20:11:30.755811+0000","last_fullsized":"2023-06-03T20:11:30.755811+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T02:42:22.071829+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,3],"acting":[4,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.0","version":"194'1836","reported_seq":1850,"reported_epoch":194,"state":"active+clean","last_fresh":"2023-06-03T20:11:48.332616+0000","last_change":"2023-06-03T20:11:11.793945+0000","last_active":"2023-06-03T20:11:48.332616+0000","last_peered":"2023-06-03T20:11:48.332616+0000","last_clean":"2023-06-03T20:11:48.332616+0000","last_became_active":"2023-06-03T20:11:11.793560+0000","last_became_peered":"2023-06-03T20:11:11.793560+0000","last_unstale":"2023-06-03T20:11:48.332616+0000","last_undegraded":"2023-06-03T20:11:48.332616+0000","last_fullsized":"2023-06-03T20:11:48.332616+0000","mapping_epoch":189,"log_start":"0'0","ondisk_log_start":"0'0","created":189,"last_epoch_clean":190,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T20:11:10.759955+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T20:11:10.759955+0000","last_clean_scrub_stamp":"2023-06-03T20:11:10.759955+0000","objects_scrubbed":0,"log_size":1836,"log_dups_size":0,"ondisk_log_size":1836,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T03:53:55.401325+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":120324096,"num_objects":1836,"num_object_clones":0,"num_object_copies":5508,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1836,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1836,"num_write_kb":117504,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":173,"reported_epoch":193,"state":"active+clean","last_fresh":"2023-06-03T20:11:31.886659+0000","last_change":"2023-06-03T20:07:19.909283+0000","last_active":"2023-06-03T20:11:31.886659+0000","last_peered":"2023-06-03T20:11:31.886659+0000","last_clean":"2023-06-03T20:11:31.886659+0000","last_became_active":"2023-06-03T20:07:19.908902+0000","last_became_peered":"2023-06-03T20:07:19.908902+0000","last_unstale":"2023-06-03T20:11:31.886659+0000","last_undegraded":"2023-06-03T20:11:31.886659+0000","last_fullsized":"2023-06-03T20:11:31.886659+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T23:08:01.870466+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,4],"acting":[3,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":106,"reported_epoch":193,"state":"active+clean","last_fresh":"2023-06-03T20:11:31.887990+0000","last_change":"2023-06-03T20:07:19.908660+0000","last_active":"2023-06-03T20:11:31.887990+0000","last_peered":"2023-06-03T20:11:31.887990+0000","last_clean":"2023-06-03T20:11:31.887990+0000","last_became_active":"2023-06-03T20:07:19.908313+0000","last_became_peered":"2023-06-03T20:07:19.908313+0000","last_unstale":"2023-06-03T20:11:31.887990+0000","last_undegraded":"2023-06-03T20:11:31.887990+0000","last_fullsized":"2023-06-03T20:11:31.887990+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T22:08:11.385111+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,7],"acting":[3,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"11.d","version":"194'1512","reported_seq":1525,"reported_epoch":194,"state":"active+clean","last_fresh":"2023-06-03T20:11:44.730832+0000","last_change":"2023-06-03T20:11:11.790003+0000","last_active":"2023-06-03T20:11:44.730832+0000","last_peered":"2023-06-03T20:11:44.730832+0000","last_clean":"2023-06-03T20:11:44.730832+0000","last_became_active":"2023-06-03T20:11:11.789662+0000","last_became_peered":"2023-06-03T20:11:11.789662+0000","last_unstale":"2023-06-03T20:11:44.730832+0000","last_undegraded":"2023-06-03T20:11:44.730832+0000","last_fullsized":"2023-06-03T20:11:44.730832+0000","mapping_epoch":189,"log_start":"0'0","ondisk_log_start":"0'0","created":189,"last_epoch_clean":190,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T20:11:10.759955+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T20:11:10.759955+0000","last_clean_scrub_stamp":"2023-06-03T20:11:10.759955+0000","objects_scrubbed":0,"log_size":1512,"log_dups_size":0,"ondisk_log_size":1512,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T22:42:13.193745+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":99090432,"num_objects":1512,"num_object_clones":0,"num_object_copies":4536,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1512,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1512,"num_write_kb":96768,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,6],"acting":[2,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":103,"reported_epoch":193,"state":"active+clean","last_fresh":"2023-06-03T20:11:31.275974+0000","last_change":"2023-06-03T20:07:19.926958+0000","last_active":"2023-06-03T20:11:31.275974+0000","last_peered":"2023-06-03T20:11:31.275974+0000","last_clean":"2023-06-03T20:11:31.275974+0000","last_became_active":"2023-06-03T20:07:19.926456+0000","last_became_peered":"2023-06-03T20:07:19.926456+0000","last_unstale":"2023-06-03T20:11:31.275974+0000","last_undegraded":"2023-06-03T20:11:31.275974+0000","last_fullsized":"2023-06-03T20:11:31.275974+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:32:20.692043+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:32:20.692043+0000","last_clean_scrub_stamp":"2023-06-03T19:32:20.692043+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T20:47:07.666782+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,2],"acting":[0,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":175,"reported_epoch":192,"state":"active+clean","last_fresh":"2023-06-03T20:11:17.988234+0000","last_change":"2023-06-03T19:56:54.512607+0000","last_active":"2023-06-03T20:11:17.988234+0000","last_peered":"2023-06-03T20:11:17.988234+0000","last_clean":"2023-06-03T20:11:17.988234+0000","last_became_active":"2023-06-03T19:56:54.512282+0000","last_became_peered":"2023-06-03T19:56:54.512282+0000","last_unstale":"2023-06-03T20:11:17.988234+0000","last_undegraded":"2023-06-03T20:11:17.988234+0000","last_fullsized":"2023-06-03T20:11:17.988234+0000","mapping_epoch":119,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":89,"reported_epoch":192,"state":"active+clean","last_fresh":"2023-06-03T20:11:17.986629+0000","last_change":"2023-06-03T20:07:19.912018+0000","last_active":"2023-06-03T20:11:17.986629+0000","last_peered":"2023-06-03T20:11:17.986629+0000","last_clean":"2023-06-03T20:11:17.986629+0000","last_became_active":"2023-06-03T20:07:19.911682+0000","last_became_peered":"2023-06-03T20:07:19.911682+0000","last_unstale":"2023-06-03T20:11:17.986629+0000","last_undegraded":"2023-06-03T20:11:17.986629+0000","last_fullsized":"2023-06-03T20:11:17.986629+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,0],"acting":[5,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"11.b","version":"194'1856","reported_seq":1870,"reported_epoch":194,"state":"active+clean","last_fresh":"2023-06-03T20:11:48.341829+0000","last_change":"2023-06-03T20:11:11.790129+0000","last_active":"2023-06-03T20:11:48.341829+0000","last_peered":"2023-06-03T20:11:48.341829+0000","last_clean":"2023-06-03T20:11:48.341829+0000","last_became_active":"2023-06-03T20:11:11.789729+0000","last_became_peered":"2023-06-03T20:11:11.789729+0000","last_unstale":"2023-06-03T20:11:48.341829+0000","last_undegraded":"2023-06-03T20:11:48.341829+0000","last_fullsized":"2023-06-03T20:11:48.341829+0000","mapping_epoch":189,"log_start":"0'0","ondisk_log_start":"0'0","created":189,"last_epoch_clean":190,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T20:11:10.759955+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T20:11:10.759955+0000","last_clean_scrub_stamp":"2023-06-03T20:11:10.759955+0000","objects_scrubbed":0,"log_size":1856,"log_dups_size":0,"ondisk_log_size":1856,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T04:59:39.200434+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":121634816,"num_objects":1856,"num_object_clones":0,"num_object_copies":5568,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1856,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1856,"num_write_kb":118784,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":173,"reported_epoch":192,"state":"active+clean","last_fresh":"2023-06-03T20:11:17.986928+0000","last_change":"2023-06-03T20:07:19.906785+0000","last_active":"2023-06-03T20:11:17.986928+0000","last_peered":"2023-06-03T20:11:17.986928+0000","last_clean":"2023-06-03T20:11:17.986928+0000","last_became_active":"2023-06-03T20:07:19.906412+0000","last_became_peered":"2023-06-03T20:07:19.906412+0000","last_unstale":"2023-06-03T20:11:17.986928+0000","last_undegraded":"2023-06-03T20:11:17.986928+0000","last_fullsized":"2023-06-03T20:11:17.986928+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:32:20.692043+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:32:20.692043+0000","last_clean_scrub_stamp":"2023-06-03T19:32:20.692043+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,0],"acting":[5,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":142,"reported_epoch":193,"state":"active+clean","last_fresh":"2023-06-03T20:11:30.755151+0000","last_change":"2023-06-03T19:57:28.460195+0000","last_active":"2023-06-03T20:11:30.755151+0000","last_peered":"2023-06-03T20:11:30.755151+0000","last_clean":"2023-06-03T20:11:30.755151+0000","last_became_active":"2023-06-03T19:57:28.459823+0000","last_became_peered":"2023-06-03T19:57:28.459823+0000","last_unstale":"2023-06-03T20:11:30.755151+0000","last_undegraded":"2023-06-03T20:11:30.755151+0000","last_fullsized":"2023-06-03T20:11:30.755151+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T20:08:48.686380+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":102,"reported_epoch":193,"state":"active+clean","last_fresh":"2023-06-03T20:11:31.886161+0000","last_change":"2023-06-03T20:07:19.900602+0000","last_active":"2023-06-03T20:11:31.886161+0000","last_peered":"2023-06-03T20:11:31.886161+0000","last_clean":"2023-06-03T20:11:31.886161+0000","last_became_active":"2023-06-03T20:07:19.900046+0000","last_became_peered":"2023-06-03T20:07:19.900046+0000","last_unstale":"2023-06-03T20:11:31.886161+0000","last_undegraded":"2023-06-03T20:11:31.886161+0000","last_fullsized":"2023-06-03T20:11:31.886161+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T00:52:57.353642+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,6],"acting":[2,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"11.8","version":"194'1905","reported_seq":1919,"reported_epoch":194,"state":"active+clean","last_fresh":"2023-06-03T20:11:48.338583+0000","last_change":"2023-06-03T20:11:11.793649+0000","last_active":"2023-06-03T20:11:48.338583+0000","last_peered":"2023-06-03T20:11:48.338583+0000","last_clean":"2023-06-03T20:11:48.338583+0000","last_became_active":"2023-06-03T20:11:11.793292+0000","last_became_peered":"2023-06-03T20:11:11.793292+0000","last_unstale":"2023-06-03T20:11:48.338583+0000","last_undegraded":"2023-06-03T20:11:48.338583+0000","last_fullsized":"2023-06-03T20:11:48.338583+0000","mapping_epoch":189,"log_start":"0'0","ondisk_log_start":"0'0","created":189,"last_epoch_clean":190,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T20:11:10.759955+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T20:11:10.759955+0000","last_clean_scrub_stamp":"2023-06-03T20:11:10.759955+0000","objects_scrubbed":0,"log_size":1905,"log_dups_size":0,"ondisk_log_size":1905,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T07:27:16.604437+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":124846080,"num_objects":1905,"num_object_clones":0,"num_object_copies":5715,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1905,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1905,"num_write_kb":121920,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,5],"acting":[3,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":172,"reported_epoch":193,"state":"active+clean","last_fresh":"2023-06-03T20:11:31.885998+0000","last_change":"2023-06-03T19:56:54.504859+0000","last_active":"2023-06-03T20:11:31.885998+0000","last_peered":"2023-06-03T20:11:31.885998+0000","last_clean":"2023-06-03T20:11:31.885998+0000","last_became_active":"2023-06-03T19:56:54.504455+0000","last_became_peered":"2023-06-03T19:56:54.504455+0000","last_unstale":"2023-06-03T20:11:31.885998+0000","last_undegraded":"2023-06-03T20:11:31.885998+0000","last_fullsized":"2023-06-03T20:11:31.885998+0000","mapping_epoch":119,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:32:20.692043+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:32:20.692043+0000","last_clean_scrub_stamp":"2023-06-03T19:32:20.692043+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T06:25:45.522100+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":123,"reported_epoch":193,"state":"active+clean","last_fresh":"2023-06-03T20:11:31.886171+0000","last_change":"2023-06-03T20:07:19.906703+0000","last_active":"2023-06-03T20:11:31.886171+0000","last_peered":"2023-06-03T20:11:31.886171+0000","last_clean":"2023-06-03T20:11:31.886171+0000","last_became_active":"2023-06-03T20:07:19.906373+0000","last_became_peered":"2023-06-03T20:07:19.906373+0000","last_unstale":"2023-06-03T20:11:31.886171+0000","last_undegraded":"2023-06-03T20:11:31.886171+0000","last_fullsized":"2023-06-03T20:11:31.886171+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T21:06:05.945328+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6,0],"acting":[2,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":95,"reported_epoch":192,"state":"active+clean","last_fresh":"2023-06-03T20:11:17.988722+0000","last_change":"2023-06-03T20:07:19.906757+0000","last_active":"2023-06-03T20:11:17.988722+0000","last_peered":"2023-06-03T20:11:17.988722+0000","last_clean":"2023-06-03T20:11:17.988722+0000","last_became_active":"2023-06-03T20:07:19.906380+0000","last_became_peered":"2023-06-03T20:07:19.906380+0000","last_unstale":"2023-06-03T20:11:17.988722+0000","last_undegraded":"2023-06-03T20:11:17.988722+0000","last_fullsized":"2023-06-03T20:11:17.988722+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,7],"acting":[5,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"11.a","version":"194'1839","reported_seq":1853,"reported_epoch":194,"state":"active+clean","last_fresh":"2023-06-03T20:11:48.353769+0000","last_change":"2023-06-03T20:11:11.793791+0000","last_active":"2023-06-03T20:11:48.353769+0000","last_peered":"2023-06-03T20:11:48.353769+0000","last_clean":"2023-06-03T20:11:48.353769+0000","last_became_active":"2023-06-03T20:11:11.793127+0000","last_became_peered":"2023-06-03T20:11:11.793127+0000","last_unstale":"2023-06-03T20:11:48.353769+0000","last_undegraded":"2023-06-03T20:11:48.353769+0000","last_fullsized":"2023-06-03T20:11:48.353769+0000","mapping_epoch":189,"log_start":"0'0","ondisk_log_start":"0'0","created":189,"last_epoch_clean":190,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T20:11:10.759955+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T20:11:10.759955+0000","last_clean_scrub_stamp":"2023-06-03T20:11:10.759955+0000","objects_scrubbed":0,"log_size":1839,"log_dups_size":0,"ondisk_log_size":1839,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T22:21:19.854335+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":120520704,"num_objects":1839,"num_object_clones":0,"num_object_copies":5517,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1839,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1839,"num_write_kb":117696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,5],"acting":[3,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":164,"reported_epoch":192,"state":"active+clean","last_fresh":"2023-06-03T20:11:17.987551+0000","last_change":"2023-06-03T19:56:11.752151+0000","last_active":"2023-06-03T20:11:17.987551+0000","last_peered":"2023-06-03T20:11:17.987551+0000","last_clean":"2023-06-03T20:11:17.987551+0000","last_became_active":"2023-06-03T19:56:11.751767+0000","last_became_peered":"2023-06-03T19:56:11.751767+0000","last_unstale":"2023-06-03T20:11:17.987551+0000","last_undegraded":"2023-06-03T20:11:17.987551+0000","last_fullsized":"2023-06-03T20:11:17.987551+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":115,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:32:20.692043+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:32:20.692043+0000","last_clean_scrub_stamp":"2023-06-03T19:32:20.692043+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"72'139","reported_seq":394,"reported_epoch":193,"state":"active+clean","last_fresh":"2023-06-03T20:11:31.883989+0000","last_change":"2023-06-03T20:07:47.872942+0000","last_active":"2023-06-03T20:11:31.883989+0000","last_peered":"2023-06-03T20:11:31.883989+0000","last_clean":"2023-06-03T20:11:31.883989+0000","last_became_active":"2023-06-03T20:07:47.872426+0000","last_became_peered":"2023-06-03T20:07:47.872426+0000","last_unstale":"2023-06-03T20:11:31.883989+0000","last_undegraded":"2023-06-03T20:11:31.883989+0000","last_fullsized":"2023-06-03T20:11:31.883989+0000","mapping_epoch":172,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":173,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:27:28.100808+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:27:28.100808+0000","last_clean_scrub_stamp":"2023-06-03T19:27:28.100808+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":145,"reported_epoch":193,"state":"active+clean","last_fresh":"2023-06-03T20:11:31.886985+0000","last_change":"2023-06-03T19:56:54.509571+0000","last_active":"2023-06-03T20:11:31.886985+0000","last_peered":"2023-06-03T20:11:31.886985+0000","last_clean":"2023-06-03T20:11:31.886985+0000","last_became_active":"2023-06-03T19:56:54.509213+0000","last_became_peered":"2023-06-03T19:56:54.509213+0000","last_unstale":"2023-06-03T20:11:31.886985+0000","last_undegraded":"2023-06-03T20:11:31.886985+0000","last_fullsized":"2023-06-03T20:11:31.886985+0000","mapping_epoch":119,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T21:26:56.003203+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":160,"reported_epoch":193,"state":"active+clean","last_fresh":"2023-06-03T20:11:30.758492+0000","last_change":"2023-06-03T19:57:28.459664+0000","last_active":"2023-06-03T20:11:30.758492+0000","last_peered":"2023-06-03T20:11:30.758492+0000","last_clean":"2023-06-03T20:11:30.758492+0000","last_became_active":"2023-06-03T19:57:28.459289+0000","last_became_peered":"2023-06-03T19:57:28.459289+0000","last_unstale":"2023-06-03T20:11:30.758492+0000","last_undegraded":"2023-06-03T20:11:30.758492+0000","last_fullsized":"2023-06-03T20:11:30.758492+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"11.9","version":"194'1912","reported_seq":1926,"reported_epoch":194,"state":"active+clean","last_fresh":"2023-06-03T20:11:48.341542+0000","last_change":"2023-06-03T20:11:11.792411+0000","last_active":"2023-06-03T20:11:48.341542+0000","last_peered":"2023-06-03T20:11:48.341542+0000","last_clean":"2023-06-03T20:11:48.341542+0000","last_became_active":"2023-06-03T20:11:11.792109+0000","last_became_peered":"2023-06-03T20:11:11.792109+0000","last_unstale":"2023-06-03T20:11:48.341542+0000","last_undegraded":"2023-06-03T20:11:48.341542+0000","last_fullsized":"2023-06-03T20:11:48.341542+0000","mapping_epoch":189,"log_start":"0'0","ondisk_log_start":"0'0","created":189,"last_epoch_clean":190,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T20:11:10.759955+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T20:11:10.759955+0000","last_clean_scrub_stamp":"2023-06-03T20:11:10.759955+0000","objects_scrubbed":0,"log_size":1912,"log_dups_size":0,"ondisk_log_size":1912,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T02:10:39.470012+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":125304832,"num_objects":1912,"num_object_clones":0,"num_object_copies":5736,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1912,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1912,"num_write_kb":122368,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,0],"acting":[3,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":177,"reported_epoch":193,"state":"active+clean","last_fresh":"2023-06-03T20:11:31.883861+0000","last_change":"2023-06-03T20:07:47.855767+0000","last_active":"2023-06-03T20:11:31.883861+0000","last_peered":"2023-06-03T20:11:31.883861+0000","last_clean":"2023-06-03T20:11:31.883861+0000","last_became_active":"2023-06-03T20:07:47.855444+0000","last_became_peered":"2023-06-03T20:07:47.855444+0000","last_unstale":"2023-06-03T20:11:31.883861+0000","last_undegraded":"2023-06-03T20:11:31.883861+0000","last_fullsized":"2023-06-03T20:11:31.883861+0000","mapping_epoch":172,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":173,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:32:20.692043+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:32:20.692043+0000","last_clean_scrub_stamp":"2023-06-03T19:32:20.692043+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,3],"acting":[7,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":169,"reported_epoch":193,"state":"active+clean","last_fresh":"2023-06-03T20:11:31.275062+0000","last_change":"2023-06-03T19:57:28.459611+0000","last_active":"2023-06-03T20:11:31.275062+0000","last_peered":"2023-06-03T20:11:31.275062+0000","last_clean":"2023-06-03T20:11:31.275062+0000","last_became_active":"2023-06-03T19:57:28.459264+0000","last_became_peered":"2023-06-03T19:57:28.459264+0000","last_unstale":"2023-06-03T20:11:31.275062+0000","last_undegraded":"2023-06-03T20:11:31.275062+0000","last_fullsized":"2023-06-03T20:11:31.275062+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T22:15:06.557231+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":179,"reported_epoch":193,"state":"active+clean","last_fresh":"2023-06-03T20:11:31.275875+0000","last_change":"2023-06-03T20:07:19.925864+0000","last_active":"2023-06-03T20:11:31.275875+0000","last_peered":"2023-06-03T20:11:31.275875+0000","last_clean":"2023-06-03T20:11:31.275875+0000","last_became_active":"2023-06-03T20:07:19.925541+0000","last_became_peered":"2023-06-03T20:07:19.925541+0000","last_unstale":"2023-06-03T20:11:31.275875+0000","last_undegraded":"2023-06-03T20:11:31.275875+0000","last_fullsized":"2023-06-03T20:11:31.275875+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T00:10:11.790831+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,7],"acting":[0,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"11.c","version":"194'1485","reported_seq":1499,"reported_epoch":194,"state":"active+clean","last_fresh":"2023-06-03T20:11:44.731450+0000","last_change":"2023-06-03T20:11:11.792633+0000","last_active":"2023-06-03T20:11:44.731450+0000","last_peered":"2023-06-03T20:11:44.731450+0000","last_clean":"2023-06-03T20:11:44.731450+0000","last_became_active":"2023-06-03T20:11:11.792266+0000","last_became_peered":"2023-06-03T20:11:11.792266+0000","last_unstale":"2023-06-03T20:11:44.731450+0000","last_undegraded":"2023-06-03T20:11:44.731450+0000","last_fullsized":"2023-06-03T20:11:44.731450+0000","mapping_epoch":189,"log_start":"0'0","ondisk_log_start":"0'0","created":189,"last_epoch_clean":190,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T20:11:10.759955+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T20:11:10.759955+0000","last_clean_scrub_stamp":"2023-06-03T20:11:10.759955+0000","objects_scrubbed":0,"log_size":1485,"log_dups_size":0,"ondisk_log_size":1485,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T07:35:53.770713+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":97320960,"num_objects":1485,"num_object_clones":0,"num_object_copies":4455,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1485,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1485,"num_write_kb":95040,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,7],"acting":[2,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":180,"reported_epoch":193,"state":"active+clean","last_fresh":"2023-06-03T20:11:31.884863+0000","last_change":"2023-06-03T19:57:28.466096+0000","last_active":"2023-06-03T20:11:31.884863+0000","last_peered":"2023-06-03T20:11:31.884863+0000","last_clean":"2023-06-03T20:11:31.884863+0000","last_became_active":"2023-06-03T19:57:28.464954+0000","last_became_peered":"2023-06-03T19:57:28.464954+0000","last_unstale":"2023-06-03T20:11:31.884863+0000","last_undegraded":"2023-06-03T20:11:31.884863+0000","last_fullsized":"2023-06-03T20:11:31.884863+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:32:20.692043+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:32:20.692043+0000","last_clean_scrub_stamp":"2023-06-03T19:32:20.692043+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":123,"reported_epoch":193,"state":"active+clean","last_fresh":"2023-06-03T20:11:31.886381+0000","last_change":"2023-06-03T20:07:19.915065+0000","last_active":"2023-06-03T20:11:31.886381+0000","last_peered":"2023-06-03T20:11:31.886381+0000","last_clean":"2023-06-03T20:11:31.886381+0000","last_became_active":"2023-06-03T20:07:19.914699+0000","last_became_peered":"2023-06-03T20:07:19.914699+0000","last_unstale":"2023-06-03T20:11:31.886381+0000","last_undegraded":"2023-06-03T20:11:31.886381+0000","last_fullsized":"2023-06-03T20:11:31.886381+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T07:16:09.077184+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,0],"acting":[2,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":140,"reported_epoch":193,"state":"active+clean","last_fresh":"2023-06-03T20:11:31.275505+0000","last_change":"2023-06-03T19:47:06.733532+0000","last_active":"2023-06-03T20:11:31.275505+0000","last_peered":"2023-06-03T20:11:31.275505+0000","last_clean":"2023-06-03T20:11:31.275505+0000","last_became_active":"2023-06-03T19:47:06.733131+0000","last_became_peered":"2023-06-03T19:47:06.733131+0000","last_unstale":"2023-06-03T20:11:31.275505+0000","last_undegraded":"2023-06-03T20:11:31.275505+0000","last_fullsized":"2023-06-03T20:11:31.275505+0000","mapping_epoch":86,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":87,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T23:46:03.788646+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"11.f","version":"194'1777","reported_seq":1790,"reported_epoch":194,"state":"active+clean","last_fresh":"2023-06-03T20:11:47.939408+0000","last_change":"2023-06-03T20:11:11.794557+0000","last_active":"2023-06-03T20:11:47.939408+0000","last_peered":"2023-06-03T20:11:47.939408+0000","last_clean":"2023-06-03T20:11:47.939408+0000","last_became_active":"2023-06-03T20:11:11.794234+0000","last_became_peered":"2023-06-03T20:11:11.794234+0000","last_unstale":"2023-06-03T20:11:47.939408+0000","last_undegraded":"2023-06-03T20:11:47.939408+0000","last_fullsized":"2023-06-03T20:11:47.939408+0000","mapping_epoch":189,"log_start":"0'0","ondisk_log_start":"0'0","created":189,"last_epoch_clean":190,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T20:11:10.759955+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T20:11:10.759955+0000","last_clean_scrub_stamp":"2023-06-03T20:11:10.759955+0000","objects_scrubbed":0,"log_size":1777,"log_dups_size":0,"ondisk_log_size":1777,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":116457472,"num_objects":1777,"num_object_clones":0,"num_object_copies":5331,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1777,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1777,"num_write_kb":113728,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,0],"acting":[7,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.7","version":"60'2","reported_seq":149,"reported_epoch":193,"state":"active+clean","last_fresh":"2023-06-03T20:11:31.888051+0000","last_change":"2023-06-03T19:56:54.510148+0000","last_active":"2023-06-03T20:11:31.888051+0000","last_peered":"2023-06-03T20:11:31.888051+0000","last_clean":"2023-06-03T20:11:31.888051+0000","last_became_active":"2023-06-03T19:56:54.509771+0000","last_became_peered":"2023-06-03T19:56:54.509771+0000","last_unstale":"2023-06-03T20:11:31.888051+0000","last_undegraded":"2023-06-03T20:11:31.888051+0000","last_fullsized":"2023-06-03T20:11:31.888051+0000","mapping_epoch":119,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T00:52:42.807432+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":131,"reported_epoch":193,"state":"active+clean","last_fresh":"2023-06-03T20:11:30.758593+0000","last_change":"2023-06-03T20:07:19.911591+0000","last_active":"2023-06-03T20:11:30.758593+0000","last_peered":"2023-06-03T20:11:30.758593+0000","last_clean":"2023-06-03T20:11:30.758593+0000","last_became_active":"2023-06-03T20:07:19.911172+0000","last_became_peered":"2023-06-03T20:07:19.911172+0000","last_unstale":"2023-06-03T20:11:30.758593+0000","last_undegraded":"2023-06-03T20:11:30.758593+0000","last_fullsized":"2023-06-03T20:11:30.758593+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:32:20.692043+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:32:20.692043+0000","last_clean_scrub_stamp":"2023-06-03T19:32:20.692043+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,3],"acting":[6,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":147,"reported_epoch":193,"state":"active+clean","last_fresh":"2023-06-03T20:11:31.886557+0000","last_change":"2023-06-03T19:56:54.507670+0000","last_active":"2023-06-03T20:11:31.886557+0000","last_peered":"2023-06-03T20:11:31.886557+0000","last_clean":"2023-06-03T20:11:31.886557+0000","last_became_active":"2023-06-03T19:56:54.507293+0000","last_became_peered":"2023-06-03T19:56:54.507293+0000","last_unstale":"2023-06-03T20:11:31.886557+0000","last_undegraded":"2023-06-03T20:11:31.886557+0000","last_fullsized":"2023-06-03T20:11:31.886557+0000","mapping_epoch":119,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T01:49:19.966054+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"11.e","version":"194'1784","reported_seq":1798,"reported_epoch":194,"state":"active+clean","last_fresh":"2023-06-03T20:11:47.165398+0000","last_change":"2023-06-03T20:11:11.790808+0000","last_active":"2023-06-03T20:11:47.165398+0000","last_peered":"2023-06-03T20:11:47.165398+0000","last_clean":"2023-06-03T20:11:47.165398+0000","last_became_active":"2023-06-03T20:11:11.790533+0000","last_became_peered":"2023-06-03T20:11:11.790533+0000","last_unstale":"2023-06-03T20:11:47.165398+0000","last_undegraded":"2023-06-03T20:11:47.165398+0000","last_fullsized":"2023-06-03T20:11:47.165398+0000","mapping_epoch":189,"log_start":"0'0","ondisk_log_start":"0'0","created":189,"last_epoch_clean":190,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T20:11:10.759955+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T20:11:10.759955+0000","last_clean_scrub_stamp":"2023-06-03T20:11:10.759955+0000","objects_scrubbed":0,"log_size":1784,"log_dups_size":0,"ondisk_log_size":1784,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T20:55:44.304940+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":116916224,"num_objects":1784,"num_object_clones":0,"num_object_copies":5352,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1784,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1784,"num_write_kb":114176,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":165,"reported_epoch":193,"state":"active+clean","last_fresh":"2023-06-03T20:11:30.758618+0000","last_change":"2023-06-03T19:56:11.752730+0000","last_active":"2023-06-03T20:11:30.758618+0000","last_peered":"2023-06-03T20:11:30.758618+0000","last_clean":"2023-06-03T20:11:30.758618+0000","last_became_active":"2023-06-03T19:56:11.752441+0000","last_became_peered":"2023-06-03T19:56:11.752441+0000","last_unstale":"2023-06-03T20:11:30.758618+0000","last_undegraded":"2023-06-03T20:11:30.758618+0000","last_fullsized":"2023-06-03T20:11:30.758618+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":115,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:32:20.692043+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:32:20.692043+0000","last_clean_scrub_stamp":"2023-06-03T19:32:20.692043+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"60'1","reported_seq":186,"reported_epoch":193,"state":"active+clean","last_fresh":"2023-06-03T20:11:31.275743+0000","last_change":"2023-06-03T20:07:19.965503+0000","last_active":"2023-06-03T20:11:31.275743+0000","last_peered":"2023-06-03T20:11:31.275743+0000","last_clean":"2023-06-03T20:11:31.275743+0000","last_became_active":"2023-06-03T20:07:19.913879+0000","last_became_peered":"2023-06-03T20:07:19.913879+0000","last_unstale":"2023-06-03T20:11:31.275743+0000","last_undegraded":"2023-06-03T20:11:31.275743+0000","last_fullsized":"2023-06-03T20:11:31.275743+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T20:37:07.402876+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":140,"reported_epoch":193,"state":"active+clean","last_fresh":"2023-06-03T20:11:30.754373+0000","last_change":"2023-06-03T19:57:28.456467+0000","last_active":"2023-06-03T20:11:30.754373+0000","last_peered":"2023-06-03T20:11:30.754373+0000","last_clean":"2023-06-03T20:11:30.754373+0000","last_became_active":"2023-06-03T19:57:28.456120+0000","last_became_peered":"2023-06-03T19:57:28.456120+0000","last_unstale":"2023-06-03T20:11:30.754373+0000","last_undegraded":"2023-06-03T20:11:30.754373+0000","last_fullsized":"2023-06-03T20:11:30.754373+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T07:28:14.306624+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"60'1","reported_seq":156,"reported_epoch":193,"state":"active+clean","last_fresh":"2023-06-03T20:11:31.275766+0000","last_change":"2023-06-03T19:57:28.465124+0000","last_active":"2023-06-03T20:11:31.275766+0000","last_peered":"2023-06-03T20:11:31.275766+0000","last_clean":"2023-06-03T20:11:31.275766+0000","last_became_active":"2023-06-03T19:57:28.464654+0000","last_became_peered":"2023-06-03T19:57:28.464654+0000","last_unstale":"2023-06-03T20:11:31.275766+0000","last_undegraded":"2023-06-03T20:11:31.275766+0000","last_fullsized":"2023-06-03T20:11:31.275766+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T01:12:32.305507+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":192,"reported_epoch":193,"state":"active+clean","last_fresh":"2023-06-03T20:11:31.886776+0000","last_change":"2023-06-03T20:07:19.914985+0000","last_active":"2023-06-03T20:11:31.886776+0000","last_peered":"2023-06-03T20:11:31.886776+0000","last_clean":"2023-06-03T20:11:31.886776+0000","last_became_active":"2023-06-03T20:07:19.914624+0000","last_became_peered":"2023-06-03T20:07:19.914624+0000","last_unstale":"2023-06-03T20:11:31.886776+0000","last_undegraded":"2023-06-03T20:11:31.886776+0000","last_fullsized":"2023-06-03T20:11:31.886776+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T07:02:50.515170+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"60'2","reported_seq":190,"reported_epoch":192,"state":"active+clean","last_fresh":"2023-06-03T20:11:17.987999+0000","last_change":"2023-06-03T20:07:19.960260+0000","last_active":"2023-06-03T20:11:17.987999+0000","last_peered":"2023-06-03T20:11:17.987999+0000","last_clean":"2023-06-03T20:11:17.987999+0000","last_became_active":"2023-06-03T20:07:19.911480+0000","last_became_peered":"2023-06-03T20:07:19.911480+0000","last_unstale":"2023-06-03T20:11:17.987999+0000","last_undegraded":"2023-06-03T20:11:17.987999+0000","last_fullsized":"2023-06-03T20:11:17.987999+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":195,"reported_epoch":193,"state":"active+clean","last_fresh":"2023-06-03T20:11:30.754991+0000","last_change":"2023-06-03T20:07:19.902344+0000","last_active":"2023-06-03T20:11:30.754991+0000","last_peered":"2023-06-03T20:11:30.754991+0000","last_clean":"2023-06-03T20:11:30.754991+0000","last_became_active":"2023-06-03T20:07:19.901903+0000","last_became_peered":"2023-06-03T20:07:19.901903+0000","last_unstale":"2023-06-03T20:11:30.754991+0000","last_undegraded":"2023-06-03T20:11:30.754991+0000","last_fullsized":"2023-06-03T20:11:30.754991+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T23:17:54.574533+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[4,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"60'2","reported_seq":162,"reported_epoch":192,"state":"active+clean","last_fresh":"2023-06-03T20:11:17.988176+0000","last_change":"2023-06-03T19:57:28.462610+0000","last_active":"2023-06-03T20:11:17.988176+0000","last_peered":"2023-06-03T20:11:17.988176+0000","last_clean":"2023-06-03T20:11:17.988176+0000","last_became_active":"2023-06-03T19:57:28.460655+0000","last_became_peered":"2023-06-03T19:57:28.460655+0000","last_unstale":"2023-06-03T20:11:17.988176+0000","last_undegraded":"2023-06-03T20:11:17.988176+0000","last_fullsized":"2023-06-03T20:11:17.988176+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":186,"reported_epoch":193,"state":"active+clean","last_fresh":"2023-06-03T20:11:30.754671+0000","last_change":"2023-06-03T19:57:28.464690+0000","last_active":"2023-06-03T20:11:30.754671+0000","last_peered":"2023-06-03T20:11:30.754671+0000","last_clean":"2023-06-03T20:11:30.754671+0000","last_became_active":"2023-06-03T19:57:28.463856+0000","last_became_peered":"2023-06-03T19:57:28.463856+0000","last_unstale":"2023-06-03T20:11:30.754671+0000","last_undegraded":"2023-06-03T20:11:30.754671+0000","last_fullsized":"2023-06-03T20:11:30.754671+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T00:11:00.991088+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"60'1","reported_seq":176,"reported_epoch":193,"state":"active+clean","last_fresh":"2023-06-03T20:11:31.887862+0000","last_change":"2023-06-03T19:56:54.505096+0000","last_active":"2023-06-03T20:11:31.887862+0000","last_peered":"2023-06-03T20:11:31.887862+0000","last_clean":"2023-06-03T20:11:31.887862+0000","last_became_active":"2023-06-03T19:56:54.504255+0000","last_became_peered":"2023-06-03T19:56:54.504255+0000","last_unstale":"2023-06-03T20:11:31.887862+0000","last_undegraded":"2023-06-03T20:11:31.887862+0000","last_fullsized":"2023-06-03T20:11:31.887862+0000","mapping_epoch":119,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T19:52:25.922308+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":188,"reported_epoch":193,"state":"active+clean","last_fresh":"2023-06-03T20:11:31.886995+0000","last_change":"2023-06-03T19:56:54.503416+0000","last_active":"2023-06-03T20:11:31.886995+0000","last_peered":"2023-06-03T20:11:31.886995+0000","last_clean":"2023-06-03T20:11:31.886995+0000","last_became_active":"2023-06-03T19:56:54.503018+0000","last_became_peered":"2023-06-03T19:56:54.503018+0000","last_unstale":"2023-06-03T20:11:31.886995+0000","last_undegraded":"2023-06-03T20:11:31.886995+0000","last_fullsized":"2023-06-03T20:11:31.886995+0000","mapping_epoch":119,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T20:14:00.330616+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"60'4","reported_seq":158,"reported_epoch":193,"state":"active+clean","last_fresh":"2023-06-03T20:11:31.276058+0000","last_change":"2023-06-03T19:56:11.748116+0000","last_active":"2023-06-03T20:11:31.276058+0000","last_peered":"2023-06-03T20:11:31.276058+0000","last_clean":"2023-06-03T20:11:31.276058+0000","last_became_active":"2023-06-03T19:56:11.747636+0000","last_became_peered":"2023-06-03T19:56:11.747636+0000","last_unstale":"2023-06-03T20:11:31.276058+0000","last_undegraded":"2023-06-03T20:11:31.276058+0000","last_fullsized":"2023-06-03T20:11:31.276058+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":115,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T06:57:48.052055+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":175,"reported_epoch":193,"state":"active+clean","last_fresh":"2023-06-03T20:11:30.754237+0000","last_change":"2023-06-03T20:07:23.956460+0000","last_active":"2023-06-03T20:11:30.754237+0000","last_peered":"2023-06-03T20:11:30.754237+0000","last_clean":"2023-06-03T20:11:30.754237+0000","last_became_active":"2023-06-03T20:07:23.956110+0000","last_became_peered":"2023-06-03T20:07:23.956110+0000","last_unstale":"2023-06-03T20:11:30.754237+0000","last_undegraded":"2023-06-03T20:11:30.754237+0000","last_fullsized":"2023-06-03T20:11:30.754237+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T19:35:07.921742+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":11,"num_pg":16,"stat_sum":{"num_bytes":1808334848,"num_objects":27593,"num_object_clones":0,"num_object_copies":82779,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":27593,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":27593,"num_write_kb":1765952,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":5331288064,"data_stored":5331288064,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":27593,"ondisk_log_size":27593,"up":48,"acting":48,"num_store_stats":7},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5486,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":62,"num_write_kb":30,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16458,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":30319,"internal_metadata":0},"log_size":56,"ondisk_log_size":56,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":3,"num_store_stats":3},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":48,"seq":206158430697,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1311464,"kb_used_data":796960,"kb_used_omap":0,"kb_used_meta":514496,"kb_avail":92433688,"statfs":{"total":95995035648,"available":94652096512,"internally_reserved":0,"allocated":816087040,"data_stored":815579491,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":526843904},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 3 20:10:54 2023","interfaces":[{"interface":"back","average":{"1min":1.411,"5min":2.5750000000000002,"15min":1.5609999999999999},"min":{"1min":0.42299999999999999,"5min":0.26200000000000001,"15min":0.25900000000000001},"max":{"1min":6.5289999999999999,"5min":48.338999999999999,"15min":48.338999999999999},"last":2.0619999999999998},{"interface":"front","average":{"1min":1.371,"5min":1.8939999999999999,"15min":1.494},"min":{"1min":0.33600000000000002,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":4.9630000000000001,"5min":46.412999999999997,"15min":46.412999999999997},"last":0.42299999999999999}]},{"osd":1,"last update":"Sat Jun 3 20:11:41 2023","interfaces":[{"interface":"back","average":{"1min":0.90000000000000002,"5min":1.321,"15min":1.6299999999999999},"min":{"1min":0.39700000000000002,"5min":0.30099999999999999,"15min":0.29699999999999999},"max":{"1min":1.302,"5min":11.4,"15min":91.811000000000007},"last":2.0299999999999998},{"interface":"front","average":{"1min":0.92500000000000004,"5min":1.448,"15min":1.8109999999999999},"min":{"1min":0.496,"5min":0.34000000000000002,"15min":0.32200000000000001},"max":{"1min":1.3,"5min":4.9850000000000003,"15min":91.695999999999998},"last":0.55400000000000005}]},{"osd":2,"last update":"Sat Jun 3 20:11:41 2023","interfaces":[{"interface":"back","average":{"1min":0.90700000000000003,"5min":2.0019999999999998,"15min":1.3759999999999999},"min":{"1min":0.52500000000000002,"5min":0.34799999999999998,"15min":0.32800000000000001},"max":{"1min":1.218,"5min":46.331000000000003,"15min":46.331000000000003},"last":2.0899999999999999},{"interface":"front","average":{"1min":0.95399999999999996,"5min":1.9890000000000001,"15min":1.5009999999999999},"min":{"1min":0.71199999999999997,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":1.3049999999999999,"5min":48.216000000000001,"15min":48.216000000000001},"last":0.45900000000000002}]},{"osd":3,"last update":"Sat Jun 3 20:11:35 2023","interfaces":[{"interface":"back","average":{"1min":0.95299999999999996,"5min":1.698,"15min":1.8200000000000001},"min":{"1min":0.52700000000000002,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":1.3819999999999999,"5min":11.483000000000001,"15min":95.962000000000003},"last":1.927},{"interface":"front","average":{"1min":1.0129999999999999,"5min":1.7010000000000001,"15min":1.97},"min":{"1min":0.629,"5min":0.34399999999999997,"15min":0.34399999999999997},"max":{"1min":1.3500000000000001,"5min":8.0310000000000006,"15min":94.174999999999997},"last":0.50700000000000001}]},{"osd":4,"last update":"Sat Jun 3 20:10:54 2023","interfaces":[{"interface":"back","average":{"1min":1.417,"5min":1.8879999999999999,"15min":1.3260000000000001},"min":{"1min":0.33900000000000002,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":7.1189999999999998,"5min":21.888999999999999,"15min":21.888999999999999},"last":1.97},{"interface":"front","average":{"1min":1.484,"5min":2.1739999999999999,"15min":1.671},"min":{"1min":0.60799999999999998,"5min":0.443,"15min":0.38800000000000001},"max":{"1min":4.8600000000000003,"5min":10.507999999999999,"15min":12.917},"last":1.839}]},{"osd":5,"last update":"Sat Jun 3 20:11:35 2023","interfaces":[{"interface":"back","average":{"1min":1.034,"5min":1.9239999999999999,"15min":1.407},"min":{"1min":0.63700000000000001,"5min":0.36399999999999999,"15min":0.36399999999999999},"max":{"1min":1.4350000000000001,"5min":11.443,"15min":11.443},"last":2.0009999999999999},{"interface":"front","average":{"1min":1.0760000000000001,"5min":2.1139999999999999,"15min":1.7849999999999999},"min":{"1min":0.64800000000000002,"5min":0.35799999999999998,"15min":0.33100000000000002},"max":{"1min":1.4430000000000001,"5min":8.6300000000000008,"15min":20.306999999999999},"last":0.48299999999999998}]},{"osd":6,"last update":"Sat Jun 3 20:11:35 2023","interfaces":[{"interface":"back","average":{"1min":1.145,"5min":2.4020000000000001,"15min":1.716},"min":{"1min":0.74399999999999999,"5min":0.41299999999999998,"15min":0.41299999999999998},"max":{"1min":1.528,"5min":11.256,"15min":11.256},"last":0.52900000000000003},{"interface":"front","average":{"1min":1.026,"5min":2.4830000000000001,"15min":1.78},"min":{"1min":0.69099999999999995,"5min":0.34899999999999998,"15min":0.32500000000000001},"max":{"1min":2.746,"5min":10.894,"15min":10.894},"last":3.5950000000000002}]}]},{"osd":6,"up_from":42,"seq":180388626930,"num_pgs":42,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1458264,"kb_used_data":1016464,"kb_used_omap":0,"kb_used_meta":441792,"kb_avail":92286888,"statfs":{"total":95995035648,"available":94501773312,"internally_reserved":0,"allocated":1040859136,"data_stored":1040367510,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":452395008},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 3 20:11:01 2023","interfaces":[{"interface":"back","average":{"1min":2.754,"5min":3.7519999999999998,"15min":2.157},"min":{"1min":0.39500000000000002,"5min":0.26600000000000001,"15min":0.26400000000000001},"max":{"1min":28.117000000000001,"5min":54.283999999999999,"15min":54.786000000000001},"last":0.29499999999999998},{"interface":"front","average":{"1min":3.2730000000000001,"5min":4.2510000000000003,"15min":2.3290000000000002},"min":{"1min":0.38800000000000001,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":28.138000000000002,"5min":54.314999999999998,"15min":54.936},"last":1.825}]},{"osd":1,"last update":"Sat Jun 3 20:11:41 2023","interfaces":[{"interface":"back","average":{"1min":1.056,"5min":1.444,"15min":2.2149999999999999},"min":{"1min":0.38300000000000001,"5min":0.34300000000000003,"15min":0.28000000000000003},"max":{"1min":1.762,"5min":6.7510000000000003,"15min":160.38499999999999},"last":1.1559999999999999},{"interface":"front","average":{"1min":1.214,"5min":2.0049999999999999,"15min":2.1899999999999999},"min":{"1min":0.307,"5min":0.307,"15min":0.29599999999999999},"max":{"1min":4.0190000000000001,"5min":12.132999999999999,"15min":160.434},"last":1.1699999999999999}]},{"osd":2,"last update":"Sat Jun 3 20:11:41 2023","interfaces":[{"interface":"back","average":{"1min":0.873,"5min":2.347,"15min":1.353},"min":{"1min":0.47799999999999998,"5min":0.32500000000000001,"15min":0.27200000000000002},"max":{"1min":1.28,"5min":42.091999999999999,"15min":42.091999999999999},"last":0.88400000000000001},{"interface":"front","average":{"1min":1.0549999999999999,"5min":2.6890000000000001,"15min":1.54},"min":{"1min":0.64300000000000002,"5min":0.33700000000000002,"15min":0.33700000000000002},"max":{"1min":1.792,"5min":43.448,"15min":43.448},"last":0.34999999999999998}]},{"osd":3,"last update":"Sat Jun 3 20:11:17 2023","interfaces":[{"interface":"back","average":{"1min":1.0680000000000001,"5min":1.458,"15min":2.2829999999999999},"min":{"1min":0.49299999999999999,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":3.7080000000000002,"5min":9.5950000000000006,"15min":164.99199999999999},"last":3.573},{"interface":"front","average":{"1min":0.94399999999999995,"5min":1.5620000000000001,"15min":2.2949999999999999},"min":{"1min":0.44,"5min":0.38400000000000001,"15min":0.316},"max":{"1min":1.2769999999999999,"5min":6.7770000000000001,"15min":163.459},"last":0.439}]},{"osd":4,"last update":"Sat Jun 3 20:10:57 2023","interfaces":[{"interface":"back","average":{"1min":2.0510000000000002,"5min":2.0350000000000001,"15min":1.3029999999999999},"min":{"1min":0.35999999999999999,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":8.2230000000000008,"5min":14.098000000000001,"15min":14.098000000000001},"last":0.83699999999999997},{"interface":"front","average":{"1min":1.929,"5min":1.738,"15min":1.2569999999999999},"min":{"1min":0.52900000000000003,"5min":0.39600000000000002,"15min":0.377},"max":{"1min":8.452,"5min":8.452,"15min":8.452},"last":0.40600000000000003}]},{"osd":5,"last update":"Sat Jun 3 20:10:56 2023","interfaces":[{"interface":"back","average":{"1min":1.4850000000000001,"5min":1.736,"15min":1.284},"min":{"1min":0.57999999999999996,"5min":0.36099999999999999,"15min":0.32200000000000001},"max":{"1min":3.302,"5min":14.731999999999999,"15min":14.731999999999999},"last":0.38600000000000001},{"interface":"front","average":{"1min":2.4039999999999999,"5min":2.504,"15min":1.6779999999999999},"min":{"1min":0.41799999999999998,"5min":0.35899999999999999,"15min":0.29499999999999998},"max":{"1min":10.786,"5min":15.295,"15min":15.295},"last":0.81799999999999995}]},{"osd":7,"last update":"Sat Jun 3 20:11:26 2023","interfaces":[{"interface":"back","average":{"1min":1.21,"5min":1.907,"15min":1.506},"min":{"1min":0.55300000000000005,"5min":0.42799999999999999,"15min":0.39400000000000002},"max":{"1min":1.881,"5min":8.4269999999999996,"15min":11.362},"last":0.42399999999999999},{"interface":"front","average":{"1min":1.105,"5min":2.2709999999999999,"15min":1.6000000000000001},"min":{"1min":0.50800000000000001,"5min":0.38900000000000001,"15min":0.33900000000000002},"max":{"1min":1.597,"5min":21.202000000000002,"15min":21.202000000000002},"last":0.91200000000000003}]}]},{"osd":1,"up_from":0,"seq":442381631728,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":0,"up_from":86,"seq":369367187754,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1207364,"kb_used_data":799996,"kb_used_omap":5,"kb_used_meta":407354,"kb_avail":92537788,"statfs":{"total":95995035648,"available":94758694912,"internally_reserved":0,"allocated":819195904,"data_stored":818431327,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":417131130},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Jun 3 20:11:26 2023","interfaces":[{"interface":"back","average":{"1min":0.60399999999999998,"5min":0.91600000000000004,"15min":1.411},"min":{"1min":0.313,"5min":0.249,"15min":0.192},"max":{"1min":0.96999999999999997,"5min":9.3040000000000003,"15min":81.310000000000002},"last":0.33700000000000002},{"interface":"front","average":{"1min":0.58299999999999996,"5min":1.3140000000000001,"15min":1.5820000000000001},"min":{"1min":0.32600000000000001,"5min":0.22600000000000001,"15min":0.187},"max":{"1min":0.86499999999999999,"5min":11.736000000000001,"15min":76.971999999999994},"last":3.2610000000000001}]},{"osd":2,"last update":"Sat Jun 3 20:11:26 2023","interfaces":[{"interface":"back","average":{"1min":0.60499999999999998,"5min":1.0489999999999999,"15min":1.006},"min":{"1min":0.28100000000000003,"5min":0.193,"15min":0.193},"max":{"1min":0.92400000000000004,"5min":11.901,"15min":11.901},"last":1.6519999999999999},{"interface":"front","average":{"1min":0.64600000000000002,"5min":1.214,"15min":1.0840000000000001},"min":{"1min":0.25800000000000001,"5min":0.255,"15min":0.218},"max":{"1min":1.131,"5min":17.315999999999999,"15min":17.315999999999999},"last":0.27800000000000002}]},{"osd":3,"last update":"Sat Jun 3 20:11:14 2023","interfaces":[{"interface":"back","average":{"1min":0.68000000000000005,"5min":1.04,"15min":1.107},"min":{"1min":0.36399999999999999,"5min":0.26800000000000002,"15min":0.216},"max":{"1min":1.843,"5min":4.8570000000000002,"15min":7.5990000000000002},"last":0.879},{"interface":"front","average":{"1min":0.78600000000000003,"5min":1.157,"15min":1.165},"min":{"1min":0.32000000000000001,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":1.978,"5min":6.3769999999999998,"15min":8.4459999999999997},"last":0.97299999999999998}]},{"osd":4,"last update":"Sat Jun 3 20:10:59 2023","interfaces":[{"interface":"back","average":{"1min":1.9370000000000001,"5min":1.7110000000000001,"15min":1.3400000000000001},"min":{"1min":0.45200000000000001,"5min":0.318,"15min":0.318},"max":{"1min":10.025,"5min":10.025,"15min":10.025},"last":0.92300000000000004},{"interface":"front","average":{"1min":1.673,"5min":1.8160000000000001,"15min":1.429},"min":{"1min":0.56699999999999995,"5min":0.375,"15min":0.375},"max":{"1min":4.6059999999999999,"5min":13.058999999999999,"15min":13.058999999999999},"last":0.90500000000000003}]},{"osd":5,"last update":"Sat Jun 3 20:10:59 2023","interfaces":[{"interface":"back","average":{"1min":1.516,"5min":2.3239999999999998,"15min":1.6439999999999999},"min":{"1min":0.442,"5min":0.42099999999999999,"15min":0.34899999999999998},"max":{"1min":4.633,"5min":8.3249999999999993,"15min":8.3249999999999993},"last":1.492},{"interface":"front","average":{"1min":1.4670000000000001,"5min":1.542,"15min":1.4319999999999999},"min":{"1min":0.51200000000000001,"5min":0.34000000000000002,"15min":0.34000000000000002},"max":{"1min":7.0529999999999999,"5min":7.0529999999999999,"15min":8.2189999999999994},"last":0.54900000000000004}]},{"osd":6,"last update":"Sat Jun 3 20:10:59 2023","interfaces":[{"interface":"back","average":{"1min":1.3460000000000001,"5min":1.5449999999999999,"15min":1.3480000000000001},"min":{"1min":0.439,"5min":0.38,"15min":0.38},"max":{"1min":7.3220000000000001,"5min":10.140000000000001,"15min":10.140000000000001},"last":0.51900000000000002},{"interface":"front","average":{"1min":1.2190000000000001,"5min":1.7949999999999999,"15min":1.4830000000000001},"min":{"1min":0.48199999999999998,"5min":0.432,"15min":0.42799999999999999},"max":{"1min":2.968,"5min":13.117000000000001,"15min":13.117000000000001},"last":0.44600000000000001}]},{"osd":7,"last update":"Sat Jun 3 20:10:59 2023","interfaces":[{"interface":"back","average":{"1min":1.5029999999999999,"5min":1.546,"15min":1.4990000000000001},"min":{"1min":0.60499999999999998,"5min":0.39800000000000002,"15min":0.35099999999999998},"max":{"1min":5.609,"5min":8.4329999999999998,"15min":11.196},"last":0.56699999999999995},{"interface":"front","average":{"1min":1.5529999999999999,"5min":1.79,"15min":1.6359999999999999},"min":{"1min":0.64700000000000002,"5min":0.47899999999999998,"15min":0.47899999999999998},"max":{"1min":5.6289999999999996,"5min":8.5960000000000001,"15min":8.891},"last":0.51200000000000001}]}]},{"osd":2,"up_from":114,"seq":489626271933,"num_pgs":29,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":652664,"kb_used_data":530096,"kb_used_omap":7,"kb_used_meta":122552,"kb_avail":93092488,"statfs":{"total":95995035648,"available":95326707712,"internally_reserved":0,"allocated":542818304,"data_stored":542116508,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8111,"internal_metadata":125493329},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 3 20:11:35 2023","interfaces":[{"interface":"back","average":{"1min":0.65000000000000002,"5min":4.649,"15min":2.5630000000000002},"min":{"1min":0.47499999999999998,"5min":0.222,"15min":0.20899999999999999},"max":{"1min":0.83299999999999996,"5min":273.51100000000002,"15min":273.51100000000002},"last":0.57799999999999996},{"interface":"front","average":{"1min":0.67900000000000005,"5min":4.7990000000000004,"15min":2.7130000000000001},"min":{"1min":0.34000000000000002,"5min":0.247,"15min":0.221},"max":{"1min":1.1679999999999999,"5min":273.67399999999998,"15min":273.67399999999998},"last":2.6160000000000001}]},{"osd":1,"last update":"Sat Jun 3 20:11:35 2023","interfaces":[{"interface":"back","average":{"1min":0.68300000000000005,"5min":4.4820000000000002,"15min":2.21},"min":{"1min":0.32100000000000001,"5min":0.254,"15min":0.22900000000000001},"max":{"1min":1.0109999999999999,"5min":273.46499999999997,"15min":273.46499999999997},"last":0.96699999999999997},{"interface":"front","average":{"1min":0.629,"5min":4.71,"15min":2.2930000000000001},"min":{"1min":0.44900000000000001,"5min":0.246,"15min":0.23699999999999999},"max":{"1min":0.94799999999999995,"5min":274.38400000000001,"15min":274.38400000000001},"last":0.40999999999999998}]},{"osd":3,"last update":"Sat Jun 3 20:11:23 2023","interfaces":[{"interface":"back","average":{"1min":16.120999999999999,"5min":4.3410000000000002,"15min":2.1669999999999998},"min":{"1min":0.41699999999999998,"5min":0.28799999999999998,"15min":0.22700000000000001},"max":{"1min":273.71899999999999,"5min":273.71899999999999,"15min":273.71899999999999},"last":0.61499999999999999},{"interface":"front","average":{"1min":15.863,"5min":4.4160000000000004,"15min":2.5049999999999999},"min":{"1min":0.30299999999999999,"5min":0.24199999999999999,"15min":0.24199999999999999},"max":{"1min":274.21100000000001,"5min":274.21100000000001,"15min":274.21100000000001},"last":1.377}]},{"osd":4,"last update":"Sat Jun 3 20:10:59 2023","interfaces":[{"interface":"back","average":{"1min":16.218,"5min":4.9749999999999996,"15min":2.2519999999999998},"min":{"1min":0.48699999999999999,"5min":0.32800000000000001,"15min":0.32700000000000001},"max":{"1min":273.75700000000001,"5min":273.75700000000001,"15min":273.75700000000001},"last":1.0069999999999999},{"interface":"front","average":{"1min":15.512,"5min":4.7480000000000002,"15min":2.1850000000000001},"min":{"1min":0.36199999999999999,"5min":0.35699999999999998,"15min":0.32600000000000001},"max":{"1min":274.27300000000002,"5min":274.27300000000002,"15min":274.27300000000002},"last":1.3400000000000001}]},{"osd":5,"last update":"Sat Jun 3 20:11:35 2023","interfaces":[{"interface":"back","average":{"1min":1.0049999999999999,"5min":4.1020000000000003,"15min":2.0299999999999998},"min":{"1min":0.59099999999999997,"5min":0.36099999999999999,"15min":0.36099999999999999},"max":{"1min":1.274,"5min":273.78800000000001,"15min":273.78800000000001},"last":0.53400000000000003},{"interface":"front","average":{"1min":1.0740000000000001,"5min":4.2270000000000003,"15min":2.2869999999999999},"min":{"1min":0.75900000000000001,"5min":0.377,"15min":0.308},"max":{"1min":1.373,"5min":274.15800000000002,"15min":274.15800000000002},"last":1.0569999999999999}]},{"osd":6,"last update":"Sat Jun 3 20:11:35 2023","interfaces":[{"interface":"back","average":{"1min":1.026,"5min":4.569,"15min":2.2090000000000001},"min":{"1min":0.63,"5min":0.376,"15min":0.376},"max":{"1min":1.4119999999999999,"5min":274.03899999999999,"15min":274.03899999999999},"last":4.8380000000000001},{"interface":"front","average":{"1min":1.05,"5min":4.71,"15min":2.266},"min":{"1min":0.70299999999999996,"5min":0.39100000000000001,"15min":0.34899999999999998},"max":{"1min":1.6499999999999999,"5min":274.06299999999999,"15min":274.06299999999999},"last":2.702}]},{"osd":7,"last update":"Sat Jun 3 20:11:35 2023","interfaces":[{"interface":"back","average":{"1min":1.044,"5min":4.3140000000000001,"15min":2.411},"min":{"1min":0.78700000000000003,"5min":0.39600000000000002,"15min":0.39600000000000002},"max":{"1min":1.504,"5min":274.54000000000002,"15min":274.54000000000002},"last":2.2410000000000001},{"interface":"front","average":{"1min":1.147,"5min":4.1989999999999998,"15min":2.3580000000000001},"min":{"1min":0.70699999999999996,"5min":0.40200000000000002,"15min":0.35699999999999998},"max":{"1min":1.7130000000000001,"5min":274.12700000000001,"15min":274.12700000000001},"last":1.0960000000000001}]}]},{"osd":3,"up_from":119,"seq":511101108405,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1009224,"kb_used_data":794304,"kb_used_omap":2,"kb_used_meta":214909,"kb_avail":92735928,"statfs":{"total":95995035648,"available":94961590272,"internally_reserved":0,"allocated":813367296,"data_stored":812654788,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":220066869},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 3 20:11:24 2023","interfaces":[{"interface":"back","average":{"1min":0.73799999999999999,"5min":2.3580000000000001,"15min":1.6819999999999999},"min":{"1min":0.34899999999999998,"5min":0.223,"15min":0.19},"max":{"1min":3.0299999999999998,"5min":58.106999999999999,"15min":68.647999999999996},"last":2.5129999999999999},{"interface":"front","average":{"1min":0.80100000000000005,"5min":2.948,"15min":1.867},"min":{"1min":0.441,"5min":0.23400000000000001,"15min":0.214},"max":{"1min":2.9340000000000002,"5min":62.585000000000001,"15min":69.691000000000003},"last":0.83299999999999996}]},{"osd":1,"last update":"Sat Jun 3 20:11:24 2023","interfaces":[{"interface":"back","average":{"1min":0.84999999999999998,"5min":0.998,"15min":1.0549999999999999},"min":{"1min":0.29799999999999999,"5min":0.214,"15min":0.214},"max":{"1min":4.0069999999999997,"5min":10.749000000000001,"15min":10.749000000000001},"last":1.198},{"interface":"front","average":{"1min":0.86099999999999999,"5min":0.75,"15min":0.97699999999999998},"min":{"1min":0.38700000000000001,"5min":0.24199999999999999,"15min":0.23499999999999999},"max":{"1min":3.1619999999999999,"5min":5.4960000000000004,"15min":10.792999999999999},"last":2.1320000000000001}]},{"osd":2,"last update":"Sat Jun 3 20:11:24 2023","interfaces":[{"interface":"back","average":{"1min":0.91800000000000004,"5min":1.827,"15min":1.246},"min":{"1min":0.495,"5min":0.217,"15min":0.217},"max":{"1min":3.7719999999999998,"5min":62.527999999999999,"15min":62.527999999999999},"last":0.248},{"interface":"front","average":{"1min":0.84799999999999998,"5min":1.827,"15min":1.2789999999999999},"min":{"1min":0.36899999999999999,"5min":0.214,"15min":0.214},"max":{"1min":4.0410000000000004,"5min":57.935000000000002,"15min":57.935000000000002},"last":1.323}]},{"osd":4,"last update":"Sat Jun 3 20:10:53 2023","interfaces":[{"interface":"back","average":{"1min":2.0379999999999998,"5min":1.98,"15min":1.413},"min":{"1min":0.42399999999999999,"5min":0.38200000000000001,"15min":0.38200000000000001},"max":{"1min":9.8360000000000003,"5min":12.757,"15min":12.757},"last":1.3009999999999999},{"interface":"front","average":{"1min":2.1749999999999998,"5min":1.744,"15min":1.333},"min":{"1min":0.39000000000000001,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":10.122,"5min":10.122,"15min":10.122},"last":2.1859999999999999}]},{"osd":5,"last update":"Sat Jun 3 20:11:24 2023","interfaces":[{"interface":"back","average":{"1min":1.079,"5min":1.151,"15min":1.204},"min":{"1min":0.54500000000000004,"5min":0.36299999999999999,"15min":0.32900000000000001},"max":{"1min":3.0939999999999999,"5min":5.4269999999999996,"15min":7.1230000000000002},"last":2.4390000000000001},{"interface":"front","average":{"1min":1.252,"5min":1.5289999999999999,"15min":1.518},"min":{"1min":0.52700000000000002,"5min":0.314,"15min":0.314},"max":{"1min":3.7229999999999999,"5min":12.56,"15min":12.56},"last":0.746}]},{"osd":6,"last update":"Sat Jun 3 20:11:24 2023","interfaces":[{"interface":"back","average":{"1min":1.498,"5min":1.996,"15min":1.556},"min":{"1min":0.79700000000000004,"5min":0.379,"15min":0.379},"max":{"1min":5.1440000000000001,"5min":11.093999999999999,"15min":11.093999999999999},"last":0.35999999999999999},{"interface":"front","average":{"1min":1.486,"5min":2.214,"15min":1.5389999999999999},"min":{"1min":0.61199999999999999,"5min":0.35599999999999998,"15min":0.35599999999999998},"max":{"1min":5.1600000000000001,"5min":11.116,"15min":11.116},"last":1.27}]},{"osd":7,"last update":"Sat Jun 3 20:11:24 2023","interfaces":[{"interface":"back","average":{"1min":1.2030000000000001,"5min":1.637,"15min":1.4019999999999999},"min":{"1min":0.63100000000000001,"5min":0.39100000000000001,"15min":0.39100000000000001},"max":{"1min":4.2380000000000004,"5min":10.858000000000001,"15min":10.858000000000001},"last":1.3460000000000001},{"interface":"front","average":{"1min":1.4790000000000001,"5min":1.776,"15min":1.6419999999999999},"min":{"1min":0.51200000000000001,"5min":0.39700000000000002,"15min":0.39700000000000002},"max":{"1min":7.798,"5min":11.084,"15min":12.715999999999999},"last":2.3290000000000002}]}]},{"osd":4,"up_from":124,"seq":532575944878,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":823240,"kb_used_data":647680,"kb_used_omap":10,"kb_used_meta":175541,"kb_avail":92921912,"statfs":{"total":95995035648,"available":95152037888,"internally_reserved":0,"allocated":663224320,"data_stored":662454454,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":10652,"internal_metadata":179754596},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 3 20:11:03 2023","interfaces":[{"interface":"back","average":{"1min":1.002,"5min":4.8399999999999999,"15min":2.468},"min":{"1min":0.58599999999999997,"5min":0.29499999999999998,"15min":0.254},"max":{"1min":3.0139999999999998,"5min":214.72999999999999,"15min":214.72999999999999},"last":1.8480000000000001},{"interface":"front","average":{"1min":2.04,"5min":5.327,"15min":2.6459999999999999},"min":{"1min":0.41399999999999998,"5min":0.40200000000000002,"15min":0.32400000000000001},"max":{"1min":15,"5min":214.76499999999999,"15min":214.76499999999999},"last":1.833}]},{"osd":1,"last update":"Sat Jun 3 20:11:03 2023","interfaces":[{"interface":"back","average":{"1min":1.1439999999999999,"5min":1.639,"15min":1.55},"min":{"1min":0.371,"5min":0.35199999999999998,"15min":0.28100000000000003},"max":{"1min":3.8210000000000002,"5min":10.446,"15min":58.32},"last":0.58199999999999996},{"interface":"front","average":{"1min":0.878,"5min":1.4990000000000001,"15min":1.4610000000000001},"min":{"1min":0.498,"5min":0.34999999999999998,"15min":0.307},"max":{"1min":1.8,"5min":12.919,"15min":58.286000000000001},"last":0.59599999999999997}]},{"osd":2,"last update":"Sat Jun 3 20:11:03 2023","interfaces":[{"interface":"back","average":{"1min":13.406000000000001,"5min":4.6070000000000002,"15min":2.1419999999999999},"min":{"1min":0.55000000000000004,"5min":0.317,"15min":0.30299999999999999},"max":{"1min":248.65799999999999,"5min":248.65799999999999,"15min":248.65799999999999},"last":0.54800000000000004},{"interface":"front","average":{"1min":14.144,"5min":4.7720000000000002,"15min":2.1890000000000001},"min":{"1min":0.46700000000000003,"5min":0.373,"15min":0.34499999999999997},"max":{"1min":249.077,"5min":249.077,"15min":249.077},"last":1.885}]},{"osd":3,"last update":"Sat Jun 3 20:11:03 2023","interfaces":[{"interface":"back","average":{"1min":1.2330000000000001,"5min":1.778,"15min":1.6240000000000001},"min":{"1min":0.40600000000000003,"5min":0.35199999999999998,"15min":0.32500000000000001},"max":{"1min":3.7759999999999998,"5min":10.301,"15min":57.954999999999998},"last":0.85799999999999998},{"interface":"front","average":{"1min":1.226,"5min":1.6479999999999999,"15min":1.613},"min":{"1min":0.50700000000000001,"5min":0.40300000000000002,"15min":0.33700000000000002},"max":{"1min":4.9210000000000003,"5min":12.890000000000001,"15min":57.889000000000003},"last":1.897}]},{"osd":5,"last update":"Sat Jun 3 20:11:03 2023","interfaces":[{"interface":"back","average":{"1min":1.4590000000000001,"5min":2.0819999999999999,"15min":1.623},"min":{"1min":0.35899999999999999,"5min":0.32000000000000001,"15min":0.313},"max":{"1min":6.6859999999999999,"5min":11.311,"15min":86.576999999999998},"last":0.47499999999999998},{"interface":"front","average":{"1min":1.5269999999999999,"5min":2.1760000000000002,"15min":1.7390000000000001},"min":{"1min":0.32900000000000001,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":7.9269999999999996,"5min":10.997999999999999,"15min":88.013000000000005},"last":0.69699999999999995}]},{"osd":6,"last update":"Sat Jun 3 20:11:03 2023","interfaces":[{"interface":"back","average":{"1min":1.8140000000000001,"5min":2.7429999999999999,"15min":1.5309999999999999},"min":{"1min":0.36099999999999999,"5min":0.34599999999999997,"15min":0.34599999999999997},"max":{"1min":6.6520000000000001,"5min":14.244999999999999,"15min":14.244999999999999},"last":0.66100000000000003},{"interface":"front","average":{"1min":2.4329999999999998,"5min":2.8679999999999999,"15min":1.629},"min":{"1min":0.42599999999999999,"5min":0.40100000000000002,"15min":0.36499999999999999},"max":{"1min":10.359,"5min":12.145,"15min":12.145},"last":1.0049999999999999}]},{"osd":7,"last update":"Sat Jun 3 20:11:03 2023","interfaces":[{"interface":"back","average":{"1min":1.7270000000000001,"5min":2.052,"15min":1.55},"min":{"1min":0.44,"5min":0.41199999999999998,"15min":0.39700000000000002},"max":{"1min":9.6210000000000004,"5min":10.388,"15min":10.388},"last":0.61699999999999999},{"interface":"front","average":{"1min":1.0629999999999999,"5min":1.7729999999999999,"15min":1.486},"min":{"1min":0.44800000000000001,"5min":0.34200000000000003,"15min":0.31900000000000001},"max":{"1min":1.8260000000000001,"5min":12.951000000000001,"15min":12.951000000000001},"last":0.51200000000000001}]}]},{"osd":5,"up_from":36,"seq":154618823163,"num_pgs":28,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":994564,"kb_used_data":425468,"kb_used_omap":0,"kb_used_meta":569088,"kb_avail":92750588,"statfs":{"total":95995035648,"available":94976602112,"internally_reserved":0,"allocated":435679232,"data_stored":435195627,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":582746112},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 3 20:10:46 2023","interfaces":[{"interface":"back","average":{"1min":6.1239999999999997,"5min":4.0549999999999997,"15min":2.1429999999999998},"min":{"1min":0.28100000000000003,"5min":0.27000000000000002,"15min":0.22800000000000001},"max":{"1min":92.706999999999994,"5min":185.29900000000001,"15min":185.29900000000001},"last":1.1060000000000001},{"interface":"front","average":{"1min":6.2569999999999997,"5min":4.3380000000000001,"15min":2.504},"min":{"1min":0.307,"5min":0.26400000000000001,"15min":0.26300000000000001},"max":{"1min":92.858999999999995,"5min":185.44200000000001,"15min":185.44200000000001},"last":0.28399999999999997}]},{"osd":1,"last update":"Sat Jun 3 20:10:46 2023","interfaces":[{"interface":"back","average":{"1min":1.6539999999999999,"5min":1.4359999999999999,"15min":2.181},"min":{"1min":0.41199999999999998,"5min":0.32900000000000001,"15min":0.30199999999999999},"max":{"1min":6.5300000000000002,"5min":8.5050000000000008,"15min":114.681},"last":1.3480000000000001},{"interface":"front","average":{"1min":1.2290000000000001,"5min":1.488,"15min":2.2029999999999998},"min":{"1min":0.41099999999999998,"5min":0.307,"15min":0.307},"max":{"1min":6.3239999999999998,"5min":9.2739999999999991,"15min":115.366},"last":0.32200000000000001}]},{"osd":2,"last update":"Sat Jun 3 20:10:46 2023","interfaces":[{"interface":"back","average":{"1min":6.5,"5min":2.798,"15min":1.5700000000000001},"min":{"1min":0.48099999999999998,"5min":0.318,"15min":0.318},"max":{"1min":92.412000000000006,"5min":92.412000000000006,"15min":92.412000000000006},"last":1.04},{"interface":"front","average":{"1min":6.4409999999999998,"5min":3.0230000000000001,"15min":1.8560000000000001},"min":{"1min":0.39300000000000002,"5min":0.32100000000000001,"15min":0.31900000000000001},"max":{"1min":94.552000000000007,"5min":94.552000000000007,"15min":94.552000000000007},"last":0.53400000000000003}]},{"osd":3,"last update":"Sat Jun 3 20:11:23 2023","interfaces":[{"interface":"back","average":{"1min":1.4530000000000001,"5min":1.8100000000000001,"15min":2.7360000000000002},"min":{"1min":0.57599999999999996,"5min":0.317,"15min":0.317},"max":{"1min":6.8440000000000003,"5min":8.4179999999999993,"15min":261.14100000000002},"last":1.534},{"interface":"front","average":{"1min":0.88100000000000001,"5min":1.4199999999999999,"15min":2.4180000000000001},"min":{"1min":0.39500000000000002,"5min":0.35799999999999998,"15min":0.35399999999999998},"max":{"1min":1.2509999999999999,"5min":9.0990000000000002,"15min":261.27100000000002},"last":1.423}]},{"osd":4,"last update":"Sat Jun 3 20:11:00 2023","interfaces":[{"interface":"back","average":{"1min":2.2930000000000001,"5min":2.4510000000000001,"15min":1.6839999999999999},"min":{"1min":0.44,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":7.5910000000000002,"5min":14.951000000000001,"15min":14.951000000000001},"last":1.51},{"interface":"front","average":{"1min":1.619,"5min":2.0129999999999999,"15min":1.3799999999999999},"min":{"1min":0.42299999999999999,"5min":0.376,"15min":0.376},"max":{"1min":7.2000000000000002,"5min":12.109,"15min":12.109},"last":1.2}]},{"osd":6,"last update":"Sat Jun 3 20:10:46 2023","interfaces":[{"interface":"back","average":{"1min":2.0049999999999999,"5min":2.4980000000000002,"15min":1.7649999999999999},"min":{"1min":0.47499999999999998,"5min":0.35799999999999998,"15min":0.33000000000000002},"max":{"1min":6.8849999999999998,"5min":22.945,"15min":22.945},"last":1.4870000000000001},{"interface":"front","average":{"1min":2.9079999999999999,"5min":2.895,"15min":1.9359999999999999},"min":{"1min":0.56699999999999995,"5min":0.38300000000000001,"15min":0.38300000000000001},"max":{"1min":10.708,"5min":22.428000000000001,"15min":22.428000000000001},"last":2.7029999999999998}]},{"osd":7,"last update":"Sat Jun 3 20:11:31 2023","interfaces":[{"interface":"back","average":{"1min":1.2729999999999999,"5min":2.0920000000000001,"15min":1.8740000000000001},"min":{"1min":0.76200000000000001,"5min":0.379,"15min":0.379},"max":{"1min":3.8279999999999998,"5min":14.223000000000001,"15min":14.223000000000001},"last":3.0049999999999999},{"interface":"front","average":{"1min":1.119,"5min":1.8029999999999999,"15min":1.393},"min":{"1min":0.48499999999999999,"5min":0.38100000000000001,"15min":0.38100000000000001},"max":{"1min":3.9950000000000001,"5min":9.0719999999999992,"15min":9.0719999999999992},"last":1.1599999999999999}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6928,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4862,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":10652,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4786,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4932,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":844300288,"data_stored":844300288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":580190208,"data_stored":580190208,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":854130688,"data_stored":854130688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":687603712,"data_stored":687603712,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":450494464,"data_stored":450494464,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":1079967744,"data_stored":1079967744,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":834600960,"data_stored":834600960,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-06-03T20:11:51.853 INFO:tasks.cephadm.ceph_manager.ceph:recovered! 2023-06-03T20:11:51.854 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-06-03T20:11:51.854 INFO:tasks.thrashosds.thrasher:Killing osd 4, live_osds are [0, 1, 2, 3, 4, 5, 6, 7] 2023-06-03T20:11:51.854 INFO:tasks.cephadm.osd.4:Stopping osd.4... 2023-06-03T20:11:51.854 DEBUG:teuthology.orchestra.run.smithi164:> sudo systemctl stop ceph-f1db3478-0243-11ee-9b1d-001a4aab830c@osd.4 2023-06-03T20:11:51.884 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:11:51.919 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:11:51.922 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:11:51.923 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:11:52.024 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:11:52.055 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:11:52.058 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:11:52.059 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:11:52.078 INFO:journalctl@ceph.osd.4.smithi164.stdout:Jun 03 20:11:51 smithi164 systemd[1]: Stopping Ceph osd.4 for f1db3478-0243-11ee-9b1d-001a4aab830c... 2023-06-03T20:11:52.079 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:11:51 smithi164 bash[59462]: audit 1685823110.188561 mgr.y (mgr.34104) 1072 : audit 0 from='client.36249 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-06-03T20:11:52.079 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:11:51 smithi164 bash[59462]: cluster 1685823110.6807258 mgr.y (mgr.34104) 1073 : cluster 0 pgmap v1039: 89 pgs: 89 active+clean; 1.8 GiB data, 7.5 GiB used, 618 GiB / 626 GiB avail; 111 MiB/s wr, 1.77k op/s 2023-06-03T20:11:52.159 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:11:52.169 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:11:51 smithi089 bash[75068]: audit 1685823110.188561 mgr.y (mgr.34104) 1072 : audit 0 from='client.36249 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-06-03T20:11:52.169 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:11:51 smithi089 bash[75068]: cluster 1685823110.6807258 mgr.y (mgr.34104) 1073 : cluster 0 pgmap v1039: 89 pgs: 89 active+clean; 1.8 GiB data, 7.5 GiB used, 618 GiB / 626 GiB avail; 111 MiB/s wr, 1.77k op/s 2023-06-03T20:11:52.169 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:11:51 smithi089 bash[78452]: audit 1685823110.188561 mgr.y (mgr.34104) 1072 : audit 0 from='client.36249 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-06-03T20:11:52.169 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:11:51 smithi089 bash[78452]: cluster 1685823110.6807258 mgr.y (mgr.34104) 1073 : cluster 0 pgmap v1039: 89 pgs: 89 active+clean; 1.8 GiB data, 7.5 GiB used, 618 GiB / 626 GiB avail; 111 MiB/s wr, 1.77k op/s 2023-06-03T20:11:52.191 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:11:52.196 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:11:52.196 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:11:52.297 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:11:52.333 INFO:journalctl@ceph.osd.4.smithi164.stdout:Jun 03 20:11:52 smithi164 bash[66063]: debug 2023-06-03T20:11:52.063+0000 7fb9aa84b700 -1 received signal: Terminated from /sbin/docker-init -- /usr/bin/ceph-osd -n osd.4 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug (PID: 1) UID: 0 2023-06-03T20:11:52.333 INFO:journalctl@ceph.osd.4.smithi164.stdout:Jun 03 20:11:52 smithi164 bash[66063]: debug 2023-06-03T20:11:52.063+0000 7fb9aa84b700 -1 osd.4 194 *** Got signal Terminated *** 2023-06-03T20:11:52.333 INFO:journalctl@ceph.osd.4.smithi164.stdout:Jun 03 20:11:52 smithi164 bash[66063]: debug 2023-06-03T20:11:52.063+0000 7fb9aa84b700 -1 osd.4 194 *** Immediate shutdown (osd_fast_shutdown=true) *** 2023-06-03T20:11:52.335 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:11:52.338 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:11:52.339 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:11:52.439 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:11:52.473 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:11:52.476 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:11:52.476 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:11:52.554 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 19 16 35922 35906 118.084 99.0625 0.0117849 0.008462 2023-06-03T20:11:52.578 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:11:52.612 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:11:52.616 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:11:52.617 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:11:52.718 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:11:52.755 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:11:52.758 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:11:52.759 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:11:52.860 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:11:52.900 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:11:52.905 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:11:52.906 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:11:52.991 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:11:52 smithi164 bash[59462]: cluster 1685823112.0764797 mon.a (mon.0) 1010 : cluster 1 osd.4 marked itself down and dead 2023-06-03T20:11:52.992 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:11:52 smithi164 bash[59462]: audit 1685823112.1234376 mon.a (mon.0) 1011 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:11:52.992 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:11:52 smithi164 bash[59462]: audit 1685823112.129243 mon.c (mon.2) 502 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T20:11:53.007 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:11:53.038 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:11:53.041 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:11:53.042 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:11:53.143 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:11:53.181 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:11:53.183 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:11:53.184 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:11:53.278 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:11:52 smithi089 bash[75068]: cluster 1685823112.0764797 mon.a (mon.0) 1010 : cluster 1 osd.4 marked itself down and dead 2023-06-03T20:11:53.279 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:11:52 smithi089 bash[75068]: audit 1685823112.1234376 mon.a (mon.0) 1011 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:11:53.279 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:11:52 smithi089 bash[75068]: audit 1685823112.129243 mon.c (mon.2) 502 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T20:11:53.279 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:11:52 smithi089 bash[78452]: cluster 1685823112.0764797 mon.a (mon.0) 1010 : cluster 1 osd.4 marked itself down and dead 2023-06-03T20:11:53.279 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:11:52 smithi089 bash[78452]: audit 1685823112.1234376 mon.a (mon.0) 1011 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:11:53.279 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:11:52 smithi089 bash[78452]: audit 1685823112.129243 mon.c (mon.2) 502 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T20:11:53.286 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:11:53.325 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:11:53.328 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:11:53.328 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:11:53.429 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:11:53.469 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:11:53.472 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:11:53.473 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:11:53.554 INFO:tasks.radosbench.radosbench.0.smithi089.stdout:2023-06-03T20:11:53.550973+0000 min lat: 0.00135638 max lat: 0.339663 avg lat: 0.00850366 2023-06-03T20:11:53.555 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-06-03T20:11:53.555 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 20 16 36647 36631 114.445 45.3125 0.00309965 0.00850366 2023-06-03T20:11:53.574 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:11:53.610 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:11:53.613 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:11:53.614 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:11:53.715 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:11:53.747 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:11:53.750 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:11:53.750 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:11:53.851 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:11:53.885 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:11:53.889 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:11:53.889 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:11:53.990 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:11:54.028 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:11:54.030 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:11:54.031 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:11:54.132 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:11:54.171 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:11:54.175 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:11:54.175 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:11:54.276 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:11:54.278 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:11:53 smithi089 bash[78452]: cluster 1685823112.6813827 mgr.y (mgr.34104) 1074 : cluster 0 pgmap v1040: 89 pgs: 89 active+clean; 2.0 GiB data, 8.0 GiB used, 618 GiB / 626 GiB avail; 120 MiB/s wr, 1.92k op/s 2023-06-03T20:11:54.278 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:11:53 smithi089 bash[78452]: cluster 1685823112.9831338 mon.a (mon.0) 1012 : cluster 3 Health check failed: 1 osds down (OSD_DOWN) 2023-06-03T20:11:54.278 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:11:53 smithi089 bash[78452]: cluster 1685823113.0002544 osd.6 (osd.6) 141 : cluster 3 failed to encode map e195 with expected crc 2023-06-03T20:11:54.279 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:11:53 smithi089 bash[78452]: cluster 1685823113.001093 osd.5 (osd.5) 127 : cluster 3 failed to encode map e195 with expected crc 2023-06-03T20:11:54.279 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:11:53 smithi089 bash[78452]: cluster 1685823113.0041392 mon.a (mon.0) 1013 : cluster 0 osdmap e195: 8 total, 7 up, 7 in 2023-06-03T20:11:54.279 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:11:53 smithi089 bash[75068]: cluster 1685823112.6813827 mgr.y (mgr.34104) 1074 : cluster 0 pgmap v1040: 89 pgs: 89 active+clean; 2.0 GiB data, 8.0 GiB used, 618 GiB / 626 GiB avail; 120 MiB/s wr, 1.92k op/s 2023-06-03T20:11:54.279 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:11:53 smithi089 bash[75068]: cluster 1685823112.9831338 mon.a (mon.0) 1012 : cluster 3 Health check failed: 1 osds down (OSD_DOWN) 2023-06-03T20:11:54.279 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:11:53 smithi089 bash[75068]: cluster 1685823113.0002544 osd.6 (osd.6) 141 : cluster 3 failed to encode map e195 with expected crc 2023-06-03T20:11:54.279 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:11:53 smithi089 bash[75068]: cluster 1685823113.001093 osd.5 (osd.5) 127 : cluster 3 failed to encode map e195 with expected crc 2023-06-03T20:11:54.279 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:11:53 smithi089 bash[75068]: cluster 1685823113.0041392 mon.a (mon.0) 1013 : cluster 0 osdmap e195: 8 total, 7 up, 7 in 2023-06-03T20:11:54.286 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:11:53 smithi164 bash[59462]: cluster 1685823112.6813827 mgr.y (mgr.34104) 1074 : cluster 0 pgmap v1040: 89 pgs: 89 active+clean; 2.0 GiB data, 8.0 GiB used, 618 GiB / 626 GiB avail; 120 MiB/s wr, 1.92k op/s 2023-06-03T20:11:54.286 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:11:53 smithi164 bash[59462]: cluster 1685823112.9831338 mon.a (mon.0) 1012 : cluster 3 Health check failed: 1 osds down (OSD_DOWN) 2023-06-03T20:11:54.287 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:11:53 smithi164 bash[59462]: cluster 1685823113.0002544 osd.6 (osd.6) 141 : cluster 3 failed to encode map e195 with expected crc 2023-06-03T20:11:54.287 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:11:53 smithi164 bash[59462]: cluster 1685823113.001093 osd.5 (osd.5) 127 : cluster 3 failed to encode map e195 with expected crc 2023-06-03T20:11:54.287 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:11:53 smithi164 bash[59462]: cluster 1685823113.0041392 mon.a (mon.0) 1013 : cluster 0 osdmap e195: 8 total, 7 up, 7 in 2023-06-03T20:11:54.307 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:11:54.311 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:11:54.312 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:11:54.413 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:11:54.445 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:11:54.448 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:11:54.449 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:11:54.550 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:11:54.554 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 21 16 37611 37595 111.865 60.25 0.00792092 0.00893396 2023-06-03T20:11:54.592 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:11:54.595 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:11:54.596 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:11:54.697 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:11:54.727 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:11:54.729 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:11:54.730 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:11:54.831 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:11:54.833 INFO:journalctl@ceph.osd.4.smithi164.stdout:Jun 03 20:11:54 smithi164 bash[101294]: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd-4 2023-06-03T20:11:54.872 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:11:54.877 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:11:54.878 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:11:54.979 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:11:55.007 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:11:55.013 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:11:55.014 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:11:55.083 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:11:55 smithi164 bash[59462]: cluster 1685823113.9995823 osd.6 (osd.6) 142 : cluster 3 failed to encode map e196 with expected crc 2023-06-03T20:11:55.084 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:11:55 smithi164 bash[59462]: cluster 1685823113.9997647 osd.5 (osd.5) 128 : cluster 3 failed to encode map e196 with expected crc 2023-06-03T20:11:55.084 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:11:55 smithi164 bash[59462]: cluster 1685823114.0034223 osd.6 (osd.6) 143 : cluster 3 failed to encode map e196 with expected crc 2023-06-03T20:11:55.084 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:11:55 smithi164 bash[59462]: cluster 1685823114.0037272 osd.5 (osd.5) 129 : cluster 3 failed to encode map e196 with expected crc 2023-06-03T20:11:55.084 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:11:55 smithi164 bash[59462]: cluster 1685823114.0037453 osd.7 (osd.7) 127 : cluster 3 failed to encode map e196 with expected crc 2023-06-03T20:11:55.084 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:11:55 smithi164 bash[59462]: cluster 1685823114.0049834 mon.a (mon.0) 1014 : cluster 0 osdmap e196: 8 total, 7 up, 7 in 2023-06-03T20:11:55.115 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:11:55.148 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:11:55.149 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:11:55.150 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:11:55.175 INFO:teuthology.orchestra.run.smithi164.stderr:Inferring config /var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/mon.b/config 2023-06-03T20:11:55.251 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:11:55.278 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:11:55 smithi089 bash[78452]: cluster 1685823113.9995823 osd.6 (osd.6) 142 : cluster 3 failed to encode map e196 with expected crc 2023-06-03T20:11:55.278 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:11:55 smithi089 bash[78452]: cluster 1685823113.9997647 osd.5 (osd.5) 128 : cluster 3 failed to encode map e196 with expected crc 2023-06-03T20:11:55.279 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:11:55 smithi089 bash[78452]: cluster 1685823114.0034223 osd.6 (osd.6) 143 : cluster 3 failed to encode map e196 with expected crc 2023-06-03T20:11:55.279 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:11:55 smithi089 bash[78452]: cluster 1685823114.0037272 osd.5 (osd.5) 129 : cluster 3 failed to encode map e196 with expected crc 2023-06-03T20:11:55.279 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:11:55 smithi089 bash[78452]: cluster 1685823114.0037453 osd.7 (osd.7) 127 : cluster 3 failed to encode map e196 with expected crc 2023-06-03T20:11:55.279 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:11:55 smithi089 bash[78452]: cluster 1685823114.0049834 mon.a (mon.0) 1014 : cluster 0 osdmap e196: 8 total, 7 up, 7 in 2023-06-03T20:11:55.279 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:11:55 smithi089 bash[75068]: cluster 1685823113.9995823 osd.6 (osd.6) 142 : cluster 3 failed to encode map e196 with expected crc 2023-06-03T20:11:55.279 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:11:55 smithi089 bash[75068]: cluster 1685823113.9997647 osd.5 (osd.5) 128 : cluster 3 failed to encode map e196 with expected crc 2023-06-03T20:11:55.280 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:11:55 smithi089 bash[75068]: cluster 1685823114.0034223 osd.6 (osd.6) 143 : cluster 3 failed to encode map e196 with expected crc 2023-06-03T20:11:55.280 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:11:55 smithi089 bash[75068]: cluster 1685823114.0037272 osd.5 (osd.5) 129 : cluster 3 failed to encode map e196 with expected crc 2023-06-03T20:11:55.280 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:11:55 smithi089 bash[75068]: cluster 1685823114.0037453 osd.7 (osd.7) 127 : cluster 3 failed to encode map e196 with expected crc 2023-06-03T20:11:55.280 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:11:55 smithi089 bash[75068]: cluster 1685823114.0049834 mon.a (mon.0) 1014 : cluster 0 osdmap e196: 8 total, 7 up, 7 in 2023-06-03T20:11:55.285 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:11:55.289 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:11:55.290 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:11:55.367 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/mon.a/config 2023-06-03T20:11:55.371 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/mon.a/config 2023-06-03T20:11:55.391 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:11:55.418 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:11:55.421 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:11:55.422 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:11:55.523 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:11:55.553 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:11:55.556 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 22 16 39237 39221 111.393 101.625 0.00919923 0.00897115 2023-06-03T20:11:55.558 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:11:55.558 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:11:55.659 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:11:55.693 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:11:55.696 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:11:55.697 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:11:55.798 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:11:55.828 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:11:55.833 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:11:55.834 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:11:55.935 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:11:55.976 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:11:55.979 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:11:55.980 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:11:56.081 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:11:56.117 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:11:56.118 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:11:56.119 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:11:56.220 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:11:56.264 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:11:56.269 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:11:56.270 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:11:56.271 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 20:11:56 smithi089 bash[71163]: ::ffff:172.21.15.164 - - [03/Jun/2023:20:11:56] "GET /metrics HTTP/1.1" 200 228795 "" "Prometheus/2.33.4" 2023-06-03T20:11:56.271 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:11:56 smithi089 bash[78452]: cluster 1685823114.6818147 mgr.y (mgr.34104) 1075 : cluster 0 pgmap v1043: 89 pgs: 6 peering, 13 stale+active+clean, 70 active+clean; 2.1 GiB data, 8.6 GiB used, 617 GiB / 626 GiB avail; 105 MiB/s wr, 1.68k op/s 2023-06-03T20:11:56.271 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:11:56 smithi089 bash[78452]: cluster 1685823115.00027 mon.a (mon.0) 1015 : cluster 3 Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2023-06-03T20:11:56.271 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:11:56 smithi089 bash[75068]: cluster 1685823114.6818147 mgr.y (mgr.34104) 1075 : cluster 0 pgmap v1043: 89 pgs: 6 peering, 13 stale+active+clean, 70 active+clean; 2.1 GiB data, 8.6 GiB used, 617 GiB / 626 GiB avail; 105 MiB/s wr, 1.68k op/s 2023-06-03T20:11:56.272 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:11:56 smithi089 bash[75068]: cluster 1685823115.00027 mon.a (mon.0) 1015 : cluster 3 Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2023-06-03T20:11:56.333 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:11:56 smithi164 bash[59462]: cluster 1685823114.6818147 mgr.y (mgr.34104) 1075 : cluster 0 pgmap v1043: 89 pgs: 6 peering, 13 stale+active+clean, 70 active+clean; 2.1 GiB data, 8.6 GiB used, 617 GiB / 626 GiB avail; 105 MiB/s wr, 1.68k op/s 2023-06-03T20:11:56.333 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:11:56 smithi164 bash[59462]: cluster 1685823115.00027 mon.a (mon.0) 1015 : cluster 3 Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2023-06-03T20:11:56.371 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:11:56.402 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:11:56.404 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:11:56.405 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:11:56.506 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:11:56.535 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:11:56.537 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:11:56.537 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:11:56.556 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 23 16 41026 41010 111.411 111.812 0.0108036 0.00897077 2023-06-03T20:11:56.638 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:11:56.669 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:11:56.676 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:11:56.677 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:11:56.777 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:11:56.807 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:11:56.809 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:11:56.810 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:11:56.910 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:11:56.938 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:11:56.941 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:11:56.941 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:11:57.042 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:11:57.074 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:11:57.076 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:11:57.076 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:11:57.177 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:11:57.184 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:11:57 smithi089 bash[75068]: cluster 1685823117.0098646 mon.a (mon.0) 1016 : cluster 3 Health check failed: Degraded data redundancy: 7205/110097 objects degraded (6.544%), 4 pgs degraded (PG_DEGRADED) 2023-06-03T20:11:57.184 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:11:57 smithi089 bash[78452]: cluster 1685823117.0098646 mon.a (mon.0) 1016 : cluster 3 Health check failed: Degraded data redundancy: 7205/110097 objects degraded (6.544%), 4 pgs degraded (PG_DEGRADED) 2023-06-03T20:11:57.201 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:11:57.203 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:11:57.204 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:11:57.305 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:11:57.333 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:11:57 smithi164 bash[59462]: cluster 1685823117.0098646 mon.a (mon.0) 1016 : cluster 3 Health check failed: Degraded data redundancy: 7205/110097 objects degraded (6.544%), 4 pgs degraded (PG_DEGRADED) 2023-06-03T20:11:57.339 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:11:57.342 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:11:57.343 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:11:57.444 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:11:57.475 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:11:57.478 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:11:57.479 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:11:57.556 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 24 16 42136 42120 109.659 69.375 0.0147957 0.0091127 2023-06-03T20:11:57.580 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:11:57.618 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:11:57.621 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:11:57.622 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:11:57.723 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:11:57.757 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:11:57.763 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:11:57.764 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:11:57.865 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:11:57.898 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:11:57.902 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:11:57.902 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:11:58.005 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:11:58.034 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:11:58.038 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:11:58.038 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:11:58.139 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:11:58.171 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:11:58.174 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:11:58.175 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:11:58.275 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:11:58.304 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:11:58.306 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:11:58.307 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:11:58.333 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:11:58 smithi164 bash[59462]: cluster 1685823116.68252 mgr.y (mgr.34104) 1076 : cluster 0 pgmap v1044: 89 pgs: 4 active+undersized+degraded, 7 active+undersized, 6 peering, 9 stale+active+clean, 63 active+clean; 2.2 GiB data, 9.0 GiB used, 617 GiB / 626 GiB avail; 71 MiB/s wr, 1.13k op/s; 7205/110097 objects degraded (6.544%); 16 KiB/s, 0 objects/s recovering 2023-06-03T20:11:58.408 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:11:58.417 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:11:58 smithi089 bash[78452]: cluster 1685823116.68252 mgr.y (mgr.34104) 1076 : cluster 0 pgmap v1044: 89 pgs: 4 active+undersized+degraded, 7 active+undersized, 6 peering, 9 stale+active+clean, 63 active+clean; 2.2 GiB data, 9.0 GiB used, 617 GiB / 626 GiB avail; 71 MiB/s wr, 1.13k op/s; 7205/110097 objects degraded (6.544%); 16 KiB/s, 0 objects/s recovering 2023-06-03T20:11:58.417 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:11:58 smithi089 bash[75068]: cluster 1685823116.68252 mgr.y (mgr.34104) 1076 : cluster 0 pgmap v1044: 89 pgs: 4 active+undersized+degraded, 7 active+undersized, 6 peering, 9 stale+active+clean, 63 active+clean; 2.2 GiB data, 9.0 GiB used, 617 GiB / 626 GiB avail; 71 MiB/s wr, 1.13k op/s; 7205/110097 objects degraded (6.544%); 16 KiB/s, 0 objects/s recovering 2023-06-03T20:11:58.443 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:11:58.445 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:11:58.446 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:11:58.547 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:11:58.556 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 25 16 43761 43745 109.335 101.562 0.00546185 0.00913991 2023-06-03T20:11:58.581 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:11:58.582 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:11:58.582 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:11:58.683 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:11:58.713 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:11:58.716 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:11:58.717 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:11:58.818 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:11:58.855 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:11:58.858 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:11:58.859 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:11:58.960 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:11:58.996 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:11:59.000 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:11:59.001 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:11:59.102 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:11:59.137 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:11:59.141 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:11:59.142 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:11:59.243 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:11:59.276 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:11:59.281 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:11:59.282 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:11:59.382 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:11:59.411 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:11:59.414 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:11:59.414 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:11:59.515 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:11:59.545 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:11:59.549 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:11:59.550 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:11:59.556 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 26 16 45362 45346 108.978 100.062 0.00811161 0.00917049 2023-06-03T20:11:59.652 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:11:59.693 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:11:59.697 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:11:59.698 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:11:59.799 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:11:59.831 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:11:59.834 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:11:59.835 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:11:59.936 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:11:59.980 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:11:59.982 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:11:59.983 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:12:00.084 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:12:00.117 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:12:00.120 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:12:00.121 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:12:00.222 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:12:00.254 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:12:00.258 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:12:00.259 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:12:00.259 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:12:00 smithi164 bash[59462]: cluster 1685823118.6846936 mgr.y (mgr.34104) 1077 : cluster 0 pgmap v1045: 89 pgs: 12 active+undersized+degraded, 23 active+undersized, 6 peering, 48 active+clean; 2.5 GiB data, 9.5 GiB used, 616 GiB / 626 GiB avail; 90 MiB/s wr, 1.43k op/s; 12572/123288 objects degraded (10.197%); 16 KiB/s, 0 objects/s recovering 2023-06-03T20:12:00.278 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:12:00 smithi089 bash[78452]: cluster 1685823118.6846936 mgr.y (mgr.34104) 1077 : cluster 0 pgmap v1045: 89 pgs: 12 active+undersized+degraded, 23 active+undersized, 6 peering, 48 active+clean; 2.5 GiB data, 9.5 GiB used, 616 GiB / 626 GiB avail; 90 MiB/s wr, 1.43k op/s; 12572/123288 objects degraded (10.197%); 16 KiB/s, 0 objects/s recovering 2023-06-03T20:12:00.279 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:12:00 smithi089 bash[75068]: cluster 1685823118.6846936 mgr.y (mgr.34104) 1077 : cluster 0 pgmap v1045: 89 pgs: 12 active+undersized+degraded, 23 active+undersized, 6 peering, 48 active+clean; 2.5 GiB data, 9.5 GiB used, 616 GiB / 626 GiB avail; 90 MiB/s wr, 1.43k op/s; 12572/123288 objects degraded (10.197%); 16 KiB/s, 0 objects/s recovering 2023-06-03T20:12:00.360 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:12:00.388 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:12:00.390 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:12:00.391 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:12:00.491 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:12:00.525 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:12:00.528 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:12:00.529 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:12:00.556 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 27 16 46876 46860 108.446 94.625 0.00912923 0.00921559 2023-06-03T20:12:00.630 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:12:00.663 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:12:00.669 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:12:00.670 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:12:00.771 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:12:00.799 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:12:00.801 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:12:00.802 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:12:00.903 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:12:00.934 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:12:00.936 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:12:00.937 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:12:01.037 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:12:01.074 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:12:01.078 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:12:01.079 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:12:01.180 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:12:01.188 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:12:01 smithi164 bash[59462]: cluster 1685823120.6854708 mgr.y (mgr.34104) 1078 : cluster 0 pgmap v1046: 89 pgs: 15 active+undersized+degraded, 26 active+undersized, 48 active+clean; 2.6 GiB data, 9.8 GiB used, 616 GiB / 626 GiB avail; 83 MiB/s wr, 1.33k op/s; 16306/129054 objects degraded (12.635%); 16 KiB/s, 0 objects/s recovering 2023-06-03T20:12:01.188 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:12:01 smithi164 bash[59462]: cluster 1685823121.0660903 mon.a (mon.0) 1017 : cluster 1 Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2023-06-03T20:12:01.214 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:12:01.216 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:12:01.217 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:12:01.318 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:12:01.327 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:12:01 smithi089 bash[78452]: cluster 1685823120.6854708 mgr.y (mgr.34104) 1078 : cluster 0 pgmap v1046: 89 pgs: 15 active+undersized+degraded, 26 active+undersized, 48 active+clean; 2.6 GiB data, 9.8 GiB used, 616 GiB / 626 GiB avail; 83 MiB/s wr, 1.33k op/s; 16306/129054 objects degraded (12.635%); 16 KiB/s, 0 objects/s recovering 2023-06-03T20:12:01.327 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:12:01 smithi089 bash[78452]: cluster 1685823121.0660903 mon.a (mon.0) 1017 : cluster 1 Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2023-06-03T20:12:01.328 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:12:01 smithi089 bash[75068]: cluster 1685823120.6854708 mgr.y (mgr.34104) 1078 : cluster 0 pgmap v1046: 89 pgs: 15 active+undersized+degraded, 26 active+undersized, 48 active+clean; 2.6 GiB data, 9.8 GiB used, 616 GiB / 626 GiB avail; 83 MiB/s wr, 1.33k op/s; 16306/129054 objects degraded (12.635%); 16 KiB/s, 0 objects/s recovering 2023-06-03T20:12:01.328 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:12:01 smithi089 bash[75068]: cluster 1685823121.0660903 mon.a (mon.0) 1017 : cluster 1 Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2023-06-03T20:12:01.351 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:12:01.353 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:12:01.354 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:12:01.455 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:12:01.502 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:12:01.505 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:12:01.506 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:12:01.556 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 28 16 48444 48428 108.073 98 0.0112977 0.0092475 2023-06-03T20:12:01.607 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:12:01.638 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:12:01.645 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:12:01.645 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:12:01.746 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:12:01.781 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:12:01.784 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:12:01.785 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:12:01.886 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:12:01.930 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:12:01.933 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:12:01.934 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:12:02.014 DEBUG:teuthology.orchestra.run.smithi164:> sudo pkill -f 'journalctl -f -n 0 -u ceph-f1db3478-0243-11ee-9b1d-001a4aab830c@osd.4.service' 2023-06-03T20:12:02.034 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:12:02.051 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-06-03T20:12:02.052 INFO:tasks.cephadm.osd.4:Stopped osd.4 2023-06-03T20:12:02.131 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:12:02.137 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:12:02.138 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:12:02.239 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:12:02.268 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:12:02.270 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:12:02.270 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:12:02.371 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:12:02.397 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:12:02.399 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:12:02.399 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:12:02.500 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:12:02.536 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:12:02.538 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:12:02.539 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:12:02.556 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 29 16 49486 49470 106.592 65.125 0.0135052 0.00937641 2023-06-03T20:12:02.640 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:12:02.672 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:12:02.676 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:12:02.677 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:12:02.778 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:12:02.781 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:12:02 smithi164 bash[59462]: cluster 1685823122.7218792 mon.a (mon.0) 1018 : cluster 3 Health check update: Degraded data redundancy: 16306/129054 objects degraded (12.635%), 15 pgs degraded (PG_DEGRADED) 2023-06-03T20:12:02.807 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:12:02.809 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:12:02.810 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:12:02.911 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:12:02.950 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:12:02.951 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:12:02.952 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:12:03.052 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:12:03.060 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:12:02 smithi089 bash[78452]: cluster 1685823122.7218792 mon.a (mon.0) 1018 : cluster 3 Health check update: Degraded data redundancy: 16306/129054 objects degraded (12.635%), 15 pgs degraded (PG_DEGRADED) 2023-06-03T20:12:03.061 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:12:02 smithi089 bash[75068]: cluster 1685823122.7218792 mon.a (mon.0) 1018 : cluster 3 Health check update: Degraded data redundancy: 16306/129054 objects degraded (12.635%), 15 pgs degraded (PG_DEGRADED) 2023-06-03T20:12:03.081 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:12:03.084 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:12:03.085 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:12:03.186 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:12:03.218 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:12:03.225 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:12:03.226 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:12:03.327 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:12:03.355 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:12:03.360 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:12:03.361 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:12:03.427 INFO:teuthology.orchestra.run.smithi164.stdout:ERROR: (22) Invalid argument 2023-06-03T20:12:03.428 INFO:teuthology.orchestra.run.smithi164.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-06-03T20:12:03.461 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:12:03.493 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:12:03.497 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:12:03.498 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:12:03.556 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 30 16 50959 50943 106.108 92.0625 0.00851642 0.00941952 2023-06-03T20:12:03.599 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:12:03.634 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:12:03.638 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:12:03.638 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:12:03.739 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:12:03.771 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:12:03.773 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:12:03.774 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:12:03.781 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:12:03 smithi164 bash[59462]: cluster 1685823122.6861231 mgr.y (mgr.34104) 1079 : cluster 0 pgmap v1047: 89 pgs: 15 active+undersized+degraded, 26 active+undersized, 48 active+clean; 2.8 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 72 MiB/s wr, 1.14k op/s; 17232/137223 objects degraded (12.558%); 13 KiB/s, 0 objects/s recovering 2023-06-03T20:12:03.876 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:12:03.918 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:12:03.922 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:12:03.923 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:12:04.025 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:12:04.028 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:12:03 smithi089 bash[75068]: cluster 1685823122.6861231 mgr.y (mgr.34104) 1079 : cluster 0 pgmap v1047: 89 pgs: 15 active+undersized+degraded, 26 active+undersized, 48 active+clean; 2.8 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 72 MiB/s wr, 1.14k op/s; 17232/137223 objects degraded (12.558%); 13 KiB/s, 0 objects/s recovering 2023-06-03T20:12:04.029 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:12:03 smithi089 bash[78452]: cluster 1685823122.6861231 mgr.y (mgr.34104) 1079 : cluster 0 pgmap v1047: 89 pgs: 15 active+undersized+degraded, 26 active+undersized, 48 active+clean; 2.8 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 72 MiB/s wr, 1.14k op/s; 17232/137223 objects degraded (12.558%); 13 KiB/s, 0 objects/s recovering 2023-06-03T20:12:04.059 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:12:04.062 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:12:04.063 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:12:04.164 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:12:04.199 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:12:04.204 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:12:04.205 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:12:04.306 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:12:04.345 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:12:04.349 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:12:04.350 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:12:04.451 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:12:04.486 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:12:04.496 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:12:04.496 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:12:04.556 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 31 16 52447 52431 105.684 93 0.00355517 0.00945624 2023-06-03T20:12:04.597 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:12:04.630 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:12:04.633 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:12:04.634 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:12:04.735 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:12:04.772 DEBUG:teuthology.orchestra.run.smithi164:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph daemon osd.5 dump_historic_ops 2023-06-03T20:12:04.775 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:12:04.776 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:12:04.777 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:12:04.879 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:12:04.921 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:12:04.923 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:12:04.924 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:12:05.025 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:12:05.067 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:12:05.068 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:12:05.068 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:12:05.169 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:12:05.208 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:12:05.212 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:12:05.212 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:12:05.313 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:12:05.358 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:12:05.360 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:12:05.361 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:12:05.462 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:12:05.507 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:12:05.510 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:12:05.511 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:12:05.557 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 32 16 53884 53868 105.188 89.8125 0.00581304 0.00950103 2023-06-03T20:12:05.612 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:12:05.640 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:12:05.642 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:12:05.643 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:12:05.743 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:12:05.777 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:12:05.780 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:12:05.780 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:12:05.881 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:12:05.917 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:12:05.920 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:12:05.921 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:12:06.022 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:12:06.028 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:12:05 smithi089 bash[78452]: cluster 1685823124.686847 mgr.y (mgr.34104) 1080 : cluster 0 pgmap v1048: 89 pgs: 15 active+undersized+degraded, 26 active+undersized, 48 active+clean; 3.0 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 81 MiB/s wr, 1.30k op/s; 18142/146784 objects degraded (12.360%); 12 KiB/s, 0 objects/s recovering 2023-06-03T20:12:06.028 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:12:05 smithi089 bash[75068]: cluster 1685823124.686847 mgr.y (mgr.34104) 1080 : cluster 0 pgmap v1048: 89 pgs: 15 active+undersized+degraded, 26 active+undersized, 48 active+clean; 3.0 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 81 MiB/s wr, 1.30k op/s; 18142/146784 objects degraded (12.360%); 12 KiB/s, 0 objects/s recovering 2023-06-03T20:12:06.059 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:12:06.061 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:12:05 smithi164 bash[59462]: cluster 1685823124.686847 mgr.y (mgr.34104) 1080 : cluster 0 pgmap v1048: 89 pgs: 15 active+undersized+degraded, 26 active+undersized, 48 active+clean; 3.0 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 81 MiB/s wr, 1.30k op/s; 18142/146784 objects degraded (12.360%); 12 KiB/s, 0 objects/s recovering 2023-06-03T20:12:06.062 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:12:06.062 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:12:06.163 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:12:06.196 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:12:06.199 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:12:06.199 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:12:06.300 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:12:06.328 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:12:06.330 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:12:06.331 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:12:06.431 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:12:06.463 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:12:06.468 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:12:06.469 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:12:06.528 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 20:12:06 smithi089 bash[71163]: ::ffff:172.21.15.164 - - [03/Jun/2023:20:12:06] "GET /metrics HTTP/1.1" 200 228795 "" "Prometheus/2.33.4" 2023-06-03T20:12:06.540 INFO:teuthology.orchestra.run.smithi089.stdout:osd.0: {} 2023-06-03T20:12:06.557 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 33 16 55248 55232 104.584 85.25 0.0211992 0.00955597 2023-06-03T20:12:06.569 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:12:06.586 INFO:teuthology.orchestra.run.smithi089.stdout:osd.1: {} 2023-06-03T20:12:06.603 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:12:06.607 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:12:06.608 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:12:06.625 INFO:teuthology.orchestra.run.smithi089.stdout:osd.2: {} 2023-06-03T20:12:06.672 INFO:teuthology.orchestra.run.smithi089.stdout:osd.3: {} 2023-06-03T20:12:06.706 INFO:teuthology.orchestra.run.smithi089.stdout:osd.5: {} 2023-06-03T20:12:06.709 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:12:06.745 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:12:06.750 INFO:teuthology.orchestra.run.smithi089.stdout:osd.6: {} 2023-06-03T20:12:06.750 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:12:06.750 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:12:06.792 INFO:teuthology.orchestra.run.smithi089.stdout:osd.7: {} 2023-06-03T20:12:06.792 INFO:teuthology.orchestra.run.smithi089.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-06-03T20:12:06.793 INFO:teuthology.orchestra.run.smithi089.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-06-03T20:12:06.793 INFO:teuthology.orchestra.run.smithi089.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-06-03T20:12:06.793 INFO:teuthology.orchestra.run.smithi089.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-06-03T20:12:06.793 INFO:teuthology.orchestra.run.smithi089.stderr:Error ENXIO: problem getting command descriptions from osd.4 2023-06-03T20:12:06.793 INFO:teuthology.orchestra.run.smithi089.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-06-03T20:12:06.793 INFO:teuthology.orchestra.run.smithi089.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-06-03T20:12:06.793 INFO:teuthology.orchestra.run.smithi089.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-06-03T20:12:06.852 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:12:06.892 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:12:06.894 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:12:06.895 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:12:06.996 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:12:07.003 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:12:06 smithi164 bash[59462]: audit 1685823126.4755135 mon.c (mon.2) 503 : audit 0 from='client.? 172.21.15.89:0/3216193866' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-03T20:12:07.028 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:12:06 smithi089 bash[78452]: audit 1685823126.4755135 mon.c (mon.2) 503 : audit 0 from='client.? 172.21.15.89:0/3216193866' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-03T20:12:07.028 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:12:06 smithi089 bash[75068]: audit 1685823126.4755135 mon.c (mon.2) 503 : audit 0 from='client.? 172.21.15.89:0/3216193866' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-03T20:12:07.030 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:12:07.037 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:12:07.037 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:12:07.053 INFO:tasks.thrashosds.thrasher:in_osds: [0, 2, 3, 4, 5, 6, 7] out_osds: [1] dead_osds: [4] live_osds: [0, 1, 2, 3, 5, 6, 7] 2023-06-03T20:12:07.053 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-06-03T20:12:07.053 INFO:tasks.thrashosds.thrasher:Reweighting osd 6 to 0.5804197722268256 2023-06-03T20:12:07.054 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph osd reweight 6 0.5804197722268256 2023-06-03T20:12:07.138 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:12:07.184 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:12:07.188 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:12:07.189 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:12:07.289 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:12:07.326 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:12:07.326 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:12:07.327 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:12:07.429 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:12:07.471 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:12:07.474 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:12:07.475 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:12:07.557 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 34 16 56513 56497 103.833 79.0625 0.00406272 0.00962421 2023-06-03T20:12:07.576 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:12:07.621 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:12:07.623 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:12:07.624 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:12:07.725 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:12:07.758 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:12:07.761 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:12:07.762 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:12:07.812 INFO:teuthology.orchestra.run.smithi089.stderr:nodeep-scrub is set 2023-06-03T20:12:07.863 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:12:07.897 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:12:07.901 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:12:07.901 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:12:08.002 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:12:08.029 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:12:08.031 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:12:08.032 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:12:08.083 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:12:07 smithi164 bash[59462]: cluster 1685823126.6876037 mgr.y (mgr.34104) 1081 : cluster 0 pgmap v1049: 89 pgs: 15 active+undersized+degraded, 26 active+undersized, 48 active+clean; 3.1 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 82 MiB/s wr, 1.31k op/s; 19508/152301 objects degraded (12.809%); 11 KiB/s, 0 objects/s recovering 2023-06-03T20:12:08.084 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:12:07 smithi164 bash[59462]: audit 1685823126.827719 mon.c (mon.2) 504 : audit 1 from='client.? 172.21.15.89:0/3138351158' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-06-03T20:12:08.084 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:12:07 smithi164 bash[59462]: audit 1685823126.8283384 mon.a (mon.0) 1019 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-06-03T20:12:08.084 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:12:07 smithi164 bash[59462]: audit 1685823127.121972 mon.a (mon.0) 1020 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:12:08.084 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:12:07 smithi164 bash[59462]: audit 1685823127.1262755 mon.c (mon.2) 505 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T20:12:08.084 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:12:07 smithi164 bash[59462]: cluster 1685823127.723139 mon.a (mon.0) 1021 : cluster 3 Health check update: Degraded data redundancy: 19508/152301 objects degraded (12.809%), 15 pgs degraded (PG_DEGRADED) 2023-06-03T20:12:08.133 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:12:08.162 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:12:08.167 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:12:08.168 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:12:08.268 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:12:08.278 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:12:07 smithi089 bash[75068]: cluster 1685823126.6876037 mgr.y (mgr.34104) 1081 : cluster 0 pgmap v1049: 89 pgs: 15 active+undersized+degraded, 26 active+undersized, 48 active+clean; 3.1 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 82 MiB/s wr, 1.31k op/s; 19508/152301 objects degraded (12.809%); 11 KiB/s, 0 objects/s recovering 2023-06-03T20:12:08.278 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:12:07 smithi089 bash[75068]: audit 1685823126.827719 mon.c (mon.2) 504 : audit 1 from='client.? 172.21.15.89:0/3138351158' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-06-03T20:12:08.278 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:12:07 smithi089 bash[75068]: audit 1685823126.8283384 mon.a (mon.0) 1019 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-06-03T20:12:08.279 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:12:07 smithi089 bash[75068]: audit 1685823127.121972 mon.a (mon.0) 1020 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:12:08.279 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:12:07 smithi089 bash[75068]: audit 1685823127.1262755 mon.c (mon.2) 505 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T20:12:08.279 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:12:07 smithi089 bash[75068]: cluster 1685823127.723139 mon.a (mon.0) 1021 : cluster 3 Health check update: Degraded data redundancy: 19508/152301 objects degraded (12.809%), 15 pgs degraded (PG_DEGRADED) 2023-06-03T20:12:08.279 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:12:07 smithi089 bash[78452]: cluster 1685823126.6876037 mgr.y (mgr.34104) 1081 : cluster 0 pgmap v1049: 89 pgs: 15 active+undersized+degraded, 26 active+undersized, 48 active+clean; 3.1 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 82 MiB/s wr, 1.31k op/s; 19508/152301 objects degraded (12.809%); 11 KiB/s, 0 objects/s recovering 2023-06-03T20:12:08.279 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:12:07 smithi089 bash[78452]: audit 1685823126.827719 mon.c (mon.2) 504 : audit 1 from='client.? 172.21.15.89:0/3138351158' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-06-03T20:12:08.279 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:12:07 smithi089 bash[78452]: audit 1685823126.8283384 mon.a (mon.0) 1019 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-06-03T20:12:08.280 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:12:07 smithi089 bash[78452]: audit 1685823127.121972 mon.a (mon.0) 1020 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:12:08.280 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:12:07 smithi089 bash[78452]: audit 1685823127.1262755 mon.c (mon.2) 505 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T20:12:08.280 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:12:07 smithi089 bash[78452]: cluster 1685823127.723139 mon.a (mon.0) 1021 : cluster 3 Health check update: Degraded data redundancy: 19508/152301 objects degraded (12.809%), 15 pgs degraded (PG_DEGRADED) 2023-06-03T20:12:08.302 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:12:08.307 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:12:08.308 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:12:08.408 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:12:08.445 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:12:08.448 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:12:08.448 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:12:08.549 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:12:08.557 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 35 16 57815 57799 103.191 81.375 0.00252075 0.00968065 2023-06-03T20:12:08.586 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:12:08.591 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:12:08.591 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:12:08.692 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:12:08.729 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:12:08.732 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:12:08.733 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:12:08.819 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:12:08 smithi164 bash[59462]: cluster 1685823127.7906704 mon.a (mon.0) 1022 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-06-03T20:12:08.819 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:12:08 smithi164 bash[59462]: cluster 1685823127.7921236 osd.6 (osd.6) 144 : cluster 3 failed to encode map e197 with expected crc 2023-06-03T20:12:08.820 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:12:08 smithi164 bash[59462]: cluster 1685823127.7957702 osd.5 (osd.5) 130 : cluster 3 failed to encode map e197 with expected crc 2023-06-03T20:12:08.820 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:12:08 smithi164 bash[59462]: audit 1685823127.7961216 mon.a (mon.0) 1023 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-06-03T20:12:08.820 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:12:08 smithi164 bash[59462]: cluster 1685823127.7962015 mon.a (mon.0) 1024 : cluster 0 osdmap e197: 8 total, 7 up, 7 in 2023-06-03T20:12:08.833 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:12:08.882 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:12:08.886 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:12:08.887 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:12:08.988 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:12:09.036 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:12:09.041 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:12:09.042 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:12:09.048 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-06-03T20:12:09.049 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-06-03T20:12:09.144 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:12:09.161 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:12:08 smithi089 bash[75068]: cluster 1685823127.7906704 mon.a (mon.0) 1022 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-06-03T20:12:09.161 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:12:08 smithi089 bash[75068]: cluster 1685823127.7921236 osd.6 (osd.6) 144 : cluster 3 failed to encode map e197 with expected crc 2023-06-03T20:12:09.161 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:12:08 smithi089 bash[75068]: cluster 1685823127.7957702 osd.5 (osd.5) 130 : cluster 3 failed to encode map e197 with expected crc 2023-06-03T20:12:09.161 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:12:08 smithi089 bash[75068]: audit 1685823127.7961216 mon.a (mon.0) 1023 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-06-03T20:12:09.162 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:12:08 smithi089 bash[75068]: cluster 1685823127.7962015 mon.a (mon.0) 1024 : cluster 0 osdmap e197: 8 total, 7 up, 7 in 2023-06-03T20:12:09.162 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:12:08 smithi089 bash[78452]: cluster 1685823127.7906704 mon.a (mon.0) 1022 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-06-03T20:12:09.162 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:12:08 smithi089 bash[78452]: cluster 1685823127.7921236 osd.6 (osd.6) 144 : cluster 3 failed to encode map e197 with expected crc 2023-06-03T20:12:09.162 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:12:08 smithi089 bash[78452]: cluster 1685823127.7957702 osd.5 (osd.5) 130 : cluster 3 failed to encode map e197 with expected crc 2023-06-03T20:12:09.162 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:12:08 smithi089 bash[78452]: audit 1685823127.7961216 mon.a (mon.0) 1023 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-06-03T20:12:09.162 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:12:08 smithi089 bash[78452]: cluster 1685823127.7962015 mon.a (mon.0) 1024 : cluster 0 osdmap e197: 8 total, 7 up, 7 in 2023-06-03T20:12:09.183 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:12:09.186 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:12:09.186 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:12:09.287 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:12:09.331 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:12:09.336 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:12:09.336 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:12:09.437 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:12:09.473 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:12:09.475 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:12:09.476 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:12:09.557 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 36 16 58944 58928 102.285 70.5625 0.0117755 0.00977119 2023-06-03T20:12:09.577 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:12:09.611 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:12:09.616 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:12:09.617 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:12:09.624 INFO:teuthology.orchestra.run.smithi164.stderr:Inferring config /var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/mon.b/config 2023-06-03T20:12:09.718 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:12:09.746 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:12:09.750 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:12:09.751 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:12:09.851 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:12:09.887 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:12:09.891 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:12:09.892 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:12:09.993 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:12:10.036 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:12:10.039 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:12:10.040 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:12:10.083 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:12:09 smithi164 bash[59462]: cluster 1685823128.6881835 mgr.y (mgr.34104) 1082 : cluster 0 pgmap v1051: 89 pgs: 15 active+undersized+degraded, 26 active+undersized, 48 active+clean; 3.4 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 90 MiB/s wr, 1.44k op/s; 20337/166503 objects degraded (12.214%) 2023-06-03T20:12:10.141 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:12:10.149 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:12:09 smithi089 bash[75068]: cluster 1685823128.6881835 mgr.y (mgr.34104) 1082 : cluster 0 pgmap v1051: 89 pgs: 15 active+undersized+degraded, 26 active+undersized, 48 active+clean; 3.4 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 90 MiB/s wr, 1.44k op/s; 20337/166503 objects degraded (12.214%) 2023-06-03T20:12:10.149 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:12:09 smithi089 bash[78452]: cluster 1685823128.6881835 mgr.y (mgr.34104) 1082 : cluster 0 pgmap v1051: 89 pgs: 15 active+undersized+degraded, 26 active+undersized, 48 active+clean; 3.4 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 90 MiB/s wr, 1.44k op/s; 20337/166503 objects degraded (12.214%) 2023-06-03T20:12:10.172 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:12:10.178 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:12:10.179 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:12:10.280 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:12:10.310 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:12:10.313 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:12:10.314 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:12:10.414 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:12:10.450 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:12:10.452 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:12:10.453 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:12:10.554 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:12:10.557 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 37 16 60272 60256 101.764 83 0.011866 0.0098212 2023-06-03T20:12:10.598 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:12:10.601 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:12:10.602 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:12:10.702 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:12:10.731 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:12:10.733 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:12:10.733 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:12:10.834 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:12:10.863 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:12:10.866 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:12:10.866 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:12:10.967 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:12:11.011 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:12:11.015 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:12:11.015 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:12:11.051 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-06-03T20:12:11.117 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:12:11.157 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:12:11.161 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:12:11.161 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:12:11.262 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:12:11.307 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:12:11.315 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:12:11.316 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:12:11.417 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:12:11.450 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:12:11.453 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:12:11.453 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:12:11.554 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:12:11.557 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 38 16 61220 61204 100.645 59.25 0.0110875 0.0099298 2023-06-03T20:12:11.591 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:12:11.597 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:12:11.598 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:12:11.699 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:12:11.732 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:12:11.736 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:12:11.736 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:12:11.837 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:12:11.872 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:12:11.874 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:12:11.875 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:12:11.976 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:12:11.986 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:12:11 smithi164 bash[59462]: cluster 1685823130.689002 mgr.y (mgr.34104) 1083 : cluster 0 pgmap v1052: 89 pgs: 15 active+undersized+degraded, 26 active+undersized, 48 active+clean; 3.5 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 88 MiB/s wr, 1.42k op/s; 21592/171525 objects degraded (12.588%) 2023-06-03T20:12:12.012 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph osd unset noscrub 2023-06-03T20:12:12.013 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:12:12.015 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:12:12.016 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:12:12.117 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:12:12.150 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:12:12.153 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:12:12.154 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:12:12.255 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:12:12.267 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:12:11 smithi089 bash[78452]: cluster 1685823130.689002 mgr.y (mgr.34104) 1083 : cluster 0 pgmap v1052: 89 pgs: 15 active+undersized+degraded, 26 active+undersized, 48 active+clean; 3.5 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 88 MiB/s wr, 1.42k op/s; 21592/171525 objects degraded (12.588%) 2023-06-03T20:12:12.267 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:12:11 smithi089 bash[75068]: cluster 1685823130.689002 mgr.y (mgr.34104) 1083 : cluster 0 pgmap v1052: 89 pgs: 15 active+undersized+degraded, 26 active+undersized, 48 active+clean; 3.5 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 88 MiB/s wr, 1.42k op/s; 21592/171525 objects degraded (12.588%) 2023-06-03T20:12:12.296 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:12:12.298 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:12:12.299 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:12:12.400 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:12:12.431 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:12:12.439 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:12:12.440 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:12:12.541 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:12:12.557 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 39 16 62492 62476 100.103 79.5 0.0180756 0.00998399 2023-06-03T20:12:12.577 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:12:12.581 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:12:12.581 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:12:12.682 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:12:12.715 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:12:12.721 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:12:12.721 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:12:12.822 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:12:12.861 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:12:12.863 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:12:12.864 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:12:12.965 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:12:12.974 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:12:12 smithi164 bash[59462]: cluster 1685823132.7243304 mon.a (mon.0) 1025 : cluster 3 Health check update: Degraded data redundancy: 21592/171525 objects degraded (12.588%), 15 pgs degraded (PG_DEGRADED) 2023-06-03T20:12:13.000 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:12:13.004 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:12:13.005 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:12:13.106 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:12:13.115 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:12:12 smithi089 bash[78452]: cluster 1685823132.7243304 mon.a (mon.0) 1025 : cluster 3 Health check update: Degraded data redundancy: 21592/171525 objects degraded (12.588%), 15 pgs degraded (PG_DEGRADED) 2023-06-03T20:12:13.115 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:12:12 smithi089 bash[75068]: cluster 1685823132.7243304 mon.a (mon.0) 1025 : cluster 3 Health check update: Degraded data redundancy: 21592/171525 objects degraded (12.588%), 15 pgs degraded (PG_DEGRADED) 2023-06-03T20:12:13.158 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:12:13.161 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:12:13.161 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:12:13.262 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:12:13.310 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:12:13.314 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:12:13.314 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:12:13.415 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:12:13.453 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:12:13.455 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:12:13.456 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:12:13.556 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:12:13.557 INFO:tasks.radosbench.radosbench.0.smithi089.stdout:2023-06-03T20:12:13.553941+0000 min lat: 0.00135638 max lat: 1.05422 avg lat: 0.0100298 2023-06-03T20:12:13.558 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-06-03T20:12:13.558 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 40 16 63803 63787 99.6483 81.9375 0.0190786 0.0100298 2023-06-03T20:12:13.584 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/mon.a/config 2023-06-03T20:12:13.605 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:12:13.616 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:12:13.616 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:12:13.717 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:12:13.762 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:12:13.764 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:12:13.765 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:12:13.834 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:12:13 smithi089 bash[75068]: cluster 1685823132.6898024 mgr.y (mgr.34104) 1084 : cluster 0 pgmap v1053: 89 pgs: 15 active+undersized+degraded, 26 active+undersized, 48 active+clean; 3.6 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 85 MiB/s wr, 1.37k op/s; 22331/178233 objects degraded (12.529%) 2023-06-03T20:12:13.866 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:12:13.899 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:12:13.904 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:12:13.905 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:12:14.006 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:12:14.014 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:12:13 smithi164 bash[59462]: cluster 1685823132.6898024 mgr.y (mgr.34104) 1084 : cluster 0 pgmap v1053: 89 pgs: 15 active+undersized+degraded, 26 active+undersized, 48 active+clean; 3.6 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 85 MiB/s wr, 1.37k op/s; 22331/178233 objects degraded (12.529%) 2023-06-03T20:12:14.035 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:12:14.040 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:12:14.041 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:12:14.142 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:12:14.149 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:12:13 smithi089 bash[78452]: cluster 1685823132.6898024 mgr.y (mgr.34104) 1084 : cluster 0 pgmap v1053: 89 pgs: 15 active+undersized+degraded, 26 active+undersized, 48 active+clean; 3.6 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 85 MiB/s wr, 1.37k op/s; 22331/178233 objects degraded (12.529%) 2023-06-03T20:12:14.176 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:12:14.178 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:12:14.178 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:12:14.279 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:12:14.311 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:12:14.315 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:12:14.316 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:12:14.416 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:12:14.450 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:12:14.453 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:12:14.453 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:12:14.554 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:12:14.557 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 41 16 65031 65015 99.0897 76.75 0.0114708 0.0100859 2023-06-03T20:12:14.589 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:12:14.597 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:12:14.598 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:12:14.699 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:12:14.733 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:12:14.735 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:12:14.735 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:12:14.836 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:12:14.865 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:12:14.868 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:12:14.869 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:12:14.970 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:12:15.003 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:12:15.010 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:12:15.011 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:12:15.111 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:12:15.141 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:12:15.144 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:12:15.144 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:12:15.245 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:12:15.279 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:12:15.282 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:12:15.283 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:12:15.383 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:12:15.409 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:12:15.412 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:12:15.413 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:12:15.514 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:12:15.546 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:12:15.549 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:12:15.550 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:12:15.557 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 42 15 66254 66239 98.5517 76.5 0.010551 0.0101404 2023-06-03T20:12:15.635 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/mon.a/config 2023-06-03T20:12:15.651 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:12:15.683 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:12:15.686 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:12:15.687 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:12:15.739 INFO:teuthology.orchestra.run.smithi164.stdout:{ 2023-06-03T20:12:15.739 INFO:teuthology.orchestra.run.smithi164.stdout: "size": 20, 2023-06-03T20:12:15.739 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 600, 2023-06-03T20:12:15.739 INFO:teuthology.orchestra.run.smithi164.stdout: "ops": [ 2023-06-03T20:12:15.740 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:12:15.740 INFO:teuthology.orchestra.run.smithi164.stdout: "description": "osd_op(client.45713.0:145272 9.6 9:60b8b740:::benchmark_data_smithi089_115580_object35138:head [delete] snapc 0=[] ondisk+write+known_if_redirected e138)", 2023-06-03T20:12:15.740 INFO:teuthology.orchestra.run.smithi164.stdout: "initiated_at": "2023-06-03T20:02:57.731374+0000", 2023-06-03T20:12:15.740 INFO:teuthology.orchestra.run.smithi164.stdout: "age": 557.99720737099994, 2023-06-03T20:12:15.740 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.22439163300000001, 2023-06-03T20:12:15.740 INFO:teuthology.orchestra.run.smithi164.stdout: "type_data": { 2023-06-03T20:12:15.740 INFO:teuthology.orchestra.run.smithi164.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-03T20:12:15.741 INFO:teuthology.orchestra.run.smithi164.stdout: "client_info": { 2023-06-03T20:12:15.741 INFO:teuthology.orchestra.run.smithi164.stdout: "client": "client.45713", 2023-06-03T20:12:15.741 INFO:teuthology.orchestra.run.smithi164.stdout: "client_addr": "172.21.15.89:0/2400821448", 2023-06-03T20:12:15.741 INFO:teuthology.orchestra.run.smithi164.stdout: "tid": 145272 2023-06-03T20:12:15.741 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:12:15.741 INFO:teuthology.orchestra.run.smithi164.stdout: "events": [ 2023-06-03T20:12:15.741 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:12:15.741 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "initiated", 2023-06-03T20:12:15.742 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:02:57.731374+0000", 2023-06-03T20:12:15.742 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0 2023-06-03T20:12:15.742 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:12:15.742 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:12:15.742 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "throttled", 2023-06-03T20:12:15.742 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:02:57.731374+0000", 2023-06-03T20:12:15.742 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 4294967295.9999995 2023-06-03T20:12:15.743 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:12:15.743 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:12:15.743 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "header_read", 2023-06-03T20:12:15.743 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:02:57.731373+0000", 2023-06-03T20:12:15.743 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 4.1500000000000001e-06 2023-06-03T20:12:15.743 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:12:15.743 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:12:15.743 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "all_read", 2023-06-03T20:12:15.744 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:02:57.731377+0000", 2023-06-03T20:12:15.744 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 7.1049999999999997e-06 2023-06-03T20:12:15.744 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:12:15.744 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:12:15.744 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "dispatched", 2023-06-03T20:12:15.744 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:02:57.731385+0000", 2023-06-03T20:12:15.744 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 4.3560000000000003e-06 2023-06-03T20:12:15.744 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:12:15.745 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:12:15.745 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "queued_for_pg", 2023-06-03T20:12:15.745 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:02:57.731389+0000", 2023-06-03T20:12:15.745 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.00087223200000000002 2023-06-03T20:12:15.745 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:12:15.745 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:12:15.745 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "reached_pg", 2023-06-03T20:12:15.746 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:02:57.732261+0000", 2023-06-03T20:12:15.746 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.000111885 2023-06-03T20:12:15.746 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:12:15.746 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:12:15.746 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "started", 2023-06-03T20:12:15.746 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:02:57.732373+0000", 2023-06-03T20:12:15.746 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.000117779 2023-06-03T20:12:15.746 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:12:15.747 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:12:15.747 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "waiting for subops from 1,2", 2023-06-03T20:12:15.747 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:02:57.732491+0000", 2023-06-03T20:12:15.747 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.0048627050000000002 2023-06-03T20:12:15.747 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:12:15.747 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:12:15.747 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:12:15.747 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:02:57.737353+0000", 2023-06-03T20:12:15.748 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.00013928399999999999 2023-06-03T20:12:15.748 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:12:15.748 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:12:15.748 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "op_commit", 2023-06-03T20:12:15.748 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:02:57.737493+0000", 2023-06-03T20:12:15.748 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.21809168400000001 2023-06-03T20:12:15.748 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:12:15.748 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:12:15.749 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:12:15.749 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:02:57.955584+0000", 2023-06-03T20:12:15.749 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 8.7686e-05 2023-06-03T20:12:15.749 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:12:15.749 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:12:15.749 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "commit_sent", 2023-06-03T20:12:15.749 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:02:57.955672+0000", 2023-06-03T20:12:15.749 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 9.3391000000000001e-05 2023-06-03T20:12:15.750 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:12:15.750 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:12:15.750 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "done", 2023-06-03T20:12:15.750 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:02:57.955766+0000", 2023-06-03T20:12:15.750 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.22439163300000001 2023-06-03T20:12:15.750 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:12:15.750 INFO:teuthology.orchestra.run.smithi164.stdout: ] 2023-06-03T20:12:15.750 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:12:15.751 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:12:15.751 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:12:15.751 INFO:teuthology.orchestra.run.smithi164.stdout: "description": "osd_op(client.45713.0:145282 9.6 9:6a07824c:::benchmark_data_smithi089_115580_object35148:head [delete] snapc 0=[] ondisk+write+known_if_redirected e138)", 2023-06-03T20:12:15.751 INFO:teuthology.orchestra.run.smithi164.stdout: "initiated_at": "2023-06-03T20:02:57.745785+0000", 2023-06-03T20:12:15.751 INFO:teuthology.orchestra.run.smithi164.stdout: "age": 557.98279584399995, 2023-06-03T20:12:15.751 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.210114734, 2023-06-03T20:12:15.751 INFO:teuthology.orchestra.run.smithi164.stdout: "type_data": { 2023-06-03T20:12:15.752 INFO:teuthology.orchestra.run.smithi164.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-03T20:12:15.752 INFO:teuthology.orchestra.run.smithi164.stdout: "client_info": { 2023-06-03T20:12:15.752 INFO:teuthology.orchestra.run.smithi164.stdout: "client": "client.45713", 2023-06-03T20:12:15.752 INFO:teuthology.orchestra.run.smithi164.stdout: "client_addr": "172.21.15.89:0/2400821448", 2023-06-03T20:12:15.752 INFO:teuthology.orchestra.run.smithi164.stdout: "tid": 145282 2023-06-03T20:12:15.752 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:12:15.752 INFO:teuthology.orchestra.run.smithi164.stdout: "events": [ 2023-06-03T20:12:15.752 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:12:15.753 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "initiated", 2023-06-03T20:12:15.753 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:02:57.745785+0000", 2023-06-03T20:12:15.753 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0 2023-06-03T20:12:15.753 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:12:15.753 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:12:15.753 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "throttled", 2023-06-03T20:12:15.753 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:02:57.745785+0000", 2023-06-03T20:12:15.753 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 4294967295.9999981 2023-06-03T20:12:15.754 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:12:15.754 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:12:15.754 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "header_read", 2023-06-03T20:12:15.754 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:02:57.745783+0000", 2023-06-03T20:12:15.754 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 1.0104e-05 2023-06-03T20:12:15.754 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:12:15.754 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:12:15.754 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "all_read", 2023-06-03T20:12:15.755 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:02:57.745794+0000", 2023-06-03T20:12:15.755 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 2.9686999999999999e-05 2023-06-03T20:12:15.755 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:12:15.755 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:12:15.755 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "dispatched", 2023-06-03T20:12:15.755 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:02:57.745823+0000", 2023-06-03T20:12:15.755 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 4.3881000000000002e-05 2023-06-03T20:12:15.755 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:12:15.756 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:12:15.756 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "queued_for_pg", 2023-06-03T20:12:15.756 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:02:57.745867+0000", 2023-06-03T20:12:15.756 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 7.3192e-05 2023-06-03T20:12:15.756 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:12:15.756 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:12:15.756 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "reached_pg", 2023-06-03T20:12:15.757 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:02:57.745940+0000", 2023-06-03T20:12:15.757 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.000217044 2023-06-03T20:12:15.757 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:12:15.757 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:12:15.757 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "started", 2023-06-03T20:12:15.757 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:02:57.746157+0000", 2023-06-03T20:12:15.757 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.000143931 2023-06-03T20:12:15.757 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:12:15.758 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:12:15.758 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "waiting for subops from 1,2", 2023-06-03T20:12:15.758 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:02:57.746301+0000", 2023-06-03T20:12:15.758 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.0051458989999999998 2023-06-03T20:12:15.758 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:12:15.758 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:12:15.758 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:12:15.758 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:02:57.751447+0000", 2023-06-03T20:12:15.759 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.00021424200000000001 2023-06-03T20:12:15.759 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:12:15.759 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:12:15.759 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "op_commit", 2023-06-03T20:12:15.759 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:02:57.751661+0000", 2023-06-03T20:12:15.759 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.20416706200000001 2023-06-03T20:12:15.759 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:12:15.759 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:12:15.760 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:12:15.760 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:02:57.955828+0000", 2023-06-03T20:12:15.760 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 3.3130000000000003e-05 2023-06-03T20:12:15.760 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:12:15.760 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:12:15.760 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "commit_sent", 2023-06-03T20:12:15.760 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:02:57.955862+0000", 2023-06-03T20:12:15.761 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 3.8513000000000002e-05 2023-06-03T20:12:15.761 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:12:15.761 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:12:15.761 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "done", 2023-06-03T20:12:15.761 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:02:57.955900+0000", 2023-06-03T20:12:15.761 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.210114734 2023-06-03T20:12:15.761 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:12:15.761 INFO:teuthology.orchestra.run.smithi164.stdout: ] 2023-06-03T20:12:15.762 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:12:15.762 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:12:15.762 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:12:15.762 INFO:teuthology.orchestra.run.smithi164.stdout: "description": "osd_op(client.45713.0:146207 9.f 9:f0b9f141:::benchmark_data_smithi089_115580_object36073:head [delete] snapc 0=[] ondisk+write+known_if_redirected e139)", 2023-06-03T20:12:15.762 INFO:teuthology.orchestra.run.smithi164.stdout: "initiated_at": "2023-06-03T20:02:59.158798+0000", 2023-06-03T20:12:15.762 INFO:teuthology.orchestra.run.smithi164.stdout: "age": 556.569782885, 2023-06-03T20:12:15.762 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.18282954800000001, 2023-06-03T20:12:15.762 INFO:teuthology.orchestra.run.smithi164.stdout: "type_data": { 2023-06-03T20:12:15.763 INFO:teuthology.orchestra.run.smithi164.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-03T20:12:15.763 INFO:teuthology.orchestra.run.smithi164.stdout: "client_info": { 2023-06-03T20:12:15.763 INFO:teuthology.orchestra.run.smithi164.stdout: "client": "client.45713", 2023-06-03T20:12:15.763 INFO:teuthology.orchestra.run.smithi164.stdout: "client_addr": "172.21.15.89:0/2400821448", 2023-06-03T20:12:15.763 INFO:teuthology.orchestra.run.smithi164.stdout: "tid": 146207 2023-06-03T20:12:15.763 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:12:15.763 INFO:teuthology.orchestra.run.smithi164.stdout: "events": [ 2023-06-03T20:12:15.763 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:12:15.764 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "initiated", 2023-06-03T20:12:15.764 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:02:59.158798+0000", 2023-06-03T20:12:15.764 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0 2023-06-03T20:12:15.764 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:12:15.764 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:12:15.764 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "throttled", 2023-06-03T20:12:15.764 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:02:59.158798+0000", 2023-06-03T20:12:15.765 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 4294967295.9999986 2023-06-03T20:12:15.765 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:12:15.765 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:12:15.765 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "header_read", 2023-06-03T20:12:15.765 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:02:59.158797+0000", 2023-06-03T20:12:15.765 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 6.5239999999999997e-06 2023-06-03T20:12:15.765 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:12:15.765 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:12:15.766 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "all_read", 2023-06-03T20:12:15.766 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:02:59.158804+0000", 2023-06-03T20:12:15.766 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.12569430400000001 2023-06-03T20:12:15.766 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:12:15.766 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:12:15.766 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "dispatched", 2023-06-03T20:12:15.766 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:02:59.284498+0000", 2023-06-03T20:12:15.766 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 2.0556000000000001e-05 2023-06-03T20:12:15.767 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:12:15.767 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:12:15.767 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "queued_for_pg", 2023-06-03T20:12:15.767 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:02:59.284518+0000", 2023-06-03T20:12:15.767 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 9.2949999999999996e-05 2023-06-03T20:12:15.767 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:12:15.767 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:12:15.767 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "reached_pg", 2023-06-03T20:12:15.768 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:02:59.284611+0000", 2023-06-03T20:12:15.768 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.00033299000000000002 2023-06-03T20:12:15.768 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:12:15.768 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:12:15.768 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "started", 2023-06-03T20:12:15.768 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:02:59.284944+0000", 2023-06-03T20:12:15.768 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.00034682799999999998 2023-06-03T20:12:15.768 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:12:15.769 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:12:15.769 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "waiting for subops from 1,3", 2023-06-03T20:12:15.769 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:02:59.285291+0000", 2023-06-03T20:12:15.769 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.020359854 2023-06-03T20:12:15.769 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:12:15.769 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:12:15.769 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:12:15.770 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:02:59.305651+0000", 2023-06-03T20:12:15.770 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 3.6431999999999999e-05 2023-06-03T20:12:15.770 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:12:15.770 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:12:15.770 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:12:15.770 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:02:59.305687+0000", 2023-06-03T20:12:15.770 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.034524036000000001 2023-06-03T20:12:15.770 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:12:15.771 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:12:15.771 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "op_commit", 2023-06-03T20:12:15.771 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:02:59.340212+0000", 2023-06-03T20:12:15.771 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.00019661599999999999 2023-06-03T20:12:15.771 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:12:15.771 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:12:15.771 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "commit_sent", 2023-06-03T20:12:15.771 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:02:59.340408+0000", 2023-06-03T20:12:15.772 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.001219765 2023-06-03T20:12:15.772 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:12:15.772 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:12:15.772 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "done", 2023-06-03T20:12:15.772 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:02:59.341628+0000", 2023-06-03T20:12:15.772 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.18282954800000001 2023-06-03T20:12:15.772 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:12:15.772 INFO:teuthology.orchestra.run.smithi164.stdout: ] 2023-06-03T20:12:15.773 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:12:15.773 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:12:15.773 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:12:15.773 INFO:teuthology.orchestra.run.smithi164.stdout: "description": "osd_op(client.45713.0:146673 9.f 9:fdfbee0d:::benchmark_data_smithi089_115580_object36539:head [delete] snapc 0=[] ondisk+write+known_if_redirected e139)", 2023-06-03T20:12:15.773 INFO:teuthology.orchestra.run.smithi164.stdout: "initiated_at": "2023-06-03T20:02:59.953758+0000", 2023-06-03T20:12:15.773 INFO:teuthology.orchestra.run.smithi164.stdout: "age": 555.77482276499995, 2023-06-03T20:12:15.773 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.18546722299999999, 2023-06-03T20:12:15.773 INFO:teuthology.orchestra.run.smithi164.stdout: "type_data": { 2023-06-03T20:12:15.774 INFO:teuthology.orchestra.run.smithi164.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-03T20:12:15.774 INFO:teuthology.orchestra.run.smithi164.stdout: "client_info": { 2023-06-03T20:12:15.774 INFO:teuthology.orchestra.run.smithi164.stdout: "client": "client.45713", 2023-06-03T20:12:15.774 INFO:teuthology.orchestra.run.smithi164.stdout: "client_addr": "172.21.15.89:0/2400821448", 2023-06-03T20:12:15.774 INFO:teuthology.orchestra.run.smithi164.stdout: "tid": 146673 2023-06-03T20:12:15.774 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:12:15.774 INFO:teuthology.orchestra.run.smithi164.stdout: "events": [ 2023-06-03T20:12:15.774 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:12:15.775 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "initiated", 2023-06-03T20:12:15.775 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:02:59.953758+0000", 2023-06-03T20:12:15.775 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0 2023-06-03T20:12:15.775 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:12:15.775 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:12:15.776 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "throttled", 2023-06-03T20:12:15.776 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:02:59.953758+0000", 2023-06-03T20:12:15.776 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 4294967295.9999986 2023-06-03T20:12:15.776 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:12:15.776 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:12:15.776 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "header_read", 2023-06-03T20:12:15.776 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:02:59.953757+0000", 2023-06-03T20:12:15.776 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 8.6940000000000001e-06 2023-06-03T20:12:15.777 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:12:15.777 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:12:15.777 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "all_read", 2023-06-03T20:12:15.777 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:02:59.953766+0000", 2023-06-03T20:12:15.777 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 1.3572e-05 2023-06-03T20:12:15.777 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:12:15.777 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:12:15.777 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "dispatched", 2023-06-03T20:12:15.778 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:02:59.953779+0000", 2023-06-03T20:12:15.778 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 8.1289999999999996e-06 2023-06-03T20:12:15.778 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:12:15.778 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:12:15.778 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "queued_for_pg", 2023-06-03T20:12:15.778 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:02:59.953787+0000", 2023-06-03T20:12:15.778 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.0094506539999999993 2023-06-03T20:12:15.778 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:12:15.779 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:12:15.779 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "reached_pg", 2023-06-03T20:12:15.779 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:02:59.963238+0000", 2023-06-03T20:12:15.779 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.00012383899999999999 2023-06-03T20:12:15.779 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:12:15.779 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:12:15.779 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "started", 2023-06-03T20:12:15.779 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:02:59.963362+0000", 2023-06-03T20:12:15.780 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.000130104 2023-06-03T20:12:15.780 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:12:15.780 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:12:15.780 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "waiting for subops from 1,3", 2023-06-03T20:12:15.780 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:02:59.963492+0000", 2023-06-03T20:12:15.780 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.010105143 2023-06-03T20:12:15.780 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:12:15.780 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:12:15.781 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:12:15.781 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:02:59.973597+0000", 2023-06-03T20:12:15.781 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 4.3477999999999999e-05 2023-06-03T20:12:15.781 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:12:15.781 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:12:15.781 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "op_commit", 2023-06-03T20:12:15.781 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:02:59.973640+0000", 2023-06-03T20:12:15.781 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.16549918999999999 2023-06-03T20:12:15.782 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:12:15.782 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:12:15.782 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:12:15.782 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:03:00.139140+0000", 2023-06-03T20:12:15.782 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 4.4567000000000002e-05 2023-06-03T20:12:15.782 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:12:15.782 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:12:15.782 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "commit_sent", 2023-06-03T20:12:15.783 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:03:00.139184+0000", 2023-06-03T20:12:15.783 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 4.1511999999999998e-05 2023-06-03T20:12:15.783 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:12:15.783 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:12:15.783 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "done", 2023-06-03T20:12:15.783 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:03:00.139226+0000", 2023-06-03T20:12:15.783 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.18546722299999999 2023-06-03T20:12:15.784 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:12:15.784 INFO:teuthology.orchestra.run.smithi164.stdout: ] 2023-06-03T20:12:15.784 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:12:15.784 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:12:15.784 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:12:15.784 INFO:teuthology.orchestra.run.smithi164.stdout: "description": "osd_op(client.45713.0:146676 9.7 9:ee40c199:::benchmark_data_smithi089_115580_object36542:head [delete] snapc 0=[] ondisk+write+known_if_redirected e139)", 2023-06-03T20:12:15.784 INFO:teuthology.orchestra.run.smithi164.stdout: "initiated_at": "2023-06-03T20:02:59.957858+0000", 2023-06-03T20:12:15.784 INFO:teuthology.orchestra.run.smithi164.stdout: "age": 555.77072366799996, 2023-06-03T20:12:15.785 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.181236649, 2023-06-03T20:12:15.785 INFO:teuthology.orchestra.run.smithi164.stdout: "type_data": { 2023-06-03T20:12:15.785 INFO:teuthology.orchestra.run.smithi164.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-03T20:12:15.785 INFO:teuthology.orchestra.run.smithi164.stdout: "client_info": { 2023-06-03T20:12:15.785 INFO:teuthology.orchestra.run.smithi164.stdout: "client": "client.45713", 2023-06-03T20:12:15.785 INFO:teuthology.orchestra.run.smithi164.stdout: "client_addr": "172.21.15.89:0/2400821448", 2023-06-03T20:12:15.785 INFO:teuthology.orchestra.run.smithi164.stdout: "tid": 146676 2023-06-03T20:12:15.785 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:12:15.786 INFO:teuthology.orchestra.run.smithi164.stdout: "events": [ 2023-06-03T20:12:15.786 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:12:15.786 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "initiated", 2023-06-03T20:12:15.786 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:02:59.957858+0000", 2023-06-03T20:12:15.786 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0 2023-06-03T20:12:15.786 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:12:15.786 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:12:15.786 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "throttled", 2023-06-03T20:12:15.787 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:02:59.957858+0000", 2023-06-03T20:12:15.787 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 4294967295.9999943 2023-06-03T20:12:15.787 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:12:15.787 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:12:15.787 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "header_read", 2023-06-03T20:12:15.787 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:02:59.957852+0000", 2023-06-03T20:12:15.787 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 1.5852e-05 2023-06-03T20:12:15.787 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:12:15.788 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:12:15.788 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "all_read", 2023-06-03T20:12:15.788 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:02:59.957868+0000", 2023-06-03T20:12:15.788 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 2.1109999999999999e-05 2023-06-03T20:12:15.788 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:12:15.788 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:12:15.788 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "dispatched", 2023-06-03T20:12:15.789 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:02:59.957889+0000", 2023-06-03T20:12:15.789 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 9.6010000000000001e-06 2023-06-03T20:12:15.789 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:12:15.789 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:12:15.789 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "queued_for_pg", 2023-06-03T20:12:15.789 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:02:59.957898+0000", 2023-06-03T20:12:15.789 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.010351582999999999 2023-06-03T20:12:15.789 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:12:15.790 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:12:15.790 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "reached_pg", 2023-06-03T20:12:15.790 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:02:59.968250+0000", 2023-06-03T20:12:15.790 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 7.9942000000000001e-05 2023-06-03T20:12:15.790 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:12:15.790 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:12:15.790 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "started", 2023-06-03T20:12:15.790 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:02:59.968330+0000", 2023-06-03T20:12:15.791 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 9.0809000000000004e-05 2023-06-03T20:12:15.791 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:12:15.791 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:12:15.791 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "waiting for subops from 3,7", 2023-06-03T20:12:15.791 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:02:59.968421+0000", 2023-06-03T20:12:15.791 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.0052237940000000004 2023-06-03T20:12:15.791 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:12:15.791 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:12:15.792 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "op_commit", 2023-06-03T20:12:15.792 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:02:59.973644+0000", 2023-06-03T20:12:15.792 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.0016236009999999999 2023-06-03T20:12:15.792 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:12:15.792 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:12:15.792 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:12:15.792 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:02:59.975268+0000", 2023-06-03T20:12:15.792 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.163722327 2023-06-03T20:12:15.793 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:12:15.793 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:12:15.793 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:12:15.793 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:03:00.138990+0000", 2023-06-03T20:12:15.793 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 5.3659999999999998e-05 2023-06-03T20:12:15.793 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:12:15.793 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:12:15.793 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "commit_sent", 2023-06-03T20:12:15.794 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:03:00.139044+0000", 2023-06-03T20:12:15.794 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 5.0173999999999998e-05 2023-06-03T20:12:15.794 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:12:15.794 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:12:15.794 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "done", 2023-06-03T20:12:15.794 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:03:00.139094+0000", 2023-06-03T20:12:15.794 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.181236649 2023-06-03T20:12:15.794 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:12:15.795 INFO:teuthology.orchestra.run.smithi164.stdout: ] 2023-06-03T20:12:15.795 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:12:15.795 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:12:15.795 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:12:15.795 INFO:teuthology.orchestra.run.smithi164.stdout: "description": "osd_op(client.45713.0:181560 9.f 9:f2718c3c:::benchmark_data_smithi089_115580_object71426:head [delete] snapc 0=[] ondisk+write+known_if_redirected e142)", 2023-06-03T20:12:15.795 INFO:teuthology.orchestra.run.smithi164.stdout: "initiated_at": "2023-06-03T20:03:49.142637+0000", 2023-06-03T20:12:15.795 INFO:teuthology.orchestra.run.smithi164.stdout: "age": 506.58594434299999, 2023-06-03T20:12:15.795 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.17728545100000001, 2023-06-03T20:12:15.796 INFO:teuthology.orchestra.run.smithi164.stdout: "type_data": { 2023-06-03T20:12:15.796 INFO:teuthology.orchestra.run.smithi164.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-03T20:12:15.796 INFO:teuthology.orchestra.run.smithi164.stdout: "client_info": { 2023-06-03T20:12:15.796 INFO:teuthology.orchestra.run.smithi164.stdout: "client": "client.45713", 2023-06-03T20:12:15.796 INFO:teuthology.orchestra.run.smithi164.stdout: "client_addr": "172.21.15.89:0/2400821448", 2023-06-03T20:12:15.796 INFO:teuthology.orchestra.run.smithi164.stdout: "tid": 181560 2023-06-03T20:12:15.796 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:12:15.797 INFO:teuthology.orchestra.run.smithi164.stdout: "events": [ 2023-06-03T20:12:15.797 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:12:15.797 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "initiated", 2023-06-03T20:12:15.797 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:03:49.142637+0000", 2023-06-03T20:12:15.797 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0 2023-06-03T20:12:15.797 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:12:15.797 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:12:15.797 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "throttled", 2023-06-03T20:12:15.798 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:03:49.142637+0000", 2023-06-03T20:12:15.798 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 4294967295.9999976 2023-06-03T20:12:15.798 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:12:15.798 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:12:15.798 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "header_read", 2023-06-03T20:12:15.798 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:03:49.142635+0000", 2023-06-03T20:12:15.798 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 7.8860000000000004e-06 2023-06-03T20:12:15.798 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:12:15.799 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:12:15.799 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "all_read", 2023-06-03T20:12:15.799 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:03:49.142643+0000", 2023-06-03T20:12:15.799 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 1.7051999999999999e-05 2023-06-03T20:12:15.799 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:12:15.799 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:12:15.799 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "dispatched", 2023-06-03T20:12:15.799 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:03:49.142660+0000", 2023-06-03T20:12:15.800 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 6.7499999999999997e-06 2023-06-03T20:12:15.800 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:12:15.800 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:12:15.800 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "queued_for_pg", 2023-06-03T20:12:15.800 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:03:49.142666+0000", 2023-06-03T20:12:15.800 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.028141283 2023-06-03T20:12:15.800 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:12:15.800 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:12:15.801 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "reached_pg", 2023-06-03T20:12:15.801 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:03:49.170808+0000", 2023-06-03T20:12:15.801 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.00013038599999999999 2023-06-03T20:12:15.801 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:12:15.801 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:12:15.801 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "started", 2023-06-03T20:12:15.801 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:03:49.170938+0000", 2023-06-03T20:12:15.801 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.000155038 2023-06-03T20:12:15.802 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:12:15.802 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:12:15.802 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "waiting for subops from 1,3", 2023-06-03T20:12:15.802 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:03:49.171093+0000", 2023-06-03T20:12:15.802 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.010356103 2023-06-03T20:12:15.802 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:12:15.802 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:12:15.802 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "op_commit", 2023-06-03T20:12:15.803 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:03:49.181449+0000", 2023-06-03T20:12:15.803 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.00092883199999999999 2023-06-03T20:12:15.803 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:12:15.803 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:12:15.803 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:12:15.803 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:03:49.182378+0000", 2023-06-03T20:12:15.803 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.13740876699999999 2023-06-03T20:12:15.803 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:12:15.804 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:12:15.804 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:12:15.804 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:03:49.319787+0000", 2023-06-03T20:12:15.804 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 5.0352999999999997e-05 2023-06-03T20:12:15.804 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:12:15.804 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:12:15.804 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "commit_sent", 2023-06-03T20:12:15.804 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:03:49.319837+0000", 2023-06-03T20:12:15.805 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 8.5277000000000004e-05 2023-06-03T20:12:15.805 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:12:15.805 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:12:15.805 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "done", 2023-06-03T20:12:15.805 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:03:49.319922+0000", 2023-06-03T20:12:15.805 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.17728545100000001 2023-06-03T20:12:15.805 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:12:15.805 INFO:teuthology.orchestra.run.smithi164.stdout: ] 2023-06-03T20:12:15.806 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:12:15.806 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:12:15.806 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:12:15.806 INFO:teuthology.orchestra.run.smithi164.stdout: "description": "osd_op(client.45713.0:181561 9.7 9:e86e2fa5:::benchmark_data_smithi089_115580_object71427:head [delete] snapc 0=[] ondisk+write+known_if_redirected e142)", 2023-06-03T20:12:15.806 INFO:teuthology.orchestra.run.smithi164.stdout: "initiated_at": "2023-06-03T20:03:49.143583+0000", 2023-06-03T20:12:15.807 INFO:teuthology.orchestra.run.smithi164.stdout: "age": 506.58499834899999, 2023-06-03T20:12:15.807 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.176389351, 2023-06-03T20:12:15.807 INFO:teuthology.orchestra.run.smithi164.stdout: "type_data": { 2023-06-03T20:12:15.807 INFO:teuthology.orchestra.run.smithi164.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-03T20:12:15.807 INFO:teuthology.orchestra.run.smithi164.stdout: "client_info": { 2023-06-03T20:12:15.807 INFO:teuthology.orchestra.run.smithi164.stdout: "client": "client.45713", 2023-06-03T20:12:15.808 INFO:teuthology.orchestra.run.smithi164.stdout: "client_addr": "172.21.15.89:0/2400821448", 2023-06-03T20:12:15.808 INFO:teuthology.orchestra.run.smithi164.stdout: "tid": 181561 2023-06-03T20:12:15.808 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:12:15.808 INFO:teuthology.orchestra.run.smithi164.stdout: "events": [ 2023-06-03T20:12:15.808 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:12:15.808 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "initiated", 2023-06-03T20:12:15.808 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:03:49.143583+0000", 2023-06-03T20:12:15.808 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0 2023-06-03T20:12:15.809 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:12:15.809 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:12:15.809 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "throttled", 2023-06-03T20:12:15.809 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:03:49.143583+0000", 2023-06-03T20:12:15.809 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 4294967295.999999 2023-06-03T20:12:15.809 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:12:15.809 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:12:15.809 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "header_read", 2023-06-03T20:12:15.810 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:03:49.143582+0000", 2023-06-03T20:12:15.810 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 5.2440000000000001e-06 2023-06-03T20:12:15.810 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:12:15.810 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:12:15.810 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "all_read", 2023-06-03T20:12:15.810 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:03:49.143587+0000", 2023-06-03T20:12:15.810 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 1.2071e-05 2023-06-03T20:12:15.811 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:12:15.811 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:12:15.811 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "dispatched", 2023-06-03T20:12:15.811 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:03:49.143599+0000", 2023-06-03T20:12:15.811 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 8.2090000000000005e-06 2023-06-03T20:12:15.811 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:12:15.811 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:12:15.811 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "queued_for_pg", 2023-06-03T20:12:15.812 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:03:49.143607+0000", 2023-06-03T20:12:15.812 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.037988091000000002 2023-06-03T20:12:15.812 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:12:15.812 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:12:15.812 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "reached_pg", 2023-06-03T20:12:15.812 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:03:49.181596+0000", 2023-06-03T20:12:15.812 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.000142857 2023-06-03T20:12:15.812 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:12:15.813 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:12:15.813 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "started", 2023-06-03T20:12:15.813 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:03:49.181738+0000", 2023-06-03T20:12:15.813 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.000164086 2023-06-03T20:12:15.813 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:12:15.813 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:12:15.813 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "waiting for subops from 3,7", 2023-06-03T20:12:15.813 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:03:49.181902+0000", 2023-06-03T20:12:15.814 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.011042893 2023-06-03T20:12:15.814 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:12:15.814 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:12:15.814 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:12:15.814 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:03:49.192945+0000", 2023-06-03T20:12:15.814 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.0061257509999999996 2023-06-03T20:12:15.814 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:12:15.814 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:12:15.815 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "op_commit", 2023-06-03T20:12:15.815 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:03:49.199071+0000", 2023-06-03T20:12:15.815 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.120692695 2023-06-03T20:12:15.815 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:12:15.815 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:12:15.815 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:12:15.815 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:03:49.319764+0000", 2023-06-03T20:12:15.816 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.000162607 2023-06-03T20:12:15.816 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:12:15.816 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:12:15.816 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "commit_sent", 2023-06-03T20:12:15.816 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:03:49.319926+0000", 2023-06-03T20:12:15.816 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 4.5803000000000002e-05 2023-06-03T20:12:15.816 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:12:15.816 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:12:15.817 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "done", 2023-06-03T20:12:15.817 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:03:49.319972+0000", 2023-06-03T20:12:15.817 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.176389351 2023-06-03T20:12:15.817 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:12:15.817 INFO:teuthology.orchestra.run.smithi164.stdout: ] 2023-06-03T20:12:15.817 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:12:15.817 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:12:15.817 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:12:15.818 INFO:teuthology.orchestra.run.smithi164.stdout: "description": "osd_op(client.45713.0:181562 9.7 9:e8ebcc70:::benchmark_data_smithi089_115580_object71428:head [delete] snapc 0=[] ondisk+write+known_if_redirected e142)", 2023-06-03T20:12:15.818 INFO:teuthology.orchestra.run.smithi164.stdout: "initiated_at": "2023-06-03T20:03:49.144007+0000", 2023-06-03T20:12:15.818 INFO:teuthology.orchestra.run.smithi164.stdout: "age": 506.58457411699999, 2023-06-03T20:12:15.818 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.20303618000000001, 2023-06-03T20:12:15.818 INFO:teuthology.orchestra.run.smithi164.stdout: "type_data": { 2023-06-03T20:12:15.818 INFO:teuthology.orchestra.run.smithi164.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-03T20:12:15.818 INFO:teuthology.orchestra.run.smithi164.stdout: "client_info": { 2023-06-03T20:12:15.818 INFO:teuthology.orchestra.run.smithi164.stdout: "client": "client.45713", 2023-06-03T20:12:15.819 INFO:teuthology.orchestra.run.smithi164.stdout: "client_addr": "172.21.15.89:0/2400821448", 2023-06-03T20:12:15.819 INFO:teuthology.orchestra.run.smithi164.stdout: "tid": 181562 2023-06-03T20:12:15.819 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:12:15.819 INFO:teuthology.orchestra.run.smithi164.stdout: "events": [ 2023-06-03T20:12:15.819 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:12:15.819 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "initiated", 2023-06-03T20:12:15.819 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:03:49.144007+0000", 2023-06-03T20:12:15.819 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0 2023-06-03T20:12:15.820 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:12:15.820 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:12:15.820 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "throttled", 2023-06-03T20:12:15.820 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:03:49.144007+0000", 2023-06-03T20:12:15.820 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 4294967295.999999 2023-06-03T20:12:15.820 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:12:15.820 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:12:15.820 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "header_read", 2023-06-03T20:12:15.821 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:03:49.144006+0000", 2023-06-03T20:12:15.821 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 5.152e-06 2023-06-03T20:12:15.821 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:12:15.821 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:12:15.821 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "all_read", 2023-06-03T20:12:15.821 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:03:49.144011+0000", 2023-06-03T20:12:15.821 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 1.0902e-05 2023-06-03T20:12:15.821 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:12:15.822 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:12:15.822 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "dispatched", 2023-06-03T20:12:15.822 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:03:49.144022+0000", 2023-06-03T20:12:15.822 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 5.0200000000000002e-06 2023-06-03T20:12:15.822 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:12:15.822 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:12:15.822 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "queued_for_pg", 2023-06-03T20:12:15.822 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:03:49.144027+0000", 2023-06-03T20:12:15.823 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.043237734999999999 2023-06-03T20:12:15.823 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:12:15.823 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:12:15.823 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "reached_pg", 2023-06-03T20:12:15.823 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:03:49.187265+0000", 2023-06-03T20:12:15.823 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 8.9528000000000001e-05 2023-06-03T20:12:15.823 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:12:15.823 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:12:15.824 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "started", 2023-06-03T20:12:15.824 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:03:49.187355+0000", 2023-06-03T20:12:15.824 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 9.2496000000000003e-05 2023-06-03T20:12:15.824 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:12:15.824 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:12:15.824 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "waiting for subops from 3,7", 2023-06-03T20:12:15.824 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:03:49.187447+0000", 2023-06-03T20:12:15.824 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.011630410000000001 2023-06-03T20:12:15.825 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:12:15.825 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:12:15.825 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "op_commit", 2023-06-03T20:12:15.825 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:03:49.199077+0000", 2023-06-03T20:12:15.825 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 2.9845e-05 2023-06-03T20:12:15.825 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:12:15.825 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:12:15.825 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:12:15.826 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:03:49.199107+0000", 2023-06-03T20:12:15.826 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.14782999699999999 2023-06-03T20:12:15.826 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:12:15.826 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:12:15.826 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:12:15.826 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:03:49.346937+0000", 2023-06-03T20:12:15.826 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 6.0843e-05 2023-06-03T20:12:15.826 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:12:15.827 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:12:15.827 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "commit_sent", 2023-06-03T20:12:15.827 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:03:49.346998+0000", 2023-06-03T20:12:15.827 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 4.5147e-05 2023-06-03T20:12:15.827 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:12:15.827 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:12:15.827 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "done", 2023-06-03T20:12:15.828 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:03:49.347043+0000", 2023-06-03T20:12:15.828 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.20303618000000001 2023-06-03T20:12:15.828 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:12:15.828 INFO:teuthology.orchestra.run.smithi164.stdout: ] 2023-06-03T20:12:15.828 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:12:15.828 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:12:15.828 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:12:15.828 INFO:teuthology.orchestra.run.smithi164.stdout: "description": "osd_op(client.45713.0:181566 9.f 9:fe4085bb:::benchmark_data_smithi089_115580_object71432:head [delete] snapc 0=[] ondisk+write+known_if_redirected e142)", 2023-06-03T20:12:15.829 INFO:teuthology.orchestra.run.smithi164.stdout: "initiated_at": "2023-06-03T20:03:49.148124+0000", 2023-06-03T20:12:15.829 INFO:teuthology.orchestra.run.smithi164.stdout: "age": 506.58045686100002, 2023-06-03T20:12:15.829 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.19016216899999999, 2023-06-03T20:12:15.829 INFO:teuthology.orchestra.run.smithi164.stdout: "type_data": { 2023-06-03T20:12:15.829 INFO:teuthology.orchestra.run.smithi164.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-03T20:12:15.829 INFO:teuthology.orchestra.run.smithi164.stdout: "client_info": { 2023-06-03T20:12:15.829 INFO:teuthology.orchestra.run.smithi164.stdout: "client": "client.45713", 2023-06-03T20:12:15.829 INFO:teuthology.orchestra.run.smithi164.stdout: "client_addr": "172.21.15.89:0/2400821448", 2023-06-03T20:12:15.830 INFO:teuthology.orchestra.run.smithi164.stdout: "tid": 181566 2023-06-03T20:12:15.830 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:12:15.830 INFO:teuthology.orchestra.run.smithi164.stdout: "events": [ 2023-06-03T20:12:15.830 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:12:15.830 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "initiated", 2023-06-03T20:12:15.830 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:03:49.148124+0000", 2023-06-03T20:12:15.830 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0 2023-06-03T20:12:15.830 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:12:15.831 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:12:15.831 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "throttled", 2023-06-03T20:12:15.831 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:03:49.148124+0000", 2023-06-03T20:12:15.831 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 4294967295.999999 2023-06-03T20:12:15.831 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:12:15.831 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:12:15.831 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "header_read", 2023-06-03T20:12:15.831 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:03:49.148123+0000", 2023-06-03T20:12:15.832 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 5.49e-06 2023-06-03T20:12:15.832 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:12:15.832 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:12:15.832 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "all_read", 2023-06-03T20:12:15.832 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:03:49.148129+0000", 2023-06-03T20:12:15.832 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 1.0582e-05 2023-06-03T20:12:15.832 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:12:15.833 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:12:15.833 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "dispatched", 2023-06-03T20:12:15.833 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:03:49.148139+0000", 2023-06-03T20:12:15.833 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 4.9540000000000003e-06 2023-06-03T20:12:15.833 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:12:15.833 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:12:15.833 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "queued_for_pg", 2023-06-03T20:12:15.833 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:03:49.148144+0000", 2023-06-03T20:12:15.834 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.039371446999999997 2023-06-03T20:12:15.834 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:12:15.834 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:12:15.834 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "reached_pg", 2023-06-03T20:12:15.834 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:03:49.187516+0000", 2023-06-03T20:12:15.834 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.00010843100000000001 2023-06-03T20:12:15.834 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:12:15.834 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:12:15.835 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "started", 2023-06-03T20:12:15.835 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:03:49.187624+0000", 2023-06-03T20:12:15.835 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.00012288199999999999 2023-06-03T20:12:15.835 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:12:15.835 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:12:15.835 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "waiting for subops from 1,3", 2023-06-03T20:12:15.835 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:03:49.187747+0000", 2023-06-03T20:12:15.835 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.01135738 2023-06-03T20:12:15.836 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:12:15.836 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:12:15.836 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:12:15.836 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:03:49.199105+0000", 2023-06-03T20:12:15.836 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 8.7879999999999995e-06 2023-06-03T20:12:15.836 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:12:15.836 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:12:15.836 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "op_commit", 2023-06-03T20:12:15.837 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:03:49.199113+0000", 2023-06-03T20:12:15.837 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.13906349100000001 2023-06-03T20:12:15.837 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:12:15.837 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:12:15.837 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:12:15.837 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:03:49.338177+0000", 2023-06-03T20:12:15.837 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 5.9175999999999997e-05 2023-06-03T20:12:15.837 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:12:15.838 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:12:15.838 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "commit_sent", 2023-06-03T20:12:15.838 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:03:49.338236+0000", 2023-06-03T20:12:15.838 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 5.0590000000000002e-05 2023-06-03T20:12:15.838 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:12:15.838 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:12:15.838 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "done", 2023-06-03T20:12:15.838 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:03:49.338287+0000", 2023-06-03T20:12:15.839 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.19016216899999999 2023-06-03T20:12:15.839 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:12:15.839 INFO:teuthology.orchestra.run.smithi164.stdout: ] 2023-06-03T20:12:15.839 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:12:15.839 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:12:15.839 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:12:15.839 INFO:teuthology.orchestra.run.smithi164.stdout: "description": "osd_op(client.45713.0:181586 9.f 9:fec709f3:::benchmark_data_smithi089_115580_object71452:head [delete] snapc 0=[] ondisk+write+known_if_redirected e142)", 2023-06-03T20:12:15.840 INFO:teuthology.orchestra.run.smithi164.stdout: "initiated_at": "2023-06-03T20:03:49.181706+0000", 2023-06-03T20:12:15.840 INFO:teuthology.orchestra.run.smithi164.stdout: "age": 506.54687492599999, 2023-06-03T20:12:15.840 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.18809337000000001, 2023-06-03T20:12:15.840 INFO:teuthology.orchestra.run.smithi164.stdout: "type_data": { 2023-06-03T20:12:15.840 INFO:teuthology.orchestra.run.smithi164.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-03T20:12:15.840 INFO:teuthology.orchestra.run.smithi164.stdout: "client_info": { 2023-06-03T20:12:15.840 INFO:teuthology.orchestra.run.smithi164.stdout: "client": "client.45713", 2023-06-03T20:12:15.840 INFO:teuthology.orchestra.run.smithi164.stdout: "client_addr": "172.21.15.89:0/2400821448", 2023-06-03T20:12:15.841 INFO:teuthology.orchestra.run.smithi164.stdout: "tid": 181586 2023-06-03T20:12:15.841 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:12:15.841 INFO:teuthology.orchestra.run.smithi164.stdout: "events": [ 2023-06-03T20:12:15.841 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:12:15.841 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "initiated", 2023-06-03T20:12:15.841 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:03:49.181706+0000", 2023-06-03T20:12:15.841 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0 2023-06-03T20:12:15.841 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:12:15.842 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:12:15.842 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "throttled", 2023-06-03T20:12:15.842 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:03:49.181706+0000", 2023-06-03T20:12:15.842 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 4294967295.999999 2023-06-03T20:12:15.842 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:12:15.842 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:12:15.842 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "header_read", 2023-06-03T20:12:15.842 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:03:49.181705+0000", 2023-06-03T20:12:15.843 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 5.0409999999999997e-06 2023-06-03T20:12:15.843 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:12:15.843 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:12:15.843 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "all_read", 2023-06-03T20:12:15.843 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:03:49.181710+0000", 2023-06-03T20:12:15.843 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 9.0629999999999995e-06 2023-06-03T20:12:15.843 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:12:15.843 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:12:15.844 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "dispatched", 2023-06-03T20:12:15.844 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:03:49.181719+0000", 2023-06-03T20:12:15.844 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 1.5795999999999999e-05 2023-06-03T20:12:15.844 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:12:15.844 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:12:15.844 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "queued_for_pg", 2023-06-03T20:12:15.844 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:03:49.181735+0000", 2023-06-03T20:12:15.845 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.011428341999999999 2023-06-03T20:12:15.845 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:12:15.845 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:12:15.845 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "reached_pg", 2023-06-03T20:12:15.845 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:03:49.193164+0000", 2023-06-03T20:12:15.845 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.00013727699999999999 2023-06-03T20:12:15.845 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:12:15.845 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:12:15.846 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "started", 2023-06-03T20:12:15.846 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:03:49.193301+0000", 2023-06-03T20:12:15.846 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.00015319300000000001 2023-06-03T20:12:15.846 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:12:15.846 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:12:15.846 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "waiting for subops from 1,3", 2023-06-03T20:12:15.846 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:03:49.193454+0000", 2023-06-03T20:12:15.846 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.0056874530000000003 2023-06-03T20:12:15.847 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:12:15.847 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:12:15.847 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:12:15.847 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:03:49.199142+0000", 2023-06-03T20:12:15.847 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.0053824650000000003 2023-06-03T20:12:15.847 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:12:15.847 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:12:15.847 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "op_commit", 2023-06-03T20:12:15.848 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:03:49.204524+0000", 2023-06-03T20:12:15.848 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.165183205 2023-06-03T20:12:15.848 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:12:15.848 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:12:15.848 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:12:15.848 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:03:49.369707+0000", 2023-06-03T20:12:15.848 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 5.0173999999999998e-05 2023-06-03T20:12:15.848 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:12:15.849 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:12:15.849 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "commit_sent", 2023-06-03T20:12:15.849 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:03:49.369757+0000", 2023-06-03T20:12:15.849 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 4.2320999999999997e-05 2023-06-03T20:12:15.849 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:12:15.849 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:12:15.849 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "done", 2023-06-03T20:12:15.849 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:03:49.369800+0000", 2023-06-03T20:12:15.850 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.18809337000000001 2023-06-03T20:12:15.850 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:12:15.850 INFO:teuthology.orchestra.run.smithi164.stdout: ] 2023-06-03T20:12:15.850 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:12:15.850 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:12:15.850 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:12:15.850 INFO:teuthology.orchestra.run.smithi164.stdout: "description": "osd_op(client.45713.0:181587 9.7 9:ed129227:::benchmark_data_smithi089_115580_object71453:head [delete] snapc 0=[] ondisk+write+known_if_redirected e142)", 2023-06-03T20:12:15.850 INFO:teuthology.orchestra.run.smithi164.stdout: "initiated_at": "2023-06-03T20:03:49.182186+0000", 2023-06-03T20:12:15.851 INFO:teuthology.orchestra.run.smithi164.stdout: "age": 506.546395297, 2023-06-03T20:12:15.851 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.18761771899999999, 2023-06-03T20:12:15.851 INFO:teuthology.orchestra.run.smithi164.stdout: "type_data": { 2023-06-03T20:12:15.851 INFO:teuthology.orchestra.run.smithi164.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-03T20:12:15.851 INFO:teuthology.orchestra.run.smithi164.stdout: "client_info": { 2023-06-03T20:12:15.851 INFO:teuthology.orchestra.run.smithi164.stdout: "client": "client.45713", 2023-06-03T20:12:15.851 INFO:teuthology.orchestra.run.smithi164.stdout: "client_addr": "172.21.15.89:0/2400821448", 2023-06-03T20:12:15.852 INFO:teuthology.orchestra.run.smithi164.stdout: "tid": 181587 2023-06-03T20:12:15.852 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:12:15.852 INFO:teuthology.orchestra.run.smithi164.stdout: "events": [ 2023-06-03T20:12:15.852 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:12:15.852 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "initiated", 2023-06-03T20:12:15.852 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:03:49.182186+0000", 2023-06-03T20:12:15.852 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0 2023-06-03T20:12:15.852 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:12:15.853 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:12:15.853 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "throttled", 2023-06-03T20:12:15.853 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:03:49.182186+0000", 2023-06-03T20:12:15.853 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 4294967295.999999 2023-06-03T20:12:15.853 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:12:15.853 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:12:15.853 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "header_read", 2023-06-03T20:12:15.853 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:03:49.182185+0000", 2023-06-03T20:12:15.854 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 5.2610000000000001e-06 2023-06-03T20:12:15.854 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:12:15.854 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:12:15.854 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "all_read", 2023-06-03T20:12:15.854 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:03:49.182190+0000", 2023-06-03T20:12:15.854 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 1.063e-05 2023-06-03T20:12:15.854 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:12:15.854 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:12:15.855 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "dispatched", 2023-06-03T20:12:15.855 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:03:49.182201+0000", 2023-06-03T20:12:15.855 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 6.0310000000000004e-06 2023-06-03T20:12:15.855 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:12:15.855 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:12:15.855 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "queued_for_pg", 2023-06-03T20:12:15.855 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:03:49.182207+0000", 2023-06-03T20:12:15.855 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.010932816 2023-06-03T20:12:15.856 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:12:15.856 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:12:15.856 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "reached_pg", 2023-06-03T20:12:15.856 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:03:49.193140+0000", 2023-06-03T20:12:15.856 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.000215019 2023-06-03T20:12:15.856 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:12:15.856 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:12:15.856 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "started", 2023-06-03T20:12:15.857 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:03:49.193355+0000", 2023-06-03T20:12:15.857 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.000118583 2023-06-03T20:12:15.857 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:12:15.857 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:12:15.857 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "waiting for subops from 3,7", 2023-06-03T20:12:15.857 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:03:49.193473+0000", 2023-06-03T20:12:15.857 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.011057813 2023-06-03T20:12:15.857 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:12:15.858 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:12:15.858 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "op_commit", 2023-06-03T20:12:15.858 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:03:49.204531+0000", 2023-06-03T20:12:15.858 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.00069448700000000003 2023-06-03T20:12:15.858 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:12:15.858 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:12:15.858 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:12:15.858 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:03:49.205226+0000", 2023-06-03T20:12:15.859 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.164484939 2023-06-03T20:12:15.859 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:12:15.859 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:12:15.859 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:12:15.859 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:03:49.369711+0000", 2023-06-03T20:12:15.859 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 6.1407000000000003e-05 2023-06-03T20:12:15.859 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:12:15.859 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:12:15.860 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "commit_sent", 2023-06-03T20:12:15.860 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:03:49.369772+0000", 2023-06-03T20:12:15.860 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 3.1643999999999997e-05 2023-06-03T20:12:15.860 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:12:15.860 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:12:15.860 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "done", 2023-06-03T20:12:15.860 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:03:49.369804+0000", 2023-06-03T20:12:15.861 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.18761771899999999 2023-06-03T20:12:15.861 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:12:15.861 INFO:teuthology.orchestra.run.smithi164.stdout: ] 2023-06-03T20:12:15.861 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:12:15.861 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:12:15.861 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:12:15.861 INFO:teuthology.orchestra.run.smithi164.stdout: "description": "osd_op(client.36009.0:126090 10.6 10:62077c05:::benchmark_data_smithi089_140876_object20955:head [delete] snapc 0=[] ondisk+write+known_if_redirected e174)", 2023-06-03T20:12:15.861 INFO:teuthology.orchestra.run.smithi164.stdout: "initiated_at": "2023-06-03T20:08:16.517668+0000", 2023-06-03T20:12:15.862 INFO:teuthology.orchestra.run.smithi164.stdout: "age": 239.21091313299999, 2023-06-03T20:12:15.862 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.18350506999999999, 2023-06-03T20:12:15.862 INFO:teuthology.orchestra.run.smithi164.stdout: "type_data": { 2023-06-03T20:12:15.862 INFO:teuthology.orchestra.run.smithi164.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-03T20:12:15.862 INFO:teuthology.orchestra.run.smithi164.stdout: "client_info": { 2023-06-03T20:12:15.862 INFO:teuthology.orchestra.run.smithi164.stdout: "client": "client.36009", 2023-06-03T20:12:15.862 INFO:teuthology.orchestra.run.smithi164.stdout: "client_addr": "172.21.15.89:0/2550116169", 2023-06-03T20:12:15.862 INFO:teuthology.orchestra.run.smithi164.stdout: "tid": 126090 2023-06-03T20:12:15.863 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:12:15.863 INFO:teuthology.orchestra.run.smithi164.stdout: "events": [ 2023-06-03T20:12:15.863 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:12:15.863 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "initiated", 2023-06-03T20:12:15.863 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:08:16.517668+0000", 2023-06-03T20:12:15.863 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0 2023-06-03T20:12:15.863 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:12:15.863 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:12:15.864 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "throttled", 2023-06-03T20:12:15.864 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:08:16.517668+0000", 2023-06-03T20:12:15.864 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 4294967295.9999971 2023-06-03T20:12:15.864 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:12:15.864 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:12:15.864 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "header_read", 2023-06-03T20:12:15.864 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:08:16.517665+0000", 2023-06-03T20:12:15.865 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 1.3027000000000001e-05 2023-06-03T20:12:15.865 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:12:15.865 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:12:15.865 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "all_read", 2023-06-03T20:12:15.865 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:08:16.517678+0000", 2023-06-03T20:12:15.865 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 1.9488999999999999e-05 2023-06-03T20:12:15.865 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:12:15.865 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:12:15.866 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "dispatched", 2023-06-03T20:12:15.866 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:08:16.517698+0000", 2023-06-03T20:12:15.866 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 8.3010000000000007e-06 2023-06-03T20:12:15.866 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:12:15.866 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:12:15.866 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "queued_for_pg", 2023-06-03T20:12:15.866 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:08:16.517706+0000", 2023-06-03T20:12:15.866 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.006973562 2023-06-03T20:12:15.867 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:12:15.867 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:12:15.867 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "reached_pg", 2023-06-03T20:12:15.867 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:08:16.524680+0000", 2023-06-03T20:12:15.867 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 9.9160999999999995e-05 2023-06-03T20:12:15.867 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:12:15.867 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:12:15.867 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "started", 2023-06-03T20:12:15.868 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:08:16.524779+0000", 2023-06-03T20:12:15.868 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 8.2677999999999997e-05 2023-06-03T20:12:15.868 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:12:15.868 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:12:15.868 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "waiting for subops from 0,4", 2023-06-03T20:12:15.868 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:08:16.524862+0000", 2023-06-03T20:12:15.868 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.010156762 2023-06-03T20:12:15.868 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:12:15.869 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:12:15.869 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "op_commit", 2023-06-03T20:12:15.869 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:08:16.535018+0000", 2023-06-03T20:12:15.869 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.0072027999999999997 2023-06-03T20:12:15.869 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:12:15.869 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:12:15.869 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:12:15.869 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:08:16.542221+0000", 2023-06-03T20:12:15.870 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.15878830599999999 2023-06-03T20:12:15.870 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:12:15.870 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:12:15.870 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:12:15.870 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:08:16.701009+0000", 2023-06-03T20:12:15.870 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 9.3574999999999998e-05 2023-06-03T20:12:15.870 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:12:15.870 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:12:15.871 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "commit_sent", 2023-06-03T20:12:15.871 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:08:16.701103+0000", 2023-06-03T20:12:15.871 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 7.0171000000000003e-05 2023-06-03T20:12:15.871 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:12:15.871 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:12:15.871 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "done", 2023-06-03T20:12:15.871 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:08:16.701173+0000", 2023-06-03T20:12:15.871 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.18350506999999999 2023-06-03T20:12:15.872 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:12:15.872 INFO:teuthology.orchestra.run.smithi164.stdout: ] 2023-06-03T20:12:15.872 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:12:15.872 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:12:15.872 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:12:15.872 INFO:teuthology.orchestra.run.smithi164.stdout: "description": "osd_op(client.36009.0:159979 10.6 10:6510875e:::benchmark_data_smithi089_140876_object54844:head [delete] snapc 0=[] ondisk+write+known_if_redirected e176)", 2023-06-03T20:12:15.872 INFO:teuthology.orchestra.run.smithi164.stdout: "initiated_at": "2023-06-03T20:09:10.081329+0000", 2023-06-03T20:12:15.873 INFO:teuthology.orchestra.run.smithi164.stdout: "age": 185.64725203699999, 2023-06-03T20:12:15.873 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.21788642799999999, 2023-06-03T20:12:15.873 INFO:teuthology.orchestra.run.smithi164.stdout: "type_data": { 2023-06-03T20:12:15.873 INFO:teuthology.orchestra.run.smithi164.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-03T20:12:15.873 INFO:teuthology.orchestra.run.smithi164.stdout: "client_info": { 2023-06-03T20:12:15.873 INFO:teuthology.orchestra.run.smithi164.stdout: "client": "client.36009", 2023-06-03T20:12:15.873 INFO:teuthology.orchestra.run.smithi164.stdout: "client_addr": "172.21.15.89:0/2550116169", 2023-06-03T20:12:15.873 INFO:teuthology.orchestra.run.smithi164.stdout: "tid": 159979 2023-06-03T20:12:15.874 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:12:15.874 INFO:teuthology.orchestra.run.smithi164.stdout: "events": [ 2023-06-03T20:12:15.874 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:12:15.874 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "initiated", 2023-06-03T20:12:15.874 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:09:10.081329+0000", 2023-06-03T20:12:15.874 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0 2023-06-03T20:12:15.874 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:12:15.874 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:12:15.875 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "throttled", 2023-06-03T20:12:15.875 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:09:10.081329+0000", 2023-06-03T20:12:15.875 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 4294967295.9999976 2023-06-03T20:12:15.875 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:12:15.875 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:12:15.875 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "header_read", 2023-06-03T20:12:15.875 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:09:10.081327+0000", 2023-06-03T20:12:15.875 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 1.2822e-05 2023-06-03T20:12:15.876 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:12:15.876 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:12:15.876 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "all_read", 2023-06-03T20:12:15.876 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:09:10.081340+0000", 2023-06-03T20:12:15.876 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 1.7034999999999999e-05 2023-06-03T20:12:15.876 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:12:15.876 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:12:15.876 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "dispatched", 2023-06-03T20:12:15.877 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:09:10.081357+0000", 2023-06-03T20:12:15.877 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 8.0239999999999997e-06 2023-06-03T20:12:15.877 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:12:15.877 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:12:15.877 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "queued_for_pg", 2023-06-03T20:12:15.877 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:09:10.081365+0000", 2023-06-03T20:12:15.877 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 6.8447000000000004e-05 2023-06-03T20:12:15.877 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:12:15.878 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:12:15.878 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "reached_pg", 2023-06-03T20:12:15.878 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:09:10.081433+0000", 2023-06-03T20:12:15.878 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.00016985700000000001 2023-06-03T20:12:15.878 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:12:15.878 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:12:15.878 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "started", 2023-06-03T20:12:15.878 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:09:10.081603+0000", 2023-06-03T20:12:15.879 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.00019754000000000001 2023-06-03T20:12:15.879 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:12:15.879 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:12:15.879 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "waiting for subops from 0,4", 2023-06-03T20:12:15.879 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:09:10.081801+0000", 2023-06-03T20:12:15.879 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.0063214839999999996 2023-06-03T20:12:15.879 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:12:15.879 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:12:15.880 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "op_commit", 2023-06-03T20:12:15.880 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:09:10.088122+0000", 2023-06-03T20:12:15.880 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.0086637999999999993 2023-06-03T20:12:15.880 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:12:15.880 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:12:15.880 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:12:15.880 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:09:10.096786+0000", 2023-06-03T20:12:15.880 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.20234137999999999 2023-06-03T20:12:15.881 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:12:15.881 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:12:15.881 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:12:15.881 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:09:10.299127+0000", 2023-06-03T20:12:15.881 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 4.3959999999999999e-05 2023-06-03T20:12:15.881 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:12:15.881 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:12:15.881 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "commit_sent", 2023-06-03T20:12:15.882 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:09:10.299171+0000", 2023-06-03T20:12:15.882 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 4.4280999999999998e-05 2023-06-03T20:12:15.882 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:12:15.882 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:12:15.882 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "done", 2023-06-03T20:12:15.882 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:09:10.299216+0000", 2023-06-03T20:12:15.882 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.21788642799999999 2023-06-03T20:12:15.882 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:12:15.883 INFO:teuthology.orchestra.run.smithi164.stdout: ] 2023-06-03T20:12:15.883 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:12:15.883 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:12:15.883 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:12:15.883 INFO:teuthology.orchestra.run.smithi164.stdout: "description": "osd_op(client.46187.0:60459 11.7 11:e813e1ef:::benchmark_data_smithi089_165412_object60458:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e197)", 2023-06-03T20:12:15.883 INFO:teuthology.orchestra.run.smithi164.stdout: "initiated_at": "2023-06-03T20:12:10.679785+0000", 2023-06-03T20:12:15.883 INFO:teuthology.orchestra.run.smithi164.stdout: "age": 5.048796061, 2023-06-03T20:12:15.883 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.304126226, 2023-06-03T20:12:15.884 INFO:teuthology.orchestra.run.smithi164.stdout: "type_data": { 2023-06-03T20:12:15.884 INFO:teuthology.orchestra.run.smithi164.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-03T20:12:15.884 INFO:teuthology.orchestra.run.smithi164.stdout: "client_info": { 2023-06-03T20:12:15.884 INFO:teuthology.orchestra.run.smithi164.stdout: "client": "client.46187", 2023-06-03T20:12:15.884 INFO:teuthology.orchestra.run.smithi164.stdout: "client_addr": "172.21.15.89:0/880684210", 2023-06-03T20:12:15.884 INFO:teuthology.orchestra.run.smithi164.stdout: "tid": 60459 2023-06-03T20:12:15.884 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:12:15.885 INFO:teuthology.orchestra.run.smithi164.stdout: "events": [ 2023-06-03T20:12:15.885 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:12:15.885 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "initiated", 2023-06-03T20:12:15.885 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:12:10.679785+0000", 2023-06-03T20:12:15.885 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0 2023-06-03T20:12:15.885 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:12:15.885 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:12:15.885 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "throttled", 2023-06-03T20:12:15.886 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:12:10.679785+0000", 2023-06-03T20:12:15.886 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 4294967295.9999981 2023-06-03T20:12:15.886 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:12:15.886 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:12:15.886 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "header_read", 2023-06-03T20:12:15.886 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:12:10.679783+0000", 2023-06-03T20:12:15.886 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 3.3775000000000001e-05 2023-06-03T20:12:15.886 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:12:15.887 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:12:15.887 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "all_read", 2023-06-03T20:12:15.887 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:12:10.679817+0000", 2023-06-03T20:12:15.887 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 1.3056e-05 2023-06-03T20:12:15.887 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:12:15.887 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:12:15.887 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "dispatched", 2023-06-03T20:12:15.887 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:12:10.679830+0000", 2023-06-03T20:12:15.888 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 6.4570000000000002e-06 2023-06-03T20:12:15.888 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:12:15.888 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:12:15.888 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "queued_for_pg", 2023-06-03T20:12:15.888 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:12:10.679837+0000", 2023-06-03T20:12:15.888 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 5.2102999999999999e-05 2023-06-03T20:12:15.888 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:12:15.888 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:12:15.889 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "reached_pg", 2023-06-03T20:12:15.889 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:12:10.679889+0000", 2023-06-03T20:12:15.889 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.000128832 2023-06-03T20:12:15.889 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:12:15.889 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:12:15.889 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "started", 2023-06-03T20:12:15.889 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:12:10.680018+0000", 2023-06-03T20:12:15.890 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.00015569999999999999 2023-06-03T20:12:15.890 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:12:15.890 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:12:15.890 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "waiting for subops from 0", 2023-06-03T20:12:15.890 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:12:10.680173+0000", 2023-06-03T20:12:15.890 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.0040138099999999996 2023-06-03T20:12:15.890 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:12:15.890 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:12:15.891 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "op_commit", 2023-06-03T20:12:15.891 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:12:10.684187+0000", 2023-06-03T20:12:15.891 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.29957187400000002 2023-06-03T20:12:15.891 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:12:15.891 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:12:15.891 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:12:15.891 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:12:10.983759+0000", 2023-06-03T20:12:15.891 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 5.6368e-05 2023-06-03T20:12:15.892 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:12:15.892 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:12:15.892 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "commit_sent", 2023-06-03T20:12:15.892 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:12:10.983815+0000", 2023-06-03T20:12:15.892 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 9.5947999999999997e-05 2023-06-03T20:12:15.892 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:12:15.892 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:12:15.892 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "done", 2023-06-03T20:12:15.893 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:12:10.983911+0000", 2023-06-03T20:12:15.893 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.304126226 2023-06-03T20:12:15.893 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:12:15.893 INFO:teuthology.orchestra.run.smithi164.stdout: ] 2023-06-03T20:12:15.893 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:12:15.893 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:12:15.893 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:12:15.893 INFO:teuthology.orchestra.run.smithi164.stdout: "description": "osd_op(client.46187.0:60468 11.7 11:e244c030:::benchmark_data_smithi089_165412_object60467:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e197)", 2023-06-03T20:12:15.894 INFO:teuthology.orchestra.run.smithi164.stdout: "initiated_at": "2023-06-03T20:12:10.686093+0000", 2023-06-03T20:12:15.894 INFO:teuthology.orchestra.run.smithi164.stdout: "age": 5.0424877759999998, 2023-06-03T20:12:15.894 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.29801150900000001, 2023-06-03T20:12:15.894 INFO:teuthology.orchestra.run.smithi164.stdout: "type_data": { 2023-06-03T20:12:15.894 INFO:teuthology.orchestra.run.smithi164.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-03T20:12:15.894 INFO:teuthology.orchestra.run.smithi164.stdout: "client_info": { 2023-06-03T20:12:15.894 INFO:teuthology.orchestra.run.smithi164.stdout: "client": "client.46187", 2023-06-03T20:12:15.894 INFO:teuthology.orchestra.run.smithi164.stdout: "client_addr": "172.21.15.89:0/880684210", 2023-06-03T20:12:15.895 INFO:teuthology.orchestra.run.smithi164.stdout: "tid": 60468 2023-06-03T20:12:15.895 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:12:15.895 INFO:teuthology.orchestra.run.smithi164.stdout: "events": [ 2023-06-03T20:12:15.895 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:12:15.895 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "initiated", 2023-06-03T20:12:15.895 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:12:10.686093+0000", 2023-06-03T20:12:15.895 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0 2023-06-03T20:12:15.895 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:12:15.896 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:12:15.896 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "throttled", 2023-06-03T20:12:15.896 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:12:10.686093+0000", 2023-06-03T20:12:15.896 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 4294967295.9999981 2023-06-03T20:12:15.896 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:12:15.896 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:12:15.896 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "header_read", 2023-06-03T20:12:15.896 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:12:10.686091+0000", 2023-06-03T20:12:15.897 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 3.3797999999999997e-05 2023-06-03T20:12:15.897 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:12:15.897 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:12:15.897 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "all_read", 2023-06-03T20:12:15.897 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:12:10.686125+0000", 2023-06-03T20:12:15.897 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 1.2563e-05 2023-06-03T20:12:15.897 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:12:15.897 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:12:15.898 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "dispatched", 2023-06-03T20:12:15.898 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:12:10.686138+0000", 2023-06-03T20:12:15.898 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 6.4919999999999998e-06 2023-06-03T20:12:15.898 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:12:15.898 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:12:15.898 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "queued_for_pg", 2023-06-03T20:12:15.898 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:12:10.686144+0000", 2023-06-03T20:12:15.898 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 5.0581999999999998e-05 2023-06-03T20:12:15.899 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:12:15.899 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:12:15.899 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "reached_pg", 2023-06-03T20:12:15.899 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:12:10.686195+0000", 2023-06-03T20:12:15.899 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.000144315 2023-06-03T20:12:15.899 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:12:15.899 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:12:15.899 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "started", 2023-06-03T20:12:15.900 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:12:10.686339+0000", 2023-06-03T20:12:15.900 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.00017369900000000001 2023-06-03T20:12:15.900 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:12:15.900 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:12:15.900 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "waiting for subops from 0", 2023-06-03T20:12:15.900 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:12:10.686513+0000", 2023-06-03T20:12:15.900 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.29741215300000001 2023-06-03T20:12:15.901 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:12:15.901 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:12:15.901 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "op_commit", 2023-06-03T20:12:15.901 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:12:10.983925+0000", 2023-06-03T20:12:15.901 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 4.3424000000000003e-05 2023-06-03T20:12:15.901 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:12:15.901 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:12:15.901 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:12:15.902 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:12:10.983968+0000", 2023-06-03T20:12:15.902 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 4.6980000000000001e-05 2023-06-03T20:12:15.902 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:12:15.902 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:12:15.902 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "commit_sent", 2023-06-03T20:12:15.902 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:12:10.984015+0000", 2023-06-03T20:12:15.902 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 8.9596000000000001e-05 2023-06-03T20:12:15.902 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:12:15.903 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:12:15.903 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "done", 2023-06-03T20:12:15.903 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:12:10.984105+0000", 2023-06-03T20:12:15.903 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.29801150900000001 2023-06-03T20:12:15.903 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:12:15.903 INFO:teuthology.orchestra.run.smithi164.stdout: ] 2023-06-03T20:12:15.903 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:12:15.903 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:12:15.904 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:12:15.904 INFO:teuthology.orchestra.run.smithi164.stdout: "description": "osd_repop(client.46187.0:60467 11.3 e197/189 11:c659ad50:::benchmark_data_smithi089_165412_object60466:head v 197'3834, mlcod=197'3831)", 2023-06-03T20:12:15.904 INFO:teuthology.orchestra.run.smithi164.stdout: "initiated_at": "2023-06-03T20:12:10.686399+0000", 2023-06-03T20:12:15.904 INFO:teuthology.orchestra.run.smithi164.stdout: "age": 5.0421821109999998, 2023-06-03T20:12:15.904 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.29427065099999999, 2023-06-03T20:12:15.904 INFO:teuthology.orchestra.run.smithi164.stdout: "type_data": { 2023-06-03T20:12:15.904 INFO:teuthology.orchestra.run.smithi164.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-03T20:12:15.904 INFO:teuthology.orchestra.run.smithi164.stdout: "events": [ 2023-06-03T20:12:15.905 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:12:15.905 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "initiated", 2023-06-03T20:12:15.905 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:12:10.686399+0000", 2023-06-03T20:12:15.905 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0 2023-06-03T20:12:15.905 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:12:15.905 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:12:15.905 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "throttled", 2023-06-03T20:12:15.905 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:12:10.686399+0000", 2023-06-03T20:12:15.906 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 4294967295.999999 2023-06-03T20:12:15.906 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:12:15.906 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:12:15.906 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "header_read", 2023-06-03T20:12:15.906 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:12:10.686398+0000", 2023-06-03T20:12:15.906 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 2.5877999999999999e-05 2023-06-03T20:12:15.906 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:12:15.906 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:12:15.907 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "all_read", 2023-06-03T20:12:15.907 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:12:10.686424+0000", 2023-06-03T20:12:15.907 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 1.0124e-05 2023-06-03T20:12:15.907 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:12:15.907 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:12:15.908 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "dispatched", 2023-06-03T20:12:15.908 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:12:10.686434+0000", 2023-06-03T20:12:15.908 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 5.7610000000000004e-06 2023-06-03T20:12:15.908 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:12:15.908 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:12:15.908 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "queued_for_pg", 2023-06-03T20:12:15.908 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:12:10.686440+0000", 2023-06-03T20:12:15.908 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 4.9316e-05 2023-06-03T20:12:15.909 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:12:15.909 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:12:15.909 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "reached_pg", 2023-06-03T20:12:15.909 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:12:10.686489+0000", 2023-06-03T20:12:15.909 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 2.4464000000000002e-05 2023-06-03T20:12:15.909 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:12:15.909 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:12:15.909 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "started", 2023-06-03T20:12:15.910 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:12:10.686514+0000", 2023-06-03T20:12:15.910 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.29411429100000003 2023-06-03T20:12:15.910 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:12:15.910 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:12:15.910 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "commit_sent", 2023-06-03T20:12:15.910 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:12:10.980628+0000", 2023-06-03T20:12:15.910 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 4.1584e-05 2023-06-03T20:12:15.910 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:12:15.911 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:12:15.911 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "done", 2023-06-03T20:12:15.911 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:12:10.980670+0000", 2023-06-03T20:12:15.911 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.29427065099999999 2023-06-03T20:12:15.911 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:12:15.911 INFO:teuthology.orchestra.run.smithi164.stdout: ] 2023-06-03T20:12:15.911 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:12:15.912 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:12:15.912 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:12:15.912 INFO:teuthology.orchestra.run.smithi164.stdout: "description": "osd_repop(client.46187.0:60470 11.8 e197/189 11:116c2bad:::benchmark_data_smithi089_165412_object60469:head v 197'3819, mlcod=197'3816)", 2023-06-03T20:12:15.912 INFO:teuthology.orchestra.run.smithi164.stdout: "initiated_at": "2023-06-03T20:12:10.687701+0000", 2023-06-03T20:12:15.912 INFO:teuthology.orchestra.run.smithi164.stdout: "age": 5.0408800999999999, 2023-06-03T20:12:15.912 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.29296406600000002, 2023-06-03T20:12:15.912 INFO:teuthology.orchestra.run.smithi164.stdout: "type_data": { 2023-06-03T20:12:15.912 INFO:teuthology.orchestra.run.smithi164.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-03T20:12:15.913 INFO:teuthology.orchestra.run.smithi164.stdout: "events": [ 2023-06-03T20:12:15.913 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:12:15.913 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "initiated", 2023-06-03T20:12:15.913 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:12:10.687701+0000", 2023-06-03T20:12:15.913 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0 2023-06-03T20:12:15.913 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:12:15.913 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:12:15.913 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "throttled", 2023-06-03T20:12:15.914 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:12:10.687701+0000", 2023-06-03T20:12:15.914 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 4294967295.999999 2023-06-03T20:12:15.914 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:12:15.914 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:12:15.914 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "header_read", 2023-06-03T20:12:15.914 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:12:10.687700+0000", 2023-06-03T20:12:15.914 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.000126904 2023-06-03T20:12:15.914 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:12:15.915 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:12:15.915 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "all_read", 2023-06-03T20:12:15.915 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:12:10.687827+0000", 2023-06-03T20:12:15.915 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 1.3742e-05 2023-06-03T20:12:15.915 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:12:15.915 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:12:15.915 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "dispatched", 2023-06-03T20:12:15.915 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:12:10.687841+0000", 2023-06-03T20:12:15.916 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 7.2919999999999998e-06 2023-06-03T20:12:15.916 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:12:15.916 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:12:15.916 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "queued_for_pg", 2023-06-03T20:12:15.916 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:12:10.687848+0000", 2023-06-03T20:12:15.916 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 6.2892999999999996e-05 2023-06-03T20:12:15.916 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:12:15.916 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:12:15.917 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "reached_pg", 2023-06-03T20:12:15.917 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:12:10.687911+0000", 2023-06-03T20:12:15.917 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 2.0224000000000002e-05 2023-06-03T20:12:15.917 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:12:15.917 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:12:15.917 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "started", 2023-06-03T20:12:15.917 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:12:10.687931+0000", 2023-06-03T20:12:15.917 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.29267958399999999 2023-06-03T20:12:15.918 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:12:15.918 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:12:15.918 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "commit_sent", 2023-06-03T20:12:15.918 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:12:10.980611+0000", 2023-06-03T20:12:15.918 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 5.4381e-05 2023-06-03T20:12:15.918 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:12:15.918 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:12:15.918 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "done", 2023-06-03T20:12:15.919 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:12:10.980665+0000", 2023-06-03T20:12:15.919 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.29296406600000002 2023-06-03T20:12:15.919 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:12:15.919 INFO:teuthology.orchestra.run.smithi164.stdout: ] 2023-06-03T20:12:15.919 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:12:15.919 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:12:15.919 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:12:15.920 INFO:teuthology.orchestra.run.smithi164.stdout: "description": "osd_op(client.46187.0:60471 11.7 11:ea4b9657:::benchmark_data_smithi089_165412_object60470:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e197)", 2023-06-03T20:12:15.920 INFO:teuthology.orchestra.run.smithi164.stdout: "initiated_at": "2023-06-03T20:12:10.688000+0000", 2023-06-03T20:12:15.920 INFO:teuthology.orchestra.run.smithi164.stdout: "age": 5.0405811299999996, 2023-06-03T20:12:15.920 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.30822460800000001, 2023-06-03T20:12:15.920 INFO:teuthology.orchestra.run.smithi164.stdout: "type_data": { 2023-06-03T20:12:15.920 INFO:teuthology.orchestra.run.smithi164.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-03T20:12:15.920 INFO:teuthology.orchestra.run.smithi164.stdout: "client_info": { 2023-06-03T20:12:15.920 INFO:teuthology.orchestra.run.smithi164.stdout: "client": "client.46187", 2023-06-03T20:12:15.921 INFO:teuthology.orchestra.run.smithi164.stdout: "client_addr": "172.21.15.89:0/880684210", 2023-06-03T20:12:15.921 INFO:teuthology.orchestra.run.smithi164.stdout: "tid": 60471 2023-06-03T20:12:15.921 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:12:15.921 INFO:teuthology.orchestra.run.smithi164.stdout: "events": [ 2023-06-03T20:12:15.921 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:12:15.921 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "initiated", 2023-06-03T20:12:15.921 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:12:10.688000+0000", 2023-06-03T20:12:15.921 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0 2023-06-03T20:12:15.922 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:12:15.922 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:12:15.922 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "throttled", 2023-06-03T20:12:15.922 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:12:10.688000+0000", 2023-06-03T20:12:15.922 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 4294967295.999999 2023-06-03T20:12:15.922 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:12:15.922 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:12:15.922 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "header_read", 2023-06-03T20:12:15.923 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:12:10.687999+0000", 2023-06-03T20:12:15.923 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 8.9674999999999998e-05 2023-06-03T20:12:15.923 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:12:15.923 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:12:15.923 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "all_read", 2023-06-03T20:12:15.923 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:12:10.688089+0000", 2023-06-03T20:12:15.923 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 1.3740000000000001e-05 2023-06-03T20:12:15.923 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:12:15.924 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:12:15.924 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "dispatched", 2023-06-03T20:12:15.924 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:12:10.688102+0000", 2023-06-03T20:12:15.924 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 5.6280000000000002e-06 2023-06-03T20:12:15.924 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:12:15.924 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:12:15.924 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "queued_for_pg", 2023-06-03T20:12:15.924 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:12:10.688108+0000", 2023-06-03T20:12:15.925 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.00060365500000000003 2023-06-03T20:12:15.925 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:12:15.925 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:12:15.925 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "reached_pg", 2023-06-03T20:12:15.925 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:12:10.688712+0000", 2023-06-03T20:12:15.925 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.00013657900000000001 2023-06-03T20:12:15.925 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:12:15.925 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:12:15.926 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "started", 2023-06-03T20:12:15.926 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:12:10.688848+0000", 2023-06-03T20:12:15.926 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.29192311799999998 2023-06-03T20:12:15.926 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:12:15.926 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:12:15.926 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "waiting for subops from 0", 2023-06-03T20:12:15.926 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:12:10.980771+0000", 2023-06-03T20:12:15.926 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.0061675640000000004 2023-06-03T20:12:15.927 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:12:15.927 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:12:15.927 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "op_commit", 2023-06-03T20:12:15.927 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:12:10.986939+0000", 2023-06-03T20:12:15.927 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.0091243100000000001 2023-06-03T20:12:15.927 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:12:15.927 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:12:15.927 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:12:15.928 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:12:10.996063+0000", 2023-06-03T20:12:15.928 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 6.1641000000000002e-05 2023-06-03T20:12:15.928 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:12:15.928 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:12:15.928 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "commit_sent", 2023-06-03T20:12:15.928 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:12:10.996125+0000", 2023-06-03T20:12:15.928 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 9.9853999999999996e-05 2023-06-03T20:12:15.928 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:12:15.929 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:12:15.929 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "done", 2023-06-03T20:12:15.929 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:12:10.996225+0000", 2023-06-03T20:12:15.929 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.30822460800000001 2023-06-03T20:12:15.929 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:12:15.929 INFO:teuthology.orchestra.run.smithi164.stdout: ] 2023-06-03T20:12:15.929 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:12:15.929 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:12:15.930 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:12:15.930 INFO:teuthology.orchestra.run.smithi164.stdout: "description": "osd_op(client.46187.0:60473 11.7 11:e4006b35:::benchmark_data_smithi089_165412_object60472:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e197)", 2023-06-03T20:12:15.930 INFO:teuthology.orchestra.run.smithi164.stdout: "initiated_at": "2023-06-03T20:12:10.689610+0000", 2023-06-03T20:12:15.930 INFO:teuthology.orchestra.run.smithi164.stdout: "age": 5.0389712470000001, 2023-06-03T20:12:15.930 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.306784532, 2023-06-03T20:12:15.930 INFO:teuthology.orchestra.run.smithi164.stdout: "type_data": { 2023-06-03T20:12:15.930 INFO:teuthology.orchestra.run.smithi164.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-03T20:12:15.931 INFO:teuthology.orchestra.run.smithi164.stdout: "client_info": { 2023-06-03T20:12:15.931 INFO:teuthology.orchestra.run.smithi164.stdout: "client": "client.46187", 2023-06-03T20:12:15.931 INFO:teuthology.orchestra.run.smithi164.stdout: "client_addr": "172.21.15.89:0/880684210", 2023-06-03T20:12:15.931 INFO:teuthology.orchestra.run.smithi164.stdout: "tid": 60473 2023-06-03T20:12:15.931 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:12:15.931 INFO:teuthology.orchestra.run.smithi164.stdout: "events": [ 2023-06-03T20:12:15.931 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:12:15.931 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "initiated", 2023-06-03T20:12:15.932 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:12:10.689610+0000", 2023-06-03T20:12:15.932 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0 2023-06-03T20:12:15.932 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:12:15.932 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:12:15.932 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "throttled", 2023-06-03T20:12:15.932 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:12:10.689610+0000", 2023-06-03T20:12:15.932 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 4294967295.9999986 2023-06-03T20:12:15.932 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:12:15.933 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:12:15.933 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "header_read", 2023-06-03T20:12:15.933 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:12:10.689608+0000", 2023-06-03T20:12:15.933 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 7.3188999999999993e-05 2023-06-03T20:12:15.933 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:12:15.933 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:12:15.933 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "all_read", 2023-06-03T20:12:15.933 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:12:10.689682+0000", 2023-06-03T20:12:15.934 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.29092713199999998 2023-06-03T20:12:15.934 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:12:15.934 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:12:15.934 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "dispatched", 2023-06-03T20:12:15.934 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:12:10.980609+0000", 2023-06-03T20:12:15.934 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 3.2396e-05 2023-06-03T20:12:15.934 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:12:15.934 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:12:15.935 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "queued_for_pg", 2023-06-03T20:12:15.935 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:12:10.980641+0000", 2023-06-03T20:12:15.935 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.003502327 2023-06-03T20:12:15.935 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:12:15.935 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:12:15.935 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "reached_pg", 2023-06-03T20:12:15.935 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:12:10.984143+0000", 2023-06-03T20:12:15.935 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.00016113999999999999 2023-06-03T20:12:15.936 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:12:15.936 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:12:15.936 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "started", 2023-06-03T20:12:15.936 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:12:10.984305+0000", 2023-06-03T20:12:15.936 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.00018914800000000001 2023-06-03T20:12:15.936 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:12:15.936 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:12:15.936 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "waiting for subops from 0", 2023-06-03T20:12:15.937 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:12:10.984494+0000", 2023-06-03T20:12:15.937 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.0055146079999999998 2023-06-03T20:12:15.937 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:12:15.937 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:12:15.937 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "op_commit", 2023-06-03T20:12:15.937 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:12:10.990008+0000", 2023-06-03T20:12:15.937 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.0062579009999999997 2023-06-03T20:12:15.938 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:12:15.938 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:12:15.938 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:12:15.938 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:12:10.996266+0000", 2023-06-03T20:12:15.938 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 4.6075999999999997e-05 2023-06-03T20:12:15.938 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:12:15.938 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:12:15.938 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "commit_sent", 2023-06-03T20:12:15.939 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:12:10.996312+0000", 2023-06-03T20:12:15.939 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 8.2223000000000002e-05 2023-06-03T20:12:15.939 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:12:15.939 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:12:15.939 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "done", 2023-06-03T20:12:15.939 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:12:10.996395+0000", 2023-06-03T20:12:15.939 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.306784532 2023-06-03T20:12:15.939 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:12:15.940 INFO:teuthology.orchestra.run.smithi164.stdout: ] 2023-06-03T20:12:15.940 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:12:15.940 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:12:15.940 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:12:15.940 INFO:teuthology.orchestra.run.smithi164.stdout: "description": "osd_repop_reply(client.46187.0:60459 11.7 e197/195 ondisk, result = 0)", 2023-06-03T20:12:15.940 INFO:teuthology.orchestra.run.smithi164.stdout: "initiated_at": "2023-06-03T20:12:10.691698+0000", 2023-06-03T20:12:15.940 INFO:teuthology.orchestra.run.smithi164.stdout: "age": 5.0368836799999999, 2023-06-03T20:12:15.940 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.29222863999999998, 2023-06-03T20:12:15.941 INFO:teuthology.orchestra.run.smithi164.stdout: "type_data": { 2023-06-03T20:12:15.941 INFO:teuthology.orchestra.run.smithi164.stdout: "flag_point": "started", 2023-06-03T20:12:15.941 INFO:teuthology.orchestra.run.smithi164.stdout: "events": [ 2023-06-03T20:12:15.941 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:12:15.941 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "initiated", 2023-06-03T20:12:15.941 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:12:10.691698+0000", 2023-06-03T20:12:15.941 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0 2023-06-03T20:12:15.941 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:12:15.942 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:12:15.942 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "throttled", 2023-06-03T20:12:15.942 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:12:10.691698+0000", 2023-06-03T20:12:15.942 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 4294967295.999999 2023-06-03T20:12:15.942 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:12:15.942 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:12:15.942 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "header_read", 2023-06-03T20:12:15.942 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:12:10.691697+0000", 2023-06-03T20:12:15.943 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 1.2357999999999999e-05 2023-06-03T20:12:15.943 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:12:15.943 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:12:15.943 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "all_read", 2023-06-03T20:12:15.943 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:12:10.691709+0000", 2023-06-03T20:12:15.943 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.288913368 2023-06-03T20:12:15.943 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:12:15.943 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:12:15.944 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "dispatched", 2023-06-03T20:12:15.944 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:12:10.980622+0000", 2023-06-03T20:12:15.944 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 1.2622999999999999e-05 2023-06-03T20:12:15.944 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:12:15.944 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:12:15.944 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "queued_for_pg", 2023-06-03T20:12:15.944 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:12:10.980635+0000", 2023-06-03T20:12:15.944 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.0031077740000000002 2023-06-03T20:12:15.945 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:12:15.945 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:12:15.945 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "reached_pg", 2023-06-03T20:12:15.945 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:12:10.983743+0000", 2023-06-03T20:12:15.945 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 7.9100000000000005e-06 2023-06-03T20:12:15.945 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:12:15.945 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:12:15.945 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "started", 2023-06-03T20:12:15.946 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:12:10.983751+0000", 2023-06-03T20:12:15.946 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.000175639 2023-06-03T20:12:15.946 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:12:15.946 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:12:15.946 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "done", 2023-06-03T20:12:15.946 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:12:10.983926+0000", 2023-06-03T20:12:15.946 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.29222863999999998 2023-06-03T20:12:15.946 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:12:15.947 INFO:teuthology.orchestra.run.smithi164.stdout: ] 2023-06-03T20:12:15.947 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:12:15.947 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:12:15.947 INFO:teuthology.orchestra.run.smithi164.stdout: ] 2023-06-03T20:12:15.947 INFO:teuthology.orchestra.run.smithi164.stdout:} 2023-06-03T20:12:15.947 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:12:15.978 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:12:15.981 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:12:15.981 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:12:16.082 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:12:16.083 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:12:15 smithi164 bash[59462]: cluster 1685823134.6905818 mgr.y (mgr.34104) 1085 : cluster 0 pgmap v1054: 89 pgs: 15 active+undersized+degraded, 26 active+undersized, 48 active+clean; 3.8 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 82 MiB/s wr, 1.31k op/s; 23088/186087 objects degraded (12.407%) 2023-06-03T20:12:16.092 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:12:15 smithi089 bash[75068]: cluster 1685823134.6905818 mgr.y (mgr.34104) 1085 : cluster 0 pgmap v1054: 89 pgs: 15 active+undersized+degraded, 26 active+undersized, 48 active+clean; 3.8 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 82 MiB/s wr, 1.31k op/s; 23088/186087 objects degraded (12.407%) 2023-06-03T20:12:16.092 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:12:15 smithi089 bash[78452]: cluster 1685823134.6905818 mgr.y (mgr.34104) 1085 : cluster 0 pgmap v1054: 89 pgs: 15 active+undersized+degraded, 26 active+undersized, 48 active+clean; 3.8 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 82 MiB/s wr, 1.31k op/s; 23088/186087 objects degraded (12.407%) 2023-06-03T20:12:16.115 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:12:16.118 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:12:16.119 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:12:16.220 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:12:16.258 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:12:16.264 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:12:16.265 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:12:16.366 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:12:16.401 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:12:16.405 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:12:16.406 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:12:16.507 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:12:16.528 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 20:12:16 smithi089 bash[71163]: ::ffff:172.21.15.164 - - [03/Jun/2023:20:12:16] "GET /metrics HTTP/1.1" 200 228852 "" "Prometheus/2.33.4" 2023-06-03T20:12:16.534 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:12:16.535 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:12:16.536 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:12:16.557 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 43 16 67473 67457 98.0301 76.125 0.0035384 0.0101856 2023-06-03T20:12:16.637 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:12:16.662 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/mon.a/config 2023-06-03T20:12:16.687 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:12:16.692 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:12:16.693 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:12:16.794 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:12:16.829 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:12:16.832 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:12:16.833 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:12:16.933 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:12:16.966 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:12:16.969 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:12:16.969 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:12:17.070 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:12:17.096 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:12:17.098 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:12:17.098 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:12:17.199 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:12:17.230 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:12:17.232 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:12:17.233 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:12:17.286 DEBUG:teuthology.orchestra.run.smithi164:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph daemon osd.7 dump_ops_in_flight 2023-06-03T20:12:17.334 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:12:17.376 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:12:17.383 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:12:17.384 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:12:17.484 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:12:17.523 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:12:17.527 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:12:17.528 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:12:17.558 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 44 16 68519 68503 97.2878 65.375 0.0192481 0.0102731 2023-06-03T20:12:17.629 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:12:17.659 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:12:17.661 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:12:17.661 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:12:17.762 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:12:17.807 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:12:17.809 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:12:17.810 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:12:17.911 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:12:17.943 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:12:17.947 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:12:17.948 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:12:18.028 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:12:17 smithi089 bash[75068]: cluster 1685823136.691265 mgr.y (mgr.34104) 1086 : cluster 0 pgmap v1055: 89 pgs: 15 active+undersized+degraded, 26 active+undersized, 48 active+clean; 3.9 GiB data, 13 GiB used, 612 GiB / 626 GiB avail; 80 MiB/s wr, 1.28k op/s; 24293/190848 objects degraded (12.729%) 2023-06-03T20:12:18.028 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:12:17 smithi089 bash[75068]: cluster 1685823137.7257476 mon.a (mon.0) 1026 : cluster 3 Health check update: Degraded data redundancy: 24293/190848 objects degraded (12.729%), 15 pgs degraded (PG_DEGRADED) 2023-06-03T20:12:18.029 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:12:17 smithi089 bash[78452]: cluster 1685823136.691265 mgr.y (mgr.34104) 1086 : cluster 0 pgmap v1055: 89 pgs: 15 active+undersized+degraded, 26 active+undersized, 48 active+clean; 3.9 GiB data, 13 GiB used, 612 GiB / 626 GiB avail; 80 MiB/s wr, 1.28k op/s; 24293/190848 objects degraded (12.729%) 2023-06-03T20:12:18.029 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:12:17 smithi089 bash[78452]: cluster 1685823137.7257476 mon.a (mon.0) 1026 : cluster 3 Health check update: Degraded data redundancy: 24293/190848 objects degraded (12.729%), 15 pgs degraded (PG_DEGRADED) 2023-06-03T20:12:18.048 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:12:18.081 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:12:18.083 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:12:17 smithi164 bash[59462]: cluster 1685823136.691265 mgr.y (mgr.34104) 1086 : cluster 0 pgmap v1055: 89 pgs: 15 active+undersized+degraded, 26 active+undersized, 48 active+clean; 3.9 GiB data, 13 GiB used, 612 GiB / 626 GiB avail; 80 MiB/s wr, 1.28k op/s; 24293/190848 objects degraded (12.729%) 2023-06-03T20:12:18.083 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:12:17 smithi164 bash[59462]: cluster 1685823137.7257476 mon.a (mon.0) 1026 : cluster 3 Health check update: Degraded data redundancy: 24293/190848 objects degraded (12.729%), 15 pgs degraded (PG_DEGRADED) 2023-06-03T20:12:18.087 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:12:18.088 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:12:18.189 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:12:18.220 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:12:18.223 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:12:18.224 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:12:18.325 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:12:18.368 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:12:18.369 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:12:18.369 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:12:18.470 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:12:18.506 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:12:18.512 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:12:18.512 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:12:18.558 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 45 16 69604 69588 96.6327 67.8125 0.00369557 0.0103358 2023-06-03T20:12:18.613 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:12:18.651 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:12:18.653 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:12:18.654 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:12:18.754 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:12:18.787 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:12:18.788 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:12:18.788 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:12:18.889 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:12:18.921 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:12:18.923 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:12:18.924 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:12:19.025 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:12:19.061 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:12:19.066 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:12:19.067 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:12:19.167 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:12:19.204 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:12:19.213 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:12:19.214 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:12:19.315 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:12:19.349 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:12:19.353 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:12:19.354 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:12:19.456 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:12:19.487 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:12:19.489 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:12:19.489 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:12:19.558 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 46 16 70684 70668 95.9993 67.5 0.00790407 0.010411 2023-06-03T20:12:19.591 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:12:19.622 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:12:19.625 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:12:19.625 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:12:19.726 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:12:19.755 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:12:19.758 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:12:19.759 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:12:19.860 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:12:19.896 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:12:19.900 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:12:19.900 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:12:20.001 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:12:20.028 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:12:19 smithi089 bash[75068]: cluster 1685823138.6920326 mgr.y (mgr.34104) 1087 : cluster 0 pgmap v1056: 89 pgs: 15 active+undersized+degraded, 26 active+undersized, 48 active+clean; 4.1 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 84 MiB/s wr, 1.35k op/s; 24993/202842 objects degraded (12.321%) 2023-06-03T20:12:20.029 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:12:19 smithi089 bash[78452]: cluster 1685823138.6920326 mgr.y (mgr.34104) 1087 : cluster 0 pgmap v1056: 89 pgs: 15 active+undersized+degraded, 26 active+undersized, 48 active+clean; 4.1 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 84 MiB/s wr, 1.35k op/s; 24993/202842 objects degraded (12.321%) 2023-06-03T20:12:20.035 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:12:20.038 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:12:20.038 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:12:20.139 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:12:20.150 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:12:19 smithi164 bash[59462]: cluster 1685823138.6920326 mgr.y (mgr.34104) 1087 : cluster 0 pgmap v1056: 89 pgs: 15 active+undersized+degraded, 26 active+undersized, 48 active+clean; 4.1 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 84 MiB/s wr, 1.35k op/s; 24993/202842 objects degraded (12.321%) 2023-06-03T20:12:20.174 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:12:20.179 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:12:20.179 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:12:20.280 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:12:20.316 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:12:20.318 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:12:20.319 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:12:20.420 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:12:20.452 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:12:20.456 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:12:20.456 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:12:20.557 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:12:20.558 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 47 16 71668 71652 95.2652 61.5 0.00546647 0.0104915 2023-06-03T20:12:20.591 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:12:20.595 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:12:20.596 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:12:20.697 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:12:20.729 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:12:20.733 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:12:20.734 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:12:20.834 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:12:20.873 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:12:20.877 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:12:20.877 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:12:20.978 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:12:21.008 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:12:21.010 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:12:21.011 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:12:21.112 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:12:21.137 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:12:21.139 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:12:21.140 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:12:21.241 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:12:21.278 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:12:21.280 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:12:21.281 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:12:21.382 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:12:21.431 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:12:21.435 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:12:21.436 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:12:21.537 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:12:21.558 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 48 16 72750 72734 94.6893 67.625 0.0109004 0.010554 2023-06-03T20:12:21.568 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:12:21.574 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:12:21.575 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:12:21.675 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:12:21.715 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:12:21.717 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:12:21.718 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:12:21.819 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:12:21.857 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:12:21.862 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:12:21.863 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:12:21.964 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:12:21.972 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:12:21 smithi164 bash[59462]: cluster 1685823140.6927018 mgr.y (mgr.34104) 1088 : cluster 0 pgmap v1057: 89 pgs: 15 active+undersized+degraded, 26 active+undersized, 48 active+clean; 4.2 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 71 MiB/s wr, 1.13k op/s; 26076/207216 objects degraded (12.584%) 2023-06-03T20:12:21.997 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:12:22.003 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:12:22.004 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:12:22.105 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:12:22.115 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:12:21 smithi089 bash[78452]: cluster 1685823140.6927018 mgr.y (mgr.34104) 1088 : cluster 0 pgmap v1057: 89 pgs: 15 active+undersized+degraded, 26 active+undersized, 48 active+clean; 4.2 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 71 MiB/s wr, 1.13k op/s; 26076/207216 objects degraded (12.584%) 2023-06-03T20:12:22.116 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:12:21 smithi089 bash[75068]: cluster 1685823140.6927018 mgr.y (mgr.34104) 1088 : cluster 0 pgmap v1057: 89 pgs: 15 active+undersized+degraded, 26 active+undersized, 48 active+clean; 4.2 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 71 MiB/s wr, 1.13k op/s; 26076/207216 objects degraded (12.584%) 2023-06-03T20:12:22.142 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:12:22.147 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:12:22.148 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:12:22.166 INFO:teuthology.orchestra.run.smithi164.stderr:Inferring config /var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/mon.b/config 2023-06-03T20:12:22.248 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:12:22.271 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:12:22.273 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:12:22.274 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:12:22.374 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:12:22.403 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:12:22.405 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:12:22.406 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:12:22.507 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:12:22.536 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:12:22.540 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:12:22.541 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:12:22.558 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 49 16 73817 73801 94.1177 66.6875 0.0315732 0.0106193 2023-06-03T20:12:22.642 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:12:22.689 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:12:22.693 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:12:22.693 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:12:22.794 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:12:22.840 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:12:22.844 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:12:22.845 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:12:22.946 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:12:22.971 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:12:22.973 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:12:22.974 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:12:23.028 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:12:22 smithi089 bash[78452]: audit 1685823142.1187568 mon.c (mon.2) 506 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T20:12:23.029 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:12:22 smithi089 bash[78452]: cluster 1685823142.726791 mon.a (mon.0) 1027 : cluster 3 Health check update: Degraded data redundancy: 26076/207216 objects degraded (12.584%), 15 pgs degraded (PG_DEGRADED) 2023-06-03T20:12:23.029 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:12:22 smithi089 bash[75068]: audit 1685823142.1187568 mon.c (mon.2) 506 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T20:12:23.029 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:12:22 smithi089 bash[75068]: cluster 1685823142.726791 mon.a (mon.0) 1027 : cluster 3 Health check update: Degraded data redundancy: 26076/207216 objects degraded (12.584%), 15 pgs degraded (PG_DEGRADED) 2023-06-03T20:12:23.075 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:12:23.083 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:12:22 smithi164 bash[59462]: audit 1685823142.1187568 mon.c (mon.2) 506 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T20:12:23.083 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:12:22 smithi164 bash[59462]: cluster 1685823142.726791 mon.a (mon.0) 1027 : cluster 3 Health check update: Degraded data redundancy: 26076/207216 objects degraded (12.584%), 15 pgs degraded (PG_DEGRADED) 2023-06-03T20:12:23.113 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:12:23.115 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:12:23.116 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:12:23.217 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:12:23.251 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:12:23.254 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:12:23.255 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:12:23.356 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:12:23.397 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:12:23.400 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:12:23.401 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:12:23.501 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:12:23.530 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:12:23.532 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:12:23.533 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:12:23.558 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 50 16 74854 74838 93.5316 64.8125 0.0126288 0.0106868 2023-06-03T20:12:23.635 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:12:23.675 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:12:23.680 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:12:23.680 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:12:23.781 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:12:23.819 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:12:23.822 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:12:23.823 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:12:23.862 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:12:23 smithi089 bash[75068]: cluster 1685823142.6954098 mgr.y (mgr.34104) 1089 : cluster 0 pgmap v1058: 89 pgs: 15 active+undersized+degraded, 26 active+undersized, 48 active+clean; 4.3 GiB data, 14 GiB used, 611 GiB / 626 GiB avail; 72 MiB/s wr, 1.16k op/s; 26733/213276 objects degraded (12.534%) 2023-06-03T20:12:23.924 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:12:23.955 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:12:23.957 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:12:23.958 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:12:24.059 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:12:24.067 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:12:23 smithi164 bash[59462]: cluster 1685823142.6954098 mgr.y (mgr.34104) 1089 : cluster 0 pgmap v1058: 89 pgs: 15 active+undersized+degraded, 26 active+undersized, 48 active+clean; 4.3 GiB data, 14 GiB used, 611 GiB / 626 GiB avail; 72 MiB/s wr, 1.16k op/s; 26733/213276 objects degraded (12.534%) 2023-06-03T20:12:24.091 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:12:24.096 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:12:24.097 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:12:24.198 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:12:24.223 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:12:23 smithi089 bash[78452]: cluster 1685823142.6954098 mgr.y (mgr.34104) 1089 : cluster 0 pgmap v1058: 89 pgs: 15 active+undersized+degraded, 26 active+undersized, 48 active+clean; 4.3 GiB data, 14 GiB used, 611 GiB / 626 GiB avail; 72 MiB/s wr, 1.16k op/s; 26733/213276 objects degraded (12.534%) 2023-06-03T20:12:24.228 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:12:24.232 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:12:24.232 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:12:24.333 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:12:24.366 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:12:24.372 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:12:24.373 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:12:24.473 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:12:24.510 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:12:24.515 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:12:24.516 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:12:24.558 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 51 16 76062 76046 93.1779 75.5 0.011567 0.0107262 2023-06-03T20:12:24.616 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:12:24.653 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:12:24.655 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:12:24.655 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:12:24.757 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:12:24.787 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:12:24.790 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:12:24.791 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:12:24.892 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:12:24.930 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:12:24.933 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:12:24.933 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:12:25.035 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:12:25.064 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:12:25.069 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:12:25.069 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:12:25.171 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:12:25.212 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:12:25.215 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:12:25.216 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:12:25.317 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:12:25.350 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:12:25.355 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:12:25.356 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:12:25.457 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:12:25.493 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:12:25.496 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:12:25.496 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:12:25.558 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 52 16 77263 77247 92.8294 75.0625 0.0106603 0.0107659 2023-06-03T20:12:25.597 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:12:25.625 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:12:25.628 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:12:25.629 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:12:25.730 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:12:25.764 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:12:25.765 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:12:25.766 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:12:25.867 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:12:25.910 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:12:25.915 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:12:25.916 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:12:26.016 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:12:26.028 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:12:25 smithi089 bash[78452]: cluster 1685823144.6960514 mgr.y (mgr.34104) 1090 : cluster 0 pgmap v1059: 89 pgs: 15 active+undersized+degraded, 26 active+undersized, 48 active+clean; 4.5 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 73 MiB/s wr, 1.17k op/s; 27406/220260 objects degraded (12.443%) 2023-06-03T20:12:26.029 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:12:25 smithi089 bash[75068]: cluster 1685823144.6960514 mgr.y (mgr.34104) 1090 : cluster 0 pgmap v1059: 89 pgs: 15 active+undersized+degraded, 26 active+undersized, 48 active+clean; 4.5 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 73 MiB/s wr, 1.17k op/s; 27406/220260 objects degraded (12.443%) 2023-06-03T20:12:26.055 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:12:26.061 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:12:26.061 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:12:26.162 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:12:26.189 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:12:26.192 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:12:26.192 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:12:26.293 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:12:26.326 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:12:26.333 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:12:25 smithi164 bash[59462]: cluster 1685823144.6960514 mgr.y (mgr.34104) 1090 : cluster 0 pgmap v1059: 89 pgs: 15 active+undersized+degraded, 26 active+undersized, 48 active+clean; 4.5 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 73 MiB/s wr, 1.17k op/s; 27406/220260 objects degraded (12.443%) 2023-06-03T20:12:26.333 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:12:26.334 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:12:26.435 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:12:26.444 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 20:12:26 smithi089 bash[71163]: ::ffff:172.21.15.164 - - [03/Jun/2023:20:12:26] "GET /metrics HTTP/1.1" 200 228830 "" "Prometheus/2.33.4" 2023-06-03T20:12:26.466 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:12:26.469 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:12:26.469 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:12:26.559 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 53 16 78215 78199 92.2005 59.5 0.0121414 0.0108407 2023-06-03T20:12:26.570 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:12:26.607 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:12:26.615 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:12:26.616 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:12:26.717 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:12:26.754 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:12:26.759 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:12:26.760 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:12:26.861 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:12:26.901 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:12:26.908 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:12:26.909 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:12:27.009 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:12:27.043 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:12:27.045 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:12:27.046 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:12:27.147 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:12:27.180 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:12:27.183 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:12:27.184 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:12:27.285 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:12:27.313 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:12:27.315 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:12:27.316 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:12:27.417 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:12:27.452 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:12:27.456 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:12:27.457 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:12:27.558 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:12:27.559 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 54 16 79300 79284 91.7488 67.8125 0.0126234 0.0108931 2023-06-03T20:12:27.593 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:12:27.599 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:12:27.600 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:12:27.701 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:12:27.728 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:12:27.730 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:12:27.731 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:12:27.737 INFO:teuthology.orchestra.run.smithi164.stdout:{ 2023-06-03T20:12:27.737 INFO:teuthology.orchestra.run.smithi164.stdout: "ops": [ 2023-06-03T20:12:27.737 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:12:27.737 INFO:teuthology.orchestra.run.smithi164.stdout: "description": "osd_op(client.46187.0:79395 11.f 11:f4e2d72b:::benchmark_data_smithi089_165412_object79394:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected e197)", 2023-06-03T20:12:27.737 INFO:teuthology.orchestra.run.smithi164.stdout: "initiated_at": "2023-06-03T20:12:27.651105+0000", 2023-06-03T20:12:27.737 INFO:teuthology.orchestra.run.smithi164.stdout: "age": 0.078944049000000002, 2023-06-03T20:12:27.738 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.079008011000000003, 2023-06-03T20:12:27.738 INFO:teuthology.orchestra.run.smithi164.stdout: "type_data": { 2023-06-03T20:12:27.738 INFO:teuthology.orchestra.run.smithi164.stdout: "flag_point": "waiting for sub ops", 2023-06-03T20:12:27.738 INFO:teuthology.orchestra.run.smithi164.stdout: "client_info": { 2023-06-03T20:12:27.738 INFO:teuthology.orchestra.run.smithi164.stdout: "client": "client.46187", 2023-06-03T20:12:27.738 INFO:teuthology.orchestra.run.smithi164.stdout: "client_addr": "172.21.15.89:0/880684210", 2023-06-03T20:12:27.738 INFO:teuthology.orchestra.run.smithi164.stdout: "tid": 79395 2023-06-03T20:12:27.738 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:12:27.739 INFO:teuthology.orchestra.run.smithi164.stdout: "events": [ 2023-06-03T20:12:27.739 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:12:27.739 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "initiated", 2023-06-03T20:12:27.739 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:12:27.651105+0000", 2023-06-03T20:12:27.739 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0 2023-06-03T20:12:27.739 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:12:27.739 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:12:27.740 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "throttled", 2023-06-03T20:12:27.740 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:12:27.651105+0000", 2023-06-03T20:12:27.740 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 4294967295.9999986 2023-06-03T20:12:27.740 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:12:27.740 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:12:27.740 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "header_read", 2023-06-03T20:12:27.740 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:12:27.651103+0000", 2023-06-03T20:12:27.741 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 5.6657999999999999e-05 2023-06-03T20:12:27.741 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:12:27.741 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:12:27.741 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "all_read", 2023-06-03T20:12:27.741 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:12:27.651160+0000", 2023-06-03T20:12:27.741 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 9.961e-06 2023-06-03T20:12:27.741 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:12:27.741 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:12:27.742 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "dispatched", 2023-06-03T20:12:27.742 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:12:27.651170+0000", 2023-06-03T20:12:27.742 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 5.8839999999999999e-06 2023-06-03T20:12:27.742 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:12:27.742 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:12:27.742 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "queued_for_pg", 2023-06-03T20:12:27.742 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:12:27.651176+0000", 2023-06-03T20:12:27.742 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.002721944 2023-06-03T20:12:27.743 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:12:27.743 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:12:27.743 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "reached_pg", 2023-06-03T20:12:27.743 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:12:27.653898+0000", 2023-06-03T20:12:27.743 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.00014195899999999999 2023-06-03T20:12:27.743 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:12:27.743 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:12:27.743 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "started", 2023-06-03T20:12:27.744 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:12:27.654040+0000", 2023-06-03T20:12:27.744 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.00019181499999999999 2023-06-03T20:12:27.744 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:12:27.744 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:12:27.744 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "waiting for subops from 0,2", 2023-06-03T20:12:27.744 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:12:27.654231+0000", 2023-06-03T20:12:27.744 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.0025168090000000001 2023-06-03T20:12:27.745 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:12:27.745 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:12:27.745 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "op_commit", 2023-06-03T20:12:27.745 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:12:27.656748+0000", 2023-06-03T20:12:27.745 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.0010992739999999999 2023-06-03T20:12:27.745 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:12:27.745 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:12:27.746 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:12:27.746 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:12:27.657848+0000", 2023-06-03T20:12:27.746 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.0067429580000000003 2023-06-03T20:12:27.746 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:12:27.746 INFO:teuthology.orchestra.run.smithi164.stdout: ] 2023-06-03T20:12:27.747 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:12:27.747 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:12:27.747 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:12:27.747 INFO:teuthology.orchestra.run.smithi164.stdout: "description": "osd_op(client.46187.0:79405 11.4 11:2a6b8981:::benchmark_data_smithi089_165412_object79404:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected e197)", 2023-06-03T20:12:27.747 INFO:teuthology.orchestra.run.smithi164.stdout: "initiated_at": "2023-06-03T20:12:27.657719+0000", 2023-06-03T20:12:27.747 INFO:teuthology.orchestra.run.smithi164.stdout: "age": 0.072330106000000005, 2023-06-03T20:12:27.747 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.072575213, 2023-06-03T20:12:27.748 INFO:teuthology.orchestra.run.smithi164.stdout: "type_data": { 2023-06-03T20:12:27.748 INFO:teuthology.orchestra.run.smithi164.stdout: "flag_point": "waiting for sub ops", 2023-06-03T20:12:27.748 INFO:teuthology.orchestra.run.smithi164.stdout: "client_info": { 2023-06-03T20:12:27.748 INFO:teuthology.orchestra.run.smithi164.stdout: "client": "client.46187", 2023-06-03T20:12:27.748 INFO:teuthology.orchestra.run.smithi164.stdout: "client_addr": "172.21.15.89:0/880684210", 2023-06-03T20:12:27.748 INFO:teuthology.orchestra.run.smithi164.stdout: "tid": 79405 2023-06-03T20:12:27.748 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:12:27.749 INFO:teuthology.orchestra.run.smithi164.stdout: "events": [ 2023-06-03T20:12:27.749 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:12:27.749 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "initiated", 2023-06-03T20:12:27.749 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:12:27.657719+0000", 2023-06-03T20:12:27.749 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0 2023-06-03T20:12:27.749 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:12:27.749 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:12:27.750 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "throttled", 2023-06-03T20:12:27.750 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:12:27.657719+0000", 2023-06-03T20:12:27.750 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 4294967295.9999986 2023-06-03T20:12:27.750 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:12:27.750 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:12:27.750 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "header_read", 2023-06-03T20:12:27.750 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:12:27.657717+0000", 2023-06-03T20:12:27.750 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 8.7504999999999997e-05 2023-06-03T20:12:27.751 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:12:27.751 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:12:27.751 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "all_read", 2023-06-03T20:12:27.751 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:12:27.657805+0000", 2023-06-03T20:12:27.751 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 1.1603999999999999e-05 2023-06-03T20:12:27.751 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:12:27.751 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:12:27.751 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "dispatched", 2023-06-03T20:12:27.752 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:12:27.657816+0000", 2023-06-03T20:12:27.752 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 5.0340000000000004e-06 2023-06-03T20:12:27.752 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:12:27.752 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:12:27.752 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "queued_for_pg", 2023-06-03T20:12:27.752 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:12:27.657821+0000", 2023-06-03T20:12:27.752 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 4.8412000000000003e-05 2023-06-03T20:12:27.753 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:12:27.753 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:12:27.753 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "reached_pg", 2023-06-03T20:12:27.753 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:12:27.657870+0000", 2023-06-03T20:12:27.753 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.000153599 2023-06-03T20:12:27.753 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:12:27.753 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:12:27.753 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "started", 2023-06-03T20:12:27.754 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:12:27.658023+0000", 2023-06-03T20:12:27.754 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.00012569100000000001 2023-06-03T20:12:27.754 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:12:27.754 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:12:27.754 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "waiting for subops from 2,6", 2023-06-03T20:12:27.754 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:12:27.658149+0000", 2023-06-03T20:12:27.754 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.0055703749999999998 2023-06-03T20:12:27.754 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:12:27.755 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:12:27.755 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:12:27.755 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:12:27.663719+0000", 2023-06-03T20:12:27.755 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 6.0704e-05 2023-06-03T20:12:27.755 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:12:27.755 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:12:27.755 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "op_commit", 2023-06-03T20:12:27.755 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:12:27.663780+0000", 2023-06-03T20:12:27.756 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.0060615850000000004 2023-06-03T20:12:27.756 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:12:27.756 INFO:teuthology.orchestra.run.smithi164.stdout: ] 2023-06-03T20:12:27.756 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:12:27.756 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:12:27.756 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:12:27.756 INFO:teuthology.orchestra.run.smithi164.stdout: "description": "osd_op(client.46187.0:79406 11.4 11:24fb7738:::benchmark_data_smithi089_165412_object79405:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected e197)", 2023-06-03T20:12:27.756 INFO:teuthology.orchestra.run.smithi164.stdout: "initiated_at": "2023-06-03T20:12:27.657835+0000", 2023-06-03T20:12:27.757 INFO:teuthology.orchestra.run.smithi164.stdout: "age": 0.072213949999999999, 2023-06-03T20:12:27.757 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.072617701000000007, 2023-06-03T20:12:27.757 INFO:teuthology.orchestra.run.smithi164.stdout: "type_data": { 2023-06-03T20:12:27.757 INFO:teuthology.orchestra.run.smithi164.stdout: "flag_point": "waiting for sub ops", 2023-06-03T20:12:27.757 INFO:teuthology.orchestra.run.smithi164.stdout: "client_info": { 2023-06-03T20:12:27.757 INFO:teuthology.orchestra.run.smithi164.stdout: "client": "client.46187", 2023-06-03T20:12:27.757 INFO:teuthology.orchestra.run.smithi164.stdout: "client_addr": "172.21.15.89:0/880684210", 2023-06-03T20:12:27.758 INFO:teuthology.orchestra.run.smithi164.stdout: "tid": 79406 2023-06-03T20:12:27.758 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:12:27.758 INFO:teuthology.orchestra.run.smithi164.stdout: "events": [ 2023-06-03T20:12:27.758 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:12:27.758 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "initiated", 2023-06-03T20:12:27.758 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:12:27.657835+0000", 2023-06-03T20:12:27.758 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0 2023-06-03T20:12:27.759 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:12:27.759 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:12:27.759 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "throttled", 2023-06-03T20:12:27.759 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:12:27.657835+0000", 2023-06-03T20:12:27.759 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 4294967295.999999 2023-06-03T20:12:27.759 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:12:27.760 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:12:27.760 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "header_read", 2023-06-03T20:12:27.760 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:12:27.657834+0000", 2023-06-03T20:12:27.760 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 7.4196999999999995e-05 2023-06-03T20:12:27.760 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:12:27.760 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:12:27.760 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "all_read", 2023-06-03T20:12:27.761 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:12:27.657908+0000", 2023-06-03T20:12:27.761 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 9.7229999999999999e-06 2023-06-03T20:12:27.761 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:12:27.761 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:12:27.761 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "dispatched", 2023-06-03T20:12:27.761 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:12:27.657918+0000", 2023-06-03T20:12:27.761 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 5.2580000000000003e-06 2023-06-03T20:12:27.761 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:12:27.762 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:12:27.762 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "queued_for_pg", 2023-06-03T20:12:27.762 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:12:27.657923+0000", 2023-06-03T20:12:27.762 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.002546455 2023-06-03T20:12:27.762 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:12:27.762 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:12:27.762 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "reached_pg", 2023-06-03T20:12:27.762 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:12:27.660469+0000", 2023-06-03T20:12:27.763 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.00016783999999999999 2023-06-03T20:12:27.763 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:12:27.763 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:12:27.763 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "started", 2023-06-03T20:12:27.763 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:12:27.660637+0000", 2023-06-03T20:12:27.763 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.000184711 2023-06-03T20:12:27.763 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:12:27.764 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:12:27.764 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "waiting for subops from 2,6", 2023-06-03T20:12:27.764 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:12:27.660822+0000", 2023-06-03T20:12:27.764 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.0029491449999999998 2023-06-03T20:12:27.764 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:12:27.764 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:12:27.764 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:12:27.765 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:12:27.663771+0000", 2023-06-03T20:12:27.765 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 2.1250000000000002e-05 2023-06-03T20:12:27.765 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:12:27.765 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:12:27.765 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "op_commit", 2023-06-03T20:12:27.765 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:12:27.663792+0000", 2023-06-03T20:12:27.765 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.0059577140000000002 2023-06-03T20:12:27.765 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:12:27.766 INFO:teuthology.orchestra.run.smithi164.stdout: ] 2023-06-03T20:12:27.766 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:12:27.766 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:12:27.766 INFO:teuthology.orchestra.run.smithi164.stdout: ], 2023-06-03T20:12:27.766 INFO:teuthology.orchestra.run.smithi164.stdout: "num_ops": 3 2023-06-03T20:12:27.766 INFO:teuthology.orchestra.run.smithi164.stdout:} 2023-06-03T20:12:27.832 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:12:27.868 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:12:27.872 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:12:27.873 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:12:27.974 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:12:28.005 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:12:28.013 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:12:28.014 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:12:28.083 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:12:27 smithi164 bash[59462]: cluster 1685823146.696716 mgr.y (mgr.34104) 1091 : cluster 0 pgmap v1060: 89 pgs: 15 active+undersized+degraded, 26 active+undersized, 48 active+clean; 4.6 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 66 MiB/s wr, 1.06k op/s; 28453/224346 objects degraded (12.683%) 2023-06-03T20:12:28.083 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:12:27 smithi164 bash[59462]: cluster 1685823147.7281017 mon.a (mon.0) 1028 : cluster 3 Health check update: Degraded data redundancy: 28453/224346 objects degraded (12.683%), 15 pgs degraded (PG_DEGRADED) 2023-06-03T20:12:28.115 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:12:28.140 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:12:28.141 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:12:28.142 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:12:28.244 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:12:28.250 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:12:27 smithi089 bash[75068]: cluster 1685823146.696716 mgr.y (mgr.34104) 1091 : cluster 0 pgmap v1060: 89 pgs: 15 active+undersized+degraded, 26 active+undersized, 48 active+clean; 4.6 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 66 MiB/s wr, 1.06k op/s; 28453/224346 objects degraded (12.683%) 2023-06-03T20:12:28.250 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:12:27 smithi089 bash[75068]: cluster 1685823147.7281017 mon.a (mon.0) 1028 : cluster 3 Health check update: Degraded data redundancy: 28453/224346 objects degraded (12.683%), 15 pgs degraded (PG_DEGRADED) 2023-06-03T20:12:28.250 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:12:27 smithi089 bash[78452]: cluster 1685823146.696716 mgr.y (mgr.34104) 1091 : cluster 0 pgmap v1060: 89 pgs: 15 active+undersized+degraded, 26 active+undersized, 48 active+clean; 4.6 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 66 MiB/s wr, 1.06k op/s; 28453/224346 objects degraded (12.683%) 2023-06-03T20:12:28.250 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:12:27 smithi089 bash[78452]: cluster 1685823147.7281017 mon.a (mon.0) 1028 : cluster 3 Health check update: Degraded data redundancy: 28453/224346 objects degraded (12.683%), 15 pgs degraded (PG_DEGRADED) 2023-06-03T20:12:28.280 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:12:28.284 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:12:28.285 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:12:28.386 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:12:28.412 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:12:28.419 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:12:28.420 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:12:28.521 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:12:28.559 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 55 16 80078 80062 90.9646 48.625 0.012564 0.0109874 2023-06-03T20:12:28.560 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:12:28.562 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:12:28.562 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:12:28.664 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:12:28.694 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:12:28.696 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:12:28.697 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:12:28.777 DEBUG:teuthology.orchestra.run.smithi164:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph daemon osd.7 dump_blocked_ops 2023-06-03T20:12:28.798 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:12:28.864 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:12:28.870 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:12:28.871 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:12:28.971 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:12:29.006 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:12:29.008 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:12:29.009 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:12:29.110 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:12:29.146 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:12:29.152 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:12:29.153 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:12:29.254 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:12:29.296 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:12:29.301 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:12:29.301 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:12:29.402 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:12:29.437 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:12:29.440 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:12:29.441 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:12:29.542 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:12:29.559 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 56 16 81141 81125 90.5265 66.4375 0.0184565 0.0110408 2023-06-03T20:12:29.569 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:12:29.571 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:12:29.572 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:12:29.673 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:12:29.702 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:12:29.705 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:12:29.706 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:12:29.807 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:12:29.839 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:12:29.847 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:12:29.848 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:12:29.885 INFO:teuthology.orchestra.run.smithi089.stderr:reweighted osd.6 to 0.58042 (9496) 2023-06-03T20:12:29.891 INFO:teuthology.orchestra.run.smithi089.stdout:osd.0: {} 2023-06-03T20:12:29.931 INFO:teuthology.orchestra.run.smithi089.stdout:osd.1: {} 2023-06-03T20:12:29.948 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:12:29.964 INFO:teuthology.orchestra.run.smithi089.stdout:osd.2: {} 2023-06-03T20:12:29.977 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:12:29.984 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:12:29.986 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:12:29.992 INFO:teuthology.orchestra.run.smithi089.stdout:osd.3: {} 2023-06-03T20:12:30.020 INFO:teuthology.orchestra.run.smithi089.stdout:osd.5: {} 2023-06-03T20:12:30.043 INFO:teuthology.orchestra.run.smithi089.stdout:osd.6: {} 2023-06-03T20:12:30.069 INFO:teuthology.orchestra.run.smithi089.stdout:osd.7: {} 2023-06-03T20:12:30.069 INFO:teuthology.orchestra.run.smithi089.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-06-03T20:12:30.070 INFO:teuthology.orchestra.run.smithi089.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-06-03T20:12:30.070 INFO:teuthology.orchestra.run.smithi089.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-06-03T20:12:30.070 INFO:teuthology.orchestra.run.smithi089.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-06-03T20:12:30.070 INFO:teuthology.orchestra.run.smithi089.stderr:Error ENXIO: problem getting command descriptions from osd.4 2023-06-03T20:12:30.070 INFO:teuthology.orchestra.run.smithi089.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-06-03T20:12:30.070 INFO:teuthology.orchestra.run.smithi089.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-06-03T20:12:30.070 INFO:teuthology.orchestra.run.smithi089.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-06-03T20:12:30.083 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:12:29 smithi164 bash[59462]: cluster 1685823148.6973023 mgr.y (mgr.34104) 1092 : cluster 0 pgmap v1061: 89 pgs: 15 active+undersized+degraded, 26 active+undersized, 48 active+clean; 4.8 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 77 MiB/s wr, 1.24k op/s; 29152/235404 objects degraded (12.384%) 2023-06-03T20:12:30.083 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:12:29 smithi164 bash[59462]: audit 1685823149.6746771 mon.a (mon.0) 1029 : audit 1 from='client.? 172.21.15.89:0/2901378872' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 0.5804197722268256}]: dispatch 2023-06-03T20:12:30.083 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:12:29 smithi164 bash[59462]: audit 1685823149.8307066 mon.a (mon.0) 1030 : audit 0 from='client.? 172.21.15.89:0/1462624624' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-03T20:12:30.087 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:12:30.112 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:12:30.114 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:12:30.114 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:12:30.215 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:12:30.222 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:12:29 smithi089 bash[78452]: cluster 1685823148.6973023 mgr.y (mgr.34104) 1092 : cluster 0 pgmap v1061: 89 pgs: 15 active+undersized+degraded, 26 active+undersized, 48 active+clean; 4.8 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 77 MiB/s wr, 1.24k op/s; 29152/235404 objects degraded (12.384%) 2023-06-03T20:12:30.222 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:12:29 smithi089 bash[78452]: audit 1685823149.6746771 mon.a (mon.0) 1029 : audit 1 from='client.? 172.21.15.89:0/2901378872' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 0.5804197722268256}]: dispatch 2023-06-03T20:12:30.222 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:12:29 smithi089 bash[78452]: audit 1685823149.8307066 mon.a (mon.0) 1030 : audit 0 from='client.? 172.21.15.89:0/1462624624' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-03T20:12:30.223 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:12:29 smithi089 bash[75068]: cluster 1685823148.6973023 mgr.y (mgr.34104) 1092 : cluster 0 pgmap v1061: 89 pgs: 15 active+undersized+degraded, 26 active+undersized, 48 active+clean; 4.8 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 77 MiB/s wr, 1.24k op/s; 29152/235404 objects degraded (12.384%) 2023-06-03T20:12:30.223 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:12:29 smithi089 bash[75068]: audit 1685823149.6746771 mon.a (mon.0) 1029 : audit 1 from='client.? 172.21.15.89:0/2901378872' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 0.5804197722268256}]: dispatch 2023-06-03T20:12:30.223 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:12:29 smithi089 bash[75068]: audit 1685823149.8307066 mon.a (mon.0) 1030 : audit 0 from='client.? 172.21.15.89:0/1462624624' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-03T20:12:30.240 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:12:30.242 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:12:30.243 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:12:30.344 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:12:30.366 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:12:30.368 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:12:30.368 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:12:30.469 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:12:30.502 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:12:30.505 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:12:30.505 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:12:30.559 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 57 16 81533 81517 89.3681 24.5 0.00640673 0.0110651 2023-06-03T20:12:30.606 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:12:30.641 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:12:30.644 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:12:30.644 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:12:30.745 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:12:30.780 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:12:30.782 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:12:30.783 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:12:30.884 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:12:30.912 INFO:teuthology.orchestra.run.smithi089.stderr:noscrub is unset 2023-06-03T20:12:30.921 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:12:30.923 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:12:30.924 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:12:30.937 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:12:30 smithi089 bash[75068]: cluster 1685823149.875884 osd.5 (osd.5) 131 : cluster 3 failed to encode map e198 with expected crc 2023-06-03T20:12:30.937 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:12:30 smithi089 bash[75068]: audit 1685823149.8760672 mon.a (mon.0) 1031 : audit 1 from='client.? 172.21.15.89:0/2901378872' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 0.5804197722268256}]': finished 2023-06-03T20:12:30.937 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:12:30 smithi089 bash[75068]: cluster 1685823149.8762503 mon.a (mon.0) 1032 : cluster 0 osdmap e198: 8 total, 7 up, 7 in 2023-06-03T20:12:30.937 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:12:30 smithi089 bash[75068]: audit 1685823150.2420685 mon.a (mon.0) 1033 : audit 1 from='client.? 172.21.15.89:0/831988679' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-06-03T20:12:30.937 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:12:30 smithi089 bash[78452]: cluster 1685823149.875884 osd.5 (osd.5) 131 : cluster 3 failed to encode map e198 with expected crc 2023-06-03T20:12:30.938 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:12:30 smithi089 bash[78452]: audit 1685823149.8760672 mon.a (mon.0) 1031 : audit 1 from='client.? 172.21.15.89:0/2901378872' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 0.5804197722268256}]': finished 2023-06-03T20:12:30.938 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:12:30 smithi089 bash[78452]: cluster 1685823149.8762503 mon.a (mon.0) 1032 : cluster 0 osdmap e198: 8 total, 7 up, 7 in 2023-06-03T20:12:30.938 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:12:30 smithi089 bash[78452]: audit 1685823150.2420685 mon.a (mon.0) 1033 : audit 1 from='client.? 172.21.15.89:0/831988679' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-06-03T20:12:31.025 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:12:31.057 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:12:31.060 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:12:31.060 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:12:31.161 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:12:31.194 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:12:31.196 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:12:31.197 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:12:31.298 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:12:31.331 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:12:31.333 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:12:30 smithi164 bash[59462]: cluster 1685823149.875884 osd.5 (osd.5) 131 : cluster 3 failed to encode map e198 with expected crc 2023-06-03T20:12:31.333 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:12:30 smithi164 bash[59462]: audit 1685823149.8760672 mon.a (mon.0) 1031 : audit 1 from='client.? 172.21.15.89:0/2901378872' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 0.5804197722268256}]': finished 2023-06-03T20:12:31.333 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:12:30 smithi164 bash[59462]: cluster 1685823149.8762503 mon.a (mon.0) 1032 : cluster 0 osdmap e198: 8 total, 7 up, 7 in 2023-06-03T20:12:31.334 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:12:30 smithi164 bash[59462]: audit 1685823150.2420685 mon.a (mon.0) 1033 : audit 1 from='client.? 172.21.15.89:0/831988679' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-06-03T20:12:31.334 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:12:31.335 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:12:31.435 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:12:31.478 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:12:31.480 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:12:31.481 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:12:31.559 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 58 16 81533 81517 87.8274 0 - 0.0110651 2023-06-03T20:12:31.582 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:12:31.618 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:12:31.620 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:12:31.621 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:12:31.722 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:12:31.762 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:12:31.764 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:12:31.765 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:12:31.866 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:12:31.896 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:12:31.899 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:12:31.899 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:12:32.000 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:12:32.024 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:12:32.026 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:12:32.027 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:12:32.083 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:12:31 smithi164 bash[59462]: cluster 1685823150.6979506 mgr.y (mgr.34104) 1093 : cluster 0 pgmap v1063: 89 pgs: 15 active+undersized+degraded, 26 active+undersized, 48 active+clean; 4.9 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 67 MiB/s wr, 1.07k op/s; 30135/239319 objects degraded (12.592%) 2023-06-03T20:12:32.083 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:12:31 smithi164 bash[59462]: cluster 1685823150.8849282 mon.a (mon.0) 1034 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-06-03T20:12:32.083 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:12:31 smithi164 bash[59462]: cluster 1685823150.893047 osd.6 (osd.6) 145 : cluster 3 failed to encode map e199 with expected crc 2023-06-03T20:12:32.084 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:12:31 smithi164 bash[59462]: cluster 1685823150.895236 osd.7 (osd.7) 128 : cluster 3 failed to encode map e199 with expected crc 2023-06-03T20:12:32.084 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:12:31 smithi164 bash[59462]: audit 1685823150.8968675 mon.a (mon.0) 1035 : audit 1 from='client.? 172.21.15.89:0/831988679' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-06-03T20:12:32.084 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:12:31 smithi164 bash[59462]: cluster 1685823150.8969624 mon.a (mon.0) 1036 : cluster 0 osdmap e199: 8 total, 7 up, 7 in 2023-06-03T20:12:32.084 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:12:31 smithi164 bash[59462]: cluster 1685823150.8988898 osd.5 (osd.5) 132 : cluster 3 failed to encode map e199 with expected crc 2023-06-03T20:12:32.127 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:12:32.157 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:12:32.161 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:12:32.162 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:12:32.263 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:12:32.271 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:12:31 smithi089 bash[78452]: cluster 1685823150.6979506 mgr.y (mgr.34104) 1093 : cluster 0 pgmap v1063: 89 pgs: 15 active+undersized+degraded, 26 active+undersized, 48 active+clean; 4.9 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 67 MiB/s wr, 1.07k op/s; 30135/239319 objects degraded (12.592%) 2023-06-03T20:12:32.271 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:12:31 smithi089 bash[78452]: cluster 1685823150.8849282 mon.a (mon.0) 1034 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-06-03T20:12:32.271 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:12:31 smithi089 bash[78452]: cluster 1685823150.893047 osd.6 (osd.6) 145 : cluster 3 failed to encode map e199 with expected crc 2023-06-03T20:12:32.271 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:12:31 smithi089 bash[78452]: cluster 1685823150.895236 osd.7 (osd.7) 128 : cluster 3 failed to encode map e199 with expected crc 2023-06-03T20:12:32.272 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:12:31 smithi089 bash[78452]: audit 1685823150.8968675 mon.a (mon.0) 1035 : audit 1 from='client.? 172.21.15.89:0/831988679' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-06-03T20:12:32.272 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:12:31 smithi089 bash[78452]: cluster 1685823150.8969624 mon.a (mon.0) 1036 : cluster 0 osdmap e199: 8 total, 7 up, 7 in 2023-06-03T20:12:32.272 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:12:31 smithi089 bash[78452]: cluster 1685823150.8988898 osd.5 (osd.5) 132 : cluster 3 failed to encode map e199 with expected crc 2023-06-03T20:12:32.272 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:12:31 smithi089 bash[75068]: cluster 1685823150.6979506 mgr.y (mgr.34104) 1093 : cluster 0 pgmap v1063: 89 pgs: 15 active+undersized+degraded, 26 active+undersized, 48 active+clean; 4.9 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 67 MiB/s wr, 1.07k op/s; 30135/239319 objects degraded (12.592%) 2023-06-03T20:12:32.272 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:12:31 smithi089 bash[75068]: cluster 1685823150.8849282 mon.a (mon.0) 1034 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-06-03T20:12:32.272 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:12:31 smithi089 bash[75068]: cluster 1685823150.893047 osd.6 (osd.6) 145 : cluster 3 failed to encode map e199 with expected crc 2023-06-03T20:12:32.273 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:12:31 smithi089 bash[75068]: cluster 1685823150.895236 osd.7 (osd.7) 128 : cluster 3 failed to encode map e199 with expected crc 2023-06-03T20:12:32.273 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:12:31 smithi089 bash[75068]: audit 1685823150.8968675 mon.a (mon.0) 1035 : audit 1 from='client.? 172.21.15.89:0/831988679' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-06-03T20:12:32.273 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:12:31 smithi089 bash[75068]: cluster 1685823150.8969624 mon.a (mon.0) 1036 : cluster 0 osdmap e199: 8 total, 7 up, 7 in 2023-06-03T20:12:32.273 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:12:31 smithi089 bash[75068]: cluster 1685823150.8988898 osd.5 (osd.5) 132 : cluster 3 failed to encode map e199 with expected crc 2023-06-03T20:12:32.291 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:12:32.294 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:12:32.295 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:12:32.395 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:12:32.432 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:12:32.436 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:12:32.437 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:12:32.538 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:12:32.559 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 59 16 81844 81828 86.6683 9.71875 0.0424794 0.0115321 2023-06-03T20:12:32.570 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:12:32.574 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:12:32.574 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:12:32.675 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:12:32.707 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:12:32.710 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:12:32.711 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:12:32.812 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:12:32.851 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:12:32.853 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:12:32.854 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:12:32.955 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:12:32.965 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:12:32 smithi089 bash[78452]: cluster 1685823151.893969 osd.7 (osd.7) 129 : cluster 3 failed to encode map e200 with expected crc 2023-06-03T20:12:32.965 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:12:32 smithi089 bash[78452]: cluster 1685823151.8964403 osd.7 (osd.7) 130 : cluster 3 failed to encode map e200 with expected crc 2023-06-03T20:12:32.965 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:12:32 smithi089 bash[78452]: cluster 1685823151.8980203 mon.a (mon.0) 1037 : cluster 0 osdmap e200: 8 total, 7 up, 7 in 2023-06-03T20:12:32.966 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:12:32 smithi089 bash[78452]: cluster 1685823151.8980222 osd.5 (osd.5) 133 : cluster 3 failed to encode map e200 with expected crc 2023-06-03T20:12:32.966 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:12:32 smithi089 bash[78452]: cluster 1685823152.7289777 mon.a (mon.0) 1038 : cluster 3 Health check update: Degraded data redundancy: 30135/239319 objects degraded (12.592%), 15 pgs degraded (PG_DEGRADED) 2023-06-03T20:12:32.966 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:12:32 smithi089 bash[75068]: cluster 1685823151.893969 osd.7 (osd.7) 129 : cluster 3 failed to encode map e200 with expected crc 2023-06-03T20:12:32.966 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:12:32 smithi089 bash[75068]: cluster 1685823151.8964403 osd.7 (osd.7) 130 : cluster 3 failed to encode map e200 with expected crc 2023-06-03T20:12:32.966 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:12:32 smithi089 bash[75068]: cluster 1685823151.8980203 mon.a (mon.0) 1037 : cluster 0 osdmap e200: 8 total, 7 up, 7 in 2023-06-03T20:12:32.966 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:12:32 smithi089 bash[75068]: cluster 1685823151.8980222 osd.5 (osd.5) 133 : cluster 3 failed to encode map e200 with expected crc 2023-06-03T20:12:32.967 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:12:32 smithi089 bash[75068]: cluster 1685823152.7289777 mon.a (mon.0) 1038 : cluster 3 Health check update: Degraded data redundancy: 30135/239319 objects degraded (12.592%), 15 pgs degraded (PG_DEGRADED) 2023-06-03T20:12:32.986 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:12:32.989 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:12:32.990 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:12:33.090 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:12:33.142 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:12:33.145 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:12:33.145 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:12:33.246 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:12:33.254 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:12:32 smithi164 bash[59462]: cluster 1685823151.893969 osd.7 (osd.7) 129 : cluster 3 failed to encode map e200 with expected crc 2023-06-03T20:12:33.254 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:12:32 smithi164 bash[59462]: cluster 1685823151.8964403 osd.7 (osd.7) 130 : cluster 3 failed to encode map e200 with expected crc 2023-06-03T20:12:33.254 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:12:32 smithi164 bash[59462]: cluster 1685823151.8980203 mon.a (mon.0) 1037 : cluster 0 osdmap e200: 8 total, 7 up, 7 in 2023-06-03T20:12:33.255 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:12:32 smithi164 bash[59462]: cluster 1685823151.8980222 osd.5 (osd.5) 133 : cluster 3 failed to encode map e200 with expected crc 2023-06-03T20:12:33.255 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:12:32 smithi164 bash[59462]: cluster 1685823152.7289777 mon.a (mon.0) 1038 : cluster 3 Health check update: Degraded data redundancy: 30135/239319 objects degraded (12.592%), 15 pgs degraded (PG_DEGRADED) 2023-06-03T20:12:33.275 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:12:33.277 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:12:33.278 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:12:33.322 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-06-03T20:12:33.323 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-06-03T20:12:33.378 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:12:33.415 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:12:33.417 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:12:33.417 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:12:33.518 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:12:33.553 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:12:33.558 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:12:33.559 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:12:33.559 INFO:tasks.radosbench.radosbench.0.smithi089.stdout:2023-06-03T20:12:33.555946+0000 min lat: 0.00135638 max lat: 2.42958 avg lat: 0.0116082 2023-06-03T20:12:33.559 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-06-03T20:12:33.559 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 60 16 82687 82671 86.1019 52.6875 0.0202883 0.0116082 2023-06-03T20:12:33.626 INFO:teuthology.orchestra.run.smithi164.stderr:Inferring config /var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/mon.b/config 2023-06-03T20:12:33.659 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:12:33.694 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:12:33.698 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:12:33.698 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:12:33.799 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:12:33.832 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:12:33.834 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:12:33.835 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:12:33.936 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:12:33.966 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:12:33.968 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:12:33.969 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:12:34.069 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:12:34.078 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:12:33 smithi089 bash[78452]: cluster 1685823152.6986656 mgr.y (mgr.34104) 1094 : cluster 0 pgmap v1066: 89 pgs: 5 active+clean+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 15 active+undersized+degraded, 1 active+recovering+undersized+remapped, 21 active+undersized, 46 active+clean; 4.9 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 63 MiB/s wr, 1.00k op/s; 35710/242445 objects degraded (14.729%); 5044/242445 objects misplaced (2.080%); 64 KiB/s, 0 objects/s recovering 2023-06-03T20:12:34.078 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:12:33 smithi089 bash[75068]: cluster 1685823152.6986656 mgr.y (mgr.34104) 1094 : cluster 0 pgmap v1066: 89 pgs: 5 active+clean+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 15 active+undersized+degraded, 1 active+recovering+undersized+remapped, 21 active+undersized, 46 active+clean; 4.9 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 63 MiB/s wr, 1.00k op/s; 35710/242445 objects degraded (14.729%); 5044/242445 objects misplaced (2.080%); 64 KiB/s, 0 objects/s recovering 2023-06-03T20:12:34.083 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:12:33 smithi164 bash[59462]: cluster 1685823152.6986656 mgr.y (mgr.34104) 1094 : cluster 0 pgmap v1066: 89 pgs: 5 active+clean+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 15 active+undersized+degraded, 1 active+recovering+undersized+remapped, 21 active+undersized, 46 active+clean; 4.9 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 63 MiB/s wr, 1.00k op/s; 35710/242445 objects degraded (14.729%); 5044/242445 objects misplaced (2.080%); 64 KiB/s, 0 objects/s recovering 2023-06-03T20:12:34.109 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:12:34.113 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:12:34.113 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:12:34.214 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:12:34.247 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:12:34.253 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:12:34.253 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:12:34.354 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:12:34.404 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:12:34.407 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:12:34.408 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:12:34.508 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:12:34.549 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:12:34.553 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:12:34.554 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:12:34.559 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 61 16 83531 83515 85.555 52.75 0.0761887 0.0116816 2023-06-03T20:12:34.655 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:12:34.684 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:12:34.686 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:12:34.686 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:12:34.787 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:12:34.826 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:12:34.830 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:12:34.831 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:12:34.932 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:12:34.972 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:12:34.975 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:12:34.975 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:12:35.076 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:12:35.116 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:12:35.119 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:12:35.119 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:12:35.220 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:12:35.253 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:12:35.257 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:12:35.258 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:12:35.324 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-06-03T20:12:35.359 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:12:35.395 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:12:35.398 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:12:35.399 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:12:35.414 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph osd unset nodeep-scrub 2023-06-03T20:12:35.500 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:12:35.547 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:12:35.555 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:12:35.556 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:12:35.561 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 62 16 84334 84318 84.982 50.1875 0.0383318 0.0117595 2023-06-03T20:12:35.657 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:12:35.703 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:12:35.707 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:12:35.708 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:12:35.809 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:12:35.853 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:12:35.854 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:12:35.855 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:12:35.956 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:12:35.966 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:12:35 smithi089 bash[78452]: cluster 1685823154.6994257 mgr.y (mgr.34104) 1095 : cluster 0 pgmap v1067: 89 pgs: 9 active+clean+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 13 active+undersized+degraded, 2 active+recovering+undersized+remapped, 20 active+undersized, 44 active+clean; 5.0 GiB data, 16 GiB used, 609 GiB / 626 GiB avail; 36 MiB/s wr, 577 op/s; 36020/245805 objects degraded (14.654%); 10159/245805 objects misplaced (4.133%); 192 KiB/s, 3 objects/s recovering 2023-06-03T20:12:35.966 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:12:35 smithi089 bash[75068]: cluster 1685823154.6994257 mgr.y (mgr.34104) 1095 : cluster 0 pgmap v1067: 89 pgs: 9 active+clean+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 13 active+undersized+degraded, 2 active+recovering+undersized+remapped, 20 active+undersized, 44 active+clean; 5.0 GiB data, 16 GiB used, 609 GiB / 626 GiB avail; 36 MiB/s wr, 577 op/s; 36020/245805 objects degraded (14.654%); 10159/245805 objects misplaced (4.133%); 192 KiB/s, 3 objects/s recovering 2023-06-03T20:12:35.998 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:12:36.002 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:12:36.003 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:12:36.083 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:12:35 smithi164 bash[59462]: cluster 1685823154.6994257 mgr.y (mgr.34104) 1095 : cluster 0 pgmap v1067: 89 pgs: 9 active+clean+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 13 active+undersized+degraded, 2 active+recovering+undersized+remapped, 20 active+undersized, 44 active+clean; 5.0 GiB data, 16 GiB used, 609 GiB / 626 GiB avail; 36 MiB/s wr, 577 op/s; 36020/245805 objects degraded (14.654%); 10159/245805 objects misplaced (4.133%); 192 KiB/s, 3 objects/s recovering 2023-06-03T20:12:36.104 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:12:36.136 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:12:36.137 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:12:36.138 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:12:36.230 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 20:12:36 smithi089 bash[71163]: ::ffff:172.21.15.164 - - [03/Jun/2023:20:12:36] "GET /metrics HTTP/1.1" 200 228830 "" "Prometheus/2.33.4" 2023-06-03T20:12:36.239 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:12:36.266 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:12:36.269 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:12:36.270 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:12:36.370 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:12:36.400 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:12:36.406 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:12:36.407 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:12:36.508 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:12:36.536 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:12:36.540 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:12:36.541 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:12:36.562 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 63 16 85131 85115 84.4237 49.8125 0.0600781 0.0118387 2023-06-03T20:12:36.642 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:12:36.678 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:12:36.683 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:12:36.684 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:12:36.785 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:12:36.821 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:12:36.822 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:12:36.823 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:12:36.924 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:12:36.957 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:12:36.959 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:12:36.960 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:12:37.061 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:12:37.098 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:12:37.102 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:12:37.103 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:12:37.204 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:12:37.244 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:12:37.248 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:12:37.248 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:12:37.349 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:12:37.381 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:12:37.382 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:12:37.383 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:12:37.484 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:12:37.522 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:12:37.530 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:12:37.531 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:12:37.562 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 64 16 85947 85931 83.9015 51 0.00722648 0.0119114 2023-06-03T20:12:37.632 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:12:37.660 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:12:37.663 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:12:37.663 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:12:37.764 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:12:37.806 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:12:37.809 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:12:37.809 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:12:37.910 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:12:37.938 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:12:37.942 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:12:37.942 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:12:38.043 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:12:38.077 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:12:38.082 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:12:38.083 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:12:38.083 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:12:37 smithi164 bash[59462]: cluster 1685823156.7001603 mgr.y (mgr.34104) 1096 : cluster 0 pgmap v1068: 89 pgs: 9 active+clean+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 13 active+undersized+degraded, 2 active+recovering+undersized+remapped, 20 active+undersized, 44 active+clean; 5.0 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 25 MiB/s wr, 407 op/s; 36490/247668 objects degraded (14.733%); 10159/247668 objects misplaced (4.102%); 169 KiB/s, 2 objects/s recovering 2023-06-03T20:12:38.084 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:12:37 smithi164 bash[59462]: audit 1685823157.1188896 mon.c (mon.2) 507 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T20:12:38.084 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:12:37 smithi164 bash[59462]: cluster 1685823157.7303393 mon.a (mon.0) 1039 : cluster 3 Health check update: Degraded data redundancy: 36490/247668 objects degraded (14.733%), 14 pgs degraded (PG_DEGRADED) 2023-06-03T20:12:38.183 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:12:38.219 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:12:38.224 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:12:38.225 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:12:38.278 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:12:37 smithi089 bash[78452]: cluster 1685823156.7001603 mgr.y (mgr.34104) 1096 : cluster 0 pgmap v1068: 89 pgs: 9 active+clean+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 13 active+undersized+degraded, 2 active+recovering+undersized+remapped, 20 active+undersized, 44 active+clean; 5.0 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 25 MiB/s wr, 407 op/s; 36490/247668 objects degraded (14.733%); 10159/247668 objects misplaced (4.102%); 169 KiB/s, 2 objects/s recovering 2023-06-03T20:12:38.279 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:12:37 smithi089 bash[78452]: audit 1685823157.1188896 mon.c (mon.2) 507 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T20:12:38.279 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:12:37 smithi089 bash[78452]: cluster 1685823157.7303393 mon.a (mon.0) 1039 : cluster 3 Health check update: Degraded data redundancy: 36490/247668 objects degraded (14.733%), 14 pgs degraded (PG_DEGRADED) 2023-06-03T20:12:38.279 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:12:37 smithi089 bash[75068]: cluster 1685823156.7001603 mgr.y (mgr.34104) 1096 : cluster 0 pgmap v1068: 89 pgs: 9 active+clean+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 13 active+undersized+degraded, 2 active+recovering+undersized+remapped, 20 active+undersized, 44 active+clean; 5.0 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 25 MiB/s wr, 407 op/s; 36490/247668 objects degraded (14.733%); 10159/247668 objects misplaced (4.102%); 169 KiB/s, 2 objects/s recovering 2023-06-03T20:12:38.279 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:12:37 smithi089 bash[75068]: audit 1685823157.1188896 mon.c (mon.2) 507 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T20:12:38.279 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:12:37 smithi089 bash[75068]: cluster 1685823157.7303393 mon.a (mon.0) 1039 : cluster 3 Health check update: Degraded data redundancy: 36490/247668 objects degraded (14.733%), 14 pgs degraded (PG_DEGRADED) 2023-06-03T20:12:38.298 INFO:tasks.thrashosds.thrasher:in_osds: [0, 2, 3, 4, 5, 6, 7] out_osds: [1] dead_osds: [4] live_osds: [0, 1, 2, 3, 5, 6, 7] 2023-06-03T20:12:38.298 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-06-03T20:12:38.298 INFO:tasks.thrashosds.thrasher:primary_affinity 2023-06-03T20:12:38.298 INFO:tasks.thrashosds.thrasher:Setting osd 3 primary_affinity to 1.000000 2023-06-03T20:12:38.299 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph osd primary-affinity 3 1 2023-06-03T20:12:38.325 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:12:38.385 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:12:38.392 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:12:38.393 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:12:38.494 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:12:38.545 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:12:38.555 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:12:38.556 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:12:38.562 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 65 16 86687 86671 83.3222 46.25 0.0176997 0.011993 2023-06-03T20:12:38.657 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:12:38.712 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:12:38.715 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:12:38.716 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:12:38.817 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:12:38.850 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:12:38.853 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:12:38.853 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:12:38.954 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:12:38.992 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:12:38.996 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:12:38.996 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:12:39.097 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:12:39.125 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:12:39.129 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:12:39.130 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:12:39.231 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:12:39.265 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:12:39.269 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:12:39.270 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:12:39.371 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:12:39.406 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:12:39.409 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:12:39.409 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:12:39.510 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:12:39.549 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:12:39.551 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:12:39.552 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:12:39.562 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 66 16 87494 87478 82.8239 50.4375 0.0246978 0.0120675 2023-06-03T20:12:39.653 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:12:39.704 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:12:39.705 INFO:teuthology.orchestra.run.smithi164.stdout:ERROR: (22) Invalid argument 2023-06-03T20:12:39.705 INFO:teuthology.orchestra.run.smithi164.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-06-03T20:12:39.708 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:12:39.709 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:12:39.809 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:12:39.836 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:12:39.838 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:12:39.838 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:12:39.939 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:12:39.972 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:12:39.974 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:12:39.975 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:12:40.075 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:12:40.083 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:12:39 smithi164 bash[59462]: cluster 1685823158.7009819 mgr.y (mgr.34104) 1097 : cluster 0 pgmap v1069: 89 pgs: 9 active+clean+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 13 active+undersized+degraded, 2 active+recovering+undersized+remapped, 20 active+undersized, 44 active+clean; 5.2 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 44 MiB/s wr, 701 op/s; 37040/256155 objects degraded (14.460%); 9971/256155 objects misplaced (3.893%); 1.6 MiB/s, 25 objects/s recovering 2023-06-03T20:12:40.085 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:12:39 smithi089 bash[78452]: cluster 1685823158.7009819 mgr.y (mgr.34104) 1097 : cluster 0 pgmap v1069: 89 pgs: 9 active+clean+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 13 active+undersized+degraded, 2 active+recovering+undersized+remapped, 20 active+undersized, 44 active+clean; 5.2 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 44 MiB/s wr, 701 op/s; 37040/256155 objects degraded (14.460%); 9971/256155 objects misplaced (3.893%); 1.6 MiB/s, 25 objects/s recovering 2023-06-03T20:12:40.085 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:12:39 smithi089 bash[75068]: cluster 1685823158.7009819 mgr.y (mgr.34104) 1097 : cluster 0 pgmap v1069: 89 pgs: 9 active+clean+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 13 active+undersized+degraded, 2 active+recovering+undersized+remapped, 20 active+undersized, 44 active+clean; 5.2 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 44 MiB/s wr, 701 op/s; 37040/256155 objects degraded (14.460%); 9971/256155 objects misplaced (3.893%); 1.6 MiB/s, 25 objects/s recovering 2023-06-03T20:12:40.106 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:12:40.109 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:12:40.110 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:12:40.211 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:12:40.241 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:12:40.247 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:12:40.247 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:12:40.250 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/mon.a/config 2023-06-03T20:12:40.252 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/mon.a/config 2023-06-03T20:12:40.348 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:12:40.385 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:12:40.390 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:12:40.391 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:12:40.492 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:12:40.524 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:12:40.528 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:12:40.529 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:12:40.562 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 67 16 88276 88260 82.3172 48.875 0.0130375 0.0121422 2023-06-03T20:12:40.630 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:12:40.677 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:12:40.682 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:12:40.683 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:12:40.784 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:12:40.819 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:12:40.825 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:12:40.826 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:12:40.858 DEBUG:teuthology.orchestra.run.smithi164:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph daemon osd.7 dump_historic_ops 2023-06-03T20:12:40.927 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:12:40.983 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:12:40.989 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:12:40.990 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:12:41.091 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:12:41.128 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:12:41.133 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:12:41.133 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:12:41.234 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:12:41.276 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:12:41.279 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:12:41.280 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:12:41.380 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:12:41.419 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:12:41.421 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:12:41.422 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:12:41.522 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:12:41.558 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:12:41.562 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:12:41.563 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:12:41.563 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 68 16 89036 89020 81.8051 47.5 0.00934366 0.0122145 2023-06-03T20:12:41.664 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:12:41.703 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:12:41.709 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:12:41.710 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:12:41.810 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:12:41.847 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:12:41.852 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:12:41.852 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:12:41.934 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:12:41 smithi089 bash[75068]: cluster 1685823160.7017066 mgr.y (mgr.34104) 1098 : cluster 0 pgmap v1070: 89 pgs: 9 active+clean+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 13 active+undersized+degraded, 2 active+recovering+undersized+remapped, 20 active+undersized, 44 active+clean; 5.3 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 41 MiB/s wr, 648 op/s; 37782/259128 objects degraded (14.580%); 9971/259128 objects misplaced (3.848%); 1.3 MiB/s, 21 objects/s recovering 2023-06-03T20:12:41.953 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:12:41.986 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:12:41.992 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:12:41.993 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:12:42.093 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:12:42.125 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:12:42.126 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:12:42.127 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:12:42.228 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:12:42.239 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:12:41 smithi089 bash[78452]: cluster 1685823160.7017066 mgr.y (mgr.34104) 1098 : cluster 0 pgmap v1070: 89 pgs: 9 active+clean+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 13 active+undersized+degraded, 2 active+recovering+undersized+remapped, 20 active+undersized, 44 active+clean; 5.3 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 41 MiB/s wr, 648 op/s; 37782/259128 objects degraded (14.580%); 9971/259128 objects misplaced (3.848%); 1.3 MiB/s, 21 objects/s recovering 2023-06-03T20:12:42.267 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:12:42.270 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:12:42.270 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:12:42.310 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/mon.a/config 2023-06-03T20:12:42.333 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:12:41 smithi164 bash[59462]: cluster 1685823160.7017066 mgr.y (mgr.34104) 1098 : cluster 0 pgmap v1070: 89 pgs: 9 active+clean+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 13 active+undersized+degraded, 2 active+recovering+undersized+remapped, 20 active+undersized, 44 active+clean; 5.3 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 41 MiB/s wr, 648 op/s; 37782/259128 objects degraded (14.580%); 9971/259128 objects misplaced (3.848%); 1.3 MiB/s, 21 objects/s recovering 2023-06-03T20:12:42.372 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:12:42.407 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:12:42.410 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:12:42.410 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:12:42.511 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:12:42.547 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:12:42.551 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:12:42.553 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:12:42.562 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 69 16 89874 89858 81.3786 52.375 0.0151017 0.0122819 2023-06-03T20:12:42.654 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:12:42.689 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:12:42.691 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:12:42.692 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:12:42.792 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:12:42.829 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:12:42.834 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:12:42.835 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:12:42.936 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:12:42.974 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:12:42.977 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:12:42.977 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:12:43.078 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:12:43.095 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:12:42 smithi089 bash[75068]: cluster 1685823162.7313058 mon.a (mon.0) 1040 : cluster 3 Health check update: Degraded data redundancy: 37782/259128 objects degraded (14.580%), 14 pgs degraded (PG_DEGRADED) 2023-06-03T20:12:43.095 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:12:42 smithi089 bash[75068]: audit 1685823162.9109755 mon.c (mon.2) 508 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-03T20:12:43.095 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:12:42 smithi089 bash[78452]: cluster 1685823162.7313058 mon.a (mon.0) 1040 : cluster 3 Health check update: Degraded data redundancy: 37782/259128 objects degraded (14.580%), 14 pgs degraded (PG_DEGRADED) 2023-06-03T20:12:43.095 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:12:42 smithi089 bash[78452]: audit 1685823162.9109755 mon.c (mon.2) 508 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-03T20:12:43.118 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:12:43.125 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:12:43.126 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:12:43.227 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:12:43.237 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:12:42 smithi164 bash[59462]: cluster 1685823162.7313058 mon.a (mon.0) 1040 : cluster 3 Health check update: Degraded data redundancy: 37782/259128 objects degraded (14.580%), 14 pgs degraded (PG_DEGRADED) 2023-06-03T20:12:43.237 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:12:42 smithi164 bash[59462]: audit 1685823162.9109755 mon.c (mon.2) 508 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-03T20:12:43.259 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:12:43.265 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:12:43.266 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:12:43.367 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:12:43.409 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:12:43.411 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:12:43.412 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:12:43.513 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:12:43.545 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:12:43.550 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:12:43.550 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:12:43.562 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 70 16 90615 90599 80.8776 46.3125 0.0597259 0.0123569 2023-06-03T20:12:43.652 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:12:43.702 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:12:43.706 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:12:43.707 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:12:43.808 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:12:43.846 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:12:43.850 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:12:43.850 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:12:43.946 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:12:43 smithi089 bash[78452]: cluster 1685823162.7025871 mgr.y (mgr.34104) 1099 : cluster 0 pgmap v1071: 89 pgs: 9 active+clean+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 13 active+undersized+degraded, 2 active+recovering+undersized+remapped, 20 active+undersized, 44 active+clean; 5.4 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 45 MiB/s wr, 727 op/s; 38271/263619 objects degraded (14.518%); 9886/263619 objects misplaced (3.750%); 1.7 MiB/s, 27 objects/s recovering 2023-06-03T20:12:43.947 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:12:43 smithi089 bash[75068]: cluster 1685823162.7025871 mgr.y (mgr.34104) 1099 : cluster 0 pgmap v1071: 89 pgs: 9 active+clean+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 13 active+undersized+degraded, 2 active+recovering+undersized+remapped, 20 active+undersized, 44 active+clean; 5.4 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 45 MiB/s wr, 727 op/s; 38271/263619 objects degraded (14.518%); 9886/263619 objects misplaced (3.750%); 1.7 MiB/s, 27 objects/s recovering 2023-06-03T20:12:43.951 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:12:43.983 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:12:43.986 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:12:43.987 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:12:44.088 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:12:44.123 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:12:44.125 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:12:44.125 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:12:44.226 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:12:44.235 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:12:43 smithi164 bash[59462]: cluster 1685823162.7025871 mgr.y (mgr.34104) 1099 : cluster 0 pgmap v1071: 89 pgs: 9 active+clean+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 13 active+undersized+degraded, 2 active+recovering+undersized+remapped, 20 active+undersized, 44 active+clean; 5.4 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 45 MiB/s wr, 727 op/s; 38271/263619 objects degraded (14.518%); 9886/263619 objects misplaced (3.750%); 1.7 MiB/s, 27 objects/s recovering 2023-06-03T20:12:44.270 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:12:44.273 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:12:44.274 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:12:44.376 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:12:44.407 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:12:44.409 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:12:44.410 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:12:44.511 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:12:44.547 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:12:44.550 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:12:44.551 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:12:44.563 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 71 16 91458 91442 80.4805 52.6875 0.0223689 0.0124193 2023-06-03T20:12:44.653 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:12:44.703 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:12:44.707 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:12:44.708 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:12:44.809 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:12:44.895 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:12:44.898 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:12:44.899 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:12:45.000 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:12:45.040 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:12:45.044 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:12:45.045 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:12:45.146 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:12:45.200 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:12:45.205 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:12:45.205 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:12:45.306 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:12:45.339 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:12:45.342 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:12:45.342 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:12:45.443 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:12:45.477 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:12:45.482 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:12:45.483 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:12:45.562 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 72 16 92276 92260 80.0728 51.125 0.0128536 0.0124797 2023-06-03T20:12:45.584 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:12:45.624 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:12:45.627 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:12:45.628 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:12:45.728 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:12:45.749 INFO:teuthology.orchestra.run.smithi164.stderr:Inferring config /var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/mon.b/config 2023-06-03T20:12:45.761 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:12:45.763 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:12:45.764 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:12:45.865 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:12:45.904 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:12:45.913 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:12:45.914 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:12:46.015 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:12:46.028 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:12:45 smithi089 bash[75068]: cluster 1685823164.7037413 mgr.y (mgr.34104) 1100 : cluster 0 pgmap v1072: 89 pgs: 9 active+clean+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 13 active+undersized+degraded, 2 active+recovering+undersized+remapped, 20 active+undersized, 44 active+clean; 5.5 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 46 MiB/s wr, 731 op/s; 38743/268791 objects degraded (14.414%); 9784/268791 objects misplaced (3.640%); 2.0 MiB/s, 32 objects/s recovering 2023-06-03T20:12:46.029 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:12:45 smithi089 bash[78452]: cluster 1685823164.7037413 mgr.y (mgr.34104) 1100 : cluster 0 pgmap v1072: 89 pgs: 9 active+clean+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 13 active+undersized+degraded, 2 active+recovering+undersized+remapped, 20 active+undersized, 44 active+clean; 5.5 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 46 MiB/s wr, 731 op/s; 38743/268791 objects degraded (14.414%); 9784/268791 objects misplaced (3.640%); 2.0 MiB/s, 32 objects/s recovering 2023-06-03T20:12:46.052 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:12:46.056 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:12:46.056 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:12:46.083 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:12:45 smithi164 bash[59462]: cluster 1685823164.7037413 mgr.y (mgr.34104) 1100 : cluster 0 pgmap v1072: 89 pgs: 9 active+clean+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 13 active+undersized+degraded, 2 active+recovering+undersized+remapped, 20 active+undersized, 44 active+clean; 5.5 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 46 MiB/s wr, 731 op/s; 38743/268791 objects degraded (14.414%); 9784/268791 objects misplaced (3.640%); 2.0 MiB/s, 32 objects/s recovering 2023-06-03T20:12:46.157 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:12:46.201 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:12:46.204 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:12:46.205 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:12:46.305 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:12:46.314 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 20:12:46 smithi089 bash[71163]: ::ffff:172.21.15.164 - - [03/Jun/2023:20:12:46] "GET /metrics HTTP/1.1" 200 228905 "" "Prometheus/2.33.4" 2023-06-03T20:12:46.341 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:12:46.346 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:12:46.347 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:12:46.448 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:12:46.487 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:12:46.490 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:12:46.491 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:12:46.563 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 73 16 93057 93041 79.6445 48.8125 0.00588965 0.0125467 2023-06-03T20:12:46.592 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:12:46.622 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:12:46.626 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:12:46.627 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:12:46.728 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:12:46.757 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:12:46.761 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:12:46.762 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:12:46.863 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:12:46.889 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:12:46.891 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:12:46.892 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:12:46.993 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:12:47.022 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:12:47.024 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:12:47.025 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:12:47.126 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:12:47.161 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:12:47.164 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:12:47.165 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:12:47.266 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:12:47.308 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:12:47.310 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:12:47.311 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:12:47.411 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:12:47.444 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:12:47.448 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:12:47.449 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:12:47.550 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:12:47.563 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 74 16 93805 93789 79.2 46.75 0.0405847 0.0126185 2023-06-03T20:12:47.594 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:12:47.598 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:12:47.599 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:12:47.699 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:12:47.730 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:12:47.732 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:12:47.733 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:12:47.834 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:12:47.881 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:12:47.886 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:12:47.887 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:12:47.988 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:12:48.016 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:12:48.018 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:12:48.019 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:12:48.083 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:12:47 smithi164 bash[59462]: cluster 1685823166.7044134 mgr.y (mgr.34104) 1101 : cluster 0 pgmap v1073: 89 pgs: 9 active+clean+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 13 active+undersized+degraded, 2 active+recovering+undersized+remapped, 20 active+undersized, 44 active+clean; 5.5 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 45 MiB/s wr, 720 op/s; 39455/271737 objects degraded (14.520%); 9784/271737 objects misplaced (3.601%); 2.0 MiB/s, 31 objects/s recovering 2023-06-03T20:12:48.083 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:12:47 smithi164 bash[59462]: cluster 1685823167.7352378 mon.a (mon.0) 1041 : cluster 3 Health check update: Degraded data redundancy: 39455/271737 objects degraded (14.520%), 14 pgs degraded (PG_DEGRADED) 2023-06-03T20:12:48.120 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:12:48.166 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:12:48.168 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:12:48.169 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:12:48.270 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:12:48.277 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:12:47 smithi089 bash[75068]: cluster 1685823166.7044134 mgr.y (mgr.34104) 1101 : cluster 0 pgmap v1073: 89 pgs: 9 active+clean+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 13 active+undersized+degraded, 2 active+recovering+undersized+remapped, 20 active+undersized, 44 active+clean; 5.5 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 45 MiB/s wr, 720 op/s; 39455/271737 objects degraded (14.520%); 9784/271737 objects misplaced (3.601%); 2.0 MiB/s, 31 objects/s recovering 2023-06-03T20:12:48.277 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:12:47 smithi089 bash[75068]: cluster 1685823167.7352378 mon.a (mon.0) 1041 : cluster 3 Health check update: Degraded data redundancy: 39455/271737 objects degraded (14.520%), 14 pgs degraded (PG_DEGRADED) 2023-06-03T20:12:48.278 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:12:47 smithi089 bash[78452]: cluster 1685823166.7044134 mgr.y (mgr.34104) 1101 : cluster 0 pgmap v1073: 89 pgs: 9 active+clean+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 13 active+undersized+degraded, 2 active+recovering+undersized+remapped, 20 active+undersized, 44 active+clean; 5.5 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 45 MiB/s wr, 720 op/s; 39455/271737 objects degraded (14.520%); 9784/271737 objects misplaced (3.601%); 2.0 MiB/s, 31 objects/s recovering 2023-06-03T20:12:48.278 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:12:47 smithi089 bash[78452]: cluster 1685823167.7352378 mon.a (mon.0) 1041 : cluster 3 Health check update: Degraded data redundancy: 39455/271737 objects degraded (14.520%), 14 pgs degraded (PG_DEGRADED) 2023-06-03T20:12:48.307 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:12:48.309 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:12:48.309 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:12:48.410 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:12:48.441 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:12:48.445 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:12:48.446 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:12:48.547 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:12:48.563 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 75 16 94596 94580 78.8031 49.4375 0.0117671 0.012683 2023-06-03T20:12:48.578 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:12:48.583 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:12:48.583 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:12:48.684 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:12:48.711 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:12:48.714 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:12:48.715 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:12:48.816 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:12:48.849 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:12:48.856 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:12:48.857 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:12:48.958 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:12:48.989 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:12:48.992 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:12:48.992 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:12:49.093 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:12:49.124 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:12:49.127 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:12:49.128 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:12:49.228 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:12:49.258 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:12:49.259 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:12:49.260 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:12:49.361 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:12:49.390 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:12:49.391 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:12:49.392 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:12:49.493 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:12:49.516 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:12:49.519 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:12:49.519 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:12:49.563 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 76 16 95257 95241 78.3098 41.3125 0.00461578 0.012742 2023-06-03T20:12:49.620 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:12:49.652 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:12:49.654 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:12:49.655 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:12:49.756 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:12:49.803 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:12:49.811 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:12:49.813 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:12:49.914 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:12:49.968 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:12:49.969 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:12:49.970 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:12:50.071 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:12:50.076 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:12:49 smithi164 bash[59462]: cluster 1685823168.7050302 mgr.y (mgr.34104) 1102 : cluster 0 pgmap v1074: 89 pgs: 9 active+clean+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 13 active+undersized+degraded, 2 active+recovering+undersized+remapped, 20 active+undersized, 44 active+clean; 5.7 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 56 MiB/s wr, 893 op/s; 39971/279831 objects degraded (14.284%); 9598/279831 objects misplaced (3.430%); 2.9 MiB/s, 46 objects/s recovering 2023-06-03T20:12:50.076 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:12:49 smithi164 bash[59462]: audit 1685823169.485566 mon.a (mon.0) 1042 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:12:50.076 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:12:49 smithi164 bash[59462]: audit 1685823169.493897 mon.a (mon.0) 1043 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:12:50.112 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:12:50.113 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:12:50.113 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:12:50.214 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:12:50.254 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:12:50.256 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:12:50.257 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:12:50.278 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:12:49 smithi089 bash[75068]: cluster 1685823168.7050302 mgr.y (mgr.34104) 1102 : cluster 0 pgmap v1074: 89 pgs: 9 active+clean+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 13 active+undersized+degraded, 2 active+recovering+undersized+remapped, 20 active+undersized, 44 active+clean; 5.7 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 56 MiB/s wr, 893 op/s; 39971/279831 objects degraded (14.284%); 9598/279831 objects misplaced (3.430%); 2.9 MiB/s, 46 objects/s recovering 2023-06-03T20:12:50.279 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:12:49 smithi089 bash[75068]: audit 1685823169.485566 mon.a (mon.0) 1042 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:12:50.279 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:12:49 smithi089 bash[75068]: audit 1685823169.493897 mon.a (mon.0) 1043 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:12:50.279 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:12:49 smithi089 bash[78452]: cluster 1685823168.7050302 mgr.y (mgr.34104) 1102 : cluster 0 pgmap v1074: 89 pgs: 9 active+clean+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 13 active+undersized+degraded, 2 active+recovering+undersized+remapped, 20 active+undersized, 44 active+clean; 5.7 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 56 MiB/s wr, 893 op/s; 39971/279831 objects degraded (14.284%); 9598/279831 objects misplaced (3.430%); 2.9 MiB/s, 46 objects/s recovering 2023-06-03T20:12:50.279 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:12:49 smithi089 bash[78452]: audit 1685823169.485566 mon.a (mon.0) 1042 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:12:50.280 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:12:49 smithi089 bash[78452]: audit 1685823169.493897 mon.a (mon.0) 1043 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:12:50.358 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:12:50.389 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:12:50.393 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:12:50.394 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:12:50.495 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:12:50.544 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:12:50.551 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:12:50.553 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:12:50.563 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 77 16 95962 95946 77.865 44.0625 0.0115838 0.0128367 2023-06-03T20:12:50.654 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:12:50.683 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:12:50.692 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:12:50.693 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:12:50.793 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:12:50.827 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:12:50.830 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:12:50.831 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:12:50.932 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:12:50.966 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:12:50.969 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:12:50.969 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:12:51.070 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:12:51.108 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:12:51.112 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:12:51.113 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:12:51.214 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:12:51.248 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:12:51.252 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:12:51.253 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:12:51.354 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:12:51.399 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:12:51.402 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:12:51.403 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:12:51.503 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:12:51.535 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:12:51.540 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:12:51.540 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:12:51.566 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 78 16 96748 96732 77.4953 49.125 0.00438531 0.0128973 2023-06-03T20:12:51.641 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:12:51.682 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:12:51.684 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:12:51.684 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:12:51.785 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:12:51.838 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:12:51.843 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:12:51.844 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:12:51.945 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:12:51.989 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:12:51.991 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:12:51.992 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:12:52.083 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:12:51 smithi164 bash[59462]: cluster 1685823170.7056444 mgr.y (mgr.34104) 1103 : cluster 0 pgmap v1075: 89 pgs: 9 active+clean+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 13 active+undersized+degraded, 2 active+recovering+undersized+remapped, 20 active+undersized, 44 active+clean; 5.8 GiB data, 18 GiB used, 607 GiB / 626 GiB avail; 46 MiB/s wr, 738 op/s; 40719/282765 objects degraded (14.400%); 9598/282765 objects misplaced (3.394%); 1.9 MiB/s, 31 objects/s recovering 2023-06-03T20:12:52.093 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:12:52.145 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:12:52.153 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:12:52.154 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:12:52.207 INFO:teuthology.orchestra.run.smithi164.stdout:ERROR: (22) Invalid argument 2023-06-03T20:12:52.207 INFO:teuthology.orchestra.run.smithi164.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-06-03T20:12:52.254 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:12:52.279 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:12:51 smithi089 bash[75068]: cluster 1685823170.7056444 mgr.y (mgr.34104) 1103 : cluster 0 pgmap v1075: 89 pgs: 9 active+clean+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 13 active+undersized+degraded, 2 active+recovering+undersized+remapped, 20 active+undersized, 44 active+clean; 5.8 GiB data, 18 GiB used, 607 GiB / 626 GiB avail; 46 MiB/s wr, 738 op/s; 40719/282765 objects degraded (14.400%); 9598/282765 objects misplaced (3.394%); 1.9 MiB/s, 31 objects/s recovering 2023-06-03T20:12:52.279 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:12:51 smithi089 bash[78452]: cluster 1685823170.7056444 mgr.y (mgr.34104) 1103 : cluster 0 pgmap v1075: 89 pgs: 9 active+clean+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 13 active+undersized+degraded, 2 active+recovering+undersized+remapped, 20 active+undersized, 44 active+clean; 5.8 GiB data, 18 GiB used, 607 GiB / 626 GiB avail; 46 MiB/s wr, 738 op/s; 40719/282765 objects degraded (14.400%); 9598/282765 objects misplaced (3.394%); 1.9 MiB/s, 31 objects/s recovering 2023-06-03T20:12:52.292 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:12:52.294 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:12:52.295 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:12:52.396 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:12:52.443 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:12:52.449 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:12:52.449 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:12:52.550 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:12:52.564 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 79 16 97459 97443 77.0769 44.4375 0.0223665 0.0129682 2023-06-03T20:12:52.588 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:12:52.590 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:12:52.591 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:12:52.692 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:12:52.724 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:12:52.727 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:12:52.728 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:12:52.829 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:12:52.870 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:12:52.876 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:12:52.876 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:12:52.970 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:12:52 smithi164 bash[59462]: audit 1685823172.1215284 mon.c (mon.2) 509 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T20:12:52.970 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:12:52 smithi164 bash[59462]: cluster 1685823172.7376986 mon.a (mon.0) 1044 : cluster 3 Health check update: Degraded data redundancy: 40719/282765 objects degraded (14.400%), 14 pgs degraded (PG_DEGRADED) 2023-06-03T20:12:52.977 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:12:53.020 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:12:53.025 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:12:53.025 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:12:53.126 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:12:53.172 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:12:53.177 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:12:53.178 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:12:53.254 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:12:52 smithi089 bash[75068]: audit 1685823172.1215284 mon.c (mon.2) 509 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T20:12:53.255 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:12:52 smithi089 bash[75068]: cluster 1685823172.7376986 mon.a (mon.0) 1044 : cluster 3 Health check update: Degraded data redundancy: 40719/282765 objects degraded (14.400%), 14 pgs degraded (PG_DEGRADED) 2023-06-03T20:12:53.255 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:12:52 smithi089 bash[78452]: audit 1685823172.1215284 mon.c (mon.2) 509 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T20:12:53.255 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:12:52 smithi089 bash[78452]: cluster 1685823172.7376986 mon.a (mon.0) 1044 : cluster 3 Health check update: Degraded data redundancy: 40719/282765 objects degraded (14.400%), 14 pgs degraded (PG_DEGRADED) 2023-06-03T20:12:53.269 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph daemon osd.0 dump_ops_in_flight 2023-06-03T20:12:53.278 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:12:53.305 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:12:53.310 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:12:53.310 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:12:53.411 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:12:53.443 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:12:53.450 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:12:53.451 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:12:53.552 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:12:53.565 INFO:tasks.radosbench.radosbench.0.smithi089.stdout:2023-06-03T20:12:53.562125+0000 min lat: 0.00135638 max lat: 2.42958 avg lat: 0.0130351 2023-06-03T20:12:53.566 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-06-03T20:12:53.566 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 80 16 98183 98167 76.6779 45.25 0.0109734 0.0130351 2023-06-03T20:12:53.598 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:12:53.601 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:12:53.601 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:12:53.702 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:12:53.731 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:12:53.734 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:12:53.735 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:12:53.835 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:12:53.873 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:12:53.876 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:12:53.877 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:12:53.978 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:12:54.012 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:12:54.015 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:12:54.015 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:12:54.116 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:12:54.146 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:12:54.148 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:12:54.149 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:12:54.249 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:12:54.258 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:12:53 smithi164 bash[59462]: cluster 1685823172.706254 mgr.y (mgr.34104) 1104 : cluster 0 pgmap v1076: 89 pgs: 9 active+clean+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 13 active+undersized+degraded, 2 active+recovering+undersized+remapped, 20 active+undersized, 44 active+clean; 5.8 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 48 MiB/s wr, 767 op/s; 41135/286779 objects degraded (14.344%); 9496/286779 objects misplaced (3.311%); 2.5 MiB/s, 39 objects/s recovering 2023-06-03T20:12:54.258 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:12:53 smithi164 bash[59462]: audit 1685823172.9963722 mon.a (mon.0) 1045 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:12:54.258 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:12:53 smithi164 bash[59462]: audit 1685823173.0094924 mon.a (mon.0) 1046 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:12:54.258 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:12:53 smithi164 bash[59462]: audit 1685823173.6066463 mon.c (mon.2) 510 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-03T20:12:54.258 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:12:53 smithi164 bash[59462]: audit 1685823173.6099908 mon.c (mon.2) 511 : audit 1 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-03T20:12:54.258 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:12:53 smithi164 bash[59462]: audit 1685823173.6234283 mon.a (mon.0) 1047 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:12:54.278 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:12:53 smithi089 bash[75068]: cluster 1685823172.706254 mgr.y (mgr.34104) 1104 : cluster 0 pgmap v1076: 89 pgs: 9 active+clean+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 13 active+undersized+degraded, 2 active+recovering+undersized+remapped, 20 active+undersized, 44 active+clean; 5.8 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 48 MiB/s wr, 767 op/s; 41135/286779 objects degraded (14.344%); 9496/286779 objects misplaced (3.311%); 2.5 MiB/s, 39 objects/s recovering 2023-06-03T20:12:54.279 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:12:53 smithi089 bash[75068]: audit 1685823172.9963722 mon.a (mon.0) 1045 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:12:54.279 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:12:53 smithi089 bash[75068]: audit 1685823173.0094924 mon.a (mon.0) 1046 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:12:54.279 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:12:53 smithi089 bash[75068]: audit 1685823173.6066463 mon.c (mon.2) 510 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-03T20:12:54.279 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:12:53 smithi089 bash[75068]: audit 1685823173.6099908 mon.c (mon.2) 511 : audit 1 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-03T20:12:54.279 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:12:53 smithi089 bash[75068]: audit 1685823173.6234283 mon.a (mon.0) 1047 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:12:54.279 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:12:53 smithi089 bash[78452]: cluster 1685823172.706254 mgr.y (mgr.34104) 1104 : cluster 0 pgmap v1076: 89 pgs: 9 active+clean+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 13 active+undersized+degraded, 2 active+recovering+undersized+remapped, 20 active+undersized, 44 active+clean; 5.8 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 48 MiB/s wr, 767 op/s; 41135/286779 objects degraded (14.344%); 9496/286779 objects misplaced (3.311%); 2.5 MiB/s, 39 objects/s recovering 2023-06-03T20:12:54.280 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:12:53 smithi089 bash[78452]: audit 1685823172.9963722 mon.a (mon.0) 1045 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:12:54.280 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:12:53 smithi089 bash[78452]: audit 1685823173.0094924 mon.a (mon.0) 1046 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:12:54.280 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:12:53 smithi089 bash[78452]: audit 1685823173.6066463 mon.c (mon.2) 510 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-03T20:12:54.280 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:12:53 smithi089 bash[78452]: audit 1685823173.6099908 mon.c (mon.2) 511 : audit 1 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-03T20:12:54.280 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:12:53 smithi089 bash[78452]: audit 1685823173.6234283 mon.a (mon.0) 1047 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:12:54.283 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:12:54.289 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:12:54.290 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:12:54.391 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:12:54.420 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:12:54.423 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:12:54.424 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:12:54.525 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:12:54.566 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 81 16 98976 98960 76.3431 49.5625 0.00569517 0.0130924 2023-06-03T20:12:54.571 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:12:54.578 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:12:54.578 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:12:54.679 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:12:54.733 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:12:54.738 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:12:54.739 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:12:54.840 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:12:54.881 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:12:54.883 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:12:54.884 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:12:54.984 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:12:55.013 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:12:55.017 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:12:55.018 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:12:55.119 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:12:55.154 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:12:55.157 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:12:55.157 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:12:55.258 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:12:55.295 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:12:55.298 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:12:55.299 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:12:55.400 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:12:55.431 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:12:55.435 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:12:55.436 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:12:55.537 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:12:55.566 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 82 16 99766 99750 76.0142 49.375 0.0104443 0.0131494 2023-06-03T20:12:55.577 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:12:55.580 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:12:55.581 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:12:55.682 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:12:55.719 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:12:55.724 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:12:55.725 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:12:55.826 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:12:55.859 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:12:55.864 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:12:55.864 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:12:55.965 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:12:56.005 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:12:56.011 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:12:56.011 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:12:56 smithi089 bash[75068]: cluster 1685823174.7090976 mgr.y (mgr.34104) 1105 : cluster 0 pgmap v1077: 89 pgs: 9 active+clean+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 13 active+undersized+degraded, 2 active+recovering+undersized+remapped, 20 active+undersized, 44 active+clean; 5.9 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 49 MiB/s wr, 777 op/s; 41596/291636 objects degraded (14.263%); 9395/291636 objects misplaced (3.221%); 2.6 MiB/s, 40 objects/s recovering 2023-06-03T20:12:56.012 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:12:56.012 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:12:56 smithi089 bash[78452]: cluster 1685823174.7090976 mgr.y (mgr.34104) 1105 : cluster 0 pgmap v1077: 89 pgs: 9 active+clean+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 13 active+undersized+degraded, 2 active+recovering+undersized+remapped, 20 active+undersized, 44 active+clean; 5.9 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 49 MiB/s wr, 777 op/s; 41596/291636 objects degraded (14.263%); 9395/291636 objects misplaced (3.221%); 2.6 MiB/s, 40 objects/s recovering 2023-06-03T20:12:56.078 INFO:teuthology.orchestra.run.smithi089.stdout:osd.0: {} 2023-06-03T20:12:56.113 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:12:56.120 INFO:teuthology.orchestra.run.smithi089.stdout:osd.1: {} 2023-06-03T20:12:56.158 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:12:56.165 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:12:56.166 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:12:56.187 INFO:teuthology.orchestra.run.smithi089.stdout:osd.2: {} 2023-06-03T20:12:56.234 INFO:teuthology.orchestra.run.smithi089.stdout:osd.3: {} 2023-06-03T20:12:56.267 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:12:56.273 INFO:teuthology.orchestra.run.smithi089.stdout:osd.5: {} 2023-06-03T20:12:56.308 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:12:56.312 INFO:teuthology.orchestra.run.smithi089.stdout:osd.6: {} 2023-06-03T20:12:56.314 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:12:56.315 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:12:56.333 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:12:56 smithi164 bash[59462]: cluster 1685823174.7090976 mgr.y (mgr.34104) 1105 : cluster 0 pgmap v1077: 89 pgs: 9 active+clean+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 13 active+undersized+degraded, 2 active+recovering+undersized+remapped, 20 active+undersized, 44 active+clean; 5.9 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 49 MiB/s wr, 777 op/s; 41596/291636 objects degraded (14.263%); 9395/291636 objects misplaced (3.221%); 2.6 MiB/s, 40 objects/s recovering 2023-06-03T20:12:56.345 INFO:teuthology.orchestra.run.smithi089.stdout:osd.7: {} 2023-06-03T20:12:56.345 INFO:teuthology.orchestra.run.smithi089.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-06-03T20:12:56.345 INFO:teuthology.orchestra.run.smithi089.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-06-03T20:12:56.346 INFO:teuthology.orchestra.run.smithi089.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-06-03T20:12:56.346 INFO:teuthology.orchestra.run.smithi089.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-06-03T20:12:56.346 INFO:teuthology.orchestra.run.smithi089.stderr:Error ENXIO: problem getting command descriptions from osd.4 2023-06-03T20:12:56.346 INFO:teuthology.orchestra.run.smithi089.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-06-03T20:12:56.346 INFO:teuthology.orchestra.run.smithi089.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-06-03T20:12:56.346 INFO:teuthology.orchestra.run.smithi089.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-06-03T20:12:56.416 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:12:56.451 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:12:56.453 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:12:56.454 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:12:56.490 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 20:12:56 smithi089 bash[71163]: ::ffff:172.21.15.164 - - [03/Jun/2023:20:12:56] "GET /metrics HTTP/1.1" 200 228917 "" "Prometheus/2.33.4" 2023-06-03T20:12:56.554 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:12:56.567 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 83 16 100402 100386 75.576 39.75 0.0122403 0.0132255 2023-06-03T20:12:56.591 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:12:56.595 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:12:56.596 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:12:56.697 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:12:56.733 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:12:56.738 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:12:56.738 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:12:56.839 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:12:56.883 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:12:56.889 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:12:56.889 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:12:56.990 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:12:57.032 INFO:teuthology.orchestra.run.smithi089.stderr:set osd.3 primary-affinity to 1 (8655362) 2023-06-03T20:12:57.036 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:12:57.037 INFO:teuthology.orchestra.run.smithi089.stderr:nodeep-scrub is unset 2023-06-03T20:12:57.038 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:12:57.039 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:12:57.140 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:12:57.169 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:12:57.174 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:12:57.175 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:12:57.276 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:12:57.278 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:12:57 smithi089 bash[78452]: audit 1685823175.9994185 mon.c (mon.2) 512 : audit 0 from='client.? 172.21.15.89:0/1583425517' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-03T20:12:57.279 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:12:57 smithi089 bash[78452]: audit 1685823176.2303202 mon.b (mon.1) 56 : audit 1 from='client.? 172.21.15.89:0/2628909707' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-06-03T20:12:57.279 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:12:57 smithi089 bash[78452]: audit 1685823176.233655 mon.a (mon.0) 1048 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-06-03T20:12:57.279 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:12:57 smithi089 bash[78452]: audit 1685823176.522097 mon.a (mon.0) 1049 : audit 1 from='client.? 172.21.15.89:0/4155364196' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 3, "weight": 1.0}]: dispatch 2023-06-03T20:12:57.279 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:12:57 smithi089 bash[75068]: audit 1685823175.9994185 mon.c (mon.2) 512 : audit 0 from='client.? 172.21.15.89:0/1583425517' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-03T20:12:57.280 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:12:57 smithi089 bash[75068]: audit 1685823176.2303202 mon.b (mon.1) 56 : audit 1 from='client.? 172.21.15.89:0/2628909707' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-06-03T20:12:57.280 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:12:57 smithi089 bash[75068]: audit 1685823176.233655 mon.a (mon.0) 1048 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-06-03T20:12:57.280 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:12:57 smithi089 bash[75068]: audit 1685823176.522097 mon.a (mon.0) 1049 : audit 1 from='client.? 172.21.15.89:0/4155364196' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 3, "weight": 1.0}]: dispatch 2023-06-03T20:12:57.311 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:12:57.314 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:12:57.314 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:12:57.333 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:12:57 smithi164 bash[59462]: audit 1685823175.9994185 mon.c (mon.2) 512 : audit 0 from='client.? 172.21.15.89:0/1583425517' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-03T20:12:57.333 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:12:57 smithi164 bash[59462]: audit 1685823176.2303202 mon.b (mon.1) 56 : audit 1 from='client.? 172.21.15.89:0/2628909707' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-06-03T20:12:57.333 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:12:57 smithi164 bash[59462]: audit 1685823176.233655 mon.a (mon.0) 1048 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-06-03T20:12:57.333 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:12:57 smithi164 bash[59462]: audit 1685823176.522097 mon.a (mon.0) 1049 : audit 1 from='client.? 172.21.15.89:0/4155364196' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 3, "weight": 1.0}]: dispatch 2023-06-03T20:12:57.415 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:12:57.448 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:12:57.450 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:12:57.450 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:12:57.551 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:12:57.572 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 84 16 101182 101166 75.2567 48.75 0.0134448 0.0132822 2023-06-03T20:12:57.595 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:12:57.598 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:12:57.599 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:12:57.700 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:12:57.733 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:12:57.744 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:12:57.745 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:12:57.845 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:12:57.868 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:12:57.870 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:12:57.870 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:12:57.971 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:12:58.010 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:12:58.013 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:12:58.014 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:12:58.115 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:12:58.164 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:12:58.170 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:12:58.171 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:12:58.272 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:12:58.278 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:12:58 smithi089 bash[75068]: cluster 1685823176.7100487 mgr.y (mgr.34104) 1106 : cluster 0 pgmap v1078: 89 pgs: 9 active+clean+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 13 active+undersized+degraded, 2 active+recovering+undersized+remapped, 20 active+undersized, 44 active+clean; 6.0 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 45 MiB/s wr, 714 op/s; 42325/294519 objects degraded (14.371%); 9395/294519 objects misplaced (3.190%); 2.0 MiB/s, 32 objects/s recovering 2023-06-03T20:12:58.278 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:12:58 smithi089 bash[75068]: cluster 1685823177.002113 mon.a (mon.0) 1050 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-06-03T20:12:58.279 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:12:58 smithi089 bash[75068]: cluster 1685823177.0168555 osd.6 (osd.6) 146 : cluster 3 failed to encode map e201 with expected crc 2023-06-03T20:12:58.279 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:12:58 smithi089 bash[75068]: audit 1685823177.018806 mon.a (mon.0) 1051 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-06-03T20:12:58.279 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:12:58 smithi089 bash[75068]: audit 1685823177.0189047 mon.a (mon.0) 1052 : audit 1 from='client.? 172.21.15.89:0/4155364196' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 3, "weight": 1.0}]': finished 2023-06-03T20:12:58.279 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:12:58 smithi089 bash[75068]: cluster 1685823177.018975 mon.a (mon.0) 1053 : cluster 0 osdmap e201: 8 total, 7 up, 7 in 2023-06-03T20:12:58.279 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:12:58 smithi089 bash[75068]: cluster 1685823177.738888 mon.a (mon.0) 1054 : cluster 3 Health check update: Degraded data redundancy: 42325/294519 objects degraded (14.371%), 14 pgs degraded, 33 pgs undersized (PG_DEGRADED) 2023-06-03T20:12:58.279 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:12:58 smithi089 bash[78452]: cluster 1685823176.7100487 mgr.y (mgr.34104) 1106 : cluster 0 pgmap v1078: 89 pgs: 9 active+clean+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 13 active+undersized+degraded, 2 active+recovering+undersized+remapped, 20 active+undersized, 44 active+clean; 6.0 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 45 MiB/s wr, 714 op/s; 42325/294519 objects degraded (14.371%); 9395/294519 objects misplaced (3.190%); 2.0 MiB/s, 32 objects/s recovering 2023-06-03T20:12:58.280 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:12:58 smithi089 bash[78452]: cluster 1685823177.002113 mon.a (mon.0) 1050 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-06-03T20:12:58.280 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:12:58 smithi089 bash[78452]: cluster 1685823177.0168555 osd.6 (osd.6) 146 : cluster 3 failed to encode map e201 with expected crc 2023-06-03T20:12:58.280 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:12:58 smithi089 bash[78452]: audit 1685823177.018806 mon.a (mon.0) 1051 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-06-03T20:12:58.280 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:12:58 smithi089 bash[78452]: audit 1685823177.0189047 mon.a (mon.0) 1052 : audit 1 from='client.? 172.21.15.89:0/4155364196' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 3, "weight": 1.0}]': finished 2023-06-03T20:12:58.280 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:12:58 smithi089 bash[78452]: cluster 1685823177.018975 mon.a (mon.0) 1053 : cluster 0 osdmap e201: 8 total, 7 up, 7 in 2023-06-03T20:12:58.280 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:12:58 smithi089 bash[78452]: cluster 1685823177.738888 mon.a (mon.0) 1054 : cluster 3 Health check update: Degraded data redundancy: 42325/294519 objects degraded (14.371%), 14 pgs degraded, 33 pgs undersized (PG_DEGRADED) 2023-06-03T20:12:58.302 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:12:58.307 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:12:58.308 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:12:58.333 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:12:58 smithi164 bash[59462]: cluster 1685823176.7100487 mgr.y (mgr.34104) 1106 : cluster 0 pgmap v1078: 89 pgs: 9 active+clean+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 13 active+undersized+degraded, 2 active+recovering+undersized+remapped, 20 active+undersized, 44 active+clean; 6.0 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 45 MiB/s wr, 714 op/s; 42325/294519 objects degraded (14.371%); 9395/294519 objects misplaced (3.190%); 2.0 MiB/s, 32 objects/s recovering 2023-06-03T20:12:58.333 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:12:58 smithi164 bash[59462]: cluster 1685823177.002113 mon.a (mon.0) 1050 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-06-03T20:12:58.333 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:12:58 smithi164 bash[59462]: cluster 1685823177.0168555 osd.6 (osd.6) 146 : cluster 3 failed to encode map e201 with expected crc 2023-06-03T20:12:58.334 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:12:58 smithi164 bash[59462]: audit 1685823177.018806 mon.a (mon.0) 1051 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-06-03T20:12:58.334 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:12:58 smithi164 bash[59462]: audit 1685823177.0189047 mon.a (mon.0) 1052 : audit 1 from='client.? 172.21.15.89:0/4155364196' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 3, "weight": 1.0}]': finished 2023-06-03T20:12:58.334 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:12:58 smithi164 bash[59462]: cluster 1685823177.018975 mon.a (mon.0) 1053 : cluster 0 osdmap e201: 8 total, 7 up, 7 in 2023-06-03T20:12:58.334 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:12:58 smithi164 bash[59462]: cluster 1685823177.738888 mon.a (mon.0) 1054 : cluster 3 Health check update: Degraded data redundancy: 42325/294519 objects degraded (14.371%), 14 pgs degraded, 33 pgs undersized (PG_DEGRADED) 2023-06-03T20:12:58.410 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:12:58.439 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:12:58.441 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:12:58.441 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:12:58.474 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-06-03T20:12:58.474 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-06-03T20:12:58.542 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:12:58.568 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 85 16 101901 101885 74.9 44.9375 0.024633 0.0133454 2023-06-03T20:12:58.573 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:12:58.578 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:12:58.579 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:12:58.680 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:12:58.722 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:12:58.727 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:12:58.727 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:12:58.828 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:12:58.861 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:12:58.863 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:12:58.864 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:12:58.965 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:12:59.006 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:12:59.013 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:12:59.014 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:12:59.115 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:12:59.145 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:12:59.149 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:12:59.150 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:12:59.251 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:12:59.296 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:12:59.298 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:12:59.299 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:12:59.400 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:12:59.430 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:12:59.435 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:12:59.436 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:12:59.537 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:12:59.568 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 86 16 102682 102666 74.5966 48.8125 0.015905 0.0133991 2023-06-03T20:12:59.569 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:12:59.578 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:12:59.579 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:12:59.680 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:12:59.717 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:12:59.719 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:12:59.720 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:12:59.820 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:12:59.849 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:12:59.854 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:12:59.854 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:12:59.956 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:12:59.997 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:13:00.002 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:13:00.002 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:13:00.103 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:13:00.139 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:13:00.142 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:13:00.143 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:13:00.244 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:13:00.252 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:13:00 smithi089 bash[78452]: cluster 1685823178.7109904 mgr.y (mgr.34104) 1107 : cluster 0 pgmap v1080: 89 pgs: 9 active+clean+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 13 active+undersized+degraded, 2 active+recovering+undersized+remapped, 20 active+undersized, 44 active+clean; 6.1 GiB data, 19 GiB used, 606 GiB / 626 GiB avail; 46 MiB/s wr, 743 op/s; 42795/302148 objects degraded (14.164%); 9205/302148 objects misplaced (3.047%); 2.5 MiB/s, 39 objects/s recovering 2023-06-03T20:13:00.253 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:13:00 smithi089 bash[75068]: cluster 1685823178.7109904 mgr.y (mgr.34104) 1107 : cluster 0 pgmap v1080: 89 pgs: 9 active+clean+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 13 active+undersized+degraded, 2 active+recovering+undersized+remapped, 20 active+undersized, 44 active+clean; 6.1 GiB data, 19 GiB used, 606 GiB / 626 GiB avail; 46 MiB/s wr, 743 op/s; 42795/302148 objects degraded (14.164%); 9205/302148 objects misplaced (3.047%); 2.5 MiB/s, 39 objects/s recovering 2023-06-03T20:13:00.281 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:13:00.284 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:13:00.285 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:13:00.333 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:13:00 smithi164 bash[59462]: cluster 1685823178.7109904 mgr.y (mgr.34104) 1107 : cluster 0 pgmap v1080: 89 pgs: 9 active+clean+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 13 active+undersized+degraded, 2 active+recovering+undersized+remapped, 20 active+undersized, 44 active+clean; 6.1 GiB data, 19 GiB used, 606 GiB / 626 GiB avail; 46 MiB/s wr, 743 op/s; 42795/302148 objects degraded (14.164%); 9205/302148 objects misplaced (3.047%); 2.5 MiB/s, 39 objects/s recovering 2023-06-03T20:13:00.386 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:13:00.440 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:13:00.443 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:13:00.444 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:13:00.475 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-06-03T20:13:00.544 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:13:00.569 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 87 16 103376 103360 74.2371 43.375 0.0203067 0.0134641 2023-06-03T20:13:00.588 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:13:00.591 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:13:00.591 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:13:00.692 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:13:00.726 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:13:00.732 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:13:00.732 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:13:00.833 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:13:00.873 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:13:00.875 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:13:00.876 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:13:00.977 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:13:01.020 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:13:01.023 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:13:01.024 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:13:01.124 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:13:01.160 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:13:01.162 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:13:01.163 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:13:01.264 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:13:01.299 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:13:01.303 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:13:01.303 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:13:01.404 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:13:01.446 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:13:01.451 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:13:01.452 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:13:01.553 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:13:01.568 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 88 16 104110 104094 73.9147 45.875 0.0318766 0.0135236 2023-06-03T20:13:01.589 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:13:01.593 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:13:01.593 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:13:01.694 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:13:01.733 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:13:01.739 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:13:01.740 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:13:01.841 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:13:01.875 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:13:01.878 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:13:01.879 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:13:01.956 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/mon.a/config 2023-06-03T20:13:01.980 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:13:02.015 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:13:02.018 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:13:02.019 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:13:02.120 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:13:02.163 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:13:02.163 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:13:02.164 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:13:02.265 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:13:02.274 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:13:02 smithi089 bash[75068]: cluster 1685823180.7118812 mgr.y (mgr.34104) 1108 : cluster 0 pgmap v1081: 89 pgs: 9 active+clean+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 13 active+undersized+degraded, 2 active+recovering+undersized+remapped, 20 active+undersized, 44 active+clean; 6.2 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 46 MiB/s wr, 737 op/s; 43480/304893 objects degraded (14.261%); 9205/304893 objects misplaced (3.019%); 2.5 MiB/s, 39 objects/s recovering 2023-06-03T20:13:02.275 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:13:02 smithi089 bash[78452]: cluster 1685823180.7118812 mgr.y (mgr.34104) 1108 : cluster 0 pgmap v1081: 89 pgs: 9 active+clean+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 13 active+undersized+degraded, 2 active+recovering+undersized+remapped, 20 active+undersized, 44 active+clean; 6.2 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 46 MiB/s wr, 737 op/s; 43480/304893 objects degraded (14.261%); 9205/304893 objects misplaced (3.019%); 2.5 MiB/s, 39 objects/s recovering 2023-06-03T20:13:02.297 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:13:02.300 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:13:02.301 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:13:02.333 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:13:02 smithi164 bash[59462]: cluster 1685823180.7118812 mgr.y (mgr.34104) 1108 : cluster 0 pgmap v1081: 89 pgs: 9 active+clean+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 13 active+undersized+degraded, 2 active+recovering+undersized+remapped, 20 active+undersized, 44 active+clean; 6.2 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 46 MiB/s wr, 737 op/s; 43480/304893 objects degraded (14.261%); 9205/304893 objects misplaced (3.019%); 2.5 MiB/s, 39 objects/s recovering 2023-06-03T20:13:02.383 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph osd set noscrub 2023-06-03T20:13:02.402 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:13:02.438 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:13:02.441 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:13:02.442 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:13:02.543 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:13:02.569 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 89 16 104840 104824 73.5963 45.625 0.0258443 0.0135801 2023-06-03T20:13:02.572 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:13:02.574 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:13:02.574 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:13:02.675 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:13:02.708 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:13:02.711 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:13:02.712 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:13:02.813 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:13:02.853 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:13:02.858 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:13:02.858 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:13:02.959 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:13:02.999 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:13:03.009 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:13:03.010 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:13:03.111 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:13:03.120 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:13:03 smithi089 bash[75068]: cluster 1685823182.74014 mon.a (mon.0) 1055 : cluster 3 Health check update: Degraded data redundancy: 43480/304893 objects degraded (14.261%), 14 pgs degraded, 33 pgs undersized (PG_DEGRADED) 2023-06-03T20:13:03.120 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:13:03 smithi089 bash[78452]: cluster 1685823182.74014 mon.a (mon.0) 1055 : cluster 3 Health check update: Degraded data redundancy: 43480/304893 objects degraded (14.261%), 14 pgs degraded, 33 pgs undersized (PG_DEGRADED) 2023-06-03T20:13:03.139 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:13:03.146 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:13:03.147 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:13:03.247 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:13:03.281 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:13:03.291 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:13:03.291 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:13:03.333 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:13:03 smithi164 bash[59462]: cluster 1685823182.74014 mon.a (mon.0) 1055 : cluster 3 Health check update: Degraded data redundancy: 43480/304893 objects degraded (14.261%), 14 pgs degraded, 33 pgs undersized (PG_DEGRADED) 2023-06-03T20:13:03.392 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:13:03.419 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:13:03.421 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:13:03.422 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:13:03.523 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:13:03.553 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:13:03.556 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:13:03.557 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:13:03.569 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 90 2 105525 105523 73.264 43.6875 0.028854 0.0136431 2023-06-03T20:13:03.658 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:13:03.692 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:13:03.694 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:13:03.694 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:13:03.795 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:13:03.828 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:13:03.830 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:13:03.831 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:13:03.932 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:13:03.961 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:13:03.963 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:13:03.964 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:13:04.065 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:13:04.077 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:13:04 smithi164 bash[59462]: cluster 1685823182.7130435 mgr.y (mgr.34104) 1109 : cluster 0 pgmap v1082: 89 pgs: 9 active+clean+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 13 active+undersized+degraded, 2 active+recovering+undersized+remapped, 20 active+undersized, 44 active+clean; 6.2 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 39 MiB/s wr, 626 op/s; 43710/305583 objects degraded (14.304%); 9205/305583 objects misplaced (3.012%); 1.8 MiB/s, 29 objects/s recovering 2023-06-03T20:13:04.100 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:13:04.103 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:13:04.104 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:13:04.205 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:13:04.215 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:13:04 smithi089 bash[78452]: cluster 1685823182.7130435 mgr.y (mgr.34104) 1109 : cluster 0 pgmap v1082: 89 pgs: 9 active+clean+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 13 active+undersized+degraded, 2 active+recovering+undersized+remapped, 20 active+undersized, 44 active+clean; 6.2 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 39 MiB/s wr, 626 op/s; 43710/305583 objects degraded (14.304%); 9205/305583 objects misplaced (3.012%); 1.8 MiB/s, 29 objects/s recovering 2023-06-03T20:13:04.216 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:13:04 smithi089 bash[75068]: cluster 1685823182.7130435 mgr.y (mgr.34104) 1109 : cluster 0 pgmap v1082: 89 pgs: 9 active+clean+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 13 active+undersized+degraded, 2 active+recovering+undersized+remapped, 20 active+undersized, 44 active+clean; 6.2 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 39 MiB/s wr, 626 op/s; 43710/305583 objects degraded (14.304%); 9205/305583 objects misplaced (3.012%); 1.8 MiB/s, 29 objects/s recovering 2023-06-03T20:13:04.233 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:13:04.236 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:13:04.237 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:13:04.338 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:13:04.366 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:13:04.370 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:13:04.371 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:13:04.472 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:13:04.502 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:13:04.504 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:13:04.505 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:13:04.569 INFO:tasks.radosbench.radosbench.0.smithi089.stdout:Total time run: 90.0281 2023-06-03T20:13:04.570 INFO:tasks.radosbench.radosbench.0.smithi089.stdout:Total writes made: 105525 2023-06-03T20:13:04.570 INFO:tasks.radosbench.radosbench.0.smithi089.stdout:Write size: 65536 2023-06-03T20:13:04.570 INFO:tasks.radosbench.radosbench.0.smithi089.stdout:Object size: 65536 2023-06-03T20:13:04.570 INFO:tasks.radosbench.radosbench.0.smithi089.stdout:Bandwidth (MB/sec): 73.2584 2023-06-03T20:13:04.570 INFO:tasks.radosbench.radosbench.0.smithi089.stdout:Stddev Bandwidth: 32.6276 2023-06-03T20:13:04.571 INFO:tasks.radosbench.radosbench.0.smithi089.stdout:Max bandwidth (MB/sec): 181.562 2023-06-03T20:13:04.571 INFO:tasks.radosbench.radosbench.0.smithi089.stdout:Min bandwidth (MB/sec): 0 2023-06-03T20:13:04.571 INFO:tasks.radosbench.radosbench.0.smithi089.stdout:Average IOPS: 1172 2023-06-03T20:13:04.571 INFO:tasks.radosbench.radosbench.0.smithi089.stdout:Stddev IOPS: 522.052 2023-06-03T20:13:04.572 INFO:tasks.radosbench.radosbench.0.smithi089.stdout:Max IOPS: 2905 2023-06-03T20:13:04.572 INFO:tasks.radosbench.radosbench.0.smithi089.stdout:Min IOPS: 0 2023-06-03T20:13:04.572 INFO:tasks.radosbench.radosbench.0.smithi089.stdout:Average Latency(s): 0.0136435 2023-06-03T20:13:04.573 INFO:tasks.radosbench.radosbench.0.smithi089.stdout:Stddev Latency(s): 0.0330146 2023-06-03T20:13:04.573 INFO:tasks.radosbench.radosbench.0.smithi089.stdout:Max latency(s): 2.42958 2023-06-03T20:13:04.573 INFO:tasks.radosbench.radosbench.0.smithi089.stdout:Min latency(s): 0.00135638 2023-06-03T20:13:04.580 INFO:tasks.radosbench.radosbench.0.smithi089.stdout:Cleaning up (deleting benchmark objects) 2023-06-03T20:13:04.606 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:13:04.654 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:13:04.656 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:13:04.657 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:13:04.758 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:13:04.807 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:13:04.809 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:13:04.810 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:13:04.911 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:13:04.948 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:13:04.955 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:13:04.956 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:13:05.040 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/mon.a/config 2023-06-03T20:13:05.057 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:13:05.085 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:13:05.087 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:13:05.088 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:13:05.189 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:13:05.221 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:13:05.225 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:13:05.226 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:13:05.326 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:13:05.359 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:13:05.362 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:13:05.363 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:13:05.453 INFO:tasks.thrashosds.thrasher:in_osds: [0, 2, 3, 4, 5, 6, 7] out_osds: [1] dead_osds: [4] live_osds: [0, 1, 2, 3, 5, 6, 7] 2023-06-03T20:13:05.453 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-06-03T20:13:05.453 INFO:tasks.thrashosds.thrasher:fixing pg num pool device_health_metrics 2023-06-03T20:13:05.454 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph pg dump --format=json 2023-06-03T20:13:05.464 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:13:05.543 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:13:05.546 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:13:05.546 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:13:05.647 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:13:05.688 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:13:05.691 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:13:05.692 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:13:05.793 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:13:05.820 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:13:05.821 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:13:05.822 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:13:05.922 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:13:05.948 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:13:05.951 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:13:05.951 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:13:06.052 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:13:06.086 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:13:06.089 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:13:06.090 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:13:06.190 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:13:06.199 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:13:06 smithi089 bash[75068]: cluster 1685823184.7140567 mgr.y (mgr.34104) 1110 : cluster 0 pgmap v1083: 89 pgs: 9 active+clean+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 13 active+undersized+degraded, 2 active+recovering+undersized+remapped, 20 active+undersized, 44 active+clean; 6.4 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 46 MiB/s wr, 739 op/s; 44362/313833 objects degraded (14.136%); 9012/313833 objects misplaced (2.872%); 2.4 MiB/s, 38 objects/s recovering 2023-06-03T20:13:06.200 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:13:06 smithi089 bash[78452]: cluster 1685823184.7140567 mgr.y (mgr.34104) 1110 : cluster 0 pgmap v1083: 89 pgs: 9 active+clean+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 13 active+undersized+degraded, 2 active+recovering+undersized+remapped, 20 active+undersized, 44 active+clean; 6.4 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 46 MiB/s wr, 739 op/s; 44362/313833 objects degraded (14.136%); 9012/313833 objects misplaced (2.872%); 2.4 MiB/s, 38 objects/s recovering 2023-06-03T20:13:06.230 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:13:06.233 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:13:06.234 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:13:06.333 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:13:06 smithi164 bash[59462]: cluster 1685823184.7140567 mgr.y (mgr.34104) 1110 : cluster 0 pgmap v1083: 89 pgs: 9 active+clean+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 13 active+undersized+degraded, 2 active+recovering+undersized+remapped, 20 active+undersized, 44 active+clean; 6.4 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 46 MiB/s wr, 739 op/s; 44362/313833 objects degraded (14.136%); 9012/313833 objects misplaced (2.872%); 2.4 MiB/s, 38 objects/s recovering 2023-06-03T20:13:06.335 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:13:06.371 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:13:06.374 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:13:06.375 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:13:06.475 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:13:06.510 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:13:06.518 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:13:06.518 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:13:06.528 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 20:13:06 smithi089 bash[71163]: ::ffff:172.21.15.164 - - [03/Jun/2023:20:13:06] "GET /metrics HTTP/1.1" 200 228917 "" "Prometheus/2.33.4" 2023-06-03T20:13:06.619 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:13:06.639 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:13:06.640 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:13:06.641 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:13:06.742 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:13:06.774 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:13:06.778 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:13:06.779 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:13:06.880 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:13:06.920 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:13:06.924 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:13:06.925 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:13:07.026 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:13:07.067 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:13:07.074 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:13:07.074 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:13:07.175 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:13:07.220 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:13:07.222 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:13:07.223 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:13:07.324 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:13:07.356 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:13:07.360 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:13:07.361 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:13:07.463 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:13:07.490 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:13:07.495 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:13:07.496 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:13:07.549 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/mon.a/config 2023-06-03T20:13:07.597 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:13:07.636 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:13:07.643 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:13:07.644 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:13:07.745 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:13:07.778 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:13:07.778 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:13:07.779 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:13:07.880 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:13:07.908 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:13:07.914 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:13:07.915 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:13:08.015 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:13:08.059 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:13:08.061 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:13:08.062 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:13:08.163 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:13:08.200 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:13:08.209 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:13:08.210 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:13:08.278 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:13:08 smithi089 bash[78452]: cluster 1685823186.7148776 mgr.y (mgr.34104) 1111 : cluster 0 pgmap v1084: 89 pgs: 9 active+clean+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 13 active+undersized+degraded, 2 active+recovering+undersized+remapped, 20 active+undersized, 44 active+clean; 6.4 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 102 B/s rd, 44 MiB/s wr, 712 op/s; 44737/315396 objects degraded (14.184%); 9012/315396 objects misplaced (2.857%); 2.4 MiB/s, 38 objects/s recovering 2023-06-03T20:13:08.279 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:13:08 smithi089 bash[78452]: audit 1685823187.1235962 mon.a (mon.0) 1056 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:13:08.279 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:13:08 smithi089 bash[78452]: audit 1685823187.1250184 mon.c (mon.2) 513 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T20:13:08.279 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:13:08 smithi089 bash[78452]: cluster 1685823187.741112 mon.a (mon.0) 1057 : cluster 3 Health check update: Degraded data redundancy: 44737/315396 objects degraded (14.184%), 14 pgs degraded, 33 pgs undersized (PG_DEGRADED) 2023-06-03T20:13:08.279 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:13:08 smithi089 bash[75068]: cluster 1685823186.7148776 mgr.y (mgr.34104) 1111 : cluster 0 pgmap v1084: 89 pgs: 9 active+clean+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 13 active+undersized+degraded, 2 active+recovering+undersized+remapped, 20 active+undersized, 44 active+clean; 6.4 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 102 B/s rd, 44 MiB/s wr, 712 op/s; 44737/315396 objects degraded (14.184%); 9012/315396 objects misplaced (2.857%); 2.4 MiB/s, 38 objects/s recovering 2023-06-03T20:13:08.279 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:13:08 smithi089 bash[75068]: audit 1685823187.1235962 mon.a (mon.0) 1056 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:13:08.280 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:13:08 smithi089 bash[75068]: audit 1685823187.1250184 mon.c (mon.2) 513 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T20:13:08.280 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:13:08 smithi089 bash[75068]: cluster 1685823187.741112 mon.a (mon.0) 1057 : cluster 3 Health check update: Degraded data redundancy: 44737/315396 objects degraded (14.184%), 14 pgs degraded, 33 pgs undersized (PG_DEGRADED) 2023-06-03T20:13:08.311 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:13:08.334 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:13:08 smithi164 bash[59462]: cluster 1685823186.7148776 mgr.y (mgr.34104) 1111 : cluster 0 pgmap v1084: 89 pgs: 9 active+clean+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 13 active+undersized+degraded, 2 active+recovering+undersized+remapped, 20 active+undersized, 44 active+clean; 6.4 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 102 B/s rd, 44 MiB/s wr, 712 op/s; 44737/315396 objects degraded (14.184%); 9012/315396 objects misplaced (2.857%); 2.4 MiB/s, 38 objects/s recovering 2023-06-03T20:13:08.334 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:13:08 smithi164 bash[59462]: audit 1685823187.1235962 mon.a (mon.0) 1056 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:13:08.334 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:13:08 smithi164 bash[59462]: audit 1685823187.1250184 mon.c (mon.2) 513 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T20:13:08.335 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:13:08 smithi164 bash[59462]: cluster 1685823187.741112 mon.a (mon.0) 1057 : cluster 3 Health check update: Degraded data redundancy: 44737/315396 objects degraded (14.184%), 14 pgs degraded, 33 pgs undersized (PG_DEGRADED) 2023-06-03T20:13:08.340 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:13:08.347 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:13:08.347 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:13:08.448 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:13:08.481 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:13:08.485 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:13:08.486 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:13:08.586 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:13:08.627 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:13:08.631 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:13:08.631 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:13:08.732 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:13:08.763 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:13:08.769 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:13:08.770 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:13:08.870 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:13:08.895 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:13:08.897 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:13:08.898 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:13:08.999 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:13:09.042 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:13:09.050 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:13:09.051 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:13:09.152 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:13:09.183 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:13:09.188 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:13:09.189 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:13:09.290 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:13:09.317 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:13:09.322 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:13:09.323 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:13:09.423 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:13:09.472 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:13:09.474 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:13:09.474 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:13:09.575 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:13:09.631 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:13:09.637 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:13:09.638 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:13:09.739 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:13:09.780 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:13:09.783 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:13:09.784 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:13:09.885 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:13:09.933 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:13:09.936 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:13:09.936 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:13:10.037 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:13:10.075 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:13:10.077 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:13:10.078 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:13:10.179 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:13:10.208 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:13:10.210 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:13:10.211 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:13:10.219 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/mon.a/config 2023-06-03T20:13:10.312 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:13:10.326 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:13:10 smithi089 bash[78452]: cluster 1685823188.7157652 mgr.y (mgr.34104) 1112 : cluster 0 pgmap v1085: 89 pgs: 9 active+clean+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 13 active+undersized+degraded, 2 active+recovering+undersized+remapped, 20 active+undersized, 44 active+clean; 6.3 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 87 B/s rd, 39 MiB/s wr, 778 op/s; 44523/311478 objects degraded (14.294%); 8787/311478 objects misplaced (2.821%); 3.2 MiB/s, 51 objects/s recovering 2023-06-03T20:13:10.327 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:13:10 smithi089 bash[75068]: cluster 1685823188.7157652 mgr.y (mgr.34104) 1112 : cluster 0 pgmap v1085: 89 pgs: 9 active+clean+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 13 active+undersized+degraded, 2 active+recovering+undersized+remapped, 20 active+undersized, 44 active+clean; 6.3 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 87 B/s rd, 39 MiB/s wr, 778 op/s; 44523/311478 objects degraded (14.294%); 8787/311478 objects misplaced (2.821%); 3.2 MiB/s, 51 objects/s recovering 2023-06-03T20:13:10.333 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:13:10 smithi164 bash[59462]: cluster 1685823188.7157652 mgr.y (mgr.34104) 1112 : cluster 0 pgmap v1085: 89 pgs: 9 active+clean+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 13 active+undersized+degraded, 2 active+recovering+undersized+remapped, 20 active+undersized, 44 active+clean; 6.3 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 87 B/s rd, 39 MiB/s wr, 778 op/s; 44523/311478 objects degraded (14.294%); 8787/311478 objects misplaced (2.821%); 3.2 MiB/s, 51 objects/s recovering 2023-06-03T20:13:10.350 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:13:10.352 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:13:10.353 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:13:10.454 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:13:10.483 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:13:10.485 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:13:10.486 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:13:10.587 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:13:10.612 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:13:10.613 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:13:10.614 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:13:10.715 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:13:10.749 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:13:10.756 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:13:10.756 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:13:10.857 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:13:10.907 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:13:10.910 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:13:10.911 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:13:11.012 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:13:11.043 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:13:11.045 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:13:11.045 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:13:11.147 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:13:11.181 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:13:11.184 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:13:11.185 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:13:11.286 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:13:11.331 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:13:11.335 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:13:11.336 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:13:11.438 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:13:11.468 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:13:11.473 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:13:11.474 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:13:11.575 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:13:11.607 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:13:11.615 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:13:11.616 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:13:11.718 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:13:11.771 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:13:11.772 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:13:11.773 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:13:11.873 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:13:11.906 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:13:11.908 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:13:11.909 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:13:12.009 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:13:12.035 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:13:12.037 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:13:12.037 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:13:12.138 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:13:12.143 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:13:12 smithi089 bash[78452]: cluster 1685823190.7164648 mgr.y (mgr.34104) 1113 : cluster 0 pgmap v1086: 89 pgs: 9 active+clean+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 13 active+undersized+degraded, 2 active+recovering+undersized+remapped, 20 active+undersized, 44 active+clean; 6.3 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 85 B/s rd, 25 MiB/s wr, 614 op/s; 43919/309012 objects degraded (14.213%); 8787/309012 objects misplaced (2.844%); 2.2 MiB/s, 34 objects/s recovering 2023-06-03T20:13:12.145 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:13:12 smithi089 bash[75068]: cluster 1685823190.7164648 mgr.y (mgr.34104) 1113 : cluster 0 pgmap v1086: 89 pgs: 9 active+clean+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 13 active+undersized+degraded, 2 active+recovering+undersized+remapped, 20 active+undersized, 44 active+clean; 6.3 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 85 B/s rd, 25 MiB/s wr, 614 op/s; 43919/309012 objects degraded (14.213%); 8787/309012 objects misplaced (2.844%); 2.2 MiB/s, 34 objects/s recovering 2023-06-03T20:13:12.178 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:13:12.182 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:13:12.182 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:13:12.284 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:13:12.331 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:13:12.333 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:13:12 smithi164 bash[59462]: cluster 1685823190.7164648 mgr.y (mgr.34104) 1113 : cluster 0 pgmap v1086: 89 pgs: 9 active+clean+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 13 active+undersized+degraded, 2 active+recovering+undersized+remapped, 20 active+undersized, 44 active+clean; 6.3 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 85 B/s rd, 25 MiB/s wr, 614 op/s; 43919/309012 objects degraded (14.213%); 8787/309012 objects misplaced (2.844%); 2.2 MiB/s, 34 objects/s recovering 2023-06-03T20:13:12.335 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:13:12.335 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:13:12.436 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:13:12.479 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:13:12.484 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:13:12.485 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:13:12.586 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:13:12.618 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:13:12.622 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:13:12.623 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:13:12.724 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:13:12.773 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:13:12.779 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:13:12.780 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:13:12.880 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:13:12.912 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:13:12.923 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:13:12.924 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:13:13.025 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:13:13.054 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:13:13.058 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:13:13.059 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:13:13.160 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:13:13.170 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:13:13 smithi164 bash[59462]: cluster 1685823192.7422478 mon.a (mon.0) 1058 : cluster 3 Health check update: Degraded data redundancy: 43919/309012 objects degraded (14.213%), 14 pgs degraded, 33 pgs undersized (PG_DEGRADED) 2023-06-03T20:13:13.192 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:13:13.194 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:13:13.195 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:13:13.296 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:13:13.306 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:13:13 smithi089 bash[75068]: cluster 1685823192.7422478 mon.a (mon.0) 1058 : cluster 3 Health check update: Degraded data redundancy: 43919/309012 objects degraded (14.213%), 14 pgs degraded, 33 pgs undersized (PG_DEGRADED) 2023-06-03T20:13:13.306 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:13:13 smithi089 bash[78452]: cluster 1685823192.7422478 mon.a (mon.0) 1058 : cluster 3 Health check update: Degraded data redundancy: 43919/309012 objects degraded (14.213%), 14 pgs degraded, 33 pgs undersized (PG_DEGRADED) 2023-06-03T20:13:13.330 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:13:13.333 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:13:13.334 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:13:13.434 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:13:13.463 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:13:13.466 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:13:13.467 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:13:13.568 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:13:13.597 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:13:13.601 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:13:13.602 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:13:13.703 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:13:13.732 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:13:13.734 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:13:13.735 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:13:13.836 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:13:13.864 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:13:13.866 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:13:13.867 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:13:13.968 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:13:14.001 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:13:14.005 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:13:14.005 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:13:14.106 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:13:14.110 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:13:14 smithi089 bash[75068]: cluster 1685823192.7172804 mgr.y (mgr.34104) 1114 : cluster 0 pgmap v1087: 89 pgs: 9 active+clean+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 13 active+undersized+degraded, 2 active+recovering+undersized+remapped, 20 active+undersized, 44 active+clean; 6.3 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 85 B/s rd, 20 MiB/s wr, 557 op/s; 43692/308331 objects degraded (14.170%); 8787/308331 objects misplaced (2.850%); 2.2 MiB/s, 34 objects/s recovering 2023-06-03T20:13:14.111 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:13:14 smithi089 bash[78452]: cluster 1685823192.7172804 mgr.y (mgr.34104) 1114 : cluster 0 pgmap v1087: 89 pgs: 9 active+clean+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 13 active+undersized+degraded, 2 active+recovering+undersized+remapped, 20 active+undersized, 44 active+clean; 6.3 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 85 B/s rd, 20 MiB/s wr, 557 op/s; 43692/308331 objects degraded (14.170%); 8787/308331 objects misplaced (2.850%); 2.2 MiB/s, 34 objects/s recovering 2023-06-03T20:13:14.146 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:13:14.150 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:13:14.151 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:13:14.251 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:13:14.260 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:13:14 smithi164 bash[59462]: cluster 1685823192.7172804 mgr.y (mgr.34104) 1114 : cluster 0 pgmap v1087: 89 pgs: 9 active+clean+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 13 active+undersized+degraded, 2 active+recovering+undersized+remapped, 20 active+undersized, 44 active+clean; 6.3 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 85 B/s rd, 20 MiB/s wr, 557 op/s; 43692/308331 objects degraded (14.170%); 8787/308331 objects misplaced (2.850%); 2.2 MiB/s, 34 objects/s recovering 2023-06-03T20:13:14.276 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:13:14.279 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:13:14.279 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:13:14.380 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:13:14.402 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:13:14.407 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:13:14.408 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:13:14.509 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:13:14.541 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:13:14.546 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:13:14.547 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:13:14.648 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:13:14.677 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:13:14.680 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:13:14.680 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:13:14.781 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:13:14.822 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:13:14.824 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:13:14.825 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:13:14.926 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:13:14.958 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:13:14.965 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:13:14.965 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:13:15.060 INFO:teuthology.orchestra.run.smithi089.stdout:{ 2023-06-03T20:13:15.061 INFO:teuthology.orchestra.run.smithi089.stdout: "ops": [ 2023-06-03T20:13:15.061 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:13:15.061 INFO:teuthology.orchestra.run.smithi089.stdout: "description": "osd_op(client.46187.0:113230 11.6 11:60960b79:::benchmark_data_smithi089_165412_object7702:head [delete] snapc 0=[] ondisk+write+known_if_redirected e201)", 2023-06-03T20:13:15.061 INFO:teuthology.orchestra.run.smithi089.stdout: "initiated_at": "2023-06-03T20:13:15.032732+0000", 2023-06-03T20:13:15.061 INFO:teuthology.orchestra.run.smithi089.stdout: "age": 0.023558322, 2023-06-03T20:13:15.061 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.023583007999999999, 2023-06-03T20:13:15.061 INFO:teuthology.orchestra.run.smithi089.stdout: "type_data": { 2023-06-03T20:13:15.062 INFO:teuthology.orchestra.run.smithi089.stdout: "flag_point": "waiting for sub ops", 2023-06-03T20:13:15.062 INFO:teuthology.orchestra.run.smithi089.stdout: "client_info": { 2023-06-03T20:13:15.062 INFO:teuthology.orchestra.run.smithi089.stdout: "client": "client.46187", 2023-06-03T20:13:15.062 INFO:teuthology.orchestra.run.smithi089.stdout: "client_addr": "172.21.15.89:0/880684210", 2023-06-03T20:13:15.062 INFO:teuthology.orchestra.run.smithi089.stdout: "tid": 113230 2023-06-03T20:13:15.062 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:13:15.062 INFO:teuthology.orchestra.run.smithi089.stdout: "events": [ 2023-06-03T20:13:15.062 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:13:15.063 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "initiated", 2023-06-03T20:13:15.063 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:13:15.032732+0000", 2023-06-03T20:13:15.063 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0 2023-06-03T20:13:15.063 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:13:15.063 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:13:15.063 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "throttled", 2023-06-03T20:13:15.063 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:13:15.032732+0000", 2023-06-03T20:13:15.064 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0 2023-06-03T20:13:15.064 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:13:15.064 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:13:15.064 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "header_read", 2023-06-03T20:13:15.064 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:13:15.032730+0000", 2023-06-03T20:13:15.064 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 4294967295.9999981 2023-06-03T20:13:15.064 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:13:15.064 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:13:15.065 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "all_read", 2023-06-03T20:13:15.065 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:13:15.032741+0000", 2023-06-03T20:13:15.065 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 1.0735e-05 2023-06-03T20:13:15.065 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:13:15.065 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:13:15.065 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "dispatched", 2023-06-03T20:13:15.065 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:13:15.032759+0000", 2023-06-03T20:13:15.066 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 1.8325000000000002e-05 2023-06-03T20:13:15.066 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:13:15.066 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:13:15.066 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "queued_for_pg", 2023-06-03T20:13:15.066 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:13:15.032771+0000", 2023-06-03T20:13:15.066 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 1.1943999999999999e-05 2023-06-03T20:13:15.066 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:13:15.067 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:13:15.067 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "reached_pg", 2023-06-03T20:13:15.067 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:13:15.032934+0000", 2023-06-03T20:13:15.067 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.00016303699999999999 2023-06-03T20:13:15.068 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:13:15.068 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:13:15.068 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "started", 2023-06-03T20:13:15.068 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:13:15.033069+0000", 2023-06-03T20:13:15.069 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.00013468399999999999 2023-06-03T20:13:15.069 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:13:15.069 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:13:15.069 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "waiting for subops from 2", 2023-06-03T20:13:15.070 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:13:15.033197+0000", 2023-06-03T20:13:15.070 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.000128663 2023-06-03T20:13:15.070 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:13:15.070 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:13:15.070 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "op_commit", 2023-06-03T20:13:15.070 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:13:15.036130+0000", 2023-06-03T20:13:15.070 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.0029326449999999998 2023-06-03T20:13:15.071 INFO:teuthology.orchestra.run.smithi089.stdout: } 2023-06-03T20:13:15.071 INFO:teuthology.orchestra.run.smithi089.stdout: ] 2023-06-03T20:13:15.071 INFO:teuthology.orchestra.run.smithi089.stdout: } 2023-06-03T20:13:15.071 INFO:teuthology.orchestra.run.smithi089.stdout: } 2023-06-03T20:13:15.071 INFO:teuthology.orchestra.run.smithi089.stdout: ], 2023-06-03T20:13:15.071 INFO:teuthology.orchestra.run.smithi089.stdout: "num_ops": 1 2023-06-03T20:13:15.071 INFO:teuthology.orchestra.run.smithi089.stdout:} 2023-06-03T20:13:15.072 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:13:15.106 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:13:15.109 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:13:15.110 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:13:15.210 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:13:15.246 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:13:15.250 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:13:15.251 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:13:15.352 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:13:15.375 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:13:15.376 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:13:15.377 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:13:15.478 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:13:15.516 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:13:15.522 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:13:15.523 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:13:15.624 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:13:15.680 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:13:15.685 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:13:15.686 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:13:15.787 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:13:15.816 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:13:15.821 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:13:15.822 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:13:15.922 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:13:15.959 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:13:15.962 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:13:15.962 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:13:16.063 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:13:16.088 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:13:16.095 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:13:16.096 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:13:16.197 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:13:16.206 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:13:16 smithi164 bash[59462]: cluster 1685823194.7182214 mgr.y (mgr.34104) 1115 : cluster 0 pgmap v1088: 89 pgs: 9 active+clean+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 13 active+undersized+degraded, 2 active+recovering+undersized+remapped, 20 active+undersized, 44 active+clean; 6.1 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 85 B/s rd, 19 MiB/s wr, 775 op/s; 42961/299787 objects degraded (14.331%); 8579/299787 objects misplaced (2.862%); 3.3 MiB/s, 52 objects/s recovering 2023-06-03T20:13:16.239 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:13:16.244 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:13:16.244 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:13:16.345 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:13:16.353 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:13:16 smithi089 bash[75068]: cluster 1685823194.7182214 mgr.y (mgr.34104) 1115 : cluster 0 pgmap v1088: 89 pgs: 9 active+clean+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 13 active+undersized+degraded, 2 active+recovering+undersized+remapped, 20 active+undersized, 44 active+clean; 6.1 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 85 B/s rd, 19 MiB/s wr, 775 op/s; 42961/299787 objects degraded (14.331%); 8579/299787 objects misplaced (2.862%); 3.3 MiB/s, 52 objects/s recovering 2023-06-03T20:13:16.353 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 20:13:16 smithi089 bash[71163]: ::ffff:172.21.15.164 - - [03/Jun/2023:20:13:16] "GET /metrics HTTP/1.1" 200 228951 "" "Prometheus/2.33.4" 2023-06-03T20:13:16.354 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:13:16 smithi089 bash[78452]: cluster 1685823194.7182214 mgr.y (mgr.34104) 1115 : cluster 0 pgmap v1088: 89 pgs: 9 active+clean+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 13 active+undersized+degraded, 2 active+recovering+undersized+remapped, 20 active+undersized, 44 active+clean; 6.1 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 85 B/s rd, 19 MiB/s wr, 775 op/s; 42961/299787 objects degraded (14.331%); 8579/299787 objects misplaced (2.862%); 3.3 MiB/s, 52 objects/s recovering 2023-06-03T20:13:16.384 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:13:16.386 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:13:16.387 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:13:16.488 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:13:16.524 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:13:16.526 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:13:16.527 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:13:16.627 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:13:16.659 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:13:16.662 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:13:16.662 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:13:16.764 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:13:16.786 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:13:16.787 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:13:16.788 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:13:16.889 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:13:16.930 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:13:16.933 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:13:16.933 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:13:17.034 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:13:17.065 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:13:17.071 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:13:17.071 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:13:17.172 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:13:17.201 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:13:17.204 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:13:17.205 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:13:17.306 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:13:17.349 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:13:17.353 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:13:17.354 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:13:17.455 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:13:17.488 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:13:17.490 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:13:17.490 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:13:17.592 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:13:17.623 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:13:17.628 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:13:17.629 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:13:17.730 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:13:17.762 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:13:17.765 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:13:17.766 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:13:17.867 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:13:17.905 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:13:17.908 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:13:17.909 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:13:18.010 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:13:18.044 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:13:18.051 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:13:18.052 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:13:18.153 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:13:18.193 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:13:18.195 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:13:18.196 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:13:18.297 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:13:18.320 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:13:18.323 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:13:18 smithi089 bash[78452]: cluster 1685823196.719086 mgr.y (mgr.34104) 1116 : cluster 0 pgmap v1089: 89 pgs: 9 active+clean+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 13 active+undersized+degraded, 2 active+recovering+undersized+remapped, 20 active+undersized, 44 active+clean; 6.0 GiB data, 19 GiB used, 606 GiB / 626 GiB avail; 85 B/s rd, 4.9 MiB/s wr, 624 op/s; 42243/296970 objects degraded (14.225%); 8579/296970 objects misplaced (2.889%); 2.3 MiB/s, 36 objects/s recovering 2023-06-03T20:13:18.323 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:13:18 smithi089 bash[78452]: cluster 1685823197.7433445 mon.a (mon.0) 1059 : cluster 3 Health check update: Degraded data redundancy: 42243/296970 objects degraded (14.225%), 14 pgs degraded, 33 pgs undersized (PG_DEGRADED) 2023-06-03T20:13:18.323 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:13:18.324 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:13:18.324 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:13:18 smithi089 bash[75068]: cluster 1685823196.719086 mgr.y (mgr.34104) 1116 : cluster 0 pgmap v1089: 89 pgs: 9 active+clean+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 13 active+undersized+degraded, 2 active+recovering+undersized+remapped, 20 active+undersized, 44 active+clean; 6.0 GiB data, 19 GiB used, 606 GiB / 626 GiB avail; 85 B/s rd, 4.9 MiB/s wr, 624 op/s; 42243/296970 objects degraded (14.225%); 8579/296970 objects misplaced (2.889%); 2.3 MiB/s, 36 objects/s recovering 2023-06-03T20:13:18.325 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:13:18 smithi089 bash[75068]: cluster 1685823197.7433445 mon.a (mon.0) 1059 : cluster 3 Health check update: Degraded data redundancy: 42243/296970 objects degraded (14.225%), 14 pgs degraded, 33 pgs undersized (PG_DEGRADED) 2023-06-03T20:13:18.425 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:13:18.455 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:13:18.461 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:13:18.461 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:13:18.563 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:13:18.583 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:13:18 smithi164 bash[59462]: cluster 1685823196.719086 mgr.y (mgr.34104) 1116 : cluster 0 pgmap v1089: 89 pgs: 9 active+clean+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 13 active+undersized+degraded, 2 active+recovering+undersized+remapped, 20 active+undersized, 44 active+clean; 6.0 GiB data, 19 GiB used, 606 GiB / 626 GiB avail; 85 B/s rd, 4.9 MiB/s wr, 624 op/s; 42243/296970 objects degraded (14.225%); 8579/296970 objects misplaced (2.889%); 2.3 MiB/s, 36 objects/s recovering 2023-06-03T20:13:18.583 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:13:18 smithi164 bash[59462]: cluster 1685823197.7433445 mon.a (mon.0) 1059 : cluster 3 Health check update: Degraded data redundancy: 42243/296970 objects degraded (14.225%), 14 pgs degraded, 33 pgs undersized (PG_DEGRADED) 2023-06-03T20:13:18.601 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:13:18.603 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:13:18.603 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:13:18.704 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:13:18.736 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:13:18.741 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:13:18.742 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:13:18.843 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:13:18.875 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:13:18.881 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:13:18.881 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:13:18.982 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:13:19.013 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:13:19.017 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:13:19.018 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:13:19.118 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:13:19.153 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:13:19.158 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:13:19.158 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:13:19.259 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:13:19.299 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:13:19.302 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:13:19.302 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:13:19.403 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:13:19.462 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:13:19.468 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:13:19.468 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:13:19.569 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:13:19.597 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:13:19.601 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:13:19.602 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:13:19.702 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:13:19.725 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:13:19.726 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:13:19.727 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:13:19.828 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:13:19.870 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:13:19.872 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:13:19.873 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:13:19.974 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:13:20.012 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:13:20.015 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:13:20.015 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:13:20.116 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:13:20.158 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:13:20.161 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:13:20.162 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:13:20.262 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:13:20.278 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:13:20 smithi089 bash[78452]: cluster 1685823198.7200758 mgr.y (mgr.34104) 1117 : cluster 0 pgmap v1090: 89 pgs: 9 active+clean+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 13 active+undersized+degraded, 2 active+recovering+undersized+remapped, 20 active+undersized, 44 active+clean; 5.9 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 1.7 MiB/s wr, 786 op/s; 41750/289083 objects degraded (14.442%); 8373/289083 objects misplaced (2.896%); 3.3 MiB/s, 53 objects/s recovering 2023-06-03T20:13:20.279 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:13:20 smithi089 bash[75068]: cluster 1685823198.7200758 mgr.y (mgr.34104) 1117 : cluster 0 pgmap v1090: 89 pgs: 9 active+clean+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 13 active+undersized+degraded, 2 active+recovering+undersized+remapped, 20 active+undersized, 44 active+clean; 5.9 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 1.7 MiB/s wr, 786 op/s; 41750/289083 objects degraded (14.442%); 8373/289083 objects misplaced (2.896%); 3.3 MiB/s, 53 objects/s recovering 2023-06-03T20:13:20.295 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:13:20.305 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:13:20.306 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:13:20.407 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:13:20.416 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:13:20 smithi164 bash[59462]: cluster 1685823198.7200758 mgr.y (mgr.34104) 1117 : cluster 0 pgmap v1090: 89 pgs: 9 active+clean+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 13 active+undersized+degraded, 2 active+recovering+undersized+remapped, 20 active+undersized, 44 active+clean; 5.9 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 1.7 MiB/s wr, 786 op/s; 41750/289083 objects degraded (14.442%); 8373/289083 objects misplaced (2.896%); 3.3 MiB/s, 53 objects/s recovering 2023-06-03T20:13:20.445 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:13:20.449 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:13:20.450 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:13:20.551 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:13:20.592 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:13:20.602 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:13:20.603 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:13:20.703 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:13:20.742 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:13:20.747 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:13:20.747 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:13:20.827 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph daemon osd.0 dump_blocked_ops 2023-06-03T20:13:20.848 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:13:20.918 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:13:20.929 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:13:20.930 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:13:21.030 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:13:21.068 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:13:21.069 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:13:21.070 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:13:21.171 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:13:21.216 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:13:21.221 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:13:21.222 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:13:21.323 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:13:21.362 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:13:21.364 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:13:21.365 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:13:21.466 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:13:21.501 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:13:21.505 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:13:21.506 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:13:21.607 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:13:21.637 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:13:21.642 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:13:21.643 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:13:21.743 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:13:21.787 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:13:21.790 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:13:21.791 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:13:21.892 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:13:21.923 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:13:21.929 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:13:21.929 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:13:22.031 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:13:22.084 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:13:22.089 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:13:22.090 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:13:22.191 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:13:22.200 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:13:22 smithi164 bash[59462]: cluster 1685823200.7209315 mgr.y (mgr.34104) 1118 : cluster 0 pgmap v1091: 89 pgs: 9 active+clean+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 13 active+undersized+degraded, 2 active+recovering+undersized+remapped, 20 active+undersized, 44 active+clean; 5.8 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 702 op/s; 41031/286188 objects degraded (14.337%); 8373/286188 objects misplaced (2.926%); 2.2 MiB/s, 34 objects/s recovering 2023-06-03T20:13:22.223 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:13:22.228 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:13:22.228 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:13:22.329 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:13:22.365 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:13:22.369 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:13:22 smithi089 bash[75068]: cluster 1685823200.7209315 mgr.y (mgr.34104) 1118 : cluster 0 pgmap v1091: 89 pgs: 9 active+clean+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 13 active+undersized+degraded, 2 active+recovering+undersized+remapped, 20 active+undersized, 44 active+clean; 5.8 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 702 op/s; 41031/286188 objects degraded (14.337%); 8373/286188 objects misplaced (2.926%); 2.2 MiB/s, 34 objects/s recovering 2023-06-03T20:13:22.369 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:13:22 smithi089 bash[78452]: cluster 1685823200.7209315 mgr.y (mgr.34104) 1118 : cluster 0 pgmap v1091: 89 pgs: 9 active+clean+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 13 active+undersized+degraded, 2 active+recovering+undersized+remapped, 20 active+undersized, 44 active+clean; 5.8 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 702 op/s; 41031/286188 objects degraded (14.337%); 8373/286188 objects misplaced (2.926%); 2.2 MiB/s, 34 objects/s recovering 2023-06-03T20:13:22.370 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:13:22.370 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:13:22.471 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:13:22.512 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:13:22.515 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:13:22.515 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:13:22.616 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:13:22.659 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:13:22.665 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:13:22.666 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:13:22.767 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:13:22.799 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:13:22.802 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:13:22.803 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:13:22.896 INFO:teuthology.orchestra.run.smithi089.stdout:osd.0: {} 2023-06-03T20:13:22.904 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:13:22.935 INFO:teuthology.orchestra.run.smithi089.stdout:osd.1: {} 2023-06-03T20:13:22.937 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:13:22.943 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:13:22.944 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:13:22.980 INFO:teuthology.orchestra.run.smithi089.stdout:osd.2: {} 2023-06-03T20:13:23.014 INFO:teuthology.orchestra.run.smithi089.stdout:osd.3: {} 2023-06-03T20:13:23.045 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:13:23.066 INFO:teuthology.orchestra.run.smithi089.stdout:osd.5: {} 2023-06-03T20:13:23.080 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:13:23.081 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:13:23.082 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:13:23.105 INFO:teuthology.orchestra.run.smithi089.stdout:osd.6: {} 2023-06-03T20:13:23.147 INFO:teuthology.orchestra.run.smithi089.stdout:osd.7: {} 2023-06-03T20:13:23.147 INFO:teuthology.orchestra.run.smithi089.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-06-03T20:13:23.147 INFO:teuthology.orchestra.run.smithi089.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-06-03T20:13:23.147 INFO:teuthology.orchestra.run.smithi089.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-06-03T20:13:23.147 INFO:teuthology.orchestra.run.smithi089.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-06-03T20:13:23.148 INFO:teuthology.orchestra.run.smithi089.stderr:Error ENXIO: problem getting command descriptions from osd.4 2023-06-03T20:13:23.148 INFO:teuthology.orchestra.run.smithi089.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-06-03T20:13:23.148 INFO:teuthology.orchestra.run.smithi089.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-06-03T20:13:23.148 INFO:teuthology.orchestra.run.smithi089.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-06-03T20:13:23.182 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:13:23.216 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:13:23.218 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:13:23.219 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:13:23.320 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:13:23.330 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:13:23 smithi164 bash[59462]: audit 1685823202.119569 mon.c (mon.2) 514 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T20:13:23.330 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:13:23 smithi164 bash[59462]: cluster 1685823202.744622 mon.a (mon.0) 1060 : cluster 3 Health check update: Degraded data redundancy: 41031/286188 objects degraded (14.337%), 14 pgs degraded, 33 pgs undersized (PG_DEGRADED) 2023-06-03T20:13:23.330 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:13:23 smithi164 bash[59462]: audit 1685823202.8420608 mon.b (mon.1) 57 : audit 0 from='client.? 172.21.15.89:0/1331034076' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-03T20:13:23.356 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:13:23.358 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:13:23.358 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:13:23.460 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:13:23.468 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:13:23 smithi089 bash[75068]: audit 1685823202.119569 mon.c (mon.2) 514 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T20:13:23.469 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:13:23 smithi089 bash[75068]: cluster 1685823202.744622 mon.a (mon.0) 1060 : cluster 3 Health check update: Degraded data redundancy: 41031/286188 objects degraded (14.337%), 14 pgs degraded, 33 pgs undersized (PG_DEGRADED) 2023-06-03T20:13:23.469 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:13:23 smithi089 bash[75068]: audit 1685823202.8420608 mon.b (mon.1) 57 : audit 0 from='client.? 172.21.15.89:0/1331034076' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-03T20:13:23.469 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:13:23 smithi089 bash[78452]: audit 1685823202.119569 mon.c (mon.2) 514 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T20:13:23.469 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:13:23 smithi089 bash[78452]: cluster 1685823202.744622 mon.a (mon.0) 1060 : cluster 3 Health check update: Degraded data redundancy: 41031/286188 objects degraded (14.337%), 14 pgs degraded, 33 pgs undersized (PG_DEGRADED) 2023-06-03T20:13:23.469 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:13:23 smithi089 bash[78452]: audit 1685823202.8420608 mon.b (mon.1) 57 : audit 0 from='client.? 172.21.15.89:0/1331034076' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-03T20:13:23.489 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:13:23.498 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:13:23.499 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:13:23.600 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:13:23.634 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:13:23.640 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:13:23.640 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:13:23.741 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:13:23.781 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:13:23.785 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:13:23.786 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:13:23.887 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:13:23.915 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:13:23.919 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:13:23.920 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:13:24.021 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:13:24.056 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:13:24.061 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:13:24.062 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:13:24.162 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:13:24.171 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:13:24 smithi164 bash[59462]: cluster 1685823202.7218075 mgr.y (mgr.34104) 1119 : cluster 0 pgmap v1092: 89 pgs: 9 active+clean+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 13 active+undersized+degraded, 2 active+recovering+undersized+remapped, 20 active+undersized, 44 active+clean; 5.8 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 653 op/s; 40796/285483 objects degraded (14.290%); 8373/285483 objects misplaced (2.933%); 2.2 MiB/s, 34 objects/s recovering 2023-06-03T20:13:24.209 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:13:24.213 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:13:24.213 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:13:24.314 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:13:24.352 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:13:24.358 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:13:24.358 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:13:24.459 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:13:24.469 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:13:24 smithi089 bash[75068]: cluster 1685823202.7218075 mgr.y (mgr.34104) 1119 : cluster 0 pgmap v1092: 89 pgs: 9 active+clean+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 13 active+undersized+degraded, 2 active+recovering+undersized+remapped, 20 active+undersized, 44 active+clean; 5.8 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 653 op/s; 40796/285483 objects degraded (14.290%); 8373/285483 objects misplaced (2.933%); 2.2 MiB/s, 34 objects/s recovering 2023-06-03T20:13:24.469 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:13:24 smithi089 bash[78452]: cluster 1685823202.7218075 mgr.y (mgr.34104) 1119 : cluster 0 pgmap v1092: 89 pgs: 9 active+clean+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 13 active+undersized+degraded, 2 active+recovering+undersized+remapped, 20 active+undersized, 44 active+clean; 5.8 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 653 op/s; 40796/285483 objects degraded (14.290%); 8373/285483 objects misplaced (2.933%); 2.2 MiB/s, 34 objects/s recovering 2023-06-03T20:13:24.515 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:13:24.517 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:13:24.518 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:13:24.619 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:13:24.768 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:13:24.776 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:13:24.776 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:13:24.877 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:13:24.925 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:13:24.928 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:13:24.929 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:13:25.029 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:13:25.068 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:13:25.070 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:13:25.071 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:13:25.172 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:13:25.206 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:13:25.208 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:13:25.209 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:13:25.309 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:13:25.350 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:13:25.352 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:13:25.353 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:13:25.454 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:13:25.487 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:13:25.494 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:13:25.494 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:13:25.528 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:13:25 smithi089 bash[78452]: cluster 1685823204.72276 mgr.y (mgr.34104) 1120 : cluster 0 pgmap v1093: 89 pgs: 9 active+clean+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 13 active+undersized+degraded, 2 active+recovering+undersized+remapped, 20 active+undersized, 44 active+clean; 5.6 GiB data, 18 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 864 op/s; 40094/277212 objects degraded (14.463%); 8168/277212 objects misplaced (2.946%); 3.2 MiB/s, 51 objects/s recovering 2023-06-03T20:13:25.529 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:13:25 smithi089 bash[75068]: cluster 1685823204.72276 mgr.y (mgr.34104) 1120 : cluster 0 pgmap v1093: 89 pgs: 9 active+clean+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 13 active+undersized+degraded, 2 active+recovering+undersized+remapped, 20 active+undersized, 44 active+clean; 5.6 GiB data, 18 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 864 op/s; 40094/277212 objects degraded (14.463%); 8168/277212 objects misplaced (2.946%); 3.2 MiB/s, 51 objects/s recovering 2023-06-03T20:13:25.583 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:13:25 smithi164 bash[59462]: cluster 1685823204.72276 mgr.y (mgr.34104) 1120 : cluster 0 pgmap v1093: 89 pgs: 9 active+clean+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 13 active+undersized+degraded, 2 active+recovering+undersized+remapped, 20 active+undersized, 44 active+clean; 5.6 GiB data, 18 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 864 op/s; 40094/277212 objects degraded (14.463%); 8168/277212 objects misplaced (2.946%); 3.2 MiB/s, 51 objects/s recovering 2023-06-03T20:13:25.596 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:13:25.625 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:13:25.629 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:13:25.629 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:13:25.730 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:13:25.776 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:13:25.779 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:13:25.779 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:13:25.880 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:13:25.920 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:13:25.921 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:13:25.922 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:13:25.993 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-06-03T20:13:25.994 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-06-03T20:13:26.023 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:13:26.053 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:13:26.058 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:13:26.059 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:13:26.160 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:13:26.195 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:13:26.199 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:13:26.200 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:13:26.278 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 20:13:26 smithi089 bash[71163]: ::ffff:172.21.15.164 - - [03/Jun/2023:20:13:26] "GET /metrics HTTP/1.1" 200 228936 "" "Prometheus/2.33.4" 2023-06-03T20:13:26.301 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:13:26.339 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:13:26.347 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:13:26.347 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:13:26.448 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:13:26.497 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:13:26.500 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:13:26.501 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:13:26.601 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:13:26.633 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:13:26.638 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:13:26.638 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:13:26.739 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:13:26.776 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:13:26.778 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:13:26.778 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:13:26.879 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:13:26.910 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:13:26.913 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:13:26.913 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:13:26.991 INFO:teuthology.orchestra.run.smithi089.stdout: 2023-06-03T20:13:27.004 INFO:teuthology.orchestra.run.smithi089.stderr:dumped all 2023-06-03T20:13:27.014 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:13:27.056 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:13:27.062 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:13:27.063 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:13:27.163 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:13:27.197 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:13:27.202 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:13:27.202 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:13:27.303 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:13:27.337 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:13:27.339 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:13:27.340 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:13:27.441 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:13:27.471 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:13:27.474 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:13:27.474 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:13:27.575 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:13:27.618 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:13:27.624 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:13:27.625 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:13:27.725 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:13:27.768 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:13:27.773 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:13:27.774 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:13:27.803 INFO:teuthology.orchestra.run.smithi089.stderr:noscrub is set 2023-06-03T20:13:27.843 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:13:27 smithi089 bash[78452]: cluster 1685823206.7236087 mgr.y (mgr.34104) 1121 : cluster 0 pgmap v1094: 89 pgs: 9 active+clean+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 13 active+undersized+degraded, 2 active+recovering+undersized+remapped, 20 active+undersized, 44 active+clean; 5.6 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 705 op/s; 39368/274377 objects degraded (14.348%); 8168/274377 objects misplaced (2.977%); 2.1 MiB/s, 34 objects/s recovering 2023-06-03T20:13:27.844 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:13:27 smithi089 bash[78452]: audit 1685823206.967441 mgr.y (mgr.34104) 1122 : audit 0 from='client.46256 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-06-03T20:13:27.844 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:13:27 smithi089 bash[78452]: audit 1685823206.9804485 mon.c (mon.2) 515 : audit 1 from='client.? 172.21.15.89:0/2597805336' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-06-03T20:13:27.844 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:13:27 smithi089 bash[78452]: audit 1685823206.9814985 mon.a (mon.0) 1061 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-06-03T20:13:27.844 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:13:27 smithi089 bash[78452]: cluster 1685823207.7457087 mon.a (mon.0) 1062 : cluster 3 Health check update: Degraded data redundancy: 39368/274377 objects degraded (14.348%), 14 pgs degraded, 33 pgs undersized (PG_DEGRADED) 2023-06-03T20:13:27.845 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:13:27 smithi089 bash[75068]: cluster 1685823206.7236087 mgr.y (mgr.34104) 1121 : cluster 0 pgmap v1094: 89 pgs: 9 active+clean+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 13 active+undersized+degraded, 2 active+recovering+undersized+remapped, 20 active+undersized, 44 active+clean; 5.6 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 705 op/s; 39368/274377 objects degraded (14.348%); 8168/274377 objects misplaced (2.977%); 2.1 MiB/s, 34 objects/s recovering 2023-06-03T20:13:27.845 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:13:27 smithi089 bash[75068]: audit 1685823206.967441 mgr.y (mgr.34104) 1122 : audit 0 from='client.46256 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-06-03T20:13:27.845 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:13:27 smithi089 bash[75068]: audit 1685823206.9804485 mon.c (mon.2) 515 : audit 1 from='client.? 172.21.15.89:0/2597805336' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-06-03T20:13:27.845 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:13:27 smithi089 bash[75068]: audit 1685823206.9814985 mon.a (mon.0) 1061 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-06-03T20:13:27.845 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:13:27 smithi089 bash[75068]: cluster 1685823207.7457087 mon.a (mon.0) 1062 : cluster 3 Health check update: Degraded data redundancy: 39368/274377 objects degraded (14.348%), 14 pgs degraded, 33 pgs undersized (PG_DEGRADED) 2023-06-03T20:13:27.875 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:13:27.904 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:13:27.907 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:13:27.908 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:13:27.995 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-06-03T20:13:28.009 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:13:28.084 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:13:27 smithi164 bash[59462]: cluster 1685823206.7236087 mgr.y (mgr.34104) 1121 : cluster 0 pgmap v1094: 89 pgs: 9 active+clean+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 13 active+undersized+degraded, 2 active+recovering+undersized+remapped, 20 active+undersized, 44 active+clean; 5.6 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 705 op/s; 39368/274377 objects degraded (14.348%); 8168/274377 objects misplaced (2.977%); 2.1 MiB/s, 34 objects/s recovering 2023-06-03T20:13:28.084 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:13:27 smithi164 bash[59462]: audit 1685823206.967441 mgr.y (mgr.34104) 1122 : audit 0 from='client.46256 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-06-03T20:13:28.084 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:13:27 smithi164 bash[59462]: audit 1685823206.9804485 mon.c (mon.2) 515 : audit 1 from='client.? 172.21.15.89:0/2597805336' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-06-03T20:13:28.084 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:13:27 smithi164 bash[59462]: audit 1685823206.9814985 mon.a (mon.0) 1061 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-06-03T20:13:28.085 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:13:27 smithi164 bash[59462]: cluster 1685823207.7457087 mon.a (mon.0) 1062 : cluster 3 Health check update: Degraded data redundancy: 39368/274377 objects degraded (14.348%), 14 pgs degraded, 33 pgs undersized (PG_DEGRADED) 2023-06-03T20:13:28.088 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:13:28.093 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:13:28.094 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:13:28.195 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:13:28.223 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:13:28.227 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:13:28.228 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:13:28.329 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:13:28.360 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:13:28.361 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:13:28.362 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:13:28.463 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:13:28.502 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:13:28.506 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:13:28.507 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:13:28.608 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:13:28.643 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:13:28.645 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:13:28.646 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:13:28.746 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:13:28.780 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:13:28.782 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:13:28.783 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:13:28.884 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:13:28.893 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:13:28 smithi164 bash[59462]: cluster 1685823207.781069 mon.a (mon.0) 1063 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-06-03T20:13:28.893 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:13:28 smithi164 bash[59462]: cluster 1685823207.7824767 osd.6 (osd.6) 147 : cluster 3 failed to encode map e202 with expected crc 2023-06-03T20:13:28.893 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:13:28 smithi164 bash[59462]: audit 1685823207.7866557 mon.a (mon.0) 1064 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-06-03T20:13:28.893 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:13:28 smithi164 bash[59462]: cluster 1685823207.7867327 mon.a (mon.0) 1065 : cluster 0 osdmap e202: 8 total, 7 up, 7 in 2023-06-03T20:13:28.893 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:13:28 smithi164 bash[59462]: cluster 1685823207.7914844 osd.5 (osd.5) 134 : cluster 3 failed to encode map e202 with expected crc 2023-06-03T20:13:28.911 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:13:28.916 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:13:28.916 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:13:29.017 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:13:29.043 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:13:29.048 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:13:29.049 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:13:29.150 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:13:29.158 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:13:28 smithi089 bash[75068]: cluster 1685823207.781069 mon.a (mon.0) 1063 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-06-03T20:13:29.159 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:13:28 smithi089 bash[75068]: cluster 1685823207.7824767 osd.6 (osd.6) 147 : cluster 3 failed to encode map e202 with expected crc 2023-06-03T20:13:29.159 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:13:28 smithi089 bash[75068]: audit 1685823207.7866557 mon.a (mon.0) 1064 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-06-03T20:13:29.159 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:13:28 smithi089 bash[75068]: cluster 1685823207.7867327 mon.a (mon.0) 1065 : cluster 0 osdmap e202: 8 total, 7 up, 7 in 2023-06-03T20:13:29.159 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:13:28 smithi089 bash[75068]: cluster 1685823207.7914844 osd.5 (osd.5) 134 : cluster 3 failed to encode map e202 with expected crc 2023-06-03T20:13:29.160 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:13:28 smithi089 bash[78452]: cluster 1685823207.781069 mon.a (mon.0) 1063 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-06-03T20:13:29.160 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:13:28 smithi089 bash[78452]: cluster 1685823207.7824767 osd.6 (osd.6) 147 : cluster 3 failed to encode map e202 with expected crc 2023-06-03T20:13:29.160 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:13:28 smithi089 bash[78452]: audit 1685823207.7866557 mon.a (mon.0) 1064 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-06-03T20:13:29.160 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:13:28 smithi089 bash[78452]: cluster 1685823207.7867327 mon.a (mon.0) 1065 : cluster 0 osdmap e202: 8 total, 7 up, 7 in 2023-06-03T20:13:29.161 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:13:28 smithi089 bash[78452]: cluster 1685823207.7914844 osd.5 (osd.5) 134 : cluster 3 failed to encode map e202 with expected crc 2023-06-03T20:13:29.190 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:13:29.193 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:13:29.193 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:13:29.228 INFO:teuthology.orchestra.run.smithi089.stdout:{"pg_ready":true,"pg_map":{"version":1094,"stamp":"2023-06-03T20:13:26.722942+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":5992814502,"num_objects":91459,"num_object_clones":0,"num_object_copies":274377,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":39368,"num_objects_misplaced":8168,"num_objects_unfound":0,"num_objects_dirty":91459,"num_whiteouts":0,"num_read":309,"num_read_kb":513,"num_write":119878,"num_write_kb":6756047,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2016,"num_bytes_recovered":131792896,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":110112,"ondisk_log_size":110112,"up":225,"acting":231,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":278,"num_osds":7,"num_per_pool_osds":7,"num_per_pool_omap_osds":7,"kb":656216064,"kb_used":19147104,"kb_used_data":16369640,"kb_used_omap":26,"kb_used_meta":2777253,"kb_avail":637068960,"statfs":{"total":671965249536,"available":652358615040,"internally_reserved":0,"allocated":16762511360,"data_stored":16758023174,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":27292,"internal_metadata":2843907428},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,1,2,4,2,0,2,2],"upper_bound":256},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-555089920,"num_objects":-8470,"num_object_clones":0,"num_object_copies":-25410,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":-3593,"num_objects_misplaced":-411,"num_objects_unfound":0,"num_objects_dirty":-8470,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8470,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":411,"num_bytes_recovered":26935296,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.005480"},"pg_stats":[{"pgid":"3.1a","version":"60'1","reported_seq":14,"reported_epoch":200,"state":"active+undersized","last_fresh":"2023-06-03T20:12:32.093246+0000","last_change":"2023-06-03T20:11:54.019728+0000","last_active":"2023-06-03T20:12:32.093246+0000","last_peered":"2023-06-03T20:12:32.093246+0000","last_clean":"2023-06-03T19:34:15.767991+0000","last_became_active":"2023-06-03T20:11:54.019728+0000","last_became_peered":"2023-06-03T20:11:54.019728+0000","last_unstale":"2023-06-03T20:12:32.093246+0000","last_undegraded":"2023-06-03T20:12:32.093246+0000","last_fullsized":"2023-06-03T20:11:54.009512+0000","mapping_epoch":195,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T19:58:26.738637+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":["2","0"],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":14,"reported_epoch":200,"state":"active+clean+remapped","last_fresh":"2023-06-03T20:12:32.546798+0000","last_change":"2023-06-03T20:12:31.911399+0000","last_active":"2023-06-03T20:12:32.546798+0000","last_peered":"2023-06-03T20:12:32.546798+0000","last_clean":"2023-06-03T20:12:32.546798+0000","last_became_active":"2023-06-03T20:12:31.911029+0000","last_became_peered":"2023-06-03T20:12:31.911029+0000","last_unstale":"2023-06-03T20:12:32.546798+0000","last_undegraded":"2023-06-03T20:12:32.546798+0000","last_fullsized":"2023-06-03T20:12:32.546798+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T05:04:34.770935+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5],"acting":[3,5,6],"avail_no_missing":["3","5","6"],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":161,"reported_epoch":200,"state":"active+undersized","last_fresh":"2023-06-03T20:12:31.897773+0000","last_change":"2023-06-03T20:11:54.017045+0000","last_active":"2023-06-03T20:12:31.897773+0000","last_peered":"2023-06-03T20:12:31.897773+0000","last_clean":"2023-06-03T20:11:43.978421+0000","last_became_active":"2023-06-03T20:11:54.017045+0000","last_became_peered":"2023-06-03T20:11:54.017045+0000","last_unstale":"2023-06-03T20:12:31.897773+0000","last_undegraded":"2023-06-03T20:12:31.897773+0000","last_fullsized":"2023-06-03T20:11:54.008056+0000","mapping_epoch":195,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T23:56:32.827460+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6],"acting":[3,6],"avail_no_missing":["3","6"],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":164,"reported_epoch":201,"state":"active+clean","last_fresh":"2023-06-03T20:12:57.050846+0000","last_change":"2023-06-03T19:56:54.507653+0000","last_active":"2023-06-03T20:12:57.050846+0000","last_peered":"2023-06-03T20:12:57.050846+0000","last_clean":"2023-06-03T20:12:57.050846+0000","last_became_active":"2023-06-03T19:56:54.507350+0000","last_became_peered":"2023-06-03T19:56:54.507350+0000","last_unstale":"2023-06-03T20:12:57.050846+0000","last_undegraded":"2023-06-03T20:12:57.050846+0000","last_fullsized":"2023-06-03T20:12:57.050846+0000","mapping_epoch":119,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T22:33:20.065844+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":195,"reported_epoch":200,"state":"active+clean","last_fresh":"2023-06-03T20:12:31.897916+0000","last_change":"2023-06-03T20:07:19.912004+0000","last_active":"2023-06-03T20:12:31.897916+0000","last_peered":"2023-06-03T20:12:31.897916+0000","last_clean":"2023-06-03T20:12:31.897916+0000","last_became_active":"2023-06-03T20:07:19.910895+0000","last_became_peered":"2023-06-03T20:07:19.910895+0000","last_unstale":"2023-06-03T20:12:31.897916+0000","last_undegraded":"2023-06-03T20:12:31.897916+0000","last_fullsized":"2023-06-03T20:12:31.897916+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T06:46:44.564002+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,2],"acting":[3,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":176,"reported_epoch":200,"state":"active+clean","last_fresh":"2023-06-03T20:12:31.906717+0000","last_change":"2023-06-03T20:07:19.918531+0000","last_active":"2023-06-03T20:12:31.906717+0000","last_peered":"2023-06-03T20:12:31.906717+0000","last_clean":"2023-06-03T20:12:31.906717+0000","last_became_active":"2023-06-03T20:07:19.918173+0000","last_became_peered":"2023-06-03T20:07:19.918173+0000","last_unstale":"2023-06-03T20:12:31.906717+0000","last_undegraded":"2023-06-03T20:12:31.906717+0000","last_fullsized":"2023-06-03T20:12:31.906717+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,0],"acting":[6,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":160,"reported_epoch":200,"state":"active+clean","last_fresh":"2023-06-03T20:12:31.940683+0000","last_change":"2023-06-03T19:56:54.509490+0000","last_active":"2023-06-03T20:12:31.940683+0000","last_peered":"2023-06-03T20:12:31.940683+0000","last_clean":"2023-06-03T20:12:31.940683+0000","last_became_active":"2023-06-03T19:56:54.509145+0000","last_became_peered":"2023-06-03T19:56:54.509145+0000","last_unstale":"2023-06-03T20:12:31.940683+0000","last_undegraded":"2023-06-03T20:12:31.940683+0000","last_fullsized":"2023-06-03T20:12:31.940683+0000","mapping_epoch":119,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T07:12:03.964110+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":196,"reported_epoch":201,"state":"active+clean+remapped","last_fresh":"2023-06-03T20:12:57.042409+0000","last_change":"2023-06-03T20:12:31.937956+0000","last_active":"2023-06-03T20:12:57.042409+0000","last_peered":"2023-06-03T20:12:57.042409+0000","last_clean":"2023-06-03T20:12:57.042409+0000","last_became_active":"2023-06-03T20:12:31.937551+0000","last_became_peered":"2023-06-03T20:12:31.937551+0000","last_unstale":"2023-06-03T20:12:57.042409+0000","last_undegraded":"2023-06-03T20:12:57.042409+0000","last_fullsized":"2023-06-03T20:12:57.042409+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T21:49:12.058781+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5],"acting":[3,5,6],"avail_no_missing":["3","5","6"],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":177,"reported_epoch":200,"state":"active+clean","last_fresh":"2023-06-03T20:12:31.903290+0000","last_change":"2023-06-03T20:07:19.912362+0000","last_active":"2023-06-03T20:12:31.903290+0000","last_peered":"2023-06-03T20:12:31.903290+0000","last_clean":"2023-06-03T20:12:31.903290+0000","last_became_active":"2023-06-03T20:07:19.907385+0000","last_became_peered":"2023-06-03T20:07:19.907385+0000","last_unstale":"2023-06-03T20:12:31.903290+0000","last_undegraded":"2023-06-03T20:12:31.903290+0000","last_fullsized":"2023-06-03T20:12:31.903290+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":152,"reported_epoch":201,"state":"active+clean","last_fresh":"2023-06-03T20:12:57.038143+0000","last_change":"2023-06-03T19:56:54.510462+0000","last_active":"2023-06-03T20:12:57.038143+0000","last_peered":"2023-06-03T20:12:57.038143+0000","last_clean":"2023-06-03T20:12:57.038143+0000","last_became_active":"2023-06-03T19:56:54.509783+0000","last_became_peered":"2023-06-03T19:56:54.509783+0000","last_unstale":"2023-06-03T20:12:57.038143+0000","last_undegraded":"2023-06-03T20:12:57.038143+0000","last_fullsized":"2023-06-03T20:12:57.038143+0000","mapping_epoch":119,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T00:16:09.624276+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"60'3","reported_seq":189,"reported_epoch":200,"state":"active+undersized+degraded","last_fresh":"2023-06-03T20:12:31.900977+0000","last_change":"2023-06-03T20:11:54.025696+0000","last_active":"2023-06-03T20:12:31.900977+0000","last_peered":"2023-06-03T20:12:31.900977+0000","last_clean":"2023-06-03T20:11:43.973370+0000","last_became_active":"2023-06-03T20:11:54.025696+0000","last_became_peered":"2023-06-03T20:11:54.025696+0000","last_unstale":"2023-06-03T20:12:31.900977+0000","last_undegraded":"2023-06-03T20:11:54.012202+0000","last_fullsized":"2023-06-03T20:11:54.012054+0000","mapping_epoch":195,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":173,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3],"acting":[7,3],"avail_no_missing":["7","3"],"object_location_counts":[{"shards":"3,7","objects":2}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":57,"reported_epoch":200,"state":"active+clean","last_fresh":"2023-06-03T20:12:34.791444+0000","last_change":"2023-06-03T20:12:30.911538+0000","last_active":"2023-06-03T20:12:34.791444+0000","last_peered":"2023-06-03T20:12:34.791444+0000","last_clean":"2023-06-03T20:12:34.791444+0000","last_became_active":"2023-06-03T20:12:30.911082+0000","last_became_peered":"2023-06-03T20:12:30.911082+0000","last_unstale":"2023-06-03T20:12:34.791444+0000","last_undegraded":"2023-06-03T20:12:34.791444+0000","last_fullsized":"2023-06-03T20:12:34.791444+0000","mapping_epoch":198,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":199,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T05:22:01.454415+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,5],"acting":[2,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.14","version":"77'23","reported_seq":71,"reported_epoch":200,"state":"active+undersized+degraded","last_fresh":"2023-06-03T20:12:31.899969+0000","last_change":"2023-06-03T20:11:54.018006+0000","last_active":"2023-06-03T20:12:31.899969+0000","last_peered":"2023-06-03T20:12:31.899969+0000","last_clean":"2023-06-03T19:41:47.150753+0000","last_became_active":"2023-06-03T20:11:54.018006+0000","last_became_peered":"2023-06-03T20:11:54.018006+0000","last_unstale":"2023-06-03T20:12:31.899969+0000","last_undegraded":"2023-06-03T20:11:54.016070+0000","last_fullsized":"2023-06-03T20:11:54.015933+0000","mapping_epoch":195,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":23,"log_dups_size":0,"ondisk_log_size":23,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4716,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":23,"num_write_kb":20,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6],"acting":[7,6],"avail_no_missing":["7","6"],"object_location_counts":[{"shards":"6,7","objects":4}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":143,"reported_epoch":200,"state":"active+undersized","last_fresh":"2023-06-03T20:12:31.907429+0000","last_change":"2023-06-03T20:11:54.013229+0000","last_active":"2023-06-03T20:12:31.907429+0000","last_peered":"2023-06-03T20:12:31.907429+0000","last_clean":"2023-06-03T20:07:14.870826+0000","last_became_active":"2023-06-03T20:11:54.013229+0000","last_became_peered":"2023-06-03T20:11:54.013229+0000","last_unstale":"2023-06-03T20:12:31.907429+0000","last_undegraded":"2023-06-03T20:12:31.907429+0000","last_fullsized":"2023-06-03T20:11:54.007837+0000","mapping_epoch":195,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5],"acting":[6,5],"avail_no_missing":["6","5"],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.13","version":"60'2","reported_seq":191,"reported_epoch":200,"state":"active+undersized+degraded","last_fresh":"2023-06-03T20:12:31.900009+0000","last_change":"2023-06-03T20:11:54.019051+0000","last_active":"2023-06-03T20:12:31.900009+0000","last_peered":"2023-06-03T20:12:31.900009+0000","last_clean":"2023-06-03T20:11:43.972488+0000","last_became_active":"2023-06-03T20:11:54.019051+0000","last_became_peered":"2023-06-03T20:11:54.019051+0000","last_unstale":"2023-06-03T20:12:31.900009+0000","last_undegraded":"2023-06-03T20:11:54.012366+0000","last_fullsized":"2023-06-03T20:11:54.012217+0000","mapping_epoch":195,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2],"acting":[7,2],"avail_no_missing":["7","2"],"object_location_counts":[{"shards":"2,7","objects":2}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":180,"reported_epoch":201,"state":"active+clean","last_fresh":"2023-06-03T20:12:57.040661+0000","last_change":"2023-06-03T20:07:19.907827+0000","last_active":"2023-06-03T20:12:57.040661+0000","last_peered":"2023-06-03T20:12:57.040661+0000","last_clean":"2023-06-03T20:12:57.040661+0000","last_became_active":"2023-06-03T20:07:19.907296+0000","last_became_peered":"2023-06-03T20:07:19.907296+0000","last_unstale":"2023-06-03T20:12:57.040661+0000","last_undegraded":"2023-06-03T20:12:57.040661+0000","last_fullsized":"2023-06-03T20:12:57.040661+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T00:34:13.230147+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":161,"reported_epoch":200,"state":"active+clean","last_fresh":"2023-06-03T20:12:45.843659+0000","last_change":"2023-06-03T19:56:54.507479+0000","last_active":"2023-06-03T20:12:45.843659+0000","last_peered":"2023-06-03T20:12:45.843659+0000","last_clean":"2023-06-03T20:12:45.843659+0000","last_became_active":"2023-06-03T19:56:54.507149+0000","last_became_peered":"2023-06-03T19:56:54.507149+0000","last_unstale":"2023-06-03T20:12:45.843659+0000","last_undegraded":"2023-06-03T20:12:45.843659+0000","last_fullsized":"2023-06-03T20:12:45.843659+0000","mapping_epoch":119,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T20:22:36.676317+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":167,"reported_epoch":200,"state":"active+clean","last_fresh":"2023-06-03T20:12:31.904059+0000","last_change":"2023-06-03T19:56:54.512472+0000","last_active":"2023-06-03T20:12:31.904059+0000","last_peered":"2023-06-03T20:12:31.904059+0000","last_clean":"2023-06-03T20:12:31.904059+0000","last_became_active":"2023-06-03T19:56:54.512051+0000","last_became_peered":"2023-06-03T19:56:54.512051+0000","last_unstale":"2023-06-03T20:12:31.904059+0000","last_undegraded":"2023-06-03T20:12:31.904059+0000","last_fullsized":"2023-06-03T20:12:31.904059+0000","mapping_epoch":119,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":171,"reported_epoch":200,"state":"active+undersized","last_fresh":"2023-06-03T20:12:31.899310+0000","last_change":"2023-06-03T20:11:54.014913+0000","last_active":"2023-06-03T20:12:31.899310+0000","last_peered":"2023-06-03T20:12:31.899310+0000","last_clean":"2023-06-03T20:11:43.971870+0000","last_became_active":"2023-06-03T20:11:54.014913+0000","last_became_peered":"2023-06-03T20:11:54.014913+0000","last_unstale":"2023-06-03T20:12:31.899310+0000","last_undegraded":"2023-06-03T20:12:31.899310+0000","last_fullsized":"2023-06-03T20:11:54.009603+0000","mapping_epoch":195,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":173,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6],"acting":[7,6],"avail_no_missing":["7","6"],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":161,"reported_epoch":200,"state":"active+clean","last_fresh":"2023-06-03T20:12:31.940602+0000","last_change":"2023-06-03T19:56:54.510289+0000","last_active":"2023-06-03T20:12:31.940602+0000","last_peered":"2023-06-03T20:12:31.940602+0000","last_clean":"2023-06-03T20:12:31.940602+0000","last_became_active":"2023-06-03T19:56:54.509905+0000","last_became_peered":"2023-06-03T19:56:54.509905+0000","last_unstale":"2023-06-03T20:12:31.940602+0000","last_undegraded":"2023-06-03T20:12:31.940602+0000","last_fullsized":"2023-06-03T20:12:31.940602+0000","mapping_epoch":119,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T02:54:26.792401+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"60'8","reported_seq":176,"reported_epoch":200,"state":"active+clean","last_fresh":"2023-06-03T20:12:31.906953+0000","last_change":"2023-06-03T19:47:06.727778+0000","last_active":"2023-06-03T20:12:31.906953+0000","last_peered":"2023-06-03T20:12:31.906953+0000","last_clean":"2023-06-03T20:12:31.906953+0000","last_became_active":"2023-06-03T19:47:06.727305+0000","last_became_peered":"2023-06-03T19:47:06.727305+0000","last_unstale":"2023-06-03T20:12:31.906953+0000","last_undegraded":"2023-06-03T20:12:31.906953+0000","last_fullsized":"2023-06-03T20:12:31.906953+0000","mapping_epoch":86,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":87,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":179,"reported_epoch":200,"state":"active+clean","last_fresh":"2023-06-03T20:12:31.898920+0000","last_change":"2023-06-03T20:07:19.907151+0000","last_active":"2023-06-03T20:12:31.898920+0000","last_peered":"2023-06-03T20:12:31.898920+0000","last_clean":"2023-06-03T20:12:31.898920+0000","last_became_active":"2023-06-03T20:07:19.906479+0000","last_became_peered":"2023-06-03T20:07:19.906479+0000","last_unstale":"2023-06-03T20:12:31.898920+0000","last_undegraded":"2023-06-03T20:12:31.898920+0000","last_fullsized":"2023-06-03T20:12:31.898920+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T03:18:22.525692+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"11.7","version":"201'7568","reported_seq":7594,"reported_epoch":201,"state":"active+undersized+degraded","last_fresh":"2023-06-03T20:13:23.660266+0000","last_change":"2023-06-03T20:11:54.017911+0000","last_active":"2023-06-03T20:13:23.660266+0000","last_peered":"2023-06-03T20:13:23.660266+0000","last_clean":"2023-06-03T20:11:53.003160+0000","last_became_active":"2023-06-03T20:11:54.017911+0000","last_became_peered":"2023-06-03T20:11:54.017911+0000","last_unstale":"2023-06-03T20:13:23.660266+0000","last_undegraded":"2023-06-03T20:11:54.010796+0000","last_fullsized":"2023-06-03T20:11:54.010661+0000","mapping_epoch":195,"log_start":"0'0","ondisk_log_start":"0'0","created":189,"last_epoch_clean":190,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T20:11:10.759955+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T20:11:10.759955+0000","last_clean_scrub_stamp":"2023-06-03T20:11:10.759955+0000","objects_scrubbed":0,"log_size":7568,"log_dups_size":0,"ondisk_log_size":7568,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":380633088,"num_objects":5808,"num_object_clones":0,"num_object_copies":17424,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5808,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5808,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7568,"num_write_kb":428032,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0],"acting":[5,0],"avail_no_missing":["5","0"],"object_location_counts":[{"shards":"0,5","objects":5808}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"60'1","reported_seq":190,"reported_epoch":200,"state":"active+undersized+degraded","last_fresh":"2023-06-03T20:12:31.900241+0000","last_change":"2023-06-03T20:11:54.016915+0000","last_active":"2023-06-03T20:12:31.900241+0000","last_peered":"2023-06-03T20:12:31.900241+0000","last_clean":"2023-06-03T20:11:43.972655+0000","last_became_active":"2023-06-03T20:11:54.016915+0000","last_became_peered":"2023-06-03T20:11:54.016915+0000","last_unstale":"2023-06-03T20:12:31.900241+0000","last_undegraded":"2023-06-03T20:11:54.009884+0000","last_fullsized":"2023-06-03T20:11:54.009713+0000","mapping_epoch":195,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0],"acting":[7,0],"avail_no_missing":["7","0"],"object_location_counts":[{"shards":"0,7","objects":1}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":162,"reported_epoch":200,"state":"active+clean+remapped","last_fresh":"2023-06-03T20:12:31.911263+0000","last_change":"2023-06-03T20:12:31.910410+0000","last_active":"2023-06-03T20:12:31.911263+0000","last_peered":"2023-06-03T20:12:31.911263+0000","last_clean":"2023-06-03T20:12:31.911263+0000","last_became_active":"2023-06-03T20:12:31.909769+0000","last_became_peered":"2023-06-03T20:12:31.909769+0000","last_unstale":"2023-06-03T20:12:31.911263+0000","last_undegraded":"2023-06-03T20:12:31.911263+0000","last_fullsized":"2023-06-03T20:12:31.911263+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7],"acting":[5,7,6],"avail_no_missing":["5","6","7"],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"11.6","version":"201'7541","reported_seq":7567,"reported_epoch":201,"state":"active+undersized+degraded","last_fresh":"2023-06-03T20:13:25.944900+0000","last_change":"2023-06-03T20:11:54.020693+0000","last_active":"2023-06-03T20:13:25.944900+0000","last_peered":"2023-06-03T20:13:25.944900+0000","last_clean":"2023-06-03T20:11:53.009007+0000","last_became_active":"2023-06-03T20:11:54.020693+0000","last_became_peered":"2023-06-03T20:11:54.020693+0000","last_unstale":"2023-06-03T20:13:25.944900+0000","last_undegraded":"2023-06-03T20:11:54.011201+0000","last_fullsized":"2023-06-03T20:11:54.011064+0000","mapping_epoch":195,"log_start":"193'1100","ondisk_log_start":"193'1100","created":189,"last_epoch_clean":190,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T20:11:10.759955+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T20:11:10.759955+0000","last_clean_scrub_stamp":"2023-06-03T20:11:10.759955+0000","objects_scrubbed":0,"log_size":6441,"log_dups_size":0,"ondisk_log_size":6441,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T23:28:23.377635+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":365494272,"num_objects":5577,"num_object_clones":0,"num_object_copies":16731,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5577,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5577,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7541,"num_write_kb":419776,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2],"acting":[0,2],"avail_no_missing":["0","2"],"object_location_counts":[{"shards":"0,2","objects":5577}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.e","version":"60'2","reported_seq":204,"reported_epoch":200,"state":"active+undersized+degraded","last_fresh":"2023-06-03T20:12:31.900843+0000","last_change":"2023-06-03T20:11:54.012607+0000","last_active":"2023-06-03T20:12:31.900843+0000","last_peered":"2023-06-03T20:12:31.900843+0000","last_clean":"2023-06-03T20:11:43.972261+0000","last_became_active":"2023-06-03T20:11:54.012607+0000","last_became_peered":"2023-06-03T20:11:54.012607+0000","last_unstale":"2023-06-03T20:12:31.900843+0000","last_undegraded":"2023-06-03T20:11:54.010373+0000","last_fullsized":"2023-06-03T20:11:54.010224+0000","mapping_epoch":195,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0],"acting":[7,0],"avail_no_missing":["7","0"],"object_location_counts":[{"shards":"0,7","objects":1}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":191,"reported_epoch":200,"state":"active+clean+remapped","last_fresh":"2023-06-03T20:12:31.943652+0000","last_change":"2023-06-03T20:12:31.941492+0000","last_active":"2023-06-03T20:12:31.943652+0000","last_peered":"2023-06-03T20:12:31.943652+0000","last_clean":"2023-06-03T20:12:31.943652+0000","last_became_active":"2023-06-03T20:12:31.941076+0000","last_became_peered":"2023-06-03T20:12:31.941076+0000","last_unstale":"2023-06-03T20:12:31.943652+0000","last_undegraded":"2023-06-03T20:12:31.943652+0000","last_fullsized":"2023-06-03T20:12:31.943652+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T03:59:11.007424+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0],"acting":[3,0,6],"avail_no_missing":["3","0","6"],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"11.5","version":"201'7491","reported_seq":7511,"reported_epoch":201,"state":"active+clean","last_fresh":"2023-06-03T20:13:23.050082+0000","last_change":"2023-06-03T20:11:11.796552+0000","last_active":"2023-06-03T20:13:23.050082+0000","last_peered":"2023-06-03T20:13:23.050082+0000","last_clean":"2023-06-03T20:13:23.050082+0000","last_became_active":"2023-06-03T20:11:11.796144+0000","last_became_peered":"2023-06-03T20:11:11.796144+0000","last_unstale":"2023-06-03T20:13:23.050082+0000","last_undegraded":"2023-06-03T20:13:23.050082+0000","last_fullsized":"2023-06-03T20:13:23.050082+0000","mapping_epoch":189,"log_start":"193'800","ondisk_log_start":"193'800","created":189,"last_epoch_clean":190,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T20:11:10.759955+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T20:11:10.759955+0000","last_clean_scrub_stamp":"2023-06-03T20:11:10.759955+0000","objects_scrubbed":0,"log_size":6691,"log_dups_size":0,"ondisk_log_size":6691,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":377290752,"num_objects":5757,"num_object_clones":0,"num_object_copies":17271,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5757,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7491,"num_write_kb":423936,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.d","version":"60'1","reported_seq":151,"reported_epoch":200,"state":"active+clean","last_fresh":"2023-06-03T20:12:31.900732+0000","last_change":"2023-06-03T20:07:47.857710+0000","last_active":"2023-06-03T20:12:31.900732+0000","last_peered":"2023-06-03T20:12:31.900732+0000","last_clean":"2023-06-03T20:12:31.900732+0000","last_became_active":"2023-06-03T20:07:47.857320+0000","last_became_peered":"2023-06-03T20:07:47.857320+0000","last_unstale":"2023-06-03T20:12:31.900732+0000","last_undegraded":"2023-06-03T20:12:31.900732+0000","last_fullsized":"2023-06-03T20:12:31.900732+0000","mapping_epoch":172,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":173,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":147,"reported_epoch":200,"state":"active+clean","last_fresh":"2023-06-03T20:12:31.899574+0000","last_change":"2023-06-03T20:12:30.910738+0000","last_active":"2023-06-03T20:12:31.899574+0000","last_peered":"2023-06-03T20:12:31.899574+0000","last_clean":"2023-06-03T20:12:31.899574+0000","last_became_active":"2023-06-03T20:12:30.910145+0000","last_became_peered":"2023-06-03T20:12:30.910145+0000","last_unstale":"2023-06-03T20:12:31.899574+0000","last_undegraded":"2023-06-03T20:12:31.899574+0000","last_fullsized":"2023-06-03T20:12:31.899574+0000","mapping_epoch":198,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":199,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,0],"acting":[7,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"11.4","version":"201'7393","reported_seq":7412,"reported_epoch":201,"state":"active+clean","last_fresh":"2023-06-03T20:13:23.015470+0000","last_change":"2023-06-03T20:11:11.796227+0000","last_active":"2023-06-03T20:13:23.015470+0000","last_peered":"2023-06-03T20:13:23.015470+0000","last_clean":"2023-06-03T20:13:23.015470+0000","last_became_active":"2023-06-03T20:11:11.795900+0000","last_became_peered":"2023-06-03T20:11:11.795900+0000","last_unstale":"2023-06-03T20:13:23.015470+0000","last_undegraded":"2023-06-03T20:13:23.015470+0000","last_fullsized":"2023-06-03T20:13:23.015470+0000","mapping_epoch":189,"log_start":"193'700","ondisk_log_start":"193'700","created":189,"last_epoch_clean":190,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T20:11:10.759955+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T20:11:10.759955+0000","last_clean_scrub_stamp":"2023-06-03T20:11:10.759955+0000","objects_scrubbed":0,"log_size":6693,"log_dups_size":0,"ondisk_log_size":6693,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":373882880,"num_objects":5705,"num_object_clones":0,"num_object_copies":17115,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5705,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7393,"num_write_kb":419136,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,2],"acting":[7,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":168,"reported_epoch":200,"state":"active+clean","last_fresh":"2023-06-03T20:12:31.904131+0000","last_change":"2023-06-03T19:56:54.507432+0000","last_active":"2023-06-03T20:12:31.904131+0000","last_peered":"2023-06-03T20:12:31.904131+0000","last_clean":"2023-06-03T20:12:31.904131+0000","last_became_active":"2023-06-03T19:56:54.507012+0000","last_became_peered":"2023-06-03T19:56:54.507012+0000","last_unstale":"2023-06-03T20:12:31.904131+0000","last_undegraded":"2023-06-03T20:12:31.904131+0000","last_fullsized":"2023-06-03T20:12:31.904131+0000","mapping_epoch":119,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":204,"reported_epoch":200,"state":"active+undersized","last_fresh":"2023-06-03T20:12:31.940147+0000","last_change":"2023-06-03T20:11:54.017375+0000","last_active":"2023-06-03T20:12:31.940147+0000","last_peered":"2023-06-03T20:12:31.940147+0000","last_clean":"2023-06-03T20:11:50.798782+0000","last_became_active":"2023-06-03T20:11:54.017375+0000","last_became_peered":"2023-06-03T20:11:54.017375+0000","last_unstale":"2023-06-03T20:12:31.940147+0000","last_undegraded":"2023-06-03T20:12:31.940147+0000","last_fullsized":"2023-06-03T20:11:54.008913+0000","mapping_epoch":195,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T21:44:01.134376+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3],"acting":[0,3],"avail_no_missing":["0","3"],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"11.3","version":"201'7545","reported_seq":9704,"reported_epoch":201,"state":"active+recovering+undersized+remapped","last_fresh":"2023-06-03T20:13:23.007267+0000","last_change":"2023-06-03T20:12:32.444069+0000","last_active":"2023-06-03T20:13:23.007267+0000","last_peered":"2023-06-03T20:13:23.007267+0000","last_clean":"2023-06-03T20:12:29.830029+0000","last_became_active":"2023-06-03T20:12:32.285822+0000","last_became_peered":"2023-06-03T20:12:32.285822+0000","last_unstale":"2023-06-03T20:13:23.007267+0000","last_undegraded":"2023-06-03T20:13:23.007267+0000","last_fullsized":"2023-06-03T20:12:31.899666+0000","mapping_epoch":199,"log_start":"193'800","ondisk_log_start":"193'800","created":189,"last_epoch_clean":190,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T20:11:10.759955+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T20:11:10.759955+0000","last_clean_scrub_stamp":"2023-06-03T20:11:10.759955+0000","objects_scrubbed":0,"log_size":6745,"log_dups_size":0,"ondisk_log_size":6745,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":379650048,"num_objects":5793,"num_object_clones":0,"num_object_copies":17379,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":4059,"num_objects_unfound":0,"num_objects_dirty":5793,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7545,"num_write_kb":426816,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1066,"num_bytes_recovered":69861376,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,2],"acting":[7,5],"avail_no_missing":["7","5","6"],"object_location_counts":[{"shards":"5,6,7","objects":4059},{"shards":"5,7","objects":1734}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.b","version":"60'2","reported_seq":178,"reported_epoch":200,"state":"active+undersized+degraded","last_fresh":"2023-06-03T20:12:37.575664+0000","last_change":"2023-06-03T20:11:54.019446+0000","last_active":"2023-06-03T20:12:37.575664+0000","last_peered":"2023-06-03T20:12:37.575664+0000","last_clean":"2023-06-03T20:11:43.977155+0000","last_became_active":"2023-06-03T20:11:54.019446+0000","last_became_peered":"2023-06-03T20:11:54.019446+0000","last_unstale":"2023-06-03T20:12:37.575664+0000","last_undegraded":"2023-06-03T20:11:54.013169+0000","last_fullsized":"2023-06-03T20:11:54.012960+0000","mapping_epoch":195,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T20:48:53.735497+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0],"acting":[3,0],"avail_no_missing":["3","0"],"object_location_counts":[{"shards":"0,3","objects":1}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":102,"reported_epoch":200,"state":"active+undersized","last_fresh":"2023-06-03T20:12:31.899809+0000","last_change":"2023-06-03T20:11:54.015780+0000","last_active":"2023-06-03T20:12:31.899809+0000","last_peered":"2023-06-03T20:12:31.899809+0000","last_clean":"2023-06-03T19:56:11.745804+0000","last_became_active":"2023-06-03T20:11:54.015780+0000","last_became_peered":"2023-06-03T20:11:54.015780+0000","last_unstale":"2023-06-03T20:12:31.899809+0000","last_undegraded":"2023-06-03T20:12:31.899809+0000","last_fullsized":"2023-06-03T20:11:54.008666+0000","mapping_epoch":195,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T06:13:29.481421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6],"acting":[3,6],"avail_no_missing":["3","6"],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"11.2","version":"201'7591","reported_seq":7613,"reported_epoch":201,"state":"active+clean","last_fresh":"2023-06-03T20:13:24.859171+0000","last_change":"2023-06-03T20:11:11.788495+0000","last_active":"2023-06-03T20:13:24.859171+0000","last_peered":"2023-06-03T20:13:24.859171+0000","last_clean":"2023-06-03T20:13:24.859171+0000","last_became_active":"2023-06-03T20:11:11.788122+0000","last_became_peered":"2023-06-03T20:11:11.788122+0000","last_unstale":"2023-06-03T20:13:24.859171+0000","last_undegraded":"2023-06-03T20:13:24.859171+0000","last_fullsized":"2023-06-03T20:13:24.859171+0000","mapping_epoch":189,"log_start":"0'0","ondisk_log_start":"0'0","created":189,"last_epoch_clean":190,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T20:11:10.759955+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T20:11:10.759955+0000","last_clean_scrub_stamp":"2023-06-03T20:11:10.759955+0000","objects_scrubbed":0,"log_size":7591,"log_dups_size":0,"ondisk_log_size":7591,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T05:43:08.101794+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":377683968,"num_objects":5763,"num_object_clones":0,"num_object_copies":17289,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5763,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7591,"num_write_kb":427328,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":198,"reported_epoch":200,"state":"active+undersized","last_fresh":"2023-06-03T20:12:31.905987+0000","last_change":"2023-06-03T20:11:54.015192+0000","last_active":"2023-06-03T20:12:31.905987+0000","last_peered":"2023-06-03T20:12:31.905987+0000","last_clean":"2023-06-03T20:11:43.969614+0000","last_became_active":"2023-06-03T20:11:54.015192+0000","last_became_peered":"2023-06-03T20:11:54.015192+0000","last_unstale":"2023-06-03T20:12:31.905987+0000","last_undegraded":"2023-06-03T20:12:31.905987+0000","last_fullsized":"2023-06-03T20:11:54.005632+0000","mapping_epoch":195,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3],"acting":[6,3],"avail_no_missing":["6","3"],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":162,"reported_epoch":200,"state":"active+undersized","last_fresh":"2023-06-03T20:12:32.098420+0000","last_change":"2023-06-03T20:11:54.028687+0000","last_active":"2023-06-03T20:12:32.098420+0000","last_peered":"2023-06-03T20:12:32.098420+0000","last_clean":"2023-06-03T20:07:14.871385+0000","last_became_active":"2023-06-03T20:11:54.028687+0000","last_became_peered":"2023-06-03T20:11:54.028687+0000","last_unstale":"2023-06-03T20:12:32.098420+0000","last_undegraded":"2023-06-03T20:12:32.098420+0000","last_fullsized":"2023-06-03T20:11:54.009673+0000","mapping_epoch":195,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T04:36:58.188469+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6],"acting":[2,6],"avail_no_missing":["2","6"],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"11.1","version":"201'7222","reported_seq":7242,"reported_epoch":201,"state":"active+clean","last_fresh":"2023-06-03T20:13:22.898855+0000","last_change":"2023-06-03T20:11:11.787443+0000","last_active":"2023-06-03T20:13:22.898855+0000","last_peered":"2023-06-03T20:13:22.898855+0000","last_clean":"2023-06-03T20:13:22.898855+0000","last_became_active":"2023-06-03T20:11:11.787117+0000","last_became_peered":"2023-06-03T20:11:11.787117+0000","last_unstale":"2023-06-03T20:13:22.898855+0000","last_undegraded":"2023-06-03T20:13:22.898855+0000","last_fullsized":"2023-06-03T20:13:22.898855+0000","mapping_epoch":189,"log_start":"0'0","ondisk_log_start":"0'0","created":189,"last_epoch_clean":190,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T20:11:10.759955+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T20:11:10.759955+0000","last_clean_scrub_stamp":"2023-06-03T20:11:10.759955+0000","objects_scrubbed":0,"log_size":7222,"log_dups_size":0,"ondisk_log_size":7222,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":364249088,"num_objects":5558,"num_object_clones":0,"num_object_copies":16674,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5558,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7222,"num_write_kb":408960,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,0],"acting":[6,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":99,"reported_epoch":200,"state":"active+undersized","last_fresh":"2023-06-03T20:12:32.092325+0000","last_change":"2023-06-03T20:11:54.018578+0000","last_active":"2023-06-03T20:12:32.092325+0000","last_peered":"2023-06-03T20:12:32.092325+0000","last_clean":"2023-06-03T19:55:41.905941+0000","last_became_active":"2023-06-03T20:11:54.018578+0000","last_became_peered":"2023-06-03T20:11:54.018578+0000","last_unstale":"2023-06-03T20:12:32.092325+0000","last_undegraded":"2023-06-03T20:12:32.092325+0000","last_fullsized":"2023-06-03T20:11:54.009386+0000","mapping_epoch":195,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T21:13:40.683176+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7],"acting":[2,7],"avail_no_missing":["2","7"],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":147,"reported_epoch":200,"state":"active+clean+remapped","last_fresh":"2023-06-03T20:12:32.546875+0000","last_change":"2023-06-03T20:12:31.910070+0000","last_active":"2023-06-03T20:12:32.546875+0000","last_peered":"2023-06-03T20:12:32.546875+0000","last_clean":"2023-06-03T20:12:32.546875+0000","last_became_active":"2023-06-03T20:12:31.909692+0000","last_became_peered":"2023-06-03T20:12:31.909692+0000","last_unstale":"2023-06-03T20:12:32.546875+0000","last_undegraded":"2023-06-03T20:12:32.546875+0000","last_fullsized":"2023-06-03T20:12:32.546875+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T07:20:52.604735+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5],"acting":[3,5,6],"avail_no_missing":["3","5","6"],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"11.0","version":"201'7380","reported_seq":9297,"reported_epoch":201,"state":"active+recovering+undersized+remapped","last_fresh":"2023-06-03T20:13:22.830704+0000","last_change":"2023-06-03T20:12:32.149687+0000","last_active":"2023-06-03T20:13:22.830704+0000","last_peered":"2023-06-03T20:13:22.830704+0000","last_clean":"2023-06-03T20:12:29.878942+0000","last_became_active":"2023-06-03T20:12:32.063527+0000","last_became_peered":"2023-06-03T20:12:32.063527+0000","last_unstale":"2023-06-03T20:13:22.830704+0000","last_undegraded":"2023-06-03T20:13:22.830704+0000","last_fullsized":"2023-06-03T20:12:31.897715+0000","mapping_epoch":199,"log_start":"193'900","ondisk_log_start":"193'900","created":189,"last_epoch_clean":190,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T20:11:10.759955+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T20:11:10.759955+0000","last_clean_scrub_stamp":"2023-06-03T20:11:10.759955+0000","objects_scrubbed":0,"log_size":6480,"log_dups_size":0,"ondisk_log_size":6480,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T06:38:41.925797+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":374472704,"num_objects":5714,"num_object_clones":0,"num_object_copies":17142,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":4107,"num_objects_unfound":0,"num_objects_dirty":5714,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7380,"num_write_kb":419008,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":943,"num_bytes_recovered":61800448,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7],"avail_no_missing":["3","6","7"],"object_location_counts":[{"shards":"3,6,7","objects":4107},{"shards":"3,7","objects":1607}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":187,"reported_epoch":200,"state":"active+undersized","last_fresh":"2023-06-03T20:12:31.897529+0000","last_change":"2023-06-03T20:11:54.026212+0000","last_active":"2023-06-03T20:12:31.897529+0000","last_peered":"2023-06-03T20:12:31.897529+0000","last_clean":"2023-06-03T20:11:43.974120+0000","last_became_active":"2023-06-03T20:11:54.026212+0000","last_became_peered":"2023-06-03T20:11:54.026212+0000","last_unstale":"2023-06-03T20:12:31.897529+0000","last_undegraded":"2023-06-03T20:12:31.897529+0000","last_fullsized":"2023-06-03T20:11:54.007103+0000","mapping_epoch":195,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T03:49:35.033521+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7],"acting":[3,7],"avail_no_missing":["3","7"],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":120,"reported_epoch":200,"state":"active+undersized","last_fresh":"2023-06-03T20:12:31.898596+0000","last_change":"2023-06-03T20:11:54.026421+0000","last_active":"2023-06-03T20:12:31.898596+0000","last_peered":"2023-06-03T20:12:31.898596+0000","last_clean":"2023-06-03T20:11:43.974605+0000","last_became_active":"2023-06-03T20:11:54.026421+0000","last_became_peered":"2023-06-03T20:11:54.026421+0000","last_unstale":"2023-06-03T20:12:31.898596+0000","last_undegraded":"2023-06-03T20:12:31.898596+0000","last_fullsized":"2023-06-03T20:11:54.009062+0000","mapping_epoch":195,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T02:07:21.901385+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7],"acting":[3,7],"avail_no_missing":["3","7"],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"11.d","version":"201'7679","reported_seq":7712,"reported_epoch":201,"state":"active+undersized+degraded","last_fresh":"2023-06-03T20:13:24.871136+0000","last_change":"2023-06-03T20:11:54.061046+0000","last_active":"2023-06-03T20:13:24.871136+0000","last_peered":"2023-06-03T20:13:24.871136+0000","last_clean":"2023-06-03T20:11:52.999600+0000","last_became_active":"2023-06-03T20:11:54.040113+0000","last_became_peered":"2023-06-03T20:11:54.040113+0000","last_unstale":"2023-06-03T20:13:24.871136+0000","last_undegraded":"2023-06-03T20:11:54.028082+0000","last_fullsized":"2023-06-03T20:11:54.010095+0000","mapping_epoch":195,"log_start":"0'0","ondisk_log_start":"0'0","created":189,"last_epoch_clean":190,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T20:11:10.759955+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T20:11:10.759955+0000","last_clean_scrub_stamp":"2023-06-03T20:11:10.759955+0000","objects_scrubbed":0,"log_size":7679,"log_dups_size":0,"ondisk_log_size":7679,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T21:38:25.157657+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":375521304,"num_objects":5731,"num_object_clones":0,"num_object_copies":17193,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5731,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5731,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":7679,"num_write_kb":429057,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":131072,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6],"acting":[2,6],"avail_no_missing":["2","6"],"object_location_counts":[{"shards":"2,6","objects":5731}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":110,"reported_epoch":200,"state":"active+clean","last_fresh":"2023-06-03T20:12:31.940001+0000","last_change":"2023-06-03T20:07:19.926958+0000","last_active":"2023-06-03T20:12:31.940001+0000","last_peered":"2023-06-03T20:12:31.940001+0000","last_clean":"2023-06-03T20:12:31.940001+0000","last_became_active":"2023-06-03T20:07:19.926456+0000","last_became_peered":"2023-06-03T20:07:19.926456+0000","last_unstale":"2023-06-03T20:12:31.940001+0000","last_undegraded":"2023-06-03T20:12:31.940001+0000","last_fullsized":"2023-06-03T20:12:31.940001+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:32:20.692043+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:32:20.692043+0000","last_clean_scrub_stamp":"2023-06-03T19:32:20.692043+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T20:47:07.666782+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,2],"acting":[0,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":183,"reported_epoch":200,"state":"active+clean","last_fresh":"2023-06-03T20:12:31.903959+0000","last_change":"2023-06-03T19:56:54.512607+0000","last_active":"2023-06-03T20:12:31.903959+0000","last_peered":"2023-06-03T20:12:31.903959+0000","last_clean":"2023-06-03T20:12:31.903959+0000","last_became_active":"2023-06-03T19:56:54.512282+0000","last_became_peered":"2023-06-03T19:56:54.512282+0000","last_unstale":"2023-06-03T20:12:31.903959+0000","last_undegraded":"2023-06-03T20:12:31.903959+0000","last_fullsized":"2023-06-03T20:12:31.903959+0000","mapping_epoch":119,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":103,"reported_epoch":200,"state":"active+undersized","last_fresh":"2023-06-03T20:12:31.904153+0000","last_change":"2023-06-03T20:11:54.017774+0000","last_active":"2023-06-03T20:12:31.904153+0000","last_peered":"2023-06-03T20:12:31.904153+0000","last_clean":"2023-06-03T20:11:43.981149+0000","last_became_active":"2023-06-03T20:11:54.017774+0000","last_became_peered":"2023-06-03T20:11:54.017774+0000","last_unstale":"2023-06-03T20:12:31.904153+0000","last_undegraded":"2023-06-03T20:12:31.904153+0000","last_fullsized":"2023-06-03T20:11:54.008207+0000","mapping_epoch":195,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0],"acting":[5,0],"avail_no_missing":["5","0"],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"11.b","version":"201'7351","reported_seq":7376,"reported_epoch":201,"state":"active+undersized+degraded","last_fresh":"2023-06-03T20:13:22.769564+0000","last_change":"2023-06-03T20:11:54.019454+0000","last_active":"2023-06-03T20:13:22.769564+0000","last_peered":"2023-06-03T20:13:22.769564+0000","last_clean":"2023-06-03T20:11:52.997407+0000","last_became_active":"2023-06-03T20:11:54.019454+0000","last_became_peered":"2023-06-03T20:11:54.019454+0000","last_unstale":"2023-06-03T20:13:22.769564+0000","last_undegraded":"2023-06-03T20:11:54.014863+0000","last_fullsized":"2023-06-03T20:11:54.014785+0000","mapping_epoch":195,"log_start":"193'900","ondisk_log_start":"193'900","created":189,"last_epoch_clean":190,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T20:11:10.759955+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T20:11:10.759955+0000","last_clean_scrub_stamp":"2023-06-03T20:11:10.759955+0000","objects_scrubbed":0,"log_size":6451,"log_dups_size":0,"ondisk_log_size":6451,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T05:26:41.801270+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":366542848,"num_objects":5593,"num_object_clones":0,"num_object_copies":16779,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5593,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5593,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7351,"num_write_kb":414208,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0],"acting":[3,0],"avail_no_missing":["3","0"],"object_location_counts":[{"shards":"0,3","objects":5593}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":190,"reported_epoch":200,"state":"active+clean","last_fresh":"2023-06-03T20:12:31.904448+0000","last_change":"2023-06-03T20:12:30.910088+0000","last_active":"2023-06-03T20:12:31.904448+0000","last_peered":"2023-06-03T20:12:31.904448+0000","last_clean":"2023-06-03T20:12:31.904448+0000","last_became_active":"2023-06-03T20:12:30.909574+0000","last_became_peered":"2023-06-03T20:12:30.909574+0000","last_unstale":"2023-06-03T20:12:31.904448+0000","last_undegraded":"2023-06-03T20:12:31.904448+0000","last_fullsized":"2023-06-03T20:12:31.904448+0000","mapping_epoch":198,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":199,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:32:20.692043+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:32:20.692043+0000","last_clean_scrub_stamp":"2023-06-03T19:32:20.692043+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":72,"reported_epoch":200,"state":"active+undersized","last_fresh":"2023-06-03T20:12:31.939755+0000","last_change":"2023-06-03T20:11:54.020579+0000","last_active":"2023-06-03T20:12:31.939755+0000","last_peered":"2023-06-03T20:12:31.939755+0000","last_clean":"2023-06-03T19:46:38.226313+0000","last_became_active":"2023-06-03T20:11:54.020579+0000","last_became_peered":"2023-06-03T20:11:54.020579+0000","last_unstale":"2023-06-03T20:12:31.939755+0000","last_undegraded":"2023-06-03T20:12:31.939755+0000","last_fullsized":"2023-06-03T20:11:54.008467+0000","mapping_epoch":195,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T21:09:23.253653+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6],"acting":[0,6],"avail_no_missing":["0","6"],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":118,"reported_epoch":200,"state":"active+clean","last_fresh":"2023-06-03T20:12:32.092585+0000","last_change":"2023-06-03T20:12:30.911946+0000","last_active":"2023-06-03T20:12:32.092585+0000","last_peered":"2023-06-03T20:12:32.092585+0000","last_clean":"2023-06-03T20:12:32.092585+0000","last_became_active":"2023-06-03T20:12:30.911513+0000","last_became_peered":"2023-06-03T20:12:30.911513+0000","last_unstale":"2023-06-03T20:12:32.092585+0000","last_undegraded":"2023-06-03T20:12:32.092585+0000","last_fullsized":"2023-06-03T20:12:32.092585+0000","mapping_epoch":198,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":199,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T07:22:01.480443+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,7],"acting":[2,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"11.8","version":"201'7570","reported_seq":7592,"reported_epoch":201,"state":"active+clean","last_fresh":"2023-06-03T20:13:22.860681+0000","last_change":"2023-06-03T20:11:11.793649+0000","last_active":"2023-06-03T20:13:22.860681+0000","last_peered":"2023-06-03T20:13:22.860681+0000","last_clean":"2023-06-03T20:13:22.860681+0000","last_became_active":"2023-06-03T20:11:11.793292+0000","last_became_peered":"2023-06-03T20:11:11.793292+0000","last_unstale":"2023-06-03T20:13:22.860681+0000","last_undegraded":"2023-06-03T20:13:22.860681+0000","last_fullsized":"2023-06-03T20:13:22.860681+0000","mapping_epoch":189,"log_start":"193'1100","ondisk_log_start":"193'1100","created":189,"last_epoch_clean":190,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T20:11:10.759955+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T20:11:10.759955+0000","last_clean_scrub_stamp":"2023-06-03T20:11:10.759955+0000","objects_scrubbed":0,"log_size":6470,"log_dups_size":0,"ondisk_log_size":6470,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T07:27:16.604437+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":377880576,"num_objects":5766,"num_object_clones":0,"num_object_copies":17298,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5766,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7570,"num_write_kb":426752,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,5],"acting":[3,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":180,"reported_epoch":200,"state":"active+clean","last_fresh":"2023-06-03T20:12:49.810028+0000","last_change":"2023-06-03T19:56:54.504859+0000","last_active":"2023-06-03T20:12:49.810028+0000","last_peered":"2023-06-03T20:12:49.810028+0000","last_clean":"2023-06-03T20:12:49.810028+0000","last_became_active":"2023-06-03T19:56:54.504455+0000","last_became_peered":"2023-06-03T19:56:54.504455+0000","last_unstale":"2023-06-03T20:12:49.810028+0000","last_undegraded":"2023-06-03T20:12:49.810028+0000","last_fullsized":"2023-06-03T20:12:49.810028+0000","mapping_epoch":119,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:32:20.692043+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:32:20.692043+0000","last_clean_scrub_stamp":"2023-06-03T19:32:20.692043+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T06:25:45.522100+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":131,"reported_epoch":201,"state":"active+clean","last_fresh":"2023-06-03T20:12:57.045773+0000","last_change":"2023-06-03T20:07:19.906703+0000","last_active":"2023-06-03T20:12:57.045773+0000","last_peered":"2023-06-03T20:12:57.045773+0000","last_clean":"2023-06-03T20:12:57.045773+0000","last_became_active":"2023-06-03T20:07:19.906373+0000","last_became_peered":"2023-06-03T20:07:19.906373+0000","last_unstale":"2023-06-03T20:12:57.045773+0000","last_undegraded":"2023-06-03T20:12:57.045773+0000","last_fullsized":"2023-06-03T20:12:57.045773+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T21:06:05.945328+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6,0],"acting":[2,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":103,"reported_epoch":200,"state":"active+clean","last_fresh":"2023-06-03T20:12:31.904269+0000","last_change":"2023-06-03T20:07:19.906757+0000","last_active":"2023-06-03T20:12:31.904269+0000","last_peered":"2023-06-03T20:12:31.904269+0000","last_clean":"2023-06-03T20:12:31.904269+0000","last_became_active":"2023-06-03T20:07:19.906380+0000","last_became_peered":"2023-06-03T20:07:19.906380+0000","last_unstale":"2023-06-03T20:12:31.904269+0000","last_undegraded":"2023-06-03T20:12:31.904269+0000","last_fullsized":"2023-06-03T20:12:31.904269+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,7],"acting":[5,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"11.a","version":"201'7358","reported_seq":7379,"reported_epoch":201,"state":"active+clean","last_fresh":"2023-06-03T20:13:22.819456+0000","last_change":"2023-06-03T20:11:11.793791+0000","last_active":"2023-06-03T20:13:22.819456+0000","last_peered":"2023-06-03T20:13:22.819456+0000","last_clean":"2023-06-03T20:13:22.819456+0000","last_became_active":"2023-06-03T20:11:11.793127+0000","last_became_peered":"2023-06-03T20:11:11.793127+0000","last_unstale":"2023-06-03T20:13:22.819456+0000","last_undegraded":"2023-06-03T20:13:22.819456+0000","last_fullsized":"2023-06-03T20:13:22.819456+0000","mapping_epoch":189,"log_start":"193'900","ondisk_log_start":"193'900","created":189,"last_epoch_clean":190,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T20:11:10.759955+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T20:11:10.759955+0000","last_clean_scrub_stamp":"2023-06-03T20:11:10.759955+0000","objects_scrubbed":0,"log_size":6458,"log_dups_size":0,"ondisk_log_size":6458,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T22:21:19.854335+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":374341632,"num_objects":5712,"num_object_clones":0,"num_object_copies":17136,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5712,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7358,"num_write_kb":418240,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,5],"acting":[3,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":172,"reported_epoch":200,"state":"active+clean","last_fresh":"2023-06-03T20:12:31.903405+0000","last_change":"2023-06-03T19:56:11.752151+0000","last_active":"2023-06-03T20:12:31.903405+0000","last_peered":"2023-06-03T20:12:31.903405+0000","last_clean":"2023-06-03T20:12:31.903405+0000","last_became_active":"2023-06-03T19:56:11.751767+0000","last_became_peered":"2023-06-03T19:56:11.751767+0000","last_unstale":"2023-06-03T20:12:31.903405+0000","last_undegraded":"2023-06-03T20:12:31.903405+0000","last_fullsized":"2023-06-03T20:12:31.903405+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":115,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:32:20.692043+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:32:20.692043+0000","last_clean_scrub_stamp":"2023-06-03T19:32:20.692043+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"72'139","reported_seq":401,"reported_epoch":200,"state":"active+clean","last_fresh":"2023-06-03T20:12:31.899160+0000","last_change":"2023-06-03T20:07:47.872942+0000","last_active":"2023-06-03T20:12:31.899160+0000","last_peered":"2023-06-03T20:12:31.899160+0000","last_clean":"2023-06-03T20:12:31.899160+0000","last_became_active":"2023-06-03T20:07:47.872426+0000","last_became_peered":"2023-06-03T20:07:47.872426+0000","last_unstale":"2023-06-03T20:12:31.899160+0000","last_undegraded":"2023-06-03T20:12:31.899160+0000","last_fullsized":"2023-06-03T20:12:31.899160+0000","mapping_epoch":172,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":173,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:27:28.100808+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:27:28.100808+0000","last_clean_scrub_stamp":"2023-06-03T19:27:28.100808+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":152,"reported_epoch":200,"state":"active+clean","last_fresh":"2023-06-03T20:12:31.898973+0000","last_change":"2023-06-03T19:56:54.509571+0000","last_active":"2023-06-03T20:12:31.898973+0000","last_peered":"2023-06-03T20:12:31.898973+0000","last_clean":"2023-06-03T20:12:31.898973+0000","last_became_active":"2023-06-03T19:56:54.509213+0000","last_became_peered":"2023-06-03T19:56:54.509213+0000","last_unstale":"2023-06-03T20:12:31.898973+0000","last_undegraded":"2023-06-03T20:12:31.898973+0000","last_fullsized":"2023-06-03T20:12:31.898973+0000","mapping_epoch":119,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T21:26:56.003203+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":177,"reported_epoch":200,"state":"active+clean+remapped","last_fresh":"2023-06-03T20:12:31.942737+0000","last_change":"2023-06-03T20:12:31.939716+0000","last_active":"2023-06-03T20:12:31.942737+0000","last_peered":"2023-06-03T20:12:31.942737+0000","last_clean":"2023-06-03T20:12:31.942737+0000","last_became_active":"2023-06-03T20:12:31.938876+0000","last_became_peered":"2023-06-03T20:12:31.938876+0000","last_unstale":"2023-06-03T20:12:31.942737+0000","last_undegraded":"2023-06-03T20:12:31.942737+0000","last_fullsized":"2023-06-03T20:12:31.942737+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0],"acting":[7,0,6],"avail_no_missing":["7","0","6"],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"11.9","version":"201'7515","reported_seq":7541,"reported_epoch":201,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-06-03T20:13:22.845718+0000","last_change":"2023-06-03T20:12:32.085781+0000","last_active":"2023-06-03T20:13:22.845718+0000","last_peered":"2023-06-03T20:13:22.845718+0000","last_clean":"2023-06-03T20:12:29.842853+0000","last_became_active":"2023-06-03T20:12:32.076473+0000","last_became_peered":"2023-06-03T20:12:32.076473+0000","last_unstale":"2023-06-03T20:13:22.845718+0000","last_undegraded":"2023-06-03T20:12:31.934803+0000","last_fullsized":"2023-06-03T20:12:31.898014+0000","mapping_epoch":199,"log_start":"193'1100","ondisk_log_start":"193'1100","created":189,"last_epoch_clean":190,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T20:11:10.759955+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T20:11:10.759955+0000","last_clean_scrub_stamp":"2023-06-03T20:11:10.759955+0000","objects_scrubbed":0,"log_size":6415,"log_dups_size":0,"ondisk_log_size":6415,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T00:47:40.082595+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":381091840,"num_objects":5815,"num_object_clones":0,"num_object_copies":17445,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5168,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5815,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7515,"num_write_kb":426560,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,7],"acting":[3,0],"avail_no_missing":["3","0","6"],"object_location_counts":[{"shards":"0,3","objects":647},{"shards":"0,3,6","objects":4317}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":184,"reported_epoch":200,"state":"active+clean","last_fresh":"2023-06-03T20:12:31.899016+0000","last_change":"2023-06-03T20:07:47.855767+0000","last_active":"2023-06-03T20:12:31.899016+0000","last_peered":"2023-06-03T20:12:31.899016+0000","last_clean":"2023-06-03T20:12:31.899016+0000","last_became_active":"2023-06-03T20:07:47.855444+0000","last_became_peered":"2023-06-03T20:07:47.855444+0000","last_unstale":"2023-06-03T20:12:31.899016+0000","last_undegraded":"2023-06-03T20:12:31.899016+0000","last_fullsized":"2023-06-03T20:12:31.899016+0000","mapping_epoch":172,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":173,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:32:20.692043+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:32:20.692043+0000","last_clean_scrub_stamp":"2023-06-03T19:32:20.692043+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,3],"acting":[7,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":183,"reported_epoch":200,"state":"active+undersized","last_fresh":"2023-06-03T20:12:31.939670+0000","last_change":"2023-06-03T20:11:54.017331+0000","last_active":"2023-06-03T20:12:31.939670+0000","last_peered":"2023-06-03T20:12:31.939670+0000","last_clean":"2023-06-03T20:11:43.977931+0000","last_became_active":"2023-06-03T20:11:54.017331+0000","last_became_peered":"2023-06-03T20:11:54.017331+0000","last_unstale":"2023-06-03T20:12:31.939670+0000","last_undegraded":"2023-06-03T20:12:31.939670+0000","last_fullsized":"2023-06-03T20:11:54.007149+0000","mapping_epoch":195,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T04:51:15.533105+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3],"acting":[0,3],"avail_no_missing":["0","3"],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":187,"reported_epoch":201,"state":"active+clean","last_fresh":"2023-06-03T20:12:57.050921+0000","last_change":"2023-06-03T20:07:19.925864+0000","last_active":"2023-06-03T20:12:57.050921+0000","last_peered":"2023-06-03T20:12:57.050921+0000","last_clean":"2023-06-03T20:12:57.050921+0000","last_became_active":"2023-06-03T20:07:19.925541+0000","last_became_peered":"2023-06-03T20:07:19.925541+0000","last_unstale":"2023-06-03T20:12:57.050921+0000","last_undegraded":"2023-06-03T20:12:57.050921+0000","last_fullsized":"2023-06-03T20:12:57.050921+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T00:10:11.790831+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,7],"acting":[0,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"11.c","version":"201'7558","reported_seq":7585,"reported_epoch":201,"state":"active+undersized+degraded","last_fresh":"2023-06-03T20:13:24.844062+0000","last_change":"2023-06-03T20:11:54.021153+0000","last_active":"2023-06-03T20:13:24.844062+0000","last_peered":"2023-06-03T20:13:24.844062+0000","last_clean":"2023-06-03T20:11:53.003830+0000","last_became_active":"2023-06-03T20:11:54.021153+0000","last_became_peered":"2023-06-03T20:11:54.021153+0000","last_unstale":"2023-06-03T20:13:24.844062+0000","last_undegraded":"2023-06-03T20:11:54.009999+0000","last_fullsized":"2023-06-03T20:11:54.009855+0000","mapping_epoch":195,"log_start":"0'0","ondisk_log_start":"0'0","created":189,"last_epoch_clean":190,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T20:11:10.759955+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T20:11:10.759955+0000","last_clean_scrub_stamp":"2023-06-03T20:11:10.759955+0000","objects_scrubbed":0,"log_size":7558,"log_dups_size":0,"ondisk_log_size":7558,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T00:13:39.997844+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":372113408,"num_objects":5678,"num_object_clones":0,"num_object_copies":17034,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5678,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5678,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7558,"num_write_kb":423552,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7],"acting":[2,7],"avail_no_missing":["2","7"],"object_location_counts":[{"shards":"2,7","objects":5678}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":193,"reported_epoch":200,"state":"active+undersized","last_fresh":"2023-06-03T20:12:31.901113+0000","last_change":"2023-06-03T20:11:54.012820+0000","last_active":"2023-06-03T20:12:31.901113+0000","last_peered":"2023-06-03T20:12:31.901113+0000","last_clean":"2023-06-03T20:11:43.973280+0000","last_became_active":"2023-06-03T20:11:54.012820+0000","last_became_peered":"2023-06-03T20:11:54.012820+0000","last_unstale":"2023-06-03T20:12:31.901113+0000","last_undegraded":"2023-06-03T20:12:31.901113+0000","last_fullsized":"2023-06-03T20:11:54.011295+0000","mapping_epoch":195,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:32:20.692043+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:32:20.692043+0000","last_clean_scrub_stamp":"2023-06-03T19:32:20.692043+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0],"acting":[7,0],"avail_no_missing":["7","0"],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":130,"reported_epoch":200,"state":"active+clean","last_fresh":"2023-06-03T20:12:32.089482+0000","last_change":"2023-06-03T20:07:19.915065+0000","last_active":"2023-06-03T20:12:32.089482+0000","last_peered":"2023-06-03T20:12:32.089482+0000","last_clean":"2023-06-03T20:12:32.089482+0000","last_became_active":"2023-06-03T20:07:19.914699+0000","last_became_peered":"2023-06-03T20:07:19.914699+0000","last_unstale":"2023-06-03T20:12:32.089482+0000","last_undegraded":"2023-06-03T20:12:32.089482+0000","last_fullsized":"2023-06-03T20:12:32.089482+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T07:16:09.077184+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,0],"acting":[2,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":148,"reported_epoch":200,"state":"active+clean","last_fresh":"2023-06-03T20:12:31.939888+0000","last_change":"2023-06-03T19:47:06.733532+0000","last_active":"2023-06-03T20:12:31.939888+0000","last_peered":"2023-06-03T20:12:31.939888+0000","last_clean":"2023-06-03T20:12:31.939888+0000","last_became_active":"2023-06-03T19:47:06.733131+0000","last_became_peered":"2023-06-03T19:47:06.733131+0000","last_unstale":"2023-06-03T20:12:31.939888+0000","last_undegraded":"2023-06-03T20:12:31.939888+0000","last_fullsized":"2023-06-03T20:12:31.939888+0000","mapping_epoch":86,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":87,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T23:46:03.788646+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"11.f","version":"201'7358","reported_seq":7378,"reported_epoch":201,"state":"active+clean","last_fresh":"2023-06-03T20:13:22.968613+0000","last_change":"2023-06-03T20:11:11.794557+0000","last_active":"2023-06-03T20:13:22.968613+0000","last_peered":"2023-06-03T20:13:22.968613+0000","last_clean":"2023-06-03T20:13:22.968613+0000","last_became_active":"2023-06-03T20:11:11.794234+0000","last_became_peered":"2023-06-03T20:11:11.794234+0000","last_unstale":"2023-06-03T20:13:22.968613+0000","last_undegraded":"2023-06-03T20:13:22.968613+0000","last_fullsized":"2023-06-03T20:13:22.968613+0000","mapping_epoch":189,"log_start":"193'600","ondisk_log_start":"193'600","created":189,"last_epoch_clean":190,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T20:11:10.759955+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T20:11:10.759955+0000","last_clean_scrub_stamp":"2023-06-03T20:11:10.759955+0000","objects_scrubbed":0,"log_size":6758,"log_dups_size":0,"ondisk_log_size":6758,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":371195904,"num_objects":5664,"num_object_clones":0,"num_object_copies":16992,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5664,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7358,"num_write_kb":416704,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,0],"acting":[7,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.7","version":"60'2","reported_seq":157,"reported_epoch":200,"state":"active+clean","last_fresh":"2023-06-03T20:12:31.898683+0000","last_change":"2023-06-03T19:56:54.510148+0000","last_active":"2023-06-03T20:12:31.898683+0000","last_peered":"2023-06-03T20:12:31.898683+0000","last_clean":"2023-06-03T20:12:31.898683+0000","last_became_active":"2023-06-03T19:56:54.509771+0000","last_became_peered":"2023-06-03T19:56:54.509771+0000","last_unstale":"2023-06-03T20:12:31.898683+0000","last_undegraded":"2023-06-03T20:12:31.898683+0000","last_fullsized":"2023-06-03T20:12:31.898683+0000","mapping_epoch":119,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T00:52:42.807432+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":144,"reported_epoch":200,"state":"active+undersized","last_fresh":"2023-06-03T20:12:31.906463+0000","last_change":"2023-06-03T20:11:54.015128+0000","last_active":"2023-06-03T20:12:31.906463+0000","last_peered":"2023-06-03T20:12:31.906463+0000","last_clean":"2023-06-03T20:11:43.976310+0000","last_became_active":"2023-06-03T20:11:54.015128+0000","last_became_peered":"2023-06-03T20:11:54.015128+0000","last_unstale":"2023-06-03T20:12:31.906463+0000","last_undegraded":"2023-06-03T20:12:31.906463+0000","last_fullsized":"2023-06-03T20:11:54.006226+0000","mapping_epoch":195,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:32:20.692043+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:32:20.692043+0000","last_clean_scrub_stamp":"2023-06-03T19:32:20.692043+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3],"acting":[6,3],"avail_no_missing":["6","3"],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":155,"reported_epoch":201,"state":"active+clean","last_fresh":"2023-06-03T20:12:57.037790+0000","last_change":"2023-06-03T19:56:54.507670+0000","last_active":"2023-06-03T20:12:57.037790+0000","last_peered":"2023-06-03T20:12:57.037790+0000","last_clean":"2023-06-03T20:12:57.037790+0000","last_became_active":"2023-06-03T19:56:54.507293+0000","last_became_peered":"2023-06-03T19:56:54.507293+0000","last_unstale":"2023-06-03T20:12:57.037790+0000","last_undegraded":"2023-06-03T20:12:57.037790+0000","last_fullsized":"2023-06-03T20:12:57.037790+0000","mapping_epoch":119,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T01:49:19.966054+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"11.e","version":"201'7497","reported_seq":7522,"reported_epoch":201,"state":"active+undersized+degraded","last_fresh":"2023-06-03T20:13:23.036570+0000","last_change":"2023-06-03T20:11:54.013399+0000","last_active":"2023-06-03T20:13:23.036570+0000","last_peered":"2023-06-03T20:13:23.036570+0000","last_clean":"2023-06-03T20:11:52.975326+0000","last_became_active":"2023-06-03T20:11:54.013399+0000","last_became_peered":"2023-06-03T20:11:54.013399+0000","last_unstale":"2023-06-03T20:13:23.036570+0000","last_undegraded":"2023-06-03T20:11:54.011155+0000","last_fullsized":"2023-06-03T20:11:54.010995+0000","mapping_epoch":195,"log_start":"193'800","ondisk_log_start":"193'800","created":189,"last_epoch_clean":190,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T20:11:10.759955+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T20:11:10.759955+0000","last_clean_scrub_stamp":"2023-06-03T20:11:10.759955+0000","objects_scrubbed":0,"log_size":6697,"log_dups_size":0,"ondisk_log_size":6697,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":380174336,"num_objects":5801,"num_object_clones":0,"num_object_copies":17403,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5801,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5801,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7497,"num_write_kb":425536,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0],"acting":[7,0],"avail_no_missing":["7","0"],"object_location_counts":[{"shards":"0,7","objects":5801}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":172,"reported_epoch":200,"state":"active+clean","last_fresh":"2023-06-03T20:12:31.906261+0000","last_change":"2023-06-03T19:56:11.752730+0000","last_active":"2023-06-03T20:12:31.906261+0000","last_peered":"2023-06-03T20:12:31.906261+0000","last_clean":"2023-06-03T20:12:31.906261+0000","last_became_active":"2023-06-03T19:56:11.752441+0000","last_became_peered":"2023-06-03T19:56:11.752441+0000","last_unstale":"2023-06-03T20:12:31.906261+0000","last_undegraded":"2023-06-03T20:12:31.906261+0000","last_fullsized":"2023-06-03T20:12:31.906261+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":115,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:32:20.692043+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:32:20.692043+0000","last_clean_scrub_stamp":"2023-06-03T19:32:20.692043+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"60'1","reported_seq":200,"reported_epoch":201,"state":"active+undersized+degraded","last_fresh":"2023-06-03T20:12:57.050715+0000","last_change":"2023-06-03T20:11:54.014758+0000","last_active":"2023-06-03T20:12:57.050715+0000","last_peered":"2023-06-03T20:12:57.050715+0000","last_clean":"2023-06-03T20:11:43.980210+0000","last_became_active":"2023-06-03T20:11:54.014758+0000","last_became_peered":"2023-06-03T20:11:54.014758+0000","last_unstale":"2023-06-03T20:12:57.050715+0000","last_undegraded":"2023-06-03T20:11:54.011442+0000","last_fullsized":"2023-06-03T20:11:54.011320+0000","mapping_epoch":195,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T20:43:34.505545+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7],"acting":[0,7],"avail_no_missing":["0","7"],"object_location_counts":[{"shards":"0,7","objects":1}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":70,"reported_epoch":200,"state":"active+undersized","last_fresh":"2023-06-03T20:12:31.903367+0000","last_change":"2023-06-03T20:11:54.015754+0000","last_active":"2023-06-03T20:12:31.903367+0000","last_peered":"2023-06-03T20:12:31.903367+0000","last_clean":"2023-06-03T19:34:16.776254+0000","last_became_active":"2023-06-03T20:11:54.015754+0000","last_became_peered":"2023-06-03T20:11:54.015754+0000","last_unstale":"2023-06-03T20:12:31.903367+0000","last_undegraded":"2023-06-03T20:12:31.903367+0000","last_fullsized":"2023-06-03T20:11:54.008549+0000","mapping_epoch":195,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6],"acting":[5,6],"avail_no_missing":["5","6"],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1b","version":"60'1","reported_seq":170,"reported_epoch":200,"state":"active+undersized","last_fresh":"2023-06-03T20:12:31.940270+0000","last_change":"2023-06-03T20:11:54.016103+0000","last_active":"2023-06-03T20:12:31.940270+0000","last_peered":"2023-06-03T20:12:31.940270+0000","last_clean":"2023-06-03T20:11:43.979629+0000","last_became_active":"2023-06-03T20:11:54.016103+0000","last_became_peered":"2023-06-03T20:11:54.016103+0000","last_unstale":"2023-06-03T20:12:31.940270+0000","last_undegraded":"2023-06-03T20:12:31.940270+0000","last_fullsized":"2023-06-03T20:11:54.009298+0000","mapping_epoch":195,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T04:35:48.454086+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7],"acting":[0,7],"avail_no_missing":["0","7"],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":199,"reported_epoch":200,"state":"active+clean","last_fresh":"2023-06-03T20:12:32.089299+0000","last_change":"2023-06-03T20:07:19.914985+0000","last_active":"2023-06-03T20:12:32.089299+0000","last_peered":"2023-06-03T20:12:32.089299+0000","last_clean":"2023-06-03T20:12:32.089299+0000","last_became_active":"2023-06-03T20:07:19.914624+0000","last_became_peered":"2023-06-03T20:07:19.914624+0000","last_unstale":"2023-06-03T20:12:32.089299+0000","last_undegraded":"2023-06-03T20:12:32.089299+0000","last_fullsized":"2023-06-03T20:12:32.089299+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T07:02:50.515170+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"60'2","reported_seq":218,"reported_epoch":200,"state":"active+clean+remapped","last_fresh":"2023-06-03T20:12:31.955948+0000","last_change":"2023-06-03T20:12:31.955413+0000","last_active":"2023-06-03T20:12:31.955948+0000","last_peered":"2023-06-03T20:12:31.955948+0000","last_clean":"2023-06-03T20:12:31.955948+0000","last_became_active":"2023-06-03T20:12:31.909460+0000","last_became_peered":"2023-06-03T20:12:31.909460+0000","last_unstale":"2023-06-03T20:12:31.955948+0000","last_undegraded":"2023-06-03T20:12:31.955948+0000","last_fullsized":"2023-06-03T20:12:31.955948+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":1,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7],"acting":[5,7,6],"avail_no_missing":["5","6","7"],"object_location_counts":[{"shards":"5,6,7","objects":1}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":171,"reported_epoch":200,"state":"active+undersized","last_fresh":"2023-06-03T20:12:31.899779+0000","last_change":"2023-06-03T20:11:54.028489+0000","last_active":"2023-06-03T20:12:31.899779+0000","last_peered":"2023-06-03T20:12:31.899779+0000","last_clean":"2023-06-03T20:07:14.870985+0000","last_became_active":"2023-06-03T20:11:54.028489+0000","last_became_peered":"2023-06-03T20:11:54.028489+0000","last_unstale":"2023-06-03T20:12:31.899779+0000","last_undegraded":"2023-06-03T20:12:31.899779+0000","last_fullsized":"2023-06-03T20:11:54.011931+0000","mapping_epoch":195,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T04:24:31.291273+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2],"acting":[3,2],"avail_no_missing":["3","2"],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1d","version":"60'2","reported_seq":190,"reported_epoch":200,"state":"active+clean+remapped","last_fresh":"2023-06-03T20:12:32.002467+0000","last_change":"2023-06-03T20:12:32.001796+0000","last_active":"2023-06-03T20:12:32.002467+0000","last_peered":"2023-06-03T20:12:32.002467+0000","last_clean":"2023-06-03T20:12:32.002467+0000","last_became_active":"2023-06-03T20:12:31.909615+0000","last_became_peered":"2023-06-03T20:12:31.909615+0000","last_unstale":"2023-06-03T20:12:32.002467+0000","last_undegraded":"2023-06-03T20:12:32.002467+0000","last_fullsized":"2023-06-03T20:12:32.002467+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":1,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3],"acting":[5,3,6],"avail_no_missing":["5","3","6"],"object_location_counts":[{"shards":"3,5,6","objects":1}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":117,"reported_epoch":200,"state":"active+undersized","last_fresh":"2023-06-03T20:12:31.898486+0000","last_change":"2023-06-03T20:11:54.015791+0000","last_active":"2023-06-03T20:12:31.898486+0000","last_peered":"2023-06-03T20:12:31.898486+0000","last_clean":"2023-06-03T19:56:11.746795+0000","last_became_active":"2023-06-03T20:11:54.015791+0000","last_became_peered":"2023-06-03T20:11:54.015791+0000","last_unstale":"2023-06-03T20:12:31.898486+0000","last_undegraded":"2023-06-03T20:12:31.898486+0000","last_fullsized":"2023-06-03T20:11:54.007458+0000","mapping_epoch":195,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T19:54:52.259092+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0],"acting":[3,0],"avail_no_missing":["3","0"],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"60'1","reported_seq":193,"reported_epoch":201,"state":"active+clean","last_fresh":"2023-06-03T20:12:57.041378+0000","last_change":"2023-06-03T20:12:30.909386+0000","last_active":"2023-06-03T20:12:57.041378+0000","last_peered":"2023-06-03T20:12:57.041378+0000","last_clean":"2023-06-03T20:12:57.041378+0000","last_became_active":"2023-06-03T20:12:30.908991+0000","last_became_peered":"2023-06-03T20:12:30.908991+0000","last_unstale":"2023-06-03T20:12:57.041378+0000","last_undegraded":"2023-06-03T20:12:57.041378+0000","last_fullsized":"2023-06-03T20:12:57.041378+0000","mapping_epoch":198,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":199,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T04:33:12.721959+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,7],"acting":[3,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":195,"reported_epoch":200,"state":"active+clean","last_fresh":"2023-06-03T20:12:31.897564+0000","last_change":"2023-06-03T19:56:54.503416+0000","last_active":"2023-06-03T20:12:31.897564+0000","last_peered":"2023-06-03T20:12:31.897564+0000","last_clean":"2023-06-03T20:12:31.897564+0000","last_became_active":"2023-06-03T19:56:54.503018+0000","last_became_peered":"2023-06-03T19:56:54.503018+0000","last_unstale":"2023-06-03T20:12:31.897564+0000","last_undegraded":"2023-06-03T20:12:31.897564+0000","last_fullsized":"2023-06-03T20:12:31.897564+0000","mapping_epoch":119,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T20:14:00.330616+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"60'4","reported_seq":166,"reported_epoch":200,"state":"active+clean","last_fresh":"2023-06-03T20:12:31.940483+0000","last_change":"2023-06-03T19:56:11.748116+0000","last_active":"2023-06-03T20:12:31.940483+0000","last_peered":"2023-06-03T20:12:31.940483+0000","last_clean":"2023-06-03T20:12:31.940483+0000","last_became_active":"2023-06-03T19:56:11.747636+0000","last_became_peered":"2023-06-03T19:56:11.747636+0000","last_unstale":"2023-06-03T20:12:31.940483+0000","last_undegraded":"2023-06-03T20:12:31.940483+0000","last_fullsized":"2023-06-03T20:12:31.940483+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":115,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T06:57:48.052055+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":14,"reported_epoch":200,"state":"active+clean+remapped","last_fresh":"2023-06-03T20:12:32.530841+0000","last_change":"2023-06-03T20:12:32.093827+0000","last_active":"2023-06-03T20:12:32.530841+0000","last_peered":"2023-06-03T20:12:32.530841+0000","last_clean":"2023-06-03T20:12:32.530841+0000","last_became_active":"2023-06-03T20:12:32.093400+0000","last_became_peered":"2023-06-03T20:12:32.093400+0000","last_unstale":"2023-06-03T20:12:32.530841+0000","last_undegraded":"2023-06-03T20:12:32.530841+0000","last_fullsized":"2023-06-03T20:12:32.530841+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T23:22:58.054742+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2],"acting":[3,2,6],"avail_no_missing":["3","2","6"],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]}],"pool_stats":[{"poolid":11,"num_pg":16,"stat_sum":{"num_bytes":5992218648,"num_objects":91435,"num_object_clones":0,"num_object_copies":274305,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":39356,"num_objects_misplaced":8166,"num_objects_unfound":0,"num_objects_dirty":91435,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":119617,"num_write_kb":6753601,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2011,"num_bytes_recovered":131792896,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":16723943424,"data_stored":16723935280,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":109917,"ondisk_log_size":109917,"up":42,"acting":39,"num_store_stats":7},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":80,"acting":87,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5486,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":12,"num_objects_misplaced":2,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":62,"num_write_kb":30,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16458,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":30319,"internal_metadata":0},"log_size":56,"ondisk_log_size":56,"up":78,"acting":80,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":3,"num_store_stats":3},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":22,"acting":22,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":48,"seq":206158430716,"num_pgs":45,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3250272,"kb_used_data":2678168,"kb_used_omap":0,"kb_used_meta":572096,"kb_avail":90494880,"statfs":{"total":95995035648,"available":92666757120,"internally_reserved":0,"allocated":2742444032,"data_stored":2741934562,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":585826304},"hb_peers":[0,1,2,3,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,0,0,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 3 20:12:56 2023","interfaces":[{"interface":"back","average":{"1min":1.8160000000000001,"5min":2.0449999999999999,"15min":1.635},"min":{"1min":0.34200000000000003,"5min":0.26200000000000001,"15min":0.25900000000000001},"max":{"1min":5.5640000000000001,"5min":11.311999999999999,"15min":48.338999999999999},"last":0.49399999999999999},{"interface":"front","average":{"1min":1.4810000000000001,"5min":1.4850000000000001,"15min":1.532},"min":{"1min":0.27800000000000002,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":5.2089999999999996,"5min":5.2089999999999996,"15min":46.412999999999997},"last":0.68899999999999995}]},{"osd":1,"last update":"Sat Jun 3 20:12:45 2023","interfaces":[{"interface":"back","average":{"1min":1.405,"5min":1.4570000000000001,"15min":1.657},"min":{"1min":0.33200000000000002,"5min":0.33200000000000002,"15min":0.29699999999999999},"max":{"1min":5.8099999999999996,"5min":11.4,"15min":91.811000000000007},"last":0.45400000000000001},{"interface":"front","average":{"1min":1.0589999999999999,"5min":1.448,"15min":1.819},"min":{"1min":0.34100000000000003,"5min":0.34000000000000002,"15min":0.32200000000000001},"max":{"1min":3.3090000000000002,"5min":4.9850000000000003,"15min":91.695999999999998},"last":0.47599999999999998}]},{"osd":2,"last update":"Sat Jun 3 20:12:45 2023","interfaces":[{"interface":"back","average":{"1min":16.513999999999999,"5min":4.5949999999999998,"15min":2.4089999999999998},"min":{"1min":0.32400000000000001,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":286.83600000000001,"5min":286.83600000000001,"15min":286.83600000000001},"last":4.577},{"interface":"front","average":{"1min":16.219000000000001,"5min":4.4900000000000002,"15min":2.5169999999999999},"min":{"1min":0.41599999999999998,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":287.32900000000001,"5min":287.32900000000001,"15min":287.32900000000001},"last":4.6660000000000004}]},{"osd":3,"last update":"Sat Jun 3 20:12:35 2023","interfaces":[{"interface":"back","average":{"1min":7.2400000000000002,"5min":2.927,"15min":2.226},"min":{"1min":0.30299999999999999,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":98.763000000000005,"5min":98.763000000000005,"15min":98.763000000000005},"last":0.38700000000000001},{"interface":"front","average":{"1min":7.1109999999999998,"5min":2.9289999999999998,"15min":2.3639999999999999},"min":{"1min":0.38500000000000001,"5min":0.34399999999999997,"15min":0.34399999999999997},"max":{"1min":97.882999999999996,"5min":97.882999999999996,"15min":97.882999999999996},"last":1.3700000000000001}]},{"osd":4,"last update":"Sat Jun 3 20:10:54 2023","interfaces":[{"interface":"back","average":{"1min":1.417,"5min":1.8879999999999999,"15min":1.3260000000000001},"min":{"1min":0.33900000000000002,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":7.1189999999999998,"5min":21.888999999999999,"15min":21.888999999999999},"last":0.40799999999999997},{"interface":"front","average":{"1min":1.484,"5min":2.1739999999999999,"15min":1.671},"min":{"1min":0.60799999999999998,"5min":0.443,"15min":0.38800000000000001},"max":{"1min":4.8600000000000003,"5min":10.507999999999999,"15min":12.917},"last":0.48899999999999999}]},{"osd":5,"last update":"Sat Jun 3 20:12:35 2023","interfaces":[{"interface":"back","average":{"1min":1.5289999999999999,"5min":2.04,"15min":1.4430000000000001},"min":{"1min":0.28899999999999998,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":5.6360000000000001,"5min":11.443,"15min":11.443},"last":0.42699999999999999},{"interface":"front","average":{"1min":1.27,"5min":2.1389999999999998,"15min":1.8029999999999999},"min":{"1min":0.442,"5min":0.439,"15min":0.33100000000000002},"max":{"1min":3.3519999999999999,"5min":8.6300000000000008,"15min":20.306999999999999},"last":0.51700000000000002}]},{"osd":6,"last update":"Sat Jun 3 20:12:35 2023","interfaces":[{"interface":"back","average":{"1min":1.8959999999999999,"5min":2.5699999999999998,"15min":1.7729999999999999},"min":{"1min":0.39300000000000002,"5min":0.39300000000000002,"15min":0.39300000000000002},"max":{"1min":10.183,"5min":11.256,"15min":11.256},"last":0.54300000000000004},{"interface":"front","average":{"1min":1.7729999999999999,"5min":2.4550000000000001,"15min":1.8320000000000001},"min":{"1min":0.33300000000000002,"5min":0.33300000000000002,"15min":0.32500000000000001},"max":{"1min":4.3630000000000004,"5min":10.894,"15min":10.894},"last":3.3039999999999998}]}]},{"osd":6,"up_from":42,"seq":180388626949,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3685652,"kb_used_data":3192012,"kb_used_omap":0,"kb_used_meta":493632,"kb_avail":90059500,"statfs":{"total":95995035648,"available":92220928000,"internally_reserved":0,"allocated":3268620288,"data_stored":3268122669,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":505479168},"hb_peers":[0,1,2,3,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1,0,1],"upper_bound":16},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 3 20:13:08 2023","interfaces":[{"interface":"back","average":{"1min":1.244,"5min":3.1219999999999999,"15min":2.21},"min":{"1min":0.25600000000000001,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":4.5410000000000004,"5min":54.283999999999999,"15min":54.786000000000001},"last":4.2960000000000003},{"interface":"front","average":{"1min":1.8149999999999999,"5min":3.5179999999999998,"15min":2.4319999999999999},"min":{"1min":0.29399999999999998,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":5.194,"5min":54.314999999999998,"15min":54.936},"last":4.2839999999999998}]},{"osd":1,"last update":"Sat Jun 3 20:12:42 2023","interfaces":[{"interface":"back","average":{"1min":1.0209999999999999,"5min":1.4350000000000001,"15min":2.2149999999999999},"min":{"1min":0.318,"5min":0.318,"15min":0.28000000000000003},"max":{"1min":3.5310000000000001,"5min":6.7510000000000003,"15min":160.38499999999999},"last":0.42199999999999999},{"interface":"front","average":{"1min":1.3620000000000001,"5min":1.863,"15min":2.2149999999999999},"min":{"1min":0.35099999999999998,"5min":0.307,"15min":0.29599999999999999},"max":{"1min":3.968,"5min":12.132999999999999,"15min":160.434},"last":0.41599999999999998}]},{"osd":2,"last update":"Sat Jun 3 20:12:42 2023","interfaces":[{"interface":"back","average":{"1min":8.4730000000000008,"5min":3.831,"15min":1.8620000000000001},"min":{"1min":0.34999999999999998,"5min":0.32500000000000001,"15min":0.27200000000000002},"max":{"1min":136.22,"5min":136.22,"15min":136.22},"last":3.6240000000000001},{"interface":"front","average":{"1min":7.4500000000000002,"5min":3.9020000000000001,"15min":1.974},"min":{"1min":0.34999999999999998,"5min":0.33700000000000002,"15min":0.33700000000000002},"max":{"1min":135.68799999999999,"5min":135.68799999999999,"15min":135.68799999999999},"last":3.6869999999999998}]},{"osd":3,"last update":"Sat Jun 3 20:13:21 2023","interfaces":[{"interface":"back","average":{"1min":1.694,"5min":1.546,"15min":2.4100000000000001},"min":{"1min":0.33600000000000002,"5min":0.33600000000000002,"15min":0.32200000000000001},"max":{"1min":7.1900000000000004,"5min":10.398,"15min":164.99199999999999},"last":0.34100000000000003},{"interface":"front","average":{"1min":1.2729999999999999,"5min":1.524,"15min":2.335},"min":{"1min":0.39100000000000001,"5min":0.38400000000000001,"15min":0.316},"max":{"1min":3.621,"5min":6.7770000000000001,"15min":163.459},"last":1.3069999999999999}]},{"osd":4,"last update":"Sat Jun 3 20:10:57 2023","interfaces":[{"interface":"back","average":{"1min":2.0510000000000002,"5min":2.0350000000000001,"15min":1.3029999999999999},"min":{"1min":0.35999999999999999,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":8.2230000000000008,"5min":14.098000000000001,"15min":14.098000000000001},"last":0.69699999999999995},{"interface":"front","average":{"1min":1.929,"5min":1.738,"15min":1.2569999999999999},"min":{"1min":0.52900000000000003,"5min":0.39600000000000002,"15min":0.377},"max":{"1min":8.452,"5min":8.452,"15min":8.452},"last":0.55700000000000005}]},{"osd":5,"last update":"Sat Jun 3 20:13:03 2023","interfaces":[{"interface":"back","average":{"1min":1.095,"5min":1.673,"15min":1.3029999999999999},"min":{"1min":0.35599999999999998,"5min":0.35599999999999998,"15min":0.32200000000000001},"max":{"1min":4.6909999999999998,"5min":14.731999999999999,"15min":14.731999999999999},"last":0.49099999999999999},{"interface":"front","average":{"1min":1.3720000000000001,"5min":2.1499999999999999,"15min":1.718},"min":{"1min":0.34799999999999998,"5min":0.34000000000000002,"15min":0.29499999999999998},"max":{"1min":5.4900000000000002,"5min":15.295,"15min":15.295},"last":0.38600000000000001}]},{"osd":7,"last update":"Sat Jun 3 20:12:27 2023","interfaces":[{"interface":"back","average":{"1min":1.28,"5min":1.855,"15min":1.518},"min":{"1min":0.33200000000000002,"5min":0.33200000000000002,"15min":0.33200000000000002},"max":{"1min":3.968,"5min":8.4269999999999996,"15min":11.362},"last":0.46500000000000002},{"interface":"front","average":{"1min":1.78,"5min":2.367,"15min":1.653},"min":{"1min":0.40200000000000002,"5min":0.40200000000000002,"15min":0.33900000000000002},"max":{"1min":10.489000000000001,"5min":21.202000000000002,"15min":21.202000000000002},"last":1.6359999999999999}]}]},{"osd":1,"up_from":0,"seq":442381631747,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":0,"up_from":86,"seq":369367187774,"num_pgs":47,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3432716,"kb_used_data":2963588,"kb_used_omap":5,"kb_used_meta":468986,"kb_avail":90312436,"statfs":{"total":95995035648,"available":92479934464,"internally_reserved":0,"allocated":3034714112,"data_stored":3033933291,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":480242298},"hb_peers":[1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Jun 3 20:12:31 2023","interfaces":[{"interface":"back","average":{"1min":0.86299999999999999,"5min":0.76900000000000002,"15min":1.4279999999999999},"min":{"1min":0.24399999999999999,"5min":0.24399999999999999,"15min":0.192},"max":{"1min":3.8700000000000001,"5min":6.016,"15min":81.310000000000002},"last":0.46700000000000003},{"interface":"front","average":{"1min":1.673,"5min":1.1930000000000001,"15min":1.6499999999999999},"min":{"1min":0.23100000000000001,"5min":0.22600000000000001,"15min":0.187},"max":{"1min":8.4320000000000004,"5min":8.4320000000000004,"15min":76.971999999999994},"last":0.39200000000000002}]},{"osd":2,"last update":"Sat Jun 3 20:12:31 2023","interfaces":[{"interface":"back","average":{"1min":1.1970000000000001,"5min":0.96799999999999997,"15min":1.0409999999999999},"min":{"1min":0.32200000000000001,"5min":0.193,"15min":0.193},"max":{"1min":3.6360000000000001,"5min":6.0019999999999998,"15min":11.901},"last":0.78200000000000003},{"interface":"front","average":{"1min":1.889,"5min":1.163,"15min":1.1599999999999999},"min":{"1min":0.27800000000000002,"5min":0.255,"15min":0.218},"max":{"1min":7.569,"5min":7.569,"15min":17.315999999999999},"last":0.52900000000000003}]},{"osd":3,"last update":"Sat Jun 3 20:13:20 2023","interfaces":[{"interface":"back","average":{"1min":2.7599999999999998,"5min":1.3480000000000001,"15min":1.2509999999999999},"min":{"1min":0.40300000000000002,"5min":0.28999999999999998,"15min":0.216},"max":{"1min":8.6539999999999999,"5min":8.6539999999999999,"15min":8.6539999999999999},"last":0.73099999999999998},{"interface":"front","average":{"1min":2.726,"5min":1.538,"15min":1.3720000000000001},"min":{"1min":0.36099999999999999,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":8.2449999999999992,"5min":9.1349999999999998,"15min":9.1349999999999998},"last":0.92000000000000004}]},{"osd":4,"last update":"Sat Jun 3 20:10:59 2023","interfaces":[{"interface":"back","average":{"1min":1.9370000000000001,"5min":1.7110000000000001,"15min":1.3400000000000001},"min":{"1min":0.45200000000000001,"5min":0.318,"15min":0.318},"max":{"1min":10.025,"5min":10.025,"15min":10.025},"last":0.497},{"interface":"front","average":{"1min":1.673,"5min":1.8160000000000001,"15min":1.429},"min":{"1min":0.56699999999999995,"5min":0.375,"15min":0.375},"max":{"1min":4.6059999999999999,"5min":13.058999999999999,"15min":13.058999999999999},"last":0.75900000000000001}]},{"osd":5,"last update":"Sat Jun 3 20:13:01 2023","interfaces":[{"interface":"back","average":{"1min":1.758,"5min":2.0699999999999998,"15min":1.7370000000000001},"min":{"1min":0.36699999999999999,"5min":0.36699999999999999,"15min":0.34899999999999998},"max":{"1min":9.3829999999999991,"5min":9.3829999999999991,"15min":9.3829999999999991},"last":2.1019999999999999},{"interface":"front","average":{"1min":1.964,"5min":1.5920000000000001,"15min":1.516},"min":{"1min":0.378,"5min":0.378,"15min":0.34000000000000002},"max":{"1min":9.1820000000000004,"5min":10.159000000000001,"15min":10.159000000000001},"last":0.60999999999999999}]},{"osd":6,"last update":"Sat Jun 3 20:13:01 2023","interfaces":[{"interface":"back","average":{"1min":1.9810000000000001,"5min":1.623,"15min":1.448},"min":{"1min":0.39800000000000002,"5min":0.35499999999999998,"15min":0.35499999999999998},"max":{"1min":9.1489999999999991,"5min":10.140000000000001,"15min":10.140000000000001},"last":0.68500000000000005},{"interface":"front","average":{"1min":1.5109999999999999,"5min":1.825,"15min":1.5860000000000001},"min":{"1min":0.45700000000000002,"5min":0.44600000000000001,"15min":0.42799999999999999},"max":{"1min":6.1639999999999997,"5min":13.117000000000001,"15min":13.117000000000001},"last":0.86399999999999999}]},{"osd":7,"last update":"Sat Jun 3 20:13:01 2023","interfaces":[{"interface":"back","average":{"1min":2.9489999999999998,"5min":1.732,"15min":1.629},"min":{"1min":0.46600000000000003,"5min":0.39800000000000002,"15min":0.35099999999999998},"max":{"1min":11.436999999999999,"5min":11.436999999999999,"15min":11.436999999999999},"last":0.76500000000000001},{"interface":"front","average":{"1min":2.0739999999999998,"5min":1.7110000000000001,"15min":1.7130000000000001},"min":{"1min":0.51700000000000002,"5min":0.51200000000000001,"15min":0.47899999999999998},"max":{"1min":6.4539999999999997,"5min":6.4740000000000002,"15min":8.891},"last":0.89600000000000002}]}]},{"osd":2,"up_from":114,"seq":489626271953,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2488776,"kb_used_data":2302464,"kb_used_omap":7,"kb_used_meta":186296,"kb_avail":91256376,"statfs":{"total":95995035648,"available":93446529024,"internally_reserved":0,"allocated":2357723136,"data_stored":2357004014,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8111,"internal_metadata":190767185},"hb_peers":[0,1,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 3 20:12:38 2023","interfaces":[{"interface":"back","average":{"1min":0.76200000000000001,"5min":3.3780000000000001,"15min":2.5720000000000001},"min":{"1min":0.20999999999999999,"5min":0.20999999999999999,"15min":0.20899999999999999},"max":{"1min":2.4089999999999998,"5min":273.51100000000002,"15min":273.51100000000002},"last":0.20799999999999999},{"interface":"front","average":{"1min":1.4830000000000001,"5min":3.7679999999999998,"15min":2.7639999999999998},"min":{"1min":0.26200000000000001,"5min":0.247,"15min":0.221},"max":{"1min":5.1239999999999997,"5min":273.67399999999998,"15min":273.67399999999998},"last":4.3550000000000004}]},{"osd":1,"last update":"Sat Jun 3 20:12:38 2023","interfaces":[{"interface":"back","average":{"1min":1.3180000000000001,"5min":3.528,"15min":2.25},"min":{"1min":0.29399999999999998,"5min":0.254,"15min":0.22900000000000001},"max":{"1min":5.5499999999999998,"5min":273.46499999999997,"15min":273.46499999999997},"last":4.46},{"interface":"front","average":{"1min":0.91200000000000003,"5min":3.6179999999999999,"15min":2.3079999999999998},"min":{"1min":0.28399999999999997,"5min":0.26300000000000001,"15min":0.23699999999999999},"max":{"1min":4.1779999999999999,"5min":274.38400000000001,"15min":274.38400000000001},"last":3.2570000000000001}]},{"osd":3,"last update":"Sat Jun 3 20:12:26 2023","interfaces":[{"interface":"back","average":{"1min":0.90500000000000003,"5min":4.1609999999999996,"15min":2.177},"min":{"1min":0.248,"5min":0.248,"15min":0.22700000000000001},"max":{"1min":2.4420000000000002,"5min":273.71899999999999,"15min":273.71899999999999},"last":0.24399999999999999},{"interface":"front","average":{"1min":1.2549999999999999,"5min":4.3540000000000001,"15min":2.5379999999999998},"min":{"1min":0.35699999999999998,"5min":0.24199999999999999,"15min":0.24199999999999999},"max":{"1min":5.0069999999999997,"5min":274.21100000000001,"15min":274.21100000000001},"last":1.4410000000000001}]},{"osd":4,"last update":"Sat Jun 3 20:10:59 2023","interfaces":[{"interface":"back","average":{"1min":16.218,"5min":4.9749999999999996,"15min":2.2519999999999998},"min":{"1min":0.48699999999999999,"5min":0.32800000000000001,"15min":0.32700000000000001},"max":{"1min":273.75700000000001,"5min":273.75700000000001,"15min":273.75700000000001},"last":1.6679999999999999},{"interface":"front","average":{"1min":15.512,"5min":4.7480000000000002,"15min":2.1850000000000001},"min":{"1min":0.36199999999999999,"5min":0.35699999999999998,"15min":0.32600000000000001},"max":{"1min":274.27300000000002,"5min":274.27300000000002,"15min":274.27300000000002},"last":1.585}]},{"osd":5,"last update":"Sat Jun 3 20:12:38 2023","interfaces":[{"interface":"back","average":{"1min":1.071,"5min":3.9289999999999998,"15min":2.032},"min":{"1min":0.317,"5min":0.317,"15min":0.317},"max":{"1min":2.5209999999999999,"5min":273.78800000000001,"15min":273.78800000000001},"last":0.379},{"interface":"front","average":{"1min":1.607,"5min":4.2560000000000002,"15min":2.3239999999999998},"min":{"1min":0.32800000000000001,"5min":0.32800000000000001,"15min":0.308},"max":{"1min":5.6059999999999999,"5min":274.15800000000002,"15min":274.15800000000002},"last":4.4279999999999999}]},{"osd":6,"last update":"Sat Jun 3 20:12:38 2023","interfaces":[{"interface":"back","average":{"1min":1.7230000000000001,"5min":4.4969999999999999,"15min":2.2549999999999999},"min":{"1min":0.41499999999999998,"5min":0.376,"15min":0.376},"max":{"1min":5.0970000000000004,"5min":274.03899999999999,"15min":274.03899999999999},"last":0.35099999999999998},{"interface":"front","average":{"1min":1.994,"5min":4.782,"15min":2.3290000000000002},"min":{"1min":0.39700000000000002,"5min":0.39100000000000001,"15min":0.34899999999999998},"max":{"1min":5.6399999999999997,"5min":274.06299999999999,"15min":274.06299999999999},"last":4.4059999999999997}]},{"osd":7,"last update":"Sat Jun 3 20:12:38 2023","interfaces":[{"interface":"back","average":{"1min":1.794,"5min":4.2830000000000004,"15min":2.452},"min":{"1min":0.41799999999999998,"5min":0.39600000000000002,"15min":0.39600000000000002},"max":{"1min":6.1609999999999996,"5min":274.54000000000002,"15min":274.54000000000002},"last":0.40799999999999997},{"interface":"front","average":{"1min":2.625,"5min":4.5019999999999998,"15min":2.4550000000000001},"min":{"1min":0.39900000000000002,"5min":0.39900000000000002,"15min":0.35699999999999998},"max":{"1min":7.1760000000000002,"5min":274.12700000000001,"15min":274.12700000000001},"last":4.3869999999999996}]}]},{"osd":3,"up_from":119,"seq":511101108424,"num_pgs":47,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2832972,"kb_used_data":2567556,"kb_used_omap":2,"kb_used_meta":265405,"kb_avail":90912180,"statfs":{"total":95995035648,"available":93094072320,"internally_reserved":0,"allocated":2629177344,"data_stored":2628452021,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":271774773},"hb_peers":[0,1,2,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1,3,1,0,2,2],"upper_bound":256},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 3 20:12:25 2023","interfaces":[{"interface":"back","average":{"1min":1.353,"5min":1.6919999999999999,"15min":1.73},"min":{"1min":0.192,"5min":0.192,"15min":0.19},"max":{"1min":11.164,"5min":36.918999999999997,"15min":68.647999999999996},"last":7.6200000000000001},{"interface":"front","average":{"1min":1.0029999999999999,"5min":2.2040000000000002,"15min":1.885},"min":{"1min":0.22600000000000001,"5min":0.22600000000000001,"15min":0.214},"max":{"1min":5.8609999999999998,"5min":42.914999999999999,"15min":69.691000000000003},"last":0.76900000000000002}]},{"osd":1,"last update":"Sat Jun 3 20:12:25 2023","interfaces":[{"interface":"back","average":{"1min":0.96499999999999997,"5min":1.0449999999999999,"15min":1.0720000000000001},"min":{"1min":0.30599999999999999,"5min":0.23100000000000001,"15min":0.214},"max":{"1min":3.851,"5min":10.749000000000001,"15min":10.749000000000001},"last":0.35499999999999998},{"interface":"front","average":{"1min":1.056,"5min":0.81299999999999994,"15min":0.996},"min":{"1min":0.24099999999999999,"5min":0.24099999999999999,"15min":0.23499999999999999},"max":{"1min":5.1769999999999996,"5min":5.4960000000000004,"15min":10.792999999999999},"last":0.29299999999999998}]},{"osd":2,"last update":"Sat Jun 3 20:12:25 2023","interfaces":[{"interface":"back","average":{"1min":1.1819999999999999,"5min":1.0820000000000001,"15min":1.282},"min":{"1min":0.248,"5min":0.217,"15min":0.217},"max":{"1min":8.8230000000000004,"5min":8.8230000000000004,"15min":62.527999999999999},"last":0.46200000000000002},{"interface":"front","average":{"1min":1.0780000000000001,"5min":1.151,"15min":1.3},"min":{"1min":0.32200000000000001,"5min":0.28399999999999997,"15min":0.214},"max":{"1min":4.4800000000000004,"5min":10.704000000000001,"15min":57.935000000000002},"last":0.55300000000000005}]},{"osd":4,"last update":"Sat Jun 3 20:10:53 2023","interfaces":[{"interface":"back","average":{"1min":2.0379999999999998,"5min":1.98,"15min":1.413},"min":{"1min":0.42399999999999999,"5min":0.38200000000000001,"15min":0.38200000000000001},"max":{"1min":9.8360000000000003,"5min":12.757,"15min":12.757},"last":1.968},{"interface":"front","average":{"1min":2.1749999999999998,"5min":1.744,"15min":1.333},"min":{"1min":0.39000000000000001,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":10.122,"5min":10.122,"15min":10.122},"last":0.29099999999999998}]},{"osd":5,"last update":"Sat Jun 3 20:12:25 2023","interfaces":[{"interface":"back","average":{"1min":1.355,"5min":1.224,"15min":1.2290000000000001},"min":{"1min":0.41399999999999998,"5min":0.38100000000000001,"15min":0.32900000000000001},"max":{"1min":5.1340000000000003,"5min":5.4269999999999996,"15min":7.1230000000000002},"last":0.38700000000000001},{"interface":"front","average":{"1min":0.94799999999999995,"5min":1.488,"15min":1.5129999999999999},"min":{"1min":0.32700000000000001,"5min":0.32700000000000001,"15min":0.314},"max":{"1min":4.8109999999999999,"5min":12.56,"15min":12.56},"last":0.48799999999999999}]},{"osd":6,"last update":"Sat Jun 3 20:12:25 2023","interfaces":[{"interface":"back","average":{"1min":1.625,"5min":1.9139999999999999,"15min":1.5960000000000001},"min":{"1min":0.35099999999999998,"5min":0.35099999999999998,"15min":0.35099999999999998},"max":{"1min":13.718,"5min":13.718,"15min":13.718},"last":0.435},{"interface":"front","average":{"1min":1.964,"5min":2.1619999999999999,"15min":1.599},"min":{"1min":0.40699999999999997,"5min":0.40699999999999997,"15min":0.35599999999999998},"max":{"1min":13.734999999999999,"5min":13.734999999999999,"15min":13.734999999999999},"last":0.58399999999999996}]},{"osd":7,"last update":"Sat Jun 3 20:12:25 2023","interfaces":[{"interface":"back","average":{"1min":1.3089999999999999,"5min":1.696,"15min":1.4199999999999999},"min":{"1min":0.438,"5min":0.42399999999999999,"15min":0.39100000000000001},"max":{"1min":3.774,"5min":10.858000000000001,"15min":10.858000000000001},"last":4.4240000000000004},{"interface":"front","average":{"1min":1.792,"5min":1.913,"15min":1.6930000000000001},"min":{"1min":0.38100000000000001,"5min":0.38100000000000001,"15min":0.38100000000000001},"max":{"1min":5.2119999999999997,"5min":11.084,"15min":12.715999999999999},"last":0.47799999999999998}]}]},{"osd":4,"up_from":124,"seq":532575944879,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1005320,"kb_used_data":818944,"kb_used_omap":10,"kb_used_meta":186357,"kb_avail":92739832,"statfs":{"total":95995035648,"available":94965587968,"internally_reserved":0,"allocated":838598656,"data_stored":837828790,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":10652,"internal_metadata":190830180},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 3 20:11:03 2023","interfaces":[{"interface":"back","average":{"1min":1.002,"5min":4.8399999999999999,"15min":2.468},"min":{"1min":0.58599999999999997,"5min":0.29499999999999998,"15min":0.254},"max":{"1min":3.0139999999999998,"5min":214.72999999999999,"15min":214.72999999999999},"last":2.1539999999999999},{"interface":"front","average":{"1min":2.04,"5min":5.327,"15min":2.6459999999999999},"min":{"1min":0.41399999999999998,"5min":0.40200000000000002,"15min":0.32400000000000001},"max":{"1min":15,"5min":214.76499999999999,"15min":214.76499999999999},"last":0.625}]},{"osd":1,"last update":"Sat Jun 3 20:11:03 2023","interfaces":[{"interface":"back","average":{"1min":1.1439999999999999,"5min":1.639,"15min":1.55},"min":{"1min":0.371,"5min":0.35199999999999998,"15min":0.28100000000000003},"max":{"1min":3.8210000000000002,"5min":10.446,"15min":58.32},"last":0.84599999999999997},{"interface":"front","average":{"1min":0.878,"5min":1.4990000000000001,"15min":1.4610000000000001},"min":{"1min":0.498,"5min":0.34999999999999998,"15min":0.307},"max":{"1min":1.8,"5min":12.919,"15min":58.286000000000001},"last":2.2429999999999999}]},{"osd":2,"last update":"Sat Jun 3 20:11:03 2023","interfaces":[{"interface":"back","average":{"1min":13.406000000000001,"5min":4.6070000000000002,"15min":2.1419999999999999},"min":{"1min":0.55000000000000004,"5min":0.317,"15min":0.30299999999999999},"max":{"1min":248.65799999999999,"5min":248.65799999999999,"15min":248.65799999999999},"last":2.2770000000000001},{"interface":"front","average":{"1min":14.144,"5min":4.7720000000000002,"15min":2.1890000000000001},"min":{"1min":0.46700000000000003,"5min":0.373,"15min":0.34499999999999997},"max":{"1min":249.077,"5min":249.077,"15min":249.077},"last":0.66200000000000003}]},{"osd":3,"last update":"Sat Jun 3 20:11:03 2023","interfaces":[{"interface":"back","average":{"1min":1.2330000000000001,"5min":1.778,"15min":1.6240000000000001},"min":{"1min":0.40600000000000003,"5min":0.35199999999999998,"15min":0.32500000000000001},"max":{"1min":3.7759999999999998,"5min":10.301,"15min":57.954999999999998},"last":1.0920000000000001},{"interface":"front","average":{"1min":1.226,"5min":1.6479999999999999,"15min":1.613},"min":{"1min":0.50700000000000001,"5min":0.40300000000000002,"15min":0.33700000000000002},"max":{"1min":4.9210000000000003,"5min":12.890000000000001,"15min":57.889000000000003},"last":2.2050000000000001}]},{"osd":5,"last update":"Sat Jun 3 20:11:03 2023","interfaces":[{"interface":"back","average":{"1min":1.4590000000000001,"5min":2.0819999999999999,"15min":1.623},"min":{"1min":0.35899999999999999,"5min":0.32000000000000001,"15min":0.313},"max":{"1min":6.6859999999999999,"5min":11.311,"15min":86.576999999999998},"last":0.47599999999999998},{"interface":"front","average":{"1min":1.5269999999999999,"5min":2.1760000000000002,"15min":1.7390000000000001},"min":{"1min":0.32900000000000001,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":7.9269999999999996,"5min":10.997999999999999,"15min":88.013000000000005},"last":4.9119999999999999}]},{"osd":6,"last update":"Sat Jun 3 20:11:03 2023","interfaces":[{"interface":"back","average":{"1min":1.8140000000000001,"5min":2.7429999999999999,"15min":1.5309999999999999},"min":{"1min":0.36099999999999999,"5min":0.34599999999999997,"15min":0.34599999999999997},"max":{"1min":6.6520000000000001,"5min":14.244999999999999,"15min":14.244999999999999},"last":2.109},{"interface":"front","average":{"1min":2.4329999999999998,"5min":2.8679999999999999,"15min":1.629},"min":{"1min":0.42599999999999999,"5min":0.40100000000000002,"15min":0.36499999999999999},"max":{"1min":10.359,"5min":12.145,"15min":12.145},"last":0.81200000000000006}]},{"osd":7,"last update":"Sat Jun 3 20:11:03 2023","interfaces":[{"interface":"back","average":{"1min":1.7270000000000001,"5min":2.052,"15min":1.55},"min":{"1min":0.44,"5min":0.41199999999999998,"15min":0.39700000000000002},"max":{"1min":9.6210000000000004,"5min":10.388,"15min":10.388},"last":4.8150000000000004},{"interface":"front","average":{"1min":1.0629999999999999,"5min":1.7729999999999999,"15min":1.486},"min":{"1min":0.44800000000000001,"5min":0.34200000000000003,"15min":0.31900000000000001},"max":{"1min":1.8260000000000001,"5min":12.951000000000001,"15min":12.951000000000001},"last":2.3109999999999999}]}]},{"osd":5,"up_from":36,"seq":154618823183,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2451396,"kb_used_data":1846908,"kb_used_omap":0,"kb_used_meta":604480,"kb_avail":91293756,"statfs":{"total":95995035648,"available":93484806144,"internally_reserved":0,"allocated":1891233792,"data_stored":1890747827,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":618987520},"hb_peers":[0,1,2,3,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1],"upper_bound":8},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 3 20:12:52 2023","interfaces":[{"interface":"back","average":{"1min":1.0449999999999999,"5min":4.0899999999999999,"15min":2.1309999999999998},"min":{"1min":0.28799999999999998,"5min":0.28100000000000003,"15min":0.22800000000000001},"max":{"1min":5.734,"5min":185.29900000000001,"15min":185.29900000000001},"last":0.28399999999999997},{"interface":"front","average":{"1min":1.0680000000000001,"5min":4.3630000000000004,"15min":2.488},"min":{"1min":0.27700000000000002,"5min":0.27700000000000002,"15min":0.26300000000000001},"max":{"1min":5.7460000000000004,"5min":185.44200000000001,"15min":185.44200000000001},"last":0.66200000000000003}]},{"osd":1,"last update":"Sat Jun 3 20:12:52 2023","interfaces":[{"interface":"back","average":{"1min":1.2070000000000001,"5min":1.5309999999999999,"15min":2.145},"min":{"1min":0.34200000000000003,"5min":0.34200000000000003,"15min":0.30199999999999999},"max":{"1min":3.7749999999999999,"5min":8.5050000000000008,"15min":114.681},"last":0.71599999999999997},{"interface":"front","average":{"1min":0.94199999999999995,"5min":1.399,"15min":2.129},"min":{"1min":0.34599999999999997,"5min":0.32200000000000001,"15min":0.307},"max":{"1min":3.427,"5min":9.2739999999999991,"15min":115.366},"last":0.80000000000000004}]},{"osd":2,"last update":"Sat Jun 3 20:12:52 2023","interfaces":[{"interface":"back","average":{"1min":0.996,"5min":2.6259999999999999,"15min":1.575},"min":{"1min":0.32500000000000001,"5min":0.32500000000000001,"15min":0.318},"max":{"1min":4.9800000000000004,"5min":92.412000000000006,"15min":92.412000000000006},"last":2.0699999999999998},{"interface":"front","average":{"1min":1.7270000000000001,"5min":3.0049999999999999,"15min":1.9099999999999999},"min":{"1min":0.314,"5min":0.314,"15min":0.314},"max":{"1min":5.5010000000000003,"5min":94.552000000000007,"15min":94.552000000000007},"last":2.4039999999999999}]},{"osd":3,"last update":"Sat Jun 3 20:12:25 2023","interfaces":[{"interface":"back","average":{"1min":1.3859999999999999,"5min":1.8560000000000001,"15min":2.7599999999999998},"min":{"1min":0.377,"5min":0.37,"15min":0.317},"max":{"1min":4.2229999999999999,"5min":8.4179999999999993,"15min":261.14100000000002},"last":0.749},{"interface":"front","average":{"1min":1.1799999999999999,"5min":1.4399999999999999,"15min":2.431},"min":{"1min":0.38500000000000001,"5min":0.35799999999999998,"15min":0.35399999999999998},"max":{"1min":3.7450000000000001,"5min":9.0990000000000002,"15min":261.27100000000002},"last":0.38800000000000001}]},{"osd":4,"last update":"Sat Jun 3 20:11:00 2023","interfaces":[{"interface":"back","average":{"1min":2.2930000000000001,"5min":2.4510000000000001,"15min":1.6839999999999999},"min":{"1min":0.44,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":7.5910000000000002,"5min":14.951000000000001,"15min":14.951000000000001},"last":0.81899999999999995},{"interface":"front","average":{"1min":1.619,"5min":2.0129999999999999,"15min":1.3799999999999999},"min":{"1min":0.42299999999999999,"5min":0.376,"15min":0.376},"max":{"1min":7.2000000000000002,"5min":12.109,"15min":12.109},"last":0.66900000000000004}]},{"osd":6,"last update":"Sat Jun 3 20:12:52 2023","interfaces":[{"interface":"back","average":{"1min":1.506,"5min":2.2919999999999998,"15min":1.8},"min":{"1min":0.42899999999999999,"5min":0.374,"15min":0.35799999999999998},"max":{"1min":6.4480000000000004,"5min":11.167,"15min":22.945},"last":0.45200000000000001},{"interface":"front","average":{"1min":1.4319999999999999,"5min":2.508,"15min":1.982},"min":{"1min":0.38400000000000001,"5min":0.38400000000000001,"15min":0.38300000000000001},"max":{"1min":6.3710000000000004,"5min":12.140000000000001,"15min":22.428000000000001},"last":0.46899999999999997}]},{"osd":7,"last update":"Sat Jun 3 20:12:32 2023","interfaces":[{"interface":"back","average":{"1min":1.4830000000000001,"5min":2.1640000000000001,"15min":1.8959999999999999},"min":{"1min":0.36299999999999999,"5min":0.36299999999999999,"15min":0.36299999999999999},"max":{"1min":3.839,"5min":14.223000000000001,"15min":14.223000000000001},"last":0.42699999999999999},{"interface":"front","average":{"1min":1.0680000000000001,"5min":1.7749999999999999,"15min":1.393},"min":{"1min":0.33500000000000002,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":4.0789999999999997,"5min":9.0719999999999992,"15min":9.0719999999999992},"last":0.50800000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6928,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4862,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":10652,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4786,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4932,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":3027042304,"data_stored":3027042304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":2338459648,"data_stored":2338455576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":2621308928,"data_stored":2621308928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":871366656,"data_stored":871366656,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":1879048192,"data_stored":1879048192,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":3253932032,"data_stored":3253927960,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":2732785664,"data_stored":2732785664,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-06-03T20:13:29.235 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph osd pool set device_health_metrics pgp_num 1 2023-06-03T20:13:29.294 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:13:29.326 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:13:29.329 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:13:29.330 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:13:29.430 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:13:29.457 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:13:29.461 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:13:29.461 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:13:29.562 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:13:29.591 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:13:29.594 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:13:29.594 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:13:29.695 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:13:29.735 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:13:29.737 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:13:29.737 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:13:29.833 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:13:29 smithi164 bash[59462]: cluster 1685823208.7245626 mgr.y (mgr.34104) 1123 : cluster 0 pgmap v1096: 89 pgs: 9 active+clean+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 13 active+undersized+degraded, 1 active+recovering+undersized+remapped, 20 active+undersized, 44 active+clean; 5.4 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 752 op/s; 38922/266484 objects degraded (14.606%); 7941/266484 objects misplaced (2.980%); 2.6 MiB/s, 41 objects/s recovering 2023-06-03T20:13:29.838 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:13:29.874 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:13:29.876 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:13:29.877 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:13:29.978 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:13:29.986 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:13:29 smithi089 bash[78452]: cluster 1685823208.7245626 mgr.y (mgr.34104) 1123 : cluster 0 pgmap v1096: 89 pgs: 9 active+clean+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 13 active+undersized+degraded, 1 active+recovering+undersized+remapped, 20 active+undersized, 44 active+clean; 5.4 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 752 op/s; 38922/266484 objects degraded (14.606%); 7941/266484 objects misplaced (2.980%); 2.6 MiB/s, 41 objects/s recovering 2023-06-03T20:13:29.986 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:13:29 smithi089 bash[75068]: cluster 1685823208.7245626 mgr.y (mgr.34104) 1123 : cluster 0 pgmap v1096: 89 pgs: 9 active+clean+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 13 active+undersized+degraded, 1 active+recovering+undersized+remapped, 20 active+undersized, 44 active+clean; 5.4 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 752 op/s; 38922/266484 objects degraded (14.606%); 7941/266484 objects misplaced (2.980%); 2.6 MiB/s, 41 objects/s recovering 2023-06-03T20:13:30.010 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:13:30.014 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:13:30.015 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:13:30.116 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:13:30.161 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:13:30.167 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:13:30.168 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:13:30.269 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:13:30.302 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:13:30.307 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:13:30.307 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:13:30.409 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:13:30.448 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:13:30.453 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:13:30.453 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:13:30.554 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:13:30.610 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:13:30.617 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:13:30.618 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:13:30.718 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:13:30.735 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/mon.a/config 2023-06-03T20:13:30.756 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:13:30.765 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:13:30.766 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:13:30.867 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:13:30.971 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:13:30.978 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:13:30.978 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:13:31.079 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:13:31.113 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:13:31.114 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:13:31.114 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:13:31.215 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:13:31.252 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:13:31.257 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:13:31.258 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:13:31.359 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:13:31.390 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:13:31.393 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:13:31.394 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:13:31.495 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:13:31.526 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:13:31.529 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:13:31.530 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:13:31.630 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:13:31.668 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:13:31.672 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:13:31.673 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:13:31.774 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:13:31.805 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:13:31.810 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:13:31.811 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:13:31.912 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:13:31.951 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:13:31.956 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:13:31.957 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:13:32.058 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:13:32.067 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:13:31 smithi164 bash[59462]: cluster 1685823210.725482 mgr.y (mgr.34104) 1124 : cluster 0 pgmap v1097: 89 pgs: 9 active+clean+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 13 active+undersized+degraded, 1 active+recovering+undersized+remapped, 20 active+undersized, 44 active+clean; 5.4 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 748 op/s; 38223/263730 objects degraded (14.493%); 7941/263730 objects misplaced (3.011%); 2.6 MiB/s, 41 objects/s recovering 2023-06-03T20:13:32.095 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:13:32.101 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:13:32.102 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:13:32.203 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:13:32.230 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:13:32.237 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:13:32.238 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:13:32.278 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:13:31 smithi089 bash[75068]: cluster 1685823210.725482 mgr.y (mgr.34104) 1124 : cluster 0 pgmap v1097: 89 pgs: 9 active+clean+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 13 active+undersized+degraded, 1 active+recovering+undersized+remapped, 20 active+undersized, 44 active+clean; 5.4 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 748 op/s; 38223/263730 objects degraded (14.493%); 7941/263730 objects misplaced (3.011%); 2.6 MiB/s, 41 objects/s recovering 2023-06-03T20:13:32.279 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:13:31 smithi089 bash[78452]: cluster 1685823210.725482 mgr.y (mgr.34104) 1124 : cluster 0 pgmap v1097: 89 pgs: 9 active+clean+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 13 active+undersized+degraded, 1 active+recovering+undersized+remapped, 20 active+undersized, 44 active+clean; 5.4 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 748 op/s; 38223/263730 objects degraded (14.493%); 7941/263730 objects misplaced (3.011%); 2.6 MiB/s, 41 objects/s recovering 2023-06-03T20:13:32.303 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph osd set nodeep-scrub 2023-06-03T20:13:32.339 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:13:32.403 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:13:32.405 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:13:32.406 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:13:32.506 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:13:32.554 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:13:32.559 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:13:32.560 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:13:32.661 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:13:32.705 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:13:32.708 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:13:32.708 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:13:32.809 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:13:32.858 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:13:32.864 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:13:32.865 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:13:32.966 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:13:32.976 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:13:32 smithi164 bash[59462]: cluster 1685823212.7467947 mon.a (mon.0) 1066 : cluster 3 Health check update: Degraded data redundancy: 38223/263730 objects degraded (14.493%), 15 pgs degraded, 33 pgs undersized (PG_DEGRADED) 2023-06-03T20:13:33.004 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:13:33.008 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:13:33.008 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:13:33.109 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:13:33.119 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:13:32 smithi089 bash[78452]: cluster 1685823212.7467947 mon.a (mon.0) 1066 : cluster 3 Health check update: Degraded data redundancy: 38223/263730 objects degraded (14.493%), 15 pgs degraded, 33 pgs undersized (PG_DEGRADED) 2023-06-03T20:13:33.119 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:13:32 smithi089 bash[75068]: cluster 1685823212.7467947 mon.a (mon.0) 1066 : cluster 3 Health check update: Degraded data redundancy: 38223/263730 objects degraded (14.493%), 15 pgs degraded, 33 pgs undersized (PG_DEGRADED) 2023-06-03T20:13:33.154 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:13:33.156 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:13:33.157 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:13:33.257 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:13:33.293 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:13:33.296 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:13:33.297 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:13:33.398 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:13:33.436 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:13:33.443 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:13:33.444 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:13:33.545 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:13:33.582 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:13:33.585 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:13:33.586 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:13:33.687 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:13:33.728 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:13:33.733 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:13:33.733 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:13:33.834 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:13:33.875 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:13:33.879 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:13:33.880 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:13:33.981 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:13:34.006 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:13:34.010 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:13:34.011 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:13:34.083 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:13:33 smithi164 bash[59462]: cluster 1685823212.7265184 mgr.y (mgr.34104) 1125 : cluster 0 pgmap v1098: 89 pgs: 9 active+clean+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 13 active+undersized+degraded, 1 active+recovering+undersized+remapped, 20 active+undersized, 44 active+clean; 5.3 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 749 op/s; 37974/262983 objects degraded (14.440%); 7941/262983 objects misplaced (3.020%); 2.6 MiB/s, 41 objects/s recovering 2023-06-03T20:13:34.112 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:13:34.145 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:13:34.148 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:13:34.149 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:13:34.250 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:13:34.280 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:13:34.282 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:13:34.283 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:13:34.383 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:13:34.419 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:13:34.425 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:13:34.426 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:13:34.527 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:13:34.528 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:13:34 smithi089 bash[75068]: cluster 1685823212.7265184 mgr.y (mgr.34104) 1125 : cluster 0 pgmap v1098: 89 pgs: 9 active+clean+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 13 active+undersized+degraded, 1 active+recovering+undersized+remapped, 20 active+undersized, 44 active+clean; 5.3 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 749 op/s; 37974/262983 objects degraded (14.440%); 7941/262983 objects misplaced (3.020%); 2.6 MiB/s, 41 objects/s recovering 2023-06-03T20:13:34.529 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:13:34 smithi089 bash[78452]: cluster 1685823212.7265184 mgr.y (mgr.34104) 1125 : cluster 0 pgmap v1098: 89 pgs: 9 active+clean+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 13 active+undersized+degraded, 1 active+recovering+undersized+remapped, 20 active+undersized, 44 active+clean; 5.3 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 749 op/s; 37974/262983 objects degraded (14.440%); 7941/262983 objects misplaced (3.020%); 2.6 MiB/s, 41 objects/s recovering 2023-06-03T20:13:34.568 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:13:34.573 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:13:34.574 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:13:34.676 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:13:34.729 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:13:34.735 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:13:34.735 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:13:34.837 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:13:34.881 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:13:34.883 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:13:34.884 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:13:34.985 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:13:35.019 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:13:35.021 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:13:35.022 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:13:35.100 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/mon.a/config 2023-06-03T20:13:35.101 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/mon.a/config 2023-06-03T20:13:35.123 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:13:35.174 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:13:35.181 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:13:35.182 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:13:35.283 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:13:35.315 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:13:35.318 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:13:35.319 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:13:35.420 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:13:35.462 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:13:35.463 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:13:35.464 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:13:35.565 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:13:35.597 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:13:35.602 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:13:35.604 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:13:35.705 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:13:35.750 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:13:35.755 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:13:35.756 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:13:35.833 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:13:35 smithi089 bash[75068]: cluster 1685823214.7275789 mgr.y (mgr.34104) 1126 : cluster 0 pgmap v1099: 89 pgs: 9 active+clean+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 13 active+undersized+degraded, 20 active+undersized, 44 active+clean; 5.2 GiB data, 17 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 755 op/s; 37414/254535 objects degraded (14.699%); 7550/254535 objects misplaced (2.966%); 2.7 MiB/s, 43 objects/s recovering 2023-06-03T20:13:35.857 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:13:35.888 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:13:35.893 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:13:35.894 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:13:35.995 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:13:36.026 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:13:36.028 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:13:36.029 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:13:36.083 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:13:35 smithi164 bash[59462]: cluster 1685823214.7275789 mgr.y (mgr.34104) 1126 : cluster 0 pgmap v1099: 89 pgs: 9 active+clean+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 13 active+undersized+degraded, 20 active+undersized, 44 active+clean; 5.2 GiB data, 17 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 755 op/s; 37414/254535 objects degraded (14.699%); 7550/254535 objects misplaced (2.966%); 2.7 MiB/s, 43 objects/s recovering 2023-06-03T20:13:36.130 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:13:36.138 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:13:35 smithi089 bash[78452]: cluster 1685823214.7275789 mgr.y (mgr.34104) 1126 : cluster 0 pgmap v1099: 89 pgs: 9 active+clean+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 13 active+undersized+degraded, 20 active+undersized, 44 active+clean; 5.2 GiB data, 17 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 755 op/s; 37414/254535 objects degraded (14.699%); 7550/254535 objects misplaced (2.966%); 2.7 MiB/s, 43 objects/s recovering 2023-06-03T20:13:36.174 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:13:36.178 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:13:36.179 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:13:36.280 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:13:36.311 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:13:36.313 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:13:36.314 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:13:36.415 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:13:36.423 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 20:13:36 smithi089 bash[71163]: ::ffff:172.21.15.164 - - [03/Jun/2023:20:13:36] "GET /metrics HTTP/1.1" 200 228936 "" "Prometheus/2.33.4" 2023-06-03T20:13:36.449 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:13:36.451 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:13:36.451 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:13:36.552 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:13:36.584 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:13:36.587 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:13:36.588 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:13:36.689 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:13:36.731 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:13:36.736 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:13:36.737 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:13:36.838 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:13:36.881 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:13:36.883 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:13:36.884 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:13:36.940 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/mon.a/config 2023-06-03T20:13:36.985 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:13:37.038 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:13:37.042 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:13:37.043 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:13:37.144 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:13:37.186 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:13:37.191 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:13:37.191 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:13:37.292 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:13:37.323 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:13:37.326 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:13:37.326 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:13:37.427 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:13:37.459 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:13:37.463 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:13:37.463 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:13:37.564 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:13:37.601 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:13:37.604 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:13:37.605 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:13:37.706 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:13:37.739 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:13:37.745 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:13:37.746 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:13:37.846 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:13:37.876 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:13:37.878 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:13:37.879 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:13:37.980 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:13:38.016 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:13:38.021 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:13:38.022 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:13:38.028 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:13:37 smithi089 bash[78452]: cluster 1685823216.728568 mgr.y (mgr.34104) 1127 : cluster 0 pgmap v1100: 89 pgs: 9 active+clean+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 13 active+undersized+degraded, 20 active+undersized, 44 active+clean; 5.1 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 756 op/s; 36707/251682 objects degraded (14.585%); 7550/251682 objects misplaced (3.000%); 2.7 MiB/s, 43 objects/s recovering 2023-06-03T20:13:38.028 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:13:37 smithi089 bash[78452]: audit 1685823217.1248748 mon.a (mon.0) 1067 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:13:38.028 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:13:37 smithi089 bash[78452]: audit 1685823217.1275187 mon.c (mon.2) 516 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T20:13:38.029 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:13:37 smithi089 bash[78452]: cluster 1685823217.747728 mon.a (mon.0) 1068 : cluster 3 Health check update: Degraded data redundancy: 36707/251682 objects degraded (14.585%), 16 pgs degraded, 36 pgs undersized (PG_DEGRADED) 2023-06-03T20:13:38.029 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:13:37 smithi089 bash[75068]: cluster 1685823216.728568 mgr.y (mgr.34104) 1127 : cluster 0 pgmap v1100: 89 pgs: 9 active+clean+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 13 active+undersized+degraded, 20 active+undersized, 44 active+clean; 5.1 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 756 op/s; 36707/251682 objects degraded (14.585%); 7550/251682 objects misplaced (3.000%); 2.7 MiB/s, 43 objects/s recovering 2023-06-03T20:13:38.029 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:13:37 smithi089 bash[75068]: audit 1685823217.1248748 mon.a (mon.0) 1067 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:13:38.029 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:13:37 smithi089 bash[75068]: audit 1685823217.1275187 mon.c (mon.2) 516 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T20:13:38.029 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:13:37 smithi089 bash[75068]: cluster 1685823217.747728 mon.a (mon.0) 1068 : cluster 3 Health check update: Degraded data redundancy: 36707/251682 objects degraded (14.585%), 16 pgs degraded, 36 pgs undersized (PG_DEGRADED) 2023-06-03T20:13:38.123 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:13:38.153 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:13:38.160 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:13:38.161 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:13:38.262 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:13:38.286 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:13:38.288 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:13:38.288 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:13:38.333 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:13:37 smithi164 bash[59462]: cluster 1685823216.728568 mgr.y (mgr.34104) 1127 : cluster 0 pgmap v1100: 89 pgs: 9 active+clean+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 13 active+undersized+degraded, 20 active+undersized, 44 active+clean; 5.1 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 756 op/s; 36707/251682 objects degraded (14.585%); 7550/251682 objects misplaced (3.000%); 2.7 MiB/s, 43 objects/s recovering 2023-06-03T20:13:38.334 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:13:37 smithi164 bash[59462]: audit 1685823217.1248748 mon.a (mon.0) 1067 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:13:38.334 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:13:37 smithi164 bash[59462]: audit 1685823217.1275187 mon.c (mon.2) 516 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T20:13:38.334 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:13:37 smithi164 bash[59462]: cluster 1685823217.747728 mon.a (mon.0) 1068 : cluster 3 Health check update: Degraded data redundancy: 36707/251682 objects degraded (14.585%), 16 pgs degraded, 36 pgs undersized (PG_DEGRADED) 2023-06-03T20:13:38.389 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:13:38.421 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:13:38.425 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:13:38.425 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:13:38.526 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:13:38.562 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:13:38.564 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:13:38.565 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:13:38.666 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:13:38.702 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:13:38.706 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:13:38.707 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:13:38.807 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:13:38.842 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:13:38.848 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:13:38.848 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:13:38.949 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:13:38.980 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:13:38.985 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:13:38.986 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:13:39.087 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:13:39.129 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:13:39.134 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:13:39.135 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:13:39.235 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:13:39.274 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:13:39.278 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:13:39.279 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:13:39.380 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:13:39.409 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:13:39.411 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:13:39.411 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:13:39.512 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:13:39.538 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:13:39.539 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:13:39.540 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:13:39.641 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:13:39.666 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:13:39.670 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:13:39.671 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:13:39.772 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:13:39.804 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:13:39.809 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:13:39.810 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:13:39.910 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:13:39.918 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:13:39 smithi089 bash[78452]: cluster 1685823218.729612 mgr.y (mgr.34104) 1128 : cluster 0 pgmap v1101: 89 pgs: 9 active+clean+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 13 active+undersized+degraded, 20 active+undersized, 44 active+clean; 5.0 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 907 op/s; 36496/244593 objects degraded (14.921%); 7103/244593 objects misplaced (2.904%); 3.7 MiB/s, 59 objects/s recovering 2023-06-03T20:13:39.919 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:13:39 smithi089 bash[75068]: cluster 1685823218.729612 mgr.y (mgr.34104) 1128 : cluster 0 pgmap v1101: 89 pgs: 9 active+clean+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 13 active+undersized+degraded, 20 active+undersized, 44 active+clean; 5.0 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 907 op/s; 36496/244593 objects degraded (14.921%); 7103/244593 objects misplaced (2.904%); 3.7 MiB/s, 59 objects/s recovering 2023-06-03T20:13:39.941 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:13:39.947 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:13:39.947 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:13:40.048 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:13:40.085 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:13:40.094 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:13:40.095 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:13:40.196 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:13:40.224 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:13:40.231 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:13:40.232 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:13:40.333 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:13:40.334 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:13:39 smithi164 bash[59462]: cluster 1685823218.729612 mgr.y (mgr.34104) 1128 : cluster 0 pgmap v1101: 89 pgs: 9 active+clean+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 13 active+undersized+degraded, 20 active+undersized, 44 active+clean; 5.0 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 907 op/s; 36496/244593 objects degraded (14.921%); 7103/244593 objects misplaced (2.904%); 3.7 MiB/s, 59 objects/s recovering 2023-06-03T20:13:40.379 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:13:40.382 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:13:40.383 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:13:40.484 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:13:40.518 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:13:40.520 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:13:40.520 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:13:40.621 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:13:40.655 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:13:40.661 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:13:40.661 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:13:40.762 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:13:40.783 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:13:40.785 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:13:40.786 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:13:40.886 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:13:40.931 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:13:40.936 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:13:40.936 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:13:41.037 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:13:41.061 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:13:41.062 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:13:41.063 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:13:41.164 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:13:41.204 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:13:41.211 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:13:41.212 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:13:41.313 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:13:41.353 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:13:41.357 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:13:41.358 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:13:41.459 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:13:41.495 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:13:41.498 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:13:41.499 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:13:41.600 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:13:41.632 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:13:41.637 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:13:41.638 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:13:41.739 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:13:41.775 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:13:41.782 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:13:41.783 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:13:41.884 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:13:41.917 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:13:41.922 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:13:41.923 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:13:42.023 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:13:42.028 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:13:41 smithi089 bash[78452]: cluster 1685823220.7310102 mgr.y (mgr.34104) 1129 : cluster 0 pgmap v1102: 89 pgs: 9 active+clean+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 13 active+undersized+degraded, 20 active+undersized, 44 active+clean; 4.9 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 684 op/s; 35819/241833 objects degraded (14.811%); 7103/241833 objects misplaced (2.937%); 2.3 MiB/s, 36 objects/s recovering 2023-06-03T20:13:42.028 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:13:41 smithi089 bash[75068]: cluster 1685823220.7310102 mgr.y (mgr.34104) 1129 : cluster 0 pgmap v1102: 89 pgs: 9 active+clean+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 13 active+undersized+degraded, 20 active+undersized, 44 active+clean; 4.9 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 684 op/s; 35819/241833 objects degraded (14.811%); 7103/241833 objects misplaced (2.937%); 2.3 MiB/s, 36 objects/s recovering 2023-06-03T20:13:42.033 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:13:41 smithi164 bash[59462]: cluster 1685823220.7310102 mgr.y (mgr.34104) 1129 : cluster 0 pgmap v1102: 89 pgs: 9 active+clean+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 13 active+undersized+degraded, 20 active+undersized, 44 active+clean; 4.9 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 684 op/s; 35819/241833 objects degraded (14.811%); 7103/241833 objects misplaced (2.937%); 2.3 MiB/s, 36 objects/s recovering 2023-06-03T20:13:42.060 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:13:42.066 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:13:42.066 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:13:42.167 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:13:42.200 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:13:42.205 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:13:42.206 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:13:42.306 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:13:42.342 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:13:42.342 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:13:42.343 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:13:42.444 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:13:42.487 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:13:42.491 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:13:42.492 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:13:42.593 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:13:42.622 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:13:42.625 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:13:42.626 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:13:42.727 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:13:42.759 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:13:42.765 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:13:42.765 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:13:42.866 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:13:42.895 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:13:42.899 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:13:42.900 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:13:43.001 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:13:43.010 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:13:42 smithi164 bash[59462]: cluster 1685823222.7491865 mon.a (mon.0) 1069 : cluster 3 Health check update: Degraded data redundancy: 35819/241833 objects degraded (14.811%), 16 pgs degraded, 36 pgs undersized (PG_DEGRADED) 2023-06-03T20:13:43.028 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:13:42 smithi089 bash[75068]: cluster 1685823222.7491865 mon.a (mon.0) 1069 : cluster 3 Health check update: Degraded data redundancy: 35819/241833 objects degraded (14.811%), 16 pgs degraded, 36 pgs undersized (PG_DEGRADED) 2023-06-03T20:13:43.029 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:13:42 smithi089 bash[78452]: cluster 1685823222.7491865 mon.a (mon.0) 1069 : cluster 3 Health check update: Degraded data redundancy: 35819/241833 objects degraded (14.811%), 16 pgs degraded, 36 pgs undersized (PG_DEGRADED) 2023-06-03T20:13:43.041 INFO:teuthology.orchestra.run.smithi089.stdout:ERROR: (22) Invalid argument 2023-06-03T20:13:43.041 INFO:teuthology.orchestra.run.smithi089.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-06-03T20:13:43.046 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:13:43.052 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:13:43.053 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:13:43.154 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:13:43.189 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:13:43.195 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:13:43.195 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:13:43.296 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:13:43.324 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:13:43.330 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:13:43.331 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:13:43.432 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:13:43.468 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:13:43.475 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:13:43.476 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:13:43.578 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:13:43.616 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:13:43.620 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:13:43.621 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:13:43.722 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:13:43.752 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:13:43.753 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:13:43.754 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:13:43.855 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:13:43.857 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:13:43 smithi089 bash[75068]: cluster 1685823222.7319314 mgr.y (mgr.34104) 1130 : cluster 0 pgmap v1103: 89 pgs: 9 active+clean+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 13 active+undersized+degraded, 20 active+undersized, 44 active+clean; 4.9 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 625 op/s; 35602/241182 objects degraded (14.761%); 7103/241182 objects misplaced (2.945%); 2.3 MiB/s, 36 objects/s recovering 2023-06-03T20:13:43.897 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:13:43.900 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:13:43.900 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:13:44.001 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:13:44.015 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:13:43 smithi164 bash[59462]: cluster 1685823222.7319314 mgr.y (mgr.34104) 1130 : cluster 0 pgmap v1103: 89 pgs: 9 active+clean+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 13 active+undersized+degraded, 20 active+undersized, 44 active+clean; 4.9 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 625 op/s; 35602/241182 objects degraded (14.761%); 7103/241182 objects misplaced (2.945%); 2.3 MiB/s, 36 objects/s recovering 2023-06-03T20:13:44.028 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:13:43 smithi089 bash[78452]: cluster 1685823222.7319314 mgr.y (mgr.34104) 1130 : cluster 0 pgmap v1103: 89 pgs: 9 active+clean+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 13 active+undersized+degraded, 20 active+undersized, 44 active+clean; 4.9 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 625 op/s; 35602/241182 objects degraded (14.761%); 7103/241182 objects misplaced (2.945%); 2.3 MiB/s, 36 objects/s recovering 2023-06-03T20:13:44.051 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:13:44.053 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:13:44.054 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:13:44.155 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:13:44.186 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:13:44.192 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:13:44.193 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:13:44.294 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:13:44.325 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:13:44.330 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:13:44.331 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:13:44.432 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:13:44.460 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:13:44.463 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:13:44.464 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:13:44.565 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:13:44.600 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:13:44.604 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:13:44.605 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:13:44.706 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:13:44.732 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:13:44.734 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:13:44.735 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:13:44.835 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:13:44.865 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:13:44.869 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:13:44.870 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:13:44.971 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:13:45.006 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:13:45.011 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:13:45.012 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:13:45.113 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:13:45.144 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:13:45.150 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:13:45.150 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:13:45.251 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:13:45.288 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:13:45.292 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:13:45.293 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:13:45.393 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:13:45.429 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:13:45.431 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:13:45.432 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:13:45.533 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:13:45.575 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:13:45.578 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:13:45.579 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:13:45.679 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:13:45.724 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:13:45.726 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:13:45.727 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:13:45.828 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:13:45.860 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:13:45.866 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:13:45.866 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:13:45.967 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:13:45.992 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:13:45.996 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:13:45.997 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:13:46.098 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:13:46.110 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:13:45 smithi164 bash[59462]: cluster 1685823224.7329094 mgr.y (mgr.34104) 1131 : cluster 0 pgmap v1104: 89 pgs: 9 active+clean+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 13 active+undersized+degraded, 20 active+undersized, 44 active+clean; 4.7 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 838 op/s; 35194/232794 objects degraded (15.118%); 6612/232794 objects misplaced (2.840%); 3.4 MiB/s, 54 objects/s recovering 2023-06-03T20:13:46.129 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:13:46.132 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:13:46.133 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:13:46.234 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:13:46.235 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:13:45 smithi089 bash[75068]: cluster 1685823224.7329094 mgr.y (mgr.34104) 1131 : cluster 0 pgmap v1104: 89 pgs: 9 active+clean+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 13 active+undersized+degraded, 20 active+undersized, 44 active+clean; 4.7 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 838 op/s; 35194/232794 objects degraded (15.118%); 6612/232794 objects misplaced (2.840%); 3.4 MiB/s, 54 objects/s recovering 2023-06-03T20:13:46.235 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:13:45 smithi089 bash[78452]: cluster 1685823224.7329094 mgr.y (mgr.34104) 1131 : cluster 0 pgmap v1104: 89 pgs: 9 active+clean+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 13 active+undersized+degraded, 20 active+undersized, 44 active+clean; 4.7 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 838 op/s; 35194/232794 objects degraded (15.118%); 6612/232794 objects misplaced (2.840%); 3.4 MiB/s, 54 objects/s recovering 2023-06-03T20:13:46.264 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:13:46.269 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:13:46.270 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:13:46.371 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:13:46.402 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:13:46.408 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:13:46.409 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:13:46.509 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:13:46.518 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 20:13:46 smithi089 bash[71163]: ::ffff:172.21.15.164 - - [03/Jun/2023:20:13:46] "GET /metrics HTTP/1.1" 200 228922 "" "Prometheus/2.33.4" 2023-06-03T20:13:46.544 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:13:46.550 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:13:46.551 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:13:46.652 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:13:46.683 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:13:46.688 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:13:46.689 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:13:46.790 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:13:46.830 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:13:46.833 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:13:46.833 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:13:46.934 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:13:46.974 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:13:46.978 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:13:46.979 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:13:47.080 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:13:47.108 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:13:47.110 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:13:47.110 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:13:47.211 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:13:47.244 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:13:47.246 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:13:47.247 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:13:47.347 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:13:47.391 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:13:47.394 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:13:47.395 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:13:47.496 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:13:47.526 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:13:47.531 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:13:47.532 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:13:47.633 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:13:47.636 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph daemon osd.0 dump_historic_ops 2023-06-03T20:13:47.664 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:13:47.670 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:13:47.670 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:13:47.771 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:13:47.807 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:13:47.809 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:13:47.810 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:13:47.911 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:13:47.946 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:13:47.949 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:13:47.950 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:13:48.050 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:13:48.082 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:13:48.090 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:13:48.091 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:13:48.192 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:13:48.221 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:13:48.226 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:13:48.227 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:13:48.328 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:13:48.333 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:13:47 smithi164 bash[59462]: cluster 1685823226.7337875 mgr.y (mgr.34104) 1132 : cluster 0 pgmap v1105: 89 pgs: 9 active+clean+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 13 active+undersized+degraded, 20 active+undersized, 44 active+clean; 4.7 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 680 op/s; 34516/230019 objects degraded (15.006%); 6612/230019 objects misplaced (2.875%); 2.2 MiB/s, 35 objects/s recovering 2023-06-03T20:13:48.333 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:13:47 smithi164 bash[59462]: cluster 1685823227.7502646 mon.a (mon.0) 1070 : cluster 3 Health check update: Degraded data redundancy: 34516/230019 objects degraded (15.006%), 16 pgs degraded, 36 pgs undersized (PG_DEGRADED) 2023-06-03T20:13:48.372 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:13:48.377 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:13:48.378 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:13:48.479 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:13:48.520 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:13:48.528 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:13:48.529 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:13:48.530 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:13:48 smithi089 bash[78452]: cluster 1685823226.7337875 mgr.y (mgr.34104) 1132 : cluster 0 pgmap v1105: 89 pgs: 9 active+clean+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 13 active+undersized+degraded, 20 active+undersized, 44 active+clean; 4.7 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 680 op/s; 34516/230019 objects degraded (15.006%); 6612/230019 objects misplaced (2.875%); 2.2 MiB/s, 35 objects/s recovering 2023-06-03T20:13:48.531 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:13:48 smithi089 bash[78452]: cluster 1685823227.7502646 mon.a (mon.0) 1070 : cluster 3 Health check update: Degraded data redundancy: 34516/230019 objects degraded (15.006%), 16 pgs degraded, 36 pgs undersized (PG_DEGRADED) 2023-06-03T20:13:48.531 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:13:48 smithi089 bash[75068]: cluster 1685823226.7337875 mgr.y (mgr.34104) 1132 : cluster 0 pgmap v1105: 89 pgs: 9 active+clean+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 13 active+undersized+degraded, 20 active+undersized, 44 active+clean; 4.7 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 680 op/s; 34516/230019 objects degraded (15.006%); 6612/230019 objects misplaced (2.875%); 2.2 MiB/s, 35 objects/s recovering 2023-06-03T20:13:48.531 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:13:48 smithi089 bash[75068]: cluster 1685823227.7502646 mon.a (mon.0) 1070 : cluster 3 Health check update: Degraded data redundancy: 34516/230019 objects degraded (15.006%), 16 pgs degraded, 36 pgs undersized (PG_DEGRADED) 2023-06-03T20:13:48.630 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:13:48.664 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:13:48.669 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:13:48.670 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:13:48.770 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:13:48.807 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:13:48.811 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:13:48.812 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:13:48.913 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:13:48.952 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:13:48.962 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:13:48.963 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:13:49.064 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:13:49.092 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:13:49.094 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:13:49.094 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:13:49.195 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:13:49.236 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:13:49.242 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:13:49.243 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:13:49.344 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:13:49.370 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:13:49.376 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:13:49.377 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:13:49.478 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:13:49.507 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:13:49.511 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:13:49.512 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:13:49.613 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:13:49.649 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:13:49.653 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:13:49.654 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:13:49.754 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:13:49.791 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:13:49.796 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:13:49.796 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:13:49.897 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:13:49.910 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:13:49 smithi089 bash[75068]: cluster 1685823228.7348855 mgr.y (mgr.34104) 1133 : cluster 0 pgmap v1106: 89 pgs: 9 active+clean+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 13 active+undersized+degraded, 20 active+undersized, 44 active+clean; 4.5 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 807 op/s; 34301/222606 objects degraded (15.409%); 6161/222606 objects misplaced (2.768%); 3.4 MiB/s, 54 objects/s recovering 2023-06-03T20:13:49.911 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:13:49 smithi089 bash[78452]: cluster 1685823228.7348855 mgr.y (mgr.34104) 1133 : cluster 0 pgmap v1106: 89 pgs: 9 active+clean+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 13 active+undersized+degraded, 20 active+undersized, 44 active+clean; 4.5 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 807 op/s; 34301/222606 objects degraded (15.409%); 6161/222606 objects misplaced (2.768%); 3.4 MiB/s, 54 objects/s recovering 2023-06-03T20:13:49.924 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:13:49.929 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:13:49.929 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:13:50.030 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:13:50.067 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:13:50.073 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:13:50.073 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:13:50.083 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:13:49 smithi164 bash[59462]: cluster 1685823228.7348855 mgr.y (mgr.34104) 1133 : cluster 0 pgmap v1106: 89 pgs: 9 active+clean+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 13 active+undersized+degraded, 20 active+undersized, 44 active+clean; 4.5 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 807 op/s; 34301/222606 objects degraded (15.409%); 6161/222606 objects misplaced (2.768%); 3.4 MiB/s, 54 objects/s recovering 2023-06-03T20:13:50.174 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:13:50.207 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:13:50.213 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:13:50.214 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:13:50.315 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:13:50.350 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:13:50.353 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:13:50.354 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:13:50.454 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:13:50.496 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:13:50.503 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:13:50.504 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:13:50.605 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:13:50.640 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:13:50.648 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:13:50.649 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:13:50.750 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:13:50.783 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:13:50.786 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:13:50.787 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:13:50.888 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:13:50.933 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:13:50.941 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:13:50.942 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:13:51.042 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:13:51.071 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:13:51.073 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:13:51.074 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:13:51.175 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:13:51.203 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:13:51.207 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:13:51.208 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:13:51.309 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:13:51.351 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:13:51.354 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:13:51.355 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:13:51.456 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:13:51.483 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:13:51.489 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:13:51.489 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:13:51.590 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:13:51.628 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:13:51.631 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:13:51.632 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:13:51.732 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:13:51.763 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:13:51.765 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:13:51.766 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:13:51.867 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:13:51.872 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:13:51 smithi089 bash[78452]: cluster 1685823230.7357965 mgr.y (mgr.34104) 1134 : cluster 0 pgmap v1107: 89 pgs: 9 active+clean+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 13 active+undersized+degraded, 20 active+undersized, 44 active+clean; 4.5 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 686 op/s; 33618/219864 objects degraded (15.290%); 6161/219864 objects misplaced (2.802%); 2.3 MiB/s, 36 objects/s recovering 2023-06-03T20:13:51.873 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:13:51 smithi089 bash[75068]: cluster 1685823230.7357965 mgr.y (mgr.34104) 1134 : cluster 0 pgmap v1107: 89 pgs: 9 active+clean+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 13 active+undersized+degraded, 20 active+undersized, 44 active+clean; 4.5 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 686 op/s; 33618/219864 objects degraded (15.290%); 6161/219864 objects misplaced (2.802%); 2.3 MiB/s, 36 objects/s recovering 2023-06-03T20:13:51.904 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:13:51.906 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:13:51.907 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:13:52.008 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:13:52.045 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:13:52.049 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:13:52.050 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:13:52.151 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:13:52.162 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:13:51 smithi164 bash[59462]: cluster 1685823230.7357965 mgr.y (mgr.34104) 1134 : cluster 0 pgmap v1107: 89 pgs: 9 active+clean+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 13 active+undersized+degraded, 20 active+undersized, 44 active+clean; 4.5 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 686 op/s; 33618/219864 objects degraded (15.290%); 6161/219864 objects misplaced (2.802%); 2.3 MiB/s, 36 objects/s recovering 2023-06-03T20:13:52.184 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:13:52.186 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:13:52.187 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:13:52.288 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:13:52.319 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:13:52.323 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:13:52.324 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:13:52.425 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:13:52.458 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:13:52.465 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:13:52.466 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:13:52.538 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/mon.a/config 2023-06-03T20:13:52.567 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:13:52.600 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:13:52.606 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:13:52.607 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:13:52.708 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:13:52.750 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:13:52.755 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:13:52.756 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:13:52.857 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:13:52.890 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:13:52.891 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:13:52.892 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:13:52.993 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:13:53.024 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:13:53.027 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:13:53.028 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:13:53.129 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:13:53.138 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:13:52 smithi164 bash[59462]: audit 1685823232.122067 mon.c (mon.2) 517 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T20:13:53.138 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:13:52 smithi164 bash[59462]: cluster 1685823232.7513576 mon.a (mon.0) 1071 : cluster 3 Health check update: Degraded data redundancy: 33618/219864 objects degraded (15.290%), 16 pgs degraded, 36 pgs undersized (PG_DEGRADED) 2023-06-03T20:13:53.158 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:13:53.161 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:13:53.162 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:13:53.263 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:13:53.270 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:13:52 smithi089 bash[78452]: audit 1685823232.122067 mon.c (mon.2) 517 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T20:13:53.270 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:13:52 smithi089 bash[78452]: cluster 1685823232.7513576 mon.a (mon.0) 1071 : cluster 3 Health check update: Degraded data redundancy: 33618/219864 objects degraded (15.290%), 16 pgs degraded, 36 pgs undersized (PG_DEGRADED) 2023-06-03T20:13:53.271 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:13:52 smithi089 bash[75068]: audit 1685823232.122067 mon.c (mon.2) 517 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T20:13:53.271 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:13:52 smithi089 bash[75068]: cluster 1685823232.7513576 mon.a (mon.0) 1071 : cluster 3 Health check update: Degraded data redundancy: 33618/219864 objects degraded (15.290%), 16 pgs degraded, 36 pgs undersized (PG_DEGRADED) 2023-06-03T20:13:53.291 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:13:53.297 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:13:53.297 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:13:53.398 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:13:53.429 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:13:53.432 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:13:53.433 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:13:53.534 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:13:53.562 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:13:53.565 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:13:53.566 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:13:53.667 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:13:53.697 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:13:53.701 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:13:53.702 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:13:53.802 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:13:53.847 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:13:53.849 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:13:53.850 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:13:53.951 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:13:53.990 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:13:53.993 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:13:53.994 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:13:54.095 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:13:54.131 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:13:54.134 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:13:54.135 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:13:54.135 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:13:53 smithi164 bash[59462]: cluster 1685823232.7366278 mgr.y (mgr.34104) 1135 : cluster 0 pgmap v1108: 89 pgs: 9 active+clean+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 13 active+undersized+degraded, 20 active+undersized, 44 active+clean; 4.5 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 628 op/s; 33391/219183 objects degraded (15.234%); 6161/219183 objects misplaced (2.811%); 2.3 MiB/s, 36 objects/s recovering 2023-06-03T20:13:54.136 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:13:53 smithi164 bash[59462]: audit 1685823233.7102873 mon.c (mon.2) 518 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-03T20:13:54.236 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:13:54.246 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:13:53 smithi089 bash[78452]: cluster 1685823232.7366278 mgr.y (mgr.34104) 1135 : cluster 0 pgmap v1108: 89 pgs: 9 active+clean+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 13 active+undersized+degraded, 20 active+undersized, 44 active+clean; 4.5 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 628 op/s; 33391/219183 objects degraded (15.234%); 6161/219183 objects misplaced (2.811%); 2.3 MiB/s, 36 objects/s recovering 2023-06-03T20:13:54.246 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:13:53 smithi089 bash[78452]: audit 1685823233.7102873 mon.c (mon.2) 518 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-03T20:13:54.247 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:13:53 smithi089 bash[75068]: cluster 1685823232.7366278 mgr.y (mgr.34104) 1135 : cluster 0 pgmap v1108: 89 pgs: 9 active+clean+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 13 active+undersized+degraded, 20 active+undersized, 44 active+clean; 4.5 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 628 op/s; 33391/219183 objects degraded (15.234%); 6161/219183 objects misplaced (2.811%); 2.3 MiB/s, 36 objects/s recovering 2023-06-03T20:13:54.247 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:13:53 smithi089 bash[75068]: audit 1685823233.7102873 mon.c (mon.2) 518 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-03T20:13:54.268 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:13:54.273 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:13:54.274 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:13:54.375 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:13:54.412 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:13:54.416 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:13:54.417 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:13:54.518 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:13:54.552 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:13:54.558 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:13:54.559 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:13:54.660 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:13:54.705 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:13:54.709 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:13:54.709 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:13:54.810 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:13:54.844 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:13:54.852 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:13:54.853 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:13:54.954 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:13:54.993 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:13:54.998 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:13:54.999 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:13:55.100 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:13:55.133 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:13:55.138 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:13:55.138 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:13:55.239 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:13:55.268 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:13:55.272 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:13:55.272 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:13:55.373 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:13:55.428 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:13:55.436 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:13:55.437 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:13:55.537 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:13:55.572 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:13:55.574 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:13:55.575 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:13:55.676 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:13:55.706 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:13:55.712 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:13:55.713 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:13:55.814 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:13:55.837 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:13:55.841 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:13:55.842 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:13:55.943 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:13:55.979 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:13:55.984 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:13:55.985 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:13:56.086 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:13:56.133 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:13:56.136 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:13:56.137 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:13:56.186 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:13:55 smithi089 bash[75068]: cluster 1685823234.7376876 mgr.y (mgr.34104) 1136 : cluster 0 pgmap v1109: 89 pgs: 9 active+clean+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 13 active+undersized+degraded, 20 active+undersized, 44 active+clean; 4.3 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 848 op/s; 32873/210639 objects degraded (15.606%); 5715/210639 objects misplaced (2.713%); 3.5 MiB/s, 56 objects/s recovering 2023-06-03T20:13:56.187 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:13:55 smithi089 bash[78452]: cluster 1685823234.7376876 mgr.y (mgr.34104) 1136 : cluster 0 pgmap v1109: 89 pgs: 9 active+clean+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 13 active+undersized+degraded, 20 active+undersized, 44 active+clean; 4.3 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 848 op/s; 32873/210639 objects degraded (15.606%); 5715/210639 objects misplaced (2.713%); 3.5 MiB/s, 56 objects/s recovering 2023-06-03T20:13:56.238 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:13:56.333 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:13:55 smithi164 bash[59462]: cluster 1685823234.7376876 mgr.y (mgr.34104) 1136 : cluster 0 pgmap v1109: 89 pgs: 9 active+clean+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 13 active+undersized+degraded, 20 active+undersized, 44 active+clean; 4.3 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 848 op/s; 32873/210639 objects degraded (15.606%); 5715/210639 objects misplaced (2.713%); 3.5 MiB/s, 56 objects/s recovering 2023-06-03T20:13:56.369 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:13:56.376 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:13:56.376 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:13:56.477 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:13:56.486 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 20:13:56 smithi089 bash[71163]: ::ffff:172.21.15.164 - - [03/Jun/2023:20:13:56] "GET /metrics HTTP/1.1" 200 228916 "" "Prometheus/2.33.4" 2023-06-03T20:13:56.529 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:13:56.533 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:13:56.533 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:13:56.634 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:13:56.667 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:13:56.676 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:13:56.677 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:13:56.778 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:13:56.819 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:13:56.827 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:13:56.827 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:13:56.928 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:13:56.972 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:13:56.979 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:13:56.980 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:13:56.985 INFO:teuthology.orchestra.run.smithi089.stdout:osd.0: {} 2023-06-03T20:13:57.024 INFO:teuthology.orchestra.run.smithi089.stdout:osd.1: {} 2023-06-03T20:13:57.081 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:13:57.094 INFO:teuthology.orchestra.run.smithi089.stdout:osd.2: {} 2023-06-03T20:13:57.120 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:13:57.128 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:13:57.129 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:13:57.147 INFO:teuthology.orchestra.run.smithi089.stdout:osd.3: {} 2023-06-03T20:13:57.184 INFO:teuthology.orchestra.run.smithi089.stdout:osd.5: {} 2023-06-03T20:13:57.225 INFO:teuthology.orchestra.run.smithi089.stdout:osd.6: {} 2023-06-03T20:13:57.231 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:13:57.263 INFO:teuthology.orchestra.run.smithi089.stdout:osd.7: {} 2023-06-03T20:13:57.264 INFO:teuthology.orchestra.run.smithi089.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-06-03T20:13:57.264 INFO:teuthology.orchestra.run.smithi089.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-06-03T20:13:57.264 INFO:teuthology.orchestra.run.smithi089.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-06-03T20:13:57.264 INFO:teuthology.orchestra.run.smithi089.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-06-03T20:13:57.264 INFO:teuthology.orchestra.run.smithi089.stderr:Error ENXIO: problem getting command descriptions from osd.4 2023-06-03T20:13:57.264 INFO:teuthology.orchestra.run.smithi089.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-06-03T20:13:57.265 INFO:teuthology.orchestra.run.smithi089.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-06-03T20:13:57.265 INFO:teuthology.orchestra.run.smithi089.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-06-03T20:13:57.276 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:13:57.278 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:13:57.279 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:13:57.380 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:13:57.419 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:13:57.421 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:13:57.421 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:13:57.522 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:13:57.566 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:13:57.571 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:13:57.572 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:13:57.673 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:13:57.694 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:13:57.695 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:13:57.696 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:13:57.797 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:13:57.831 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:13:57.835 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:13:57.835 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:13:57.846 INFO:teuthology.orchestra.run.smithi089.stderr:Error ENOENT: unrecognized pool 'device_health_metrics' 2023-06-03T20:13:57.896 INFO:teuthology.orchestra.run.smithi089.stderr:nodeep-scrub is set 2023-06-03T20:13:57.936 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:13:57.969 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:13:57.975 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:13:57.976 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:13:58.077 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:13:58.083 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:13:57 smithi164 bash[59462]: cluster 1685823236.7385628 mgr.y (mgr.34104) 1137 : cluster 0 pgmap v1110: 89 pgs: 9 active+clean+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 13 active+undersized+degraded, 20 active+undersized, 44 active+clean; 4.2 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 694 op/s; 32181/207792 objects degraded (15.487%); 5715/207792 objects misplaced (2.750%); 2.4 MiB/s, 38 objects/s recovering 2023-06-03T20:13:58.083 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:13:57 smithi164 bash[59462]: audit 1685823236.9297447 mon.a (mon.0) 1072 : audit 0 from='client.? 172.21.15.89:0/1128439264' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-03T20:13:58.083 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:13:57 smithi164 bash[59462]: audit 1685823237.6857538 mon.c (mon.2) 519 : audit 1 from='client.? 172.21.15.89:0/2953928083' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-06-03T20:13:58.084 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:13:57 smithi164 bash[59462]: audit 1685823237.6862104 mon.a (mon.0) 1073 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-06-03T20:13:58.084 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:13:57 smithi164 bash[59462]: cluster 1685823237.752669 mon.a (mon.0) 1074 : cluster 3 Health check update: Degraded data redundancy: 32181/207792 objects degraded (15.487%), 16 pgs degraded, 36 pgs undersized (PG_DEGRADED) 2023-06-03T20:13:58.084 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:13:57 smithi164 bash[59462]: audit 1685823237.8358676 mon.c (mon.2) 520 : audit 1 from='client.? 172.21.15.89:0/2570741176' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "device_health_metrics", "var": "pgp_num", "val": "1"}]: dispatch 2023-06-03T20:13:58.084 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:13:57 smithi164 bash[59462]: audit 1685823237.83639 mon.a (mon.0) 1075 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "device_health_metrics", "var": "pgp_num", "val": "1"}]: dispatch 2023-06-03T20:13:58.110 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:13:58.114 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:13:58.114 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:13:58.215 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:13:58.224 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:13:57 smithi089 bash[75068]: cluster 1685823236.7385628 mgr.y (mgr.34104) 1137 : cluster 0 pgmap v1110: 89 pgs: 9 active+clean+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 13 active+undersized+degraded, 20 active+undersized, 44 active+clean; 4.2 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 694 op/s; 32181/207792 objects degraded (15.487%); 5715/207792 objects misplaced (2.750%); 2.4 MiB/s, 38 objects/s recovering 2023-06-03T20:13:58.224 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:13:57 smithi089 bash[75068]: audit 1685823236.9297447 mon.a (mon.0) 1072 : audit 0 from='client.? 172.21.15.89:0/1128439264' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-03T20:13:58.224 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:13:57 smithi089 bash[75068]: audit 1685823237.6857538 mon.c (mon.2) 519 : audit 1 from='client.? 172.21.15.89:0/2953928083' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-06-03T20:13:58.225 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:13:57 smithi089 bash[75068]: audit 1685823237.6862104 mon.a (mon.0) 1073 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-06-03T20:13:58.225 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:13:57 smithi089 bash[75068]: cluster 1685823237.752669 mon.a (mon.0) 1074 : cluster 3 Health check update: Degraded data redundancy: 32181/207792 objects degraded (15.487%), 16 pgs degraded, 36 pgs undersized (PG_DEGRADED) 2023-06-03T20:13:58.225 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:13:57 smithi089 bash[75068]: audit 1685823237.8358676 mon.c (mon.2) 520 : audit 1 from='client.? 172.21.15.89:0/2570741176' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "device_health_metrics", "var": "pgp_num", "val": "1"}]: dispatch 2023-06-03T20:13:58.225 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:13:57 smithi089 bash[75068]: audit 1685823237.83639 mon.a (mon.0) 1075 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "device_health_metrics", "var": "pgp_num", "val": "1"}]: dispatch 2023-06-03T20:13:58.225 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:13:57 smithi089 bash[78452]: cluster 1685823236.7385628 mgr.y (mgr.34104) 1137 : cluster 0 pgmap v1110: 89 pgs: 9 active+clean+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 13 active+undersized+degraded, 20 active+undersized, 44 active+clean; 4.2 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 694 op/s; 32181/207792 objects degraded (15.487%); 5715/207792 objects misplaced (2.750%); 2.4 MiB/s, 38 objects/s recovering 2023-06-03T20:13:58.226 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:13:57 smithi089 bash[78452]: audit 1685823236.9297447 mon.a (mon.0) 1072 : audit 0 from='client.? 172.21.15.89:0/1128439264' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-03T20:13:58.226 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:13:57 smithi089 bash[78452]: audit 1685823237.6857538 mon.c (mon.2) 519 : audit 1 from='client.? 172.21.15.89:0/2953928083' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-06-03T20:13:58.226 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:13:57 smithi089 bash[78452]: audit 1685823237.6862104 mon.a (mon.0) 1073 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-06-03T20:13:58.227 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:13:57 smithi089 bash[78452]: cluster 1685823237.752669 mon.a (mon.0) 1074 : cluster 3 Health check update: Degraded data redundancy: 32181/207792 objects degraded (15.487%), 16 pgs degraded, 36 pgs undersized (PG_DEGRADED) 2023-06-03T20:13:58.227 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:13:57 smithi089 bash[78452]: audit 1685823237.8358676 mon.c (mon.2) 520 : audit 1 from='client.? 172.21.15.89:0/2570741176' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "device_health_metrics", "var": "pgp_num", "val": "1"}]: dispatch 2023-06-03T20:13:58.227 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:13:57 smithi089 bash[78452]: audit 1685823237.83639 mon.a (mon.0) 1075 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "device_health_metrics", "var": "pgp_num", "val": "1"}]: dispatch 2023-06-03T20:13:58.249 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:13:58.251 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:13:58.252 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:13:58.353 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:13:58.389 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:13:58.395 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:13:58.396 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:13:58.496 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:13:58.524 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:13:58.525 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:13:58.526 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:13:58.627 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:13:58.669 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:13:58.674 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:13:58.675 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:13:58.776 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:13:58.802 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:13:58.804 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:13:58.805 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:13:58.906 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:13:58.937 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:13:58.944 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:13:58.944 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:13:59.045 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:13:59.074 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:13:59.076 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:13:59.077 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:13:59.078 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:13:58 smithi089 bash[78452]: cluster 1685823237.8836267 mon.a (mon.0) 1076 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-06-03T20:13:59.078 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:13:58 smithi089 bash[78452]: cluster 1685823237.885095 osd.6 (osd.6) 148 : cluster 3 failed to encode map e203 with expected crc 2023-06-03T20:13:59.078 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:13:58 smithi089 bash[78452]: cluster 1685823237.8853543 osd.5 (osd.5) 135 : cluster 3 failed to encode map e203 with expected crc 2023-06-03T20:13:59.078 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:13:58 smithi089 bash[78452]: audit 1685823237.888655 mon.a (mon.0) 1077 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-06-03T20:13:59.078 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:13:58 smithi089 bash[78452]: cluster 1685823237.8888206 mon.a (mon.0) 1078 : cluster 0 osdmap e203: 8 total, 7 up, 7 in 2023-06-03T20:13:59.078 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:13:58 smithi089 bash[75068]: cluster 1685823237.8836267 mon.a (mon.0) 1076 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-06-03T20:13:59.079 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:13:58 smithi089 bash[75068]: cluster 1685823237.885095 osd.6 (osd.6) 148 : cluster 3 failed to encode map e203 with expected crc 2023-06-03T20:13:59.079 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:13:58 smithi089 bash[75068]: cluster 1685823237.8853543 osd.5 (osd.5) 135 : cluster 3 failed to encode map e203 with expected crc 2023-06-03T20:13:59.079 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:13:58 smithi089 bash[75068]: audit 1685823237.888655 mon.a (mon.0) 1077 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-06-03T20:13:59.079 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:13:58 smithi089 bash[75068]: cluster 1685823237.8888206 mon.a (mon.0) 1078 : cluster 0 osdmap e203: 8 total, 7 up, 7 in 2023-06-03T20:13:59.179 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:13:59.208 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:13:59.210 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:13:59.211 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:13:59.312 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:13:59.320 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:13:58 smithi164 bash[59462]: cluster 1685823237.8836267 mon.a (mon.0) 1076 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-06-03T20:13:59.320 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:13:58 smithi164 bash[59462]: cluster 1685823237.885095 osd.6 (osd.6) 148 : cluster 3 failed to encode map e203 with expected crc 2023-06-03T20:13:59.320 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:13:58 smithi164 bash[59462]: cluster 1685823237.8853543 osd.5 (osd.5) 135 : cluster 3 failed to encode map e203 with expected crc 2023-06-03T20:13:59.320 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:13:58 smithi164 bash[59462]: audit 1685823237.888655 mon.a (mon.0) 1077 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-06-03T20:13:59.320 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:13:58 smithi164 bash[59462]: cluster 1685823237.8888206 mon.a (mon.0) 1078 : cluster 0 osdmap e203: 8 total, 7 up, 7 in 2023-06-03T20:13:59.341 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:13:59.343 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:13:59.344 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:13:59.444 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:13:59.478 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:13:59.487 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:13:59.488 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:13:59.589 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:13:59.616 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:13:59.620 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:13:59.621 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:13:59.722 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:13:59.779 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:13:59.790 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:13:59.791 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:13:59.892 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:13:59.910 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:13:59 smithi164 bash[59462]: cluster 1685823238.7393367 mgr.y (mgr.34104) 1138 : cluster 0 pgmap v1112: 89 pgs: 9 active+clean+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 13 active+undersized+degraded, 20 active+undersized, 44 active+clean; 4.1 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 721 op/s; 31965/200943 objects degraded (15.907%); 5304/200943 objects misplaced (2.640%); 2.8 MiB/s, 44 objects/s recovering 2023-06-03T20:13:59.920 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:13:59.923 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:13:59.924 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:14:00.025 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:14:00.064 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:14:00.070 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:14:00.070 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:14:00.171 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:14:00.216 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:14:00.221 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:14:00.222 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:14:00.278 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:13:59 smithi089 bash[78452]: cluster 1685823238.7393367 mgr.y (mgr.34104) 1138 : cluster 0 pgmap v1112: 89 pgs: 9 active+clean+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 13 active+undersized+degraded, 20 active+undersized, 44 active+clean; 4.1 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 721 op/s; 31965/200943 objects degraded (15.907%); 5304/200943 objects misplaced (2.640%); 2.8 MiB/s, 44 objects/s recovering 2023-06-03T20:14:00.279 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:13:59 smithi089 bash[75068]: cluster 1685823238.7393367 mgr.y (mgr.34104) 1138 : cluster 0 pgmap v1112: 89 pgs: 9 active+clean+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 13 active+undersized+degraded, 20 active+undersized, 44 active+clean; 4.1 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 721 op/s; 31965/200943 objects degraded (15.907%); 5304/200943 objects misplaced (2.640%); 2.8 MiB/s, 44 objects/s recovering 2023-06-03T20:14:00.323 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:14:00.364 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:14:00.366 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:14:00.367 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:14:00.468 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:14:00.514 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:14:00.523 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:14:00.524 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:14:00.625 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:14:00.660 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:14:00.664 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:14:00.665 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:14:00.766 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:14:00.807 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:14:00.809 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:14:00.809 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:14:00.910 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:14:00.948 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:14:00.954 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:14:00.954 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:14:01.055 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:14:01.083 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:14:01.087 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:14:01.088 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:14:01.188 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:14:01.235 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:14:01.245 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:14:01.246 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:14:01.347 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:14:01.385 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:14:01.388 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:14:01.389 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:14:01.490 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:14:01.533 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:14:01.538 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:14:01.539 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:14:01.640 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:14:01.672 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:14:01.676 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:14:01.677 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:14:01.778 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:14:01.814 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:14:01.818 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:14:01.819 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:14:01.920 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:14:01.953 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:14:01.957 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:14:01.958 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:14:02.012 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:14:02 smithi164 bash[59462]: cluster 1685823240.740014 mgr.y (mgr.34104) 1139 : cluster 0 pgmap v1113: 89 pgs: 9 active+clean+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 13 active+undersized+degraded, 20 active+undersized, 44 active+clean; 4.0 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 715 op/s; 31316/198396 objects degraded (15.785%); 5304/198396 objects misplaced (2.673%); 2.8 MiB/s, 44 objects/s recovering 2023-06-03T20:14:02.012 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:14:02 smithi164 bash[59462]: audit 1685823241.007654 mon.a (mon.0) 1079 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:14:02.013 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:14:02 smithi164 bash[59462]: audit 1685823241.0300117 mon.a (mon.0) 1080 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:14:02.045 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:14:02 smithi089 bash[78452]: cluster 1685823240.740014 mgr.y (mgr.34104) 1139 : cluster 0 pgmap v1113: 89 pgs: 9 active+clean+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 13 active+undersized+degraded, 20 active+undersized, 44 active+clean; 4.0 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 715 op/s; 31316/198396 objects degraded (15.785%); 5304/198396 objects misplaced (2.673%); 2.8 MiB/s, 44 objects/s recovering 2023-06-03T20:14:02.046 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:14:02 smithi089 bash[78452]: audit 1685823241.007654 mon.a (mon.0) 1079 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:14:02.046 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:14:02 smithi089 bash[78452]: audit 1685823241.0300117 mon.a (mon.0) 1080 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:14:02.046 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:14:02 smithi089 bash[75068]: cluster 1685823240.740014 mgr.y (mgr.34104) 1139 : cluster 0 pgmap v1113: 89 pgs: 9 active+clean+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 13 active+undersized+degraded, 20 active+undersized, 44 active+clean; 4.0 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 715 op/s; 31316/198396 objects degraded (15.785%); 5304/198396 objects misplaced (2.673%); 2.8 MiB/s, 44 objects/s recovering 2023-06-03T20:14:02.046 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:14:02 smithi089 bash[75068]: audit 1685823241.007654 mon.a (mon.0) 1079 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:14:02.047 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:14:02 smithi089 bash[75068]: audit 1685823241.0300117 mon.a (mon.0) 1080 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:14:02.047 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-06-03T20:14:02.048 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-06-03T20:14:02.059 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:14:02.090 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:14:02.094 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:14:02.094 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:14:02.195 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:14:02.225 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:14:02.231 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:14:02.231 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:14:02.284 DEBUG:teuthology.orchestra.run:got remote process result: 2 2023-06-03T20:14:02.332 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:14:02.390 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:14:02.395 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:14:02.395 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:14:02.496 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:14:02.525 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:14:02.529 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:14:02.529 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:14:02.630 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:14:02.668 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:14:02.672 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:14:02.673 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:14:02.774 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:14:02.803 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:14:02.805 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:14:02.806 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:14:02.907 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:14:02.948 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:14:02.953 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:14:02.954 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:14:03.026 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:14:03 smithi164 bash[59462]: cluster 1685823242.753827 mon.a (mon.0) 1081 : cluster 3 Health check update: Degraded data redundancy: 31316/198396 objects degraded (15.785%), 16 pgs degraded, 36 pgs undersized (PG_DEGRADED) 2023-06-03T20:14:03.055 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:14:03.092 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:14:03.097 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:14:03.098 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:14:03.199 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:14:03.240 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:14:03.249 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:14:03.250 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:14:03.278 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:14:03 smithi089 bash[78452]: cluster 1685823242.753827 mon.a (mon.0) 1081 : cluster 3 Health check update: Degraded data redundancy: 31316/198396 objects degraded (15.785%), 16 pgs degraded, 36 pgs undersized (PG_DEGRADED) 2023-06-03T20:14:03.278 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:14:03 smithi089 bash[75068]: cluster 1685823242.753827 mon.a (mon.0) 1081 : cluster 3 Health check update: Degraded data redundancy: 31316/198396 objects degraded (15.785%), 16 pgs degraded, 36 pgs undersized (PG_DEGRADED) 2023-06-03T20:14:03.351 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:14:03.399 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:14:03.401 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:14:03.402 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:14:03.503 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:14:03.542 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:14:03.548 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:14:03.549 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:14:03.650 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:14:03.693 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:14:03.698 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:14:03.699 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:14:03.799 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:14:03.829 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:14:03.839 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:14:03.840 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:14:03.941 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:14:03.984 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:14:03.989 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:14:03.990 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:14:04.027 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:14:04 smithi089 bash[78452]: cluster 1685823242.7406673 mgr.y (mgr.34104) 1140 : cluster 0 pgmap v1114: 89 pgs: 9 active+clean+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 13 active+undersized+degraded, 20 active+undersized, 44 active+clean; 4.0 GiB data, 14 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 713 op/s; 31109/197775 objects degraded (15.729%); 5304/197775 objects misplaced (2.682%); 2.8 MiB/s, 44 objects/s recovering 2023-06-03T20:14:04.027 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:14:04 smithi089 bash[75068]: cluster 1685823242.7406673 mgr.y (mgr.34104) 1140 : cluster 0 pgmap v1114: 89 pgs: 9 active+clean+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 13 active+undersized+degraded, 20 active+undersized, 44 active+clean; 4.0 GiB data, 14 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 713 op/s; 31109/197775 objects degraded (15.729%); 5304/197775 objects misplaced (2.682%); 2.8 MiB/s, 44 objects/s recovering 2023-06-03T20:14:04.049 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-06-03T20:14:04.083 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:14:04 smithi164 bash[59462]: cluster 1685823242.7406673 mgr.y (mgr.34104) 1140 : cluster 0 pgmap v1114: 89 pgs: 9 active+clean+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 13 active+undersized+degraded, 20 active+undersized, 44 active+clean; 4.0 GiB data, 14 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 713 op/s; 31109/197775 objects degraded (15.729%); 5304/197775 objects misplaced (2.682%); 2.8 MiB/s, 44 objects/s recovering 2023-06-03T20:14:04.090 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:14:04.118 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:14:04.121 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:14:04.122 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:14:04.223 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:14:04.251 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:14:04.254 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:14:04.254 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:14:04.310 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph osd unset noscrub 2023-06-03T20:14:04.356 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:14:04.406 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:14:04.412 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:14:04.413 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:14:04.514 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:14:04.559 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:14:04.565 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:14:04.566 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:14:04.621 INFO:teuthology.orchestra.run.smithi089.stdout:{ 2023-06-03T20:14:04.621 INFO:teuthology.orchestra.run.smithi089.stdout: "size": 20, 2023-06-03T20:14:04.621 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 600, 2023-06-03T20:14:04.622 INFO:teuthology.orchestra.run.smithi089.stdout: "ops": [ 2023-06-03T20:14:04.622 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:14:04.622 INFO:teuthology.orchestra.run.smithi089.stdout: "description": "osd_op(client.36009.0:93787 10.8 10:1ed0e12f:::benchmark_data_smithi089_140876_object93786:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e170)", 2023-06-03T20:14:04.622 INFO:teuthology.orchestra.run.smithi089.stdout: "initiated_at": "2023-06-03T20:07:27.763800+0000", 2023-06-03T20:14:04.622 INFO:teuthology.orchestra.run.smithi089.stdout: "age": 396.84542347399997, 2023-06-03T20:14:04.622 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.24366188, 2023-06-03T20:14:04.622 INFO:teuthology.orchestra.run.smithi089.stdout: "type_data": { 2023-06-03T20:14:04.623 INFO:teuthology.orchestra.run.smithi089.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-03T20:14:04.623 INFO:teuthology.orchestra.run.smithi089.stdout: "client_info": { 2023-06-03T20:14:04.623 INFO:teuthology.orchestra.run.smithi089.stdout: "client": "client.36009", 2023-06-03T20:14:04.623 INFO:teuthology.orchestra.run.smithi089.stdout: "client_addr": "172.21.15.89:0/2550116169", 2023-06-03T20:14:04.623 INFO:teuthology.orchestra.run.smithi089.stdout: "tid": 93787 2023-06-03T20:14:04.623 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:14:04.623 INFO:teuthology.orchestra.run.smithi089.stdout: "events": [ 2023-06-03T20:14:04.624 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:14:04.624 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "initiated", 2023-06-03T20:14:04.624 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:07:27.763800+0000", 2023-06-03T20:14:04.624 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0 2023-06-03T20:14:04.624 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:14:04.624 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:14:04.624 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "throttled", 2023-06-03T20:14:04.625 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:07:27.763800+0000", 2023-06-03T20:14:04.625 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0 2023-06-03T20:14:04.625 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:14:04.625 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:14:04.625 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "header_read", 2023-06-03T20:14:04.625 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:07:27.763798+0000", 2023-06-03T20:14:04.625 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 4294967295.9999981 2023-06-03T20:14:04.625 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:14:04.626 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:14:04.626 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "all_read", 2023-06-03T20:14:04.626 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:07:27.763823+0000", 2023-06-03T20:14:04.626 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 2.4756000000000002e-05 2023-06-03T20:14:04.626 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:14:04.626 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:14:04.626 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "dispatched", 2023-06-03T20:14:04.626 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:07:27.763834+0000", 2023-06-03T20:14:04.627 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 1.0482999999999999e-05 2023-06-03T20:14:04.627 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:14:04.627 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:14:04.627 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "queued_for_pg", 2023-06-03T20:14:04.627 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:07:27.763840+0000", 2023-06-03T20:14:04.627 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 6.2790000000000004e-06 2023-06-03T20:14:04.627 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:14:04.628 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:14:04.628 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "reached_pg", 2023-06-03T20:14:04.628 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:07:27.763890+0000", 2023-06-03T20:14:04.628 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 5.0318000000000002e-05 2023-06-03T20:14:04.628 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:14:04.628 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:14:04.628 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "started", 2023-06-03T20:14:04.628 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:07:27.764008+0000", 2023-06-03T20:14:04.629 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.000118196 2023-06-03T20:14:04.629 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:14:04.629 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:14:04.629 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "waiting for subops from 2,7", 2023-06-03T20:14:04.629 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:07:27.764108+0000", 2023-06-03T20:14:04.629 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 9.9907000000000004e-05 2023-06-03T20:14:04.629 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:14:04.629 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:14:04.630 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:14:04.630 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:07:27.767246+0000", 2023-06-03T20:14:04.630 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.0031374509999999999 2023-06-03T20:14:04.630 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:14:04.630 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:14:04.630 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "op_commit", 2023-06-03T20:14:04.630 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:07:27.767285+0000", 2023-06-03T20:14:04.630 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 3.9320000000000003e-05 2023-06-03T20:14:04.631 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:14:04.631 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:14:04.631 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:14:04.631 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:07:28.007200+0000", 2023-06-03T20:14:04.631 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.23991483299999999 2023-06-03T20:14:04.631 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:14:04.631 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:14:04.631 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "commit_sent", 2023-06-03T20:14:04.632 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:07:28.007254+0000", 2023-06-03T20:14:04.632 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 5.4221999999999997e-05 2023-06-03T20:14:04.632 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:14:04.632 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:14:04.632 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "done", 2023-06-03T20:14:04.632 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:07:28.007462+0000", 2023-06-03T20:14:04.632 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.000207911 2023-06-03T20:14:04.633 INFO:teuthology.orchestra.run.smithi089.stdout: } 2023-06-03T20:14:04.633 INFO:teuthology.orchestra.run.smithi089.stdout: ] 2023-06-03T20:14:04.633 INFO:teuthology.orchestra.run.smithi089.stdout: } 2023-06-03T20:14:04.633 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:14:04.633 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:14:04.633 INFO:teuthology.orchestra.run.smithi089.stdout: "description": "osd_repop(client.36009.0:93788 10.d e170/157 10:bfe7c435:::benchmark_data_smithi089_140876_object93787:head v 170'5945, mlcod=170'5942)", 2023-06-03T20:14:04.633 INFO:teuthology.orchestra.run.smithi089.stdout: "initiated_at": "2023-06-03T20:07:27.769502+0000", 2023-06-03T20:14:04.633 INFO:teuthology.orchestra.run.smithi089.stdout: "age": 396.83972131899998, 2023-06-03T20:14:04.634 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.23818402999999999, 2023-06-03T20:14:04.634 INFO:teuthology.orchestra.run.smithi089.stdout: "type_data": { 2023-06-03T20:14:04.634 INFO:teuthology.orchestra.run.smithi089.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-03T20:14:04.634 INFO:teuthology.orchestra.run.smithi089.stdout: "events": [ 2023-06-03T20:14:04.634 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:14:04.634 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "initiated", 2023-06-03T20:14:04.634 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:07:27.769502+0000", 2023-06-03T20:14:04.635 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0 2023-06-03T20:14:04.635 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:14:04.635 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:14:04.635 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "throttled", 2023-06-03T20:14:04.635 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:07:27.769502+0000", 2023-06-03T20:14:04.635 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0 2023-06-03T20:14:04.635 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:14:04.636 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:14:04.636 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "header_read", 2023-06-03T20:14:04.636 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:07:27.769502+0000", 2023-06-03T20:14:04.636 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 4294967295.9999995 2023-06-03T20:14:04.636 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:14:04.636 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:14:04.636 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "all_read", 2023-06-03T20:14:04.636 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:07:27.769566+0000", 2023-06-03T20:14:04.637 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 6.4240999999999997e-05 2023-06-03T20:14:04.637 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:14:04.637 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:14:04.637 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "dispatched", 2023-06-03T20:14:04.637 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:07:27.769575+0000", 2023-06-03T20:14:04.637 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 8.7579999999999998e-06 2023-06-03T20:14:04.637 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:14:04.637 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:14:04.638 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "queued_for_pg", 2023-06-03T20:14:04.638 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:07:27.769580+0000", 2023-06-03T20:14:04.638 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 5.6200000000000004e-06 2023-06-03T20:14:04.638 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:14:04.639 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:14:04.639 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "reached_pg", 2023-06-03T20:14:04.639 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:07:27.769626+0000", 2023-06-03T20:14:04.639 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 4.5466000000000001e-05 2023-06-03T20:14:04.640 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:14:04.640 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:14:04.640 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "started", 2023-06-03T20:14:04.641 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:07:27.769644+0000", 2023-06-03T20:14:04.641 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 1.8671000000000001e-05 2023-06-03T20:14:04.641 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:14:04.641 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:14:04.642 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "commit_sent", 2023-06-03T20:14:04.642 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:07:28.007643+0000", 2023-06-03T20:14:04.642 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.23799832800000001 2023-06-03T20:14:04.643 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:14:04.643 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:14:04.644 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "done", 2023-06-03T20:14:04.644 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:07:28.007686+0000", 2023-06-03T20:14:04.644 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 4.3526999999999998e-05 2023-06-03T20:14:04.644 INFO:teuthology.orchestra.run.smithi089.stdout: } 2023-06-03T20:14:04.645 INFO:teuthology.orchestra.run.smithi089.stdout: ] 2023-06-03T20:14:04.645 INFO:teuthology.orchestra.run.smithi089.stdout: } 2023-06-03T20:14:04.645 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:14:04.645 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:14:04.645 INFO:teuthology.orchestra.run.smithi089.stdout: "description": "osd_op(client.36009.0:93794 10.4 10:2bc5e295:::benchmark_data_smithi089_140876_object93793:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e170)", 2023-06-03T20:14:04.645 INFO:teuthology.orchestra.run.smithi089.stdout: "initiated_at": "2023-06-03T20:07:27.771618+0000", 2023-06-03T20:14:04.645 INFO:teuthology.orchestra.run.smithi089.stdout: "age": 396.83760528200003, 2023-06-03T20:14:04.646 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.24791359499999999, 2023-06-03T20:14:04.646 INFO:teuthology.orchestra.run.smithi089.stdout: "type_data": { 2023-06-03T20:14:04.646 INFO:teuthology.orchestra.run.smithi089.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-03T20:14:04.646 INFO:teuthology.orchestra.run.smithi089.stdout: "client_info": { 2023-06-03T20:14:04.646 INFO:teuthology.orchestra.run.smithi089.stdout: "client": "client.36009", 2023-06-03T20:14:04.646 INFO:teuthology.orchestra.run.smithi089.stdout: "client_addr": "172.21.15.89:0/2550116169", 2023-06-03T20:14:04.646 INFO:teuthology.orchestra.run.smithi089.stdout: "tid": 93794 2023-06-03T20:14:04.647 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:14:04.647 INFO:teuthology.orchestra.run.smithi089.stdout: "events": [ 2023-06-03T20:14:04.647 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:14:04.647 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "initiated", 2023-06-03T20:14:04.647 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:07:27.771618+0000", 2023-06-03T20:14:04.648 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0 2023-06-03T20:14:04.648 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:14:04.648 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:14:04.649 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "throttled", 2023-06-03T20:14:04.649 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:07:27.771618+0000", 2023-06-03T20:14:04.649 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0 2023-06-03T20:14:04.649 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:14:04.650 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:14:04.650 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "header_read", 2023-06-03T20:14:04.650 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:07:27.771616+0000", 2023-06-03T20:14:04.650 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 4294967295.9999976 2023-06-03T20:14:04.650 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:14:04.650 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:14:04.650 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "all_read", 2023-06-03T20:14:04.650 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:07:27.771653+0000", 2023-06-03T20:14:04.651 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 3.7571000000000003e-05 2023-06-03T20:14:04.651 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:14:04.651 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:14:04.651 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "dispatched", 2023-06-03T20:14:04.652 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:07:27.771667+0000", 2023-06-03T20:14:04.652 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 1.3981999999999999e-05 2023-06-03T20:14:04.653 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:14:04.653 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:14:04.653 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "queued_for_pg", 2023-06-03T20:14:04.653 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:07:27.771674+0000", 2023-06-03T20:14:04.653 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 6.7129999999999999e-06 2023-06-03T20:14:04.653 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:14:04.653 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:14:04.654 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "reached_pg", 2023-06-03T20:14:04.654 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:07:27.771717+0000", 2023-06-03T20:14:04.654 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 4.3219000000000001e-05 2023-06-03T20:14:04.654 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:14:04.655 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:14:04.655 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "started", 2023-06-03T20:14:04.655 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:07:27.771848+0000", 2023-06-03T20:14:04.655 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.00013075399999999999 2023-06-03T20:14:04.656 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:14:04.656 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:14:04.656 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "waiting for subops from 3,4", 2023-06-03T20:14:04.657 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:07:27.771986+0000", 2023-06-03T20:14:04.657 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.000138145 2023-06-03T20:14:04.657 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:14:04.658 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:14:04.658 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:14:04.658 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:07:28.011224+0000", 2023-06-03T20:14:04.659 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.23923821100000001 2023-06-03T20:14:04.659 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:14:04.659 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:14:04.659 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:14:04.660 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:07:28.011294+0000", 2023-06-03T20:14:04.660 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 7.0158000000000002e-05 2023-06-03T20:14:04.660 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:14:04.661 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:14:04.661 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "op_commit", 2023-06-03T20:14:04.661 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:07:28.019454+0000", 2023-06-03T20:14:04.661 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.0081596599999999991 2023-06-03T20:14:04.662 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:14:04.662 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:14:04.662 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "commit_sent", 2023-06-03T20:14:04.662 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:07:28.019491+0000", 2023-06-03T20:14:04.662 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 3.6511000000000003e-05 2023-06-03T20:14:04.662 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:14:04.662 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:14:04.664 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "done", 2023-06-03T20:14:04.664 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:07:28.019532+0000", 2023-06-03T20:14:04.664 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 4.1196999999999999e-05 2023-06-03T20:14:04.664 INFO:teuthology.orchestra.run.smithi089.stdout: } 2023-06-03T20:14:04.665 INFO:teuthology.orchestra.run.smithi089.stdout: ] 2023-06-03T20:14:04.665 INFO:teuthology.orchestra.run.smithi089.stdout: } 2023-06-03T20:14:04.665 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:14:04.665 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:14:04.665 INFO:teuthology.orchestra.run.smithi089.stdout: "description": "osd_repop_reply(client.36009.0:93787 10.8 e170/157 ondisk, result = 0)", 2023-06-03T20:14:04.666 INFO:teuthology.orchestra.run.smithi089.stdout: "initiated_at": "2023-06-03T20:07:27.773023+0000", 2023-06-03T20:14:04.666 INFO:teuthology.orchestra.run.smithi089.stdout: "age": 396.83620080700001, 2023-06-03T20:14:04.666 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.234450609, 2023-06-03T20:14:04.667 INFO:teuthology.orchestra.run.smithi089.stdout: "type_data": { 2023-06-03T20:14:04.667 INFO:teuthology.orchestra.run.smithi089.stdout: "flag_point": "started", 2023-06-03T20:14:04.667 INFO:teuthology.orchestra.run.smithi089.stdout: "events": [ 2023-06-03T20:14:04.667 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:14:04.668 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "initiated", 2023-06-03T20:14:04.668 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:07:27.773023+0000", 2023-06-03T20:14:04.668 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0 2023-06-03T20:14:04.668 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:14:04.669 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:14:04.669 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "throttled", 2023-06-03T20:14:04.669 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:07:27.773023+0000", 2023-06-03T20:14:04.670 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0 2023-06-03T20:14:04.670 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:14:04.670 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:14:04.670 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "header_read", 2023-06-03T20:14:04.670 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:07:27.773022+0000", 2023-06-03T20:14:04.670 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 4294967295.999999 2023-06-03T20:14:04.670 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:14:04.670 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:14:04.671 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "all_read", 2023-06-03T20:14:04.671 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:07:27.773030+0000", 2023-06-03T20:14:04.671 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 7.7319999999999998e-06 2023-06-03T20:14:04.671 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:14:04.671 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:14:04.671 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "dispatched", 2023-06-03T20:14:04.671 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:07:28.006956+0000", 2023-06-03T20:14:04.672 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.233926412 2023-06-03T20:14:04.672 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:14:04.672 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:14:04.672 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "queued_for_pg", 2023-06-03T20:14:04.672 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:07:28.006980+0000", 2023-06-03T20:14:04.673 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 2.4019000000000002e-05 2023-06-03T20:14:04.673 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:14:04.673 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:14:04.673 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "reached_pg", 2023-06-03T20:14:04.673 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:07:28.007174+0000", 2023-06-03T20:14:04.673 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.000194413 2023-06-03T20:14:04.673 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:14:04.674 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:14:04.674 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "started", 2023-06-03T20:14:04.674 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:07:28.007189+0000", 2023-06-03T20:14:04.674 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 1.4314000000000001e-05 2023-06-03T20:14:04.674 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:14:04.674 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:14:04.674 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "done", 2023-06-03T20:14:04.675 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:07:28.007473+0000", 2023-06-03T20:14:04.675 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.00028461900000000001 2023-06-03T20:14:04.675 INFO:teuthology.orchestra.run.smithi089.stdout: } 2023-06-03T20:14:04.675 INFO:teuthology.orchestra.run.smithi089.stdout: ] 2023-06-03T20:14:04.675 INFO:teuthology.orchestra.run.smithi089.stdout: } 2023-06-03T20:14:04.675 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:14:04.675 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:14:04.676 INFO:teuthology.orchestra.run.smithi089.stdout: "description": "osd_op(client.36009.0:93797 10.4 10:2ab04804:::benchmark_data_smithi089_140876_object93796:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e170)", 2023-06-03T20:14:04.676 INFO:teuthology.orchestra.run.smithi089.stdout: "initiated_at": "2023-06-03T20:07:27.774857+0000", 2023-06-03T20:14:04.676 INFO:teuthology.orchestra.run.smithi089.stdout: "age": 396.83436650599998, 2023-06-03T20:14:04.676 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.248770939, 2023-06-03T20:14:04.676 INFO:teuthology.orchestra.run.smithi089.stdout: "type_data": { 2023-06-03T20:14:04.676 INFO:teuthology.orchestra.run.smithi089.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-03T20:14:04.677 INFO:teuthology.orchestra.run.smithi089.stdout: "client_info": { 2023-06-03T20:14:04.677 INFO:teuthology.orchestra.run.smithi089.stdout: "client": "client.36009", 2023-06-03T20:14:04.677 INFO:teuthology.orchestra.run.smithi089.stdout: "client_addr": "172.21.15.89:0/2550116169", 2023-06-03T20:14:04.677 INFO:teuthology.orchestra.run.smithi089.stdout: "tid": 93797 2023-06-03T20:14:04.677 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:14:04.677 INFO:teuthology.orchestra.run.smithi089.stdout: "events": [ 2023-06-03T20:14:04.677 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:14:04.677 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "initiated", 2023-06-03T20:14:04.678 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:07:27.774857+0000", 2023-06-03T20:14:04.678 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0 2023-06-03T20:14:04.678 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:14:04.678 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:14:04.678 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "throttled", 2023-06-03T20:14:04.678 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:07:27.774857+0000", 2023-06-03T20:14:04.679 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0 2023-06-03T20:14:04.679 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:14:04.679 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:14:04.679 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "header_read", 2023-06-03T20:14:04.680 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:07:27.774855+0000", 2023-06-03T20:14:04.680 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 4294967295.9999986 2023-06-03T20:14:04.680 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:14:04.680 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:14:04.681 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "all_read", 2023-06-03T20:14:04.681 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:07:27.774880+0000", 2023-06-03T20:14:04.681 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 2.4372e-05 2023-06-03T20:14:04.681 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:14:04.681 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:14:04.681 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "dispatched", 2023-06-03T20:14:04.682 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:07:28.006970+0000", 2023-06-03T20:14:04.682 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.23209026099999999 2023-06-03T20:14:04.682 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:14:04.682 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:14:04.682 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "queued_for_pg", 2023-06-03T20:14:04.682 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:07:28.006990+0000", 2023-06-03T20:14:04.682 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 2.0344999999999999e-05 2023-06-03T20:14:04.683 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:14:04.683 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:14:04.683 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "reached_pg", 2023-06-03T20:14:04.683 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:07:28.011461+0000", 2023-06-03T20:14:04.683 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.0044709290000000002 2023-06-03T20:14:04.683 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:14:04.683 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:14:04.683 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "started", 2023-06-03T20:14:04.684 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:07:28.011655+0000", 2023-06-03T20:14:04.684 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.000193969 2023-06-03T20:14:04.684 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:14:04.684 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:14:04.684 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "waiting for subops from 3,4", 2023-06-03T20:14:04.684 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:07:28.011814+0000", 2023-06-03T20:14:04.684 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.00015902200000000001 2023-06-03T20:14:04.685 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:14:04.685 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:14:04.685 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:14:04.685 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:07:28.023473+0000", 2023-06-03T20:14:04.685 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.011658236000000001 2023-06-03T20:14:04.685 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:14:04.685 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:14:04.685 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "op_commit", 2023-06-03T20:14:04.686 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:07:28.023484+0000", 2023-06-03T20:14:04.686 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 1.1401e-05 2023-06-03T20:14:04.686 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:14:04.686 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:14:04.686 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:14:04.686 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:07:28.023531+0000", 2023-06-03T20:14:04.686 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 4.7058000000000002e-05 2023-06-03T20:14:04.686 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:14:04.687 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:14:04.687 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "commit_sent", 2023-06-03T20:14:04.687 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:07:28.023581+0000", 2023-06-03T20:14:04.687 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 5.0064e-05 2023-06-03T20:14:04.687 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:14:04.687 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:14:04.687 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "done", 2023-06-03T20:14:04.687 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:07:28.023628+0000", 2023-06-03T20:14:04.688 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 4.6897999999999997e-05 2023-06-03T20:14:04.688 INFO:teuthology.orchestra.run.smithi089.stdout: } 2023-06-03T20:14:04.688 INFO:teuthology.orchestra.run.smithi089.stdout: ] 2023-06-03T20:14:04.688 INFO:teuthology.orchestra.run.smithi089.stdout: } 2023-06-03T20:14:04.688 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:14:04.688 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:14:04.688 INFO:teuthology.orchestra.run.smithi089.stdout: "description": "osd_repop(client.36009.0:159969 10.b e176/167 10:db66c7c6:::benchmark_data_smithi089_140876_object54834:head v 176'9756, mlcod=163'3266)", 2023-06-03T20:14:04.689 INFO:teuthology.orchestra.run.smithi089.stdout: "initiated_at": "2023-06-03T20:09:10.065868+0000", 2023-06-03T20:14:04.689 INFO:teuthology.orchestra.run.smithi089.stdout: "age": 294.543355152, 2023-06-03T20:14:04.689 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.222988926, 2023-06-03T20:14:04.689 INFO:teuthology.orchestra.run.smithi089.stdout: "type_data": { 2023-06-03T20:14:04.689 INFO:teuthology.orchestra.run.smithi089.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-03T20:14:04.689 INFO:teuthology.orchestra.run.smithi089.stdout: "events": [ 2023-06-03T20:14:04.690 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:14:04.690 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "initiated", 2023-06-03T20:14:04.690 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:09:10.065868+0000", 2023-06-03T20:14:04.690 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0 2023-06-03T20:14:04.691 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:14:04.691 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:14:04.691 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "throttled", 2023-06-03T20:14:04.691 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:09:10.065868+0000", 2023-06-03T20:14:04.691 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0 2023-06-03T20:14:04.692 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:14:04.692 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:14:04.692 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "header_read", 2023-06-03T20:14:04.692 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:09:10.065868+0000", 2023-06-03T20:14:04.693 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 4294967295.999999 2023-06-03T20:14:04.693 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:14:04.693 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:14:04.693 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "all_read", 2023-06-03T20:14:04.693 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:09:10.065875+0000", 2023-06-03T20:14:04.693 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 7.4590000000000001e-06 2023-06-03T20:14:04.694 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:14:04.694 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:14:04.694 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "dispatched", 2023-06-03T20:14:04.694 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:09:10.065885+0000", 2023-06-03T20:14:04.694 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 1.0147e-05 2023-06-03T20:14:04.694 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:14:04.694 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:14:04.694 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "queued_for_pg", 2023-06-03T20:14:04.695 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:09:10.065892+0000", 2023-06-03T20:14:04.695 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 6.5950000000000004e-06 2023-06-03T20:14:04.695 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:14:04.695 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:14:04.695 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "reached_pg", 2023-06-03T20:14:04.695 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:09:10.065954+0000", 2023-06-03T20:14:04.695 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 6.2216000000000002e-05 2023-06-03T20:14:04.695 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:14:04.696 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:14:04.696 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "started", 2023-06-03T20:14:04.696 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:09:10.065972+0000", 2023-06-03T20:14:04.696 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 1.8159999999999999e-05 2023-06-03T20:14:04.696 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:14:04.696 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:14:04.696 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "commit_sent", 2023-06-03T20:14:04.697 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:09:10.288743+0000", 2023-06-03T20:14:04.697 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.22277042499999999 2023-06-03T20:14:04.697 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:14:04.697 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:14:04.697 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "done", 2023-06-03T20:14:04.697 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:09:10.288857+0000", 2023-06-03T20:14:04.697 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.000114789 2023-06-03T20:14:04.697 INFO:teuthology.orchestra.run.smithi089.stdout: } 2023-06-03T20:14:04.698 INFO:teuthology.orchestra.run.smithi089.stdout: ] 2023-06-03T20:14:04.698 INFO:teuthology.orchestra.run.smithi089.stdout: } 2023-06-03T20:14:04.698 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:14:04.698 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:14:04.698 INFO:teuthology.orchestra.run.smithi089.stdout: "description": "osd_repop(client.36009.0:159965 10.2 e176/167 10:4c8f1693:::benchmark_data_smithi089_140876_object54830:head v 176'10151, mlcod=0'0)", 2023-06-03T20:14:04.698 INFO:teuthology.orchestra.run.smithi089.stdout: "initiated_at": "2023-06-03T20:09:10.068090+0000", 2023-06-03T20:14:04.698 INFO:teuthology.orchestra.run.smithi089.stdout: "age": 294.54113332899999, 2023-06-03T20:14:04.699 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.25234304000000002, 2023-06-03T20:14:04.699 INFO:teuthology.orchestra.run.smithi089.stdout: "type_data": { 2023-06-03T20:14:04.699 INFO:teuthology.orchestra.run.smithi089.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-03T20:14:04.699 INFO:teuthology.orchestra.run.smithi089.stdout: "events": [ 2023-06-03T20:14:04.699 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:14:04.699 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "initiated", 2023-06-03T20:14:04.699 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:09:10.068090+0000", 2023-06-03T20:14:04.699 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0 2023-06-03T20:14:04.700 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:14:04.700 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:14:04.700 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "throttled", 2023-06-03T20:14:04.700 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:09:10.068090+0000", 2023-06-03T20:14:04.700 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0 2023-06-03T20:14:04.700 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:14:04.700 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:14:04.700 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "header_read", 2023-06-03T20:14:04.701 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:09:10.068089+0000", 2023-06-03T20:14:04.701 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 4294967295.999999 2023-06-03T20:14:04.701 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:14:04.701 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:14:04.701 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "all_read", 2023-06-03T20:14:04.701 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:09:10.068102+0000", 2023-06-03T20:14:04.701 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 1.2658e-05 2023-06-03T20:14:04.702 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:14:04.702 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:14:04.702 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "dispatched", 2023-06-03T20:14:04.702 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:09:10.068111+0000", 2023-06-03T20:14:04.702 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 8.9169999999999995e-06 2023-06-03T20:14:04.702 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:14:04.702 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:14:04.702 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "queued_for_pg", 2023-06-03T20:14:04.703 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:09:10.068116+0000", 2023-06-03T20:14:04.703 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 5.1900000000000003e-06 2023-06-03T20:14:04.703 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:14:04.703 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:14:04.703 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "reached_pg", 2023-06-03T20:14:04.703 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:09:10.289577+0000", 2023-06-03T20:14:04.703 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.22146059700000001 2023-06-03T20:14:04.703 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:14:04.704 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:14:04.704 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "started", 2023-06-03T20:14:04.704 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:09:10.289648+0000", 2023-06-03T20:14:04.704 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 7.1181999999999997e-05 2023-06-03T20:14:04.704 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:14:04.704 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:14:04.704 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "commit_sent", 2023-06-03T20:14:04.704 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:09:10.320387+0000", 2023-06-03T20:14:04.705 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.030739628000000001 2023-06-03T20:14:04.705 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:14:04.705 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:14:04.705 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "done", 2023-06-03T20:14:04.705 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:09:10.320433+0000", 2023-06-03T20:14:04.705 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 4.5853999999999998e-05 2023-06-03T20:14:04.705 INFO:teuthology.orchestra.run.smithi089.stdout: } 2023-06-03T20:14:04.706 INFO:teuthology.orchestra.run.smithi089.stdout: ] 2023-06-03T20:14:04.706 INFO:teuthology.orchestra.run.smithi089.stdout: } 2023-06-03T20:14:04.706 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:14:04.706 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:14:04.706 INFO:teuthology.orchestra.run.smithi089.stdout: "description": "osd_repop(client.36009.0:159972 10.1 e176/157 10:89d299ee:::benchmark_data_smithi089_140876_object54837:head v 176'9770, mlcod=176'9767)", 2023-06-03T20:14:04.706 INFO:teuthology.orchestra.run.smithi089.stdout: "initiated_at": "2023-06-03T20:09:10.068129+0000", 2023-06-03T20:14:04.706 INFO:teuthology.orchestra.run.smithi089.stdout: "age": 294.54109483799999, 2023-06-03T20:14:04.706 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.23239045899999999, 2023-06-03T20:14:04.707 INFO:teuthology.orchestra.run.smithi089.stdout: "type_data": { 2023-06-03T20:14:04.707 INFO:teuthology.orchestra.run.smithi089.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-03T20:14:04.707 INFO:teuthology.orchestra.run.smithi089.stdout: "events": [ 2023-06-03T20:14:04.707 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:14:04.707 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "initiated", 2023-06-03T20:14:04.707 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:09:10.068129+0000", 2023-06-03T20:14:04.708 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0 2023-06-03T20:14:04.708 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:14:04.708 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:14:04.709 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "throttled", 2023-06-03T20:14:04.709 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:09:10.068129+0000", 2023-06-03T20:14:04.709 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0 2023-06-03T20:14:04.709 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:14:04.709 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:14:04.710 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "header_read", 2023-06-03T20:14:04.710 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:09:10.068129+0000", 2023-06-03T20:14:04.710 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 4294967296 2023-06-03T20:14:04.710 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:14:04.710 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:14:04.710 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "all_read", 2023-06-03T20:14:04.710 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:09:10.068131+0000", 2023-06-03T20:14:04.710 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 2.8899999999999999e-06 2023-06-03T20:14:04.711 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:14:04.711 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:14:04.711 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "dispatched", 2023-06-03T20:14:04.711 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:09:10.068135+0000", 2023-06-03T20:14:04.711 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 3.8840000000000004e-06 2023-06-03T20:14:04.711 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:14:04.711 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:14:04.711 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "queued_for_pg", 2023-06-03T20:14:04.712 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:09:10.068138+0000", 2023-06-03T20:14:04.712 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 2.6089999999999999e-06 2023-06-03T20:14:04.712 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:14:04.712 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:14:04.712 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "reached_pg", 2023-06-03T20:14:04.712 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:09:10.288673+0000", 2023-06-03T20:14:04.712 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.220535126 2023-06-03T20:14:04.713 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:14:04.713 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:14:04.713 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "started", 2023-06-03T20:14:04.713 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:09:10.288804+0000", 2023-06-03T20:14:04.713 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.000131388 2023-06-03T20:14:04.713 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:14:04.713 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:14:04.713 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "commit_sent", 2023-06-03T20:14:04.714 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:09:10.300427+0000", 2023-06-03T20:14:04.714 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.011622721000000001 2023-06-03T20:14:04.714 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:14:04.714 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:14:04.714 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "done", 2023-06-03T20:14:04.714 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:09:10.300519+0000", 2023-06-03T20:14:04.714 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 9.2033999999999993e-05 2023-06-03T20:14:04.714 INFO:teuthology.orchestra.run.smithi089.stdout: } 2023-06-03T20:14:04.715 INFO:teuthology.orchestra.run.smithi089.stdout: ] 2023-06-03T20:14:04.715 INFO:teuthology.orchestra.run.smithi089.stdout: } 2023-06-03T20:14:04.715 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:14:04.715 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:14:04.715 INFO:teuthology.orchestra.run.smithi089.stdout: "description": "osd_repop(client.36009.0:159975 10.1 e176/157 10:851a60fd:::benchmark_data_smithi089_140876_object54840:head v 176'9771, mlcod=176'9768)", 2023-06-03T20:14:04.715 INFO:teuthology.orchestra.run.smithi089.stdout: "initiated_at": "2023-06-03T20:09:10.070839+0000", 2023-06-03T20:14:04.716 INFO:teuthology.orchestra.run.smithi089.stdout: "age": 294.53838491300002, 2023-06-03T20:14:04.716 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.23080806300000001, 2023-06-03T20:14:04.716 INFO:teuthology.orchestra.run.smithi089.stdout: "type_data": { 2023-06-03T20:14:04.716 INFO:teuthology.orchestra.run.smithi089.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-03T20:14:04.716 INFO:teuthology.orchestra.run.smithi089.stdout: "events": [ 2023-06-03T20:14:04.716 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:14:04.716 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "initiated", 2023-06-03T20:14:04.717 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:09:10.070839+0000", 2023-06-03T20:14:04.717 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0 2023-06-03T20:14:04.717 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:14:04.717 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:14:04.717 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "throttled", 2023-06-03T20:14:04.717 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:09:10.070839+0000", 2023-06-03T20:14:04.717 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0 2023-06-03T20:14:04.718 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:14:04.718 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:14:04.718 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "header_read", 2023-06-03T20:14:04.718 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:09:10.070838+0000", 2023-06-03T20:14:04.718 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 4294967295.999999 2023-06-03T20:14:04.718 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:14:04.718 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:14:04.718 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "all_read", 2023-06-03T20:14:04.719 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:09:10.070845+0000", 2023-06-03T20:14:04.719 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 7.199e-06 2023-06-03T20:14:04.719 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:14:04.719 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:14:04.719 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "dispatched", 2023-06-03T20:14:04.719 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:09:10.288741+0000", 2023-06-03T20:14:04.719 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.21789615900000001 2023-06-03T20:14:04.719 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:14:04.720 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:14:04.720 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "queued_for_pg", 2023-06-03T20:14:04.720 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:09:10.288776+0000", 2023-06-03T20:14:04.720 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 3.5277000000000001e-05 2023-06-03T20:14:04.720 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:14:04.720 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:14:04.720 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "reached_pg", 2023-06-03T20:14:04.721 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:09:10.292886+0000", 2023-06-03T20:14:04.721 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.0041099480000000004 2023-06-03T20:14:04.721 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:14:04.721 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:14:04.721 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "started", 2023-06-03T20:14:04.721 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:09:10.292939+0000", 2023-06-03T20:14:04.721 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 5.2172999999999997e-05 2023-06-03T20:14:04.722 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:14:04.722 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:14:04.722 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "commit_sent", 2023-06-03T20:14:04.722 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:09:10.301612+0000", 2023-06-03T20:14:04.722 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.0086729519999999994 2023-06-03T20:14:04.722 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:14:04.722 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:14:04.722 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "done", 2023-06-03T20:14:04.723 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:09:10.301647+0000", 2023-06-03T20:14:04.723 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 3.5194000000000002e-05 2023-06-03T20:14:04.723 INFO:teuthology.orchestra.run.smithi089.stdout: } 2023-06-03T20:14:04.723 INFO:teuthology.orchestra.run.smithi089.stdout: ] 2023-06-03T20:14:04.723 INFO:teuthology.orchestra.run.smithi089.stdout: } 2023-06-03T20:14:04.723 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:14:04.723 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:14:04.723 INFO:teuthology.orchestra.run.smithi089.stdout: "description": "osd_op(client.46187.0:11084 11.6 11:6960d7a5:::benchmark_data_smithi089_165412_object11083:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e193)", 2023-06-03T20:14:04.724 INFO:teuthology.orchestra.run.smithi089.stdout: "initiated_at": "2023-06-03T20:11:37.817163+0000", 2023-06-03T20:14:04.724 INFO:teuthology.orchestra.run.smithi089.stdout: "age": 146.79206094400001, 2023-06-03T20:14:04.724 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.15708528999999999, 2023-06-03T20:14:04.724 INFO:teuthology.orchestra.run.smithi089.stdout: "type_data": { 2023-06-03T20:14:04.724 INFO:teuthology.orchestra.run.smithi089.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-03T20:14:04.724 INFO:teuthology.orchestra.run.smithi089.stdout: "client_info": { 2023-06-03T20:14:04.724 INFO:teuthology.orchestra.run.smithi089.stdout: "client": "client.46187", 2023-06-03T20:14:04.725 INFO:teuthology.orchestra.run.smithi089.stdout: "client_addr": "172.21.15.89:0/880684210", 2023-06-03T20:14:04.725 INFO:teuthology.orchestra.run.smithi089.stdout: "tid": 11084 2023-06-03T20:14:04.725 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:14:04.725 INFO:teuthology.orchestra.run.smithi089.stdout: "events": [ 2023-06-03T20:14:04.725 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:14:04.725 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "initiated", 2023-06-03T20:14:04.725 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:11:37.817163+0000", 2023-06-03T20:14:04.725 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0 2023-06-03T20:14:04.726 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:14:04.726 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:14:04.726 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "throttled", 2023-06-03T20:14:04.726 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:11:37.817163+0000", 2023-06-03T20:14:04.726 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0 2023-06-03T20:14:04.726 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:14:04.726 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:14:04.726 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "header_read", 2023-06-03T20:14:04.727 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:11:37.817161+0000", 2023-06-03T20:14:04.727 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 4294967295.9999986 2023-06-03T20:14:04.727 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:14:04.727 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:14:04.727 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "all_read", 2023-06-03T20:14:04.727 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:11:37.817183+0000", 2023-06-03T20:14:04.727 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 2.2061000000000001e-05 2023-06-03T20:14:04.727 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:14:04.728 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:14:04.728 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "dispatched", 2023-06-03T20:14:04.728 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:11:37.817195+0000", 2023-06-03T20:14:04.728 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 1.2139e-05 2023-06-03T20:14:04.728 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:14:04.728 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:14:04.728 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "queued_for_pg", 2023-06-03T20:14:04.728 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:11:37.817204+0000", 2023-06-03T20:14:04.729 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 8.8969999999999997e-06 2023-06-03T20:14:04.729 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:14:04.729 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:14:04.729 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "reached_pg", 2023-06-03T20:14:04.729 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:11:37.817248+0000", 2023-06-03T20:14:04.729 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 4.3782999999999997e-05 2023-06-03T20:14:04.729 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:14:04.729 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:14:04.730 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "started", 2023-06-03T20:14:04.730 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:11:37.817387+0000", 2023-06-03T20:14:04.730 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.000139226 2023-06-03T20:14:04.730 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:14:04.730 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:14:04.730 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "waiting for subops from 2,4", 2023-06-03T20:14:04.730 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:11:37.817514+0000", 2023-06-03T20:14:04.730 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.00012689299999999999 2023-06-03T20:14:04.731 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:14:04.731 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:14:04.731 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "op_commit", 2023-06-03T20:14:04.731 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:11:37.818238+0000", 2023-06-03T20:14:04.731 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.00072384900000000004 2023-06-03T20:14:04.731 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:14:04.731 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:14:04.732 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:14:04.732 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:11:37.819079+0000", 2023-06-03T20:14:04.732 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.00084151500000000004 2023-06-03T20:14:04.732 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:14:04.732 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:14:04.732 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:14:04.732 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:11:37.974014+0000", 2023-06-03T20:14:04.732 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.15493417400000001 2023-06-03T20:14:04.733 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:14:04.733 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:14:04.733 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "commit_sent", 2023-06-03T20:14:04.733 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:11:37.974146+0000", 2023-06-03T20:14:04.733 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.00013221399999999999 2023-06-03T20:14:04.733 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:14:04.733 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:14:04.733 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "done", 2023-06-03T20:14:04.734 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:11:37.974248+0000", 2023-06-03T20:14:04.734 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.000102021 2023-06-03T20:14:04.734 INFO:teuthology.orchestra.run.smithi089.stdout: } 2023-06-03T20:14:04.734 INFO:teuthology.orchestra.run.smithi089.stdout: ] 2023-06-03T20:14:04.734 INFO:teuthology.orchestra.run.smithi089.stdout: } 2023-06-03T20:14:04.734 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:14:04.734 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:14:04.734 INFO:teuthology.orchestra.run.smithi089.stdout: "description": "osd_op(client.46187.0:11091 11.6 11:6964d10d:::benchmark_data_smithi089_165412_object11090:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e193)", 2023-06-03T20:14:04.735 INFO:teuthology.orchestra.run.smithi089.stdout: "initiated_at": "2023-06-03T20:11:37.820735+0000", 2023-06-03T20:14:04.735 INFO:teuthology.orchestra.run.smithi089.stdout: "age": 146.78848834199999, 2023-06-03T20:14:04.735 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.15647836600000001, 2023-06-03T20:14:04.735 INFO:teuthology.orchestra.run.smithi089.stdout: "type_data": { 2023-06-03T20:14:04.735 INFO:teuthology.orchestra.run.smithi089.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-03T20:14:04.735 INFO:teuthology.orchestra.run.smithi089.stdout: "client_info": { 2023-06-03T20:14:04.735 INFO:teuthology.orchestra.run.smithi089.stdout: "client": "client.46187", 2023-06-03T20:14:04.736 INFO:teuthology.orchestra.run.smithi089.stdout: "client_addr": "172.21.15.89:0/880684210", 2023-06-03T20:14:04.736 INFO:teuthology.orchestra.run.smithi089.stdout: "tid": 11091 2023-06-03T20:14:04.736 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:14:04.736 INFO:teuthology.orchestra.run.smithi089.stdout: "events": [ 2023-06-03T20:14:04.736 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:14:04.736 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "initiated", 2023-06-03T20:14:04.736 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:11:37.820735+0000", 2023-06-03T20:14:04.737 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0 2023-06-03T20:14:04.737 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:14:04.737 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:14:04.737 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "throttled", 2023-06-03T20:14:04.737 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:11:37.820735+0000", 2023-06-03T20:14:04.737 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0 2023-06-03T20:14:04.737 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:14:04.737 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:14:04.738 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "header_read", 2023-06-03T20:14:04.738 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:11:37.820734+0000", 2023-06-03T20:14:04.738 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 4294967295.9999986 2023-06-03T20:14:04.738 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:14:04.738 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:14:04.738 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "all_read", 2023-06-03T20:14:04.738 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:11:37.820758+0000", 2023-06-03T20:14:04.738 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 2.3876e-05 2023-06-03T20:14:04.739 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:14:04.739 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:14:04.739 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "dispatched", 2023-06-03T20:14:04.739 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:11:37.820769+0000", 2023-06-03T20:14:04.739 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 1.1881e-05 2023-06-03T20:14:04.739 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:14:04.739 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:14:04.739 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "queued_for_pg", 2023-06-03T20:14:04.740 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:11:37.820777+0000", 2023-06-03T20:14:04.740 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 7.4320000000000002e-06 2023-06-03T20:14:04.740 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:14:04.740 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:14:04.740 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "reached_pg", 2023-06-03T20:14:04.740 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:11:37.820820+0000", 2023-06-03T20:14:04.740 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 4.2886000000000002e-05 2023-06-03T20:14:04.740 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:14:04.741 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:14:04.741 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "started", 2023-06-03T20:14:04.741 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:11:37.820975+0000", 2023-06-03T20:14:04.741 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.00015515200000000001 2023-06-03T20:14:04.741 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:14:04.741 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:14:04.741 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "waiting for subops from 2,4", 2023-06-03T20:14:04.741 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:11:37.821150+0000", 2023-06-03T20:14:04.742 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.00017504 2023-06-03T20:14:04.742 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:14:04.742 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:14:04.742 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:14:04.742 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:11:37.823262+0000", 2023-06-03T20:14:04.742 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.0021122469999999998 2023-06-03T20:14:04.742 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:14:04.743 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:14:04.743 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "op_commit", 2023-06-03T20:14:04.743 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:11:37.976219+0000", 2023-06-03T20:14:04.743 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.15295651599999999 2023-06-03T20:14:04.743 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:14:04.743 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:14:04.743 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:14:04.743 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:11:37.977124+0000", 2023-06-03T20:14:04.744 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.00090513599999999996 2023-06-03T20:14:04.744 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:14:04.744 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:14:04.744 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "commit_sent", 2023-06-03T20:14:04.744 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:11:37.977178+0000", 2023-06-03T20:14:04.745 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 5.3851000000000003e-05 2023-06-03T20:14:04.745 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:14:04.745 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:14:04.745 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "done", 2023-06-03T20:14:04.745 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:11:37.977214+0000", 2023-06-03T20:14:04.745 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 3.5899000000000003e-05 2023-06-03T20:14:04.746 INFO:teuthology.orchestra.run.smithi089.stdout: } 2023-06-03T20:14:04.746 INFO:teuthology.orchestra.run.smithi089.stdout: ] 2023-06-03T20:14:04.746 INFO:teuthology.orchestra.run.smithi089.stdout: } 2023-06-03T20:14:04.746 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:14:04.746 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:14:04.746 INFO:teuthology.orchestra.run.smithi089.stdout: "description": "osd_op(client.46187.0:11094 11.6 11:6b083a6d:::benchmark_data_smithi089_165412_object11093:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e193)", 2023-06-03T20:14:04.746 INFO:teuthology.orchestra.run.smithi089.stdout: "initiated_at": "2023-06-03T20:11:37.821972+0000", 2023-06-03T20:14:04.746 INFO:teuthology.orchestra.run.smithi089.stdout: "age": 146.787251435, 2023-06-03T20:14:04.747 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.15774666300000001, 2023-06-03T20:14:04.747 INFO:teuthology.orchestra.run.smithi089.stdout: "type_data": { 2023-06-03T20:14:04.747 INFO:teuthology.orchestra.run.smithi089.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-03T20:14:04.747 INFO:teuthology.orchestra.run.smithi089.stdout: "client_info": { 2023-06-03T20:14:04.747 INFO:teuthology.orchestra.run.smithi089.stdout: "client": "client.46187", 2023-06-03T20:14:04.747 INFO:teuthology.orchestra.run.smithi089.stdout: "client_addr": "172.21.15.89:0/880684210", 2023-06-03T20:14:04.747 INFO:teuthology.orchestra.run.smithi089.stdout: "tid": 11094 2023-06-03T20:14:04.747 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:14:04.748 INFO:teuthology.orchestra.run.smithi089.stdout: "events": [ 2023-06-03T20:14:04.748 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:14:04.748 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "initiated", 2023-06-03T20:14:04.748 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:11:37.821972+0000", 2023-06-03T20:14:04.748 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0 2023-06-03T20:14:04.748 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:14:04.749 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:14:04.749 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "throttled", 2023-06-03T20:14:04.749 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:11:37.821972+0000", 2023-06-03T20:14:04.749 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0 2023-06-03T20:14:04.749 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:14:04.749 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:14:04.749 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "header_read", 2023-06-03T20:14:04.749 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:11:37.821971+0000", 2023-06-03T20:14:04.750 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 4294967295.999999 2023-06-03T20:14:04.750 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:14:04.750 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:14:04.750 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "all_read", 2023-06-03T20:14:04.750 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:11:37.821991+0000", 2023-06-03T20:14:04.750 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 1.9536000000000001e-05 2023-06-03T20:14:04.750 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:14:04.750 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:14:04.751 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "dispatched", 2023-06-03T20:14:04.751 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:11:37.822000+0000", 2023-06-03T20:14:04.751 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 9.825e-06 2023-06-03T20:14:04.751 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:14:04.751 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:14:04.751 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "queued_for_pg", 2023-06-03T20:14:04.751 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:11:37.822008+0000", 2023-06-03T20:14:04.751 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 7.5859999999999999e-06 2023-06-03T20:14:04.752 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:14:04.752 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:14:04.752 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "reached_pg", 2023-06-03T20:14:04.752 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:11:37.974339+0000", 2023-06-03T20:14:04.752 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.152330458 2023-06-03T20:14:04.752 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:14:04.752 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:14:04.752 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "started", 2023-06-03T20:14:04.753 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:11:37.974652+0000", 2023-06-03T20:14:04.753 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.00031328200000000001 2023-06-03T20:14:04.753 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:14:04.753 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:14:04.753 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "waiting for subops from 2,4", 2023-06-03T20:14:04.753 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:11:37.974981+0000", 2023-06-03T20:14:04.753 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.00032898500000000001 2023-06-03T20:14:04.753 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:14:04.754 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:14:04.754 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "op_commit", 2023-06-03T20:14:04.754 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:11:37.977356+0000", 2023-06-03T20:14:04.754 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.0023755170000000002 2023-06-03T20:14:04.754 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:14:04.754 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:14:04.754 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:14:04.754 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:11:37.978064+0000", 2023-06-03T20:14:04.755 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.00070721599999999999 2023-06-03T20:14:04.755 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:14:04.755 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:14:04.755 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:14:04.755 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:11:37.979502+0000", 2023-06-03T20:14:04.755 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.0014384560000000001 2023-06-03T20:14:04.755 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:14:04.756 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:14:04.756 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "commit_sent", 2023-06-03T20:14:04.756 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:11:37.979653+0000", 2023-06-03T20:14:04.756 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.00015089899999999999 2023-06-03T20:14:04.756 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:14:04.756 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:14:04.756 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "done", 2023-06-03T20:14:04.756 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:11:37.979719+0000", 2023-06-03T20:14:04.757 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 6.5919000000000004e-05 2023-06-03T20:14:04.757 INFO:teuthology.orchestra.run.smithi089.stdout: } 2023-06-03T20:14:04.757 INFO:teuthology.orchestra.run.smithi089.stdout: ] 2023-06-03T20:14:04.757 INFO:teuthology.orchestra.run.smithi089.stdout: } 2023-06-03T20:14:04.757 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:14:04.757 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:14:04.757 INFO:teuthology.orchestra.run.smithi089.stdout: "description": "osd_repop(client.46187.0:11093 11.9 e193/189 11:90fc17a6:::benchmark_data_smithi089_165412_object11092:head v 193'682, mlcod=193'680)", 2023-06-03T20:14:04.757 INFO:teuthology.orchestra.run.smithi089.stdout: "initiated_at": "2023-06-03T20:11:37.822322+0000", 2023-06-03T20:14:04.758 INFO:teuthology.orchestra.run.smithi089.stdout: "age": 146.786901387, 2023-06-03T20:14:04.758 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.15356304300000001, 2023-06-03T20:14:04.758 INFO:teuthology.orchestra.run.smithi089.stdout: "type_data": { 2023-06-03T20:14:04.758 INFO:teuthology.orchestra.run.smithi089.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-03T20:14:04.758 INFO:teuthology.orchestra.run.smithi089.stdout: "events": [ 2023-06-03T20:14:04.758 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:14:04.758 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "initiated", 2023-06-03T20:14:04.758 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:11:37.822322+0000", 2023-06-03T20:14:04.759 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0 2023-06-03T20:14:04.759 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:14:04.759 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:14:04.759 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "throttled", 2023-06-03T20:14:04.759 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:11:37.822322+0000", 2023-06-03T20:14:04.759 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0 2023-06-03T20:14:04.759 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:14:04.759 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:14:04.760 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "header_read", 2023-06-03T20:14:04.760 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:11:37.822321+0000", 2023-06-03T20:14:04.760 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 4294967295.9999995 2023-06-03T20:14:04.760 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:14:04.760 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:14:04.760 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "all_read", 2023-06-03T20:14:04.760 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:11:37.822347+0000", 2023-06-03T20:14:04.760 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 2.5534999999999999e-05 2023-06-03T20:14:04.761 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:14:04.761 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:14:04.761 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "dispatched", 2023-06-03T20:14:04.761 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:11:37.822357+0000", 2023-06-03T20:14:04.761 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 1.0017000000000001e-05 2023-06-03T20:14:04.762 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:14:04.762 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:14:04.762 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "queued_for_pg", 2023-06-03T20:14:04.762 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:11:37.822364+0000", 2023-06-03T20:14:04.762 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 6.6590000000000001e-06 2023-06-03T20:14:04.762 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:14:04.762 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:14:04.762 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "reached_pg", 2023-06-03T20:14:04.763 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:11:37.822411+0000", 2023-06-03T20:14:04.763 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 4.6859e-05 2023-06-03T20:14:04.763 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:14:04.763 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:14:04.763 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "started", 2023-06-03T20:14:04.764 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:11:37.822431+0000", 2023-06-03T20:14:04.764 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 2.0321000000000001e-05 2023-06-03T20:14:04.764 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:14:04.764 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:14:04.764 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "commit_sent", 2023-06-03T20:14:04.764 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:11:37.975843+0000", 2023-06-03T20:14:04.765 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.15341263499999999 2023-06-03T20:14:04.765 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:14:04.765 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:14:04.765 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "done", 2023-06-03T20:14:04.765 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:11:37.975885+0000", 2023-06-03T20:14:04.765 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 4.1712000000000003e-05 2023-06-03T20:14:04.765 INFO:teuthology.orchestra.run.smithi089.stdout: } 2023-06-03T20:14:04.765 INFO:teuthology.orchestra.run.smithi089.stdout: ] 2023-06-03T20:14:04.766 INFO:teuthology.orchestra.run.smithi089.stdout: } 2023-06-03T20:14:04.766 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:14:04.766 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:14:04.766 INFO:teuthology.orchestra.run.smithi089.stdout: "description": "osd_repop(client.46187.0:11086 11.e e193/189 11:761bcf85:::benchmark_data_smithi089_165412_object11085:head v 193'685, mlcod=193'682)", 2023-06-03T20:14:04.766 INFO:teuthology.orchestra.run.smithi089.stdout: "initiated_at": "2023-06-03T20:11:37.822441+0000", 2023-06-03T20:14:04.766 INFO:teuthology.orchestra.run.smithi089.stdout: "age": 146.78678270099999, 2023-06-03T20:14:04.766 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.15489956099999999, 2023-06-03T20:14:04.766 INFO:teuthology.orchestra.run.smithi089.stdout: "type_data": { 2023-06-03T20:14:04.767 INFO:teuthology.orchestra.run.smithi089.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-03T20:14:04.767 INFO:teuthology.orchestra.run.smithi089.stdout: "events": [ 2023-06-03T20:14:04.767 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:14:04.767 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "initiated", 2023-06-03T20:14:04.767 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:11:37.822441+0000", 2023-06-03T20:14:04.767 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0 2023-06-03T20:14:04.767 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:14:04.768 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:14:04.768 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "throttled", 2023-06-03T20:14:04.768 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:11:37.822441+0000", 2023-06-03T20:14:04.768 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0 2023-06-03T20:14:04.768 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:14:04.768 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:14:04.768 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "header_read", 2023-06-03T20:14:04.768 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:11:37.822440+0000", 2023-06-03T20:14:04.769 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 4294967295.9999995 2023-06-03T20:14:04.769 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:14:04.769 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:14:04.769 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "all_read", 2023-06-03T20:14:04.769 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:11:37.822487+0000", 2023-06-03T20:14:04.769 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 4.6545999999999998e-05 2023-06-03T20:14:04.769 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:14:04.770 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:14:04.770 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "dispatched", 2023-06-03T20:14:04.770 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:11:37.822495+0000", 2023-06-03T20:14:04.770 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 7.6299999999999998e-06 2023-06-03T20:14:04.770 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:14:04.770 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:14:04.770 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "queued_for_pg", 2023-06-03T20:14:04.770 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:11:37.822500+0000", 2023-06-03T20:14:04.771 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 5.75e-06 2023-06-03T20:14:04.771 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:14:04.771 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:14:04.771 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "reached_pg", 2023-06-03T20:14:04.771 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:11:37.822847+0000", 2023-06-03T20:14:04.771 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.00034713700000000002 2023-06-03T20:14:04.771 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:14:04.771 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:14:04.772 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "started", 2023-06-03T20:14:04.772 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:11:37.822861+0000", 2023-06-03T20:14:04.772 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 1.3247e-05 2023-06-03T20:14:04.772 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:14:04.772 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:14:04.772 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "commit_sent", 2023-06-03T20:14:04.772 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:11:37.977271+0000", 2023-06-03T20:14:04.772 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.15441036799999999 2023-06-03T20:14:04.773 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:14:04.773 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:14:04.773 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "done", 2023-06-03T20:14:04.773 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:11:37.977340+0000", 2023-06-03T20:14:04.773 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 6.9387999999999994e-05 2023-06-03T20:14:04.773 INFO:teuthology.orchestra.run.smithi089.stdout: } 2023-06-03T20:14:04.773 INFO:teuthology.orchestra.run.smithi089.stdout: ] 2023-06-03T20:14:04.773 INFO:teuthology.orchestra.run.smithi089.stdout: } 2023-06-03T20:14:04.774 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:14:04.774 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:14:04.774 INFO:teuthology.orchestra.run.smithi089.stdout: "description": "osd_repop(client.46187.0:11098 11.2 e193/189 11:4e5c3f33:::benchmark_data_smithi089_165412_object11097:head v 193'680, mlcod=193'677)", 2023-06-03T20:14:04.774 INFO:teuthology.orchestra.run.smithi089.stdout: "initiated_at": "2023-06-03T20:11:37.823761+0000", 2023-06-03T20:14:04.774 INFO:teuthology.orchestra.run.smithi089.stdout: "age": 146.785462927, 2023-06-03T20:14:04.774 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.15337098900000001, 2023-06-03T20:14:04.774 INFO:teuthology.orchestra.run.smithi089.stdout: "type_data": { 2023-06-03T20:14:04.775 INFO:teuthology.orchestra.run.smithi089.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-03T20:14:04.775 INFO:teuthology.orchestra.run.smithi089.stdout: "events": [ 2023-06-03T20:14:04.775 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:14:04.775 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "initiated", 2023-06-03T20:14:04.775 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:11:37.823761+0000", 2023-06-03T20:14:04.775 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0 2023-06-03T20:14:04.775 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:14:04.775 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:14:04.776 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "throttled", 2023-06-03T20:14:04.776 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:11:37.823761+0000", 2023-06-03T20:14:04.776 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0 2023-06-03T20:14:04.776 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:14:04.776 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:14:04.776 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "header_read", 2023-06-03T20:14:04.776 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:11:37.823760+0000", 2023-06-03T20:14:04.776 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 4294967295.999999 2023-06-03T20:14:04.777 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:14:04.777 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:14:04.777 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "all_read", 2023-06-03T20:14:04.777 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:11:37.823783+0000", 2023-06-03T20:14:04.777 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 2.3368999999999999e-05 2023-06-03T20:14:04.777 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:14:04.777 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:14:04.777 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "dispatched", 2023-06-03T20:14:04.778 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:11:37.974003+0000", 2023-06-03T20:14:04.778 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.150219462 2023-06-03T20:14:04.778 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:14:04.778 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:14:04.778 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "queued_for_pg", 2023-06-03T20:14:04.778 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:11:37.974008+0000", 2023-06-03T20:14:04.778 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 5.7379999999999999e-06 2023-06-03T20:14:04.778 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:14:04.779 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:14:04.779 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "reached_pg", 2023-06-03T20:14:04.779 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:11:37.974243+0000", 2023-06-03T20:14:04.779 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.000234548 2023-06-03T20:14:04.779 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:14:04.779 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:14:04.779 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "started", 2023-06-03T20:14:04.779 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:11:37.974294+0000", 2023-06-03T20:14:04.780 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 5.0646999999999998e-05 2023-06-03T20:14:04.780 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:14:04.780 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:14:04.780 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "commit_sent", 2023-06-03T20:14:04.780 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:11:37.977078+0000", 2023-06-03T20:14:04.780 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.0027841950000000002 2023-06-03T20:14:04.780 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:14:04.781 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:14:04.781 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "done", 2023-06-03T20:14:04.781 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:11:37.977132+0000", 2023-06-03T20:14:04.781 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 5.3844999999999997e-05 2023-06-03T20:14:04.781 INFO:teuthology.orchestra.run.smithi089.stdout: } 2023-06-03T20:14:04.781 INFO:teuthology.orchestra.run.smithi089.stdout: ] 2023-06-03T20:14:04.781 INFO:teuthology.orchestra.run.smithi089.stdout: } 2023-06-03T20:14:04.782 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:14:04.782 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:14:04.782 INFO:teuthology.orchestra.run.smithi089.stdout: "description": "osd_repop(client.46187.0:11099 11.9 e193/189 11:9072c63d:::benchmark_data_smithi089_165412_object11098:head v 193'683, mlcod=193'680)", 2023-06-03T20:14:04.782 INFO:teuthology.orchestra.run.smithi089.stdout: "initiated_at": "2023-06-03T20:11:37.823773+0000", 2023-06-03T20:14:04.782 INFO:teuthology.orchestra.run.smithi089.stdout: "age": 146.78545036700001, 2023-06-03T20:14:04.783 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.152133993, 2023-06-03T20:14:04.783 INFO:teuthology.orchestra.run.smithi089.stdout: "type_data": { 2023-06-03T20:14:04.783 INFO:teuthology.orchestra.run.smithi089.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-03T20:14:04.783 INFO:teuthology.orchestra.run.smithi089.stdout: "events": [ 2023-06-03T20:14:04.783 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:14:04.783 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "initiated", 2023-06-03T20:14:04.784 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:11:37.823773+0000", 2023-06-03T20:14:04.784 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0 2023-06-03T20:14:04.784 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:14:04.784 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:14:04.784 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "throttled", 2023-06-03T20:14:04.785 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:11:37.823773+0000", 2023-06-03T20:14:04.785 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0 2023-06-03T20:14:04.785 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:14:04.785 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:14:04.785 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "header_read", 2023-06-03T20:14:04.785 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:11:37.823772+0000", 2023-06-03T20:14:04.785 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 4294967295.999999 2023-06-03T20:14:04.785 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:14:04.786 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:14:04.786 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "all_read", 2023-06-03T20:14:04.786 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:11:37.823795+0000", 2023-06-03T20:14:04.786 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 2.2830999999999999e-05 2023-06-03T20:14:04.786 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:14:04.786 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:14:04.786 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "dispatched", 2023-06-03T20:14:04.786 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:11:37.973936+0000", 2023-06-03T20:14:04.787 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.15014085899999999 2023-06-03T20:14:04.787 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:14:04.787 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:14:04.787 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "queued_for_pg", 2023-06-03T20:14:04.787 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:11:37.973950+0000", 2023-06-03T20:14:04.787 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 1.4028e-05 2023-06-03T20:14:04.787 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:14:04.788 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:14:04.788 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "reached_pg", 2023-06-03T20:14:04.788 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:11:37.974035+0000", 2023-06-03T20:14:04.788 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 8.5295000000000002e-05 2023-06-03T20:14:04.788 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:14:04.788 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:14:04.788 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "started", 2023-06-03T20:14:04.788 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:11:37.974094+0000", 2023-06-03T20:14:04.789 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 5.9119000000000001e-05 2023-06-03T20:14:04.789 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:14:04.789 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:14:04.789 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "commit_sent", 2023-06-03T20:14:04.789 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:11:37.975889+0000", 2023-06-03T20:14:04.789 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.0017949140000000001 2023-06-03T20:14:04.789 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:14:04.789 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:14:04.790 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "done", 2023-06-03T20:14:04.790 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:11:37.975907+0000", 2023-06-03T20:14:04.790 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 1.7893e-05 2023-06-03T20:14:04.790 INFO:teuthology.orchestra.run.smithi089.stdout: } 2023-06-03T20:14:04.790 INFO:teuthology.orchestra.run.smithi089.stdout: ] 2023-06-03T20:14:04.790 INFO:teuthology.orchestra.run.smithi089.stdout: } 2023-06-03T20:14:04.790 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:14:04.790 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:14:04.791 INFO:teuthology.orchestra.run.smithi089.stdout: "description": "osd_repop(client.46187.0:11096 11.7 e193/189 11:ec330442:::benchmark_data_smithi089_165412_object11095:head v 193'671, mlcod=193'669)", 2023-06-03T20:14:04.791 INFO:teuthology.orchestra.run.smithi089.stdout: "initiated_at": "2023-06-03T20:11:37.823830+0000", 2023-06-03T20:14:04.791 INFO:teuthology.orchestra.run.smithi089.stdout: "age": 146.785393497, 2023-06-03T20:14:04.791 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.15156946399999999, 2023-06-03T20:14:04.791 INFO:teuthology.orchestra.run.smithi089.stdout: "type_data": { 2023-06-03T20:14:04.791 INFO:teuthology.orchestra.run.smithi089.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-03T20:14:04.791 INFO:teuthology.orchestra.run.smithi089.stdout: "events": [ 2023-06-03T20:14:04.792 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:14:04.792 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "initiated", 2023-06-03T20:14:04.792 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:11:37.823830+0000", 2023-06-03T20:14:04.792 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0 2023-06-03T20:14:04.792 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:14:04.792 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:14:04.792 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "throttled", 2023-06-03T20:14:04.792 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:11:37.823830+0000", 2023-06-03T20:14:04.793 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0 2023-06-03T20:14:04.793 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:14:04.793 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:14:04.793 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "header_read", 2023-06-03T20:14:04.793 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:11:37.823829+0000", 2023-06-03T20:14:04.793 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 4294967295.9999995 2023-06-03T20:14:04.793 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:14:04.793 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:14:04.794 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "all_read", 2023-06-03T20:14:04.794 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:11:37.823888+0000", 2023-06-03T20:14:04.794 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 5.8743999999999999e-05 2023-06-03T20:14:04.794 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:14:04.794 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:14:04.794 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "dispatched", 2023-06-03T20:14:04.795 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:11:37.973977+0000", 2023-06-03T20:14:04.795 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.15008929200000001 2023-06-03T20:14:04.795 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:14:04.796 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:14:04.796 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "queued_for_pg", 2023-06-03T20:14:04.796 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:11:37.973985+0000", 2023-06-03T20:14:04.796 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 7.7659999999999998e-06 2023-06-03T20:14:04.796 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:14:04.797 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:14:04.797 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "reached_pg", 2023-06-03T20:14:04.797 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:11:37.974053+0000", 2023-06-03T20:14:04.797 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 6.7748000000000003e-05 2023-06-03T20:14:04.797 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:14:04.797 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:14:04.797 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "started", 2023-06-03T20:14:04.798 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:11:37.974077+0000", 2023-06-03T20:14:04.798 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 2.4453000000000001e-05 2023-06-03T20:14:04.798 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:14:04.798 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:14:04.798 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "commit_sent", 2023-06-03T20:14:04.798 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:11:37.975325+0000", 2023-06-03T20:14:04.798 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.0012478369999999999 2023-06-03T20:14:04.799 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:14:04.799 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:14:04.799 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "done", 2023-06-03T20:14:04.799 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:11:37.975400+0000", 2023-06-03T20:14:04.799 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 7.4270000000000006e-05 2023-06-03T20:14:04.799 INFO:teuthology.orchestra.run.smithi089.stdout: } 2023-06-03T20:14:04.799 INFO:teuthology.orchestra.run.smithi089.stdout: ] 2023-06-03T20:14:04.799 INFO:teuthology.orchestra.run.smithi089.stdout: } 2023-06-03T20:14:04.800 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:14:04.800 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:14:04.800 INFO:teuthology.orchestra.run.smithi089.stdout: "description": "osd_op(client.46187.0:71105 11.6 11:64c546c9:::benchmark_data_smithi089_165412_object71104:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e197)", 2023-06-03T20:14:04.800 INFO:teuthology.orchestra.run.smithi089.stdout: "initiated_at": "2023-06-03T20:12:19.898029+0000", 2023-06-03T20:14:04.800 INFO:teuthology.orchestra.run.smithi089.stdout: "age": 104.71119407099999, 2023-06-03T20:14:04.800 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.22681279600000001, 2023-06-03T20:14:04.801 INFO:teuthology.orchestra.run.smithi089.stdout: "type_data": { 2023-06-03T20:14:04.801 INFO:teuthology.orchestra.run.smithi089.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-03T20:14:04.801 INFO:teuthology.orchestra.run.smithi089.stdout: "client_info": { 2023-06-03T20:14:04.801 INFO:teuthology.orchestra.run.smithi089.stdout: "client": "client.46187", 2023-06-03T20:14:04.801 INFO:teuthology.orchestra.run.smithi089.stdout: "client_addr": "172.21.15.89:0/880684210", 2023-06-03T20:14:04.801 INFO:teuthology.orchestra.run.smithi089.stdout: "tid": 71105 2023-06-03T20:14:04.802 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:14:04.802 INFO:teuthology.orchestra.run.smithi089.stdout: "events": [ 2023-06-03T20:14:04.802 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:14:04.802 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "initiated", 2023-06-03T20:14:04.802 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:12:19.898029+0000", 2023-06-03T20:14:04.802 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0 2023-06-03T20:14:04.802 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:14:04.802 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:14:04.803 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "throttled", 2023-06-03T20:14:04.803 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:12:19.898029+0000", 2023-06-03T20:14:04.803 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0 2023-06-03T20:14:04.803 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:14:04.803 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:14:04.803 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "header_read", 2023-06-03T20:14:04.803 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:12:19.898028+0000", 2023-06-03T20:14:04.803 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 4294967295.9999981 2023-06-03T20:14:04.804 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:14:04.804 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:14:04.804 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "all_read", 2023-06-03T20:14:04.804 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:12:19.898056+0000", 2023-06-03T20:14:04.804 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 2.7883999999999999e-05 2023-06-03T20:14:04.804 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:14:04.804 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:14:04.805 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "dispatched", 2023-06-03T20:14:04.805 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:12:19.898070+0000", 2023-06-03T20:14:04.805 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 1.4498000000000001e-05 2023-06-03T20:14:04.805 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:14:04.805 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:14:04.805 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "queued_for_pg", 2023-06-03T20:14:04.805 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:12:19.898079+0000", 2023-06-03T20:14:04.805 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 9.1759999999999999e-06 2023-06-03T20:14:04.806 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:14:04.806 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:14:04.806 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "reached_pg", 2023-06-03T20:14:04.806 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:12:19.900335+0000", 2023-06-03T20:14:04.806 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.0022557829999999999 2023-06-03T20:14:04.806 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:14:04.806 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:14:04.806 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "started", 2023-06-03T20:14:04.807 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:12:19.900492+0000", 2023-06-03T20:14:04.807 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.000157412 2023-06-03T20:14:04.807 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:14:04.807 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:14:04.807 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "waiting for subops from 2", 2023-06-03T20:14:04.807 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:12:19.900646+0000", 2023-06-03T20:14:04.807 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.00015353900000000001 2023-06-03T20:14:04.807 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:14:04.808 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:14:04.808 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "op_commit", 2023-06-03T20:14:04.808 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:12:19.903638+0000", 2023-06-03T20:14:04.808 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.002992523 2023-06-03T20:14:04.808 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:14:04.808 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:14:04.808 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:14:04.808 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:12:20.124679+0000", 2023-06-03T20:14:04.809 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.22104043600000001 2023-06-03T20:14:04.809 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:14:04.809 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:14:04.809 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "commit_sent", 2023-06-03T20:14:04.809 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:12:20.124756+0000", 2023-06-03T20:14:04.809 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 7.7199000000000004e-05 2023-06-03T20:14:04.810 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:14:04.810 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:14:04.810 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "done", 2023-06-03T20:14:04.810 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:12:20.124842+0000", 2023-06-03T20:14:04.811 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 8.6172000000000001e-05 2023-06-03T20:14:04.811 INFO:teuthology.orchestra.run.smithi089.stdout: } 2023-06-03T20:14:04.811 INFO:teuthology.orchestra.run.smithi089.stdout: ] 2023-06-03T20:14:04.811 INFO:teuthology.orchestra.run.smithi089.stdout: } 2023-06-03T20:14:04.811 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:14:04.811 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:14:04.812 INFO:teuthology.orchestra.run.smithi089.stdout: "description": "osd_op(client.46187.0:71133 11.6 11:6f3e72f6:::benchmark_data_smithi089_165412_object71132:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e197)", 2023-06-03T20:14:04.812 INFO:teuthology.orchestra.run.smithi089.stdout: "initiated_at": "2023-06-03T20:12:19.925462+0000", 2023-06-03T20:14:04.812 INFO:teuthology.orchestra.run.smithi089.stdout: "age": 104.683761878, 2023-06-03T20:14:04.812 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.19957854999999999, 2023-06-03T20:14:04.813 INFO:teuthology.orchestra.run.smithi089.stdout: "type_data": { 2023-06-03T20:14:04.813 INFO:teuthology.orchestra.run.smithi089.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-03T20:14:04.813 INFO:teuthology.orchestra.run.smithi089.stdout: "client_info": { 2023-06-03T20:14:04.813 INFO:teuthology.orchestra.run.smithi089.stdout: "client": "client.46187", 2023-06-03T20:14:04.813 INFO:teuthology.orchestra.run.smithi089.stdout: "client_addr": "172.21.15.89:0/880684210", 2023-06-03T20:14:04.813 INFO:teuthology.orchestra.run.smithi089.stdout: "tid": 71133 2023-06-03T20:14:04.813 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:14:04.813 INFO:teuthology.orchestra.run.smithi089.stdout: "events": [ 2023-06-03T20:14:04.814 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:14:04.814 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "initiated", 2023-06-03T20:14:04.814 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:12:19.925462+0000", 2023-06-03T20:14:04.814 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0 2023-06-03T20:14:04.814 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:14:04.814 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:14:04.814 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "throttled", 2023-06-03T20:14:04.814 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:12:19.925462+0000", 2023-06-03T20:14:04.815 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0 2023-06-03T20:14:04.815 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:14:04.815 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:14:04.815 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "header_read", 2023-06-03T20:14:04.815 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:12:19.925460+0000", 2023-06-03T20:14:04.815 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 4294967295.9999981 2023-06-03T20:14:04.815 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:14:04.816 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:14:04.816 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "all_read", 2023-06-03T20:14:04.816 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:12:19.925486+0000", 2023-06-03T20:14:04.816 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 2.5829999999999998e-05 2023-06-03T20:14:04.816 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:14:04.816 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:14:04.816 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "dispatched", 2023-06-03T20:14:04.816 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:12:19.925500+0000", 2023-06-03T20:14:04.817 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 1.4214e-05 2023-06-03T20:14:04.817 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:14:04.817 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:14:04.817 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "queued_for_pg", 2023-06-03T20:14:04.817 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:12:19.925508+0000", 2023-06-03T20:14:04.817 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 7.9179999999999994e-06 2023-06-03T20:14:04.817 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:14:04.817 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:14:04.818 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "reached_pg", 2023-06-03T20:14:04.818 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:12:19.925569+0000", 2023-06-03T20:14:04.818 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 6.1140000000000001e-05 2023-06-03T20:14:04.818 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:14:04.818 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:14:04.819 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "started", 2023-06-03T20:14:04.819 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:12:19.925735+0000", 2023-06-03T20:14:04.819 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.00016555300000000001 2023-06-03T20:14:04.819 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:14:04.819 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:14:04.819 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "waiting for subops from 2", 2023-06-03T20:14:04.819 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:12:19.925893+0000", 2023-06-03T20:14:04.820 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.00015806800000000001 2023-06-03T20:14:04.820 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:14:04.820 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:14:04.820 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "op_commit", 2023-06-03T20:14:04.820 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:12:19.928451+0000", 2023-06-03T20:14:04.820 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.00255887 2023-06-03T20:14:04.820 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:14:04.820 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:14:04.821 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:14:04.821 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:12:20.124889+0000", 2023-06-03T20:14:04.821 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.196437531 2023-06-03T20:14:04.821 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:14:04.821 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:14:04.821 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "commit_sent", 2023-06-03T20:14:04.821 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:12:20.124939+0000", 2023-06-03T20:14:04.822 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 4.9545000000000001e-05 2023-06-03T20:14:04.822 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:14:04.822 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:14:04.822 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "done", 2023-06-03T20:14:04.822 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:12:20.125040+0000", 2023-06-03T20:14:04.822 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.000101711 2023-06-03T20:14:04.822 INFO:teuthology.orchestra.run.smithi089.stdout: } 2023-06-03T20:14:04.823 INFO:teuthology.orchestra.run.smithi089.stdout: ] 2023-06-03T20:14:04.823 INFO:teuthology.orchestra.run.smithi089.stdout: } 2023-06-03T20:14:04.823 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:14:04.823 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:14:04.823 INFO:teuthology.orchestra.run.smithi089.stdout: "description": "osd_op(client.46187.0:71137 11.6 11:634eac00:::benchmark_data_smithi089_165412_object71136:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e197)", 2023-06-03T20:14:04.823 INFO:teuthology.orchestra.run.smithi089.stdout: "initiated_at": "2023-06-03T20:12:19.934812+0000", 2023-06-03T20:14:04.823 INFO:teuthology.orchestra.run.smithi089.stdout: "age": 104.67441110999999, 2023-06-03T20:14:04.823 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.19735046000000001, 2023-06-03T20:14:04.824 INFO:teuthology.orchestra.run.smithi089.stdout: "type_data": { 2023-06-03T20:14:04.824 INFO:teuthology.orchestra.run.smithi089.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-03T20:14:04.824 INFO:teuthology.orchestra.run.smithi089.stdout: "client_info": { 2023-06-03T20:14:04.824 INFO:teuthology.orchestra.run.smithi089.stdout: "client": "client.46187", 2023-06-03T20:14:04.824 INFO:teuthology.orchestra.run.smithi089.stdout: "client_addr": "172.21.15.89:0/880684210", 2023-06-03T20:14:04.824 INFO:teuthology.orchestra.run.smithi089.stdout: "tid": 71137 2023-06-03T20:14:04.824 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:14:04.824 INFO:teuthology.orchestra.run.smithi089.stdout: "events": [ 2023-06-03T20:14:04.825 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:14:04.825 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "initiated", 2023-06-03T20:14:04.825 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:12:19.934812+0000", 2023-06-03T20:14:04.825 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0 2023-06-03T20:14:04.825 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:14:04.825 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:14:04.825 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "throttled", 2023-06-03T20:14:04.825 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:12:19.934812+0000", 2023-06-03T20:14:04.826 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0 2023-06-03T20:14:04.826 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:14:04.826 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:14:04.826 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "header_read", 2023-06-03T20:14:04.826 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:12:19.934810+0000", 2023-06-03T20:14:04.826 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 4294967295.9999981 2023-06-03T20:14:04.826 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:14:04.826 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:14:04.827 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "all_read", 2023-06-03T20:14:04.827 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:12:19.934844+0000", 2023-06-03T20:14:04.827 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 3.3302000000000001e-05 2023-06-03T20:14:04.827 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:14:04.827 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:14:04.827 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "dispatched", 2023-06-03T20:14:04.827 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:12:19.934859+0000", 2023-06-03T20:14:04.827 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 1.5461999999999999e-05 2023-06-03T20:14:04.828 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:14:04.828 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:14:04.828 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "queued_for_pg", 2023-06-03T20:14:04.828 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:12:19.934867+0000", 2023-06-03T20:14:04.828 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 8.0059999999999992e-06 2023-06-03T20:14:04.828 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:14:04.828 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:14:04.829 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "reached_pg", 2023-06-03T20:14:04.829 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:12:19.934917+0000", 2023-06-03T20:14:04.829 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 5.0241000000000002e-05 2023-06-03T20:14:04.829 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:14:04.829 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:14:04.829 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "started", 2023-06-03T20:14:04.829 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:12:19.935057+0000", 2023-06-03T20:14:04.829 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.00013991399999999999 2023-06-03T20:14:04.830 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:14:04.830 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:14:04.830 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "waiting for subops from 2", 2023-06-03T20:14:04.830 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:12:19.935199+0000", 2023-06-03T20:14:04.830 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.00014178899999999999 2023-06-03T20:14:04.830 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:14:04.830 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:14:04.830 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "op_commit", 2023-06-03T20:14:04.831 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:12:19.937569+0000", 2023-06-03T20:14:04.831 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.0023702039999999999 2023-06-03T20:14:04.831 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:14:04.831 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:14:04.831 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:14:04.831 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:12:20.132009+0000", 2023-06-03T20:14:04.831 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.19443953899999999 2023-06-03T20:14:04.831 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:14:04.832 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:14:04.832 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "commit_sent", 2023-06-03T20:14:04.832 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:12:20.132065+0000", 2023-06-03T20:14:04.832 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 5.6209999999999999e-05 2023-06-03T20:14:04.832 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:14:04.832 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:14:04.832 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "done", 2023-06-03T20:14:04.832 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:12:20.132163+0000", 2023-06-03T20:14:04.833 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 9.7781000000000005e-05 2023-06-03T20:14:04.833 INFO:teuthology.orchestra.run.smithi089.stdout: } 2023-06-03T20:14:04.833 INFO:teuthology.orchestra.run.smithi089.stdout: ] 2023-06-03T20:14:04.833 INFO:teuthology.orchestra.run.smithi089.stdout: } 2023-06-03T20:14:04.833 INFO:teuthology.orchestra.run.smithi089.stdout: } 2023-06-03T20:14:04.833 INFO:teuthology.orchestra.run.smithi089.stdout: ] 2023-06-03T20:14:04.833 INFO:teuthology.orchestra.run.smithi089.stdout:} 2023-06-03T20:14:04.834 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:14:04.878 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:14:04.884 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:14:04.885 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:14:04.986 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:14:05.023 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:14:05.027 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:14:05.028 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:14:05.129 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:14:05.174 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:14:05.179 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:14:05.180 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:14:05.281 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:14:05.317 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:14:05.320 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:14:05.321 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:14:05.422 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:14:05.451 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:14:05.455 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:14:05.456 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:14:05.557 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:14:05.596 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:14:05.601 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:14:05.602 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:14:05.703 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:14:05.712 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:14:05 smithi089 bash[75068]: audit 1685823244.6143148 mon.a (mon.0) 1082 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:14:05.713 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:14:05 smithi089 bash[75068]: audit 1685823244.6357987 mon.a (mon.0) 1083 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:14:05.713 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:14:05 smithi089 bash[75068]: cluster 1685823244.7419844 mgr.y (mgr.34104) 1141 : cluster 0 pgmap v1115: 89 pgs: 9 active+clean+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 13 active+undersized+degraded, 20 active+undersized, 44 active+clean; 3.9 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 701 op/s; 30642/189585 objects degraded (16.163%); 4838/189585 objects misplaced (2.552%); 2.9 MiB/s, 45 objects/s recovering 2023-06-03T20:14:05.713 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:14:05 smithi089 bash[75068]: audit 1685823245.2173436 mon.c (mon.2) 521 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-03T20:14:05.714 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:14:05 smithi089 bash[75068]: audit 1685823245.2212417 mon.c (mon.2) 522 : audit 1 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-03T20:14:05.714 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:14:05 smithi089 bash[75068]: audit 1685823245.2284698 mon.a (mon.0) 1084 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:14:05.714 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:14:05 smithi089 bash[78452]: audit 1685823244.6143148 mon.a (mon.0) 1082 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:14:05.715 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:14:05 smithi089 bash[78452]: audit 1685823244.6357987 mon.a (mon.0) 1083 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:14:05.715 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:14:05 smithi089 bash[78452]: cluster 1685823244.7419844 mgr.y (mgr.34104) 1141 : cluster 0 pgmap v1115: 89 pgs: 9 active+clean+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 13 active+undersized+degraded, 20 active+undersized, 44 active+clean; 3.9 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 701 op/s; 30642/189585 objects degraded (16.163%); 4838/189585 objects misplaced (2.552%); 2.9 MiB/s, 45 objects/s recovering 2023-06-03T20:14:05.715 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:14:05 smithi089 bash[78452]: audit 1685823245.2173436 mon.c (mon.2) 521 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-03T20:14:05.716 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:14:05 smithi089 bash[78452]: audit 1685823245.2212417 mon.c (mon.2) 522 : audit 1 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-03T20:14:05.716 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:14:05 smithi089 bash[78452]: audit 1685823245.2284698 mon.a (mon.0) 1084 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:14:05.736 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:14:05.741 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:14:05.742 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:14:05.833 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:14:05 smithi164 bash[59462]: audit 1685823244.6143148 mon.a (mon.0) 1082 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:14:05.833 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:14:05 smithi164 bash[59462]: audit 1685823244.6357987 mon.a (mon.0) 1083 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:14:05.834 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:14:05 smithi164 bash[59462]: cluster 1685823244.7419844 mgr.y (mgr.34104) 1141 : cluster 0 pgmap v1115: 89 pgs: 9 active+clean+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 13 active+undersized+degraded, 20 active+undersized, 44 active+clean; 3.9 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 701 op/s; 30642/189585 objects degraded (16.163%); 4838/189585 objects misplaced (2.552%); 2.9 MiB/s, 45 objects/s recovering 2023-06-03T20:14:05.834 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:14:05 smithi164 bash[59462]: audit 1685823245.2173436 mon.c (mon.2) 521 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-03T20:14:05.834 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:14:05 smithi164 bash[59462]: audit 1685823245.2212417 mon.c (mon.2) 522 : audit 1 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-03T20:14:05.834 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:14:05 smithi164 bash[59462]: audit 1685823245.2284698 mon.a (mon.0) 1084 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:14:05.844 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:14:05.874 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:14:05.877 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:14:05.877 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:14:05.978 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:14:06.005 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:14:06.008 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:14:06.009 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:14:06.110 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:14:06.147 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:14:06.152 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:14:06.152 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:14:06.207 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph daemon osd.1 dump_ops_in_flight 2023-06-03T20:14:06.253 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:14:06.283 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:14:06.285 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:14:06.285 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:14:06.386 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:14:06.438 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:14:06.445 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:14:06.446 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:14:06.528 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 20:14:06 smithi089 bash[71163]: ::ffff:172.21.15.164 - - [03/Jun/2023:20:14:06] "GET /metrics HTTP/1.1" 200 228916 "" "Prometheus/2.33.4" 2023-06-03T20:14:06.547 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:14:06.587 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:14:06.591 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:14:06.592 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:14:06.693 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:14:06.728 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:14:06.729 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:14:06.729 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:14:06.830 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:14:06.866 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:14:06.869 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:14:06.870 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:14:06.971 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:14:07.008 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:14:07.013 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:14:07.013 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:14:07.114 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:14:07.149 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:14:07.151 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:14:07.152 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:14:07.252 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:14:07.284 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:14:07.286 INFO:tasks.thrashosds.thrasher:in_osds: [0, 2, 3, 4, 5, 6, 7] out_osds: [1] dead_osds: [4] live_osds: [0, 1, 2, 3, 5, 6, 7] 2023-06-03T20:14:07.286 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-06-03T20:14:07.287 INFO:tasks.thrashosds.thrasher:Growing pool device_health_metrics 2023-06-03T20:14:07.287 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph pg dump --format=json 2023-06-03T20:14:07.288 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:14:07.289 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:14:07.390 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:14:07.429 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:14:07.433 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:14:07.434 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:14:07.535 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:14:07.567 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:14:07.569 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:14:07.570 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:14:07.671 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:14:07.704 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:14:07.707 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:14:07.708 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:14:07.802 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:14:07 smithi164 bash[59462]: cluster 1685823246.7426968 mgr.y (mgr.34104) 1142 : cluster 0 pgmap v1116: 89 pgs: 9 active+clean+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 13 active+undersized+degraded, 20 active+undersized, 44 active+clean; 3.8 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 690 op/s; 29999/187056 objects degraded (16.037%); 4838/187056 objects misplaced (2.586%); 2.9 MiB/s, 45 objects/s recovering 2023-06-03T20:14:07.802 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:14:07 smithi164 bash[59462]: audit 1685823247.1245031 mon.c (mon.2) 523 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T20:14:07.803 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:14:07 smithi164 bash[59462]: cluster 1685823247.7550046 mon.a (mon.0) 1085 : cluster 3 Health check update: Degraded data redundancy: 29999/187056 objects degraded (16.037%), 16 pgs degraded, 36 pgs undersized (PG_DEGRADED) 2023-06-03T20:14:07.809 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:14:07.850 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:14:07.853 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:14:07.854 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:14:07.955 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:14:07.981 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:14:07.986 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:14:07.986 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:14:08.028 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:14:07 smithi089 bash[75068]: cluster 1685823246.7426968 mgr.y (mgr.34104) 1142 : cluster 0 pgmap v1116: 89 pgs: 9 active+clean+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 13 active+undersized+degraded, 20 active+undersized, 44 active+clean; 3.8 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 690 op/s; 29999/187056 objects degraded (16.037%); 4838/187056 objects misplaced (2.586%); 2.9 MiB/s, 45 objects/s recovering 2023-06-03T20:14:08.029 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:14:07 smithi089 bash[75068]: audit 1685823247.1245031 mon.c (mon.2) 523 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T20:14:08.029 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:14:07 smithi089 bash[75068]: cluster 1685823247.7550046 mon.a (mon.0) 1085 : cluster 3 Health check update: Degraded data redundancy: 29999/187056 objects degraded (16.037%), 16 pgs degraded, 36 pgs undersized (PG_DEGRADED) 2023-06-03T20:14:08.029 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:14:07 smithi089 bash[78452]: cluster 1685823246.7426968 mgr.y (mgr.34104) 1142 : cluster 0 pgmap v1116: 89 pgs: 9 active+clean+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 13 active+undersized+degraded, 20 active+undersized, 44 active+clean; 3.8 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 690 op/s; 29999/187056 objects degraded (16.037%); 4838/187056 objects misplaced (2.586%); 2.9 MiB/s, 45 objects/s recovering 2023-06-03T20:14:08.030 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:14:07 smithi089 bash[78452]: audit 1685823247.1245031 mon.c (mon.2) 523 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T20:14:08.030 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:14:07 smithi089 bash[78452]: cluster 1685823247.7550046 mon.a (mon.0) 1085 : cluster 3 Health check update: Degraded data redundancy: 29999/187056 objects degraded (16.037%), 16 pgs degraded, 36 pgs undersized (PG_DEGRADED) 2023-06-03T20:14:08.087 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:14:08.125 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:14:08.133 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:14:08.133 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:14:08.234 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:14:08.271 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:14:08.278 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:14:08.278 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:14:08.379 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:14:08.432 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:14:08.434 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:14:08.435 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:14:08.536 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:14:08.577 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:14:08.582 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:14:08.583 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:14:08.684 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:14:08.720 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:14:08.726 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:14:08.727 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:14:08.828 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:14:08.876 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:14:08.880 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:14:08.881 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:14:08.982 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:14:09.015 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:14:09.018 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:14:09.019 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:14:09.120 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:14:09.149 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:14:09.153 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:14:09.153 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:14:09.254 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:14:09.299 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:14:09.301 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:14:09.302 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:14:09.403 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:14:09.431 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:14:09.435 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:14:09.435 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:14:09.536 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:14:09.571 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:14:09.580 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:14:09.581 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:14:09.681 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:14:09.716 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:14:09.718 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:14:09.718 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:14:09.804 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/mon.a/config 2023-06-03T20:14:09.805 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/mon.a/config 2023-06-03T20:14:09.819 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:14:09.856 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:14:09.861 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:14:09.862 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:14:09.962 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:14:10.000 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:14:10.005 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:14:10.006 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:14:10.083 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:14:09 smithi164 bash[59462]: cluster 1685823248.7435572 mgr.y (mgr.34104) 1143 : cluster 0 pgmap v1117: 89 pgs: 9 active+clean+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 13 active+undersized+degraded, 20 active+undersized, 44 active+clean; 3.7 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 854 op/s; 29771/179964 objects degraded (16.543%); 4391/179964 objects misplaced (2.440%); 4.0 MiB/s, 64 objects/s recovering 2023-06-03T20:14:10.107 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:14:10.116 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:14:09 smithi089 bash[78452]: cluster 1685823248.7435572 mgr.y (mgr.34104) 1143 : cluster 0 pgmap v1117: 89 pgs: 9 active+clean+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 13 active+undersized+degraded, 20 active+undersized, 44 active+clean; 3.7 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 854 op/s; 29771/179964 objects degraded (16.543%); 4391/179964 objects misplaced (2.440%); 4.0 MiB/s, 64 objects/s recovering 2023-06-03T20:14:10.116 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:14:09 smithi089 bash[75068]: cluster 1685823248.7435572 mgr.y (mgr.34104) 1143 : cluster 0 pgmap v1117: 89 pgs: 9 active+clean+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 13 active+undersized+degraded, 20 active+undersized, 44 active+clean; 3.7 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 854 op/s; 29771/179964 objects degraded (16.543%); 4391/179964 objects misplaced (2.440%); 4.0 MiB/s, 64 objects/s recovering 2023-06-03T20:14:10.143 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:14:10.148 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:14:10.148 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:14:10.249 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:14:10.286 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:14:10.288 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:14:10.289 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:14:10.390 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:14:10.435 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:14:10.442 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:14:10.443 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:14:10.543 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:14:10.584 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:14:10.587 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:14:10.588 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:14:10.689 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:14:10.728 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:14:10.731 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:14:10.731 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:14:10.828 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/mon.a/config 2023-06-03T20:14:10.832 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:14:10.871 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:14:10.873 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:14:10.874 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:14:10.975 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:14:11.012 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:14:11.018 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:14:11.019 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:14:11.120 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:14:11.153 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:14:11.157 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:14:11.158 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:14:11.259 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:14:11.293 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:14:11.298 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:14:11.299 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:14:11.400 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:14:11.436 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:14:11.439 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:14:11.440 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:14:11.541 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:14:11.577 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:14:11.584 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:14:11.585 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:14:11.686 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:14:11.720 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:14:11.725 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:14:11.726 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:14:11.815 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:14:11 smithi164 bash[59462]: cluster 1685823250.7446778 mgr.y (mgr.34104) 1144 : cluster 0 pgmap v1118: 89 pgs: 9 active+clean+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 13 active+undersized+degraded, 20 active+undersized, 44 active+clean; 3.6 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 658 op/s; 29087/177228 objects degraded (16.412%); 4391/177228 objects misplaced (2.478%); 2.6 MiB/s, 41 objects/s recovering 2023-06-03T20:14:11.827 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:14:11.857 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:14:11.859 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:14:11.860 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:14:11.864 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/mon.a/config 2023-06-03T20:14:11.961 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:14:11.993 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:14:11.995 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:14:11.996 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:14:12.097 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:14:12.133 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:14:12.135 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:14:12.136 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:14:12.237 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:14:12.247 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:14:11 smithi089 bash[78452]: cluster 1685823250.7446778 mgr.y (mgr.34104) 1144 : cluster 0 pgmap v1118: 89 pgs: 9 active+clean+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 13 active+undersized+degraded, 20 active+undersized, 44 active+clean; 3.6 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 658 op/s; 29087/177228 objects degraded (16.412%); 4391/177228 objects misplaced (2.478%); 2.6 MiB/s, 41 objects/s recovering 2023-06-03T20:14:12.247 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:14:11 smithi089 bash[75068]: cluster 1685823250.7446778 mgr.y (mgr.34104) 1144 : cluster 0 pgmap v1118: 89 pgs: 9 active+clean+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 13 active+undersized+degraded, 20 active+undersized, 44 active+clean; 3.6 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 658 op/s; 29087/177228 objects degraded (16.412%); 4391/177228 objects misplaced (2.478%); 2.6 MiB/s, 41 objects/s recovering 2023-06-03T20:14:12.280 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:14:12.284 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:14:12.285 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:14:12.386 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:14:12.428 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:14:12.431 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:14:12.431 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:14:12.532 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:14:12.566 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:14:12.575 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:14:12.576 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:14:12.677 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:14:12.717 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:14:12.719 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:14:12.720 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:14:12.821 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:14:12.850 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:14:12.852 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:14:12.853 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:14:12.903 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:14:12 smithi089 bash[75068]: cluster 1685823252.756172 mon.a (mon.0) 1086 : cluster 3 Health check update: Degraded data redundancy: 29087/177228 objects degraded (16.412%), 16 pgs degraded, 36 pgs undersized (PG_DEGRADED) 2023-06-03T20:14:12.954 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:14:12.989 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:14:12.999 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:14:12.999 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:14:13.083 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:14:12 smithi164 bash[59462]: cluster 1685823252.756172 mon.a (mon.0) 1086 : cluster 3 Health check update: Degraded data redundancy: 29087/177228 objects degraded (16.412%), 16 pgs degraded, 36 pgs undersized (PG_DEGRADED) 2023-06-03T20:14:13.101 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:14:13.135 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:14:13.137 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:14:13.137 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:14:13.238 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:14:13.245 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:14:12 smithi089 bash[78452]: cluster 1685823252.756172 mon.a (mon.0) 1086 : cluster 3 Health check update: Degraded data redundancy: 29087/177228 objects degraded (16.412%), 16 pgs degraded, 36 pgs undersized (PG_DEGRADED) 2023-06-03T20:14:13.273 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:14:13.277 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:14:13.277 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:14:13.378 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:14:13.413 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:14:13.415 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:14:13.416 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:14:13.517 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:14:13.578 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:14:13.582 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:14:13.582 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:14:13.683 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:14:13.724 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:14:13.727 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:14:13.727 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:14:13.828 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:14:13.857 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:14:13.862 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:14:13.862 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:14:13.963 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:14:14.000 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:14:14.004 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:14:14.005 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:14:14.083 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:14:13 smithi164 bash[59462]: cluster 1685823252.7455468 mgr.y (mgr.34104) 1145 : cluster 0 pgmap v1119: 89 pgs: 9 active+clean+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 13 active+undersized+degraded, 20 active+undersized, 44 active+clean; 3.6 GiB data, 13 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 605 op/s; 28879/176604 objects degraded (16.352%); 4391/176604 objects misplaced (2.486%); 2.6 MiB/s, 41 objects/s recovering 2023-06-03T20:14:14.106 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:14:14.114 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:14:13 smithi089 bash[75068]: cluster 1685823252.7455468 mgr.y (mgr.34104) 1145 : cluster 0 pgmap v1119: 89 pgs: 9 active+clean+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 13 active+undersized+degraded, 20 active+undersized, 44 active+clean; 3.6 GiB data, 13 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 605 op/s; 28879/176604 objects degraded (16.352%); 4391/176604 objects misplaced (2.486%); 2.6 MiB/s, 41 objects/s recovering 2023-06-03T20:14:14.115 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:14:13 smithi089 bash[78452]: cluster 1685823252.7455468 mgr.y (mgr.34104) 1145 : cluster 0 pgmap v1119: 89 pgs: 9 active+clean+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 13 active+undersized+degraded, 20 active+undersized, 44 active+clean; 3.6 GiB data, 13 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 605 op/s; 28879/176604 objects degraded (16.352%); 4391/176604 objects misplaced (2.486%); 2.6 MiB/s, 41 objects/s recovering 2023-06-03T20:14:14.134 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:14:14.140 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:14:14.141 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:14:14.242 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:14:14.270 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:14:14.273 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:14:14.273 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:14:14.374 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:14:14.403 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:14:14.408 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:14:14.409 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:14:14.509 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:14:14.549 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:14:14.550 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:14:14.551 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:14:14.652 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:14:14.685 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:14:14.689 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:14:14.689 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:14:14.790 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:14:14.842 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:14:14.846 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:14:14.847 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:14:14.948 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:14:14.979 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:14:14.981 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:14:14.982 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:14:15.083 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:14:15.126 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:14:15.129 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:14:15.130 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:14:15.231 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:14:15.260 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:14:15.262 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:14:15.263 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:14:15.363 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:14:15.400 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:14:15.404 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:14:15.405 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:14:15.506 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:14:15.544 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:14:15.550 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:14:15.551 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:14:15.652 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:14:15.688 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:14:15.693 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:14:15.693 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:14:15.794 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:14:15.829 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:14:15.831 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:14:15.832 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:14:15.933 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:14:15.969 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:14:15.972 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:14:15.973 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:14:16.074 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:14:16.080 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:14:15 smithi089 bash[78452]: cluster 1685823254.7465358 mgr.y (mgr.34104) 1146 : cluster 0 pgmap v1120: 89 pgs: 9 active+clean+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 13 active+undersized+degraded, 20 active+undersized, 44 active+clean; 3.4 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 806 op/s; 28411/168732 objects degraded (16.838%); 3951/168732 objects misplaced (2.342%); 3.8 MiB/s, 61 objects/s recovering 2023-06-03T20:14:16.081 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:14:15 smithi089 bash[75068]: cluster 1685823254.7465358 mgr.y (mgr.34104) 1146 : cluster 0 pgmap v1120: 89 pgs: 9 active+clean+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 13 active+undersized+degraded, 20 active+undersized, 44 active+clean; 3.4 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 806 op/s; 28411/168732 objects degraded (16.838%); 3951/168732 objects misplaced (2.342%); 3.8 MiB/s, 61 objects/s recovering 2023-06-03T20:14:16.083 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:14:15 smithi164 bash[59462]: cluster 1685823254.7465358 mgr.y (mgr.34104) 1146 : cluster 0 pgmap v1120: 89 pgs: 9 active+clean+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 13 active+undersized+degraded, 20 active+undersized, 44 active+clean; 3.4 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 806 op/s; 28411/168732 objects degraded (16.838%); 3951/168732 objects misplaced (2.342%); 3.8 MiB/s, 61 objects/s recovering 2023-06-03T20:14:16.096 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:14:16.097 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:14:16.098 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:14:16.199 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:14:16.226 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:14:16.229 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:14:16.229 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:14:16.330 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:14:16.338 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 20:14:16 smithi089 bash[71163]: ::ffff:172.21.15.164 - - [03/Jun/2023:20:14:16] "GET /metrics HTTP/1.1" 200 228926 "" "Prometheus/2.33.4" 2023-06-03T20:14:16.372 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:14:16.379 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:14:16.380 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:14:16.481 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:14:16.515 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:14:16.518 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:14:16.518 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:14:16.619 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:14:16.644 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:14:16.646 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:14:16.646 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:14:16.747 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:14:16.778 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:14:16.780 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:14:16.780 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:14:16.881 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:14:16.914 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:14:16.921 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:14:16.922 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:14:17.022 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:14:17.057 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:14:17.062 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:14:17.063 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:14:17.164 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:14:17.195 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:14:17.198 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:14:17.199 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:14:17.300 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:14:17.330 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:14:17.333 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:14:17.333 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:14:17.434 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:14:17.479 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:14:17.481 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:14:17.481 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:14:17.582 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:14:17.615 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:14:17.617 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:14:17.618 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:14:17.719 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:14:17.759 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:14:17.763 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:14:17.764 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:14:17.832 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:14:17 smithi164 bash[59462]: cluster 1685823256.749405 mgr.y (mgr.34104) 1147 : cluster 0 pgmap v1121: 89 pgs: 9 active+clean+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 13 active+undersized+degraded, 20 active+undersized, 44 active+clean; 3.4 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 657 op/s; 27732/165912 objects degraded (16.715%); 3951/165912 objects misplaced (2.381%); 2.5 MiB/s, 40 objects/s recovering 2023-06-03T20:14:17.833 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:14:17 smithi164 bash[59462]: cluster 1685823257.7571228 mon.a (mon.0) 1087 : cluster 3 Health check update: Degraded data redundancy: 27732/165912 objects degraded (16.715%), 16 pgs degraded, 36 pgs undersized (PG_DEGRADED) 2023-06-03T20:14:17.865 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:14:17.908 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:14:17.914 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:14:17.915 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:14:18.016 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:14:18.028 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:14:17 smithi089 bash[75068]: cluster 1685823256.749405 mgr.y (mgr.34104) 1147 : cluster 0 pgmap v1121: 89 pgs: 9 active+clean+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 13 active+undersized+degraded, 20 active+undersized, 44 active+clean; 3.4 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 657 op/s; 27732/165912 objects degraded (16.715%); 3951/165912 objects misplaced (2.381%); 2.5 MiB/s, 40 objects/s recovering 2023-06-03T20:14:18.029 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:14:17 smithi089 bash[75068]: cluster 1685823257.7571228 mon.a (mon.0) 1087 : cluster 3 Health check update: Degraded data redundancy: 27732/165912 objects degraded (16.715%), 16 pgs degraded, 36 pgs undersized (PG_DEGRADED) 2023-06-03T20:14:18.029 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:14:17 smithi089 bash[78452]: cluster 1685823256.749405 mgr.y (mgr.34104) 1147 : cluster 0 pgmap v1121: 89 pgs: 9 active+clean+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 13 active+undersized+degraded, 20 active+undersized, 44 active+clean; 3.4 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 657 op/s; 27732/165912 objects degraded (16.715%); 3951/165912 objects misplaced (2.381%); 2.5 MiB/s, 40 objects/s recovering 2023-06-03T20:14:18.029 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:14:17 smithi089 bash[78452]: cluster 1685823257.7571228 mon.a (mon.0) 1087 : cluster 3 Health check update: Degraded data redundancy: 27732/165912 objects degraded (16.715%), 16 pgs degraded, 36 pgs undersized (PG_DEGRADED) 2023-06-03T20:14:18.050 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:14:18.052 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:14:18.053 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:14:18.154 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:14:18.194 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:14:18.198 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:14:18.199 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:14:18.300 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:14:18.342 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:14:18.344 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:14:18.345 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:14:18.445 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:14:18.472 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:14:18.478 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:14:18.479 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:14:18.580 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:14:18.614 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:14:18.621 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:14:18.622 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:14:18.723 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:14:18.762 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:14:18.769 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:14:18.769 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:14:18.870 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:14:18.899 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:14:18.903 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:14:18.904 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:14:19.006 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:14:19.038 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:14:19.045 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:14:19.046 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:14:19.147 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:14:19.184 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:14:19.190 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:14:19.191 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:14:19.292 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:14:19.319 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:14:19.321 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:14:19.322 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:14:19.423 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:14:19.455 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:14:19.457 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:14:19.458 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:14:19.559 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:14:19.596 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:14:19.602 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:14:19.602 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:14:19.703 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:14:19.744 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:14:19.752 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:14:19.752 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:14:19.853 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:14:19.881 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:14:19.884 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:14:19.885 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:14:19.986 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:14:20.017 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:14:20.019 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:14:20.020 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:14:20.083 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:14:19 smithi164 bash[59462]: cluster 1685823258.7503295 mgr.y (mgr.34104) 1148 : cluster 0 pgmap v1122: 89 pgs: 9 active+clean+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 13 active+undersized+degraded, 20 active+undersized, 44 active+clean; 3.2 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 784 op/s; 27520/158781 objects degraded (17.332%); 3514/158781 objects misplaced (2.213%); 3.8 MiB/s, 60 objects/s recovering 2023-06-03T20:14:20.121 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:14:20.166 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:14:20.172 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:14:20.173 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:14:20.273 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:14:20.300 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:14:20.303 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:14:20.304 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:14:20.304 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:14:19 smithi089 bash[75068]: cluster 1685823258.7503295 mgr.y (mgr.34104) 1148 : cluster 0 pgmap v1122: 89 pgs: 9 active+clean+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 13 active+undersized+degraded, 20 active+undersized, 44 active+clean; 3.2 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 784 op/s; 27520/158781 objects degraded (17.332%); 3514/158781 objects misplaced (2.213%); 3.8 MiB/s, 60 objects/s recovering 2023-06-03T20:14:20.305 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:14:19 smithi089 bash[78452]: cluster 1685823258.7503295 mgr.y (mgr.34104) 1148 : cluster 0 pgmap v1122: 89 pgs: 9 active+clean+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 13 active+undersized+degraded, 20 active+undersized, 44 active+clean; 3.2 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 784 op/s; 27520/158781 objects degraded (17.332%); 3514/158781 objects misplaced (2.213%); 3.8 MiB/s, 60 objects/s recovering 2023-06-03T20:14:20.404 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:14:20.440 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:14:20.442 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:14:20.443 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:14:20.544 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:14:20.578 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:14:20.584 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:14:20.584 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:14:20.685 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:14:20.714 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:14:20.720 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:14:20.721 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:14:20.821 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:14:20.851 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:14:20.854 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:14:20.855 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:14:20.956 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:14:20.994 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:14:20.999 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:14:20.999 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:14:21.100 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:14:21.129 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:14:21.134 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:14:21.135 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:14:21.236 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:14:21.271 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:14:21.273 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:14:21.273 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:14:21.374 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:14:21.409 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:14:21.413 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:14:21.414 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:14:21.515 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:14:21.548 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:14:21.551 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:14:21.551 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:14:21.652 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:14:21.684 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:14:21.686 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:14:21.687 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:14:21.788 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:14:21.827 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:14:21.829 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:14:21.830 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:14:21.931 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:14:21.938 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:14:21 smithi089 bash[78452]: cluster 1685823260.7510974 mgr.y (mgr.34104) 1149 : cluster 0 pgmap v1123: 89 pgs: 9 active+clean+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 13 active+undersized+degraded, 20 active+undersized, 44 active+clean; 3.2 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 661 op/s; 26868/156141 objects degraded (17.208%); 3514/156141 objects misplaced (2.251%); 2.5 MiB/s, 40 objects/s recovering 2023-06-03T20:14:21.939 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:14:21 smithi089 bash[75068]: cluster 1685823260.7510974 mgr.y (mgr.34104) 1149 : cluster 0 pgmap v1123: 89 pgs: 9 active+clean+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 13 active+undersized+degraded, 20 active+undersized, 44 active+clean; 3.2 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 661 op/s; 26868/156141 objects degraded (17.208%); 3514/156141 objects misplaced (2.251%); 2.5 MiB/s, 40 objects/s recovering 2023-06-03T20:14:21.970 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:14:21.975 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:14:21.976 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:14:22.077 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:14:22.083 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:14:21 smithi164 bash[59462]: cluster 1685823260.7510974 mgr.y (mgr.34104) 1149 : cluster 0 pgmap v1123: 89 pgs: 9 active+clean+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 13 active+undersized+degraded, 20 active+undersized, 44 active+clean; 3.2 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 661 op/s; 26868/156141 objects degraded (17.208%); 3514/156141 objects misplaced (2.251%); 2.5 MiB/s, 40 objects/s recovering 2023-06-03T20:14:22.109 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:14:22.115 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:14:22.116 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:14:22.217 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:14:22.246 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:14:22.248 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:14:22.249 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:14:22.350 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:14:22.383 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:14:22.386 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:14:22.386 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:14:22.487 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:14:22.520 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:14:22.529 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:14:22.529 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:14:22.630 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:14:22.660 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:14:22.661 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:14:22.662 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:14:22.763 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:14:22.791 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:14:22.794 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:14:22.795 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:14:22.896 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:14:22.932 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:14:22.936 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:14:22.938 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:14:23.029 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:14:22 smithi089 bash[75068]: audit 1685823262.1249843 mon.c (mon.2) 524 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T20:14:23.029 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:14:22 smithi089 bash[75068]: cluster 1685823262.758318 mon.a (mon.0) 1088 : cluster 3 Health check update: Degraded data redundancy: 26868/156141 objects degraded (17.208%), 16 pgs degraded, 36 pgs undersized (PG_DEGRADED) 2023-06-03T20:14:23.029 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:14:22 smithi089 bash[78452]: audit 1685823262.1249843 mon.c (mon.2) 524 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T20:14:23.030 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:14:22 smithi089 bash[78452]: cluster 1685823262.758318 mon.a (mon.0) 1088 : cluster 3 Health check update: Degraded data redundancy: 26868/156141 objects degraded (17.208%), 16 pgs degraded, 36 pgs undersized (PG_DEGRADED) 2023-06-03T20:14:23.039 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:14:23.073 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:14:23.077 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:14:23.077 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:14:23.178 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:14:23.211 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:14:23.215 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:14:23.215 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:14:23.316 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:14:23.333 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:14:22 smithi164 bash[59462]: audit 1685823262.1249843 mon.c (mon.2) 524 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T20:14:23.333 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:14:22 smithi164 bash[59462]: cluster 1685823262.758318 mon.a (mon.0) 1088 : cluster 3 Health check update: Degraded data redundancy: 26868/156141 objects degraded (17.208%), 16 pgs degraded, 36 pgs undersized (PG_DEGRADED) 2023-06-03T20:14:23.357 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:14:23.362 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:14:23.362 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:14:23.463 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:14:23.497 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:14:23.501 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:14:23.501 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:14:23.602 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:14:23.640 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:14:23.643 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:14:23.643 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:14:23.744 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:14:23.786 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:14:23.790 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:14:23.791 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:14:23.891 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:14:23.918 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:14:23.921 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:14:23.922 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:14:23.935 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:14:23 smithi089 bash[75068]: cluster 1685823262.7518363 mgr.y (mgr.34104) 1150 : cluster 0 pgmap v1124: 89 pgs: 9 active+clean+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 13 active+undersized+degraded, 20 active+undersized, 44 active+clean; 3.2 GiB data, 12 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 605 op/s; 26621/155400 objects degraded (17.131%); 3514/155400 objects misplaced (2.261%); 2.5 MiB/s, 40 objects/s recovering 2023-06-03T20:14:23.936 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:14:23 smithi089 bash[78452]: cluster 1685823262.7518363 mgr.y (mgr.34104) 1150 : cluster 0 pgmap v1124: 89 pgs: 9 active+clean+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 13 active+undersized+degraded, 20 active+undersized, 44 active+clean; 3.2 GiB data, 12 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 605 op/s; 26621/155400 objects degraded (17.131%); 3514/155400 objects misplaced (2.261%); 2.5 MiB/s, 40 objects/s recovering 2023-06-03T20:14:24.023 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:14:24.058 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:14:24.064 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:14:24.065 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:14:24.165 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:14:24.201 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:14:24.206 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:14:24.206 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:14:24.307 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:14:24.332 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:14:24.333 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:14:23 smithi164 bash[59462]: cluster 1685823262.7518363 mgr.y (mgr.34104) 1150 : cluster 0 pgmap v1124: 89 pgs: 9 active+clean+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 13 active+undersized+degraded, 20 active+undersized, 44 active+clean; 3.2 GiB data, 12 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 605 op/s; 26621/155400 objects degraded (17.131%); 3514/155400 objects misplaced (2.261%); 2.5 MiB/s, 40 objects/s recovering 2023-06-03T20:14:24.334 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:14:24.335 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:14:24.436 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:14:24.468 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:14:24.476 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:14:24.477 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:14:24.578 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:14:24.614 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:14:24.618 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:14:24.619 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:14:24.720 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:14:24.765 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:14:24.770 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:14:24.771 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:14:24.872 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:14:24.899 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:14:24.903 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:14:24.904 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:14:25.005 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:14:25.040 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:14:25.048 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:14:25.049 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:14:25.150 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:14:25.183 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:14:25.186 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:14:25.186 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:14:25.287 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:14:25.317 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:14:25.322 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:14:25.323 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:14:25.423 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:14:25.460 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:14:25.462 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:14:25.463 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:14:25.564 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:14:25.585 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:14:25.587 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:14:25.588 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:14:25.689 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:14:25.710 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:14:25.711 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:14:25.712 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:14:25.813 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:14:25.857 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:14:25 smithi089 bash[75068]: cluster 1685823264.7538874 mgr.y (mgr.34104) 1151 : cluster 0 pgmap v1125: 89 pgs: 9 active+clean+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 13 active+undersized+degraded, 20 active+undersized, 44 active+clean; 3.0 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 811 op/s; 26153/147375 objects degraded (17.746%); 3043/147375 objects misplaced (2.065%); 3.9 MiB/s, 61 objects/s recovering 2023-06-03T20:14:25.859 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:14:25.864 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:14:25.864 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:14:25.965 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:14:26.000 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:14:26.006 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:14:26.006 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:14:26.083 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:14:25 smithi164 bash[59462]: cluster 1685823264.7538874 mgr.y (mgr.34104) 1151 : cluster 0 pgmap v1125: 89 pgs: 9 active+clean+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 13 active+undersized+degraded, 20 active+undersized, 44 active+clean; 3.0 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 811 op/s; 26153/147375 objects degraded (17.746%); 3043/147375 objects misplaced (2.065%); 3.9 MiB/s, 61 objects/s recovering 2023-06-03T20:14:26.108 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:14:26.155 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:14:26.159 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:14:26.160 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:14:26.232 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:14:25 smithi089 bash[78452]: cluster 1685823264.7538874 mgr.y (mgr.34104) 1151 : cluster 0 pgmap v1125: 89 pgs: 9 active+clean+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 13 active+undersized+degraded, 20 active+undersized, 44 active+clean; 3.0 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 811 op/s; 26153/147375 objects degraded (17.746%); 3043/147375 objects misplaced (2.065%); 3.9 MiB/s, 61 objects/s recovering 2023-06-03T20:14:26.260 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:14:26.291 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:14:26.299 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:14:26.300 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:14:26.401 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:14:26.436 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:14:26.439 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:14:26.440 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:14:26.528 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 20:14:26 smithi089 bash[71163]: ::ffff:172.21.15.164 - - [03/Jun/2023:20:14:26] "GET /metrics HTTP/1.1" 200 228932 "" "Prometheus/2.33.4" 2023-06-03T20:14:26.541 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:14:26.580 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:14:26.582 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:14:26.583 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:14:26.684 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:14:26.726 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:14:26.729 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:14:26.730 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:14:26.831 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:14:26.872 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:14:26.874 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:14:26.875 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:14:26.976 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:14:27.015 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:14:27.018 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:14:27.019 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:14:27.120 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:14:27.152 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:14:27.154 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:14:27.155 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:14:27.255 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:14:27.293 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:14:27.300 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:14:27.300 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:14:27.401 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:14:27.443 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:14:27.445 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:14:27.446 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:14:27.547 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:14:27.590 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:14:27.593 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:14:27.594 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:14:27.694 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:14:27.730 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:14:27.733 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:14:27.734 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:14:27.835 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:14:27.870 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:14:27.872 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:14:27.873 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:14:27.974 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:14:28.004 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:14:28.006 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:14:28.007 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:14:28.029 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:14:27 smithi089 bash[75068]: cluster 1685823266.754814 mgr.y (mgr.34104) 1152 : cluster 0 pgmap v1126: 89 pgs: 9 active+clean+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 13 active+undersized+degraded, 20 active+undersized, 44 active+clean; 2.9 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 667 op/s; 25490/144684 objects degraded (17.618%); 3043/144684 objects misplaced (2.103%); 2.6 MiB/s, 41 objects/s recovering 2023-06-03T20:14:28.029 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:14:27 smithi089 bash[75068]: cluster 1685823267.7593758 mon.a (mon.0) 1089 : cluster 3 Health check update: Degraded data redundancy: 25490/144684 objects degraded (17.618%), 16 pgs degraded, 36 pgs undersized (PG_DEGRADED) 2023-06-03T20:14:28.029 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:14:27 smithi089 bash[78452]: cluster 1685823266.754814 mgr.y (mgr.34104) 1152 : cluster 0 pgmap v1126: 89 pgs: 9 active+clean+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 13 active+undersized+degraded, 20 active+undersized, 44 active+clean; 2.9 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 667 op/s; 25490/144684 objects degraded (17.618%); 3043/144684 objects misplaced (2.103%); 2.6 MiB/s, 41 objects/s recovering 2023-06-03T20:14:28.029 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:14:27 smithi089 bash[78452]: cluster 1685823267.7593758 mon.a (mon.0) 1089 : cluster 3 Health check update: Degraded data redundancy: 25490/144684 objects degraded (17.618%), 16 pgs degraded, 36 pgs undersized (PG_DEGRADED) 2023-06-03T20:14:28.108 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:14:28.117 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:14:27 smithi164 bash[59462]: cluster 1685823266.754814 mgr.y (mgr.34104) 1152 : cluster 0 pgmap v1126: 89 pgs: 9 active+clean+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 13 active+undersized+degraded, 20 active+undersized, 44 active+clean; 2.9 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 667 op/s; 25490/144684 objects degraded (17.618%); 3043/144684 objects misplaced (2.103%); 2.6 MiB/s, 41 objects/s recovering 2023-06-03T20:14:28.117 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:14:27 smithi164 bash[59462]: cluster 1685823267.7593758 mon.a (mon.0) 1089 : cluster 3 Health check update: Degraded data redundancy: 25490/144684 objects degraded (17.618%), 16 pgs degraded, 36 pgs undersized (PG_DEGRADED) 2023-06-03T20:14:28.139 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:14:28.141 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:14:28.142 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:14:28.243 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:14:28.275 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:14:28.280 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:14:28.281 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:14:28.382 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:14:28.413 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:14:28.416 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:14:28.416 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:14:28.517 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:14:28.557 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:14:28.559 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:14:28.560 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:14:28.661 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:14:28.694 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:14:28.699 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:14:28.699 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:14:28.800 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:14:28.830 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:14:28.834 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:14:28.835 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:14:28.936 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:14:28.967 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:14:28.972 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:14:28.973 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:14:29.074 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:14:29.105 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:14:29.108 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:14:29.109 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:14:29.210 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:14:29.244 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:14:29.246 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:14:29.247 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:14:29.348 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:14:29.380 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:14:29.383 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:14:29.383 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:14:29.484 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:14:29.531 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:14:29.538 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:14:29.538 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:14:29.639 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:14:29.671 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:14:29.676 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:14:29.677 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:14:29.778 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:14:29.814 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:14:29.818 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:14:29.819 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:14:29.920 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:14:29.958 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:14:29.960 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:14:29.962 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:14:30.028 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:14:29 smithi089 bash[78452]: cluster 1685823268.7559874 mgr.y (mgr.34104) 1153 : cluster 0 pgmap v1127: 89 pgs: 9 active+clean+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 13 active+undersized+degraded, 20 active+undersized, 44 active+clean; 2.8 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 792 op/s; 25234/137370 objects degraded (18.369%); 2589/137370 objects misplaced (1.885%); 4.0 MiB/s, 63 objects/s recovering 2023-06-03T20:14:30.029 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:14:29 smithi089 bash[75068]: cluster 1685823268.7559874 mgr.y (mgr.34104) 1153 : cluster 0 pgmap v1127: 89 pgs: 9 active+clean+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 13 active+undersized+degraded, 20 active+undersized, 44 active+clean; 2.8 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 792 op/s; 25234/137370 objects degraded (18.369%); 2589/137370 objects misplaced (1.885%); 4.0 MiB/s, 63 objects/s recovering 2023-06-03T20:14:30.062 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:14:30.110 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:14:30.115 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:14:30.116 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:14:30.217 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:14:30.228 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:14:29 smithi164 bash[59462]: cluster 1685823268.7559874 mgr.y (mgr.34104) 1153 : cluster 0 pgmap v1127: 89 pgs: 9 active+clean+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 13 active+undersized+degraded, 20 active+undersized, 44 active+clean; 2.8 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 792 op/s; 25234/137370 objects degraded (18.369%); 2589/137370 objects misplaced (1.885%); 4.0 MiB/s, 63 objects/s recovering 2023-06-03T20:14:30.251 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:14:30.253 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:14:30.254 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:14:30.355 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:14:30.387 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:14:30.389 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:14:30.389 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:14:30.491 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:14:30.535 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:14:30.537 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:14:30.537 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:14:30.638 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:14:30.681 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:14:30.689 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:14:30.690 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:14:30.791 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:14:30.828 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:14:30.830 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:14:30.831 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:14:30.932 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:14:30.976 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:14:30.981 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:14:30.982 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:14:31.083 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:14:31.113 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:14:31.115 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:14:31.116 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:14:31.217 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:14:31.249 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:14:31.253 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:14:31.254 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:14:31.355 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:14:31.385 INFO:teuthology.orchestra.run.smithi089.stdout:osd.0: {} 2023-06-03T20:14:31.401 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:14:31.406 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:14:31.407 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:14:31.422 INFO:teuthology.orchestra.run.smithi089.stdout:osd.1: {} 2023-06-03T20:14:31.472 INFO:teuthology.orchestra.run.smithi089.stdout:osd.2: {} 2023-06-03T20:14:31.507 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:14:31.517 INFO:teuthology.orchestra.run.smithi089.stdout:osd.3: {} 2023-06-03T20:14:31.540 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:14:31.543 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:14:31.543 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:14:31.557 INFO:teuthology.orchestra.run.smithi089.stdout:osd.5: {} 2023-06-03T20:14:31.586 INFO:teuthology.orchestra.run.smithi089.stdout:osd.6: {} 2023-06-03T20:14:31.641 INFO:teuthology.orchestra.run.smithi089.stdout:osd.7: {} 2023-06-03T20:14:31.641 INFO:teuthology.orchestra.run.smithi089.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-06-03T20:14:31.641 INFO:teuthology.orchestra.run.smithi089.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-06-03T20:14:31.641 INFO:teuthology.orchestra.run.smithi089.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-06-03T20:14:31.642 INFO:teuthology.orchestra.run.smithi089.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-06-03T20:14:31.642 INFO:teuthology.orchestra.run.smithi089.stderr:Error ENXIO: problem getting command descriptions from osd.4 2023-06-03T20:14:31.642 INFO:teuthology.orchestra.run.smithi089.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-06-03T20:14:31.642 INFO:teuthology.orchestra.run.smithi089.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-06-03T20:14:31.642 INFO:teuthology.orchestra.run.smithi089.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-06-03T20:14:31.644 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:14:31.674 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:14:31.680 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:14:31.681 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:14:31.782 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:14:31.820 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:14:31.824 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:14:31.824 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:14:31.925 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:14:31.955 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:14:31 smithi089 bash[78452]: cluster 1685823270.7569146 mgr.y (mgr.34104) 1154 : cluster 0 pgmap v1128: 89 pgs: 9 active+clean+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 13 active+undersized+degraded, 20 active+undersized, 44 active+clean; 2.7 GiB data, 11 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 672 op/s; 24512/134544 objects degraded (18.219%); 2589/134544 objects misplaced (1.924%); 2.7 MiB/s, 43 objects/s recovering 2023-06-03T20:14:31.955 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:14:31 smithi089 bash[78452]: audit 1685823271.3386683 mon.a (mon.0) 1090 : audit 0 from='client.? 172.21.15.89:0/1768089916' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-03T20:14:31.955 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:14:31 smithi089 bash[75068]: cluster 1685823270.7569146 mgr.y (mgr.34104) 1154 : cluster 0 pgmap v1128: 89 pgs: 9 active+clean+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 13 active+undersized+degraded, 20 active+undersized, 44 active+clean; 2.7 GiB data, 11 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 672 op/s; 24512/134544 objects degraded (18.219%); 2589/134544 objects misplaced (1.924%); 2.7 MiB/s, 43 objects/s recovering 2023-06-03T20:14:31.956 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:14:31 smithi089 bash[75068]: audit 1685823271.3386683 mon.a (mon.0) 1090 : audit 0 from='client.? 172.21.15.89:0/1768089916' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-03T20:14:31.964 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:14:31.969 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:14:31.970 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:14:32.071 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:14:32.102 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:14:32.105 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:14:32.106 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:14:32.207 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:14:32.262 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:14:32.266 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:14:32.267 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:14:32.333 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:14:31 smithi164 bash[59462]: cluster 1685823270.7569146 mgr.y (mgr.34104) 1154 : cluster 0 pgmap v1128: 89 pgs: 9 active+clean+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 13 active+undersized+degraded, 20 active+undersized, 44 active+clean; 2.7 GiB data, 11 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 672 op/s; 24512/134544 objects degraded (18.219%); 2589/134544 objects misplaced (1.924%); 2.7 MiB/s, 43 objects/s recovering 2023-06-03T20:14:32.393 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:14:31 smithi164 bash[59462]: audit 1685823271.3386683 mon.a (mon.0) 1090 : audit 0 from='client.? 172.21.15.89:0/1768089916' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-03T20:14:32.394 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:14:32.427 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:14:32.431 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:14:32.432 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:14:32.481 INFO:teuthology.orchestra.run.smithi089.stdout: 2023-06-03T20:14:32.486 INFO:teuthology.orchestra.run.smithi089.stderr:dumped all 2023-06-03T20:14:32.516 INFO:teuthology.orchestra.run.smithi089.stdout:ERROR: (22) Invalid argument 2023-06-03T20:14:32.517 INFO:teuthology.orchestra.run.smithi089.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-06-03T20:14:32.533 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:14:32.572 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:14:32.578 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:14:32.578 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:14:32.679 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:14:32.720 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:14:32.724 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:14:32.724 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:14:32.825 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:14:32.868 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:14:32.870 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:14:32.870 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:14:32.898 INFO:teuthology.orchestra.run.smithi089.stderr:noscrub is unset 2023-06-03T20:14:32.972 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:14:33.003 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:14:33.007 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:14:33.008 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:14:33.083 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:14:32 smithi164 bash[59462]: audit 1685823272.0898597 mon.b (mon.1) 58 : audit 1 from='client.? 172.21.15.89:0/3637391650' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-06-03T20:14:33.084 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:14:32 smithi164 bash[59462]: audit 1685823272.092322 mon.a (mon.0) 1091 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-06-03T20:14:33.084 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:14:32 smithi164 bash[59462]: cluster 1685823272.760489 mon.a (mon.0) 1092 : cluster 3 Health check update: Degraded data redundancy: 24512/134544 objects degraded (18.219%), 16 pgs degraded, 36 pgs undersized (PG_DEGRADED) 2023-06-03T20:14:33.109 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:14:33.146 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:14:33.150 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:14:33.150 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:14:33.251 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:14:33.259 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:14:32 smithi089 bash[75068]: audit 1685823272.0898597 mon.b (mon.1) 58 : audit 1 from='client.? 172.21.15.89:0/3637391650' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-06-03T20:14:33.260 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:14:32 smithi089 bash[75068]: audit 1685823272.092322 mon.a (mon.0) 1091 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-06-03T20:14:33.260 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:14:32 smithi089 bash[75068]: cluster 1685823272.760489 mon.a (mon.0) 1092 : cluster 3 Health check update: Degraded data redundancy: 24512/134544 objects degraded (18.219%), 16 pgs degraded, 36 pgs undersized (PG_DEGRADED) 2023-06-03T20:14:33.260 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:14:32 smithi089 bash[78452]: audit 1685823272.0898597 mon.b (mon.1) 58 : audit 1 from='client.? 172.21.15.89:0/3637391650' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-06-03T20:14:33.260 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:14:32 smithi089 bash[78452]: audit 1685823272.092322 mon.a (mon.0) 1091 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-06-03T20:14:33.260 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:14:32 smithi089 bash[78452]: cluster 1685823272.760489 mon.a (mon.0) 1092 : cluster 3 Health check update: Degraded data redundancy: 24512/134544 objects degraded (18.219%), 16 pgs degraded, 36 pgs undersized (PG_DEGRADED) 2023-06-03T20:14:33.286 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:14:33.291 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:14:33.291 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:14:33.392 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:14:33.420 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:14:33.429 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:14:33.430 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:14:33.531 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:14:33.566 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:14:33.568 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:14:33.569 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:14:33.669 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:14:33.705 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:14:33.710 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:14:33.711 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:14:33.812 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:14:33.850 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:14:33.853 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:14:33.854 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:14:33.955 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:14:33.995 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:14:33.999 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:14:34.000 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:14:34.101 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:14:34.144 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:14:34.146 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:14:33 smithi164 bash[59462]: audit 1685823272.4530656 mgr.y (mgr.34104) 1155 : audit 0 from='client.55780 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-06-03T20:14:34.146 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:14:33 smithi164 bash[59462]: cluster 1685823272.7577538 mgr.y (mgr.34104) 1156 : cluster 0 pgmap v1129: 89 pgs: 9 active+clean+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 13 active+undersized+degraded, 20 active+undersized, 44 active+clean; 2.7 GiB data, 11 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 618 op/s; 24282/133854 objects degraded (18.141%); 2589/133854 objects misplaced (1.934%); 2.7 MiB/s, 43 objects/s recovering 2023-06-03T20:14:34.146 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:14:33 smithi164 bash[59462]: cluster 1685823272.8667333 mon.a (mon.0) 1093 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-06-03T20:14:34.146 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:14:33 smithi164 bash[59462]: cluster 1685823272.872915 osd.6 (osd.6) 149 : cluster 3 failed to encode map e204 with expected crc 2023-06-03T20:14:34.146 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:14:33 smithi164 bash[59462]: audit 1685823272.8790205 mon.a (mon.0) 1094 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-06-03T20:14:34.146 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:14:33 smithi164 bash[59462]: cluster 1685823272.8791614 mon.a (mon.0) 1095 : cluster 0 osdmap e204: 8 total, 7 up, 7 in 2023-06-03T20:14:34.147 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:14:33 smithi164 bash[59462]: cluster 1685823272.8836155 osd.5 (osd.5) 136 : cluster 3 failed to encode map e204 with expected crc 2023-06-03T20:14:34.147 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:14:33 smithi164 bash[59462]: cluster 1685823272.885546 osd.7 (osd.7) 131 : cluster 3 failed to encode map e204 with expected crc 2023-06-03T20:14:34.147 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:14:33 smithi164 bash[59462]: cluster 1685823272.8900912 osd.7 (osd.7) 132 : cluster 3 failed to encode map e204 with expected crc 2023-06-03T20:14:34.147 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:14:34.148 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:14:34.249 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:14:34.257 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:14:33 smithi089 bash[78452]: audit 1685823272.4530656 mgr.y (mgr.34104) 1155 : audit 0 from='client.55780 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-06-03T20:14:34.257 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:14:33 smithi089 bash[78452]: cluster 1685823272.7577538 mgr.y (mgr.34104) 1156 : cluster 0 pgmap v1129: 89 pgs: 9 active+clean+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 13 active+undersized+degraded, 20 active+undersized, 44 active+clean; 2.7 GiB data, 11 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 618 op/s; 24282/133854 objects degraded (18.141%); 2589/133854 objects misplaced (1.934%); 2.7 MiB/s, 43 objects/s recovering 2023-06-03T20:14:34.258 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:14:33 smithi089 bash[78452]: cluster 1685823272.8667333 mon.a (mon.0) 1093 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-06-03T20:14:34.258 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:14:33 smithi089 bash[78452]: cluster 1685823272.872915 osd.6 (osd.6) 149 : cluster 3 failed to encode map e204 with expected crc 2023-06-03T20:14:34.258 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:14:33 smithi089 bash[78452]: audit 1685823272.8790205 mon.a (mon.0) 1094 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-06-03T20:14:34.258 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:14:33 smithi089 bash[78452]: cluster 1685823272.8791614 mon.a (mon.0) 1095 : cluster 0 osdmap e204: 8 total, 7 up, 7 in 2023-06-03T20:14:34.258 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:14:33 smithi089 bash[78452]: cluster 1685823272.8836155 osd.5 (osd.5) 136 : cluster 3 failed to encode map e204 with expected crc 2023-06-03T20:14:34.258 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:14:33 smithi089 bash[78452]: cluster 1685823272.885546 osd.7 (osd.7) 131 : cluster 3 failed to encode map e204 with expected crc 2023-06-03T20:14:34.258 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:14:33 smithi089 bash[78452]: cluster 1685823272.8900912 osd.7 (osd.7) 132 : cluster 3 failed to encode map e204 with expected crc 2023-06-03T20:14:34.259 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:14:33 smithi089 bash[75068]: audit 1685823272.4530656 mgr.y (mgr.34104) 1155 : audit 0 from='client.55780 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-06-03T20:14:34.259 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:14:33 smithi089 bash[75068]: cluster 1685823272.7577538 mgr.y (mgr.34104) 1156 : cluster 0 pgmap v1129: 89 pgs: 9 active+clean+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 13 active+undersized+degraded, 20 active+undersized, 44 active+clean; 2.7 GiB data, 11 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 618 op/s; 24282/133854 objects degraded (18.141%); 2589/133854 objects misplaced (1.934%); 2.7 MiB/s, 43 objects/s recovering 2023-06-03T20:14:34.259 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:14:33 smithi089 bash[75068]: cluster 1685823272.8667333 mon.a (mon.0) 1093 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-06-03T20:14:34.259 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:14:33 smithi089 bash[75068]: cluster 1685823272.872915 osd.6 (osd.6) 149 : cluster 3 failed to encode map e204 with expected crc 2023-06-03T20:14:34.259 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:14:33 smithi089 bash[75068]: audit 1685823272.8790205 mon.a (mon.0) 1094 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-06-03T20:14:34.259 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:14:33 smithi089 bash[75068]: cluster 1685823272.8791614 mon.a (mon.0) 1095 : cluster 0 osdmap e204: 8 total, 7 up, 7 in 2023-06-03T20:14:34.260 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:14:33 smithi089 bash[75068]: cluster 1685823272.8836155 osd.5 (osd.5) 136 : cluster 3 failed to encode map e204 with expected crc 2023-06-03T20:14:34.260 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:14:33 smithi089 bash[75068]: cluster 1685823272.885546 osd.7 (osd.7) 131 : cluster 3 failed to encode map e204 with expected crc 2023-06-03T20:14:34.260 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:14:33 smithi089 bash[75068]: cluster 1685823272.8900912 osd.7 (osd.7) 132 : cluster 3 failed to encode map e204 with expected crc 2023-06-03T20:14:34.287 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:14:34.288 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:14:34.288 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:14:34.389 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:14:34.426 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:14:34.427 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:14:34.428 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:14:34.529 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:14:34.560 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:14:34.564 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:14:34.565 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:14:34.666 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:14:34.704 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:14:34.706 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:14:34.707 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:14:34.808 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:14:34.833 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:14:34.836 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:14:34.837 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:14:34.938 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:14:34.940 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-06-03T20:14:34.941 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-06-03T20:14:34.972 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:14:34.974 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:14:34.975 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:14:35.076 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:14:35.111 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:14:35.114 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:14:35.115 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:14:35.216 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:14:35.245 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:14:35.249 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:14:35.252 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:14:35.352 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:14:35.383 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:14:35.393 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:14:35.394 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:14:35.495 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:14:35.528 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:14:35.530 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:14:35.531 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:14:35.632 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:14:35.653 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:14:35.655 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:14:35.655 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:14:35.756 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:14:35.778 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:14:35.780 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:14:35.781 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:14:35.882 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:14:35.891 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:14:35 smithi089 bash[75068]: cluster 1685823274.75851 mgr.y (mgr.34104) 1157 : cluster 0 pgmap v1131: 89 pgs: 9 active+clean+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 13 active+undersized+degraded, 20 active+undersized, 44 active+clean; 2.6 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 705 op/s; 23818/126186 objects degraded (18.875%); 2171/126186 objects misplaced (1.720%); 3.4 MiB/s, 53 objects/s recovering 2023-06-03T20:14:35.891 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:14:35 smithi089 bash[78452]: cluster 1685823274.75851 mgr.y (mgr.34104) 1157 : cluster 0 pgmap v1131: 89 pgs: 9 active+clean+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 13 active+undersized+degraded, 20 active+undersized, 44 active+clean; 2.6 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 705 op/s; 23818/126186 objects degraded (18.875%); 2171/126186 objects misplaced (1.720%); 3.4 MiB/s, 53 objects/s recovering 2023-06-03T20:14:35.928 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:14:35.933 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:14:35.933 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:14:36.034 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:14:36.075 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:14:36.080 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:14:36.080 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:14:36.181 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:14:36.191 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:14:35 smithi164 bash[59462]: cluster 1685823274.75851 mgr.y (mgr.34104) 1157 : cluster 0 pgmap v1131: 89 pgs: 9 active+clean+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 13 active+undersized+degraded, 20 active+undersized, 44 active+clean; 2.6 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 705 op/s; 23818/126186 objects degraded (18.875%); 2171/126186 objects misplaced (1.720%); 3.4 MiB/s, 53 objects/s recovering 2023-06-03T20:14:36.212 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:14:36.216 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:14:36.217 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:14:36.318 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:14:36.356 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:14:36.364 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:14:36.365 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:14:36.465 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:14:36.501 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:14:36.507 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:14:36.507 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:14:36.510 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 20:14:36 smithi089 bash[71163]: ::ffff:172.21.15.164 - - [03/Jun/2023:20:14:36] "GET /metrics HTTP/1.1" 200 228932 "" "Prometheus/2.33.4" 2023-06-03T20:14:36.608 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:14:36.634 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:14:36.637 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:14:36.637 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:14:36.738 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:14:36.768 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:14:36.771 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:14:36.772 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:14:36.873 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:14:36.915 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:14:36.921 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:14:36.922 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:14:36.942 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-06-03T20:14:37.023 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:14:37.059 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:14:37.063 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:14:37.064 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:14:37.164 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:14:37.209 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:14:37.216 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:14:37.217 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:14:37.318 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:14:37.359 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:14:37.366 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:14:37.366 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:14:37.467 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:14:37.507 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:14:37.510 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:14:37.511 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:14:37.612 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:14:37.647 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:14:37.651 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:14:37.652 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:14:37.752 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:14:37.779 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:14:37.782 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:14:37.782 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:14:37.883 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:14:37.918 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:14:37.923 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:14:37.923 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:14:37.936 INFO:teuthology.orchestra.run.smithi089.stdout:{"pg_ready":true,"pg_map":{"version":1128,"stamp":"2023-06-03T20:14:30.756307+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":2938116006,"num_objects":44848,"num_object_clones":0,"num_object_copies":134544,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":24512,"num_objects_misplaced":2589,"num_objects_unfound":0,"num_objects_dirty":44848,"num_whiteouts":0,"num_read":309,"num_read_kb":513,"num_write":166489,"num_write_kb":6756047,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5050,"num_bytes_recovered":330629120,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":120323,"ondisk_log_size":120323,"up":225,"acting":231,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":278,"num_osds":7,"num_per_pool_osds":7,"num_per_pool_omap_osds":7,"kb":656216064,"kb_used":12057608,"kb_used_data":9188688,"kb_used_omap":26,"kb_used_meta":2868453,"kb_avail":644158456,"statfs":{"total":671965249536,"available":659618258944,"internally_reserved":0,"allocated":9409216512,"data_stored":9404720078,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":27292,"internal_metadata":2937296228},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[3,1,1,6,7,0,0,0,0,9],"upper_bound":1024},"perf_stat":{"commit_latency_ms":5,"apply_latency_ms":5,"commit_latency_ns":5000000,"apply_latency_ns":5000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-529465344,"num_objects":-8079,"num_object_clones":0,"num_object_copies":-24237,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":-3008,"num_objects_misplaced":-925,"num_objects_unfound":0,"num_objects_dirty":-8079,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8079,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":522,"num_bytes_recovered":34209792,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.006722"},"pg_stats":[{"pgid":"3.1a","version":"60'1","reported_seq":17,"reported_epoch":203,"state":"active+undersized","last_fresh":"2023-06-03T20:13:57.918426+0000","last_change":"2023-06-03T20:11:54.019728+0000","last_active":"2023-06-03T20:13:57.918426+0000","last_peered":"2023-06-03T20:13:57.918426+0000","last_clean":"2023-06-03T19:34:15.767991+0000","last_became_active":"2023-06-03T20:11:54.019728+0000","last_became_peered":"2023-06-03T20:11:54.019728+0000","last_unstale":"2023-06-03T20:13:57.918426+0000","last_undegraded":"2023-06-03T20:13:57.918426+0000","last_fullsized":"2023-06-03T20:11:54.009512+0000","mapping_epoch":195,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T19:58:26.738637+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":["2","0"],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":16,"reported_epoch":202,"state":"active+clean+remapped","last_fresh":"2023-06-03T20:13:27.788152+0000","last_change":"2023-06-03T20:12:31.911399+0000","last_active":"2023-06-03T20:13:27.788152+0000","last_peered":"2023-06-03T20:13:27.788152+0000","last_clean":"2023-06-03T20:13:27.788152+0000","last_became_active":"2023-06-03T20:12:31.911029+0000","last_became_peered":"2023-06-03T20:12:31.911029+0000","last_unstale":"2023-06-03T20:13:27.788152+0000","last_undegraded":"2023-06-03T20:13:27.788152+0000","last_fullsized":"2023-06-03T20:13:27.788152+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T05:04:34.770935+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5],"acting":[3,5,6],"avail_no_missing":["3","5","6"],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":163,"reported_epoch":202,"state":"active+undersized","last_fresh":"2023-06-03T20:13:27.787820+0000","last_change":"2023-06-03T20:11:54.017045+0000","last_active":"2023-06-03T20:13:27.787820+0000","last_peered":"2023-06-03T20:13:27.787820+0000","last_clean":"2023-06-03T20:11:43.978421+0000","last_became_active":"2023-06-03T20:11:54.017045+0000","last_became_peered":"2023-06-03T20:11:54.017045+0000","last_unstale":"2023-06-03T20:13:27.787820+0000","last_undegraded":"2023-06-03T20:13:27.787820+0000","last_fullsized":"2023-06-03T20:11:54.008056+0000","mapping_epoch":195,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T23:56:32.827460+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6],"acting":[3,6],"avail_no_missing":["3","6"],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":166,"reported_epoch":202,"state":"active+clean","last_fresh":"2023-06-03T20:13:27.803330+0000","last_change":"2023-06-03T19:56:54.507653+0000","last_active":"2023-06-03T20:13:27.803330+0000","last_peered":"2023-06-03T20:13:27.803330+0000","last_clean":"2023-06-03T20:13:27.803330+0000","last_became_active":"2023-06-03T19:56:54.507350+0000","last_became_peered":"2023-06-03T19:56:54.507350+0000","last_unstale":"2023-06-03T20:13:27.803330+0000","last_undegraded":"2023-06-03T20:13:27.803330+0000","last_fullsized":"2023-06-03T20:13:27.803330+0000","mapping_epoch":119,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T22:33:20.065844+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":198,"reported_epoch":202,"state":"active+clean","last_fresh":"2023-06-03T20:13:47.950461+0000","last_change":"2023-06-03T20:07:19.912004+0000","last_active":"2023-06-03T20:13:47.950461+0000","last_peered":"2023-06-03T20:13:47.950461+0000","last_clean":"2023-06-03T20:13:47.950461+0000","last_became_active":"2023-06-03T20:07:19.910895+0000","last_became_peered":"2023-06-03T20:07:19.910895+0000","last_unstale":"2023-06-03T20:13:47.950461+0000","last_undegraded":"2023-06-03T20:13:47.950461+0000","last_fullsized":"2023-06-03T20:13:47.950461+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T06:46:44.564002+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,2],"acting":[3,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":178,"reported_epoch":202,"state":"active+clean","last_fresh":"2023-06-03T20:13:27.788232+0000","last_change":"2023-06-03T20:07:19.918531+0000","last_active":"2023-06-03T20:13:27.788232+0000","last_peered":"2023-06-03T20:13:27.788232+0000","last_clean":"2023-06-03T20:13:27.788232+0000","last_became_active":"2023-06-03T20:07:19.918173+0000","last_became_peered":"2023-06-03T20:07:19.918173+0000","last_unstale":"2023-06-03T20:13:27.788232+0000","last_undegraded":"2023-06-03T20:13:27.788232+0000","last_fullsized":"2023-06-03T20:13:27.788232+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,0],"acting":[6,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":163,"reported_epoch":202,"state":"active+clean","last_fresh":"2023-06-03T20:13:41.014309+0000","last_change":"2023-06-03T19:56:54.509490+0000","last_active":"2023-06-03T20:13:41.014309+0000","last_peered":"2023-06-03T20:13:41.014309+0000","last_clean":"2023-06-03T20:13:41.014309+0000","last_became_active":"2023-06-03T19:56:54.509145+0000","last_became_peered":"2023-06-03T19:56:54.509145+0000","last_unstale":"2023-06-03T20:13:41.014309+0000","last_undegraded":"2023-06-03T20:13:41.014309+0000","last_fullsized":"2023-06-03T20:13:41.014309+0000","mapping_epoch":119,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T07:12:03.964110+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":198,"reported_epoch":202,"state":"active+clean+remapped","last_fresh":"2023-06-03T20:13:27.789006+0000","last_change":"2023-06-03T20:12:31.937956+0000","last_active":"2023-06-03T20:13:27.789006+0000","last_peered":"2023-06-03T20:13:27.789006+0000","last_clean":"2023-06-03T20:13:27.789006+0000","last_became_active":"2023-06-03T20:12:31.937551+0000","last_became_peered":"2023-06-03T20:12:31.937551+0000","last_unstale":"2023-06-03T20:13:27.789006+0000","last_undegraded":"2023-06-03T20:13:27.789006+0000","last_fullsized":"2023-06-03T20:13:27.789006+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T21:49:12.058781+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5],"acting":[3,5,6],"avail_no_missing":["3","5","6"],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":179,"reported_epoch":202,"state":"active+clean","last_fresh":"2023-06-03T20:13:27.802332+0000","last_change":"2023-06-03T20:07:19.912362+0000","last_active":"2023-06-03T20:13:27.802332+0000","last_peered":"2023-06-03T20:13:27.802332+0000","last_clean":"2023-06-03T20:13:27.802332+0000","last_became_active":"2023-06-03T20:07:19.907385+0000","last_became_peered":"2023-06-03T20:07:19.907385+0000","last_unstale":"2023-06-03T20:13:27.802332+0000","last_undegraded":"2023-06-03T20:13:27.802332+0000","last_fullsized":"2023-06-03T20:13:27.802332+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":154,"reported_epoch":202,"state":"active+clean","last_fresh":"2023-06-03T20:13:27.787706+0000","last_change":"2023-06-03T19:56:54.510462+0000","last_active":"2023-06-03T20:13:27.787706+0000","last_peered":"2023-06-03T20:13:27.787706+0000","last_clean":"2023-06-03T20:13:27.787706+0000","last_became_active":"2023-06-03T19:56:54.509783+0000","last_became_peered":"2023-06-03T19:56:54.509783+0000","last_unstale":"2023-06-03T20:13:27.787706+0000","last_undegraded":"2023-06-03T20:13:27.787706+0000","last_fullsized":"2023-06-03T20:13:27.787706+0000","mapping_epoch":119,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T00:16:09.624276+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"60'3","reported_seq":191,"reported_epoch":202,"state":"active+undersized+degraded","last_fresh":"2023-06-03T20:13:27.794124+0000","last_change":"2023-06-03T20:11:54.025696+0000","last_active":"2023-06-03T20:13:27.794124+0000","last_peered":"2023-06-03T20:13:27.794124+0000","last_clean":"2023-06-03T20:11:43.973370+0000","last_became_active":"2023-06-03T20:11:54.025696+0000","last_became_peered":"2023-06-03T20:11:54.025696+0000","last_unstale":"2023-06-03T20:13:27.794124+0000","last_undegraded":"2023-06-03T20:11:54.012202+0000","last_fullsized":"2023-06-03T20:11:54.012054+0000","mapping_epoch":195,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":173,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3],"acting":[7,3],"avail_no_missing":["7","3"],"object_location_counts":[{"shards":"3,7","objects":2}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":59,"reported_epoch":202,"state":"active+clean","last_fresh":"2023-06-03T20:13:27.793857+0000","last_change":"2023-06-03T20:12:30.911538+0000","last_active":"2023-06-03T20:13:27.793857+0000","last_peered":"2023-06-03T20:13:27.793857+0000","last_clean":"2023-06-03T20:13:27.793857+0000","last_became_active":"2023-06-03T20:12:30.911082+0000","last_became_peered":"2023-06-03T20:12:30.911082+0000","last_unstale":"2023-06-03T20:13:27.793857+0000","last_undegraded":"2023-06-03T20:13:27.793857+0000","last_fullsized":"2023-06-03T20:13:27.793857+0000","mapping_epoch":198,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":199,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T05:22:01.454415+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,5],"acting":[2,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.14","version":"77'23","reported_seq":73,"reported_epoch":202,"state":"active+undersized+degraded","last_fresh":"2023-06-03T20:13:27.792315+0000","last_change":"2023-06-03T20:11:54.018006+0000","last_active":"2023-06-03T20:13:27.792315+0000","last_peered":"2023-06-03T20:13:27.792315+0000","last_clean":"2023-06-03T19:41:47.150753+0000","last_became_active":"2023-06-03T20:11:54.018006+0000","last_became_peered":"2023-06-03T20:11:54.018006+0000","last_unstale":"2023-06-03T20:13:27.792315+0000","last_undegraded":"2023-06-03T20:11:54.016070+0000","last_fullsized":"2023-06-03T20:11:54.015933+0000","mapping_epoch":195,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":23,"log_dups_size":0,"ondisk_log_size":23,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4716,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":23,"num_write_kb":20,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6],"acting":[7,6],"avail_no_missing":["7","6"],"object_location_counts":[{"shards":"6,7","objects":4}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":145,"reported_epoch":202,"state":"active+undersized","last_fresh":"2023-06-03T20:13:27.788224+0000","last_change":"2023-06-03T20:11:54.013229+0000","last_active":"2023-06-03T20:13:27.788224+0000","last_peered":"2023-06-03T20:13:27.788224+0000","last_clean":"2023-06-03T20:07:14.870826+0000","last_became_active":"2023-06-03T20:11:54.013229+0000","last_became_peered":"2023-06-03T20:11:54.013229+0000","last_unstale":"2023-06-03T20:13:27.788224+0000","last_undegraded":"2023-06-03T20:13:27.788224+0000","last_fullsized":"2023-06-03T20:11:54.007837+0000","mapping_epoch":195,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5],"acting":[6,5],"avail_no_missing":["6","5"],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.13","version":"60'2","reported_seq":193,"reported_epoch":202,"state":"active+undersized+degraded","last_fresh":"2023-06-03T20:13:27.809926+0000","last_change":"2023-06-03T20:11:54.019051+0000","last_active":"2023-06-03T20:13:27.809926+0000","last_peered":"2023-06-03T20:13:27.809926+0000","last_clean":"2023-06-03T20:11:43.972488+0000","last_became_active":"2023-06-03T20:11:54.019051+0000","last_became_peered":"2023-06-03T20:11:54.019051+0000","last_unstale":"2023-06-03T20:13:27.809926+0000","last_undegraded":"2023-06-03T20:11:54.012366+0000","last_fullsized":"2023-06-03T20:11:54.012217+0000","mapping_epoch":195,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2],"acting":[7,2],"avail_no_missing":["7","2"],"object_location_counts":[{"shards":"2,7","objects":2}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":182,"reported_epoch":202,"state":"active+clean","last_fresh":"2023-06-03T20:13:27.788882+0000","last_change":"2023-06-03T20:07:19.907827+0000","last_active":"2023-06-03T20:13:27.788882+0000","last_peered":"2023-06-03T20:13:27.788882+0000","last_clean":"2023-06-03T20:13:27.788882+0000","last_became_active":"2023-06-03T20:07:19.907296+0000","last_became_peered":"2023-06-03T20:07:19.907296+0000","last_unstale":"2023-06-03T20:13:27.788882+0000","last_undegraded":"2023-06-03T20:13:27.788882+0000","last_fullsized":"2023-06-03T20:13:27.788882+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T00:34:13.230147+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":163,"reported_epoch":202,"state":"active+clean","last_fresh":"2023-06-03T20:13:27.805157+0000","last_change":"2023-06-03T19:56:54.507479+0000","last_active":"2023-06-03T20:13:27.805157+0000","last_peered":"2023-06-03T20:13:27.805157+0000","last_clean":"2023-06-03T20:13:27.805157+0000","last_became_active":"2023-06-03T19:56:54.507149+0000","last_became_peered":"2023-06-03T19:56:54.507149+0000","last_unstale":"2023-06-03T20:13:27.805157+0000","last_undegraded":"2023-06-03T20:13:27.805157+0000","last_fullsized":"2023-06-03T20:13:27.805157+0000","mapping_epoch":119,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T20:22:36.676317+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":169,"reported_epoch":202,"state":"active+clean","last_fresh":"2023-06-03T20:13:27.803766+0000","last_change":"2023-06-03T19:56:54.512472+0000","last_active":"2023-06-03T20:13:27.803766+0000","last_peered":"2023-06-03T20:13:27.803766+0000","last_clean":"2023-06-03T20:13:27.803766+0000","last_became_active":"2023-06-03T19:56:54.512051+0000","last_became_peered":"2023-06-03T19:56:54.512051+0000","last_unstale":"2023-06-03T20:13:27.803766+0000","last_undegraded":"2023-06-03T20:13:27.803766+0000","last_fullsized":"2023-06-03T20:13:27.803766+0000","mapping_epoch":119,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":173,"reported_epoch":202,"state":"active+undersized","last_fresh":"2023-06-03T20:13:27.798512+0000","last_change":"2023-06-03T20:11:54.014913+0000","last_active":"2023-06-03T20:13:27.798512+0000","last_peered":"2023-06-03T20:13:27.798512+0000","last_clean":"2023-06-03T20:11:43.971870+0000","last_became_active":"2023-06-03T20:11:54.014913+0000","last_became_peered":"2023-06-03T20:11:54.014913+0000","last_unstale":"2023-06-03T20:13:27.798512+0000","last_undegraded":"2023-06-03T20:13:27.798512+0000","last_fullsized":"2023-06-03T20:11:54.009603+0000","mapping_epoch":195,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":173,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6],"acting":[7,6],"avail_no_missing":["7","6"],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":164,"reported_epoch":202,"state":"active+clean","last_fresh":"2023-06-03T20:13:41.014290+0000","last_change":"2023-06-03T19:56:54.510289+0000","last_active":"2023-06-03T20:13:41.014290+0000","last_peered":"2023-06-03T20:13:41.014290+0000","last_clean":"2023-06-03T20:13:41.014290+0000","last_became_active":"2023-06-03T19:56:54.509905+0000","last_became_peered":"2023-06-03T19:56:54.509905+0000","last_unstale":"2023-06-03T20:13:41.014290+0000","last_undegraded":"2023-06-03T20:13:41.014290+0000","last_fullsized":"2023-06-03T20:13:41.014290+0000","mapping_epoch":119,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T02:54:26.792401+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"60'8","reported_seq":178,"reported_epoch":202,"state":"active+clean","last_fresh":"2023-06-03T20:13:27.787881+0000","last_change":"2023-06-03T19:47:06.727778+0000","last_active":"2023-06-03T20:13:27.787881+0000","last_peered":"2023-06-03T20:13:27.787881+0000","last_clean":"2023-06-03T20:13:27.787881+0000","last_became_active":"2023-06-03T19:47:06.727305+0000","last_became_peered":"2023-06-03T19:47:06.727305+0000","last_unstale":"2023-06-03T20:13:27.787881+0000","last_undegraded":"2023-06-03T20:13:27.787881+0000","last_fullsized":"2023-06-03T20:13:27.787881+0000","mapping_epoch":86,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":87,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":182,"reported_epoch":202,"state":"active+clean","last_fresh":"2023-06-03T20:13:47.951346+0000","last_change":"2023-06-03T20:07:19.907151+0000","last_active":"2023-06-03T20:13:47.951346+0000","last_peered":"2023-06-03T20:13:47.951346+0000","last_clean":"2023-06-03T20:13:47.951346+0000","last_became_active":"2023-06-03T20:07:19.906479+0000","last_became_peered":"2023-06-03T20:07:19.906479+0000","last_unstale":"2023-06-03T20:13:47.951346+0000","last_undegraded":"2023-06-03T20:13:47.951346+0000","last_fullsized":"2023-06-03T20:13:47.951346+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T03:18:22.525692+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"11.7","version":"203'10532","reported_seq":10560,"reported_epoch":203,"state":"active+undersized+degraded","last_fresh":"2023-06-03T20:14:28.599287+0000","last_change":"2023-06-03T20:11:54.017911+0000","last_active":"2023-06-03T20:14:28.599287+0000","last_peered":"2023-06-03T20:14:28.599287+0000","last_clean":"2023-06-03T20:11:53.003160+0000","last_became_active":"2023-06-03T20:11:54.017911+0000","last_became_peered":"2023-06-03T20:11:54.017911+0000","last_unstale":"2023-06-03T20:14:28.599287+0000","last_undegraded":"2023-06-03T20:11:54.010796+0000","last_fullsized":"2023-06-03T20:11:54.010661+0000","mapping_epoch":195,"log_start":"193'800","ondisk_log_start":"193'800","created":189,"last_epoch_clean":190,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T20:11:10.759955+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T20:11:10.759955+0000","last_clean_scrub_stamp":"2023-06-03T20:11:10.759955+0000","objects_scrubbed":0,"log_size":9732,"log_dups_size":0,"ondisk_log_size":9732,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":186384384,"num_objects":2844,"num_object_clones":0,"num_object_copies":8532,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2844,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2844,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10532,"num_write_kb":428032,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0],"acting":[5,0],"avail_no_missing":["5","0"],"object_location_counts":[{"shards":"0,5","objects":2844}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"60'1","reported_seq":192,"reported_epoch":202,"state":"active+undersized+degraded","last_fresh":"2023-06-03T20:13:27.791520+0000","last_change":"2023-06-03T20:11:54.016915+0000","last_active":"2023-06-03T20:13:27.791520+0000","last_peered":"2023-06-03T20:13:27.791520+0000","last_clean":"2023-06-03T20:11:43.972655+0000","last_became_active":"2023-06-03T20:11:54.016915+0000","last_became_peered":"2023-06-03T20:11:54.016915+0000","last_unstale":"2023-06-03T20:13:27.791520+0000","last_undegraded":"2023-06-03T20:11:54.009884+0000","last_fullsized":"2023-06-03T20:11:54.009713+0000","mapping_epoch":195,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0],"acting":[7,0],"avail_no_missing":["7","0"],"object_location_counts":[{"shards":"0,7","objects":1}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":164,"reported_epoch":202,"state":"active+clean+remapped","last_fresh":"2023-06-03T20:13:27.802263+0000","last_change":"2023-06-03T20:12:31.910410+0000","last_active":"2023-06-03T20:13:27.802263+0000","last_peered":"2023-06-03T20:13:27.802263+0000","last_clean":"2023-06-03T20:13:27.802263+0000","last_became_active":"2023-06-03T20:12:31.909769+0000","last_became_peered":"2023-06-03T20:12:31.909769+0000","last_unstale":"2023-06-03T20:13:27.802263+0000","last_undegraded":"2023-06-03T20:13:27.802263+0000","last_fullsized":"2023-06-03T20:13:27.802263+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7],"acting":[5,7,6],"avail_no_missing":["5","6","7"],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"11.6","version":"203'10200","reported_seq":10228,"reported_epoch":203,"state":"active+undersized+degraded","last_fresh":"2023-06-03T20:14:26.218997+0000","last_change":"2023-06-03T20:11:54.020693+0000","last_active":"2023-06-03T20:14:26.218997+0000","last_peered":"2023-06-03T20:14:26.218997+0000","last_clean":"2023-06-03T20:11:53.009007+0000","last_became_active":"2023-06-03T20:11:54.020693+0000","last_became_peered":"2023-06-03T20:11:54.020693+0000","last_unstale":"2023-06-03T20:14:26.218997+0000","last_undegraded":"2023-06-03T20:11:54.011201+0000","last_fullsized":"2023-06-03T20:11:54.011064+0000","mapping_epoch":195,"log_start":"197'3800","ondisk_log_start":"197'3800","created":189,"last_epoch_clean":190,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T20:11:10.759955+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T20:11:10.759955+0000","last_clean_scrub_stamp":"2023-06-03T20:11:10.759955+0000","objects_scrubbed":0,"log_size":6400,"log_dups_size":0,"ondisk_log_size":6400,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T23:28:23.377635+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":191234048,"num_objects":2918,"num_object_clones":0,"num_object_copies":8754,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2918,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2918,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10200,"num_write_kb":419776,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2],"acting":[0,2],"avail_no_missing":["0","2"],"object_location_counts":[{"shards":"0,2","objects":2918}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.e","version":"60'2","reported_seq":206,"reported_epoch":202,"state":"active+undersized+degraded","last_fresh":"2023-06-03T20:13:27.791498+0000","last_change":"2023-06-03T20:11:54.012607+0000","last_active":"2023-06-03T20:13:27.791498+0000","last_peered":"2023-06-03T20:13:27.791498+0000","last_clean":"2023-06-03T20:11:43.972261+0000","last_became_active":"2023-06-03T20:11:54.012607+0000","last_became_peered":"2023-06-03T20:11:54.012607+0000","last_unstale":"2023-06-03T20:13:27.791498+0000","last_undegraded":"2023-06-03T20:11:54.010373+0000","last_fullsized":"2023-06-03T20:11:54.010224+0000","mapping_epoch":195,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0],"acting":[7,0],"avail_no_missing":["7","0"],"object_location_counts":[{"shards":"0,7","objects":1}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":194,"reported_epoch":203,"state":"active+clean+remapped","last_fresh":"2023-06-03T20:13:57.910462+0000","last_change":"2023-06-03T20:12:31.941492+0000","last_active":"2023-06-03T20:13:57.910462+0000","last_peered":"2023-06-03T20:13:57.910462+0000","last_clean":"2023-06-03T20:13:57.910462+0000","last_became_active":"2023-06-03T20:12:31.941076+0000","last_became_peered":"2023-06-03T20:12:31.941076+0000","last_unstale":"2023-06-03T20:13:57.910462+0000","last_undegraded":"2023-06-03T20:13:57.910462+0000","last_fullsized":"2023-06-03T20:13:57.910462+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T03:59:11.007424+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0],"acting":[3,0,6],"avail_no_missing":["3","0","6"],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"11.5","version":"203'10411","reported_seq":10433,"reported_epoch":203,"state":"active+clean","last_fresh":"2023-06-03T20:14:28.157580+0000","last_change":"2023-06-03T20:11:11.796552+0000","last_active":"2023-06-03T20:14:28.157580+0000","last_peered":"2023-06-03T20:14:28.157580+0000","last_clean":"2023-06-03T20:14:28.157580+0000","last_became_active":"2023-06-03T20:11:11.796144+0000","last_became_peered":"2023-06-03T20:11:11.796144+0000","last_unstale":"2023-06-03T20:14:28.157580+0000","last_undegraded":"2023-06-03T20:14:28.157580+0000","last_fullsized":"2023-06-03T20:14:28.157580+0000","mapping_epoch":189,"log_start":"197'3700","ondisk_log_start":"197'3700","created":189,"last_epoch_clean":190,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T20:11:10.759955+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T20:11:10.759955+0000","last_clean_scrub_stamp":"2023-06-03T20:11:10.759955+0000","objects_scrubbed":0,"log_size":6711,"log_dups_size":0,"ondisk_log_size":6711,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":185925632,"num_objects":2837,"num_object_clones":0,"num_object_copies":8511,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2837,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10411,"num_write_kb":423936,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.d","version":"60'1","reported_seq":153,"reported_epoch":202,"state":"active+clean","last_fresh":"2023-06-03T20:13:27.793853+0000","last_change":"2023-06-03T20:07:47.857710+0000","last_active":"2023-06-03T20:13:27.793853+0000","last_peered":"2023-06-03T20:13:27.793853+0000","last_clean":"2023-06-03T20:13:27.793853+0000","last_became_active":"2023-06-03T20:07:47.857320+0000","last_became_peered":"2023-06-03T20:07:47.857320+0000","last_unstale":"2023-06-03T20:13:27.793853+0000","last_undegraded":"2023-06-03T20:13:27.793853+0000","last_fullsized":"2023-06-03T20:13:27.793853+0000","mapping_epoch":172,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":173,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":149,"reported_epoch":202,"state":"active+clean","last_fresh":"2023-06-03T20:13:27.791219+0000","last_change":"2023-06-03T20:12:30.910738+0000","last_active":"2023-06-03T20:13:27.791219+0000","last_peered":"2023-06-03T20:13:27.791219+0000","last_clean":"2023-06-03T20:13:27.791219+0000","last_became_active":"2023-06-03T20:12:30.910145+0000","last_became_peered":"2023-06-03T20:12:30.910145+0000","last_unstale":"2023-06-03T20:13:27.791219+0000","last_undegraded":"2023-06-03T20:13:27.791219+0000","last_fullsized":"2023-06-03T20:13:27.791219+0000","mapping_epoch":198,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":199,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,0],"acting":[7,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"11.4","version":"203'10306","reported_seq":10328,"reported_epoch":203,"state":"active+clean","last_fresh":"2023-06-03T20:14:28.127835+0000","last_change":"2023-06-03T20:11:11.796227+0000","last_active":"2023-06-03T20:14:28.127835+0000","last_peered":"2023-06-03T20:14:28.127835+0000","last_clean":"2023-06-03T20:14:28.127835+0000","last_became_active":"2023-06-03T20:11:11.795900+0000","last_became_peered":"2023-06-03T20:11:11.795900+0000","last_unstale":"2023-06-03T20:14:28.127835+0000","last_undegraded":"2023-06-03T20:14:28.127835+0000","last_fullsized":"2023-06-03T20:14:28.127835+0000","mapping_epoch":189,"log_start":"197'3600","ondisk_log_start":"197'3600","created":189,"last_epoch_clean":190,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T20:11:10.759955+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T20:11:10.759955+0000","last_clean_scrub_stamp":"2023-06-03T20:11:10.759955+0000","objects_scrubbed":0,"log_size":6706,"log_dups_size":0,"ondisk_log_size":6706,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":182976512,"num_objects":2792,"num_object_clones":0,"num_object_copies":8376,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2792,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10306,"num_write_kb":419136,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,2],"acting":[7,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":170,"reported_epoch":202,"state":"active+clean","last_fresh":"2023-06-03T20:13:27.802222+0000","last_change":"2023-06-03T19:56:54.507432+0000","last_active":"2023-06-03T20:13:27.802222+0000","last_peered":"2023-06-03T20:13:27.802222+0000","last_clean":"2023-06-03T20:13:27.802222+0000","last_became_active":"2023-06-03T19:56:54.507012+0000","last_became_peered":"2023-06-03T19:56:54.507012+0000","last_unstale":"2023-06-03T20:13:27.802222+0000","last_undegraded":"2023-06-03T20:13:27.802222+0000","last_fullsized":"2023-06-03T20:13:27.802222+0000","mapping_epoch":119,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":207,"reported_epoch":203,"state":"active+undersized","last_fresh":"2023-06-03T20:13:57.918647+0000","last_change":"2023-06-03T20:11:54.017375+0000","last_active":"2023-06-03T20:13:57.918647+0000","last_peered":"2023-06-03T20:13:57.918647+0000","last_clean":"2023-06-03T20:11:50.798782+0000","last_became_active":"2023-06-03T20:11:54.017375+0000","last_became_peered":"2023-06-03T20:11:54.017375+0000","last_unstale":"2023-06-03T20:13:57.918647+0000","last_undegraded":"2023-06-03T20:13:57.918647+0000","last_fullsized":"2023-06-03T20:11:54.008913+0000","mapping_epoch":195,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T21:44:01.134376+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3],"acting":[0,3],"avail_no_missing":["0","3"],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"11.3","version":"203'10506","reported_seq":15585,"reported_epoch":203,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2023-06-03T20:14:28.161919+0000","last_change":"2023-06-03T20:13:31.111903+0000","last_active":"2023-06-03T20:14:28.161919+0000","last_peered":"2023-06-03T20:14:28.161919+0000","last_clean":"2023-06-03T20:12:29.830029+0000","last_became_active":"2023-06-03T20:12:32.285822+0000","last_became_peered":"2023-06-03T20:12:32.285822+0000","last_unstale":"2023-06-03T20:14:28.161919+0000","last_undegraded":"2023-06-03T20:13:31.108675+0000","last_fullsized":"2023-06-03T20:12:31.899666+0000","mapping_epoch":199,"log_start":"197'3800","ondisk_log_start":"197'3800","created":189,"last_epoch_clean":190,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T20:11:10.759955+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T20:11:10.759955+0000","last_clean_scrub_stamp":"2023-06-03T20:11:10.759955+0000","objects_scrubbed":0,"log_size":6706,"log_dups_size":0,"ondisk_log_size":6706,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":185597952,"num_objects":2832,"num_object_clones":0,"num_object_copies":8496,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1312,"num_objects_misplaced":1288,"num_objects_unfound":0,"num_objects_dirty":2832,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10506,"num_write_kb":426816,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2525,"num_bytes_recovered":165478400,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,2],"acting":[7,5],"avail_no_missing":["7","5","6"],"object_location_counts":[{"shards":"5,6,7","objects":1288},{"shards":"5,7","objects":232}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.b","version":"60'2","reported_seq":180,"reported_epoch":202,"state":"active+undersized+degraded","last_fresh":"2023-06-03T20:13:27.786397+0000","last_change":"2023-06-03T20:11:54.019446+0000","last_active":"2023-06-03T20:13:27.786397+0000","last_peered":"2023-06-03T20:13:27.786397+0000","last_clean":"2023-06-03T20:11:43.977155+0000","last_became_active":"2023-06-03T20:11:54.019446+0000","last_became_peered":"2023-06-03T20:11:54.019446+0000","last_unstale":"2023-06-03T20:13:27.786397+0000","last_undegraded":"2023-06-03T20:11:54.013169+0000","last_fullsized":"2023-06-03T20:11:54.012960+0000","mapping_epoch":195,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T20:48:53.735497+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0],"acting":[3,0],"avail_no_missing":["3","0"],"object_location_counts":[{"shards":"0,3","objects":1}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":105,"reported_epoch":203,"state":"active+undersized","last_fresh":"2023-06-03T20:13:57.907580+0000","last_change":"2023-06-03T20:11:54.015780+0000","last_active":"2023-06-03T20:13:57.907580+0000","last_peered":"2023-06-03T20:13:57.907580+0000","last_clean":"2023-06-03T19:56:11.745804+0000","last_became_active":"2023-06-03T20:11:54.015780+0000","last_became_peered":"2023-06-03T20:11:54.015780+0000","last_unstale":"2023-06-03T20:13:57.907580+0000","last_undegraded":"2023-06-03T20:13:57.907580+0000","last_fullsized":"2023-06-03T20:11:54.008666+0000","mapping_epoch":195,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T06:13:29.481421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6],"acting":[3,6],"avail_no_missing":["3","6"],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"11.2","version":"203'10603","reported_seq":10627,"reported_epoch":203,"state":"active+clean","last_fresh":"2023-06-03T20:14:30.097954+0000","last_change":"2023-06-03T20:11:11.788495+0000","last_active":"2023-06-03T20:14:30.097954+0000","last_peered":"2023-06-03T20:14:30.097954+0000","last_clean":"2023-06-03T20:14:30.097954+0000","last_became_active":"2023-06-03T20:11:11.788122+0000","last_became_peered":"2023-06-03T20:11:11.788122+0000","last_unstale":"2023-06-03T20:14:30.097954+0000","last_undegraded":"2023-06-03T20:14:30.097954+0000","last_fullsized":"2023-06-03T20:14:30.097954+0000","mapping_epoch":189,"log_start":"193'600","ondisk_log_start":"193'600","created":189,"last_epoch_clean":190,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T20:11:10.759955+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T20:11:10.759955+0000","last_clean_scrub_stamp":"2023-06-03T20:11:10.759955+0000","objects_scrubbed":0,"log_size":10003,"log_dups_size":0,"ondisk_log_size":10003,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T05:43:08.101794+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":180289536,"num_objects":2751,"num_object_clones":0,"num_object_copies":8253,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2751,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10603,"num_write_kb":427328,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":200,"reported_epoch":202,"state":"active+undersized","last_fresh":"2023-06-03T20:13:27.787997+0000","last_change":"2023-06-03T20:11:54.015192+0000","last_active":"2023-06-03T20:13:27.787997+0000","last_peered":"2023-06-03T20:13:27.787997+0000","last_clean":"2023-06-03T20:11:43.969614+0000","last_became_active":"2023-06-03T20:11:54.015192+0000","last_became_peered":"2023-06-03T20:11:54.015192+0000","last_unstale":"2023-06-03T20:13:27.787997+0000","last_undegraded":"2023-06-03T20:13:27.787997+0000","last_fullsized":"2023-06-03T20:11:54.005632+0000","mapping_epoch":195,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3],"acting":[6,3],"avail_no_missing":["6","3"],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":165,"reported_epoch":203,"state":"active+undersized","last_fresh":"2023-06-03T20:13:57.917785+0000","last_change":"2023-06-03T20:11:54.028687+0000","last_active":"2023-06-03T20:13:57.917785+0000","last_peered":"2023-06-03T20:13:57.917785+0000","last_clean":"2023-06-03T20:07:14.871385+0000","last_became_active":"2023-06-03T20:11:54.028687+0000","last_became_peered":"2023-06-03T20:11:54.028687+0000","last_unstale":"2023-06-03T20:13:57.917785+0000","last_undegraded":"2023-06-03T20:13:57.917785+0000","last_fullsized":"2023-06-03T20:11:54.009673+0000","mapping_epoch":195,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T04:36:58.188469+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6],"acting":[2,6],"avail_no_missing":["2","6"],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"11.1","version":"203'10007","reported_seq":10027,"reported_epoch":203,"state":"active+clean","last_fresh":"2023-06-03T20:14:28.006205+0000","last_change":"2023-06-03T20:11:11.787443+0000","last_active":"2023-06-03T20:14:28.006205+0000","last_peered":"2023-06-03T20:14:28.006205+0000","last_clean":"2023-06-03T20:14:28.006205+0000","last_became_active":"2023-06-03T20:11:11.787117+0000","last_became_peered":"2023-06-03T20:11:11.787117+0000","last_unstale":"2023-06-03T20:14:28.006205+0000","last_undegraded":"2023-06-03T20:14:28.006205+0000","last_fullsized":"2023-06-03T20:14:28.006205+0000","mapping_epoch":189,"log_start":"194'1800","ondisk_log_start":"194'1800","created":189,"last_epoch_clean":190,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T20:11:10.759955+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T20:11:10.759955+0000","last_clean_scrub_stamp":"2023-06-03T20:11:10.759955+0000","objects_scrubbed":0,"log_size":8207,"log_dups_size":0,"ondisk_log_size":8207,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":181731328,"num_objects":2773,"num_object_clones":0,"num_object_copies":8319,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2773,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10007,"num_write_kb":408960,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,0],"acting":[6,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":102,"reported_epoch":203,"state":"active+undersized","last_fresh":"2023-06-03T20:13:57.917625+0000","last_change":"2023-06-03T20:11:54.018578+0000","last_active":"2023-06-03T20:13:57.917625+0000","last_peered":"2023-06-03T20:13:57.917625+0000","last_clean":"2023-06-03T19:55:41.905941+0000","last_became_active":"2023-06-03T20:11:54.018578+0000","last_became_peered":"2023-06-03T20:11:54.018578+0000","last_unstale":"2023-06-03T20:13:57.917625+0000","last_undegraded":"2023-06-03T20:13:57.917625+0000","last_fullsized":"2023-06-03T20:11:54.009386+0000","mapping_epoch":195,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T21:13:40.683176+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7],"acting":[2,7],"avail_no_missing":["2","7"],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":149,"reported_epoch":202,"state":"active+clean+remapped","last_fresh":"2023-06-03T20:13:27.787886+0000","last_change":"2023-06-03T20:12:31.910070+0000","last_active":"2023-06-03T20:13:27.787886+0000","last_peered":"2023-06-03T20:13:27.787886+0000","last_clean":"2023-06-03T20:13:27.787886+0000","last_became_active":"2023-06-03T20:12:31.909692+0000","last_became_peered":"2023-06-03T20:12:31.909692+0000","last_unstale":"2023-06-03T20:13:27.787886+0000","last_undegraded":"2023-06-03T20:13:27.787886+0000","last_fullsized":"2023-06-03T20:13:27.787886+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T07:20:52.604735+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5],"acting":[3,5,6],"avail_no_missing":["3","5","6"],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"11.0","version":"203'10294","reported_seq":15363,"reported_epoch":203,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2023-06-03T20:14:28.144954+0000","last_change":"2023-06-03T20:13:27.251536+0000","last_active":"2023-06-03T20:14:28.144954+0000","last_peered":"2023-06-03T20:14:28.144954+0000","last_clean":"2023-06-03T20:12:29.878942+0000","last_became_active":"2023-06-03T20:12:32.063527+0000","last_became_peered":"2023-06-03T20:12:32.063527+0000","last_unstale":"2023-06-03T20:14:28.144954+0000","last_undegraded":"2023-06-03T20:13:27.243151+0000","last_fullsized":"2023-06-03T20:12:31.897715+0000","mapping_epoch":199,"log_start":"197'3900","ondisk_log_start":"197'3900","created":189,"last_epoch_clean":190,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T20:11:10.759955+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T20:11:10.759955+0000","last_clean_scrub_stamp":"2023-06-03T20:11:10.759955+0000","objects_scrubbed":0,"log_size":6394,"log_dups_size":0,"ondisk_log_size":6394,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T06:38:41.925797+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":183500800,"num_objects":2800,"num_object_clones":0,"num_object_copies":8400,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1233,"num_objects_misplaced":1299,"num_objects_unfound":0,"num_objects_dirty":2800,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10294,"num_write_kb":419008,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2518,"num_bytes_recovered":165019648,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7],"avail_no_missing":["3","6","7"],"object_location_counts":[{"shards":"3,6,7","objects":1299},{"shards":"3,7","objects":268}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":190,"reported_epoch":203,"state":"active+undersized","last_fresh":"2023-06-03T20:13:57.911390+0000","last_change":"2023-06-03T20:11:54.026212+0000","last_active":"2023-06-03T20:13:57.911390+0000","last_peered":"2023-06-03T20:13:57.911390+0000","last_clean":"2023-06-03T20:11:43.974120+0000","last_became_active":"2023-06-03T20:11:54.026212+0000","last_became_peered":"2023-06-03T20:11:54.026212+0000","last_unstale":"2023-06-03T20:13:57.911390+0000","last_undegraded":"2023-06-03T20:13:57.911390+0000","last_fullsized":"2023-06-03T20:11:54.007103+0000","mapping_epoch":195,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T03:49:35.033521+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7],"acting":[3,7],"avail_no_missing":["3","7"],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":123,"reported_epoch":203,"state":"active+undersized","last_fresh":"2023-06-03T20:13:57.908391+0000","last_change":"2023-06-03T20:11:54.026421+0000","last_active":"2023-06-03T20:13:57.908391+0000","last_peered":"2023-06-03T20:13:57.908391+0000","last_clean":"2023-06-03T20:11:43.974605+0000","last_became_active":"2023-06-03T20:11:54.026421+0000","last_became_peered":"2023-06-03T20:11:54.026421+0000","last_unstale":"2023-06-03T20:13:57.908391+0000","last_undegraded":"2023-06-03T20:13:57.908391+0000","last_fullsized":"2023-06-03T20:11:54.009062+0000","mapping_epoch":195,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T02:07:21.901385+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7],"acting":[3,7],"avail_no_missing":["3","7"],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"11.d","version":"203'10682","reported_seq":10718,"reported_epoch":203,"state":"active+undersized+degraded","last_fresh":"2023-06-03T20:14:30.058304+0000","last_change":"2023-06-03T20:11:54.061046+0000","last_active":"2023-06-03T20:14:30.058304+0000","last_peered":"2023-06-03T20:14:30.058304+0000","last_clean":"2023-06-03T20:11:52.999600+0000","last_became_active":"2023-06-03T20:11:54.040113+0000","last_became_peered":"2023-06-03T20:11:54.040113+0000","last_unstale":"2023-06-03T20:14:30.058304+0000","last_undegraded":"2023-06-03T20:11:54.028082+0000","last_fullsized":"2023-06-03T20:11:54.010095+0000","mapping_epoch":195,"log_start":"193'600","ondisk_log_start":"193'600","created":189,"last_epoch_clean":190,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T20:11:10.759955+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T20:11:10.759955+0000","last_clean_scrub_stamp":"2023-06-03T20:11:10.759955+0000","objects_scrubbed":0,"log_size":10082,"log_dups_size":0,"ondisk_log_size":10082,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T21:38:25.157657+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":178716696,"num_objects":2728,"num_object_clones":0,"num_object_copies":8184,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2728,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2728,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":10682,"num_write_kb":429057,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":131072,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6],"acting":[2,6],"avail_no_missing":["2","6"],"object_location_counts":[{"shards":"2,6","objects":2728}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":113,"reported_epoch":202,"state":"active+clean","last_fresh":"2023-06-03T20:13:36.000571+0000","last_change":"2023-06-03T20:07:19.926958+0000","last_active":"2023-06-03T20:13:36.000571+0000","last_peered":"2023-06-03T20:13:36.000571+0000","last_clean":"2023-06-03T20:13:36.000571+0000","last_became_active":"2023-06-03T20:07:19.926456+0000","last_became_peered":"2023-06-03T20:07:19.926456+0000","last_unstale":"2023-06-03T20:13:36.000571+0000","last_undegraded":"2023-06-03T20:13:36.000571+0000","last_fullsized":"2023-06-03T20:13:36.000571+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:32:20.692043+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:32:20.692043+0000","last_clean_scrub_stamp":"2023-06-03T19:32:20.692043+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T20:47:07.666782+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,2],"acting":[0,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":185,"reported_epoch":202,"state":"active+clean","last_fresh":"2023-06-03T20:13:27.803172+0000","last_change":"2023-06-03T19:56:54.512607+0000","last_active":"2023-06-03T20:13:27.803172+0000","last_peered":"2023-06-03T20:13:27.803172+0000","last_clean":"2023-06-03T20:13:27.803172+0000","last_became_active":"2023-06-03T19:56:54.512282+0000","last_became_peered":"2023-06-03T19:56:54.512282+0000","last_unstale":"2023-06-03T20:13:27.803172+0000","last_undegraded":"2023-06-03T20:13:27.803172+0000","last_fullsized":"2023-06-03T20:13:27.803172+0000","mapping_epoch":119,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":105,"reported_epoch":202,"state":"active+undersized","last_fresh":"2023-06-03T20:13:27.802429+0000","last_change":"2023-06-03T20:11:54.017774+0000","last_active":"2023-06-03T20:13:27.802429+0000","last_peered":"2023-06-03T20:13:27.802429+0000","last_clean":"2023-06-03T20:11:43.981149+0000","last_became_active":"2023-06-03T20:11:54.017774+0000","last_became_peered":"2023-06-03T20:11:54.017774+0000","last_unstale":"2023-06-03T20:13:27.802429+0000","last_undegraded":"2023-06-03T20:13:27.802429+0000","last_fullsized":"2023-06-03T20:11:54.008207+0000","mapping_epoch":195,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0],"acting":[5,0],"avail_no_missing":["5","0"],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"11.b","version":"203'10156","reported_seq":10184,"reported_epoch":203,"state":"active+undersized+degraded","last_fresh":"2023-06-03T20:14:28.111194+0000","last_change":"2023-06-03T20:11:54.019454+0000","last_active":"2023-06-03T20:14:28.111194+0000","last_peered":"2023-06-03T20:14:28.111194+0000","last_clean":"2023-06-03T20:11:52.997407+0000","last_became_active":"2023-06-03T20:11:54.019454+0000","last_became_peered":"2023-06-03T20:11:54.019454+0000","last_unstale":"2023-06-03T20:14:28.111194+0000","last_undegraded":"2023-06-03T20:11:54.014863+0000","last_fullsized":"2023-06-03T20:11:54.014785+0000","mapping_epoch":195,"log_start":"197'3700","ondisk_log_start":"197'3700","created":189,"last_epoch_clean":190,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T20:11:10.759955+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T20:11:10.759955+0000","last_clean_scrub_stamp":"2023-06-03T20:11:10.759955+0000","objects_scrubbed":0,"log_size":6456,"log_dups_size":0,"ondisk_log_size":6456,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T05:26:41.801270+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":182714368,"num_objects":2788,"num_object_clones":0,"num_object_copies":8364,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2788,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2788,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10156,"num_write_kb":414208,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0],"acting":[3,0],"avail_no_missing":["3","0"],"object_location_counts":[{"shards":"0,3","objects":2788}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":192,"reported_epoch":202,"state":"active+clean","last_fresh":"2023-06-03T20:13:27.802574+0000","last_change":"2023-06-03T20:12:30.910088+0000","last_active":"2023-06-03T20:13:27.802574+0000","last_peered":"2023-06-03T20:13:27.802574+0000","last_clean":"2023-06-03T20:13:27.802574+0000","last_became_active":"2023-06-03T20:12:30.909574+0000","last_became_peered":"2023-06-03T20:12:30.909574+0000","last_unstale":"2023-06-03T20:13:27.802574+0000","last_undegraded":"2023-06-03T20:13:27.802574+0000","last_fullsized":"2023-06-03T20:13:27.802574+0000","mapping_epoch":198,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":199,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:32:20.692043+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:32:20.692043+0000","last_clean_scrub_stamp":"2023-06-03T19:32:20.692043+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":75,"reported_epoch":203,"state":"active+undersized","last_fresh":"2023-06-03T20:13:57.918472+0000","last_change":"2023-06-03T20:11:54.020579+0000","last_active":"2023-06-03T20:13:57.918472+0000","last_peered":"2023-06-03T20:13:57.918472+0000","last_clean":"2023-06-03T19:46:38.226313+0000","last_became_active":"2023-06-03T20:11:54.020579+0000","last_became_peered":"2023-06-03T20:11:54.020579+0000","last_unstale":"2023-06-03T20:13:57.918472+0000","last_undegraded":"2023-06-03T20:13:57.918472+0000","last_fullsized":"2023-06-03T20:11:54.008467+0000","mapping_epoch":195,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T21:09:23.253653+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6],"acting":[0,6],"avail_no_missing":["0","6"],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":121,"reported_epoch":202,"state":"active+clean","last_fresh":"2023-06-03T20:13:34.940137+0000","last_change":"2023-06-03T20:12:30.911946+0000","last_active":"2023-06-03T20:13:34.940137+0000","last_peered":"2023-06-03T20:13:34.940137+0000","last_clean":"2023-06-03T20:13:34.940137+0000","last_became_active":"2023-06-03T20:12:30.911513+0000","last_became_peered":"2023-06-03T20:12:30.911513+0000","last_unstale":"2023-06-03T20:13:34.940137+0000","last_undegraded":"2023-06-03T20:13:34.940137+0000","last_fullsized":"2023-06-03T20:13:34.940137+0000","mapping_epoch":198,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":199,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T07:22:01.480443+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,7],"acting":[2,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"11.8","version":"203'10492","reported_seq":10516,"reported_epoch":203,"state":"active+clean","last_fresh":"2023-06-03T20:14:28.154958+0000","last_change":"2023-06-03T20:11:11.793649+0000","last_active":"2023-06-03T20:14:28.154958+0000","last_peered":"2023-06-03T20:14:28.154958+0000","last_clean":"2023-06-03T20:14:28.154958+0000","last_became_active":"2023-06-03T20:11:11.793292+0000","last_became_peered":"2023-06-03T20:11:11.793292+0000","last_unstale":"2023-06-03T20:14:28.154958+0000","last_undegraded":"2023-06-03T20:14:28.154958+0000","last_fullsized":"2023-06-03T20:14:28.154958+0000","mapping_epoch":189,"log_start":"197'4100","ondisk_log_start":"197'4100","created":189,"last_epoch_clean":190,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T20:11:10.759955+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T20:11:10.759955+0000","last_clean_scrub_stamp":"2023-06-03T20:11:10.759955+0000","objects_scrubbed":0,"log_size":6392,"log_dups_size":0,"ondisk_log_size":6392,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T07:27:16.604437+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":186384384,"num_objects":2844,"num_object_clones":0,"num_object_copies":8532,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2844,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10492,"num_write_kb":426752,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,5],"acting":[3,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":182,"reported_epoch":202,"state":"active+clean","last_fresh":"2023-06-03T20:13:27.793875+0000","last_change":"2023-06-03T19:56:54.504859+0000","last_active":"2023-06-03T20:13:27.793875+0000","last_peered":"2023-06-03T20:13:27.793875+0000","last_clean":"2023-06-03T20:13:27.793875+0000","last_became_active":"2023-06-03T19:56:54.504455+0000","last_became_peered":"2023-06-03T19:56:54.504455+0000","last_unstale":"2023-06-03T20:13:27.793875+0000","last_undegraded":"2023-06-03T20:13:27.793875+0000","last_fullsized":"2023-06-03T20:13:27.793875+0000","mapping_epoch":119,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:32:20.692043+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:32:20.692043+0000","last_clean_scrub_stamp":"2023-06-03T19:32:20.692043+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T06:25:45.522100+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":133,"reported_epoch":202,"state":"active+clean","last_fresh":"2023-06-03T20:13:27.793963+0000","last_change":"2023-06-03T20:07:19.906703+0000","last_active":"2023-06-03T20:13:27.793963+0000","last_peered":"2023-06-03T20:13:27.793963+0000","last_clean":"2023-06-03T20:13:27.793963+0000","last_became_active":"2023-06-03T20:07:19.906373+0000","last_became_peered":"2023-06-03T20:07:19.906373+0000","last_unstale":"2023-06-03T20:13:27.793963+0000","last_undegraded":"2023-06-03T20:13:27.793963+0000","last_fullsized":"2023-06-03T20:13:27.793963+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T21:06:05.945328+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6,0],"acting":[2,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":105,"reported_epoch":202,"state":"active+clean","last_fresh":"2023-06-03T20:13:27.802783+0000","last_change":"2023-06-03T20:07:19.906757+0000","last_active":"2023-06-03T20:13:27.802783+0000","last_peered":"2023-06-03T20:13:27.802783+0000","last_clean":"2023-06-03T20:13:27.802783+0000","last_became_active":"2023-06-03T20:07:19.906380+0000","last_became_peered":"2023-06-03T20:07:19.906380+0000","last_unstale":"2023-06-03T20:13:27.802783+0000","last_undegraded":"2023-06-03T20:13:27.802783+0000","last_fullsized":"2023-06-03T20:13:27.802783+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,7],"acting":[5,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"11.a","version":"203'10279","reported_seq":10303,"reported_epoch":203,"state":"active+clean","last_fresh":"2023-06-03T20:14:28.173287+0000","last_change":"2023-06-03T20:11:11.793791+0000","last_active":"2023-06-03T20:14:28.173287+0000","last_peered":"2023-06-03T20:14:28.173287+0000","last_clean":"2023-06-03T20:14:28.173287+0000","last_became_active":"2023-06-03T20:11:11.793127+0000","last_became_peered":"2023-06-03T20:11:11.793127+0000","last_unstale":"2023-06-03T20:14:28.173287+0000","last_undegraded":"2023-06-03T20:14:28.173287+0000","last_fullsized":"2023-06-03T20:14:28.173287+0000","mapping_epoch":189,"log_start":"197'3800","ondisk_log_start":"197'3800","created":189,"last_epoch_clean":190,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T20:11:10.759955+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T20:11:10.759955+0000","last_clean_scrub_stamp":"2023-06-03T20:11:10.759955+0000","objects_scrubbed":0,"log_size":6479,"log_dups_size":0,"ondisk_log_size":6479,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T22:21:19.854335+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":182910976,"num_objects":2791,"num_object_clones":0,"num_object_copies":8373,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2791,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10279,"num_write_kb":418240,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,5],"acting":[3,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":174,"reported_epoch":202,"state":"active+clean","last_fresh":"2023-06-03T20:13:27.802388+0000","last_change":"2023-06-03T19:56:11.752151+0000","last_active":"2023-06-03T20:13:27.802388+0000","last_peered":"2023-06-03T20:13:27.802388+0000","last_clean":"2023-06-03T20:13:27.802388+0000","last_became_active":"2023-06-03T19:56:11.751767+0000","last_became_peered":"2023-06-03T19:56:11.751767+0000","last_unstale":"2023-06-03T20:13:27.802388+0000","last_undegraded":"2023-06-03T20:13:27.802388+0000","last_fullsized":"2023-06-03T20:13:27.802388+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":115,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:32:20.692043+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:32:20.692043+0000","last_clean_scrub_stamp":"2023-06-03T19:32:20.692043+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"72'139","reported_seq":403,"reported_epoch":202,"state":"active+clean","last_fresh":"2023-06-03T20:13:27.790964+0000","last_change":"2023-06-03T20:07:47.872942+0000","last_active":"2023-06-03T20:13:27.790964+0000","last_peered":"2023-06-03T20:13:27.790964+0000","last_clean":"2023-06-03T20:13:27.790964+0000","last_became_active":"2023-06-03T20:07:47.872426+0000","last_became_peered":"2023-06-03T20:07:47.872426+0000","last_unstale":"2023-06-03T20:13:27.790964+0000","last_undegraded":"2023-06-03T20:13:27.790964+0000","last_fullsized":"2023-06-03T20:13:27.790964+0000","mapping_epoch":172,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":173,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:27:28.100808+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:27:28.100808+0000","last_clean_scrub_stamp":"2023-06-03T19:27:28.100808+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":155,"reported_epoch":202,"state":"active+clean","last_fresh":"2023-06-03T20:13:32.905957+0000","last_change":"2023-06-03T19:56:54.509571+0000","last_active":"2023-06-03T20:13:32.905957+0000","last_peered":"2023-06-03T20:13:32.905957+0000","last_clean":"2023-06-03T20:13:32.905957+0000","last_became_active":"2023-06-03T19:56:54.509213+0000","last_became_peered":"2023-06-03T19:56:54.509213+0000","last_unstale":"2023-06-03T20:13:32.905957+0000","last_undegraded":"2023-06-03T20:13:32.905957+0000","last_fullsized":"2023-06-03T20:13:32.905957+0000","mapping_epoch":119,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T21:26:56.003203+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":179,"reported_epoch":202,"state":"active+clean+remapped","last_fresh":"2023-06-03T20:13:27.794290+0000","last_change":"2023-06-03T20:12:31.939716+0000","last_active":"2023-06-03T20:13:27.794290+0000","last_peered":"2023-06-03T20:13:27.794290+0000","last_clean":"2023-06-03T20:13:27.794290+0000","last_became_active":"2023-06-03T20:12:31.938876+0000","last_became_peered":"2023-06-03T20:12:31.938876+0000","last_unstale":"2023-06-03T20:13:27.794290+0000","last_undegraded":"2023-06-03T20:13:27.794290+0000","last_fullsized":"2023-06-03T20:13:27.794290+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0],"acting":[7,0,6],"avail_no_missing":["7","0","6"],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"11.9","version":"203'10495","reported_seq":10524,"reported_epoch":203,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-06-03T20:14:28.163593+0000","last_change":"2023-06-03T20:12:32.085781+0000","last_active":"2023-06-03T20:14:28.163593+0000","last_peered":"2023-06-03T20:14:28.163593+0000","last_clean":"2023-06-03T20:12:29.842853+0000","last_became_active":"2023-06-03T20:12:32.076473+0000","last_became_peered":"2023-06-03T20:12:32.076473+0000","last_unstale":"2023-06-03T20:14:28.163593+0000","last_undegraded":"2023-06-03T20:12:31.934803+0000","last_fullsized":"2023-06-03T20:12:31.898014+0000","mapping_epoch":199,"log_start":"197'4100","ondisk_log_start":"197'4100","created":189,"last_epoch_clean":190,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T20:11:10.759955+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T20:11:10.759955+0000","last_clean_scrub_stamp":"2023-06-03T20:11:10.759955+0000","objects_scrubbed":0,"log_size":6395,"log_dups_size":0,"ondisk_log_size":6395,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T00:47:40.082595+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":185794560,"num_objects":2835,"num_object_clones":0,"num_object_copies":8505,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5168,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2835,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10495,"num_write_kb":426560,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,7],"acting":[3,0],"avail_no_missing":["3","0","6"],"object_location_counts":[{"shards":"0,3","objects":-2333},{"shards":"0,3,6","objects":1337}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":186,"reported_epoch":202,"state":"active+clean","last_fresh":"2023-06-03T20:13:27.790929+0000","last_change":"2023-06-03T20:07:47.855767+0000","last_active":"2023-06-03T20:13:27.790929+0000","last_peered":"2023-06-03T20:13:27.790929+0000","last_clean":"2023-06-03T20:13:27.790929+0000","last_became_active":"2023-06-03T20:07:47.855444+0000","last_became_peered":"2023-06-03T20:07:47.855444+0000","last_unstale":"2023-06-03T20:13:27.790929+0000","last_undegraded":"2023-06-03T20:13:27.790929+0000","last_fullsized":"2023-06-03T20:13:27.790929+0000","mapping_epoch":172,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":173,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:32:20.692043+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:32:20.692043+0000","last_clean_scrub_stamp":"2023-06-03T19:32:20.692043+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,3],"acting":[7,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":186,"reported_epoch":203,"state":"active+undersized","last_fresh":"2023-06-03T20:13:57.917610+0000","last_change":"2023-06-03T20:11:54.017331+0000","last_active":"2023-06-03T20:13:57.917610+0000","last_peered":"2023-06-03T20:13:57.917610+0000","last_clean":"2023-06-03T20:11:43.977931+0000","last_became_active":"2023-06-03T20:11:54.017331+0000","last_became_peered":"2023-06-03T20:11:54.017331+0000","last_unstale":"2023-06-03T20:13:57.917610+0000","last_undegraded":"2023-06-03T20:13:57.917610+0000","last_fullsized":"2023-06-03T20:11:54.007149+0000","mapping_epoch":195,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T04:51:15.533105+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3],"acting":[0,3],"avail_no_missing":["0","3"],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":189,"reported_epoch":202,"state":"active+clean","last_fresh":"2023-06-03T20:13:27.803437+0000","last_change":"2023-06-03T20:07:19.925864+0000","last_active":"2023-06-03T20:13:27.803437+0000","last_peered":"2023-06-03T20:13:27.803437+0000","last_clean":"2023-06-03T20:13:27.803437+0000","last_became_active":"2023-06-03T20:07:19.925541+0000","last_became_peered":"2023-06-03T20:07:19.925541+0000","last_unstale":"2023-06-03T20:13:27.803437+0000","last_undegraded":"2023-06-03T20:13:27.803437+0000","last_fullsized":"2023-06-03T20:13:27.803437+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T00:10:11.790831+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,7],"acting":[0,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"11.c","version":"203'10502","reported_seq":10531,"reported_epoch":203,"state":"active+undersized+degraded","last_fresh":"2023-06-03T20:14:30.115288+0000","last_change":"2023-06-03T20:11:54.021153+0000","last_active":"2023-06-03T20:14:30.115288+0000","last_peered":"2023-06-03T20:14:30.115288+0000","last_clean":"2023-06-03T20:11:53.003830+0000","last_became_active":"2023-06-03T20:11:54.021153+0000","last_became_peered":"2023-06-03T20:11:54.021153+0000","last_unstale":"2023-06-03T20:14:30.115288+0000","last_undegraded":"2023-06-03T20:11:54.009999+0000","last_fullsized":"2023-06-03T20:11:54.009855+0000","mapping_epoch":195,"log_start":"193'500","ondisk_log_start":"193'500","created":189,"last_epoch_clean":190,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T20:11:10.759955+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T20:11:10.759955+0000","last_clean_scrub_stamp":"2023-06-03T20:11:10.759955+0000","objects_scrubbed":0,"log_size":10002,"log_dups_size":0,"ondisk_log_size":10002,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T00:13:39.997844+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":179175424,"num_objects":2734,"num_object_clones":0,"num_object_copies":8202,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2734,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2734,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10502,"num_write_kb":423552,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7],"acting":[2,7],"avail_no_missing":["2","7"],"object_location_counts":[{"shards":"2,7","objects":2734}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":195,"reported_epoch":202,"state":"active+undersized","last_fresh":"2023-06-03T20:13:27.793964+0000","last_change":"2023-06-03T20:11:54.012820+0000","last_active":"2023-06-03T20:13:27.793964+0000","last_peered":"2023-06-03T20:13:27.793964+0000","last_clean":"2023-06-03T20:11:43.973280+0000","last_became_active":"2023-06-03T20:11:54.012820+0000","last_became_peered":"2023-06-03T20:11:54.012820+0000","last_unstale":"2023-06-03T20:13:27.793964+0000","last_undegraded":"2023-06-03T20:13:27.793964+0000","last_fullsized":"2023-06-03T20:11:54.011295+0000","mapping_epoch":195,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:32:20.692043+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:32:20.692043+0000","last_clean_scrub_stamp":"2023-06-03T19:32:20.692043+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0],"acting":[7,0],"avail_no_missing":["7","0"],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":133,"reported_epoch":202,"state":"active+clean","last_fresh":"2023-06-03T20:13:34.940157+0000","last_change":"2023-06-03T20:07:19.915065+0000","last_active":"2023-06-03T20:13:34.940157+0000","last_peered":"2023-06-03T20:13:34.940157+0000","last_clean":"2023-06-03T20:13:34.940157+0000","last_became_active":"2023-06-03T20:07:19.914699+0000","last_became_peered":"2023-06-03T20:07:19.914699+0000","last_unstale":"2023-06-03T20:13:34.940157+0000","last_undegraded":"2023-06-03T20:13:34.940157+0000","last_fullsized":"2023-06-03T20:13:34.940157+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T07:16:09.077184+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,0],"acting":[2,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":151,"reported_epoch":203,"state":"active+clean","last_fresh":"2023-06-03T20:13:57.918493+0000","last_change":"2023-06-03T19:47:06.733532+0000","last_active":"2023-06-03T20:13:57.918493+0000","last_peered":"2023-06-03T20:13:57.918493+0000","last_clean":"2023-06-03T20:13:57.918493+0000","last_became_active":"2023-06-03T19:47:06.733131+0000","last_became_peered":"2023-06-03T19:47:06.733131+0000","last_unstale":"2023-06-03T20:13:57.918493+0000","last_undegraded":"2023-06-03T20:13:57.918493+0000","last_fullsized":"2023-06-03T20:13:57.918493+0000","mapping_epoch":86,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":87,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T23:46:03.788646+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"11.f","version":"203'10240","reported_seq":10262,"reported_epoch":203,"state":"active+clean","last_fresh":"2023-06-03T20:14:28.166404+0000","last_change":"2023-06-03T20:11:11.794557+0000","last_active":"2023-06-03T20:14:28.166404+0000","last_peered":"2023-06-03T20:14:28.166404+0000","last_clean":"2023-06-03T20:14:28.166404+0000","last_became_active":"2023-06-03T20:11:11.794234+0000","last_became_peered":"2023-06-03T20:11:11.794234+0000","last_unstale":"2023-06-03T20:14:28.166404+0000","last_undegraded":"2023-06-03T20:14:28.166404+0000","last_fullsized":"2023-06-03T20:14:28.166404+0000","mapping_epoch":189,"log_start":"196'3500","ondisk_log_start":"196'3500","created":189,"last_epoch_clean":190,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T20:11:10.759955+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T20:11:10.759955+0000","last_clean_scrub_stamp":"2023-06-03T20:11:10.759955+0000","objects_scrubbed":0,"log_size":6740,"log_dups_size":0,"ondisk_log_size":6740,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":182321152,"num_objects":2782,"num_object_clones":0,"num_object_copies":8346,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2782,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10240,"num_write_kb":416704,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,0],"acting":[7,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.7","version":"60'2","reported_seq":160,"reported_epoch":203,"state":"active+clean","last_fresh":"2023-06-03T20:13:57.908502+0000","last_change":"2023-06-03T19:56:54.510148+0000","last_active":"2023-06-03T20:13:57.908502+0000","last_peered":"2023-06-03T20:13:57.908502+0000","last_clean":"2023-06-03T20:13:57.908502+0000","last_became_active":"2023-06-03T19:56:54.509771+0000","last_became_peered":"2023-06-03T19:56:54.509771+0000","last_unstale":"2023-06-03T20:13:57.908502+0000","last_undegraded":"2023-06-03T20:13:57.908502+0000","last_fullsized":"2023-06-03T20:13:57.908502+0000","mapping_epoch":119,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T00:52:42.807432+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":146,"reported_epoch":202,"state":"active+undersized","last_fresh":"2023-06-03T20:13:27.788477+0000","last_change":"2023-06-03T20:11:54.015128+0000","last_active":"2023-06-03T20:13:27.788477+0000","last_peered":"2023-06-03T20:13:27.788477+0000","last_clean":"2023-06-03T20:11:43.976310+0000","last_became_active":"2023-06-03T20:11:54.015128+0000","last_became_peered":"2023-06-03T20:11:54.015128+0000","last_unstale":"2023-06-03T20:13:27.788477+0000","last_undegraded":"2023-06-03T20:13:27.788477+0000","last_fullsized":"2023-06-03T20:11:54.006226+0000","mapping_epoch":195,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:32:20.692043+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:32:20.692043+0000","last_clean_scrub_stamp":"2023-06-03T19:32:20.692043+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3],"acting":[6,3],"avail_no_missing":["6","3"],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":157,"reported_epoch":202,"state":"active+clean","last_fresh":"2023-06-03T20:13:27.785968+0000","last_change":"2023-06-03T19:56:54.507670+0000","last_active":"2023-06-03T20:13:27.785968+0000","last_peered":"2023-06-03T20:13:27.785968+0000","last_clean":"2023-06-03T20:13:27.785968+0000","last_became_active":"2023-06-03T19:56:54.507293+0000","last_became_peered":"2023-06-03T19:56:54.507293+0000","last_unstale":"2023-06-03T20:13:27.785968+0000","last_undegraded":"2023-06-03T20:13:27.785968+0000","last_fullsized":"2023-06-03T20:13:27.785968+0000","mapping_epoch":119,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T01:49:19.966054+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"11.e","version":"203'10523","reported_seq":10551,"reported_epoch":203,"state":"active+undersized+degraded","last_fresh":"2023-06-03T20:14:28.107231+0000","last_change":"2023-06-03T20:11:54.013399+0000","last_active":"2023-06-03T20:14:28.107231+0000","last_peered":"2023-06-03T20:14:28.107231+0000","last_clean":"2023-06-03T20:11:52.975326+0000","last_became_active":"2023-06-03T20:11:54.013399+0000","last_became_peered":"2023-06-03T20:11:54.013399+0000","last_unstale":"2023-06-03T20:14:28.107231+0000","last_undegraded":"2023-06-03T20:11:54.011155+0000","last_fullsized":"2023-06-03T20:11:54.010995+0000","mapping_epoch":195,"log_start":"197'3800","ondisk_log_start":"197'3800","created":189,"last_epoch_clean":190,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T20:11:10.759955+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T20:11:10.759955+0000","last_clean_scrub_stamp":"2023-06-03T20:11:10.759955+0000","objects_scrubbed":0,"log_size":6723,"log_dups_size":0,"ondisk_log_size":6723,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":181862400,"num_objects":2775,"num_object_clones":0,"num_object_copies":8325,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2775,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2775,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10523,"num_write_kb":425536,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0],"acting":[7,0],"avail_no_missing":["7","0"],"object_location_counts":[{"shards":"0,7","objects":2775}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":174,"reported_epoch":202,"state":"active+clean","last_fresh":"2023-06-03T20:13:27.788149+0000","last_change":"2023-06-03T19:56:11.752730+0000","last_active":"2023-06-03T20:13:27.788149+0000","last_peered":"2023-06-03T20:13:27.788149+0000","last_clean":"2023-06-03T20:13:27.788149+0000","last_became_active":"2023-06-03T19:56:11.752441+0000","last_became_peered":"2023-06-03T19:56:11.752441+0000","last_unstale":"2023-06-03T20:13:27.788149+0000","last_undegraded":"2023-06-03T20:13:27.788149+0000","last_fullsized":"2023-06-03T20:13:27.788149+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":115,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:32:20.692043+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:32:20.692043+0000","last_clean_scrub_stamp":"2023-06-03T19:32:20.692043+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"60'1","reported_seq":202,"reported_epoch":202,"state":"active+undersized+degraded","last_fresh":"2023-06-03T20:13:27.803205+0000","last_change":"2023-06-03T20:11:54.014758+0000","last_active":"2023-06-03T20:13:27.803205+0000","last_peered":"2023-06-03T20:13:27.803205+0000","last_clean":"2023-06-03T20:11:43.980210+0000","last_became_active":"2023-06-03T20:11:54.014758+0000","last_became_peered":"2023-06-03T20:11:54.014758+0000","last_unstale":"2023-06-03T20:13:27.803205+0000","last_undegraded":"2023-06-03T20:11:54.011442+0000","last_fullsized":"2023-06-03T20:11:54.011320+0000","mapping_epoch":195,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T20:43:34.505545+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7],"acting":[0,7],"avail_no_missing":["0","7"],"object_location_counts":[{"shards":"0,7","objects":1}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":72,"reported_epoch":202,"state":"active+undersized","last_fresh":"2023-06-03T20:13:27.802075+0000","last_change":"2023-06-03T20:11:54.015754+0000","last_active":"2023-06-03T20:13:27.802075+0000","last_peered":"2023-06-03T20:13:27.802075+0000","last_clean":"2023-06-03T19:34:16.776254+0000","last_became_active":"2023-06-03T20:11:54.015754+0000","last_became_peered":"2023-06-03T20:11:54.015754+0000","last_unstale":"2023-06-03T20:13:27.802075+0000","last_undegraded":"2023-06-03T20:13:27.802075+0000","last_fullsized":"2023-06-03T20:11:54.008549+0000","mapping_epoch":195,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6],"acting":[5,6],"avail_no_missing":["5","6"],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1b","version":"60'1","reported_seq":172,"reported_epoch":202,"state":"active+undersized","last_fresh":"2023-06-03T20:13:27.802627+0000","last_change":"2023-06-03T20:11:54.016103+0000","last_active":"2023-06-03T20:13:27.802627+0000","last_peered":"2023-06-03T20:13:27.802627+0000","last_clean":"2023-06-03T20:11:43.979629+0000","last_became_active":"2023-06-03T20:11:54.016103+0000","last_became_peered":"2023-06-03T20:11:54.016103+0000","last_unstale":"2023-06-03T20:13:27.802627+0000","last_undegraded":"2023-06-03T20:13:27.802627+0000","last_fullsized":"2023-06-03T20:11:54.009298+0000","mapping_epoch":195,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T04:35:48.454086+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7],"acting":[0,7],"avail_no_missing":["0","7"],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":202,"reported_epoch":202,"state":"active+clean","last_fresh":"2023-06-03T20:13:39.957340+0000","last_change":"2023-06-03T20:07:19.914985+0000","last_active":"2023-06-03T20:13:39.957340+0000","last_peered":"2023-06-03T20:13:39.957340+0000","last_clean":"2023-06-03T20:13:39.957340+0000","last_became_active":"2023-06-03T20:07:19.914624+0000","last_became_peered":"2023-06-03T20:07:19.914624+0000","last_unstale":"2023-06-03T20:13:39.957340+0000","last_undegraded":"2023-06-03T20:13:39.957340+0000","last_fullsized":"2023-06-03T20:13:39.957340+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T07:02:50.515170+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"60'2","reported_seq":220,"reported_epoch":202,"state":"active+clean+remapped","last_fresh":"2023-06-03T20:13:27.802295+0000","last_change":"2023-06-03T20:12:31.955413+0000","last_active":"2023-06-03T20:13:27.802295+0000","last_peered":"2023-06-03T20:13:27.802295+0000","last_clean":"2023-06-03T20:13:27.802295+0000","last_became_active":"2023-06-03T20:12:31.909460+0000","last_became_peered":"2023-06-03T20:12:31.909460+0000","last_unstale":"2023-06-03T20:13:27.802295+0000","last_undegraded":"2023-06-03T20:13:27.802295+0000","last_fullsized":"2023-06-03T20:13:27.802295+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":1,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7],"acting":[5,7,6],"avail_no_missing":["5","6","7"],"object_location_counts":[{"shards":"5,6,7","objects":1}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":174,"reported_epoch":203,"state":"active+undersized","last_fresh":"2023-06-03T20:13:57.910426+0000","last_change":"2023-06-03T20:11:54.028489+0000","last_active":"2023-06-03T20:13:57.910426+0000","last_peered":"2023-06-03T20:13:57.910426+0000","last_clean":"2023-06-03T20:07:14.870985+0000","last_became_active":"2023-06-03T20:11:54.028489+0000","last_became_peered":"2023-06-03T20:11:54.028489+0000","last_unstale":"2023-06-03T20:13:57.910426+0000","last_undegraded":"2023-06-03T20:13:57.910426+0000","last_fullsized":"2023-06-03T20:11:54.011931+0000","mapping_epoch":195,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T04:24:31.291273+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2],"acting":[3,2],"avail_no_missing":["3","2"],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1d","version":"60'2","reported_seq":192,"reported_epoch":202,"state":"active+clean+remapped","last_fresh":"2023-06-03T20:13:27.803025+0000","last_change":"2023-06-03T20:12:32.001796+0000","last_active":"2023-06-03T20:13:27.803025+0000","last_peered":"2023-06-03T20:13:27.803025+0000","last_clean":"2023-06-03T20:13:27.803025+0000","last_became_active":"2023-06-03T20:12:31.909615+0000","last_became_peered":"2023-06-03T20:12:31.909615+0000","last_unstale":"2023-06-03T20:13:27.803025+0000","last_undegraded":"2023-06-03T20:13:27.803025+0000","last_fullsized":"2023-06-03T20:13:27.803025+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":1,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3],"acting":[5,3,6],"avail_no_missing":["5","3","6"],"object_location_counts":[{"shards":"3,5,6","objects":1}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":120,"reported_epoch":203,"state":"active+undersized","last_fresh":"2023-06-03T20:13:57.912562+0000","last_change":"2023-06-03T20:11:54.015791+0000","last_active":"2023-06-03T20:13:57.912562+0000","last_peered":"2023-06-03T20:13:57.912562+0000","last_clean":"2023-06-03T19:56:11.746795+0000","last_became_active":"2023-06-03T20:11:54.015791+0000","last_became_peered":"2023-06-03T20:11:54.015791+0000","last_unstale":"2023-06-03T20:13:57.912562+0000","last_undegraded":"2023-06-03T20:13:57.912562+0000","last_fullsized":"2023-06-03T20:11:54.007458+0000","mapping_epoch":195,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T19:54:52.259092+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0],"acting":[3,0],"avail_no_missing":["3","0"],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"60'1","reported_seq":195,"reported_epoch":202,"state":"active+clean","last_fresh":"2023-06-03T20:13:27.789072+0000","last_change":"2023-06-03T20:12:30.909386+0000","last_active":"2023-06-03T20:13:27.789072+0000","last_peered":"2023-06-03T20:13:27.789072+0000","last_clean":"2023-06-03T20:13:27.789072+0000","last_became_active":"2023-06-03T20:12:30.908991+0000","last_became_peered":"2023-06-03T20:12:30.908991+0000","last_unstale":"2023-06-03T20:13:27.789072+0000","last_undegraded":"2023-06-03T20:13:27.789072+0000","last_fullsized":"2023-06-03T20:13:27.789072+0000","mapping_epoch":198,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":199,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T04:33:12.721959+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,7],"acting":[3,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":198,"reported_epoch":202,"state":"active+clean","last_fresh":"2023-06-03T20:13:52.991276+0000","last_change":"2023-06-03T19:56:54.503416+0000","last_active":"2023-06-03T20:13:52.991276+0000","last_peered":"2023-06-03T20:13:52.991276+0000","last_clean":"2023-06-03T20:13:52.991276+0000","last_became_active":"2023-06-03T19:56:54.503018+0000","last_became_peered":"2023-06-03T19:56:54.503018+0000","last_unstale":"2023-06-03T20:13:52.991276+0000","last_undegraded":"2023-06-03T20:13:52.991276+0000","last_fullsized":"2023-06-03T20:13:52.991276+0000","mapping_epoch":119,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T20:14:00.330616+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"60'4","reported_seq":169,"reported_epoch":203,"state":"active+clean","last_fresh":"2023-06-03T20:13:57.918083+0000","last_change":"2023-06-03T19:56:11.748116+0000","last_active":"2023-06-03T20:13:57.918083+0000","last_peered":"2023-06-03T20:13:57.918083+0000","last_clean":"2023-06-03T20:13:57.918083+0000","last_became_active":"2023-06-03T19:56:11.747636+0000","last_became_peered":"2023-06-03T19:56:11.747636+0000","last_unstale":"2023-06-03T20:13:57.918083+0000","last_undegraded":"2023-06-03T20:13:57.918083+0000","last_fullsized":"2023-06-03T20:13:57.918083+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":115,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T06:57:48.052055+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":16,"reported_epoch":202,"state":"active+clean+remapped","last_fresh":"2023-06-03T20:13:27.786097+0000","last_change":"2023-06-03T20:12:32.093827+0000","last_active":"2023-06-03T20:13:27.786097+0000","last_peered":"2023-06-03T20:13:27.786097+0000","last_clean":"2023-06-03T20:13:27.786097+0000","last_became_active":"2023-06-03T20:12:32.093400+0000","last_became_peered":"2023-06-03T20:12:32.093400+0000","last_unstale":"2023-06-03T20:13:27.786097+0000","last_undegraded":"2023-06-03T20:13:27.786097+0000","last_fullsized":"2023-06-03T20:13:27.786097+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T23:22:58.054742+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2],"acting":[3,2,6],"avail_no_missing":["3","2","6"],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]}],"pool_stats":[{"poolid":11,"num_pg":16,"stat_sum":{"num_bytes":2937520152,"num_objects":44824,"num_object_clones":0,"num_object_copies":134472,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":24500,"num_objects_misplaced":2587,"num_objects_unfound":0,"num_objects_dirty":44824,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":166228,"num_write_kb":6753601,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5045,"num_bytes_recovered":330629120,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":9367986176,"data_stored":9367978032,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":120128,"ondisk_log_size":120128,"up":42,"acting":39,"num_store_stats":7},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":80,"acting":87,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5486,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":12,"num_objects_misplaced":2,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":62,"num_write_kb":30,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16458,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":30319,"internal_metadata":0},"log_size":56,"ondisk_log_size":56,"up":78,"acting":80,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":3,"num_store_stats":3},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":22,"acting":22,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":48,"seq":206158430729,"num_pgs":45,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1944752,"kb_used_data":1360232,"kb_used_omap":0,"kb_used_meta":584320,"kb_avail":91800400,"statfs":{"total":95995035648,"available":94003609600,"internally_reserved":0,"allocated":1392877568,"data_stored":1392367988,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":598343680},"hb_peers":[0,1,2,3,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[2,0,0,2,4],"upper_bound":32},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 3 20:13:58 2023","interfaces":[{"interface":"back","average":{"1min":2.3039999999999998,"5min":1.6259999999999999,"15min":1.7310000000000001},"min":{"1min":0.25800000000000001,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":7.2300000000000004,"5min":11.311999999999999,"15min":48.338999999999999},"last":0.96699999999999997},{"interface":"front","average":{"1min":1.6930000000000001,"5min":1.492,"15min":1.5840000000000001},"min":{"1min":0.28000000000000003,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":7.2670000000000003,"5min":7.2670000000000003,"15min":46.412999999999997},"last":0.65200000000000002}]},{"osd":1,"last update":"Sat Jun 3 20:13:45 2023","interfaces":[{"interface":"back","average":{"1min":1.734,"5min":1.3859999999999999,"15min":1.716},"min":{"1min":0.31,"5min":0.31,"15min":0.29699999999999999},"max":{"1min":5.4809999999999999,"5min":11.4,"15min":91.811000000000007},"last":0.89200000000000002},{"interface":"front","average":{"1min":1.3400000000000001,"5min":1.337,"15min":1.8380000000000001},"min":{"1min":0.41199999999999998,"5min":0.34000000000000002,"15min":0.32200000000000001},"max":{"1min":3.649,"5min":4.9850000000000003,"15min":91.695999999999998},"last":0.69599999999999995}]},{"osd":2,"last update":"Sat Jun 3 20:13:45 2023","interfaces":[{"interface":"back","average":{"1min":2.7629999999999999,"5min":4.6959999999999997,"15min":2.52},"min":{"1min":0.53000000000000003,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":5.5199999999999996,"5min":286.83600000000001,"15min":286.83600000000001},"last":0.92200000000000004},{"interface":"front","average":{"1min":1.8069999999999999,"5min":4.4630000000000001,"15min":2.5649999999999999},"min":{"1min":0.38900000000000001,"5min":0.375,"15min":0.33500000000000002},"max":{"1min":4.6660000000000004,"5min":287.32900000000001,"15min":287.32900000000001},"last":0.67500000000000004}]},{"osd":3,"last update":"Sat Jun 3 20:13:37 2023","interfaces":[{"interface":"back","average":{"1min":2.2000000000000002,"5min":2.871,"15min":2.2959999999999998},"min":{"1min":0.38700000000000001,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":7.6840000000000002,"5min":98.763000000000005,"15min":98.763000000000005},"last":265.40499999999997},{"interface":"front","average":{"1min":1.464,"5min":2.802,"15min":2.3809999999999998},"min":{"1min":0.53700000000000003,"5min":0.34399999999999997,"15min":0.34399999999999997},"max":{"1min":3.7879999999999998,"5min":97.882999999999996,"15min":97.882999999999996},"last":265.43599999999998}]},{"osd":4,"last update":"Sat Jun 3 20:10:54 2023","interfaces":[{"interface":"back","average":{"1min":1.417,"5min":1.8879999999999999,"15min":1.3260000000000001},"min":{"1min":0.33900000000000002,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":7.1189999999999998,"5min":21.888999999999999,"15min":21.888999999999999},"last":0.40799999999999997},{"interface":"front","average":{"1min":1.484,"5min":2.1739999999999999,"15min":1.671},"min":{"1min":0.60799999999999998,"5min":0.443,"15min":0.38800000000000001},"max":{"1min":4.8600000000000003,"5min":10.507999999999999,"15min":12.917},"last":0.48899999999999999}]},{"osd":5,"last update":"Sat Jun 3 20:13:37 2023","interfaces":[{"interface":"back","average":{"1min":1.6819999999999999,"5min":1.7689999999999999,"15min":1.49},"min":{"1min":0.375,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":7.726,"5min":11.443,"15min":11.443},"last":0.94499999999999995},{"interface":"front","average":{"1min":1.2729999999999999,"5min":1.9830000000000001,"15min":1.8129999999999999},"min":{"1min":0.33600000000000002,"5min":0.33600000000000002,"15min":0.33100000000000002},"max":{"1min":3.6920000000000002,"5min":8.6300000000000008,"15min":20.306999999999999},"last":0.621}]},{"osd":6,"last update":"Sat Jun 3 20:13:37 2023","interfaces":[{"interface":"back","average":{"1min":1.8640000000000001,"5min":2.3650000000000002,"15min":1.823},"min":{"1min":0.46300000000000002,"5min":0.39300000000000002,"15min":0.39300000000000002},"max":{"1min":4.5270000000000001,"5min":11.256,"15min":11.256},"last":0.83799999999999997},{"interface":"front","average":{"1min":1.319,"5min":2.1299999999999999,"15min":1.8580000000000001},"min":{"1min":0.26900000000000002,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":7.4779999999999998,"5min":10.894,"15min":10.894},"last":1.8260000000000001}]}]},{"osd":6,"up_from":42,"seq":180388626962,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2569636,"kb_used_data":2078428,"kb_used_omap":0,"kb_used_meta":491136,"kb_avail":91175516,"statfs":{"total":95995035648,"available":93363728384,"internally_reserved":0,"allocated":2128310272,"data_stored":2127812543,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":502923264},"hb_peers":[0,1,2,3,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,1],"upper_bound":4},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 3 20:14:09 2023","interfaces":[{"interface":"back","average":{"1min":1.4419999999999999,"5min":2.7989999999999999,"15min":2.2400000000000002},"min":{"1min":0.29799999999999999,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":8.0239999999999991,"5min":54.283999999999999,"15min":54.786000000000001},"last":0.53500000000000003},{"interface":"front","average":{"1min":1.621,"5min":3.0539999999999998,"15min":2.4769999999999999},"min":{"1min":0.317,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":4.6429999999999998,"5min":54.314999999999998,"15min":54.936},"last":6.5990000000000002}]},{"osd":1,"last update":"Sat Jun 3 20:13:46 2023","interfaces":[{"interface":"back","average":{"1min":0.879,"5min":1.329,"15min":2.21},"min":{"1min":0.33200000000000002,"5min":0.318,"15min":0.28000000000000003},"max":{"1min":5.2939999999999996,"5min":6.7510000000000003,"15min":160.38499999999999},"last":0.59199999999999997},{"interface":"front","average":{"1min":1.026,"5min":1.7390000000000001,"15min":2.218},"min":{"1min":0.36099999999999999,"5min":0.307,"15min":0.29599999999999999},"max":{"1min":5.4290000000000003,"5min":12.132999999999999,"15min":160.434},"last":6.6349999999999998}]},{"osd":2,"last update":"Sat Jun 3 20:13:46 2023","interfaces":[{"interface":"back","average":{"1min":2.1709999999999998,"5min":3.6819999999999999,"15min":1.9470000000000001},"min":{"1min":0.32900000000000001,"5min":0.32500000000000001,"15min":0.27200000000000002},"max":{"1min":6.8419999999999996,"5min":136.22,"15min":136.22},"last":2.222},{"interface":"front","average":{"1min":1.1919999999999999,"5min":3.5630000000000002,"15min":1.9890000000000001},"min":{"1min":0.40000000000000002,"5min":0.33700000000000002,"15min":0.33700000000000002},"max":{"1min":5.2460000000000004,"5min":135.68799999999999,"15min":135.68799999999999},"last":2.4550000000000001}]},{"osd":3,"last update":"Sat Jun 3 20:14:23 2023","interfaces":[{"interface":"back","average":{"1min":1.849,"5min":1.6890000000000001,"15min":2.4729999999999999},"min":{"1min":0.34100000000000003,"5min":0.33600000000000002,"15min":0.32200000000000001},"max":{"1min":4.5810000000000004,"5min":10.398,"15min":164.99199999999999},"last":2.0489999999999999},{"interface":"front","average":{"1min":1.448,"5min":1.4730000000000001,"15min":2.3580000000000001},"min":{"1min":0.433,"5min":0.38400000000000001,"15min":0.316},"max":{"1min":5.2709999999999999,"5min":6.7770000000000001,"15min":163.459},"last":0.93799999999999994}]},{"osd":4,"last update":"Sat Jun 3 20:10:57 2023","interfaces":[{"interface":"back","average":{"1min":2.0510000000000002,"5min":2.0350000000000001,"15min":1.3029999999999999},"min":{"1min":0.35999999999999999,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":8.2230000000000008,"5min":14.098000000000001,"15min":14.098000000000001},"last":0.69699999999999995},{"interface":"front","average":{"1min":1.929,"5min":1.738,"15min":1.2569999999999999},"min":{"1min":0.52900000000000003,"5min":0.39600000000000002,"15min":0.377},"max":{"1min":8.452,"5min":8.452,"15min":8.452},"last":0.55700000000000005}]},{"osd":5,"last update":"Sat Jun 3 20:14:03 2023","interfaces":[{"interface":"back","average":{"1min":0.86799999999999999,"5min":1.3999999999999999,"15min":1.2829999999999999},"min":{"1min":0.38300000000000001,"5min":0.35599999999999998,"15min":0.32200000000000001},"max":{"1min":5.2060000000000004,"5min":6.6980000000000004,"15min":14.731999999999999},"last":0.61399999999999999},{"interface":"front","average":{"1min":1.3280000000000001,"5min":1.802,"15min":1.734},"min":{"1min":0.34100000000000003,"5min":0.34000000000000002,"15min":0.29499999999999998},"max":{"1min":4.7359999999999998,"5min":10.786,"15min":15.295},"last":6.806}]},{"osd":7,"last update":"Sat Jun 3 20:13:31 2023","interfaces":[{"interface":"back","average":{"1min":0.91700000000000004,"5min":1.649,"15min":1.512},"min":{"1min":0.39400000000000002,"5min":0.33200000000000002,"15min":0.33200000000000002},"max":{"1min":3.5190000000000001,"5min":8.4269999999999996,"15min":11.362},"last":0.56899999999999995},{"interface":"front","average":{"1min":1.748,"5min":2.0720000000000001,"15min":1.7050000000000001},"min":{"1min":0.36299999999999999,"5min":0.36299999999999999,"15min":0.33900000000000002},"max":{"1min":5.7939999999999996,"5min":21.202000000000002,"15min":21.202000000000002},"last":0.64800000000000002}]}]},{"osd":1,"up_from":0,"seq":442381631760,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":0,"up_from":86,"seq":369367187786,"num_pgs":47,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2099296,"kb_used_data":1595352,"kb_used_omap":5,"kb_used_meta":503930,"kb_avail":91645856,"statfs":{"total":95995035648,"available":93845356544,"internally_reserved":0,"allocated":1633640448,"data_stored":1632855723,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":516024954},"hb_peers":[1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,3,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Jun 3 20:13:34 2023","interfaces":[{"interface":"back","average":{"1min":1.3120000000000001,"5min":0.87,"15min":1.466},"min":{"1min":0.20799999999999999,"5min":0.20799999999999999,"15min":0.192},"max":{"1min":8.2110000000000003,"5min":8.2110000000000003,"15min":81.310000000000002},"last":0.21099999999999999},{"interface":"front","average":{"1min":1.6100000000000001,"5min":1.224,"15min":1.7130000000000001},"min":{"1min":0.26400000000000001,"5min":0.22600000000000001,"15min":0.187},"max":{"1min":6.7990000000000004,"5min":8.4320000000000004,"15min":76.971999999999994},"last":0.26400000000000001}]},{"osd":2,"last update":"Sat Jun 3 20:13:34 2023","interfaces":[{"interface":"back","average":{"1min":2.4820000000000002,"5min":1.2669999999999999,"15min":1.1599999999999999},"min":{"1min":0.23599999999999999,"5min":0.193,"15min":0.193},"max":{"1min":9.1050000000000004,"5min":9.1050000000000004,"15min":11.901},"last":2.7320000000000002},{"interface":"front","average":{"1min":2.0169999999999999,"5min":1.379,"15min":1.2410000000000001},"min":{"1min":0.31900000000000001,"5min":0.255,"15min":0.218},"max":{"1min":8.3409999999999993,"5min":8.3409999999999993,"15min":17.315999999999999},"last":0.51100000000000001}]},{"osd":3,"last update":"Sat Jun 3 20:14:25 2023","interfaces":[{"interface":"back","average":{"1min":2.6840000000000002,"5min":1.649,"15min":1.383},"min":{"1min":0.251,"5min":0.251,"15min":0.216},"max":{"1min":13.926,"5min":13.926,"15min":13.926},"last":2.7050000000000001},{"interface":"front","average":{"1min":2.3180000000000001,"5min":1.72,"15min":1.4750000000000001},"min":{"1min":0.39300000000000002,"5min":0.32000000000000001,"15min":0.28100000000000003},"max":{"1min":8.9949999999999992,"5min":9.1349999999999998,"15min":9.1349999999999998},"last":0.39300000000000002}]},{"osd":4,"last update":"Sat Jun 3 20:10:59 2023","interfaces":[{"interface":"back","average":{"1min":1.9370000000000001,"5min":1.7110000000000001,"15min":1.3400000000000001},"min":{"1min":0.45200000000000001,"5min":0.318,"15min":0.318},"max":{"1min":10.025,"5min":10.025,"15min":10.025},"last":0.497},{"interface":"front","average":{"1min":1.673,"5min":1.8160000000000001,"15min":1.429},"min":{"1min":0.56699999999999995,"5min":0.375,"15min":0.375},"max":{"1min":4.6059999999999999,"5min":13.058999999999999,"15min":13.058999999999999},"last":0.75900000000000001}]},{"osd":5,"last update":"Sat Jun 3 20:14:06 2023","interfaces":[{"interface":"back","average":{"1min":2.9500000000000002,"5min":2.0859999999999999,"15min":1.867},"min":{"1min":0.51400000000000001,"5min":0.36699999999999999,"15min":0.34899999999999998},"max":{"1min":8.3350000000000009,"5min":9.3829999999999991,"15min":9.3829999999999991},"last":2.4750000000000001},{"interface":"front","average":{"1min":2.1299999999999999,"5min":1.6759999999999999,"15min":1.583},"min":{"1min":0.32300000000000001,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":8.6950000000000003,"5min":10.159000000000001,"15min":10.159000000000001},"last":2.7530000000000001}]},{"osd":6,"last update":"Sat Jun 3 20:14:06 2023","interfaces":[{"interface":"back","average":{"1min":1.9470000000000001,"5min":1.6910000000000001,"15min":1.5},"min":{"1min":0.35399999999999998,"5min":0.35399999999999998,"15min":0.35399999999999998},"max":{"1min":10.192,"5min":10.192,"15min":10.192},"last":2.7869999999999999},{"interface":"front","average":{"1min":2.137,"5min":1.7370000000000001,"15min":1.6499999999999999},"min":{"1min":0.38300000000000001,"5min":0.38300000000000001,"15min":0.38300000000000001},"max":{"1min":7.5800000000000001,"5min":10.115,"15min":13.117000000000001},"last":0.66700000000000004}]},{"osd":7,"last update":"Sat Jun 3 20:14:06 2023","interfaces":[{"interface":"back","average":{"1min":2.3900000000000001,"5min":1.9350000000000001,"15min":1.7110000000000001},"min":{"1min":0.39400000000000002,"5min":0.39400000000000002,"15min":0.35099999999999998},"max":{"1min":8.7200000000000006,"5min":11.436999999999999,"15min":11.436999999999999},"last":2.7709999999999999},{"interface":"front","average":{"1min":2.2389999999999999,"5min":1.7350000000000001,"15min":1.7849999999999999},"min":{"1min":0.40300000000000002,"5min":0.40300000000000002,"15min":0.40300000000000002},"max":{"1min":7.444,"5min":7.444,"15min":8.891},"last":0.71599999999999997}]}]},{"osd":2,"up_from":114,"seq":489626271966,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1381208,"kb_used_data":1162960,"kb_used_omap":7,"kb_used_meta":218104,"kb_avail":92363944,"statfs":{"total":95995035648,"available":94580678656,"internally_reserved":0,"allocated":1190871040,"data_stored":1190151808,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8111,"internal_metadata":223338577},"hb_peers":[0,1,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,1,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 3 20:13:40 2023","interfaces":[{"interface":"back","average":{"1min":2.419,"5min":3.6819999999999999,"15min":2.694},"min":{"1min":0.19900000000000001,"5min":0.19900000000000001,"15min":0.19900000000000001},"max":{"1min":13.032999999999999,"5min":273.51100000000002,"15min":273.51100000000002},"last":0.38500000000000001},{"interface":"front","average":{"1min":2.226,"5min":3.9929999999999999,"15min":2.8650000000000002},"min":{"1min":0.30099999999999999,"5min":0.247,"15min":0.221},"max":{"1min":8.0559999999999992,"5min":273.67399999999998,"15min":273.67399999999998},"last":0.86899999999999999}]},{"osd":1,"last update":"Sat Jun 3 20:13:40 2023","interfaces":[{"interface":"back","average":{"1min":1.337,"5min":3.593,"15min":2.294},"min":{"1min":0.28899999999999998,"5min":0.254,"15min":0.22900000000000001},"max":{"1min":5.0369999999999999,"5min":273.46499999999997,"15min":273.46499999999997},"last":0.90600000000000003},{"interface":"front","average":{"1min":1.5960000000000001,"5min":3.7080000000000002,"15min":2.371},"min":{"1min":0.27400000000000002,"5min":0.26300000000000001,"15min":0.23699999999999999},"max":{"1min":7.4969999999999999,"5min":274.38400000000001,"15min":274.38400000000001},"last":0.45800000000000002}]},{"osd":3,"last update":"Sat Jun 3 20:13:29 2023","interfaces":[{"interface":"back","average":{"1min":3.169,"5min":4.5289999999999999,"15min":2.3380000000000001},"min":{"1min":0.24399999999999999,"5min":0.24399999999999999,"15min":0.22700000000000001},"max":{"1min":25.497,"5min":273.71899999999999,"15min":273.71899999999999},"last":0.251},{"interface":"front","average":{"1min":3.4209999999999998,"5min":4.7770000000000001,"15min":2.7160000000000002},"min":{"1min":0.23899999999999999,"5min":0.23899999999999999,"15min":0.23899999999999999},"max":{"1min":25.306999999999999,"5min":274.21100000000001,"15min":274.21100000000001},"last":0.51100000000000001}]},{"osd":4,"last update":"Sat Jun 3 20:10:59 2023","interfaces":[{"interface":"back","average":{"1min":16.218,"5min":4.9749999999999996,"15min":2.2519999999999998},"min":{"1min":0.48699999999999999,"5min":0.32800000000000001,"15min":0.32700000000000001},"max":{"1min":273.75700000000001,"5min":273.75700000000001,"15min":273.75700000000001},"last":1.6679999999999999},{"interface":"front","average":{"1min":15.512,"5min":4.7480000000000002,"15min":2.1850000000000001},"min":{"1min":0.36199999999999999,"5min":0.35699999999999998,"15min":0.32600000000000001},"max":{"1min":274.27300000000002,"5min":274.27300000000002,"15min":274.27300000000002},"last":1.585}]},{"osd":5,"last update":"Sat Jun 3 20:13:40 2023","interfaces":[{"interface":"back","average":{"1min":1.9239999999999999,"5min":3.9289999999999998,"15min":2.093},"min":{"1min":0.379,"5min":0.317,"15min":0.317},"max":{"1min":6.9199999999999999,"5min":273.78800000000001,"15min":273.78800000000001},"last":0.34699999999999998},{"interface":"front","average":{"1min":1.718,"5min":4.1970000000000001,"15min":2.3700000000000001},"min":{"1min":0.38400000000000001,"5min":0.32800000000000001,"15min":0.308},"max":{"1min":5.008,"5min":274.15800000000002,"15min":274.15800000000002},"last":0.95999999999999996}]},{"osd":6,"last update":"Sat Jun 3 20:13:40 2023","interfaces":[{"interface":"back","average":{"1min":1.879,"5min":4.585,"15min":2.3100000000000001},"min":{"1min":0.34200000000000003,"5min":0.34200000000000003,"15min":0.34200000000000003},"max":{"1min":9.4350000000000005,"5min":274.03899999999999,"15min":274.03899999999999},"last":0.42099999999999999},{"interface":"front","average":{"1min":1.74,"5min":4.6369999999999996,"15min":2.3730000000000002},"min":{"1min":0.433,"5min":0.39700000000000002,"15min":0.34899999999999998},"max":{"1min":4.9809999999999999,"5min":274.06299999999999,"15min":274.06299999999999},"last":0.98599999999999999}]},{"osd":7,"last update":"Sat Jun 3 20:13:40 2023","interfaces":[{"interface":"back","average":{"1min":2.3279999999999998,"5min":4.3769999999999998,"15min":2.5329999999999999},"min":{"1min":0.317,"5min":0.317,"15min":0.317},"max":{"1min":6.8710000000000004,"5min":274.54000000000002,"15min":274.54000000000002},"last":0.40500000000000003},{"interface":"front","average":{"1min":2.3969999999999998,"5min":4.5590000000000002,"15min":2.54},"min":{"1min":0.53000000000000003,"5min":0.39900000000000002,"15min":0.35699999999999998},"max":{"1min":4.9260000000000002,"5min":274.12700000000001,"15min":274.12700000000001},"last":0.93200000000000005}]}]},{"osd":3,"up_from":119,"seq":511101108437,"num_pgs":47,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1541792,"kb_used_data":1266968,"kb_used_omap":2,"kb_used_meta":274813,"kb_avail":92203360,"statfs":{"total":95995035648,"available":94416240640,"internally_reserved":0,"allocated":1297375232,"data_stored":1296646005,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":281408565},"hb_peers":[0,1,2,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,0,0,0,0,0,9],"upper_bound":1024},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 3 20:13:25 2023","interfaces":[{"interface":"back","average":{"1min":2.3479999999999999,"5min":1.6439999999999999,"15min":1.8440000000000001},"min":{"1min":0.245,"5min":0.192,"15min":0.19},"max":{"1min":7.6200000000000001,"5min":36.918999999999997,"15min":68.647999999999996},"last":0.63300000000000001},{"interface":"front","average":{"1min":1.49,"5min":1.891,"15min":1.9350000000000001},"min":{"1min":0.27800000000000002,"5min":0.22600000000000001,"15min":0.214},"max":{"1min":6.6520000000000001,"5min":42.914999999999999,"15min":69.691000000000003},"last":0.70799999999999996}]},{"osd":1,"last update":"Sat Jun 3 20:13:25 2023","interfaces":[{"interface":"back","average":{"1min":1.3149999999999999,"5min":1.101,"15min":1.1120000000000001},"min":{"1min":0.28199999999999997,"5min":0.23100000000000001,"15min":0.214},"max":{"1min":7.2089999999999996,"5min":10.749000000000001,"15min":10.749000000000001},"last":1.355},{"interface":"front","average":{"1min":1.071,"5min":0.88200000000000001,"15min":1.0169999999999999},"min":{"1min":0.25,"5min":0.24099999999999999,"15min":0.23499999999999999},"max":{"1min":3.9300000000000002,"5min":5.1769999999999996,"15min":10.792999999999999},"last":0.72999999999999998}]},{"osd":2,"last update":"Sat Jun 3 20:13:25 2023","interfaces":[{"interface":"back","average":{"1min":1.6439999999999999,"5min":1.2490000000000001,"15min":1.349},"min":{"1min":0.309,"5min":0.217,"15min":0.217},"max":{"1min":8.7729999999999997,"5min":8.8230000000000004,"15min":62.527999999999999},"last":0.76100000000000001},{"interface":"front","average":{"1min":1.6799999999999999,"5min":1.3100000000000001,"15min":1.361},"min":{"1min":0.28699999999999998,"5min":0.28399999999999997,"15min":0.214},"max":{"1min":8.0220000000000002,"5min":10.704000000000001,"15min":57.935000000000002},"last":1.399}]},{"osd":4,"last update":"Sat Jun 3 20:10:53 2023","interfaces":[{"interface":"back","average":{"1min":2.0379999999999998,"5min":1.98,"15min":1.413},"min":{"1min":0.42399999999999999,"5min":0.38200000000000001,"15min":0.38200000000000001},"max":{"1min":9.8360000000000003,"5min":12.757,"15min":12.757},"last":1.968},{"interface":"front","average":{"1min":2.1749999999999998,"5min":1.744,"15min":1.333},"min":{"1min":0.39000000000000001,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":10.122,"5min":10.122,"15min":10.122},"last":0.29099999999999998}]},{"osd":5,"last update":"Sat Jun 3 20:13:25 2023","interfaces":[{"interface":"back","average":{"1min":1.3240000000000001,"5min":1.2569999999999999,"15min":1.2509999999999999},"min":{"1min":0.38700000000000001,"5min":0.38700000000000001,"15min":0.32900000000000001},"max":{"1min":3.8580000000000001,"5min":5.1340000000000003,"15min":7.1230000000000002},"last":0.80000000000000004},{"interface":"front","average":{"1min":0.74199999999999999,"5min":1.256,"15min":1.494},"min":{"1min":0.39100000000000001,"5min":0.32700000000000001,"15min":0.314},"max":{"1min":2.5059999999999998,"5min":9.5359999999999996,"15min":12.56},"last":0.94699999999999995}]},{"osd":6,"last update":"Sat Jun 3 20:13:25 2023","interfaces":[{"interface":"back","average":{"1min":0.93700000000000006,"5min":1.607,"15min":1.589},"min":{"1min":0.41299999999999998,"5min":0.35099999999999998,"15min":0.35099999999999998},"max":{"1min":2.7440000000000002,"5min":13.718,"15min":13.718},"last":1.0269999999999999},{"interface":"front","average":{"1min":1.643,"5min":1.9470000000000001,"15min":1.6379999999999999},"min":{"1min":0.33400000000000002,"5min":0.33400000000000002,"15min":0.33400000000000002},"max":{"1min":7.2919999999999998,"5min":13.734999999999999,"15min":13.734999999999999},"last":1.623}]},{"osd":7,"last update":"Sat Jun 3 20:13:25 2023","interfaces":[{"interface":"back","average":{"1min":1.827,"5min":1.794,"15min":1.472},"min":{"1min":0.36599999999999999,"5min":0.36599999999999999,"15min":0.36599999999999999},"max":{"1min":7.2590000000000003,"5min":10.858000000000001,"15min":10.858000000000001},"last":1.5860000000000001},{"interface":"front","average":{"1min":1.504,"5min":1.9470000000000001,"15min":1.7250000000000001},"min":{"1min":0.44900000000000001,"5min":0.38100000000000001,"15min":0.38100000000000001},"max":{"1min":3.903,"5min":11.084,"15min":12.715999999999999},"last":0.86399999999999999}]}]},{"osd":4,"up_from":124,"seq":532575944879,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1005320,"kb_used_data":818944,"kb_used_omap":10,"kb_used_meta":186357,"kb_avail":92739832,"statfs":{"total":95995035648,"available":94965587968,"internally_reserved":0,"allocated":838598656,"data_stored":837828790,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":10652,"internal_metadata":190830180},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 3 20:11:03 2023","interfaces":[{"interface":"back","average":{"1min":1.002,"5min":4.8399999999999999,"15min":2.468},"min":{"1min":0.58599999999999997,"5min":0.29499999999999998,"15min":0.254},"max":{"1min":3.0139999999999998,"5min":214.72999999999999,"15min":214.72999999999999},"last":2.1539999999999999},{"interface":"front","average":{"1min":2.04,"5min":5.327,"15min":2.6459999999999999},"min":{"1min":0.41399999999999998,"5min":0.40200000000000002,"15min":0.32400000000000001},"max":{"1min":15,"5min":214.76499999999999,"15min":214.76499999999999},"last":0.625}]},{"osd":1,"last update":"Sat Jun 3 20:11:03 2023","interfaces":[{"interface":"back","average":{"1min":1.1439999999999999,"5min":1.639,"15min":1.55},"min":{"1min":0.371,"5min":0.35199999999999998,"15min":0.28100000000000003},"max":{"1min":3.8210000000000002,"5min":10.446,"15min":58.32},"last":0.84599999999999997},{"interface":"front","average":{"1min":0.878,"5min":1.4990000000000001,"15min":1.4610000000000001},"min":{"1min":0.498,"5min":0.34999999999999998,"15min":0.307},"max":{"1min":1.8,"5min":12.919,"15min":58.286000000000001},"last":2.2429999999999999}]},{"osd":2,"last update":"Sat Jun 3 20:11:03 2023","interfaces":[{"interface":"back","average":{"1min":13.406000000000001,"5min":4.6070000000000002,"15min":2.1419999999999999},"min":{"1min":0.55000000000000004,"5min":0.317,"15min":0.30299999999999999},"max":{"1min":248.65799999999999,"5min":248.65799999999999,"15min":248.65799999999999},"last":2.2770000000000001},{"interface":"front","average":{"1min":14.144,"5min":4.7720000000000002,"15min":2.1890000000000001},"min":{"1min":0.46700000000000003,"5min":0.373,"15min":0.34499999999999997},"max":{"1min":249.077,"5min":249.077,"15min":249.077},"last":0.66200000000000003}]},{"osd":3,"last update":"Sat Jun 3 20:11:03 2023","interfaces":[{"interface":"back","average":{"1min":1.2330000000000001,"5min":1.778,"15min":1.6240000000000001},"min":{"1min":0.40600000000000003,"5min":0.35199999999999998,"15min":0.32500000000000001},"max":{"1min":3.7759999999999998,"5min":10.301,"15min":57.954999999999998},"last":1.0920000000000001},{"interface":"front","average":{"1min":1.226,"5min":1.6479999999999999,"15min":1.613},"min":{"1min":0.50700000000000001,"5min":0.40300000000000002,"15min":0.33700000000000002},"max":{"1min":4.9210000000000003,"5min":12.890000000000001,"15min":57.889000000000003},"last":2.2050000000000001}]},{"osd":5,"last update":"Sat Jun 3 20:11:03 2023","interfaces":[{"interface":"back","average":{"1min":1.4590000000000001,"5min":2.0819999999999999,"15min":1.623},"min":{"1min":0.35899999999999999,"5min":0.32000000000000001,"15min":0.313},"max":{"1min":6.6859999999999999,"5min":11.311,"15min":86.576999999999998},"last":0.47599999999999998},{"interface":"front","average":{"1min":1.5269999999999999,"5min":2.1760000000000002,"15min":1.7390000000000001},"min":{"1min":0.32900000000000001,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":7.9269999999999996,"5min":10.997999999999999,"15min":88.013000000000005},"last":4.9119999999999999}]},{"osd":6,"last update":"Sat Jun 3 20:11:03 2023","interfaces":[{"interface":"back","average":{"1min":1.8140000000000001,"5min":2.7429999999999999,"15min":1.5309999999999999},"min":{"1min":0.36099999999999999,"5min":0.34599999999999997,"15min":0.34599999999999997},"max":{"1min":6.6520000000000001,"5min":14.244999999999999,"15min":14.244999999999999},"last":2.109},{"interface":"front","average":{"1min":2.4329999999999998,"5min":2.8679999999999999,"15min":1.629},"min":{"1min":0.42599999999999999,"5min":0.40100000000000002,"15min":0.36499999999999999},"max":{"1min":10.359,"5min":12.145,"15min":12.145},"last":0.81200000000000006}]},{"osd":7,"last update":"Sat Jun 3 20:11:03 2023","interfaces":[{"interface":"back","average":{"1min":1.7270000000000001,"5min":2.052,"15min":1.55},"min":{"1min":0.44,"5min":0.41199999999999998,"15min":0.39700000000000002},"max":{"1min":9.6210000000000004,"5min":10.388,"15min":10.388},"last":4.8150000000000004},{"interface":"front","average":{"1min":1.0629999999999999,"5min":1.7729999999999999,"15min":1.486},"min":{"1min":0.44800000000000001,"5min":0.34200000000000003,"15min":0.31900000000000001},"max":{"1min":1.8260000000000001,"5min":12.951000000000001,"15min":12.951000000000001},"last":2.3109999999999999}]}]},{"osd":5,"up_from":36,"seq":154618823196,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1515604,"kb_used_data":905804,"kb_used_omap":0,"kb_used_meta":609792,"kb_avail":92229548,"statfs":{"total":95995035648,"available":94443057152,"internally_reserved":0,"allocated":927543296,"data_stored":927057221,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":624427008},"hb_peers":[0,1,2,3,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1,0,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 3 20:13:53 2023","interfaces":[{"interface":"back","average":{"1min":1.548,"5min":2.198,"15min":2.173},"min":{"1min":0.28399999999999997,"5min":0.28100000000000003,"15min":0.22800000000000001},"max":{"1min":9.0809999999999995,"5min":92.706999999999994,"15min":185.29900000000001},"last":0.48499999999999999},{"interface":"front","average":{"1min":1.294,"5min":2.403,"15min":2.5089999999999999},"min":{"1min":0.35799999999999998,"5min":0.27700000000000002,"15min":0.26300000000000001},"max":{"1min":9.0999999999999996,"5min":92.858999999999995,"15min":185.44200000000001},"last":4.6070000000000002}]},{"osd":1,"last update":"Sat Jun 3 20:13:53 2023","interfaces":[{"interface":"back","average":{"1min":0.66400000000000003,"5min":1.347,"15min":2.1230000000000002},"min":{"1min":0.30599999999999999,"5min":0.30599999999999999,"15min":0.30199999999999999},"max":{"1min":3.266,"5min":8.5050000000000008,"15min":114.681},"last":0.41599999999999998},{"interface":"front","average":{"1min":1.226,"5min":1.3280000000000001,"15min":2.153},"min":{"1min":0.33500000000000002,"5min":0.32200000000000001,"15min":0.307},"max":{"1min":5.8209999999999997,"5min":6.3239999999999998,"15min":115.366},"last":4.6710000000000003}]},{"osd":2,"last update":"Sat Jun 3 20:13:53 2023","interfaces":[{"interface":"back","average":{"1min":5.0869999999999997,"5min":3.0550000000000002,"15min":1.853},"min":{"1min":0.40100000000000002,"5min":0.32500000000000001,"15min":0.318},"max":{"1min":63.762999999999998,"5min":92.412000000000006,"15min":92.412000000000006},"last":2.0059999999999998},{"interface":"front","average":{"1min":5.0990000000000002,"5min":3.4049999999999998,"15min":2.1869999999999998},"min":{"1min":0.38800000000000001,"5min":0.314,"15min":0.314},"max":{"1min":69.697000000000003,"5min":94.552000000000007,"15min":94.552000000000007},"last":4.718}]},{"osd":3,"last update":"Sat Jun 3 20:13:27 2023","interfaces":[{"interface":"back","average":{"1min":1.401,"5min":1.843,"15min":2.786},"min":{"1min":0.38200000000000001,"5min":0.37,"15min":0.317},"max":{"1min":4.2359999999999998,"5min":8.4179999999999993,"15min":261.14100000000002},"last":0.71299999999999997},{"interface":"front","average":{"1min":1.4179999999999999,"5min":1.3069999999999999,"15min":2.4590000000000001},"min":{"1min":0.373,"5min":0.35799999999999998,"15min":0.35399999999999998},"max":{"1min":4.383,"5min":7.2320000000000002,"15min":261.27100000000002},"last":0.59399999999999997}]},{"osd":4,"last update":"Sat Jun 3 20:11:00 2023","interfaces":[{"interface":"back","average":{"1min":2.2930000000000001,"5min":2.4510000000000001,"15min":1.6839999999999999},"min":{"1min":0.44,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":7.5910000000000002,"5min":14.951000000000001,"15min":14.951000000000001},"last":0.81899999999999995},{"interface":"front","average":{"1min":1.619,"5min":2.0129999999999999,"15min":1.3799999999999999},"min":{"1min":0.42299999999999999,"5min":0.376,"15min":0.376},"max":{"1min":7.2000000000000002,"5min":12.109,"15min":12.109},"last":0.66900000000000004}]},{"osd":6,"last update":"Sat Jun 3 20:13:53 2023","interfaces":[{"interface":"back","average":{"1min":1.131,"5min":1.9470000000000001,"15min":1.802},"min":{"1min":0.34300000000000003,"5min":0.34300000000000003,"15min":0.34300000000000003},"max":{"1min":3.8260000000000001,"5min":11.167,"15min":22.945},"last":4.6859999999999999},{"interface":"front","average":{"1min":1.0780000000000001,"5min":2.161,"15min":1.9770000000000001},"min":{"1min":0.44600000000000001,"5min":0.38400000000000001,"15min":0.38300000000000001},"max":{"1min":4.7830000000000004,"5min":12.140000000000001,"15min":22.428000000000001},"last":0.752}]},{"osd":7,"last update":"Sat Jun 3 20:13:34 2023","interfaces":[{"interface":"back","average":{"1min":1.226,"5min":1.9790000000000001,"15min":1.9079999999999999},"min":{"1min":0.41599999999999998,"5min":0.36299999999999999,"15min":0.36299999999999999},"max":{"1min":4.7210000000000001,"5min":14.223000000000001,"15min":14.223000000000001},"last":1.03},{"interface":"front","average":{"1min":0.874,"5min":1.538,"15min":1.387},"min":{"1min":0.28799999999999998,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":4.3109999999999999,"5min":9.0039999999999996,"15min":9.0719999999999992},"last":0.54900000000000004}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6928,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4862,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":10652,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4786,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4932,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":1619263488,"data_stored":1619263488,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":1172508672,"data_stored":1172504600,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":1287520256,"data_stored":1287520256,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":871366656,"data_stored":871366656,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":920453120,"data_stored":920453120,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":2111836160,"data_stored":2111832088,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":1385037824,"data_stored":1385037824,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-06-03T20:14:37.942 INFO:tasks.cephadm.ceph_manager.ceph:increase pool size by 10 2023-06-03T20:14:37.942 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph osd pool set device_health_metrics pg_num 11 2023-06-03T20:14:38.025 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:14:38.028 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:14:37 smithi089 bash[75068]: cluster 1685823276.7593582 mgr.y (mgr.34104) 1158 : cluster 0 pgmap v1132: 89 pgs: 9 active+clean+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 13 active+undersized+degraded, 20 active+undersized, 44 active+clean; 2.5 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 704 op/s; 23163/123552 objects degraded (18.748%); 2171/123552 objects misplaced (1.757%); 3.4 MiB/s, 53 objects/s recovering 2023-06-03T20:14:38.028 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:14:37 smithi089 bash[75068]: audit 1685823277.126157 mon.c (mon.2) 525 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T20:14:38.029 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:14:37 smithi089 bash[75068]: cluster 1685823277.7615592 mon.a (mon.0) 1096 : cluster 3 Health check update: Degraded data redundancy: 23163/123552 objects degraded (18.748%), 16 pgs degraded, 36 pgs undersized (PG_DEGRADED) 2023-06-03T20:14:38.029 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:14:37 smithi089 bash[78452]: cluster 1685823276.7593582 mgr.y (mgr.34104) 1158 : cluster 0 pgmap v1132: 89 pgs: 9 active+clean+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 13 active+undersized+degraded, 20 active+undersized, 44 active+clean; 2.5 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 704 op/s; 23163/123552 objects degraded (18.748%); 2171/123552 objects misplaced (1.757%); 3.4 MiB/s, 53 objects/s recovering 2023-06-03T20:14:38.029 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:14:37 smithi089 bash[78452]: audit 1685823277.126157 mon.c (mon.2) 525 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T20:14:38.029 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:14:37 smithi089 bash[78452]: cluster 1685823277.7615592 mon.a (mon.0) 1096 : cluster 3 Health check update: Degraded data redundancy: 23163/123552 objects degraded (18.748%), 16 pgs degraded, 36 pgs undersized (PG_DEGRADED) 2023-06-03T20:14:38.067 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:14:38.073 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:14:38.073 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:14:38.119 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph daemon osd.1 dump_blocked_ops 2023-06-03T20:14:38.175 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:14:38.272 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:14:38.273 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:14:38.274 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:14:38.333 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:14:37 smithi164 bash[59462]: cluster 1685823276.7593582 mgr.y (mgr.34104) 1158 : cluster 0 pgmap v1132: 89 pgs: 9 active+clean+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 13 active+undersized+degraded, 20 active+undersized, 44 active+clean; 2.5 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 704 op/s; 23163/123552 objects degraded (18.748%); 2171/123552 objects misplaced (1.757%); 3.4 MiB/s, 53 objects/s recovering 2023-06-03T20:14:38.334 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:14:37 smithi164 bash[59462]: audit 1685823277.126157 mon.c (mon.2) 525 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T20:14:38.334 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:14:37 smithi164 bash[59462]: cluster 1685823277.7615592 mon.a (mon.0) 1096 : cluster 3 Health check update: Degraded data redundancy: 23163/123552 objects degraded (18.748%), 16 pgs degraded, 36 pgs undersized (PG_DEGRADED) 2023-06-03T20:14:38.374 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:14:38.418 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:14:38.429 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:14:38.430 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:14:38.531 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:14:38.561 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:14:38.563 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:14:38.564 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:14:38.665 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:14:38.695 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:14:38.699 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:14:38.700 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:14:38.801 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:14:38.835 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:14:38.837 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:14:38.838 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:14:38.939 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:14:38.976 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:14:38.979 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:14:38.980 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:14:39.081 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:14:39.129 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:14:39.135 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:14:39.135 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:14:39.236 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:14:39.267 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:14:39.269 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:14:39.270 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:14:39.371 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:14:39.404 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:14:39.406 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:14:39.407 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:14:39.507 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:14:39.542 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:14:39.548 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:14:39.549 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:14:39.650 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:14:39.692 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:14:39.696 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:14:39.697 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:14:39.797 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:14:39.844 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:14:39.849 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:14:39.850 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:14:39.950 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:14:39.997 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:14:40.004 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:14:40.004 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:14:40.083 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:14:39 smithi164 bash[59462]: cluster 1685823278.7603917 mgr.y (mgr.34104) 1159 : cluster 0 pgmap v1133: 89 pgs: 9 active+clean+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 13 active+undersized+degraded, 20 active+undersized, 44 active+clean; 2.4 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 684 op/s; 22906/116811 objects degraded (19.609%); 1742/116811 objects misplaced (1.491%); 3.6 MiB/s, 56 objects/s recovering 2023-06-03T20:14:40.106 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:14:40.127 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph osd unset nodeep-scrub 2023-06-03T20:14:40.138 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:14:40.140 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:14:39 smithi089 bash[78452]: cluster 1685823278.7603917 mgr.y (mgr.34104) 1159 : cluster 0 pgmap v1133: 89 pgs: 9 active+clean+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 13 active+undersized+degraded, 20 active+undersized, 44 active+clean; 2.4 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 684 op/s; 22906/116811 objects degraded (19.609%); 1742/116811 objects misplaced (1.491%); 3.6 MiB/s, 56 objects/s recovering 2023-06-03T20:14:40.140 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:14:40.141 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:14:40.141 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:14:39 smithi089 bash[75068]: cluster 1685823278.7603917 mgr.y (mgr.34104) 1159 : cluster 0 pgmap v1133: 89 pgs: 9 active+clean+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 13 active+undersized+degraded, 20 active+undersized, 44 active+clean; 2.4 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 684 op/s; 22906/116811 objects degraded (19.609%); 1742/116811 objects misplaced (1.491%); 3.6 MiB/s, 56 objects/s recovering 2023-06-03T20:14:40.242 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:14:40.281 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:14:40.285 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:14:40.285 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:14:40.386 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:14:40.426 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:14:40.430 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:14:40.431 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:14:40.532 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:14:40.591 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:14:40.597 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:14:40.598 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:14:40.699 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:14:40.733 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:14:40.737 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:14:40.738 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:14:40.839 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:14:40.876 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:14:40.878 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:14:40.879 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:14:40.980 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:14:41.036 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:14:41.043 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:14:41.044 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:14:41.144 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:14:41.179 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:14:41.182 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:14:41.182 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:14:41.283 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:14:41.326 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:14:41.331 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:14:41.331 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:14:41.432 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:14:41.471 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:14:41.472 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:14:41.473 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:14:41.574 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:14:41.609 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:14:41.612 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:14:41.612 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:14:41.713 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:14:41.755 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:14:41.757 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:14:41.758 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:14:41.858 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:14:41.892 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:14:41.900 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:14:41.900 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:14:42.001 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:14:42.039 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:14:42.041 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:14:42.042 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:14:42.050 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/mon.a/config 2023-06-03T20:14:42.056 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/mon.a/config 2023-06-03T20:14:42.083 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:14:41 smithi164 bash[59462]: cluster 1685823280.7614603 mgr.y (mgr.34104) 1160 : cluster 0 pgmap v1134: 89 pgs: 9 active+clean+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 13 active+undersized+degraded, 20 active+undersized, 44 active+clean; 2.3 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 679 op/s; 22256/114156 objects degraded (19.496%); 1742/114156 objects misplaced (1.526%); 3.6 MiB/s, 56 objects/s recovering 2023-06-03T20:14:42.121 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:14:41 smithi089 bash[75068]: cluster 1685823280.7614603 mgr.y (mgr.34104) 1160 : cluster 0 pgmap v1134: 89 pgs: 9 active+clean+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 13 active+undersized+degraded, 20 active+undersized, 44 active+clean; 2.3 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 679 op/s; 22256/114156 objects degraded (19.496%); 1742/114156 objects misplaced (1.526%); 3.6 MiB/s, 56 objects/s recovering 2023-06-03T20:14:42.121 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:14:41 smithi089 bash[78452]: cluster 1685823280.7614603 mgr.y (mgr.34104) 1160 : cluster 0 pgmap v1134: 89 pgs: 9 active+clean+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 13 active+undersized+degraded, 20 active+undersized, 44 active+clean; 2.3 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 679 op/s; 22256/114156 objects degraded (19.496%); 1742/114156 objects misplaced (1.526%); 3.6 MiB/s, 56 objects/s recovering 2023-06-03T20:14:42.143 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:14:42.177 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:14:42.179 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:14:42.179 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:14:42.280 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:14:42.314 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:14:42.323 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:14:42.324 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:14:42.425 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:14:42.457 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:14:42.466 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:14:42.467 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:14:42.568 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:14:42.595 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:14:42.599 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:14:42.600 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:14:42.701 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:14:42.734 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:14:42.739 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:14:42.740 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:14:42.841 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:14:42.884 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:14:42.887 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:14:42.888 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:14:42.989 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:14:43.021 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:14:43.029 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:14:43.030 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:14:43.081 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/mon.a/config 2023-06-03T20:14:43.131 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:14:43.170 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:14:43.173 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:14:43.173 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:14:43.274 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:14:43.301 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:14:43.305 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:14:43.306 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:14:43.407 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:14:43.442 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:14:43.444 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:14:43.445 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:14:43.546 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:14:43.584 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:14:43.588 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:14:43.589 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:14:43.690 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:14:43.716 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:14:43.717 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:14:43.718 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:14:43.773 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:14:43 smithi164 bash[59462]: cluster 1685823282.7621448 mgr.y (mgr.34104) 1161 : cluster 0 pgmap v1135: 89 pgs: 9 active+clean+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 13 active+undersized+degraded, 20 active+undersized, 44 active+clean; 2.3 GiB data, 10 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 680 op/s; 22009/113415 objects degraded (19.406%); 1742/113415 objects misplaced (1.536%); 3.6 MiB/s, 56 objects/s recovering 2023-06-03T20:14:43.774 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:14:43 smithi164 bash[59462]: cluster 1685823282.7626169 mon.a (mon.0) 1097 : cluster 3 Health check update: Degraded data redundancy: 22256/114156 objects degraded (19.496%), 16 pgs degraded, 36 pgs undersized (PG_DEGRADED) 2023-06-03T20:14:43.819 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:14:43.852 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:14:43.856 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:14:43.857 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:14:43.958 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:14:44.000 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:14:44.005 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:14:44.006 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:14:44.028 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:14:43 smithi089 bash[78452]: cluster 1685823282.7621448 mgr.y (mgr.34104) 1161 : cluster 0 pgmap v1135: 89 pgs: 9 active+clean+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 13 active+undersized+degraded, 20 active+undersized, 44 active+clean; 2.3 GiB data, 10 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 680 op/s; 22009/113415 objects degraded (19.406%); 1742/113415 objects misplaced (1.536%); 3.6 MiB/s, 56 objects/s recovering 2023-06-03T20:14:44.028 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:14:43 smithi089 bash[78452]: cluster 1685823282.7626169 mon.a (mon.0) 1097 : cluster 3 Health check update: Degraded data redundancy: 22256/114156 objects degraded (19.496%), 16 pgs degraded, 36 pgs undersized (PG_DEGRADED) 2023-06-03T20:14:44.029 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:14:43 smithi089 bash[75068]: cluster 1685823282.7621448 mgr.y (mgr.34104) 1161 : cluster 0 pgmap v1135: 89 pgs: 9 active+clean+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 13 active+undersized+degraded, 20 active+undersized, 44 active+clean; 2.3 GiB data, 10 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 680 op/s; 22009/113415 objects degraded (19.406%); 1742/113415 objects misplaced (1.536%); 3.6 MiB/s, 56 objects/s recovering 2023-06-03T20:14:44.029 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:14:43 smithi089 bash[75068]: cluster 1685823282.7626169 mon.a (mon.0) 1097 : cluster 3 Health check update: Degraded data redundancy: 22256/114156 objects degraded (19.496%), 16 pgs degraded, 36 pgs undersized (PG_DEGRADED) 2023-06-03T20:14:44.106 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:14:44.113 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/mon.a/config 2023-06-03T20:14:44.138 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:14:44.140 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:14:44.141 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:14:44.242 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:14:44.280 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:14:44.285 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:14:44.286 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:14:44.387 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:14:44.431 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:14:44.436 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:14:44.437 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:14:44.538 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:14:44.584 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:14:44.590 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:14:44.591 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:14:44.691 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:14:44.728 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:14:44.730 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:14:44.731 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:14:44.831 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:14:44.863 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:14:44.868 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:14:44.868 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:14:44.969 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:14:45.013 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:14:45.015 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:14:45.016 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:14:45.117 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:14:45.156 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:14:45.160 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:14:45.161 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:14:45.262 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:14:45.307 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:14:45.311 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:14:45.312 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:14:45.413 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:14:45.447 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:14:45.449 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:14:45.449 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:14:45.550 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:14:45.588 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:14:45.589 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:14:45.589 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:14:45.690 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:14:45.719 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:14:45.727 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:14:45.728 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:14:45.828 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:14:45.838 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:14:45 smithi089 bash[75068]: cluster 1685823284.763121 mgr.y (mgr.34104) 1162 : cluster 0 pgmap v1136: 89 pgs: 9 active+clean+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 13 active+undersized+degraded, 20 active+undersized, 44 active+clean; 2.1 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 808 op/s; 21473/105042 objects degraded (20.442%); 1285/105042 objects misplaced (1.223%); 4.5 MiB/s, 72 objects/s recovering 2023-06-03T20:14:45.838 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:14:45 smithi089 bash[78452]: cluster 1685823284.763121 mgr.y (mgr.34104) 1162 : cluster 0 pgmap v1136: 89 pgs: 9 active+clean+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 13 active+undersized+degraded, 20 active+undersized, 44 active+clean; 2.1 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 808 op/s; 21473/105042 objects degraded (20.442%); 1285/105042 objects misplaced (1.223%); 4.5 MiB/s, 72 objects/s recovering 2023-06-03T20:14:45.859 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:14:45.860 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:14:45.861 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:14:45.962 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:14:45.995 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:14:46.000 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:14:46.001 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:14:46.083 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:14:45 smithi164 bash[59462]: cluster 1685823284.763121 mgr.y (mgr.34104) 1162 : cluster 0 pgmap v1136: 89 pgs: 9 active+clean+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 13 active+undersized+degraded, 20 active+undersized, 44 active+clean; 2.1 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 808 op/s; 21473/105042 objects degraded (20.442%); 1285/105042 objects misplaced (1.223%); 4.5 MiB/s, 72 objects/s recovering 2023-06-03T20:14:46.102 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:14:46.134 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:14:46.140 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:14:46.141 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:14:46.242 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:14:46.278 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:14:46.281 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:14:46.281 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:14:46.382 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:14:46.391 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 20:14:46 smithi089 bash[71163]: ::ffff:172.21.15.164 - - [03/Jun/2023:20:14:46] "GET /metrics HTTP/1.1" 200 228934 "" "Prometheus/2.33.4" 2023-06-03T20:14:46.412 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:14:46.417 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:14:46.418 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:14:46.518 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:14:46.556 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:14:46.558 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:14:46.559 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:14:46.660 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:14:46.692 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:14:46.695 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:14:46.696 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:14:46.797 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:14:46.833 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:14:46.838 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:14:46.838 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:14:46.939 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:14:46.971 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:14:46.974 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:14:46.975 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:14:47.075 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:14:47.102 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:14:47.104 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:14:47.104 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:14:47.205 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:14:47.243 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:14:47.244 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:14:47.245 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:14:47.348 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:14:47.384 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:14:47.388 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:14:47.389 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:14:47.490 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:14:47.534 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:14:47.538 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:14:47.539 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:14:47.640 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:14:47.680 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:14:47.683 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:14:47.683 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:14:47.784 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:14:47.811 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:14:47.815 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:14:47.816 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:14:47.917 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:14:47.927 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:14:47 smithi164 bash[59462]: cluster 1685823286.7639484 mgr.y (mgr.34104) 1163 : cluster 0 pgmap v1137: 89 pgs: 9 active+clean+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 13 active+undersized+degraded, 20 active+undersized, 44 active+clean; 2.1 GiB data, 9.9 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 667 op/s; 20754/102156 objects degraded (20.316%); 1285/102156 objects misplaced (1.258%); 3.1 MiB/s, 49 objects/s recovering 2023-06-03T20:14:47.928 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:14:47 smithi164 bash[59462]: cluster 1685823287.7638357 mon.a (mon.0) 1098 : cluster 3 Health check update: Degraded data redundancy: 20754/102156 objects degraded (20.316%), 16 pgs degraded, 36 pgs undersized (PG_DEGRADED) 2023-06-03T20:14:47.950 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:14:47.953 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:14:47.953 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:14:48.054 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:14:48.064 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:14:47 smithi089 bash[75068]: cluster 1685823286.7639484 mgr.y (mgr.34104) 1163 : cluster 0 pgmap v1137: 89 pgs: 9 active+clean+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 13 active+undersized+degraded, 20 active+undersized, 44 active+clean; 2.1 GiB data, 9.9 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 667 op/s; 20754/102156 objects degraded (20.316%); 1285/102156 objects misplaced (1.258%); 3.1 MiB/s, 49 objects/s recovering 2023-06-03T20:14:48.064 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:14:47 smithi089 bash[75068]: cluster 1685823287.7638357 mon.a (mon.0) 1098 : cluster 3 Health check update: Degraded data redundancy: 20754/102156 objects degraded (20.316%), 16 pgs degraded, 36 pgs undersized (PG_DEGRADED) 2023-06-03T20:14:48.064 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:14:47 smithi089 bash[78452]: cluster 1685823286.7639484 mgr.y (mgr.34104) 1163 : cluster 0 pgmap v1137: 89 pgs: 9 active+clean+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 13 active+undersized+degraded, 20 active+undersized, 44 active+clean; 2.1 GiB data, 9.9 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 667 op/s; 20754/102156 objects degraded (20.316%); 1285/102156 objects misplaced (1.258%); 3.1 MiB/s, 49 objects/s recovering 2023-06-03T20:14:48.064 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:14:47 smithi089 bash[78452]: cluster 1685823287.7638357 mon.a (mon.0) 1098 : cluster 3 Health check update: Degraded data redundancy: 20754/102156 objects degraded (20.316%), 16 pgs degraded, 36 pgs undersized (PG_DEGRADED) 2023-06-03T20:14:48.091 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:14:48.094 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:14:48.095 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:14:48.196 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:14:48.224 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:14:48.226 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:14:48.226 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:14:48.327 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:14:48.372 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:14:48.376 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:14:48.376 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:14:48.477 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:14:48.512 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:14:48.515 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:14:48.516 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:14:48.617 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:14:48.654 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:14:48.661 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:14:48.662 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:14:48.763 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:14:48.796 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:14:48.799 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:14:48.800 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:14:48.901 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:14:48.947 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:14:48.953 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:14:48.953 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:14:49.054 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:14:49.084 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:14:49.093 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:14:49.094 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:14:49.195 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:14:49.228 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:14:49.235 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:14:49.236 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:14:49.337 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:14:49.373 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:14:49.375 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:14:49.376 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:14:49.477 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:14:49.512 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:14:49.514 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:14:49.515 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:14:49.616 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:14:49.649 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:14:49.652 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:14:49.652 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:14:49.753 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:14:49.789 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:14:49.793 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:14:49.793 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:14:49.894 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:14:49.930 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:14:49.933 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:14:49.935 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:14:50.029 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:14:49 smithi089 bash[75068]: cluster 1685823288.7649302 mgr.y (mgr.34104) 1164 : cluster 0 pgmap v1138: 89 pgs: 9 active+clean+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 13 active+undersized+degraded, 20 active+undersized, 44 active+clean; 1.9 GiB data, 9.6 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 809 op/s; 20493/94389 objects degraded (21.711%); 786/94389 objects misplaced (0.833%); 4.6 MiB/s, 74 objects/s recovering 2023-06-03T20:14:50.029 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:14:49 smithi089 bash[78452]: cluster 1685823288.7649302 mgr.y (mgr.34104) 1164 : cluster 0 pgmap v1138: 89 pgs: 9 active+clean+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 13 active+undersized+degraded, 20 active+undersized, 44 active+clean; 1.9 GiB data, 9.6 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 809 op/s; 20493/94389 objects degraded (21.711%); 786/94389 objects misplaced (0.833%); 4.6 MiB/s, 74 objects/s recovering 2023-06-03T20:14:50.035 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:14:50.062 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:14:50.066 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:14:50.066 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:14:50.084 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:14:49 smithi164 bash[59462]: cluster 1685823288.7649302 mgr.y (mgr.34104) 1164 : cluster 0 pgmap v1138: 89 pgs: 9 active+clean+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 13 active+undersized+degraded, 20 active+undersized, 44 active+clean; 1.9 GiB data, 9.6 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 809 op/s; 20493/94389 objects degraded (21.711%); 786/94389 objects misplaced (0.833%); 4.6 MiB/s, 74 objects/s recovering 2023-06-03T20:14:50.167 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:14:50.199 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:14:50.209 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:14:50.210 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:14:50.311 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:14:50.347 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:14:50.355 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:14:50.355 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:14:50.456 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:14:50.484 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:14:50.493 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:14:50.494 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:14:50.595 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:14:50.623 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:14:50.625 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:14:50.626 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:14:50.727 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:14:50.757 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:14:50.760 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:14:50.761 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:14:50.862 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:14:50.895 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:14:50.901 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:14:50.901 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:14:51.002 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:14:51.031 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:14:51.041 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:14:51.042 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:14:51.143 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:14:51.172 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:14:51.180 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:14:51.181 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:14:51.281 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:14:51.322 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:14:51.326 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:14:51.327 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:14:51.428 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:14:51.453 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:14:51.457 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:14:51.458 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:14:51.559 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:14:51.597 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:14:51.602 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:14:51.602 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:14:51.703 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:14:51.737 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:14:51.741 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:14:51.741 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:14:51.842 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:14:51.851 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:14:51 smithi089 bash[75068]: cluster 1685823290.7658179 mgr.y (mgr.34104) 1165 : cluster 0 pgmap v1139: 89 pgs: 9 active+clean+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 13 active+undersized+degraded, 20 active+undersized, 44 active+clean; 1.9 GiB data, 9.4 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 698 op/s; 19781/91641 objects degraded (21.585%); 786/91641 objects misplaced (0.858%); 3.1 MiB/s, 49 objects/s recovering 2023-06-03T20:14:51.852 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:14:51 smithi089 bash[78452]: cluster 1685823290.7658179 mgr.y (mgr.34104) 1165 : cluster 0 pgmap v1139: 89 pgs: 9 active+clean+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 13 active+undersized+degraded, 20 active+undersized, 44 active+clean; 1.9 GiB data, 9.4 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 698 op/s; 19781/91641 objects degraded (21.585%); 786/91641 objects misplaced (0.858%); 3.1 MiB/s, 49 objects/s recovering 2023-06-03T20:14:51.879 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:14:51.881 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:14:51.882 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:14:51.983 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:14:52.016 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:14:52.020 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:14:52.021 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:14:52.083 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:14:51 smithi164 bash[59462]: cluster 1685823290.7658179 mgr.y (mgr.34104) 1165 : cluster 0 pgmap v1139: 89 pgs: 9 active+clean+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 13 active+undersized+degraded, 20 active+undersized, 44 active+clean; 1.9 GiB data, 9.4 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 698 op/s; 19781/91641 objects degraded (21.585%); 786/91641 objects misplaced (0.858%); 3.1 MiB/s, 49 objects/s recovering 2023-06-03T20:14:52.122 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:14:52.157 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:14:52.160 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:14:52.160 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:14:52.261 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:14:52.307 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:14:52.312 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:14:52.313 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:14:52.414 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:14:52.445 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:14:52.450 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:14:52.451 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:14:52.552 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:14:52.597 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:14:52.601 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:14:52.602 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:14:52.703 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:14:52.739 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:14:52.744 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:14:52.745 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:14:52.846 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:14:52.897 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:14:52.899 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:14:52.900 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:14:53.000 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:14:53.009 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:14:52 smithi089 bash[75068]: audit 1685823292.1255577 mon.c (mon.2) 526 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T20:14:53.009 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:14:52 smithi089 bash[75068]: cluster 1685823292.7647436 mon.a (mon.0) 1099 : cluster 3 Health check update: Degraded data redundancy: 19781/91641 objects degraded (21.585%), 16 pgs degraded, 36 pgs undersized (PG_DEGRADED) 2023-06-03T20:14:53.009 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:14:52 smithi089 bash[78452]: audit 1685823292.1255577 mon.c (mon.2) 526 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T20:14:53.010 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:14:52 smithi089 bash[78452]: cluster 1685823292.7647436 mon.a (mon.0) 1099 : cluster 3 Health check update: Degraded data redundancy: 19781/91641 objects degraded (21.585%), 16 pgs degraded, 36 pgs undersized (PG_DEGRADED) 2023-06-03T20:14:53.040 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:14:53.044 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:14:53.045 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:14:53.083 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:14:52 smithi164 bash[59462]: audit 1685823292.1255577 mon.c (mon.2) 526 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T20:14:53.084 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:14:52 smithi164 bash[59462]: cluster 1685823292.7647436 mon.a (mon.0) 1099 : cluster 3 Health check update: Degraded data redundancy: 19781/91641 objects degraded (21.585%), 16 pgs degraded, 36 pgs undersized (PG_DEGRADED) 2023-06-03T20:14:53.146 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:14:53.189 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:14:53.192 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:14:53.193 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:14:53.294 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:14:53.330 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:14:53.333 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:14:53.334 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:14:53.435 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:14:53.469 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:14:53.472 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:14:53.473 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:14:53.574 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:14:53.606 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:14:53.608 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:14:53.609 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:14:53.710 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:14:53.756 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:14:53.762 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:14:53.762 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:14:53.863 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:14:53.897 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:14:53.901 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:14:53.901 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:14:54.002 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:14:54.029 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:14:54.033 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:14:54.034 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:14:54.083 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:14:53 smithi164 bash[59462]: cluster 1685823292.7665646 mgr.y (mgr.34104) 1166 : cluster 0 pgmap v1140: 89 pgs: 9 active+clean+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 13 active+undersized+degraded, 20 active+undersized, 44 active+clean; 1.8 GiB data, 9.3 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 645 op/s; 19534/90900 objects degraded (21.490%); 786/90900 objects misplaced (0.865%); 3.1 MiB/s, 49 objects/s recovering 2023-06-03T20:14:54.135 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:14:54.144 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:14:53 smithi089 bash[75068]: cluster 1685823292.7665646 mgr.y (mgr.34104) 1166 : cluster 0 pgmap v1140: 89 pgs: 9 active+clean+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 13 active+undersized+degraded, 20 active+undersized, 44 active+clean; 1.8 GiB data, 9.3 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 645 op/s; 19534/90900 objects degraded (21.490%); 786/90900 objects misplaced (0.865%); 3.1 MiB/s, 49 objects/s recovering 2023-06-03T20:14:54.145 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:14:53 smithi089 bash[78452]: cluster 1685823292.7665646 mgr.y (mgr.34104) 1166 : cluster 0 pgmap v1140: 89 pgs: 9 active+clean+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 13 active+undersized+degraded, 20 active+undersized, 44 active+clean; 1.8 GiB data, 9.3 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 645 op/s; 19534/90900 objects degraded (21.490%); 786/90900 objects misplaced (0.865%); 3.1 MiB/s, 49 objects/s recovering 2023-06-03T20:14:54.171 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:14:54.175 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:14:54.176 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:14:54.277 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:14:54.307 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:14:54.313 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:14:54.313 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:14:54.414 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:14:54.455 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:14:54.456 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:14:54.456 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:14:54.557 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:14:54.592 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:14:54.596 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:14:54.597 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:14:54.698 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:14:54.732 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:14:54.739 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:14:54.739 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:14:54.840 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:14:54.883 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:14:54.885 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:14:54.886 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:14:54.987 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:14:55.024 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:14:55.026 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:14:55.026 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:14:55.127 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:14:55.153 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:14:55.160 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:14:55.161 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:14:55.262 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:14:55.310 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:14:55.314 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:14:55.316 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:14:55.417 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:14:55.454 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:14:55.457 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:14:55.457 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:14:55.558 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:14:55.589 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:14:55.593 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:14:55.594 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:14:55.695 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:14:55.723 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:14:55.728 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:14:55.729 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:14:55.830 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:14:55.859 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:14:55.862 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:14:55.862 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:14:55.963 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:14:56.004 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:14:56.009 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:14:56.010 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:14:56.083 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:14:55 smithi164 bash[59462]: cluster 1685823294.7675836 mgr.y (mgr.34104) 1167 : cluster 0 pgmap v1141: 89 pgs: 9 active+clean+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 13 active+undersized+degraded, 20 active+undersized, 44 active+clean; 1.7 GiB data, 8.9 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 859 op/s; 18947/82473 objects degraded (22.974%); 333/82473 objects misplaced (0.404%); 4.7 MiB/s, 74 objects/s recovering 2023-06-03T20:14:56.111 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:14:56.156 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:14:56.161 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:14:56.162 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:14:56.230 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:14:55 smithi089 bash[75068]: cluster 1685823294.7675836 mgr.y (mgr.34104) 1167 : cluster 0 pgmap v1141: 89 pgs: 9 active+clean+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 13 active+undersized+degraded, 20 active+undersized, 44 active+clean; 1.7 GiB data, 8.9 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 859 op/s; 18947/82473 objects degraded (22.974%); 333/82473 objects misplaced (0.404%); 4.7 MiB/s, 74 objects/s recovering 2023-06-03T20:14:56.231 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:14:55 smithi089 bash[78452]: cluster 1685823294.7675836 mgr.y (mgr.34104) 1167 : cluster 0 pgmap v1141: 89 pgs: 9 active+clean+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 13 active+undersized+degraded, 20 active+undersized, 44 active+clean; 1.7 GiB data, 8.9 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 859 op/s; 18947/82473 objects degraded (22.974%); 333/82473 objects misplaced (0.404%); 4.7 MiB/s, 74 objects/s recovering 2023-06-03T20:14:56.263 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:14:56.292 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:14:56.295 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:14:56.296 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:14:56.397 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:14:56.431 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:14:56.437 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:14:56.437 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:14:56.528 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 20:14:56 smithi089 bash[71163]: ::ffff:172.21.15.164 - - [03/Jun/2023:20:14:56] "GET /metrics HTTP/1.1" 200 228932 "" "Prometheus/2.33.4" 2023-06-03T20:14:56.539 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:14:56.578 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:14:56.581 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:14:56.582 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:14:56.683 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:14:56.715 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:14:56.720 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:14:56.721 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:14:56.822 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:14:56.854 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:14:56.857 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:14:56.858 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:14:56.959 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:14:56.999 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:14:57.006 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:14:57.006 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:14:57.108 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:14:57.157 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:14:57.162 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:14:57.162 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:14:57.263 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:14:57.289 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:14:57.290 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:14:57.291 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:14:57.392 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:14:57.428 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:14:57.429 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:14:57.429 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:14:57.530 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:14:57.565 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:14:57.573 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:14:57.573 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:14:57.674 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:14:57.709 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:14:57.713 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:14:57.714 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:14:57.815 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:14:57.857 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:14:57.861 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:14:57.862 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:14:57.963 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:14:57.991 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:14:57.996 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:14:57.997 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:14:58.083 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:14:57 smithi164 bash[59462]: cluster 1685823296.7684572 mgr.y (mgr.34104) 1168 : cluster 0 pgmap v1142: 89 pgs: 9 active+clean+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 13 active+undersized+degraded, 20 active+undersized, 44 active+clean; 1.6 GiB data, 8.8 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 710 op/s; 18195/79470 objects degraded (22.895%); 333/79470 objects misplaced (0.419%); 3.2 MiB/s, 50 objects/s recovering 2023-06-03T20:14:58.083 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:14:57 smithi164 bash[59462]: cluster 1685823297.7660952 mon.a (mon.0) 1100 : cluster 3 Health check update: Degraded data redundancy: 18195/79470 objects degraded (22.895%), 16 pgs degraded, 36 pgs undersized (PG_DEGRADED) 2023-06-03T20:14:58.098 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:14:58.107 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:14:57 smithi089 bash[75068]: cluster 1685823296.7684572 mgr.y (mgr.34104) 1168 : cluster 0 pgmap v1142: 89 pgs: 9 active+clean+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 13 active+undersized+degraded, 20 active+undersized, 44 active+clean; 1.6 GiB data, 8.8 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 710 op/s; 18195/79470 objects degraded (22.895%); 333/79470 objects misplaced (0.419%); 3.2 MiB/s, 50 objects/s recovering 2023-06-03T20:14:58.107 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:14:57 smithi089 bash[75068]: cluster 1685823297.7660952 mon.a (mon.0) 1100 : cluster 3 Health check update: Degraded data redundancy: 18195/79470 objects degraded (22.895%), 16 pgs degraded, 36 pgs undersized (PG_DEGRADED) 2023-06-03T20:14:58.107 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:14:57 smithi089 bash[78452]: cluster 1685823296.7684572 mgr.y (mgr.34104) 1168 : cluster 0 pgmap v1142: 89 pgs: 9 active+clean+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 13 active+undersized+degraded, 20 active+undersized, 44 active+clean; 1.6 GiB data, 8.8 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 710 op/s; 18195/79470 objects degraded (22.895%); 333/79470 objects misplaced (0.419%); 3.2 MiB/s, 50 objects/s recovering 2023-06-03T20:14:58.107 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:14:57 smithi089 bash[78452]: cluster 1685823297.7660952 mon.a (mon.0) 1100 : cluster 3 Health check update: Degraded data redundancy: 18195/79470 objects degraded (22.895%), 16 pgs degraded, 36 pgs undersized (PG_DEGRADED) 2023-06-03T20:14:58.128 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:14:58.134 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:14:58.134 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:14:58.235 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:14:58.263 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:14:58.266 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:14:58.267 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:14:58.367 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:14:58.401 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:14:58.406 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:14:58.407 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:14:58.508 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:14:58.542 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:14:58.546 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:14:58.547 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:14:58.648 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:14:58.677 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:14:58.681 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:14:58.682 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:14:58.782 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:14:58.830 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:14:58.830 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:14:58.831 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:14:58.932 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:14:58.966 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:14:58.969 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:14:58.970 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:14:59.071 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:14:59.103 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:14:59.105 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:14:59.105 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:14:59.206 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:14:59.245 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:14:59.250 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:14:59.251 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:14:59.351 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:14:59.394 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:14:59.397 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:14:59.397 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:14:59.498 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:14:59.536 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:14:59.541 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:14:59.542 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:14:59.643 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:14:59.676 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:14:59.681 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:14:59.682 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:14:59.783 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:14:59.813 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:14:59.817 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:14:59.818 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:14:59.918 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:14:59.927 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:14:59 smithi164 bash[59462]: cluster 1685823298.7692845 mgr.y (mgr.34104) 1169 : cluster 0 pgmap v1143: 89 pgs: 9 active+clean+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 13 active+undersized+degraded, 20 active+undersized, 44 active+clean; 1.5 GiB data, 8.5 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 849 op/s; 17708/71559 objects degraded (24.746%); 2/71559 objects misplaced (0.003%); 4.3 MiB/s, 79 objects/s recovering 2023-06-03T20:14:59.953 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:14:59.956 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:14:59.957 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:15:00.058 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:15:00.067 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:14:59 smithi089 bash[78452]: cluster 1685823298.7692845 mgr.y (mgr.34104) 1169 : cluster 0 pgmap v1143: 89 pgs: 9 active+clean+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 13 active+undersized+degraded, 20 active+undersized, 44 active+clean; 1.5 GiB data, 8.5 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 849 op/s; 17708/71559 objects degraded (24.746%); 2/71559 objects misplaced (0.003%); 4.3 MiB/s, 79 objects/s recovering 2023-06-03T20:15:00.067 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:14:59 smithi089 bash[75068]: cluster 1685823298.7692845 mgr.y (mgr.34104) 1169 : cluster 0 pgmap v1143: 89 pgs: 9 active+clean+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 13 active+undersized+degraded, 20 active+undersized, 44 active+clean; 1.5 GiB data, 8.5 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 849 op/s; 17708/71559 objects degraded (24.746%); 2/71559 objects misplaced (0.003%); 4.3 MiB/s, 79 objects/s recovering 2023-06-03T20:15:00.100 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:15:00.104 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:15:00.105 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:15:00.206 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:15:00.233 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:15:00.238 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:15:00.238 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:15:00.340 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:15:00.385 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:15:00.392 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:15:00.393 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:15:00.494 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:15:00.533 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:15:00.537 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:15:00.538 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:15:00.639 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:15:00.667 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:15:00.671 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:15:00.672 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:15:00.773 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:15:00.810 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:15:00.813 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:15:00.814 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:15:00.915 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:15:00.965 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:15:00.969 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:15:00.970 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:15:01.071 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:15:01.107 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:15:01.108 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:15:01.108 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:15:01.209 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:15:01.238 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:15:01.240 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:15:01.240 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:15:01.341 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:15:01.385 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:15:01.390 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:15:01.391 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:15:01.492 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:15:01.525 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:15:01.530 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:15:01.531 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:15:01.632 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:15:01.671 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:15:01.673 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:15:01.675 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:15:01.776 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:15:01.819 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:15:01.826 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:15:01.827 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:15:01.928 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:15:01.932 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:15:01 smithi164 bash[59462]: cluster 1685823300.770491 mgr.y (mgr.34104) 1170 : cluster 0 pgmap v1144: 89 pgs: 9 active+clean+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 13 active+undersized+degraded, 20 active+undersized, 44 active+clean; 1.4 GiB data, 8.3 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 712 op/s; 17001/68712 objects degraded (24.742%); 2/68712 objects misplaced (0.003%); 2.7 MiB/s, 53 objects/s recovering 2023-06-03T20:15:01.975 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:15:01.976 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:15:01.977 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:15:02.013 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:15:01 smithi089 bash[75068]: cluster 1685823300.770491 mgr.y (mgr.34104) 1170 : cluster 0 pgmap v1144: 89 pgs: 9 active+clean+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 13 active+undersized+degraded, 20 active+undersized, 44 active+clean; 1.4 GiB data, 8.3 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 712 op/s; 17001/68712 objects degraded (24.742%); 2/68712 objects misplaced (0.003%); 2.7 MiB/s, 53 objects/s recovering 2023-06-03T20:15:02.014 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:15:01 smithi089 bash[78452]: cluster 1685823300.770491 mgr.y (mgr.34104) 1170 : cluster 0 pgmap v1144: 89 pgs: 9 active+clean+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 13 active+undersized+degraded, 20 active+undersized, 44 active+clean; 1.4 GiB data, 8.3 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 712 op/s; 17001/68712 objects degraded (24.742%); 2/68712 objects misplaced (0.003%); 2.7 MiB/s, 53 objects/s recovering 2023-06-03T20:15:02.078 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:15:02.118 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:15:02.120 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:15:02.120 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:15:02.221 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:15:02.252 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:15:02.254 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:15:02.254 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:15:02.349 INFO:teuthology.orchestra.run.smithi089.stdout:osd.0: {} 2023-06-03T20:15:02.356 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:15:02.383 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:15:02.384 INFO:teuthology.orchestra.run.smithi089.stdout:osd.1: {} 2023-06-03T20:15:02.389 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:15:02.390 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:15:02.415 INFO:teuthology.orchestra.run.smithi089.stdout:osd.2: {} 2023-06-03T20:15:02.454 INFO:teuthology.orchestra.run.smithi089.stdout:osd.3: {} 2023-06-03T20:15:02.491 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:15:02.497 INFO:teuthology.orchestra.run.smithi089.stdout:osd.5: {} 2023-06-03T20:15:02.534 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:15:02.538 INFO:teuthology.orchestra.run.smithi089.stdout:osd.6: {} 2023-06-03T20:15:02.539 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:15:02.540 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:15:02.583 INFO:teuthology.orchestra.run.smithi089.stdout:osd.7: {} 2023-06-03T20:15:02.583 INFO:teuthology.orchestra.run.smithi089.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-06-03T20:15:02.583 INFO:teuthology.orchestra.run.smithi089.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-06-03T20:15:02.584 INFO:teuthology.orchestra.run.smithi089.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-06-03T20:15:02.584 INFO:teuthology.orchestra.run.smithi089.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-06-03T20:15:02.584 INFO:teuthology.orchestra.run.smithi089.stderr:Error ENXIO: problem getting command descriptions from osd.4 2023-06-03T20:15:02.584 INFO:teuthology.orchestra.run.smithi089.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-06-03T20:15:02.584 INFO:teuthology.orchestra.run.smithi089.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-06-03T20:15:02.584 INFO:teuthology.orchestra.run.smithi089.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-06-03T20:15:02.641 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:15:02.686 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:15:02.689 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:15:02.690 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:15:02.791 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:15:02.828 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:15:02.831 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:15:02.831 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:15:02.932 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:15:02.941 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:15:02 smithi089 bash[75068]: audit 1685823302.289659 mon.c (mon.2) 527 : audit 0 from='client.? 172.21.15.89:0/549678251' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-03T20:15:02.941 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:15:02 smithi089 bash[75068]: cluster 1685823302.7671602 mon.a (mon.0) 1101 : cluster 3 Health check update: Degraded data redundancy: 17001/68712 objects degraded (24.742%), 16 pgs degraded, 36 pgs undersized (PG_DEGRADED) 2023-06-03T20:15:02.942 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:15:02 smithi089 bash[78452]: audit 1685823302.289659 mon.c (mon.2) 527 : audit 0 from='client.? 172.21.15.89:0/549678251' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-03T20:15:02.942 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:15:02 smithi089 bash[78452]: cluster 1685823302.7671602 mon.a (mon.0) 1101 : cluster 3 Health check update: Degraded data redundancy: 17001/68712 objects degraded (24.742%), 16 pgs degraded, 36 pgs undersized (PG_DEGRADED) 2023-06-03T20:15:02.968 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:15:02.973 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:15:02.974 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:15:03.074 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:15:03.083 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:15:02 smithi164 bash[59462]: audit 1685823302.289659 mon.c (mon.2) 527 : audit 0 from='client.? 172.21.15.89:0/549678251' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-03T20:15:03.084 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:15:02 smithi164 bash[59462]: cluster 1685823302.7671602 mon.a (mon.0) 1101 : cluster 3 Health check update: Degraded data redundancy: 17001/68712 objects degraded (24.742%), 16 pgs degraded, 36 pgs undersized (PG_DEGRADED) 2023-06-03T20:15:03.102 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:15:03.105 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:15:03.106 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:15:03.129 INFO:teuthology.orchestra.run.smithi089.stdout:{ 2023-06-03T20:15:03.129 INFO:teuthology.orchestra.run.smithi089.stdout: "ops": [], 2023-06-03T20:15:03.129 INFO:teuthology.orchestra.run.smithi089.stdout: "complaint_time": 30, 2023-06-03T20:15:03.129 INFO:teuthology.orchestra.run.smithi089.stdout: "num_blocked_ops": 0 2023-06-03T20:15:03.130 INFO:teuthology.orchestra.run.smithi089.stdout:} 2023-06-03T20:15:03.207 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:15:03.246 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:15:03.253 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:15:03.254 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:15:03.355 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:15:03.383 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:15:03.390 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:15:03.391 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:15:03.424 INFO:teuthology.orchestra.run.smithi089.stderr:Error ENOENT: unrecognized pool 'device_health_metrics' 2023-06-03T20:15:03.492 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:15:03.528 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:15:03.530 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:15:03.530 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:15:03.631 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:15:03.668 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:15:03.673 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:15:03.675 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:15:03.776 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:15:03.804 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:15:03.810 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:15:03.811 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:15:03.859 INFO:teuthology.orchestra.run.smithi089.stderr:nodeep-scrub is unset 2023-06-03T20:15:03.912 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:15:03.921 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:15:03 smithi164 bash[59462]: cluster 1685823302.771418 mgr.y (mgr.34104) 1171 : cluster 0 pgmap v1145: 89 pgs: 9 active+clean+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 13 active+undersized+degraded, 20 active+undersized, 44 active+clean; 1.4 GiB data, 8.2 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 654 op/s; 16787/68070 objects degraded (24.661%); 2/68070 objects misplaced (0.003%); 2.7 MiB/s, 53 objects/s recovering 2023-06-03T20:15:03.921 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:15:03 smithi164 bash[59462]: audit 1685823303.0741918 mon.a (mon.0) 1102 : audit 1 from='client.? 172.21.15.89:0/625322008' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-06-03T20:15:03.921 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:15:03 smithi164 bash[59462]: audit 1685823303.4160907 mon.a (mon.0) 1103 : audit 1 from='client.? 172.21.15.89:0/2528254666' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "device_health_metrics", "var": "pg_num", "val": "11"}]: dispatch 2023-06-03T20:15:03.955 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:15:03.958 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:15:03.959 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:15:04.059 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:15:04.103 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:15:04.110 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:15:04.111 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:15:04.211 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:15:04.257 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:15:04.258 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:15:04.258 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:15:04.279 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:15:03 smithi089 bash[78452]: cluster 1685823302.771418 mgr.y (mgr.34104) 1171 : cluster 0 pgmap v1145: 89 pgs: 9 active+clean+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 13 active+undersized+degraded, 20 active+undersized, 44 active+clean; 1.4 GiB data, 8.2 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 654 op/s; 16787/68070 objects degraded (24.661%); 2/68070 objects misplaced (0.003%); 2.7 MiB/s, 53 objects/s recovering 2023-06-03T20:15:04.279 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:15:03 smithi089 bash[78452]: audit 1685823303.0741918 mon.a (mon.0) 1102 : audit 1 from='client.? 172.21.15.89:0/625322008' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-06-03T20:15:04.279 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:15:03 smithi089 bash[78452]: audit 1685823303.4160907 mon.a (mon.0) 1103 : audit 1 from='client.? 172.21.15.89:0/2528254666' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "device_health_metrics", "var": "pg_num", "val": "11"}]: dispatch 2023-06-03T20:15:04.279 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:15:03 smithi089 bash[75068]: cluster 1685823302.771418 mgr.y (mgr.34104) 1171 : cluster 0 pgmap v1145: 89 pgs: 9 active+clean+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 13 active+undersized+degraded, 20 active+undersized, 44 active+clean; 1.4 GiB data, 8.2 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 654 op/s; 16787/68070 objects degraded (24.661%); 2/68070 objects misplaced (0.003%); 2.7 MiB/s, 53 objects/s recovering 2023-06-03T20:15:04.280 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:15:03 smithi089 bash[75068]: audit 1685823303.0741918 mon.a (mon.0) 1102 : audit 1 from='client.? 172.21.15.89:0/625322008' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-06-03T20:15:04.280 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:15:03 smithi089 bash[75068]: audit 1685823303.4160907 mon.a (mon.0) 1103 : audit 1 from='client.? 172.21.15.89:0/2528254666' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "device_health_metrics", "var": "pg_num", "val": "11"}]: dispatch 2023-06-03T20:15:04.359 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:15:04.401 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:15:04.405 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:15:04.406 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:15:04.507 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:15:04.537 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:15:04.542 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:15:04.543 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:15:04.644 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:15:04.689 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:15:04.693 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:15:04.694 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:15:04.795 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:15:04.829 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:15:04.835 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:15:04.836 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:15:04.936 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:15:04.945 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:15:04 smithi089 bash[75068]: cluster 1685823303.8271878 mon.a (mon.0) 1104 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-06-03T20:15:04.945 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:15:04 smithi089 bash[75068]: cluster 1685823303.8440847 osd.6 (osd.6) 150 : cluster 3 failed to encode map e205 with expected crc 2023-06-03T20:15:04.946 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:15:04 smithi089 bash[75068]: cluster 1685823303.8467681 osd.5 (osd.5) 137 : cluster 3 failed to encode map e205 with expected crc 2023-06-03T20:15:04.946 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:15:04 smithi089 bash[75068]: audit 1685823303.84906 mon.a (mon.0) 1105 : audit 1 from='client.? 172.21.15.89:0/625322008' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-06-03T20:15:04.946 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:15:04 smithi089 bash[75068]: cluster 1685823303.8494194 mon.a (mon.0) 1106 : cluster 0 osdmap e205: 8 total, 7 up, 7 in 2023-06-03T20:15:04.946 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:15:04 smithi089 bash[78452]: cluster 1685823303.8271878 mon.a (mon.0) 1104 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-06-03T20:15:04.946 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:15:04 smithi089 bash[78452]: cluster 1685823303.8440847 osd.6 (osd.6) 150 : cluster 3 failed to encode map e205 with expected crc 2023-06-03T20:15:04.946 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:15:04 smithi089 bash[78452]: cluster 1685823303.8467681 osd.5 (osd.5) 137 : cluster 3 failed to encode map e205 with expected crc 2023-06-03T20:15:04.946 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:15:04 smithi089 bash[78452]: audit 1685823303.84906 mon.a (mon.0) 1105 : audit 1 from='client.? 172.21.15.89:0/625322008' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-06-03T20:15:04.947 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:15:04 smithi089 bash[78452]: cluster 1685823303.8494194 mon.a (mon.0) 1106 : cluster 0 osdmap e205: 8 total, 7 up, 7 in 2023-06-03T20:15:04.967 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:15:04.970 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:15:04.971 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:15:05.071 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:15:05.079 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:15:04 smithi164 bash[59462]: cluster 1685823303.8271878 mon.a (mon.0) 1104 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-06-03T20:15:05.079 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:15:04 smithi164 bash[59462]: cluster 1685823303.8440847 osd.6 (osd.6) 150 : cluster 3 failed to encode map e205 with expected crc 2023-06-03T20:15:05.079 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:15:04 smithi164 bash[59462]: cluster 1685823303.8467681 osd.5 (osd.5) 137 : cluster 3 failed to encode map e205 with expected crc 2023-06-03T20:15:05.080 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:15:04 smithi164 bash[59462]: audit 1685823303.84906 mon.a (mon.0) 1105 : audit 1 from='client.? 172.21.15.89:0/625322008' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-06-03T20:15:05.080 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:15:04 smithi164 bash[59462]: cluster 1685823303.8494194 mon.a (mon.0) 1106 : cluster 0 osdmap e205: 8 total, 7 up, 7 in 2023-06-03T20:15:05.103 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:15:05.105 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:15:05.106 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:15:05.207 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:15:05.245 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:15:05.249 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:15:05.250 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:15:05.350 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:15:05.385 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:15:05.394 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:15:05.395 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:15:05.496 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:15:05.528 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:15:05.530 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:15:05.530 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:15:05.631 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:15:05.668 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:15:05.677 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:15:05.678 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:15:05.779 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:15:05.812 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:15:05.814 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:15:05.815 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:15:05.916 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:15:05.920 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-06-03T20:15:05.921 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-06-03T20:15:05.952 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:15:05.958 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:15:05.959 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:15:06.060 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:15:06.083 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:15:05 smithi164 bash[59462]: cluster 1685823304.7721577 mgr.y (mgr.34104) 1172 : cluster 0 pgmap v1147: 89 pgs: 9 active+clean+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 13 active+undersized+degraded, 20 active+undersized, 44 active+clean; 1.2 GiB data, 7.8 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 744 op/s; 15712/60129 objects degraded (26.130%); 2/60129 objects misplaced (0.003%); 1.4 MiB/s, 72 objects/s recovering 2023-06-03T20:15:06.083 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:15:05 smithi164 bash[59462]: audit 1685823305.311682 mon.c (mon.2) 528 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-03T20:15:06.113 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:15:06.115 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:15:06.116 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:15:06.217 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:15:06.230 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 20:15:06 smithi089 bash[71163]: ::ffff:172.21.15.164 - - [03/Jun/2023:20:15:06] "GET /metrics HTTP/1.1" 200 228932 "" "Prometheus/2.33.4" 2023-06-03T20:15:06.231 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:15:06 smithi089 bash[78452]: cluster 1685823304.7721577 mgr.y (mgr.34104) 1172 : cluster 0 pgmap v1147: 89 pgs: 9 active+clean+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 13 active+undersized+degraded, 20 active+undersized, 44 active+clean; 1.2 GiB data, 7.8 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 744 op/s; 15712/60129 objects degraded (26.130%); 2/60129 objects misplaced (0.003%); 1.4 MiB/s, 72 objects/s recovering 2023-06-03T20:15:06.231 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:15:06 smithi089 bash[78452]: audit 1685823305.311682 mon.c (mon.2) 528 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-03T20:15:06.231 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:15:06 smithi089 bash[75068]: cluster 1685823304.7721577 mgr.y (mgr.34104) 1172 : cluster 0 pgmap v1147: 89 pgs: 9 active+clean+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 13 active+undersized+degraded, 20 active+undersized, 44 active+clean; 1.2 GiB data, 7.8 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 744 op/s; 15712/60129 objects degraded (26.130%); 2/60129 objects misplaced (0.003%); 1.4 MiB/s, 72 objects/s recovering 2023-06-03T20:15:06.231 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:15:06 smithi089 bash[75068]: audit 1685823305.311682 mon.c (mon.2) 528 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-03T20:15:06.255 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:15:06.258 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:15:06.258 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:15:06.360 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:15:06.392 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:15:06.399 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:15:06.399 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:15:06.500 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:15:06.529 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:15:06.531 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:15:06.531 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:15:06.632 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:15:06.695 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:15:06.698 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:15:06.699 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:15:06.800 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:15:06.832 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:15:06.835 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:15:06.835 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:15:06.936 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:15:06.967 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:15:06.969 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:15:06.970 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:15:07.071 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:15:07.099 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:15:07.106 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:15:07.106 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:15:07.207 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:15:07.240 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:15:07.244 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:15:07.244 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:15:07.345 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:15:07.393 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:15:07.398 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:15:07.399 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:15:07.500 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:15:07.540 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:15:07.541 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:15:07.541 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:15:07.642 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:15:07.684 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:15:07.688 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:15:07.689 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:15:07.790 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:15:07.816 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:15:07.818 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:15:07.820 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:15:07.921 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:15:07.922 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-06-03T20:15:07.968 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:15:07.972 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:15:07.973 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:15:08.073 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:15:08.117 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:15:08.119 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:15:08.120 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:15:08.221 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:15:08.258 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:15:08.264 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:15:08.265 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:15:08.296 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph daemon osd.1 dump_historic_ops 2023-06-03T20:15:08.313 DEBUG:teuthology.orchestra.run:got remote process result: 2 2023-06-03T20:15:08.333 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:15:08 smithi164 bash[59462]: cluster 1685823306.7734952 mgr.y (mgr.34104) 1173 : cluster 0 pgmap v1148: 89 pgs: 9 active+clean+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 13 active+undersized+degraded, 20 active+undersized, 44 active+clean; 1.2 GiB data, 7.6 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 732 op/s; 15052/57492 objects degraded (26.181%); 2/57492 objects misplaced (0.003%); 1.4 MiB/s, 72 objects/s recovering 2023-06-03T20:15:08.333 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:15:08 smithi164 bash[59462]: audit 1685823307.1317956 mon.a (mon.0) 1107 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:15:08.334 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:15:08 smithi164 bash[59462]: audit 1685823307.1356726 mon.c (mon.2) 529 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T20:15:08.334 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:15:08 smithi164 bash[59462]: cluster 1685823307.7682774 mon.a (mon.0) 1108 : cluster 3 Health check update: Degraded data redundancy: 15052/57492 objects degraded (26.181%), 16 pgs degraded, 36 pgs undersized (PG_DEGRADED) 2023-06-03T20:15:08.334 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:15:08 smithi089 bash[75068]: cluster 1685823306.7734952 mgr.y (mgr.34104) 1173 : cluster 0 pgmap v1148: 89 pgs: 9 active+clean+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 13 active+undersized+degraded, 20 active+undersized, 44 active+clean; 1.2 GiB data, 7.6 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 732 op/s; 15052/57492 objects degraded (26.181%); 2/57492 objects misplaced (0.003%); 1.4 MiB/s, 72 objects/s recovering 2023-06-03T20:15:08.335 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:15:08 smithi089 bash[75068]: audit 1685823307.1317956 mon.a (mon.0) 1107 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:15:08.335 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:15:08 smithi089 bash[75068]: audit 1685823307.1356726 mon.c (mon.2) 529 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T20:15:08.335 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:15:08 smithi089 bash[75068]: cluster 1685823307.7682774 mon.a (mon.0) 1108 : cluster 3 Health check update: Degraded data redundancy: 15052/57492 objects degraded (26.181%), 16 pgs degraded, 36 pgs undersized (PG_DEGRADED) 2023-06-03T20:15:08.335 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:15:08 smithi089 bash[78452]: cluster 1685823306.7734952 mgr.y (mgr.34104) 1173 : cluster 0 pgmap v1148: 89 pgs: 9 active+clean+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 13 active+undersized+degraded, 20 active+undersized, 44 active+clean; 1.2 GiB data, 7.6 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 732 op/s; 15052/57492 objects degraded (26.181%); 2/57492 objects misplaced (0.003%); 1.4 MiB/s, 72 objects/s recovering 2023-06-03T20:15:08.336 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:15:08 smithi089 bash[78452]: audit 1685823307.1317956 mon.a (mon.0) 1107 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:15:08.336 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:15:08 smithi089 bash[78452]: audit 1685823307.1356726 mon.c (mon.2) 529 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T20:15:08.336 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:15:08 smithi089 bash[78452]: cluster 1685823307.7682774 mon.a (mon.0) 1108 : cluster 3 Health check update: Degraded data redundancy: 15052/57492 objects degraded (26.181%), 16 pgs degraded, 36 pgs undersized (PG_DEGRADED) 2023-06-03T20:15:08.366 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:15:08.417 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:15:08.425 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:15:08.425 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:15:08.526 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:15:08.560 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:15:08.565 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:15:08.566 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:15:08.667 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:15:08.712 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:15:08.717 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:15:08.718 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:15:08.819 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:15:08.874 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:15:08.880 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:15:08.880 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:15:08.981 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:15:09.022 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:15:09.024 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:15:09.025 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:15:09.126 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:15:09.156 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:15:09.158 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:15:09.159 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:15:09.259 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:15:09.299 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:15:09.304 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:15:09.305 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:15:09.406 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:15:09.457 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:15:09.463 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:15:09.463 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:15:09.564 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:15:09.591 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:15:09.592 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:15:09.593 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:15:09.694 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:15:09.728 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:15:09.735 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:15:09.736 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:15:09.836 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:15:09.869 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:15:09.876 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:15:09.876 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:15:09.977 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:15:10.019 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:15:10.023 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:15:10.024 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:15:10.125 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:15:10.166 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:15:10.175 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:15:10.176 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:15:10.277 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:15:10.279 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:15:10 smithi089 bash[75068]: cluster 1685823308.7748642 mgr.y (mgr.34104) 1174 : cluster 0 pgmap v1149: 89 pgs: 9 active+clean+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 13 active+undersized+degraded, 20 active+undersized, 44 active+clean; 1.0 GiB data, 7.3 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 732 op/s; 14234/49566 objects degraded (28.717%); 2/49566 objects misplaced (0.004%); 0 B/s, 73 objects/s recovering 2023-06-03T20:15:10.279 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:15:10 smithi089 bash[78452]: cluster 1685823308.7748642 mgr.y (mgr.34104) 1174 : cluster 0 pgmap v1149: 89 pgs: 9 active+clean+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 13 active+undersized+degraded, 20 active+undersized, 44 active+clean; 1.0 GiB data, 7.3 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 732 op/s; 14234/49566 objects degraded (28.717%); 2/49566 objects misplaced (0.004%); 0 B/s, 73 objects/s recovering 2023-06-03T20:15:10.314 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:15:10.320 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:15:10.321 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:15:10.338 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph osd set noscrub 2023-06-03T20:15:10.422 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:15:10.435 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:15:10 smithi164 bash[59462]: cluster 1685823308.7748642 mgr.y (mgr.34104) 1174 : cluster 0 pgmap v1149: 89 pgs: 9 active+clean+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 13 active+undersized+degraded, 20 active+undersized, 44 active+clean; 1.0 GiB data, 7.3 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 732 op/s; 14234/49566 objects degraded (28.717%); 2/49566 objects misplaced (0.004%); 0 B/s, 73 objects/s recovering 2023-06-03T20:15:10.461 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:15:10.465 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:15:10.465 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:15:10.566 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:15:10.606 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:15:10.608 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:15:10.609 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:15:10.710 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:15:10.766 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:15:10.774 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:15:10.776 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:15:10.877 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:15:10.905 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:15:10.907 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:15:10.907 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:15:11.008 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:15:11.040 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:15:11.046 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:15:11.047 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:15:11.148 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:15:11.201 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:15:11.206 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:15:11.207 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:15:11.307 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:15:11.383 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:15:11.385 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:15:11.385 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:15:11.486 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:15:11.499 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:15:11 smithi089 bash[75068]: cluster 1685823310.7758863 mgr.y (mgr.34104) 1175 : cluster 0 pgmap v1150: 89 pgs: 9 active+clean+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 13 active+undersized+degraded, 20 active+undersized, 44 active+clean; 975 MiB data, 7.1 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 729 op/s; 13580/46824 objects degraded (29.002%); 2/46824 objects misplaced (0.004%); 0 B/s, 73 objects/s recovering 2023-06-03T20:15:11.500 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:15:11 smithi089 bash[78452]: cluster 1685823310.7758863 mgr.y (mgr.34104) 1175 : cluster 0 pgmap v1150: 89 pgs: 9 active+clean+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 13 active+undersized+degraded, 20 active+undersized, 44 active+clean; 975 MiB data, 7.1 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 729 op/s; 13580/46824 objects degraded (29.002%); 2/46824 objects misplaced (0.004%); 0 B/s, 73 objects/s recovering 2023-06-03T20:15:11.519 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:15:11.522 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:15:11.523 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:15:11.584 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:15:11 smithi164 bash[59462]: cluster 1685823310.7758863 mgr.y (mgr.34104) 1175 : cluster 0 pgmap v1150: 89 pgs: 9 active+clean+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 13 active+undersized+degraded, 20 active+undersized, 44 active+clean; 975 MiB data, 7.1 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 729 op/s; 13580/46824 objects degraded (29.002%); 2/46824 objects misplaced (0.004%); 0 B/s, 73 objects/s recovering 2023-06-03T20:15:11.624 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:15:11.656 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:15:11.658 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:15:11.659 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:15:11.760 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:15:11.840 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:15:11.848 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:15:11.849 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:15:11.950 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:15:11.994 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:15:11.996 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:15:11.997 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:15:12.098 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:15:12.136 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:15:12.138 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:15:12.139 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:15:12.240 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:15:12.276 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:15:12.279 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:15:12.280 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:15:12.381 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:15:12.416 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/mon.a/config 2023-06-03T20:15:12.417 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/mon.a/config 2023-06-03T20:15:12.422 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:15:12.424 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:15:12.425 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:15:12.526 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:15:12.577 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:15:12.580 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:15:12.581 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:15:12.681 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:15:12.733 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:15:12.738 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:15:12.738 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:15:12.839 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:15:12.867 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:15:12.869 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:15:12.870 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:15:12.971 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:15:13.003 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:15:13.006 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:15:13.007 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:15:13.108 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:15:13.148 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:15:13.152 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:15:13.153 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:15:13.254 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:15:13.293 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:15:13.298 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:15:13.299 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:15:13.315 INFO:tasks.thrashosds.thrasher:in_osds: [0, 2, 3, 4, 5, 6, 7] out_osds: [1] dead_osds: [4] live_osds: [0, 1, 2, 3, 5, 6, 7] 2023-06-03T20:15:13.315 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-06-03T20:15:13.316 INFO:tasks.thrashosds.thrasher:inject_pause on osd.3 2023-06-03T20:15:13.316 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2023-06-03T20:15:13.316 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2023-06-03T20:15:13.316 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph daemon osd.3 config set filestore_inject_stall 3 2023-06-03T20:15:13.400 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:15:13.469 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:15:13.477 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:15:13.478 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:15:13.579 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:15:13.623 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:15:13.626 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:15:13.627 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:15:13.728 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:15:13.737 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:15:13 smithi089 bash[75068]: audit 1685823312.4681077 mon.a (mon.0) 1109 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:15:13.738 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:15:13 smithi089 bash[75068]: audit 1685823312.480659 mon.a (mon.0) 1110 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:15:13.738 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:15:13 smithi089 bash[75068]: cluster 1685823312.7708855 mon.a (mon.0) 1111 : cluster 3 Health check update: Degraded data redundancy: 13580/46824 objects degraded (29.002%), 16 pgs degraded, 36 pgs undersized (PG_DEGRADED) 2023-06-03T20:15:13.738 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:15:13 smithi089 bash[75068]: cluster 1685823312.77675 mgr.y (mgr.34104) 1176 : cluster 0 pgmap v1151: 89 pgs: 9 active+clean+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 13 active+undersized+degraded, 20 active+undersized, 44 active+clean; 960 MiB data, 7.0 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 731 op/s; 13347/46125 objects degraded (28.937%); 2/46125 objects misplaced (0.004%); 0 B/s, 73 objects/s recovering 2023-06-03T20:15:13.738 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:15:13 smithi089 bash[75068]: audit 1685823313.1902037 mon.a (mon.0) 1112 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:15:13.738 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:15:13 smithi089 bash[75068]: audit 1685823313.2000985 mon.a (mon.0) 1113 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:15:13.739 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:15:13 smithi089 bash[78452]: audit 1685823312.4681077 mon.a (mon.0) 1109 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:15:13.739 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:15:13 smithi089 bash[78452]: audit 1685823312.480659 mon.a (mon.0) 1110 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:15:13.739 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:15:13 smithi089 bash[78452]: cluster 1685823312.7708855 mon.a (mon.0) 1111 : cluster 3 Health check update: Degraded data redundancy: 13580/46824 objects degraded (29.002%), 16 pgs degraded, 36 pgs undersized (PG_DEGRADED) 2023-06-03T20:15:13.739 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:15:13 smithi089 bash[78452]: cluster 1685823312.77675 mgr.y (mgr.34104) 1176 : cluster 0 pgmap v1151: 89 pgs: 9 active+clean+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 13 active+undersized+degraded, 20 active+undersized, 44 active+clean; 960 MiB data, 7.0 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 731 op/s; 13347/46125 objects degraded (28.937%); 2/46125 objects misplaced (0.004%); 0 B/s, 73 objects/s recovering 2023-06-03T20:15:13.739 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:15:13 smithi089 bash[78452]: audit 1685823313.1902037 mon.a (mon.0) 1112 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:15:13.739 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:15:13 smithi089 bash[78452]: audit 1685823313.2000985 mon.a (mon.0) 1113 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:15:13.761 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:15:13.763 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:15:13.764 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:15:13.833 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:15:13 smithi164 bash[59462]: audit 1685823312.4681077 mon.a (mon.0) 1109 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:15:13.834 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:15:13 smithi164 bash[59462]: audit 1685823312.480659 mon.a (mon.0) 1110 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:15:13.834 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:15:13 smithi164 bash[59462]: cluster 1685823312.7708855 mon.a (mon.0) 1111 : cluster 3 Health check update: Degraded data redundancy: 13580/46824 objects degraded (29.002%), 16 pgs degraded, 36 pgs undersized (PG_DEGRADED) 2023-06-03T20:15:13.834 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:15:13 smithi164 bash[59462]: cluster 1685823312.77675 mgr.y (mgr.34104) 1176 : cluster 0 pgmap v1151: 89 pgs: 9 active+clean+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 13 active+undersized+degraded, 20 active+undersized, 44 active+clean; 960 MiB data, 7.0 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 731 op/s; 13347/46125 objects degraded (28.937%); 2/46125 objects misplaced (0.004%); 0 B/s, 73 objects/s recovering 2023-06-03T20:15:13.834 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:15:13 smithi164 bash[59462]: audit 1685823313.1902037 mon.a (mon.0) 1112 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:15:13.834 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:15:13 smithi164 bash[59462]: audit 1685823313.2000985 mon.a (mon.0) 1113 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:15:13.865 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:15:13.898 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:15:13.903 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:15:13.904 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:15:14.005 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:15:14.029 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:15:14.031 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:15:14.031 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:15:14.132 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:15:14.169 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:15:14.173 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:15:14.173 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:15:14.274 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:15:14.309 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:15:14.315 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:15:14.316 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:15:14.417 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:15:14.454 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:15:14.458 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:15:14.459 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:15:14.488 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/mon.a/config 2023-06-03T20:15:14.560 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:15:14.595 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:15:14.601 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:15:14.602 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:15:14.703 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:15:14.712 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:15:14 smithi089 bash[75068]: audit 1685823313.832341 mon.c (mon.2) 530 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-03T20:15:14.712 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:15:14 smithi089 bash[75068]: audit 1685823313.8390143 mon.c (mon.2) 531 : audit 1 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-03T20:15:14.712 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:15:14 smithi089 bash[75068]: audit 1685823313.846265 mon.a (mon.0) 1114 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:15:14.713 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:15:14 smithi089 bash[78452]: audit 1685823313.832341 mon.c (mon.2) 530 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-03T20:15:14.713 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:15:14 smithi089 bash[78452]: audit 1685823313.8390143 mon.c (mon.2) 531 : audit 1 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-03T20:15:14.713 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:15:14 smithi089 bash[78452]: audit 1685823313.846265 mon.a (mon.0) 1114 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:15:14.732 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:15:14.734 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:15:14.735 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:15:14.833 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:15:14 smithi164 bash[59462]: audit 1685823313.832341 mon.c (mon.2) 530 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-03T20:15:14.834 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:15:14 smithi164 bash[59462]: audit 1685823313.8390143 mon.c (mon.2) 531 : audit 1 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-03T20:15:14.834 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:15:14 smithi164 bash[59462]: audit 1685823313.846265 mon.a (mon.0) 1114 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:15:14.836 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:15:14.862 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:15:14.864 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:15:14.864 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:15:14.965 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:15:14.996 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:15:15.000 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:15:15.000 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:15:15.102 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:15:15.135 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:15:15.141 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:15:15.142 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:15:15.243 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:15:15.273 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:15:15.279 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:15:15.280 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:15:15.381 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:15:15.418 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:15:15.423 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:15:15.424 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:15:15.524 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:15:15.529 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:15:15 smithi089 bash[75068]: cluster 1685823314.7778924 mgr.y (mgr.34104) 1177 : cluster 0 pgmap v1152: 89 pgs: 9 active+clean+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 13 active+undersized+degraded, 20 active+undersized, 44 active+clean; 797 MiB data, 6.7 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 666 op/s; 12327/38286 objects degraded (32.197%); 2/38286 objects misplaced (0.005%); 0 B/s, 63 objects/s recovering 2023-06-03T20:15:15.529 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:15:15 smithi089 bash[78452]: cluster 1685823314.7778924 mgr.y (mgr.34104) 1177 : cluster 0 pgmap v1152: 89 pgs: 9 active+clean+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 13 active+undersized+degraded, 20 active+undersized, 44 active+clean; 797 MiB data, 6.7 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 666 op/s; 12327/38286 objects degraded (32.197%); 2/38286 objects misplaced (0.005%); 0 B/s, 63 objects/s recovering 2023-06-03T20:15:15.568 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:15:15.573 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:15:15.574 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:15:15.675 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:15:15.707 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:15:15.710 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:15:15.710 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:15:15.811 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:15:15.821 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:15:15 smithi164 bash[59462]: cluster 1685823314.7778924 mgr.y (mgr.34104) 1177 : cluster 0 pgmap v1152: 89 pgs: 9 active+clean+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 13 active+undersized+degraded, 20 active+undersized, 44 active+clean; 797 MiB data, 6.7 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 666 op/s; 12327/38286 objects degraded (32.197%); 2/38286 objects misplaced (0.005%); 0 B/s, 63 objects/s recovering 2023-06-03T20:15:15.863 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:15:15.866 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:15:15.867 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:15:15.968 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:15:16.004 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:15:16.009 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:15:16.010 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:15:16.110 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:15:16.137 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:15:16.139 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:15:16.140 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:15:16.241 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:15:16.271 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:15:16.274 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:15:16.275 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:15:16.376 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:15:16.396 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 20:15:16 smithi089 bash[71163]: ::ffff:172.21.15.164 - - [03/Jun/2023:20:15:16] "GET /metrics HTTP/1.1" 200 228914 "" "Prometheus/2.33.4" 2023-06-03T20:15:16.416 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:15:16.422 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:15:16.423 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:15:16.524 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:15:16.561 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:15:16.565 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:15:16.566 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:15:16.666 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:15:16.708 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:15:16.709 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:15:16.710 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:15:16.811 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:15:16.841 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:15:16.845 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:15:16.846 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:15:16.947 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:15:16.997 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:15:17.001 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:15:17.002 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:15:17.103 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:15:17.140 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:15:17.142 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:15:17.142 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:15:17.243 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:15:17.285 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:15:17.287 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:15:17.288 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:15:17.389 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:15:17.420 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:15:17.425 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:15:17.426 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:15:17.527 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:15:17.552 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:15:17.556 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:15:17.557 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:15:17.568 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/mon.a/config 2023-06-03T20:15:17.658 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:15:17.696 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:15:17.703 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:15:17.704 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:15:17.805 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:15:17.836 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:15:17.842 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:15:17.842 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:15:17.943 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:15:17.978 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:15:17.984 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:15:17.984 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:15:18.029 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:15:17 smithi089 bash[75068]: cluster 1685823316.783084 mgr.y (mgr.34104) 1178 : cluster 0 pgmap v1153: 89 pgs: 9 active+clean+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 13 active+undersized+degraded, 20 active+undersized, 44 active+clean; 744 MiB data, 6.5 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 676 op/s; 11698/35757 objects degraded (32.715%); 2/35757 objects misplaced (0.006%); 0 B/s, 58 objects/s recovering 2023-06-03T20:15:18.029 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:15:17 smithi089 bash[75068]: cluster 1685823317.7772517 mon.a (mon.0) 1115 : cluster 3 Health check update: Degraded data redundancy: 11698/35757 objects degraded (32.715%), 16 pgs degraded, 36 pgs undersized (PG_DEGRADED) 2023-06-03T20:15:18.029 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:15:17 smithi089 bash[78452]: cluster 1685823316.783084 mgr.y (mgr.34104) 1178 : cluster 0 pgmap v1153: 89 pgs: 9 active+clean+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 13 active+undersized+degraded, 20 active+undersized, 44 active+clean; 744 MiB data, 6.5 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 676 op/s; 11698/35757 objects degraded (32.715%); 2/35757 objects misplaced (0.006%); 0 B/s, 58 objects/s recovering 2023-06-03T20:15:18.030 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:15:17 smithi089 bash[78452]: cluster 1685823317.7772517 mon.a (mon.0) 1115 : cluster 3 Health check update: Degraded data redundancy: 11698/35757 objects degraded (32.715%), 16 pgs degraded, 36 pgs undersized (PG_DEGRADED) 2023-06-03T20:15:18.083 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:15:17 smithi164 bash[59462]: cluster 1685823316.783084 mgr.y (mgr.34104) 1178 : cluster 0 pgmap v1153: 89 pgs: 9 active+clean+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 13 active+undersized+degraded, 20 active+undersized, 44 active+clean; 744 MiB data, 6.5 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 676 op/s; 11698/35757 objects degraded (32.715%); 2/35757 objects misplaced (0.006%); 0 B/s, 58 objects/s recovering 2023-06-03T20:15:18.084 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:15:17 smithi164 bash[59462]: cluster 1685823317.7772517 mon.a (mon.0) 1115 : cluster 3 Health check update: Degraded data redundancy: 11698/35757 objects degraded (32.715%), 16 pgs degraded, 36 pgs undersized (PG_DEGRADED) 2023-06-03T20:15:18.085 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:15:18.112 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:15:18.115 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:15:18.116 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:15:18.217 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:15:18.253 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:15:18.257 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:15:18.258 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:15:18.359 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:15:18.385 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:15:18.388 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:15:18.389 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:15:18.490 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:15:18.511 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:15:18.512 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:15:18.512 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:15:18.613 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:15:18.634 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:15:18.635 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:15:18.636 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:15:18.737 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:15:18.774 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:15:18.778 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:15:18.778 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:15:18.879 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:15:18.903 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:15:18.904 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:15:18.905 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:15:19.006 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:15:19.028 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:15:19.029 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:15:19.029 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:15:19.130 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:15:19.173 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:15:19.179 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:15:19.179 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:15:19.280 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:15:19.312 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:15:19.314 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:15:19.315 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:15:19.416 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:15:19.447 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:15:19.451 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:15:19.451 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:15:19.552 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:15:19.592 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:15:19.594 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:15:19.595 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:15:19.696 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:15:19.729 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:15:19.736 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:15:19.737 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:15:19.837 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:15:19.883 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:15:19.887 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:15:19.887 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:15:19.988 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:15:20.011 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:15:20.012 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:15:20.012 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:15:20.106 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:15:19 smithi089 bash[78452]: cluster 1685823318.784016 mgr.y (mgr.34104) 1179 : cluster 0 pgmap v1154: 89 pgs: 9 active+clean+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 13 active+undersized+degraded, 20 active+undersized, 44 active+clean; 594 MiB data, 6.2 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 802 op/s; 10910/28560 objects degraded (38.200%); 2/28560 objects misplaced (0.007%); 0 B/s, 89 objects/s recovering 2023-06-03T20:15:20.107 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:15:19 smithi089 bash[75068]: cluster 1685823318.784016 mgr.y (mgr.34104) 1179 : cluster 0 pgmap v1154: 89 pgs: 9 active+clean+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 13 active+undersized+degraded, 20 active+undersized, 44 active+clean; 594 MiB data, 6.2 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 802 op/s; 10910/28560 objects degraded (38.200%); 2/28560 objects misplaced (0.007%); 0 B/s, 89 objects/s recovering 2023-06-03T20:15:20.113 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:15:20.156 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:15:20.160 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:15:20.161 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:15:20.262 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:15:20.295 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:15:20.297 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:15:20.298 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:15:20.333 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:15:19 smithi164 bash[59462]: cluster 1685823318.784016 mgr.y (mgr.34104) 1179 : cluster 0 pgmap v1154: 89 pgs: 9 active+clean+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 13 active+undersized+degraded, 20 active+undersized, 44 active+clean; 594 MiB data, 6.2 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 802 op/s; 10910/28560 objects degraded (38.200%); 2/28560 objects misplaced (0.007%); 0 B/s, 89 objects/s recovering 2023-06-03T20:15:20.399 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:15:20.432 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:15:20.437 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:15:20.438 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:15:20.539 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:15:20.568 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:15:20.574 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:15:20.574 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:15:20.675 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:15:20.712 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:15:20.714 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:15:20.714 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:15:20.815 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:15:20.855 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:15:20.856 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:15:20.856 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:15:20.957 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:15:21.003 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:15:21.009 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:15:21.010 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:15:21.111 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:15:21.157 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:15:21.160 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:15:21.161 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:15:21.261 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:15:21.300 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:15:21.301 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:15:21.302 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:15:21.402 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:15:21.443 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:15:21.445 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:15:21.446 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:15:21.547 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:15:21.575 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:15:21.581 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:15:21.582 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:15:21.683 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:15:21.713 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:15:21.717 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:15:21.718 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:15:21.819 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:15:21.844 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:15:21.850 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:15:21.851 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:15:21.952 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:15:21.961 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:15:21 smithi089 bash[75068]: cluster 1685823320.784959 mgr.y (mgr.34104) 1180 : cluster 0 pgmap v1155: 89 pgs: 9 active+clean+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 13 active+undersized+degraded, 20 active+undersized, 44 active+clean; 540 MiB data, 6.1 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 655 op/s; 10241/25965 objects degraded (39.442%); 2/25965 objects misplaced (0.008%); 0 B/s, 60 objects/s recovering 2023-06-03T20:15:21.962 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:15:21 smithi089 bash[78452]: cluster 1685823320.784959 mgr.y (mgr.34104) 1180 : cluster 0 pgmap v1155: 89 pgs: 9 active+clean+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 13 active+undersized+degraded, 20 active+undersized, 44 active+clean; 540 MiB data, 6.1 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 655 op/s; 10241/25965 objects degraded (39.442%); 2/25965 objects misplaced (0.008%); 0 B/s, 60 objects/s recovering 2023-06-03T20:15:21.997 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:15:22.004 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:15:22.005 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:15:22.106 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:15:22.114 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:15:21 smithi164 bash[59462]: cluster 1685823320.784959 mgr.y (mgr.34104) 1180 : cluster 0 pgmap v1155: 89 pgs: 9 active+clean+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 13 active+undersized+degraded, 20 active+undersized, 44 active+clean; 540 MiB data, 6.1 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 655 op/s; 10241/25965 objects degraded (39.442%); 2/25965 objects misplaced (0.008%); 0 B/s, 60 objects/s recovering 2023-06-03T20:15:22.136 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:15:22.141 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:15:22.142 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:15:22.243 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:15:22.283 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:15:22.285 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:15:22.286 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:15:22.387 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:15:22.418 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:15:22.421 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:15:22.421 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:15:22.522 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:15:22.559 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:15:22.561 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:15:22.562 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:15:22.663 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:15:22.696 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:15:22.702 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:15:22.702 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:15:22.803 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:15:22.851 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:15:22.853 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:15:22.854 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:15:22.954 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:15:22.974 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:15:22.976 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:15:22.976 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:15:23.077 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:15:23.083 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:15:22 smithi164 bash[59462]: audit 1685823322.126999 mon.c (mon.2) 532 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T20:15:23.083 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:15:22 smithi164 bash[59462]: cluster 1685823322.7788556 mon.a (mon.0) 1116 : cluster 3 Health check update: Degraded data redundancy: 10241/25965 objects degraded (39.442%), 16 pgs degraded, 36 pgs undersized (PG_DEGRADED) 2023-06-03T20:15:23.107 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:15:23.116 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:15:23.117 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:15:23.217 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:15:23.263 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:15:23.269 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:15:23.270 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:15:23.304 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:15:22 smithi089 bash[78452]: audit 1685823322.126999 mon.c (mon.2) 532 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T20:15:23.304 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:15:22 smithi089 bash[78452]: cluster 1685823322.7788556 mon.a (mon.0) 1116 : cluster 3 Health check update: Degraded data redundancy: 10241/25965 objects degraded (39.442%), 16 pgs degraded, 36 pgs undersized (PG_DEGRADED) 2023-06-03T20:15:23.304 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:15:22 smithi089 bash[75068]: audit 1685823322.126999 mon.c (mon.2) 532 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T20:15:23.305 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:15:22 smithi089 bash[75068]: cluster 1685823322.7788556 mon.a (mon.0) 1116 : cluster 3 Health check update: Degraded data redundancy: 10241/25965 objects degraded (39.442%), 16 pgs degraded, 36 pgs undersized (PG_DEGRADED) 2023-06-03T20:15:23.371 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:15:23.399 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:15:23.402 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:15:23.402 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:15:23.503 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:15:23.534 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:15:23.535 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:15:23.535 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:15:23.636 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:15:23.671 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:15:23.673 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:15:23.674 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:15:23.775 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:15:23.804 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:15:23.809 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:15:23.810 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:15:23.911 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:15:23.920 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:15:23 smithi089 bash[75068]: cluster 1685823322.7859206 mgr.y (mgr.34104) 1181 : cluster 0 pgmap v1156: 89 pgs: 9 active+clean+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 13 active+undersized+degraded, 20 active+undersized, 44 active+clean; 526 MiB data, 5.9 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 597 op/s; 10022/25308 objects degraded (39.600%); 2/25308 objects misplaced (0.008%); 0 B/s, 60 objects/s recovering 2023-06-03T20:15:23.920 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:15:23 smithi089 bash[78452]: cluster 1685823322.7859206 mgr.y (mgr.34104) 1181 : cluster 0 pgmap v1156: 89 pgs: 9 active+clean+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 13 active+undersized+degraded, 20 active+undersized, 44 active+clean; 526 MiB data, 5.9 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 597 op/s; 10022/25308 objects degraded (39.600%); 2/25308 objects misplaced (0.008%); 0 B/s, 60 objects/s recovering 2023-06-03T20:15:23.949 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:15:23.953 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:15:23.954 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:15:24.054 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:15:24.084 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:15:24.092 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:15:24.093 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:15:24.194 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:15:24.228 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:15:24.230 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:15:24.231 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:15:24.332 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:15:24.333 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:15:23 smithi164 bash[59462]: cluster 1685823322.7859206 mgr.y (mgr.34104) 1181 : cluster 0 pgmap v1156: 89 pgs: 9 active+clean+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 13 active+undersized+degraded, 20 active+undersized, 44 active+clean; 526 MiB data, 5.9 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 597 op/s; 10022/25308 objects degraded (39.600%); 2/25308 objects misplaced (0.008%); 0 B/s, 60 objects/s recovering 2023-06-03T20:15:24.370 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:15:24.374 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:15:24.375 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:15:24.476 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:15:24.516 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:15:24.520 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:15:24.521 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:15:24.622 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:15:24.653 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:15:24.656 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:15:24.657 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:15:24.758 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:15:24.793 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:15:24.798 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:15:24.799 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:15:24.900 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:15:24.931 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:15:24.941 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:15:24.942 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:15:25.043 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:15:25.077 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:15:25.082 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:15:25.082 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:15:25.183 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:15:25.236 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:15:25.240 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:15:25.241 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:15:25.341 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:15:25.391 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:15:25.394 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:15:25.395 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:15:25.496 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:15:25.525 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:15:25.528 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:15:25.529 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:15:25.630 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:15:25.663 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:15:25.669 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:15:25.670 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:15:25.771 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:15:25.806 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:15:25.810 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:15:25.811 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:15:25.912 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:15:25.940 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:15:25.942 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:15:25.943 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:15:26.044 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:15:26.053 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:15:25 smithi089 bash[78452]: cluster 1685823324.786943 mgr.y (mgr.34104) 1182 : cluster 0 pgmap v1157: 89 pgs: 9 active+clean+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 13 active+undersized+degraded, 20 active+undersized, 44 active+clean; 360 MiB data, 5.6 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 799 op/s; 8941/17316 objects degraded (51.634%); 2/17316 objects misplaced (0.012%); 0 B/s, 93 objects/s recovering 2023-06-03T20:15:26.054 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:15:25 smithi089 bash[75068]: cluster 1685823324.786943 mgr.y (mgr.34104) 1182 : cluster 0 pgmap v1157: 89 pgs: 9 active+clean+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 13 active+undersized+degraded, 20 active+undersized, 44 active+clean; 360 MiB data, 5.6 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 799 op/s; 8941/17316 objects degraded (51.634%); 2/17316 objects misplaced (0.012%); 0 B/s, 93 objects/s recovering 2023-06-03T20:15:26.072 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:15:26.079 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:15:26.079 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:15:26.083 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:15:25 smithi164 bash[59462]: cluster 1685823324.786943 mgr.y (mgr.34104) 1182 : cluster 0 pgmap v1157: 89 pgs: 9 active+clean+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 13 active+undersized+degraded, 20 active+undersized, 44 active+clean; 360 MiB data, 5.6 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 799 op/s; 8941/17316 objects degraded (51.634%); 2/17316 objects misplaced (0.012%); 0 B/s, 93 objects/s recovering 2023-06-03T20:15:26.181 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:15:26.217 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:15:26.219 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:15:26.220 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:15:26.321 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:15:26.328 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 20:15:26 smithi089 bash[71163]: ::ffff:172.21.15.164 - - [03/Jun/2023:20:15:26] "GET /metrics HTTP/1.1" 200 228892 "" "Prometheus/2.33.4" 2023-06-03T20:15:26.345 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:15:26.351 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:15:26.351 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:15:26.452 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:15:26.480 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:15:26.483 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:15:26.484 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:15:26.584 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:15:26.620 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:15:26.622 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:15:26.623 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:15:26.724 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:15:26.756 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:15:26.758 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:15:26.759 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:15:26.859 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:15:26.891 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:15:26.892 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:15:26.893 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:15:26.994 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:15:27.025 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:15:27.030 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:15:27.030 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:15:27.131 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:15:27.170 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:15:27.171 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:15:27.172 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:15:27.272 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:15:27.280 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:15:26 smithi089 bash[75068]: cluster 1685823325.8682072 osd.6 (osd.6) 151 : cluster 3 failed to encode map e206 with expected crc 2023-06-03T20:15:27.280 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:15:26 smithi089 bash[75068]: cluster 1685823325.8726528 mon.a (mon.0) 1117 : cluster 0 osdmap e206: 8 total, 7 up, 7 in 2023-06-03T20:15:27.280 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:15:26 smithi089 bash[75068]: cluster 1685823325.8747404 osd.5 (osd.5) 138 : cluster 3 failed to encode map e206 with expected crc 2023-06-03T20:15:27.280 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:15:26 smithi089 bash[78452]: cluster 1685823325.8682072 osd.6 (osd.6) 151 : cluster 3 failed to encode map e206 with expected crc 2023-06-03T20:15:27.281 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:15:26 smithi089 bash[78452]: cluster 1685823325.8726528 mon.a (mon.0) 1117 : cluster 0 osdmap e206: 8 total, 7 up, 7 in 2023-06-03T20:15:27.281 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:15:26 smithi089 bash[78452]: cluster 1685823325.8747404 osd.5 (osd.5) 138 : cluster 3 failed to encode map e206 with expected crc 2023-06-03T20:15:27.282 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:15:26 smithi164 bash[59462]: cluster 1685823325.8682072 osd.6 (osd.6) 151 : cluster 3 failed to encode map e206 with expected crc 2023-06-03T20:15:27.282 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:15:26 smithi164 bash[59462]: cluster 1685823325.8726528 mon.a (mon.0) 1117 : cluster 0 osdmap e206: 8 total, 7 up, 7 in 2023-06-03T20:15:27.282 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:15:26 smithi164 bash[59462]: cluster 1685823325.8747404 osd.5 (osd.5) 138 : cluster 3 failed to encode map e206 with expected crc 2023-06-03T20:15:27.312 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:15:27.320 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:15:27.320 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:15:27.421 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:15:27.457 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:15:27.462 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:15:27.463 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:15:27.563 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:15:27.593 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:15:27.595 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:15:27.596 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:15:27.697 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:15:27.730 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:15:27.735 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:15:27.736 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:15:27.837 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:15:27.874 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:15:27.877 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:15:27.877 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:15:27.978 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:15:27.987 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:15:27 smithi089 bash[78452]: cluster 1685823326.7875938 mgr.y (mgr.34104) 1183 : cluster 0 pgmap v1159: 89 pgs: 9 active+clean+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 13 active+undersized+degraded, 20 active+undersized, 44 active+clean; 302 MiB data, 5.4 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 707 op/s; 8256/14529 objects degraded (56.824%); 2/14529 objects misplaced (0.014%); 0 B/s, 77 objects/s recovering 2023-06-03T20:15:27.987 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:15:27 smithi089 bash[78452]: cluster 1685823326.874968 osd.7 (osd.7) 133 : cluster 3 failed to encode map e207 with expected crc 2023-06-03T20:15:27.987 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:15:27 smithi089 bash[78452]: cluster 1685823326.8787107 mon.a (mon.0) 1118 : cluster 0 osdmap e207: 8 total, 7 up, 7 in 2023-06-03T20:15:27.987 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:15:27 smithi089 bash[78452]: cluster 1685823327.0026557 osd.5 (osd.5) 139 : cluster 3 failed to encode map e207 with expected crc 2023-06-03T20:15:27.987 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:15:27 smithi089 bash[78452]: cluster 1685823327.7803922 mon.a (mon.0) 1119 : cluster 3 Health check update: Degraded data redundancy: 8256/14529 objects degraded (56.824%), 16 pgs degraded, 36 pgs undersized (PG_DEGRADED) 2023-06-03T20:15:27.988 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:15:27 smithi089 bash[75068]: cluster 1685823326.7875938 mgr.y (mgr.34104) 1183 : cluster 0 pgmap v1159: 89 pgs: 9 active+clean+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 13 active+undersized+degraded, 20 active+undersized, 44 active+clean; 302 MiB data, 5.4 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 707 op/s; 8256/14529 objects degraded (56.824%); 2/14529 objects misplaced (0.014%); 0 B/s, 77 objects/s recovering 2023-06-03T20:15:27.988 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:15:27 smithi089 bash[75068]: cluster 1685823326.874968 osd.7 (osd.7) 133 : cluster 3 failed to encode map e207 with expected crc 2023-06-03T20:15:27.988 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:15:27 smithi089 bash[75068]: cluster 1685823326.8787107 mon.a (mon.0) 1118 : cluster 0 osdmap e207: 8 total, 7 up, 7 in 2023-06-03T20:15:27.988 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:15:27 smithi089 bash[75068]: cluster 1685823327.0026557 osd.5 (osd.5) 139 : cluster 3 failed to encode map e207 with expected crc 2023-06-03T20:15:27.988 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:15:27 smithi089 bash[75068]: cluster 1685823327.7803922 mon.a (mon.0) 1119 : cluster 3 Health check update: Degraded data redundancy: 8256/14529 objects degraded (56.824%), 16 pgs degraded, 36 pgs undersized (PG_DEGRADED) 2023-06-03T20:15:28.017 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:15:28.023 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:15:28.024 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:15:28.125 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:15:28.158 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:15:28.159 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:15:28.159 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:15:28.260 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:15:28.268 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:15:27 smithi164 bash[59462]: cluster 1685823326.7875938 mgr.y (mgr.34104) 1183 : cluster 0 pgmap v1159: 89 pgs: 9 active+clean+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 13 active+undersized+degraded, 20 active+undersized, 44 active+clean; 302 MiB data, 5.4 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 707 op/s; 8256/14529 objects degraded (56.824%); 2/14529 objects misplaced (0.014%); 0 B/s, 77 objects/s recovering 2023-06-03T20:15:28.268 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:15:27 smithi164 bash[59462]: cluster 1685823326.874968 osd.7 (osd.7) 133 : cluster 3 failed to encode map e207 with expected crc 2023-06-03T20:15:28.268 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:15:27 smithi164 bash[59462]: cluster 1685823326.8787107 mon.a (mon.0) 1118 : cluster 0 osdmap e207: 8 total, 7 up, 7 in 2023-06-03T20:15:28.269 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:15:27 smithi164 bash[59462]: cluster 1685823327.0026557 osd.5 (osd.5) 139 : cluster 3 failed to encode map e207 with expected crc 2023-06-03T20:15:28.269 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:15:27 smithi164 bash[59462]: cluster 1685823327.7803922 mon.a (mon.0) 1119 : cluster 3 Health check update: Degraded data redundancy: 8256/14529 objects degraded (56.824%), 16 pgs degraded, 36 pgs undersized (PG_DEGRADED) 2023-06-03T20:15:28.291 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:15:28.295 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:15:28.295 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:15:28.397 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:15:28.430 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:15:28.437 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:15:28.437 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:15:28.538 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:15:28.572 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:15:28.578 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:15:28.579 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:15:28.680 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:15:28.724 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:15:28.730 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:15:28.730 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:15:28.831 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:15:28.861 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:15:28.871 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:15:28.872 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:15:28.973 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:15:29.007 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:15:29.009 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:15:29.010 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:15:29.111 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:15:29.155 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:15:29.158 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:15:29.158 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:15:29.259 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:15:29.290 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:15:29.295 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:15:29.296 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:15:29.397 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:15:29.422 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:15:29.427 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:15:29.429 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:15:29.529 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:15:29.561 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:15:29.565 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:15:29.566 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:15:29.667 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:15:29.706 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:15:29.712 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:15:29.713 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:15:29.814 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:15:29.846 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:15:29.849 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:15:29.849 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:15:29.950 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:15:30.000 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:15:30.005 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:15:30.006 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:15:30.083 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:15:29 smithi164 bash[59462]: cluster 1685823328.7885463 mgr.y (mgr.34104) 1184 : cluster 0 pgmap v1161: 89 pgs: 9 active+clean+remapped, 2 active+recovering+undersized+degraded+remapped, 13 active+undersized+degraded, 20 active+undersized, 45 active+clean; 160 MiB data, 5.1 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 760 op/s; 7516/7713 objects degraded (97.446%); 2/7713 objects misplaced (0.026%); 0 B/s, 90 objects/s recovering 2023-06-03T20:15:30.107 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:15:30.137 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:15:30.141 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:15:30.141 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:15:30.212 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:15:29 smithi089 bash[75068]: cluster 1685823328.7885463 mgr.y (mgr.34104) 1184 : cluster 0 pgmap v1161: 89 pgs: 9 active+clean+remapped, 2 active+recovering+undersized+degraded+remapped, 13 active+undersized+degraded, 20 active+undersized, 45 active+clean; 160 MiB data, 5.1 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 760 op/s; 7516/7713 objects degraded (97.446%); 2/7713 objects misplaced (0.026%); 0 B/s, 90 objects/s recovering 2023-06-03T20:15:30.212 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:15:29 smithi089 bash[78452]: cluster 1685823328.7885463 mgr.y (mgr.34104) 1184 : cluster 0 pgmap v1161: 89 pgs: 9 active+clean+remapped, 2 active+recovering+undersized+degraded+remapped, 13 active+undersized+degraded, 20 active+undersized, 45 active+clean; 160 MiB data, 5.1 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 760 op/s; 7516/7713 objects degraded (97.446%); 2/7713 objects misplaced (0.026%); 0 B/s, 90 objects/s recovering 2023-06-03T20:15:30.243 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:15:30.278 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:15:30.285 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:15:30.286 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:15:30.386 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:15:30.428 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:15:30.434 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:15:30.434 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:15:30.535 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:15:30.578 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:15:30.580 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:15:30.581 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:15:30.682 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:15:30.716 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:15:30.720 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:15:30.721 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:15:30.822 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:15:30.851 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:15:30.853 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:15:30.854 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:15:30.955 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:15:30.984 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:15:30.990 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:15:30.990 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:15:31.036 INFO:teuthology.orchestra.run.smithi089.stdout:osd.0: {} 2023-06-03T20:15:31.045 INFO:tasks.radosbench.radosbench.0.smithi089.stdout:Removed 105525 objects 2023-06-03T20:15:31.045 INFO:tasks.radosbench.radosbench.0.smithi089.stdout:Clean up completed and total clean up time :146.465 2023-06-03T20:15:31.058 INFO:teuthology.orchestra.run.smithi089.stdout:osd.1: {} 2023-06-03T20:15:31.079 INFO:teuthology.orchestra.run.smithi089.stdout:osd.2: {} 2023-06-03T20:15:31.092 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:15:31.105 INFO:teuthology.orchestra.run.smithi089.stdout:osd.3: {} 2023-06-03T20:15:31.123 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:15:31.126 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:15:31.126 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:15:31.133 INFO:teuthology.orchestra.run.smithi089.stdout:osd.5: {} 2023-06-03T20:15:31.156 INFO:teuthology.orchestra.run.smithi089.stdout:osd.6: {} 2023-06-03T20:15:31.185 INFO:teuthology.orchestra.run.smithi089.stdout:osd.7: {} 2023-06-03T20:15:31.185 INFO:teuthology.orchestra.run.smithi089.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-06-03T20:15:31.185 INFO:teuthology.orchestra.run.smithi089.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-06-03T20:15:31.186 INFO:teuthology.orchestra.run.smithi089.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-06-03T20:15:31.186 INFO:teuthology.orchestra.run.smithi089.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-06-03T20:15:31.186 INFO:teuthology.orchestra.run.smithi089.stderr:Error ENXIO: problem getting command descriptions from osd.4 2023-06-03T20:15:31.186 INFO:teuthology.orchestra.run.smithi089.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-06-03T20:15:31.186 INFO:teuthology.orchestra.run.smithi089.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-06-03T20:15:31.186 INFO:teuthology.orchestra.run.smithi089.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-06-03T20:15:31.227 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:15:31.252 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:15:31.255 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:15:31.256 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:15:31.357 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:15:31.393 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:15:31.395 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:15:31.396 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:15:31.458 INFO:tasks.cephadm.ceph_manager.ceph:removing pool_name unique_pool_6 2023-06-03T20:15:31.459 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph osd pool rm unique_pool_6 unique_pool_6 --yes-i-really-really-mean-it 2023-06-03T20:15:31.497 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:15:31.533 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:15:31.535 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:15:31.535 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:15:31.636 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:15:31.661 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:15:31.664 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:15:31.665 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:15:31.766 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:15:31.795 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:15:31.798 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:15:31.798 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:15:31.899 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:15:31.924 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:15:31.928 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:15:31.929 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:15:32.029 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:15:31 smithi089 bash[75068]: cluster 1685823330.789208 mgr.y (mgr.34104) 1185 : cluster 0 pgmap v1162: 89 pgs: 9 active+clean+remapped, 2 active+recovering+undersized+degraded+remapped, 13 active+undersized+degraded, 20 active+undersized, 45 active+clean; 110 MiB data, 4.9 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 832 op/s; 6906/5313 objects degraded (129.983%); 2/5313 objects misplaced (0.038%); 0 B/s, 90 objects/s recovering 2023-06-03T20:15:32.029 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:15:31 smithi089 bash[75068]: audit 1685823330.9991486 mon.a (mon.0) 1120 : audit 0 from='client.? 172.21.15.89:0/1499104916' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-03T20:15:32.029 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:15:31 smithi089 bash[78452]: cluster 1685823330.789208 mgr.y (mgr.34104) 1185 : cluster 0 pgmap v1162: 89 pgs: 9 active+clean+remapped, 2 active+recovering+undersized+degraded+remapped, 13 active+undersized+degraded, 20 active+undersized, 45 active+clean; 110 MiB data, 4.9 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 832 op/s; 6906/5313 objects degraded (129.983%); 2/5313 objects misplaced (0.038%); 0 B/s, 90 objects/s recovering 2023-06-03T20:15:32.029 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:15:31 smithi089 bash[78452]: audit 1685823330.9991486 mon.a (mon.0) 1120 : audit 0 from='client.? 172.21.15.89:0/1499104916' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-03T20:15:32.030 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:15:32.041 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:15:31 smithi164 bash[59462]: cluster 1685823330.789208 mgr.y (mgr.34104) 1185 : cluster 0 pgmap v1162: 89 pgs: 9 active+clean+remapped, 2 active+recovering+undersized+degraded+remapped, 13 active+undersized+degraded, 20 active+undersized, 45 active+clean; 110 MiB data, 4.9 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 832 op/s; 6906/5313 objects degraded (129.983%); 2/5313 objects misplaced (0.038%); 0 B/s, 90 objects/s recovering 2023-06-03T20:15:32.041 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:15:31 smithi164 bash[59462]: audit 1685823330.9991486 mon.a (mon.0) 1120 : audit 0 from='client.? 172.21.15.89:0/1499104916' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-03T20:15:32.063 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:15:32.067 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:15:32.068 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:15:32.168 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:15:32.199 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:15:32.202 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:15:32.202 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:15:32.303 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:15:32.337 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:15:32.339 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:15:32.340 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:15:32.441 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:15:32.469 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:15:32.471 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:15:32.471 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:15:32.572 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:15:32.599 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:15:32.601 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:15:32.601 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:15:32.702 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:15:32.736 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:15:32.738 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:15:32.739 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:15:32.839 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:15:32.865 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:15:32.867 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:15:32.867 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:15:32.969 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:15:32.992 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:15:32.995 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:15:32.995 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:15:33.028 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:15:32 smithi089 bash[78452]: cluster 1685823332.7815917 mon.a (mon.0) 1121 : cluster 3 Health check update: Degraded data redundancy: 6906/5313 objects degraded (129.983%), 15 pgs degraded, 35 pgs undersized (PG_DEGRADED) 2023-06-03T20:15:33.029 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:15:32 smithi089 bash[75068]: cluster 1685823332.7815917 mon.a (mon.0) 1121 : cluster 3 Health check update: Degraded data redundancy: 6906/5313 objects degraded (129.983%), 15 pgs degraded, 35 pgs undersized (PG_DEGRADED) 2023-06-03T20:15:33.096 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:15:33.127 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:15:33.130 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:15:33.131 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:15:33.232 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:15:33.257 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:15:33.259 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:15:33.260 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:15:33.333 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:15:32 smithi164 bash[59462]: cluster 1685823332.7815917 mon.a (mon.0) 1121 : cluster 3 Health check update: Degraded data redundancy: 6906/5313 objects degraded (129.983%), 15 pgs degraded, 35 pgs undersized (PG_DEGRADED) 2023-06-03T20:15:33.361 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:15:33.384 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:15:33.386 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:15:33.386 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:15:33.487 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:15:33.514 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:15:33.516 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:15:33.516 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:15:33.617 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:15:33.659 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:15:33.662 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:15:33.662 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:15:33.763 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:15:33.821 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:15:33.823 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:15:33.824 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:15:33.925 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:15:33.953 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:15:33.955 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:15:33.956 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:15:34.057 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:15:34.086 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:15:34.088 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:15:34.089 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:15:34.191 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:15:34.205 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:15:33 smithi089 bash[78452]: cluster 1685823332.7899244 mgr.y (mgr.34104) 1186 : cluster 0 pgmap v1163: 89 pgs: 9 active+clean+remapped, 2 active+recovering+undersized+degraded+remapped, 12 active+undersized+degraded, 21 active+undersized, 45 active+clean; 98 MiB data, 4.8 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 523 op/s; 6714/4737 objects degraded (141.735%); 2/4737 objects misplaced (0.042%); 0 B/s, 40 objects/s recovering 2023-06-03T20:15:34.206 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:15:33 smithi089 bash[75068]: cluster 1685823332.7899244 mgr.y (mgr.34104) 1186 : cluster 0 pgmap v1163: 89 pgs: 9 active+clean+remapped, 2 active+recovering+undersized+degraded+remapped, 12 active+undersized+degraded, 21 active+undersized, 45 active+clean; 98 MiB data, 4.8 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 523 op/s; 6714/4737 objects degraded (141.735%); 2/4737 objects misplaced (0.042%); 0 B/s, 40 objects/s recovering 2023-06-03T20:15:34.224 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:15:34.226 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:15:34.227 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:15:34.328 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:15:34.333 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:15:33 smithi164 bash[59462]: cluster 1685823332.7899244 mgr.y (mgr.34104) 1186 : cluster 0 pgmap v1163: 89 pgs: 9 active+clean+remapped, 2 active+recovering+undersized+degraded+remapped, 12 active+undersized+degraded, 21 active+undersized, 45 active+clean; 98 MiB data, 4.8 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 523 op/s; 6714/4737 objects degraded (141.735%); 2/4737 objects misplaced (0.042%); 0 B/s, 40 objects/s recovering 2023-06-03T20:15:34.358 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:15:34.360 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:15:34.360 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:15:34.461 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:15:34.487 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:15:34.490 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:15:34.490 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:15:34.591 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:15:34.622 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:15:34.624 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:15:34.625 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:15:34.726 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:15:34.756 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:15:34.758 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:15:34.759 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:15:34.861 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:15:34.894 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:15:34.896 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:15:34.897 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:15:34.998 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:15:35.033 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:15:35.035 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:15:35.036 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:15:35.137 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:15:35.169 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:15:35.171 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:15:35.172 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:15:35.273 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:15:35.308 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:15:35.311 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:15:35.311 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:15:35.412 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:15:35.440 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:15:35.442 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:15:35.442 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:15:35.543 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:15:35.572 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:15:35.574 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:15:35.575 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:15:35.676 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:15:35.702 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:15:35.704 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:15:35.704 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:15:35.805 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:15:35.827 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:15:35.829 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:15:35.830 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:15:35.931 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:15:35.969 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:15:35.972 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:15:35.972 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:15:36.028 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:15:35 smithi089 bash[75068]: cluster 1685823334.7913432 mgr.y (mgr.34104) 1187 : cluster 0 pgmap v1164: 89 pgs: 9 active+clean+remapped, 2 active+recovering+undersized+degraded+remapped, 12 active+undersized+degraded, 21 active+undersized, 45 active+clean; 6.8 MiB data, 4.6 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 555 op/s; 6021/375 objects degraded (1605.600%); 2/375 objects misplaced (0.533%); 0 B/s, 77 objects/s recovering 2023-06-03T20:15:36.029 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:15:35 smithi089 bash[78452]: cluster 1685823334.7913432 mgr.y (mgr.34104) 1187 : cluster 0 pgmap v1164: 89 pgs: 9 active+clean+remapped, 2 active+recovering+undersized+degraded+remapped, 12 active+undersized+degraded, 21 active+undersized, 45 active+clean; 6.8 MiB data, 4.6 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 555 op/s; 6021/375 objects degraded (1605.600%); 2/375 objects misplaced (0.533%); 0 B/s, 77 objects/s recovering 2023-06-03T20:15:36.073 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:15:36.108 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:15:36.110 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:15:36.111 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:15:36.212 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:15:36.222 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:15:35 smithi164 bash[59462]: cluster 1685823334.7913432 mgr.y (mgr.34104) 1187 : cluster 0 pgmap v1164: 89 pgs: 9 active+clean+remapped, 2 active+recovering+undersized+degraded+remapped, 12 active+undersized+degraded, 21 active+undersized, 45 active+clean; 6.8 MiB data, 4.6 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 555 op/s; 6021/375 objects degraded (1605.600%); 2/375 objects misplaced (0.533%); 0 B/s, 77 objects/s recovering 2023-06-03T20:15:36.244 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:15:36.246 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:15:36.246 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:15:36.347 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:15:36.354 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 20:15:36 smithi089 bash[71163]: ::ffff:172.21.15.164 - - [03/Jun/2023:20:15:36] "GET /metrics HTTP/1.1" 200 228892 "" "Prometheus/2.33.4" 2023-06-03T20:15:36.370 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:15:36.373 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:15:36.373 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:15:36.474 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:15:36.514 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:15:36.516 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:15:36.517 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:15:36.618 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:15:36.640 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:15:36.642 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:15:36.643 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:15:36.691 INFO:teuthology.orchestra.run.smithi089.stdout:ERROR: (22) Invalid argument 2023-06-03T20:15:36.691 INFO:teuthology.orchestra.run.smithi089.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-06-03T20:15:36.744 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:15:36.769 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:15:36.770 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:15:36.771 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:15:36.872 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:15:36.908 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:15:36.910 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:15:36.911 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:15:36.911 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:15:36 smithi089 bash[78452]: audit 1685823335.9605277 mon.c (mon.2) 533 : audit 1 from='client.? 172.21.15.89:0/444202585' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-06-03T20:15:36.911 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:15:36 smithi089 bash[78452]: audit 1685823335.9611704 mon.a (mon.0) 1122 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-06-03T20:15:36.919 INFO:teuthology.orchestra.run.smithi089.stderr:noscrub is set 2023-06-03T20:15:37.011 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:15:37.037 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:15:37.039 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:15:37.040 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:15:37.141 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:15:37.150 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:15:36 smithi164 bash[59462]: audit 1685823335.9605277 mon.c (mon.2) 533 : audit 1 from='client.? 172.21.15.89:0/444202585' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-06-03T20:15:37.150 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:15:36 smithi164 bash[59462]: audit 1685823335.9611704 mon.a (mon.0) 1122 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-06-03T20:15:37.170 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:15:37.177 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:15:37.177 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:15:37.278 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:15:37.313 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:15:37.315 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:15:37.316 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:15:37.323 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:15:36 smithi089 bash[75068]: audit 1685823335.9605277 mon.c (mon.2) 533 : audit 1 from='client.? 172.21.15.89:0/444202585' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-06-03T20:15:37.323 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:15:36 smithi089 bash[75068]: audit 1685823335.9611704 mon.a (mon.0) 1122 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-06-03T20:15:37.417 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:15:37.443 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:15:37.444 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:15:37.445 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:15:37.546 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:15:37.575 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:15:37.577 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:15:37.578 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:15:37.679 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:15:37.702 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:15:37.706 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:15:37.707 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:15:37.761 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-06-03T20:15:37.762 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-06-03T20:15:37.808 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:15:37.837 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:15:37.838 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:15:37.839 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:15:37.940 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:15:37.970 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:15:37.972 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:15:37.973 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:15:38.029 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:15:37 smithi089 bash[78452]: cluster 1685823336.7920516 mgr.y (mgr.34104) 1188 : cluster 0 pgmap v1165: 89 pgs: 9 active+clean+remapped, 2 active+recovering+undersized+degraded+remapped, 11 active+undersized+degraded, 22 active+undersized, 45 active+clean; 582 KiB data, 4.6 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 481 op/s; 5952/72 objects degraded (8266.667%); 2/72 objects misplaced (2.778%); 0 B/s, 69 objects/s recovering 2023-06-03T20:15:38.029 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:15:37 smithi089 bash[78452]: cluster 1685823336.8991861 mon.a (mon.0) 1123 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-06-03T20:15:38.029 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:15:37 smithi089 bash[78452]: cluster 1685823336.905017 osd.7 (osd.7) 134 : cluster 3 failed to encode map e208 with expected crc 2023-06-03T20:15:38.029 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:15:37 smithi089 bash[78452]: audit 1685823336.911214 mon.a (mon.0) 1124 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-06-03T20:15:38.029 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:15:37 smithi089 bash[78452]: cluster 1685823336.9113045 mon.a (mon.0) 1125 : cluster 0 osdmap e208: 8 total, 7 up, 7 in 2023-06-03T20:15:38.030 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:15:37 smithi089 bash[78452]: cluster 1685823337.1225607 osd.5 (osd.5) 140 : cluster 3 failed to encode map e208 with expected crc 2023-06-03T20:15:38.030 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:15:37 smithi089 bash[78452]: audit 1685823337.129375 mon.a (mon.0) 1126 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:15:38.030 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:15:37 smithi089 bash[78452]: audit 1685823337.1310718 mon.c (mon.2) 534 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T20:15:38.030 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:15:37 smithi089 bash[78452]: cluster 1685823337.3508317 osd.6 (osd.6) 152 : cluster 3 failed to encode map e208 with expected crc 2023-06-03T20:15:38.030 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:15:37 smithi089 bash[78452]: cluster 1685823337.7827096 mon.a (mon.0) 1127 : cluster 3 Health check update: Degraded data redundancy: 5952/72 objects degraded (8266.667%), 13 pgs degraded, 35 pgs undersized (PG_DEGRADED) 2023-06-03T20:15:38.030 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:15:37 smithi089 bash[75068]: cluster 1685823336.7920516 mgr.y (mgr.34104) 1188 : cluster 0 pgmap v1165: 89 pgs: 9 active+clean+remapped, 2 active+recovering+undersized+degraded+remapped, 11 active+undersized+degraded, 22 active+undersized, 45 active+clean; 582 KiB data, 4.6 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 481 op/s; 5952/72 objects degraded (8266.667%); 2/72 objects misplaced (2.778%); 0 B/s, 69 objects/s recovering 2023-06-03T20:15:38.031 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:15:37 smithi089 bash[75068]: cluster 1685823336.8991861 mon.a (mon.0) 1123 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-06-03T20:15:38.031 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:15:37 smithi089 bash[75068]: cluster 1685823336.905017 osd.7 (osd.7) 134 : cluster 3 failed to encode map e208 with expected crc 2023-06-03T20:15:38.031 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:15:37 smithi089 bash[75068]: audit 1685823336.911214 mon.a (mon.0) 1124 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-06-03T20:15:38.031 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:15:37 smithi089 bash[75068]: cluster 1685823336.9113045 mon.a (mon.0) 1125 : cluster 0 osdmap e208: 8 total, 7 up, 7 in 2023-06-03T20:15:38.031 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:15:37 smithi089 bash[75068]: cluster 1685823337.1225607 osd.5 (osd.5) 140 : cluster 3 failed to encode map e208 with expected crc 2023-06-03T20:15:38.031 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:15:37 smithi089 bash[75068]: audit 1685823337.129375 mon.a (mon.0) 1126 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:15:38.031 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:15:37 smithi089 bash[75068]: audit 1685823337.1310718 mon.c (mon.2) 534 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T20:15:38.031 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:15:37 smithi089 bash[75068]: cluster 1685823337.3508317 osd.6 (osd.6) 152 : cluster 3 failed to encode map e208 with expected crc 2023-06-03T20:15:38.032 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:15:37 smithi089 bash[75068]: cluster 1685823337.7827096 mon.a (mon.0) 1127 : cluster 3 Health check update: Degraded data redundancy: 5952/72 objects degraded (8266.667%), 13 pgs degraded, 35 pgs undersized (PG_DEGRADED) 2023-06-03T20:15:38.074 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:15:38.102 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:15:38.105 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:15:38.105 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:15:38.206 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:15:38.237 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:15:38.239 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:15:38.240 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:15:38.333 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:15:37 smithi164 bash[59462]: cluster 1685823336.7920516 mgr.y (mgr.34104) 1188 : cluster 0 pgmap v1165: 89 pgs: 9 active+clean+remapped, 2 active+recovering+undersized+degraded+remapped, 11 active+undersized+degraded, 22 active+undersized, 45 active+clean; 582 KiB data, 4.6 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 481 op/s; 5952/72 objects degraded (8266.667%); 2/72 objects misplaced (2.778%); 0 B/s, 69 objects/s recovering 2023-06-03T20:15:38.334 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:15:37 smithi164 bash[59462]: cluster 1685823336.8991861 mon.a (mon.0) 1123 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-06-03T20:15:38.334 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:15:37 smithi164 bash[59462]: cluster 1685823336.905017 osd.7 (osd.7) 134 : cluster 3 failed to encode map e208 with expected crc 2023-06-03T20:15:38.334 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:15:37 smithi164 bash[59462]: audit 1685823336.911214 mon.a (mon.0) 1124 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-06-03T20:15:38.334 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:15:37 smithi164 bash[59462]: cluster 1685823336.9113045 mon.a (mon.0) 1125 : cluster 0 osdmap e208: 8 total, 7 up, 7 in 2023-06-03T20:15:38.334 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:15:37 smithi164 bash[59462]: cluster 1685823337.1225607 osd.5 (osd.5) 140 : cluster 3 failed to encode map e208 with expected crc 2023-06-03T20:15:38.334 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:15:37 smithi164 bash[59462]: audit 1685823337.129375 mon.a (mon.0) 1126 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:15:38.335 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:15:37 smithi164 bash[59462]: audit 1685823337.1310718 mon.c (mon.2) 534 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T20:15:38.335 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:15:37 smithi164 bash[59462]: cluster 1685823337.3508317 osd.6 (osd.6) 152 : cluster 3 failed to encode map e208 with expected crc 2023-06-03T20:15:38.335 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:15:37 smithi164 bash[59462]: cluster 1685823337.7827096 mon.a (mon.0) 1127 : cluster 3 Health check update: Degraded data redundancy: 5952/72 objects degraded (8266.667%), 13 pgs degraded, 35 pgs undersized (PG_DEGRADED) 2023-06-03T20:15:38.340 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:15:38.370 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:15:38.374 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:15:38.375 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:15:38.476 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:15:38.502 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:15:38.504 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:15:38.505 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:15:38.606 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:15:38.634 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:15:38.636 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:15:38.637 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:15:38.738 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:15:38.772 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:15:38.776 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:15:38.777 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:15:38.878 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:15:38.907 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:15:38.909 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:15:38.910 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:15:39.011 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:15:39.046 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:15:39.049 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:15:39.050 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:15:39.151 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:15:39.181 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:15:39.183 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:15:39.183 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:15:39.285 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:15:39.312 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:15:39.314 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:15:39.315 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:15:39.417 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:15:39.450 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:15:39.454 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:15:39.454 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:15:39.556 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:15:39.590 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:15:39.592 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:15:39.593 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:15:39.694 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:15:39.723 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:15:39.726 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:15:39.726 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:15:39.764 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-06-03T20:15:39.828 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:15:39.861 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:15:39.864 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:15:39.865 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:15:39.915 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:15:39 smithi089 bash[75068]: cluster 1685823338.7935643 mgr.y (mgr.34104) 1189 : cluster 0 pgmap v1167: 89 pgs: 9 active+clean+remapped, 2 active+recovering+undersized+degraded+remapped, 9 active+undersized+degraded, 24 active+undersized, 45 active+clean; 582 KiB data, 4.6 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 254 op/s; 5369/72 objects degraded (7456.944%); 2/72 objects misplaced (2.778%); 0 B/s, 94 objects/s recovering 2023-06-03T20:15:39.966 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:15:39.995 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:15:39.997 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:15:39.997 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:15:40.098 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:15:40.109 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:15:39 smithi164 bash[59462]: cluster 1685823338.7935643 mgr.y (mgr.34104) 1189 : cluster 0 pgmap v1167: 89 pgs: 9 active+clean+remapped, 2 active+recovering+undersized+degraded+remapped, 9 active+undersized+degraded, 24 active+undersized, 45 active+clean; 582 KiB data, 4.6 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 254 op/s; 5369/72 objects degraded (7456.944%); 2/72 objects misplaced (2.778%); 0 B/s, 94 objects/s recovering 2023-06-03T20:15:40.133 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:15:40.135 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:15:40.135 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:15:40.236 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:15:40.247 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:15:39 smithi089 bash[78452]: cluster 1685823338.7935643 mgr.y (mgr.34104) 1189 : cluster 0 pgmap v1167: 89 pgs: 9 active+clean+remapped, 2 active+recovering+undersized+degraded+remapped, 9 active+undersized+degraded, 24 active+undersized, 45 active+clean; 582 KiB data, 4.6 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 254 op/s; 5369/72 objects degraded (7456.944%); 2/72 objects misplaced (2.778%); 0 B/s, 94 objects/s recovering 2023-06-03T20:15:40.254 INFO:teuthology.orchestra.run.smithi089.stdout:{ 2023-06-03T20:15:40.254 INFO:teuthology.orchestra.run.smithi089.stdout: "success": "filestore_inject_stall = '3' (not observed, change may require restart) " 2023-06-03T20:15:40.254 INFO:teuthology.orchestra.run.smithi089.stdout:} 2023-06-03T20:15:40.264 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:15:40.266 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:15:40.267 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:15:40.368 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:15:40.403 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:15:40.405 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:15:40.406 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:15:40.507 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:15:40.537 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:15:40.539 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:15:40.540 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:15:40.641 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:15:40.677 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:15:40.680 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:15:40.681 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:15:40.782 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:15:40.816 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:15:40.819 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:15:40.819 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:15:40.920 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:15:40.948 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:15:40.949 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:15:40.950 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:15:41.051 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:15:41.079 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:15:41.081 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:15:41.081 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:15:41.182 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:15:41.207 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:15:41.210 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:15:41.211 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:15:41.312 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:15:41.348 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:15:41.351 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:15:41.351 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:15:41.452 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:15:41.487 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:15:41.490 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:15:41.490 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:15:41.591 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:15:41.621 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:15:41.623 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:15:41.624 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:15:41.724 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:15:41.758 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:15:41.761 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:15:41.761 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:15:41.862 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:15:41.894 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:15:41.896 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:15:41.897 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:15:41.926 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:15:41 smithi089 bash[75068]: cluster 1685823340.7943478 mgr.y (mgr.34104) 1190 : cluster 0 pgmap v1168: 89 pgs: 9 active+clean+remapped, 2 active+recovering+undersized+degraded+remapped, 7 active+undersized+degraded, 26 active+undersized, 45 active+clean; 582 KiB data, 4.6 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 174 op/s; 5369/72 objects degraded (7456.944%); 2/72 objects misplaced (2.778%); 0 B/s, 94 objects/s recovering 2023-06-03T20:15:41.998 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:15:42.010 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:15:41 smithi164 bash[59462]: cluster 1685823340.7943478 mgr.y (mgr.34104) 1190 : cluster 0 pgmap v1168: 89 pgs: 9 active+clean+remapped, 2 active+recovering+undersized+degraded+remapped, 7 active+undersized+degraded, 26 active+undersized, 45 active+clean; 582 KiB data, 4.6 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 174 op/s; 5369/72 objects degraded (7456.944%); 2/72 objects misplaced (2.778%); 0 B/s, 94 objects/s recovering 2023-06-03T20:15:42.031 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:15:42.033 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:15:42.034 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:15:42.135 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:15:42.166 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:15:42.169 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:15:42.170 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:15:42.271 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:15:42.278 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:15:41 smithi089 bash[78452]: cluster 1685823340.7943478 mgr.y (mgr.34104) 1190 : cluster 0 pgmap v1168: 89 pgs: 9 active+clean+remapped, 2 active+recovering+undersized+degraded+remapped, 7 active+undersized+degraded, 26 active+undersized, 45 active+clean; 582 KiB data, 4.6 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 174 op/s; 5369/72 objects degraded (7456.944%); 2/72 objects misplaced (2.778%); 0 B/s, 94 objects/s recovering 2023-06-03T20:15:42.303 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:15:42.305 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:15:42.305 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:15:42.406 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:15:42.432 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:15:42.434 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:15:42.435 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:15:42.536 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:15:42.566 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:15:42.571 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:15:42.572 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:15:42.673 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:15:42.702 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:15:42.704 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:15:42.705 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:15:42.805 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:15:42.833 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:15:42.836 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:15:42.836 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:15:42.937 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:15:42.949 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:15:42 smithi164 bash[59462]: cluster 1685823342.7843232 mon.a (mon.0) 1128 : cluster 3 Health check update: Degraded data redundancy: 5369/72 objects degraded (7456.944%), 9 pgs degraded, 35 pgs undersized (PG_DEGRADED) 2023-06-03T20:15:42.961 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph daemon osd.2 dump_ops_in_flight 2023-06-03T20:15:42.971 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:15:42.973 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:15:42.974 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:15:43.075 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:15:43.113 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:15:43.116 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:15:43.117 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:15:43.217 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:15:43.229 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:15:42 smithi089 bash[78452]: cluster 1685823342.7843232 mon.a (mon.0) 1128 : cluster 3 Health check update: Degraded data redundancy: 5369/72 objects degraded (7456.944%), 9 pgs degraded, 35 pgs undersized (PG_DEGRADED) 2023-06-03T20:15:43.230 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:15:42 smithi089 bash[75068]: cluster 1685823342.7843232 mon.a (mon.0) 1128 : cluster 3 Health check update: Degraded data redundancy: 5369/72 objects degraded (7456.944%), 9 pgs degraded, 35 pgs undersized (PG_DEGRADED) 2023-06-03T20:15:43.248 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:15:43.250 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:15:43.251 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:15:43.351 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:15:43.382 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:15:43.384 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:15:43.385 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:15:43.486 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:15:43.516 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:15:43.518 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:15:43.518 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:15:43.619 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:15:43.652 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:15:43.656 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:15:43.656 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:15:43.757 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:15:43.793 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:15:43.796 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:15:43.797 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:15:43.898 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:15:43.931 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:15:43.935 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:15:43.935 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:15:44.036 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:15:44.069 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:15:44.072 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:15:44.072 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:15:44.083 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:15:43 smithi164 bash[59462]: cluster 1685823342.7952688 mgr.y (mgr.34104) 1191 : cluster 0 pgmap v1169: 89 pgs: 9 active+clean+remapped, 2 active+recovering+undersized+degraded+remapped, 7 active+undersized+degraded, 26 active+undersized, 45 active+clean; 582 KiB data, 4.6 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 155 op/s; 5369/72 objects degraded (7456.944%); 2/72 objects misplaced (2.778%); 0 B/s, 94 objects/s recovering 2023-06-03T20:15:44.173 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:15:44.206 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:15:44.209 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:15:44.210 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:15:44.279 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:15:43 smithi089 bash[75068]: cluster 1685823342.7952688 mgr.y (mgr.34104) 1191 : cluster 0 pgmap v1169: 89 pgs: 9 active+clean+remapped, 2 active+recovering+undersized+degraded+remapped, 7 active+undersized+degraded, 26 active+undersized, 45 active+clean; 582 KiB data, 4.6 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 155 op/s; 5369/72 objects degraded (7456.944%); 2/72 objects misplaced (2.778%); 0 B/s, 94 objects/s recovering 2023-06-03T20:15:44.279 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:15:43 smithi089 bash[78452]: cluster 1685823342.7952688 mgr.y (mgr.34104) 1191 : cluster 0 pgmap v1169: 89 pgs: 9 active+clean+remapped, 2 active+recovering+undersized+degraded+remapped, 7 active+undersized+degraded, 26 active+undersized, 45 active+clean; 582 KiB data, 4.6 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 155 op/s; 5369/72 objects degraded (7456.944%); 2/72 objects misplaced (2.778%); 0 B/s, 94 objects/s recovering 2023-06-03T20:15:44.310 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:15:44.342 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:15:44.346 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:15:44.346 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:15:44.447 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:15:44.484 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:15:44.486 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:15:44.487 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:15:44.588 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:15:44.619 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:15:44.621 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:15:44.622 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:15:44.723 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:15:44.751 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:15:44.753 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:15:44.754 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:15:44.779 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph osd set nodeep-scrub 2023-06-03T20:15:44.855 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:15:44.888 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:15:44.890 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:15:44.891 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:15:44.991 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:15:45.028 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:15:45.030 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:15:45.031 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:15:45.132 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:15:45.168 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:15:45.170 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:15:45.171 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:15:45.272 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:15:45.313 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:15:45.317 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:15:45.318 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:15:45.418 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:15:45.455 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:15:45.458 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:15:45.459 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:15:45.559 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:15:45.595 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:15:45.597 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:15:45.598 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:15:45.699 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:15:45.737 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:15:45.740 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:15:45.741 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:15:45.842 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:15:45.878 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:15:45.880 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:15:45.881 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:15:45.982 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:15:46.016 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:15:46.018 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:15:46.018 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:15:46.120 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:15:46.152 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:15:46.154 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:15:46.155 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:15:46.231 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:15:45 smithi089 bash[78452]: cluster 1685823344.7965066 mgr.y (mgr.34104) 1192 : cluster 0 pgmap v1170: 89 pgs: 9 active+clean+remapped, 2 active+recovering+undersized+degraded+remapped, 7 active+undersized+degraded, 26 active+undersized, 45 active+clean; 582 KiB data, 4.6 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 10 op/s; 4797/72 objects degraded (6662.500%); 2/72 objects misplaced (2.778%); 0 B/s, 115 objects/s recovering 2023-06-03T20:15:46.231 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:15:45 smithi089 bash[75068]: cluster 1685823344.7965066 mgr.y (mgr.34104) 1192 : cluster 0 pgmap v1170: 89 pgs: 9 active+clean+remapped, 2 active+recovering+undersized+degraded+remapped, 7 active+undersized+degraded, 26 active+undersized, 45 active+clean; 582 KiB data, 4.6 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 10 op/s; 4797/72 objects degraded (6662.500%); 2/72 objects misplaced (2.778%); 0 B/s, 115 objects/s recovering 2023-06-03T20:15:46.256 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:15:46.295 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:15:46.297 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:15:46.297 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:15:46.319 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/mon.a/config 2023-06-03T20:15:46.333 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:15:45 smithi164 bash[59462]: cluster 1685823344.7965066 mgr.y (mgr.34104) 1192 : cluster 0 pgmap v1170: 89 pgs: 9 active+clean+remapped, 2 active+recovering+undersized+degraded+remapped, 7 active+undersized+degraded, 26 active+undersized, 45 active+clean; 582 KiB data, 4.6 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 10 op/s; 4797/72 objects degraded (6662.500%); 2/72 objects misplaced (2.778%); 0 B/s, 115 objects/s recovering 2023-06-03T20:15:46.399 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:15:46.432 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:15:46.435 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:15:46.436 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:15:46.528 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 20:15:46 smithi089 bash[71163]: ::ffff:172.21.15.164 - - [03/Jun/2023:20:15:46] "GET /metrics HTTP/1.1" 200 228870 "" "Prometheus/2.33.4" 2023-06-03T20:15:46.537 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:15:46.573 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:15:46.575 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:15:46.576 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:15:46.677 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:15:46.712 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:15:46.715 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:15:46.716 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:15:46.817 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:15:46.847 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:15:46.849 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:15:46.849 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:15:46.950 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:15:46.980 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:15:47.376 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:15:47.377 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:15:47.379 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/mon.a/config 2023-06-03T20:15:47.379 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/mon.a/config 2023-06-03T20:15:47.479 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:15:47.511 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:15:47.514 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:15:47.515 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:15:47.616 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:15:47.650 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:15:47.653 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:15:47.654 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:15:47.756 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:15:47.785 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:15:47.787 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:15:47.788 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:15:47.889 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:15:47.919 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:15:47.921 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:15:47.921 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:15:48.022 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:15:48.059 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:15:48.062 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:15:48.063 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:15:48.164 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:15:48.195 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:15:48.196 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:15:47 smithi164 bash[59462]: cluster 1685823346.7975013 mgr.y (mgr.34104) 1193 : cluster 0 pgmap v1171: 89 pgs: 9 active+clean+remapped, 2 active+recovering+undersized+degraded+remapped, 7 active+undersized+degraded, 26 active+undersized, 45 active+clean; 582 KiB data, 4.6 GiB used, 621 GiB / 626 GiB avail; 4797/72 objects degraded (6662.500%); 2/72 objects misplaced (2.778%); 0 B/s, 115 objects/s recovering 2023-06-03T20:15:48.197 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:15:47 smithi164 bash[59462]: cluster 1685823347.785395 mon.a (mon.0) 1129 : cluster 3 Health check update: Degraded data redundancy: 4797/72 objects degraded (6662.500%), 9 pgs degraded, 35 pgs undersized (PG_DEGRADED) 2023-06-03T20:15:48.197 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:15:48.198 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:15:48.279 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:15:47 smithi089 bash[78452]: cluster 1685823346.7975013 mgr.y (mgr.34104) 1193 : cluster 0 pgmap v1171: 89 pgs: 9 active+clean+remapped, 2 active+recovering+undersized+degraded+remapped, 7 active+undersized+degraded, 26 active+undersized, 45 active+clean; 582 KiB data, 4.6 GiB used, 621 GiB / 626 GiB avail; 4797/72 objects degraded (6662.500%); 2/72 objects misplaced (2.778%); 0 B/s, 115 objects/s recovering 2023-06-03T20:15:48.279 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:15:47 smithi089 bash[78452]: cluster 1685823347.785395 mon.a (mon.0) 1129 : cluster 3 Health check update: Degraded data redundancy: 4797/72 objects degraded (6662.500%), 9 pgs degraded, 35 pgs undersized (PG_DEGRADED) 2023-06-03T20:15:48.279 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:15:47 smithi089 bash[75068]: cluster 1685823346.7975013 mgr.y (mgr.34104) 1193 : cluster 0 pgmap v1171: 89 pgs: 9 active+clean+remapped, 2 active+recovering+undersized+degraded+remapped, 7 active+undersized+degraded, 26 active+undersized, 45 active+clean; 582 KiB data, 4.6 GiB used, 621 GiB / 626 GiB avail; 4797/72 objects degraded (6662.500%); 2/72 objects misplaced (2.778%); 0 B/s, 115 objects/s recovering 2023-06-03T20:15:48.280 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:15:47 smithi089 bash[75068]: cluster 1685823347.785395 mon.a (mon.0) 1129 : cluster 3 Health check update: Degraded data redundancy: 4797/72 objects degraded (6662.500%), 9 pgs degraded, 35 pgs undersized (PG_DEGRADED) 2023-06-03T20:15:48.298 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:15:48.330 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:15:48.332 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:15:48.333 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:15:48.434 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:15:48.461 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:15:48.463 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:15:48.463 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:15:48.564 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:15:48.592 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:15:48.594 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:15:48.595 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:15:48.696 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:15:48.731 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:15:48.732 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:15:48.733 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:15:48.834 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:15:48.859 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:15:48.863 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:15:48.864 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:15:48.965 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:15:48.997 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:15:49.000 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:15:49.000 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:15:49.101 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:15:49.136 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:15:49.139 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:15:49.139 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:15:49.240 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:15:49.268 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:15:49.270 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:15:49.270 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:15:49.296 INFO:tasks.thrashosds.thrasher:in_osds: [0, 2, 3, 4, 5, 6, 7] out_osds: [1] dead_osds: [4] live_osds: [0, 1, 2, 3, 5, 6, 7] 2023-06-03T20:15:49.296 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-06-03T20:15:49.296 INFO:tasks.thrashosds.thrasher:Removing osd 6, in_osds are: [0, 2, 3, 4, 5, 6, 7] 2023-06-03T20:15:49.296 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph osd out 6 2023-06-03T20:15:49.371 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:15:49.403 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:15:49.405 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:15:49.406 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:15:49.406 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/mon.a/config 2023-06-03T20:15:49.507 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:15:49.534 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:15:49.536 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:15:49.537 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:15:49.638 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:15:49.673 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:15:49.675 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:15:49.675 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:15:49.776 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:15:49.804 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:15:49.805 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:15:49.806 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:15:49.907 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:15:49.936 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:15:49.938 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:15:49.939 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:15:50.040 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:15:50.051 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:15:49 smithi089 bash[78452]: cluster 1685823348.7987692 mgr.y (mgr.34104) 1194 : cluster 0 pgmap v1172: 89 pgs: 9 active+clean+remapped, 2 active+recovering+undersized+degraded+remapped, 7 active+undersized+degraded, 26 active+undersized, 45 active+clean; 582 KiB data, 4.5 GiB used, 621 GiB / 626 GiB avail; 4233/72 objects degraded (5879.167%); 2/72 objects misplaced (2.778%); 0 B/s, 144 objects/s recovering 2023-06-03T20:15:50.051 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:15:49 smithi089 bash[75068]: cluster 1685823348.7987692 mgr.y (mgr.34104) 1194 : cluster 0 pgmap v1172: 89 pgs: 9 active+clean+remapped, 2 active+recovering+undersized+degraded+remapped, 7 active+undersized+degraded, 26 active+undersized, 45 active+clean; 582 KiB data, 4.5 GiB used, 621 GiB / 626 GiB avail; 4233/72 objects degraded (5879.167%); 2/72 objects misplaced (2.778%); 0 B/s, 144 objects/s recovering 2023-06-03T20:15:50.071 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:15:50.073 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:15:50.074 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:15:50.175 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:15:50.204 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:15:50.206 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:15:50.206 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:15:50.307 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:15:50.318 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:15:49 smithi164 bash[59462]: cluster 1685823348.7987692 mgr.y (mgr.34104) 1194 : cluster 0 pgmap v1172: 89 pgs: 9 active+clean+remapped, 2 active+recovering+undersized+degraded+remapped, 7 active+undersized+degraded, 26 active+undersized, 45 active+clean; 582 KiB data, 4.5 GiB used, 621 GiB / 626 GiB avail; 4233/72 objects degraded (5879.167%); 2/72 objects misplaced (2.778%); 0 B/s, 144 objects/s recovering 2023-06-03T20:15:50.338 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:15:50.340 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:15:50.341 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:15:50.443 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:15:50.474 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:15:50.477 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:15:50.477 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:15:50.578 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:15:50.606 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:15:50.608 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:15:50.609 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:15:50.710 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:15:50.739 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:15:50.742 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:15:50.743 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:15:50.844 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:15:50.886 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:15:50.887 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:15:50.888 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:15:50.988 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:15:51.024 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:15:51.030 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:15:51.030 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:15:51.131 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:15:51.171 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:15:51.174 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:15:51.174 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:15:51.275 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:15:51.318 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:15:51.320 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:15:51.320 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:15:51.421 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:15:51.780 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:15:51.782 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:15:51.783 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:15:51.884 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:15:51.915 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:15:51.918 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:15:51.919 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:15:52.021 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:15:52.052 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:15:52.053 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:15:52.054 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:15:52.155 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:15:52.186 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:15:52.189 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:15:52.189 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:15:52.290 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:15:52.321 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:15:52.323 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:15:52.324 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:15:52.333 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:15:51 smithi164 bash[59462]: cluster 1685823350.7994978 mgr.y (mgr.34104) 1195 : cluster 0 pgmap v1173: 89 pgs: 9 active+clean+remapped, 2 active+recovering+undersized+degraded+remapped, 7 active+undersized+degraded, 26 active+undersized, 45 active+clean; 582 KiB data, 4.5 GiB used, 621 GiB / 626 GiB avail; 4233/72 objects degraded (5879.167%); 2/72 objects misplaced (2.778%); 0 B/s, 94 objects/s recovering 2023-06-03T20:15:52.425 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:15:52.457 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:15:52.459 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:15:52.460 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:15:52.541 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:15:52 smithi089 bash[75068]: cluster 1685823350.7994978 mgr.y (mgr.34104) 1195 : cluster 0 pgmap v1173: 89 pgs: 9 active+clean+remapped, 2 active+recovering+undersized+degraded+remapped, 7 active+undersized+degraded, 26 active+undersized, 45 active+clean; 582 KiB data, 4.5 GiB used, 621 GiB / 626 GiB avail; 4233/72 objects degraded (5879.167%); 2/72 objects misplaced (2.778%); 0 B/s, 94 objects/s recovering 2023-06-03T20:15:52.542 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:15:52 smithi089 bash[78452]: cluster 1685823350.7994978 mgr.y (mgr.34104) 1195 : cluster 0 pgmap v1173: 89 pgs: 9 active+clean+remapped, 2 active+recovering+undersized+degraded+remapped, 7 active+undersized+degraded, 26 active+undersized, 45 active+clean; 582 KiB data, 4.5 GiB used, 621 GiB / 626 GiB avail; 4233/72 objects degraded (5879.167%); 2/72 objects misplaced (2.778%); 0 B/s, 94 objects/s recovering 2023-06-03T20:15:52.560 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:15:52.590 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:15:52.592 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:15:52.592 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:15:52.694 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:15:52.734 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:15:52.738 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:15:52.738 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:15:52.839 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:15:52.870 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:15:52.873 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:15:52.874 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:15:52.975 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:15:52.998 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:15:53.000 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:15:53.001 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:15:53.083 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:15:52 smithi164 bash[59462]: audit 1685823352.1261232 mon.c (mon.2) 535 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T20:15:53.084 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:15:52 smithi164 bash[59462]: cluster 1685823352.786532 mon.a (mon.0) 1130 : cluster 3 Health check update: Degraded data redundancy: 4233/72 objects degraded (5879.167%), 9 pgs degraded, 35 pgs undersized (PG_DEGRADED) 2023-06-03T20:15:53.102 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:15:53.128 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:15:53.130 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:15:53.130 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:15:53.232 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:15:53.243 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:15:52 smithi089 bash[75068]: audit 1685823352.1261232 mon.c (mon.2) 535 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T20:15:53.244 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:15:52 smithi089 bash[75068]: cluster 1685823352.786532 mon.a (mon.0) 1130 : cluster 3 Health check update: Degraded data redundancy: 4233/72 objects degraded (5879.167%), 9 pgs degraded, 35 pgs undersized (PG_DEGRADED) 2023-06-03T20:15:53.244 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:15:52 smithi089 bash[78452]: audit 1685823352.1261232 mon.c (mon.2) 535 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T20:15:53.245 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:15:52 smithi089 bash[78452]: cluster 1685823352.786532 mon.a (mon.0) 1130 : cluster 3 Health check update: Degraded data redundancy: 4233/72 objects degraded (5879.167%), 9 pgs degraded, 35 pgs undersized (PG_DEGRADED) 2023-06-03T20:15:53.262 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:15:53.263 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:15:53.264 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:15:53.366 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:15:53.398 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:15:53.400 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:15:53.401 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:15:53.463 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/mon.a/config 2023-06-03T20:15:53.501 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:15:53.529 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:15:53.530 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:15:53.531 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:15:53.632 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:15:53.660 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:15:53.662 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:15:53.662 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:15:53.764 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:15:53.801 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:15:53.803 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:15:53.803 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:15:53.905 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:15:53.932 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:15:53.934 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:15:53.934 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:15:54.035 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:15:54.062 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:15:54.066 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:15:54.067 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:15:54.083 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:15:53 smithi164 bash[59462]: cluster 1685823352.8002748 mgr.y (mgr.34104) 1196 : cluster 0 pgmap v1174: 89 pgs: 9 active+clean+remapped, 2 active+recovering+undersized+degraded+remapped, 7 active+undersized+degraded, 26 active+undersized, 45 active+clean; 582 KiB data, 4.5 GiB used, 621 GiB / 626 GiB avail; 4233/72 objects degraded (5879.167%); 2/72 objects misplaced (2.778%); 0 B/s, 94 objects/s recovering 2023-06-03T20:15:54.168 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:15:54.177 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:15:53 smithi089 bash[78452]: cluster 1685823352.8002748 mgr.y (mgr.34104) 1196 : cluster 0 pgmap v1174: 89 pgs: 9 active+clean+remapped, 2 active+recovering+undersized+degraded+remapped, 7 active+undersized+degraded, 26 active+undersized, 45 active+clean; 582 KiB data, 4.5 GiB used, 621 GiB / 626 GiB avail; 4233/72 objects degraded (5879.167%); 2/72 objects misplaced (2.778%); 0 B/s, 94 objects/s recovering 2023-06-03T20:15:54.178 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:15:53 smithi089 bash[75068]: cluster 1685823352.8002748 mgr.y (mgr.34104) 1196 : cluster 0 pgmap v1174: 89 pgs: 9 active+clean+remapped, 2 active+recovering+undersized+degraded+remapped, 7 active+undersized+degraded, 26 active+undersized, 45 active+clean; 582 KiB data, 4.5 GiB used, 621 GiB / 626 GiB avail; 4233/72 objects degraded (5879.167%); 2/72 objects misplaced (2.778%); 0 B/s, 94 objects/s recovering 2023-06-03T20:15:54.198 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:15:54.200 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:15:54.201 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:15:54.302 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:15:54.336 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:15:54.339 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:15:54.339 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:15:54.440 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:15:54.474 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:15:54.477 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:15:54.477 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:15:54.579 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:15:54.609 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:15:54.611 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:15:54.612 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:15:54.713 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:15:54.743 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:15:54.745 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:15:54.746 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:15:54.847 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:15:54.876 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:15:54.878 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:15:54.879 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:15:54.980 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:15:55.008 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:15:55.010 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:15:55.011 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:15:55.112 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:15:55.142 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:15:55.145 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:15:55.146 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:15:55.246 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:15:55.280 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:15:55.282 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:15:55.282 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:15:55.383 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:15:55.413 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:15:55.416 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:15:55.416 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:15:55.517 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:15:55.547 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:15:55.549 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:15:55.550 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:15:55.650 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:15:55.684 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:15:55.689 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:15:55.689 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:15:55.790 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:15:55.817 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:15:55.819 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:15:55.819 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:15:55.920 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:15:55.951 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:15:55.953 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:15:55.954 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:15:56.054 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:15:56.062 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:15:56 smithi089 bash[75068]: cluster 1685823354.8011422 mgr.y (mgr.34104) 1197 : cluster 0 pgmap v1175: 89 pgs: 9 active+clean+remapped, 2 active+recovering+undersized+degraded+remapped, 7 active+undersized+degraded, 26 active+undersized, 45 active+clean; 582 KiB data, 4.5 GiB used, 621 GiB / 626 GiB avail; 3557/72 objects degraded (4940.278%); 2/72 objects misplaced (2.778%); 0 B/s, 150 objects/s recovering 2023-06-03T20:15:56.063 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:15:56 smithi089 bash[78452]: cluster 1685823354.8011422 mgr.y (mgr.34104) 1197 : cluster 0 pgmap v1175: 89 pgs: 9 active+clean+remapped, 2 active+recovering+undersized+degraded+remapped, 7 active+undersized+degraded, 26 active+undersized, 45 active+clean; 582 KiB data, 4.5 GiB used, 621 GiB / 626 GiB avail; 3557/72 objects degraded (4940.278%); 2/72 objects misplaced (2.778%); 0 B/s, 150 objects/s recovering 2023-06-03T20:15:56.082 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:15:56.085 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:15:56.086 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:15:56.187 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:15:56.215 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:15:56.219 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:15:56.219 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:15:56.320 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:15:56.332 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:15:55 smithi164 bash[59462]: cluster 1685823354.8011422 mgr.y (mgr.34104) 1197 : cluster 0 pgmap v1175: 89 pgs: 9 active+clean+remapped, 2 active+recovering+undersized+degraded+remapped, 7 active+undersized+degraded, 26 active+undersized, 45 active+clean; 582 KiB data, 4.5 GiB used, 621 GiB / 626 GiB avail; 3557/72 objects degraded (4940.278%); 2/72 objects misplaced (2.778%); 0 B/s, 150 objects/s recovering 2023-06-03T20:15:56.351 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:15:56.354 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:15:56.355 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:15:56.455 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:15:56.465 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 20:15:56 smithi089 bash[71163]: ::ffff:172.21.15.164 - - [03/Jun/2023:20:15:56] "GET /metrics HTTP/1.1" 200 228858 "" "Prometheus/2.33.4" 2023-06-03T20:15:56.488 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:15:56.491 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:15:56.492 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:15:56.593 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:15:56.628 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:15:56.630 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:15:56.631 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:15:56.732 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:15:56.765 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:15:56.766 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:15:56.767 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:15:56.868 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:15:56.893 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:15:56.895 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:15:56.895 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:15:56.996 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:15:57.030 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:15:57.032 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:15:57.033 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:15:57.134 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:15:57.166 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:15:57.168 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:15:57.169 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:15:57.270 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:15:57.295 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:15:57.298 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:15:57.298 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:15:57.399 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:15:57.425 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:15:57.426 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:15:57.427 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:15:57.528 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:15:57.557 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:15:57.559 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:15:57.560 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:15:57.661 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:15:57.689 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:15:57.691 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:15:57.691 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:15:57.792 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:15:57.826 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:15:57.828 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:15:57.829 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:15:57.930 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:15:57.967 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:15:57.969 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:15:57.970 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:15:58.071 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:15:58.098 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:15:58.100 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:15:58.101 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:15:58.201 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:15:58.209 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:15:57 smithi089 bash[78452]: cluster 1685823356.8020146 mgr.y (mgr.34104) 1198 : cluster 0 pgmap v1176: 89 pgs: 9 active+clean+remapped, 2 active+recovering+undersized+degraded+remapped, 7 active+undersized+degraded, 26 active+undersized, 45 active+clean; 582 KiB data, 4.5 GiB used, 621 GiB / 626 GiB avail; 3557/72 objects degraded (4940.278%); 2/72 objects misplaced (2.778%); 0 B/s, 103 objects/s recovering 2023-06-03T20:15:58.210 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:15:57 smithi089 bash[78452]: cluster 1685823357.787457 mon.a (mon.0) 1131 : cluster 3 Health check update: Degraded data redundancy: 3557/72 objects degraded (4940.278%), 9 pgs degraded, 35 pgs undersized (PG_DEGRADED) 2023-06-03T20:15:58.210 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:15:57 smithi089 bash[75068]: cluster 1685823356.8020146 mgr.y (mgr.34104) 1198 : cluster 0 pgmap v1176: 89 pgs: 9 active+clean+remapped, 2 active+recovering+undersized+degraded+remapped, 7 active+undersized+degraded, 26 active+undersized, 45 active+clean; 582 KiB data, 4.5 GiB used, 621 GiB / 626 GiB avail; 3557/72 objects degraded (4940.278%); 2/72 objects misplaced (2.778%); 0 B/s, 103 objects/s recovering 2023-06-03T20:15:58.210 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:15:57 smithi089 bash[75068]: cluster 1685823357.787457 mon.a (mon.0) 1131 : cluster 3 Health check update: Degraded data redundancy: 3557/72 objects degraded (4940.278%), 9 pgs degraded, 35 pgs undersized (PG_DEGRADED) 2023-06-03T20:15:58.227 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:15:58.229 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:15:58.229 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:15:58.331 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:15:58.334 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:15:57 smithi164 bash[59462]: cluster 1685823356.8020146 mgr.y (mgr.34104) 1198 : cluster 0 pgmap v1176: 89 pgs: 9 active+clean+remapped, 2 active+recovering+undersized+degraded+remapped, 7 active+undersized+degraded, 26 active+undersized, 45 active+clean; 582 KiB data, 4.5 GiB used, 621 GiB / 626 GiB avail; 3557/72 objects degraded (4940.278%); 2/72 objects misplaced (2.778%); 0 B/s, 103 objects/s recovering 2023-06-03T20:15:58.334 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:15:57 smithi164 bash[59462]: cluster 1685823357.787457 mon.a (mon.0) 1131 : cluster 3 Health check update: Degraded data redundancy: 3557/72 objects degraded (4940.278%), 9 pgs degraded, 35 pgs undersized (PG_DEGRADED) 2023-06-03T20:15:58.360 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:15:58.363 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:15:58.363 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:15:58.464 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:15:58.489 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:15:58.493 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:15:58.494 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:15:58.594 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:15:58.624 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:15:58.626 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:15:58.628 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:15:58.730 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:15:58.770 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:15:58.772 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:15:58.773 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:15:58.874 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:15:58.899 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:15:58.900 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:15:58.901 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:15:59.002 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:15:59.036 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:15:59.039 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:15:59.039 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:15:59.140 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:15:59.186 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:15:59.188 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:15:59.189 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:15:59.290 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:15:59.318 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:15:59.320 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:15:59.320 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:15:59.421 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:15:59.453 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:15:59.455 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:15:59.456 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:15:59.557 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:15:59.591 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:15:59.593 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:15:59.594 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:15:59.695 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:15:59.726 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:15:59.730 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:15:59.731 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:15:59.832 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:15:59.858 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:15:59.859 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:15:59.860 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:15:59.961 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:15:59.990 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:15:59.993 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:15:59.994 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:16:00.028 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:15:59 smithi089 bash[75068]: cluster 1685823358.802994 mgr.y (mgr.34104) 1199 : cluster 0 pgmap v1177: 89 pgs: 9 active+clean+remapped, 2 active+recovering+undersized+degraded+remapped, 7 active+undersized+degraded, 26 active+undersized, 45 active+clean; 582 KiB data, 4.5 GiB used, 621 GiB / 626 GiB avail; 2799/72 objects degraded (3887.500%); 2/72 objects misplaced (2.778%); 0 B/s, 166 objects/s recovering 2023-06-03T20:16:00.029 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:15:59 smithi089 bash[78452]: cluster 1685823358.802994 mgr.y (mgr.34104) 1199 : cluster 0 pgmap v1177: 89 pgs: 9 active+clean+remapped, 2 active+recovering+undersized+degraded+remapped, 7 active+undersized+degraded, 26 active+undersized, 45 active+clean; 582 KiB data, 4.5 GiB used, 621 GiB / 626 GiB avail; 2799/72 objects degraded (3887.500%); 2/72 objects misplaced (2.778%); 0 B/s, 166 objects/s recovering 2023-06-03T20:16:00.094 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:16:00.126 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:16:00.129 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:16:00.130 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:16:00.231 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:16:00.253 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:16:00.255 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:16:00.255 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:16:00.333 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:15:59 smithi164 bash[59462]: cluster 1685823358.802994 mgr.y (mgr.34104) 1199 : cluster 0 pgmap v1177: 89 pgs: 9 active+clean+remapped, 2 active+recovering+undersized+degraded+remapped, 7 active+undersized+degraded, 26 active+undersized, 45 active+clean; 582 KiB data, 4.5 GiB used, 621 GiB / 626 GiB avail; 2799/72 objects degraded (3887.500%); 2/72 objects misplaced (2.778%); 0 B/s, 166 objects/s recovering 2023-06-03T20:16:00.356 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:16:00.385 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:16:00.387 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:16:00.388 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:16:00.489 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:16:00.521 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:16:00.525 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:16:00.525 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:16:00.626 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:16:00.648 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:16:00.649 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:16:00.650 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:16:00.751 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:16:00.775 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:16:00.776 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:16:00.777 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:16:00.878 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:16:00.905 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:16:00.907 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:16:00.907 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:16:01.008 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:16:01.038 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:16:01.041 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:16:01.041 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:16:01.142 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:16:01.167 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:16:01.168 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:16:01.169 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:16:01.270 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:16:01.294 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:16:01.296 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:16:01.296 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:16:01.397 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:16:01.425 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:16:01.427 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:16:01.428 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:16:01.529 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:16:01.556 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:16:01.558 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:16:01.559 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:16:01.660 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:16:01.690 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:16:01.693 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:16:01.694 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:16:01.794 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:16:01.823 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:16:01.824 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:16:01.825 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:16:01.926 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:16:01.952 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:16:01.955 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:16:01.956 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:16:02.057 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:16:02.080 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:16:02.082 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:16:02.083 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:16:02.184 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:16:02.213 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:16:02.215 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:16:02.215 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:16:02.316 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:16:02.333 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:16:01 smithi164 bash[59462]: cluster 1685823360.8043644 mgr.y (mgr.34104) 1200 : cluster 0 pgmap v1178: 89 pgs: 9 active+clean+remapped, 2 active+recovering+undersized+degraded+remapped, 7 active+undersized+degraded, 26 active+undersized, 45 active+clean; 582 KiB data, 4.5 GiB used, 621 GiB / 626 GiB avail; 2799/72 objects degraded (3887.500%); 2/72 objects misplaced (2.778%); 0 B/s, 119 objects/s recovering 2023-06-03T20:16:02.355 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:16:02.357 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:16:02.358 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:16:02.459 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:16:02.493 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:16:02.495 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:16:02.496 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:16:02.508 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:16:02 smithi089 bash[78452]: cluster 1685823360.8043644 mgr.y (mgr.34104) 1200 : cluster 0 pgmap v1178: 89 pgs: 9 active+clean+remapped, 2 active+recovering+undersized+degraded+remapped, 7 active+undersized+degraded, 26 active+undersized, 45 active+clean; 582 KiB data, 4.5 GiB used, 621 GiB / 626 GiB avail; 2799/72 objects degraded (3887.500%); 2/72 objects misplaced (2.778%); 0 B/s, 119 objects/s recovering 2023-06-03T20:16:02.508 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:16:02 smithi089 bash[75068]: cluster 1685823360.8043644 mgr.y (mgr.34104) 1200 : cluster 0 pgmap v1178: 89 pgs: 9 active+clean+remapped, 2 active+recovering+undersized+degraded+remapped, 7 active+undersized+degraded, 26 active+undersized, 45 active+clean; 582 KiB data, 4.5 GiB used, 621 GiB / 626 GiB avail; 2799/72 objects degraded (3887.500%); 2/72 objects misplaced (2.778%); 0 B/s, 119 objects/s recovering 2023-06-03T20:16:02.597 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:16:02.627 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:16:02.630 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:16:02.631 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:16:02.732 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:16:02.762 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:16:02.766 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:16:02.766 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:16:02.867 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:16:02.900 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:16:02.902 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:16:02.903 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:16:03.004 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:16:03.038 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:16:03.040 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:16:03.041 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:16:03.142 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:16:03.152 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:16:02 smithi164 bash[59462]: cluster 1685823362.7883997 mon.a (mon.0) 1132 : cluster 3 Health check update: Degraded data redundancy: 2799/72 objects degraded (3887.500%), 9 pgs degraded, 35 pgs undersized (PG_DEGRADED) 2023-06-03T20:16:03.175 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:16:03.177 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:16:03.178 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:16:03.278 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:16:03.279 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:16:02 smithi089 bash[78452]: cluster 1685823362.7883997 mon.a (mon.0) 1132 : cluster 3 Health check update: Degraded data redundancy: 2799/72 objects degraded (3887.500%), 9 pgs degraded, 35 pgs undersized (PG_DEGRADED) 2023-06-03T20:16:03.280 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:16:02 smithi089 bash[75068]: cluster 1685823362.7883997 mon.a (mon.0) 1132 : cluster 3 Health check update: Degraded data redundancy: 2799/72 objects degraded (3887.500%), 9 pgs degraded, 35 pgs undersized (PG_DEGRADED) 2023-06-03T20:16:03.310 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:16:03.312 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:16:03.312 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:16:03.413 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:16:03.448 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:16:03.451 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:16:03.451 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:16:03.552 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:16:03.581 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:16:03.583 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:16:03.584 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:16:03.685 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:16:03.710 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:16:03.712 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:16:03.713 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:16:03.814 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:16:03.844 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:16:03.846 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:16:03.847 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:16:03.948 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:16:03.980 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:16:03.982 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:16:03.983 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:16:04.083 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:16:03 smithi164 bash[59462]: cluster 1685823362.8053389 mgr.y (mgr.34104) 1201 : cluster 0 pgmap v1179: 89 pgs: 9 active+clean+remapped, 2 active+recovering+undersized+degraded+remapped, 7 active+undersized+degraded, 26 active+undersized, 45 active+clean; 582 KiB data, 4.5 GiB used, 621 GiB / 626 GiB avail; 2799/72 objects degraded (3887.500%); 2/72 objects misplaced (2.778%); 0 B/s, 119 objects/s recovering 2023-06-03T20:16:04.084 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:16:04.094 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:16:03 smithi089 bash[75068]: cluster 1685823362.8053389 mgr.y (mgr.34104) 1201 : cluster 0 pgmap v1179: 89 pgs: 9 active+clean+remapped, 2 active+recovering+undersized+degraded+remapped, 7 active+undersized+degraded, 26 active+undersized, 45 active+clean; 582 KiB data, 4.5 GiB used, 621 GiB / 626 GiB avail; 2799/72 objects degraded (3887.500%); 2/72 objects misplaced (2.778%); 0 B/s, 119 objects/s recovering 2023-06-03T20:16:04.094 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:16:03 smithi089 bash[78452]: cluster 1685823362.8053389 mgr.y (mgr.34104) 1201 : cluster 0 pgmap v1179: 89 pgs: 9 active+clean+remapped, 2 active+recovering+undersized+degraded+remapped, 7 active+undersized+degraded, 26 active+undersized, 45 active+clean; 582 KiB data, 4.5 GiB used, 621 GiB / 626 GiB avail; 2799/72 objects degraded (3887.500%); 2/72 objects misplaced (2.778%); 0 B/s, 119 objects/s recovering 2023-06-03T20:16:04.113 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:16:04.115 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:16:04.116 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:16:04.217 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:16:04.247 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:16:04.249 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:16:04.250 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:16:04.351 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:16:04.379 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:16:04.383 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:16:04.383 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:16:04.484 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:16:04.514 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:16:04.516 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:16:04.517 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:16:04.617 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:16:04.655 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:16:04.657 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:16:04.657 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:16:04.758 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:16:04.785 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:16:04.787 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:16:04.788 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:16:04.889 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:16:04.922 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:16:04.924 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:16:04.924 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:16:05.025 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:16:05.060 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:16:05.062 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:16:05.063 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:16:05.164 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:16:05.195 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:16:05.198 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:16:05.199 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:16:05.299 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:16:05.334 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:16:05.337 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:16:05.338 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:16:05.439 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:16:05.473 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:16:05.475 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:16:05.476 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:16:05.577 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:16:05.606 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:16:05.609 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:16:05.609 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:16:05.710 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:16:05.738 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:16:05.740 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:16:05.741 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:16:05.842 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:16:05.868 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:16:05.871 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:16:05.872 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:16:05.973 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:16:05.986 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:16:05 smithi089 bash[75068]: cluster 1685823364.806313 mgr.y (mgr.34104) 1202 : cluster 0 pgmap v1180: 89 pgs: 9 active+clean+remapped, 2 active+recovering+undersized+degraded+remapped, 7 active+undersized+degraded, 26 active+undersized, 45 active+clean; 582 KiB data, 4.5 GiB used, 621 GiB / 626 GiB avail; 1926/72 objects degraded (2675.000%); 2/72 objects misplaced (2.778%); 0 B/s, 192 objects/s recovering 2023-06-03T20:16:05.986 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:16:05 smithi089 bash[78452]: cluster 1685823364.806313 mgr.y (mgr.34104) 1202 : cluster 0 pgmap v1180: 89 pgs: 9 active+clean+remapped, 2 active+recovering+undersized+degraded+remapped, 7 active+undersized+degraded, 26 active+undersized, 45 active+clean; 582 KiB data, 4.5 GiB used, 621 GiB / 626 GiB avail; 1926/72 objects degraded (2675.000%); 2/72 objects misplaced (2.778%); 0 B/s, 192 objects/s recovering 2023-06-03T20:16:06.006 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:16:06.009 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:16:06.009 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:16:06.110 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:16:06.146 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:16:06.149 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:16:06.150 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:16:06.250 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:16:06.279 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:16:06.281 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:16:06.282 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:16:06.297 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 20:16:06 smithi089 bash[71163]: ::ffff:172.21.15.164 - - [03/Jun/2023:20:16:06] "GET /metrics HTTP/1.1" 200 228858 "" "Prometheus/2.33.4" 2023-06-03T20:16:06.333 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:16:05 smithi164 bash[59462]: cluster 1685823364.806313 mgr.y (mgr.34104) 1202 : cluster 0 pgmap v1180: 89 pgs: 9 active+clean+remapped, 2 active+recovering+undersized+degraded+remapped, 7 active+undersized+degraded, 26 active+undersized, 45 active+clean; 582 KiB data, 4.5 GiB used, 621 GiB / 626 GiB avail; 1926/72 objects degraded (2675.000%); 2/72 objects misplaced (2.778%); 0 B/s, 192 objects/s recovering 2023-06-03T20:16:06.383 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:16:06.407 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:16:06.409 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:16:06.410 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:16:06.511 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:16:06.542 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:16:06.544 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:16:06.544 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:16:06.645 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:16:06.679 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:16:06.682 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:16:06.682 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:16:06.783 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:16:06.811 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:16:06.813 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:16:06.814 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:16:06.915 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:16:06.945 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:16:06.947 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:16:06.947 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:16:07.048 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:16:07.077 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:16:07.080 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:16:07.080 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:16:07.181 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:16:07.207 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:16:07.209 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:16:07.210 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:16:07.311 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:16:07.342 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:16:07.344 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:16:07.345 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:16:07.446 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:16:07.478 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:16:07.481 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:16:07.481 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:16:07.582 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:16:07.612 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:16:07.614 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:16:07.615 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:16:07.715 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:16:07.741 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:16:07.744 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:16:07.745 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:16:07.846 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:16:07.871 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:16:07.875 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:16:07.875 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:16:07.976 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:16:08.005 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:16:08.008 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:16:08.008 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:16:08.028 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:16:07 smithi089 bash[75068]: cluster 1685823366.807201 mgr.y (mgr.34104) 1203 : cluster 0 pgmap v1181: 89 pgs: 9 active+clean+remapped, 2 active+recovering+undersized+degraded+remapped, 7 active+undersized+degraded, 26 active+undersized, 45 active+clean; 582 KiB data, 4.5 GiB used, 621 GiB / 626 GiB avail; 1926/72 objects degraded (2675.000%); 2/72 objects misplaced (2.778%); 0 B/s, 135 objects/s recovering 2023-06-03T20:16:08.029 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:16:07 smithi089 bash[75068]: audit 1685823367.1267545 mon.c (mon.2) 536 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T20:16:08.029 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:16:07 smithi089 bash[75068]: cluster 1685823367.7894118 mon.a (mon.0) 1133 : cluster 3 Health check update: Degraded data redundancy: 1926/72 objects degraded (2675.000%), 9 pgs degraded, 35 pgs undersized (PG_DEGRADED) 2023-06-03T20:16:08.030 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:16:07 smithi089 bash[78452]: cluster 1685823366.807201 mgr.y (mgr.34104) 1203 : cluster 0 pgmap v1181: 89 pgs: 9 active+clean+remapped, 2 active+recovering+undersized+degraded+remapped, 7 active+undersized+degraded, 26 active+undersized, 45 active+clean; 582 KiB data, 4.5 GiB used, 621 GiB / 626 GiB avail; 1926/72 objects degraded (2675.000%); 2/72 objects misplaced (2.778%); 0 B/s, 135 objects/s recovering 2023-06-03T20:16:08.030 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:16:07 smithi089 bash[78452]: audit 1685823367.1267545 mon.c (mon.2) 536 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T20:16:08.030 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:16:07 smithi089 bash[78452]: cluster 1685823367.7894118 mon.a (mon.0) 1133 : cluster 3 Health check update: Degraded data redundancy: 1926/72 objects degraded (2675.000%), 9 pgs degraded, 35 pgs undersized (PG_DEGRADED) 2023-06-03T20:16:08.109 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:16:08.140 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:16:08.142 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:16:08.142 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:16:08.243 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:16:08.273 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:16:08.275 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:16:08.276 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:16:08.333 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:16:07 smithi164 bash[59462]: cluster 1685823366.807201 mgr.y (mgr.34104) 1203 : cluster 0 pgmap v1181: 89 pgs: 9 active+clean+remapped, 2 active+recovering+undersized+degraded+remapped, 7 active+undersized+degraded, 26 active+undersized, 45 active+clean; 582 KiB data, 4.5 GiB used, 621 GiB / 626 GiB avail; 1926/72 objects degraded (2675.000%); 2/72 objects misplaced (2.778%); 0 B/s, 135 objects/s recovering 2023-06-03T20:16:08.333 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:16:07 smithi164 bash[59462]: audit 1685823367.1267545 mon.c (mon.2) 536 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T20:16:08.334 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:16:07 smithi164 bash[59462]: cluster 1685823367.7894118 mon.a (mon.0) 1133 : cluster 3 Health check update: Degraded data redundancy: 1926/72 objects degraded (2675.000%), 9 pgs degraded, 35 pgs undersized (PG_DEGRADED) 2023-06-03T20:16:08.377 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:16:08.408 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:16:08.411 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:16:08.412 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:16:08.513 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:16:08.547 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:16:08.549 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:16:08.550 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:16:08.651 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:16:08.681 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:16:08.685 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:16:08.685 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:16:08.786 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:16:08.817 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:16:08.819 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:16:08.819 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:16:08.920 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:16:08.954 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:16:08.956 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:16:08.957 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:16:09.058 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:16:09.094 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:16:09.097 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:16:09.097 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:16:09.198 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:16:09.232 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:16:09.236 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:16:09.236 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:16:09.337 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:16:09.371 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:16:09.373 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:16:09.374 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:16:09.475 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:16:09.509 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:16:09.513 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:16:09.513 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:16:09.614 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:16:09.642 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:16:09.644 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:16:09.645 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:16:09.746 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:16:09.772 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:16:09.775 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:16:09.775 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:16:09.876 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:16:09.910 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:16:09.912 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:16:09.913 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:16:10.014 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:16:10.045 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:16:10.047 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:16:10.048 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:16:10.149 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:16:10.186 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:16:10.189 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:16:10.190 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:16:10.287 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:16:09 smithi089 bash[75068]: cluster 1685823368.8086922 mgr.y (mgr.34104) 1204 : cluster 0 pgmap v1182: 89 pgs: 9 active+clean+remapped, 2 active+recovering+undersized+degraded+remapped, 7 active+undersized+degraded, 26 active+undersized, 45 active+clean; 582 KiB data, 4.5 GiB used, 621 GiB / 626 GiB avail; 851/72 objects degraded (1181.944%); 2/72 objects misplaced (2.778%); 0 B/s, 225 objects/s recovering 2023-06-03T20:16:10.288 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:16:09 smithi089 bash[78452]: cluster 1685823368.8086922 mgr.y (mgr.34104) 1204 : cluster 0 pgmap v1182: 89 pgs: 9 active+clean+remapped, 2 active+recovering+undersized+degraded+remapped, 7 active+undersized+degraded, 26 active+undersized, 45 active+clean; 582 KiB data, 4.5 GiB used, 621 GiB / 626 GiB avail; 851/72 objects degraded (1181.944%); 2/72 objects misplaced (2.778%); 0 B/s, 225 objects/s recovering 2023-06-03T20:16:10.290 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:16:10.318 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:16:10.320 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:16:10.321 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:16:10.333 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:16:09 smithi164 bash[59462]: cluster 1685823368.8086922 mgr.y (mgr.34104) 1204 : cluster 0 pgmap v1182: 89 pgs: 9 active+clean+remapped, 2 active+recovering+undersized+degraded+remapped, 7 active+undersized+degraded, 26 active+undersized, 45 active+clean; 582 KiB data, 4.5 GiB used, 621 GiB / 626 GiB avail; 851/72 objects degraded (1181.944%); 2/72 objects misplaced (2.778%); 0 B/s, 225 objects/s recovering 2023-06-03T20:16:10.422 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:16:10.455 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:16:10.456 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:16:10.457 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:16:10.558 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:16:10.592 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:16:10.594 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:16:10.595 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:16:10.696 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:16:10.724 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:16:10.725 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:16:10.726 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:16:10.827 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:16:10.861 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:16:10.863 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:16:10.864 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:16:10.964 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:16:10.992 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:16:10.994 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:16:10.995 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:16:11.096 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:16:11.124 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:16:11.126 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:16:11.126 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:16:11.227 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:16:11.256 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:16:11.258 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:16:11.259 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:16:11.360 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:16:11.397 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:16:11.399 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:16:11.399 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:16:11.500 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:16:11.526 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:16:11.528 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:16:11.529 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:16:11.630 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:16:11.666 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:16:11.669 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:16:11.670 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:16:11.770 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:16:11.802 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:16:11.804 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:16:11.804 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:16:11.905 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:16:11.935 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:16:11.936 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:16:11.937 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:16:12.038 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:16:12.049 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:16:12 smithi089 bash[78452]: cluster 1685823370.810212 mgr.y (mgr.34104) 1205 : cluster 0 pgmap v1183: 89 pgs: 9 active+clean+remapped, 2 active+recovering+undersized+degraded+remapped, 7 active+undersized+degraded, 26 active+undersized, 45 active+clean; 582 KiB data, 4.5 GiB used, 621 GiB / 626 GiB avail; 851/72 objects degraded (1181.944%); 2/72 objects misplaced (2.778%); 0 B/s, 162 objects/s recovering 2023-06-03T20:16:12.049 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:16:12 smithi089 bash[78452]: cluster 1685823370.9691017 osd.5 (osd.5) 141 : cluster 3 failed to encode map e209 with expected crc 2023-06-03T20:16:12.049 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:16:12 smithi089 bash[78452]: cluster 1685823370.9737427 mon.a (mon.0) 1134 : cluster 0 osdmap e209: 8 total, 7 up, 7 in 2023-06-03T20:16:12.049 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:16:12 smithi089 bash[78452]: cluster 1685823370.980753 osd.6 (osd.6) 153 : cluster 3 failed to encode map e209 with expected crc 2023-06-03T20:16:12.049 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:16:12 smithi089 bash[78452]: cluster 1685823370.9807713 osd.7 (osd.7) 135 : cluster 3 failed to encode map e209 with expected crc 2023-06-03T20:16:12.050 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:16:12 smithi089 bash[75068]: cluster 1685823370.810212 mgr.y (mgr.34104) 1205 : cluster 0 pgmap v1183: 89 pgs: 9 active+clean+remapped, 2 active+recovering+undersized+degraded+remapped, 7 active+undersized+degraded, 26 active+undersized, 45 active+clean; 582 KiB data, 4.5 GiB used, 621 GiB / 626 GiB avail; 851/72 objects degraded (1181.944%); 2/72 objects misplaced (2.778%); 0 B/s, 162 objects/s recovering 2023-06-03T20:16:12.050 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:16:12 smithi089 bash[75068]: cluster 1685823370.9691017 osd.5 (osd.5) 141 : cluster 3 failed to encode map e209 with expected crc 2023-06-03T20:16:12.050 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:16:12 smithi089 bash[75068]: cluster 1685823370.9737427 mon.a (mon.0) 1134 : cluster 0 osdmap e209: 8 total, 7 up, 7 in 2023-06-03T20:16:12.050 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:16:12 smithi089 bash[75068]: cluster 1685823370.980753 osd.6 (osd.6) 153 : cluster 3 failed to encode map e209 with expected crc 2023-06-03T20:16:12.050 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:16:12 smithi089 bash[75068]: cluster 1685823370.9807713 osd.7 (osd.7) 135 : cluster 3 failed to encode map e209 with expected crc 2023-06-03T20:16:12.071 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:16:12.073 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:16:12.074 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:16:12.175 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:16:12.208 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:16:12.211 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:16:12.211 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:16:12.312 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:16:12.333 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:16:11 smithi164 bash[59462]: cluster 1685823370.810212 mgr.y (mgr.34104) 1205 : cluster 0 pgmap v1183: 89 pgs: 9 active+clean+remapped, 2 active+recovering+undersized+degraded+remapped, 7 active+undersized+degraded, 26 active+undersized, 45 active+clean; 582 KiB data, 4.5 GiB used, 621 GiB / 626 GiB avail; 851/72 objects degraded (1181.944%); 2/72 objects misplaced (2.778%); 0 B/s, 162 objects/s recovering 2023-06-03T20:16:12.334 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:16:11 smithi164 bash[59462]: cluster 1685823370.9691017 osd.5 (osd.5) 141 : cluster 3 failed to encode map e209 with expected crc 2023-06-03T20:16:12.334 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:16:11 smithi164 bash[59462]: cluster 1685823370.9737427 mon.a (mon.0) 1134 : cluster 0 osdmap e209: 8 total, 7 up, 7 in 2023-06-03T20:16:12.334 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:16:11 smithi164 bash[59462]: cluster 1685823370.980753 osd.6 (osd.6) 153 : cluster 3 failed to encode map e209 with expected crc 2023-06-03T20:16:12.334 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:16:11 smithi164 bash[59462]: cluster 1685823370.9807713 osd.7 (osd.7) 135 : cluster 3 failed to encode map e209 with expected crc 2023-06-03T20:16:12.340 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:16:12.343 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:16:12.344 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:16:12.445 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:16:12.471 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:16:12.475 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:16:12.475 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:16:12.576 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:16:12.602 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:16:12.605 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:16:12.606 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:16:12.707 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:16:12.746 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:16:12.748 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:16:12.749 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:16:12.850 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:16:12.884 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:16:12.886 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:16:12.887 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:16:12.988 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:16:13.017 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:16:13.018 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:16:13.019 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:16:13.120 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:16:13.154 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:16:13.156 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:16:13.157 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:16:13.258 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:16:13.266 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:16:12 smithi089 bash[75068]: cluster 1685823371.9703896 osd.6 (osd.6) 154 : cluster 3 failed to encode map e210 with expected crc 2023-06-03T20:16:13.266 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:16:12 smithi089 bash[75068]: cluster 1685823371.9708226 osd.5 (osd.5) 142 : cluster 3 failed to encode map e210 with expected crc 2023-06-03T20:16:13.267 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:16:12 smithi089 bash[75068]: cluster 1685823371.9741216 mon.a (mon.0) 1135 : cluster 0 osdmap e210: 8 total, 7 up, 7 in 2023-06-03T20:16:13.267 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:16:12 smithi089 bash[75068]: cluster 1685823371.9780223 osd.7 (osd.7) 136 : cluster 3 failed to encode map e210 with expected crc 2023-06-03T20:16:13.267 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:16:12 smithi089 bash[75068]: cluster 1685823372.7912881 mon.a (mon.0) 1136 : cluster 3 Health check update: Degraded data redundancy: 851/72 objects degraded (1181.944%), 9 pgs degraded, 35 pgs undersized (PG_DEGRADED) 2023-06-03T20:16:13.267 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:16:12 smithi089 bash[78452]: cluster 1685823371.9703896 osd.6 (osd.6) 154 : cluster 3 failed to encode map e210 with expected crc 2023-06-03T20:16:13.267 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:16:12 smithi089 bash[78452]: cluster 1685823371.9708226 osd.5 (osd.5) 142 : cluster 3 failed to encode map e210 with expected crc 2023-06-03T20:16:13.267 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:16:12 smithi089 bash[78452]: cluster 1685823371.9741216 mon.a (mon.0) 1135 : cluster 0 osdmap e210: 8 total, 7 up, 7 in 2023-06-03T20:16:13.268 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:16:12 smithi089 bash[78452]: cluster 1685823371.9780223 osd.7 (osd.7) 136 : cluster 3 failed to encode map e210 with expected crc 2023-06-03T20:16:13.268 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:16:12 smithi089 bash[78452]: cluster 1685823372.7912881 mon.a (mon.0) 1136 : cluster 3 Health check update: Degraded data redundancy: 851/72 objects degraded (1181.944%), 9 pgs degraded, 35 pgs undersized (PG_DEGRADED) 2023-06-03T20:16:13.284 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:16:13.286 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:16:13.286 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:16:13.333 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:16:12 smithi164 bash[59462]: cluster 1685823371.9703896 osd.6 (osd.6) 154 : cluster 3 failed to encode map e210 with expected crc 2023-06-03T20:16:13.334 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:16:12 smithi164 bash[59462]: cluster 1685823371.9708226 osd.5 (osd.5) 142 : cluster 3 failed to encode map e210 with expected crc 2023-06-03T20:16:13.334 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:16:12 smithi164 bash[59462]: cluster 1685823371.9741216 mon.a (mon.0) 1135 : cluster 0 osdmap e210: 8 total, 7 up, 7 in 2023-06-03T20:16:13.334 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:16:12 smithi164 bash[59462]: cluster 1685823371.9780223 osd.7 (osd.7) 136 : cluster 3 failed to encode map e210 with expected crc 2023-06-03T20:16:13.334 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:16:12 smithi164 bash[59462]: cluster 1685823372.7912881 mon.a (mon.0) 1136 : cluster 3 Health check update: Degraded data redundancy: 851/72 objects degraded (1181.944%), 9 pgs degraded, 35 pgs undersized (PG_DEGRADED) 2023-06-03T20:16:13.387 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:16:13.420 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:16:13.423 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:16:13.424 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:16:13.524 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:16:13.556 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:16:13.558 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:16:13.558 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:16:13.659 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:16:13.695 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:16:13.697 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:16:13.697 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:16:13.798 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:16:13.822 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:16:13.825 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:16:13.825 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:16:13.926 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:16:13.952 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:16:13.954 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:16:13.955 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:16:14.056 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:16:14.083 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:16:14.086 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:16:14.087 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:16:14.188 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:16:14.210 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:16:14.213 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:16:14.213 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:16:14.292 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:16:13 smithi089 bash[75068]: cluster 1685823372.811177 mgr.y (mgr.34104) 1206 : cluster 0 pgmap v1186: 89 pgs: 9 active+clean+remapped, 2 active+recovering+undersized+degraded+remapped, 7 active+undersized+degraded, 26 active+undersized, 45 active+clean; 582 KiB data, 4.5 GiB used, 621 GiB / 626 GiB avail; 851/72 objects degraded (1181.944%); 2/72 objects misplaced (2.778%); 0 B/s, 134 objects/s recovering 2023-06-03T20:16:14.292 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:16:13 smithi089 bash[75068]: audit 1685823373.93408 mon.c (mon.2) 537 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-03T20:16:14.293 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:16:13 smithi089 bash[78452]: cluster 1685823372.811177 mgr.y (mgr.34104) 1206 : cluster 0 pgmap v1186: 89 pgs: 9 active+clean+remapped, 2 active+recovering+undersized+degraded+remapped, 7 active+undersized+degraded, 26 active+undersized, 45 active+clean; 582 KiB data, 4.5 GiB used, 621 GiB / 626 GiB avail; 851/72 objects degraded (1181.944%); 2/72 objects misplaced (2.778%); 0 B/s, 134 objects/s recovering 2023-06-03T20:16:14.293 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:16:13 smithi089 bash[78452]: audit 1685823373.93408 mon.c (mon.2) 537 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-03T20:16:14.314 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:16:14.324 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:16:13 smithi164 bash[59462]: cluster 1685823372.811177 mgr.y (mgr.34104) 1206 : cluster 0 pgmap v1186: 89 pgs: 9 active+clean+remapped, 2 active+recovering+undersized+degraded+remapped, 7 active+undersized+degraded, 26 active+undersized, 45 active+clean; 582 KiB data, 4.5 GiB used, 621 GiB / 626 GiB avail; 851/72 objects degraded (1181.944%); 2/72 objects misplaced (2.778%); 0 B/s, 134 objects/s recovering 2023-06-03T20:16:14.324 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:16:13 smithi164 bash[59462]: audit 1685823373.93408 mon.c (mon.2) 537 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-03T20:16:14.343 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:16:14.346 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:16:14.347 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:16:14.448 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:16:14.484 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:16:14.487 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:16:14.487 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:16:14.588 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:16:14.614 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:16:14.616 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:16:14.617 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:16:14.717 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:16:14.754 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:16:14.756 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:16:14.756 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:16:14.857 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:16:14.894 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:16:14.949 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:16:14.950 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:16:15.050 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:16:15.076 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:16:15.077 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:16:15.078 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:16:15.179 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:16:15.214 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:16:15.217 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:16:15.217 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:16:15.318 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:16:15.350 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:16:15.351 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:16:15.352 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:16:15.453 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:16:15.466 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:16:15 smithi089 bash[75068]: audit 1685823374.306298 mon.a (mon.0) 1137 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:16:15.467 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:16:15 smithi089 bash[75068]: audit 1685823374.3161328 mon.a (mon.0) 1138 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:16:15.467 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:16:15 smithi089 bash[75068]: audit 1685823374.3565733 mon.a (mon.0) 1139 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:16:15.467 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:16:15 smithi089 bash[75068]: audit 1685823374.3655276 mon.a (mon.0) 1140 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:16:15.467 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:16:15 smithi089 bash[75068]: cluster 1685823374.8119311 mgr.y (mgr.34104) 1207 : cluster 0 pgmap v1187: 89 pgs: 9 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 7 active+undersized+degraded, 26 active+undersized, 46 active+clean; 582 KiB data, 4.5 GiB used, 621 GiB / 626 GiB avail; 223/72 objects degraded (309.722%); 2/72 objects misplaced (2.778%); 0 B/s, 212 objects/s recovering 2023-06-03T20:16:15.467 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:16:15 smithi089 bash[78452]: audit 1685823374.306298 mon.a (mon.0) 1137 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:16:15.468 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:16:15 smithi089 bash[78452]: audit 1685823374.3161328 mon.a (mon.0) 1138 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:16:15.468 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:16:15 smithi089 bash[78452]: audit 1685823374.3565733 mon.a (mon.0) 1139 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:16:15.468 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:16:15 smithi089 bash[78452]: audit 1685823374.3655276 mon.a (mon.0) 1140 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:16:15.468 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:16:15 smithi089 bash[78452]: cluster 1685823374.8119311 mgr.y (mgr.34104) 1207 : cluster 0 pgmap v1187: 89 pgs: 9 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 7 active+undersized+degraded, 26 active+undersized, 46 active+clean; 582 KiB data, 4.5 GiB used, 621 GiB / 626 GiB avail; 223/72 objects degraded (309.722%); 2/72 objects misplaced (2.778%); 0 B/s, 212 objects/s recovering 2023-06-03T20:16:15.484 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:16:15.486 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:16:15.486 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:16:15.584 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:16:15 smithi164 bash[59462]: audit 1685823374.306298 mon.a (mon.0) 1137 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:16:15.584 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:16:15 smithi164 bash[59462]: audit 1685823374.3161328 mon.a (mon.0) 1138 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:16:15.585 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:16:15 smithi164 bash[59462]: audit 1685823374.3565733 mon.a (mon.0) 1139 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:16:15.585 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:16:15 smithi164 bash[59462]: audit 1685823374.3655276 mon.a (mon.0) 1140 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:16:15.585 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:16:15 smithi164 bash[59462]: cluster 1685823374.8119311 mgr.y (mgr.34104) 1207 : cluster 0 pgmap v1187: 89 pgs: 9 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 7 active+undersized+degraded, 26 active+undersized, 46 active+clean; 582 KiB data, 4.5 GiB used, 621 GiB / 626 GiB avail; 223/72 objects degraded (309.722%); 2/72 objects misplaced (2.778%); 0 B/s, 212 objects/s recovering 2023-06-03T20:16:15.588 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:16:15.617 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:16:15.619 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:16:15.619 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:16:15.720 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:16:15.756 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:16:15.758 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:16:15.759 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:16:15.860 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:16:15.893 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:16:15.896 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:16:15.897 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:16:15.998 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:16:16.026 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:16:16.028 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:16:16.029 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:16:16.130 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:16:16.165 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:16:16.168 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:16:16.169 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:16:16.270 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:16:16.278 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 20:16:16 smithi089 bash[71163]: ::ffff:172.21.15.164 - - [03/Jun/2023:20:16:16] "GET /metrics HTTP/1.1" 200 228838 "" "Prometheus/2.33.4" 2023-06-03T20:16:16.300 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:16:16.302 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:16:16.302 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:16:16.403 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:16:16.434 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:16:16.436 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:16:16.437 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:16:16.538 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:16:16.570 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:16:16.572 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:16:16.573 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:16:16.674 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:16:16.703 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:16:16.706 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:16:16.706 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:16:16.807 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:16:16.833 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:16:16.836 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:16:16.837 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:16:16.937 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:16:16.970 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:16:16.973 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:16:16.973 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:16:17.074 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:16:17.105 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:16:17.107 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:16:17.107 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:16:17.208 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:16:17.236 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:16:17.238 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:16:17.239 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:16:17.340 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:16:17.366 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:16:17.369 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:16:17.369 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:16:17.470 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:16:17.506 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:16:17.508 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:16:17.509 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:16:17.610 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:16:17.638 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:16:17.641 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:16:17.641 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:16:17.742 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:16:17.770 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:16:17.772 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:16:17.773 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:16:17.874 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:16:17.906 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:16:17.908 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:16:17.909 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:16:18.010 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:16:18.028 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:16:17 smithi089 bash[75068]: cluster 1685823376.8134599 mgr.y (mgr.34104) 1208 : cluster 0 pgmap v1188: 89 pgs: 9 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 7 active+undersized+degraded, 26 active+undersized, 46 active+clean; 582 KiB data, 4.5 GiB used, 621 GiB / 626 GiB avail; 223/72 objects degraded (309.722%); 2/72 objects misplaced (2.778%); 0 B/s, 78 objects/s recovering 2023-06-03T20:16:18.029 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:16:17 smithi089 bash[75068]: cluster 1685823377.7928612 mon.a (mon.0) 1141 : cluster 3 Health check update: Degraded data redundancy: 223/72 objects degraded (309.722%), 8 pgs degraded, 34 pgs undersized (PG_DEGRADED) 2023-06-03T20:16:18.029 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:16:17 smithi089 bash[78452]: cluster 1685823376.8134599 mgr.y (mgr.34104) 1208 : cluster 0 pgmap v1188: 89 pgs: 9 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 7 active+undersized+degraded, 26 active+undersized, 46 active+clean; 582 KiB data, 4.5 GiB used, 621 GiB / 626 GiB avail; 223/72 objects degraded (309.722%); 2/72 objects misplaced (2.778%); 0 B/s, 78 objects/s recovering 2023-06-03T20:16:18.029 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:16:17 smithi089 bash[78452]: cluster 1685823377.7928612 mon.a (mon.0) 1141 : cluster 3 Health check update: Degraded data redundancy: 223/72 objects degraded (309.722%), 8 pgs degraded, 34 pgs undersized (PG_DEGRADED) 2023-06-03T20:16:18.046 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:16:18.049 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:16:18.049 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:16:18.150 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:16:18.180 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:16:18.182 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:16:18.183 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:16:18.284 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:16:18.295 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:16:17 smithi164 bash[59462]: cluster 1685823376.8134599 mgr.y (mgr.34104) 1208 : cluster 0 pgmap v1188: 89 pgs: 9 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 7 active+undersized+degraded, 26 active+undersized, 46 active+clean; 582 KiB data, 4.5 GiB used, 621 GiB / 626 GiB avail; 223/72 objects degraded (309.722%); 2/72 objects misplaced (2.778%); 0 B/s, 78 objects/s recovering 2023-06-03T20:16:18.296 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:16:17 smithi164 bash[59462]: cluster 1685823377.7928612 mon.a (mon.0) 1141 : cluster 3 Health check update: Degraded data redundancy: 223/72 objects degraded (309.722%), 8 pgs degraded, 34 pgs undersized (PG_DEGRADED) 2023-06-03T20:16:18.319 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:16:18.322 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:16:18.323 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:16:18.424 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:16:18.457 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:16:18.460 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:16:18.460 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:16:18.561 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:16:18.598 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:16:18.601 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:16:18.601 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:16:18.702 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:16:18.735 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:16:18.737 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:16:18.738 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:16:18.839 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:16:18.860 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:16:18.862 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:16:18.863 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:16:18.900 INFO:teuthology.orchestra.run.smithi089.stdout:ERROR: (22) Invalid argument 2023-06-03T20:16:18.900 INFO:teuthology.orchestra.run.smithi089.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-06-03T20:16:18.905 INFO:teuthology.orchestra.run.smithi089.stderr:pool 'unique_pool_6' removed 2023-06-03T20:16:18.911 INFO:teuthology.orchestra.run.smithi089.stdout:osd.0: {} 2023-06-03T20:16:18.934 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:16:18 smithi089 bash[78452]: audit 1685823378.3021214 mon.a (mon.0) 1142 : audit 1 from='client.? 172.21.15.89:0/2661280422' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_6", "pool2": "unique_pool_6", "yes_i_really_really_mean_it": true}]: dispatch 2023-06-03T20:16:18.934 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:16:18 smithi089 bash[75068]: audit 1685823378.3021214 mon.a (mon.0) 1142 : audit 1 from='client.? 172.21.15.89:0/2661280422' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_6", "pool2": "unique_pool_6", "yes_i_really_really_mean_it": true}]: dispatch 2023-06-03T20:16:18.944 INFO:teuthology.orchestra.run.smithi089.stdout:osd.1: {} 2023-06-03T20:16:18.963 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:16:18.964 INFO:teuthology.orchestra.run.smithi089.stdout:osd.2: {} 2023-06-03T20:16:18.990 INFO:teuthology.orchestra.run.smithi089.stdout:osd.3: {} 2023-06-03T20:16:18.995 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:16:18.998 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:16:18.999 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:16:19.027 INFO:teuthology.orchestra.run.smithi089.stdout:osd.5: {} 2023-06-03T20:16:19.065 INFO:teuthology.orchestra.run.smithi089.stdout:osd.6: {} 2023-06-03T20:16:19.099 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:16:19.101 INFO:teuthology.orchestra.run.smithi089.stdout:osd.7: {} 2023-06-03T20:16:19.102 INFO:teuthology.orchestra.run.smithi089.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-06-03T20:16:19.102 INFO:teuthology.orchestra.run.smithi089.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-06-03T20:16:19.102 INFO:teuthology.orchestra.run.smithi089.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-06-03T20:16:19.102 INFO:teuthology.orchestra.run.smithi089.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-06-03T20:16:19.102 INFO:teuthology.orchestra.run.smithi089.stderr:Error ENXIO: problem getting command descriptions from osd.4 2023-06-03T20:16:19.102 INFO:teuthology.orchestra.run.smithi089.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-06-03T20:16:19.103 INFO:teuthology.orchestra.run.smithi089.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-06-03T20:16:19.103 INFO:teuthology.orchestra.run.smithi089.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-06-03T20:16:19.143 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:16:19.147 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:16:19.147 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:16:19.248 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:16:19.284 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:16:19.286 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:16:19.286 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:16:19.333 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:16:18 smithi164 bash[59462]: audit 1685823378.3021214 mon.a (mon.0) 1142 : audit 1 from='client.? 172.21.15.89:0/2661280422' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_6", "pool2": "unique_pool_6", "yes_i_really_really_mean_it": true}]: dispatch 2023-06-03T20:16:19.387 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:16:19.416 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:16:19.417 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:16:19.418 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:16:19.518 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:16:19.554 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:16:19.556 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:16:19.557 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:16:19.658 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:16:19.688 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:16:19.690 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:16:19.691 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:16:19.792 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:16:19.847 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:16:19.849 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:16:19.850 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:16:19.895 INFO:teuthology.orchestra.run.smithi089.stderr:marked out osd.6. 2023-06-03T20:16:19.896 INFO:teuthology.orchestra.run.smithi089.stderr:nodeep-scrub is set 2023-06-03T20:16:19.913 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:16:19 smithi089 bash[75068]: debug 2023-06-03T20:16:19.633+0000 7f0fee4d2700 -1 mon.a@0(leader).osd e211 definitely_dead 0 2023-06-03T20:16:19.913 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:16:19 smithi089 bash[75068]: cluster 1685823378.8144224 mgr.y (mgr.34104) 1209 : cluster 0 pgmap v1189: 89 pgs: 9 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 7 active+undersized+degraded, 26 active+undersized, 46 active+clean; 582 KiB data, 4.5 GiB used, 621 GiB / 626 GiB avail; 107/72 objects degraded (148.611%); 2/72 objects misplaced (2.778%); 0 B/s, 92 objects/s recovering 2023-06-03T20:16:19.914 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:16:19 smithi089 bash[75068]: audit 1685823378.8698847 mon.c (mon.2) 538 : audit 0 from='client.? 172.21.15.89:0/429857825' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-03T20:16:19.914 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:16:19 smithi089 bash[75068]: cluster 1685823378.8803215 osd.6 (osd.6) 155 : cluster 3 failed to encode map e211 with expected crc 2023-06-03T20:16:19.914 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:16:19 smithi089 bash[75068]: audit 1685823378.8845625 mon.a (mon.0) 1143 : audit 1 from='client.? 172.21.15.89:0/2661280422' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_6", "pool2": "unique_pool_6", "yes_i_really_really_mean_it": true}]': finished 2023-06-03T20:16:19.914 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:16:19 smithi089 bash[75068]: cluster 1685823378.8846078 mon.a (mon.0) 1144 : cluster 0 osdmap e211: 8 total, 7 up, 7 in 2023-06-03T20:16:19.914 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:16:19 smithi089 bash[75068]: audit 1685823379.4126217 mon.a (mon.0) 1145 : audit 1 from='client.? 172.21.15.89:0/2907305462' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-06-03T20:16:19.914 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:16:19 smithi089 bash[75068]: audit 1685823379.6383502 mon.a (mon.0) 1146 : audit 1 from='client.? 172.21.15.89:0/2115400778' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["6"]}]: dispatch 2023-06-03T20:16:19.914 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:16:19 smithi089 bash[75068]: cluster 1685823379.6385314 mon.a (mon.0) 1147 : cluster 1 Client client.admin marked osd.6 out, while it was still marked up 2023-06-03T20:16:19.915 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:16:19 smithi089 bash[78452]: cluster 1685823378.8144224 mgr.y (mgr.34104) 1209 : cluster 0 pgmap v1189: 89 pgs: 9 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 7 active+undersized+degraded, 26 active+undersized, 46 active+clean; 582 KiB data, 4.5 GiB used, 621 GiB / 626 GiB avail; 107/72 objects degraded (148.611%); 2/72 objects misplaced (2.778%); 0 B/s, 92 objects/s recovering 2023-06-03T20:16:19.915 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:16:19 smithi089 bash[78452]: audit 1685823378.8698847 mon.c (mon.2) 538 : audit 0 from='client.? 172.21.15.89:0/429857825' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-03T20:16:19.915 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:16:19 smithi089 bash[78452]: cluster 1685823378.8803215 osd.6 (osd.6) 155 : cluster 3 failed to encode map e211 with expected crc 2023-06-03T20:16:19.915 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:16:19 smithi089 bash[78452]: audit 1685823378.8845625 mon.a (mon.0) 1143 : audit 1 from='client.? 172.21.15.89:0/2661280422' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_6", "pool2": "unique_pool_6", "yes_i_really_really_mean_it": true}]': finished 2023-06-03T20:16:19.915 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:16:19 smithi089 bash[78452]: cluster 1685823378.8846078 mon.a (mon.0) 1144 : cluster 0 osdmap e211: 8 total, 7 up, 7 in 2023-06-03T20:16:19.915 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:16:19 smithi089 bash[78452]: audit 1685823379.4126217 mon.a (mon.0) 1145 : audit 1 from='client.? 172.21.15.89:0/2907305462' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-06-03T20:16:19.916 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:16:19 smithi089 bash[78452]: audit 1685823379.6383502 mon.a (mon.0) 1146 : audit 1 from='client.? 172.21.15.89:0/2115400778' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["6"]}]: dispatch 2023-06-03T20:16:19.916 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:16:19 smithi089 bash[78452]: cluster 1685823379.6385314 mon.a (mon.0) 1147 : cluster 1 Client client.admin marked osd.6 out, while it was still marked up 2023-06-03T20:16:19.950 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:16:19.978 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:16:19.979 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:16:19.980 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:16:20.080 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:16:20.084 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:16:19 smithi164 bash[59462]: cluster 1685823378.8144224 mgr.y (mgr.34104) 1209 : cluster 0 pgmap v1189: 89 pgs: 9 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 7 active+undersized+degraded, 26 active+undersized, 46 active+clean; 582 KiB data, 4.5 GiB used, 621 GiB / 626 GiB avail; 107/72 objects degraded (148.611%); 2/72 objects misplaced (2.778%); 0 B/s, 92 objects/s recovering 2023-06-03T20:16:20.085 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:16:19 smithi164 bash[59462]: audit 1685823378.8698847 mon.c (mon.2) 538 : audit 0 from='client.? 172.21.15.89:0/429857825' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-03T20:16:20.085 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:16:19 smithi164 bash[59462]: cluster 1685823378.8803215 osd.6 (osd.6) 155 : cluster 3 failed to encode map e211 with expected crc 2023-06-03T20:16:20.085 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:16:19 smithi164 bash[59462]: audit 1685823378.8845625 mon.a (mon.0) 1143 : audit 1 from='client.? 172.21.15.89:0/2661280422' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_6", "pool2": "unique_pool_6", "yes_i_really_really_mean_it": true}]': finished 2023-06-03T20:16:20.085 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:16:19 smithi164 bash[59462]: cluster 1685823378.8846078 mon.a (mon.0) 1144 : cluster 0 osdmap e211: 8 total, 7 up, 7 in 2023-06-03T20:16:20.085 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:16:19 smithi164 bash[59462]: audit 1685823379.4126217 mon.a (mon.0) 1145 : audit 1 from='client.? 172.21.15.89:0/2907305462' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-06-03T20:16:20.085 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:16:19 smithi164 bash[59462]: audit 1685823379.6383502 mon.a (mon.0) 1146 : audit 1 from='client.? 172.21.15.89:0/2115400778' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["6"]}]: dispatch 2023-06-03T20:16:20.088 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:16:19 smithi164 bash[59462]: cluster 1685823379.6385314 mon.a (mon.0) 1147 : cluster 1 Client client.admin marked osd.6 out, while it was still marked up 2023-06-03T20:16:20.129 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:16:20.130 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:16:20.131 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:16:20.232 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:16:20.262 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:16:20.264 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:16:20.265 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:16:20.366 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:16:20.407 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:16:20.410 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:16:20.411 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:16:20.512 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:16:20.551 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:16:20.553 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:16:20.553 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:16:20.654 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:16:20.683 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:16:20.686 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:16:20.686 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:16:20.787 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:16:20.818 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:16:20.822 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:16:20.823 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:16:20.924 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:16:20.937 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:16:20 smithi164 bash[59462]: cluster 1685823379.8770897 mon.a (mon.0) 1148 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-06-03T20:16:20.937 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:16:20 smithi164 bash[59462]: cluster 1685823379.8830078 osd.6 (osd.6) 156 : cluster 3 failed to encode map e212 with expected crc 2023-06-03T20:16:20.937 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:16:20 smithi164 bash[59462]: audit 1685823379.8864758 mon.a (mon.0) 1149 : audit 1 from='client.? 172.21.15.89:0/2907305462' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-06-03T20:16:20.937 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:16:20 smithi164 bash[59462]: audit 1685823379.8865263 mon.a (mon.0) 1150 : audit 1 from='client.? 172.21.15.89:0/2115400778' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["6"]}]': finished 2023-06-03T20:16:20.937 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:16:20 smithi164 bash[59462]: cluster 1685823379.886574 mon.a (mon.0) 1151 : cluster 0 osdmap e212: 8 total, 7 up, 6 in 2023-06-03T20:16:20.938 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:16:20 smithi164 bash[59462]: audit 1685823380.8038075 mon.a (mon.0) 1152 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:16:20.938 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:16:20 smithi164 bash[59462]: audit 1685823380.8122127 mon.a (mon.0) 1153 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:16:20.956 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:16:20.959 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:16:20.960 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:16:21.061 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:16:21.099 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:16:21.101 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:16:21.102 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:16:21.203 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:16:21.214 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:16:20 smithi089 bash[75068]: cluster 1685823379.8770897 mon.a (mon.0) 1148 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-06-03T20:16:21.214 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:16:20 smithi089 bash[75068]: cluster 1685823379.8830078 osd.6 (osd.6) 156 : cluster 3 failed to encode map e212 with expected crc 2023-06-03T20:16:21.214 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:16:20 smithi089 bash[75068]: audit 1685823379.8864758 mon.a (mon.0) 1149 : audit 1 from='client.? 172.21.15.89:0/2907305462' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-06-03T20:16:21.214 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:16:20 smithi089 bash[75068]: audit 1685823379.8865263 mon.a (mon.0) 1150 : audit 1 from='client.? 172.21.15.89:0/2115400778' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["6"]}]': finished 2023-06-03T20:16:21.214 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:16:20 smithi089 bash[75068]: cluster 1685823379.886574 mon.a (mon.0) 1151 : cluster 0 osdmap e212: 8 total, 7 up, 6 in 2023-06-03T20:16:21.215 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:16:20 smithi089 bash[75068]: audit 1685823380.8038075 mon.a (mon.0) 1152 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:16:21.215 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:16:20 smithi089 bash[75068]: audit 1685823380.8122127 mon.a (mon.0) 1153 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:16:21.215 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:16:20 smithi089 bash[78452]: cluster 1685823379.8770897 mon.a (mon.0) 1148 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-06-03T20:16:21.215 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:16:20 smithi089 bash[78452]: cluster 1685823379.8830078 osd.6 (osd.6) 156 : cluster 3 failed to encode map e212 with expected crc 2023-06-03T20:16:21.215 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:16:20 smithi089 bash[78452]: audit 1685823379.8864758 mon.a (mon.0) 1149 : audit 1 from='client.? 172.21.15.89:0/2907305462' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-06-03T20:16:21.215 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:16:20 smithi089 bash[78452]: audit 1685823379.8865263 mon.a (mon.0) 1150 : audit 1 from='client.? 172.21.15.89:0/2115400778' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["6"]}]': finished 2023-06-03T20:16:21.216 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:16:20 smithi089 bash[78452]: cluster 1685823379.886574 mon.a (mon.0) 1151 : cluster 0 osdmap e212: 8 total, 7 up, 6 in 2023-06-03T20:16:21.216 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:16:20 smithi089 bash[78452]: audit 1685823380.8038075 mon.a (mon.0) 1152 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:16:21.216 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:16:20 smithi089 bash[78452]: audit 1685823380.8122127 mon.a (mon.0) 1153 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:16:21.234 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:16:21.236 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:16:21.237 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:16:21.337 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:16:21.364 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:16:21.366 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:16:21.366 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:16:21.467 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:16:21.502 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:16:21.504 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:16:21.505 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:16:21.606 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:16:21.632 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:16:21.634 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:16:21.634 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:16:21.735 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:16:21.761 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:16:21.763 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:16:21.764 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:16:21.865 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:16:21.892 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:16:21.894 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:16:21.895 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:16:21.996 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:16:22.030 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:16:22.032 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:16:22.033 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:16:22.134 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:16:22.147 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:16:21 smithi089 bash[75068]: cluster 1685823380.815 mgr.y (mgr.34104) 1210 : cluster 0 pgmap v1192: 73 pgs: 2 peering, 9 active+clean+remapped, 7 active+undersized+degraded, 19 active+undersized, 36 active+clean; 582 KiB data, 3.2 GiB used, 533 GiB / 536 GiB avail; 12/72 objects degraded (16.667%); 2/72 objects misplaced (2.778%) 2023-06-03T20:16:22.148 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:16:21 smithi089 bash[75068]: cluster 1685823380.8913527 osd.6 (osd.6) 157 : cluster 3 failed to encode map e213 with expected crc 2023-06-03T20:16:22.148 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:16:21 smithi089 bash[75068]: cluster 1685823380.8914587 osd.5 (osd.5) 143 : cluster 3 failed to encode map e213 with expected crc 2023-06-03T20:16:22.148 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:16:21 smithi089 bash[75068]: cluster 1685823380.8935878 osd.7 (osd.7) 137 : cluster 3 failed to encode map e213 with expected crc 2023-06-03T20:16:22.149 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:16:21 smithi089 bash[75068]: cluster 1685823380.8938355 osd.5 (osd.5) 144 : cluster 3 failed to encode map e213 with expected crc 2023-06-03T20:16:22.149 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:16:21 smithi089 bash[75068]: cluster 1685823380.8948941 mon.a (mon.0) 1154 : cluster 0 osdmap e213: 8 total, 7 up, 6 in 2023-06-03T20:16:22.149 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:16:21 smithi089 bash[75068]: cluster 1685823380.8971398 osd.5 (osd.5) 145 : cluster 3 failed to encode map e213 with expected crc 2023-06-03T20:16:22.150 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:16:21 smithi089 bash[78452]: cluster 1685823380.815 mgr.y (mgr.34104) 1210 : cluster 0 pgmap v1192: 73 pgs: 2 peering, 9 active+clean+remapped, 7 active+undersized+degraded, 19 active+undersized, 36 active+clean; 582 KiB data, 3.2 GiB used, 533 GiB / 536 GiB avail; 12/72 objects degraded (16.667%); 2/72 objects misplaced (2.778%) 2023-06-03T20:16:22.150 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:16:21 smithi089 bash[78452]: cluster 1685823380.8913527 osd.6 (osd.6) 157 : cluster 3 failed to encode map e213 with expected crc 2023-06-03T20:16:22.150 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:16:21 smithi089 bash[78452]: cluster 1685823380.8914587 osd.5 (osd.5) 143 : cluster 3 failed to encode map e213 with expected crc 2023-06-03T20:16:22.150 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:16:21 smithi089 bash[78452]: cluster 1685823380.8935878 osd.7 (osd.7) 137 : cluster 3 failed to encode map e213 with expected crc 2023-06-03T20:16:22.151 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:16:21 smithi089 bash[78452]: cluster 1685823380.8938355 osd.5 (osd.5) 144 : cluster 3 failed to encode map e213 with expected crc 2023-06-03T20:16:22.151 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:16:21 smithi089 bash[78452]: cluster 1685823380.8948941 mon.a (mon.0) 1154 : cluster 0 osdmap e213: 8 total, 7 up, 6 in 2023-06-03T20:16:22.151 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:16:21 smithi089 bash[78452]: cluster 1685823380.8971398 osd.5 (osd.5) 145 : cluster 3 failed to encode map e213 with expected crc 2023-06-03T20:16:22.167 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:16:22.169 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:16:22.170 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:16:22.271 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:16:22.285 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:16:21 smithi164 bash[59462]: cluster 1685823380.815 mgr.y (mgr.34104) 1210 : cluster 0 pgmap v1192: 73 pgs: 2 peering, 9 active+clean+remapped, 7 active+undersized+degraded, 19 active+undersized, 36 active+clean; 582 KiB data, 3.2 GiB used, 533 GiB / 536 GiB avail; 12/72 objects degraded (16.667%); 2/72 objects misplaced (2.778%) 2023-06-03T20:16:22.285 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:16:21 smithi164 bash[59462]: cluster 1685823380.8913527 osd.6 (osd.6) 157 : cluster 3 failed to encode map e213 with expected crc 2023-06-03T20:16:22.285 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:16:21 smithi164 bash[59462]: cluster 1685823380.8914587 osd.5 (osd.5) 143 : cluster 3 failed to encode map e213 with expected crc 2023-06-03T20:16:22.285 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:16:21 smithi164 bash[59462]: cluster 1685823380.8935878 osd.7 (osd.7) 137 : cluster 3 failed to encode map e213 with expected crc 2023-06-03T20:16:22.286 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:16:21 smithi164 bash[59462]: cluster 1685823380.8938355 osd.5 (osd.5) 144 : cluster 3 failed to encode map e213 with expected crc 2023-06-03T20:16:22.286 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:16:21 smithi164 bash[59462]: cluster 1685823380.8948941 mon.a (mon.0) 1154 : cluster 0 osdmap e213: 8 total, 7 up, 6 in 2023-06-03T20:16:22.286 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:16:21 smithi164 bash[59462]: cluster 1685823380.8971398 osd.5 (osd.5) 145 : cluster 3 failed to encode map e213 with expected crc 2023-06-03T20:16:22.314 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:16:22.317 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:16:22.317 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:16:22.419 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:16:22.453 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:16:22.455 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:16:22.456 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:16:22.557 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:16:22.598 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:16:22.601 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:16:22.601 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:16:22.702 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:16:22.741 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:16:22.744 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:16:22.744 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:16:22.845 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:16:22.877 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:16:22.880 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:16:22.881 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:16:22.982 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:16:23.017 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:16:23.019 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:16:23.020 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:16:23.099 INFO:teuthology.task.full_sequential:In full_sequential, running task radosbench... 2023-06-03T20:16:23.100 INFO:tasks.radosbench:Beginning radosbench... 2023-06-03T20:16:23.101 INFO:tasks.cephadm.ceph_manager.ceph:creating pool_name unique_pool_7 2023-06-03T20:16:23.102 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph osd pool create unique_pool_7 16 2023-06-03T20:16:23.109 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:16:22 smithi089 bash[78452]: cluster 1685823381.8995268 osd.6 (osd.6) 158 : cluster 3 failed to encode map e214 with expected crc 2023-06-03T20:16:23.110 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:16:22 smithi089 bash[78452]: cluster 1685823381.9001155 osd.5 (osd.5) 146 : cluster 3 failed to encode map e214 with expected crc 2023-06-03T20:16:23.110 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:16:22 smithi089 bash[78452]: cluster 1685823381.9015155 osd.7 (osd.7) 138 : cluster 3 failed to encode map e214 with expected crc 2023-06-03T20:16:23.110 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:16:22 smithi089 bash[78452]: cluster 1685823381.902418 osd.5 (osd.5) 147 : cluster 3 failed to encode map e214 with expected crc 2023-06-03T20:16:23.110 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:16:22 smithi089 bash[78452]: cluster 1685823381.9029427 mon.a (mon.0) 1155 : cluster 0 osdmap e214: 8 total, 7 up, 6 in 2023-06-03T20:16:23.111 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:16:22 smithi089 bash[78452]: cluster 1685823381.9038498 osd.7 (osd.7) 139 : cluster 3 failed to encode map e214 with expected crc 2023-06-03T20:16:23.111 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:16:22 smithi089 bash[78452]: audit 1685823382.126789 mon.c (mon.2) 539 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T20:16:23.111 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:16:22 smithi089 bash[78452]: cluster 1685823382.7939806 mon.a (mon.0) 1156 : cluster 3 Health check update: Degraded data redundancy: 12/72 objects degraded (16.667%), 7 pgs degraded, 26 pgs undersized (PG_DEGRADED) 2023-06-03T20:16:23.112 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:16:22 smithi089 bash[78452]: cluster 1685823382.8022387 mon.a (mon.0) 1157 : cluster 0 osdmap e215: 8 total, 7 up, 6 in 2023-06-03T20:16:23.112 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:16:22 smithi089 bash[75068]: cluster 1685823381.8995268 osd.6 (osd.6) 158 : cluster 3 failed to encode map e214 with expected crc 2023-06-03T20:16:23.112 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:16:22 smithi089 bash[75068]: cluster 1685823381.9001155 osd.5 (osd.5) 146 : cluster 3 failed to encode map e214 with expected crc 2023-06-03T20:16:23.113 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:16:22 smithi089 bash[75068]: cluster 1685823381.9015155 osd.7 (osd.7) 138 : cluster 3 failed to encode map e214 with expected crc 2023-06-03T20:16:23.113 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:16:22 smithi089 bash[75068]: cluster 1685823381.902418 osd.5 (osd.5) 147 : cluster 3 failed to encode map e214 with expected crc 2023-06-03T20:16:23.113 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:16:22 smithi089 bash[75068]: cluster 1685823381.9029427 mon.a (mon.0) 1155 : cluster 0 osdmap e214: 8 total, 7 up, 6 in 2023-06-03T20:16:23.113 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:16:22 smithi089 bash[75068]: cluster 1685823381.9038498 osd.7 (osd.7) 139 : cluster 3 failed to encode map e214 with expected crc 2023-06-03T20:16:23.114 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:16:22 smithi089 bash[75068]: audit 1685823382.126789 mon.c (mon.2) 539 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T20:16:23.114 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:16:22 smithi089 bash[75068]: cluster 1685823382.7939806 mon.a (mon.0) 1156 : cluster 3 Health check update: Degraded data redundancy: 12/72 objects degraded (16.667%), 7 pgs degraded, 26 pgs undersized (PG_DEGRADED) 2023-06-03T20:16:23.114 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:16:22 smithi089 bash[75068]: cluster 1685823382.8022387 mon.a (mon.0) 1157 : cluster 0 osdmap e215: 8 total, 7 up, 6 in 2023-06-03T20:16:23.121 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:16:23.150 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:16:23.152 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:16:23.153 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:16:23.254 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:16:23.281 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:16:23.283 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:16:23.284 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:16:23.332 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph daemon osd.2 dump_blocked_ops 2023-06-03T20:16:23.334 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:16:22 smithi164 bash[59462]: cluster 1685823381.8995268 osd.6 (osd.6) 158 : cluster 3 failed to encode map e214 with expected crc 2023-06-03T20:16:23.334 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:16:22 smithi164 bash[59462]: cluster 1685823381.9001155 osd.5 (osd.5) 146 : cluster 3 failed to encode map e214 with expected crc 2023-06-03T20:16:23.334 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:16:22 smithi164 bash[59462]: cluster 1685823381.9015155 osd.7 (osd.7) 138 : cluster 3 failed to encode map e214 with expected crc 2023-06-03T20:16:23.335 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:16:22 smithi164 bash[59462]: cluster 1685823381.902418 osd.5 (osd.5) 147 : cluster 3 failed to encode map e214 with expected crc 2023-06-03T20:16:23.335 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:16:22 smithi164 bash[59462]: cluster 1685823381.9029427 mon.a (mon.0) 1155 : cluster 0 osdmap e214: 8 total, 7 up, 6 in 2023-06-03T20:16:23.335 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:16:22 smithi164 bash[59462]: cluster 1685823381.9038498 osd.7 (osd.7) 139 : cluster 3 failed to encode map e214 with expected crc 2023-06-03T20:16:23.335 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:16:22 smithi164 bash[59462]: audit 1685823382.126789 mon.c (mon.2) 539 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T20:16:23.336 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:16:22 smithi164 bash[59462]: cluster 1685823382.7939806 mon.a (mon.0) 1156 : cluster 3 Health check update: Degraded data redundancy: 12/72 objects degraded (16.667%), 7 pgs degraded, 26 pgs undersized (PG_DEGRADED) 2023-06-03T20:16:23.336 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:16:22 smithi164 bash[59462]: cluster 1685823382.8022387 mon.a (mon.0) 1157 : cluster 0 osdmap e215: 8 total, 7 up, 6 in 2023-06-03T20:16:23.385 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:16:23.411 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:16:23.413 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:16:23.415 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:16:23.517 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:16:23.542 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:16:23.545 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:16:23.546 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:16:23.572 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-06-03T20:16:23.573 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-06-03T20:16:23.647 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:16:23.706 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:16:23.710 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:16:23.712 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:16:23.813 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:16:23.847 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:16:23.849 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:16:23.850 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:16:23.952 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:16:23.965 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:16:23 smithi089 bash[78452]: cluster 1685823382.797057 osd.5 (osd.5) 148 : cluster 3 failed to encode map e215 with expected crc 2023-06-03T20:16:23.965 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:16:23 smithi089 bash[78452]: cluster 1685823382.7970734 osd.6 (osd.6) 159 : cluster 3 failed to encode map e215 with expected crc 2023-06-03T20:16:23.965 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:16:23 smithi089 bash[78452]: cluster 1685823382.8010945 osd.7 (osd.7) 140 : cluster 3 failed to encode map e215 with expected crc 2023-06-03T20:16:23.966 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:16:23 smithi089 bash[78452]: cluster 1685823382.8157122 mgr.y (mgr.34104) 1211 : cluster 0 pgmap v1196: 73 pgs: 1 remapped+peering, 2 peering, 9 active+clean+remapped, 7 active+undersized+degraded, 18 active+undersized, 36 active+clean; 582 KiB data, 3.2 GiB used, 533 GiB / 536 GiB avail; 12/72 objects degraded (16.667%); 2/72 objects misplaced (2.778%); 0 B/s, 1 objects/s recovering 2023-06-03T20:16:23.966 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:16:23 smithi089 bash[78452]: cluster 1685823383.820414 mon.a (mon.0) 1158 : cluster 0 osdmap e216: 8 total, 7 up, 6 in 2023-06-03T20:16:23.966 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:16:23 smithi089 bash[75068]: cluster 1685823382.797057 osd.5 (osd.5) 148 : cluster 3 failed to encode map e215 with expected crc 2023-06-03T20:16:23.966 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:16:23 smithi089 bash[75068]: cluster 1685823382.7970734 osd.6 (osd.6) 159 : cluster 3 failed to encode map e215 with expected crc 2023-06-03T20:16:23.966 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:16:23 smithi089 bash[75068]: cluster 1685823382.8010945 osd.7 (osd.7) 140 : cluster 3 failed to encode map e215 with expected crc 2023-06-03T20:16:23.967 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:16:23 smithi089 bash[75068]: cluster 1685823382.8157122 mgr.y (mgr.34104) 1211 : cluster 0 pgmap v1196: 73 pgs: 1 remapped+peering, 2 peering, 9 active+clean+remapped, 7 active+undersized+degraded, 18 active+undersized, 36 active+clean; 582 KiB data, 3.2 GiB used, 533 GiB / 536 GiB avail; 12/72 objects degraded (16.667%); 2/72 objects misplaced (2.778%); 0 B/s, 1 objects/s recovering 2023-06-03T20:16:23.967 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:16:23 smithi089 bash[75068]: cluster 1685823383.820414 mon.a (mon.0) 1158 : cluster 0 osdmap e216: 8 total, 7 up, 6 in 2023-06-03T20:16:23.986 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:16:23.989 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:16:23.990 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:16:24.091 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:16:24.127 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:16:24.131 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:16:24.132 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:16:24.233 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:16:24.246 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:16:23 smithi164 bash[59462]: cluster 1685823382.797057 osd.5 (osd.5) 148 : cluster 3 failed to encode map e215 with expected crc 2023-06-03T20:16:24.247 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:16:23 smithi164 bash[59462]: cluster 1685823382.7970734 osd.6 (osd.6) 159 : cluster 3 failed to encode map e215 with expected crc 2023-06-03T20:16:24.247 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:16:23 smithi164 bash[59462]: cluster 1685823382.8010945 osd.7 (osd.7) 140 : cluster 3 failed to encode map e215 with expected crc 2023-06-03T20:16:24.247 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:16:23 smithi164 bash[59462]: cluster 1685823382.8157122 mgr.y (mgr.34104) 1211 : cluster 0 pgmap v1196: 73 pgs: 1 remapped+peering, 2 peering, 9 active+clean+remapped, 7 active+undersized+degraded, 18 active+undersized, 36 active+clean; 582 KiB data, 3.2 GiB used, 533 GiB / 536 GiB avail; 12/72 objects degraded (16.667%); 2/72 objects misplaced (2.778%); 0 B/s, 1 objects/s recovering 2023-06-03T20:16:24.247 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:16:23 smithi164 bash[59462]: cluster 1685823383.820414 mon.a (mon.0) 1158 : cluster 0 osdmap e216: 8 total, 7 up, 6 in 2023-06-03T20:16:24.272 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:16:24.275 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:16:24.276 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:16:24.377 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:16:24.414 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:16:24.417 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:16:24.418 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:16:24.519 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:16:24.552 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:16:24.554 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:16:24.555 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:16:24.655 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:16:24.685 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:16:24.687 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:16:24.688 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:16:24.789 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:16:24.826 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:16:24.828 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:16:24.828 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:16:24.929 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:16:24.957 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:16:24.960 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:16:24.961 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:16:25.062 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:16:25.095 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:16:25.098 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:16:25.099 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:16:25.200 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:16:25.209 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:16:24 smithi089 bash[75068]: cluster 1685823383.8162558 osd.6 (osd.6) 160 : cluster 3 failed to encode map e216 with expected crc 2023-06-03T20:16:25.209 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:16:24 smithi089 bash[75068]: cluster 1685823383.816446 osd.5 (osd.5) 149 : cluster 3 failed to encode map e216 with expected crc 2023-06-03T20:16:25.209 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:16:24 smithi089 bash[75068]: cluster 1685823383.819187 osd.7 (osd.7) 141 : cluster 3 failed to encode map e216 with expected crc 2023-06-03T20:16:25.210 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:16:24 smithi089 bash[78452]: cluster 1685823383.8162558 osd.6 (osd.6) 160 : cluster 3 failed to encode map e216 with expected crc 2023-06-03T20:16:25.210 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:16:24 smithi089 bash[78452]: cluster 1685823383.816446 osd.5 (osd.5) 149 : cluster 3 failed to encode map e216 with expected crc 2023-06-03T20:16:25.210 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:16:24 smithi089 bash[78452]: cluster 1685823383.819187 osd.7 (osd.7) 141 : cluster 3 failed to encode map e216 with expected crc 2023-06-03T20:16:25.226 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:16:25.252 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:16:25.253 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:16:25.333 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:16:24 smithi164 bash[59462]: cluster 1685823383.8162558 osd.6 (osd.6) 160 : cluster 3 failed to encode map e216 with expected crc 2023-06-03T20:16:25.334 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:16:24 smithi164 bash[59462]: cluster 1685823383.816446 osd.5 (osd.5) 149 : cluster 3 failed to encode map e216 with expected crc 2023-06-03T20:16:25.334 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:16:24 smithi164 bash[59462]: cluster 1685823383.819187 osd.7 (osd.7) 141 : cluster 3 failed to encode map e216 with expected crc 2023-06-03T20:16:25.355 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:16:25.391 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:16:25.393 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:16:25.393 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:16:25.494 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:16:25.526 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:16:25.529 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:16:25.530 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:16:25.574 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-06-03T20:16:25.630 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph osd unset noscrub 2023-06-03T20:16:25.631 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:16:25.667 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:16:25.670 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:16:25.671 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:16:25.771 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:16:25.801 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:16:25.804 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:16:25.804 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:16:25.905 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:16:25.943 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:16:25.946 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:16:25.946 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:16:26.047 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:16:26.081 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:16:26.084 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:16:26.084 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:16:26.185 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:16:26.197 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:16:25 smithi164 bash[59462]: cluster 1685823384.8168392 mgr.y (mgr.34104) 1212 : cluster 0 pgmap v1198: 73 pgs: 1 remapped+peering, 3 peering, 18 active+clean+remapped, 6 active+undersized+degraded, 11 active+undersized, 34 active+clean; 582 KiB data, 3.2 GiB used, 533 GiB / 536 GiB avail; 8/72 objects degraded (11.111%); 6/72 objects misplaced (8.333%); 145 KiB/s, 2 objects/s recovering 2023-06-03T20:16:26.198 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:16:25 smithi164 bash[59462]: audit 1685823385.4802325 mon.a (mon.0) 1159 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:16:26.198 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:16:25 smithi164 bash[59462]: audit 1685823385.4940362 mon.a (mon.0) 1160 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:16:26.222 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:16:26.225 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:16:26.226 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:16:26.232 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:16:25 smithi089 bash[75068]: cluster 1685823384.8168392 mgr.y (mgr.34104) 1212 : cluster 0 pgmap v1198: 73 pgs: 1 remapped+peering, 3 peering, 18 active+clean+remapped, 6 active+undersized+degraded, 11 active+undersized, 34 active+clean; 582 KiB data, 3.2 GiB used, 533 GiB / 536 GiB avail; 8/72 objects degraded (11.111%); 6/72 objects misplaced (8.333%); 145 KiB/s, 2 objects/s recovering 2023-06-03T20:16:26.232 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:16:25 smithi089 bash[75068]: audit 1685823385.4802325 mon.a (mon.0) 1159 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:16:26.232 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:16:25 smithi089 bash[75068]: audit 1685823385.4940362 mon.a (mon.0) 1160 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:16:26.232 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:16:25 smithi089 bash[78452]: cluster 1685823384.8168392 mgr.y (mgr.34104) 1212 : cluster 0 pgmap v1198: 73 pgs: 1 remapped+peering, 3 peering, 18 active+clean+remapped, 6 active+undersized+degraded, 11 active+undersized, 34 active+clean; 582 KiB data, 3.2 GiB used, 533 GiB / 536 GiB avail; 8/72 objects degraded (11.111%); 6/72 objects misplaced (8.333%); 145 KiB/s, 2 objects/s recovering 2023-06-03T20:16:26.233 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:16:25 smithi089 bash[78452]: audit 1685823385.4802325 mon.a (mon.0) 1159 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:16:26.233 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:16:25 smithi089 bash[78452]: audit 1685823385.4940362 mon.a (mon.0) 1160 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:16:26.326 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:16:26.361 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:16:26.364 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:16:26.365 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:16:26.466 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:16:26.500 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:16:26.502 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 20:16:26 smithi089 bash[71163]: ::ffff:172.21.15.164 - - [03/Jun/2023:20:16:26] "GET /metrics HTTP/1.1" 200 226428 "" "Prometheus/2.33.4" 2023-06-03T20:16:26.503 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:16:26.504 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:16:26.604 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:16:26.642 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:16:26.645 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:16:26.646 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:16:26.746 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:16:26.786 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:16:26.789 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:16:26.790 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:16:26.891 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:16:26.927 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:16:26.930 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:16:26.931 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:16:27.031 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:16:27.068 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:16:27.071 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:16:27.072 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:16:27.083 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:16:26 smithi164 bash[59462]: audit 1685823385.964869 mon.c (mon.2) 540 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-03T20:16:27.083 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:16:26 smithi164 bash[59462]: audit 1685823385.966709 mon.c (mon.2) 541 : audit 1 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-03T20:16:27.084 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:16:26 smithi164 bash[59462]: audit 1685823385.9762716 mon.a (mon.0) 1161 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:16:27.173 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:16:27.184 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:16:26 smithi089 bash[78452]: audit 1685823385.964869 mon.c (mon.2) 540 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-03T20:16:27.184 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:16:26 smithi089 bash[78452]: audit 1685823385.966709 mon.c (mon.2) 541 : audit 1 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-03T20:16:27.185 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:16:26 smithi089 bash[78452]: audit 1685823385.9762716 mon.a (mon.0) 1161 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:16:27.185 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:16:26 smithi089 bash[75068]: audit 1685823385.964869 mon.c (mon.2) 540 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-03T20:16:27.185 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:16:26 smithi089 bash[75068]: audit 1685823385.966709 mon.c (mon.2) 541 : audit 1 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-03T20:16:27.185 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:16:26 smithi089 bash[75068]: audit 1685823385.9762716 mon.a (mon.0) 1161 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:16:27.208 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:16:27.211 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:16:27.212 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:16:27.313 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:16:27.347 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:16:27.349 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:16:27.350 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:16:27.451 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:16:27.484 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:16:27.489 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:16:27.489 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:16:27.591 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:16:27.631 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:16:27.634 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:16:27.635 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:16:27.736 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:16:27.771 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:16:27.774 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:16:27.774 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:16:27.875 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:16:27.915 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:16:27.918 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:16:27.919 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:16:28.020 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:16:28.053 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:16:28.057 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:16:28.057 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:16:28.158 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:16:28.194 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:16:28.197 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:16:28.198 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:16:28.209 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/mon.a/config 2023-06-03T20:16:28.210 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/mon.a/config 2023-06-03T20:16:28.239 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:16:27 smithi089 bash[78452]: cluster 1685823386.8180523 mgr.y (mgr.34104) 1213 : cluster 0 pgmap v1199: 73 pgs: 1 peering, 20 active+clean+remapped, 6 active+undersized+degraded, 11 active+undersized, 35 active+clean; 582 KiB data, 3.2 GiB used, 533 GiB / 536 GiB avail; 8/72 objects degraded (11.111%); 6/72 objects misplaced (8.333%); 98 KiB/s, 1 objects/s recovering 2023-06-03T20:16:28.240 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:16:27 smithi089 bash[78452]: cluster 1685823387.7958539 mon.a (mon.0) 1162 : cluster 3 Health check update: Degraded data redundancy: 8/72 objects degraded (11.111%), 6 pgs degraded, 17 pgs undersized (PG_DEGRADED) 2023-06-03T20:16:28.241 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:16:27 smithi089 bash[75068]: cluster 1685823386.8180523 mgr.y (mgr.34104) 1213 : cluster 0 pgmap v1199: 73 pgs: 1 peering, 20 active+clean+remapped, 6 active+undersized+degraded, 11 active+undersized, 35 active+clean; 582 KiB data, 3.2 GiB used, 533 GiB / 536 GiB avail; 8/72 objects degraded (11.111%); 6/72 objects misplaced (8.333%); 98 KiB/s, 1 objects/s recovering 2023-06-03T20:16:28.241 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:16:27 smithi089 bash[75068]: cluster 1685823387.7958539 mon.a (mon.0) 1162 : cluster 3 Health check update: Degraded data redundancy: 8/72 objects degraded (11.111%), 6 pgs degraded, 17 pgs undersized (PG_DEGRADED) 2023-06-03T20:16:28.298 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:16:28.334 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:16:27 smithi164 bash[59462]: cluster 1685823386.8180523 mgr.y (mgr.34104) 1213 : cluster 0 pgmap v1199: 73 pgs: 1 peering, 20 active+clean+remapped, 6 active+undersized+degraded, 11 active+undersized, 35 active+clean; 582 KiB data, 3.2 GiB used, 533 GiB / 536 GiB avail; 8/72 objects degraded (11.111%); 6/72 objects misplaced (8.333%); 98 KiB/s, 1 objects/s recovering 2023-06-03T20:16:28.334 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:16:27 smithi164 bash[59462]: cluster 1685823387.7958539 mon.a (mon.0) 1162 : cluster 3 Health check update: Degraded data redundancy: 8/72 objects degraded (11.111%), 6 pgs degraded, 17 pgs undersized (PG_DEGRADED) 2023-06-03T20:16:28.335 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:16:28.337 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:16:28.338 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:16:28.439 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:16:28.476 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:16:28.478 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:16:28.479 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:16:28.580 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:16:28.597 INFO:tasks.thrashosds.thrasher:in_osds: [0, 2, 3, 4, 5, 7] out_osds: [1, 6] dead_osds: [4] live_osds: [0, 1, 2, 3, 5, 6, 7] 2023-06-03T20:16:28.598 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-06-03T20:16:28.598 INFO:tasks.thrashosds.thrasher:inject_pause on osd.1 2023-06-03T20:16:28.598 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2023-06-03T20:16:28.598 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2023-06-03T20:16:28.599 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph daemon osd.1 config set filestore_inject_stall 3 2023-06-03T20:16:28.615 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:16:28.618 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:16:28.619 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:16:28.720 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:16:28.758 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:16:28.761 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:16:28.762 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:16:28.863 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:16:28.894 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:16:28.896 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:16:28.897 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:16:28.998 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:16:29.031 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:16:29.033 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:16:29.034 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:16:29.134 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:16:29.172 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:16:29.175 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:16:29.175 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:16:29.276 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:16:29.315 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:16:29.318 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:16:29.319 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:16:29.420 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:16:29.460 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:16:29.463 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:16:29.463 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:16:29.564 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:16:29.602 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:16:29.606 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:16:29.607 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:16:29.708 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:16:29.747 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:16:29.749 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:16:29.750 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:16:29.851 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:16:29.894 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:16:29.896 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:16:29.897 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:16:29.998 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:16:30.032 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:16:30.034 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:16:30.035 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:16:30.136 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:16:30.175 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:16:30.178 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:16:30.178 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:16:30.250 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/mon.a/config 2023-06-03T20:16:30.251 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/mon.a/config 2023-06-03T20:16:30.278 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:16:29 smithi089 bash[75068]: cluster 1685823388.819356 mgr.y (mgr.34104) 1214 : cluster 0 pgmap v1200: 73 pgs: 20 active+clean+remapped, 6 active+undersized+degraded, 11 active+undersized, 36 active+clean; 582 KiB data, 3.2 GiB used, 533 GiB / 536 GiB avail; 8/72 objects degraded (11.111%); 6/72 objects misplaced (8.333%); 84 KiB/s, 1 objects/s recovering 2023-06-03T20:16:30.279 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:16:29 smithi089 bash[78452]: cluster 1685823388.819356 mgr.y (mgr.34104) 1214 : cluster 0 pgmap v1200: 73 pgs: 20 active+clean+remapped, 6 active+undersized+degraded, 11 active+undersized, 36 active+clean; 582 KiB data, 3.2 GiB used, 533 GiB / 536 GiB avail; 8/72 objects degraded (11.111%); 6/72 objects misplaced (8.333%); 84 KiB/s, 1 objects/s recovering 2023-06-03T20:16:30.279 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:16:30.292 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:16:29 smithi164 bash[59462]: cluster 1685823388.819356 mgr.y (mgr.34104) 1214 : cluster 0 pgmap v1200: 73 pgs: 20 active+clean+remapped, 6 active+undersized+degraded, 11 active+undersized, 36 active+clean; 582 KiB data, 3.2 GiB used, 533 GiB / 536 GiB avail; 8/72 objects degraded (11.111%); 6/72 objects misplaced (8.333%); 84 KiB/s, 1 objects/s recovering 2023-06-03T20:16:30.316 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:16:30.319 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:16:30.320 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:16:30.420 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:16:30.460 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:16:30.462 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:16:30.463 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:16:30.563 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:16:30.594 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:16:30.596 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:16:30.597 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:16:30.698 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:16:30.738 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:16:30.740 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:16:30.741 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:16:30.841 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:16:30.871 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:16:30.875 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:16:30.875 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:16:30.976 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:16:31.009 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:16:31.013 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:16:31.013 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:16:31.114 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:16:31.143 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:16:31.146 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:16:31.146 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:16:31.247 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:16:31.274 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:16:31.277 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:16:31.278 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:16:31.379 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:16:31.420 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:16:31.423 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:16:31.424 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:16:31.525 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:16:31.560 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:16:31.563 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:16:31.563 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:16:31.664 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:16:31.690 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:16:31.692 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:16:31.693 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:16:31.793 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:16:31.834 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:16:31.837 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:16:31.837 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:16:31.938 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:16:31.974 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:16:31.977 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:16:31.978 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:16:32.078 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:16:32.083 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:16:31 smithi164 bash[59462]: cluster 1685823390.820183 mgr.y (mgr.34104) 1215 : cluster 0 pgmap v1201: 73 pgs: 20 active+clean+remapped, 6 active+undersized+degraded, 11 active+undersized, 36 active+clean; 582 KiB data, 3.2 GiB used, 533 GiB / 536 GiB avail; 8/72 objects degraded (11.111%); 6/72 objects misplaced (8.333%); 72 KiB/s, 0 objects/s recovering 2023-06-03T20:16:32.122 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:16:32.125 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:16:32.126 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:16:32.227 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:16:32.264 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:16:32.267 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:16:32.268 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:16:32.278 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:16:31 smithi089 bash[75068]: cluster 1685823390.820183 mgr.y (mgr.34104) 1215 : cluster 0 pgmap v1201: 73 pgs: 20 active+clean+remapped, 6 active+undersized+degraded, 11 active+undersized, 36 active+clean; 582 KiB data, 3.2 GiB used, 533 GiB / 536 GiB avail; 8/72 objects degraded (11.111%); 6/72 objects misplaced (8.333%); 72 KiB/s, 0 objects/s recovering 2023-06-03T20:16:32.279 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:16:31 smithi089 bash[78452]: cluster 1685823390.820183 mgr.y (mgr.34104) 1215 : cluster 0 pgmap v1201: 73 pgs: 20 active+clean+remapped, 6 active+undersized+degraded, 11 active+undersized, 36 active+clean; 582 KiB data, 3.2 GiB used, 533 GiB / 536 GiB avail; 8/72 objects degraded (11.111%); 6/72 objects misplaced (8.333%); 72 KiB/s, 0 objects/s recovering 2023-06-03T20:16:32.368 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:16:32.399 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:16:32.402 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:16:32.403 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:16:32.504 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:16:32.531 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:16:32.532 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:16:32.533 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:16:32.634 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:16:32.679 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:16:32.681 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:16:32.682 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:16:32.783 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:16:32.815 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:16:32.817 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:16:32.818 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:16:32.919 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:16:32.955 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:16:32.958 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:16:32.958 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:16:33.059 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:16:33.088 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:16:33.090 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:16:33.091 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:16:33.192 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:16:33.227 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:16:33.230 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:16:33.231 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:16:33.332 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:16:33.366 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:16:33.370 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:16:33.371 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:16:33.472 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:16:33.508 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:16:33.510 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:16:33.510 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:16:33.611 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:16:33.645 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:16:33.647 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:16:33.648 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:16:33.749 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:16:33.782 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:16:33.785 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:16:33.785 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:16:33.886 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:16:33.897 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:16:33 smithi089 bash[75068]: cluster 1685823392.8209217 mgr.y (mgr.34104) 1216 : cluster 0 pgmap v1202: 73 pgs: 20 active+clean+remapped, 6 active+undersized+degraded, 11 active+undersized, 36 active+clean; 582 KiB data, 3.2 GiB used, 533 GiB / 536 GiB avail; 8/72 objects degraded (11.111%); 6/72 objects misplaced (8.333%); 58 KiB/s, 0 objects/s recovering 2023-06-03T20:16:33.898 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:16:33 smithi089 bash[78452]: cluster 1685823392.8209217 mgr.y (mgr.34104) 1216 : cluster 0 pgmap v1202: 73 pgs: 20 active+clean+remapped, 6 active+undersized+degraded, 11 active+undersized, 36 active+clean; 582 KiB data, 3.2 GiB used, 533 GiB / 536 GiB avail; 8/72 objects degraded (11.111%); 6/72 objects misplaced (8.333%); 58 KiB/s, 0 objects/s recovering 2023-06-03T20:16:33.919 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:16:33.921 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:16:33.922 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:16:34.023 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:16:34.057 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:16:34.060 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:16:34.061 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:16:34.083 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:16:33 smithi164 bash[59462]: cluster 1685823392.8209217 mgr.y (mgr.34104) 1216 : cluster 0 pgmap v1202: 73 pgs: 20 active+clean+remapped, 6 active+undersized+degraded, 11 active+undersized, 36 active+clean; 582 KiB data, 3.2 GiB used, 533 GiB / 536 GiB avail; 8/72 objects degraded (11.111%); 6/72 objects misplaced (8.333%); 58 KiB/s, 0 objects/s recovering 2023-06-03T20:16:34.162 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:16:34.202 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:16:34.205 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:16:34.206 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:16:34.307 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:16:34.346 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:16:34.349 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:16:34.349 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:16:34.450 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:16:34.483 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:16:34.487 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:16:34.487 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:16:34.588 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:16:34.613 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:16:34.615 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:16:34.616 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:16:34.717 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:16:34.754 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:16:34.756 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:16:34.757 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:16:34.859 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:16:34.895 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:16:34.898 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:16:34.899 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:16:35.001 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:16:35.032 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:16:35.035 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:16:35.036 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:16:35.138 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:16:35.175 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:16:35.178 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:16:35.178 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:16:35.279 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:16:35.315 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:16:35.318 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:16:35.319 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:16:35.420 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:16:35.459 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:16:35.462 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:16:35.464 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:16:35.565 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:16:35.600 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:16:35.603 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:16:35.603 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:16:35.704 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:16:35.742 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:16:35.746 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:16:35.746 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:16:35.847 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:16:35.887 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:16:35.889 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:16:35.890 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:16:35.991 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:16:36.022 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:16:36.025 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:16:36.026 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:16:36.083 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:16:35 smithi164 bash[59462]: cluster 1685823394.821617 mgr.y (mgr.34104) 1217 : cluster 0 pgmap v1203: 73 pgs: 20 active+clean+remapped, 6 active+undersized+degraded, 11 active+undersized, 36 active+clean; 582 KiB data, 3.2 GiB used, 533 GiB / 536 GiB avail; 8/72 objects degraded (11.111%); 6/72 objects misplaced (8.333%) 2023-06-03T20:16:36.126 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:16:36.170 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:16:36.173 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:16:36.173 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:16:36.232 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:16:35 smithi089 bash[75068]: cluster 1685823394.821617 mgr.y (mgr.34104) 1217 : cluster 0 pgmap v1203: 73 pgs: 20 active+clean+remapped, 6 active+undersized+degraded, 11 active+undersized, 36 active+clean; 582 KiB data, 3.2 GiB used, 533 GiB / 536 GiB avail; 8/72 objects degraded (11.111%); 6/72 objects misplaced (8.333%) 2023-06-03T20:16:36.233 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:16:35 smithi089 bash[78452]: cluster 1685823394.821617 mgr.y (mgr.34104) 1217 : cluster 0 pgmap v1203: 73 pgs: 20 active+clean+remapped, 6 active+undersized+degraded, 11 active+undersized, 36 active+clean; 582 KiB data, 3.2 GiB used, 533 GiB / 536 GiB avail; 8/72 objects degraded (11.111%); 6/72 objects misplaced (8.333%) 2023-06-03T20:16:36.274 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:16:36.307 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:16:36.309 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:16:36.310 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:16:36.411 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:16:36.437 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:16:36.440 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:16:36.441 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:16:36.508 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 20:16:36 smithi089 bash[71163]: ::ffff:172.21.15.164 - - [03/Jun/2023:20:16:36] "GET /metrics HTTP/1.1" 200 226428 "" "Prometheus/2.33.4" 2023-06-03T20:16:36.543 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:16:36.586 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:16:36.588 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:16:36.589 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:16:36.690 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:16:36.733 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:16:36.735 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:16:36.735 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:16:36.836 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:16:36.866 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:16:36.868 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:16:36.868 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:16:36.911 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/mon.a/config 2023-06-03T20:16:36.970 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:16:36.998 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:16:37.000 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:16:37.001 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:16:37.102 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:16:37.127 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:16:37.131 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:16:37.131 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:16:37.232 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:16:37.261 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:16:37.263 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:16:37.264 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:16:37.365 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:16:37.399 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:16:37.402 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:16:37.402 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:16:37.503 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:16:37.539 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:16:37.542 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:16:37.543 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:16:37.644 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:16:37.683 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:16:37.686 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:16:37.688 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:16:37.789 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:16:37.827 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:16:37.830 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:16:37.830 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:16:37.931 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:16:37.962 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:16:37.965 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:16:37.965 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:16:38.066 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:16:38.080 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:16:37 smithi164 bash[59462]: cluster 1685823396.8225245 mgr.y (mgr.34104) 1218 : cluster 0 pgmap v1204: 73 pgs: 20 active+clean+remapped, 6 active+undersized+degraded, 11 active+undersized, 36 active+clean; 582 KiB data, 3.2 GiB used, 533 GiB / 536 GiB avail; 8/72 objects degraded (11.111%); 6/72 objects misplaced (8.333%) 2023-06-03T20:16:38.080 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:16:37 smithi164 bash[59462]: audit 1685823397.1270137 mon.c (mon.2) 542 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T20:16:38.104 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:16:38.107 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:16:38.108 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:16:38.209 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:16:38.216 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:16:37 smithi089 bash[75068]: cluster 1685823396.8225245 mgr.y (mgr.34104) 1218 : cluster 0 pgmap v1204: 73 pgs: 20 active+clean+remapped, 6 active+undersized+degraded, 11 active+undersized, 36 active+clean; 582 KiB data, 3.2 GiB used, 533 GiB / 536 GiB avail; 8/72 objects degraded (11.111%); 6/72 objects misplaced (8.333%) 2023-06-03T20:16:38.217 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:16:37 smithi089 bash[75068]: audit 1685823397.1270137 mon.c (mon.2) 542 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T20:16:38.217 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:16:37 smithi089 bash[78452]: cluster 1685823396.8225245 mgr.y (mgr.34104) 1218 : cluster 0 pgmap v1204: 73 pgs: 20 active+clean+remapped, 6 active+undersized+degraded, 11 active+undersized, 36 active+clean; 582 KiB data, 3.2 GiB used, 533 GiB / 536 GiB avail; 8/72 objects degraded (11.111%); 6/72 objects misplaced (8.333%) 2023-06-03T20:16:38.217 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:16:37 smithi089 bash[78452]: audit 1685823397.1270137 mon.c (mon.2) 542 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T20:16:38.238 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:16:38.240 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:16:38.241 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:16:38.342 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:16:38.371 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:16:38.374 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:16:38.375 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:16:38.476 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:16:38.514 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:16:38.517 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:16:38.518 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:16:38.619 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:16:38.651 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:16:38.654 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:16:38.654 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:16:38.755 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:16:38.799 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:16:38.802 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:16:38.803 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:16:38.904 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:16:38.938 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:16:38.940 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:16:38.941 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:16:39.042 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:16:39.085 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:16:39.088 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:16:39.088 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:16:39.189 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:16:39.229 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:16:39.232 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:16:39.233 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:16:39.334 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:16:39.365 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:16:39.368 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:16:39.368 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:16:39.469 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:16:39.504 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:16:39.507 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:16:39.508 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:16:39.609 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:16:39.642 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:16:39.645 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:16:39.645 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:16:39.746 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:16:39.776 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:16:39.779 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:16:39.780 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:16:39.880 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:16:39.906 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:16:39.908 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:16:39.908 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:16:40.009 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:16:40.036 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:16:40.038 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:16:40.039 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:16:40.140 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:16:40.148 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:16:39 smithi089 bash[78452]: cluster 1685823398.8237677 mgr.y (mgr.34104) 1219 : cluster 0 pgmap v1205: 73 pgs: 20 active+clean+remapped, 6 active+undersized+degraded, 11 active+undersized, 36 active+clean; 582 KiB data, 3.2 GiB used, 533 GiB / 536 GiB avail; 8/72 objects degraded (11.111%); 6/72 objects misplaced (8.333%) 2023-06-03T20:16:40.148 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:16:39 smithi089 bash[75068]: cluster 1685823398.8237677 mgr.y (mgr.34104) 1219 : cluster 0 pgmap v1205: 73 pgs: 20 active+clean+remapped, 6 active+undersized+degraded, 11 active+undersized, 36 active+clean; 582 KiB data, 3.2 GiB used, 533 GiB / 536 GiB avail; 8/72 objects degraded (11.111%); 6/72 objects misplaced (8.333%) 2023-06-03T20:16:40.168 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:16:40.171 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:16:40.172 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:16:40.273 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:16:40.287 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:16:39 smithi164 bash[59462]: cluster 1685823398.8237677 mgr.y (mgr.34104) 1219 : cluster 0 pgmap v1205: 73 pgs: 20 active+clean+remapped, 6 active+undersized+degraded, 11 active+undersized, 36 active+clean; 582 KiB data, 3.2 GiB used, 533 GiB / 536 GiB avail; 8/72 objects degraded (11.111%); 6/72 objects misplaced (8.333%) 2023-06-03T20:16:40.309 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:16:40.312 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:16:40.313 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:16:40.414 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:16:40.448 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:16:40.452 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:16:40.452 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:16:40.553 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:16:40.579 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:16:40.581 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:16:40.583 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:16:40.683 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:16:40.717 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:16:40.720 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:16:40.721 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:16:40.822 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:16:40.862 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:16:40.865 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:16:40.866 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:16:40.967 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:16:41.007 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:16:41.010 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:16:41.011 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:16:41.112 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:16:41.143 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:16:41.145 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:16:41.145 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:16:41.246 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:16:41.284 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:16:41.288 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:16:41.288 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:16:41.389 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:16:41.414 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:16:41.416 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:16:41.417 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:16:41.518 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:16:41.548 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:16:41.551 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:16:41.551 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:16:41.652 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:16:41.695 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:16:41.697 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:16:41.698 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:16:41.799 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:16:41.829 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:16:41.831 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:16:41.832 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:16:41.933 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:16:41.945 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:16:41 smithi164 bash[59462]: cluster 1685823400.8249831 mgr.y (mgr.34104) 1220 : cluster 0 pgmap v1206: 73 pgs: 20 active+clean+remapped, 6 active+undersized+degraded, 11 active+undersized, 36 active+clean; 582 KiB data, 3.2 GiB used, 533 GiB / 536 GiB avail; 8/72 objects degraded (11.111%); 6/72 objects misplaced (8.333%) 2023-06-03T20:16:41.969 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:16:41.973 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:16:41.974 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:16:42.074 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:16:42.106 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:16:42.109 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:16:42.110 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:16:42.132 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:16:41 smithi089 bash[78452]: cluster 1685823400.8249831 mgr.y (mgr.34104) 1220 : cluster 0 pgmap v1206: 73 pgs: 20 active+clean+remapped, 6 active+undersized+degraded, 11 active+undersized, 36 active+clean; 582 KiB data, 3.2 GiB used, 533 GiB / 536 GiB avail; 8/72 objects degraded (11.111%); 6/72 objects misplaced (8.333%) 2023-06-03T20:16:42.133 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:16:41 smithi089 bash[75068]: cluster 1685823400.8249831 mgr.y (mgr.34104) 1220 : cluster 0 pgmap v1206: 73 pgs: 20 active+clean+remapped, 6 active+undersized+degraded, 11 active+undersized, 36 active+clean; 582 KiB data, 3.2 GiB used, 533 GiB / 536 GiB avail; 8/72 objects degraded (11.111%); 6/72 objects misplaced (8.333%) 2023-06-03T20:16:42.211 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:16:42.239 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:16:42.241 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:16:42.242 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:16:42.256 INFO:teuthology.orchestra.run.smithi089.stdout:{ 2023-06-03T20:16:42.256 INFO:teuthology.orchestra.run.smithi089.stdout: "ops": [], 2023-06-03T20:16:42.256 INFO:teuthology.orchestra.run.smithi089.stdout: "complaint_time": 30, 2023-06-03T20:16:42.256 INFO:teuthology.orchestra.run.smithi089.stdout: "num_blocked_ops": 0 2023-06-03T20:16:42.256 INFO:teuthology.orchestra.run.smithi089.stdout:} 2023-06-03T20:16:42.343 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:16:42.366 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:16:42.369 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:16:42.369 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:16:42.470 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:16:42.506 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:16:42.509 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:16:42.509 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:16:42.610 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:16:42.642 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:16:42.645 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:16:42.645 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:16:42.746 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:16:42.786 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:16:42.788 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:16:42.789 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:16:42.890 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:16:42.926 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:16:42.929 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:16:42.930 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:16:43.030 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:16:43.071 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:16:43.074 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:16:43.074 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:16:43.175 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:16:43.209 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:16:43.211 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:16:43.212 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:16:43.313 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:16:43.339 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:16:43.341 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:16:43.342 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:16:43.443 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:16:43.482 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:16:43.486 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:16:43.487 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:16:43.588 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:16:43.627 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:16:43.630 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:16:43.630 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:16:43.731 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:16:43.774 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:16:43.778 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:16:43.778 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:16:43.879 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:16:43.905 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:16:43.907 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:16:43.907 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:16:44.008 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:16:44.021 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:16:43 smithi164 bash[59462]: cluster 1685823402.8255508 mgr.y (mgr.34104) 1221 : cluster 0 pgmap v1207: 73 pgs: 20 active+clean+remapped, 6 active+undersized+degraded, 11 active+undersized, 36 active+clean; 582 KiB data, 3.2 GiB used, 533 GiB / 536 GiB avail; 8/72 objects degraded (11.111%); 6/72 objects misplaced (8.333%) 2023-06-03T20:16:44.043 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:16:44.046 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:16:44.046 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:16:44.147 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:16:44.180 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:16:44.182 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:16:44.182 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:16:44.240 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:16:43 smithi089 bash[75068]: cluster 1685823402.8255508 mgr.y (mgr.34104) 1221 : cluster 0 pgmap v1207: 73 pgs: 20 active+clean+remapped, 6 active+undersized+degraded, 11 active+undersized, 36 active+clean; 582 KiB data, 3.2 GiB used, 533 GiB / 536 GiB avail; 8/72 objects degraded (11.111%); 6/72 objects misplaced (8.333%) 2023-06-03T20:16:44.240 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:16:43 smithi089 bash[78452]: cluster 1685823402.8255508 mgr.y (mgr.34104) 1221 : cluster 0 pgmap v1207: 73 pgs: 20 active+clean+remapped, 6 active+undersized+degraded, 11 active+undersized, 36 active+clean; 582 KiB data, 3.2 GiB used, 533 GiB / 536 GiB avail; 8/72 objects degraded (11.111%); 6/72 objects misplaced (8.333%) 2023-06-03T20:16:44.283 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:16:44.323 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:16:44.325 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:16:44.326 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:16:44.427 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:16:44.465 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:16:44.468 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:16:44.468 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:16:44.569 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:16:44.603 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:16:44.606 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:16:44.607 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:16:44.707 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:16:44.746 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:16:44.750 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:16:44.751 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:16:44.851 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:16:44.888 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:16:44.892 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:16:44.893 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:16:44.994 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:16:45.033 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:16:45.035 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:16:45.037 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:16:45.138 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:16:45.169 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:16:45.171 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:16:45.187 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:16:45.288 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:16:45.316 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:16:45.318 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:16:45.319 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:16:45.419 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:16:45.451 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:16:45.454 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:16:45.455 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:16:45.556 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:16:45.582 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:16:45.583 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:16:45.584 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:16:45.685 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:16:45.722 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:16:45.725 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:16:45.725 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:16:45.782 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph daemon osd.2 dump_historic_ops 2023-06-03T20:16:45.826 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:16:45.863 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:16:45.866 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:16:45.867 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:16:45.968 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:16:45.995 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:16:45.997 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:16:45.998 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:16:46.099 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:16:46.122 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:16:46.123 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:16:46.124 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:16:46.178 INFO:teuthology.orchestra.run.smithi089.stdout:osd.0: {} 2023-06-03T20:16:46.192 INFO:teuthology.orchestra.run.smithi089.stderr:pool 'unique_pool_7' created 2023-06-03T20:16:46.206 INFO:teuthology.orchestra.run.smithi089.stdout:osd.1: {} 2023-06-03T20:16:46.225 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:16:46.228 INFO:teuthology.orchestra.run.smithi089.stdout:osd.2: {} 2023-06-03T20:16:46.246 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:16:46.248 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:16:46.249 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:16:46.249 INFO:teuthology.orchestra.run.smithi089.stdout:osd.3: {} 2023-06-03T20:16:46.271 INFO:teuthology.orchestra.run.smithi089.stdout:osd.5: {} 2023-06-03T20:16:46.294 INFO:teuthology.orchestra.run.smithi089.stdout:osd.6: {} 2023-06-03T20:16:46.314 INFO:teuthology.orchestra.run.smithi089.stdout:osd.7: {} 2023-06-03T20:16:46.315 INFO:teuthology.orchestra.run.smithi089.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-06-03T20:16:46.315 INFO:teuthology.orchestra.run.smithi089.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-06-03T20:16:46.315 INFO:teuthology.orchestra.run.smithi089.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-06-03T20:16:46.315 INFO:teuthology.orchestra.run.smithi089.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-06-03T20:16:46.315 INFO:teuthology.orchestra.run.smithi089.stderr:Error ENXIO: problem getting command descriptions from osd.4 2023-06-03T20:16:46.315 INFO:teuthology.orchestra.run.smithi089.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-06-03T20:16:46.315 INFO:teuthology.orchestra.run.smithi089.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-06-03T20:16:46.316 INFO:teuthology.orchestra.run.smithi089.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-06-03T20:16:46.321 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:16:46 smithi089 bash[75068]: cluster 1685823404.826329 mgr.y (mgr.34104) 1222 : cluster 0 pgmap v1208: 73 pgs: 20 active+clean+remapped, 6 active+undersized+degraded, 11 active+undersized, 36 active+clean; 582 KiB data, 3.2 GiB used, 533 GiB / 536 GiB avail; 8/72 objects degraded (11.111%); 6/72 objects misplaced (8.333%) 2023-06-03T20:16:46.322 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:16:46 smithi089 bash[78452]: cluster 1685823404.826329 mgr.y (mgr.34104) 1222 : cluster 0 pgmap v1208: 73 pgs: 20 active+clean+remapped, 6 active+undersized+degraded, 11 active+undersized, 36 active+clean; 582 KiB data, 3.2 GiB used, 533 GiB / 536 GiB avail; 8/72 objects degraded (11.111%); 6/72 objects misplaced (8.333%) 2023-06-03T20:16:46.322 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 20:16:46 smithi089 bash[71163]: ::ffff:172.21.15.164 - - [03/Jun/2023:20:16:46] "GET /metrics HTTP/1.1" 200 226432 "" "Prometheus/2.33.4" 2023-06-03T20:16:46.350 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:16:46.373 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:16:46.375 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:16:46.375 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:16:46.476 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:16:46.489 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:16:46 smithi164 bash[59462]: cluster 1685823404.826329 mgr.y (mgr.34104) 1222 : cluster 0 pgmap v1208: 73 pgs: 20 active+clean+remapped, 6 active+undersized+degraded, 11 active+undersized, 36 active+clean; 582 KiB data, 3.2 GiB used, 533 GiB / 536 GiB avail; 8/72 objects degraded (11.111%); 6/72 objects misplaced (8.333%) 2023-06-03T20:16:46.513 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:16:46.516 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:16:46.517 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:16:46.618 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:16:46.657 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:16:46.659 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:16:46.660 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:16:46.761 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:16:46.786 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:16:46.789 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:16:46.790 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:16:46.891 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:16:46.920 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:16:46.922 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:16:46.923 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:16:47.024 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:16:47.050 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:16:47.053 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:16:47.053 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:16:47.154 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:16:47.176 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:16:47.178 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:16:47.179 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:16:47.280 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:16:47.293 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:16:47 smithi089 bash[75068]: audit 1685823405.6771133 mon.a (mon.0) 1163 : audit 1 from='client.? 172.21.15.89:0/1652177811' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_7", "pg_num": 16}]: dispatch 2023-06-03T20:16:47.293 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:16:47 smithi089 bash[75068]: audit 1685823406.1499145 mon.a (mon.0) 1164 : audit 0 from='client.? 172.21.15.89:0/2503129055' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-03T20:16:47.293 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:16:47 smithi089 bash[75068]: audit 1685823406.1787128 mon.a (mon.0) 1165 : audit 1 from='client.? 172.21.15.89:0/1652177811' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_7", "pg_num": 16}]': finished 2023-06-03T20:16:47.294 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:16:47 smithi089 bash[75068]: cluster 1685823406.1787853 mon.a (mon.0) 1166 : cluster 0 osdmap e217: 8 total, 7 up, 6 in 2023-06-03T20:16:47.294 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:16:47 smithi089 bash[75068]: cluster 1685823406.1798422 osd.5 (osd.5) 150 : cluster 3 failed to encode map e217 with expected crc 2023-06-03T20:16:47.294 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:16:47 smithi089 bash[75068]: cluster 1685823406.1824656 osd.5 (osd.5) 151 : cluster 3 failed to encode map e217 with expected crc 2023-06-03T20:16:47.294 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:16:47 smithi089 bash[78452]: audit 1685823405.6771133 mon.a (mon.0) 1163 : audit 1 from='client.? 172.21.15.89:0/1652177811' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_7", "pg_num": 16}]: dispatch 2023-06-03T20:16:47.294 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:16:47 smithi089 bash[78452]: audit 1685823406.1499145 mon.a (mon.0) 1164 : audit 0 from='client.? 172.21.15.89:0/2503129055' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-03T20:16:47.295 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:16:47 smithi089 bash[78452]: audit 1685823406.1787128 mon.a (mon.0) 1165 : audit 1 from='client.? 172.21.15.89:0/1652177811' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_7", "pg_num": 16}]': finished 2023-06-03T20:16:47.295 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:16:47 smithi089 bash[78452]: cluster 1685823406.1787853 mon.a (mon.0) 1166 : cluster 0 osdmap e217: 8 total, 7 up, 6 in 2023-06-03T20:16:47.295 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:16:47 smithi089 bash[78452]: cluster 1685823406.1798422 osd.5 (osd.5) 150 : cluster 3 failed to encode map e217 with expected crc 2023-06-03T20:16:47.295 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:16:47 smithi089 bash[78452]: cluster 1685823406.1824656 osd.5 (osd.5) 151 : cluster 3 failed to encode map e217 with expected crc 2023-06-03T20:16:47.311 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:16:47.313 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:16:47.314 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:16:47.415 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:16:47.435 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:16:47.437 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:16:47.437 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:16:47.538 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:16:47.559 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:16:47.562 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:16:47.562 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:16:47.583 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:16:47 smithi164 bash[59462]: audit 1685823405.6771133 mon.a (mon.0) 1163 : audit 1 from='client.? 172.21.15.89:0/1652177811' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_7", "pg_num": 16}]: dispatch 2023-06-03T20:16:47.584 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:16:47 smithi164 bash[59462]: audit 1685823406.1499145 mon.a (mon.0) 1164 : audit 0 from='client.? 172.21.15.89:0/2503129055' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-03T20:16:47.584 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:16:47 smithi164 bash[59462]: audit 1685823406.1787128 mon.a (mon.0) 1165 : audit 1 from='client.? 172.21.15.89:0/1652177811' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_7", "pg_num": 16}]': finished 2023-06-03T20:16:47.584 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:16:47 smithi164 bash[59462]: cluster 1685823406.1787853 mon.a (mon.0) 1166 : cluster 0 osdmap e217: 8 total, 7 up, 6 in 2023-06-03T20:16:47.584 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:16:47 smithi164 bash[59462]: cluster 1685823406.1798422 osd.5 (osd.5) 150 : cluster 3 failed to encode map e217 with expected crc 2023-06-03T20:16:47.584 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:16:47 smithi164 bash[59462]: cluster 1685823406.1824656 osd.5 (osd.5) 151 : cluster 3 failed to encode map e217 with expected crc 2023-06-03T20:16:47.663 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:16:47.685 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:16:47.687 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:16:47.688 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:16:47.789 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:16:47.824 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:16:47.826 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:16:47.827 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:16:47.928 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:16:47.963 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:16:47.965 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:16:47.965 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:16:48.066 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:16:48.106 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:16:48.108 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:16:48.109 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:16:48.203 INFO:teuthology.orchestra.run.smithi089.stderr:noscrub is unset 2023-06-03T20:16:48.209 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:16:48.233 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:16:48.235 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:16:48.236 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:16:48.334 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:16:48 smithi164 bash[59462]: cluster 1685823406.1766024 osd.7 (osd.7) 142 : cluster 3 failed to encode map e217 with expected crc 2023-06-03T20:16:48.334 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:16:48 smithi164 bash[59462]: cluster 1685823406.1799984 osd.7 (osd.7) 143 : cluster 3 failed to encode map e217 with expected crc 2023-06-03T20:16:48.334 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:16:48 smithi164 bash[59462]: cluster 1685823406.8274503 mgr.y (mgr.34104) 1223 : cluster 0 pgmap v1210: 89 pgs: 4 creating+peering, 12 unknown, 20 active+clean+remapped, 6 active+undersized+degraded, 11 active+undersized, 36 active+clean; 582 KiB data, 3.2 GiB used, 533 GiB / 536 GiB avail; 8/72 objects degraded (11.111%); 6/72 objects misplaced (8.333%) 2023-06-03T20:16:48.334 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:16:48 smithi164 bash[59462]: cluster 1685823407.178175 osd.5 (osd.5) 152 : cluster 3 failed to encode map e218 with expected crc 2023-06-03T20:16:48.334 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:16:48 smithi164 bash[59462]: cluster 1685823407.1813223 osd.5 (osd.5) 153 : cluster 3 failed to encode map e218 with expected crc 2023-06-03T20:16:48.334 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:16:48 smithi164 bash[59462]: cluster 1685823407.1825697 mon.a (mon.0) 1167 : cluster 0 osdmap e218: 8 total, 7 up, 6 in 2023-06-03T20:16:48.335 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:16:48 smithi164 bash[59462]: cluster 1685823407.3328016 osd.6 (osd.6) 161 : cluster 3 failed to encode map e218 with expected crc 2023-06-03T20:16:48.335 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:16:48 smithi164 bash[59462]: audit 1685823407.4531422 mon.a (mon.0) 1168 : audit 1 from='client.? 172.21.15.89:0/2025494477' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-06-03T20:16:48.337 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:16:48.364 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:16:48.366 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:16:48.367 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:16:48.467 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:16:48.503 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:16:48.506 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:16:48.506 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:16:48.531 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:16:48 smithi089 bash[78452]: cluster 1685823406.1766024 osd.7 (osd.7) 142 : cluster 3 failed to encode map e217 with expected crc 2023-06-03T20:16:48.531 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:16:48 smithi089 bash[78452]: cluster 1685823406.1799984 osd.7 (osd.7) 143 : cluster 3 failed to encode map e217 with expected crc 2023-06-03T20:16:48.531 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:16:48 smithi089 bash[78452]: cluster 1685823406.8274503 mgr.y (mgr.34104) 1223 : cluster 0 pgmap v1210: 89 pgs: 4 creating+peering, 12 unknown, 20 active+clean+remapped, 6 active+undersized+degraded, 11 active+undersized, 36 active+clean; 582 KiB data, 3.2 GiB used, 533 GiB / 536 GiB avail; 8/72 objects degraded (11.111%); 6/72 objects misplaced (8.333%) 2023-06-03T20:16:48.532 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:16:48 smithi089 bash[78452]: cluster 1685823407.178175 osd.5 (osd.5) 152 : cluster 3 failed to encode map e218 with expected crc 2023-06-03T20:16:48.532 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:16:48 smithi089 bash[78452]: cluster 1685823407.1813223 osd.5 (osd.5) 153 : cluster 3 failed to encode map e218 with expected crc 2023-06-03T20:16:48.532 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:16:48 smithi089 bash[78452]: cluster 1685823407.1825697 mon.a (mon.0) 1167 : cluster 0 osdmap e218: 8 total, 7 up, 6 in 2023-06-03T20:16:48.532 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:16:48 smithi089 bash[78452]: cluster 1685823407.3328016 osd.6 (osd.6) 161 : cluster 3 failed to encode map e218 with expected crc 2023-06-03T20:16:48.532 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:16:48 smithi089 bash[78452]: audit 1685823407.4531422 mon.a (mon.0) 1168 : audit 1 from='client.? 172.21.15.89:0/2025494477' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-06-03T20:16:48.532 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:16:48 smithi089 bash[75068]: cluster 1685823406.1766024 osd.7 (osd.7) 142 : cluster 3 failed to encode map e217 with expected crc 2023-06-03T20:16:48.533 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:16:48 smithi089 bash[75068]: cluster 1685823406.1799984 osd.7 (osd.7) 143 : cluster 3 failed to encode map e217 with expected crc 2023-06-03T20:16:48.533 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:16:48 smithi089 bash[75068]: cluster 1685823406.8274503 mgr.y (mgr.34104) 1223 : cluster 0 pgmap v1210: 89 pgs: 4 creating+peering, 12 unknown, 20 active+clean+remapped, 6 active+undersized+degraded, 11 active+undersized, 36 active+clean; 582 KiB data, 3.2 GiB used, 533 GiB / 536 GiB avail; 8/72 objects degraded (11.111%); 6/72 objects misplaced (8.333%) 2023-06-03T20:16:48.533 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:16:48 smithi089 bash[75068]: cluster 1685823407.178175 osd.5 (osd.5) 152 : cluster 3 failed to encode map e218 with expected crc 2023-06-03T20:16:48.533 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:16:48 smithi089 bash[75068]: cluster 1685823407.1813223 osd.5 (osd.5) 153 : cluster 3 failed to encode map e218 with expected crc 2023-06-03T20:16:48.533 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:16:48 smithi089 bash[75068]: cluster 1685823407.1825697 mon.a (mon.0) 1167 : cluster 0 osdmap e218: 8 total, 7 up, 6 in 2023-06-03T20:16:48.533 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:16:48 smithi089 bash[75068]: cluster 1685823407.3328016 osd.6 (osd.6) 161 : cluster 3 failed to encode map e218 with expected crc 2023-06-03T20:16:48.533 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:16:48 smithi089 bash[75068]: audit 1685823407.4531422 mon.a (mon.0) 1168 : audit 1 from='client.? 172.21.15.89:0/2025494477' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-06-03T20:16:48.607 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:16:48.635 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:16:48.639 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:16:48.640 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:16:48.741 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:16:48.771 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:16:48.773 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:16:48.774 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:16:48.874 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:16:48.909 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:16:48.911 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:16:48.911 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:16:49.012 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:16:49.044 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:16:49.048 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:16:49.048 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:16:49.149 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:16:49.186 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:16:49.190 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:16:49.191 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:16:49.279 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:16:49 smithi089 bash[78452]: cluster 1685823407.1810992 osd.7 (osd.7) 144 : cluster 3 failed to encode map e218 with expected crc 2023-06-03T20:16:49.279 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:16:49 smithi089 bash[78452]: cluster 1685823408.1867032 mon.a (mon.0) 1169 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-06-03T20:16:49.280 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:16:49 smithi089 bash[78452]: audit 1685823408.1938415 mon.a (mon.0) 1170 : audit 1 from='client.? 172.21.15.89:0/2025494477' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-06-03T20:16:49.280 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:16:49 smithi089 bash[78452]: cluster 1685823408.1938996 mon.a (mon.0) 1171 : cluster 0 osdmap e219: 8 total, 7 up, 6 in 2023-06-03T20:16:49.280 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:16:49 smithi089 bash[78452]: cluster 1685823408.454705 osd.5 (osd.5) 154 : cluster 3 failed to encode map e219 with expected crc 2023-06-03T20:16:49.280 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:16:49 smithi089 bash[78452]: cluster 1685823408.5017488 osd.6 (osd.6) 162 : cluster 3 failed to encode map e219 with expected crc 2023-06-03T20:16:49.280 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:16:49 smithi089 bash[75068]: cluster 1685823407.1810992 osd.7 (osd.7) 144 : cluster 3 failed to encode map e218 with expected crc 2023-06-03T20:16:49.281 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:16:49 smithi089 bash[75068]: cluster 1685823408.1867032 mon.a (mon.0) 1169 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-06-03T20:16:49.281 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:16:49 smithi089 bash[75068]: audit 1685823408.1938415 mon.a (mon.0) 1170 : audit 1 from='client.? 172.21.15.89:0/2025494477' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-06-03T20:16:49.281 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:16:49 smithi089 bash[75068]: cluster 1685823408.1938996 mon.a (mon.0) 1171 : cluster 0 osdmap e219: 8 total, 7 up, 6 in 2023-06-03T20:16:49.281 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:16:49 smithi089 bash[75068]: cluster 1685823408.454705 osd.5 (osd.5) 154 : cluster 3 failed to encode map e219 with expected crc 2023-06-03T20:16:49.281 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:16:49 smithi089 bash[75068]: cluster 1685823408.5017488 osd.6 (osd.6) 162 : cluster 3 failed to encode map e219 with expected crc 2023-06-03T20:16:49.292 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:16:49.324 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:16:49.327 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:16:49.328 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:16:49.428 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:16:49.465 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:16:49.468 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:16:49.469 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:16:49.569 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:16:49.583 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:16:49 smithi164 bash[59462]: cluster 1685823407.1810992 osd.7 (osd.7) 144 : cluster 3 failed to encode map e218 with expected crc 2023-06-03T20:16:49.584 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:16:49 smithi164 bash[59462]: cluster 1685823408.1867032 mon.a (mon.0) 1169 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-06-03T20:16:49.584 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:16:49 smithi164 bash[59462]: audit 1685823408.1938415 mon.a (mon.0) 1170 : audit 1 from='client.? 172.21.15.89:0/2025494477' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-06-03T20:16:49.584 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:16:49 smithi164 bash[59462]: cluster 1685823408.1938996 mon.a (mon.0) 1171 : cluster 0 osdmap e219: 8 total, 7 up, 6 in 2023-06-03T20:16:49.585 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:16:49 smithi164 bash[59462]: cluster 1685823408.454705 osd.5 (osd.5) 154 : cluster 3 failed to encode map e219 with expected crc 2023-06-03T20:16:49.585 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:16:49 smithi164 bash[59462]: cluster 1685823408.5017488 osd.6 (osd.6) 162 : cluster 3 failed to encode map e219 with expected crc 2023-06-03T20:16:49.605 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:16:49.608 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:16:49.609 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:16:49.710 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:16:49.743 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:16:49.745 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:16:49.746 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:16:49.847 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:16:49.880 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:16:49.883 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:16:49.883 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:16:49.984 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:16:50.018 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:16:50.021 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:16:50.022 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:16:50.123 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:16:50.156 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:16:50.159 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:16:50.160 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:16:50.261 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:16:50.299 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:16:50.301 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:16:50.301 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:16:50.321 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:16:50 smithi089 bash[75068]: cluster 1685823408.1881018 osd.7 (osd.7) 145 : cluster 3 failed to encode map e219 with expected crc 2023-06-03T20:16:50.321 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:16:50 smithi089 bash[75068]: cluster 1685823408.8288639 mgr.y (mgr.34104) 1224 : cluster 0 pgmap v1213: 89 pgs: 4 creating+peering, 4 unknown, 20 active+clean+remapped, 6 active+undersized+degraded, 16 active+undersized, 39 active+clean; 582 KiB data, 3.2 GiB used, 533 GiB / 536 GiB avail; 8/72 objects degraded (11.111%); 6/72 objects misplaced (8.333%) 2023-06-03T20:16:50.403 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:16:50.437 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:16:50.439 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:16:50.440 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:16:50.541 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:16:50.553 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:16:50 smithi164 bash[59462]: cluster 1685823408.1881018 osd.7 (osd.7) 145 : cluster 3 failed to encode map e219 with expected crc 2023-06-03T20:16:50.554 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:16:50 smithi164 bash[59462]: cluster 1685823408.8288639 mgr.y (mgr.34104) 1224 : cluster 0 pgmap v1213: 89 pgs: 4 creating+peering, 4 unknown, 20 active+clean+remapped, 6 active+undersized+degraded, 16 active+undersized, 39 active+clean; 582 KiB data, 3.2 GiB used, 533 GiB / 536 GiB avail; 8/72 objects degraded (11.111%); 6/72 objects misplaced (8.333%) 2023-06-03T20:16:50.574 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:16:50.576 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:16:50.577 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:16:50.678 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:16:50.691 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:16:50 smithi089 bash[78452]: cluster 1685823408.1881018 osd.7 (osd.7) 145 : cluster 3 failed to encode map e219 with expected crc 2023-06-03T20:16:50.691 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:16:50 smithi089 bash[78452]: cluster 1685823408.8288639 mgr.y (mgr.34104) 1224 : cluster 0 pgmap v1213: 89 pgs: 4 creating+peering, 4 unknown, 20 active+clean+remapped, 6 active+undersized+degraded, 16 active+undersized, 39 active+clean; 582 KiB data, 3.2 GiB used, 533 GiB / 536 GiB avail; 8/72 objects degraded (11.111%); 6/72 objects misplaced (8.333%) 2023-06-03T20:16:50.714 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:16:50.717 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:16:50.717 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:16:50.818 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:16:50.857 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:16:50.860 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:16:50.861 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:16:50.911 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph osd pool application enable unique_pool_7 rados --yes-i-really-mean-it || true 2023-06-03T20:16:50.961 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:16:51.002 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:16:51.004 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:16:51.005 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:16:51.106 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:16:51.137 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:16:51.140 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:16:51.141 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:16:51.241 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:16:51.274 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:16:51.276 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:16:51.277 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:16:51.378 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:16:51.413 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:16:51.417 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:16:51.417 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:16:51.518 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:16:51.552 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:16:51.553 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:16:51.554 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:16:51.655 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:16:51.689 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:16:51.691 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:16:51.692 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:16:51.793 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:16:51.823 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:16:51.825 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:16:51.826 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:16:51.927 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:16:51.955 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-06-03T20:16:51.956 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-06-03T20:16:51.957 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:16:51.958 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:16:51.959 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:16:52.060 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:16:52.094 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:16:52.098 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:16:52.099 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:16:52.200 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:16:52.237 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:16:52.239 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:16:52.239 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:16:52.341 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:16:52.352 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:16:52 smithi089 bash[75068]: cluster 1685823410.829927 mgr.y (mgr.34104) 1225 : cluster 0 pgmap v1214: 89 pgs: 4 creating+peering, 20 active+clean+remapped, 6 active+undersized+degraded, 18 active+undersized, 41 active+clean; 582 KiB data, 3.2 GiB used, 533 GiB / 536 GiB avail; 8/72 objects degraded (11.111%); 6/72 objects misplaced (8.333%) 2023-06-03T20:16:52.353 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:16:52 smithi089 bash[75068]: audit 1685823412.1272743 mon.c (mon.2) 543 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T20:16:52.353 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:16:52 smithi089 bash[78452]: cluster 1685823410.829927 mgr.y (mgr.34104) 1225 : cluster 0 pgmap v1214: 89 pgs: 4 creating+peering, 20 active+clean+remapped, 6 active+undersized+degraded, 18 active+undersized, 41 active+clean; 582 KiB data, 3.2 GiB used, 533 GiB / 536 GiB avail; 8/72 objects degraded (11.111%); 6/72 objects misplaced (8.333%) 2023-06-03T20:16:52.353 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:16:52 smithi089 bash[78452]: audit 1685823412.1272743 mon.c (mon.2) 543 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T20:16:52.378 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:16:52.381 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:16:52.382 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:16:52.483 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:16:52.496 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:16:52 smithi164 bash[59462]: cluster 1685823410.829927 mgr.y (mgr.34104) 1225 : cluster 0 pgmap v1214: 89 pgs: 4 creating+peering, 20 active+clean+remapped, 6 active+undersized+degraded, 18 active+undersized, 41 active+clean; 582 KiB data, 3.2 GiB used, 533 GiB / 536 GiB avail; 8/72 objects degraded (11.111%); 6/72 objects misplaced (8.333%) 2023-06-03T20:16:52.497 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:16:52 smithi164 bash[59462]: audit 1685823412.1272743 mon.c (mon.2) 543 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T20:16:52.522 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:16:52.525 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:16:52.526 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:16:52.627 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:16:52.670 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:16:52.672 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:16:52.673 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:16:52.771 INFO:teuthology.orchestra.run.smithi089.stdout:{ 2023-06-03T20:16:52.772 INFO:teuthology.orchestra.run.smithi089.stdout: "success": "" 2023-06-03T20:16:52.772 INFO:teuthology.orchestra.run.smithi089.stdout:} 2023-06-03T20:16:52.775 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:16:52.815 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:16:52.818 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:16:52.819 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:16:52.919 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:16:52.946 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:16:52.948 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:16:52.949 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:16:53.050 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:16:53.086 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:16:53.089 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:16:53.089 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:16:53.190 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:16:53.232 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:16:53.235 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:16:53.235 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:16:53.336 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:16:53.371 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:16:53.373 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:16:53.374 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:16:53.475 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:16:53.515 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:16:53.518 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:16:53.519 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:16:53.620 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:16:53.656 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:16:53.659 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:16:53.660 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:16:53.761 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:16:53.794 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:16:53.796 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:16:53.797 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:16:53.898 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:16:53.931 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:16:53.933 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:16:53.934 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:16:53.957 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-06-03T20:16:54.029 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:16:53 smithi089 bash[78452]: cluster 1685823412.8310065 mgr.y (mgr.34104) 1226 : cluster 0 pgmap v1215: 89 pgs: 20 active+clean+remapped, 6 active+undersized+degraded, 19 active+undersized, 44 active+clean; 582 KiB data, 3.2 GiB used, 533 GiB / 536 GiB avail; 8/72 objects degraded (11.111%); 6/72 objects misplaced (8.333%) 2023-06-03T20:16:54.029 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:16:53 smithi089 bash[75068]: cluster 1685823412.8310065 mgr.y (mgr.34104) 1226 : cluster 0 pgmap v1215: 89 pgs: 20 active+clean+remapped, 6 active+undersized+degraded, 19 active+undersized, 44 active+clean; 582 KiB data, 3.2 GiB used, 533 GiB / 536 GiB avail; 8/72 objects degraded (11.111%); 6/72 objects misplaced (8.333%) 2023-06-03T20:16:54.035 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:16:54.064 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:16:54.067 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:16:54.068 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:16:54.083 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:16:53 smithi164 bash[59462]: cluster 1685823412.8310065 mgr.y (mgr.34104) 1226 : cluster 0 pgmap v1215: 89 pgs: 20 active+clean+remapped, 6 active+undersized+degraded, 19 active+undersized, 44 active+clean; 582 KiB data, 3.2 GiB used, 533 GiB / 536 GiB avail; 8/72 objects degraded (11.111%); 6/72 objects misplaced (8.333%) 2023-06-03T20:16:54.122 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph osd unset nodeep-scrub 2023-06-03T20:16:54.169 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:16:54.197 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:16:54.200 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:16:54.201 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:16:54.302 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:16:54.334 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:16:54.336 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:16:54.337 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:16:54.438 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:16:54.475 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:16:54.479 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:16:54.479 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:16:54.580 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:16:54.616 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:16:54.619 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:16:54.620 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:16:54.721 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:16:54.760 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:16:54.764 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:16:54.764 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:16:54.865 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:16:54.901 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:16:54.904 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:16:54.904 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:16:55.005 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:16:55.041 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:16:55.044 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:16:55.044 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:16:55.145 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:16:55.180 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:16:55.183 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:16:55.184 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:16:55.285 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:16:55.325 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:16:55.329 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:16:55.330 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:16:55.428 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/mon.a/config 2023-06-03T20:16:55.430 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:16:55.472 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:16:55.475 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:16:55.476 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:16:55.577 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:16:55.606 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:16:55.609 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:16:55.609 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:16:55.711 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:16:55.751 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:16:55.755 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:16:55.757 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:16:55.858 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:16:55.907 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:16:55.911 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:16:55.911 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:16:56.012 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:16:56.029 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:16:55 smithi089 bash[75068]: cluster 1685823414.8324027 mgr.y (mgr.34104) 1227 : cluster 0 pgmap v1216: 89 pgs: 20 active+clean+remapped, 6 active+undersized+degraded, 19 active+undersized, 44 active+clean; 582 KiB data, 3.2 GiB used, 533 GiB / 536 GiB avail; 8/72 objects degraded (11.111%); 6/72 objects misplaced (8.333%) 2023-06-03T20:16:56.029 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:16:55 smithi089 bash[78452]: cluster 1685823414.8324027 mgr.y (mgr.34104) 1227 : cluster 0 pgmap v1216: 89 pgs: 20 active+clean+remapped, 6 active+undersized+degraded, 19 active+undersized, 44 active+clean; 582 KiB data, 3.2 GiB used, 533 GiB / 536 GiB avail; 8/72 objects degraded (11.111%); 6/72 objects misplaced (8.333%) 2023-06-03T20:16:56.046 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:16:56.049 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:16:56.050 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:16:56.151 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:16:56.162 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:16:55 smithi164 bash[59462]: cluster 1685823414.8324027 mgr.y (mgr.34104) 1227 : cluster 0 pgmap v1216: 89 pgs: 20 active+clean+remapped, 6 active+undersized+degraded, 19 active+undersized, 44 active+clean; 582 KiB data, 3.2 GiB used, 533 GiB / 536 GiB avail; 8/72 objects degraded (11.111%); 6/72 objects misplaced (8.333%) 2023-06-03T20:16:56.183 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:16:56.185 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:16:56.186 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:16:56.287 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:16:56.320 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:16:56.323 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:16:56.323 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:16:56.424 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:16:56.451 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/mon.a/config 2023-06-03T20:16:56.461 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:16:56.465 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:16:56.465 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:16:56.483 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 20:16:56 smithi089 bash[71163]: ::ffff:172.21.15.164 - - [03/Jun/2023:20:16:56] "GET /metrics HTTP/1.1" 200 228782 "" "Prometheus/2.33.4" 2023-06-03T20:16:56.567 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:16:56.596 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:16:56.598 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:16:56.599 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:16:56.700 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:16:56.732 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:16:56.734 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:16:56.735 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:16:56.836 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:16:56.872 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:16:56.875 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:16:56.876 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:16:56.977 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:16:57.011 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:16:57.014 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:16:57.015 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:16:57.116 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:16:57.150 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:16:57.151 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:16:57.152 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:16:57.253 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:16:57.290 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:16:57.292 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:16:57.293 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:16:57.394 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:16:57.422 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:16:57.425 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:16:57.425 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:16:57.526 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:16:57.564 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:16:57.567 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:16:57.568 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:16:57.668 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:16:57.703 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:16:57.705 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:16:57.706 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:16:57.807 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:16:57.833 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:16:57.836 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:16:57.837 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:16:57.938 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:16:57.979 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:16:57.982 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:16:57.983 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:16:58.029 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:16:57 smithi089 bash[78452]: cluster 1685823416.8330917 mgr.y (mgr.34104) 1228 : cluster 0 pgmap v1217: 89 pgs: 20 active+clean+remapped, 6 active+undersized+degraded, 19 active+undersized, 44 active+clean; 582 KiB data, 3.2 GiB used, 533 GiB / 536 GiB avail; 8/72 objects degraded (11.111%); 6/72 objects misplaced (8.333%) 2023-06-03T20:16:58.029 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:16:57 smithi089 bash[75068]: cluster 1685823416.8330917 mgr.y (mgr.34104) 1228 : cluster 0 pgmap v1217: 89 pgs: 20 active+clean+remapped, 6 active+undersized+degraded, 19 active+undersized, 44 active+clean; 582 KiB data, 3.2 GiB used, 533 GiB / 536 GiB avail; 8/72 objects degraded (11.111%); 6/72 objects misplaced (8.333%) 2023-06-03T20:16:58.084 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:16:58.120 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:16:58.122 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:16:58.123 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:16:58.224 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:16:58.236 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:16:57 smithi164 bash[59462]: cluster 1685823416.8330917 mgr.y (mgr.34104) 1228 : cluster 0 pgmap v1217: 89 pgs: 20 active+clean+remapped, 6 active+undersized+degraded, 19 active+undersized, 44 active+clean; 582 KiB data, 3.2 GiB used, 533 GiB / 536 GiB avail; 8/72 objects degraded (11.111%); 6/72 objects misplaced (8.333%) 2023-06-03T20:16:58.259 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:16:58.262 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:16:58.263 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:16:58.364 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:16:58.396 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:16:58.399 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:16:58.400 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:16:58.498 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/mon.a/config 2023-06-03T20:16:58.499 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/mon.a/config 2023-06-03T20:16:58.501 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:16:58.540 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:16:58.543 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:16:58.543 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:16:58.644 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:16:58.682 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:16:58.685 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:16:58.685 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:16:58.786 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:16:58.812 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:16:58.814 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:16:58.814 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:16:58.907 INFO:tasks.thrashosds.thrasher:in_osds: [0, 2, 3, 4, 5, 7] out_osds: [1, 6] dead_osds: [4] live_osds: [0, 1, 2, 3, 5, 6, 7] 2023-06-03T20:16:58.908 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-06-03T20:16:58.908 INFO:tasks.thrashosds.thrasher:primary_affinity 2023-06-03T20:16:58.908 INFO:tasks.thrashosds.thrasher:Setting osd 5 primary_affinity to 0.000000 2023-06-03T20:16:58.908 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph osd primary-affinity 5 0 2023-06-03T20:16:58.916 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:16:58.950 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:16:58.953 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:16:58.954 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:16:59.055 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:16:59.077 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:16:59.078 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:16:59.079 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:16:59.180 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:16:59.210 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:16:59.212 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:16:59.213 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:16:59.314 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:16:59.349 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:16:59.352 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:16:59.352 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:16:59.453 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:16:59.484 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:16:59.487 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:16:59.487 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:16:59.588 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:16:59.617 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:16:59.619 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:16:59.620 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:16:59.721 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:16:59.758 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:16:59.761 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:16:59.761 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:16:59.862 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:16:59.894 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:16:59.898 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:16:59.898 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:16:59.999 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:17:00.030 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:17:00.032 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:17:00.033 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:17:00.083 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:16:59 smithi164 bash[59462]: cluster 1685823418.834432 mgr.y (mgr.34104) 1229 : cluster 0 pgmap v1218: 89 pgs: 20 active+clean+remapped, 6 active+undersized+degraded, 19 active+undersized, 44 active+clean; 582 KiB data, 3.2 GiB used, 533 GiB / 536 GiB avail; 8/72 objects degraded (11.111%); 6/72 objects misplaced (8.333%) 2023-06-03T20:17:00.134 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:17:00.170 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:17:00.173 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:17:00.174 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:17:00.178 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:16:59 smithi089 bash[78452]: cluster 1685823418.834432 mgr.y (mgr.34104) 1229 : cluster 0 pgmap v1218: 89 pgs: 20 active+clean+remapped, 6 active+undersized+degraded, 19 active+undersized, 44 active+clean; 582 KiB data, 3.2 GiB used, 533 GiB / 536 GiB avail; 8/72 objects degraded (11.111%); 6/72 objects misplaced (8.333%) 2023-06-03T20:17:00.178 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:16:59 smithi089 bash[75068]: cluster 1685823418.834432 mgr.y (mgr.34104) 1229 : cluster 0 pgmap v1218: 89 pgs: 20 active+clean+remapped, 6 active+undersized+degraded, 19 active+undersized, 44 active+clean; 582 KiB data, 3.2 GiB used, 533 GiB / 536 GiB avail; 8/72 objects degraded (11.111%); 6/72 objects misplaced (8.333%) 2023-06-03T20:17:00.275 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:17:00.310 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:17:00.313 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:17:00.314 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:17:00.415 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:17:00.457 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:17:00.460 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:17:00.461 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:17:00.562 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:17:00.592 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:17:00.593 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:17:00.594 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:17:00.695 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:17:00.731 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:17:00.734 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:17:00.735 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:17:00.836 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:17:00.862 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:17:00.864 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:17:00.865 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:17:00.966 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:17:01.001 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:17:01.004 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:17:01.004 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:17:01.105 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:17:01.133 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:17:01.135 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:17:01.136 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:17:01.237 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:17:01.267 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:17:01.270 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:17:01.271 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:17:01.371 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:17:01.409 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:17:01.412 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:17:01.413 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:17:01.514 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:17:01.542 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:17:01.545 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:17:01.545 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:17:01.646 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:17:01.685 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:17:01.688 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:17:01.689 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:17:01.790 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:17:01.814 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:17:01.816 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:17:01.816 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:17:01.917 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:17:01.945 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:17:01.948 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:17:01.948 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:17:01.949 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:17:01 smithi164 bash[59462]: cluster 1685823420.8355296 mgr.y (mgr.34104) 1230 : cluster 0 pgmap v1219: 89 pgs: 20 active+clean+remapped, 6 active+undersized+degraded, 19 active+undersized, 44 active+clean; 582 KiB data, 3.2 GiB used, 533 GiB / 536 GiB avail; 8/72 objects degraded (11.111%); 6/72 objects misplaced (8.333%) 2023-06-03T20:17:02.049 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:17:02.086 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:17:02.089 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:17:02.089 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:17:02.190 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:17:02.233 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:17:02.236 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:17:02.237 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:17:02.278 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:17:01 smithi089 bash[75068]: cluster 1685823420.8355296 mgr.y (mgr.34104) 1230 : cluster 0 pgmap v1219: 89 pgs: 20 active+clean+remapped, 6 active+undersized+degraded, 19 active+undersized, 44 active+clean; 582 KiB data, 3.2 GiB used, 533 GiB / 536 GiB avail; 8/72 objects degraded (11.111%); 6/72 objects misplaced (8.333%) 2023-06-03T20:17:02.279 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:17:01 smithi089 bash[78452]: cluster 1685823420.8355296 mgr.y (mgr.34104) 1230 : cluster 0 pgmap v1219: 89 pgs: 20 active+clean+remapped, 6 active+undersized+degraded, 19 active+undersized, 44 active+clean; 582 KiB data, 3.2 GiB used, 533 GiB / 536 GiB avail; 8/72 objects degraded (11.111%); 6/72 objects misplaced (8.333%) 2023-06-03T20:17:02.337 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:17:02.379 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:17:02.382 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:17:02.382 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:17:02.483 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:17:02.524 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:17:02.527 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:17:02.528 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:17:02.628 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:17:02.670 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:17:02.673 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:17:02.674 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:17:02.775 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:17:02.802 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:17:02.805 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:17:02.805 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:17:02.906 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:17:02.937 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:17:02.940 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:17:02.941 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:17:03.041 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:17:03.066 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:17:03.069 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:17:03.069 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:17:03.170 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:17:03.211 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:17:03.213 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:17:03.214 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:17:03.315 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:17:03.342 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:17:03.344 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:17:03.345 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:17:03.446 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:17:03.483 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:17:03.485 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:17:03.486 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:17:03.586 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:17:03.612 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:17:03.614 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:17:03.615 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:17:03.716 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:17:03.741 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:17:03.743 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:17:03.744 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:17:03.845 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:17:03.855 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:17:03 smithi089 bash[75068]: cluster 1685823422.8361342 mgr.y (mgr.34104) 1231 : cluster 0 pgmap v1220: 89 pgs: 20 active+clean+remapped, 6 active+undersized+degraded, 19 active+undersized, 44 active+clean; 582 KiB data, 3.2 GiB used, 533 GiB / 536 GiB avail; 8/72 objects degraded (11.111%); 6/72 objects misplaced (8.333%) 2023-06-03T20:17:03.855 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:17:03 smithi089 bash[78452]: cluster 1685823422.8361342 mgr.y (mgr.34104) 1231 : cluster 0 pgmap v1220: 89 pgs: 20 active+clean+remapped, 6 active+undersized+degraded, 19 active+undersized, 44 active+clean; 582 KiB data, 3.2 GiB used, 533 GiB / 536 GiB avail; 8/72 objects degraded (11.111%); 6/72 objects misplaced (8.333%) 2023-06-03T20:17:03.875 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:17:03.877 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:17:03.878 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:17:03.979 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:17:04.011 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:17:04.013 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:17:04.014 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:17:04.083 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:17:03 smithi164 bash[59462]: cluster 1685823422.8361342 mgr.y (mgr.34104) 1231 : cluster 0 pgmap v1220: 89 pgs: 20 active+clean+remapped, 6 active+undersized+degraded, 19 active+undersized, 44 active+clean; 582 KiB data, 3.2 GiB used, 533 GiB / 536 GiB avail; 8/72 objects degraded (11.111%); 6/72 objects misplaced (8.333%) 2023-06-03T20:17:04.115 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:17:04.145 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:17:04.147 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:17:04.147 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:17:04.248 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:17:04.281 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:17:04.283 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:17:04.284 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:17:04.385 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:17:04.411 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:17:04.413 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:17:04.414 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:17:04.515 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:17:04.546 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:17:04.548 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:17:04.548 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:17:04.649 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:17:04.678 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:17:04.680 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:17:04.681 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:17:04.781 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:17:04.820 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:17:04.823 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:17:04.823 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:17:04.924 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:17:04.955 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:17:04.957 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:17:04.957 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:17:05.058 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:17:05.088 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:17:05.090 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:17:05.090 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:17:05.191 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:17:05.224 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:17:05.226 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:17:05.227 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:17:05.328 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:17:05.356 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:17:05.358 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:17:05.359 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:17:05.459 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:17:05.488 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:17:05.489 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:17:05.490 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:17:05.591 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:17:05.616 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:17:05.618 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:17:05.618 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:17:05.719 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:17:05.746 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:17:05.749 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:17:05.749 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:17:05.850 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:17:05.884 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:17:05.887 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:17:05.887 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:17:05.988 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:17:06.028 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:17:06.029 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:17:05 smithi089 bash[78452]: cluster 1685823424.8371863 mgr.y (mgr.34104) 1232 : cluster 0 pgmap v1221: 89 pgs: 20 active+clean+remapped, 6 active+undersized+degraded, 19 active+undersized, 44 active+clean; 582 KiB data, 3.2 GiB used, 533 GiB / 536 GiB avail; 8/72 objects degraded (11.111%); 6/72 objects misplaced (8.333%) 2023-06-03T20:17:06.029 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:17:05 smithi089 bash[75068]: cluster 1685823424.8371863 mgr.y (mgr.34104) 1232 : cluster 0 pgmap v1221: 89 pgs: 20 active+clean+remapped, 6 active+undersized+degraded, 19 active+undersized, 44 active+clean; 582 KiB data, 3.2 GiB used, 533 GiB / 536 GiB avail; 8/72 objects degraded (11.111%); 6/72 objects misplaced (8.333%) 2023-06-03T20:17:06.031 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:17:06.031 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:17:06.132 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:17:06.159 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:17:06.161 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:17:06.162 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:17:06.263 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:17:06.276 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:17:05 smithi164 bash[59462]: cluster 1685823424.8371863 mgr.y (mgr.34104) 1232 : cluster 0 pgmap v1221: 89 pgs: 20 active+clean+remapped, 6 active+undersized+degraded, 19 active+undersized, 44 active+clean; 582 KiB data, 3.2 GiB used, 533 GiB / 536 GiB avail; 8/72 objects degraded (11.111%); 6/72 objects misplaced (8.333%) 2023-06-03T20:17:06.301 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:17:06.305 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:17:06.306 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:17:06.407 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:17:06.436 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:17:06.437 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:17:06.438 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:17:06.512 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 20:17:06 smithi089 bash[71163]: ::ffff:172.21.15.164 - - [03/Jun/2023:20:17:06] "GET /metrics HTTP/1.1" 200 228782 "" "Prometheus/2.33.4" 2023-06-03T20:17:06.514 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/mon.a/config 2023-06-03T20:17:06.539 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:17:06.579 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:17:06.582 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:17:06.582 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:17:06.683 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:17:06.717 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:17:06.720 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:17:06.721 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:17:06.822 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:17:06.861 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:17:06.863 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:17:06.864 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:17:06.965 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:17:07.004 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:17:07.006 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:17:07.007 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:17:07.108 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:17:07.142 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:17:07.145 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:17:07.145 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:17:07.246 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:17:07.278 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:17:07.280 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:17:07.281 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:17:07.381 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:17:07.412 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:17:07.414 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:17:07.415 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:17:07.516 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:17:07.556 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:17:07.559 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:17:07.560 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:17:07.661 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:17:07.698 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:17:07.701 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:17:07.701 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:17:07.802 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:17:07.838 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:17:07.841 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:17:07.842 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:17:07.943 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:17:07.972 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:17:07.975 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:17:07.975 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:17:07.984 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:17:07 smithi089 bash[78452]: cluster 1685823426.83781 mgr.y (mgr.34104) 1233 : cluster 0 pgmap v1222: 89 pgs: 20 active+clean+remapped, 6 active+undersized+degraded, 19 active+undersized, 44 active+clean; 582 KiB data, 3.2 GiB used, 533 GiB / 536 GiB avail; 8/72 objects degraded (11.111%); 6/72 objects misplaced (8.333%) 2023-06-03T20:17:07.984 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:17:07 smithi089 bash[78452]: audit 1685823427.1279864 mon.c (mon.2) 544 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T20:17:07.984 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:17:07 smithi089 bash[75068]: cluster 1685823426.83781 mgr.y (mgr.34104) 1233 : cluster 0 pgmap v1222: 89 pgs: 20 active+clean+remapped, 6 active+undersized+degraded, 19 active+undersized, 44 active+clean; 582 KiB data, 3.2 GiB used, 533 GiB / 536 GiB avail; 8/72 objects degraded (11.111%); 6/72 objects misplaced (8.333%) 2023-06-03T20:17:07.984 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:17:07 smithi089 bash[75068]: audit 1685823427.1279864 mon.c (mon.2) 544 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T20:17:08.076 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:17:08.083 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:17:07 smithi164 bash[59462]: cluster 1685823426.83781 mgr.y (mgr.34104) 1233 : cluster 0 pgmap v1222: 89 pgs: 20 active+clean+remapped, 6 active+undersized+degraded, 19 active+undersized, 44 active+clean; 582 KiB data, 3.2 GiB used, 533 GiB / 536 GiB avail; 8/72 objects degraded (11.111%); 6/72 objects misplaced (8.333%) 2023-06-03T20:17:08.083 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:17:07 smithi164 bash[59462]: audit 1685823427.1279864 mon.c (mon.2) 544 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T20:17:08.115 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:17:08.117 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:17:08.118 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:17:08.219 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:17:08.248 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:17:08.250 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:17:08.250 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:17:08.351 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:17:08.390 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:17:08.394 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:17:08.394 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:17:08.495 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:17:08.529 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:17:08.532 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:17:08.532 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:17:08.633 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:17:08.663 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:17:08.665 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:17:08.666 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:17:08.767 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:17:08.792 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:17:08.793 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:17:08.794 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:17:08.895 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:17:08.921 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:17:08.924 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:17:08.925 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:17:09.025 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:17:09.053 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:17:09.054 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:17:09.055 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:17:09.156 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:17:09.185 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:17:09.188 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:17:09.188 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:17:09.289 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:17:09.330 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:17:09.333 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:17:09.333 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:17:09.434 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:17:09.474 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:17:09.477 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:17:09.477 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:17:09.578 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:17:09.611 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:17:09.616 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:17:09.617 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:17:09.718 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:17:09.747 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:17:09.748 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:17:09.749 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:17:09.850 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:17:09.875 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:17:09.877 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:17:09.877 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:17:09.978 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:17:10.007 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:17:10.008 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:17:10.009 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:17:10.110 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:17:10.139 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:17:10.141 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:17:10.142 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:17:10.242 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:17:10.267 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:17:10.269 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:17:10.270 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:17:10.333 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:17:09 smithi164 bash[59462]: cluster 1685823428.8386548 mgr.y (mgr.34104) 1234 : cluster 0 pgmap v1223: 89 pgs: 20 active+clean+remapped, 6 active+undersized+degraded, 19 active+undersized, 44 active+clean; 582 KiB data, 3.2 GiB used, 533 GiB / 536 GiB avail; 8/72 objects degraded (11.111%); 6/72 objects misplaced (8.333%) 2023-06-03T20:17:10.337 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:17:09 smithi089 bash[75068]: cluster 1685823428.8386548 mgr.y (mgr.34104) 1234 : cluster 0 pgmap v1223: 89 pgs: 20 active+clean+remapped, 6 active+undersized+degraded, 19 active+undersized, 44 active+clean; 582 KiB data, 3.2 GiB used, 533 GiB / 536 GiB avail; 8/72 objects degraded (11.111%); 6/72 objects misplaced (8.333%) 2023-06-03T20:17:10.338 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:17:09 smithi089 bash[78452]: cluster 1685823428.8386548 mgr.y (mgr.34104) 1234 : cluster 0 pgmap v1223: 89 pgs: 20 active+clean+remapped, 6 active+undersized+degraded, 19 active+undersized, 44 active+clean; 582 KiB data, 3.2 GiB used, 533 GiB / 536 GiB avail; 8/72 objects degraded (11.111%); 6/72 objects misplaced (8.333%) 2023-06-03T20:17:10.371 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:17:10.400 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:17:10.403 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:17:10.403 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:17:10.504 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:17:10.530 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:17:10.532 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:17:10.532 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:17:10.633 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:17:10.674 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:17:10.677 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:17:10.678 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:17:10.779 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:17:10.812 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:17:10.814 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:17:10.815 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:17:10.916 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:17:10.943 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:17:10.946 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:17:10.946 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:17:11.047 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:17:11.076 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:17:11.078 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:17:11.079 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:17:11.179 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:17:11.207 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:17:11.209 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:17:11.209 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:17:11.220 INFO:teuthology.orchestra.run.smithi089.stdout:ERROR: (22) Invalid argument 2023-06-03T20:17:11.221 INFO:teuthology.orchestra.run.smithi089.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-06-03T20:17:11.310 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:17:11.337 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:17:11.341 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:17:11.341 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:17:11.442 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:17:11.481 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:17:11.484 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:17:11.485 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:17:11.586 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:17:11.622 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:17:11.624 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:17:11.624 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:17:11.725 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:17:11.754 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:17:11.757 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:17:11.758 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:17:11.859 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:17:11.893 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:17:11.896 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:17:11.896 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:17:11.917 INFO:teuthology.orchestra.run.smithi089.stderr:enabled application 'rados' on pool 'unique_pool_7' 2023-06-03T20:17:11.997 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:17:12.029 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:17:11 smithi089 bash[78452]: cluster 1685823430.8397825 mgr.y (mgr.34104) 1235 : cluster 0 pgmap v1224: 89 pgs: 20 active+clean+remapped, 6 active+undersized+degraded, 19 active+undersized, 44 active+clean; 582 KiB data, 3.2 GiB used, 533 GiB / 536 GiB avail; 8/72 objects degraded (11.111%); 6/72 objects misplaced (8.333%) 2023-06-03T20:17:12.029 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:17:11 smithi089 bash[78452]: audit 1685823431.5451014 mon.c (mon.2) 545 : audit 1 from='client.? 172.21.15.89:0/744617642' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_7", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-06-03T20:17:12.029 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:17:11 smithi089 bash[78452]: audit 1685823431.545466 mon.a (mon.0) 1172 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_7", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-06-03T20:17:12.030 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:17:11 smithi089 bash[75068]: cluster 1685823430.8397825 mgr.y (mgr.34104) 1235 : cluster 0 pgmap v1224: 89 pgs: 20 active+clean+remapped, 6 active+undersized+degraded, 19 active+undersized, 44 active+clean; 582 KiB data, 3.2 GiB used, 533 GiB / 536 GiB avail; 8/72 objects degraded (11.111%); 6/72 objects misplaced (8.333%) 2023-06-03T20:17:12.030 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:17:11 smithi089 bash[75068]: audit 1685823431.5451014 mon.c (mon.2) 545 : audit 1 from='client.? 172.21.15.89:0/744617642' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_7", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-06-03T20:17:12.030 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:17:11 smithi089 bash[75068]: audit 1685823431.545466 mon.a (mon.0) 1172 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_7", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-06-03T20:17:12.031 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:17:12.035 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:17:12.036 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:17:12.136 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:17:12.171 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:17:12.174 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:17:11 smithi164 bash[59462]: cluster 1685823430.8397825 mgr.y (mgr.34104) 1235 : cluster 0 pgmap v1224: 89 pgs: 20 active+clean+remapped, 6 active+undersized+degraded, 19 active+undersized, 44 active+clean; 582 KiB data, 3.2 GiB used, 533 GiB / 536 GiB avail; 8/72 objects degraded (11.111%); 6/72 objects misplaced (8.333%) 2023-06-03T20:17:12.175 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:17:11 smithi164 bash[59462]: audit 1685823431.5451014 mon.c (mon.2) 545 : audit 1 from='client.? 172.21.15.89:0/744617642' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_7", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-06-03T20:17:12.175 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:17:11 smithi164 bash[59462]: audit 1685823431.545466 mon.a (mon.0) 1172 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_7", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-06-03T20:17:12.175 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:17:12.176 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:17:12.277 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:17:12.314 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:17:12.316 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:17:12.317 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:17:12.418 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:17:12.450 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:17:12.452 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:17:12.453 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:17:12.554 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:17:12.580 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:17:12.582 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:17:12.582 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:17:12.683 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:17:12.708 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:17:12.710 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:17:12.711 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:17:12.812 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:17:12.852 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:17:12.855 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:17:12.856 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:17:12.957 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:17:12.994 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:17:12.996 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:17:12.997 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:17:13.084 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:17:12 smithi164 bash[59462]: cluster 1685823431.9054148 osd.6 (osd.6) 163 : cluster 3 failed to encode map e220 with expected crc 2023-06-03T20:17:13.084 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:17:12 smithi164 bash[59462]: audit 1685823431.9091117 mon.a (mon.0) 1173 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_7", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-06-03T20:17:13.084 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:17:12 smithi164 bash[59462]: cluster 1685823431.909169 mon.a (mon.0) 1174 : cluster 0 osdmap e220: 8 total, 7 up, 6 in 2023-06-03T20:17:13.084 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:17:12 smithi164 bash[59462]: cluster 1685823432.082787 osd.7 (osd.7) 146 : cluster 3 failed to encode map e220 with expected crc 2023-06-03T20:17:13.084 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:17:12 smithi164 bash[59462]: cluster 1685823432.539427 osd.5 (osd.5) 155 : cluster 3 failed to encode map e220 with expected crc 2023-06-03T20:17:13.098 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:17:13.135 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:17:13.139 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:17:13.140 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:17:13.228 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:17:12 smithi089 bash[75068]: cluster 1685823431.9054148 osd.6 (osd.6) 163 : cluster 3 failed to encode map e220 with expected crc 2023-06-03T20:17:13.228 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:17:12 smithi089 bash[75068]: audit 1685823431.9091117 mon.a (mon.0) 1173 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_7", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-06-03T20:17:13.228 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:17:12 smithi089 bash[75068]: cluster 1685823431.909169 mon.a (mon.0) 1174 : cluster 0 osdmap e220: 8 total, 7 up, 6 in 2023-06-03T20:17:13.229 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:17:12 smithi089 bash[75068]: cluster 1685823432.082787 osd.7 (osd.7) 146 : cluster 3 failed to encode map e220 with expected crc 2023-06-03T20:17:13.229 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:17:12 smithi089 bash[75068]: cluster 1685823432.539427 osd.5 (osd.5) 155 : cluster 3 failed to encode map e220 with expected crc 2023-06-03T20:17:13.229 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:17:12 smithi089 bash[78452]: cluster 1685823431.9054148 osd.6 (osd.6) 163 : cluster 3 failed to encode map e220 with expected crc 2023-06-03T20:17:13.229 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:17:12 smithi089 bash[78452]: audit 1685823431.9091117 mon.a (mon.0) 1173 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_7", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-06-03T20:17:13.229 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:17:12 smithi089 bash[78452]: cluster 1685823431.909169 mon.a (mon.0) 1174 : cluster 0 osdmap e220: 8 total, 7 up, 6 in 2023-06-03T20:17:13.229 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:17:12 smithi089 bash[78452]: cluster 1685823432.082787 osd.7 (osd.7) 146 : cluster 3 failed to encode map e220 with expected crc 2023-06-03T20:17:13.230 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:17:12 smithi089 bash[78452]: cluster 1685823432.539427 osd.5 (osd.5) 155 : cluster 3 failed to encode map e220 with expected crc 2023-06-03T20:17:13.241 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:17:13.276 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:17:13.279 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:17:13.279 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:17:13.381 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:17:13.407 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:17:13.410 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:17:13.410 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:17:13.511 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:17:13.546 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:17:13.550 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:17:13.551 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:17:13.652 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:17:13.677 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:17:13.679 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:17:13.679 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:17:13.780 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:17:13.811 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:17:13.813 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:17:13.814 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:17:13.915 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:17:13.946 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:17:13.949 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:17:13.950 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:17:14.029 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:17:13 smithi089 bash[78452]: cluster 1685823432.8404114 mgr.y (mgr.34104) 1236 : cluster 0 pgmap v1226: 89 pgs: 20 active+clean+remapped, 6 active+undersized+degraded, 19 active+undersized, 44 active+clean; 582 KiB data, 3.2 GiB used, 533 GiB / 536 GiB avail; 8/72 objects degraded (11.111%); 6/72 objects misplaced (8.333%) 2023-06-03T20:17:14.029 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:17:13 smithi089 bash[75068]: cluster 1685823432.8404114 mgr.y (mgr.34104) 1236 : cluster 0 pgmap v1226: 89 pgs: 20 active+clean+remapped, 6 active+undersized+degraded, 19 active+undersized, 44 active+clean; 582 KiB data, 3.2 GiB used, 533 GiB / 536 GiB avail; 8/72 objects degraded (11.111%); 6/72 objects misplaced (8.333%) 2023-06-03T20:17:14.051 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:17:14.088 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:17:14.091 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:17:14.092 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:17:14.193 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:17:14.205 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:17:13 smithi164 bash[59462]: cluster 1685823432.8404114 mgr.y (mgr.34104) 1236 : cluster 0 pgmap v1226: 89 pgs: 20 active+clean+remapped, 6 active+undersized+degraded, 19 active+undersized, 44 active+clean; 582 KiB data, 3.2 GiB used, 533 GiB / 536 GiB avail; 8/72 objects degraded (11.111%); 6/72 objects misplaced (8.333%) 2023-06-03T20:17:14.233 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:17:14.237 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:17:14.237 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:17:14.338 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:17:14.380 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:17:14.382 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:17:14.383 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:17:14.484 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:17:14.517 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:17:14.519 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:17:14.520 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:17:14.621 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:17:14.655 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:17:14.658 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:17:14.659 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:17:14.760 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:17:14.796 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:17:14.800 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:17:14.800 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:17:14.901 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:17:14.937 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:17:14.940 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:17:14.941 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:17:15.042 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:17:15.075 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:17:15.077 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:17:15.078 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:17:15.179 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:17:15.217 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:17:15.220 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:17:15.220 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:17:15.321 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:17:15.355 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:17:15.357 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:17:15.357 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:17:15.458 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:17:15.489 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:17:15.491 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:17:15.492 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:17:15.593 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:17:15.626 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:17:15.628 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:17:15.629 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:17:15.730 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:17:15.761 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:17:15.765 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:17:15.765 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:17:15.851 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph daemon osd.3 dump_ops_in_flight 2023-06-03T20:17:15.866 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:17:15.905 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:17:15.908 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:17:15.909 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:17:16.010 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:17:16.051 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:17:16.052 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:17:16.053 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:17:16.154 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:17:16.168 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:17:15 smithi164 bash[59462]: cluster 1685823434.8413634 mgr.y (mgr.34104) 1237 : cluster 0 pgmap v1227: 89 pgs: 20 active+clean+remapped, 6 active+undersized+degraded, 19 active+undersized, 44 active+clean; 582 KiB data, 3.2 GiB used, 533 GiB / 536 GiB avail; 8/72 objects degraded (11.111%); 6/72 objects misplaced (8.333%) 2023-06-03T20:17:16.189 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:17:16.193 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:17:16.194 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:17:16.231 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:17:15 smithi089 bash[78452]: cluster 1685823434.8413634 mgr.y (mgr.34104) 1237 : cluster 0 pgmap v1227: 89 pgs: 20 active+clean+remapped, 6 active+undersized+degraded, 19 active+undersized, 44 active+clean; 582 KiB data, 3.2 GiB used, 533 GiB / 536 GiB avail; 8/72 objects degraded (11.111%); 6/72 objects misplaced (8.333%) 2023-06-03T20:17:16.232 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:17:15 smithi089 bash[75068]: cluster 1685823434.8413634 mgr.y (mgr.34104) 1237 : cluster 0 pgmap v1227: 89 pgs: 20 active+clean+remapped, 6 active+undersized+degraded, 19 active+undersized, 44 active+clean; 582 KiB data, 3.2 GiB used, 533 GiB / 536 GiB avail; 8/72 objects degraded (11.111%); 6/72 objects misplaced (8.333%) 2023-06-03T20:17:16.296 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:17:16.328 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:17:16.331 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:17:16.331 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:17:16.432 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:17:16.463 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:17:16.465 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:17:16.466 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:17:16.534 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 20:17:16 smithi089 bash[71163]: ::ffff:172.21.15.164 - - [03/Jun/2023:20:17:16] "GET /metrics HTTP/1.1" 200 228764 "" "Prometheus/2.33.4" 2023-06-03T20:17:16.567 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:17:16.606 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:17:16.609 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:17:16.610 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:17:16.711 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:17:16.742 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:17:16.744 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:17:16.745 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:17:16.846 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:17:16.881 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:17:16.885 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:17:16.886 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:17:16.987 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:17:17.023 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:17:17.026 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:17:17.026 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:17:17.127 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:17:17.166 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:17:17.169 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:17:17.170 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:17:17.270 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:17:17.296 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:17:17.299 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:17:17.299 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:17:17.353 DEBUG:teuthology.orchestra.run.smithi089:> /bin/sh -c 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage rados --no-log-to-stderr --name client.0 -b 65536 --object-size 65536 -p unique_pool_7 bench 90 write' 2023-06-03T20:17:17.355 INFO:tasks.radosbench:joining radosbench (timing out after 3000s) 2023-06-03T20:17:17.355 INFO:teuthology.orchestra.run:waiting for 3000 2023-06-03T20:17:17.387 INFO:tasks.radosbench.radosbench.0.smithi089.stdout:hints = 1 2023-06-03T20:17:17.388 INFO:tasks.radosbench.radosbench.0.smithi089.stdout:Maintaining 16 concurrent writes of 65536 bytes to objects of size 65536 for up to 90 seconds or 0 objects 2023-06-03T20:17:17.388 INFO:tasks.radosbench.radosbench.0.smithi089.stdout:Object prefix: benchmark_data_smithi089_191647 2023-06-03T20:17:17.388 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-06-03T20:17:17.389 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 0 0 0 0 0 0 - 0 2023-06-03T20:17:17.400 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:17:17.446 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:17:17.451 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:17:17.451 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:17:17.552 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:17:17.583 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:17:17.586 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:17:17.586 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:17:17.688 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:17:17.724 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:17:17.726 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:17:17.727 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:17:17.828 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:17:17.862 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:17:17.865 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:17:17.866 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:17:17.940 INFO:teuthology.orchestra.run.smithi089.stderr:nodeep-scrub is unset 2023-06-03T20:17:17.967 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:17:17.996 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:17:17.999 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:17:18.000 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:17:18.101 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:17:18.134 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:17:18.136 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:17:18.137 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:17:18.238 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:17:18.262 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:17:18.264 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:17:18.264 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:17:18.333 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:17:17 smithi164 bash[59462]: cluster 1685823436.8423023 mgr.y (mgr.34104) 1238 : cluster 0 pgmap v1228: 89 pgs: 20 active+clean+remapped, 6 active+undersized+degraded, 19 active+undersized, 44 active+clean; 582 KiB data, 3.2 GiB used, 533 GiB / 536 GiB avail; 8/72 objects degraded (11.111%); 6/72 objects misplaced (8.333%) 2023-06-03T20:17:18.334 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:17:17 smithi164 bash[59462]: audit 1685823437.715054 mon.c (mon.2) 546 : audit 1 from='client.? 172.21.15.89:0/4103143478' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-06-03T20:17:18.334 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:17:17 smithi164 bash[59462]: audit 1685823437.7155468 mon.a (mon.0) 1175 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-06-03T20:17:18.334 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:17:17 smithi089 bash[75068]: cluster 1685823436.8423023 mgr.y (mgr.34104) 1238 : cluster 0 pgmap v1228: 89 pgs: 20 active+clean+remapped, 6 active+undersized+degraded, 19 active+undersized, 44 active+clean; 582 KiB data, 3.2 GiB used, 533 GiB / 536 GiB avail; 8/72 objects degraded (11.111%); 6/72 objects misplaced (8.333%) 2023-06-03T20:17:18.335 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:17:17 smithi089 bash[75068]: audit 1685823437.715054 mon.c (mon.2) 546 : audit 1 from='client.? 172.21.15.89:0/4103143478' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-06-03T20:17:18.335 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:17:17 smithi089 bash[75068]: audit 1685823437.7155468 mon.a (mon.0) 1175 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-06-03T20:17:18.335 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:17:17 smithi089 bash[78452]: cluster 1685823436.8423023 mgr.y (mgr.34104) 1238 : cluster 0 pgmap v1228: 89 pgs: 20 active+clean+remapped, 6 active+undersized+degraded, 19 active+undersized, 44 active+clean; 582 KiB data, 3.2 GiB used, 533 GiB / 536 GiB avail; 8/72 objects degraded (11.111%); 6/72 objects misplaced (8.333%) 2023-06-03T20:17:18.335 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:17:17 smithi089 bash[78452]: audit 1685823437.715054 mon.c (mon.2) 546 : audit 1 from='client.? 172.21.15.89:0/4103143478' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-06-03T20:17:18.335 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:17:17 smithi089 bash[78452]: audit 1685823437.7155468 mon.a (mon.0) 1175 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-06-03T20:17:18.366 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:17:18.379 INFO:teuthology.orchestra.run.smithi089.stdout:osd.0: {} 2023-06-03T20:17:18.388 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 1 16 2317 2301 143.804 143.812 0.00286923 0.00692704 2023-06-03T20:17:18.399 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:17:18.402 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:17:18.403 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:17:18.419 INFO:teuthology.orchestra.run.smithi089.stdout:osd.1: {} 2023-06-03T20:17:18.458 INFO:teuthology.orchestra.run.smithi089.stdout:osd.2: {} 2023-06-03T20:17:18.497 INFO:teuthology.orchestra.run.smithi089.stdout:osd.3: {} 2023-06-03T20:17:18.504 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:17:18.532 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:17:18.535 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:17:18.536 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:17:18.544 INFO:teuthology.orchestra.run.smithi089.stdout:osd.5: {} 2023-06-03T20:17:18.584 INFO:teuthology.orchestra.run.smithi089.stdout:osd.6: {} 2023-06-03T20:17:18.620 INFO:teuthology.orchestra.run.smithi089.stdout:osd.7: {} 2023-06-03T20:17:18.620 INFO:teuthology.orchestra.run.smithi089.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-06-03T20:17:18.620 INFO:teuthology.orchestra.run.smithi089.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-06-03T20:17:18.620 INFO:teuthology.orchestra.run.smithi089.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-06-03T20:17:18.620 INFO:teuthology.orchestra.run.smithi089.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-06-03T20:17:18.621 INFO:teuthology.orchestra.run.smithi089.stderr:Error ENXIO: problem getting command descriptions from osd.4 2023-06-03T20:17:18.621 INFO:teuthology.orchestra.run.smithi089.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-06-03T20:17:18.621 INFO:teuthology.orchestra.run.smithi089.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-06-03T20:17:18.621 INFO:teuthology.orchestra.run.smithi089.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-06-03T20:17:18.637 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:17:18.673 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:17:18.675 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:17:18.676 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:17:18.777 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:17:18.812 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:17:18.814 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:17:18.815 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:17:18.916 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:17:18.949 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:17:18.950 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:17:18 smithi089 bash[75068]: cluster 1685823437.9209745 mon.a (mon.0) 1176 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-06-03T20:17:18.950 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:17:18 smithi089 bash[75068]: cluster 1685823437.9265888 osd.6 (osd.6) 164 : cluster 3 failed to encode map e221 with expected crc 2023-06-03T20:17:18.950 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:17:18 smithi089 bash[75068]: audit 1685823437.9290657 mon.a (mon.0) 1177 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-06-03T20:17:18.950 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:17:18 smithi089 bash[75068]: cluster 1685823437.9291568 mon.a (mon.0) 1178 : cluster 0 osdmap e221: 8 total, 7 up, 6 in 2023-06-03T20:17:18.950 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:17:18 smithi089 bash[75068]: cluster 1685823437.9293194 osd.5 (osd.5) 156 : cluster 3 failed to encode map e221 with expected crc 2023-06-03T20:17:18.950 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:17:18 smithi089 bash[75068]: audit 1685823438.2959766 mon.c (mon.2) 547 : audit 0 from='client.? 172.21.15.89:0/2637647373' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-03T20:17:18.951 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:17:18 smithi089 bash[78452]: cluster 1685823437.9209745 mon.a (mon.0) 1176 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-06-03T20:17:18.951 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:17:18 smithi089 bash[78452]: cluster 1685823437.9265888 osd.6 (osd.6) 164 : cluster 3 failed to encode map e221 with expected crc 2023-06-03T20:17:18.951 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:17:18 smithi089 bash[78452]: audit 1685823437.9290657 mon.a (mon.0) 1177 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-06-03T20:17:18.951 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:17:18 smithi089 bash[78452]: cluster 1685823437.9291568 mon.a (mon.0) 1178 : cluster 0 osdmap e221: 8 total, 7 up, 6 in 2023-06-03T20:17:18.951 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:17:18 smithi089 bash[78452]: cluster 1685823437.9293194 osd.5 (osd.5) 156 : cluster 3 failed to encode map e221 with expected crc 2023-06-03T20:17:18.951 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:17:18 smithi089 bash[78452]: audit 1685823438.2959766 mon.c (mon.2) 547 : audit 0 from='client.? 172.21.15.89:0/2637647373' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-03T20:17:18.953 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:17:18.954 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:17:19.055 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:17:19.090 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:17:19.093 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:17:19.094 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:17:19.195 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:17:19.205 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:17:18 smithi164 bash[59462]: cluster 1685823437.9209745 mon.a (mon.0) 1176 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-06-03T20:17:19.205 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:17:18 smithi164 bash[59462]: cluster 1685823437.9265888 osd.6 (osd.6) 164 : cluster 3 failed to encode map e221 with expected crc 2023-06-03T20:17:19.206 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:17:18 smithi164 bash[59462]: audit 1685823437.9290657 mon.a (mon.0) 1177 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-06-03T20:17:19.206 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:17:18 smithi164 bash[59462]: cluster 1685823437.9291568 mon.a (mon.0) 1178 : cluster 0 osdmap e221: 8 total, 7 up, 6 in 2023-06-03T20:17:19.206 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:17:18 smithi164 bash[59462]: cluster 1685823437.9293194 osd.5 (osd.5) 156 : cluster 3 failed to encode map e221 with expected crc 2023-06-03T20:17:19.206 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:17:18 smithi164 bash[59462]: audit 1685823438.2959766 mon.c (mon.2) 547 : audit 0 from='client.? 172.21.15.89:0/2637647373' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-03T20:17:19.230 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:17:19.233 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:17:19.233 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:17:19.334 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:17:19.368 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:17:19.370 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:17:19.371 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:17:19.388 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 2 16 5146 5130 160.301 176.812 0.0113952 0.00622389 2023-06-03T20:17:19.472 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:17:19.505 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:17:19.509 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:17:19.509 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:17:19.610 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:17:19.641 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:17:19.643 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:17:19.644 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:17:19.745 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:17:19.781 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:17:19.783 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:17:19.783 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:17:19.884 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:17:19.918 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:17:19.920 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:17:19.920 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:17:19.954 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:17:19 smithi089 bash[75068]: cluster 1685823438.843159 mgr.y (mgr.34104) 1239 : cluster 0 pgmap v1230: 89 pgs: 20 active+clean+remapped, 11 active+undersized+degraded, 14 active+undersized, 44 active+clean; 87 MiB data, 3.4 GiB used, 533 GiB / 536 GiB avail; 11 MiB/s wr, 173 op/s; 898/4230 objects degraded (21.229%); 6/4230 objects misplaced (0.142%) 2023-06-03T20:17:19.954 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:17:19 smithi089 bash[75068]: cluster 1685823438.9349551 mon.a (mon.0) 1179 : cluster 3 Health check update: Degraded data redundancy: 898/4230 objects degraded (21.229%), 11 pgs degraded, 17 pgs undersized (PG_DEGRADED) 2023-06-03T20:17:19.955 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:17:19 smithi089 bash[78452]: cluster 1685823438.843159 mgr.y (mgr.34104) 1239 : cluster 0 pgmap v1230: 89 pgs: 20 active+clean+remapped, 11 active+undersized+degraded, 14 active+undersized, 44 active+clean; 87 MiB data, 3.4 GiB used, 533 GiB / 536 GiB avail; 11 MiB/s wr, 173 op/s; 898/4230 objects degraded (21.229%); 6/4230 objects misplaced (0.142%) 2023-06-03T20:17:19.955 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:17:19 smithi089 bash[78452]: cluster 1685823438.9349551 mon.a (mon.0) 1179 : cluster 3 Health check update: Degraded data redundancy: 898/4230 objects degraded (21.229%), 11 pgs degraded, 17 pgs undersized (PG_DEGRADED) 2023-06-03T20:17:20.021 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:17:20.046 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:17:20.048 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:17:20.048 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:17:20.083 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:17:19 smithi164 bash[59462]: cluster 1685823438.843159 mgr.y (mgr.34104) 1239 : cluster 0 pgmap v1230: 89 pgs: 20 active+clean+remapped, 11 active+undersized+degraded, 14 active+undersized, 44 active+clean; 87 MiB data, 3.4 GiB used, 533 GiB / 536 GiB avail; 11 MiB/s wr, 173 op/s; 898/4230 objects degraded (21.229%); 6/4230 objects misplaced (0.142%) 2023-06-03T20:17:20.084 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:17:19 smithi164 bash[59462]: cluster 1685823438.9349551 mon.a (mon.0) 1179 : cluster 3 Health check update: Degraded data redundancy: 898/4230 objects degraded (21.229%), 11 pgs degraded, 17 pgs undersized (PG_DEGRADED) 2023-06-03T20:17:20.149 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:17:20.185 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:17:20.187 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:17:20.188 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:17:20.289 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:17:20.321 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:17:20.324 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:17:20.325 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:17:20.388 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 3 16 7869 7853 163.591 170.188 0.00700414 0.00610176 2023-06-03T20:17:20.425 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:17:20.461 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:17:20.464 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:17:20.465 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:17:20.566 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:17:20.602 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:17:20.604 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:17:20.604 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:17:20.706 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:17:20.737 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:17:20.738 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:17:20.739 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:17:20.840 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:17:20.871 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:17:20.875 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:17:20.875 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:17:20.976 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:17:21.010 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:17:21.013 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:17:21.014 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:17:21.114 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:17:21.139 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:17:21.141 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:17:21.143 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:17:21.244 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:17:21.280 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:17:21.282 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:17:21.283 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:17:21.383 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:17:21.388 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 4 16 10282 10266 160.393 150.812 0.00368629 0.00622032 2023-06-03T20:17:21.415 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:17:21.418 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:17:21.418 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:17:21.519 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:17:21.558 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:17:21.561 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:17:21.562 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:17:21.663 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:17:21.694 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:17:21.698 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:17:21.698 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:17:21.799 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:17:21.835 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:17:21.838 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:17:21.839 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:17:21.940 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:17:21.971 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:17:21.972 INFO:teuthology.orchestra.run.smithi089.stderr:set osd.5 primary-affinity to 0 (802) 2023-06-03T20:17:21.973 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:17:21 smithi164 bash[59462]: cluster 1685823440.8438702 mgr.y (mgr.34104) 1240 : cluster 0 pgmap v1231: 89 pgs: 20 active+clean+remapped, 13 active+undersized+degraded, 12 active+undersized, 44 active+clean; 203 MiB data, 3.7 GiB used, 533 GiB / 536 GiB avail; 23 MiB/s wr, 362 op/s; 1830/9804 objects degraded (18.666%); 6/9804 objects misplaced (0.061%) 2023-06-03T20:17:21.973 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:17:21 smithi164 bash[59462]: audit 1685823441.5662713 mon.a (mon.0) 1180 : audit 1 from='client.? 172.21.15.89:0/3398334929' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 5, "weight": 0.0}]: dispatch 2023-06-03T20:17:21.975 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:17:21.975 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:17:22.076 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:17:22.111 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:17:22.114 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:17:22.114 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:17:22.215 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:17:22.228 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:17:21 smithi089 bash[75068]: cluster 1685823440.8438702 mgr.y (mgr.34104) 1240 : cluster 0 pgmap v1231: 89 pgs: 20 active+clean+remapped, 13 active+undersized+degraded, 12 active+undersized, 44 active+clean; 203 MiB data, 3.7 GiB used, 533 GiB / 536 GiB avail; 23 MiB/s wr, 362 op/s; 1830/9804 objects degraded (18.666%); 6/9804 objects misplaced (0.061%) 2023-06-03T20:17:22.228 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:17:21 smithi089 bash[75068]: audit 1685823441.5662713 mon.a (mon.0) 1180 : audit 1 from='client.? 172.21.15.89:0/3398334929' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 5, "weight": 0.0}]: dispatch 2023-06-03T20:17:22.228 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:17:21 smithi089 bash[78452]: cluster 1685823440.8438702 mgr.y (mgr.34104) 1240 : cluster 0 pgmap v1231: 89 pgs: 20 active+clean+remapped, 13 active+undersized+degraded, 12 active+undersized, 44 active+clean; 203 MiB data, 3.7 GiB used, 533 GiB / 536 GiB avail; 23 MiB/s wr, 362 op/s; 1830/9804 objects degraded (18.666%); 6/9804 objects misplaced (0.061%) 2023-06-03T20:17:22.228 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:17:21 smithi089 bash[78452]: audit 1685823441.5662713 mon.a (mon.0) 1180 : audit 1 from='client.? 172.21.15.89:0/3398334929' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 5, "weight": 0.0}]: dispatch 2023-06-03T20:17:22.245 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:17:22.248 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:17:22.248 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:17:22.349 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:17:22.385 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:17:22.387 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:17:22.388 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:17:22.388 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 5 16 11807 11791 147.376 95.3125 0.00199324 0.0063073 2023-06-03T20:17:22.489 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:17:22.524 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:17:22.527 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:17:22.528 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:17:22.629 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:17:22.667 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:17:22.669 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:17:22.670 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:17:22.771 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:17:22.797 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:17:22.799 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:17:22.800 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:17:22.901 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:17:22.933 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:17:22.936 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:17:22.936 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:17:23.037 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:17:23.067 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:17:23.071 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:17:23.071 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:17:23.172 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:17:23.183 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:17:22 smithi164 bash[59462]: cluster 1685823441.9540994 osd.7 (osd.7) 147 : cluster 3 failed to encode map e222 with expected crc 2023-06-03T20:17:23.183 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:17:22 smithi164 bash[59462]: audit 1685823441.9584002 mon.a (mon.0) 1181 : audit 1 from='client.? 172.21.15.89:0/3398334929' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 5, "weight": 0.0}]': finished 2023-06-03T20:17:23.183 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:17:22 smithi164 bash[59462]: cluster 1685823441.9584694 mon.a (mon.0) 1182 : cluster 0 osdmap e222: 8 total, 7 up, 6 in 2023-06-03T20:17:23.183 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:17:22 smithi164 bash[59462]: audit 1685823442.1340106 mon.a (mon.0) 1183 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:17:23.183 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:17:22 smithi164 bash[59462]: audit 1685823442.1353612 mon.c (mon.2) 548 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T20:17:23.184 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:17:22 smithi164 bash[59462]: cluster 1685823442.8182895 mon.a (mon.0) 1184 : cluster 0 osdmap e223: 8 total, 7 up, 6 in 2023-06-03T20:17:23.208 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:17:23.211 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:17:23.211 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:17:23.312 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:17:23.345 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:17:23.349 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:17:23.351 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:17:23.389 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 6 16 12632 12616 131.406 51.5625 0.00439219 0.00760192 2023-06-03T20:17:23.395 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:17:23 smithi089 bash[78452]: cluster 1685823441.9540994 osd.7 (osd.7) 147 : cluster 3 failed to encode map e222 with expected crc 2023-06-03T20:17:23.395 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:17:23 smithi089 bash[78452]: audit 1685823441.9584002 mon.a (mon.0) 1181 : audit 1 from='client.? 172.21.15.89:0/3398334929' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 5, "weight": 0.0}]': finished 2023-06-03T20:17:23.395 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:17:23 smithi089 bash[78452]: cluster 1685823441.9584694 mon.a (mon.0) 1182 : cluster 0 osdmap e222: 8 total, 7 up, 6 in 2023-06-03T20:17:23.395 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:17:23 smithi089 bash[78452]: audit 1685823442.1340106 mon.a (mon.0) 1183 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:17:23.395 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:17:23 smithi089 bash[78452]: audit 1685823442.1353612 mon.c (mon.2) 548 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T20:17:23.396 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:17:23 smithi089 bash[78452]: cluster 1685823442.8182895 mon.a (mon.0) 1184 : cluster 0 osdmap e223: 8 total, 7 up, 6 in 2023-06-03T20:17:23.396 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:17:23 smithi089 bash[75068]: cluster 1685823441.9540994 osd.7 (osd.7) 147 : cluster 3 failed to encode map e222 with expected crc 2023-06-03T20:17:23.396 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:17:23 smithi089 bash[75068]: audit 1685823441.9584002 mon.a (mon.0) 1181 : audit 1 from='client.? 172.21.15.89:0/3398334929' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 5, "weight": 0.0}]': finished 2023-06-03T20:17:23.396 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:17:23 smithi089 bash[75068]: cluster 1685823441.9584694 mon.a (mon.0) 1182 : cluster 0 osdmap e222: 8 total, 7 up, 6 in 2023-06-03T20:17:23.396 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:17:23 smithi089 bash[75068]: audit 1685823442.1340106 mon.a (mon.0) 1183 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:17:23.396 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:17:23 smithi089 bash[75068]: audit 1685823442.1353612 mon.c (mon.2) 548 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T20:17:23.397 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:17:23 smithi089 bash[75068]: cluster 1685823442.8182895 mon.a (mon.0) 1184 : cluster 0 osdmap e223: 8 total, 7 up, 6 in 2023-06-03T20:17:23.452 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:17:23.483 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:17:23.485 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:17:23.486 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:17:23.587 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:17:23.625 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:17:23.628 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:17:23.628 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:17:23.729 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:17:23.765 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:17:23.768 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:17:23.769 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:17:23.870 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:17:23.905 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:17:23.908 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:17:23.909 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:17:24.010 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:17:24.021 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:17:23 smithi164 bash[59462]: cluster 1685823442.8145394 osd.6 (osd.6) 165 : cluster 3 failed to encode map e223 with expected crc 2023-06-03T20:17:24.022 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:17:23 smithi164 bash[59462]: cluster 1685823442.8152306 osd.5 (osd.5) 157 : cluster 3 failed to encode map e223 with expected crc 2023-06-03T20:17:24.022 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:17:23 smithi164 bash[59462]: cluster 1685823442.8171198 osd.7 (osd.7) 148 : cluster 3 failed to encode map e223 with expected crc 2023-06-03T20:17:24.022 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:17:23 smithi164 bash[59462]: cluster 1685823442.8203988 osd.5 (osd.5) 158 : cluster 3 failed to encode map e223 with expected crc 2023-06-03T20:17:24.022 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:17:23 smithi164 bash[59462]: cluster 1685823442.8207262 osd.6 (osd.6) 166 : cluster 3 failed to encode map e223 with expected crc 2023-06-03T20:17:24.022 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:17:23 smithi164 bash[59462]: cluster 1685823442.8230844 osd.5 (osd.5) 159 : cluster 3 failed to encode map e223 with expected crc 2023-06-03T20:17:24.023 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:17:23 smithi164 bash[59462]: cluster 1685823442.8261871 osd.5 (osd.5) 160 : cluster 3 failed to encode map e223 with expected crc 2023-06-03T20:17:24.023 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:17:23 smithi164 bash[59462]: cluster 1685823442.845135 mgr.y (mgr.34104) 1241 : cluster 0 pgmap v1234: 89 pgs: 20 active+clean+remapped, 14 active+undersized+degraded, 11 active+undersized, 44 active+clean; 363 MiB data, 3.9 GiB used, 532 GiB / 536 GiB avail; 60 MiB/s wr, 965 op/s; 2487/17466 objects degraded (14.239%); 6/17466 objects misplaced (0.034%) 2023-06-03T20:17:24.023 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:17:23 smithi164 bash[59462]: cluster 1685823443.8201747 mon.a (mon.0) 1185 : cluster 0 osdmap e224: 8 total, 7 up, 6 in 2023-06-03T20:17:24.047 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:17:24.050 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:17:24.051 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:17:24.152 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:17:24.183 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:17:24.185 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:17:24.186 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:17:24.279 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:17:23 smithi089 bash[75068]: cluster 1685823442.8145394 osd.6 (osd.6) 165 : cluster 3 failed to encode map e223 with expected crc 2023-06-03T20:17:24.279 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:17:23 smithi089 bash[75068]: cluster 1685823442.8152306 osd.5 (osd.5) 157 : cluster 3 failed to encode map e223 with expected crc 2023-06-03T20:17:24.279 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:17:23 smithi089 bash[75068]: cluster 1685823442.8171198 osd.7 (osd.7) 148 : cluster 3 failed to encode map e223 with expected crc 2023-06-03T20:17:24.279 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:17:23 smithi089 bash[75068]: cluster 1685823442.8203988 osd.5 (osd.5) 158 : cluster 3 failed to encode map e223 with expected crc 2023-06-03T20:17:24.280 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:17:23 smithi089 bash[75068]: cluster 1685823442.8207262 osd.6 (osd.6) 166 : cluster 3 failed to encode map e223 with expected crc 2023-06-03T20:17:24.280 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:17:23 smithi089 bash[75068]: cluster 1685823442.8230844 osd.5 (osd.5) 159 : cluster 3 failed to encode map e223 with expected crc 2023-06-03T20:17:24.280 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:17:23 smithi089 bash[75068]: cluster 1685823442.8261871 osd.5 (osd.5) 160 : cluster 3 failed to encode map e223 with expected crc 2023-06-03T20:17:24.280 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:17:23 smithi089 bash[75068]: cluster 1685823442.845135 mgr.y (mgr.34104) 1241 : cluster 0 pgmap v1234: 89 pgs: 20 active+clean+remapped, 14 active+undersized+degraded, 11 active+undersized, 44 active+clean; 363 MiB data, 3.9 GiB used, 532 GiB / 536 GiB avail; 60 MiB/s wr, 965 op/s; 2487/17466 objects degraded (14.239%); 6/17466 objects misplaced (0.034%) 2023-06-03T20:17:24.280 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:17:23 smithi089 bash[75068]: cluster 1685823443.8201747 mon.a (mon.0) 1185 : cluster 0 osdmap e224: 8 total, 7 up, 6 in 2023-06-03T20:17:24.281 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:17:23 smithi089 bash[78452]: cluster 1685823442.8145394 osd.6 (osd.6) 165 : cluster 3 failed to encode map e223 with expected crc 2023-06-03T20:17:24.281 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:17:23 smithi089 bash[78452]: cluster 1685823442.8152306 osd.5 (osd.5) 157 : cluster 3 failed to encode map e223 with expected crc 2023-06-03T20:17:24.281 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:17:23 smithi089 bash[78452]: cluster 1685823442.8171198 osd.7 (osd.7) 148 : cluster 3 failed to encode map e223 with expected crc 2023-06-03T20:17:24.281 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:17:23 smithi089 bash[78452]: cluster 1685823442.8203988 osd.5 (osd.5) 158 : cluster 3 failed to encode map e223 with expected crc 2023-06-03T20:17:24.281 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:17:23 smithi089 bash[78452]: cluster 1685823442.8207262 osd.6 (osd.6) 166 : cluster 3 failed to encode map e223 with expected crc 2023-06-03T20:17:24.281 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:17:23 smithi089 bash[78452]: cluster 1685823442.8230844 osd.5 (osd.5) 159 : cluster 3 failed to encode map e223 with expected crc 2023-06-03T20:17:24.282 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:17:23 smithi089 bash[78452]: cluster 1685823442.8261871 osd.5 (osd.5) 160 : cluster 3 failed to encode map e223 with expected crc 2023-06-03T20:17:24.282 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:17:23 smithi089 bash[78452]: cluster 1685823442.845135 mgr.y (mgr.34104) 1241 : cluster 0 pgmap v1234: 89 pgs: 20 active+clean+remapped, 14 active+undersized+degraded, 11 active+undersized, 44 active+clean; 363 MiB data, 3.9 GiB used, 532 GiB / 536 GiB avail; 60 MiB/s wr, 965 op/s; 2487/17466 objects degraded (14.239%); 6/17466 objects misplaced (0.034%) 2023-06-03T20:17:24.282 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:17:23 smithi089 bash[78452]: cluster 1685823443.8201747 mon.a (mon.0) 1185 : cluster 0 osdmap e224: 8 total, 7 up, 6 in 2023-06-03T20:17:24.287 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:17:24.321 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:17:24.323 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:17:24.324 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:17:24.389 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 7 16 14874 14858 132.65 140.125 0.00282189 0.00753031 2023-06-03T20:17:24.425 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:17:24.668 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:17:24.672 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:17:24.673 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:17:24.774 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:17:24.815 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:17:24.818 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:17:24.818 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:17:24.919 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:17:24.954 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:17:24.956 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:17:24.957 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:17:25.057 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:17:25.083 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:17:24 smithi164 bash[59462]: cluster 1685823443.816691 osd.6 (osd.6) 167 : cluster 3 failed to encode map e224 with expected crc 2023-06-03T20:17:25.084 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:17:24 smithi164 bash[59462]: cluster 1685823443.8187888 osd.7 (osd.7) 149 : cluster 3 failed to encode map e224 with expected crc 2023-06-03T20:17:25.084 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:17:24 smithi164 bash[59462]: cluster 1685823443.8233845 osd.7 (osd.7) 150 : cluster 3 failed to encode map e224 with expected crc 2023-06-03T20:17:25.084 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:17:24 smithi164 bash[59462]: cluster 1685823443.8243122 osd.5 (osd.5) 161 : cluster 3 failed to encode map e224 with expected crc 2023-06-03T20:17:25.084 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:17:25.086 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:17:25.087 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:17:25.188 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:17:25.197 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:17:24 smithi089 bash[75068]: cluster 1685823443.816691 osd.6 (osd.6) 167 : cluster 3 failed to encode map e224 with expected crc 2023-06-03T20:17:25.197 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:17:24 smithi089 bash[75068]: cluster 1685823443.8187888 osd.7 (osd.7) 149 : cluster 3 failed to encode map e224 with expected crc 2023-06-03T20:17:25.197 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:17:24 smithi089 bash[75068]: cluster 1685823443.8233845 osd.7 (osd.7) 150 : cluster 3 failed to encode map e224 with expected crc 2023-06-03T20:17:25.197 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:17:24 smithi089 bash[75068]: cluster 1685823443.8243122 osd.5 (osd.5) 161 : cluster 3 failed to encode map e224 with expected crc 2023-06-03T20:17:25.197 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:17:24 smithi089 bash[78452]: cluster 1685823443.816691 osd.6 (osd.6) 167 : cluster 3 failed to encode map e224 with expected crc 2023-06-03T20:17:25.197 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:17:24 smithi089 bash[78452]: cluster 1685823443.8187888 osd.7 (osd.7) 149 : cluster 3 failed to encode map e224 with expected crc 2023-06-03T20:17:25.198 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:17:24 smithi089 bash[78452]: cluster 1685823443.8233845 osd.7 (osd.7) 150 : cluster 3 failed to encode map e224 with expected crc 2023-06-03T20:17:25.198 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:17:24 smithi089 bash[78452]: cluster 1685823443.8243122 osd.5 (osd.5) 161 : cluster 3 failed to encode map e224 with expected crc 2023-06-03T20:17:25.216 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:17:25.219 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:17:25.219 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:17:25.321 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:17:25.352 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:17:25.357 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:17:25.357 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:17:25.389 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 8 16 16945 16929 132.247 129.438 0.006778 0.007554 2023-06-03T20:17:25.459 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:17:25.492 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:17:25.495 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:17:25.496 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:17:25.597 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:17:25.630 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:17:25.635 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:17:25.635 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:17:25.736 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:17:25.768 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:17:25.771 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:17:25.772 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:17:25.873 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:17:25.916 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:17:25.918 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:17:25.919 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:17:25.995 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:17:25 smithi089 bash[78452]: cluster 1685823444.8458717 mgr.y (mgr.34104) 1242 : cluster 0 pgmap v1236: 89 pgs: 3 remapped+peering, 17 active+clean+remapped, 14 active+undersized+degraded, 11 active+undersized, 44 active+clean; 646 MiB data, 4.8 GiB used, 532 GiB / 536 GiB avail; 93 MiB/s wr, 1.49k op/s; 5084/31044 objects degraded (16.377%); 4/31044 objects misplaced (0.013%) 2023-06-03T20:17:25.996 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:17:25 smithi089 bash[78452]: cluster 1685823444.9703486 mon.a (mon.0) 1186 : cluster 3 Health check failed: Reduced data availability: 3 pgs inactive, 3 pgs peering (PG_AVAILABILITY) 2023-06-03T20:17:25.996 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:17:25 smithi089 bash[78452]: cluster 1685823444.9703867 mon.a (mon.0) 1187 : cluster 3 Health check update: Degraded data redundancy: 5084/31044 objects degraded (16.377%), 14 pgs degraded, 17 pgs undersized (PG_DEGRADED) 2023-06-03T20:17:25.996 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:17:25 smithi089 bash[75068]: cluster 1685823444.8458717 mgr.y (mgr.34104) 1242 : cluster 0 pgmap v1236: 89 pgs: 3 remapped+peering, 17 active+clean+remapped, 14 active+undersized+degraded, 11 active+undersized, 44 active+clean; 646 MiB data, 4.8 GiB used, 532 GiB / 536 GiB avail; 93 MiB/s wr, 1.49k op/s; 5084/31044 objects degraded (16.377%); 4/31044 objects misplaced (0.013%) 2023-06-03T20:17:25.997 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:17:25 smithi089 bash[75068]: cluster 1685823444.9703486 mon.a (mon.0) 1186 : cluster 3 Health check failed: Reduced data availability: 3 pgs inactive, 3 pgs peering (PG_AVAILABILITY) 2023-06-03T20:17:25.997 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:17:25 smithi089 bash[75068]: cluster 1685823444.9703867 mon.a (mon.0) 1187 : cluster 3 Health check update: Degraded data redundancy: 5084/31044 objects degraded (16.377%), 14 pgs degraded, 17 pgs undersized (PG_DEGRADED) 2023-06-03T20:17:26.020 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:17:26.053 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:17:26.055 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:17:26.056 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:17:26.157 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:17:26.187 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:17:26.190 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:17:26.191 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:17:26.278 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 20:17:26 smithi089 bash[71163]: ::ffff:172.21.15.164 - - [03/Jun/2023:20:17:26] "GET /metrics HTTP/1.1" 200 228869 "" "Prometheus/2.33.4" 2023-06-03T20:17:26.292 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:17:26.302 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:17:25 smithi164 bash[59462]: cluster 1685823444.8458717 mgr.y (mgr.34104) 1242 : cluster 0 pgmap v1236: 89 pgs: 3 remapped+peering, 17 active+clean+remapped, 14 active+undersized+degraded, 11 active+undersized, 44 active+clean; 646 MiB data, 4.8 GiB used, 532 GiB / 536 GiB avail; 93 MiB/s wr, 1.49k op/s; 5084/31044 objects degraded (16.377%); 4/31044 objects misplaced (0.013%) 2023-06-03T20:17:26.303 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:17:25 smithi164 bash[59462]: cluster 1685823444.9703486 mon.a (mon.0) 1186 : cluster 3 Health check failed: Reduced data availability: 3 pgs inactive, 3 pgs peering (PG_AVAILABILITY) 2023-06-03T20:17:26.303 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:17:25 smithi164 bash[59462]: cluster 1685823444.9703867 mon.a (mon.0) 1187 : cluster 3 Health check update: Degraded data redundancy: 5084/31044 objects degraded (16.377%), 14 pgs degraded, 17 pgs undersized (PG_DEGRADED) 2023-06-03T20:17:26.330 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:17:26.333 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:17:26.334 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:17:26.389 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 9 16 18872 18856 130.933 120.438 0.00621902 0.00762939 2023-06-03T20:17:26.436 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:17:26.477 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:17:26.479 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:17:26.479 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:17:26.580 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:17:26.613 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:17:26.614 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:17:26.614 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:17:26.674 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph osd set noscrub 2023-06-03T20:17:26.716 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:17:26.803 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:17:26.809 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:17:26.809 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:17:26.810 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-06-03T20:17:26.810 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-06-03T20:17:26.910 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:17:26.944 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:17:26.946 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:17:26.947 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:17:27.000 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:17:26 smithi089 bash[75068]: audit 1685823446.0598595 mon.c (mon.2) 549 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-03T20:17:27.048 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:17:27.080 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:17:27.086 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:17:27.087 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:17:27.187 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:17:27.220 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:17:27.222 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:17:27.223 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:17:27.278 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:17:26 smithi089 bash[78452]: audit 1685823446.0598595 mon.c (mon.2) 549 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-03T20:17:27.323 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:17:27.333 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:17:26 smithi164 bash[59462]: audit 1685823446.0598595 mon.c (mon.2) 549 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-03T20:17:27.360 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:17:27.363 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:17:27.363 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:17:27.389 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 10 16 20696 20680 129.239 114 0.00758918 0.0077309 2023-06-03T20:17:27.465 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:17:27.498 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:17:27.501 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:17:27.502 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:17:27.603 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:17:27.636 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:17:27.639 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:17:27.639 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:17:27.740 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:17:27.779 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:17:27.781 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:17:27.782 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:17:27.883 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:17:27.917 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:17:27.920 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:17:27.921 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:17:28.022 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:17:28.055 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:17:28.058 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:17:28.058 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:17:28.159 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:17:28.192 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:17:28.194 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:17:28.194 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:17:28.279 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:17:27 smithi089 bash[78452]: cluster 1685823446.8466828 mgr.y (mgr.34104) 1243 : cluster 0 pgmap v1237: 89 pgs: 3 remapped+peering, 17 active+clean+remapped, 14 active+undersized+degraded, 11 active+undersized, 44 active+clean; 976 MiB data, 5.4 GiB used, 531 GiB / 536 GiB avail; 129 MiB/s wr, 2.06k op/s; 7615/46875 objects degraded (16.245%); 4/46875 objects misplaced (0.009%) 2023-06-03T20:17:28.279 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:17:27 smithi089 bash[75068]: cluster 1685823446.8466828 mgr.y (mgr.34104) 1243 : cluster 0 pgmap v1237: 89 pgs: 3 remapped+peering, 17 active+clean+remapped, 14 active+undersized+degraded, 11 active+undersized, 44 active+clean; 976 MiB data, 5.4 GiB used, 531 GiB / 536 GiB avail; 129 MiB/s wr, 2.06k op/s; 7615/46875 objects degraded (16.245%); 4/46875 objects misplaced (0.009%) 2023-06-03T20:17:28.295 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:17:28.331 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:17:28.333 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:17:27 smithi164 bash[59462]: cluster 1685823446.8466828 mgr.y (mgr.34104) 1243 : cluster 0 pgmap v1237: 89 pgs: 3 remapped+peering, 17 active+clean+remapped, 14 active+undersized+degraded, 11 active+undersized, 44 active+clean; 976 MiB data, 5.4 GiB used, 531 GiB / 536 GiB avail; 129 MiB/s wr, 2.06k op/s; 7615/46875 objects degraded (16.245%); 4/46875 objects misplaced (0.009%) 2023-06-03T20:17:28.334 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:17:28.335 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:17:28.389 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 11 16 22594 22578 128.273 118.625 0.00341925 0.00778921 2023-06-03T20:17:28.436 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:17:28.473 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:17:28.479 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:17:28.480 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:17:28.581 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:17:28.619 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:17:28.622 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:17:28.623 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:17:28.724 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:17:28.757 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:17:28.759 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:17:28.760 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:17:28.811 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-06-03T20:17:28.861 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:17:28.907 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:17:28.909 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:17:28.910 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:17:29.011 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:17:29.049 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:17:29.049 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:17:29.050 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:17:29.058 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:17:29 smithi164 bash[59462]: cluster 1685823449.0001364 mon.a (mon.0) 1188 : cluster 1 Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 3 pgs inactive, 3 pgs peering) 2023-06-03T20:17:29.151 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:17:29.162 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:17:29 smithi089 bash[75068]: cluster 1685823449.0001364 mon.a (mon.0) 1188 : cluster 1 Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 3 pgs inactive, 3 pgs peering) 2023-06-03T20:17:29.162 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:17:29 smithi089 bash[78452]: cluster 1685823449.0001364 mon.a (mon.0) 1188 : cluster 1 Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 3 pgs inactive, 3 pgs peering) 2023-06-03T20:17:29.190 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:17:29.194 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:17:29.194 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:17:29.295 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:17:29.336 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:17:29.338 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:17:29.339 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:17:29.390 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 12 16 24269 24253 126.307 104.688 0.0104698 0.00790978 2023-06-03T20:17:29.441 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:17:29.473 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:17:29.476 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:17:29.477 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:17:29.578 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:17:29.607 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:17:29.610 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:17:29.610 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:17:29.640 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/mon.a/config 2023-06-03T20:17:29.711 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:17:29.740 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:17:29.743 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:17:29.744 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:17:29.846 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:17:29.883 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:17:29.887 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:17:29.887 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:17:29.988 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:17:30.018 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:17:30.020 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:17:30.021 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:17:30.121 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:17:30.154 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:17:30.157 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:17:30.158 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:17:30.259 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:17:30.291 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:17:30.294 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:17:30.295 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:17:30.333 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:17:30 smithi164 bash[59462]: cluster 1685823448.8475304 mgr.y (mgr.34104) 1244 : cluster 0 pgmap v1238: 89 pgs: 20 active+clean+remapped, 14 active+undersized+degraded, 11 active+undersized, 44 active+clean; 1.2 GiB data, 6.3 GiB used, 530 GiB / 536 GiB avail; 128 MiB/s wr, 2.05k op/s; 10064/59844 objects degraded (16.817%); 6/59844 objects misplaced (0.010%) 2023-06-03T20:17:30.389 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 13 16 25777 25761 123.84 94.25 0.00902628 0.0080695 2023-06-03T20:17:30.396 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:17:30.428 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:17:30.431 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:17:30.431 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:17:30.529 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:17:30 smithi089 bash[78452]: cluster 1685823448.8475304 mgr.y (mgr.34104) 1244 : cluster 0 pgmap v1238: 89 pgs: 20 active+clean+remapped, 14 active+undersized+degraded, 11 active+undersized, 44 active+clean; 1.2 GiB data, 6.3 GiB used, 530 GiB / 536 GiB avail; 128 MiB/s wr, 2.05k op/s; 10064/59844 objects degraded (16.817%); 6/59844 objects misplaced (0.010%) 2023-06-03T20:17:30.529 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:17:30 smithi089 bash[75068]: cluster 1685823448.8475304 mgr.y (mgr.34104) 1244 : cluster 0 pgmap v1238: 89 pgs: 20 active+clean+remapped, 14 active+undersized+degraded, 11 active+undersized, 44 active+clean; 1.2 GiB data, 6.3 GiB used, 530 GiB / 536 GiB avail; 128 MiB/s wr, 2.05k op/s; 10064/59844 objects degraded (16.817%); 6/59844 objects misplaced (0.010%) 2023-06-03T20:17:30.533 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:17:30.564 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:17:30.567 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:17:30.568 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:17:30.668 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:17:30.696 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:17:30.699 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:17:30.700 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:17:30.801 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:17:30.839 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:17:30.843 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:17:30.844 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:17:30.945 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:17:30.976 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:17:30.978 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:17:30.979 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:17:31.080 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:17:31.121 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:17:31.124 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:17:31.125 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:17:31.225 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:17:31.263 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:17:31.266 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:17:31.267 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:17:31.368 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:17:31.376 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:17:31 smithi089 bash[78452]: cluster 1685823451.054366 mon.a (mon.0) 1189 : cluster 3 Health check update: Degraded data redundancy: 11747/68166 objects degraded (17.233%), 14 pgs degraded, 16 pgs undersized (PG_DEGRADED) 2023-06-03T20:17:31.376 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:17:31 smithi089 bash[75068]: cluster 1685823451.054366 mon.a (mon.0) 1189 : cluster 3 Health check update: Degraded data redundancy: 11747/68166 objects degraded (17.233%), 14 pgs degraded, 16 pgs undersized (PG_DEGRADED) 2023-06-03T20:17:31.390 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 14 16 27621 27605 123.216 115.25 0.00953269 0.00810864 2023-06-03T20:17:31.401 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:17:31.403 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:17:31.404 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:17:31.505 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:17:31.535 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:17:31.537 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:17:31.537 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:17:31.587 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:17:31 smithi164 bash[59462]: cluster 1685823451.054366 mon.a (mon.0) 1189 : cluster 3 Health check update: Degraded data redundancy: 11747/68166 objects degraded (17.233%), 14 pgs degraded, 16 pgs undersized (PG_DEGRADED) 2023-06-03T20:17:31.639 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:17:31.670 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:17:31.673 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:17:31.673 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:17:31.705 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/mon.a/config 2023-06-03T20:17:31.775 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:17:31.814 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:17:31.816 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:17:31.817 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:17:31.918 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:17:31.963 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:17:31.967 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:17:31.968 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:17:32.069 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:17:32.114 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:17:32.116 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:17:32.117 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:17:32.164 INFO:tasks.thrashosds.thrasher:in_osds: [0, 2, 3, 4, 5, 7] out_osds: [1, 6] dead_osds: [4] live_osds: [0, 1, 2, 3, 5, 6, 7] 2023-06-03T20:17:32.165 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-06-03T20:17:32.165 INFO:tasks.thrashosds.thrasher:inject_pause on osd.5 2023-06-03T20:17:32.166 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2023-06-03T20:17:32.166 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2023-06-03T20:17:32.167 DEBUG:teuthology.orchestra.run.smithi164:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph daemon osd.5 config set filestore_inject_stall 3 2023-06-03T20:17:32.218 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:17:32.228 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:17:32 smithi089 bash[78452]: cluster 1685823450.8482485 mgr.y (mgr.34104) 1245 : cluster 0 pgmap v1239: 89 pgs: 20 active+clean+remapped, 14 active+undersized+degraded, 11 active+undersized, 44 active+clean; 1.4 GiB data, 6.7 GiB used, 530 GiB / 536 GiB avail; 132 MiB/s wr, 2.10k op/s; 11747/68166 objects degraded (17.233%); 6/68166 objects misplaced (0.009%) 2023-06-03T20:17:32.228 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:17:32 smithi089 bash[75068]: cluster 1685823450.8482485 mgr.y (mgr.34104) 1245 : cluster 0 pgmap v1239: 89 pgs: 20 active+clean+remapped, 14 active+undersized+degraded, 11 active+undersized, 44 active+clean; 1.4 GiB data, 6.7 GiB used, 530 GiB / 536 GiB avail; 132 MiB/s wr, 2.10k op/s; 11747/68166 objects degraded (17.233%); 6/68166 objects misplaced (0.009%) 2023-06-03T20:17:32.253 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:17:32.263 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:17:32.265 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:17:32.366 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:17:32.390 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 15 16 29112 29096 121.214 93.1875 0.0129887 0.00824023 2023-06-03T20:17:32.399 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:17:32.401 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:17:32.402 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:17:32.503 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:17:32.554 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:17:32.558 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:17:32.559 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:17:32.583 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:17:32 smithi164 bash[59462]: cluster 1685823450.8482485 mgr.y (mgr.34104) 1245 : cluster 0 pgmap v1239: 89 pgs: 20 active+clean+remapped, 14 active+undersized+degraded, 11 active+undersized, 44 active+clean; 1.4 GiB data, 6.7 GiB used, 530 GiB / 536 GiB avail; 132 MiB/s wr, 2.10k op/s; 11747/68166 objects degraded (17.233%); 6/68166 objects misplaced (0.009%) 2023-06-03T20:17:32.661 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:17:32.698 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:17:32.702 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:17:32.703 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:17:32.804 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:17:32.832 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:17:32.836 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:17:32.837 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:17:32.938 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:17:32.965 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:17:32.968 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:17:32.969 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:17:33.070 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:17:33.127 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:17:33.135 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:17:33.136 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:17:33.238 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:17:33.272 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:17:33.274 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:17:33.275 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:17:33.376 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:17:33.390 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 16 16 30461 30445 118.907 84.3125 0.0120651 0.00840131 2023-06-03T20:17:33.409 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:17:33.411 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:17:33.412 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:17:33.513 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:17:33.567 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:17:33.569 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:17:33.570 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:17:33.670 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:17:33.714 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:17:33.718 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:17:33.719 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:17:33.779 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:17:33 smithi089 bash[78452]: audit 1685823452.5788882 mon.a (mon.0) 1190 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:17:33.779 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:17:33 smithi089 bash[78452]: audit 1685823452.6052382 mon.a (mon.0) 1191 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:17:33.779 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:17:33 smithi089 bash[78452]: cluster 1685823452.8490264 mgr.y (mgr.34104) 1246 : cluster 0 pgmap v1240: 89 pgs: 20 active+clean+remapped, 14 active+undersized+degraded, 11 active+undersized, 44 active+clean; 1.5 GiB data, 6.9 GiB used, 529 GiB / 536 GiB avail; 120 MiB/s wr, 1.91k op/s; 12279/74901 objects degraded (16.394%); 6/74901 objects misplaced (0.008%) 2023-06-03T20:17:33.779 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:17:33 smithi089 bash[78452]: audit 1685823453.262971 mon.a (mon.0) 1192 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:17:33.779 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:17:33 smithi089 bash[78452]: audit 1685823453.2788305 mon.a (mon.0) 1193 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:17:33.780 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:17:33 smithi089 bash[75068]: audit 1685823452.5788882 mon.a (mon.0) 1190 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:17:33.780 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:17:33 smithi089 bash[75068]: audit 1685823452.6052382 mon.a (mon.0) 1191 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:17:33.780 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:17:33 smithi089 bash[75068]: cluster 1685823452.8490264 mgr.y (mgr.34104) 1246 : cluster 0 pgmap v1240: 89 pgs: 20 active+clean+remapped, 14 active+undersized+degraded, 11 active+undersized, 44 active+clean; 1.5 GiB data, 6.9 GiB used, 529 GiB / 536 GiB avail; 120 MiB/s wr, 1.91k op/s; 12279/74901 objects degraded (16.394%); 6/74901 objects misplaced (0.008%) 2023-06-03T20:17:33.780 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:17:33 smithi089 bash[75068]: audit 1685823453.262971 mon.a (mon.0) 1192 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:17:33.780 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:17:33 smithi089 bash[75068]: audit 1685823453.2788305 mon.a (mon.0) 1193 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:17:33.781 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/mon.a/config 2023-06-03T20:17:33.820 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:17:33.833 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:17:33 smithi164 bash[59462]: audit 1685823452.5788882 mon.a (mon.0) 1190 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:17:33.834 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:17:33 smithi164 bash[59462]: audit 1685823452.6052382 mon.a (mon.0) 1191 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:17:33.834 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:17:33 smithi164 bash[59462]: cluster 1685823452.8490264 mgr.y (mgr.34104) 1246 : cluster 0 pgmap v1240: 89 pgs: 20 active+clean+remapped, 14 active+undersized+degraded, 11 active+undersized, 44 active+clean; 1.5 GiB data, 6.9 GiB used, 529 GiB / 536 GiB avail; 120 MiB/s wr, 1.91k op/s; 12279/74901 objects degraded (16.394%); 6/74901 objects misplaced (0.008%) 2023-06-03T20:17:33.834 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:17:33 smithi164 bash[59462]: audit 1685823453.262971 mon.a (mon.0) 1192 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:17:33.834 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:17:33 smithi164 bash[59462]: audit 1685823453.2788305 mon.a (mon.0) 1193 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:17:33.856 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:17:33.859 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:17:33.859 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:17:33.960 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:17:33.993 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:17:33.996 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:17:33.997 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:17:34.098 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:17:34.132 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:17:34.135 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:17:34.136 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:17:34.237 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:17:34.273 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:17:34.276 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:17:34.277 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:17:34.378 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:17:34.391 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 17 16 31999 31983 117.567 96.125 0.0129368 0.00849763 2023-06-03T20:17:34.408 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:17:34.412 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:17:34.413 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:17:34.514 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:17:34.542 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:17:34.545 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:17:34.545 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:17:34.646 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:17:34.683 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:17:34.686 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:17:34.686 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:17:34.787 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:17:34.822 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:17:34.825 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:17:34.826 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:17:34.926 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:17:34.937 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:17:34 smithi164 bash[59462]: audit 1685823453.9113965 mon.c (mon.2) 550 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-03T20:17:34.937 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:17:34 smithi164 bash[59462]: audit 1685823453.9135962 mon.c (mon.2) 551 : audit 1 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-03T20:17:34.937 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:17:34 smithi164 bash[59462]: audit 1685823453.9243364 mon.a (mon.0) 1194 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:17:34.961 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:17:34.963 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:17:34.963 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:17:35.064 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:17:35.091 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:17:35.091 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:17:34 smithi089 bash[75068]: audit 1685823453.9113965 mon.c (mon.2) 550 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-03T20:17:35.092 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:17:34 smithi089 bash[75068]: audit 1685823453.9135962 mon.c (mon.2) 551 : audit 1 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-03T20:17:35.092 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:17:34 smithi089 bash[75068]: audit 1685823453.9243364 mon.a (mon.0) 1194 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:17:35.092 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:17:34 smithi089 bash[78452]: audit 1685823453.9113965 mon.c (mon.2) 550 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-03T20:17:35.092 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:17:34 smithi089 bash[78452]: audit 1685823453.9135962 mon.c (mon.2) 551 : audit 1 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-03T20:17:35.092 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:17:34 smithi089 bash[78452]: audit 1685823453.9243364 mon.a (mon.0) 1194 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:17:35.093 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:17:35.093 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:17:35.194 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:17:35.225 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:17:35.230 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:17:35.231 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:17:35.332 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:17:35.367 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:17:35.372 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:17:35.373 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:17:35.391 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 18 16 33677 33661 116.861 104.875 0.0141855 0.00855169 2023-06-03T20:17:35.474 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:17:35.511 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:17:35.513 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:17:35.513 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:17:35.614 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:17:35.658 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:17:35.660 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:17:35.661 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:17:35.762 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:17:35.793 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:17:35.796 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:17:35.796 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:17:35.897 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:17:35.930 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:17:35.932 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:17:35.933 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:17:36.021 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:17:35 smithi089 bash[75068]: cluster 1685823454.8498318 mgr.y (mgr.34104) 1247 : cluster 0 pgmap v1241: 89 pgs: 20 active+clean+remapped, 14 active+undersized+degraded, 11 active+undersized, 44 active+clean; 1.7 GiB data, 7.6 GiB used, 529 GiB / 536 GiB avail; 102 MiB/s wr, 1.63k op/s; 14224/85122 objects degraded (16.710%); 6/85122 objects misplaced (0.007%) 2023-06-03T20:17:36.022 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:17:35 smithi089 bash[78452]: cluster 1685823454.8498318 mgr.y (mgr.34104) 1247 : cluster 0 pgmap v1241: 89 pgs: 20 active+clean+remapped, 14 active+undersized+degraded, 11 active+undersized, 44 active+clean; 1.7 GiB data, 7.6 GiB used, 529 GiB / 536 GiB avail; 102 MiB/s wr, 1.63k op/s; 14224/85122 objects degraded (16.710%); 6/85122 objects misplaced (0.007%) 2023-06-03T20:17:36.034 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:17:36.043 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:17:35 smithi164 bash[59462]: cluster 1685823454.8498318 mgr.y (mgr.34104) 1247 : cluster 0 pgmap v1241: 89 pgs: 20 active+clean+remapped, 14 active+undersized+degraded, 11 active+undersized, 44 active+clean; 1.7 GiB data, 7.6 GiB used, 529 GiB / 536 GiB avail; 102 MiB/s wr, 1.63k op/s; 14224/85122 objects degraded (16.710%); 6/85122 objects misplaced (0.007%) 2023-06-03T20:17:36.067 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:17:36.070 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:17:36.070 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:17:36.171 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:17:36.206 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:17:36.209 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:17:36.210 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:17:36.279 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 20:17:36 smithi089 bash[71163]: ::ffff:172.21.15.164 - - [03/Jun/2023:20:17:36] "GET /metrics HTTP/1.1" 200 228869 "" "Prometheus/2.33.4" 2023-06-03T20:17:36.311 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:17:36.343 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:17:36.345 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:17:36.346 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:17:36.391 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 19 16 35329 35313 116.144 103.25 0.0117454 0.0086028 2023-06-03T20:17:36.446 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:17:36.481 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:17:36.484 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:17:36.485 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:17:36.586 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:17:36.619 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:17:36.622 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:17:36.623 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:17:36.723 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:17:36.757 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:17:36.760 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:17:36.761 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:17:36.862 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:17:36.899 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:17:36.902 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:17:36.903 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:17:37.004 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:17:37.029 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:17:36 smithi089 bash[75068]: cluster 1685823456.902231 mon.a (mon.0) 1195 : cluster 3 Health check update: Degraded data redundancy: 16150/98223 objects degraded (16.442%), 14 pgs degraded, 16 pgs undersized (PG_DEGRADED) 2023-06-03T20:17:37.029 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:17:36 smithi089 bash[78452]: cluster 1685823456.902231 mon.a (mon.0) 1195 : cluster 3 Health check update: Degraded data redundancy: 16150/98223 objects degraded (16.442%), 14 pgs degraded, 16 pgs undersized (PG_DEGRADED) 2023-06-03T20:17:37.036 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:17:37.038 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:17:37.039 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:17:37.052 INFO:teuthology.orchestra.run.smithi164.stderr:Inferring config /var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/mon.b/config 2023-06-03T20:17:37.140 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:17:37.170 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:17:37.174 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:17:37.174 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:17:37.275 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:17:37.306 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:17:37.310 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:17:37.311 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:17:37.333 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:17:36 smithi164 bash[59462]: cluster 1685823456.902231 mon.a (mon.0) 1195 : cluster 3 Health check update: Degraded data redundancy: 16150/98223 objects degraded (16.442%), 14 pgs degraded, 16 pgs undersized (PG_DEGRADED) 2023-06-03T20:17:37.391 INFO:tasks.radosbench.radosbench.0.smithi089.stdout:2023-06-03T20:17:37.387206+0000 min lat: 0.00129872 max lat: 0.870111 avg lat: 0.00866111 2023-06-03T20:17:37.391 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-06-03T20:17:37.392 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 20 16 36942 36926 115.377 100.812 0.00789646 0.00866111 2023-06-03T20:17:37.411 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:17:37.439 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:17:37.441 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:17:37.441 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:17:37.542 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:17:37.572 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:17:37.574 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:17:37.574 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:17:37.675 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:17:37.711 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:17:37.715 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:17:37.716 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:17:37.817 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:17:37.879 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:17:37.883 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:17:37.884 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:17:37.985 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:17:38.018 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:17:38.021 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:17:38.021 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:17:38.122 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:17:38.131 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:17:37 smithi089 bash[75068]: cluster 1685823456.8506203 mgr.y (mgr.34104) 1248 : cluster 0 pgmap v1242: 89 pgs: 20 active+clean+remapped, 14 active+undersized+degraded, 11 active+undersized, 44 active+clean; 2.0 GiB data, 8.1 GiB used, 528 GiB / 536 GiB avail; 117 MiB/s wr, 1.86k op/s; 16150/98223 objects degraded (16.442%); 6/98223 objects misplaced (0.006%) 2023-06-03T20:17:38.132 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:17:37 smithi089 bash[75068]: audit 1685823457.1285615 mon.c (mon.2) 552 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T20:17:38.132 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:17:37 smithi089 bash[78452]: cluster 1685823456.8506203 mgr.y (mgr.34104) 1248 : cluster 0 pgmap v1242: 89 pgs: 20 active+clean+remapped, 14 active+undersized+degraded, 11 active+undersized, 44 active+clean; 2.0 GiB data, 8.1 GiB used, 528 GiB / 536 GiB avail; 117 MiB/s wr, 1.86k op/s; 16150/98223 objects degraded (16.442%); 6/98223 objects misplaced (0.006%) 2023-06-03T20:17:38.132 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:17:37 smithi089 bash[78452]: audit 1685823457.1285615 mon.c (mon.2) 552 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T20:17:38.153 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:17:38.156 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:17:38.157 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:17:38.258 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:17:38.302 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:17:38.305 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:17:38.306 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:17:38.333 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:17:37 smithi164 bash[59462]: cluster 1685823456.8506203 mgr.y (mgr.34104) 1248 : cluster 0 pgmap v1242: 89 pgs: 20 active+clean+remapped, 14 active+undersized+degraded, 11 active+undersized, 44 active+clean; 2.0 GiB data, 8.1 GiB used, 528 GiB / 536 GiB avail; 117 MiB/s wr, 1.86k op/s; 16150/98223 objects degraded (16.442%); 6/98223 objects misplaced (0.006%) 2023-06-03T20:17:38.333 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:17:37 smithi164 bash[59462]: audit 1685823457.1285615 mon.c (mon.2) 552 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T20:17:38.391 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 21 16 38449 38433 114.368 94.1875 0.0126584 0.00873771 2023-06-03T20:17:38.407 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:17:38.436 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:17:38.438 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:17:38.439 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:17:38.540 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:17:38.569 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:17:38.572 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:17:38.573 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:17:38.674 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:17:38.705 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:17:38.713 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:17:38.714 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:17:38.815 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:17:38.844 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:17:38.847 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:17:38.848 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:17:38.948 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:17:38.976 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:17:38.978 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:17:38.978 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:17:39.079 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:17:39.113 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:17:39.115 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:17:39.116 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:17:39.216 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:17:39.251 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:17:39.252 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:17:39.253 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:17:39.354 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:17:39.382 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:17:39.384 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:17:39.385 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:17:39.391 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 22 16 40019 40003 113.629 98.125 0.00821989 0.00879407 2023-06-03T20:17:39.485 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:17:39.519 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:17:39.523 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:17:39.524 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:17:39.625 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:17:39.656 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:17:39.658 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:17:39.659 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:17:39.760 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:17:39.792 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:17:39.796 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:17:39.796 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:17:39.897 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:17:39.925 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:17:39.927 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:17:39.927 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:17:40.028 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:17:40.060 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:17:40.063 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:17:39 smithi164 bash[59462]: cluster 1685823458.8514338 mgr.y (mgr.34104) 1249 : cluster 0 pgmap v1243: 89 pgs: 20 active+clean+remapped, 14 active+undersized+degraded, 11 active+undersized, 44 active+clean; 2.2 GiB data, 8.9 GiB used, 528 GiB / 536 GiB avail; 107 MiB/s wr, 1.72k op/s; 18083/108747 objects degraded (16.629%); 6/108747 objects misplaced (0.006%) 2023-06-03T20:17:40.063 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:17:40.064 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:17:40.165 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:17:40.195 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:17:40.198 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:17:40.198 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:17:40.279 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:17:39 smithi089 bash[78452]: cluster 1685823458.8514338 mgr.y (mgr.34104) 1249 : cluster 0 pgmap v1243: 89 pgs: 20 active+clean+remapped, 14 active+undersized+degraded, 11 active+undersized, 44 active+clean; 2.2 GiB data, 8.9 GiB used, 528 GiB / 536 GiB avail; 107 MiB/s wr, 1.72k op/s; 18083/108747 objects degraded (16.629%); 6/108747 objects misplaced (0.006%) 2023-06-03T20:17:40.279 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:17:39 smithi089 bash[75068]: cluster 1685823458.8514338 mgr.y (mgr.34104) 1249 : cluster 0 pgmap v1243: 89 pgs: 20 active+clean+remapped, 14 active+undersized+degraded, 11 active+undersized, 44 active+clean; 2.2 GiB data, 8.9 GiB used, 528 GiB / 536 GiB avail; 107 MiB/s wr, 1.72k op/s; 18083/108747 objects degraded (16.629%); 6/108747 objects misplaced (0.006%) 2023-06-03T20:17:40.299 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:17:40.333 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:17:40.338 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:17:40.339 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:17:40.391 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 23 16 41369 41353 112.357 84.375 0.00742902 0.00889426 2023-06-03T20:17:40.440 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:17:40.470 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:17:40.473 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:17:40.473 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:17:40.574 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:17:40.602 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:17:40.606 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:17:40.606 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:17:40.707 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:17:40.743 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:17:40.745 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:17:40.746 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:17:40.847 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:17:40.885 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:17:40.887 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:17:40.888 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:17:40.989 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:17:41.024 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:17:41.027 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:17:41.028 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:17:41.130 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:17:41.162 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:17:41.165 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:17:41.166 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:17:41.268 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:17:41.307 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:17:41.310 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:17:41.310 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:17:41.391 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 24 16 42831 42815 111.482 91.375 0.00883653 0.00896486 2023-06-03T20:17:41.411 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:17:41.439 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:17:41.442 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:17:41.442 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:17:41.543 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:17:41.571 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:17:41.574 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:17:41.575 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:17:41.676 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:17:41.705 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:17:41.708 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:17:41.709 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:17:41.810 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:17:41.844 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:17:41.848 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:17:41.848 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:17:41.949 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:17:41.980 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:17:41.982 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:17:41.982 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:17:42.083 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:17:42.092 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:17:41 smithi089 bash[75068]: cluster 1685823460.8526514 mgr.y (mgr.34104) 1250 : cluster 0 pgmap v1244: 89 pgs: 20 active+clean+remapped, 14 active+undersized+degraded, 11 active+undersized, 44 active+clean; 2.4 GiB data, 9.2 GiB used, 527 GiB / 536 GiB avail; 98 MiB/s wr, 1.56k op/s; 19600/116211 objects degraded (16.866%); 6/116211 objects misplaced (0.005%) 2023-06-03T20:17:42.093 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:17:41 smithi089 bash[78452]: cluster 1685823460.8526514 mgr.y (mgr.34104) 1250 : cluster 0 pgmap v1244: 89 pgs: 20 active+clean+remapped, 14 active+undersized+degraded, 11 active+undersized, 44 active+clean; 2.4 GiB data, 9.2 GiB used, 527 GiB / 536 GiB avail; 98 MiB/s wr, 1.56k op/s; 19600/116211 objects degraded (16.866%); 6/116211 objects misplaced (0.005%) 2023-06-03T20:17:42.118 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:17:42.123 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:17:42.124 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:17:42.225 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:17:42.249 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:17:41 smithi164 bash[59462]: cluster 1685823460.8526514 mgr.y (mgr.34104) 1250 : cluster 0 pgmap v1244: 89 pgs: 20 active+clean+remapped, 14 active+undersized+degraded, 11 active+undersized, 44 active+clean; 2.4 GiB data, 9.2 GiB used, 527 GiB / 536 GiB avail; 98 MiB/s wr, 1.56k op/s; 19600/116211 objects degraded (16.866%); 6/116211 objects misplaced (0.005%) 2023-06-03T20:17:42.257 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:17:42.261 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:17:42.262 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:17:42.363 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:17:42.391 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 25 16 44268 44252 110.615 89.8125 0.00504962 0.00903413 2023-06-03T20:17:42.395 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:17:42.398 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:17:42.398 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:17:42.499 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:17:42.527 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:17:42.530 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:17:42.531 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:17:42.545 INFO:teuthology.orchestra.run.smithi164.stdout:{ 2023-06-03T20:17:42.546 INFO:teuthology.orchestra.run.smithi164.stdout: "success": "filestore_inject_stall = '3' (not observed, change may require restart) " 2023-06-03T20:17:42.546 INFO:teuthology.orchestra.run.smithi164.stdout:} 2023-06-03T20:17:42.631 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:17:42.663 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:17:42.666 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:17:42.667 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:17:42.768 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:17:42.807 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:17:42.811 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:17:42.811 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:17:42.912 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:17:43.183 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:17:42 smithi089 bash[75068]: cluster 1685823462.8179133 mon.a (mon.0) 1196 : cluster 3 Health check update: Degraded data redundancy: 19600/116211 objects degraded (16.866%), 14 pgs degraded, 16 pgs undersized (PG_DEGRADED) 2023-06-03T20:17:43.184 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:17:42 smithi089 bash[78452]: cluster 1685823462.8179133 mon.a (mon.0) 1196 : cluster 3 Health check update: Degraded data redundancy: 19600/116211 objects degraded (16.866%), 14 pgs degraded, 16 pgs undersized (PG_DEGRADED) 2023-06-03T20:17:43.184 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:17:42 smithi164 bash[59462]: cluster 1685823462.8179133 mon.a (mon.0) 1196 : cluster 3 Health check update: Degraded data redundancy: 19600/116211 objects degraded (16.866%), 14 pgs degraded, 16 pgs undersized (PG_DEGRADED) 2023-06-03T20:17:43.227 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:17:43.229 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:17:43.230 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:17:43.331 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:17:43.358 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:17:43.361 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:17:43.361 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:17:43.391 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 26 15 45379 45364 109.034 69.5 0.00591803 0.00916602 2023-06-03T20:17:43.462 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:17:43.494 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:17:43.498 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:17:43.499 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:17:43.600 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:17:43.631 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:17:43.636 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:17:43.636 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:17:43.737 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:17:43.768 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:17:43.773 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:17:43.773 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:17:43.874 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:17:43.905 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:17:43.907 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:17:43.908 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:17:44.009 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:17:44.044 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:17:44.046 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:17:44.047 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:17:44.083 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:17:43 smithi164 bash[59462]: cluster 1685823462.8534987 mgr.y (mgr.34104) 1251 : cluster 0 pgmap v1245: 89 pgs: 20 active+clean+remapped, 14 active+undersized+degraded, 11 active+undersized, 44 active+clean; 2.5 GiB data, 9.4 GiB used, 527 GiB / 536 GiB avail; 93 MiB/s wr, 1.49k op/s; 20063/121758 objects degraded (16.478%); 6/121758 objects misplaced (0.005%) 2023-06-03T20:17:44.148 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:17:44.156 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:17:43 smithi089 bash[75068]: cluster 1685823462.8534987 mgr.y (mgr.34104) 1251 : cluster 0 pgmap v1245: 89 pgs: 20 active+clean+remapped, 14 active+undersized+degraded, 11 active+undersized, 44 active+clean; 2.5 GiB data, 9.4 GiB used, 527 GiB / 536 GiB avail; 93 MiB/s wr, 1.49k op/s; 20063/121758 objects degraded (16.478%); 6/121758 objects misplaced (0.005%) 2023-06-03T20:17:44.156 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:17:43 smithi089 bash[78452]: cluster 1685823462.8534987 mgr.y (mgr.34104) 1251 : cluster 0 pgmap v1245: 89 pgs: 20 active+clean+remapped, 14 active+undersized+degraded, 11 active+undersized, 44 active+clean; 2.5 GiB data, 9.4 GiB used, 527 GiB / 536 GiB avail; 93 MiB/s wr, 1.49k op/s; 20063/121758 objects degraded (16.478%); 6/121758 objects misplaced (0.005%) 2023-06-03T20:17:44.183 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:17:44.185 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:17:44.186 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:17:44.287 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:17:44.312 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:17:44.314 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:17:44.315 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:17:44.391 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 27 16 46711 46695 108.076 83.1875 0.00868164 0.00923302 2023-06-03T20:17:44.416 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:17:44.450 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:17:44.453 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:17:44.454 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:17:44.554 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:17:44.587 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:17:44.590 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:17:44.591 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:17:44.692 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:17:44.725 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:17:44.728 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:17:44.728 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:17:44.829 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:17:44.861 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:17:44.866 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:17:44.867 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:17:44.968 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:17:45.022 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:17:45.027 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:17:45.028 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:17:45.129 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:17:45.158 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:17:45.161 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:17:45.161 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:17:45.262 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:17:45.305 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:17:45.312 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:17:45.312 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:17:45.392 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 28 16 47980 47964 107.049 79.3125 0.0133674 0.00933532 2023-06-03T20:17:45.413 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:17:45.442 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:17:45.445 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:17:45.445 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:17:45.546 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:17:45.581 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:17:45.587 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:17:45.587 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:17:45.688 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:17:45.859 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:17:45.862 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:17:45.864 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:17:45.965 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:17:45.997 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:17:45.999 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:17:46.000 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:17:46.101 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:17:46.110 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:17:45 smithi164 bash[59462]: cluster 1685823464.8557744 mgr.y (mgr.34104) 1252 : cluster 0 pgmap v1246: 89 pgs: 20 active+clean+remapped, 14 active+undersized+degraded, 11 active+undersized, 44 active+clean; 2.7 GiB data, 10 GiB used, 526 GiB / 536 GiB avail; 97 MiB/s wr, 1.55k op/s; 21758/130743 objects degraded (16.642%); 6/130743 objects misplaced (0.005%) 2023-06-03T20:17:46.137 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:17:46.139 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:17:46.139 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:17:46.231 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:17:45 smithi089 bash[75068]: cluster 1685823464.8557744 mgr.y (mgr.34104) 1252 : cluster 0 pgmap v1246: 89 pgs: 20 active+clean+remapped, 14 active+undersized+degraded, 11 active+undersized, 44 active+clean; 2.7 GiB data, 10 GiB used, 526 GiB / 536 GiB avail; 97 MiB/s wr, 1.55k op/s; 21758/130743 objects degraded (16.642%); 6/130743 objects misplaced (0.005%) 2023-06-03T20:17:46.232 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 20:17:46 smithi089 bash[71163]: ::ffff:172.21.15.164 - - [03/Jun/2023:20:17:46] "GET /metrics HTTP/1.1" 200 228926 "" "Prometheus/2.33.4" 2023-06-03T20:17:46.233 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:17:46 smithi089 bash[78452]: cluster 1685823464.8557744 mgr.y (mgr.34104) 1252 : cluster 0 pgmap v1246: 89 pgs: 20 active+clean+remapped, 14 active+undersized+degraded, 11 active+undersized, 44 active+clean; 2.7 GiB data, 10 GiB used, 526 GiB / 536 GiB avail; 97 MiB/s wr, 1.55k op/s; 21758/130743 objects degraded (16.642%); 6/130743 objects misplaced (0.005%) 2023-06-03T20:17:46.240 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:17:46.270 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:17:46.274 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:17:46.274 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:17:46.375 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:17:46.392 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 29 16 49340 49324 106.288 85 0.0145373 0.00940237 2023-06-03T20:17:46.407 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:17:46.410 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:17:46.412 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:17:46.513 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:17:46.537 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:17:46.539 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:17:46.540 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:17:46.641 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:17:46.671 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:17:46.676 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:17:46.677 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:17:46.778 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:17:46.811 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:17:46.813 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:17:46.813 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:17:46.914 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:17:46.949 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:17:46.953 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:17:46.954 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:17:47.054 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:17:47.092 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:17:47.095 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:17:47.095 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:17:47.196 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:17:47.226 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:17:47.231 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:17:47.231 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:17:47.332 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:17:47.367 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:17:47.369 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:17:47.370 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:17:47.392 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 30 16 50620 50604 105.412 80 0.0103514 0.00948087 2023-06-03T20:17:47.471 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:17:47.513 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:17:47.516 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:17:47.517 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:17:47.618 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:17:47.657 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:17:47.660 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:17:47.661 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:17:47.762 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:17:47.809 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:17:47.813 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:17:47.814 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:17:47.916 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:17:47.948 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:17:47.951 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:17:47.952 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:17:48.053 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:17:48.090 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:17:48.094 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:17:48.094 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:17:48.195 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:17:48.209 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:17:47 smithi089 bash[78452]: cluster 1685823466.8564594 mgr.y (mgr.34104) 1253 : cluster 0 pgmap v1247: 89 pgs: 20 active+clean+remapped, 14 active+undersized+degraded, 11 active+undersized, 44 active+clean; 2.9 GiB data, 10 GiB used, 526 GiB / 536 GiB avail; 98 MiB/s wr, 1.57k op/s; 23471/141855 objects degraded (16.546%); 6/141855 objects misplaced (0.004%) 2023-06-03T20:17:48.210 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:17:47 smithi089 bash[78452]: cluster 1685823467.819045 mon.a (mon.0) 1197 : cluster 3 Health check update: Degraded data redundancy: 23471/141855 objects degraded (16.546%), 14 pgs degraded, 16 pgs undersized (PG_DEGRADED) 2023-06-03T20:17:48.210 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:17:47 smithi089 bash[75068]: cluster 1685823466.8564594 mgr.y (mgr.34104) 1253 : cluster 0 pgmap v1247: 89 pgs: 20 active+clean+remapped, 14 active+undersized+degraded, 11 active+undersized, 44 active+clean; 2.9 GiB data, 10 GiB used, 526 GiB / 536 GiB avail; 98 MiB/s wr, 1.57k op/s; 23471/141855 objects degraded (16.546%); 6/141855 objects misplaced (0.004%) 2023-06-03T20:17:48.210 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:17:47 smithi089 bash[75068]: cluster 1685823467.819045 mon.a (mon.0) 1197 : cluster 3 Health check update: Degraded data redundancy: 23471/141855 objects degraded (16.546%), 14 pgs degraded, 16 pgs undersized (PG_DEGRADED) 2023-06-03T20:17:48.234 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:17:48.239 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:17:48.246 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:17:48.333 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:17:47 smithi164 bash[59462]: cluster 1685823466.8564594 mgr.y (mgr.34104) 1253 : cluster 0 pgmap v1247: 89 pgs: 20 active+clean+remapped, 14 active+undersized+degraded, 11 active+undersized, 44 active+clean; 2.9 GiB data, 10 GiB used, 526 GiB / 536 GiB avail; 98 MiB/s wr, 1.57k op/s; 23471/141855 objects degraded (16.546%); 6/141855 objects misplaced (0.004%) 2023-06-03T20:17:48.334 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:17:47 smithi164 bash[59462]: cluster 1685823467.819045 mon.a (mon.0) 1197 : cluster 3 Health check update: Degraded data redundancy: 23471/141855 objects degraded (16.546%), 14 pgs degraded, 16 pgs undersized (PG_DEGRADED) 2023-06-03T20:17:48.347 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:17:48.384 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:17:48.387 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:17:48.388 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:17:48.392 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 31 16 51951 51935 104.695 83.1875 0.0117285 0.00954562 2023-06-03T20:17:48.489 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:17:48.520 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:17:48.524 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:17:48.525 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:17:48.528 INFO:tasks.thrashosds.thrasher:in_osds: [0, 2, 3, 4, 5, 7] out_osds: [1, 6] dead_osds: [4] live_osds: [0, 1, 2, 3, 5, 6, 7] 2023-06-03T20:17:48.528 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-06-03T20:17:48.528 INFO:tasks.thrashosds.thrasher:inject_pause on osd.0 2023-06-03T20:17:48.528 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2023-06-03T20:17:48.529 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2023-06-03T20:17:48.529 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph daemon osd.0 config set filestore_inject_stall 3 2023-06-03T20:17:48.626 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:17:48.661 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:17:48.663 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:17:48.664 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:17:48.765 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:17:48.801 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:17:48.807 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:17:48.808 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:17:48.909 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:17:48.942 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:17:48.945 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:17:48.946 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:17:49.047 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:17:49.085 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:17:49.088 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:17:49.089 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:17:49.190 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:17:49.221 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:17:49.224 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:17:49.225 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:17:49.325 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:17:49.359 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:17:49.365 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:17:49.366 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:17:49.392 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 32 16 52877 52861 103.231 57.875 0.00508536 0.00967992 2023-06-03T20:17:49.467 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:17:49.506 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:17:49.509 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:17:49.510 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:17:49.611 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:17:49.641 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:17:49.645 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:17:49.646 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:17:49.747 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:17:49.789 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:17:49.796 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:17:49.797 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:17:49.898 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:17:49.931 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:17:49.934 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:17:49.935 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:17:50.036 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:17:50.065 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:17:50.068 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:17:50.069 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:17:50.170 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:17:50.211 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:17:50.213 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:17:50.213 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:17:50.278 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:17:49 smithi089 bash[78452]: cluster 1685823468.8571622 mgr.y (mgr.34104) 1254 : cluster 0 pgmap v1248: 89 pgs: 20 active+clean+remapped, 14 active+undersized+degraded, 11 active+undersized, 44 active+clean; 3.1 GiB data, 11 GiB used, 525 GiB / 536 GiB avail; 91 MiB/s wr, 1.45k op/s; 25175/150564 objects degraded (16.720%); 6/150564 objects misplaced (0.004%) 2023-06-03T20:17:50.279 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:17:49 smithi089 bash[75068]: cluster 1685823468.8571622 mgr.y (mgr.34104) 1254 : cluster 0 pgmap v1248: 89 pgs: 20 active+clean+remapped, 14 active+undersized+degraded, 11 active+undersized, 44 active+clean; 3.1 GiB data, 11 GiB used, 525 GiB / 536 GiB avail; 91 MiB/s wr, 1.45k op/s; 25175/150564 objects degraded (16.720%); 6/150564 objects misplaced (0.004%) 2023-06-03T20:17:50.314 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:17:50.327 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:17:49 smithi164 bash[59462]: cluster 1685823468.8571622 mgr.y (mgr.34104) 1254 : cluster 0 pgmap v1248: 89 pgs: 20 active+clean+remapped, 14 active+undersized+degraded, 11 active+undersized, 44 active+clean; 3.1 GiB data, 11 GiB used, 525 GiB / 536 GiB avail; 91 MiB/s wr, 1.45k op/s; 25175/150564 objects degraded (16.720%); 6/150564 objects misplaced (0.004%) 2023-06-03T20:17:50.351 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:17:50.354 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:17:50.355 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:17:50.392 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 33 15 53758 53743 101.774 55.125 0.0106714 0.00981934 2023-06-03T20:17:50.429 INFO:teuthology.orchestra.run.smithi089.stdout:{ 2023-06-03T20:17:50.429 INFO:teuthology.orchestra.run.smithi089.stdout: "ops": [ 2023-06-03T20:17:50.429 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:17:50.429 INFO:teuthology.orchestra.run.smithi089.stdout: "description": "osd_repop(client.46373.0:53777 12.9 e224/217 12:9e30570c:::benchmark_data_smithi089_191647_object53776:head v 224'3294, mlcod=224'3288)", 2023-06-03T20:17:50.430 INFO:teuthology.orchestra.run.smithi089.stdout: "initiated_at": "2023-06-03T20:17:50.407344+0000", 2023-06-03T20:17:50.430 INFO:teuthology.orchestra.run.smithi089.stdout: "age": 0.013812988999999999, 2023-06-03T20:17:50.430 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.01384809, 2023-06-03T20:17:50.430 INFO:teuthology.orchestra.run.smithi089.stdout: "type_data": { 2023-06-03T20:17:50.430 INFO:teuthology.orchestra.run.smithi089.stdout: "flag_point": "started", 2023-06-03T20:17:50.430 INFO:teuthology.orchestra.run.smithi089.stdout: "events": [ 2023-06-03T20:17:50.430 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:17:50.431 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "initiated", 2023-06-03T20:17:50.431 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:17:50.407344+0000", 2023-06-03T20:17:50.431 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0 2023-06-03T20:17:50.431 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:17:50.431 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:17:50.431 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "throttled", 2023-06-03T20:17:50.431 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:17:50.407344+0000", 2023-06-03T20:17:50.432 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0 2023-06-03T20:17:50.432 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:17:50.432 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:17:50.432 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "header_read", 2023-06-03T20:17:50.432 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:17:50.407343+0000", 2023-06-03T20:17:50.432 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 4294967295.999999 2023-06-03T20:17:50.432 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:17:50.432 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:17:50.433 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "all_read", 2023-06-03T20:17:50.433 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:17:50.407377+0000", 2023-06-03T20:17:50.433 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 3.379e-05 2023-06-03T20:17:50.433 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:17:50.433 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:17:50.433 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "dispatched", 2023-06-03T20:17:50.434 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:17:50.407390+0000", 2023-06-03T20:17:50.434 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 1.3266e-05 2023-06-03T20:17:50.434 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:17:50.434 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:17:50.434 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "queued_for_pg", 2023-06-03T20:17:50.434 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:17:50.407398+0000", 2023-06-03T20:17:50.434 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 7.7940000000000003e-06 2023-06-03T20:17:50.435 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:17:50.435 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:17:50.435 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "reached_pg", 2023-06-03T20:17:50.435 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:17:50.411167+0000", 2023-06-03T20:17:50.436 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.0037686199999999999 2023-06-03T20:17:50.436 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:17:50.436 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:17:50.436 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "started", 2023-06-03T20:17:50.436 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:17:50.411186+0000", 2023-06-03T20:17:50.437 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 1.9412999999999999e-05 2023-06-03T20:17:50.437 INFO:teuthology.orchestra.run.smithi089.stdout: } 2023-06-03T20:17:50.437 INFO:teuthology.orchestra.run.smithi089.stdout: ] 2023-06-03T20:17:50.437 INFO:teuthology.orchestra.run.smithi089.stdout: } 2023-06-03T20:17:50.437 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:17:50.438 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:17:50.438 INFO:teuthology.orchestra.run.smithi089.stdout: "description": "osd_repop(client.46373.0:53784 12.6 e224/217 12:61c54c34:::benchmark_data_smithi089_191647_object53783:head v 224'3359, mlcod=224'3356)", 2023-06-03T20:17:50.438 INFO:teuthology.orchestra.run.smithi089.stdout: "initiated_at": "2023-06-03T20:17:50.410856+0000", 2023-06-03T20:17:50.438 INFO:teuthology.orchestra.run.smithi089.stdout: "age": 0.010301233, 2023-06-03T20:17:50.438 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.010413047, 2023-06-03T20:17:50.438 INFO:teuthology.orchestra.run.smithi089.stdout: "type_data": { 2023-06-03T20:17:50.438 INFO:teuthology.orchestra.run.smithi089.stdout: "flag_point": "started", 2023-06-03T20:17:50.438 INFO:teuthology.orchestra.run.smithi089.stdout: "events": [ 2023-06-03T20:17:50.439 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:17:50.439 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "initiated", 2023-06-03T20:17:50.439 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:17:50.410856+0000", 2023-06-03T20:17:50.439 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0 2023-06-03T20:17:50.439 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:17:50.439 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:17:50.439 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "throttled", 2023-06-03T20:17:50.439 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:17:50.410856+0000", 2023-06-03T20:17:50.440 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0 2023-06-03T20:17:50.440 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:17:50.440 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:17:50.440 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "header_read", 2023-06-03T20:17:50.440 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:17:50.410856+0000", 2023-06-03T20:17:50.441 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 4294967295.9999995 2023-06-03T20:17:50.441 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:17:50.441 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:17:50.441 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "all_read", 2023-06-03T20:17:50.441 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:17:50.410881+0000", 2023-06-03T20:17:50.442 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 2.5602999999999999e-05 2023-06-03T20:17:50.442 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:17:50.442 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:17:50.442 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "dispatched", 2023-06-03T20:17:50.442 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:17:50.410888+0000", 2023-06-03T20:17:50.442 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 7.1430000000000001e-06 2023-06-03T20:17:50.443 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:17:50.443 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:17:50.443 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "queued_for_pg", 2023-06-03T20:17:50.443 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:17:50.410894+0000", 2023-06-03T20:17:50.443 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 5.5090000000000001e-06 2023-06-03T20:17:50.443 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:17:50.444 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:17:50.444 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "reached_pg", 2023-06-03T20:17:50.444 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:17:50.412783+0000", 2023-06-03T20:17:50.444 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.0018891629999999999 2023-06-03T20:17:50.444 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:17:50.444 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:17:50.444 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "started", 2023-06-03T20:17:50.445 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:17:50.412805+0000", 2023-06-03T20:17:50.445 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 2.2280999999999999e-05 2023-06-03T20:17:50.445 INFO:teuthology.orchestra.run.smithi089.stdout: } 2023-06-03T20:17:50.445 INFO:teuthology.orchestra.run.smithi089.stdout: ] 2023-06-03T20:17:50.445 INFO:teuthology.orchestra.run.smithi089.stdout: } 2023-06-03T20:17:50.445 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:17:50.445 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:17:50.446 INFO:teuthology.orchestra.run.smithi089.stdout: "description": "osd_repop(client.46373.0:53782 12.9 e224/217 12:905a1c0d:::benchmark_data_smithi089_191647_object53781:head v 224'3295, mlcod=224'3290)", 2023-06-03T20:17:50.446 INFO:teuthology.orchestra.run.smithi089.stdout: "initiated_at": "2023-06-03T20:17:50.411123+0000", 2023-06-03T20:17:50.446 INFO:teuthology.orchestra.run.smithi089.stdout: "age": 0.010034319, 2023-06-03T20:17:50.446 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.010216302, 2023-06-03T20:17:50.446 INFO:teuthology.orchestra.run.smithi089.stdout: "type_data": { 2023-06-03T20:17:50.446 INFO:teuthology.orchestra.run.smithi089.stdout: "flag_point": "started", 2023-06-03T20:17:50.447 INFO:teuthology.orchestra.run.smithi089.stdout: "events": [ 2023-06-03T20:17:50.447 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:17:50.447 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "initiated", 2023-06-03T20:17:50.447 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:17:50.411123+0000", 2023-06-03T20:17:50.447 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0 2023-06-03T20:17:50.447 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:17:50.448 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:17:50.448 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "throttled", 2023-06-03T20:17:50.448 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:17:50.411123+0000", 2023-06-03T20:17:50.448 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0 2023-06-03T20:17:50.448 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:17:50.449 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:17:50.449 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "header_read", 2023-06-03T20:17:50.449 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:17:50.411122+0000", 2023-06-03T20:17:50.449 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 4294967295.999999 2023-06-03T20:17:50.449 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:17:50.449 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:17:50.449 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "all_read", 2023-06-03T20:17:50.449 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:17:50.411150+0000", 2023-06-03T20:17:50.450 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 2.7962000000000001e-05 2023-06-03T20:17:50.450 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:17:50.450 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:17:50.450 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "dispatched", 2023-06-03T20:17:50.450 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:17:50.411177+0000", 2023-06-03T20:17:50.450 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 2.7305e-05 2023-06-03T20:17:50.450 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:17:50.450 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:17:50.451 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "queued_for_pg", 2023-06-03T20:17:50.451 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:17:50.411185+0000", 2023-06-03T20:17:50.451 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 8.0229999999999992e-06 2023-06-03T20:17:50.451 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:17:50.451 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:17:50.451 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "reached_pg", 2023-06-03T20:17:50.451 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:17:50.413521+0000", 2023-06-03T20:17:50.452 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.00233563 2023-06-03T20:17:50.452 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:17:50.452 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:17:50.452 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "started", 2023-06-03T20:17:50.452 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:17:50.413541+0000", 2023-06-03T20:17:50.452 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 1.9544000000000001e-05 2023-06-03T20:17:50.452 INFO:teuthology.orchestra.run.smithi089.stdout: } 2023-06-03T20:17:50.452 INFO:teuthology.orchestra.run.smithi089.stdout: ] 2023-06-03T20:17:50.453 INFO:teuthology.orchestra.run.smithi089.stdout: } 2023-06-03T20:17:50.453 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:17:50.453 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:17:50.453 INFO:teuthology.orchestra.run.smithi089.stdout: "description": "osd_repop(client.46373.0:53783 12.5 e224/217 12:a8894ad6:::benchmark_data_smithi089_191647_object53782:head v 224'3325, mlcod=224'3322)", 2023-06-03T20:17:50.453 INFO:teuthology.orchestra.run.smithi089.stdout: "initiated_at": "2023-06-03T20:17:50.411194+0000", 2023-06-03T20:17:50.453 INFO:teuthology.orchestra.run.smithi089.stdout: "age": 0.0099635899999999996, 2023-06-03T20:17:50.453 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.010210954, 2023-06-03T20:17:50.453 INFO:teuthology.orchestra.run.smithi089.stdout: "type_data": { 2023-06-03T20:17:50.454 INFO:teuthology.orchestra.run.smithi089.stdout: "flag_point": "started", 2023-06-03T20:17:50.454 INFO:teuthology.orchestra.run.smithi089.stdout: "events": [ 2023-06-03T20:17:50.454 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:17:50.454 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "initiated", 2023-06-03T20:17:50.454 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:17:50.411194+0000", 2023-06-03T20:17:50.454 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0 2023-06-03T20:17:50.454 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:17:50.454 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:17:50.455 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "throttled", 2023-06-03T20:17:50.455 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:17:50.411194+0000", 2023-06-03T20:17:50.455 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0 2023-06-03T20:17:50.455 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:17:50.455 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:17:50.455 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "header_read", 2023-06-03T20:17:50.455 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:17:50.411193+0000", 2023-06-03T20:17:50.456 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 4294967295.9999995 2023-06-03T20:17:50.456 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:17:50.456 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:17:50.456 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "all_read", 2023-06-03T20:17:50.456 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:17:50.411219+0000", 2023-06-03T20:17:50.456 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 2.5570000000000001e-05 2023-06-03T20:17:50.456 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:17:50.457 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:17:50.457 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "dispatched", 2023-06-03T20:17:50.457 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:17:50.411231+0000", 2023-06-03T20:17:50.457 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 1.1968e-05 2023-06-03T20:17:50.457 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:17:50.457 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:17:50.457 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "queued_for_pg", 2023-06-03T20:17:50.457 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:17:50.411237+0000", 2023-06-03T20:17:50.458 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 6.5060000000000001e-06 2023-06-03T20:17:50.458 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:17:50.458 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:17:50.458 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "reached_pg", 2023-06-03T20:17:50.458 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:17:50.411276+0000", 2023-06-03T20:17:50.458 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 3.8318000000000001e-05 2023-06-03T20:17:50.458 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:17:50.458 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:17:50.459 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "started", 2023-06-03T20:17:50.459 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:17:50.411292+0000", 2023-06-03T20:17:50.459 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 1.6884e-05 2023-06-03T20:17:50.459 INFO:teuthology.orchestra.run.smithi089.stdout: } 2023-06-03T20:17:50.459 INFO:teuthology.orchestra.run.smithi089.stdout: ] 2023-06-03T20:17:50.459 INFO:teuthology.orchestra.run.smithi089.stdout: } 2023-06-03T20:17:50.459 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:17:50.460 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:17:50.460 INFO:teuthology.orchestra.run.smithi089.stdout: "description": "osd_op(client.46373.0:53788 12.1 12:8b710e5b:::benchmark_data_smithi089_191647_object53787:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected e224)", 2023-06-03T20:17:50.460 INFO:teuthology.orchestra.run.smithi089.stdout: "initiated_at": "2023-06-03T20:17:50.413452+0000", 2023-06-03T20:17:50.460 INFO:teuthology.orchestra.run.smithi089.stdout: "age": 0.0077050240000000004, 2023-06-03T20:17:50.460 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.0080193620000000004, 2023-06-03T20:17:50.460 INFO:teuthology.orchestra.run.smithi089.stdout: "type_data": { 2023-06-03T20:17:50.460 INFO:teuthology.orchestra.run.smithi089.stdout: "flag_point": "waiting for sub ops", 2023-06-03T20:17:50.461 INFO:teuthology.orchestra.run.smithi089.stdout: "client_info": { 2023-06-03T20:17:50.461 INFO:teuthology.orchestra.run.smithi089.stdout: "client": "client.46373", 2023-06-03T20:17:50.461 INFO:teuthology.orchestra.run.smithi089.stdout: "client_addr": "172.21.15.89:0/3737977806", 2023-06-03T20:17:50.461 INFO:teuthology.orchestra.run.smithi089.stdout: "tid": 53788 2023-06-03T20:17:50.461 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:17:50.461 INFO:teuthology.orchestra.run.smithi089.stdout: "events": [ 2023-06-03T20:17:50.461 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:17:50.461 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "initiated", 2023-06-03T20:17:50.462 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:17:50.413452+0000", 2023-06-03T20:17:50.462 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0 2023-06-03T20:17:50.462 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:17:50.463 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:17:50.463 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "throttled", 2023-06-03T20:17:50.463 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:17:50.413452+0000", 2023-06-03T20:17:50.464 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0 2023-06-03T20:17:50.464 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:17:50.464 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:17:50.464 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "header_read", 2023-06-03T20:17:50.464 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:17:50.413451+0000", 2023-06-03T20:17:50.465 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 4294967295.9999986 2023-06-03T20:17:50.465 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:17:50.465 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:17:50.465 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "all_read", 2023-06-03T20:17:50.465 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:17:50.413477+0000", 2023-06-03T20:17:50.465 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 2.6574999999999999e-05 2023-06-03T20:17:50.465 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:17:50.465 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:17:50.466 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "dispatched", 2023-06-03T20:17:50.466 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:17:50.413493+0000", 2023-06-03T20:17:50.466 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 1.5383000000000002e-05 2023-06-03T20:17:50.466 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:17:50.466 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:17:50.466 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "queued_for_pg", 2023-06-03T20:17:50.467 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:17:50.413502+0000", 2023-06-03T20:17:50.467 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 8.9109999999999999e-06 2023-06-03T20:17:50.467 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:17:50.467 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:17:50.467 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "reached_pg", 2023-06-03T20:17:50.467 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:17:50.413615+0000", 2023-06-03T20:17:50.468 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.00011340000000000001 2023-06-03T20:17:50.468 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:17:50.468 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:17:50.468 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "started", 2023-06-03T20:17:50.468 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:17:50.413766+0000", 2023-06-03T20:17:50.468 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.00015108600000000001 2023-06-03T20:17:50.469 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:17:50.469 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:17:50.469 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "waiting for subops from 5,7", 2023-06-03T20:17:50.469 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:17:50.413914+0000", 2023-06-03T20:17:50.469 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.000148429 2023-06-03T20:17:50.469 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:17:50.469 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:17:50.469 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:17:50.470 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:17:50.419119+0000", 2023-06-03T20:17:50.470 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.0052047350000000003 2023-06-03T20:17:50.470 INFO:teuthology.orchestra.run.smithi089.stdout: } 2023-06-03T20:17:50.470 INFO:teuthology.orchestra.run.smithi089.stdout: ] 2023-06-03T20:17:50.470 INFO:teuthology.orchestra.run.smithi089.stdout: } 2023-06-03T20:17:50.471 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:17:50.471 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:17:50.471 INFO:teuthology.orchestra.run.smithi089.stdout: "description": "osd_op(client.46373.0:53789 12.4 12:28e0f36e:::benchmark_data_smithi089_191647_object53788:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected e224)", 2023-06-03T20:17:50.471 INFO:teuthology.orchestra.run.smithi089.stdout: "initiated_at": "2023-06-03T20:17:50.413524+0000", 2023-06-03T20:17:50.471 INFO:teuthology.orchestra.run.smithi089.stdout: "age": 0.0076336529999999998, 2023-06-03T20:17:50.471 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.0080385630000000003, 2023-06-03T20:17:50.472 INFO:teuthology.orchestra.run.smithi089.stdout: "type_data": { 2023-06-03T20:17:50.472 INFO:teuthology.orchestra.run.smithi089.stdout: "flag_point": "waiting for sub ops", 2023-06-03T20:17:50.472 INFO:teuthology.orchestra.run.smithi089.stdout: "client_info": { 2023-06-03T20:17:50.473 INFO:teuthology.orchestra.run.smithi089.stdout: "client": "client.46373", 2023-06-03T20:17:50.473 INFO:teuthology.orchestra.run.smithi089.stdout: "client_addr": "172.21.15.89:0/3737977806", 2023-06-03T20:17:50.473 INFO:teuthology.orchestra.run.smithi089.stdout: "tid": 53789 2023-06-03T20:17:50.474 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:17:50.474 INFO:teuthology.orchestra.run.smithi089.stdout: "events": [ 2023-06-03T20:17:50.474 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:17:50.475 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "initiated", 2023-06-03T20:17:50.475 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:17:50.413524+0000", 2023-06-03T20:17:50.475 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0 2023-06-03T20:17:50.476 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:17:50.477 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:17:50.477 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "throttled", 2023-06-03T20:17:50.477 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:17:50.413524+0000", 2023-06-03T20:17:50.478 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0 2023-06-03T20:17:50.478 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:17:50.478 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:17:50.479 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "header_read", 2023-06-03T20:17:50.479 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:17:50.413523+0000", 2023-06-03T20:17:50.480 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 4294967295.9999995 2023-06-03T20:17:50.480 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:17:50.480 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:17:50.481 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "all_read", 2023-06-03T20:17:50.481 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:17:50.413547+0000", 2023-06-03T20:17:50.482 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 2.3964999999999998e-05 2023-06-03T20:17:50.482 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:17:50.483 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:17:50.484 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "dispatched", 2023-06-03T20:17:50.484 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:17:50.413555+0000", 2023-06-03T20:17:50.485 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 8.2770000000000005e-06 2023-06-03T20:17:50.485 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:17:50.486 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:17:50.486 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "queued_for_pg", 2023-06-03T20:17:50.487 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:17:50.413562+0000", 2023-06-03T20:17:50.487 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 6.9970000000000001e-06 2023-06-03T20:17:50.487 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:17:50.488 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:17:50.488 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "reached_pg", 2023-06-03T20:17:50.489 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:17:50.415152+0000", 2023-06-03T20:17:50.490 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.0015898500000000001 2023-06-03T20:17:50.490 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:17:50.490 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:17:50.491 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "started", 2023-06-03T20:17:50.491 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:17:50.415309+0000", 2023-06-03T20:17:50.491 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.00015704200000000001 2023-06-03T20:17:50.492 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:17:50.493 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:17:50.493 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "waiting for subops from 7", 2023-06-03T20:17:50.493 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:17:50.415439+0000", 2023-06-03T20:17:50.494 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.00012996000000000001 2023-06-03T20:17:50.494 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:17:50.494 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:17:50.495 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:17:50.495 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:17:50.420169+0000", 2023-06-03T20:17:50.495 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.004730143 2023-06-03T20:17:50.496 INFO:teuthology.orchestra.run.smithi089.stdout: } 2023-06-03T20:17:50.496 INFO:teuthology.orchestra.run.smithi089.stdout: ] 2023-06-03T20:17:50.496 INFO:teuthology.orchestra.run.smithi089.stdout: } 2023-06-03T20:17:50.496 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:17:50.497 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:17:50.497 INFO:teuthology.orchestra.run.smithi089.stdout: "description": "osd_op(client.46373.0:53794 12.b 12:df023caa:::benchmark_data_smithi089_191647_object53793:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected e224)", 2023-06-03T20:17:50.497 INFO:teuthology.orchestra.run.smithi089.stdout: "initiated_at": "2023-06-03T20:17:50.416650+0000", 2023-06-03T20:17:50.497 INFO:teuthology.orchestra.run.smithi089.stdout: "age": 0.0045070919999999999, 2023-06-03T20:17:50.497 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.0049687669999999998, 2023-06-03T20:17:50.497 INFO:teuthology.orchestra.run.smithi089.stdout: "type_data": { 2023-06-03T20:17:50.497 INFO:teuthology.orchestra.run.smithi089.stdout: "flag_point": "waiting for sub ops", 2023-06-03T20:17:50.498 INFO:teuthology.orchestra.run.smithi089.stdout: "client_info": { 2023-06-03T20:17:50.498 INFO:teuthology.orchestra.run.smithi089.stdout: "client": "client.46373", 2023-06-03T20:17:50.498 INFO:teuthology.orchestra.run.smithi089.stdout: "client_addr": "172.21.15.89:0/3737977806", 2023-06-03T20:17:50.498 INFO:teuthology.orchestra.run.smithi089.stdout: "tid": 53794 2023-06-03T20:17:50.498 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:17:50.498 INFO:teuthology.orchestra.run.smithi089.stdout: "events": [ 2023-06-03T20:17:50.498 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:17:50.498 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "initiated", 2023-06-03T20:17:50.499 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:17:50.416650+0000", 2023-06-03T20:17:50.499 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0 2023-06-03T20:17:50.499 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:17:50.499 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:17:50.499 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "throttled", 2023-06-03T20:17:50.499 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:17:50.416650+0000", 2023-06-03T20:17:50.499 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0 2023-06-03T20:17:50.500 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:17:50.500 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:17:50.500 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "header_read", 2023-06-03T20:17:50.500 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:17:50.416649+0000", 2023-06-03T20:17:50.500 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 4294967295.9999986 2023-06-03T20:17:50.500 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:17:50.500 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:17:50.500 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "all_read", 2023-06-03T20:17:50.501 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:17:50.416676+0000", 2023-06-03T20:17:50.501 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 2.7022e-05 2023-06-03T20:17:50.501 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:17:50.501 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:17:50.501 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "dispatched", 2023-06-03T20:17:50.501 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:17:50.419005+0000", 2023-06-03T20:17:50.501 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.0023293379999999998 2023-06-03T20:17:50.501 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:17:50.502 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:17:50.502 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "queued_for_pg", 2023-06-03T20:17:50.502 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:17:50.419019+0000", 2023-06-03T20:17:50.502 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 1.3757999999999999e-05 2023-06-03T20:17:50.502 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:17:50.502 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:17:50.502 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "reached_pg", 2023-06-03T20:17:50.503 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:17:50.419070+0000", 2023-06-03T20:17:50.503 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 5.0942000000000001e-05 2023-06-03T20:17:50.503 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:17:50.503 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:17:50.503 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "started", 2023-06-03T20:17:50.503 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:17:50.419215+0000", 2023-06-03T20:17:50.503 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.000145434 2023-06-03T20:17:50.503 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:17:50.504 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:17:50.504 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "waiting for subops from 2", 2023-06-03T20:17:50.504 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:17:50.419348+0000", 2023-06-03T20:17:50.504 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.00013301099999999999 2023-06-03T20:17:50.504 INFO:teuthology.orchestra.run.smithi089.stdout: } 2023-06-03T20:17:50.504 INFO:teuthology.orchestra.run.smithi089.stdout: ] 2023-06-03T20:17:50.504 INFO:teuthology.orchestra.run.smithi089.stdout: } 2023-06-03T20:17:50.505 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:17:50.505 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:17:50.505 INFO:teuthology.orchestra.run.smithi089.stdout: "description": "osd_repop(client.46373.0:53791 12.3 e224/217 12:cfc38288:::benchmark_data_smithi089_191647_object53790:head v 224'3388, mlcod=224'3386)", 2023-06-03T20:17:50.505 INFO:teuthology.orchestra.run.smithi089.stdout: "initiated_at": "2023-06-03T20:17:50.419066+0000", 2023-06-03T20:17:50.505 INFO:teuthology.orchestra.run.smithi089.stdout: "age": 0.0020913709999999999, 2023-06-03T20:17:50.505 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.0026109530000000001, 2023-06-03T20:17:50.505 INFO:teuthology.orchestra.run.smithi089.stdout: "type_data": { 2023-06-03T20:17:50.505 INFO:teuthology.orchestra.run.smithi089.stdout: "flag_point": "started", 2023-06-03T20:17:50.506 INFO:teuthology.orchestra.run.smithi089.stdout: "events": [ 2023-06-03T20:17:50.506 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:17:50.506 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "initiated", 2023-06-03T20:17:50.506 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:17:50.419066+0000", 2023-06-03T20:17:50.506 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0 2023-06-03T20:17:50.506 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:17:50.506 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:17:50.506 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "throttled", 2023-06-03T20:17:50.507 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:17:50.419066+0000", 2023-06-03T20:17:50.507 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0 2023-06-03T20:17:50.507 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:17:50.507 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:17:50.507 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "header_read", 2023-06-03T20:17:50.507 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:17:50.419065+0000", 2023-06-03T20:17:50.507 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 4294967295.999999 2023-06-03T20:17:50.507 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:17:50.508 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:17:50.508 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "all_read", 2023-06-03T20:17:50.508 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:17:50.419096+0000", 2023-06-03T20:17:50.508 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 3.1584000000000001e-05 2023-06-03T20:17:50.508 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:17:50.508 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:17:50.508 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "dispatched", 2023-06-03T20:17:50.508 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:17:50.419108+0000", 2023-06-03T20:17:50.509 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 1.1566000000000001e-05 2023-06-03T20:17:50.509 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:17:50.509 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:17:50.509 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "queued_for_pg", 2023-06-03T20:17:50.509 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:17:50.419115+0000", 2023-06-03T20:17:50.509 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 6.9840000000000004e-06 2023-06-03T20:17:50.509 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:17:50.510 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:17:50.510 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "reached_pg", 2023-06-03T20:17:50.510 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:17:50.419188+0000", 2023-06-03T20:17:50.510 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 7.3343000000000006e-05 2023-06-03T20:17:50.510 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:17:50.510 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:17:50.510 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "started", 2023-06-03T20:17:50.510 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:17:50.419210+0000", 2023-06-03T20:17:50.511 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 2.1603999999999998e-05 2023-06-03T20:17:50.511 INFO:teuthology.orchestra.run.smithi089.stdout: } 2023-06-03T20:17:50.511 INFO:teuthology.orchestra.run.smithi089.stdout: ] 2023-06-03T20:17:50.511 INFO:teuthology.orchestra.run.smithi089.stdout: } 2023-06-03T20:17:50.511 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:17:50.511 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:17:50.511 INFO:teuthology.orchestra.run.smithi089.stdout: "description": "osd_repop(client.46373.0:53786 12.a e224/217)", 2023-06-03T20:17:50.511 INFO:teuthology.orchestra.run.smithi089.stdout: "initiated_at": "2023-06-03T20:17:50.421603+0000", 2023-06-03T20:17:50.512 INFO:teuthology.orchestra.run.smithi089.stdout: "age": 4294967295.9995537, 2023-06-03T20:17:50.512 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.00011348799999999999, 2023-06-03T20:17:50.512 INFO:teuthology.orchestra.run.smithi089.stdout: "type_data": { 2023-06-03T20:17:50.512 INFO:teuthology.orchestra.run.smithi089.stdout: "flag_point": "reached pg", 2023-06-03T20:17:50.512 INFO:teuthology.orchestra.run.smithi089.stdout: "events": [ 2023-06-03T20:17:50.512 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:17:50.512 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "initiated", 2023-06-03T20:17:50.512 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:17:50.421603+0000", 2023-06-03T20:17:50.513 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0 2023-06-03T20:17:50.513 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:17:50.513 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:17:50.513 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "throttled", 2023-06-03T20:17:50.513 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:17:50.421603+0000", 2023-06-03T20:17:50.513 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0 2023-06-03T20:17:50.513 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:17:50.513 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:17:50.514 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "header_read", 2023-06-03T20:17:50.514 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:17:50.421602+0000", 2023-06-03T20:17:50.514 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 4294967295.999999 2023-06-03T20:17:50.514 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:17:50.514 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:17:50.514 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "all_read", 2023-06-03T20:17:50.514 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:17:50.421642+0000", 2023-06-03T20:17:50.515 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 3.9054000000000003e-05 2023-06-03T20:17:50.515 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:17:50.515 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:17:50.515 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "dispatched", 2023-06-03T20:17:50.515 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:17:50.421657+0000", 2023-06-03T20:17:50.515 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 1.5116999999999999e-05 2023-06-03T20:17:50.515 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:17:50.515 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:17:50.516 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "queued_for_pg", 2023-06-03T20:17:50.516 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:17:50.421665+0000", 2023-06-03T20:17:50.516 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 8.6049999999999998e-06 2023-06-03T20:17:50.516 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:17:50.516 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:17:50.516 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "reached_pg", 2023-06-03T20:17:50.516 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:17:50.421705+0000", 2023-06-03T20:17:50.516 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 3.9301999999999997e-05 2023-06-03T20:17:50.517 INFO:teuthology.orchestra.run.smithi089.stdout: } 2023-06-03T20:17:50.517 INFO:teuthology.orchestra.run.smithi089.stdout: ] 2023-06-03T20:17:50.517 INFO:teuthology.orchestra.run.smithi089.stdout: } 2023-06-03T20:17:50.517 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:17:50.517 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:17:50.517 INFO:teuthology.orchestra.run.smithi089.stdout: "description": "osd_repop(client.46373.0:53793 12.a e224/217)", 2023-06-03T20:17:50.517 INFO:teuthology.orchestra.run.smithi089.stdout: "initiated_at": "2023-06-03T20:17:50.421678+0000", 2023-06-03T20:17:50.517 INFO:teuthology.orchestra.run.smithi089.stdout: "age": 4294967295.9994788, 2023-06-03T20:17:50.518 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 7.4740000000000006e-05, 2023-06-03T20:17:50.518 INFO:teuthology.orchestra.run.smithi089.stdout: "type_data": { 2023-06-03T20:17:50.518 INFO:teuthology.orchestra.run.smithi089.stdout: "flag_point": "queued for pg", 2023-06-03T20:17:50.518 INFO:teuthology.orchestra.run.smithi089.stdout: "events": [ 2023-06-03T20:17:50.518 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:17:50.518 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "initiated", 2023-06-03T20:17:50.518 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:17:50.421678+0000", 2023-06-03T20:17:50.518 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0 2023-06-03T20:17:50.519 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:17:50.519 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:17:50.519 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "throttled", 2023-06-03T20:17:50.519 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:17:50.421678+0000", 2023-06-03T20:17:50.519 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0 2023-06-03T20:17:50.519 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:17:50.519 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:17:50.520 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "header_read", 2023-06-03T20:17:50.520 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:17:50.421678+0000", 2023-06-03T20:17:50.520 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 4294967295.9999995 2023-06-03T20:17:50.520 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:17:50.520 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:17:50.520 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "all_read", 2023-06-03T20:17:50.520 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:17:50.421710+0000", 2023-06-03T20:17:50.520 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 3.2444000000000003e-05 2023-06-03T20:17:50.521 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:17:50.521 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:17:50.521 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "dispatched", 2023-06-03T20:17:50.521 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:17:50.421718+0000", 2023-06-03T20:17:50.521 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 7.9240000000000007e-06 2023-06-03T20:17:50.521 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:17:50.521 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:17:50.521 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "queued_for_pg", 2023-06-03T20:17:50.522 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:17:50.421724+0000", 2023-06-03T20:17:50.522 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 5.9190000000000004e-06 2023-06-03T20:17:50.522 INFO:teuthology.orchestra.run.smithi089.stdout: } 2023-06-03T20:17:50.522 INFO:teuthology.orchestra.run.smithi089.stdout: ] 2023-06-03T20:17:50.522 INFO:teuthology.orchestra.run.smithi089.stdout: } 2023-06-03T20:17:50.522 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:17:50.522 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:17:50.522 INFO:teuthology.orchestra.run.smithi089.stdout: "description": "osd_op(client.46373.0:53798 12.b 12.ccf87c7b (undecoded) ondisk+write+known_if_redirected e224)", 2023-06-03T20:17:50.523 INFO:teuthology.orchestra.run.smithi089.stdout: "initiated_at": "2023-06-03T20:17:50.421716+0000", 2023-06-03T20:17:50.523 INFO:teuthology.orchestra.run.smithi089.stdout: "age": 4294967295.9994407, 2023-06-03T20:17:50.523 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 8.0508999999999998e-05, 2023-06-03T20:17:50.523 INFO:teuthology.orchestra.run.smithi089.stdout: "type_data": { 2023-06-03T20:17:50.523 INFO:teuthology.orchestra.run.smithi089.stdout: "flag_point": "queued for pg", 2023-06-03T20:17:50.523 INFO:teuthology.orchestra.run.smithi089.stdout: "client_info": { 2023-06-03T20:17:50.523 INFO:teuthology.orchestra.run.smithi089.stdout: "client": "client.46373", 2023-06-03T20:17:50.523 INFO:teuthology.orchestra.run.smithi089.stdout: "client_addr": "172.21.15.89:0/3737977806", 2023-06-03T20:17:50.524 INFO:teuthology.orchestra.run.smithi089.stdout: "tid": 53798 2023-06-03T20:17:50.524 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:17:50.524 INFO:teuthology.orchestra.run.smithi089.stdout: "events": [ 2023-06-03T20:17:50.524 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:17:50.524 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "initiated", 2023-06-03T20:17:50.524 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:17:50.421716+0000", 2023-06-03T20:17:50.524 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0 2023-06-03T20:17:50.524 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:17:50.525 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:17:50.525 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "throttled", 2023-06-03T20:17:50.525 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:17:50.421716+0000", 2023-06-03T20:17:50.525 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0 2023-06-03T20:17:50.525 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:17:50.526 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:17:50.526 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "header_read", 2023-06-03T20:17:50.526 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:17:50.421715+0000", 2023-06-03T20:17:50.526 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 4294967295.9999981 2023-06-03T20:17:50.526 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:17:50.526 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:17:50.526 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "all_read", 2023-06-03T20:17:50.526 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:17:50.421741+0000", 2023-06-03T20:17:50.527 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 2.6752000000000001e-05 2023-06-03T20:17:50.527 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:17:50.527 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:17:50.527 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "dispatched", 2023-06-03T20:17:50.527 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:17:50.421753+0000", 2023-06-03T20:17:50.527 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 1.1634000000000001e-05 2023-06-03T20:17:50.527 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:17:50.527 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:17:50.528 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "queued_for_pg", 2023-06-03T20:17:50.528 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:17:50.421763+0000", 2023-06-03T20:17:50.528 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 9.9820000000000003e-06 2023-06-03T20:17:50.528 INFO:teuthology.orchestra.run.smithi089.stdout: } 2023-06-03T20:17:50.528 INFO:teuthology.orchestra.run.smithi089.stdout: ] 2023-06-03T20:17:50.528 INFO:teuthology.orchestra.run.smithi089.stdout: } 2023-06-03T20:17:50.528 INFO:teuthology.orchestra.run.smithi089.stdout: } 2023-06-03T20:17:50.529 INFO:teuthology.orchestra.run.smithi089.stdout: ], 2023-06-03T20:17:50.529 INFO:teuthology.orchestra.run.smithi089.stdout: "num_ops": 11 2023-06-03T20:17:50.529 INFO:teuthology.orchestra.run.smithi089.stdout:} 2023-06-03T20:17:50.529 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:17:50.565 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:17:50.568 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:17:50.568 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:17:50.669 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:17:50.704 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:17:50.706 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:17:50.707 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:17:50.809 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:17:50.842 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:17:50.845 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:17:50.845 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:17:50.946 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:17:50.979 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:17:50.980 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:17:50.981 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:17:51.082 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:17:51.122 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:17:51.125 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:17:51.126 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:17:51.227 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:17:51.264 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:17:51.269 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:17:51.270 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:17:51.371 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:17:51.394 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 34 16 54970 54954 101.001 75.6875 0.00656092 0.00989436 2023-06-03T20:17:51.401 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:17:51.405 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:17:51.406 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:17:51.506 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:17:51.542 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:17:51.546 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:17:51.547 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:17:51.647 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:17:51.692 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:17:51.695 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:17:51.696 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:17:51.797 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:17:51.829 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:17:51.831 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:17:51.832 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:17:51.933 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:17:51.961 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:17:51.968 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:17:51.968 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:17:52.069 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:17:52.083 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:17:51 smithi164 bash[59462]: cluster 1685823470.8580956 mgr.y (mgr.34104) 1255 : cluster 0 pgmap v1249: 89 pgs: 20 active+clean+remapped, 14 active+undersized+degraded, 11 active+undersized, 44 active+clean; 3.2 GiB data, 11 GiB used, 525 GiB / 536 GiB avail; 82 MiB/s wr, 1.31k op/s; 26224/155847 objects degraded (16.827%); 6/155847 objects misplaced (0.004%) 2023-06-03T20:17:52.090 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:17:51 smithi089 bash[78452]: cluster 1685823470.8580956 mgr.y (mgr.34104) 1255 : cluster 0 pgmap v1249: 89 pgs: 20 active+clean+remapped, 14 active+undersized+degraded, 11 active+undersized, 44 active+clean; 3.2 GiB data, 11 GiB used, 525 GiB / 536 GiB avail; 82 MiB/s wr, 1.31k op/s; 26224/155847 objects degraded (16.827%); 6/155847 objects misplaced (0.004%) 2023-06-03T20:17:52.091 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:17:51 smithi089 bash[75068]: cluster 1685823470.8580956 mgr.y (mgr.34104) 1255 : cluster 0 pgmap v1249: 89 pgs: 20 active+clean+remapped, 14 active+undersized+degraded, 11 active+undersized, 44 active+clean; 3.2 GiB data, 11 GiB used, 525 GiB / 536 GiB avail; 82 MiB/s wr, 1.31k op/s; 26224/155847 objects degraded (16.827%); 6/155847 objects misplaced (0.004%) 2023-06-03T20:17:52.109 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:17:52.114 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:17:52.115 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:17:52.216 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:17:52.249 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:17:52.251 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:17:52.252 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:17:52.353 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:17:52.378 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:17:52.383 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:17:52.384 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:17:52.394 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 35 16 56037 56021 100.021 66.6875 0.00464066 0.0099923 2023-06-03T20:17:52.486 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:17:52.519 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:17:52.522 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:17:52.523 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:17:52.624 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:17:52.653 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:17:52.655 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:17:52.656 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:17:52.757 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:17:52.790 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:17:52.792 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:17:52.793 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:17:52.894 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:17:52.915 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:17:52.917 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:17:52.918 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:17:53.019 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:17:53.060 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:17:53.060 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:17:53.061 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:17:53.162 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:17:53.194 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:17:53.198 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:17:53.199 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:17:53.279 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:17:52 smithi089 bash[78452]: audit 1685823472.1287463 mon.c (mon.2) 553 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T20:17:53.279 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:17:52 smithi089 bash[78452]: cluster 1685823472.822787 mon.a (mon.0) 1198 : cluster 3 Health check update: Degraded data redundancy: 26224/155847 objects degraded (16.827%), 14 pgs degraded, 23 pgs undersized (PG_DEGRADED) 2023-06-03T20:17:53.279 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:17:52 smithi089 bash[75068]: audit 1685823472.1287463 mon.c (mon.2) 553 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T20:17:53.279 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:17:52 smithi089 bash[75068]: cluster 1685823472.822787 mon.a (mon.0) 1198 : cluster 3 Health check update: Degraded data redundancy: 26224/155847 objects degraded (16.827%), 14 pgs degraded, 23 pgs undersized (PG_DEGRADED) 2023-06-03T20:17:53.300 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:17:53.333 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:17:53.334 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:17:52 smithi164 bash[59462]: audit 1685823472.1287463 mon.c (mon.2) 553 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T20:17:53.334 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:17:52 smithi164 bash[59462]: cluster 1685823472.822787 mon.a (mon.0) 1198 : cluster 3 Health check update: Degraded data redundancy: 26224/155847 objects degraded (16.827%), 14 pgs degraded, 23 pgs undersized (PG_DEGRADED) 2023-06-03T20:17:53.336 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:17:53.337 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:17:53.394 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 36 16 57129 57113 99.1383 68.25 0.0174226 0.0100811 2023-06-03T20:17:53.438 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:17:53.478 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:17:53.482 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:17:53.483 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:17:53.584 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:17:53.617 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:17:53.622 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:17:53.622 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:17:53.723 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:17:53.759 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:17:53.762 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:17:53.762 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:17:53.863 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:17:53.901 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:17:53.904 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:17:53.905 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:17:54.006 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:17:54.030 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:17:54.032 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:17:54.033 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:17:54.133 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:17:54.168 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:17:54.172 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:17:54.172 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:17:54.273 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:17:54.279 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:17:54 smithi089 bash[78452]: cluster 1685823472.8588586 mgr.y (mgr.34104) 1256 : cluster 0 pgmap v1250: 89 pgs: 20 active+clean+remapped, 14 active+undersized+degraded, 11 active+undersized, 44 active+clean; 3.3 GiB data, 11 GiB used, 525 GiB / 536 GiB avail; 76 MiB/s wr, 1.21k op/s; 26554/159936 objects degraded (16.603%); 6/159936 objects misplaced (0.004%) 2023-06-03T20:17:54.279 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:17:53 smithi089 bash[75068]: cluster 1685823472.8588586 mgr.y (mgr.34104) 1256 : cluster 0 pgmap v1250: 89 pgs: 20 active+clean+remapped, 14 active+undersized+degraded, 11 active+undersized, 44 active+clean; 3.3 GiB data, 11 GiB used, 525 GiB / 536 GiB avail; 76 MiB/s wr, 1.21k op/s; 26554/159936 objects degraded (16.603%); 6/159936 objects misplaced (0.004%) 2023-06-03T20:17:54.306 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:17:54.309 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:17:54.310 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:17:54.333 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:17:53 smithi164 bash[59462]: cluster 1685823472.8588586 mgr.y (mgr.34104) 1256 : cluster 0 pgmap v1250: 89 pgs: 20 active+clean+remapped, 14 active+undersized+degraded, 11 active+undersized, 44 active+clean; 3.3 GiB data, 11 GiB used, 525 GiB / 536 GiB avail; 76 MiB/s wr, 1.21k op/s; 26554/159936 objects degraded (16.603%); 6/159936 objects misplaced (0.004%) 2023-06-03T20:17:54.394 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 37 16 58133 58117 98.1548 62.75 0.00519866 0.0101817 2023-06-03T20:17:54.411 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:17:54.442 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:17:54.445 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:17:54.445 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:17:54.547 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:17:54.587 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:17:54.592 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:17:54.592 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:17:54.693 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:17:54.725 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:17:54.729 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:17:54.729 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:17:54.830 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:17:54.864 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:17:54.867 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:17:54.867 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:17:54.968 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:17:54.999 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:17:55.003 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:17:55.004 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:17:55.027 INFO:teuthology.orchestra.run.smithi089.stderr:noscrub is set 2023-06-03T20:17:55.105 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:17:55.138 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:17:55.142 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:17:55.143 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:17:55.244 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:17:55.256 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:17:55 smithi164 bash[59462]: audit 1685823474.03379 mon.a (mon.0) 1199 : audit 1 from='client.? 172.21.15.89:0/2156321404' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-06-03T20:17:55.277 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:17:55.278 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:17:55 smithi089 bash[75068]: audit 1685823474.03379 mon.a (mon.0) 1199 : audit 1 from='client.? 172.21.15.89:0/2156321404' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-06-03T20:17:55.279 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:17:55 smithi089 bash[78452]: audit 1685823474.03379 mon.a (mon.0) 1199 : audit 1 from='client.? 172.21.15.89:0/2156321404' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-06-03T20:17:55.283 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:17:55.284 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:17:55.289 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph daemon osd.3 dump_blocked_ops 2023-06-03T20:17:55.385 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:17:55.396 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 38 16 59333 59317 97.5414 75 0.0146649 0.0102449 2023-06-03T20:17:55.437 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:17:55.439 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:17:55.440 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:17:55.541 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:17:55.586 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:17:55.590 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:17:55.591 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:17:55.692 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:17:55.725 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:17:55.728 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:17:55.728 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:17:55.829 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:17:55.859 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:17:55.862 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:17:55.863 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:17:55.963 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:17:55.997 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:17:55.999 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:17:56.000 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:17:56.009 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:17:56 smithi164 bash[59462]: cluster 1685823474.8594959 mgr.y (mgr.34104) 1257 : cluster 0 pgmap v1251: 89 pgs: 20 active+clean+remapped, 14 active+undersized+degraded, 11 active+undersized, 44 active+clean; 3.4 GiB data, 12 GiB used, 525 GiB / 536 GiB avail; 78 MiB/s wr, 1.25k op/s; 27824/166722 objects degraded (16.689%); 6/166722 objects misplaced (0.004%) 2023-06-03T20:17:56.009 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:17:56 smithi164 bash[59462]: cluster 1685823475.0002575 mon.a (mon.0) 1200 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-06-03T20:17:56.010 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:17:56 smithi164 bash[59462]: audit 1685823475.015581 mon.a (mon.0) 1201 : audit 1 from='client.? 172.21.15.89:0/2156321404' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-06-03T20:17:56.010 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:17:56 smithi164 bash[59462]: cluster 1685823475.0156224 mon.a (mon.0) 1202 : cluster 0 osdmap e225: 8 total, 7 up, 6 in 2023-06-03T20:17:56.010 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:17:56 smithi164 bash[59462]: cluster 1685823475.0211055 osd.5 (osd.5) 162 : cluster 3 failed to encode map e225 with expected crc 2023-06-03T20:17:56.010 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:17:56 smithi164 bash[59462]: cluster 1685823475.023176 osd.7 (osd.7) 151 : cluster 3 failed to encode map e225 with expected crc 2023-06-03T20:17:56.010 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:17:56 smithi164 bash[59462]: cluster 1685823475.0745351 osd.6 (osd.6) 168 : cluster 3 failed to encode map e225 with expected crc 2023-06-03T20:17:56.101 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:17:56.110 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:17:56 smithi089 bash[75068]: cluster 1685823474.8594959 mgr.y (mgr.34104) 1257 : cluster 0 pgmap v1251: 89 pgs: 20 active+clean+remapped, 14 active+undersized+degraded, 11 active+undersized, 44 active+clean; 3.4 GiB data, 12 GiB used, 525 GiB / 536 GiB avail; 78 MiB/s wr, 1.25k op/s; 27824/166722 objects degraded (16.689%); 6/166722 objects misplaced (0.004%) 2023-06-03T20:17:56.110 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:17:56 smithi089 bash[75068]: cluster 1685823475.0002575 mon.a (mon.0) 1200 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-06-03T20:17:56.111 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:17:56 smithi089 bash[75068]: audit 1685823475.015581 mon.a (mon.0) 1201 : audit 1 from='client.? 172.21.15.89:0/2156321404' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-06-03T20:17:56.111 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:17:56 smithi089 bash[75068]: cluster 1685823475.0156224 mon.a (mon.0) 1202 : cluster 0 osdmap e225: 8 total, 7 up, 6 in 2023-06-03T20:17:56.111 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:17:56 smithi089 bash[75068]: cluster 1685823475.0211055 osd.5 (osd.5) 162 : cluster 3 failed to encode map e225 with expected crc 2023-06-03T20:17:56.111 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:17:56 smithi089 bash[75068]: cluster 1685823475.023176 osd.7 (osd.7) 151 : cluster 3 failed to encode map e225 with expected crc 2023-06-03T20:17:56.111 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:17:56 smithi089 bash[75068]: cluster 1685823475.0745351 osd.6 (osd.6) 168 : cluster 3 failed to encode map e225 with expected crc 2023-06-03T20:17:56.112 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:17:56 smithi089 bash[78452]: cluster 1685823474.8594959 mgr.y (mgr.34104) 1257 : cluster 0 pgmap v1251: 89 pgs: 20 active+clean+remapped, 14 active+undersized+degraded, 11 active+undersized, 44 active+clean; 3.4 GiB data, 12 GiB used, 525 GiB / 536 GiB avail; 78 MiB/s wr, 1.25k op/s; 27824/166722 objects degraded (16.689%); 6/166722 objects misplaced (0.004%) 2023-06-03T20:17:56.112 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:17:56 smithi089 bash[78452]: cluster 1685823475.0002575 mon.a (mon.0) 1200 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-06-03T20:17:56.112 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:17:56 smithi089 bash[78452]: audit 1685823475.015581 mon.a (mon.0) 1201 : audit 1 from='client.? 172.21.15.89:0/2156321404' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-06-03T20:17:56.112 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:17:56 smithi089 bash[78452]: cluster 1685823475.0156224 mon.a (mon.0) 1202 : cluster 0 osdmap e225: 8 total, 7 up, 6 in 2023-06-03T20:17:56.112 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:17:56 smithi089 bash[78452]: cluster 1685823475.0211055 osd.5 (osd.5) 162 : cluster 3 failed to encode map e225 with expected crc 2023-06-03T20:17:56.112 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:17:56 smithi089 bash[78452]: cluster 1685823475.023176 osd.7 (osd.7) 151 : cluster 3 failed to encode map e225 with expected crc 2023-06-03T20:17:56.113 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:17:56 smithi089 bash[78452]: cluster 1685823475.0745351 osd.6 (osd.6) 168 : cluster 3 failed to encode map e225 with expected crc 2023-06-03T20:17:56.129 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:17:56.132 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:17:56.133 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:17:56.234 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:17:56.267 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:17:56.269 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:17:56.270 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:17:56.371 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:17:56.379 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 20:17:56 smithi089 bash[71163]: ::ffff:172.21.15.164 - - [03/Jun/2023:20:17:56] "GET /metrics HTTP/1.1" 200 228941 "" "Prometheus/2.33.4" 2023-06-03T20:17:56.396 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 39 16 60505 60489 96.9185 73.25 0.00529722 0.0103119 2023-06-03T20:17:56.400 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:17:56.403 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:17:56.403 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:17:56.504 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:17:56.554 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:17:56.559 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:17:56.560 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:17:56.661 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:17:56.698 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:17:56.703 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:17:56.704 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:17:56.805 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:17:56.838 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:17:56.842 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:17:56.843 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:17:56.876 INFO:teuthology.orchestra.run.smithi089.stdout:osd.0: {} 2023-06-03T20:17:56.913 INFO:teuthology.orchestra.run.smithi089.stdout:osd.1: {} 2023-06-03T20:17:56.944 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:17:56.966 INFO:teuthology.orchestra.run.smithi089.stdout:osd.2: {} 2023-06-03T20:17:56.987 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:17:56.991 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:17:56.992 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:17:57.005 INFO:teuthology.orchestra.run.smithi089.stdout:osd.3: {} 2023-06-03T20:17:57.049 INFO:teuthology.orchestra.run.smithi089.stdout:osd.5: {} 2023-06-03T20:17:57.078 INFO:teuthology.orchestra.run.smithi089.stdout:osd.6: {} 2023-06-03T20:17:57.093 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:17:57.122 INFO:teuthology.orchestra.run.smithi089.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-06-03T20:17:57.122 INFO:teuthology.orchestra.run.smithi089.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-06-03T20:17:57.122 INFO:teuthology.orchestra.run.smithi089.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-06-03T20:17:57.122 INFO:teuthology.orchestra.run.smithi089.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-06-03T20:17:57.123 INFO:teuthology.orchestra.run.smithi089.stderr:Error ENXIO: problem getting command descriptions from osd.4 2023-06-03T20:17:57.123 INFO:teuthology.orchestra.run.smithi089.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-06-03T20:17:57.123 INFO:teuthology.orchestra.run.smithi089.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-06-03T20:17:57.123 INFO:teuthology.orchestra.run.smithi089.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-06-03T20:17:57.123 INFO:teuthology.orchestra.run.smithi089.stdout:osd.7: {} 2023-06-03T20:17:57.129 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:17:57.131 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:17:57.132 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:17:57.233 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:17:57.262 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:17:57.266 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:17:57.266 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:17:57.278 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:17:57 smithi089 bash[75068]: audit 1685823476.8316793 mon.b (mon.1) 59 : audit 0 from='client.? 172.21.15.89:0/2980248724' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-03T20:17:57.279 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:17:57 smithi089 bash[78452]: audit 1685823476.8316793 mon.b (mon.1) 59 : audit 0 from='client.? 172.21.15.89:0/2980248724' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-03T20:17:57.333 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:17:57 smithi164 bash[59462]: audit 1685823476.8316793 mon.b (mon.1) 59 : audit 0 from='client.? 172.21.15.89:0/2980248724' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-03T20:17:57.367 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:17:57.396 INFO:tasks.radosbench.radosbench.0.smithi089.stdout:2023-06-03T20:17:57.392738+0000 min lat: 0.00129872 max lat: 0.870111 avg lat: 0.0103929 2023-06-03T20:17:57.397 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-06-03T20:17:57.397 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 40 16 61582 61566 96.1767 67.3125 0.0162105 0.0103929 2023-06-03T20:17:57.404 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:17:57.407 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:17:57.407 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:17:57.508 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:17:57.536 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:17:57.539 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:17:57.540 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:17:57.642 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:17:57.685 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:17:57.688 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:17:57.688 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:17:57.789 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:17:57.830 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:17:57.832 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:17:57.833 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:17:57.933 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:17:57.959 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:17:57.960 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:17:57.961 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:17:58.062 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:17:58.099 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:17:58.102 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:17:58.102 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:17:58.203 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:17:58.211 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:17:58 smithi089 bash[78452]: cluster 1685823476.8603141 mgr.y (mgr.34104) 1258 : cluster 0 pgmap v1253: 89 pgs: 20 active+clean+remapped, 14 active+undersized+degraded, 11 active+undersized, 44 active+clean; 3.6 GiB data, 12 GiB used, 524 GiB / 536 GiB avail; 72 MiB/s wr, 1.15k op/s; 29291/176337 objects degraded (16.611%); 6/176337 objects misplaced (0.003%) 2023-06-03T20:17:58.212 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:17:58 smithi089 bash[78452]: cluster 1685823477.8242197 mon.a (mon.0) 1203 : cluster 3 Health check update: Degraded data redundancy: 29291/176337 objects degraded (16.611%), 14 pgs degraded, 23 pgs undersized (PG_DEGRADED) 2023-06-03T20:17:58.212 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:17:58 smithi089 bash[75068]: cluster 1685823476.8603141 mgr.y (mgr.34104) 1258 : cluster 0 pgmap v1253: 89 pgs: 20 active+clean+remapped, 14 active+undersized+degraded, 11 active+undersized, 44 active+clean; 3.6 GiB data, 12 GiB used, 524 GiB / 536 GiB avail; 72 MiB/s wr, 1.15k op/s; 29291/176337 objects degraded (16.611%); 6/176337 objects misplaced (0.003%) 2023-06-03T20:17:58.212 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:17:58 smithi089 bash[75068]: cluster 1685823477.8242197 mon.a (mon.0) 1203 : cluster 3 Health check update: Degraded data redundancy: 29291/176337 objects degraded (16.611%), 14 pgs degraded, 23 pgs undersized (PG_DEGRADED) 2023-06-03T20:17:58.241 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:17:58.248 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:17:58.249 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:17:58.333 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:17:58 smithi164 bash[59462]: cluster 1685823476.8603141 mgr.y (mgr.34104) 1258 : cluster 0 pgmap v1253: 89 pgs: 20 active+clean+remapped, 14 active+undersized+degraded, 11 active+undersized, 44 active+clean; 3.6 GiB data, 12 GiB used, 524 GiB / 536 GiB avail; 72 MiB/s wr, 1.15k op/s; 29291/176337 objects degraded (16.611%); 6/176337 objects misplaced (0.003%) 2023-06-03T20:17:58.334 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:17:58 smithi164 bash[59462]: cluster 1685823477.8242197 mon.a (mon.0) 1203 : cluster 3 Health check update: Degraded data redundancy: 29291/176337 objects degraded (16.611%), 14 pgs degraded, 23 pgs undersized (PG_DEGRADED) 2023-06-03T20:17:58.350 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:17:58.382 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:17:58.385 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:17:58.386 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:17:58.396 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 41 16 62633 62617 95.4329 65.6875 0.00900413 0.0104726 2023-06-03T20:17:58.486 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:17:58.516 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:17:58.518 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:17:58.518 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:17:58.619 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:17:58.652 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:17:58.658 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:17:58.658 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:17:58.759 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:17:58.800 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:17:58.805 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:17:58.806 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:17:58.907 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:17:58.946 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:17:58.949 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:17:58.949 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:17:59.050 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:17:59.106 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:17:59.108 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:17:59.109 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:17:59.210 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:17:59.243 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:17:59.249 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:17:59.250 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:17:59.321 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-06-03T20:17:59.321 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-06-03T20:17:59.351 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:17:59.386 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:17:59.388 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:17:59.389 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:17:59.397 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 42 16 63743 63727 94.8124 69.375 0.00923814 0.0105402 2023-06-03T20:17:59.490 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:17:59.525 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:17:59.527 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:17:59.528 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:17:59.628 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:17:59.664 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:17:59.667 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:17:59.668 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:17:59.739 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/mon.a/config 2023-06-03T20:17:59.769 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:17:59.802 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:17:59.804 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:17:59.805 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:17:59.906 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:17:59.950 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:17:59.952 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:17:59.953 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:18:00.054 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:18:00.083 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:18:00 smithi164 bash[59462]: cluster 1685823478.8634505 mgr.y (mgr.34104) 1259 : cluster 0 pgmap v1254: 89 pgs: 20 active+clean+remapped, 14 active+undersized+degraded, 11 active+undersized, 44 active+clean; 3.7 GiB data, 13 GiB used, 524 GiB / 536 GiB avail; 69 MiB/s wr, 1.10k op/s; 30659/183522 objects degraded (16.706%); 6/183522 objects misplaced (0.003%) 2023-06-03T20:18:00.098 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:18:00.104 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:18:00.105 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:18:00.206 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:18:00.238 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:18:00.240 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:18:00.241 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:18:00.261 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph osd set nodeep-scrub 2023-06-03T20:18:00.272 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:18:00 smithi089 bash[75068]: cluster 1685823478.8634505 mgr.y (mgr.34104) 1259 : cluster 0 pgmap v1254: 89 pgs: 20 active+clean+remapped, 14 active+undersized+degraded, 11 active+undersized, 44 active+clean; 3.7 GiB data, 13 GiB used, 524 GiB / 536 GiB avail; 69 MiB/s wr, 1.10k op/s; 30659/183522 objects degraded (16.706%); 6/183522 objects misplaced (0.003%) 2023-06-03T20:18:00.272 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:18:00 smithi089 bash[78452]: cluster 1685823478.8634505 mgr.y (mgr.34104) 1259 : cluster 0 pgmap v1254: 89 pgs: 20 active+clean+remapped, 14 active+undersized+degraded, 11 active+undersized, 44 active+clean; 3.7 GiB data, 13 GiB used, 524 GiB / 536 GiB avail; 69 MiB/s wr, 1.10k op/s; 30659/183522 objects degraded (16.706%); 6/183522 objects misplaced (0.003%) 2023-06-03T20:18:00.342 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:18:00.389 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:18:00.392 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:18:00.393 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:18:00.397 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 43 16 64771 64755 94.1016 64.25 0.0322062 0.0106199 2023-06-03T20:18:00.494 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:18:00.532 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:18:00.535 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:18:00.536 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:18:00.638 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:18:00.667 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:18:00.674 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:18:00.676 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:18:00.777 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:18:00.807 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:18:00.810 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:18:00.811 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:18:00.912 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:18:00.946 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:18:00.950 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:18:00.950 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:18:01.051 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:18:01.083 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:18:01.087 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:18:01.088 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:18:01.188 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:18:01.245 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:18:01.250 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:18:01.251 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:18:01.322 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-06-03T20:18:01.353 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:18:01.387 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:18:01.389 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:18:01.390 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:18:01.397 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 44 16 65877 65861 93.5339 69.125 0.0193762 0.0106854 2023-06-03T20:18:01.490 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:18:01.530 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:18:01.532 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:18:01.533 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:18:01.633 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:18:01.668 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:18:01.672 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:18:01.673 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:18:01.774 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:18:01.809 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:18:01.813 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:18:01.814 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:18:01.840 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/mon.a/config 2023-06-03T20:18:01.915 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:18:01.944 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:18:01.947 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:18:01.948 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:18:02.037 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:18:02 smithi089 bash[75068]: cluster 1685823480.864165 mgr.y (mgr.34104) 1260 : cluster 0 pgmap v1255: 89 pgs: 20 active+clean+remapped, 14 active+undersized+degraded, 11 active+undersized, 44 active+clean; 3.8 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 68 MiB/s wr, 1.09k op/s; 31647/188592 objects degraded (16.781%); 6/188592 objects misplaced (0.003%) 2023-06-03T20:18:02.049 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:18:02.087 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:18:02.091 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:18:02.092 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:18:02.193 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:18:02.224 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:18:02.227 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:18:02.227 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:18:02.328 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:18:02.333 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:18:02 smithi164 bash[59462]: cluster 1685823480.864165 mgr.y (mgr.34104) 1260 : cluster 0 pgmap v1255: 89 pgs: 20 active+clean+remapped, 14 active+undersized+degraded, 11 active+undersized, 44 active+clean; 3.8 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 68 MiB/s wr, 1.09k op/s; 31647/188592 objects degraded (16.781%); 6/188592 objects misplaced (0.003%) 2023-06-03T20:18:02.337 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:18:02 smithi089 bash[78452]: cluster 1685823480.864165 mgr.y (mgr.34104) 1260 : cluster 0 pgmap v1255: 89 pgs: 20 active+clean+remapped, 14 active+undersized+degraded, 11 active+undersized, 44 active+clean; 3.8 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 68 MiB/s wr, 1.09k op/s; 31647/188592 objects degraded (16.781%); 6/188592 objects misplaced (0.003%) 2023-06-03T20:18:02.365 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:18:02.366 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:18:02.366 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:18:02.397 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 45 16 66950 66934 92.9456 67.0625 0.0196149 0.0107529 2023-06-03T20:18:02.467 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:18:02.503 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:18:02.505 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:18:02.506 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:18:02.607 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:18:02.640 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:18:02.642 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:18:02.643 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:18:02.744 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:18:02.779 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:18:02.782 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:18:02.782 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:18:02.883 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:18:02.914 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:18:02.916 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:18:02.916 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:18:03.017 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:18:03.056 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:18:03.060 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:18:03.061 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:18:03.162 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:18:03.196 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:18:03.199 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:18:03.199 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:18:03.279 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:18:03 smithi089 bash[75068]: cluster 1685823482.8269122 mon.a (mon.0) 1204 : cluster 3 Health check update: Degraded data redundancy: 31647/188592 objects degraded (16.781%), 14 pgs degraded, 23 pgs undersized (PG_DEGRADED) 2023-06-03T20:18:03.279 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:18:03 smithi089 bash[78452]: cluster 1685823482.8269122 mon.a (mon.0) 1204 : cluster 3 Health check update: Degraded data redundancy: 31647/188592 objects degraded (16.781%), 14 pgs degraded, 23 pgs undersized (PG_DEGRADED) 2023-06-03T20:18:03.301 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:18:03.333 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:18:03 smithi164 bash[59462]: cluster 1685823482.8269122 mon.a (mon.0) 1204 : cluster 3 Health check update: Degraded data redundancy: 31647/188592 objects degraded (16.781%), 14 pgs degraded, 23 pgs undersized (PG_DEGRADED) 2023-06-03T20:18:03.342 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:18:03.344 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:18:03.345 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:18:03.397 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 46 16 67931 67915 92.2578 61.3125 0.0207398 0.0108337 2023-06-03T20:18:03.446 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:18:03.479 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:18:03.480 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:18:03.481 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:18:03.581 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:18:03.614 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:18:03.616 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:18:03.617 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:18:03.718 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:18:03.758 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:18:03.759 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:18:03.759 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:18:03.860 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:18:03.897 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:18:03.901 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:18:03.902 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:18:04.003 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:18:04.039 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:18:04.043 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:18:04.044 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:18:04.125 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:18:04 smithi089 bash[75068]: cluster 1685823482.8650355 mgr.y (mgr.34104) 1261 : cluster 0 pgmap v1256: 89 pgs: 20 active+clean+remapped, 14 active+undersized+degraded, 11 active+undersized, 44 active+clean; 3.9 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 68 MiB/s wr, 1.09k op/s; 32014/192612 objects degraded (16.621%); 6/192612 objects misplaced (0.003%) 2023-06-03T20:18:04.125 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:18:04 smithi089 bash[78452]: cluster 1685823482.8650355 mgr.y (mgr.34104) 1261 : cluster 0 pgmap v1256: 89 pgs: 20 active+clean+remapped, 14 active+undersized+degraded, 11 active+undersized, 44 active+clean; 3.9 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 68 MiB/s wr, 1.09k op/s; 32014/192612 objects degraded (16.621%); 6/192612 objects misplaced (0.003%) 2023-06-03T20:18:04.145 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:18:04.176 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:18:04.178 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:18:04.179 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:18:04.280 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:18:04.290 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:18:04 smithi164 bash[59462]: cluster 1685823482.8650355 mgr.y (mgr.34104) 1261 : cluster 0 pgmap v1256: 89 pgs: 20 active+clean+remapped, 14 active+undersized+degraded, 11 active+undersized, 44 active+clean; 3.9 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 68 MiB/s wr, 1.09k op/s; 32014/192612 objects degraded (16.621%); 6/192612 objects misplaced (0.003%) 2023-06-03T20:18:04.310 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:18:04.313 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:18:04.313 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:18:04.398 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 47 16 68832 68816 91.4925 56.3125 0.0171361 0.0109243 2023-06-03T20:18:04.415 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:18:04.446 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:18:04.448 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:18:04.449 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:18:04.550 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:18:04.580 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:18:04.587 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:18:04.587 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:18:04.688 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:18:04.726 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:18:04.730 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:18:04.731 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:18:04.832 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:18:04.863 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:18:04.866 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:18:04.867 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:18:04.932 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/mon.a/config 2023-06-03T20:18:04.968 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:18:04.999 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:18:05.001 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:18:05.002 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:18:05.104 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:18:05.137 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:18:05.141 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:18:05.142 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:18:05.243 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:18:05.278 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:18:05.282 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:18:05.282 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:18:05.383 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:18:05.397 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 48 16 69892 69876 90.9665 66.25 0.00812655 0.0109867 2023-06-03T20:18:05.418 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:18:05.423 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:18:05.423 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:18:05.524 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:18:05.557 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:18:05.561 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:18:05.562 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:18:05.664 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:18:05.699 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:18:05.704 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:18:05.705 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:18:05.806 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:18:05.838 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:18:05.841 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:18:05.841 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:18:05.942 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:18:05.980 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:18:05.984 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:18:05.984 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:18:06.083 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:18:06 smithi164 bash[59462]: cluster 1685823484.8657305 mgr.y (mgr.34104) 1262 : cluster 0 pgmap v1257: 89 pgs: 20 active+clean+remapped, 14 active+undersized+degraded, 11 active+undersized, 44 active+clean; 4.1 GiB data, 14 GiB used, 523 GiB / 536 GiB avail; 68 MiB/s wr, 1.09k op/s; 33402/199572 objects degraded (16.737%); 6/199572 objects misplaced (0.003%) 2023-06-03T20:18:06.085 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:18:06.111 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:18:06.113 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:18:06.114 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:18:06.215 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:18:06.240 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:18:06.242 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:18:06.242 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:18:06.277 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:18:06 smithi089 bash[78452]: cluster 1685823484.8657305 mgr.y (mgr.34104) 1262 : cluster 0 pgmap v1257: 89 pgs: 20 active+clean+remapped, 14 active+undersized+degraded, 11 active+undersized, 44 active+clean; 4.1 GiB data, 14 GiB used, 523 GiB / 536 GiB avail; 68 MiB/s wr, 1.09k op/s; 33402/199572 objects degraded (16.737%); 6/199572 objects misplaced (0.003%) 2023-06-03T20:18:06.278 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:18:06 smithi089 bash[75068]: cluster 1685823484.8657305 mgr.y (mgr.34104) 1262 : cluster 0 pgmap v1257: 89 pgs: 20 active+clean+remapped, 14 active+undersized+degraded, 11 active+undersized, 44 active+clean; 4.1 GiB data, 14 GiB used, 523 GiB / 536 GiB avail; 68 MiB/s wr, 1.09k op/s; 33402/199572 objects degraded (16.737%); 6/199572 objects misplaced (0.003%) 2023-06-03T20:18:06.343 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:18:06.373 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:18:06.376 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:18:06.376 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:18:06.397 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 49 16 70933 70917 90.4378 65.0625 0.0188379 0.0110525 2023-06-03T20:18:06.477 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:18:06.509 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:18:06.511 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:18:06.511 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:18:06.528 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 20:18:06 smithi089 bash[71163]: ::ffff:172.21.15.164 - - [03/Jun/2023:20:18:06] "GET /metrics HTTP/1.1" 200 228941 "" "Prometheus/2.33.4" 2023-06-03T20:18:06.612 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:18:06.713 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:18:06.721 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:18:06.722 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:18:06.823 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:18:06.843 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:18:06.845 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:18:06.845 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:18:06.946 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:18:06.990 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:18:06.993 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:18:06.994 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:18:07.095 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:18:07.143 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:18:07.145 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:18:07.146 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:18:07.247 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:18:07.280 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:18:07.284 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:18:07.285 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:18:07.385 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:18:07.397 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 50 16 71829 71813 89.749 56 0.0118377 0.0111369 2023-06-03T20:18:07.414 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:18:07.418 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:18:07.419 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:18:07.520 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:18:07.547 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:18:07.550 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:18:07.550 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:18:07.651 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:18:07.684 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:18:07.688 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:18:07.689 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:18:07.790 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:18:07.837 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:18:07.841 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:18:07.841 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:18:07.942 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:18:07.980 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:18:07.982 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:18:07.982 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:18:08.083 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:18:08.121 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:18:08.125 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:18:08.126 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:18:08.138 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:18:08 smithi089 bash[75068]: cluster 1685823486.8664157 mgr.y (mgr.34104) 1263 : cluster 0 pgmap v1258: 89 pgs: 20 active+clean+remapped, 14 active+undersized+degraded, 11 active+undersized, 44 active+clean; 4.2 GiB data, 14 GiB used, 523 GiB / 536 GiB avail; 73 MiB/s wr, 1.17k op/s; 34738/208311 objects degraded (16.676%); 6/208311 objects misplaced (0.003%) 2023-06-03T20:18:08.139 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:18:08 smithi089 bash[75068]: audit 1685823487.134307 mon.a (mon.0) 1205 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:18:08.139 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:18:08 smithi089 bash[75068]: audit 1685823487.1423402 mon.c (mon.2) 554 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T20:18:08.139 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:18:08 smithi089 bash[75068]: cluster 1685823487.8278682 mon.a (mon.0) 1206 : cluster 3 Health check update: Degraded data redundancy: 34738/208311 objects degraded (16.676%), 14 pgs degraded, 23 pgs undersized (PG_DEGRADED) 2023-06-03T20:18:08.227 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:18:08.260 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:18:08.262 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:18:08.262 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:18:08.278 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:18:08 smithi089 bash[78452]: cluster 1685823486.8664157 mgr.y (mgr.34104) 1263 : cluster 0 pgmap v1258: 89 pgs: 20 active+clean+remapped, 14 active+undersized+degraded, 11 active+undersized, 44 active+clean; 4.2 GiB data, 14 GiB used, 523 GiB / 536 GiB avail; 73 MiB/s wr, 1.17k op/s; 34738/208311 objects degraded (16.676%); 6/208311 objects misplaced (0.003%) 2023-06-03T20:18:08.279 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:18:08 smithi089 bash[78452]: audit 1685823487.134307 mon.a (mon.0) 1205 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:18:08.279 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:18:08 smithi089 bash[78452]: audit 1685823487.1423402 mon.c (mon.2) 554 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T20:18:08.279 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:18:08 smithi089 bash[78452]: cluster 1685823487.8278682 mon.a (mon.0) 1206 : cluster 3 Health check update: Degraded data redundancy: 34738/208311 objects degraded (16.676%), 14 pgs degraded, 23 pgs undersized (PG_DEGRADED) 2023-06-03T20:18:08.363 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:18:08.374 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:18:08 smithi164 bash[59462]: cluster 1685823486.8664157 mgr.y (mgr.34104) 1263 : cluster 0 pgmap v1258: 89 pgs: 20 active+clean+remapped, 14 active+undersized+degraded, 11 active+undersized, 44 active+clean; 4.2 GiB data, 14 GiB used, 523 GiB / 536 GiB avail; 73 MiB/s wr, 1.17k op/s; 34738/208311 objects degraded (16.676%); 6/208311 objects misplaced (0.003%) 2023-06-03T20:18:08.375 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:18:08 smithi164 bash[59462]: audit 1685823487.134307 mon.a (mon.0) 1205 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:18:08.375 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:18:08 smithi164 bash[59462]: audit 1685823487.1423402 mon.c (mon.2) 554 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T20:18:08.375 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:18:08 smithi164 bash[59462]: cluster 1685823487.8278682 mon.a (mon.0) 1206 : cluster 3 Health check update: Degraded data redundancy: 34738/208311 objects degraded (16.676%), 14 pgs degraded, 23 pgs undersized (PG_DEGRADED) 2023-06-03T20:18:08.397 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:18:08.397 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 51 16 72872 72856 89.2674 65.1875 0.0126985 0.0111961 2023-06-03T20:18:08.399 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:18:08.400 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:18:08.501 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:18:08.533 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:18:08.539 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:18:08.539 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:18:08.640 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:18:08.670 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:18:08.674 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:18:08.674 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:18:08.775 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:18:08.805 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:18:08.808 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:18:08.809 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:18:08.910 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:18:08.943 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:18:08.951 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:18:08.951 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:18:09.052 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:18:09.083 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:18:09.088 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:18:09.089 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:18:09.190 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:18:09.222 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:18:09.226 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:18:09.226 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:18:09.327 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:18:09.368 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:18:09.371 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:18:09.372 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:18:09.398 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 52 16 73869 73853 88.749 62.3125 0.0445592 0.0112576 2023-06-03T20:18:09.473 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:18:09.505 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:18:09.508 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:18:09.509 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:18:09.610 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:18:09.641 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:18:09.643 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:18:09.645 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:18:09.746 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:18:09.777 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:18:09.781 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:18:09.782 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:18:09.883 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:18:09.912 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:18:09.914 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:18:09.915 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:18:10.016 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:18:10.047 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:18:10.053 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:18:10.054 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:18:10.155 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:18:10.175 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:18:10.177 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:18:10.177 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:18:10.278 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:18:10.279 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:18:10 smithi089 bash[75068]: cluster 1685823488.8673177 mgr.y (mgr.34104) 1264 : cluster 0 pgmap v1259: 89 pgs: 20 active+clean+remapped, 14 active+undersized+degraded, 11 active+undersized, 44 active+clean; 4.4 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 67 MiB/s wr, 1.06k op/s; 35908/214668 objects degraded (16.727%); 6/214668 objects misplaced (0.003%) 2023-06-03T20:18:10.280 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:18:10 smithi089 bash[78452]: cluster 1685823488.8673177 mgr.y (mgr.34104) 1264 : cluster 0 pgmap v1259: 89 pgs: 20 active+clean+remapped, 14 active+undersized+degraded, 11 active+undersized, 44 active+clean; 4.4 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 67 MiB/s wr, 1.06k op/s; 35908/214668 objects degraded (16.727%); 6/214668 objects misplaced (0.003%) 2023-06-03T20:18:10.322 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:18:10.325 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:18:10.326 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:18:10.398 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 53 16 74791 74775 88.1616 57.625 0.00800644 0.011337 2023-06-03T20:18:10.427 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:18:10.437 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:18:10 smithi164 bash[59462]: cluster 1685823488.8673177 mgr.y (mgr.34104) 1264 : cluster 0 pgmap v1259: 89 pgs: 20 active+clean+remapped, 14 active+undersized+degraded, 11 active+undersized, 44 active+clean; 4.4 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 67 MiB/s wr, 1.06k op/s; 35908/214668 objects degraded (16.727%); 6/214668 objects misplaced (0.003%) 2023-06-03T20:18:10.464 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:18:10.468 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:18:10.469 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:18:10.570 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:18:10.604 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:18:10.606 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:18:10.607 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:18:10.708 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:18:10.737 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:18:10.738 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:18:10.739 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:18:10.840 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:18:10.878 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:18:10.881 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:18:10.882 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:18:10.983 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:18:11.025 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:18:11.029 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:18:11.030 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:18:11.131 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:18:11.158 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:18:11.159 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:18:11.160 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:18:11.261 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:18:11.300 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:18:11.306 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:18:11.307 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:18:11.398 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 54 16 75526 75510 87.3797 45.9375 0.0118816 0.0114391 2023-06-03T20:18:11.408 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:18:11.444 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:18:11.449 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:18:11.450 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:18:11.551 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:18:11.590 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:18:11.593 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:18:11.594 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:18:11.695 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:18:11.728 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:18:11.733 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:18:11.733 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:18:11.834 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:18:11.876 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:18:11.882 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:18:11.883 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:18:11.983 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:18:12.011 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:18:12.014 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:18:12.014 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:18:12.115 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:18:12.153 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:18:12.155 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:18:12.155 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:18:12.256 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:18:12.278 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:18:12 smithi089 bash[75068]: cluster 1685823490.8681211 mgr.y (mgr.34104) 1265 : cluster 0 pgmap v1260: 89 pgs: 20 active+clean+remapped, 14 active+undersized+degraded, 11 active+undersized, 44 active+clean; 4.5 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 62 MiB/s wr, 993 op/s; 36802/219297 objects degraded (16.782%); 6/219297 objects misplaced (0.003%) 2023-06-03T20:18:12.279 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:18:12 smithi089 bash[78452]: cluster 1685823490.8681211 mgr.y (mgr.34104) 1265 : cluster 0 pgmap v1260: 89 pgs: 20 active+clean+remapped, 14 active+undersized+degraded, 11 active+undersized, 44 active+clean; 4.5 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 62 MiB/s wr, 993 op/s; 36802/219297 objects degraded (16.782%); 6/219297 objects misplaced (0.003%) 2023-06-03T20:18:12.292 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:18:12.296 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:18:12.296 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:18:12.397 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:18:12.398 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 55 16 76310 76294 86.6819 49 0.0098245 0.0115311 2023-06-03T20:18:12.432 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:18:12.435 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:18:12.436 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:18:12.536 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:18:12.548 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/mon.a/config 2023-06-03T20:18:12.575 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:18:12.583 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:18:12.584 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:18:12.584 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:18:12 smithi164 bash[59462]: cluster 1685823490.8681211 mgr.y (mgr.34104) 1265 : cluster 0 pgmap v1260: 89 pgs: 20 active+clean+remapped, 14 active+undersized+degraded, 11 active+undersized, 44 active+clean; 4.5 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 62 MiB/s wr, 993 op/s; 36802/219297 objects degraded (16.782%); 6/219297 objects misplaced (0.003%) 2023-06-03T20:18:12.685 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:18:12.714 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:18:12.716 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:18:12.716 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:18:12.817 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:18:12.852 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:18:12.855 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:18:12.855 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:18:12.956 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:18:12.987 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:18:12.990 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:18:12.990 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:18:13.091 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:18:13.130 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:18:13.133 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:18:13.134 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:18:13.235 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:18:13.267 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:18:13.270 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:18:13.271 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:18:13.334 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:18:13 smithi164 bash[59462]: cluster 1685823492.831312 mon.a (mon.0) 1207 : cluster 3 Health check update: Degraded data redundancy: 36802/219297 objects degraded (16.782%), 14 pgs degraded, 23 pgs undersized (PG_DEGRADED) 2023-06-03T20:18:13.355 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:18:13 smithi089 bash[78452]: cluster 1685823492.831312 mon.a (mon.0) 1207 : cluster 3 Health check update: Degraded data redundancy: 36802/219297 objects degraded (16.782%), 14 pgs degraded, 23 pgs undersized (PG_DEGRADED) 2023-06-03T20:18:13.355 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:18:13 smithi089 bash[75068]: cluster 1685823492.831312 mon.a (mon.0) 1207 : cluster 3 Health check update: Degraded data redundancy: 36802/219297 objects degraded (16.782%), 14 pgs degraded, 23 pgs undersized (PG_DEGRADED) 2023-06-03T20:18:13.372 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:18:13.398 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 56 16 77290 77274 86.2272 61.25 0.024997 0.0115919 2023-06-03T20:18:13.406 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:18:13.409 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:18:13.410 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:18:13.511 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:18:13.540 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:18:13.548 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:18:13.549 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:18:13.650 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:18:13.683 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:18:13.686 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:18:13.687 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:18:13.788 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:18:13.819 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:18:13.821 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:18:13.822 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:18:13.923 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:18:13.960 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:18:13.966 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:18:13.967 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:18:14.068 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:18:14.104 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:18:14.106 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:18:14.107 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:18:14.208 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:18:14.216 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:18:14 smithi089 bash[78452]: cluster 1685823492.8689535 mgr.y (mgr.34104) 1266 : cluster 0 pgmap v1261: 89 pgs: 20 active+clean+remapped, 14 active+undersized+degraded, 11 active+undersized, 44 active+clean; 4.5 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 59 MiB/s wr, 949 op/s; 37096/222801 objects degraded (16.650%); 6/222801 objects misplaced (0.003%) 2023-06-03T20:18:14.216 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:18:14 smithi089 bash[75068]: cluster 1685823492.8689535 mgr.y (mgr.34104) 1266 : cluster 0 pgmap v1261: 89 pgs: 20 active+clean+remapped, 14 active+undersized+degraded, 11 active+undersized, 44 active+clean; 4.5 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 59 MiB/s wr, 949 op/s; 37096/222801 objects degraded (16.650%); 6/222801 objects misplaced (0.003%) 2023-06-03T20:18:14.236 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:18:14.238 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:18:14.239 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:18:14.333 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:18:14 smithi164 bash[59462]: cluster 1685823492.8689535 mgr.y (mgr.34104) 1266 : cluster 0 pgmap v1261: 89 pgs: 20 active+clean+remapped, 14 active+undersized+degraded, 11 active+undersized, 44 active+clean; 4.5 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 59 MiB/s wr, 949 op/s; 37096/222801 objects degraded (16.650%); 6/222801 objects misplaced (0.003%) 2023-06-03T20:18:14.340 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:18:14.371 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:18:14.373 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:18:14.373 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:18:14.399 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 57 15 78289 78274 85.8109 62.5 0.0121712 0.0116485 2023-06-03T20:18:14.474 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:18:14.509 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:18:14.511 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:18:14.511 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:18:14.612 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:18:14.644 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:18:14.647 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:18:14.648 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:18:14.749 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:18:14.784 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:18:14.787 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:18:14.788 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:18:14.889 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:18:14.920 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:18:14.923 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:18:14.924 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:18:15.025 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:18:15.060 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:18:15.067 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:18:15.067 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:18:15.168 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:18:15.199 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:18:15.202 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:18:15.203 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:18:15.304 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:18:15.330 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:18:15.331 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:18:15.332 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:18:15.399 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 58 16 79227 79211 85.341 58.5625 0.0069635 0.011695 2023-06-03T20:18:15.433 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:18:15.464 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:18:15.466 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:18:15.466 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:18:15.567 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:18:15.609 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:18:15.612 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:18:15.612 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:18:15.713 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:18:15.740 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:18:15.744 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:18:15.744 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:18:15.845 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:18:15.881 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:18:15.886 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:18:15.887 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:18:15.988 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:18:16.013 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:18:16.015 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:18:16.015 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:18:16.116 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:18:16.155 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:18:16 smithi164 bash[59462]: cluster 1685823494.8698666 mgr.y (mgr.34104) 1267 : cluster 0 pgmap v1262: 89 pgs: 20 active+clean+remapped, 14 active+undersized+degraded, 11 active+undersized, 44 active+clean; 4.6 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 62 MiB/s wr, 998 op/s; 38204/228579 objects degraded (16.714%); 6/228579 objects misplaced (0.003%) 2023-06-03T20:18:16.155 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:18:16.159 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:18:16.160 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:18:16.261 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:18:16.271 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:18:16 smithi089 bash[75068]: cluster 1685823494.8698666 mgr.y (mgr.34104) 1267 : cluster 0 pgmap v1262: 89 pgs: 20 active+clean+remapped, 14 active+undersized+degraded, 11 active+undersized, 44 active+clean; 4.6 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 62 MiB/s wr, 998 op/s; 38204/228579 objects degraded (16.714%); 6/228579 objects misplaced (0.003%) 2023-06-03T20:18:16.272 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 20:18:16 smithi089 bash[71163]: ::ffff:172.21.15.164 - - [03/Jun/2023:20:18:16] "GET /metrics HTTP/1.1" 200 228952 "" "Prometheus/2.33.4" 2023-06-03T20:18:16.272 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:18:16 smithi089 bash[78452]: cluster 1685823494.8698666 mgr.y (mgr.34104) 1267 : cluster 0 pgmap v1262: 89 pgs: 20 active+clean+remapped, 14 active+undersized+degraded, 11 active+undersized, 44 active+clean; 4.6 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 62 MiB/s wr, 998 op/s; 38204/228579 objects degraded (16.714%); 6/228579 objects misplaced (0.003%) 2023-06-03T20:18:16.297 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:18:16.303 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:18:16.304 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:18:16.399 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 59 16 80082 80066 84.8003 53.4375 0.00675232 0.0117867 2023-06-03T20:18:16.405 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:18:16.436 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:18:16.442 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:18:16.442 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:18:16.543 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:18:16.575 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:18:16.577 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:18:16.577 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:18:16.678 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:18:16.706 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:18:16.708 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:18:16.708 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:18:16.809 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:18:16.841 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:18:16.843 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:18:16.843 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:18:16.944 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:18:16.975 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:18:16.979 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:18:16.979 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:18:17.081 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:18:17.112 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:18:17.117 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:18:17.118 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:18:17.219 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:18:17.252 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:18:17.257 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:18:17.258 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:18:17.359 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:18:17.397 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:18:17.399 INFO:tasks.radosbench.radosbench.0.smithi089.stdout:2023-06-03T20:18:17.395141+0000 min lat: 0.00129872 max lat: 0.870111 avg lat: 0.0118442 2023-06-03T20:18:17.399 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-06-03T20:18:17.399 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 60 16 81038 81022 84.3827 59.75 0.008816 0.0118442 2023-06-03T20:18:17.402 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:18:17.403 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:18:17.504 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:18:17.537 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:18:17.544 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:18:17.545 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:18:17.571 INFO:teuthology.orchestra.run.smithi089.stdout:{ 2023-06-03T20:18:17.571 INFO:teuthology.orchestra.run.smithi089.stdout: "success": "" 2023-06-03T20:18:17.571 INFO:teuthology.orchestra.run.smithi089.stdout:} 2023-06-03T20:18:17.646 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:18:17.670 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:18:17.671 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:18:17.672 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:18:17.773 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:18:17.806 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:18:17.809 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:18:17.809 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:18:17.910 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:18:17.941 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:18:17.946 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:18:17.947 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:18:18.048 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:18:18.069 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:18:18.070 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:18:18.071 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:18:18.172 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:18:18.203 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:18:18.205 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:18:18.205 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:18:18.279 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:18:18 smithi089 bash[78452]: cluster 1685823496.8706622 mgr.y (mgr.34104) 1268 : cluster 0 pgmap v1263: 89 pgs: 20 active+clean+remapped, 14 active+undersized+degraded, 11 active+undersized, 44 active+clean; 4.8 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 63 MiB/s wr, 1.01k op/s; 39329/236112 objects degraded (16.657%); 6/236112 objects misplaced (0.003%) 2023-06-03T20:18:18.279 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:18:18 smithi089 bash[78452]: cluster 1685823497.8323615 mon.a (mon.0) 1208 : cluster 3 Health check update: Degraded data redundancy: 39329/236112 objects degraded (16.657%), 14 pgs degraded, 23 pgs undersized (PG_DEGRADED) 2023-06-03T20:18:18.279 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:18:18 smithi089 bash[75068]: cluster 1685823496.8706622 mgr.y (mgr.34104) 1268 : cluster 0 pgmap v1263: 89 pgs: 20 active+clean+remapped, 14 active+undersized+degraded, 11 active+undersized, 44 active+clean; 4.8 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 63 MiB/s wr, 1.01k op/s; 39329/236112 objects degraded (16.657%); 6/236112 objects misplaced (0.003%) 2023-06-03T20:18:18.280 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:18:18 smithi089 bash[75068]: cluster 1685823497.8323615 mon.a (mon.0) 1208 : cluster 3 Health check update: Degraded data redundancy: 39329/236112 objects degraded (16.657%), 14 pgs degraded, 23 pgs undersized (PG_DEGRADED) 2023-06-03T20:18:18.306 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:18:18.337 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:18:18.339 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:18:18.340 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:18:18.399 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 61 16 81871 81855 83.8528 52.0625 0.020399 0.0119188 2023-06-03T20:18:18.441 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:18:18.454 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:18:18 smithi164 bash[59462]: cluster 1685823496.8706622 mgr.y (mgr.34104) 1268 : cluster 0 pgmap v1263: 89 pgs: 20 active+clean+remapped, 14 active+undersized+degraded, 11 active+undersized, 44 active+clean; 4.8 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 63 MiB/s wr, 1.01k op/s; 39329/236112 objects degraded (16.657%); 6/236112 objects misplaced (0.003%) 2023-06-03T20:18:18.455 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:18:18 smithi164 bash[59462]: cluster 1685823497.8323615 mon.a (mon.0) 1208 : cluster 3 Health check update: Degraded data redundancy: 39329/236112 objects degraded (16.657%), 14 pgs degraded, 23 pgs undersized (PG_DEGRADED) 2023-06-03T20:18:18.481 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:18:18.487 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:18:18.487 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:18:18.588 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:18:18.623 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:18:18.627 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:18:18.627 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:18:18.728 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:18:18.759 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:18:18.760 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:18:18.761 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:18:18.862 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:18:18.895 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:18:18.897 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:18:18.898 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:18:18.998 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:18:19.030 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:18:19.031 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:18:19.031 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:18:19.132 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:18:19.237 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:18:19.240 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:18:19.241 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:18:19.342 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:18:19.376 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:18:19.378 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:18:19.379 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:18:19.399 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 62 16 82745 82729 83.3814 54.625 0.0217925 0.011987 2023-06-03T20:18:19.480 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:18:19.509 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:18:19.511 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:18:19.511 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:18:19.612 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:18:19.644 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:18:19.646 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:18:19.647 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:18:19.748 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:18:19.783 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:18:19.785 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:18:19.786 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:18:19.887 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:18:19.923 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:18:19.928 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:18:19.929 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:18:20.030 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:18:20.062 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:18:20.064 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:18:20.065 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:18:20.165 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:18:20 smithi089 bash[75068]: cluster 1685823498.8714912 mgr.y (mgr.34104) 1269 : cluster 0 pgmap v1264: 89 pgs: 20 active+clean+remapped, 14 active+undersized+degraded, 11 active+undersized, 44 active+clean; 4.9 GiB data, 16 GiB used, 521 GiB / 536 GiB avail; 59 MiB/s wr, 936 op/s; 40457/242025 objects degraded (16.716%); 6/242025 objects misplaced (0.002%) 2023-06-03T20:18:20.165 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:18:20.205 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:18:20.208 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:18:20.209 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:18:20.310 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:18:20.334 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:18:20 smithi164 bash[59462]: cluster 1685823498.8714912 mgr.y (mgr.34104) 1269 : cluster 0 pgmap v1264: 89 pgs: 20 active+clean+remapped, 14 active+undersized+degraded, 11 active+undersized, 44 active+clean; 4.9 GiB data, 16 GiB used, 521 GiB / 536 GiB avail; 59 MiB/s wr, 936 op/s; 40457/242025 objects degraded (16.716%); 6/242025 objects misplaced (0.002%) 2023-06-03T20:18:20.338 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:18:20.343 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:18:20.344 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:18:20.399 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 63 16 83455 83439 82.7622 44.375 0.00850831 0.0120769 2023-06-03T20:18:20.445 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:18:20.455 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:18:20 smithi089 bash[78452]: cluster 1685823498.8714912 mgr.y (mgr.34104) 1269 : cluster 0 pgmap v1264: 89 pgs: 20 active+clean+remapped, 14 active+undersized+degraded, 11 active+undersized, 44 active+clean; 4.9 GiB data, 16 GiB used, 521 GiB / 536 GiB avail; 59 MiB/s wr, 936 op/s; 40457/242025 objects degraded (16.716%); 6/242025 objects misplaced (0.002%) 2023-06-03T20:18:20.480 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:18:20.485 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:18:20.486 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:18:20.587 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:18:20.633 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:18:20.637 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:18:20.638 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:18:20.739 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:18:20.776 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:18:20.777 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:18:20.778 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:18:20.878 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:18:20.912 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:18:20.917 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:18:20.917 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:18:21.018 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:18:21.061 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:18:21.065 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:18:21.066 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:18:21.167 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:18:21.200 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:18:21.206 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:18:21.206 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:18:21.307 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:18:21.327 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:18:21.329 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:18:21.330 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:18:21.399 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 64 16 84246 84230 82.2415 49.4375 0.128186 0.0121513 2023-06-03T20:18:21.431 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:18:21.463 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:18:21.465 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:18:21.466 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:18:21.567 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:18:21.596 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:18:21.602 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:18:21.602 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:18:21.703 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:18:21.745 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:18:21.747 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:18:21.748 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:18:21.849 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:18:21.879 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:18:21.882 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:18:21.882 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:18:21.983 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:18:22.013 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:18:22.019 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:18:22.020 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:18:22.121 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:18:22.155 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:18:22.157 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:18:22.158 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:18:22.168 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:18:22 smithi164 bash[59462]: cluster 1685823500.8726451 mgr.y (mgr.34104) 1270 : cluster 0 pgmap v1265: 89 pgs: 20 active+clean+remapped, 14 active+undersized+degraded, 11 active+undersized, 44 active+clean; 5.0 GiB data, 16 GiB used, 521 GiB / 536 GiB avail; 54 MiB/s wr, 869 op/s; 41224/245997 objects degraded (16.758%); 6/245997 objects misplaced (0.002%) 2023-06-03T20:18:22.168 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:18:22 smithi164 bash[59462]: audit 1685823502.1311889 mon.c (mon.2) 555 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T20:18:22.259 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:18:22.267 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:18:22 smithi089 bash[78452]: cluster 1685823500.8726451 mgr.y (mgr.34104) 1270 : cluster 0 pgmap v1265: 89 pgs: 20 active+clean+remapped, 14 active+undersized+degraded, 11 active+undersized, 44 active+clean; 5.0 GiB data, 16 GiB used, 521 GiB / 536 GiB avail; 54 MiB/s wr, 869 op/s; 41224/245997 objects degraded (16.758%); 6/245997 objects misplaced (0.002%) 2023-06-03T20:18:22.268 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:18:22 smithi089 bash[78452]: audit 1685823502.1311889 mon.c (mon.2) 555 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T20:18:22.268 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:18:22 smithi089 bash[75068]: cluster 1685823500.8726451 mgr.y (mgr.34104) 1270 : cluster 0 pgmap v1265: 89 pgs: 20 active+clean+remapped, 14 active+undersized+degraded, 11 active+undersized, 44 active+clean; 5.0 GiB data, 16 GiB used, 521 GiB / 536 GiB avail; 54 MiB/s wr, 869 op/s; 41224/245997 objects degraded (16.758%); 6/245997 objects misplaced (0.002%) 2023-06-03T20:18:22.268 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:18:22 smithi089 bash[75068]: audit 1685823502.1311889 mon.c (mon.2) 555 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T20:18:22.287 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:18:22.290 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:18:22.291 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:18:22.391 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:18:22.399 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 65 16 85198 85182 81.8916 59.5 0.00654555 0.0122055 2023-06-03T20:18:22.418 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:18:22.420 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:18:22.420 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:18:22.512 INFO:teuthology.orchestra.run.smithi089.stdout:ERROR: (22) Invalid argument 2023-06-03T20:18:22.512 INFO:teuthology.orchestra.run.smithi089.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-06-03T20:18:22.522 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:18:22.552 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:18:22.554 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:18:22.555 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:18:22.656 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:18:22.681 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:18:22.688 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:18:22.689 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:18:22.790 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:18:22.822 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:18:22.825 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:18:22.826 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:18:22.927 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:18:22.963 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:18:22.966 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:18:22.966 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:18:23.067 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:18:23.094 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:18:23.097 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:18:23.097 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:18:23.198 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:18:23.242 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:18:23.247 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:18:23 smithi089 bash[75068]: cluster 1685823502.8333147 mon.a (mon.0) 1209 : cluster 3 Health check update: Degraded data redundancy: 41224/245997 objects degraded (16.758%), 14 pgs degraded, 23 pgs undersized (PG_DEGRADED) 2023-06-03T20:18:23.248 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:18:23 smithi089 bash[78452]: cluster 1685823502.8333147 mon.a (mon.0) 1209 : cluster 3 Health check update: Degraded data redundancy: 41224/245997 objects degraded (16.758%), 14 pgs degraded, 23 pgs undersized (PG_DEGRADED) 2023-06-03T20:18:23.248 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:18:23.249 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:18:23.349 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:18:23.379 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:18:23.381 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:18:23.382 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:18:23.399 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 66 16 85987 85971 81.3979 49.3125 0.00692341 0.0122805 2023-06-03T20:18:23.482 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:18:23.513 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:18:23.517 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:18:23.517 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:18:23.583 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:18:23 smithi164 bash[59462]: cluster 1685823502.8333147 mon.a (mon.0) 1209 : cluster 3 Health check update: Degraded data redundancy: 41224/245997 objects degraded (16.758%), 14 pgs degraded, 23 pgs undersized (PG_DEGRADED) 2023-06-03T20:18:23.618 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:18:23.661 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:18:23.663 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:18:23.664 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:18:23.765 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:18:23.785 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:18:23.786 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:18:23.787 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:18:23.888 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:18:23.922 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:18:23.926 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:18:23.926 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:18:24.027 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:18:24.072 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:18:24.077 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:18:24.078 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:18:24.171 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:18:24 smithi089 bash[75068]: cluster 1685823502.8735702 mgr.y (mgr.34104) 1271 : cluster 0 pgmap v1266: 89 pgs: 20 active+clean+remapped, 14 active+undersized+degraded, 11 active+undersized, 44 active+clean; 5.1 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 52 MiB/s wr, 826 op/s; 41472/249066 objects degraded (16.651%); 6/249066 objects misplaced (0.002%) 2023-06-03T20:18:24.179 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:18:24.214 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:18:24.217 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:18:24.217 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:18:24.318 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:18:24.346 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:18:24.348 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:18:24.349 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:18:24.399 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 67 16 86854 86838 80.9917 54.1875 0.00743087 0.0123415 2023-06-03T20:18:24.450 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:18:24.459 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:18:24 smithi164 bash[59462]: cluster 1685823502.8735702 mgr.y (mgr.34104) 1271 : cluster 0 pgmap v1266: 89 pgs: 20 active+clean+remapped, 14 active+undersized+degraded, 11 active+undersized, 44 active+clean; 5.1 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 52 MiB/s wr, 826 op/s; 41472/249066 objects degraded (16.651%); 6/249066 objects misplaced (0.002%) 2023-06-03T20:18:24.480 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:18:24.483 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:18:24.484 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:18:24.528 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:18:24 smithi089 bash[78452]: cluster 1685823502.8735702 mgr.y (mgr.34104) 1271 : cluster 0 pgmap v1266: 89 pgs: 20 active+clean+remapped, 14 active+undersized+degraded, 11 active+undersized, 44 active+clean; 5.1 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 52 MiB/s wr, 826 op/s; 41472/249066 objects degraded (16.651%); 6/249066 objects misplaced (0.002%) 2023-06-03T20:18:24.586 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:18:24.616 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:18:24.620 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:18:24.621 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:18:24.721 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:18:24.754 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:18:24.756 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:18:24.757 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:18:24.857 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:18:24.890 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:18:24.893 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:18:24.894 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:18:24.995 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:18:25.028 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:18:25.035 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:18:25.035 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:18:25.136 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:18:25.168 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:18:25.174 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:18:25.175 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:18:25.276 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:18:25.308 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:18:25.312 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:18:25.313 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:18:25.400 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 68 16 87815 87799 80.6839 60.0625 0.0304013 0.0123897 2023-06-03T20:18:25.415 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:18:25.448 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:18:25.450 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:18:25.450 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:18:25.551 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:18:25.576 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:18:25.580 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:18:25.581 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:18:25.682 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:18:25.722 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:18:25.724 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:18:25.725 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:18:25.825 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:18:25.873 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:18:25.876 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:18:25.876 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:18:25.977 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:18:26.008 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:18:26.010 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:18:26.011 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:18:26.112 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:18:26.141 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:18:26.145 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:18:26.146 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:18:26.247 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:18:26.277 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:18:26.280 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:18:26.281 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:18:26.382 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:18:26.400 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 69 16 88587 88571 80.2138 48.25 0.0117638 0.0124607 2023-06-03T20:18:26.412 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:18:26.414 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:18:26.415 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:18:26.516 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:18:26.522 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:18:26 smithi089 bash[75068]: cluster 1685823504.8744066 mgr.y (mgr.34104) 1272 : cluster 0 pgmap v1267: 89 pgs: 20 active+clean+remapped, 14 active+undersized+degraded, 11 active+undersized, 44 active+clean; 5.2 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 55 MiB/s wr, 886 op/s; 42569/254718 objects degraded (16.712%); 6/254718 objects misplaced (0.002%) 2023-06-03T20:18:26.523 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 20:18:26 smithi089 bash[71163]: ::ffff:172.21.15.164 - - [03/Jun/2023:20:18:26] "GET /metrics HTTP/1.1" 200 228936 "" "Prometheus/2.33.4" 2023-06-03T20:18:26.523 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:18:26 smithi089 bash[78452]: cluster 1685823504.8744066 mgr.y (mgr.34104) 1272 : cluster 0 pgmap v1267: 89 pgs: 20 active+clean+remapped, 14 active+undersized+degraded, 11 active+undersized, 44 active+clean; 5.2 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 55 MiB/s wr, 886 op/s; 42569/254718 objects degraded (16.712%); 6/254718 objects misplaced (0.002%) 2023-06-03T20:18:26.536 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:18:26.538 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:18:26.538 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:18:26.583 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:18:26 smithi164 bash[59462]: cluster 1685823504.8744066 mgr.y (mgr.34104) 1272 : cluster 0 pgmap v1267: 89 pgs: 20 active+clean+remapped, 14 active+undersized+degraded, 11 active+undersized, 44 active+clean; 5.2 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 55 MiB/s wr, 886 op/s; 42569/254718 objects degraded (16.712%); 6/254718 objects misplaced (0.002%) 2023-06-03T20:18:26.639 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:18:26.674 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:18:26.677 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:18:26.678 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:18:26.779 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:18:26.809 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:18:26.810 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:18:26.811 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:18:26.855 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph daemon osd.3 dump_historic_ops 2023-06-03T20:18:26.912 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:18:26.951 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:18:26.954 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:18:26.954 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:18:27.055 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:18:27.091 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:18:27.098 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:18:27.098 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:18:27.199 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:18:27.228 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:18:27.231 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:18:27.232 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:18:27.333 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:18:27.371 INFO:tasks.thrashosds.thrasher:in_osds: [0, 2, 3, 4, 5, 7] out_osds: [1, 6] dead_osds: [4] live_osds: [0, 1, 2, 3, 5, 6, 7] 2023-06-03T20:18:27.371 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-06-03T20:18:27.372 INFO:tasks.thrashosds.thrasher:Growing pool unique_pool_7 2023-06-03T20:18:27.372 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph pg dump --format=json 2023-06-03T20:18:27.375 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:18:27.377 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:18:27.378 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:18:27.400 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 70 16 89274 89258 79.6805 42.9375 0.00706853 0.0125437 2023-06-03T20:18:27.479 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:18:27.511 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:18:27.513 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:18:27.513 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:18:27.614 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:18:27.647 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:18:27.656 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:18:27.657 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:18:27.758 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:18:27.793 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:18:27.802 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:18:27.803 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:18:27.904 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:18:27.943 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:18:27.946 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:18:27.947 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:18:28.048 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:18:28.084 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:18:28.088 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:18:28.089 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:18:28.190 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:18:28.216 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:18:28.220 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:18:28.221 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:18:28.317 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:18:28 smithi089 bash[75068]: cluster 1685823506.8752754 mgr.y (mgr.34104) 1273 : cluster 0 pgmap v1268: 89 pgs: 20 active+clean+remapped, 14 active+undersized+degraded, 11 active+undersized, 44 active+clean; 5.3 GiB data, 17 GiB used, 520 GiB / 536 GiB avail; 58 MiB/s wr, 927 op/s; 43641/261990 objects degraded (16.658%); 6/261990 objects misplaced (0.002%) 2023-06-03T20:18:28.317 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:18:28 smithi089 bash[75068]: cluster 1685823507.8345566 mon.a (mon.0) 1210 : cluster 3 Health check update: Degraded data redundancy: 43641/261990 objects degraded (16.658%), 14 pgs degraded, 25 pgs undersized (PG_DEGRADED) 2023-06-03T20:18:28.317 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:18:28 smithi089 bash[75068]: audit 1685823508.0832884 mon.a (mon.0) 1211 : audit 1 from='client.? 172.21.15.89:0/3516764658' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-06-03T20:18:28.322 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:18:28.333 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:18:28 smithi164 bash[59462]: cluster 1685823506.8752754 mgr.y (mgr.34104) 1273 : cluster 0 pgmap v1268: 89 pgs: 20 active+clean+remapped, 14 active+undersized+degraded, 11 active+undersized, 44 active+clean; 5.3 GiB data, 17 GiB used, 520 GiB / 536 GiB avail; 58 MiB/s wr, 927 op/s; 43641/261990 objects degraded (16.658%); 6/261990 objects misplaced (0.002%) 2023-06-03T20:18:28.334 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:18:28 smithi164 bash[59462]: cluster 1685823507.8345566 mon.a (mon.0) 1210 : cluster 3 Health check update: Degraded data redundancy: 43641/261990 objects degraded (16.658%), 14 pgs degraded, 25 pgs undersized (PG_DEGRADED) 2023-06-03T20:18:28.334 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:18:28 smithi164 bash[59462]: audit 1685823508.0832884 mon.a (mon.0) 1211 : audit 1 from='client.? 172.21.15.89:0/3516764658' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-06-03T20:18:28.370 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:18:28.376 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:18:28.377 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:18:28.400 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 71 15 90102 90087 79.288 51.8125 0.0112158 0.0126058 2023-06-03T20:18:28.478 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:18:28.508 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:18:28.512 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:18:28.513 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:18:28.614 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:18:28.622 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:18:28 smithi089 bash[78452]: cluster 1685823506.8752754 mgr.y (mgr.34104) 1273 : cluster 0 pgmap v1268: 89 pgs: 20 active+clean+remapped, 14 active+undersized+degraded, 11 active+undersized, 44 active+clean; 5.3 GiB data, 17 GiB used, 520 GiB / 536 GiB avail; 58 MiB/s wr, 927 op/s; 43641/261990 objects degraded (16.658%); 6/261990 objects misplaced (0.002%) 2023-06-03T20:18:28.623 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:18:28 smithi089 bash[78452]: cluster 1685823507.8345566 mon.a (mon.0) 1210 : cluster 3 Health check update: Degraded data redundancy: 43641/261990 objects degraded (16.658%), 14 pgs degraded, 25 pgs undersized (PG_DEGRADED) 2023-06-03T20:18:28.623 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:18:28 smithi089 bash[78452]: audit 1685823508.0832884 mon.a (mon.0) 1211 : audit 1 from='client.? 172.21.15.89:0/3516764658' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-06-03T20:18:28.651 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:18:28.655 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:18:28.656 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:18:28.757 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:18:28.793 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:18:28.798 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:18:28.798 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:18:28.855 INFO:teuthology.orchestra.run.smithi089.stderr:nodeep-scrub is set 2023-06-03T20:18:28.899 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:18:28.932 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:18:28.934 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:18:28.935 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:18:29.036 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:18:29.066 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:18:29.069 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:18:29.070 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:18:29.170 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:18:29.333 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:18:29 smithi164 bash[59462]: cluster 1685823508.8369958 mon.a (mon.0) 1212 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-06-03T20:18:29.334 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:18:29 smithi164 bash[59462]: audit 1685823508.8464289 mon.a (mon.0) 1213 : audit 1 from='client.? 172.21.15.89:0/3516764658' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-06-03T20:18:29.334 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:18:29 smithi164 bash[59462]: cluster 1685823508.8466473 mon.a (mon.0) 1214 : cluster 0 osdmap e226: 8 total, 7 up, 6 in 2023-06-03T20:18:29.398 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:18:29.401 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 72 16 91032 91016 78.9931 58.0625 0.0147696 0.0126543 2023-06-03T20:18:29.408 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:18:29.409 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:18:29.510 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:18:29.528 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:18:29 smithi089 bash[78452]: cluster 1685823508.8369958 mon.a (mon.0) 1212 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-06-03T20:18:29.529 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:18:29 smithi089 bash[78452]: audit 1685823508.8464289 mon.a (mon.0) 1213 : audit 1 from='client.? 172.21.15.89:0/3516764658' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-06-03T20:18:29.529 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:18:29 smithi089 bash[78452]: cluster 1685823508.8466473 mon.a (mon.0) 1214 : cluster 0 osdmap e226: 8 total, 7 up, 6 in 2023-06-03T20:18:29.529 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:18:29 smithi089 bash[75068]: cluster 1685823508.8369958 mon.a (mon.0) 1212 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-06-03T20:18:29.529 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:18:29 smithi089 bash[75068]: audit 1685823508.8464289 mon.a (mon.0) 1213 : audit 1 from='client.? 172.21.15.89:0/3516764658' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-06-03T20:18:29.529 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:18:29 smithi089 bash[75068]: cluster 1685823508.8466473 mon.a (mon.0) 1214 : cluster 0 osdmap e226: 8 total, 7 up, 6 in 2023-06-03T20:18:29.537 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:18:29.539 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:18:29.540 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:18:29.640 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:18:29.673 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:18:29.679 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:18:29.680 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:18:29.781 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:18:29.808 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:18:29.811 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:18:29.811 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:18:29.912 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:18:29.952 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:18:29.960 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:18:29.961 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:18:30.062 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:18:30.107 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:18:30.111 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:18:30.111 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:18:30.212 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:18:30.249 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:18:30.258 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:18:30.258 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:18:30.359 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:18:30.387 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:18:30.394 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:18:30.395 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:18:30.401 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 73 16 91941 91925 78.6892 56.8125 0.0236175 0.0127027 2023-06-03T20:18:30.496 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:18:30.504 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:18:30 smithi089 bash[78452]: cluster 1685823508.8412118 osd.6 (osd.6) 169 : cluster 3 failed to encode map e226 with expected crc 2023-06-03T20:18:30.504 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:18:30 smithi089 bash[78452]: cluster 1685823508.8428438 osd.5 (osd.5) 163 : cluster 3 failed to encode map e226 with expected crc 2023-06-03T20:18:30.505 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:18:30 smithi089 bash[78452]: cluster 1685823508.875981 mgr.y (mgr.34104) 1274 : cluster 0 pgmap v1270: 89 pgs: 20 active+clean+remapped, 14 active+undersized+degraded, 11 active+undersized, 44 active+clean; 5.4 GiB data, 17 GiB used, 519 GiB / 536 GiB avail; 53 MiB/s wr, 842 op/s; 44675/267324 objects degraded (16.712%); 6/267324 objects misplaced (0.002%) 2023-06-03T20:18:30.507 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:18:30 smithi089 bash[75068]: cluster 1685823508.8412118 osd.6 (osd.6) 169 : cluster 3 failed to encode map e226 with expected crc 2023-06-03T20:18:30.507 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:18:30 smithi089 bash[75068]: cluster 1685823508.8428438 osd.5 (osd.5) 163 : cluster 3 failed to encode map e226 with expected crc 2023-06-03T20:18:30.508 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:18:30 smithi089 bash[75068]: cluster 1685823508.875981 mgr.y (mgr.34104) 1274 : cluster 0 pgmap v1270: 89 pgs: 20 active+clean+remapped, 14 active+undersized+degraded, 11 active+undersized, 44 active+clean; 5.4 GiB data, 17 GiB used, 519 GiB / 536 GiB avail; 53 MiB/s wr, 842 op/s; 44675/267324 objects degraded (16.712%); 6/267324 objects misplaced (0.002%) 2023-06-03T20:18:30.534 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:18:30.538 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:18:30.538 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:18:30.583 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:18:30 smithi164 bash[59462]: cluster 1685823508.8412118 osd.6 (osd.6) 169 : cluster 3 failed to encode map e226 with expected crc 2023-06-03T20:18:30.584 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:18:30 smithi164 bash[59462]: cluster 1685823508.8428438 osd.5 (osd.5) 163 : cluster 3 failed to encode map e226 with expected crc 2023-06-03T20:18:30.584 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:18:30 smithi164 bash[59462]: cluster 1685823508.875981 mgr.y (mgr.34104) 1274 : cluster 0 pgmap v1270: 89 pgs: 20 active+clean+remapped, 14 active+undersized+degraded, 11 active+undersized, 44 active+clean; 5.4 GiB data, 17 GiB used, 519 GiB / 536 GiB avail; 53 MiB/s wr, 842 op/s; 44675/267324 objects degraded (16.712%); 6/267324 objects misplaced (0.002%) 2023-06-03T20:18:30.639 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:18:30.678 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:18:30.683 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:18:30.684 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:18:30.785 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:18:30.815 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:18:30.822 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:18:30.822 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:18:30.923 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:18:30.959 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:18:30.963 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:18:30.963 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:18:31.064 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:18:31.097 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:18:31.100 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:18:31.101 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:18:31.202 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:18:31.230 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:18:31.233 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:18:31.233 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:18:31.334 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:18:31.367 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:18:31.371 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:18:31.371 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:18:31.401 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 74 16 92858 92842 78.4003 57.3125 0.0132536 0.0127492 2023-06-03T20:18:31.402 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:18:31 smithi089 bash[75068]: cluster 1685823510.8774295 mgr.y (mgr.34104) 1275 : cluster 0 pgmap v1271: 89 pgs: 20 active+clean+remapped, 14 active+undersized+degraded, 11 active+undersized, 44 active+clean; 5.5 GiB data, 17 GiB used, 519 GiB / 536 GiB avail; 52 MiB/s wr, 838 op/s; 45431/271167 objects degraded (16.754%); 6/271167 objects misplaced (0.002%) 2023-06-03T20:18:31.402 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:18:31 smithi089 bash[78452]: cluster 1685823510.8774295 mgr.y (mgr.34104) 1275 : cluster 0 pgmap v1271: 89 pgs: 20 active+clean+remapped, 14 active+undersized+degraded, 11 active+undersized, 44 active+clean; 5.5 GiB data, 17 GiB used, 519 GiB / 536 GiB avail; 52 MiB/s wr, 838 op/s; 45431/271167 objects degraded (16.754%); 6/271167 objects misplaced (0.002%) 2023-06-03T20:18:31.472 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:18:31.517 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:18:31.518 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:18:31.518 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:18:31.584 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:18:31 smithi164 bash[59462]: cluster 1685823510.8774295 mgr.y (mgr.34104) 1275 : cluster 0 pgmap v1271: 89 pgs: 20 active+clean+remapped, 14 active+undersized+degraded, 11 active+undersized, 44 active+clean; 5.5 GiB data, 17 GiB used, 519 GiB / 536 GiB avail; 52 MiB/s wr, 838 op/s; 45431/271167 objects degraded (16.754%); 6/271167 objects misplaced (0.002%) 2023-06-03T20:18:31.620 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:18:31.649 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:18:31.651 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:18:31.652 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:18:31.753 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:18:31.779 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:18:31.781 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:18:31.782 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:18:31.788 INFO:teuthology.orchestra.run.smithi089.stdout:osd.0: {} 2023-06-03T20:18:31.828 INFO:teuthology.orchestra.run.smithi089.stdout:osd.1: {} 2023-06-03T20:18:31.871 INFO:teuthology.orchestra.run.smithi089.stdout:osd.2: {} 2023-06-03T20:18:31.883 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:18:31.911 INFO:teuthology.orchestra.run.smithi089.stdout:osd.3: {} 2023-06-03T20:18:31.914 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:18:31.916 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:18:31.917 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:18:31.944 INFO:teuthology.orchestra.run.smithi089.stdout:osd.5: {} 2023-06-03T20:18:31.981 INFO:teuthology.orchestra.run.smithi089.stdout:osd.6: {} 2023-06-03T20:18:32.017 INFO:teuthology.orchestra.run.smithi089.stdout:osd.7: {} 2023-06-03T20:18:32.017 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:18:32.018 INFO:teuthology.orchestra.run.smithi089.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-06-03T20:18:32.018 INFO:teuthology.orchestra.run.smithi089.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-06-03T20:18:32.019 INFO:teuthology.orchestra.run.smithi089.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-06-03T20:18:32.019 INFO:teuthology.orchestra.run.smithi089.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-06-03T20:18:32.019 INFO:teuthology.orchestra.run.smithi089.stderr:Error ENXIO: problem getting command descriptions from osd.4 2023-06-03T20:18:32.019 INFO:teuthology.orchestra.run.smithi089.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-06-03T20:18:32.019 INFO:teuthology.orchestra.run.smithi089.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-06-03T20:18:32.019 INFO:teuthology.orchestra.run.smithi089.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-06-03T20:18:32.072 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:18:32.076 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:18:32.076 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:18:32.177 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:18:32.202 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:18:32 smithi164 bash[59462]: audit 1685823511.7361186 mon.a (mon.0) 1215 : audit 0 from='client.? 172.21.15.89:0/666786864' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-03T20:18:32.217 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:18:32.222 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:18:32.223 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:18:32.279 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:18:32 smithi089 bash[78452]: audit 1685823511.7361186 mon.a (mon.0) 1215 : audit 0 from='client.? 172.21.15.89:0/666786864' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-03T20:18:32.279 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:18:32 smithi089 bash[75068]: audit 1685823511.7361186 mon.a (mon.0) 1215 : audit 0 from='client.? 172.21.15.89:0/666786864' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-03T20:18:32.324 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:18:32.354 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:18:32.358 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:18:32.359 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:18:32.401 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 75 16 93704 93688 78.0599 52.875 0.0266446 0.0128044 2023-06-03T20:18:32.460 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:18:32.489 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:18:32.492 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:18:32.493 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:18:32.593 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:18:32.623 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:18:32.629 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:18:32.630 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:18:32.731 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:18:32.766 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:18:32.768 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:18:32.769 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:18:32.869 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:18:33.302 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:18:33.304 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:18:33.305 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:18:33.401 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 76 16 94632 94616 77.7959 58 0.00650946 0.012848 2023-06-03T20:18:33.403 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph osd unset noscrub 2023-06-03T20:18:33.406 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:18:33.412 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:18:33 smithi089 bash[75068]: cluster 1685823512.835669 mon.a (mon.0) 1216 : cluster 3 Health check update: Degraded data redundancy: 45431/271167 objects degraded (16.754%), 14 pgs degraded, 25 pgs undersized (PG_DEGRADED) 2023-06-03T20:18:33.412 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:18:33 smithi089 bash[75068]: cluster 1685823512.8782158 mgr.y (mgr.34104) 1276 : cluster 0 pgmap v1272: 89 pgs: 20 active+clean+remapped, 14 active+undersized+degraded, 11 active+undersized, 44 active+clean; 5.6 GiB data, 17 GiB used, 519 GiB / 536 GiB avail; 53 MiB/s wr, 845 op/s; 45723/274452 objects degraded (16.660%); 6/274452 objects misplaced (0.002%) 2023-06-03T20:18:33.412 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:18:33 smithi089 bash[78452]: cluster 1685823512.835669 mon.a (mon.0) 1216 : cluster 3 Health check update: Degraded data redundancy: 45431/271167 objects degraded (16.754%), 14 pgs degraded, 25 pgs undersized (PG_DEGRADED) 2023-06-03T20:18:33.412 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:18:33 smithi089 bash[78452]: cluster 1685823512.8782158 mgr.y (mgr.34104) 1276 : cluster 0 pgmap v1272: 89 pgs: 20 active+clean+remapped, 14 active+undersized+degraded, 11 active+undersized, 44 active+clean; 5.6 GiB data, 17 GiB used, 519 GiB / 536 GiB avail; 53 MiB/s wr, 845 op/s; 45723/274452 objects degraded (16.660%); 6/274452 objects misplaced (0.002%) 2023-06-03T20:18:33.438 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:18:33.441 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:18:33.441 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:18:33.543 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:18:33.554 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:18:33 smithi164 bash[59462]: cluster 1685823512.835669 mon.a (mon.0) 1216 : cluster 3 Health check update: Degraded data redundancy: 45431/271167 objects degraded (16.754%), 14 pgs degraded, 25 pgs undersized (PG_DEGRADED) 2023-06-03T20:18:33.554 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:18:33 smithi164 bash[59462]: cluster 1685823512.8782158 mgr.y (mgr.34104) 1276 : cluster 0 pgmap v1272: 89 pgs: 20 active+clean+remapped, 14 active+undersized+degraded, 11 active+undersized, 44 active+clean; 5.6 GiB data, 17 GiB used, 519 GiB / 536 GiB avail; 53 MiB/s wr, 845 op/s; 45723/274452 objects degraded (16.660%); 6/274452 objects misplaced (0.002%) 2023-06-03T20:18:33.575 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:18:33.578 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:18:33.578 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:18:33.630 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-06-03T20:18:33.631 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-06-03T20:18:33.679 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:18:33.709 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:18:33.715 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:18:33.715 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:18:33.816 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:18:33.848 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:18:33.851 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:18:33.852 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:18:33.953 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:18:33.983 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:18:33.987 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:18:33.987 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:18:34.088 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:18:34.124 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:18:34.129 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:18:34.129 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:18:34.230 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:18:34.279 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:18:34.281 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:18:34.282 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:18:34.383 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:18:34.401 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 77 16 95468 95452 77.464 52.25 0.0229139 0.0129037 2023-06-03T20:18:34.421 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:18:34.424 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:18:34.424 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:18:34.525 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:18:34.557 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:18:34.561 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:18:34.562 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:18:34.663 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:18:34.700 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:18:34.706 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:18:34.706 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:18:34.807 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:18:34.839 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:18:34.843 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:18:34.843 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:18:34.944 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:18:34.980 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:18:34.984 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:18:34.984 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:18:35.029 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:18:34 smithi089 bash[75068]: audit 1685823514.0070155 mon.c (mon.2) 556 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-03T20:18:35.029 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:18:34 smithi089 bash[78452]: audit 1685823514.0070155 mon.c (mon.2) 556 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-03T20:18:35.083 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:18:34 smithi164 bash[59462]: audit 1685823514.0070155 mon.c (mon.2) 556 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-03T20:18:35.085 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:18:35.133 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:18:35.134 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/mon.a/config 2023-06-03T20:18:35.134 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/mon.a/config 2023-06-03T20:18:35.136 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:18:35.136 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:18:35.237 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:18:35.314 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:18:35.320 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:18:35.321 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:18:35.401 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 78 16 96328 96312 77.16 53.75 0.00885606 0.0129545 2023-06-03T20:18:35.422 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:18:35.456 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:18:35.461 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:18:35.462 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:18:35.563 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:18:35.625 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:18:35.632 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:18:35.632 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:18:35.633 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-06-03T20:18:35.734 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:18:35.867 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:18:35 smithi089 bash[75068]: cluster 1685823514.8791406 mgr.y (mgr.34104) 1277 : cluster 0 pgmap v1273: 89 pgs: 20 active+clean+remapped, 14 active+undersized+degraded, 11 active+undersized, 44 active+clean; 5.7 GiB data, 18 GiB used, 519 GiB / 536 GiB avail; 53 MiB/s wr, 853 op/s; 46876/280344 objects degraded (16.721%); 6/280344 objects misplaced (0.002%) 2023-06-03T20:18:36.059 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:18:36.064 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:18:36.065 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:18:36.166 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:18:36.175 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:18:35 smithi164 bash[59462]: cluster 1685823514.8791406 mgr.y (mgr.34104) 1277 : cluster 0 pgmap v1273: 89 pgs: 20 active+clean+remapped, 14 active+undersized+degraded, 11 active+undersized, 44 active+clean; 5.7 GiB data, 18 GiB used, 519 GiB / 536 GiB avail; 53 MiB/s wr, 853 op/s; 46876/280344 objects degraded (16.721%); 6/280344 objects misplaced (0.002%) 2023-06-03T20:18:36.193 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:18:36.197 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:18:36.197 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:18:36.232 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 20:18:36 smithi089 bash[71163]: ::ffff:172.21.15.164 - - [03/Jun/2023:20:18:36] "GET /metrics HTTP/1.1" 200 228936 "" "Prometheus/2.33.4" 2023-06-03T20:18:36.232 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:18:35 smithi089 bash[78452]: cluster 1685823514.8791406 mgr.y (mgr.34104) 1277 : cluster 0 pgmap v1273: 89 pgs: 20 active+clean+remapped, 14 active+undersized+degraded, 11 active+undersized, 44 active+clean; 5.7 GiB data, 18 GiB used, 519 GiB / 536 GiB avail; 53 MiB/s wr, 853 op/s; 46876/280344 objects degraded (16.721%); 6/280344 objects misplaced (0.002%) 2023-06-03T20:18:36.298 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:18:36.331 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:18:36.337 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:18:36.337 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:18:36.401 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 79 16 97153 97137 76.8359 51.5625 0.00804203 0.0130091 2023-06-03T20:18:36.439 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:18:36.474 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:18:36.477 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:18:36.478 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:18:36.578 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:18:36.609 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:18:36.611 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:18:36.612 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:18:36.713 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:18:36.742 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:18:36.745 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:18:36.746 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:18:36.847 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:18:36.876 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:18:36.878 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:18:36.879 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:18:36.979 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:18:37.008 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:18:37.010 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:18:37.011 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:18:37.112 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:18:37.142 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:18:37.144 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:18:37.145 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:18:37.246 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:18:37.277 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:18:37.282 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:18:37.282 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:18:37.383 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:18:37.401 INFO:tasks.radosbench.radosbench.0.smithi089.stdout:2023-06-03T20:18:37.397770+0000 min lat: 0.00129872 max lat: 0.870111 avg lat: 0.0130543 2023-06-03T20:18:37.402 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-06-03T20:18:37.402 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 80 16 98038 98022 76.5668 55.3125 0.00642758 0.0130543 2023-06-03T20:18:37.419 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:18:37.422 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:18:37.423 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:18:37.524 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:18:37.563 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:18:37.566 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:18:37.566 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:18:37.667 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:18:37.798 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:18:37.801 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:18:37.802 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:18:37.903 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:18:37.935 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:18:37.937 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:18:37.938 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:18:38.038 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:18:38.065 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:18:38.069 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:18:38.070 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:18:38.170 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:18:38.201 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:18:38.205 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:18:37 smithi164 bash[59462]: cluster 1685823516.8801038 mgr.y (mgr.34104) 1278 : cluster 0 pgmap v1274: 89 pgs: 20 active+clean+remapped, 14 active+undersized+degraded, 11 active+undersized, 44 active+clean; 5.9 GiB data, 18 GiB used, 518 GiB / 536 GiB avail; 54 MiB/s wr, 863 op/s; 48024/287922 objects degraded (16.680%); 6/287922 objects misplaced (0.002%) 2023-06-03T20:18:38.205 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:18:37 smithi164 bash[59462]: audit 1685823517.1298456 mon.c (mon.2) 557 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T20:18:38.205 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:18:37 smithi164 bash[59462]: cluster 1685823517.8370414 mon.a (mon.0) 1217 : cluster 3 Health check update: Degraded data redundancy: 48024/287922 objects degraded (16.680%), 14 pgs degraded, 25 pgs undersized (PG_DEGRADED) 2023-06-03T20:18:38.206 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:18:38.206 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:18:38.237 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/mon.a/config 2023-06-03T20:18:38.276 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:18:37 smithi089 bash[78452]: cluster 1685823516.8801038 mgr.y (mgr.34104) 1278 : cluster 0 pgmap v1274: 89 pgs: 20 active+clean+remapped, 14 active+undersized+degraded, 11 active+undersized, 44 active+clean; 5.9 GiB data, 18 GiB used, 518 GiB / 536 GiB avail; 54 MiB/s wr, 863 op/s; 48024/287922 objects degraded (16.680%); 6/287922 objects misplaced (0.002%) 2023-06-03T20:18:38.277 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:18:37 smithi089 bash[78452]: audit 1685823517.1298456 mon.c (mon.2) 557 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T20:18:38.277 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:18:37 smithi089 bash[78452]: cluster 1685823517.8370414 mon.a (mon.0) 1217 : cluster 3 Health check update: Degraded data redundancy: 48024/287922 objects degraded (16.680%), 14 pgs degraded, 25 pgs undersized (PG_DEGRADED) 2023-06-03T20:18:38.277 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:18:37 smithi089 bash[75068]: cluster 1685823516.8801038 mgr.y (mgr.34104) 1278 : cluster 0 pgmap v1274: 89 pgs: 20 active+clean+remapped, 14 active+undersized+degraded, 11 active+undersized, 44 active+clean; 5.9 GiB data, 18 GiB used, 518 GiB / 536 GiB avail; 54 MiB/s wr, 863 op/s; 48024/287922 objects degraded (16.680%); 6/287922 objects misplaced (0.002%) 2023-06-03T20:18:38.277 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:18:37 smithi089 bash[75068]: audit 1685823517.1298456 mon.c (mon.2) 557 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T20:18:38.278 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:18:37 smithi089 bash[75068]: cluster 1685823517.8370414 mon.a (mon.0) 1217 : cluster 3 Health check update: Degraded data redundancy: 48024/287922 objects degraded (16.680%), 14 pgs degraded, 25 pgs undersized (PG_DEGRADED) 2023-06-03T20:18:38.307 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:18:38.357 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:18:38.365 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:18:38.366 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:18:38.402 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 81 16 98865 98849 76.2589 51.6875 0.00880605 0.0131076 2023-06-03T20:18:38.468 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:18:38.496 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:18:38.498 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:18:38.499 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:18:38.600 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:18:38.631 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:18:38.634 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:18:38.635 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:18:38.736 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:18:38.772 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:18:38.775 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:18:38.776 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:18:38.877 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:18:38.910 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:18:38.912 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:18:38.913 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:18:39.014 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:18:39.049 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:18:39.057 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:18:39.057 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:18:39.158 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:18:39.180 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:18:39.182 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:18:39.182 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:18:39.283 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:18:39.371 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:18:39.377 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:18:39.378 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:18:39.402 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 82 16 99557 99541 75.8564 43.25 0.0165635 0.013177 2023-06-03T20:18:39.479 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:18:39.536 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:18:39.538 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:18:39.539 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:18:39.640 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:18:39.701 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:18:39.703 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:18:39.704 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:18:39.805 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:18:39.835 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:18:39.864 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:18:39.865 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:18:39.966 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:18:40.037 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:18:40.040 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:18:40.041 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:18:40.084 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:18:39 smithi164 bash[59462]: cluster 1685823518.8813727 mgr.y (mgr.34104) 1279 : cluster 0 pgmap v1275: 89 pgs: 20 active+clean+remapped, 14 active+undersized+degraded, 11 active+undersized, 44 active+clean; 6.0 GiB data, 18 GiB used, 518 GiB / 536 GiB avail; 54 MiB/s wr, 864 op/s; 49051/293367 objects degraded (16.720%); 6/293367 objects misplaced (0.002%) 2023-06-03T20:18:40.141 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:18:40.204 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:18:40.209 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:18:40.210 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:18:39 smithi089 bash[75068]: cluster 1685823518.8813727 mgr.y (mgr.34104) 1279 : cluster 0 pgmap v1275: 89 pgs: 20 active+clean+remapped, 14 active+undersized+degraded, 11 active+undersized, 44 active+clean; 6.0 GiB data, 18 GiB used, 518 GiB / 536 GiB avail; 54 MiB/s wr, 864 op/s; 49051/293367 objects degraded (16.720%); 6/293367 objects misplaced (0.002%) 2023-06-03T20:18:40.210 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:18:39 smithi089 bash[78452]: cluster 1685823518.8813727 mgr.y (mgr.34104) 1279 : cluster 0 pgmap v1275: 89 pgs: 20 active+clean+remapped, 14 active+undersized+degraded, 11 active+undersized, 44 active+clean; 6.0 GiB data, 18 GiB used, 518 GiB / 536 GiB avail; 54 MiB/s wr, 864 op/s; 49051/293367 objects degraded (16.720%); 6/293367 objects misplaced (0.002%) 2023-06-03T20:18:40.211 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:18:40.306 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/mon.a/config 2023-06-03T20:18:40.312 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:18:40.352 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:18:40.352 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:18:40.353 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:18:40.404 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 83 15 100256 100241 75.4684 43.75 0.0188963 0.013245 2023-06-03T20:18:40.454 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:18:40.527 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:18:40.531 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:18:40.532 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:18:40.632 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:18:40.663 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:18:40.668 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:18:40.669 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:18:40.769 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:18:40.899 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:18:40.905 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:18:40.905 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:18:41.006 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:18:41.038 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:18:41.040 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:18:41.041 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:18:41.142 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:18:41.176 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:18:41.178 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:18:41.180 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:18:41.281 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:18:41.321 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:18:41.330 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:18:41.331 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:18:41.404 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 84 16 100945 100929 75.0819 43 0.00692389 0.0133135 2023-06-03T20:18:41.432 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:18:41.463 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:18:41.464 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:18:41.465 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:18:41.500 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:18:41 smithi089 bash[78452]: audit 1685823520.2799435 mon.a (mon.0) 1218 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:18:41.501 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:18:41 smithi089 bash[78452]: audit 1685823520.294288 mon.a (mon.0) 1219 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:18:41.501 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:18:41 smithi089 bash[78452]: audit 1685823520.4322462 mon.a (mon.0) 1220 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:18:41.501 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:18:41 smithi089 bash[78452]: audit 1685823520.4455547 mon.a (mon.0) 1221 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:18:41.501 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:18:41 smithi089 bash[78452]: cluster 1685823520.882379 mgr.y (mgr.34104) 1280 : cluster 0 pgmap v1276: 89 pgs: 20 active+clean+remapped, 14 active+undersized+degraded, 11 active+undersized, 44 active+clean; 6.0 GiB data, 19 GiB used, 518 GiB / 536 GiB avail; 52 MiB/s wr, 824 op/s; 49785/297027 objects degraded (16.761%); 6/297027 objects misplaced (0.002%) 2023-06-03T20:18:41.502 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:18:41 smithi089 bash[78452]: audit 1685823521.028076 mon.c (mon.2) 558 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-03T20:18:41.502 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:18:41 smithi089 bash[78452]: audit 1685823521.0293293 mon.c (mon.2) 559 : audit 1 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-03T20:18:41.502 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:18:41 smithi089 bash[78452]: audit 1685823521.0373466 mon.a (mon.0) 1222 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:18:41.502 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:18:41 smithi089 bash[75068]: audit 1685823520.2799435 mon.a (mon.0) 1218 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:18:41.503 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:18:41 smithi089 bash[75068]: audit 1685823520.294288 mon.a (mon.0) 1219 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:18:41.503 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:18:41 smithi089 bash[75068]: audit 1685823520.4322462 mon.a (mon.0) 1220 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:18:41.503 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:18:41 smithi089 bash[75068]: audit 1685823520.4455547 mon.a (mon.0) 1221 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:18:41.503 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:18:41 smithi089 bash[75068]: cluster 1685823520.882379 mgr.y (mgr.34104) 1280 : cluster 0 pgmap v1276: 89 pgs: 20 active+clean+remapped, 14 active+undersized+degraded, 11 active+undersized, 44 active+clean; 6.0 GiB data, 19 GiB used, 518 GiB / 536 GiB avail; 52 MiB/s wr, 824 op/s; 49785/297027 objects degraded (16.761%); 6/297027 objects misplaced (0.002%) 2023-06-03T20:18:41.504 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:18:41 smithi089 bash[75068]: audit 1685823521.028076 mon.c (mon.2) 558 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-03T20:18:41.504 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:18:41 smithi089 bash[75068]: audit 1685823521.0293293 mon.c (mon.2) 559 : audit 1 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-03T20:18:41.504 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:18:41 smithi089 bash[75068]: audit 1685823521.0373466 mon.a (mon.0) 1222 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:18:41.566 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:18:41.584 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:18:41 smithi164 bash[59462]: audit 1685823520.2799435 mon.a (mon.0) 1218 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:18:41.584 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:18:41 smithi164 bash[59462]: audit 1685823520.294288 mon.a (mon.0) 1219 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:18:41.584 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:18:41 smithi164 bash[59462]: audit 1685823520.4322462 mon.a (mon.0) 1220 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:18:41.584 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:18:41 smithi164 bash[59462]: audit 1685823520.4455547 mon.a (mon.0) 1221 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:18:41.585 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:18:41 smithi164 bash[59462]: cluster 1685823520.882379 mgr.y (mgr.34104) 1280 : cluster 0 pgmap v1276: 89 pgs: 20 active+clean+remapped, 14 active+undersized+degraded, 11 active+undersized, 44 active+clean; 6.0 GiB data, 19 GiB used, 518 GiB / 536 GiB avail; 52 MiB/s wr, 824 op/s; 49785/297027 objects degraded (16.761%); 6/297027 objects misplaced (0.002%) 2023-06-03T20:18:41.585 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:18:41 smithi164 bash[59462]: audit 1685823521.028076 mon.c (mon.2) 558 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-03T20:18:41.585 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:18:41 smithi164 bash[59462]: audit 1685823521.0293293 mon.c (mon.2) 559 : audit 1 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-03T20:18:41.585 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:18:41 smithi164 bash[59462]: audit 1685823521.0373466 mon.a (mon.0) 1222 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:18:41.595 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:18:41.597 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:18:41.598 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:18:41.699 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:18:41.746 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:18:41.749 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:18:41.750 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:18:41.851 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:18:41.883 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:18:41.886 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:18:41.886 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:18:41.987 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:18:42.018 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:18:42.024 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:18:42.024 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:18:42.125 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:18:42.153 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:18:42.161 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:18:42.162 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:18:42.263 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:18:42.291 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:18:42.294 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:18:42.294 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:18:42.395 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:18:42.404 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 85 16 101805 101789 74.8309 53.75 0.0157793 0.0133578 2023-06-03T20:18:42.426 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:18:42.429 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:18:42.429 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:18:42.531 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:18:42.565 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:18:42.569 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:18:42.569 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:18:42.670 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:18:42.707 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:18:42.709 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:18:42.710 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:18:42.810 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:18:42.848 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:18:42.854 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:18:42.855 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:18:42.956 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:18:42.986 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:18:42.988 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:18:42.988 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:18:43.075 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:18:42 smithi089 bash[75068]: cluster 1685823522.8381455 mon.a (mon.0) 1223 : cluster 3 Health check update: Degraded data redundancy: 49785/297027 objects degraded (16.761%), 14 pgs degraded, 25 pgs undersized (PG_DEGRADED) 2023-06-03T20:18:43.075 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:18:42 smithi089 bash[78452]: cluster 1685823522.8381455 mon.a (mon.0) 1223 : cluster 3 Health check update: Degraded data redundancy: 49785/297027 objects degraded (16.761%), 14 pgs degraded, 25 pgs undersized (PG_DEGRADED) 2023-06-03T20:18:43.089 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:18:43.117 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:18:43.124 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:18:43.125 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:18:43.226 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:18:43.260 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:18:43.263 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:18:43.263 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:18:43.333 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:18:42 smithi164 bash[59462]: cluster 1685823522.8381455 mon.a (mon.0) 1223 : cluster 3 Health check update: Degraded data redundancy: 49785/297027 objects degraded (16.761%), 14 pgs degraded, 25 pgs undersized (PG_DEGRADED) 2023-06-03T20:18:43.364 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:18:43.395 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:18:43.397 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:18:43.398 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:18:43.404 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 86 16 102649 102633 74.5741 52.75 0.0187157 0.0134036 2023-06-03T20:18:43.498 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:18:43.527 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:18:43.530 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:18:43.531 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:18:43.632 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:18:43.669 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:18:43.671 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:18:43.672 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:18:43.772 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:18:43.809 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:18:43.816 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:18:43.817 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:18:43.917 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:18:43.953 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:18:43.957 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:18:43.958 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:18:44.059 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:18:44.068 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:18:43 smithi089 bash[75068]: cluster 1685823522.883145 mgr.y (mgr.34104) 1281 : cluster 0 pgmap v1277: 89 pgs: 20 active+clean+remapped, 14 active+undersized+degraded, 11 active+undersized, 44 active+clean; 6.1 GiB data, 19 GiB used, 518 GiB / 536 GiB avail; 50 MiB/s wr, 797 op/s; 50034/299901 objects degraded (16.684%); 6/299901 objects misplaced (0.002%) 2023-06-03T20:18:44.068 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:18:43 smithi089 bash[78452]: cluster 1685823522.883145 mgr.y (mgr.34104) 1281 : cluster 0 pgmap v1277: 89 pgs: 20 active+clean+remapped, 14 active+undersized+degraded, 11 active+undersized, 44 active+clean; 6.1 GiB data, 19 GiB used, 518 GiB / 536 GiB avail; 50 MiB/s wr, 797 op/s; 50034/299901 objects degraded (16.684%); 6/299901 objects misplaced (0.002%) 2023-06-03T20:18:44.096 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:18:44.100 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:18:44.101 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:18:44.202 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:18:44.228 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:18:44.230 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:18:44.230 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:18:44.331 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:18:44.334 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:18:43 smithi164 bash[59462]: cluster 1685823522.883145 mgr.y (mgr.34104) 1281 : cluster 0 pgmap v1277: 89 pgs: 20 active+clean+remapped, 14 active+undersized+degraded, 11 active+undersized, 44 active+clean; 6.1 GiB data, 19 GiB used, 518 GiB / 536 GiB avail; 50 MiB/s wr, 797 op/s; 50034/299901 objects degraded (16.684%); 6/299901 objects misplaced (0.002%) 2023-06-03T20:18:44.368 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:18:44.370 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:18:44.371 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:18:44.404 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 87 16 103288 103272 74.176 39.9375 0.00819425 0.0134627 2023-06-03T20:18:44.472 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:18:44.606 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:18:44.610 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:18:44.611 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:18:44.712 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:18:44.746 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:18:44.749 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:18:44.750 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:18:44.850 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:18:44.878 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:18:44.882 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:18:44.883 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:18:44.985 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:18:45.018 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:18:45.022 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:18:45.022 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:18:45.123 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:18:45.145 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:18:45.146 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:18:45.147 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:18:45.248 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:18:45.284 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:18:45.290 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:18:45.291 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:18:45.392 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:18:45.404 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 88 16 103988 103972 73.8302 43.75 0.00813255 0.0135387 2023-06-03T20:18:45.429 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:18:45.434 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:18:45.435 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:18:45.536 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:18:45.575 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:18:45.583 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:18:45.583 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:18:45.684 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:18:45.715 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:18:45.718 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:18:45.718 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:18:45.819 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:18:45.847 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:18:45.849 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:18:45.850 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:18:45.902 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:18:45 smithi089 bash[75068]: cluster 1685823524.8839955 mgr.y (mgr.34104) 1282 : cluster 0 pgmap v1278: 89 pgs: 20 active+clean+remapped, 14 active+undersized+degraded, 11 active+undersized, 44 active+clean; 6.2 GiB data, 19 GiB used, 517 GiB / 536 GiB avail; 53 MiB/s wr, 843 op/s; 51004/304836 objects degraded (16.732%); 6/304836 objects misplaced (0.002%) 2023-06-03T20:18:45.951 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:18:45.960 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:18:45 smithi164 bash[59462]: cluster 1685823524.8839955 mgr.y (mgr.34104) 1282 : cluster 0 pgmap v1278: 89 pgs: 20 active+clean+remapped, 14 active+undersized+degraded, 11 active+undersized, 44 active+clean; 6.2 GiB data, 19 GiB used, 517 GiB / 536 GiB avail; 53 MiB/s wr, 843 op/s; 51004/304836 objects degraded (16.732%); 6/304836 objects misplaced (0.002%) 2023-06-03T20:18:45.982 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:18:45.985 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:18:45.985 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:18:46.086 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:18:46.119 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:18:46.121 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:18:46.122 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:18:46.223 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:18:46.231 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 20:18:46 smithi089 bash[71163]: ::ffff:172.21.15.164 - - [03/Jun/2023:20:18:46] "GET /metrics HTTP/1.1" 200 228914 "" "Prometheus/2.33.4" 2023-06-03T20:18:46.231 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:18:45 smithi089 bash[78452]: cluster 1685823524.8839955 mgr.y (mgr.34104) 1282 : cluster 0 pgmap v1278: 89 pgs: 20 active+clean+remapped, 14 active+undersized+degraded, 11 active+undersized, 44 active+clean; 6.2 GiB data, 19 GiB used, 517 GiB / 536 GiB avail; 53 MiB/s wr, 843 op/s; 51004/304836 objects degraded (16.732%); 6/304836 objects misplaced (0.002%) 2023-06-03T20:18:46.259 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:18:46.262 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:18:46.263 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:18:46.364 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:18:46.396 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:18:46.398 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:18:46.399 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:18:46.404 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 89 16 104907 104891 73.646 57.4375 0.0206433 0.0135729 2023-06-03T20:18:46.500 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:18:46.529 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:18:46.533 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:18:46.534 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:18:46.635 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:18:46.672 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:18:46.675 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:18:46.675 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:18:46.776 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:18:46.812 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:18:46.814 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:18:46.815 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:18:46.915 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:18:46.937 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:18:46.939 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:18:46.940 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:18:47.041 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:18:47.072 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:18:47.075 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:18:47.076 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:18:47.177 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:18:47.210 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:18:47.215 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:18:47.216 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:18:47.317 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:18:47.351 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:18:47.353 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:18:47.354 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:18:47.405 INFO:tasks.radosbench.radosbench.0.smithi089.stdout:Total time run: 90.0068 2023-06-03T20:18:47.405 INFO:tasks.radosbench.radosbench.0.smithi089.stdout:Total writes made: 105606 2023-06-03T20:18:47.405 INFO:tasks.radosbench.radosbench.0.smithi089.stdout:Write size: 65536 2023-06-03T20:18:47.405 INFO:tasks.radosbench.radosbench.0.smithi089.stdout:Object size: 65536 2023-06-03T20:18:47.405 INFO:tasks.radosbench.radosbench.0.smithi089.stdout:Bandwidth (MB/sec): 73.332 2023-06-03T20:18:47.406 INFO:tasks.radosbench.radosbench.0.smithi089.stdout:Stddev Bandwidth: 29.017 2023-06-03T20:18:47.406 INFO:tasks.radosbench.radosbench.0.smithi089.stdout:Max bandwidth (MB/sec): 176.812 2023-06-03T20:18:47.406 INFO:tasks.radosbench.radosbench.0.smithi089.stdout:Min bandwidth (MB/sec): 39.9375 2023-06-03T20:18:47.406 INFO:tasks.radosbench.radosbench.0.smithi089.stdout:Average IOPS: 1173 2023-06-03T20:18:47.406 INFO:tasks.radosbench.radosbench.0.smithi089.stdout:Stddev IOPS: 464.271 2023-06-03T20:18:47.406 INFO:tasks.radosbench.radosbench.0.smithi089.stdout:Max IOPS: 2829 2023-06-03T20:18:47.406 INFO:tasks.radosbench.radosbench.0.smithi089.stdout:Min IOPS: 639 2023-06-03T20:18:47.407 INFO:tasks.radosbench.radosbench.0.smithi089.stdout:Average Latency(s): 0.0136322 2023-06-03T20:18:47.407 INFO:tasks.radosbench.radosbench.0.smithi089.stdout:Stddev Latency(s): 0.0169975 2023-06-03T20:18:47.407 INFO:tasks.radosbench.radosbench.0.smithi089.stdout:Max latency(s): 0.870111 2023-06-03T20:18:47.407 INFO:tasks.radosbench.radosbench.0.smithi089.stdout:Min latency(s): 0.00129872 2023-06-03T20:18:47.413 INFO:tasks.radosbench.radosbench.0.smithi089.stdout:Cleaning up (deleting benchmark objects) 2023-06-03T20:18:47.455 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:18:47.484 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:18:47.488 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:18:47.488 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:18:47.589 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:18:47.615 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:18:47.618 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:18:47.619 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:18:47.720 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:18:47.760 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:18:47.762 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:18:47.763 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:18:47.864 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:18:47.899 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:18:47.901 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:18:47.902 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:18:47.905 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:18:47 smithi089 bash[75068]: cluster 1685823526.8846934 mgr.y (mgr.34104) 1283 : cluster 0 pgmap v1279: 89 pgs: 20 active+clean+remapped, 14 active+undersized+degraded, 11 active+undersized, 44 active+clean; 6.3 GiB data, 19 GiB used, 517 GiB / 536 GiB avail; 54 MiB/s wr, 859 op/s; 51968/311316 objects degraded (16.693%); 6/311316 objects misplaced (0.002%) 2023-06-03T20:18:47.905 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:18:47 smithi089 bash[75068]: cluster 1685823527.8396494 mon.a (mon.0) 1224 : cluster 3 Health check update: Degraded data redundancy: 51968/311316 objects degraded (16.693%), 14 pgs degraded, 25 pgs undersized (PG_DEGRADED) 2023-06-03T20:18:48.003 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:18:48.037 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:18:48.043 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:18:48.043 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:18:48.083 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:18:47 smithi164 bash[59462]: cluster 1685823526.8846934 mgr.y (mgr.34104) 1283 : cluster 0 pgmap v1279: 89 pgs: 20 active+clean+remapped, 14 active+undersized+degraded, 11 active+undersized, 44 active+clean; 6.3 GiB data, 19 GiB used, 517 GiB / 536 GiB avail; 54 MiB/s wr, 859 op/s; 51968/311316 objects degraded (16.693%); 6/311316 objects misplaced (0.002%) 2023-06-03T20:18:48.084 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:18:47 smithi164 bash[59462]: cluster 1685823527.8396494 mon.a (mon.0) 1224 : cluster 3 Health check update: Degraded data redundancy: 51968/311316 objects degraded (16.693%), 14 pgs degraded, 25 pgs undersized (PG_DEGRADED) 2023-06-03T20:18:48.144 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:18:48.172 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:18:48.178 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:18:47 smithi089 bash[78452]: cluster 1685823526.8846934 mgr.y (mgr.34104) 1283 : cluster 0 pgmap v1279: 89 pgs: 20 active+clean+remapped, 14 active+undersized+degraded, 11 active+undersized, 44 active+clean; 6.3 GiB data, 19 GiB used, 517 GiB / 536 GiB avail; 54 MiB/s wr, 859 op/s; 51968/311316 objects degraded (16.693%); 6/311316 objects misplaced (0.002%) 2023-06-03T20:18:48.178 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:18:47 smithi089 bash[78452]: cluster 1685823527.8396494 mon.a (mon.0) 1224 : cluster 3 Health check update: Degraded data redundancy: 51968/311316 objects degraded (16.693%), 14 pgs degraded, 25 pgs undersized (PG_DEGRADED) 2023-06-03T20:18:48.179 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:18:48.179 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:18:48.280 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:18:48.313 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:18:48.316 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:18:48.317 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:18:48.418 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:18:48.451 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:18:48.454 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:18:48.454 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:18:48.555 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:18:48.589 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:18:48.592 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:18:48.592 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:18:48.694 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:18:48.723 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:18:48.725 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:18:48.726 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:18:48.826 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:18:48.865 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:18:48.871 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:18:48.872 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:18:48.974 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:18:49.016 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:18:49.023 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:18:49.024 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:18:49.125 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:18:49.156 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:18:49.159 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:18:49.159 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:18:49.260 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:18:49.291 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:18:49.293 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:18:49.294 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:18:49.395 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:18:49.422 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:18:49.425 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:18:49.426 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:18:49.527 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:18:49.559 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:18:49.564 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:18:49.564 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:18:49.665 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:18:49.713 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:18:49.717 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:18:49.718 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:18:49.819 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:18:49.857 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:18:49.862 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:18:49.863 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:18:49.964 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:18:50.003 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:18:50.007 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:18:50.008 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:18:50.084 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:18:49 smithi164 bash[59462]: cluster 1685823528.885554 mgr.y (mgr.34104) 1284 : cluster 0 pgmap v1280: 89 pgs: 20 active+clean+remapped, 14 active+undersized+degraded, 11 active+undersized, 44 active+clean; 6.4 GiB data, 19 GiB used, 517 GiB / 536 GiB avail; 85 B/s rd, 47 MiB/s wr, 786 op/s; 52426/313800 objects degraded (16.707%); 6/313800 objects misplaced (0.002%) 2023-06-03T20:18:50.109 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:18:50.141 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:18:50.143 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:18:50.144 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:18:50.245 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:18:50.268 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:18:50.270 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:18:50.270 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:18:50.307 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:18:49 smithi089 bash[78452]: cluster 1685823528.885554 mgr.y (mgr.34104) 1284 : cluster 0 pgmap v1280: 89 pgs: 20 active+clean+remapped, 14 active+undersized+degraded, 11 active+undersized, 44 active+clean; 6.4 GiB data, 19 GiB used, 517 GiB / 536 GiB avail; 85 B/s rd, 47 MiB/s wr, 786 op/s; 52426/313800 objects degraded (16.707%); 6/313800 objects misplaced (0.002%) 2023-06-03T20:18:50.307 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:18:49 smithi089 bash[75068]: cluster 1685823528.885554 mgr.y (mgr.34104) 1284 : cluster 0 pgmap v1280: 89 pgs: 20 active+clean+remapped, 14 active+undersized+degraded, 11 active+undersized, 44 active+clean; 6.4 GiB data, 19 GiB used, 517 GiB / 536 GiB avail; 85 B/s rd, 47 MiB/s wr, 786 op/s; 52426/313800 objects degraded (16.707%); 6/313800 objects misplaced (0.002%) 2023-06-03T20:18:50.372 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:18:50.504 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:18:50.505 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:18:50.506 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:18:50.607 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:18:50.829 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:18:50.831 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:18:50.832 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:18:50.933 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:18:50.958 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:18:50.960 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:18:50.961 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:18:51.062 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:18:51.099 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:18:51.102 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:18:51.103 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:18:51.204 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:18:51.261 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:18:51.267 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:18:51.268 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:18:51.369 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:18:51.403 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:18:51.408 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:18:51.408 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:18:51.509 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:18:51.541 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:18:51.545 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:18:51.545 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:18:51.646 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:18:51.679 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:18:51.684 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:18:51.685 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:18:51.786 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:18:51.814 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:18:51.817 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:18:51.818 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:18:51.912 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:18:51 smithi164 bash[59462]: cluster 1685823530.8861604 mgr.y (mgr.34104) 1285 : cluster 0 pgmap v1281: 89 pgs: 20 active+clean+remapped, 14 active+undersized+degraded, 11 active+undersized, 44 active+clean; 6.4 GiB data, 19 GiB used, 517 GiB / 536 GiB avail; 85 B/s rd, 40 MiB/s wr, 737 op/s; 52258/312981 objects degraded (16.697%); 6/312981 objects misplaced (0.002%) 2023-06-03T20:18:51.919 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:18:51.983 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:18:51.986 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:18:51.987 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:18:52.088 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:18:52.119 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:18:52.122 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:18:52.123 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:18:52.224 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:18:52.257 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:18:52.259 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:18:52.260 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:18:52.262 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:18:51 smithi089 bash[75068]: cluster 1685823530.8861604 mgr.y (mgr.34104) 1285 : cluster 0 pgmap v1281: 89 pgs: 20 active+clean+remapped, 14 active+undersized+degraded, 11 active+undersized, 44 active+clean; 6.4 GiB data, 19 GiB used, 517 GiB / 536 GiB avail; 85 B/s rd, 40 MiB/s wr, 737 op/s; 52258/312981 objects degraded (16.697%); 6/312981 objects misplaced (0.002%) 2023-06-03T20:18:52.263 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:18:51 smithi089 bash[78452]: cluster 1685823530.8861604 mgr.y (mgr.34104) 1285 : cluster 0 pgmap v1281: 89 pgs: 20 active+clean+remapped, 14 active+undersized+degraded, 11 active+undersized, 44 active+clean; 6.4 GiB data, 19 GiB used, 517 GiB / 536 GiB avail; 85 B/s rd, 40 MiB/s wr, 737 op/s; 52258/312981 objects degraded (16.697%); 6/312981 objects misplaced (0.002%) 2023-06-03T20:18:52.361 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:18:52.390 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:18:52.394 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:18:52.394 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:18:52.495 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:18:52.528 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:18:52.529 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:18:52.530 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:18:52.631 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:18:52.670 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:18:52.673 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:18:52.673 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:18:52.774 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:18:52.808 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:18:52.814 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:18:52.815 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:18:52.914 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:18:52 smithi164 bash[59462]: audit 1685823532.129972 mon.c (mon.2) 560 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T20:18:52.914 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:18:52 smithi164 bash[59462]: cluster 1685823532.8411205 mon.a (mon.0) 1225 : cluster 3 Health check update: Degraded data redundancy: 52258/312981 objects degraded (16.697%), 14 pgs degraded, 25 pgs undersized (PG_DEGRADED) 2023-06-03T20:18:52.916 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:18:52.971 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:18:52.972 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:18:52.973 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:18:53.073 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:18:53.110 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:18:53.115 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:18:53.116 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:18:53.217 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:18:53.251 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:18:53.254 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:18:53.254 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:18:53.278 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:18:52 smithi089 bash[78452]: audit 1685823532.129972 mon.c (mon.2) 560 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T20:18:53.279 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:18:52 smithi089 bash[78452]: cluster 1685823532.8411205 mon.a (mon.0) 1225 : cluster 3 Health check update: Degraded data redundancy: 52258/312981 objects degraded (16.697%), 14 pgs degraded, 25 pgs undersized (PG_DEGRADED) 2023-06-03T20:18:53.279 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:18:52 smithi089 bash[75068]: audit 1685823532.129972 mon.c (mon.2) 560 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T20:18:53.279 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:18:52 smithi089 bash[75068]: cluster 1685823532.8411205 mon.a (mon.0) 1225 : cluster 3 Health check update: Degraded data redundancy: 52258/312981 objects degraded (16.697%), 14 pgs degraded, 25 pgs undersized (PG_DEGRADED) 2023-06-03T20:18:53.355 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:18:53.387 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:18:53.390 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:18:53.390 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:18:53.491 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:18:53.554 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:18:53.557 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:18:53.557 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:18:53.658 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:18:53.734 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:18:53.737 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:18:53.737 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:18:53.838 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:18:53.876 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:18:53.879 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:18:53.880 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:18:53.981 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:18:54.016 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:18:54.020 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:18:54.020 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:18:54.121 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:18:54.130 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:18:53 smithi089 bash[75068]: cluster 1685823532.886757 mgr.y (mgr.34104) 1286 : cluster 0 pgmap v1282: 89 pgs: 20 active+clean+remapped, 14 active+undersized+degraded, 11 active+undersized, 44 active+clean; 6.3 GiB data, 19 GiB used, 517 GiB / 536 GiB avail; 85 B/s rd, 34 MiB/s wr, 715 op/s; 52097/310977 objects degraded (16.753%); 6/310977 objects misplaced (0.002%) 2023-06-03T20:18:54.130 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:18:53 smithi089 bash[78452]: cluster 1685823532.886757 mgr.y (mgr.34104) 1286 : cluster 0 pgmap v1282: 89 pgs: 20 active+clean+remapped, 14 active+undersized+degraded, 11 active+undersized, 44 active+clean; 6.3 GiB data, 19 GiB used, 517 GiB / 536 GiB avail; 85 B/s rd, 34 MiB/s wr, 715 op/s; 52097/310977 objects degraded (16.753%); 6/310977 objects misplaced (0.002%) 2023-06-03T20:18:54.244 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:18:54.246 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:18:54.246 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:18:54.333 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:18:53 smithi164 bash[59462]: cluster 1685823532.886757 mgr.y (mgr.34104) 1286 : cluster 0 pgmap v1282: 89 pgs: 20 active+clean+remapped, 14 active+undersized+degraded, 11 active+undersized, 44 active+clean; 6.3 GiB data, 19 GiB used, 517 GiB / 536 GiB avail; 85 B/s rd, 34 MiB/s wr, 715 op/s; 52097/310977 objects degraded (16.753%); 6/310977 objects misplaced (0.002%) 2023-06-03T20:18:54.347 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:18:54.368 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:18:54.369 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:18:54.370 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:18:54.471 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:18:54.588 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:18:54.590 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:18:54.591 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:18:54.692 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:18:54.775 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:18:54.776 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:18:54.777 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:18:54.878 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:18:54.967 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:18:54.968 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:18:54.969 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:18:55.070 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:18:55.101 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:18:55.106 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:18:55.106 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:18:55.207 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:18:55.300 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:18:55.304 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:18:55.305 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:18:55.406 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:18:55.480 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:18:55.482 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:18:55.483 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:18:55.584 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:18:55.617 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:18:55.619 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:18:55.620 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:18:55.721 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:18:55.749 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:18:55.759 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:18:55.760 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:18:55.860 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:18:56.004 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:18:56.006 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:18:56.007 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:18:56.091 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:18:55 smithi089 bash[75068]: cluster 1685823534.8894565 mgr.y (mgr.34104) 1287 : cluster 0 pgmap v1283: 89 pgs: 20 active+clean+remapped, 14 active+undersized+degraded, 11 active+undersized, 44 active+clean; 6.2 GiB data, 19 GiB used, 517 GiB / 536 GiB avail; 85 B/s rd, 29 MiB/s wr, 791 op/s; 51055/305364 objects degraded (16.719%); 6/305364 objects misplaced (0.002%) 2023-06-03T20:18:56.092 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:18:55 smithi089 bash[78452]: cluster 1685823534.8894565 mgr.y (mgr.34104) 1287 : cluster 0 pgmap v1283: 89 pgs: 20 active+clean+remapped, 14 active+undersized+degraded, 11 active+undersized, 44 active+clean; 6.2 GiB data, 19 GiB used, 517 GiB / 536 GiB avail; 85 B/s rd, 29 MiB/s wr, 791 op/s; 51055/305364 objects degraded (16.719%); 6/305364 objects misplaced (0.002%) 2023-06-03T20:18:56.108 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:18:56.144 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:18:56.148 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:18:56.149 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:18:56.249 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:18:56.260 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:18:55 smithi164 bash[59462]: cluster 1685823534.8894565 mgr.y (mgr.34104) 1287 : cluster 0 pgmap v1283: 89 pgs: 20 active+clean+remapped, 14 active+undersized+degraded, 11 active+undersized, 44 active+clean; 6.2 GiB data, 19 GiB used, 517 GiB / 536 GiB avail; 85 B/s rd, 29 MiB/s wr, 791 op/s; 51055/305364 objects degraded (16.719%); 6/305364 objects misplaced (0.002%) 2023-06-03T20:18:56.289 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:18:56.291 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:18:56.291 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:18:56.358 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 20:18:56 smithi089 bash[71163]: ::ffff:172.21.15.164 - - [03/Jun/2023:20:18:56] "GET /metrics HTTP/1.1" 200 228931 "" "Prometheus/2.33.4" 2023-06-03T20:18:56.392 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:18:56.431 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:18:56.433 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:18:56.434 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:18:56.535 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:18:56.563 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:18:56.567 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:18:56.568 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:18:56.595 INFO:teuthology.orchestra.run.smithi089.stdout:{ 2023-06-03T20:18:56.595 INFO:teuthology.orchestra.run.smithi089.stdout: "size": 20, 2023-06-03T20:18:56.595 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 600, 2023-06-03T20:18:56.595 INFO:teuthology.orchestra.run.smithi089.stdout: "ops": [ 2023-06-03T20:18:56.596 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:18:56.596 INFO:teuthology.orchestra.run.smithi089.stdout: "description": "osd_op(client.46187.0:164203 11.9 11:9230de01:::benchmark_data_smithi089_165412_object58675:head [delete] snapc 0=[] ondisk+write+known_if_redirected e203)", 2023-06-03T20:18:56.596 INFO:teuthology.orchestra.run.smithi089.stdout: "initiated_at": "2023-06-03T20:14:25.503815+0000", 2023-06-03T20:18:56.596 INFO:teuthology.orchestra.run.smithi089.stdout: "age": 271.08280009200001, 2023-06-03T20:18:56.596 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.314191479, 2023-06-03T20:18:56.597 INFO:teuthology.orchestra.run.smithi089.stdout: "type_data": { 2023-06-03T20:18:56.597 INFO:teuthology.orchestra.run.smithi089.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-03T20:18:56.597 INFO:teuthology.orchestra.run.smithi089.stdout: "client_info": { 2023-06-03T20:18:56.597 INFO:teuthology.orchestra.run.smithi089.stdout: "client": "client.46187", 2023-06-03T20:18:56.597 INFO:teuthology.orchestra.run.smithi089.stdout: "client_addr": "172.21.15.89:0/880684210", 2023-06-03T20:18:56.597 INFO:teuthology.orchestra.run.smithi089.stdout: "tid": 164203 2023-06-03T20:18:56.597 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:18:56.597 INFO:teuthology.orchestra.run.smithi089.stdout: "events": [ 2023-06-03T20:18:56.598 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:18:56.598 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "initiated", 2023-06-03T20:18:56.598 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:14:25.503815+0000", 2023-06-03T20:18:56.598 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0 2023-06-03T20:18:56.598 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:18:56.598 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:18:56.598 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "throttled", 2023-06-03T20:18:56.599 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:14:25.503815+0000", 2023-06-03T20:18:56.599 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0 2023-06-03T20:18:56.599 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:18:56.599 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:18:56.600 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "header_read", 2023-06-03T20:18:56.600 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:14:25.503813+0000", 2023-06-03T20:18:56.600 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 4294967295.9999976 2023-06-03T20:18:56.600 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:18:56.601 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:18:56.601 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "all_read", 2023-06-03T20:18:56.601 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:14:25.503820+0000", 2023-06-03T20:18:56.601 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 7.6490000000000008e-06 2023-06-03T20:18:56.601 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:18:56.601 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:18:56.602 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "dispatched", 2023-06-03T20:18:56.602 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:14:25.503831+0000", 2023-06-03T20:18:56.602 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 1.0406e-05 2023-06-03T20:18:56.602 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:18:56.602 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:18:56.603 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "queued_for_pg", 2023-06-03T20:18:56.603 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:14:25.503839+0000", 2023-06-03T20:18:56.603 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 8.1510000000000004e-06 2023-06-03T20:18:56.603 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:18:56.603 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:18:56.603 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "reached_pg", 2023-06-03T20:18:56.603 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:14:25.778496+0000", 2023-06-03T20:18:56.603 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.27465759899999997 2023-06-03T20:18:56.604 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:18:56.604 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:18:56.604 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "started", 2023-06-03T20:18:56.604 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:14:25.778686+0000", 2023-06-03T20:18:56.604 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.0001895 2023-06-03T20:18:56.604 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:18:56.604 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:18:56.605 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "waiting for subops from 0,7", 2023-06-03T20:18:56.605 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:14:25.779183+0000", 2023-06-03T20:18:56.605 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.000497183 2023-06-03T20:18:56.605 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:18:56.605 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:18:56.605 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "op_commit", 2023-06-03T20:18:56.605 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:14:25.799499+0000", 2023-06-03T20:18:56.606 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.020315612 2023-06-03T20:18:56.606 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:18:56.606 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:18:56.606 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:18:56.606 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:14:25.799590+0000", 2023-06-03T20:18:56.606 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 9.1461999999999999e-05 2023-06-03T20:18:56.606 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:18:56.606 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:18:56.607 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:18:56.607 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:14:25.815167+0000", 2023-06-03T20:18:56.607 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.015577288999999999 2023-06-03T20:18:56.607 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:18:56.607 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:18:56.607 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "commit_sent", 2023-06-03T20:18:56.607 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:14:25.815223+0000", 2023-06-03T20:18:56.608 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 5.5151000000000001e-05 2023-06-03T20:18:56.608 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:18:56.608 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:18:56.608 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "done", 2023-06-03T20:18:56.608 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:14:25.818006+0000", 2023-06-03T20:18:56.608 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.0027836660000000002 2023-06-03T20:18:56.608 INFO:teuthology.orchestra.run.smithi089.stdout: } 2023-06-03T20:18:56.608 INFO:teuthology.orchestra.run.smithi089.stdout: ] 2023-06-03T20:18:56.609 INFO:teuthology.orchestra.run.smithi089.stdout: } 2023-06-03T20:18:56.609 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:18:56.609 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:18:56.609 INFO:teuthology.orchestra.run.smithi089.stdout: "description": "osd_op(client.46187.0:164206 11.9 11:95fb5add:::benchmark_data_smithi089_165412_object58678:head [delete] snapc 0=[] ondisk+write+known_if_redirected e203)", 2023-06-03T20:18:56.609 INFO:teuthology.orchestra.run.smithi089.stdout: "initiated_at": "2023-06-03T20:14:25.503999+0000", 2023-06-03T20:18:56.609 INFO:teuthology.orchestra.run.smithi089.stdout: "age": 271.08261563899998, 2023-06-03T20:18:56.609 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.335831822, 2023-06-03T20:18:56.609 INFO:teuthology.orchestra.run.smithi089.stdout: "type_data": { 2023-06-03T20:18:56.610 INFO:teuthology.orchestra.run.smithi089.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-03T20:18:56.610 INFO:teuthology.orchestra.run.smithi089.stdout: "client_info": { 2023-06-03T20:18:56.610 INFO:teuthology.orchestra.run.smithi089.stdout: "client": "client.46187", 2023-06-03T20:18:56.610 INFO:teuthology.orchestra.run.smithi089.stdout: "client_addr": "172.21.15.89:0/880684210", 2023-06-03T20:18:56.611 INFO:teuthology.orchestra.run.smithi089.stdout: "tid": 164206 2023-06-03T20:18:56.611 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:18:56.611 INFO:teuthology.orchestra.run.smithi089.stdout: "events": [ 2023-06-03T20:18:56.611 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:18:56.611 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "initiated", 2023-06-03T20:18:56.611 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:14:25.503999+0000", 2023-06-03T20:18:56.611 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0 2023-06-03T20:18:56.612 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:18:56.612 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:18:56.612 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "throttled", 2023-06-03T20:18:56.612 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:14:25.503999+0000", 2023-06-03T20:18:56.612 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0 2023-06-03T20:18:56.612 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:18:56.613 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:18:56.613 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "header_read", 2023-06-03T20:18:56.613 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:14:25.503999+0000", 2023-06-03T20:18:56.613 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 4294967295.9999995 2023-06-03T20:18:56.613 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:18:56.613 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:18:56.614 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "all_read", 2023-06-03T20:18:56.614 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:14:25.504003+0000", 2023-06-03T20:18:56.614 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 4.0990000000000001e-06 2023-06-03T20:18:56.614 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:18:56.614 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:18:56.614 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "dispatched", 2023-06-03T20:18:56.614 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:14:25.504013+0000", 2023-06-03T20:18:56.614 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 1.0631999999999999e-05 2023-06-03T20:18:56.615 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:18:56.615 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:18:56.615 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "queued_for_pg", 2023-06-03T20:18:56.615 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:14:25.504020+0000", 2023-06-03T20:18:56.615 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 6.3010000000000003e-06 2023-06-03T20:18:56.615 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:18:56.615 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:18:56.615 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "reached_pg", 2023-06-03T20:18:56.616 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:14:25.783600+0000", 2023-06-03T20:18:56.616 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.27958067599999997 2023-06-03T20:18:56.616 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:18:56.616 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:18:56.616 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "started", 2023-06-03T20:18:56.616 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:14:25.783885+0000", 2023-06-03T20:18:56.616 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.00028459999999999998 2023-06-03T20:18:56.616 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:18:56.617 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:18:56.617 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "waiting for subops from 0,7", 2023-06-03T20:18:56.617 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:14:25.784060+0000", 2023-06-03T20:18:56.617 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.00017522100000000001 2023-06-03T20:18:56.617 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:18:56.617 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:18:56.617 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "op_commit", 2023-06-03T20:18:56.617 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:14:25.799599+0000", 2023-06-03T20:18:56.618 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.015538817 2023-06-03T20:18:56.618 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:18:56.618 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:18:56.618 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:18:56.618 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:14:25.799675+0000", 2023-06-03T20:18:56.618 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 7.6278000000000004e-05 2023-06-03T20:18:56.618 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:18:56.619 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:18:56.619 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:18:56.619 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:14:25.836958+0000", 2023-06-03T20:18:56.619 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.037283206999999999 2023-06-03T20:18:56.619 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:18:56.619 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:18:56.619 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "commit_sent", 2023-06-03T20:18:56.619 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:14:25.837286+0000", 2023-06-03T20:18:56.620 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.00032740800000000001 2023-06-03T20:18:56.620 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:18:56.620 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:18:56.620 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "done", 2023-06-03T20:18:56.620 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:14:25.839831+0000", 2023-06-03T20:18:56.620 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.0025452090000000001 2023-06-03T20:18:56.620 INFO:teuthology.orchestra.run.smithi089.stdout: } 2023-06-03T20:18:56.621 INFO:teuthology.orchestra.run.smithi089.stdout: ] 2023-06-03T20:18:56.621 INFO:teuthology.orchestra.run.smithi089.stdout: } 2023-06-03T20:18:56.621 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:18:56.621 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:18:56.621 INFO:teuthology.orchestra.run.smithi089.stdout: "description": "osd_op(client.46187.0:164207 11.9 11:96f7e159:::benchmark_data_smithi089_165412_object58679:head [delete] snapc 0=[] ondisk+write+known_if_redirected e203)", 2023-06-03T20:18:56.621 INFO:teuthology.orchestra.run.smithi089.stdout: "initiated_at": "2023-06-03T20:14:25.504098+0000", 2023-06-03T20:18:56.621 INFO:teuthology.orchestra.run.smithi089.stdout: "age": 271.08251683600002, 2023-06-03T20:18:56.621 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.33812297000000002, 2023-06-03T20:18:56.622 INFO:teuthology.orchestra.run.smithi089.stdout: "type_data": { 2023-06-03T20:18:56.622 INFO:teuthology.orchestra.run.smithi089.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-03T20:18:56.622 INFO:teuthology.orchestra.run.smithi089.stdout: "client_info": { 2023-06-03T20:18:56.622 INFO:teuthology.orchestra.run.smithi089.stdout: "client": "client.46187", 2023-06-03T20:18:56.622 INFO:teuthology.orchestra.run.smithi089.stdout: "client_addr": "172.21.15.89:0/880684210", 2023-06-03T20:18:56.622 INFO:teuthology.orchestra.run.smithi089.stdout: "tid": 164207 2023-06-03T20:18:56.622 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:18:56.622 INFO:teuthology.orchestra.run.smithi089.stdout: "events": [ 2023-06-03T20:18:56.623 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:18:56.623 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "initiated", 2023-06-03T20:18:56.623 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:14:25.504098+0000", 2023-06-03T20:18:56.623 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0 2023-06-03T20:18:56.623 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:18:56.623 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:18:56.624 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "throttled", 2023-06-03T20:18:56.624 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:14:25.504098+0000", 2023-06-03T20:18:56.624 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0 2023-06-03T20:18:56.624 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:18:56.624 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:18:56.624 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "header_read", 2023-06-03T20:18:56.624 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:14:25.504097+0000", 2023-06-03T20:18:56.625 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 4294967295.9999995 2023-06-03T20:18:56.625 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:18:56.625 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:18:56.625 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "all_read", 2023-06-03T20:18:56.625 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:14:25.504101+0000", 2023-06-03T20:18:56.625 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 3.7120000000000002e-06 2023-06-03T20:18:56.625 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:18:56.625 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:18:56.626 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "dispatched", 2023-06-03T20:18:56.626 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:14:25.504109+0000", 2023-06-03T20:18:56.626 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 7.3660000000000003e-06 2023-06-03T20:18:56.626 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:18:56.626 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:18:56.626 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "queued_for_pg", 2023-06-03T20:18:56.626 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:14:25.504115+0000", 2023-06-03T20:18:56.627 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 6.1569999999999997e-06 2023-06-03T20:18:56.627 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:18:56.627 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:18:56.627 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "reached_pg", 2023-06-03T20:18:56.627 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:14:25.787507+0000", 2023-06-03T20:18:56.627 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.28339241599999998 2023-06-03T20:18:56.627 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:18:56.627 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:18:56.628 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "started", 2023-06-03T20:18:56.628 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:14:25.787710+0000", 2023-06-03T20:18:56.628 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.000202876 2023-06-03T20:18:56.628 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:18:56.628 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:18:56.628 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "waiting for subops from 0,7", 2023-06-03T20:18:56.628 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:14:25.787880+0000", 2023-06-03T20:18:56.629 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.00016971 2023-06-03T20:18:56.629 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:18:56.629 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:18:56.629 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "op_commit", 2023-06-03T20:18:56.629 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:14:25.799627+0000", 2023-06-03T20:18:56.629 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.011747226 2023-06-03T20:18:56.629 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:18:56.629 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:18:56.630 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:18:56.630 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:14:25.804525+0000", 2023-06-03T20:18:56.630 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.004897958 2023-06-03T20:18:56.630 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:18:56.630 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:18:56.630 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:18:56.630 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:14:25.839899+0000", 2023-06-03T20:18:56.630 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.035373588999999997 2023-06-03T20:18:56.631 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:18:56.631 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:18:56.631 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "commit_sent", 2023-06-03T20:18:56.631 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:14:25.839955+0000", 2023-06-03T20:18:56.631 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 5.6025e-05 2023-06-03T20:18:56.631 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:18:56.631 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:18:56.632 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "done", 2023-06-03T20:18:56.632 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:14:25.842221+0000", 2023-06-03T20:18:56.632 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.0022664759999999999 2023-06-03T20:18:56.633 INFO:teuthology.orchestra.run.smithi089.stdout: } 2023-06-03T20:18:56.633 INFO:teuthology.orchestra.run.smithi089.stdout: ] 2023-06-03T20:18:56.633 INFO:teuthology.orchestra.run.smithi089.stdout: } 2023-06-03T20:18:56.633 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:18:56.633 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:18:56.633 INFO:teuthology.orchestra.run.smithi089.stdout: "description": "MOSDPGRecoveryDeleteReply(11.0 e205,199 11:00dd3436:::benchmark_data_smithi089_165412_object49614:head,203'9602)", 2023-06-03T20:18:56.633 INFO:teuthology.orchestra.run.smithi089.stdout: "initiated_at": "2023-06-03T20:15:16.047868+0000", 2023-06-03T20:18:56.634 INFO:teuthology.orchestra.run.smithi089.stdout: "age": 220.53874731299999, 2023-06-03T20:18:56.634 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.31431026200000001, 2023-06-03T20:18:56.634 INFO:teuthology.orchestra.run.smithi089.stdout: "type_data": { 2023-06-03T20:18:56.634 INFO:teuthology.orchestra.run.smithi089.stdout: "flag_point": "reached pg", 2023-06-03T20:18:56.634 INFO:teuthology.orchestra.run.smithi089.stdout: "events": [ 2023-06-03T20:18:56.634 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:18:56.634 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "initiated", 2023-06-03T20:18:56.635 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:15:16.047868+0000", 2023-06-03T20:18:56.635 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0 2023-06-03T20:18:56.635 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:18:56.635 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:18:56.635 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "throttled", 2023-06-03T20:18:56.635 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:15:16.047868+0000", 2023-06-03T20:18:56.635 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0 2023-06-03T20:18:56.636 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:18:56.636 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:18:56.636 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "header_read", 2023-06-03T20:18:56.636 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:15:16.047867+0000", 2023-06-03T20:18:56.636 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 4294967295.999999 2023-06-03T20:18:56.636 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:18:56.636 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:18:56.637 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "all_read", 2023-06-03T20:18:56.637 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:15:16.047873+0000", 2023-06-03T20:18:56.637 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 6.4880000000000004e-06 2023-06-03T20:18:56.637 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:18:56.637 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:18:56.637 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "dispatched", 2023-06-03T20:18:56.637 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:15:16.047886+0000", 2023-06-03T20:18:56.637 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 1.2520999999999999e-05 2023-06-03T20:18:56.638 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:18:56.638 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:18:56.638 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "queued_for_pg", 2023-06-03T20:18:56.638 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:15:16.047894+0000", 2023-06-03T20:18:56.638 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 8.7800000000000006e-06 2023-06-03T20:18:56.638 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:18:56.638 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:18:56.639 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "reached_pg", 2023-06-03T20:18:56.639 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:15:16.361461+0000", 2023-06-03T20:18:56.639 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.31356641800000001 2023-06-03T20:18:56.639 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:18:56.639 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:18:56.639 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "done", 2023-06-03T20:18:56.639 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:15:16.362178+0000", 2023-06-03T20:18:56.639 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.00071698400000000005 2023-06-03T20:18:56.640 INFO:teuthology.orchestra.run.smithi089.stdout: } 2023-06-03T20:18:56.640 INFO:teuthology.orchestra.run.smithi089.stdout: ] 2023-06-03T20:18:56.640 INFO:teuthology.orchestra.run.smithi089.stdout: } 2023-06-03T20:18:56.640 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:18:56.640 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:18:56.640 INFO:teuthology.orchestra.run.smithi089.stdout: "description": "osd_op(client.46187.0:207727 11.0 11:0c7be78e:::benchmark_data_smithi089_165412_object102199:head [delete] snapc 0=[] ondisk+write+known_if_redirected e205)", 2023-06-03T20:18:56.641 INFO:teuthology.orchestra.run.smithi089.stdout: "initiated_at": "2023-06-03T20:15:25.860149+0000", 2023-06-03T20:18:56.641 INFO:teuthology.orchestra.run.smithi089.stdout: "age": 210.72646594700001, 2023-06-03T20:18:56.641 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 1.127864835, 2023-06-03T20:18:56.641 INFO:teuthology.orchestra.run.smithi089.stdout: "type_data": { 2023-06-03T20:18:56.641 INFO:teuthology.orchestra.run.smithi089.stdout: "flag_point": "reached pg", 2023-06-03T20:18:56.641 INFO:teuthology.orchestra.run.smithi089.stdout: "client_info": { 2023-06-03T20:18:56.641 INFO:teuthology.orchestra.run.smithi089.stdout: "client": "client.46187", 2023-06-03T20:18:56.642 INFO:teuthology.orchestra.run.smithi089.stdout: "client_addr": "172.21.15.89:0/880684210", 2023-06-03T20:18:56.642 INFO:teuthology.orchestra.run.smithi089.stdout: "tid": 207727 2023-06-03T20:18:56.642 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:18:56.642 INFO:teuthology.orchestra.run.smithi089.stdout: "events": [ 2023-06-03T20:18:56.642 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:18:56.643 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "initiated", 2023-06-03T20:18:56.643 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:15:25.860149+0000", 2023-06-03T20:18:56.643 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0 2023-06-03T20:18:56.643 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:18:56.643 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:18:56.643 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "throttled", 2023-06-03T20:18:56.643 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:15:25.860149+0000", 2023-06-03T20:18:56.644 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0 2023-06-03T20:18:56.644 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:18:56.644 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:18:56.644 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "header_read", 2023-06-03T20:18:56.644 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:15:25.860147+0000", 2023-06-03T20:18:56.644 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 4294967295.9999981 2023-06-03T20:18:56.645 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:18:56.645 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:18:56.645 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "all_read", 2023-06-03T20:18:56.645 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:15:25.860153+0000", 2023-06-03T20:18:56.646 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 6.2419999999999997e-06 2023-06-03T20:18:56.646 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:18:56.646 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:18:56.646 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "dispatched", 2023-06-03T20:18:56.646 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:15:25.860180+0000", 2023-06-03T20:18:56.647 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 2.6211e-05 2023-06-03T20:18:56.647 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:18:56.647 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:18:56.647 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "queued_for_pg", 2023-06-03T20:18:56.647 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:15:25.860193+0000", 2023-06-03T20:18:56.647 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 1.3370000000000001e-05 2023-06-03T20:18:56.647 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:18:56.647 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:18:56.648 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "reached_pg", 2023-06-03T20:18:56.648 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:15:25.860237+0000", 2023-06-03T20:18:56.648 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 4.4212999999999998e-05 2023-06-03T20:18:56.648 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:18:56.648 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:18:56.648 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "started", 2023-06-03T20:18:56.648 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:15:25.860361+0000", 2023-06-03T20:18:56.649 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.000124258 2023-06-03T20:18:56.649 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:18:56.649 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:18:56.649 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "waiting for subops from 0,7", 2023-06-03T20:18:56.649 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:15:25.860488+0000", 2023-06-03T20:18:56.649 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.000126332 2023-06-03T20:18:56.649 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:18:56.649 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:18:56.650 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:18:56.650 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:15:25.866680+0000", 2023-06-03T20:18:56.650 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.0061926990000000003 2023-06-03T20:18:56.650 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:18:56.650 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:18:56.650 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "op_commit", 2023-06-03T20:18:56.650 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:15:25.866691+0000", 2023-06-03T20:18:56.650 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 1.0581000000000001e-05 2023-06-03T20:18:56.651 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:18:56.651 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:18:56.651 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "reached_pg", 2023-06-03T20:18:56.651 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:15:25.880387+0000", 2023-06-03T20:18:56.651 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.013696033999999999 2023-06-03T20:18:56.651 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:18:56.651 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:18:56.651 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "waiting for peered", 2023-06-03T20:18:56.652 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:15:25.880398+0000", 2023-06-03T20:18:56.652 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 1.1015e-05 2023-06-03T20:18:56.652 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:18:56.652 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:18:56.652 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "reached_pg", 2023-06-03T20:18:56.652 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:15:26.987920+0000", 2023-06-03T20:18:56.652 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 1.1075220100000001 2023-06-03T20:18:56.653 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:18:56.653 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:18:56.653 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "done", 2023-06-03T20:18:56.653 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:15:26.988014+0000", 2023-06-03T20:18:56.653 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 9.3659999999999994e-05 2023-06-03T20:18:56.653 INFO:teuthology.orchestra.run.smithi089.stdout: } 2023-06-03T20:18:56.653 INFO:teuthology.orchestra.run.smithi089.stdout: ] 2023-06-03T20:18:56.653 INFO:teuthology.orchestra.run.smithi089.stdout: } 2023-06-03T20:18:56.654 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:18:56.654 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:18:56.654 INFO:teuthology.orchestra.run.smithi089.stdout: "description": "osd_op(client.46187.0:207785 11.0 11:0989b269:::benchmark_data_smithi089_165412_object102257:head [delete] snapc 0=[] ondisk+write+known_if_redirected e206)", 2023-06-03T20:18:56.654 INFO:teuthology.orchestra.run.smithi089.stdout: "initiated_at": "2023-06-03T20:15:25.945346+0000", 2023-06-03T20:18:56.654 INFO:teuthology.orchestra.run.smithi089.stdout: "age": 210.64126868400001, 2023-06-03T20:18:56.654 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 1.0581214370000001, 2023-06-03T20:18:56.654 INFO:teuthology.orchestra.run.smithi089.stdout: "type_data": { 2023-06-03T20:18:56.655 INFO:teuthology.orchestra.run.smithi089.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-03T20:18:56.655 INFO:teuthology.orchestra.run.smithi089.stdout: "client_info": { 2023-06-03T20:18:56.655 INFO:teuthology.orchestra.run.smithi089.stdout: "client": "client.46187", 2023-06-03T20:18:56.655 INFO:teuthology.orchestra.run.smithi089.stdout: "client_addr": "172.21.15.89:0/880684210", 2023-06-03T20:18:56.656 INFO:teuthology.orchestra.run.smithi089.stdout: "tid": 207785 2023-06-03T20:18:56.656 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:18:56.656 INFO:teuthology.orchestra.run.smithi089.stdout: "events": [ 2023-06-03T20:18:56.656 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:18:56.656 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "initiated", 2023-06-03T20:18:56.656 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:15:25.945346+0000", 2023-06-03T20:18:56.656 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0 2023-06-03T20:18:56.657 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:18:56.657 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:18:56.657 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "throttled", 2023-06-03T20:18:56.657 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:15:25.945346+0000", 2023-06-03T20:18:56.657 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0 2023-06-03T20:18:56.657 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:18:56.657 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:18:56.658 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "header_read", 2023-06-03T20:18:56.658 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:15:25.945345+0000", 2023-06-03T20:18:56.658 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 4294967295.999999 2023-06-03T20:18:56.658 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:18:56.658 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:18:56.658 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "all_read", 2023-06-03T20:18:56.658 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:15:25.945350+0000", 2023-06-03T20:18:56.658 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 4.4789999999999998e-06 2023-06-03T20:18:56.659 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:18:56.659 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:18:56.659 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "dispatched", 2023-06-03T20:18:56.659 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:15:25.945362+0000", 2023-06-03T20:18:56.659 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 1.1756999999999999e-05 2023-06-03T20:18:56.659 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:18:56.659 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:18:56.659 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "queued_for_pg", 2023-06-03T20:18:56.660 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:15:25.945369+0000", 2023-06-03T20:18:56.660 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 7.0400000000000004e-06 2023-06-03T20:18:56.660 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:18:56.660 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:18:56.660 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "reached_pg", 2023-06-03T20:18:56.661 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:15:25.948763+0000", 2023-06-03T20:18:56.661 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.0033944420000000001 2023-06-03T20:18:56.661 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:18:56.661 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:18:56.662 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "waiting for peered", 2023-06-03T20:18:56.662 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:15:25.948773+0000", 2023-06-03T20:18:56.662 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 9.9529999999999994e-06 2023-06-03T20:18:56.662 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:18:56.663 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:18:56.663 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "reached_pg", 2023-06-03T20:18:56.663 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:15:26.989154+0000", 2023-06-03T20:18:56.663 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 1.0403810520000001 2023-06-03T20:18:56.663 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:18:56.663 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:18:56.663 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "started", 2023-06-03T20:18:56.663 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:15:26.989293+0000", 2023-06-03T20:18:56.664 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.00013865500000000001 2023-06-03T20:18:56.664 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:18:56.664 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:18:56.664 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "waiting for subops from 0,7", 2023-06-03T20:18:56.664 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:15:26.989450+0000", 2023-06-03T20:18:56.665 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.00015770500000000001 2023-06-03T20:18:56.665 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:18:56.665 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:18:56.665 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:18:56.666 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:15:26.997017+0000", 2023-06-03T20:18:56.666 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.0075669609999999997 2023-06-03T20:18:56.666 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:18:56.666 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:18:56.666 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "op_commit", 2023-06-03T20:18:56.667 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:15:26.997149+0000", 2023-06-03T20:18:56.667 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.00013187999999999999 2023-06-03T20:18:56.667 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:18:56.667 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:18:56.668 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:18:56.668 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:15:27.003398+0000", 2023-06-03T20:18:56.668 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.0062484669999999997 2023-06-03T20:18:56.668 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:18:56.668 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:18:56.668 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "commit_sent", 2023-06-03T20:18:56.669 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:15:27.003433+0000", 2023-06-03T20:18:56.669 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 3.5320000000000001e-05 2023-06-03T20:18:56.669 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:18:56.669 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:18:56.669 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "done", 2023-06-03T20:18:56.669 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:15:27.003468+0000", 2023-06-03T20:18:56.669 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 3.4524999999999999e-05 2023-06-03T20:18:56.670 INFO:teuthology.orchestra.run.smithi089.stdout: } 2023-06-03T20:18:56.670 INFO:teuthology.orchestra.run.smithi089.stdout: ] 2023-06-03T20:18:56.670 INFO:teuthology.orchestra.run.smithi089.stdout: } 2023-06-03T20:18:56.670 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:18:56.670 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:18:56.670 INFO:teuthology.orchestra.run.smithi089.stdout: "description": "osd_op(client.46187.0:207792 11.0 11:09e736e9:::benchmark_data_smithi089_165412_object102264:head [delete] snapc 0=[] ondisk+write+known_if_redirected e206)", 2023-06-03T20:18:56.670 INFO:teuthology.orchestra.run.smithi089.stdout: "initiated_at": "2023-06-03T20:15:25.955125+0000", 2023-06-03T20:18:56.670 INFO:teuthology.orchestra.run.smithi089.stdout: "age": 210.631489692, 2023-06-03T20:18:56.671 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 1.048451072, 2023-06-03T20:18:56.671 INFO:teuthology.orchestra.run.smithi089.stdout: "type_data": { 2023-06-03T20:18:56.671 INFO:teuthology.orchestra.run.smithi089.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-03T20:18:56.671 INFO:teuthology.orchestra.run.smithi089.stdout: "client_info": { 2023-06-03T20:18:56.671 INFO:teuthology.orchestra.run.smithi089.stdout: "client": "client.46187", 2023-06-03T20:18:56.671 INFO:teuthology.orchestra.run.smithi089.stdout: "client_addr": "172.21.15.89:0/880684210", 2023-06-03T20:18:56.671 INFO:teuthology.orchestra.run.smithi089.stdout: "tid": 207792 2023-06-03T20:18:56.671 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:18:56.672 INFO:teuthology.orchestra.run.smithi089.stdout: "events": [ 2023-06-03T20:18:56.672 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:18:56.672 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "initiated", 2023-06-03T20:18:56.672 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:15:25.955125+0000", 2023-06-03T20:18:56.673 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0 2023-06-03T20:18:56.673 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:18:56.673 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:18:56.673 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "throttled", 2023-06-03T20:18:56.673 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:15:25.955125+0000", 2023-06-03T20:18:56.673 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0 2023-06-03T20:18:56.673 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:18:56.673 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:18:56.674 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "header_read", 2023-06-03T20:18:56.674 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:15:25.955124+0000", 2023-06-03T20:18:56.674 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 4294967295.9999986 2023-06-03T20:18:56.674 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:18:56.674 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:18:56.674 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "all_read", 2023-06-03T20:18:56.674 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:15:25.955129+0000", 2023-06-03T20:18:56.674 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 5.7069999999999997e-06 2023-06-03T20:18:56.675 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:18:56.675 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:18:56.675 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "dispatched", 2023-06-03T20:18:56.675 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:15:25.955140+0000", 2023-06-03T20:18:56.675 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 1.0903e-05 2023-06-03T20:18:56.675 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:18:56.675 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:18:56.676 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "queued_for_pg", 2023-06-03T20:18:56.676 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:15:25.955146+0000", 2023-06-03T20:18:56.676 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 5.5369999999999997e-06 2023-06-03T20:18:56.676 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:18:56.676 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:18:56.676 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "reached_pg", 2023-06-03T20:18:56.676 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:15:25.955187+0000", 2023-06-03T20:18:56.676 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 4.1287e-05 2023-06-03T20:18:56.677 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:18:56.677 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:18:56.677 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "waiting for peered", 2023-06-03T20:18:56.677 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:15:25.955195+0000", 2023-06-03T20:18:56.677 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 7.3810000000000001e-06 2023-06-03T20:18:56.677 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:18:56.677 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:18:56.678 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "reached_pg", 2023-06-03T20:18:56.678 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:15:26.992378+0000", 2023-06-03T20:18:56.678 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 1.0371836290000001 2023-06-03T20:18:56.678 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:18:56.678 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:18:56.678 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "started", 2023-06-03T20:18:56.679 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:15:26.992521+0000", 2023-06-03T20:18:56.679 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.000142519 2023-06-03T20:18:56.679 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:18:56.679 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:18:56.680 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "waiting for subops from 0,7", 2023-06-03T20:18:56.680 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:15:26.992659+0000", 2023-06-03T20:18:56.680 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.000138009 2023-06-03T20:18:56.680 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:18:56.680 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:18:56.680 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:18:56.680 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:15:26.997141+0000", 2023-06-03T20:18:56.681 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.0044827240000000004 2023-06-03T20:18:56.681 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:18:56.681 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:18:56.681 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "op_commit", 2023-06-03T20:18:56.681 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:15:27.003475+0000", 2023-06-03T20:18:56.681 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.0063338910000000003 2023-06-03T20:18:56.682 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:18:56.682 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:18:56.682 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:18:56.682 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:15:27.003517+0000", 2023-06-03T20:18:56.683 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 4.1882999999999999e-05 2023-06-03T20:18:56.683 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:18:56.683 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:18:56.683 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "commit_sent", 2023-06-03T20:18:56.684 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:15:27.003544+0000", 2023-06-03T20:18:56.684 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 2.7157000000000001e-05 2023-06-03T20:18:56.684 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:18:56.685 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:18:56.685 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "done", 2023-06-03T20:18:56.685 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:15:27.003576+0000", 2023-06-03T20:18:56.685 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 3.1856000000000001e-05 2023-06-03T20:18:56.686 INFO:teuthology.orchestra.run.smithi089.stdout: } 2023-06-03T20:18:56.686 INFO:teuthology.orchestra.run.smithi089.stdout: ] 2023-06-03T20:18:56.686 INFO:teuthology.orchestra.run.smithi089.stdout: } 2023-06-03T20:18:56.686 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:18:56.687 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:18:56.687 INFO:teuthology.orchestra.run.smithi089.stdout: "description": "osd_op(client.46187.0:207796 11.0 11:01105bae:::benchmark_data_smithi089_165412_object102268:head [delete] snapc 0=[] ondisk+write+known_if_redirected e206)", 2023-06-03T20:18:56.687 INFO:teuthology.orchestra.run.smithi089.stdout: "initiated_at": "2023-06-03T20:15:25.959825+0000", 2023-06-03T20:18:56.687 INFO:teuthology.orchestra.run.smithi089.stdout: "age": 210.626789909, 2023-06-03T20:18:56.688 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 1.0574886130000001, 2023-06-03T20:18:56.688 INFO:teuthology.orchestra.run.smithi089.stdout: "type_data": { 2023-06-03T20:18:56.688 INFO:teuthology.orchestra.run.smithi089.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-03T20:18:56.688 INFO:teuthology.orchestra.run.smithi089.stdout: "client_info": { 2023-06-03T20:18:56.688 INFO:teuthology.orchestra.run.smithi089.stdout: "client": "client.46187", 2023-06-03T20:18:56.689 INFO:teuthology.orchestra.run.smithi089.stdout: "client_addr": "172.21.15.89:0/880684210", 2023-06-03T20:18:56.689 INFO:teuthology.orchestra.run.smithi089.stdout: "tid": 207796 2023-06-03T20:18:56.689 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:18:56.689 INFO:teuthology.orchestra.run.smithi089.stdout: "events": [ 2023-06-03T20:18:56.689 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:18:56.689 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "initiated", 2023-06-03T20:18:56.689 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:15:25.959825+0000", 2023-06-03T20:18:56.689 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0 2023-06-03T20:18:56.690 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:18:56.690 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:18:56.690 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "throttled", 2023-06-03T20:18:56.690 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:15:25.959825+0000", 2023-06-03T20:18:56.690 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0 2023-06-03T20:18:56.690 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:18:56.690 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:18:56.691 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "header_read", 2023-06-03T20:18:56.691 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:15:25.959824+0000", 2023-06-03T20:18:56.691 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 4294967295.999999 2023-06-03T20:18:56.691 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:18:56.691 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:18:56.691 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "all_read", 2023-06-03T20:18:56.691 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:15:25.959831+0000", 2023-06-03T20:18:56.691 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 7.1579999999999999e-06 2023-06-03T20:18:56.692 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:18:56.692 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:18:56.692 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "dispatched", 2023-06-03T20:18:56.692 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:15:25.959841+0000", 2023-06-03T20:18:56.692 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 9.8220000000000002e-06 2023-06-03T20:18:56.692 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:18:56.692 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:18:56.693 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "queued_for_pg", 2023-06-03T20:18:56.693 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:15:25.959848+0000", 2023-06-03T20:18:56.693 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 7.1470000000000004e-06 2023-06-03T20:18:56.693 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:18:56.693 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:18:56.693 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "reached_pg", 2023-06-03T20:18:56.693 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:15:25.959896+0000", 2023-06-03T20:18:56.693 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 4.7707999999999998e-05 2023-06-03T20:18:56.694 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:18:56.694 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:18:56.694 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "waiting for peered", 2023-06-03T20:18:56.694 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:15:25.959905+0000", 2023-06-03T20:18:56.694 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 8.9760000000000001e-06 2023-06-03T20:18:56.694 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:18:56.694 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:18:56.694 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "reached_pg", 2023-06-03T20:18:56.695 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:15:26.997172+0000", 2023-06-03T20:18:56.695 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 1.0372673530000001 2023-06-03T20:18:56.695 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:18:56.695 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:18:56.695 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "started", 2023-06-03T20:18:56.695 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:15:26.997298+0000", 2023-06-03T20:18:56.695 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.00012557099999999999 2023-06-03T20:18:56.696 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:18:56.696 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:18:56.696 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "waiting for subops from 0,7", 2023-06-03T20:18:56.696 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:15:26.997454+0000", 2023-06-03T20:18:56.696 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.00015632100000000001 2023-06-03T20:18:56.696 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:18:56.696 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:18:56.697 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:18:56.697 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:15:27.003496+0000", 2023-06-03T20:18:56.697 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.0060421709999999998 2023-06-03T20:18:56.697 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:18:56.697 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:18:56.697 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:18:56.697 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:15:27.003610+0000", 2023-06-03T20:18:56.697 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.00011393899999999999 2023-06-03T20:18:56.698 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:18:56.698 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:18:56.698 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "op_commit", 2023-06-03T20:18:56.698 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:15:27.017215+0000", 2023-06-03T20:18:56.698 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.013604298000000001 2023-06-03T20:18:56.698 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:18:56.698 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:18:56.698 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "commit_sent", 2023-06-03T20:18:56.699 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:15:27.017271+0000", 2023-06-03T20:18:56.699 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 5.6124e-05 2023-06-03T20:18:56.699 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:18:56.699 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:18:56.699 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "done", 2023-06-03T20:18:56.699 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:15:27.017314+0000", 2023-06-03T20:18:56.699 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 4.2865999999999999e-05 2023-06-03T20:18:56.699 INFO:teuthology.orchestra.run.smithi089.stdout: } 2023-06-03T20:18:56.700 INFO:teuthology.orchestra.run.smithi089.stdout: ] 2023-06-03T20:18:56.700 INFO:teuthology.orchestra.run.smithi089.stdout: } 2023-06-03T20:18:56.700 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:18:56.700 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:18:56.700 INFO:teuthology.orchestra.run.smithi089.stdout: "description": "osd_op(client.46187.0:207835 11.0 11:03b92e3e:::benchmark_data_smithi089_165412_object102307:head [delete] snapc 0=[] ondisk+write+known_if_redirected e206)", 2023-06-03T20:18:56.700 INFO:teuthology.orchestra.run.smithi089.stdout: "initiated_at": "2023-06-03T20:15:26.041016+0000", 2023-06-03T20:18:56.700 INFO:teuthology.orchestra.run.smithi089.stdout: "age": 210.54559913400001, 2023-06-03T20:18:56.701 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.97637812300000004, 2023-06-03T20:18:56.701 INFO:teuthology.orchestra.run.smithi089.stdout: "type_data": { 2023-06-03T20:18:56.701 INFO:teuthology.orchestra.run.smithi089.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-03T20:18:56.701 INFO:teuthology.orchestra.run.smithi089.stdout: "client_info": { 2023-06-03T20:18:56.701 INFO:teuthology.orchestra.run.smithi089.stdout: "client": "client.46187", 2023-06-03T20:18:56.701 INFO:teuthology.orchestra.run.smithi089.stdout: "client_addr": "172.21.15.89:0/880684210", 2023-06-03T20:18:56.701 INFO:teuthology.orchestra.run.smithi089.stdout: "tid": 207835 2023-06-03T20:18:56.701 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:18:56.702 INFO:teuthology.orchestra.run.smithi089.stdout: "events": [ 2023-06-03T20:18:56.702 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:18:56.702 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "initiated", 2023-06-03T20:18:56.702 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:15:26.041016+0000", 2023-06-03T20:18:56.702 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0 2023-06-03T20:18:56.702 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:18:56.702 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:18:56.703 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "throttled", 2023-06-03T20:18:56.703 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:15:26.041016+0000", 2023-06-03T20:18:56.703 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0 2023-06-03T20:18:56.703 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:18:56.703 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:18:56.703 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "header_read", 2023-06-03T20:18:56.703 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:15:26.041014+0000", 2023-06-03T20:18:56.703 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 4294967295.9999981 2023-06-03T20:18:56.704 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:18:56.704 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:18:56.704 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "all_read", 2023-06-03T20:18:56.704 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:15:26.041022+0000", 2023-06-03T20:18:56.704 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 7.8849999999999999e-06 2023-06-03T20:18:56.704 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:18:56.704 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:18:56.705 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "dispatched", 2023-06-03T20:18:56.705 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:15:26.041035+0000", 2023-06-03T20:18:56.705 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 1.3275000000000001e-05 2023-06-03T20:18:56.705 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:18:56.705 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:18:56.705 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "queued_for_pg", 2023-06-03T20:18:56.705 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:15:26.041044+0000", 2023-06-03T20:18:56.705 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 8.7390000000000005e-06 2023-06-03T20:18:56.706 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:18:56.706 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:18:56.706 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "reached_pg", 2023-06-03T20:18:56.706 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:15:26.047821+0000", 2023-06-03T20:18:56.706 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.0067766789999999999 2023-06-03T20:18:56.706 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:18:56.706 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:18:56.707 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "waiting for peered", 2023-06-03T20:18:56.707 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:15:26.047830+0000", 2023-06-03T20:18:56.707 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 9.2420000000000006e-06 2023-06-03T20:18:56.707 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:18:56.707 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:18:56.707 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "reached_pg", 2023-06-03T20:18:56.707 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:15:27.003632+0000", 2023-06-03T20:18:56.707 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.95580217300000003 2023-06-03T20:18:56.708 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:18:56.708 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:18:56.708 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "started", 2023-06-03T20:18:56.708 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:15:27.003726+0000", 2023-06-03T20:18:56.708 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 9.3993999999999995e-05 2023-06-03T20:18:56.708 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:18:56.708 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:18:56.709 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "waiting for subops from 0,7", 2023-06-03T20:18:56.709 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:15:27.003808+0000", 2023-06-03T20:18:56.709 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 8.1712999999999995e-05 2023-06-03T20:18:56.709 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:18:56.709 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:18:56.709 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:18:56.709 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:15:27.011327+0000", 2023-06-03T20:18:56.709 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.0075194390000000002 2023-06-03T20:18:56.710 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:18:56.710 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:18:56.710 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:18:56.710 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:15:27.011361+0000", 2023-06-03T20:18:56.710 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 3.3522999999999997e-05 2023-06-03T20:18:56.710 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:18:56.711 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:18:56.711 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "op_commit", 2023-06-03T20:18:56.711 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:15:27.017323+0000", 2023-06-03T20:18:56.711 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.0059623690000000003 2023-06-03T20:18:56.711 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:18:56.711 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:18:56.711 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "commit_sent", 2023-06-03T20:18:56.711 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:15:27.017360+0000", 2023-06-03T20:18:56.712 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 3.6461999999999998e-05 2023-06-03T20:18:56.712 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:18:56.712 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:18:56.712 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "done", 2023-06-03T20:18:56.712 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:15:27.017394+0000", 2023-06-03T20:18:56.712 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 3.4350000000000001e-05 2023-06-03T20:18:56.712 INFO:teuthology.orchestra.run.smithi089.stdout: } 2023-06-03T20:18:56.712 INFO:teuthology.orchestra.run.smithi089.stdout: ] 2023-06-03T20:18:56.713 INFO:teuthology.orchestra.run.smithi089.stdout: } 2023-06-03T20:18:56.713 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:18:56.713 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:18:56.713 INFO:teuthology.orchestra.run.smithi089.stdout: "description": "osd_op(client.46187.0:207841 11.0 11:0b5f0426:::benchmark_data_smithi089_165412_object102313:head [delete] snapc 0=[] ondisk+write+known_if_redirected e206)", 2023-06-03T20:18:56.713 INFO:teuthology.orchestra.run.smithi089.stdout: "initiated_at": "2023-06-03T20:15:26.047532+0000", 2023-06-03T20:18:56.713 INFO:teuthology.orchestra.run.smithi089.stdout: "age": 210.53908253500001, 2023-06-03T20:18:56.713 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.99484972100000002, 2023-06-03T20:18:56.714 INFO:teuthology.orchestra.run.smithi089.stdout: "type_data": { 2023-06-03T20:18:56.714 INFO:teuthology.orchestra.run.smithi089.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-03T20:18:56.714 INFO:teuthology.orchestra.run.smithi089.stdout: "client_info": { 2023-06-03T20:18:56.714 INFO:teuthology.orchestra.run.smithi089.stdout: "client": "client.46187", 2023-06-03T20:18:56.714 INFO:teuthology.orchestra.run.smithi089.stdout: "client_addr": "172.21.15.89:0/880684210", 2023-06-03T20:18:56.714 INFO:teuthology.orchestra.run.smithi089.stdout: "tid": 207841 2023-06-03T20:18:56.714 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:18:56.714 INFO:teuthology.orchestra.run.smithi089.stdout: "events": [ 2023-06-03T20:18:56.715 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:18:56.715 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "initiated", 2023-06-03T20:18:56.715 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:15:26.047532+0000", 2023-06-03T20:18:56.715 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0 2023-06-03T20:18:56.715 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:18:56.715 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:18:56.715 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "throttled", 2023-06-03T20:18:56.716 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:15:26.047532+0000", 2023-06-03T20:18:56.716 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0 2023-06-03T20:18:56.716 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:18:56.716 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:18:56.716 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "header_read", 2023-06-03T20:18:56.716 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:15:26.047532+0000", 2023-06-03T20:18:56.716 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 4294967295.9999995 2023-06-03T20:18:56.716 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:18:56.717 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:18:56.717 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "all_read", 2023-06-03T20:18:56.717 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:15:26.047536+0000", 2023-06-03T20:18:56.717 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 3.8920000000000002e-06 2023-06-03T20:18:56.717 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:18:56.717 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:18:56.717 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "dispatched", 2023-06-03T20:18:56.717 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:15:26.047543+0000", 2023-06-03T20:18:56.718 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 7.3660000000000003e-06 2023-06-03T20:18:56.718 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:18:56.718 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:18:56.718 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "queued_for_pg", 2023-06-03T20:18:56.718 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:15:26.047549+0000", 2023-06-03T20:18:56.718 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 5.7250000000000002e-06 2023-06-03T20:18:56.718 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:18:56.719 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:18:56.719 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "reached_pg", 2023-06-03T20:18:56.719 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:15:26.047852+0000", 2023-06-03T20:18:56.719 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.00030297400000000002 2023-06-03T20:18:56.719 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:18:56.719 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:18:56.719 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "waiting for peered", 2023-06-03T20:18:56.719 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:15:26.047858+0000", 2023-06-03T20:18:56.720 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 6.1249999999999998e-06 2023-06-03T20:18:56.720 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:18:56.720 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:18:56.720 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "reached_pg", 2023-06-03T20:18:56.720 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:15:27.007420+0000", 2023-06-03T20:18:56.720 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.95956176999999998 2023-06-03T20:18:56.721 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:18:56.721 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:18:56.721 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "started", 2023-06-03T20:18:56.721 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:15:27.007550+0000", 2023-06-03T20:18:56.721 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.00013030800000000001 2023-06-03T20:18:56.721 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:18:56.721 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:18:56.722 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "waiting for subops from 0,7", 2023-06-03T20:18:56.722 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:15:27.007639+0000", 2023-06-03T20:18:56.722 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 8.8787000000000002e-05 2023-06-03T20:18:56.722 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:18:56.722 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:18:56.722 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:18:56.722 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:15:27.017205+0000", 2023-06-03T20:18:56.722 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.0095666610000000006 2023-06-03T20:18:56.723 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:18:56.723 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:18:56.723 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "op_commit", 2023-06-03T20:18:56.723 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:15:27.017402+0000", 2023-06-03T20:18:56.723 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.00019635099999999999 2023-06-03T20:18:56.723 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:18:56.723 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:18:56.723 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:18:56.724 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:15:27.042294+0000", 2023-06-03T20:18:56.724 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.024892239 2023-06-03T20:18:56.724 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:18:56.724 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:18:56.724 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "commit_sent", 2023-06-03T20:18:56.724 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:15:27.042342+0000", 2023-06-03T20:18:56.724 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 4.7728000000000001e-05 2023-06-03T20:18:56.725 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:18:56.725 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:18:56.725 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "done", 2023-06-03T20:18:56.725 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:15:27.042382+0000", 2023-06-03T20:18:56.725 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 4.0380999999999998e-05 2023-06-03T20:18:56.725 INFO:teuthology.orchestra.run.smithi089.stdout: } 2023-06-03T20:18:56.725 INFO:teuthology.orchestra.run.smithi089.stdout: ] 2023-06-03T20:18:56.726 INFO:teuthology.orchestra.run.smithi089.stdout: } 2023-06-03T20:18:56.726 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:18:56.726 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:18:56.726 INFO:teuthology.orchestra.run.smithi089.stdout: "description": "osd_op(client.46187.0:207842 11.0 11:06f54139:::benchmark_data_smithi089_165412_object102314:head [delete] snapc 0=[] ondisk+write+known_if_redirected e206)", 2023-06-03T20:18:56.726 INFO:teuthology.orchestra.run.smithi089.stdout: "initiated_at": "2023-06-03T20:15:26.047766+0000", 2023-06-03T20:18:56.726 INFO:teuthology.orchestra.run.smithi089.stdout: "age": 210.53884849900001, 2023-06-03T20:18:56.726 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.99475771800000001, 2023-06-03T20:18:56.726 INFO:teuthology.orchestra.run.smithi089.stdout: "type_data": { 2023-06-03T20:18:56.727 INFO:teuthology.orchestra.run.smithi089.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-03T20:18:56.727 INFO:teuthology.orchestra.run.smithi089.stdout: "client_info": { 2023-06-03T20:18:56.727 INFO:teuthology.orchestra.run.smithi089.stdout: "client": "client.46187", 2023-06-03T20:18:56.727 INFO:teuthology.orchestra.run.smithi089.stdout: "client_addr": "172.21.15.89:0/880684210", 2023-06-03T20:18:56.727 INFO:teuthology.orchestra.run.smithi089.stdout: "tid": 207842 2023-06-03T20:18:56.727 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:18:56.728 INFO:teuthology.orchestra.run.smithi089.stdout: "events": [ 2023-06-03T20:18:56.728 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:18:56.728 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "initiated", 2023-06-03T20:18:56.728 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:15:26.047766+0000", 2023-06-03T20:18:56.728 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0 2023-06-03T20:18:56.728 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:18:56.728 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:18:56.728 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "throttled", 2023-06-03T20:18:56.729 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:15:26.047766+0000", 2023-06-03T20:18:56.729 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0 2023-06-03T20:18:56.729 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:18:56.729 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:18:56.729 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "header_read", 2023-06-03T20:18:56.729 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:15:26.047765+0000", 2023-06-03T20:18:56.729 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 4294967295.999999 2023-06-03T20:18:56.729 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:18:56.730 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:18:56.730 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "all_read", 2023-06-03T20:18:56.730 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:15:26.047771+0000", 2023-06-03T20:18:56.730 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 5.9970000000000004e-06 2023-06-03T20:18:56.730 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:18:56.730 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:18:56.730 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "dispatched", 2023-06-03T20:18:56.730 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:15:26.047781+0000", 2023-06-03T20:18:56.731 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 1.0030000000000001e-05 2023-06-03T20:18:56.731 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:18:56.731 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:18:56.731 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "queued_for_pg", 2023-06-03T20:18:56.731 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:15:26.047789+0000", 2023-06-03T20:18:56.732 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 7.2309999999999999e-06 2023-06-03T20:18:56.732 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:18:56.732 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:18:56.732 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "reached_pg", 2023-06-03T20:18:56.733 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:15:26.047876+0000", 2023-06-03T20:18:56.733 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 8.7441999999999994e-05 2023-06-03T20:18:56.733 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:18:56.733 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:18:56.733 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "waiting for peered", 2023-06-03T20:18:56.734 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:15:26.047883+0000", 2023-06-03T20:18:56.734 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 6.5610000000000004e-06 2023-06-03T20:18:56.734 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:18:56.734 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:18:56.735 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "reached_pg", 2023-06-03T20:18:56.735 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:15:27.011383+0000", 2023-06-03T20:18:56.735 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.96350037899999996 2023-06-03T20:18:56.735 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:18:56.736 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:18:56.736 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "started", 2023-06-03T20:18:56.736 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:15:27.011479+0000", 2023-06-03T20:18:56.736 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 9.6182999999999997e-05 2023-06-03T20:18:56.736 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:18:56.736 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:18:56.737 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "waiting for subops from 0,7", 2023-06-03T20:18:56.737 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:15:27.011594+0000", 2023-06-03T20:18:56.737 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.00011521 2023-06-03T20:18:56.737 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:18:56.737 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:18:56.738 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "op_commit", 2023-06-03T20:18:56.738 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:15:27.027490+0000", 2023-06-03T20:18:56.738 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.015895167000000002 2023-06-03T20:18:56.738 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:18:56.738 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:18:56.738 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:18:56.738 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:15:27.027559+0000", 2023-06-03T20:18:56.739 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 6.9498000000000004e-05 2023-06-03T20:18:56.739 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:18:56.739 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:18:56.739 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:18:56.739 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:15:27.042451+0000", 2023-06-03T20:18:56.740 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.014892392000000001 2023-06-03T20:18:56.740 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:18:56.740 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:18:56.740 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "commit_sent", 2023-06-03T20:18:56.740 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:15:27.042490+0000", 2023-06-03T20:18:56.740 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 3.8501999999999998e-05 2023-06-03T20:18:56.740 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:18:56.741 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:18:56.741 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "done", 2023-06-03T20:18:56.741 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:15:27.042524+0000", 2023-06-03T20:18:56.741 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 3.4167e-05 2023-06-03T20:18:56.741 INFO:teuthology.orchestra.run.smithi089.stdout: } 2023-06-03T20:18:56.741 INFO:teuthology.orchestra.run.smithi089.stdout: ] 2023-06-03T20:18:56.742 INFO:teuthology.orchestra.run.smithi089.stdout: } 2023-06-03T20:18:56.742 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:18:56.742 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:18:56.742 INFO:teuthology.orchestra.run.smithi089.stdout: "description": "osd_op(client.46187.0:207870 11.0 11:096247d7:::benchmark_data_smithi089_165412_object102342:head [delete] snapc 0=[] ondisk+write+known_if_redirected e206)", 2023-06-03T20:18:56.742 INFO:teuthology.orchestra.run.smithi089.stdout: "initiated_at": "2023-06-03T20:15:26.088859+0000", 2023-06-03T20:18:56.742 INFO:teuthology.orchestra.run.smithi089.stdout: "age": 210.49775575300001, 2023-06-03T20:18:56.742 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.95380226400000001, 2023-06-03T20:18:56.742 INFO:teuthology.orchestra.run.smithi089.stdout: "type_data": { 2023-06-03T20:18:56.743 INFO:teuthology.orchestra.run.smithi089.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-03T20:18:56.743 INFO:teuthology.orchestra.run.smithi089.stdout: "client_info": { 2023-06-03T20:18:56.743 INFO:teuthology.orchestra.run.smithi089.stdout: "client": "client.46187", 2023-06-03T20:18:56.743 INFO:teuthology.orchestra.run.smithi089.stdout: "client_addr": "172.21.15.89:0/880684210", 2023-06-03T20:18:56.743 INFO:teuthology.orchestra.run.smithi089.stdout: "tid": 207870 2023-06-03T20:18:56.743 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:18:56.743 INFO:teuthology.orchestra.run.smithi089.stdout: "events": [ 2023-06-03T20:18:56.743 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:18:56.744 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "initiated", 2023-06-03T20:18:56.744 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:15:26.088859+0000", 2023-06-03T20:18:56.744 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0 2023-06-03T20:18:56.744 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:18:56.744 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:18:56.744 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "throttled", 2023-06-03T20:18:56.744 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:15:26.088859+0000", 2023-06-03T20:18:56.745 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0 2023-06-03T20:18:56.745 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:18:56.745 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:18:56.745 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "header_read", 2023-06-03T20:18:56.745 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:15:26.088858+0000", 2023-06-03T20:18:56.745 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 4294967295.9999986 2023-06-03T20:18:56.745 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:18:56.745 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:18:56.746 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "all_read", 2023-06-03T20:18:56.746 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:15:26.088864+0000", 2023-06-03T20:18:56.746 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 5.9089999999999997e-06 2023-06-03T20:18:56.746 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:18:56.746 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:18:56.747 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "dispatched", 2023-06-03T20:18:56.747 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:15:26.088878+0000", 2023-06-03T20:18:56.747 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 1.3947999999999999e-05 2023-06-03T20:18:56.747 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:18:56.748 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:18:56.748 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "queued_for_pg", 2023-06-03T20:18:56.748 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:15:26.088886+0000", 2023-06-03T20:18:56.748 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 8.5839999999999995e-06 2023-06-03T20:18:56.748 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:18:56.749 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:18:56.749 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "reached_pg", 2023-06-03T20:18:56.749 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:15:26.089626+0000", 2023-06-03T20:18:56.749 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.00073987900000000004 2023-06-03T20:18:56.749 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:18:56.749 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:18:56.750 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "waiting for peered", 2023-06-03T20:18:56.750 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:15:26.089635+0000", 2023-06-03T20:18:56.750 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 8.918e-06 2023-06-03T20:18:56.750 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:18:56.750 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:18:56.751 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "reached_pg", 2023-06-03T20:18:56.751 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:15:27.014628+0000", 2023-06-03T20:18:56.751 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.92499317700000006 2023-06-03T20:18:56.751 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:18:56.751 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:18:56.751 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "started", 2023-06-03T20:18:56.751 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:15:27.014732+0000", 2023-06-03T20:18:56.751 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.000103738 2023-06-03T20:18:56.752 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:18:56.752 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:18:56.752 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "waiting for subops from 0,7", 2023-06-03T20:18:56.752 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:15:27.014814+0000", 2023-06-03T20:18:56.752 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 8.2237999999999994e-05 2023-06-03T20:18:56.752 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:18:56.752 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:18:56.753 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "op_commit", 2023-06-03T20:18:56.753 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:15:27.027500+0000", 2023-06-03T20:18:56.753 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.012686011000000001 2023-06-03T20:18:56.753 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:18:56.753 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:18:56.753 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:18:56.753 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:15:27.027615+0000", 2023-06-03T20:18:56.753 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.00011421099999999999 2023-06-03T20:18:56.754 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:18:56.754 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:18:56.754 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:18:56.754 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:15:27.042591+0000", 2023-06-03T20:18:56.754 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.014976893999999999 2023-06-03T20:18:56.754 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:18:56.754 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:18:56.755 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "commit_sent", 2023-06-03T20:18:56.755 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:15:27.042627+0000", 2023-06-03T20:18:56.755 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 3.5438999999999997e-05 2023-06-03T20:18:56.755 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:18:56.755 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:18:56.755 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "done", 2023-06-03T20:18:56.755 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:15:27.042661+0000", 2023-06-03T20:18:56.755 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 3.4542000000000002e-05 2023-06-03T20:18:56.756 INFO:teuthology.orchestra.run.smithi089.stdout: } 2023-06-03T20:18:56.756 INFO:teuthology.orchestra.run.smithi089.stdout: ] 2023-06-03T20:18:56.756 INFO:teuthology.orchestra.run.smithi089.stdout: } 2023-06-03T20:18:56.756 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:18:56.756 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:18:56.756 INFO:teuthology.orchestra.run.smithi089.stdout: "description": "osd_op(client.46187.0:207889 11.0 11:0ab182e7:::benchmark_data_smithi089_165412_object102361:head [delete] snapc 0=[] ondisk+write+known_if_redirected e206)", 2023-06-03T20:18:56.756 INFO:teuthology.orchestra.run.smithi089.stdout: "initiated_at": "2023-06-03T20:15:26.114929+0000", 2023-06-03T20:18:56.757 INFO:teuthology.orchestra.run.smithi089.stdout: "age": 210.471685522, 2023-06-03T20:18:56.757 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.92786148400000001, 2023-06-03T20:18:56.757 INFO:teuthology.orchestra.run.smithi089.stdout: "type_data": { 2023-06-03T20:18:56.757 INFO:teuthology.orchestra.run.smithi089.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-03T20:18:56.757 INFO:teuthology.orchestra.run.smithi089.stdout: "client_info": { 2023-06-03T20:18:56.757 INFO:teuthology.orchestra.run.smithi089.stdout: "client": "client.46187", 2023-06-03T20:18:56.757 INFO:teuthology.orchestra.run.smithi089.stdout: "client_addr": "172.21.15.89:0/880684210", 2023-06-03T20:18:56.757 INFO:teuthology.orchestra.run.smithi089.stdout: "tid": 207889 2023-06-03T20:18:56.758 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:18:56.758 INFO:teuthology.orchestra.run.smithi089.stdout: "events": [ 2023-06-03T20:18:56.758 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:18:56.758 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "initiated", 2023-06-03T20:18:56.758 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:15:26.114929+0000", 2023-06-03T20:18:56.758 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0 2023-06-03T20:18:56.758 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:18:56.758 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:18:56.759 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "throttled", 2023-06-03T20:18:56.759 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:15:26.114929+0000", 2023-06-03T20:18:56.759 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0 2023-06-03T20:18:56.759 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:18:56.759 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:18:56.759 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "header_read", 2023-06-03T20:18:56.759 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:15:26.114928+0000", 2023-06-03T20:18:56.759 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 4294967295.999999 2023-06-03T20:18:56.760 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:18:56.760 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:18:56.760 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "all_read", 2023-06-03T20:18:56.760 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:15:26.114935+0000", 2023-06-03T20:18:56.760 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 6.4969999999999998e-06 2023-06-03T20:18:56.760 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:18:56.760 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:18:56.761 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "dispatched", 2023-06-03T20:18:56.761 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:15:26.114944+0000", 2023-06-03T20:18:56.761 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 9.4779999999999997e-06 2023-06-03T20:18:56.761 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:18:56.761 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:18:56.761 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "queued_for_pg", 2023-06-03T20:18:56.761 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:15:26.114954+0000", 2023-06-03T20:18:56.761 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 9.5990000000000008e-06 2023-06-03T20:18:56.762 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:18:56.762 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:18:56.762 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "reached_pg", 2023-06-03T20:18:56.762 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:15:26.115012+0000", 2023-06-03T20:18:56.762 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 5.8452999999999998e-05 2023-06-03T20:18:56.762 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:18:56.762 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:18:56.762 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "waiting for peered", 2023-06-03T20:18:56.763 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:15:26.115025+0000", 2023-06-03T20:18:56.763 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 1.2359e-05 2023-06-03T20:18:56.763 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:18:56.763 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:18:56.763 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "reached_pg", 2023-06-03T20:18:56.763 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:15:27.017437+0000", 2023-06-03T20:18:56.763 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.90241278199999997 2023-06-03T20:18:56.763 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:18:56.764 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:18:56.764 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "started", 2023-06-03T20:18:56.764 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:15:27.017521+0000", 2023-06-03T20:18:56.764 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 8.3844999999999994e-05 2023-06-03T20:18:56.764 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:18:56.764 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:18:56.764 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "waiting for subops from 0,7", 2023-06-03T20:18:56.765 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:15:27.017601+0000", 2023-06-03T20:18:56.765 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 7.9478e-05 2023-06-03T20:18:56.765 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:18:56.765 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:18:56.765 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:18:56.765 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:15:27.027673+0000", 2023-06-03T20:18:56.765 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.010071762 2023-06-03T20:18:56.766 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:18:56.766 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:18:56.766 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "op_commit", 2023-06-03T20:18:56.766 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:15:27.027682+0000", 2023-06-03T20:18:56.766 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 9.7569999999999999e-06 2023-06-03T20:18:56.766 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:18:56.766 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:18:56.766 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:18:56.767 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:15:27.042723+0000", 2023-06-03T20:18:56.767 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.015040278000000001 2023-06-03T20:18:56.767 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:18:56.767 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:18:56.767 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "commit_sent", 2023-06-03T20:18:56.767 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:15:27.042760+0000", 2023-06-03T20:18:56.767 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 3.7379999999999998e-05 2023-06-03T20:18:56.768 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:18:56.768 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:18:56.768 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "done", 2023-06-03T20:18:56.768 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:15:27.042791+0000", 2023-06-03T20:18:56.768 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 3.0836000000000001e-05 2023-06-03T20:18:56.768 INFO:teuthology.orchestra.run.smithi089.stdout: } 2023-06-03T20:18:56.768 INFO:teuthology.orchestra.run.smithi089.stdout: ] 2023-06-03T20:18:56.769 INFO:teuthology.orchestra.run.smithi089.stdout: } 2023-06-03T20:18:56.769 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:18:56.769 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:18:56.769 INFO:teuthology.orchestra.run.smithi089.stdout: "description": "osd_op(client.46187.0:207922 11.0 11:0346b820:::benchmark_data_smithi089_165412_object102394:head [delete] snapc 0=[] ondisk+write+known_if_redirected e206)", 2023-06-03T20:18:56.769 INFO:teuthology.orchestra.run.smithi089.stdout: "initiated_at": "2023-06-03T20:15:26.160587+0000", 2023-06-03T20:18:56.769 INFO:teuthology.orchestra.run.smithi089.stdout: "age": 210.42602763900001, 2023-06-03T20:18:56.769 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.88230225600000001, 2023-06-03T20:18:56.769 INFO:teuthology.orchestra.run.smithi089.stdout: "type_data": { 2023-06-03T20:18:56.770 INFO:teuthology.orchestra.run.smithi089.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-03T20:18:56.770 INFO:teuthology.orchestra.run.smithi089.stdout: "client_info": { 2023-06-03T20:18:56.770 INFO:teuthology.orchestra.run.smithi089.stdout: "client": "client.46187", 2023-06-03T20:18:56.770 INFO:teuthology.orchestra.run.smithi089.stdout: "client_addr": "172.21.15.89:0/880684210", 2023-06-03T20:18:56.770 INFO:teuthology.orchestra.run.smithi089.stdout: "tid": 207922 2023-06-03T20:18:56.770 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:18:56.770 INFO:teuthology.orchestra.run.smithi089.stdout: "events": [ 2023-06-03T20:18:56.770 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:18:56.771 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "initiated", 2023-06-03T20:18:56.771 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:15:26.160587+0000", 2023-06-03T20:18:56.771 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0 2023-06-03T20:18:56.771 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:18:56.771 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:18:56.771 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "throttled", 2023-06-03T20:18:56.772 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:15:26.160587+0000", 2023-06-03T20:18:56.772 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0 2023-06-03T20:18:56.772 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:18:56.772 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:18:56.772 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "header_read", 2023-06-03T20:18:56.773 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:15:26.160586+0000", 2023-06-03T20:18:56.773 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 4294967295.9999986 2023-06-03T20:18:56.773 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:18:56.773 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:18:56.773 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "all_read", 2023-06-03T20:18:56.773 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:15:26.160602+0000", 2023-06-03T20:18:56.773 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 1.6376999999999999e-05 2023-06-03T20:18:56.773 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:18:56.774 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:18:56.774 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "dispatched", 2023-06-03T20:18:56.774 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:15:26.160612+0000", 2023-06-03T20:18:56.774 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 9.6930000000000002e-06 2023-06-03T20:18:56.774 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:18:56.774 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:18:56.774 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "queued_for_pg", 2023-06-03T20:18:56.774 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:15:26.160619+0000", 2023-06-03T20:18:56.775 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 6.8140000000000003e-06 2023-06-03T20:18:56.775 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:18:56.775 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:18:56.775 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "reached_pg", 2023-06-03T20:18:56.775 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:15:26.160663+0000", 2023-06-03T20:18:56.775 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 4.4084e-05 2023-06-03T20:18:56.775 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:18:56.776 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:18:56.776 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "waiting for peered", 2023-06-03T20:18:56.776 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:15:26.160676+0000", 2023-06-03T20:18:56.776 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 1.3475000000000001e-05 2023-06-03T20:18:56.776 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:18:56.776 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:18:56.776 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "reached_pg", 2023-06-03T20:18:56.777 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:15:27.020592+0000", 2023-06-03T20:18:56.777 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.85991571200000005 2023-06-03T20:18:56.777 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:18:56.777 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:18:56.777 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "started", 2023-06-03T20:18:56.777 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:15:27.020732+0000", 2023-06-03T20:18:56.777 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.000140288 2023-06-03T20:18:56.778 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:18:56.778 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:18:56.778 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "waiting for subops from 0,7", 2023-06-03T20:18:56.778 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:15:27.020862+0000", 2023-06-03T20:18:56.778 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.00013021500000000001 2023-06-03T20:18:56.778 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:18:56.778 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:18:56.778 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "op_commit", 2023-06-03T20:18:56.779 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:15:27.027689+0000", 2023-06-03T20:18:56.779 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.0068270079999999999 2023-06-03T20:18:56.779 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:18:56.779 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:18:56.779 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:18:56.780 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:15:27.035008+0000", 2023-06-03T20:18:56.780 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.0073185230000000004 2023-06-03T20:18:56.780 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:18:56.780 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:18:56.780 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:18:56.780 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:15:27.042825+0000", 2023-06-03T20:18:56.780 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.0078167889999999993 2023-06-03T20:18:56.781 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:18:56.781 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:18:56.781 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "commit_sent", 2023-06-03T20:18:56.781 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:15:27.042858+0000", 2023-06-03T20:18:56.781 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 3.3463000000000001e-05 2023-06-03T20:18:56.781 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:18:56.781 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:18:56.781 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "done", 2023-06-03T20:18:56.782 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:15:27.042889+0000", 2023-06-03T20:18:56.782 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 3.1322999999999998e-05 2023-06-03T20:18:56.782 INFO:teuthology.orchestra.run.smithi089.stdout: } 2023-06-03T20:18:56.782 INFO:teuthology.orchestra.run.smithi089.stdout: ] 2023-06-03T20:18:56.782 INFO:teuthology.orchestra.run.smithi089.stdout: } 2023-06-03T20:18:56.782 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:18:56.782 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:18:56.782 INFO:teuthology.orchestra.run.smithi089.stdout: "description": "osd_op(client.46187.0:207960 11.0 11:0725ba7b:::benchmark_data_smithi089_165412_object102432:head [delete] snapc 0=[] ondisk+write+known_if_redirected e206)", 2023-06-03T20:18:56.783 INFO:teuthology.orchestra.run.smithi089.stdout: "initiated_at": "2023-06-03T20:15:26.219256+0000", 2023-06-03T20:18:56.783 INFO:teuthology.orchestra.run.smithi089.stdout: "age": 210.367358798, 2023-06-03T20:18:56.783 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.82374393099999998, 2023-06-03T20:18:56.783 INFO:teuthology.orchestra.run.smithi089.stdout: "type_data": { 2023-06-03T20:18:56.783 INFO:teuthology.orchestra.run.smithi089.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-03T20:18:56.783 INFO:teuthology.orchestra.run.smithi089.stdout: "client_info": { 2023-06-03T20:18:56.783 INFO:teuthology.orchestra.run.smithi089.stdout: "client": "client.46187", 2023-06-03T20:18:56.783 INFO:teuthology.orchestra.run.smithi089.stdout: "client_addr": "172.21.15.89:0/880684210", 2023-06-03T20:18:56.784 INFO:teuthology.orchestra.run.smithi089.stdout: "tid": 207960 2023-06-03T20:18:56.784 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:18:56.784 INFO:teuthology.orchestra.run.smithi089.stdout: "events": [ 2023-06-03T20:18:56.784 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:18:56.784 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "initiated", 2023-06-03T20:18:56.784 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:15:26.219256+0000", 2023-06-03T20:18:56.785 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0 2023-06-03T20:18:56.785 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:18:56.785 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:18:56.785 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "throttled", 2023-06-03T20:18:56.785 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:15:26.219256+0000", 2023-06-03T20:18:56.785 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0 2023-06-03T20:18:56.785 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:18:56.785 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:18:56.786 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "header_read", 2023-06-03T20:18:56.786 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:15:26.219255+0000", 2023-06-03T20:18:56.786 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 4294967295.9999986 2023-06-03T20:18:56.786 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:18:56.786 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:18:56.786 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "all_read", 2023-06-03T20:18:56.786 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:15:26.219261+0000", 2023-06-03T20:18:56.787 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 6.0009999999999998e-06 2023-06-03T20:18:56.787 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:18:56.787 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:18:56.787 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "dispatched", 2023-06-03T20:18:56.787 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:15:26.219271+0000", 2023-06-03T20:18:56.787 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 1.0274e-05 2023-06-03T20:18:56.787 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:18:56.787 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:18:56.788 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "queued_for_pg", 2023-06-03T20:18:56.788 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:15:26.219277+0000", 2023-06-03T20:18:56.788 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 5.8039999999999998e-06 2023-06-03T20:18:56.788 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:18:56.788 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:18:56.788 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "reached_pg", 2023-06-03T20:18:56.788 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:15:26.219338+0000", 2023-06-03T20:18:56.789 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 6.0943999999999998e-05 2023-06-03T20:18:56.789 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:18:56.789 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:18:56.789 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "waiting for peered", 2023-06-03T20:18:56.789 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:15:26.219351+0000", 2023-06-03T20:18:56.789 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 1.3828999999999999e-05 2023-06-03T20:18:56.789 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:18:56.789 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:18:56.790 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "reached_pg", 2023-06-03T20:18:56.790 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:15:27.024117+0000", 2023-06-03T20:18:56.790 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.80476597500000002 2023-06-03T20:18:56.790 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:18:56.790 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:18:56.790 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "started", 2023-06-03T20:18:56.790 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:15:27.024296+0000", 2023-06-03T20:18:56.790 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.000178593 2023-06-03T20:18:56.791 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:18:56.791 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:18:56.791 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "waiting for subops from 0,7", 2023-06-03T20:18:56.791 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:15:27.024495+0000", 2023-06-03T20:18:56.791 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.000198786 2023-06-03T20:18:56.791 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:18:56.791 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:18:56.792 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "op_commit", 2023-06-03T20:18:56.792 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:15:27.034969+0000", 2023-06-03T20:18:56.792 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.010474641999999999 2023-06-03T20:18:56.792 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:18:56.792 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:18:56.792 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:18:56.792 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:15:27.035043+0000", 2023-06-03T20:18:56.792 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 7.3569e-05 2023-06-03T20:18:56.793 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:18:56.793 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:18:56.793 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:18:56.793 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:15:27.042935+0000", 2023-06-03T20:18:56.793 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.0078917669999999992 2023-06-03T20:18:56.793 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:18:56.793 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:18:56.793 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "commit_sent", 2023-06-03T20:18:56.794 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:15:27.042969+0000", 2023-06-03T20:18:56.794 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 3.3680000000000003e-05 2023-06-03T20:18:56.794 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:18:56.794 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:18:56.794 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "done", 2023-06-03T20:18:56.794 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:15:27.043000+0000", 2023-06-03T20:18:56.794 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 3.1494e-05 2023-06-03T20:18:56.795 INFO:teuthology.orchestra.run.smithi089.stdout: } 2023-06-03T20:18:56.795 INFO:teuthology.orchestra.run.smithi089.stdout: ] 2023-06-03T20:18:56.795 INFO:teuthology.orchestra.run.smithi089.stdout: } 2023-06-03T20:18:56.795 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:18:56.795 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:18:56.795 INFO:teuthology.orchestra.run.smithi089.stdout: "description": "osd_op(client.46187.0:207961 11.0 11:079d2404:::benchmark_data_smithi089_165412_object102433:head [delete] snapc 0=[] ondisk+write+known_if_redirected e206)", 2023-06-03T20:18:56.795 INFO:teuthology.orchestra.run.smithi089.stdout: "initiated_at": "2023-06-03T20:15:26.219597+0000", 2023-06-03T20:18:56.795 INFO:teuthology.orchestra.run.smithi089.stdout: "age": 210.36701815500001, 2023-06-03T20:18:56.796 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.83177192799999999, 2023-06-03T20:18:56.796 INFO:teuthology.orchestra.run.smithi089.stdout: "type_data": { 2023-06-03T20:18:56.796 INFO:teuthology.orchestra.run.smithi089.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-03T20:18:56.796 INFO:teuthology.orchestra.run.smithi089.stdout: "client_info": { 2023-06-03T20:18:56.796 INFO:teuthology.orchestra.run.smithi089.stdout: "client": "client.46187", 2023-06-03T20:18:56.796 INFO:teuthology.orchestra.run.smithi089.stdout: "client_addr": "172.21.15.89:0/880684210", 2023-06-03T20:18:56.796 INFO:teuthology.orchestra.run.smithi089.stdout: "tid": 207961 2023-06-03T20:18:56.797 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:18:56.797 INFO:teuthology.orchestra.run.smithi089.stdout: "events": [ 2023-06-03T20:18:56.797 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:18:56.797 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "initiated", 2023-06-03T20:18:56.797 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:15:26.219597+0000", 2023-06-03T20:18:56.797 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0 2023-06-03T20:18:56.797 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:18:56.797 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:18:56.798 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "throttled", 2023-06-03T20:18:56.798 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:15:26.219597+0000", 2023-06-03T20:18:56.798 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0 2023-06-03T20:18:56.798 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:18:56.798 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:18:56.799 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "header_read", 2023-06-03T20:18:56.799 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:15:26.219596+0000", 2023-06-03T20:18:56.799 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 4294967295.9999995 2023-06-03T20:18:56.799 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:18:56.800 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:18:56.800 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "all_read", 2023-06-03T20:18:56.800 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:15:26.219600+0000", 2023-06-03T20:18:56.800 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 3.946e-06 2023-06-03T20:18:56.801 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:18:56.801 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:18:56.801 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "dispatched", 2023-06-03T20:18:56.801 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:15:26.219607+0000", 2023-06-03T20:18:56.801 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 6.956e-06 2023-06-03T20:18:56.801 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:18:56.801 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:18:56.802 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "queued_for_pg", 2023-06-03T20:18:56.802 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:15:26.219612+0000", 2023-06-03T20:18:56.802 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 4.5680000000000001e-06 2023-06-03T20:18:56.802 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:18:56.802 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:18:56.802 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "reached_pg", 2023-06-03T20:18:56.802 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:15:26.219641+0000", 2023-06-03T20:18:56.803 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 2.9881000000000001e-05 2023-06-03T20:18:56.803 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:18:56.803 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:18:56.803 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "waiting for peered", 2023-06-03T20:18:56.803 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:15:26.219647+0000", 2023-06-03T20:18:56.803 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 5.5690000000000004e-06 2023-06-03T20:18:56.803 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:18:56.803 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:18:56.804 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "reached_pg", 2023-06-03T20:18:56.804 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:15:27.027711+0000", 2023-06-03T20:18:56.804 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.80806405299999995 2023-06-03T20:18:56.804 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:18:56.804 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:18:56.804 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "started", 2023-06-03T20:18:56.804 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:15:27.027829+0000", 2023-06-03T20:18:56.805 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.00011808599999999999 2023-06-03T20:18:56.805 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:18:56.805 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:18:56.805 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "waiting for subops from 0,7", 2023-06-03T20:18:56.805 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:15:27.027933+0000", 2023-06-03T20:18:56.805 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.00010414 2023-06-03T20:18:56.806 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:18:56.806 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:18:56.806 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "op_commit", 2023-06-03T20:18:56.806 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:15:27.035014+0000", 2023-06-03T20:18:56.806 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.0070811010000000002 2023-06-03T20:18:56.806 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:18:56.806 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:18:56.806 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:18:56.807 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:15:27.035073+0000", 2023-06-03T20:18:56.807 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 5.8375999999999998e-05 2023-06-03T20:18:56.807 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:18:56.807 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:18:56.807 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:18:56.807 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:15:27.051226+0000", 2023-06-03T20:18:56.807 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.016153758000000001 2023-06-03T20:18:56.807 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:18:56.808 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:18:56.808 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "commit_sent", 2023-06-03T20:18:56.808 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:15:27.051325+0000", 2023-06-03T20:18:56.808 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 9.8156e-05 2023-06-03T20:18:56.808 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:18:56.808 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:18:56.808 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "done", 2023-06-03T20:18:56.809 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:15:27.051369+0000", 2023-06-03T20:18:56.809 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 4.4023000000000002e-05 2023-06-03T20:18:56.809 INFO:teuthology.orchestra.run.smithi089.stdout: } 2023-06-03T20:18:56.809 INFO:teuthology.orchestra.run.smithi089.stdout: ] 2023-06-03T20:18:56.809 INFO:teuthology.orchestra.run.smithi089.stdout: } 2023-06-03T20:18:56.809 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:18:56.810 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:18:56.810 INFO:teuthology.orchestra.run.smithi089.stdout: "description": "osd_op(client.46187.0:207969 11.0 11:0dfc9717:::benchmark_data_smithi089_165412_object102441:head [delete] snapc 0=[] ondisk+write+known_if_redirected e206)", 2023-06-03T20:18:56.810 INFO:teuthology.orchestra.run.smithi089.stdout: "initiated_at": "2023-06-03T20:15:26.237796+0000", 2023-06-03T20:18:56.810 INFO:teuthology.orchestra.run.smithi089.stdout: "age": 210.34881894099999, 2023-06-03T20:18:56.810 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.81369875199999997, 2023-06-03T20:18:56.810 INFO:teuthology.orchestra.run.smithi089.stdout: "type_data": { 2023-06-03T20:18:56.810 INFO:teuthology.orchestra.run.smithi089.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-03T20:18:56.810 INFO:teuthology.orchestra.run.smithi089.stdout: "client_info": { 2023-06-03T20:18:56.811 INFO:teuthology.orchestra.run.smithi089.stdout: "client": "client.46187", 2023-06-03T20:18:56.811 INFO:teuthology.orchestra.run.smithi089.stdout: "client_addr": "172.21.15.89:0/880684210", 2023-06-03T20:18:56.811 INFO:teuthology.orchestra.run.smithi089.stdout: "tid": 207969 2023-06-03T20:18:56.811 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:18:56.811 INFO:teuthology.orchestra.run.smithi089.stdout: "events": [ 2023-06-03T20:18:56.811 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:18:56.811 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "initiated", 2023-06-03T20:18:56.812 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:15:26.237796+0000", 2023-06-03T20:18:56.812 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0 2023-06-03T20:18:56.812 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:18:56.812 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:18:56.812 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "throttled", 2023-06-03T20:18:56.812 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:15:26.237796+0000", 2023-06-03T20:18:56.812 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0 2023-06-03T20:18:56.813 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:18:56.813 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:18:56.813 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "header_read", 2023-06-03T20:18:56.813 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:15:26.237795+0000", 2023-06-03T20:18:56.813 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 4294967295.9999986 2023-06-03T20:18:56.813 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:18:56.813 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:18:56.813 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "all_read", 2023-06-03T20:18:56.814 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:15:26.237801+0000", 2023-06-03T20:18:56.814 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 6.1140000000000002e-06 2023-06-03T20:18:56.814 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:18:56.814 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:18:56.814 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "dispatched", 2023-06-03T20:18:56.814 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:15:26.237811+0000", 2023-06-03T20:18:56.814 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 1.0359e-05 2023-06-03T20:18:56.814 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:18:56.815 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:18:56.815 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "queued_for_pg", 2023-06-03T20:18:56.815 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:15:26.237817+0000", 2023-06-03T20:18:56.815 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 5.8560000000000003e-06 2023-06-03T20:18:56.815 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:18:56.815 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:18:56.815 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "reached_pg", 2023-06-03T20:18:56.815 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:15:26.237862+0000", 2023-06-03T20:18:56.816 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 4.4993000000000001e-05 2023-06-03T20:18:56.816 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:18:56.816 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:18:56.816 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "waiting for peered", 2023-06-03T20:18:56.816 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:15:26.237873+0000", 2023-06-03T20:18:56.816 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 1.1392999999999999e-05 2023-06-03T20:18:56.816 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:18:56.817 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:18:56.817 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "reached_pg", 2023-06-03T20:18:56.817 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:15:27.031294+0000", 2023-06-03T20:18:56.817 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.79342085399999995 2023-06-03T20:18:56.817 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:18:56.817 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:18:56.817 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "started", 2023-06-03T20:18:56.817 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:15:27.031421+0000", 2023-06-03T20:18:56.818 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.00012627999999999999 2023-06-03T20:18:56.818 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:18:56.818 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:18:56.818 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "waiting for subops from 0,7", 2023-06-03T20:18:56.818 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:15:27.031531+0000", 2023-06-03T20:18:56.818 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.000110682 2023-06-03T20:18:56.818 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:18:56.819 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:18:56.819 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "op_commit", 2023-06-03T20:18:56.819 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:15:27.042249+0000", 2023-06-03T20:18:56.819 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.010718243000000001 2023-06-03T20:18:56.819 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:18:56.820 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:18:56.820 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:18:56.820 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:15:27.042422+0000", 2023-06-03T20:18:56.820 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.000172459 2023-06-03T20:18:56.821 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:18:56.821 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:18:56.821 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:18:56.821 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:15:27.051420+0000", 2023-06-03T20:18:56.822 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.0089985329999999995 2023-06-03T20:18:56.822 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:18:56.822 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:18:56.823 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "commit_sent", 2023-06-03T20:18:56.823 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:15:27.051461+0000", 2023-06-03T20:18:56.823 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 4.0127999999999999e-05 2023-06-03T20:18:56.823 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:18:56.824 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:18:56.824 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "done", 2023-06-03T20:18:56.824 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:15:27.051495+0000", 2023-06-03T20:18:56.824 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 3.4094999999999998e-05 2023-06-03T20:18:56.825 INFO:teuthology.orchestra.run.smithi089.stdout: } 2023-06-03T20:18:56.825 INFO:teuthology.orchestra.run.smithi089.stdout: ] 2023-06-03T20:18:56.825 INFO:teuthology.orchestra.run.smithi089.stdout: } 2023-06-03T20:18:56.825 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:18:56.825 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:18:56.825 INFO:teuthology.orchestra.run.smithi089.stdout: "description": "osd_op(client.46187.0:207970 11.0 11:0928d0d2:::benchmark_data_smithi089_165412_object102442:head [delete] snapc 0=[] ondisk+write+known_if_redirected e206)", 2023-06-03T20:18:56.825 INFO:teuthology.orchestra.run.smithi089.stdout: "initiated_at": "2023-06-03T20:15:26.238175+0000", 2023-06-03T20:18:56.826 INFO:teuthology.orchestra.run.smithi089.stdout: "age": 210.348439958, 2023-06-03T20:18:56.826 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.81344859899999999, 2023-06-03T20:18:56.826 INFO:teuthology.orchestra.run.smithi089.stdout: "type_data": { 2023-06-03T20:18:56.826 INFO:teuthology.orchestra.run.smithi089.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-03T20:18:56.826 INFO:teuthology.orchestra.run.smithi089.stdout: "client_info": { 2023-06-03T20:18:56.826 INFO:teuthology.orchestra.run.smithi089.stdout: "client": "client.46187", 2023-06-03T20:18:56.826 INFO:teuthology.orchestra.run.smithi089.stdout: "client_addr": "172.21.15.89:0/880684210", 2023-06-03T20:18:56.827 INFO:teuthology.orchestra.run.smithi089.stdout: "tid": 207970 2023-06-03T20:18:56.827 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:18:56.827 INFO:teuthology.orchestra.run.smithi089.stdout: "events": [ 2023-06-03T20:18:56.827 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:18:56.827 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "initiated", 2023-06-03T20:18:56.827 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:15:26.238175+0000", 2023-06-03T20:18:56.827 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0 2023-06-03T20:18:56.827 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:18:56.828 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:18:56.828 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "throttled", 2023-06-03T20:18:56.828 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:15:26.238175+0000", 2023-06-03T20:18:56.828 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0 2023-06-03T20:18:56.828 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:18:56.828 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:18:56.828 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "header_read", 2023-06-03T20:18:56.829 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:15:26.238174+0000", 2023-06-03T20:18:56.829 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 4294967295.999999 2023-06-03T20:18:56.829 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:18:56.829 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:18:56.829 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "all_read", 2023-06-03T20:18:56.829 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:15:26.238179+0000", 2023-06-03T20:18:56.829 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 5.1050000000000003e-06 2023-06-03T20:18:56.829 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:18:56.830 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:18:56.830 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "dispatched", 2023-06-03T20:18:56.830 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:15:26.238189+0000", 2023-06-03T20:18:56.830 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 9.6690000000000001e-06 2023-06-03T20:18:56.830 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:18:56.830 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:18:56.830 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "queued_for_pg", 2023-06-03T20:18:56.830 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:15:26.238195+0000", 2023-06-03T20:18:56.831 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 6.737e-06 2023-06-03T20:18:56.831 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:18:56.831 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:18:56.831 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "reached_pg", 2023-06-03T20:18:56.831 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:15:26.238231+0000", 2023-06-03T20:18:56.831 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 3.5358999999999998e-05 2023-06-03T20:18:56.831 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:18:56.831 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:18:56.832 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "waiting for peered", 2023-06-03T20:18:56.832 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:15:26.238239+0000", 2023-06-03T20:18:56.832 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 7.994e-06 2023-06-03T20:18:56.832 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:18:56.832 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:18:56.832 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "reached_pg", 2023-06-03T20:18:56.832 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:15:27.035093+0000", 2023-06-03T20:18:56.833 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.79685445799999999 2023-06-03T20:18:56.833 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:18:56.833 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:18:56.833 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "started", 2023-06-03T20:18:56.833 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:15:27.035205+0000", 2023-06-03T20:18:56.833 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.000111803 2023-06-03T20:18:56.833 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:18:56.834 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:18:56.834 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "waiting for subops from 0,7", 2023-06-03T20:18:56.834 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:15:27.035320+0000", 2023-06-03T20:18:56.834 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.000115084 2023-06-03T20:18:56.834 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:18:56.834 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:18:56.834 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:18:56.834 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:15:27.042559+0000", 2023-06-03T20:18:56.835 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.0072386129999999996 2023-06-03T20:18:56.835 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:18:56.835 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:18:56.835 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:18:56.835 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:15:27.051547+0000", 2023-06-03T20:18:56.835 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.0089878879999999994 2023-06-03T20:18:56.835 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:18:56.836 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:18:56.836 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "op_commit", 2023-06-03T20:18:56.836 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:15:27.051556+0000", 2023-06-03T20:18:56.836 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 9.3749999999999992e-06 2023-06-03T20:18:56.836 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:18:56.836 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:18:56.836 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "commit_sent", 2023-06-03T20:18:56.837 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:15:27.051588+0000", 2023-06-03T20:18:56.837 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 3.2048999999999998e-05 2023-06-03T20:18:56.837 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:18:56.837 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:18:56.837 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "done", 2023-06-03T20:18:56.837 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:15:27.051624+0000", 2023-06-03T20:18:56.837 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 3.5398999999999998e-05 2023-06-03T20:18:56.837 INFO:teuthology.orchestra.run.smithi089.stdout: } 2023-06-03T20:18:56.838 INFO:teuthology.orchestra.run.smithi089.stdout: ] 2023-06-03T20:18:56.838 INFO:teuthology.orchestra.run.smithi089.stdout: } 2023-06-03T20:18:56.838 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:18:56.838 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:18:56.838 INFO:teuthology.orchestra.run.smithi089.stdout: "description": "osd_op(client.46187.0:207981 11.0 11:0e4706dc:::benchmark_data_smithi089_165412_object102453:head [delete] snapc 0=[] ondisk+write+known_if_redirected e206)", 2023-06-03T20:18:56.838 INFO:teuthology.orchestra.run.smithi089.stdout: "initiated_at": "2023-06-03T20:15:26.274604+0000", 2023-06-03T20:18:56.838 INFO:teuthology.orchestra.run.smithi089.stdout: "age": 210.31201074699999, 2023-06-03T20:18:56.839 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.77719452700000002, 2023-06-03T20:18:56.839 INFO:teuthology.orchestra.run.smithi089.stdout: "type_data": { 2023-06-03T20:18:56.839 INFO:teuthology.orchestra.run.smithi089.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-03T20:18:56.839 INFO:teuthology.orchestra.run.smithi089.stdout: "client_info": { 2023-06-03T20:18:56.839 INFO:teuthology.orchestra.run.smithi089.stdout: "client": "client.46187", 2023-06-03T20:18:56.839 INFO:teuthology.orchestra.run.smithi089.stdout: "client_addr": "172.21.15.89:0/880684210", 2023-06-03T20:18:56.839 INFO:teuthology.orchestra.run.smithi089.stdout: "tid": 207981 2023-06-03T20:18:56.840 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:18:56.840 INFO:teuthology.orchestra.run.smithi089.stdout: "events": [ 2023-06-03T20:18:56.840 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:18:56.840 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "initiated", 2023-06-03T20:18:56.840 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:15:26.274604+0000", 2023-06-03T20:18:56.840 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0 2023-06-03T20:18:56.840 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:18:56.840 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:18:56.841 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "throttled", 2023-06-03T20:18:56.841 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:15:26.274604+0000", 2023-06-03T20:18:56.841 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0 2023-06-03T20:18:56.841 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:18:56.841 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:18:56.841 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "header_read", 2023-06-03T20:18:56.842 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:15:26.274603+0000", 2023-06-03T20:18:56.842 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 4294967295.999999 2023-06-03T20:18:56.842 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:18:56.842 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:18:56.842 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "all_read", 2023-06-03T20:18:56.842 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:15:26.274608+0000", 2023-06-03T20:18:56.842 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 5.3070000000000002e-06 2023-06-03T20:18:56.842 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:18:56.843 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:18:56.843 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "dispatched", 2023-06-03T20:18:56.843 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:15:26.274617+0000", 2023-06-03T20:18:56.843 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 8.9099999999999994e-06 2023-06-03T20:18:56.843 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:18:56.843 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:18:56.843 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "queued_for_pg", 2023-06-03T20:18:56.843 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:15:26.274623+0000", 2023-06-03T20:18:56.844 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 5.4720000000000003e-06 2023-06-03T20:18:56.844 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:18:56.844 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:18:56.844 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "reached_pg", 2023-06-03T20:18:56.844 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:15:26.274678+0000", 2023-06-03T20:18:56.844 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 5.5031000000000002e-05 2023-06-03T20:18:56.845 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:18:56.845 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:18:56.845 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "waiting for peered", 2023-06-03T20:18:56.845 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:15:26.274690+0000", 2023-06-03T20:18:56.845 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 1.2227e-05 2023-06-03T20:18:56.845 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:18:56.845 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:18:56.846 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "reached_pg", 2023-06-03T20:18:56.846 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:15:27.038684+0000", 2023-06-03T20:18:56.846 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.76399384599999998 2023-06-03T20:18:56.846 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:18:56.846 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:18:56.846 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "started", 2023-06-03T20:18:56.846 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:15:27.038786+0000", 2023-06-03T20:18:56.846 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.00010241799999999999 2023-06-03T20:18:56.847 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:18:56.847 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:18:56.847 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "waiting for subops from 0,7", 2023-06-03T20:18:56.847 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:15:27.038901+0000", 2023-06-03T20:18:56.847 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.000114962 2023-06-03T20:18:56.847 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:18:56.847 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:18:56.847 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:18:56.848 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:15:27.042695+0000", 2023-06-03T20:18:56.848 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.003793388 2023-06-03T20:18:56.848 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:18:56.848 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:18:56.848 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "op_commit", 2023-06-03T20:18:56.848 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:15:27.051632+0000", 2023-06-03T20:18:56.848 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.0089377480000000006 2023-06-03T20:18:56.849 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:18:56.849 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:18:56.849 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:18:56.849 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:15:27.051687+0000", 2023-06-03T20:18:56.849 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 5.4965e-05 2023-06-03T20:18:56.849 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:18:56.849 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:18:56.849 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "commit_sent", 2023-06-03T20:18:56.850 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:15:27.051763+0000", 2023-06-03T20:18:56.850 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 7.5652e-05 2023-06-03T20:18:56.850 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:18:56.850 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:18:56.850 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "done", 2023-06-03T20:18:56.850 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:15:27.051799+0000", 2023-06-03T20:18:56.850 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 3.5714999999999999e-05 2023-06-03T20:18:56.850 INFO:teuthology.orchestra.run.smithi089.stdout: } 2023-06-03T20:18:56.851 INFO:teuthology.orchestra.run.smithi089.stdout: ] 2023-06-03T20:18:56.851 INFO:teuthology.orchestra.run.smithi089.stdout: } 2023-06-03T20:18:56.851 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:18:56.851 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:18:56.851 INFO:teuthology.orchestra.run.smithi089.stdout: "description": "osd_op(client.46187.0:208033 11.0 11:0d76188e:::benchmark_data_smithi089_165412_object102505:head [delete] snapc 0=[] ondisk+write+known_if_redirected e206)", 2023-06-03T20:18:56.851 INFO:teuthology.orchestra.run.smithi089.stdout: "initiated_at": "2023-06-03T20:15:26.611707+0000", 2023-06-03T20:18:56.851 INFO:teuthology.orchestra.run.smithi089.stdout: "age": 209.97490764599999, 2023-06-03T20:18:56.851 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.45519212999999997, 2023-06-03T20:18:56.852 INFO:teuthology.orchestra.run.smithi089.stdout: "type_data": { 2023-06-03T20:18:56.852 INFO:teuthology.orchestra.run.smithi089.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-03T20:18:56.852 INFO:teuthology.orchestra.run.smithi089.stdout: "client_info": { 2023-06-03T20:18:56.852 INFO:teuthology.orchestra.run.smithi089.stdout: "client": "client.46187", 2023-06-03T20:18:56.852 INFO:teuthology.orchestra.run.smithi089.stdout: "client_addr": "172.21.15.89:0/880684210", 2023-06-03T20:18:56.852 INFO:teuthology.orchestra.run.smithi089.stdout: "tid": 208033 2023-06-03T20:18:56.852 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:18:56.852 INFO:teuthology.orchestra.run.smithi089.stdout: "events": [ 2023-06-03T20:18:56.853 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:18:56.853 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "initiated", 2023-06-03T20:18:56.853 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:15:26.611707+0000", 2023-06-03T20:18:56.853 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0 2023-06-03T20:18:56.853 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:18:56.853 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:18:56.853 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "throttled", 2023-06-03T20:18:56.854 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:15:26.611707+0000", 2023-06-03T20:18:56.854 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0 2023-06-03T20:18:56.854 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:18:56.854 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:18:56.854 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "header_read", 2023-06-03T20:18:56.854 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:15:26.611705+0000", 2023-06-03T20:18:56.854 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 4294967295.9999976 2023-06-03T20:18:56.854 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:18:56.855 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:18:56.855 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "all_read", 2023-06-03T20:18:56.855 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:15:26.611717+0000", 2023-06-03T20:18:56.855 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 1.2431e-05 2023-06-03T20:18:56.855 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:18:56.855 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:18:56.855 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "dispatched", 2023-06-03T20:18:56.856 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:15:26.611745+0000", 2023-06-03T20:18:56.856 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 2.7625000000000001e-05 2023-06-03T20:18:56.856 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:18:56.856 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:18:56.856 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "queued_for_pg", 2023-06-03T20:18:56.856 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:15:26.611759+0000", 2023-06-03T20:18:56.856 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 1.4403000000000001e-05 2023-06-03T20:18:56.857 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:18:56.857 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:18:56.857 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "reached_pg", 2023-06-03T20:18:56.857 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:15:26.611819+0000", 2023-06-03T20:18:56.857 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 5.9811e-05 2023-06-03T20:18:56.857 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:18:56.857 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:18:56.857 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "waiting for peered", 2023-06-03T20:18:56.858 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:15:26.611829+0000", 2023-06-03T20:18:56.858 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 1.0135999999999999e-05 2023-06-03T20:18:56.858 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:18:56.858 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:18:56.858 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "reached_pg", 2023-06-03T20:18:56.858 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:15:27.043024+0000", 2023-06-03T20:18:56.858 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.431194823 2023-06-03T20:18:56.858 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:18:56.859 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:18:56.859 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "started", 2023-06-03T20:18:56.859 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:15:27.043128+0000", 2023-06-03T20:18:56.859 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.000103906 2023-06-03T20:18:56.859 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:18:56.859 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:18:56.859 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "waiting for subops from 0,7", 2023-06-03T20:18:56.860 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:15:27.043226+0000", 2023-06-03T20:18:56.860 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 9.7837999999999994e-05 2023-06-03T20:18:56.860 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:18:56.860 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:18:56.860 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "op_commit", 2023-06-03T20:18:56.860 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:15:27.051639+0000", 2023-06-03T20:18:56.860 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.0084136130000000003 2023-06-03T20:18:56.860 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:18:56.861 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:18:56.861 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:18:56.861 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:15:27.066822+0000", 2023-06-03T20:18:56.861 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.015183139999999999 2023-06-03T20:18:56.861 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:18:56.861 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:18:56.861 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:18:56.861 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:15:27.066842+0000", 2023-06-03T20:18:56.862 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 1.9126e-05 2023-06-03T20:18:56.862 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:18:56.862 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:18:56.862 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "commit_sent", 2023-06-03T20:18:56.862 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:15:27.066874+0000", 2023-06-03T20:18:56.862 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 3.2361999999999999e-05 2023-06-03T20:18:56.862 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:18:56.862 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:18:56.863 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "done", 2023-06-03T20:18:56.863 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:15:27.066899+0000", 2023-06-03T20:18:56.863 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 2.5476000000000001e-05 2023-06-03T20:18:56.863 INFO:teuthology.orchestra.run.smithi089.stdout: } 2023-06-03T20:18:56.863 INFO:teuthology.orchestra.run.smithi089.stdout: ] 2023-06-03T20:18:56.863 INFO:teuthology.orchestra.run.smithi089.stdout: } 2023-06-03T20:18:56.863 INFO:teuthology.orchestra.run.smithi089.stdout: } 2023-06-03T20:18:56.863 INFO:teuthology.orchestra.run.smithi089.stdout: ] 2023-06-03T20:18:56.864 INFO:teuthology.orchestra.run.smithi089.stdout:} 2023-06-03T20:18:56.864 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:18:56.905 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:18:56.907 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:18:56.908 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:18:57.009 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:18:57.043 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:18:57.046 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:18:57.047 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:18:57.148 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:18:57.188 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:18:57.192 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:18:57.193 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:18:57.294 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:18:57.326 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:18:57.332 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:18:57.333 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:18:57.434 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:18:57.468 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:18:57.470 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:18:57.470 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:18:57.571 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:18:57.597 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:18:57.598 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:18:57.599 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:18:57.700 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:18:57.730 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:18:57.734 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:18:57.735 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:18:57.836 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:18:57.863 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:18:57.865 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:18:57.866 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:18:57.967 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:18:57.997 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:18:58.000 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:18:58.001 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:18:58.084 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:18:57 smithi164 bash[59462]: cluster 1685823536.8902502 mgr.y (mgr.34104) 1288 : cluster 0 pgmap v1284: 89 pgs: 20 active+clean+remapped, 14 active+undersized+degraded, 11 active+undersized, 44 active+clean; 6.1 GiB data, 19 GiB used, 518 GiB / 536 GiB avail; 85 B/s rd, 21 MiB/s wr, 848 op/s; 50004/298377 objects degraded (16.759%); 6/298377 objects misplaced (0.002%) 2023-06-03T20:18:58.084 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:18:57 smithi164 bash[59462]: cluster 1685823537.8424644 mon.a (mon.0) 1226 : cluster 3 Health check update: Degraded data redundancy: 50004/298377 objects degraded (16.759%), 14 pgs degraded, 25 pgs undersized (PG_DEGRADED) 2023-06-03T20:18:58.101 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:18:58.136 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:18:58.139 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:18:58.139 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:18:58.241 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:18:58.274 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:18:58.281 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:18:58.282 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:18:58.382 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:18:58.414 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:18:58.417 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:18:58.418 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:18:58.519 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:18:58.547 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:18:58.553 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:18:58.553 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:18:58.654 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:18:58.690 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:18:58.692 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:18:58.693 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:18:58.693 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:18:58 smithi089 bash[75068]: cluster 1685823536.8902502 mgr.y (mgr.34104) 1288 : cluster 0 pgmap v1284: 89 pgs: 20 active+clean+remapped, 14 active+undersized+degraded, 11 active+undersized, 44 active+clean; 6.1 GiB data, 19 GiB used, 518 GiB / 536 GiB avail; 85 B/s rd, 21 MiB/s wr, 848 op/s; 50004/298377 objects degraded (16.759%); 6/298377 objects misplaced (0.002%) 2023-06-03T20:18:58.694 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:18:58 smithi089 bash[75068]: cluster 1685823537.8424644 mon.a (mon.0) 1226 : cluster 3 Health check update: Degraded data redundancy: 50004/298377 objects degraded (16.759%), 14 pgs degraded, 25 pgs undersized (PG_DEGRADED) 2023-06-03T20:18:58.694 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:18:58 smithi089 bash[78452]: cluster 1685823536.8902502 mgr.y (mgr.34104) 1288 : cluster 0 pgmap v1284: 89 pgs: 20 active+clean+remapped, 14 active+undersized+degraded, 11 active+undersized, 44 active+clean; 6.1 GiB data, 19 GiB used, 518 GiB / 536 GiB avail; 85 B/s rd, 21 MiB/s wr, 848 op/s; 50004/298377 objects degraded (16.759%); 6/298377 objects misplaced (0.002%) 2023-06-03T20:18:58.694 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:18:58 smithi089 bash[78452]: cluster 1685823537.8424644 mon.a (mon.0) 1226 : cluster 3 Health check update: Degraded data redundancy: 50004/298377 objects degraded (16.759%), 14 pgs degraded, 25 pgs undersized (PG_DEGRADED) 2023-06-03T20:18:58.793 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:18:58.836 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:18:58.837 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:18:58.838 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:18:58.939 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:18:58.969 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:18:58.972 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:18:58.973 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:18:59.064 INFO:teuthology.orchestra.run.smithi089.stdout: 2023-06-03T20:18:59.074 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:18:59.084 INFO:teuthology.orchestra.run.smithi089.stderr:dumped all 2023-06-03T20:18:59.103 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:18:59.105 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:18:59.106 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:18:59.207 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:18:59.235 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:18:59.238 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:18:59.238 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:18:59.339 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:18:59.376 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:18:59.378 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:18:59.379 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:18:59.479 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:18:59.518 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:18:59.522 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:18:59.523 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:18:59.624 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:18:59.651 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:18:59.656 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:18:59.657 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:18:59.758 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:18:59.778 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:18:59.780 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:18:59.780 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:18:59.881 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:18:59.921 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:18:59.924 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:18:59.924 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:19:00.025 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:19:00.056 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:19:00.060 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:19:00.061 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:19:00.162 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:19:00.171 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:18:59 smithi089 bash[75068]: cluster 1685823538.8911674 mgr.y (mgr.34104) 1289 : cluster 0 pgmap v1285: 89 pgs: 20 active+clean+remapped, 14 active+undersized+degraded, 11 active+undersized, 44 active+clean; 6.0 GiB data, 18 GiB used, 518 GiB / 536 GiB avail; 85 B/s rd, 9.7 MiB/s wr, 814 op/s; 49000/293121 objects degraded (16.717%); 6/293121 objects misplaced (0.002%) 2023-06-03T20:19:00.171 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:18:59 smithi089 bash[75068]: audit 1685823539.0479736 mgr.y (mgr.34104) 1290 : audit 0 from='client.36456 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-06-03T20:19:00.171 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:18:59 smithi089 bash[78452]: cluster 1685823538.8911674 mgr.y (mgr.34104) 1289 : cluster 0 pgmap v1285: 89 pgs: 20 active+clean+remapped, 14 active+undersized+degraded, 11 active+undersized, 44 active+clean; 6.0 GiB data, 18 GiB used, 518 GiB / 536 GiB avail; 85 B/s rd, 9.7 MiB/s wr, 814 op/s; 49000/293121 objects degraded (16.717%); 6/293121 objects misplaced (0.002%) 2023-06-03T20:19:00.172 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:18:59 smithi089 bash[78452]: audit 1685823539.0479736 mgr.y (mgr.34104) 1290 : audit 0 from='client.36456 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-06-03T20:19:00.198 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:19:00.203 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:19:00.204 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:19:00.305 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:19:00.313 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:18:59 smithi164 bash[59462]: cluster 1685823538.8911674 mgr.y (mgr.34104) 1289 : cluster 0 pgmap v1285: 89 pgs: 20 active+clean+remapped, 14 active+undersized+degraded, 11 active+undersized, 44 active+clean; 6.0 GiB data, 18 GiB used, 518 GiB / 536 GiB avail; 85 B/s rd, 9.7 MiB/s wr, 814 op/s; 49000/293121 objects degraded (16.717%); 6/293121 objects misplaced (0.002%) 2023-06-03T20:19:00.313 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:18:59 smithi164 bash[59462]: audit 1685823539.0479736 mgr.y (mgr.34104) 1290 : audit 0 from='client.36456 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-06-03T20:19:00.342 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:19:00.345 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:19:00.346 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:19:00.447 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:19:00.478 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:19:00.479 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:19:00.480 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:19:00.581 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:19:00.623 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:19:00.627 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:19:00.628 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:19:00.729 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:19:00.763 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:19:00.766 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:19:00.766 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:19:00.867 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:19:00.898 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:19:00.900 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:19:00.901 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:19:01.002 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:19:01.031 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:19:01.033 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:19:01.034 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:19:01.135 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:19:01.185 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:19:01.189 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:19:01.190 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:19:01.290 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:19:01.320 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:19:01.324 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:19:01.324 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:19:01.425 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:19:01.455 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:19:01.459 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:19:01.459 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:19:01.560 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:19:01.588 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:19:01.594 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:19:01.595 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:19:01.696 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:19:01.728 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:19:01.730 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:19:01.730 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:19:01.831 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:19:01.876 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:19:01.881 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:19:01.882 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:19:01.983 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:19:02.012 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:19:02.018 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:19:02.018 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:19:02.094 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:19:02 smithi089 bash[75068]: cluster 1685823540.8918602 mgr.y (mgr.34104) 1291 : cluster 0 pgmap v1286: 89 pgs: 20 active+clean+remapped, 14 active+undersized+degraded, 11 active+undersized, 44 active+clean; 5.9 GiB data, 18 GiB used, 518 GiB / 536 GiB avail; 3.2 MiB/s wr, 782 op/s; 48256/289341 objects degraded (16.678%); 6/289341 objects misplaced (0.002%) 2023-06-03T20:19:02.095 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:19:02 smithi089 bash[78452]: cluster 1685823540.8918602 mgr.y (mgr.34104) 1291 : cluster 0 pgmap v1286: 89 pgs: 20 active+clean+remapped, 14 active+undersized+degraded, 11 active+undersized, 44 active+clean; 5.9 GiB data, 18 GiB used, 518 GiB / 536 GiB avail; 3.2 MiB/s wr, 782 op/s; 48256/289341 objects degraded (16.678%); 6/289341 objects misplaced (0.002%) 2023-06-03T20:19:02.119 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:19:02.156 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:19:02.159 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:19:02.160 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:19:02.260 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:19:02.294 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:19:02.297 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:19:02.297 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:19:02.334 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:19:01 smithi164 bash[59462]: cluster 1685823540.8918602 mgr.y (mgr.34104) 1291 : cluster 0 pgmap v1286: 89 pgs: 20 active+clean+remapped, 14 active+undersized+degraded, 11 active+undersized, 44 active+clean; 5.9 GiB data, 18 GiB used, 518 GiB / 536 GiB avail; 3.2 MiB/s wr, 782 op/s; 48256/289341 objects degraded (16.678%); 6/289341 objects misplaced (0.002%) 2023-06-03T20:19:02.398 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:19:02.444 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:19:02.448 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:19:02.449 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:19:02.550 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:19:02.583 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:19:02.586 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:19:02.587 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:19:02.626 DEBUG:teuthology.orchestra.run.smithi164:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph daemon osd.5 dump_ops_in_flight 2023-06-03T20:19:02.688 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:19:02.730 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:19:02.734 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:19:02.734 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:19:02.835 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:19:02.866 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:19:02.871 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:19:02.872 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:19:02.943 INFO:teuthology.orchestra.run.smithi089.stdout:{"pg_ready":true,"pg_map":{"version":1285,"stamp":"2023-06-03T20:18:58.890443+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":6402283430,"num_objects":97707,"num_object_clones":0,"num_object_copies":293121,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":49000,"num_objects_misplaced":6,"num_objects_unfound":0,"num_objects_dirty":97707,"num_whiteouts":0,"num_read":309,"num_read_kb":513,"num_write":113792,"num_write_kb":6761231,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":14,"num_bytes_recovered":595154,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":99626,"ondisk_log_size":99626,"up":222,"acting":242,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":263,"num_osds":6,"num_per_pool_osds":6,"num_per_pool_omap_osds":6,"kb":562470912,"kb_used":19331800,"kb_used_data":16474664,"kb_used_omap":26,"kb_used_meta":2856869,"kb_avail":543139112,"statfs":{"total":575970213888,"available":556174450688,"internally_reserved":0,"allocated":16870055936,"data_stored":16865660772,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":27292,"internal_metadata":2925434212},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[2,1,4,9,4,3,3],"upper_bound":128},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-397541352,"num_objects":-6065,"num_object_clones":0,"num_object_copies":-18195,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":-2968,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":-6065,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":9783,"num_write_kb":118913,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.006216"},"pg_stats":[{"pgid":"3.1a","version":"60'1","reported_seq":41,"reported_epoch":225,"state":"active+undersized","last_fresh":"2023-06-03T20:17:55.029037+0000","last_change":"2023-06-03T20:11:54.019728+0000","last_active":"2023-06-03T20:17:55.029037+0000","last_peered":"2023-06-03T20:17:55.029037+0000","last_clean":"2023-06-03T19:34:15.767991+0000","last_became_active":"2023-06-03T20:11:54.019728+0000","last_became_peered":"2023-06-03T20:11:54.019728+0000","last_unstale":"2023-06-03T20:17:55.029037+0000","last_undegraded":"2023-06-03T20:17:55.029037+0000","last_fullsized":"2023-06-03T20:11:54.009512+0000","mapping_epoch":195,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T19:58:26.738637+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":["2","0"],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":41,"reported_epoch":225,"state":"active+clean+remapped","last_fresh":"2023-06-03T20:17:55.031928+0000","last_change":"2023-06-03T20:12:31.911399+0000","last_active":"2023-06-03T20:17:55.031928+0000","last_peered":"2023-06-03T20:17:55.031928+0000","last_clean":"2023-06-03T20:17:55.031928+0000","last_became_active":"2023-06-03T20:12:31.911029+0000","last_became_peered":"2023-06-03T20:12:31.911029+0000","last_unstale":"2023-06-03T20:17:55.031928+0000","last_undegraded":"2023-06-03T20:17:55.031928+0000","last_fullsized":"2023-06-03T20:17:55.031928+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T05:04:34.770935+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5],"acting":[3,5,6],"avail_no_missing":["3","5","6"],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":198,"reported_epoch":225,"state":"active+clean+remapped","last_fresh":"2023-06-03T20:17:55.032905+0000","last_change":"2023-06-03T20:16:21.917001+0000","last_active":"2023-06-03T20:17:55.032905+0000","last_peered":"2023-06-03T20:17:55.032905+0000","last_clean":"2023-06-03T20:17:55.032905+0000","last_became_active":"2023-06-03T20:16:21.916538+0000","last_became_peered":"2023-06-03T20:16:21.916538+0000","last_unstale":"2023-06-03T20:17:55.032905+0000","last_undegraded":"2023-06-03T20:17:55.032905+0000","last_fullsized":"2023-06-03T20:17:55.032905+0000","mapping_epoch":213,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":214,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T20:44:25.048340+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7],"acting":[3,7,6],"avail_no_missing":["3","6","7"],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":191,"reported_epoch":225,"state":"active+clean","last_fresh":"2023-06-03T20:17:55.021576+0000","last_change":"2023-06-03T19:56:54.507653+0000","last_active":"2023-06-03T20:17:55.021576+0000","last_peered":"2023-06-03T20:17:55.021576+0000","last_clean":"2023-06-03T20:17:55.021576+0000","last_became_active":"2023-06-03T19:56:54.507350+0000","last_became_peered":"2023-06-03T19:56:54.507350+0000","last_unstale":"2023-06-03T20:17:55.021576+0000","last_undegraded":"2023-06-03T20:17:55.021576+0000","last_fullsized":"2023-06-03T20:17:55.021576+0000","mapping_epoch":119,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T22:33:20.065844+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":223,"reported_epoch":226,"state":"active+clean","last_fresh":"2023-06-03T20:18:28.855685+0000","last_change":"2023-06-03T20:07:19.912004+0000","last_active":"2023-06-03T20:18:28.855685+0000","last_peered":"2023-06-03T20:18:28.855685+0000","last_clean":"2023-06-03T20:18:28.855685+0000","last_became_active":"2023-06-03T20:07:19.910895+0000","last_became_peered":"2023-06-03T20:07:19.910895+0000","last_unstale":"2023-06-03T20:18:28.855685+0000","last_undegraded":"2023-06-03T20:18:28.855685+0000","last_fullsized":"2023-06-03T20:18:28.855685+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T06:46:44.564002+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,2],"acting":[3,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":151,"reported_epoch":225,"state":"active+clean","last_fresh":"2023-06-03T20:17:55.021150+0000","last_change":"2023-06-03T20:17:22.834646+0000","last_active":"2023-06-03T20:17:55.021150+0000","last_peered":"2023-06-03T20:17:55.021150+0000","last_clean":"2023-06-03T20:17:55.021150+0000","last_became_active":"2023-06-03T20:17:22.834324+0000","last_became_peered":"2023-06-03T20:17:22.834324+0000","last_unstale":"2023-06-03T20:17:55.021150+0000","last_undegraded":"2023-06-03T20:17:55.021150+0000","last_fullsized":"2023-06-03T20:17:55.021150+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T02:18:49.246420+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":188,"reported_epoch":226,"state":"active+clean","last_fresh":"2023-06-03T20:18:28.863069+0000","last_change":"2023-06-03T19:56:54.509490+0000","last_active":"2023-06-03T20:18:28.863069+0000","last_peered":"2023-06-03T20:18:28.863069+0000","last_clean":"2023-06-03T20:18:28.863069+0000","last_became_active":"2023-06-03T19:56:54.509145+0000","last_became_peered":"2023-06-03T19:56:54.509145+0000","last_unstale":"2023-06-03T20:18:28.863069+0000","last_undegraded":"2023-06-03T20:18:28.863069+0000","last_fullsized":"2023-06-03T20:18:28.863069+0000","mapping_epoch":119,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T07:12:03.964110+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":223,"reported_epoch":225,"state":"active+clean+remapped","last_fresh":"2023-06-03T20:17:55.031143+0000","last_change":"2023-06-03T20:12:31.937956+0000","last_active":"2023-06-03T20:17:55.031143+0000","last_peered":"2023-06-03T20:17:55.031143+0000","last_clean":"2023-06-03T20:17:55.031143+0000","last_became_active":"2023-06-03T20:12:31.937551+0000","last_became_peered":"2023-06-03T20:12:31.937551+0000","last_unstale":"2023-06-03T20:17:55.031143+0000","last_undegraded":"2023-06-03T20:17:55.031143+0000","last_fullsized":"2023-06-03T20:17:55.031143+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T21:49:12.058781+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5],"acting":[3,5,6],"avail_no_missing":["3","5","6"],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":149,"reported_epoch":225,"state":"active+clean","last_fresh":"2023-06-03T20:17:55.027601+0000","last_change":"2023-06-03T20:17:22.832811+0000","last_active":"2023-06-03T20:17:55.027601+0000","last_peered":"2023-06-03T20:17:55.027601+0000","last_clean":"2023-06-03T20:17:55.027601+0000","last_became_active":"2023-06-03T20:17:22.832474+0000","last_became_peered":"2023-06-03T20:17:22.832474+0000","last_unstale":"2023-06-03T20:17:55.027601+0000","last_undegraded":"2023-06-03T20:17:55.027601+0000","last_fullsized":"2023-06-03T20:17:55.027601+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,3],"acting":[7,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":188,"reported_epoch":225,"state":"active+clean","last_fresh":"2023-06-03T20:17:55.032784+0000","last_change":"2023-06-03T20:16:20.909781+0000","last_active":"2023-06-03T20:17:55.032784+0000","last_peered":"2023-06-03T20:17:55.032784+0000","last_clean":"2023-06-03T20:17:55.032784+0000","last_became_active":"2023-06-03T20:16:20.909496+0000","last_became_peered":"2023-06-03T20:16:20.909496+0000","last_unstale":"2023-06-03T20:17:55.032784+0000","last_undegraded":"2023-06-03T20:17:55.032784+0000","last_fullsized":"2023-06-03T20:17:55.032784+0000","mapping_epoch":212,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":213,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T20:32:26.886172+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,5],"acting":[3,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"60'3","reported_seq":214,"reported_epoch":225,"state":"active+undersized+degraded","last_fresh":"2023-06-03T20:17:55.027076+0000","last_change":"2023-06-03T20:11:54.025696+0000","last_active":"2023-06-03T20:17:55.027076+0000","last_peered":"2023-06-03T20:17:55.027076+0000","last_clean":"2023-06-03T20:11:43.973370+0000","last_became_active":"2023-06-03T20:11:54.025696+0000","last_became_peered":"2023-06-03T20:11:54.025696+0000","last_unstale":"2023-06-03T20:17:55.027076+0000","last_undegraded":"2023-06-03T20:11:54.012202+0000","last_fullsized":"2023-06-03T20:11:54.012054+0000","mapping_epoch":195,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":173,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3],"acting":[7,3],"avail_no_missing":["7","3"],"object_location_counts":[{"shards":"3,7","objects":2}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":84,"reported_epoch":225,"state":"active+clean","last_fresh":"2023-06-03T20:17:55.028928+0000","last_change":"2023-06-03T20:12:30.911538+0000","last_active":"2023-06-03T20:17:55.028928+0000","last_peered":"2023-06-03T20:17:55.028928+0000","last_clean":"2023-06-03T20:17:55.028928+0000","last_became_active":"2023-06-03T20:12:30.911082+0000","last_became_peered":"2023-06-03T20:12:30.911082+0000","last_unstale":"2023-06-03T20:17:55.028928+0000","last_undegraded":"2023-06-03T20:17:55.028928+0000","last_fullsized":"2023-06-03T20:17:55.028928+0000","mapping_epoch":198,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":199,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T05:22:01.454415+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,5],"acting":[2,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.14","version":"77'23","reported_seq":116,"reported_epoch":225,"state":"active+clean+remapped","last_fresh":"2023-06-03T20:17:55.027311+0000","last_change":"2023-06-03T20:16:22.089746+0000","last_active":"2023-06-03T20:17:55.027311+0000","last_peered":"2023-06-03T20:17:55.027311+0000","last_clean":"2023-06-03T20:17:55.027311+0000","last_became_active":"2023-06-03T20:16:21.916216+0000","last_became_peered":"2023-06-03T20:16:21.916216+0000","last_unstale":"2023-06-03T20:17:55.027311+0000","last_undegraded":"2023-06-03T20:17:55.027311+0000","last_fullsized":"2023-06-03T20:17:55.027311+0000","mapping_epoch":213,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":214,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":23,"log_dups_size":0,"ondisk_log_size":23,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4716,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":4,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":23,"num_write_kb":20,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4716,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2],"acting":[7,2,6],"avail_no_missing":["7","2","6"],"object_location_counts":[{"shards":"2,6,7","objects":4}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":151,"reported_epoch":225,"state":"active+clean+remapped","last_fresh":"2023-06-03T20:17:55.027240+0000","last_change":"2023-06-03T20:17:23.856188+0000","last_active":"2023-06-03T20:17:55.027240+0000","last_peered":"2023-06-03T20:17:55.027240+0000","last_clean":"2023-06-03T20:17:55.027240+0000","last_became_active":"2023-06-03T20:17:23.855783+0000","last_became_peered":"2023-06-03T20:17:23.855783+0000","last_unstale":"2023-06-03T20:17:55.027240+0000","last_undegraded":"2023-06-03T20:17:55.027240+0000","last_fullsized":"2023-06-03T20:17:55.027240+0000","mapping_epoch":223,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":224,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T20:50:46.483487+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5],"acting":[2,5,6],"avail_no_missing":["2","5","6"],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.13","version":"60'2","reported_seq":216,"reported_epoch":225,"state":"active+undersized+degraded","last_fresh":"2023-06-03T20:17:55.030535+0000","last_change":"2023-06-03T20:11:54.019051+0000","last_active":"2023-06-03T20:17:55.030535+0000","last_peered":"2023-06-03T20:17:55.030535+0000","last_clean":"2023-06-03T20:11:43.972488+0000","last_became_active":"2023-06-03T20:11:54.019051+0000","last_became_peered":"2023-06-03T20:11:54.019051+0000","last_unstale":"2023-06-03T20:17:55.030535+0000","last_undegraded":"2023-06-03T20:11:54.012366+0000","last_fullsized":"2023-06-03T20:11:54.012217+0000","mapping_epoch":195,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2],"acting":[7,2],"avail_no_missing":["7","2"],"object_location_counts":[{"shards":"2,7","objects":2}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":207,"reported_epoch":225,"state":"active+clean","last_fresh":"2023-06-03T20:17:58.566786+0000","last_change":"2023-06-03T20:07:19.907827+0000","last_active":"2023-06-03T20:17:58.566786+0000","last_peered":"2023-06-03T20:17:58.566786+0000","last_clean":"2023-06-03T20:17:58.566786+0000","last_became_active":"2023-06-03T20:07:19.907296+0000","last_became_peered":"2023-06-03T20:07:19.907296+0000","last_unstale":"2023-06-03T20:17:58.566786+0000","last_undegraded":"2023-06-03T20:17:58.566786+0000","last_fullsized":"2023-06-03T20:17:58.566786+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T00:34:13.230147+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":188,"reported_epoch":225,"state":"active+clean","last_fresh":"2023-06-03T20:17:55.022294+0000","last_change":"2023-06-03T19:56:54.507479+0000","last_active":"2023-06-03T20:17:55.022294+0000","last_peered":"2023-06-03T20:17:55.022294+0000","last_clean":"2023-06-03T20:17:55.022294+0000","last_became_active":"2023-06-03T19:56:54.507149+0000","last_became_peered":"2023-06-03T19:56:54.507149+0000","last_unstale":"2023-06-03T20:17:55.022294+0000","last_undegraded":"2023-06-03T20:17:55.022294+0000","last_fullsized":"2023-06-03T20:17:55.022294+0000","mapping_epoch":119,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T20:22:36.676317+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":92,"reported_epoch":225,"state":"active+clean","last_fresh":"2023-06-03T20:17:55.027458+0000","last_change":"2023-06-03T20:17:22.832844+0000","last_active":"2023-06-03T20:17:55.027458+0000","last_peered":"2023-06-03T20:17:55.027458+0000","last_clean":"2023-06-03T20:17:55.027458+0000","last_became_active":"2023-06-03T20:17:22.832473+0000","last_became_peered":"2023-06-03T20:17:22.832473+0000","last_unstale":"2023-06-03T20:17:55.027458+0000","last_undegraded":"2023-06-03T20:17:55.027458+0000","last_fullsized":"2023-06-03T20:17:55.027458+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,3],"acting":[7,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":206,"reported_epoch":225,"state":"active+clean+remapped","last_fresh":"2023-06-03T20:17:55.026740+0000","last_change":"2023-06-03T20:16:21.916599+0000","last_active":"2023-06-03T20:17:55.026740+0000","last_peered":"2023-06-03T20:17:55.026740+0000","last_clean":"2023-06-03T20:17:55.026740+0000","last_became_active":"2023-06-03T20:16:21.916145+0000","last_became_peered":"2023-06-03T20:16:21.916145+0000","last_unstale":"2023-06-03T20:17:55.026740+0000","last_undegraded":"2023-06-03T20:17:55.026740+0000","last_fullsized":"2023-06-03T20:17:55.026740+0000","mapping_epoch":213,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":214,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0],"acting":[7,0,6],"avail_no_missing":["7","0","6"],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":189,"reported_epoch":226,"state":"active+clean","last_fresh":"2023-06-03T20:18:28.865135+0000","last_change":"2023-06-03T19:56:54.510289+0000","last_active":"2023-06-03T20:18:28.865135+0000","last_peered":"2023-06-03T20:18:28.865135+0000","last_clean":"2023-06-03T20:18:28.865135+0000","last_became_active":"2023-06-03T19:56:54.509905+0000","last_became_peered":"2023-06-03T19:56:54.509905+0000","last_unstale":"2023-06-03T20:18:28.865135+0000","last_undegraded":"2023-06-03T20:18:28.865135+0000","last_fullsized":"2023-06-03T20:18:28.865135+0000","mapping_epoch":119,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T02:54:26.792401+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"60'8","reported_seq":47,"reported_epoch":226,"state":"active+clean","last_fresh":"2023-06-03T20:18:28.868614+0000","last_change":"2023-06-03T20:16:20.914195+0000","last_active":"2023-06-03T20:18:28.868614+0000","last_peered":"2023-06-03T20:18:28.868614+0000","last_clean":"2023-06-03T20:18:28.868614+0000","last_became_active":"2023-06-03T20:16:20.906350+0000","last_became_peered":"2023-06-03T20:16:20.906350+0000","last_unstale":"2023-06-03T20:18:28.868614+0000","last_undegraded":"2023-06-03T20:18:28.868614+0000","last_fullsized":"2023-06-03T20:18:28.868614+0000","mapping_epoch":212,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":213,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T20:33:32.314202+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":207,"reported_epoch":226,"state":"active+clean","last_fresh":"2023-06-03T20:18:28.854364+0000","last_change":"2023-06-03T20:07:19.907151+0000","last_active":"2023-06-03T20:18:28.854364+0000","last_peered":"2023-06-03T20:18:28.854364+0000","last_clean":"2023-06-03T20:18:28.854364+0000","last_became_active":"2023-06-03T20:07:19.906479+0000","last_became_peered":"2023-06-03T20:07:19.906479+0000","last_unstale":"2023-06-03T20:18:28.854364+0000","last_undegraded":"2023-06-03T20:18:28.854364+0000","last_fullsized":"2023-06-03T20:18:28.854364+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T03:18:22.525692+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"12.0","version":"226'7071","reported_seq":7087,"reported_epoch":226,"state":"active+undersized+degraded","last_fresh":"2023-06-03T20:18:55.506113+0000","last_change":"2023-06-03T20:17:17.413985+0000","last_active":"2023-06-03T20:18:55.506113+0000","last_peered":"2023-06-03T20:18:55.506113+0000","last_clean":"2023-06-03T20:16:46.173157+0000","last_became_active":"2023-06-03T20:16:47.243639+0000","last_became_peered":"2023-06-03T20:16:47.243639+0000","last_unstale":"2023-06-03T20:18:55.506113+0000","last_undegraded":"2023-06-03T20:17:17.399054+0000","last_fullsized":"2023-06-03T20:16:47.186301+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":217,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T20:16:46.173157+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T20:16:46.173157+0000","last_clean_scrub_stamp":"2023-06-03T20:16:46.173157+0000","objects_scrubbed":0,"log_size":7071,"log_dups_size":0,"ondisk_log_size":7071,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T02:35:37.796501+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":407699456,"num_objects":6221,"num_object_clones":0,"num_object_copies":18663,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":6221,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6221,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7071,"num_write_kb":425344,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7],"acting":[2,7],"avail_no_missing":["2","7"],"object_location_counts":[{"shards":"2,7","objects":6221}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.f","version":"60'1","reported_seq":215,"reported_epoch":225,"state":"active+undersized+degraded","last_fresh":"2023-06-03T20:17:55.027190+0000","last_change":"2023-06-03T20:11:54.016915+0000","last_active":"2023-06-03T20:17:55.027190+0000","last_peered":"2023-06-03T20:17:55.027190+0000","last_clean":"2023-06-03T20:11:43.972655+0000","last_became_active":"2023-06-03T20:11:54.016915+0000","last_became_peered":"2023-06-03T20:11:54.016915+0000","last_unstale":"2023-06-03T20:17:55.027190+0000","last_undegraded":"2023-06-03T20:11:54.009884+0000","last_fullsized":"2023-06-03T20:11:54.009713+0000","mapping_epoch":195,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0],"acting":[7,0],"avail_no_missing":["7","0"],"object_location_counts":[{"shards":"0,7","objects":1}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":166,"reported_epoch":225,"state":"active+clean+remapped","last_fresh":"2023-06-03T20:17:55.026732+0000","last_change":"2023-06-03T20:17:23.832761+0000","last_active":"2023-06-03T20:17:55.026732+0000","last_peered":"2023-06-03T20:17:55.026732+0000","last_clean":"2023-06-03T20:17:55.026732+0000","last_became_active":"2023-06-03T20:17:23.832000+0000","last_became_peered":"2023-06-03T20:17:23.832000+0000","last_unstale":"2023-06-03T20:17:55.026732+0000","last_undegraded":"2023-06-03T20:17:55.026732+0000","last_fullsized":"2023-06-03T20:17:55.026732+0000","mapping_epoch":223,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":224,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5],"acting":[7,5,6],"avail_no_missing":["7","5","6"],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"12.1","version":"226'7420","reported_seq":7439,"reported_epoch":226,"state":"active+clean","last_fresh":"2023-06-03T20:18:58.744852+0000","last_change":"2023-06-03T20:16:47.253397+0000","last_active":"2023-06-03T20:18:58.744852+0000","last_peered":"2023-06-03T20:18:58.744852+0000","last_clean":"2023-06-03T20:18:58.744852+0000","last_became_active":"2023-06-03T20:16:47.253078+0000","last_became_peered":"2023-06-03T20:16:47.253078+0000","last_unstale":"2023-06-03T20:18:58.744852+0000","last_undegraded":"2023-06-03T20:18:58.744852+0000","last_fullsized":"2023-06-03T20:18:58.744852+0000","mapping_epoch":217,"log_start":"224'1900","ondisk_log_start":"224'1900","created":217,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T20:16:46.173157+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T20:16:46.173157+0000","last_clean_scrub_stamp":"2023-06-03T20:16:46.173157+0000","objects_scrubbed":0,"log_size":5520,"log_dups_size":0,"ondisk_log_size":5520,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T05:27:02.520483+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":408289280,"num_objects":6230,"num_object_clones":0,"num_object_copies":18690,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6230,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7420,"num_write_kb":436800,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,5],"acting":[3,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"60'2","reported_seq":229,"reported_epoch":225,"state":"active+undersized+degraded","last_fresh":"2023-06-03T20:17:55.027318+0000","last_change":"2023-06-03T20:11:54.012607+0000","last_active":"2023-06-03T20:17:55.027318+0000","last_peered":"2023-06-03T20:17:55.027318+0000","last_clean":"2023-06-03T20:11:43.972261+0000","last_became_active":"2023-06-03T20:11:54.012607+0000","last_became_peered":"2023-06-03T20:11:54.012607+0000","last_unstale":"2023-06-03T20:17:55.027318+0000","last_undegraded":"2023-06-03T20:11:54.010373+0000","last_fullsized":"2023-06-03T20:11:54.010224+0000","mapping_epoch":195,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0],"acting":[7,0],"avail_no_missing":["7","0"],"object_location_counts":[{"shards":"0,7","objects":1}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":219,"reported_epoch":225,"state":"active+clean+remapped","last_fresh":"2023-06-03T20:18:28.642438+0000","last_change":"2023-06-03T20:12:31.941492+0000","last_active":"2023-06-03T20:18:28.642438+0000","last_peered":"2023-06-03T20:18:28.642438+0000","last_clean":"2023-06-03T20:18:28.642438+0000","last_became_active":"2023-06-03T20:12:31.941076+0000","last_became_peered":"2023-06-03T20:12:31.941076+0000","last_unstale":"2023-06-03T20:18:28.642438+0000","last_undegraded":"2023-06-03T20:18:28.642438+0000","last_fullsized":"2023-06-03T20:18:28.642438+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T03:59:11.007424+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0],"acting":[3,0,6],"avail_no_missing":["3","0","6"],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"12.2","version":"226'7217","reported_seq":7233,"reported_epoch":226,"state":"active+undersized+degraded","last_fresh":"2023-06-03T20:18:58.777022+0000","last_change":"2023-06-03T20:17:17.393673+0000","last_active":"2023-06-03T20:18:58.777022+0000","last_peered":"2023-06-03T20:18:58.777022+0000","last_clean":"2023-06-03T20:16:46.173157+0000","last_became_active":"2023-06-03T20:16:47.245039+0000","last_became_peered":"2023-06-03T20:16:47.245039+0000","last_unstale":"2023-06-03T20:18:58.777022+0000","last_undegraded":"2023-06-03T20:17:17.392270+0000","last_fullsized":"2023-06-03T20:16:47.186792+0000","mapping_epoch":217,"log_start":"224'1700","ondisk_log_start":"224'1700","created":217,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T20:16:46.173157+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T20:16:46.173157+0000","last_clean_scrub_stamp":"2023-06-03T20:16:46.173157+0000","objects_scrubbed":0,"log_size":5517,"log_dups_size":0,"ondisk_log_size":5517,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T21:53:32.172346+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":397737984,"num_objects":6069,"num_object_clones":0,"num_object_copies":18207,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":6069,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6069,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7217,"num_write_kb":425152,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7],"acting":[3,7],"avail_no_missing":["3","7"],"object_location_counts":[{"shards":"3,7","objects":6069}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.d","version":"60'1","reported_seq":189,"reported_epoch":225,"state":"active+clean","last_fresh":"2023-06-03T20:17:55.026864+0000","last_change":"2023-06-03T20:16:20.915741+0000","last_active":"2023-06-03T20:17:55.026864+0000","last_peered":"2023-06-03T20:17:55.026864+0000","last_clean":"2023-06-03T20:17:55.026864+0000","last_became_active":"2023-06-03T20:16:20.908091+0000","last_became_peered":"2023-06-03T20:16:20.908091+0000","last_unstale":"2023-06-03T20:17:55.026864+0000","last_undegraded":"2023-06-03T20:17:55.026864+0000","last_fullsized":"2023-06-03T20:17:55.026864+0000","mapping_epoch":212,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":213,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":70,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,3],"acting":[7,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":172,"reported_epoch":225,"state":"active+clean","last_fresh":"2023-06-03T20:17:55.026925+0000","last_change":"2023-06-03T20:12:30.910738+0000","last_active":"2023-06-03T20:17:55.026925+0000","last_peered":"2023-06-03T20:17:55.026925+0000","last_clean":"2023-06-03T20:17:55.026925+0000","last_became_active":"2023-06-03T20:12:30.910145+0000","last_became_peered":"2023-06-03T20:12:30.910145+0000","last_unstale":"2023-06-03T20:17:55.026925+0000","last_undegraded":"2023-06-03T20:17:55.026925+0000","last_fullsized":"2023-06-03T20:17:55.026925+0000","mapping_epoch":198,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":199,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,0],"acting":[7,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"12.3","version":"226'7079","reported_seq":7095,"reported_epoch":226,"state":"active+clean","last_fresh":"2023-06-03T20:18:56.597814+0000","last_change":"2023-06-03T20:16:47.247006+0000","last_active":"2023-06-03T20:18:56.597814+0000","last_peered":"2023-06-03T20:18:56.597814+0000","last_clean":"2023-06-03T20:18:56.597814+0000","last_became_active":"2023-06-03T20:16:47.246661+0000","last_became_peered":"2023-06-03T20:16:47.246661+0000","last_unstale":"2023-06-03T20:18:56.597814+0000","last_undegraded":"2023-06-03T20:18:56.597814+0000","last_fullsized":"2023-06-03T20:18:56.597814+0000","mapping_epoch":217,"log_start":"224'1000","ondisk_log_start":"224'1000","created":217,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T20:16:46.173157+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T20:16:46.173157+0000","last_clean_scrub_stamp":"2023-06-03T20:16:46.173157+0000","objects_scrubbed":0,"log_size":6079,"log_dups_size":0,"ondisk_log_size":6079,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T07:23:06.138157+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":399966208,"num_objects":6103,"num_object_clones":0,"num_object_copies":18309,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6103,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7079,"num_write_kb":421824,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":195,"reported_epoch":225,"state":"active+clean","last_fresh":"2023-06-03T20:17:55.032097+0000","last_change":"2023-06-03T20:17:22.835469+0000","last_active":"2023-06-03T20:17:55.032097+0000","last_peered":"2023-06-03T20:17:55.032097+0000","last_clean":"2023-06-03T20:17:55.032097+0000","last_became_active":"2023-06-03T20:17:22.835172+0000","last_became_peered":"2023-06-03T20:17:22.835172+0000","last_unstale":"2023-06-03T20:17:55.032097+0000","last_undegraded":"2023-06-03T20:17:55.032097+0000","last_fullsized":"2023-06-03T20:17:55.032097+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T21:59:45.737240+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,0],"acting":[3,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":232,"reported_epoch":226,"state":"active+undersized","last_fresh":"2023-06-03T20:18:28.863981+0000","last_change":"2023-06-03T20:11:54.017375+0000","last_active":"2023-06-03T20:18:28.863981+0000","last_peered":"2023-06-03T20:18:28.863981+0000","last_clean":"2023-06-03T20:11:50.798782+0000","last_became_active":"2023-06-03T20:11:54.017375+0000","last_became_peered":"2023-06-03T20:11:54.017375+0000","last_unstale":"2023-06-03T20:18:28.863981+0000","last_undegraded":"2023-06-03T20:18:28.863981+0000","last_fullsized":"2023-06-03T20:11:54.008913+0000","mapping_epoch":195,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T21:44:01.134376+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3],"acting":[0,3],"avail_no_missing":["0","3"],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"12.4","version":"226'7170","reported_seq":7186,"reported_epoch":226,"state":"active+undersized+degraded","last_fresh":"2023-06-03T20:18:58.765764+0000","last_change":"2023-06-03T20:17:17.411278+0000","last_active":"2023-06-03T20:18:58.765764+0000","last_peered":"2023-06-03T20:18:58.765764+0000","last_clean":"2023-06-03T20:16:46.173157+0000","last_became_active":"2023-06-03T20:16:47.245348+0000","last_became_peered":"2023-06-03T20:16:47.245348+0000","last_unstale":"2023-06-03T20:18:58.765764+0000","last_undegraded":"2023-06-03T20:17:17.394222+0000","last_fullsized":"2023-06-03T20:16:47.186407+0000","mapping_epoch":217,"log_start":"224'1700","ondisk_log_start":"224'1700","created":217,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T20:16:46.173157+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T20:16:46.173157+0000","last_clean_scrub_stamp":"2023-06-03T20:16:46.173157+0000","objects_scrubbed":0,"log_size":5470,"log_dups_size":0,"ondisk_log_size":5470,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T06:20:25.995223+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":397279232,"num_objects":6062,"num_object_clones":0,"num_object_copies":18186,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":6062,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6062,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7170,"num_write_kb":423424,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7],"acting":[3,7],"avail_no_missing":["3","7"],"object_location_counts":[{"shards":"3,7","objects":6062}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.b","version":"60'2","reported_seq":205,"reported_epoch":225,"state":"active+undersized+degraded","last_fresh":"2023-06-03T20:17:55.035628+0000","last_change":"2023-06-03T20:11:54.019446+0000","last_active":"2023-06-03T20:17:55.035628+0000","last_peered":"2023-06-03T20:17:55.035628+0000","last_clean":"2023-06-03T20:11:43.977155+0000","last_became_active":"2023-06-03T20:11:54.019446+0000","last_became_peered":"2023-06-03T20:11:54.019446+0000","last_unstale":"2023-06-03T20:17:55.035628+0000","last_undegraded":"2023-06-03T20:11:54.013169+0000","last_fullsized":"2023-06-03T20:11:54.012960+0000","mapping_epoch":195,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T20:48:53.735497+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0],"acting":[3,0],"avail_no_missing":["3","0"],"object_location_counts":[{"shards":"0,3","objects":1}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":140,"reported_epoch":226,"state":"active+clean+remapped","last_fresh":"2023-06-03T20:18:28.855468+0000","last_change":"2023-06-03T20:16:21.917354+0000","last_active":"2023-06-03T20:18:28.855468+0000","last_peered":"2023-06-03T20:18:28.855468+0000","last_clean":"2023-06-03T20:18:28.855468+0000","last_became_active":"2023-06-03T20:16:21.916856+0000","last_became_peered":"2023-06-03T20:16:21.916856+0000","last_unstale":"2023-06-03T20:18:28.855468+0000","last_undegraded":"2023-06-03T20:18:28.855468+0000","last_fullsized":"2023-06-03T20:18:28.855468+0000","mapping_epoch":213,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":214,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T06:32:46.792786+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7],"acting":[3,7,6],"avail_no_missing":["3","6","7"],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"12.5","version":"226'6910","reported_seq":6929,"reported_epoch":226,"state":"active+clean","last_fresh":"2023-06-03T20:18:55.506578+0000","last_change":"2023-06-03T20:16:47.247897+0000","last_active":"2023-06-03T20:18:55.506578+0000","last_peered":"2023-06-03T20:18:55.506578+0000","last_clean":"2023-06-03T20:18:55.506578+0000","last_became_active":"2023-06-03T20:16:47.247585+0000","last_became_peered":"2023-06-03T20:16:47.247585+0000","last_unstale":"2023-06-03T20:18:55.506578+0000","last_undegraded":"2023-06-03T20:18:55.506578+0000","last_fullsized":"2023-06-03T20:18:55.506578+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":217,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T20:16:46.173157+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T20:16:46.173157+0000","last_clean_scrub_stamp":"2023-06-03T20:16:46.173157+0000","objects_scrubbed":0,"log_size":6910,"log_dups_size":0,"ondisk_log_size":6910,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T00:04:38.960679+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":401735680,"num_objects":6130,"num_object_clones":0,"num_object_copies":18390,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6130,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6910,"num_write_kb":417280,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":214,"reported_epoch":226,"state":"active+clean+remapped","last_fresh":"2023-06-03T20:18:28.857582+0000","last_change":"2023-06-03T20:16:21.913184+0000","last_active":"2023-06-03T20:18:28.857582+0000","last_peered":"2023-06-03T20:18:28.857582+0000","last_clean":"2023-06-03T20:18:28.857582+0000","last_became_active":"2023-06-03T20:16:21.912223+0000","last_became_peered":"2023-06-03T20:16:21.912223+0000","last_unstale":"2023-06-03T20:18:28.857582+0000","last_undegraded":"2023-06-03T20:18:28.857582+0000","last_fullsized":"2023-06-03T20:18:28.857582+0000","mapping_epoch":213,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":214,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T23:07:23.671860+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5],"acting":[3,5,6],"avail_no_missing":["3","5","6"],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":200,"reported_epoch":226,"state":"active+clean+remapped","last_fresh":"2023-06-03T20:18:28.861620+0000","last_change":"2023-06-03T20:16:21.909200+0000","last_active":"2023-06-03T20:18:28.861620+0000","last_peered":"2023-06-03T20:18:28.861620+0000","last_clean":"2023-06-03T20:18:28.861620+0000","last_became_active":"2023-06-03T20:16:21.908774+0000","last_became_peered":"2023-06-03T20:16:21.908774+0000","last_unstale":"2023-06-03T20:18:28.861620+0000","last_undegraded":"2023-06-03T20:18:28.861620+0000","last_fullsized":"2023-06-03T20:18:28.861620+0000","mapping_epoch":213,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":214,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T22:38:01.732512+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0,6],"avail_no_missing":["2","0","6"],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"12.6","version":"226'7045","reported_seq":7064,"reported_epoch":226,"state":"active+clean","last_fresh":"2023-06-03T20:18:56.589435+0000","last_change":"2023-06-03T20:16:47.254017+0000","last_active":"2023-06-03T20:18:56.589435+0000","last_peered":"2023-06-03T20:18:56.589435+0000","last_clean":"2023-06-03T20:18:56.589435+0000","last_became_active":"2023-06-03T20:16:47.253781+0000","last_became_peered":"2023-06-03T20:16:47.253781+0000","last_unstale":"2023-06-03T20:18:56.589435+0000","last_undegraded":"2023-06-03T20:18:56.589435+0000","last_fullsized":"2023-06-03T20:18:56.589435+0000","mapping_epoch":217,"log_start":"224'1000","ondisk_log_start":"224'1000","created":217,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T20:16:46.173157+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T20:16:46.173157+0000","last_clean_scrub_stamp":"2023-06-03T20:16:46.173157+0000","objects_scrubbed":0,"log_size":6045,"log_dups_size":0,"ondisk_log_size":6045,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T20:50:22.970205+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":398524416,"num_objects":6081,"num_object_clones":0,"num_object_copies":18243,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6081,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7045,"num_write_kb":420032,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":127,"reported_epoch":225,"state":"active+undersized","last_fresh":"2023-06-03T20:18:25.435094+0000","last_change":"2023-06-03T20:11:54.018578+0000","last_active":"2023-06-03T20:18:25.435094+0000","last_peered":"2023-06-03T20:18:25.435094+0000","last_clean":"2023-06-03T19:55:41.905941+0000","last_became_active":"2023-06-03T20:11:54.018578+0000","last_became_peered":"2023-06-03T20:11:54.018578+0000","last_unstale":"2023-06-03T20:18:25.435094+0000","last_undegraded":"2023-06-03T20:18:25.435094+0000","last_fullsized":"2023-06-03T20:11:54.009386+0000","mapping_epoch":195,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T21:13:40.683176+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7],"acting":[2,7],"avail_no_missing":["2","7"],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":174,"reported_epoch":225,"state":"active+clean+remapped","last_fresh":"2023-06-03T20:17:55.037397+0000","last_change":"2023-06-03T20:12:31.910070+0000","last_active":"2023-06-03T20:17:55.037397+0000","last_peered":"2023-06-03T20:17:55.037397+0000","last_clean":"2023-06-03T20:17:55.037397+0000","last_became_active":"2023-06-03T20:12:31.909692+0000","last_became_peered":"2023-06-03T20:12:31.909692+0000","last_unstale":"2023-06-03T20:17:55.037397+0000","last_undegraded":"2023-06-03T20:17:55.037397+0000","last_fullsized":"2023-06-03T20:17:55.037397+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T07:20:52.604735+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5],"acting":[3,5,6],"avail_no_missing":["3","5","6"],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"12.7","version":"226'7227","reported_seq":7240,"reported_epoch":226,"state":"active+undersized+degraded","last_fresh":"2023-06-03T20:18:58.396447+0000","last_change":"2023-06-03T20:17:17.396134+0000","last_active":"2023-06-03T20:18:58.396447+0000","last_peered":"2023-06-03T20:18:58.396447+0000","last_clean":"2023-06-03T20:16:46.173157+0000","last_became_active":"2023-06-03T20:16:47.247445+0000","last_became_peered":"2023-06-03T20:16:47.247445+0000","last_unstale":"2023-06-03T20:18:58.396447+0000","last_undegraded":"2023-06-03T20:17:17.388520+0000","last_fullsized":"2023-06-03T20:16:47.186781+0000","mapping_epoch":217,"log_start":"224'900","ondisk_log_start":"224'900","created":217,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T20:16:46.173157+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T20:16:46.173157+0000","last_clean_scrub_stamp":"2023-06-03T20:16:46.173157+0000","objects_scrubbed":0,"log_size":6327,"log_dups_size":0,"ondisk_log_size":6327,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":400752640,"num_objects":6115,"num_object_clones":0,"num_object_copies":18345,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":6115,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6115,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7227,"num_write_kb":426944,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2],"acting":[7,2],"avail_no_missing":["7","2"],"object_location_counts":[{"shards":"2,7","objects":6115}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":215,"reported_epoch":226,"state":"active+undersized","last_fresh":"2023-06-03T20:18:28.855561+0000","last_change":"2023-06-03T20:11:54.026212+0000","last_active":"2023-06-03T20:18:28.855561+0000","last_peered":"2023-06-03T20:18:28.855561+0000","last_clean":"2023-06-03T20:11:43.974120+0000","last_became_active":"2023-06-03T20:11:54.026212+0000","last_became_peered":"2023-06-03T20:11:54.026212+0000","last_unstale":"2023-06-03T20:18:28.855561+0000","last_undegraded":"2023-06-03T20:18:28.855561+0000","last_fullsized":"2023-06-03T20:11:54.007103+0000","mapping_epoch":195,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T03:49:35.033521+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7],"acting":[3,7],"avail_no_missing":["3","7"],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":148,"reported_epoch":226,"state":"active+undersized","last_fresh":"2023-06-03T20:18:28.854169+0000","last_change":"2023-06-03T20:11:54.026421+0000","last_active":"2023-06-03T20:18:28.854169+0000","last_peered":"2023-06-03T20:18:28.854169+0000","last_clean":"2023-06-03T20:11:43.974605+0000","last_became_active":"2023-06-03T20:11:54.026421+0000","last_became_peered":"2023-06-03T20:11:54.026421+0000","last_unstale":"2023-06-03T20:18:28.854169+0000","last_undegraded":"2023-06-03T20:18:28.854169+0000","last_fullsized":"2023-06-03T20:11:54.009062+0000","mapping_epoch":195,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T02:07:21.901385+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7],"acting":[3,7],"avail_no_missing":["3","7"],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"12.a","version":"226'7075","reported_seq":7093,"reported_epoch":226,"state":"active+clean","last_fresh":"2023-06-03T20:18:55.451544+0000","last_change":"2023-06-03T20:16:47.247803+0000","last_active":"2023-06-03T20:18:55.451544+0000","last_peered":"2023-06-03T20:18:55.451544+0000","last_clean":"2023-06-03T20:18:55.451544+0000","last_became_active":"2023-06-03T20:16:47.247523+0000","last_became_peered":"2023-06-03T20:16:47.247523+0000","last_unstale":"2023-06-03T20:18:55.451544+0000","last_undegraded":"2023-06-03T20:18:55.451544+0000","last_fullsized":"2023-06-03T20:18:55.451544+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":217,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T20:16:46.173157+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T20:16:46.173157+0000","last_clean_scrub_stamp":"2023-06-03T20:16:46.173157+0000","objects_scrubbed":0,"log_size":7075,"log_dups_size":0,"ondisk_log_size":7075,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T06:47:22.934291+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":411238400,"num_objects":6275,"num_object_clones":0,"num_object_copies":18825,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6275,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7075,"num_write_kb":427200,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,3],"acting":[2,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":138,"reported_epoch":225,"state":"active+clean","last_fresh":"2023-06-03T20:17:55.021852+0000","last_change":"2023-06-03T20:07:19.926958+0000","last_active":"2023-06-03T20:17:55.021852+0000","last_peered":"2023-06-03T20:17:55.021852+0000","last_clean":"2023-06-03T20:17:55.021852+0000","last_became_active":"2023-06-03T20:07:19.926456+0000","last_became_peered":"2023-06-03T20:07:19.926456+0000","last_unstale":"2023-06-03T20:17:55.021852+0000","last_undegraded":"2023-06-03T20:17:55.021852+0000","last_fullsized":"2023-06-03T20:17:55.021852+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:32:20.692043+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:32:20.692043+0000","last_clean_scrub_stamp":"2023-06-03T19:32:20.692043+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T20:47:07.666782+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,2],"acting":[0,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":108,"reported_epoch":225,"state":"active+clean","last_fresh":"2023-06-03T20:17:55.031847+0000","last_change":"2023-06-03T20:17:22.834746+0000","last_active":"2023-06-03T20:17:55.031847+0000","last_peered":"2023-06-03T20:17:55.031847+0000","last_clean":"2023-06-03T20:17:55.031847+0000","last_became_active":"2023-06-03T20:17:22.834448+0000","last_became_peered":"2023-06-03T20:17:22.834448+0000","last_unstale":"2023-06-03T20:17:55.031847+0000","last_undegraded":"2023-06-03T20:17:55.031847+0000","last_fullsized":"2023-06-03T20:17:55.031847+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T21:16:31.167063+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,2],"acting":[3,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":104,"reported_epoch":225,"state":"active+undersized","last_fresh":"2023-06-03T20:17:55.022173+0000","last_change":"2023-06-03T20:17:22.834796+0000","last_active":"2023-06-03T20:17:55.022173+0000","last_peered":"2023-06-03T20:17:55.022173+0000","last_clean":"2023-06-03T20:11:43.981149+0000","last_became_active":"2023-06-03T20:17:22.834796+0000","last_became_peered":"2023-06-03T20:17:22.834796+0000","last_unstale":"2023-06-03T20:17:55.022173+0000","last_undegraded":"2023-06-03T20:17:55.022173+0000","last_fullsized":"2023-06-03T20:17:22.820764+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T22:16:00.682284+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5],"acting":[0,5],"avail_no_missing":["0","5"],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"12.c","version":"226'6964","reported_seq":6982,"reported_epoch":226,"state":"active+clean","last_fresh":"2023-06-03T20:18:56.575852+0000","last_change":"2023-06-03T20:16:47.246691+0000","last_active":"2023-06-03T20:18:56.575852+0000","last_peered":"2023-06-03T20:18:56.575852+0000","last_clean":"2023-06-03T20:18:56.575852+0000","last_became_active":"2023-06-03T20:16:47.246401+0000","last_became_peered":"2023-06-03T20:16:47.246401+0000","last_unstale":"2023-06-03T20:18:56.575852+0000","last_undegraded":"2023-06-03T20:18:56.575852+0000","last_fullsized":"2023-06-03T20:18:56.575852+0000","mapping_epoch":217,"log_start":"224'900","ondisk_log_start":"224'900","created":217,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T20:16:46.173157+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T20:16:46.173157+0000","last_clean_scrub_stamp":"2023-06-03T20:16:46.173157+0000","objects_scrubbed":0,"log_size":6064,"log_dups_size":0,"ondisk_log_size":6064,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T22:59:40.915657+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":400293888,"num_objects":6108,"num_object_clones":0,"num_object_copies":18324,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6108,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6964,"num_write_kb":418304,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":194,"reported_epoch":225,"state":"active+clean","last_fresh":"2023-06-03T20:17:55.022458+0000","last_change":"2023-06-03T20:17:22.834516+0000","last_active":"2023-06-03T20:17:55.022458+0000","last_peered":"2023-06-03T20:17:55.022458+0000","last_clean":"2023-06-03T20:17:55.022458+0000","last_became_active":"2023-06-03T20:17:22.833893+0000","last_became_peered":"2023-06-03T20:17:22.833893+0000","last_unstale":"2023-06-03T20:17:55.022458+0000","last_undegraded":"2023-06-03T20:17:55.022458+0000","last_fullsized":"2023-06-03T20:17:55.022458+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:32:20.692043+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:32:20.692043+0000","last_clean_scrub_stamp":"2023-06-03T19:32:20.692043+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T19:57:46.959122+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":110,"reported_epoch":225,"state":"active+clean+remapped","last_fresh":"2023-06-03T20:18:11.484192+0000","last_change":"2023-06-03T20:16:21.912393+0000","last_active":"2023-06-03T20:18:11.484192+0000","last_peered":"2023-06-03T20:18:11.484192+0000","last_clean":"2023-06-03T20:18:11.484192+0000","last_became_active":"2023-06-03T20:16:21.911851+0000","last_became_peered":"2023-06-03T20:16:21.911851+0000","last_unstale":"2023-06-03T20:18:11.484192+0000","last_undegraded":"2023-06-03T20:18:11.484192+0000","last_fullsized":"2023-06-03T20:18:11.484192+0000","mapping_epoch":213,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":214,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T06:21:00.069019+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5],"acting":[0,5,6],"avail_no_missing":["0","5","6"],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":146,"reported_epoch":225,"state":"active+clean","last_fresh":"2023-06-03T20:17:55.026665+0000","last_change":"2023-06-03T20:12:30.911946+0000","last_active":"2023-06-03T20:17:55.026665+0000","last_peered":"2023-06-03T20:17:55.026665+0000","last_clean":"2023-06-03T20:17:55.026665+0000","last_became_active":"2023-06-03T20:12:30.911513+0000","last_became_peered":"2023-06-03T20:12:30.911513+0000","last_unstale":"2023-06-03T20:17:55.026665+0000","last_undegraded":"2023-06-03T20:17:55.026665+0000","last_fullsized":"2023-06-03T20:17:55.026665+0000","mapping_epoch":198,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":199,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T07:22:01.480443+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,7],"acting":[2,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"12.f","version":"226'7137","reported_seq":7156,"reported_epoch":226,"state":"active+undersized+degraded","last_fresh":"2023-06-03T20:18:55.497545+0000","last_change":"2023-06-03T20:17:22.835038+0000","last_active":"2023-06-03T20:18:55.497545+0000","last_peered":"2023-06-03T20:18:55.497545+0000","last_clean":"2023-06-03T20:16:46.173157+0000","last_became_active":"2023-06-03T20:17:22.835038+0000","last_became_peered":"2023-06-03T20:17:22.835038+0000","last_unstale":"2023-06-03T20:18:55.497545+0000","last_undegraded":"2023-06-03T20:17:22.825008+0000","last_fullsized":"2023-06-03T20:17:22.824833+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":217,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T20:16:46.173157+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T20:16:46.173157+0000","last_clean_scrub_stamp":"2023-06-03T20:16:46.173157+0000","objects_scrubbed":0,"log_size":7137,"log_dups_size":0,"ondisk_log_size":7137,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T02:06:54.311682+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":411500544,"num_objects":6279,"num_object_clones":0,"num_object_copies":18837,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":6279,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6279,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7137,"num_write_kb":429312,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5],"acting":[2,5],"avail_no_missing":["2","5"],"object_location_counts":[{"shards":"2,5","objects":6279}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":207,"reported_epoch":225,"state":"active+clean","last_fresh":"2023-06-03T20:17:55.025925+0000","last_change":"2023-06-03T19:56:54.504859+0000","last_active":"2023-06-03T20:17:55.025925+0000","last_peered":"2023-06-03T20:17:55.025925+0000","last_clean":"2023-06-03T20:17:55.025925+0000","last_became_active":"2023-06-03T19:56:54.504455+0000","last_became_peered":"2023-06-03T19:56:54.504455+0000","last_unstale":"2023-06-03T20:17:55.025925+0000","last_undegraded":"2023-06-03T20:17:55.025925+0000","last_fullsized":"2023-06-03T20:17:55.025925+0000","mapping_epoch":119,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:32:20.692043+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:32:20.692043+0000","last_clean_scrub_stamp":"2023-06-03T19:32:20.692043+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T06:25:45.522100+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":167,"reported_epoch":225,"state":"active+clean","last_fresh":"2023-06-03T20:17:55.026774+0000","last_change":"2023-06-03T20:16:20.905988+0000","last_active":"2023-06-03T20:17:55.026774+0000","last_peered":"2023-06-03T20:17:55.026774+0000","last_clean":"2023-06-03T20:17:55.026774+0000","last_became_active":"2023-06-03T20:16:20.905535+0000","last_became_peered":"2023-06-03T20:16:20.905535+0000","last_unstale":"2023-06-03T20:17:55.026774+0000","last_undegraded":"2023-06-03T20:17:55.026774+0000","last_fullsized":"2023-06-03T20:17:55.026774+0000","mapping_epoch":212,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":213,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T22:12:58.208178+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,5],"acting":[2,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":75,"reported_epoch":225,"state":"active+clean","last_fresh":"2023-06-03T20:17:55.021447+0000","last_change":"2023-06-03T20:17:22.834837+0000","last_active":"2023-06-03T20:17:55.021447+0000","last_peered":"2023-06-03T20:17:55.021447+0000","last_clean":"2023-06-03T20:17:55.021447+0000","last_became_active":"2023-06-03T20:17:22.834493+0000","last_became_peered":"2023-06-03T20:17:22.834493+0000","last_unstale":"2023-06-03T20:17:55.021447+0000","last_undegraded":"2023-06-03T20:17:55.021447+0000","last_fullsized":"2023-06-03T20:17:55.021447+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T23:57:05.248165+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"12.d","version":"226'6913","reported_seq":6929,"reported_epoch":226,"state":"active+clean","last_fresh":"2023-06-03T20:18:58.343730+0000","last_change":"2023-06-03T20:16:47.245232+0000","last_active":"2023-06-03T20:18:58.343730+0000","last_peered":"2023-06-03T20:18:58.343730+0000","last_clean":"2023-06-03T20:18:58.343730+0000","last_became_active":"2023-06-03T20:16:47.244812+0000","last_became_peered":"2023-06-03T20:16:47.244812+0000","last_unstale":"2023-06-03T20:18:58.343730+0000","last_undegraded":"2023-06-03T20:18:58.343730+0000","last_fullsized":"2023-06-03T20:18:58.343730+0000","mapping_epoch":217,"log_start":"221'600","ondisk_log_start":"221'600","created":217,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T20:16:46.173157+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T20:16:46.173157+0000","last_clean_scrub_stamp":"2023-06-03T20:16:46.173157+0000","objects_scrubbed":0,"log_size":6313,"log_dups_size":0,"ondisk_log_size":6313,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":378011672,"num_objects":5769,"num_object_clones":0,"num_object_copies":17307,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5769,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":6913,"num_write_kb":405761,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,3],"acting":[7,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":92,"reported_epoch":225,"state":"active+clean","last_fresh":"2023-06-03T20:17:55.026949+0000","last_change":"2023-06-03T20:17:22.834467+0000","last_active":"2023-06-03T20:17:55.026949+0000","last_peered":"2023-06-03T20:17:55.026949+0000","last_clean":"2023-06-03T20:17:55.026949+0000","last_became_active":"2023-06-03T20:17:22.834147+0000","last_became_peered":"2023-06-03T20:17:22.834147+0000","last_unstale":"2023-06-03T20:17:55.026949+0000","last_undegraded":"2023-06-03T20:17:55.026949+0000","last_fullsized":"2023-06-03T20:17:55.026949+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:32:20.692043+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:32:20.692043+0000","last_clean_scrub_stamp":"2023-06-03T19:32:20.692043+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T04:36:29.567051+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,7],"acting":[2,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"72'139","reported_seq":449,"reported_epoch":225,"state":"active+clean","last_fresh":"2023-06-03T20:17:55.026492+0000","last_change":"2023-06-03T20:16:23.829575+0000","last_active":"2023-06-03T20:17:55.026492+0000","last_peered":"2023-06-03T20:17:55.026492+0000","last_clean":"2023-06-03T20:17:55.026492+0000","last_became_active":"2023-06-03T20:16:23.829028+0000","last_became_peered":"2023-06-03T20:16:23.829028+0000","last_unstale":"2023-06-03T20:17:55.026492+0000","last_undegraded":"2023-06-03T20:17:55.026492+0000","last_fullsized":"2023-06-03T20:17:55.026492+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:27:28.100808+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:27:28.100808+0000","last_clean_scrub_stamp":"2023-06-03T19:27:28.100808+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":590368,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,2],"acting":[7,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":190,"reported_epoch":225,"state":"active+clean+remapped","last_fresh":"2023-06-03T20:17:55.032156+0000","last_change":"2023-06-03T20:16:21.913608+0000","last_active":"2023-06-03T20:17:55.032156+0000","last_peered":"2023-06-03T20:17:55.032156+0000","last_clean":"2023-06-03T20:17:55.032156+0000","last_became_active":"2023-06-03T20:16:21.912814+0000","last_became_peered":"2023-06-03T20:16:21.912814+0000","last_unstale":"2023-06-03T20:17:55.032156+0000","last_undegraded":"2023-06-03T20:17:55.032156+0000","last_fullsized":"2023-06-03T20:17:55.032156+0000","mapping_epoch":213,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":214,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T06:29:01.991511+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5],"acting":[3,5,6],"avail_no_missing":["3","5","6"],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":202,"reported_epoch":225,"state":"active+clean+remapped","last_fresh":"2023-06-03T20:17:55.027388+0000","last_change":"2023-06-03T20:12:31.939716+0000","last_active":"2023-06-03T20:17:55.027388+0000","last_peered":"2023-06-03T20:17:55.027388+0000","last_clean":"2023-06-03T20:17:55.027388+0000","last_became_active":"2023-06-03T20:12:31.938876+0000","last_became_peered":"2023-06-03T20:12:31.938876+0000","last_unstale":"2023-06-03T20:17:55.027388+0000","last_undegraded":"2023-06-03T20:17:55.027388+0000","last_fullsized":"2023-06-03T20:17:55.027388+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0],"acting":[7,0,6],"avail_no_missing":["7","0","6"],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"12.e","version":"226'7164","reported_seq":7179,"reported_epoch":226,"state":"active+undersized+degraded","last_fresh":"2023-06-03T20:18:56.542842+0000","last_change":"2023-06-03T20:17:17.423740+0000","last_active":"2023-06-03T20:18:56.542842+0000","last_peered":"2023-06-03T20:18:56.542842+0000","last_clean":"2023-06-03T20:16:46.173157+0000","last_became_active":"2023-06-03T20:16:47.253524+0000","last_became_peered":"2023-06-03T20:16:47.253524+0000","last_unstale":"2023-06-03T20:18:56.542842+0000","last_undegraded":"2023-06-03T20:17:17.408943+0000","last_fullsized":"2023-06-03T20:16:47.245201+0000","mapping_epoch":217,"log_start":"224'1100","ondisk_log_start":"224'1100","created":217,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T20:16:46.173157+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T20:16:46.173157+0000","last_clean_scrub_stamp":"2023-06-03T20:16:46.173157+0000","objects_scrubbed":0,"log_size":6064,"log_dups_size":0,"ondisk_log_size":6064,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T00:13:48.032998+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":409337856,"num_objects":6246,"num_object_clones":0,"num_object_copies":18738,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":6246,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6246,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7164,"num_write_kb":429120,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7],"acting":[0,7],"avail_no_missing":["0","7"],"object_location_counts":[{"shards":"0,7","objects":6246}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":209,"reported_epoch":225,"state":"active+clean","last_fresh":"2023-06-03T20:17:55.026460+0000","last_change":"2023-06-03T20:07:47.855767+0000","last_active":"2023-06-03T20:17:55.026460+0000","last_peered":"2023-06-03T20:17:55.026460+0000","last_clean":"2023-06-03T20:17:55.026460+0000","last_became_active":"2023-06-03T20:07:47.855444+0000","last_became_peered":"2023-06-03T20:07:47.855444+0000","last_unstale":"2023-06-03T20:17:55.026460+0000","last_undegraded":"2023-06-03T20:17:55.026460+0000","last_fullsized":"2023-06-03T20:17:55.026460+0000","mapping_epoch":172,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":173,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:32:20.692043+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:32:20.692043+0000","last_clean_scrub_stamp":"2023-06-03T19:32:20.692043+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,3],"acting":[7,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":211,"reported_epoch":225,"state":"active+undersized","last_fresh":"2023-06-03T20:18:21.490579+0000","last_change":"2023-06-03T20:11:54.017331+0000","last_active":"2023-06-03T20:18:21.490579+0000","last_peered":"2023-06-03T20:18:21.490579+0000","last_clean":"2023-06-03T20:11:43.977931+0000","last_became_active":"2023-06-03T20:11:54.017331+0000","last_became_peered":"2023-06-03T20:11:54.017331+0000","last_unstale":"2023-06-03T20:18:21.490579+0000","last_undegraded":"2023-06-03T20:18:21.490579+0000","last_fullsized":"2023-06-03T20:11:54.007149+0000","mapping_epoch":195,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T04:51:15.533105+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3],"acting":[0,3],"avail_no_missing":["0","3"],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":214,"reported_epoch":225,"state":"active+clean","last_fresh":"2023-06-03T20:17:55.021783+0000","last_change":"2023-06-03T20:07:19.925864+0000","last_active":"2023-06-03T20:17:55.021783+0000","last_peered":"2023-06-03T20:17:55.021783+0000","last_clean":"2023-06-03T20:17:55.021783+0000","last_became_active":"2023-06-03T20:07:19.925541+0000","last_became_peered":"2023-06-03T20:07:19.925541+0000","last_unstale":"2023-06-03T20:17:55.021783+0000","last_undegraded":"2023-06-03T20:17:55.021783+0000","last_fullsized":"2023-06-03T20:17:55.021783+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T00:10:11.790831+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,7],"acting":[0,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"12.b","version":"226'7056","reported_seq":7071,"reported_epoch":226,"state":"active+undersized+degraded","last_fresh":"2023-06-03T20:18:58.724791+0000","last_change":"2023-06-03T20:17:17.433790+0000","last_active":"2023-06-03T20:18:58.724791+0000","last_peered":"2023-06-03T20:18:58.724791+0000","last_clean":"2023-06-03T20:16:46.173157+0000","last_became_active":"2023-06-03T20:16:47.248675+0000","last_became_peered":"2023-06-03T20:16:47.248675+0000","last_unstale":"2023-06-03T20:18:58.724791+0000","last_undegraded":"2023-06-03T20:17:17.407850+0000","last_fullsized":"2023-06-03T20:16:47.187599+0000","mapping_epoch":217,"log_start":"224'1600","ondisk_log_start":"224'1600","created":217,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T20:16:46.173157+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T20:16:46.173157+0000","last_clean_scrub_stamp":"2023-06-03T20:16:46.173157+0000","objects_scrubbed":0,"log_size":5456,"log_dups_size":0,"ondisk_log_size":5456,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T21:43:59.039606+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":383516672,"num_objects":5852,"num_object_clones":0,"num_object_copies":17556,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5852,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5852,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7056,"num_write_kb":413056,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2],"acting":[3,2],"avail_no_missing":["3","2"],"object_location_counts":[{"shards":"2,3","objects":5852}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":218,"reported_epoch":225,"state":"active+undersized","last_fresh":"2023-06-03T20:17:55.027192+0000","last_change":"2023-06-03T20:11:54.012820+0000","last_active":"2023-06-03T20:17:55.027192+0000","last_peered":"2023-06-03T20:17:55.027192+0000","last_clean":"2023-06-03T20:11:43.973280+0000","last_became_active":"2023-06-03T20:11:54.012820+0000","last_became_peered":"2023-06-03T20:11:54.012820+0000","last_unstale":"2023-06-03T20:17:55.027192+0000","last_undegraded":"2023-06-03T20:17:55.027192+0000","last_fullsized":"2023-06-03T20:11:54.011295+0000","mapping_epoch":195,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:32:20.692043+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:32:20.692043+0000","last_clean_scrub_stamp":"2023-06-03T19:32:20.692043+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0],"acting":[7,0],"avail_no_missing":["7","0"],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":158,"reported_epoch":225,"state":"active+clean","last_fresh":"2023-06-03T20:17:55.026834+0000","last_change":"2023-06-03T20:07:19.915065+0000","last_active":"2023-06-03T20:17:55.026834+0000","last_peered":"2023-06-03T20:17:55.026834+0000","last_clean":"2023-06-03T20:17:55.026834+0000","last_became_active":"2023-06-03T20:07:19.914699+0000","last_became_peered":"2023-06-03T20:07:19.914699+0000","last_unstale":"2023-06-03T20:17:55.026834+0000","last_undegraded":"2023-06-03T20:17:55.026834+0000","last_fullsized":"2023-06-03T20:17:55.026834+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T07:16:09.077184+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,0],"acting":[2,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":176,"reported_epoch":225,"state":"active+clean","last_fresh":"2023-06-03T20:18:21.490656+0000","last_change":"2023-06-03T19:47:06.733532+0000","last_active":"2023-06-03T20:18:21.490656+0000","last_peered":"2023-06-03T20:18:21.490656+0000","last_clean":"2023-06-03T20:18:21.490656+0000","last_became_active":"2023-06-03T19:47:06.733131+0000","last_became_peered":"2023-06-03T19:47:06.733131+0000","last_unstale":"2023-06-03T20:18:21.490656+0000","last_undegraded":"2023-06-03T20:18:21.490656+0000","last_fullsized":"2023-06-03T20:18:21.490656+0000","mapping_epoch":86,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":87,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T23:46:03.788646+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"12.8","version":"226'7155","reported_seq":7173,"reported_epoch":226,"state":"active+clean","last_fresh":"2023-06-03T20:18:58.772934+0000","last_change":"2023-06-03T20:16:47.250053+0000","last_active":"2023-06-03T20:18:58.772934+0000","last_peered":"2023-06-03T20:18:58.772934+0000","last_clean":"2023-06-03T20:18:58.772934+0000","last_became_active":"2023-06-03T20:16:47.249779+0000","last_became_peered":"2023-06-03T20:16:47.249779+0000","last_unstale":"2023-06-03T20:18:58.772934+0000","last_undegraded":"2023-06-03T20:18:58.772934+0000","last_fullsized":"2023-06-03T20:18:58.772934+0000","mapping_epoch":217,"log_start":"224'1700","ondisk_log_start":"224'1700","created":217,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T20:16:46.173157+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T20:16:46.173157+0000","last_clean_scrub_stamp":"2023-06-03T20:16:46.173157+0000","objects_scrubbed":0,"log_size":5455,"log_dups_size":0,"ondisk_log_size":5455,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T06:42:14.674064+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":392888320,"num_objects":5995,"num_object_clones":0,"num_object_copies":17985,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5995,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7155,"num_write_kb":420800,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.7","version":"60'2","reported_seq":185,"reported_epoch":226,"state":"active+clean","last_fresh":"2023-06-03T20:18:28.854288+0000","last_change":"2023-06-03T19:56:54.510148+0000","last_active":"2023-06-03T20:18:28.854288+0000","last_peered":"2023-06-03T20:18:28.854288+0000","last_clean":"2023-06-03T20:18:28.854288+0000","last_became_active":"2023-06-03T19:56:54.509771+0000","last_became_peered":"2023-06-03T19:56:54.509771+0000","last_unstale":"2023-06-03T20:18:28.854288+0000","last_undegraded":"2023-06-03T20:18:28.854288+0000","last_fullsized":"2023-06-03T20:18:28.854288+0000","mapping_epoch":119,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T00:52:42.807432+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":160,"reported_epoch":226,"state":"active+clean+remapped","last_fresh":"2023-06-03T20:18:28.854271+0000","last_change":"2023-06-03T20:16:21.910628+0000","last_active":"2023-06-03T20:18:28.854271+0000","last_peered":"2023-06-03T20:18:28.854271+0000","last_clean":"2023-06-03T20:18:28.854271+0000","last_became_active":"2023-06-03T20:16:21.910183+0000","last_became_peered":"2023-06-03T20:16:21.910183+0000","last_unstale":"2023-06-03T20:18:28.854271+0000","last_undegraded":"2023-06-03T20:18:28.854271+0000","last_fullsized":"2023-06-03T20:18:28.854271+0000","mapping_epoch":213,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":214,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:32:20.692043+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:32:20.692043+0000","last_clean_scrub_stamp":"2023-06-03T19:32:20.692043+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T22:12:12.845109+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0],"acting":[3,0,6],"avail_no_missing":["3","0","6"],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":183,"reported_epoch":226,"state":"active+clean","last_fresh":"2023-06-03T20:18:28.855298+0000","last_change":"2023-06-03T19:56:54.507670+0000","last_active":"2023-06-03T20:18:28.855298+0000","last_peered":"2023-06-03T20:18:28.855298+0000","last_clean":"2023-06-03T20:18:28.855298+0000","last_became_active":"2023-06-03T19:56:54.507293+0000","last_became_peered":"2023-06-03T19:56:54.507293+0000","last_unstale":"2023-06-03T20:18:28.855298+0000","last_undegraded":"2023-06-03T20:18:28.855298+0000","last_fullsized":"2023-06-03T20:18:28.855298+0000","mapping_epoch":119,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T01:49:19.966054+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"12.9","version":"226'6928","reported_seq":6943,"reported_epoch":226,"state":"active+undersized+degraded","last_fresh":"2023-06-03T20:18:55.439671+0000","last_change":"2023-06-03T20:17:17.488439+0000","last_active":"2023-06-03T20:18:55.439671+0000","last_peered":"2023-06-03T20:18:55.439671+0000","last_clean":"2023-06-03T20:16:46.173157+0000","last_became_active":"2023-06-03T20:16:47.247014+0000","last_became_peered":"2023-06-03T20:16:47.247014+0000","last_unstale":"2023-06-03T20:18:55.439671+0000","last_undegraded":"2023-06-03T20:17:17.485553+0000","last_fullsized":"2023-06-03T20:16:47.186572+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":217,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T20:16:46.173157+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T20:16:46.173157+0000","last_clean_scrub_stamp":"2023-06-03T20:16:46.173157+0000","objects_scrubbed":0,"log_size":6928,"log_dups_size":0,"ondisk_log_size":6928,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T00:37:52.758979+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":402915328,"num_objects":6148,"num_object_clones":0,"num_object_copies":18444,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":6148,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6148,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6928,"num_write_kb":418432,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3],"acting":[2,3],"avail_no_missing":["2","3"],"object_location_counts":[{"shards":"2,3","objects":6148}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":102,"reported_epoch":225,"state":"active+clean","last_fresh":"2023-06-03T20:17:55.027155+0000","last_change":"2023-06-03T20:16:20.907573+0000","last_active":"2023-06-03T20:17:55.027155+0000","last_peered":"2023-06-03T20:17:55.027155+0000","last_clean":"2023-06-03T20:17:55.027155+0000","last_became_active":"2023-06-03T20:16:20.907200+0000","last_became_peered":"2023-06-03T20:16:20.907200+0000","last_unstale":"2023-06-03T20:17:55.027155+0000","last_undegraded":"2023-06-03T20:17:55.027155+0000","last_fullsized":"2023-06-03T20:17:55.027155+0000","mapping_epoch":212,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":213,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:32:20.692043+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:32:20.692043+0000","last_clean_scrub_stamp":"2023-06-03T19:32:20.692043+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,3],"acting":[7,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.6","version":"60'1","reported_seq":227,"reported_epoch":225,"state":"active+undersized+degraded","last_fresh":"2023-06-03T20:17:55.021454+0000","last_change":"2023-06-03T20:11:54.014758+0000","last_active":"2023-06-03T20:17:55.021454+0000","last_peered":"2023-06-03T20:17:55.021454+0000","last_clean":"2023-06-03T20:11:43.980210+0000","last_became_active":"2023-06-03T20:11:54.014758+0000","last_became_peered":"2023-06-03T20:11:54.014758+0000","last_unstale":"2023-06-03T20:17:55.021454+0000","last_undegraded":"2023-06-03T20:11:54.011442+0000","last_fullsized":"2023-06-03T20:11:54.011320+0000","mapping_epoch":195,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T20:43:34.505545+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7],"acting":[0,7],"avail_no_missing":["0","7"],"object_location_counts":[{"shards":"0,7","objects":1}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":98,"reported_epoch":225,"state":"active+clean+remapped","last_fresh":"2023-06-03T20:17:55.020933+0000","last_change":"2023-06-03T20:17:23.833953+0000","last_active":"2023-06-03T20:17:55.020933+0000","last_peered":"2023-06-03T20:17:55.020933+0000","last_clean":"2023-06-03T20:17:55.020933+0000","last_became_active":"2023-06-03T20:17:23.833627+0000","last_became_peered":"2023-06-03T20:17:23.833627+0000","last_unstale":"2023-06-03T20:17:55.020933+0000","last_undegraded":"2023-06-03T20:17:55.020933+0000","last_fullsized":"2023-06-03T20:17:55.020933+0000","mapping_epoch":223,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":224,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T06:35:53.082804+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5],"acting":[0,5,6],"avail_no_missing":["0","5","6"],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1b","version":"60'1","reported_seq":197,"reported_epoch":225,"state":"active+undersized","last_fresh":"2023-06-03T20:17:55.021557+0000","last_change":"2023-06-03T20:11:54.016103+0000","last_active":"2023-06-03T20:17:55.021557+0000","last_peered":"2023-06-03T20:17:55.021557+0000","last_clean":"2023-06-03T20:11:43.979629+0000","last_became_active":"2023-06-03T20:11:54.016103+0000","last_became_peered":"2023-06-03T20:11:54.016103+0000","last_unstale":"2023-06-03T20:17:55.021557+0000","last_undegraded":"2023-06-03T20:17:55.021557+0000","last_fullsized":"2023-06-03T20:11:54.009298+0000","mapping_epoch":195,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T04:35:48.454086+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7],"acting":[0,7],"avail_no_missing":["0","7"],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":227,"reported_epoch":226,"state":"active+clean","last_fresh":"2023-06-03T20:18:28.861939+0000","last_change":"2023-06-03T20:07:19.914985+0000","last_active":"2023-06-03T20:18:28.861939+0000","last_peered":"2023-06-03T20:18:28.861939+0000","last_clean":"2023-06-03T20:18:28.861939+0000","last_became_active":"2023-06-03T20:07:19.914624+0000","last_became_peered":"2023-06-03T20:07:19.914624+0000","last_unstale":"2023-06-03T20:18:28.861939+0000","last_undegraded":"2023-06-03T20:18:28.861939+0000","last_fullsized":"2023-06-03T20:18:28.861939+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T07:02:50.515170+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"60'2","reported_seq":218,"reported_epoch":225,"state":"active+clean+remapped","last_fresh":"2023-06-03T20:17:55.027218+0000","last_change":"2023-06-03T20:17:23.833394+0000","last_active":"2023-06-03T20:17:55.027218+0000","last_peered":"2023-06-03T20:17:55.027218+0000","last_clean":"2023-06-03T20:17:55.027218+0000","last_became_active":"2023-06-03T20:17:23.832836+0000","last_became_peered":"2023-06-03T20:17:23.832836+0000","last_unstale":"2023-06-03T20:17:55.027218+0000","last_undegraded":"2023-06-03T20:17:55.027218+0000","last_fullsized":"2023-06-03T20:17:55.027218+0000","mapping_epoch":223,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":224,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":1,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5],"acting":[7,5,6],"avail_no_missing":["7","5","6"],"object_location_counts":[{"shards":"5,6,7","objects":1}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":198,"reported_epoch":225,"state":"active+undersized","last_fresh":"2023-06-03T20:17:55.035402+0000","last_change":"2023-06-03T20:11:54.028489+0000","last_active":"2023-06-03T20:17:55.035402+0000","last_peered":"2023-06-03T20:17:55.035402+0000","last_clean":"2023-06-03T20:07:14.870985+0000","last_became_active":"2023-06-03T20:11:54.028489+0000","last_became_peered":"2023-06-03T20:11:54.028489+0000","last_unstale":"2023-06-03T20:17:55.035402+0000","last_undegraded":"2023-06-03T20:17:55.035402+0000","last_fullsized":"2023-06-03T20:11:54.011931+0000","mapping_epoch":195,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T04:24:31.291273+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2],"acting":[3,2],"avail_no_missing":["3","2"],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1d","version":"60'2","reported_seq":190,"reported_epoch":225,"state":"active+clean+remapped","last_fresh":"2023-06-03T20:17:55.031990+0000","last_change":"2023-06-03T20:17:23.862021+0000","last_active":"2023-06-03T20:17:55.031990+0000","last_peered":"2023-06-03T20:17:55.031990+0000","last_clean":"2023-06-03T20:17:55.031990+0000","last_became_active":"2023-06-03T20:17:23.861592+0000","last_became_peered":"2023-06-03T20:17:23.861592+0000","last_unstale":"2023-06-03T20:17:55.031990+0000","last_undegraded":"2023-06-03T20:17:55.031990+0000","last_fullsized":"2023-06-03T20:17:55.031990+0000","mapping_epoch":223,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":224,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T23:22:31.887030+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":1,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5],"acting":[3,5,6],"avail_no_missing":["3","5","6"],"object_location_counts":[{"shards":"3,5,6","objects":1}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":144,"reported_epoch":225,"state":"active+undersized","last_fresh":"2023-06-03T20:17:55.031900+0000","last_change":"2023-06-03T20:11:54.015791+0000","last_active":"2023-06-03T20:17:55.031900+0000","last_peered":"2023-06-03T20:17:55.031900+0000","last_clean":"2023-06-03T19:56:11.746795+0000","last_became_active":"2023-06-03T20:11:54.015791+0000","last_became_peered":"2023-06-03T20:11:54.015791+0000","last_unstale":"2023-06-03T20:17:55.031900+0000","last_undegraded":"2023-06-03T20:17:55.031900+0000","last_fullsized":"2023-06-03T20:11:54.007458+0000","mapping_epoch":195,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T19:54:52.259092+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0],"acting":[3,0],"avail_no_missing":["3","0"],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"60'1","reported_seq":221,"reported_epoch":226,"state":"active+clean","last_fresh":"2023-06-03T20:18:28.853958+0000","last_change":"2023-06-03T20:12:30.909386+0000","last_active":"2023-06-03T20:18:28.853958+0000","last_peered":"2023-06-03T20:18:28.853958+0000","last_clean":"2023-06-03T20:18:28.853958+0000","last_became_active":"2023-06-03T20:12:30.908991+0000","last_became_peered":"2023-06-03T20:12:30.908991+0000","last_unstale":"2023-06-03T20:18:28.853958+0000","last_undegraded":"2023-06-03T20:18:28.853958+0000","last_fullsized":"2023-06-03T20:18:28.853958+0000","mapping_epoch":198,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":199,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T04:33:12.721959+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,7],"acting":[3,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":223,"reported_epoch":226,"state":"active+clean","last_fresh":"2023-06-03T20:18:28.855311+0000","last_change":"2023-06-03T19:56:54.503416+0000","last_active":"2023-06-03T20:18:28.855311+0000","last_peered":"2023-06-03T20:18:28.855311+0000","last_clean":"2023-06-03T20:18:28.855311+0000","last_became_active":"2023-06-03T19:56:54.503018+0000","last_became_peered":"2023-06-03T19:56:54.503018+0000","last_unstale":"2023-06-03T20:18:28.855311+0000","last_undegraded":"2023-06-03T20:18:28.855311+0000","last_fullsized":"2023-06-03T20:18:28.855311+0000","mapping_epoch":119,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T20:14:00.330616+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"60'4","reported_seq":194,"reported_epoch":225,"state":"active+clean","last_fresh":"2023-06-03T20:18:21.490744+0000","last_change":"2023-06-03T19:56:11.748116+0000","last_active":"2023-06-03T20:18:21.490744+0000","last_peered":"2023-06-03T20:18:21.490744+0000","last_clean":"2023-06-03T20:18:21.490744+0000","last_became_active":"2023-06-03T19:56:11.747636+0000","last_became_peered":"2023-06-03T19:56:11.747636+0000","last_unstale":"2023-06-03T20:18:21.490744+0000","last_undegraded":"2023-06-03T20:18:21.490744+0000","last_fullsized":"2023-06-03T20:18:21.490744+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":115,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T06:57:48.052055+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":41,"reported_epoch":225,"state":"active+clean+remapped","last_fresh":"2023-06-03T20:17:55.030808+0000","last_change":"2023-06-03T20:12:32.093827+0000","last_active":"2023-06-03T20:17:55.030808+0000","last_peered":"2023-06-03T20:17:55.030808+0000","last_clean":"2023-06-03T20:17:55.030808+0000","last_became_active":"2023-06-03T20:12:32.093400+0000","last_became_peered":"2023-06-03T20:12:32.093400+0000","last_unstale":"2023-06-03T20:17:55.030808+0000","last_undegraded":"2023-06-03T20:17:55.030808+0000","last_fullsized":"2023-06-03T20:17:55.030808+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T23:22:58.054742+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2],"acting":[3,2,6],"avail_no_missing":["3","2","6"],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]}],"pool_stats":[{"poolid":12,"num_pg":16,"stat_sum":{"num_bytes":6401687576,"num_objects":97683,"num_object_clones":0,"num_object_copies":293049,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":48992,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":97683,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":113531,"num_write_kb":6758785,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":15955406848,"data_stored":15955394632,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":99431,"ondisk_log_size":99431,"up":40,"acting":40,"num_store_stats":5},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":80,"acting":91,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5486,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":8,"num_objects_misplaced":6,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":62,"num_write_kb":30,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":4786,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":110592,"data_stored":21174,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":30319,"internal_metadata":0},"log_size":56,"ondisk_log_size":56,"up":77,"acting":85,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":590368,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2375680,"data_stored":2361472,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":3,"num_store_stats":4},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":22,"acting":23,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":48,"seq":206158430783,"num_pgs":48,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":4216856,"kb_used_data":3501904,"kb_used_omap":0,"kb_used_meta":714944,"kb_avail":89528296,"statfs":{"total":95995035648,"available":91676975104,"internally_reserved":0,"allocated":3585949696,"data_stored":3585391424,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":732102656},"hb_peers":[0,1,2,3,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,1,0,3],"upper_bound":16},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 3 20:18:04 2023","interfaces":[{"interface":"back","average":{"1min":1.1870000000000001,"5min":1.5629999999999999,"15min":1.819},"min":{"1min":0.312,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":4.5410000000000004,"5min":20.981999999999999,"15min":48.338999999999999},"last":0.309},{"interface":"front","average":{"1min":1.153,"5min":1.4490000000000001,"15min":1.5760000000000001},"min":{"1min":0.30199999999999999,"5min":0.28000000000000003,"15min":0.27800000000000002},"max":{"1min":4.9820000000000002,"5min":21.167000000000002,"15min":46.412999999999997},"last":0.68400000000000005}]},{"osd":1,"last update":"Sat Jun 3 20:18:55 2023","interfaces":[{"interface":"back","average":{"1min":0.56499999999999995,"5min":0.88400000000000001,"15min":1.1930000000000001},"min":{"1min":0.34899999999999998,"5min":0.308,"15min":0.30099999999999999},"max":{"1min":1.3280000000000001,"5min":6.9619999999999997,"15min":11.4},"last":0.34899999999999998},{"interface":"front","average":{"1min":0.996,"5min":0.89300000000000002,"15min":1.2330000000000001},"min":{"1min":0.34999999999999998,"5min":0.34200000000000003,"15min":0.33000000000000002},"max":{"1min":6.6760000000000002,"5min":6.6760000000000002,"15min":9.2080000000000002},"last":0.43099999999999999}]},{"osd":2,"last update":"Sat Jun 3 20:18:55 2023","interfaces":[{"interface":"back","average":{"1min":4.4829999999999997,"5min":4.835,"15min":3.746},"min":{"1min":0.35899999999999999,"5min":0.33600000000000002,"15min":0.32400000000000001},"max":{"1min":72.831000000000003,"5min":216.93700000000001,"15min":286.83600000000001},"last":1.7789999999999999},{"interface":"front","average":{"1min":5.4450000000000003,"5min":5.0529999999999999,"15min":3.8149999999999999},"min":{"1min":0.315,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":72.787000000000006,"5min":218.119,"15min":287.32900000000001},"last":2.6579999999999999}]},{"osd":3,"last update":"Sat Jun 3 20:18:44 2023","interfaces":[{"interface":"back","average":{"1min":5.1859999999999999,"5min":5.8639999999999999,"15min":3.4460000000000002},"min":{"1min":0.38200000000000001,"5min":0.38200000000000001,"15min":0.30299999999999999},"max":{"1min":74.039000000000001,"5min":265.40499999999997,"15min":265.40499999999997},"last":2.0710000000000002},{"interface":"front","average":{"1min":5.6790000000000003,"5min":5.9690000000000003,"15min":3.516},"min":{"1min":0.41199999999999998,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":73.001999999999995,"5min":265.43599999999998,"15min":265.43599999999998},"last":0.49099999999999999}]},{"osd":4,"last update":"Sat Jun 3 20:10:54 2023","interfaces":[{"interface":"back","average":{"1min":1.417,"5min":1.8879999999999999,"15min":1.3260000000000001},"min":{"1min":0.33900000000000002,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":7.1189999999999998,"5min":21.888999999999999,"15min":21.888999999999999},"last":0.40799999999999997},{"interface":"front","average":{"1min":1.484,"5min":2.1739999999999999,"15min":1.671},"min":{"1min":0.60799999999999998,"5min":0.443,"15min":0.38800000000000001},"max":{"1min":4.8600000000000003,"5min":10.507999999999999,"15min":12.917},"last":0.48899999999999999}]},{"osd":5,"last update":"Sat Jun 3 20:18:44 2023","interfaces":[{"interface":"back","average":{"1min":0.56999999999999995,"5min":0.96499999999999997,"15min":1.423},"min":{"1min":0.38800000000000001,"5min":0.376,"15min":0.28899999999999998},"max":{"1min":0.93799999999999994,"5min":6.9269999999999996,"15min":11.443},"last":2.1829999999999998},{"interface":"front","average":{"1min":0.94499999999999995,"5min":1.0880000000000001,"15min":1.667},"min":{"1min":0.33500000000000002,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":6.6920000000000002,"5min":9.1099999999999994,"15min":20.306999999999999},"last":0.52900000000000003}]},{"osd":6,"last update":"Sat Jun 3 20:18:44 2023","interfaces":[{"interface":"back","average":{"1min":0.996,"5min":1.1830000000000001,"15min":1.7410000000000001},"min":{"1min":0.46000000000000002,"5min":0.34799999999999998,"15min":0.34799999999999998},"max":{"1min":6.6559999999999997,"5min":9.2200000000000006,"15min":11.256},"last":0.59299999999999997},{"interface":"front","average":{"1min":0.63,"5min":1.1990000000000001,"15min":1.7749999999999999},"min":{"1min":0.435,"5min":0.38700000000000001,"15min":0.26900000000000002},"max":{"1min":0.97599999999999998,"5min":9.3219999999999992,"15min":10.894},"last":0.57599999999999996}]}]},{"osd":6,"up_from":0,"seq":180388627016,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":1,"up_from":0,"seq":442381631814,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":0,"up_from":86,"seq":369367187840,"num_pgs":50,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3344060,"kb_used_data":2761332,"kb_used_omap":5,"kb_used_meta":582586,"kb_avail":90401092,"statfs":{"total":95995035648,"available":92570718208,"internally_reserved":0,"allocated":2827603968,"data_stored":2826723320,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":596568698},"hb_peers":[1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,2,2,2,1],"upper_bound":64},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Jun 3 20:18:44 2023","interfaces":[{"interface":"back","average":{"1min":0.79800000000000004,"5min":0.77000000000000002,"15min":0.91600000000000004},"min":{"1min":0.23999999999999999,"5min":0.21099999999999999,"15min":0.19400000000000001},"max":{"1min":3.2080000000000002,"5min":6.8819999999999997,"15min":9.3040000000000003},"last":4.1109999999999998},{"interface":"front","average":{"1min":1.2769999999999999,"5min":1.1020000000000001,"15min":1.3109999999999999},"min":{"1min":0.23699999999999999,"5min":0.23699999999999999,"15min":0.22600000000000001},"max":{"1min":6.7750000000000004,"5min":7.46,"15min":11.736000000000001},"last":0.42099999999999999}]},{"osd":2,"last update":"Sat Jun 3 20:18:44 2023","interfaces":[{"interface":"back","average":{"1min":1.8720000000000001,"5min":1.2490000000000001,"15min":1.2569999999999999},"min":{"1min":0.223,"5min":0.22,"15min":0.193},"max":{"1min":6.8570000000000002,"5min":10.303000000000001,"15min":11.901},"last":4.2300000000000004},{"interface":"front","average":{"1min":2.5739999999999998,"5min":1.5529999999999999,"15min":1.4059999999999999},"min":{"1min":0.30199999999999999,"5min":0.25800000000000001,"15min":0.255},"max":{"1min":11.988,"5min":11.988,"15min":17.315999999999999},"last":4.0759999999999996}]},{"osd":3,"last update":"Sat Jun 3 20:18:31 2023","interfaces":[{"interface":"back","average":{"1min":4.4249999999999998,"5min":2.9180000000000001,"15min":1.859},"min":{"1min":0.30499999999999999,"5min":0.251,"15min":0.251},"max":{"1min":55.505000000000003,"5min":57.790999999999997,"15min":57.790999999999997},"last":4.2000000000000002},{"interface":"front","average":{"1min":4.4240000000000004,"5min":3.0230000000000001,"15min":1.9910000000000001},"min":{"1min":0.33900000000000002,"5min":0.33900000000000002,"15min":0.28100000000000003},"max":{"1min":55.710999999999999,"5min":58.646000000000001,"15min":58.646000000000001},"last":5.0030000000000001}]},{"osd":4,"last update":"Sat Jun 3 20:10:59 2023","interfaces":[{"interface":"back","average":{"1min":1.9370000000000001,"5min":1.7110000000000001,"15min":1.3400000000000001},"min":{"1min":0.45200000000000001,"5min":0.318,"15min":0.318},"max":{"1min":10.025,"5min":10.025,"15min":10.025},"last":0.497},{"interface":"front","average":{"1min":1.673,"5min":1.8160000000000001,"15min":1.429},"min":{"1min":0.56699999999999995,"5min":0.375,"15min":0.375},"max":{"1min":4.6059999999999999,"5min":13.058999999999999,"15min":13.058999999999999},"last":0.75900000000000001}]},{"osd":5,"last update":"Sat Jun 3 20:18:11 2023","interfaces":[{"interface":"back","average":{"1min":1.655,"5min":1.615,"15min":1.915},"min":{"1min":0.432,"5min":0.317,"15min":0.317},"max":{"1min":5.7670000000000003,"5min":8.3350000000000009,"15min":9.3829999999999991},"last":0.61599999999999999},{"interface":"front","average":{"1min":1.129,"5min":1.224,"15min":1.5409999999999999},"min":{"1min":0.42499999999999999,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":4.6799999999999997,"5min":8.6950000000000003,"15min":10.159000000000001},"last":4.2999999999999998}]},{"osd":6,"last update":"Sat Jun 3 20:18:11 2023","interfaces":[{"interface":"back","average":{"1min":1.137,"5min":1.2410000000000001,"15min":1.4970000000000001},"min":{"1min":0.36299999999999999,"5min":0.35399999999999998,"15min":0.35399999999999998},"max":{"1min":4.6440000000000001,"5min":10.192,"15min":10.192},"last":4.2649999999999997},{"interface":"front","average":{"1min":1.1299999999999999,"5min":1.2490000000000001,"15min":1.5640000000000001},"min":{"1min":0.47299999999999998,"5min":0.38300000000000001,"15min":0.38300000000000001},"max":{"1min":3.133,"5min":7.5800000000000001,"15min":13.117000000000001},"last":4.1449999999999996}]},{"osd":7,"last update":"Sat Jun 3 20:18:11 2023","interfaces":[{"interface":"back","average":{"1min":1.1619999999999999,"5min":1.8200000000000001,"15min":1.8280000000000001},"min":{"1min":0.39800000000000002,"5min":0.39400000000000002,"15min":0.39400000000000002},"max":{"1min":4.6020000000000003,"5min":13.470000000000001,"15min":13.470000000000001},"last":4.3280000000000003},{"interface":"front","average":{"1min":1.1910000000000001,"5min":1.744,"15min":1.8169999999999999},"min":{"1min":0.52000000000000002,"5min":0.40300000000000002,"15min":0.40300000000000002},"max":{"1min":3.1579999999999999,"5min":12.41,"15min":12.41},"last":4.1719999999999997}]}]},{"osd":2,"up_from":114,"seq":489626272019,"num_pgs":36,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":4288624,"kb_used_data":3967208,"kb_used_omap":7,"kb_used_meta":321336,"kb_avail":89456528,"statfs":{"total":95995035648,"available":91603484672,"internally_reserved":0,"allocated":4062420992,"data_stored":4061613638,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8111,"internal_metadata":329048145},"hb_peers":[0,1,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,0,0,3],"upper_bound":128},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 3 20:18:49 2023","interfaces":[{"interface":"back","average":{"1min":1.895,"5min":2.5350000000000001,"15min":2.7719999999999998},"min":{"1min":0.20300000000000001,"5min":0.17799999999999999,"15min":0.17799999999999999},"max":{"1min":7.585,"5min":116.919,"15min":273.51100000000002},"last":0.40899999999999997},{"interface":"front","average":{"1min":1.2709999999999999,"5min":2.5249999999999999,"15min":2.9119999999999999},"min":{"1min":0.20399999999999999,"5min":0.20399999999999999,"15min":0.20399999999999999},"max":{"1min":6.2300000000000004,"5min":117.09099999999999,"15min":273.67399999999998},"last":0.503}]},{"osd":1,"last update":"Sat Jun 3 20:18:49 2023","interfaces":[{"interface":"back","average":{"1min":0.85599999999999998,"5min":0.85599999999999998,"15min":2.1469999999999998},"min":{"1min":0.253,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":2.96,"5min":5.6280000000000001,"15min":273.46499999999997},"last":0.42499999999999999},{"interface":"front","average":{"1min":2.1190000000000002,"5min":1.2,"15min":2.375},"min":{"1min":0.27100000000000002,"5min":0.25700000000000001,"15min":0.246},"max":{"1min":8.0679999999999996,"5min":8.0679999999999996,"15min":274.38400000000001},"last":0.439}]},{"osd":3,"last update":"Sat Jun 3 20:18:41 2023","interfaces":[{"interface":"back","average":{"1min":2.0139999999999998,"5min":2.5609999999999999,"15min":2.794},"min":{"1min":0.26800000000000002,"5min":0.245,"15min":0.24399999999999999},"max":{"1min":13.558,"5min":103.714,"15min":273.71899999999999},"last":4.9329999999999998},{"interface":"front","average":{"1min":2.9460000000000002,"5min":2.9500000000000002,"15min":2.9929999999999999},"min":{"1min":0.31,"5min":0.29699999999999999,"15min":0.23899999999999999},"max":{"1min":9.3699999999999992,"5min":104.05200000000001,"15min":274.21100000000001},"last":7.9980000000000002}]},{"osd":4,"last update":"Sat Jun 3 20:10:59 2023","interfaces":[{"interface":"back","average":{"1min":16.218,"5min":4.9749999999999996,"15min":2.2519999999999998},"min":{"1min":0.48699999999999999,"5min":0.32800000000000001,"15min":0.32700000000000001},"max":{"1min":273.75700000000001,"5min":273.75700000000001,"15min":273.75700000000001},"last":1.6679999999999999},{"interface":"front","average":{"1min":15.512,"5min":4.7480000000000002,"15min":2.1850000000000001},"min":{"1min":0.36199999999999999,"5min":0.35699999999999998,"15min":0.32600000000000001},"max":{"1min":274.27300000000002,"5min":274.27300000000002,"15min":274.27300000000002},"last":1.585}]},{"osd":5,"last update":"Sat Jun 3 20:18:49 2023","interfaces":[{"interface":"back","average":{"1min":1.508,"5min":1.1519999999999999,"15min":2.1440000000000001},"min":{"1min":0.47299999999999998,"5min":0.32100000000000001,"15min":0.317},"max":{"1min":4.2039999999999997,"5min":7.54,"15min":273.78800000000001},"last":0.85699999999999998},{"interface":"front","average":{"1min":0.94799999999999995,"5min":1.129,"15min":2.3039999999999998},"min":{"1min":0.33300000000000002,"5min":0.33300000000000002,"15min":0.32800000000000001},"max":{"1min":3.7909999999999999,"5min":5.7569999999999997,"15min":274.15800000000002},"last":1.01}]},{"osd":6,"last update":"Sat Jun 3 20:18:49 2023","interfaces":[{"interface":"back","average":{"1min":1.577,"5min":1.2150000000000001,"15min":2.371},"min":{"1min":0.54500000000000004,"5min":0.35299999999999998,"15min":0.34200000000000003},"max":{"1min":4.1719999999999997,"5min":7.4119999999999999,"15min":274.03899999999999},"last":0.999},{"interface":"front","average":{"1min":0.98899999999999999,"5min":1.1200000000000001,"15min":2.4100000000000001},"min":{"1min":0.39300000000000002,"5min":0.35099999999999998,"15min":0.34899999999999998},"max":{"1min":3.839,"5min":5.6630000000000003,"15min":274.06299999999999},"last":0.89200000000000002}]},{"osd":7,"last update":"Sat Jun 3 20:18:49 2023","interfaces":[{"interface":"back","average":{"1min":1.6000000000000001,"5min":1.464,"15min":2.5209999999999999},"min":{"1min":0.48499999999999999,"5min":0.34799999999999998,"15min":0.317},"max":{"1min":4.234,"5min":9.0220000000000002,"15min":274.54000000000002},"last":0.97099999999999997},{"interface":"front","average":{"1min":1.02,"5min":1.464,"15min":2.5369999999999999},"min":{"1min":0.39400000000000002,"5min":0.39400000000000002,"15min":0.39400000000000002},"max":{"1min":3.8759999999999999,"5min":9.0039999999999996,"15min":274.12700000000001},"last":1.032}]}]},{"osd":3,"up_from":119,"seq":511101108491,"num_pgs":55,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":4629908,"kb_used_data":4241740,"kb_used_omap":2,"kb_used_meta":388157,"kb_avail":89115244,"statfs":{"total":95995035648,"available":91254009856,"internally_reserved":0,"allocated":4343541760,"data_stored":4342693431,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":397472821},"hb_peers":[0,1,2,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,2,4,2,2],"upper_bound":64},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 3 20:18:38 2023","interfaces":[{"interface":"back","average":{"1min":6.8760000000000003,"5min":2.4609999999999999,"15min":2.0600000000000001},"min":{"1min":0.224,"5min":0.20499999999999999,"15min":0.19},"max":{"1min":97.164000000000001,"5min":97.164000000000001,"15min":97.164000000000001},"last":1.9610000000000001},{"interface":"front","average":{"1min":6.7359999999999998,"5min":2.351,"15min":2.1520000000000001},"min":{"1min":0.183,"5min":0.183,"15min":0.183},"max":{"1min":95.816000000000003,"5min":95.816000000000003,"15min":95.816000000000003},"last":1.5549999999999999}]},{"osd":1,"last update":"Sat Jun 3 20:18:38 2023","interfaces":[{"interface":"back","average":{"1min":1.9370000000000001,"5min":1.1120000000000001,"15min":1.0920000000000001},"min":{"1min":0.26900000000000002,"5min":0.249,"15min":0.214},"max":{"1min":8.5299999999999994,"5min":8.5299999999999994,"15min":10.749000000000001},"last":1.7010000000000001},{"interface":"front","average":{"1min":1.8,"5min":1.2849999999999999,"15min":1.044},"min":{"1min":0.23499999999999999,"5min":0.20699999999999999,"15min":0.20699999999999999},"max":{"1min":9.4969999999999999,"5min":10.304,"15min":10.304},"last":1.587}]},{"osd":2,"last update":"Sat Jun 3 20:18:38 2023","interfaces":[{"interface":"back","average":{"1min":2.0419999999999998,"5min":1.7969999999999999,"15min":1.6120000000000001},"min":{"1min":0.25700000000000001,"5min":0.22900000000000001,"15min":0.217},"max":{"1min":5.8970000000000002,"5min":29.106999999999999,"15min":62.527999999999999},"last":1.7270000000000001},{"interface":"front","average":{"1min":2.915,"5min":1.8069999999999999,"15min":1.603},"min":{"1min":0.25,"5min":0.23599999999999999,"15min":0.214},"max":{"1min":8.7219999999999995,"5min":29.175999999999998,"15min":57.935000000000002},"last":1.7629999999999999}]},{"osd":4,"last update":"Sat Jun 3 20:10:53 2023","interfaces":[{"interface":"back","average":{"1min":2.0379999999999998,"5min":1.98,"15min":1.413},"min":{"1min":0.42399999999999999,"5min":0.38200000000000001,"15min":0.38200000000000001},"max":{"1min":9.8360000000000003,"5min":12.757,"15min":12.757},"last":1.968},{"interface":"front","average":{"1min":2.1749999999999998,"5min":1.744,"15min":1.333},"min":{"1min":0.39000000000000001,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":10.122,"5min":10.122,"15min":10.122},"last":0.29099999999999998}]},{"osd":5,"last update":"Sat Jun 3 20:18:38 2023","interfaces":[{"interface":"back","average":{"1min":1.784,"5min":1.7190000000000001,"15min":1.3859999999999999},"min":{"1min":0.35899999999999999,"5min":0.313,"15min":0.313},"max":{"1min":7.2489999999999997,"5min":10.275,"15min":10.275},"last":1.665},{"interface":"front","average":{"1min":1.649,"5min":1.198,"15min":1.391},"min":{"1min":0.373,"5min":0.34100000000000003,"15min":0.314},"max":{"1min":5.8760000000000003,"5min":5.8760000000000003,"15min":12.56},"last":0.32900000000000001}]},{"osd":6,"last update":"Sat Jun 3 20:18:38 2023","interfaces":[{"interface":"back","average":{"1min":1.6890000000000001,"5min":1.4379999999999999,"15min":1.607},"min":{"1min":0.42899999999999999,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":5.9139999999999997,"5min":8.5500000000000007,"15min":13.718},"last":0.376},{"interface":"front","average":{"1min":2.7690000000000001,"5min":1.6899999999999999,"15min":1.778},"min":{"1min":0.44900000000000001,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":8.7279999999999998,"5min":8.7279999999999998,"15min":13.734999999999999},"last":1.8380000000000001}]},{"osd":7,"last update":"Sat Jun 3 20:18:38 2023","interfaces":[{"interface":"back","average":{"1min":2.8239999999999998,"5min":1.6739999999999999,"15min":1.583},"min":{"1min":0.41499999999999998,"5min":0.35599999999999998,"15min":0.35599999999999998},"max":{"1min":8.7149999999999999,"5min":8.7149999999999999,"15min":10.858000000000001},"last":1.8700000000000001},{"interface":"front","average":{"1min":1.802,"5min":2.016,"15min":1.8380000000000001},"min":{"1min":0.38,"5min":0.34899999999999998,"15min":0.34899999999999998},"max":{"1min":7.2199999999999998,"5min":18.431999999999999,"15min":18.431999999999999},"last":1.6339999999999999}]}]},{"osd":4,"up_from":124,"seq":532575944879,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1005320,"kb_used_data":818944,"kb_used_omap":10,"kb_used_meta":186357,"kb_avail":92739832,"statfs":{"total":95995035648,"available":94965587968,"internally_reserved":0,"allocated":838598656,"data_stored":837828790,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":10652,"internal_metadata":190830180},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 3 20:11:03 2023","interfaces":[{"interface":"back","average":{"1min":1.002,"5min":4.8399999999999999,"15min":2.468},"min":{"1min":0.58599999999999997,"5min":0.29499999999999998,"15min":0.254},"max":{"1min":3.0139999999999998,"5min":214.72999999999999,"15min":214.72999999999999},"last":2.1539999999999999},{"interface":"front","average":{"1min":2.04,"5min":5.327,"15min":2.6459999999999999},"min":{"1min":0.41399999999999998,"5min":0.40200000000000002,"15min":0.32400000000000001},"max":{"1min":15,"5min":214.76499999999999,"15min":214.76499999999999},"last":0.625}]},{"osd":1,"last update":"Sat Jun 3 20:11:03 2023","interfaces":[{"interface":"back","average":{"1min":1.1439999999999999,"5min":1.639,"15min":1.55},"min":{"1min":0.371,"5min":0.35199999999999998,"15min":0.28100000000000003},"max":{"1min":3.8210000000000002,"5min":10.446,"15min":58.32},"last":0.84599999999999997},{"interface":"front","average":{"1min":0.878,"5min":1.4990000000000001,"15min":1.4610000000000001},"min":{"1min":0.498,"5min":0.34999999999999998,"15min":0.307},"max":{"1min":1.8,"5min":12.919,"15min":58.286000000000001},"last":2.2429999999999999}]},{"osd":2,"last update":"Sat Jun 3 20:11:03 2023","interfaces":[{"interface":"back","average":{"1min":13.406000000000001,"5min":4.6070000000000002,"15min":2.1419999999999999},"min":{"1min":0.55000000000000004,"5min":0.317,"15min":0.30299999999999999},"max":{"1min":248.65799999999999,"5min":248.65799999999999,"15min":248.65799999999999},"last":2.2770000000000001},{"interface":"front","average":{"1min":14.144,"5min":4.7720000000000002,"15min":2.1890000000000001},"min":{"1min":0.46700000000000003,"5min":0.373,"15min":0.34499999999999997},"max":{"1min":249.077,"5min":249.077,"15min":249.077},"last":0.66200000000000003}]},{"osd":3,"last update":"Sat Jun 3 20:11:03 2023","interfaces":[{"interface":"back","average":{"1min":1.2330000000000001,"5min":1.778,"15min":1.6240000000000001},"min":{"1min":0.40600000000000003,"5min":0.35199999999999998,"15min":0.32500000000000001},"max":{"1min":3.7759999999999998,"5min":10.301,"15min":57.954999999999998},"last":1.0920000000000001},{"interface":"front","average":{"1min":1.226,"5min":1.6479999999999999,"15min":1.613},"min":{"1min":0.50700000000000001,"5min":0.40300000000000002,"15min":0.33700000000000002},"max":{"1min":4.9210000000000003,"5min":12.890000000000001,"15min":57.889000000000003},"last":2.2050000000000001}]},{"osd":5,"last update":"Sat Jun 3 20:11:03 2023","interfaces":[{"interface":"back","average":{"1min":1.4590000000000001,"5min":2.0819999999999999,"15min":1.623},"min":{"1min":0.35899999999999999,"5min":0.32000000000000001,"15min":0.313},"max":{"1min":6.6859999999999999,"5min":11.311,"15min":86.576999999999998},"last":0.47599999999999998},{"interface":"front","average":{"1min":1.5269999999999999,"5min":2.1760000000000002,"15min":1.7390000000000001},"min":{"1min":0.32900000000000001,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":7.9269999999999996,"5min":10.997999999999999,"15min":88.013000000000005},"last":4.9119999999999999}]},{"osd":6,"last update":"Sat Jun 3 20:11:03 2023","interfaces":[{"interface":"back","average":{"1min":1.8140000000000001,"5min":2.7429999999999999,"15min":1.5309999999999999},"min":{"1min":0.36099999999999999,"5min":0.34599999999999997,"15min":0.34599999999999997},"max":{"1min":6.6520000000000001,"5min":14.244999999999999,"15min":14.244999999999999},"last":2.109},{"interface":"front","average":{"1min":2.4329999999999998,"5min":2.8679999999999999,"15min":1.629},"min":{"1min":0.42599999999999999,"5min":0.40100000000000002,"15min":0.36499999999999999},"max":{"1min":10.359,"5min":12.145,"15min":12.145},"last":0.81200000000000006}]},{"osd":7,"last update":"Sat Jun 3 20:11:03 2023","interfaces":[{"interface":"back","average":{"1min":1.7270000000000001,"5min":2.052,"15min":1.55},"min":{"1min":0.44,"5min":0.41199999999999998,"15min":0.39700000000000002},"max":{"1min":9.6210000000000004,"5min":10.388,"15min":10.388},"last":4.8150000000000004},{"interface":"front","average":{"1min":1.0629999999999999,"5min":1.7729999999999999,"15min":1.486},"min":{"1min":0.44800000000000001,"5min":0.34200000000000003,"15min":0.31900000000000001},"max":{"1min":1.8260000000000001,"5min":12.951000000000001,"15min":12.951000000000001},"last":2.3109999999999999}]}]},{"osd":5,"up_from":36,"seq":154618823250,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1847032,"kb_used_data":1183536,"kb_used_omap":0,"kb_used_meta":663488,"kb_avail":91898120,"statfs":{"total":95995035648,"available":94103674880,"internally_reserved":0,"allocated":1211940864,"data_stored":1211410169,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":679411712},"hb_peers":[0,1,2,3,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1],"upper_bound":2},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 3 20:17:58 2023","interfaces":[{"interface":"back","average":{"1min":4.923,"5min":1.988,"15min":2.3599999999999999},"min":{"1min":0.312,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":86.424999999999997,"5min":86.424999999999997,"15min":185.29900000000001},"last":0.37},{"interface":"front","average":{"1min":4.9880000000000004,"5min":2.04,"15min":2.6379999999999999},"min":{"1min":0.36199999999999999,"5min":0.316,"15min":0.26300000000000001},"max":{"1min":86.388999999999996,"5min":86.388999999999996,"15min":185.44200000000001},"last":0.39300000000000002}]},{"osd":1,"last update":"Sat Jun 3 20:17:58 2023","interfaces":[{"interface":"back","average":{"1min":0.64500000000000002,"5min":0.98799999999999999,"15min":1.6000000000000001},"min":{"1min":0.36499999999999999,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":1.772,"5min":6.1189999999999998,"15min":47.747999999999998},"last":0.41499999999999998},{"interface":"front","average":{"1min":0.89600000000000002,"5min":0.94499999999999995,"15min":1.556},"min":{"1min":0.32800000000000001,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":5.7469999999999999,"5min":5.8209999999999997,"15min":48.015000000000001},"last":0.45500000000000002}]},{"osd":2,"last update":"Sat Jun 3 20:17:58 2023","interfaces":[{"interface":"back","average":{"1min":4.8029999999999999,"5min":5.4130000000000003,"15min":3.0720000000000001},"min":{"1min":0.40300000000000002,"5min":0.33400000000000002,"15min":0.318},"max":{"1min":86.757999999999996,"5min":216.881,"15min":216.881},"last":1.421},{"interface":"front","average":{"1min":5.5679999999999996,"5min":5.6520000000000001,"15min":3.419},"min":{"1min":0.372,"5min":0.34100000000000003,"15min":0.314},"max":{"1min":87.400999999999996,"5min":216.751,"15min":216.751},"last":0.503}]},{"osd":3,"last update":"Sat Jun 3 20:18:34 2023","interfaces":[{"interface":"back","average":{"1min":5.1710000000000003,"5min":6.593,"15min":3.3900000000000001},"min":{"1min":0.49099999999999999,"5min":0.36299999999999999,"15min":0.317},"max":{"1min":54.430999999999997,"5min":264.822,"15min":264.822},"last":0.56499999999999995},{"interface":"front","average":{"1min":5.1289999999999996,"5min":6.3689999999999998,"15min":3.1070000000000002},"min":{"1min":0.40999999999999998,"5min":0.379,"15min":0.35799999999999998},"max":{"1min":54.539999999999999,"5min":264.75,"15min":264.75},"last":0.58299999999999996}]},{"osd":4,"last update":"Sat Jun 3 20:11:00 2023","interfaces":[{"interface":"back","average":{"1min":2.2930000000000001,"5min":2.4510000000000001,"15min":1.6839999999999999},"min":{"1min":0.44,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":7.5910000000000002,"5min":14.951000000000001,"15min":14.951000000000001},"last":0.81899999999999995},{"interface":"front","average":{"1min":1.619,"5min":2.0129999999999999,"15min":1.3799999999999999},"min":{"1min":0.42299999999999999,"5min":0.376,"15min":0.376},"max":{"1min":7.2000000000000002,"5min":12.109,"15min":12.109},"last":0.66900000000000004}]},{"osd":6,"last update":"Sat Jun 3 20:17:58 2023","interfaces":[{"interface":"back","average":{"1min":0.79600000000000004,"5min":1.234,"15min":1.7589999999999999},"min":{"1min":0.44700000000000001,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":2.056,"5min":9.7029999999999994,"15min":22.945},"last":0.60699999999999998},{"interface":"front","average":{"1min":0.84499999999999997,"5min":1.264,"15min":1.9159999999999999},"min":{"1min":0.51300000000000001,"5min":0.35799999999999998,"15min":0.35799999999999998},"max":{"1min":1.857,"5min":5.625,"15min":22.428000000000001},"last":0.64500000000000002}]},{"osd":7,"last update":"Sat Jun 3 20:18:44 2023","interfaces":[{"interface":"back","average":{"1min":0.77000000000000002,"5min":1.3160000000000001,"15min":1.6990000000000001},"min":{"1min":0.48399999999999999,"5min":0.42699999999999999,"15min":0.36299999999999999},"max":{"1min":1.1879999999999999,"5min":6.9619999999999997,"15min":14.223000000000001},"last":0.71499999999999997},{"interface":"front","average":{"1min":0.873,"5min":1.1910000000000001,"15min":1.3600000000000001},"min":{"1min":0.42999999999999999,"5min":0.373,"15min":0.28799999999999998},"max":{"1min":4.4809999999999999,"5min":8.4489999999999998,"15min":9.0719999999999992},"last":2.5739999999999998}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6928,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4862,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":10652,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":4716,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4932,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":2814246912,"data_stored":2814246912,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":4034596864,"data_stored":4034592792,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4326887424,"data_stored":4326883352,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":1204486144,"data_stored":1204486144,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":3575189504,"data_stored":3575185432,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-06-03T20:19:02.952 INFO:tasks.cephadm.ceph_manager.ceph:increase pool size by 10 2023-06-03T20:19:02.953 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph osd pool set unique_pool_7 pg_num 26 2023-06-03T20:19:02.962 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:19:02 smithi089 bash[75068]: cluster 1685823542.8434372 mon.a (mon.0) 1227 : cluster 3 Health check update: Degraded data redundancy: 48256/289341 objects degraded (16.678%), 14 pgs degraded, 25 pgs undersized (PG_DEGRADED) 2023-06-03T20:19:02.963 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:19:02 smithi089 bash[78452]: cluster 1685823542.8434372 mon.a (mon.0) 1227 : cluster 3 Health check update: Degraded data redundancy: 48256/289341 objects degraded (16.678%), 14 pgs degraded, 25 pgs undersized (PG_DEGRADED) 2023-06-03T20:19:02.973 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:19:03.006 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:19:03.009 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:19:03.010 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:19:03.111 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:19:03.144 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:19:03.146 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:19:03.147 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:19:03.248 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:19:03.259 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:19:02 smithi164 bash[59462]: cluster 1685823542.8434372 mon.a (mon.0) 1227 : cluster 3 Health check update: Degraded data redundancy: 48256/289341 objects degraded (16.678%), 14 pgs degraded, 25 pgs undersized (PG_DEGRADED) 2023-06-03T20:19:03.279 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:19:03.281 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:19:03.281 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:19:03.345 INFO:teuthology.orchestra.run.smithi089.stdout:osd.0: {} 2023-06-03T20:19:03.382 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:19:03.385 INFO:teuthology.orchestra.run.smithi089.stdout:osd.1: {} 2023-06-03T20:19:03.412 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:19:03.416 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:19:03.417 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:19:03.425 INFO:teuthology.orchestra.run.smithi089.stdout:osd.2: {} 2023-06-03T20:19:03.465 INFO:teuthology.orchestra.run.smithi089.stdout:osd.3: {} 2023-06-03T20:19:03.509 INFO:teuthology.orchestra.run.smithi089.stdout:osd.5: {} 2023-06-03T20:19:03.518 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:19:03.543 INFO:teuthology.orchestra.run.smithi089.stdout:osd.6: {} 2023-06-03T20:19:03.557 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:19:03.564 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:19:03.565 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:19:03.580 INFO:teuthology.orchestra.run.smithi089.stdout:osd.7: {} 2023-06-03T20:19:03.580 INFO:teuthology.orchestra.run.smithi089.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-06-03T20:19:03.581 INFO:teuthology.orchestra.run.smithi089.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-06-03T20:19:03.581 INFO:teuthology.orchestra.run.smithi089.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-06-03T20:19:03.581 INFO:teuthology.orchestra.run.smithi089.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-06-03T20:19:03.581 INFO:teuthology.orchestra.run.smithi089.stderr:Error ENXIO: problem getting command descriptions from osd.4 2023-06-03T20:19:03.581 INFO:teuthology.orchestra.run.smithi089.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-06-03T20:19:03.581 INFO:teuthology.orchestra.run.smithi089.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-06-03T20:19:03.582 INFO:teuthology.orchestra.run.smithi089.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-06-03T20:19:03.666 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:19:03.702 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:19:03.709 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:19:03.709 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:19:03.810 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:19:03.837 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:19:03.840 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:19:03.841 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:19:03.942 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:19:03.954 INFO:teuthology.orchestra.run.smithi089.stderr:noscrub is unset 2023-06-03T20:19:03.969 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:19:03 smithi089 bash[75068]: cluster 1685823542.89259 mgr.y (mgr.34104) 1292 : cluster 0 pgmap v1287: 89 pgs: 20 active+clean+remapped, 14 active+undersized+degraded, 11 active+undersized, 44 active+clean; 5.8 GiB data, 18 GiB used, 518 GiB / 536 GiB avail; 778 KiB/s wr, 762 op/s; 47988/286410 objects degraded (16.755%); 6/286410 objects misplaced (0.002%) 2023-06-03T20:19:03.969 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:19:03 smithi089 bash[75068]: audit 1685823543.2857578 mon.a (mon.0) 1228 : audit 0 from='client.? 172.21.15.89:0/3627127526' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-03T20:19:03.970 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:19:03 smithi089 bash[75068]: audit 1685823543.3250587 mon.c (mon.2) 561 : audit 1 from='client.? 172.21.15.89:0/3841688828' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-06-03T20:19:03.970 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:19:03 smithi089 bash[75068]: audit 1685823543.3263664 mon.a (mon.0) 1229 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-06-03T20:19:03.974 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:19:03.976 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:19:03.976 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:19:04.077 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:19:04.083 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:19:03 smithi164 bash[59462]: cluster 1685823542.89259 mgr.y (mgr.34104) 1292 : cluster 0 pgmap v1287: 89 pgs: 20 active+clean+remapped, 14 active+undersized+degraded, 11 active+undersized, 44 active+clean; 5.8 GiB data, 18 GiB used, 518 GiB / 536 GiB avail; 778 KiB/s wr, 762 op/s; 47988/286410 objects degraded (16.755%); 6/286410 objects misplaced (0.002%) 2023-06-03T20:19:04.084 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:19:03 smithi164 bash[59462]: audit 1685823543.2857578 mon.a (mon.0) 1228 : audit 0 from='client.? 172.21.15.89:0/3627127526' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-03T20:19:04.084 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:19:03 smithi164 bash[59462]: audit 1685823543.3250587 mon.c (mon.2) 561 : audit 1 from='client.? 172.21.15.89:0/3841688828' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-06-03T20:19:04.084 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:19:03 smithi164 bash[59462]: audit 1685823543.3263664 mon.a (mon.0) 1229 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-06-03T20:19:04.108 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:19:04.111 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:19:04.112 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:19:04.212 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:19:04.221 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:19:03 smithi089 bash[78452]: cluster 1685823542.89259 mgr.y (mgr.34104) 1292 : cluster 0 pgmap v1287: 89 pgs: 20 active+clean+remapped, 14 active+undersized+degraded, 11 active+undersized, 44 active+clean; 5.8 GiB data, 18 GiB used, 518 GiB / 536 GiB avail; 778 KiB/s wr, 762 op/s; 47988/286410 objects degraded (16.755%); 6/286410 objects misplaced (0.002%) 2023-06-03T20:19:04.221 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:19:03 smithi089 bash[78452]: audit 1685823543.2857578 mon.a (mon.0) 1228 : audit 0 from='client.? 172.21.15.89:0/3627127526' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-03T20:19:04.222 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:19:03 smithi089 bash[78452]: audit 1685823543.3250587 mon.c (mon.2) 561 : audit 1 from='client.? 172.21.15.89:0/3841688828' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-06-03T20:19:04.222 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:19:03 smithi089 bash[78452]: audit 1685823543.3263664 mon.a (mon.0) 1229 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-06-03T20:19:04.250 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:19:04.252 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:19:04.253 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:19:04.354 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:19:04.386 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:19:04.388 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:19:04.389 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:19:04.490 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:19:04.525 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:19:04.528 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:19:04.529 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:19:04.630 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:19:04.654 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:19:04.656 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:19:04.657 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:19:04.757 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:19:04.794 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:19:04.797 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:19:04.797 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:19:04.898 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:19:04.938 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:19:04.944 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:19:04.945 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:19:05.028 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:19:04 smithi089 bash[78452]: cluster 1685823543.9292955 mon.a (mon.0) 1230 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-06-03T20:19:05.029 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:19:04 smithi089 bash[78452]: cluster 1685823543.9376273 osd.6 (osd.6) 170 : cluster 3 failed to encode map e227 with expected crc 2023-06-03T20:19:05.029 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:19:04 smithi089 bash[78452]: cluster 1685823543.9386268 osd.5 (osd.5) 164 : cluster 3 failed to encode map e227 with expected crc 2023-06-03T20:19:05.029 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:19:04 smithi089 bash[78452]: audit 1685823543.9435034 mon.a (mon.0) 1231 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-06-03T20:19:05.029 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:19:04 smithi089 bash[78452]: cluster 1685823543.9438682 mon.a (mon.0) 1232 : cluster 0 osdmap e227: 8 total, 7 up, 6 in 2023-06-03T20:19:05.029 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:19:04 smithi089 bash[75068]: cluster 1685823543.9292955 mon.a (mon.0) 1230 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-06-03T20:19:05.030 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:19:04 smithi089 bash[75068]: cluster 1685823543.9376273 osd.6 (osd.6) 170 : cluster 3 failed to encode map e227 with expected crc 2023-06-03T20:19:05.030 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:19:04 smithi089 bash[75068]: cluster 1685823543.9386268 osd.5 (osd.5) 164 : cluster 3 failed to encode map e227 with expected crc 2023-06-03T20:19:05.030 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:19:04 smithi089 bash[75068]: audit 1685823543.9435034 mon.a (mon.0) 1231 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-06-03T20:19:05.030 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:19:04 smithi089 bash[75068]: cluster 1685823543.9438682 mon.a (mon.0) 1232 : cluster 0 osdmap e227: 8 total, 7 up, 6 in 2023-06-03T20:19:05.046 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:19:05.075 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:19:05.078 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:19:05.078 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:19:05.179 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:19:05.208 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:19:05.214 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:19:04 smithi164 bash[59462]: cluster 1685823543.9292955 mon.a (mon.0) 1230 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-06-03T20:19:05.214 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:19:04 smithi164 bash[59462]: cluster 1685823543.9376273 osd.6 (osd.6) 170 : cluster 3 failed to encode map e227 with expected crc 2023-06-03T20:19:05.214 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:19:04 smithi164 bash[59462]: cluster 1685823543.9386268 osd.5 (osd.5) 164 : cluster 3 failed to encode map e227 with expected crc 2023-06-03T20:19:05.214 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:19:04 smithi164 bash[59462]: audit 1685823543.9435034 mon.a (mon.0) 1231 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-06-03T20:19:05.215 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:19:04 smithi164 bash[59462]: cluster 1685823543.9438682 mon.a (mon.0) 1232 : cluster 0 osdmap e227: 8 total, 7 up, 6 in 2023-06-03T20:19:05.215 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:19:05.216 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:19:05.317 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:19:05.358 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:19:05.362 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:19:05.362 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:19:05.463 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:19:05.494 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:19:05.497 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:19:05.498 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:19:05.599 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:19:05.632 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:19:05.638 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:19:05.638 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:19:05.739 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:19:05.778 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:19:05.781 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:19:05.782 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:19:05.883 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:19:05.919 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:19:05.924 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:19:05.925 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:19:05.956 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:19:05 smithi089 bash[75068]: cluster 1685823544.8931742 mgr.y (mgr.34104) 1293 : cluster 0 pgmap v1289: 89 pgs: 20 active+clean+remapped, 14 active+undersized+degraded, 11 active+undersized, 44 active+clean; 5.7 GiB data, 18 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 818 op/s; 46880/280806 objects degraded (16.695%); 6/280806 objects misplaced (0.002%) 2023-06-03T20:19:06.025 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:19:06.058 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:19:06.064 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:19:06.065 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:19:06.165 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:19:06.213 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:19:06.217 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:19:05 smithi089 bash[78452]: cluster 1685823544.8931742 mgr.y (mgr.34104) 1293 : cluster 0 pgmap v1289: 89 pgs: 20 active+clean+remapped, 14 active+undersized+degraded, 11 active+undersized, 44 active+clean; 5.7 GiB data, 18 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 818 op/s; 46880/280806 objects degraded (16.695%); 6/280806 objects misplaced (0.002%) 2023-06-03T20:19:06.219 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:19:06.219 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:19:06.320 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:19:06.333 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:19:05 smithi164 bash[59462]: cluster 1685823544.8931742 mgr.y (mgr.34104) 1293 : cluster 0 pgmap v1289: 89 pgs: 20 active+clean+remapped, 14 active+undersized+degraded, 11 active+undersized, 44 active+clean; 5.7 GiB data, 18 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 818 op/s; 46880/280806 objects degraded (16.695%); 6/280806 objects misplaced (0.002%) 2023-06-03T20:19:06.357 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:19:06.361 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:19:06.362 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:19:06.463 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:19:06.471 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 20:19:06 smithi089 bash[71163]: ::ffff:172.21.15.164 - - [03/Jun/2023:20:19:06] "GET /metrics HTTP/1.1" 200 228931 "" "Prometheus/2.33.4" 2023-06-03T20:19:06.504 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:19:06.509 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:19:06.510 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:19:06.611 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:19:06.656 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-06-03T20:19:06.657 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-06-03T20:19:06.660 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:19:06.666 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:19:06.667 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:19:06.768 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:19:06.805 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:19:06.808 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:19:06.809 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:19:06.910 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:19:06.946 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:19:06.949 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:19:06.951 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:19:07.051 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:19:07.082 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:19:07.085 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:19:07.085 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:19:07.186 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:19:07.221 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:19:07.224 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:19:07.226 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:19:07.327 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:19:07.362 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:19:07.366 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:19:07.366 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:19:07.449 INFO:teuthology.orchestra.run.smithi164.stderr:Inferring config /var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/mon.b/config 2023-06-03T20:19:07.467 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:19:07.502 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:19:07.505 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:19:07.506 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:19:07.607 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:19:07.641 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:19:07.648 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:19:07.649 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:19:07.751 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:19:07.780 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:19:07.783 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:19:07.785 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:19:07.887 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:19:07.923 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:19:07.929 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:19:07.929 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:19:08.030 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:19:08.060 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:19:08.063 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:19:08.063 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:19:08.164 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:19:08.173 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:19:07 smithi089 bash[75068]: cluster 1685823546.894056 mgr.y (mgr.34104) 1294 : cluster 0 pgmap v1290: 89 pgs: 20 active+clean+remapped, 14 active+undersized+degraded, 11 active+undersized, 44 active+clean; 5.6 GiB data, 18 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 819 op/s; 45818/273777 objects degraded (16.736%); 6/273777 objects misplaced (0.002%) 2023-06-03T20:19:08.173 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:19:07 smithi089 bash[75068]: audit 1685823547.1305723 mon.c (mon.2) 562 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T20:19:08.173 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:19:07 smithi089 bash[75068]: cluster 1685823547.8449683 mon.a (mon.0) 1233 : cluster 3 Health check update: Degraded data redundancy: 45818/273777 objects degraded (16.736%), 14 pgs degraded, 25 pgs undersized (PG_DEGRADED) 2023-06-03T20:19:08.174 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:19:07 smithi089 bash[78452]: cluster 1685823546.894056 mgr.y (mgr.34104) 1294 : cluster 0 pgmap v1290: 89 pgs: 20 active+clean+remapped, 14 active+undersized+degraded, 11 active+undersized, 44 active+clean; 5.6 GiB data, 18 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 819 op/s; 45818/273777 objects degraded (16.736%); 6/273777 objects misplaced (0.002%) 2023-06-03T20:19:08.174 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:19:07 smithi089 bash[78452]: audit 1685823547.1305723 mon.c (mon.2) 562 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T20:19:08.174 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:19:07 smithi089 bash[78452]: cluster 1685823547.8449683 mon.a (mon.0) 1233 : cluster 3 Health check update: Degraded data redundancy: 45818/273777 objects degraded (16.736%), 14 pgs degraded, 25 pgs undersized (PG_DEGRADED) 2023-06-03T20:19:08.201 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:19:08.204 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:19:08.205 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:19:08.306 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:19:08.316 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:19:07 smithi164 bash[59462]: cluster 1685823546.894056 mgr.y (mgr.34104) 1294 : cluster 0 pgmap v1290: 89 pgs: 20 active+clean+remapped, 14 active+undersized+degraded, 11 active+undersized, 44 active+clean; 5.6 GiB data, 18 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 819 op/s; 45818/273777 objects degraded (16.736%); 6/273777 objects misplaced (0.002%) 2023-06-03T20:19:08.316 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:19:07 smithi164 bash[59462]: audit 1685823547.1305723 mon.c (mon.2) 562 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T20:19:08.316 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:19:07 smithi164 bash[59462]: cluster 1685823547.8449683 mon.a (mon.0) 1233 : cluster 3 Health check update: Degraded data redundancy: 45818/273777 objects degraded (16.736%), 14 pgs degraded, 25 pgs undersized (PG_DEGRADED) 2023-06-03T20:19:08.339 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:19:08.341 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:19:08.344 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:19:08.445 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:19:08.476 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:19:08.480 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:19:08.481 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:19:08.582 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:19:08.616 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:19:08.622 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:19:08.623 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:19:08.658 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-06-03T20:19:08.724 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:19:08.751 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph osd unset nodeep-scrub 2023-06-03T20:19:08.779 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:19:08.782 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:19:08.783 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:19:08.884 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:19:08.913 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:19:08.916 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:19:08.917 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:19:09.018 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:19:09.062 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:19:09.071 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:19:09.072 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:19:09.173 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:19:09.217 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:19:09.220 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:19:09.221 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:19:09.322 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:19:09.350 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:19:09.352 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:19:09.352 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:19:09.453 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:19:09.503 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:19:09.509 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:19:09.510 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:19:09.611 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:19:09.662 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:19:09.669 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:19:09.670 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:19:09.770 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:19:09.799 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:19:09.801 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:19:09.801 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:19:09.902 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:19:09.931 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:19:09.933 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:19:09.934 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:19:09.978 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:19:09 smithi089 bash[75068]: cluster 1685823548.8949594 mgr.y (mgr.34104) 1295 : cluster 0 pgmap v1291: 89 pgs: 20 active+clean+remapped, 14 active+undersized+degraded, 11 active+undersized, 44 active+clean; 5.5 GiB data, 17 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 835 op/s; 44732/268050 objects degraded (16.688%); 6/268050 objects misplaced (0.002%) 2023-06-03T20:19:10.036 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:19:10.078 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:19:10.084 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:19:10.085 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:19:10.186 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:19:10.221 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:19:10.225 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:19:10.226 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:19:10.242 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:19:09 smithi164 bash[59462]: cluster 1685823548.8949594 mgr.y (mgr.34104) 1295 : cluster 0 pgmap v1291: 89 pgs: 20 active+clean+remapped, 14 active+undersized+degraded, 11 active+undersized, 44 active+clean; 5.5 GiB data, 17 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 835 op/s; 44732/268050 objects degraded (16.688%); 6/268050 objects misplaced (0.002%) 2023-06-03T20:19:10.279 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:19:09 smithi089 bash[78452]: cluster 1685823548.8949594 mgr.y (mgr.34104) 1295 : cluster 0 pgmap v1291: 89 pgs: 20 active+clean+remapped, 14 active+undersized+degraded, 11 active+undersized, 44 active+clean; 5.5 GiB data, 17 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 835 op/s; 44732/268050 objects degraded (16.688%); 6/268050 objects misplaced (0.002%) 2023-06-03T20:19:10.312 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/mon.a/config 2023-06-03T20:19:10.327 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:19:10.356 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:19:10.358 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:19:10.358 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:19:10.459 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:19:10.488 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:19:10.489 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:19:10.490 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:19:10.591 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:19:10.625 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:19:10.629 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:19:10.629 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:19:10.730 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:19:10.782 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:19:10.787 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:19:10.788 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:19:10.889 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:19:10.934 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:19:10.938 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:19:10.939 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:19:11.040 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:19:11.081 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:19:11.085 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:19:11.086 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:19:11.187 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:19:11.225 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:19:11.231 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:19:11.231 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:19:11.332 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:19:11.359 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:19:11.361 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:19:11.362 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:19:11.463 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:19:11.497 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:19:11.503 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:19:11.504 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:19:11.605 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:19:11.633 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:19:11.635 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:19:11.636 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:19:11.737 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:19:11.769 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:19:11.772 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:19:11.773 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:19:11.874 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:19:11.908 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:19:11.908 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:19:11.909 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:19:11.928 INFO:teuthology.orchestra.run.smithi164.stdout:ERROR: (22) Invalid argument 2023-06-03T20:19:11.929 INFO:teuthology.orchestra.run.smithi164.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-06-03T20:19:12.009 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:19:12.021 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:19:11 smithi089 bash[75068]: cluster 1685823550.8956811 mgr.y (mgr.34104) 1296 : cluster 0 pgmap v1292: 89 pgs: 20 active+clean+remapped, 14 active+undersized+degraded, 11 active+undersized, 44 active+clean; 5.4 GiB data, 17 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 845 op/s; 43898/263961 objects degraded (16.630%); 6/263961 objects misplaced (0.002%) 2023-06-03T20:19:12.021 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:19:11 smithi089 bash[78452]: cluster 1685823550.8956811 mgr.y (mgr.34104) 1296 : cluster 0 pgmap v1292: 89 pgs: 20 active+clean+remapped, 14 active+undersized+degraded, 11 active+undersized, 44 active+clean; 5.4 GiB data, 17 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 845 op/s; 43898/263961 objects degraded (16.630%); 6/263961 objects misplaced (0.002%) 2023-06-03T20:19:12.044 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:19:12.047 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:19:12.048 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:19:12.140 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:19:11 smithi164 bash[59462]: cluster 1685823550.8956811 mgr.y (mgr.34104) 1296 : cluster 0 pgmap v1292: 89 pgs: 20 active+clean+remapped, 14 active+undersized+degraded, 11 active+undersized, 44 active+clean; 5.4 GiB data, 17 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 845 op/s; 43898/263961 objects degraded (16.630%); 6/263961 objects misplaced (0.002%) 2023-06-03T20:19:12.150 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:19:12.181 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:19:12.186 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:19:12.187 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:19:12.288 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:19:12.334 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:19:12.339 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:19:12.340 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:19:12.441 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:19:12.480 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:19:12.481 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:19:12.482 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:19:12.583 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:19:12.613 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:19:12.618 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:19:12.619 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:19:12.720 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:19:12.752 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:19:12.756 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:19:12.757 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:19:12.858 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:19:12.903 DEBUG:teuthology.orchestra.run.smithi164:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph daemon osd.5 dump_blocked_ops 2023-06-03T20:19:12.914 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:19:12.917 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:19:12.918 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:19:13.019 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:19:13.059 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:19:13.068 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:19:13.069 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:19:13.170 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:19:13.204 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:19:13.207 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:19:13.208 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:19:13.279 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:19:12 smithi089 bash[75068]: cluster 1685823552.846665 mon.a (mon.0) 1234 : cluster 3 Health check update: Degraded data redundancy: 43898/263961 objects degraded (16.630%), 14 pgs degraded, 25 pgs undersized (PG_DEGRADED) 2023-06-03T20:19:13.279 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:19:12 smithi089 bash[78452]: cluster 1685823552.846665 mon.a (mon.0) 1234 : cluster 3 Health check update: Degraded data redundancy: 43898/263961 objects degraded (16.630%), 14 pgs degraded, 25 pgs undersized (PG_DEGRADED) 2023-06-03T20:19:13.309 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:19:13.319 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:19:12 smithi164 bash[59462]: cluster 1685823552.846665 mon.a (mon.0) 1234 : cluster 3 Health check update: Degraded data redundancy: 43898/263961 objects degraded (16.630%), 14 pgs degraded, 25 pgs undersized (PG_DEGRADED) 2023-06-03T20:19:13.340 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:19:13.343 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:19:13.344 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:19:13.395 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/mon.a/config 2023-06-03T20:19:13.397 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/mon.a/config 2023-06-03T20:19:13.445 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:19:13.478 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:19:13.480 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:19:13.481 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:19:13.582 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:19:13.643 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:19:13.652 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:19:13.653 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:19:13.754 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:19:13.785 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:19:13.788 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:19:13.789 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:19:13.890 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:19:13.922 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:19:13.925 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:19:13.926 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:19:13.989 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:19:13 smithi089 bash[75068]: cluster 1685823552.896667 mgr.y (mgr.34104) 1297 : cluster 0 pgmap v1293: 89 pgs: 20 active+clean+remapped, 14 active+undersized+degraded, 11 active+undersized, 44 active+clean; 5.3 GiB data, 17 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 851 op/s; 43644/260850 objects degraded (16.731%); 6/260850 objects misplaced (0.002%) 2023-06-03T20:19:14.026 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:19:14.035 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:19:13 smithi164 bash[59462]: cluster 1685823552.896667 mgr.y (mgr.34104) 1297 : cluster 0 pgmap v1293: 89 pgs: 20 active+clean+remapped, 14 active+undersized+degraded, 11 active+undersized, 44 active+clean; 5.3 GiB data, 17 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 851 op/s; 43644/260850 objects degraded (16.731%); 6/260850 objects misplaced (0.002%) 2023-06-03T20:19:14.056 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:19:14.059 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:19:14.060 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:19:14.160 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:19:14.201 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:19:14.206 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:19:14.206 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:19:14.279 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:19:13 smithi089 bash[78452]: cluster 1685823552.896667 mgr.y (mgr.34104) 1297 : cluster 0 pgmap v1293: 89 pgs: 20 active+clean+remapped, 14 active+undersized+degraded, 11 active+undersized, 44 active+clean; 5.3 GiB data, 17 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 851 op/s; 43644/260850 objects degraded (16.731%); 6/260850 objects misplaced (0.002%) 2023-06-03T20:19:14.308 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:19:14.345 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:19:14.350 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:19:14.351 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:19:14.452 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:19:14.530 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:19:14.534 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:19:14.535 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:19:14.636 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:19:14.791 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:19:14.796 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:19:14.797 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:19:14.898 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:19:14.931 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:19:14.935 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:19:14.936 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:19:15.037 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:19:15.098 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:19:15.103 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:19:15.103 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:19:15.204 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:19:15.241 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:19:15.243 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:19:15.244 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:19:15.345 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:19:15.374 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:19:15.376 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:19:15.377 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:19:15.478 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:19:15.508 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:19:15.513 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:19:15.514 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:19:15.614 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:19:15.646 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:19:15.655 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:19:15.655 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:19:15.756 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:19:16.012 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:19:16.016 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:19:16.017 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:19:16.118 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:19:16.155 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:19:16.157 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:19:16.157 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:19:16.258 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:19:16.269 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:19:15 smithi164 bash[59462]: cluster 1685823554.8977675 mgr.y (mgr.34104) 1298 : cluster 0 pgmap v1294: 89 pgs: 20 active+clean+remapped, 14 active+undersized+degraded, 11 active+undersized, 44 active+clean; 5.2 GiB data, 17 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 764 op/s; 42681/255681 objects degraded (16.693%); 6/255681 objects misplaced (0.002%) 2023-06-03T20:19:16.278 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 20:19:16 smithi089 bash[71163]: ::ffff:172.21.15.164 - - [03/Jun/2023:20:19:16] "GET /metrics HTTP/1.1" 200 228935 "" "Prometheus/2.33.4" 2023-06-03T20:19:16.279 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:19:15 smithi089 bash[78452]: cluster 1685823554.8977675 mgr.y (mgr.34104) 1298 : cluster 0 pgmap v1294: 89 pgs: 20 active+clean+remapped, 14 active+undersized+degraded, 11 active+undersized, 44 active+clean; 5.2 GiB data, 17 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 764 op/s; 42681/255681 objects degraded (16.693%); 6/255681 objects misplaced (0.002%) 2023-06-03T20:19:16.279 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:19:15 smithi089 bash[75068]: cluster 1685823554.8977675 mgr.y (mgr.34104) 1298 : cluster 0 pgmap v1294: 89 pgs: 20 active+clean+remapped, 14 active+undersized+degraded, 11 active+undersized, 44 active+clean; 5.2 GiB data, 17 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 764 op/s; 42681/255681 objects degraded (16.693%); 6/255681 objects misplaced (0.002%) 2023-06-03T20:19:16.291 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:19:16.293 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:19:16.293 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:19:16.394 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:19:16.431 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:19:16.437 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:19:16.438 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:19:16.539 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:19:16.573 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:19:16.577 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:19:16.578 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:19:16.679 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:19:16.711 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:19:16.713 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:19:16.714 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:19:16.815 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:19:16.845 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:19:16.848 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:19:16.848 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:19:16.949 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:19:17.143 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:19:17.146 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:19:17.147 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:19:17.248 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:19:17.283 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:19:17.286 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:19:17.287 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:19:17.387 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:19:17.432 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:19:17.434 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:19:17.434 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:19:17.536 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:19:17.563 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:19:17.567 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:19:17.567 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:19:17.668 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:19:17.696 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:19:17.698 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:19:17.698 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:19:17.709 INFO:teuthology.orchestra.run.smithi164.stderr:Inferring config /var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/mon.b/config 2023-06-03T20:19:17.799 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:19:17.828 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:19:17.832 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:19:17.833 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:19:17.934 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:19:17.972 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:19:17.974 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:19:17.975 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:19:18.076 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:19:18.111 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:19:18.113 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:19:18.114 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:19:18.215 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:19:18.223 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:19:18 smithi089 bash[75068]: cluster 1685823556.8992975 mgr.y (mgr.34104) 1299 : cluster 0 pgmap v1295: 89 pgs: 20 active+clean+remapped, 14 active+undersized+degraded, 11 active+undersized, 44 active+clean; 5.1 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 881 op/s; 41711/249072 objects degraded (16.747%); 6/249072 objects misplaced (0.002%) 2023-06-03T20:19:18.223 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:19:18 smithi089 bash[75068]: cluster 1685823557.8508015 mon.a (mon.0) 1235 : cluster 3 Health check update: Degraded data redundancy: 41711/249072 objects degraded (16.747%), 14 pgs degraded, 25 pgs undersized (PG_DEGRADED) 2023-06-03T20:19:18.224 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:19:18 smithi089 bash[78452]: cluster 1685823556.8992975 mgr.y (mgr.34104) 1299 : cluster 0 pgmap v1295: 89 pgs: 20 active+clean+remapped, 14 active+undersized+degraded, 11 active+undersized, 44 active+clean; 5.1 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 881 op/s; 41711/249072 objects degraded (16.747%); 6/249072 objects misplaced (0.002%) 2023-06-03T20:19:18.224 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:19:18 smithi089 bash[78452]: cluster 1685823557.8508015 mon.a (mon.0) 1235 : cluster 3 Health check update: Degraded data redundancy: 41711/249072 objects degraded (16.747%), 14 pgs degraded, 25 pgs undersized (PG_DEGRADED) 2023-06-03T20:19:18.244 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:19:18.249 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:19:18.250 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:19:18.333 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:19:17 smithi164 bash[59462]: cluster 1685823556.8992975 mgr.y (mgr.34104) 1299 : cluster 0 pgmap v1295: 89 pgs: 20 active+clean+remapped, 14 active+undersized+degraded, 11 active+undersized, 44 active+clean; 5.1 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 881 op/s; 41711/249072 objects degraded (16.747%); 6/249072 objects misplaced (0.002%) 2023-06-03T20:19:18.334 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:19:17 smithi164 bash[59462]: cluster 1685823557.8508015 mon.a (mon.0) 1235 : cluster 3 Health check update: Degraded data redundancy: 41711/249072 objects degraded (16.747%), 14 pgs degraded, 25 pgs undersized (PG_DEGRADED) 2023-06-03T20:19:18.351 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:19:18.378 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:19:18.380 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:19:18.380 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:19:18.481 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:19:18.511 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:19:18.513 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:19:18.514 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:19:18.615 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:19:18.654 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:19:18.658 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:19:18.659 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:19:18.760 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:19:18.782 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:19:18.783 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:19:18.784 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:19:18.885 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:19:18.913 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:19:18.916 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:19:18.916 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:19:19.017 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:19:19.047 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:19:19.056 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:19:19.056 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:19:19.157 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:19:19.195 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:19:19.197 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:19:19.198 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:19:19.299 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:19:19.329 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:19:19.331 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:19:19.332 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:19:19.433 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:19:19.481 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:19:19.487 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:19:19.488 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:19:19.589 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:19:19.621 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:19:19.625 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:19:19.625 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:19:19.727 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:19:19.762 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:19:19.766 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:19:19.766 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:19:19.867 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:19:19.899 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:19:19.903 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:19:19.904 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:19:20.005 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:19:20.049 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:19:20.050 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:19:20.051 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:19:20.152 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:19:20.160 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:19:19 smithi164 bash[59462]: cluster 1685823558.90042 mgr.y (mgr.34104) 1300 : cluster 0 pgmap v1296: 89 pgs: 20 active+clean+remapped, 14 active+undersized+degraded, 11 active+undersized, 44 active+clean; 5.0 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 823 op/s; 40769/244128 objects degraded (16.700%); 6/244128 objects misplaced (0.002%) 2023-06-03T20:19:20.179 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:19:20.181 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:19:20.182 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:19:20.278 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:19:19 smithi089 bash[78452]: cluster 1685823558.90042 mgr.y (mgr.34104) 1300 : cluster 0 pgmap v1296: 89 pgs: 20 active+clean+remapped, 14 active+undersized+degraded, 11 active+undersized, 44 active+clean; 5.0 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 823 op/s; 40769/244128 objects degraded (16.700%); 6/244128 objects misplaced (0.002%) 2023-06-03T20:19:20.279 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:19:19 smithi089 bash[75068]: cluster 1685823558.90042 mgr.y (mgr.34104) 1300 : cluster 0 pgmap v1296: 89 pgs: 20 active+clean+remapped, 14 active+undersized+degraded, 11 active+undersized, 44 active+clean; 5.0 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 823 op/s; 40769/244128 objects degraded (16.700%); 6/244128 objects misplaced (0.002%) 2023-06-03T20:19:20.282 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:19:20.309 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:19:20.315 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:19:20.316 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:19:20.417 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:19:20.448 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:19:20.451 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:19:20.452 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:19:20.553 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:19:20.603 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:19:20.605 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:19:20.606 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:19:20.707 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:19:20.736 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:19:20.738 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:19:20.738 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:19:20.839 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:19:20.868 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:19:20.871 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:19:20.872 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:19:20.973 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:19:21.007 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:19:21.013 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:19:21.014 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:19:21.115 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:19:21.148 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:19:21.150 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:19:21.151 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:19:21.252 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:19:21.284 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:19:21.286 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:19:21.287 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:19:21.387 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:19:21.426 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:19:21.431 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:19:21.432 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:19:21.533 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:19:21.565 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:19:21.570 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:19:21.571 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:19:21.672 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:19:21.709 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:19:21.715 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:19:21.716 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:19:21.817 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:19:21.850 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:19:21.852 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:19:21.853 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:19:21.954 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:19:21.993 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:19:21.994 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:19:21.994 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:19:22.095 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:19:22.108 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:19:22 smithi089 bash[75068]: cluster 1685823560.9011981 mgr.y (mgr.34104) 1301 : cluster 0 pgmap v1297: 89 pgs: 20 active+clean+remapped, 14 active+undersized+degraded, 11 active+undersized, 44 active+clean; 4.9 GiB data, 16 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 769 op/s; 40010/240330 objects degraded (16.648%); 6/240330 objects misplaced (0.002%) 2023-06-03T20:19:22.150 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:19:22.153 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:19:22.154 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:19:22.255 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:19:22.280 INFO:teuthology.orchestra.run.smithi164.stdout:ERROR: (22) Invalid argument 2023-06-03T20:19:22.281 INFO:teuthology.orchestra.run.smithi164.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-06-03T20:19:22.288 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:19:22.292 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:19:22.293 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:19:22.327 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:19:21 smithi164 bash[59462]: cluster 1685823560.9011981 mgr.y (mgr.34104) 1301 : cluster 0 pgmap v1297: 89 pgs: 20 active+clean+remapped, 14 active+undersized+degraded, 11 active+undersized, 44 active+clean; 4.9 GiB data, 16 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 769 op/s; 40010/240330 objects degraded (16.648%); 6/240330 objects misplaced (0.002%) 2023-06-03T20:19:22.393 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:19:22.402 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:19:22 smithi089 bash[78452]: cluster 1685823560.9011981 mgr.y (mgr.34104) 1301 : cluster 0 pgmap v1297: 89 pgs: 20 active+clean+remapped, 14 active+undersized+degraded, 11 active+undersized, 44 active+clean; 4.9 GiB data, 16 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 769 op/s; 40010/240330 objects degraded (16.648%); 6/240330 objects misplaced (0.002%) 2023-06-03T20:19:22.429 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:19:22.435 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:19:22.435 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:19:22.536 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:19:22.568 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:19:22.570 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:19:22.570 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:19:22.671 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:19:22.700 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:19:22.705 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:19:22.706 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:19:22.806 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:19:22.839 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:19:22.842 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:19:22.843 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:19:22.943 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:19:22.973 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:19:22.977 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:19:22.978 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:19:23.079 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:19:23.112 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:19:23.114 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:19:23.114 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:19:23.159 DEBUG:teuthology.orchestra.run.smithi164:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph daemon osd.5 dump_historic_ops 2023-06-03T20:19:23.216 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:19:23.224 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:19:22 smithi089 bash[75068]: audit 1685823562.1307247 mon.c (mon.2) 563 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T20:19:23.224 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:19:22 smithi089 bash[75068]: cluster 1685823562.8521159 mon.a (mon.0) 1236 : cluster 3 Health check update: Degraded data redundancy: 40010/240330 objects degraded (16.648%), 14 pgs degraded, 25 pgs undersized (PG_DEGRADED) 2023-06-03T20:19:23.224 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:19:22 smithi089 bash[78452]: audit 1685823562.1307247 mon.c (mon.2) 563 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T20:19:23.225 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:19:22 smithi089 bash[78452]: cluster 1685823562.8521159 mon.a (mon.0) 1236 : cluster 3 Health check update: Degraded data redundancy: 40010/240330 objects degraded (16.648%), 14 pgs degraded, 25 pgs undersized (PG_DEGRADED) 2023-06-03T20:19:23.248 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:19:23.255 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:19:23.255 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:19:23.334 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:19:22 smithi164 bash[59462]: audit 1685823562.1307247 mon.c (mon.2) 563 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T20:19:23.334 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:19:22 smithi164 bash[59462]: cluster 1685823562.8521159 mon.a (mon.0) 1236 : cluster 3 Health check update: Degraded data redundancy: 40010/240330 objects degraded (16.648%), 14 pgs degraded, 25 pgs undersized (PG_DEGRADED) 2023-06-03T20:19:23.357 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:19:23.387 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:19:23.389 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:19:23.389 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:19:23.490 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:19:23.525 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:19:23.529 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:19:23.529 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:19:23.630 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:19:23.668 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:19:23.670 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:19:23.670 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:19:23.771 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:19:23.805 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:19:23.808 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:19:23.809 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:19:23.910 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:19:23.943 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:19:23.945 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:19:23.945 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:19:24.034 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:19:24 smithi089 bash[75068]: cluster 1685823562.9019544 mgr.y (mgr.34104) 1302 : cluster 0 pgmap v1298: 89 pgs: 20 active+clean+remapped, 14 active+undersized+degraded, 11 active+undersized, 44 active+clean; 4.8 GiB data, 16 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 741 op/s; 39770/237255 objects degraded (16.763%); 6/237255 objects misplaced (0.003%) 2023-06-03T20:19:24.046 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:19:24.077 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:19:24.080 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:19:24.080 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:19:24.181 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:19:24.210 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:19:24.212 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:19:24.213 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:19:24.314 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:19:24.324 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:19:24 smithi089 bash[78452]: cluster 1685823562.9019544 mgr.y (mgr.34104) 1302 : cluster 0 pgmap v1298: 89 pgs: 20 active+clean+remapped, 14 active+undersized+degraded, 11 active+undersized, 44 active+clean; 4.8 GiB data, 16 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 741 op/s; 39770/237255 objects degraded (16.763%); 6/237255 objects misplaced (0.003%) 2023-06-03T20:19:24.334 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:19:23 smithi164 bash[59462]: cluster 1685823562.9019544 mgr.y (mgr.34104) 1302 : cluster 0 pgmap v1298: 89 pgs: 20 active+clean+remapped, 14 active+undersized+degraded, 11 active+undersized, 44 active+clean; 4.8 GiB data, 16 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 741 op/s; 39770/237255 objects degraded (16.763%); 6/237255 objects misplaced (0.003%) 2023-06-03T20:19:24.350 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:19:24.353 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:19:24.353 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:19:24.454 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:19:24.494 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:19:24.499 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:19:24.499 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:19:24.600 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:19:24.639 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:19:24.642 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:19:24.643 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:19:24.744 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:19:24.785 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:19:24.787 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:19:24.788 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:19:24.889 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:19:24.928 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:19:24.933 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:19:24.934 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:19:25.035 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:19:25.068 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:19:25.073 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:19:25.074 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:19:25.174 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:19:25.245 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:19:25.253 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:19:25.254 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:19:25.354 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:19:25.394 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:19:25.397 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:19:25.397 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:19:25.500 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:19:25.544 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:19:25.546 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:19:25.547 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:19:25.648 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:19:25.676 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:19:25.681 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:19:25.682 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:19:25.783 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:19:25.809 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:19:25.813 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:19:25.814 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:19:25.915 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:19:25.947 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:19:25.952 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:19:25.953 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:19:26.026 INFO:teuthology.orchestra.run.smithi089.stderr:set pool 12 pg_num to 26 2023-06-03T20:19:26.053 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:19:26.093 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:19:26.097 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:19:26.098 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:19:26.199 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:19:26.239 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:19:26.243 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:19:26.243 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:19:26.279 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 20:19:26 smithi089 bash[71163]: ::ffff:172.21.15.164 - - [03/Jun/2023:20:19:26] "GET /metrics HTTP/1.1" 200 228933 "" "Prometheus/2.33.4" 2023-06-03T20:19:26.279 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:19:26 smithi089 bash[78452]: cluster 1685823564.9035854 mgr.y (mgr.34104) 1303 : cluster 0 pgmap v1299: 89 pgs: 20 active+clean+remapped, 14 active+undersized+degraded, 11 active+undersized, 44 active+clean; 4.7 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 795 op/s; 38820/232209 objects degraded (16.718%); 6/232209 objects misplaced (0.003%) 2023-06-03T20:19:26.279 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:19:26 smithi089 bash[78452]: audit 1685823565.647686 mon.b (mon.1) 60 : audit 1 from='client.? 172.21.15.89:0/3043203034' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_7", "var": "pg_num", "val": "26"}]: dispatch 2023-06-03T20:19:26.279 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:19:26 smithi089 bash[78452]: audit 1685823565.6527202 mon.a (mon.0) 1237 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_7", "var": "pg_num", "val": "26"}]: dispatch 2023-06-03T20:19:26.280 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:19:26 smithi089 bash[75068]: cluster 1685823564.9035854 mgr.y (mgr.34104) 1303 : cluster 0 pgmap v1299: 89 pgs: 20 active+clean+remapped, 14 active+undersized+degraded, 11 active+undersized, 44 active+clean; 4.7 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 795 op/s; 38820/232209 objects degraded (16.718%); 6/232209 objects misplaced (0.003%) 2023-06-03T20:19:26.280 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:19:26 smithi089 bash[75068]: audit 1685823565.647686 mon.b (mon.1) 60 : audit 1 from='client.? 172.21.15.89:0/3043203034' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_7", "var": "pg_num", "val": "26"}]: dispatch 2023-06-03T20:19:26.280 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:19:26 smithi089 bash[75068]: audit 1685823565.6527202 mon.a (mon.0) 1237 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_7", "var": "pg_num", "val": "26"}]: dispatch 2023-06-03T20:19:26.334 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:19:26 smithi164 bash[59462]: cluster 1685823564.9035854 mgr.y (mgr.34104) 1303 : cluster 0 pgmap v1299: 89 pgs: 20 active+clean+remapped, 14 active+undersized+degraded, 11 active+undersized, 44 active+clean; 4.7 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 795 op/s; 38820/232209 objects degraded (16.718%); 6/232209 objects misplaced (0.003%) 2023-06-03T20:19:26.334 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:19:26 smithi164 bash[59462]: audit 1685823565.647686 mon.b (mon.1) 60 : audit 1 from='client.? 172.21.15.89:0/3043203034' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_7", "var": "pg_num", "val": "26"}]: dispatch 2023-06-03T20:19:26.334 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:19:26 smithi164 bash[59462]: audit 1685823565.6527202 mon.a (mon.0) 1237 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_7", "var": "pg_num", "val": "26"}]: dispatch 2023-06-03T20:19:26.345 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:19:26.378 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:19:26.381 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:19:26.381 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:19:26.482 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:19:26.525 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:19:26.529 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:19:26.530 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:19:26.630 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:19:26.661 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:19:26.663 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:19:26.664 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:19:26.765 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:19:26.796 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:19:26.800 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:19:26.801 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:19:26.902 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:19:26.936 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:19:26.940 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:19:26.941 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:19:27.042 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:19:27.117 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:19:27.125 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:19:27.126 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:19:27.227 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:19:27.334 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:19:27 smithi164 bash[59462]: cluster 1685823566.0042465 osd.6 (osd.6) 171 : cluster 3 failed to encode map e228 with expected crc 2023-06-03T20:19:27.335 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:19:27 smithi164 bash[59462]: cluster 1685823566.0070949 osd.5 (osd.5) 165 : cluster 3 failed to encode map e228 with expected crc 2023-06-03T20:19:27.335 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:19:27 smithi164 bash[59462]: audit 1685823566.0099082 mon.a (mon.0) 1238 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_7", "var": "pg_num", "val": "26"}]': finished 2023-06-03T20:19:27.335 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:19:27 smithi164 bash[59462]: cluster 1685823566.0101376 mon.a (mon.0) 1239 : cluster 0 osdmap e228: 8 total, 7 up, 6 in 2023-06-03T20:19:27.335 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:19:27 smithi164 bash[59462]: cluster 1685823566.0304508 osd.7 (osd.7) 152 : cluster 3 failed to encode map e228 with expected crc 2023-06-03T20:19:27.335 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:19:27 smithi164 bash[59462]: audit 1685823566.9048626 mon.c (mon.2) 564 : audit 1 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_7", "var": "pg_num_actual", "val": "26"}]: dispatch 2023-06-03T20:19:27.336 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:19:27 smithi164 bash[59462]: audit 1685823566.9062316 mon.a (mon.0) 1240 : audit 1 from='mgr.34104 ' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_7", "var": "pg_num_actual", "val": "26"}]: dispatch 2023-06-03T20:19:27.485 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:19:27 smithi089 bash[75068]: cluster 1685823566.0042465 osd.6 (osd.6) 171 : cluster 3 failed to encode map e228 with expected crc 2023-06-03T20:19:27.486 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:19:27 smithi089 bash[75068]: cluster 1685823566.0070949 osd.5 (osd.5) 165 : cluster 3 failed to encode map e228 with expected crc 2023-06-03T20:19:27.486 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:19:27 smithi089 bash[75068]: audit 1685823566.0099082 mon.a (mon.0) 1238 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_7", "var": "pg_num", "val": "26"}]': finished 2023-06-03T20:19:27.486 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:19:27 smithi089 bash[75068]: cluster 1685823566.0101376 mon.a (mon.0) 1239 : cluster 0 osdmap e228: 8 total, 7 up, 6 in 2023-06-03T20:19:27.486 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:19:27 smithi089 bash[75068]: cluster 1685823566.0304508 osd.7 (osd.7) 152 : cluster 3 failed to encode map e228 with expected crc 2023-06-03T20:19:27.487 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:19:27 smithi089 bash[75068]: audit 1685823566.9048626 mon.c (mon.2) 564 : audit 1 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_7", "var": "pg_num_actual", "val": "26"}]: dispatch 2023-06-03T20:19:27.487 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:19:27 smithi089 bash[75068]: audit 1685823566.9062316 mon.a (mon.0) 1240 : audit 1 from='mgr.34104 ' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_7", "var": "pg_num_actual", "val": "26"}]: dispatch 2023-06-03T20:19:27.487 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:19:27 smithi089 bash[78452]: cluster 1685823566.0042465 osd.6 (osd.6) 171 : cluster 3 failed to encode map e228 with expected crc 2023-06-03T20:19:27.488 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:19:27 smithi089 bash[78452]: cluster 1685823566.0070949 osd.5 (osd.5) 165 : cluster 3 failed to encode map e228 with expected crc 2023-06-03T20:19:27.488 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:19:27 smithi089 bash[78452]: audit 1685823566.0099082 mon.a (mon.0) 1238 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_7", "var": "pg_num", "val": "26"}]': finished 2023-06-03T20:19:27.488 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:19:27 smithi089 bash[78452]: cluster 1685823566.0101376 mon.a (mon.0) 1239 : cluster 0 osdmap e228: 8 total, 7 up, 6 in 2023-06-03T20:19:27.488 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:19:27 smithi089 bash[78452]: cluster 1685823566.0304508 osd.7 (osd.7) 152 : cluster 3 failed to encode map e228 with expected crc 2023-06-03T20:19:27.488 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:19:27 smithi089 bash[78452]: audit 1685823566.9048626 mon.c (mon.2) 564 : audit 1 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_7", "var": "pg_num_actual", "val": "26"}]: dispatch 2023-06-03T20:19:27.489 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:19:27 smithi089 bash[78452]: audit 1685823566.9062316 mon.a (mon.0) 1240 : audit 1 from='mgr.34104 ' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_7", "var": "pg_num_actual", "val": "26"}]: dispatch 2023-06-03T20:19:27.537 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:19:27.539 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:19:27.540 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:19:27.641 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:19:27.666 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:19:27.668 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:19:27.669 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:19:27.769 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:19:27.795 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:19:27.798 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:19:27.798 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:19:27.899 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:19:27.930 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:19:27.932 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:19:27.933 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:19:27.969 INFO:teuthology.orchestra.run.smithi164.stderr:Inferring config /var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/mon.b/config 2023-06-03T20:19:28.029 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:19:28 smithi089 bash[75068]: cluster 1685823566.9043717 mgr.y (mgr.34104) 1304 : cluster 0 pgmap v1301: 89 pgs: 20 active+clean+remapped, 14 active+undersized+degraded, 11 active+undersized, 44 active+clean; 4.6 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 768 op/s; 37947/226017 objects degraded (16.789%); 6/226017 objects misplaced (0.003%) 2023-06-03T20:19:28.029 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:19:28 smithi089 bash[75068]: cluster 1685823567.0229256 osd.6 (osd.6) 172 : cluster 3 failed to encode map e229 with expected crc 2023-06-03T20:19:28.029 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:19:28 smithi089 bash[75068]: audit 1685823567.0311246 mon.a (mon.0) 1241 : audit 1 from='mgr.34104 ' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_7", "var": "pg_num_actual", "val": "26"}]': finished 2023-06-03T20:19:28.030 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:19:28 smithi089 bash[75068]: cluster 1685823567.0315936 mon.a (mon.0) 1242 : cluster 0 osdmap e229: 8 total, 7 up, 6 in 2023-06-03T20:19:28.030 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:19:28 smithi089 bash[75068]: cluster 1685823567.035101 osd.7 (osd.7) 153 : cluster 3 failed to encode map e229 with expected crc 2023-06-03T20:19:28.030 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:19:28 smithi089 bash[75068]: cluster 1685823567.8532622 mon.a (mon.0) 1243 : cluster 3 Health check update: Degraded data redundancy: 37947/226017 objects degraded (16.789%), 14 pgs degraded, 25 pgs undersized (PG_DEGRADED) 2023-06-03T20:19:28.033 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:19:28.063 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:19:28.065 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:19:28.066 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:19:28.167 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:19:28.207 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:19:28.209 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:19:28.209 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:19:28.279 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:19:28 smithi089 bash[78452]: cluster 1685823566.9043717 mgr.y (mgr.34104) 1304 : cluster 0 pgmap v1301: 89 pgs: 20 active+clean+remapped, 14 active+undersized+degraded, 11 active+undersized, 44 active+clean; 4.6 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 768 op/s; 37947/226017 objects degraded (16.789%); 6/226017 objects misplaced (0.003%) 2023-06-03T20:19:28.279 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:19:28 smithi089 bash[78452]: cluster 1685823567.0229256 osd.6 (osd.6) 172 : cluster 3 failed to encode map e229 with expected crc 2023-06-03T20:19:28.279 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:19:28 smithi089 bash[78452]: audit 1685823567.0311246 mon.a (mon.0) 1241 : audit 1 from='mgr.34104 ' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_7", "var": "pg_num_actual", "val": "26"}]': finished 2023-06-03T20:19:28.279 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:19:28 smithi089 bash[78452]: cluster 1685823567.0315936 mon.a (mon.0) 1242 : cluster 0 osdmap e229: 8 total, 7 up, 6 in 2023-06-03T20:19:28.279 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:19:28 smithi089 bash[78452]: cluster 1685823567.035101 osd.7 (osd.7) 153 : cluster 3 failed to encode map e229 with expected crc 2023-06-03T20:19:28.280 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:19:28 smithi089 bash[78452]: cluster 1685823567.8532622 mon.a (mon.0) 1243 : cluster 3 Health check update: Degraded data redundancy: 37947/226017 objects degraded (16.789%), 14 pgs degraded, 25 pgs undersized (PG_DEGRADED) 2023-06-03T20:19:28.310 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:19:28.320 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:19:28 smithi164 bash[59462]: cluster 1685823566.9043717 mgr.y (mgr.34104) 1304 : cluster 0 pgmap v1301: 89 pgs: 20 active+clean+remapped, 14 active+undersized+degraded, 11 active+undersized, 44 active+clean; 4.6 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 768 op/s; 37947/226017 objects degraded (16.789%); 6/226017 objects misplaced (0.003%) 2023-06-03T20:19:28.320 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:19:28 smithi164 bash[59462]: cluster 1685823567.0229256 osd.6 (osd.6) 172 : cluster 3 failed to encode map e229 with expected crc 2023-06-03T20:19:28.320 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:19:28 smithi164 bash[59462]: audit 1685823567.0311246 mon.a (mon.0) 1241 : audit 1 from='mgr.34104 ' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_7", "var": "pg_num_actual", "val": "26"}]': finished 2023-06-03T20:19:28.320 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:19:28 smithi164 bash[59462]: cluster 1685823567.0315936 mon.a (mon.0) 1242 : cluster 0 osdmap e229: 8 total, 7 up, 6 in 2023-06-03T20:19:28.321 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:19:28 smithi164 bash[59462]: cluster 1685823567.035101 osd.7 (osd.7) 153 : cluster 3 failed to encode map e229 with expected crc 2023-06-03T20:19:28.321 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:19:28 smithi164 bash[59462]: cluster 1685823567.8532622 mon.a (mon.0) 1243 : cluster 3 Health check update: Degraded data redundancy: 37947/226017 objects degraded (16.789%), 14 pgs degraded, 25 pgs undersized (PG_DEGRADED) 2023-06-03T20:19:28.338 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:19:28.342 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:19:28.342 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:19:28.443 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:19:28.476 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:19:28.478 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:19:28.479 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:19:28.580 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:19:28.612 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:19:28.615 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:19:28.616 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:19:28.717 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:19:28.747 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:19:28.749 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:19:28.750 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:19:28.851 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:19:28.884 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:19:28.890 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:19:28.890 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:19:28.991 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:19:29.019 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:19:29.022 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:19:29.023 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:19:29.124 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:19:29.132 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:19:29 smithi089 bash[78452]: cluster 1685823568.0283544 osd.6 (osd.6) 173 : cluster 3 failed to encode map e230 with expected crc 2023-06-03T20:19:29.133 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:19:29 smithi089 bash[78452]: cluster 1685823568.0312445 osd.7 (osd.7) 154 : cluster 3 failed to encode map e230 with expected crc 2023-06-03T20:19:29.133 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:19:29 smithi089 bash[78452]: cluster 1685823568.0326347 mon.a (mon.0) 1244 : cluster 0 osdmap e230: 8 total, 7 up, 6 in 2023-06-03T20:19:29.133 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:19:29 smithi089 bash[78452]: cluster 1685823568.0344477 osd.7 (osd.7) 155 : cluster 3 failed to encode map e230 with expected crc 2023-06-03T20:19:29.133 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:19:29 smithi089 bash[78452]: cluster 1685823568.0351963 osd.5 (osd.5) 166 : cluster 3 failed to encode map e230 with expected crc 2023-06-03T20:19:29.133 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:19:29 smithi089 bash[78452]: cluster 1685823568.0360432 osd.7 (osd.7) 156 : cluster 3 failed to encode map e230 with expected crc 2023-06-03T20:19:29.134 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:19:29 smithi089 bash[75068]: cluster 1685823568.0283544 osd.6 (osd.6) 173 : cluster 3 failed to encode map e230 with expected crc 2023-06-03T20:19:29.134 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:19:29 smithi089 bash[75068]: cluster 1685823568.0312445 osd.7 (osd.7) 154 : cluster 3 failed to encode map e230 with expected crc 2023-06-03T20:19:29.134 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:19:29 smithi089 bash[75068]: cluster 1685823568.0326347 mon.a (mon.0) 1244 : cluster 0 osdmap e230: 8 total, 7 up, 6 in 2023-06-03T20:19:29.134 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:19:29 smithi089 bash[75068]: cluster 1685823568.0344477 osd.7 (osd.7) 155 : cluster 3 failed to encode map e230 with expected crc 2023-06-03T20:19:29.134 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:19:29 smithi089 bash[75068]: cluster 1685823568.0351963 osd.5 (osd.5) 166 : cluster 3 failed to encode map e230 with expected crc 2023-06-03T20:19:29.134 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:19:29 smithi089 bash[75068]: cluster 1685823568.0360432 osd.7 (osd.7) 156 : cluster 3 failed to encode map e230 with expected crc 2023-06-03T20:19:29.159 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:19:29.159 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:19:29.160 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:19:29.261 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:19:29.302 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:19:29.306 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:19:29.307 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:19:29.333 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:19:29 smithi164 bash[59462]: cluster 1685823568.0283544 osd.6 (osd.6) 173 : cluster 3 failed to encode map e230 with expected crc 2023-06-03T20:19:29.334 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:19:29 smithi164 bash[59462]: cluster 1685823568.0312445 osd.7 (osd.7) 154 : cluster 3 failed to encode map e230 with expected crc 2023-06-03T20:19:29.334 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:19:29 smithi164 bash[59462]: cluster 1685823568.0326347 mon.a (mon.0) 1244 : cluster 0 osdmap e230: 8 total, 7 up, 6 in 2023-06-03T20:19:29.334 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:19:29 smithi164 bash[59462]: cluster 1685823568.0344477 osd.7 (osd.7) 155 : cluster 3 failed to encode map e230 with expected crc 2023-06-03T20:19:29.334 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:19:29 smithi164 bash[59462]: cluster 1685823568.0351963 osd.5 (osd.5) 166 : cluster 3 failed to encode map e230 with expected crc 2023-06-03T20:19:29.334 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:19:29 smithi164 bash[59462]: cluster 1685823568.0360432 osd.7 (osd.7) 156 : cluster 3 failed to encode map e230 with expected crc 2023-06-03T20:19:29.408 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:19:29.440 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:19:29.443 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:19:29.444 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:19:29.545 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:19:29.572 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:19:29.574 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:19:29.574 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:19:29.675 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:19:29.710 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:19:29.715 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:19:29.715 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:19:29.816 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:19:29.849 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:19:29.853 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:19:29.854 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:19:29.876 INFO:teuthology.orchestra.run.smithi089.stdout:osd.0: {} 2023-06-03T20:19:29.931 INFO:teuthology.orchestra.run.smithi089.stdout:osd.1: {} 2023-06-03T20:19:29.955 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:19:29.974 INFO:teuthology.orchestra.run.smithi089.stdout:osd.2: {} 2023-06-03T20:19:29.985 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:19:29.988 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:19:29.988 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:19:30.030 INFO:teuthology.orchestra.run.smithi089.stdout:osd.3: {} 2023-06-03T20:19:30.073 INFO:teuthology.orchestra.run.smithi089.stdout:osd.5: {} 2023-06-03T20:19:30.089 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:19:30.116 INFO:teuthology.orchestra.run.smithi089.stdout:osd.6: {} 2023-06-03T20:19:30.117 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:19:30.121 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:19:30.123 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:19:30.160 INFO:teuthology.orchestra.run.smithi089.stdout:osd.7: {} 2023-06-03T20:19:30.160 INFO:teuthology.orchestra.run.smithi089.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-06-03T20:19:30.160 INFO:teuthology.orchestra.run.smithi089.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-06-03T20:19:30.161 INFO:teuthology.orchestra.run.smithi089.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-06-03T20:19:30.161 INFO:teuthology.orchestra.run.smithi089.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-06-03T20:19:30.161 INFO:teuthology.orchestra.run.smithi089.stderr:Error ENXIO: problem getting command descriptions from osd.4 2023-06-03T20:19:30.161 INFO:teuthology.orchestra.run.smithi089.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-06-03T20:19:30.161 INFO:teuthology.orchestra.run.smithi089.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-06-03T20:19:30.161 INFO:teuthology.orchestra.run.smithi089.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-06-03T20:19:30.224 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:19:30.264 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:19:30.267 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:19:30.267 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:19:30.333 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:19:30 smithi164 bash[59462]: cluster 1685823568.5754557 osd.3 (osd.3) 1 : cluster 0 12.8 scrub starts 2023-06-03T20:19:30.334 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:19:30 smithi164 bash[59462]: cluster 1685823568.9051259 mgr.y (mgr.34104) 1305 : cluster 0 pgmap v1304: 99 pgs: 9 unknown, 20 active+clean+remapped, 15 active+undersized+degraded, 11 active+undersized, 44 active+clean; 4.6 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 685 op/s; 37767/224913 objects degraded (16.792%); 6/224913 objects misplaced (0.003%) 2023-06-03T20:19:30.334 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:19:30 smithi164 bash[59462]: audit 1685823569.814536 mon.c (mon.2) 565 : audit 0 from='client.? 172.21.15.89:0/1838298215' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-03T20:19:30.338 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:19:30 smithi089 bash[78452]: cluster 1685823568.5754557 osd.3 (osd.3) 1 : cluster 0 12.8 scrub starts 2023-06-03T20:19:30.338 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:19:30 smithi089 bash[78452]: cluster 1685823568.9051259 mgr.y (mgr.34104) 1305 : cluster 0 pgmap v1304: 99 pgs: 9 unknown, 20 active+clean+remapped, 15 active+undersized+degraded, 11 active+undersized, 44 active+clean; 4.6 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 685 op/s; 37767/224913 objects degraded (16.792%); 6/224913 objects misplaced (0.003%) 2023-06-03T20:19:30.338 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:19:30 smithi089 bash[78452]: audit 1685823569.814536 mon.c (mon.2) 565 : audit 0 from='client.? 172.21.15.89:0/1838298215' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-03T20:19:30.339 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:19:30 smithi089 bash[75068]: cluster 1685823568.5754557 osd.3 (osd.3) 1 : cluster 0 12.8 scrub starts 2023-06-03T20:19:30.339 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:19:30 smithi089 bash[75068]: cluster 1685823568.9051259 mgr.y (mgr.34104) 1305 : cluster 0 pgmap v1304: 99 pgs: 9 unknown, 20 active+clean+remapped, 15 active+undersized+degraded, 11 active+undersized, 44 active+clean; 4.6 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 685 op/s; 37767/224913 objects degraded (16.792%); 6/224913 objects misplaced (0.003%) 2023-06-03T20:19:30.339 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:19:30 smithi089 bash[75068]: audit 1685823569.814536 mon.c (mon.2) 565 : audit 0 from='client.? 172.21.15.89:0/1838298215' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-03T20:19:30.369 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:19:30.398 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:19:30.404 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:19:30.405 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:19:30.506 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:19:30.538 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:19:30.540 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:19:30.541 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:19:30.641 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:19:30.675 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:19:30.678 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:19:30.678 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:19:30.779 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:19:30.813 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:19:30.815 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:19:30.816 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:19:30.917 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:19:30.951 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:19:30.953 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:19:30.954 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:19:31.055 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:19:31.071 INFO:teuthology.orchestra.run.smithi089.stderr:nodeep-scrub is unset 2023-06-03T20:19:31.085 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:19:31.089 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:19:31.090 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:19:31.191 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:19:31.201 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:19:31 smithi089 bash[78452]: cluster 1685823569.3899891 osd.3 (osd.3) 2 : cluster 0 12.8 scrub ok 2023-06-03T20:19:31.201 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:19:31 smithi089 bash[78452]: cluster 1685823569.6289053 osd.3 (osd.3) 3 : cluster 0 12.1 scrub starts 2023-06-03T20:19:31.201 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:19:31 smithi089 bash[78452]: cluster 1685823570.1189053 osd.0 (osd.0) 5 : cluster 0 12.6 scrub starts 2023-06-03T20:19:31.201 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:19:31 smithi089 bash[78452]: audit 1685823570.2967906 mon.c (mon.2) 566 : audit 1 from='client.? 172.21.15.89:0/1760690751' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-06-03T20:19:31.202 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:19:31 smithi089 bash[78452]: audit 1685823570.2974076 mon.a (mon.0) 1245 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-06-03T20:19:31.202 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:19:31 smithi089 bash[78452]: cluster 1685823570.552851 osd.2 (osd.2) 1 : cluster 0 12.5 scrub starts 2023-06-03T20:19:31.202 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:19:31 smithi089 bash[75068]: cluster 1685823569.3899891 osd.3 (osd.3) 2 : cluster 0 12.8 scrub ok 2023-06-03T20:19:31.203 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:19:31 smithi089 bash[75068]: cluster 1685823569.6289053 osd.3 (osd.3) 3 : cluster 0 12.1 scrub starts 2023-06-03T20:19:31.203 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:19:31 smithi089 bash[75068]: cluster 1685823570.1189053 osd.0 (osd.0) 5 : cluster 0 12.6 scrub starts 2023-06-03T20:19:31.203 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:19:31 smithi089 bash[75068]: audit 1685823570.2967906 mon.c (mon.2) 566 : audit 1 from='client.? 172.21.15.89:0/1760690751' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-06-03T20:19:31.203 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:19:31 smithi089 bash[75068]: audit 1685823570.2974076 mon.a (mon.0) 1245 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-06-03T20:19:31.203 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:19:31 smithi089 bash[75068]: cluster 1685823570.552851 osd.2 (osd.2) 1 : cluster 0 12.5 scrub starts 2023-06-03T20:19:31.229 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:19:31.231 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:19:31.232 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:19:31.333 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:19:31.335 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:19:31 smithi164 bash[59462]: cluster 1685823569.3899891 osd.3 (osd.3) 2 : cluster 0 12.8 scrub ok 2023-06-03T20:19:31.335 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:19:31 smithi164 bash[59462]: cluster 1685823569.6289053 osd.3 (osd.3) 3 : cluster 0 12.1 scrub starts 2023-06-03T20:19:31.335 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:19:31 smithi164 bash[59462]: cluster 1685823570.1189053 osd.0 (osd.0) 5 : cluster 0 12.6 scrub starts 2023-06-03T20:19:31.335 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:19:31 smithi164 bash[59462]: audit 1685823570.2967906 mon.c (mon.2) 566 : audit 1 from='client.? 172.21.15.89:0/1760690751' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-06-03T20:19:31.336 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:19:31 smithi164 bash[59462]: audit 1685823570.2974076 mon.a (mon.0) 1245 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-06-03T20:19:31.336 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:19:31 smithi164 bash[59462]: cluster 1685823570.552851 osd.2 (osd.2) 1 : cluster 0 12.5 scrub starts 2023-06-03T20:19:31.365 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:19:31.367 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:19:31.368 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:19:31.469 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:19:31.501 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:19:31.503 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:19:31.503 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:19:31.604 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:19:31.647 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:19:31.650 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:19:31.652 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:19:31.753 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:19:31.783 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:19:31.786 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:19:31.786 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:19:31.887 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:19:31.939 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:19:31.941 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:19:31.941 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:19:32.043 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:19:32.103 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:19:32.107 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:19:32.108 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:19:32.209 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:19:32.244 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:19:32.247 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:19:32.250 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:19:32.279 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:19:32 smithi089 bash[78452]: cluster 1685823570.6996062 osd.3 (osd.3) 4 : cluster 0 12.1 scrub ok 2023-06-03T20:19:32.279 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:19:32 smithi089 bash[78452]: cluster 1685823570.9061422 mgr.y (mgr.34104) 1306 : cluster 0 pgmap v1305: 99 pgs: 1 active+clean+scrubbing, 2 unknown, 20 active+clean+remapped, 19 active+undersized+degraded, 11 active+undersized, 46 active+clean; 4.5 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 743 op/s; 36549/218823 objects degraded (16.703%); 6/218823 objects misplaced (0.003%); 0 B/s, 0 objects/s recovering 2023-06-03T20:19:32.279 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:19:32 smithi089 bash[78452]: cluster 1685823571.0373428 mon.a (mon.0) 1246 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-06-03T20:19:32.279 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:19:32 smithi089 bash[78452]: cluster 1685823571.0456536 osd.6 (osd.6) 174 : cluster 3 failed to encode map e231 with expected crc 2023-06-03T20:19:32.280 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:19:32 smithi089 bash[78452]: cluster 1685823571.0473914 osd.5 (osd.5) 167 : cluster 3 failed to encode map e231 with expected crc 2023-06-03T20:19:32.280 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:19:32 smithi089 bash[78452]: audit 1685823571.0529206 mon.a (mon.0) 1247 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-06-03T20:19:32.280 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:19:32 smithi089 bash[78452]: cluster 1685823571.0529788 mon.a (mon.0) 1248 : cluster 0 osdmap e231: 8 total, 7 up, 6 in 2023-06-03T20:19:32.280 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:19:32 smithi089 bash[75068]: cluster 1685823570.6996062 osd.3 (osd.3) 4 : cluster 0 12.1 scrub ok 2023-06-03T20:19:32.280 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:19:32 smithi089 bash[75068]: cluster 1685823570.9061422 mgr.y (mgr.34104) 1306 : cluster 0 pgmap v1305: 99 pgs: 1 active+clean+scrubbing, 2 unknown, 20 active+clean+remapped, 19 active+undersized+degraded, 11 active+undersized, 46 active+clean; 4.5 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 743 op/s; 36549/218823 objects degraded (16.703%); 6/218823 objects misplaced (0.003%); 0 B/s, 0 objects/s recovering 2023-06-03T20:19:32.280 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:19:32 smithi089 bash[75068]: cluster 1685823571.0373428 mon.a (mon.0) 1246 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-06-03T20:19:32.281 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:19:32 smithi089 bash[75068]: cluster 1685823571.0456536 osd.6 (osd.6) 174 : cluster 3 failed to encode map e231 with expected crc 2023-06-03T20:19:32.281 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:19:32 smithi089 bash[75068]: cluster 1685823571.0473914 osd.5 (osd.5) 167 : cluster 3 failed to encode map e231 with expected crc 2023-06-03T20:19:32.281 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:19:32 smithi089 bash[75068]: audit 1685823571.0529206 mon.a (mon.0) 1247 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-06-03T20:19:32.281 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:19:32 smithi089 bash[75068]: cluster 1685823571.0529788 mon.a (mon.0) 1248 : cluster 0 osdmap e231: 8 total, 7 up, 6 in 2023-06-03T20:19:32.334 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:19:32 smithi164 bash[59462]: cluster 1685823570.6996062 osd.3 (osd.3) 4 : cluster 0 12.1 scrub ok 2023-06-03T20:19:32.334 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:19:32 smithi164 bash[59462]: cluster 1685823570.9061422 mgr.y (mgr.34104) 1306 : cluster 0 pgmap v1305: 99 pgs: 1 active+clean+scrubbing, 2 unknown, 20 active+clean+remapped, 19 active+undersized+degraded, 11 active+undersized, 46 active+clean; 4.5 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 743 op/s; 36549/218823 objects degraded (16.703%); 6/218823 objects misplaced (0.003%); 0 B/s, 0 objects/s recovering 2023-06-03T20:19:32.334 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:19:32 smithi164 bash[59462]: cluster 1685823571.0373428 mon.a (mon.0) 1246 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-06-03T20:19:32.335 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:19:32 smithi164 bash[59462]: cluster 1685823571.0456536 osd.6 (osd.6) 174 : cluster 3 failed to encode map e231 with expected crc 2023-06-03T20:19:32.335 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:19:32 smithi164 bash[59462]: cluster 1685823571.0473914 osd.5 (osd.5) 167 : cluster 3 failed to encode map e231 with expected crc 2023-06-03T20:19:32.335 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:19:32 smithi164 bash[59462]: audit 1685823571.0529206 mon.a (mon.0) 1247 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-06-03T20:19:32.335 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:19:32 smithi164 bash[59462]: cluster 1685823571.0529788 mon.a (mon.0) 1248 : cluster 0 osdmap e231: 8 total, 7 up, 6 in 2023-06-03T20:19:32.352 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:19:32.388 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:19:32.390 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:19:32.390 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:19:32.491 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:19:32.534 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:19:32.536 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:19:32.537 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:19:32.639 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:19:32.671 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:19:32.675 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:19:32.675 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:19:32.776 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:19:32.810 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:19:32.815 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:19:32.816 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:19:32.868 INFO:teuthology.orchestra.run.smithi164.stdout:{ 2023-06-03T20:19:32.868 INFO:teuthology.orchestra.run.smithi164.stdout: "size": 20, 2023-06-03T20:19:32.869 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 600, 2023-06-03T20:19:32.869 INFO:teuthology.orchestra.run.smithi164.stdout: "ops": [ 2023-06-03T20:19:32.869 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:19:32.869 INFO:teuthology.orchestra.run.smithi164.stdout: "description": "osd_op(client.46187.0:60459 11.7 11:e813e1ef:::benchmark_data_smithi089_165412_object60458:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e197)", 2023-06-03T20:19:32.869 INFO:teuthology.orchestra.run.smithi164.stdout: "initiated_at": "2023-06-03T20:12:10.679785+0000", 2023-06-03T20:19:32.869 INFO:teuthology.orchestra.run.smithi164.stdout: "age": 442.17868002400002, 2023-06-03T20:19:32.869 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.304126226, 2023-06-03T20:19:32.870 INFO:teuthology.orchestra.run.smithi164.stdout: "type_data": { 2023-06-03T20:19:32.870 INFO:teuthology.orchestra.run.smithi164.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-03T20:19:32.870 INFO:teuthology.orchestra.run.smithi164.stdout: "client_info": { 2023-06-03T20:19:32.870 INFO:teuthology.orchestra.run.smithi164.stdout: "client": "client.46187", 2023-06-03T20:19:32.870 INFO:teuthology.orchestra.run.smithi164.stdout: "client_addr": "172.21.15.89:0/880684210", 2023-06-03T20:19:32.870 INFO:teuthology.orchestra.run.smithi164.stdout: "tid": 60459 2023-06-03T20:19:32.870 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:19:32.870 INFO:teuthology.orchestra.run.smithi164.stdout: "events": [ 2023-06-03T20:19:32.871 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:19:32.871 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "initiated", 2023-06-03T20:19:32.871 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:12:10.679785+0000", 2023-06-03T20:19:32.871 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0 2023-06-03T20:19:32.871 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:19:32.871 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:19:32.871 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "throttled", 2023-06-03T20:19:32.871 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:12:10.679785+0000", 2023-06-03T20:19:32.872 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 4294967295.9999981 2023-06-03T20:19:32.872 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:19:32.872 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:19:32.872 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "header_read", 2023-06-03T20:19:32.872 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:12:10.679783+0000", 2023-06-03T20:19:32.872 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 3.3775000000000001e-05 2023-06-03T20:19:32.872 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:19:32.873 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:19:32.873 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "all_read", 2023-06-03T20:19:32.873 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:12:10.679817+0000", 2023-06-03T20:19:32.873 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 1.3056e-05 2023-06-03T20:19:32.873 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:19:32.873 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:19:32.873 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "dispatched", 2023-06-03T20:19:32.873 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:12:10.679830+0000", 2023-06-03T20:19:32.874 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 6.4570000000000002e-06 2023-06-03T20:19:32.874 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:19:32.874 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:19:32.874 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "queued_for_pg", 2023-06-03T20:19:32.874 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:12:10.679837+0000", 2023-06-03T20:19:32.874 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 5.2102999999999999e-05 2023-06-03T20:19:32.874 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:19:32.874 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:19:32.875 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "reached_pg", 2023-06-03T20:19:32.875 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:12:10.679889+0000", 2023-06-03T20:19:32.875 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.000128832 2023-06-03T20:19:32.875 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:19:32.875 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:19:32.875 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "started", 2023-06-03T20:19:32.875 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:12:10.680018+0000", 2023-06-03T20:19:32.876 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.00015569999999999999 2023-06-03T20:19:32.876 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:19:32.876 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:19:32.876 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "waiting for subops from 0", 2023-06-03T20:19:32.876 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:12:10.680173+0000", 2023-06-03T20:19:32.876 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.0040138099999999996 2023-06-03T20:19:32.876 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:19:32.877 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:19:32.877 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "op_commit", 2023-06-03T20:19:32.877 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:12:10.684187+0000", 2023-06-03T20:19:32.877 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.29957187400000002 2023-06-03T20:19:32.877 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:19:32.877 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:19:32.877 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:19:32.878 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:12:10.983759+0000", 2023-06-03T20:19:32.878 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 5.6368e-05 2023-06-03T20:19:32.878 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:19:32.878 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:19:32.878 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "commit_sent", 2023-06-03T20:19:32.878 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:12:10.983815+0000", 2023-06-03T20:19:32.878 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 9.5947999999999997e-05 2023-06-03T20:19:32.879 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:19:32.880 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:19:32.880 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "done", 2023-06-03T20:19:32.880 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:12:10.983911+0000", 2023-06-03T20:19:32.880 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.304126226 2023-06-03T20:19:32.880 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:19:32.880 INFO:teuthology.orchestra.run.smithi164.stdout: ] 2023-06-03T20:19:32.880 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:19:32.880 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:19:32.881 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:19:32.881 INFO:teuthology.orchestra.run.smithi164.stdout: "description": "osd_op(client.46187.0:60468 11.7 11:e244c030:::benchmark_data_smithi089_165412_object60467:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e197)", 2023-06-03T20:19:32.881 INFO:teuthology.orchestra.run.smithi164.stdout: "initiated_at": "2023-06-03T20:12:10.686093+0000", 2023-06-03T20:19:32.881 INFO:teuthology.orchestra.run.smithi164.stdout: "age": 442.17237173900003, 2023-06-03T20:19:32.881 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.29801150900000001, 2023-06-03T20:19:32.882 INFO:teuthology.orchestra.run.smithi164.stdout: "type_data": { 2023-06-03T20:19:32.882 INFO:teuthology.orchestra.run.smithi164.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-03T20:19:32.883 INFO:teuthology.orchestra.run.smithi164.stdout: "client_info": { 2023-06-03T20:19:32.883 INFO:teuthology.orchestra.run.smithi164.stdout: "client": "client.46187", 2023-06-03T20:19:32.883 INFO:teuthology.orchestra.run.smithi164.stdout: "client_addr": "172.21.15.89:0/880684210", 2023-06-03T20:19:32.883 INFO:teuthology.orchestra.run.smithi164.stdout: "tid": 60468 2023-06-03T20:19:32.883 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:19:32.883 INFO:teuthology.orchestra.run.smithi164.stdout: "events": [ 2023-06-03T20:19:32.883 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:19:32.883 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "initiated", 2023-06-03T20:19:32.884 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:12:10.686093+0000", 2023-06-03T20:19:32.884 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0 2023-06-03T20:19:32.884 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:19:32.884 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:19:32.884 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "throttled", 2023-06-03T20:19:32.885 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:12:10.686093+0000", 2023-06-03T20:19:32.885 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 4294967295.9999981 2023-06-03T20:19:32.885 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:19:32.885 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:19:32.886 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "header_read", 2023-06-03T20:19:32.886 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:12:10.686091+0000", 2023-06-03T20:19:32.886 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 3.3797999999999997e-05 2023-06-03T20:19:32.886 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:19:32.886 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:19:32.886 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "all_read", 2023-06-03T20:19:32.887 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:12:10.686125+0000", 2023-06-03T20:19:32.887 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 1.2563e-05 2023-06-03T20:19:32.887 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:19:32.887 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:19:32.887 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "dispatched", 2023-06-03T20:19:32.887 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:12:10.686138+0000", 2023-06-03T20:19:32.888 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 6.4919999999999998e-06 2023-06-03T20:19:32.888 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:19:32.888 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:19:32.888 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "queued_for_pg", 2023-06-03T20:19:32.889 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:12:10.686144+0000", 2023-06-03T20:19:32.889 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 5.0581999999999998e-05 2023-06-03T20:19:32.889 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:19:32.889 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:19:32.890 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "reached_pg", 2023-06-03T20:19:32.890 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:12:10.686195+0000", 2023-06-03T20:19:32.890 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.000144315 2023-06-03T20:19:32.890 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:19:32.890 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:19:32.890 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "started", 2023-06-03T20:19:32.891 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:12:10.686339+0000", 2023-06-03T20:19:32.891 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.00017369900000000001 2023-06-03T20:19:32.891 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:19:32.891 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:19:32.891 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "waiting for subops from 0", 2023-06-03T20:19:32.891 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:12:10.686513+0000", 2023-06-03T20:19:32.891 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.29741215300000001 2023-06-03T20:19:32.891 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:19:32.892 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:19:32.892 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "op_commit", 2023-06-03T20:19:32.892 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:12:10.983925+0000", 2023-06-03T20:19:32.892 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 4.3424000000000003e-05 2023-06-03T20:19:32.892 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:19:32.892 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:19:32.892 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:19:32.893 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:12:10.983968+0000", 2023-06-03T20:19:32.893 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 4.6980000000000001e-05 2023-06-03T20:19:32.893 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:19:32.893 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:19:32.893 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "commit_sent", 2023-06-03T20:19:32.893 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:12:10.984015+0000", 2023-06-03T20:19:32.893 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 8.9596000000000001e-05 2023-06-03T20:19:32.894 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:19:32.894 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:19:32.894 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "done", 2023-06-03T20:19:32.894 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:12:10.984105+0000", 2023-06-03T20:19:32.894 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.29801150900000001 2023-06-03T20:19:32.895 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:19:32.895 INFO:teuthology.orchestra.run.smithi164.stdout: ] 2023-06-03T20:19:32.895 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:19:32.895 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:19:32.895 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:19:32.895 INFO:teuthology.orchestra.run.smithi164.stdout: "description": "osd_repop(client.46187.0:60467 11.3 e197/189 11:c659ad50:::benchmark_data_smithi089_165412_object60466:head v 197'3834, mlcod=197'3831)", 2023-06-03T20:19:32.895 INFO:teuthology.orchestra.run.smithi164.stdout: "initiated_at": "2023-06-03T20:12:10.686399+0000", 2023-06-03T20:19:32.896 INFO:teuthology.orchestra.run.smithi164.stdout: "age": 442.17206607399999, 2023-06-03T20:19:32.896 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.29427065099999999, 2023-06-03T20:19:32.896 INFO:teuthology.orchestra.run.smithi164.stdout: "type_data": { 2023-06-03T20:19:32.896 INFO:teuthology.orchestra.run.smithi164.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-03T20:19:32.896 INFO:teuthology.orchestra.run.smithi164.stdout: "events": [ 2023-06-03T20:19:32.897 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:19:32.897 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "initiated", 2023-06-03T20:19:32.897 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:12:10.686399+0000", 2023-06-03T20:19:32.898 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0 2023-06-03T20:19:32.898 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:19:32.898 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:19:32.898 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "throttled", 2023-06-03T20:19:32.898 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:12:10.686399+0000", 2023-06-03T20:19:32.899 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 4294967295.999999 2023-06-03T20:19:32.899 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:19:32.899 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:19:32.899 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "header_read", 2023-06-03T20:19:32.899 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:12:10.686398+0000", 2023-06-03T20:19:32.899 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 2.5877999999999999e-05 2023-06-03T20:19:32.899 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:19:32.900 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:19:32.900 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "all_read", 2023-06-03T20:19:32.900 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:12:10.686424+0000", 2023-06-03T20:19:32.900 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 1.0124e-05 2023-06-03T20:19:32.900 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:19:32.900 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:19:32.901 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "dispatched", 2023-06-03T20:19:32.901 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:12:10.686434+0000", 2023-06-03T20:19:32.901 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 5.7610000000000004e-06 2023-06-03T20:19:32.901 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:19:32.901 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:19:32.901 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "queued_for_pg", 2023-06-03T20:19:32.901 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:12:10.686440+0000", 2023-06-03T20:19:32.902 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 4.9316e-05 2023-06-03T20:19:32.902 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:19:32.902 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:19:32.902 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "reached_pg", 2023-06-03T20:19:32.902 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:12:10.686489+0000", 2023-06-03T20:19:32.902 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 2.4464000000000002e-05 2023-06-03T20:19:32.902 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:19:32.903 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:19:32.903 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "started", 2023-06-03T20:19:32.903 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:12:10.686514+0000", 2023-06-03T20:19:32.903 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.29411429100000003 2023-06-03T20:19:32.903 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:19:32.904 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:19:32.904 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "commit_sent", 2023-06-03T20:19:32.904 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:12:10.980628+0000", 2023-06-03T20:19:32.904 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 4.1584e-05 2023-06-03T20:19:32.904 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:19:32.904 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:19:32.904 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "done", 2023-06-03T20:19:32.905 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:12:10.980670+0000", 2023-06-03T20:19:32.905 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.29427065099999999 2023-06-03T20:19:32.905 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:19:32.905 INFO:teuthology.orchestra.run.smithi164.stdout: ] 2023-06-03T20:19:32.905 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:19:32.905 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:19:32.905 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:19:32.906 INFO:teuthology.orchestra.run.smithi164.stdout: "description": "osd_repop(client.46187.0:60470 11.8 e197/189 11:116c2bad:::benchmark_data_smithi089_165412_object60469:head v 197'3819, mlcod=197'3816)", 2023-06-03T20:19:32.906 INFO:teuthology.orchestra.run.smithi164.stdout: "initiated_at": "2023-06-03T20:12:10.687701+0000", 2023-06-03T20:19:32.906 INFO:teuthology.orchestra.run.smithi164.stdout: "age": 442.17076406299998, 2023-06-03T20:19:32.906 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.29296406600000002, 2023-06-03T20:19:32.906 INFO:teuthology.orchestra.run.smithi164.stdout: "type_data": { 2023-06-03T20:19:32.906 INFO:teuthology.orchestra.run.smithi164.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-03T20:19:32.907 INFO:teuthology.orchestra.run.smithi164.stdout: "events": [ 2023-06-03T20:19:32.907 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:19:32.907 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "initiated", 2023-06-03T20:19:32.908 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:12:10.687701+0000", 2023-06-03T20:19:32.908 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0 2023-06-03T20:19:32.908 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:19:32.909 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:19:32.909 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "throttled", 2023-06-03T20:19:32.909 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:12:10.687701+0000", 2023-06-03T20:19:32.910 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 4294967295.999999 2023-06-03T20:19:32.910 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:19:32.910 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:19:32.910 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "header_read", 2023-06-03T20:19:32.911 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:12:10.687700+0000", 2023-06-03T20:19:32.911 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.000126904 2023-06-03T20:19:32.911 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:19:32.911 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:19:32.912 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "all_read", 2023-06-03T20:19:32.912 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:12:10.687827+0000", 2023-06-03T20:19:32.912 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 1.3742e-05 2023-06-03T20:19:32.912 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:19:32.913 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:19:32.913 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "dispatched", 2023-06-03T20:19:32.913 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:12:10.687841+0000", 2023-06-03T20:19:32.914 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 7.2919999999999998e-06 2023-06-03T20:19:32.914 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:19:32.914 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:19:32.915 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "queued_for_pg", 2023-06-03T20:19:32.915 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:12:10.687848+0000", 2023-06-03T20:19:32.915 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 6.2892999999999996e-05 2023-06-03T20:19:32.915 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:19:32.915 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:19:32.916 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "reached_pg", 2023-06-03T20:19:32.916 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:12:10.687911+0000", 2023-06-03T20:19:32.916 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 2.0224000000000002e-05 2023-06-03T20:19:32.916 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:19:32.916 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:19:32.916 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "started", 2023-06-03T20:19:32.916 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:12:10.687931+0000", 2023-06-03T20:19:32.917 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.29267958399999999 2023-06-03T20:19:32.917 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:19:32.917 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:19:32.917 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "commit_sent", 2023-06-03T20:19:32.917 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:12:10.980611+0000", 2023-06-03T20:19:32.917 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 5.4381e-05 2023-06-03T20:19:32.917 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:19:32.918 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:19:32.918 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "done", 2023-06-03T20:19:32.918 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:12:10.980665+0000", 2023-06-03T20:19:32.918 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.29296406600000002 2023-06-03T20:19:32.918 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:19:32.918 INFO:teuthology.orchestra.run.smithi164.stdout: ] 2023-06-03T20:19:32.919 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:19:32.919 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:19:32.919 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:19:32.919 INFO:teuthology.orchestra.run.smithi164.stdout: "description": "osd_op(client.46187.0:60471 11.7 11:ea4b9657:::benchmark_data_smithi089_165412_object60470:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e197)", 2023-06-03T20:19:32.920 INFO:teuthology.orchestra.run.smithi164.stdout: "initiated_at": "2023-06-03T20:12:10.688000+0000", 2023-06-03T20:19:32.920 INFO:teuthology.orchestra.run.smithi164.stdout: "age": 442.17046509300002, 2023-06-03T20:19:32.920 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.30822460800000001, 2023-06-03T20:19:32.920 INFO:teuthology.orchestra.run.smithi164.stdout: "type_data": { 2023-06-03T20:19:32.921 INFO:teuthology.orchestra.run.smithi164.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-03T20:19:32.921 INFO:teuthology.orchestra.run.smithi164.stdout: "client_info": { 2023-06-03T20:19:32.921 INFO:teuthology.orchestra.run.smithi164.stdout: "client": "client.46187", 2023-06-03T20:19:32.921 INFO:teuthology.orchestra.run.smithi164.stdout: "client_addr": "172.21.15.89:0/880684210", 2023-06-03T20:19:32.921 INFO:teuthology.orchestra.run.smithi164.stdout: "tid": 60471 2023-06-03T20:19:32.921 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:19:32.922 INFO:teuthology.orchestra.run.smithi164.stdout: "events": [ 2023-06-03T20:19:32.922 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:19:32.922 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "initiated", 2023-06-03T20:19:32.922 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:12:10.688000+0000", 2023-06-03T20:19:32.922 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0 2023-06-03T20:19:32.923 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:19:32.923 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:19:32.923 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "throttled", 2023-06-03T20:19:32.923 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:12:10.688000+0000", 2023-06-03T20:19:32.923 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 4294967295.999999 2023-06-03T20:19:32.923 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:19:32.923 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:19:32.924 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "header_read", 2023-06-03T20:19:32.924 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:12:10.687999+0000", 2023-06-03T20:19:32.924 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 8.9674999999999998e-05 2023-06-03T20:19:32.924 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:19:32.924 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:19:32.924 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "all_read", 2023-06-03T20:19:32.924 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:12:10.688089+0000", 2023-06-03T20:19:32.925 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 1.3740000000000001e-05 2023-06-03T20:19:32.925 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:19:32.925 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:19:32.925 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "dispatched", 2023-06-03T20:19:32.925 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:12:10.688102+0000", 2023-06-03T20:19:32.925 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 5.6280000000000002e-06 2023-06-03T20:19:32.925 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:19:32.926 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:19:32.926 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "queued_for_pg", 2023-06-03T20:19:32.926 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:12:10.688108+0000", 2023-06-03T20:19:32.926 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.00060365500000000003 2023-06-03T20:19:32.926 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:19:32.926 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:19:32.926 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "reached_pg", 2023-06-03T20:19:32.927 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:12:10.688712+0000", 2023-06-03T20:19:32.927 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.00013657900000000001 2023-06-03T20:19:32.927 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:19:32.927 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:19:32.927 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "started", 2023-06-03T20:19:32.927 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:12:10.688848+0000", 2023-06-03T20:19:32.927 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.29192311799999998 2023-06-03T20:19:32.928 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:19:32.928 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:19:32.928 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "waiting for subops from 0", 2023-06-03T20:19:32.928 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:12:10.980771+0000", 2023-06-03T20:19:32.928 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.0061675640000000004 2023-06-03T20:19:32.928 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:19:32.928 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:19:32.929 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "op_commit", 2023-06-03T20:19:32.929 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:12:10.986939+0000", 2023-06-03T20:19:32.929 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.0091243100000000001 2023-06-03T20:19:32.929 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:19:32.929 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:19:32.929 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:19:32.930 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:12:10.996063+0000", 2023-06-03T20:19:32.930 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 6.1641000000000002e-05 2023-06-03T20:19:32.930 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:19:32.930 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:19:32.930 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "commit_sent", 2023-06-03T20:19:32.930 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:12:10.996125+0000", 2023-06-03T20:19:32.930 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 9.9853999999999996e-05 2023-06-03T20:19:32.930 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:19:32.931 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:19:32.931 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "done", 2023-06-03T20:19:32.931 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:12:10.996225+0000", 2023-06-03T20:19:32.931 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.30822460800000001 2023-06-03T20:19:32.931 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:19:32.931 INFO:teuthology.orchestra.run.smithi164.stdout: ] 2023-06-03T20:19:32.932 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:19:32.932 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:19:32.932 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:19:32.933 INFO:teuthology.orchestra.run.smithi164.stdout: "description": "osd_op(client.46187.0:60473 11.7 11:e4006b35:::benchmark_data_smithi089_165412_object60472:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e197)", 2023-06-03T20:19:32.933 INFO:teuthology.orchestra.run.smithi164.stdout: "initiated_at": "2023-06-03T20:12:10.689610+0000", 2023-06-03T20:19:32.933 INFO:teuthology.orchestra.run.smithi164.stdout: "age": 442.16885521, 2023-06-03T20:19:32.933 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.306784532, 2023-06-03T20:19:32.933 INFO:teuthology.orchestra.run.smithi164.stdout: "type_data": { 2023-06-03T20:19:32.933 INFO:teuthology.orchestra.run.smithi164.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-03T20:19:32.934 INFO:teuthology.orchestra.run.smithi164.stdout: "client_info": { 2023-06-03T20:19:32.934 INFO:teuthology.orchestra.run.smithi164.stdout: "client": "client.46187", 2023-06-03T20:19:32.934 INFO:teuthology.orchestra.run.smithi164.stdout: "client_addr": "172.21.15.89:0/880684210", 2023-06-03T20:19:32.934 INFO:teuthology.orchestra.run.smithi164.stdout: "tid": 60473 2023-06-03T20:19:32.934 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:19:32.934 INFO:teuthology.orchestra.run.smithi164.stdout: "events": [ 2023-06-03T20:19:32.935 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:19:32.935 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "initiated", 2023-06-03T20:19:32.935 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:12:10.689610+0000", 2023-06-03T20:19:32.935 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0 2023-06-03T20:19:32.935 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:19:32.935 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:19:32.936 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "throttled", 2023-06-03T20:19:32.936 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:12:10.689610+0000", 2023-06-03T20:19:32.936 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 4294967295.9999986 2023-06-03T20:19:32.936 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:19:32.937 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:19:32.937 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "header_read", 2023-06-03T20:19:32.937 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:12:10.689608+0000", 2023-06-03T20:19:32.937 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 7.3188999999999993e-05 2023-06-03T20:19:32.938 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:19:32.938 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:19:32.938 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "all_read", 2023-06-03T20:19:32.938 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:12:10.689682+0000", 2023-06-03T20:19:32.939 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.29092713199999998 2023-06-03T20:19:32.939 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:19:32.939 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:19:32.939 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "dispatched", 2023-06-03T20:19:32.939 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:12:10.980609+0000", 2023-06-03T20:19:32.939 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 3.2396e-05 2023-06-03T20:19:32.940 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:19:32.940 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:19:32.940 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "queued_for_pg", 2023-06-03T20:19:32.940 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:12:10.980641+0000", 2023-06-03T20:19:32.940 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.003502327 2023-06-03T20:19:32.940 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:19:32.941 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:19:32.941 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "reached_pg", 2023-06-03T20:19:32.941 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:12:10.984143+0000", 2023-06-03T20:19:32.941 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.00016113999999999999 2023-06-03T20:19:32.941 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:19:32.941 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:19:32.941 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "started", 2023-06-03T20:19:32.942 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:12:10.984305+0000", 2023-06-03T20:19:32.942 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.00018914800000000001 2023-06-03T20:19:32.942 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:19:32.942 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:19:32.942 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "waiting for subops from 0", 2023-06-03T20:19:32.943 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:12:10.984494+0000", 2023-06-03T20:19:32.943 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.0055146079999999998 2023-06-03T20:19:32.943 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:19:32.943 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:19:32.943 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "op_commit", 2023-06-03T20:19:32.944 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:12:10.990008+0000", 2023-06-03T20:19:32.944 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.0062579009999999997 2023-06-03T20:19:32.944 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:19:32.944 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:19:32.944 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:19:32.944 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:12:10.996266+0000", 2023-06-03T20:19:32.944 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 4.6075999999999997e-05 2023-06-03T20:19:32.945 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:19:32.945 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:19:32.945 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "commit_sent", 2023-06-03T20:19:32.945 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:12:10.996312+0000", 2023-06-03T20:19:32.945 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 8.2223000000000002e-05 2023-06-03T20:19:32.945 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:19:32.945 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:19:32.946 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "done", 2023-06-03T20:19:32.946 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:12:10.996395+0000", 2023-06-03T20:19:32.946 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.306784532 2023-06-03T20:19:32.946 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:19:32.946 INFO:teuthology.orchestra.run.smithi164.stdout: ] 2023-06-03T20:19:32.946 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:19:32.946 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:19:32.947 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:19:32.947 INFO:teuthology.orchestra.run.smithi164.stdout: "description": "osd_repop_reply(client.46187.0:60459 11.7 e197/195 ondisk, result = 0)", 2023-06-03T20:19:32.947 INFO:teuthology.orchestra.run.smithi164.stdout: "initiated_at": "2023-06-03T20:12:10.691698+0000", 2023-06-03T20:19:32.947 INFO:teuthology.orchestra.run.smithi164.stdout: "age": 442.16676764300001, 2023-06-03T20:19:32.947 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.29222863999999998, 2023-06-03T20:19:32.947 INFO:teuthology.orchestra.run.smithi164.stdout: "type_data": { 2023-06-03T20:19:32.948 INFO:teuthology.orchestra.run.smithi164.stdout: "flag_point": "started", 2023-06-03T20:19:32.948 INFO:teuthology.orchestra.run.smithi164.stdout: "events": [ 2023-06-03T20:19:32.948 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:19:32.948 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "initiated", 2023-06-03T20:19:32.948 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:12:10.691698+0000", 2023-06-03T20:19:32.948 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0 2023-06-03T20:19:32.948 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:19:32.949 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:19:32.949 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "throttled", 2023-06-03T20:19:32.949 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:12:10.691698+0000", 2023-06-03T20:19:32.949 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 4294967295.999999 2023-06-03T20:19:32.950 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:19:32.950 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:19:32.950 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "header_read", 2023-06-03T20:19:32.950 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:12:10.691697+0000", 2023-06-03T20:19:32.951 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 1.2357999999999999e-05 2023-06-03T20:19:32.951 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:19:32.951 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:19:32.951 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "all_read", 2023-06-03T20:19:32.951 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:12:10.691709+0000", 2023-06-03T20:19:32.952 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.288913368 2023-06-03T20:19:32.952 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:19:32.952 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:19:32.952 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "dispatched", 2023-06-03T20:19:32.952 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:12:10.980622+0000", 2023-06-03T20:19:32.953 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 1.2622999999999999e-05 2023-06-03T20:19:32.953 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:19:32.953 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:19:32.953 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "queued_for_pg", 2023-06-03T20:19:32.953 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:12:10.980635+0000", 2023-06-03T20:19:32.953 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.0031077740000000002 2023-06-03T20:19:32.953 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:19:32.954 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:19:32.954 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "reached_pg", 2023-06-03T20:19:32.954 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:12:10.983743+0000", 2023-06-03T20:19:32.954 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 7.9100000000000005e-06 2023-06-03T20:19:32.954 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:19:32.954 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:19:32.954 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "started", 2023-06-03T20:19:32.955 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:12:10.983751+0000", 2023-06-03T20:19:32.955 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.000175639 2023-06-03T20:19:32.955 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:19:32.955 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:19:32.955 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "done", 2023-06-03T20:19:32.955 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:12:10.983926+0000", 2023-06-03T20:19:32.955 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.29222863999999998 2023-06-03T20:19:32.956 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:19:32.956 INFO:teuthology.orchestra.run.smithi164.stdout: ] 2023-06-03T20:19:32.956 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:19:32.956 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:19:32.956 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:19:32.956 INFO:teuthology.orchestra.run.smithi164.stdout: "description": "osd_op(client.46187.0:114528 11.7 11:ea34b056:::benchmark_data_smithi089_165412_object9000:head [delete] snapc 0=[] ondisk+write+known_if_redirected e201)", 2023-06-03T20:19:32.956 INFO:teuthology.orchestra.run.smithi164.stdout: "initiated_at": "2023-06-03T20:13:16.741556+0000", 2023-06-03T20:19:32.957 INFO:teuthology.orchestra.run.smithi164.stdout: "age": 376.11690905900002, 2023-06-03T20:19:32.957 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.071604044000000006, 2023-06-03T20:19:32.957 INFO:teuthology.orchestra.run.smithi164.stdout: "type_data": { 2023-06-03T20:19:32.957 INFO:teuthology.orchestra.run.smithi164.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-03T20:19:32.957 INFO:teuthology.orchestra.run.smithi164.stdout: "client_info": { 2023-06-03T20:19:32.957 INFO:teuthology.orchestra.run.smithi164.stdout: "client": "client.46187", 2023-06-03T20:19:32.957 INFO:teuthology.orchestra.run.smithi164.stdout: "client_addr": "172.21.15.89:0/880684210", 2023-06-03T20:19:32.958 INFO:teuthology.orchestra.run.smithi164.stdout: "tid": 114528 2023-06-03T20:19:32.958 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:19:32.958 INFO:teuthology.orchestra.run.smithi164.stdout: "events": [ 2023-06-03T20:19:32.958 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:19:32.959 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "initiated", 2023-06-03T20:19:32.959 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:13:16.741556+0000", 2023-06-03T20:19:32.959 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0 2023-06-03T20:19:32.959 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:19:32.959 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:19:32.960 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "throttled", 2023-06-03T20:19:32.960 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:13:16.741556+0000", 2023-06-03T20:19:32.960 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 4294967295.9999976 2023-06-03T20:19:32.960 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:19:32.960 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:19:32.961 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "header_read", 2023-06-03T20:19:32.961 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:13:16.741554+0000", 2023-06-03T20:19:32.961 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 1.0797e-05 2023-06-03T20:19:32.961 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:19:32.961 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:19:32.962 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "all_read", 2023-06-03T20:19:32.962 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:13:16.741565+0000", 2023-06-03T20:19:32.962 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 1.6266e-05 2023-06-03T20:19:32.963 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:19:32.963 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:19:32.963 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "dispatched", 2023-06-03T20:19:32.963 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:13:16.741581+0000", 2023-06-03T20:19:32.964 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 7.9810000000000003e-06 2023-06-03T20:19:32.964 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:19:32.964 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:19:32.964 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "queued_for_pg", 2023-06-03T20:19:32.965 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:13:16.741589+0000", 2023-06-03T20:19:32.965 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 5.8554999999999998e-05 2023-06-03T20:19:32.965 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:19:32.965 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:19:32.965 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "reached_pg", 2023-06-03T20:19:32.966 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:13:16.741647+0000", 2023-06-03T20:19:32.966 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.00013013 2023-06-03T20:19:32.966 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:19:32.966 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:19:32.967 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "started", 2023-06-03T20:19:32.967 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:13:16.741777+0000", 2023-06-03T20:19:32.967 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.000168166 2023-06-03T20:19:32.968 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:19:32.968 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:19:32.968 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "waiting for subops from 0", 2023-06-03T20:19:32.968 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:13:16.741946+0000", 2023-06-03T20:19:32.969 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.0087850540000000005 2023-06-03T20:19:32.969 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:19:32.969 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:19:32.969 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "op_commit", 2023-06-03T20:19:32.969 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:13:16.750731+0000", 2023-06-03T20:19:32.969 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.062301557 2023-06-03T20:19:32.970 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:19:32.970 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:19:32.970 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:19:32.970 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:13:16.813032+0000", 2023-06-03T20:19:32.970 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 4.9818000000000003e-05 2023-06-03T20:19:32.970 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:19:32.970 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:19:32.971 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "commit_sent", 2023-06-03T20:19:32.971 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:13:16.813082+0000", 2023-06-03T20:19:32.971 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 7.8126000000000004e-05 2023-06-03T20:19:32.971 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:19:32.971 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:19:32.971 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "done", 2023-06-03T20:19:32.972 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:13:16.813160+0000", 2023-06-03T20:19:32.972 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.071604044000000006 2023-06-03T20:19:32.972 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:19:32.972 INFO:teuthology.orchestra.run.smithi164.stdout: ] 2023-06-03T20:19:32.972 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:19:32.973 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:19:32.973 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:19:32.973 INFO:teuthology.orchestra.run.smithi164.stdout: "description": "osd_op(client.46187.0:149058 11.7 11:e931c133:::benchmark_data_smithi089_165412_object43530:head [delete] snapc 0=[] ondisk+write+known_if_redirected e203)", 2023-06-03T20:19:32.973 INFO:teuthology.orchestra.run.smithi164.stdout: "initiated_at": "2023-06-03T20:14:04.069905+0000", 2023-06-03T20:19:32.973 INFO:teuthology.orchestra.run.smithi164.stdout: "age": 328.788560523, 2023-06-03T20:19:32.973 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.068463232999999998, 2023-06-03T20:19:32.973 INFO:teuthology.orchestra.run.smithi164.stdout: "type_data": { 2023-06-03T20:19:32.974 INFO:teuthology.orchestra.run.smithi164.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-03T20:19:32.974 INFO:teuthology.orchestra.run.smithi164.stdout: "client_info": { 2023-06-03T20:19:32.974 INFO:teuthology.orchestra.run.smithi164.stdout: "client": "client.46187", 2023-06-03T20:19:32.974 INFO:teuthology.orchestra.run.smithi164.stdout: "client_addr": "172.21.15.89:0/880684210", 2023-06-03T20:19:32.974 INFO:teuthology.orchestra.run.smithi164.stdout: "tid": 149058 2023-06-03T20:19:32.974 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:19:32.974 INFO:teuthology.orchestra.run.smithi164.stdout: "events": [ 2023-06-03T20:19:32.975 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:19:32.975 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "initiated", 2023-06-03T20:19:32.975 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:14:04.069905+0000", 2023-06-03T20:19:32.975 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0 2023-06-03T20:19:32.975 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:19:32.975 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:19:32.976 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "throttled", 2023-06-03T20:19:32.976 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:14:04.069905+0000", 2023-06-03T20:19:32.976 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 4294967295.9999976 2023-06-03T20:19:32.976 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:19:32.976 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:19:32.977 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "header_read", 2023-06-03T20:19:32.977 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:14:04.069902+0000", 2023-06-03T20:19:32.977 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 1.1365e-05 2023-06-03T20:19:32.978 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:19:32.978 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:19:32.978 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "all_read", 2023-06-03T20:19:32.978 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:14:04.069914+0000", 2023-06-03T20:19:32.979 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 2.2078000000000001e-05 2023-06-03T20:19:32.979 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:19:32.979 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:19:32.980 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "dispatched", 2023-06-03T20:19:32.980 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:14:04.069936+0000", 2023-06-03T20:19:32.980 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 8.9060000000000008e-06 2023-06-03T20:19:32.981 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:19:32.981 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:19:32.981 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "queued_for_pg", 2023-06-03T20:19:32.981 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:14:04.069945+0000", 2023-06-03T20:19:32.981 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 6.6019999999999995e-05 2023-06-03T20:19:32.982 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:19:32.982 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:19:32.982 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "reached_pg", 2023-06-03T20:19:32.982 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:14:04.070011+0000", 2023-06-03T20:19:32.982 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.000152394 2023-06-03T20:19:32.982 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:19:32.982 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:19:32.983 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "started", 2023-06-03T20:19:32.983 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:14:04.070163+0000", 2023-06-03T20:19:32.983 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.00015047199999999999 2023-06-03T20:19:32.984 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:19:32.984 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:19:32.984 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "waiting for subops from 0", 2023-06-03T20:19:32.985 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:14:04.070314+0000", 2023-06-03T20:19:32.985 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.0070163559999999996 2023-06-03T20:19:32.985 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:19:32.985 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:19:32.986 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "op_commit", 2023-06-03T20:19:32.986 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:14:04.077330+0000", 2023-06-03T20:19:32.986 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.060883516999999998 2023-06-03T20:19:32.987 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:19:32.987 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:19:32.987 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:19:32.987 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:14:04.138213+0000", 2023-06-03T20:19:32.988 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 5.5334000000000002e-05 2023-06-03T20:19:32.988 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:19:32.988 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:19:32.989 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "commit_sent", 2023-06-03T20:19:32.989 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:14:04.138269+0000", 2023-06-03T20:19:32.989 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 9.9080999999999995e-05 2023-06-03T20:19:32.990 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:19:32.990 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:19:32.990 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "done", 2023-06-03T20:19:32.991 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:14:04.138368+0000", 2023-06-03T20:19:32.991 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.068463232999999998 2023-06-03T20:19:32.991 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:19:32.991 INFO:teuthology.orchestra.run.smithi164.stdout: ] 2023-06-03T20:19:32.991 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:19:32.992 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:19:32.992 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:19:32.992 INFO:teuthology.orchestra.run.smithi164.stdout: "description": "osd_op(client.46187.0:149065 11.7 11:e249fddb:::benchmark_data_smithi089_165412_object43537:head [delete] snapc 0=[] ondisk+write+known_if_redirected e203)", 2023-06-03T20:19:32.992 INFO:teuthology.orchestra.run.smithi164.stdout: "initiated_at": "2023-06-03T20:14:04.082048+0000", 2023-06-03T20:19:32.992 INFO:teuthology.orchestra.run.smithi164.stdout: "age": 328.77641693200002, 2023-06-03T20:19:32.993 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.067123820000000001, 2023-06-03T20:19:32.993 INFO:teuthology.orchestra.run.smithi164.stdout: "type_data": { 2023-06-03T20:19:32.993 INFO:teuthology.orchestra.run.smithi164.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-03T20:19:32.993 INFO:teuthology.orchestra.run.smithi164.stdout: "client_info": { 2023-06-03T20:19:32.993 INFO:teuthology.orchestra.run.smithi164.stdout: "client": "client.46187", 2023-06-03T20:19:32.993 INFO:teuthology.orchestra.run.smithi164.stdout: "client_addr": "172.21.15.89:0/880684210", 2023-06-03T20:19:32.993 INFO:teuthology.orchestra.run.smithi164.stdout: "tid": 149065 2023-06-03T20:19:32.994 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:19:32.994 INFO:teuthology.orchestra.run.smithi164.stdout: "events": [ 2023-06-03T20:19:32.994 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:19:32.994 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "initiated", 2023-06-03T20:19:32.994 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:14:04.082048+0000", 2023-06-03T20:19:32.994 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0 2023-06-03T20:19:32.994 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:19:32.994 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:19:32.995 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "throttled", 2023-06-03T20:19:32.995 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:14:04.082048+0000", 2023-06-03T20:19:32.995 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 4294967295.9999981 2023-06-03T20:19:32.995 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:19:32.995 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:19:32.995 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "header_read", 2023-06-03T20:19:32.995 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:14:04.082046+0000", 2023-06-03T20:19:32.996 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 1.9160999999999998e-05 2023-06-03T20:19:32.996 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:19:32.996 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:19:32.996 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "all_read", 2023-06-03T20:19:32.996 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:14:04.082066+0000", 2023-06-03T20:19:32.996 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 1.6167999999999998e-05 2023-06-03T20:19:32.997 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:19:32.997 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:19:32.997 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "dispatched", 2023-06-03T20:19:32.997 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:14:04.082082+0000", 2023-06-03T20:19:32.997 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 6.8630000000000002e-06 2023-06-03T20:19:32.997 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:19:32.997 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:19:32.997 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "queued_for_pg", 2023-06-03T20:19:32.998 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:14:04.082089+0000", 2023-06-03T20:19:32.998 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 7.2098e-05 2023-06-03T20:19:32.998 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:19:32.998 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:19:32.998 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "reached_pg", 2023-06-03T20:19:32.998 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:14:04.082161+0000", 2023-06-03T20:19:32.998 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.00019353400000000001 2023-06-03T20:19:32.999 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:19:32.999 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:19:32.999 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "started", 2023-06-03T20:19:32.999 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:14:04.082354+0000", 2023-06-03T20:19:32.999 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.00020690699999999999 2023-06-03T20:19:32.999 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:19:33.000 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:19:33.000 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "waiting for subops from 0", 2023-06-03T20:19:33.000 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:14:04.082561+0000", 2023-06-03T20:19:33.000 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.0048655399999999998 2023-06-03T20:19:33.000 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:19:33.000 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:19:33.000 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "op_commit", 2023-06-03T20:19:33.001 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:14:04.087427+0000", 2023-06-03T20:19:33.001 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.061516330000000001 2023-06-03T20:19:33.001 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:19:33.001 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:19:33.001 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:19:33.002 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:14:04.148943+0000", 2023-06-03T20:19:33.002 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 8.1410999999999997e-05 2023-06-03T20:19:33.002 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:19:33.002 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:19:33.003 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "commit_sent", 2023-06-03T20:19:33.003 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:14:04.149024+0000", 2023-06-03T20:19:33.003 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.00014768 2023-06-03T20:19:33.003 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:19:33.003 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:19:33.003 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "done", 2023-06-03T20:19:33.003 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:14:04.149172+0000", 2023-06-03T20:19:33.004 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.067123820000000001 2023-06-03T20:19:33.004 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:19:33.004 INFO:teuthology.orchestra.run.smithi164.stdout: ] 2023-06-03T20:19:33.004 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:19:33.004 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:19:33.004 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:19:33.005 INFO:teuthology.orchestra.run.smithi164.stdout: "description": "osd_op(client.46187.0:157557 11.7 11:e5b1ac6e:::benchmark_data_smithi089_165412_object52029:head [delete] snapc 0=[] ondisk+write+known_if_redirected e203)", 2023-06-03T20:19:33.005 INFO:teuthology.orchestra.run.smithi164.stdout: "initiated_at": "2023-06-03T20:14:16.057387+0000", 2023-06-03T20:19:33.005 INFO:teuthology.orchestra.run.smithi164.stdout: "age": 316.80107808899999, 2023-06-03T20:19:33.005 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.17980800499999999, 2023-06-03T20:19:33.005 INFO:teuthology.orchestra.run.smithi164.stdout: "type_data": { 2023-06-03T20:19:33.005 INFO:teuthology.orchestra.run.smithi164.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-03T20:19:33.005 INFO:teuthology.orchestra.run.smithi164.stdout: "client_info": { 2023-06-03T20:19:33.006 INFO:teuthology.orchestra.run.smithi164.stdout: "client": "client.46187", 2023-06-03T20:19:33.006 INFO:teuthology.orchestra.run.smithi164.stdout: "client_addr": "172.21.15.89:0/880684210", 2023-06-03T20:19:33.006 INFO:teuthology.orchestra.run.smithi164.stdout: "tid": 157557 2023-06-03T20:19:33.006 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:19:33.006 INFO:teuthology.orchestra.run.smithi164.stdout: "events": [ 2023-06-03T20:19:33.006 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:19:33.006 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "initiated", 2023-06-03T20:19:33.007 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:14:16.057387+0000", 2023-06-03T20:19:33.007 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0 2023-06-03T20:19:33.007 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:19:33.007 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:19:33.007 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "throttled", 2023-06-03T20:19:33.007 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:14:16.057387+0000", 2023-06-03T20:19:33.007 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 4294967295.9999981 2023-06-03T20:19:33.008 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:19:33.008 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:19:33.008 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "header_read", 2023-06-03T20:19:33.008 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:14:16.057385+0000", 2023-06-03T20:19:33.008 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 1.1668000000000001e-05 2023-06-03T20:19:33.008 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:19:33.008 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:19:33.009 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "all_read", 2023-06-03T20:19:33.009 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:14:16.057397+0000", 2023-06-03T20:19:33.009 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 2.0288999999999998e-05 2023-06-03T20:19:33.009 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:19:33.009 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:19:33.009 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "dispatched", 2023-06-03T20:19:33.009 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:14:16.057417+0000", 2023-06-03T20:19:33.010 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 7.3379999999999999e-06 2023-06-03T20:19:33.010 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:19:33.010 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:19:33.010 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "queued_for_pg", 2023-06-03T20:19:33.010 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:14:16.057424+0000", 2023-06-03T20:19:33.010 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 7.0810999999999997e-05 2023-06-03T20:19:33.010 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:19:33.010 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:19:33.011 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "reached_pg", 2023-06-03T20:19:33.011 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:14:16.057495+0000", 2023-06-03T20:19:33.011 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.00015740099999999999 2023-06-03T20:19:33.011 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:19:33.011 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:19:33.011 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "started", 2023-06-03T20:19:33.011 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:14:16.057653+0000", 2023-06-03T20:19:33.012 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.00020162700000000001 2023-06-03T20:19:33.012 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:19:33.012 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:19:33.012 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "waiting for subops from 0", 2023-06-03T20:19:33.012 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:14:16.057854+0000", 2023-06-03T20:19:33.012 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.0056318649999999998 2023-06-03T20:19:33.012 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:19:33.013 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:19:33.013 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "op_commit", 2023-06-03T20:19:33.013 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:14:16.063486+0000", 2023-06-03T20:19:33.013 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.173473565 2023-06-03T20:19:33.013 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:19:33.013 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:19:33.013 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:19:33.014 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:14:16.236960+0000", 2023-06-03T20:19:33.014 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 9.6502999999999994e-05 2023-06-03T20:19:33.014 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:19:33.014 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:19:33.014 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "commit_sent", 2023-06-03T20:19:33.014 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:14:16.237056+0000", 2023-06-03T20:19:33.014 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.00013898400000000001 2023-06-03T20:19:33.015 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:19:33.015 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:19:33.015 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "done", 2023-06-03T20:19:33.015 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:14:16.237195+0000", 2023-06-03T20:19:33.015 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.17980800499999999 2023-06-03T20:19:33.015 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:19:33.015 INFO:teuthology.orchestra.run.smithi164.stdout: ] 2023-06-03T20:19:33.016 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:19:33.016 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:19:33.016 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:19:33.016 INFO:teuthology.orchestra.run.smithi164.stdout: "description": "osd_op(client.46187.0:157565 11.7 11:e1b2f77a:::benchmark_data_smithi089_165412_object52037:head [delete] snapc 0=[] ondisk+write+known_if_redirected e203)", 2023-06-03T20:19:33.016 INFO:teuthology.orchestra.run.smithi164.stdout: "initiated_at": "2023-06-03T20:14:16.066525+0000", 2023-06-03T20:19:33.016 INFO:teuthology.orchestra.run.smithi164.stdout: "age": 316.791940017, 2023-06-03T20:19:33.016 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.17550977200000001, 2023-06-03T20:19:33.017 INFO:teuthology.orchestra.run.smithi164.stdout: "type_data": { 2023-06-03T20:19:33.017 INFO:teuthology.orchestra.run.smithi164.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-03T20:19:33.017 INFO:teuthology.orchestra.run.smithi164.stdout: "client_info": { 2023-06-03T20:19:33.017 INFO:teuthology.orchestra.run.smithi164.stdout: "client": "client.46187", 2023-06-03T20:19:33.017 INFO:teuthology.orchestra.run.smithi164.stdout: "client_addr": "172.21.15.89:0/880684210", 2023-06-03T20:19:33.017 INFO:teuthology.orchestra.run.smithi164.stdout: "tid": 157565 2023-06-03T20:19:33.017 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:19:33.018 INFO:teuthology.orchestra.run.smithi164.stdout: "events": [ 2023-06-03T20:19:33.018 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:19:33.018 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "initiated", 2023-06-03T20:19:33.018 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:14:16.066525+0000", 2023-06-03T20:19:33.018 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0 2023-06-03T20:19:33.018 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:19:33.018 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:19:33.018 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "throttled", 2023-06-03T20:19:33.019 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:14:16.066525+0000", 2023-06-03T20:19:33.019 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 4294967295.9999981 2023-06-03T20:19:33.019 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:19:33.019 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:19:33.019 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "header_read", 2023-06-03T20:19:33.019 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:14:16.066523+0000", 2023-06-03T20:19:33.019 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 1.1021e-05 2023-06-03T20:19:33.020 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:19:33.020 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:19:33.020 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "all_read", 2023-06-03T20:19:33.020 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:14:16.066534+0000", 2023-06-03T20:19:33.020 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 1.5911000000000001e-05 2023-06-03T20:19:33.020 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:19:33.021 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:19:33.021 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "dispatched", 2023-06-03T20:19:33.021 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:14:16.066550+0000", 2023-06-03T20:19:33.021 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 6.0560000000000001e-06 2023-06-03T20:19:33.021 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:19:33.022 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:19:33.022 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "queued_for_pg", 2023-06-03T20:19:33.022 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:14:16.066556+0000", 2023-06-03T20:19:33.023 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 5.9845e-05 2023-06-03T20:19:33.023 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:19:33.023 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:19:33.023 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "reached_pg", 2023-06-03T20:19:33.023 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:14:16.066616+0000", 2023-06-03T20:19:33.023 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.00014320899999999999 2023-06-03T20:19:33.023 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:19:33.024 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:19:33.024 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "started", 2023-06-03T20:19:33.024 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:14:16.066759+0000", 2023-06-03T20:19:33.024 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.000143409 2023-06-03T20:19:33.025 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:19:33.025 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:19:33.025 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "waiting for subops from 0", 2023-06-03T20:19:33.025 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:14:16.066903+0000", 2023-06-03T20:19:33.025 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.0054952229999999996 2023-06-03T20:19:33.025 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:19:33.025 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:19:33.026 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "op_commit", 2023-06-03T20:19:33.026 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:14:16.072398+0000", 2023-06-03T20:19:33.026 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.16948649399999999 2023-06-03T20:19:33.026 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:19:33.026 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:19:33.026 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:19:33.026 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:14:16.241884+0000", 2023-06-03T20:19:33.027 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 5.4187000000000001e-05 2023-06-03T20:19:33.027 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:19:33.027 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:19:33.027 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "commit_sent", 2023-06-03T20:19:33.027 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:14:16.241939+0000", 2023-06-03T20:19:33.027 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 9.6272000000000003e-05 2023-06-03T20:19:33.027 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:19:33.028 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:19:33.028 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "done", 2023-06-03T20:19:33.028 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:14:16.242035+0000", 2023-06-03T20:19:33.028 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.17550977200000001 2023-06-03T20:19:33.028 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:19:33.028 INFO:teuthology.orchestra.run.smithi164.stdout: ] 2023-06-03T20:19:33.028 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:19:33.029 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:19:33.029 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:19:33.029 INFO:teuthology.orchestra.run.smithi164.stdout: "description": "osd_op(client.46187.0:157567 11.7 11:e8940e02:::benchmark_data_smithi089_165412_object52039:head [delete] snapc 0=[] ondisk+write+known_if_redirected e203)", 2023-06-03T20:19:33.029 INFO:teuthology.orchestra.run.smithi164.stdout: "initiated_at": "2023-06-03T20:14:16.073472+0000", 2023-06-03T20:19:33.029 INFO:teuthology.orchestra.run.smithi164.stdout: "age": 316.78499276399998, 2023-06-03T20:19:33.029 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.17008615699999999, 2023-06-03T20:19:33.030 INFO:teuthology.orchestra.run.smithi164.stdout: "type_data": { 2023-06-03T20:19:33.030 INFO:teuthology.orchestra.run.smithi164.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-03T20:19:33.030 INFO:teuthology.orchestra.run.smithi164.stdout: "client_info": { 2023-06-03T20:19:33.030 INFO:teuthology.orchestra.run.smithi164.stdout: "client": "client.46187", 2023-06-03T20:19:33.030 INFO:teuthology.orchestra.run.smithi164.stdout: "client_addr": "172.21.15.89:0/880684210", 2023-06-03T20:19:33.030 INFO:teuthology.orchestra.run.smithi164.stdout: "tid": 157567 2023-06-03T20:19:33.030 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:19:33.031 INFO:teuthology.orchestra.run.smithi164.stdout: "events": [ 2023-06-03T20:19:33.031 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:19:33.031 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "initiated", 2023-06-03T20:19:33.031 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:14:16.073472+0000", 2023-06-03T20:19:33.031 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0 2023-06-03T20:19:33.031 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:19:33.031 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:19:33.032 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "throttled", 2023-06-03T20:19:33.032 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:14:16.073472+0000", 2023-06-03T20:19:33.032 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 4294967295.9999981 2023-06-03T20:19:33.032 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:19:33.032 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:19:33.033 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "header_read", 2023-06-03T20:19:33.033 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:14:16.073471+0000", 2023-06-03T20:19:33.033 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 7.7680000000000008e-06 2023-06-03T20:19:33.033 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:19:33.033 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:19:33.033 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "all_read", 2023-06-03T20:19:33.034 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:14:16.073478+0000", 2023-06-03T20:19:33.034 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 1.3050999999999999e-05 2023-06-03T20:19:33.034 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:19:33.034 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:19:33.034 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "dispatched", 2023-06-03T20:19:33.034 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:14:16.073491+0000", 2023-06-03T20:19:33.035 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 6.1079999999999998e-06 2023-06-03T20:19:33.035 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:19:33.035 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:19:33.035 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "queued_for_pg", 2023-06-03T20:19:33.035 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:14:16.073498+0000", 2023-06-03T20:19:33.036 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 4.6103000000000002e-05 2023-06-03T20:19:33.036 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:19:33.036 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:19:33.036 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "reached_pg", 2023-06-03T20:19:33.036 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:14:16.073544+0000", 2023-06-03T20:19:33.036 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.000101974 2023-06-03T20:19:33.036 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:19:33.037 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:19:33.037 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "started", 2023-06-03T20:19:33.037 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:14:16.073646+0000", 2023-06-03T20:19:33.037 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.000120938 2023-06-03T20:19:33.037 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:19:33.037 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:19:33.037 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "waiting for subops from 0", 2023-06-03T20:19:33.038 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:14:16.073767+0000", 2023-06-03T20:19:33.038 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.0041628070000000001 2023-06-03T20:19:33.038 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:19:33.038 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:19:33.038 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "op_commit", 2023-06-03T20:19:33.038 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:14:16.077929+0000", 2023-06-03T20:19:33.038 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.165489941 2023-06-03T20:19:33.039 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:19:33.039 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:19:33.039 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:19:33.039 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:14:16.243419+0000", 2023-06-03T20:19:33.039 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 5.0158999999999999e-05 2023-06-03T20:19:33.039 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:19:33.039 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:19:33.040 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "commit_sent", 2023-06-03T20:19:33.040 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:14:16.243470+0000", 2023-06-03T20:19:33.040 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 8.9085000000000005e-05 2023-06-03T20:19:33.040 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:19:33.040 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:19:33.040 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "done", 2023-06-03T20:19:33.040 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:14:16.243559+0000", 2023-06-03T20:19:33.041 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.17008615699999999 2023-06-03T20:19:33.041 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:19:33.041 INFO:teuthology.orchestra.run.smithi164.stdout: ] 2023-06-03T20:19:33.041 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:19:33.041 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:19:33.041 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:19:33.042 INFO:teuthology.orchestra.run.smithi164.stdout: "description": "osd_op(client.46187.0:162187 11.7 11:e80ebcaa:::benchmark_data_smithi089_165412_object56659:head [delete] snapc 0=[] ondisk+write+known_if_redirected e203)", 2023-06-03T20:19:33.042 INFO:teuthology.orchestra.run.smithi164.stdout: "initiated_at": "2023-06-03T20:14:22.714461+0000", 2023-06-03T20:19:33.042 INFO:teuthology.orchestra.run.smithi164.stdout: "age": 310.14400392300001, 2023-06-03T20:19:33.042 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.067202676000000003, 2023-06-03T20:19:33.043 INFO:teuthology.orchestra.run.smithi164.stdout: "type_data": { 2023-06-03T20:19:33.043 INFO:teuthology.orchestra.run.smithi164.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-03T20:19:33.043 INFO:teuthology.orchestra.run.smithi164.stdout: "client_info": { 2023-06-03T20:19:33.043 INFO:teuthology.orchestra.run.smithi164.stdout: "client": "client.46187", 2023-06-03T20:19:33.043 INFO:teuthology.orchestra.run.smithi164.stdout: "client_addr": "172.21.15.89:0/880684210", 2023-06-03T20:19:33.043 INFO:teuthology.orchestra.run.smithi164.stdout: "tid": 162187 2023-06-03T20:19:33.043 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:19:33.044 INFO:teuthology.orchestra.run.smithi164.stdout: "events": [ 2023-06-03T20:19:33.044 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:19:33.044 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "initiated", 2023-06-03T20:19:33.044 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:14:22.714461+0000", 2023-06-03T20:19:33.044 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0 2023-06-03T20:19:33.044 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:19:33.044 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:19:33.045 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "throttled", 2023-06-03T20:19:33.045 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:14:22.714461+0000", 2023-06-03T20:19:33.045 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 4294967295.999999 2023-06-03T20:19:33.045 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:19:33.046 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:19:33.046 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "header_read", 2023-06-03T20:19:33.046 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:14:22.714460+0000", 2023-06-03T20:19:33.047 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 5.3870000000000003e-06 2023-06-03T20:19:33.047 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:19:33.047 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:19:33.047 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "all_read", 2023-06-03T20:19:33.047 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:14:22.714466+0000", 2023-06-03T20:19:33.047 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 9.7969999999999995e-06 2023-06-03T20:19:33.047 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:19:33.048 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:19:33.048 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "dispatched", 2023-06-03T20:19:33.048 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:14:22.714475+0000", 2023-06-03T20:19:33.048 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 5.9000000000000003e-06 2023-06-03T20:19:33.048 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:19:33.048 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:19:33.049 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "queued_for_pg", 2023-06-03T20:19:33.049 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:14:22.714481+0000", 2023-06-03T20:19:33.049 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.0058075399999999999 2023-06-03T20:19:33.049 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:19:33.049 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:19:33.049 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "reached_pg", 2023-06-03T20:19:33.049 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:14:22.720289+0000", 2023-06-03T20:19:33.049 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.00015505299999999999 2023-06-03T20:19:33.050 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:19:33.050 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:19:33.050 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "started", 2023-06-03T20:19:33.050 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:14:22.720444+0000", 2023-06-03T20:19:33.050 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.00017082800000000001 2023-06-03T20:19:33.050 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:19:33.051 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:19:33.051 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "waiting for subops from 0", 2023-06-03T20:19:33.051 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:14:22.720615+0000", 2023-06-03T20:19:33.051 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.0045807230000000001 2023-06-03T20:19:33.052 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:19:33.052 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:19:33.052 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "op_commit", 2023-06-03T20:19:33.052 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:14:22.725196+0000", 2023-06-03T20:19:33.052 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.056340201999999999 2023-06-03T20:19:33.052 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:19:33.053 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:19:33.053 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:19:33.053 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:14:22.781536+0000", 2023-06-03T20:19:33.053 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 4.6949000000000001e-05 2023-06-03T20:19:33.053 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:19:33.053 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:19:33.053 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "commit_sent", 2023-06-03T20:19:33.053 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:14:22.781583+0000", 2023-06-03T20:19:33.054 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 8.1274999999999997e-05 2023-06-03T20:19:33.054 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:19:33.054 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:19:33.054 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "done", 2023-06-03T20:19:33.054 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:14:22.781664+0000", 2023-06-03T20:19:33.054 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.067202676000000003 2023-06-03T20:19:33.054 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:19:33.055 INFO:teuthology.orchestra.run.smithi164.stdout: ] 2023-06-03T20:19:33.055 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:19:33.055 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:19:33.055 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:19:33.055 INFO:teuthology.orchestra.run.smithi164.stdout: "description": "osd_op(client.46187.0:167121 11.7 11:ef4cbeae:::benchmark_data_smithi089_165412_object61593:head [delete] snapc 0=[] ondisk+write+known_if_redirected e203)", 2023-06-03T20:19:33.055 INFO:teuthology.orchestra.run.smithi164.stdout: "initiated_at": "2023-06-03T20:14:29.639659+0000", 2023-06-03T20:19:33.055 INFO:teuthology.orchestra.run.smithi164.stdout: "age": 303.218806685, 2023-06-03T20:19:33.056 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.053990353999999997, 2023-06-03T20:19:33.056 INFO:teuthology.orchestra.run.smithi164.stdout: "type_data": { 2023-06-03T20:19:33.056 INFO:teuthology.orchestra.run.smithi164.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-03T20:19:33.056 INFO:teuthology.orchestra.run.smithi164.stdout: "client_info": { 2023-06-03T20:19:33.056 INFO:teuthology.orchestra.run.smithi164.stdout: "client": "client.46187", 2023-06-03T20:19:33.056 INFO:teuthology.orchestra.run.smithi164.stdout: "client_addr": "172.21.15.89:0/880684210", 2023-06-03T20:19:33.056 INFO:teuthology.orchestra.run.smithi164.stdout: "tid": 167121 2023-06-03T20:19:33.057 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:19:33.057 INFO:teuthology.orchestra.run.smithi164.stdout: "events": [ 2023-06-03T20:19:33.057 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:19:33.057 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "initiated", 2023-06-03T20:19:33.057 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:14:29.639659+0000", 2023-06-03T20:19:33.057 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0 2023-06-03T20:19:33.057 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:19:33.058 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:19:33.058 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "throttled", 2023-06-03T20:19:33.058 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:14:29.639659+0000", 2023-06-03T20:19:33.058 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 4294967295.9999986 2023-06-03T20:19:33.058 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:19:33.058 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:19:33.058 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "header_read", 2023-06-03T20:19:33.058 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:14:29.639657+0000", 2023-06-03T20:19:33.059 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 1.1212e-05 2023-06-03T20:19:33.059 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:19:33.059 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:19:33.059 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "all_read", 2023-06-03T20:19:33.059 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:14:29.639668+0000", 2023-06-03T20:19:33.059 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 1.1627999999999999e-05 2023-06-03T20:19:33.059 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:19:33.060 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:19:33.060 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "dispatched", 2023-06-03T20:19:33.060 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:14:29.639680+0000", 2023-06-03T20:19:33.060 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 6.6939999999999998e-06 2023-06-03T20:19:33.060 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:19:33.060 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:19:33.061 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "queued_for_pg", 2023-06-03T20:19:33.061 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:14:29.639687+0000", 2023-06-03T20:19:33.061 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.0045642199999999999 2023-06-03T20:19:33.061 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:19:33.061 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:19:33.061 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "reached_pg", 2023-06-03T20:19:33.061 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:14:29.644251+0000", 2023-06-03T20:19:33.062 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.00014954399999999999 2023-06-03T20:19:33.062 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:19:33.062 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:19:33.062 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "started", 2023-06-03T20:19:33.062 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:14:29.644401+0000", 2023-06-03T20:19:33.062 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.00016934799999999999 2023-06-03T20:19:33.062 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:19:33.063 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:19:33.063 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "waiting for subops from 0", 2023-06-03T20:19:33.063 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:14:29.644570+0000", 2023-06-03T20:19:33.063 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.005954109 2023-06-03T20:19:33.063 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:19:33.063 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:19:33.063 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "op_commit", 2023-06-03T20:19:33.063 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:14:29.650524+0000", 2023-06-03T20:19:33.064 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.042899977999999998 2023-06-03T20:19:33.064 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:19:33.064 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:19:33.064 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:19:33.064 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:14:29.693424+0000", 2023-06-03T20:19:33.064 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.00010846899999999999 2023-06-03T20:19:33.065 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:19:33.065 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:19:33.065 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "commit_sent", 2023-06-03T20:19:33.065 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:14:29.693532+0000", 2023-06-03T20:19:33.066 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.00011639 2023-06-03T20:19:33.066 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:19:33.066 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:19:33.066 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "done", 2023-06-03T20:19:33.066 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:14:29.693649+0000", 2023-06-03T20:19:33.067 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.053990353999999997 2023-06-03T20:19:33.067 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:19:33.067 INFO:teuthology.orchestra.run.smithi164.stdout: ] 2023-06-03T20:19:33.067 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:19:33.067 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:19:33.067 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:19:33.068 INFO:teuthology.orchestra.run.smithi164.stdout: "description": "osd_op(client.46187.0:191253 11.7 11:e75a8a28:::benchmark_data_smithi089_165412_object85725:head [delete] snapc 0=[] ondisk+write+known_if_redirected e204)", 2023-06-03T20:19:33.068 INFO:teuthology.orchestra.run.smithi164.stdout: "initiated_at": "2023-06-03T20:15:02.817113+0000", 2023-06-03T20:19:33.068 INFO:teuthology.orchestra.run.smithi164.stdout: "age": 270.04135199900003, 2023-06-03T20:19:33.068 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.063194636999999998, 2023-06-03T20:19:33.069 INFO:teuthology.orchestra.run.smithi164.stdout: "type_data": { 2023-06-03T20:19:33.069 INFO:teuthology.orchestra.run.smithi164.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-03T20:19:33.069 INFO:teuthology.orchestra.run.smithi164.stdout: "client_info": { 2023-06-03T20:19:33.069 INFO:teuthology.orchestra.run.smithi164.stdout: "client": "client.46187", 2023-06-03T20:19:33.069 INFO:teuthology.orchestra.run.smithi164.stdout: "client_addr": "172.21.15.89:0/880684210", 2023-06-03T20:19:33.069 INFO:teuthology.orchestra.run.smithi164.stdout: "tid": 191253 2023-06-03T20:19:33.070 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:19:33.070 INFO:teuthology.orchestra.run.smithi164.stdout: "events": [ 2023-06-03T20:19:33.070 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:19:33.070 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "initiated", 2023-06-03T20:19:33.071 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:15:02.817113+0000", 2023-06-03T20:19:33.071 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0 2023-06-03T20:19:33.071 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:19:33.071 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:19:33.072 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "throttled", 2023-06-03T20:19:33.072 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:15:02.817113+0000", 2023-06-03T20:19:33.072 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 4294967295.9999981 2023-06-03T20:19:33.072 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:19:33.072 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:19:33.072 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "header_read", 2023-06-03T20:19:33.072 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:15:02.817111+0000", 2023-06-03T20:19:33.073 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 9.7839999999999998e-06 2023-06-03T20:19:33.073 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:19:33.073 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:19:33.074 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "all_read", 2023-06-03T20:19:33.074 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:15:02.817121+0000", 2023-06-03T20:19:33.074 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 1.7187e-05 2023-06-03T20:19:33.074 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:19:33.074 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:19:33.074 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "dispatched", 2023-06-03T20:19:33.074 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:15:02.817138+0000", 2023-06-03T20:19:33.075 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 1.0186999999999999e-05 2023-06-03T20:19:33.075 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:19:33.075 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:19:33.075 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "queued_for_pg", 2023-06-03T20:19:33.075 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:15:02.817149+0000", 2023-06-03T20:19:33.075 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 7.9213000000000002e-05 2023-06-03T20:19:33.075 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:19:33.076 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:19:33.076 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "reached_pg", 2023-06-03T20:19:33.076 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:15:02.817228+0000", 2023-06-03T20:19:33.076 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.00014837499999999999 2023-06-03T20:19:33.076 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:19:33.076 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:19:33.076 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "started", 2023-06-03T20:19:33.077 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:15:02.817376+0000", 2023-06-03T20:19:33.077 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.00017582700000000001 2023-06-03T20:19:33.077 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:19:33.077 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:19:33.077 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "waiting for subops from 0", 2023-06-03T20:19:33.077 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:15:02.817552+0000", 2023-06-03T20:19:33.077 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.0056557630000000003 2023-06-03T20:19:33.078 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:19:33.078 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:19:33.078 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "op_commit", 2023-06-03T20:19:33.078 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:15:02.823208+0000", 2023-06-03T20:19:33.078 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.056857980000000002 2023-06-03T20:19:33.078 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:19:33.078 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:19:33.078 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:19:33.079 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:15:02.880066+0000", 2023-06-03T20:19:33.079 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 8.6182999999999998e-05 2023-06-03T20:19:33.079 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:19:33.079 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:19:33.079 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "commit_sent", 2023-06-03T20:19:33.079 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:15:02.880152+0000", 2023-06-03T20:19:33.079 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.000155933 2023-06-03T20:19:33.080 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:19:33.080 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:19:33.080 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "done", 2023-06-03T20:19:33.080 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:15:02.880308+0000", 2023-06-03T20:19:33.080 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.063194636999999998 2023-06-03T20:19:33.080 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:19:33.080 INFO:teuthology.orchestra.run.smithi164.stdout: ] 2023-06-03T20:19:33.081 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:19:33.081 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:19:33.081 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:19:33.081 INFO:teuthology.orchestra.run.smithi164.stdout: "description": "osd_op(client.46187.0:199652 11.7 11:e0944a34:::benchmark_data_smithi089_165412_object94124:head [delete] snapc 0=[] ondisk+write+known_if_redirected e205)", 2023-06-03T20:19:33.081 INFO:teuthology.orchestra.run.smithi164.stdout: "initiated_at": "2023-06-03T20:15:14.621611+0000", 2023-06-03T20:19:33.081 INFO:teuthology.orchestra.run.smithi164.stdout: "age": 258.23685393, 2023-06-03T20:19:33.081 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.063514320999999999, 2023-06-03T20:19:33.082 INFO:teuthology.orchestra.run.smithi164.stdout: "type_data": { 2023-06-03T20:19:33.082 INFO:teuthology.orchestra.run.smithi164.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-03T20:19:33.082 INFO:teuthology.orchestra.run.smithi164.stdout: "client_info": { 2023-06-03T20:19:33.082 INFO:teuthology.orchestra.run.smithi164.stdout: "client": "client.46187", 2023-06-03T20:19:33.082 INFO:teuthology.orchestra.run.smithi164.stdout: "client_addr": "172.21.15.89:0/880684210", 2023-06-03T20:19:33.082 INFO:teuthology.orchestra.run.smithi164.stdout: "tid": 199652 2023-06-03T20:19:33.082 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:19:33.083 INFO:teuthology.orchestra.run.smithi164.stdout: "events": [ 2023-06-03T20:19:33.083 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:19:33.083 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "initiated", 2023-06-03T20:19:33.083 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:15:14.621611+0000", 2023-06-03T20:19:33.083 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0 2023-06-03T20:19:33.083 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:19:33.083 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:19:33.083 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "throttled", 2023-06-03T20:19:33.084 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:15:14.621611+0000", 2023-06-03T20:19:33.084 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 4294967295.9999976 2023-06-03T20:19:33.084 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:19:33.084 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:19:33.084 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "header_read", 2023-06-03T20:19:33.084 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:15:14.621609+0000", 2023-06-03T20:19:33.084 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 1.3423e-05 2023-06-03T20:19:33.085 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:19:33.085 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:19:33.085 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "all_read", 2023-06-03T20:19:33.085 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:15:14.621622+0000", 2023-06-03T20:19:33.085 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 1.5238999999999999e-05 2023-06-03T20:19:33.085 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:19:33.085 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:19:33.086 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "dispatched", 2023-06-03T20:19:33.086 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:15:14.621637+0000", 2023-06-03T20:19:33.086 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 7.3209999999999999e-06 2023-06-03T20:19:33.086 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:19:33.086 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:19:33.086 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "queued_for_pg", 2023-06-03T20:19:33.086 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:15:14.621645+0000", 2023-06-03T20:19:33.087 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.0040448539999999996 2023-06-03T20:19:33.087 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:19:33.087 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:19:33.087 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "reached_pg", 2023-06-03T20:19:33.087 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:15:14.625690+0000", 2023-06-03T20:19:33.087 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.00017671599999999999 2023-06-03T20:19:33.087 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:19:33.087 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:19:33.088 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "started", 2023-06-03T20:19:33.088 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:15:14.625866+0000", 2023-06-03T20:19:33.088 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.000171865 2023-06-03T20:19:33.088 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:19:33.088 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:19:33.088 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "waiting for subops from 0", 2023-06-03T20:19:33.088 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:15:14.626038+0000", 2023-06-03T20:19:33.089 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.0050553880000000001 2023-06-03T20:19:33.089 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:19:33.089 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:19:33.089 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "op_commit", 2023-06-03T20:19:33.089 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:15:14.631093+0000", 2023-06-03T20:19:33.089 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.053896292999999998 2023-06-03T20:19:33.089 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:19:33.090 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:19:33.090 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:19:33.090 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:15:14.684990+0000", 2023-06-03T20:19:33.090 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 6.2364000000000001e-05 2023-06-03T20:19:33.090 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:19:33.090 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:19:33.090 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "commit_sent", 2023-06-03T20:19:33.091 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:15:14.685052+0000", 2023-06-03T20:19:33.091 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 7.3452000000000001e-05 2023-06-03T20:19:33.091 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:19:33.091 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:19:33.091 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "done", 2023-06-03T20:19:33.091 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:15:14.685126+0000", 2023-06-03T20:19:33.091 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.063514320999999999 2023-06-03T20:19:33.091 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:19:33.092 INFO:teuthology.orchestra.run.smithi164.stdout: ] 2023-06-03T20:19:33.092 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:19:33.092 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:19:33.092 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:19:33.092 INFO:teuthology.orchestra.run.smithi164.stdout: "description": "osd_op(client.46187.0:200002 11.7 11:ee8a8b2f:::benchmark_data_smithi089_165412_object94474:head [delete] snapc 0=[] ondisk+write+known_if_redirected e205)", 2023-06-03T20:19:33.092 INFO:teuthology.orchestra.run.smithi164.stdout: "initiated_at": "2023-06-03T20:15:15.050643+0000", 2023-06-03T20:19:33.093 INFO:teuthology.orchestra.run.smithi164.stdout: "age": 257.807822192, 2023-06-03T20:19:33.093 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.054438082999999998, 2023-06-03T20:19:33.093 INFO:teuthology.orchestra.run.smithi164.stdout: "type_data": { 2023-06-03T20:19:33.093 INFO:teuthology.orchestra.run.smithi164.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-03T20:19:33.093 INFO:teuthology.orchestra.run.smithi164.stdout: "client_info": { 2023-06-03T20:19:33.093 INFO:teuthology.orchestra.run.smithi164.stdout: "client": "client.46187", 2023-06-03T20:19:33.093 INFO:teuthology.orchestra.run.smithi164.stdout: "client_addr": "172.21.15.89:0/880684210", 2023-06-03T20:19:33.094 INFO:teuthology.orchestra.run.smithi164.stdout: "tid": 200002 2023-06-03T20:19:33.094 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:19:33.094 INFO:teuthology.orchestra.run.smithi164.stdout: "events": [ 2023-06-03T20:19:33.094 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:19:33.094 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "initiated", 2023-06-03T20:19:33.094 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:15:15.050643+0000", 2023-06-03T20:19:33.094 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0 2023-06-03T20:19:33.095 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:19:33.095 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:19:33.095 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "throttled", 2023-06-03T20:19:33.095 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:15:15.050643+0000", 2023-06-03T20:19:33.095 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 4294967295.9999981 2023-06-03T20:19:33.095 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:19:33.095 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:19:33.096 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "header_read", 2023-06-03T20:19:33.096 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:15:15.050641+0000", 2023-06-03T20:19:33.096 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 9.1689999999999998e-06 2023-06-03T20:19:33.096 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:19:33.096 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:19:33.096 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "all_read", 2023-06-03T20:19:33.096 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:15:15.050650+0000", 2023-06-03T20:19:33.097 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 1.6483000000000001e-05 2023-06-03T20:19:33.097 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:19:33.097 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:19:33.097 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "dispatched", 2023-06-03T20:19:33.097 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:15:15.050667+0000", 2023-06-03T20:19:33.097 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 8.3250000000000008e-06 2023-06-03T20:19:33.097 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:19:33.098 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:19:33.098 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "queued_for_pg", 2023-06-03T20:19:33.098 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:15:15.050675+0000", 2023-06-03T20:19:33.098 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 6.1532000000000007e-05 2023-06-03T20:19:33.098 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:19:33.098 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:19:33.099 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "reached_pg", 2023-06-03T20:19:33.099 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:15:15.050737+0000", 2023-06-03T20:19:33.099 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.00012258200000000001 2023-06-03T20:19:33.099 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:19:33.099 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:19:33.099 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "started", 2023-06-03T20:19:33.099 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:15:15.050859+0000", 2023-06-03T20:19:33.100 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.000140481 2023-06-03T20:19:33.100 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:19:33.100 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:19:33.100 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "waiting for subops from 0", 2023-06-03T20:19:33.100 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:15:15.051000+0000", 2023-06-03T20:19:33.100 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.012606338 2023-06-03T20:19:33.100 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:19:33.101 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:19:33.101 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "op_commit", 2023-06-03T20:19:33.101 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:15:15.063606+0000", 2023-06-03T20:19:33.101 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.041320293000000001 2023-06-03T20:19:33.101 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:19:33.101 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:19:33.101 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:19:33.101 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:15:15.104926+0000", 2023-06-03T20:19:33.102 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 5.9085000000000001e-05 2023-06-03T20:19:33.102 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:19:33.102 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:19:33.102 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "commit_sent", 2023-06-03T20:19:33.102 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:15:15.104985+0000", 2023-06-03T20:19:33.102 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 9.5703000000000002e-05 2023-06-03T20:19:33.102 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:19:33.103 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:19:33.103 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "done", 2023-06-03T20:19:33.103 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:15:15.105081+0000", 2023-06-03T20:19:33.103 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.054438082999999998 2023-06-03T20:19:33.103 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:19:33.103 INFO:teuthology.orchestra.run.smithi164.stdout: ] 2023-06-03T20:19:33.103 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:19:33.104 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:19:33.104 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:19:33.104 INFO:teuthology.orchestra.run.smithi164.stdout: "description": "osd_repop(client.46373.0:113766 12.f e226/222 12:f07f90ed:::benchmark_data_smithi089_191647_object8157:head v 226'7240, mlcod=226'7236)", 2023-06-03T20:19:33.104 INFO:teuthology.orchestra.run.smithi164.stdout: "initiated_at": "2023-06-03T20:18:57.686484+0000", 2023-06-03T20:19:33.104 INFO:teuthology.orchestra.run.smithi164.stdout: "age": 35.171981481000003, 2023-06-03T20:19:33.104 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.016576437999999999, 2023-06-03T20:19:33.104 INFO:teuthology.orchestra.run.smithi164.stdout: "type_data": { 2023-06-03T20:19:33.105 INFO:teuthology.orchestra.run.smithi164.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-03T20:19:33.105 INFO:teuthology.orchestra.run.smithi164.stdout: "events": [ 2023-06-03T20:19:33.105 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:19:33.105 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "initiated", 2023-06-03T20:19:33.106 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:18:57.686484+0000", 2023-06-03T20:19:33.106 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0 2023-06-03T20:19:33.106 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:19:33.106 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:19:33.106 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "throttled", 2023-06-03T20:19:33.106 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:18:57.686484+0000", 2023-06-03T20:19:33.107 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 4294967295.9999981 2023-06-03T20:19:33.107 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:19:33.107 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:19:33.107 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "header_read", 2023-06-03T20:19:33.107 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:18:57.686482+0000", 2023-06-03T20:19:33.107 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 1.5451000000000002e-05 2023-06-03T20:19:33.107 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:19:33.107 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:19:33.108 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "all_read", 2023-06-03T20:19:33.108 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:18:57.686497+0000", 2023-06-03T20:19:33.108 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 2.8170999999999998e-05 2023-06-03T20:19:33.108 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:19:33.108 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:19:33.108 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "dispatched", 2023-06-03T20:19:33.108 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:18:57.686525+0000", 2023-06-03T20:19:33.109 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 1.2665e-05 2023-06-03T20:19:33.109 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:19:33.109 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:19:33.109 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "queued_for_pg", 2023-06-03T20:19:33.110 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:18:57.686538+0000", 2023-06-03T20:19:33.110 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.000120703 2023-06-03T20:19:33.110 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:19:33.111 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:19:33.111 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "reached_pg", 2023-06-03T20:19:33.111 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:18:57.686659+0000", 2023-06-03T20:19:33.111 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 7.0018999999999995e-05 2023-06-03T20:19:33.111 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:19:33.111 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:19:33.112 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "started", 2023-06-03T20:19:33.112 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:18:57.686729+0000", 2023-06-03T20:19:33.112 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.016283095000000001 2023-06-03T20:19:33.112 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:19:33.112 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:19:33.112 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "commit_sent", 2023-06-03T20:19:33.112 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:18:57.703012+0000", 2023-06-03T20:19:33.113 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 4.8270999999999999e-05 2023-06-03T20:19:33.113 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:19:33.113 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:19:33.113 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "done", 2023-06-03T20:19:33.113 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:18:57.703060+0000", 2023-06-03T20:19:33.113 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.016576437999999999 2023-06-03T20:19:33.113 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:19:33.114 INFO:teuthology.orchestra.run.smithi164.stdout: ] 2023-06-03T20:19:33.114 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:19:33.114 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:19:33.114 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:19:33.114 INFO:teuthology.orchestra.run.smithi164.stdout: "description": "osd_repop(client.46373.0:141922 12.f e231/222 12:f3504cfe:::benchmark_data_smithi089_191647_object36313:head v 231'8989, mlcod=231'8987)", 2023-06-03T20:19:33.114 INFO:teuthology.orchestra.run.smithi164.stdout: "initiated_at": "2023-06-03T20:19:32.612241+0000", 2023-06-03T20:19:33.114 INFO:teuthology.orchestra.run.smithi164.stdout: "age": 0.246223847, 2023-06-03T20:19:33.115 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.0094826270000000004, 2023-06-03T20:19:33.115 INFO:teuthology.orchestra.run.smithi164.stdout: "type_data": { 2023-06-03T20:19:33.115 INFO:teuthology.orchestra.run.smithi164.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-03T20:19:33.115 INFO:teuthology.orchestra.run.smithi164.stdout: "events": [ 2023-06-03T20:19:33.115 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:19:33.115 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "initiated", 2023-06-03T20:19:33.115 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:19:32.612241+0000", 2023-06-03T20:19:33.115 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0 2023-06-03T20:19:33.116 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:19:33.116 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:19:33.116 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "throttled", 2023-06-03T20:19:33.116 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:19:32.612241+0000", 2023-06-03T20:19:33.116 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 4294967295.9999986 2023-06-03T20:19:33.116 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:19:33.116 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:19:33.117 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "header_read", 2023-06-03T20:19:33.117 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:19:32.612240+0000", 2023-06-03T20:19:33.117 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 1.5118e-05 2023-06-03T20:19:33.117 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:19:33.117 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:19:33.117 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "all_read", 2023-06-03T20:19:33.117 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:19:32.612255+0000", 2023-06-03T20:19:33.118 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 2.3099e-05 2023-06-03T20:19:33.118 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:19:33.118 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:19:33.118 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "dispatched", 2023-06-03T20:19:33.118 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:19:32.612278+0000", 2023-06-03T20:19:33.118 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 9.5359999999999998e-06 2023-06-03T20:19:33.118 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:19:33.119 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:19:33.119 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "queued_for_pg", 2023-06-03T20:19:33.119 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:19:32.612288+0000", 2023-06-03T20:19:33.119 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 9.5586000000000003e-05 2023-06-03T20:19:33.119 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:19:33.119 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:19:33.119 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "reached_pg", 2023-06-03T20:19:33.119 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:19:32.612383+0000", 2023-06-03T20:19:33.120 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 4.5448999999999998e-05 2023-06-03T20:19:33.120 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:19:33.120 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:19:33.120 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "started", 2023-06-03T20:19:33.120 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:19:32.612429+0000", 2023-06-03T20:19:33.120 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.009241137 2023-06-03T20:19:33.120 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:19:33.121 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:19:33.121 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "commit_sent", 2023-06-03T20:19:33.121 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:19:32.621670+0000", 2023-06-03T20:19:33.121 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 5.4286999999999997e-05 2023-06-03T20:19:33.121 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:19:33.121 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:19:33.121 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "done", 2023-06-03T20:19:33.122 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:19:32.621724+0000", 2023-06-03T20:19:33.122 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.0094826270000000004 2023-06-03T20:19:33.122 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:19:33.122 INFO:teuthology.orchestra.run.smithi164.stdout: ] 2023-06-03T20:19:33.122 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:19:33.122 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:19:33.122 INFO:teuthology.orchestra.run.smithi164.stdout: ] 2023-06-03T20:19:33.123 INFO:teuthology.orchestra.run.smithi164.stdout:} 2023-06-03T20:19:33.123 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:19:33.124 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-06-03T20:19:33.125 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-06-03T20:19:33.152 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:19:33.155 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:19:33.156 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:19:33.156 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:19:33 smithi164 bash[59462]: cluster 1685823571.5939994 osd.3 (osd.3) 5 : cluster 0 12.18 deep-scrub starts 2023-06-03T20:19:33.157 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:19:33 smithi164 bash[59462]: cluster 1685823572.854233 mon.a (mon.0) 1249 : cluster 3 Health check update: Degraded data redundancy: 36549/218823 objects degraded (16.703%), 19 pgs degraded, 20 pgs undersized (PG_DEGRADED) 2023-06-03T20:19:33.256 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:19:33.299 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:19:33.302 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:19:33.304 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:19:33.405 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:19:33.434 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:19:33.436 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:19:33.437 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:19:33.529 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:19:33 smithi089 bash[78452]: cluster 1685823571.5939994 osd.3 (osd.3) 5 : cluster 0 12.18 deep-scrub starts 2023-06-03T20:19:33.529 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:19:33 smithi089 bash[78452]: cluster 1685823572.854233 mon.a (mon.0) 1249 : cluster 3 Health check update: Degraded data redundancy: 36549/218823 objects degraded (16.703%), 19 pgs degraded, 20 pgs undersized (PG_DEGRADED) 2023-06-03T20:19:33.529 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:19:33 smithi089 bash[75068]: cluster 1685823571.5939994 osd.3 (osd.3) 5 : cluster 0 12.18 deep-scrub starts 2023-06-03T20:19:33.529 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:19:33 smithi089 bash[75068]: cluster 1685823572.854233 mon.a (mon.0) 1249 : cluster 3 Health check update: Degraded data redundancy: 36549/218823 objects degraded (16.703%), 19 pgs degraded, 20 pgs undersized (PG_DEGRADED) 2023-06-03T20:19:33.538 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:19:33.577 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:19:33.581 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:19:33.582 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:19:33.684 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:19:33.724 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:19:33.731 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:19:33.732 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:19:33.832 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:19:33.869 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:19:33.869 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:19:33.870 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:19:33.954 DEBUG:teuthology.orchestra.run.smithi164:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph daemon osd.6 dump_ops_in_flight 2023-06-03T20:19:33.971 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:19:34.028 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:19:34.031 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:19:34.032 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:19:34.132 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:19:34.142 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:19:34 smithi089 bash[75068]: cluster 1685823572.906958 mgr.y (mgr.34104) 1307 : cluster 0 pgmap v1307: 99 pgs: 1 active+clean+scrubbing, 20 active+clean+remapped, 19 active+undersized+degraded, 11 active+undersized, 48 active+clean; 4.4 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 558 op/s; 36318/215955 objects degraded (16.817%); 6/215955 objects misplaced (0.003%); 0 B/s, 0 objects/s recovering 2023-06-03T20:19:34.143 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:19:34 smithi089 bash[78452]: cluster 1685823572.906958 mgr.y (mgr.34104) 1307 : cluster 0 pgmap v1307: 99 pgs: 1 active+clean+scrubbing, 20 active+clean+remapped, 19 active+undersized+degraded, 11 active+undersized, 48 active+clean; 4.4 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 558 op/s; 36318/215955 objects degraded (16.817%); 6/215955 objects misplaced (0.003%); 0 B/s, 0 objects/s recovering 2023-06-03T20:19:34.170 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:19:34.175 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:19:34.176 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:19:34.197 INFO:tasks.thrashosds.thrasher:in_osds: [0, 2, 3, 4, 5, 7] out_osds: [1, 6] dead_osds: [4] live_osds: [0, 1, 2, 3, 5, 6, 7] 2023-06-03T20:19:34.198 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-06-03T20:19:34.198 INFO:tasks.thrashosds.thrasher:fixing pg num pool device_health_metrics 2023-06-03T20:19:34.199 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph pg dump --format=json 2023-06-03T20:19:34.277 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:19:34.286 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:19:34 smithi164 bash[59462]: cluster 1685823572.906958 mgr.y (mgr.34104) 1307 : cluster 0 pgmap v1307: 99 pgs: 1 active+clean+scrubbing, 20 active+clean+remapped, 19 active+undersized+degraded, 11 active+undersized, 48 active+clean; 4.4 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 558 op/s; 36318/215955 objects degraded (16.817%); 6/215955 objects misplaced (0.003%); 0 B/s, 0 objects/s recovering 2023-06-03T20:19:34.309 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:19:34.311 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:19:34.312 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:19:34.413 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:19:34.471 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:19:34.478 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:19:34.479 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:19:34.580 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:19:34.614 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:19:34.618 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:19:34.618 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:19:34.719 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:19:34.761 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:19:34.765 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:19:34.766 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:19:34.867 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:19:34.901 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:19:34.905 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:19:34.906 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:19:35.007 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:19:35.039 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:19:35.040 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:19:35.041 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:19:35.126 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-06-03T20:19:35.142 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:19:35.175 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:19:35.179 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:19:35.180 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:19:35.280 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:19:35.337 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:19:35.343 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:19:35.344 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:19:35.445 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:19:35.476 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:19:35.479 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:19:35.480 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:19:35.581 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:19:35.615 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:19:35.617 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:19:35.618 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:19:35.719 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:19:35.753 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:19:35.755 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:19:35.756 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:19:35.857 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:19:35.889 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:19:35.893 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:19:35.894 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:19:35.995 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:19:36.022 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:19:36.025 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:19:36.026 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:19:36.127 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:19:36.157 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:19:36 smithi164 bash[59462]: cluster 1685823574.9079247 mgr.y (mgr.34104) 1308 : cluster 0 pgmap v1308: 99 pgs: 20 active+clean+remapped, 1 active+clean+scrubbing+deep, 19 active+undersized+degraded, 11 active+undersized, 48 active+clean; 4.3 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 35112/209367 objects degraded (16.771%); 6/209367 objects misplaced (0.003%); 0 B/s, 0 objects/s recovering 2023-06-03T20:19:36.164 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:19:36.166 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:19:36.167 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:19:36.213 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph osd set noscrub 2023-06-03T20:19:36.268 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:19:36.279 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:19:36 smithi089 bash[75068]: cluster 1685823574.9079247 mgr.y (mgr.34104) 1308 : cluster 0 pgmap v1308: 99 pgs: 20 active+clean+remapped, 1 active+clean+scrubbing+deep, 19 active+undersized+degraded, 11 active+undersized, 48 active+clean; 4.3 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 35112/209367 objects degraded (16.771%); 6/209367 objects misplaced (0.003%); 0 B/s, 0 objects/s recovering 2023-06-03T20:19:36.279 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 20:19:36 smithi089 bash[71163]: ::ffff:172.21.15.164 - - [03/Jun/2023:20:19:36] "GET /metrics HTTP/1.1" 200 228933 "" "Prometheus/2.33.4" 2023-06-03T20:19:36.279 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:19:36 smithi089 bash[78452]: cluster 1685823574.9079247 mgr.y (mgr.34104) 1308 : cluster 0 pgmap v1308: 99 pgs: 20 active+clean+remapped, 1 active+clean+scrubbing+deep, 19 active+undersized+degraded, 11 active+undersized, 48 active+clean; 4.3 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 35112/209367 objects degraded (16.771%); 6/209367 objects misplaced (0.003%); 0 B/s, 0 objects/s recovering 2023-06-03T20:19:36.311 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:19:36.316 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:19:36.317 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:19:36.418 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:19:36.449 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:19:36.452 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:19:36.453 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:19:36.554 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:19:36.590 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:19:36.593 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:19:36.594 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:19:36.695 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:19:36.730 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:19:36.732 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:19:36.733 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:19:36.833 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:19:36.864 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:19:36.866 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:19:36.866 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:19:36.967 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:19:36.999 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:19:37.004 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:19:37.005 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:19:37.106 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:19:37.140 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:19:37.143 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:19:37.143 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:19:37.244 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:19:37.278 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:19:37.280 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:19:37.281 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:19:37.381 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:19:37.414 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:19:37.417 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:19:37.418 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:19:37.518 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:19:37.549 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:19:37.552 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:19:37.553 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:19:37.654 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:19:37.692 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:19:37.696 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:19:37.697 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:19:37.798 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:19:37.829 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:19:37.833 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:19:37.833 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:19:37.934 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:19:37.965 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:19:37.968 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:19:37.969 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:19:38.069 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:19:38.112 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:19:38.115 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:19:38.116 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:19:38.217 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:19:38.265 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:19:38.268 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:19:38.268 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:19:38.369 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:19:38.404 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:19:38.407 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:19:38 smithi164 bash[59462]: cluster 1685823576.9088736 mgr.y (mgr.34104) 1309 : cluster 0 pgmap v1309: 99 pgs: 20 active+clean+remapped, 1 active+clean+scrubbing+deep, 19 active+undersized+degraded, 11 active+undersized, 48 active+clean; 4.1 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 33882/201438 objects degraded (16.820%); 6/201438 objects misplaced (0.003%); 0 B/s, 0 objects/s recovering 2023-06-03T20:19:38.407 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:19:38 smithi164 bash[59462]: audit 1685823577.13944 mon.a (mon.0) 1250 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:19:38.407 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:19:38 smithi164 bash[59462]: audit 1685823577.1443548 mon.c (mon.2) 567 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T20:19:38.407 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:19:38 smithi164 bash[59462]: cluster 1685823577.8554046 mon.a (mon.0) 1251 : cluster 3 Health check update: Degraded data redundancy: 33882/201438 objects degraded (16.820%), 19 pgs degraded, 20 pgs undersized (PG_DEGRADED) 2023-06-03T20:19:38.408 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:19:38.408 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:19:38.509 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:19:38.529 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:19:38 smithi089 bash[78452]: cluster 1685823576.9088736 mgr.y (mgr.34104) 1309 : cluster 0 pgmap v1309: 99 pgs: 20 active+clean+remapped, 1 active+clean+scrubbing+deep, 19 active+undersized+degraded, 11 active+undersized, 48 active+clean; 4.1 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 33882/201438 objects degraded (16.820%); 6/201438 objects misplaced (0.003%); 0 B/s, 0 objects/s recovering 2023-06-03T20:19:38.529 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:19:38 smithi089 bash[78452]: audit 1685823577.13944 mon.a (mon.0) 1250 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:19:38.529 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:19:38 smithi089 bash[78452]: audit 1685823577.1443548 mon.c (mon.2) 567 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T20:19:38.529 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:19:38 smithi089 bash[78452]: cluster 1685823577.8554046 mon.a (mon.0) 1251 : cluster 3 Health check update: Degraded data redundancy: 33882/201438 objects degraded (16.820%), 19 pgs degraded, 20 pgs undersized (PG_DEGRADED) 2023-06-03T20:19:38.530 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:19:38 smithi089 bash[75068]: cluster 1685823576.9088736 mgr.y (mgr.34104) 1309 : cluster 0 pgmap v1309: 99 pgs: 20 active+clean+remapped, 1 active+clean+scrubbing+deep, 19 active+undersized+degraded, 11 active+undersized, 48 active+clean; 4.1 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 33882/201438 objects degraded (16.820%); 6/201438 objects misplaced (0.003%); 0 B/s, 0 objects/s recovering 2023-06-03T20:19:38.530 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:19:38 smithi089 bash[75068]: audit 1685823577.13944 mon.a (mon.0) 1250 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:19:38.530 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:19:38 smithi089 bash[75068]: audit 1685823577.1443548 mon.c (mon.2) 567 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T20:19:38.530 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:19:38 smithi089 bash[75068]: cluster 1685823577.8554046 mon.a (mon.0) 1251 : cluster 3 Health check update: Degraded data redundancy: 33882/201438 objects degraded (16.820%), 19 pgs degraded, 20 pgs undersized (PG_DEGRADED) 2023-06-03T20:19:38.543 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:19:38.545 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:19:38.545 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:19:38.646 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:19:38.686 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:19:38.691 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:19:38.692 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:19:38.756 INFO:teuthology.orchestra.run.smithi164.stderr:Inferring config /var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/mon.b/config 2023-06-03T20:19:38.792 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:19:38.823 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:19:38.826 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:19:38.826 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:19:38.927 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:19:38.967 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:19:38.975 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:19:38.975 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:19:39.077 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:19:39.104 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:19:39.108 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:19:39.109 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:19:39.202 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/mon.a/config 2023-06-03T20:19:39.205 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/mon.a/config 2023-06-03T20:19:39.210 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:19:39.234 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:19:39.237 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:19:39.237 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:19:39.338 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:19:39.377 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:19:39.383 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:19:39.384 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:19:39.485 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:19:39.519 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:19:39.521 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:19:39.521 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:19:39.622 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:19:39.664 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:19:39.668 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:19:39.669 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:19:39.770 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:19:39.816 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:19:39.820 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:19:39.821 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:19:39.922 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:19:39.946 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:19:39.949 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:19:39.950 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:19:40.051 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:19:40.090 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:19:40.095 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:19:40.095 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:19:40.196 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:19:40.226 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:19:40.232 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/mon.a/config 2023-06-03T20:19:40.237 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:19:40.238 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:19:40.334 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:19:40 smithi164 bash[59462]: cluster 1685823578.9098601 mgr.y (mgr.34104) 1310 : cluster 0 pgmap v1310: 99 pgs: 20 active+clean+remapped, 1 active+clean+scrubbing+deep, 19 active+undersized+degraded, 11 active+undersized, 48 active+clean; 4.1 GiB data, 14 GiB used, 523 GiB / 536 GiB avail; 33632/199914 objects degraded (16.823%); 6/199914 objects misplaced (0.003%); 0 B/s, 0 objects/s recovering 2023-06-03T20:19:40.339 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:19:40.371 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:19:40.374 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:19:40.375 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:19:40.476 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:19:40.506 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:19:40.509 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:19:40.509 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:19:40.529 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:19:40 smithi089 bash[75068]: cluster 1685823578.9098601 mgr.y (mgr.34104) 1310 : cluster 0 pgmap v1310: 99 pgs: 20 active+clean+remapped, 1 active+clean+scrubbing+deep, 19 active+undersized+degraded, 11 active+undersized, 48 active+clean; 4.1 GiB data, 14 GiB used, 523 GiB / 536 GiB avail; 33632/199914 objects degraded (16.823%); 6/199914 objects misplaced (0.003%); 0 B/s, 0 objects/s recovering 2023-06-03T20:19:40.530 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:19:40 smithi089 bash[78452]: cluster 1685823578.9098601 mgr.y (mgr.34104) 1310 : cluster 0 pgmap v1310: 99 pgs: 20 active+clean+remapped, 1 active+clean+scrubbing+deep, 19 active+undersized+degraded, 11 active+undersized, 48 active+clean; 4.1 GiB data, 14 GiB used, 523 GiB / 536 GiB avail; 33632/199914 objects degraded (16.823%); 6/199914 objects misplaced (0.003%); 0 B/s, 0 objects/s recovering 2023-06-03T20:19:40.611 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:19:40.653 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:19:40.656 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:19:40.657 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:19:40.758 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:19:40.791 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:19:40.794 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:19:40.794 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:19:40.895 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:19:40.924 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:19:40.928 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:19:40.928 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:19:41.029 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:19:41.061 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:19:41.064 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:19:41.065 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:19:41.166 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:19:41.177 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:19:41 smithi164 bash[59462]: audit 1685823581.1082141 mon.c (mon.2) 568 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-03T20:19:41.198 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:19:41.202 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:19:41.203 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:19:41.304 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:19:41.313 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:19:41 smithi089 bash[78452]: audit 1685823581.1082141 mon.c (mon.2) 568 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-03T20:19:41.314 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:19:41 smithi089 bash[75068]: audit 1685823581.1082141 mon.c (mon.2) 568 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-03T20:19:41.348 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:19:41.354 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:19:41.355 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:19:41.455 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:19:41.482 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:19:41.484 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:19:41.484 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:19:41.586 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:19:41.617 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:19:41.621 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:19:41.622 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:19:41.723 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:19:41.757 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:19:41.761 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:19:41.762 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:19:41.863 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:19:41.891 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:19:41.893 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:19:41.894 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:19:41.995 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:19:42.045 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:19:42.049 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:19:42.050 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:19:42.150 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:19:42.190 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:19:42.194 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:19:42.194 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:19:42.295 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:19:42.330 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:19:42.332 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:19:42.332 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:19:42.433 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:19:42.467 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:19:42.470 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:19:42.470 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:19:42.487 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:19:42 smithi089 bash[78452]: cluster 1685823580.9124203 mgr.y (mgr.34104) 1311 : cluster 0 pgmap v1311: 99 pgs: 20 active+clean+remapped, 1 active+clean+scrubbing+deep, 19 active+undersized+degraded, 11 active+undersized, 48 active+clean; 3.9 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 26 op/s; 31978/191586 objects degraded (16.691%); 6/191586 objects misplaced (0.003%) 2023-06-03T20:19:42.487 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:19:42 smithi089 bash[75068]: cluster 1685823580.9124203 mgr.y (mgr.34104) 1311 : cluster 0 pgmap v1311: 99 pgs: 20 active+clean+remapped, 1 active+clean+scrubbing+deep, 19 active+undersized+degraded, 11 active+undersized, 48 active+clean; 3.9 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 26 op/s; 31978/191586 objects degraded (16.691%); 6/191586 objects misplaced (0.003%) 2023-06-03T20:19:42.572 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:19:42.587 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:19:42 smithi164 bash[59462]: cluster 1685823580.9124203 mgr.y (mgr.34104) 1311 : cluster 0 pgmap v1311: 99 pgs: 20 active+clean+remapped, 1 active+clean+scrubbing+deep, 19 active+undersized+degraded, 11 active+undersized, 48 active+clean; 3.9 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 26 op/s; 31978/191586 objects degraded (16.691%); 6/191586 objects misplaced (0.003%) 2023-06-03T20:19:42.610 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:19:42.613 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:19:42.614 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:19:42.715 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:19:42.749 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:19:42.753 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:19:42.754 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:19:42.855 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:19:42.890 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:19:42.894 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:19:42.894 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:19:42.995 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:19:43.040 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:19:43.043 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:19:43.044 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:19:43.145 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:19:43.176 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:19:43.179 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:19:43.180 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:19:43.281 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:19:43.308 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:19:43.309 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:19:43.310 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:19:43.316 INFO:teuthology.orchestra.run.smithi164.stdout:{ 2023-06-03T20:19:43.316 INFO:teuthology.orchestra.run.smithi164.stdout: "ops": [], 2023-06-03T20:19:43.316 INFO:teuthology.orchestra.run.smithi164.stdout: "num_ops": 0 2023-06-03T20:19:43.316 INFO:teuthology.orchestra.run.smithi164.stdout:} 2023-06-03T20:19:43.333 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:19:43 smithi164 bash[59462]: cluster 1685823582.8566637 mon.a (mon.0) 1252 : cluster 3 Health check update: Degraded data redundancy: 31978/191586 objects degraded (16.691%), 19 pgs degraded, 20 pgs undersized (PG_DEGRADED) 2023-06-03T20:19:43.411 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:19:43.446 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:19:43.451 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:19:43.451 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:19:43.529 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:19:43 smithi089 bash[78452]: cluster 1685823582.8566637 mon.a (mon.0) 1252 : cluster 3 Health check update: Degraded data redundancy: 31978/191586 objects degraded (16.691%), 19 pgs degraded, 20 pgs undersized (PG_DEGRADED) 2023-06-03T20:19:43.529 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:19:43 smithi089 bash[75068]: cluster 1685823582.8566637 mon.a (mon.0) 1252 : cluster 3 Health check update: Degraded data redundancy: 31978/191586 objects degraded (16.691%), 19 pgs degraded, 20 pgs undersized (PG_DEGRADED) 2023-06-03T20:19:43.552 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:19:43.579 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:19:43.581 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:19:43.582 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:19:43.683 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:19:43.710 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:19:43.713 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:19:43.713 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:19:43.814 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:19:43.845 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:19:43.847 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:19:43.847 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:19:43.948 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:19:43.977 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:19:43.980 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:19:43.980 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:19:44.081 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:19:44.110 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:19:44.119 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:19:44.120 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:19:44.220 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:19:44.227 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:19:44 smithi089 bash[78452]: cluster 1685823582.9132063 mgr.y (mgr.34104) 1312 : cluster 0 pgmap v1312: 99 pgs: 20 active+clean+remapped, 1 active+clean+scrubbing+deep, 19 active+undersized+degraded, 11 active+undersized, 48 active+clean; 3.8 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 116 op/s; 31683/188226 objects degraded (16.832%); 6/188226 objects misplaced (0.003%) 2023-06-03T20:19:44.227 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:19:44 smithi089 bash[75068]: cluster 1685823582.9132063 mgr.y (mgr.34104) 1312 : cluster 0 pgmap v1312: 99 pgs: 20 active+clean+remapped, 1 active+clean+scrubbing+deep, 19 active+undersized+degraded, 11 active+undersized, 48 active+clean; 3.8 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 116 op/s; 31683/188226 objects degraded (16.832%); 6/188226 objects misplaced (0.003%) 2023-06-03T20:19:44.231 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:19:44 smithi164 bash[59462]: cluster 1685823582.9132063 mgr.y (mgr.34104) 1312 : cluster 0 pgmap v1312: 99 pgs: 20 active+clean+remapped, 1 active+clean+scrubbing+deep, 19 active+undersized+degraded, 11 active+undersized, 48 active+clean; 3.8 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 116 op/s; 31683/188226 objects degraded (16.832%); 6/188226 objects misplaced (0.003%) 2023-06-03T20:19:44.258 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:19:44.259 DEBUG:teuthology.orchestra.run.smithi164:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph daemon osd.6 dump_blocked_ops 2023-06-03T20:19:44.264 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:19:44.265 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:19:44.366 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:19:44.397 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:19:44.401 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:19:44.401 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:19:44.502 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:19:44.538 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:19:44.545 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:19:44.546 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:19:44.647 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:19:44.676 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:19:44.678 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:19:44.679 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:19:44.780 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:19:44.809 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:19:44.813 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:19:44.814 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:19:44.914 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:19:44.948 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:19:44.954 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:19:44.954 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:19:45.055 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:19:45.090 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:19:45.093 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:19:45.093 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:19:45.194 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:19:45.228 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:19:45.231 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:19:45.231 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:19:45.332 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:19:45.333 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:19:45 smithi164 bash[59462]: cluster 1685823583.7747905 osd.3 (osd.3) 6 : cluster 0 12.18 deep-scrub ok 2023-06-03T20:19:45.354 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:19:45.355 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:19:45.356 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:19:45.423 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:19:45 smithi089 bash[78452]: cluster 1685823583.7747905 osd.3 (osd.3) 6 : cluster 0 12.18 deep-scrub ok 2023-06-03T20:19:45.424 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:19:45 smithi089 bash[75068]: cluster 1685823583.7747905 osd.3 (osd.3) 6 : cluster 0 12.18 deep-scrub ok 2023-06-03T20:19:45.456 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:19:45.489 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:19:45.493 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:19:45.494 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:19:45.594 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:19:45.628 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:19:45.631 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:19:45.631 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:19:45.732 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:19:45.764 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:19:45.765 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:19:45.766 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:19:45.867 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:19:45.903 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:19:45.905 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:19:45.906 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:19:46.007 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:19:46.042 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:19:46.048 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:19:46.050 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:19:46.151 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:19:46.190 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:19:46.194 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:19:46.195 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:19:46.296 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:19:46.329 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:19:46.331 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:19:46.332 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:19:46.433 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:19:46.443 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:19:46 smithi164 bash[59462]: cluster 1685823584.7940915 osd.3 (osd.3) 7 : cluster 0 12.11 scrub starts 2023-06-03T20:19:46.444 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:19:46 smithi164 bash[59462]: cluster 1685823584.9141955 mgr.y (mgr.34104) 1313 : cluster 0 pgmap v1313: 99 pgs: 20 active+clean+remapped, 19 active+undersized+degraded, 11 active+undersized, 49 active+clean; 3.7 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 30596/182469 objects degraded (16.768%); 6/182469 objects misplaced (0.003%) 2023-06-03T20:19:46.462 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:19:46 smithi089 bash[75068]: cluster 1685823584.7940915 osd.3 (osd.3) 7 : cluster 0 12.11 scrub starts 2023-06-03T20:19:46.463 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:19:46 smithi089 bash[75068]: cluster 1685823584.9141955 mgr.y (mgr.34104) 1313 : cluster 0 pgmap v1313: 99 pgs: 20 active+clean+remapped, 19 active+undersized+degraded, 11 active+undersized, 49 active+clean; 3.7 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 30596/182469 objects degraded (16.768%); 6/182469 objects misplaced (0.003%) 2023-06-03T20:19:46.463 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 20:19:46 smithi089 bash[71163]: ::ffff:172.21.15.164 - - [03/Jun/2023:20:19:46] "GET /metrics HTTP/1.1" 200 228969 "" "Prometheus/2.33.4" 2023-06-03T20:19:46.463 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:19:46 smithi089 bash[78452]: cluster 1685823584.7940915 osd.3 (osd.3) 7 : cluster 0 12.11 scrub starts 2023-06-03T20:19:46.463 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:19:46 smithi089 bash[78452]: cluster 1685823584.9141955 mgr.y (mgr.34104) 1313 : cluster 0 pgmap v1313: 99 pgs: 20 active+clean+remapped, 19 active+undersized+degraded, 11 active+undersized, 49 active+clean; 3.7 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 30596/182469 objects degraded (16.768%); 6/182469 objects misplaced (0.003%) 2023-06-03T20:19:46.464 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:19:46.467 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:19:46.467 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:19:46.568 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:19:46.624 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:19:46.626 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:19:46.627 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:19:46.728 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:19:46.778 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:19:46.783 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:19:46.783 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:19:46.884 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:19:46.914 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:19:46.916 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:19:46.917 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:19:47.017 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:19:47.059 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:19:47.062 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:19:47.062 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:19:47.163 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:19:47.198 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:19:47.201 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:19:47.201 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:19:47.279 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:19:47 smithi089 bash[75068]: cluster 1685823585.6682239 osd.3 (osd.3) 8 : cluster 0 12.11 scrub ok 2023-06-03T20:19:47.279 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:19:47 smithi089 bash[75068]: cluster 1685823586.4019554 osd.2 (osd.2) 2 : cluster 0 12.15 scrub starts 2023-06-03T20:19:47.279 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:19:47 smithi089 bash[78452]: cluster 1685823585.6682239 osd.3 (osd.3) 8 : cluster 0 12.11 scrub ok 2023-06-03T20:19:47.279 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:19:47 smithi089 bash[78452]: cluster 1685823586.4019554 osd.2 (osd.2) 2 : cluster 0 12.15 scrub starts 2023-06-03T20:19:47.303 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:19:47.336 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:19:47.338 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:19:47.339 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:19:47.440 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:19:47.502 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:19:47.505 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:19:47.506 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:19:47.584 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:19:47 smithi164 bash[59462]: cluster 1685823585.6682239 osd.3 (osd.3) 8 : cluster 0 12.11 scrub ok 2023-06-03T20:19:47.584 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:19:47 smithi164 bash[59462]: cluster 1685823586.4019554 osd.2 (osd.2) 2 : cluster 0 12.15 scrub starts 2023-06-03T20:19:47.606 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:19:47.647 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:19:47.654 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:19:47.654 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:19:47.755 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:19:47.791 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:19:47.794 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:19:47.797 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:19:47.899 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:19:47.958 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:19:47.963 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:19:47.963 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:19:48.064 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:19:48.085 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:19:48.087 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:19:48.087 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:19:48.188 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:19:48.218 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:19:48.221 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:19:48.221 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:19:48.322 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:19:48.353 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:19:48.357 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:19:48.358 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:19:48.426 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:19:48 smithi089 bash[75068]: cluster 1685823586.915089 mgr.y (mgr.34104) 1314 : cluster 0 pgmap v1314: 99 pgs: 20 active+clean+remapped, 19 active+undersized+degraded, 11 active+undersized, 49 active+clean; 3.6 GiB data, 13 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 580 op/s; 29403/174684 objects degraded (16.832%); 6/174684 objects misplaced (0.003%) 2023-06-03T20:19:48.426 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:19:48 smithi089 bash[75068]: cluster 1685823587.187918 osd.2 (osd.2) 3 : cluster 0 12.15 scrub ok 2023-06-03T20:19:48.427 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:19:48 smithi089 bash[75068]: cluster 1685823587.2693424 osd.0 (osd.0) 6 : cluster 0 12.3 deep-scrub starts 2023-06-03T20:19:48.427 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:19:48 smithi089 bash[75068]: audit 1685823587.7686186 mon.a (mon.0) 1253 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:19:48.427 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:19:48 smithi089 bash[75068]: audit 1685823587.781091 mon.a (mon.0) 1254 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:19:48.427 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:19:48 smithi089 bash[75068]: cluster 1685823587.8577597 mon.a (mon.0) 1255 : cluster 3 Health check update: Degraded data redundancy: 29403/174684 objects degraded (16.832%), 19 pgs degraded, 20 pgs undersized (PG_DEGRADED) 2023-06-03T20:19:48.427 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:19:48 smithi089 bash[78452]: cluster 1685823586.915089 mgr.y (mgr.34104) 1314 : cluster 0 pgmap v1314: 99 pgs: 20 active+clean+remapped, 19 active+undersized+degraded, 11 active+undersized, 49 active+clean; 3.6 GiB data, 13 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 580 op/s; 29403/174684 objects degraded (16.832%); 6/174684 objects misplaced (0.003%) 2023-06-03T20:19:48.427 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:19:48 smithi089 bash[78452]: cluster 1685823587.187918 osd.2 (osd.2) 3 : cluster 0 12.15 scrub ok 2023-06-03T20:19:48.428 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:19:48 smithi089 bash[78452]: cluster 1685823587.2693424 osd.0 (osd.0) 6 : cluster 0 12.3 deep-scrub starts 2023-06-03T20:19:48.428 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:19:48 smithi089 bash[78452]: audit 1685823587.7686186 mon.a (mon.0) 1253 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:19:48.428 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:19:48 smithi089 bash[78452]: audit 1685823587.781091 mon.a (mon.0) 1254 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:19:48.428 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:19:48 smithi089 bash[78452]: cluster 1685823587.8577597 mon.a (mon.0) 1255 : cluster 3 Health check update: Degraded data redundancy: 29403/174684 objects degraded (16.832%), 19 pgs degraded, 20 pgs undersized (PG_DEGRADED) 2023-06-03T20:19:48.458 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:19:48.467 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:19:48 smithi164 bash[59462]: cluster 1685823586.915089 mgr.y (mgr.34104) 1314 : cluster 0 pgmap v1314: 99 pgs: 20 active+clean+remapped, 19 active+undersized+degraded, 11 active+undersized, 49 active+clean; 3.6 GiB data, 13 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 580 op/s; 29403/174684 objects degraded (16.832%); 6/174684 objects misplaced (0.003%) 2023-06-03T20:19:48.467 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:19:48 smithi164 bash[59462]: cluster 1685823587.187918 osd.2 (osd.2) 3 : cluster 0 12.15 scrub ok 2023-06-03T20:19:48.468 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:19:48 smithi164 bash[59462]: cluster 1685823587.2693424 osd.0 (osd.0) 6 : cluster 0 12.3 deep-scrub starts 2023-06-03T20:19:48.468 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:19:48 smithi164 bash[59462]: audit 1685823587.7686186 mon.a (mon.0) 1253 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:19:48.468 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:19:48 smithi164 bash[59462]: audit 1685823587.781091 mon.a (mon.0) 1254 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:19:48.468 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:19:48 smithi164 bash[59462]: cluster 1685823587.8577597 mon.a (mon.0) 1255 : cluster 3 Health check update: Degraded data redundancy: 29403/174684 objects degraded (16.832%), 19 pgs degraded, 20 pgs undersized (PG_DEGRADED) 2023-06-03T20:19:48.490 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:19:48.492 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:19:48.493 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:19:48.594 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:19:48.624 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:19:48.626 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:19:48.626 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:19:48.727 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:19:48.762 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:19:48.766 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:19:48.767 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:19:48.830 INFO:teuthology.orchestra.run.smithi164.stderr:Inferring config /var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/mon.b/config 2023-06-03T20:19:48.868 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:19:48.905 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:19:48.908 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:19:48.909 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:19:49.010 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:19:49.052 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:19:49.057 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:19:49.057 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:19:49.158 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:19:49.199 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:19:49.202 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:19:49.202 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:19:49.303 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:19:49.342 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:19:49.344 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:19:49.344 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:19:49.445 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:19:49.455 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:19:49 smithi089 bash[75068]: audit 1685823588.3819127 mon.a (mon.0) 1256 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:19:49.455 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:19:49 smithi089 bash[75068]: audit 1685823588.3885918 mon.a (mon.0) 1257 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:19:49.455 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:19:49 smithi089 bash[75068]: cluster 1685823588.9156885 mgr.y (mgr.34104) 1315 : cluster 0 pgmap v1315: 99 pgs: 20 active+clean+remapped, 19 active+undersized+degraded, 11 active+undersized, 49 active+clean; 3.5 GiB data, 12 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 407 op/s; 29110/173013 objects degraded (16.825%); 6/173013 objects misplaced (0.003%) 2023-06-03T20:19:49.455 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:19:49 smithi089 bash[75068]: audit 1685823588.9708703 mon.c (mon.2) 569 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-03T20:19:49.455 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:19:49 smithi089 bash[75068]: audit 1685823588.9720464 mon.c (mon.2) 570 : audit 1 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-03T20:19:49.456 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:19:49 smithi089 bash[75068]: audit 1685823588.9788132 mon.a (mon.0) 1258 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:19:49.456 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:19:49 smithi089 bash[78452]: audit 1685823588.3819127 mon.a (mon.0) 1256 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:19:49.456 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:19:49 smithi089 bash[78452]: audit 1685823588.3885918 mon.a (mon.0) 1257 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:19:49.457 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:19:49 smithi089 bash[78452]: cluster 1685823588.9156885 mgr.y (mgr.34104) 1315 : cluster 0 pgmap v1315: 99 pgs: 20 active+clean+remapped, 19 active+undersized+degraded, 11 active+undersized, 49 active+clean; 3.5 GiB data, 12 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 407 op/s; 29110/173013 objects degraded (16.825%); 6/173013 objects misplaced (0.003%) 2023-06-03T20:19:49.457 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:19:49 smithi089 bash[78452]: audit 1685823588.9708703 mon.c (mon.2) 569 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-03T20:19:49.457 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:19:49 smithi089 bash[78452]: audit 1685823588.9720464 mon.c (mon.2) 570 : audit 1 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-03T20:19:49.457 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:19:49 smithi089 bash[78452]: audit 1685823588.9788132 mon.a (mon.0) 1258 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:19:49.485 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:19:49.490 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:19:49.491 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:19:49.592 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:19:49.622 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:19:49.624 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:19:49.625 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:19:49.725 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:19:49.735 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:19:49 smithi164 bash[59462]: audit 1685823588.3819127 mon.a (mon.0) 1256 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:19:49.735 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:19:49 smithi164 bash[59462]: audit 1685823588.3885918 mon.a (mon.0) 1257 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:19:49.735 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:19:49 smithi164 bash[59462]: cluster 1685823588.9156885 mgr.y (mgr.34104) 1315 : cluster 0 pgmap v1315: 99 pgs: 20 active+clean+remapped, 19 active+undersized+degraded, 11 active+undersized, 49 active+clean; 3.5 GiB data, 12 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 407 op/s; 29110/173013 objects degraded (16.825%); 6/173013 objects misplaced (0.003%) 2023-06-03T20:19:49.735 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:19:49 smithi164 bash[59462]: audit 1685823588.9708703 mon.c (mon.2) 569 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-03T20:19:49.735 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:19:49 smithi164 bash[59462]: audit 1685823588.9720464 mon.c (mon.2) 570 : audit 1 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-03T20:19:49.736 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:19:49 smithi164 bash[59462]: audit 1685823588.9788132 mon.a (mon.0) 1258 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:19:49.754 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:19:49.757 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:19:49.757 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:19:49.858 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:19:49.887 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:19:49.889 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:19:49.890 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:19:49.990 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:19:50.030 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:19:50.035 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:19:50.036 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:19:50.137 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:19:50.165 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:19:50.167 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:19:50.168 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:19:50.269 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:19:50.302 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:19:50.304 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:19:50.305 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:19:50.405 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:19:50.432 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:19:50.436 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:19:50.437 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:19:50.538 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:19:50.568 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:19:50.572 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:19:50.573 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:19:50.674 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:19:50.710 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:19:50.714 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:19:50.715 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:19:50.816 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:19:50.848 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:19:50.853 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:19:50.853 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:19:50.954 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:19:50.967 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:19:50 smithi164 bash[59462]: cluster 1685823590.4384203 osd.2 (osd.2) 4 : cluster 0 12.5 scrub starts 2023-06-03T20:19:50.987 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:19:50.989 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:19:50.990 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:19:51.091 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:19:51.128 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:19:51.130 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:19:51.131 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:19:51.134 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:19:50 smithi089 bash[75068]: cluster 1685823590.4384203 osd.2 (osd.2) 4 : cluster 0 12.5 scrub starts 2023-06-03T20:19:51.135 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:19:51 smithi089 bash[78452]: cluster 1685823590.4384203 osd.2 (osd.2) 4 : cluster 0 12.5 scrub starts 2023-06-03T20:19:51.231 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:19:51.261 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:19:51.263 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:19:51.263 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:19:51.364 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:19:51.390 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:19:51.393 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:19:51.394 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:19:51.495 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:19:51.523 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:19:51.533 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:19:51.534 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:19:51.635 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:19:51.664 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:19:51.668 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:19:51.669 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:19:51.770 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:19:51.796 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:19:51.799 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:19:51.799 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:19:51.900 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:19:51.931 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:19:51.937 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:19:51.938 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:19:52.039 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:19:52.049 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:19:51 smithi164 bash[59462]: cluster 1685823590.9165115 mgr.y (mgr.34104) 1316 : cluster 0 pgmap v1316: 99 pgs: 20 active+clean+remapped, 19 active+undersized+degraded, 11 active+undersized, 49 active+clean; 3.4 GiB data, 12 GiB used, 524 GiB / 536 GiB avail; 27441/164793 objects degraded (16.652%); 6/164793 objects misplaced (0.004%) 2023-06-03T20:19:52.073 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:19:52.077 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:19:52.078 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:19:52.179 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:19:52.187 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:19:51 smithi089 bash[75068]: cluster 1685823590.9165115 mgr.y (mgr.34104) 1316 : cluster 0 pgmap v1316: 99 pgs: 20 active+clean+remapped, 19 active+undersized+degraded, 11 active+undersized, 49 active+clean; 3.4 GiB data, 12 GiB used, 524 GiB / 536 GiB avail; 27441/164793 objects degraded (16.652%); 6/164793 objects misplaced (0.004%) 2023-06-03T20:19:52.188 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:19:51 smithi089 bash[78452]: cluster 1685823590.9165115 mgr.y (mgr.34104) 1316 : cluster 0 pgmap v1316: 99 pgs: 20 active+clean+remapped, 19 active+undersized+degraded, 11 active+undersized, 49 active+clean; 3.4 GiB data, 12 GiB used, 524 GiB / 536 GiB avail; 27441/164793 objects degraded (16.652%); 6/164793 objects misplaced (0.004%) 2023-06-03T20:19:52.229 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:19:52.235 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:19:52.235 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:19:52.336 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:19:52.372 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:19:52.379 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:19:52.380 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:19:52.481 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:19:52.517 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:19:52.522 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:19:52.523 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:19:52.624 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:19:52.652 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:19:52.655 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:19:52.655 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:19:52.756 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:19:52.784 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:19:52.786 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:19:52.787 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:19:52.888 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:19:52.924 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:19:52.928 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:19:52.929 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:19:52.930 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:19:52 smithi164 bash[59462]: audit 1685823592.131288 mon.c (mon.2) 571 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T20:19:52.930 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:19:52 smithi164 bash[59462]: cluster 1685823592.8587983 mon.a (mon.0) 1259 : cluster 3 Health check update: Degraded data redundancy: 27441/164793 objects degraded (16.652%), 19 pgs degraded, 20 pgs undersized (PG_DEGRADED) 2023-06-03T20:19:53.029 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:19:52 smithi089 bash[75068]: audit 1685823592.131288 mon.c (mon.2) 571 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T20:19:53.029 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:19:52 smithi089 bash[75068]: cluster 1685823592.8587983 mon.a (mon.0) 1259 : cluster 3 Health check update: Degraded data redundancy: 27441/164793 objects degraded (16.652%), 19 pgs degraded, 20 pgs undersized (PG_DEGRADED) 2023-06-03T20:19:53.030 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:19:52 smithi089 bash[78452]: audit 1685823592.131288 mon.c (mon.2) 571 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T20:19:53.030 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:19:52 smithi089 bash[78452]: cluster 1685823592.8587983 mon.a (mon.0) 1259 : cluster 3 Health check update: Degraded data redundancy: 27441/164793 objects degraded (16.652%), 19 pgs degraded, 20 pgs undersized (PG_DEGRADED) 2023-06-03T20:19:53.031 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:19:53.062 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:19:53.064 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:19:53.065 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:19:53.166 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:19:53.201 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:19:53.201 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:19:53.202 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:19:53.303 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:19:53.332 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:19:53.334 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:19:53.334 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:19:53.435 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:19:53.467 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:19:53.473 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:19:53.473 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:19:53.504 INFO:teuthology.orchestra.run.smithi164.stdout:{ 2023-06-03T20:19:53.504 INFO:teuthology.orchestra.run.smithi164.stdout: "ops": [], 2023-06-03T20:19:53.504 INFO:teuthology.orchestra.run.smithi164.stdout: "complaint_time": 30, 2023-06-03T20:19:53.504 INFO:teuthology.orchestra.run.smithi164.stdout: "num_blocked_ops": 0 2023-06-03T20:19:53.504 INFO:teuthology.orchestra.run.smithi164.stdout:} 2023-06-03T20:19:53.574 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:19:53.604 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:19:53.610 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:19:53.611 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:19:53.712 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:19:53.742 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:19:53.745 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:19:53.746 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:19:53.847 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:19:53.888 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:19:53.891 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:19:53.892 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:19:53.993 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:19:54.025 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:19:54.027 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:19:54.028 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:19:54.084 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:19:53 smithi164 bash[59462]: cluster 1685823592.9172993 mgr.y (mgr.34104) 1317 : cluster 0 pgmap v1317: 99 pgs: 20 active+clean+remapped, 1 active+clean+scrubbing+deep, 19 active+undersized+degraded, 11 active+undersized, 48 active+clean; 3.3 GiB data, 12 GiB used, 524 GiB / 536 GiB avail; 27185/161679 objects degraded (16.814%); 6/161679 objects misplaced (0.004%) 2023-06-03T20:19:54.129 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:19:54.161 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:19:54.167 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:19:54.168 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:19:54.173 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:19:54 smithi089 bash[75068]: cluster 1685823592.9172993 mgr.y (mgr.34104) 1317 : cluster 0 pgmap v1317: 99 pgs: 20 active+clean+remapped, 1 active+clean+scrubbing+deep, 19 active+undersized+degraded, 11 active+undersized, 48 active+clean; 3.3 GiB data, 12 GiB used, 524 GiB / 536 GiB avail; 27185/161679 objects degraded (16.814%); 6/161679 objects misplaced (0.004%) 2023-06-03T20:19:54.174 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:19:54 smithi089 bash[78452]: cluster 1685823592.9172993 mgr.y (mgr.34104) 1317 : cluster 0 pgmap v1317: 99 pgs: 20 active+clean+remapped, 1 active+clean+scrubbing+deep, 19 active+undersized+degraded, 11 active+undersized, 48 active+clean; 3.3 GiB data, 12 GiB used, 524 GiB / 536 GiB avail; 27185/161679 objects degraded (16.814%); 6/161679 objects misplaced (0.004%) 2023-06-03T20:19:54.269 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:19:54.304 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:19:54.309 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:19:54.310 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:19:54.411 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:19:54.440 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:19:54.447 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:19:54.448 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:19:54.549 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:19:54.581 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:19:54.584 DEBUG:teuthology.orchestra.run.smithi164:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph daemon osd.6 dump_historic_ops 2023-06-03T20:19:54.586 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:19:54.587 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:19:54.688 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:19:54.718 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:19:54.721 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:19:54.722 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:19:54.823 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:19:54.857 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:19:54.862 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:19:54.863 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:19:54.964 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:19:54.996 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:19:54.999 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:19:55.000 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:19:55.101 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:19:55.110 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:19:54 smithi089 bash[78452]: cluster 1685823594.4237483 osd.2 (osd.2) 5 : cluster 0 12.5 scrub starts 2023-06-03T20:19:55.110 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:19:54 smithi089 bash[75068]: cluster 1685823594.4237483 osd.2 (osd.2) 5 : cluster 0 12.5 scrub starts 2023-06-03T20:19:55.151 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:19:55.154 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:19:55.155 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:19:55.255 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:19:55.287 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:19:55.288 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:19:55.289 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:19:55.333 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:19:54 smithi164 bash[59462]: cluster 1685823594.4237483 osd.2 (osd.2) 5 : cluster 0 12.5 scrub starts 2023-06-03T20:19:55.390 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:19:55.423 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:19:55.427 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:19:55.428 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:19:55.529 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:19:55.561 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:19:55.567 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:19:55.568 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:19:55.669 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:19:55.705 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:19:55.707 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:19:55.708 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:19:55.809 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:19:55.840 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:19:55.845 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:19:55.846 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:19:55.947 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:19:55.947 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:19:55 smithi089 bash[75068]: cluster 1685823594.9180849 mgr.y (mgr.34104) 1318 : cluster 0 pgmap v1318: 99 pgs: 20 active+clean+remapped, 1 active+clean+scrubbing+deep, 19 active+undersized+degraded, 11 active+undersized, 48 active+clean; 3.2 GiB data, 12 GiB used, 525 GiB / 536 GiB avail; 26057/155838 objects degraded (16.721%); 6/155838 objects misplaced (0.004%) 2023-06-03T20:19:55.948 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:19:55 smithi089 bash[78452]: cluster 1685823594.9180849 mgr.y (mgr.34104) 1318 : cluster 0 pgmap v1318: 99 pgs: 20 active+clean+remapped, 1 active+clean+scrubbing+deep, 19 active+undersized+degraded, 11 active+undersized, 48 active+clean; 3.2 GiB data, 12 GiB used, 525 GiB / 536 GiB avail; 26057/155838 objects degraded (16.721%); 6/155838 objects misplaced (0.004%) 2023-06-03T20:19:55.979 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:19:55.984 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:19:55.985 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:19:56.086 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:19:56.126 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:19:56.129 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:19:56.130 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:19:56.230 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:19:56.232 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 20:19:56 smithi089 bash[71163]: ::ffff:172.21.15.164 - - [03/Jun/2023:20:19:56] "GET /metrics HTTP/1.1" 200 228955 "" "Prometheus/2.33.4" 2023-06-03T20:19:56.240 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:19:55 smithi164 bash[59462]: cluster 1685823594.9180849 mgr.y (mgr.34104) 1318 : cluster 0 pgmap v1318: 99 pgs: 20 active+clean+remapped, 1 active+clean+scrubbing+deep, 19 active+undersized+degraded, 11 active+undersized, 48 active+clean; 3.2 GiB data, 12 GiB used, 525 GiB / 536 GiB avail; 26057/155838 objects degraded (16.721%); 6/155838 objects misplaced (0.004%) 2023-06-03T20:19:56.261 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:19:56.264 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:19:56.265 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:19:56.366 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:19:56.397 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:19:56.400 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:19:56.401 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:19:56.502 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:19:56.533 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:19:56.538 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:19:56.538 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:19:56.639 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:19:56.669 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:19:56.672 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:19:56.673 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:19:56.773 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:19:56.804 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:19:56.807 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:19:56.808 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:19:56.909 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:19:56.943 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:19:56.945 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:19:56.945 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:19:56.955 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:19:56 smithi089 bash[78452]: cluster 1685823596.4042866 osd.2 (osd.2) 6 : cluster 0 12.5 scrub starts 2023-06-03T20:19:56.956 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:19:56 smithi089 bash[75068]: cluster 1685823596.4042866 osd.2 (osd.2) 6 : cluster 0 12.5 scrub starts 2023-06-03T20:19:57.046 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:19:57.055 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:19:56 smithi164 bash[59462]: cluster 1685823596.4042866 osd.2 (osd.2) 6 : cluster 0 12.5 scrub starts 2023-06-03T20:19:57.076 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:19:57.081 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:19:57.082 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:19:57.183 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:19:57.213 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:19:57.215 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:19:57.216 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:19:57.317 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:19:57.348 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:19:57.350 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:19:57.350 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:19:57.451 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:19:57.483 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:19:57.487 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:19:57.487 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:19:57.588 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:19:57.617 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:19:57.618 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:19:57.619 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:19:57.720 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:19:57.752 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:19:57.755 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:19:57.756 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:19:57.857 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:19:57.890 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:19:57.893 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:19:57.894 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:19:57.995 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:19:58.029 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:19:57 smithi089 bash[75068]: cluster 1685823596.9190016 mgr.y (mgr.34104) 1319 : cluster 0 pgmap v1319: 99 pgs: 20 active+clean+remapped, 1 active+clean+scrubbing+deep, 19 active+undersized+degraded, 11 active+undersized, 48 active+clean; 3.0 GiB data, 11 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 127 op/s; 25014/148698 objects degraded (16.822%); 6/148698 objects misplaced (0.004%) 2023-06-03T20:19:58.029 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:19:57 smithi089 bash[75068]: cluster 1685823597.8433447 osd.0 (osd.0) 7 : cluster 0 12.3 deep-scrub ok 2023-06-03T20:19:58.029 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:19:57 smithi089 bash[75068]: cluster 1685823597.8601701 mon.a (mon.0) 1260 : cluster 3 Health check update: Degraded data redundancy: 25014/148698 objects degraded (16.822%), 19 pgs degraded, 20 pgs undersized (PG_DEGRADED) 2023-06-03T20:19:58.030 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:19:57 smithi089 bash[78452]: cluster 1685823596.9190016 mgr.y (mgr.34104) 1319 : cluster 0 pgmap v1319: 99 pgs: 20 active+clean+remapped, 1 active+clean+scrubbing+deep, 19 active+undersized+degraded, 11 active+undersized, 48 active+clean; 3.0 GiB data, 11 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 127 op/s; 25014/148698 objects degraded (16.822%); 6/148698 objects misplaced (0.004%) 2023-06-03T20:19:58.030 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:19:57 smithi089 bash[78452]: cluster 1685823597.8433447 osd.0 (osd.0) 7 : cluster 0 12.3 deep-scrub ok 2023-06-03T20:19:58.030 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:19:57 smithi089 bash[78452]: cluster 1685823597.8601701 mon.a (mon.0) 1260 : cluster 3 Health check update: Degraded data redundancy: 25014/148698 objects degraded (16.822%), 19 pgs degraded, 20 pgs undersized (PG_DEGRADED) 2023-06-03T20:19:58.053 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:19:58.059 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:19:58.060 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:19:58.160 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:19:58.169 INFO:teuthology.orchestra.run.smithi089.stdout:osd.0: {} 2023-06-03T20:19:58.203 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:19:58.205 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:19:58.206 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:19:58.206 INFO:teuthology.orchestra.run.smithi089.stdout:osd.1: {} 2023-06-03T20:19:58.253 INFO:teuthology.orchestra.run.smithi089.stdout:osd.2: {} 2023-06-03T20:19:58.290 INFO:teuthology.orchestra.run.smithi089.stdout:osd.3: {} 2023-06-03T20:19:58.307 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:19:58.320 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:19:57 smithi164 bash[59462]: cluster 1685823596.9190016 mgr.y (mgr.34104) 1319 : cluster 0 pgmap v1319: 99 pgs: 20 active+clean+remapped, 1 active+clean+scrubbing+deep, 19 active+undersized+degraded, 11 active+undersized, 48 active+clean; 3.0 GiB data, 11 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 127 op/s; 25014/148698 objects degraded (16.822%); 6/148698 objects misplaced (0.004%) 2023-06-03T20:19:58.321 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:19:57 smithi164 bash[59462]: cluster 1685823597.8433447 osd.0 (osd.0) 7 : cluster 0 12.3 deep-scrub ok 2023-06-03T20:19:58.321 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:19:57 smithi164 bash[59462]: cluster 1685823597.8601701 mon.a (mon.0) 1260 : cluster 3 Health check update: Degraded data redundancy: 25014/148698 objects degraded (16.822%), 19 pgs degraded, 20 pgs undersized (PG_DEGRADED) 2023-06-03T20:19:58.332 INFO:teuthology.orchestra.run.smithi089.stdout:osd.5: {} 2023-06-03T20:19:58.346 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:19:58.348 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:19:58.348 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:19:58.377 INFO:teuthology.orchestra.run.smithi089.stdout:osd.6: {} 2023-06-03T20:19:58.411 INFO:teuthology.orchestra.run.smithi089.stdout:osd.7: {} 2023-06-03T20:19:58.411 INFO:teuthology.orchestra.run.smithi089.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-06-03T20:19:58.412 INFO:teuthology.orchestra.run.smithi089.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-06-03T20:19:58.412 INFO:teuthology.orchestra.run.smithi089.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-06-03T20:19:58.412 INFO:teuthology.orchestra.run.smithi089.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-06-03T20:19:58.412 INFO:teuthology.orchestra.run.smithi089.stderr:Error ENXIO: problem getting command descriptions from osd.4 2023-06-03T20:19:58.412 INFO:teuthology.orchestra.run.smithi089.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-06-03T20:19:58.412 INFO:teuthology.orchestra.run.smithi089.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-06-03T20:19:58.413 INFO:teuthology.orchestra.run.smithi089.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-06-03T20:19:58.449 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:19:58.489 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:19:58.493 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:19:58.494 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:19:58.595 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:19:58.626 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:19:58.629 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:19:58.629 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:19:58.730 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:19:58.753 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:19:58.756 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:19:58.757 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:19:58.858 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:19:58.890 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:19:58.894 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:19:58.895 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:19:58.996 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:19:59.023 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:19:59.026 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:19:59.027 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:19:59.081 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:19:58 smithi089 bash[78452]: audit 1685823598.109323 mon.a (mon.0) 1261 : audit 0 from='client.? 172.21.15.89:0/3122406720' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-03T20:19:59.081 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:19:58 smithi089 bash[78452]: cluster 1685823598.2639604 osd.0 (osd.0) 8 : cluster 0 12.13 scrub starts 2023-06-03T20:19:59.081 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:19:58 smithi089 bash[78452]: cluster 1685823598.7771053 osd.0 (osd.0) 9 : cluster 0 12.13 scrub ok 2023-06-03T20:19:59.082 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:19:58 smithi089 bash[75068]: audit 1685823598.109323 mon.a (mon.0) 1261 : audit 0 from='client.? 172.21.15.89:0/3122406720' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-03T20:19:59.082 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:19:58 smithi089 bash[75068]: cluster 1685823598.2639604 osd.0 (osd.0) 8 : cluster 0 12.13 scrub starts 2023-06-03T20:19:59.082 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:19:58 smithi089 bash[75068]: cluster 1685823598.7771053 osd.0 (osd.0) 9 : cluster 0 12.13 scrub ok 2023-06-03T20:19:59.128 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:19:59.135 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:19:58 smithi164 bash[59462]: audit 1685823598.109323 mon.a (mon.0) 1261 : audit 0 from='client.? 172.21.15.89:0/3122406720' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-03T20:19:59.135 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:19:58 smithi164 bash[59462]: cluster 1685823598.2639604 osd.0 (osd.0) 8 : cluster 0 12.13 scrub starts 2023-06-03T20:19:59.136 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:19:58 smithi164 bash[59462]: cluster 1685823598.7771053 osd.0 (osd.0) 9 : cluster 0 12.13 scrub ok 2023-06-03T20:19:59.155 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:19:59.158 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:19:59.158 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:19:59.259 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:19:59.294 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:19:59.296 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:19:59.297 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:19:59.337 INFO:teuthology.orchestra.run.smithi089.stdout: 2023-06-03T20:19:59.338 INFO:teuthology.orchestra.run.smithi089.stderr:dumped all 2023-06-03T20:19:59.398 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:19:59.402 INFO:teuthology.orchestra.run.smithi164.stderr:Inferring config /var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/mon.b/config 2023-06-03T20:19:59.426 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:19:59.429 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:19:59.430 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:19:59.531 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:19:59.560 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:19:59.565 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:19:59.565 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:19:59.666 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:19:59.694 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:19:59.696 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:19:59.697 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:19:59.797 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:19:59.832 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:19:59.833 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:19:59.834 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:19:59.934 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:19:59.966 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:19:59.966 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:19:59 smithi089 bash[75068]: cluster 1685823598.9195955 mgr.y (mgr.34104) 1320 : cluster 0 pgmap v1320: 99 pgs: 20 active+clean+remapped, 1 active+clean+scrubbing+deep, 19 active+undersized+degraded, 11 active+undersized, 48 active+clean; 3.0 GiB data, 11 GiB used, 525 GiB / 536 GiB avail; 24774/147291 objects degraded (16.820%); 6/147291 objects misplaced (0.004%) 2023-06-03T20:19:59.967 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:19:59 smithi089 bash[75068]: audit 1685823599.027699 mon.c (mon.2) 572 : audit 1 from='client.? 172.21.15.89:0/255378318' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-06-03T20:19:59.967 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:19:59 smithi089 bash[75068]: audit 1685823599.028002 mon.a (mon.0) 1262 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-06-03T20:19:59.969 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:19:59.970 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:19:59.996 INFO:teuthology.orchestra.run.smithi089.stderr:noscrub is set 2023-06-03T20:20:00.071 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:20:00.102 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:20:00.105 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:20:00.105 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:20:00.206 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:20:00.216 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:19:59 smithi089 bash[78452]: cluster 1685823598.9195955 mgr.y (mgr.34104) 1320 : cluster 0 pgmap v1320: 99 pgs: 20 active+clean+remapped, 1 active+clean+scrubbing+deep, 19 active+undersized+degraded, 11 active+undersized, 48 active+clean; 3.0 GiB data, 11 GiB used, 525 GiB / 536 GiB avail; 24774/147291 objects degraded (16.820%); 6/147291 objects misplaced (0.004%) 2023-06-03T20:20:00.216 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:19:59 smithi089 bash[78452]: audit 1685823599.027699 mon.c (mon.2) 572 : audit 1 from='client.? 172.21.15.89:0/255378318' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-06-03T20:20:00.216 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:19:59 smithi089 bash[78452]: audit 1685823599.028002 mon.a (mon.0) 1262 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-06-03T20:20:00.238 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:20:00.241 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:20:00.242 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:20:00.334 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:19:59 smithi164 bash[59462]: cluster 1685823598.9195955 mgr.y (mgr.34104) 1320 : cluster 0 pgmap v1320: 99 pgs: 20 active+clean+remapped, 1 active+clean+scrubbing+deep, 19 active+undersized+degraded, 11 active+undersized, 48 active+clean; 3.0 GiB data, 11 GiB used, 525 GiB / 536 GiB avail; 24774/147291 objects degraded (16.820%); 6/147291 objects misplaced (0.004%) 2023-06-03T20:20:00.334 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:19:59 smithi164 bash[59462]: audit 1685823599.027699 mon.c (mon.2) 572 : audit 1 from='client.? 172.21.15.89:0/255378318' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-06-03T20:20:00.334 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:19:59 smithi164 bash[59462]: audit 1685823599.028002 mon.a (mon.0) 1262 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-06-03T20:20:00.343 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:20:00.374 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:20:00.378 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:20:00.379 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:20:00.480 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:20:00.513 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:20:00.516 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:20:00.517 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:20:00.618 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:20:00.650 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:20:00.653 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:20:00.653 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:20:00.754 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:20:00.784 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:20:00.786 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:20:00.787 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:20:00.889 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:20:00.928 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:20:00.931 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:20:00.932 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:20:01.017 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:20:00 smithi164 bash[59462]: audit 1685823599.3113828 mgr.y (mgr.34104) 1321 : audit 0 from='client.36492 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-06-03T20:20:01.017 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:20:00 smithi164 bash[59462]: cluster 1685823599.9605312 mon.a (mon.0) 1263 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-06-03T20:20:01.017 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:20:00 smithi164 bash[59462]: cluster 1685823599.96752 osd.6 (osd.6) 175 : cluster 3 failed to encode map e232 with expected crc 2023-06-03T20:20:01.017 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:20:00 smithi164 bash[59462]: audit 1685823599.9722853 mon.a (mon.0) 1264 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-06-03T20:20:01.017 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:20:00 smithi164 bash[59462]: cluster 1685823599.9724393 mon.a (mon.0) 1265 : cluster 0 osdmap e232: 8 total, 7 up, 6 in 2023-06-03T20:20:01.018 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:20:00 smithi164 bash[59462]: cluster 1685823599.9730096 osd.7 (osd.7) 157 : cluster 3 failed to encode map e232 with expected crc 2023-06-03T20:20:01.018 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:20:00 smithi164 bash[59462]: cluster 1685823599.9862645 osd.5 (osd.5) 168 : cluster 3 failed to encode map e232 with expected crc 2023-06-03T20:20:01.018 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:20:00 smithi164 bash[59462]: cluster 1685823600.0001564 mon.a (mon.0) 1266 : cluster 3 Health detail: HEALTH_WARN noscrub flag(s) set; 1 osds down; Degraded data redundancy: 24774/147291 objects degraded (16.820%), 19 pgs degraded, 20 pgs undersized 2023-06-03T20:20:01.018 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:20:00 smithi164 bash[59462]: cluster 1685823600.0002024 mon.a (mon.0) 1267 : cluster 3 [WRN] OSDMAP_FLAGS: noscrub flag(s) set 2023-06-03T20:20:01.018 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:20:00 smithi164 bash[59462]: cluster 1685823600.0002258 mon.a (mon.0) 1268 : cluster 3 [WRN] OSD_DOWN: 1 osds down 2023-06-03T20:20:01.018 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:20:00 smithi164 bash[59462]: cluster 1685823600.0002944 mon.a (mon.0) 1269 : cluster 3 osd.4 (root=default,host=smithi164) is down 2023-06-03T20:20:01.018 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:20:00 smithi164 bash[59462]: cluster 1685823600.000308 mon.a (mon.0) 1270 : cluster 3 [WRN] PG_DEGRADED: Degraded data redundancy: 24774/147291 objects degraded (16.820%), 19 pgs degraded, 20 pgs undersized 2023-06-03T20:20:01.019 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:20:00 smithi164 bash[59462]: cluster 1685823600.0003233 mon.a (mon.0) 1271 : cluster 3 pg 2.5 is stuck undersized for 8m, current state active+undersized, last acting [7,0] 2023-06-03T20:20:01.019 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:20:00 smithi164 bash[59462]: cluster 1685823600.0003352 mon.a (mon.0) 1272 : cluster 3 pg 3.1 is stuck undersized for 8m, current state active+undersized, last acting [0,3] 2023-06-03T20:20:01.019 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:20:00 smithi164 bash[59462]: cluster 1685823600.000343 mon.a (mon.0) 1273 : cluster 3 pg 3.6 is stuck undersized for 8m, current state active+undersized+degraded, last acting [0,7] 2023-06-03T20:20:01.019 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:20:00 smithi164 bash[59462]: cluster 1685823600.000362 mon.a (mon.0) 1274 : cluster 3 pg 3.8 is stuck undersized for 8m, current state active+undersized, last acting [3,7] 2023-06-03T20:20:01.019 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:20:00 smithi164 bash[59462]: cluster 1685823600.0003767 mon.a (mon.0) 1275 : cluster 3 pg 3.9 is stuck undersized for 8m, current state active+undersized, last acting [2,7] 2023-06-03T20:20:01.019 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:20:00 smithi164 bash[59462]: cluster 1685823600.0003915 mon.a (mon.0) 1276 : cluster 3 pg 3.b is stuck undersized for 8m, current state active+undersized+degraded, last acting [3,0] 2023-06-03T20:20:01.019 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:20:00 smithi164 bash[59462]: cluster 1685823600.0004063 mon.a (mon.0) 1277 : cluster 3 pg 3.e is stuck undersized for 8m, current state active+undersized+degraded, last acting [7,0] 2023-06-03T20:20:01.019 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:20:00 smithi164 bash[59462]: cluster 1685823600.000425 mon.a (mon.0) 1278 : cluster 3 pg 3.f is stuck undersized for 8m, current state active+undersized+degraded, last acting [7,0] 2023-06-03T20:20:01.020 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:20:00 smithi164 bash[59462]: cluster 1685823600.0004392 mon.a (mon.0) 1279 : cluster 3 pg 3.13 is stuck undersized for 8m, current state active+undersized+degraded, last acting [7,2] 2023-06-03T20:20:01.020 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:20:00 smithi164 bash[59462]: cluster 1685823600.0004473 mon.a (mon.0) 1280 : cluster 3 pg 3.15 is stuck undersized for 8m, current state active+undersized+degraded, last acting [7,3] 2023-06-03T20:20:01.020 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:20:00 smithi164 bash[59462]: cluster 1685823600.0004573 mon.a (mon.0) 1281 : cluster 3 pg 3.1a is stuck undersized for 8m, current state active+undersized, last acting [2,0] 2023-06-03T20:20:01.020 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:20:00 smithi164 bash[59462]: cluster 1685823600.000468 mon.a (mon.0) 1282 : cluster 3 pg 3.1b is stuck undersized for 8m, current state active+undersized, last acting [0,7] 2023-06-03T20:20:01.020 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:20:00 smithi164 bash[59462]: cluster 1685823600.0004814 mon.a (mon.0) 1283 : cluster 3 pg 4.2 is stuck undersized for 2m, current state active+undersized, last acting [0,5] 2023-06-03T20:20:01.020 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:20:00 smithi164 bash[59462]: cluster 1685823600.0004983 mon.a (mon.0) 1284 : cluster 3 pg 4.b is stuck undersized for 8m, current state active+undersized, last acting [0,3] 2023-06-03T20:20:01.021 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:20:00 smithi164 bash[59462]: cluster 1685823600.0005171 mon.a (mon.0) 1285 : cluster 3 pg 4.f is stuck undersized for 8m, current state active+undersized, last acting [3,7] 2023-06-03T20:20:01.022 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:20:00 smithi164 bash[59462]: cluster 1685823600.000536 mon.a (mon.0) 1286 : cluster 3 pg 4.1a is stuck undersized for 8m, current state active+undersized, last acting [3,0] 2023-06-03T20:20:01.022 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:20:00 smithi164 bash[59462]: cluster 1685823600.0005538 mon.a (mon.0) 1287 : cluster 3 pg 4.1b is stuck undersized for 8m, current state active+undersized, last acting [3,2] 2023-06-03T20:20:01.022 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:20:00 smithi164 bash[59462]: cluster 1685823600.0005724 mon.a (mon.0) 1288 : cluster 3 pg 12.0 is active+undersized+degraded, acting [2,7] 2023-06-03T20:20:01.022 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:20:00 smithi164 bash[59462]: cluster 1685823600.000592 mon.a (mon.0) 1289 : cluster 3 pg 12.2 is active+undersized+degraded, acting [3,7] 2023-06-03T20:20:01.023 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:20:01 smithi164 bash[59462]: cluster 1685823600.0006115 mon.a (mon.0) 1290 : cluster 3 pg 12.4 is active+undersized+degraded, acting [3,7] 2023-06-03T20:20:01.023 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:20:01 smithi164 bash[59462]: cluster 1685823600.0006485 mon.a (mon.0) 1291 : cluster 3 pg 12.7 is active+undersized+degraded, acting [7,2] 2023-06-03T20:20:01.023 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:20:01 smithi164 bash[59462]: cluster 1685823600.0006692 mon.a (mon.0) 1292 : cluster 3 pg 12.9 is active+undersized+degraded, acting [2,3] 2023-06-03T20:20:01.024 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:20:01 smithi164 bash[59462]: cluster 1685823600.0006874 mon.a (mon.0) 1293 : cluster 3 pg 12.b is stuck undersized for 3m, current state active+undersized+degraded, last acting [3,2] 2023-06-03T20:20:01.024 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:20:01 smithi164 bash[59462]: cluster 1685823600.0007262 mon.a (mon.0) 1294 : cluster 3 pg 12.e is stuck undersized for 3m, current state active+undersized+degraded, last acting [0,7] 2023-06-03T20:20:01.024 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:20:01 smithi164 bash[59462]: cluster 1685823600.0007625 mon.a (mon.0) 1295 : cluster 3 pg 12.f is stuck undersized for 2m, current state active+undersized+degraded, last acting [2,5] 2023-06-03T20:20:01.024 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:20:01 smithi164 bash[59462]: cluster 1685823600.000781 mon.a (mon.0) 1296 : cluster 3 pg 12.10 is active+undersized+degraded, acting [2,7] 2023-06-03T20:20:01.024 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:20:01 smithi164 bash[59462]: cluster 1685823600.0007994 mon.a (mon.0) 1297 : cluster 3 pg 12.12 is active+undersized+degraded, acting [3,7] 2023-06-03T20:20:01.025 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:20:01 smithi164 bash[59462]: cluster 1685823600.0008154 mon.a (mon.0) 1298 : cluster 3 pg 12.14 is active+undersized+degraded, acting [3,7] 2023-06-03T20:20:01.025 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:20:01 smithi164 bash[59462]: cluster 1685823600.0008314 mon.a (mon.0) 1299 : cluster 3 pg 12.17 is active+undersized+degraded, acting [7,2] 2023-06-03T20:20:01.025 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:20:01 smithi164 bash[59462]: cluster 1685823600.000847 mon.a (mon.0) 1300 : cluster 3 pg 12.19 is active+undersized+degraded, acting [2,3] 2023-06-03T20:20:01.033 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:20:01.064 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:20:01.067 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:20:01.068 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:20:01.169 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:20:01.177 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:20:00 smithi089 bash[75068]: audit 1685823599.3113828 mgr.y (mgr.34104) 1321 : audit 0 from='client.36492 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-06-03T20:20:01.177 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:20:00 smithi089 bash[75068]: cluster 1685823599.9605312 mon.a (mon.0) 1263 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-06-03T20:20:01.177 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:20:00 smithi089 bash[75068]: cluster 1685823599.96752 osd.6 (osd.6) 175 : cluster 3 failed to encode map e232 with expected crc 2023-06-03T20:20:01.177 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:20:00 smithi089 bash[75068]: audit 1685823599.9722853 mon.a (mon.0) 1264 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-06-03T20:20:01.178 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:20:00 smithi089 bash[75068]: cluster 1685823599.9724393 mon.a (mon.0) 1265 : cluster 0 osdmap e232: 8 total, 7 up, 6 in 2023-06-03T20:20:01.178 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:20:00 smithi089 bash[75068]: cluster 1685823599.9730096 osd.7 (osd.7) 157 : cluster 3 failed to encode map e232 with expected crc 2023-06-03T20:20:01.178 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:20:00 smithi089 bash[75068]: cluster 1685823599.9862645 osd.5 (osd.5) 168 : cluster 3 failed to encode map e232 with expected crc 2023-06-03T20:20:01.178 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:20:00 smithi089 bash[75068]: cluster 1685823600.0001564 mon.a (mon.0) 1266 : cluster 3 Health detail: HEALTH_WARN noscrub flag(s) set; 1 osds down; Degraded data redundancy: 24774/147291 objects degraded (16.820%), 19 pgs degraded, 20 pgs undersized 2023-06-03T20:20:01.178 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:20:00 smithi089 bash[75068]: cluster 1685823600.0002024 mon.a (mon.0) 1267 : cluster 3 [WRN] OSDMAP_FLAGS: noscrub flag(s) set 2023-06-03T20:20:01.178 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:20:00 smithi089 bash[75068]: cluster 1685823600.0002258 mon.a (mon.0) 1268 : cluster 3 [WRN] OSD_DOWN: 1 osds down 2023-06-03T20:20:01.178 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:20:00 smithi089 bash[75068]: cluster 1685823600.0002944 mon.a (mon.0) 1269 : cluster 3 osd.4 (root=default,host=smithi164) is down 2023-06-03T20:20:01.179 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:20:00 smithi089 bash[75068]: cluster 1685823600.000308 mon.a (mon.0) 1270 : cluster 3 [WRN] PG_DEGRADED: Degraded data redundancy: 24774/147291 objects degraded (16.820%), 19 pgs degraded, 20 pgs undersized 2023-06-03T20:20:01.179 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:20:00 smithi089 bash[75068]: cluster 1685823600.0003233 mon.a (mon.0) 1271 : cluster 3 pg 2.5 is stuck undersized for 8m, current state active+undersized, last acting [7,0] 2023-06-03T20:20:01.179 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:20:00 smithi089 bash[75068]: cluster 1685823600.0003352 mon.a (mon.0) 1272 : cluster 3 pg 3.1 is stuck undersized for 8m, current state active+undersized, last acting [0,3] 2023-06-03T20:20:01.179 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:20:00 smithi089 bash[75068]: cluster 1685823600.000343 mon.a (mon.0) 1273 : cluster 3 pg 3.6 is stuck undersized for 8m, current state active+undersized+degraded, last acting [0,7] 2023-06-03T20:20:01.179 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:20:00 smithi089 bash[75068]: cluster 1685823600.000362 mon.a (mon.0) 1274 : cluster 3 pg 3.8 is stuck undersized for 8m, current state active+undersized, last acting [3,7] 2023-06-03T20:20:01.179 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:20:00 smithi089 bash[75068]: cluster 1685823600.0003767 mon.a (mon.0) 1275 : cluster 3 pg 3.9 is stuck undersized for 8m, current state active+undersized, last acting [2,7] 2023-06-03T20:20:01.179 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:20:00 smithi089 bash[75068]: cluster 1685823600.0003915 mon.a (mon.0) 1276 : cluster 3 pg 3.b is stuck undersized for 8m, current state active+undersized+degraded, last acting [3,0] 2023-06-03T20:20:01.180 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:20:00 smithi089 bash[75068]: cluster 1685823600.0004063 mon.a (mon.0) 1277 : cluster 3 pg 3.e is stuck undersized for 8m, current state active+undersized+degraded, last acting [7,0] 2023-06-03T20:20:01.180 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:20:00 smithi089 bash[75068]: cluster 1685823600.000425 mon.a (mon.0) 1278 : cluster 3 pg 3.f is stuck undersized for 8m, current state active+undersized+degraded, last acting [7,0] 2023-06-03T20:20:01.180 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:20:00 smithi089 bash[75068]: cluster 1685823600.0004392 mon.a (mon.0) 1279 : cluster 3 pg 3.13 is stuck undersized for 8m, current state active+undersized+degraded, last acting [7,2] 2023-06-03T20:20:01.180 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:20:00 smithi089 bash[75068]: cluster 1685823600.0004473 mon.a (mon.0) 1280 : cluster 3 pg 3.15 is stuck undersized for 8m, current state active+undersized+degraded, last acting [7,3] 2023-06-03T20:20:01.180 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:20:00 smithi089 bash[75068]: cluster 1685823600.0004573 mon.a (mon.0) 1281 : cluster 3 pg 3.1a is stuck undersized for 8m, current state active+undersized, last acting [2,0] 2023-06-03T20:20:01.180 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:20:00 smithi089 bash[75068]: cluster 1685823600.000468 mon.a (mon.0) 1282 : cluster 3 pg 3.1b is stuck undersized for 8m, current state active+undersized, last acting [0,7] 2023-06-03T20:20:01.180 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:20:00 smithi089 bash[75068]: cluster 1685823600.0004814 mon.a (mon.0) 1283 : cluster 3 pg 4.2 is stuck undersized for 2m, current state active+undersized, last acting [0,5] 2023-06-03T20:20:01.180 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:20:00 smithi089 bash[75068]: cluster 1685823600.0004983 mon.a (mon.0) 1284 : cluster 3 pg 4.b is stuck undersized for 8m, current state active+undersized, last acting [0,3] 2023-06-03T20:20:01.181 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:20:00 smithi089 bash[75068]: cluster 1685823600.0005171 mon.a (mon.0) 1285 : cluster 3 pg 4.f is stuck undersized for 8m, current state active+undersized, last acting [3,7] 2023-06-03T20:20:01.181 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:20:00 smithi089 bash[75068]: cluster 1685823600.000536 mon.a (mon.0) 1286 : cluster 3 pg 4.1a is stuck undersized for 8m, current state active+undersized, last acting [3,0] 2023-06-03T20:20:01.181 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:20:00 smithi089 bash[75068]: cluster 1685823600.0005538 mon.a (mon.0) 1287 : cluster 3 pg 4.1b is stuck undersized for 8m, current state active+undersized, last acting [3,2] 2023-06-03T20:20:01.181 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:20:00 smithi089 bash[75068]: cluster 1685823600.0005724 mon.a (mon.0) 1288 : cluster 3 pg 12.0 is active+undersized+degraded, acting [2,7] 2023-06-03T20:20:01.181 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:20:00 smithi089 bash[75068]: cluster 1685823600.000592 mon.a (mon.0) 1289 : cluster 3 pg 12.2 is active+undersized+degraded, acting [3,7] 2023-06-03T20:20:01.181 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:20:00 smithi089 bash[75068]: cluster 1685823600.0006115 mon.a (mon.0) 1290 : cluster 3 pg 12.4 is active+undersized+degraded, acting [3,7] 2023-06-03T20:20:01.181 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:20:00 smithi089 bash[75068]: cluster 1685823600.0006485 mon.a (mon.0) 1291 : cluster 3 pg 12.7 is active+undersized+degraded, acting [7,2] 2023-06-03T20:20:01.182 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:20:00 smithi089 bash[75068]: cluster 1685823600.0006692 mon.a (mon.0) 1292 : cluster 3 pg 12.9 is active+undersized+degraded, acting [2,3] 2023-06-03T20:20:01.182 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:20:00 smithi089 bash[75068]: cluster 1685823600.0006874 mon.a (mon.0) 1293 : cluster 3 pg 12.b is stuck undersized for 3m, current state active+undersized+degraded, last acting [3,2] 2023-06-03T20:20:01.182 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:20:00 smithi089 bash[75068]: cluster 1685823600.0007262 mon.a (mon.0) 1294 : cluster 3 pg 12.e is stuck undersized for 3m, current state active+undersized+degraded, last acting [0,7] 2023-06-03T20:20:01.182 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:20:00 smithi089 bash[75068]: cluster 1685823600.0007625 mon.a (mon.0) 1295 : cluster 3 pg 12.f is stuck undersized for 2m, current state active+undersized+degraded, last acting [2,5] 2023-06-03T20:20:01.182 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:20:00 smithi089 bash[75068]: cluster 1685823600.000781 mon.a (mon.0) 1296 : cluster 3 pg 12.10 is active+undersized+degraded, acting [2,7] 2023-06-03T20:20:01.182 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:20:00 smithi089 bash[75068]: cluster 1685823600.0007994 mon.a (mon.0) 1297 : cluster 3 pg 12.12 is active+undersized+degraded, acting [3,7] 2023-06-03T20:20:01.182 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:20:00 smithi089 bash[75068]: cluster 1685823600.0008154 mon.a (mon.0) 1298 : cluster 3 pg 12.14 is active+undersized+degraded, acting [3,7] 2023-06-03T20:20:01.183 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:20:00 smithi089 bash[75068]: cluster 1685823600.0008314 mon.a (mon.0) 1299 : cluster 3 pg 12.17 is active+undersized+degraded, acting [7,2] 2023-06-03T20:20:01.183 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:20:00 smithi089 bash[75068]: cluster 1685823600.000847 mon.a (mon.0) 1300 : cluster 3 pg 12.19 is active+undersized+degraded, acting [2,3] 2023-06-03T20:20:01.183 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:20:00 smithi089 bash[78452]: audit 1685823599.3113828 mgr.y (mgr.34104) 1321 : audit 0 from='client.36492 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-06-03T20:20:01.184 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:20:00 smithi089 bash[78452]: cluster 1685823599.9605312 mon.a (mon.0) 1263 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-06-03T20:20:01.184 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:20:00 smithi089 bash[78452]: cluster 1685823599.96752 osd.6 (osd.6) 175 : cluster 3 failed to encode map e232 with expected crc 2023-06-03T20:20:01.184 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:20:00 smithi089 bash[78452]: audit 1685823599.9722853 mon.a (mon.0) 1264 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-06-03T20:20:01.184 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:20:00 smithi089 bash[78452]: cluster 1685823599.9724393 mon.a (mon.0) 1265 : cluster 0 osdmap e232: 8 total, 7 up, 6 in 2023-06-03T20:20:01.184 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:20:00 smithi089 bash[78452]: cluster 1685823599.9730096 osd.7 (osd.7) 157 : cluster 3 failed to encode map e232 with expected crc 2023-06-03T20:20:01.184 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:20:00 smithi089 bash[78452]: cluster 1685823599.9862645 osd.5 (osd.5) 168 : cluster 3 failed to encode map e232 with expected crc 2023-06-03T20:20:01.184 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:20:00 smithi089 bash[78452]: cluster 1685823600.0001564 mon.a (mon.0) 1266 : cluster 3 Health detail: HEALTH_WARN noscrub flag(s) set; 1 osds down; Degraded data redundancy: 24774/147291 objects degraded (16.820%), 19 pgs degraded, 20 pgs undersized 2023-06-03T20:20:01.185 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:20:00 smithi089 bash[78452]: cluster 1685823600.0002024 mon.a (mon.0) 1267 : cluster 3 [WRN] OSDMAP_FLAGS: noscrub flag(s) set 2023-06-03T20:20:01.185 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:20:00 smithi089 bash[78452]: cluster 1685823600.0002258 mon.a (mon.0) 1268 : cluster 3 [WRN] OSD_DOWN: 1 osds down 2023-06-03T20:20:01.185 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:20:00 smithi089 bash[78452]: cluster 1685823600.0002944 mon.a (mon.0) 1269 : cluster 3 osd.4 (root=default,host=smithi164) is down 2023-06-03T20:20:01.185 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:20:00 smithi089 bash[78452]: cluster 1685823600.000308 mon.a (mon.0) 1270 : cluster 3 [WRN] PG_DEGRADED: Degraded data redundancy: 24774/147291 objects degraded (16.820%), 19 pgs degraded, 20 pgs undersized 2023-06-03T20:20:01.185 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:20:00 smithi089 bash[78452]: cluster 1685823600.0003233 mon.a (mon.0) 1271 : cluster 3 pg 2.5 is stuck undersized for 8m, current state active+undersized, last acting [7,0] 2023-06-03T20:20:01.185 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:20:00 smithi089 bash[78452]: cluster 1685823600.0003352 mon.a (mon.0) 1272 : cluster 3 pg 3.1 is stuck undersized for 8m, current state active+undersized, last acting [0,3] 2023-06-03T20:20:01.185 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:20:00 smithi089 bash[78452]: cluster 1685823600.000343 mon.a (mon.0) 1273 : cluster 3 pg 3.6 is stuck undersized for 8m, current state active+undersized+degraded, last acting [0,7] 2023-06-03T20:20:01.185 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:20:00 smithi089 bash[78452]: cluster 1685823600.000362 mon.a (mon.0) 1274 : cluster 3 pg 3.8 is stuck undersized for 8m, current state active+undersized, last acting [3,7] 2023-06-03T20:20:01.186 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:20:00 smithi089 bash[78452]: cluster 1685823600.0003767 mon.a (mon.0) 1275 : cluster 3 pg 3.9 is stuck undersized for 8m, current state active+undersized, last acting [2,7] 2023-06-03T20:20:01.186 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:20:00 smithi089 bash[78452]: cluster 1685823600.0003915 mon.a (mon.0) 1276 : cluster 3 pg 3.b is stuck undersized for 8m, current state active+undersized+degraded, last acting [3,0] 2023-06-03T20:20:01.186 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:20:00 smithi089 bash[78452]: cluster 1685823600.0004063 mon.a (mon.0) 1277 : cluster 3 pg 3.e is stuck undersized for 8m, current state active+undersized+degraded, last acting [7,0] 2023-06-03T20:20:01.186 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:20:00 smithi089 bash[78452]: cluster 1685823600.000425 mon.a (mon.0) 1278 : cluster 3 pg 3.f is stuck undersized for 8m, current state active+undersized+degraded, last acting [7,0] 2023-06-03T20:20:01.186 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:20:00 smithi089 bash[78452]: cluster 1685823600.0004392 mon.a (mon.0) 1279 : cluster 3 pg 3.13 is stuck undersized for 8m, current state active+undersized+degraded, last acting [7,2] 2023-06-03T20:20:01.186 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:20:00 smithi089 bash[78452]: cluster 1685823600.0004473 mon.a (mon.0) 1280 : cluster 3 pg 3.15 is stuck undersized for 8m, current state active+undersized+degraded, last acting [7,3] 2023-06-03T20:20:01.186 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:20:00 smithi089 bash[78452]: cluster 1685823600.0004573 mon.a (mon.0) 1281 : cluster 3 pg 3.1a is stuck undersized for 8m, current state active+undersized, last acting [2,0] 2023-06-03T20:20:01.186 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:20:00 smithi089 bash[78452]: cluster 1685823600.000468 mon.a (mon.0) 1282 : cluster 3 pg 3.1b is stuck undersized for 8m, current state active+undersized, last acting [0,7] 2023-06-03T20:20:01.187 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:20:00 smithi089 bash[78452]: cluster 1685823600.0004814 mon.a (mon.0) 1283 : cluster 3 pg 4.2 is stuck undersized for 2m, current state active+undersized, last acting [0,5] 2023-06-03T20:20:01.187 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:20:00 smithi089 bash[78452]: cluster 1685823600.0004983 mon.a (mon.0) 1284 : cluster 3 pg 4.b is stuck undersized for 8m, current state active+undersized, last acting [0,3] 2023-06-03T20:20:01.187 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:20:00 smithi089 bash[78452]: cluster 1685823600.0005171 mon.a (mon.0) 1285 : cluster 3 pg 4.f is stuck undersized for 8m, current state active+undersized, last acting [3,7] 2023-06-03T20:20:01.187 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:20:00 smithi089 bash[78452]: cluster 1685823600.000536 mon.a (mon.0) 1286 : cluster 3 pg 4.1a is stuck undersized for 8m, current state active+undersized, last acting [3,0] 2023-06-03T20:20:01.187 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:20:00 smithi089 bash[78452]: cluster 1685823600.0005538 mon.a (mon.0) 1287 : cluster 3 pg 4.1b is stuck undersized for 8m, current state active+undersized, last acting [3,2] 2023-06-03T20:20:01.187 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:20:00 smithi089 bash[78452]: cluster 1685823600.0005724 mon.a (mon.0) 1288 : cluster 3 pg 12.0 is active+undersized+degraded, acting [2,7] 2023-06-03T20:20:01.187 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:20:00 smithi089 bash[78452]: cluster 1685823600.000592 mon.a (mon.0) 1289 : cluster 3 pg 12.2 is active+undersized+degraded, acting [3,7] 2023-06-03T20:20:01.188 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:20:00 smithi089 bash[78452]: cluster 1685823600.0006115 mon.a (mon.0) 1290 : cluster 3 pg 12.4 is active+undersized+degraded, acting [3,7] 2023-06-03T20:20:01.188 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:20:00 smithi089 bash[78452]: cluster 1685823600.0006485 mon.a (mon.0) 1291 : cluster 3 pg 12.7 is active+undersized+degraded, acting [7,2] 2023-06-03T20:20:01.188 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:20:00 smithi089 bash[78452]: cluster 1685823600.0006692 mon.a (mon.0) 1292 : cluster 3 pg 12.9 is active+undersized+degraded, acting [2,3] 2023-06-03T20:20:01.188 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:20:00 smithi089 bash[78452]: cluster 1685823600.0006874 mon.a (mon.0) 1293 : cluster 3 pg 12.b is stuck undersized for 3m, current state active+undersized+degraded, last acting [3,2] 2023-06-03T20:20:01.188 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:20:00 smithi089 bash[78452]: cluster 1685823600.0007262 mon.a (mon.0) 1294 : cluster 3 pg 12.e is stuck undersized for 3m, current state active+undersized+degraded, last acting [0,7] 2023-06-03T20:20:01.188 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:20:00 smithi089 bash[78452]: cluster 1685823600.0007625 mon.a (mon.0) 1295 : cluster 3 pg 12.f is stuck undersized for 2m, current state active+undersized+degraded, last acting [2,5] 2023-06-03T20:20:01.188 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:20:00 smithi089 bash[78452]: cluster 1685823600.000781 mon.a (mon.0) 1296 : cluster 3 pg 12.10 is active+undersized+degraded, acting [2,7] 2023-06-03T20:20:01.189 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:20:00 smithi089 bash[78452]: cluster 1685823600.0007994 mon.a (mon.0) 1297 : cluster 3 pg 12.12 is active+undersized+degraded, acting [3,7] 2023-06-03T20:20:01.189 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:20:00 smithi089 bash[78452]: cluster 1685823600.0008154 mon.a (mon.0) 1298 : cluster 3 pg 12.14 is active+undersized+degraded, acting [3,7] 2023-06-03T20:20:01.189 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:20:00 smithi089 bash[78452]: cluster 1685823600.0008314 mon.a (mon.0) 1299 : cluster 3 pg 12.17 is active+undersized+degraded, acting [7,2] 2023-06-03T20:20:01.189 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:20:00 smithi089 bash[78452]: cluster 1685823600.000847 mon.a (mon.0) 1300 : cluster 3 pg 12.19 is active+undersized+degraded, acting [2,3] 2023-06-03T20:20:01.197 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:20:01.202 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:20:01.202 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:20:01.303 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:20:01.337 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:20:01.342 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:20:01.343 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:20:01.444 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:20:01.475 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:20:01.481 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:20:01.481 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:20:01.582 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:20:01.614 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:20:01.618 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:20:01.619 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:20:01.720 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:20:01.749 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:20:01.751 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:20:01.752 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:20:01.853 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:20:01.880 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:20:01.881 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:20:01.882 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:20:01.983 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:20:02.016 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:20:02.020 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:20:02.021 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:20:02.122 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:20:02.144 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:20:02 smithi164 bash[59462]: cluster 1685823600.9213712 mgr.y (mgr.34104) 1322 : cluster 0 pgmap v1322: 99 pgs: 20 active+clean+remapped, 1 active+clean+scrubbing+deep, 19 active+undersized+degraded, 11 active+undersized, 48 active+clean; 2.8 GiB data, 11 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 847 op/s; 23222/139359 objects degraded (16.663%); 6/139359 objects misplaced (0.004%) 2023-06-03T20:20:02.155 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:20:02.158 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:20:02.159 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:20:02.260 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:20:02.279 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:20:01 smithi089 bash[75068]: cluster 1685823600.9213712 mgr.y (mgr.34104) 1322 : cluster 0 pgmap v1322: 99 pgs: 20 active+clean+remapped, 1 active+clean+scrubbing+deep, 19 active+undersized+degraded, 11 active+undersized, 48 active+clean; 2.8 GiB data, 11 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 847 op/s; 23222/139359 objects degraded (16.663%); 6/139359 objects misplaced (0.004%) 2023-06-03T20:20:02.279 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:20:01 smithi089 bash[78452]: cluster 1685823600.9213712 mgr.y (mgr.34104) 1322 : cluster 0 pgmap v1322: 99 pgs: 20 active+clean+remapped, 1 active+clean+scrubbing+deep, 19 active+undersized+degraded, 11 active+undersized, 48 active+clean; 2.8 GiB data, 11 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 847 op/s; 23222/139359 objects degraded (16.663%); 6/139359 objects misplaced (0.004%) 2023-06-03T20:20:02.297 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:20:02.299 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:20:02.300 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:20:02.401 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:20:02.436 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:20:02.438 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:20:02.439 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:20:02.540 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:20:02.585 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:20:02.591 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:20:02.592 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:20:02.693 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:20:02.731 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:20:02.736 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:20:02.738 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:20:02.839 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:20:02.867 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:20:02.870 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:20:02.871 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:20:02.972 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:20:02.996 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:20:02 smithi164 bash[59462]: cluster 1685823602.8613138 mon.a (mon.0) 1301 : cluster 3 Health check update: Degraded data redundancy: 23222/139359 objects degraded (16.663%), 19 pgs degraded, 20 pgs undersized (PG_DEGRADED) 2023-06-03T20:20:03.007 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:20:03.010 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:20:03.011 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:20:03.104 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-06-03T20:20:03.105 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-06-03T20:20:03.112 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:20:03.118 INFO:teuthology.orchestra.run.smithi089.stdout:{"pg_ready":true,"pg_map":{"version":1320,"stamp":"2023-06-03T20:19:58.919171+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":3216578470,"num_objects":49097,"num_object_clones":0,"num_object_copies":147291,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":24774,"num_objects_misplaced":6,"num_objects_unfound":0,"num_objects_dirty":49097,"num_whiteouts":0,"num_read":309,"num_read_kb":513,"num_write":139284,"num_write_kb":5694991,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":18,"num_bytes_recovered":595154,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":185143,"ondisk_log_size":185143,"up":247,"acting":267,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":288,"num_osds":6,"num_per_pool_osds":6,"num_per_pool_omap_osds":6,"kb":562470912,"kb_used":11670940,"kb_used_data":8631596,"kb_used_omap":26,"kb_used_meta":3039205,"kb_avail":550799972,"statfs":{"total":575970213888,"available":564019171328,"internally_reserved":0,"allocated":8838754304,"data_stored":8834240317,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":27292,"internal_metadata":3112146276},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,3,1,2,2,3,8],"upper_bound":128},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-598376448,"num_objects":-9131,"num_object_clones":0,"num_object_copies":-27393,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":-4629,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":-9131,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":-590,"num_write_kb":-427040,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":-1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.004570"},"pg_stats":[{"pgid":"12.15","version":"231'9255","reported_seq":9337,"reported_epoch":231,"state":"active+clean","last_fresh":"2023-06-03T20:19:55.542625+0000","last_change":"2023-06-03T20:19:47.190503+0000","last_active":"2023-06-03T20:19:55.542625+0000","last_peered":"2023-06-03T20:19:55.542625+0000","last_clean":"2023-06-03T20:19:55.542625+0000","last_became_active":"2023-06-03T20:19:28.136030+0000","last_became_peered":"2023-06-03T20:19:28.136030+0000","last_unstale":"2023-06-03T20:19:55.542625+0000","last_undegraded":"2023-06-03T20:19:55.542625+0000","last_fullsized":"2023-06-03T20:19:55.542625+0000","mapping_epoch":229,"log_start":"221'100","ondisk_log_start":"221'100","created":229,"last_epoch_clean":230,"parent":"0.0","parent_split_bits":5,"last_scrub":"231'9038","last_scrub_stamp":"2023-06-03T20:19:47.187925+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T20:16:46.173157+0000","last_clean_scrub_stamp":"2023-06-03T20:19:47.187925+0000","objects_scrubbed":1746,"log_size":9155,"log_dups_size":0,"ondisk_log_size":9155,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T07:53:56.634073+0000","scrub_duration":0.78494912400000005,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":99876864,"num_objects":1524,"num_object_clones":0,"num_object_copies":4572,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1524,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":222,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1a","version":"60'1","reported_seq":47,"reported_epoch":230,"state":"active+undersized","last_fresh":"2023-06-03T20:19:28.037322+0000","last_change":"2023-06-03T20:11:54.019728+0000","last_active":"2023-06-03T20:19:28.037322+0000","last_peered":"2023-06-03T20:19:28.037322+0000","last_clean":"2023-06-03T19:34:15.767991+0000","last_became_active":"2023-06-03T20:11:54.019728+0000","last_became_peered":"2023-06-03T20:11:54.019728+0000","last_unstale":"2023-06-03T20:19:28.037322+0000","last_undegraded":"2023-06-03T20:19:28.037322+0000","last_fullsized":"2023-06-03T20:11:54.009512+0000","mapping_epoch":195,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T19:58:26.738637+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":["2","0"],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":46,"reported_epoch":230,"state":"active+clean+remapped","last_fresh":"2023-06-03T20:19:28.037377+0000","last_change":"2023-06-03T20:12:31.911399+0000","last_active":"2023-06-03T20:19:28.037377+0000","last_peered":"2023-06-03T20:19:28.037377+0000","last_clean":"2023-06-03T20:19:28.037377+0000","last_became_active":"2023-06-03T20:12:31.911029+0000","last_became_peered":"2023-06-03T20:12:31.911029+0000","last_unstale":"2023-06-03T20:19:28.037377+0000","last_undegraded":"2023-06-03T20:19:28.037377+0000","last_fullsized":"2023-06-03T20:19:28.037377+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T05:04:34.770935+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5],"acting":[3,5,6],"avail_no_missing":["3","5","6"],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"12.16","version":"231'9328","reported_seq":9362,"reported_epoch":231,"state":"active+clean","last_fresh":"2023-06-03T20:19:56.679418+0000","last_change":"2023-06-03T20:19:28.145900+0000","last_active":"2023-06-03T20:19:56.679418+0000","last_peered":"2023-06-03T20:19:56.679418+0000","last_clean":"2023-06-03T20:19:56.679418+0000","last_became_active":"2023-06-03T20:19:28.132321+0000","last_became_peered":"2023-06-03T20:19:28.132321+0000","last_unstale":"2023-06-03T20:19:56.679418+0000","last_undegraded":"2023-06-03T20:19:56.679418+0000","last_fullsized":"2023-06-03T20:19:56.679418+0000","mapping_epoch":229,"log_start":"224'2500","ondisk_log_start":"224'2500","created":229,"last_epoch_clean":230,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T20:16:46.173157+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T20:16:46.173157+0000","last_clean_scrub_stamp":"2023-06-03T20:16:46.173157+0000","objects_scrubbed":0,"log_size":6828,"log_dups_size":0,"ondisk_log_size":6828,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":100106240,"num_objects":1528,"num_object_clones":0,"num_object_copies":4584,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1528,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5036,"num_write_kb":210016,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":204,"reported_epoch":231,"state":"active+clean+remapped","last_fresh":"2023-06-03T20:19:31.081116+0000","last_change":"2023-06-03T20:16:21.917001+0000","last_active":"2023-06-03T20:19:31.081116+0000","last_peered":"2023-06-03T20:19:31.081116+0000","last_clean":"2023-06-03T20:19:31.081116+0000","last_became_active":"2023-06-03T20:16:21.916538+0000","last_became_peered":"2023-06-03T20:16:21.916538+0000","last_unstale":"2023-06-03T20:19:31.081116+0000","last_undegraded":"2023-06-03T20:19:31.081116+0000","last_fullsized":"2023-06-03T20:19:31.081116+0000","mapping_epoch":213,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":214,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T20:44:25.048340+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7],"acting":[3,7,6],"avail_no_missing":["3","6","7"],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":197,"reported_epoch":230,"state":"active+clean","last_fresh":"2023-06-03T20:19:28.044868+0000","last_change":"2023-06-03T19:56:54.507653+0000","last_active":"2023-06-03T20:19:28.044868+0000","last_peered":"2023-06-03T20:19:28.044868+0000","last_clean":"2023-06-03T20:19:28.044868+0000","last_became_active":"2023-06-03T19:56:54.507350+0000","last_became_peered":"2023-06-03T19:56:54.507350+0000","last_unstale":"2023-06-03T20:19:28.044868+0000","last_undegraded":"2023-06-03T20:19:28.044868+0000","last_fullsized":"2023-06-03T20:19:28.044868+0000","mapping_epoch":119,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T22:33:20.065844+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"12.17","version":"231'9520","reported_seq":9545,"reported_epoch":231,"state":"active+undersized+degraded","last_fresh":"2023-06-03T20:19:58.426166+0000","last_change":"2023-06-03T20:19:28.136162+0000","last_active":"2023-06-03T20:19:58.426166+0000","last_peered":"2023-06-03T20:19:58.426166+0000","last_clean":"2023-06-03T20:16:46.173157+0000","last_became_active":"2023-06-03T20:19:28.136162+0000","last_became_peered":"2023-06-03T20:19:28.136162+0000","last_unstale":"2023-06-03T20:19:58.426166+0000","last_undegraded":"2023-06-03T20:19:28.097150+0000","last_fullsized":"2023-06-03T20:19:28.041873+0000","mapping_epoch":229,"log_start":"224'2400","ondisk_log_start":"224'2400","created":229,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T20:16:46.173157+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T20:16:46.173157+0000","last_clean_scrub_stamp":"2023-06-03T20:16:46.173157+0000","objects_scrubbed":0,"log_size":7120,"log_dups_size":0,"ondisk_log_size":7120,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":98369536,"num_objects":1501,"num_object_clones":0,"num_object_copies":4503,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1501,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1501,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5170,"num_write_kb":213472,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2],"acting":[7,2],"avail_no_missing":["7","2"],"object_location_counts":[{"shards":"2,7","objects":1501}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":228,"reported_epoch":230,"state":"active+clean","last_fresh":"2023-06-03T20:19:28.035531+0000","last_change":"2023-06-03T20:07:19.912004+0000","last_active":"2023-06-03T20:19:28.035531+0000","last_peered":"2023-06-03T20:19:28.035531+0000","last_clean":"2023-06-03T20:19:28.035531+0000","last_became_active":"2023-06-03T20:07:19.910895+0000","last_became_peered":"2023-06-03T20:07:19.910895+0000","last_unstale":"2023-06-03T20:19:28.035531+0000","last_undegraded":"2023-06-03T20:19:28.035531+0000","last_fullsized":"2023-06-03T20:19:28.035531+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T06:46:44.564002+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,2],"acting":[3,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":157,"reported_epoch":231,"state":"active+clean","last_fresh":"2023-06-03T20:19:31.089484+0000","last_change":"2023-06-03T20:17:22.834646+0000","last_active":"2023-06-03T20:19:31.089484+0000","last_peered":"2023-06-03T20:19:31.089484+0000","last_clean":"2023-06-03T20:19:31.089484+0000","last_became_active":"2023-06-03T20:17:22.834324+0000","last_became_peered":"2023-06-03T20:17:22.834324+0000","last_unstale":"2023-06-03T20:19:31.089484+0000","last_undegraded":"2023-06-03T20:19:31.089484+0000","last_fullsized":"2023-06-03T20:19:31.089484+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T02:18:49.246420+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"12.18","version":"231'11274","reported_seq":11485,"reported_epoch":231,"state":"active+clean","last_fresh":"2023-06-03T20:19:53.760146+0000","last_change":"2023-06-03T20:19:43.774956+0000","last_active":"2023-06-03T20:19:53.760146+0000","last_peered":"2023-06-03T20:19:53.760146+0000","last_clean":"2023-06-03T20:19:53.760146+0000","last_became_active":"2023-06-03T20:19:28.109118+0000","last_became_peered":"2023-06-03T20:19:28.109118+0000","last_unstale":"2023-06-03T20:19:53.760146+0000","last_undegraded":"2023-06-03T20:19:53.760146+0000","last_fullsized":"2023-06-03T20:19:53.760146+0000","mapping_epoch":229,"log_start":"225'4453","ondisk_log_start":"225'4453","created":229,"last_epoch_clean":230,"parent":"0.0","parent_split_bits":5,"last_scrub":"231'11015","last_scrub_stamp":"2023-06-03T20:19:43.774799+0000","last_deep_scrub":"231'11015","last_deep_scrub_stamp":"2023-06-03T20:19:43.774799+0000","last_clean_scrub_stamp":"2023-06-03T20:19:43.774799+0000","objects_scrubbed":1991,"log_size":6821,"log_dups_size":0,"ondisk_log_size":6821,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":13,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T07:53:19.645611+0000","scrub_duration":12.180670491000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":101515264,"num_objects":1549,"num_object_clones":0,"num_object_copies":4647,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1549,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":442,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":193,"reported_epoch":230,"state":"active+clean","last_fresh":"2023-06-03T20:19:28.041279+0000","last_change":"2023-06-03T19:56:54.509490+0000","last_active":"2023-06-03T20:19:28.041279+0000","last_peered":"2023-06-03T20:19:28.041279+0000","last_clean":"2023-06-03T20:19:28.041279+0000","last_became_active":"2023-06-03T19:56:54.509145+0000","last_became_peered":"2023-06-03T19:56:54.509145+0000","last_unstale":"2023-06-03T20:19:28.041279+0000","last_undegraded":"2023-06-03T20:19:28.041279+0000","last_fullsized":"2023-06-03T20:19:28.041279+0000","mapping_epoch":119,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T07:12:03.964110+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":228,"reported_epoch":230,"state":"active+clean+remapped","last_fresh":"2023-06-03T20:19:28.035905+0000","last_change":"2023-06-03T20:12:31.937956+0000","last_active":"2023-06-03T20:19:28.035905+0000","last_peered":"2023-06-03T20:19:28.035905+0000","last_clean":"2023-06-03T20:19:28.035905+0000","last_became_active":"2023-06-03T20:12:31.937551+0000","last_became_peered":"2023-06-03T20:12:31.937551+0000","last_unstale":"2023-06-03T20:19:28.035905+0000","last_undegraded":"2023-06-03T20:19:28.035905+0000","last_fullsized":"2023-06-03T20:19:28.035905+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T21:49:12.058781+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5],"acting":[3,5,6],"avail_no_missing":["3","5","6"],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"12.19","version":"231'9245","reported_seq":9273,"reported_epoch":231,"state":"active+undersized+degraded","last_fresh":"2023-06-03T20:19:55.603754+0000","last_change":"2023-06-03T20:19:28.117046+0000","last_active":"2023-06-03T20:19:55.603754+0000","last_peered":"2023-06-03T20:19:55.603754+0000","last_clean":"2023-06-03T20:16:46.173157+0000","last_became_active":"2023-06-03T20:19:28.117046+0000","last_became_peered":"2023-06-03T20:19:28.117046+0000","last_unstale":"2023-06-03T20:19:55.603754+0000","last_undegraded":"2023-06-03T20:19:28.100684+0000","last_fullsized":"2023-06-03T20:19:28.043512+0000","mapping_epoch":229,"log_start":"221'100","ondisk_log_start":"221'100","created":229,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T20:16:46.173157+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T20:16:46.173157+0000","last_clean_scrub_stamp":"2023-06-03T20:16:46.173157+0000","objects_scrubbed":0,"log_size":9145,"log_dups_size":0,"ondisk_log_size":9145,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":99942400,"num_objects":1525,"num_object_clones":0,"num_object_copies":4575,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1525,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1525,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5013,"num_write_kb":209216,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3],"acting":[2,3],"avail_no_missing":["2","3"],"object_location_counts":[{"shards":"2,3","objects":1525}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":154,"reported_epoch":230,"state":"active+clean","last_fresh":"2023-06-03T20:19:28.040678+0000","last_change":"2023-06-03T20:17:22.832811+0000","last_active":"2023-06-03T20:19:28.040678+0000","last_peered":"2023-06-03T20:19:28.040678+0000","last_clean":"2023-06-03T20:19:28.040678+0000","last_became_active":"2023-06-03T20:17:22.832474+0000","last_became_peered":"2023-06-03T20:17:22.832474+0000","last_unstale":"2023-06-03T20:19:28.040678+0000","last_undegraded":"2023-06-03T20:19:28.040678+0000","last_fullsized":"2023-06-03T20:19:28.040678+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,3],"acting":[7,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":193,"reported_epoch":230,"state":"active+clean","last_fresh":"2023-06-03T20:19:28.037506+0000","last_change":"2023-06-03T20:16:20.909781+0000","last_active":"2023-06-03T20:19:28.037506+0000","last_peered":"2023-06-03T20:19:28.037506+0000","last_clean":"2023-06-03T20:19:28.037506+0000","last_became_active":"2023-06-03T20:16:20.909496+0000","last_became_peered":"2023-06-03T20:16:20.909496+0000","last_unstale":"2023-06-03T20:19:28.037506+0000","last_undegraded":"2023-06-03T20:19:28.037506+0000","last_fullsized":"2023-06-03T20:19:28.037506+0000","mapping_epoch":212,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":213,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T20:32:26.886172+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,5],"acting":[3,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"60'3","reported_seq":219,"reported_epoch":230,"state":"active+undersized+degraded","last_fresh":"2023-06-03T20:19:28.040427+0000","last_change":"2023-06-03T20:11:54.025696+0000","last_active":"2023-06-03T20:19:28.040427+0000","last_peered":"2023-06-03T20:19:28.040427+0000","last_clean":"2023-06-03T20:11:43.973370+0000","last_became_active":"2023-06-03T20:11:54.025696+0000","last_became_peered":"2023-06-03T20:11:54.025696+0000","last_unstale":"2023-06-03T20:19:28.040427+0000","last_undegraded":"2023-06-03T20:11:54.012202+0000","last_fullsized":"2023-06-03T20:11:54.012054+0000","mapping_epoch":195,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":173,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3],"acting":[7,3],"avail_no_missing":["7","3"],"object_location_counts":[{"shards":"3,7","objects":2}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":90,"reported_epoch":231,"state":"active+clean","last_fresh":"2023-06-03T20:19:31.071537+0000","last_change":"2023-06-03T20:12:30.911538+0000","last_active":"2023-06-03T20:19:31.071537+0000","last_peered":"2023-06-03T20:19:31.071537+0000","last_clean":"2023-06-03T20:19:31.071537+0000","last_became_active":"2023-06-03T20:12:30.911082+0000","last_became_peered":"2023-06-03T20:12:30.911082+0000","last_unstale":"2023-06-03T20:19:31.071537+0000","last_undegraded":"2023-06-03T20:19:31.071537+0000","last_fullsized":"2023-06-03T20:19:31.071537+0000","mapping_epoch":198,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":199,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T05:22:01.454415+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,5],"acting":[2,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.14","version":"77'23","reported_seq":121,"reported_epoch":230,"state":"active+clean+remapped","last_fresh":"2023-06-03T20:19:28.042337+0000","last_change":"2023-06-03T20:16:22.089746+0000","last_active":"2023-06-03T20:19:28.042337+0000","last_peered":"2023-06-03T20:19:28.042337+0000","last_clean":"2023-06-03T20:19:28.042337+0000","last_became_active":"2023-06-03T20:16:21.916216+0000","last_became_peered":"2023-06-03T20:16:21.916216+0000","last_unstale":"2023-06-03T20:19:28.042337+0000","last_undegraded":"2023-06-03T20:19:28.042337+0000","last_fullsized":"2023-06-03T20:19:28.042337+0000","mapping_epoch":213,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":214,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":23,"log_dups_size":0,"ondisk_log_size":23,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4716,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":4,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":23,"num_write_kb":20,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4716,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2],"acting":[7,2,6],"avail_no_missing":["7","2","6"],"object_location_counts":[{"shards":"2,6,7","objects":4}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":157,"reported_epoch":231,"state":"active+clean+remapped","last_fresh":"2023-06-03T20:19:31.069997+0000","last_change":"2023-06-03T20:17:23.856188+0000","last_active":"2023-06-03T20:19:31.069997+0000","last_peered":"2023-06-03T20:19:31.069997+0000","last_clean":"2023-06-03T20:19:31.069997+0000","last_became_active":"2023-06-03T20:17:23.855783+0000","last_became_peered":"2023-06-03T20:17:23.855783+0000","last_unstale":"2023-06-03T20:19:31.069997+0000","last_undegraded":"2023-06-03T20:19:31.069997+0000","last_fullsized":"2023-06-03T20:19:31.069997+0000","mapping_epoch":223,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":224,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T20:50:46.483487+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5],"acting":[2,5,6],"avail_no_missing":["2","5","6"],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.13","version":"60'2","reported_seq":221,"reported_epoch":230,"state":"active+undersized+degraded","last_fresh":"2023-06-03T20:19:28.041169+0000","last_change":"2023-06-03T20:11:54.019051+0000","last_active":"2023-06-03T20:19:28.041169+0000","last_peered":"2023-06-03T20:19:28.041169+0000","last_clean":"2023-06-03T20:11:43.972488+0000","last_became_active":"2023-06-03T20:11:54.019051+0000","last_became_peered":"2023-06-03T20:11:54.019051+0000","last_unstale":"2023-06-03T20:19:28.041169+0000","last_undegraded":"2023-06-03T20:11:54.012366+0000","last_fullsized":"2023-06-03T20:11:54.012217+0000","mapping_epoch":195,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2],"acting":[7,2],"avail_no_missing":["7","2"],"object_location_counts":[{"shards":"2,7","objects":2}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":212,"reported_epoch":230,"state":"active+clean","last_fresh":"2023-06-03T20:19:28.037596+0000","last_change":"2023-06-03T20:07:19.907827+0000","last_active":"2023-06-03T20:19:28.037596+0000","last_peered":"2023-06-03T20:19:28.037596+0000","last_clean":"2023-06-03T20:19:28.037596+0000","last_became_active":"2023-06-03T20:07:19.907296+0000","last_became_peered":"2023-06-03T20:07:19.907296+0000","last_unstale":"2023-06-03T20:19:28.037596+0000","last_undegraded":"2023-06-03T20:19:28.037596+0000","last_fullsized":"2023-06-03T20:19:28.037596+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T00:34:13.230147+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":194,"reported_epoch":231,"state":"active+clean","last_fresh":"2023-06-03T20:19:31.085069+0000","last_change":"2023-06-03T19:56:54.507479+0000","last_active":"2023-06-03T20:19:31.085069+0000","last_peered":"2023-06-03T20:19:31.085069+0000","last_clean":"2023-06-03T20:19:31.085069+0000","last_became_active":"2023-06-03T19:56:54.507149+0000","last_became_peered":"2023-06-03T19:56:54.507149+0000","last_unstale":"2023-06-03T20:19:31.085069+0000","last_undegraded":"2023-06-03T20:19:31.085069+0000","last_fullsized":"2023-06-03T20:19:31.085069+0000","mapping_epoch":119,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T20:22:36.676317+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":97,"reported_epoch":230,"state":"active+clean","last_fresh":"2023-06-03T20:19:28.040795+0000","last_change":"2023-06-03T20:17:22.832844+0000","last_active":"2023-06-03T20:19:28.040795+0000","last_peered":"2023-06-03T20:19:28.040795+0000","last_clean":"2023-06-03T20:19:28.040795+0000","last_became_active":"2023-06-03T20:17:22.832473+0000","last_became_peered":"2023-06-03T20:17:22.832473+0000","last_unstale":"2023-06-03T20:19:28.040795+0000","last_undegraded":"2023-06-03T20:19:28.040795+0000","last_fullsized":"2023-06-03T20:19:28.040795+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,3],"acting":[7,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":211,"reported_epoch":230,"state":"active+clean+remapped","last_fresh":"2023-06-03T20:19:28.042289+0000","last_change":"2023-06-03T20:16:21.916599+0000","last_active":"2023-06-03T20:19:28.042289+0000","last_peered":"2023-06-03T20:19:28.042289+0000","last_clean":"2023-06-03T20:19:28.042289+0000","last_became_active":"2023-06-03T20:16:21.916145+0000","last_became_peered":"2023-06-03T20:16:21.916145+0000","last_unstale":"2023-06-03T20:19:28.042289+0000","last_undegraded":"2023-06-03T20:19:28.042289+0000","last_fullsized":"2023-06-03T20:19:28.042289+0000","mapping_epoch":213,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":214,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0],"acting":[7,0,6],"avail_no_missing":["7","0","6"],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":194,"reported_epoch":230,"state":"active+clean","last_fresh":"2023-06-03T20:19:28.045036+0000","last_change":"2023-06-03T19:56:54.510289+0000","last_active":"2023-06-03T20:19:28.045036+0000","last_peered":"2023-06-03T20:19:28.045036+0000","last_clean":"2023-06-03T20:19:28.045036+0000","last_became_active":"2023-06-03T19:56:54.509905+0000","last_became_peered":"2023-06-03T19:56:54.509905+0000","last_unstale":"2023-06-03T20:19:28.045036+0000","last_undegraded":"2023-06-03T20:19:28.045036+0000","last_fullsized":"2023-06-03T20:19:28.045036+0000","mapping_epoch":119,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T02:54:26.792401+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"60'8","reported_seq":52,"reported_epoch":230,"state":"active+clean","last_fresh":"2023-06-03T20:19:28.034337+0000","last_change":"2023-06-03T20:16:20.914195+0000","last_active":"2023-06-03T20:19:28.034337+0000","last_peered":"2023-06-03T20:19:28.034337+0000","last_clean":"2023-06-03T20:19:28.034337+0000","last_became_active":"2023-06-03T20:16:20.906350+0000","last_became_peered":"2023-06-03T20:16:20.906350+0000","last_unstale":"2023-06-03T20:19:28.034337+0000","last_undegraded":"2023-06-03T20:19:28.034337+0000","last_fullsized":"2023-06-03T20:19:28.034337+0000","mapping_epoch":212,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":213,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T20:33:32.314202+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":212,"reported_epoch":230,"state":"active+clean","last_fresh":"2023-06-03T20:19:28.040178+0000","last_change":"2023-06-03T20:07:19.907151+0000","last_active":"2023-06-03T20:19:28.040178+0000","last_peered":"2023-06-03T20:19:28.040178+0000","last_clean":"2023-06-03T20:19:28.040178+0000","last_became_active":"2023-06-03T20:07:19.906479+0000","last_became_peered":"2023-06-03T20:07:19.906479+0000","last_unstale":"2023-06-03T20:19:28.040178+0000","last_undegraded":"2023-06-03T20:19:28.040178+0000","last_fullsized":"2023-06-03T20:19:28.040178+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T03:18:22.525692+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"12.0","version":"231'9504","reported_seq":9531,"reported_epoch":231,"state":"active+undersized+degraded","last_fresh":"2023-06-03T20:19:55.660997+0000","last_change":"2023-06-03T20:19:28.063103+0000","last_active":"2023-06-03T20:19:55.660997+0000","last_peered":"2023-06-03T20:19:55.660997+0000","last_clean":"2023-06-03T20:16:46.173157+0000","last_became_active":"2023-06-03T20:19:28.063103+0000","last_became_peered":"2023-06-03T20:19:28.063103+0000","last_unstale":"2023-06-03T20:19:55.660997+0000","last_undegraded":"2023-06-03T20:19:28.041063+0000","last_fullsized":"2023-06-03T20:19:28.035035+0000","mapping_epoch":217,"log_start":"221'300","ondisk_log_start":"221'300","created":217,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T20:16:46.173157+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T20:16:46.173157+0000","last_clean_scrub_stamp":"2023-06-03T20:16:46.173157+0000","objects_scrubbed":0,"log_size":9204,"log_dups_size":0,"ondisk_log_size":9204,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":97091584,"num_objects":1481,"num_object_clones":0,"num_object_copies":4443,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1481,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1481,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5164,"num_write_kb":212672,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7],"acting":[2,7],"avail_no_missing":["2","7"],"object_location_counts":[{"shards":"2,7","objects":1481}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.f","version":"60'1","reported_seq":220,"reported_epoch":230,"state":"active+undersized+degraded","last_fresh":"2023-06-03T20:19:28.042112+0000","last_change":"2023-06-03T20:11:54.016915+0000","last_active":"2023-06-03T20:19:28.042112+0000","last_peered":"2023-06-03T20:19:28.042112+0000","last_clean":"2023-06-03T20:11:43.972655+0000","last_became_active":"2023-06-03T20:11:54.016915+0000","last_became_peered":"2023-06-03T20:11:54.016915+0000","last_unstale":"2023-06-03T20:19:28.042112+0000","last_undegraded":"2023-06-03T20:11:54.009884+0000","last_fullsized":"2023-06-03T20:11:54.009713+0000","mapping_epoch":195,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0],"acting":[7,0],"avail_no_missing":["7","0"],"object_location_counts":[{"shards":"0,7","objects":1}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":171,"reported_epoch":230,"state":"active+clean+remapped","last_fresh":"2023-06-03T20:19:28.048822+0000","last_change":"2023-06-03T20:17:23.832761+0000","last_active":"2023-06-03T20:19:28.048822+0000","last_peered":"2023-06-03T20:19:28.048822+0000","last_clean":"2023-06-03T20:19:28.048822+0000","last_became_active":"2023-06-03T20:17:23.832000+0000","last_became_peered":"2023-06-03T20:17:23.832000+0000","last_unstale":"2023-06-03T20:19:28.048822+0000","last_undegraded":"2023-06-03T20:19:28.048822+0000","last_fullsized":"2023-06-03T20:19:28.048822+0000","mapping_epoch":223,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":224,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5],"acting":[7,5,6],"avail_no_missing":["7","5","6"],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"12.1","version":"231'9629","reported_seq":9723,"reported_epoch":231,"state":"active+clean","last_fresh":"2023-06-03T20:19:53.855235+0000","last_change":"2023-06-03T20:19:30.701717+0000","last_active":"2023-06-03T20:19:53.855235+0000","last_peered":"2023-06-03T20:19:53.855235+0000","last_clean":"2023-06-03T20:19:53.855235+0000","last_became_active":"2023-06-03T20:19:28.052763+0000","last_became_peered":"2023-06-03T20:19:28.052763+0000","last_unstale":"2023-06-03T20:19:53.855235+0000","last_undegraded":"2023-06-03T20:19:53.855235+0000","last_fullsized":"2023-06-03T20:19:53.855235+0000","mapping_epoch":217,"log_start":"224'3400","ondisk_log_start":"224'3400","created":217,"last_epoch_clean":230,"parent":"0.0","parent_split_bits":0,"last_scrub":"230'8991","last_scrub_stamp":"2023-06-03T20:19:30.701448+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T20:16:46.173157+0000","last_clean_scrub_stamp":"2023-06-03T20:19:30.701448+0000","objects_scrubbed":2287,"log_size":6229,"log_dups_size":0,"ondisk_log_size":6229,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":2,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T04:24:32.520075+0000","scrub_duration":1.0673071169999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":106692608,"num_objects":1628,"num_object_clones":0,"num_object_copies":4884,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1628,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":659,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,5],"acting":[3,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"60'2","reported_seq":234,"reported_epoch":230,"state":"active+undersized+degraded","last_fresh":"2023-06-03T20:19:28.040770+0000","last_change":"2023-06-03T20:11:54.012607+0000","last_active":"2023-06-03T20:19:28.040770+0000","last_peered":"2023-06-03T20:19:28.040770+0000","last_clean":"2023-06-03T20:11:43.972261+0000","last_became_active":"2023-06-03T20:11:54.012607+0000","last_became_peered":"2023-06-03T20:11:54.012607+0000","last_unstale":"2023-06-03T20:19:28.040770+0000","last_undegraded":"2023-06-03T20:11:54.010373+0000","last_fullsized":"2023-06-03T20:11:54.010224+0000","mapping_epoch":195,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0],"acting":[7,0],"avail_no_missing":["7","0"],"object_location_counts":[{"shards":"0,7","objects":1}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":224,"reported_epoch":230,"state":"active+clean+remapped","last_fresh":"2023-06-03T20:19:28.036845+0000","last_change":"2023-06-03T20:12:31.941492+0000","last_active":"2023-06-03T20:19:28.036845+0000","last_peered":"2023-06-03T20:19:28.036845+0000","last_clean":"2023-06-03T20:19:28.036845+0000","last_became_active":"2023-06-03T20:12:31.941076+0000","last_became_peered":"2023-06-03T20:12:31.941076+0000","last_unstale":"2023-06-03T20:19:28.036845+0000","last_undegraded":"2023-06-03T20:19:28.036845+0000","last_fullsized":"2023-06-03T20:19:28.036845+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T03:59:11.007424+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0],"acting":[3,0,6],"avail_no_missing":["3","0","6"],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"12.2","version":"231'9356","reported_seq":9385,"reported_epoch":231,"state":"active+undersized+degraded","last_fresh":"2023-06-03T20:19:53.960617+0000","last_change":"2023-06-03T20:19:28.164924+0000","last_active":"2023-06-03T20:19:53.960617+0000","last_peered":"2023-06-03T20:19:53.960617+0000","last_clean":"2023-06-03T20:16:46.173157+0000","last_became_active":"2023-06-03T20:19:28.076958+0000","last_became_peered":"2023-06-03T20:19:28.076958+0000","last_unstale":"2023-06-03T20:19:53.960617+0000","last_undegraded":"2023-06-03T20:19:28.043700+0000","last_fullsized":"2023-06-03T20:19:28.040385+0000","mapping_epoch":217,"log_start":"224'3100","ondisk_log_start":"224'3100","created":217,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T20:16:46.173157+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T20:16:46.173157+0000","last_clean_scrub_stamp":"2023-06-03T20:16:46.173157+0000","objects_scrubbed":0,"log_size":6256,"log_dups_size":0,"ondisk_log_size":6256,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":105512960,"num_objects":1610,"num_object_clones":0,"num_object_copies":4830,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1610,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1610,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5033,"num_write_kb":212576,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7],"acting":[3,7],"avail_no_missing":["3","7"],"object_location_counts":[{"shards":"3,7","objects":1610}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.d","version":"60'1","reported_seq":194,"reported_epoch":230,"state":"active+clean","last_fresh":"2023-06-03T20:19:28.040209+0000","last_change":"2023-06-03T20:16:20.915741+0000","last_active":"2023-06-03T20:19:28.040209+0000","last_peered":"2023-06-03T20:19:28.040209+0000","last_clean":"2023-06-03T20:19:28.040209+0000","last_became_active":"2023-06-03T20:16:20.908091+0000","last_became_peered":"2023-06-03T20:16:20.908091+0000","last_unstale":"2023-06-03T20:19:28.040209+0000","last_undegraded":"2023-06-03T20:19:28.040209+0000","last_fullsized":"2023-06-03T20:19:28.040209+0000","mapping_epoch":212,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":213,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":70,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,3],"acting":[7,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":177,"reported_epoch":230,"state":"active+clean","last_fresh":"2023-06-03T20:19:28.040215+0000","last_change":"2023-06-03T20:12:30.910738+0000","last_active":"2023-06-03T20:19:28.040215+0000","last_peered":"2023-06-03T20:19:28.040215+0000","last_clean":"2023-06-03T20:19:28.040215+0000","last_became_active":"2023-06-03T20:12:30.910145+0000","last_became_peered":"2023-06-03T20:12:30.910145+0000","last_unstale":"2023-06-03T20:19:28.040215+0000","last_undegraded":"2023-06-03T20:19:28.040215+0000","last_fullsized":"2023-06-03T20:19:28.040215+0000","mapping_epoch":198,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":199,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,0],"acting":[7,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"12.3","version":"231'10868","reported_seq":11014,"reported_epoch":231,"state":"active+clean+scrubbing+deep","last_fresh":"2023-06-03T20:19:56.791834+0000","last_change":"2023-06-03T20:19:47.268050+0000","last_active":"2023-06-03T20:19:56.791834+0000","last_peered":"2023-06-03T20:19:56.791834+0000","last_clean":"2023-06-03T20:19:56.791834+0000","last_became_active":"2023-06-03T20:19:28.047502+0000","last_became_peered":"2023-06-03T20:19:28.047502+0000","last_unstale":"2023-06-03T20:19:56.791834+0000","last_undegraded":"2023-06-03T20:19:56.791834+0000","last_fullsized":"2023-06-03T20:19:56.791834+0000","mapping_epoch":217,"log_start":"224'2600","ondisk_log_start":"224'2600","created":217,"last_epoch_clean":230,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T20:16:46.173157+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T20:16:46.173157+0000","last_clean_scrub_stamp":"2023-06-03T20:16:46.173157+0000","objects_scrubbed":1474,"log_size":8268,"log_dups_size":0,"ondisk_log_size":8268,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"deep scrubbing for 9s","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":99123200,"num_objects":1512,"num_object_clones":0,"num_object_copies":4536,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1512,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5078,"num_write_kb":210912,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":201,"reported_epoch":231,"state":"active+clean","last_fresh":"2023-06-03T20:19:31.082911+0000","last_change":"2023-06-03T20:17:22.835469+0000","last_active":"2023-06-03T20:19:31.082911+0000","last_peered":"2023-06-03T20:19:31.082911+0000","last_clean":"2023-06-03T20:19:31.082911+0000","last_became_active":"2023-06-03T20:17:22.835172+0000","last_became_peered":"2023-06-03T20:17:22.835172+0000","last_unstale":"2023-06-03T20:19:31.082911+0000","last_undegraded":"2023-06-03T20:19:31.082911+0000","last_fullsized":"2023-06-03T20:19:31.082911+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T21:59:45.737240+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,0],"acting":[3,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":237,"reported_epoch":230,"state":"active+undersized","last_fresh":"2023-06-03T20:19:28.035102+0000","last_change":"2023-06-03T20:11:54.017375+0000","last_active":"2023-06-03T20:19:28.035102+0000","last_peered":"2023-06-03T20:19:28.035102+0000","last_clean":"2023-06-03T20:11:50.798782+0000","last_became_active":"2023-06-03T20:11:54.017375+0000","last_became_peered":"2023-06-03T20:11:54.017375+0000","last_unstale":"2023-06-03T20:19:28.035102+0000","last_undegraded":"2023-06-03T20:19:28.035102+0000","last_fullsized":"2023-06-03T20:11:54.008913+0000","mapping_epoch":195,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T21:44:01.134376+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3],"acting":[0,3],"avail_no_missing":["0","3"],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"12.4","version":"231'9281","reported_seq":9309,"reported_epoch":231,"state":"active+undersized+degraded","last_fresh":"2023-06-03T20:19:53.960364+0000","last_change":"2023-06-03T20:19:28.045312+0000","last_active":"2023-06-03T20:19:53.960364+0000","last_peered":"2023-06-03T20:19:53.960364+0000","last_clean":"2023-06-03T20:16:46.173157+0000","last_became_active":"2023-06-03T20:19:28.045312+0000","last_became_peered":"2023-06-03T20:19:28.045312+0000","last_unstale":"2023-06-03T20:19:53.960364+0000","last_undegraded":"2023-06-03T20:19:28.037929+0000","last_fullsized":"2023-06-03T20:19:28.037826+0000","mapping_epoch":217,"log_start":"224'3000","ondisk_log_start":"224'3000","created":217,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T20:16:46.173157+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T20:16:46.173157+0000","last_clean_scrub_stamp":"2023-06-03T20:16:46.173157+0000","objects_scrubbed":0,"log_size":6281,"log_dups_size":0,"ondisk_log_size":6281,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":104923136,"num_objects":1601,"num_object_clones":0,"num_object_copies":4803,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1601,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1601,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5015,"num_write_kb":211712,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7],"acting":[3,7],"avail_no_missing":["3","7"],"object_location_counts":[{"shards":"3,7","objects":1601}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.b","version":"60'2","reported_seq":211,"reported_epoch":230,"state":"active+undersized+degraded","last_fresh":"2023-06-03T20:19:28.038838+0000","last_change":"2023-06-03T20:11:54.019446+0000","last_active":"2023-06-03T20:19:28.038838+0000","last_peered":"2023-06-03T20:19:28.038838+0000","last_clean":"2023-06-03T20:11:43.977155+0000","last_became_active":"2023-06-03T20:11:54.019446+0000","last_became_peered":"2023-06-03T20:11:54.019446+0000","last_unstale":"2023-06-03T20:19:28.038838+0000","last_undegraded":"2023-06-03T20:11:54.013169+0000","last_fullsized":"2023-06-03T20:11:54.012960+0000","mapping_epoch":195,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T20:48:53.735497+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0],"acting":[3,0],"avail_no_missing":["3","0"],"object_location_counts":[{"shards":"0,3","objects":1}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":145,"reported_epoch":230,"state":"active+clean+remapped","last_fresh":"2023-06-03T20:19:28.038153+0000","last_change":"2023-06-03T20:16:21.917354+0000","last_active":"2023-06-03T20:19:28.038153+0000","last_peered":"2023-06-03T20:19:28.038153+0000","last_clean":"2023-06-03T20:19:28.038153+0000","last_became_active":"2023-06-03T20:16:21.916856+0000","last_became_peered":"2023-06-03T20:16:21.916856+0000","last_unstale":"2023-06-03T20:19:28.038153+0000","last_undegraded":"2023-06-03T20:19:28.038153+0000","last_fullsized":"2023-06-03T20:19:28.038153+0000","mapping_epoch":213,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":214,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T06:32:46.792786+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7],"acting":[3,7,6],"avail_no_missing":["3","6","7"],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"12.5","version":"231'9241","reported_seq":9281,"reported_epoch":231,"state":"active+clean","last_fresh":"2023-06-03T20:19:55.617837+0000","last_change":"2023-06-03T20:19:54.505955+0000","last_active":"2023-06-03T20:19:55.617837+0000","last_peered":"2023-06-03T20:19:55.617837+0000","last_clean":"2023-06-03T20:19:55.617837+0000","last_became_active":"2023-06-03T20:19:28.109649+0000","last_became_peered":"2023-06-03T20:19:28.109649+0000","last_unstale":"2023-06-03T20:19:55.617837+0000","last_undegraded":"2023-06-03T20:19:55.617837+0000","last_fullsized":"2023-06-03T20:19:55.617837+0000","mapping_epoch":217,"log_start":"221'100","ondisk_log_start":"221'100","created":217,"last_epoch_clean":230,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T20:16:46.173157+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T20:16:46.173157+0000","last_clean_scrub_stamp":"2023-06-03T20:16:46.173157+0000","objects_scrubbed":0,"log_size":9141,"log_dups_size":0,"ondisk_log_size":9141,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":100139008,"num_objects":1528,"num_object_clones":0,"num_object_copies":4584,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1528,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4992,"num_write_kb":208640,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":219,"reported_epoch":230,"state":"active+clean+remapped","last_fresh":"2023-06-03T20:19:28.036357+0000","last_change":"2023-06-03T20:16:21.913184+0000","last_active":"2023-06-03T20:19:28.036357+0000","last_peered":"2023-06-03T20:19:28.036357+0000","last_clean":"2023-06-03T20:19:28.036357+0000","last_became_active":"2023-06-03T20:16:21.912223+0000","last_became_peered":"2023-06-03T20:16:21.912223+0000","last_unstale":"2023-06-03T20:19:28.036357+0000","last_undegraded":"2023-06-03T20:19:28.036357+0000","last_fullsized":"2023-06-03T20:19:28.036357+0000","mapping_epoch":213,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":214,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T23:07:23.671860+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5],"acting":[3,5,6],"avail_no_missing":["3","5","6"],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":205,"reported_epoch":230,"state":"active+clean+remapped","last_fresh":"2023-06-03T20:19:28.038926+0000","last_change":"2023-06-03T20:16:21.909200+0000","last_active":"2023-06-03T20:19:28.038926+0000","last_peered":"2023-06-03T20:19:28.038926+0000","last_clean":"2023-06-03T20:19:28.038926+0000","last_became_active":"2023-06-03T20:16:21.908774+0000","last_became_peered":"2023-06-03T20:16:21.908774+0000","last_unstale":"2023-06-03T20:19:28.038926+0000","last_undegraded":"2023-06-03T20:19:28.038926+0000","last_fullsized":"2023-06-03T20:19:28.038926+0000","mapping_epoch":213,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":214,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T22:38:01.732512+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0,6],"avail_no_missing":["2","0","6"],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"12.6","version":"231'9410","reported_seq":9445,"reported_epoch":231,"state":"active+clean","last_fresh":"2023-06-03T20:19:56.752752+0000","last_change":"2023-06-03T20:19:30.183106+0000","last_active":"2023-06-03T20:19:56.752752+0000","last_peered":"2023-06-03T20:19:56.752752+0000","last_clean":"2023-06-03T20:19:56.752752+0000","last_became_active":"2023-06-03T20:19:28.060975+0000","last_became_peered":"2023-06-03T20:19:28.060975+0000","last_unstale":"2023-06-03T20:19:56.752752+0000","last_undegraded":"2023-06-03T20:19:56.752752+0000","last_fullsized":"2023-06-03T20:19:56.752752+0000","mapping_epoch":217,"log_start":"224'2500","ondisk_log_start":"224'2500","created":217,"last_epoch_clean":230,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T20:16:46.173157+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T20:16:46.173157+0000","last_clean_scrub_stamp":"2023-06-03T20:16:46.173157+0000","objects_scrubbed":0,"log_size":6910,"log_dups_size":0,"ondisk_log_size":6910,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":94732288,"num_objects":1445,"num_object_clones":0,"num_object_copies":4335,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1445,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5117,"num_write_kb":210016,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":132,"reported_epoch":230,"state":"active+undersized","last_fresh":"2023-06-03T20:19:28.037337+0000","last_change":"2023-06-03T20:11:54.018578+0000","last_active":"2023-06-03T20:19:28.037337+0000","last_peered":"2023-06-03T20:19:28.037337+0000","last_clean":"2023-06-03T19:55:41.905941+0000","last_became_active":"2023-06-03T20:11:54.018578+0000","last_became_peered":"2023-06-03T20:11:54.018578+0000","last_unstale":"2023-06-03T20:19:28.037337+0000","last_undegraded":"2023-06-03T20:19:28.037337+0000","last_fullsized":"2023-06-03T20:11:54.009386+0000","mapping_epoch":195,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T21:13:40.683176+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7],"acting":[2,7],"avail_no_missing":["2","7"],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":180,"reported_epoch":230,"state":"active+clean+remapped","last_fresh":"2023-06-03T20:19:28.039640+0000","last_change":"2023-06-03T20:12:31.910070+0000","last_active":"2023-06-03T20:19:28.039640+0000","last_peered":"2023-06-03T20:19:28.039640+0000","last_clean":"2023-06-03T20:19:28.039640+0000","last_became_active":"2023-06-03T20:12:31.909692+0000","last_became_peered":"2023-06-03T20:12:31.909692+0000","last_unstale":"2023-06-03T20:19:28.039640+0000","last_undegraded":"2023-06-03T20:19:28.039640+0000","last_fullsized":"2023-06-03T20:19:28.039640+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T07:20:52.604735+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5],"acting":[3,5,6],"avail_no_missing":["3","5","6"],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"12.7","version":"231'9526","reported_seq":9552,"reported_epoch":231,"state":"active+undersized+degraded","last_fresh":"2023-06-03T20:19:58.379011+0000","last_change":"2023-06-03T20:19:28.098542+0000","last_active":"2023-06-03T20:19:58.379011+0000","last_peered":"2023-06-03T20:19:58.379011+0000","last_clean":"2023-06-03T20:16:46.173157+0000","last_became_active":"2023-06-03T20:19:28.098542+0000","last_became_peered":"2023-06-03T20:19:28.098542+0000","last_unstale":"2023-06-03T20:19:58.379011+0000","last_undegraded":"2023-06-03T20:19:28.040568+0000","last_fullsized":"2023-06-03T20:19:28.040378+0000","mapping_epoch":217,"log_start":"224'2400","ondisk_log_start":"224'2400","created":217,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T20:16:46.173157+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T20:16:46.173157+0000","last_clean_scrub_stamp":"2023-06-03T20:16:46.173157+0000","objects_scrubbed":0,"log_size":7126,"log_dups_size":0,"ondisk_log_size":7126,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":97976320,"num_objects":1495,"num_object_clones":0,"num_object_copies":4485,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1495,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1495,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5176,"num_write_kb":213472,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2],"acting":[7,2],"avail_no_missing":["7","2"],"object_location_counts":[{"shards":"2,7","objects":1495}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":220,"reported_epoch":230,"state":"active+undersized","last_fresh":"2023-06-03T20:19:28.035394+0000","last_change":"2023-06-03T20:11:54.026212+0000","last_active":"2023-06-03T20:19:28.035394+0000","last_peered":"2023-06-03T20:19:28.035394+0000","last_clean":"2023-06-03T20:11:43.974120+0000","last_became_active":"2023-06-03T20:11:54.026212+0000","last_became_peered":"2023-06-03T20:11:54.026212+0000","last_unstale":"2023-06-03T20:19:28.035394+0000","last_undegraded":"2023-06-03T20:19:28.035394+0000","last_fullsized":"2023-06-03T20:11:54.007103+0000","mapping_epoch":195,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T03:49:35.033521+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7],"acting":[3,7],"avail_no_missing":["3","7"],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":153,"reported_epoch":230,"state":"active+undersized","last_fresh":"2023-06-03T20:19:28.040063+0000","last_change":"2023-06-03T20:11:54.026421+0000","last_active":"2023-06-03T20:19:28.040063+0000","last_peered":"2023-06-03T20:19:28.040063+0000","last_clean":"2023-06-03T20:11:43.974605+0000","last_became_active":"2023-06-03T20:11:54.026421+0000","last_became_peered":"2023-06-03T20:11:54.026421+0000","last_unstale":"2023-06-03T20:19:28.040063+0000","last_undegraded":"2023-06-03T20:19:28.040063+0000","last_fullsized":"2023-06-03T20:11:54.009062+0000","mapping_epoch":195,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T02:07:21.901385+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7],"acting":[3,7],"avail_no_missing":["3","7"],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"12.a","version":"231'10197","reported_seq":10221,"reported_epoch":231,"state":"active+clean","last_fresh":"2023-06-03T20:19:55.604277+0000","last_change":"2023-06-03T20:16:47.247803+0000","last_active":"2023-06-03T20:19:55.604277+0000","last_peered":"2023-06-03T20:19:55.604277+0000","last_clean":"2023-06-03T20:19:55.604277+0000","last_became_active":"2023-06-03T20:16:47.247523+0000","last_became_peered":"2023-06-03T20:16:47.247523+0000","last_unstale":"2023-06-03T20:19:55.604277+0000","last_undegraded":"2023-06-03T20:19:55.604277+0000","last_fullsized":"2023-06-03T20:19:55.604277+0000","mapping_epoch":217,"log_start":"224'2858","ondisk_log_start":"224'2858","created":217,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T20:16:46.173157+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T20:16:46.173157+0000","last_clean_scrub_stamp":"2023-06-03T20:16:46.173157+0000","objects_scrubbed":0,"log_size":7339,"log_dups_size":0,"ondisk_log_size":7339,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T06:47:22.934291+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":206635008,"num_objects":3153,"num_object_clones":0,"num_object_copies":9459,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3153,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10197,"num_write_kb":427200,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,3],"acting":[2,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":144,"reported_epoch":231,"state":"active+clean","last_fresh":"2023-06-03T20:19:31.085106+0000","last_change":"2023-06-03T20:07:19.926958+0000","last_active":"2023-06-03T20:19:31.085106+0000","last_peered":"2023-06-03T20:19:31.085106+0000","last_clean":"2023-06-03T20:19:31.085106+0000","last_became_active":"2023-06-03T20:07:19.926456+0000","last_became_peered":"2023-06-03T20:07:19.926456+0000","last_unstale":"2023-06-03T20:19:31.085106+0000","last_undegraded":"2023-06-03T20:19:31.085106+0000","last_fullsized":"2023-06-03T20:19:31.085106+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:32:20.692043+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:32:20.692043+0000","last_clean_scrub_stamp":"2023-06-03T19:32:20.692043+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T20:47:07.666782+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,2],"acting":[0,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":114,"reported_epoch":230,"state":"active+clean","last_fresh":"2023-06-03T20:19:28.920058+0000","last_change":"2023-06-03T20:17:22.834746+0000","last_active":"2023-06-03T20:19:28.920058+0000","last_peered":"2023-06-03T20:19:28.920058+0000","last_clean":"2023-06-03T20:19:28.920058+0000","last_became_active":"2023-06-03T20:17:22.834448+0000","last_became_peered":"2023-06-03T20:17:22.834448+0000","last_unstale":"2023-06-03T20:19:28.920058+0000","last_undegraded":"2023-06-03T20:19:28.920058+0000","last_fullsized":"2023-06-03T20:19:28.920058+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T21:16:31.167063+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,2],"acting":[3,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":110,"reported_epoch":231,"state":"active+undersized","last_fresh":"2023-06-03T20:19:31.084774+0000","last_change":"2023-06-03T20:17:22.834796+0000","last_active":"2023-06-03T20:19:31.084774+0000","last_peered":"2023-06-03T20:19:31.084774+0000","last_clean":"2023-06-03T20:11:43.981149+0000","last_became_active":"2023-06-03T20:17:22.834796+0000","last_became_peered":"2023-06-03T20:17:22.834796+0000","last_unstale":"2023-06-03T20:19:31.084774+0000","last_undegraded":"2023-06-03T20:19:31.084774+0000","last_fullsized":"2023-06-03T20:17:22.820764+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T22:16:00.682284+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5],"acting":[0,5],"avail_no_missing":["0","5"],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"12.c","version":"231'10065","reported_seq":10089,"reported_epoch":231,"state":"active+clean","last_fresh":"2023-06-03T20:19:56.787570+0000","last_change":"2023-06-03T20:16:47.246691+0000","last_active":"2023-06-03T20:19:56.787570+0000","last_peered":"2023-06-03T20:19:56.787570+0000","last_clean":"2023-06-03T20:19:56.787570+0000","last_became_active":"2023-06-03T20:16:47.246401+0000","last_became_peered":"2023-06-03T20:16:47.246401+0000","last_unstale":"2023-06-03T20:19:56.787570+0000","last_undegraded":"2023-06-03T20:19:56.787570+0000","last_fullsized":"2023-06-03T20:19:56.787570+0000","mapping_epoch":217,"log_start":"225'4495","ondisk_log_start":"225'4495","created":217,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T20:16:46.173157+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T20:16:46.173157+0000","last_clean_scrub_stamp":"2023-06-03T20:16:46.173157+0000","objects_scrubbed":0,"log_size":5570,"log_dups_size":0,"ondisk_log_size":5570,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T22:59:40.915657+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":197066752,"num_objects":3007,"num_object_clones":0,"num_object_copies":9021,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3007,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10065,"num_write_kb":418304,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":200,"reported_epoch":231,"state":"active+clean","last_fresh":"2023-06-03T20:19:31.084957+0000","last_change":"2023-06-03T20:17:22.834516+0000","last_active":"2023-06-03T20:19:31.084957+0000","last_peered":"2023-06-03T20:19:31.084957+0000","last_clean":"2023-06-03T20:19:31.084957+0000","last_became_active":"2023-06-03T20:17:22.833893+0000","last_became_peered":"2023-06-03T20:17:22.833893+0000","last_unstale":"2023-06-03T20:19:31.084957+0000","last_undegraded":"2023-06-03T20:19:31.084957+0000","last_fullsized":"2023-06-03T20:19:31.084957+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:32:20.692043+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:32:20.692043+0000","last_clean_scrub_stamp":"2023-06-03T19:32:20.692043+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T19:57:46.959122+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":115,"reported_epoch":230,"state":"active+clean+remapped","last_fresh":"2023-06-03T20:19:28.041678+0000","last_change":"2023-06-03T20:16:21.912393+0000","last_active":"2023-06-03T20:19:28.041678+0000","last_peered":"2023-06-03T20:19:28.041678+0000","last_clean":"2023-06-03T20:19:28.041678+0000","last_became_active":"2023-06-03T20:16:21.911851+0000","last_became_peered":"2023-06-03T20:16:21.911851+0000","last_unstale":"2023-06-03T20:19:28.041678+0000","last_undegraded":"2023-06-03T20:19:28.041678+0000","last_fullsized":"2023-06-03T20:19:28.041678+0000","mapping_epoch":213,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":214,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T06:21:00.069019+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5],"acting":[0,5,6],"avail_no_missing":["0","5","6"],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":152,"reported_epoch":231,"state":"active+clean","last_fresh":"2023-06-03T20:19:31.070378+0000","last_change":"2023-06-03T20:12:30.911946+0000","last_active":"2023-06-03T20:19:31.070378+0000","last_peered":"2023-06-03T20:19:31.070378+0000","last_clean":"2023-06-03T20:19:31.070378+0000","last_became_active":"2023-06-03T20:12:30.911513+0000","last_became_peered":"2023-06-03T20:12:30.911513+0000","last_unstale":"2023-06-03T20:19:31.070378+0000","last_undegraded":"2023-06-03T20:19:31.070378+0000","last_fullsized":"2023-06-03T20:19:31.070378+0000","mapping_epoch":198,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":199,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T07:22:01.480443+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,7],"acting":[2,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"12.f","version":"231'10282","reported_seq":10307,"reported_epoch":231,"state":"active+undersized+degraded","last_fresh":"2023-06-03T20:19:55.663187+0000","last_change":"2023-06-03T20:17:22.835038+0000","last_active":"2023-06-03T20:19:55.663187+0000","last_peered":"2023-06-03T20:19:55.663187+0000","last_clean":"2023-06-03T20:16:46.173157+0000","last_became_active":"2023-06-03T20:17:22.835038+0000","last_became_peered":"2023-06-03T20:17:22.835038+0000","last_unstale":"2023-06-03T20:19:55.663187+0000","last_undegraded":"2023-06-03T20:17:22.825008+0000","last_fullsized":"2023-06-03T20:17:22.824833+0000","mapping_epoch":222,"log_start":"224'2881","ondisk_log_start":"224'2881","created":217,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T20:16:46.173157+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T20:16:46.173157+0000","last_clean_scrub_stamp":"2023-06-03T20:16:46.173157+0000","objects_scrubbed":0,"log_size":7401,"log_dups_size":0,"ondisk_log_size":7401,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T02:06:54.311682+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":205389824,"num_objects":3134,"num_object_clones":0,"num_object_copies":9402,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3134,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3134,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10282,"num_write_kb":429312,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5],"acting":[2,5],"avail_no_missing":["2","5"],"object_location_counts":[{"shards":"2,5","objects":3134}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":213,"reported_epoch":230,"state":"active+clean","last_fresh":"2023-06-03T20:19:28.038760+0000","last_change":"2023-06-03T19:56:54.504859+0000","last_active":"2023-06-03T20:19:28.038760+0000","last_peered":"2023-06-03T20:19:28.038760+0000","last_clean":"2023-06-03T20:19:28.038760+0000","last_became_active":"2023-06-03T19:56:54.504455+0000","last_became_peered":"2023-06-03T19:56:54.504455+0000","last_unstale":"2023-06-03T20:19:28.038760+0000","last_undegraded":"2023-06-03T20:19:28.038760+0000","last_fullsized":"2023-06-03T20:19:28.038760+0000","mapping_epoch":119,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:32:20.692043+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:32:20.692043+0000","last_clean_scrub_stamp":"2023-06-03T19:32:20.692043+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T06:25:45.522100+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":173,"reported_epoch":230,"state":"active+clean","last_fresh":"2023-06-03T20:19:28.035769+0000","last_change":"2023-06-03T20:16:20.905988+0000","last_active":"2023-06-03T20:19:28.035769+0000","last_peered":"2023-06-03T20:19:28.035769+0000","last_clean":"2023-06-03T20:19:28.035769+0000","last_became_active":"2023-06-03T20:16:20.905535+0000","last_became_peered":"2023-06-03T20:16:20.905535+0000","last_unstale":"2023-06-03T20:19:28.035769+0000","last_undegraded":"2023-06-03T20:19:28.035769+0000","last_fullsized":"2023-06-03T20:19:28.035769+0000","mapping_epoch":212,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":213,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T22:12:58.208178+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,5],"acting":[2,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":81,"reported_epoch":231,"state":"active+clean","last_fresh":"2023-06-03T20:19:31.089801+0000","last_change":"2023-06-03T20:17:22.834837+0000","last_active":"2023-06-03T20:19:31.089801+0000","last_peered":"2023-06-03T20:19:31.089801+0000","last_clean":"2023-06-03T20:19:31.089801+0000","last_became_active":"2023-06-03T20:17:22.834493+0000","last_became_peered":"2023-06-03T20:17:22.834493+0000","last_unstale":"2023-06-03T20:19:31.089801+0000","last_undegraded":"2023-06-03T20:19:31.089801+0000","last_fullsized":"2023-06-03T20:19:31.089801+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T23:57:05.248165+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"12.d","version":"231'9927","reported_seq":9950,"reported_epoch":231,"state":"active+clean","last_fresh":"2023-06-03T20:19:58.417457+0000","last_change":"2023-06-03T20:16:47.245232+0000","last_active":"2023-06-03T20:19:58.417457+0000","last_peered":"2023-06-03T20:19:58.417457+0000","last_clean":"2023-06-03T20:19:58.417457+0000","last_became_active":"2023-06-03T20:16:47.244812+0000","last_became_peered":"2023-06-03T20:16:47.244812+0000","last_unstale":"2023-06-03T20:19:58.417457+0000","last_undegraded":"2023-06-03T20:19:58.417457+0000","last_fullsized":"2023-06-03T20:19:58.417457+0000","mapping_epoch":217,"log_start":"225'4443","ondisk_log_start":"225'4443","created":217,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T20:16:46.173157+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T20:16:46.173157+0000","last_clean_scrub_stamp":"2023-06-03T20:16:46.173157+0000","objects_scrubbed":0,"log_size":5484,"log_dups_size":0,"ondisk_log_size":5484,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":180486168,"num_objects":2755,"num_object_clones":0,"num_object_copies":8265,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2755,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":9927,"num_write_kb":405761,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,3],"acting":[7,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":98,"reported_epoch":230,"state":"active+clean","last_fresh":"2023-06-03T20:19:30.604782+0000","last_change":"2023-06-03T20:17:22.834467+0000","last_active":"2023-06-03T20:19:30.604782+0000","last_peered":"2023-06-03T20:19:30.604782+0000","last_clean":"2023-06-03T20:19:30.604782+0000","last_became_active":"2023-06-03T20:17:22.834147+0000","last_became_peered":"2023-06-03T20:17:22.834147+0000","last_unstale":"2023-06-03T20:19:30.604782+0000","last_undegraded":"2023-06-03T20:19:30.604782+0000","last_fullsized":"2023-06-03T20:19:30.604782+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:32:20.692043+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:32:20.692043+0000","last_clean_scrub_stamp":"2023-06-03T19:32:20.692043+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T04:36:29.567051+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,7],"acting":[2,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"72'139","reported_seq":454,"reported_epoch":230,"state":"active+clean","last_fresh":"2023-06-03T20:19:28.048697+0000","last_change":"2023-06-03T20:16:23.829575+0000","last_active":"2023-06-03T20:19:28.048697+0000","last_peered":"2023-06-03T20:19:28.048697+0000","last_clean":"2023-06-03T20:19:28.048697+0000","last_became_active":"2023-06-03T20:16:23.829028+0000","last_became_peered":"2023-06-03T20:16:23.829028+0000","last_unstale":"2023-06-03T20:19:28.048697+0000","last_undegraded":"2023-06-03T20:19:28.048697+0000","last_fullsized":"2023-06-03T20:19:28.048697+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:27:28.100808+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:27:28.100808+0000","last_clean_scrub_stamp":"2023-06-03T19:27:28.100808+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":590368,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,2],"acting":[7,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":196,"reported_epoch":231,"state":"active+clean+remapped","last_fresh":"2023-06-03T20:19:31.097512+0000","last_change":"2023-06-03T20:16:21.913608+0000","last_active":"2023-06-03T20:19:31.097512+0000","last_peered":"2023-06-03T20:19:31.097512+0000","last_clean":"2023-06-03T20:19:31.097512+0000","last_became_active":"2023-06-03T20:16:21.912814+0000","last_became_peered":"2023-06-03T20:16:21.912814+0000","last_unstale":"2023-06-03T20:19:31.097512+0000","last_undegraded":"2023-06-03T20:19:31.097512+0000","last_fullsized":"2023-06-03T20:19:31.097512+0000","mapping_epoch":213,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":214,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T06:29:01.991511+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5],"acting":[3,5,6],"avail_no_missing":["3","5","6"],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":207,"reported_epoch":230,"state":"active+clean+remapped","last_fresh":"2023-06-03T20:19:28.040662+0000","last_change":"2023-06-03T20:12:31.939716+0000","last_active":"2023-06-03T20:19:28.040662+0000","last_peered":"2023-06-03T20:19:28.040662+0000","last_clean":"2023-06-03T20:19:28.040662+0000","last_became_active":"2023-06-03T20:12:31.938876+0000","last_became_peered":"2023-06-03T20:12:31.938876+0000","last_unstale":"2023-06-03T20:19:28.040662+0000","last_undegraded":"2023-06-03T20:19:28.040662+0000","last_fullsized":"2023-06-03T20:19:28.040662+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0],"acting":[7,0,6],"avail_no_missing":["7","0","6"],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"12.e","version":"231'10268","reported_seq":10287,"reported_epoch":231,"state":"active+undersized+degraded","last_fresh":"2023-06-03T20:19:56.784980+0000","last_change":"2023-06-03T20:17:17.423740+0000","last_active":"2023-06-03T20:19:56.784980+0000","last_peered":"2023-06-03T20:19:56.784980+0000","last_clean":"2023-06-03T20:16:46.173157+0000","last_became_active":"2023-06-03T20:16:47.253524+0000","last_became_peered":"2023-06-03T20:16:47.253524+0000","last_unstale":"2023-06-03T20:19:56.784980+0000","last_undegraded":"2023-06-03T20:17:17.408943+0000","last_fullsized":"2023-06-03T20:16:47.245201+0000","mapping_epoch":217,"log_start":"225'4633","ondisk_log_start":"225'4633","created":217,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T20:16:46.173157+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T20:16:46.173157+0000","last_clean_scrub_stamp":"2023-06-03T20:16:46.173157+0000","objects_scrubbed":0,"log_size":5635,"log_dups_size":0,"ondisk_log_size":5635,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T00:13:48.032998+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":205914112,"num_objects":3142,"num_object_clones":0,"num_object_copies":9426,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3142,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3142,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10268,"num_write_kb":429120,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7],"acting":[0,7],"avail_no_missing":["0","7"],"object_location_counts":[{"shards":"0,7","objects":3142}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":214,"reported_epoch":230,"state":"active+clean","last_fresh":"2023-06-03T20:19:28.048634+0000","last_change":"2023-06-03T20:07:47.855767+0000","last_active":"2023-06-03T20:19:28.048634+0000","last_peered":"2023-06-03T20:19:28.048634+0000","last_clean":"2023-06-03T20:19:28.048634+0000","last_became_active":"2023-06-03T20:07:47.855444+0000","last_became_peered":"2023-06-03T20:07:47.855444+0000","last_unstale":"2023-06-03T20:19:28.048634+0000","last_undegraded":"2023-06-03T20:19:28.048634+0000","last_fullsized":"2023-06-03T20:19:28.048634+0000","mapping_epoch":172,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":173,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:32:20.692043+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:32:20.692043+0000","last_clean_scrub_stamp":"2023-06-03T19:32:20.692043+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,3],"acting":[7,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":216,"reported_epoch":230,"state":"active+undersized","last_fresh":"2023-06-03T20:19:28.034616+0000","last_change":"2023-06-03T20:11:54.017331+0000","last_active":"2023-06-03T20:19:28.034616+0000","last_peered":"2023-06-03T20:19:28.034616+0000","last_clean":"2023-06-03T20:11:43.977931+0000","last_became_active":"2023-06-03T20:11:54.017331+0000","last_became_peered":"2023-06-03T20:11:54.017331+0000","last_unstale":"2023-06-03T20:19:28.034616+0000","last_undegraded":"2023-06-03T20:19:28.034616+0000","last_fullsized":"2023-06-03T20:11:54.007149+0000","mapping_epoch":195,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T04:51:15.533105+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3],"acting":[0,3],"avail_no_missing":["0","3"],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":220,"reported_epoch":231,"state":"active+clean","last_fresh":"2023-06-03T20:19:31.090915+0000","last_change":"2023-06-03T20:07:19.925864+0000","last_active":"2023-06-03T20:19:31.090915+0000","last_peered":"2023-06-03T20:19:31.090915+0000","last_clean":"2023-06-03T20:19:31.090915+0000","last_became_active":"2023-06-03T20:07:19.925541+0000","last_became_peered":"2023-06-03T20:07:19.925541+0000","last_unstale":"2023-06-03T20:19:31.090915+0000","last_undegraded":"2023-06-03T20:19:31.090915+0000","last_fullsized":"2023-06-03T20:19:31.090915+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T00:10:11.790831+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,7],"acting":[0,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"12.b","version":"231'9864","reported_seq":9885,"reported_epoch":231,"state":"active+undersized+degraded","last_fresh":"2023-06-03T20:19:53.971602+0000","last_change":"2023-06-03T20:17:17.433790+0000","last_active":"2023-06-03T20:19:53.971602+0000","last_peered":"2023-06-03T20:19:53.971602+0000","last_clean":"2023-06-03T20:16:46.173157+0000","last_became_active":"2023-06-03T20:16:47.248675+0000","last_became_peered":"2023-06-03T20:16:47.248675+0000","last_unstale":"2023-06-03T20:19:53.971602+0000","last_undegraded":"2023-06-03T20:17:17.407850+0000","last_fullsized":"2023-06-03T20:16:47.187599+0000","mapping_epoch":217,"log_start":"225'5078","ondisk_log_start":"225'5078","created":217,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T20:16:46.173157+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T20:16:46.173157+0000","last_clean_scrub_stamp":"2023-06-03T20:16:46.173157+0000","objects_scrubbed":0,"log_size":4786,"log_dups_size":0,"ondisk_log_size":4786,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T21:43:59.039606+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":199491584,"num_objects":3044,"num_object_clones":0,"num_object_copies":9132,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3044,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3044,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9864,"num_write_kb":413056,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2],"acting":[3,2],"avail_no_missing":["3","2"],"object_location_counts":[{"shards":"2,3","objects":3044}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":223,"reported_epoch":230,"state":"active+undersized","last_fresh":"2023-06-03T20:19:28.040386+0000","last_change":"2023-06-03T20:11:54.012820+0000","last_active":"2023-06-03T20:19:28.040386+0000","last_peered":"2023-06-03T20:19:28.040386+0000","last_clean":"2023-06-03T20:11:43.973280+0000","last_became_active":"2023-06-03T20:11:54.012820+0000","last_became_peered":"2023-06-03T20:11:54.012820+0000","last_unstale":"2023-06-03T20:19:28.040386+0000","last_undegraded":"2023-06-03T20:19:28.040386+0000","last_fullsized":"2023-06-03T20:11:54.011295+0000","mapping_epoch":195,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:32:20.692043+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:32:20.692043+0000","last_clean_scrub_stamp":"2023-06-03T19:32:20.692043+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0],"acting":[7,0],"avail_no_missing":["7","0"],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":164,"reported_epoch":231,"state":"active+clean","last_fresh":"2023-06-03T20:19:31.070454+0000","last_change":"2023-06-03T20:07:19.915065+0000","last_active":"2023-06-03T20:19:31.070454+0000","last_peered":"2023-06-03T20:19:31.070454+0000","last_clean":"2023-06-03T20:19:31.070454+0000","last_became_active":"2023-06-03T20:07:19.914699+0000","last_became_peered":"2023-06-03T20:07:19.914699+0000","last_unstale":"2023-06-03T20:19:31.070454+0000","last_undegraded":"2023-06-03T20:19:31.070454+0000","last_fullsized":"2023-06-03T20:19:31.070454+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T07:16:09.077184+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,0],"acting":[2,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":181,"reported_epoch":230,"state":"active+clean","last_fresh":"2023-06-03T20:19:28.041660+0000","last_change":"2023-06-03T19:47:06.733532+0000","last_active":"2023-06-03T20:19:28.041660+0000","last_peered":"2023-06-03T20:19:28.041660+0000","last_clean":"2023-06-03T20:19:28.041660+0000","last_became_active":"2023-06-03T19:47:06.733131+0000","last_became_peered":"2023-06-03T19:47:06.733131+0000","last_unstale":"2023-06-03T20:19:28.041660+0000","last_undegraded":"2023-06-03T20:19:28.041660+0000","last_fullsized":"2023-06-03T20:19:28.041660+0000","mapping_epoch":86,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":87,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T23:46:03.788646+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"12.8","version":"231'9283","reported_seq":9375,"reported_epoch":231,"state":"active+clean","last_fresh":"2023-06-03T20:19:53.896965+0000","last_change":"2023-06-03T20:19:29.390359+0000","last_active":"2023-06-03T20:19:53.896965+0000","last_peered":"2023-06-03T20:19:53.896965+0000","last_clean":"2023-06-03T20:19:53.896965+0000","last_became_active":"2023-06-03T20:19:28.046199+0000","last_became_peered":"2023-06-03T20:19:28.046199+0000","last_unstale":"2023-06-03T20:19:53.896965+0000","last_undegraded":"2023-06-03T20:19:53.896965+0000","last_fullsized":"2023-06-03T20:19:53.896965+0000","mapping_epoch":217,"log_start":"224'3000","ondisk_log_start":"224'3000","created":217,"last_epoch_clean":230,"parent":"0.0","parent_split_bits":0,"last_scrub":"230'8578","last_scrub_stamp":"2023-06-03T20:19:29.389997+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T20:16:46.173157+0000","last_clean_scrub_stamp":"2023-06-03T20:19:29.389997+0000","objects_scrubbed":2299,"log_size":6283,"log_dups_size":0,"ondisk_log_size":6283,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T02:23:37.014896+0000","scrub_duration":0.81442078600000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":103677952,"num_objects":1582,"num_object_clones":0,"num_object_copies":4746,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1582,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":717,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.7","version":"60'2","reported_seq":190,"reported_epoch":230,"state":"active+clean","last_fresh":"2023-06-03T20:19:28.040133+0000","last_change":"2023-06-03T19:56:54.510148+0000","last_active":"2023-06-03T20:19:28.040133+0000","last_peered":"2023-06-03T20:19:28.040133+0000","last_clean":"2023-06-03T20:19:28.040133+0000","last_became_active":"2023-06-03T19:56:54.509771+0000","last_became_peered":"2023-06-03T19:56:54.509771+0000","last_unstale":"2023-06-03T20:19:28.040133+0000","last_undegraded":"2023-06-03T20:19:28.040133+0000","last_fullsized":"2023-06-03T20:19:28.040133+0000","mapping_epoch":119,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T00:52:42.807432+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":165,"reported_epoch":230,"state":"active+clean+remapped","last_fresh":"2023-06-03T20:19:28.039774+0000","last_change":"2023-06-03T20:16:21.910628+0000","last_active":"2023-06-03T20:19:28.039774+0000","last_peered":"2023-06-03T20:19:28.039774+0000","last_clean":"2023-06-03T20:19:28.039774+0000","last_became_active":"2023-06-03T20:16:21.910183+0000","last_became_peered":"2023-06-03T20:16:21.910183+0000","last_unstale":"2023-06-03T20:19:28.039774+0000","last_undegraded":"2023-06-03T20:19:28.039774+0000","last_fullsized":"2023-06-03T20:19:28.039774+0000","mapping_epoch":213,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":214,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:32:20.692043+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:32:20.692043+0000","last_clean_scrub_stamp":"2023-06-03T19:32:20.692043+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T22:12:12.845109+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0],"acting":[3,0,6],"avail_no_missing":["3","0","6"],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":188,"reported_epoch":230,"state":"active+clean","last_fresh":"2023-06-03T20:19:28.035022+0000","last_change":"2023-06-03T19:56:54.507670+0000","last_active":"2023-06-03T20:19:28.035022+0000","last_peered":"2023-06-03T20:19:28.035022+0000","last_clean":"2023-06-03T20:19:28.035022+0000","last_became_active":"2023-06-03T19:56:54.507293+0000","last_became_peered":"2023-06-03T19:56:54.507293+0000","last_unstale":"2023-06-03T20:19:28.035022+0000","last_undegraded":"2023-06-03T20:19:28.035022+0000","last_fullsized":"2023-06-03T20:19:28.035022+0000","mapping_epoch":119,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T01:49:19.966054+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"12.9","version":"231'9221","reported_seq":9249,"reported_epoch":231,"state":"active+undersized+degraded","last_fresh":"2023-06-03T20:19:55.665359+0000","last_change":"2023-06-03T20:19:28.045828+0000","last_active":"2023-06-03T20:19:55.665359+0000","last_peered":"2023-06-03T20:19:55.665359+0000","last_clean":"2023-06-03T20:16:46.173157+0000","last_became_active":"2023-06-03T20:19:28.045828+0000","last_became_peered":"2023-06-03T20:19:28.045828+0000","last_unstale":"2023-06-03T20:19:55.665359+0000","last_undegraded":"2023-06-03T20:19:28.034601+0000","last_fullsized":"2023-06-03T20:19:28.034460+0000","mapping_epoch":217,"log_start":"221'100","ondisk_log_start":"221'100","created":217,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T20:16:46.173157+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T20:16:46.173157+0000","last_clean_scrub_stamp":"2023-06-03T20:16:46.173157+0000","objects_scrubbed":0,"log_size":9121,"log_dups_size":0,"ondisk_log_size":9121,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":101515264,"num_objects":1549,"num_object_clones":0,"num_object_copies":4647,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1549,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1549,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4989,"num_write_kb":209216,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3],"acting":[2,3],"avail_no_missing":["2","3"],"object_location_counts":[{"shards":"2,3","objects":1549}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":107,"reported_epoch":230,"state":"active+clean","last_fresh":"2023-06-03T20:19:28.042054+0000","last_change":"2023-06-03T20:16:20.907573+0000","last_active":"2023-06-03T20:19:28.042054+0000","last_peered":"2023-06-03T20:19:28.042054+0000","last_clean":"2023-06-03T20:19:28.042054+0000","last_became_active":"2023-06-03T20:16:20.907200+0000","last_became_peered":"2023-06-03T20:16:20.907200+0000","last_unstale":"2023-06-03T20:19:28.042054+0000","last_undegraded":"2023-06-03T20:19:28.042054+0000","last_fullsized":"2023-06-03T20:19:28.042054+0000","mapping_epoch":212,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":213,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:32:20.692043+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:32:20.692043+0000","last_clean_scrub_stamp":"2023-06-03T19:32:20.692043+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,3],"acting":[7,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.6","version":"60'1","reported_seq":233,"reported_epoch":231,"state":"active+undersized+degraded","last_fresh":"2023-06-03T20:19:31.090421+0000","last_change":"2023-06-03T20:11:54.014758+0000","last_active":"2023-06-03T20:19:31.090421+0000","last_peered":"2023-06-03T20:19:31.090421+0000","last_clean":"2023-06-03T20:11:43.980210+0000","last_became_active":"2023-06-03T20:11:54.014758+0000","last_became_peered":"2023-06-03T20:11:54.014758+0000","last_unstale":"2023-06-03T20:19:31.090421+0000","last_undegraded":"2023-06-03T20:11:54.011442+0000","last_fullsized":"2023-06-03T20:11:54.011320+0000","mapping_epoch":195,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T20:43:34.505545+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7],"acting":[0,7],"avail_no_missing":["0","7"],"object_location_counts":[{"shards":"0,7","objects":1}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":104,"reported_epoch":231,"state":"active+clean+remapped","last_fresh":"2023-06-03T20:19:31.082980+0000","last_change":"2023-06-03T20:17:23.833953+0000","last_active":"2023-06-03T20:19:31.082980+0000","last_peered":"2023-06-03T20:19:31.082980+0000","last_clean":"2023-06-03T20:19:31.082980+0000","last_became_active":"2023-06-03T20:17:23.833627+0000","last_became_peered":"2023-06-03T20:17:23.833627+0000","last_unstale":"2023-06-03T20:19:31.082980+0000","last_undegraded":"2023-06-03T20:19:31.082980+0000","last_fullsized":"2023-06-03T20:19:31.082980+0000","mapping_epoch":223,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":224,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T06:35:53.082804+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5],"acting":[0,5,6],"avail_no_missing":["0","5","6"],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"12.14","version":"231'9268","reported_seq":9296,"reported_epoch":231,"state":"active+undersized+degraded","last_fresh":"2023-06-03T20:19:53.980544+0000","last_change":"2023-06-03T20:19:28.104845+0000","last_active":"2023-06-03T20:19:53.980544+0000","last_peered":"2023-06-03T20:19:53.980544+0000","last_clean":"2023-06-03T20:16:46.173157+0000","last_became_active":"2023-06-03T20:19:28.104845+0000","last_became_peered":"2023-06-03T20:19:28.104845+0000","last_unstale":"2023-06-03T20:19:53.980544+0000","last_undegraded":"2023-06-03T20:19:28.089706+0000","last_fullsized":"2023-06-03T20:19:28.045033+0000","mapping_epoch":229,"log_start":"224'3000","ondisk_log_start":"224'3000","created":229,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T20:16:46.173157+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T20:16:46.173157+0000","last_clean_scrub_stamp":"2023-06-03T20:16:46.173157+0000","objects_scrubbed":0,"log_size":6268,"log_dups_size":0,"ondisk_log_size":6268,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":105775104,"num_objects":1614,"num_object_clones":0,"num_object_copies":4842,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1614,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1614,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5002,"num_write_kb":211712,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7],"acting":[3,7],"avail_no_missing":["3","7"],"object_location_counts":[{"shards":"3,7","objects":1614}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1b","version":"60'1","reported_seq":203,"reported_epoch":231,"state":"active+undersized","last_fresh":"2023-06-03T20:19:31.085907+0000","last_change":"2023-06-03T20:11:54.016103+0000","last_active":"2023-06-03T20:19:31.085907+0000","last_peered":"2023-06-03T20:19:31.085907+0000","last_clean":"2023-06-03T20:11:43.979629+0000","last_became_active":"2023-06-03T20:11:54.016103+0000","last_became_peered":"2023-06-03T20:11:54.016103+0000","last_unstale":"2023-06-03T20:19:31.085907+0000","last_undegraded":"2023-06-03T20:19:31.085907+0000","last_fullsized":"2023-06-03T20:11:54.009298+0000","mapping_epoch":195,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T04:35:48.454086+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7],"acting":[0,7],"avail_no_missing":["0","7"],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":232,"reported_epoch":230,"state":"active+clean","last_fresh":"2023-06-03T20:19:28.037114+0000","last_change":"2023-06-03T20:07:19.914985+0000","last_active":"2023-06-03T20:19:28.037114+0000","last_peered":"2023-06-03T20:19:28.037114+0000","last_clean":"2023-06-03T20:19:28.037114+0000","last_became_active":"2023-06-03T20:07:19.914624+0000","last_became_peered":"2023-06-03T20:07:19.914624+0000","last_unstale":"2023-06-03T20:19:28.037114+0000","last_undegraded":"2023-06-03T20:19:28.037114+0000","last_fullsized":"2023-06-03T20:19:28.037114+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T07:02:50.515170+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"12.13","version":"231'9418","reported_seq":9450,"reported_epoch":231,"state":"active+clean","last_fresh":"2023-06-03T20:19:56.629612+0000","last_change":"2023-06-03T20:19:28.099282+0000","last_active":"2023-06-03T20:19:56.629612+0000","last_peered":"2023-06-03T20:19:56.629612+0000","last_clean":"2023-06-03T20:19:56.629612+0000","last_became_active":"2023-06-03T20:19:28.086759+0000","last_became_peered":"2023-06-03T20:19:28.086759+0000","last_unstale":"2023-06-03T20:19:56.629612+0000","last_undegraded":"2023-06-03T20:19:56.629612+0000","last_fullsized":"2023-06-03T20:19:56.629612+0000","mapping_epoch":229,"log_start":"224'2600","ondisk_log_start":"224'2600","created":229,"last_epoch_clean":230,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T20:16:46.173157+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T20:16:46.173157+0000","last_clean_scrub_stamp":"2023-06-03T20:16:46.173157+0000","objects_scrubbed":0,"log_size":6818,"log_dups_size":0,"ondisk_log_size":6818,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":97550336,"num_objects":1489,"num_object_clones":0,"num_object_copies":4467,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1489,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5103,"num_write_kb":210912,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1c","version":"60'2","reported_seq":223,"reported_epoch":230,"state":"active+clean+remapped","last_fresh":"2023-06-03T20:19:28.042050+0000","last_change":"2023-06-03T20:17:23.833394+0000","last_active":"2023-06-03T20:19:28.042050+0000","last_peered":"2023-06-03T20:19:28.042050+0000","last_clean":"2023-06-03T20:19:28.042050+0000","last_became_active":"2023-06-03T20:17:23.832836+0000","last_became_peered":"2023-06-03T20:17:23.832836+0000","last_unstale":"2023-06-03T20:19:28.042050+0000","last_undegraded":"2023-06-03T20:19:28.042050+0000","last_fullsized":"2023-06-03T20:19:28.042050+0000","mapping_epoch":223,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":224,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":1,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5],"acting":[7,5,6],"avail_no_missing":["7","5","6"],"object_location_counts":[{"shards":"5,6,7","objects":1}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":204,"reported_epoch":230,"state":"active+undersized","last_fresh":"2023-06-03T20:19:28.919901+0000","last_change":"2023-06-03T20:11:54.028489+0000","last_active":"2023-06-03T20:19:28.919901+0000","last_peered":"2023-06-03T20:19:28.919901+0000","last_clean":"2023-06-03T20:07:14.870985+0000","last_became_active":"2023-06-03T20:11:54.028489+0000","last_became_peered":"2023-06-03T20:11:54.028489+0000","last_unstale":"2023-06-03T20:19:28.919901+0000","last_undegraded":"2023-06-03T20:19:28.919901+0000","last_fullsized":"2023-06-03T20:11:54.011931+0000","mapping_epoch":195,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T04:24:31.291273+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2],"acting":[3,2],"avail_no_missing":["3","2"],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"12.12","version":"231'9405","reported_seq":9430,"reported_epoch":231,"state":"active+undersized+degraded","last_fresh":"2023-06-03T20:19:53.930952+0000","last_change":"2023-06-03T20:19:28.136066+0000","last_active":"2023-06-03T20:19:53.930952+0000","last_peered":"2023-06-03T20:19:53.930952+0000","last_clean":"2023-06-03T20:16:46.173157+0000","last_became_active":"2023-06-03T20:19:28.136066+0000","last_became_peered":"2023-06-03T20:19:28.136066+0000","last_unstale":"2023-06-03T20:19:53.930952+0000","last_undegraded":"2023-06-03T20:19:28.100349+0000","last_fullsized":"2023-06-03T20:19:28.042422+0000","mapping_epoch":229,"log_start":"224'3100","ondisk_log_start":"224'3100","created":229,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T20:16:46.173157+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T20:16:46.173157+0000","last_clean_scrub_stamp":"2023-06-03T20:16:46.173157+0000","objects_scrubbed":0,"log_size":6305,"log_dups_size":0,"ondisk_log_size":6305,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":102301696,"num_objects":1561,"num_object_clones":0,"num_object_copies":4683,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1561,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1561,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5082,"num_write_kb":212576,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7],"acting":[3,7],"avail_no_missing":["3","7"],"object_location_counts":[{"shards":"3,7","objects":1561}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1d","version":"60'2","reported_seq":196,"reported_epoch":231,"state":"active+clean+remapped","last_fresh":"2023-06-03T20:19:31.091622+0000","last_change":"2023-06-03T20:17:23.862021+0000","last_active":"2023-06-03T20:19:31.091622+0000","last_peered":"2023-06-03T20:19:31.091622+0000","last_clean":"2023-06-03T20:19:31.091622+0000","last_became_active":"2023-06-03T20:17:23.861592+0000","last_became_peered":"2023-06-03T20:17:23.861592+0000","last_unstale":"2023-06-03T20:19:31.091622+0000","last_undegraded":"2023-06-03T20:19:31.091622+0000","last_fullsized":"2023-06-03T20:19:31.091622+0000","mapping_epoch":223,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":224,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T23:22:31.887030+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":1,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5],"acting":[3,5,6],"avail_no_missing":["3","5","6"],"object_location_counts":[{"shards":"3,5,6","objects":1}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":150,"reported_epoch":230,"state":"active+undersized","last_fresh":"2023-06-03T20:19:28.919820+0000","last_change":"2023-06-03T20:11:54.015791+0000","last_active":"2023-06-03T20:19:28.919820+0000","last_peered":"2023-06-03T20:19:28.919820+0000","last_clean":"2023-06-03T19:56:11.746795+0000","last_became_active":"2023-06-03T20:11:54.015791+0000","last_became_peered":"2023-06-03T20:11:54.015791+0000","last_unstale":"2023-06-03T20:19:28.919820+0000","last_undegraded":"2023-06-03T20:19:28.919820+0000","last_fullsized":"2023-06-03T20:11:54.007458+0000","mapping_epoch":195,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T19:54:52.259092+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0],"acting":[3,0],"avail_no_missing":["3","0"],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"12.11","version":"231'9677","reported_seq":9761,"reported_epoch":231,"state":"active+clean","last_fresh":"2023-06-03T20:19:53.755320+0000","last_change":"2023-06-03T20:19:45.668508+0000","last_active":"2023-06-03T20:19:53.755320+0000","last_peered":"2023-06-03T20:19:53.755320+0000","last_clean":"2023-06-03T20:19:53.755320+0000","last_became_active":"2023-06-03T20:19:28.113045+0000","last_became_peered":"2023-06-03T20:19:28.113045+0000","last_unstale":"2023-06-03T20:19:53.755320+0000","last_undegraded":"2023-06-03T20:19:53.755320+0000","last_fullsized":"2023-06-03T20:19:53.755320+0000","mapping_epoch":229,"log_start":"224'3400","ondisk_log_start":"224'3400","created":229,"last_epoch_clean":230,"parent":"0.0","parent_split_bits":5,"last_scrub":"231'9415","last_scrub_stamp":"2023-06-03T20:19:45.668233+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T20:16:46.173157+0000","last_clean_scrub_stamp":"2023-06-03T20:19:45.668233+0000","objects_scrubbed":1879,"log_size":6277,"log_dups_size":0,"ondisk_log_size":6277,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T22:01:58.764903+0000","scrub_duration":0.87397994800000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":105316352,"num_objects":1607,"num_object_clones":0,"num_object_copies":4821,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1607,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":272,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,5],"acting":[3,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"60'1","reported_seq":226,"reported_epoch":230,"state":"active+clean","last_fresh":"2023-06-03T20:19:28.039457+0000","last_change":"2023-06-03T20:12:30.909386+0000","last_active":"2023-06-03T20:19:28.039457+0000","last_peered":"2023-06-03T20:19:28.039457+0000","last_clean":"2023-06-03T20:19:28.039457+0000","last_became_active":"2023-06-03T20:12:30.908991+0000","last_became_peered":"2023-06-03T20:12:30.908991+0000","last_unstale":"2023-06-03T20:19:28.039457+0000","last_undegraded":"2023-06-03T20:19:28.039457+0000","last_fullsized":"2023-06-03T20:19:28.039457+0000","mapping_epoch":198,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":199,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T04:33:12.721959+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,7],"acting":[3,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":228,"reported_epoch":230,"state":"active+clean","last_fresh":"2023-06-03T20:19:28.037261+0000","last_change":"2023-06-03T19:56:54.503416+0000","last_active":"2023-06-03T20:19:28.037261+0000","last_peered":"2023-06-03T20:19:28.037261+0000","last_clean":"2023-06-03T20:19:28.037261+0000","last_became_active":"2023-06-03T19:56:54.503018+0000","last_became_peered":"2023-06-03T19:56:54.503018+0000","last_unstale":"2023-06-03T20:19:28.037261+0000","last_undegraded":"2023-06-03T20:19:28.037261+0000","last_fullsized":"2023-06-03T20:19:28.037261+0000","mapping_epoch":119,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T20:14:00.330616+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"12.10","version":"231'9477","reported_seq":9508,"reported_epoch":231,"state":"active+undersized+degraded","last_fresh":"2023-06-03T20:19:55.576873+0000","last_change":"2023-06-03T20:19:28.169683+0000","last_active":"2023-06-03T20:19:55.576873+0000","last_peered":"2023-06-03T20:19:55.576873+0000","last_clean":"2023-06-03T20:16:46.173157+0000","last_became_active":"2023-06-03T20:19:28.135231+0000","last_became_peered":"2023-06-03T20:19:28.135231+0000","last_unstale":"2023-06-03T20:19:55.576873+0000","last_undegraded":"2023-06-03T20:19:28.094158+0000","last_fullsized":"2023-06-03T20:19:28.062620+0000","mapping_epoch":229,"log_start":"221'300","ondisk_log_start":"221'300","created":229,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T20:16:46.173157+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T20:16:46.173157+0000","last_clean_scrub_stamp":"2023-06-03T20:16:46.173157+0000","objects_scrubbed":0,"log_size":9177,"log_dups_size":0,"ondisk_log_size":9177,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":98861056,"num_objects":1509,"num_object_clones":0,"num_object_copies":4527,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1509,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1509,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5138,"num_write_kb":212672,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7],"acting":[2,7],"avail_no_missing":["2","7"],"object_location_counts":[{"shards":"2,7","objects":1509}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1f","version":"60'4","reported_seq":199,"reported_epoch":230,"state":"active+clean","last_fresh":"2023-06-03T20:19:28.041184+0000","last_change":"2023-06-03T19:56:11.748116+0000","last_active":"2023-06-03T20:19:28.041184+0000","last_peered":"2023-06-03T20:19:28.041184+0000","last_clean":"2023-06-03T20:19:28.041184+0000","last_became_active":"2023-06-03T19:56:11.747636+0000","last_became_peered":"2023-06-03T19:56:11.747636+0000","last_unstale":"2023-06-03T20:19:28.041184+0000","last_undegraded":"2023-06-03T20:19:28.041184+0000","last_fullsized":"2023-06-03T20:19:28.041184+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":115,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T06:57:48.052055+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":47,"reported_epoch":231,"state":"active+clean+remapped","last_fresh":"2023-06-03T20:19:31.080305+0000","last_change":"2023-06-03T20:12:32.093827+0000","last_active":"2023-06-03T20:19:31.080305+0000","last_peered":"2023-06-03T20:19:31.080305+0000","last_clean":"2023-06-03T20:19:31.080305+0000","last_became_active":"2023-06-03T20:12:32.093400+0000","last_became_peered":"2023-06-03T20:12:32.093400+0000","last_unstale":"2023-06-03T20:19:31.080305+0000","last_undegraded":"2023-06-03T20:19:31.080305+0000","last_fullsized":"2023-06-03T20:19:31.080305+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T23:22:58.054742+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2],"acting":[3,2,6],"avail_no_missing":["3","2","6"],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]}],"pool_stats":[{"poolid":12,"num_pg":26,"stat_sum":{"num_bytes":3215982616,"num_objects":49073,"num_object_clones":0,"num_object_copies":147219,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":24766,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":49073,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":139023,"num_write_kb":5692545,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":7951953920,"data_stored":7951941704,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":184948,"ondisk_log_size":184948,"up":65,"acting":65,"num_store_stats":5},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":80,"acting":91,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5486,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":8,"num_objects_misplaced":6,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":62,"num_write_kb":30,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":4786,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":110592,"data_stored":21174,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":30319,"internal_metadata":0},"log_size":56,"ondisk_log_size":56,"up":77,"acting":85,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":590368,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2375680,"data_stored":2361472,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":3,"num_store_stats":4},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":22,"acting":23,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":48,"seq":206158430795,"num_pgs":55,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2436820,"kb_used_data":1690508,"kb_used_omap":0,"kb_used_meta":746304,"kb_avail":91308332,"statfs":{"total":95995035648,"available":93499731968,"internally_reserved":0,"allocated":1731080192,"data_stored":1730502553,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":764215296},"hb_peers":[0,1,2,3,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,1,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 3 20:19:04 2023","interfaces":[{"interface":"back","average":{"1min":4.6589999999999998,"5min":2.0339999999999998,"15min":1.9990000000000001},"min":{"1min":0.26500000000000001,"5min":0.26500000000000001,"15min":0.25800000000000001},"max":{"1min":73.810000000000002,"5min":73.810000000000002,"15min":73.810000000000002},"last":0.28399999999999997},{"interface":"front","average":{"1min":5.04,"5min":2.1190000000000002,"15min":1.776},"min":{"1min":0.309,"5min":0.29299999999999998,"15min":0.27800000000000002},"max":{"1min":72.878,"5min":72.878,"15min":72.878},"last":0.44900000000000001}]},{"osd":1,"last update":"Sat Jun 3 20:19:56 2023","interfaces":[{"interface":"back","average":{"1min":0.81599999999999995,"5min":0.82699999999999996,"15min":1.175},"min":{"1min":0.32600000000000001,"5min":0.308,"15min":0.30099999999999999},"max":{"1min":3.907,"5min":6.9619999999999997,"15min":11.4},"last":0.61499999999999999},{"interface":"front","average":{"1min":0.52100000000000002,"5min":0.80500000000000005,"15min":1.159},"min":{"1min":0.33300000000000002,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":0.91400000000000003,"5min":6.6760000000000002,"15min":9.2080000000000002},"last":0.42299999999999999}]},{"osd":2,"last update":"Sat Jun 3 20:19:56 2023","interfaces":[{"interface":"back","average":{"1min":3.7200000000000002,"5min":4.5810000000000004,"15min":3.9380000000000002},"min":{"1min":0.32200000000000001,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":48.319000000000003,"5min":216.93700000000001,"15min":286.83600000000001},"last":1.679},{"interface":"front","average":{"1min":3.8500000000000001,"5min":4.7389999999999999,"15min":3.9790000000000001},"min":{"1min":0.33600000000000002,"5min":0.315,"15min":0.28699999999999998},"max":{"1min":47.915999999999997,"5min":218.119,"15min":287.32900000000001},"last":1.7809999999999999}]},{"osd":3,"last update":"Sat Jun 3 20:19:44 2023","interfaces":[{"interface":"back","average":{"1min":0.94699999999999995,"5min":2.6779999999999999,"15min":3.367},"min":{"1min":0.39000000000000001,"5min":0.38200000000000001,"15min":0.30299999999999999},"max":{"1min":4.9720000000000004,"5min":74.039000000000001,"15min":265.40499999999997},"last":1.915},{"interface":"front","average":{"1min":1.091,"5min":2.7789999999999999,"15min":3.423},"min":{"1min":0.47099999999999997,"5min":0.39000000000000001,"15min":0.31900000000000001},"max":{"1min":3.657,"5min":73.001999999999995,"15min":265.43599999999998},"last":2.1680000000000001}]},{"osd":4,"last update":"Sat Jun 3 20:10:54 2023","interfaces":[{"interface":"back","average":{"1min":1.417,"5min":1.8879999999999999,"15min":1.3260000000000001},"min":{"1min":0.33900000000000002,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":7.1189999999999998,"5min":21.888999999999999,"15min":21.888999999999999},"last":0.40799999999999997},{"interface":"front","average":{"1min":1.484,"5min":2.1739999999999999,"15min":1.671},"min":{"1min":0.60799999999999998,"5min":0.443,"15min":0.38800000000000001},"max":{"1min":4.8600000000000003,"5min":10.507999999999999,"15min":12.917},"last":0.48899999999999999}]},{"osd":5,"last update":"Sat Jun 3 20:19:44 2023","interfaces":[{"interface":"back","average":{"1min":0.68700000000000006,"5min":0.81299999999999994,"15min":1.3759999999999999},"min":{"1min":0.42399999999999999,"5min":0.376,"15min":0.28899999999999998},"max":{"1min":2.1829999999999998,"5min":6.9269999999999996,"15min":11.443},"last":0.38700000000000001},{"interface":"front","average":{"1min":0.71399999999999997,"5min":0.90600000000000003,"15min":1.4990000000000001},"min":{"1min":0.45300000000000001,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":1.6339999999999999,"5min":6.8390000000000004,"15min":9.1449999999999996},"last":0.374}]},{"osd":6,"last update":"Sat Jun 3 20:19:44 2023","interfaces":[{"interface":"back","average":{"1min":0.67600000000000005,"5min":0.98999999999999999,"15min":1.6779999999999999},"min":{"1min":0.39800000000000002,"5min":0.34799999999999998,"15min":0.34799999999999998},"max":{"1min":1.083,"5min":6.6559999999999997,"15min":11.256},"last":0.48099999999999998},{"interface":"front","average":{"1min":0.67700000000000005,"5min":0.97399999999999998,"15min":1.629},"min":{"1min":0.437,"5min":0.39000000000000001,"15min":0.26900000000000002},"max":{"1min":1.5740000000000001,"5min":7.8179999999999996,"15min":10.894},"last":0.80400000000000005}]}]},{"osd":6,"up_from":0,"seq":180388627028,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":1,"up_from":0,"seq":442381631826,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":0,"up_from":86,"seq":369367187852,"num_pgs":54,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1985216,"kb_used_data":1371064,"kb_used_omap":5,"kb_used_meta":614138,"kb_avail":91759936,"statfs":{"total":95995035648,"available":93962174464,"internally_reserved":0,"allocated":1403969536,"data_stored":1403061965,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":628877946},"hb_peers":[1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1,0,1,1,1,8],"upper_bound":128},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Jun 3 20:19:47 2023","interfaces":[{"interface":"back","average":{"1min":0.879,"5min":0.69699999999999995,"15min":0.873},"min":{"1min":0.23400000000000001,"5min":0.22,"15min":0.19400000000000001},"max":{"1min":5.3689999999999998,"5min":5.3689999999999998,"15min":9.3040000000000003},"last":0.27000000000000002},{"interface":"front","average":{"1min":0.871,"5min":0.90500000000000003,"15min":1.2130000000000001},"min":{"1min":0.26300000000000001,"5min":0.23699999999999999,"15min":0.22600000000000001},"max":{"1min":3.9369999999999998,"5min":6.7750000000000004,"15min":11.736000000000001},"last":0.83799999999999997}]},{"osd":2,"last update":"Sat Jun 3 20:19:47 2023","interfaces":[{"interface":"back","average":{"1min":1.6080000000000001,"5min":1.125,"15min":1.23},"min":{"1min":0.26500000000000001,"5min":0.22,"15min":0.193},"max":{"1min":4.4749999999999996,"5min":6.8570000000000002,"15min":11.901},"last":2.0670000000000002},{"interface":"front","average":{"1min":1.5980000000000001,"5min":1.45,"15min":1.4199999999999999},"min":{"1min":0.314,"5min":0.25800000000000001,"15min":0.255},"max":{"1min":5.2969999999999997,"5min":11.988,"15min":17.315999999999999},"last":0.318}]},{"osd":3,"last update":"Sat Jun 3 20:19:34 2023","interfaces":[{"interface":"back","average":{"1min":2.0230000000000001,"5min":2.786,"15min":1.8420000000000001},"min":{"1min":0.248,"5min":0.248,"15min":0.248},"max":{"1min":7.5490000000000004,"5min":57.790999999999997,"15min":57.790999999999997},"last":2.6549999999999998},{"interface":"front","average":{"1min":1.7529999999999999,"5min":2.9100000000000001,"15min":1.9630000000000001},"min":{"1min":0.34899999999999998,"5min":0.33900000000000002,"15min":0.28100000000000003},"max":{"1min":5.4269999999999996,"5min":58.646000000000001,"15min":58.646000000000001},"last":1.1140000000000001}]},{"osd":4,"last update":"Sat Jun 3 20:10:59 2023","interfaces":[{"interface":"back","average":{"1min":1.9370000000000001,"5min":1.7110000000000001,"15min":1.3400000000000001},"min":{"1min":0.45200000000000001,"5min":0.318,"15min":0.318},"max":{"1min":10.025,"5min":10.025,"15min":10.025},"last":0.497},{"interface":"front","average":{"1min":1.673,"5min":1.8160000000000001,"15min":1.429},"min":{"1min":0.56699999999999995,"5min":0.375,"15min":0.375},"max":{"1min":4.6059999999999999,"5min":13.058999999999999,"15min":13.058999999999999},"last":0.75900000000000001}]},{"osd":5,"last update":"Sat Jun 3 20:19:13 2023","interfaces":[{"interface":"back","average":{"1min":1.6659999999999999,"5min":1.3580000000000001,"15min":1.859},"min":{"1min":0.316,"5min":0.316,"15min":0.316},"max":{"1min":6.9240000000000004,"5min":7.5060000000000002,"15min":9.3829999999999991},"last":0.88500000000000001},{"interface":"front","average":{"1min":1.7629999999999999,"5min":1.151,"15min":1.492},"min":{"1min":0.39000000000000001,"5min":0.38400000000000001,"15min":0.32300000000000001},"max":{"1min":6.891,"5min":6.891,"15min":10.159000000000001},"last":2.0089999999999999}]},{"osd":6,"last update":"Sat Jun 3 20:19:13 2023","interfaces":[{"interface":"back","average":{"1min":1.9690000000000001,"5min":1.2450000000000001,"15min":1.5149999999999999},"min":{"1min":0.39000000000000001,"5min":0.36299999999999999,"15min":0.35399999999999998},"max":{"1min":6.9219999999999997,"5min":6.9219999999999997,"15min":10.192},"last":2.0379999999999998},{"interface":"front","average":{"1min":1.4339999999999999,"5min":1.109,"15min":1.5569999999999999},"min":{"1min":0.5,"5min":0.40899999999999997,"15min":0.38300000000000001},"max":{"1min":4.3140000000000001,"5min":4.3140000000000001,"15min":13.117000000000001},"last":0.45300000000000001}]},{"osd":7,"last update":"Sat Jun 3 20:19:13 2023","interfaces":[{"interface":"back","average":{"1min":2.089,"5min":1.76,"15min":1.7889999999999999},"min":{"1min":0.42499999999999999,"5min":0.39800000000000002,"15min":0.39400000000000002},"max":{"1min":6.952,"5min":13.470000000000001,"15min":13.470000000000001},"last":1.966},{"interface":"front","average":{"1min":1.6559999999999999,"5min":1.6279999999999999,"15min":1.7789999999999999},"min":{"1min":0.496,"5min":0.42999999999999999,"15min":0.40300000000000002},"max":{"1min":4.2080000000000002,"5min":12.41,"15min":12.41},"last":0.41099999999999998}]}]},{"osd":2,"up_from":114,"seq":489626272031,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2343536,"kb_used_data":1962856,"kb_used_omap":7,"kb_used_meta":380600,"kb_avail":91401616,"statfs":{"total":95995035648,"available":93595254784,"internally_reserved":0,"allocated":2009964544,"data_stored":2009134061,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8111,"internal_metadata":389734481},"hb_peers":[0,1,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1,0,0,0,2],"upper_bound":64},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 3 20:19:51 2023","interfaces":[{"interface":"back","average":{"1min":2.9390000000000001,"5min":2.798,"15min":2.895},"min":{"1min":0.23999999999999999,"5min":0.20300000000000001,"15min":0.17799999999999999},"max":{"1min":13.833,"5min":116.919,"15min":273.51100000000002},"last":2.577},{"interface":"front","average":{"1min":1.5669999999999999,"5min":2.4489999999999998,"15min":2.9409999999999998},"min":{"1min":0.25700000000000001,"5min":0.20399999999999999,"15min":0.20399999999999999},"max":{"1min":8.1440000000000001,"5min":117.09099999999999,"15min":273.67399999999998},"last":0.39100000000000001}]},{"osd":1,"last update":"Sat Jun 3 20:19:51 2023","interfaces":[{"interface":"back","average":{"1min":1.0089999999999999,"5min":0.84499999999999997,"15min":2.1230000000000002},"min":{"1min":0.247,"5min":0.247,"15min":0.23300000000000001},"max":{"1min":7.1260000000000003,"5min":7.1260000000000003,"15min":273.46499999999997},"last":0.41199999999999998},{"interface":"front","average":{"1min":2.4500000000000002,"5min":1.4399999999999999,"15min":2.4420000000000002},"min":{"1min":0.26900000000000002,"5min":0.25700000000000001,"15min":0.246},"max":{"1min":13.977,"5min":13.977,"15min":274.38400000000001},"last":0.33800000000000002}]},{"osd":3,"last update":"Sat Jun 3 20:19:42 2023","interfaces":[{"interface":"back","average":{"1min":3.9390000000000001,"5min":2.8079999999999998,"15min":2.9929999999999999},"min":{"1min":0.57599999999999996,"5min":0.26700000000000002,"15min":0.24399999999999999},"max":{"1min":14.082000000000001,"5min":103.714,"15min":273.71899999999999},"last":2.6190000000000002},{"interface":"front","average":{"1min":5.2050000000000001,"5min":3.464,"15min":3.234},"min":{"1min":0.29999999999999999,"5min":0.29999999999999999,"15min":0.23899999999999999},"max":{"1min":14.298,"5min":104.05200000000001,"15min":274.21100000000001},"last":0.26800000000000002}]},{"osd":4,"last update":"Sat Jun 3 20:10:59 2023","interfaces":[{"interface":"back","average":{"1min":16.218,"5min":4.9749999999999996,"15min":2.2519999999999998},"min":{"1min":0.48699999999999999,"5min":0.32800000000000001,"15min":0.32700000000000001},"max":{"1min":273.75700000000001,"5min":273.75700000000001,"15min":273.75700000000001},"last":1.6679999999999999},{"interface":"front","average":{"1min":15.512,"5min":4.7480000000000002,"15min":2.1850000000000001},"min":{"1min":0.36199999999999999,"5min":0.35699999999999998,"15min":0.32600000000000001},"max":{"1min":274.27300000000002,"5min":274.27300000000002,"15min":274.27300000000002},"last":1.585}]},{"osd":5,"last update":"Sat Jun 3 20:19:51 2023","interfaces":[{"interface":"back","average":{"1min":2.2160000000000002,"5min":1.3,"15min":2.2290000000000001},"min":{"1min":0.47199999999999998,"5min":0.32100000000000001,"15min":0.317},"max":{"1min":14.361000000000001,"5min":14.361000000000001,"15min":273.78800000000001},"last":3.1629999999999998},{"interface":"front","average":{"1min":1.4370000000000001,"5min":1.087,"15min":2.3170000000000002},"min":{"1min":0.32600000000000001,"5min":0.32600000000000001,"15min":0.32600000000000001},"max":{"1min":7.2370000000000001,"5min":7.2370000000000001,"15min":274.15800000000002},"last":0.73199999999999998}]},{"osd":6,"last update":"Sat Jun 3 20:19:51 2023","interfaces":[{"interface":"back","average":{"1min":2.4329999999999998,"5min":1.403,"15min":2.4689999999999999},"min":{"1min":0.53600000000000003,"5min":0.40200000000000002,"15min":0.34200000000000003},"max":{"1min":14.331,"5min":14.331,"15min":274.03899999999999},"last":3.1909999999999998},{"interface":"front","average":{"1min":1.474,"5min":1.129,"15min":2.4359999999999999},"min":{"1min":0.35599999999999998,"5min":0.35599999999999998,"15min":0.34899999999999998},"max":{"1min":7.2919999999999998,"5min":7.2919999999999998,"15min":274.06299999999999},"last":0.70699999999999996}]},{"osd":7,"last update":"Sat Jun 3 20:19:51 2023","interfaces":[{"interface":"back","average":{"1min":2.4260000000000002,"5min":1.679,"15min":2.5179999999999998},"min":{"1min":0.46899999999999997,"5min":0.34799999999999998,"15min":0.317},"max":{"1min":14.243,"5min":14.243,"15min":274.54000000000002},"last":3.1230000000000002},{"interface":"front","average":{"1min":1.6319999999999999,"5min":1.4530000000000001,"15min":2.4910000000000001},"min":{"1min":0.38600000000000001,"5min":0.38600000000000001,"15min":0.38600000000000001},"max":{"1min":7.2690000000000001,"5min":9.0039999999999996,"15min":274.12700000000001},"last":0.64500000000000002}]}]},{"osd":3,"up_from":119,"seq":511101108502,"num_pgs":63,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2665116,"kb_used_data":2218772,"kb_used_omap":2,"kb_used_meta":446333,"kb_avail":91080036,"statfs":{"total":95995035648,"available":93265956864,"internally_reserved":0,"allocated":2272022528,"data_stored":2271144162,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":457045045},"hb_peers":[0,1,2,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1,1],"upper_bound":8},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 3 20:19:40 2023","interfaces":[{"interface":"back","average":{"1min":3.3180000000000001,"5min":2.778,"15min":2.177},"min":{"1min":0.214,"5min":0.214,"15min":0.192},"max":{"1min":25.992000000000001,"5min":97.164000000000001,"15min":97.164000000000001},"last":6.8159999999999998},{"interface":"front","average":{"1min":4.0019999999999998,"5min":2.8479999999999999,"15min":2.2919999999999998},"min":{"1min":0.248,"5min":0.183,"15min":0.183},"max":{"1min":28.260000000000002,"5min":95.816000000000003,"15min":95.816000000000003},"last":1.8759999999999999}]},{"osd":1,"last update":"Sat Jun 3 20:19:40 2023","interfaces":[{"interface":"back","average":{"1min":1.6850000000000001,"5min":1.2350000000000001,"15min":1.0640000000000001},"min":{"1min":0.26200000000000001,"5min":0.26200000000000001,"15min":0.214},"max":{"1min":6.1859999999999999,"5min":8.5299999999999994,"15min":10.749000000000001},"last":0.59299999999999997},{"interface":"front","average":{"1min":0.996,"5min":1.1759999999999999,"15min":0.996},"min":{"1min":0.23100000000000001,"5min":0.20699999999999999,"15min":0.20699999999999999},"max":{"1min":9.5660000000000007,"5min":10.304,"15min":10.304},"last":8.0630000000000006}]},{"osd":2,"last update":"Sat Jun 3 20:19:40 2023","interfaces":[{"interface":"back","average":{"1min":2.681,"5min":1.877,"15min":1.6679999999999999},"min":{"1min":0.28899999999999998,"5min":0.25700000000000001,"15min":0.217},"max":{"1min":10.702999999999999,"5min":29.106999999999999,"15min":62.527999999999999},"last":3.5779999999999998},{"interface":"front","average":{"1min":2.8340000000000001,"5min":2.077,"15min":1.675},"min":{"1min":0.26600000000000001,"5min":0.23599999999999999,"15min":0.214},"max":{"1min":13.491,"5min":29.175999999999998,"15min":57.935000000000002},"last":0.55900000000000005}]},{"osd":4,"last update":"Sat Jun 3 20:10:53 2023","interfaces":[{"interface":"back","average":{"1min":2.0379999999999998,"5min":1.98,"15min":1.413},"min":{"1min":0.42399999999999999,"5min":0.38200000000000001,"15min":0.38200000000000001},"max":{"1min":9.8360000000000003,"5min":12.757,"15min":12.757},"last":1.968},{"interface":"front","average":{"1min":2.1749999999999998,"5min":1.744,"15min":1.333},"min":{"1min":0.39000000000000001,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":10.122,"5min":10.122,"15min":10.122},"last":0.29099999999999998}]},{"osd":5,"last update":"Sat Jun 3 20:19:40 2023","interfaces":[{"interface":"back","average":{"1min":1.6160000000000001,"5min":1.5840000000000001,"15min":1.393},"min":{"1min":0.37,"5min":0.313,"15min":0.313},"max":{"1min":9.609,"5min":10.275,"15min":10.275},"last":6.7460000000000004},{"interface":"front","average":{"1min":1.8109999999999999,"5min":1.323,"15min":1.323},"min":{"1min":0.32900000000000001,"5min":0.32900000000000001,"15min":0.314},"max":{"1min":7.282,"5min":7.282,"15min":12.56},"last":1.9419999999999999}]},{"osd":6,"last update":"Sat Jun 3 20:19:40 2023","interfaces":[{"interface":"back","average":{"1min":1.883,"5min":1.462,"15min":1.5820000000000001},"min":{"1min":0.376,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":7.3289999999999997,"5min":8.1519999999999992,"15min":13.718},"last":1.911},{"interface":"front","average":{"1min":1.827,"5min":1.76,"15min":1.798},"min":{"1min":0.47099999999999997,"5min":0.39300000000000002,"15min":0.32500000000000001},"max":{"1min":6.5720000000000001,"5min":8.7279999999999998,"15min":13.734999999999999},"last":0.64000000000000001}]},{"osd":7,"last update":"Sat Jun 3 20:19:40 2023","interfaces":[{"interface":"back","average":{"1min":1.8029999999999999,"5min":1.744,"15min":1.5609999999999999},"min":{"1min":0.375,"5min":0.375,"15min":0.35599999999999998},"max":{"1min":6.5110000000000001,"5min":8.7149999999999999,"15min":10.858000000000001},"last":0.61699999999999999},{"interface":"front","average":{"1min":1.8839999999999999,"5min":1.8700000000000001,"15min":1.8380000000000001},"min":{"1min":0.35699999999999998,"5min":0.34899999999999998,"15min":0.34899999999999998},"max":{"1min":13.576000000000001,"5min":18.431999999999999,"15min":18.431999999999999},"last":6.7889999999999997}]}]},{"osd":4,"up_from":124,"seq":532575944879,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1005320,"kb_used_data":818944,"kb_used_omap":10,"kb_used_meta":186357,"kb_avail":92739832,"statfs":{"total":95995035648,"available":94965587968,"internally_reserved":0,"allocated":838598656,"data_stored":837828790,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":10652,"internal_metadata":190830180},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 3 20:11:03 2023","interfaces":[{"interface":"back","average":{"1min":1.002,"5min":4.8399999999999999,"15min":2.468},"min":{"1min":0.58599999999999997,"5min":0.29499999999999998,"15min":0.254},"max":{"1min":3.0139999999999998,"5min":214.72999999999999,"15min":214.72999999999999},"last":2.1539999999999999},{"interface":"front","average":{"1min":2.04,"5min":5.327,"15min":2.6459999999999999},"min":{"1min":0.41399999999999998,"5min":0.40200000000000002,"15min":0.32400000000000001},"max":{"1min":15,"5min":214.76499999999999,"15min":214.76499999999999},"last":0.625}]},{"osd":1,"last update":"Sat Jun 3 20:11:03 2023","interfaces":[{"interface":"back","average":{"1min":1.1439999999999999,"5min":1.639,"15min":1.55},"min":{"1min":0.371,"5min":0.35199999999999998,"15min":0.28100000000000003},"max":{"1min":3.8210000000000002,"5min":10.446,"15min":58.32},"last":0.84599999999999997},{"interface":"front","average":{"1min":0.878,"5min":1.4990000000000001,"15min":1.4610000000000001},"min":{"1min":0.498,"5min":0.34999999999999998,"15min":0.307},"max":{"1min":1.8,"5min":12.919,"15min":58.286000000000001},"last":2.2429999999999999}]},{"osd":2,"last update":"Sat Jun 3 20:11:03 2023","interfaces":[{"interface":"back","average":{"1min":13.406000000000001,"5min":4.6070000000000002,"15min":2.1419999999999999},"min":{"1min":0.55000000000000004,"5min":0.317,"15min":0.30299999999999999},"max":{"1min":248.65799999999999,"5min":248.65799999999999,"15min":248.65799999999999},"last":2.2770000000000001},{"interface":"front","average":{"1min":14.144,"5min":4.7720000000000002,"15min":2.1890000000000001},"min":{"1min":0.46700000000000003,"5min":0.373,"15min":0.34499999999999997},"max":{"1min":249.077,"5min":249.077,"15min":249.077},"last":0.66200000000000003}]},{"osd":3,"last update":"Sat Jun 3 20:11:03 2023","interfaces":[{"interface":"back","average":{"1min":1.2330000000000001,"5min":1.778,"15min":1.6240000000000001},"min":{"1min":0.40600000000000003,"5min":0.35199999999999998,"15min":0.32500000000000001},"max":{"1min":3.7759999999999998,"5min":10.301,"15min":57.954999999999998},"last":1.0920000000000001},{"interface":"front","average":{"1min":1.226,"5min":1.6479999999999999,"15min":1.613},"min":{"1min":0.50700000000000001,"5min":0.40300000000000002,"15min":0.33700000000000002},"max":{"1min":4.9210000000000003,"5min":12.890000000000001,"15min":57.889000000000003},"last":2.2050000000000001}]},{"osd":5,"last update":"Sat Jun 3 20:11:03 2023","interfaces":[{"interface":"back","average":{"1min":1.4590000000000001,"5min":2.0819999999999999,"15min":1.623},"min":{"1min":0.35899999999999999,"5min":0.32000000000000001,"15min":0.313},"max":{"1min":6.6859999999999999,"5min":11.311,"15min":86.576999999999998},"last":0.47599999999999998},{"interface":"front","average":{"1min":1.5269999999999999,"5min":2.1760000000000002,"15min":1.7390000000000001},"min":{"1min":0.32900000000000001,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":7.9269999999999996,"5min":10.997999999999999,"15min":88.013000000000005},"last":4.9119999999999999}]},{"osd":6,"last update":"Sat Jun 3 20:11:03 2023","interfaces":[{"interface":"back","average":{"1min":1.8140000000000001,"5min":2.7429999999999999,"15min":1.5309999999999999},"min":{"1min":0.36099999999999999,"5min":0.34599999999999997,"15min":0.34599999999999997},"max":{"1min":6.6520000000000001,"5min":14.244999999999999,"15min":14.244999999999999},"last":2.109},{"interface":"front","average":{"1min":2.4329999999999998,"5min":2.8679999999999999,"15min":1.629},"min":{"1min":0.42599999999999999,"5min":0.40100000000000002,"15min":0.36499999999999999},"max":{"1min":10.359,"5min":12.145,"15min":12.145},"last":0.81200000000000006}]},{"osd":7,"last update":"Sat Jun 3 20:11:03 2023","interfaces":[{"interface":"back","average":{"1min":1.7270000000000001,"5min":2.052,"15min":1.55},"min":{"1min":0.44,"5min":0.41199999999999998,"15min":0.39700000000000002},"max":{"1min":9.6210000000000004,"5min":10.388,"15min":10.388},"last":4.8150000000000004},{"interface":"front","average":{"1min":1.0629999999999999,"5min":1.7729999999999999,"15min":1.486},"min":{"1min":0.44800000000000001,"5min":0.34200000000000003,"15min":0.31900000000000001},"max":{"1min":1.8260000000000001,"5min":12.951000000000001,"15min":12.951000000000001},"last":2.3109999999999999}]}]},{"osd":5,"up_from":36,"seq":154618823262,"num_pgs":34,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1234932,"kb_used_data":569452,"kb_used_omap":0,"kb_used_meta":665472,"kb_avail":92510220,"statfs":{"total":95995035648,"available":94730465280,"internally_reserved":0,"allocated":583118848,"data_stored":582568786,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":681443328},"hb_peers":[0,1,2,3,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 3 20:19:00 2023","interfaces":[{"interface":"back","average":{"1min":2.1720000000000002,"5min":2.113,"15min":2.4209999999999998},"min":{"1min":0.30099999999999999,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":20.172999999999998,"5min":86.424999999999997,"15min":185.29900000000001},"last":0.58899999999999997},{"interface":"front","average":{"1min":2.226,"5min":2.226,"15min":2.6459999999999999},"min":{"1min":0.33800000000000002,"5min":0.316,"15min":0.26400000000000001},"max":{"1min":20.303000000000001,"5min":86.388999999999996,"15min":185.44200000000001},"last":0.44700000000000001}]},{"osd":1,"last update":"Sat Jun 3 20:19:00 2023","interfaces":[{"interface":"back","average":{"1min":0.60399999999999998,"5min":0.97599999999999998,"15min":1.1899999999999999},"min":{"1min":0.32300000000000001,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":1.4670000000000001,"5min":6.1189999999999998,"15min":9.2390000000000008},"last":0.53600000000000003},{"interface":"front","average":{"1min":0.498,"5min":0.79900000000000004,"15min":1.1779999999999999},"min":{"1min":0.35199999999999998,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":0.84899999999999998,"5min":5.7469999999999999,"15min":9.2739999999999991},"last":0.62}]},{"osd":2,"last update":"Sat Jun 3 20:19:00 2023","interfaces":[{"interface":"back","average":{"1min":1.077,"5min":4.6109999999999998,"15min":3.0659999999999998},"min":{"1min":0.371,"5min":0.33400000000000002,"15min":0.318},"max":{"1min":4.2300000000000004,"5min":216.881,"15min":216.881},"last":1.478},{"interface":"front","average":{"1min":1.2909999999999999,"5min":4.891,"15min":3.387},"min":{"1min":0.35399999999999998,"5min":0.34100000000000003,"15min":0.314},"max":{"1min":4.1379999999999999,"5min":216.751,"15min":216.751},"last":0.51700000000000002}]},{"osd":3,"last update":"Sat Jun 3 20:19:37 2023","interfaces":[{"interface":"back","average":{"1min":1.669,"5min":3.5600000000000001,"15min":3.258},"min":{"1min":0.42999999999999999,"5min":0.36299999999999999,"15min":0.317},"max":{"1min":6.6470000000000002,"5min":86.486999999999995,"15min":264.822},"last":2.3490000000000002},{"interface":"front","average":{"1min":1.6359999999999999,"5min":3.3580000000000001,"15min":3.0449999999999999},"min":{"1min":0.38700000000000001,"5min":0.38,"15min":0.35799999999999998},"max":{"1min":6.6360000000000001,"5min":86.512,"15min":264.75},"last":2.3260000000000001}]},{"osd":4,"last update":"Sat Jun 3 20:11:00 2023","interfaces":[{"interface":"back","average":{"1min":2.2930000000000001,"5min":2.4510000000000001,"15min":1.6839999999999999},"min":{"1min":0.44,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":7.5910000000000002,"5min":14.951000000000001,"15min":14.951000000000001},"last":0.81899999999999995},{"interface":"front","average":{"1min":1.619,"5min":2.0129999999999999,"15min":1.3799999999999999},"min":{"1min":0.42299999999999999,"5min":0.376,"15min":0.376},"max":{"1min":7.2000000000000002,"5min":12.109,"15min":12.109},"last":0.66900000000000004}]},{"osd":6,"last update":"Sat Jun 3 20:19:00 2023","interfaces":[{"interface":"back","average":{"1min":0.745,"5min":1.157,"15min":1.6399999999999999},"min":{"1min":0.39700000000000002,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":1.9630000000000001,"5min":9.7029999999999994,"15min":22.945},"last":0.49299999999999999},{"interface":"front","average":{"1min":0.751,"5min":1.198,"15min":1.8049999999999999},"min":{"1min":0.432,"5min":0.35799999999999998,"15min":0.35799999999999998},"max":{"1min":1.1830000000000001,"5min":5.625,"15min":22.428000000000001},"last":0.56799999999999995}]},{"osd":7,"last update":"Sat Jun 3 20:19:49 2023","interfaces":[{"interface":"back","average":{"1min":0.68700000000000006,"5min":1.0760000000000001,"15min":1.577},"min":{"1min":0.35999999999999999,"5min":0.35999999999999999,"15min":0.35999999999999999},"max":{"1min":1.536,"5min":6.9619999999999997,"15min":14.223000000000001},"last":0.72899999999999998},{"interface":"front","average":{"1min":0.75800000000000001,"5min":1.0429999999999999,"15min":1.3400000000000001},"min":{"1min":0.42199999999999999,"5min":0.373,"15min":0.28799999999999998},"max":{"1min":2.5739999999999998,"5min":8.4489999999999998,"15min":9.0719999999999992},"last":0.69699999999999995}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6928,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4862,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":10652,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":4716,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4932,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":1388707840,"data_stored":1388707840,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":1995837440,"data_stored":1995833368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":2262831104,"data_stored":2262827032,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":577568768,"data_stored":577568768,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":1727008768,"data_stored":1727004696,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-06-03T20:20:03.129 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph osd pool set device_health_metrics pgp_num 11 2023-06-03T20:20:03.141 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:20:03.145 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:20:03.146 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:20:03.247 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:20:03.276 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:20:03.278 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:20:02 smithi089 bash[78452]: cluster 1685823602.8613138 mon.a (mon.0) 1301 : cluster 3 Health check update: Degraded data redundancy: 23222/139359 objects degraded (16.663%), 19 pgs degraded, 20 pgs undersized (PG_DEGRADED) 2023-06-03T20:20:03.279 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:20:02 smithi089 bash[75068]: cluster 1685823602.8613138 mon.a (mon.0) 1301 : cluster 3 Health check update: Degraded data redundancy: 23222/139359 objects degraded (16.663%), 19 pgs degraded, 20 pgs undersized (PG_DEGRADED) 2023-06-03T20:20:03.281 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:20:03.282 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:20:03.383 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:20:03.420 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:20:03.424 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:20:03.425 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:20:03.526 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:20:03.563 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:20:03.570 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:20:03.571 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:20:03.672 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:20:03.704 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:20:03.706 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:20:03.707 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:20:03.807 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:20:03.855 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:20:03.860 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:20:03.861 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:20:03.962 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:20:03.995 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:20:04.000 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:20:04.001 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:20:04.001 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:20:03 smithi089 bash[75068]: cluster 1685823602.9222083 mgr.y (mgr.34104) 1323 : cluster 0 pgmap v1323: 99 pgs: 20 active+clean+remapped, 19 active+undersized+degraded, 11 active+undersized, 49 active+clean; 2.8 GiB data, 11 GiB used, 526 GiB / 536 GiB avail; 22877/135621 objects degraded (16.868%); 6/135621 objects misplaced (0.004%) 2023-06-03T20:20:04.002 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:20:03 smithi089 bash[78452]: cluster 1685823602.9222083 mgr.y (mgr.34104) 1323 : cluster 0 pgmap v1323: 99 pgs: 20 active+clean+remapped, 19 active+undersized+degraded, 11 active+undersized, 49 active+clean; 2.8 GiB data, 11 GiB used, 526 GiB / 536 GiB avail; 22877/135621 objects degraded (16.868%); 6/135621 objects misplaced (0.004%) 2023-06-03T20:20:04.102 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:20:04.135 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:20:04.138 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:20:04.139 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:20:04.240 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:20:04.248 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:20:03 smithi164 bash[59462]: cluster 1685823602.9222083 mgr.y (mgr.34104) 1323 : cluster 0 pgmap v1323: 99 pgs: 20 active+clean+remapped, 19 active+undersized+degraded, 11 active+undersized, 49 active+clean; 2.8 GiB data, 11 GiB used, 526 GiB / 536 GiB avail; 22877/135621 objects degraded (16.868%); 6/135621 objects misplaced (0.004%) 2023-06-03T20:20:04.269 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:20:04.272 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:20:04.273 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:20:04.297 INFO:teuthology.orchestra.run.smithi164.stdout:ERROR: (22) Invalid argument 2023-06-03T20:20:04.299 INFO:teuthology.orchestra.run.smithi164.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-06-03T20:20:04.375 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:20:04.405 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:20:04.407 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:20:04.408 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:20:04.510 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:20:04.540 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:20:04.542 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:20:04.542 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:20:04.643 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:20:04.680 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:20:04.682 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:20:04.683 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:20:04.784 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:20:04.821 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:20:04.825 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:20:04.825 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:20:04.926 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:20:04.958 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:20:04.966 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:20:04.967 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:20:05.068 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:20:05.102 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:20:05.104 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:20:05.104 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:20:05.107 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-06-03T20:20:05.131 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph osd set nodeep-scrub 2023-06-03T20:20:05.205 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:20:05.236 DEBUG:teuthology.orchestra.run.smithi164:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph daemon osd.7 dump_ops_in_flight 2023-06-03T20:20:05.285 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:20:05.289 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:20:05.289 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:20:05.390 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:20:05.422 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:20:05.425 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:20:05.425 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:20:05.526 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:20:05.579 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:20:05.583 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:20:05.584 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:20:05.685 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:20:05.721 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:20:05.723 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:20:05.723 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:20:05.824 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:20:05.865 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:20:05.871 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:20:05.871 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:20:05.972 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:20:06.007 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:20:06.010 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:20:06.010 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:20:06.111 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:20:06.143 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:20:06.145 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:20:06.146 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:20:06.232 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:20:05 smithi089 bash[75068]: cluster 1685823604.923032 mgr.y (mgr.34104) 1324 : cluster 0 pgmap v1324: 99 pgs: 20 active+clean+remapped, 19 active+undersized+degraded, 11 active+undersized, 49 active+clean; 2.6 GiB data, 10 GiB used, 526 GiB / 536 GiB avail; 21598/128976 objects degraded (16.746%); 6/128976 objects misplaced (0.005%) 2023-06-03T20:20:06.232 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 20:20:06 smithi089 bash[71163]: ::ffff:172.21.15.164 - - [03/Jun/2023:20:20:06] "GET /metrics HTTP/1.1" 200 228955 "" "Prometheus/2.33.4" 2023-06-03T20:20:06.233 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:20:06 smithi089 bash[78452]: cluster 1685823604.923032 mgr.y (mgr.34104) 1324 : cluster 0 pgmap v1324: 99 pgs: 20 active+clean+remapped, 19 active+undersized+degraded, 11 active+undersized, 49 active+clean; 2.6 GiB data, 10 GiB used, 526 GiB / 536 GiB avail; 21598/128976 objects degraded (16.746%); 6/128976 objects misplaced (0.005%) 2023-06-03T20:20:06.247 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:20:06.278 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:20:06.280 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:20:06.281 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:20:06.333 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:20:05 smithi164 bash[59462]: cluster 1685823604.923032 mgr.y (mgr.34104) 1324 : cluster 0 pgmap v1324: 99 pgs: 20 active+clean+remapped, 19 active+undersized+degraded, 11 active+undersized, 49 active+clean; 2.6 GiB data, 10 GiB used, 526 GiB / 536 GiB avail; 21598/128976 objects degraded (16.746%); 6/128976 objects misplaced (0.005%) 2023-06-03T20:20:06.382 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:20:06.420 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:20:06.423 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:20:06.424 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:20:06.525 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:20:06.563 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:20:06.566 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:20:06.566 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:20:06.667 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:20:06.704 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:20:06.707 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:20:06.707 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:20:06.808 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:20:06.853 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:20:06.858 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:20:06.859 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:20:06.960 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:20:07.016 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:20:07.021 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:20:07.022 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:20:07.123 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:20:07.160 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:20:07.163 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:20:07.163 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:20:07.264 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:20:07.299 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:20:07.303 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:20:07.303 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:20:07.404 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:20:07.436 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:20:07.438 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:20:07.439 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:20:07.540 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:20:07.568 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:20:07.570 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:20:07.571 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:20:07.672 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:20:07.705 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:20:07.709 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:20:07.710 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:20:07.811 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:20:07.838 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:20:07.841 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:20:07.841 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:20:07.942 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:20:07.986 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:20:07.988 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:20:07.989 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:20:08.046 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/mon.a/config 2023-06-03T20:20:08.089 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:20:08.131 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:20:08.137 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:20:08.138 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:20:08.239 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:20:08.272 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:20:08.276 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:20:08.277 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:20:08.278 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:20:08 smithi089 bash[78452]: cluster 1685823606.9238846 mgr.y (mgr.34104) 1325 : cluster 0 pgmap v1325: 99 pgs: 20 active+clean+remapped, 19 active+undersized+degraded, 11 active+undersized, 49 active+clean; 2.5 GiB data, 9.9 GiB used, 526 GiB / 536 GiB avail; 20296/120564 objects degraded (16.834%); 6/120564 objects misplaced (0.005%) 2023-06-03T20:20:08.279 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:20:08 smithi089 bash[78452]: audit 1685823607.1512725 mon.a (mon.0) 1302 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:20:08.279 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:20:08 smithi089 bash[78452]: audit 1685823607.1543872 mon.c (mon.2) 573 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T20:20:08.279 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:20:08 smithi089 bash[78452]: cluster 1685823607.8624575 mon.a (mon.0) 1303 : cluster 3 Health check update: Degraded data redundancy: 20296/120564 objects degraded (16.834%), 19 pgs degraded, 20 pgs undersized (PG_DEGRADED) 2023-06-03T20:20:08.279 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:20:08 smithi089 bash[75068]: cluster 1685823606.9238846 mgr.y (mgr.34104) 1325 : cluster 0 pgmap v1325: 99 pgs: 20 active+clean+remapped, 19 active+undersized+degraded, 11 active+undersized, 49 active+clean; 2.5 GiB data, 9.9 GiB used, 526 GiB / 536 GiB avail; 20296/120564 objects degraded (16.834%); 6/120564 objects misplaced (0.005%) 2023-06-03T20:20:08.280 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:20:08 smithi089 bash[75068]: audit 1685823607.1512725 mon.a (mon.0) 1302 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:20:08.280 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:20:08 smithi089 bash[75068]: audit 1685823607.1543872 mon.c (mon.2) 573 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T20:20:08.280 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:20:08 smithi089 bash[75068]: cluster 1685823607.8624575 mon.a (mon.0) 1303 : cluster 3 Health check update: Degraded data redundancy: 20296/120564 objects degraded (16.834%), 19 pgs degraded, 20 pgs undersized (PG_DEGRADED) 2023-06-03T20:20:08.378 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:20:08.412 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:20:08.416 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:20:08 smithi164 bash[59462]: cluster 1685823606.9238846 mgr.y (mgr.34104) 1325 : cluster 0 pgmap v1325: 99 pgs: 20 active+clean+remapped, 19 active+undersized+degraded, 11 active+undersized, 49 active+clean; 2.5 GiB data, 9.9 GiB used, 526 GiB / 536 GiB avail; 20296/120564 objects degraded (16.834%); 6/120564 objects misplaced (0.005%) 2023-06-03T20:20:08.416 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:20:08 smithi164 bash[59462]: audit 1685823607.1512725 mon.a (mon.0) 1302 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:20:08.417 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:20:08 smithi164 bash[59462]: audit 1685823607.1543872 mon.c (mon.2) 573 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T20:20:08.417 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:20:08 smithi164 bash[59462]: cluster 1685823607.8624575 mon.a (mon.0) 1303 : cluster 3 Health check update: Degraded data redundancy: 20296/120564 objects degraded (16.834%), 19 pgs degraded, 20 pgs undersized (PG_DEGRADED) 2023-06-03T20:20:08.417 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:20:08.418 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:20:08.519 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:20:08.550 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:20:08.553 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:20:08.554 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:20:08.655 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:20:08.696 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:20:08.699 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:20:08.700 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:20:08.801 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:20:08.840 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:20:08.844 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:20:08.845 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:20:08.946 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:20:08.995 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:20:08.997 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:20:08.997 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:20:09.075 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/mon.a/config 2023-06-03T20:20:09.075 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/mon.a/config 2023-06-03T20:20:09.099 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:20:09.148 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:20:09.149 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:20:09.150 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:20:09.250 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:20:09.287 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:20:09.288 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:20:09.289 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:20:09.390 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:20:09.421 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:20:09.425 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:20:09.426 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:20:09.527 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:20:09.557 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:20:09.559 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:20:09.560 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:20:09.661 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:20:09.696 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:20:09.700 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:20:09.701 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:20:09.802 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:20:09.832 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:20:09.835 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:20:09.836 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:20:09.936 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:20:09.976 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:20:09.979 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:20:09.979 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:20:10.058 INFO:teuthology.orchestra.run.smithi164.stderr:Inferring config /var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/mon.b/config 2023-06-03T20:20:10.080 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:20:10.126 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:20:10.130 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:20:10.131 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:20:10.232 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:20:10.259 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:20:10.263 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:20:10.264 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:20:10.365 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:20:10.372 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:20:10 smithi089 bash[78452]: cluster 1685823608.924661 mgr.y (mgr.34104) 1326 : cluster 0 pgmap v1326: 99 pgs: 20 active+clean+remapped, 19 active+undersized+degraded, 11 active+undersized, 49 active+clean; 2.4 GiB data, 9.7 GiB used, 527 GiB / 536 GiB avail; 19963/118794 objects degraded (16.805%); 6/118794 objects misplaced (0.005%) 2023-06-03T20:20:10.373 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:20:10 smithi089 bash[75068]: cluster 1685823608.924661 mgr.y (mgr.34104) 1326 : cluster 0 pgmap v1326: 99 pgs: 20 active+clean+remapped, 19 active+undersized+degraded, 11 active+undersized, 49 active+clean; 2.4 GiB data, 9.7 GiB used, 527 GiB / 536 GiB avail; 19963/118794 objects degraded (16.805%); 6/118794 objects misplaced (0.005%) 2023-06-03T20:20:10.396 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:20:10.398 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:20:10.399 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:20:10.500 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:20:10.549 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:20:10.554 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:20:10.555 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:20:10.583 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:20:10 smithi164 bash[59462]: cluster 1685823608.924661 mgr.y (mgr.34104) 1326 : cluster 0 pgmap v1326: 99 pgs: 20 active+clean+remapped, 19 active+undersized+degraded, 11 active+undersized, 49 active+clean; 2.4 GiB data, 9.7 GiB used, 527 GiB / 536 GiB avail; 19963/118794 objects degraded (16.805%); 6/118794 objects misplaced (0.005%) 2023-06-03T20:20:10.656 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:20:10.689 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:20:10.691 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:20:10.691 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:20:10.792 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:20:10.829 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:20:10.843 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:20:10.845 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:20:10.946 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:20:10.977 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:20:10.980 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:20:10.980 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:20:11.081 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:20:11.122 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:20:11.127 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:20:11.127 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:20:11.228 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:20:11.264 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:20:11.266 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:20:11.267 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:20:11.368 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:20:11.401 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:20:11.404 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:20:11.405 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:20:11.507 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:20:11.537 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:20:11.541 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:20:11.542 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:20:11.642 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:20:11.674 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:20:11.676 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:20:11.677 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:20:11.778 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:20:11.812 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:20:11.815 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:20:11.815 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:20:11.916 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:20:11.946 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:20:11.952 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:20:11.953 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:20:12.054 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:20:12.091 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:20:12.095 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:20:12.096 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:20:12.191 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:20:12 smithi164 bash[59462]: cluster 1685823610.925554 mgr.y (mgr.34104) 1327 : cluster 0 pgmap v1327: 99 pgs: 20 active+clean+remapped, 19 active+undersized+degraded, 11 active+undersized, 49 active+clean; 2.2 GiB data, 9.4 GiB used, 527 GiB / 536 GiB avail; 0 B/s wr, 111 op/s; 18107/109491 objects degraded (16.537%); 6/109491 objects misplaced (0.005%) 2023-06-03T20:20:12.197 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:20:12.224 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:20:12.226 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:20:12.227 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:20:12.328 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:20:12.360 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:20:12.364 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:20:12.365 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:20:12.466 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:20:12.473 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:20:12 smithi089 bash[75068]: cluster 1685823610.925554 mgr.y (mgr.34104) 1327 : cluster 0 pgmap v1327: 99 pgs: 20 active+clean+remapped, 19 active+undersized+degraded, 11 active+undersized, 49 active+clean; 2.2 GiB data, 9.4 GiB used, 527 GiB / 536 GiB avail; 0 B/s wr, 111 op/s; 18107/109491 objects degraded (16.537%); 6/109491 objects misplaced (0.005%) 2023-06-03T20:20:12.476 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:20:12 smithi089 bash[78452]: cluster 1685823610.925554 mgr.y (mgr.34104) 1327 : cluster 0 pgmap v1327: 99 pgs: 20 active+clean+remapped, 19 active+undersized+degraded, 11 active+undersized, 49 active+clean; 2.2 GiB data, 9.4 GiB used, 527 GiB / 536 GiB avail; 0 B/s wr, 111 op/s; 18107/109491 objects degraded (16.537%); 6/109491 objects misplaced (0.005%) 2023-06-03T20:20:12.505 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:20:12.512 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:20:12.512 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:20:12.613 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:20:12.648 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:20:12.651 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:20:12.651 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:20:12.752 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:20:12.779 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:20:12.781 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:20:12.782 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:20:12.883 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:20:12.909 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:20:12.912 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:20:12.912 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:20:13.013 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:20:13.052 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:20:13.059 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:20:13.059 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:20:13.160 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:20:13.189 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:20:13.192 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:20:13.193 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:20:13.278 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:20:13 smithi089 bash[75068]: cluster 1685823612.8636475 mon.a (mon.0) 1304 : cluster 3 Health check update: Degraded data redundancy: 18107/109491 objects degraded (16.537%), 19 pgs degraded, 20 pgs undersized (PG_DEGRADED) 2023-06-03T20:20:13.279 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:20:13 smithi089 bash[78452]: cluster 1685823612.8636475 mon.a (mon.0) 1304 : cluster 3 Health check update: Degraded data redundancy: 18107/109491 objects degraded (16.537%), 19 pgs degraded, 20 pgs undersized (PG_DEGRADED) 2023-06-03T20:20:13.294 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:20:13.331 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:20:13.336 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:20:13.337 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:20:13.438 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:20:13.468 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:20:13.472 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:20:13.473 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:20:13.574 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:20:13.583 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:20:13 smithi164 bash[59462]: cluster 1685823612.8636475 mon.a (mon.0) 1304 : cluster 3 Health check update: Degraded data redundancy: 18107/109491 objects degraded (16.537%), 19 pgs degraded, 20 pgs undersized (PG_DEGRADED) 2023-06-03T20:20:13.610 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:20:13.615 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:20:13.616 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:20:13.717 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:20:13.753 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:20:13.757 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:20:13.758 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:20:13.859 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:20:13.895 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:20:13.897 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:20:13.898 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:20:13.999 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:20:14.030 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:20:14.034 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:20:14.035 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:20:14.135 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:20:14.171 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:20:14.174 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:20:14.175 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:20:14.276 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:20:14.311 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:20:14.313 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:20:14.313 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:20:14.411 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:20:14 smithi089 bash[75068]: cluster 1685823612.926437 mgr.y (mgr.34104) 1328 : cluster 0 pgmap v1328: 99 pgs: 20 active+clean+remapped, 19 active+undersized+degraded, 11 active+undersized, 49 active+clean; 2.1 GiB data, 9.2 GiB used, 527 GiB / 536 GiB avail; 0 B/s wr, 94 op/s; 17755/105594 objects degraded (16.814%); 6/105594 objects misplaced (0.006%) 2023-06-03T20:20:14.411 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:20:14 smithi089 bash[78452]: cluster 1685823612.926437 mgr.y (mgr.34104) 1328 : cluster 0 pgmap v1328: 99 pgs: 20 active+clean+remapped, 19 active+undersized+degraded, 11 active+undersized, 49 active+clean; 2.1 GiB data, 9.2 GiB used, 527 GiB / 536 GiB avail; 0 B/s wr, 94 op/s; 17755/105594 objects degraded (16.814%); 6/105594 objects misplaced (0.006%) 2023-06-03T20:20:14.414 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:20:14.452 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:20:14.456 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:20:14.457 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:20:14.558 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:20:14.569 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:20:14 smithi164 bash[59462]: cluster 1685823612.926437 mgr.y (mgr.34104) 1328 : cluster 0 pgmap v1328: 99 pgs: 20 active+clean+remapped, 19 active+undersized+degraded, 11 active+undersized, 49 active+clean; 2.1 GiB data, 9.2 GiB used, 527 GiB / 536 GiB avail; 0 B/s wr, 94 op/s; 17755/105594 objects degraded (16.814%); 6/105594 objects misplaced (0.006%) 2023-06-03T20:20:14.596 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:20:14.603 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:20:14.603 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:20:14.705 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:20:14.745 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:20:14.750 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:20:14.750 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:20:14.821 INFO:teuthology.orchestra.run.smithi164.stdout:ERROR: (22) Invalid argument 2023-06-03T20:20:14.822 INFO:teuthology.orchestra.run.smithi164.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-06-03T20:20:14.851 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:20:14.881 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:20:14.883 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:20:14.883 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:20:14.984 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:20:15.019 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:20:15.021 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:20:15.021 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:20:15.122 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:20:15.161 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:20:15.163 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:20:15.164 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:20:15.265 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:20:15.294 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:20:15.297 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:20:15.298 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:20:15.399 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:20:15.449 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:20:15.451 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:20:15.452 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:20:15.553 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:20:15.593 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:20:15.599 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:20:15.599 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:20:15.700 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:20:15.708 DEBUG:teuthology.orchestra.run.smithi164:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph daemon osd.7 dump_blocked_ops 2023-06-03T20:20:15.741 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:20:15.743 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:20:15.744 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:20:15.845 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:20:15.889 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:20:15.895 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:20:15.896 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:20:15.997 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:20:16.032 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:20:16.036 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:20:16.037 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:20:16.138 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:20:16.167 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:20:16.169 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:20:16.169 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:20:16.270 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:20:16.304 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:20:16.306 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:20:16.307 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:20:16.328 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:20:16 smithi089 bash[78452]: cluster 1685823614.9273605 mgr.y (mgr.34104) 1329 : cluster 0 pgmap v1329: 99 pgs: 20 active+clean+remapped, 19 active+undersized+degraded, 11 active+undersized, 49 active+clean; 2.0 GiB data, 8.8 GiB used, 528 GiB / 536 GiB avail; 0 B/s wr, 1.02k op/s; 16540/98988 objects degraded (16.709%); 6/98988 objects misplaced (0.006%) 2023-06-03T20:20:16.408 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:20:16.421 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:20:16 smithi164 bash[59462]: cluster 1685823614.9273605 mgr.y (mgr.34104) 1329 : cluster 0 pgmap v1329: 99 pgs: 20 active+clean+remapped, 19 active+undersized+degraded, 11 active+undersized, 49 active+clean; 2.0 GiB data, 8.8 GiB used, 528 GiB / 536 GiB avail; 0 B/s wr, 1.02k op/s; 16540/98988 objects degraded (16.709%); 6/98988 objects misplaced (0.006%) 2023-06-03T20:20:16.446 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:20:16.449 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:20:16.450 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:20:16.551 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:20:16.576 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:20:16.578 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:20:16 smithi089 bash[75068]: cluster 1685823614.9273605 mgr.y (mgr.34104) 1329 : cluster 0 pgmap v1329: 99 pgs: 20 active+clean+remapped, 19 active+undersized+degraded, 11 active+undersized, 49 active+clean; 2.0 GiB data, 8.8 GiB used, 528 GiB / 536 GiB avail; 0 B/s wr, 1.02k op/s; 16540/98988 objects degraded (16.709%); 6/98988 objects misplaced (0.006%) 2023-06-03T20:20:16.578 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 20:20:16 smithi089 bash[71163]: ::ffff:172.21.15.164 - - [03/Jun/2023:20:20:16] "GET /metrics HTTP/1.1" 200 228952 "" "Prometheus/2.33.4" 2023-06-03T20:20:16.578 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:20:16.579 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:20:16.680 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:20:16.713 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:20:16.715 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:20:16.716 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:20:16.817 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:20:17.022 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:20:17.026 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:20:17.027 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:20:17.128 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:20:17.164 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:20:17.171 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:20:17.172 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:20:17.272 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:20:17.299 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:20:17.301 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:20:17.302 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:20:17.402 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:20:17.426 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:20:17.428 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:20:17.428 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:20:17.529 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:20:17.562 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:20:17.564 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:20:17.565 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:20:17.665 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:20:17.697 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:20:17.700 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:20:17.700 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:20:17.801 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:20:17.833 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:20:17.835 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:20:17.836 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:20:17.937 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:20:17.967 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:20:17.971 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:20:17.972 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:20:18.073 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:20:18.110 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:20:18.114 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:20:18.114 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:20:18.215 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:20:18.247 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:20:18.249 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:20:18.250 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:20:18.279 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:20:18 smithi089 bash[75068]: cluster 1685823616.9282267 mgr.y (mgr.34104) 1330 : cluster 0 pgmap v1330: 99 pgs: 20 active+clean+remapped, 19 active+undersized+degraded, 11 active+undersized, 49 active+clean; 1.8 GiB data, 8.5 GiB used, 528 GiB / 536 GiB avail; 0 B/s wr, 1.06k op/s; 15301/90603 objects degraded (16.888%); 6/90603 objects misplaced (0.007%) 2023-06-03T20:20:18.279 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:20:18 smithi089 bash[75068]: cluster 1685823617.8646576 mon.a (mon.0) 1305 : cluster 3 Health check update: Degraded data redundancy: 15301/90603 objects degraded (16.888%), 19 pgs degraded, 20 pgs undersized (PG_DEGRADED) 2023-06-03T20:20:18.280 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:20:18 smithi089 bash[78452]: cluster 1685823616.9282267 mgr.y (mgr.34104) 1330 : cluster 0 pgmap v1330: 99 pgs: 20 active+clean+remapped, 19 active+undersized+degraded, 11 active+undersized, 49 active+clean; 1.8 GiB data, 8.5 GiB used, 528 GiB / 536 GiB avail; 0 B/s wr, 1.06k op/s; 15301/90603 objects degraded (16.888%); 6/90603 objects misplaced (0.007%) 2023-06-03T20:20:18.280 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:20:18 smithi089 bash[78452]: cluster 1685823617.8646576 mon.a (mon.0) 1305 : cluster 3 Health check update: Degraded data redundancy: 15301/90603 objects degraded (16.888%), 19 pgs degraded, 20 pgs undersized (PG_DEGRADED) 2023-06-03T20:20:18.351 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:20:18.386 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:20:18.389 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:20:18.390 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:20:18.490 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:20:18.529 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:20:18.530 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:20:18.530 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:20:18.584 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:20:18 smithi164 bash[59462]: cluster 1685823616.9282267 mgr.y (mgr.34104) 1330 : cluster 0 pgmap v1330: 99 pgs: 20 active+clean+remapped, 19 active+undersized+degraded, 11 active+undersized, 49 active+clean; 1.8 GiB data, 8.5 GiB used, 528 GiB / 536 GiB avail; 0 B/s wr, 1.06k op/s; 15301/90603 objects degraded (16.888%); 6/90603 objects misplaced (0.007%) 2023-06-03T20:20:18.584 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:20:18 smithi164 bash[59462]: cluster 1685823617.8646576 mon.a (mon.0) 1305 : cluster 3 Health check update: Degraded data redundancy: 15301/90603 objects degraded (16.888%), 19 pgs degraded, 20 pgs undersized (PG_DEGRADED) 2023-06-03T20:20:18.631 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:20:18.665 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:20:18.669 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:20:18.670 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:20:18.771 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:20:18.801 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:20:18.805 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:20:18.805 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:20:18.906 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:20:18.934 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:20:18.937 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:20:18.937 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:20:19.038 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:20:19.076 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:20:19.078 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:20:19.079 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:20:19.180 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:20:19.211 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:20:19.214 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:20:19.215 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:20:19.316 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:20:19.352 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:20:19.354 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:20:19.355 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:20:19.456 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:20:19.490 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:20:19.493 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:20:19.494 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:20:19.595 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:20:19.636 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:20:19.639 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:20:19.639 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:20:19.740 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:20:19.771 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:20:19.773 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:20:19.774 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:20:19.875 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:20:19.911 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:20:19.913 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:20:19.913 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:20:20.014 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:20:20.052 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:20:20.055 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:20:20.056 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:20:20.157 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:20:20.195 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:20:20.198 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:20:20.199 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:20:20.300 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:20:20.332 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:20:20.333 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:20:20 smithi164 bash[59462]: cluster 1685823618.929021 mgr.y (mgr.34104) 1331 : cluster 0 pgmap v1331: 99 pgs: 20 active+clean+remapped, 19 active+undersized+degraded, 11 active+undersized, 49 active+clean; 1.8 GiB data, 8.2 GiB used, 528 GiB / 536 GiB avail; 0 B/s wr, 882 op/s; 14986/88764 objects degraded (16.883%); 6/88764 objects misplaced (0.007%) 2023-06-03T20:20:20.334 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:20:20.334 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:20:20.436 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:20:20.444 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:20:20 smithi089 bash[75068]: cluster 1685823618.929021 mgr.y (mgr.34104) 1331 : cluster 0 pgmap v1331: 99 pgs: 20 active+clean+remapped, 19 active+undersized+degraded, 11 active+undersized, 49 active+clean; 1.8 GiB data, 8.2 GiB used, 528 GiB / 536 GiB avail; 0 B/s wr, 882 op/s; 14986/88764 objects degraded (16.883%); 6/88764 objects misplaced (0.007%) 2023-06-03T20:20:20.444 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:20:20 smithi089 bash[78452]: cluster 1685823618.929021 mgr.y (mgr.34104) 1331 : cluster 0 pgmap v1331: 99 pgs: 20 active+clean+remapped, 19 active+undersized+degraded, 11 active+undersized, 49 active+clean; 1.8 GiB data, 8.2 GiB used, 528 GiB / 536 GiB avail; 0 B/s wr, 882 op/s; 14986/88764 objects degraded (16.883%); 6/88764 objects misplaced (0.007%) 2023-06-03T20:20:20.470 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:20:20.474 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:20:20.475 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:20:20.559 INFO:teuthology.orchestra.run.smithi164.stderr:Inferring config /var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/mon.b/config 2023-06-03T20:20:20.576 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:20:20.615 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:20:20.619 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:20:20.619 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:20:20.720 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:20:20.750 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:20:20.752 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:20:20.753 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:20:20.854 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:20:20.887 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:20:20.889 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:20:20.889 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:20:20.990 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:20:21.021 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:20:21.024 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:20:21.024 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:20:21.125 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:20:21.160 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:20:21.166 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:20:21.166 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:20:21.267 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:20:21.302 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:20:21.304 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:20:21.305 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:20:21.406 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:20:21.433 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:20:21.437 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:20:21.438 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:20:21.539 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:20:21.561 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:20:21.564 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:20:21.564 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:20:21.665 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:20:21.697 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:20:21.700 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:20:21.700 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:20:21.801 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:20:21.837 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:20:21.840 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:20:21.841 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:20:21.942 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:20:21.980 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:20:21.982 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:20:21.983 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:20:22.084 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:20:22.113 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:20:22.115 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:20:22.116 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:20:22.217 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:20:22.225 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:20:22 smithi089 bash[75068]: cluster 1685823620.9298236 mgr.y (mgr.34104) 1332 : cluster 0 pgmap v1332: 99 pgs: 20 active+clean+remapped, 19 active+undersized+degraded, 11 active+undersized, 49 active+clean; 1.6 GiB data, 7.9 GiB used, 529 GiB / 536 GiB avail; 0 B/s wr, 1.08k op/s; 13220/79989 objects degraded (16.527%); 6/79989 objects misplaced (0.008%) 2023-06-03T20:20:22.225 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:20:22 smithi089 bash[75068]: audit 1685823622.1320827 mon.c (mon.2) 574 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T20:20:22.225 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:20:22 smithi089 bash[78452]: cluster 1685823620.9298236 mgr.y (mgr.34104) 1332 : cluster 0 pgmap v1332: 99 pgs: 20 active+clean+remapped, 19 active+undersized+degraded, 11 active+undersized, 49 active+clean; 1.6 GiB data, 7.9 GiB used, 529 GiB / 536 GiB avail; 0 B/s wr, 1.08k op/s; 13220/79989 objects degraded (16.527%); 6/79989 objects misplaced (0.008%) 2023-06-03T20:20:22.225 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:20:22 smithi089 bash[78452]: audit 1685823622.1320827 mon.c (mon.2) 574 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T20:20:22.242 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:20:22.244 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:20:22.244 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:20:22.346 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:20:22.358 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:20:22 smithi164 bash[59462]: cluster 1685823620.9298236 mgr.y (mgr.34104) 1332 : cluster 0 pgmap v1332: 99 pgs: 20 active+clean+remapped, 19 active+undersized+degraded, 11 active+undersized, 49 active+clean; 1.6 GiB data, 7.9 GiB used, 529 GiB / 536 GiB avail; 0 B/s wr, 1.08k op/s; 13220/79989 objects degraded (16.527%); 6/79989 objects misplaced (0.008%) 2023-06-03T20:20:22.358 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:20:22 smithi164 bash[59462]: audit 1685823622.1320827 mon.c (mon.2) 574 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T20:20:22.380 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:20:22.386 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:20:22.386 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:20:22.487 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:20:22.523 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:20:22.526 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:20:22.527 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:20:22.628 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:20:22.669 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:20:22.674 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:20:22.674 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:20:22.775 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:20:22.813 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:20:22.815 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:20:22.815 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:20:22.916 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:20:22.950 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:20:22.955 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:20:22.955 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:20:23.056 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:20:23.092 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:20:23.093 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:20:23.093 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:20:23.194 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:20:23.200 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:20:23 smithi089 bash[75068]: cluster 1685823622.8657546 mon.a (mon.0) 1306 : cluster 3 Health check update: Degraded data redundancy: 13220/79989 objects degraded (16.527%), 19 pgs degraded, 20 pgs undersized (PG_DEGRADED) 2023-06-03T20:20:23.224 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:20:23.227 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:20:23.227 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:20:23.328 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:20:23.376 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:20:23.380 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:20:23.381 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:20:23.475 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:20:23 smithi164 bash[59462]: cluster 1685823622.8657546 mon.a (mon.0) 1306 : cluster 3 Health check update: Degraded data redundancy: 13220/79989 objects degraded (16.527%), 19 pgs degraded, 20 pgs undersized (PG_DEGRADED) 2023-06-03T20:20:23.483 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:20:23.504 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:20:23.505 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:20:23.505 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:20:23.606 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:20:23.633 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:20:23.635 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:20:23.636 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:20:23.671 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:20:23 smithi089 bash[78452]: cluster 1685823622.8657546 mon.a (mon.0) 1306 : cluster 3 Health check update: Degraded data redundancy: 13220/79989 objects degraded (16.527%), 19 pgs degraded, 20 pgs undersized (PG_DEGRADED) 2023-06-03T20:20:23.737 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:20:23.763 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:20:23.765 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:20:23.766 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:20:23.867 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:20:23.906 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:20:23.908 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:20:23.909 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:20:24.010 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:20:24.043 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:20:24.045 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:20:24.046 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:20:24.147 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:20:24.167 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:20:24.168 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:20:24.169 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:20:24.270 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:20:24.303 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:20:24.307 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:20:24.307 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:20:24.334 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:20:24 smithi164 bash[59462]: cluster 1685823622.9334042 mgr.y (mgr.34104) 1333 : cluster 0 pgmap v1333: 99 pgs: 20 active+clean+remapped, 19 active+undersized+degraded, 11 active+undersized, 49 active+clean; 1.6 GiB data, 7.8 GiB used, 529 GiB / 536 GiB avail; 0 B/s wr, 913 op/s; 12926/76584 objects degraded (16.878%); 6/76584 objects misplaced (0.008%) 2023-06-03T20:20:24.408 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:20:24.435 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:20:24.441 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:20:24.442 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:20:24.529 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:20:24 smithi089 bash[75068]: cluster 1685823622.9334042 mgr.y (mgr.34104) 1333 : cluster 0 pgmap v1333: 99 pgs: 20 active+clean+remapped, 19 active+undersized+degraded, 11 active+undersized, 49 active+clean; 1.6 GiB data, 7.8 GiB used, 529 GiB / 536 GiB avail; 0 B/s wr, 913 op/s; 12926/76584 objects degraded (16.878%); 6/76584 objects misplaced (0.008%) 2023-06-03T20:20:24.529 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:20:24 smithi089 bash[78452]: cluster 1685823622.9334042 mgr.y (mgr.34104) 1333 : cluster 0 pgmap v1333: 99 pgs: 20 active+clean+remapped, 19 active+undersized+degraded, 11 active+undersized, 49 active+clean; 1.6 GiB data, 7.8 GiB used, 529 GiB / 536 GiB avail; 0 B/s wr, 913 op/s; 12926/76584 objects degraded (16.878%); 6/76584 objects misplaced (0.008%) 2023-06-03T20:20:24.543 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:20:24.577 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:20:24.581 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:20:24.582 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:20:24.682 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:20:24.712 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:20:24.714 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:20:24.715 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:20:24.816 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:20:24.856 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:20:24.858 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:20:24.859 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:20:24.960 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:20:24.994 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:20:24.997 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:20:24.998 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:20:25.099 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:20:25.157 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:20:25.161 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:20:25.161 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:20:25.181 INFO:teuthology.orchestra.run.smithi164.stdout:ERROR: (22) Invalid argument 2023-06-03T20:20:25.182 INFO:teuthology.orchestra.run.smithi164.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-06-03T20:20:25.225 INFO:teuthology.orchestra.run.smithi089.stderr:nodeep-scrub is set 2023-06-03T20:20:25.262 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:20:25.293 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:20:25.296 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:20:25.297 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:20:25.398 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:20:25.407 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:20:25 smithi089 bash[75068]: audit 1685823625.1770644 mon.c (mon.2) 575 : audit 1 from='client.? 172.21.15.89:0/2372572754' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-06-03T20:20:25.407 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:20:25 smithi089 bash[75068]: audit 1685823625.177948 mon.a (mon.0) 1307 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-06-03T20:20:25.407 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:20:25 smithi089 bash[78452]: audit 1685823625.1770644 mon.c (mon.2) 575 : audit 1 from='client.? 172.21.15.89:0/2372572754' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-06-03T20:20:25.409 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:20:25 smithi089 bash[78452]: audit 1685823625.177948 mon.a (mon.0) 1307 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-06-03T20:20:25.438 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:20:25.444 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:20:25.445 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:20:25.472 INFO:teuthology.orchestra.run.smithi089.stdout:osd.0: {} 2023-06-03T20:20:25.514 INFO:teuthology.orchestra.run.smithi089.stdout:osd.1: {} 2023-06-03T20:20:25.546 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:20:25.564 INFO:teuthology.orchestra.run.smithi089.stdout:osd.2: {} 2023-06-03T20:20:25.583 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:20:25 smithi164 bash[59462]: audit 1685823625.1770644 mon.c (mon.2) 575 : audit 1 from='client.? 172.21.15.89:0/2372572754' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-06-03T20:20:25.584 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:20:25 smithi164 bash[59462]: audit 1685823625.177948 mon.a (mon.0) 1307 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-06-03T20:20:25.586 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:20:25.589 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:20:25.590 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:20:25.625 INFO:teuthology.orchestra.run.smithi089.stdout:osd.3: {} 2023-06-03T20:20:25.659 INFO:teuthology.orchestra.run.smithi089.stdout:osd.5: {} 2023-06-03T20:20:25.691 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:20:25.692 INFO:teuthology.orchestra.run.smithi089.stdout:osd.6: {} 2023-06-03T20:20:25.729 INFO:teuthology.orchestra.run.smithi089.stdout:osd.7: {} 2023-06-03T20:20:25.730 INFO:teuthology.orchestra.run.smithi089.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-06-03T20:20:25.730 INFO:teuthology.orchestra.run.smithi089.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-06-03T20:20:25.730 INFO:teuthology.orchestra.run.smithi089.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-06-03T20:20:25.730 INFO:teuthology.orchestra.run.smithi089.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-06-03T20:20:25.730 INFO:teuthology.orchestra.run.smithi089.stderr:Error ENXIO: problem getting command descriptions from osd.4 2023-06-03T20:20:25.730 INFO:teuthology.orchestra.run.smithi089.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-06-03T20:20:25.731 INFO:teuthology.orchestra.run.smithi089.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-06-03T20:20:25.731 INFO:teuthology.orchestra.run.smithi089.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-06-03T20:20:25.731 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:20:25.733 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:20:25.734 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:20:25.801 INFO:teuthology.orchestra.run.smithi089.stderr:Error ENOENT: unrecognized pool 'device_health_metrics' 2023-06-03T20:20:25.835 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:20:25.871 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:20:25.876 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:20:25.877 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:20:25.977 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:20:26.003 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:20:26.006 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:20:26.006 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:20:26.107 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:20:26.139 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:20:26.142 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:20:26.142 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:20:26.188 DEBUG:teuthology.orchestra.run.smithi164:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph daemon osd.7 dump_historic_ops 2023-06-03T20:20:26.243 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:20:26.254 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:20:26 smithi164 bash[59462]: cluster 1685823624.9342735 mgr.y (mgr.34104) 1334 : cluster 0 pgmap v1334: 99 pgs: 20 active+clean+remapped, 19 active+undersized+degraded, 11 active+undersized, 49 active+clean; 1.4 GiB data, 7.4 GiB used, 529 GiB / 536 GiB avail; 0 B/s wr, 965 op/s; 11820/70824 objects degraded (16.689%); 6/70824 objects misplaced (0.008%) 2023-06-03T20:20:26.254 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:20:26 smithi164 bash[59462]: cluster 1685823625.1971452 mon.a (mon.0) 1308 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-06-03T20:20:26.254 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:20:26 smithi164 bash[59462]: cluster 1685823625.2041986 osd.6 (osd.6) 176 : cluster 3 failed to encode map e233 with expected crc 2023-06-03T20:20:26.254 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:20:26 smithi164 bash[59462]: audit 1685823625.2137916 mon.a (mon.0) 1309 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-06-03T20:20:26.254 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:20:26 smithi164 bash[59462]: cluster 1685823625.214042 mon.a (mon.0) 1310 : cluster 0 osdmap e233: 8 total, 7 up, 6 in 2023-06-03T20:20:26.255 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:20:26 smithi164 bash[59462]: cluster 1685823625.3230681 osd.7 (osd.7) 158 : cluster 3 failed to encode map e233 with expected crc 2023-06-03T20:20:26.255 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:20:26 smithi164 bash[59462]: cluster 1685823625.3238425 osd.5 (osd.5) 169 : cluster 3 failed to encode map e233 with expected crc 2023-06-03T20:20:26.255 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:20:26 smithi164 bash[59462]: cluster 1685823625.3253908 osd.5 (osd.5) 170 : cluster 3 failed to encode map e233 with expected crc 2023-06-03T20:20:26.255 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:20:26 smithi164 bash[59462]: audit 1685823625.4217951 mon.b (mon.1) 61 : audit 0 from='client.? 172.21.15.89:0/3003959519' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-03T20:20:26.255 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:20:26 smithi164 bash[59462]: audit 1685823625.793164 mon.a (mon.0) 1311 : audit 1 from='client.? 172.21.15.89:0/1020870104' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "device_health_metrics", "var": "pgp_num", "val": "11"}]: dispatch 2023-06-03T20:20:26.272 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:20:26.276 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:20:26.277 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:20:26.279 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:20:26 smithi089 bash[75068]: cluster 1685823624.9342735 mgr.y (mgr.34104) 1334 : cluster 0 pgmap v1334: 99 pgs: 20 active+clean+remapped, 19 active+undersized+degraded, 11 active+undersized, 49 active+clean; 1.4 GiB data, 7.4 GiB used, 529 GiB / 536 GiB avail; 0 B/s wr, 965 op/s; 11820/70824 objects degraded (16.689%); 6/70824 objects misplaced (0.008%) 2023-06-03T20:20:26.279 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:20:26 smithi089 bash[75068]: cluster 1685823625.1971452 mon.a (mon.0) 1308 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-06-03T20:20:26.279 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:20:26 smithi089 bash[75068]: cluster 1685823625.2041986 osd.6 (osd.6) 176 : cluster 3 failed to encode map e233 with expected crc 2023-06-03T20:20:26.279 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:20:26 smithi089 bash[75068]: audit 1685823625.2137916 mon.a (mon.0) 1309 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-06-03T20:20:26.279 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:20:26 smithi089 bash[75068]: cluster 1685823625.214042 mon.a (mon.0) 1310 : cluster 0 osdmap e233: 8 total, 7 up, 6 in 2023-06-03T20:20:26.280 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:20:26 smithi089 bash[75068]: cluster 1685823625.3230681 osd.7 (osd.7) 158 : cluster 3 failed to encode map e233 with expected crc 2023-06-03T20:20:26.280 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:20:26 smithi089 bash[75068]: cluster 1685823625.3238425 osd.5 (osd.5) 169 : cluster 3 failed to encode map e233 with expected crc 2023-06-03T20:20:26.280 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:20:26 smithi089 bash[75068]: cluster 1685823625.3253908 osd.5 (osd.5) 170 : cluster 3 failed to encode map e233 with expected crc 2023-06-03T20:20:26.280 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:20:26 smithi089 bash[75068]: audit 1685823625.4217951 mon.b (mon.1) 61 : audit 0 from='client.? 172.21.15.89:0/3003959519' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-03T20:20:26.280 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:20:26 smithi089 bash[75068]: audit 1685823625.793164 mon.a (mon.0) 1311 : audit 1 from='client.? 172.21.15.89:0/1020870104' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "device_health_metrics", "var": "pgp_num", "val": "11"}]: dispatch 2023-06-03T20:20:26.281 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 20:20:26 smithi089 bash[71163]: ::ffff:172.21.15.164 - - [03/Jun/2023:20:20:26] "GET /metrics HTTP/1.1" 200 228944 "" "Prometheus/2.33.4" 2023-06-03T20:20:26.281 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:20:26 smithi089 bash[78452]: cluster 1685823624.9342735 mgr.y (mgr.34104) 1334 : cluster 0 pgmap v1334: 99 pgs: 20 active+clean+remapped, 19 active+undersized+degraded, 11 active+undersized, 49 active+clean; 1.4 GiB data, 7.4 GiB used, 529 GiB / 536 GiB avail; 0 B/s wr, 965 op/s; 11820/70824 objects degraded (16.689%); 6/70824 objects misplaced (0.008%) 2023-06-03T20:20:26.281 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:20:26 smithi089 bash[78452]: cluster 1685823625.1971452 mon.a (mon.0) 1308 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-06-03T20:20:26.281 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:20:26 smithi089 bash[78452]: cluster 1685823625.2041986 osd.6 (osd.6) 176 : cluster 3 failed to encode map e233 with expected crc 2023-06-03T20:20:26.281 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:20:26 smithi089 bash[78452]: audit 1685823625.2137916 mon.a (mon.0) 1309 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-06-03T20:20:26.281 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:20:26 smithi089 bash[78452]: cluster 1685823625.214042 mon.a (mon.0) 1310 : cluster 0 osdmap e233: 8 total, 7 up, 6 in 2023-06-03T20:20:26.282 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:20:26 smithi089 bash[78452]: cluster 1685823625.3230681 osd.7 (osd.7) 158 : cluster 3 failed to encode map e233 with expected crc 2023-06-03T20:20:26.282 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:20:26 smithi089 bash[78452]: cluster 1685823625.3238425 osd.5 (osd.5) 169 : cluster 3 failed to encode map e233 with expected crc 2023-06-03T20:20:26.282 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:20:26 smithi089 bash[78452]: cluster 1685823625.3253908 osd.5 (osd.5) 170 : cluster 3 failed to encode map e233 with expected crc 2023-06-03T20:20:26.282 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:20:26 smithi089 bash[78452]: audit 1685823625.4217951 mon.b (mon.1) 61 : audit 0 from='client.? 172.21.15.89:0/3003959519' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-03T20:20:26.282 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:20:26 smithi089 bash[78452]: audit 1685823625.793164 mon.a (mon.0) 1311 : audit 1 from='client.? 172.21.15.89:0/1020870104' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "device_health_metrics", "var": "pgp_num", "val": "11"}]: dispatch 2023-06-03T20:20:26.378 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:20:26.412 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:20:26.416 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:20:26.417 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:20:26.518 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:20:26.550 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:20:26.552 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:20:26.552 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:20:26.653 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:20:26.687 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:20:26.691 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:20:26.691 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:20:26.792 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:20:26.820 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:20:26.826 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:20:26.826 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:20:26.927 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:20:26.958 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:20:26.960 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:20:26.960 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:20:27.061 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:20:27.094 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:20:27.102 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:20:27.103 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:20:27.203 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:20:27.244 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:20:27.244 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:20:27.245 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:20:27.346 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:20:27.381 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:20:27.386 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:20:27.386 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:20:27.487 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:20:27.520 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:20:27.524 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:20:27.524 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:20:27.625 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:20:27.664 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:20:27.672 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:20:27.673 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:20:27.774 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:20:27.806 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:20:27.808 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:20:27.808 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:20:27.909 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:20:27.946 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:20:27.952 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:20:27.953 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:20:28.053 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:20:28.086 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:20:28.091 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:20:28.091 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:20:28.192 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:20:28.214 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:20:28 smithi089 bash[75068]: cluster 1685823626.9355361 mgr.y (mgr.34104) 1335 : cluster 0 pgmap v1336: 99 pgs: 20 active+clean+remapped, 19 active+undersized+degraded, 11 active+undersized, 49 active+clean; 1.3 GiB data, 7.1 GiB used, 529 GiB / 536 GiB avail; 0 B/s wr, 888 op/s; 10856/63942 objects degraded (16.978%); 6/63942 objects misplaced (0.009%) 2023-06-03T20:20:28.215 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:20:28 smithi089 bash[75068]: cluster 1685823627.8667316 mon.a (mon.0) 1312 : cluster 3 Health check update: Degraded data redundancy: 10856/63942 objects degraded (16.978%), 19 pgs degraded, 20 pgs undersized (PG_DEGRADED) 2023-06-03T20:20:28.224 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:20:28.226 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:20:28.227 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:20:28.328 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:20:28.360 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:20:28.364 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:20:28.364 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:20:28.465 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:20:28.475 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:20:28 smithi164 bash[59462]: cluster 1685823626.9355361 mgr.y (mgr.34104) 1335 : cluster 0 pgmap v1336: 99 pgs: 20 active+clean+remapped, 19 active+undersized+degraded, 11 active+undersized, 49 active+clean; 1.3 GiB data, 7.1 GiB used, 529 GiB / 536 GiB avail; 0 B/s wr, 888 op/s; 10856/63942 objects degraded (16.978%); 6/63942 objects misplaced (0.009%) 2023-06-03T20:20:28.475 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:20:28 smithi164 bash[59462]: cluster 1685823627.8667316 mon.a (mon.0) 1312 : cluster 3 Health check update: Degraded data redundancy: 10856/63942 objects degraded (16.978%), 19 pgs degraded, 20 pgs undersized (PG_DEGRADED) 2023-06-03T20:20:28.495 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:20:28.498 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:20:28.498 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:20:28.529 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:20:28 smithi089 bash[78452]: cluster 1685823626.9355361 mgr.y (mgr.34104) 1335 : cluster 0 pgmap v1336: 99 pgs: 20 active+clean+remapped, 19 active+undersized+degraded, 11 active+undersized, 49 active+clean; 1.3 GiB data, 7.1 GiB used, 529 GiB / 536 GiB avail; 0 B/s wr, 888 op/s; 10856/63942 objects degraded (16.978%); 6/63942 objects misplaced (0.009%) 2023-06-03T20:20:28.529 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:20:28 smithi089 bash[78452]: cluster 1685823627.8667316 mon.a (mon.0) 1312 : cluster 3 Health check update: Degraded data redundancy: 10856/63942 objects degraded (16.978%), 19 pgs degraded, 20 pgs undersized (PG_DEGRADED) 2023-06-03T20:20:28.599 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:20:28.632 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:20:28.635 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:20:28.635 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:20:28.736 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:20:28.769 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:20:28.776 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:20:28.776 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:20:28.877 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:20:28.912 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:20:28.914 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:20:28.914 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:20:29.015 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:20:29.053 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:20:29.056 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:20:29.057 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:20:29.158 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:20:29.186 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:20:29.188 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:20:29.189 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:20:29.290 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:20:29.325 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:20:29.331 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:20:29.332 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:20:29.432 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:20:29.467 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:20:29.471 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:20:29.472 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:20:29.573 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:20:29.624 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:20:29.627 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:20:29.628 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:20:29.729 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:20:29.759 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:20:29.762 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:20:29.763 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:20:29.863 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:20:29.897 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:20:29.901 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:20:29.902 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:20:29.929 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-06-03T20:20:29.930 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-06-03T20:20:29.957 DEBUG:teuthology.orchestra.run:got remote process result: 2 2023-06-03T20:20:30.003 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:20:30.032 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:20:30.035 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:20:30.036 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:20:30.137 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:20:30.179 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:20:30.183 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:20:30.184 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:20:30.279 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:20:30 smithi089 bash[78452]: cluster 1685823628.936409 mgr.y (mgr.34104) 1336 : cluster 0 pgmap v1337: 99 pgs: 20 active+clean+remapped, 19 active+undersized+degraded, 11 active+undersized, 49 active+clean; 1.3 GiB data, 6.9 GiB used, 530 GiB / 536 GiB avail; 0 B/s wr, 876 op/s; 10601/62448 objects degraded (16.976%); 6/62448 objects misplaced (0.010%) 2023-06-03T20:20:30.279 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:20:30 smithi089 bash[75068]: cluster 1685823628.936409 mgr.y (mgr.34104) 1336 : cluster 0 pgmap v1337: 99 pgs: 20 active+clean+remapped, 19 active+undersized+degraded, 11 active+undersized, 49 active+clean; 1.3 GiB data, 6.9 GiB used, 530 GiB / 536 GiB avail; 0 B/s wr, 876 op/s; 10601/62448 objects degraded (16.976%); 6/62448 objects misplaced (0.010%) 2023-06-03T20:20:30.285 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:20:30.339 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:20:30.342 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:20:30.343 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:20:30.444 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:20:30.480 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:20:30.482 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:20:30.483 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:20:30.539 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph osd unset noscrub 2023-06-03T20:20:30.583 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:20:30.584 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:20:30 smithi164 bash[59462]: cluster 1685823628.936409 mgr.y (mgr.34104) 1336 : cluster 0 pgmap v1337: 99 pgs: 20 active+clean+remapped, 19 active+undersized+degraded, 11 active+undersized, 49 active+clean; 1.3 GiB data, 6.9 GiB used, 530 GiB / 536 GiB avail; 0 B/s wr, 876 op/s; 10601/62448 objects degraded (16.976%); 6/62448 objects misplaced (0.010%) 2023-06-03T20:20:30.631 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:20:30.635 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:20:30.635 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:20:30.736 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:20:30.781 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:20:30.784 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:20:30.784 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:20:30.885 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:20:30.918 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:20:30.921 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:20:30.921 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:20:31.011 INFO:teuthology.orchestra.run.smithi164.stderr:Inferring config /var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/mon.b/config 2023-06-03T20:20:31.022 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:20:31.058 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:20:31.062 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:20:31.062 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:20:31.163 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:20:31.192 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:20:31.195 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:20:31.195 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:20:31.296 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:20:31.333 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:20:31.340 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:20:31.341 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:20:31.442 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:20:31.471 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:20:31.475 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:20:31.476 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:20:31.577 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:20:31.608 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:20:31.610 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:20:31.611 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:20:31.712 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:20:31.746 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:20:31.751 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:20:31.752 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:20:31.853 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:20:31.882 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:20:31.885 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:20:31.886 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:20:31.931 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-06-03T20:20:31.987 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:20:32.029 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:20:32.032 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:20:32.033 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:20:32.134 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:20:32.180 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:20:32.183 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:20:32.184 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:20:32.279 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:20:32 smithi089 bash[75068]: cluster 1685823630.937399 mgr.y (mgr.34104) 1337 : cluster 0 pgmap v1338: 99 pgs: 20 active+clean+remapped, 19 active+undersized+degraded, 11 active+undersized, 49 active+clean; 1.1 GiB data, 6.6 GiB used, 530 GiB / 536 GiB avail; 0 B/s wr, 884 op/s; 8795/53421 objects degraded (16.464%); 6/53421 objects misplaced (0.011%) 2023-06-03T20:20:32.280 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:20:32 smithi089 bash[78452]: cluster 1685823630.937399 mgr.y (mgr.34104) 1337 : cluster 0 pgmap v1338: 99 pgs: 20 active+clean+remapped, 19 active+undersized+degraded, 11 active+undersized, 49 active+clean; 1.1 GiB data, 6.6 GiB used, 530 GiB / 536 GiB avail; 0 B/s wr, 884 op/s; 8795/53421 objects degraded (16.464%); 6/53421 objects misplaced (0.011%) 2023-06-03T20:20:32.286 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:20:32.330 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:20:32.333 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:20:32 smithi164 bash[59462]: cluster 1685823630.937399 mgr.y (mgr.34104) 1337 : cluster 0 pgmap v1338: 99 pgs: 20 active+clean+remapped, 19 active+undersized+degraded, 11 active+undersized, 49 active+clean; 1.1 GiB data, 6.6 GiB used, 530 GiB / 536 GiB avail; 0 B/s wr, 884 op/s; 8795/53421 objects degraded (16.464%); 6/53421 objects misplaced (0.011%) 2023-06-03T20:20:32.336 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:20:32.336 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:20:32.437 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:20:32.475 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:20:32.479 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:20:32.480 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:20:32.581 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:20:32.615 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:20:32.619 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:20:32.620 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:20:32.721 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:20:32.753 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:20:32.758 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:20:32.758 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:20:32.859 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:20:32.912 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:20:32.917 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:20:32.918 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:20:33.019 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:20:33.052 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:20:33.057 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:20:33.058 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:20:33.159 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:20:33.188 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:20:33.190 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:20:33.191 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:20:33.225 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:20:33 smithi089 bash[78452]: cluster 1685823632.870134 mon.a (mon.0) 1313 : cluster 3 Health check update: Degraded data redundancy: 8795/53421 objects degraded (16.464%), 19 pgs degraded, 30 pgs undersized (PG_DEGRADED) 2023-06-03T20:20:33.225 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:20:33 smithi089 bash[75068]: cluster 1685823632.870134 mon.a (mon.0) 1313 : cluster 3 Health check update: Degraded data redundancy: 8795/53421 objects degraded (16.464%), 19 pgs degraded, 30 pgs undersized (PG_DEGRADED) 2023-06-03T20:20:33.292 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:20:33.337 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:20:33.342 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:20:33.343 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:20:33.444 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:20:33.477 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:20:33.483 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:20:33.484 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:20:33.584 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:20:33 smithi164 bash[59462]: cluster 1685823632.870134 mon.a (mon.0) 1313 : cluster 3 Health check update: Degraded data redundancy: 8795/53421 objects degraded (16.464%), 19 pgs degraded, 30 pgs undersized (PG_DEGRADED) 2023-06-03T20:20:33.584 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:20:33.619 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:20:33.622 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:20:33.623 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:20:33.724 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:20:33.753 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:20:33.756 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:20:33.757 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:20:33.858 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:20:33.891 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:20:33.894 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:20:33.894 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:20:33.995 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:20:34.042 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:20:34.046 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:20:34.047 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:20:34.148 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:20:34.178 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:20:34.184 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:20:34.185 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:20:34.279 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:20:34 smithi089 bash[78452]: cluster 1685823632.9382715 mgr.y (mgr.34104) 1338 : cluster 0 pgmap v1339: 99 pgs: 20 active+clean+remapped, 19 active+undersized+degraded, 11 active+undersized, 49 active+clean; 1.0 GiB data, 6.4 GiB used, 530 GiB / 536 GiB avail; 0 B/s wr, 890 op/s; 8500/49854 objects degraded (17.050%); 6/49854 objects misplaced (0.012%) 2023-06-03T20:20:34.279 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:20:34 smithi089 bash[75068]: cluster 1685823632.9382715 mgr.y (mgr.34104) 1338 : cluster 0 pgmap v1339: 99 pgs: 20 active+clean+remapped, 19 active+undersized+degraded, 11 active+undersized, 49 active+clean; 1.0 GiB data, 6.4 GiB used, 530 GiB / 536 GiB avail; 0 B/s wr, 890 op/s; 8500/49854 objects degraded (17.050%); 6/49854 objects misplaced (0.012%) 2023-06-03T20:20:34.286 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:20:34.313 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:20:34.316 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:20:34 smithi164 bash[59462]: cluster 1685823632.9382715 mgr.y (mgr.34104) 1338 : cluster 0 pgmap v1339: 99 pgs: 20 active+clean+remapped, 19 active+undersized+degraded, 11 active+undersized, 49 active+clean; 1.0 GiB data, 6.4 GiB used, 530 GiB / 536 GiB avail; 0 B/s wr, 890 op/s; 8500/49854 objects degraded (17.050%); 6/49854 objects misplaced (0.012%) 2023-06-03T20:20:34.317 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:20:34.317 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:20:34.418 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:20:34.455 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:20:34.461 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:20:34.462 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:20:34.563 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:20:34.592 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:20:34.594 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:20:34.595 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:20:34.695 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:20:34.733 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:20:34.740 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:20:34.741 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:20:34.842 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:20:34.881 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:20:34.884 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:20:34.884 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:20:34.959 INFO:tasks.thrashosds.thrasher:in_osds: [0, 2, 3, 4, 5, 7] out_osds: [1, 6] dead_osds: [4] live_osds: [0, 1, 2, 3, 5, 6, 7] 2023-06-03T20:20:34.960 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-06-03T20:20:34.960 INFO:tasks.thrashosds.thrasher:Adding osd 1 2023-06-03T20:20:34.960 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph osd in 1 2023-06-03T20:20:34.985 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:20:35.015 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:20:35.017 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:20:35.018 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:20:35.119 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:20:35.169 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:20:35.171 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:20:35.172 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:20:35.276 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:20:35.281 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:20:35 smithi089 bash[75068]: cluster 1685823634.9391243 mgr.y (mgr.34104) 1339 : cluster 0 pgmap v1340: 99 pgs: 20 active+clean+remapped, 19 active+undersized+degraded, 11 active+undersized, 49 active+clean; 908 MiB data, 6.0 GiB used, 530 GiB / 536 GiB avail; 0 B/s wr, 905 op/s; 7278/43638 objects degraded (16.678%); 6/43638 objects misplaced (0.014%) 2023-06-03T20:20:35.282 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:20:35 smithi089 bash[78452]: cluster 1685823634.9391243 mgr.y (mgr.34104) 1339 : cluster 0 pgmap v1340: 99 pgs: 20 active+clean+remapped, 19 active+undersized+degraded, 11 active+undersized, 49 active+clean; 908 MiB data, 6.0 GiB used, 530 GiB / 536 GiB avail; 0 B/s wr, 905 op/s; 7278/43638 objects degraded (16.678%); 6/43638 objects misplaced (0.014%) 2023-06-03T20:20:35.323 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:20:35.327 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:20:35.328 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:20:35.420 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/mon.a/config 2023-06-03T20:20:35.429 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:20:35.463 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:20:35.468 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:20:35.469 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:20:35.539 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:20:35 smithi164 bash[59462]: cluster 1685823634.9391243 mgr.y (mgr.34104) 1339 : cluster 0 pgmap v1340: 99 pgs: 20 active+clean+remapped, 19 active+undersized+degraded, 11 active+undersized, 49 active+clean; 908 MiB data, 6.0 GiB used, 530 GiB / 536 GiB avail; 0 B/s wr, 905 op/s; 7278/43638 objects degraded (16.678%); 6/43638 objects misplaced (0.014%) 2023-06-03T20:20:35.569 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:20:35.615 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:20:35.616 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:20:35.616 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:20:35.717 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:20:35.723 INFO:teuthology.orchestra.run.smithi164.stdout:{ 2023-06-03T20:20:35.724 INFO:teuthology.orchestra.run.smithi164.stdout: "size": 20, 2023-06-03T20:20:35.724 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 600, 2023-06-03T20:20:35.724 INFO:teuthology.orchestra.run.smithi164.stdout: "ops": [ 2023-06-03T20:20:35.724 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:20:35.724 INFO:teuthology.orchestra.run.smithi164.stdout: "description": "osd_op(client.46187.0:60479 11.5 11:a23097f9:::benchmark_data_smithi089_165412_object60478:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e197)", 2023-06-03T20:20:35.724 INFO:teuthology.orchestra.run.smithi164.stdout: "initiated_at": "2023-06-03T20:12:10.691874+0000", 2023-06-03T20:20:35.724 INFO:teuthology.orchestra.run.smithi164.stdout: "age": 505.02124510599998, 2023-06-03T20:20:35.725 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.29931533300000002, 2023-06-03T20:20:35.725 INFO:teuthology.orchestra.run.smithi164.stdout: "type_data": { 2023-06-03T20:20:35.725 INFO:teuthology.orchestra.run.smithi164.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-03T20:20:35.725 INFO:teuthology.orchestra.run.smithi164.stdout: "client_info": { 2023-06-03T20:20:35.725 INFO:teuthology.orchestra.run.smithi164.stdout: "client": "client.46187", 2023-06-03T20:20:35.725 INFO:teuthology.orchestra.run.smithi164.stdout: "client_addr": "172.21.15.89:0/880684210", 2023-06-03T20:20:35.725 INFO:teuthology.orchestra.run.smithi164.stdout: "tid": 60479 2023-06-03T20:20:35.726 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:20:35.726 INFO:teuthology.orchestra.run.smithi164.stdout: "events": [ 2023-06-03T20:20:35.726 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:20:35.726 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "initiated", 2023-06-03T20:20:35.726 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:12:10.691874+0000", 2023-06-03T20:20:35.726 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0 2023-06-03T20:20:35.726 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:20:35.726 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:20:35.727 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "throttled", 2023-06-03T20:20:35.727 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:12:10.691874+0000", 2023-06-03T20:20:35.727 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 4294967295.9999995 2023-06-03T20:20:35.727 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:20:35.727 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:20:35.727 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "header_read", 2023-06-03T20:20:35.727 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:12:10.691874+0000", 2023-06-03T20:20:35.727 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 9.132e-05 2023-06-03T20:20:35.728 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:20:35.728 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:20:35.728 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "all_read", 2023-06-03T20:20:35.728 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:12:10.691965+0000", 2023-06-03T20:20:35.728 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 1.2515e-05 2023-06-03T20:20:35.728 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:20:35.728 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:20:35.729 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "dispatched", 2023-06-03T20:20:35.729 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:12:10.691978+0000", 2023-06-03T20:20:35.729 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 8.1270000000000003e-06 2023-06-03T20:20:35.729 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:20:35.729 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:20:35.729 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "queued_for_pg", 2023-06-03T20:20:35.729 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:12:10.691986+0000", 2023-06-03T20:20:35.729 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 5.2308000000000001e-05 2023-06-03T20:20:35.730 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:20:35.730 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:20:35.730 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "reached_pg", 2023-06-03T20:20:35.730 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:12:10.692038+0000", 2023-06-03T20:20:35.730 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.00015591499999999999 2023-06-03T20:20:35.730 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:20:35.730 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:20:35.730 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "started", 2023-06-03T20:20:35.731 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:12:10.692194+0000", 2023-06-03T20:20:35.731 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.00023018499999999999 2023-06-03T20:20:35.731 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:20:35.731 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:20:35.731 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "waiting for subops from 5,6", 2023-06-03T20:20:35.731 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:12:10.692424+0000", 2023-06-03T20:20:35.731 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.0023787529999999999 2023-06-03T20:20:35.732 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:20:35.732 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:20:35.732 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "op_commit", 2023-06-03T20:20:35.732 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:12:10.694803+0000", 2023-06-03T20:20:35.732 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.0058327199999999996 2023-06-03T20:20:35.732 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:20:35.732 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:20:35.732 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:20:35.733 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:12:10.700636+0000", 2023-06-03T20:20:35.733 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.29045285500000001 2023-06-03T20:20:35.733 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:20:35.733 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:20:35.733 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:20:35.733 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:12:10.991089+0000", 2023-06-03T20:20:35.733 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 5.0569000000000003e-05 2023-06-03T20:20:35.734 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:20:35.734 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:20:35.734 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "commit_sent", 2023-06-03T20:20:35.734 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:12:10.991139+0000", 2023-06-03T20:20:35.734 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 5.0745999999999998e-05 2023-06-03T20:20:35.734 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:20:35.734 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:20:35.734 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "done", 2023-06-03T20:20:35.735 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:12:10.991190+0000", 2023-06-03T20:20:35.735 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.29931533300000002 2023-06-03T20:20:35.735 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:20:35.735 INFO:teuthology.orchestra.run.smithi164.stdout: ] 2023-06-03T20:20:35.735 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:20:35.735 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:20:35.735 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:20:35.735 INFO:teuthology.orchestra.run.smithi164.stdout: "description": "osd_op(client.46187.0:60483 11.5 11:a65a5150:::benchmark_data_smithi089_165412_object60482:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e197)", 2023-06-03T20:20:35.736 INFO:teuthology.orchestra.run.smithi164.stdout: "initiated_at": "2023-06-03T20:12:10.695247+0000", 2023-06-03T20:20:35.736 INFO:teuthology.orchestra.run.smithi164.stdout: "age": 505.01787215600001, 2023-06-03T20:20:35.736 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.29607891600000003, 2023-06-03T20:20:35.736 INFO:teuthology.orchestra.run.smithi164.stdout: "type_data": { 2023-06-03T20:20:35.736 INFO:teuthology.orchestra.run.smithi164.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-03T20:20:35.736 INFO:teuthology.orchestra.run.smithi164.stdout: "client_info": { 2023-06-03T20:20:35.736 INFO:teuthology.orchestra.run.smithi164.stdout: "client": "client.46187", 2023-06-03T20:20:35.737 INFO:teuthology.orchestra.run.smithi164.stdout: "client_addr": "172.21.15.89:0/880684210", 2023-06-03T20:20:35.737 INFO:teuthology.orchestra.run.smithi164.stdout: "tid": 60483 2023-06-03T20:20:35.737 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:20:35.737 INFO:teuthology.orchestra.run.smithi164.stdout: "events": [ 2023-06-03T20:20:35.737 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:20:35.737 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "initiated", 2023-06-03T20:20:35.737 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:12:10.695247+0000", 2023-06-03T20:20:35.737 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0 2023-06-03T20:20:35.738 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:20:35.738 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:20:35.738 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "throttled", 2023-06-03T20:20:35.738 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:12:10.695247+0000", 2023-06-03T20:20:35.738 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 4294967295.9999995 2023-06-03T20:20:35.738 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:20:35.738 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:20:35.738 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "header_read", 2023-06-03T20:20:35.739 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:12:10.695247+0000", 2023-06-03T20:20:35.739 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 6.2464000000000004e-05 2023-06-03T20:20:35.739 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:20:35.739 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:20:35.739 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "all_read", 2023-06-03T20:20:35.739 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:12:10.695309+0000", 2023-06-03T20:20:35.739 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 1.0056e-05 2023-06-03T20:20:35.739 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:20:35.740 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:20:35.740 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "dispatched", 2023-06-03T20:20:35.740 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:12:10.695319+0000", 2023-06-03T20:20:35.740 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 3.8903999999999999e-05 2023-06-03T20:20:35.740 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:20:35.740 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:20:35.740 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "queued_for_pg", 2023-06-03T20:20:35.740 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:12:10.695358+0000", 2023-06-03T20:20:35.741 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 4.5828000000000002e-05 2023-06-03T20:20:35.741 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:20:35.741 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:20:35.741 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "reached_pg", 2023-06-03T20:20:35.741 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:12:10.695404+0000", 2023-06-03T20:20:35.741 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 9.3615000000000004e-05 2023-06-03T20:20:35.741 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:20:35.742 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:20:35.742 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "started", 2023-06-03T20:20:35.742 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:12:10.695498+0000", 2023-06-03T20:20:35.742 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 9.9705999999999997e-05 2023-06-03T20:20:35.742 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:20:35.742 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:20:35.742 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "waiting for subops from 5,6", 2023-06-03T20:20:35.742 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:12:10.695597+0000", 2023-06-03T20:20:35.743 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.0018357239999999999 2023-06-03T20:20:35.743 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:20:35.743 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:20:35.743 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "op_commit", 2023-06-03T20:20:35.743 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:12:10.697433+0000", 2023-06-03T20:20:35.743 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.0033581549999999998 2023-06-03T20:20:35.743 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:20:35.743 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:20:35.744 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:20:35.744 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:12:10.700791+0000", 2023-06-03T20:20:35.744 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.29046200700000002 2023-06-03T20:20:35.744 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:20:35.744 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:20:35.744 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:20:35.744 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:12:10.991253+0000", 2023-06-03T20:20:35.744 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 4.1170000000000001e-05 2023-06-03T20:20:35.745 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:20:35.745 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:20:35.745 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "commit_sent", 2023-06-03T20:20:35.745 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:12:10.991294+0000", 2023-06-03T20:20:35.745 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 3.1878000000000002e-05 2023-06-03T20:20:35.745 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:20:35.745 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:20:35.745 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "done", 2023-06-03T20:20:35.746 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:12:10.991326+0000", 2023-06-03T20:20:35.746 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.29607891600000003 2023-06-03T20:20:35.746 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:20:35.746 INFO:teuthology.orchestra.run.smithi164.stdout: ] 2023-06-03T20:20:35.746 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:20:35.746 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:20:35.746 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:20:35.746 INFO:teuthology.orchestra.run.smithi164.stdout: "description": "osd_op(client.46187.0:60493 11.5 11:af1be14d:::benchmark_data_smithi089_165412_object60492:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e197)", 2023-06-03T20:20:35.747 INFO:teuthology.orchestra.run.smithi164.stdout: "initiated_at": "2023-06-03T20:12:10.710960+0000", 2023-06-03T20:20:35.747 INFO:teuthology.orchestra.run.smithi164.stdout: "age": 505.00215931700001, 2023-06-03T20:20:35.747 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.28049938899999999, 2023-06-03T20:20:35.747 INFO:teuthology.orchestra.run.smithi164.stdout: "type_data": { 2023-06-03T20:20:35.747 INFO:teuthology.orchestra.run.smithi164.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-03T20:20:35.747 INFO:teuthology.orchestra.run.smithi164.stdout: "client_info": { 2023-06-03T20:20:35.747 INFO:teuthology.orchestra.run.smithi164.stdout: "client": "client.46187", 2023-06-03T20:20:35.748 INFO:teuthology.orchestra.run.smithi164.stdout: "client_addr": "172.21.15.89:0/880684210", 2023-06-03T20:20:35.748 INFO:teuthology.orchestra.run.smithi164.stdout: "tid": 60493 2023-06-03T20:20:35.748 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:20:35.748 INFO:teuthology.orchestra.run.smithi164.stdout: "events": [ 2023-06-03T20:20:35.748 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:20:35.748 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "initiated", 2023-06-03T20:20:35.748 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:12:10.710960+0000", 2023-06-03T20:20:35.748 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0 2023-06-03T20:20:35.749 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:20:35.749 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:20:35.749 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "throttled", 2023-06-03T20:20:35.749 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:12:10.710960+0000", 2023-06-03T20:20:35.749 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 4294967295.9999981 2023-06-03T20:20:35.749 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:20:35.749 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:20:35.749 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "header_read", 2023-06-03T20:20:35.750 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:12:10.710958+0000", 2023-06-03T20:20:35.750 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 6.5641000000000004e-05 2023-06-03T20:20:35.750 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:20:35.750 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:20:35.750 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "all_read", 2023-06-03T20:20:35.750 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:12:10.711024+0000", 2023-06-03T20:20:35.750 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 2.1622000000000001e-05 2023-06-03T20:20:35.751 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:20:35.751 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:20:35.751 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "dispatched", 2023-06-03T20:20:35.751 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:12:10.711046+0000", 2023-06-03T20:20:35.751 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 1.4433000000000001e-05 2023-06-03T20:20:35.751 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:20:35.751 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:20:35.751 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "queued_for_pg", 2023-06-03T20:20:35.752 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:12:10.711060+0000", 2023-06-03T20:20:35.752 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 4.7890999999999999e-05 2023-06-03T20:20:35.752 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:20:35.752 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:20:35.752 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "reached_pg", 2023-06-03T20:20:35.752 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:12:10.711108+0000", 2023-06-03T20:20:35.752 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.000105142 2023-06-03T20:20:35.752 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:20:35.753 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:20:35.753 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "started", 2023-06-03T20:20:35.753 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:12:10.711213+0000", 2023-06-03T20:20:35.753 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.000117027 2023-06-03T20:20:35.753 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:20:35.753 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:20:35.753 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "waiting for subops from 5,6", 2023-06-03T20:20:35.753 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:12:10.711330+0000", 2023-06-03T20:20:35.754 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.001883633 2023-06-03T20:20:35.754 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:20:35.754 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:20:35.754 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "op_commit", 2023-06-03T20:20:35.754 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:12:10.713214+0000", 2023-06-03T20:20:35.754 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.00062865700000000002 2023-06-03T20:20:35.754 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:20:35.755 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:20:35.755 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:20:35.755 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:12:10.713842+0000", 2023-06-03T20:20:35.755 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.277537749 2023-06-03T20:20:35.755 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:20:35.755 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:20:35.755 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:20:35.755 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:12:10.991380+0000", 2023-06-03T20:20:35.756 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 3.6746999999999999e-05 2023-06-03T20:20:35.756 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:20:35.756 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:20:35.756 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "commit_sent", 2023-06-03T20:20:35.756 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:12:10.991417+0000", 2023-06-03T20:20:35.756 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 4.2806999999999999e-05 2023-06-03T20:20:35.756 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:20:35.756 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:20:35.757 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "done", 2023-06-03T20:20:35.757 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:12:10.991460+0000", 2023-06-03T20:20:35.757 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.28049938899999999 2023-06-03T20:20:35.757 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:20:35.757 INFO:teuthology.orchestra.run.smithi164.stdout: ] 2023-06-03T20:20:35.757 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:20:35.757 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:20:35.757 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:20:35.758 INFO:teuthology.orchestra.run.smithi164.stdout: "description": "osd_op(client.46187.0:60495 11.5 11:acf684de:::benchmark_data_smithi089_165412_object60494:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e197)", 2023-06-03T20:20:35.758 INFO:teuthology.orchestra.run.smithi164.stdout: "initiated_at": "2023-06-03T20:12:10.715064+0000", 2023-06-03T20:20:35.758 INFO:teuthology.orchestra.run.smithi164.stdout: "age": 504.99805548500001, 2023-06-03T20:20:35.758 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.27649058300000001, 2023-06-03T20:20:35.758 INFO:teuthology.orchestra.run.smithi164.stdout: "type_data": { 2023-06-03T20:20:35.758 INFO:teuthology.orchestra.run.smithi164.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-03T20:20:35.758 INFO:teuthology.orchestra.run.smithi164.stdout: "client_info": { 2023-06-03T20:20:35.758 INFO:teuthology.orchestra.run.smithi164.stdout: "client": "client.46187", 2023-06-03T20:20:35.759 INFO:teuthology.orchestra.run.smithi164.stdout: "client_addr": "172.21.15.89:0/880684210", 2023-06-03T20:20:35.759 INFO:teuthology.orchestra.run.smithi164.stdout: "tid": 60495 2023-06-03T20:20:35.759 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:20:35.759 INFO:teuthology.orchestra.run.smithi164.stdout: "events": [ 2023-06-03T20:20:35.759 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:20:35.759 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "initiated", 2023-06-03T20:20:35.759 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:12:10.715064+0000", 2023-06-03T20:20:35.759 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0 2023-06-03T20:20:35.760 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:20:35.760 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:20:35.760 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "throttled", 2023-06-03T20:20:35.760 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:12:10.715064+0000", 2023-06-03T20:20:35.760 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 4294967295.9999981 2023-06-03T20:20:35.760 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:20:35.760 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:20:35.761 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "header_read", 2023-06-03T20:20:35.761 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:12:10.715062+0000", 2023-06-03T20:20:35.761 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.000117798 2023-06-03T20:20:35.761 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:20:35.761 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:20:35.761 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "all_read", 2023-06-03T20:20:35.761 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:12:10.715180+0000", 2023-06-03T20:20:35.761 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 1.9901000000000001e-05 2023-06-03T20:20:35.762 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:20:35.762 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:20:35.762 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "dispatched", 2023-06-03T20:20:35.762 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:12:10.715200+0000", 2023-06-03T20:20:35.762 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 1.2058999999999999e-05 2023-06-03T20:20:35.762 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:20:35.762 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:20:35.762 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "queued_for_pg", 2023-06-03T20:20:35.763 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:12:10.715212+0000", 2023-06-03T20:20:35.763 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 3.3269999999999998e-05 2023-06-03T20:20:35.763 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:20:35.763 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:20:35.763 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "reached_pg", 2023-06-03T20:20:35.763 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:12:10.715245+0000", 2023-06-03T20:20:35.763 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 8.3429000000000004e-05 2023-06-03T20:20:35.763 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:20:35.764 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:20:35.764 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "started", 2023-06-03T20:20:35.764 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:12:10.715329+0000", 2023-06-03T20:20:35.764 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.000105509 2023-06-03T20:20:35.764 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:20:35.764 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:20:35.764 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "waiting for subops from 5,6", 2023-06-03T20:20:35.764 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:12:10.715434+0000", 2023-06-03T20:20:35.765 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.0020838359999999999 2023-06-03T20:20:35.765 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:20:35.765 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:20:35.765 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "op_commit", 2023-06-03T20:20:35.765 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:12:10.717518+0000", 2023-06-03T20:20:35.765 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.000117759 2023-06-03T20:20:35.765 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:20:35.766 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:20:35.766 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:20:35.766 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:12:10.717636+0000", 2023-06-03T20:20:35.766 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.27385867400000002 2023-06-03T20:20:35.766 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:20:35.766 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:20:35.766 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:20:35.766 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:12:10.991495+0000", 2023-06-03T20:20:35.767 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 3.3670999999999997e-05 2023-06-03T20:20:35.767 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:20:35.767 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:20:35.767 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "commit_sent", 2023-06-03T20:20:35.767 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:12:10.991528+0000", 2023-06-03T20:20:35.767 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 2.6460000000000001e-05 2023-06-03T20:20:35.767 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:20:35.767 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:20:35.768 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "done", 2023-06-03T20:20:35.768 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:12:10.991555+0000", 2023-06-03T20:20:35.768 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.27649058300000001 2023-06-03T20:20:35.768 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:20:35.768 INFO:teuthology.orchestra.run.smithi164.stdout: ] 2023-06-03T20:20:35.768 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:20:35.768 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:20:35.768 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:20:35.769 INFO:teuthology.orchestra.run.smithi164.stdout: "description": "osd_op(client.46187.0:71103 11.4 11:2354b94d:::benchmark_data_smithi089_165412_object71102:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e197)", 2023-06-03T20:20:35.769 INFO:teuthology.orchestra.run.smithi164.stdout: "initiated_at": "2023-06-03T20:12:19.896024+0000", 2023-06-03T20:20:35.769 INFO:teuthology.orchestra.run.smithi164.stdout: "age": 495.81709596500002, 2023-06-03T20:20:35.769 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.22640047799999999, 2023-06-03T20:20:35.769 INFO:teuthology.orchestra.run.smithi164.stdout: "type_data": { 2023-06-03T20:20:35.769 INFO:teuthology.orchestra.run.smithi164.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-03T20:20:35.769 INFO:teuthology.orchestra.run.smithi164.stdout: "client_info": { 2023-06-03T20:20:35.770 INFO:teuthology.orchestra.run.smithi164.stdout: "client": "client.46187", 2023-06-03T20:20:35.770 INFO:teuthology.orchestra.run.smithi164.stdout: "client_addr": "172.21.15.89:0/880684210", 2023-06-03T20:20:35.770 INFO:teuthology.orchestra.run.smithi164.stdout: "tid": 71103 2023-06-03T20:20:35.770 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:20:35.770 INFO:teuthology.orchestra.run.smithi164.stdout: "events": [ 2023-06-03T20:20:35.770 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:20:35.770 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "initiated", 2023-06-03T20:20:35.770 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:12:19.896024+0000", 2023-06-03T20:20:35.771 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0 2023-06-03T20:20:35.771 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:20:35.771 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:20:35.771 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "throttled", 2023-06-03T20:20:35.771 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:12:19.896024+0000", 2023-06-03T20:20:35.771 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 4294967295.9999976 2023-06-03T20:20:35.771 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:20:35.771 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:20:35.772 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "header_read", 2023-06-03T20:20:35.772 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:12:19.896021+0000", 2023-06-03T20:20:35.772 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 6.0772e-05 2023-06-03T20:20:35.772 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:20:35.772 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:20:35.772 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "all_read", 2023-06-03T20:20:35.772 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:12:19.896082+0000", 2023-06-03T20:20:35.772 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 1.2201000000000001e-05 2023-06-03T20:20:35.773 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:20:35.773 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:20:35.773 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "dispatched", 2023-06-03T20:20:35.773 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:12:19.896094+0000", 2023-06-03T20:20:35.773 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 7.6809999999999998e-06 2023-06-03T20:20:35.773 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:20:35.773 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:20:35.773 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "queued_for_pg", 2023-06-03T20:20:35.774 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:12:19.896102+0000", 2023-06-03T20:20:35.774 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 5.1476e-05 2023-06-03T20:20:35.774 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:20:35.774 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:20:35.774 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "reached_pg", 2023-06-03T20:20:35.774 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:12:19.896153+0000", 2023-06-03T20:20:35.774 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.00015325899999999999 2023-06-03T20:20:35.774 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:20:35.775 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:20:35.775 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "started", 2023-06-03T20:20:35.775 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:12:19.896307+0000", 2023-06-03T20:20:35.775 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.000177887 2023-06-03T20:20:35.775 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:20:35.775 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:20:35.775 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "waiting for subops from 2,6", 2023-06-03T20:20:35.776 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:12:19.896485+0000", 2023-06-03T20:20:35.776 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.0026035870000000001 2023-06-03T20:20:35.776 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:20:35.776 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:20:35.776 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:20:35.776 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:12:19.899088+0000", 2023-06-03T20:20:35.776 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.0032725499999999999 2023-06-03T20:20:35.776 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:20:35.777 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:20:35.777 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "op_commit", 2023-06-03T20:20:35.777 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:12:19.902361+0000", 2023-06-03T20:20:35.777 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.219975473 2023-06-03T20:20:35.777 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:20:35.777 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:20:35.777 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:20:35.777 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:12:20.122336+0000", 2023-06-03T20:20:35.778 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 4.0735000000000002e-05 2023-06-03T20:20:35.778 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:20:35.778 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:20:35.778 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "commit_sent", 2023-06-03T20:20:35.778 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:12:20.122377+0000", 2023-06-03T20:20:35.778 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 4.7187000000000001e-05 2023-06-03T20:20:35.778 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:20:35.778 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:20:35.779 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "done", 2023-06-03T20:20:35.779 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:12:20.122424+0000", 2023-06-03T20:20:35.779 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.22640047799999999 2023-06-03T20:20:35.779 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:20:35.779 INFO:teuthology.orchestra.run.smithi164.stdout: ] 2023-06-03T20:20:35.779 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:20:35.779 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:20:35.779 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:20:35.780 INFO:teuthology.orchestra.run.smithi164.stdout: "description": "osd_op(client.46187.0:71110 11.4 11:2f4d03ca:::benchmark_data_smithi089_165412_object71109:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e197)", 2023-06-03T20:20:35.780 INFO:teuthology.orchestra.run.smithi164.stdout: "initiated_at": "2023-06-03T20:12:19.899163+0000", 2023-06-03T20:20:35.780 INFO:teuthology.orchestra.run.smithi164.stdout: "age": 495.81395616200001, 2023-06-03T20:20:35.780 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.223819605, 2023-06-03T20:20:35.780 INFO:teuthology.orchestra.run.smithi164.stdout: "type_data": { 2023-06-03T20:20:35.780 INFO:teuthology.orchestra.run.smithi164.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-03T20:20:35.780 INFO:teuthology.orchestra.run.smithi164.stdout: "client_info": { 2023-06-03T20:20:35.780 INFO:teuthology.orchestra.run.smithi164.stdout: "client": "client.46187", 2023-06-03T20:20:35.781 INFO:teuthology.orchestra.run.smithi164.stdout: "client_addr": "172.21.15.89:0/880684210", 2023-06-03T20:20:35.781 INFO:teuthology.orchestra.run.smithi164.stdout: "tid": 71110 2023-06-03T20:20:35.781 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:20:35.781 INFO:teuthology.orchestra.run.smithi164.stdout: "events": [ 2023-06-03T20:20:35.781 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:20:35.781 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "initiated", 2023-06-03T20:20:35.781 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:12:19.899163+0000", 2023-06-03T20:20:35.781 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0 2023-06-03T20:20:35.782 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:20:35.782 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:20:35.782 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "throttled", 2023-06-03T20:20:35.782 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:12:19.899163+0000", 2023-06-03T20:20:35.782 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 4294967295.9999986 2023-06-03T20:20:35.782 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:20:35.782 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:20:35.782 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "header_read", 2023-06-03T20:20:35.783 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:12:19.899162+0000", 2023-06-03T20:20:35.783 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 3.7632999999999997e-05 2023-06-03T20:20:35.783 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:20:35.783 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:20:35.783 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "all_read", 2023-06-03T20:20:35.783 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:12:19.899200+0000", 2023-06-03T20:20:35.783 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 2.4368000000000001e-05 2023-06-03T20:20:35.784 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:20:35.784 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:20:35.784 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "dispatched", 2023-06-03T20:20:35.784 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:12:19.899224+0000", 2023-06-03T20:20:35.784 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 1.1826e-05 2023-06-03T20:20:35.784 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:20:35.784 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:20:35.784 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "queued_for_pg", 2023-06-03T20:20:35.785 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:12:19.899236+0000", 2023-06-03T20:20:35.785 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 4.3751000000000002e-05 2023-06-03T20:20:35.785 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:20:35.785 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:20:35.785 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "reached_pg", 2023-06-03T20:20:35.785 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:12:19.899279+0000", 2023-06-03T20:20:35.785 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.000134702 2023-06-03T20:20:35.785 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:20:35.786 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:20:35.786 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "started", 2023-06-03T20:20:35.786 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:12:19.899414+0000", 2023-06-03T20:20:35.786 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.00017672 2023-06-03T20:20:35.786 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:20:35.786 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:20:35.786 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "waiting for subops from 2,6", 2023-06-03T20:20:35.786 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:12:19.899591+0000", 2023-06-03T20:20:35.787 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.0029351049999999999 2023-06-03T20:20:35.787 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:20:35.787 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:20:35.787 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "op_commit", 2023-06-03T20:20:35.787 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:12:19.902526+0000", 2023-06-03T20:20:35.787 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.0049889230000000001 2023-06-03T20:20:35.787 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:20:35.787 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:20:35.788 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:20:35.788 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:12:19.907515+0000", 2023-06-03T20:20:35.788 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.21527191900000001 2023-06-03T20:20:35.788 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:20:35.788 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:20:35.788 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:20:35.788 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:12:20.122787+0000", 2023-06-03T20:20:35.788 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.00012214600000000001 2023-06-03T20:20:35.789 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:20:35.789 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:20:35.789 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "commit_sent", 2023-06-03T20:20:35.789 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:12:20.122909+0000", 2023-06-03T20:20:35.789 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 7.4039e-05 2023-06-03T20:20:35.789 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:20:35.789 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:20:35.789 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "done", 2023-06-03T20:20:35.790 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:12:20.122983+0000", 2023-06-03T20:20:35.790 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.223819605 2023-06-03T20:20:35.790 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:20:35.790 INFO:teuthology.orchestra.run.smithi164.stdout: ] 2023-06-03T20:20:35.790 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:20:35.790 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:20:35.790 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:20:35.790 INFO:teuthology.orchestra.run.smithi164.stdout: "description": "osd_op(client.46373.0:45028 12.d 12:b829c474:::benchmark_data_smithi089_191647_object45027:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e224)", 2023-06-03T20:20:35.791 INFO:teuthology.orchestra.run.smithi164.stdout: "initiated_at": "2023-06-03T20:17:42.901466+0000", 2023-06-03T20:20:35.791 INFO:teuthology.orchestra.run.smithi164.stdout: "age": 172.81165353599999, 2023-06-03T20:20:35.791 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.25592049100000003, 2023-06-03T20:20:35.791 INFO:teuthology.orchestra.run.smithi164.stdout: "type_data": { 2023-06-03T20:20:35.791 INFO:teuthology.orchestra.run.smithi164.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-03T20:20:35.791 INFO:teuthology.orchestra.run.smithi164.stdout: "client_info": { 2023-06-03T20:20:35.791 INFO:teuthology.orchestra.run.smithi164.stdout: "client": "client.46373", 2023-06-03T20:20:35.791 INFO:teuthology.orchestra.run.smithi164.stdout: "client_addr": "172.21.15.89:0/3737977806", 2023-06-03T20:20:35.792 INFO:teuthology.orchestra.run.smithi164.stdout: "tid": 45028 2023-06-03T20:20:35.792 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:20:35.792 INFO:teuthology.orchestra.run.smithi164.stdout: "events": [ 2023-06-03T20:20:35.792 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:20:35.792 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "initiated", 2023-06-03T20:20:35.792 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:17:42.901466+0000", 2023-06-03T20:20:35.792 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0 2023-06-03T20:20:35.792 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:20:35.793 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:20:35.793 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "throttled", 2023-06-03T20:20:35.793 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:17:42.901466+0000", 2023-06-03T20:20:35.793 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 4294967295.9999981 2023-06-03T20:20:35.793 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:20:35.793 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:20:35.793 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "header_read", 2023-06-03T20:20:35.793 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:17:42.901464+0000", 2023-06-03T20:20:35.794 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 9.4926000000000006e-05 2023-06-03T20:20:35.794 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:20:35.794 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:20:35.794 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "all_read", 2023-06-03T20:20:35.794 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:17:42.901559+0000", 2023-06-03T20:20:35.794 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 1.2079000000000001e-05 2023-06-03T20:20:35.794 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:20:35.794 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:20:35.795 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "dispatched", 2023-06-03T20:20:35.795 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:17:42.901571+0000", 2023-06-03T20:20:35.795 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 6.8369999999999999e-06 2023-06-03T20:20:35.795 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:20:35.795 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:20:35.795 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "queued_for_pg", 2023-06-03T20:20:35.795 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:17:42.901578+0000", 2023-06-03T20:20:35.796 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 3.4109999999999997e-05 2023-06-03T20:20:35.796 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:20:35.796 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:20:35.796 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "reached_pg", 2023-06-03T20:20:35.796 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:17:42.901612+0000", 2023-06-03T20:20:35.796 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 9.9675999999999999e-05 2023-06-03T20:20:35.796 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:20:35.796 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:20:35.797 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "started", 2023-06-03T20:20:35.797 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:17:42.901712+0000", 2023-06-03T20:20:35.797 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.00012689299999999999 2023-06-03T20:20:35.797 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:20:35.797 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:20:35.797 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "waiting for subops from 2,3", 2023-06-03T20:20:35.797 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:17:42.901838+0000", 2023-06-03T20:20:35.797 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.00156441 2023-06-03T20:20:35.798 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:20:35.798 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:20:35.798 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "op_commit", 2023-06-03T20:20:35.798 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:17:42.903403+0000", 2023-06-03T20:20:35.798 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.25356268799999998 2023-06-03T20:20:35.798 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:20:35.798 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:20:35.798 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:20:35.799 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:17:43.156966+0000", 2023-06-03T20:20:35.799 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.00011974899999999999 2023-06-03T20:20:35.799 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:20:35.799 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:20:35.799 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:20:35.799 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:17:43.157085+0000", 2023-06-03T20:20:35.799 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.000161905 2023-06-03T20:20:35.799 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:20:35.800 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:20:35.800 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "commit_sent", 2023-06-03T20:20:35.800 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:17:43.157247+0000", 2023-06-03T20:20:35.800 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.000139321 2023-06-03T20:20:35.800 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:20:35.800 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:20:35.800 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "done", 2023-06-03T20:20:35.801 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:17:43.157387+0000", 2023-06-03T20:20:35.801 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.25592049100000003 2023-06-03T20:20:35.801 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:20:35.801 INFO:teuthology.orchestra.run.smithi164.stdout: ] 2023-06-03T20:20:35.801 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:20:35.801 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:20:35.801 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:20:35.801 INFO:teuthology.orchestra.run.smithi164.stdout: "description": "osd_op(client.46373.0:45034 12.7 12:ea1ba3a7:::benchmark_data_smithi089_191647_object45033:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e224)", 2023-06-03T20:20:35.802 INFO:teuthology.orchestra.run.smithi164.stdout: "initiated_at": "2023-06-03T20:17:42.902482+0000", 2023-06-03T20:20:35.802 INFO:teuthology.orchestra.run.smithi164.stdout: "age": 172.810637985, 2023-06-03T20:20:35.802 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.25655267700000001, 2023-06-03T20:20:35.802 INFO:teuthology.orchestra.run.smithi164.stdout: "type_data": { 2023-06-03T20:20:35.802 INFO:teuthology.orchestra.run.smithi164.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-03T20:20:35.802 INFO:teuthology.orchestra.run.smithi164.stdout: "client_info": { 2023-06-03T20:20:35.802 INFO:teuthology.orchestra.run.smithi164.stdout: "client": "client.46373", 2023-06-03T20:20:35.802 INFO:teuthology.orchestra.run.smithi164.stdout: "client_addr": "172.21.15.89:0/3737977806", 2023-06-03T20:20:35.803 INFO:teuthology.orchestra.run.smithi164.stdout: "tid": 45034 2023-06-03T20:20:35.803 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:20:35.803 INFO:teuthology.orchestra.run.smithi164.stdout: "events": [ 2023-06-03T20:20:35.803 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:20:35.803 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "initiated", 2023-06-03T20:20:35.803 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:17:42.902482+0000", 2023-06-03T20:20:35.803 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0 2023-06-03T20:20:35.803 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:20:35.804 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:20:35.804 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "throttled", 2023-06-03T20:20:35.804 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:17:42.902482+0000", 2023-06-03T20:20:35.804 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 4294967295.999999 2023-06-03T20:20:35.804 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:20:35.804 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:20:35.804 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "header_read", 2023-06-03T20:20:35.804 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:17:42.902480+0000", 2023-06-03T20:20:35.805 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 4.4431000000000002e-05 2023-06-03T20:20:35.805 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:20:35.805 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:20:35.805 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "all_read", 2023-06-03T20:20:35.805 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:17:42.902525+0000", 2023-06-03T20:20:35.805 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 1.0227999999999999e-05 2023-06-03T20:20:35.805 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:20:35.805 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:20:35.806 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "dispatched", 2023-06-03T20:20:35.806 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:17:42.902535+0000", 2023-06-03T20:20:35.806 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 5.2020000000000003e-06 2023-06-03T20:20:35.806 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:20:35.806 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:20:35.806 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "queued_for_pg", 2023-06-03T20:20:35.806 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:17:42.902540+0000", 2023-06-03T20:20:35.807 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 6.0118000000000003e-05 2023-06-03T20:20:35.807 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:20:35.807 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:20:35.807 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "reached_pg", 2023-06-03T20:20:35.807 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:17:42.902600+0000", 2023-06-03T20:20:35.807 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.00014852600000000001 2023-06-03T20:20:35.807 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:20:35.807 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:20:35.808 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "started", 2023-06-03T20:20:35.808 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:17:42.902749+0000", 2023-06-03T20:20:35.808 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.000185358 2023-06-03T20:20:35.808 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:20:35.808 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:20:35.808 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "waiting for subops from 2", 2023-06-03T20:20:35.808 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:17:42.902934+0000", 2023-06-03T20:20:35.808 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.001622224 2023-06-03T20:20:35.809 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:20:35.809 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:20:35.809 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "op_commit", 2023-06-03T20:20:35.809 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:17:42.904557+0000", 2023-06-03T20:20:35.809 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.25395919 2023-06-03T20:20:35.809 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:20:35.809 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:20:35.809 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:20:35.810 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:17:43.158516+0000", 2023-06-03T20:20:35.810 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.000209423 2023-06-03T20:20:35.810 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:20:35.810 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:20:35.810 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "commit_sent", 2023-06-03T20:20:35.810 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:17:43.158725+0000", 2023-06-03T20:20:35.810 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.00030914899999999999 2023-06-03T20:20:35.810 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:20:35.811 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:20:35.811 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "done", 2023-06-03T20:20:35.811 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:17:43.159034+0000", 2023-06-03T20:20:35.811 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.25655267700000001 2023-06-03T20:20:35.811 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:20:35.811 INFO:teuthology.orchestra.run.smithi164.stdout: ] 2023-06-03T20:20:35.811 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:20:35.812 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:20:35.812 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:20:35.812 INFO:teuthology.orchestra.run.smithi164.stdout: "description": "osd_repop(client.46373.0:45030 12.0 e224/217 12:0211bb92:::benchmark_data_smithi089_191647_object45029:head v 224'2890, mlcod=224'2888)", 2023-06-03T20:20:35.812 INFO:teuthology.orchestra.run.smithi164.stdout: "initiated_at": "2023-06-03T20:17:42.905837+0000", 2023-06-03T20:20:35.812 INFO:teuthology.orchestra.run.smithi164.stdout: "age": 172.80728295099999, 2023-06-03T20:20:35.812 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.2481389, 2023-06-03T20:20:35.812 INFO:teuthology.orchestra.run.smithi164.stdout: "type_data": { 2023-06-03T20:20:35.812 INFO:teuthology.orchestra.run.smithi164.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-03T20:20:35.813 INFO:teuthology.orchestra.run.smithi164.stdout: "events": [ 2023-06-03T20:20:35.813 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:20:35.813 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "initiated", 2023-06-03T20:20:35.813 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:17:42.905837+0000", 2023-06-03T20:20:35.813 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0 2023-06-03T20:20:35.813 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:20:35.813 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:20:35.813 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "throttled", 2023-06-03T20:20:35.814 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:17:42.905837+0000", 2023-06-03T20:20:35.814 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 4294967295.9999995 2023-06-03T20:20:35.814 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:20:35.814 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:20:35.814 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "header_read", 2023-06-03T20:20:35.814 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:17:42.905836+0000", 2023-06-03T20:20:35.814 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 4.3982e-05 2023-06-03T20:20:35.814 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:20:35.815 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:20:35.815 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "all_read", 2023-06-03T20:20:35.815 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:17:42.905880+0000", 2023-06-03T20:20:35.815 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 7.3459999999999996e-06 2023-06-03T20:20:35.815 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:20:35.815 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:20:35.815 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "dispatched", 2023-06-03T20:20:35.815 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:17:42.905888+0000", 2023-06-03T20:20:35.816 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 5.152e-06 2023-06-03T20:20:35.816 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:20:35.816 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:20:35.816 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "queued_for_pg", 2023-06-03T20:20:35.816 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:17:42.905893+0000", 2023-06-03T20:20:35.816 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 4.8334000000000002e-05 2023-06-03T20:20:35.816 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:20:35.817 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:20:35.817 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "reached_pg", 2023-06-03T20:20:35.817 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:17:42.905941+0000", 2023-06-03T20:20:35.817 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 1.7283999999999999e-05 2023-06-03T20:20:35.817 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:20:35.817 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:20:35.817 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "started", 2023-06-03T20:20:35.817 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:17:42.905958+0000", 2023-06-03T20:20:35.818 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.24792240700000001 2023-06-03T20:20:35.818 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:20:35.818 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:20:35.818 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "commit_sent", 2023-06-03T20:20:35.818 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:17:43.153881+0000", 2023-06-03T20:20:35.818 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 9.4791999999999996e-05 2023-06-03T20:20:35.818 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:20:35.818 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:20:35.819 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "done", 2023-06-03T20:20:35.819 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:17:43.153976+0000", 2023-06-03T20:20:35.819 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.2481389 2023-06-03T20:20:35.819 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:20:35.819 INFO:teuthology.orchestra.run.smithi164.stdout: ] 2023-06-03T20:20:35.819 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:20:35.819 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:20:35.819 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:20:35.820 INFO:teuthology.orchestra.run.smithi164.stdout: "description": "osd_op(client.46373.0:45040 12.7 12:e700222b:::benchmark_data_smithi089_191647_object45039:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e224)", 2023-06-03T20:20:35.820 INFO:teuthology.orchestra.run.smithi164.stdout: "initiated_at": "2023-06-03T20:17:42.906789+0000", 2023-06-03T20:20:35.820 INFO:teuthology.orchestra.run.smithi164.stdout: "age": 172.806330817, 2023-06-03T20:20:35.820 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.25457194399999999, 2023-06-03T20:20:35.820 INFO:teuthology.orchestra.run.smithi164.stdout: "type_data": { 2023-06-03T20:20:35.820 INFO:teuthology.orchestra.run.smithi164.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-03T20:20:35.820 INFO:teuthology.orchestra.run.smithi164.stdout: "client_info": { 2023-06-03T20:20:35.821 INFO:teuthology.orchestra.run.smithi164.stdout: "client": "client.46373", 2023-06-03T20:20:35.821 INFO:teuthology.orchestra.run.smithi164.stdout: "client_addr": "172.21.15.89:0/3737977806", 2023-06-03T20:20:35.821 INFO:teuthology.orchestra.run.smithi164.stdout: "tid": 45040 2023-06-03T20:20:35.821 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:20:35.821 INFO:teuthology.orchestra.run.smithi164.stdout: "events": [ 2023-06-03T20:20:35.821 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:20:35.821 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "initiated", 2023-06-03T20:20:35.821 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:17:42.906789+0000", 2023-06-03T20:20:35.822 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0 2023-06-03T20:20:35.822 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:20:35.822 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:20:35.822 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "throttled", 2023-06-03T20:20:35.822 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:17:42.906789+0000", 2023-06-03T20:20:35.822 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 4294967295.9999986 2023-06-03T20:20:35.822 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:20:35.822 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:20:35.823 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "header_read", 2023-06-03T20:20:35.823 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:17:42.906787+0000", 2023-06-03T20:20:35.823 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.00012655399999999999 2023-06-03T20:20:35.823 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:20:35.823 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:20:35.823 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "all_read", 2023-06-03T20:20:35.823 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:17:42.906914+0000", 2023-06-03T20:20:35.823 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 1.2373e-05 2023-06-03T20:20:35.824 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:20:35.824 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:20:35.824 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "dispatched", 2023-06-03T20:20:35.824 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:17:42.906926+0000", 2023-06-03T20:20:35.824 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 6.2940000000000002e-06 2023-06-03T20:20:35.824 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:20:35.824 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:20:35.824 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "queued_for_pg", 2023-06-03T20:20:35.825 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:17:42.906933+0000", 2023-06-03T20:20:35.825 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 6.8170000000000006e-05 2023-06-03T20:20:35.825 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:20:35.825 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:20:35.825 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "reached_pg", 2023-06-03T20:20:35.825 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:17:42.907001+0000", 2023-06-03T20:20:35.825 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.00015271300000000001 2023-06-03T20:20:35.825 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:20:35.826 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:20:35.826 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "started", 2023-06-03T20:20:35.826 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:17:42.907153+0000", 2023-06-03T20:20:35.826 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.246852189 2023-06-03T20:20:35.826 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:20:35.826 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:20:35.826 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "waiting for subops from 2", 2023-06-03T20:20:35.827 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:17:43.154006+0000", 2023-06-03T20:20:35.827 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.0044045600000000001 2023-06-03T20:20:35.827 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:20:35.827 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:20:35.827 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "op_commit", 2023-06-03T20:20:35.827 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:17:43.158410+0000", 2023-06-03T20:20:35.827 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.0025058569999999998 2023-06-03T20:20:35.827 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:20:35.828 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:20:35.828 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:20:35.828 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:17:43.160916+0000", 2023-06-03T20:20:35.828 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.00015159 2023-06-03T20:20:35.828 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:20:35.828 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:20:35.828 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "commit_sent", 2023-06-03T20:20:35.828 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:17:43.161068+0000", 2023-06-03T20:20:35.829 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.00029304199999999998 2023-06-03T20:20:35.829 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:20:35.829 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:20:35.829 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "done", 2023-06-03T20:20:35.829 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:17:43.161361+0000", 2023-06-03T20:20:35.829 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.25457194399999999 2023-06-03T20:20:35.829 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:20:35.829 INFO:teuthology.orchestra.run.smithi164.stdout: ] 2023-06-03T20:20:35.830 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:20:35.830 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:20:35.830 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:20:35.830 INFO:teuthology.orchestra.run.smithi164.stdout: "description": "osd_repop(client.46373.0:45033 12.6 e224/217 12:65888d8d:::benchmark_data_smithi089_191647_object45032:head v 224'2816, mlcod=224'2812)", 2023-06-03T20:20:35.830 INFO:teuthology.orchestra.run.smithi164.stdout: "initiated_at": "2023-06-03T20:17:42.906822+0000", 2023-06-03T20:20:35.830 INFO:teuthology.orchestra.run.smithi164.stdout: "age": 172.806298043, 2023-06-03T20:20:35.830 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.253660791, 2023-06-03T20:20:35.830 INFO:teuthology.orchestra.run.smithi164.stdout: "type_data": { 2023-06-03T20:20:35.831 INFO:teuthology.orchestra.run.smithi164.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-03T20:20:35.831 INFO:teuthology.orchestra.run.smithi164.stdout: "events": [ 2023-06-03T20:20:35.831 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:20:35.831 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "initiated", 2023-06-03T20:20:35.831 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:17:42.906822+0000", 2023-06-03T20:20:35.831 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0 2023-06-03T20:20:35.831 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:20:35.832 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:20:35.832 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "throttled", 2023-06-03T20:20:35.832 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:17:42.906822+0000", 2023-06-03T20:20:35.832 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 4294967295.999999 2023-06-03T20:20:35.832 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:20:35.832 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:20:35.832 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "header_read", 2023-06-03T20:20:35.832 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:17:42.906821+0000", 2023-06-03T20:20:35.833 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.00029021899999999998 2023-06-03T20:20:35.833 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:20:35.833 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:20:35.833 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "all_read", 2023-06-03T20:20:35.833 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:17:42.907111+0000", 2023-06-03T20:20:35.833 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 5.8100000000000003e-06 2023-06-03T20:20:35.833 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:20:35.833 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:20:35.834 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "dispatched", 2023-06-03T20:20:35.834 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:17:42.907117+0000", 2023-06-03T20:20:35.834 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 3.6459999999999999e-06 2023-06-03T20:20:35.834 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:20:35.834 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:20:35.834 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "queued_for_pg", 2023-06-03T20:20:35.834 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:17:42.907120+0000", 2023-06-03T20:20:35.834 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 4.4097000000000001e-05 2023-06-03T20:20:35.835 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:20:35.835 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:20:35.835 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "reached_pg", 2023-06-03T20:20:35.835 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:17:42.907165+0000", 2023-06-03T20:20:35.835 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 2.2708e-05 2023-06-03T20:20:35.835 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:20:35.835 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:20:35.836 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "started", 2023-06-03T20:20:35.836 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:17:42.907187+0000", 2023-06-03T20:20:35.836 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.25321293099999997 2023-06-03T20:20:35.836 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:20:35.836 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:20:35.836 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "commit_sent", 2023-06-03T20:20:35.836 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:17:43.160400+0000", 2023-06-03T20:20:35.836 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 8.2125000000000004e-05 2023-06-03T20:20:35.837 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:20:35.837 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:20:35.837 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "done", 2023-06-03T20:20:35.837 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:17:43.160482+0000", 2023-06-03T20:20:35.837 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.253660791 2023-06-03T20:20:35.837 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:20:35.837 INFO:teuthology.orchestra.run.smithi164.stdout: ] 2023-06-03T20:20:35.837 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:20:35.838 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:20:35.838 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:20:35.838 INFO:teuthology.orchestra.run.smithi164.stdout: "description": "osd_repop(client.46373.0:45038 12.2 e224/217 12:48d4f529:::benchmark_data_smithi089_191647_object45037:head v 224'2830, mlcod=224'2828)", 2023-06-03T20:20:35.838 INFO:teuthology.orchestra.run.smithi164.stdout: "initiated_at": "2023-06-03T20:17:42.906842+0000", 2023-06-03T20:20:35.838 INFO:teuthology.orchestra.run.smithi164.stdout: "age": 172.80627713199999, 2023-06-03T20:20:35.838 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.247385894, 2023-06-03T20:20:35.838 INFO:teuthology.orchestra.run.smithi164.stdout: "type_data": { 2023-06-03T20:20:35.838 INFO:teuthology.orchestra.run.smithi164.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-03T20:20:35.839 INFO:teuthology.orchestra.run.smithi164.stdout: "events": [ 2023-06-03T20:20:35.839 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:20:35.839 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "initiated", 2023-06-03T20:20:35.839 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:17:42.906842+0000", 2023-06-03T20:20:35.839 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0 2023-06-03T20:20:35.839 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:20:35.839 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:20:35.840 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "throttled", 2023-06-03T20:20:35.840 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:17:42.906842+0000", 2023-06-03T20:20:35.840 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 4294967295.9999995 2023-06-03T20:20:35.840 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:20:35.840 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:20:35.840 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "header_read", 2023-06-03T20:20:35.840 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:17:42.906842+0000", 2023-06-03T20:20:35.840 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.00014259099999999999 2023-06-03T20:20:35.841 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:20:35.841 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:20:35.841 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "all_read", 2023-06-03T20:20:35.841 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:17:42.906985+0000", 2023-06-03T20:20:35.841 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 8.6149999999999997e-06 2023-06-03T20:20:35.841 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:20:35.841 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:20:35.841 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "dispatched", 2023-06-03T20:20:35.842 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:17:42.906993+0000", 2023-06-03T20:20:35.842 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 4.403e-06 2023-06-03T20:20:35.842 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:20:35.842 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:20:35.842 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "queued_for_pg", 2023-06-03T20:20:35.842 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:17:42.906998+0000", 2023-06-03T20:20:35.842 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 4.1384999999999998e-05 2023-06-03T20:20:35.843 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:20:35.843 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:20:35.843 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "reached_pg", 2023-06-03T20:20:35.843 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:17:42.907039+0000", 2023-06-03T20:20:35.843 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 1.855e-05 2023-06-03T20:20:35.843 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:20:35.843 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:20:35.843 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "started", 2023-06-03T20:20:35.844 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:17:42.907058+0000", 2023-06-03T20:20:35.844 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.24710415899999999 2023-06-03T20:20:35.844 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:20:35.844 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:20:35.844 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "commit_sent", 2023-06-03T20:20:35.844 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:17:43.154162+0000", 2023-06-03T20:20:35.844 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 6.6556000000000004e-05 2023-06-03T20:20:35.844 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:20:35.845 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:20:35.845 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "done", 2023-06-03T20:20:35.845 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:17:43.154228+0000", 2023-06-03T20:20:35.845 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.247385894 2023-06-03T20:20:35.845 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:20:35.845 INFO:teuthology.orchestra.run.smithi164.stdout: ] 2023-06-03T20:20:35.845 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:20:35.846 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:20:35.846 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:20:35.846 INFO:teuthology.orchestra.run.smithi164.stdout: "description": "osd_repop(client.46373.0:45037 12.1 e224/217 12:858e8468:::benchmark_data_smithi089_191647_object45036:head v 224'2968, mlcod=224'2966)", 2023-06-03T20:20:35.846 INFO:teuthology.orchestra.run.smithi164.stdout: "initiated_at": "2023-06-03T20:17:42.907006+0000", 2023-06-03T20:20:35.846 INFO:teuthology.orchestra.run.smithi164.stdout: "age": 172.806113944, 2023-06-03T20:20:35.846 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.24697540000000001, 2023-06-03T20:20:35.846 INFO:teuthology.orchestra.run.smithi164.stdout: "type_data": { 2023-06-03T20:20:35.846 INFO:teuthology.orchestra.run.smithi164.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-03T20:20:35.847 INFO:teuthology.orchestra.run.smithi164.stdout: "events": [ 2023-06-03T20:20:35.847 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:20:35.847 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "initiated", 2023-06-03T20:20:35.847 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:17:42.907006+0000", 2023-06-03T20:20:35.847 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0 2023-06-03T20:20:35.847 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:20:35.847 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:20:35.847 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "throttled", 2023-06-03T20:20:35.848 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:17:42.907006+0000", 2023-06-03T20:20:35.848 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 4294967295.9999995 2023-06-03T20:20:35.848 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:20:35.848 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:20:35.848 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "header_read", 2023-06-03T20:20:35.848 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:17:42.907005+0000", 2023-06-03T20:20:35.848 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 4.4783999999999997e-05 2023-06-03T20:20:35.848 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:20:35.849 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:20:35.849 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "all_read", 2023-06-03T20:20:35.849 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:17:42.907050+0000", 2023-06-03T20:20:35.849 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 8.6379999999999993e-06 2023-06-03T20:20:35.849 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:20:35.849 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:20:35.849 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "dispatched", 2023-06-03T20:20:35.849 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:17:42.907059+0000", 2023-06-03T20:20:35.850 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 5.118e-06 2023-06-03T20:20:35.850 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:20:35.850 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:20:35.850 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "queued_for_pg", 2023-06-03T20:20:35.850 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:17:42.907064+0000", 2023-06-03T20:20:35.850 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 4.3924999999999997e-05 2023-06-03T20:20:35.850 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:20:35.851 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:20:35.851 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "reached_pg", 2023-06-03T20:20:35.851 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:17:42.907108+0000", 2023-06-03T20:20:35.851 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 2.8566999999999999e-05 2023-06-03T20:20:35.851 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:20:35.851 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:20:35.851 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "started", 2023-06-03T20:20:35.851 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:17:42.907136+0000", 2023-06-03T20:20:35.852 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.24676276699999999 2023-06-03T20:20:35.852 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:20:35.852 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:20:35.852 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "commit_sent", 2023-06-03T20:20:35.852 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:17:43.153899+0000", 2023-06-03T20:20:35.852 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 8.1996000000000006e-05 2023-06-03T20:20:35.852 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:20:35.852 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:20:35.853 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "done", 2023-06-03T20:20:35.853 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:17:43.153981+0000", 2023-06-03T20:20:35.853 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.24697540000000001 2023-06-03T20:20:35.853 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:20:35.853 INFO:teuthology.orchestra.run.smithi164.stdout: ] 2023-06-03T20:20:35.853 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:20:35.853 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:20:35.853 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:20:35.854 INFO:teuthology.orchestra.run.smithi164.stdout: "description": "osd_repop(client.46373.0:45032 12.e e224/217 12:7beb6fe5:::benchmark_data_smithi089_191647_object45031:head v 224'2825, mlcod=224'2823)", 2023-06-03T20:20:35.854 INFO:teuthology.orchestra.run.smithi164.stdout: "initiated_at": "2023-06-03T20:17:42.908593+0000", 2023-06-03T20:20:35.854 INFO:teuthology.orchestra.run.smithi164.stdout: "age": 172.80452644499999, 2023-06-03T20:20:35.854 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.25100721100000001, 2023-06-03T20:20:35.854 INFO:teuthology.orchestra.run.smithi164.stdout: "type_data": { 2023-06-03T20:20:35.854 INFO:teuthology.orchestra.run.smithi164.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-03T20:20:35.854 INFO:teuthology.orchestra.run.smithi164.stdout: "events": [ 2023-06-03T20:20:35.855 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:20:35.855 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "initiated", 2023-06-03T20:20:35.855 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:17:42.908593+0000", 2023-06-03T20:20:35.855 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0 2023-06-03T20:20:35.855 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:20:35.855 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:20:35.855 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "throttled", 2023-06-03T20:20:35.855 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:17:42.908593+0000", 2023-06-03T20:20:35.856 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 4294967295.9999995 2023-06-03T20:20:35.856 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:20:35.856 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:20:35.856 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "header_read", 2023-06-03T20:20:35.856 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:17:42.908593+0000", 2023-06-03T20:20:35.856 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 7.6361999999999998e-05 2023-06-03T20:20:35.856 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:20:35.856 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:20:35.857 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "all_read", 2023-06-03T20:20:35.857 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:17:42.908669+0000", 2023-06-03T20:20:35.857 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.24518262499999999 2023-06-03T20:20:35.857 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:20:35.857 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:20:35.857 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "dispatched", 2023-06-03T20:20:35.857 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:17:43.153852+0000", 2023-06-03T20:20:35.857 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.00016243 2023-06-03T20:20:35.858 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:20:35.858 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:20:35.858 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "queued_for_pg", 2023-06-03T20:20:35.858 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:17:43.154014+0000", 2023-06-03T20:20:35.858 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 5.7476000000000003e-05 2023-06-03T20:20:35.858 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:20:35.858 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:20:35.858 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "reached_pg", 2023-06-03T20:20:35.859 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:17:43.154072+0000", 2023-06-03T20:20:35.859 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 3.0238000000000001e-05 2023-06-03T20:20:35.859 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:20:35.859 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:20:35.859 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "started", 2023-06-03T20:20:35.859 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:17:43.154102+0000", 2023-06-03T20:20:35.859 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.0054123019999999999 2023-06-03T20:20:35.859 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:20:35.860 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:20:35.860 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "commit_sent", 2023-06-03T20:20:35.860 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:17:43.159514+0000", 2023-06-03T20:20:35.860 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 8.6274999999999996e-05 2023-06-03T20:20:35.860 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:20:35.860 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:20:35.860 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "done", 2023-06-03T20:20:35.861 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:17:43.159600+0000", 2023-06-03T20:20:35.861 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.25100721100000001 2023-06-03T20:20:35.861 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:20:35.861 INFO:teuthology.orchestra.run.smithi164.stdout: ] 2023-06-03T20:20:35.861 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:20:35.861 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:20:35.861 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:20:35.861 INFO:teuthology.orchestra.run.smithi164.stdout: "description": "osd_op(client.46373.0:45050 12.7 12:e348df25:::benchmark_data_smithi089_191647_object45049:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e224)", 2023-06-03T20:20:35.862 INFO:teuthology.orchestra.run.smithi164.stdout: "initiated_at": "2023-06-03T20:17:42.916547+0000", 2023-06-03T20:20:35.862 INFO:teuthology.orchestra.run.smithi164.stdout: "age": 172.79657285100001, 2023-06-03T20:20:35.862 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.24505840200000001, 2023-06-03T20:20:35.862 INFO:teuthology.orchestra.run.smithi164.stdout: "type_data": { 2023-06-03T20:20:35.862 INFO:teuthology.orchestra.run.smithi164.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-03T20:20:35.862 INFO:teuthology.orchestra.run.smithi164.stdout: "client_info": { 2023-06-03T20:20:35.862 INFO:teuthology.orchestra.run.smithi164.stdout: "client": "client.46373", 2023-06-03T20:20:35.862 INFO:teuthology.orchestra.run.smithi164.stdout: "client_addr": "172.21.15.89:0/3737977806", 2023-06-03T20:20:35.863 INFO:teuthology.orchestra.run.smithi164.stdout: "tid": 45050 2023-06-03T20:20:35.863 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:20:35.863 INFO:teuthology.orchestra.run.smithi164.stdout: "events": [ 2023-06-03T20:20:35.863 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:20:35.863 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "initiated", 2023-06-03T20:20:35.863 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:17:42.916547+0000", 2023-06-03T20:20:35.863 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0 2023-06-03T20:20:35.863 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:20:35.864 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:20:35.864 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "throttled", 2023-06-03T20:20:35.864 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:17:42.916547+0000", 2023-06-03T20:20:35.864 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 4294967295.9999971 2023-06-03T20:20:35.864 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:20:35.864 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:20:35.864 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "header_read", 2023-06-03T20:20:35.865 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:17:42.916544+0000", 2023-06-03T20:20:35.865 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 5.6119000000000003e-05 2023-06-03T20:20:35.865 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:20:35.865 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:20:35.865 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "all_read", 2023-06-03T20:20:35.865 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:17:42.916600+0000", 2023-06-03T20:20:35.865 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.23728180199999999 2023-06-03T20:20:35.865 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:20:35.866 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:20:35.866 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "dispatched", 2023-06-03T20:20:35.866 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:17:43.153882+0000", 2023-06-03T20:20:35.866 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 2.5083e-05 2023-06-03T20:20:35.866 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:20:35.866 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:20:35.866 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "queued_for_pg", 2023-06-03T20:20:35.866 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:17:43.153907+0000", 2023-06-03T20:20:35.867 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.0023207509999999998 2023-06-03T20:20:35.867 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:20:35.867 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:20:35.867 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "reached_pg", 2023-06-03T20:20:35.867 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:17:43.156228+0000", 2023-06-03T20:20:35.867 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.00015502800000000001 2023-06-03T20:20:35.867 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:20:35.867 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:20:35.868 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "started", 2023-06-03T20:20:35.868 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:17:43.156383+0000", 2023-06-03T20:20:35.868 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.00018033099999999999 2023-06-03T20:20:35.868 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:20:35.868 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:20:35.868 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "waiting for subops from 2", 2023-06-03T20:20:35.868 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:17:43.156563+0000", 2023-06-03T20:20:35.868 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.00252637 2023-06-03T20:20:35.869 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:20:35.869 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:20:35.869 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "op_commit", 2023-06-03T20:20:35.869 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:17:43.159090+0000", 2023-06-03T20:20:35.869 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.0023624900000000001 2023-06-03T20:20:35.869 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:20:35.869 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:20:35.870 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:20:35.870 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:17:43.161452+0000", 2023-06-03T20:20:35.870 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 5.6619000000000001e-05 2023-06-03T20:20:35.870 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:20:35.870 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:20:35.870 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "commit_sent", 2023-06-03T20:20:35.870 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:17:43.161509+0000", 2023-06-03T20:20:35.870 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 9.6495000000000004e-05 2023-06-03T20:20:35.871 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:20:35.871 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:20:35.871 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "done", 2023-06-03T20:20:35.871 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:17:43.161605+0000", 2023-06-03T20:20:35.871 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.24505840200000001 2023-06-03T20:20:35.871 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:20:35.871 INFO:teuthology.orchestra.run.smithi164.stdout: ] 2023-06-03T20:20:35.871 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:20:35.872 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:20:35.872 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:20:35.872 INFO:teuthology.orchestra.run.smithi164.stdout: "description": "osd_op(client.46373.0:53683 12.d 12:b08322d6:::benchmark_data_smithi089_191647_object53682:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e224)", 2023-06-03T20:20:35.872 INFO:teuthology.orchestra.run.smithi164.stdout: "initiated_at": "2023-06-03T20:17:50.062358+0000", 2023-06-03T20:20:35.872 INFO:teuthology.orchestra.run.smithi164.stdout: "age": 165.65076130400001, 2023-06-03T20:20:35.872 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.27565993, 2023-06-03T20:20:35.872 INFO:teuthology.orchestra.run.smithi164.stdout: "type_data": { 2023-06-03T20:20:35.872 INFO:teuthology.orchestra.run.smithi164.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-03T20:20:35.873 INFO:teuthology.orchestra.run.smithi164.stdout: "client_info": { 2023-06-03T20:20:35.873 INFO:teuthology.orchestra.run.smithi164.stdout: "client": "client.46373", 2023-06-03T20:20:35.873 INFO:teuthology.orchestra.run.smithi164.stdout: "client_addr": "172.21.15.89:0/3737977806", 2023-06-03T20:20:35.873 INFO:teuthology.orchestra.run.smithi164.stdout: "tid": 53683 2023-06-03T20:20:35.873 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:20:35.873 INFO:teuthology.orchestra.run.smithi164.stdout: "events": [ 2023-06-03T20:20:35.873 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:20:35.873 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "initiated", 2023-06-03T20:20:35.874 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:17:50.062358+0000", 2023-06-03T20:20:35.874 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0 2023-06-03T20:20:35.874 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:20:35.874 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:20:35.874 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "throttled", 2023-06-03T20:20:35.874 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:17:50.062358+0000", 2023-06-03T20:20:35.874 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 4294967295.9999976 2023-06-03T20:20:35.875 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:20:35.875 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:20:35.875 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "header_read", 2023-06-03T20:20:35.875 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:17:50.062356+0000", 2023-06-03T20:20:35.875 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 8.9591000000000003e-05 2023-06-03T20:20:35.875 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:20:35.875 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:20:35.875 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "all_read", 2023-06-03T20:20:35.876 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:17:50.062445+0000", 2023-06-03T20:20:35.876 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 2.3408999999999999e-05 2023-06-03T20:20:35.876 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:20:35.876 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:20:35.876 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "dispatched", 2023-06-03T20:20:35.876 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:17:50.062469+0000", 2023-06-03T20:20:35.876 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 1.0438000000000001e-05 2023-06-03T20:20:35.876 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:20:35.877 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:20:35.877 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "queued_for_pg", 2023-06-03T20:20:35.877 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:17:50.062479+0000", 2023-06-03T20:20:35.877 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 4.7215000000000001e-05 2023-06-03T20:20:35.877 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:20:35.877 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:20:35.877 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "reached_pg", 2023-06-03T20:20:35.877 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:17:50.062527+0000", 2023-06-03T20:20:35.878 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.000120451 2023-06-03T20:20:35.878 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:20:35.878 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:20:35.878 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "started", 2023-06-03T20:20:35.878 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:17:50.062647+0000", 2023-06-03T20:20:35.878 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.00020754399999999999 2023-06-03T20:20:35.878 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:20:35.878 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:20:35.879 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "waiting for subops from 2,3", 2023-06-03T20:20:35.879 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:17:50.062855+0000", 2023-06-03T20:20:35.879 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.0020108610000000001 2023-06-03T20:20:35.879 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:20:35.879 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:20:35.879 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "op_commit", 2023-06-03T20:20:35.879 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:17:50.064865+0000", 2023-06-03T20:20:35.880 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.0073364820000000001 2023-06-03T20:20:35.880 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:20:35.880 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:20:35.880 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:20:35.880 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:17:50.072202+0000", 2023-06-03T20:20:35.880 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.26572790400000001 2023-06-03T20:20:35.880 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:20:35.880 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:20:35.881 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:20:35.881 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:17:50.337930+0000", 2023-06-03T20:20:35.881 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 5.2722000000000001e-05 2023-06-03T20:20:35.881 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:20:35.881 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:20:35.881 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "commit_sent", 2023-06-03T20:20:35.881 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:17:50.337982+0000", 2023-06-03T20:20:35.882 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 3.5743e-05 2023-06-03T20:20:35.882 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:20:35.882 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:20:35.882 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "done", 2023-06-03T20:20:35.882 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:17:50.338018+0000", 2023-06-03T20:20:35.882 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.27565993 2023-06-03T20:20:35.882 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:20:35.882 INFO:teuthology.orchestra.run.smithi164.stdout: ] 2023-06-03T20:20:35.883 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:20:35.883 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:20:35.883 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:20:35.883 INFO:teuthology.orchestra.run.smithi164.stdout: "description": "osd_op(client.46373.0:53685 12.d 12:b04155c9:::benchmark_data_smithi089_191647_object53684:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e224)", 2023-06-03T20:20:35.883 INFO:teuthology.orchestra.run.smithi164.stdout: "initiated_at": "2023-06-03T20:17:50.064853+0000", 2023-06-03T20:20:35.883 INFO:teuthology.orchestra.run.smithi164.stdout: "age": 165.64826707399999, 2023-06-03T20:20:35.883 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.27342903800000001, 2023-06-03T20:20:35.883 INFO:teuthology.orchestra.run.smithi164.stdout: "type_data": { 2023-06-03T20:20:35.884 INFO:teuthology.orchestra.run.smithi164.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-03T20:20:35.884 INFO:teuthology.orchestra.run.smithi164.stdout: "client_info": { 2023-06-03T20:20:35.884 INFO:teuthology.orchestra.run.smithi164.stdout: "client": "client.46373", 2023-06-03T20:20:35.884 INFO:teuthology.orchestra.run.smithi164.stdout: "client_addr": "172.21.15.89:0/3737977806", 2023-06-03T20:20:35.884 INFO:teuthology.orchestra.run.smithi164.stdout: "tid": 53685 2023-06-03T20:20:35.884 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:20:35.884 INFO:teuthology.orchestra.run.smithi164.stdout: "events": [ 2023-06-03T20:20:35.884 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:20:35.885 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "initiated", 2023-06-03T20:20:35.885 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:17:50.064853+0000", 2023-06-03T20:20:35.885 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0 2023-06-03T20:20:35.885 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:20:35.885 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:20:35.885 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "throttled", 2023-06-03T20:20:35.885 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:17:50.064853+0000", 2023-06-03T20:20:35.886 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 4294967295.9999981 2023-06-03T20:20:35.886 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:20:35.886 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:20:35.886 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "header_read", 2023-06-03T20:20:35.886 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:17:50.064851+0000", 2023-06-03T20:20:35.886 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 9.6500000000000001e-05 2023-06-03T20:20:35.886 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:20:35.886 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:20:35.887 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "all_read", 2023-06-03T20:20:35.887 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:17:50.064947+0000", 2023-06-03T20:20:35.887 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 1.8763999999999999e-05 2023-06-03T20:20:35.887 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:20:35.887 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:20:35.887 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "dispatched", 2023-06-03T20:20:35.887 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:17:50.064966+0000", 2023-06-03T20:20:35.887 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 8.7309999999999999e-06 2023-06-03T20:20:35.888 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:20:35.888 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:20:35.888 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "queued_for_pg", 2023-06-03T20:20:35.888 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:17:50.064975+0000", 2023-06-03T20:20:35.888 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 4.3838999999999998e-05 2023-06-03T20:20:35.888 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:20:35.888 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:20:35.889 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "reached_pg", 2023-06-03T20:20:35.889 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:17:50.065019+0000", 2023-06-03T20:20:35.889 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 9.3026e-05 2023-06-03T20:20:35.889 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:20:35.889 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:20:35.889 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "started", 2023-06-03T20:20:35.889 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:17:50.065112+0000", 2023-06-03T20:20:35.889 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.000108601 2023-06-03T20:20:35.890 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:20:35.890 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:20:35.890 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "waiting for subops from 2,3", 2023-06-03T20:20:35.890 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:17:50.065220+0000", 2023-06-03T20:20:35.890 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.0016180529999999999 2023-06-03T20:20:35.890 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:20:35.890 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:20:35.890 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "op_commit", 2023-06-03T20:20:35.891 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:17:50.066838+0000", 2023-06-03T20:20:35.891 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.0054107130000000002 2023-06-03T20:20:35.891 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:20:35.891 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:20:35.891 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:20:35.891 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:17:50.072249+0000", 2023-06-03T20:20:35.891 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.265859556 2023-06-03T20:20:35.891 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:20:35.892 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:20:35.892 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:20:35.892 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:17:50.338108+0000", 2023-06-03T20:20:35.892 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.000109067 2023-06-03T20:20:35.892 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:20:35.892 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:20:35.892 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "commit_sent", 2023-06-03T20:20:35.892 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:17:50.338218+0000", 2023-06-03T20:20:35.893 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 6.4000999999999999e-05 2023-06-03T20:20:35.893 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:20:35.893 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:20:35.893 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "done", 2023-06-03T20:20:35.893 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:17:50.338282+0000", 2023-06-03T20:20:35.893 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.27342903800000001 2023-06-03T20:20:35.893 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:20:35.894 INFO:teuthology.orchestra.run.smithi164.stdout: ] 2023-06-03T20:20:35.894 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:20:35.894 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:20:35.894 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:20:35.894 INFO:teuthology.orchestra.run.smithi164.stdout: "description": "osd_op(client.46373.0:53693 12.7 12:e7e1a137:::benchmark_data_smithi089_191647_object53692:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e224)", 2023-06-03T20:20:35.894 INFO:teuthology.orchestra.run.smithi164.stdout: "initiated_at": "2023-06-03T20:17:50.071660+0000", 2023-06-03T20:20:35.894 INFO:teuthology.orchestra.run.smithi164.stdout: "age": 165.641460024, 2023-06-03T20:20:35.894 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.27451203299999999, 2023-06-03T20:20:35.895 INFO:teuthology.orchestra.run.smithi164.stdout: "type_data": { 2023-06-03T20:20:35.895 INFO:teuthology.orchestra.run.smithi164.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-03T20:20:35.895 INFO:teuthology.orchestra.run.smithi164.stdout: "client_info": { 2023-06-03T20:20:35.895 INFO:teuthology.orchestra.run.smithi164.stdout: "client": "client.46373", 2023-06-03T20:20:35.895 INFO:teuthology.orchestra.run.smithi164.stdout: "client_addr": "172.21.15.89:0/3737977806", 2023-06-03T20:20:35.895 INFO:teuthology.orchestra.run.smithi164.stdout: "tid": 53693 2023-06-03T20:20:35.895 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:20:35.895 INFO:teuthology.orchestra.run.smithi164.stdout: "events": [ 2023-06-03T20:20:35.896 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:20:35.896 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "initiated", 2023-06-03T20:20:35.896 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:17:50.071660+0000", 2023-06-03T20:20:35.896 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0 2023-06-03T20:20:35.896 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:20:35.896 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:20:35.896 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "throttled", 2023-06-03T20:20:35.897 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:17:50.071660+0000", 2023-06-03T20:20:35.897 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 4294967295.9999981 2023-06-03T20:20:35.897 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:20:35.897 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:20:35.897 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "header_read", 2023-06-03T20:20:35.897 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:17:50.071658+0000", 2023-06-03T20:20:35.897 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 6.2694000000000007e-05 2023-06-03T20:20:35.897 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:20:35.898 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:20:35.898 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "all_read", 2023-06-03T20:20:35.898 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:17:50.071720+0000", 2023-06-03T20:20:35.898 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 1.6758000000000001e-05 2023-06-03T20:20:35.898 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:20:35.898 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:20:35.898 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "dispatched", 2023-06-03T20:20:35.898 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:17:50.071737+0000", 2023-06-03T20:20:35.899 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 7.8679999999999999e-06 2023-06-03T20:20:35.899 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:20:35.899 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:20:35.899 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "queued_for_pg", 2023-06-03T20:20:35.899 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:17:50.071745+0000", 2023-06-03T20:20:35.899 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 4.905e-05 2023-06-03T20:20:35.899 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:20:35.900 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:20:35.900 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "reached_pg", 2023-06-03T20:20:35.900 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:17:50.071794+0000", 2023-06-03T20:20:35.900 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.00018128600000000001 2023-06-03T20:20:35.900 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:20:35.900 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:20:35.900 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "started", 2023-06-03T20:20:35.900 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:17:50.071975+0000", 2023-06-03T20:20:35.901 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.00017199199999999999 2023-06-03T20:20:35.901 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:20:35.901 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:20:35.901 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "waiting for subops from 2", 2023-06-03T20:20:35.901 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:17:50.072147+0000", 2023-06-03T20:20:35.901 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.0021150639999999998 2023-06-03T20:20:35.901 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:20:35.901 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:20:35.902 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "op_commit", 2023-06-03T20:20:35.902 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:17:50.074262+0000", 2023-06-03T20:20:35.902 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.27133335200000003 2023-06-03T20:20:35.902 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:20:35.902 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:20:35.902 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:20:35.902 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:17:50.345596+0000", 2023-06-03T20:20:35.902 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.00030316799999999998 2023-06-03T20:20:35.903 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:20:35.903 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:20:35.903 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "commit_sent", 2023-06-03T20:20:35.903 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:17:50.345899+0000", 2023-06-03T20:20:35.903 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.000272816 2023-06-03T20:20:35.903 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:20:35.903 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:20:35.903 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "done", 2023-06-03T20:20:35.904 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:17:50.346172+0000", 2023-06-03T20:20:35.904 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.27451203299999999 2023-06-03T20:20:35.904 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:20:35.904 INFO:teuthology.orchestra.run.smithi164.stdout: ] 2023-06-03T20:20:35.904 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:20:35.904 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:20:35.904 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:20:35.904 INFO:teuthology.orchestra.run.smithi164.stdout: "description": "osd_op(client.46373.0:53700 12.d 12:bc5fa220:::benchmark_data_smithi089_191647_object53699:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e224)", 2023-06-03T20:20:35.905 INFO:teuthology.orchestra.run.smithi164.stdout: "initiated_at": "2023-06-03T20:17:50.084325+0000", 2023-06-03T20:20:35.905 INFO:teuthology.orchestra.run.smithi164.stdout: "age": 165.628794567, 2023-06-03T20:20:35.905 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.26224864599999997, 2023-06-03T20:20:35.905 INFO:teuthology.orchestra.run.smithi164.stdout: "type_data": { 2023-06-03T20:20:35.905 INFO:teuthology.orchestra.run.smithi164.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-03T20:20:35.905 INFO:teuthology.orchestra.run.smithi164.stdout: "client_info": { 2023-06-03T20:20:35.905 INFO:teuthology.orchestra.run.smithi164.stdout: "client": "client.46373", 2023-06-03T20:20:35.905 INFO:teuthology.orchestra.run.smithi164.stdout: "client_addr": "172.21.15.89:0/3737977806", 2023-06-03T20:20:35.906 INFO:teuthology.orchestra.run.smithi164.stdout: "tid": 53700 2023-06-03T20:20:35.906 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:20:35.906 INFO:teuthology.orchestra.run.smithi164.stdout: "events": [ 2023-06-03T20:20:35.906 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:20:35.906 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "initiated", 2023-06-03T20:20:35.906 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:17:50.084325+0000", 2023-06-03T20:20:35.906 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0 2023-06-03T20:20:35.907 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:20:35.907 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:20:35.907 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "throttled", 2023-06-03T20:20:35.907 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:17:50.084325+0000", 2023-06-03T20:20:35.907 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 4294967295.9999981 2023-06-03T20:20:35.907 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:20:35.907 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:20:35.907 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "header_read", 2023-06-03T20:20:35.908 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:17:50.084323+0000", 2023-06-03T20:20:35.908 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 3.9051999999999998e-05 2023-06-03T20:20:35.908 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:20:35.908 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:20:35.908 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "all_read", 2023-06-03T20:20:35.908 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:17:50.084362+0000", 2023-06-03T20:20:35.908 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 2.0472e-05 2023-06-03T20:20:35.908 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:20:35.909 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:20:35.909 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "dispatched", 2023-06-03T20:20:35.909 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:17:50.084383+0000", 2023-06-03T20:20:35.909 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 1.3942e-05 2023-06-03T20:20:35.909 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:20:35.909 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:20:35.909 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "queued_for_pg", 2023-06-03T20:20:35.909 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:17:50.084396+0000", 2023-06-03T20:20:35.910 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 4.7311999999999997e-05 2023-06-03T20:20:35.910 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:20:35.910 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:20:35.910 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "reached_pg", 2023-06-03T20:20:35.910 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:17:50.084444+0000", 2023-06-03T20:20:35.910 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.000175947 2023-06-03T20:20:35.910 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:20:35.910 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:20:35.911 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "started", 2023-06-03T20:20:35.911 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:17:50.084620+0000", 2023-06-03T20:20:35.911 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.00024496400000000002 2023-06-03T20:20:35.911 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:20:35.911 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:20:35.911 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "waiting for subops from 2,3", 2023-06-03T20:20:35.911 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:17:50.084865+0000", 2023-06-03T20:20:35.911 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.0034482390000000001 2023-06-03T20:20:35.912 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:20:35.912 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:20:35.912 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:20:35.912 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:17:50.088313+0000", 2023-06-03T20:20:35.912 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 9.5483999999999996e-05 2023-06-03T20:20:35.912 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:20:35.912 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:20:35.913 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "op_commit", 2023-06-03T20:20:35.913 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:17:50.088408+0000", 2023-06-03T20:20:35.913 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.25807205799999999 2023-06-03T20:20:35.913 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:20:35.913 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:20:35.913 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:20:35.913 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:17:50.346480+0000", 2023-06-03T20:20:35.913 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 4.3943000000000003e-05 2023-06-03T20:20:35.914 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:20:35.914 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:20:35.914 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "commit_sent", 2023-06-03T20:20:35.914 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:17:50.346524+0000", 2023-06-03T20:20:35.914 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 4.9245e-05 2023-06-03T20:20:35.914 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:20:35.914 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:20:35.914 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "done", 2023-06-03T20:20:35.915 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:17:50.346574+0000", 2023-06-03T20:20:35.915 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.26224864599999997 2023-06-03T20:20:35.915 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:20:35.915 INFO:teuthology.orchestra.run.smithi164.stdout: ] 2023-06-03T20:20:35.915 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:20:35.915 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:20:35.915 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:20:35.915 INFO:teuthology.orchestra.run.smithi164.stdout: "description": "osd_op(client.46373.0:105812 12.d 12:b7fe3a01:::benchmark_data_smithi089_191647_object203:head [delete] snapc 0=[] ondisk+write+known_if_redirected e226)", 2023-06-03T20:20:35.916 INFO:teuthology.orchestra.run.smithi164.stdout: "initiated_at": "2023-06-03T20:18:47.617893+0000", 2023-06-03T20:20:35.916 INFO:teuthology.orchestra.run.smithi164.stdout: "age": 108.095226829, 2023-06-03T20:20:35.916 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.24625012399999999, 2023-06-03T20:20:35.916 INFO:teuthology.orchestra.run.smithi164.stdout: "type_data": { 2023-06-03T20:20:35.916 INFO:teuthology.orchestra.run.smithi164.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-03T20:20:35.916 INFO:teuthology.orchestra.run.smithi164.stdout: "client_info": { 2023-06-03T20:20:35.916 INFO:teuthology.orchestra.run.smithi164.stdout: "client": "client.46373", 2023-06-03T20:20:35.917 INFO:teuthology.orchestra.run.smithi164.stdout: "client_addr": "172.21.15.89:0/3737977806", 2023-06-03T20:20:35.917 INFO:teuthology.orchestra.run.smithi164.stdout: "tid": 105812 2023-06-03T20:20:35.917 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:20:35.917 INFO:teuthology.orchestra.run.smithi164.stdout: "events": [ 2023-06-03T20:20:35.917 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:20:35.917 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "initiated", 2023-06-03T20:20:35.917 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:18:47.617893+0000", 2023-06-03T20:20:35.917 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0 2023-06-03T20:20:35.918 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:20:35.918 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:20:35.918 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "throttled", 2023-06-03T20:20:35.918 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:18:47.617893+0000", 2023-06-03T20:20:35.918 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 4294967295.9999976 2023-06-03T20:20:35.918 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:20:35.918 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:20:35.919 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "header_read", 2023-06-03T20:20:35.919 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:18:47.617890+0000", 2023-06-03T20:20:35.919 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 1.0424e-05 2023-06-03T20:20:35.919 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:20:35.919 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:20:35.919 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "all_read", 2023-06-03T20:20:35.919 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:18:47.617901+0000", 2023-06-03T20:20:35.919 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 1.6702999999999999e-05 2023-06-03T20:20:35.920 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:20:35.920 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:20:35.920 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "dispatched", 2023-06-03T20:20:35.920 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:18:47.617918+0000", 2023-06-03T20:20:35.920 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 9.7219999999999994e-06 2023-06-03T20:20:35.920 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:20:35.920 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:20:35.920 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "queued_for_pg", 2023-06-03T20:20:35.921 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:18:47.617927+0000", 2023-06-03T20:20:35.921 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 5.8968000000000002e-05 2023-06-03T20:20:35.921 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:20:35.921 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:20:35.921 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "reached_pg", 2023-06-03T20:20:35.921 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:18:47.617986+0000", 2023-06-03T20:20:35.921 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.000116347 2023-06-03T20:20:35.921 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:20:35.922 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:20:35.922 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "started", 2023-06-03T20:20:35.922 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:18:47.618103+0000", 2023-06-03T20:20:35.922 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.000136618 2023-06-03T20:20:35.922 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:20:35.922 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:20:35.922 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "waiting for subops from 2,3", 2023-06-03T20:20:35.922 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:18:47.618239+0000", 2023-06-03T20:20:35.923 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.0043872499999999997 2023-06-03T20:20:35.923 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:20:35.923 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:20:35.923 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:20:35.923 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:18:47.622627+0000", 2023-06-03T20:20:35.923 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.00013117200000000001 2023-06-03T20:20:35.923 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:20:35.923 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:20:35.924 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "op_commit", 2023-06-03T20:20:35.924 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:18:47.622758+0000", 2023-06-03T20:20:35.924 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.24107920099999999 2023-06-03T20:20:35.924 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:20:35.924 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:20:35.924 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:20:35.924 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:18:47.863837+0000", 2023-06-03T20:20:35.925 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.000169908 2023-06-03T20:20:35.925 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:20:35.925 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:20:35.925 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "commit_sent", 2023-06-03T20:20:35.925 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:18:47.864007+0000", 2023-06-03T20:20:35.925 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.000136098 2023-06-03T20:20:35.925 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:20:35.925 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:20:35.926 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "done", 2023-06-03T20:20:35.926 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:18:47.864143+0000", 2023-06-03T20:20:35.926 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.24625012399999999 2023-06-03T20:20:35.926 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:20:35.926 INFO:teuthology.orchestra.run.smithi164.stdout: ] 2023-06-03T20:20:35.926 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:20:35.926 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:20:35.926 INFO:teuthology.orchestra.run.smithi164.stdout: ] 2023-06-03T20:20:35.927 INFO:teuthology.orchestra.run.smithi164.stdout:} 2023-06-03T20:20:35.927 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:20:35.928 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:20:35.928 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:20:36.029 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:20:36.070 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:20:36.072 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:20:36.073 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:20:36.174 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:20:36.226 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:20:36.229 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:20:36.230 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:20:36.331 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:20:36.361 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:20:36.363 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:20:36.364 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:20:36.446 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/mon.a/config 2023-06-03T20:20:36.465 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:20:36.478 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 20:20:36 smithi089 bash[71163]: ::ffff:172.21.15.164 - - [03/Jun/2023:20:20:36] "GET /metrics HTTP/1.1" 200 228944 "" "Prometheus/2.33.4" 2023-06-03T20:20:36.494 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:20:36.496 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:20:36.497 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:20:36.598 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:20:36.628 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:20:36.630 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:20:36.630 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:20:36.665 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph daemon osd.0 dump_ops_in_flight 2023-06-03T20:20:36.732 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:20:36.777 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:20:36.782 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:20:36.783 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:20:36.884 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:20:36.919 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:20:36.921 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:20:36.922 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:20:37.023 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:20:37.060 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:20:37.062 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:20:37.063 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:20:37.164 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:20:37.209 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:20:37.210 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:20:37.211 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:20:37.312 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:20:37.351 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:20:37.356 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:20:37.357 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:20:37.458 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:20:37.489 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:20:37.491 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:20:37.492 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:20:37.593 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:20:37.625 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:20:37.632 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:20:37.633 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:20:37.734 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:20:37.767 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:20:37.769 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:20:37.770 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:20:37.871 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:20:37.902 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:20:37.907 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:20:37.908 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:20:38.009 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:20:38.040 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:20:38.046 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:20:38.047 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:20:38.148 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:20:38.157 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:20:37 smithi164 bash[59462]: cluster 1685823636.9400382 mgr.y (mgr.34104) 1340 : cluster 0 pgmap v1341: 99 pgs: 20 active+clean+remapped, 19 active+undersized+degraded, 11 active+undersized, 49 active+clean; 749 MiB data, 5.7 GiB used, 531 GiB / 536 GiB avail; 0 B/s wr, 989 op/s; 6138/35982 objects degraded (17.059%); 6/35982 objects misplaced (0.017%) 2023-06-03T20:20:38.157 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:20:37 smithi164 bash[59462]: audit 1685823637.1324182 mon.c (mon.2) 576 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T20:20:38.157 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:20:37 smithi164 bash[59462]: cluster 1685823637.8714097 mon.a (mon.0) 1314 : cluster 3 Health check update: Degraded data redundancy: 6138/35982 objects degraded (17.059%), 19 pgs degraded, 30 pgs undersized (PG_DEGRADED) 2023-06-03T20:20:38.177 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:20:38.181 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:20:38.182 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:20:38.279 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:20:37 smithi089 bash[78452]: cluster 1685823636.9400382 mgr.y (mgr.34104) 1340 : cluster 0 pgmap v1341: 99 pgs: 20 active+clean+remapped, 19 active+undersized+degraded, 11 active+undersized, 49 active+clean; 749 MiB data, 5.7 GiB used, 531 GiB / 536 GiB avail; 0 B/s wr, 989 op/s; 6138/35982 objects degraded (17.059%); 6/35982 objects misplaced (0.017%) 2023-06-03T20:20:38.279 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:20:37 smithi089 bash[78452]: audit 1685823637.1324182 mon.c (mon.2) 576 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T20:20:38.279 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:20:37 smithi089 bash[78452]: cluster 1685823637.8714097 mon.a (mon.0) 1314 : cluster 3 Health check update: Degraded data redundancy: 6138/35982 objects degraded (17.059%), 19 pgs degraded, 30 pgs undersized (PG_DEGRADED) 2023-06-03T20:20:38.279 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:20:37 smithi089 bash[75068]: cluster 1685823636.9400382 mgr.y (mgr.34104) 1340 : cluster 0 pgmap v1341: 99 pgs: 20 active+clean+remapped, 19 active+undersized+degraded, 11 active+undersized, 49 active+clean; 749 MiB data, 5.7 GiB used, 531 GiB / 536 GiB avail; 0 B/s wr, 989 op/s; 6138/35982 objects degraded (17.059%); 6/35982 objects misplaced (0.017%) 2023-06-03T20:20:38.280 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:20:37 smithi089 bash[75068]: audit 1685823637.1324182 mon.c (mon.2) 576 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T20:20:38.280 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:20:37 smithi089 bash[75068]: cluster 1685823637.8714097 mon.a (mon.0) 1314 : cluster 3 Health check update: Degraded data redundancy: 6138/35982 objects degraded (17.059%), 19 pgs degraded, 30 pgs undersized (PG_DEGRADED) 2023-06-03T20:20:38.282 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:20:38.318 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:20:38.320 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:20:38.321 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:20:38.422 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:20:38.453 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:20:38.458 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:20:38.458 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:20:38.559 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:20:38.586 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:20:38.589 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:20:38.590 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:20:38.691 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:20:38.721 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:20:38.723 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:20:38.724 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:20:38.825 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:20:38.854 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:20:38.858 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:20:38.858 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:20:38.959 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:20:38.989 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:20:38.994 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:20:38.995 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:20:39.096 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:20:39.122 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:20:39.124 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:20:39.125 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:20:39.225 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:20:39.252 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:20:39.254 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:20:39.255 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:20:39.356 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:20:39.405 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:20:39.409 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:20:39.409 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:20:39.510 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:20:39.537 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:20:39.540 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/mon.a/config 2023-06-03T20:20:39.540 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:20:39.541 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:20:39.641 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:20:39.673 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:20:39.675 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:20:39.676 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:20:39.777 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:20:39.831 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:20:39.835 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:20:39.836 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:20:39.936 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:20:39.973 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:20:39.978 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:20:39.979 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:20:40.080 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:20:40.113 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:20:40.116 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:20:40.117 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:20:40.218 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:20:40.226 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:20:40 smithi089 bash[75068]: cluster 1685823638.9407246 mgr.y (mgr.34104) 1341 : cluster 0 pgmap v1342: 99 pgs: 20 active+clean+remapped, 19 active+undersized+degraded, 11 active+undersized, 49 active+clean; 716 MiB data, 5.5 GiB used, 531 GiB / 536 GiB avail; 0 B/s wr, 819 op/s; 5864/34416 objects degraded (17.039%); 6/34416 objects misplaced (0.017%) 2023-06-03T20:20:40.227 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:20:40 smithi089 bash[78452]: cluster 1685823638.9407246 mgr.y (mgr.34104) 1341 : cluster 0 pgmap v1342: 99 pgs: 20 active+clean+remapped, 19 active+undersized+degraded, 11 active+undersized, 49 active+clean; 716 MiB data, 5.5 GiB used, 531 GiB / 536 GiB avail; 0 B/s wr, 819 op/s; 5864/34416 objects degraded (17.039%); 6/34416 objects misplaced (0.017%) 2023-06-03T20:20:40.252 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:20:40.254 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:20:40.255 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:20:40.334 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:20:40 smithi164 bash[59462]: cluster 1685823638.9407246 mgr.y (mgr.34104) 1341 : cluster 0 pgmap v1342: 99 pgs: 20 active+clean+remapped, 19 active+undersized+degraded, 11 active+undersized, 49 active+clean; 716 MiB data, 5.5 GiB used, 531 GiB / 536 GiB avail; 0 B/s wr, 819 op/s; 5864/34416 objects degraded (17.039%); 6/34416 objects misplaced (0.017%) 2023-06-03T20:20:40.356 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:20:40.396 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:20:40.398 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:20:40.399 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:20:40.500 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:20:40.537 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:20:40.541 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:20:40.542 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:20:40.642 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:20:40.672 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:20:40.674 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:20:40.675 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:20:40.776 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:20:40.805 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:20:40.808 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:20:40.808 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:20:40.909 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:20:40.941 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:20:40.945 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:20:40.946 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:20:41.047 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:20:41.087 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:20:41.090 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:20:41.091 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:20:41.192 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:20:41.223 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:20:41.226 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:20:41.227 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:20:41.327 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:20:41.360 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:20:41.362 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:20:41.363 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:20:41.464 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:20:41.500 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:20:41.504 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:20:41.504 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:20:41.593 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/mon.a/config 2023-06-03T20:20:41.605 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:20:41.650 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:20:41.656 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:20:41.657 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:20:41.757 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:20:41.791 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:20:41.794 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:20:41.794 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:20:41.895 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:20:41.935 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:20:41.938 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:20:41.938 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:20:42.039 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:20:42.052 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:20:42 smithi164 bash[59462]: cluster 1685823640.9416103 mgr.y (mgr.34104) 1342 : cluster 0 pgmap v1343: 99 pgs: 20 active+clean+remapped, 19 active+undersized+degraded, 11 active+undersized, 49 active+clean; 534 MiB data, 5.2 GiB used, 531 GiB / 536 GiB avail; 0 B/s wr, 1.02k op/s; 4136/25689 objects degraded (16.100%); 6/25689 objects misplaced (0.023%) 2023-06-03T20:20:42.078 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:20:42.082 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:20:42.082 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:20:42.183 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:20:42.203 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:20:42.205 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:20:42.205 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:20:42.251 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:20:42 smithi089 bash[75068]: cluster 1685823640.9416103 mgr.y (mgr.34104) 1342 : cluster 0 pgmap v1343: 99 pgs: 20 active+clean+remapped, 19 active+undersized+degraded, 11 active+undersized, 49 active+clean; 534 MiB data, 5.2 GiB used, 531 GiB / 536 GiB avail; 0 B/s wr, 1.02k op/s; 4136/25689 objects degraded (16.100%); 6/25689 objects misplaced (0.023%) 2023-06-03T20:20:42.251 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:20:42 smithi089 bash[78452]: cluster 1685823640.9416103 mgr.y (mgr.34104) 1342 : cluster 0 pgmap v1343: 99 pgs: 20 active+clean+remapped, 19 active+undersized+degraded, 11 active+undersized, 49 active+clean; 534 MiB data, 5.2 GiB used, 531 GiB / 536 GiB avail; 0 B/s wr, 1.02k op/s; 4136/25689 objects degraded (16.100%); 6/25689 objects misplaced (0.023%) 2023-06-03T20:20:42.306 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:20:42.337 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:20:42.340 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:20:42.341 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:20:42.442 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:20:42.473 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:20:42.476 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:20:42.476 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:20:42.577 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:20:42.623 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:20:42.626 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:20:42.626 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:20:42.727 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:20:42.756 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:20:42.762 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:20:42.762 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:20:42.863 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:20:42.894 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:20:42.899 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:20:42.900 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:20:43.001 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:20:43.033 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:20:43.035 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:20:43.036 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:20:43.137 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:20:43.147 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:20:43 smithi164 bash[59462]: cluster 1685823642.8725386 mon.a (mon.0) 1315 : cluster 3 Health check update: Degraded data redundancy: 4136/25689 objects degraded (16.100%), 19 pgs degraded, 30 pgs undersized (PG_DEGRADED) 2023-06-03T20:20:43.168 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:20:43.171 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:20:43.172 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:20:43.273 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:20:43.279 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:20:43 smithi089 bash[75068]: cluster 1685823642.8725386 mon.a (mon.0) 1315 : cluster 3 Health check update: Degraded data redundancy: 4136/25689 objects degraded (16.100%), 19 pgs degraded, 30 pgs undersized (PG_DEGRADED) 2023-06-03T20:20:43.279 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:20:43 smithi089 bash[78452]: cluster 1685823642.8725386 mon.a (mon.0) 1315 : cluster 3 Health check update: Degraded data redundancy: 4136/25689 objects degraded (16.100%), 19 pgs degraded, 30 pgs undersized (PG_DEGRADED) 2023-06-03T20:20:43.310 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:20:43.312 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:20:43.313 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:20:43.414 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:20:43.452 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:20:43.457 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:20:43.457 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:20:43.558 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:20:43.595 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:20:43.598 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:20:43.599 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:20:43.700 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:20:43.733 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:20:43.737 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:20:43.738 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:20:43.839 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:20:43.872 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:20:43.874 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:20:43.875 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:20:43.976 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:20:44.009 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:20:44.014 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:20:44.015 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:20:44.102 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:20:44 smithi089 bash[75068]: cluster 1685823642.9422908 mgr.y (mgr.34104) 1343 : cluster 0 pgmap v1344: 99 pgs: 20 active+clean+remapped, 19 active+undersized+degraded, 11 active+undersized, 49 active+clean; 461 MiB data, 5.1 GiB used, 531 GiB / 536 GiB avail; 0 B/s wr, 867 op/s; 3840/22191 objects degraded (17.304%); 6/22191 objects misplaced (0.027%) 2023-06-03T20:20:44.102 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:20:44 smithi089 bash[78452]: cluster 1685823642.9422908 mgr.y (mgr.34104) 1343 : cluster 0 pgmap v1344: 99 pgs: 20 active+clean+remapped, 19 active+undersized+degraded, 11 active+undersized, 49 active+clean; 461 MiB data, 5.1 GiB used, 531 GiB / 536 GiB avail; 0 B/s wr, 867 op/s; 3840/22191 objects degraded (17.304%); 6/22191 objects misplaced (0.027%) 2023-06-03T20:20:44.116 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:20:44.151 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:20:44.154 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:20:44.155 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:20:44.256 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:20:44.287 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:20:44.290 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:20:44.290 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:20:44.334 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:20:44 smithi164 bash[59462]: cluster 1685823642.9422908 mgr.y (mgr.34104) 1343 : cluster 0 pgmap v1344: 99 pgs: 20 active+clean+remapped, 19 active+undersized+degraded, 11 active+undersized, 49 active+clean; 461 MiB data, 5.1 GiB used, 531 GiB / 536 GiB avail; 0 B/s wr, 867 op/s; 3840/22191 objects degraded (17.304%); 6/22191 objects misplaced (0.027%) 2023-06-03T20:20:44.391 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:20:44.420 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:20:44.423 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:20:44.423 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:20:44.524 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:20:44.552 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:20:44.556 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:20:44.557 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:20:44.658 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:20:44.689 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:20:44.692 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:20:44.693 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:20:44.793 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:20:44.822 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:20:44.825 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:20:44.826 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:20:44.928 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:20:44.968 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:20:44.971 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:20:44.972 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:20:45.073 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:20:45.103 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:20:45.106 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:20:45.107 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:20:45.208 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:20:45.239 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:20:45.240 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:20:45.241 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:20:45.342 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:20:45.379 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:20:45.384 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:20:45.385 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:20:45.486 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:20:45.520 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:20:45.523 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:20:45.523 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:20:45.624 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:20:45.654 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:20:45.655 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:20:45.656 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:20:45.757 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:20:45.793 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:20:45.803 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:20:45.803 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:20:45.904 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:20:45.944 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:20:45.947 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:20:45.948 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:20:46.049 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:20:46.081 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:20:46.084 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:20:46.085 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:20:46.186 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:20:46.228 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:20:46.230 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:20:46.231 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:20:46.323 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:20:46 smithi089 bash[78452]: cluster 1685823644.9431713 mgr.y (mgr.34104) 1344 : cluster 0 pgmap v1345: 99 pgs: 20 active+clean+remapped, 19 active+undersized+degraded, 11 active+undersized, 49 active+clean; 341 MiB data, 4.7 GiB used, 532 GiB / 536 GiB avail; 0 B/s wr, 928 op/s; 2731/16422 objects degraded (16.630%); 6/16422 objects misplaced (0.037%) 2023-06-03T20:20:46.324 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:20:46 smithi089 bash[75068]: cluster 1685823644.9431713 mgr.y (mgr.34104) 1344 : cluster 0 pgmap v1345: 99 pgs: 20 active+clean+remapped, 19 active+undersized+degraded, 11 active+undersized, 49 active+clean; 341 MiB data, 4.7 GiB used, 532 GiB / 536 GiB avail; 0 B/s wr, 928 op/s; 2731/16422 objects degraded (16.630%); 6/16422 objects misplaced (0.037%) 2023-06-03T20:20:46.332 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:20:46.334 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:20:46 smithi164 bash[59462]: cluster 1685823644.9431713 mgr.y (mgr.34104) 1344 : cluster 0 pgmap v1345: 99 pgs: 20 active+clean+remapped, 19 active+undersized+degraded, 11 active+undersized, 49 active+clean; 341 MiB data, 4.7 GiB used, 532 GiB / 536 GiB avail; 0 B/s wr, 928 op/s; 2731/16422 objects degraded (16.630%); 6/16422 objects misplaced (0.037%) 2023-06-03T20:20:46.375 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:20:46.380 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:20:46.380 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:20:46.481 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:20:46.517 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:20:46.519 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:20:46.520 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:20:46.620 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:20:46.629 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 20:20:46 smithi089 bash[71163]: ::ffff:172.21.15.164 - - [03/Jun/2023:20:20:46] "GET /metrics HTTP/1.1" 200 228962 "" "Prometheus/2.33.4" 2023-06-03T20:20:46.650 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:20:46.653 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:20:46.653 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:20:46.754 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:20:46.788 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:20:46.791 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:20:46.791 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:20:46.892 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:20:46.929 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:20:46.932 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:20:46.933 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:20:47.034 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:20:47.062 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:20:47.064 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:20:47.065 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:20:47.165 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:20:47.209 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:20:47.211 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:20:47.212 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:20:47.312 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:20:47.356 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:20:47.358 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:20:47.359 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:20:47.460 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:20:47.495 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:20:47.498 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:20:47.499 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:20:47.600 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:20:47.633 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:20:47.638 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:20:47.638 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:20:47.739 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:20:47.770 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:20:47.772 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:20:47.773 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:20:47.874 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:20:47.913 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:20:47.916 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:20:47.917 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:20:48.017 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:20:48.019 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:20:48 smithi089 bash[78452]: cluster 1685823646.9444807 mgr.y (mgr.34104) 1345 : cluster 0 pgmap v1346: 99 pgs: 20 active+clean+remapped, 19 active+undersized+degraded, 11 active+undersized, 49 active+clean; 258 MiB data, 4.5 GiB used, 532 GiB / 536 GiB avail; 0 B/s wr, 866 op/s; 1919/12435 objects degraded (15.432%); 6/12435 objects misplaced (0.048%) 2023-06-03T20:20:48.020 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:20:48 smithi089 bash[78452]: cluster 1685823647.8737156 mon.a (mon.0) 1316 : cluster 3 Health check update: Degraded data redundancy: 1919/12435 objects degraded (15.432%), 19 pgs degraded, 30 pgs undersized (PG_DEGRADED) 2023-06-03T20:20:48.020 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:20:48 smithi089 bash[75068]: cluster 1685823646.9444807 mgr.y (mgr.34104) 1345 : cluster 0 pgmap v1346: 99 pgs: 20 active+clean+remapped, 19 active+undersized+degraded, 11 active+undersized, 49 active+clean; 258 MiB data, 4.5 GiB used, 532 GiB / 536 GiB avail; 0 B/s wr, 866 op/s; 1919/12435 objects degraded (15.432%); 6/12435 objects misplaced (0.048%) 2023-06-03T20:20:48.021 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:20:48 smithi089 bash[75068]: cluster 1685823647.8737156 mon.a (mon.0) 1316 : cluster 3 Health check update: Degraded data redundancy: 1919/12435 objects degraded (15.432%), 19 pgs degraded, 30 pgs undersized (PG_DEGRADED) 2023-06-03T20:20:48.049 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:20:48.055 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:20:48.056 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:20:48.157 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:20:48.169 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:20:48 smithi164 bash[59462]: cluster 1685823646.9444807 mgr.y (mgr.34104) 1345 : cluster 0 pgmap v1346: 99 pgs: 20 active+clean+remapped, 19 active+undersized+degraded, 11 active+undersized, 49 active+clean; 258 MiB data, 4.5 GiB used, 532 GiB / 536 GiB avail; 0 B/s wr, 866 op/s; 1919/12435 objects degraded (15.432%); 6/12435 objects misplaced (0.048%) 2023-06-03T20:20:48.170 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:20:48 smithi164 bash[59462]: cluster 1685823647.8737156 mon.a (mon.0) 1316 : cluster 3 Health check update: Degraded data redundancy: 1919/12435 objects degraded (15.432%), 19 pgs degraded, 30 pgs undersized (PG_DEGRADED) 2023-06-03T20:20:48.191 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:20:48.194 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:20:48.194 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:20:48.295 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:20:48.322 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:20:48.328 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:20:48.329 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:20:48.430 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:20:48.462 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:20:48.468 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:20:48.469 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:20:48.570 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:20:48.607 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:20:48.612 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:20:48.613 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:20:48.714 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:20:48.744 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:20:48.747 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:20:48.748 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:20:48.849 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:20:48.879 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:20:48.882 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:20:48.883 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:20:48.951 INFO:tasks.radosbench.radosbench.0.smithi089.stdout:Removed 105606 objects 2023-06-03T20:20:48.951 INFO:tasks.radosbench.radosbench.0.smithi089.stdout:Clean up completed and total clean up time :121.538 2023-06-03T20:20:48.984 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:20:49.022 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:20:49.025 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:20:49.025 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:20:49.127 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:20:49.160 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:20:49.163 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:20:49.164 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:20:49.266 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:20:49.286 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:20:49.289 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:20:49.289 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:20:49.390 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:20:49.418 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:20:49.421 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:20:49.422 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:20:49.522 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:20:49.569 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:20:49.572 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:20:49.573 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:20:49.673 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:20:49.707 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:20:49.710 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:20:49.711 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:20:49.811 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:20:49.840 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:20:49.842 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:20:49.843 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:20:49.944 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:20:49.981 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:20:49.983 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:20:49.984 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:20:50.085 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:20:50.119 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:20:50.121 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:20:50.122 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:20:50.223 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:20:50.253 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:20:50.254 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:20:50.255 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:20:50.279 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:20:50 smithi089 bash[75068]: cluster 1685823648.945284 mgr.y (mgr.34104) 1346 : cluster 0 pgmap v1347: 99 pgs: 20 active+clean+remapped, 19 active+undersized+degraded, 11 active+undersized, 49 active+clean; 162 MiB data, 4.2 GiB used, 532 GiB / 536 GiB avail; 0 B/s wr, 785 op/s; 1371/7839 objects degraded (17.489%); 6/7839 objects misplaced (0.077%) 2023-06-03T20:20:50.279 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:20:50 smithi089 bash[75068]: audit 1685823649.0679197 mon.c (mon.2) 577 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-03T20:20:50.279 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:20:50 smithi089 bash[78452]: cluster 1685823648.945284 mgr.y (mgr.34104) 1346 : cluster 0 pgmap v1347: 99 pgs: 20 active+clean+remapped, 19 active+undersized+degraded, 11 active+undersized, 49 active+clean; 162 MiB data, 4.2 GiB used, 532 GiB / 536 GiB avail; 0 B/s wr, 785 op/s; 1371/7839 objects degraded (17.489%); 6/7839 objects misplaced (0.077%) 2023-06-03T20:20:50.279 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:20:50 smithi089 bash[78452]: audit 1685823649.0679197 mon.c (mon.2) 577 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-03T20:20:50.334 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:20:50 smithi164 bash[59462]: cluster 1685823648.945284 mgr.y (mgr.34104) 1346 : cluster 0 pgmap v1347: 99 pgs: 20 active+clean+remapped, 19 active+undersized+degraded, 11 active+undersized, 49 active+clean; 162 MiB data, 4.2 GiB used, 532 GiB / 536 GiB avail; 0 B/s wr, 785 op/s; 1371/7839 objects degraded (17.489%); 6/7839 objects misplaced (0.077%) 2023-06-03T20:20:50.334 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:20:50 smithi164 bash[59462]: audit 1685823649.0679197 mon.c (mon.2) 577 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-03T20:20:50.356 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:20:50.386 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:20:50.388 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:20:50.388 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:20:50.489 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:20:50.519 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:20:50.522 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:20:50.522 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:20:50.623 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:20:50.653 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:20:50.655 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:20:50.656 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:20:50.756 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:20:50.795 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:20:50.797 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:20:50.798 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:20:50.899 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:20:50.902 INFO:tasks.cephadm.ceph_manager.ceph:removing pool_name unique_pool_7 2023-06-03T20:20:50.903 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph osd pool rm unique_pool_7 unique_pool_7 --yes-i-really-really-mean-it 2023-06-03T20:20:50.932 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:20:50.935 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:20:50.936 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:20:51.037 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:20:51.072 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:20:51.075 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:20:51.075 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:20:51.176 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:20:51.212 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:20:51.215 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:20:51.215 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:20:51.317 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:20:51.341 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:20:51.343 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:20:51.344 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:20:51.445 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:20:51.484 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:20:51.487 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:20:51.488 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:20:51.589 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:20:51.625 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:20:51.628 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:20:51.629 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:20:51.730 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:20:51.766 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:20:51.770 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:20:51.770 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:20:51.871 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:20:51.905 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:20:51.908 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:20:51.908 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:20:52.009 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:20:52.036 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:20:52.038 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:20:52.039 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:20:52.140 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:20:52.179 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:20:52.181 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:20:52.181 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:20:52.280 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:20:52 smithi089 bash[75068]: cluster 1685823650.9467807 mgr.y (mgr.34104) 1347 : cluster 0 pgmap v1348: 99 pgs: 20 active+clean+remapped, 15 active+undersized+degraded, 15 active+undersized, 49 active+clean; 38 MiB data, 3.9 GiB used, 532 GiB / 536 GiB avail; 0 B/s wr, 914 op/s; 211/1890 objects degraded (11.164%); 6/1890 objects misplaced (0.317%) 2023-06-03T20:20:52.280 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:20:52 smithi089 bash[78452]: cluster 1685823650.9467807 mgr.y (mgr.34104) 1347 : cluster 0 pgmap v1348: 99 pgs: 20 active+clean+remapped, 15 active+undersized+degraded, 15 active+undersized, 49 active+clean; 38 MiB data, 3.9 GiB used, 532 GiB / 536 GiB avail; 0 B/s wr, 914 op/s; 211/1890 objects degraded (11.164%); 6/1890 objects misplaced (0.317%) 2023-06-03T20:20:52.283 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:20:52.296 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:20:52 smithi164 bash[59462]: cluster 1685823650.9467807 mgr.y (mgr.34104) 1347 : cluster 0 pgmap v1348: 99 pgs: 20 active+clean+remapped, 15 active+undersized+degraded, 15 active+undersized, 49 active+clean; 38 MiB data, 3.9 GiB used, 532 GiB / 536 GiB avail; 0 B/s wr, 914 op/s; 211/1890 objects degraded (11.164%); 6/1890 objects misplaced (0.317%) 2023-06-03T20:20:52.321 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:20:52.324 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:20:52.324 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:20:52.425 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:20:52.461 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:20:52.464 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:20:52.465 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:20:52.566 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:20:52.602 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:20:52.605 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:20:52.605 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:20:52.706 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:20:52.744 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:20:52.746 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:20:52.747 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:20:52.848 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:20:52.873 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:20:52.875 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:20:52.876 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:20:52.977 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:20:53.007 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:20:53.009 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:20:53.009 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:20:53.110 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:20:53.145 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:20:53.147 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:20:53.148 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:20:53.222 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:20:53 smithi089 bash[75068]: audit 1685823652.1340935 mon.c (mon.2) 578 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T20:20:53.223 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:20:53 smithi089 bash[75068]: cluster 1685823652.8747394 mon.a (mon.0) 1317 : cluster 3 Health check update: Degraded data redundancy: 211/1890 objects degraded (11.164%), 15 pgs degraded, 30 pgs undersized (PG_DEGRADED) 2023-06-03T20:20:53.223 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:20:53 smithi089 bash[78452]: audit 1685823652.1340935 mon.c (mon.2) 578 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T20:20:53.223 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:20:53 smithi089 bash[78452]: cluster 1685823652.8747394 mon.a (mon.0) 1317 : cluster 3 Health check update: Degraded data redundancy: 211/1890 objects degraded (11.164%), 15 pgs degraded, 30 pgs undersized (PG_DEGRADED) 2023-06-03T20:20:53.250 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:20:53.276 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:20:53.278 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:20:53.278 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:20:53.334 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:20:53 smithi164 bash[59462]: audit 1685823652.1340935 mon.c (mon.2) 578 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T20:20:53.334 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:20:53 smithi164 bash[59462]: cluster 1685823652.8747394 mon.a (mon.0) 1317 : cluster 3 Health check update: Degraded data redundancy: 211/1890 objects degraded (11.164%), 15 pgs degraded, 30 pgs undersized (PG_DEGRADED) 2023-06-03T20:20:53.380 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:20:53.416 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:20:53.417 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:20:53.418 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:20:53.519 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:20:53.560 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:20:53.562 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:20:53.563 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:20:53.664 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:20:53.699 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:20:53.702 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:20:53.702 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:20:53.803 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:20:53.832 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:20:53.835 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:20:53.835 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:20:53.936 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:20:53.969 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:20:53.971 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:20:53.971 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:20:54.072 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:20:54.102 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:20:54.104 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:20:54.105 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:20:54.206 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:20:54.218 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:20:54 smithi089 bash[75068]: cluster 1685823652.947751 mgr.y (mgr.34104) 1348 : cluster 0 pgmap v1349: 99 pgs: 20 active+clean+remapped, 15 active+undersized+degraded, 15 active+undersized, 49 active+clean; 16 MiB data, 3.9 GiB used, 533 GiB / 536 GiB avail; 0 B/s wr, 706 op/s; 119/822 objects degraded (14.477%); 6/822 objects misplaced (0.730%) 2023-06-03T20:20:54.219 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:20:54 smithi089 bash[78452]: cluster 1685823652.947751 mgr.y (mgr.34104) 1348 : cluster 0 pgmap v1349: 99 pgs: 20 active+clean+remapped, 15 active+undersized+degraded, 15 active+undersized, 49 active+clean; 16 MiB data, 3.9 GiB used, 533 GiB / 536 GiB avail; 0 B/s wr, 706 op/s; 119/822 objects degraded (14.477%); 6/822 objects misplaced (0.730%) 2023-06-03T20:20:54.244 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:20:54.246 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:20:54.247 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:20:54.334 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:20:54 smithi164 bash[59462]: cluster 1685823652.947751 mgr.y (mgr.34104) 1348 : cluster 0 pgmap v1349: 99 pgs: 20 active+clean+remapped, 15 active+undersized+degraded, 15 active+undersized, 49 active+clean; 16 MiB data, 3.9 GiB used, 533 GiB / 536 GiB avail; 0 B/s wr, 706 op/s; 119/822 objects degraded (14.477%); 6/822 objects misplaced (0.730%) 2023-06-03T20:20:54.347 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:20:54.378 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:20:54.380 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:20:54.381 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:20:54.482 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:20:54.538 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:20:54.586 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:20:54.587 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:20:54.687 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:20:54.712 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:20:54.714 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:20:54.715 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:20:54.816 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:20:54.843 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:20:54.846 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:20:54.846 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:20:54.947 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:20:54.972 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:20:54.974 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:20:54.974 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:20:55.075 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:20:55.132 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:20:55.134 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:20:55.135 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:20:55.236 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:20:55.263 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:20:55.265 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:20:55.265 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:20:55.366 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:20:55.391 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:20:55.395 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:20:55.396 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:20:55.496 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:20:55.532 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:20:55.534 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:20:55.535 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:20:55.636 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:20:55.667 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:20:55.669 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:20:55.669 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:20:55.770 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:20:55.804 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:20:55.807 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:20:55.808 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:20:55.908 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:20:55.945 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:20:55.948 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:20:55.949 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:20:56.049 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:20:56.089 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:20:56.091 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:20:56.092 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:20:56.192 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:20:56.226 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:20:56.228 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:20:56.229 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:20:56.317 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/mon.a/config 2023-06-03T20:20:56.330 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:20:56.334 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:20:56 smithi164 bash[59462]: cluster 1685823654.9485571 mgr.y (mgr.34104) 1349 : cluster 0 pgmap v1350: 99 pgs: 20 active+clean+remapped, 14 active+undersized+degraded, 16 active+undersized, 49 active+clean; 14 MiB data, 3.8 GiB used, 533 GiB / 536 GiB avail; 0 B/s wr, 613 op/s; 107/699 objects degraded (15.308%); 6/699 objects misplaced (0.858%) 2023-06-03T20:20:56.334 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:20:56 smithi164 bash[59462]: audit 1685823655.4582186 mon.a (mon.0) 1318 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:20:56.334 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:20:56 smithi164 bash[59462]: audit 1685823655.468595 mon.a (mon.0) 1319 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:20:56.345 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 20:20:56 smithi089 bash[71163]: ::ffff:172.21.15.164 - - [03/Jun/2023:20:20:56] "GET /metrics HTTP/1.1" 200 228934 "" "Prometheus/2.33.4" 2023-06-03T20:20:56.346 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:20:56 smithi089 bash[78452]: cluster 1685823654.9485571 mgr.y (mgr.34104) 1349 : cluster 0 pgmap v1350: 99 pgs: 20 active+clean+remapped, 14 active+undersized+degraded, 16 active+undersized, 49 active+clean; 14 MiB data, 3.8 GiB used, 533 GiB / 536 GiB avail; 0 B/s wr, 613 op/s; 107/699 objects degraded (15.308%); 6/699 objects misplaced (0.858%) 2023-06-03T20:20:56.346 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:20:56 smithi089 bash[78452]: audit 1685823655.4582186 mon.a (mon.0) 1318 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:20:56.346 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:20:56 smithi089 bash[78452]: audit 1685823655.468595 mon.a (mon.0) 1319 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:20:56.346 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:20:56 smithi089 bash[75068]: cluster 1685823654.9485571 mgr.y (mgr.34104) 1349 : cluster 0 pgmap v1350: 99 pgs: 20 active+clean+remapped, 14 active+undersized+degraded, 16 active+undersized, 49 active+clean; 14 MiB data, 3.8 GiB used, 533 GiB / 536 GiB avail; 0 B/s wr, 613 op/s; 107/699 objects degraded (15.308%); 6/699 objects misplaced (0.858%) 2023-06-03T20:20:56.346 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:20:56 smithi089 bash[75068]: audit 1685823655.4582186 mon.a (mon.0) 1318 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:20:56.347 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:20:56 smithi089 bash[75068]: audit 1685823655.468595 mon.a (mon.0) 1319 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:20:56.366 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:20:56.368 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:20:56.369 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:20:56.470 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:20:56.507 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:20:56.509 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:20:56.510 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:20:56.611 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:20:56.655 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:20:56.658 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:20:56.658 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:20:56.759 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:20:56.798 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:20:56.800 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:20:56.801 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:20:56.902 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:20:56.923 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:20:56.925 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:20:56.925 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:20:57.026 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:20:57.064 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:20:57.067 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:20:57.067 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:20:57.168 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:20:57.239 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:20:57.242 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:20:57.243 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:20:57.343 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:20:57.380 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:20:57.383 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:20:57.384 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:20:57.485 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:20:57.515 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:20:57.518 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:20:57.519 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:20:57.620 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:20:57.647 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:20:57.649 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:20:57.650 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:20:57.751 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:20:57.778 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:20:57.780 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:20:57.780 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:20:57.881 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:20:57.907 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:20:57.909 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:20:57.910 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:20:58.011 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:20:58.047 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:20:58.049 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:20:58.050 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:20:58.151 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:20:58.185 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:20:58.189 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:20:58.190 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:20:58.279 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:20:58 smithi089 bash[75068]: cluster 1685823656.9492614 mgr.y (mgr.34104) 1350 : cluster 0 pgmap v1351: 99 pgs: 20 active+clean+remapped, 14 active+undersized+degraded, 16 active+undersized, 49 active+clean; 14 MiB data, 3.8 GiB used, 533 GiB / 536 GiB avail; 0 B/s wr, 453 op/s; 107/699 objects degraded (15.308%); 6/699 objects misplaced (0.858%) 2023-06-03T20:20:58.279 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:20:58 smithi089 bash[75068]: audit 1685823657.7167554 mon.a (mon.0) 1320 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:20:58.279 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:20:58 smithi089 bash[75068]: audit 1685823657.728225 mon.a (mon.0) 1321 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:20:58.279 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:20:58 smithi089 bash[75068]: cluster 1685823657.8762407 mon.a (mon.0) 1322 : cluster 3 Health check update: Degraded data redundancy: 107/699 objects degraded (15.308%), 14 pgs degraded, 30 pgs undersized (PG_DEGRADED) 2023-06-03T20:20:58.280 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:20:58 smithi089 bash[78452]: cluster 1685823656.9492614 mgr.y (mgr.34104) 1350 : cluster 0 pgmap v1351: 99 pgs: 20 active+clean+remapped, 14 active+undersized+degraded, 16 active+undersized, 49 active+clean; 14 MiB data, 3.8 GiB used, 533 GiB / 536 GiB avail; 0 B/s wr, 453 op/s; 107/699 objects degraded (15.308%); 6/699 objects misplaced (0.858%) 2023-06-03T20:20:58.280 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:20:58 smithi089 bash[78452]: audit 1685823657.7167554 mon.a (mon.0) 1320 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:20:58.280 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:20:58 smithi089 bash[78452]: audit 1685823657.728225 mon.a (mon.0) 1321 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:20:58.280 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:20:58 smithi089 bash[78452]: cluster 1685823657.8762407 mon.a (mon.0) 1322 : cluster 3 Health check update: Degraded data redundancy: 107/699 objects degraded (15.308%), 14 pgs degraded, 30 pgs undersized (PG_DEGRADED) 2023-06-03T20:20:58.285 INFO:teuthology.orchestra.run.smithi089.stdout:osd.0: {} 2023-06-03T20:20:58.291 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:20:58.305 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:20:58 smithi164 bash[59462]: cluster 1685823656.9492614 mgr.y (mgr.34104) 1350 : cluster 0 pgmap v1351: 99 pgs: 20 active+clean+remapped, 14 active+undersized+degraded, 16 active+undersized, 49 active+clean; 14 MiB data, 3.8 GiB used, 533 GiB / 536 GiB avail; 0 B/s wr, 453 op/s; 107/699 objects degraded (15.308%); 6/699 objects misplaced (0.858%) 2023-06-03T20:20:58.305 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:20:58 smithi164 bash[59462]: audit 1685823657.7167554 mon.a (mon.0) 1320 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:20:58.305 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:20:58 smithi164 bash[59462]: audit 1685823657.728225 mon.a (mon.0) 1321 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:20:58.305 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:20:58 smithi164 bash[59462]: cluster 1685823657.8762407 mon.a (mon.0) 1322 : cluster 3 Health check update: Degraded data redundancy: 107/699 objects degraded (15.308%), 14 pgs degraded, 30 pgs undersized (PG_DEGRADED) 2023-06-03T20:20:58.306 INFO:teuthology.orchestra.run.smithi089.stdout:osd.1: {} 2023-06-03T20:20:58.326 INFO:teuthology.orchestra.run.smithi089.stdout:osd.2: {} 2023-06-03T20:20:58.327 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:20:58.330 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:20:58.331 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:20:58.346 INFO:teuthology.orchestra.run.smithi089.stdout:osd.3: {} 2023-06-03T20:20:58.376 INFO:teuthology.orchestra.run.smithi089.stdout:osd.5: {} 2023-06-03T20:20:58.399 INFO:teuthology.orchestra.run.smithi089.stdout:osd.6: {} 2023-06-03T20:20:58.432 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:20:58.436 INFO:teuthology.orchestra.run.smithi089.stdout:osd.7: {} 2023-06-03T20:20:58.437 INFO:teuthology.orchestra.run.smithi089.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-06-03T20:20:58.437 INFO:teuthology.orchestra.run.smithi089.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-06-03T20:20:58.437 INFO:teuthology.orchestra.run.smithi089.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-06-03T20:20:58.437 INFO:teuthology.orchestra.run.smithi089.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-06-03T20:20:58.438 INFO:teuthology.orchestra.run.smithi089.stderr:Error ENXIO: problem getting command descriptions from osd.4 2023-06-03T20:20:58.438 INFO:teuthology.orchestra.run.smithi089.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-06-03T20:20:58.438 INFO:teuthology.orchestra.run.smithi089.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-06-03T20:20:58.438 INFO:teuthology.orchestra.run.smithi089.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-06-03T20:20:58.459 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:20:58.461 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:20:58.462 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:20:58.542 INFO:teuthology.orchestra.run.smithi089.stdout:ERROR: (22) Invalid argument 2023-06-03T20:20:58.543 INFO:teuthology.orchestra.run.smithi089.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-06-03T20:20:58.563 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:20:58.591 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:20:58.593 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:20:58.594 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:20:58.694 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:20:58.720 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:20:58.723 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:20:58.723 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:20:58.824 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:20:58.849 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:20:58.850 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:20:58.851 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:20:58.952 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:20:58.990 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:20:58.993 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:20:58.993 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:20:59.094 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:20:59.127 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:20:59 smithi089 bash[75068]: audit 1685823658.1968484 mon.c (mon.2) 579 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-03T20:20:59.127 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:20:59 smithi089 bash[75068]: audit 1685823658.1978664 mon.c (mon.2) 580 : audit 1 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-03T20:20:59.128 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:20:59 smithi089 bash[75068]: audit 1685823658.204599 mon.a (mon.0) 1323 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:20:59.128 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:20:59 smithi089 bash[75068]: audit 1685823658.26069 mon.c (mon.2) 581 : audit 0 from='client.? 172.21.15.89:0/372545320' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-03T20:20:59.128 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:20:59 smithi089 bash[75068]: audit 1685823658.284831 mon.a (mon.0) 1324 : audit 1 from='client.? 172.21.15.89:0/3192280124' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-06-03T20:20:59.129 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:20:59 smithi089 bash[78452]: audit 1685823658.1968484 mon.c (mon.2) 579 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-03T20:20:59.129 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:20:59 smithi089 bash[78452]: audit 1685823658.1978664 mon.c (mon.2) 580 : audit 1 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-03T20:20:59.130 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:20:59 smithi089 bash[78452]: audit 1685823658.204599 mon.a (mon.0) 1323 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:20:59.130 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:20:59 smithi089 bash[78452]: audit 1685823658.26069 mon.c (mon.2) 581 : audit 0 from='client.? 172.21.15.89:0/372545320' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-03T20:20:59.130 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:20:59 smithi089 bash[78452]: audit 1685823658.284831 mon.a (mon.0) 1324 : audit 1 from='client.? 172.21.15.89:0/3192280124' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-06-03T20:20:59.131 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:20:59.131 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:20:59.132 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:20:59.221 INFO:teuthology.orchestra.run.smithi089.stderr:noscrub is unset 2023-06-03T20:20:59.234 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:20:59.263 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:20:59.265 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:20:59.266 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:20:59.334 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:20:59 smithi164 bash[59462]: audit 1685823658.1968484 mon.c (mon.2) 579 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-03T20:20:59.334 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:20:59 smithi164 bash[59462]: audit 1685823658.1978664 mon.c (mon.2) 580 : audit 1 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-03T20:20:59.334 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:20:59 smithi164 bash[59462]: audit 1685823658.204599 mon.a (mon.0) 1323 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:20:59.334 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:20:59 smithi164 bash[59462]: audit 1685823658.26069 mon.c (mon.2) 581 : audit 0 from='client.? 172.21.15.89:0/372545320' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-03T20:20:59.335 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:20:59 smithi164 bash[59462]: audit 1685823658.284831 mon.a (mon.0) 1324 : audit 1 from='client.? 172.21.15.89:0/3192280124' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-06-03T20:20:59.367 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:20:59.406 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:20:59.408 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:20:59.409 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:20:59.510 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:20:59.548 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:20:59.551 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:20:59.551 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:20:59.652 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:20:59.690 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:20:59.693 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:20:59.694 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:20:59.779 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:20:59 smithi089 bash[75068]: debug 2023-06-03T20:20:59.403+0000 7f0fee4d2700 -1 mon.a@0(leader).osd e234 definitely_dead 0 2023-06-03T20:20:59.794 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:20:59.826 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:20:59.829 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:20:59.830 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:20:59.930 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:20:59.966 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:20:59.968 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:20:59.969 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:21:00.070 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:21:00.109 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:21:00.110 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:21:00.111 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:21:00.212 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:21:00.222 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:21:00 smithi089 bash[75068]: cluster 1685823658.950734 mgr.y (mgr.34104) 1351 : cluster 0 pgmap v1352: 99 pgs: 20 active+clean+remapped, 14 active+undersized+degraded, 16 active+undersized, 49 active+clean; 14 MiB data, 3.8 GiB used, 533 GiB / 536 GiB avail; 0 B/s wr, 342 op/s; 107/699 objects degraded (15.308%); 6/699 objects misplaced (0.858%) 2023-06-03T20:21:00.223 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:21:00 smithi089 bash[75068]: cluster 1685823659.2020986 mon.a (mon.0) 1325 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-06-03T20:21:00.223 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:21:00 smithi089 bash[75068]: cluster 1685823659.2074766 osd.6 (osd.6) 177 : cluster 3 failed to encode map e234 with expected crc 2023-06-03T20:21:00.223 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:21:00 smithi089 bash[75068]: audit 1685823659.2110443 mon.a (mon.0) 1326 : audit 1 from='client.? 172.21.15.89:0/3192280124' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-06-03T20:21:00.223 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:21:00 smithi089 bash[75068]: cluster 1685823659.2111065 mon.a (mon.0) 1327 : cluster 0 osdmap e234: 8 total, 7 up, 6 in 2023-06-03T20:21:00.223 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:21:00 smithi089 bash[75068]: audit 1685823659.4065983 mon.a (mon.0) 1328 : audit 0 from='client.? 172.21.15.89:0/929306835' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["1"]}]: dispatch 2023-06-03T20:21:00.223 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:21:00 smithi089 bash[78452]: cluster 1685823658.950734 mgr.y (mgr.34104) 1351 : cluster 0 pgmap v1352: 99 pgs: 20 active+clean+remapped, 14 active+undersized+degraded, 16 active+undersized, 49 active+clean; 14 MiB data, 3.8 GiB used, 533 GiB / 536 GiB avail; 0 B/s wr, 342 op/s; 107/699 objects degraded (15.308%); 6/699 objects misplaced (0.858%) 2023-06-03T20:21:00.224 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:21:00 smithi089 bash[78452]: cluster 1685823659.2020986 mon.a (mon.0) 1325 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-06-03T20:21:00.224 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:21:00 smithi089 bash[78452]: cluster 1685823659.2074766 osd.6 (osd.6) 177 : cluster 3 failed to encode map e234 with expected crc 2023-06-03T20:21:00.224 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:21:00 smithi089 bash[78452]: audit 1685823659.2110443 mon.a (mon.0) 1326 : audit 1 from='client.? 172.21.15.89:0/3192280124' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-06-03T20:21:00.224 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:21:00 smithi089 bash[78452]: cluster 1685823659.2111065 mon.a (mon.0) 1327 : cluster 0 osdmap e234: 8 total, 7 up, 6 in 2023-06-03T20:21:00.224 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:21:00 smithi089 bash[78452]: audit 1685823659.4065983 mon.a (mon.0) 1328 : audit 0 from='client.? 172.21.15.89:0/929306835' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["1"]}]: dispatch 2023-06-03T20:21:00.234 INFO:teuthology.orchestra.run.smithi089.stderr:marked in osd.1. 2023-06-03T20:21:00.251 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:21:00.254 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:21:00.255 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:21:00.334 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:21:00 smithi164 bash[59462]: cluster 1685823658.950734 mgr.y (mgr.34104) 1351 : cluster 0 pgmap v1352: 99 pgs: 20 active+clean+remapped, 14 active+undersized+degraded, 16 active+undersized, 49 active+clean; 14 MiB data, 3.8 GiB used, 533 GiB / 536 GiB avail; 0 B/s wr, 342 op/s; 107/699 objects degraded (15.308%); 6/699 objects misplaced (0.858%) 2023-06-03T20:21:00.334 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:21:00 smithi164 bash[59462]: cluster 1685823659.2020986 mon.a (mon.0) 1325 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-06-03T20:21:00.334 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:21:00 smithi164 bash[59462]: cluster 1685823659.2074766 osd.6 (osd.6) 177 : cluster 3 failed to encode map e234 with expected crc 2023-06-03T20:21:00.335 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:21:00 smithi164 bash[59462]: audit 1685823659.2110443 mon.a (mon.0) 1326 : audit 1 from='client.? 172.21.15.89:0/3192280124' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-06-03T20:21:00.335 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:21:00 smithi164 bash[59462]: cluster 1685823659.2111065 mon.a (mon.0) 1327 : cluster 0 osdmap e234: 8 total, 7 up, 6 in 2023-06-03T20:21:00.335 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:21:00 smithi164 bash[59462]: audit 1685823659.4065983 mon.a (mon.0) 1328 : audit 0 from='client.? 172.21.15.89:0/929306835' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["1"]}]: dispatch 2023-06-03T20:21:00.355 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:21:00.394 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:21:00.396 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:21:00.397 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:21:00.498 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:21:00.534 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:21:00.537 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:21:00.537 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:21:00.638 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:21:00.679 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:21:00.681 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:21:00.682 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:21:00.783 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:21:00.820 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:21:00.823 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:21:00.823 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:21:00.924 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:21:00.949 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:21:00.951 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:21:00.952 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:21:01.052 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:21:01.078 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:21:01.081 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:21:01.082 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:21:01.182 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:21:01.194 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:21:01 smithi089 bash[75068]: cluster 1685823659.4796271 osd.7 (osd.7) 159 : cluster 3 failed to encode map e234 with expected crc 2023-06-03T20:21:01.195 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:21:01 smithi089 bash[75068]: cluster 1685823660.2112856 osd.6 (osd.6) 178 : cluster 3 failed to encode map e235 with expected crc 2023-06-03T20:21:01.195 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:21:01 smithi089 bash[75068]: cluster 1685823660.215039 osd.6 (osd.6) 179 : cluster 3 failed to encode map e235 with expected crc 2023-06-03T20:21:01.195 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:21:01 smithi089 bash[75068]: audit 1685823660.2176368 mon.a (mon.0) 1329 : audit 1 from='client.? 172.21.15.89:0/929306835' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["1"]}]': finished 2023-06-03T20:21:01.195 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:21:01 smithi089 bash[75068]: cluster 1685823660.2177374 mon.a (mon.0) 1330 : cluster 0 osdmap e235: 8 total, 7 up, 7 in 2023-06-03T20:21:01.196 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:21:01 smithi089 bash[75068]: cluster 1685823660.2177565 osd.5 (osd.5) 171 : cluster 3 failed to encode map e234 with expected crc 2023-06-03T20:21:01.196 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:21:01 smithi089 bash[75068]: cluster 1685823660.2200124 osd.5 (osd.5) 172 : cluster 3 failed to encode map e234 with expected crc 2023-06-03T20:21:01.196 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:21:01 smithi089 bash[78452]: cluster 1685823659.4796271 osd.7 (osd.7) 159 : cluster 3 failed to encode map e234 with expected crc 2023-06-03T20:21:01.197 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:21:01 smithi089 bash[78452]: cluster 1685823660.2112856 osd.6 (osd.6) 178 : cluster 3 failed to encode map e235 with expected crc 2023-06-03T20:21:01.197 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:21:01 smithi089 bash[78452]: cluster 1685823660.215039 osd.6 (osd.6) 179 : cluster 3 failed to encode map e235 with expected crc 2023-06-03T20:21:01.197 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:21:01 smithi089 bash[78452]: audit 1685823660.2176368 mon.a (mon.0) 1329 : audit 1 from='client.? 172.21.15.89:0/929306835' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["1"]}]': finished 2023-06-03T20:21:01.197 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:21:01 smithi089 bash[78452]: cluster 1685823660.2177374 mon.a (mon.0) 1330 : cluster 0 osdmap e235: 8 total, 7 up, 7 in 2023-06-03T20:21:01.197 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:21:01 smithi089 bash[78452]: cluster 1685823660.2177565 osd.5 (osd.5) 171 : cluster 3 failed to encode map e234 with expected crc 2023-06-03T20:21:01.198 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:21:01 smithi089 bash[78452]: cluster 1685823660.2200124 osd.5 (osd.5) 172 : cluster 3 failed to encode map e234 with expected crc 2023-06-03T20:21:01.219 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:21:01.223 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:21:01.223 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:21:01.324 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:21:01.334 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:21:01 smithi164 bash[59462]: cluster 1685823659.4796271 osd.7 (osd.7) 159 : cluster 3 failed to encode map e234 with expected crc 2023-06-03T20:21:01.334 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:21:01 smithi164 bash[59462]: cluster 1685823660.2112856 osd.6 (osd.6) 178 : cluster 3 failed to encode map e235 with expected crc 2023-06-03T20:21:01.334 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:21:01 smithi164 bash[59462]: cluster 1685823660.215039 osd.6 (osd.6) 179 : cluster 3 failed to encode map e235 with expected crc 2023-06-03T20:21:01.334 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:21:01 smithi164 bash[59462]: audit 1685823660.2176368 mon.a (mon.0) 1329 : audit 1 from='client.? 172.21.15.89:0/929306835' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["1"]}]': finished 2023-06-03T20:21:01.334 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:21:01 smithi164 bash[59462]: cluster 1685823660.2177374 mon.a (mon.0) 1330 : cluster 0 osdmap e235: 8 total, 7 up, 7 in 2023-06-03T20:21:01.335 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:21:01 smithi164 bash[59462]: cluster 1685823660.2177565 osd.5 (osd.5) 171 : cluster 3 failed to encode map e234 with expected crc 2023-06-03T20:21:01.335 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:21:01 smithi164 bash[59462]: cluster 1685823660.2200124 osd.5 (osd.5) 172 : cluster 3 failed to encode map e234 with expected crc 2023-06-03T20:21:01.349 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:21:01.351 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:21:01.352 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:21:01.453 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:21:01.489 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:21:01.491 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:21:01.491 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:21:01.592 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:21:01.628 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:21:01.632 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:21:01.632 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:21:01.733 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:21:01.774 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:21:01.777 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:21:01.778 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:21:01.878 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:21:01.903 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:21:01.905 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:21:01.906 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:21:02.007 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:21:02.040 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:21:02.042 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:21:02.042 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:21:02.143 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:21:02.166 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:21:02 smithi089 bash[78452]: cluster 1685823660.215827 osd.7 (osd.7) 160 : cluster 3 failed to encode map e235 with expected crc 2023-06-03T20:21:02.166 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:21:02 smithi089 bash[78452]: cluster 1685823660.9515567 mgr.y (mgr.34104) 1352 : cluster 0 pgmap v1355: 99 pgs: 1 unknown, 4 peering, 2 remapped+peering, 18 active+clean+remapped, 12 active+undersized+degraded, 16 active+undersized, 46 active+clean; 14 MiB data, 3.8 GiB used, 533 GiB / 536 GiB avail; 0 B/s wr, 6 op/s; 87/699 objects degraded (12.446%); 6/699 objects misplaced (0.858%) 2023-06-03T20:21:02.167 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:21:02 smithi089 bash[78452]: cluster 1685823661.2203355 osd.6 (osd.6) 180 : cluster 3 failed to encode map e236 with expected crc 2023-06-03T20:21:02.167 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:21:02 smithi089 bash[78452]: cluster 1685823661.2253678 mon.a (mon.0) 1331 : cluster 0 osdmap e236: 8 total, 7 up, 7 in 2023-06-03T20:21:02.167 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:21:02 smithi089 bash[78452]: cluster 1685823661.230595 osd.5 (osd.5) 173 : cluster 3 failed to encode map e236 with expected crc 2023-06-03T20:21:02.167 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:21:02 smithi089 bash[78452]: cluster 1685823661.2330883 osd.5 (osd.5) 174 : cluster 3 failed to encode map e236 with expected crc 2023-06-03T20:21:02.167 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:21:02 smithi089 bash[75068]: cluster 1685823660.215827 osd.7 (osd.7) 160 : cluster 3 failed to encode map e235 with expected crc 2023-06-03T20:21:02.167 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:21:02 smithi089 bash[75068]: cluster 1685823660.9515567 mgr.y (mgr.34104) 1352 : cluster 0 pgmap v1355: 99 pgs: 1 unknown, 4 peering, 2 remapped+peering, 18 active+clean+remapped, 12 active+undersized+degraded, 16 active+undersized, 46 active+clean; 14 MiB data, 3.8 GiB used, 533 GiB / 536 GiB avail; 0 B/s wr, 6 op/s; 87/699 objects degraded (12.446%); 6/699 objects misplaced (0.858%) 2023-06-03T20:21:02.168 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:21:02 smithi089 bash[75068]: cluster 1685823661.2203355 osd.6 (osd.6) 180 : cluster 3 failed to encode map e236 with expected crc 2023-06-03T20:21:02.168 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:21:02 smithi089 bash[75068]: cluster 1685823661.2253678 mon.a (mon.0) 1331 : cluster 0 osdmap e236: 8 total, 7 up, 7 in 2023-06-03T20:21:02.168 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:21:02 smithi089 bash[75068]: cluster 1685823661.230595 osd.5 (osd.5) 173 : cluster 3 failed to encode map e236 with expected crc 2023-06-03T20:21:02.168 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:21:02 smithi089 bash[75068]: cluster 1685823661.2330883 osd.5 (osd.5) 174 : cluster 3 failed to encode map e236 with expected crc 2023-06-03T20:21:02.182 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:21:02.185 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:21:02.185 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:21:02.193 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-06-03T20:21:02.193 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-06-03T20:21:02.286 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:21:02.295 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:21:02 smithi164 bash[59462]: cluster 1685823660.215827 osd.7 (osd.7) 160 : cluster 3 failed to encode map e235 with expected crc 2023-06-03T20:21:02.296 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:21:02 smithi164 bash[59462]: cluster 1685823660.9515567 mgr.y (mgr.34104) 1352 : cluster 0 pgmap v1355: 99 pgs: 1 unknown, 4 peering, 2 remapped+peering, 18 active+clean+remapped, 12 active+undersized+degraded, 16 active+undersized, 46 active+clean; 14 MiB data, 3.8 GiB used, 533 GiB / 536 GiB avail; 0 B/s wr, 6 op/s; 87/699 objects degraded (12.446%); 6/699 objects misplaced (0.858%) 2023-06-03T20:21:02.296 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:21:02 smithi164 bash[59462]: cluster 1685823661.2203355 osd.6 (osd.6) 180 : cluster 3 failed to encode map e236 with expected crc 2023-06-03T20:21:02.296 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:21:02 smithi164 bash[59462]: cluster 1685823661.2253678 mon.a (mon.0) 1331 : cluster 0 osdmap e236: 8 total, 7 up, 7 in 2023-06-03T20:21:02.296 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:21:02 smithi164 bash[59462]: cluster 1685823661.230595 osd.5 (osd.5) 173 : cluster 3 failed to encode map e236 with expected crc 2023-06-03T20:21:02.296 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:21:02 smithi164 bash[59462]: cluster 1685823661.2330883 osd.5 (osd.5) 174 : cluster 3 failed to encode map e236 with expected crc 2023-06-03T20:21:02.313 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:21:02.316 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:21:02.316 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:21:02.361 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph daemon osd.0 dump_blocked_ops 2023-06-03T20:21:02.417 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:21:02.453 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:21:02.456 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:21:02.457 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:21:02.558 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:21:02.592 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:21:02.596 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:21:02.597 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:21:02.698 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:21:02.738 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:21:02.740 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:21:02.741 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:21:02.842 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:21:02.869 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:21:02.871 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:21:02.871 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:21:02.972 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:21:03.010 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:21:03.013 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:21:03.014 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:21:03.115 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:21:03.144 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:21:03.146 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:21:03.146 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:21:03.247 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:21:03.260 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:21:03 smithi164 bash[59462]: cluster 1685823661.223636 osd.7 (osd.7) 161 : cluster 3 failed to encode map e236 with expected crc 2023-06-03T20:21:03.260 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:21:03 smithi164 bash[59462]: cluster 1685823661.2275188 osd.7 (osd.7) 162 : cluster 3 failed to encode map e236 with expected crc 2023-06-03T20:21:03.261 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:21:03 smithi164 bash[59462]: cluster 1685823662.222987 osd.6 (osd.6) 181 : cluster 3 failed to encode map e237 with expected crc 2023-06-03T20:21:03.261 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:21:03 smithi164 bash[59462]: cluster 1685823662.2230706 osd.5 (osd.5) 175 : cluster 3 failed to encode map e237 with expected crc 2023-06-03T20:21:03.261 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:21:03 smithi164 bash[59462]: cluster 1685823662.229184 mon.a (mon.0) 1332 : cluster 0 osdmap e237: 8 total, 7 up, 7 in 2023-06-03T20:21:03.261 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:21:03 smithi164 bash[59462]: cluster 1685823662.2323496 osd.3 (osd.3) 9 : cluster 0 12.1 starting backfill to osd.1 from (0'0,0'0] MAX to 233'11257 2023-06-03T20:21:03.261 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:21:03 smithi164 bash[59462]: cluster 1685823662.234711 osd.2 (osd.2) 7 : cluster 0 12.9 starting backfill to osd.1 from (0'0,0'0] MAX to 233'10790 2023-06-03T20:21:03.261 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:21:03 smithi164 bash[59462]: cluster 1685823662.2353508 osd.2 (osd.2) 8 : cluster 0 12.7 starting backfill to osd.1 from (0'0,0'0] MAX to 233'11011 2023-06-03T20:21:03.261 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:21:03 smithi164 bash[59462]: cluster 1685823662.2353911 osd.2 (osd.2) 9 : cluster 0 12.19 starting backfill to osd.1 from (0'0,0'0] MAX to 233'10750 2023-06-03T20:21:03.262 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:21:03 smithi164 bash[59462]: cluster 1685823662.236111 osd.2 (osd.2) 10 : cluster 0 12.17 starting backfill to osd.1 from (0'0,0'0] MAX to 233'11031 2023-06-03T20:21:03.262 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:21:03 smithi164 bash[59462]: cluster 1685823662.2363436 osd.2 (osd.2) 11 : cluster 0 12.5 starting backfill to osd.1 from (0'0,0'0] MAX to 233'10759 2023-06-03T20:21:03.262 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:21:03 smithi164 bash[59462]: cluster 1685823662.2370934 osd.2 (osd.2) 12 : cluster 0 12.15 starting backfill to osd.1 from (0'0,0'0] MAX to 233'10779 2023-06-03T20:21:03.262 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:21:03 smithi164 bash[59462]: cluster 1685823662.2454274 osd.3 (osd.3) 10 : cluster 0 12.11 starting backfill to osd.1 from (0'0,0'0] MAX to 233'11284 2023-06-03T20:21:03.262 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:21:03 smithi164 bash[59462]: cluster 1685823662.247229 osd.3 (osd.3) 11 : cluster 0 12.4 starting backfill to osd.1 from (0'0,0'0] MAX to 233'10892 2023-06-03T20:21:03.262 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:21:03 smithi164 bash[59462]: cluster 1685823662.248065 osd.3 (osd.3) 12 : cluster 0 12.14 starting backfill to osd.1 from (0'0,0'0] MAX to 233'10872 2023-06-03T20:21:03.262 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:21:03 smithi164 bash[59462]: cluster 1685823662.8773496 mon.a (mon.0) 1333 : cluster 3 Health check update: Degraded data redundancy: 87/699 objects degraded (12.446%), 12 pgs degraded, 28 pgs undersized (PG_DEGRADED) 2023-06-03T20:21:03.263 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:21:03 smithi164 bash[59462]: cluster 1685823662.884233 mon.a (mon.0) 1334 : cluster 0 osdmap e238: 8 total, 7 up, 7 in 2023-06-03T20:21:03.287 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:21:03.289 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:21:03.291 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:21:03.392 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:21:03.429 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:21:03.433 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:21:03.433 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:21:03.521 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:21:03 smithi089 bash[75068]: cluster 1685823661.223636 osd.7 (osd.7) 161 : cluster 3 failed to encode map e236 with expected crc 2023-06-03T20:21:03.521 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:21:03 smithi089 bash[75068]: cluster 1685823661.2275188 osd.7 (osd.7) 162 : cluster 3 failed to encode map e236 with expected crc 2023-06-03T20:21:03.522 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:21:03 smithi089 bash[75068]: cluster 1685823662.222987 osd.6 (osd.6) 181 : cluster 3 failed to encode map e237 with expected crc 2023-06-03T20:21:03.522 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:21:03 smithi089 bash[75068]: cluster 1685823662.2230706 osd.5 (osd.5) 175 : cluster 3 failed to encode map e237 with expected crc 2023-06-03T20:21:03.522 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:21:03 smithi089 bash[75068]: cluster 1685823662.229184 mon.a (mon.0) 1332 : cluster 0 osdmap e237: 8 total, 7 up, 7 in 2023-06-03T20:21:03.522 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:21:03 smithi089 bash[75068]: cluster 1685823662.2323496 osd.3 (osd.3) 9 : cluster 0 12.1 starting backfill to osd.1 from (0'0,0'0] MAX to 233'11257 2023-06-03T20:21:03.522 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:21:03 smithi089 bash[75068]: cluster 1685823662.234711 osd.2 (osd.2) 7 : cluster 0 12.9 starting backfill to osd.1 from (0'0,0'0] MAX to 233'10790 2023-06-03T20:21:03.522 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:21:03 smithi089 bash[75068]: cluster 1685823662.2353508 osd.2 (osd.2) 8 : cluster 0 12.7 starting backfill to osd.1 from (0'0,0'0] MAX to 233'11011 2023-06-03T20:21:03.522 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:21:03 smithi089 bash[75068]: cluster 1685823662.2353911 osd.2 (osd.2) 9 : cluster 0 12.19 starting backfill to osd.1 from (0'0,0'0] MAX to 233'10750 2023-06-03T20:21:03.523 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:21:03 smithi089 bash[75068]: cluster 1685823662.236111 osd.2 (osd.2) 10 : cluster 0 12.17 starting backfill to osd.1 from (0'0,0'0] MAX to 233'11031 2023-06-03T20:21:03.523 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:21:03 smithi089 bash[75068]: cluster 1685823662.2363436 osd.2 (osd.2) 11 : cluster 0 12.5 starting backfill to osd.1 from (0'0,0'0] MAX to 233'10759 2023-06-03T20:21:03.523 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:21:03 smithi089 bash[75068]: cluster 1685823662.2370934 osd.2 (osd.2) 12 : cluster 0 12.15 starting backfill to osd.1 from (0'0,0'0] MAX to 233'10779 2023-06-03T20:21:03.523 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:21:03 smithi089 bash[75068]: cluster 1685823662.2454274 osd.3 (osd.3) 10 : cluster 0 12.11 starting backfill to osd.1 from (0'0,0'0] MAX to 233'11284 2023-06-03T20:21:03.523 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:21:03 smithi089 bash[75068]: cluster 1685823662.247229 osd.3 (osd.3) 11 : cluster 0 12.4 starting backfill to osd.1 from (0'0,0'0] MAX to 233'10892 2023-06-03T20:21:03.523 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:21:03 smithi089 bash[75068]: cluster 1685823662.248065 osd.3 (osd.3) 12 : cluster 0 12.14 starting backfill to osd.1 from (0'0,0'0] MAX to 233'10872 2023-06-03T20:21:03.523 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:21:03 smithi089 bash[75068]: cluster 1685823662.8773496 mon.a (mon.0) 1333 : cluster 3 Health check update: Degraded data redundancy: 87/699 objects degraded (12.446%), 12 pgs degraded, 28 pgs undersized (PG_DEGRADED) 2023-06-03T20:21:03.524 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:21:03 smithi089 bash[75068]: cluster 1685823662.884233 mon.a (mon.0) 1334 : cluster 0 osdmap e238: 8 total, 7 up, 7 in 2023-06-03T20:21:03.524 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:21:03 smithi089 bash[78452]: cluster 1685823661.223636 osd.7 (osd.7) 161 : cluster 3 failed to encode map e236 with expected crc 2023-06-03T20:21:03.524 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:21:03 smithi089 bash[78452]: cluster 1685823661.2275188 osd.7 (osd.7) 162 : cluster 3 failed to encode map e236 with expected crc 2023-06-03T20:21:03.524 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:21:03 smithi089 bash[78452]: cluster 1685823662.222987 osd.6 (osd.6) 181 : cluster 3 failed to encode map e237 with expected crc 2023-06-03T20:21:03.524 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:21:03 smithi089 bash[78452]: cluster 1685823662.2230706 osd.5 (osd.5) 175 : cluster 3 failed to encode map e237 with expected crc 2023-06-03T20:21:03.524 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:21:03 smithi089 bash[78452]: cluster 1685823662.229184 mon.a (mon.0) 1332 : cluster 0 osdmap e237: 8 total, 7 up, 7 in 2023-06-03T20:21:03.525 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:21:03 smithi089 bash[78452]: cluster 1685823662.2323496 osd.3 (osd.3) 9 : cluster 0 12.1 starting backfill to osd.1 from (0'0,0'0] MAX to 233'11257 2023-06-03T20:21:03.525 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:21:03 smithi089 bash[78452]: cluster 1685823662.234711 osd.2 (osd.2) 7 : cluster 0 12.9 starting backfill to osd.1 from (0'0,0'0] MAX to 233'10790 2023-06-03T20:21:03.525 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:21:03 smithi089 bash[78452]: cluster 1685823662.2353508 osd.2 (osd.2) 8 : cluster 0 12.7 starting backfill to osd.1 from (0'0,0'0] MAX to 233'11011 2023-06-03T20:21:03.525 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:21:03 smithi089 bash[78452]: cluster 1685823662.2353911 osd.2 (osd.2) 9 : cluster 0 12.19 starting backfill to osd.1 from (0'0,0'0] MAX to 233'10750 2023-06-03T20:21:03.525 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:21:03 smithi089 bash[78452]: cluster 1685823662.236111 osd.2 (osd.2) 10 : cluster 0 12.17 starting backfill to osd.1 from (0'0,0'0] MAX to 233'11031 2023-06-03T20:21:03.525 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:21:03 smithi089 bash[78452]: cluster 1685823662.2363436 osd.2 (osd.2) 11 : cluster 0 12.5 starting backfill to osd.1 from (0'0,0'0] MAX to 233'10759 2023-06-03T20:21:03.525 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:21:03 smithi089 bash[78452]: cluster 1685823662.2370934 osd.2 (osd.2) 12 : cluster 0 12.15 starting backfill to osd.1 from (0'0,0'0] MAX to 233'10779 2023-06-03T20:21:03.525 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:21:03 smithi089 bash[78452]: cluster 1685823662.2454274 osd.3 (osd.3) 10 : cluster 0 12.11 starting backfill to osd.1 from (0'0,0'0] MAX to 233'11284 2023-06-03T20:21:03.526 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:21:03 smithi089 bash[78452]: cluster 1685823662.247229 osd.3 (osd.3) 11 : cluster 0 12.4 starting backfill to osd.1 from (0'0,0'0] MAX to 233'10892 2023-06-03T20:21:03.526 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:21:03 smithi089 bash[78452]: cluster 1685823662.248065 osd.3 (osd.3) 12 : cluster 0 12.14 starting backfill to osd.1 from (0'0,0'0] MAX to 233'10872 2023-06-03T20:21:03.526 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:21:03 smithi089 bash[78452]: cluster 1685823662.8773496 mon.a (mon.0) 1333 : cluster 3 Health check update: Degraded data redundancy: 87/699 objects degraded (12.446%), 12 pgs degraded, 28 pgs undersized (PG_DEGRADED) 2023-06-03T20:21:03.526 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:21:03 smithi089 bash[78452]: cluster 1685823662.884233 mon.a (mon.0) 1334 : cluster 0 osdmap e238: 8 total, 7 up, 7 in 2023-06-03T20:21:03.534 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:21:03.571 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:21:03.574 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:21:03.575 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:21:03.675 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:21:03.697 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:21:03.698 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:21:03.699 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:21:03.800 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:21:03.831 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:21:03.833 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:21:03.833 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:21:03.934 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:21:03.972 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:21:03.975 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:21:03.975 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:21:04.076 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:21:04.113 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:21:04.115 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:21:04.116 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:21:04.171 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:21:04 smithi089 bash[78452]: cluster 1685823662.2274694 osd.7 (osd.7) 163 : cluster 3 failed to encode map e237 with expected crc 2023-06-03T20:21:04.172 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:21:04 smithi089 bash[78452]: cluster 1685823662.2306266 osd.7 (osd.7) 164 : cluster 3 failed to encode map e237 with expected crc 2023-06-03T20:21:04.172 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:21:04 smithi089 bash[78452]: cluster 1685823662.2328346 osd.0 (osd.0) 10 : cluster 0 12.6 starting backfill to osd.1 from (0'0,0'0] MAX to 233'10955 2023-06-03T20:21:04.172 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:21:04 smithi089 bash[78452]: cluster 1685823662.2329578 osd.7 (osd.7) 165 : cluster 3 failed to encode map e237 with expected crc 2023-06-03T20:21:04.172 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:21:04 smithi089 bash[78452]: cluster 1685823662.2343931 osd.7 (osd.7) 166 : cluster 3 failed to encode map e237 with expected crc 2023-06-03T20:21:04.172 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:21:04 smithi089 bash[78452]: cluster 1685823662.234821 osd.0 (osd.0) 11 : cluster 0 12.16 starting backfill to osd.1 from (0'0,0'0] MAX to 233'10756 2023-06-03T20:21:04.173 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:21:04 smithi089 bash[78452]: cluster 1685823662.2385976 osd.7 (osd.7) 167 : cluster 0 12.d starting backfill to osd.1 from (0'0,0'0] MAX to 233'12682 2023-06-03T20:21:04.173 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:21:04 smithi089 bash[78452]: cluster 1685823662.8807342 osd.6 (osd.6) 182 : cluster 3 failed to encode map e238 with expected crc 2023-06-03T20:21:04.173 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:21:04 smithi089 bash[78452]: cluster 1685823662.8812754 osd.5 (osd.5) 176 : cluster 3 failed to encode map e238 with expected crc 2023-06-03T20:21:04.173 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:21:04 smithi089 bash[78452]: cluster 1685823662.882674 osd.7 (osd.7) 168 : cluster 3 failed to encode map e238 with expected crc 2023-06-03T20:21:04.173 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:21:04 smithi089 bash[78452]: cluster 1685823662.886075 osd.7 (osd.7) 169 : cluster 3 failed to encode map e238 with expected crc 2023-06-03T20:21:04.173 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:21:04 smithi089 bash[78452]: cluster 1685823662.8875751 osd.7 (osd.7) 170 : cluster 3 failed to encode map e238 with expected crc 2023-06-03T20:21:04.173 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:21:04 smithi089 bash[78452]: cluster 1685823662.952174 mgr.y (mgr.34104) 1353 : cluster 0 pgmap v1359: 99 pgs: 2 active+recovery_wait+degraded, 2 activating, 1 unknown, 4 peering, 6 remapped+peering, 11 active+clean+remapped, 10 active+undersized+degraded, 13 active+undersized, 50 active+clean; 14 MiB data, 4.1 GiB used, 622 GiB / 626 GiB avail; 88/699 objects degraded (12.589%); 6/699 objects misplaced (0.858%); 0 B/s, 0 objects/s recovering 2023-06-03T20:21:04.173 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:21:04 smithi089 bash[78452]: cluster 1685823663.8925707 mon.a (mon.0) 1335 : cluster 3 Health check failed: Reduced data availability: 1 pg inactive (PG_AVAILABILITY) 2023-06-03T20:21:04.174 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:21:04 smithi089 bash[78452]: cluster 1685823663.904796 mon.a (mon.0) 1336 : cluster 0 osdmap e239: 8 total, 7 up, 7 in 2023-06-03T20:21:04.174 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:21:04 smithi089 bash[75068]: cluster 1685823662.2274694 osd.7 (osd.7) 163 : cluster 3 failed to encode map e237 with expected crc 2023-06-03T20:21:04.174 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:21:04 smithi089 bash[75068]: cluster 1685823662.2306266 osd.7 (osd.7) 164 : cluster 3 failed to encode map e237 with expected crc 2023-06-03T20:21:04.174 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:21:04 smithi089 bash[75068]: cluster 1685823662.2328346 osd.0 (osd.0) 10 : cluster 0 12.6 starting backfill to osd.1 from (0'0,0'0] MAX to 233'10955 2023-06-03T20:21:04.174 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:21:04 smithi089 bash[75068]: cluster 1685823662.2329578 osd.7 (osd.7) 165 : cluster 3 failed to encode map e237 with expected crc 2023-06-03T20:21:04.174 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:21:04 smithi089 bash[75068]: cluster 1685823662.2343931 osd.7 (osd.7) 166 : cluster 3 failed to encode map e237 with expected crc 2023-06-03T20:21:04.175 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:21:04 smithi089 bash[75068]: cluster 1685823662.234821 osd.0 (osd.0) 11 : cluster 0 12.16 starting backfill to osd.1 from (0'0,0'0] MAX to 233'10756 2023-06-03T20:21:04.175 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:21:04 smithi089 bash[75068]: cluster 1685823662.2385976 osd.7 (osd.7) 167 : cluster 0 12.d starting backfill to osd.1 from (0'0,0'0] MAX to 233'12682 2023-06-03T20:21:04.175 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:21:04 smithi089 bash[75068]: cluster 1685823662.8807342 osd.6 (osd.6) 182 : cluster 3 failed to encode map e238 with expected crc 2023-06-03T20:21:04.175 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:21:04 smithi089 bash[75068]: cluster 1685823662.8812754 osd.5 (osd.5) 176 : cluster 3 failed to encode map e238 with expected crc 2023-06-03T20:21:04.175 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:21:04 smithi089 bash[75068]: cluster 1685823662.882674 osd.7 (osd.7) 168 : cluster 3 failed to encode map e238 with expected crc 2023-06-03T20:21:04.175 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:21:04 smithi089 bash[75068]: cluster 1685823662.886075 osd.7 (osd.7) 169 : cluster 3 failed to encode map e238 with expected crc 2023-06-03T20:21:04.175 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:21:04 smithi089 bash[75068]: cluster 1685823662.8875751 osd.7 (osd.7) 170 : cluster 3 failed to encode map e238 with expected crc 2023-06-03T20:21:04.176 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:21:04 smithi089 bash[75068]: cluster 1685823662.952174 mgr.y (mgr.34104) 1353 : cluster 0 pgmap v1359: 99 pgs: 2 active+recovery_wait+degraded, 2 activating, 1 unknown, 4 peering, 6 remapped+peering, 11 active+clean+remapped, 10 active+undersized+degraded, 13 active+undersized, 50 active+clean; 14 MiB data, 4.1 GiB used, 622 GiB / 626 GiB avail; 88/699 objects degraded (12.589%); 6/699 objects misplaced (0.858%); 0 B/s, 0 objects/s recovering 2023-06-03T20:21:04.176 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:21:04 smithi089 bash[75068]: cluster 1685823663.8925707 mon.a (mon.0) 1335 : cluster 3 Health check failed: Reduced data availability: 1 pg inactive (PG_AVAILABILITY) 2023-06-03T20:21:04.176 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:21:04 smithi089 bash[75068]: cluster 1685823663.904796 mon.a (mon.0) 1336 : cluster 0 osdmap e239: 8 total, 7 up, 7 in 2023-06-03T20:21:04.194 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-06-03T20:21:04.217 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:21:04.232 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:21:04 smithi164 bash[59462]: cluster 1685823662.2274694 osd.7 (osd.7) 163 : cluster 3 failed to encode map e237 with expected crc 2023-06-03T20:21:04.232 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:21:04 smithi164 bash[59462]: cluster 1685823662.2306266 osd.7 (osd.7) 164 : cluster 3 failed to encode map e237 with expected crc 2023-06-03T20:21:04.233 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:21:04 smithi164 bash[59462]: cluster 1685823662.2328346 osd.0 (osd.0) 10 : cluster 0 12.6 starting backfill to osd.1 from (0'0,0'0] MAX to 233'10955 2023-06-03T20:21:04.233 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:21:04 smithi164 bash[59462]: cluster 1685823662.2329578 osd.7 (osd.7) 165 : cluster 3 failed to encode map e237 with expected crc 2023-06-03T20:21:04.233 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:21:04 smithi164 bash[59462]: cluster 1685823662.2343931 osd.7 (osd.7) 166 : cluster 3 failed to encode map e237 with expected crc 2023-06-03T20:21:04.233 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:21:04 smithi164 bash[59462]: cluster 1685823662.234821 osd.0 (osd.0) 11 : cluster 0 12.16 starting backfill to osd.1 from (0'0,0'0] MAX to 233'10756 2023-06-03T20:21:04.233 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:21:04 smithi164 bash[59462]: cluster 1685823662.2385976 osd.7 (osd.7) 167 : cluster 0 12.d starting backfill to osd.1 from (0'0,0'0] MAX to 233'12682 2023-06-03T20:21:04.233 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:21:04 smithi164 bash[59462]: cluster 1685823662.8807342 osd.6 (osd.6) 182 : cluster 3 failed to encode map e238 with expected crc 2023-06-03T20:21:04.233 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:21:04 smithi164 bash[59462]: cluster 1685823662.8812754 osd.5 (osd.5) 176 : cluster 3 failed to encode map e238 with expected crc 2023-06-03T20:21:04.233 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:21:04 smithi164 bash[59462]: cluster 1685823662.882674 osd.7 (osd.7) 168 : cluster 3 failed to encode map e238 with expected crc 2023-06-03T20:21:04.234 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:21:04 smithi164 bash[59462]: cluster 1685823662.886075 osd.7 (osd.7) 169 : cluster 3 failed to encode map e238 with expected crc 2023-06-03T20:21:04.234 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:21:04 smithi164 bash[59462]: cluster 1685823662.8875751 osd.7 (osd.7) 170 : cluster 3 failed to encode map e238 with expected crc 2023-06-03T20:21:04.234 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:21:04 smithi164 bash[59462]: cluster 1685823662.952174 mgr.y (mgr.34104) 1353 : cluster 0 pgmap v1359: 99 pgs: 2 active+recovery_wait+degraded, 2 activating, 1 unknown, 4 peering, 6 remapped+peering, 11 active+clean+remapped, 10 active+undersized+degraded, 13 active+undersized, 50 active+clean; 14 MiB data, 4.1 GiB used, 622 GiB / 626 GiB avail; 88/699 objects degraded (12.589%); 6/699 objects misplaced (0.858%); 0 B/s, 0 objects/s recovering 2023-06-03T20:21:04.234 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:21:04 smithi164 bash[59462]: cluster 1685823663.8925707 mon.a (mon.0) 1335 : cluster 3 Health check failed: Reduced data availability: 1 pg inactive (PG_AVAILABILITY) 2023-06-03T20:21:04.234 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:21:04 smithi164 bash[59462]: cluster 1685823663.904796 mon.a (mon.0) 1336 : cluster 0 osdmap e239: 8 total, 7 up, 7 in 2023-06-03T20:21:04.256 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:21:04.259 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:21:04.259 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:21:04.360 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:21:04.398 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:21:04.401 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:21:04.401 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:21:04.502 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:21:04.527 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:21:04.529 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:21:04.530 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:21:04.572 INFO:tasks.thrashosds.thrasher:Added osd 1 2023-06-03T20:21:04.630 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:21:04.659 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:21:04.662 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:21:04.662 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:21:04.763 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:21:04.797 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:21:04.800 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:21:04.801 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:21:04.902 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:21:04.944 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:21:04.947 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:21:04.947 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:21:05.048 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:21:05.084 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:21:05.086 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:21:05.087 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:21:05.188 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:21:05.218 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:21:05.221 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:21:05.221 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:21:05.322 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:21:05.333 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:21:05 smithi089 bash[78452]: cluster 1685823663.9012125 osd.5 (osd.5) 177 : cluster 3 failed to encode map e239 with expected crc 2023-06-03T20:21:05.334 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:21:05 smithi089 bash[78452]: cluster 1685823663.902782 osd.7 (osd.7) 171 : cluster 3 failed to encode map e239 with expected crc 2023-06-03T20:21:05.334 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:21:05 smithi089 bash[78452]: cluster 1685823663.9043493 osd.7 (osd.7) 172 : cluster 3 failed to encode map e239 with expected crc 2023-06-03T20:21:05.334 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:21:05 smithi089 bash[78452]: cluster 1685823663.9082994 osd.7 (osd.7) 173 : cluster 3 failed to encode map e239 with expected crc 2023-06-03T20:21:05.334 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:21:05 smithi089 bash[78452]: cluster 1685823663.9098747 osd.7 (osd.7) 174 : cluster 3 failed to encode map e239 with expected crc 2023-06-03T20:21:05.334 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:21:05 smithi089 bash[78452]: cluster 1685823664.2679386 osd.3 (osd.3) 13 : cluster 0 12.4 scrub starts 2023-06-03T20:21:05.334 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:21:05 smithi089 bash[78452]: cluster 1685823664.2871 osd.3 (osd.3) 14 : cluster 0 12.4 scrub ok 2023-06-03T20:21:05.335 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:21:05 smithi089 bash[78452]: cluster 1685823664.486712 osd.6 (osd.6) 183 : cluster 3 failed to encode map e239 with expected crc 2023-06-03T20:21:05.335 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:21:05 smithi089 bash[78452]: cluster 1685823664.5440433 osd.1 (osd.1) 7 : cluster 0 12.17 scrub starts 2023-06-03T20:21:05.335 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:21:05 smithi089 bash[78452]: cluster 1685823664.5536442 osd.1 (osd.1) 8 : cluster 0 12.17 scrub ok 2023-06-03T20:21:05.335 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:21:05 smithi089 bash[75068]: cluster 1685823663.9012125 osd.5 (osd.5) 177 : cluster 3 failed to encode map e239 with expected crc 2023-06-03T20:21:05.335 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:21:05 smithi089 bash[75068]: cluster 1685823663.902782 osd.7 (osd.7) 171 : cluster 3 failed to encode map e239 with expected crc 2023-06-03T20:21:05.335 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:21:05 smithi089 bash[75068]: cluster 1685823663.9043493 osd.7 (osd.7) 172 : cluster 3 failed to encode map e239 with expected crc 2023-06-03T20:21:05.336 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:21:05 smithi089 bash[75068]: cluster 1685823663.9082994 osd.7 (osd.7) 173 : cluster 3 failed to encode map e239 with expected crc 2023-06-03T20:21:05.336 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:21:05 smithi089 bash[75068]: cluster 1685823663.9098747 osd.7 (osd.7) 174 : cluster 3 failed to encode map e239 with expected crc 2023-06-03T20:21:05.336 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:21:05 smithi089 bash[75068]: cluster 1685823664.2679386 osd.3 (osd.3) 13 : cluster 0 12.4 scrub starts 2023-06-03T20:21:05.336 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:21:05 smithi089 bash[75068]: cluster 1685823664.2871 osd.3 (osd.3) 14 : cluster 0 12.4 scrub ok 2023-06-03T20:21:05.336 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:21:05 smithi089 bash[75068]: cluster 1685823664.486712 osd.6 (osd.6) 183 : cluster 3 failed to encode map e239 with expected crc 2023-06-03T20:21:05.336 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:21:05 smithi089 bash[75068]: cluster 1685823664.5440433 osd.1 (osd.1) 7 : cluster 0 12.17 scrub starts 2023-06-03T20:21:05.336 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:21:05 smithi089 bash[75068]: cluster 1685823664.5536442 osd.1 (osd.1) 8 : cluster 0 12.17 scrub ok 2023-06-03T20:21:05.337 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:21:05 smithi164 bash[59462]: cluster 1685823663.9012125 osd.5 (osd.5) 177 : cluster 3 failed to encode map e239 with expected crc 2023-06-03T20:21:05.337 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:21:05 smithi164 bash[59462]: cluster 1685823663.902782 osd.7 (osd.7) 171 : cluster 3 failed to encode map e239 with expected crc 2023-06-03T20:21:05.337 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:21:05 smithi164 bash[59462]: cluster 1685823663.9043493 osd.7 (osd.7) 172 : cluster 3 failed to encode map e239 with expected crc 2023-06-03T20:21:05.337 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:21:05 smithi164 bash[59462]: cluster 1685823663.9082994 osd.7 (osd.7) 173 : cluster 3 failed to encode map e239 with expected crc 2023-06-03T20:21:05.337 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:21:05 smithi164 bash[59462]: cluster 1685823663.9098747 osd.7 (osd.7) 174 : cluster 3 failed to encode map e239 with expected crc 2023-06-03T20:21:05.338 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:21:05 smithi164 bash[59462]: cluster 1685823664.2679386 osd.3 (osd.3) 13 : cluster 0 12.4 scrub starts 2023-06-03T20:21:05.338 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:21:05 smithi164 bash[59462]: cluster 1685823664.2871 osd.3 (osd.3) 14 : cluster 0 12.4 scrub ok 2023-06-03T20:21:05.338 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:21:05 smithi164 bash[59462]: cluster 1685823664.486712 osd.6 (osd.6) 183 : cluster 3 failed to encode map e239 with expected crc 2023-06-03T20:21:05.338 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:21:05 smithi164 bash[59462]: cluster 1685823664.5440433 osd.1 (osd.1) 7 : cluster 0 12.17 scrub starts 2023-06-03T20:21:05.338 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:21:05 smithi164 bash[59462]: cluster 1685823664.5536442 osd.1 (osd.1) 8 : cluster 0 12.17 scrub ok 2023-06-03T20:21:05.352 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:21:05.356 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:21:05.356 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:21:05.457 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:21:05.489 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:21:05.491 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:21:05.492 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:21:05.593 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:21:05.631 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:21:05.634 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:21:05.635 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:21:05.736 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:21:05.770 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:21:05.773 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:21:05.773 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:21:05.874 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:21:05.913 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:21:05.916 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:21:05.916 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:21:06.017 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:21:06.054 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:21:06.058 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:21:06.059 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:21:06.160 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:21:06.173 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:21:06 smithi164 bash[59462]: cluster 1685823664.9528794 mgr.y (mgr.34104) 1354 : cluster 0 pgmap v1361: 99 pgs: 2 active+recovery_wait+degraded, 2 activating, 1 unknown, 6 peering, 6 remapped+peering, 13 active+clean+remapped, 7 active+undersized+degraded, 11 active+undersized, 51 active+clean; 14 MiB data, 4.1 GiB used, 622 GiB / 626 GiB avail; 77/699 objects degraded (11.016%); 7/699 objects misplaced (1.001%); 144 KiB/s, 0 objects/s recovering 2023-06-03T20:21:06.173 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:21:06 smithi164 bash[59462]: cluster 1685823665.459172 osd.2 (osd.2) 13 : cluster 0 12.9 scrub starts 2023-06-03T20:21:06.173 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:21:06 smithi164 bash[59462]: cluster 1685823665.4734094 osd.2 (osd.2) 14 : cluster 0 12.9 scrub ok 2023-06-03T20:21:06.196 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:21:06.199 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:21:06.199 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:21:06.300 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:21:06.336 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:21:06.339 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:21:06.339 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:21:06.440 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:21:06.475 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:21:06.478 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:21:06.478 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:21:06.529 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:21:06 smithi089 bash[75068]: cluster 1685823664.9528794 mgr.y (mgr.34104) 1354 : cluster 0 pgmap v1361: 99 pgs: 2 active+recovery_wait+degraded, 2 activating, 1 unknown, 6 peering, 6 remapped+peering, 13 active+clean+remapped, 7 active+undersized+degraded, 11 active+undersized, 51 active+clean; 14 MiB data, 4.1 GiB used, 622 GiB / 626 GiB avail; 77/699 objects degraded (11.016%); 7/699 objects misplaced (1.001%); 144 KiB/s, 0 objects/s recovering 2023-06-03T20:21:06.529 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:21:06 smithi089 bash[75068]: cluster 1685823665.459172 osd.2 (osd.2) 13 : cluster 0 12.9 scrub starts 2023-06-03T20:21:06.529 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:21:06 smithi089 bash[75068]: cluster 1685823665.4734094 osd.2 (osd.2) 14 : cluster 0 12.9 scrub ok 2023-06-03T20:21:06.530 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 20:21:06 smithi089 bash[71163]: ::ffff:172.21.15.164 - - [03/Jun/2023:20:21:06] "GET /metrics HTTP/1.1" 200 228934 "" "Prometheus/2.33.4" 2023-06-03T20:21:06.530 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:21:06 smithi089 bash[78452]: cluster 1685823664.9528794 mgr.y (mgr.34104) 1354 : cluster 0 pgmap v1361: 99 pgs: 2 active+recovery_wait+degraded, 2 activating, 1 unknown, 6 peering, 6 remapped+peering, 13 active+clean+remapped, 7 active+undersized+degraded, 11 active+undersized, 51 active+clean; 14 MiB data, 4.1 GiB used, 622 GiB / 626 GiB avail; 77/699 objects degraded (11.016%); 7/699 objects misplaced (1.001%); 144 KiB/s, 0 objects/s recovering 2023-06-03T20:21:06.530 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:21:06 smithi089 bash[78452]: cluster 1685823665.459172 osd.2 (osd.2) 13 : cluster 0 12.9 scrub starts 2023-06-03T20:21:06.530 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:21:06 smithi089 bash[78452]: cluster 1685823665.4734094 osd.2 (osd.2) 14 : cluster 0 12.9 scrub ok 2023-06-03T20:21:06.557 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph osd unset nodeep-scrub 2023-06-03T20:21:06.580 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:21:06.614 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:21:06.616 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:21:06.617 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:21:06.718 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:21:06.744 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:21:06.746 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:21:06.747 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:21:06.848 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:21:06.886 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:21:06.889 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:21:06.890 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:21:06.991 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:21:07.032 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:21:07.034 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:21:07.035 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:21:07.136 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:21:07.148 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:21:07 smithi164 bash[59462]: cluster 1685823666.2313979 osd.3 (osd.3) 15 : cluster 0 12.14 scrub starts 2023-06-03T20:21:07.149 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:21:07 smithi164 bash[59462]: cluster 1685823666.2496233 osd.3 (osd.3) 16 : cluster 0 12.14 scrub ok 2023-06-03T20:21:07.173 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:21:07.178 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:21:07.179 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:21:07.280 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:21:07.313 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:21:07.319 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:21:07 smithi089 bash[78452]: cluster 1685823666.2313979 osd.3 (osd.3) 15 : cluster 0 12.14 scrub starts 2023-06-03T20:21:07.319 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:21:07 smithi089 bash[78452]: cluster 1685823666.2496233 osd.3 (osd.3) 16 : cluster 0 12.14 scrub ok 2023-06-03T20:21:07.319 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:21:07 smithi089 bash[75068]: cluster 1685823666.2313979 osd.3 (osd.3) 15 : cluster 0 12.14 scrub starts 2023-06-03T20:21:07.319 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:21:07 smithi089 bash[75068]: cluster 1685823666.2496233 osd.3 (osd.3) 16 : cluster 0 12.14 scrub ok 2023-06-03T20:21:07.320 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:21:07.320 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:21:07.421 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:21:07.450 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:21:07.452 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:21:07.453 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:21:07.554 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:21:07.595 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:21:07.597 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:21:07.598 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:21:07.699 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:21:07.732 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:21:07.735 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:21:07.735 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:21:07.836 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:21:07.874 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:21:07.877 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:21:07.877 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:21:07.978 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:21:08.015 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:21:08.019 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:21:08.019 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:21:08.120 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:21:08.157 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:21:08.163 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:21:08.164 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:21:08.169 INFO:teuthology.orchestra.run.smithi089.stderr:pool 'unique_pool_7' removed 2023-06-03T20:21:08.265 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:21:08.305 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:21:08.310 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:21:08.310 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:21:08.334 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:21:08 smithi164 bash[59462]: cluster 1685823666.9539793 mgr.y (mgr.34104) 1355 : cluster 0 pgmap v1362: 99 pgs: 2 active+recovery_wait+degraded, 2 activating, 2 peering, 5 remapped+peering, 14 active+clean+remapped, 7 active+undersized+degraded, 11 active+undersized, 56 active+clean; 14 MiB data, 4.1 GiB used, 622 GiB / 626 GiB avail; 77/699 objects degraded (11.016%); 7/699 objects misplaced (1.001%); 101 KiB/s, 0 objects/s recovering 2023-06-03T20:21:08.334 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:21:08 smithi164 bash[59462]: audit 1685823667.1390753 mon.a (mon.0) 1337 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:21:08.335 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:21:08 smithi164 bash[59462]: audit 1685823667.1422112 mon.c (mon.2) 582 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T20:21:08.335 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:21:08 smithi164 bash[59462]: cluster 1685823667.4669824 osd.2 (osd.2) 15 : cluster 0 12.5 scrub starts 2023-06-03T20:21:08.335 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:21:08 smithi164 bash[59462]: cluster 1685823667.4926956 osd.2 (osd.2) 16 : cluster 0 12.5 scrub ok 2023-06-03T20:21:08.336 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:21:08 smithi164 bash[59462]: audit 1685823667.5206823 mon.c (mon.2) 583 : audit 1 from='client.? 172.21.15.89:0/138972759' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_7", "pool2": "unique_pool_7", "yes_i_really_really_mean_it": true}]: dispatch 2023-06-03T20:21:08.336 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:21:08 smithi164 bash[59462]: audit 1685823667.5215478 mon.a (mon.0) 1338 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_7", "pool2": "unique_pool_7", "yes_i_really_really_mean_it": true}]: dispatch 2023-06-03T20:21:08.336 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:21:08 smithi164 bash[59462]: cluster 1685823667.525147 osd.1 (osd.1) 9 : cluster 0 12.16 scrub starts 2023-06-03T20:21:08.336 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:21:08 smithi164 bash[59462]: cluster 1685823667.5382986 osd.1 (osd.1) 10 : cluster 0 12.16 scrub ok 2023-06-03T20:21:08.336 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:21:08 smithi164 bash[59462]: cluster 1685823667.87858 mon.a (mon.0) 1339 : cluster 3 Health check update: Degraded data redundancy: 77/699 objects degraded (11.016%), 9 pgs degraded, 18 pgs undersized (PG_DEGRADED) 2023-06-03T20:21:08.411 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:21:08.449 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:21:08.451 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:21:08.451 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:21:08.529 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:21:08 smithi089 bash[75068]: cluster 1685823666.9539793 mgr.y (mgr.34104) 1355 : cluster 0 pgmap v1362: 99 pgs: 2 active+recovery_wait+degraded, 2 activating, 2 peering, 5 remapped+peering, 14 active+clean+remapped, 7 active+undersized+degraded, 11 active+undersized, 56 active+clean; 14 MiB data, 4.1 GiB used, 622 GiB / 626 GiB avail; 77/699 objects degraded (11.016%); 7/699 objects misplaced (1.001%); 101 KiB/s, 0 objects/s recovering 2023-06-03T20:21:08.529 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:21:08 smithi089 bash[75068]: audit 1685823667.1390753 mon.a (mon.0) 1337 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:21:08.529 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:21:08 smithi089 bash[75068]: audit 1685823667.1422112 mon.c (mon.2) 582 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T20:21:08.530 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:21:08 smithi089 bash[75068]: cluster 1685823667.4669824 osd.2 (osd.2) 15 : cluster 0 12.5 scrub starts 2023-06-03T20:21:08.530 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:21:08 smithi089 bash[75068]: cluster 1685823667.4926956 osd.2 (osd.2) 16 : cluster 0 12.5 scrub ok 2023-06-03T20:21:08.530 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:21:08 smithi089 bash[75068]: audit 1685823667.5206823 mon.c (mon.2) 583 : audit 1 from='client.? 172.21.15.89:0/138972759' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_7", "pool2": "unique_pool_7", "yes_i_really_really_mean_it": true}]: dispatch 2023-06-03T20:21:08.530 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:21:08 smithi089 bash[75068]: audit 1685823667.5215478 mon.a (mon.0) 1338 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_7", "pool2": "unique_pool_7", "yes_i_really_really_mean_it": true}]: dispatch 2023-06-03T20:21:08.530 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:21:08 smithi089 bash[75068]: cluster 1685823667.525147 osd.1 (osd.1) 9 : cluster 0 12.16 scrub starts 2023-06-03T20:21:08.530 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:21:08 smithi089 bash[75068]: cluster 1685823667.5382986 osd.1 (osd.1) 10 : cluster 0 12.16 scrub ok 2023-06-03T20:21:08.530 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:21:08 smithi089 bash[75068]: cluster 1685823667.87858 mon.a (mon.0) 1339 : cluster 3 Health check update: Degraded data redundancy: 77/699 objects degraded (11.016%), 9 pgs degraded, 18 pgs undersized (PG_DEGRADED) 2023-06-03T20:21:08.531 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:21:08 smithi089 bash[78452]: cluster 1685823666.9539793 mgr.y (mgr.34104) 1355 : cluster 0 pgmap v1362: 99 pgs: 2 active+recovery_wait+degraded, 2 activating, 2 peering, 5 remapped+peering, 14 active+clean+remapped, 7 active+undersized+degraded, 11 active+undersized, 56 active+clean; 14 MiB data, 4.1 GiB used, 622 GiB / 626 GiB avail; 77/699 objects degraded (11.016%); 7/699 objects misplaced (1.001%); 101 KiB/s, 0 objects/s recovering 2023-06-03T20:21:08.531 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:21:08 smithi089 bash[78452]: audit 1685823667.1390753 mon.a (mon.0) 1337 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:21:08.531 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:21:08 smithi089 bash[78452]: audit 1685823667.1422112 mon.c (mon.2) 582 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T20:21:08.531 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:21:08 smithi089 bash[78452]: cluster 1685823667.4669824 osd.2 (osd.2) 15 : cluster 0 12.5 scrub starts 2023-06-03T20:21:08.531 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:21:08 smithi089 bash[78452]: cluster 1685823667.4926956 osd.2 (osd.2) 16 : cluster 0 12.5 scrub ok 2023-06-03T20:21:08.531 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:21:08 smithi089 bash[78452]: audit 1685823667.5206823 mon.c (mon.2) 583 : audit 1 from='client.? 172.21.15.89:0/138972759' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_7", "pool2": "unique_pool_7", "yes_i_really_really_mean_it": true}]: dispatch 2023-06-03T20:21:08.532 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:21:08 smithi089 bash[78452]: audit 1685823667.5215478 mon.a (mon.0) 1338 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_7", "pool2": "unique_pool_7", "yes_i_really_really_mean_it": true}]: dispatch 2023-06-03T20:21:08.532 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:21:08 smithi089 bash[78452]: cluster 1685823667.525147 osd.1 (osd.1) 9 : cluster 0 12.16 scrub starts 2023-06-03T20:21:08.532 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:21:08 smithi089 bash[78452]: cluster 1685823667.5382986 osd.1 (osd.1) 10 : cluster 0 12.16 scrub ok 2023-06-03T20:21:08.532 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:21:08 smithi089 bash[78452]: cluster 1685823667.87858 mon.a (mon.0) 1339 : cluster 3 Health check update: Degraded data redundancy: 77/699 objects degraded (11.016%), 9 pgs degraded, 18 pgs undersized (PG_DEGRADED) 2023-06-03T20:21:08.553 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:21:08.581 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:21:08.583 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:21:08.584 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:21:08.685 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:21:08.718 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:21:08.720 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:21:08.721 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:21:08.821 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:21:08.854 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:21:08.857 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:21:08.858 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:21:08.958 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:21:08.988 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:21:08.991 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:21:08.992 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:21:09.093 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:21:09.130 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:21:09.133 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:21:09.134 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:21:09.235 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:21:09.272 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:21:09.278 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:21:09.279 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:21:09.325 INFO:teuthology.task.full_sequential:In full_sequential, running task radosbench... 2023-06-03T20:21:09.327 INFO:tasks.radosbench:Beginning radosbench... 2023-06-03T20:21:09.327 INFO:tasks.cephadm.ceph_manager.ceph:creating pool_name unique_pool_8 2023-06-03T20:21:09.328 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph osd pool create unique_pool_8 16 2023-06-03T20:21:09.379 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:21:09.414 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:21:09.418 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:21:09.418 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:21:09 smithi089 bash[75068]: cluster 1685823668.1506073 osd.6 (osd.6) 184 : cluster 3 failed to encode map e240 with expected crc 2023-06-03T20:21:09.418 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:21:09 smithi089 bash[75068]: audit 1685823668.155449 mon.a (mon.0) 1340 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_7", "pool2": "unique_pool_7", "yes_i_really_really_mean_it": true}]': finished 2023-06-03T20:21:09.419 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:21:09 smithi089 bash[75068]: cluster 1685823668.1555362 mon.a (mon.0) 1341 : cluster 0 osdmap e240: 8 total, 7 up, 7 in 2023-06-03T20:21:09.419 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:21:09 smithi089 bash[75068]: cluster 1685823668.4787643 osd.5 (osd.5) 178 : cluster 3 failed to encode map e240 with expected crc 2023-06-03T20:21:09.419 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:21:09 smithi089 bash[75068]: cluster 1685823668.4818046 osd.5 (osd.5) 179 : cluster 3 failed to encode map e240 with expected crc 2023-06-03T20:21:09.419 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:21:09 smithi089 bash[78452]: cluster 1685823668.1506073 osd.6 (osd.6) 184 : cluster 3 failed to encode map e240 with expected crc 2023-06-03T20:21:09.419 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:21:09 smithi089 bash[78452]: audit 1685823668.155449 mon.a (mon.0) 1340 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_7", "pool2": "unique_pool_7", "yes_i_really_really_mean_it": true}]': finished 2023-06-03T20:21:09.419 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:21:09 smithi089 bash[78452]: cluster 1685823668.1555362 mon.a (mon.0) 1341 : cluster 0 osdmap e240: 8 total, 7 up, 7 in 2023-06-03T20:21:09.420 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:21:09 smithi089 bash[78452]: cluster 1685823668.4787643 osd.5 (osd.5) 178 : cluster 3 failed to encode map e240 with expected crc 2023-06-03T20:21:09.420 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:21:09 smithi089 bash[78452]: cluster 1685823668.4818046 osd.5 (osd.5) 179 : cluster 3 failed to encode map e240 with expected crc 2023-06-03T20:21:09.420 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:21:09.521 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:21:09.533 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:21:09 smithi164 bash[59462]: cluster 1685823668.1506073 osd.6 (osd.6) 184 : cluster 3 failed to encode map e240 with expected crc 2023-06-03T20:21:09.534 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:21:09 smithi164 bash[59462]: audit 1685823668.155449 mon.a (mon.0) 1340 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_7", "pool2": "unique_pool_7", "yes_i_really_really_mean_it": true}]': finished 2023-06-03T20:21:09.534 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:21:09 smithi164 bash[59462]: cluster 1685823668.1555362 mon.a (mon.0) 1341 : cluster 0 osdmap e240: 8 total, 7 up, 7 in 2023-06-03T20:21:09.534 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:21:09 smithi164 bash[59462]: cluster 1685823668.4787643 osd.5 (osd.5) 178 : cluster 3 failed to encode map e240 with expected crc 2023-06-03T20:21:09.534 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:21:09 smithi164 bash[59462]: cluster 1685823668.4818046 osd.5 (osd.5) 179 : cluster 3 failed to encode map e240 with expected crc 2023-06-03T20:21:09.554 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:21:09.557 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:21:09.558 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:21:09.573 INFO:tasks.thrashosds.thrasher:in_osds: [0, 2, 3, 4, 5, 7, 1] out_osds: [6] dead_osds: [4] live_osds: [0, 1, 2, 3, 5, 6, 7] 2023-06-03T20:21:09.574 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-06-03T20:21:09.658 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:21:09.694 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:21:09.696 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:21:09.697 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:21:09.752 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/mon.a/config 2023-06-03T20:21:09.752 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/mon.a/config 2023-06-03T20:21:09.798 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:21:09.837 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:21:09.840 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:21:09.840 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:21:09.941 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:21:09.973 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:21:09.976 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:21:09.977 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:21:10.078 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:21:10.118 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:21:10.122 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:21:10.122 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:21:10.223 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:21:10.262 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:21:10.265 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:21:10.265 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:21:10.366 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:21:10.396 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:21:10.399 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:21:10.399 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:21:10.500 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:21:10.511 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:21:10 smithi089 bash[78452]: cluster 1685823668.955046 mgr.y (mgr.34104) 1356 : cluster 0 pgmap v1364: 73 pgs: 17 active+clean+remapped, 4 active+undersized+degraded, 5 active+undersized, 47 active+clean; 582 KiB data, 4.1 GiB used, 622 GiB / 626 GiB avail; 6/72 objects degraded (8.333%); 6/72 objects misplaced (8.333%); 95 KiB/s, 0 objects/s recovering 2023-06-03T20:21:10.511 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:21:10 smithi089 bash[78452]: cluster 1685823669.1459618 mon.a (mon.0) 1342 : cluster 1 Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive) 2023-06-03T20:21:10.511 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:21:10 smithi089 bash[75068]: cluster 1685823668.955046 mgr.y (mgr.34104) 1356 : cluster 0 pgmap v1364: 73 pgs: 17 active+clean+remapped, 4 active+undersized+degraded, 5 active+undersized, 47 active+clean; 582 KiB data, 4.1 GiB used, 622 GiB / 626 GiB avail; 6/72 objects degraded (8.333%); 6/72 objects misplaced (8.333%); 95 KiB/s, 0 objects/s recovering 2023-06-03T20:21:10.512 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:21:10 smithi089 bash[75068]: cluster 1685823669.1459618 mon.a (mon.0) 1342 : cluster 1 Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive) 2023-06-03T20:21:10.534 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:21:10.536 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:21:10.537 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:21:10.584 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:21:10 smithi164 bash[59462]: cluster 1685823668.955046 mgr.y (mgr.34104) 1356 : cluster 0 pgmap v1364: 73 pgs: 17 active+clean+remapped, 4 active+undersized+degraded, 5 active+undersized, 47 active+clean; 582 KiB data, 4.1 GiB used, 622 GiB / 626 GiB avail; 6/72 objects degraded (8.333%); 6/72 objects misplaced (8.333%); 95 KiB/s, 0 objects/s recovering 2023-06-03T20:21:10.584 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:21:10 smithi164 bash[59462]: cluster 1685823669.1459618 mon.a (mon.0) 1342 : cluster 1 Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive) 2023-06-03T20:21:10.638 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:21:10.667 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:21:10.670 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:21:10.670 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:21:10.771 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:21:10.808 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:21:10.810 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:21:10.811 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:21:10.911 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:21:10.952 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:21:10.956 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:21:10.956 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:21:11.057 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:21:11.087 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:21:11.089 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:21:11.090 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:21:11.191 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:21:11.229 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:21:11.232 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:21:11.233 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:21:11.334 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:21:11.370 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:21:11.373 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:21:11.373 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:21:11.474 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:21:11.485 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:21:11 smithi089 bash[75068]: cluster 1685823669.225457 osd.7 (osd.7) 175 : cluster 3 failed to encode map e240 with expected crc 2023-06-03T20:21:11.485 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:21:11 smithi089 bash[78452]: cluster 1685823669.225457 osd.7 (osd.7) 175 : cluster 3 failed to encode map e240 with expected crc 2023-06-03T20:21:11.512 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:21:11.515 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:21:11.515 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:21:11.584 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:21:11 smithi164 bash[59462]: cluster 1685823669.225457 osd.7 (osd.7) 175 : cluster 3 failed to encode map e240 with expected crc 2023-06-03T20:21:11.616 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:21:11.643 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:21:11.646 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:21:11.647 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:21:11.748 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:21:11.787 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:21:11.789 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:21:11.790 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:21:11.818 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/mon.a/config 2023-06-03T20:21:11.891 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:21:11.926 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:21:11.929 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:21:11.930 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:21:12.031 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:21:12.070 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:21:12.072 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:21:12.073 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:21:12.174 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:21:12.212 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:21:12.215 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:21:12.215 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:21:12.316 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:21:12.334 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:21:12 smithi164 bash[59462]: cluster 1685823670.9558532 mgr.y (mgr.34104) 1357 : cluster 0 pgmap v1365: 73 pgs: 17 active+clean+remapped, 4 active+undersized+degraded, 5 active+undersized, 47 active+clean; 582 KiB data, 4.1 GiB used, 622 GiB / 626 GiB avail; 6/72 objects degraded (8.333%); 6/72 objects misplaced (8.333%); 72 KiB/s, 0 objects/s recovering 2023-06-03T20:21:12.346 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:21:12.348 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:21:12.348 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:21:12.432 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:21:12 smithi089 bash[78452]: cluster 1685823670.9558532 mgr.y (mgr.34104) 1357 : cluster 0 pgmap v1365: 73 pgs: 17 active+clean+remapped, 4 active+undersized+degraded, 5 active+undersized, 47 active+clean; 582 KiB data, 4.1 GiB used, 622 GiB / 626 GiB avail; 6/72 objects degraded (8.333%); 6/72 objects misplaced (8.333%); 72 KiB/s, 0 objects/s recovering 2023-06-03T20:21:12.450 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:21:12.460 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:21:12 smithi089 bash[75068]: cluster 1685823670.9558532 mgr.y (mgr.34104) 1357 : cluster 0 pgmap v1365: 73 pgs: 17 active+clean+remapped, 4 active+undersized+degraded, 5 active+undersized, 47 active+clean; 582 KiB data, 4.1 GiB used, 622 GiB / 626 GiB avail; 6/72 objects degraded (8.333%); 6/72 objects misplaced (8.333%); 72 KiB/s, 0 objects/s recovering 2023-06-03T20:21:12.494 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:21:12.497 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:21:12.498 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:21:12.598 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:21:12.630 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:21:12.632 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:21:12.633 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:21:12.734 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:21:12.770 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:21:12.773 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:21:12.774 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:21:12.875 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:21:12.916 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:21:12.919 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:21:12.920 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:21:13.020 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:21:13.055 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:21:13.058 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:21:13.059 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:21:13.160 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:21:13.198 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:21:13.200 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:21:13.201 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:21:13.302 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:21:13.316 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:21:13 smithi164 bash[59462]: cluster 1685823672.8800497 mon.a (mon.0) 1343 : cluster 3 Health check update: Degraded data redundancy: 6/72 objects degraded (8.333%), 4 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-06-03T20:21:13.347 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:21:13.350 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:21:13.350 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:21:13.451 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:21:13.461 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:21:13 smithi089 bash[78452]: cluster 1685823672.8800497 mon.a (mon.0) 1343 : cluster 3 Health check update: Degraded data redundancy: 6/72 objects degraded (8.333%), 4 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-06-03T20:21:13.461 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:21:13 smithi089 bash[75068]: cluster 1685823672.8800497 mon.a (mon.0) 1343 : cluster 3 Health check update: Degraded data redundancy: 6/72 objects degraded (8.333%), 4 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-06-03T20:21:13.482 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:21:13.486 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:21:13.487 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:21:13.587 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:21:13.613 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:21:13.615 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:21:13.616 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:21:13.717 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:21:13.743 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:21:13.745 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:21:13.746 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:21:13.847 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:21:13.878 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:21:13.880 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:21:13.881 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:21:13.982 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:21:14.022 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:21:14.025 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:21:14.026 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:21:14.128 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:21:14.161 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:21:14.163 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:21:14.164 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:21:14.265 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:21:14.276 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:21:14 smithi164 bash[59462]: cluster 1685823672.9567552 mgr.y (mgr.34104) 1358 : cluster 0 pgmap v1366: 73 pgs: 17 active+clean+remapped, 4 active+undersized+degraded, 5 active+undersized, 47 active+clean; 582 KiB data, 4.1 GiB used, 622 GiB / 626 GiB avail; 6/72 objects degraded (8.333%); 6/72 objects misplaced (8.333%) 2023-06-03T20:21:14.299 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:21:14.302 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:21:14.303 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:21:14.404 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:21:14.441 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:21:14.444 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:21:14.445 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:21:14.529 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:21:14 smithi089 bash[75068]: cluster 1685823672.9567552 mgr.y (mgr.34104) 1358 : cluster 0 pgmap v1366: 73 pgs: 17 active+clean+remapped, 4 active+undersized+degraded, 5 active+undersized, 47 active+clean; 582 KiB data, 4.1 GiB used, 622 GiB / 626 GiB avail; 6/72 objects degraded (8.333%); 6/72 objects misplaced (8.333%) 2023-06-03T20:21:14.529 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:21:14 smithi089 bash[78452]: cluster 1685823672.9567552 mgr.y (mgr.34104) 1358 : cluster 0 pgmap v1366: 73 pgs: 17 active+clean+remapped, 4 active+undersized+degraded, 5 active+undersized, 47 active+clean; 582 KiB data, 4.1 GiB used, 622 GiB / 626 GiB avail; 6/72 objects degraded (8.333%); 6/72 objects misplaced (8.333%) 2023-06-03T20:21:14.546 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:21:14.585 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:21:14.587 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:21:14.588 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:21:14.689 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:21:14.720 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:21:14.722 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:21:14.722 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:21:14.823 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:21:14.848 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:21:14.850 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:21:14.850 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:21:14.951 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:21:14.986 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:21:14.989 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:21:14.990 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:21:15.090 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:21:15.122 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:21:15.124 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:21:15.125 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:21:15.226 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:21:15.261 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:21:15.263 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:21:15.264 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:21:15.365 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:21:15.399 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:21:15.402 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:21:15.403 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:21:15.504 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:21:15.542 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:21:15.545 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:21:15.546 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:21:15.647 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:21:15.674 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:21:15.677 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:21:15.677 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:21:15.779 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:21:15.808 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:21:15.811 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:21:15.811 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:21:15.913 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:21:15.941 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:21:15.942 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:21:15.943 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:21:16.044 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:21:16.083 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:21:16.085 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:21:16.086 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:21:16.187 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:21:16.222 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:21:16.225 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:21:16.225 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:21:16.326 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:21:16.334 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:21:16 smithi164 bash[59462]: cluster 1685823674.9577725 mgr.y (mgr.34104) 1359 : cluster 0 pgmap v1367: 73 pgs: 17 active+clean+remapped, 4 active+undersized+degraded, 5 active+undersized, 47 active+clean; 582 KiB data, 4.1 GiB used, 622 GiB / 626 GiB avail; 6/72 objects degraded (8.333%); 6/72 objects misplaced (8.333%) 2023-06-03T20:21:16.367 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:21:16.370 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:21:16.370 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:21:16.471 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:21:16.484 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:21:16 smithi089 bash[75068]: cluster 1685823674.9577725 mgr.y (mgr.34104) 1359 : cluster 0 pgmap v1367: 73 pgs: 17 active+clean+remapped, 4 active+undersized+degraded, 5 active+undersized, 47 active+clean; 582 KiB data, 4.1 GiB used, 622 GiB / 626 GiB avail; 6/72 objects degraded (8.333%); 6/72 objects misplaced (8.333%) 2023-06-03T20:21:16.485 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:21:16 smithi089 bash[78452]: cluster 1685823674.9577725 mgr.y (mgr.34104) 1359 : cluster 0 pgmap v1367: 73 pgs: 17 active+clean+remapped, 4 active+undersized+degraded, 5 active+undersized, 47 active+clean; 582 KiB data, 4.1 GiB used, 622 GiB / 626 GiB avail; 6/72 objects degraded (8.333%); 6/72 objects misplaced (8.333%) 2023-06-03T20:21:16.485 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 20:21:16 smithi089 bash[71163]: ::ffff:172.21.15.164 - - [03/Jun/2023:20:21:16] "GET /metrics HTTP/1.1" 200 228890 "" "Prometheus/2.33.4" 2023-06-03T20:21:16.508 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:21:16.510 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:21:16.511 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:21:16.585 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/mon.a/config 2023-06-03T20:21:16.612 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:21:16.646 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:21:16.649 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:21:16.650 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:21:16.751 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:21:16.790 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:21:16.793 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:21:16.794 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:21:16.894 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:21:16.933 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:21:16.936 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:21:16.936 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:21:17.037 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:21:17.070 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:21:17.073 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:21:17.073 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:21:17.174 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:21:17.199 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:21:17.200 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:21:17.201 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:21:17.302 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:21:17.339 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:21:17.342 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:21:17.343 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:21:17.443 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:21:17.485 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:21:17.488 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:21:17.489 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:21:17.589 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:21:17.626 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:21:17.629 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:21:17.630 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:21:17.731 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:21:17.759 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:21:17.763 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:21:17.763 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:21:17.864 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:21:17.902 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:21:17.905 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:21:17.905 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:21:18.006 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:21:18.045 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:21:18.048 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:21:18.048 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:21:18.149 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:21:18.174 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:21:18.177 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:21:18.178 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:21:18.278 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:21:18.279 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:21:18 smithi089 bash[78452]: cluster 1685823676.9586995 mgr.y (mgr.34104) 1360 : cluster 0 pgmap v1368: 73 pgs: 17 active+clean+remapped, 4 active+undersized+degraded, 5 active+undersized, 47 active+clean; 582 KiB data, 4.1 GiB used, 622 GiB / 626 GiB avail; 6/72 objects degraded (8.333%); 6/72 objects misplaced (8.333%) 2023-06-03T20:21:18.280 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:21:18 smithi089 bash[75068]: cluster 1685823676.9586995 mgr.y (mgr.34104) 1360 : cluster 0 pgmap v1368: 73 pgs: 17 active+clean+remapped, 4 active+undersized+degraded, 5 active+undersized, 47 active+clean; 582 KiB data, 4.1 GiB used, 622 GiB / 626 GiB avail; 6/72 objects degraded (8.333%); 6/72 objects misplaced (8.333%) 2023-06-03T20:21:18.291 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:21:18 smithi164 bash[59462]: cluster 1685823676.9586995 mgr.y (mgr.34104) 1360 : cluster 0 pgmap v1368: 73 pgs: 17 active+clean+remapped, 4 active+undersized+degraded, 5 active+undersized, 47 active+clean; 582 KiB data, 4.1 GiB used, 622 GiB / 626 GiB avail; 6/72 objects degraded (8.333%); 6/72 objects misplaced (8.333%) 2023-06-03T20:21:18.314 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:21:18.317 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:21:18.318 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:21:18.419 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:21:18.447 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:21:18.449 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:21:18.451 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:21:18.552 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:21:18.585 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:21:18.588 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:21:18.589 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:21:18.689 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:21:18.723 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:21:18.726 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:21:18.727 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:21:18.828 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:21:18.859 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:21:18.860 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:21:18.861 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:21:18.962 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:21:19.000 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:21:19.003 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:21:19.003 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:21:19.104 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:21:19.135 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:21:19.137 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:21:19.138 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:21:19.239 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:21:19.269 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:21:19.271 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:21:19.272 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:21:19.373 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:21:19.406 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:21:19.409 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:21:19.409 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:21:19.510 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:21:19.548 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:21:19.551 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:21:19.551 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:21:19.652 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:21:19.692 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:21:19.694 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:21:19.695 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:21:19.796 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:21:19.831 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:21:19.834 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:21:19.835 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:21:19.936 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:21:19.963 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:21:19.965 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:21:19.966 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:21:20.067 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:21:20.106 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:21:20.110 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:21:20.111 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:21:20.212 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:21:20.250 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:21:20.253 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:21:20.254 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:21:20.277 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:21:20 smithi089 bash[75068]: cluster 1685823678.9594045 mgr.y (mgr.34104) 1361 : cluster 0 pgmap v1369: 73 pgs: 17 active+clean+remapped, 4 active+undersized+degraded, 5 active+undersized, 47 active+clean; 582 KiB data, 4.1 GiB used, 622 GiB / 626 GiB avail; 6/72 objects degraded (8.333%); 6/72 objects misplaced (8.333%) 2023-06-03T20:21:20.277 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:21:20 smithi089 bash[78452]: cluster 1685823678.9594045 mgr.y (mgr.34104) 1361 : cluster 0 pgmap v1369: 73 pgs: 17 active+clean+remapped, 4 active+undersized+degraded, 5 active+undersized, 47 active+clean; 582 KiB data, 4.1 GiB used, 622 GiB / 626 GiB avail; 6/72 objects degraded (8.333%); 6/72 objects misplaced (8.333%) 2023-06-03T20:21:20.334 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:21:20 smithi164 bash[59462]: cluster 1685823678.9594045 mgr.y (mgr.34104) 1361 : cluster 0 pgmap v1369: 73 pgs: 17 active+clean+remapped, 4 active+undersized+degraded, 5 active+undersized, 47 active+clean; 582 KiB data, 4.1 GiB used, 622 GiB / 626 GiB avail; 6/72 objects degraded (8.333%); 6/72 objects misplaced (8.333%) 2023-06-03T20:21:20.356 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:21:20.387 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:21:20.390 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:21:20.390 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:21:20.491 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:21:20.525 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:21:20.528 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:21:20.529 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:21:20.630 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:21:20.667 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:21:20.670 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:21:20.670 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:21:20.771 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:21:20.800 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:21:20.804 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:21:20.804 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:21:20.905 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:21:20.940 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:21:20.942 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:21:20.943 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:21:21.044 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:21:21.085 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:21:21.088 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:21:21.088 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:21:21.189 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:21:21.226 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:21:21.229 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:21:21.230 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:21:21.331 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:21:21.366 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:21:21.368 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:21:21.369 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:21:21.470 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:21:21.511 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:21:21.514 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:21:21.515 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:21:21.616 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:21:21.649 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:21:21.651 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:21:21.652 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:21:21.753 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:21:21.790 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:21:21.792 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:21:21.793 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:21:21.893 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:21:21.923 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:21:21.925 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:21:21.926 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:21:22.027 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:21:22.058 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:21:22.061 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:21:22.061 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:21:22.162 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:21:22.201 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:21:22.204 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:21:22.205 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:21:22.306 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:21:22.320 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:21:22 smithi164 bash[59462]: cluster 1685823680.9600055 mgr.y (mgr.34104) 1362 : cluster 0 pgmap v1370: 73 pgs: 17 active+clean+remapped, 4 active+undersized+degraded, 5 active+undersized, 47 active+clean; 582 KiB data, 4.1 GiB used, 622 GiB / 626 GiB avail; 6/72 objects degraded (8.333%); 6/72 objects misplaced (8.333%) 2023-06-03T20:21:22.320 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:21:22 smithi164 bash[59462]: audit 1685823682.1379378 mon.a (mon.0) 1344 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:21:22.320 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:21:22 smithi164 bash[59462]: audit 1685823682.1393201 mon.c (mon.2) 584 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T20:21:22.348 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:21:22.351 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:21:22.352 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:21:22.453 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:21:22.491 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:21:22.493 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:21:22.494 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:21:22.529 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:21:22 smithi089 bash[78452]: cluster 1685823680.9600055 mgr.y (mgr.34104) 1362 : cluster 0 pgmap v1370: 73 pgs: 17 active+clean+remapped, 4 active+undersized+degraded, 5 active+undersized, 47 active+clean; 582 KiB data, 4.1 GiB used, 622 GiB / 626 GiB avail; 6/72 objects degraded (8.333%); 6/72 objects misplaced (8.333%) 2023-06-03T20:21:22.529 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:21:22 smithi089 bash[78452]: audit 1685823682.1379378 mon.a (mon.0) 1344 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:21:22.530 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:21:22 smithi089 bash[78452]: audit 1685823682.1393201 mon.c (mon.2) 584 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T20:21:22.530 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:21:22 smithi089 bash[75068]: cluster 1685823680.9600055 mgr.y (mgr.34104) 1362 : cluster 0 pgmap v1370: 73 pgs: 17 active+clean+remapped, 4 active+undersized+degraded, 5 active+undersized, 47 active+clean; 582 KiB data, 4.1 GiB used, 622 GiB / 626 GiB avail; 6/72 objects degraded (8.333%); 6/72 objects misplaced (8.333%) 2023-06-03T20:21:22.531 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:21:22 smithi089 bash[75068]: audit 1685823682.1379378 mon.a (mon.0) 1344 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:21:22.531 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:21:22 smithi089 bash[75068]: audit 1685823682.1393201 mon.c (mon.2) 584 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T20:21:22.582 INFO:teuthology.orchestra.run.smithi089.stdout:osd.0: {} 2023-06-03T20:21:22.595 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:21:22.605 INFO:teuthology.orchestra.run.smithi089.stdout:osd.1: {} 2023-06-03T20:21:22.626 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:21:22.629 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:21:22.630 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:21:22.632 INFO:teuthology.orchestra.run.smithi089.stdout:osd.2: {} 2023-06-03T20:21:22.658 INFO:teuthology.orchestra.run.smithi089.stdout:osd.3: {} 2023-06-03T20:21:22.684 INFO:teuthology.orchestra.run.smithi089.stdout:osd.5: {} 2023-06-03T20:21:22.713 INFO:teuthology.orchestra.run.smithi089.stdout:osd.6: {} 2023-06-03T20:21:22.731 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:21:22.735 INFO:teuthology.orchestra.run.smithi089.stdout:osd.7: {} 2023-06-03T20:21:22.736 INFO:teuthology.orchestra.run.smithi089.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-06-03T20:21:22.736 INFO:teuthology.orchestra.run.smithi089.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-06-03T20:21:22.736 INFO:teuthology.orchestra.run.smithi089.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-06-03T20:21:22.736 INFO:teuthology.orchestra.run.smithi089.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-06-03T20:21:22.736 INFO:teuthology.orchestra.run.smithi089.stderr:Error ENXIO: problem getting command descriptions from osd.4 2023-06-03T20:21:22.736 INFO:teuthology.orchestra.run.smithi089.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-06-03T20:21:22.737 INFO:teuthology.orchestra.run.smithi089.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-06-03T20:21:22.737 INFO:teuthology.orchestra.run.smithi089.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-06-03T20:21:22.767 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:21:22.769 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:21:22.769 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:21:22.869 INFO:teuthology.orchestra.run.smithi089.stdout:{ 2023-06-03T20:21:22.870 INFO:teuthology.orchestra.run.smithi089.stdout: "ops": [], 2023-06-03T20:21:22.870 INFO:teuthology.orchestra.run.smithi089.stdout: "complaint_time": 30, 2023-06-03T20:21:22.870 INFO:teuthology.orchestra.run.smithi089.stdout: "num_blocked_ops": 0 2023-06-03T20:21:22.870 INFO:teuthology.orchestra.run.smithi089.stdout:} 2023-06-03T20:21:22.870 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:21:22.898 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:21:22.900 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:21:22.901 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:21:23.002 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:21:23.031 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:21:23.034 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:21:23.035 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:21:23.136 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:21:23.159 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:21:23.161 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:21:23.162 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:21:23.263 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:21:23.303 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:21:23.305 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:21:23.306 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:21:23.407 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:21:23.442 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:21:23.446 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:21:23.447 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:21:23.529 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:21:23 smithi089 bash[78452]: audit 1685823682.5522964 mon.a (mon.0) 1345 : audit 0 from='client.? 172.21.15.89:0/4211080074' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-03T20:21:23.529 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:21:23 smithi089 bash[78452]: audit 1685823683.1345642 mon.a (mon.0) 1346 : audit 1 from='client.? 172.21.15.89:0/2139527603' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-06-03T20:21:23.530 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:21:23 smithi089 bash[75068]: audit 1685823682.5522964 mon.a (mon.0) 1345 : audit 0 from='client.? 172.21.15.89:0/4211080074' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-03T20:21:23.530 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:21:23 smithi089 bash[75068]: audit 1685823683.1345642 mon.a (mon.0) 1346 : audit 1 from='client.? 172.21.15.89:0/2139527603' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-06-03T20:21:23.547 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:21:23.561 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:21:23 smithi164 bash[59462]: audit 1685823682.5522964 mon.a (mon.0) 1345 : audit 0 from='client.? 172.21.15.89:0/4211080074' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-03T20:21:23.561 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:21:23 smithi164 bash[59462]: audit 1685823683.1345642 mon.a (mon.0) 1346 : audit 1 from='client.? 172.21.15.89:0/2139527603' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-06-03T20:21:23.584 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:21:23.587 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:21:23.588 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:21:23.688 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:21:23.722 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:21:23.724 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:21:23.725 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:21:23.826 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:21:23.864 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:21:23.868 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:21:23.869 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:21:23.904 INFO:teuthology.orchestra.run.smithi089.stderr:nodeep-scrub is unset 2023-06-03T20:21:23.969 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:21:24.009 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:21:24.012 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:21:24.012 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:21:24.113 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:21:24.152 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:21:24.155 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:21:24.156 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:21:24.203 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:21:24 smithi089 bash[75068]: cluster 1685823682.960695 mgr.y (mgr.34104) 1363 : cluster 0 pgmap v1371: 73 pgs: 17 active+clean+remapped, 4 active+undersized+degraded, 5 active+undersized, 47 active+clean; 582 KiB data, 4.1 GiB used, 622 GiB / 626 GiB avail; 6/72 objects degraded (8.333%); 6/72 objects misplaced (8.333%) 2023-06-03T20:21:24.204 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:21:24 smithi089 bash[75068]: cluster 1685823683.8857255 mon.a (mon.0) 1347 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-06-03T20:21:24.204 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:21:24 smithi089 bash[75068]: audit 1685823683.8958771 mon.a (mon.0) 1348 : audit 1 from='client.? 172.21.15.89:0/2139527603' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-06-03T20:21:24.204 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:21:24 smithi089 bash[75068]: cluster 1685823683.8959439 mon.a (mon.0) 1349 : cluster 0 osdmap e241: 8 total, 7 up, 7 in 2023-06-03T20:21:24.258 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:21:24.294 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:21:24.296 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:21:24.296 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:21:24.397 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:21:24.411 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:21:24 smithi164 bash[59462]: cluster 1685823682.960695 mgr.y (mgr.34104) 1363 : cluster 0 pgmap v1371: 73 pgs: 17 active+clean+remapped, 4 active+undersized+degraded, 5 active+undersized, 47 active+clean; 582 KiB data, 4.1 GiB used, 622 GiB / 626 GiB avail; 6/72 objects degraded (8.333%); 6/72 objects misplaced (8.333%) 2023-06-03T20:21:24.411 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:21:24 smithi164 bash[59462]: cluster 1685823683.8857255 mon.a (mon.0) 1347 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-06-03T20:21:24.412 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:21:24 smithi164 bash[59462]: audit 1685823683.8958771 mon.a (mon.0) 1348 : audit 1 from='client.? 172.21.15.89:0/2139527603' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-06-03T20:21:24.412 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:21:24 smithi164 bash[59462]: cluster 1685823683.8959439 mon.a (mon.0) 1349 : cluster 0 osdmap e241: 8 total, 7 up, 7 in 2023-06-03T20:21:24.434 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:21:24.437 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:21:24.438 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:21:24.529 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:21:24 smithi089 bash[78452]: cluster 1685823682.960695 mgr.y (mgr.34104) 1363 : cluster 0 pgmap v1371: 73 pgs: 17 active+clean+remapped, 4 active+undersized+degraded, 5 active+undersized, 47 active+clean; 582 KiB data, 4.1 GiB used, 622 GiB / 626 GiB avail; 6/72 objects degraded (8.333%); 6/72 objects misplaced (8.333%) 2023-06-03T20:21:24.529 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:21:24 smithi089 bash[78452]: cluster 1685823683.8857255 mon.a (mon.0) 1347 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-06-03T20:21:24.530 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:21:24 smithi089 bash[78452]: audit 1685823683.8958771 mon.a (mon.0) 1348 : audit 1 from='client.? 172.21.15.89:0/2139527603' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-06-03T20:21:24.530 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:21:24 smithi089 bash[78452]: cluster 1685823683.8959439 mon.a (mon.0) 1349 : cluster 0 osdmap e241: 8 total, 7 up, 7 in 2023-06-03T20:21:24.539 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:21:24.581 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:21:24.583 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:21:24.583 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:21:24.685 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:21:24.713 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:21:24.716 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:21:24.716 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:21:24.818 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:21:24.843 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:21:24.844 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:21:24.845 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:21:24.946 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:21:24.974 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:21:24.976 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:21:24.977 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:21:25.078 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:21:25.106 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:21:25.109 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:21:25.110 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:21:25.211 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:21:25.245 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:21:25.248 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:21:25.248 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:21:25.349 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:21:25.395 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:21:25.398 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:21:25.399 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:21:25.500 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:21:25.511 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:21:25 smithi164 bash[59462]: cluster 1685823683.8916354 osd.6 (osd.6) 185 : cluster 3 failed to encode map e241 with expected crc 2023-06-03T20:21:25.529 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:21:25 smithi089 bash[78452]: cluster 1685823683.8916354 osd.6 (osd.6) 185 : cluster 3 failed to encode map e241 with expected crc 2023-06-03T20:21:25.529 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:21:25 smithi089 bash[75068]: cluster 1685823683.8916354 osd.6 (osd.6) 185 : cluster 3 failed to encode map e241 with expected crc 2023-06-03T20:21:25.537 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:21:25.541 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:21:25.541 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:21:25.642 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:21:25.676 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:21:25.679 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:21:25.679 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:21:25.780 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:21:25.818 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:21:25.821 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:21:25.822 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:21:25.923 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:21:25.956 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:21:25.959 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:21:25.960 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:21:26.061 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:21:26.098 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:21:26.100 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:21:26.101 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:21:26.202 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:21:26.228 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:21:26.231 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:21:26.231 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:21:26.233 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-06-03T20:21:26.234 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-06-03T20:21:26.333 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:21:26.334 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:21:26 smithi164 bash[59462]: cluster 1685823684.961594 mgr.y (mgr.34104) 1364 : cluster 0 pgmap v1373: 73 pgs: 17 active+clean+remapped, 4 active+undersized+degraded, 5 active+undersized, 47 active+clean; 582 KiB data, 4.1 GiB used, 622 GiB / 626 GiB avail; 6/72 objects degraded (8.333%); 6/72 objects misplaced (8.333%) 2023-06-03T20:21:26.363 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:21:26.365 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:21:26.366 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:21:26.466 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:21:26.499 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:21:26.502 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:21:26.503 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:21:26.529 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:21:26 smithi089 bash[75068]: cluster 1685823684.961594 mgr.y (mgr.34104) 1364 : cluster 0 pgmap v1373: 73 pgs: 17 active+clean+remapped, 4 active+undersized+degraded, 5 active+undersized, 47 active+clean; 582 KiB data, 4.1 GiB used, 622 GiB / 626 GiB avail; 6/72 objects degraded (8.333%); 6/72 objects misplaced (8.333%) 2023-06-03T20:21:26.529 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 20:21:26 smithi089 bash[71163]: ::ffff:172.21.15.164 - - [03/Jun/2023:20:21:26] "GET /metrics HTTP/1.1" 200 226499 "" "Prometheus/2.33.4" 2023-06-03T20:21:26.530 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:21:26 smithi089 bash[78452]: cluster 1685823684.961594 mgr.y (mgr.34104) 1364 : cluster 0 pgmap v1373: 73 pgs: 17 active+clean+remapped, 4 active+undersized+degraded, 5 active+undersized, 47 active+clean; 582 KiB data, 4.1 GiB used, 622 GiB / 626 GiB avail; 6/72 objects degraded (8.333%); 6/72 objects misplaced (8.333%) 2023-06-03T20:21:26.604 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:21:26.632 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:21:26.635 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:21:26.635 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:21:26.736 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:21:26.776 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:21:26.779 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:21:26.780 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:21:26.881 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:21:26.916 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:21:26.920 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:21:26.920 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:21:26.956 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph daemon osd.0 dump_historic_ops 2023-06-03T20:21:27.021 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:21:27.047 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:21:27.050 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:21:27.050 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:21:27.151 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:21:27.193 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:21:27.196 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:21:27.197 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:21:27.298 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:21:27.334 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:21:27.337 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:21:27.337 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:21:27.438 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:21:27.479 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:21:27.482 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:21:27.482 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:21:27.583 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:21:27.618 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:21:27.621 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:21:27.622 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:21:27.723 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:21:27.761 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:21:27.764 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:21:27.764 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:21:27.865 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:21:27.907 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:21:27.910 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:21:27.910 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:21:28.011 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:21:28.047 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:21:28.050 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:21:28.050 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:21:28.151 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:21:28.186 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:21:28.188 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:21:28.189 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:21:28.235 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-06-03T20:21:28.290 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:21:28.325 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:21:28.329 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:21:28.330 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:21:28.431 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:21:28.438 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:21:28 smithi089 bash[78452]: cluster 1685823686.9623983 mgr.y (mgr.34104) 1365 : cluster 0 pgmap v1374: 73 pgs: 17 active+clean+remapped, 4 active+undersized+degraded, 5 active+undersized, 47 active+clean; 582 KiB data, 4.1 GiB used, 622 GiB / 626 GiB avail; 6/72 objects degraded (8.333%); 6/72 objects misplaced (8.333%) 2023-06-03T20:21:28.438 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:21:28 smithi089 bash[75068]: cluster 1685823686.9623983 mgr.y (mgr.34104) 1365 : cluster 0 pgmap v1374: 73 pgs: 17 active+clean+remapped, 4 active+undersized+degraded, 5 active+undersized, 47 active+clean; 582 KiB data, 4.1 GiB used, 622 GiB / 626 GiB avail; 6/72 objects degraded (8.333%); 6/72 objects misplaced (8.333%) 2023-06-03T20:21:28.455 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:21:28.458 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:21:28.459 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:21:28.560 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:21:28.571 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:21:28 smithi164 bash[59462]: cluster 1685823686.9623983 mgr.y (mgr.34104) 1365 : cluster 0 pgmap v1374: 73 pgs: 17 active+clean+remapped, 4 active+undersized+degraded, 5 active+undersized, 47 active+clean; 582 KiB data, 4.1 GiB used, 622 GiB / 626 GiB avail; 6/72 objects degraded (8.333%); 6/72 objects misplaced (8.333%) 2023-06-03T20:21:28.597 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:21:28.600 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:21:28.600 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:21:28.621 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph osd set noscrub 2023-06-03T20:21:28.701 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:21:28.735 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:21:28.737 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:21:28.738 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:21:28.839 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:21:28.875 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:21:28.878 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:21:28.879 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:21:28.980 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:21:29.019 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:21:29.021 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:21:29.022 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:21:29.123 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:21:29.158 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:21:29.161 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:21:29.162 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:21:29.263 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:21:29.299 INFO:teuthology.orchestra.run.smithi089.stderr:pool 'unique_pool_8' created 2023-06-03T20:21:29.302 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:21:29.305 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:21:29.305 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:21:29.407 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:21:29.443 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:21:29.446 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:21:29.447 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:21:29.547 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:21:29.561 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:21:29 smithi164 bash[59462]: audit 1685823688.7715127 mon.a (mon.0) 1350 : audit 1 from='client.? 172.21.15.89:0/1169680017' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_8", "pg_num": 16}]: dispatch 2023-06-03T20:21:29.561 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:21:29 smithi164 bash[59462]: cluster 1685823688.9634302 mgr.y (mgr.34104) 1366 : cluster 0 pgmap v1375: 73 pgs: 17 active+clean+remapped, 4 active+undersized+degraded, 5 active+undersized, 47 active+clean; 582 KiB data, 4.1 GiB used, 622 GiB / 626 GiB avail; 6/72 objects degraded (8.333%); 6/72 objects misplaced (8.333%) 2023-06-03T20:21:29.589 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:21:29.592 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:21:29.592 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:21:29.693 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:21:29.702 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:21:29 smithi089 bash[75068]: audit 1685823688.7715127 mon.a (mon.0) 1350 : audit 1 from='client.? 172.21.15.89:0/1169680017' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_8", "pg_num": 16}]: dispatch 2023-06-03T20:21:29.702 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:21:29 smithi089 bash[75068]: cluster 1685823688.9634302 mgr.y (mgr.34104) 1366 : cluster 0 pgmap v1375: 73 pgs: 17 active+clean+remapped, 4 active+undersized+degraded, 5 active+undersized, 47 active+clean; 582 KiB data, 4.1 GiB used, 622 GiB / 626 GiB avail; 6/72 objects degraded (8.333%); 6/72 objects misplaced (8.333%) 2023-06-03T20:21:29.702 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:21:29 smithi089 bash[78452]: audit 1685823688.7715127 mon.a (mon.0) 1350 : audit 1 from='client.? 172.21.15.89:0/1169680017' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_8", "pg_num": 16}]: dispatch 2023-06-03T20:21:29.702 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:21:29 smithi089 bash[78452]: cluster 1685823688.9634302 mgr.y (mgr.34104) 1366 : cluster 0 pgmap v1375: 73 pgs: 17 active+clean+remapped, 4 active+undersized+degraded, 5 active+undersized, 47 active+clean; 582 KiB data, 4.1 GiB used, 622 GiB / 626 GiB avail; 6/72 objects degraded (8.333%); 6/72 objects misplaced (8.333%) 2023-06-03T20:21:29.721 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:21:29.723 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:21:29.724 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:21:29.825 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:21:29.862 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:21:29.865 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:21:29.865 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:21:29.966 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:21:30.006 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:21:30.009 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:21:30.009 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:21:30.110 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:21:30.144 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:21:30.148 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:21:30.148 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:21:30.249 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:21:30.287 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:21:30.290 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:21:30.290 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:21:30.391 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:21:30.402 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:21:30 smithi089 bash[75068]: cluster 1685823689.2802098 osd.6 (osd.6) 186 : cluster 3 failed to encode map e242 with expected crc 2023-06-03T20:21:30.402 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:21:30 smithi089 bash[75068]: cluster 1685823689.2856283 osd.7 (osd.7) 176 : cluster 3 failed to encode map e242 with expected crc 2023-06-03T20:21:30.402 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:21:30 smithi089 bash[75068]: cluster 1685823689.285863 osd.5 (osd.5) 180 : cluster 3 failed to encode map e242 with expected crc 2023-06-03T20:21:30.403 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:21:30 smithi089 bash[75068]: audit 1685823689.2873 mon.a (mon.0) 1351 : audit 1 from='client.? 172.21.15.89:0/1169680017' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_8", "pg_num": 16}]': finished 2023-06-03T20:21:30.403 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:21:30 smithi089 bash[75068]: cluster 1685823689.2874403 mon.a (mon.0) 1352 : cluster 0 osdmap e242: 8 total, 7 up, 7 in 2023-06-03T20:21:30.403 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:21:30 smithi089 bash[75068]: cluster 1685823689.2893085 osd.5 (osd.5) 181 : cluster 3 failed to encode map e242 with expected crc 2023-06-03T20:21:30.403 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:21:30 smithi089 bash[78452]: cluster 1685823689.2802098 osd.6 (osd.6) 186 : cluster 3 failed to encode map e242 with expected crc 2023-06-03T20:21:30.403 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:21:30 smithi089 bash[78452]: cluster 1685823689.2856283 osd.7 (osd.7) 176 : cluster 3 failed to encode map e242 with expected crc 2023-06-03T20:21:30.403 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:21:30 smithi089 bash[78452]: cluster 1685823689.285863 osd.5 (osd.5) 180 : cluster 3 failed to encode map e242 with expected crc 2023-06-03T20:21:30.404 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:21:30 smithi089 bash[78452]: audit 1685823689.2873 mon.a (mon.0) 1351 : audit 1 from='client.? 172.21.15.89:0/1169680017' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_8", "pg_num": 16}]': finished 2023-06-03T20:21:30.404 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:21:30 smithi089 bash[78452]: cluster 1685823689.2874403 mon.a (mon.0) 1352 : cluster 0 osdmap e242: 8 total, 7 up, 7 in 2023-06-03T20:21:30.404 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:21:30 smithi089 bash[78452]: cluster 1685823689.2893085 osd.5 (osd.5) 181 : cluster 3 failed to encode map e242 with expected crc 2023-06-03T20:21:30.421 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:21:30.425 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:21:30.425 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:21:30.437 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph osd pool application enable unique_pool_8 rados --yes-i-really-mean-it || true 2023-06-03T20:21:30.526 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:21:30.562 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:21:30.565 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:21:30.566 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:21:30.584 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:21:30 smithi164 bash[59462]: cluster 1685823689.2802098 osd.6 (osd.6) 186 : cluster 3 failed to encode map e242 with expected crc 2023-06-03T20:21:30.584 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:21:30 smithi164 bash[59462]: cluster 1685823689.2856283 osd.7 (osd.7) 176 : cluster 3 failed to encode map e242 with expected crc 2023-06-03T20:21:30.585 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:21:30 smithi164 bash[59462]: cluster 1685823689.285863 osd.5 (osd.5) 180 : cluster 3 failed to encode map e242 with expected crc 2023-06-03T20:21:30.585 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:21:30 smithi164 bash[59462]: audit 1685823689.2873 mon.a (mon.0) 1351 : audit 1 from='client.? 172.21.15.89:0/1169680017' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_8", "pg_num": 16}]': finished 2023-06-03T20:21:30.585 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:21:30 smithi164 bash[59462]: cluster 1685823689.2874403 mon.a (mon.0) 1352 : cluster 0 osdmap e242: 8 total, 7 up, 7 in 2023-06-03T20:21:30.585 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:21:30 smithi164 bash[59462]: cluster 1685823689.2893085 osd.5 (osd.5) 181 : cluster 3 failed to encode map e242 with expected crc 2023-06-03T20:21:30.667 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:21:30.700 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:21:30.702 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:21:30.703 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:21:30.804 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:21:30.840 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:21:30.843 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:21:30.844 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:21:30.945 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:21:30.983 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:21:30.985 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:21:30.986 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:21:31.086 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:21:31.123 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:21:31.125 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:21:31.127 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:21:31.228 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:21:31.266 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:21:31.269 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:21:31.269 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:21:31.370 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:21:31.383 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:21:31 smithi089 bash[78452]: cluster 1685823690.2928638 osd.6 (osd.6) 187 : cluster 3 failed to encode map e243 with expected crc 2023-06-03T20:21:31.383 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:21:31 smithi089 bash[78452]: cluster 1685823690.297538 osd.7 (osd.7) 177 : cluster 3 failed to encode map e243 with expected crc 2023-06-03T20:21:31.383 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:21:31 smithi089 bash[78452]: cluster 1685823690.2991621 mon.a (mon.0) 1353 : cluster 0 osdmap e243: 8 total, 7 up, 7 in 2023-06-03T20:21:31.384 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:21:31 smithi089 bash[78452]: cluster 1685823690.9643023 mgr.y (mgr.34104) 1367 : cluster 0 pgmap v1378: 89 pgs: 12 unknown, 17 active+clean+remapped, 4 active+undersized+degraded, 7 active+undersized, 49 active+clean; 582 KiB data, 4.1 GiB used, 622 GiB / 626 GiB avail; 6/72 objects degraded (8.333%); 6/72 objects misplaced (8.333%) 2023-06-03T20:21:31.384 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:21:31 smithi089 bash[75068]: cluster 1685823690.2928638 osd.6 (osd.6) 187 : cluster 3 failed to encode map e243 with expected crc 2023-06-03T20:21:31.385 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:21:31 smithi089 bash[75068]: cluster 1685823690.297538 osd.7 (osd.7) 177 : cluster 3 failed to encode map e243 with expected crc 2023-06-03T20:21:31.385 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:21:31 smithi089 bash[75068]: cluster 1685823690.2991621 mon.a (mon.0) 1353 : cluster 0 osdmap e243: 8 total, 7 up, 7 in 2023-06-03T20:21:31.385 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:21:31 smithi089 bash[75068]: cluster 1685823690.9643023 mgr.y (mgr.34104) 1367 : cluster 0 pgmap v1378: 89 pgs: 12 unknown, 17 active+clean+remapped, 4 active+undersized+degraded, 7 active+undersized, 49 active+clean; 582 KiB data, 4.1 GiB used, 622 GiB / 626 GiB avail; 6/72 objects degraded (8.333%); 6/72 objects misplaced (8.333%) 2023-06-03T20:21:31.406 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:21:31.408 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:21:31.409 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:21:31.510 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:21:31.546 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:21:31.549 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:21:31.550 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:21:31.584 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:21:31 smithi164 bash[59462]: cluster 1685823690.2928638 osd.6 (osd.6) 187 : cluster 3 failed to encode map e243 with expected crc 2023-06-03T20:21:31.585 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:21:31 smithi164 bash[59462]: cluster 1685823690.297538 osd.7 (osd.7) 177 : cluster 3 failed to encode map e243 with expected crc 2023-06-03T20:21:31.585 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:21:31 smithi164 bash[59462]: cluster 1685823690.2991621 mon.a (mon.0) 1353 : cluster 0 osdmap e243: 8 total, 7 up, 7 in 2023-06-03T20:21:31.585 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:21:31 smithi164 bash[59462]: cluster 1685823690.9643023 mgr.y (mgr.34104) 1367 : cluster 0 pgmap v1378: 89 pgs: 12 unknown, 17 active+clean+remapped, 4 active+undersized+degraded, 7 active+undersized, 49 active+clean; 582 KiB data, 4.1 GiB used, 622 GiB / 626 GiB avail; 6/72 objects degraded (8.333%); 6/72 objects misplaced (8.333%) 2023-06-03T20:21:31.651 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:21:31.692 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:21:31.695 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:21:31.695 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:21:31.796 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:21:31.839 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:21:31.842 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:21:31.842 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:21:31.943 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:21:31.983 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:21:31.986 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:21:31.986 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:21:32.087 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:21:32.122 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:21:32.124 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:21:32.125 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:21:32.226 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:21:32.261 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:21:32.264 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:21:32.265 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:21:32.365 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:21:32.406 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:21:32.409 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:21:32.409 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:21:32.510 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:21:32.523 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:21:32 smithi089 bash[78452]: cluster 1685823691.31058 osd.6 (osd.6) 188 : cluster 3 failed to encode map e244 with expected crc 2023-06-03T20:21:32.524 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:21:32 smithi089 bash[78452]: cluster 1685823691.317515 mon.a (mon.0) 1354 : cluster 0 osdmap e244: 8 total, 7 up, 7 in 2023-06-03T20:21:32.524 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:21:32 smithi089 bash[75068]: cluster 1685823691.31058 osd.6 (osd.6) 188 : cluster 3 failed to encode map e244 with expected crc 2023-06-03T20:21:32.524 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:21:32 smithi089 bash[75068]: cluster 1685823691.317515 mon.a (mon.0) 1354 : cluster 0 osdmap e244: 8 total, 7 up, 7 in 2023-06-03T20:21:32.550 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:21:32.552 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:21:32.553 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:21:32.584 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:21:32 smithi164 bash[59462]: cluster 1685823691.31058 osd.6 (osd.6) 188 : cluster 3 failed to encode map e244 with expected crc 2023-06-03T20:21:32.584 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:21:32 smithi164 bash[59462]: cluster 1685823691.317515 mon.a (mon.0) 1354 : cluster 0 osdmap e244: 8 total, 7 up, 7 in 2023-06-03T20:21:32.654 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:21:32.691 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:21:32.694 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:21:32.695 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:21:32.796 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:21:32.832 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:21:32.834 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:21:32.835 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:21:32.924 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/mon.a/config 2023-06-03T20:21:32.925 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/mon.a/config 2023-06-03T20:21:32.926 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/mon.a/config 2023-06-03T20:21:32.936 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:21:32.971 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:21:32.974 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:21:32.974 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:21:33.076 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:21:33.111 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:21:33.114 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:21:33.115 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:21:33.216 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:21:33.252 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:21:33.254 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:21:33.255 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:21:33.356 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:21:33.385 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:21:33.387 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:21:33.388 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:21:33.489 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:21:33.497 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:21:33 smithi089 bash[75068]: cluster 1685823692.5607002 osd.7 (osd.7) 178 : cluster 3 failed to encode map e244 with expected crc 2023-06-03T20:21:33.497 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:21:33 smithi089 bash[75068]: cluster 1685823692.6293337 osd.5 (osd.5) 182 : cluster 3 failed to encode map e244 with expected crc 2023-06-03T20:21:33.497 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:21:33 smithi089 bash[75068]: cluster 1685823692.9651837 mgr.y (mgr.34104) 1368 : cluster 0 pgmap v1380: 89 pgs: 7 unknown, 17 active+clean+remapped, 4 active+undersized+degraded, 10 active+undersized, 51 active+clean; 582 KiB data, 4.1 GiB used, 622 GiB / 626 GiB avail; 6/72 objects degraded (8.333%); 6/72 objects misplaced (8.333%) 2023-06-03T20:21:33.498 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:21:33 smithi089 bash[78452]: cluster 1685823692.5607002 osd.7 (osd.7) 178 : cluster 3 failed to encode map e244 with expected crc 2023-06-03T20:21:33.498 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:21:33 smithi089 bash[78452]: cluster 1685823692.6293337 osd.5 (osd.5) 182 : cluster 3 failed to encode map e244 with expected crc 2023-06-03T20:21:33.498 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:21:33 smithi089 bash[78452]: cluster 1685823692.9651837 mgr.y (mgr.34104) 1368 : cluster 0 pgmap v1380: 89 pgs: 7 unknown, 17 active+clean+remapped, 4 active+undersized+degraded, 10 active+undersized, 51 active+clean; 582 KiB data, 4.1 GiB used, 622 GiB / 626 GiB avail; 6/72 objects degraded (8.333%); 6/72 objects misplaced (8.333%) 2023-06-03T20:21:33.517 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:21:33.520 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:21:33.521 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:21:33.584 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:21:33 smithi164 bash[59462]: cluster 1685823692.5607002 osd.7 (osd.7) 178 : cluster 3 failed to encode map e244 with expected crc 2023-06-03T20:21:33.584 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:21:33 smithi164 bash[59462]: cluster 1685823692.6293337 osd.5 (osd.5) 182 : cluster 3 failed to encode map e244 with expected crc 2023-06-03T20:21:33.585 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:21:33 smithi164 bash[59462]: cluster 1685823692.9651837 mgr.y (mgr.34104) 1368 : cluster 0 pgmap v1380: 89 pgs: 7 unknown, 17 active+clean+remapped, 4 active+undersized+degraded, 10 active+undersized, 51 active+clean; 582 KiB data, 4.1 GiB used, 622 GiB / 626 GiB avail; 6/72 objects degraded (8.333%); 6/72 objects misplaced (8.333%) 2023-06-03T20:21:33.622 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:21:33.657 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:21:33.659 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:21:33.660 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:21:33.761 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:21:33.792 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:21:33.795 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:21:33.796 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:21:33.897 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:21:33.924 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:21:33.926 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:21:33.926 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:21:34.027 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:21:34.056 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:21:34.058 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:21:34.059 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:21:34.160 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:21:34.199 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:21:34.202 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:21:34.202 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:21:34.303 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:21:34.341 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:21:34.343 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:21:34.344 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:21:34.445 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:21:34.479 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:21:34.481 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:21:34.482 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:21:34.582 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:21:34.616 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:21:34.618 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:21:34.619 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:21:34.720 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:21:34.753 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:21:34.756 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:21:34.757 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:21:34.858 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:21:34.884 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:21:34.886 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:21:34.887 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:21:34.964 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/mon.a/config 2023-06-03T20:21:34.988 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:21:35.014 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:21:35.017 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:21:35.017 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:21:35.119 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:21:35.158 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:21:35.160 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:21:35.161 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:21:35.262 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:21:35.296 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:21:35.298 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:21:35.299 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:21:35.400 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:21:35.434 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:21:35.437 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:21:35.437 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:21:35.538 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:21:35.576 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:21:35.579 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:21:35.580 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:21:35.681 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:21:35.719 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:21:35.722 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:21:35.723 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:21:35.824 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:21:35.866 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:21:35.869 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:21:35.869 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:21:35.970 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:21:36.005 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:21:36.009 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:21:36.009 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:21:36.110 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:21:36.123 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:21:36 smithi089 bash[78452]: cluster 1685823694.9658315 mgr.y (mgr.34104) 1369 : cluster 0 pgmap v1381: 89 pgs: 17 active+clean+remapped, 4 active+undersized+degraded, 14 active+undersized, 54 active+clean; 582 KiB data, 4.1 GiB used, 622 GiB / 626 GiB avail; 6/72 objects degraded (8.333%); 6/72 objects misplaced (8.333%) 2023-06-03T20:21:36.124 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:21:36 smithi089 bash[75068]: cluster 1685823694.9658315 mgr.y (mgr.34104) 1369 : cluster 0 pgmap v1381: 89 pgs: 17 active+clean+remapped, 4 active+undersized+degraded, 14 active+undersized, 54 active+clean; 582 KiB data, 4.1 GiB used, 622 GiB / 626 GiB avail; 6/72 objects degraded (8.333%); 6/72 objects misplaced (8.333%) 2023-06-03T20:21:36.150 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:21:36.153 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:21:36.153 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:21:36.254 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:21:36.284 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:21:36.286 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:21:36.287 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:21:36.334 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:21:36 smithi164 bash[59462]: cluster 1685823694.9658315 mgr.y (mgr.34104) 1369 : cluster 0 pgmap v1381: 89 pgs: 17 active+clean+remapped, 4 active+undersized+degraded, 14 active+undersized, 54 active+clean; 582 KiB data, 4.1 GiB used, 622 GiB / 626 GiB avail; 6/72 objects degraded (8.333%); 6/72 objects misplaced (8.333%) 2023-06-03T20:21:36.387 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:21:36.423 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:21:36.425 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:21:36.426 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:21:36.527 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:21:36.528 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 20:21:36 smithi089 bash[71163]: ::ffff:172.21.15.164 - - [03/Jun/2023:20:21:36] "GET /metrics HTTP/1.1" 200 226499 "" "Prometheus/2.33.4" 2023-06-03T20:21:36.560 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:21:36.562 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:21:36.563 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:21:36.664 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:21:36.695 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:21:36.698 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:21:36.699 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:21:36.800 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:21:36.833 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:21:36.836 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:21:36.837 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:21:36.938 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:21:36.971 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:21:36.973 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:21:36.974 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:21:37.074 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:21:37.115 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:21:37.118 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:21:37.119 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:21:37.220 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:21:37.251 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:21:37.253 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:21:37.254 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:21:37.355 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:21:37.389 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:21:37.393 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:21:37.394 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:21:37.495 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:21:37.520 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:21:37.522 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:21:37.523 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:21:37.624 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:21:37.662 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:21:37.665 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:21:37.666 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:21:37.767 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:21:37.800 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:21:37.800 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:21:37.801 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:21:37.902 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:21:37.938 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:21:37.940 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:21:37.941 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:21:38.042 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:21:38.072 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:21:38.074 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:21:38.075 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:21:38.175 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:21:38.204 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:21:38.206 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:21:38.207 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:21:38.293 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:21:38 smithi089 bash[75068]: cluster 1685823696.966377 mgr.y (mgr.34104) 1370 : cluster 0 pgmap v1382: 89 pgs: 17 active+clean+remapped, 4 active+undersized+degraded, 14 active+undersized, 54 active+clean; 582 KiB data, 4.1 GiB used, 622 GiB / 626 GiB avail; 6/72 objects degraded (8.333%); 6/72 objects misplaced (8.333%) 2023-06-03T20:21:38.293 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:21:38 smithi089 bash[75068]: audit 1685823697.1412427 mon.a (mon.0) 1355 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:21:38.293 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:21:38 smithi089 bash[75068]: audit 1685823697.1425645 mon.c (mon.2) 585 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T20:21:38.293 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:21:38 smithi089 bash[78452]: cluster 1685823696.966377 mgr.y (mgr.34104) 1370 : cluster 0 pgmap v1382: 89 pgs: 17 active+clean+remapped, 4 active+undersized+degraded, 14 active+undersized, 54 active+clean; 582 KiB data, 4.1 GiB used, 622 GiB / 626 GiB avail; 6/72 objects degraded (8.333%); 6/72 objects misplaced (8.333%) 2023-06-03T20:21:38.293 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:21:38 smithi089 bash[78452]: audit 1685823697.1412427 mon.a (mon.0) 1355 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:21:38.294 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:21:38 smithi089 bash[78452]: audit 1685823697.1425645 mon.c (mon.2) 585 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T20:21:38.308 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:21:38.334 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:21:38 smithi164 bash[59462]: cluster 1685823696.966377 mgr.y (mgr.34104) 1370 : cluster 0 pgmap v1382: 89 pgs: 17 active+clean+remapped, 4 active+undersized+degraded, 14 active+undersized, 54 active+clean; 582 KiB data, 4.1 GiB used, 622 GiB / 626 GiB avail; 6/72 objects degraded (8.333%); 6/72 objects misplaced (8.333%) 2023-06-03T20:21:38.334 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:21:38 smithi164 bash[59462]: audit 1685823697.1412427 mon.a (mon.0) 1355 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:21:38.334 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:21:38 smithi164 bash[59462]: audit 1685823697.1425645 mon.c (mon.2) 585 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T20:21:38.335 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:21:38.337 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:21:38.338 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:21:38.439 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:21:38.466 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:21:38.468 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:21:38.468 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:21:38.569 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:21:38.594 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:21:38.597 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:21:38.598 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:21:38.698 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:21:38.733 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:21:38.735 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:21:38.736 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:21:38.837 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:21:38.875 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:21:38.878 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:21:38.878 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:21:38.979 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:21:39.008 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:21:39.011 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:21:39.011 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:21:39.112 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:21:39.145 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:21:39.152 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:21:39.153 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:21:39.253 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:21:39.296 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:21:39.299 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:21:39.300 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:21:39.401 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:21:39.434 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:21:39.437 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:21:39.438 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:21:39.539 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:21:39.568 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:21:39.569 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:21:39.570 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:21:39.671 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:21:39.702 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:21:39.704 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:21:39.705 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:21:39.806 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:21:39.836 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:21:39.838 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:21:39.839 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:21:39.940 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:21:39.970 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:21:39.973 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:21:39.974 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:21:40.074 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:21:40.107 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:21:40.108 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:21:40.109 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:21:40.210 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:21:40.250 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:21:40.253 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:21:40.253 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:21:40.354 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:21:40.393 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:21:40.397 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:21:40.397 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:21:40.445 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:21:40 smithi089 bash[75068]: cluster 1685823698.9674015 mgr.y (mgr.34104) 1371 : cluster 0 pgmap v1383: 89 pgs: 17 active+clean+remapped, 4 active+undersized+degraded, 14 active+undersized, 54 active+clean; 582 KiB data, 4.1 GiB used, 622 GiB / 626 GiB avail; 6/72 objects degraded (8.333%); 6/72 objects misplaced (8.333%) 2023-06-03T20:21:40.446 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:21:40 smithi089 bash[78452]: cluster 1685823698.9674015 mgr.y (mgr.34104) 1371 : cluster 0 pgmap v1383: 89 pgs: 17 active+clean+remapped, 4 active+undersized+degraded, 14 active+undersized, 54 active+clean; 582 KiB data, 4.1 GiB used, 622 GiB / 626 GiB avail; 6/72 objects degraded (8.333%); 6/72 objects misplaced (8.333%) 2023-06-03T20:21:40.498 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:21:40.510 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:21:40 smithi164 bash[59462]: cluster 1685823698.9674015 mgr.y (mgr.34104) 1371 : cluster 0 pgmap v1383: 89 pgs: 17 active+clean+remapped, 4 active+undersized+degraded, 14 active+undersized, 54 active+clean; 582 KiB data, 4.1 GiB used, 622 GiB / 626 GiB avail; 6/72 objects degraded (8.333%); 6/72 objects misplaced (8.333%) 2023-06-03T20:21:40.533 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:21:40.536 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:21:40.536 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:21:40.637 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:21:40.674 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:21:40.677 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:21:40.678 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:21:40.778 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:21:40.804 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:21:40.807 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:21:40.807 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:21:40.908 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:21:40.939 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:21:40.941 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:21:40.941 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:21:41.042 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:21:41.074 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:21:41.076 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:21:41.077 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:21:41.178 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:21:41.212 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:21:41.215 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:21:41.216 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:21:41.317 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:21:41.356 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:21:41.360 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:21:41.361 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:21:41.461 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:21:41.497 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:21:41.499 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:21:41.500 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:21:41.601 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:21:41.640 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:21:41.643 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:21:41.644 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:21:41.745 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:21:41.776 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:21:41.779 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:21:41.780 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:21:41.881 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:21:41.913 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:21:41.916 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:21:41.917 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:21:42.017 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:21:42.053 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:21:42.055 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:21:42.056 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:21:42.157 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:21:42.194 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:21:42.197 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:21:42.198 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:21:42.279 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:21:42 smithi089 bash[78452]: cluster 1685823700.9680517 mgr.y (mgr.34104) 1372 : cluster 0 pgmap v1384: 89 pgs: 17 active+clean+remapped, 4 active+undersized+degraded, 14 active+undersized, 54 active+clean; 582 KiB data, 4.1 GiB used, 622 GiB / 626 GiB avail; 6/72 objects degraded (8.333%); 6/72 objects misplaced (8.333%) 2023-06-03T20:21:42.279 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:21:42 smithi089 bash[75068]: cluster 1685823700.9680517 mgr.y (mgr.34104) 1372 : cluster 0 pgmap v1384: 89 pgs: 17 active+clean+remapped, 4 active+undersized+degraded, 14 active+undersized, 54 active+clean; 582 KiB data, 4.1 GiB used, 622 GiB / 626 GiB avail; 6/72 objects degraded (8.333%); 6/72 objects misplaced (8.333%) 2023-06-03T20:21:42.299 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:21:42.338 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:21:42.341 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:21:42.342 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:21:42.443 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:21:42.478 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:21:42.480 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:21:42.481 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:21:42.581 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:21:42.584 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:21:42 smithi164 bash[59462]: cluster 1685823700.9680517 mgr.y (mgr.34104) 1372 : cluster 0 pgmap v1384: 89 pgs: 17 active+clean+remapped, 4 active+undersized+degraded, 14 active+undersized, 54 active+clean; 582 KiB data, 4.1 GiB used, 622 GiB / 626 GiB avail; 6/72 objects degraded (8.333%); 6/72 objects misplaced (8.333%) 2023-06-03T20:21:42.616 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:21:42.618 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:21:42.619 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:21:42.720 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:21:42.754 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:21:42.756 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:21:42.757 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:21:42.858 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:21:42.888 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:21:42.890 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:21:42.891 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:21:42.992 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:21:43.031 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:21:43.035 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:21:43.035 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:21:43.136 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:21:43.172 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:21:43.174 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:21:43.174 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:21:43.275 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:21:43.307 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:21:43.310 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:21:43.311 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:21:43.412 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:21:43.445 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:21:43.447 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:21:43.447 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:21:43.548 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:21:43.580 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:21:43.582 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:21:43.583 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:21:43.683 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:21:43.711 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:21:43.713 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:21:43.714 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:21:43.815 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:21:43.842 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:21:43.844 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:21:43.845 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:21:43.946 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:21:43.979 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:21:43.981 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:21:43.981 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:21:44.074 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:21:43 smithi089 bash[75068]: cluster 1685823702.9692438 mgr.y (mgr.34104) 1373 : cluster 0 pgmap v1385: 89 pgs: 17 active+clean+remapped, 4 active+undersized+degraded, 14 active+undersized, 54 active+clean; 582 KiB data, 4.1 GiB used, 622 GiB / 626 GiB avail; 6/72 objects degraded (8.333%); 6/72 objects misplaced (8.333%) 2023-06-03T20:21:44.074 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:21:43 smithi089 bash[78452]: cluster 1685823702.9692438 mgr.y (mgr.34104) 1373 : cluster 0 pgmap v1385: 89 pgs: 17 active+clean+remapped, 4 active+undersized+degraded, 14 active+undersized, 54 active+clean; 582 KiB data, 4.1 GiB used, 622 GiB / 626 GiB avail; 6/72 objects degraded (8.333%); 6/72 objects misplaced (8.333%) 2023-06-03T20:21:44.083 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:21:44.111 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:21:44.114 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:21:44.115 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:21:44.216 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:21:44.229 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:21:43 smithi164 bash[59462]: cluster 1685823702.9692438 mgr.y (mgr.34104) 1373 : cluster 0 pgmap v1385: 89 pgs: 17 active+clean+remapped, 4 active+undersized+degraded, 14 active+undersized, 54 active+clean; 582 KiB data, 4.1 GiB used, 622 GiB / 626 GiB avail; 6/72 objects degraded (8.333%); 6/72 objects misplaced (8.333%) 2023-06-03T20:21:44.251 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:21:44.254 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:21:44.254 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:21:44.355 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:21:44.393 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:21:44.396 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:21:44.396 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:21:44.497 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:21:44.532 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:21:44.534 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:21:44.535 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:21:44.635 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:21:44.672 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:21:44.675 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:21:44.675 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:21:44.776 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:21:44.809 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:21:44.812 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:21:44.813 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:21:44.914 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:21:44.955 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:21:44.959 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:21:44.960 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:21:45.061 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:21:45.094 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:21:45.097 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:21:45.098 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:21:45.199 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:21:45.236 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:21:45.239 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:21:45.240 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:21:45.341 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:21:45.374 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:21:45.377 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:21:45.378 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:21:45.479 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:21:45.508 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:21:45.512 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:21:45.513 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:21:45.613 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:21:45.650 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:21:45.652 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:21:45.653 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:21:45.754 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:21:45.823 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:21:45.827 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:21:45.829 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:21:45.931 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:21:45.971 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:21:45.974 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:21:45.975 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:21:46.076 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:21:46.106 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:21:46.108 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:21:46.109 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:21:46.210 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:21:46.243 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:21:46.245 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:21:46.246 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:21:46.334 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:21:46 smithi164 bash[59462]: cluster 1685823704.9698963 mgr.y (mgr.34104) 1374 : cluster 0 pgmap v1386: 89 pgs: 17 active+clean+remapped, 4 active+undersized+degraded, 14 active+undersized, 54 active+clean; 582 KiB data, 4.1 GiB used, 622 GiB / 626 GiB avail; 6/72 objects degraded (8.333%); 6/72 objects misplaced (8.333%) 2023-06-03T20:21:46.347 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:21:46.358 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:21:46 smithi089 bash[75068]: cluster 1685823704.9698963 mgr.y (mgr.34104) 1374 : cluster 0 pgmap v1386: 89 pgs: 17 active+clean+remapped, 4 active+undersized+degraded, 14 active+undersized, 54 active+clean; 582 KiB data, 4.1 GiB used, 622 GiB / 626 GiB avail; 6/72 objects degraded (8.333%); 6/72 objects misplaced (8.333%) 2023-06-03T20:21:46.359 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 20:21:46 smithi089 bash[71163]: ::ffff:172.21.15.164 - - [03/Jun/2023:20:21:46] "GET /metrics HTTP/1.1" 200 228852 "" "Prometheus/2.33.4" 2023-06-03T20:21:46.359 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:21:46 smithi089 bash[78452]: cluster 1685823704.9698963 mgr.y (mgr.34104) 1374 : cluster 0 pgmap v1386: 89 pgs: 17 active+clean+remapped, 4 active+undersized+degraded, 14 active+undersized, 54 active+clean; 582 KiB data, 4.1 GiB used, 622 GiB / 626 GiB avail; 6/72 objects degraded (8.333%); 6/72 objects misplaced (8.333%) 2023-06-03T20:21:46.382 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:21:46.384 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:21:46.385 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:21:46.485 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:21:46.520 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:21:46.523 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:21:46.523 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:21:46.624 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:21:46.661 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:21:46.664 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:21:46.665 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:21:46.766 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:21:46.797 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:21:46.799 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:21:46.799 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:21:46.900 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:21:46.936 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:21:46.939 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:21:46.940 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:21:47.041 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:21:47.069 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:21:47.072 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:21:47.073 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:21:47.174 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:21:47.206 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:21:47.209 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:21:47.210 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:21:47.311 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:21:47.341 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:21:47.344 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:21:47.346 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:21:47.447 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:21:47.483 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:21:47.487 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:21:47.488 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:21:47.589 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:21:47.624 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:21:47.627 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:21:47.628 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:21:47.729 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:21:47.770 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:21:47.772 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:21:47.773 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:21:47.874 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:21:47.910 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:21:47.913 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:21:47.913 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:21:48.014 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:21:48.042 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:21:48.047 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:21:48.048 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:21:48.149 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:21:48.182 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:21:48.186 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:21:48.186 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:21:48.232 INFO:teuthology.orchestra.run.smithi089.stdout:osd.0: {} 2023-06-03T20:21:48.255 INFO:teuthology.orchestra.run.smithi089.stdout:osd.1: {} 2023-06-03T20:21:48.278 INFO:teuthology.orchestra.run.smithi089.stdout:ERROR: (22) Invalid argument 2023-06-03T20:21:48.278 INFO:teuthology.orchestra.run.smithi089.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-06-03T20:21:48.279 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:21:48 smithi089 bash[75068]: cluster 1685823706.9705634 mgr.y (mgr.34104) 1375 : cluster 0 pgmap v1387: 89 pgs: 17 active+clean+remapped, 4 active+undersized+degraded, 14 active+undersized, 54 active+clean; 582 KiB data, 4.1 GiB used, 622 GiB / 626 GiB avail; 6/72 objects degraded (8.333%); 6/72 objects misplaced (8.333%) 2023-06-03T20:21:48.279 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:21:48 smithi089 bash[78452]: cluster 1685823706.9705634 mgr.y (mgr.34104) 1375 : cluster 0 pgmap v1387: 89 pgs: 17 active+clean+remapped, 4 active+undersized+degraded, 14 active+undersized, 54 active+clean; 582 KiB data, 4.1 GiB used, 622 GiB / 626 GiB avail; 6/72 objects degraded (8.333%); 6/72 objects misplaced (8.333%) 2023-06-03T20:21:48.287 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:21:48.294 INFO:teuthology.orchestra.run.smithi089.stdout:osd.2: {} 2023-06-03T20:21:48.298 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:21:48 smithi164 bash[59462]: cluster 1685823706.9705634 mgr.y (mgr.34104) 1375 : cluster 0 pgmap v1387: 89 pgs: 17 active+clean+remapped, 4 active+undersized+degraded, 14 active+undersized, 54 active+clean; 582 KiB data, 4.1 GiB used, 622 GiB / 626 GiB avail; 6/72 objects degraded (8.333%); 6/72 objects misplaced (8.333%) 2023-06-03T20:21:48.319 INFO:teuthology.orchestra.run.smithi089.stdout:osd.3: {} 2023-06-03T20:21:48.321 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:21:48.325 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:21:48.325 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:21:48.344 INFO:teuthology.orchestra.run.smithi089.stdout:osd.5: {} 2023-06-03T20:21:48.364 INFO:teuthology.orchestra.run.smithi089.stdout:osd.6: {} 2023-06-03T20:21:48.391 INFO:teuthology.orchestra.run.smithi089.stdout:osd.7: {} 2023-06-03T20:21:48.392 INFO:teuthology.orchestra.run.smithi089.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-06-03T20:21:48.392 INFO:teuthology.orchestra.run.smithi089.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-06-03T20:21:48.392 INFO:teuthology.orchestra.run.smithi089.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-06-03T20:21:48.392 INFO:teuthology.orchestra.run.smithi089.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-06-03T20:21:48.392 INFO:teuthology.orchestra.run.smithi089.stderr:Error ENXIO: problem getting command descriptions from osd.4 2023-06-03T20:21:48.393 INFO:teuthology.orchestra.run.smithi089.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-06-03T20:21:48.393 INFO:teuthology.orchestra.run.smithi089.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-06-03T20:21:48.393 INFO:teuthology.orchestra.run.smithi089.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-06-03T20:21:48.426 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:21:48.457 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:21:48.460 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:21:48.461 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:21:48.561 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:21:48.599 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:21:48.602 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:21:48.603 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:21:48.704 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:21:48.735 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:21:48.737 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:21:48.737 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:21:48.838 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:21:48.868 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:21:48.871 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:21:48.871 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:21:48.972 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:21:49.002 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:21:49.004 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:21:49.004 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:21:49.049 INFO:teuthology.orchestra.run.smithi089.stderr:enabled application 'rados' on pool 'unique_pool_8' 2023-06-03T20:21:49.050 INFO:teuthology.orchestra.run.smithi089.stderr:noscrub is set 2023-06-03T20:21:49.105 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:21:49.132 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:21:49.134 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:21:49.135 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:21:49.236 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:21:49.270 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:21:49.273 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:21:49.274 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:21:49.279 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:21:49 smithi089 bash[75068]: audit 1685823708.1888092 mon.a (mon.0) 1356 : audit 0 from='client.? 172.21.15.89:0/4106256319' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-03T20:21:49.279 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:21:49 smithi089 bash[75068]: audit 1685823708.3014514 mon.a (mon.0) 1357 : audit 1 from='client.? 172.21.15.89:0/2978035338' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-06-03T20:21:49.279 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:21:49 smithi089 bash[75068]: audit 1685823708.4094698 mon.c (mon.2) 586 : audit 1 from='client.? 172.21.15.89:0/3927925592' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_8", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-06-03T20:21:49.279 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:21:49 smithi089 bash[75068]: audit 1685823708.4100919 mon.a (mon.0) 1358 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_8", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-06-03T20:21:49.280 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:21:49 smithi089 bash[78452]: audit 1685823708.1888092 mon.a (mon.0) 1356 : audit 0 from='client.? 172.21.15.89:0/4106256319' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-03T20:21:49.280 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:21:49 smithi089 bash[78452]: audit 1685823708.3014514 mon.a (mon.0) 1357 : audit 1 from='client.? 172.21.15.89:0/2978035338' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-06-03T20:21:49.280 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:21:49 smithi089 bash[78452]: audit 1685823708.4094698 mon.c (mon.2) 586 : audit 1 from='client.? 172.21.15.89:0/3927925592' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_8", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-06-03T20:21:49.280 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:21:49 smithi089 bash[78452]: audit 1685823708.4100919 mon.a (mon.0) 1358 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_8", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-06-03T20:21:49.334 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:21:49 smithi164 bash[59462]: audit 1685823708.1888092 mon.a (mon.0) 1356 : audit 0 from='client.? 172.21.15.89:0/4106256319' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-03T20:21:49.334 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:21:49 smithi164 bash[59462]: audit 1685823708.3014514 mon.a (mon.0) 1357 : audit 1 from='client.? 172.21.15.89:0/2978035338' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-06-03T20:21:49.334 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:21:49 smithi164 bash[59462]: audit 1685823708.4094698 mon.c (mon.2) 586 : audit 1 from='client.? 172.21.15.89:0/3927925592' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_8", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-06-03T20:21:49.334 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:21:49 smithi164 bash[59462]: audit 1685823708.4100919 mon.a (mon.0) 1358 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_8", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-06-03T20:21:49.375 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:21:49.416 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:21:49.419 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:21:49.419 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:21:49.520 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:21:49.551 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:21:49.553 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:21:49.554 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:21:49.654 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:21:49.690 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:21:49.693 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:21:49.693 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:21:49.794 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:21:49.834 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:21:49.837 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:21:49.838 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:21:49.939 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:21:49.964 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:21:49.966 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:21:49.967 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:21:50.067 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:21:50.081 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:21:50 smithi164 bash[59462]: cluster 1685823708.9712355 mgr.y (mgr.34104) 1376 : cluster 0 pgmap v1388: 89 pgs: 17 active+clean+remapped, 4 active+undersized+degraded, 14 active+undersized, 54 active+clean; 582 KiB data, 4.1 GiB used, 622 GiB / 626 GiB avail; 6/72 objects degraded (8.333%); 6/72 objects misplaced (8.333%) 2023-06-03T20:21:50.081 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:21:50 smithi164 bash[59462]: cluster 1685823709.0292277 mon.a (mon.0) 1359 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-06-03T20:21:50.081 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:21:50 smithi164 bash[59462]: cluster 1685823709.035153 osd.6 (osd.6) 189 : cluster 3 failed to encode map e245 with expected crc 2023-06-03T20:21:50.081 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:21:50 smithi164 bash[59462]: cluster 1685823709.036119 osd.5 (osd.5) 183 : cluster 3 failed to encode map e245 with expected crc 2023-06-03T20:21:50.081 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:21:50 smithi164 bash[59462]: audit 1685823709.0402217 mon.a (mon.0) 1360 : audit 1 from='client.? 172.21.15.89:0/2978035338' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-06-03T20:21:50.082 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:21:50 smithi164 bash[59462]: audit 1685823709.0402923 mon.a (mon.0) 1361 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_8", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-06-03T20:21:50.082 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:21:50 smithi164 bash[59462]: cluster 1685823709.0403605 mon.a (mon.0) 1362 : cluster 0 osdmap e245: 8 total, 7 up, 7 in 2023-06-03T20:21:50.103 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:21:50.105 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:21:50.106 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:21:50.207 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:21:50.247 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:21:50.249 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:21:50.250 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:21:50.279 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:21:50 smithi089 bash[75068]: cluster 1685823708.9712355 mgr.y (mgr.34104) 1376 : cluster 0 pgmap v1388: 89 pgs: 17 active+clean+remapped, 4 active+undersized+degraded, 14 active+undersized, 54 active+clean; 582 KiB data, 4.1 GiB used, 622 GiB / 626 GiB avail; 6/72 objects degraded (8.333%); 6/72 objects misplaced (8.333%) 2023-06-03T20:21:50.279 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:21:50 smithi089 bash[75068]: cluster 1685823709.0292277 mon.a (mon.0) 1359 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-06-03T20:21:50.279 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:21:50 smithi089 bash[75068]: cluster 1685823709.035153 osd.6 (osd.6) 189 : cluster 3 failed to encode map e245 with expected crc 2023-06-03T20:21:50.279 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:21:50 smithi089 bash[75068]: cluster 1685823709.036119 osd.5 (osd.5) 183 : cluster 3 failed to encode map e245 with expected crc 2023-06-03T20:21:50.279 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:21:50 smithi089 bash[75068]: audit 1685823709.0402217 mon.a (mon.0) 1360 : audit 1 from='client.? 172.21.15.89:0/2978035338' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-06-03T20:21:50.280 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:21:50 smithi089 bash[75068]: audit 1685823709.0402923 mon.a (mon.0) 1361 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_8", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-06-03T20:21:50.280 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:21:50 smithi089 bash[75068]: cluster 1685823709.0403605 mon.a (mon.0) 1362 : cluster 0 osdmap e245: 8 total, 7 up, 7 in 2023-06-03T20:21:50.280 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:21:50 smithi089 bash[78452]: cluster 1685823708.9712355 mgr.y (mgr.34104) 1376 : cluster 0 pgmap v1388: 89 pgs: 17 active+clean+remapped, 4 active+undersized+degraded, 14 active+undersized, 54 active+clean; 582 KiB data, 4.1 GiB used, 622 GiB / 626 GiB avail; 6/72 objects degraded (8.333%); 6/72 objects misplaced (8.333%) 2023-06-03T20:21:50.280 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:21:50 smithi089 bash[78452]: cluster 1685823709.0292277 mon.a (mon.0) 1359 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-06-03T20:21:50.280 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:21:50 smithi089 bash[78452]: cluster 1685823709.035153 osd.6 (osd.6) 189 : cluster 3 failed to encode map e245 with expected crc 2023-06-03T20:21:50.280 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:21:50 smithi089 bash[78452]: cluster 1685823709.036119 osd.5 (osd.5) 183 : cluster 3 failed to encode map e245 with expected crc 2023-06-03T20:21:50.281 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:21:50 smithi089 bash[78452]: audit 1685823709.0402217 mon.a (mon.0) 1360 : audit 1 from='client.? 172.21.15.89:0/2978035338' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-06-03T20:21:50.281 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:21:50 smithi089 bash[78452]: audit 1685823709.0402923 mon.a (mon.0) 1361 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_8", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-06-03T20:21:50.281 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:21:50 smithi089 bash[78452]: cluster 1685823709.0403605 mon.a (mon.0) 1362 : cluster 0 osdmap e245: 8 total, 7 up, 7 in 2023-06-03T20:21:50.350 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:21:50.391 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:21:50.395 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:21:50.395 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:21:50.496 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:21:50.539 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:21:50.541 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:21:50.542 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:21:50.643 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:21:50.669 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:21:50.671 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:21:50.671 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:21:50.772 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:21:50.807 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:21:50.811 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:21:50.812 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:21:50.913 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:21:50.947 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:21:50.949 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:21:50.950 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:21:51.050 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:21:51.079 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:21:51.081 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:21:51.082 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:21:51.184 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:21:51.224 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:21:51.227 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:21:51.228 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:21:51.279 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:21:51 smithi089 bash[75068]: cluster 1685823709.711016 osd.7 (osd.7) 179 : cluster 3 failed to encode map e245 with expected crc 2023-06-03T20:21:51.279 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:21:51 smithi089 bash[78452]: cluster 1685823709.711016 osd.7 (osd.7) 179 : cluster 3 failed to encode map e245 with expected crc 2023-06-03T20:21:51.329 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:21:51.333 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:21:51 smithi164 bash[59462]: cluster 1685823709.711016 osd.7 (osd.7) 179 : cluster 3 failed to encode map e245 with expected crc 2023-06-03T20:21:51.349 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph daemon osd.1 dump_ops_in_flight 2023-06-03T20:21:51.365 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:21:51.369 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:21:51.370 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:21:51.471 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:21:51.496 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:21:51.498 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:21:51.499 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:21:51.555 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-06-03T20:21:51.556 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-06-03T20:21:51.600 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:21:51.638 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:21:51.642 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:21:51.642 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:21:51.743 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:21:51.771 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:21:51.773 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:21:51.774 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:21:51.875 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:21:51.906 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:21:51.908 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:21:51.909 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:21:52.010 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:21:52.048 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:21:52.050 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:21:52.050 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:21:52.151 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:21:52.180 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:21:52.183 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:21:52.183 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:21:52.284 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:21:52.316 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:21:52.318 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:21:52 smithi089 bash[78452]: cluster 1685823710.9721057 mgr.y (mgr.34104) 1377 : cluster 0 pgmap v1390: 89 pgs: 17 active+clean+remapped, 4 active+undersized+degraded, 14 active+undersized, 54 active+clean; 582 KiB data, 4.1 GiB used, 622 GiB / 626 GiB avail; 6/72 objects degraded (8.333%); 6/72 objects misplaced (8.333%) 2023-06-03T20:21:52.318 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:21:52 smithi089 bash[75068]: cluster 1685823710.9721057 mgr.y (mgr.34104) 1377 : cluster 0 pgmap v1390: 89 pgs: 17 active+clean+remapped, 4 active+undersized+degraded, 14 active+undersized, 54 active+clean; 582 KiB data, 4.1 GiB used, 622 GiB / 626 GiB avail; 6/72 objects degraded (8.333%); 6/72 objects misplaced (8.333%) 2023-06-03T20:21:52.319 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:21:52.319 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:21:52.334 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:21:52 smithi164 bash[59462]: cluster 1685823710.9721057 mgr.y (mgr.34104) 1377 : cluster 0 pgmap v1390: 89 pgs: 17 active+clean+remapped, 4 active+undersized+degraded, 14 active+undersized, 54 active+clean; 582 KiB data, 4.1 GiB used, 622 GiB / 626 GiB avail; 6/72 objects degraded (8.333%); 6/72 objects misplaced (8.333%) 2023-06-03T20:21:52.420 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:21:52.456 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:21:52.458 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:21:52.459 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:21:52.560 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:21:52.625 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:21:52.628 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:21:52.629 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:21:52.730 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:21:52.768 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:21:52.771 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:21:52.771 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:21:52.872 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:21:52.902 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:21:52.904 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:21:52.905 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:21:53.006 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:21:53.045 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:21:53.048 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:21:53.049 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:21:53.150 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:21:53.192 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:21:53.195 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:21:53.196 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:21:53.297 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:21:53.333 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:21:53.336 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:21:53.337 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:21:53.438 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:21:53.450 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:21:53 smithi164 bash[59462]: audit 1685823712.1410704 mon.a (mon.0) 1363 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:21:53.451 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:21:53 smithi164 bash[59462]: audit 1685823712.142492 mon.c (mon.2) 587 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T20:21:53.476 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:21:53.478 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:21:53.479 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:21:53.529 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:21:53 smithi089 bash[75068]: audit 1685823712.1410704 mon.a (mon.0) 1363 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:21:53.529 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:21:53 smithi089 bash[75068]: audit 1685823712.142492 mon.c (mon.2) 587 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T20:21:53.530 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:21:53 smithi089 bash[78452]: audit 1685823712.1410704 mon.a (mon.0) 1363 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:21:53.530 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:21:53 smithi089 bash[78452]: audit 1685823712.142492 mon.c (mon.2) 587 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T20:21:53.534 DEBUG:teuthology.orchestra.run.smithi089:> /bin/sh -c 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage rados --no-log-to-stderr --name client.0 -b 65536 --object-size 65536 -p unique_pool_8 bench 90 write' 2023-06-03T20:21:53.535 INFO:tasks.thrashosds.thrasher:Growing pool device_health_metrics 2023-06-03T20:21:53.535 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph pg dump --format=json 2023-06-03T20:21:53.538 INFO:tasks.radosbench:joining radosbench (timing out after 3000s) 2023-06-03T20:21:53.539 INFO:teuthology.orchestra.run:waiting for 3000 2023-06-03T20:21:53.558 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-06-03T20:21:53.579 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:21:53.598 INFO:tasks.radosbench.radosbench.0.smithi089.stdout:hints = 1 2023-06-03T20:21:53.598 INFO:tasks.radosbench.radosbench.0.smithi089.stdout:Maintaining 16 concurrent writes of 65536 bytes to objects of size 65536 for up to 90 seconds or 0 objects 2023-06-03T20:21:53.598 INFO:tasks.radosbench.radosbench.0.smithi089.stdout:Object prefix: benchmark_data_smithi089_211581 2023-06-03T20:21:53.599 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-06-03T20:21:53.599 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 0 0 0 0 0 0 - 0 2023-06-03T20:21:53.627 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:21:53.635 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:21:53.636 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:21:53.737 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:21:53.791 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:21:53.793 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:21:53.794 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:21:53.895 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:21:53.929 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:21:53.931 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:21:53.932 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:21:54.033 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:21:54.069 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:21:54.072 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:21:54.073 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:21:54.174 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:21:54.204 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:21:54.207 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:21:54.207 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:21:54.308 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:21:54.318 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:21:54 smithi089 bash[78452]: cluster 1685823712.9732385 mgr.y (mgr.34104) 1378 : cluster 0 pgmap v1391: 89 pgs: 17 active+clean+remapped, 4 active+undersized+degraded, 14 active+undersized, 54 active+clean; 582 KiB data, 4.1 GiB used, 622 GiB / 626 GiB avail; 6/72 objects degraded (8.333%); 6/72 objects misplaced (8.333%) 2023-06-03T20:21:54.320 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:21:54 smithi089 bash[75068]: cluster 1685823712.9732385 mgr.y (mgr.34104) 1378 : cluster 0 pgmap v1391: 89 pgs: 17 active+clean+remapped, 4 active+undersized+degraded, 14 active+undersized, 54 active+clean; 582 KiB data, 4.1 GiB used, 622 GiB / 626 GiB avail; 6/72 objects degraded (8.333%); 6/72 objects misplaced (8.333%) 2023-06-03T20:21:54.334 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:21:54 smithi164 bash[59462]: cluster 1685823712.9732385 mgr.y (mgr.34104) 1378 : cluster 0 pgmap v1391: 89 pgs: 17 active+clean+remapped, 4 active+undersized+degraded, 14 active+undersized, 54 active+clean; 582 KiB data, 4.1 GiB used, 622 GiB / 626 GiB avail; 6/72 objects degraded (8.333%); 6/72 objects misplaced (8.333%) 2023-06-03T20:21:54.341 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:21:54.347 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:21:54.348 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:21:54.449 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:21:54.486 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:21:54.490 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:21:54.491 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:21:54.551 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph osd set nodeep-scrub 2023-06-03T20:21:54.593 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:21:54.600 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 1 16 2800 2784 173.71 174 0.00810827 0.00569961 2023-06-03T20:21:54.641 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:21:54.644 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:21:54.644 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:21:54.745 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:21:54.786 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:21:54.789 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:21:54.789 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:21:54.890 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:21:54.923 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:21:54.926 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:21:54.927 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:21:55.028 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:21:55.061 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:21:55.066 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:21:55.067 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:21:55.168 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:21:55.199 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:21:55.203 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:21:55.204 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:21:55.305 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:21:55.334 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:21:55.336 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:21:55.337 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:21:55.437 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:21:55.456 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:21:55 smithi164 bash[59462]: cluster 1685823715.13987 mon.a (mon.0) 1364 : cluster 3 Health check update: Degraded data redundancy: 168/819 objects degraded (20.513%), 6 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-06-03T20:21:55.473 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:21:55.476 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:21:55.476 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:21:55.529 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:21:55 smithi089 bash[75068]: cluster 1685823715.13987 mon.a (mon.0) 1364 : cluster 3 Health check update: Degraded data redundancy: 168/819 objects degraded (20.513%), 6 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-06-03T20:21:55.529 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:21:55 smithi089 bash[78452]: cluster 1685823715.13987 mon.a (mon.0) 1364 : cluster 3 Health check update: Degraded data redundancy: 168/819 objects degraded (20.513%), 6 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-06-03T20:21:55.578 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:21:55.600 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 2 16 5725 5709 178.25 182.812 0.00280094 0.00559856 2023-06-03T20:21:55.612 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:21:55.615 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:21:55.616 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:21:55.717 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:21:55.748 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:21:55.750 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:21:55.751 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:21:55.852 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:21:55.886 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:21:55.888 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:21:55.889 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:21:55.989 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:21:56.023 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:21:56.026 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:21:56.026 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:21:56.127 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:21:56.156 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/mon.a/config 2023-06-03T20:21:56.158 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:21:56.166 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:21:56.166 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:21:56.267 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:21:56.300 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:21:56.304 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:21:56.305 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:21:56.334 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:21:56 smithi164 bash[59462]: cluster 1685823714.9741013 mgr.y (mgr.34104) 1379 : cluster 0 pgmap v1392: 89 pgs: 17 active+clean+remapped, 6 active+undersized+degraded, 12 active+undersized, 54 active+clean; 16 MiB data, 4.1 GiB used, 622 GiB / 626 GiB avail; 1.6 MiB/s wr, 24 op/s; 168/819 objects degraded (20.513%); 6/819 objects misplaced (0.733%) 2023-06-03T20:21:56.406 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:21:56.415 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:21:56 smithi089 bash[75068]: cluster 1685823714.9741013 mgr.y (mgr.34104) 1379 : cluster 0 pgmap v1392: 89 pgs: 17 active+clean+remapped, 6 active+undersized+degraded, 12 active+undersized, 54 active+clean; 16 MiB data, 4.1 GiB used, 622 GiB / 626 GiB avail; 1.6 MiB/s wr, 24 op/s; 168/819 objects degraded (20.513%); 6/819 objects misplaced (0.733%) 2023-06-03T20:21:56.416 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 20:21:56 smithi089 bash[71163]: ::ffff:172.21.15.164 - - [03/Jun/2023:20:21:56] "GET /metrics HTTP/1.1" 200 228817 "" "Prometheus/2.33.4" 2023-06-03T20:21:56.416 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:21:56 smithi089 bash[78452]: cluster 1685823714.9741013 mgr.y (mgr.34104) 1379 : cluster 0 pgmap v1392: 89 pgs: 17 active+clean+remapped, 6 active+undersized+degraded, 12 active+undersized, 54 active+clean; 16 MiB data, 4.1 GiB used, 622 GiB / 626 GiB avail; 1.6 MiB/s wr, 24 op/s; 168/819 objects degraded (20.513%); 6/819 objects misplaced (0.733%) 2023-06-03T20:21:56.448 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:21:56.451 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:21:56.452 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:21:56.553 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:21:56.587 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:21:56.592 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:21:56.592 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:21:56.600 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 3 16 8472 8456 176.058 171.688 0.0064851 0.00566885 2023-06-03T20:21:56.694 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:21:56.728 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:21:56.732 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:21:56.733 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:21:56.833 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:21:56.870 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:21:56.872 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:21:56.873 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:21:56.974 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:21:57.012 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:21:57.015 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:21:57.015 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:21:57.117 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:21:57.149 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:21:57.151 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:21:57.151 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:21:57.252 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:21:57.288 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:21:57.290 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:21:57.291 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:21:57.392 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:21:57.425 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:21:57.428 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:21:57.429 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:21:57.529 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:21:57.566 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:21:57.569 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:21:57.570 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:21:57.600 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 4 16 11013 10997 171.744 158.812 0.00500553 0.00581503 2023-06-03T20:21:57.670 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:21:57.700 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:21:57.702 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:21:57.703 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:21:57.804 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:21:57.835 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:21:57.838 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:21:57.839 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:21:57.940 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:21:57.971 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:21:57.974 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:21:57.974 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:21:58.075 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:21:58.107 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:21:58.112 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:21:58.113 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:21:58.208 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:21:58 smithi089 bash[75068]: cluster 1685823716.9749887 mgr.y (mgr.34104) 1380 : cluster 0 pgmap v1393: 89 pgs: 17 active+clean+remapped, 8 active+undersized+degraded, 10 active+undersized, 54 active+clean; 113 MiB data, 4.3 GiB used, 621 GiB / 626 GiB avail; 11 MiB/s wr, 179 op/s; 938/5460 objects degraded (17.179%); 6/5460 objects misplaced (0.110%) 2023-06-03T20:21:58.214 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:21:58.226 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/mon.a/config 2023-06-03T20:21:58.230 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/mon.a/config 2023-06-03T20:21:58.246 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:21:58.249 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:21:58.250 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:21:58.351 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:21:58.388 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:21:58.390 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:21:58.391 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:21:58.492 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:21:58.504 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:21:58 smithi089 bash[78452]: cluster 1685823716.9749887 mgr.y (mgr.34104) 1380 : cluster 0 pgmap v1393: 89 pgs: 17 active+clean+remapped, 8 active+undersized+degraded, 10 active+undersized, 54 active+clean; 113 MiB data, 4.3 GiB used, 621 GiB / 626 GiB avail; 11 MiB/s wr, 179 op/s; 938/5460 objects degraded (17.179%); 6/5460 objects misplaced (0.110%) 2023-06-03T20:21:58.541 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:21:58.544 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:21:58.545 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:21:58.584 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:21:58 smithi164 bash[59462]: cluster 1685823716.9749887 mgr.y (mgr.34104) 1380 : cluster 0 pgmap v1393: 89 pgs: 17 active+clean+remapped, 8 active+undersized+degraded, 10 active+undersized, 54 active+clean; 113 MiB data, 4.3 GiB used, 621 GiB / 626 GiB avail; 11 MiB/s wr, 179 op/s; 938/5460 objects degraded (17.179%); 6/5460 objects misplaced (0.110%) 2023-06-03T20:21:58.601 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 5 16 13518 13502 168.689 156.562 0.00353207 0.00591994 2023-06-03T20:21:58.646 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:21:58.684 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:21:58.690 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:21:58.691 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:21:58.791 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:21:58.824 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:21:58.828 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:21:58.829 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:21:58.931 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:21:58.976 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:21:58.981 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:21:58.981 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:21:59.082 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:21:59.115 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:21:59.122 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:21:59.123 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:21:59.223 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:21:59.254 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/mon.a/config 2023-06-03T20:21:59.258 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:21:59.260 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:21:59.261 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:21:59.362 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:21:59.396 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:21:59.402 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:21:59.403 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:21:59.504 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:21:59.513 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:21:59 smithi089 bash[75068]: audit 1685823718.263515 mon.c (mon.2) 588 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-03T20:21:59.513 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:21:59 smithi089 bash[78452]: audit 1685823718.263515 mon.c (mon.2) 588 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-03T20:21:59.542 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:21:59.545 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:21:59.546 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:21:59.584 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:21:59 smithi164 bash[59462]: audit 1685823718.263515 mon.c (mon.2) 588 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-03T20:21:59.601 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 6 16 15843 15827 164.791 145.312 0.00541058 0.00606106 2023-06-03T20:21:59.648 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:21:59.684 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:21:59.689 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:21:59.690 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:21:59.791 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:21:59.824 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:21:59.827 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:21:59.827 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:21:59.928 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:21:59.962 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:21:59.964 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:21:59.965 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:22:00.066 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:22:00.099 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:22:00.103 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:22:00.104 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:22:00.205 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:22:00.233 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:22:00.235 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:22:00.236 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:22:00.334 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:22:00 smithi164 bash[59462]: cluster 1685823718.9759352 mgr.y (mgr.34104) 1381 : cluster 0 pgmap v1394: 89 pgs: 17 active+clean+remapped, 13 active+undersized+degraded, 5 active+undersized, 54 active+clean; 522 MiB data, 5.5 GiB used, 620 GiB / 626 GiB avail; 52 MiB/s wr, 833 op/s; 4549/25092 objects degraded (18.129%); 6/25092 objects misplaced (0.024%) 2023-06-03T20:22:00.337 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:22:00.373 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:22:00.377 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:22:00.377 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:22:00.478 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:22:00.487 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:22:00 smithi089 bash[78452]: cluster 1685823718.9759352 mgr.y (mgr.34104) 1381 : cluster 0 pgmap v1394: 89 pgs: 17 active+clean+remapped, 13 active+undersized+degraded, 5 active+undersized, 54 active+clean; 522 MiB data, 5.5 GiB used, 620 GiB / 626 GiB avail; 52 MiB/s wr, 833 op/s; 4549/25092 objects degraded (18.129%); 6/25092 objects misplaced (0.024%) 2023-06-03T20:22:00.487 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:22:00 smithi089 bash[75068]: cluster 1685823718.9759352 mgr.y (mgr.34104) 1381 : cluster 0 pgmap v1394: 89 pgs: 17 active+clean+remapped, 13 active+undersized+degraded, 5 active+undersized, 54 active+clean; 522 MiB data, 5.5 GiB used, 620 GiB / 626 GiB avail; 52 MiB/s wr, 833 op/s; 4549/25092 objects degraded (18.129%); 6/25092 objects misplaced (0.024%) 2023-06-03T20:22:00.507 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:22:00.510 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:22:00.510 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:22:00.602 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 7 16 18070 18054 161.133 139.188 0.00781697 0.00619996 2023-06-03T20:22:00.611 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:22:00.636 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:22:00.638 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:22:00.638 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:22:00.739 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:22:00.784 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:22:00.790 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:22:00.791 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:22:00.891 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:22:00.924 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:22:00.927 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:22:00.928 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:22:01.029 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:22:01.070 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:22:01.074 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:22:01.075 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:22:01.176 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:22:01.210 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:22:01.213 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:22:01.214 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:22:01.315 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:22:01.324 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:22:01 smithi089 bash[78452]: cluster 1685823720.976736 mgr.y (mgr.34104) 1382 : cluster 0 pgmap v1395: 89 pgs: 17 active+clean+remapped, 13 active+undersized+degraded, 5 active+undersized, 54 active+clean; 868 MiB data, 6.1 GiB used, 620 GiB / 626 GiB avail; 73 MiB/s wr, 1.16k op/s; 7710/41703 objects degraded (18.488%); 6/41703 objects misplaced (0.014%) 2023-06-03T20:22:01.324 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:22:01 smithi089 bash[78452]: cluster 1685823721.2120323 mon.a (mon.0) 1365 : cluster 3 Health check update: Degraded data redundancy: 7710/41703 objects degraded (18.488%), 13 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-06-03T20:22:01.324 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:22:01 smithi089 bash[75068]: cluster 1685823720.976736 mgr.y (mgr.34104) 1382 : cluster 0 pgmap v1395: 89 pgs: 17 active+clean+remapped, 13 active+undersized+degraded, 5 active+undersized, 54 active+clean; 868 MiB data, 6.1 GiB used, 620 GiB / 626 GiB avail; 73 MiB/s wr, 1.16k op/s; 7710/41703 objects degraded (18.488%); 6/41703 objects misplaced (0.014%) 2023-06-03T20:22:01.324 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:22:01 smithi089 bash[75068]: cluster 1685823721.2120323 mon.a (mon.0) 1365 : cluster 3 Health check update: Degraded data redundancy: 7710/41703 objects degraded (18.488%), 13 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-06-03T20:22:01.343 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:22:01.346 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:22:01.346 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:22:01.447 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:22:01.470 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:22:01.471 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:22:01.472 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:22:01.573 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:22:01.584 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:22:01 smithi164 bash[59462]: cluster 1685823720.976736 mgr.y (mgr.34104) 1382 : cluster 0 pgmap v1395: 89 pgs: 17 active+clean+remapped, 13 active+undersized+degraded, 5 active+undersized, 54 active+clean; 868 MiB data, 6.1 GiB used, 620 GiB / 626 GiB avail; 73 MiB/s wr, 1.16k op/s; 7710/41703 objects degraded (18.488%); 6/41703 objects misplaced (0.014%) 2023-06-03T20:22:01.584 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:22:01 smithi164 bash[59462]: cluster 1685823721.2120323 mon.a (mon.0) 1365 : cluster 3 Health check update: Degraded data redundancy: 7710/41703 objects degraded (18.488%), 13 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-06-03T20:22:01.598 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:22:01.599 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:22:01.600 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:22:01.601 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 8 16 19709 19693 153.796 102.438 0.00215534 0.00626092 2023-06-03T20:22:01.701 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:22:01.734 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:22:01.737 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:22:01.737 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:22:01.839 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:22:01.871 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:22:01.876 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:22:01.876 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:22:01.977 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:22:02.009 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:22:02.011 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:22:02.012 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:22:02.113 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:22:02.153 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:22:02.155 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:22:02.156 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:22:02.257 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:22:02.287 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:22:02.288 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:22:02.289 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:22:02.390 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:22:02.443 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:22:02.446 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:22:02.447 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:22:02.548 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:22:02.584 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:22:02.586 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:22:02.587 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:22:02.601 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 9 16 21741 21725 150.819 127 0.00297041 0.00662302 2023-06-03T20:22:02.688 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:22:02.719 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:22:02.721 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:22:02.721 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:22:02.822 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:22:02.855 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:22:02.858 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:22:02.859 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:22:02.960 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:22:02.994 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:22:02.997 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:22:02.997 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:22:03.098 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:22:03.133 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:22:03.136 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:22:03.137 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:22:03.238 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:22:03.318 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:22:03.323 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:22:03.323 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:22:03.424 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:22:03.465 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:22:03.466 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:22:03.467 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:22:03.568 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:22:03.605 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 10 16 23542 23526 146.969 112.562 0.00606101 0.0067977 2023-06-03T20:22:03.608 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:22:03.615 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:22:03.616 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:22:03.717 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:22:03.757 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:22:03.761 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:22:03.762 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:22:03.863 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:22:03.912 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:22:03.915 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:22:03.916 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:22:04.016 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:22:04.045 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:22:04.049 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:22:04.050 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:22:04.084 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:22:03 smithi164 bash[59462]: cluster 1685823722.977463 mgr.y (mgr.34104) 1383 : cluster 0 pgmap v1396: 89 pgs: 17 active+clean+remapped, 13 active+undersized+degraded, 5 active+undersized, 54 active+clean; 1.1 GiB data, 6.5 GiB used, 619 GiB / 626 GiB avail; 90 MiB/s wr, 1.44k op/s; 9771/51978 objects degraded (18.798%); 6/51978 objects misplaced (0.012%) 2023-06-03T20:22:04.084 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:22:03 smithi164 bash[59462]: audit 1685823723.7773478 mon.a (mon.0) 1366 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:22:04.084 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:22:03 smithi164 bash[59462]: audit 1685823723.7921066 mon.a (mon.0) 1367 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:22:04.151 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:22:04.181 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:22:04.186 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:22:04.186 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:22:04.279 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:22:03 smithi089 bash[78452]: cluster 1685823722.977463 mgr.y (mgr.34104) 1383 : cluster 0 pgmap v1396: 89 pgs: 17 active+clean+remapped, 13 active+undersized+degraded, 5 active+undersized, 54 active+clean; 1.1 GiB data, 6.5 GiB used, 619 GiB / 626 GiB avail; 90 MiB/s wr, 1.44k op/s; 9771/51978 objects degraded (18.798%); 6/51978 objects misplaced (0.012%) 2023-06-03T20:22:04.279 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:22:03 smithi089 bash[78452]: audit 1685823723.7773478 mon.a (mon.0) 1366 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:22:04.279 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:22:03 smithi089 bash[78452]: audit 1685823723.7921066 mon.a (mon.0) 1367 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:22:04.279 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:22:03 smithi089 bash[75068]: cluster 1685823722.977463 mgr.y (mgr.34104) 1383 : cluster 0 pgmap v1396: 89 pgs: 17 active+clean+remapped, 13 active+undersized+degraded, 5 active+undersized, 54 active+clean; 1.1 GiB data, 6.5 GiB used, 619 GiB / 626 GiB avail; 90 MiB/s wr, 1.44k op/s; 9771/51978 objects degraded (18.798%); 6/51978 objects misplaced (0.012%) 2023-06-03T20:22:04.280 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:22:03 smithi089 bash[75068]: audit 1685823723.7773478 mon.a (mon.0) 1366 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:22:04.280 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:22:03 smithi089 bash[75068]: audit 1685823723.7921066 mon.a (mon.0) 1367 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:22:04.287 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:22:04.328 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:22:04.334 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:22:04.334 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:22:04.435 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:22:04.470 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:22:04.478 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:22:04.480 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:22:04.581 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:22:04.603 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 11 16 25373 25357 144.011 114.438 0.00788269 0.00693796 2023-06-03T20:22:04.616 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:22:04.618 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:22:04.619 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:22:04.720 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:22:04.751 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:22:04.752 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:22:04.753 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:22:04.853 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:22:04.877 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:22:04.880 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:22:04.881 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:22:04.982 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:22:05.026 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:22:05.029 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:22:05.030 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:22:05.131 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:22:05.165 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:22:05.170 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:22:05.170 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:22:05.271 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:22:05.301 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:22:05.303 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:22:05.303 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:22:05.404 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:22:05.435 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:22:05.438 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:22:05.438 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:22:05.539 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:22:05.570 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:22:05.571 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:22:05.572 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:22:05.603 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 12 16 27024 27008 140.61 103.188 0.00424812 0.00710452 2023-06-03T20:22:05.673 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:22:05.707 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:22:05.712 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:22:05.713 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:22:05.814 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:22:05.847 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:22:05.851 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:22:05.852 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:22:05.953 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:22:05.983 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:22:05.985 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:22:05.986 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:22:06.087 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:22:06.120 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:22:06.125 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:22:06.125 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:22:06.226 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:22:06.234 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:22:05 smithi089 bash[75068]: audit 1685823724.893396 mon.a (mon.0) 1368 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:22:06.234 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:22:05 smithi089 bash[75068]: audit 1685823724.90249 mon.a (mon.0) 1369 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:22:06.235 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:22:05 smithi089 bash[75068]: cluster 1685823724.9782665 mgr.y (mgr.34104) 1384 : cluster 0 pgmap v1397: 89 pgs: 17 active+clean+remapped, 13 active+undersized+degraded, 5 active+undersized, 54 active+clean; 1.3 GiB data, 7.3 GiB used, 618 GiB / 626 GiB avail; 112 MiB/s wr, 1.79k op/s; 12215/64734 objects degraded (18.870%); 6/64734 objects misplaced (0.009%) 2023-06-03T20:22:06.235 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:22:05 smithi089 bash[75068]: audit 1685823725.5308847 mon.c (mon.2) 589 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-03T20:22:06.235 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:22:05 smithi089 bash[75068]: audit 1685823725.5323684 mon.c (mon.2) 590 : audit 1 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-03T20:22:06.235 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:22:05 smithi089 bash[75068]: audit 1685823725.54798 mon.a (mon.0) 1370 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:22:06.235 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 20:22:06 smithi089 bash[71163]: ::ffff:172.21.15.164 - - [03/Jun/2023:20:22:06] "GET /metrics HTTP/1.1" 200 228817 "" "Prometheus/2.33.4" 2023-06-03T20:22:06.236 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:22:05 smithi089 bash[78452]: audit 1685823724.893396 mon.a (mon.0) 1368 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:22:06.236 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:22:05 smithi089 bash[78452]: audit 1685823724.90249 mon.a (mon.0) 1369 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:22:06.236 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:22:05 smithi089 bash[78452]: cluster 1685823724.9782665 mgr.y (mgr.34104) 1384 : cluster 0 pgmap v1397: 89 pgs: 17 active+clean+remapped, 13 active+undersized+degraded, 5 active+undersized, 54 active+clean; 1.3 GiB data, 7.3 GiB used, 618 GiB / 626 GiB avail; 112 MiB/s wr, 1.79k op/s; 12215/64734 objects degraded (18.870%); 6/64734 objects misplaced (0.009%) 2023-06-03T20:22:06.236 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:22:05 smithi089 bash[78452]: audit 1685823725.5308847 mon.c (mon.2) 589 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-03T20:22:06.236 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:22:05 smithi089 bash[78452]: audit 1685823725.5323684 mon.c (mon.2) 590 : audit 1 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-03T20:22:06.236 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:22:05 smithi089 bash[78452]: audit 1685823725.54798 mon.a (mon.0) 1370 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:22:06.259 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:22:06.262 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:22:06.263 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:22:06.334 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:22:05 smithi164 bash[59462]: audit 1685823724.893396 mon.a (mon.0) 1368 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:22:06.334 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:22:05 smithi164 bash[59462]: audit 1685823724.90249 mon.a (mon.0) 1369 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:22:06.334 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:22:05 smithi164 bash[59462]: cluster 1685823724.9782665 mgr.y (mgr.34104) 1384 : cluster 0 pgmap v1397: 89 pgs: 17 active+clean+remapped, 13 active+undersized+degraded, 5 active+undersized, 54 active+clean; 1.3 GiB data, 7.3 GiB used, 618 GiB / 626 GiB avail; 112 MiB/s wr, 1.79k op/s; 12215/64734 objects degraded (18.870%); 6/64734 objects misplaced (0.009%) 2023-06-03T20:22:06.335 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:22:05 smithi164 bash[59462]: audit 1685823725.5308847 mon.c (mon.2) 589 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-03T20:22:06.335 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:22:05 smithi164 bash[59462]: audit 1685823725.5323684 mon.c (mon.2) 590 : audit 1 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-03T20:22:06.335 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:22:05 smithi164 bash[59462]: audit 1685823725.54798 mon.a (mon.0) 1370 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:22:06.363 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:22:06.397 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:22:06.401 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:22:06.401 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:22:06.502 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:22:06.537 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:22:06.540 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:22:06.541 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:22:06.603 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 13 16 28974 28958 139.168 121.875 0.00355896 0.00717982 2023-06-03T20:22:06.642 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:22:06.677 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:22:06.680 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:22:06.680 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:22:06.781 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:22:06.813 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:22:06.816 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:22:06.817 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:22:06.917 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:22:06.951 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:22:06.957 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:22:06.957 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:22:07.058 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:22:07.089 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:22:07.091 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:22:07.092 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:22:07.193 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:22:07.225 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:22:07.228 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:22:07.229 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:22:07.330 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:22:07.333 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:22:07 smithi164 bash[59462]: cluster 1685823727.0308957 mon.a (mon.0) 1371 : cluster 3 Health check update: Degraded data redundancy: 13331/71448 objects degraded (18.658%), 13 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-06-03T20:22:07.339 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:22:07 smithi089 bash[75068]: cluster 1685823727.0308957 mon.a (mon.0) 1371 : cluster 3 Health check update: Degraded data redundancy: 13331/71448 objects degraded (18.658%), 13 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-06-03T20:22:07.339 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:22:07 smithi089 bash[78452]: cluster 1685823727.0308957 mon.a (mon.0) 1371 : cluster 3 Health check update: Degraded data redundancy: 13331/71448 objects degraded (18.658%), 13 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-06-03T20:22:07.358 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:22:07.364 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:22:07.365 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:22:07.465 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:22:07.502 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:22:07.507 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:22:07.508 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:22:07.604 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 14 16 31001 30985 138.276 126.688 0.00929119 0.00722492 2023-06-03T20:22:07.608 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:22:07.640 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:22:07.644 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:22:07.645 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:22:07.745 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:22:07.769 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:22:07.771 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:22:07.772 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:22:07.873 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:22:07.905 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:22:07.909 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:22:07.910 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:22:08.011 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:22:08.042 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:22:08.044 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:22:08.045 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:22:08.146 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:22:08.178 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:22:08.181 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:22:08.182 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:22:08.279 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:22:08 smithi089 bash[75068]: cluster 1685823726.9789236 mgr.y (mgr.34104) 1385 : cluster 0 pgmap v1398: 89 pgs: 17 active+clean+remapped, 13 active+undersized+degraded, 5 active+undersized, 54 active+clean; 1.5 GiB data, 7.6 GiB used, 618 GiB / 626 GiB avail; 123 MiB/s wr, 1.96k op/s; 13331/71448 objects degraded (18.658%); 6/71448 objects misplaced (0.008%) 2023-06-03T20:22:08.279 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:22:08 smithi089 bash[75068]: audit 1685823727.134871 mon.c (mon.2) 591 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T20:22:08.280 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:22:08 smithi089 bash[78452]: cluster 1685823726.9789236 mgr.y (mgr.34104) 1385 : cluster 0 pgmap v1398: 89 pgs: 17 active+clean+remapped, 13 active+undersized+degraded, 5 active+undersized, 54 active+clean; 1.5 GiB data, 7.6 GiB used, 618 GiB / 626 GiB avail; 123 MiB/s wr, 1.96k op/s; 13331/71448 objects degraded (18.658%); 6/71448 objects misplaced (0.008%) 2023-06-03T20:22:08.280 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:22:08 smithi089 bash[78452]: audit 1685823727.134871 mon.c (mon.2) 591 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T20:22:08.283 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:22:08.318 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:22:08.321 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:22:08.322 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:22:08.423 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:22:08.458 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:22:08.461 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:22:08.462 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:22:08.562 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:22:08.570 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:22:08 smithi164 bash[59462]: cluster 1685823726.9789236 mgr.y (mgr.34104) 1385 : cluster 0 pgmap v1398: 89 pgs: 17 active+clean+remapped, 13 active+undersized+degraded, 5 active+undersized, 54 active+clean; 1.5 GiB data, 7.6 GiB used, 618 GiB / 626 GiB avail; 123 MiB/s wr, 1.96k op/s; 13331/71448 objects degraded (18.658%); 6/71448 objects misplaced (0.008%) 2023-06-03T20:22:08.571 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:22:08 smithi164 bash[59462]: audit 1685823727.134871 mon.c (mon.2) 591 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T20:22:08.592 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:22:08.594 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:22:08.594 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:22:08.604 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 15 16 32893 32877 136.941 118.25 0.0107239 0.0072955 2023-06-03T20:22:08.695 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:22:08.726 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:22:08.731 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:22:08.731 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:22:08.832 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:22:08.863 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:22:08.867 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:22:08.867 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:22:08.968 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:22:08.999 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:22:09.001 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:22:09.002 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:22:09.103 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:22:09.134 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:22:09.137 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:22:09.138 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:22:09.239 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:22:09.266 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:22:09.268 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:22:09.269 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:22:09.370 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:22:09.403 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:22:09.407 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:22:09.408 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:22:09.509 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:22:09.539 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:22:09.543 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:22:09.543 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:22:09.604 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 16 16 34685 34669 135.382 112 0.00881912 0.007381 2023-06-03T20:22:09.644 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:22:09.678 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:22:09.681 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:22:09.682 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:22:09.783 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:22:09.811 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:22:09.816 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:22:09.817 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:22:09.918 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:22:09.949 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:22:09.951 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:22:09.951 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:22:10.052 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:22:10.100 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:22:10.103 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:22:10.104 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:22:10.205 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:22:10.236 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:22:10.239 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:22:10.240 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:22:10.339 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:22:10 smithi089 bash[75068]: cluster 1685823728.979862 mgr.y (mgr.34104) 1386 : cluster 0 pgmap v1399: 89 pgs: 17 active+clean+remapped, 13 active+undersized+degraded, 5 active+undersized, 54 active+clean; 1.8 GiB data, 8.6 GiB used, 617 GiB / 626 GiB avail; 142 MiB/s wr, 2.27k op/s; 16170/87237 objects degraded (18.536%); 6/87237 objects misplaced (0.007%) 2023-06-03T20:22:10.340 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:22:10 smithi089 bash[78452]: cluster 1685823728.979862 mgr.y (mgr.34104) 1386 : cluster 0 pgmap v1399: 89 pgs: 17 active+clean+remapped, 13 active+undersized+degraded, 5 active+undersized, 54 active+clean; 1.8 GiB data, 8.6 GiB used, 617 GiB / 626 GiB avail; 142 MiB/s wr, 2.27k op/s; 16170/87237 objects degraded (18.536%); 6/87237 objects misplaced (0.007%) 2023-06-03T20:22:10.341 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:22:10.373 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:22:10.376 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:22:10.377 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:22:10.478 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:22:10.508 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:22:10.511 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:22:10.512 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:22:10.584 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:22:10 smithi164 bash[59462]: cluster 1685823728.979862 mgr.y (mgr.34104) 1386 : cluster 0 pgmap v1399: 89 pgs: 17 active+clean+remapped, 13 active+undersized+degraded, 5 active+undersized, 54 active+clean; 1.8 GiB data, 8.6 GiB used, 617 GiB / 626 GiB avail; 142 MiB/s wr, 2.27k op/s; 16170/87237 objects degraded (18.536%); 6/87237 objects misplaced (0.007%) 2023-06-03T20:22:10.604 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 17 16 36431 36415 133.837 109.125 0.007427 0.00746466 2023-06-03T20:22:10.613 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:22:10.649 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:22:10.652 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:22:10.653 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:22:10.754 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:22:10.795 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:22:10.797 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:22:10.798 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:22:10.899 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:22:10.937 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:22:10.940 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:22:10.941 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:22:11.042 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:22:11.075 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:22:11.077 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:22:11.078 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:22:11.179 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:22:11.213 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:22:11.215 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:22:11.216 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:22:11.318 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:22:11.354 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:22:11.359 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:22:11.360 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:22:11.461 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:22:11.491 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:22:11.493 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:22:11.494 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:22:11.595 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:22:11.604 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 18 16 38054 38038 132.037 101.438 0.00241656 0.00755796 2023-06-03T20:22:11.641 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:22:11.643 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:22:11.644 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:22:11.745 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:22:11.779 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:22:11.782 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:22:11.783 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:22:11.884 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:22:11.926 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:22:11.930 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:22:11.931 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:22:12.032 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:22:12.064 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:22:12.066 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:22:12.067 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:22:12.167 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:22:12.199 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:22:12.203 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:22:12.204 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:22:12.285 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:22:12 smithi089 bash[75068]: cluster 1685823730.9806128 mgr.y (mgr.34104) 1387 : cluster 0 pgmap v1400: 89 pgs: 17 active+clean+remapped, 13 active+undersized+degraded, 5 active+undersized, 54 active+clean; 2.0 GiB data, 9.0 GiB used, 617 GiB / 626 GiB avail; 129 MiB/s wr, 2.06k op/s; 18495/99393 objects degraded (18.608%); 6/99393 objects misplaced (0.006%) 2023-06-03T20:22:12.285 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:22:12 smithi089 bash[78452]: cluster 1685823730.9806128 mgr.y (mgr.34104) 1387 : cluster 0 pgmap v1400: 89 pgs: 17 active+clean+remapped, 13 active+undersized+degraded, 5 active+undersized, 54 active+clean; 2.0 GiB data, 9.0 GiB used, 617 GiB / 626 GiB avail; 129 MiB/s wr, 2.06k op/s; 18495/99393 objects degraded (18.608%); 6/99393 objects misplaced (0.006%) 2023-06-03T20:22:12.304 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:22:12.313 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:22:12 smithi164 bash[59462]: cluster 1685823730.9806128 mgr.y (mgr.34104) 1387 : cluster 0 pgmap v1400: 89 pgs: 17 active+clean+remapped, 13 active+undersized+degraded, 5 active+undersized, 54 active+clean; 2.0 GiB data, 9.0 GiB used, 617 GiB / 626 GiB avail; 129 MiB/s wr, 2.06k op/s; 18495/99393 objects degraded (18.608%); 6/99393 objects misplaced (0.006%) 2023-06-03T20:22:12.334 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:22:12.338 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:22:12.339 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:22:12.440 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:22:12.473 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:22:12.475 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:22:12.476 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:22:12.576 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:22:12.600 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:22:12.604 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 19 16 39438 39422 129.64 86.5 0.0609548 0.00769051 2023-06-03T20:22:12.605 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:22:12.606 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:22:12.706 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:22:12.738 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:22:12.745 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:22:12.746 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:22:12.846 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:22:12.884 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:22:12.887 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:22:12.888 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:22:12.988 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:22:13.027 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:22:13.029 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:22:13.030 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:22:13.131 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:22:13.164 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:22:13.168 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:22:13.168 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:22:13.269 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:22:13.279 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:22:13 smithi164 bash[59462]: cluster 1685823732.897842 mon.a (mon.0) 1372 : cluster 3 Health check update: Degraded data redundancy: 18495/99393 objects degraded (18.608%), 13 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-06-03T20:22:13.304 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:22:13.306 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:22:13.307 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:22:13.359 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:22:13 smithi089 bash[75068]: cluster 1685823732.897842 mon.a (mon.0) 1372 : cluster 3 Health check update: Degraded data redundancy: 18495/99393 objects degraded (18.608%), 13 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-06-03T20:22:13.360 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:22:13 smithi089 bash[78452]: cluster 1685823732.897842 mon.a (mon.0) 1372 : cluster 3 Health check update: Degraded data redundancy: 18495/99393 objects degraded (18.608%), 13 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-06-03T20:22:13.408 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:22:13.443 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:22:13.446 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:22:13.447 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:22:13.547 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:22:13.575 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:22:13.578 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:22:13.579 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:22:13.604 INFO:tasks.radosbench.radosbench.0.smithi089.stdout:2023-06-03T20:22:13.600276+0000 min lat: 0.0012167 max lat: 0.341691 avg lat: 0.00777718 2023-06-03T20:22:13.604 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-06-03T20:22:13.605 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 20 16 41035 41019 128.149 99.8125 0.00585749 0.00777718 2023-06-03T20:22:13.680 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:22:13.728 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:22:13.730 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:22:13.731 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:22:13.832 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:22:13.864 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:22:13.867 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:22:13.869 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:22:13.970 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:22:14.000 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:22:14.003 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:22:14.003 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:22:14.104 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:22:14.135 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:22:14.139 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:22:14.140 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:22:14.242 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:22:14.250 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:22:14 smithi089 bash[78452]: cluster 1685823732.981272 mgr.y (mgr.34104) 1388 : cluster 0 pgmap v1401: 89 pgs: 17 active+clean+remapped, 13 active+undersized+degraded, 5 active+undersized, 54 active+clean; 2.2 GiB data, 9.3 GiB used, 616 GiB / 626 GiB avail; 115 MiB/s wr, 1.83k op/s; 20168/107802 objects degraded (18.708%); 6/107802 objects misplaced (0.006%) 2023-06-03T20:22:14.251 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:22:14 smithi089 bash[75068]: cluster 1685823732.981272 mgr.y (mgr.34104) 1388 : cluster 0 pgmap v1401: 89 pgs: 17 active+clean+remapped, 13 active+undersized+degraded, 5 active+undersized, 54 active+clean; 2.2 GiB data, 9.3 GiB used, 616 GiB / 626 GiB avail; 115 MiB/s wr, 1.83k op/s; 20168/107802 objects degraded (18.708%); 6/107802 objects misplaced (0.006%) 2023-06-03T20:22:14.271 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:22:14.273 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:22:14.274 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:22:14.375 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:22:14.386 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:22:14 smithi164 bash[59462]: cluster 1685823732.981272 mgr.y (mgr.34104) 1388 : cluster 0 pgmap v1401: 89 pgs: 17 active+clean+remapped, 13 active+undersized+degraded, 5 active+undersized, 54 active+clean; 2.2 GiB data, 9.3 GiB used, 616 GiB / 626 GiB avail; 115 MiB/s wr, 1.83k op/s; 20168/107802 objects degraded (18.708%); 6/107802 objects misplaced (0.006%) 2023-06-03T20:22:14.409 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:22:14.412 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:22:14.413 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:22:14.514 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:22:14.548 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:22:14.550 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:22:14.551 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:22:14.604 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 21 16 42600 42584 126.704 97.8125 0.00769576 0.00788636 2023-06-03T20:22:14.652 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:22:14.684 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:22:14.687 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:22:14.687 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:22:14.788 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:22:14.824 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:22:14.827 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:22:14.827 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:22:14.928 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:22:14.962 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:22:14.966 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:22:14.967 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:22:15.068 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:22:15.102 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:22:15.107 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:22:15.108 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:22:15.209 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:22:15.241 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:22:15.244 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:22:15.245 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:22:15.346 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:22:15.385 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:22:15.387 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:22:15.388 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:22:15.489 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:22:15.521 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:22:15.523 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:22:15.523 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:22:15.605 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 22 16 44112 44096 125.24 94.5 0.0947847 0.00795081 2023-06-03T20:22:15.624 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:22:15.657 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:22:15.661 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:22:15.663 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:22:15.764 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:22:15.793 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:22:15.797 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:22:15.797 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:22:15.898 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:22:15.934 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:22:15.936 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:22:15.937 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:22:16.038 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:22:16.071 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:22:16.073 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:22:16.074 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:22:16.175 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:22:16.184 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:22:16 smithi164 bash[59462]: cluster 1685823734.9823115 mgr.y (mgr.34104) 1389 : cluster 0 pgmap v1402: 89 pgs: 17 active+clean+remapped, 13 active+undersized+degraded, 5 active+undersized, 54 active+clean; 2.4 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 115 MiB/s wr, 1.84k op/s; 22221/118356 objects degraded (18.775%); 6/118356 objects misplaced (0.005%) 2023-06-03T20:22:16.204 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:22:16.208 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:22:16.209 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:22:16.309 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:22:16.341 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:22:16.345 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:22:16.346 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:22:16.447 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:22:16.457 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 20:22:16 smithi089 bash[71163]: ::ffff:172.21.15.164 - - [03/Jun/2023:20:22:16] "GET /metrics HTTP/1.1" 200 228978 "" "Prometheus/2.33.4" 2023-06-03T20:22:16.457 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:22:16 smithi089 bash[78452]: cluster 1685823734.9823115 mgr.y (mgr.34104) 1389 : cluster 0 pgmap v1402: 89 pgs: 17 active+clean+remapped, 13 active+undersized+degraded, 5 active+undersized, 54 active+clean; 2.4 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 115 MiB/s wr, 1.84k op/s; 22221/118356 objects degraded (18.775%); 6/118356 objects misplaced (0.005%) 2023-06-03T20:22:16.458 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:22:16 smithi089 bash[75068]: cluster 1685823734.9823115 mgr.y (mgr.34104) 1389 : cluster 0 pgmap v1402: 89 pgs: 17 active+clean+remapped, 13 active+undersized+degraded, 5 active+undersized, 54 active+clean; 2.4 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 115 MiB/s wr, 1.84k op/s; 22221/118356 objects degraded (18.775%); 6/118356 objects misplaced (0.005%) 2023-06-03T20:22:16.485 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:22:16.490 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:22:16.491 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:22:16.592 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:22:16.604 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 23 16 45558 45542 123.724 90.375 0.00614676 0.008076 2023-06-03T20:22:16.645 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:22:16.647 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:22:16.648 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:22:16.748 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:22:16.777 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:22:16.779 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:22:16.780 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:22:16.881 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:22:16.916 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:22:16.917 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:22:16.917 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:22:17.018 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:22:17.054 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:22:17.057 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:22:17.058 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:22:17.159 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:22:17.197 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:22:17.202 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:22:17.203 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:22:17.304 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:22:17.343 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:22:17.346 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:22:17.347 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:22:17.449 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:22:17.483 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:22:17.487 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:22:17.488 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:22:17.590 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:22:17.605 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 24 16 46931 46915 122.144 85.8125 0.0188759 0.00818027 2023-06-03T20:22:17.622 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:22:17.625 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:22:17.626 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:22:17.727 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:22:17.762 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:22:17.764 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:22:17.765 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:22:17.866 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:22:17.900 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:22:17.906 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:22:17.907 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:22:18.007 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:22:18.044 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:22:18.046 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:22:18.047 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:22:18.147 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:22:18.178 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:22:18.181 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:22:18.181 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:22:18.282 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:22:18.292 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:22:18 smithi164 bash[59462]: cluster 1685823736.9830778 mgr.y (mgr.34104) 1390 : cluster 0 pgmap v1403: 89 pgs: 17 active+clean+remapped, 13 active+undersized+degraded, 5 active+undersized, 54 active+clean; 2.5 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 103 MiB/s wr, 1.65k op/s; 23208/124125 objects degraded (18.697%); 6/124125 objects misplaced (0.005%) 2023-06-03T20:22:18.292 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:22:18 smithi164 bash[59462]: cluster 1685823737.8990204 mon.a (mon.0) 1373 : cluster 3 Health check update: Degraded data redundancy: 23208/124125 objects degraded (18.697%), 13 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-06-03T20:22:18.314 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:22:18.318 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:22:18.319 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:22:18.420 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:22:18.455 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:22:18.457 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:22:18.457 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:22:18.529 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:22:18 smithi089 bash[75068]: cluster 1685823736.9830778 mgr.y (mgr.34104) 1390 : cluster 0 pgmap v1403: 89 pgs: 17 active+clean+remapped, 13 active+undersized+degraded, 5 active+undersized, 54 active+clean; 2.5 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 103 MiB/s wr, 1.65k op/s; 23208/124125 objects degraded (18.697%); 6/124125 objects misplaced (0.005%) 2023-06-03T20:22:18.529 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:22:18 smithi089 bash[75068]: cluster 1685823737.8990204 mon.a (mon.0) 1373 : cluster 3 Health check update: Degraded data redundancy: 23208/124125 objects degraded (18.697%), 13 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-06-03T20:22:18.529 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:22:18 smithi089 bash[78452]: cluster 1685823736.9830778 mgr.y (mgr.34104) 1390 : cluster 0 pgmap v1403: 89 pgs: 17 active+clean+remapped, 13 active+undersized+degraded, 5 active+undersized, 54 active+clean; 2.5 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 103 MiB/s wr, 1.65k op/s; 23208/124125 objects degraded (18.697%); 6/124125 objects misplaced (0.005%) 2023-06-03T20:22:18.530 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:22:18 smithi089 bash[78452]: cluster 1685823737.8990204 mon.a (mon.0) 1373 : cluster 3 Health check update: Degraded data redundancy: 23208/124125 objects degraded (18.697%), 13 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-06-03T20:22:18.558 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:22:18.592 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:22:18.594 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:22:18.595 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:22:18.604 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 25 15 48545 48530 121.296 100.938 0.0200304 0.00823778 2023-06-03T20:22:18.696 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:22:18.732 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:22:18.737 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:22:18.737 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:22:18.838 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:22:18.875 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:22:18.878 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:22:18.879 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:22:18.980 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:22:19.011 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:22:19.013 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:22:19.014 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:22:19.115 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:22:19.151 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:22:19.153 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:22:19.154 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:22:19.255 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:22:19.291 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:22:19.294 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:22:19.295 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:22:19.396 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:22:19.425 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:22:19.429 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:22:19.430 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:22:19.531 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:22:19.564 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:22:19.570 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:22:19.571 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:22:19.605 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 26 16 49714 49698 119.438 73 0.00857931 0.0083672 2023-06-03T20:22:19.672 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:22:19.703 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:22:19.709 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:22:19.710 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:22:19.811 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:22:19.848 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:22:19.852 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:22:19.853 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:22:19.954 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:22:19.992 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:22:19.995 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:22:19.996 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:22:20.097 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:22:20.138 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:22:20.141 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:22:20.142 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:22:20.243 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:22:20.276 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:22:20.280 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:22:20.281 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:22:20.382 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:22:20.390 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:22:20 smithi089 bash[78452]: cluster 1685823738.983887 mgr.y (mgr.34104) 1391 : cluster 0 pgmap v1404: 89 pgs: 17 active+clean+remapped, 13 active+undersized+degraded, 5 active+undersized, 54 active+clean; 2.8 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 113 MiB/s wr, 1.81k op/s; 25527/136593 objects degraded (18.688%); 6/136593 objects misplaced (0.004%) 2023-06-03T20:22:20.390 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:22:20 smithi089 bash[75068]: cluster 1685823738.983887 mgr.y (mgr.34104) 1391 : cluster 0 pgmap v1404: 89 pgs: 17 active+clean+remapped, 13 active+undersized+degraded, 5 active+undersized, 54 active+clean; 2.8 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 113 MiB/s wr, 1.81k op/s; 25527/136593 objects degraded (18.688%); 6/136593 objects misplaced (0.004%) 2023-06-03T20:22:20.411 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:22:20.412 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:22:20.412 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:22:20.513 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:22:20.526 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:22:20 smithi164 bash[59462]: cluster 1685823738.983887 mgr.y (mgr.34104) 1391 : cluster 0 pgmap v1404: 89 pgs: 17 active+clean+remapped, 13 active+undersized+degraded, 5 active+undersized, 54 active+clean; 2.8 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 113 MiB/s wr, 1.81k op/s; 25527/136593 objects degraded (18.688%); 6/136593 objects misplaced (0.004%) 2023-06-03T20:22:20.543 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:22:20.545 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:22:20.545 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:22:20.605 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 27 15 50867 50852 117.686 72.125 0.00606782 0.00849265 2023-06-03T20:22:20.647 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:22:20.683 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:22:20.687 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:22:20.687 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:22:20.788 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:22:20.819 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:22:20.824 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:22:20.825 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:22:20.926 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:22:20.964 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:22:20.966 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:22:20.967 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:22:21.068 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:22:21.100 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:22:21.103 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:22:21.104 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:22:21.205 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:22:21.249 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:22:21.256 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:22:21.257 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:22:21.291 INFO:teuthology.orchestra.run.smithi089.stdout:ERROR: (22) Invalid argument 2023-06-03T20:22:21.291 INFO:teuthology.orchestra.run.smithi089.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-06-03T20:22:21.359 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:22:21.389 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:22:21.393 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:22:21.394 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:22:21.495 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:22:21.529 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:22:21.531 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:22:21.532 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:22:21.605 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 28 16 52294 52278 116.666 89.125 0.00978331 0.00856637 2023-06-03T20:22:21.633 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:22:21.666 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:22:21.669 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:22:21.670 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:22:21.771 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:22:21.799 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:22:21.803 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:22:21.804 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:22:21.905 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:22:21.937 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:22:21.940 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:22:21.941 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:22:22.042 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:22:22.072 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:22:22.076 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:22:22.076 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:22:22.177 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:22:22.185 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:22:22 smithi089 bash[75068]: cluster 1685823740.9845726 mgr.y (mgr.34104) 1392 : cluster 0 pgmap v1405: 89 pgs: 17 active+clean+remapped, 13 active+undersized+degraded, 5 active+undersized, 54 active+clean; 3.0 GiB data, 11 GiB used, 614 GiB / 626 GiB avail; 101 MiB/s wr, 1.62k op/s; 27279/145680 objects degraded (18.725%); 6/145680 objects misplaced (0.004%) 2023-06-03T20:22:22.185 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:22:22 smithi089 bash[78452]: cluster 1685823740.9845726 mgr.y (mgr.34104) 1392 : cluster 0 pgmap v1405: 89 pgs: 17 active+clean+remapped, 13 active+undersized+degraded, 5 active+undersized, 54 active+clean; 3.0 GiB data, 11 GiB used, 614 GiB / 626 GiB avail; 101 MiB/s wr, 1.62k op/s; 27279/145680 objects degraded (18.725%); 6/145680 objects misplaced (0.004%) 2023-06-03T20:22:22.209 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:22:22.210 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:22:22.210 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:22:22.311 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:22:22.334 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:22:22 smithi164 bash[59462]: cluster 1685823740.9845726 mgr.y (mgr.34104) 1392 : cluster 0 pgmap v1405: 89 pgs: 17 active+clean+remapped, 13 active+undersized+degraded, 5 active+undersized, 54 active+clean; 3.0 GiB data, 11 GiB used, 614 GiB / 626 GiB avail; 101 MiB/s wr, 1.62k op/s; 27279/145680 objects degraded (18.725%); 6/145680 objects misplaced (0.004%) 2023-06-03T20:22:22.348 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:22:22.354 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:22:22.355 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:22:22.456 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:22:22.487 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:22:22.491 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:22:22.492 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:22:22.593 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:22:22.605 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 29 16 53767 53751 115.817 92.0625 0.0170228 0.00862909 2023-06-03T20:22:22.622 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:22:22.627 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:22:22.628 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:22:22.729 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:22:22.763 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:22:22.765 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:22:22.766 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:22:22.867 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:22:22.900 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:22:22.904 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:22:22.905 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:22:23.006 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:22:23.037 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:22:23.041 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:22:23.042 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:22:23.143 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:22:23.177 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:22:23.180 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:22:23.181 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:22:23.282 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:22:23.324 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:22:23.328 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:22:23.329 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:22:23.334 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:22:23 smithi164 bash[59462]: audit 1685823742.1349182 mon.c (mon.2) 592 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T20:22:23.334 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:22:23 smithi164 bash[59462]: cluster 1685823742.9001498 mon.a (mon.0) 1374 : cluster 3 Health check update: Degraded data redundancy: 27279/145680 objects degraded (18.725%), 13 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-06-03T20:22:23.429 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:22:23.439 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:22:23 smithi089 bash[75068]: audit 1685823742.1349182 mon.c (mon.2) 592 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T20:22:23.439 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:22:23 smithi089 bash[75068]: cluster 1685823742.9001498 mon.a (mon.0) 1374 : cluster 3 Health check update: Degraded data redundancy: 27279/145680 objects degraded (18.725%), 13 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-06-03T20:22:23.439 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:22:23 smithi089 bash[78452]: audit 1685823742.1349182 mon.c (mon.2) 592 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T20:22:23.440 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:22:23 smithi089 bash[78452]: cluster 1685823742.9001498 mon.a (mon.0) 1374 : cluster 3 Health check update: Degraded data redundancy: 27279/145680 objects degraded (18.725%), 13 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-06-03T20:22:23.463 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:22:23.469 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:22:23.470 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:22:23.571 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:22:23.605 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 30 16 55274 55258 115.096 94.1875 0.0168541 0.0086831 2023-06-03T20:22:23.605 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:22:23.609 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:22:23.609 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:22:23.710 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:22:23.743 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:22:23.745 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:22:23.746 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:22:23.847 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:22:23.871 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:22:23.873 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:22:23.874 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:22:23.975 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:22:24.008 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:22:24.011 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:22:24.012 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:22:24.113 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:22:24.149 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:22:24.154 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:22:24.155 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:22:24.256 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:22:24.288 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:22:24.292 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:22:24.292 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:22:24.391 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:22:24 smithi089 bash[75068]: cluster 1685823742.9853368 mgr.y (mgr.34104) 1393 : cluster 0 pgmap v1406: 89 pgs: 17 active+clean+remapped, 13 active+undersized+degraded, 5 active+undersized, 54 active+clean; 3.1 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 92 MiB/s wr, 1.47k op/s; 28539/152157 objects degraded (18.756%); 6/152157 objects misplaced (0.004%) 2023-06-03T20:22:24.392 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:22:24 smithi089 bash[78452]: cluster 1685823742.9853368 mgr.y (mgr.34104) 1393 : cluster 0 pgmap v1406: 89 pgs: 17 active+clean+remapped, 13 active+undersized+degraded, 5 active+undersized, 54 active+clean; 3.1 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 92 MiB/s wr, 1.47k op/s; 28539/152157 objects degraded (18.756%); 6/152157 objects misplaced (0.004%) 2023-06-03T20:22:24.393 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:22:24.435 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:22:24.438 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:22:24.439 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:22:24.540 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:22:24.571 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:22:24.575 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:22:24.576 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:22:24.584 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:22:24 smithi164 bash[59462]: cluster 1685823742.9853368 mgr.y (mgr.34104) 1393 : cluster 0 pgmap v1406: 89 pgs: 17 active+clean+remapped, 13 active+undersized+degraded, 5 active+undersized, 54 active+clean; 3.1 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 92 MiB/s wr, 1.47k op/s; 28539/152157 objects degraded (18.756%); 6/152157 objects misplaced (0.004%) 2023-06-03T20:22:24.604 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph daemon osd.1 dump_blocked_ops 2023-06-03T20:22:24.605 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 31 16 56300 56284 113.452 64.125 0.0100044 0.00880657 2023-06-03T20:22:24.677 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:22:24.726 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:22:24.728 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:22:24.729 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:22:24.830 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:22:24.887 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:22:24.898 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:22:24.898 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:22:24.973 INFO:teuthology.orchestra.run.smithi089.stdout:osd.0: {} 2023-06-03T20:22:24.999 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:22:25.020 INFO:teuthology.orchestra.run.smithi089.stdout:osd.1: {} 2023-06-03T20:22:25.033 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:22:25.038 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:22:25.039 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:22:25.069 INFO:teuthology.orchestra.run.smithi089.stdout:osd.2: {} 2023-06-03T20:22:25.112 INFO:teuthology.orchestra.run.smithi089.stdout:osd.3: {} 2023-06-03T20:22:25.139 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:22:25.164 INFO:teuthology.orchestra.run.smithi089.stdout:osd.5: {} 2023-06-03T20:22:25.169 INFO:teuthology.orchestra.run.smithi089.stderr:nodeep-scrub is set 2023-06-03T20:22:25.190 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:22:25.196 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:22:25.197 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:22:25.224 INFO:teuthology.orchestra.run.smithi089.stdout:osd.6: {} 2023-06-03T20:22:25.262 INFO:teuthology.orchestra.run.smithi089.stdout:osd.7: {} 2023-06-03T20:22:25.263 INFO:teuthology.orchestra.run.smithi089.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-06-03T20:22:25.263 INFO:teuthology.orchestra.run.smithi089.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-06-03T20:22:25.263 INFO:teuthology.orchestra.run.smithi089.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-06-03T20:22:25.263 INFO:teuthology.orchestra.run.smithi089.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-06-03T20:22:25.263 INFO:teuthology.orchestra.run.smithi089.stderr:Error ENXIO: problem getting command descriptions from osd.4 2023-06-03T20:22:25.263 INFO:teuthology.orchestra.run.smithi089.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-06-03T20:22:25.264 INFO:teuthology.orchestra.run.smithi089.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-06-03T20:22:25.264 INFO:teuthology.orchestra.run.smithi089.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-06-03T20:22:25.300 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:22:25.332 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:22:25.334 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:22:25 smithi164 bash[59462]: audit 1685823744.88739 mon.a (mon.0) 1375 : audit 0 from='client.? 172.21.15.89:0/2477621306' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-03T20:22:25.334 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:22:25 smithi164 bash[59462]: audit 1685823745.002042 mon.c (mon.2) 593 : audit 1 from='client.? 172.21.15.89:0/1134536646' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-06-03T20:22:25.334 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:22:25 smithi164 bash[59462]: audit 1685823745.005159 mon.a (mon.0) 1376 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-06-03T20:22:25.335 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:22:25.335 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:22:25.350 INFO:teuthology.orchestra.run.smithi089.stdout: 2023-06-03T20:22:25.352 INFO:teuthology.orchestra.run.smithi089.stderr:dumped all 2023-06-03T20:22:25.398 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:22:25 smithi089 bash[75068]: audit 1685823744.88739 mon.a (mon.0) 1375 : audit 0 from='client.? 172.21.15.89:0/2477621306' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-03T20:22:25.398 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:22:25 smithi089 bash[75068]: audit 1685823745.002042 mon.c (mon.2) 593 : audit 1 from='client.? 172.21.15.89:0/1134536646' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-06-03T20:22:25.398 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:22:25 smithi089 bash[75068]: audit 1685823745.005159 mon.a (mon.0) 1376 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-06-03T20:22:25.398 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:22:25 smithi089 bash[78452]: audit 1685823744.88739 mon.a (mon.0) 1375 : audit 0 from='client.? 172.21.15.89:0/2477621306' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-03T20:22:25.398 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:22:25 smithi089 bash[78452]: audit 1685823745.002042 mon.c (mon.2) 593 : audit 1 from='client.? 172.21.15.89:0/1134536646' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-06-03T20:22:25.399 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:22:25 smithi089 bash[78452]: audit 1685823745.005159 mon.a (mon.0) 1376 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-06-03T20:22:25.438 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:22:25.470 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:22:25.474 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:22:25.475 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:22:25.575 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:22:25.605 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 32 16 57341 57325 111.939 65.0625 0.00670022 0.00892822 2023-06-03T20:22:25.606 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:22:25.610 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:22:25.611 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:22:25.711 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:22:25.752 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:22:25.757 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:22:25.757 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:22:25.858 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:22:25.888 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:22:25.888 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:22:25.889 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:22:25.990 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:22:26.017 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:22:26.020 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:22:26.021 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:22:26.121 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:22:26.150 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:22:26.153 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:22:26.154 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:22:26.256 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:22:26.279 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:22:26 smithi089 bash[75068]: cluster 1685823744.9861104 mgr.y (mgr.34104) 1394 : cluster 0 pgmap v1407: 89 pgs: 17 active+clean+remapped, 13 active+undersized+degraded, 5 active+undersized, 54 active+clean; 3.3 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 92 MiB/s wr, 1.47k op/s; 30189/160830 objects degraded (18.771%); 6/160830 objects misplaced (0.004%) 2023-06-03T20:22:26.280 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:22:26 smithi089 bash[75068]: cluster 1685823745.1246657 mon.a (mon.0) 1377 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-06-03T20:22:26.280 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:22:26 smithi089 bash[75068]: cluster 1685823745.139043 osd.6 (osd.6) 190 : cluster 3 failed to encode map e246 with expected crc 2023-06-03T20:22:26.280 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:22:26 smithi089 bash[75068]: audit 1685823745.1431482 mon.a (mon.0) 1378 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-06-03T20:22:26.281 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:22:26 smithi089 bash[75068]: cluster 1685823745.1433187 mon.a (mon.0) 1379 : cluster 0 osdmap e246: 8 total, 7 up, 7 in 2023-06-03T20:22:26.281 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:22:26 smithi089 bash[75068]: cluster 1685823745.1608524 osd.7 (osd.7) 180 : cluster 3 failed to encode map e246 with expected crc 2023-06-03T20:22:26.281 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:22:26 smithi089 bash[75068]: cluster 1685823745.1669216 osd.5 (osd.5) 184 : cluster 3 failed to encode map e246 with expected crc 2023-06-03T20:22:26.282 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 20:22:26 smithi089 bash[71163]: ::ffff:172.21.15.164 - - [03/Jun/2023:20:22:26] "GET /metrics HTTP/1.1" 200 228999 "" "Prometheus/2.33.4" 2023-06-03T20:22:26.282 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:22:26 smithi089 bash[78452]: cluster 1685823744.9861104 mgr.y (mgr.34104) 1394 : cluster 0 pgmap v1407: 89 pgs: 17 active+clean+remapped, 13 active+undersized+degraded, 5 active+undersized, 54 active+clean; 3.3 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 92 MiB/s wr, 1.47k op/s; 30189/160830 objects degraded (18.771%); 6/160830 objects misplaced (0.004%) 2023-06-03T20:22:26.282 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:22:26 smithi089 bash[78452]: cluster 1685823745.1246657 mon.a (mon.0) 1377 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-06-03T20:22:26.282 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:22:26 smithi089 bash[78452]: cluster 1685823745.139043 osd.6 (osd.6) 190 : cluster 3 failed to encode map e246 with expected crc 2023-06-03T20:22:26.283 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:22:26 smithi089 bash[78452]: audit 1685823745.1431482 mon.a (mon.0) 1378 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-06-03T20:22:26.283 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:22:26 smithi089 bash[78452]: cluster 1685823745.1433187 mon.a (mon.0) 1379 : cluster 0 osdmap e246: 8 total, 7 up, 7 in 2023-06-03T20:22:26.283 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:22:26 smithi089 bash[78452]: cluster 1685823745.1608524 osd.7 (osd.7) 180 : cluster 3 failed to encode map e246 with expected crc 2023-06-03T20:22:26.283 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:22:26 smithi089 bash[78452]: cluster 1685823745.1669216 osd.5 (osd.5) 184 : cluster 3 failed to encode map e246 with expected crc 2023-06-03T20:22:26.288 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:22:26.292 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:22:26.293 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:22:26.394 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:22:26.404 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:22:26 smithi164 bash[59462]: cluster 1685823744.9861104 mgr.y (mgr.34104) 1394 : cluster 0 pgmap v1407: 89 pgs: 17 active+clean+remapped, 13 active+undersized+degraded, 5 active+undersized, 54 active+clean; 3.3 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 92 MiB/s wr, 1.47k op/s; 30189/160830 objects degraded (18.771%); 6/160830 objects misplaced (0.004%) 2023-06-03T20:22:26.404 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:22:26 smithi164 bash[59462]: cluster 1685823745.1246657 mon.a (mon.0) 1377 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-06-03T20:22:26.404 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:22:26 smithi164 bash[59462]: cluster 1685823745.139043 osd.6 (osd.6) 190 : cluster 3 failed to encode map e246 with expected crc 2023-06-03T20:22:26.405 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:22:26 smithi164 bash[59462]: audit 1685823745.1431482 mon.a (mon.0) 1378 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-06-03T20:22:26.405 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:22:26 smithi164 bash[59462]: cluster 1685823745.1433187 mon.a (mon.0) 1379 : cluster 0 osdmap e246: 8 total, 7 up, 7 in 2023-06-03T20:22:26.405 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:22:26 smithi164 bash[59462]: cluster 1685823745.1608524 osd.7 (osd.7) 180 : cluster 3 failed to encode map e246 with expected crc 2023-06-03T20:22:26.405 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:22:26 smithi164 bash[59462]: cluster 1685823745.1669216 osd.5 (osd.5) 184 : cluster 3 failed to encode map e246 with expected crc 2023-06-03T20:22:26.429 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:22:26.434 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:22:26.435 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:22:26.536 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:22:26.570 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:22:26.573 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:22:26.573 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:22:26.605 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 33 16 58743 58727 111.202 87.625 0.0146425 0.00898687 2023-06-03T20:22:26.674 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:22:26.716 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:22:26.720 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:22:26.721 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:22:26.822 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:22:26.867 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:22:26.869 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:22:26.870 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:22:26.971 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:22:27.003 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:22:27.005 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:22:27.006 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:22:27.107 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:22:27.130 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:22:27.133 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:22:27.134 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:22:27.147 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:22:27 smithi089 bash[75068]: audit 1685823745.3300629 mgr.y (mgr.34104) 1395 : audit 0 from='client.36591 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-06-03T20:22:27.236 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:22:27.266 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:22:27.274 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:22:27.275 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:22:27.334 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:22:27 smithi164 bash[59462]: audit 1685823745.3300629 mgr.y (mgr.34104) 1395 : audit 0 from='client.36591 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-06-03T20:22:27.376 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:22:27.411 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:22:27.414 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:22:27.415 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:22:27.516 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:22:27.524 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:22:27 smithi089 bash[78452]: audit 1685823745.3300629 mgr.y (mgr.34104) 1395 : audit 0 from='client.36591 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-06-03T20:22:27.549 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:22:27.550 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:22:27.551 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:22:27.605 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 34 16 60040 60024 110.316 81.0625 0.00571643 0.00905964 2023-06-03T20:22:27.652 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:22:27.674 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:22:27.676 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:22:27.676 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:22:27.777 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:22:27.813 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:22:27.818 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:22:27.820 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:22:27.921 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:22:27.954 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:22:27.956 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:22:27.957 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:22:28.058 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:22:28.087 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:22:28.089 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:22:28.091 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:22:28.192 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:22:28.231 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:22:28.236 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:22:28.237 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:22:28.338 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:22:28.376 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:22:28.378 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:22:28.379 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:22:28.481 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:22:28.490 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:22:28 smithi164 bash[59462]: cluster 1685823746.9866426 mgr.y (mgr.34104) 1396 : cluster 0 pgmap v1409: 89 pgs: 17 active+clean+remapped, 13 active+undersized+degraded, 5 active+undersized, 54 active+clean; 3.4 GiB data, 12 GiB used, 613 GiB / 626 GiB avail; 87 MiB/s wr, 1.38k op/s; 30976/165681 objects degraded (18.696%); 6/165681 objects misplaced (0.004%) 2023-06-03T20:22:28.490 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:22:28 smithi164 bash[59462]: cluster 1685823747.9013298 mon.a (mon.0) 1380 : cluster 3 Health check update: Degraded data redundancy: 30976/165681 objects degraded (18.696%), 13 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-06-03T20:22:28.515 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:22:28.517 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:22:28.518 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:22:28.529 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:22:28 smithi089 bash[78452]: cluster 1685823746.9866426 mgr.y (mgr.34104) 1396 : cluster 0 pgmap v1409: 89 pgs: 17 active+clean+remapped, 13 active+undersized+degraded, 5 active+undersized, 54 active+clean; 3.4 GiB data, 12 GiB used, 613 GiB / 626 GiB avail; 87 MiB/s wr, 1.38k op/s; 30976/165681 objects degraded (18.696%); 6/165681 objects misplaced (0.004%) 2023-06-03T20:22:28.529 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:22:28 smithi089 bash[78452]: cluster 1685823747.9013298 mon.a (mon.0) 1380 : cluster 3 Health check update: Degraded data redundancy: 30976/165681 objects degraded (18.696%), 13 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-06-03T20:22:28.529 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:22:28 smithi089 bash[75068]: cluster 1685823746.9866426 mgr.y (mgr.34104) 1396 : cluster 0 pgmap v1409: 89 pgs: 17 active+clean+remapped, 13 active+undersized+degraded, 5 active+undersized, 54 active+clean; 3.4 GiB data, 12 GiB used, 613 GiB / 626 GiB avail; 87 MiB/s wr, 1.38k op/s; 30976/165681 objects degraded (18.696%); 6/165681 objects misplaced (0.004%) 2023-06-03T20:22:28.529 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:22:28 smithi089 bash[75068]: cluster 1685823747.9013298 mon.a (mon.0) 1380 : cluster 3 Health check update: Degraded data redundancy: 30976/165681 objects degraded (18.696%), 13 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-06-03T20:22:28.605 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 35 16 61392 61376 109.578 84.5 0.0327009 0.00911994 2023-06-03T20:22:28.619 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:22:28.675 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:22:28.682 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:22:28.683 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:22:28.701 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-06-03T20:22:28.703 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-06-03T20:22:28.742 INFO:teuthology.orchestra.run.smithi089.stdout:{"pg_ready":true,"pg_map":{"version":1408,"stamp":"2023-06-03T20:22:25.147242+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":3512407950,"num_objects":53610,"num_object_clones":0,"num_object_copies":160830,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":30189,"num_objects_misplaced":6,"num_objects_unfound":0,"num_objects_dirty":53610,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":53847,"num_write_kb":3431950,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":21,"num_bytes_recovered":1185522,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":53781,"ondisk_log_size":53781,"up":222,"acting":249,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":284,"num_osds":7,"num_per_pool_osds":7,"num_per_pool_omap_osds":7,"kb":656216064,"kb_used":12803068,"kb_used_data":9145156,"kb_used_omap":30,"kb_used_meta":3657825,"kb_avail":643412996,"statfs":{"total":671965249536,"available":658854907904,"internally_reserved":0,"allocated":9364639744,"data_stored":9358753016,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":31502,"internal_metadata":3745613042},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":927858688,"num_objects":14158,"num_object_clones":0,"num_object_copies":42474,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":7968,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":14158,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":14158,"num_write_kb":906112,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.165765"},"pg_stats":[{"pgid":"3.1a","version":"60'1","reported_seq":21,"reported_epoch":244,"state":"active+clean+remapped","last_fresh":"2023-06-03T20:21:31.315843+0000","last_change":"2023-06-03T20:21:02.300388+0000","last_active":"2023-06-03T20:21:31.315843+0000","last_peered":"2023-06-03T20:21:31.315843+0000","last_clean":"2023-06-03T20:21:31.315843+0000","last_became_active":"2023-06-03T20:21:02.299987+0000","last_became_peered":"2023-06-03T20:21:02.299987+0000","last_unstale":"2023-06-03T20:21:31.315843+0000","last_undegraded":"2023-06-03T20:21:31.315843+0000","last_fullsized":"2023-06-03T20:21:31.315843+0000","mapping_epoch":236,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":237,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T20:28:23.481980+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2,0],"avail_no_missing":["1","0","2"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":19,"reported_epoch":244,"state":"active+clean","last_fresh":"2023-06-03T20:21:31.316190+0000","last_change":"2023-06-03T20:21:02.260968+0000","last_active":"2023-06-03T20:21:31.316190+0000","last_peered":"2023-06-03T20:21:31.316190+0000","last_clean":"2023-06-03T20:21:31.316190+0000","last_became_active":"2023-06-03T20:21:02.260270+0000","last_became_peered":"2023-06-03T20:21:02.260270+0000","last_unstale":"2023-06-03T20:21:31.316190+0000","last_undegraded":"2023-06-03T20:21:31.316190+0000","last_fullsized":"2023-06-03T20:21:31.316190+0000","mapping_epoch":236,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":237,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T05:47:03.678153+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5],"acting":[1,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":19,"reported_epoch":244,"state":"active+clean","last_fresh":"2023-06-03T20:21:31.315506+0000","last_change":"2023-06-03T20:21:02.559424+0000","last_active":"2023-06-03T20:21:31.315506+0000","last_peered":"2023-06-03T20:21:31.315506+0000","last_clean":"2023-06-03T20:21:31.315506+0000","last_became_active":"2023-06-03T20:21:02.558351+0000","last_became_peered":"2023-06-03T20:21:02.558351+0000","last_unstale":"2023-06-03T20:21:31.315506+0000","last_undegraded":"2023-06-03T20:21:31.315506+0000","last_fullsized":"2023-06-03T20:21:31.315506+0000","mapping_epoch":236,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":237,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T23:29:05.523399+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3],"acting":[1,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":211,"reported_epoch":244,"state":"active+clean","last_fresh":"2023-06-03T20:21:32.218760+0000","last_change":"2023-06-03T19:56:54.507653+0000","last_active":"2023-06-03T20:21:32.218760+0000","last_peered":"2023-06-03T20:21:32.218760+0000","last_clean":"2023-06-03T20:21:32.218760+0000","last_became_active":"2023-06-03T19:56:54.507350+0000","last_became_peered":"2023-06-03T19:56:54.507350+0000","last_unstale":"2023-06-03T20:21:32.218760+0000","last_undegraded":"2023-06-03T20:21:32.218760+0000","last_fullsized":"2023-06-03T20:21:32.218760+0000","mapping_epoch":119,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T22:33:20.065844+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":252,"reported_epoch":244,"state":"active+clean","last_fresh":"2023-06-03T20:21:32.633766+0000","last_change":"2023-06-03T20:21:01.256894+0000","last_active":"2023-06-03T20:21:32.633766+0000","last_peered":"2023-06-03T20:21:32.633766+0000","last_clean":"2023-06-03T20:21:32.633766+0000","last_became_active":"2023-06-03T20:21:01.255903+0000","last_became_peered":"2023-06-03T20:21:01.255903+0000","last_unstale":"2023-06-03T20:21:32.633766+0000","last_undegraded":"2023-06-03T20:21:32.633766+0000","last_fullsized":"2023-06-03T20:21:32.633766+0000","mapping_epoch":235,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":236,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T20:35:27.810860+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":20,"reported_epoch":244,"state":"active+clean","last_fresh":"2023-06-03T20:21:31.316641+0000","last_change":"2023-06-03T20:21:01.263102+0000","last_active":"2023-06-03T20:21:31.316641+0000","last_peered":"2023-06-03T20:21:31.316641+0000","last_clean":"2023-06-03T20:21:31.316641+0000","last_became_active":"2023-06-03T20:21:01.262794+0000","last_became_peered":"2023-06-03T20:21:01.262794+0000","last_unstale":"2023-06-03T20:21:31.316641+0000","last_undegraded":"2023-06-03T20:21:31.316641+0000","last_fullsized":"2023-06-03T20:21:31.316641+0000","mapping_epoch":235,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":236,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T23:37:02.207295+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":207,"reported_epoch":244,"state":"active+clean","last_fresh":"2023-06-03T20:21:32.218963+0000","last_change":"2023-06-03T19:56:54.509490+0000","last_active":"2023-06-03T20:21:32.218963+0000","last_peered":"2023-06-03T20:21:32.218963+0000","last_clean":"2023-06-03T20:21:32.218963+0000","last_became_active":"2023-06-03T19:56:54.509145+0000","last_became_peered":"2023-06-03T19:56:54.509145+0000","last_unstale":"2023-06-03T20:21:32.218963+0000","last_undegraded":"2023-06-03T20:21:32.218963+0000","last_fullsized":"2023-06-03T20:21:32.218963+0000","mapping_epoch":119,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T07:12:03.964110+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":253,"reported_epoch":244,"state":"active+clean+remapped","last_fresh":"2023-06-03T20:21:32.633895+0000","last_change":"2023-06-03T20:21:02.282661+0000","last_active":"2023-06-03T20:21:32.633895+0000","last_peered":"2023-06-03T20:21:32.633895+0000","last_clean":"2023-06-03T20:21:32.633895+0000","last_became_active":"2023-06-03T20:21:02.282053+0000","last_became_peered":"2023-06-03T20:21:02.282053+0000","last_unstale":"2023-06-03T20:21:32.633895+0000","last_undegraded":"2023-06-03T20:21:32.633895+0000","last_fullsized":"2023-06-03T20:21:32.633895+0000","mapping_epoch":236,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":237,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T20:44:36.999754+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1],"acting":[3,1,5],"avail_no_missing":["3","1","5"],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":177,"reported_epoch":244,"state":"active+clean","last_fresh":"2023-06-03T20:21:32.567088+0000","last_change":"2023-06-03T20:21:01.258441+0000","last_active":"2023-06-03T20:21:32.567088+0000","last_peered":"2023-06-03T20:21:32.567088+0000","last_clean":"2023-06-03T20:21:32.567088+0000","last_became_active":"2023-06-03T20:21:01.257779+0000","last_became_peered":"2023-06-03T20:21:01.257779+0000","last_unstale":"2023-06-03T20:21:32.567088+0000","last_undegraded":"2023-06-03T20:21:32.567088+0000","last_fullsized":"2023-06-03T20:21:32.567088+0000","mapping_epoch":235,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":236,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,1],"acting":[7,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":208,"reported_epoch":244,"state":"active+clean","last_fresh":"2023-06-03T20:21:32.634637+0000","last_change":"2023-06-03T20:16:20.909781+0000","last_active":"2023-06-03T20:21:32.634637+0000","last_peered":"2023-06-03T20:21:32.634637+0000","last_clean":"2023-06-03T20:21:32.634637+0000","last_became_active":"2023-06-03T20:16:20.909496+0000","last_became_peered":"2023-06-03T20:16:20.909496+0000","last_unstale":"2023-06-03T20:21:32.634637+0000","last_undegraded":"2023-06-03T20:21:32.634637+0000","last_fullsized":"2023-06-03T20:21:32.634637+0000","mapping_epoch":212,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":213,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T20:32:26.886172+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,5],"acting":[3,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"60'3","reported_seq":233,"reported_epoch":244,"state":"active+undersized+degraded","last_fresh":"2023-06-03T20:21:32.567120+0000","last_change":"2023-06-03T20:11:54.025696+0000","last_active":"2023-06-03T20:21:32.567120+0000","last_peered":"2023-06-03T20:21:32.567120+0000","last_clean":"2023-06-03T20:11:43.973370+0000","last_became_active":"2023-06-03T20:11:54.025696+0000","last_became_peered":"2023-06-03T20:11:54.025696+0000","last_unstale":"2023-06-03T20:21:32.567120+0000","last_undegraded":"2023-06-03T20:11:54.012202+0000","last_fullsized":"2023-06-03T20:11:54.012054+0000","mapping_epoch":195,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":173,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3],"acting":[7,3],"avail_no_missing":["7","3"],"object_location_counts":[{"shards":"3,7","objects":2}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":20,"reported_epoch":244,"state":"active+clean","last_fresh":"2023-06-03T20:21:31.316165+0000","last_change":"2023-06-03T20:21:01.265588+0000","last_active":"2023-06-03T20:21:31.316165+0000","last_peered":"2023-06-03T20:21:31.316165+0000","last_clean":"2023-06-03T20:21:31.316165+0000","last_became_active":"2023-06-03T20:21:01.265292+0000","last_became_peered":"2023-06-03T20:21:01.265292+0000","last_unstale":"2023-06-03T20:21:31.316165+0000","last_undegraded":"2023-06-03T20:21:31.316165+0000","last_fullsized":"2023-06-03T20:21:31.316165+0000","mapping_epoch":235,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":236,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T01:29:11.398318+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,0],"acting":[1,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"77'23","reported_seq":135,"reported_epoch":244,"state":"active+clean+remapped","last_fresh":"2023-06-03T20:21:32.567496+0000","last_change":"2023-06-03T20:16:22.089746+0000","last_active":"2023-06-03T20:21:32.567496+0000","last_peered":"2023-06-03T20:21:32.567496+0000","last_clean":"2023-06-03T20:21:32.567496+0000","last_became_active":"2023-06-03T20:16:21.916216+0000","last_became_peered":"2023-06-03T20:16:21.916216+0000","last_unstale":"2023-06-03T20:21:32.567496+0000","last_undegraded":"2023-06-03T20:21:32.567496+0000","last_fullsized":"2023-06-03T20:21:32.567496+0000","mapping_epoch":213,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":214,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":23,"log_dups_size":0,"ondisk_log_size":23,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4716,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":4,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":23,"num_write_kb":20,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4716,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2],"acting":[7,2,6],"avail_no_missing":["7","2","6"],"object_location_counts":[{"shards":"2,6,7","objects":4}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":19,"reported_epoch":244,"state":"active+clean","last_fresh":"2023-06-03T20:21:31.315905+0000","last_change":"2023-06-03T20:21:02.264449+0000","last_active":"2023-06-03T20:21:31.315905+0000","last_peered":"2023-06-03T20:21:31.315905+0000","last_clean":"2023-06-03T20:21:31.315905+0000","last_became_active":"2023-06-03T20:21:02.264020+0000","last_became_peered":"2023-06-03T20:21:02.264020+0000","last_unstale":"2023-06-03T20:21:31.315905+0000","last_undegraded":"2023-06-03T20:21:31.315905+0000","last_fullsized":"2023-06-03T20:21:31.315905+0000","mapping_epoch":236,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":237,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T07:33:32.525277+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5],"acting":[1,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.13","version":"60'2","reported_seq":235,"reported_epoch":244,"state":"active+undersized+degraded","last_fresh":"2023-06-03T20:21:32.566913+0000","last_change":"2023-06-03T20:11:54.019051+0000","last_active":"2023-06-03T20:21:32.566913+0000","last_peered":"2023-06-03T20:21:32.566913+0000","last_clean":"2023-06-03T20:11:43.972488+0000","last_became_active":"2023-06-03T20:11:54.019051+0000","last_became_peered":"2023-06-03T20:11:54.019051+0000","last_unstale":"2023-06-03T20:21:32.566913+0000","last_undegraded":"2023-06-03T20:11:54.012366+0000","last_fullsized":"2023-06-03T20:11:54.012217+0000","mapping_epoch":195,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2],"acting":[7,2],"avail_no_missing":["7","2"],"object_location_counts":[{"shards":"2,7","objects":2}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":236,"reported_epoch":244,"state":"active+clean","last_fresh":"2023-06-03T20:21:32.633739+0000","last_change":"2023-06-03T20:21:01.256612+0000","last_active":"2023-06-03T20:21:32.633739+0000","last_peered":"2023-06-03T20:21:32.633739+0000","last_clean":"2023-06-03T20:21:32.633739+0000","last_became_active":"2023-06-03T20:21:01.256243+0000","last_became_peered":"2023-06-03T20:21:01.256243+0000","last_unstale":"2023-06-03T20:21:32.633739+0000","last_undegraded":"2023-06-03T20:21:32.633739+0000","last_fullsized":"2023-06-03T20:21:32.633739+0000","mapping_epoch":235,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":236,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T05:24:47.054342+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":208,"reported_epoch":245,"state":"active+clean","last_fresh":"2023-06-03T20:21:49.793699+0000","last_change":"2023-06-03T19:56:54.507479+0000","last_active":"2023-06-03T20:21:49.793699+0000","last_peered":"2023-06-03T20:21:49.793699+0000","last_clean":"2023-06-03T20:21:49.793699+0000","last_became_active":"2023-06-03T19:56:54.507149+0000","last_became_peered":"2023-06-03T19:56:54.507149+0000","last_unstale":"2023-06-03T20:21:49.793699+0000","last_undegraded":"2023-06-03T20:21:49.793699+0000","last_fullsized":"2023-06-03T20:21:49.793699+0000","mapping_epoch":119,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T20:22:36.676317+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":111,"reported_epoch":244,"state":"active+clean","last_fresh":"2023-06-03T20:21:32.567832+0000","last_change":"2023-06-03T20:17:22.832844+0000","last_active":"2023-06-03T20:21:32.567832+0000","last_peered":"2023-06-03T20:21:32.567832+0000","last_clean":"2023-06-03T20:21:32.567832+0000","last_became_active":"2023-06-03T20:17:22.832473+0000","last_became_peered":"2023-06-03T20:17:22.832473+0000","last_unstale":"2023-06-03T20:21:32.567832+0000","last_undegraded":"2023-06-03T20:21:32.567832+0000","last_fullsized":"2023-06-03T20:21:32.567832+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,3],"acting":[7,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":225,"reported_epoch":244,"state":"active+clean+remapped","last_fresh":"2023-06-03T20:21:32.566454+0000","last_change":"2023-06-03T20:16:21.916599+0000","last_active":"2023-06-03T20:21:32.566454+0000","last_peered":"2023-06-03T20:21:32.566454+0000","last_clean":"2023-06-03T20:21:32.566454+0000","last_became_active":"2023-06-03T20:16:21.916145+0000","last_became_peered":"2023-06-03T20:16:21.916145+0000","last_unstale":"2023-06-03T20:21:32.566454+0000","last_undegraded":"2023-06-03T20:21:32.566454+0000","last_fullsized":"2023-06-03T20:21:32.566454+0000","mapping_epoch":213,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":214,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0],"acting":[7,0,6],"avail_no_missing":["7","0","6"],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":208,"reported_epoch":244,"state":"active+clean","last_fresh":"2023-06-03T20:21:32.219145+0000","last_change":"2023-06-03T19:56:54.510289+0000","last_active":"2023-06-03T20:21:32.219145+0000","last_peered":"2023-06-03T20:21:32.219145+0000","last_clean":"2023-06-03T20:21:32.219145+0000","last_became_active":"2023-06-03T19:56:54.509905+0000","last_became_peered":"2023-06-03T19:56:54.509905+0000","last_unstale":"2023-06-03T20:21:32.219145+0000","last_undegraded":"2023-06-03T20:21:32.219145+0000","last_fullsized":"2023-06-03T20:21:32.219145+0000","mapping_epoch":119,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T02:54:26.792401+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"60'8","reported_seq":66,"reported_epoch":244,"state":"active+clean","last_fresh":"2023-06-03T20:21:32.217662+0000","last_change":"2023-06-03T20:16:20.914195+0000","last_active":"2023-06-03T20:21:32.217662+0000","last_peered":"2023-06-03T20:21:32.217662+0000","last_clean":"2023-06-03T20:21:32.217662+0000","last_became_active":"2023-06-03T20:16:20.906350+0000","last_became_peered":"2023-06-03T20:16:20.906350+0000","last_unstale":"2023-06-03T20:21:32.217662+0000","last_undegraded":"2023-06-03T20:21:32.217662+0000","last_fullsized":"2023-06-03T20:21:32.217662+0000","mapping_epoch":212,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":213,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T20:33:32.314202+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":236,"reported_epoch":244,"state":"active+clean","last_fresh":"2023-06-03T20:21:32.635678+0000","last_change":"2023-06-03T20:21:01.256880+0000","last_active":"2023-06-03T20:21:32.635678+0000","last_peered":"2023-06-03T20:21:32.635678+0000","last_clean":"2023-06-03T20:21:32.635678+0000","last_became_active":"2023-06-03T20:21:01.256478+0000","last_became_peered":"2023-06-03T20:21:01.256478+0000","last_unstale":"2023-06-03T20:21:32.635678+0000","last_undegraded":"2023-06-03T20:21:32.635678+0000","last_fullsized":"2023-06-03T20:21:32.635678+0000","mapping_epoch":235,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":236,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T01:09:33.462743+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"13.1","version":"245'3211","reported_seq":3220,"reported_epoch":245,"state":"active+undersized+degraded","last_fresh":"2023-06-03T20:22:20.821456+0000","last_change":"2023-06-03T20:21:53.618190+0000","last_active":"2023-06-03T20:22:20.821456+0000","last_peered":"2023-06-03T20:22:20.821456+0000","last_clean":"2023-06-03T20:21:29.277668+0000","last_became_active":"2023-06-03T20:21:30.305612+0000","last_became_peered":"2023-06-03T20:21:30.305612+0000","last_unstale":"2023-06-03T20:22:20.821456+0000","last_undegraded":"2023-06-03T20:21:53.604572+0000","last_fullsized":"2023-06-03T20:21:30.296329+0000","mapping_epoch":242,"log_start":"0'0","ondisk_log_start":"0'0","created":242,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T20:21:29.277668+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T20:21:29.277668+0000","last_clean_scrub_stamp":"2023-06-03T20:21:29.277668+0000","objects_scrubbed":0,"log_size":3211,"log_dups_size":0,"ondisk_log_size":3211,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T23:28:41.807393+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":210436096,"num_objects":3211,"num_object_clones":0,"num_object_copies":9633,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3211,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3211,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3211,"num_write_kb":205504,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7],"acting":[2,7],"avail_no_missing":["2","7"],"object_location_counts":[{"shards":"2,7","objects":3211}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.f","version":"60'1","reported_seq":234,"reported_epoch":244,"state":"active+undersized+degraded","last_fresh":"2023-06-03T20:21:32.567349+0000","last_change":"2023-06-03T20:11:54.016915+0000","last_active":"2023-06-03T20:21:32.567349+0000","last_peered":"2023-06-03T20:21:32.567349+0000","last_clean":"2023-06-03T20:11:43.972655+0000","last_became_active":"2023-06-03T20:11:54.016915+0000","last_became_peered":"2023-06-03T20:11:54.016915+0000","last_unstale":"2023-06-03T20:21:32.567349+0000","last_undegraded":"2023-06-03T20:11:54.009884+0000","last_fullsized":"2023-06-03T20:11:54.009713+0000","mapping_epoch":195,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0],"acting":[7,0],"avail_no_missing":["7","0"],"object_location_counts":[{"shards":"0,7","objects":1}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":185,"reported_epoch":244,"state":"active+clean+remapped","last_fresh":"2023-06-03T20:21:32.565958+0000","last_change":"2023-06-03T20:17:23.832761+0000","last_active":"2023-06-03T20:21:32.565958+0000","last_peered":"2023-06-03T20:21:32.565958+0000","last_clean":"2023-06-03T20:21:32.565958+0000","last_became_active":"2023-06-03T20:17:23.832000+0000","last_became_peered":"2023-06-03T20:17:23.832000+0000","last_unstale":"2023-06-03T20:21:32.565958+0000","last_undegraded":"2023-06-03T20:21:32.565958+0000","last_fullsized":"2023-06-03T20:21:32.565958+0000","mapping_epoch":223,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":224,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5],"acting":[7,5,6],"avail_no_missing":["7","5","6"],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"13.0","version":"245'3447","reported_seq":3457,"reported_epoch":245,"state":"active+clean","last_fresh":"2023-06-03T20:22:23.484051+0000","last_change":"2023-06-03T20:21:30.309649+0000","last_active":"2023-06-03T20:22:23.484051+0000","last_peered":"2023-06-03T20:22:23.484051+0000","last_clean":"2023-06-03T20:22:23.484051+0000","last_became_active":"2023-06-03T20:21:30.309118+0000","last_became_peered":"2023-06-03T20:21:30.309118+0000","last_unstale":"2023-06-03T20:22:23.484051+0000","last_undegraded":"2023-06-03T20:22:23.484051+0000","last_fullsized":"2023-06-03T20:22:23.484051+0000","mapping_epoch":242,"log_start":"0'0","ondisk_log_start":"0'0","created":242,"last_epoch_clean":243,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T20:21:29.277668+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T20:21:29.277668+0000","last_clean_scrub_stamp":"2023-06-03T20:21:29.277668+0000","objects_scrubbed":0,"log_size":3447,"log_dups_size":0,"ondisk_log_size":3447,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":225902592,"num_objects":3447,"num_object_clones":0,"num_object_copies":10341,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3447,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3447,"num_write_kb":220608,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,2],"acting":[7,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.e","version":"60'2","reported_seq":262,"reported_epoch":244,"state":"active+clean+remapped","last_fresh":"2023-06-03T20:21:32.567362+0000","last_change":"2023-06-03T20:21:02.485665+0000","last_active":"2023-06-03T20:21:32.567362+0000","last_peered":"2023-06-03T20:21:32.567362+0000","last_clean":"2023-06-03T20:21:32.567362+0000","last_became_active":"2023-06-03T20:21:02.280395+0000","last_became_peered":"2023-06-03T20:21:02.280395+0000","last_unstale":"2023-06-03T20:21:32.567362+0000","last_undegraded":"2023-06-03T20:21:32.567362+0000","last_fullsized":"2023-06-03T20:21:32.567362+0000","mapping_epoch":236,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":237,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":1,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1],"acting":[7,1,0],"avail_no_missing":["7","0","1"],"object_location_counts":[{"shards":"0,1,7","objects":1}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":19,"reported_epoch":244,"state":"active+clean","last_fresh":"2023-06-03T20:21:31.315667+0000","last_change":"2023-06-03T20:21:02.560114+0000","last_active":"2023-06-03T20:21:31.315667+0000","last_peered":"2023-06-03T20:21:31.315667+0000","last_clean":"2023-06-03T20:21:31.315667+0000","last_became_active":"2023-06-03T20:21:02.558981+0000","last_became_peered":"2023-06-03T20:21:02.558981+0000","last_unstale":"2023-06-03T20:21:31.315667+0000","last_undegraded":"2023-06-03T20:21:31.315667+0000","last_fullsized":"2023-06-03T20:21:31.315667+0000","mapping_epoch":236,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":237,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T05:46:03.325996+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3],"acting":[1,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"13.3","version":"245'3241","reported_seq":3249,"reported_epoch":245,"state":"active+undersized+degraded","last_fresh":"2023-06-03T20:22:22.023757+0000","last_change":"2023-06-03T20:21:53.622680+0000","last_active":"2023-06-03T20:22:22.023757+0000","last_peered":"2023-06-03T20:22:22.023757+0000","last_clean":"2023-06-03T20:21:29.277668+0000","last_became_active":"2023-06-03T20:21:30.308896+0000","last_became_peered":"2023-06-03T20:21:30.308896+0000","last_unstale":"2023-06-03T20:22:22.023757+0000","last_undegraded":"2023-06-03T20:21:53.600254+0000","last_fullsized":"2023-06-03T20:21:30.301109+0000","mapping_epoch":242,"log_start":"0'0","ondisk_log_start":"0'0","created":242,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T20:21:29.277668+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T20:21:29.277668+0000","last_clean_scrub_stamp":"2023-06-03T20:21:29.277668+0000","objects_scrubbed":0,"log_size":3241,"log_dups_size":0,"ondisk_log_size":3241,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T21:10:38.914255+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":212402176,"num_objects":3241,"num_object_clones":0,"num_object_copies":9723,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3241,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3241,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3241,"num_write_kb":207424,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3],"acting":[0,3],"avail_no_missing":["0","3"],"object_location_counts":[{"shards":"0,3","objects":3241}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.d","version":"60'1","reported_seq":208,"reported_epoch":244,"state":"active+clean","last_fresh":"2023-06-03T20:21:32.566773+0000","last_change":"2023-06-03T20:16:20.915741+0000","last_active":"2023-06-03T20:21:32.566773+0000","last_peered":"2023-06-03T20:21:32.566773+0000","last_clean":"2023-06-03T20:21:32.566773+0000","last_became_active":"2023-06-03T20:16:20.908091+0000","last_became_peered":"2023-06-03T20:16:20.908091+0000","last_unstale":"2023-06-03T20:21:32.566773+0000","last_undegraded":"2023-06-03T20:21:32.566773+0000","last_fullsized":"2023-06-03T20:21:32.566773+0000","mapping_epoch":212,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":213,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":70,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,3],"acting":[7,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":20,"reported_epoch":244,"state":"active+clean","last_fresh":"2023-06-03T20:21:31.316332+0000","last_change":"2023-06-03T20:21:01.261609+0000","last_active":"2023-06-03T20:21:31.316332+0000","last_peered":"2023-06-03T20:21:31.316332+0000","last_clean":"2023-06-03T20:21:31.316332+0000","last_became_active":"2023-06-03T20:21:01.261293+0000","last_became_peered":"2023-06-03T20:21:01.261293+0000","last_unstale":"2023-06-03T20:21:31.316332+0000","last_undegraded":"2023-06-03T20:21:31.316332+0000","last_fullsized":"2023-06-03T20:21:31.316332+0000","mapping_epoch":235,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":236,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T06:20:39.958989+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,5],"acting":[1,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"13.2","version":"245'3438","reported_seq":3447,"reported_epoch":245,"state":"active+undersized+degraded","last_fresh":"2023-06-03T20:22:24.088244+0000","last_change":"2023-06-03T20:21:53.606828+0000","last_active":"2023-06-03T20:22:24.088244+0000","last_peered":"2023-06-03T20:22:24.088244+0000","last_clean":"2023-06-03T20:21:29.277668+0000","last_became_active":"2023-06-03T20:21:30.618072+0000","last_became_peered":"2023-06-03T20:21:30.618072+0000","last_unstale":"2023-06-03T20:22:24.088244+0000","last_undegraded":"2023-06-03T20:21:53.600553+0000","last_fullsized":"2023-06-03T20:21:30.302591+0000","mapping_epoch":242,"log_start":"0'0","ondisk_log_start":"0'0","created":242,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T20:21:29.277668+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T20:21:29.277668+0000","last_clean_scrub_stamp":"2023-06-03T20:21:29.277668+0000","objects_scrubbed":0,"log_size":3438,"log_dups_size":0,"ondisk_log_size":3438,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T06:50:06.082160+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":225312768,"num_objects":3438,"num_object_clones":0,"num_object_copies":10314,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3438,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3438,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3438,"num_write_kb":220032,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5],"acting":[3,5],"avail_no_missing":["3","5"],"object_location_counts":[{"shards":"3,5","objects":3438}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":225,"reported_epoch":245,"state":"active+clean","last_fresh":"2023-06-03T20:21:49.290721+0000","last_change":"2023-06-03T20:21:01.262006+0000","last_active":"2023-06-03T20:21:49.290721+0000","last_peered":"2023-06-03T20:21:49.290721+0000","last_clean":"2023-06-03T20:21:49.290721+0000","last_became_active":"2023-06-03T20:21:01.261491+0000","last_became_peered":"2023-06-03T20:21:01.261491+0000","last_unstale":"2023-06-03T20:21:49.290721+0000","last_undegraded":"2023-06-03T20:21:49.290721+0000","last_fullsized":"2023-06-03T20:21:49.290721+0000","mapping_epoch":235,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":236,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T03:01:27.196941+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,1],"acting":[3,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":261,"reported_epoch":244,"state":"active+clean+remapped","last_fresh":"2023-06-03T20:21:32.218168+0000","last_change":"2023-06-03T20:21:02.263607+0000","last_active":"2023-06-03T20:21:32.218168+0000","last_peered":"2023-06-03T20:21:32.218168+0000","last_clean":"2023-06-03T20:21:32.218168+0000","last_became_active":"2023-06-03T20:21:02.262921+0000","last_became_peered":"2023-06-03T20:21:02.262921+0000","last_unstale":"2023-06-03T20:21:32.218168+0000","last_undegraded":"2023-06-03T20:21:32.218168+0000","last_fullsized":"2023-06-03T20:21:32.218168+0000","mapping_epoch":236,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":237,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T20:18:05.486358+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1,3],"avail_no_missing":["0","1","3"],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"13.5","version":"245'3363","reported_seq":3372,"reported_epoch":245,"state":"active+undersized+degraded","last_fresh":"2023-06-03T20:22:21.988297+0000","last_change":"2023-06-03T20:21:53.623414+0000","last_active":"2023-06-03T20:22:21.988297+0000","last_peered":"2023-06-03T20:22:21.988297+0000","last_clean":"2023-06-03T20:21:29.277668+0000","last_became_active":"2023-06-03T20:21:30.616362+0000","last_became_peered":"2023-06-03T20:21:30.616362+0000","last_unstale":"2023-06-03T20:22:21.988297+0000","last_undegraded":"2023-06-03T20:21:53.610764+0000","last_fullsized":"2023-06-03T20:21:30.301434+0000","mapping_epoch":242,"log_start":"0'0","ondisk_log_start":"0'0","created":242,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T20:21:29.277668+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T20:21:29.277668+0000","last_clean_scrub_stamp":"2023-06-03T20:21:29.277668+0000","objects_scrubbed":0,"log_size":3363,"log_dups_size":0,"ondisk_log_size":3363,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T01:37:35.279813+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":220397568,"num_objects":3363,"num_object_clones":0,"num_object_copies":10089,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3363,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3363,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3363,"num_write_kb":215232,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5],"acting":[0,5],"avail_no_missing":["0","5"],"object_location_counts":[{"shards":"0,5","objects":3363}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.b","version":"60'2","reported_seq":226,"reported_epoch":244,"state":"active+undersized+degraded","last_fresh":"2023-06-03T20:21:39.120694+0000","last_change":"2023-06-03T20:11:54.019446+0000","last_active":"2023-06-03T20:21:39.120694+0000","last_peered":"2023-06-03T20:21:39.120694+0000","last_clean":"2023-06-03T20:11:43.977155+0000","last_became_active":"2023-06-03T20:11:54.019446+0000","last_became_peered":"2023-06-03T20:11:54.019446+0000","last_unstale":"2023-06-03T20:21:39.120694+0000","last_undegraded":"2023-06-03T20:11:54.013169+0000","last_fullsized":"2023-06-03T20:11:54.012960+0000","mapping_epoch":195,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T20:48:53.735497+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0],"acting":[3,0],"avail_no_missing":["3","0"],"object_location_counts":[{"shards":"0,3","objects":1}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":170,"reported_epoch":244,"state":"active+clean+remapped","last_fresh":"2023-06-03T20:21:32.633978+0000","last_change":"2023-06-03T20:21:02.295273+0000","last_active":"2023-06-03T20:21:32.633978+0000","last_peered":"2023-06-03T20:21:32.633978+0000","last_clean":"2023-06-03T20:21:32.633978+0000","last_became_active":"2023-06-03T20:21:02.294202+0000","last_became_peered":"2023-06-03T20:21:02.294202+0000","last_unstale":"2023-06-03T20:21:32.633978+0000","last_undegraded":"2023-06-03T20:21:32.633978+0000","last_fullsized":"2023-06-03T20:21:32.633978+0000","mapping_epoch":236,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":237,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T06:30:35.038844+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1],"acting":[3,1,7],"avail_no_missing":["3","1","7"],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"13.4","version":"245'3166","reported_seq":3178,"reported_epoch":245,"state":"active+clean","last_fresh":"2023-06-03T20:22:20.826871+0000","last_change":"2023-06-03T20:21:30.618079+0000","last_active":"2023-06-03T20:22:20.826871+0000","last_peered":"2023-06-03T20:22:20.826871+0000","last_clean":"2023-06-03T20:22:20.826871+0000","last_became_active":"2023-06-03T20:21:30.617468+0000","last_became_peered":"2023-06-03T20:21:30.617468+0000","last_unstale":"2023-06-03T20:22:20.826871+0000","last_undegraded":"2023-06-03T20:22:20.826871+0000","last_fullsized":"2023-06-03T20:22:20.826871+0000","mapping_epoch":242,"log_start":"0'0","ondisk_log_start":"0'0","created":242,"last_epoch_clean":243,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T20:21:29.277668+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T20:21:29.277668+0000","last_clean_scrub_stamp":"2023-06-03T20:21:29.277668+0000","objects_scrubbed":0,"log_size":3166,"log_dups_size":0,"ondisk_log_size":3166,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T22:31:44.994683+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":207486976,"num_objects":3166,"num_object_clones":0,"num_object_copies":9498,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3166,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3166,"num_write_kb":202624,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,5],"acting":[2,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":19,"reported_epoch":244,"state":"active+clean","last_fresh":"2023-06-03T20:21:31.315716+0000","last_change":"2023-06-03T20:21:02.262217+0000","last_active":"2023-06-03T20:21:31.315716+0000","last_peered":"2023-06-03T20:21:31.315716+0000","last_clean":"2023-06-03T20:21:31.315716+0000","last_became_active":"2023-06-03T20:21:02.261851+0000","last_became_peered":"2023-06-03T20:21:02.261851+0000","last_unstale":"2023-06-03T20:21:31.315716+0000","last_undegraded":"2023-06-03T20:21:31.315716+0000","last_fullsized":"2023-06-03T20:21:31.315716+0000","mapping_epoch":236,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":237,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T06:47:53.406518+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3],"acting":[1,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":230,"reported_epoch":244,"state":"active+clean+remapped","last_fresh":"2023-06-03T20:21:32.635487+0000","last_change":"2023-06-03T20:21:02.335231+0000","last_active":"2023-06-03T20:21:32.635487+0000","last_peered":"2023-06-03T20:21:32.635487+0000","last_clean":"2023-06-03T20:21:32.635487+0000","last_became_active":"2023-06-03T20:21:02.334326+0000","last_became_peered":"2023-06-03T20:21:02.334326+0000","last_unstale":"2023-06-03T20:21:32.635487+0000","last_undegraded":"2023-06-03T20:21:32.635487+0000","last_fullsized":"2023-06-03T20:21:32.635487+0000","mapping_epoch":236,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":237,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T23:46:08.667477+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1,0],"avail_no_missing":["2","0","1"],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"13.7","version":"245'3470","reported_seq":3478,"reported_epoch":245,"state":"active+undersized+degraded","last_fresh":"2023-06-03T20:22:23.453923+0000","last_change":"2023-06-03T20:21:53.602739+0000","last_active":"2023-06-03T20:22:23.453923+0000","last_peered":"2023-06-03T20:22:23.453923+0000","last_clean":"2023-06-03T20:21:29.277668+0000","last_became_active":"2023-06-03T20:21:30.614923+0000","last_became_peered":"2023-06-03T20:21:30.614923+0000","last_unstale":"2023-06-03T20:22:23.453923+0000","last_undegraded":"2023-06-03T20:21:53.602595+0000","last_fullsized":"2023-06-03T20:21:30.302797+0000","mapping_epoch":242,"log_start":"0'0","ondisk_log_start":"0'0","created":242,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T20:21:29.277668+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T20:21:29.277668+0000","last_clean_scrub_stamp":"2023-06-03T20:21:29.277668+0000","objects_scrubbed":0,"log_size":3470,"log_dups_size":0,"ondisk_log_size":3470,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":227409920,"num_objects":3470,"num_object_clones":0,"num_object_copies":10410,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3470,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3470,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3470,"num_write_kb":222080,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5],"acting":[7,5],"avail_no_missing":["7","5"],"object_location_counts":[{"shards":"5,7","objects":3470}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":147,"reported_epoch":244,"state":"active+undersized","last_fresh":"2023-06-03T20:21:32.635256+0000","last_change":"2023-06-03T20:11:54.018578+0000","last_active":"2023-06-03T20:21:32.635256+0000","last_peered":"2023-06-03T20:21:32.635256+0000","last_clean":"2023-06-03T19:55:41.905941+0000","last_became_active":"2023-06-03T20:11:54.018578+0000","last_became_peered":"2023-06-03T20:11:54.018578+0000","last_unstale":"2023-06-03T20:21:32.635256+0000","last_undegraded":"2023-06-03T20:21:32.635256+0000","last_fullsized":"2023-06-03T20:11:54.009386+0000","mapping_epoch":195,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T21:13:40.683176+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7],"acting":[2,7],"avail_no_missing":["2","7"],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":19,"reported_epoch":244,"state":"active+clean","last_fresh":"2023-06-03T20:21:31.316754+0000","last_change":"2023-06-03T20:21:02.261425+0000","last_active":"2023-06-03T20:21:31.316754+0000","last_peered":"2023-06-03T20:21:31.316754+0000","last_clean":"2023-06-03T20:21:31.316754+0000","last_became_active":"2023-06-03T20:21:02.260954+0000","last_became_peered":"2023-06-03T20:21:02.260954+0000","last_unstale":"2023-06-03T20:21:31.316754+0000","last_undegraded":"2023-06-03T20:21:31.316754+0000","last_fullsized":"2023-06-03T20:21:31.316754+0000","mapping_epoch":236,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":237,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T01:13:29.939775+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3],"acting":[1,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"13.6","version":"245'3414","reported_seq":3423,"reported_epoch":245,"state":"active+undersized+degraded","last_fresh":"2023-06-03T20:22:24.081387+0000","last_change":"2023-06-03T20:21:53.605824+0000","last_active":"2023-06-03T20:22:24.081387+0000","last_peered":"2023-06-03T20:22:24.081387+0000","last_clean":"2023-06-03T20:21:29.277668+0000","last_became_active":"2023-06-03T20:21:30.619987+0000","last_became_peered":"2023-06-03T20:21:30.619987+0000","last_unstale":"2023-06-03T20:22:24.081387+0000","last_undegraded":"2023-06-03T20:21:53.599987+0000","last_fullsized":"2023-06-03T20:21:30.302526+0000","mapping_epoch":242,"log_start":"0'0","ondisk_log_start":"0'0","created":242,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T20:21:29.277668+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T20:21:29.277668+0000","last_clean_scrub_stamp":"2023-06-03T20:21:29.277668+0000","objects_scrubbed":0,"log_size":3414,"log_dups_size":0,"ondisk_log_size":3414,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T00:51:33.781333+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":223739904,"num_objects":3414,"num_object_clones":0,"num_object_copies":10242,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3414,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3414,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3414,"num_write_kb":218496,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5],"acting":[3,5],"avail_no_missing":["3","5"],"object_location_counts":[{"shards":"3,5","objects":3414}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":244,"reported_epoch":244,"state":"active+clean","last_fresh":"2023-06-03T20:21:32.633680+0000","last_change":"2023-06-03T20:21:01.256958+0000","last_active":"2023-06-03T20:21:32.633680+0000","last_peered":"2023-06-03T20:21:32.633680+0000","last_clean":"2023-06-03T20:21:32.633680+0000","last_became_active":"2023-06-03T20:21:01.255452+0000","last_became_peered":"2023-06-03T20:21:01.255452+0000","last_unstale":"2023-06-03T20:21:32.633680+0000","last_undegraded":"2023-06-03T20:21:32.633680+0000","last_fullsized":"2023-06-03T20:21:32.633680+0000","mapping_epoch":235,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":236,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T02:05:32.589969+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":21,"reported_epoch":244,"state":"active+clean+remapped","last_fresh":"2023-06-03T20:21:31.316468+0000","last_change":"2023-06-03T20:21:02.266984+0000","last_active":"2023-06-03T20:21:31.316468+0000","last_peered":"2023-06-03T20:21:31.316468+0000","last_clean":"2023-06-03T20:21:31.316468+0000","last_became_active":"2023-06-03T20:21:02.266623+0000","last_became_peered":"2023-06-03T20:21:02.266623+0000","last_unstale":"2023-06-03T20:21:31.316468+0000","last_undegraded":"2023-06-03T20:21:31.316468+0000","last_fullsized":"2023-06-03T20:21:31.316468+0000","mapping_epoch":236,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":237,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T05:04:02.475656+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3],"acting":[1,3,7],"avail_no_missing":["1","3","7"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"13.b","version":"245'3361","reported_seq":3373,"reported_epoch":245,"state":"active+clean","last_fresh":"2023-06-03T20:22:24.106965+0000","last_change":"2023-06-03T20:21:30.617377+0000","last_active":"2023-06-03T20:22:24.106965+0000","last_peered":"2023-06-03T20:22:24.106965+0000","last_clean":"2023-06-03T20:22:24.106965+0000","last_became_active":"2023-06-03T20:21:30.616650+0000","last_became_peered":"2023-06-03T20:21:30.616650+0000","last_unstale":"2023-06-03T20:22:24.106965+0000","last_undegraded":"2023-06-03T20:22:24.106965+0000","last_fullsized":"2023-06-03T20:22:24.106965+0000","mapping_epoch":242,"log_start":"0'0","ondisk_log_start":"0'0","created":242,"last_epoch_clean":243,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T20:21:29.277668+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T20:21:29.277668+0000","last_clean_scrub_stamp":"2023-06-03T20:21:29.277668+0000","objects_scrubbed":0,"log_size":3361,"log_dups_size":0,"ondisk_log_size":3361,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T20:35:22.419539+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":220266496,"num_objects":3361,"num_object_clones":0,"num_object_copies":10083,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3361,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3361,"num_write_kb":215104,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,2],"acting":[3,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":20,"reported_epoch":244,"state":"active+clean","last_fresh":"2023-06-03T20:21:31.316940+0000","last_change":"2023-06-03T20:21:01.258182+0000","last_active":"2023-06-03T20:21:31.316940+0000","last_peered":"2023-06-03T20:21:31.316940+0000","last_clean":"2023-06-03T20:21:31.316940+0000","last_became_active":"2023-06-03T20:21:01.254049+0000","last_became_peered":"2023-06-03T20:21:01.254049+0000","last_unstale":"2023-06-03T20:21:31.316940+0000","last_undegraded":"2023-06-03T20:21:31.316940+0000","last_fullsized":"2023-06-03T20:21:31.316940+0000","mapping_epoch":235,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":236,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:32:20.692043+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:32:20.692043+0000","last_clean_scrub_stamp":"2023-06-03T19:32:20.692043+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T05:16:30.078473+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":129,"reported_epoch":244,"state":"active+clean","last_fresh":"2023-06-03T20:21:34.119708+0000","last_change":"2023-06-03T20:17:22.834746+0000","last_active":"2023-06-03T20:21:34.119708+0000","last_peered":"2023-06-03T20:21:34.119708+0000","last_clean":"2023-06-03T20:21:34.119708+0000","last_became_active":"2023-06-03T20:17:22.834448+0000","last_became_peered":"2023-06-03T20:17:22.834448+0000","last_unstale":"2023-06-03T20:21:34.119708+0000","last_undegraded":"2023-06-03T20:21:34.119708+0000","last_fullsized":"2023-06-03T20:21:34.119708+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T21:16:31.167063+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,2],"acting":[3,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":21,"reported_epoch":244,"state":"active+clean+remapped","last_fresh":"2023-06-03T20:21:31.315948+0000","last_change":"2023-06-03T20:21:02.281169+0000","last_active":"2023-06-03T20:21:31.315948+0000","last_peered":"2023-06-03T20:21:31.315948+0000","last_clean":"2023-06-03T20:21:31.315948+0000","last_became_active":"2023-06-03T20:21:02.280727+0000","last_became_peered":"2023-06-03T20:21:02.280727+0000","last_unstale":"2023-06-03T20:21:31.315948+0000","last_undegraded":"2023-06-03T20:21:31.315948+0000","last_fullsized":"2023-06-03T20:21:31.315948+0000","mapping_epoch":236,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":237,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T21:12:24.122859+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5],"acting":[1,5,0],"avail_no_missing":["1","0","5"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"13.d","version":"245'3376","reported_seq":3385,"reported_epoch":245,"state":"active+undersized+degraded","last_fresh":"2023-06-03T20:22:22.314482+0000","last_change":"2023-06-03T20:21:53.607757+0000","last_active":"2023-06-03T20:22:22.314482+0000","last_peered":"2023-06-03T20:22:22.314482+0000","last_clean":"2023-06-03T20:21:29.277668+0000","last_became_active":"2023-06-03T20:21:30.310864+0000","last_became_peered":"2023-06-03T20:21:30.310864+0000","last_unstale":"2023-06-03T20:22:22.314482+0000","last_undegraded":"2023-06-03T20:21:53.605211+0000","last_fullsized":"2023-06-03T20:21:30.308343+0000","mapping_epoch":242,"log_start":"0'0","ondisk_log_start":"0'0","created":242,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T20:21:29.277668+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T20:21:29.277668+0000","last_clean_scrub_stamp":"2023-06-03T20:21:29.277668+0000","objects_scrubbed":0,"log_size":3376,"log_dups_size":0,"ondisk_log_size":3376,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T23:17:35.458543+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":221249536,"num_objects":3376,"num_object_clones":0,"num_object_copies":10128,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3376,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3376,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3376,"num_write_kb":216064,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7],"acting":[1,7],"avail_no_missing":["1","7"],"object_location_counts":[{"shards":"1,7","objects":3376}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":20,"reported_epoch":244,"state":"active+clean","last_fresh":"2023-06-03T20:21:31.316470+0000","last_change":"2023-06-03T20:21:01.262334+0000","last_active":"2023-06-03T20:21:31.316470+0000","last_peered":"2023-06-03T20:21:31.316470+0000","last_clean":"2023-06-03T20:21:31.316470+0000","last_became_active":"2023-06-03T20:21:01.262013+0000","last_became_peered":"2023-06-03T20:21:01.262013+0000","last_unstale":"2023-06-03T20:21:31.316470+0000","last_undegraded":"2023-06-03T20:21:31.316470+0000","last_fullsized":"2023-06-03T20:21:31.316470+0000","mapping_epoch":235,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":236,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:32:20.692043+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:32:20.692043+0000","last_clean_scrub_stamp":"2023-06-03T19:32:20.692043+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T03:19:31.249552+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":140,"reported_epoch":245,"state":"active+clean+remapped","last_fresh":"2023-06-03T20:21:49.793912+0000","last_change":"2023-06-03T20:21:02.267322+0000","last_active":"2023-06-03T20:21:49.793912+0000","last_peered":"2023-06-03T20:21:49.793912+0000","last_clean":"2023-06-03T20:21:49.793912+0000","last_became_active":"2023-06-03T20:21:02.266003+0000","last_became_peered":"2023-06-03T20:21:02.266003+0000","last_unstale":"2023-06-03T20:21:49.793912+0000","last_undegraded":"2023-06-03T20:21:49.793912+0000","last_fullsized":"2023-06-03T20:21:49.793912+0000","mapping_epoch":236,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":237,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T02:48:12.423480+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1,5],"avail_no_missing":["0","1","5"],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":20,"reported_epoch":244,"state":"active+clean","last_fresh":"2023-06-03T20:21:31.317027+0000","last_change":"2023-06-03T20:21:01.259331+0000","last_active":"2023-06-03T20:21:31.317027+0000","last_peered":"2023-06-03T20:21:31.317027+0000","last_clean":"2023-06-03T20:21:31.317027+0000","last_became_active":"2023-06-03T20:21:01.258836+0000","last_became_peered":"2023-06-03T20:21:01.258836+0000","last_unstale":"2023-06-03T20:21:31.317027+0000","last_undegraded":"2023-06-03T20:21:31.317027+0000","last_fullsized":"2023-06-03T20:21:31.317027+0000","mapping_epoch":235,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":236,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T01:29:19.471600+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"13.e","version":"245'3457","reported_seq":3465,"reported_epoch":245,"state":"active+undersized+degraded","last_fresh":"2023-06-03T20:22:23.492036+0000","last_change":"2023-06-03T20:21:53.605782+0000","last_active":"2023-06-03T20:22:23.492036+0000","last_peered":"2023-06-03T20:22:23.492036+0000","last_clean":"2023-06-03T20:21:29.277668+0000","last_became_active":"2023-06-03T20:21:30.615924+0000","last_became_peered":"2023-06-03T20:21:30.615924+0000","last_unstale":"2023-06-03T20:22:23.492036+0000","last_undegraded":"2023-06-03T20:21:53.602605+0000","last_fullsized":"2023-06-03T20:21:30.303082+0000","mapping_epoch":242,"log_start":"0'0","ondisk_log_start":"0'0","created":242,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T20:21:29.277668+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T20:21:29.277668+0000","last_clean_scrub_stamp":"2023-06-03T20:21:29.277668+0000","objects_scrubbed":0,"log_size":3457,"log_dups_size":0,"ondisk_log_size":3457,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":226557952,"num_objects":3457,"num_object_clones":0,"num_object_copies":10371,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3457,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3457,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3457,"num_write_kb":221248,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5],"acting":[7,5],"avail_no_missing":["7","5"],"object_location_counts":[{"shards":"5,7","objects":3457}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":228,"reported_epoch":244,"state":"active+clean","last_fresh":"2023-06-03T20:21:32.633465+0000","last_change":"2023-06-03T19:56:54.504859+0000","last_active":"2023-06-03T20:21:32.633465+0000","last_peered":"2023-06-03T20:21:32.633465+0000","last_clean":"2023-06-03T20:21:32.633465+0000","last_became_active":"2023-06-03T19:56:54.504455+0000","last_became_peered":"2023-06-03T19:56:54.504455+0000","last_unstale":"2023-06-03T20:21:32.633465+0000","last_undegraded":"2023-06-03T20:21:32.633465+0000","last_fullsized":"2023-06-03T20:21:32.633465+0000","mapping_epoch":119,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:32:20.692043+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:32:20.692043+0000","last_clean_scrub_stamp":"2023-06-03T19:32:20.692043+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T06:25:45.522100+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":20,"reported_epoch":244,"state":"active+clean","last_fresh":"2023-06-03T20:21:31.316065+0000","last_change":"2023-06-03T20:21:01.251128+0000","last_active":"2023-06-03T20:21:31.316065+0000","last_peered":"2023-06-03T20:21:31.316065+0000","last_clean":"2023-06-03T20:21:31.316065+0000","last_became_active":"2023-06-03T20:21:01.250758+0000","last_became_peered":"2023-06-03T20:21:01.250758+0000","last_unstale":"2023-06-03T20:21:31.316065+0000","last_undegraded":"2023-06-03T20:21:31.316065+0000","last_fullsized":"2023-06-03T20:21:31.316065+0000","mapping_epoch":235,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":236,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T22:02:15.202291+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,0],"acting":[1,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":20,"reported_epoch":244,"state":"active+clean","last_fresh":"2023-06-03T20:21:31.316533+0000","last_change":"2023-06-03T20:21:01.261473+0000","last_active":"2023-06-03T20:21:31.316533+0000","last_peered":"2023-06-03T20:21:31.316533+0000","last_clean":"2023-06-03T20:21:31.316533+0000","last_became_active":"2023-06-03T20:21:01.261098+0000","last_became_peered":"2023-06-03T20:21:01.261098+0000","last_unstale":"2023-06-03T20:21:31.316533+0000","last_undegraded":"2023-06-03T20:21:31.316533+0000","last_fullsized":"2023-06-03T20:21:31.316533+0000","mapping_epoch":235,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":236,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T03:56:22.986549+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"13.c","version":"245'3424","reported_seq":3435,"reported_epoch":245,"state":"active+clean","last_fresh":"2023-06-03T20:22:22.305533+0000","last_change":"2023-06-03T20:21:30.617912+0000","last_active":"2023-06-03T20:22:22.305533+0000","last_peered":"2023-06-03T20:22:22.305533+0000","last_clean":"2023-06-03T20:22:22.305533+0000","last_became_active":"2023-06-03T20:21:30.617203+0000","last_became_peered":"2023-06-03T20:21:30.617203+0000","last_unstale":"2023-06-03T20:22:22.305533+0000","last_undegraded":"2023-06-03T20:22:22.305533+0000","last_fullsized":"2023-06-03T20:22:22.305533+0000","mapping_epoch":242,"log_start":"0'0","ondisk_log_start":"0'0","created":242,"last_epoch_clean":243,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T20:21:29.277668+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T20:21:29.277668+0000","last_clean_scrub_stamp":"2023-06-03T20:21:29.277668+0000","objects_scrubbed":0,"log_size":3424,"log_dups_size":0,"ondisk_log_size":3424,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T03:16:35.236413+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":224395264,"num_objects":3424,"num_object_clones":0,"num_object_copies":10272,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3424,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3424,"num_write_kb":219136,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,2],"acting":[1,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":113,"reported_epoch":244,"state":"active+clean","last_fresh":"2023-06-03T20:21:35.794449+0000","last_change":"2023-06-03T20:17:22.834467+0000","last_active":"2023-06-03T20:21:35.794449+0000","last_peered":"2023-06-03T20:21:35.794449+0000","last_clean":"2023-06-03T20:21:35.794449+0000","last_became_active":"2023-06-03T20:17:22.834147+0000","last_became_peered":"2023-06-03T20:17:22.834147+0000","last_unstale":"2023-06-03T20:21:35.794449+0000","last_undegraded":"2023-06-03T20:21:35.794449+0000","last_fullsized":"2023-06-03T20:21:35.794449+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:32:20.692043+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:32:20.692043+0000","last_clean_scrub_stamp":"2023-06-03T19:32:20.692043+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T04:36:29.567051+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,7],"acting":[2,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"72'139","reported_seq":494,"reported_epoch":244,"state":"active+clean","last_fresh":"2023-06-03T20:21:32.566747+0000","last_change":"2023-06-03T20:21:03.955881+0000","last_active":"2023-06-03T20:21:32.566747+0000","last_peered":"2023-06-03T20:21:32.566747+0000","last_clean":"2023-06-03T20:21:32.566747+0000","last_became_active":"2023-06-03T20:21:03.954963+0000","last_became_peered":"2023-06-03T20:21:03.954963+0000","last_unstale":"2023-06-03T20:21:32.566747+0000","last_undegraded":"2023-06-03T20:21:32.566747+0000","last_fullsized":"2023-06-03T20:21:32.566747+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:27:28.100808+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:27:28.100808+0000","last_clean_scrub_stamp":"2023-06-03T19:27:28.100808+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1180736,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,1],"acting":[7,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":211,"reported_epoch":245,"state":"active+clean+remapped","last_fresh":"2023-06-03T20:21:49.291492+0000","last_change":"2023-06-03T20:16:21.913608+0000","last_active":"2023-06-03T20:21:49.291492+0000","last_peered":"2023-06-03T20:21:49.291492+0000","last_clean":"2023-06-03T20:21:49.291492+0000","last_became_active":"2023-06-03T20:16:21.912814+0000","last_became_peered":"2023-06-03T20:16:21.912814+0000","last_unstale":"2023-06-03T20:21:49.291492+0000","last_undegraded":"2023-06-03T20:21:49.291492+0000","last_fullsized":"2023-06-03T20:21:49.291492+0000","mapping_epoch":213,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":214,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T06:29:01.991511+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5],"acting":[3,5,6],"avail_no_missing":["3","5","6"],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":221,"reported_epoch":244,"state":"active+clean+remapped","last_fresh":"2023-06-03T20:21:32.567416+0000","last_change":"2023-06-03T20:12:31.939716+0000","last_active":"2023-06-03T20:21:32.567416+0000","last_peered":"2023-06-03T20:21:32.567416+0000","last_clean":"2023-06-03T20:21:32.567416+0000","last_became_active":"2023-06-03T20:12:31.938876+0000","last_became_peered":"2023-06-03T20:12:31.938876+0000","last_unstale":"2023-06-03T20:21:32.567416+0000","last_undegraded":"2023-06-03T20:21:32.567416+0000","last_fullsized":"2023-06-03T20:21:32.567416+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0],"acting":[7,0,6],"avail_no_missing":["7","0","6"],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"13.f","version":"245'3475","reported_seq":3486,"reported_epoch":245,"state":"active+clean","last_fresh":"2023-06-03T20:22:23.494836+0000","last_change":"2023-06-03T20:21:30.617044+0000","last_active":"2023-06-03T20:22:23.494836+0000","last_peered":"2023-06-03T20:22:23.494836+0000","last_clean":"2023-06-03T20:22:23.494836+0000","last_became_active":"2023-06-03T20:21:30.615846+0000","last_became_peered":"2023-06-03T20:21:30.615846+0000","last_unstale":"2023-06-03T20:22:23.494836+0000","last_undegraded":"2023-06-03T20:22:23.494836+0000","last_fullsized":"2023-06-03T20:22:23.494836+0000","mapping_epoch":242,"log_start":"0'0","ondisk_log_start":"0'0","created":242,"last_epoch_clean":243,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T20:21:29.277668+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T20:21:29.277668+0000","last_clean_scrub_stamp":"2023-06-03T20:21:29.277668+0000","objects_scrubbed":0,"log_size":3475,"log_dups_size":0,"ondisk_log_size":3475,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":227737600,"num_objects":3475,"num_object_clones":0,"num_object_copies":10425,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3475,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3475,"num_write_kb":222400,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,5],"acting":[7,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":237,"reported_epoch":244,"state":"active+clean","last_fresh":"2023-06-03T20:21:32.566455+0000","last_change":"2023-06-03T20:21:01.258113+0000","last_active":"2023-06-03T20:21:32.566455+0000","last_peered":"2023-06-03T20:21:32.566455+0000","last_clean":"2023-06-03T20:21:32.566455+0000","last_became_active":"2023-06-03T20:21:01.257488+0000","last_became_peered":"2023-06-03T20:21:01.257488+0000","last_unstale":"2023-06-03T20:21:32.566455+0000","last_undegraded":"2023-06-03T20:21:32.566455+0000","last_fullsized":"2023-06-03T20:21:32.566455+0000","mapping_epoch":235,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":236,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:32:20.692043+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:32:20.692043+0000","last_clean_scrub_stamp":"2023-06-03T19:32:20.692043+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":230,"reported_epoch":244,"state":"active+undersized","last_fresh":"2023-06-03T20:21:32.217978+0000","last_change":"2023-06-03T20:11:54.017331+0000","last_active":"2023-06-03T20:21:32.217978+0000","last_peered":"2023-06-03T20:21:32.217978+0000","last_clean":"2023-06-03T20:11:43.977931+0000","last_became_active":"2023-06-03T20:11:54.017331+0000","last_became_peered":"2023-06-03T20:11:54.017331+0000","last_unstale":"2023-06-03T20:21:32.217978+0000","last_undegraded":"2023-06-03T20:21:32.217978+0000","last_fullsized":"2023-06-03T20:11:54.007149+0000","mapping_epoch":195,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T04:51:15.533105+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3],"acting":[0,3],"avail_no_missing":["0","3"],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":243,"reported_epoch":244,"state":"active+clean","last_fresh":"2023-06-03T20:21:47.017370+0000","last_change":"2023-06-03T20:21:01.250266+0000","last_active":"2023-06-03T20:21:47.017370+0000","last_peered":"2023-06-03T20:21:47.017370+0000","last_clean":"2023-06-03T20:21:47.017370+0000","last_became_active":"2023-06-03T20:21:01.249839+0000","last_became_peered":"2023-06-03T20:21:01.249839+0000","last_unstale":"2023-06-03T20:21:47.017370+0000","last_undegraded":"2023-06-03T20:21:47.017370+0000","last_fullsized":"2023-06-03T20:21:47.017370+0000","mapping_epoch":235,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":236,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T05:13:26.584607+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"13.a","version":"245'3213","reported_seq":3222,"reported_epoch":245,"state":"active+undersized+degraded","last_fresh":"2023-06-03T20:22:20.791931+0000","last_change":"2023-06-03T20:21:53.604366+0000","last_active":"2023-06-03T20:22:20.791931+0000","last_peered":"2023-06-03T20:22:20.791931+0000","last_clean":"2023-06-03T20:21:29.277668+0000","last_became_active":"2023-06-03T20:21:30.311019+0000","last_became_peered":"2023-06-03T20:21:30.311019+0000","last_unstale":"2023-06-03T20:22:20.791931+0000","last_undegraded":"2023-06-03T20:21:53.600353+0000","last_fullsized":"2023-06-03T20:21:30.296388+0000","mapping_epoch":242,"log_start":"0'0","ondisk_log_start":"0'0","created":242,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T20:21:29.277668+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T20:21:29.277668+0000","last_clean_scrub_stamp":"2023-06-03T20:21:29.277668+0000","objects_scrubbed":0,"log_size":3213,"log_dups_size":0,"ondisk_log_size":3213,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T07:14:31.234061+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":210567168,"num_objects":3213,"num_object_clones":0,"num_object_copies":9639,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3213,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3213,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3213,"num_write_kb":205632,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["2","1"],"object_location_counts":[{"shards":"1,2","objects":3213}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":237,"reported_epoch":244,"state":"active+undersized","last_fresh":"2023-06-03T20:21:32.567000+0000","last_change":"2023-06-03T20:11:54.012820+0000","last_active":"2023-06-03T20:21:32.567000+0000","last_peered":"2023-06-03T20:21:32.567000+0000","last_clean":"2023-06-03T20:11:43.973280+0000","last_became_active":"2023-06-03T20:11:54.012820+0000","last_became_peered":"2023-06-03T20:11:54.012820+0000","last_unstale":"2023-06-03T20:21:32.567000+0000","last_undegraded":"2023-06-03T20:21:32.567000+0000","last_fullsized":"2023-06-03T20:11:54.011295+0000","mapping_epoch":195,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:32:20.692043+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:32:20.692043+0000","last_clean_scrub_stamp":"2023-06-03T19:32:20.692043+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0],"acting":[7,0],"avail_no_missing":["7","0"],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":20,"reported_epoch":244,"state":"active+clean","last_fresh":"2023-06-03T20:21:31.316944+0000","last_change":"2023-06-03T20:21:01.252859+0000","last_active":"2023-06-03T20:21:31.316944+0000","last_peered":"2023-06-03T20:21:31.316944+0000","last_clean":"2023-06-03T20:21:31.316944+0000","last_became_active":"2023-06-03T20:21:01.251222+0000","last_became_peered":"2023-06-03T20:21:01.251222+0000","last_unstale":"2023-06-03T20:21:31.316944+0000","last_undegraded":"2023-06-03T20:21:31.316944+0000","last_fullsized":"2023-06-03T20:21:31.316944+0000","mapping_epoch":235,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":236,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T05:19:58.100226+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":195,"reported_epoch":244,"state":"active+clean","last_fresh":"2023-06-03T20:21:32.218964+0000","last_change":"2023-06-03T19:47:06.733532+0000","last_active":"2023-06-03T20:21:32.218964+0000","last_peered":"2023-06-03T20:21:32.218964+0000","last_clean":"2023-06-03T20:21:32.218964+0000","last_became_active":"2023-06-03T19:47:06.733131+0000","last_became_peered":"2023-06-03T19:47:06.733131+0000","last_unstale":"2023-06-03T20:21:32.218964+0000","last_undegraded":"2023-06-03T20:21:32.218964+0000","last_fullsized":"2023-06-03T20:21:32.218964+0000","mapping_epoch":86,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":87,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T23:46:03.788646+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"13.9","version":"245'3370","reported_seq":3382,"reported_epoch":245,"state":"active+clean","last_fresh":"2023-06-03T20:22:22.003115+0000","last_change":"2023-06-03T20:21:30.617754+0000","last_active":"2023-06-03T20:22:22.003115+0000","last_peered":"2023-06-03T20:22:22.003115+0000","last_clean":"2023-06-03T20:22:22.003115+0000","last_became_active":"2023-06-03T20:21:30.616735+0000","last_became_peered":"2023-06-03T20:21:30.616735+0000","last_unstale":"2023-06-03T20:22:22.003115+0000","last_undegraded":"2023-06-03T20:22:22.003115+0000","last_fullsized":"2023-06-03T20:22:22.003115+0000","mapping_epoch":242,"log_start":"0'0","ondisk_log_start":"0'0","created":242,"last_epoch_clean":243,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T20:21:29.277668+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T20:21:29.277668+0000","last_clean_scrub_stamp":"2023-06-03T20:21:29.277668+0000","objects_scrubbed":0,"log_size":3370,"log_dups_size":0,"ondisk_log_size":3370,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T01:05:03.356423+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":220856320,"num_objects":3370,"num_object_clones":0,"num_object_copies":10110,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3370,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3370,"num_write_kb":215680,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.7","version":"60'2","reported_seq":205,"reported_epoch":244,"state":"active+clean","last_fresh":"2023-06-03T20:21:32.635637+0000","last_change":"2023-06-03T19:56:54.510148+0000","last_active":"2023-06-03T20:21:32.635637+0000","last_peered":"2023-06-03T20:21:32.635637+0000","last_clean":"2023-06-03T20:21:32.635637+0000","last_became_active":"2023-06-03T19:56:54.509771+0000","last_became_peered":"2023-06-03T19:56:54.509771+0000","last_unstale":"2023-06-03T20:21:32.635637+0000","last_undegraded":"2023-06-03T20:21:32.635637+0000","last_fullsized":"2023-06-03T20:21:32.635637+0000","mapping_epoch":119,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T00:52:42.807432+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":19,"reported_epoch":244,"state":"active+clean","last_fresh":"2023-06-03T20:21:31.316729+0000","last_change":"2023-06-03T20:21:02.431550+0000","last_active":"2023-06-03T20:21:31.316729+0000","last_peered":"2023-06-03T20:21:31.316729+0000","last_clean":"2023-06-03T20:21:31.316729+0000","last_became_active":"2023-06-03T20:21:02.429444+0000","last_became_peered":"2023-06-03T20:21:02.429444+0000","last_unstale":"2023-06-03T20:21:31.316729+0000","last_undegraded":"2023-06-03T20:21:31.316729+0000","last_fullsized":"2023-06-03T20:21:31.316729+0000","mapping_epoch":236,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":237,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:32:20.692043+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:32:20.692043+0000","last_clean_scrub_stamp":"2023-06-03T19:32:20.692043+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T20:13:43.306762+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3],"acting":[1,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":203,"reported_epoch":244,"state":"active+clean","last_fresh":"2023-06-03T20:21:32.633441+0000","last_change":"2023-06-03T19:56:54.507670+0000","last_active":"2023-06-03T20:21:32.633441+0000","last_peered":"2023-06-03T20:21:32.633441+0000","last_clean":"2023-06-03T20:21:32.633441+0000","last_became_active":"2023-06-03T19:56:54.507293+0000","last_became_peered":"2023-06-03T19:56:54.507293+0000","last_unstale":"2023-06-03T20:21:32.633441+0000","last_undegraded":"2023-06-03T20:21:32.633441+0000","last_fullsized":"2023-06-03T20:21:32.633441+0000","mapping_epoch":119,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T01:49:19.966054+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"13.8","version":"245'3160","reported_seq":3170,"reported_epoch":245,"state":"active+clean","last_fresh":"2023-06-03T20:22:20.813848+0000","last_change":"2023-06-03T20:21:30.617209+0000","last_active":"2023-06-03T20:22:20.813848+0000","last_peered":"2023-06-03T20:22:20.813848+0000","last_clean":"2023-06-03T20:22:20.813848+0000","last_became_active":"2023-06-03T20:21:30.616334+0000","last_became_peered":"2023-06-03T20:21:30.616334+0000","last_unstale":"2023-06-03T20:22:20.813848+0000","last_undegraded":"2023-06-03T20:22:20.813848+0000","last_fullsized":"2023-06-03T20:22:20.813848+0000","mapping_epoch":242,"log_start":"0'0","ondisk_log_start":"0'0","created":242,"last_epoch_clean":243,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T20:21:29.277668+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T20:21:29.277668+0000","last_clean_scrub_stamp":"2023-06-03T20:21:29.277668+0000","objects_scrubbed":0,"log_size":3160,"log_dups_size":0,"ondisk_log_size":3160,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T03:41:48.527300+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":207093760,"num_objects":3160,"num_object_clones":0,"num_object_copies":9480,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3160,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3160,"num_write_kb":202240,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,1],"acting":[2,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":121,"reported_epoch":244,"state":"active+clean","last_fresh":"2023-06-03T20:21:32.567374+0000","last_change":"2023-06-03T20:16:20.907573+0000","last_active":"2023-06-03T20:21:32.567374+0000","last_peered":"2023-06-03T20:21:32.567374+0000","last_clean":"2023-06-03T20:21:32.567374+0000","last_became_active":"2023-06-03T20:16:20.907200+0000","last_became_peered":"2023-06-03T20:16:20.907200+0000","last_unstale":"2023-06-03T20:21:32.567374+0000","last_undegraded":"2023-06-03T20:21:32.567374+0000","last_fullsized":"2023-06-03T20:21:32.567374+0000","mapping_epoch":212,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":213,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:32:20.692043+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:32:20.692043+0000","last_clean_scrub_stamp":"2023-06-03T19:32:20.692043+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,3],"acting":[7,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.6","version":"60'1","reported_seq":261,"reported_epoch":244,"state":"active+clean+remapped","last_fresh":"2023-06-03T20:21:47.017217+0000","last_change":"2023-06-03T20:21:02.574463+0000","last_active":"2023-06-03T20:21:47.017217+0000","last_peered":"2023-06-03T20:21:47.017217+0000","last_clean":"2023-06-03T20:21:47.017217+0000","last_became_active":"2023-06-03T20:21:02.273146+0000","last_became_peered":"2023-06-03T20:21:02.273146+0000","last_unstale":"2023-06-03T20:21:47.017217+0000","last_undegraded":"2023-06-03T20:21:47.017217+0000","last_fullsized":"2023-06-03T20:21:47.017217+0000","mapping_epoch":236,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":237,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T22:08:23.963860+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":1,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1,7],"avail_no_missing":["0","1","7"],"object_location_counts":[{"shards":"0,1,7","objects":1}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":118,"reported_epoch":244,"state":"active+clean+remapped","last_fresh":"2023-06-03T20:21:37.014965+0000","last_change":"2023-06-03T20:17:23.833953+0000","last_active":"2023-06-03T20:21:37.014965+0000","last_peered":"2023-06-03T20:21:37.014965+0000","last_clean":"2023-06-03T20:21:37.014965+0000","last_became_active":"2023-06-03T20:17:23.833627+0000","last_became_peered":"2023-06-03T20:17:23.833627+0000","last_unstale":"2023-06-03T20:21:37.014965+0000","last_undegraded":"2023-06-03T20:21:37.014965+0000","last_fullsized":"2023-06-03T20:21:37.014965+0000","mapping_epoch":223,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":224,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T06:35:53.082804+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5],"acting":[0,5,6],"avail_no_missing":["0","5","6"],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1b","version":"60'1","reported_seq":217,"reported_epoch":245,"state":"active+undersized","last_fresh":"2023-06-03T20:21:49.793747+0000","last_change":"2023-06-03T20:11:54.016103+0000","last_active":"2023-06-03T20:21:49.793747+0000","last_peered":"2023-06-03T20:21:49.793747+0000","last_clean":"2023-06-03T20:11:43.979629+0000","last_became_active":"2023-06-03T20:11:54.016103+0000","last_became_peered":"2023-06-03T20:11:54.016103+0000","last_unstale":"2023-06-03T20:21:49.793747+0000","last_undegraded":"2023-06-03T20:21:49.793747+0000","last_fullsized":"2023-06-03T20:11:54.009298+0000","mapping_epoch":195,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T04:35:48.454086+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7],"acting":[0,7],"avail_no_missing":["0","7"],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":256,"reported_epoch":244,"state":"active+clean","last_fresh":"2023-06-03T20:21:32.635796+0000","last_change":"2023-06-03T20:21:01.257656+0000","last_active":"2023-06-03T20:21:32.635796+0000","last_peered":"2023-06-03T20:21:32.635796+0000","last_clean":"2023-06-03T20:21:32.635796+0000","last_became_active":"2023-06-03T20:21:01.257107+0000","last_became_peered":"2023-06-03T20:21:01.257107+0000","last_unstale":"2023-06-03T20:21:32.635796+0000","last_undegraded":"2023-06-03T20:21:32.635796+0000","last_fullsized":"2023-06-03T20:21:32.635796+0000","mapping_epoch":235,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":236,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T06:08:18.954539+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"60'2","reported_seq":23,"reported_epoch":244,"state":"active+clean","last_fresh":"2023-06-03T20:21:31.316828+0000","last_change":"2023-06-03T20:21:02.580723+0000","last_active":"2023-06-03T20:21:31.316828+0000","last_peered":"2023-06-03T20:21:31.316828+0000","last_clean":"2023-06-03T20:21:31.316828+0000","last_became_active":"2023-06-03T20:21:02.264936+0000","last_became_peered":"2023-06-03T20:21:02.264936+0000","last_unstale":"2023-06-03T20:21:31.316828+0000","last_undegraded":"2023-06-03T20:21:31.316828+0000","last_fullsized":"2023-06-03T20:21:31.316828+0000","mapping_epoch":236,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":237,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T03:05:15.388300+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5],"acting":[1,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":229,"reported_epoch":245,"state":"active+clean+remapped","last_fresh":"2023-06-03T20:21:49.291268+0000","last_change":"2023-06-03T20:21:02.281782+0000","last_active":"2023-06-03T20:21:49.291268+0000","last_peered":"2023-06-03T20:21:49.291268+0000","last_clean":"2023-06-03T20:21:49.291268+0000","last_became_active":"2023-06-03T20:21:02.279247+0000","last_became_peered":"2023-06-03T20:21:02.279247+0000","last_unstale":"2023-06-03T20:21:49.291268+0000","last_undegraded":"2023-06-03T20:21:49.291268+0000","last_fullsized":"2023-06-03T20:21:49.291268+0000","mapping_epoch":236,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":237,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T00:59:13.999557+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1],"acting":[3,1,2],"avail_no_missing":["3","1","2"],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1d","version":"60'2","reported_seq":23,"reported_epoch":244,"state":"active+clean","last_fresh":"2023-06-03T20:21:31.316420+0000","last_change":"2023-06-03T20:21:02.574995+0000","last_active":"2023-06-03T20:21:31.316420+0000","last_peered":"2023-06-03T20:21:31.316420+0000","last_clean":"2023-06-03T20:21:31.316420+0000","last_became_active":"2023-06-03T20:21:02.260413+0000","last_became_peered":"2023-06-03T20:21:02.260413+0000","last_unstale":"2023-06-03T20:21:31.316420+0000","last_undegraded":"2023-06-03T20:21:31.316420+0000","last_fullsized":"2023-06-03T20:21:31.316420+0000","mapping_epoch":236,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":237,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T20:54:51.829431+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5],"acting":[1,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":165,"reported_epoch":245,"state":"active+undersized","last_fresh":"2023-06-03T20:21:49.291463+0000","last_change":"2023-06-03T20:11:54.015791+0000","last_active":"2023-06-03T20:21:49.291463+0000","last_peered":"2023-06-03T20:21:49.291463+0000","last_clean":"2023-06-03T19:56:11.746795+0000","last_became_active":"2023-06-03T20:11:54.015791+0000","last_became_peered":"2023-06-03T20:11:54.015791+0000","last_unstale":"2023-06-03T20:21:49.291463+0000","last_undegraded":"2023-06-03T20:21:49.291463+0000","last_fullsized":"2023-06-03T20:11:54.007458+0000","mapping_epoch":195,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T19:54:52.259092+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0],"acting":[3,0],"avail_no_missing":["3","0"],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"60'1","reported_seq":250,"reported_epoch":244,"state":"active+clean","last_fresh":"2023-06-03T20:21:32.634930+0000","last_change":"2023-06-03T20:21:01.256517+0000","last_active":"2023-06-03T20:21:32.634930+0000","last_peered":"2023-06-03T20:21:32.634930+0000","last_clean":"2023-06-03T20:21:32.634930+0000","last_became_active":"2023-06-03T20:21:01.256116+0000","last_became_peered":"2023-06-03T20:21:01.256116+0000","last_unstale":"2023-06-03T20:21:32.634930+0000","last_undegraded":"2023-06-03T20:21:32.634930+0000","last_fullsized":"2023-06-03T20:21:32.634930+0000","mapping_epoch":235,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":236,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T06:47:35.144094+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,1],"acting":[3,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":243,"reported_epoch":244,"state":"active+clean","last_fresh":"2023-06-03T20:21:32.634411+0000","last_change":"2023-06-03T19:56:54.503416+0000","last_active":"2023-06-03T20:21:32.634411+0000","last_peered":"2023-06-03T20:21:32.634411+0000","last_clean":"2023-06-03T20:21:32.634411+0000","last_became_active":"2023-06-03T19:56:54.503018+0000","last_became_peered":"2023-06-03T19:56:54.503018+0000","last_unstale":"2023-06-03T20:21:32.634411+0000","last_undegraded":"2023-06-03T20:21:32.634411+0000","last_fullsized":"2023-06-03T20:21:32.634411+0000","mapping_epoch":119,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T20:14:00.330616+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"60'4","reported_seq":213,"reported_epoch":244,"state":"active+clean","last_fresh":"2023-06-03T20:21:32.218728+0000","last_change":"2023-06-03T19:56:11.748116+0000","last_active":"2023-06-03T20:21:32.218728+0000","last_peered":"2023-06-03T20:21:32.218728+0000","last_clean":"2023-06-03T20:21:32.218728+0000","last_became_active":"2023-06-03T19:56:11.747636+0000","last_became_peered":"2023-06-03T19:56:11.747636+0000","last_unstale":"2023-06-03T20:21:32.218728+0000","last_undegraded":"2023-06-03T20:21:32.218728+0000","last_fullsized":"2023-06-03T20:21:32.218728+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":115,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T06:57:48.052055+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":19,"reported_epoch":244,"state":"active+clean","last_fresh":"2023-06-03T20:21:31.315907+0000","last_change":"2023-06-03T20:21:02.263339+0000","last_active":"2023-06-03T20:21:31.315907+0000","last_peered":"2023-06-03T20:21:31.315907+0000","last_clean":"2023-06-03T20:21:31.315907+0000","last_became_active":"2023-06-03T20:21:02.262926+0000","last_became_peered":"2023-06-03T20:21:02.262926+0000","last_unstale":"2023-06-03T20:21:31.315907+0000","last_undegraded":"2023-06-03T20:21:31.315907+0000","last_fullsized":"2023-06-03T20:21:31.315907+0000","mapping_epoch":236,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":237,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T06:04:23.525380+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":13,"num_pg":16,"stat_sum":{"num_bytes":3511812096,"num_objects":53586,"num_object_clones":0,"num_object_copies":160758,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":30183,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":53586,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":53586,"num_write_kb":3429504,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":8612478976,"data_stored":8612478976,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":53586,"ondisk_log_size":53586,"up":39,"acting":39,"num_store_stats":6},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":80,"acting":95,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5486,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":6,"num_objects_misplaced":6,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":62,"num_write_kb":30,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":17,"num_bytes_recovered":4786,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":110592,"data_stored":21174,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":30319,"internal_metadata":0},"log_size":56,"ondisk_log_size":56,"up":78,"acting":89,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1180736,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2969600,"data_stored":2951840,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":3,"num_store_stats":5},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":22,"acting":23,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":48,"seq":206158430824,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2159612,"kb_used_data":1311092,"kb_used_omap":0,"kb_used_meta":848512,"kb_avail":91585540,"statfs":{"total":95995035648,"available":93783592960,"internally_reserved":0,"allocated":1342558208,"data_stored":1341929089,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":868876288},"hb_peers":[0,1,2,3,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 3 20:22:12 2023","interfaces":[{"interface":"back","average":{"1min":0.77700000000000002,"5min":1.9019999999999999,"15min":1.871},"min":{"1min":0.373,"5min":0.26500000000000001,"15min":0.25800000000000001},"max":{"1min":1.2370000000000001,"5min":73.810000000000002,"15min":73.810000000000002},"last":1.597},{"interface":"front","average":{"1min":1.0920000000000001,"5min":1.8859999999999999,"15min":1.617},"min":{"1min":0.34100000000000003,"5min":0.30099999999999999,"15min":0.27800000000000002},"max":{"1min":4.8300000000000001,"5min":72.878,"15min":72.878},"last":3.024}]},{"osd":1,"last update":"Sat Jun 3 20:21:58 2023","interfaces":[{"interface":"back","average":{"1min":0.89300000000000002,"5min":0.73099999999999998,"15min":1.099},"min":{"1min":0.48999999999999999,"5min":0.32100000000000001,"15min":0.30099999999999999},"max":{"1min":1.2589999999999999,"5min":3.907,"15min":11.4},"last":1.5800000000000001},{"interface":"front","average":{"1min":1.8320000000000001,"5min":0.93300000000000005,"15min":1.1359999999999999},"min":{"1min":0.66800000000000004,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":13.5,"5min":13.5,"15min":13.5},"last":0.31900000000000001}]},{"osd":2,"last update":"Sat Jun 3 20:21:58 2023","interfaces":[{"interface":"back","average":{"1min":0.91200000000000003,"5min":2.4969999999999999,"15min":3.9470000000000001},"min":{"1min":0.61499999999999999,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":1.752,"5min":72.831000000000003,"15min":286.83600000000001},"last":1.411},{"interface":"front","average":{"1min":1.071,"5min":2.927,"15min":4.016},"min":{"1min":0.46800000000000003,"5min":0.315,"15min":0.28699999999999998},"max":{"1min":2.5059999999999998,"5min":72.787000000000006,"15min":287.32900000000001},"last":0.35699999999999998}]},{"osd":3,"last update":"Sat Jun 3 20:21:51 2023","interfaces":[{"interface":"back","average":{"1min":1.655,"5min":2.528,"15min":3.3999999999999999},"min":{"1min":0.73399999999999999,"5min":0.371,"15min":0.30299999999999999},"max":{"1min":11.992000000000001,"5min":74.039000000000001,"15min":265.40499999999997},"last":1.5549999999999999},{"interface":"front","average":{"1min":0.98799999999999999,"5min":2.7389999999999999,"15min":3.3719999999999999},"min":{"1min":0.61399999999999999,"5min":0.41199999999999998,"15min":0.31900000000000001},"max":{"1min":1.3200000000000001,"5min":73.001999999999995,"15min":265.43599999999998},"last":0.79400000000000004}]},{"osd":4,"last update":"Sat Jun 3 20:10:54 2023","interfaces":[{"interface":"back","average":{"1min":1.417,"5min":1.8879999999999999,"15min":1.3260000000000001},"min":{"1min":0.33900000000000002,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":7.1189999999999998,"5min":21.888999999999999,"15min":21.888999999999999},"last":0.40799999999999997},{"interface":"front","average":{"1min":1.484,"5min":2.1739999999999999,"15min":1.671},"min":{"1min":0.60799999999999998,"5min":0.443,"15min":0.38800000000000001},"max":{"1min":4.8600000000000003,"5min":10.507999999999999,"15min":12.917},"last":0.48899999999999999}]},{"osd":5,"last update":"Sat Jun 3 20:21:51 2023","interfaces":[{"interface":"back","average":{"1min":0.97599999999999998,"5min":0.71299999999999997,"15min":1.325},"min":{"1min":0.73599999999999999,"5min":0.38700000000000001,"15min":0.28899999999999998},"max":{"1min":1.296,"5min":2.1829999999999998,"15min":11.443},"last":1.4810000000000001},{"interface":"front","average":{"1min":1.0589999999999999,"5min":0.82099999999999995,"15min":1.393},"min":{"1min":0.67400000000000004,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":1.4019999999999999,"5min":6.6920000000000002,"15min":9.1099999999999994},"last":0.42499999999999999}]},{"osd":6,"last update":"Sat Jun 3 20:21:51 2023","interfaces":[{"interface":"back","average":{"1min":1.093,"5min":0.84999999999999998,"15min":1.6060000000000001},"min":{"1min":0.74099999999999999,"5min":0.39800000000000002,"15min":0.34799999999999998},"max":{"1min":1.4730000000000001,"5min":6.6559999999999997,"15min":11.256},"last":0.39800000000000002},{"interface":"front","average":{"1min":0.96999999999999997,"5min":0.78200000000000003,"15min":1.591},"min":{"1min":0.64300000000000002,"5min":0.372,"15min":0.26900000000000002},"max":{"1min":1.167,"5min":2.3660000000000001,"15min":10.894},"last":0.46600000000000003}]}]},{"osd":6,"up_from":0,"seq":180388627057,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":1,"up_from":103,"seq":442381631855,"num_pgs":47,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1610164,"kb_used_data":1280236,"kb_used_omap":4,"kb_used_meta":329915,"kb_avail":92134988,"statfs":{"total":95995035648,"available":94346227712,"internally_reserved":0,"allocated":1310961664,"data_stored":1309992765,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":337833870},"hb_peers":[0,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 3 20:22:09 2023","interfaces":[{"interface":"back","average":{"1min":0.82799999999999996,"5min":4.0179999999999998,"15min":2.371},"min":{"1min":0.30399999999999999,"5min":0.20100000000000001,"15min":0.193},"max":{"1min":2.665,"5min":119.94499999999999,"15min":119.94499999999999},"last":0.17100000000000001},{"interface":"front","average":{"1min":0.82499999999999996,"5min":4.0940000000000003,"15min":2.4590000000000001},"min":{"1min":0.24399999999999999,"5min":0.22500000000000001,"15min":0.20799999999999999},"max":{"1min":4.4740000000000002,"5min":120.04300000000001,"15min":120.04300000000001},"last":0.26900000000000002}]},{"osd":2,"last update":"Sat Jun 3 20:22:09 2023","interfaces":[{"interface":"back","average":{"1min":0.88200000000000001,"5min":3.7360000000000002,"15min":2.5950000000000002},"min":{"1min":0.27900000000000003,"5min":0.247,"15min":0.23499999999999999},"max":{"1min":2.4409999999999998,"5min":77.719999999999999,"15min":219.72300000000001},"last":2.6709999999999998},{"interface":"front","average":{"1min":0.91800000000000004,"5min":3.6819999999999999,"15min":2.5960000000000001},"min":{"1min":0.26400000000000001,"5min":0.22700000000000001,"15min":0.224},"max":{"1min":3.5169999999999999,"5min":78.007999999999996,"15min":219.82900000000001},"last":2.597}]},{"osd":3,"last update":"Sat Jun 3 20:21:39 2023","interfaces":[{"interface":"back","average":{"1min":3.27,"5min":2.2959999999999998,"15min":1.8400000000000001},"min":{"1min":0.28599999999999998,"5min":0.23599999999999999,"15min":0.23599999999999999},"max":{"1min":53.685000000000002,"5min":96.864999999999995,"15min":96.864999999999995},"last":0.51400000000000001},{"interface":"front","average":{"1min":3.016,"5min":2.4239999999999999,"15min":1.8360000000000001},"min":{"1min":0.432,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":49.18,"5min":96.679000000000002,"15min":96.679000000000002},"last":0.55900000000000005}]},{"osd":4,"last update":"Sat Jun 3 20:10:54 2023","interfaces":[{"interface":"back","average":{"1min":0.97099999999999997,"5min":1.6100000000000001,"15min":1.4330000000000001},"min":{"1min":0.34100000000000003,"5min":0.34100000000000003,"15min":0.34100000000000003},"max":{"1min":2.9430000000000001,"5min":9.2929999999999993,"15min":9.9589999999999996},"last":1.319},{"interface":"front","average":{"1min":1.032,"5min":1.4299999999999999,"15min":1.6759999999999999},"min":{"1min":0.40500000000000003,"5min":0.376,"15min":0.32700000000000001},"max":{"1min":3.4630000000000001,"5min":9.0749999999999993,"15min":120.084},"last":0.78100000000000003}]},{"osd":5,"last update":"Sat Jun 3 20:22:09 2023","interfaces":[{"interface":"back","average":{"1min":1.1220000000000001,"5min":0.73899999999999999,"15min":0.80300000000000005},"min":{"1min":0.57099999999999995,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":2.8109999999999999,"5min":3.7349999999999999,"15min":4.3840000000000003},"last":0.30599999999999999},{"interface":"front","average":{"1min":0.98799999999999999,"5min":0.80700000000000005,"15min":0.88100000000000001},"min":{"1min":0.40200000000000002,"5min":0.372,"15min":0.35999999999999999},"max":{"1min":2.2519999999999998,"5min":5.9779999999999998,"15min":5.9779999999999998},"last":0.65800000000000003}]},{"osd":6,"last update":"Sat Jun 3 20:22:09 2023","interfaces":[{"interface":"back","average":{"1min":0.95499999999999996,"5min":0.81999999999999995,"15min":1.163},"min":{"1min":0.41999999999999998,"5min":0.41599999999999998,"15min":0.33900000000000002},"max":{"1min":2.2709999999999999,"5min":6.0140000000000002,"15min":8.6099999999999994},"last":0.70099999999999996},{"interface":"front","average":{"1min":1.3160000000000001,"5min":0.88600000000000001,"15min":1.175},"min":{"1min":0.46000000000000002,"5min":0.40699999999999997,"15min":0.38400000000000001},"max":{"1min":4.8280000000000003,"5min":4.8280000000000003,"15min":11.106},"last":0.79700000000000004}]},{"osd":7,"last update":"Sat Jun 3 20:22:09 2023","interfaces":[{"interface":"back","average":{"1min":1.3220000000000001,"5min":0.90000000000000002,"15min":1.232},"min":{"1min":0.42699999999999999,"5min":0.372,"15min":0.372},"max":{"1min":4.8620000000000001,"5min":4.8620000000000001,"15min":13.298999999999999},"last":0.64100000000000001},{"interface":"front","average":{"1min":1.089,"5min":0.77300000000000002,"15min":1.044},"min":{"1min":0.624,"5min":0.34000000000000002,"15min":0.34000000000000002},"max":{"1min":2.8450000000000002,"5min":3.7719999999999998,"15min":5.2880000000000003},"last":0.378}]}]},{"osd":0,"up_from":86,"seq":369367187881,"num_pgs":42,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1275616,"kb_used_data":627224,"kb_used_omap":5,"kb_used_meta":648378,"kb_avail":92469536,"statfs":{"total":95995035648,"available":94688804864,"internally_reserved":0,"allocated":642277376,"data_stored":641268121,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":663939706},"hb_peers":[1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Jun 3 20:21:53 2023","interfaces":[{"interface":"back","average":{"1min":0.64000000000000001,"5min":0.82399999999999995,"15min":0.89100000000000001},"min":{"1min":0.33700000000000002,"5min":0.22600000000000001,"15min":0.20799999999999999},"max":{"1min":1.9370000000000001,"5min":6.1890000000000001,"15min":9.3040000000000003},"last":0.51600000000000001},{"interface":"front","average":{"1min":0.55100000000000005,"5min":1.0389999999999999,"15min":1.2070000000000001},"min":{"1min":0.33500000000000002,"5min":0.20899999999999999,"15min":0.20899999999999999},"max":{"1min":0.82999999999999996,"5min":6.7750000000000004,"15min":11.736000000000001},"last":0.56999999999999995}]},{"osd":2,"last update":"Sat Jun 3 20:21:53 2023","interfaces":[{"interface":"back","average":{"1min":0.66700000000000004,"5min":1.401,"15min":1.302},"min":{"1min":0.38800000000000001,"5min":0.218,"15min":0.193},"max":{"1min":2.113,"5min":11.811,"15min":11.901},"last":2.653},{"interface":"front","average":{"1min":0.70499999999999996,"5min":1.7410000000000001,"15min":1.5249999999999999},"min":{"1min":0.40699999999999997,"5min":0.30199999999999999,"15min":0.255},"max":{"1min":1.819,"5min":11.988,"15min":17.315999999999999},"last":0.434}]},{"osd":3,"last update":"Sat Jun 3 20:21:35 2023","interfaces":[{"interface":"back","average":{"1min":1.3979999999999999,"5min":2.27,"15min":1.9950000000000001},"min":{"1min":0.38,"5min":0.248,"15min":0.248},"max":{"1min":12.757,"5min":55.505000000000003,"15min":57.790999999999997},"last":2.4470000000000001},{"interface":"front","average":{"1min":1.6859999999999999,"5min":2.0680000000000001,"15min":2.0470000000000002},"min":{"1min":0.434,"5min":0.33400000000000002,"15min":0.28100000000000003},"max":{"1min":14.613,"5min":55.710999999999999,"15min":58.646000000000001},"last":0.49199999999999999}]},{"osd":4,"last update":"Sat Jun 3 20:10:59 2023","interfaces":[{"interface":"back","average":{"1min":1.9370000000000001,"5min":1.7110000000000001,"15min":1.3400000000000001},"min":{"1min":0.45200000000000001,"5min":0.318,"15min":0.318},"max":{"1min":10.025,"5min":10.025,"15min":10.025},"last":0.497},{"interface":"front","average":{"1min":1.673,"5min":1.8160000000000001,"15min":1.429},"min":{"1min":0.56699999999999995,"5min":0.375,"15min":0.375},"max":{"1min":4.6059999999999999,"5min":13.058999999999999,"15min":13.058999999999999},"last":0.75900000000000001}]},{"osd":5,"last update":"Sat Jun 3 20:22:15 2023","interfaces":[{"interface":"back","average":{"1min":0.90400000000000003,"5min":1.3069999999999999,"15min":1.712},"min":{"1min":0.42499999999999999,"5min":0.316,"15min":0.316},"max":{"1min":2.6509999999999998,"5min":6.9240000000000004,"15min":9.3829999999999991},"last":0.54800000000000004},{"interface":"front","average":{"1min":1.095,"5min":1.2909999999999999,"15min":1.427},"min":{"1min":0.61599999999999999,"5min":0.39000000000000001,"15min":0.32300000000000001},"max":{"1min":2.8639999999999999,"5min":6.891,"15min":10.159000000000001},"last":2.3769999999999998}]},{"osd":6,"last update":"Sat Jun 3 20:22:15 2023","interfaces":[{"interface":"back","average":{"1min":1.036,"5min":1.3260000000000001,"15min":1.46},"min":{"1min":0.66700000000000004,"5min":0.36299999999999999,"15min":0.35399999999999998},"max":{"1min":2.0670000000000002,"5min":6.9219999999999997,"15min":10.192},"last":2.419},{"interface":"front","average":{"1min":1.0229999999999999,"5min":1.236,"15min":1.4870000000000001},"min":{"1min":0.41599999999999998,"5min":0.41599999999999998,"15min":0.38300000000000001},"max":{"1min":1.8640000000000001,"5min":6.2999999999999998,"15min":13.117000000000001},"last":0.46899999999999997}]},{"osd":7,"last update":"Sat Jun 3 20:22:15 2023","interfaces":[{"interface":"back","average":{"1min":1.127,"5min":1.552,"15min":1.786},"min":{"1min":0.35499999999999998,"5min":0.35499999999999998,"15min":0.35499999999999998},"max":{"1min":3.6080000000000001,"5min":11.755000000000001,"15min":13.470000000000001},"last":2.8980000000000001},{"interface":"front","average":{"1min":1.2330000000000001,"5min":1.4199999999999999,"15min":1.702},"min":{"1min":0.40000000000000002,"5min":0.40000000000000002,"15min":0.40000000000000002},"max":{"1min":4.6139999999999999,"5min":6.4139999999999997,"15min":12.41},"last":1.0249999999999999}]}]},{"osd":2,"up_from":114,"seq":489626272060,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2272584,"kb_used_data":1839168,"kb_used_omap":7,"kb_used_meta":433400,"kb_avail":91472568,"statfs":{"total":95995035648,"available":93667909632,"internally_reserved":0,"allocated":1883308032,"data_stored":1882383489,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8111,"internal_metadata":443801681},"hb_peers":[0,1,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 3 20:21:52 2023","interfaces":[{"interface":"back","average":{"1min":0.58999999999999997,"5min":2.3090000000000002,"15min":3.1890000000000001},"min":{"1min":0.29499999999999998,"5min":0.20300000000000001,"15min":0.17799999999999999},"max":{"1min":0.98599999999999999,"5min":57.470999999999997,"15min":273.51100000000002},"last":0.251},{"interface":"front","average":{"1min":0.60599999999999998,"5min":1.9470000000000001,"15min":3.1680000000000001},"min":{"1min":0.32200000000000001,"5min":0.20399999999999999,"15min":0.20399999999999999},"max":{"1min":0.86099999999999999,"5min":57.765999999999998,"15min":273.67399999999998},"last":0.32500000000000001}]},{"osd":1,"last update":"Sat Jun 3 20:21:52 2023","interfaces":[{"interface":"back","average":{"1min":0.56200000000000006,"5min":0.88800000000000001,"15min":2.1219999999999999},"min":{"1min":0.377,"5min":0.247,"15min":0.23300000000000001},"max":{"1min":0.81999999999999995,"5min":7.1260000000000003,"15min":273.46499999999997},"last":0.64300000000000002},{"interface":"front","average":{"1min":0.57199999999999995,"5min":1.4810000000000001,"15min":2.4199999999999999},"min":{"1min":0.33300000000000002,"5min":0.25700000000000001,"15min":0.246},"max":{"1min":0.92500000000000004,"5min":13.977,"15min":274.38400000000001},"last":1.042}]},{"osd":3,"last update":"Sat Jun 3 20:21:46 2023","interfaces":[{"interface":"back","average":{"1min":0.82799999999999996,"5min":2.617,"15min":3.2490000000000001},"min":{"1min":0.36599999999999999,"5min":0.26800000000000002,"15min":0.24399999999999999},"max":{"1min":4.6769999999999996,"5min":74.703000000000003,"15min":273.71899999999999},"last":0.29599999999999999},{"interface":"front","average":{"1min":0.58399999999999996,"5min":2.548,"15min":3.339},"min":{"1min":0.23000000000000001,"5min":0.23000000000000001,"15min":0.23000000000000001},"max":{"1min":0.85399999999999998,"5min":14.298,"15min":274.21100000000001},"last":0.84799999999999998}]},{"osd":4,"last update":"Sat Jun 3 20:10:59 2023","interfaces":[{"interface":"back","average":{"1min":16.218,"5min":4.9749999999999996,"15min":2.2519999999999998},"min":{"1min":0.48699999999999999,"5min":0.32800000000000001,"15min":0.32700000000000001},"max":{"1min":273.75700000000001,"5min":273.75700000000001,"15min":273.75700000000001},"last":1.6679999999999999},{"interface":"front","average":{"1min":15.512,"5min":4.7480000000000002,"15min":2.1850000000000001},"min":{"1min":0.36199999999999999,"5min":0.35699999999999998,"15min":0.32600000000000001},"max":{"1min":274.27300000000002,"5min":274.27300000000002,"15min":274.27300000000002},"last":1.585}]},{"osd":5,"last update":"Sat Jun 3 20:21:52 2023","interfaces":[{"interface":"back","average":{"1min":0.89000000000000001,"5min":1.448,"15min":2.258},"min":{"1min":0.49399999999999999,"5min":0.32100000000000001,"15min":0.317},"max":{"1min":1.2669999999999999,"5min":14.361000000000001,"15min":273.78800000000001},"last":0.42799999999999999},{"interface":"front","average":{"1min":0.874,"5min":1.111,"15min":2.2290000000000001},"min":{"1min":0.51000000000000001,"5min":0.32600000000000001,"15min":0.32600000000000001},"max":{"1min":1.1000000000000001,"5min":7.2370000000000001,"15min":274.15800000000002},"last":0.378}]},{"osd":6,"last update":"Sat Jun 3 20:21:52 2023","interfaces":[{"interface":"back","average":{"1min":0.93600000000000005,"5min":1.5289999999999999,"15min":2.4940000000000002},"min":{"1min":0.64000000000000001,"5min":0.40200000000000002,"15min":0.34200000000000003},"max":{"1min":1.4299999999999999,"5min":14.331,"15min":274.03899999999999},"last":0.44700000000000001},{"interface":"front","average":{"1min":0.93500000000000005,"5min":1.2549999999999999,"15min":2.456},"min":{"1min":0.58999999999999997,"5min":0.35599999999999998,"15min":0.35099999999999998},"max":{"1min":1.319,"5min":7.4630000000000001,"15min":274.06299999999999},"last":0.40500000000000003}]},{"osd":7,"last update":"Sat Jun 3 20:21:52 2023","interfaces":[{"interface":"back","average":{"1min":1.0309999999999999,"5min":1.5820000000000001,"15min":2.5329999999999999},"min":{"1min":0.72599999999999998,"5min":0.42899999999999999,"15min":0.317},"max":{"1min":1.369,"5min":14.243,"15min":274.54000000000002},"last":0.59899999999999998},{"interface":"front","average":{"1min":0.96199999999999997,"5min":1.254,"15min":2.4780000000000002},"min":{"1min":0.56000000000000005,"5min":0.38600000000000001,"15min":0.38600000000000001},"max":{"1min":1.2629999999999999,"5min":7.2690000000000001,"15min":274.12700000000001},"last":0.61199999999999999}]}]},{"osd":3,"up_from":119,"seq":511101108532,"num_pgs":44,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1353596,"kb_used_data":866612,"kb_used_omap":2,"kb_used_meta":486973,"kb_avail":92391556,"statfs":{"total":95995035648,"available":94608953344,"internally_reserved":0,"allocated":887410688,"data_stored":886431256,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":498660405},"hb_peers":[0,1,2,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 3 20:21:45 2023","interfaces":[{"interface":"back","average":{"1min":0.53700000000000003,"5min":3.6080000000000001,"15min":2.5009999999999999},"min":{"1min":0.314,"5min":0.214,"15min":0.192},"max":{"1min":0.75700000000000001,"5min":97.164000000000001,"15min":97.164000000000001},"last":0.69799999999999995},{"interface":"front","average":{"1min":0.63900000000000001,"5min":3.7109999999999999,"15min":2.6619999999999999},"min":{"1min":0.25800000000000001,"5min":0.183,"15min":0.183},"max":{"1min":1.1739999999999999,"5min":95.816000000000003,"15min":95.816000000000003},"last":1.054}]},{"osd":1,"last update":"Sat Jun 3 20:21:45 2023","interfaces":[{"interface":"back","average":{"1min":0.71799999999999997,"5min":1.3660000000000001,"15min":1.1279999999999999},"min":{"1min":0.35199999999999998,"5min":0.22500000000000001,"15min":0.214},"max":{"1min":2.6230000000000002,"5min":9.4009999999999998,"15min":10.749000000000001},"last":3.2170000000000001},{"interface":"front","average":{"1min":0.61699999999999999,"5min":1.2170000000000001,"15min":1.0289999999999999},"min":{"1min":0.35699999999999998,"5min":0.23100000000000001,"15min":0.20699999999999999},"max":{"1min":0.97799999999999998,"5min":10.304,"15min":10.304},"last":3.1869999999999998}]},{"osd":2,"last update":"Sat Jun 3 20:21:45 2023","interfaces":[{"interface":"back","average":{"1min":0.59599999999999997,"5min":2.6989999999999998,"15min":2.097},"min":{"1min":0.32200000000000001,"5min":0.25700000000000001,"15min":0.217},"max":{"1min":0.86199999999999999,"5min":85.614000000000004,"15min":85.614000000000004},"last":1.1240000000000001},{"interface":"front","average":{"1min":0.81499999999999995,"5min":2.8119999999999998,"15min":2.0779999999999998},"min":{"1min":0.27200000000000002,"5min":0.25,"15min":0.214},"max":{"1min":2.5899999999999999,"5min":77.927000000000007,"15min":77.927000000000007},"last":2.347}]},{"osd":4,"last update":"Sat Jun 3 20:10:53 2023","interfaces":[{"interface":"back","average":{"1min":2.0379999999999998,"5min":1.98,"15min":1.413},"min":{"1min":0.42399999999999999,"5min":0.38200000000000001,"15min":0.38200000000000001},"max":{"1min":9.8360000000000003,"5min":12.757,"15min":12.757},"last":1.968},{"interface":"front","average":{"1min":2.1749999999999998,"5min":1.744,"15min":1.333},"min":{"1min":0.39000000000000001,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":10.122,"5min":10.122,"15min":10.122},"last":0.29099999999999998}]},{"osd":5,"last update":"Sat Jun 3 20:21:45 2023","interfaces":[{"interface":"back","average":{"1min":1.131,"5min":1.53,"15min":1.4319999999999999},"min":{"1min":0.50800000000000001,"5min":0.313,"15min":0.313},"max":{"1min":6.859,"5min":10.275,"15min":10.275},"last":0.73699999999999999},{"interface":"front","average":{"1min":0.82799999999999996,"5min":1.468,"15min":1.327},"min":{"1min":0.55600000000000005,"5min":0.32900000000000001,"15min":0.314},"max":{"1min":1.1879999999999999,"5min":9.1059999999999999,"15min":12.56},"last":1.0780000000000001}]},{"osd":6,"last update":"Sat Jun 3 20:21:45 2023","interfaces":[{"interface":"back","average":{"1min":0.91500000000000004,"5min":1.5249999999999999,"15min":1.6319999999999999},"min":{"1min":0.503,"5min":0.34799999999999998,"15min":0.33500000000000002},"max":{"1min":1.351,"5min":9.0719999999999992,"15min":13.718},"last":1.095},{"interface":"front","average":{"1min":1.135,"5min":1.804,"15min":1.883},"min":{"1min":0.68100000000000005,"5min":0.39300000000000002,"15min":0.32500000000000001},"max":{"1min":2.871,"5min":8.7279999999999998,"15min":13.734999999999999},"last":0.57099999999999995}]},{"osd":7,"last update":"Sat Jun 3 20:21:45 2023","interfaces":[{"interface":"back","average":{"1min":1.0900000000000001,"5min":1.8180000000000001,"15min":1.655},"min":{"1min":0.49299999999999999,"5min":0.36599999999999999,"15min":0.35599999999999998},"max":{"1min":2.9159999999999999,"5min":8.7149999999999999,"15min":10.858000000000001},"last":0.60599999999999998},{"interface":"front","average":{"1min":1.21,"5min":1.623,"15min":1.806},"min":{"1min":0.56200000000000006,"5min":0.34899999999999998,"15min":0.34899999999999998},"max":{"1min":6.8179999999999996,"5min":13.576000000000001,"15min":18.431999999999999},"last":0.76700000000000002}]}]},{"osd":4,"up_from":124,"seq":532575944879,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1005320,"kb_used_data":818944,"kb_used_omap":10,"kb_used_meta":186357,"kb_avail":92739832,"statfs":{"total":95995035648,"available":94965587968,"internally_reserved":0,"allocated":838598656,"data_stored":837828790,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":10652,"internal_metadata":190830180},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 3 20:11:03 2023","interfaces":[{"interface":"back","average":{"1min":1.002,"5min":4.8399999999999999,"15min":2.468},"min":{"1min":0.58599999999999997,"5min":0.29499999999999998,"15min":0.254},"max":{"1min":3.0139999999999998,"5min":214.72999999999999,"15min":214.72999999999999},"last":2.1539999999999999},{"interface":"front","average":{"1min":2.04,"5min":5.327,"15min":2.6459999999999999},"min":{"1min":0.41399999999999998,"5min":0.40200000000000002,"15min":0.32400000000000001},"max":{"1min":15,"5min":214.76499999999999,"15min":214.76499999999999},"last":0.625}]},{"osd":1,"last update":"Sat Jun 3 20:11:03 2023","interfaces":[{"interface":"back","average":{"1min":1.1439999999999999,"5min":1.639,"15min":1.55},"min":{"1min":0.371,"5min":0.35199999999999998,"15min":0.28100000000000003},"max":{"1min":3.8210000000000002,"5min":10.446,"15min":58.32},"last":0.84599999999999997},{"interface":"front","average":{"1min":0.878,"5min":1.4990000000000001,"15min":1.4610000000000001},"min":{"1min":0.498,"5min":0.34999999999999998,"15min":0.307},"max":{"1min":1.8,"5min":12.919,"15min":58.286000000000001},"last":2.2429999999999999}]},{"osd":2,"last update":"Sat Jun 3 20:11:03 2023","interfaces":[{"interface":"back","average":{"1min":13.406000000000001,"5min":4.6070000000000002,"15min":2.1419999999999999},"min":{"1min":0.55000000000000004,"5min":0.317,"15min":0.30299999999999999},"max":{"1min":248.65799999999999,"5min":248.65799999999999,"15min":248.65799999999999},"last":2.2770000000000001},{"interface":"front","average":{"1min":14.144,"5min":4.7720000000000002,"15min":2.1890000000000001},"min":{"1min":0.46700000000000003,"5min":0.373,"15min":0.34499999999999997},"max":{"1min":249.077,"5min":249.077,"15min":249.077},"last":0.66200000000000003}]},{"osd":3,"last update":"Sat Jun 3 20:11:03 2023","interfaces":[{"interface":"back","average":{"1min":1.2330000000000001,"5min":1.778,"15min":1.6240000000000001},"min":{"1min":0.40600000000000003,"5min":0.35199999999999998,"15min":0.32500000000000001},"max":{"1min":3.7759999999999998,"5min":10.301,"15min":57.954999999999998},"last":1.0920000000000001},{"interface":"front","average":{"1min":1.226,"5min":1.6479999999999999,"15min":1.613},"min":{"1min":0.50700000000000001,"5min":0.40300000000000002,"15min":0.33700000000000002},"max":{"1min":4.9210000000000003,"5min":12.890000000000001,"15min":57.889000000000003},"last":2.2050000000000001}]},{"osd":5,"last update":"Sat Jun 3 20:11:03 2023","interfaces":[{"interface":"back","average":{"1min":1.4590000000000001,"5min":2.0819999999999999,"15min":1.623},"min":{"1min":0.35899999999999999,"5min":0.32000000000000001,"15min":0.313},"max":{"1min":6.6859999999999999,"5min":11.311,"15min":86.576999999999998},"last":0.47599999999999998},{"interface":"front","average":{"1min":1.5269999999999999,"5min":2.1760000000000002,"15min":1.7390000000000001},"min":{"1min":0.32900000000000001,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":7.9269999999999996,"5min":10.997999999999999,"15min":88.013000000000005},"last":4.9119999999999999}]},{"osd":6,"last update":"Sat Jun 3 20:11:03 2023","interfaces":[{"interface":"back","average":{"1min":1.8140000000000001,"5min":2.7429999999999999,"15min":1.5309999999999999},"min":{"1min":0.36099999999999999,"5min":0.34599999999999997,"15min":0.34599999999999997},"max":{"1min":6.6520000000000001,"5min":14.244999999999999,"15min":14.244999999999999},"last":2.109},{"interface":"front","average":{"1min":2.4329999999999998,"5min":2.8679999999999999,"15min":1.629},"min":{"1min":0.42599999999999999,"5min":0.40100000000000002,"15min":0.36499999999999999},"max":{"1min":10.359,"5min":12.145,"15min":12.145},"last":0.81200000000000006}]},{"osd":7,"last update":"Sat Jun 3 20:11:03 2023","interfaces":[{"interface":"back","average":{"1min":1.7270000000000001,"5min":2.052,"15min":1.55},"min":{"1min":0.44,"5min":0.41199999999999998,"15min":0.39700000000000002},"max":{"1min":9.6210000000000004,"5min":10.388,"15min":10.388},"last":4.8150000000000004},{"interface":"front","average":{"1min":1.0629999999999999,"5min":1.7729999999999999,"15min":1.486},"min":{"1min":0.44800000000000001,"5min":0.34200000000000003,"15min":0.31900000000000001},"max":{"1min":1.8260000000000001,"5min":12.951000000000001,"15min":12.951000000000001},"last":2.3109999999999999}]}]},{"osd":5,"up_from":36,"seq":154618823291,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3126176,"kb_used_data":2401880,"kb_used_omap":0,"kb_used_meta":724288,"kb_avail":90618976,"statfs":{"total":95995035648,"available":92793831424,"internally_reserved":0,"allocated":2459525120,"data_stored":2458919506,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":741670912},"hb_peers":[0,1,2,3,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 3 20:22:06 2023","interfaces":[{"interface":"back","average":{"1min":7.8010000000000002,"5min":3.8410000000000002,"15min":3.0470000000000002},"min":{"1min":0.29199999999999998,"5min":0.27400000000000002,"15min":0.26700000000000002},"max":{"1min":126.461,"5min":126.461,"15min":185.29900000000001},"last":0.47499999999999998},{"interface":"front","average":{"1min":7.8620000000000001,"5min":3.9100000000000001,"15min":3.1749999999999998},"min":{"1min":0.41599999999999998,"5min":0.33800000000000002,"15min":0.26400000000000001},"max":{"1min":126.497,"5min":126.497,"15min":185.44200000000001},"last":0.58599999999999997}]},{"osd":1,"last update":"Sat Jun 3 20:22:06 2023","interfaces":[{"interface":"back","average":{"1min":0.85799999999999998,"5min":0.68400000000000005,"15min":1.0880000000000001},"min":{"1min":0.497,"5min":0.32300000000000001,"15min":0.28799999999999998},"max":{"1min":1.234,"5min":2.859,"15min":8.5050000000000008},"last":102.205},{"interface":"front","average":{"1min":0.94899999999999995,"5min":0.72699999999999998,"15min":1.0489999999999999},"min":{"1min":0.57199999999999995,"5min":0.32800000000000001,"15min":0.27600000000000002},"max":{"1min":1.3380000000000001,"5min":5.7469999999999999,"15min":9.2739999999999991},"last":102.265}]},{"osd":2,"last update":"Sat Jun 3 20:22:06 2023","interfaces":[{"interface":"back","average":{"1min":0.92400000000000004,"5min":2.726,"15min":3.379},"min":{"1min":0.42799999999999999,"5min":0.34599999999999997,"15min":0.318},"max":{"1min":3.1000000000000001,"5min":86.757999999999996,"15min":216.881},"last":0.63200000000000001},{"interface":"front","average":{"1min":0.92600000000000005,"5min":3.0510000000000002,"15min":3.6440000000000001},"min":{"1min":0.55300000000000005,"5min":0.35199999999999998,"15min":0.314},"max":{"1min":1.2809999999999999,"5min":87.400999999999996,"15min":216.751},"last":0.68500000000000005}]},{"osd":3,"last update":"Sat Jun 3 20:21:39 2023","interfaces":[{"interface":"back","average":{"1min":0.94499999999999995,"5min":2.919,"15min":3.2709999999999999},"min":{"1min":0.51600000000000001,"5min":0.41899999999999998,"15min":0.317},"max":{"1min":1.4199999999999999,"5min":86.486999999999995,"15min":264.822},"last":0.55600000000000005},{"interface":"front","average":{"1min":0.98699999999999999,"5min":3.3410000000000002,"15min":3.1949999999999998},"min":{"1min":0.56000000000000005,"5min":0.37,"15min":0.35799999999999998},"max":{"1min":1.272,"5min":86.512,"15min":264.75},"last":0.74399999999999999}]},{"osd":4,"last update":"Sat Jun 3 20:11:00 2023","interfaces":[{"interface":"back","average":{"1min":2.2930000000000001,"5min":2.4510000000000001,"15min":1.6839999999999999},"min":{"1min":0.44,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":7.5910000000000002,"5min":14.951000000000001,"15min":14.951000000000001},"last":0.81899999999999995},{"interface":"front","average":{"1min":1.619,"5min":2.0129999999999999,"15min":1.3799999999999999},"min":{"1min":0.42299999999999999,"5min":0.376,"15min":0.376},"max":{"1min":7.2000000000000002,"5min":12.109,"15min":12.109},"last":0.66900000000000004}]},{"osd":6,"last update":"Sat Jun 3 20:22:06 2023","interfaces":[{"interface":"back","average":{"1min":0.91300000000000003,"5min":0.746,"15min":1.538},"min":{"1min":0.52600000000000002,"5min":0.35199999999999998,"15min":0.29099999999999998},"max":{"1min":1.3839999999999999,"5min":2.056,"15min":22.945},"last":0.81399999999999995},{"interface":"front","average":{"1min":1.05,"5min":0.84299999999999997,"15min":1.698},"min":{"1min":0.623,"5min":0.432,"15min":0.35799999999999998},"max":{"1min":1.45,"5min":2.899,"15min":22.428000000000001},"last":0.77600000000000002}]},{"osd":7,"last update":"Sat Jun 3 20:21:54 2023","interfaces":[{"interface":"back","average":{"1min":1.177,"5min":0.88400000000000001,"15min":1.4970000000000001},"min":{"1min":0.81899999999999995,"5min":0.35999999999999999,"15min":0.35999999999999999},"max":{"1min":2.0390000000000001,"5min":2.0739999999999998,"15min":14.223000000000001},"last":0.71299999999999997},{"interface":"front","average":{"1min":1.105,"5min":0.84999999999999998,"15min":1.296},"min":{"1min":0.71699999999999997,"5min":0.373,"15min":0.28799999999999998},"max":{"1min":2.0960000000000001,"5min":4.4809999999999999,"15min":9.0719999999999992},"last":0.90900000000000003}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6928,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4862,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":10652,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":4716,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4932,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":653721600,"data_stored":653721600,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":1318060032,"data_stored":1318060032,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":1888944128,"data_stored":1888944128,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":892010496,"data_stored":892010496,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":2497380352,"data_stored":2497380352,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":1362362368,"data_stored":1362362368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-06-03T20:22:28.749 INFO:tasks.cephadm.ceph_manager.ceph:increase pool size by 10 2023-06-03T20:22:28.749 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph osd pool set device_health_metrics pg_num 21 2023-06-03T20:22:28.784 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:22:28.817 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:22:28.819 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:22:28.820 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:22:28.921 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:22:28.953 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:22:28.955 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:22:28.955 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:22:29.056 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:22:29.092 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:22:29.094 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:22:29.095 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:22:29.196 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:22:29.234 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:22:29.236 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:22:29.237 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:22:29.338 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:22:29.366 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:22:29.370 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:22:29.370 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:22:29.471 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:22:29.507 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:22:29.512 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:22:29.513 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:22:29.606 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 36 16 62660 62644 108.736 79.25 0.00330633 0.00919243 2023-06-03T20:22:29.614 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:22:29.650 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:22:29.654 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:22:29.655 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:22:29.755 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:22:29.799 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:22:29.802 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:22:29.802 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:22:29.903 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:22:29.937 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:22:29.944 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:22:29.944 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:22:30.045 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:22:30.080 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:22:30.082 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:22:30.083 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:22:30.184 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:22:30.193 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:22:30 smithi164 bash[59462]: cluster 1685823748.9894724 mgr.y (mgr.34104) 1397 : cluster 0 pgmap v1410: 89 pgs: 17 active+clean+remapped, 13 active+undersized+degraded, 5 active+undersized, 54 active+clean; 3.6 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 83 MiB/s wr, 1.33k op/s; 33003/176421 objects degraded (18.707%); 6/176421 objects misplaced (0.003%) 2023-06-03T20:22:30.217 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:22:30.220 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:22:30.221 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:22:30.322 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:22:30.359 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:22:30.362 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:22:30.363 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:22:30.464 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:22:30.473 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:22:30 smithi089 bash[75068]: cluster 1685823748.9894724 mgr.y (mgr.34104) 1397 : cluster 0 pgmap v1410: 89 pgs: 17 active+clean+remapped, 13 active+undersized+degraded, 5 active+undersized, 54 active+clean; 3.6 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 83 MiB/s wr, 1.33k op/s; 33003/176421 objects degraded (18.707%); 6/176421 objects misplaced (0.003%) 2023-06-03T20:22:30.474 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:22:30 smithi089 bash[78452]: cluster 1685823748.9894724 mgr.y (mgr.34104) 1397 : cluster 0 pgmap v1410: 89 pgs: 17 active+clean+remapped, 13 active+undersized+degraded, 5 active+undersized, 54 active+clean; 3.6 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 83 MiB/s wr, 1.33k op/s; 33003/176421 objects degraded (18.707%); 6/176421 objects misplaced (0.003%) 2023-06-03T20:22:30.496 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:22:30.499 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:22:30.500 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:22:30.600 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:22:30.606 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 37 16 63910 63894 107.908 78.125 0.00877521 0.00926194 2023-06-03T20:22:30.633 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:22:30.638 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:22:30.638 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:22:30.703 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph osd unset noscrub 2023-06-03T20:22:30.704 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-06-03T20:22:30.740 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:22:30.813 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:22:30.818 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:22:30.819 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:22:30.919 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:22:30.969 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:22:30.972 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:22:30.973 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:22:31.074 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:22:31.113 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:22:31.117 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:22:31.118 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:22:31.219 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:22:31.257 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:22:31.260 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:22:31.261 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:22:31.362 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:22:31.396 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:22:31.399 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:22:31.400 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:22:31.500 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:22:31.532 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:22:31.537 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:22:31.537 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:22:31.606 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 38 16 65057 65041 106.955 71.6875 0.00886549 0.00934274 2023-06-03T20:22:31.638 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:22:31.679 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:22:31.681 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:22:31.682 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:22:31.783 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:22:31.822 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:22:31.826 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:22:31.827 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:22:31.928 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:22:31.965 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:22:31.968 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:22:31.969 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:22:32.070 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:22:32.102 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:22:32.106 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:22:32.107 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:22:32.207 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:22:32.217 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:22:32 smithi089 bash[75068]: cluster 1685823750.9902213 mgr.y (mgr.34104) 1398 : cluster 0 pgmap v1411: 89 pgs: 17 active+clean+remapped, 13 active+undersized+degraded, 5 active+undersized, 54 active+clean; 3.8 GiB data, 13 GiB used, 612 GiB / 626 GiB avail; 82 MiB/s wr, 1.31k op/s; 34633/184929 objects degraded (18.728%); 6/184929 objects misplaced (0.003%) 2023-06-03T20:22:32.217 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:22:32 smithi089 bash[78452]: cluster 1685823750.9902213 mgr.y (mgr.34104) 1398 : cluster 0 pgmap v1411: 89 pgs: 17 active+clean+remapped, 13 active+undersized+degraded, 5 active+undersized, 54 active+clean; 3.8 GiB data, 13 GiB used, 612 GiB / 626 GiB avail; 82 MiB/s wr, 1.31k op/s; 34633/184929 objects degraded (18.728%); 6/184929 objects misplaced (0.003%) 2023-06-03T20:22:32.240 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:22:32.243 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:22:32.243 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:22:32.303 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/mon.a/config 2023-06-03T20:22:32.334 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:22:32 smithi164 bash[59462]: cluster 1685823750.9902213 mgr.y (mgr.34104) 1398 : cluster 0 pgmap v1411: 89 pgs: 17 active+clean+remapped, 13 active+undersized+degraded, 5 active+undersized, 54 active+clean; 3.8 GiB data, 13 GiB used, 612 GiB / 626 GiB avail; 82 MiB/s wr, 1.31k op/s; 34633/184929 objects degraded (18.728%); 6/184929 objects misplaced (0.003%) 2023-06-03T20:22:32.344 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:22:32.375 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:22:32.378 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:22:32.379 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:22:32.480 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:22:32.517 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:22:32.522 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:22:32.523 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:22:32.606 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 39 16 66249 66233 106.123 74.5 0.0138297 0.0094171 2023-06-03T20:22:32.624 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:22:32.668 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:22:32.673 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:22:32.674 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:22:32.775 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:22:32.817 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:22:32.823 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:22:32.824 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:22:32.924 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:22:32.957 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:22:32.962 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:22:32.963 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:22:33.064 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:22:33.104 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:22:33.110 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:22:33.111 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:22:33.212 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:22:33.253 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:22:33.257 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:22:33.258 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:22:33.334 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:22:33 smithi164 bash[59462]: cluster 1685823752.9027014 mon.a (mon.0) 1381 : cluster 3 Health check update: Degraded data redundancy: 34633/184929 objects degraded (18.728%), 13 pgs degraded, 18 pgs undersized (PG_DEGRADED) 2023-06-03T20:22:33.335 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/mon.a/config 2023-06-03T20:22:33.359 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:22:33.369 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:22:33 smithi089 bash[78452]: cluster 1685823752.9027014 mon.a (mon.0) 1381 : cluster 3 Health check update: Degraded data redundancy: 34633/184929 objects degraded (18.728%), 13 pgs degraded, 18 pgs undersized (PG_DEGRADED) 2023-06-03T20:22:33.369 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:22:33 smithi089 bash[75068]: cluster 1685823752.9027014 mon.a (mon.0) 1381 : cluster 3 Health check update: Degraded data redundancy: 34633/184929 objects degraded (18.728%), 13 pgs degraded, 18 pgs undersized (PG_DEGRADED) 2023-06-03T20:22:33.391 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:22:33.396 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:22:33.397 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:22:33.498 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:22:33.525 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:22:33.527 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:22:33.527 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:22:33.606 INFO:tasks.radosbench.radosbench.0.smithi089.stdout:2023-06-03T20:22:33.602157+0000 min lat: 0.0012167 max lat: 0.341691 avg lat: 0.00950164 2023-06-03T20:22:33.606 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-06-03T20:22:33.606 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 40 16 67354 67338 105.196 69.0625 0.0142609 0.00950164 2023-06-03T20:22:33.628 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:22:33.672 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:22:33.677 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:22:33.678 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:22:33.779 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:22:33.818 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:22:33.820 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:22:33.821 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:22:33.922 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:22:33.952 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:22:33.954 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:22:33.955 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:22:34.056 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:22:34.089 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:22:34.094 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:22:34.095 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:22:34.196 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:22:34.227 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:22:34.228 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:22:34.229 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:22:34.304 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:22:34 smithi089 bash[75068]: cluster 1685823752.9910238 mgr.y (mgr.34104) 1399 : cluster 0 pgmap v1412: 89 pgs: 17 active+clean+remapped, 13 active+undersized+degraded, 5 active+undersized, 54 active+clean; 3.9 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 80 MiB/s wr, 1.28k op/s; 35805/190698 objects degraded (18.776%); 6/190698 objects misplaced (0.003%) 2023-06-03T20:22:34.330 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:22:34.334 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:22:34 smithi164 bash[59462]: cluster 1685823752.9910238 mgr.y (mgr.34104) 1399 : cluster 0 pgmap v1412: 89 pgs: 17 active+clean+remapped, 13 active+undersized+degraded, 5 active+undersized, 54 active+clean; 3.9 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 80 MiB/s wr, 1.28k op/s; 35805/190698 objects degraded (18.776%); 6/190698 objects misplaced (0.003%) 2023-06-03T20:22:34.363 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:22:34.367 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:22:34.368 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:22:34.468 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:22:34.516 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:22:34.519 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:22:34.519 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:22:34.606 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 41 16 68488 68472 104.359 70.875 0.00401739 0.00957676 2023-06-03T20:22:34.621 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:22:34.653 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:22:34.655 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:22:34.656 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:22:34.757 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:22:34.779 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:22:34 smithi089 bash[78452]: cluster 1685823752.9910238 mgr.y (mgr.34104) 1399 : cluster 0 pgmap v1412: 89 pgs: 17 active+clean+remapped, 13 active+undersized+degraded, 5 active+undersized, 54 active+clean; 3.9 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 80 MiB/s wr, 1.28k op/s; 35805/190698 objects degraded (18.776%); 6/190698 objects misplaced (0.003%) 2023-06-03T20:22:34.790 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:22:34.794 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:22:34.795 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:22:34.895 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:22:34.937 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:22:34.938 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:22:34.938 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:22:35.039 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:22:35.076 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:22:35.080 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:22:35.081 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:22:35.182 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:22:35.216 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:22:35.216 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:22:35.217 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:22:35.318 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:22:35.356 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:22:35.361 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:22:35.362 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:22:35.390 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/mon.a/config 2023-06-03T20:22:35.391 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/mon.a/config 2023-06-03T20:22:35.463 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:22:35.491 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:22:35.495 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:22:35.496 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:22:35.597 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:22:35.606 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 42 16 69648 69632 103.6 72.5 0.00707952 0.00964734 2023-06-03T20:22:35.637 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:22:35.644 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:22:35.645 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:22:35.746 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:22:35.785 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:22:35.787 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:22:35.788 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:22:35.889 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:22:35.919 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:22:35.922 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:22:35.923 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:22:36.024 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:22:36.057 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:22:36.060 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:22:36.061 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:22:36.162 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:22:36.194 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:22:36.199 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:22:36.199 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:22:36.301 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:22:36.310 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:22:36 smithi164 bash[59462]: cluster 1685823754.9916713 mgr.y (mgr.34104) 1400 : cluster 0 pgmap v1413: 89 pgs: 17 active+clean+remapped, 13 active+undersized+degraded, 5 active+undersized, 54 active+clean; 4.0 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 78 MiB/s wr, 1.25k op/s; 37288/198531 objects degraded (18.782%); 6/198531 objects misplaced (0.003%) 2023-06-03T20:22:36.334 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:22:36.338 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:22:36.339 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:22:36.440 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:22:36.449 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 20:22:36 smithi089 bash[71163]: ::ffff:172.21.15.164 - - [03/Jun/2023:20:22:36] "GET /metrics HTTP/1.1" 200 228999 "" "Prometheus/2.33.4" 2023-06-03T20:22:36.449 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:22:36 smithi089 bash[78452]: cluster 1685823754.9916713 mgr.y (mgr.34104) 1400 : cluster 0 pgmap v1413: 89 pgs: 17 active+clean+remapped, 13 active+undersized+degraded, 5 active+undersized, 54 active+clean; 4.0 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 78 MiB/s wr, 1.25k op/s; 37288/198531 objects degraded (18.782%); 6/198531 objects misplaced (0.003%) 2023-06-03T20:22:36.450 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:22:36 smithi089 bash[75068]: cluster 1685823754.9916713 mgr.y (mgr.34104) 1400 : cluster 0 pgmap v1413: 89 pgs: 17 active+clean+remapped, 13 active+undersized+degraded, 5 active+undersized, 54 active+clean; 4.0 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 78 MiB/s wr, 1.25k op/s; 37288/198531 objects degraded (18.782%); 6/198531 objects misplaced (0.003%) 2023-06-03T20:22:36.472 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:22:36.475 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:22:36.476 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:22:36.577 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:22:36.606 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 43 16 70934 70918 103.06 80.375 0.0118531 0.0096977 2023-06-03T20:22:36.615 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:22:36.618 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:22:36.619 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:22:36.720 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:22:36.754 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:22:36.762 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:22:36.763 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:22:36.864 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:22:36.894 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:22:36.896 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:22:36.897 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:22:36.997 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:22:37.053 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:22:37.055 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:22:37.056 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:22:37.157 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:22:37.190 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:22:37.193 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:22:37.193 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:22:37.291 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:22:37 smithi089 bash[75068]: audit 1685823757.1354609 mon.c (mon.2) 594 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T20:22:37.291 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:22:37 smithi089 bash[78452]: audit 1685823757.1354609 mon.c (mon.2) 594 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T20:22:37.295 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:22:37.332 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:22:37.335 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:22:37.335 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:22:37.436 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:22:37.471 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:22:37.476 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:22:37.476 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:22:37.577 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:22:37.584 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:22:37 smithi164 bash[59462]: audit 1685823757.1354609 mon.c (mon.2) 594 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T20:22:37.607 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 44 16 72127 72111 102.412 74.5625 0.01585 0.00975781 2023-06-03T20:22:37.617 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:22:37.624 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:22:37.625 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:22:37.726 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:22:37.755 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:22:37.758 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:22:37.759 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:22:37.859 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:22:37.890 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:22:37.892 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:22:37.893 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:22:37.995 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:22:38.028 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:22:38.030 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:22:38.031 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:22:38.132 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:22:38.168 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:22:38.169 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:22:38.170 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:22:38.271 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:22:38.279 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:22:38 smithi089 bash[78452]: cluster 1685823756.9922068 mgr.y (mgr.34104) 1401 : cluster 0 pgmap v1414: 89 pgs: 17 active+clean+remapped, 13 active+undersized+degraded, 5 active+undersized, 54 active+clean; 4.1 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 74 MiB/s wr, 1.19k op/s; 37984/202941 objects degraded (18.717%); 6/202941 objects misplaced (0.003%) 2023-06-03T20:22:38.279 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:22:38 smithi089 bash[78452]: cluster 1685823757.9037812 mon.a (mon.0) 1382 : cluster 3 Health check update: Degraded data redundancy: 37984/202941 objects degraded (18.717%), 13 pgs degraded, 18 pgs undersized (PG_DEGRADED) 2023-06-03T20:22:38.280 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:22:38 smithi089 bash[75068]: cluster 1685823756.9922068 mgr.y (mgr.34104) 1401 : cluster 0 pgmap v1414: 89 pgs: 17 active+clean+remapped, 13 active+undersized+degraded, 5 active+undersized, 54 active+clean; 4.1 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 74 MiB/s wr, 1.19k op/s; 37984/202941 objects degraded (18.717%); 6/202941 objects misplaced (0.003%) 2023-06-03T20:22:38.280 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:22:38 smithi089 bash[75068]: cluster 1685823757.9037812 mon.a (mon.0) 1382 : cluster 3 Health check update: Degraded data redundancy: 37984/202941 objects degraded (18.717%), 13 pgs degraded, 18 pgs undersized (PG_DEGRADED) 2023-06-03T20:22:38.299 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:22:38.301 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:22:38.302 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:22:38.403 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:22:38.434 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:22:38.436 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:22:38.437 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:22:38.538 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:22:38.548 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:22:38 smithi164 bash[59462]: cluster 1685823756.9922068 mgr.y (mgr.34104) 1401 : cluster 0 pgmap v1414: 89 pgs: 17 active+clean+remapped, 13 active+undersized+degraded, 5 active+undersized, 54 active+clean; 4.1 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 74 MiB/s wr, 1.19k op/s; 37984/202941 objects degraded (18.717%); 6/202941 objects misplaced (0.003%) 2023-06-03T20:22:38.548 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:22:38 smithi164 bash[59462]: cluster 1685823757.9037812 mon.a (mon.0) 1382 : cluster 3 Health check update: Degraded data redundancy: 37984/202941 objects degraded (18.717%), 13 pgs degraded, 18 pgs undersized (PG_DEGRADED) 2023-06-03T20:22:38.570 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:22:38.574 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:22:38.575 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:22:38.607 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 45 16 73374 73358 101.868 77.9375 0.0153699 0.00981012 2023-06-03T20:22:38.675 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:22:38.724 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:22:38.730 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:22:38.730 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:22:38.831 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:22:38.868 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:22:38.871 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:22:38.872 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:22:38.973 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:22:39.022 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:22:39.026 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:22:39.027 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:22:39.128 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:22:39.162 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:22:39.163 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:22:39.164 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:22:39.265 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:22:39.295 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:22:39.302 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:22:39.303 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:22:39.403 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:22:39.437 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:22:39.442 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:22:39.443 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:22:39.544 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:22:39.589 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:22:39.592 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:22:39.593 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:22:39.606 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 46 16 74497 74481 101.179 70.1875 0.0165823 0.00987822 2023-06-03T20:22:39.694 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:22:39.719 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:22:39.721 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:22:39.722 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:22:39.822 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:22:39.861 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:22:39.866 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:22:39.867 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:22:39.968 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:22:39.988 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:22:39.989 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:22:39.990 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:22:40.091 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:22:40.121 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:22:40.126 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:22:40.127 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:22:40.228 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:22:40.260 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:22:40.265 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:22:40.266 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:22:40.367 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:22:40.404 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:22:40.409 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:22:40.410 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:22:40.510 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:22:40.521 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:22:40 smithi164 bash[59462]: cluster 1685823758.9933798 mgr.y (mgr.34104) 1402 : cluster 0 pgmap v1415: 89 pgs: 17 active+clean+remapped, 13 active+undersized+degraded, 5 active+undersized, 54 active+clean; 4.3 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 82 MiB/s wr, 1.31k op/s; 39803/212973 objects degraded (18.689%); 6/212973 objects misplaced (0.003%) 2023-06-03T20:22:40.529 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:22:40 smithi089 bash[75068]: cluster 1685823758.9933798 mgr.y (mgr.34104) 1402 : cluster 0 pgmap v1415: 89 pgs: 17 active+clean+remapped, 13 active+undersized+degraded, 5 active+undersized, 54 active+clean; 4.3 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 82 MiB/s wr, 1.31k op/s; 39803/212973 objects degraded (18.689%); 6/212973 objects misplaced (0.003%) 2023-06-03T20:22:40.529 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:22:40 smithi089 bash[78452]: cluster 1685823758.9933798 mgr.y (mgr.34104) 1402 : cluster 0 pgmap v1415: 89 pgs: 17 active+clean+remapped, 13 active+undersized+degraded, 5 active+undersized, 54 active+clean; 4.3 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 82 MiB/s wr, 1.31k op/s; 39803/212973 objects degraded (18.689%); 6/212973 objects misplaced (0.003%) 2023-06-03T20:22:40.546 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:22:40.550 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:22:40.551 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:22:40.607 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 47 16 75394 75378 100.219 56.0625 0.0343487 0.00997257 2023-06-03T20:22:40.652 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:22:40.683 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:22:40.685 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:22:40.686 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:22:40.787 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:22:40.821 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:22:40.824 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:22:40.825 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:22:40.926 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:22:40.956 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:22:40.958 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:22:40.959 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:22:41.060 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:22:41.100 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:22:41.102 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:22:41.103 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:22:41.204 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:22:41.240 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:22:41.244 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:22:41.245 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:22:41.346 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:22:41.382 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:22:41.384 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:22:41.385 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:22:41.486 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:22:41.519 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:22:41.523 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:22:41.524 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:22:41.607 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 48 16 76422 76406 99.4699 64.25 0.0224758 0.0100476 2023-06-03T20:22:41.625 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:22:41.661 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:22:41.663 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:22:41.664 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:22:41.765 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:22:41.804 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:22:41.807 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:22:41.807 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:22:41.908 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:22:41.946 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:22:41.949 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:22:41.950 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:22:42.051 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:22:42.090 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:22:42.094 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:22:42.095 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:22:42.196 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:22:42.231 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:22:42.234 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:22:42.234 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:22:42.279 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:22:42 smithi089 bash[75068]: cluster 1685823760.9941654 mgr.y (mgr.34104) 1403 : cluster 0 pgmap v1416: 89 pgs: 17 active+clean+remapped, 13 active+undersized+degraded, 5 active+undersized, 54 active+clean; 4.5 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 76 MiB/s wr, 1.22k op/s; 41195/220299 objects degraded (18.700%); 6/220299 objects misplaced (0.003%) 2023-06-03T20:22:42.280 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:22:42 smithi089 bash[78452]: cluster 1685823760.9941654 mgr.y (mgr.34104) 1403 : cluster 0 pgmap v1416: 89 pgs: 17 active+clean+remapped, 13 active+undersized+degraded, 5 active+undersized, 54 active+clean; 4.5 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 76 MiB/s wr, 1.22k op/s; 41195/220299 objects degraded (18.700%); 6/220299 objects misplaced (0.003%) 2023-06-03T20:22:42.335 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:22:42.372 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:22:42.375 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:22:42.375 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:22:42.476 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:22:42.484 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:22:42 smithi164 bash[59462]: cluster 1685823760.9941654 mgr.y (mgr.34104) 1403 : cluster 0 pgmap v1416: 89 pgs: 17 active+clean+remapped, 13 active+undersized+degraded, 5 active+undersized, 54 active+clean; 4.5 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 76 MiB/s wr, 1.22k op/s; 41195/220299 objects degraded (18.700%); 6/220299 objects misplaced (0.003%) 2023-06-03T20:22:42.503 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:22:42.505 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:22:42.505 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:22:42.608 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:22:42.610 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 49 15 77442 77427 98.7422 63.8125 0.0130397 0.0101225 2023-06-03T20:22:42.650 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:22:42.654 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:22:42.655 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:22:42.756 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:22:42.798 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:22:42.803 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:22:42.803 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:22:42.904 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:22:42.945 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:22:42.947 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:22:42.948 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:22:43.049 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:22:43.085 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:22:43.088 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:22:43.089 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:22:43.190 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:22:43.197 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:22:43 smithi164 bash[59462]: cluster 1685823762.904881 mon.a (mon.0) 1383 : cluster 3 Health check update: Degraded data redundancy: 41195/220299 objects degraded (18.700%), 13 pgs degraded, 18 pgs undersized (PG_DEGRADED) 2023-06-03T20:22:43.198 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:22:43 smithi089 bash[75068]: cluster 1685823762.904881 mon.a (mon.0) 1383 : cluster 3 Health check update: Degraded data redundancy: 41195/220299 objects degraded (18.700%), 13 pgs degraded, 18 pgs undersized (PG_DEGRADED) 2023-06-03T20:22:43.229 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:22:43.233 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:22:43.234 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:22:43.335 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:22:43.373 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:22:43.377 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:22:43.377 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:22:43.478 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:22:43.517 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:22:43.521 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:22:43.522 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:22:43.529 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:22:43 smithi089 bash[78452]: cluster 1685823762.904881 mon.a (mon.0) 1383 : cluster 3 Health check update: Degraded data redundancy: 41195/220299 objects degraded (18.700%), 13 pgs degraded, 18 pgs undersized (PG_DEGRADED) 2023-06-03T20:22:43.607 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 50 16 78551 78535 98.1522 69.25 0.00578794 0.0101832 2023-06-03T20:22:43.623 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:22:43.661 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:22:43.664 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:22:43.665 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:22:43.766 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:22:43.797 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:22:43.801 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:22:43.801 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:22:43.902 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:22:43.938 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:22:43.943 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:22:43.944 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:22:44.044 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:22:44.078 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:22:44.082 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:22:44.082 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:22:44.183 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:22:44.199 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:22:44 smithi164 bash[59462]: cluster 1685823762.9947855 mgr.y (mgr.34104) 1404 : cluster 0 pgmap v1417: 89 pgs: 17 active+clean+remapped, 13 active+undersized+degraded, 5 active+undersized, 54 active+clean; 4.6 GiB data, 15 GiB used, 610 GiB / 626 GiB avail; 70 MiB/s wr, 1.13k op/s; 42232/225513 objects degraded (18.727%); 6/225513 objects misplaced (0.003%) 2023-06-03T20:22:44.219 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:22:44.223 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:22:44.224 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:22:44.279 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:22:44 smithi089 bash[75068]: cluster 1685823762.9947855 mgr.y (mgr.34104) 1404 : cluster 0 pgmap v1417: 89 pgs: 17 active+clean+remapped, 13 active+undersized+degraded, 5 active+undersized, 54 active+clean; 4.6 GiB data, 15 GiB used, 610 GiB / 626 GiB avail; 70 MiB/s wr, 1.13k op/s; 42232/225513 objects degraded (18.727%); 6/225513 objects misplaced (0.003%) 2023-06-03T20:22:44.280 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:22:44 smithi089 bash[78452]: cluster 1685823762.9947855 mgr.y (mgr.34104) 1404 : cluster 0 pgmap v1417: 89 pgs: 17 active+clean+remapped, 13 active+undersized+degraded, 5 active+undersized, 54 active+clean; 4.6 GiB data, 15 GiB used, 610 GiB / 626 GiB avail; 70 MiB/s wr, 1.13k op/s; 42232/225513 objects degraded (18.727%); 6/225513 objects misplaced (0.003%) 2023-06-03T20:22:44.324 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:22:44.361 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:22:44.364 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:22:44.365 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:22:44.466 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:22:44.495 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:22:44.500 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:22:44.500 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:22:44.601 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:22:44.607 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 51 16 79714 79698 97.6528 72.6875 0.0115501 0.0102338 2023-06-03T20:22:44.637 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:22:44.642 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:22:44.642 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:22:44.743 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:22:44.770 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:22:44.772 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:22:44.772 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:22:44.873 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:22:44.912 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:22:44.918 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:22:44.918 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:22:45.019 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:22:45.051 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:22:45.052 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:22:45.053 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:22:45.153 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:22:45.193 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:22:45.197 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:22:45.198 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:22:45.298 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:22:45.332 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:22:45.336 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:22:45.337 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:22:45.438 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:22:45.481 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:22:45.486 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:22:45.487 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:22:45.588 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:22:45.607 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 52 16 80400 80384 96.5994 42.875 0.0225432 0.0103469 2023-06-03T20:22:45.614 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:22:45.616 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:22:45.617 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:22:45.717 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:22:45.750 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:22:45.753 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:22:45.754 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:22:45.854 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:22:45.897 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:22:45.901 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:22:45.901 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:22:46.002 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:22:46.049 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:22:46.053 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:22:46.054 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:22:46.155 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:22:46.186 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:22:46.189 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:22:46.190 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:22:46.290 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:22:46.320 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:22:46.322 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:22:46.322 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:22:46.423 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:22:46.444 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:22:46.446 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:22:46.446 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:22:46.547 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:22:46.582 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:22:46.584 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:22:46 smithi164 bash[59462]: cluster 1685823764.995406 mgr.y (mgr.34104) 1405 : cluster 0 pgmap v1418: 89 pgs: 17 active+clean+remapped, 13 active+undersized+degraded, 5 active+undersized, 54 active+clean; 4.7 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 72 MiB/s wr, 1.16k op/s; 43527/232398 objects degraded (18.730%); 6/232398 objects misplaced (0.003%) 2023-06-03T20:22:46.586 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 20:22:46 smithi089 bash[71163]: ::ffff:172.21.15.164 - - [03/Jun/2023:20:22:46] "GET /metrics HTTP/1.1" 200 229015 "" "Prometheus/2.33.4" 2023-06-03T20:22:46.586 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:22:46 smithi089 bash[78452]: cluster 1685823764.995406 mgr.y (mgr.34104) 1405 : cluster 0 pgmap v1418: 89 pgs: 17 active+clean+remapped, 13 active+undersized+degraded, 5 active+undersized, 54 active+clean; 4.7 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 72 MiB/s wr, 1.16k op/s; 43527/232398 objects degraded (18.730%); 6/232398 objects misplaced (0.003%) 2023-06-03T20:22:46.587 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:22:46 smithi089 bash[75068]: cluster 1685823764.995406 mgr.y (mgr.34104) 1405 : cluster 0 pgmap v1418: 89 pgs: 17 active+clean+remapped, 13 active+undersized+degraded, 5 active+undersized, 54 active+clean; 4.7 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 72 MiB/s wr, 1.16k op/s; 43527/232398 objects degraded (18.730%); 6/232398 objects misplaced (0.003%) 2023-06-03T20:22:46.587 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:22:46.588 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:22:46.607 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 53 16 81383 81367 95.9359 61.4375 0.00426808 0.0104183 2023-06-03T20:22:46.689 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:22:46.724 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:22:46.729 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:22:46.730 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:22:46.831 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:22:46.859 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:22:46.863 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:22:46.863 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:22:46.964 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:22:47.004 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:22:47.006 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:22:47.007 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:22:47.108 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:22:47.141 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:22:47.147 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:22:47.147 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:22:47.248 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:22:47.280 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:22:47.289 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:22:47.289 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:22:47.390 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:22:47.412 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:22:47.415 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:22:47.415 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:22:47.516 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:22:47.548 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:22:47.555 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:22:47.556 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:22:47.607 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 54 16 82409 82393 95.3468 64.125 0.0168728 0.0104831 2023-06-03T20:22:47.657 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:22:47.690 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:22:47.695 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:22:47.696 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:22:47.796 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:22:47.835 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:22:47.841 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:22:47.842 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:22:47.943 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:22:48.047 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:22:48.054 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:22:48.055 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:22:48.156 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:22:48.202 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:22:48.205 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:22:48.206 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:22:48.279 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:22:48 smithi089 bash[78452]: cluster 1685823766.9961088 mgr.y (mgr.34104) 1406 : cluster 0 pgmap v1419: 89 pgs: 17 active+clean+remapped, 13 active+undersized+degraded, 5 active+undersized, 54 active+clean; 4.8 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 65 MiB/s wr, 1.04k op/s; 44177/236181 objects degraded (18.705%); 6/236181 objects misplaced (0.003%) 2023-06-03T20:22:48.279 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:22:48 smithi089 bash[78452]: cluster 1685823767.9059608 mon.a (mon.0) 1384 : cluster 3 Health check update: Degraded data redundancy: 44177/236181 objects degraded (18.705%), 13 pgs degraded, 18 pgs undersized (PG_DEGRADED) 2023-06-03T20:22:48.279 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:22:48 smithi089 bash[75068]: cluster 1685823766.9961088 mgr.y (mgr.34104) 1406 : cluster 0 pgmap v1419: 89 pgs: 17 active+clean+remapped, 13 active+undersized+degraded, 5 active+undersized, 54 active+clean; 4.8 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 65 MiB/s wr, 1.04k op/s; 44177/236181 objects degraded (18.705%); 6/236181 objects misplaced (0.003%) 2023-06-03T20:22:48.280 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:22:48 smithi089 bash[75068]: cluster 1685823767.9059608 mon.a (mon.0) 1384 : cluster 3 Health check update: Degraded data redundancy: 44177/236181 objects degraded (18.705%), 13 pgs degraded, 18 pgs undersized (PG_DEGRADED) 2023-06-03T20:22:48.308 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:22:48.340 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:22:48.343 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:22:48.344 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:22:48.444 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:22:48.483 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:22:48.488 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:22:48 smithi164 bash[59462]: cluster 1685823766.9961088 mgr.y (mgr.34104) 1406 : cluster 0 pgmap v1419: 89 pgs: 17 active+clean+remapped, 13 active+undersized+degraded, 5 active+undersized, 54 active+clean; 4.8 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 65 MiB/s wr, 1.04k op/s; 44177/236181 objects degraded (18.705%); 6/236181 objects misplaced (0.003%) 2023-06-03T20:22:48.488 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:22:48 smithi164 bash[59462]: cluster 1685823767.9059608 mon.a (mon.0) 1384 : cluster 3 Health check update: Degraded data redundancy: 44177/236181 objects degraded (18.705%), 13 pgs degraded, 18 pgs undersized (PG_DEGRADED) 2023-06-03T20:22:48.489 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:22:48.489 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:22:48.590 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:22:48.607 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 55 16 83411 83395 94.7518 62.625 0.0240776 0.0105485 2023-06-03T20:22:48.624 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:22:48.629 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:22:48.630 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:22:48.731 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:22:48.762 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:22:48.764 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:22:48.765 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:22:48.866 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:22:48.900 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:22:48.905 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:22:48.906 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:22:49.007 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:22:49.033 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:22:49.035 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:22:49.036 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:22:49.137 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:22:49.165 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:22:49.167 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:22:49.168 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:22:49.269 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:22:49.303 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:22:49.310 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:22:49.311 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:22:49.412 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:22:49.442 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:22:49.448 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:22:49.449 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:22:49.550 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:22:49.594 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:22:49.599 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:22:49.600 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:22:49.607 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 56 16 84295 84279 94.0464 55.25 0.0160232 0.0106271 2023-06-03T20:22:49.701 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:22:49.731 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:22:49.735 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:22:49.736 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:22:49.837 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:22:49.879 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:22:49.882 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:22:49.882 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:22:49.983 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:22:50.020 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:22:50.023 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:22:50.023 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:22:50.124 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:22:50.153 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:22:50.161 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:22:50.161 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:22:50.262 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:22:50.287 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:22:50.291 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:22:50.292 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:22:50.394 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:22:50.403 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:22:50 smithi089 bash[75068]: cluster 1685823768.9970245 mgr.y (mgr.34104) 1407 : cluster 0 pgmap v1420: 89 pgs: 17 active+clean+remapped, 13 active+undersized+degraded, 5 active+undersized, 54 active+clean; 5.0 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 72 MiB/s wr, 1.16k op/s; 45690/244557 objects degraded (18.683%); 6/244557 objects misplaced (0.002%) 2023-06-03T20:22:50.403 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:22:50 smithi089 bash[78452]: cluster 1685823768.9970245 mgr.y (mgr.34104) 1407 : cluster 0 pgmap v1420: 89 pgs: 17 active+clean+remapped, 13 active+undersized+degraded, 5 active+undersized, 54 active+clean; 5.0 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 72 MiB/s wr, 1.16k op/s; 45690/244557 objects degraded (18.683%); 6/244557 objects misplaced (0.002%) 2023-06-03T20:22:50.440 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:22:50.443 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:22:50.444 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:22:50.544 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:22:50.573 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:22:50.577 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:22:50.578 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:22:50.584 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:22:50 smithi164 bash[59462]: cluster 1685823768.9970245 mgr.y (mgr.34104) 1407 : cluster 0 pgmap v1420: 89 pgs: 17 active+clean+remapped, 13 active+undersized+degraded, 5 active+undersized, 54 active+clean; 5.0 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 72 MiB/s wr, 1.16k op/s; 45690/244557 objects degraded (18.683%); 6/244557 objects misplaced (0.002%) 2023-06-03T20:22:50.608 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 57 16 85348 85332 93.551 65.8125 0.0150169 0.0106846 2023-06-03T20:22:50.679 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:22:50.718 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:22:50.725 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:22:50.726 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:22:50.826 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:22:50.861 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:22:50.865 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:22:50.866 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:22:50.967 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:22:51.007 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:22:51.009 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:22:51.010 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:22:51.111 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:22:51.142 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:22:51.147 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:22:51.147 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:22:51.248 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:22:51.283 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:22:51.285 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:22:51.286 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:22:51.387 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:22:51.415 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:22:51.416 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:22:51.417 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:22:51.518 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:22:51.553 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:22:51.561 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:22:51.561 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:22:51.608 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 58 16 86346 86330 93.0134 62.375 0.010968 0.0107463 2023-06-03T20:22:51.663 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:22:51.694 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:22:51.702 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:22:51.702 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:22:51.803 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:22:51.835 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:22:51.842 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:22:51.843 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:22:51.944 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:22:51.976 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:22:51.980 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:22:51.981 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:22:52.082 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:22:52.122 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:22:52.126 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:22:52.127 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:22:52.228 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:22:52.261 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:22:52.264 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:22:52.265 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:22:52.292 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:22:52 smithi089 bash[75068]: cluster 1685823770.9976423 mgr.y (mgr.34104) 1408 : cluster 0 pgmap v1421: 89 pgs: 17 active+clean+remapped, 13 active+undersized+degraded, 5 active+undersized, 54 active+clean; 5.1 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 66 MiB/s wr, 1.05k op/s; 46898/250902 objects degraded (18.692%); 6/250902 objects misplaced (0.002%) 2023-06-03T20:22:52.292 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:22:52 smithi089 bash[75068]: audit 1685823772.1358097 mon.c (mon.2) 595 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T20:22:52.293 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:22:52 smithi089 bash[78452]: cluster 1685823770.9976423 mgr.y (mgr.34104) 1408 : cluster 0 pgmap v1421: 89 pgs: 17 active+clean+remapped, 13 active+undersized+degraded, 5 active+undersized, 54 active+clean; 5.1 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 66 MiB/s wr, 1.05k op/s; 46898/250902 objects degraded (18.692%); 6/250902 objects misplaced (0.002%) 2023-06-03T20:22:52.293 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:22:52 smithi089 bash[78452]: audit 1685823772.1358097 mon.c (mon.2) 595 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T20:22:52.366 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:22:52.404 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:22:52.408 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:22:52.409 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:22:52.510 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:22:52.557 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:22:52.559 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:22:52.559 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:22:52.584 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:22:52 smithi164 bash[59462]: cluster 1685823770.9976423 mgr.y (mgr.34104) 1408 : cluster 0 pgmap v1421: 89 pgs: 17 active+clean+remapped, 13 active+undersized+degraded, 5 active+undersized, 54 active+clean; 5.1 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 66 MiB/s wr, 1.05k op/s; 46898/250902 objects degraded (18.692%); 6/250902 objects misplaced (0.002%) 2023-06-03T20:22:52.584 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:22:52 smithi164 bash[59462]: audit 1685823772.1358097 mon.c (mon.2) 595 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T20:22:52.608 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 59 16 87409 87393 92.5629 66.4375 0.0145275 0.0107975 2023-06-03T20:22:52.660 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:22:52.696 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:22:52.699 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:22:52.699 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:22:52.800 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:22:52.830 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:22:52.835 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:22:52.836 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:22:52.937 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:22:52.970 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:22:52.972 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:22:52.973 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:22:53.074 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:22:53.106 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:22:53.112 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:22:53.112 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:22:53.213 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:22:53.223 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:22:53 smithi164 bash[59462]: cluster 1685823772.9089947 mon.a (mon.0) 1385 : cluster 3 Health check update: Degraded data redundancy: 46898/250902 objects degraded (18.692%), 13 pgs degraded, 18 pgs undersized (PG_DEGRADED) 2023-06-03T20:22:53.237 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:22:53.240 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:22:53.240 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:22:53.341 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:22:53.350 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:22:53 smithi089 bash[75068]: cluster 1685823772.9089947 mon.a (mon.0) 1385 : cluster 3 Health check update: Degraded data redundancy: 46898/250902 objects degraded (18.692%), 13 pgs degraded, 18 pgs undersized (PG_DEGRADED) 2023-06-03T20:22:53.351 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:22:53 smithi089 bash[78452]: cluster 1685823772.9089947 mon.a (mon.0) 1385 : cluster 3 Health check update: Degraded data redundancy: 46898/250902 objects degraded (18.692%), 13 pgs degraded, 18 pgs undersized (PG_DEGRADED) 2023-06-03T20:22:53.375 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:22:53.377 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:22:53.378 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:22:53.479 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:22:53.525 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:22:53.529 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:22:53.530 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:22:53.608 INFO:tasks.radosbench.radosbench.0.smithi089.stdout:2023-06-03T20:22:53.604017+0000 min lat: 0.0012167 max lat: 0.341691 avg lat: 0.0108544 2023-06-03T20:22:53.608 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-06-03T20:22:53.608 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 60 16 88431 88415 92.0847 63.875 0.00690796 0.0108544 2023-06-03T20:22:53.631 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:22:53.663 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:22:53.665 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:22:53.666 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:22:53.767 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:22:53.798 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:22:53.805 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:22:53.805 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:22:53.906 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:22:53.943 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:22:53.947 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:22:53.947 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:22:54.048 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:22:54.080 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:22:54.083 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:22:54.084 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:22:54.185 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:22:54.212 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:22:54.214 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:22:54.214 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:22:54.315 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:22:54.326 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:22:54 smithi089 bash[75068]: cluster 1685823772.9984138 mgr.y (mgr.34104) 1409 : cluster 0 pgmap v1422: 89 pgs: 17 active+clean+remapped, 13 active+undersized+degraded, 5 active+undersized, 54 active+clean; 5.2 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 61 MiB/s wr, 977 op/s; 47816/255492 objects degraded (18.715%); 6/255492 objects misplaced (0.002%) 2023-06-03T20:22:54.326 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:22:54 smithi089 bash[78452]: cluster 1685823772.9984138 mgr.y (mgr.34104) 1409 : cluster 0 pgmap v1422: 89 pgs: 17 active+clean+remapped, 13 active+undersized+degraded, 5 active+undersized, 54 active+clean; 5.2 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 61 MiB/s wr, 977 op/s; 47816/255492 objects degraded (18.715%); 6/255492 objects misplaced (0.002%) 2023-06-03T20:22:54.355 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:22:54.361 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:22:54.361 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:22:54.462 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:22:54.488 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:22:54.490 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:22:54.490 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:22:54.584 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:22:54 smithi164 bash[59462]: cluster 1685823772.9984138 mgr.y (mgr.34104) 1409 : cluster 0 pgmap v1422: 89 pgs: 17 active+clean+remapped, 13 active+undersized+degraded, 5 active+undersized, 54 active+clean; 5.2 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 61 MiB/s wr, 977 op/s; 47816/255492 objects degraded (18.715%); 6/255492 objects misplaced (0.002%) 2023-06-03T20:22:54.592 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:22:54.608 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 61 16 89247 89231 91.4111 51 0.01129 0.0109337 2023-06-03T20:22:54.622 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:22:54.625 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:22:54.625 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:22:54.726 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:22:54.756 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:22:54.763 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:22:54.763 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:22:54.864 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:22:54.901 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:22:54.904 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:22:54.905 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:22:55.006 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:22:55.033 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:22:55.038 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:22:55.038 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:22:55.139 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:22:55.179 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:22:55.183 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:22:55.184 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:22:55.285 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:22:55.314 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:22:55.318 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:22:55.319 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:22:55.420 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:22:55.452 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:22:55.456 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:22:55.457 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:22:55.558 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:22:55.587 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:22:55.589 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:22:55.589 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:22:55.608 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 62 16 90282 90266 90.9799 64.6875 0.00415599 0.0109795 2023-06-03T20:22:55.690 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:22:55.718 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:22:55.720 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:22:55.721 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:22:55.821 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:22:55.861 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:22:55.866 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:22:55.867 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:22:55.968 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:22:56.004 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:22:56.006 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:22:56.007 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:22:56.108 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:22:56.144 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:22:56.146 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:22:56.147 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:22:56.248 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:22:56.274 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:22:56.278 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:22:56.279 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:22:56.380 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:22:56.414 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:22:56.419 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:22:56.420 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:22:56.521 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:22:56.529 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:22:56 smithi089 bash[75068]: cluster 1685823774.9991794 mgr.y (mgr.34104) 1410 : cluster 0 pgmap v1423: 89 pgs: 17 active+clean+remapped, 13 active+undersized+degraded, 5 active+undersized, 54 active+clean; 5.3 GiB data, 17 GiB used, 608 GiB / 626 GiB avail; 63 MiB/s wr, 1.01k op/s; 49061/262062 objects degraded (18.721%); 6/262062 objects misplaced (0.002%) 2023-06-03T20:22:56.529 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 20:22:56 smithi089 bash[71163]: ::ffff:172.21.15.164 - - [03/Jun/2023:20:22:56] "GET /metrics HTTP/1.1" 200 229013 "" "Prometheus/2.33.4" 2023-06-03T20:22:56.529 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:22:56 smithi089 bash[78452]: cluster 1685823774.9991794 mgr.y (mgr.34104) 1410 : cluster 0 pgmap v1423: 89 pgs: 17 active+clean+remapped, 13 active+undersized+degraded, 5 active+undersized, 54 active+clean; 5.3 GiB data, 17 GiB used, 608 GiB / 626 GiB avail; 63 MiB/s wr, 1.01k op/s; 49061/262062 objects degraded (18.721%); 6/262062 objects misplaced (0.002%) 2023-06-03T20:22:56.556 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:22:56.559 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:22:56.559 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:22:56.584 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:22:56 smithi164 bash[59462]: cluster 1685823774.9991794 mgr.y (mgr.34104) 1410 : cluster 0 pgmap v1423: 89 pgs: 17 active+clean+remapped, 13 active+undersized+degraded, 5 active+undersized, 54 active+clean; 5.3 GiB data, 17 GiB used, 608 GiB / 626 GiB avail; 63 MiB/s wr, 1.01k op/s; 49061/262062 objects degraded (18.721%); 6/262062 objects misplaced (0.002%) 2023-06-03T20:22:56.608 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 63 16 91228 91212 90.4742 59.125 0.00382006 0.0110475 2023-06-03T20:22:56.660 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:22:56.697 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:22:56.702 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:22:56.702 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:22:56.803 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:22:56.836 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:22:56.840 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:22:56.840 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:22:56.941 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:22:56.985 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:22:56.987 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:22:56.988 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:22:57.042 INFO:teuthology.orchestra.run.smithi089.stdout:{ 2023-06-03T20:22:57.042 INFO:teuthology.orchestra.run.smithi089.stdout: "ops": [], 2023-06-03T20:22:57.043 INFO:teuthology.orchestra.run.smithi089.stdout: "complaint_time": 30, 2023-06-03T20:22:57.043 INFO:teuthology.orchestra.run.smithi089.stdout: "num_blocked_ops": 0 2023-06-03T20:22:57.043 INFO:teuthology.orchestra.run.smithi089.stdout:} 2023-06-03T20:22:57.089 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:22:57.131 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:22:57.133 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:22:57.133 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:22:57.236 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:22:57.245 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:22:57 smithi089 bash[75068]: audit 1685823777.2160087 mon.c (mon.2) 596 : audit 0 from='client.? 172.21.15.89:0/2298149583' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-03T20:22:57.268 INFO:teuthology.orchestra.run.smithi089.stdout:osd.0: {} 2023-06-03T20:22:57.270 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:22:57.275 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:22:57.276 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:22:57.313 INFO:teuthology.orchestra.run.smithi089.stdout:osd.1: {} 2023-06-03T20:22:57.372 INFO:teuthology.orchestra.run.smithi089.stderr:Error ENOENT: unrecognized pool 'device_health_metrics' 2023-06-03T20:22:57.373 INFO:teuthology.orchestra.run.smithi089.stdout:osd.2: {} 2023-06-03T20:22:57.377 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:22:57.385 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:22:57 smithi089 bash[78452]: audit 1685823777.2160087 mon.c (mon.2) 596 : audit 0 from='client.? 172.21.15.89:0/2298149583' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-03T20:22:57.419 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:22:57.419 INFO:teuthology.orchestra.run.smithi089.stdout:osd.3: {} 2023-06-03T20:22:57.420 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:22:57.421 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:22:57.464 INFO:teuthology.orchestra.run.smithi089.stdout:osd.5: {} 2023-06-03T20:22:57.504 INFO:teuthology.orchestra.run.smithi089.stdout:osd.6: {} 2023-06-03T20:22:57.522 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:22:57.547 INFO:teuthology.orchestra.run.smithi089.stdout:osd.7: {} 2023-06-03T20:22:57.548 INFO:teuthology.orchestra.run.smithi089.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-06-03T20:22:57.548 INFO:teuthology.orchestra.run.smithi089.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-06-03T20:22:57.549 INFO:teuthology.orchestra.run.smithi089.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-06-03T20:22:57.549 INFO:teuthology.orchestra.run.smithi089.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-06-03T20:22:57.549 INFO:teuthology.orchestra.run.smithi089.stderr:Error ENXIO: problem getting command descriptions from osd.4 2023-06-03T20:22:57.549 INFO:teuthology.orchestra.run.smithi089.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-06-03T20:22:57.549 INFO:teuthology.orchestra.run.smithi089.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-06-03T20:22:57.549 INFO:teuthology.orchestra.run.smithi089.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-06-03T20:22:57.570 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:22:57.572 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:22:57.573 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:22:57.584 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:22:57 smithi164 bash[59462]: audit 1685823777.2160087 mon.c (mon.2) 596 : audit 0 from='client.? 172.21.15.89:0/2298149583' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-03T20:22:57.608 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 64 16 92010 91994 89.8242 48.875 0.0112342 0.0111272 2023-06-03T20:22:57.674 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:22:57.704 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:22:57.706 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:22:57.707 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:22:57.807 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:22:57.836 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:22:57.840 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:22:57.841 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:22:57.942 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:22:57.983 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:22:57.990 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:22:57.990 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:22:58.091 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:22:58.127 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:22:58.130 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:22:58.131 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:22:58.233 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:22:58.264 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:22:58.269 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:22:58.270 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:22:58.275 INFO:teuthology.orchestra.run.smithi089.stderr:noscrub is unset 2023-06-03T20:22:58.371 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:22:58.401 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:22:58.406 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:22:58.406 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:22:58.507 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:22:58.516 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:22:58 smithi089 bash[78452]: cluster 1685823776.9998498 mgr.y (mgr.34104) 1411 : cluster 0 pgmap v1424: 89 pgs: 17 active+clean+remapped, 13 active+undersized+degraded, 5 active+undersized, 54 active+clean; 5.4 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 58 MiB/s wr, 924 op/s; 49640/265686 objects degraded (18.684%); 6/265686 objects misplaced (0.002%) 2023-06-03T20:22:58.516 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:22:58 smithi089 bash[78452]: audit 1685823777.361564 mon.c (mon.2) 597 : audit 1 from='client.? 172.21.15.89:0/1851994146' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "device_health_metrics", "var": "pg_num", "val": "21"}]: dispatch 2023-06-03T20:22:58.517 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:22:58 smithi089 bash[78452]: audit 1685823777.3629036 mon.a (mon.0) 1386 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "device_health_metrics", "var": "pg_num", "val": "21"}]: dispatch 2023-06-03T20:22:58.517 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:22:58 smithi089 bash[78452]: audit 1685823777.6722667 mon.a (mon.0) 1387 : audit 1 from='client.? 172.21.15.89:0/1485137449' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-06-03T20:22:58.517 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:22:58 smithi089 bash[78452]: cluster 1685823777.9101691 mon.a (mon.0) 1388 : cluster 3 Health check update: Degraded data redundancy: 49640/265686 objects degraded (18.684%), 13 pgs degraded, 18 pgs undersized (PG_DEGRADED) 2023-06-03T20:22:58.518 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:22:58 smithi089 bash[75068]: cluster 1685823776.9998498 mgr.y (mgr.34104) 1411 : cluster 0 pgmap v1424: 89 pgs: 17 active+clean+remapped, 13 active+undersized+degraded, 5 active+undersized, 54 active+clean; 5.4 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 58 MiB/s wr, 924 op/s; 49640/265686 objects degraded (18.684%); 6/265686 objects misplaced (0.002%) 2023-06-03T20:22:58.518 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:22:58 smithi089 bash[75068]: audit 1685823777.361564 mon.c (mon.2) 597 : audit 1 from='client.? 172.21.15.89:0/1851994146' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "device_health_metrics", "var": "pg_num", "val": "21"}]: dispatch 2023-06-03T20:22:58.518 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:22:58 smithi089 bash[75068]: audit 1685823777.3629036 mon.a (mon.0) 1386 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "device_health_metrics", "var": "pg_num", "val": "21"}]: dispatch 2023-06-03T20:22:58.518 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:22:58 smithi089 bash[75068]: audit 1685823777.6722667 mon.a (mon.0) 1387 : audit 1 from='client.? 172.21.15.89:0/1485137449' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-06-03T20:22:58.519 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:22:58 smithi089 bash[75068]: cluster 1685823777.9101691 mon.a (mon.0) 1388 : cluster 3 Health check update: Degraded data redundancy: 49640/265686 objects degraded (18.684%), 13 pgs degraded, 18 pgs undersized (PG_DEGRADED) 2023-06-03T20:22:58.543 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:22:58.545 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:22:58.546 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:22:58.584 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:22:58 smithi164 bash[59462]: cluster 1685823776.9998498 mgr.y (mgr.34104) 1411 : cluster 0 pgmap v1424: 89 pgs: 17 active+clean+remapped, 13 active+undersized+degraded, 5 active+undersized, 54 active+clean; 5.4 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 58 MiB/s wr, 924 op/s; 49640/265686 objects degraded (18.684%); 6/265686 objects misplaced (0.002%) 2023-06-03T20:22:58.584 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:22:58 smithi164 bash[59462]: audit 1685823777.361564 mon.c (mon.2) 597 : audit 1 from='client.? 172.21.15.89:0/1851994146' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "device_health_metrics", "var": "pg_num", "val": "21"}]: dispatch 2023-06-03T20:22:58.584 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:22:58 smithi164 bash[59462]: audit 1685823777.3629036 mon.a (mon.0) 1386 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "device_health_metrics", "var": "pg_num", "val": "21"}]: dispatch 2023-06-03T20:22:58.585 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:22:58 smithi164 bash[59462]: audit 1685823777.6722667 mon.a (mon.0) 1387 : audit 1 from='client.? 172.21.15.89:0/1485137449' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-06-03T20:22:58.585 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:22:58 smithi164 bash[59462]: cluster 1685823777.9101691 mon.a (mon.0) 1388 : cluster 3 Health check update: Degraded data redundancy: 49640/265686 objects degraded (18.684%), 13 pgs degraded, 18 pgs undersized (PG_DEGRADED) 2023-06-03T20:22:58.608 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 65 16 93037 93021 89.4297 64.1875 0.0042022 0.0111735 2023-06-03T20:22:58.647 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:22:58.667 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:22:58.669 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:22:58.670 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:22:58.770 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:22:58.805 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:22:58.812 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:22:58.813 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:22:58.913 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:22:58.945 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:22:58.948 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:22:58.949 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:22:59.050 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:22:59.081 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:22:59.085 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:22:59.086 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:22:59.187 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:22:59.221 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:22:59.226 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:22:59.227 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:22:59.328 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:22:59.360 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:22:59.363 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:22:59.364 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:22:59.464 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:22:59.473 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:22:59 smithi164 bash[59462]: cluster 1685823778.2401347 mon.a (mon.0) 1389 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-06-03T20:22:59.474 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:22:59 smithi164 bash[59462]: cluster 1685823778.255082 osd.6 (osd.6) 191 : cluster 3 failed to encode map e247 with expected crc 2023-06-03T20:22:59.474 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:22:59 smithi164 bash[59462]: cluster 1685823778.2582924 osd.5 (osd.5) 185 : cluster 3 failed to encode map e247 with expected crc 2023-06-03T20:22:59.474 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:22:59 smithi164 bash[59462]: audit 1685823778.259393 mon.a (mon.0) 1390 : audit 1 from='client.? 172.21.15.89:0/1485137449' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-06-03T20:22:59.474 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:22:59 smithi164 bash[59462]: cluster 1685823778.2595186 mon.a (mon.0) 1391 : cluster 0 osdmap e247: 8 total, 7 up, 7 in 2023-06-03T20:22:59.500 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:22:59.502 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:22:59.503 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:22:59.529 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:22:59 smithi089 bash[75068]: cluster 1685823778.2401347 mon.a (mon.0) 1389 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-06-03T20:22:59.529 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:22:59 smithi089 bash[75068]: cluster 1685823778.255082 osd.6 (osd.6) 191 : cluster 3 failed to encode map e247 with expected crc 2023-06-03T20:22:59.530 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:22:59 smithi089 bash[75068]: cluster 1685823778.2582924 osd.5 (osd.5) 185 : cluster 3 failed to encode map e247 with expected crc 2023-06-03T20:22:59.530 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:22:59 smithi089 bash[75068]: audit 1685823778.259393 mon.a (mon.0) 1390 : audit 1 from='client.? 172.21.15.89:0/1485137449' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-06-03T20:22:59.530 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:22:59 smithi089 bash[75068]: cluster 1685823778.2595186 mon.a (mon.0) 1391 : cluster 0 osdmap e247: 8 total, 7 up, 7 in 2023-06-03T20:22:59.531 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:22:59 smithi089 bash[78452]: cluster 1685823778.2401347 mon.a (mon.0) 1389 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-06-03T20:22:59.531 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:22:59 smithi089 bash[78452]: cluster 1685823778.255082 osd.6 (osd.6) 191 : cluster 3 failed to encode map e247 with expected crc 2023-06-03T20:22:59.531 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:22:59 smithi089 bash[78452]: cluster 1685823778.2582924 osd.5 (osd.5) 185 : cluster 3 failed to encode map e247 with expected crc 2023-06-03T20:22:59.531 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:22:59 smithi089 bash[78452]: audit 1685823778.259393 mon.a (mon.0) 1390 : audit 1 from='client.? 172.21.15.89:0/1485137449' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-06-03T20:22:59.531 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:22:59 smithi089 bash[78452]: cluster 1685823778.2595186 mon.a (mon.0) 1391 : cluster 0 osdmap e247: 8 total, 7 up, 7 in 2023-06-03T20:22:59.604 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:22:59.608 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 66 16 94024 94008 89.0093 61.6875 0.0164311 0.0112294 2023-06-03T20:22:59.636 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:22:59.638 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:22:59.639 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:22:59.740 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:22:59.768 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:22:59.772 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:22:59.773 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:22:59.874 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:22:59.917 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:22:59.921 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:22:59.921 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:23:00.022 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:23:00.059 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:23:00.064 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:23:00.065 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:23:00.165 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:23:00.196 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:23:00.198 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:23:00.198 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:23:00.299 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:23:00.337 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:23:00.340 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:23:00.341 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:23:00.442 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:23:00.454 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:23:00 smithi089 bash[75068]: cluster 1685823779.00063 mgr.y (mgr.34104) 1412 : cluster 0 pgmap v1426: 89 pgs: 17 active+clean+remapped, 13 active+undersized+degraded, 5 active+undersized, 54 active+clean; 5.6 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 61 MiB/s wr, 970 op/s; 51083/273678 objects degraded (18.665%); 6/273678 objects misplaced (0.002%) 2023-06-03T20:23:00.454 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:23:00 smithi089 bash[78452]: cluster 1685823779.00063 mgr.y (mgr.34104) 1412 : cluster 0 pgmap v1426: 89 pgs: 17 active+clean+remapped, 13 active+undersized+degraded, 5 active+undersized, 54 active+clean; 5.6 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 61 MiB/s wr, 970 op/s; 51083/273678 objects degraded (18.665%); 6/273678 objects misplaced (0.002%) 2023-06-03T20:23:00.479 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:23:00.485 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:23:00.486 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:23:00.584 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:23:00 smithi164 bash[59462]: cluster 1685823779.00063 mgr.y (mgr.34104) 1412 : cluster 0 pgmap v1426: 89 pgs: 17 active+clean+remapped, 13 active+undersized+degraded, 5 active+undersized, 54 active+clean; 5.6 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 61 MiB/s wr, 970 op/s; 51083/273678 objects degraded (18.665%); 6/273678 objects misplaced (0.002%) 2023-06-03T20:23:00.587 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:23:00.609 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 67 15 95036 95021 88.6257 63.3125 0.0270918 0.0112776 2023-06-03T20:23:00.626 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:23:00.632 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:23:00.632 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:23:00.734 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph daemon osd.1 dump_historic_ops 2023-06-03T20:23:00.734 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:23:00.773 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:23:00.776 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:23:00.777 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:23:00.877 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:23:00.910 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:23:00.913 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:23:00.913 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:23:01.014 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:23:01.047 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:23:01.049 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:23:01.050 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:23:01.151 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:23:01.184 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:23:01.187 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:23:01.187 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:23:01.282 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:23:01 smithi164 bash[59462]: cluster 1685823781.001442 mgr.y (mgr.34104) 1413 : cluster 0 pgmap v1427: 89 pgs: 17 active+clean+remapped, 13 active+undersized+degraded, 5 active+undersized, 54 active+clean; 5.7 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 60 MiB/s wr, 964 op/s; 52221/279855 objects degraded (18.660%); 6/279855 objects misplaced (0.002%) 2023-06-03T20:23:01.288 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:23:01.314 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:23:01.319 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:23:01.320 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:23:01.420 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:23:01.472 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:23:01.478 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:23:01.479 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:23:01.580 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:23:01.609 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 68 16 96042 96026 88.246 62.8125 0.00565303 0.0113264 2023-06-03T20:23:01.633 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:23:01.637 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:23:01.638 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:23:01.739 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:23:01.776 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:23:01.778 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:23:01.779 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:23:01.780 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:23:01 smithi089 bash[75068]: cluster 1685823781.001442 mgr.y (mgr.34104) 1413 : cluster 0 pgmap v1427: 89 pgs: 17 active+clean+remapped, 13 active+undersized+degraded, 5 active+undersized, 54 active+clean; 5.7 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 60 MiB/s wr, 964 op/s; 52221/279855 objects degraded (18.660%); 6/279855 objects misplaced (0.002%) 2023-06-03T20:23:01.780 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:23:01 smithi089 bash[78452]: cluster 1685823781.001442 mgr.y (mgr.34104) 1413 : cluster 0 pgmap v1427: 89 pgs: 17 active+clean+remapped, 13 active+undersized+degraded, 5 active+undersized, 54 active+clean; 5.7 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 60 MiB/s wr, 964 op/s; 52221/279855 objects degraded (18.660%); 6/279855 objects misplaced (0.002%) 2023-06-03T20:23:01.880 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:23:01.912 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:23:01.918 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:23:01.919 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:23:02.020 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:23:02.050 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:23:02.053 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:23:02.054 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:23:02.155 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:23:02.186 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:23:02.187 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:23:02.188 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:23:02.289 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:23:02.336 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:23:02.339 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:23:02.340 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:23:02.441 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:23:02.482 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:23:02.485 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:23:02.486 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:23:02.587 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:23:02.609 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 69 16 96801 96785 87.6546 47.4375 0.0204091 0.0114025 2023-06-03T20:23:02.610 DEBUG:teuthology.orchestra.run:got remote process result: 2 2023-06-03T20:23:02.636 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:23:02.640 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:23:02.641 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:23:02.698 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-06-03T20:23:02.699 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-06-03T20:23:02.742 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:23:02.773 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:23:02.778 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:23:02.780 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:23:02.881 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:23:02.912 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:23:02.919 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:23:02.920 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:23:03.021 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:23:03.052 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:23:03.056 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:23:03.057 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:23:03.084 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:23:02 smithi164 bash[59462]: cluster 1685823782.9114432 mon.a (mon.0) 1392 : cluster 3 Health check update: Degraded data redundancy: 52221/279855 objects degraded (18.660%), 13 pgs degraded, 18 pgs undersized (PG_DEGRADED) 2023-06-03T20:23:03.157 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:23:03.168 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:23:02 smithi089 bash[78452]: cluster 1685823782.9114432 mon.a (mon.0) 1392 : cluster 3 Health check update: Degraded data redundancy: 52221/279855 objects degraded (18.660%), 13 pgs degraded, 18 pgs undersized (PG_DEGRADED) 2023-06-03T20:23:03.168 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:23:02 smithi089 bash[75068]: cluster 1685823782.9114432 mon.a (mon.0) 1392 : cluster 3 Health check update: Degraded data redundancy: 52221/279855 objects degraded (18.660%), 13 pgs degraded, 18 pgs undersized (PG_DEGRADED) 2023-06-03T20:23:03.199 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:23:03.205 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:23:03.206 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:23:03.308 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:23:03.347 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:23:03.349 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:23:03.352 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:23:03.454 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:23:03.491 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:23:03.493 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:23:03.494 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:23:03.595 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:23:03.609 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 70 16 97829 97813 87.3198 64.25 0.0335962 0.011446 2023-06-03T20:23:03.638 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:23:03.641 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:23:03.641 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:23:03.742 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:23:03.773 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:23:03.775 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:23:03.776 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:23:03.877 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:23:03.909 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:23:03.912 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:23:03.912 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:23:03.985 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:23:03 smithi089 bash[75068]: cluster 1685823783.00232 mgr.y (mgr.34104) 1414 : cluster 0 pgmap v1428: 89 pgs: 17 active+clean+remapped, 13 active+undersized+degraded, 5 active+undersized, 54 active+clean; 5.8 GiB data, 18 GiB used, 607 GiB / 626 GiB avail; 60 MiB/s wr, 964 op/s; 53156/284424 objects degraded (18.689%); 6/284424 objects misplaced (0.002%) 2023-06-03T20:23:04.014 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:23:04.053 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:23:04.061 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:23:04.062 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:23:04.163 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:23:04.171 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:23:03 smithi164 bash[59462]: cluster 1685823783.00232 mgr.y (mgr.34104) 1414 : cluster 0 pgmap v1428: 89 pgs: 17 active+clean+remapped, 13 active+undersized+degraded, 5 active+undersized, 54 active+clean; 5.8 GiB data, 18 GiB used, 607 GiB / 626 GiB avail; 60 MiB/s wr, 964 op/s; 53156/284424 objects degraded (18.689%); 6/284424 objects misplaced (0.002%) 2023-06-03T20:23:04.198 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:23:04.205 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:23:04.206 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:23:04.279 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:23:03 smithi089 bash[78452]: cluster 1685823783.00232 mgr.y (mgr.34104) 1414 : cluster 0 pgmap v1428: 89 pgs: 17 active+clean+remapped, 13 active+undersized+degraded, 5 active+undersized, 54 active+clean; 5.8 GiB data, 18 GiB used, 607 GiB / 626 GiB avail; 60 MiB/s wr, 964 op/s; 53156/284424 objects degraded (18.689%); 6/284424 objects misplaced (0.002%) 2023-06-03T20:23:04.307 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:23:04.337 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:23:04.344 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:23:04.345 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:23:04.447 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:23:04.502 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:23:04.506 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:23:04.506 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:23:04.607 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:23:04.609 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 71 16 98791 98775 86.9367 60.125 0.03344 0.0114967 2023-06-03T20:23:04.642 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:23:04.647 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:23:04.647 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:23:04.700 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-06-03T20:23:04.715 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph osd unset nodeep-scrub 2023-06-03T20:23:04.749 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:23:04.779 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:23:04.781 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:23:04.781 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:23:04.882 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:23:04.915 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:23:04.916 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:23:04.917 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:23:05.018 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:23:05.057 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:23:05.067 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:23:05.068 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:23:05.169 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:23:05.204 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:23:05.207 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:23:05.207 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:23:05.308 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:23:05.343 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:23:05.345 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:23:05.346 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:23:05.447 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:23:05.489 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:23:05.494 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:23:05.494 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:23:05.595 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:23:05.609 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 72 16 99691 99675 86.5105 56.25 0.0104531 0.0115535 2023-06-03T20:23:05.633 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:23:05.638 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:23:05.639 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:23:05.740 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:23:05.784 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:23:05.787 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:23:05.788 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:23:05.889 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:23:05.938 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:23:05.946 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:23:05.947 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:23:05.970 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/mon.a/config 2023-06-03T20:23:06.049 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:23:06.090 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:23:06.094 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:23:06.094 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:23:06.195 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:23:06.204 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:23:06 smithi089 bash[75068]: cluster 1685823785.00546 mgr.y (mgr.34104) 1415 : cluster 0 pgmap v1429: 89 pgs: 17 active+clean+remapped, 13 active+undersized+degraded, 5 active+undersized, 54 active+clean; 5.9 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 60 MiB/s wr, 955 op/s; 54359/290733 objects degraded (18.697%); 6/290733 objects misplaced (0.002%) 2023-06-03T20:23:06.205 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:23:06 smithi089 bash[75068]: audit 1685823785.6326544 mon.c (mon.2) 598 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-03T20:23:06.205 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:23:06 smithi089 bash[78452]: cluster 1685823785.00546 mgr.y (mgr.34104) 1415 : cluster 0 pgmap v1429: 89 pgs: 17 active+clean+remapped, 13 active+undersized+degraded, 5 active+undersized, 54 active+clean; 5.9 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 60 MiB/s wr, 955 op/s; 54359/290733 objects degraded (18.697%); 6/290733 objects misplaced (0.002%) 2023-06-03T20:23:06.206 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:23:06 smithi089 bash[78452]: audit 1685823785.6326544 mon.c (mon.2) 598 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-03T20:23:06.226 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:23:06.228 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:23:06.228 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:23:06.279 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 20:23:06 smithi089 bash[71163]: ::ffff:172.21.15.164 - - [03/Jun/2023:20:23:06] "GET /metrics HTTP/1.1" 200 229013 "" "Prometheus/2.33.4" 2023-06-03T20:23:06.330 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:23:06.334 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:23:06 smithi164 bash[59462]: cluster 1685823785.00546 mgr.y (mgr.34104) 1415 : cluster 0 pgmap v1429: 89 pgs: 17 active+clean+remapped, 13 active+undersized+degraded, 5 active+undersized, 54 active+clean; 5.9 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 60 MiB/s wr, 955 op/s; 54359/290733 objects degraded (18.697%); 6/290733 objects misplaced (0.002%) 2023-06-03T20:23:06.335 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:23:06 smithi164 bash[59462]: audit 1685823785.6326544 mon.c (mon.2) 598 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-03T20:23:06.367 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:23:06.369 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:23:06.370 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:23:06.471 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:23:06.496 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:23:06.498 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:23:06.498 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:23:06.600 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:23:06.610 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 73 16 100466 100450 85.9889 48.4375 0.00656398 0.0116239 2023-06-03T20:23:06.641 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:23:06.646 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:23:06.647 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:23:06.748 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:23:06.780 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:23:06.785 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:23:06.785 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:23:06.887 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:23:06.932 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:23:06.935 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:23:06.935 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:23:07.036 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:23:07.068 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:23:07.070 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:23:07.071 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:23:07.172 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:23:07.206 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:23:07.210 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:23:07.211 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:23:07.312 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:23:07.353 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:23:07.358 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:23:07.358 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:23:07.459 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:23:07.499 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:23:07.504 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:23:07.505 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:23:07.606 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:23:07.610 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 74 16 101429 101413 85.6401 60.1875 0.00754499 0.0116714 2023-06-03T20:23:07.612 INFO:tasks.thrashosds.thrasher:in_osds: [0, 2, 3, 4, 5, 7, 1] out_osds: [6] dead_osds: [4] live_osds: [0, 1, 2, 3, 5, 6, 7] 2023-06-03T20:23:07.612 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-06-03T20:23:07.613 INFO:tasks.thrashosds.thrasher:Reviving osd 4 2023-06-03T20:23:07.613 INFO:tasks.cephadm.osd.4:Restarting osd.4 (starting--it wasn't running)... 2023-06-03T20:23:07.613 DEBUG:teuthology.orchestra.run.smithi164:osd.4> sudo journalctl -f -n 0 -u ceph-f1db3478-0243-11ee-9b1d-001a4aab830c@osd.4.service 2023-06-03T20:23:07.645 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:23:07.648 DEBUG:teuthology.orchestra.run.smithi164:> sudo systemctl start ceph-f1db3478-0243-11ee-9b1d-001a4aab830c@osd.4 2023-06-03T20:23:07.651 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:23:07.654 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:23:07.724 DEBUG:teuthology.orchestra.run.smithi164:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph daemon osd.4 dump_ops_in_flight 2023-06-03T20:23:07.741 INFO:journalctl@ceph.osd.4.smithi164.stdout:-- Logs begin at Sun 2022-08-14 12:20:24 UTC. -- 2023-06-03T20:23:07.755 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:23:07.792 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:23:07.794 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:23:07.795 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:23:07.896 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:23:07.935 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:23:07.937 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:23:07.938 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:23:08.039 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:23:08.070 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:23:08 smithi164 bash[59462]: cluster 1685823787.0060065 mgr.y (mgr.34104) 1416 : cluster 0 pgmap v1430: 89 pgs: 17 active+clean+remapped, 13 active+undersized+degraded, 5 active+undersized, 54 active+clean; 6.0 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 59 MiB/s wr, 951 op/s; 54952/294252 objects degraded (18.675%); 6/294252 objects misplaced (0.002%) 2023-06-03T20:23:08.070 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:23:08 smithi164 bash[59462]: audit 1685823787.1356373 mon.c (mon.2) 599 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T20:23:08.070 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:23:08 smithi164 bash[59462]: cluster 1685823787.9126015 mon.a (mon.0) 1393 : cluster 3 Health check update: Degraded data redundancy: 54952/294252 objects degraded (18.675%), 13 pgs degraded, 18 pgs undersized (PG_DEGRADED) 2023-06-03T20:23:08.076 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:23:08.079 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:23:08.080 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:23:08.182 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:23:08.214 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:23:08.220 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:23:08.220 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:23:08.321 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:23:08.334 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:23:08 smithi089 bash[78452]: cluster 1685823787.0060065 mgr.y (mgr.34104) 1416 : cluster 0 pgmap v1430: 89 pgs: 17 active+clean+remapped, 13 active+undersized+degraded, 5 active+undersized, 54 active+clean; 6.0 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 59 MiB/s wr, 951 op/s; 54952/294252 objects degraded (18.675%); 6/294252 objects misplaced (0.002%) 2023-06-03T20:23:08.336 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:23:08 smithi089 bash[78452]: audit 1685823787.1356373 mon.c (mon.2) 599 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T20:23:08.336 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:23:08 smithi089 bash[78452]: cluster 1685823787.9126015 mon.a (mon.0) 1393 : cluster 3 Health check update: Degraded data redundancy: 54952/294252 objects degraded (18.675%), 13 pgs degraded, 18 pgs undersized (PG_DEGRADED) 2023-06-03T20:23:08.337 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:23:08 smithi089 bash[75068]: cluster 1685823787.0060065 mgr.y (mgr.34104) 1416 : cluster 0 pgmap v1430: 89 pgs: 17 active+clean+remapped, 13 active+undersized+degraded, 5 active+undersized, 54 active+clean; 6.0 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 59 MiB/s wr, 951 op/s; 54952/294252 objects degraded (18.675%); 6/294252 objects misplaced (0.002%) 2023-06-03T20:23:08.337 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:23:08 smithi089 bash[75068]: audit 1685823787.1356373 mon.c (mon.2) 599 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T20:23:08.338 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:23:08 smithi089 bash[75068]: cluster 1685823787.9126015 mon.a (mon.0) 1393 : cluster 3 Health check update: Degraded data redundancy: 54952/294252 objects degraded (18.675%), 13 pgs degraded, 18 pgs undersized (PG_DEGRADED) 2023-06-03T20:23:08.356 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:23:08.361 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:23:08.361 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:23:08.462 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:23:08.501 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:23:08.506 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:23:08.507 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:23:08.608 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:23:08.610 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 75 16 102370 102354 85.2824 58.8125 0.0054198 0.0117202 2023-06-03T20:23:08.643 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:23:08.649 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:23:08.650 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:23:08.751 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:23:08.785 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:23:08.786 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:23:08.787 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:23:08.888 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:23:08.931 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:23:08.933 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:23:08.934 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:23:09.035 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:23:09.064 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:23:09.070 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:23:09.071 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:23:09.172 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:23:09.204 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:23:09.206 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:23:09.207 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:23:09.307 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:23:09.341 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:23:09.345 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:23:09.347 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:23:09.448 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:23:09.477 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:23:09.484 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:23:09.484 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:23:09.585 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:23:09.610 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 76 16 103317 103301 84.939 59.1875 0.0136053 0.011768 2023-06-03T20:23:09.627 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:23:09.632 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:23:09.632 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:23:09.733 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:23:09.771 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:23:09.774 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:23:09.775 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:23:09.876 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:23:09.909 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:23:09.914 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:23:09.915 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:23:10.016 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:23:10.048 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:23:10.050 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:23:10.051 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:23:10.151 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:23:10.190 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:23:10.195 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:23:10.195 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:23:10.296 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:23:10.334 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:23:10 smithi164 bash[59462]: cluster 1685823789.0068772 mgr.y (mgr.34104) 1417 : cluster 0 pgmap v1431: 89 pgs: 17 active+clean+remapped, 13 active+undersized+degraded, 5 active+undersized, 54 active+clean; 6.1 GiB data, 19 GiB used, 606 GiB / 626 GiB avail; 61 MiB/s wr, 983 op/s; 56411/301920 objects degraded (18.684%); 6/301920 objects misplaced (0.002%) 2023-06-03T20:23:10.335 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:23:10.341 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:23:10.342 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:23:10.404 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/mon.a/config 2023-06-03T20:23:10.405 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/mon.a/config 2023-06-03T20:23:10.429 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:23:10 smithi089 bash[75068]: cluster 1685823789.0068772 mgr.y (mgr.34104) 1417 : cluster 0 pgmap v1431: 89 pgs: 17 active+clean+remapped, 13 active+undersized+degraded, 5 active+undersized, 54 active+clean; 6.1 GiB data, 19 GiB used, 606 GiB / 626 GiB avail; 61 MiB/s wr, 983 op/s; 56411/301920 objects degraded (18.684%); 6/301920 objects misplaced (0.002%) 2023-06-03T20:23:10.430 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:23:10 smithi089 bash[78452]: cluster 1685823789.0068772 mgr.y (mgr.34104) 1417 : cluster 0 pgmap v1431: 89 pgs: 17 active+clean+remapped, 13 active+undersized+degraded, 5 active+undersized, 54 active+clean; 6.1 GiB data, 19 GiB used, 606 GiB / 626 GiB avail; 61 MiB/s wr, 983 op/s; 56411/301920 objects degraded (18.684%); 6/301920 objects misplaced (0.002%) 2023-06-03T20:23:10.442 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:23:10.472 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:23:10.475 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:23:10.476 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:23:10.577 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:23:10.610 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 77 16 104215 104199 84.5647 56.125 0.0118544 0.0118192 2023-06-03T20:23:10.642 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:23:10.644 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:23:10.645 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:23:10.746 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:23:10.754 INFO:journalctl@ceph.osd.4.smithi164.stdout:Jun 03 20:23:10 smithi164 bash[130347]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-4 2023-06-03T20:23:10.754 INFO:journalctl@ceph.osd.4.smithi164.stdout:Jun 03 20:23:10 smithi164 bash[130347]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-4 --no-mon-config --dev /dev/mapper/vg_nvme-lv_4 2023-06-03T20:23:10.754 INFO:journalctl@ceph.osd.4.smithi164.stdout:Jun 03 20:23:10 smithi164 bash[130347]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_4 2023-06-03T20:23:10.754 INFO:journalctl@ceph.osd.4.smithi164.stdout:Jun 03 20:23:10 smithi164 bash[130347]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-3 2023-06-03T20:23:10.754 INFO:journalctl@ceph.osd.4.smithi164.stdout:Jun 03 20:23:10 smithi164 bash[130347]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_4 /var/lib/ceph/osd/ceph-4/block 2023-06-03T20:23:10.754 INFO:journalctl@ceph.osd.4.smithi164.stdout:Jun 03 20:23:10 smithi164 bash[130347]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-4 2023-06-03T20:23:10.755 INFO:journalctl@ceph.osd.4.smithi164.stdout:Jun 03 20:23:10 smithi164 bash[130347]: --> ceph-volume raw activate successful for osd ID: 4 2023-06-03T20:23:10.777 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:23:10.779 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:23:10.779 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:23:10.880 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:23:10.912 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:23:10.914 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:23:10.914 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:23:11.015 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:23:11.052 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:23:11.064 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:23:11.065 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:23:11.166 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:23:11.213 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:23:11.216 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:23:11.217 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:23:11.318 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:23:11.350 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:23:11.351 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:23:11.351 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:23:11.452 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:23:11.509 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:23:11.513 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:23:11.514 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:23:11.610 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 78 16 105026 105010 84.1304 50.6875 0.0157102 0.0118807 2023-06-03T20:23:11.615 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:23:11.645 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:23:11.648 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:23:11.648 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:23:11.749 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:23:11.781 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:23:11.783 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:23:11.784 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:23:11.885 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:23:11.923 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:23:11.927 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:23:11.928 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:23:12.000 INFO:teuthology.orchestra.run.smithi164.stderr:Inferring config /var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/mon.b/config 2023-06-03T20:23:12.029 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:23:12.058 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:23:12.068 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:23:12.069 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:23:12.169 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:23:12.214 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:23:12.219 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:23:12.219 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:23:12.279 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:23:12 smithi089 bash[75068]: cluster 1685823791.0077932 mgr.y (mgr.34104) 1418 : cluster 0 pgmap v1432: 89 pgs: 17 active+clean+remapped, 13 active+undersized+degraded, 5 active+undersized, 54 active+clean; 6.3 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 59 MiB/s wr, 944 op/s; 57501/307716 objects degraded (18.686%); 6/307716 objects misplaced (0.002%) 2023-06-03T20:23:12.280 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:23:12 smithi089 bash[75068]: audit 1685823791.9586594 mon.a (mon.0) 1394 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:23:12.280 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:23:12 smithi089 bash[75068]: audit 1685823791.9668999 mon.a (mon.0) 1395 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:23:12.281 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:23:12 smithi089 bash[78452]: cluster 1685823791.0077932 mgr.y (mgr.34104) 1418 : cluster 0 pgmap v1432: 89 pgs: 17 active+clean+remapped, 13 active+undersized+degraded, 5 active+undersized, 54 active+clean; 6.3 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 59 MiB/s wr, 944 op/s; 57501/307716 objects degraded (18.686%); 6/307716 objects misplaced (0.002%) 2023-06-03T20:23:12.281 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:23:12 smithi089 bash[78452]: audit 1685823791.9586594 mon.a (mon.0) 1394 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:23:12.281 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:23:12 smithi089 bash[78452]: audit 1685823791.9668999 mon.a (mon.0) 1395 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:23:12.320 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:23:12.334 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:23:12 smithi164 bash[59462]: cluster 1685823791.0077932 mgr.y (mgr.34104) 1418 : cluster 0 pgmap v1432: 89 pgs: 17 active+clean+remapped, 13 active+undersized+degraded, 5 active+undersized, 54 active+clean; 6.3 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 59 MiB/s wr, 944 op/s; 57501/307716 objects degraded (18.686%); 6/307716 objects misplaced (0.002%) 2023-06-03T20:23:12.334 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:23:12 smithi164 bash[59462]: audit 1685823791.9586594 mon.a (mon.0) 1394 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:23:12.335 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:23:12 smithi164 bash[59462]: audit 1685823791.9668999 mon.a (mon.0) 1395 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:23:12.362 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:23:12.363 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:23:12.363 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:23:12.464 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:23:12.511 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:23:12.519 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:23:12.520 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:23:12.610 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 79 16 105865 105849 83.7291 52.4375 0.00637823 0.0119366 2023-06-03T20:23:12.622 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:23:12.660 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:23:12.668 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:23:12.669 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:23:12.769 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:23:12.802 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:23:12.806 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:23:12.807 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:23:12.908 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:23:12.964 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:23:12.968 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:23:12.969 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:23:13.069 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:23:13.111 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:23:13.115 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:23:13.116 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:23:13.216 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:23:13.249 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:23:13.253 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:23:13.254 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:23:13.334 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:23:13 smithi164 bash[59462]: cluster 1685823792.9144907 mon.a (mon.0) 1396 : cluster 3 Health check update: Degraded data redundancy: 57501/307716 objects degraded (18.686%), 13 pgs degraded, 18 pgs undersized (PG_DEGRADED) 2023-06-03T20:23:13.355 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:23:13.384 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:23:13.388 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:23:13.390 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:23:13.407 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:23:13 smithi089 bash[75068]: cluster 1685823792.9144907 mon.a (mon.0) 1396 : cluster 3 Health check update: Degraded data redundancy: 57501/307716 objects degraded (18.686%), 13 pgs degraded, 18 pgs undersized (PG_DEGRADED) 2023-06-03T20:23:13.407 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:23:13 smithi089 bash[78452]: cluster 1685823792.9144907 mon.a (mon.0) 1396 : cluster 3 Health check update: Degraded data redundancy: 57501/307716 objects degraded (18.686%), 13 pgs degraded, 18 pgs undersized (PG_DEGRADED) 2023-06-03T20:23:13.491 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:23:13.549 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:23:13.549 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:23:13.550 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:23:13.610 INFO:tasks.radosbench.radosbench.0.smithi089.stdout:2023-06-03T20:23:13.606246+0000 min lat: 0.0012167 max lat: 0.341691 avg lat: 0.0119838 2023-06-03T20:23:13.610 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-06-03T20:23:13.611 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 80 16 106790 106774 83.4051 57.8125 0.0128108 0.0119838 2023-06-03T20:23:13.650 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:23:13.685 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:23:13.693 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:23:13.694 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:23:13.795 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:23:13.841 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:23:13.843 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:23:13.844 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:23:13.944 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:23:13.975 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:23:13.979 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:23:13.979 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:23:14.080 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:23:14.083 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:23:14 smithi164 bash[59462]: cluster 1685823793.008607 mgr.y (mgr.34104) 1419 : cluster 0 pgmap v1433: 89 pgs: 17 active+clean+remapped, 13 active+undersized+degraded, 5 active+undersized, 54 active+clean; 6.3 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 56 MiB/s wr, 892 op/s; 58327/312018 objects degraded (18.693%); 6/312018 objects misplaced (0.002%) 2023-06-03T20:23:14.083 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:23:14 smithi164 bash[59462]: audit 1685823793.5718153 mon.a (mon.0) 1397 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:23:14.084 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:23:14 smithi164 bash[59462]: audit 1685823793.582033 mon.a (mon.0) 1398 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:23:14.127 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:23:14.130 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:23:14.131 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:23:14.232 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:23:14.242 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:23:14 smithi089 bash[78452]: cluster 1685823793.008607 mgr.y (mgr.34104) 1419 : cluster 0 pgmap v1433: 89 pgs: 17 active+clean+remapped, 13 active+undersized+degraded, 5 active+undersized, 54 active+clean; 6.3 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 56 MiB/s wr, 892 op/s; 58327/312018 objects degraded (18.693%); 6/312018 objects misplaced (0.002%) 2023-06-03T20:23:14.242 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:23:14 smithi089 bash[78452]: audit 1685823793.5718153 mon.a (mon.0) 1397 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:23:14.242 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:23:14 smithi089 bash[78452]: audit 1685823793.582033 mon.a (mon.0) 1398 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:23:14.242 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:23:14 smithi089 bash[75068]: cluster 1685823793.008607 mgr.y (mgr.34104) 1419 : cluster 0 pgmap v1433: 89 pgs: 17 active+clean+remapped, 13 active+undersized+degraded, 5 active+undersized, 54 active+clean; 6.3 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 56 MiB/s wr, 892 op/s; 58327/312018 objects degraded (18.693%); 6/312018 objects misplaced (0.002%) 2023-06-03T20:23:14.243 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:23:14 smithi089 bash[75068]: audit 1685823793.5718153 mon.a (mon.0) 1397 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:23:14.243 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:23:14 smithi089 bash[75068]: audit 1685823793.582033 mon.a (mon.0) 1398 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:23:14.271 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:23:14.275 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:23:14.276 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:23:14.377 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:23:14.404 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:23:14.410 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:23:14.411 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:23:14.512 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:23:14.549 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:23:14.555 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:23:14.556 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:23:14.610 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 81 16 107702 107686 83.0791 57 0.0146971 0.0120316 2023-06-03T20:23:14.657 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:23:14.695 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:23:14.698 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:23:14.699 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:23:14.800 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:23:14.830 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:23:14.832 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:23:14.833 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:23:14.934 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:23:14.962 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:23:14.970 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:23:14.970 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:23:15.071 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:23:15.112 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:23:15.115 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:23:15.115 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:23:15.216 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:23:15.240 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:23:15.242 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:23:15.242 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:23:15.279 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:23:15 smithi089 bash[75068]: audit 1685823794.1671612 mon.c (mon.2) 600 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-03T20:23:15.279 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:23:15 smithi089 bash[75068]: audit 1685823794.1711135 mon.c (mon.2) 601 : audit 1 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-03T20:23:15.279 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:23:15 smithi089 bash[75068]: audit 1685823794.183163 mon.a (mon.0) 1399 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:23:15.280 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:23:15 smithi089 bash[78452]: audit 1685823794.1671612 mon.c (mon.2) 600 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-03T20:23:15.280 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:23:15 smithi089 bash[78452]: audit 1685823794.1711135 mon.c (mon.2) 601 : audit 1 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-03T20:23:15.280 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:23:15 smithi089 bash[78452]: audit 1685823794.183163 mon.a (mon.0) 1399 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:23:15.292 INFO:teuthology.orchestra.run.smithi089.stdout:ERROR: (22) Invalid argument 2023-06-03T20:23:15.292 INFO:teuthology.orchestra.run.smithi089.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-06-03T20:23:15.334 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:23:15 smithi164 bash[59462]: audit 1685823794.1671612 mon.c (mon.2) 600 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-03T20:23:15.334 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:23:15 smithi164 bash[59462]: audit 1685823794.1711135 mon.c (mon.2) 601 : audit 1 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-03T20:23:15.334 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:23:15 smithi164 bash[59462]: audit 1685823794.183163 mon.a (mon.0) 1399 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:23:15.344 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:23:15.610 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 82 16 108536 108520 82.7016 52.125 0.0209151 0.0120863 2023-06-03T20:23:15.821 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:23:15.824 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:23:15.824 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:23:15.925 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:23:15.963 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:23:15.966 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:23:15.967 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:23:16.068 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:23:16.104 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:23:16.107 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:23:16.108 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:23:16.209 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:23:16.248 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:23:16.251 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:23:16.252 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:23:16.334 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:23:16 smithi164 bash[59462]: cluster 1685823795.0093775 mgr.y (mgr.34104) 1420 : cluster 0 pgmap v1434: 89 pgs: 17 active+clean+remapped, 13 active+undersized+degraded, 5 active+undersized, 54 active+clean; 6.5 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 58 MiB/s wr, 930 op/s; 59461/317931 objects degraded (18.702%); 6/317931 objects misplaced (0.002%) 2023-06-03T20:23:16.353 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:23:16.388 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:23:16.392 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:23:16.393 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:23:16.494 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:23:16.502 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 20:23:16 smithi089 bash[71163]: ::ffff:172.21.15.164 - - [03/Jun/2023:20:23:16] "GET /metrics HTTP/1.1" 200 229008 "" "Prometheus/2.33.4" 2023-06-03T20:23:16.503 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:23:16 smithi089 bash[78452]: cluster 1685823795.0093775 mgr.y (mgr.34104) 1420 : cluster 0 pgmap v1434: 89 pgs: 17 active+clean+remapped, 13 active+undersized+degraded, 5 active+undersized, 54 active+clean; 6.5 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 58 MiB/s wr, 930 op/s; 59461/317931 objects degraded (18.702%); 6/317931 objects misplaced (0.002%) 2023-06-03T20:23:16.503 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:23:16 smithi089 bash[75068]: cluster 1685823795.0093775 mgr.y (mgr.34104) 1420 : cluster 0 pgmap v1434: 89 pgs: 17 active+clean+remapped, 13 active+undersized+degraded, 5 active+undersized, 54 active+clean; 6.5 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 58 MiB/s wr, 930 op/s; 59461/317931 objects degraded (18.702%); 6/317931 objects misplaced (0.002%) 2023-06-03T20:23:16.523 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:23:16.525 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:23:16.526 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:23:16.610 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 83 16 109477 109461 82.4137 58.8125 0.0203332 0.0121286 2023-06-03T20:23:16.627 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:23:16.659 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:23:16.661 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:23:16.662 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:23:16.763 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:23:16.801 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:23:16.804 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:23:16.805 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:23:16.906 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:23:16.933 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:23:16.935 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:23:16.936 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:23:17.037 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:23:17.077 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:23:17.082 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:23:17.083 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:23:17.184 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:23:17.226 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:23:17.229 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:23:17.230 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:23:17.331 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:23:17.364 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:23:17.370 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:23:17.371 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:23:17.472 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:23:17.504 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:23:17.509 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:23:17.511 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:23:17.610 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 84 16 110356 110340 82.0866 54.9375 0.0252936 0.0121767 2023-06-03T20:23:17.612 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:23:17.638 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:23:17.640 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:23:17.640 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:23:17.741 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:23:17.784 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:23:17.792 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:23:17.793 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:23:17.894 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:23:17.922 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:23:17.926 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:23:17.927 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:23:18.028 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:23:18.059 INFO:journalctl@ceph.osd.4.smithi164.stdout:Jun 03 20:23:17 smithi164 bash[130758]: debug 2023-06-03T20:23:17.724+0000 7fd6d68ca640 -1 Falling back to public interface 2023-06-03T20:23:18.066 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:23:18.067 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:23:18.068 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:23:18.169 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:23:18.203 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:23:18.211 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:23:18.212 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:23:18.313 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:23:18.322 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:23:18 smithi164 bash[59462]: cluster 1685823797.0101726 mgr.y (mgr.34104) 1421 : cluster 0 pgmap v1435: 89 pgs: 17 active+clean+remapped, 13 active+undersized+degraded, 5 active+undersized, 54 active+clean; 6.5 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 53 MiB/s wr, 845 op/s; 60006/321180 objects degraded (18.683%); 6/321180 objects misplaced (0.002%) 2023-06-03T20:23:18.323 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:23:18 smithi164 bash[59462]: cluster 1685823797.9155116 mon.a (mon.0) 1400 : cluster 3 Health check update: Degraded data redundancy: 60006/321180 objects degraded (18.683%), 13 pgs degraded, 18 pgs undersized (PG_DEGRADED) 2023-06-03T20:23:18.394 INFO:teuthology.orchestra.run.smithi164.stderr:no valid command found; 10 closest matches: 2023-06-03T20:23:18.395 INFO:teuthology.orchestra.run.smithi164.stderr:0 2023-06-03T20:23:18.396 INFO:teuthology.orchestra.run.smithi164.stderr:1 2023-06-03T20:23:18.396 INFO:teuthology.orchestra.run.smithi164.stderr:2 2023-06-03T20:23:18.397 INFO:teuthology.orchestra.run.smithi164.stderr:abort 2023-06-03T20:23:18.397 INFO:teuthology.orchestra.run.smithi164.stderr:assert 2023-06-03T20:23:18.397 INFO:teuthology.orchestra.run.smithi164.stderr:config diff 2023-06-03T20:23:18.397 INFO:teuthology.orchestra.run.smithi164.stderr:config diff get 2023-06-03T20:23:18.397 INFO:teuthology.orchestra.run.smithi164.stderr:config get 2023-06-03T20:23:18.398 INFO:teuthology.orchestra.run.smithi164.stderr:config help [] 2023-06-03T20:23:18.398 INFO:teuthology.orchestra.run.smithi164.stderr:config set ... 2023-06-03T20:23:18.398 INFO:teuthology.orchestra.run.smithi164.stderr:admin_socket: invalid command 2023-06-03T20:23:18.411 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:23:18.418 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:23:18.418 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:23:18.483 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:23:18 smithi089 bash[78452]: cluster 1685823797.0101726 mgr.y (mgr.34104) 1421 : cluster 0 pgmap v1435: 89 pgs: 17 active+clean+remapped, 13 active+undersized+degraded, 5 active+undersized, 54 active+clean; 6.5 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 53 MiB/s wr, 845 op/s; 60006/321180 objects degraded (18.683%); 6/321180 objects misplaced (0.002%) 2023-06-03T20:23:18.483 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:23:18 smithi089 bash[78452]: cluster 1685823797.9155116 mon.a (mon.0) 1400 : cluster 3 Health check update: Degraded data redundancy: 60006/321180 objects degraded (18.683%), 13 pgs degraded, 18 pgs undersized (PG_DEGRADED) 2023-06-03T20:23:18.487 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:23:18 smithi089 bash[75068]: cluster 1685823797.0101726 mgr.y (mgr.34104) 1421 : cluster 0 pgmap v1435: 89 pgs: 17 active+clean+remapped, 13 active+undersized+degraded, 5 active+undersized, 54 active+clean; 6.5 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 53 MiB/s wr, 845 op/s; 60006/321180 objects degraded (18.683%); 6/321180 objects misplaced (0.002%) 2023-06-03T20:23:18.488 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:23:18 smithi089 bash[75068]: cluster 1685823797.9155116 mon.a (mon.0) 1400 : cluster 3 Health check update: Degraded data redundancy: 60006/321180 objects degraded (18.683%), 13 pgs degraded, 18 pgs undersized (PG_DEGRADED) 2023-06-03T20:23:18.519 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:23:18.551 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:23:18.553 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:23:18.554 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:23:18.611 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 85 16 111165 111149 81.7157 50.5625 0.0446489 0.0122319 2023-06-03T20:23:18.628 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph daemon osd.2 dump_ops_in_flight 2023-06-03T20:23:18.655 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:23:18.690 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:23:18.695 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:23:18.695 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:23:18.796 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:23:18.830 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:23:18.835 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:23:18.836 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:23:18.937 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:23:18.981 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:23:18.984 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:23:18.984 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:23:19.085 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:23:19.119 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:23:19.120 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:23:19.122 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:23:19.223 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:23:19.260 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:23:19.264 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:23:19.265 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:23:19.366 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:23:19.405 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:23:19.407 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:23:19.408 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:23:19.486 DEBUG:teuthology.orchestra.run:got remote process result: 22 2023-06-03T20:23:19.487 INFO:tasks.cephadm.ceph_manager.ceph:waiting on admin_socket for osd-4, ['dump_ops_in_flight'] 2023-06-03T20:23:19.508 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:23:19.539 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:23:19.543 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:23:19.544 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:23:19.611 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 86 16 112090 112074 81.4377 57.8125 0.0275588 0.0122738 2023-06-03T20:23:19.645 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:23:19.682 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:23:19.686 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:23:19.687 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:23:19.788 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:23:19.816 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:23:19.820 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:23:19.821 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:23:19.922 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:23:19.965 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:23:19.970 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:23:19.971 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:23:20.071 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:23:20.101 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:23:20.106 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:23:20.107 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:23:20.208 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:23:20.242 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:23:20.246 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:23:20.247 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:23:20.347 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:23:20.388 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:23:20.392 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:23:20.393 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:23:20.493 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:23:20.502 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:23:20 smithi089 bash[75068]: cluster 1685823799.0127878 mgr.y (mgr.34104) 1422 : cluster 0 pgmap v1436: 89 pgs: 17 active+clean+remapped, 13 active+undersized+degraded, 5 active+undersized, 54 active+clean; 6.7 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 60 MiB/s wr, 956 op/s; 61409/328707 objects degraded (18.682%); 6/328707 objects misplaced (0.002%) 2023-06-03T20:23:20.503 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:23:20 smithi089 bash[78452]: cluster 1685823799.0127878 mgr.y (mgr.34104) 1422 : cluster 0 pgmap v1436: 89 pgs: 17 active+clean+remapped, 13 active+undersized+degraded, 5 active+undersized, 54 active+clean; 6.7 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 60 MiB/s wr, 956 op/s; 61409/328707 objects degraded (18.682%); 6/328707 objects misplaced (0.002%) 2023-06-03T20:23:20.521 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:23:20.523 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:23:20.523 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:23:20.584 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:23:20 smithi164 bash[59462]: cluster 1685823799.0127878 mgr.y (mgr.34104) 1422 : cluster 0 pgmap v1436: 89 pgs: 17 active+clean+remapped, 13 active+undersized+degraded, 5 active+undersized, 54 active+clean; 6.7 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 60 MiB/s wr, 956 op/s; 61409/328707 objects degraded (18.682%); 6/328707 objects misplaced (0.002%) 2023-06-03T20:23:20.611 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 87 16 113032 113016 81.1783 58.875 0.0140152 0.0123132 2023-06-03T20:23:20.624 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:23:20.652 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:23:20.657 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:23:20.658 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:23:20.759 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:23:20.787 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:23:20.789 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:23:20.790 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:23:20.891 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:23:20.923 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:23:20.925 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:23:20.926 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:23:21.027 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:23:21.054 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:23:21.057 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:23:21.057 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:23:21.158 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:23:21.195 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:23:21.201 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:23:21.201 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:23:21.302 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:23:21.342 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:23:21.347 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:23:21.347 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:23:21.448 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:23:21.489 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:23:21.494 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:23:21.495 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:23:21.596 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:23:21.611 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 88 16 113885 113869 80.8616 53.3125 0.0427488 0.0123611 2023-06-03T20:23:21.633 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:23:21.639 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:23:21.639 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:23:21.740 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:23:21.773 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:23:21.780 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:23:21.781 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:23:21.881 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:23:21.918 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:23:21.922 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:23:21.923 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:23:22.024 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:23:22.075 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:23:22.079 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:23:22.079 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:23:22.180 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:23:22.218 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:23:22.223 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:23:22.224 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:23:22.325 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:23:22.368 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:23:22.376 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:23:22.378 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:23:22.407 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:23:22 smithi089 bash[75068]: cluster 1685823801.0135674 mgr.y (mgr.34104) 1423 : cluster 0 pgmap v1437: 89 pgs: 17 active+clean+remapped, 13 active+undersized+degraded, 5 active+undersized, 54 active+clean; 6.7 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 51 MiB/s wr, 810 op/s; 61935/331113 objects degraded (18.705%); 6/331113 objects misplaced (0.002%) 2023-06-03T20:23:22.408 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:23:22 smithi089 bash[75068]: audit 1685823802.1364164 mon.c (mon.2) 602 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T20:23:22.408 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:23:22 smithi089 bash[78452]: cluster 1685823801.0135674 mgr.y (mgr.34104) 1423 : cluster 0 pgmap v1437: 89 pgs: 17 active+clean+remapped, 13 active+undersized+degraded, 5 active+undersized, 54 active+clean; 6.7 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 51 MiB/s wr, 810 op/s; 61935/331113 objects degraded (18.705%); 6/331113 objects misplaced (0.002%) 2023-06-03T20:23:22.408 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:23:22 smithi089 bash[78452]: audit 1685823802.1364164 mon.c (mon.2) 602 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T20:23:22.479 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:23:22.488 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:23:22 smithi164 bash[59462]: cluster 1685823801.0135674 mgr.y (mgr.34104) 1423 : cluster 0 pgmap v1437: 89 pgs: 17 active+clean+remapped, 13 active+undersized+degraded, 5 active+undersized, 54 active+clean; 6.7 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 51 MiB/s wr, 810 op/s; 61935/331113 objects degraded (18.705%); 6/331113 objects misplaced (0.002%) 2023-06-03T20:23:22.488 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:23:22 smithi164 bash[59462]: audit 1685823802.1364164 mon.c (mon.2) 602 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T20:23:22.557 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:23:22.563 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:23:22.564 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:23:22.611 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 89 16 114767 114751 80.5724 55.125 0.0135848 0.0124055 2023-06-03T20:23:22.665 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:23:22.701 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:23:22.709 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:23:22.709 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:23:22.810 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:23:22.846 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:23:22.851 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:23:22.852 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:23:22.855 INFO:teuthology.orchestra.run.smithi089.stdout:osd.0: {} 2023-06-03T20:23:22.900 INFO:teuthology.orchestra.run.smithi089.stdout:osd.1: {} 2023-06-03T20:23:22.949 INFO:teuthology.orchestra.run.smithi089.stdout:osd.2: {} 2023-06-03T20:23:22.952 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:23:23.001 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:23:23.005 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:23:23.005 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:23:23.023 INFO:teuthology.orchestra.run.smithi089.stdout:osd.3: {} 2023-06-03T20:23:23.060 INFO:teuthology.orchestra.run.smithi089.stdout:osd.5: {} 2023-06-03T20:23:23.096 INFO:teuthology.orchestra.run.smithi089.stdout:osd.6: {} 2023-06-03T20:23:23.106 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:23:23.141 INFO:teuthology.orchestra.run.smithi089.stdout:osd.7: {} 2023-06-03T20:23:23.142 INFO:teuthology.orchestra.run.smithi089.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-06-03T20:23:23.143 INFO:teuthology.orchestra.run.smithi089.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-06-03T20:23:23.143 INFO:teuthology.orchestra.run.smithi089.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-06-03T20:23:23.143 INFO:teuthology.orchestra.run.smithi089.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-06-03T20:23:23.143 INFO:teuthology.orchestra.run.smithi089.stderr:Error ENXIO: problem getting command descriptions from osd.4 2023-06-03T20:23:23.143 INFO:teuthology.orchestra.run.smithi089.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-06-03T20:23:23.143 INFO:teuthology.orchestra.run.smithi089.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-06-03T20:23:23.144 INFO:teuthology.orchestra.run.smithi089.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-06-03T20:23:23.144 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:23:23.147 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:23:23.148 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:23:23.215 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:23:23 smithi089 bash[75068]: audit 1685823802.7827792 mon.b (mon.1) 62 : audit 0 from='client.? 172.21.15.89:0/2340596235' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-03T20:23:23.215 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:23:23 smithi089 bash[75068]: cluster 1685823802.9165435 mon.a (mon.0) 1401 : cluster 3 Health check update: Degraded data redundancy: 61935/331113 objects degraded (18.705%), 13 pgs degraded, 18 pgs undersized (PG_DEGRADED) 2023-06-03T20:23:23.216 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:23:23 smithi089 bash[75068]: audit 1685823803.1159518 mon.c (mon.2) 603 : audit 1 from='client.? 172.21.15.89:0/4112647469' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-06-03T20:23:23.216 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:23:23 smithi089 bash[75068]: audit 1685823803.1170995 mon.a (mon.0) 1402 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-06-03T20:23:23.216 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:23:23 smithi089 bash[78452]: audit 1685823802.7827792 mon.b (mon.1) 62 : audit 0 from='client.? 172.21.15.89:0/2340596235' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-03T20:23:23.216 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:23:23 smithi089 bash[78452]: cluster 1685823802.9165435 mon.a (mon.0) 1401 : cluster 3 Health check update: Degraded data redundancy: 61935/331113 objects degraded (18.705%), 13 pgs degraded, 18 pgs undersized (PG_DEGRADED) 2023-06-03T20:23:23.217 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:23:23 smithi089 bash[78452]: audit 1685823803.1159518 mon.c (mon.2) 603 : audit 1 from='client.? 172.21.15.89:0/4112647469' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-06-03T20:23:23.217 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:23:23 smithi089 bash[78452]: audit 1685823803.1170995 mon.a (mon.0) 1402 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-06-03T20:23:23.249 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:23:23.301 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:23:23.307 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:23:23.308 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:23:23.409 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:23:23.446 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:23:23.452 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:23:23.453 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:23:23.554 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:23:23.584 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:23:23 smithi164 bash[59462]: audit 1685823802.7827792 mon.b (mon.1) 62 : audit 0 from='client.? 172.21.15.89:0/2340596235' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-03T20:23:23.584 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:23:23 smithi164 bash[59462]: cluster 1685823802.9165435 mon.a (mon.0) 1401 : cluster 3 Health check update: Degraded data redundancy: 61935/331113 objects degraded (18.705%), 13 pgs degraded, 18 pgs undersized (PG_DEGRADED) 2023-06-03T20:23:23.585 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:23:23 smithi164 bash[59462]: audit 1685823803.1159518 mon.c (mon.2) 603 : audit 1 from='client.? 172.21.15.89:0/4112647469' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-06-03T20:23:23.585 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:23:23 smithi164 bash[59462]: audit 1685823803.1170995 mon.a (mon.0) 1402 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-06-03T20:23:23.598 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:23:23.603 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:23:23.604 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:23:23.611 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 90 4 115522 115518 80.2098 47.9375 0.0179158 0.0124623 2023-06-03T20:23:23.705 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:23:23.743 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:23:23.746 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:23:23.746 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:23:23.847 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:23:23.881 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:23:23.883 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:23:23.884 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:23:23.942 INFO:teuthology.orchestra.run.smithi089.stderr:nodeep-scrub is unset 2023-06-03T20:23:23.985 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:23:24.014 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:23:24.016 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:23:24.016 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:23:24.117 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:23:24.147 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:23:24.149 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:23:24.150 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:23:24.251 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:23:24.286 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:23:24.288 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:23:24.289 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:23:24.390 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:23:24.404 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:23:24 smithi089 bash[78452]: cluster 1685823803.014405 mgr.y (mgr.34104) 1424 : cluster 0 pgmap v1438: 89 pgs: 17 active+clean+remapped, 13 active+undersized+degraded, 5 active+undersized, 54 active+clean; 6.9 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 54 MiB/s wr, 856 op/s; 63316/338562 objects degraded (18.701%); 6/338562 objects misplaced (0.002%) 2023-06-03T20:23:24.404 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:23:24 smithi089 bash[78452]: cluster 1685823803.91979 mon.a (mon.0) 1403 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-06-03T20:23:24.404 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:23:24 smithi089 bash[78452]: audit 1685823803.933217 mon.a (mon.0) 1404 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-06-03T20:23:24.405 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:23:24 smithi089 bash[78452]: cluster 1685823803.933317 mon.a (mon.0) 1405 : cluster 0 osdmap e248: 8 total, 7 up, 7 in 2023-06-03T20:23:24.405 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:23:24 smithi089 bash[75068]: cluster 1685823803.014405 mgr.y (mgr.34104) 1424 : cluster 0 pgmap v1438: 89 pgs: 17 active+clean+remapped, 13 active+undersized+degraded, 5 active+undersized, 54 active+clean; 6.9 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 54 MiB/s wr, 856 op/s; 63316/338562 objects degraded (18.701%); 6/338562 objects misplaced (0.002%) 2023-06-03T20:23:24.405 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:23:24 smithi089 bash[75068]: cluster 1685823803.91979 mon.a (mon.0) 1403 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-06-03T20:23:24.405 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:23:24 smithi089 bash[75068]: audit 1685823803.933217 mon.a (mon.0) 1404 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-06-03T20:23:24.405 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:23:24 smithi089 bash[75068]: cluster 1685823803.933317 mon.a (mon.0) 1405 : cluster 0 osdmap e248: 8 total, 7 up, 7 in 2023-06-03T20:23:24.429 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:23:24.432 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:23:24.433 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:23:24.488 DEBUG:teuthology.orchestra.run.smithi164:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph daemon osd.4 dump_ops_in_flight 2023-06-03T20:23:24.501 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:23:24 smithi164 bash[59462]: cluster 1685823803.014405 mgr.y (mgr.34104) 1424 : cluster 0 pgmap v1438: 89 pgs: 17 active+clean+remapped, 13 active+undersized+degraded, 5 active+undersized, 54 active+clean; 6.9 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 54 MiB/s wr, 856 op/s; 63316/338562 objects degraded (18.701%); 6/338562 objects misplaced (0.002%) 2023-06-03T20:23:24.501 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:23:24 smithi164 bash[59462]: cluster 1685823803.91979 mon.a (mon.0) 1403 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-06-03T20:23:24.501 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:23:24 smithi164 bash[59462]: audit 1685823803.933217 mon.a (mon.0) 1404 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-06-03T20:23:24.502 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:23:24 smithi164 bash[59462]: cluster 1685823803.933317 mon.a (mon.0) 1405 : cluster 0 osdmap e248: 8 total, 7 up, 7 in 2023-06-03T20:23:24.534 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:23:24.563 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:23:24.566 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:23:24.566 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:23:24.611 INFO:tasks.radosbench.radosbench.0.smithi089.stdout:Total time run: 90.0145 2023-06-03T20:23:24.612 INFO:tasks.radosbench.radosbench.0.smithi089.stdout:Total writes made: 115522 2023-06-03T20:23:24.612 INFO:tasks.radosbench.radosbench.0.smithi089.stdout:Write size: 65536 2023-06-03T20:23:24.612 INFO:tasks.radosbench.radosbench.0.smithi089.stdout:Object size: 65536 2023-06-03T20:23:24.612 INFO:tasks.radosbench.radosbench.0.smithi089.stdout:Bandwidth (MB/sec): 80.2107 2023-06-03T20:23:24.612 INFO:tasks.radosbench.radosbench.0.smithi089.stdout:Stddev Bandwidth: 31.1771 2023-06-03T20:23:24.613 INFO:tasks.radosbench.radosbench.0.smithi089.stdout:Max bandwidth (MB/sec): 182.812 2023-06-03T20:23:24.613 INFO:tasks.radosbench.radosbench.0.smithi089.stdout:Min bandwidth (MB/sec): 42.875 2023-06-03T20:23:24.613 INFO:tasks.radosbench.radosbench.0.smithi089.stdout:Average IOPS: 1283 2023-06-03T20:23:24.613 INFO:tasks.radosbench.radosbench.0.smithi089.stdout:Stddev IOPS: 498.834 2023-06-03T20:23:24.613 INFO:tasks.radosbench.radosbench.0.smithi089.stdout:Max IOPS: 2925 2023-06-03T20:23:24.613 INFO:tasks.radosbench.radosbench.0.smithi089.stdout:Min IOPS: 686 2023-06-03T20:23:24.614 INFO:tasks.radosbench.radosbench.0.smithi089.stdout:Average Latency(s): 0.0124626 2023-06-03T20:23:24.614 INFO:tasks.radosbench.radosbench.0.smithi089.stdout:Stddev Latency(s): 0.0133985 2023-06-03T20:23:24.614 INFO:tasks.radosbench.radosbench.0.smithi089.stdout:Max latency(s): 0.341691 2023-06-03T20:23:24.614 INFO:tasks.radosbench.radosbench.0.smithi089.stdout:Min latency(s): 0.0012167 2023-06-03T20:23:24.619 INFO:tasks.radosbench.radosbench.0.smithi089.stdout:Cleaning up (deleting benchmark objects) 2023-06-03T20:23:24.668 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:23:24.707 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:23:24.712 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:23:24.713 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:23:24.814 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:23:24.851 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:23:24.859 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:23:24.859 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:23:24.960 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:23:24.995 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:23:25.002 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:23:25.003 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:23:25.092 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-06-03T20:23:25.092 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-06-03T20:23:25.104 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:23:25.137 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:23:25.143 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:23:25.144 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:23:25.245 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:23:25.254 INFO:journalctl@ceph.osd.4.smithi164.stdout:Jun 03 20:23:25 smithi164 bash[130758]: debug 2023-06-03T20:23:25.100+0000 7fd6d68ca640 -1 osd.4 194 log_to_monitors true 2023-06-03T20:23:25.254 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:23:25 smithi164 bash[59462]: cluster 1685823803.926228 osd.6 (osd.6) 192 : cluster 3 failed to encode map e248 with expected crc 2023-06-03T20:23:25.254 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:23:25 smithi164 bash[59462]: cluster 1685823804.301144 osd.5 (osd.5) 186 : cluster 3 failed to encode map e248 with expected crc 2023-06-03T20:23:25.255 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:23:25 smithi164 bash[59462]: audit 1685823805.1247363 mon.a (mon.0) 1406 : audit 1 from='osd.4 [v2:172.21.15.164:6800/3945899737,v1:172.21.15.164:6801/3945899737]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2023-06-03T20:23:25.278 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:23:25.280 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:23:25.281 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:23:25.381 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:23:25.399 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:23:25 smithi089 bash[75068]: cluster 1685823803.926228 osd.6 (osd.6) 192 : cluster 3 failed to encode map e248 with expected crc 2023-06-03T20:23:25.400 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:23:25 smithi089 bash[75068]: cluster 1685823804.301144 osd.5 (osd.5) 186 : cluster 3 failed to encode map e248 with expected crc 2023-06-03T20:23:25.400 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:23:25 smithi089 bash[75068]: audit 1685823805.1247363 mon.a (mon.0) 1406 : audit 1 from='osd.4 [v2:172.21.15.164:6800/3945899737,v1:172.21.15.164:6801/3945899737]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2023-06-03T20:23:25.400 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:23:25 smithi089 bash[78452]: cluster 1685823803.926228 osd.6 (osd.6) 192 : cluster 3 failed to encode map e248 with expected crc 2023-06-03T20:23:25.401 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:23:25 smithi089 bash[78452]: cluster 1685823804.301144 osd.5 (osd.5) 186 : cluster 3 failed to encode map e248 with expected crc 2023-06-03T20:23:25.401 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:23:25 smithi089 bash[78452]: audit 1685823805.1247363 mon.a (mon.0) 1406 : audit 1 from='osd.4 [v2:172.21.15.164:6800/3945899737,v1:172.21.15.164:6801/3945899737]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2023-06-03T20:23:25.430 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:23:25.435 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:23:25.436 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:23:25.537 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:23:25.572 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:23:25.575 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:23:25.575 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:23:25.584 INFO:journalctl@ceph.osd.4.smithi164.stdout:Jun 03 20:23:25 smithi164 bash[130758]: debug 2023-06-03T20:23:25.372+0000 7fd6c5d32700 -1 osd.4 249 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2023-06-03T20:23:25.676 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:23:25.710 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:23:25.713 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:23:25.714 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:23:25.815 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:23:25.849 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:23:25.853 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:23:25.854 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:23:25.955 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:23:25.992 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:23:25.995 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:23:25.996 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:23:26.097 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:23:26.141 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:23:26.145 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:23:26.146 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:23:26.247 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:23:26.276 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:23:26.278 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:23:26.278 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:23:26.379 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:23:26.414 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:23:26.417 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:23:26.418 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:23:26.492 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/mon.a/config 2023-06-03T20:23:26.519 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:23:26.527 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:23:26 smithi089 bash[78452]: cluster 1685823804.6062438 osd.7 (osd.7) 181 : cluster 3 failed to encode map e248 with expected crc 2023-06-03T20:23:26.527 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:23:26 smithi089 bash[78452]: cluster 1685823805.0150373 mgr.y (mgr.34104) 1425 : cluster 0 pgmap v1440: 89 pgs: 17 active+clean+remapped, 13 active+undersized+degraded, 5 active+undersized, 54 active+clean; 7.0 GiB data, 21 GiB used, 604 GiB / 626 GiB avail; 54 MiB/s wr, 870 op/s; 64360/344055 objects degraded (18.706%); 6/344055 objects misplaced (0.002%) 2023-06-03T20:23:26.527 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:23:26 smithi089 bash[78452]: cluster 1685823805.2180386 osd.5 (osd.5) 187 : cluster 3 failed to encode map e249 with expected crc 2023-06-03T20:23:26.527 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:23:26 smithi089 bash[78452]: audit 1685823805.2187457 mon.a (mon.0) 1407 : audit 1 from='osd.4 [v2:172.21.15.164:6800/3945899737,v1:172.21.15.164:6801/3945899737]' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2023-06-03T20:23:26.527 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:23:26 smithi089 bash[78452]: cluster 1685823805.2188573 mon.a (mon.0) 1408 : cluster 0 osdmap e249: 8 total, 7 up, 7 in 2023-06-03T20:23:26.528 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:23:26 smithi089 bash[78452]: cluster 1685823805.2203286 osd.5 (osd.5) 188 : cluster 3 failed to encode map e249 with expected crc 2023-06-03T20:23:26.528 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:23:26 smithi089 bash[78452]: audit 1685823805.221063 mon.a (mon.0) 1409 : audit 1 from='osd.4 [v2:172.21.15.164:6800/3945899737,v1:172.21.15.164:6801/3945899737]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi164", "root=default"]}]: dispatch 2023-06-03T20:23:26.528 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 20:23:26 smithi089 bash[71163]: ::ffff:172.21.15.164 - - [03/Jun/2023:20:23:26] "GET /metrics HTTP/1.1" 200 228996 "" "Prometheus/2.33.4" 2023-06-03T20:23:26.528 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:23:26 smithi089 bash[75068]: cluster 1685823804.6062438 osd.7 (osd.7) 181 : cluster 3 failed to encode map e248 with expected crc 2023-06-03T20:23:26.529 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:23:26 smithi089 bash[75068]: cluster 1685823805.0150373 mgr.y (mgr.34104) 1425 : cluster 0 pgmap v1440: 89 pgs: 17 active+clean+remapped, 13 active+undersized+degraded, 5 active+undersized, 54 active+clean; 7.0 GiB data, 21 GiB used, 604 GiB / 626 GiB avail; 54 MiB/s wr, 870 op/s; 64360/344055 objects degraded (18.706%); 6/344055 objects misplaced (0.002%) 2023-06-03T20:23:26.529 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:23:26 smithi089 bash[75068]: cluster 1685823805.2180386 osd.5 (osd.5) 187 : cluster 3 failed to encode map e249 with expected crc 2023-06-03T20:23:26.529 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:23:26 smithi089 bash[75068]: audit 1685823805.2187457 mon.a (mon.0) 1407 : audit 1 from='osd.4 [v2:172.21.15.164:6800/3945899737,v1:172.21.15.164:6801/3945899737]' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2023-06-03T20:23:26.530 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:23:26 smithi089 bash[75068]: cluster 1685823805.2188573 mon.a (mon.0) 1408 : cluster 0 osdmap e249: 8 total, 7 up, 7 in 2023-06-03T20:23:26.531 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:23:26 smithi089 bash[75068]: cluster 1685823805.2203286 osd.5 (osd.5) 188 : cluster 3 failed to encode map e249 with expected crc 2023-06-03T20:23:26.531 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:23:26 smithi089 bash[75068]: audit 1685823805.221063 mon.a (mon.0) 1409 : audit 1 from='osd.4 [v2:172.21.15.164:6800/3945899737,v1:172.21.15.164:6801/3945899737]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi164", "root=default"]}]: dispatch 2023-06-03T20:23:26.547 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:23:26.549 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:23:26.550 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:23:26.584 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:23:26 smithi164 bash[59462]: cluster 1685823804.6062438 osd.7 (osd.7) 181 : cluster 3 failed to encode map e248 with expected crc 2023-06-03T20:23:26.585 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:23:26 smithi164 bash[59462]: cluster 1685823805.0150373 mgr.y (mgr.34104) 1425 : cluster 0 pgmap v1440: 89 pgs: 17 active+clean+remapped, 13 active+undersized+degraded, 5 active+undersized, 54 active+clean; 7.0 GiB data, 21 GiB used, 604 GiB / 626 GiB avail; 54 MiB/s wr, 870 op/s; 64360/344055 objects degraded (18.706%); 6/344055 objects misplaced (0.002%) 2023-06-03T20:23:26.585 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:23:26 smithi164 bash[59462]: cluster 1685823805.2180386 osd.5 (osd.5) 187 : cluster 3 failed to encode map e249 with expected crc 2023-06-03T20:23:26.586 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:23:26 smithi164 bash[59462]: audit 1685823805.2187457 mon.a (mon.0) 1407 : audit 1 from='osd.4 [v2:172.21.15.164:6800/3945899737,v1:172.21.15.164:6801/3945899737]' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2023-06-03T20:23:26.586 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:23:26 smithi164 bash[59462]: cluster 1685823805.2188573 mon.a (mon.0) 1408 : cluster 0 osdmap e249: 8 total, 7 up, 7 in 2023-06-03T20:23:26.586 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:23:26 smithi164 bash[59462]: cluster 1685823805.2203286 osd.5 (osd.5) 188 : cluster 3 failed to encode map e249 with expected crc 2023-06-03T20:23:26.586 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:23:26 smithi164 bash[59462]: audit 1685823805.221063 mon.a (mon.0) 1409 : audit 1 from='osd.4 [v2:172.21.15.164:6800/3945899737,v1:172.21.15.164:6801/3945899737]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi164", "root=default"]}]: dispatch 2023-06-03T20:23:26.651 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:23:26.692 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:23:26.695 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:23:26.695 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:23:26.796 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:23:26.825 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:23:26.829 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:23:26.830 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:23:26.930 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:23:26.964 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:23:26.966 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:23:26.967 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:23:27.068 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:23:27.093 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-06-03T20:23:27.104 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:23:27.106 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:23:27.107 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:23:27.208 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:23:27.258 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:23:27.261 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:23:27.262 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:23:27.362 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:23:27.374 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:23:27 smithi089 bash[78452]: cluster 1685823806.2138596 mon.a (mon.0) 1410 : cluster 1 Health check cleared: OSD_DOWN (was: 1 osds down) 2023-06-03T20:23:27.374 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:23:27 smithi089 bash[78452]: cluster 1685823806.223498 mon.a (mon.0) 1411 : cluster 1 osd.4 [v2:172.21.15.164:6800/3945899737,v1:172.21.15.164:6801/3945899737] boot 2023-06-03T20:23:27.375 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:23:27 smithi089 bash[78452]: cluster 1685823806.2236304 mon.a (mon.0) 1412 : cluster 0 osdmap e250: 8 total, 8 up, 7 in 2023-06-03T20:23:27.375 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:23:27 smithi089 bash[78452]: cluster 1685823806.22515 osd.5 (osd.5) 189 : cluster 3 failed to encode map e250 with expected crc 2023-06-03T20:23:27.375 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:23:27 smithi089 bash[78452]: audit 1685823806.228738 mon.c (mon.2) 604 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-06-03T20:23:27.375 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:23:27 smithi089 bash[78452]: cluster 1685823807.227051 mon.a (mon.0) 1413 : cluster 0 osdmap e251: 8 total, 8 up, 7 in 2023-06-03T20:23:27.375 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:23:27 smithi089 bash[75068]: cluster 1685823806.2138596 mon.a (mon.0) 1410 : cluster 1 Health check cleared: OSD_DOWN (was: 1 osds down) 2023-06-03T20:23:27.375 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:23:27 smithi089 bash[75068]: cluster 1685823806.223498 mon.a (mon.0) 1411 : cluster 1 osd.4 [v2:172.21.15.164:6800/3945899737,v1:172.21.15.164:6801/3945899737] boot 2023-06-03T20:23:27.376 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:23:27 smithi089 bash[75068]: cluster 1685823806.2236304 mon.a (mon.0) 1412 : cluster 0 osdmap e250: 8 total, 8 up, 7 in 2023-06-03T20:23:27.376 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:23:27 smithi089 bash[75068]: cluster 1685823806.22515 osd.5 (osd.5) 189 : cluster 3 failed to encode map e250 with expected crc 2023-06-03T20:23:27.376 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:23:27 smithi089 bash[75068]: audit 1685823806.228738 mon.c (mon.2) 604 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-06-03T20:23:27.376 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:23:27 smithi089 bash[75068]: cluster 1685823807.227051 mon.a (mon.0) 1413 : cluster 0 osdmap e251: 8 total, 8 up, 7 in 2023-06-03T20:23:27.396 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:23:27.398 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:23:27.399 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:23:27.500 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:23:27.511 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:23:27 smithi164 bash[59462]: cluster 1685823806.2138596 mon.a (mon.0) 1410 : cluster 1 Health check cleared: OSD_DOWN (was: 1 osds down) 2023-06-03T20:23:27.512 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:23:27 smithi164 bash[59462]: cluster 1685823806.223498 mon.a (mon.0) 1411 : cluster 1 osd.4 [v2:172.21.15.164:6800/3945899737,v1:172.21.15.164:6801/3945899737] boot 2023-06-03T20:23:27.512 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:23:27 smithi164 bash[59462]: cluster 1685823806.2236304 mon.a (mon.0) 1412 : cluster 0 osdmap e250: 8 total, 8 up, 7 in 2023-06-03T20:23:27.512 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:23:27 smithi164 bash[59462]: cluster 1685823806.22515 osd.5 (osd.5) 189 : cluster 3 failed to encode map e250 with expected crc 2023-06-03T20:23:27.512 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:23:27 smithi164 bash[59462]: audit 1685823806.228738 mon.c (mon.2) 604 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-06-03T20:23:27.512 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:23:27 smithi164 bash[59462]: cluster 1685823807.227051 mon.a (mon.0) 1413 : cluster 0 osdmap e251: 8 total, 8 up, 7 in 2023-06-03T20:23:27.531 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:23:27.534 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:23:27.534 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:23:27.635 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:23:27.675 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:23:27.678 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:23:27.679 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:23:27.780 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:23:27.818 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:23:27.820 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:23:27.821 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:23:27.922 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:23:27.955 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:23:27.958 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:23:27.959 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:23:28.060 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:23:28.072 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph osd set noscrub 2023-06-03T20:23:28.092 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:23:28.094 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:23:28.095 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:23:28.196 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:23:28.287 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:23:28.291 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:23:28.292 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:23:28.334 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:23:28 smithi164 bash[59462]: cluster 1685823807.0160809 mgr.y (mgr.34104) 1426 : cluster 0 pgmap v1443: 89 pgs: 17 active+clean+remapped, 13 active+undersized+degraded, 5 active+undersized, 54 active+clean; 7.0 GiB data, 21 GiB used, 604 GiB / 626 GiB avail; 50 MiB/s wr, 862 op/s; 64437/344559 objects degraded (18.701%); 6/344559 objects misplaced (0.002%) 2023-06-03T20:23:28.335 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:23:28 smithi164 bash[59462]: cluster 1685823807.2245722 osd.5 (osd.5) 190 : cluster 3 failed to encode map e251 with expected crc 2023-06-03T20:23:28.335 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:23:28 smithi164 bash[59462]: cluster 1685823807.224952 osd.7 (osd.7) 182 : cluster 3 failed to encode map e251 with expected crc 2023-06-03T20:23:28.335 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:23:28 smithi164 bash[59462]: cluster 1685823807.2299976 osd.7 (osd.7) 183 : cluster 3 failed to encode map e251 with expected crc 2023-06-03T20:23:28.335 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:23:28 smithi164 bash[59462]: cluster 1685823807.2300344 osd.5 (osd.5) 191 : cluster 3 failed to encode map e251 with expected crc 2023-06-03T20:23:28.335 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:23:28 smithi164 bash[59462]: cluster 1685823807.234041 osd.5 (osd.5) 192 : cluster 3 failed to encode map e251 with expected crc 2023-06-03T20:23:28.335 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:23:28 smithi164 bash[59462]: cluster 1685823807.9177482 mon.a (mon.0) 1414 : cluster 3 Health check update: Degraded data redundancy: 64437/344559 objects degraded (18.701%), 13 pgs degraded, 18 pgs undersized (PG_DEGRADED) 2023-06-03T20:23:28.336 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:23:28 smithi164 bash[59462]: cluster 1685823808.2263815 mon.a (mon.0) 1415 : cluster 0 osdmap e252: 8 total, 8 up, 7 in 2023-06-03T20:23:28.393 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:23:28.408 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:23:28 smithi089 bash[75068]: cluster 1685823807.0160809 mgr.y (mgr.34104) 1426 : cluster 0 pgmap v1443: 89 pgs: 17 active+clean+remapped, 13 active+undersized+degraded, 5 active+undersized, 54 active+clean; 7.0 GiB data, 21 GiB used, 604 GiB / 626 GiB avail; 50 MiB/s wr, 862 op/s; 64437/344559 objects degraded (18.701%); 6/344559 objects misplaced (0.002%) 2023-06-03T20:23:28.409 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:23:28 smithi089 bash[75068]: cluster 1685823807.2245722 osd.5 (osd.5) 190 : cluster 3 failed to encode map e251 with expected crc 2023-06-03T20:23:28.409 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:23:28 smithi089 bash[75068]: cluster 1685823807.224952 osd.7 (osd.7) 182 : cluster 3 failed to encode map e251 with expected crc 2023-06-03T20:23:28.409 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:23:28 smithi089 bash[75068]: cluster 1685823807.2299976 osd.7 (osd.7) 183 : cluster 3 failed to encode map e251 with expected crc 2023-06-03T20:23:28.409 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:23:28 smithi089 bash[75068]: cluster 1685823807.2300344 osd.5 (osd.5) 191 : cluster 3 failed to encode map e251 with expected crc 2023-06-03T20:23:28.409 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:23:28 smithi089 bash[75068]: cluster 1685823807.234041 osd.5 (osd.5) 192 : cluster 3 failed to encode map e251 with expected crc 2023-06-03T20:23:28.409 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:23:28 smithi089 bash[75068]: cluster 1685823807.9177482 mon.a (mon.0) 1414 : cluster 3 Health check update: Degraded data redundancy: 64437/344559 objects degraded (18.701%), 13 pgs degraded, 18 pgs undersized (PG_DEGRADED) 2023-06-03T20:23:28.410 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:23:28 smithi089 bash[75068]: cluster 1685823808.2263815 mon.a (mon.0) 1415 : cluster 0 osdmap e252: 8 total, 8 up, 7 in 2023-06-03T20:23:28.410 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:23:28 smithi089 bash[78452]: cluster 1685823807.0160809 mgr.y (mgr.34104) 1426 : cluster 0 pgmap v1443: 89 pgs: 17 active+clean+remapped, 13 active+undersized+degraded, 5 active+undersized, 54 active+clean; 7.0 GiB data, 21 GiB used, 604 GiB / 626 GiB avail; 50 MiB/s wr, 862 op/s; 64437/344559 objects degraded (18.701%); 6/344559 objects misplaced (0.002%) 2023-06-03T20:23:28.410 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:23:28 smithi089 bash[78452]: cluster 1685823807.2245722 osd.5 (osd.5) 190 : cluster 3 failed to encode map e251 with expected crc 2023-06-03T20:23:28.410 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:23:28 smithi089 bash[78452]: cluster 1685823807.224952 osd.7 (osd.7) 182 : cluster 3 failed to encode map e251 with expected crc 2023-06-03T20:23:28.411 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:23:28 smithi089 bash[78452]: cluster 1685823807.2299976 osd.7 (osd.7) 183 : cluster 3 failed to encode map e251 with expected crc 2023-06-03T20:23:28.411 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:23:28 smithi089 bash[78452]: cluster 1685823807.2300344 osd.5 (osd.5) 191 : cluster 3 failed to encode map e251 with expected crc 2023-06-03T20:23:28.411 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:23:28 smithi089 bash[78452]: cluster 1685823807.234041 osd.5 (osd.5) 192 : cluster 3 failed to encode map e251 with expected crc 2023-06-03T20:23:28.411 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:23:28 smithi089 bash[78452]: cluster 1685823807.9177482 mon.a (mon.0) 1414 : cluster 3 Health check update: Degraded data redundancy: 64437/344559 objects degraded (18.701%), 13 pgs degraded, 18 pgs undersized (PG_DEGRADED) 2023-06-03T20:23:28.411 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:23:28 smithi089 bash[78452]: cluster 1685823808.2263815 mon.a (mon.0) 1415 : cluster 0 osdmap e252: 8 total, 8 up, 7 in 2023-06-03T20:23:28.424 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:23:28.427 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:23:28.427 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:23:28.529 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:23:28.577 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:23:28.581 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:23:28.582 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:23:28.683 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:23:28.710 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:23:28.712 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:23:28.713 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:23:28.814 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:23:28.850 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:23:28.856 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:23:28.857 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:23:28.958 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:23:29.004 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:23:29.009 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:23:29.010 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:23:29.111 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:23:29.142 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:23:29.146 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:23:29.147 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:23:29.248 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:23:29.272 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:23:29 smithi089 bash[78452]: cluster 1685823808.2219734 osd.7 (osd.7) 184 : cluster 3 failed to encode map e252 with expected crc 2023-06-03T20:23:29.273 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:23:29 smithi089 bash[78452]: cluster 1685823808.2248435 osd.7 (osd.7) 185 : cluster 3 failed to encode map e252 with expected crc 2023-06-03T20:23:29.273 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:23:29 smithi089 bash[78452]: cluster 1685823808.2279663 osd.3 (osd.3) 17 : cluster 0 13.2 starting backfill to osd.4 from (0'0,0'0] MAX to 249'7298 2023-06-03T20:23:29.273 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:23:29 smithi089 bash[78452]: cluster 1685823808.2284064 osd.0 (osd.0) 12 : cluster 0 13.3 starting backfill to osd.4 from (0'0,0'0] MAX to 249'7263 2023-06-03T20:23:29.273 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:23:29 smithi089 bash[78452]: cluster 1685823808.2303462 osd.5 (osd.5) 193 : cluster 3 failed to encode map e252 with expected crc 2023-06-03T20:23:29.274 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:23:29 smithi089 bash[78452]: cluster 1685823808.2305787 osd.0 (osd.0) 13 : cluster 0 13.5 starting backfill to osd.4 from (0'0,0'0] MAX to 249'7292 2023-06-03T20:23:29.274 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:23:29 smithi089 bash[78452]: cluster 1685823808.2325642 osd.5 (osd.5) 194 : cluster 3 failed to encode map e252 with expected crc 2023-06-03T20:23:29.274 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:23:29 smithi089 bash[78452]: cluster 1685823808.2339606 osd.7 (osd.7) 186 : cluster 0 13.7 starting backfill to osd.4 from (0'0,0'0] MAX to 249'7431 2023-06-03T20:23:29.274 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:23:29 smithi089 bash[78452]: cluster 1685823808.2382896 osd.1 (osd.1) 11 : cluster 0 13.d starting backfill to osd.4 from (0'0,0'0] MAX to 249'7397 2023-06-03T20:23:29.274 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:23:29 smithi089 bash[78452]: cluster 1685823808.244379 osd.3 (osd.3) 18 : cluster 0 13.6 starting backfill to osd.4 from (0'0,0'0] MAX to 249'7171 2023-06-03T20:23:29.275 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:23:29 smithi089 bash[75068]: cluster 1685823808.2219734 osd.7 (osd.7) 184 : cluster 3 failed to encode map e252 with expected crc 2023-06-03T20:23:29.275 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:23:29 smithi089 bash[75068]: cluster 1685823808.2248435 osd.7 (osd.7) 185 : cluster 3 failed to encode map e252 with expected crc 2023-06-03T20:23:29.275 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:23:29 smithi089 bash[75068]: cluster 1685823808.2279663 osd.3 (osd.3) 17 : cluster 0 13.2 starting backfill to osd.4 from (0'0,0'0] MAX to 249'7298 2023-06-03T20:23:29.275 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:23:29 smithi089 bash[75068]: cluster 1685823808.2284064 osd.0 (osd.0) 12 : cluster 0 13.3 starting backfill to osd.4 from (0'0,0'0] MAX to 249'7263 2023-06-03T20:23:29.275 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:23:29 smithi089 bash[75068]: cluster 1685823808.2303462 osd.5 (osd.5) 193 : cluster 3 failed to encode map e252 with expected crc 2023-06-03T20:23:29.275 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:23:29 smithi089 bash[75068]: cluster 1685823808.2305787 osd.0 (osd.0) 13 : cluster 0 13.5 starting backfill to osd.4 from (0'0,0'0] MAX to 249'7292 2023-06-03T20:23:29.276 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:23:29 smithi089 bash[75068]: cluster 1685823808.2325642 osd.5 (osd.5) 194 : cluster 3 failed to encode map e252 with expected crc 2023-06-03T20:23:29.276 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:23:29 smithi089 bash[75068]: cluster 1685823808.2339606 osd.7 (osd.7) 186 : cluster 0 13.7 starting backfill to osd.4 from (0'0,0'0] MAX to 249'7431 2023-06-03T20:23:29.276 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:23:29 smithi089 bash[75068]: cluster 1685823808.2382896 osd.1 (osd.1) 11 : cluster 0 13.d starting backfill to osd.4 from (0'0,0'0] MAX to 249'7397 2023-06-03T20:23:29.276 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:23:29 smithi089 bash[75068]: cluster 1685823808.244379 osd.3 (osd.3) 18 : cluster 0 13.6 starting backfill to osd.4 from (0'0,0'0] MAX to 249'7171 2023-06-03T20:23:29.286 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:23:29.291 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:23:29.292 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:23:29.332 INFO:teuthology.orchestra.run.smithi164.stderr:Inferring config /var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/mon.b/config 2023-06-03T20:23:29.391 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:23:29 smithi164 bash[59462]: cluster 1685823808.2219734 osd.7 (osd.7) 184 : cluster 3 failed to encode map e252 with expected crc 2023-06-03T20:23:29.392 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:23:29 smithi164 bash[59462]: cluster 1685823808.2248435 osd.7 (osd.7) 185 : cluster 3 failed to encode map e252 with expected crc 2023-06-03T20:23:29.392 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:23:29 smithi164 bash[59462]: cluster 1685823808.2279663 osd.3 (osd.3) 17 : cluster 0 13.2 starting backfill to osd.4 from (0'0,0'0] MAX to 249'7298 2023-06-03T20:23:29.392 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:23:29 smithi164 bash[59462]: cluster 1685823808.2284064 osd.0 (osd.0) 12 : cluster 0 13.3 starting backfill to osd.4 from (0'0,0'0] MAX to 249'7263 2023-06-03T20:23:29.392 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:23:29 smithi164 bash[59462]: cluster 1685823808.2303462 osd.5 (osd.5) 193 : cluster 3 failed to encode map e252 with expected crc 2023-06-03T20:23:29.393 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:23:29 smithi164 bash[59462]: cluster 1685823808.2305787 osd.0 (osd.0) 13 : cluster 0 13.5 starting backfill to osd.4 from (0'0,0'0] MAX to 249'7292 2023-06-03T20:23:29.393 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:23:29 smithi164 bash[59462]: cluster 1685823808.2325642 osd.5 (osd.5) 194 : cluster 3 failed to encode map e252 with expected crc 2023-06-03T20:23:29.393 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:23:29 smithi164 bash[59462]: cluster 1685823808.2339606 osd.7 (osd.7) 186 : cluster 0 13.7 starting backfill to osd.4 from (0'0,0'0] MAX to 249'7431 2023-06-03T20:23:29.393 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:23:29 smithi164 bash[59462]: cluster 1685823808.2382896 osd.1 (osd.1) 11 : cluster 0 13.d starting backfill to osd.4 from (0'0,0'0] MAX to 249'7397 2023-06-03T20:23:29.393 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:23:29 smithi164 bash[59462]: cluster 1685823808.244379 osd.3 (osd.3) 18 : cluster 0 13.6 starting backfill to osd.4 from (0'0,0'0] MAX to 249'7171 2023-06-03T20:23:29.393 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:23:29.439 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:23:29.444 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:23:29.445 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:23:29.546 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:23:29.583 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:23:29.589 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:23:29.590 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:23:29.691 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:23:29.729 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:23:29.734 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:23:29.736 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:23:29.836 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:23:29.877 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:23:29.883 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:23:29.884 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:23:29.986 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:23:30.025 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:23:30.028 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:23:30.029 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:23:30.130 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:23:30.171 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:23:30.173 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:23:30.174 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:23:30.275 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:23:30.319 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:23:30.324 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:23:30.325 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:23:30.427 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:23:30.437 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:23:30 smithi089 bash[75068]: cluster 1685823809.0167248 mgr.y (mgr.34104) 1427 : cluster 0 pgmap v1446: 89 pgs: 1 activating+undersized+degraded+remapped, 1 active, 3 remapped+peering, 7 peering, 10 active+clean+remapped, 5 active+undersized+degraded, 1 active+undersized+degraded+remapped+backfill_wait, 2 active+undersized, 59 active+clean; 7.0 GiB data, 21 GiB used, 604 GiB / 626 GiB avail; 255 B/s rd, 13 MiB/s wr, 512 op/s; 42876/343074 objects degraded (12.498%); 4/343074 objects misplaced (0.001%) 2023-06-03T20:23:30.438 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:23:30 smithi089 bash[78452]: cluster 1685823809.0167248 mgr.y (mgr.34104) 1427 : cluster 0 pgmap v1446: 89 pgs: 1 activating+undersized+degraded+remapped, 1 active, 3 remapped+peering, 7 peering, 10 active+clean+remapped, 5 active+undersized+degraded, 1 active+undersized+degraded+remapped+backfill_wait, 2 active+undersized, 59 active+clean; 7.0 GiB data, 21 GiB used, 604 GiB / 626 GiB avail; 255 B/s rd, 13 MiB/s wr, 512 op/s; 42876/343074 objects degraded (12.498%); 4/343074 objects misplaced (0.001%) 2023-06-03T20:23:30.465 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:23:30.468 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:23:30.469 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:23:30.571 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:23:30.584 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:23:30 smithi164 bash[59462]: cluster 1685823809.0167248 mgr.y (mgr.34104) 1427 : cluster 0 pgmap v1446: 89 pgs: 1 activating+undersized+degraded+remapped, 1 active, 3 remapped+peering, 7 peering, 10 active+clean+remapped, 5 active+undersized+degraded, 1 active+undersized+degraded+remapped+backfill_wait, 2 active+undersized, 59 active+clean; 7.0 GiB data, 21 GiB used, 604 GiB / 626 GiB avail; 255 B/s rd, 13 MiB/s wr, 512 op/s; 42876/343074 objects degraded (12.498%); 4/343074 objects misplaced (0.001%) 2023-06-03T20:23:30.612 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:23:30.613 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:23:30.615 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:23:30.717 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:23:30.758 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:23:30.761 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:23:30.762 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:23:30.862 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:23:30.911 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:23:30.919 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:23:30.921 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:23:31.022 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:23:31.072 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:23:31.076 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:23:31.076 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:23:31.177 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:23:31.225 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:23:31.230 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:23:31.231 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:23:31.332 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:23:31.366 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:23:31.375 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:23:31.376 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:23:31.475 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:23:31 smithi164 bash[59462]: cluster 1685823811.0172663 mgr.y (mgr.34104) 1428 : cluster 0 pgmap v1447: 89 pgs: 1 activating+undersized+degraded+remapped, 1 active, 3 remapped+peering, 7 peering, 2 active+undersized+degraded, 3 active+undersized+degraded+remapped+backfill_wait, 72 active+clean; 7.0 GiB data, 21 GiB used, 604 GiB / 626 GiB avail; 176 B/s rd, 9.3 MiB/s wr, 408 op/s; 42669/342123 objects degraded (12.472%) 2023-06-03T20:23:31.476 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:23:31.519 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:23:31.522 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:23:31.523 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:23:31.624 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:23:31.633 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:23:31 smithi089 bash[75068]: cluster 1685823811.0172663 mgr.y (mgr.34104) 1428 : cluster 0 pgmap v1447: 89 pgs: 1 activating+undersized+degraded+remapped, 1 active, 3 remapped+peering, 7 peering, 2 active+undersized+degraded, 3 active+undersized+degraded+remapped+backfill_wait, 72 active+clean; 7.0 GiB data, 21 GiB used, 604 GiB / 626 GiB avail; 176 B/s rd, 9.3 MiB/s wr, 408 op/s; 42669/342123 objects degraded (12.472%) 2023-06-03T20:23:31.634 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:23:31 smithi089 bash[78452]: cluster 1685823811.0172663 mgr.y (mgr.34104) 1428 : cluster 0 pgmap v1447: 89 pgs: 1 activating+undersized+degraded+remapped, 1 active, 3 remapped+peering, 7 peering, 2 active+undersized+degraded, 3 active+undersized+degraded+remapped+backfill_wait, 72 active+clean; 7.0 GiB data, 21 GiB used, 604 GiB / 626 GiB avail; 176 B/s rd, 9.3 MiB/s wr, 408 op/s; 42669/342123 objects degraded (12.472%) 2023-06-03T20:23:31.664 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:23:31.666 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:23:31.667 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:23:31.768 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:23:31.810 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:23:31.814 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:23:31.815 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:23:31.916 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:23:31.978 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:23:31.985 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:23:31.986 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:23:32.081 INFO:teuthology.orchestra.run.smithi089.stdout:{ 2023-06-03T20:23:32.082 INFO:teuthology.orchestra.run.smithi089.stdout: "ops": [ 2023-06-03T20:23:32.082 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:23:32.082 INFO:teuthology.orchestra.run.smithi089.stdout: "description": "osd_op(client.36573.0:119145 13.a 13:576e7dfb:::benchmark_data_smithi089_211581_object3620:head [delete] snapc 0=[] ondisk+write+known_if_redirected e252)", 2023-06-03T20:23:32.082 INFO:teuthology.orchestra.run.smithi089.stdout: "initiated_at": "2023-06-03T20:23:32.059252+0000", 2023-06-03T20:23:32.082 INFO:teuthology.orchestra.run.smithi089.stdout: "age": 0.015964807000000001, 2023-06-03T20:23:32.082 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.015999297999999999, 2023-06-03T20:23:32.083 INFO:teuthology.orchestra.run.smithi089.stdout: "type_data": { 2023-06-03T20:23:32.083 INFO:teuthology.orchestra.run.smithi089.stdout: "flag_point": "waiting for sub ops", 2023-06-03T20:23:32.083 INFO:teuthology.orchestra.run.smithi089.stdout: "client_info": { 2023-06-03T20:23:32.083 INFO:teuthology.orchestra.run.smithi089.stdout: "client": "client.36573", 2023-06-03T20:23:32.083 INFO:teuthology.orchestra.run.smithi089.stdout: "client_addr": "172.21.15.89:0/616445942", 2023-06-03T20:23:32.083 INFO:teuthology.orchestra.run.smithi089.stdout: "tid": 119145 2023-06-03T20:23:32.083 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:23:32.084 INFO:teuthology.orchestra.run.smithi089.stdout: "events": [ 2023-06-03T20:23:32.084 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:23:32.084 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "initiated", 2023-06-03T20:23:32.084 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:23:32.059252+0000", 2023-06-03T20:23:32.084 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0 2023-06-03T20:23:32.084 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:23:32.084 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:23:32.084 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "throttled", 2023-06-03T20:23:32.085 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:23:32.059252+0000", 2023-06-03T20:23:32.085 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0 2023-06-03T20:23:32.085 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:23:32.085 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:23:32.085 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "header_read", 2023-06-03T20:23:32.085 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:23:32.059250+0000", 2023-06-03T20:23:32.085 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 4294967295.9999986 2023-06-03T20:23:32.085 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:23:32.086 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:23:32.086 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "all_read", 2023-06-03T20:23:32.086 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:23:32.059257+0000", 2023-06-03T20:23:32.086 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 6.63e-06 2023-06-03T20:23:32.086 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:23:32.086 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:23:32.086 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "dispatched", 2023-06-03T20:23:32.086 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:23:32.059271+0000", 2023-06-03T20:23:32.087 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 1.4126e-05 2023-06-03T20:23:32.087 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:23:32.087 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:23:32.087 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "queued_for_pg", 2023-06-03T20:23:32.087 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:23:32.059283+0000", 2023-06-03T20:23:32.087 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 1.1843e-05 2023-06-03T20:23:32.087 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:23:32.088 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:23:32.088 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "reached_pg", 2023-06-03T20:23:32.088 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:23:32.059340+0000", 2023-06-03T20:23:32.088 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 5.6878e-05 2023-06-03T20:23:32.088 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:23:32.088 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:23:32.088 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "started", 2023-06-03T20:23:32.088 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:23:32.059461+0000", 2023-06-03T20:23:32.089 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.000121531 2023-06-03T20:23:32.089 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:23:32.089 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:23:32.089 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "waiting for subops from 1,4", 2023-06-03T20:23:32.089 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:23:32.059559+0000", 2023-06-03T20:23:32.089 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 9.7589000000000004e-05 2023-06-03T20:23:32.089 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:23:32.089 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:23:32.090 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "op_commit", 2023-06-03T20:23:32.090 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:23:32.063854+0000", 2023-06-03T20:23:32.090 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.0042953210000000004 2023-06-03T20:23:32.090 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:23:32.090 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:23:32.090 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:23:32.090 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:23:32.063918+0000", 2023-06-03T20:23:32.090 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 6.3880999999999994e-05 2023-06-03T20:23:32.091 INFO:teuthology.orchestra.run.smithi089.stdout: } 2023-06-03T20:23:32.091 INFO:teuthology.orchestra.run.smithi089.stdout: ] 2023-06-03T20:23:32.091 INFO:teuthology.orchestra.run.smithi089.stdout: } 2023-06-03T20:23:32.091 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:23:32.091 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:23:32.092 INFO:teuthology.orchestra.run.smithi089.stdout: "description": "osd_repop(client.36573.0:119147 13.9 e252/242 13:9a4dba7e:::benchmark_data_smithi089_211581_object3622:head v 252'7514, mlcod=252'7512)", 2023-06-03T20:23:32.092 INFO:teuthology.orchestra.run.smithi089.stdout: "initiated_at": "2023-06-03T20:23:32.071407+0000", 2023-06-03T20:23:32.092 INFO:teuthology.orchestra.run.smithi089.stdout: "age": 0.0038094219999999998, 2023-06-03T20:23:32.092 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.0039410740000000001, 2023-06-03T20:23:32.092 INFO:teuthology.orchestra.run.smithi089.stdout: "type_data": { 2023-06-03T20:23:32.092 INFO:teuthology.orchestra.run.smithi089.stdout: "flag_point": "started", 2023-06-03T20:23:32.092 INFO:teuthology.orchestra.run.smithi089.stdout: "events": [ 2023-06-03T20:23:32.092 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:23:32.093 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "initiated", 2023-06-03T20:23:32.093 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:23:32.071407+0000", 2023-06-03T20:23:32.093 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0 2023-06-03T20:23:32.093 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:23:32.093 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:23:32.093 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "throttled", 2023-06-03T20:23:32.093 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:23:32.071407+0000", 2023-06-03T20:23:32.093 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0 2023-06-03T20:23:32.094 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:23:32.094 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:23:32.094 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "header_read", 2023-06-03T20:23:32.094 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:23:32.071406+0000", 2023-06-03T20:23:32.094 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 4294967295.999999 2023-06-03T20:23:32.094 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:23:32.094 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:23:32.094 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "all_read", 2023-06-03T20:23:32.095 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:23:32.071415+0000", 2023-06-03T20:23:32.095 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 8.4549999999999995e-06 2023-06-03T20:23:32.095 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:23:32.095 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:23:32.095 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "dispatched", 2023-06-03T20:23:32.095 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:23:32.071428+0000", 2023-06-03T20:23:32.095 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 1.3594999999999999e-05 2023-06-03T20:23:32.096 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:23:32.096 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:23:32.096 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "queued_for_pg", 2023-06-03T20:23:32.096 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:23:32.071437+0000", 2023-06-03T20:23:32.096 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 8.9490000000000002e-06 2023-06-03T20:23:32.096 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:23:32.096 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:23:32.096 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "reached_pg", 2023-06-03T20:23:32.097 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:23:32.071487+0000", 2023-06-03T20:23:32.097 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 4.9877999999999999e-05 2023-06-03T20:23:32.097 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:23:32.097 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:23:32.097 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "started", 2023-06-03T20:23:32.097 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:23:32.071519+0000", 2023-06-03T20:23:32.098 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 3.2171000000000001e-05 2023-06-03T20:23:32.098 INFO:teuthology.orchestra.run.smithi089.stdout: } 2023-06-03T20:23:32.098 INFO:teuthology.orchestra.run.smithi089.stdout: ] 2023-06-03T20:23:32.098 INFO:teuthology.orchestra.run.smithi089.stdout: } 2023-06-03T20:23:32.098 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:23:32.098 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:23:32.099 INFO:teuthology.orchestra.run.smithi089.stdout: "description": "osd_op(client.36573.0:119150 13.4 13:20a6fd8e:::benchmark_data_smithi089_211581_object3625:head [delete] snapc 0=[] ondisk+write+known_if_redirected e252)", 2023-06-03T20:23:32.099 INFO:teuthology.orchestra.run.smithi089.stdout: "initiated_at": "2023-06-03T20:23:32.073039+0000", 2023-06-03T20:23:32.099 INFO:teuthology.orchestra.run.smithi089.stdout: "age": 0.0021779799999999999, 2023-06-03T20:23:32.099 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.0023736220000000001, 2023-06-03T20:23:32.099 INFO:teuthology.orchestra.run.smithi089.stdout: "type_data": { 2023-06-03T20:23:32.099 INFO:teuthology.orchestra.run.smithi089.stdout: "flag_point": "waiting for sub ops", 2023-06-03T20:23:32.100 INFO:teuthology.orchestra.run.smithi089.stdout: "client_info": { 2023-06-03T20:23:32.100 INFO:teuthology.orchestra.run.smithi089.stdout: "client": "client.36573", 2023-06-03T20:23:32.100 INFO:teuthology.orchestra.run.smithi089.stdout: "client_addr": "172.21.15.89:0/616445942", 2023-06-03T20:23:32.100 INFO:teuthology.orchestra.run.smithi089.stdout: "tid": 119150 2023-06-03T20:23:32.100 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:23:32.100 INFO:teuthology.orchestra.run.smithi089.stdout: "events": [ 2023-06-03T20:23:32.100 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:23:32.101 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "initiated", 2023-06-03T20:23:32.101 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:23:32.073039+0000", 2023-06-03T20:23:32.101 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0 2023-06-03T20:23:32.101 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:23:32.101 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:23:32.101 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "throttled", 2023-06-03T20:23:32.101 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:23:32.073039+0000", 2023-06-03T20:23:32.101 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0 2023-06-03T20:23:32.102 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:23:32.102 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:23:32.102 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "header_read", 2023-06-03T20:23:32.102 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:23:32.073037+0000", 2023-06-03T20:23:32.102 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 4294967295.9999986 2023-06-03T20:23:32.102 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:23:32.102 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:23:32.102 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "all_read", 2023-06-03T20:23:32.103 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:23:32.073044+0000", 2023-06-03T20:23:32.103 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 6.8759999999999999e-06 2023-06-03T20:23:32.103 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:23:32.103 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:23:32.103 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "dispatched", 2023-06-03T20:23:32.103 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:23:32.073057+0000", 2023-06-03T20:23:32.103 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 1.292e-05 2023-06-03T20:23:32.103 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:23:32.104 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:23:32.104 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "queued_for_pg", 2023-06-03T20:23:32.104 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:23:32.073064+0000", 2023-06-03T20:23:32.104 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 6.4370000000000004e-06 2023-06-03T20:23:32.104 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:23:32.104 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:23:32.104 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "reached_pg", 2023-06-03T20:23:32.104 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:23:32.073111+0000", 2023-06-03T20:23:32.105 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 4.7437e-05 2023-06-03T20:23:32.105 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:23:32.105 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:23:32.105 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "started", 2023-06-03T20:23:32.105 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:23:32.073219+0000", 2023-06-03T20:23:32.105 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.000108076 2023-06-03T20:23:32.105 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:23:32.105 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:23:32.106 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "waiting for subops from 1,5", 2023-06-03T20:23:32.106 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:23:32.073306+0000", 2023-06-03T20:23:32.106 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 8.6773000000000004e-05 2023-06-03T20:23:32.106 INFO:teuthology.orchestra.run.smithi089.stdout: } 2023-06-03T20:23:32.106 INFO:teuthology.orchestra.run.smithi089.stdout: ] 2023-06-03T20:23:32.106 INFO:teuthology.orchestra.run.smithi089.stdout: } 2023-06-03T20:23:32.106 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:23:32.106 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:23:32.107 INFO:teuthology.orchestra.run.smithi089.stdout: "description": "osd_repop(client.36573.0:119146 13.0 e252/242 13:0c073768:::benchmark_data_smithi089_211581_object3621:head v 252'7562, mlcod=252'7559)", 2023-06-03T20:23:32.107 INFO:teuthology.orchestra.run.smithi089.stdout: "initiated_at": "2023-06-03T20:23:32.073507+0000", 2023-06-03T20:23:32.107 INFO:teuthology.orchestra.run.smithi089.stdout: "age": 0.001709803, 2023-06-03T20:23:32.107 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.0019806479999999998, 2023-06-03T20:23:32.107 INFO:teuthology.orchestra.run.smithi089.stdout: "type_data": { 2023-06-03T20:23:32.107 INFO:teuthology.orchestra.run.smithi089.stdout: "flag_point": "started", 2023-06-03T20:23:32.107 INFO:teuthology.orchestra.run.smithi089.stdout: "events": [ 2023-06-03T20:23:32.108 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:23:32.108 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "initiated", 2023-06-03T20:23:32.108 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:23:32.073507+0000", 2023-06-03T20:23:32.108 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0 2023-06-03T20:23:32.108 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:23:32.108 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:23:32.108 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "throttled", 2023-06-03T20:23:32.108 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:23:32.073507+0000", 2023-06-03T20:23:32.109 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0 2023-06-03T20:23:32.109 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:23:32.109 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:23:32.109 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "header_read", 2023-06-03T20:23:32.109 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:23:32.073506+0000", 2023-06-03T20:23:32.109 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 4294967295.9999995 2023-06-03T20:23:32.109 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:23:32.109 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:23:32.110 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "all_read", 2023-06-03T20:23:32.110 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:23:32.073513+0000", 2023-06-03T20:23:32.110 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 6.7109999999999998e-06 2023-06-03T20:23:32.110 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:23:32.110 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:23:32.110 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "dispatched", 2023-06-03T20:23:32.110 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:23:32.073525+0000", 2023-06-03T20:23:32.110 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 1.1973999999999999e-05 2023-06-03T20:23:32.111 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:23:32.111 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:23:32.111 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "queued_for_pg", 2023-06-03T20:23:32.111 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:23:32.073532+0000", 2023-06-03T20:23:32.111 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 6.7909999999999999e-06 2023-06-03T20:23:32.111 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:23:32.111 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:23:32.112 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "reached_pg", 2023-06-03T20:23:32.112 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:23:32.073616+0000", 2023-06-03T20:23:32.112 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 8.4431999999999994e-05 2023-06-03T20:23:32.112 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:23:32.112 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:23:32.112 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "started", 2023-06-03T20:23:32.112 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:23:32.073638+0000", 2023-06-03T20:23:32.112 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 2.1678000000000001e-05 2023-06-03T20:23:32.113 INFO:teuthology.orchestra.run.smithi089.stdout: } 2023-06-03T20:23:32.113 INFO:teuthology.orchestra.run.smithi089.stdout: ] 2023-06-03T20:23:32.113 INFO:teuthology.orchestra.run.smithi089.stdout: } 2023-06-03T20:23:32.113 INFO:teuthology.orchestra.run.smithi089.stdout: } 2023-06-03T20:23:32.113 INFO:teuthology.orchestra.run.smithi089.stdout: ], 2023-06-03T20:23:32.113 INFO:teuthology.orchestra.run.smithi089.stdout: "num_ops": 4 2023-06-03T20:23:32.113 INFO:teuthology.orchestra.run.smithi089.stdout:} 2023-06-03T20:23:32.114 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:23:32.148 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:23:32.150 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:23:32.151 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:23:32.252 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:23:32.283 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:23:32.290 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:23:32.291 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:23:32.392 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:23:32.455 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:23:32.458 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:23:32.459 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:23:32.560 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:23:32.598 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:23:32.604 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:23:32.605 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:23:32.706 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:23:32.750 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:23:32.752 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:23:32.753 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:23:32.854 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:23:32.890 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:23:32.893 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:23:32.893 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:23:32.986 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:23:32 smithi089 bash[75068]: cluster 1685823812.9193254 mon.a (mon.0) 1416 : cluster 3 Health check update: Degraded data redundancy: 42669/342123 objects degraded (12.472%), 6 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-06-03T20:23:32.995 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:23:33.037 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:23:33.045 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:23:33.046 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:23:33.147 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:23:33.175 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:23:33.181 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:23:33.181 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:23:33.279 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:23:32 smithi089 bash[78452]: cluster 1685823812.9193254 mon.a (mon.0) 1416 : cluster 3 Health check update: Degraded data redundancy: 42669/342123 objects degraded (12.472%), 6 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-06-03T20:23:33.282 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:23:33.312 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph daemon osd.2 dump_blocked_ops 2023-06-03T20:23:33.324 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:23:33.327 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:23:33.328 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:23:33.334 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:23:32 smithi164 bash[59462]: cluster 1685823812.9193254 mon.a (mon.0) 1416 : cluster 3 Health check update: Degraded data redundancy: 42669/342123 objects degraded (12.472%), 6 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-06-03T20:23:33.430 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:23:33.473 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:23:33.474 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:23:33.475 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:23:33.576 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:23:33.615 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:23:33.620 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:23:33.621 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:23:33.722 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:23:33.765 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:23:33.772 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:23:33.773 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:23:33.808 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/mon.a/config 2023-06-03T20:23:33.808 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/mon.a/config 2023-06-03T20:23:33.874 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:23:33.913 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:23:33.917 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:23:33.917 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:23:34.018 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:23:34.068 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:23:34.071 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:23:34.071 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:23:34.155 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:23:33 smithi164 bash[59462]: cluster 1685823813.0180306 mgr.y (mgr.34104) 1429 : cluster 0 pgmap v1448: 89 pgs: 1 activating+undersized+degraded+remapped, 1 active, 2 active+recovery_wait+undersized+degraded+remapped, 6 active+undersized+degraded+remapped+backfill_wait, 79 active+clean; 6.9 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 150 B/s rd, 3.2 MiB/s wr, 385 op/s; 63780/338799 objects degraded (18.825%); 235 KiB/s, 3 objects/s recovering 2023-06-03T20:23:34.173 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:23:34.199 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:23:34.201 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:23:34.201 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:23:34.279 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:23:33 smithi089 bash[75068]: cluster 1685823813.0180306 mgr.y (mgr.34104) 1429 : cluster 0 pgmap v1448: 89 pgs: 1 activating+undersized+degraded+remapped, 1 active, 2 active+recovery_wait+undersized+degraded+remapped, 6 active+undersized+degraded+remapped+backfill_wait, 79 active+clean; 6.9 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 150 B/s rd, 3.2 MiB/s wr, 385 op/s; 63780/338799 objects degraded (18.825%); 235 KiB/s, 3 objects/s recovering 2023-06-03T20:23:34.279 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:23:33 smithi089 bash[78452]: cluster 1685823813.0180306 mgr.y (mgr.34104) 1429 : cluster 0 pgmap v1448: 89 pgs: 1 activating+undersized+degraded+remapped, 1 active, 2 active+recovery_wait+undersized+degraded+remapped, 6 active+undersized+degraded+remapped+backfill_wait, 79 active+clean; 6.9 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 150 B/s rd, 3.2 MiB/s wr, 385 op/s; 63780/338799 objects degraded (18.825%); 235 KiB/s, 3 objects/s recovering 2023-06-03T20:23:34.303 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:23:34.343 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:23:34.349 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:23:34.349 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:23:34.411 INFO:teuthology.orchestra.run.smithi164.stdout:{ 2023-06-03T20:23:34.411 INFO:teuthology.orchestra.run.smithi164.stdout: "ops": [ 2023-06-03T20:23:34.412 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:23:34.412 INFO:teuthology.orchestra.run.smithi164.stdout: "description": "osd_repop(client.36573.0:120681 13.1 e252/251 13:8ece5731:::benchmark_data_smithi089_211581_object5156:head v 252'7565, mlcod=245'102)", 2023-06-03T20:23:34.412 INFO:teuthology.orchestra.run.smithi164.stdout: "initiated_at": "2023-06-03T20:23:34.377880+0000", 2023-06-03T20:23:34.412 INFO:teuthology.orchestra.run.smithi164.stdout: "age": 0.025260054000000001, 2023-06-03T20:23:34.412 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.025296666999999998, 2023-06-03T20:23:34.412 INFO:teuthology.orchestra.run.smithi164.stdout: "type_data": { 2023-06-03T20:23:34.412 INFO:teuthology.orchestra.run.smithi164.stdout: "flag_point": "started", 2023-06-03T20:23:34.413 INFO:teuthology.orchestra.run.smithi164.stdout: "events": [ 2023-06-03T20:23:34.413 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:23:34.413 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "initiated", 2023-06-03T20:23:34.413 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:23:34.377880+0000", 2023-06-03T20:23:34.413 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0 2023-06-03T20:23:34.413 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:23:34.413 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:23:34.414 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "throttled", 2023-06-03T20:23:34.414 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:23:34.377880+0000", 2023-06-03T20:23:34.414 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0 2023-06-03T20:23:34.414 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:23:34.414 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:23:34.414 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "header_read", 2023-06-03T20:23:34.414 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:23:34.377878+0000", 2023-06-03T20:23:34.414 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 4294967295.9999981 2023-06-03T20:23:34.415 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:23:34.415 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:23:34.415 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "all_read", 2023-06-03T20:23:34.415 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:23:34.377896+0000", 2023-06-03T20:23:34.415 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 1.7439e-05 2023-06-03T20:23:34.415 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:23:34.415 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:23:34.415 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "dispatched", 2023-06-03T20:23:34.416 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:23:34.377919+0000", 2023-06-03T20:23:34.416 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 2.2818000000000001e-05 2023-06-03T20:23:34.416 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:23:34.416 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:23:34.416 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "queued_for_pg", 2023-06-03T20:23:34.416 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:23:34.377931+0000", 2023-06-03T20:23:34.416 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 1.2084e-05 2023-06-03T20:23:34.416 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:23:34.417 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:23:34.417 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "reached_pg", 2023-06-03T20:23:34.417 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:23:34.391431+0000", 2023-06-03T20:23:34.417 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.013500003 2023-06-03T20:23:34.417 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:23:34.417 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:23:34.417 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "started", 2023-06-03T20:23:34.418 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:23:34.391497+0000", 2023-06-03T20:23:34.418 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 6.6062999999999994e-05 2023-06-03T20:23:34.418 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:23:34.418 INFO:teuthology.orchestra.run.smithi164.stdout: ] 2023-06-03T20:23:34.418 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:23:34.418 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:23:34.418 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:23:34.418 INFO:teuthology.orchestra.run.smithi164.stdout: "description": "osd_repop(client.36573.0:120691 13.7 e252/251 13:e990b7a6:::benchmark_data_smithi089_211581_object5166:head v 252'7649, mlcod=252'7647)", 2023-06-03T20:23:34.419 INFO:teuthology.orchestra.run.smithi164.stdout: "initiated_at": "2023-06-03T20:23:34.398315+0000", 2023-06-03T20:23:34.419 INFO:teuthology.orchestra.run.smithi164.stdout: "age": 0.0048254839999999997, 2023-06-03T20:23:34.419 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.004928417, 2023-06-03T20:23:34.419 INFO:teuthology.orchestra.run.smithi164.stdout: "type_data": { 2023-06-03T20:23:34.419 INFO:teuthology.orchestra.run.smithi164.stdout: "flag_point": "started", 2023-06-03T20:23:34.419 INFO:teuthology.orchestra.run.smithi164.stdout: "events": [ 2023-06-03T20:23:34.419 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:23:34.419 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "initiated", 2023-06-03T20:23:34.420 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:23:34.398315+0000", 2023-06-03T20:23:34.420 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0 2023-06-03T20:23:34.420 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:23:34.420 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:23:34.420 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "throttled", 2023-06-03T20:23:34.420 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:23:34.398315+0000", 2023-06-03T20:23:34.420 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0 2023-06-03T20:23:34.420 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:23:34.421 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:23:34.421 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "header_read", 2023-06-03T20:23:34.421 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:23:34.398313+0000", 2023-06-03T20:23:34.421 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 4294967295.9999981 2023-06-03T20:23:34.421 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:23:34.421 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:23:34.421 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "all_read", 2023-06-03T20:23:34.422 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:23:34.398328+0000", 2023-06-03T20:23:34.422 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 1.4929999999999999e-05 2023-06-03T20:23:34.422 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:23:34.422 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:23:34.422 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "dispatched", 2023-06-03T20:23:34.422 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:23:34.398353+0000", 2023-06-03T20:23:34.422 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 2.5255000000000001e-05 2023-06-03T20:23:34.422 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:23:34.423 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:23:34.423 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "queued_for_pg", 2023-06-03T20:23:34.423 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:23:34.398365+0000", 2023-06-03T20:23:34.423 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 1.2044e-05 2023-06-03T20:23:34.423 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:23:34.423 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:23:34.423 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "reached_pg", 2023-06-03T20:23:34.423 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:23:34.398446+0000", 2023-06-03T20:23:34.424 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 8.0536000000000003e-05 2023-06-03T20:23:34.424 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:23:34.424 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:23:34.424 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "started", 2023-06-03T20:23:34.424 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:23:34.398470+0000", 2023-06-03T20:23:34.424 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 2.3676000000000002e-05 2023-06-03T20:23:34.424 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:23:34.424 INFO:teuthology.orchestra.run.smithi164.stdout: ] 2023-06-03T20:23:34.425 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:23:34.425 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:23:34.425 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:23:34.425 INFO:teuthology.orchestra.run.smithi164.stdout: "description": "osd_repop(client.36573.0:120693 13.e e252/251 13:7841e66d:::benchmark_data_smithi089_211581_object5168:head v 252'7477, mlcod=245'344)", 2023-06-03T20:23:34.425 INFO:teuthology.orchestra.run.smithi164.stdout: "initiated_at": "2023-06-03T20:23:34.402581+0000", 2023-06-03T20:23:34.425 INFO:teuthology.orchestra.run.smithi164.stdout: "age": 0.00055939200000000003, 2023-06-03T20:23:34.425 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.00076631799999999999, 2023-06-03T20:23:34.425 INFO:teuthology.orchestra.run.smithi164.stdout: "type_data": { 2023-06-03T20:23:34.426 INFO:teuthology.orchestra.run.smithi164.stdout: "flag_point": "started", 2023-06-03T20:23:34.426 INFO:teuthology.orchestra.run.smithi164.stdout: "events": [ 2023-06-03T20:23:34.426 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:23:34.426 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "initiated", 2023-06-03T20:23:34.426 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:23:34.402581+0000", 2023-06-03T20:23:34.426 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0 2023-06-03T20:23:34.426 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:23:34.427 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:23:34.427 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "throttled", 2023-06-03T20:23:34.427 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:23:34.402581+0000", 2023-06-03T20:23:34.427 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0 2023-06-03T20:23:34.427 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:23:34.427 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:23:34.427 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "header_read", 2023-06-03T20:23:34.427 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:23:34.402580+0000", 2023-06-03T20:23:34.428 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 4294967295.9999986 2023-06-03T20:23:34.428 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:23:34.428 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:23:34.428 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "all_read", 2023-06-03T20:23:34.428 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:23:34.402593+0000", 2023-06-03T20:23:34.428 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 1.358e-05 2023-06-03T20:23:34.428 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:23:34.428 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:23:34.429 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "dispatched", 2023-06-03T20:23:34.429 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:23:34.402614+0000", 2023-06-03T20:23:34.429 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 2.0894999999999999e-05 2023-06-03T20:23:34.429 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:23:34.429 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:23:34.429 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "queued_for_pg", 2023-06-03T20:23:34.429 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:23:34.402625+0000", 2023-06-03T20:23:34.429 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 1.113e-05 2023-06-03T20:23:34.430 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:23:34.430 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:23:34.430 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "reached_pg", 2023-06-03T20:23:34.430 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:23:34.402694+0000", 2023-06-03T20:23:34.430 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 6.8269999999999995e-05 2023-06-03T20:23:34.430 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:23:34.430 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:23:34.430 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "started", 2023-06-03T20:23:34.431 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:23:34.402734+0000", 2023-06-03T20:23:34.431 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 4.0268000000000001e-05 2023-06-03T20:23:34.431 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:23:34.431 INFO:teuthology.orchestra.run.smithi164.stdout: ] 2023-06-03T20:23:34.431 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:23:34.431 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:23:34.431 INFO:teuthology.orchestra.run.smithi164.stdout: ], 2023-06-03T20:23:34.431 INFO:teuthology.orchestra.run.smithi164.stdout: "num_ops": 3 2023-06-03T20:23:34.432 INFO:teuthology.orchestra.run.smithi164.stdout:} 2023-06-03T20:23:34.450 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:23:34.494 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:23:34.498 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:23:34.498 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:23:34.599 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:23:34.643 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:23:34.645 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:23:34.645 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:23:34.746 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:23:34.781 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:23:34.788 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:23:34.789 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:23:34.890 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:23:34.932 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:23:34.936 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:23:34.937 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:23:35.038 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:23:35.066 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:23:35.068 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:23:35.069 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:23:35.170 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:23:35.204 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:23:35.208 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:23:35.208 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:23:35.309 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:23:35.347 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:23:35.355 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:23:35.356 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:23:35.457 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:23:35.485 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:23:35.487 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:23:35.488 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:23:35.589 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:23:35.622 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:23:35.625 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:23:35.626 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:23:35.727 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:23:35.756 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:23:35.763 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:23:35.764 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:23:35.864 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:23:35.902 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:23:35.903 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:23:35.904 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:23:36.005 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:23:36.046 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:23:36.050 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:23:36.051 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:23:36.151 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:23:36.160 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:23:36 smithi089 bash[75068]: cluster 1685823815.0189786 mgr.y (mgr.34104) 1430 : cluster 0 pgmap v1449: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 6 active+undersized+degraded+remapped+backfill_wait, 80 active+clean; 6.8 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 127 B/s rd, 2.7 MiB/s wr, 506 op/s; 63053/334482 objects degraded (18.851%); 679 KiB/s, 10 objects/s recovering 2023-06-03T20:23:36.161 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:23:36 smithi089 bash[78452]: cluster 1685823815.0189786 mgr.y (mgr.34104) 1430 : cluster 0 pgmap v1449: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 6 active+undersized+degraded+remapped+backfill_wait, 80 active+clean; 6.8 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 127 B/s rd, 2.7 MiB/s wr, 506 op/s; 63053/334482 objects degraded (18.851%); 679 KiB/s, 10 objects/s recovering 2023-06-03T20:23:36.196 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:23:36.199 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:23:36.199 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:23:36.300 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:23:36.330 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:23:36.334 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:23:36 smithi164 bash[59462]: cluster 1685823815.0189786 mgr.y (mgr.34104) 1430 : cluster 0 pgmap v1449: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 6 active+undersized+degraded+remapped+backfill_wait, 80 active+clean; 6.8 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 127 B/s rd, 2.7 MiB/s wr, 506 op/s; 63053/334482 objects degraded (18.851%); 679 KiB/s, 10 objects/s recovering 2023-06-03T20:23:36.334 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:23:36.335 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:23:36.436 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:23:36.440 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 20:23:36 smithi089 bash[71163]: ::ffff:172.21.15.164 - - [03/Jun/2023:20:23:36] "GET /metrics HTTP/1.1" 200 228996 "" "Prometheus/2.33.4" 2023-06-03T20:23:36.466 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:23:36.470 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:23:36.471 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:23:36.571 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:23:36.605 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:23:36.611 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:23:36.612 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:23:36.712 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:23:36.751 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:23:36.753 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:23:36.754 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:23:36.855 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:23:36.883 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:23:36.887 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:23:36.888 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:23:36.989 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:23:37.026 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:23:37.032 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:23:37.032 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:23:37.133 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:23:37.171 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:23:37.174 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:23:37.174 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:23:37.275 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:23:37.310 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:23:37.312 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:23:37.312 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:23:37.413 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:23:37.451 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:23:37.454 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:23:37.454 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:23:37.555 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:23:37.584 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:23:37.587 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:23:37.588 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:23:37.689 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:23:37.720 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:23:37.722 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:23:37.722 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:23:37.823 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:23:37.858 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:23:37.863 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:23:37.864 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:23:37.965 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:23:37.997 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:23:38.001 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:23:38.002 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:23:38.103 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:23:38.136 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:23:38.142 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:23:38.143 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:23:38.244 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:23:38.270 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:23:38.272 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:23:38.273 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:23:38.334 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:23:38 smithi164 bash[59462]: cluster 1685823817.019638 mgr.y (mgr.34104) 1431 : cluster 0 pgmap v1450: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 6 active+undersized+degraded+remapped+backfill_wait, 80 active+clean; 6.8 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 104 B/s rd, 2.2 MiB/s wr, 496 op/s; 63053/332076 objects degraded (18.988%); 555 KiB/s, 8 objects/s recovering 2023-06-03T20:23:38.335 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:23:38 smithi164 bash[59462]: audit 1685823817.1417842 mon.a (mon.0) 1417 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:23:38.335 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:23:38 smithi164 bash[59462]: audit 1685823817.148533 mon.c (mon.2) 605 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T20:23:38.335 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:23:38 smithi164 bash[59462]: cluster 1685823817.9248016 mon.a (mon.0) 1418 : cluster 3 Health check update: Degraded data redundancy: 63053/332076 objects degraded (18.988%), 9 pgs degraded (PG_DEGRADED) 2023-06-03T20:23:38.375 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:23:38.408 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:23:38.409 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:23:38.410 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:23:38.511 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:23:38.529 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:23:38 smithi089 bash[75068]: cluster 1685823817.019638 mgr.y (mgr.34104) 1431 : cluster 0 pgmap v1450: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 6 active+undersized+degraded+remapped+backfill_wait, 80 active+clean; 6.8 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 104 B/s rd, 2.2 MiB/s wr, 496 op/s; 63053/332076 objects degraded (18.988%); 555 KiB/s, 8 objects/s recovering 2023-06-03T20:23:38.529 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:23:38 smithi089 bash[75068]: audit 1685823817.1417842 mon.a (mon.0) 1417 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:23:38.529 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:23:38 smithi089 bash[75068]: audit 1685823817.148533 mon.c (mon.2) 605 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T20:23:38.530 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:23:38 smithi089 bash[75068]: cluster 1685823817.9248016 mon.a (mon.0) 1418 : cluster 3 Health check update: Degraded data redundancy: 63053/332076 objects degraded (18.988%), 9 pgs degraded (PG_DEGRADED) 2023-06-03T20:23:38.530 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:23:38 smithi089 bash[78452]: cluster 1685823817.019638 mgr.y (mgr.34104) 1431 : cluster 0 pgmap v1450: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 6 active+undersized+degraded+remapped+backfill_wait, 80 active+clean; 6.8 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 104 B/s rd, 2.2 MiB/s wr, 496 op/s; 63053/332076 objects degraded (18.988%); 555 KiB/s, 8 objects/s recovering 2023-06-03T20:23:38.530 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:23:38 smithi089 bash[78452]: audit 1685823817.1417842 mon.a (mon.0) 1417 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:23:38.530 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:23:38 smithi089 bash[78452]: audit 1685823817.148533 mon.c (mon.2) 605 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T20:23:38.530 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:23:38 smithi089 bash[78452]: cluster 1685823817.9248016 mon.a (mon.0) 1418 : cluster 3 Health check update: Degraded data redundancy: 63053/332076 objects degraded (18.988%), 9 pgs degraded (PG_DEGRADED) 2023-06-03T20:23:38.548 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:23:38.553 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:23:38.553 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:23:38.654 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:23:38.696 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:23:38.702 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:23:38.703 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:23:38.804 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:23:38.842 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:23:38.849 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:23:38.850 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:23:38.951 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:23:38.992 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:23:38.997 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:23:38.997 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:23:39.098 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:23:39.126 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:23:39.130 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:23:39.131 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:23:39.232 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:23:39.276 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:23:39.278 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:23:39.279 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:23:39.367 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/mon.a/config 2023-06-03T20:23:39.380 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:23:39.419 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:23:39.421 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:23:39.422 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:23:39.523 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:23:39.574 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:23:39.575 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:23:39.576 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:23:39.677 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:23:39.714 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:23:39.717 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:23:39.718 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:23:39.819 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:23:39.851 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:23:39.853 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:23:39.853 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:23:39.954 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:23:40.006 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:23:40.011 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:23:40.110 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:23:40.148 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:23:40 smithi089 bash[75068]: cluster 1685823819.0203493 mgr.y (mgr.34104) 1432 : cluster 0 pgmap v1451: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 6 active+undersized+degraded+remapped+backfill_wait, 80 active+clean; 6.6 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 113 KiB/s wr, 553 op/s; 62091/326295 objects degraded (19.029%); 948 KiB/s, 15 objects/s recovering 2023-06-03T20:23:40.212 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:23:40.246 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:23:40.251 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:23:40.252 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:23:40.334 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:23:40 smithi164 bash[59462]: cluster 1685823819.0203493 mgr.y (mgr.34104) 1432 : cluster 0 pgmap v1451: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 6 active+undersized+degraded+remapped+backfill_wait, 80 active+clean; 6.6 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 113 KiB/s wr, 553 op/s; 62091/326295 objects degraded (19.029%); 948 KiB/s, 15 objects/s recovering 2023-06-03T20:23:40.353 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:23:40.390 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:23:40.392 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:23:40.393 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:23:40.494 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:23:40.503 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:23:40 smithi089 bash[78452]: cluster 1685823819.0203493 mgr.y (mgr.34104) 1432 : cluster 0 pgmap v1451: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 6 active+undersized+degraded+remapped+backfill_wait, 80 active+clean; 6.6 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 113 KiB/s wr, 553 op/s; 62091/326295 objects degraded (19.029%); 948 KiB/s, 15 objects/s recovering 2023-06-03T20:23:40.526 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:23:40.528 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:23:40.528 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:23:40.609 INFO:tasks.thrashosds.thrasher:in_osds: [0, 2, 3, 4, 5, 7, 1] out_osds: [6] dead_osds: [] live_osds: [0, 1, 2, 3, 5, 6, 7, 4] 2023-06-03T20:23:40.610 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-06-03T20:23:40.610 INFO:tasks.thrashosds.thrasher:Adding osd 6 2023-06-03T20:23:40.610 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph osd in 6 2023-06-03T20:23:40.629 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:23:40.713 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:23:40.719 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:23:40.720 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:23:40.820 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:23:40.859 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:23:40.861 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:23:40.862 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:23:40.963 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:23:41.019 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:23:41.024 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:23:41.024 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:23:41.125 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:23:41.163 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:23:41.173 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:23:41.173 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:23:41.274 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:23:41.303 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:23:41.308 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:23:41.309 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:23:41.410 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:23:41.436 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:23:41.441 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:23:41.442 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:23:41.543 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:23:41.578 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:23:41.581 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:23:41.582 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:23:41.683 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:23:41.722 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:23:41.730 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:23:41.731 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:23:41.832 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:23:41.870 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:23:41.875 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:23:41.876 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:23:41.977 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:23:42.026 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:23:42.031 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:23:42.033 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:23:42.134 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:23:42.169 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:23:42.174 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:23:42.175 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:23:42.276 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:23:42.286 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:23:42 smithi164 bash[59462]: cluster 1685823821.021083 mgr.y (mgr.34104) 1433 : cluster 0 pgmap v1452: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 6 active+undersized+degraded+remapped+backfill_wait, 80 active+clean; 6.6 GiB data, 20 GiB used, 605 GiB / 626 GiB avail; 0 B/s wr, 521 op/s; 61654/324276 objects degraded (19.013%); 853 KiB/s, 13 objects/s recovering 2023-06-03T20:23:42.337 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:23:42.342 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:23:42.343 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:23:42.444 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:23:42.452 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:23:42 smithi089 bash[75068]: cluster 1685823821.021083 mgr.y (mgr.34104) 1433 : cluster 0 pgmap v1452: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 6 active+undersized+degraded+remapped+backfill_wait, 80 active+clean; 6.6 GiB data, 20 GiB used, 605 GiB / 626 GiB avail; 0 B/s wr, 521 op/s; 61654/324276 objects degraded (19.013%); 853 KiB/s, 13 objects/s recovering 2023-06-03T20:23:42.453 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:23:42 smithi089 bash[78452]: cluster 1685823821.021083 mgr.y (mgr.34104) 1433 : cluster 0 pgmap v1452: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 6 active+undersized+degraded+remapped+backfill_wait, 80 active+clean; 6.6 GiB data, 20 GiB used, 605 GiB / 626 GiB avail; 0 B/s wr, 521 op/s; 61654/324276 objects degraded (19.013%); 853 KiB/s, 13 objects/s recovering 2023-06-03T20:23:42.482 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:23:42.488 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:23:42.489 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:23:42.590 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:23:42.695 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:23:42.700 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:23:42.701 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:23:42.802 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:23:42.848 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:23:42.854 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:23:42.855 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:23:42.956 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:23:43.005 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:23:43.007 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:23:43.008 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:23:43.109 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:23:43.143 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:23:43.146 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:23:43.147 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:23:43.160 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:23:43 smithi089 bash[75068]: cluster 1685823822.9263449 mon.a (mon.0) 1419 : cluster 3 Health check update: Degraded data redundancy: 61654/324276 objects degraded (19.013%), 9 pgs degraded (PG_DEGRADED) 2023-06-03T20:23:43.247 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:23:43.289 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:23:43.291 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:23:43.292 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:23:43.393 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:23:43.415 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:23:43 smithi089 bash[78452]: cluster 1685823822.9263449 mon.a (mon.0) 1419 : cluster 3 Health check update: Degraded data redundancy: 61654/324276 objects degraded (19.013%), 9 pgs degraded (PG_DEGRADED) 2023-06-03T20:23:43.450 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:23:43.454 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:23:43.455 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:23:43.556 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:23:43.584 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:23:43 smithi164 bash[59462]: cluster 1685823822.9263449 mon.a (mon.0) 1419 : cluster 3 Health check update: Degraded data redundancy: 61654/324276 objects degraded (19.013%), 9 pgs degraded (PG_DEGRADED) 2023-06-03T20:23:43.606 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:23:43.606 INFO:teuthology.orchestra.run.smithi089.stdout:osd.0: {} 2023-06-03T20:23:43.609 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:23:43.610 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:23:43.645 INFO:teuthology.orchestra.run.smithi089.stdout:osd.1: {} 2023-06-03T20:23:43.692 INFO:teuthology.orchestra.run.smithi089.stdout:osd.2: {} 2023-06-03T20:23:43.712 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:23:43.726 INFO:teuthology.orchestra.run.smithi089.stdout:osd.3: {} 2023-06-03T20:23:43.758 INFO:teuthology.orchestra.run.smithi089.stdout:osd.4: {} 2023-06-03T20:23:43.790 INFO:teuthology.orchestra.run.smithi089.stdout:osd.5: {} 2023-06-03T20:23:43.807 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:23:43.811 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:23:43.812 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:23:43.834 INFO:teuthology.orchestra.run.smithi089.stdout:osd.6: {} 2023-06-03T20:23:43.866 INFO:teuthology.orchestra.run.smithi089.stdout:osd.7: {} 2023-06-03T20:23:43.870 INFO:teuthology.orchestra.run.smithi089.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-06-03T20:23:43.870 INFO:teuthology.orchestra.run.smithi089.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-06-03T20:23:43.870 INFO:teuthology.orchestra.run.smithi089.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-06-03T20:23:43.871 INFO:teuthology.orchestra.run.smithi089.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-06-03T20:23:43.871 INFO:teuthology.orchestra.run.smithi089.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-06-03T20:23:43.871 INFO:teuthology.orchestra.run.smithi089.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-06-03T20:23:43.871 INFO:teuthology.orchestra.run.smithi089.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-06-03T20:23:43.871 INFO:teuthology.orchestra.run.smithi089.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-06-03T20:23:43.913 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:23:43.964 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:23:43.966 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:23:43.967 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:23:44.068 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:23:44.103 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:23:44.107 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:23:44.108 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:23:44.178 INFO:teuthology.orchestra.run.smithi089.stderr:noscrub is set 2023-06-03T20:23:44.210 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:23:44.239 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:23:44.243 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:23:44.244 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:23:44.344 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:23:44.378 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:23:44.380 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:23:44.381 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:23:44.482 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:23:44.491 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:23:44 smithi164 bash[59462]: cluster 1685823823.02182 mgr.y (mgr.34104) 1434 : cluster 0 pgmap v1453: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 6 active+undersized+degraded+remapped+backfill_wait, 80 active+clean; 6.5 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 652 op/s; 61012/318621 objects degraded (19.149%); 853 KiB/s, 13 objects/s recovering 2023-06-03T20:23:44.491 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:23:44 smithi164 bash[59462]: audit 1685823823.542966 mon.a (mon.0) 1420 : audit 0 from='client.? 172.21.15.89:0/2213941085' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-03T20:23:44.491 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:23:44 smithi164 bash[59462]: audit 1685823823.5430098 mon.c (mon.2) 606 : audit 1 from='client.? 172.21.15.89:0/2867379289' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-06-03T20:23:44.491 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:23:44 smithi164 bash[59462]: audit 1685823823.543883 mon.a (mon.0) 1421 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-06-03T20:23:44.529 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:23:44 smithi089 bash[75068]: cluster 1685823823.02182 mgr.y (mgr.34104) 1434 : cluster 0 pgmap v1453: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 6 active+undersized+degraded+remapped+backfill_wait, 80 active+clean; 6.5 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 652 op/s; 61012/318621 objects degraded (19.149%); 853 KiB/s, 13 objects/s recovering 2023-06-03T20:23:44.529 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:23:44 smithi089 bash[75068]: audit 1685823823.542966 mon.a (mon.0) 1420 : audit 0 from='client.? 172.21.15.89:0/2213941085' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-03T20:23:44.529 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:23:44 smithi089 bash[75068]: audit 1685823823.5430098 mon.c (mon.2) 606 : audit 1 from='client.? 172.21.15.89:0/2867379289' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-06-03T20:23:44.529 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:23:44 smithi089 bash[75068]: audit 1685823823.543883 mon.a (mon.0) 1421 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-06-03T20:23:44.530 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:23:44 smithi089 bash[78452]: cluster 1685823823.02182 mgr.y (mgr.34104) 1434 : cluster 0 pgmap v1453: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 6 active+undersized+degraded+remapped+backfill_wait, 80 active+clean; 6.5 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 652 op/s; 61012/318621 objects degraded (19.149%); 853 KiB/s, 13 objects/s recovering 2023-06-03T20:23:44.530 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:23:44 smithi089 bash[78452]: audit 1685823823.542966 mon.a (mon.0) 1420 : audit 0 from='client.? 172.21.15.89:0/2213941085' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-03T20:23:44.530 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:23:44 smithi089 bash[78452]: audit 1685823823.5430098 mon.c (mon.2) 606 : audit 1 from='client.? 172.21.15.89:0/2867379289' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-06-03T20:23:44.530 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:23:44 smithi089 bash[78452]: audit 1685823823.543883 mon.a (mon.0) 1421 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-06-03T20:23:44.531 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:23:44.534 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:23:44.534 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:23:44.635 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:23:44.675 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:23:44.682 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:23:44.682 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:23:44.783 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:23:44.826 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:23:44.833 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:23:44.834 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:23:44.935 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:23:44.975 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:23:44.980 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:23:44.981 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:23:45.082 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:23:45.117 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:23:45.123 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:23:45.123 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:23:45.186 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:23:45 smithi089 bash[78452]: cluster 1685823824.1547153 mon.a (mon.0) 1422 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-06-03T20:23:45.187 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:23:45 smithi089 bash[78452]: audit 1685823824.165854 mon.a (mon.0) 1423 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-06-03T20:23:45.187 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:23:45 smithi089 bash[78452]: cluster 1685823824.165891 osd.7 (osd.7) 187 : cluster 3 failed to encode map e253 with expected crc 2023-06-03T20:23:45.187 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:23:45 smithi089 bash[78452]: cluster 1685823824.16611 mon.a (mon.0) 1424 : cluster 0 osdmap e253: 8 total, 8 up, 7 in 2023-06-03T20:23:45.187 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:23:45 smithi089 bash[78452]: cluster 1685823824.1734622 osd.5 (osd.5) 195 : cluster 3 failed to encode map e253 with expected crc 2023-06-03T20:23:45.187 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:23:45 smithi089 bash[78452]: cluster 1685823824.1811173 osd.5 (osd.5) 196 : cluster 3 failed to encode map e253 with expected crc 2023-06-03T20:23:45.187 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:23:45 smithi089 bash[78452]: cluster 1685823824.2783494 osd.6 (osd.6) 193 : cluster 3 failed to encode map e253 with expected crc 2023-06-03T20:23:45.188 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:23:45 smithi089 bash[75068]: cluster 1685823824.1547153 mon.a (mon.0) 1422 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-06-03T20:23:45.188 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:23:45 smithi089 bash[75068]: audit 1685823824.165854 mon.a (mon.0) 1423 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-06-03T20:23:45.188 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:23:45 smithi089 bash[75068]: cluster 1685823824.165891 osd.7 (osd.7) 187 : cluster 3 failed to encode map e253 with expected crc 2023-06-03T20:23:45.188 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:23:45 smithi089 bash[75068]: cluster 1685823824.16611 mon.a (mon.0) 1424 : cluster 0 osdmap e253: 8 total, 8 up, 7 in 2023-06-03T20:23:45.188 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:23:45 smithi089 bash[75068]: cluster 1685823824.1734622 osd.5 (osd.5) 195 : cluster 3 failed to encode map e253 with expected crc 2023-06-03T20:23:45.188 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:23:45 smithi089 bash[75068]: cluster 1685823824.1811173 osd.5 (osd.5) 196 : cluster 3 failed to encode map e253 with expected crc 2023-06-03T20:23:45.188 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:23:45 smithi089 bash[75068]: cluster 1685823824.2783494 osd.6 (osd.6) 193 : cluster 3 failed to encode map e253 with expected crc 2023-06-03T20:23:45.224 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:23:45.263 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:23:45.269 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:23:45.270 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:23:45.371 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:23:45.407 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:23:45.411 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:23:45.413 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:23:45.514 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:23:45.544 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:23:45.550 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:23:45.551 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:23:45.584 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:23:45 smithi164 bash[59462]: cluster 1685823824.1547153 mon.a (mon.0) 1422 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-06-03T20:23:45.643 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:23:45 smithi164 bash[59462]: audit 1685823824.165854 mon.a (mon.0) 1423 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-06-03T20:23:45.643 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:23:45 smithi164 bash[59462]: cluster 1685823824.165891 osd.7 (osd.7) 187 : cluster 3 failed to encode map e253 with expected crc 2023-06-03T20:23:45.643 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:23:45 smithi164 bash[59462]: cluster 1685823824.16611 mon.a (mon.0) 1424 : cluster 0 osdmap e253: 8 total, 8 up, 7 in 2023-06-03T20:23:45.644 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:23:45 smithi164 bash[59462]: cluster 1685823824.1734622 osd.5 (osd.5) 195 : cluster 3 failed to encode map e253 with expected crc 2023-06-03T20:23:45.644 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:23:45 smithi164 bash[59462]: cluster 1685823824.1811173 osd.5 (osd.5) 196 : cluster 3 failed to encode map e253 with expected crc 2023-06-03T20:23:45.644 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:23:45 smithi164 bash[59462]: cluster 1685823824.2783494 osd.6 (osd.6) 193 : cluster 3 failed to encode map e253 with expected crc 2023-06-03T20:23:45.653 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:23:45.689 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:23:45.691 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:23:45.692 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:23:45.793 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:23:45.846 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:23:45.853 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:23:45.854 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:23:45.955 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:23:45.996 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:23:46.003 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:23:46.004 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:23:46.105 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:23:46.156 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:23:46.158 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:23:46.159 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:23:46.194 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:23:46 smithi089 bash[75068]: cluster 1685823825.0223718 mgr.y (mgr.34104) 1435 : cluster 0 pgmap v1455: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 6 active+undersized+degraded+remapped+backfill_wait, 80 active+clean; 6.4 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 669 op/s; 60357/314376 objects degraded (19.199%); 934 KiB/s, 14 objects/s recovering 2023-06-03T20:23:46.260 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:23:46.270 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:23:46 smithi164 bash[59462]: cluster 1685823825.0223718 mgr.y (mgr.34104) 1435 : cluster 0 pgmap v1455: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 6 active+undersized+degraded+remapped+backfill_wait, 80 active+clean; 6.4 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 669 op/s; 60357/314376 objects degraded (19.199%); 934 KiB/s, 14 objects/s recovering 2023-06-03T20:23:46.308 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:23:46.315 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:23:46.316 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:23:46.417 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:23:46.468 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:23:46.473 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:23:46.474 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:23:46.529 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 20:23:46 smithi089 bash[71163]: ::ffff:172.21.15.164 - - [03/Jun/2023:20:23:46] "GET /metrics HTTP/1.1" 200 228840 "" "Prometheus/2.33.4" 2023-06-03T20:23:46.530 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:23:46 smithi089 bash[78452]: cluster 1685823825.0223718 mgr.y (mgr.34104) 1435 : cluster 0 pgmap v1455: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 6 active+undersized+degraded+remapped+backfill_wait, 80 active+clean; 6.4 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 669 op/s; 60357/314376 objects degraded (19.199%); 934 KiB/s, 14 objects/s recovering 2023-06-03T20:23:46.575 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:23:46.616 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:23:46.621 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:23:46.622 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:23:46.723 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:23:46.763 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:23:46.769 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:23:46.770 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:23:46.871 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:23:46.917 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:23:46.928 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:23:46.929 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:23:47.030 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:23:47.067 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:23:47.071 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:23:47.072 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:23:47.172 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:23:47.205 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:23:47.207 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:23:47.207 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:23:47.308 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:23:47.346 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:23:47.347 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:23:47.348 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:23:47.450 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:23:47.482 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:23:47.486 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:23:47.487 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:23:47.589 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:23:47.638 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:23:47.642 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:23:47.643 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:23:47.744 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:23:47.791 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:23:47.798 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:23:47.799 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:23:47.900 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:23:47.939 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:23:47.941 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:23:47.942 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:23:48.043 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:23:48.079 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:23:48.087 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:23:48.088 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:23:48.189 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:23:48.227 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:23:48.234 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:23:48.235 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:23:48.334 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:23:48 smithi164 bash[59462]: cluster 1685823827.0230138 mgr.y (mgr.34104) 1436 : cluster 0 pgmap v1456: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 6 active+undersized+degraded+remapped+backfill_wait, 80 active+clean; 6.3 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 666 op/s; 60357/312081 objects degraded (19.340%); 934 KiB/s, 14 objects/s recovering 2023-06-03T20:23:48.335 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:23:48 smithi164 bash[59462]: cluster 1685823827.9275124 mon.a (mon.0) 1425 : cluster 3 Health check update: Degraded data redundancy: 60357/312081 objects degraded (19.340%), 9 pgs degraded (PG_DEGRADED) 2023-06-03T20:23:48.336 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:23:48.371 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:23:48.376 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:23:48.378 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:23:48.479 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:23:48.525 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:23:48.529 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:23:48 smithi089 bash[75068]: cluster 1685823827.0230138 mgr.y (mgr.34104) 1436 : cluster 0 pgmap v1456: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 6 active+undersized+degraded+remapped+backfill_wait, 80 active+clean; 6.3 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 666 op/s; 60357/312081 objects degraded (19.340%); 934 KiB/s, 14 objects/s recovering 2023-06-03T20:23:48.529 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:23:48 smithi089 bash[75068]: cluster 1685823827.9275124 mon.a (mon.0) 1425 : cluster 3 Health check update: Degraded data redundancy: 60357/312081 objects degraded (19.340%), 9 pgs degraded (PG_DEGRADED) 2023-06-03T20:23:48.529 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:23:48 smithi089 bash[78452]: cluster 1685823827.0230138 mgr.y (mgr.34104) 1436 : cluster 0 pgmap v1456: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 6 active+undersized+degraded+remapped+backfill_wait, 80 active+clean; 6.3 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 666 op/s; 60357/312081 objects degraded (19.340%); 934 KiB/s, 14 objects/s recovering 2023-06-03T20:23:48.530 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:23:48 smithi089 bash[78452]: cluster 1685823827.9275124 mon.a (mon.0) 1425 : cluster 3 Health check update: Degraded data redundancy: 60357/312081 objects degraded (19.340%), 9 pgs degraded (PG_DEGRADED) 2023-06-03T20:23:48.530 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:23:48.531 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:23:48.604 INFO:teuthology.orchestra.run.smithi089.stdout:ERROR: (22) Invalid argument 2023-06-03T20:23:48.605 INFO:teuthology.orchestra.run.smithi089.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-06-03T20:23:48.632 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:23:48.662 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:23:48.666 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:23:48.667 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:23:48.768 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:23:48.811 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:23:48.815 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:23:48.816 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:23:48.916 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:23:48.960 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:23:48.966 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:23:48.966 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:23:49.067 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:23:49.113 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:23:49.118 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:23:49.119 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:23:49.220 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:23:49.263 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:23:49.265 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:23:49.266 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:23:49.367 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:23:49.403 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:23:49.407 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:23:49.407 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:23:49.508 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:23:49.553 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:23:49.558 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:23:49.559 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:23:49.596 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph osd set nodeep-scrub 2023-06-03T20:23:49.660 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:23:49.709 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:23:49.710 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:23:49.711 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:23:49.811 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:23:49.818 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-06-03T20:23:49.825 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph daemon osd.2 dump_historic_ops 2023-06-03T20:23:49.846 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:23:49.851 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:23:49.852 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:23:49.952 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:23:49.999 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:23:50.008 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:23:50.009 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:23:50.110 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:23:50.149 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:23:50.151 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:23:50.152 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:23:50.208 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:23:50 smithi089 bash[75068]: cluster 1685823829.0236633 mgr.y (mgr.34104) 1437 : cluster 0 pgmap v1457: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 6 active+undersized+degraded+remapped+backfill_wait, 80 active+clean; 6.2 GiB data, 19 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 655 op/s; 59488/306618 objects degraded (19.401%); 902 KiB/s, 14 objects/s recovering 2023-06-03T20:23:50.238 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/mon.a/config 2023-06-03T20:23:50.253 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:23:50.291 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:23:50.296 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:23:50.297 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:23:50.397 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:23:50.406 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:23:50 smithi164 bash[59462]: cluster 1685823829.0236633 mgr.y (mgr.34104) 1437 : cluster 0 pgmap v1457: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 6 active+undersized+degraded+remapped+backfill_wait, 80 active+clean; 6.2 GiB data, 19 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 655 op/s; 59488/306618 objects degraded (19.401%); 902 KiB/s, 14 objects/s recovering 2023-06-03T20:23:50.455 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:23:50.462 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:23:50.463 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:23:50.529 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:23:50 smithi089 bash[78452]: cluster 1685823829.0236633 mgr.y (mgr.34104) 1437 : cluster 0 pgmap v1457: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 6 active+undersized+degraded+remapped+backfill_wait, 80 active+clean; 6.2 GiB data, 19 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 655 op/s; 59488/306618 objects degraded (19.401%); 902 KiB/s, 14 objects/s recovering 2023-06-03T20:23:50.564 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:23:50.608 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:23:50.613 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:23:50.613 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:23:50.714 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:23:50.744 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:23:50.747 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:23:50.748 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:23:50.849 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:23:50.893 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:23:50.896 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:23:50.897 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:23:50.998 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:23:51.040 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:23:51.044 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:23:51.045 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:23:51.146 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:23:51.182 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:23:51.187 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:23:51.188 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:23:51.289 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:23:51.340 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:23:51.344 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:23:51.345 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:23:51.446 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:23:51.477 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:23:51.482 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:23:51.483 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:23:51.584 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:23:51.614 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:23:51.620 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:23:51.621 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:23:51.722 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:23:51.752 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:23:51.755 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:23:51.755 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:23:51.856 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:23:51.901 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:23:51.903 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:23:51.904 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:23:52.004 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:23:52.043 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:23:52.047 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:23:52.047 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:23:52.148 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:23:52.195 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:23:52.198 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:23:52.198 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:23:52.279 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:23:52 smithi089 bash[78452]: cluster 1685823831.0242121 mgr.y (mgr.34104) 1438 : cluster 0 pgmap v1458: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 6 active+undersized+degraded+remapped+backfill_wait, 80 active+clean; 6.2 GiB data, 19 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 649 op/s; 59079/304794 objects degraded (19.383%); 902 KiB/s, 14 objects/s recovering 2023-06-03T20:23:52.280 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:23:52 smithi089 bash[78452]: audit 1685823832.1412485 mon.a (mon.0) 1426 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:23:52.280 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:23:52 smithi089 bash[78452]: audit 1685823832.1439366 mon.c (mon.2) 607 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T20:23:52.280 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:23:52 smithi089 bash[75068]: cluster 1685823831.0242121 mgr.y (mgr.34104) 1438 : cluster 0 pgmap v1458: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 6 active+undersized+degraded+remapped+backfill_wait, 80 active+clean; 6.2 GiB data, 19 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 649 op/s; 59079/304794 objects degraded (19.383%); 902 KiB/s, 14 objects/s recovering 2023-06-03T20:23:52.280 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:23:52 smithi089 bash[75068]: audit 1685823832.1412485 mon.a (mon.0) 1426 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:23:52.281 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:23:52 smithi089 bash[75068]: audit 1685823832.1439366 mon.c (mon.2) 607 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T20:23:52.299 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:23:52.342 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:23:52.346 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:23:52.347 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:23:52.447 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:23:52.479 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:23:52.482 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:23:52.483 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:23:52.584 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:23:52.585 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:23:52 smithi164 bash[59462]: cluster 1685823831.0242121 mgr.y (mgr.34104) 1438 : cluster 0 pgmap v1458: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 6 active+undersized+degraded+remapped+backfill_wait, 80 active+clean; 6.2 GiB data, 19 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 649 op/s; 59079/304794 objects degraded (19.383%); 902 KiB/s, 14 objects/s recovering 2023-06-03T20:23:52.585 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:23:52 smithi164 bash[59462]: audit 1685823832.1412485 mon.a (mon.0) 1426 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:23:52.586 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:23:52 smithi164 bash[59462]: audit 1685823832.1439366 mon.c (mon.2) 607 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T20:23:52.612 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:23:52.614 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:23:52.615 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:23:52.716 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:23:52.744 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:23:52.747 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:23:52.747 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:23:52.848 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:23:52.893 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:23:52.895 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:23:52.895 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:23:52.996 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:23:53.038 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:23:53.043 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:23:53.043 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:23:53.144 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:23:53.175 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:23:53.181 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:23:53.181 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:23:53.212 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:23:53 smithi089 bash[75068]: cluster 1685823832.92854 mon.a (mon.0) 1427 : cluster 3 Health check update: Degraded data redundancy: 59079/304794 objects degraded (19.383%), 9 pgs degraded (PG_DEGRADED) 2023-06-03T20:23:53.282 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:23:53.323 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:23:53.325 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:23:53.326 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:23:53.427 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:23:53.464 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:23:53.466 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:23:53 smithi089 bash[78452]: cluster 1685823832.92854 mon.a (mon.0) 1427 : cluster 3 Health check update: Degraded data redundancy: 59079/304794 objects degraded (19.383%), 9 pgs degraded (PG_DEGRADED) 2023-06-03T20:23:53.467 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:23:53.467 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:23:53.568 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:23:53.584 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:23:53 smithi164 bash[59462]: cluster 1685823832.92854 mon.a (mon.0) 1427 : cluster 3 Health check update: Degraded data redundancy: 59079/304794 objects degraded (19.383%), 9 pgs degraded (PG_DEGRADED) 2023-06-03T20:23:53.595 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:23:53.599 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:23:53.600 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:23:53.700 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:23:53.745 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:23:53.751 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:23:53.752 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:23:53.853 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:23:53.882 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:23:53.889 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:23:53.890 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:23:53.991 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:23:54.036 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:23:54.044 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:23:54.045 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:23:54.146 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:23:54.185 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:23:54.188 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:23:54.189 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:23:54.290 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:23:54.297 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:23:54 smithi089 bash[78452]: cluster 1685823833.0249712 mgr.y (mgr.34104) 1439 : cluster 0 pgmap v1459: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 6 active+undersized+degraded+remapped+backfill_wait, 80 active+clean; 6.1 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 642 op/s; 58477/299334 objects degraded (19.536%); 902 KiB/s, 14 objects/s recovering 2023-06-03T20:23:54.298 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:23:54 smithi089 bash[75068]: cluster 1685823833.0249712 mgr.y (mgr.34104) 1439 : cluster 0 pgmap v1459: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 6 active+undersized+degraded+remapped+backfill_wait, 80 active+clean; 6.1 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 642 op/s; 58477/299334 objects degraded (19.536%); 902 KiB/s, 14 objects/s recovering 2023-06-03T20:23:54.316 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:23:54.319 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:23:54.320 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:23:54.405 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/mon.a/config 2023-06-03T20:23:54.407 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/mon.a/config 2023-06-03T20:23:54.407 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/mon.a/config 2023-06-03T20:23:54.421 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:23:54.450 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:23:54.453 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:23:54.454 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:23:54.555 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:23:54.584 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:23:54 smithi164 bash[59462]: cluster 1685823833.0249712 mgr.y (mgr.34104) 1439 : cluster 0 pgmap v1459: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 6 active+undersized+degraded+remapped+backfill_wait, 80 active+clean; 6.1 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 642 op/s; 58477/299334 objects degraded (19.536%); 902 KiB/s, 14 objects/s recovering 2023-06-03T20:23:54.590 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:23:54.592 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:23:54.593 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:23:54.694 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:23:54.734 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:23:54.738 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:23:54.738 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:23:54.839 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:23:54.886 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:23:54.893 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:23:54.893 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:23:54.994 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:23:55.029 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:23:55.030 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:23:55.031 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:23:55.132 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:23:55.160 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:23:55.164 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:23:55.164 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:23:55.265 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:23:55.308 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:23:55.315 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:23:55.315 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:23:55.416 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:23:55.454 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:23:55.457 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:23:55.458 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:23:55.560 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:23:55.592 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:23:55.594 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:23:55.595 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:23:55.702 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:23:55.735 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:23:55.739 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:23:55.740 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:23:55.841 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:23:55.881 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:23:55.887 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:23:55.887 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:23:55.988 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:23:56.041 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:23:56.047 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:23:56.048 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:23:56.149 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:23:56.192 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:23:56.197 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:23:56.198 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:23:56.299 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:23:56.308 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:23:56 smithi164 bash[59462]: cluster 1685823835.025657 mgr.y (mgr.34104) 1440 : cluster 0 pgmap v1460: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 6 active+undersized+degraded+remapped+backfill_wait, 80 active+clean; 6.0 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 635 op/s; 57810/295329 objects degraded (19.575%); 884 KiB/s, 13 objects/s recovering 2023-06-03T20:23:56.342 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:23:56.344 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:23:56.345 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:23:56.447 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:23:56.482 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:23:56.487 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:23:56.487 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:23:56 smithi089 bash[75068]: cluster 1685823835.025657 mgr.y (mgr.34104) 1440 : cluster 0 pgmap v1460: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 6 active+undersized+degraded+remapped+backfill_wait, 80 active+clean; 6.0 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 635 op/s; 57810/295329 objects degraded (19.575%); 884 KiB/s, 13 objects/s recovering 2023-06-03T20:23:56.488 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 20:23:56 smithi089 bash[71163]: ::ffff:172.21.15.164 - - [03/Jun/2023:20:23:56] "GET /metrics HTTP/1.1" 200 228880 "" "Prometheus/2.33.4" 2023-06-03T20:23:56.488 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:23:56 smithi089 bash[78452]: cluster 1685823835.025657 mgr.y (mgr.34104) 1440 : cluster 0 pgmap v1460: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 6 active+undersized+degraded+remapped+backfill_wait, 80 active+clean; 6.0 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 635 op/s; 57810/295329 objects degraded (19.575%); 884 KiB/s, 13 objects/s recovering 2023-06-03T20:23:56.489 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:23:56.590 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:23:56.630 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:23:56.637 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:23:56.638 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:23:56.739 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:23:56.772 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:23:56.777 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:23:56.777 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:23:56.878 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:23:56.921 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:23:56.925 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:23:56.926 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:23:57.027 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:23:57.059 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:23:57.065 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:23:57.066 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:23:57.167 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:23:57.204 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:23:57.208 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:23:57.209 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:23:57.310 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:23:57.348 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:23:57.350 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:23:57.351 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:23:57.453 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:23:57.490 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:23:57.495 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:23:57.496 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:23:57.597 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:23:57.623 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:23:57.627 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:23:57.627 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:23:57.728 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:23:57.761 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:23:57.762 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:23:57.762 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:23:57.863 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:23:57.900 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:23:57.903 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:23:57.903 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:23:58.004 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:23:58.039 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:23:58.041 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:23:58.042 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:23:58.143 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:23:58.176 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:23:58.178 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:23:58.179 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:23:58.280 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:23:58.311 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:23:58.317 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:23:58.318 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:23:58.419 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:23:58.423 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:23:58 smithi089 bash[75068]: cluster 1685823837.0262978 mgr.y (mgr.34104) 1441 : cluster 0 pgmap v1461: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 6 active+undersized+degraded+remapped+backfill_wait, 80 active+clean; 6.0 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 598 op/s; 57810/292839 objects degraded (19.741%); 800 KiB/s, 12 objects/s recovering 2023-06-03T20:23:58.423 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:23:58 smithi089 bash[75068]: cluster 1685823837.9298024 mon.a (mon.0) 1428 : cluster 3 Health check update: Degraded data redundancy: 57810/292839 objects degraded (19.741%), 9 pgs degraded (PG_DEGRADED) 2023-06-03T20:23:58.448 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:23:58.453 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:23:58.454 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:23:58.555 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:23:58.565 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:23:58 smithi164 bash[59462]: cluster 1685823837.0262978 mgr.y (mgr.34104) 1441 : cluster 0 pgmap v1461: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 6 active+undersized+degraded+remapped+backfill_wait, 80 active+clean; 6.0 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 598 op/s; 57810/292839 objects degraded (19.741%); 800 KiB/s, 12 objects/s recovering 2023-06-03T20:23:58.566 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:23:58 smithi164 bash[59462]: cluster 1685823837.9298024 mon.a (mon.0) 1428 : cluster 3 Health check update: Degraded data redundancy: 57810/292839 objects degraded (19.741%), 9 pgs degraded (PG_DEGRADED) 2023-06-03T20:23:58.594 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:23:58.604 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:23:58.605 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:23:58.706 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:23:58.713 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:23:58 smithi089 bash[78452]: cluster 1685823837.0262978 mgr.y (mgr.34104) 1441 : cluster 0 pgmap v1461: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 6 active+undersized+degraded+remapped+backfill_wait, 80 active+clean; 6.0 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 598 op/s; 57810/292839 objects degraded (19.741%); 800 KiB/s, 12 objects/s recovering 2023-06-03T20:23:58.714 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:23:58 smithi089 bash[78452]: cluster 1685823837.9298024 mon.a (mon.0) 1428 : cluster 3 Health check update: Degraded data redundancy: 57810/292839 objects degraded (19.741%), 9 pgs degraded (PG_DEGRADED) 2023-06-03T20:23:58.742 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:23:58.746 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:23:58.747 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:23:58.847 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:23:58.897 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:23:58.900 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:23:58.901 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:23:59.002 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:23:59.048 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:23:59.055 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:23:59.055 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:23:59.156 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:23:59.204 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:23:59.210 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:23:59.211 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:23:59.312 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:23:59.368 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:23:59.374 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:23:59.375 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:23:59.476 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:23:59.499 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:23:59.503 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:23:59.504 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:23:59.605 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:23:59.637 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:23:59.642 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:23:59.643 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:23:59.744 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:23:59.782 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:23:59.785 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:23:59.785 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:23:59.886 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:23:59.920 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:23:59.921 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:23:59.922 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:24:00.023 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:24:00.053 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:24:00.054 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:24:00.055 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:24:00.156 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:24:00.187 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:24:00.193 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:24:00.194 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:24:00.295 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:24:00.307 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:24:00 smithi089 bash[78452]: cluster 1685823839.0270784 mgr.y (mgr.34104) 1442 : cluster 0 pgmap v1462: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 6 active+undersized+degraded+remapped+backfill_wait, 80 active+clean; 5.9 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 676 op/s; 57002/287736 objects degraded (19.811%); 1.2 MiB/s, 18 objects/s recovering 2023-06-03T20:24:00.308 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:24:00 smithi089 bash[75068]: cluster 1685823839.0270784 mgr.y (mgr.34104) 1442 : cluster 0 pgmap v1462: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 6 active+undersized+degraded+remapped+backfill_wait, 80 active+clean; 5.9 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 676 op/s; 57002/287736 objects degraded (19.811%); 1.2 MiB/s, 18 objects/s recovering 2023-06-03T20:24:00.329 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:24:00.336 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:24:00.337 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:24:00.438 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:24:00.473 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:24:00.478 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:24:00.480 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:24:00.581 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:24:00.584 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:24:00 smithi164 bash[59462]: cluster 1685823839.0270784 mgr.y (mgr.34104) 1442 : cluster 0 pgmap v1462: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 6 active+undersized+degraded+remapped+backfill_wait, 80 active+clean; 5.9 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 676 op/s; 57002/287736 objects degraded (19.811%); 1.2 MiB/s, 18 objects/s recovering 2023-06-03T20:24:00.624 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:24:00.655 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:24:00.656 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:24:00.758 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:24:00.799 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:24:00.800 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:24:00.801 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:24:00.902 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:24:00.940 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:24:00.947 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:24:00.948 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:24:01.049 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:24:01.085 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:24:01.088 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:24:01.089 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:24:01.190 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:24:01.199 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:24:00 smithi089 bash[75068]: debug 2023-06-03T20:24:00.902+0000 7f0fee4d2700 -1 mon.a@0(leader).osd e253 definitely_dead 0 2023-06-03T20:24:01.225 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:24:01.230 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:24:01.231 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:24:01.257 INFO:teuthology.orchestra.run.smithi089.stderr:marked in osd.6. 2023-06-03T20:24:01.332 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:24:01.334 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:24:01 smithi164 bash[59462]: audit 1685823840.9069965 mon.c (mon.2) 608 : audit 0 from='client.? 172.21.15.89:0/698717114' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["6"]}]: dispatch 2023-06-03T20:24:01.335 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:24:01 smithi164 bash[59462]: audit 1685823840.9082513 mon.a (mon.0) 1429 : audit 0 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["6"]}]: dispatch 2023-06-03T20:24:01.362 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:24:01.364 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:24:01.365 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:24:01.466 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:24:01.507 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:24:01.510 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:24:01.511 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:24:01.529 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:24:01 smithi089 bash[78452]: audit 1685823840.9069965 mon.c (mon.2) 608 : audit 0 from='client.? 172.21.15.89:0/698717114' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["6"]}]: dispatch 2023-06-03T20:24:01.529 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:24:01 smithi089 bash[78452]: audit 1685823840.9082513 mon.a (mon.0) 1429 : audit 0 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["6"]}]: dispatch 2023-06-03T20:24:01.530 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:24:01 smithi089 bash[75068]: audit 1685823840.9069965 mon.c (mon.2) 608 : audit 0 from='client.? 172.21.15.89:0/698717114' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["6"]}]: dispatch 2023-06-03T20:24:01.530 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:24:01 smithi089 bash[75068]: audit 1685823840.9082513 mon.a (mon.0) 1429 : audit 0 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["6"]}]: dispatch 2023-06-03T20:24:01.612 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:24:01.646 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:24:01.648 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:24:01.649 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:24:01.750 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:24:01.787 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:24:01.790 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:24:01.791 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:24:01.892 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:24:01.921 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:24:01.923 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:24:01.924 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:24:02.025 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:24:02.056 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:24:02.059 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:24:02.060 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:24:02.161 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:24:02.200 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:24:02.202 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:24:02.203 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:24:02.304 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:24:02.337 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:24:02.343 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:24:02.344 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:24:02.445 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:24:02.455 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:24:02 smithi164 bash[59462]: cluster 1685823841.0277922 mgr.y (mgr.34104) 1443 : cluster 0 pgmap v1463: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 6 active+undersized+degraded+remapped+backfill_wait, 80 active+clean; 5.8 GiB data, 18 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 566 op/s; 56653/286209 objects degraded (19.794%); 837 KiB/s, 13 objects/s recovering 2023-06-03T20:24:02.455 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:24:02 smithi164 bash[59462]: cluster 1685823841.2341554 osd.5 (osd.5) 197 : cluster 3 failed to encode map e254 with expected crc 2023-06-03T20:24:02.455 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:24:02 smithi164 bash[59462]: cluster 1685823841.241148 osd.6 (osd.6) 194 : cluster 3 failed to encode map e254 with expected crc 2023-06-03T20:24:02.456 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:24:02 smithi164 bash[59462]: audit 1685823841.241162 mon.a (mon.0) 1430 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["6"]}]': finished 2023-06-03T20:24:02.456 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:24:02 smithi164 bash[59462]: cluster 1685823841.2412517 mon.a (mon.0) 1431 : cluster 0 osdmap e254: 8 total, 8 up, 8 in 2023-06-03T20:24:02.456 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:24:02 smithi164 bash[59462]: cluster 1685823841.2455542 osd.7 (osd.7) 188 : cluster 3 failed to encode map e254 with expected crc 2023-06-03T20:24:02.456 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:24:02 smithi164 bash[59462]: cluster 1685823841.247596 osd.7 (osd.7) 189 : cluster 3 failed to encode map e254 with expected crc 2023-06-03T20:24:02.492 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:24:02.494 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:24:02.495 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:24:02.529 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:24:02 smithi089 bash[78452]: cluster 1685823841.0277922 mgr.y (mgr.34104) 1443 : cluster 0 pgmap v1463: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 6 active+undersized+degraded+remapped+backfill_wait, 80 active+clean; 5.8 GiB data, 18 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 566 op/s; 56653/286209 objects degraded (19.794%); 837 KiB/s, 13 objects/s recovering 2023-06-03T20:24:02.530 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:24:02 smithi089 bash[78452]: cluster 1685823841.2341554 osd.5 (osd.5) 197 : cluster 3 failed to encode map e254 with expected crc 2023-06-03T20:24:02.530 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:24:02 smithi089 bash[78452]: cluster 1685823841.241148 osd.6 (osd.6) 194 : cluster 3 failed to encode map e254 with expected crc 2023-06-03T20:24:02.530 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:24:02 smithi089 bash[78452]: audit 1685823841.241162 mon.a (mon.0) 1430 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["6"]}]': finished 2023-06-03T20:24:02.530 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:24:02 smithi089 bash[78452]: cluster 1685823841.2412517 mon.a (mon.0) 1431 : cluster 0 osdmap e254: 8 total, 8 up, 8 in 2023-06-03T20:24:02.531 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:24:02 smithi089 bash[78452]: cluster 1685823841.2455542 osd.7 (osd.7) 188 : cluster 3 failed to encode map e254 with expected crc 2023-06-03T20:24:02.531 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:24:02 smithi089 bash[78452]: cluster 1685823841.247596 osd.7 (osd.7) 189 : cluster 3 failed to encode map e254 with expected crc 2023-06-03T20:24:02.531 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:24:02 smithi089 bash[75068]: cluster 1685823841.0277922 mgr.y (mgr.34104) 1443 : cluster 0 pgmap v1463: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 6 active+undersized+degraded+remapped+backfill_wait, 80 active+clean; 5.8 GiB data, 18 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 566 op/s; 56653/286209 objects degraded (19.794%); 837 KiB/s, 13 objects/s recovering 2023-06-03T20:24:02.531 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:24:02 smithi089 bash[75068]: cluster 1685823841.2341554 osd.5 (osd.5) 197 : cluster 3 failed to encode map e254 with expected crc 2023-06-03T20:24:02.532 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:24:02 smithi089 bash[75068]: cluster 1685823841.241148 osd.6 (osd.6) 194 : cluster 3 failed to encode map e254 with expected crc 2023-06-03T20:24:02.532 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:24:02 smithi089 bash[75068]: audit 1685823841.241162 mon.a (mon.0) 1430 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["6"]}]': finished 2023-06-03T20:24:02.532 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:24:02 smithi089 bash[75068]: cluster 1685823841.2412517 mon.a (mon.0) 1431 : cluster 0 osdmap e254: 8 total, 8 up, 8 in 2023-06-03T20:24:02.532 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:24:02 smithi089 bash[75068]: cluster 1685823841.2455542 osd.7 (osd.7) 188 : cluster 3 failed to encode map e254 with expected crc 2023-06-03T20:24:02.532 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:24:02 smithi089 bash[75068]: cluster 1685823841.247596 osd.7 (osd.7) 189 : cluster 3 failed to encode map e254 with expected crc 2023-06-03T20:24:02.596 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:24:02.627 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:24:02.631 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:24:02.631 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:24:02.732 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:24:02.758 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:24:02.760 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:24:02.762 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:24:02.863 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:24:02.900 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:24:02.903 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:24:02.904 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:24:03.005 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:24:03.036 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:24:03.038 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:24:03.039 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:24:03.140 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:24:03.169 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:24:03.172 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:24:03.172 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:24:03.274 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:24:03.299 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:24:03.301 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:24:03.302 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:24:03.403 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:24:03.427 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:24:03.428 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:24:03.429 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:24:03.436 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:24:03 smithi089 bash[75068]: cluster 1685823842.2417781 osd.6 (osd.6) 195 : cluster 3 failed to encode map e255 with expected crc 2023-06-03T20:24:03.437 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:24:03 smithi089 bash[75068]: cluster 1685823842.2442322 osd.6 (osd.6) 196 : cluster 3 failed to encode map e255 with expected crc 2023-06-03T20:24:03.437 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:24:03 smithi089 bash[75068]: cluster 1685823842.2447038 osd.7 (osd.7) 190 : cluster 3 failed to encode map e255 with expected crc 2023-06-03T20:24:03.437 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:24:03 smithi089 bash[75068]: cluster 1685823842.2462742 mon.a (mon.0) 1432 : cluster 0 osdmap e255: 8 total, 8 up, 8 in 2023-06-03T20:24:03.437 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:24:03 smithi089 bash[75068]: cluster 1685823842.2474825 osd.6 (osd.6) 197 : cluster 3 failed to encode map e255 with expected crc 2023-06-03T20:24:03.437 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:24:03 smithi089 bash[75068]: cluster 1685823842.2496243 osd.5 (osd.5) 198 : cluster 3 failed to encode map e255 with expected crc 2023-06-03T20:24:03.437 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:24:03 smithi089 bash[75068]: cluster 1685823842.2527082 osd.7 (osd.7) 191 : cluster 3 failed to encode map e255 with expected crc 2023-06-03T20:24:03.438 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:24:03 smithi089 bash[75068]: cluster 1685823842.254947 osd.7 (osd.7) 192 : cluster 3 failed to encode map e255 with expected crc 2023-06-03T20:24:03.438 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:24:03 smithi089 bash[75068]: cluster 1685823842.2571254 osd.7 (osd.7) 193 : cluster 3 failed to encode map e255 with expected crc 2023-06-03T20:24:03.438 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:24:03 smithi089 bash[75068]: cluster 1685823842.259195 osd.7 (osd.7) 194 : cluster 3 failed to encode map e255 with expected crc 2023-06-03T20:24:03.438 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:24:03 smithi089 bash[75068]: cluster 1685823842.263778 osd.7 (osd.7) 195 : cluster 0 13.7 starting backfill to osd.4 from (245'300,253'8732] MIN to 253'8733 2023-06-03T20:24:03.438 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:24:03 smithi089 bash[75068]: cluster 1685823842.9309468 mon.a (mon.0) 1433 : cluster 3 Health check update: Degraded data redundancy: 56653/286209 objects degraded (19.794%), 9 pgs degraded (PG_DEGRADED) 2023-06-03T20:24:03.438 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:24:03 smithi089 bash[78452]: cluster 1685823842.2417781 osd.6 (osd.6) 195 : cluster 3 failed to encode map e255 with expected crc 2023-06-03T20:24:03.439 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:24:03 smithi089 bash[78452]: cluster 1685823842.2442322 osd.6 (osd.6) 196 : cluster 3 failed to encode map e255 with expected crc 2023-06-03T20:24:03.439 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:24:03 smithi089 bash[78452]: cluster 1685823842.2447038 osd.7 (osd.7) 190 : cluster 3 failed to encode map e255 with expected crc 2023-06-03T20:24:03.439 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:24:03 smithi089 bash[78452]: cluster 1685823842.2462742 mon.a (mon.0) 1432 : cluster 0 osdmap e255: 8 total, 8 up, 8 in 2023-06-03T20:24:03.439 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:24:03 smithi089 bash[78452]: cluster 1685823842.2474825 osd.6 (osd.6) 197 : cluster 3 failed to encode map e255 with expected crc 2023-06-03T20:24:03.439 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:24:03 smithi089 bash[78452]: cluster 1685823842.2496243 osd.5 (osd.5) 198 : cluster 3 failed to encode map e255 with expected crc 2023-06-03T20:24:03.439 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:24:03 smithi089 bash[78452]: cluster 1685823842.2527082 osd.7 (osd.7) 191 : cluster 3 failed to encode map e255 with expected crc 2023-06-03T20:24:03.439 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:24:03 smithi089 bash[78452]: cluster 1685823842.254947 osd.7 (osd.7) 192 : cluster 3 failed to encode map e255 with expected crc 2023-06-03T20:24:03.439 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:24:03 smithi089 bash[78452]: cluster 1685823842.2571254 osd.7 (osd.7) 193 : cluster 3 failed to encode map e255 with expected crc 2023-06-03T20:24:03.440 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:24:03 smithi089 bash[78452]: cluster 1685823842.259195 osd.7 (osd.7) 194 : cluster 3 failed to encode map e255 with expected crc 2023-06-03T20:24:03.440 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:24:03 smithi089 bash[78452]: cluster 1685823842.263778 osd.7 (osd.7) 195 : cluster 0 13.7 starting backfill to osd.4 from (245'300,253'8732] MIN to 253'8733 2023-06-03T20:24:03.440 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:24:03 smithi089 bash[78452]: cluster 1685823842.9309468 mon.a (mon.0) 1433 : cluster 3 Health check update: Degraded data redundancy: 56653/286209 objects degraded (19.794%), 9 pgs degraded (PG_DEGRADED) 2023-06-03T20:24:03.530 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:24:03.564 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:24:03.566 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:24:03.567 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:24:03.585 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:24:03 smithi164 bash[59462]: cluster 1685823842.2417781 osd.6 (osd.6) 195 : cluster 3 failed to encode map e255 with expected crc 2023-06-03T20:24:03.585 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:24:03 smithi164 bash[59462]: cluster 1685823842.2442322 osd.6 (osd.6) 196 : cluster 3 failed to encode map e255 with expected crc 2023-06-03T20:24:03.585 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:24:03 smithi164 bash[59462]: cluster 1685823842.2447038 osd.7 (osd.7) 190 : cluster 3 failed to encode map e255 with expected crc 2023-06-03T20:24:03.586 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:24:03 smithi164 bash[59462]: cluster 1685823842.2462742 mon.a (mon.0) 1432 : cluster 0 osdmap e255: 8 total, 8 up, 8 in 2023-06-03T20:24:03.586 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:24:03 smithi164 bash[59462]: cluster 1685823842.2474825 osd.6 (osd.6) 197 : cluster 3 failed to encode map e255 with expected crc 2023-06-03T20:24:03.586 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:24:03 smithi164 bash[59462]: cluster 1685823842.2496243 osd.5 (osd.5) 198 : cluster 3 failed to encode map e255 with expected crc 2023-06-03T20:24:03.586 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:24:03 smithi164 bash[59462]: cluster 1685823842.2527082 osd.7 (osd.7) 191 : cluster 3 failed to encode map e255 with expected crc 2023-06-03T20:24:03.586 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:24:03 smithi164 bash[59462]: cluster 1685823842.254947 osd.7 (osd.7) 192 : cluster 3 failed to encode map e255 with expected crc 2023-06-03T20:24:03.586 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:24:03 smithi164 bash[59462]: cluster 1685823842.2571254 osd.7 (osd.7) 193 : cluster 3 failed to encode map e255 with expected crc 2023-06-03T20:24:03.587 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:24:03 smithi164 bash[59462]: cluster 1685823842.259195 osd.7 (osd.7) 194 : cluster 3 failed to encode map e255 with expected crc 2023-06-03T20:24:03.587 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:24:03 smithi164 bash[59462]: cluster 1685823842.263778 osd.7 (osd.7) 195 : cluster 0 13.7 starting backfill to osd.4 from (245'300,253'8732] MIN to 253'8733 2023-06-03T20:24:03.587 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:24:03 smithi164 bash[59462]: cluster 1685823842.9309468 mon.a (mon.0) 1433 : cluster 3 Health check update: Degraded data redundancy: 56653/286209 objects degraded (19.794%), 9 pgs degraded (PG_DEGRADED) 2023-06-03T20:24:03.667 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:24:03.706 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:24:03.709 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:24:03.709 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:24:03.810 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:24:03.846 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:24:03.851 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:24:03.852 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:24:03.953 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:24:03.987 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:24:03.995 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:24:03.996 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:24:04.097 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:24:04.138 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:24:04.143 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:24:04.144 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:24:04.245 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:24:04.261 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:24:04 smithi164 bash[59462]: cluster 1685823842.2853177 osd.7 (osd.7) 196 : cluster 0 13.7 starting backfill to osd.6 from (0'0,0'0] MAX to 253'8733 2023-06-03T20:24:04.263 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:24:04 smithi164 bash[59462]: cluster 1685823843.0288227 mgr.y (mgr.34104) 1444 : cluster 0 pgmap v1466: 89 pgs: 1 peering, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 6 active+undersized+degraded+remapped+backfill_wait, 79 active+clean; 5.7 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 576 op/s; 56200/281490 objects degraded (19.965%); 616 KiB/s, 9 objects/s recovering 2023-06-03T20:24:04.263 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:24:04 smithi164 bash[59462]: cluster 1685823843.253258 osd.7 (osd.7) 197 : cluster 3 failed to encode map e256 with expected crc 2023-06-03T20:24:04.263 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:24:04 smithi164 bash[59462]: cluster 1685823843.2552042 mon.a (mon.0) 1434 : cluster 0 osdmap e256: 8 total, 8 up, 8 in 2023-06-03T20:24:04.263 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:24:04 smithi164 bash[59462]: cluster 1685823843.2587214 osd.2 (osd.2) 17 : cluster 0 13.f starting backfill to osd.6 from (0'0,0'0] MAX to 253'8729 2023-06-03T20:24:04.264 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:24:04 smithi164 bash[59462]: cluster 1685823843.2623973 osd.5 (osd.5) 199 : cluster 3 failed to encode map e256 with expected crc 2023-06-03T20:24:04.280 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:24:04.283 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:24:04.284 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:24:04.385 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:24:04.421 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:24:04.428 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:24:04.429 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:24:04.530 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:24:04.582 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:24:04.586 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:24:04.587 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:24:04.688 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:24:04.737 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:24:04.739 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:24:04.740 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:24:04.779 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:24:04 smithi089 bash[78452]: cluster 1685823842.2853177 osd.7 (osd.7) 196 : cluster 0 13.7 starting backfill to osd.6 from (0'0,0'0] MAX to 253'8733 2023-06-03T20:24:04.779 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:24:04 smithi089 bash[78452]: cluster 1685823843.0288227 mgr.y (mgr.34104) 1444 : cluster 0 pgmap v1466: 89 pgs: 1 peering, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 6 active+undersized+degraded+remapped+backfill_wait, 79 active+clean; 5.7 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 576 op/s; 56200/281490 objects degraded (19.965%); 616 KiB/s, 9 objects/s recovering 2023-06-03T20:24:04.780 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:24:04 smithi089 bash[78452]: cluster 1685823843.253258 osd.7 (osd.7) 197 : cluster 3 failed to encode map e256 with expected crc 2023-06-03T20:24:04.780 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:24:04 smithi089 bash[78452]: cluster 1685823843.2552042 mon.a (mon.0) 1434 : cluster 0 osdmap e256: 8 total, 8 up, 8 in 2023-06-03T20:24:04.780 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:24:04 smithi089 bash[78452]: cluster 1685823843.2587214 osd.2 (osd.2) 17 : cluster 0 13.f starting backfill to osd.6 from (0'0,0'0] MAX to 253'8729 2023-06-03T20:24:04.780 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:24:04 smithi089 bash[78452]: cluster 1685823843.2623973 osd.5 (osd.5) 199 : cluster 3 failed to encode map e256 with expected crc 2023-06-03T20:24:04.780 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:24:04 smithi089 bash[75068]: cluster 1685823842.2853177 osd.7 (osd.7) 196 : cluster 0 13.7 starting backfill to osd.6 from (0'0,0'0] MAX to 253'8733 2023-06-03T20:24:04.781 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:24:04 smithi089 bash[75068]: cluster 1685823843.0288227 mgr.y (mgr.34104) 1444 : cluster 0 pgmap v1466: 89 pgs: 1 peering, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 6 active+undersized+degraded+remapped+backfill_wait, 79 active+clean; 5.7 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 576 op/s; 56200/281490 objects degraded (19.965%); 616 KiB/s, 9 objects/s recovering 2023-06-03T20:24:04.781 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:24:04 smithi089 bash[75068]: cluster 1685823843.253258 osd.7 (osd.7) 197 : cluster 3 failed to encode map e256 with expected crc 2023-06-03T20:24:04.781 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:24:04 smithi089 bash[75068]: cluster 1685823843.2552042 mon.a (mon.0) 1434 : cluster 0 osdmap e256: 8 total, 8 up, 8 in 2023-06-03T20:24:04.781 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:24:04 smithi089 bash[75068]: cluster 1685823843.2587214 osd.2 (osd.2) 17 : cluster 0 13.f starting backfill to osd.6 from (0'0,0'0] MAX to 253'8729 2023-06-03T20:24:04.781 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:24:04 smithi089 bash[75068]: cluster 1685823843.2623973 osd.5 (osd.5) 199 : cluster 3 failed to encode map e256 with expected crc 2023-06-03T20:24:04.841 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:24:04.869 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:24:04.874 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:24:04.876 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:24:04.977 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:24:05.011 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:24:05.016 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:24:05.017 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:24:05.118 INFO:tasks.thrashosds.thrasher:Added osd 6 2023-06-03T20:24:05.119 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:24:05.164 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:24:05.165 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:24:05.166 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:24:05.267 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:24:05.319 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:24:05.330 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:24:05.331 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:24:05.433 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:24:05.463 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:24:05.468 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:24:05.469 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:24:05.571 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:24:05.614 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:24:05.619 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:24:05.620 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:24:05.721 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:24:05.752 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:24:05.753 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:24:05.754 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:24:05.855 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:24:05.913 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:24:05.918 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:24:05.919 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:24:06.020 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:24:06.061 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:24:06.064 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:24:06.065 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:24:06.166 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:24:06.213 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:24:06.219 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:24:06.220 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:24:06.271 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:24:06 smithi164 bash[59462]: cluster 1685823845.029519 mgr.y (mgr.34104) 1445 : cluster 0 pgmap v1468: 89 pgs: 1 activating+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 1 peering, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 6 active+undersized+degraded+remapped+backfill_wait, 76 active+clean; 5.7 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 443 op/s; 55754/279747 objects degraded (19.930%); 5933/279747 objects misplaced (2.121%); 1.6 MiB/s, 25 objects/s recovering 2023-06-03T20:24:06.320 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:24:06.347 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:24:06.352 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:24:06.353 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:24:06.429 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:24:06 smithi089 bash[75068]: cluster 1685823845.029519 mgr.y (mgr.34104) 1445 : cluster 0 pgmap v1468: 89 pgs: 1 activating+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 1 peering, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 6 active+undersized+degraded+remapped+backfill_wait, 76 active+clean; 5.7 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 443 op/s; 55754/279747 objects degraded (19.930%); 5933/279747 objects misplaced (2.121%); 1.6 MiB/s, 25 objects/s recovering 2023-06-03T20:24:06.429 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 20:24:06 smithi089 bash[71163]: ::ffff:172.21.15.164 - - [03/Jun/2023:20:24:06] "GET /metrics HTTP/1.1" 200 228880 "" "Prometheus/2.33.4" 2023-06-03T20:24:06.431 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:24:06 smithi089 bash[78452]: cluster 1685823845.029519 mgr.y (mgr.34104) 1445 : cluster 0 pgmap v1468: 89 pgs: 1 activating+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 1 peering, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 6 active+undersized+degraded+remapped+backfill_wait, 76 active+clean; 5.7 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 443 op/s; 55754/279747 objects degraded (19.930%); 5933/279747 objects misplaced (2.121%); 1.6 MiB/s, 25 objects/s recovering 2023-06-03T20:24:06.454 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:24:06.481 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:24:06.486 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:24:06.487 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:24:06.588 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:24:06.627 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:24:06.631 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:24:06.632 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:24:06.732 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:24:06.765 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:24:06.769 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:24:06.770 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:24:06.871 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:24:06.909 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:24:06.916 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:24:06.917 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:24:07.018 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:24:07.062 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:24:07.068 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:24:07.069 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:24:07.170 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:24:07.208 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:24:07.212 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:24:07.212 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:24:07.313 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:24:07.341 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:24:07.351 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:24:07.352 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:24:07.453 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:24:07.490 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:24:07.496 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:24:07.498 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:24:07.584 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:24:07 smithi164 bash[59462]: audit 1685823847.137414 mon.c (mon.2) 609 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T20:24:07.599 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:24:07.631 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:24:07.635 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:24:07.636 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:24:07.654 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:24:07 smithi089 bash[78452]: audit 1685823847.137414 mon.c (mon.2) 609 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T20:24:07.655 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:24:07 smithi089 bash[75068]: audit 1685823847.137414 mon.c (mon.2) 609 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T20:24:07.737 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:24:07.775 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:24:07.778 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:24:07.778 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:24:07.879 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:24:07.912 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:24:07.918 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:24:07.919 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:24:08.020 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:24:08.046 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:24:08.050 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:24:08.050 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:24:08.151 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:24:08.207 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:24:08.213 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:24:08.214 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:24:08.315 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:24:08.360 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:24:08.362 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:24:08.363 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:24:08.464 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:24:08.473 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:24:08 smithi164 bash[59462]: cluster 1685823847.030563 mgr.y (mgr.34104) 1446 : cluster 0 pgmap v1469: 89 pgs: 1 activating+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 6 active+undersized+degraded+remapped+backfill_wait, 75 active+clean; 5.7 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 451 op/s; 62989/278085 objects degraded (22.651%); 11706/278085 objects misplaced (4.210%); 1.6 MiB/s, 26 objects/s recovering 2023-06-03T20:24:08.473 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:24:08 smithi164 bash[59462]: cluster 1685823847.9321024 mon.a (mon.0) 1435 : cluster 3 Health check update: Degraded data redundancy: 62989/278085 objects degraded (22.651%), 13 pgs degraded (PG_DEGRADED) 2023-06-03T20:24:08.509 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:24:08.514 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:24:08.515 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:24:08.616 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:24:08.664 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:24:08.666 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:24:08.667 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:24:08.768 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:24:08.779 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:24:08 smithi089 bash[75068]: cluster 1685823847.030563 mgr.y (mgr.34104) 1446 : cluster 0 pgmap v1469: 89 pgs: 1 activating+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 6 active+undersized+degraded+remapped+backfill_wait, 75 active+clean; 5.7 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 451 op/s; 62989/278085 objects degraded (22.651%); 11706/278085 objects misplaced (4.210%); 1.6 MiB/s, 26 objects/s recovering 2023-06-03T20:24:08.779 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:24:08 smithi089 bash[75068]: cluster 1685823847.9321024 mon.a (mon.0) 1435 : cluster 3 Health check update: Degraded data redundancy: 62989/278085 objects degraded (22.651%), 13 pgs degraded (PG_DEGRADED) 2023-06-03T20:24:08.779 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:24:08 smithi089 bash[78452]: cluster 1685823847.030563 mgr.y (mgr.34104) 1446 : cluster 0 pgmap v1469: 89 pgs: 1 activating+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 6 active+undersized+degraded+remapped+backfill_wait, 75 active+clean; 5.7 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 451 op/s; 62989/278085 objects degraded (22.651%); 11706/278085 objects misplaced (4.210%); 1.6 MiB/s, 26 objects/s recovering 2023-06-03T20:24:08.779 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:24:08 smithi089 bash[78452]: cluster 1685823847.9321024 mon.a (mon.0) 1435 : cluster 3 Health check update: Degraded data redundancy: 62989/278085 objects degraded (22.651%), 13 pgs degraded (PG_DEGRADED) 2023-06-03T20:24:08.812 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:24:08.815 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:24:08.816 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:24:08.917 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:24:08.954 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:24:08.957 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:24:08.957 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:24:09.058 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:24:09.093 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:24:09.098 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:24:09.099 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:24:09.200 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:24:09.243 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:24:09.249 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:24:09.250 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:24:09.334 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:24:09 smithi164 bash[59462]: cluster 1685823849.0312667 mgr.y (mgr.34104) 1447 : cluster 0 pgmap v1470: 89 pgs: 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 6 active+undersized+degraded+remapped+backfill_wait, 75 active+clean; 5.6 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 505 op/s; 62324/274392 objects degraded (22.713%); 11528/274392 objects misplaced (4.201%); 1.9 MiB/s, 30 objects/s recovering 2023-06-03T20:24:09.350 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:24:09.385 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:24:09.387 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:24:09.388 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:24:09.490 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:24:09.524 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:24:09.529 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:24:09 smithi089 bash[75068]: cluster 1685823849.0312667 mgr.y (mgr.34104) 1447 : cluster 0 pgmap v1470: 89 pgs: 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 6 active+undersized+degraded+remapped+backfill_wait, 75 active+clean; 5.6 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 505 op/s; 62324/274392 objects degraded (22.713%); 11528/274392 objects misplaced (4.201%); 1.9 MiB/s, 30 objects/s recovering 2023-06-03T20:24:09.529 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:24:09 smithi089 bash[78452]: cluster 1685823849.0312667 mgr.y (mgr.34104) 1447 : cluster 0 pgmap v1470: 89 pgs: 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 6 active+undersized+degraded+remapped+backfill_wait, 75 active+clean; 5.6 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 505 op/s; 62324/274392 objects degraded (22.713%); 11528/274392 objects misplaced (4.201%); 1.9 MiB/s, 30 objects/s recovering 2023-06-03T20:24:09.532 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:24:09.532 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:24:09.633 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:24:09.676 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:24:09.678 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:24:09.679 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:24:09.780 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:24:09.829 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:24:09.834 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:24:09.835 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:24:09.936 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:24:09.973 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:24:09.975 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:24:09.975 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:24:10.076 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:24:10.107 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:24:10.109 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:24:10.110 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:24:10.119 INFO:tasks.thrashosds.thrasher:in_osds: [0, 2, 3, 4, 5, 7, 1, 6] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 5, 6, 7, 4] 2023-06-03T20:24:10.119 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-06-03T20:24:10.120 INFO:tasks.thrashosds.thrasher:Growing pool unique_pool_8 2023-06-03T20:24:10.120 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph pg dump --format=json 2023-06-03T20:24:10.211 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:24:10.239 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:24:10.241 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:24:10.241 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:24:10.342 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:24:10.376 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:24:10.385 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:24:10.386 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:24:10.487 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:24:10.526 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:24:10.528 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:24:10.528 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:24:10.629 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:24:10.691 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:24:10.694 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:24:10.695 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:24:10.796 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:24:10.843 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:24:10.845 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:24:10.846 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:24:10.899 INFO:teuthology.orchestra.run.smithi089.stdout:ERROR: (22) Invalid argument 2023-06-03T20:24:10.899 INFO:teuthology.orchestra.run.smithi089.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-06-03T20:24:10.947 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:24:10.979 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:24:10.981 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:24:10.981 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:24:11.082 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:24:11.119 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:24:11.121 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:24:11.122 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:24:11.223 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:24:11.257 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:24:11.265 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:24:11.266 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:24:11.367 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:24:11.412 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:24:11.414 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:24:11.415 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:24:11.516 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:24:11.549 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:24:11.552 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:24:11.553 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:24:11.654 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:24:11.688 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:24:11.691 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:24:11.691 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:24:11.792 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:24:11.829 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:24:11.834 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:24:11.835 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:24:11.936 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:24:11.981 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:24:11.987 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:24:11.987 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:24:12.088 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:24:12.114 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:24:12.117 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:24:12.117 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:24:12.172 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:24:12 smithi089 bash[75068]: cluster 1685823851.0320985 mgr.y (mgr.34104) 1448 : cluster 0 pgmap v1471: 89 pgs: 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 6 active+undersized+degraded+remapped+backfill_wait, 75 active+clean; 5.5 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 420 op/s; 61946/272751 objects degraded (22.712%); 11528/272751 objects misplaced (4.227%); 1.6 MiB/s, 26 objects/s recovering 2023-06-03T20:24:12.172 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:24:12 smithi089 bash[78452]: cluster 1685823851.0320985 mgr.y (mgr.34104) 1448 : cluster 0 pgmap v1471: 89 pgs: 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 6 active+undersized+degraded+remapped+backfill_wait, 75 active+clean; 5.5 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 420 op/s; 61946/272751 objects degraded (22.712%); 11528/272751 objects misplaced (4.227%); 1.6 MiB/s, 26 objects/s recovering 2023-06-03T20:24:12.218 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:24:12.249 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:24:12.255 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:24:12.256 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:24:12.334 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:24:12 smithi164 bash[59462]: cluster 1685823851.0320985 mgr.y (mgr.34104) 1448 : cluster 0 pgmap v1471: 89 pgs: 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 6 active+undersized+degraded+remapped+backfill_wait, 75 active+clean; 5.5 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 420 op/s; 61946/272751 objects degraded (22.712%); 11528/272751 objects misplaced (4.227%); 1.6 MiB/s, 26 objects/s recovering 2023-06-03T20:24:12.357 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:24:12.408 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:24:12.409 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:24:12.409 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:24:12.510 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:24:12.546 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:24:12.550 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:24:12.551 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:24:12.652 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:24:12.676 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:24:12.678 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:24:12.678 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:24:12.779 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:24:12.812 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:24:12.815 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:24:12.815 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:24:12.916 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:24:12.944 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:24:12.947 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:24:12.947 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:24:13.048 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:24:13.084 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:24:13.090 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:24:13.091 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:24:13.149 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph daemon osd.3 dump_ops_in_flight 2023-06-03T20:24:13.192 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:24:13.229 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:24:13.234 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:24:13.235 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:24:13.254 INFO:teuthology.orchestra.run.smithi089.stdout:osd.0: {} 2023-06-03T20:24:13.279 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:24:13 smithi089 bash[78452]: cluster 1685823852.9332583 mon.a (mon.0) 1436 : cluster 3 Health check update: Degraded data redundancy: 61946/272751 objects degraded (22.712%), 13 pgs degraded (PG_DEGRADED) 2023-06-03T20:24:13.279 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:24:13 smithi089 bash[75068]: cluster 1685823852.9332583 mon.a (mon.0) 1436 : cluster 3 Health check update: Degraded data redundancy: 61946/272751 objects degraded (22.712%), 13 pgs degraded (PG_DEGRADED) 2023-06-03T20:24:13.312 INFO:teuthology.orchestra.run.smithi089.stdout:osd.1: {} 2023-06-03T20:24:13.334 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:24:13 smithi164 bash[59462]: cluster 1685823852.9332583 mon.a (mon.0) 1436 : cluster 3 Health check update: Degraded data redundancy: 61946/272751 objects degraded (22.712%), 13 pgs degraded (PG_DEGRADED) 2023-06-03T20:24:13.336 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:24:13.384 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:24:13.386 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:24:13.387 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:24:13.389 INFO:teuthology.orchestra.run.smithi089.stdout:osd.2: {} 2023-06-03T20:24:13.439 INFO:teuthology.orchestra.run.smithi089.stdout:osd.3: {} 2023-06-03T20:24:13.488 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:24:13.491 INFO:teuthology.orchestra.run.smithi089.stdout:osd.4: {} 2023-06-03T20:24:13.523 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:24:13.527 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:24:13.528 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:24:13.530 INFO:teuthology.orchestra.run.smithi089.stdout:osd.5: {} 2023-06-03T20:24:13.572 INFO:teuthology.orchestra.run.smithi089.stdout:osd.6: {} 2023-06-03T20:24:13.607 INFO:teuthology.orchestra.run.smithi089.stdout:osd.7: {} 2023-06-03T20:24:13.614 INFO:teuthology.orchestra.run.smithi089.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-06-03T20:24:13.614 INFO:teuthology.orchestra.run.smithi089.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-06-03T20:24:13.615 INFO:teuthology.orchestra.run.smithi089.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-06-03T20:24:13.615 INFO:teuthology.orchestra.run.smithi089.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-06-03T20:24:13.615 INFO:teuthology.orchestra.run.smithi089.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-06-03T20:24:13.615 INFO:teuthology.orchestra.run.smithi089.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-06-03T20:24:13.615 INFO:teuthology.orchestra.run.smithi089.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-06-03T20:24:13.615 INFO:teuthology.orchestra.run.smithi089.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-06-03T20:24:13.629 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:24:13.662 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:24:13.668 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:24:13.668 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:24:13.769 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:24:13.821 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:24:13.828 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:24:13.829 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:24:13.930 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:24:13.965 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:24:13.968 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:24:13.969 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:24:14.070 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:24:14.106 INFO:teuthology.orchestra.run.smithi089.stderr:nodeep-scrub is set 2023-06-03T20:24:14.107 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:24:14.110 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:24:14.110 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:24:14.211 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:24:14.220 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:24:14 smithi089 bash[75068]: cluster 1685823853.0327384 mgr.y (mgr.34104) 1449 : cluster 0 pgmap v1472: 89 pgs: 2 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 6 active+undersized+degraded+remapped+backfill_wait, 75 active+clean; 5.4 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 469 op/s; 61431/267402 objects degraded (22.973%); 11323/267402 objects misplaced (4.234%); 1.4 MiB/s, 23 objects/s recovering 2023-06-03T20:24:14.221 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:24:14 smithi089 bash[75068]: audit 1685823853.2040365 mon.c (mon.2) 610 : audit 0 from='client.? 172.21.15.89:0/127481029' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-03T20:24:14.221 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:24:14 smithi089 bash[75068]: audit 1685823853.6323178 mon.c (mon.2) 611 : audit 1 from='client.? 172.21.15.89:0/1793499964' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-06-03T20:24:14.221 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:24:14 smithi089 bash[75068]: audit 1685823853.6343248 mon.a (mon.0) 1437 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-06-03T20:24:14.222 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:24:14 smithi089 bash[78452]: cluster 1685823853.0327384 mgr.y (mgr.34104) 1449 : cluster 0 pgmap v1472: 89 pgs: 2 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 6 active+undersized+degraded+remapped+backfill_wait, 75 active+clean; 5.4 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 469 op/s; 61431/267402 objects degraded (22.973%); 11323/267402 objects misplaced (4.234%); 1.4 MiB/s, 23 objects/s recovering 2023-06-03T20:24:14.222 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:24:14 smithi089 bash[78452]: audit 1685823853.2040365 mon.c (mon.2) 610 : audit 0 from='client.? 172.21.15.89:0/127481029' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-03T20:24:14.223 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:24:14 smithi089 bash[78452]: audit 1685823853.6323178 mon.c (mon.2) 611 : audit 1 from='client.? 172.21.15.89:0/1793499964' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-06-03T20:24:14.223 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:24:14 smithi089 bash[78452]: audit 1685823853.6343248 mon.a (mon.0) 1437 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-06-03T20:24:14.240 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:24:14.243 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:24:14.243 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:24:14.335 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:24:14 smithi164 bash[59462]: cluster 1685823853.0327384 mgr.y (mgr.34104) 1449 : cluster 0 pgmap v1472: 89 pgs: 2 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 6 active+undersized+degraded+remapped+backfill_wait, 75 active+clean; 5.4 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 469 op/s; 61431/267402 objects degraded (22.973%); 11323/267402 objects misplaced (4.234%); 1.4 MiB/s, 23 objects/s recovering 2023-06-03T20:24:14.336 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:24:14 smithi164 bash[59462]: audit 1685823853.2040365 mon.c (mon.2) 610 : audit 0 from='client.? 172.21.15.89:0/127481029' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-03T20:24:14.336 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:24:14 smithi164 bash[59462]: audit 1685823853.6323178 mon.c (mon.2) 611 : audit 1 from='client.? 172.21.15.89:0/1793499964' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-06-03T20:24:14.336 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:24:14 smithi164 bash[59462]: audit 1685823853.6343248 mon.a (mon.0) 1437 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-06-03T20:24:14.344 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:24:14.399 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:24:14.402 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:24:14.403 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:24:14.504 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:24:14.556 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:24:14.559 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:24:14.560 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:24:14.661 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:24:14.689 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:24:14.691 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:24:14.692 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:24:14.793 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:24:14.825 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:24:14.827 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:24:14.828 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:24:14.929 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:24:14.960 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:24:14.962 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:24:14.962 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:24:15.063 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:24:15.095 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:24:15.099 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:24:15.100 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:24:15.201 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:24:15.242 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:24:15.245 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:24:15.246 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:24:15.279 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:24:15 smithi089 bash[75068]: cluster 1685823854.0874321 mon.a (mon.0) 1438 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-06-03T20:24:15.279 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:24:15 smithi089 bash[75068]: cluster 1685823854.0935197 osd.5 (osd.5) 200 : cluster 3 failed to encode map e257 with expected crc 2023-06-03T20:24:15.280 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:24:15 smithi089 bash[75068]: audit 1685823854.0946202 mon.a (mon.0) 1439 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-06-03T20:24:15.280 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:24:15 smithi089 bash[75068]: cluster 1685823854.0947094 mon.a (mon.0) 1440 : cluster 0 osdmap e257: 8 total, 8 up, 8 in 2023-06-03T20:24:15.280 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:24:15 smithi089 bash[75068]: audit 1685823854.2683303 mon.c (mon.2) 612 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-03T20:24:15.280 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:24:15 smithi089 bash[78452]: cluster 1685823854.0874321 mon.a (mon.0) 1438 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-06-03T20:24:15.280 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:24:15 smithi089 bash[78452]: cluster 1685823854.0935197 osd.5 (osd.5) 200 : cluster 3 failed to encode map e257 with expected crc 2023-06-03T20:24:15.281 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:24:15 smithi089 bash[78452]: audit 1685823854.0946202 mon.a (mon.0) 1439 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-06-03T20:24:15.281 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:24:15 smithi089 bash[78452]: cluster 1685823854.0947094 mon.a (mon.0) 1440 : cluster 0 osdmap e257: 8 total, 8 up, 8 in 2023-06-03T20:24:15.281 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:24:15 smithi089 bash[78452]: audit 1685823854.2683303 mon.c (mon.2) 612 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-03T20:24:15.334 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:24:15 smithi164 bash[59462]: cluster 1685823854.0874321 mon.a (mon.0) 1438 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-06-03T20:24:15.335 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:24:15 smithi164 bash[59462]: cluster 1685823854.0935197 osd.5 (osd.5) 200 : cluster 3 failed to encode map e257 with expected crc 2023-06-03T20:24:15.335 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:24:15 smithi164 bash[59462]: audit 1685823854.0946202 mon.a (mon.0) 1439 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-06-03T20:24:15.335 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:24:15 smithi164 bash[59462]: cluster 1685823854.0947094 mon.a (mon.0) 1440 : cluster 0 osdmap e257: 8 total, 8 up, 8 in 2023-06-03T20:24:15.336 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:24:15 smithi164 bash[59462]: audit 1685823854.2683303 mon.c (mon.2) 612 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-03T20:24:15.348 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:24:15.384 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:24:15.387 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:24:15.388 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:24:15.489 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:24:15.540 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:24:15.541 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:24:15.541 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:24:15.642 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:24:15.670 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:24:15.672 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:24:15.673 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:24:15.774 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:24:15.805 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:24:15.808 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:24:15.809 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:24:15.910 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:24:15.953 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:24:15.959 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:24:15.960 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:24:16.060 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:24:16.100 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:24:16.104 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:24:16.105 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:24:16.206 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:24:16.215 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:24:16 smithi089 bash[78452]: cluster 1685823855.0333943 mgr.y (mgr.34104) 1450 : cluster 0 pgmap v1474: 89 pgs: 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 6 active+undersized+degraded+remapped+backfill_wait, 75 active+clean; 5.4 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 510 op/s; 60893/264435 objects degraded (23.028%); 11172/264435 objects misplaced (4.225%); 998 KiB/s, 15 objects/s recovering 2023-06-03T20:24:16.217 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:24:16 smithi089 bash[75068]: cluster 1685823855.0333943 mgr.y (mgr.34104) 1450 : cluster 0 pgmap v1474: 89 pgs: 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 6 active+undersized+degraded+remapped+backfill_wait, 75 active+clean; 5.4 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 510 op/s; 60893/264435 objects degraded (23.028%); 11172/264435 objects misplaced (4.225%); 998 KiB/s, 15 objects/s recovering 2023-06-03T20:24:16.239 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:24:16.245 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:24:16.246 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:24:16.334 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:24:16 smithi164 bash[59462]: cluster 1685823855.0333943 mgr.y (mgr.34104) 1450 : cluster 0 pgmap v1474: 89 pgs: 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 6 active+undersized+degraded+remapped+backfill_wait, 75 active+clean; 5.4 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 510 op/s; 60893/264435 objects degraded (23.028%); 11172/264435 objects misplaced (4.225%); 998 KiB/s, 15 objects/s recovering 2023-06-03T20:24:16.347 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:24:16.388 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:24:16.393 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:24:16.394 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:24:16.495 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:24:16.507 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 20:24:16 smithi089 bash[71163]: ::ffff:172.21.15.164 - - [03/Jun/2023:20:24:16] "GET /metrics HTTP/1.1" 200 228936 "" "Prometheus/2.33.4" 2023-06-03T20:24:16.546 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:24:16.550 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:24:16.551 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:24:16.652 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:24:16.680 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:24:16.683 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:24:16.684 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:24:16.785 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:24:16.816 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:24:16.823 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:24:16.824 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:24:16.925 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:24:16.968 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:24:16.974 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:24:16.975 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:24:17.076 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:24:17.126 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:24:17.129 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:24:17.130 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:24:17.231 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:24:17.271 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:24:17.275 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:24:17.276 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:24:17.377 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:24:17.412 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:24:17.417 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:24:17.418 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:24:17.519 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:24:17.520 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-06-03T20:24:17.550 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:24:17.552 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:24:17.553 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:24:17.654 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:24:17.701 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:24:17.708 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:24:17.709 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:24:17.810 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:24:17.860 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:24:17.862 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:24:17.863 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:24:17.964 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:24:18.009 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:24:18.009 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/mon.a/config 2023-06-03T20:24:18.014 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:24:18.014 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:24:18.115 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:24:18.157 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:24:18.167 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:24:18.168 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:24:18.269 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:24:18.279 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:24:18 smithi089 bash[75068]: cluster 1685823857.0341415 mgr.y (mgr.34104) 1451 : cluster 0 pgmap v1475: 89 pgs: 2 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 6 active+undersized+degraded+remapped+backfill_wait, 75 active+clean; 5.3 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 538 op/s; 60893/261927 objects degraded (23.248%); 10998/261927 objects misplaced (4.199%); 998 KiB/s, 15 objects/s recovering 2023-06-03T20:24:18.279 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:24:18 smithi089 bash[75068]: cluster 1685823857.9342606 mon.a (mon.0) 1441 : cluster 3 Health check update: Degraded data redundancy: 60893/261927 objects degraded (23.248%), 13 pgs degraded (PG_DEGRADED) 2023-06-03T20:24:18.279 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:24:18 smithi089 bash[78452]: cluster 1685823857.0341415 mgr.y (mgr.34104) 1451 : cluster 0 pgmap v1475: 89 pgs: 2 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 6 active+undersized+degraded+remapped+backfill_wait, 75 active+clean; 5.3 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 538 op/s; 60893/261927 objects degraded (23.248%); 10998/261927 objects misplaced (4.199%); 998 KiB/s, 15 objects/s recovering 2023-06-03T20:24:18.280 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:24:18 smithi089 bash[78452]: cluster 1685823857.9342606 mon.a (mon.0) 1441 : cluster 3 Health check update: Degraded data redundancy: 60893/261927 objects degraded (23.248%), 13 pgs degraded (PG_DEGRADED) 2023-06-03T20:24:18.312 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:24:18.313 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:24:18.313 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:24:18.414 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:24:18.423 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:24:18 smithi164 bash[59462]: cluster 1685823857.0341415 mgr.y (mgr.34104) 1451 : cluster 0 pgmap v1475: 89 pgs: 2 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 6 active+undersized+degraded+remapped+backfill_wait, 75 active+clean; 5.3 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 538 op/s; 60893/261927 objects degraded (23.248%); 10998/261927 objects misplaced (4.199%); 998 KiB/s, 15 objects/s recovering 2023-06-03T20:24:18.423 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:24:18 smithi164 bash[59462]: cluster 1685823857.9342606 mon.a (mon.0) 1441 : cluster 3 Health check update: Degraded data redundancy: 60893/261927 objects degraded (23.248%), 13 pgs degraded (PG_DEGRADED) 2023-06-03T20:24:18.453 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:24:18.459 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:24:18.459 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:24:18.509 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph osd unset noscrub 2023-06-03T20:24:18.560 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:24:18.618 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:24:18.623 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:24:18.624 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:24:18.725 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:24:18.760 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:24:18.764 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:24:18.765 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:24:18.867 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:24:18.900 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:24:18.905 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:24:18.906 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:24:19.007 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:24:19.042 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:24:19.044 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:24:19.045 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:24:19.146 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:24:19.176 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:24:19.178 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:24:19.179 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:24:19.280 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:24:19.317 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:24:19.322 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:24:19.323 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:24:19.424 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:24:19.468 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:24:19.475 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:24:19.475 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:24:19.576 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:24:19.604 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:24:19.608 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:24:19.608 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:24:19.709 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:24:19.737 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:24:19.778 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:24:19.779 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:24:19.880 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:24:19.918 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:24:19.922 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:24:19.923 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:24:20.024 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:24:20.053 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/mon.a/config 2023-06-03T20:24:20.071 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:24:20.075 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:24:20.076 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:24:20.177 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:24:20.209 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:24:20.212 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:24:20.213 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:24:20.316 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:24:20.327 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:24:20 smithi164 bash[59462]: cluster 1685823859.0374088 mgr.y (mgr.34104) 1452 : cluster 0 pgmap v1476: 89 pgs: 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 6 active+undersized+degraded+remapped+backfill_wait, 75 active+clean; 5.2 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 547 op/s; 60193/257958 objects degraded (23.334%); 10829/257958 objects misplaced (4.198%); 940 KiB/s, 14 objects/s recovering 2023-06-03T20:24:20.358 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:24:20.365 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:24:20.366 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:24:20.466 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:24:20.513 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:24:20.520 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:24:20.521 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:24:20.529 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:24:20 smithi089 bash[75068]: cluster 1685823859.0374088 mgr.y (mgr.34104) 1452 : cluster 0 pgmap v1476: 89 pgs: 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 6 active+undersized+degraded+remapped+backfill_wait, 75 active+clean; 5.2 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 547 op/s; 60193/257958 objects degraded (23.334%); 10829/257958 objects misplaced (4.198%); 940 KiB/s, 14 objects/s recovering 2023-06-03T20:24:20.529 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:24:20 smithi089 bash[78452]: cluster 1685823859.0374088 mgr.y (mgr.34104) 1452 : cluster 0 pgmap v1476: 89 pgs: 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 6 active+undersized+degraded+remapped+backfill_wait, 75 active+clean; 5.2 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 547 op/s; 60193/257958 objects degraded (23.334%); 10829/257958 objects misplaced (4.198%); 940 KiB/s, 14 objects/s recovering 2023-06-03T20:24:20.622 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:24:20.660 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:24:20.668 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:24:20.669 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:24:20.770 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:24:20.799 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:24:20.802 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:24:20.804 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:24:20.905 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:24:20.972 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:24:20.973 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:24:20.974 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:24:21.075 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:24:21.118 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:24:21.123 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:24:21.125 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:24:21.229 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:24:21.277 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:24:21.285 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:24:21.286 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:24:21.387 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:24:21.431 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:24:21.438 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:24:21.439 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:24:21.539 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:24:21.574 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:24:21.579 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:24:21.580 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:24:21.681 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:24:21.722 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:24:21.727 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:24:21.728 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:24:21.829 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:24:21.869 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:24:21.870 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:24:21.872 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:24:21.974 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:24:22.012 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:24:22.015 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:24:22.017 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:24:22.118 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:24:22.165 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:24:22.168 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:24:22.169 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:24:22.269 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:24:22.307 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:24:22.311 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:24:22.312 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:24:22.413 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:24:22.429 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:24:22 smithi089 bash[78452]: cluster 1685823861.0381923 mgr.y (mgr.34104) 1453 : cluster 0 pgmap v1477: 89 pgs: 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 6 active+undersized+degraded+remapped+backfill_wait, 75 active+clean; 5.2 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 541 op/s; 59874/256497 objects degraded (23.343%); 10829/256497 objects misplaced (4.222%); 940 KiB/s, 14 objects/s recovering 2023-06-03T20:24:22.430 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:24:22 smithi089 bash[78452]: audit 1685823861.572034 mon.a (mon.0) 1442 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:24:22.430 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:24:22 smithi089 bash[78452]: audit 1685823861.5852952 mon.a (mon.0) 1443 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:24:22.430 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:24:22 smithi089 bash[75068]: cluster 1685823861.0381923 mgr.y (mgr.34104) 1453 : cluster 0 pgmap v1477: 89 pgs: 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 6 active+undersized+degraded+remapped+backfill_wait, 75 active+clean; 5.2 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 541 op/s; 59874/256497 objects degraded (23.343%); 10829/256497 objects misplaced (4.222%); 940 KiB/s, 14 objects/s recovering 2023-06-03T20:24:22.430 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:24:22 smithi089 bash[75068]: audit 1685823861.572034 mon.a (mon.0) 1442 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:24:22.431 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:24:22 smithi089 bash[75068]: audit 1685823861.5852952 mon.a (mon.0) 1443 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:24:22.466 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:24:22.470 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:24:22.472 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:24:22.574 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:24:22.584 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:24:22 smithi164 bash[59462]: cluster 1685823861.0381923 mgr.y (mgr.34104) 1453 : cluster 0 pgmap v1477: 89 pgs: 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 6 active+undersized+degraded+remapped+backfill_wait, 75 active+clean; 5.2 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 541 op/s; 59874/256497 objects degraded (23.343%); 10829/256497 objects misplaced (4.222%); 940 KiB/s, 14 objects/s recovering 2023-06-03T20:24:22.585 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:24:22 smithi164 bash[59462]: audit 1685823861.572034 mon.a (mon.0) 1442 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:24:22.585 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:24:22 smithi164 bash[59462]: audit 1685823861.5852952 mon.a (mon.0) 1443 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:24:22.617 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:24:22.620 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:24:22.621 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:24:22.722 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:24:22.759 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:24:22.761 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:24:22.762 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:24:22.864 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:24:22.926 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:24:22.932 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:24:22.933 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:24:23.034 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:24:23.067 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:24:23.069 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:24:23.070 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:24:23.145 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:24:23 smithi164 bash[59462]: audit 1685823862.1455874 mon.c (mon.2) 613 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T20:24:23.145 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:24:23 smithi164 bash[59462]: audit 1685823862.8443742 mon.a (mon.0) 1444 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:24:23.165 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/mon.a/config 2023-06-03T20:24:23.166 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/mon.a/config 2023-06-03T20:24:23.167 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:24:23 smithi089 bash[78452]: audit 1685823862.1455874 mon.c (mon.2) 613 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T20:24:23.171 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:24:23.171 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:24:23 smithi089 bash[75068]: audit 1685823862.1455874 mon.c (mon.2) 613 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T20:24:23.172 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:24:23 smithi089 bash[75068]: audit 1685823862.8443742 mon.a (mon.0) 1444 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:24:23.172 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:24:23 smithi089 bash[75068]: audit 1685823862.8583713 mon.a (mon.0) 1445 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:24:23.172 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:24:23 smithi089 bash[75068]: cluster 1685823862.9407892 mon.a (mon.0) 1446 : cluster 3 Health check update: Degraded data redundancy: 59874/256497 objects degraded (23.343%), 13 pgs degraded (PG_DEGRADED) 2023-06-03T20:24:23.248 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:24:23.251 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:24:23.252 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:24:23.353 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:24:23.397 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:24:23.400 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:24:23.401 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:24:23.402 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:24:23 smithi164 bash[59462]: audit 1685823862.8583713 mon.a (mon.0) 1445 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:24:23.402 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:24:23 smithi164 bash[59462]: cluster 1685823862.9407892 mon.a (mon.0) 1446 : cluster 3 Health check update: Degraded data redundancy: 59874/256497 objects degraded (23.343%), 13 pgs degraded (PG_DEGRADED) 2023-06-03T20:24:23.503 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:24:23.529 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:24:23 smithi089 bash[78452]: audit 1685823862.8443742 mon.a (mon.0) 1444 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:24:23.531 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:24:23 smithi089 bash[78452]: audit 1685823862.8583713 mon.a (mon.0) 1445 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:24:23.531 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:24:23 smithi089 bash[78452]: cluster 1685823862.9407892 mon.a (mon.0) 1446 : cluster 3 Health check update: Degraded data redundancy: 59874/256497 objects degraded (23.343%), 13 pgs degraded (PG_DEGRADED) 2023-06-03T20:24:23.543 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:24:23.549 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:24:23.550 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:24:23.651 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:24:23.691 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:24:23.696 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:24:23.697 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:24:23.798 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:24:23.856 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:24:23.857 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:24:23.858 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:24:23.960 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:24:24.006 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:24:24.008 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:24:24.009 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:24:24.110 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:24:24.138 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:24:24.144 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:24:24.145 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:24:24.246 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:24:24.305 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:24:24.308 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:24:24.309 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:24:24.410 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:24:24.436 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:24:24.437 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:24:24.439 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:24:24.529 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:24:24 smithi089 bash[78452]: cluster 1685823863.039015 mgr.y (mgr.34104) 1454 : cluster 0 pgmap v1478: 89 pgs: 2 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 6 active+undersized+degraded+remapped+backfill_wait, 75 active+clean; 5.1 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 527 op/s; 59388/251556 objects degraded (23.608%); 10665/251556 objects misplaced (4.240%); 940 KiB/s, 14 objects/s recovering 2023-06-03T20:24:24.531 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:24:24 smithi089 bash[78452]: audit 1685823863.4382565 mon.c (mon.2) 614 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-03T20:24:24.531 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:24:24 smithi089 bash[78452]: audit 1685823863.4395003 mon.c (mon.2) 615 : audit 1 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-03T20:24:24.531 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:24:24 smithi089 bash[78452]: audit 1685823863.4475503 mon.a (mon.0) 1447 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:24:24.531 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:24:24 smithi089 bash[75068]: cluster 1685823863.039015 mgr.y (mgr.34104) 1454 : cluster 0 pgmap v1478: 89 pgs: 2 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 6 active+undersized+degraded+remapped+backfill_wait, 75 active+clean; 5.1 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 527 op/s; 59388/251556 objects degraded (23.608%); 10665/251556 objects misplaced (4.240%); 940 KiB/s, 14 objects/s recovering 2023-06-03T20:24:24.532 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:24:24 smithi089 bash[75068]: audit 1685823863.4382565 mon.c (mon.2) 614 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-03T20:24:24.532 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:24:24 smithi089 bash[75068]: audit 1685823863.4395003 mon.c (mon.2) 615 : audit 1 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-03T20:24:24.532 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:24:24 smithi089 bash[75068]: audit 1685823863.4475503 mon.a (mon.0) 1447 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:24:24.540 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:24:24.550 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:24:24 smithi164 bash[59462]: cluster 1685823863.039015 mgr.y (mgr.34104) 1454 : cluster 0 pgmap v1478: 89 pgs: 2 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 6 active+undersized+degraded+remapped+backfill_wait, 75 active+clean; 5.1 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 527 op/s; 59388/251556 objects degraded (23.608%); 10665/251556 objects misplaced (4.240%); 940 KiB/s, 14 objects/s recovering 2023-06-03T20:24:24.550 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:24:24 smithi164 bash[59462]: audit 1685823863.4382565 mon.c (mon.2) 614 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-03T20:24:24.550 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:24:24 smithi164 bash[59462]: audit 1685823863.4395003 mon.c (mon.2) 615 : audit 1 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-03T20:24:24.551 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:24:24 smithi164 bash[59462]: audit 1685823863.4475503 mon.a (mon.0) 1447 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:24:24.582 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:24:24.586 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:24:24.587 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:24:24.688 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:24:24.736 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:24:24.743 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:24:24.743 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:24:24.845 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:24:24.874 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:24:24.876 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:24:24.877 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:24:24.978 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:24:25.013 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:24:25.017 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:24:25.018 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:24:25.120 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:24:25.152 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:24:25.154 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:24:25.155 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:24:25.257 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:24:25.305 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:24:25.311 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:24:25.315 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:24:25.416 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:24:25.479 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:24:25.483 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:24:25.484 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:24:25.585 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:24:25.632 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:24:25.636 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:24:25.637 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:24:25.738 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:24:25.766 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:24:25.768 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:24:25.768 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:24:25.869 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:24:25.923 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:24:25.928 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:24:25.929 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:24:26.031 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:24:26.063 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:24:26.066 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:24:26.067 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:24:26.158 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:24:26 smithi164 bash[59462]: cluster 1685823865.0397751 mgr.y (mgr.34104) 1455 : cluster 0 pgmap v1479: 89 pgs: 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 6 active+undersized+degraded+remapped+backfill_wait, 75 active+clean; 5.1 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 526 op/s; 58805/248592 objects degraded (23.655%); 10492/248592 objects misplaced (4.221%); 789 KiB/s, 12 objects/s recovering 2023-06-03T20:24:26.168 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:24:26.197 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:24:26.203 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:24:26.204 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:24:26.305 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:24:26.345 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:24:26.347 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:24:26.348 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:24:26.449 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:24:26.504 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:24:26.509 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:24:26.510 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:24:26.530 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:24:26 smithi089 bash[75068]: cluster 1685823865.0397751 mgr.y (mgr.34104) 1455 : cluster 0 pgmap v1479: 89 pgs: 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 6 active+undersized+degraded+remapped+backfill_wait, 75 active+clean; 5.1 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 526 op/s; 58805/248592 objects degraded (23.655%); 10492/248592 objects misplaced (4.221%); 789 KiB/s, 12 objects/s recovering 2023-06-03T20:24:26.531 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 20:24:26 smithi089 bash[71163]: ::ffff:172.21.15.164 - - [03/Jun/2023:20:24:26] "GET /metrics HTTP/1.1" 200 228957 "" "Prometheus/2.33.4" 2023-06-03T20:24:26.531 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:24:26 smithi089 bash[78452]: cluster 1685823865.0397751 mgr.y (mgr.34104) 1455 : cluster 0 pgmap v1479: 89 pgs: 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 6 active+undersized+degraded+remapped+backfill_wait, 75 active+clean; 5.1 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 526 op/s; 58805/248592 objects degraded (23.655%); 10492/248592 objects misplaced (4.221%); 789 KiB/s, 12 objects/s recovering 2023-06-03T20:24:26.611 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:24:26.656 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:24:26.662 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:24:26.663 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:24:26.764 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:24:26.811 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:24:26.817 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:24:26.818 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:24:26.919 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:24:26.960 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:24:26.963 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:24:26.964 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:24:27.066 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:24:27.107 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:24:27.111 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:24:27.112 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:24:27.214 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:24:27.250 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:24:27.254 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:24:27.255 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:24:27.356 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:24:27.404 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:24:27.409 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:24:27.410 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:24:27.512 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:24:27.544 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:24:27.548 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:24:27.549 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:24:27.650 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:24:27.692 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:24:27.693 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:24:27.694 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:24:27.795 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:24:27.822 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:24:27.829 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:24:27.830 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:24:27.931 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:24:27.965 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:24:27.969 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:24:27.970 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:24:28.072 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:24:28.117 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:24:28.120 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:24:28.121 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:24:28.163 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:24:28 smithi089 bash[75068]: cluster 1685823867.0405226 mgr.y (mgr.34104) 1456 : cluster 0 pgmap v1480: 89 pgs: 2 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 6 active+undersized+degraded+remapped+backfill_wait, 75 active+clean; 5.0 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 508 op/s; 58805/246105 objects degraded (23.894%); 10325/246105 objects misplaced (4.195%); 720 KiB/s, 11 objects/s recovering 2023-06-03T20:24:28.163 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:24:28 smithi089 bash[75068]: cluster 1685823867.9421425 mon.a (mon.0) 1448 : cluster 3 Health check update: Degraded data redundancy: 58805/246105 objects degraded (23.894%), 13 pgs degraded (PG_DEGRADED) 2023-06-03T20:24:28.223 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:24:28.256 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:24:28.259 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:24:28.260 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:24:28.334 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:24:28 smithi164 bash[59462]: cluster 1685823867.0405226 mgr.y (mgr.34104) 1456 : cluster 0 pgmap v1480: 89 pgs: 2 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 6 active+undersized+degraded+remapped+backfill_wait, 75 active+clean; 5.0 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 508 op/s; 58805/246105 objects degraded (23.894%); 10325/246105 objects misplaced (4.195%); 720 KiB/s, 11 objects/s recovering 2023-06-03T20:24:28.334 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:24:28 smithi164 bash[59462]: cluster 1685823867.9421425 mon.a (mon.0) 1448 : cluster 3 Health check update: Degraded data redundancy: 58805/246105 objects degraded (23.894%), 13 pgs degraded (PG_DEGRADED) 2023-06-03T20:24:28.361 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:24:28.385 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:24:28.388 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:24:28.389 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:24:28.490 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:24:28.529 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:24:28 smithi089 bash[78452]: cluster 1685823867.0405226 mgr.y (mgr.34104) 1456 : cluster 0 pgmap v1480: 89 pgs: 2 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 6 active+undersized+degraded+remapped+backfill_wait, 75 active+clean; 5.0 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 508 op/s; 58805/246105 objects degraded (23.894%); 10325/246105 objects misplaced (4.195%); 720 KiB/s, 11 objects/s recovering 2023-06-03T20:24:28.529 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:24:28 smithi089 bash[78452]: cluster 1685823867.9421425 mon.a (mon.0) 1448 : cluster 3 Health check update: Degraded data redundancy: 58805/246105 objects degraded (23.894%), 13 pgs degraded (PG_DEGRADED) 2023-06-03T20:24:28.531 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:24:28.534 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:24:28.535 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:24:28.636 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:24:28.661 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:24:28.663 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:24:28.664 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:24:28.765 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:24:28.808 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:24:28.815 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:24:28.816 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:24:28.917 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:24:28.967 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:24:28.969 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:24:28.970 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:24:29.071 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:24:29.102 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:24:29.109 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:24:29.110 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:24:29.210 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:24:29.246 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:24:29.251 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:24:29.252 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:24:29.353 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:24:29.388 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:24:29.395 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:24:29.396 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:24:29.497 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:24:29.527 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:24:29.529 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:24:29.530 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:24:29.631 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:24:29.670 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:24:29.673 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:24:29.674 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:24:29.775 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:24:29.818 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:24:29.826 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:24:29.827 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:24:29.929 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:24:29.966 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:24:29.968 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:24:29.979 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:24:30.080 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:24:30.128 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:24:30.135 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:24:30.136 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:24:30.238 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:24:30.266 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:24:30.269 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:24:30.270 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:24:30.371 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:24:30.381 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:24:30 smithi089 bash[75068]: cluster 1685823869.0412986 mgr.y (mgr.34104) 1457 : cluster 0 pgmap v1481: 89 pgs: 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 6 active+undersized+degraded+remapped+backfill_wait, 75 active+clean; 4.9 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 551 op/s; 58034/242055 objects degraded (23.976%); 10155/242055 objects misplaced (4.195%); 1.1 MiB/s, 17 objects/s recovering 2023-06-03T20:24:30.381 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:24:30 smithi089 bash[78452]: cluster 1685823869.0412986 mgr.y (mgr.34104) 1457 : cluster 0 pgmap v1481: 89 pgs: 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 6 active+undersized+degraded+remapped+backfill_wait, 75 active+clean; 4.9 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 551 op/s; 58034/242055 objects degraded (23.976%); 10155/242055 objects misplaced (4.195%); 1.1 MiB/s, 17 objects/s recovering 2023-06-03T20:24:30.405 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:24:30.411 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:24:30.412 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:24:30.513 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:24:30.563 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:24:30.567 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:24:30.568 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:24:30.584 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:24:30 smithi164 bash[59462]: cluster 1685823869.0412986 mgr.y (mgr.34104) 1457 : cluster 0 pgmap v1481: 89 pgs: 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 6 active+undersized+degraded+remapped+backfill_wait, 75 active+clean; 4.9 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 551 op/s; 58034/242055 objects degraded (23.976%); 10155/242055 objects misplaced (4.195%); 1.1 MiB/s, 17 objects/s recovering 2023-06-03T20:24:30.669 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:24:30.703 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:24:30.705 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:24:30.706 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:24:30.806 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:24:30.837 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:24:30.840 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:24:30.840 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:24:30.941 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:24:30.979 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:24:30.982 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:24:30.982 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:24:31.083 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:24:31.087 INFO:teuthology.orchestra.run.smithi089.stdout: 2023-06-03T20:24:31.094 INFO:teuthology.orchestra.run.smithi089.stderr:dumped all 2023-06-03T20:24:31.123 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:24:31.126 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:24:31.129 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:24:31.230 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:24:31.272 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:24:31.276 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:24:31.276 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:24:31.377 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:24:31.421 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:24:31.426 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:24:31.427 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:24:31.528 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:24:31.572 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:24:31.574 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:24:31.574 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:24:31.675 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:24:31.720 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:24:31.727 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:24:31.728 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:24:31.829 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:24:31.871 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:24:31.876 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:24:31.876 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:24:31.977 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:24:32.005 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:24:32.007 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:24:32.008 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:24:32.109 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:24:32.139 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:24:32.141 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:24:32.141 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:24:32.242 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:24:32.279 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:24:32 smithi089 bash[75068]: cluster 1685823871.0420606 mgr.y (mgr.34104) 1458 : cluster 0 pgmap v1482: 89 pgs: 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 6 active+undersized+degraded+remapped+backfill_wait, 75 active+clean; 4.9 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 482 op/s; 57711/240582 objects degraded (23.988%); 10155/240582 objects misplaced (4.221%); 757 KiB/s, 11 objects/s recovering 2023-06-03T20:24:32.279 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:24:32 smithi089 bash[75068]: audit 1685823871.0627248 mgr.y (mgr.34104) 1459 : audit 0 from='client.36636 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-06-03T20:24:32.280 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:24:32 smithi089 bash[78452]: cluster 1685823871.0420606 mgr.y (mgr.34104) 1458 : cluster 0 pgmap v1482: 89 pgs: 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 6 active+undersized+degraded+remapped+backfill_wait, 75 active+clean; 4.9 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 482 op/s; 57711/240582 objects degraded (23.988%); 10155/240582 objects misplaced (4.221%); 757 KiB/s, 11 objects/s recovering 2023-06-03T20:24:32.280 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:24:32 smithi089 bash[78452]: audit 1685823871.0627248 mgr.y (mgr.34104) 1459 : audit 0 from='client.36636 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-06-03T20:24:32.280 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:24:32.284 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:24:32.285 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:24:32.385 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:24:32.420 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:24:32.425 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:24:32.426 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:24:32.528 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:24:32.569 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:24:32.573 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:24:32.574 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:24:32.584 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:24:32 smithi164 bash[59462]: cluster 1685823871.0420606 mgr.y (mgr.34104) 1458 : cluster 0 pgmap v1482: 89 pgs: 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 6 active+undersized+degraded+remapped+backfill_wait, 75 active+clean; 4.9 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 482 op/s; 57711/240582 objects degraded (23.988%); 10155/240582 objects misplaced (4.221%); 757 KiB/s, 11 objects/s recovering 2023-06-03T20:24:32.585 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:24:32 smithi164 bash[59462]: audit 1685823871.0627248 mgr.y (mgr.34104) 1459 : audit 0 from='client.36636 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-06-03T20:24:32.675 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:24:32.724 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:24:32.726 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:24:32.727 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:24:32.828 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:24:32.866 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:24:32.868 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:24:32.869 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:24:32.970 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:24:32.999 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:24:33.002 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:24:33.003 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:24:33.104 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:24:33.145 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:24:33.151 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:24:33.151 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:24:33.252 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:24:33.259 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:24:33 smithi089 bash[78452]: cluster 1685823872.9432566 mon.a (mon.0) 1449 : cluster 3 Health check update: Degraded data redundancy: 57711/240582 objects degraded (23.988%), 13 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2023-06-03T20:24:33.260 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:24:33 smithi089 bash[75068]: cluster 1685823872.9432566 mon.a (mon.0) 1449 : cluster 3 Health check update: Degraded data redundancy: 57711/240582 objects degraded (23.988%), 13 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2023-06-03T20:24:33.280 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:24:33.281 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:24:33.282 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:24:33.383 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:24:33.410 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:24:33.414 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:24:33.415 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:24:33.516 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:24:33.558 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:24:33.560 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:24:33.560 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:24:33.584 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:24:33 smithi164 bash[59462]: cluster 1685823872.9432566 mon.a (mon.0) 1449 : cluster 3 Health check update: Degraded data redundancy: 57711/240582 objects degraded (23.988%), 13 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2023-06-03T20:24:33.661 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:24:33.690 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:24:33.693 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:24:33.694 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:24:33.795 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:24:33.833 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:24:33.839 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:24:33.840 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:24:33.941 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:24:33.998 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:24:34.007 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:24:34.008 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:24:34.110 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:24:34.145 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:24:34.150 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:24:34.151 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:24:34.252 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:24:34.295 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:24:34.297 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:24:34.298 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:24:34.399 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:24:34.408 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:24:34 smithi089 bash[78452]: cluster 1685823873.0428333 mgr.y (mgr.34104) 1460 : cluster 0 pgmap v1483: 89 pgs: 2 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 6 active+undersized+degraded+remapped+backfill_wait, 75 active+clean; 4.8 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 579 op/s; 57195/235626 objects degraded (24.274%); 9997/235626 objects misplaced (4.243%); 757 KiB/s, 11 objects/s recovering 2023-06-03T20:24:34.408 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:24:34 smithi089 bash[75068]: cluster 1685823873.0428333 mgr.y (mgr.34104) 1460 : cluster 0 pgmap v1483: 89 pgs: 2 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 6 active+undersized+degraded+remapped+backfill_wait, 75 active+clean; 4.8 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 579 op/s; 57195/235626 objects degraded (24.274%); 9997/235626 objects misplaced (4.243%); 757 KiB/s, 11 objects/s recovering 2023-06-03T20:24:34.424 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:24:34.428 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:24:34.429 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:24:34.531 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:24:34.563 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:24:34.565 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:24:34.566 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:24:34.584 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:24:34 smithi164 bash[59462]: cluster 1685823873.0428333 mgr.y (mgr.34104) 1460 : cluster 0 pgmap v1483: 89 pgs: 2 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 6 active+undersized+degraded+remapped+backfill_wait, 75 active+clean; 4.8 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 579 op/s; 57195/235626 objects degraded (24.274%); 9997/235626 objects misplaced (4.243%); 757 KiB/s, 11 objects/s recovering 2023-06-03T20:24:34.668 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:24:34.709 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:24:34.714 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:24:34.715 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:24:34.816 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:24:34.848 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:24:34.851 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:24:34.851 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:24:34.953 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:24:34.988 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:24:34.989 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:24:34.990 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:24:35.091 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:24:35.119 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:24:35.121 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:24:35.122 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:24:35.223 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:24:35.261 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:24:35.267 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:24:35.267 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:24:35.368 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:24:35.421 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:24:35.423 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:24:35.424 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:24:35.525 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:24:35.585 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:24:35.589 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:24:35.590 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:24:35.691 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:24:35.728 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:24:35.730 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:24:35.731 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:24:35.832 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:24:35.869 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:24:35.874 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:24:35.874 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:24:35.897 INFO:teuthology.orchestra.run.smithi089.stdout:{ 2023-06-03T20:24:35.897 INFO:teuthology.orchestra.run.smithi089.stdout: "ops": [ 2023-06-03T20:24:35.898 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:24:35.898 INFO:teuthology.orchestra.run.smithi089.stdout: "description": "osd_op(client.36573.0:155192 13.b 13:da522470:::benchmark_data_smithi089_211581_object39667:head [delete] snapc 0=[] ondisk+write+known_if_redirected e257)", 2023-06-03T20:24:35.898 INFO:teuthology.orchestra.run.smithi089.stdout: "initiated_at": "2023-06-03T20:24:35.865259+0000", 2023-06-03T20:24:35.898 INFO:teuthology.orchestra.run.smithi089.stdout: "age": 0.026790880999999999, 2023-06-03T20:24:35.898 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.026819912000000001, 2023-06-03T20:24:35.898 INFO:teuthology.orchestra.run.smithi089.stdout: "type_data": { 2023-06-03T20:24:35.898 INFO:teuthology.orchestra.run.smithi089.stdout: "flag_point": "waiting for sub ops", 2023-06-03T20:24:35.898 INFO:teuthology.orchestra.run.smithi089.stdout: "client_info": { 2023-06-03T20:24:35.899 INFO:teuthology.orchestra.run.smithi089.stdout: "client": "client.36573", 2023-06-03T20:24:35.899 INFO:teuthology.orchestra.run.smithi089.stdout: "client_addr": "172.21.15.89:0/616445942", 2023-06-03T20:24:35.899 INFO:teuthology.orchestra.run.smithi089.stdout: "tid": 155192 2023-06-03T20:24:35.899 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:24:35.899 INFO:teuthology.orchestra.run.smithi089.stdout: "events": [ 2023-06-03T20:24:35.899 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:24:35.899 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "initiated", 2023-06-03T20:24:35.900 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:24:35.865259+0000", 2023-06-03T20:24:35.900 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0 2023-06-03T20:24:35.900 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:24:35.900 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:24:35.900 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "throttled", 2023-06-03T20:24:35.900 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:24:35.865259+0000", 2023-06-03T20:24:35.900 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0 2023-06-03T20:24:35.901 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:24:35.901 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:24:35.901 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "header_read", 2023-06-03T20:24:35.901 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:24:35.865257+0000", 2023-06-03T20:24:35.901 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 4294967295.9999976 2023-06-03T20:24:35.901 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:24:35.901 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:24:35.901 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "all_read", 2023-06-03T20:24:35.902 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:24:35.865266+0000", 2023-06-03T20:24:35.902 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 8.8890000000000008e-06 2023-06-03T20:24:35.902 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:24:35.902 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:24:35.902 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "dispatched", 2023-06-03T20:24:35.902 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:24:35.865280+0000", 2023-06-03T20:24:35.902 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 1.4212999999999999e-05 2023-06-03T20:24:35.902 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:24:35.903 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:24:35.903 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "queued_for_pg", 2023-06-03T20:24:35.903 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:24:35.865291+0000", 2023-06-03T20:24:35.903 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 1.0583e-05 2023-06-03T20:24:35.903 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:24:35.903 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:24:35.903 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "reached_pg", 2023-06-03T20:24:35.903 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:24:35.865349+0000", 2023-06-03T20:24:35.904 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 5.8823000000000002e-05 2023-06-03T20:24:35.904 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:24:35.904 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:24:35.904 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "started", 2023-06-03T20:24:35.904 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:24:35.865513+0000", 2023-06-03T20:24:35.904 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.000163769 2023-06-03T20:24:35.904 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:24:35.905 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:24:35.905 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "waiting for subops from 2,5", 2023-06-03T20:24:35.905 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:24:35.865666+0000", 2023-06-03T20:24:35.905 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.000152943 2023-06-03T20:24:35.905 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:24:35.905 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:24:35.905 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:24:35.906 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:24:35.875607+0000", 2023-06-03T20:24:35.906 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.0099412249999999997 2023-06-03T20:24:35.906 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:24:35.906 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:24:35.906 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "op_commit", 2023-06-03T20:24:35.906 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:24:35.875620+0000", 2023-06-03T20:24:35.906 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 1.2184000000000001e-05 2023-06-03T20:24:35.906 INFO:teuthology.orchestra.run.smithi089.stdout: } 2023-06-03T20:24:35.907 INFO:teuthology.orchestra.run.smithi089.stdout: ] 2023-06-03T20:24:35.907 INFO:teuthology.orchestra.run.smithi089.stdout: } 2023-06-03T20:24:35.907 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:24:35.907 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:24:35.907 INFO:teuthology.orchestra.run.smithi089.stdout: "description": "osd_op(client.36573.0:155193 13.b 13:d04662fb:::benchmark_data_smithi089_211581_object39668:head [delete] snapc 0=[] ondisk+write+known_if_redirected e257)", 2023-06-03T20:24:35.907 INFO:teuthology.orchestra.run.smithi089.stdout: "initiated_at": "2023-06-03T20:24:35.869526+0000", 2023-06-03T20:24:35.907 INFO:teuthology.orchestra.run.smithi089.stdout: "age": 0.022523838000000001, 2023-06-03T20:24:35.908 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.022613274999999999, 2023-06-03T20:24:35.908 INFO:teuthology.orchestra.run.smithi089.stdout: "type_data": { 2023-06-03T20:24:35.908 INFO:teuthology.orchestra.run.smithi089.stdout: "flag_point": "waiting for sub ops", 2023-06-03T20:24:35.908 INFO:teuthology.orchestra.run.smithi089.stdout: "client_info": { 2023-06-03T20:24:35.908 INFO:teuthology.orchestra.run.smithi089.stdout: "client": "client.36573", 2023-06-03T20:24:35.908 INFO:teuthology.orchestra.run.smithi089.stdout: "client_addr": "172.21.15.89:0/616445942", 2023-06-03T20:24:35.908 INFO:teuthology.orchestra.run.smithi089.stdout: "tid": 155193 2023-06-03T20:24:35.908 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:24:35.909 INFO:teuthology.orchestra.run.smithi089.stdout: "events": [ 2023-06-03T20:24:35.909 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:24:35.910 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "initiated", 2023-06-03T20:24:35.910 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:24:35.869526+0000", 2023-06-03T20:24:35.910 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0 2023-06-03T20:24:35.910 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:24:35.910 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:24:35.911 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "throttled", 2023-06-03T20:24:35.911 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:24:35.869526+0000", 2023-06-03T20:24:35.911 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0 2023-06-03T20:24:35.911 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:24:35.911 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:24:35.911 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "header_read", 2023-06-03T20:24:35.911 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:24:35.869524+0000", 2023-06-03T20:24:35.912 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 4294967295.9999976 2023-06-03T20:24:35.912 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:24:35.912 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:24:35.912 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "all_read", 2023-06-03T20:24:35.912 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:24:35.869534+0000", 2023-06-03T20:24:35.912 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 9.747e-06 2023-06-03T20:24:35.912 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:24:35.912 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:24:35.913 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "dispatched", 2023-06-03T20:24:35.913 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:24:35.869550+0000", 2023-06-03T20:24:35.913 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 1.6220999999999999e-05 2023-06-03T20:24:35.913 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:24:35.913 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:24:35.913 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "queued_for_pg", 2023-06-03T20:24:35.913 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:24:35.869561+0000", 2023-06-03T20:24:35.914 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 1.1385e-05 2023-06-03T20:24:35.914 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:24:35.914 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:24:35.914 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "reached_pg", 2023-06-03T20:24:35.914 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:24:35.870738+0000", 2023-06-03T20:24:35.914 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.001177048 2023-06-03T20:24:35.914 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:24:35.914 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:24:35.915 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "started", 2023-06-03T20:24:35.915 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:24:35.870820+0000", 2023-06-03T20:24:35.915 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 8.1683999999999999e-05 2023-06-03T20:24:35.915 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:24:35.915 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:24:35.915 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "waiting for subops from 2,5", 2023-06-03T20:24:35.916 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:24:35.870910+0000", 2023-06-03T20:24:35.916 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 8.9732000000000001e-05 2023-06-03T20:24:35.916 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:24:35.916 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:24:35.916 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "op_commit", 2023-06-03T20:24:35.916 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:24:35.875624+0000", 2023-06-03T20:24:35.916 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.0047138220000000003 2023-06-03T20:24:35.917 INFO:teuthology.orchestra.run.smithi089.stdout: } 2023-06-03T20:24:35.917 INFO:teuthology.orchestra.run.smithi089.stdout: ] 2023-06-03T20:24:35.917 INFO:teuthology.orchestra.run.smithi089.stdout: } 2023-06-03T20:24:35.917 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:24:35.917 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:24:35.917 INFO:teuthology.orchestra.run.smithi089.stdout: "description": "osd_op(client.36573.0:155196 13.b 13:d695adec:::benchmark_data_smithi089_211581_object39671:head [delete] snapc 0=[] ondisk+write+known_if_redirected e257)", 2023-06-03T20:24:35.917 INFO:teuthology.orchestra.run.smithi089.stdout: "initiated_at": "2023-06-03T20:24:35.875671+0000", 2023-06-03T20:24:35.917 INFO:teuthology.orchestra.run.smithi089.stdout: "age": 0.016378670000000001, 2023-06-03T20:24:35.918 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.016516349999999999, 2023-06-03T20:24:35.918 INFO:teuthology.orchestra.run.smithi089.stdout: "type_data": { 2023-06-03T20:24:35.918 INFO:teuthology.orchestra.run.smithi089.stdout: "flag_point": "waiting for sub ops", 2023-06-03T20:24:35.918 INFO:teuthology.orchestra.run.smithi089.stdout: "client_info": { 2023-06-03T20:24:35.918 INFO:teuthology.orchestra.run.smithi089.stdout: "client": "client.36573", 2023-06-03T20:24:35.918 INFO:teuthology.orchestra.run.smithi089.stdout: "client_addr": "172.21.15.89:0/616445942", 2023-06-03T20:24:35.918 INFO:teuthology.orchestra.run.smithi089.stdout: "tid": 155196 2023-06-03T20:24:35.918 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:24:35.919 INFO:teuthology.orchestra.run.smithi089.stdout: "events": [ 2023-06-03T20:24:35.919 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:24:35.919 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "initiated", 2023-06-03T20:24:35.919 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:24:35.875671+0000", 2023-06-03T20:24:35.919 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0 2023-06-03T20:24:35.919 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:24:35.919 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:24:35.919 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "throttled", 2023-06-03T20:24:35.920 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:24:35.875671+0000", 2023-06-03T20:24:35.920 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0 2023-06-03T20:24:35.920 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:24:35.921 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:24:35.921 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "header_read", 2023-06-03T20:24:35.921 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:24:35.875670+0000", 2023-06-03T20:24:35.921 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 4294967295.9999986 2023-06-03T20:24:35.921 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:24:35.921 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:24:35.921 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "all_read", 2023-06-03T20:24:35.921 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:24:35.875678+0000", 2023-06-03T20:24:35.922 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 8.3529999999999995e-06 2023-06-03T20:24:35.922 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:24:35.922 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:24:35.922 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "dispatched", 2023-06-03T20:24:35.922 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:24:35.875690+0000", 2023-06-03T20:24:35.922 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 1.1950000000000001e-05 2023-06-03T20:24:35.922 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:24:35.922 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:24:35.923 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "queued_for_pg", 2023-06-03T20:24:35.923 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:24:35.875699+0000", 2023-06-03T20:24:35.924 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 8.7110000000000001e-06 2023-06-03T20:24:35.924 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:24:35.924 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:24:35.924 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "reached_pg", 2023-06-03T20:24:35.924 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:24:35.875813+0000", 2023-06-03T20:24:35.924 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.000114009 2023-06-03T20:24:35.925 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:24:35.925 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:24:35.925 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "started", 2023-06-03T20:24:35.925 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:24:35.875943+0000", 2023-06-03T20:24:35.925 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.00012978800000000001 2023-06-03T20:24:35.925 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:24:35.925 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:24:35.926 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "waiting for subops from 2,5", 2023-06-03T20:24:35.926 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:24:35.876038+0000", 2023-06-03T20:24:35.926 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 9.5309999999999994e-05 2023-06-03T20:24:35.926 INFO:teuthology.orchestra.run.smithi089.stdout: } 2023-06-03T20:24:35.926 INFO:teuthology.orchestra.run.smithi089.stdout: ] 2023-06-03T20:24:35.926 INFO:teuthology.orchestra.run.smithi089.stdout: } 2023-06-03T20:24:35.926 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:24:35.927 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:24:35.927 INFO:teuthology.orchestra.run.smithi089.stdout: "description": "osd_repop_reply(client.36573.0:155193 13.b e257/242)", 2023-06-03T20:24:35.927 INFO:teuthology.orchestra.run.smithi089.stdout: "initiated_at": "2023-06-03T20:24:35.875742+0000", 2023-06-03T20:24:35.927 INFO:teuthology.orchestra.run.smithi089.stdout: "age": 0.01630817, 2023-06-03T20:24:35.927 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.016487783999999998, 2023-06-03T20:24:35.927 INFO:teuthology.orchestra.run.smithi089.stdout: "type_data": { 2023-06-03T20:24:35.927 INFO:teuthology.orchestra.run.smithi089.stdout: "flag_point": "queued for pg", 2023-06-03T20:24:35.927 INFO:teuthology.orchestra.run.smithi089.stdout: "events": [ 2023-06-03T20:24:35.928 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:24:35.928 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "initiated", 2023-06-03T20:24:35.928 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:24:35.875742+0000", 2023-06-03T20:24:35.928 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0 2023-06-03T20:24:35.928 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:24:35.928 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:24:35.928 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "throttled", 2023-06-03T20:24:35.929 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:24:35.875742+0000", 2023-06-03T20:24:35.929 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0 2023-06-03T20:24:35.929 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:24:35.929 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:24:35.929 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "header_read", 2023-06-03T20:24:35.929 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:24:35.875741+0000", 2023-06-03T20:24:35.929 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 4294967295.999999 2023-06-03T20:24:35.929 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:24:35.930 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:24:35.930 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "all_read", 2023-06-03T20:24:35.930 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:24:35.875747+0000", 2023-06-03T20:24:35.930 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 6.1029999999999998e-06 2023-06-03T20:24:35.930 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:24:35.930 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:24:35.930 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "dispatched", 2023-06-03T20:24:35.930 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:24:35.875758+0000", 2023-06-03T20:24:35.931 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 1.1127999999999999e-05 2023-06-03T20:24:35.931 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:24:35.931 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:24:35.931 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "queued_for_pg", 2023-06-03T20:24:35.931 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:24:35.875766+0000", 2023-06-03T20:24:35.931 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 7.5820000000000004e-06 2023-06-03T20:24:35.931 INFO:teuthology.orchestra.run.smithi089.stdout: } 2023-06-03T20:24:35.932 INFO:teuthology.orchestra.run.smithi089.stdout: ] 2023-06-03T20:24:35.932 INFO:teuthology.orchestra.run.smithi089.stdout: } 2023-06-03T20:24:35.932 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:24:35.932 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:24:35.932 INFO:teuthology.orchestra.run.smithi089.stdout: "description": "osd_op(client.36573.0:155202 13.b 13.525cf21b (undecoded) ondisk+write+known_if_redirected e257)", 2023-06-03T20:24:35.933 INFO:teuthology.orchestra.run.smithi089.stdout: "initiated_at": "2023-06-03T20:24:35.877445+0000", 2023-06-03T20:24:35.933 INFO:teuthology.orchestra.run.smithi089.stdout: "age": 0.014604758000000001, 2023-06-03T20:24:35.933 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.0148124, 2023-06-03T20:24:35.933 INFO:teuthology.orchestra.run.smithi089.stdout: "type_data": { 2023-06-03T20:24:35.933 INFO:teuthology.orchestra.run.smithi089.stdout: "flag_point": "queued for pg", 2023-06-03T20:24:35.933 INFO:teuthology.orchestra.run.smithi089.stdout: "client_info": { 2023-06-03T20:24:35.933 INFO:teuthology.orchestra.run.smithi089.stdout: "client": "client.36573", 2023-06-03T20:24:35.934 INFO:teuthology.orchestra.run.smithi089.stdout: "client_addr": "172.21.15.89:0/616445942", 2023-06-03T20:24:35.934 INFO:teuthology.orchestra.run.smithi089.stdout: "tid": 155202 2023-06-03T20:24:35.934 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:24:35.934 INFO:teuthology.orchestra.run.smithi089.stdout: "events": [ 2023-06-03T20:24:35.934 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:24:35.934 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "initiated", 2023-06-03T20:24:35.935 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:24:35.877445+0000", 2023-06-03T20:24:35.935 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0 2023-06-03T20:24:35.935 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:24:35.935 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:24:35.935 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "throttled", 2023-06-03T20:24:35.935 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:24:35.877445+0000", 2023-06-03T20:24:35.935 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0 2023-06-03T20:24:35.936 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:24:35.936 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:24:35.936 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "header_read", 2023-06-03T20:24:35.936 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:24:35.877444+0000", 2023-06-03T20:24:35.936 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 4294967295.9999986 2023-06-03T20:24:35.936 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:24:35.936 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:24:35.937 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "all_read", 2023-06-03T20:24:35.937 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:24:35.877450+0000", 2023-06-03T20:24:35.937 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 6.4420000000000003e-06 2023-06-03T20:24:35.937 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:24:35.937 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:24:35.937 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "dispatched", 2023-06-03T20:24:35.938 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:24:35.877463+0000", 2023-06-03T20:24:35.938 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 1.2312e-05 2023-06-03T20:24:35.938 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:24:35.938 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:24:35.938 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "queued_for_pg", 2023-06-03T20:24:35.938 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:24:35.877471+0000", 2023-06-03T20:24:35.939 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 7.8199999999999997e-06 2023-06-03T20:24:35.939 INFO:teuthology.orchestra.run.smithi089.stdout: } 2023-06-03T20:24:35.939 INFO:teuthology.orchestra.run.smithi089.stdout: ] 2023-06-03T20:24:35.939 INFO:teuthology.orchestra.run.smithi089.stdout: } 2023-06-03T20:24:35.939 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:24:35.939 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:24:35.940 INFO:teuthology.orchestra.run.smithi089.stdout: "description": "osd_op(client.36573.0:155206 13.6 13:606c73eb:::benchmark_data_smithi089_211581_object39681:head [delete] snapc 0=[] ondisk+write+known_if_redirected e257)", 2023-06-03T20:24:35.940 INFO:teuthology.orchestra.run.smithi089.stdout: "initiated_at": "2023-06-03T20:24:35.883125+0000", 2023-06-03T20:24:35.940 INFO:teuthology.orchestra.run.smithi089.stdout: "age": 0.0089252629999999993, 2023-06-03T20:24:35.940 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.0091660730000000003, 2023-06-03T20:24:35.940 INFO:teuthology.orchestra.run.smithi089.stdout: "type_data": { 2023-06-03T20:24:35.940 INFO:teuthology.orchestra.run.smithi089.stdout: "flag_point": "started", 2023-06-03T20:24:35.941 INFO:teuthology.orchestra.run.smithi089.stdout: "client_info": { 2023-06-03T20:24:35.941 INFO:teuthology.orchestra.run.smithi089.stdout: "client": "client.36573", 2023-06-03T20:24:35.941 INFO:teuthology.orchestra.run.smithi089.stdout: "client_addr": "172.21.15.89:0/616445942", 2023-06-03T20:24:35.941 INFO:teuthology.orchestra.run.smithi089.stdout: "tid": 155206 2023-06-03T20:24:35.941 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:24:35.941 INFO:teuthology.orchestra.run.smithi089.stdout: "events": [ 2023-06-03T20:24:35.941 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:24:35.941 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "initiated", 2023-06-03T20:24:35.942 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:24:35.883125+0000", 2023-06-03T20:24:35.942 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0 2023-06-03T20:24:35.942 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:24:35.942 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:24:35.942 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "throttled", 2023-06-03T20:24:35.942 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:24:35.883125+0000", 2023-06-03T20:24:35.942 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0 2023-06-03T20:24:35.943 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:24:35.943 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:24:35.943 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "header_read", 2023-06-03T20:24:35.943 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:24:35.883123+0000", 2023-06-03T20:24:35.943 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 4294967295.9999981 2023-06-03T20:24:35.943 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:24:35.943 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:24:35.943 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "all_read", 2023-06-03T20:24:35.944 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:24:35.883131+0000", 2023-06-03T20:24:35.944 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 7.9060000000000002e-06 2023-06-03T20:24:35.944 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:24:35.944 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:24:35.944 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "dispatched", 2023-06-03T20:24:35.944 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:24:35.883145+0000", 2023-06-03T20:24:35.944 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 1.4036000000000001e-05 2023-06-03T20:24:35.945 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:24:35.945 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:24:35.945 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "queued_for_pg", 2023-06-03T20:24:35.945 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:24:35.883155+0000", 2023-06-03T20:24:35.945 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 1.0475999999999999e-05 2023-06-03T20:24:35.946 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:24:35.946 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:24:35.946 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "reached_pg", 2023-06-03T20:24:35.946 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:24:35.883209+0000", 2023-06-03T20:24:35.946 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 5.4004e-05 2023-06-03T20:24:35.946 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:24:35.947 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:24:35.947 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "started", 2023-06-03T20:24:35.947 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:24:35.883373+0000", 2023-06-03T20:24:35.947 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.000163425 2023-06-03T20:24:35.947 INFO:teuthology.orchestra.run.smithi089.stdout: } 2023-06-03T20:24:35.947 INFO:teuthology.orchestra.run.smithi089.stdout: ] 2023-06-03T20:24:35.947 INFO:teuthology.orchestra.run.smithi089.stdout: } 2023-06-03T20:24:35.948 INFO:teuthology.orchestra.run.smithi089.stdout: } 2023-06-03T20:24:35.948 INFO:teuthology.orchestra.run.smithi089.stdout: ], 2023-06-03T20:24:35.948 INFO:teuthology.orchestra.run.smithi089.stdout: "num_ops": 6 2023-06-03T20:24:35.948 INFO:teuthology.orchestra.run.smithi089.stdout:} 2023-06-03T20:24:35.975 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:24:36.017 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:24:36.025 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:24:36.026 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:24:36.127 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:24:36.166 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:24:36.170 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:24:36.171 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:24:36.272 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:24:36.305 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:24:36.310 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:24:36.311 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:24:36.407 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 20:24:36 smithi089 bash[71163]: ::ffff:172.21.15.164 - - [03/Jun/2023:20:24:36] "GET /metrics HTTP/1.1" 200 228957 "" "Prometheus/2.33.4" 2023-06-03T20:24:36.408 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:24:36 smithi089 bash[78452]: cluster 1685823875.0435846 mgr.y (mgr.34104) 1461 : cluster 0 pgmap v1484: 89 pgs: 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 6 active+undersized+degraded+remapped+backfill_wait, 75 active+clean; 4.7 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 523 op/s; 56642/232719 objects degraded (24.339%); 9828/232719 objects misplaced (4.223%); 1.1 MiB/s, 18 objects/s recovering 2023-06-03T20:24:36.408 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:24:36 smithi089 bash[75068]: cluster 1685823875.0435846 mgr.y (mgr.34104) 1461 : cluster 0 pgmap v1484: 89 pgs: 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 6 active+undersized+degraded+remapped+backfill_wait, 75 active+clean; 4.7 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 523 op/s; 56642/232719 objects degraded (24.339%); 9828/232719 objects misplaced (4.223%); 1.1 MiB/s, 18 objects/s recovering 2023-06-03T20:24:36.412 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:24:36.493 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:24:36.495 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:24:36.496 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:24:36.598 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:24:36.631 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:24:36.633 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:24:36.634 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:24:36.736 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:24:36.745 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:24:36 smithi164 bash[59462]: cluster 1685823875.0435846 mgr.y (mgr.34104) 1461 : cluster 0 pgmap v1484: 89 pgs: 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 6 active+undersized+degraded+remapped+backfill_wait, 75 active+clean; 4.7 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 523 op/s; 56642/232719 objects degraded (24.339%); 9828/232719 objects misplaced (4.223%); 1.1 MiB/s, 18 objects/s recovering 2023-06-03T20:24:36.785 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:24:36.789 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:24:36.791 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:24:36.892 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:24:36.923 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:24:36.927 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:24:36.929 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:24:37.031 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:24:37.072 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:24:37.074 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:24:37.075 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:24:37.176 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:24:37.219 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:24:37.221 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:24:37.222 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:24:37.303 INFO:teuthology.orchestra.run.smithi089.stdout:{"pg_ready":true,"pg_map":{"version":1482,"stamp":"2023-06-03T20:24:31.041482+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":5254551462,"num_objects":80194,"num_object_clones":0,"num_object_copies":240582,"num_objects_missing_on_primary":4,"num_objects_missing":4,"num_objects_degraded":57711,"num_objects_misplaced":10155,"num_objects_unfound":0,"num_objects_dirty":80194,"num_whiteouts":0,"num_read":309,"num_read_kb":513,"num_write":151137,"num_write_kb":7395855,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1005,"num_bytes_recovered":65279730,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":137871,"ondisk_log_size":137871,"up":267,"acting":256,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":280,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":17148892,"kb_used_data":12767132,"kb_used_omap":30,"kb_used_meta":4381473,"kb_avail":732812324,"statfs":{"total":767960285184,"available":750399819776,"internally_reserved":0,"allocated":13073543168,"data_stored":13066423752,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":31502,"internal_metadata":4486628594},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[3,0,1,7,9,7,10,4],"upper_bound":256},"perf_stat":{"commit_latency_ms":7,"apply_latency_ms":7,"commit_latency_ns":7000000,"apply_latency_ns":7000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-379584512,"num_objects":-5792,"num_object_clones":0,"num_object_copies":-17376,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":-2482,"num_objects_misplaced":-674,"num_objects_unfound":0,"num_objects_dirty":-5792,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5792,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":142,"num_bytes_recovered":9306112,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.004702"},"pg_stats":[{"pgid":"3.1a","version":"60'1","reported_seq":181,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-06-03T20:24:03.252118+0000","last_change":"2023-06-03T20:23:28.242261+0000","last_active":"2023-06-03T20:24:03.252118+0000","last_peered":"2023-06-03T20:24:03.252118+0000","last_clean":"2023-06-03T20:24:03.252118+0000","last_became_active":"2023-06-03T20:23:28.240045+0000","last_became_peered":"2023-06-03T20:23:28.240045+0000","last_unstale":"2023-06-03T20:24:03.252118+0000","last_undegraded":"2023-06-03T20:24:03.252118+0000","last_fullsized":"2023-06-03T20:24:03.252118+0000","mapping_epoch":251,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":252,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T05:54:33.215385+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":42,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-06-03T20:24:03.262954+0000","last_change":"2023-06-03T20:23:28.283021+0000","last_active":"2023-06-03T20:24:03.262954+0000","last_peered":"2023-06-03T20:24:03.262954+0000","last_clean":"2023-06-03T20:24:03.262954+0000","last_became_active":"2023-06-03T20:23:28.282428+0000","last_became_peered":"2023-06-03T20:23:28.282428+0000","last_unstale":"2023-06-03T20:24:03.262954+0000","last_undegraded":"2023-06-03T20:24:03.262954+0000","last_fullsized":"2023-06-03T20:24:03.262954+0000","mapping_epoch":251,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":252,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T05:01:33.115332+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":42,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-06-03T20:24:03.262678+0000","last_change":"2023-06-03T20:23:28.253821+0000","last_active":"2023-06-03T20:24:03.262678+0000","last_peered":"2023-06-03T20:24:03.262678+0000","last_clean":"2023-06-03T20:24:03.262678+0000","last_became_active":"2023-06-03T20:23:28.253448+0000","last_became_peered":"2023-06-03T20:23:28.253448+0000","last_unstale":"2023-06-03T20:24:03.262678+0000","last_undegraded":"2023-06-03T20:24:03.262678+0000","last_fullsized":"2023-06-03T20:24:03.262678+0000","mapping_epoch":251,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":252,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T23:40:02.264778+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":224,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-06-03T20:24:03.262263+0000","last_change":"2023-06-03T19:56:54.507653+0000","last_active":"2023-06-03T20:24:03.262263+0000","last_peered":"2023-06-03T20:24:03.262263+0000","last_clean":"2023-06-03T20:24:03.262263+0000","last_became_active":"2023-06-03T19:56:54.507350+0000","last_became_peered":"2023-06-03T19:56:54.507350+0000","last_unstale":"2023-06-03T20:24:03.262263+0000","last_undegraded":"2023-06-03T20:24:03.262263+0000","last_fullsized":"2023-06-03T20:24:03.262263+0000","mapping_epoch":119,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T22:33:20.065844+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":265,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-06-03T20:24:03.254492+0000","last_change":"2023-06-03T20:21:01.256894+0000","last_active":"2023-06-03T20:24:03.254492+0000","last_peered":"2023-06-03T20:24:03.254492+0000","last_clean":"2023-06-03T20:24:03.254492+0000","last_became_active":"2023-06-03T20:21:01.255903+0000","last_became_peered":"2023-06-03T20:21:01.255903+0000","last_unstale":"2023-06-03T20:24:03.254492+0000","last_undegraded":"2023-06-03T20:24:03.254492+0000","last_fullsized":"2023-06-03T20:24:03.254492+0000","mapping_epoch":235,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":236,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T20:35:27.810860+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":12,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-06-03T20:24:03.620864+0000","last_change":"2023-06-03T20:24:02.259545+0000","last_active":"2023-06-03T20:24:03.620864+0000","last_peered":"2023-06-03T20:24:03.620864+0000","last_clean":"2023-06-03T20:24:03.620864+0000","last_became_active":"2023-06-03T20:24:02.259180+0000","last_became_peered":"2023-06-03T20:24:02.259180+0000","last_unstale":"2023-06-03T20:24:03.620864+0000","last_undegraded":"2023-06-03T20:24:03.620864+0000","last_fullsized":"2023-06-03T20:24:03.620864+0000","mapping_epoch":254,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":255,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":220,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-06-03T20:24:03.262314+0000","last_change":"2023-06-03T19:56:54.509490+0000","last_active":"2023-06-03T20:24:03.262314+0000","last_peered":"2023-06-03T20:24:03.262314+0000","last_clean":"2023-06-03T20:24:03.262314+0000","last_became_active":"2023-06-03T19:56:54.509145+0000","last_became_peered":"2023-06-03T19:56:54.509145+0000","last_unstale":"2023-06-03T20:24:03.262314+0000","last_undegraded":"2023-06-03T20:24:03.262314+0000","last_fullsized":"2023-06-03T20:24:03.262314+0000","mapping_epoch":119,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T07:12:03.964110+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":276,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-06-03T20:24:03.254514+0000","last_change":"2023-06-03T20:23:28.243147+0000","last_active":"2023-06-03T20:24:03.254514+0000","last_peered":"2023-06-03T20:24:03.254514+0000","last_clean":"2023-06-03T20:24:03.254514+0000","last_became_active":"2023-06-03T20:23:28.242716+0000","last_became_peered":"2023-06-03T20:23:28.242716+0000","last_unstale":"2023-06-03T20:24:03.254514+0000","last_undegraded":"2023-06-03T20:24:03.254514+0000","last_fullsized":"2023-06-03T20:24:03.254514+0000","mapping_epoch":251,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":252,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T05:22:16.707501+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":189,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-06-03T20:24:03.256924+0000","last_change":"2023-06-03T20:21:01.258441+0000","last_active":"2023-06-03T20:24:03.256924+0000","last_peered":"2023-06-03T20:24:03.256924+0000","last_clean":"2023-06-03T20:24:03.256924+0000","last_became_active":"2023-06-03T20:21:01.257779+0000","last_became_peered":"2023-06-03T20:21:01.257779+0000","last_unstale":"2023-06-03T20:24:03.256924+0000","last_undegraded":"2023-06-03T20:24:03.256924+0000","last_fullsized":"2023-06-03T20:24:03.256924+0000","mapping_epoch":235,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":236,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,1],"acting":[7,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":230,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-06-03T20:24:03.254509+0000","last_change":"2023-06-03T20:24:02.267331+0000","last_active":"2023-06-03T20:24:03.254509+0000","last_peered":"2023-06-03T20:24:03.254509+0000","last_clean":"2023-06-03T20:24:03.254509+0000","last_became_active":"2023-06-03T20:24:02.266997+0000","last_became_peered":"2023-06-03T20:24:02.266997+0000","last_unstale":"2023-06-03T20:24:03.254509+0000","last_undegraded":"2023-06-03T20:24:03.254509+0000","last_fullsized":"2023-06-03T20:24:03.254509+0000","mapping_epoch":254,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":255,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T22:09:18.993656+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"60'3","reported_seq":254,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-06-03T20:24:03.257298+0000","last_change":"2023-06-03T20:23:27.261246+0000","last_active":"2023-06-03T20:24:03.257298+0000","last_peered":"2023-06-03T20:24:03.257298+0000","last_clean":"2023-06-03T20:24:03.257298+0000","last_became_active":"2023-06-03T20:23:27.259201+0000","last_became_peered":"2023-06-03T20:23:27.259201+0000","last_unstale":"2023-06-03T20:24:03.257298+0000","last_undegraded":"2023-06-03T20:24:03.257298+0000","last_fullsized":"2023-06-03T20:24:03.257298+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":33,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-06-03T20:24:03.262428+0000","last_change":"2023-06-03T20:21:01.265588+0000","last_active":"2023-06-03T20:24:03.262428+0000","last_peered":"2023-06-03T20:24:03.262428+0000","last_clean":"2023-06-03T20:24:03.262428+0000","last_became_active":"2023-06-03T20:21:01.265292+0000","last_became_peered":"2023-06-03T20:21:01.265292+0000","last_unstale":"2023-06-03T20:24:03.262428+0000","last_undegraded":"2023-06-03T20:24:03.262428+0000","last_fullsized":"2023-06-03T20:24:03.262428+0000","mapping_epoch":235,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":236,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T01:29:11.398318+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,0],"acting":[1,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"77'23","reported_seq":188,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-06-03T20:24:03.252210+0000","last_change":"2023-06-03T20:24:02.280274+0000","last_active":"2023-06-03T20:24:03.252210+0000","last_peered":"2023-06-03T20:24:03.252210+0000","last_clean":"2023-06-03T20:24:03.252210+0000","last_became_active":"2023-06-03T20:24:02.264914+0000","last_became_peered":"2023-06-03T20:24:02.264914+0000","last_unstale":"2023-06-03T20:24:03.252210+0000","last_undegraded":"2023-06-03T20:24:03.252210+0000","last_fullsized":"2023-06-03T20:24:03.252210+0000","mapping_epoch":254,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":255,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":23,"log_dups_size":0,"ondisk_log_size":23,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T07:00:35.888220+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4716,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":23,"num_write_kb":20,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4716,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":164,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-06-03T20:24:03.251850+0000","last_change":"2023-06-03T20:24:02.254850+0000","last_active":"2023-06-03T20:24:03.251850+0000","last_peered":"2023-06-03T20:24:03.251850+0000","last_clean":"2023-06-03T20:24:03.251850+0000","last_became_active":"2023-06-03T20:24:02.254409+0000","last_became_peered":"2023-06-03T20:24:02.254409+0000","last_unstale":"2023-06-03T20:24:03.251850+0000","last_undegraded":"2023-06-03T20:24:03.251850+0000","last_fullsized":"2023-06-03T20:24:03.251850+0000","mapping_epoch":254,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":255,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T19:49:53.393121+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"60'2","reported_seq":256,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-06-03T20:24:03.256964+0000","last_change":"2023-06-03T20:23:27.258749+0000","last_active":"2023-06-03T20:24:03.256964+0000","last_peered":"2023-06-03T20:24:03.256964+0000","last_clean":"2023-06-03T20:24:03.256964+0000","last_became_active":"2023-06-03T20:23:27.256899+0000","last_became_peered":"2023-06-03T20:23:27.256899+0000","last_unstale":"2023-06-03T20:24:03.256964+0000","last_undegraded":"2023-06-03T20:24:03.256964+0000","last_fullsized":"2023-06-03T20:24:03.256964+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":249,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-06-03T20:24:03.254829+0000","last_change":"2023-06-03T20:21:01.256612+0000","last_active":"2023-06-03T20:24:03.254829+0000","last_peered":"2023-06-03T20:24:03.254829+0000","last_clean":"2023-06-03T20:24:03.254829+0000","last_became_active":"2023-06-03T20:21:01.256243+0000","last_became_peered":"2023-06-03T20:21:01.256243+0000","last_unstale":"2023-06-03T20:24:03.254829+0000","last_undegraded":"2023-06-03T20:24:03.254829+0000","last_fullsized":"2023-06-03T20:24:03.254829+0000","mapping_epoch":235,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":236,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T05:24:47.054342+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":221,"reported_epoch":257,"state":"active+clean","last_fresh":"2023-06-03T20:24:14.107942+0000","last_change":"2023-06-03T19:56:54.507479+0000","last_active":"2023-06-03T20:24:14.107942+0000","last_peered":"2023-06-03T20:24:14.107942+0000","last_clean":"2023-06-03T20:24:14.107942+0000","last_became_active":"2023-06-03T19:56:54.507149+0000","last_became_peered":"2023-06-03T19:56:54.507149+0000","last_unstale":"2023-06-03T20:24:14.107942+0000","last_undegraded":"2023-06-03T20:24:14.107942+0000","last_fullsized":"2023-06-03T20:24:14.107942+0000","mapping_epoch":119,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T20:22:36.676317+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":123,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-06-03T20:24:03.257529+0000","last_change":"2023-06-03T20:17:22.832844+0000","last_active":"2023-06-03T20:24:03.257529+0000","last_peered":"2023-06-03T20:24:03.257529+0000","last_clean":"2023-06-03T20:24:03.257529+0000","last_became_active":"2023-06-03T20:17:22.832473+0000","last_became_peered":"2023-06-03T20:17:22.832473+0000","last_unstale":"2023-06-03T20:24:03.257529+0000","last_undegraded":"2023-06-03T20:24:03.257529+0000","last_fullsized":"2023-06-03T20:24:03.257529+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,3],"acting":[7,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":256,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-06-03T20:24:03.256560+0000","last_change":"2023-06-03T20:24:02.267531+0000","last_active":"2023-06-03T20:24:03.256560+0000","last_peered":"2023-06-03T20:24:03.256560+0000","last_clean":"2023-06-03T20:24:03.256560+0000","last_became_active":"2023-06-03T20:24:02.267159+0000","last_became_peered":"2023-06-03T20:24:02.267159+0000","last_unstale":"2023-06-03T20:24:03.256560+0000","last_undegraded":"2023-06-03T20:24:03.256560+0000","last_fullsized":"2023-06-03T20:24:03.256560+0000","mapping_epoch":254,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":255,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":221,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-06-03T20:24:03.262428+0000","last_change":"2023-06-03T19:56:54.510289+0000","last_active":"2023-06-03T20:24:03.262428+0000","last_peered":"2023-06-03T20:24:03.262428+0000","last_clean":"2023-06-03T20:24:03.262428+0000","last_became_active":"2023-06-03T19:56:54.509905+0000","last_became_peered":"2023-06-03T19:56:54.509905+0000","last_unstale":"2023-06-03T20:24:03.262428+0000","last_undegraded":"2023-06-03T20:24:03.262428+0000","last_fullsized":"2023-06-03T20:24:03.262428+0000","mapping_epoch":119,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T02:54:26.792401+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"60'8","reported_seq":10,"reported_epoch":256,"state":"active+recovery_wait+degraded","last_fresh":"2023-06-03T20:24:03.835364+0000","last_change":"2023-06-03T20:24:02.260404+0000","last_active":"2023-06-03T20:24:03.835364+0000","last_peered":"2023-06-03T20:24:03.835364+0000","last_clean":"2023-06-03T20:23:44.176828+0000","last_became_active":"2023-06-03T20:24:02.260105+0000","last_became_peered":"2023-06-03T20:24:02.260105+0000","last_unstale":"2023-06-03T20:24:03.835364+0000","last_undegraded":"2023-06-03T20:24:02.252731+0000","last_fullsized":"2023-06-03T20:24:03.835364+0000","mapping_epoch":254,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":213,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":4,"num_objects_missing":4,"num_objects_degraded":4,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":["0","3","5"],"object_location_counts":[{"shards":"0,3,5","objects":4}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":249,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-06-03T20:24:03.255543+0000","last_change":"2023-06-03T20:21:01.256880+0000","last_active":"2023-06-03T20:24:03.255543+0000","last_peered":"2023-06-03T20:24:03.255543+0000","last_clean":"2023-06-03T20:24:03.255543+0000","last_became_active":"2023-06-03T20:21:01.256478+0000","last_became_peered":"2023-06-03T20:21:01.256478+0000","last_unstale":"2023-06-03T20:24:03.255543+0000","last_undegraded":"2023-06-03T20:24:03.255543+0000","last_fullsized":"2023-06-03T20:24:03.255543+0000","mapping_epoch":235,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":236,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T01:09:33.462743+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"13.1","version":"257'9406","reported_seq":9441,"reported_epoch":257,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-06-03T20:24:26.320681+0000","last_change":"2023-06-03T20:23:28.776292+0000","last_active":"2023-06-03T20:24:26.320681+0000","last_peered":"2023-06-03T20:24:26.320681+0000","last_clean":"2023-06-03T20:21:29.277668+0000","last_became_active":"2023-06-03T20:23:28.755432+0000","last_became_peered":"2023-06-03T20:23:28.755432+0000","last_unstale":"2023-06-03T20:24:26.320681+0000","last_undegraded":"2023-06-03T20:23:28.456733+0000","last_fullsized":"2023-06-03T20:23:28.234041+0000","mapping_epoch":251,"log_start":"0'0","ondisk_log_start":"0'0","created":242,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T20:21:29.277668+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T20:21:29.277668+0000","last_clean_scrub_stamp":"2023-06-03T20:21:29.277668+0000","objects_scrubbed":0,"log_size":9406,"log_dups_size":0,"ondisk_log_size":9406,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T00:53:35.555868+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":332267520,"num_objects":5070,"num_object_clones":0,"num_object_copies":15210,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":7236,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5070,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9406,"num_write_kb":463232,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,7],"acting":[2,7],"avail_no_missing":["2","7"],"object_location_counts":[{"shards":"2,7","objects":-2067}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.f","version":"60'1","reported_seq":255,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-06-03T20:24:03.256760+0000","last_change":"2023-06-03T20:23:27.263667+0000","last_active":"2023-06-03T20:24:03.256760+0000","last_peered":"2023-06-03T20:24:03.256760+0000","last_clean":"2023-06-03T20:24:03.256760+0000","last_became_active":"2023-06-03T20:23:27.263110+0000","last_became_peered":"2023-06-03T20:23:27.263110+0000","last_unstale":"2023-06-03T20:24:03.256760+0000","last_undegraded":"2023-06-03T20:24:03.256760+0000","last_fullsized":"2023-06-03T20:24:03.256760+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":207,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-06-03T20:24:03.256459+0000","last_change":"2023-06-03T20:23:28.238989+0000","last_active":"2023-06-03T20:24:03.256459+0000","last_peered":"2023-06-03T20:24:03.256459+0000","last_clean":"2023-06-03T20:24:03.256459+0000","last_became_active":"2023-06-03T20:23:28.238533+0000","last_became_peered":"2023-06-03T20:23:28.238533+0000","last_unstale":"2023-06-03T20:24:03.256459+0000","last_undegraded":"2023-06-03T20:24:03.256459+0000","last_fullsized":"2023-06-03T20:24:03.256459+0000","mapping_epoch":251,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":252,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,4],"acting":[7,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"13.0","version":"257'9563","reported_seq":9586,"reported_epoch":257,"state":"active+clean","last_fresh":"2023-06-03T20:24:28.710905+0000","last_change":"2023-06-03T20:21:30.309649+0000","last_active":"2023-06-03T20:24:28.710905+0000","last_peered":"2023-06-03T20:24:28.710905+0000","last_clean":"2023-06-03T20:24:28.710905+0000","last_became_active":"2023-06-03T20:21:30.309118+0000","last_became_peered":"2023-06-03T20:21:30.309118+0000","last_unstale":"2023-06-03T20:24:28.710905+0000","last_undegraded":"2023-06-03T20:24:28.710905+0000","last_fullsized":"2023-06-03T20:24:28.710905+0000","mapping_epoch":242,"log_start":"245'1200","ondisk_log_start":"245'1200","created":242,"last_epoch_clean":243,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T20:21:29.277668+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T20:21:29.277668+0000","last_clean_scrub_stamp":"2023-06-03T20:21:29.277668+0000","objects_scrubbed":0,"log_size":8363,"log_dups_size":0,"ondisk_log_size":8363,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":334036992,"num_objects":5097,"num_object_clones":0,"num_object_copies":15291,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5097,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9563,"num_write_kb":469120,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,2],"acting":[7,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.e","version":"60'2","reported_seq":284,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-06-03T20:24:03.257075+0000","last_change":"2023-06-03T20:23:28.606751+0000","last_active":"2023-06-03T20:24:03.257075+0000","last_peered":"2023-06-03T20:24:03.257075+0000","last_clean":"2023-06-03T20:24:03.257075+0000","last_became_active":"2023-06-03T20:23:28.605968+0000","last_became_peered":"2023-06-03T20:23:28.605968+0000","last_unstale":"2023-06-03T20:24:03.257075+0000","last_undegraded":"2023-06-03T20:24:03.257075+0000","last_fullsized":"2023-06-03T20:24:03.257075+0000","mapping_epoch":251,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":252,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":193,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-06-03T20:24:03.252088+0000","last_change":"2023-06-03T20:23:28.249694+0000","last_active":"2023-06-03T20:24:03.252088+0000","last_peered":"2023-06-03T20:24:03.252088+0000","last_clean":"2023-06-03T20:24:03.252088+0000","last_became_active":"2023-06-03T20:23:28.249388+0000","last_became_peered":"2023-06-03T20:23:28.249388+0000","last_unstale":"2023-06-03T20:24:03.252088+0000","last_undegraded":"2023-06-03T20:24:03.252088+0000","last_fullsized":"2023-06-03T20:24:03.252088+0000","mapping_epoch":251,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":252,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T05:44:01.835586+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"13.3","version":"257'9282","reported_seq":9312,"reported_epoch":257,"state":"active+undersized+degraded+remapped+backfill_wait","last_fresh":"2023-06-03T20:24:27.477176+0000","last_change":"2023-06-03T20:23:28.597169+0000","last_active":"2023-06-03T20:24:27.477176+0000","last_peered":"2023-06-03T20:24:27.477176+0000","last_clean":"2023-06-03T20:21:29.277668+0000","last_became_active":"2023-06-03T20:23:28.590834+0000","last_became_peered":"2023-06-03T20:23:28.590834+0000","last_unstale":"2023-06-03T20:24:27.477176+0000","last_undegraded":"2023-06-03T20:23:28.282286+0000","last_fullsized":"2023-06-03T20:23:28.228267+0000","mapping_epoch":251,"log_start":"245'100","ondisk_log_start":"245'100","created":242,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T20:21:29.277668+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T20:21:29.277668+0000","last_clean_scrub_stamp":"2023-06-03T20:21:29.277668+0000","objects_scrubbed":0,"log_size":9182,"log_dups_size":0,"ondisk_log_size":9182,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T01:44:15.805543+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":330694656,"num_objects":5046,"num_object_clones":0,"num_object_copies":15138,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5046,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5046,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9282,"num_write_kb":458496,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,3],"avail_no_missing":["0","3"],"object_location_counts":[{"shards":"0,3","objects":5046}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.d","version":"60'1","reported_seq":227,"reported_epoch":256,"state":"active+recovery_wait+degraded","last_fresh":"2023-06-03T20:24:03.256904+0000","last_change":"2023-06-03T20:24:02.268778+0000","last_active":"2023-06-03T20:24:03.256904+0000","last_peered":"2023-06-03T20:24:03.256904+0000","last_clean":"2023-06-03T20:23:44.178265+0000","last_became_active":"2023-06-03T20:24:02.268002+0000","last_became_peered":"2023-06-03T20:24:02.268002+0000","last_unstale":"2023-06-03T20:24:03.256904+0000","last_undegraded":"2023-06-03T20:24:02.264537+0000","last_fullsized":"2023-06-03T20:24:03.256904+0000","mapping_epoch":254,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":213,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":70,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":["7","3","5"],"object_location_counts":[{"shards":"3,5,7","objects":1}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":33,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-06-03T20:24:03.262514+0000","last_change":"2023-06-03T20:21:01.261609+0000","last_active":"2023-06-03T20:24:03.262514+0000","last_peered":"2023-06-03T20:24:03.262514+0000","last_clean":"2023-06-03T20:24:03.262514+0000","last_became_active":"2023-06-03T20:21:01.261293+0000","last_became_peered":"2023-06-03T20:21:01.261293+0000","last_unstale":"2023-06-03T20:24:03.262514+0000","last_undegraded":"2023-06-03T20:24:03.262514+0000","last_fullsized":"2023-06-03T20:24:03.262514+0000","mapping_epoch":235,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":236,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T06:20:39.958989+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,5],"acting":[1,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"13.2","version":"257'9454","reported_seq":9485,"reported_epoch":257,"state":"active+undersized+degraded+remapped+backfill_wait","last_fresh":"2023-06-03T20:24:29.201135+0000","last_change":"2023-06-03T20:23:28.745629+0000","last_active":"2023-06-03T20:24:29.201135+0000","last_peered":"2023-06-03T20:24:29.201135+0000","last_clean":"2023-06-03T20:21:29.277668+0000","last_became_active":"2023-06-03T20:23:28.732875+0000","last_became_peered":"2023-06-03T20:23:28.732875+0000","last_unstale":"2023-06-03T20:24:29.201135+0000","last_undegraded":"2023-06-03T20:23:28.270285+0000","last_fullsized":"2023-06-03T20:23:28.227831+0000","mapping_epoch":251,"log_start":"245'1300","ondisk_log_start":"245'1300","created":242,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T20:21:29.277668+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T20:21:29.277668+0000","last_clean_scrub_stamp":"2023-06-03T20:21:29.277668+0000","objects_scrubbed":0,"log_size":8154,"log_dups_size":0,"ondisk_log_size":8154,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T22:07:33.536331+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":324665344,"num_objects":4954,"num_object_clones":0,"num_object_copies":14862,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4954,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4954,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9454,"num_write_kb":461056,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,4],"acting":[3,5],"avail_no_missing":["3","5"],"object_location_counts":[{"shards":"3,5","objects":4954}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":247,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-06-03T20:24:09.226938+0000","last_change":"2023-06-03T20:24:02.261252+0000","last_active":"2023-06-03T20:24:09.226938+0000","last_peered":"2023-06-03T20:24:09.226938+0000","last_clean":"2023-06-03T20:24:09.226938+0000","last_became_active":"2023-06-03T20:24:02.260824+0000","last_became_peered":"2023-06-03T20:24:02.260824+0000","last_unstale":"2023-06-03T20:24:09.226938+0000","last_undegraded":"2023-06-03T20:24:09.226938+0000","last_fullsized":"2023-06-03T20:24:09.226938+0000","mapping_epoch":254,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":255,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T20:00:53.584753+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":284,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-06-03T20:24:03.262738+0000","last_change":"2023-06-03T20:23:28.283360+0000","last_active":"2023-06-03T20:24:03.262738+0000","last_peered":"2023-06-03T20:24:03.262738+0000","last_clean":"2023-06-03T20:24:03.262738+0000","last_became_active":"2023-06-03T20:23:28.283035+0000","last_became_peered":"2023-06-03T20:23:28.283035+0000","last_unstale":"2023-06-03T20:24:03.262738+0000","last_undegraded":"2023-06-03T20:24:03.262738+0000","last_fullsized":"2023-06-03T20:24:03.262738+0000","mapping_epoch":251,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":252,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T00:22:59.446515+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"13.5","version":"257'9439","reported_seq":9519,"reported_epoch":257,"state":"active+undersized+degraded+remapped+backfill_wait","last_fresh":"2023-06-03T20:24:27.424531+0000","last_change":"2023-06-03T20:23:28.788572+0000","last_active":"2023-06-03T20:24:27.424531+0000","last_peered":"2023-06-03T20:24:27.424531+0000","last_clean":"2023-06-03T20:21:29.277668+0000","last_became_active":"2023-06-03T20:23:28.569129+0000","last_became_peered":"2023-06-03T20:23:28.569129+0000","last_unstale":"2023-06-03T20:24:27.424531+0000","last_undegraded":"2023-06-03T20:23:28.282211+0000","last_fullsized":"2023-06-03T20:23:28.230455+0000","mapping_epoch":251,"log_start":"245'200","ondisk_log_start":"245'200","created":242,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T20:21:29.277668+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T20:21:29.277668+0000","last_clean_scrub_stamp":"2023-06-03T20:21:29.277668+0000","objects_scrubbed":0,"log_size":9239,"log_dups_size":0,"ondisk_log_size":9239,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T04:38:04.553910+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":323944448,"num_objects":4943,"num_object_clones":0,"num_object_copies":14829,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4929,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4943,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9439,"num_write_kb":460224,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":25,"num_bytes_recovered":1638400,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,5],"acting":[0,5],"avail_no_missing":["0","5"],"object_location_counts":[{"shards":"0,5","objects":4943}],"blocked_by":[],"up_primary":4,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.b","version":"60'2","reported_seq":248,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-06-03T20:24:03.255064+0000","last_change":"2023-06-03T20:23:27.254179+0000","last_active":"2023-06-03T20:24:03.255064+0000","last_peered":"2023-06-03T20:24:03.255064+0000","last_clean":"2023-06-03T20:24:03.255064+0000","last_became_active":"2023-06-03T20:23:27.253791+0000","last_became_peered":"2023-06-03T20:23:27.253791+0000","last_unstale":"2023-06-03T20:24:03.255064+0000","last_undegraded":"2023-06-03T20:24:03.255064+0000","last_fullsized":"2023-06-03T20:24:03.255064+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T00:31:27.336760+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":169,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-06-03T20:24:03.252104+0000","last_change":"2023-06-03T20:24:02.255551+0000","last_active":"2023-06-03T20:24:03.252104+0000","last_peered":"2023-06-03T20:24:03.252104+0000","last_clean":"2023-06-03T20:24:03.252104+0000","last_became_active":"2023-06-03T20:24:02.255209+0000","last_became_peered":"2023-06-03T20:24:02.255209+0000","last_unstale":"2023-06-03T20:24:03.252104+0000","last_undegraded":"2023-06-03T20:24:03.252104+0000","last_fullsized":"2023-06-03T20:24:03.252104+0000","mapping_epoch":254,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":255,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T20:05:02.741217+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"13.4","version":"257'9347","reported_seq":9372,"reported_epoch":257,"state":"active+clean","last_fresh":"2023-06-03T20:24:26.340188+0000","last_change":"2023-06-03T20:21:30.618079+0000","last_active":"2023-06-03T20:24:26.340188+0000","last_peered":"2023-06-03T20:24:26.340188+0000","last_clean":"2023-06-03T20:24:26.340188+0000","last_became_active":"2023-06-03T20:21:30.617468+0000","last_became_peered":"2023-06-03T20:21:30.617468+0000","last_unstale":"2023-06-03T20:24:26.340188+0000","last_undegraded":"2023-06-03T20:24:26.340188+0000","last_fullsized":"2023-06-03T20:24:26.340188+0000","mapping_epoch":242,"log_start":"0'0","ondisk_log_start":"0'0","created":242,"last_epoch_clean":243,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T20:21:29.277668+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T20:21:29.277668+0000","last_clean_scrub_stamp":"2023-06-03T20:21:29.277668+0000","objects_scrubbed":0,"log_size":9347,"log_dups_size":0,"ondisk_log_size":9347,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T22:31:44.994683+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":330235904,"num_objects":5039,"num_object_clones":0,"num_object_copies":15117,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5039,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9347,"num_write_kb":460352,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,5],"acting":[2,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":12,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-06-03T20:24:03.620377+0000","last_change":"2023-06-03T20:24:02.254172+0000","last_active":"2023-06-03T20:24:03.620377+0000","last_peered":"2023-06-03T20:24:03.620377+0000","last_clean":"2023-06-03T20:24:03.620377+0000","last_became_active":"2023-06-03T20:24:02.253797+0000","last_became_peered":"2023-06-03T20:24:02.253797+0000","last_unstale":"2023-06-03T20:24:03.620377+0000","last_undegraded":"2023-06-03T20:24:03.620377+0000","last_fullsized":"2023-06-03T20:24:03.620377+0000","mapping_epoch":254,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":255,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":174,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-06-03T20:24:03.252450+0000","last_change":"2023-06-03T20:23:28.291250+0000","last_active":"2023-06-03T20:24:03.252450+0000","last_peered":"2023-06-03T20:24:03.252450+0000","last_clean":"2023-06-03T20:24:03.252450+0000","last_became_active":"2023-06-03T20:23:28.290120+0000","last_became_peered":"2023-06-03T20:23:28.290120+0000","last_unstale":"2023-06-03T20:24:03.252450+0000","last_undegraded":"2023-06-03T20:24:03.252450+0000","last_fullsized":"2023-06-03T20:24:03.252450+0000","mapping_epoch":251,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":252,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T22:37:56.033327+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"13.7","version":"257'9586","reported_seq":9619,"reported_epoch":257,"state":"active+undersized+degraded+remapped+backfill_wait","last_fresh":"2023-06-03T20:24:28.691287+0000","last_change":"2023-06-03T20:24:02.454906+0000","last_active":"2023-06-03T20:24:28.691287+0000","last_peered":"2023-06-03T20:24:28.691287+0000","last_clean":"2023-06-03T20:21:29.277668+0000","last_became_active":"2023-06-03T20:24:02.436714+0000","last_became_peered":"2023-06-03T20:24:02.436714+0000","last_unstale":"2023-06-03T20:24:28.691287+0000","last_undegraded":"2023-06-03T20:24:02.301992+0000","last_fullsized":"2023-06-03T20:24:02.263673+0000","mapping_epoch":254,"log_start":"245'1200","ondisk_log_start":"245'1200","created":242,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T20:21:29.277668+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T20:21:29.277668+0000","last_clean_scrub_stamp":"2023-06-03T20:21:29.277668+0000","objects_scrubbed":0,"log_size":8386,"log_dups_size":0,"ondisk_log_size":8386,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":333709312,"num_objects":5092,"num_object_clones":0,"num_object_copies":15276,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5092,"num_objects_misplaced":5092,"num_objects_unfound":0,"num_objects_dirty":5092,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9586,"num_write_kb":469696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,4],"acting":[7,5],"avail_no_missing":["7","5"],"object_location_counts":[{"shards":"5,7","objects":5092}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":159,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-06-03T20:24:03.251995+0000","last_change":"2023-06-03T20:23:27.276693+0000","last_active":"2023-06-03T20:24:03.251995+0000","last_peered":"2023-06-03T20:24:03.251995+0000","last_clean":"2023-06-03T20:24:03.251995+0000","last_became_active":"2023-06-03T20:23:27.276305+0000","last_became_peered":"2023-06-03T20:23:27.276305+0000","last_unstale":"2023-06-03T20:24:03.251995+0000","last_undegraded":"2023-06-03T20:24:03.251995+0000","last_fullsized":"2023-06-03T20:24:03.251995+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T20:34:50.944936+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":155,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-06-03T20:24:03.252759+0000","last_change":"2023-06-03T20:23:28.722550+0000","last_active":"2023-06-03T20:24:03.252759+0000","last_peered":"2023-06-03T20:24:03.252759+0000","last_clean":"2023-06-03T20:24:03.252759+0000","last_became_active":"2023-06-03T20:23:28.722072+0000","last_became_peered":"2023-06-03T20:23:28.722072+0000","last_unstale":"2023-06-03T20:24:03.252759+0000","last_undegraded":"2023-06-03T20:24:03.252759+0000","last_fullsized":"2023-06-03T20:24:03.252759+0000","mapping_epoch":251,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":252,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T21:00:13.021915+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"13.6","version":"257'9276","reported_seq":9304,"reported_epoch":257,"state":"active+undersized+degraded+remapped+backfill_wait","last_fresh":"2023-06-03T20:24:29.166637+0000","last_change":"2023-06-03T20:23:28.836420+0000","last_active":"2023-06-03T20:24:29.166637+0000","last_peered":"2023-06-03T20:24:29.166637+0000","last_clean":"2023-06-03T20:21:29.277668+0000","last_became_active":"2023-06-03T20:23:28.828659+0000","last_became_peered":"2023-06-03T20:23:28.828659+0000","last_unstale":"2023-06-03T20:24:29.166637+0000","last_undegraded":"2023-06-03T20:23:28.273568+0000","last_fullsized":"2023-06-03T20:23:28.231987+0000","mapping_epoch":251,"log_start":"245'1100","ondisk_log_start":"245'1100","created":242,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T20:21:29.277668+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T20:21:29.277668+0000","last_clean_scrub_stamp":"2023-06-03T20:21:29.277668+0000","objects_scrubbed":0,"log_size":8176,"log_dups_size":0,"ondisk_log_size":8176,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T00:44:56.298777+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":321781760,"num_objects":4910,"num_object_clones":0,"num_object_copies":14730,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4910,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4910,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9276,"num_write_kb":453952,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,3],"acting":[3,5],"avail_no_missing":["3","5"],"object_location_counts":[{"shards":"3,5","objects":4910}],"blocked_by":[],"up_primary":4,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":257,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-06-03T20:24:03.254371+0000","last_change":"2023-06-03T20:21:01.256958+0000","last_active":"2023-06-03T20:24:03.254371+0000","last_peered":"2023-06-03T20:24:03.254371+0000","last_clean":"2023-06-03T20:24:03.254371+0000","last_became_active":"2023-06-03T20:21:01.255452+0000","last_became_peered":"2023-06-03T20:21:01.255452+0000","last_unstale":"2023-06-03T20:24:03.254371+0000","last_undegraded":"2023-06-03T20:24:03.254371+0000","last_fullsized":"2023-06-03T20:24:03.254371+0000","mapping_epoch":235,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":236,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T01:56:59.777763+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":44,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-06-03T20:24:03.263146+0000","last_change":"2023-06-03T20:23:28.250884+0000","last_active":"2023-06-03T20:24:03.263146+0000","last_peered":"2023-06-03T20:24:03.263146+0000","last_clean":"2023-06-03T20:24:03.263146+0000","last_became_active":"2023-06-03T20:23:28.243793+0000","last_became_peered":"2023-06-03T20:23:28.243793+0000","last_unstale":"2023-06-03T20:24:03.263146+0000","last_undegraded":"2023-06-03T20:24:03.263146+0000","last_fullsized":"2023-06-03T20:24:03.263146+0000","mapping_epoch":251,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":252,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T03:12:27.946002+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"13.b","version":"257'9192","reported_seq":9217,"reported_epoch":257,"state":"active+clean","last_fresh":"2023-06-03T20:24:29.193569+0000","last_change":"2023-06-03T20:21:30.617377+0000","last_active":"2023-06-03T20:24:29.193569+0000","last_peered":"2023-06-03T20:24:29.193569+0000","last_clean":"2023-06-03T20:24:29.193569+0000","last_became_active":"2023-06-03T20:21:30.616650+0000","last_became_peered":"2023-06-03T20:21:30.616650+0000","last_unstale":"2023-06-03T20:24:29.193569+0000","last_undegraded":"2023-06-03T20:24:29.193569+0000","last_fullsized":"2023-06-03T20:24:29.193569+0000","mapping_epoch":242,"log_start":"245'1000","ondisk_log_start":"245'1000","created":242,"last_epoch_clean":243,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T20:21:29.277668+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T20:21:29.277668+0000","last_clean_scrub_stamp":"2023-06-03T20:21:29.277668+0000","objects_scrubbed":0,"log_size":8192,"log_dups_size":0,"ondisk_log_size":8192,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T20:35:22.419539+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":315097088,"num_objects":4808,"num_object_clones":0,"num_object_copies":14424,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4808,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9192,"num_write_kb":448000,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,2],"acting":[3,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":33,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-06-03T20:24:03.263049+0000","last_change":"2023-06-03T20:21:01.258182+0000","last_active":"2023-06-03T20:24:03.263049+0000","last_peered":"2023-06-03T20:24:03.263049+0000","last_clean":"2023-06-03T20:24:03.263049+0000","last_became_active":"2023-06-03T20:21:01.254049+0000","last_became_peered":"2023-06-03T20:21:01.254049+0000","last_unstale":"2023-06-03T20:24:03.263049+0000","last_undegraded":"2023-06-03T20:24:03.263049+0000","last_fullsized":"2023-06-03T20:24:03.263049+0000","mapping_epoch":235,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":236,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:32:20.692043+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:32:20.692043+0000","last_clean_scrub_stamp":"2023-06-03T19:32:20.692043+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T05:16:30.078473+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":142,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-06-03T20:24:03.255241+0000","last_change":"2023-06-03T20:17:22.834746+0000","last_active":"2023-06-03T20:24:03.255241+0000","last_peered":"2023-06-03T20:24:03.255241+0000","last_clean":"2023-06-03T20:24:03.255241+0000","last_became_active":"2023-06-03T20:17:22.834448+0000","last_became_peered":"2023-06-03T20:17:22.834448+0000","last_unstale":"2023-06-03T20:24:03.255241+0000","last_undegraded":"2023-06-03T20:24:03.255241+0000","last_fullsized":"2023-06-03T20:24:03.255241+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T21:16:31.167063+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,2],"acting":[3,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":44,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-06-03T20:24:03.262331+0000","last_change":"2023-06-03T20:23:28.254434+0000","last_active":"2023-06-03T20:24:03.262331+0000","last_peered":"2023-06-03T20:24:03.262331+0000","last_clean":"2023-06-03T20:24:03.262331+0000","last_became_active":"2023-06-03T20:23:28.254012+0000","last_became_peered":"2023-06-03T20:23:28.254012+0000","last_unstale":"2023-06-03T20:24:03.262331+0000","last_undegraded":"2023-06-03T20:24:03.262331+0000","last_fullsized":"2023-06-03T20:24:03.262331+0000","mapping_epoch":251,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":252,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T06:22:09.540298+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"13.d","version":"257'9551","reported_seq":9584,"reported_epoch":257,"state":"active+undersized+degraded+remapped+backfill_wait","last_fresh":"2023-06-03T20:24:27.257375+0000","last_change":"2023-06-03T20:23:28.629277+0000","last_active":"2023-06-03T20:24:27.257375+0000","last_peered":"2023-06-03T20:24:27.257375+0000","last_clean":"2023-06-03T20:21:29.277668+0000","last_became_active":"2023-06-03T20:23:28.560153+0000","last_became_peered":"2023-06-03T20:23:28.560153+0000","last_unstale":"2023-06-03T20:24:27.257375+0000","last_undegraded":"2023-06-03T20:23:28.629147+0000","last_fullsized":"2023-06-03T20:23:28.238064+0000","mapping_epoch":251,"log_start":"245'2700","ondisk_log_start":"245'2700","created":242,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T20:21:29.277668+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T20:21:29.277668+0000","last_clean_scrub_stamp":"2023-06-03T20:21:29.277668+0000","objects_scrubbed":0,"log_size":6851,"log_dups_size":0,"ondisk_log_size":6851,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T03:14:56.090099+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":331874328,"num_objects":5065,"num_object_clones":0,"num_object_copies":15195,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5065,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5065,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":9551,"num_write_kb":467649,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,7],"acting":[1,7],"avail_no_missing":["1","7"],"object_location_counts":[{"shards":"1,7","objects":5065}],"blocked_by":[],"up_primary":4,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":33,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-06-03T20:24:03.262599+0000","last_change":"2023-06-03T20:21:01.262334+0000","last_active":"2023-06-03T20:24:03.262599+0000","last_peered":"2023-06-03T20:24:03.262599+0000","last_clean":"2023-06-03T20:24:03.262599+0000","last_became_active":"2023-06-03T20:21:01.262013+0000","last_became_peered":"2023-06-03T20:21:01.262013+0000","last_unstale":"2023-06-03T20:24:03.262599+0000","last_undegraded":"2023-06-03T20:24:03.262599+0000","last_fullsized":"2023-06-03T20:24:03.262599+0000","mapping_epoch":235,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":236,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:32:20.692043+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:32:20.692043+0000","last_clean_scrub_stamp":"2023-06-03T19:32:20.692043+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T03:19:31.249552+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":139,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-06-03T20:24:03.252863+0000","last_change":"2023-06-03T20:24:02.258311+0000","last_active":"2023-06-03T20:24:03.252863+0000","last_peered":"2023-06-03T20:24:03.252863+0000","last_clean":"2023-06-03T20:24:03.252863+0000","last_became_active":"2023-06-03T20:24:02.257676+0000","last_became_peered":"2023-06-03T20:24:02.257676+0000","last_unstale":"2023-06-03T20:24:03.252863+0000","last_undegraded":"2023-06-03T20:24:03.252863+0000","last_fullsized":"2023-06-03T20:24:03.252863+0000","mapping_epoch":254,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":255,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T01:47:46.383038+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":33,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-06-03T20:24:03.263146+0000","last_change":"2023-06-03T20:21:01.259331+0000","last_active":"2023-06-03T20:24:03.263146+0000","last_peered":"2023-06-03T20:24:03.263146+0000","last_clean":"2023-06-03T20:24:03.263146+0000","last_became_active":"2023-06-03T20:21:01.258836+0000","last_became_peered":"2023-06-03T20:21:01.258836+0000","last_unstale":"2023-06-03T20:24:03.263146+0000","last_undegraded":"2023-06-03T20:24:03.263146+0000","last_fullsized":"2023-06-03T20:24:03.263146+0000","mapping_epoch":235,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":236,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T01:29:19.471600+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"13.e","version":"257'9420","reported_seq":11360,"reported_epoch":257,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2023-06-03T20:24:28.734598+0000","last_change":"2023-06-03T20:23:28.812784+0000","last_active":"2023-06-03T20:24:28.734598+0000","last_peered":"2023-06-03T20:24:28.734598+0000","last_clean":"2023-06-03T20:21:29.277668+0000","last_became_active":"2023-06-03T20:23:28.735246+0000","last_became_peered":"2023-06-03T20:23:28.735246+0000","last_unstale":"2023-06-03T20:24:28.734598+0000","last_undegraded":"2023-06-03T20:23:28.593703+0000","last_fullsized":"2023-06-03T20:23:28.232492+0000","mapping_epoch":251,"log_start":"245'1000","ondisk_log_start":"245'1000","created":242,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T20:21:29.277668+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T20:21:29.277668+0000","last_clean_scrub_stamp":"2023-06-03T20:21:29.277668+0000","objects_scrubbed":0,"log_size":8420,"log_dups_size":0,"ondisk_log_size":8420,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":317325312,"num_objects":4842,"num_object_clones":0,"num_object_copies":14526,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":6097,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4842,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9420,"num_write_kb":456384,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":956,"num_bytes_recovered":62455808,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,4],"acting":[7,5],"avail_no_missing":["7","5"],"object_location_counts":[{"shards":"5,7","objects":-1255}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":241,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-06-03T20:24:03.258376+0000","last_change":"2023-06-03T19:56:54.504859+0000","last_active":"2023-06-03T20:24:03.258376+0000","last_peered":"2023-06-03T20:24:03.258376+0000","last_clean":"2023-06-03T20:24:03.258376+0000","last_became_active":"2023-06-03T19:56:54.504455+0000","last_became_peered":"2023-06-03T19:56:54.504455+0000","last_unstale":"2023-06-03T20:24:03.258376+0000","last_undegraded":"2023-06-03T20:24:03.258376+0000","last_fullsized":"2023-06-03T20:24:03.258376+0000","mapping_epoch":119,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:32:20.692043+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:32:20.692043+0000","last_clean_scrub_stamp":"2023-06-03T19:32:20.692043+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T06:25:45.522100+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":42,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-06-03T20:24:03.264016+0000","last_change":"2023-06-03T20:24:02.256677+0000","last_active":"2023-06-03T20:24:03.264016+0000","last_peered":"2023-06-03T20:24:03.264016+0000","last_clean":"2023-06-03T20:24:03.264016+0000","last_became_active":"2023-06-03T20:24:02.256216+0000","last_became_peered":"2023-06-03T20:24:02.256216+0000","last_unstale":"2023-06-03T20:24:03.264016+0000","last_undegraded":"2023-06-03T20:24:03.264016+0000","last_fullsized":"2023-06-03T20:24:03.264016+0000","mapping_epoch":254,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":255,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T02:32:45.186204+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":33,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-06-03T20:24:03.263054+0000","last_change":"2023-06-03T20:21:01.261473+0000","last_active":"2023-06-03T20:24:03.263054+0000","last_peered":"2023-06-03T20:24:03.263054+0000","last_clean":"2023-06-03T20:24:03.263054+0000","last_became_active":"2023-06-03T20:21:01.261098+0000","last_became_peered":"2023-06-03T20:21:01.261098+0000","last_unstale":"2023-06-03T20:24:03.263054+0000","last_undegraded":"2023-06-03T20:24:03.263054+0000","last_fullsized":"2023-06-03T20:24:03.263054+0000","mapping_epoch":235,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":236,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T03:56:22.986549+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"13.c","version":"257'9576","reported_seq":9601,"reported_epoch":257,"state":"active+clean","last_fresh":"2023-06-03T20:24:27.380760+0000","last_change":"2023-06-03T20:21:30.617912+0000","last_active":"2023-06-03T20:24:27.380760+0000","last_peered":"2023-06-03T20:24:27.380760+0000","last_clean":"2023-06-03T20:24:27.380760+0000","last_became_active":"2023-06-03T20:21:30.617203+0000","last_became_peered":"2023-06-03T20:21:30.617203+0000","last_unstale":"2023-06-03T20:24:27.380760+0000","last_undegraded":"2023-06-03T20:24:27.380760+0000","last_fullsized":"2023-06-03T20:24:27.380760+0000","mapping_epoch":242,"log_start":"245'2700","ondisk_log_start":"245'2700","created":242,"last_epoch_clean":243,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T20:21:29.277668+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T20:21:29.277668+0000","last_clean_scrub_stamp":"2023-06-03T20:21:29.277668+0000","objects_scrubbed":0,"log_size":6876,"log_dups_size":0,"ondisk_log_size":6876,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T03:16:35.236413+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":334626816,"num_objects":5106,"num_object_clones":0,"num_object_copies":15318,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5106,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9576,"num_write_kb":469824,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,2],"acting":[1,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":126,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-06-03T20:24:03.258364+0000","last_change":"2023-06-03T20:17:22.834467+0000","last_active":"2023-06-03T20:24:03.258364+0000","last_peered":"2023-06-03T20:24:03.258364+0000","last_clean":"2023-06-03T20:24:03.258364+0000","last_became_active":"2023-06-03T20:17:22.834147+0000","last_became_peered":"2023-06-03T20:17:22.834147+0000","last_unstale":"2023-06-03T20:24:03.258364+0000","last_undegraded":"2023-06-03T20:24:03.258364+0000","last_fullsized":"2023-06-03T20:24:03.258364+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:32:20.692043+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:32:20.692043+0000","last_clean_scrub_stamp":"2023-06-03T19:32:20.692043+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T04:36:29.567051+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,7],"acting":[2,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"72'139","reported_seq":514,"reported_epoch":256,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-06-03T20:24:03.836300+0000","last_change":"2023-06-03T20:24:03.836300+0000","last_active":"2023-06-03T20:24:03.836300+0000","last_peered":"2023-06-03T20:24:03.836300+0000","last_clean":"2023-06-03T20:23:44.179074+0000","last_became_active":"2023-06-03T20:24:03.835911+0000","last_became_peered":"2023-06-03T20:24:03.835911+0000","last_unstale":"2023-06-03T20:24:03.836300+0000","last_undegraded":"2023-06-03T20:24:03.257647+0000","last_fullsized":"2023-06-03T20:24:03.257150+0000","mapping_epoch":255,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:27:28.100808+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:27:28.100808+0000","last_clean_scrub_stamp":"2023-06-03T19:27:28.100808+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1180736,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0],"avail_no_missing":["7","0","1"],"object_location_counts":[{"shards":"0,1,7","objects":2}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":243,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-06-03T20:24:03.254895+0000","last_change":"2023-06-03T20:24:02.263360+0000","last_active":"2023-06-03T20:24:03.254895+0000","last_peered":"2023-06-03T20:24:03.254895+0000","last_clean":"2023-06-03T20:24:03.254895+0000","last_became_active":"2023-06-03T20:24:02.262954+0000","last_became_peered":"2023-06-03T20:24:02.262954+0000","last_unstale":"2023-06-03T20:24:03.254895+0000","last_undegraded":"2023-06-03T20:24:03.254895+0000","last_fullsized":"2023-06-03T20:24:03.254895+0000","mapping_epoch":254,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":255,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T19:38:46.084731+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":100,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-06-03T20:24:03.252413+0000","last_change":"2023-06-03T20:23:28.514811+0000","last_active":"2023-06-03T20:24:03.252413+0000","last_peered":"2023-06-03T20:24:03.252413+0000","last_clean":"2023-06-03T20:24:03.252413+0000","last_became_active":"2023-06-03T20:23:28.512870+0000","last_became_peered":"2023-06-03T20:23:28.512870+0000","last_unstale":"2023-06-03T20:24:03.252413+0000","last_undegraded":"2023-06-03T20:24:03.252413+0000","last_fullsized":"2023-06-03T20:24:03.252413+0000","mapping_epoch":251,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":252,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T21:20:56.980045+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"13.f","version":"257'9537","reported_seq":9566,"reported_epoch":257,"state":"active+remapped+backfill_wait","last_fresh":"2023-06-03T20:24:26.346512+0000","last_change":"2023-06-03T20:24:03.767965+0000","last_active":"2023-06-03T20:24:26.346512+0000","last_peered":"2023-06-03T20:24:26.346512+0000","last_clean":"2023-06-03T20:24:01.053624+0000","last_became_active":"2023-06-03T20:24:03.745317+0000","last_became_peered":"2023-06-03T20:24:03.745317+0000","last_unstale":"2023-06-03T20:24:26.346512+0000","last_undegraded":"2023-06-03T20:24:26.346512+0000","last_fullsized":"2023-06-03T20:24:26.346512+0000","mapping_epoch":255,"log_start":"245'300","ondisk_log_start":"245'300","created":242,"last_epoch_clean":243,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T20:21:29.277668+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T20:21:29.277668+0000","last_clean_scrub_stamp":"2023-06-03T20:21:29.277668+0000","objects_scrubbed":0,"log_size":9237,"log_dups_size":0,"ondisk_log_size":9237,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T06:00:42.514105+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":331808768,"num_objects":5063,"num_object_clones":0,"num_object_copies":15189,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":5063,"num_objects_unfound":0,"num_objects_dirty":5063,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9537,"num_write_kb":467200,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[2,7,5],"avail_no_missing":["2","5","7"],"object_location_counts":[{"shards":"2,5,7","objects":5063}],"blocked_by":[],"up_primary":6,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":249,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-06-03T20:24:03.256180+0000","last_change":"2023-06-03T20:21:01.258113+0000","last_active":"2023-06-03T20:24:03.256180+0000","last_peered":"2023-06-03T20:24:03.256180+0000","last_clean":"2023-06-03T20:24:03.256180+0000","last_became_active":"2023-06-03T20:21:01.257488+0000","last_became_peered":"2023-06-03T20:21:01.257488+0000","last_unstale":"2023-06-03T20:24:03.256180+0000","last_undegraded":"2023-06-03T20:24:03.256180+0000","last_fullsized":"2023-06-03T20:24:03.256180+0000","mapping_epoch":235,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":236,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:32:20.692043+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:32:20.692043+0000","last_clean_scrub_stamp":"2023-06-03T19:32:20.692043+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":252,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-06-03T20:24:03.262680+0000","last_change":"2023-06-03T20:23:27.255164+0000","last_active":"2023-06-03T20:24:03.262680+0000","last_peered":"2023-06-03T20:24:03.262680+0000","last_clean":"2023-06-03T20:24:03.262680+0000","last_became_active":"2023-06-03T20:23:27.253932+0000","last_became_peered":"2023-06-03T20:23:27.253932+0000","last_unstale":"2023-06-03T20:24:03.262680+0000","last_undegraded":"2023-06-03T20:24:03.262680+0000","last_fullsized":"2023-06-03T20:24:03.262680+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T06:42:02.303554+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":256,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-06-03T20:24:03.262345+0000","last_change":"2023-06-03T20:21:01.250266+0000","last_active":"2023-06-03T20:24:03.262345+0000","last_peered":"2023-06-03T20:24:03.262345+0000","last_clean":"2023-06-03T20:24:03.262345+0000","last_became_active":"2023-06-03T20:21:01.249839+0000","last_became_peered":"2023-06-03T20:21:01.249839+0000","last_unstale":"2023-06-03T20:24:03.262345+0000","last_undegraded":"2023-06-03T20:24:03.262345+0000","last_fullsized":"2023-06-03T20:24:03.262345+0000","mapping_epoch":235,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":236,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T05:13:26.584607+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"13.a","version":"257'9319","reported_seq":9349,"reported_epoch":257,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-06-03T20:24:26.391163+0000","last_change":"2023-06-03T20:23:28.764466+0000","last_active":"2023-06-03T20:24:26.391163+0000","last_peered":"2023-06-03T20:24:26.391163+0000","last_clean":"2023-06-03T20:21:29.277668+0000","last_became_active":"2023-06-03T20:23:28.743460+0000","last_became_peered":"2023-06-03T20:23:28.743460+0000","last_unstale":"2023-06-03T20:24:26.391163+0000","last_undegraded":"2023-06-03T20:23:28.317352+0000","last_fullsized":"2023-06-03T20:23:28.234877+0000","mapping_epoch":251,"log_start":"0'0","ondisk_log_start":"0'0","created":242,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T20:21:29.277668+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T20:21:29.277668+0000","last_clean_scrub_stamp":"2023-06-03T20:21:29.277668+0000","objects_scrubbed":0,"log_size":9319,"log_dups_size":0,"ondisk_log_size":9319,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T07:54:17.271110+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":325124096,"num_objects":4961,"num_object_clones":0,"num_object_copies":14883,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":7140,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4961,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9319,"num_write_kb":456960,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,4],"acting":[2,1],"avail_no_missing":["2","1"],"object_location_counts":[{"shards":"1,2","objects":-2091}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":258,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-06-03T20:24:03.257162+0000","last_change":"2023-06-03T20:23:27.261511+0000","last_active":"2023-06-03T20:24:03.257162+0000","last_peered":"2023-06-03T20:24:03.257162+0000","last_clean":"2023-06-03T20:24:03.257162+0000","last_became_active":"2023-06-03T20:23:27.259622+0000","last_became_peered":"2023-06-03T20:23:27.259622+0000","last_unstale":"2023-06-03T20:24:03.257162+0000","last_undegraded":"2023-06-03T20:24:03.257162+0000","last_fullsized":"2023-06-03T20:24:03.257162+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:32:20.692043+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:32:20.692043+0000","last_clean_scrub_stamp":"2023-06-03T19:32:20.692043+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":33,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-06-03T20:24:03.262964+0000","last_change":"2023-06-03T20:21:01.252859+0000","last_active":"2023-06-03T20:24:03.262964+0000","last_peered":"2023-06-03T20:24:03.262964+0000","last_clean":"2023-06-03T20:24:03.262964+0000","last_became_active":"2023-06-03T20:21:01.251222+0000","last_became_peered":"2023-06-03T20:21:01.251222+0000","last_unstale":"2023-06-03T20:24:03.262964+0000","last_undegraded":"2023-06-03T20:24:03.262964+0000","last_fullsized":"2023-06-03T20:24:03.262964+0000","mapping_epoch":235,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":236,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T05:19:58.100226+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":208,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-06-03T20:24:03.262917+0000","last_change":"2023-06-03T19:47:06.733532+0000","last_active":"2023-06-03T20:24:03.262917+0000","last_peered":"2023-06-03T20:24:03.262917+0000","last_clean":"2023-06-03T20:24:03.262917+0000","last_became_active":"2023-06-03T19:47:06.733131+0000","last_became_peered":"2023-06-03T19:47:06.733131+0000","last_unstale":"2023-06-03T20:24:03.262917+0000","last_undegraded":"2023-06-03T20:24:03.262917+0000","last_fullsized":"2023-06-03T20:24:03.262917+0000","mapping_epoch":86,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":87,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T23:46:03.788646+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"13.9","version":"257'9537","reported_seq":9562,"reported_epoch":257,"state":"active+clean","last_fresh":"2023-06-03T20:24:27.429500+0000","last_change":"2023-06-03T20:21:30.617754+0000","last_active":"2023-06-03T20:24:27.429500+0000","last_peered":"2023-06-03T20:24:27.429500+0000","last_clean":"2023-06-03T20:24:27.429500+0000","last_became_active":"2023-06-03T20:21:30.616735+0000","last_became_peered":"2023-06-03T20:21:30.616735+0000","last_unstale":"2023-06-03T20:24:27.429500+0000","last_undegraded":"2023-06-03T20:24:27.429500+0000","last_fullsized":"2023-06-03T20:24:27.429500+0000","mapping_epoch":242,"log_start":"245'400","ondisk_log_start":"245'400","created":242,"last_epoch_clean":243,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T20:21:29.277668+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T20:21:29.277668+0000","last_clean_scrub_stamp":"2023-06-03T20:21:29.277668+0000","objects_scrubbed":0,"log_size":9137,"log_dups_size":0,"ondisk_log_size":9137,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T01:05:03.356423+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":333905920,"num_objects":5095,"num_object_clones":0,"num_object_copies":15285,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5095,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9537,"num_write_kb":468224,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.7","version":"60'2","reported_seq":218,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-06-03T20:24:03.255440+0000","last_change":"2023-06-03T19:56:54.510148+0000","last_active":"2023-06-03T20:24:03.255440+0000","last_peered":"2023-06-03T20:24:03.255440+0000","last_clean":"2023-06-03T20:24:03.255440+0000","last_became_active":"2023-06-03T19:56:54.509771+0000","last_became_peered":"2023-06-03T19:56:54.509771+0000","last_unstale":"2023-06-03T20:24:03.255440+0000","last_undegraded":"2023-06-03T20:24:03.255440+0000","last_fullsized":"2023-06-03T20:24:03.255440+0000","mapping_epoch":119,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T00:52:42.807432+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":51,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-06-03T20:24:03.263349+0000","last_change":"2023-06-03T20:24:02.257142+0000","last_active":"2023-06-03T20:24:03.263349+0000","last_peered":"2023-06-03T20:24:03.263349+0000","last_clean":"2023-06-03T20:24:03.263349+0000","last_became_active":"2023-06-03T20:24:02.256483+0000","last_became_peered":"2023-06-03T20:24:02.256483+0000","last_unstale":"2023-06-03T20:24:03.263349+0000","last_undegraded":"2023-06-03T20:24:03.263349+0000","last_fullsized":"2023-06-03T20:24:03.263349+0000","mapping_epoch":254,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":255,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:32:20.692043+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:32:20.692043+0000","last_clean_scrub_stamp":"2023-06-03T19:32:20.692043+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T00:53:37.197795+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":216,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-06-03T20:24:03.254361+0000","last_change":"2023-06-03T19:56:54.507670+0000","last_active":"2023-06-03T20:24:03.254361+0000","last_peered":"2023-06-03T20:24:03.254361+0000","last_clean":"2023-06-03T20:24:03.254361+0000","last_became_active":"2023-06-03T19:56:54.507293+0000","last_became_peered":"2023-06-03T19:56:54.507293+0000","last_unstale":"2023-06-03T20:24:03.254361+0000","last_undegraded":"2023-06-03T20:24:03.254361+0000","last_fullsized":"2023-06-03T20:24:03.254361+0000","mapping_epoch":119,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T01:49:19.966054+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"13.8","version":"257'9391","reported_seq":9424,"reported_epoch":257,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-06-03T20:24:26.349969+0000","last_change":"2023-06-03T20:24:03.876735+0000","last_active":"2023-06-03T20:24:26.349969+0000","last_peered":"2023-06-03T20:24:26.349969+0000","last_clean":"2023-06-03T20:24:01.253250+0000","last_became_active":"2023-06-03T20:24:03.848438+0000","last_became_peered":"2023-06-03T20:24:03.848438+0000","last_unstale":"2023-06-03T20:24:26.349969+0000","last_undegraded":"2023-06-03T20:24:03.309476+0000","last_fullsized":"2023-06-03T20:24:03.258094+0000","mapping_epoch":255,"log_start":"0'0","ondisk_log_start":"0'0","created":242,"last_epoch_clean":243,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T20:21:29.277668+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T20:21:29.277668+0000","last_clean_scrub_stamp":"2023-06-03T20:21:29.277668+0000","objects_scrubbed":0,"log_size":9391,"log_dups_size":0,"ondisk_log_size":9391,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T06:51:11.715942+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":332857344,"num_objects":5079,"num_object_clones":0,"num_object_copies":15237,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":7235,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5079,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9391,"num_write_kb":463040,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,6],"acting":[2,5],"avail_no_missing":["2","1","5"],"object_location_counts":[{"shards":"1,2,5","objects":5079},{"shards":"2,5","objects":-780}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":12,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-06-03T20:24:03.740157+0000","last_change":"2023-06-03T20:24:02.319230+0000","last_active":"2023-06-03T20:24:03.740157+0000","last_peered":"2023-06-03T20:24:03.740157+0000","last_clean":"2023-06-03T20:24:03.740157+0000","last_became_active":"2023-06-03T20:24:02.318926+0000","last_became_peered":"2023-06-03T20:24:02.318926+0000","last_unstale":"2023-06-03T20:24:03.740157+0000","last_undegraded":"2023-06-03T20:24:03.740157+0000","last_fullsized":"2023-06-03T20:24:03.740157+0000","mapping_epoch":254,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":255,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:32:20.692043+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:32:20.692043+0000","last_clean_scrub_stamp":"2023-06-03T19:32:20.692043+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"60'1","reported_seq":284,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-06-03T20:24:03.262174+0000","last_change":"2023-06-03T20:23:28.239664+0000","last_active":"2023-06-03T20:24:03.262174+0000","last_peered":"2023-06-03T20:24:03.262174+0000","last_clean":"2023-06-03T20:24:03.262174+0000","last_became_active":"2023-06-03T20:23:28.239252+0000","last_became_peered":"2023-06-03T20:23:28.239252+0000","last_unstale":"2023-06-03T20:24:03.262174+0000","last_undegraded":"2023-06-03T20:24:03.262174+0000","last_fullsized":"2023-06-03T20:24:03.262174+0000","mapping_epoch":251,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":252,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T06:52:20.862520+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":137,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-06-03T20:24:03.251763+0000","last_change":"2023-06-03T20:24:02.259850+0000","last_active":"2023-06-03T20:24:03.251763+0000","last_peered":"2023-06-03T20:24:03.251763+0000","last_clean":"2023-06-03T20:24:03.251763+0000","last_became_active":"2023-06-03T20:24:02.258764+0000","last_became_peered":"2023-06-03T20:24:02.258764+0000","last_unstale":"2023-06-03T20:24:03.251763+0000","last_undegraded":"2023-06-03T20:24:03.251763+0000","last_fullsized":"2023-06-03T20:24:03.251763+0000","mapping_epoch":254,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":255,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T21:54:11.487473+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"60'1","reported_seq":238,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-06-03T20:24:03.262833+0000","last_change":"2023-06-03T20:23:27.254853+0000","last_active":"2023-06-03T20:24:03.262833+0000","last_peered":"2023-06-03T20:24:03.262833+0000","last_clean":"2023-06-03T20:24:03.262833+0000","last_became_active":"2023-06-03T20:23:27.254410+0000","last_became_peered":"2023-06-03T20:23:27.254410+0000","last_unstale":"2023-06-03T20:24:03.262833+0000","last_undegraded":"2023-06-03T20:24:03.262833+0000","last_fullsized":"2023-06-03T20:24:03.262833+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T22:46:19.423939+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":269,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-06-03T20:24:03.258647+0000","last_change":"2023-06-03T20:21:01.257656+0000","last_active":"2023-06-03T20:24:03.258647+0000","last_peered":"2023-06-03T20:24:03.258647+0000","last_clean":"2023-06-03T20:24:03.258647+0000","last_became_active":"2023-06-03T20:21:01.257107+0000","last_became_peered":"2023-06-03T20:21:01.257107+0000","last_unstale":"2023-06-03T20:24:03.258647+0000","last_undegraded":"2023-06-03T20:24:03.258647+0000","last_fullsized":"2023-06-03T20:24:03.258647+0000","mapping_epoch":235,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":236,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T06:08:18.954539+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"60'2","reported_seq":29,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-06-03T20:24:03.251949+0000","last_change":"2023-06-03T20:23:28.243048+0000","last_active":"2023-06-03T20:24:03.251949+0000","last_peered":"2023-06-03T20:24:03.251949+0000","last_clean":"2023-06-03T20:24:03.251949+0000","last_became_active":"2023-06-03T20:23:28.241128+0000","last_became_peered":"2023-06-03T20:23:28.241128+0000","last_unstale":"2023-06-03T20:24:03.251949+0000","last_undegraded":"2023-06-03T20:24:03.251949+0000","last_fullsized":"2023-06-03T20:24:03.251949+0000","mapping_epoch":251,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":252,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T05:14:54.701329+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,1],"acting":[4,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":183,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-06-03T20:24:03.252691+0000","last_change":"2023-06-03T20:23:28.243038+0000","last_active":"2023-06-03T20:24:03.252691+0000","last_peered":"2023-06-03T20:24:03.252691+0000","last_clean":"2023-06-03T20:24:03.252691+0000","last_became_active":"2023-06-03T20:23:28.242008+0000","last_became_peered":"2023-06-03T20:23:28.242008+0000","last_unstale":"2023-06-03T20:24:03.252691+0000","last_undegraded":"2023-06-03T20:24:03.252691+0000","last_fullsized":"2023-06-03T20:24:03.252691+0000","mapping_epoch":251,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":252,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T23:11:11.920712+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"60'2","reported_seq":29,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-06-03T20:24:03.252185+0000","last_change":"2023-06-03T20:23:28.291544+0000","last_active":"2023-06-03T20:24:03.252185+0000","last_peered":"2023-06-03T20:24:03.252185+0000","last_clean":"2023-06-03T20:24:03.252185+0000","last_became_active":"2023-06-03T20:23:28.290655+0000","last_became_peered":"2023-06-03T20:23:28.290655+0000","last_unstale":"2023-06-03T20:24:03.252185+0000","last_undegraded":"2023-06-03T20:24:03.252185+0000","last_fullsized":"2023-06-03T20:24:03.252185+0000","mapping_epoch":251,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":252,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T23:28:09.467638+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,1],"acting":[4,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":176,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-06-03T20:24:03.251953+0000","last_change":"2023-06-03T20:23:27.254468+0000","last_active":"2023-06-03T20:24:03.251953+0000","last_peered":"2023-06-03T20:24:03.251953+0000","last_clean":"2023-06-03T20:24:03.251953+0000","last_became_active":"2023-06-03T20:23:27.253859+0000","last_became_peered":"2023-06-03T20:23:27.253859+0000","last_unstale":"2023-06-03T20:24:03.251953+0000","last_undegraded":"2023-06-03T20:24:03.251953+0000","last_fullsized":"2023-06-03T20:24:03.251953+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T20:15:25.066125+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"60'1","reported_seq":263,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-06-03T20:24:03.255498+0000","last_change":"2023-06-03T20:21:01.256517+0000","last_active":"2023-06-03T20:24:03.255498+0000","last_peered":"2023-06-03T20:24:03.255498+0000","last_clean":"2023-06-03T20:24:03.255498+0000","last_became_active":"2023-06-03T20:21:01.256116+0000","last_became_peered":"2023-06-03T20:21:01.256116+0000","last_unstale":"2023-06-03T20:24:03.255498+0000","last_undegraded":"2023-06-03T20:24:03.255498+0000","last_fullsized":"2023-06-03T20:24:03.255498+0000","mapping_epoch":235,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":236,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T06:47:35.144094+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,1],"acting":[3,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":256,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-06-03T20:24:03.254516+0000","last_change":"2023-06-03T19:56:54.503416+0000","last_active":"2023-06-03T20:24:03.254516+0000","last_peered":"2023-06-03T20:24:03.254516+0000","last_clean":"2023-06-03T20:24:03.254516+0000","last_became_active":"2023-06-03T19:56:54.503018+0000","last_became_peered":"2023-06-03T19:56:54.503018+0000","last_unstale":"2023-06-03T20:24:03.254516+0000","last_undegraded":"2023-06-03T20:24:03.254516+0000","last_fullsized":"2023-06-03T20:24:03.254516+0000","mapping_epoch":119,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T20:14:00.330616+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"60'4","reported_seq":226,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-06-03T20:24:03.262212+0000","last_change":"2023-06-03T19:56:11.748116+0000","last_active":"2023-06-03T20:24:03.262212+0000","last_peered":"2023-06-03T20:24:03.262212+0000","last_clean":"2023-06-03T20:24:03.262212+0000","last_became_active":"2023-06-03T19:56:11.747636+0000","last_became_peered":"2023-06-03T19:56:11.747636+0000","last_unstale":"2023-06-03T20:24:03.262212+0000","last_undegraded":"2023-06-03T20:24:03.262212+0000","last_fullsized":"2023-06-03T20:24:03.262212+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":115,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T06:57:48.052055+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":167,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-06-03T20:24:03.251671+0000","last_change":"2023-06-03T20:23:28.238140+0000","last_active":"2023-06-03T20:24:03.251671+0000","last_peered":"2023-06-03T20:24:03.251671+0000","last_clean":"2023-06-03T20:24:03.251671+0000","last_became_active":"2023-06-03T20:23:28.237535+0000","last_became_peered":"2023-06-03T20:23:28.237535+0000","last_unstale":"2023-06-03T20:24:03.251671+0000","last_undegraded":"2023-06-03T20:24:03.251671+0000","last_fullsized":"2023-06-03T20:24:03.251671+0000","mapping_epoch":251,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":252,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T04:02:16.349126+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":13,"num_pg":16,"stat_sum":{"num_bytes":5253955608,"num_objects":80170,"num_object_clones":0,"num_object_copies":240510,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":57704,"num_objects_misplaced":10155,"num_objects_unfound":0,"num_objects_dirty":80170,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":150876,"num_write_kb":7393409,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":984,"num_bytes_recovered":64094208,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12835561472,"data_stored":12835553328,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":137676,"ondisk_log_size":137676,"up":48,"acting":38,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5486,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":4,"num_objects_missing":4,"num_objects_degraded":5,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":62,"num_write_kb":30,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":17,"num_bytes_recovered":4786,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":110592,"data_stored":21174,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":30319,"internal_metadata":0},"log_size":56,"ondisk_log_size":56,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1180736,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2375680,"data_stored":2361472,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":2,"num_store_stats":5},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":48,"seq":206158430849,"num_pgs":36,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2817524,"kb_used_data":1934124,"kb_used_omap":0,"kb_used_meta":883328,"kb_avail":90927628,"statfs":{"total":95995035648,"available":93109891072,"internally_reserved":0,"allocated":1980542976,"data_stored":1979890592,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":904527872},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,1,4],"upper_bound":32},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 3 20:24:17 2023","interfaces":[{"interface":"back","average":{"1min":1.3700000000000001,"5min":1.262,"15min":1.641},"min":{"1min":0.26800000000000002,"5min":0.26800000000000002,"15min":0.25800000000000001},"max":{"1min":3.5150000000000001,"5min":8.5289999999999999,"15min":73.810000000000002},"last":2.363},{"interface":"front","average":{"1min":1.829,"5min":1.254,"15min":1.621},"min":{"1min":0.27100000000000002,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":6.9109999999999996,"5min":8.4939999999999998,"15min":72.878},"last":0.67900000000000005}]},{"osd":1,"last update":"Sat Jun 3 20:24:03 2023","interfaces":[{"interface":"back","average":{"1min":2.1139999999999999,"5min":1.131,"15min":1.133},"min":{"1min":0.311,"5min":0.311,"15min":0.308},"max":{"1min":19.341999999999999,"5min":19.341999999999999,"15min":19.341999999999999},"last":2.3969999999999998},{"interface":"front","average":{"1min":2.4220000000000002,"5min":1.288,"15min":1.173},"min":{"1min":0.34899999999999998,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":15.319000000000001,"5min":15.319000000000001,"15min":15.319000000000001},"last":0.58599999999999997}]},{"osd":2,"last update":"Sat Jun 3 20:24:03 2023","interfaces":[{"interface":"back","average":{"1min":1.7609999999999999,"5min":1.9259999999999999,"15min":3.819},"min":{"1min":0.29499999999999998,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":6.29,"5min":48.319000000000003,"15min":286.83600000000001},"last":2.3149999999999999},{"interface":"front","average":{"1min":1.5449999999999999,"5min":1.9650000000000001,"15min":3.827},"min":{"1min":0.33600000000000002,"5min":0.32500000000000001,"15min":0.28699999999999998},"max":{"1min":5.3239999999999998,"5min":47.915999999999997,"15min":287.32900000000001},"last":0.63400000000000001}]},{"osd":3,"last update":"Sat Jun 3 20:23:56 2023","interfaces":[{"interface":"back","average":{"1min":1.585,"5min":1.464,"15min":3.399},"min":{"1min":0.40200000000000002,"5min":0.371,"15min":0.30299999999999999},"max":{"1min":5.4329999999999998,"5min":11.992000000000001,"15min":265.40499999999997},"last":2.46},{"interface":"front","average":{"1min":1.454,"5min":1.212,"15min":3.3279999999999998},"min":{"1min":0.41499999999999998,"5min":0.36499999999999999,"15min":0.31900000000000001},"max":{"1min":5.4770000000000003,"5min":8.6739999999999995,"15min":265.43599999999998},"last":0.70799999999999996}]},{"osd":4,"last update":"Sat Jun 3 20:24:27 2023","interfaces":[{"interface":"back","average":{"1min":2.282,"5min":2.282,"15min":2.282},"min":{"1min":0.438,"5min":0.438,"15min":0.438},"max":{"1min":7.149,"5min":7.149,"15min":7.149},"last":5.0419999999999998},{"interface":"front","average":{"1min":1.9730000000000001,"5min":1.9730000000000001,"15min":1.9730000000000001},"min":{"1min":0.27600000000000002,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":8.7569999999999997,"5min":8.7569999999999997,"15min":8.7569999999999997},"last":3.1419999999999999}]},{"osd":5,"last update":"Sat Jun 3 20:23:56 2023","interfaces":[{"interface":"back","average":{"1min":1.845,"5min":0.97299999999999998,"15min":1.236},"min":{"1min":0.35699999999999998,"5min":0.35699999999999998,"15min":0.28899999999999998},"max":{"1min":5.5490000000000004,"5min":5.5490000000000004,"15min":11.443},"last":2.4289999999999998},{"interface":"front","average":{"1min":2.048,"5min":1.036,"15min":1.369},"min":{"1min":0.29399999999999998,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":9.3710000000000004,"5min":9.3710000000000004,"15min":9.3710000000000004},"last":5.0949999999999998}]},{"osd":6,"last update":"Sat Jun 3 20:23:56 2023","interfaces":[{"interface":"back","average":{"1min":0.92900000000000005,"5min":0.81699999999999995,"15min":1.4550000000000001},"min":{"1min":0.379,"5min":0.379,"15min":0.34799999999999998},"max":{"1min":2.855,"5min":2.855,"15min":11.256},"last":0.73399999999999999},{"interface":"front","average":{"1min":0.89200000000000002,"5min":0.76700000000000002,"15min":1.365},"min":{"1min":0.34399999999999997,"5min":0.34399999999999997,"15min":0.26900000000000002},"max":{"1min":3.7320000000000002,"5min":3.7320000000000002,"15min":10.894},"last":3.1000000000000001}]}]},{"osd":6,"up_from":42,"seq":180388627082,"num_pgs":20,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":535640,"kb_used_data":2384,"kb_used_omap":0,"kb_used_meta":533248,"kb_avail":93209512,"statfs":{"total":95995035648,"available":95446540288,"internally_reserved":0,"allocated":2441216,"data_stored":1816405,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":546045952},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1],"upper_bound":8},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 3 20:23:28 2023","interfaces":[{"interface":"back","average":{"1min":1.0780000000000001,"5min":1.7230000000000001,"15min":1.962},"min":{"1min":0.32500000000000001,"5min":0.26600000000000001,"15min":0.25600000000000001},"max":{"1min":5.117,"5min":56.259,"15min":56.259},"last":0.38500000000000001},{"interface":"front","average":{"1min":1.1279999999999999,"5min":1.7569999999999999,"15min":2.1789999999999998},"min":{"1min":0.35899999999999999,"5min":0.32700000000000001,"15min":0.28299999999999997},"max":{"1min":5.0839999999999996,"5min":56.234000000000002,"15min":56.234000000000002},"last":0.45500000000000002}]},{"osd":1,"last update":"Sat Jun 3 20:24:10 2023","interfaces":[{"interface":"back","average":{"1min":1.2509999999999999,"5min":1.2949999999999999,"15min":1.093},"min":{"1min":0.36699999999999999,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":5.6859999999999999,"5min":30.736000000000001,"15min":30.736000000000001},"last":0.50700000000000001},{"interface":"front","average":{"1min":0.82199999999999995,"5min":1.24,"15min":1.3},"min":{"1min":0.36799999999999999,"5min":0.32700000000000001,"15min":0.307},"max":{"1min":3.0369999999999999,"5min":30.788,"15min":30.788},"last":0.40899999999999997}]},{"osd":2,"last update":"Sat Jun 3 20:24:10 2023","interfaces":[{"interface":"back","average":{"1min":1.4119999999999999,"5min":2.5409999999999999,"15min":3.3300000000000001},"min":{"1min":0.35499999999999998,"5min":0.35499999999999998,"15min":0.315},"max":{"1min":6.6429999999999998,"5min":58.201000000000001,"15min":136.22},"last":0.376},{"interface":"front","average":{"1min":1.333,"5min":2.2570000000000001,"15min":3.129},"min":{"1min":0.36699999999999999,"5min":0.317,"15min":0.317},"max":{"1min":5.7590000000000003,"5min":58.103000000000002,"15min":135.68799999999999},"last":0.51400000000000001}]},{"osd":3,"last update":"Sat Jun 3 20:23:43 2023","interfaces":[{"interface":"back","average":{"1min":1.109,"5min":2.3740000000000001,"15min":1.772},"min":{"1min":0.435,"5min":0.41899999999999998,"15min":0.33600000000000002},"max":{"1min":4.8010000000000002,"5min":58.615000000000002,"15min":58.615000000000002},"last":0.436},{"interface":"front","average":{"1min":1.423,"5min":2.552,"15min":1.776},"min":{"1min":0.42899999999999999,"5min":0.40799999999999997,"15min":0.32500000000000001},"max":{"1min":5.4809999999999999,"5min":58.546999999999997,"15min":58.546999999999997},"last":0.54100000000000004}]},{"osd":4,"last update":"Sat Jun 3 20:24:28 2023","interfaces":[{"interface":"back","average":{"1min":1.8660000000000001,"5min":1.8660000000000001,"15min":1.8660000000000001},"min":{"1min":0.39000000000000001,"5min":0.39000000000000001,"15min":0.39000000000000001},"max":{"1min":9.6210000000000004,"5min":9.6210000000000004,"15min":9.6210000000000004},"last":0.495},{"interface":"front","average":{"1min":1.2869999999999999,"5min":1.2869999999999999,"15min":1.2869999999999999},"min":{"1min":0.34499999999999997,"5min":0.34499999999999997,"15min":0.34499999999999997},"max":{"1min":8.4990000000000006,"5min":8.4990000000000006,"15min":8.4990000000000006},"last":0.52000000000000002}]},{"osd":5,"last update":"Sat Jun 3 20:24:21 2023","interfaces":[{"interface":"back","average":{"1min":1.972,"5min":0.96799999999999997,"15min":1.0569999999999999},"min":{"1min":0.29899999999999999,"5min":0.29899999999999999,"15min":0.27700000000000002},"max":{"1min":10.625999999999999,"5min":10.625999999999999,"15min":10.625999999999999},"last":4.5499999999999998},{"interface":"front","average":{"1min":2.1139999999999999,"5min":1.0249999999999999,"15min":1.3089999999999999},"min":{"1min":0.436,"5min":0.41399999999999998,"15min":0.309},"max":{"1min":10.856,"5min":10.856,"15min":10.856},"last":0.432}]},{"osd":7,"last update":"Sat Jun 3 20:23:58 2023","interfaces":[{"interface":"back","average":{"1min":1.921,"5min":1.032,"15min":1.258},"min":{"1min":0.52200000000000002,"5min":0.37,"15min":0.315},"max":{"1min":6.4580000000000002,"5min":6.4580000000000002,"15min":9.0640000000000001},"last":0.54100000000000004},{"interface":"front","average":{"1min":1.4630000000000001,"5min":0.91500000000000004,"15min":1.347},"min":{"1min":0.45000000000000001,"5min":0.38,"15min":0.36299999999999999},"max":{"1min":9.5220000000000002,"5min":9.5220000000000002,"15min":21.202000000000002},"last":0.70999999999999996}]}]},{"osd":1,"up_from":103,"seq":442381631880,"num_pgs":44,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2365448,"kb_used_data":1997504,"kb_used_omap":4,"kb_used_meta":367931,"kb_avail":91379704,"statfs":{"total":95995035648,"available":93572816896,"internally_reserved":0,"allocated":2045444096,"data_stored":2044426894,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":376762254},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,0,0,1,0,2],"upper_bound":128},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 3 20:24:12 2023","interfaces":[{"interface":"back","average":{"1min":0.77400000000000002,"5min":2.363,"15min":2.2719999999999998},"min":{"1min":0.214,"5min":0.17100000000000001,"15min":0.17100000000000001},"max":{"1min":4.5919999999999996,"5min":119.94499999999999,"15min":119.94499999999999},"last":0.26100000000000001},{"interface":"front","average":{"1min":0.86699999999999999,"5min":2.2999999999999998,"15min":2.3439999999999999},"min":{"1min":0.23699999999999999,"5min":0.22500000000000001,"15min":0.20799999999999999},"max":{"1min":3.8940000000000001,"5min":120.04300000000001,"15min":120.04300000000001},"last":0.312}]},{"osd":2,"last update":"Sat Jun 3 20:24:12 2023","interfaces":[{"interface":"back","average":{"1min":1.4019999999999999,"5min":2.1440000000000001,"15min":2.6949999999999998},"min":{"1min":0.255,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":6.9500000000000002,"5min":49.595999999999997,"15min":219.72300000000001},"last":0.379},{"interface":"front","average":{"1min":1.3300000000000001,"5min":2.169,"15min":2.7440000000000002},"min":{"1min":0.33400000000000002,"5min":0.23499999999999999,"15min":0.22500000000000001},"max":{"1min":7.3079999999999998,"5min":49.360999999999997,"15min":219.82900000000001},"last":6.8540000000000001}]},{"osd":3,"last update":"Sat Jun 3 20:23:44 2023","interfaces":[{"interface":"back","average":{"1min":1.488,"5min":2.6139999999999999,"15min":1.9139999999999999},"min":{"1min":0.23400000000000001,"5min":0.23400000000000001,"15min":0.23400000000000001},"max":{"1min":4.883,"5min":96.864999999999995,"15min":96.864999999999995},"last":6.9059999999999997},{"interface":"front","average":{"1min":1.8440000000000001,"5min":2.8029999999999999,"15min":1.9430000000000001},"min":{"1min":0.439,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":5.4749999999999996,"5min":96.679000000000002,"15min":96.679000000000002},"last":6.9429999999999996}]},{"osd":4,"last update":"Sat Jun 3 20:10:54 2023","interfaces":[{"interface":"back","average":{"1min":0.97099999999999997,"5min":1.6100000000000001,"15min":1.4330000000000001},"min":{"1min":0.34100000000000003,"5min":0.34100000000000003,"15min":0.34100000000000003},"max":{"1min":2.9430000000000001,"5min":9.2929999999999993,"15min":9.9589999999999996},"last":0.45900000000000002},{"interface":"front","average":{"1min":1.032,"5min":1.4299999999999999,"15min":1.6759999999999999},"min":{"1min":0.40500000000000003,"5min":0.376,"15min":0.32700000000000001},"max":{"1min":3.4630000000000001,"5min":9.0749999999999993,"15min":120.084},"last":0.47399999999999998}]},{"osd":5,"last update":"Sat Jun 3 20:24:12 2023","interfaces":[{"interface":"back","average":{"1min":1.5700000000000001,"5min":1.0669999999999999,"15min":0.83499999999999996},"min":{"1min":0.36299999999999999,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":5.0449999999999999,"5min":5.0449999999999999,"15min":5.0449999999999999},"last":5.4939999999999998},{"interface":"front","average":{"1min":1.5549999999999999,"5min":0.98999999999999999,"15min":0.90700000000000003},"min":{"1min":0.45900000000000002,"5min":0.372,"15min":0.35999999999999999},"max":{"1min":4.0170000000000003,"5min":4.0170000000000003,"15min":5.9779999999999998},"last":5.4779999999999998}]},{"osd":6,"last update":"Sat Jun 3 20:24:12 2023","interfaces":[{"interface":"back","average":{"1min":1.151,"5min":0.90500000000000003,"15min":1.105},"min":{"1min":0.46000000000000002,"5min":0.41999999999999998,"15min":0.33900000000000002},"max":{"1min":3.9630000000000001,"5min":3.9630000000000001,"15min":8.6099999999999994},"last":0.44500000000000001},{"interface":"front","average":{"1min":1.4199999999999999,"5min":1.171,"15min":1.1579999999999999},"min":{"1min":0.48799999999999999,"5min":0.371,"15min":0.371},"max":{"1min":4.7729999999999997,"5min":7.9900000000000002,"15min":11.106},"last":7.0620000000000003}]},{"osd":7,"last update":"Sat Jun 3 20:24:12 2023","interfaces":[{"interface":"back","average":{"1min":1.7290000000000001,"5min":1.2869999999999999,"15min":1.2869999999999999},"min":{"1min":0.58399999999999996,"5min":0.39000000000000001,"15min":0.372},"max":{"1min":4.8220000000000001,"5min":8.0239999999999991,"15min":13.298999999999999},"last":6.9980000000000002},{"interface":"front","average":{"1min":1.9259999999999999,"5min":1.1319999999999999,"15min":1.0660000000000001},"min":{"1min":0.39200000000000002,"5min":0.34000000000000002,"15min":0.34000000000000002},"max":{"1min":7.5099999999999998,"5min":7.5099999999999998,"15min":7.5099999999999998},"last":0.436}]}]},{"osd":0,"up_from":86,"seq":369367187906,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1638708,"kb_used_data":975404,"kb_used_omap":5,"kb_used_meta":663290,"kb_avail":92106444,"statfs":{"total":95995035648,"available":94316998656,"internally_reserved":0,"allocated":998813696,"data_stored":997757215,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":679209594},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,0,1],"upper_bound":16},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Jun 3 20:23:59 2023","interfaces":[{"interface":"back","average":{"1min":1.095,"5min":3.444,"15min":1.694},"min":{"1min":0.185,"5min":0.185,"15min":0.185},"max":{"1min":6.75,"5min":243.03200000000001,"15min":243.03200000000001},"last":1.038},{"interface":"front","average":{"1min":0.99099999999999999,"5min":3.5369999999999999,"15min":1.954},"min":{"1min":0.20699999999999999,"5min":0.20699999999999999,"15min":0.20699999999999999},"max":{"1min":5.4779999999999998,"5min":242.78399999999999,"15min":242.78399999999999},"last":0.25}]},{"osd":2,"last update":"Sat Jun 3 20:23:59 2023","interfaces":[{"interface":"back","average":{"1min":1.2450000000000001,"5min":1.4299999999999999,"15min":1.3160000000000001},"min":{"1min":0.224,"5min":0.218,"15min":0.193},"max":{"1min":5.7000000000000002,"5min":11.811,"15min":11.811},"last":0.73099999999999998},{"interface":"front","average":{"1min":2.0760000000000001,"5min":4.4020000000000001,"15min":2.4449999999999998},"min":{"1min":0.312,"5min":0.312,"15min":0.255},"max":{"1min":8.3149999999999995,"5min":243.30099999999999,"15min":243.30099999999999},"last":1.0800000000000001}]},{"osd":3,"last update":"Sat Jun 3 20:23:38 2023","interfaces":[{"interface":"back","average":{"1min":11.933999999999999,"5min":3.8759999999999999,"15min":2.714},"min":{"1min":0.29199999999999998,"5min":0.248,"15min":0.248},"max":{"1min":243.38900000000001,"5min":243.38900000000001,"15min":243.38900000000001},"last":0.307},{"interface":"front","average":{"1min":0.78700000000000003,"5min":1.413,"15min":1.9910000000000001},"min":{"1min":0.23799999999999999,"5min":0.23799999999999999,"15min":0.23799999999999999},"max":{"1min":2.4430000000000001,"5min":14.613,"15min":58.646000000000001},"last":1.411}]},{"osd":4,"last update":"Sat Jun 3 20:10:59 2023","interfaces":[{"interface":"back","average":{"1min":1.9370000000000001,"5min":1.7110000000000001,"15min":1.3400000000000001},"min":{"1min":0.45200000000000001,"5min":0.318,"15min":0.318},"max":{"1min":10.025,"5min":10.025,"15min":10.025},"last":0.42899999999999999},{"interface":"front","average":{"1min":1.673,"5min":1.8160000000000001,"15min":1.429},"min":{"1min":0.56699999999999995,"5min":0.375,"15min":0.375},"max":{"1min":4.6059999999999999,"5min":13.058999999999999,"15min":13.058999999999999},"last":0.48399999999999999}]},{"osd":5,"last update":"Sat Jun 3 20:24:19 2023","interfaces":[{"interface":"back","average":{"1min":1.5800000000000001,"5min":3.6520000000000001,"15min":2.3650000000000002},"min":{"1min":0.40899999999999997,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":10.09,"5min":243.15899999999999,"15min":243.15899999999999},"last":0.39800000000000002},{"interface":"front","average":{"1min":1.373,"5min":3.7410000000000001,"15min":2.1890000000000001},"min":{"1min":0.33400000000000002,"5min":0.33400000000000002,"15min":0.32300000000000001},"max":{"1min":5.101,"5min":243.32599999999999,"15min":243.32599999999999},"last":0.57099999999999995}]},{"osd":6,"last update":"Sat Jun 3 20:24:19 2023","interfaces":[{"interface":"back","average":{"1min":1.095,"5min":3.5760000000000001,"15min":2.1709999999999998},"min":{"1min":0.41599999999999998,"5min":0.40999999999999998,"15min":0.35399999999999998},"max":{"1min":4.9610000000000003,"5min":243.25,"15min":243.25},"last":0.44700000000000001},{"interface":"front","average":{"1min":1.387,"5min":3.5819999999999999,"15min":2.1419999999999999},"min":{"1min":0.38600000000000001,"5min":0.38600000000000001,"15min":0.38300000000000001},"max":{"1min":6.8419999999999996,"5min":242.84399999999999,"15min":242.84399999999999},"last":1.1759999999999999}]},{"osd":7,"last update":"Sat Jun 3 20:24:19 2023","interfaces":[{"interface":"back","average":{"1min":1.1830000000000001,"5min":3.8069999999999999,"15min":2.5009999999999999},"min":{"1min":0.40799999999999997,"5min":0.35499999999999998,"15min":0.35499999999999998},"max":{"1min":4.9349999999999996,"5min":242.93600000000001,"15min":242.93600000000001},"last":0.69799999999999995},{"interface":"front","average":{"1min":1.506,"5min":3.7709999999999999,"15min":2.3780000000000001},"min":{"1min":0.58299999999999996,"5min":0.40000000000000002,"15min":0.40000000000000002},"max":{"1min":6.9000000000000004,"5min":243.21299999999999,"15min":243.21299999999999},"last":1.214}]}]},{"osd":2,"up_from":114,"seq":489626272085,"num_pgs":28,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3431008,"kb_used_data":2930392,"kb_used_omap":7,"kb_used_meta":500600,"kb_avail":90314144,"statfs":{"total":95995035648,"available":92481683456,"internally_reserved":0,"allocated":3000721408,"data_stored":2999746000,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8111,"internal_metadata":512614481},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,1,0,4,3,4],"upper_bound":256},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 3 20:23:52 2023","interfaces":[{"interface":"back","average":{"1min":2.8260000000000001,"5min":3.8610000000000002,"15min":3.359},"min":{"1min":0.27300000000000002,"5min":0.23999999999999999,"15min":0.17799999999999999},"max":{"1min":10.444000000000001,"5min":117.57899999999999,"15min":273.51100000000002},"last":0.307},{"interface":"front","average":{"1min":1.931,"5min":3.379,"15min":3.2989999999999999},"min":{"1min":0.248,"5min":0.23200000000000001,"15min":0.20399999999999999},"max":{"1min":12.76,"5min":117.38500000000001,"15min":273.67399999999998},"last":2.081}]},{"osd":1,"last update":"Sat Jun 3 20:23:52 2023","interfaces":[{"interface":"back","average":{"1min":2.6120000000000001,"5min":1.256,"15min":1.901},"min":{"1min":0.40300000000000002,"5min":0.247,"15min":0.23300000000000001},"max":{"1min":12.798,"5min":12.798,"15min":273.46499999999997},"last":0.27700000000000002},{"interface":"front","average":{"1min":2.6040000000000001,"5min":1.8069999999999999,"15min":2.238},"min":{"1min":0.25800000000000001,"5min":0.25800000000000001,"15min":0.25700000000000001},"max":{"1min":16.355,"5min":16.355,"15min":274.38400000000001},"last":0.56899999999999995}]},{"osd":3,"last update":"Sat Jun 3 20:23:50 2023","interfaces":[{"interface":"back","average":{"1min":3.593,"5min":2.956,"15min":3.3490000000000002},"min":{"1min":0.249,"5min":0.249,"15min":0.24399999999999999},"max":{"1min":13.058,"5min":74.703000000000003,"15min":273.71899999999999},"last":0.63},{"interface":"front","average":{"1min":2.3479999999999999,"5min":2.6230000000000002,"15min":3.4500000000000002},"min":{"1min":0.36499999999999999,"5min":0.23000000000000001,"15min":0.23000000000000001},"max":{"1min":16.463999999999999,"5min":16.463999999999999,"15min":274.21100000000001},"last":0.61499999999999999}]},{"osd":4,"last update":"Sat Jun 3 20:10:59 2023","interfaces":[{"interface":"back","average":{"1min":16.218,"5min":4.9749999999999996,"15min":2.2519999999999998},"min":{"1min":0.48699999999999999,"5min":0.32800000000000001,"15min":0.32700000000000001},"max":{"1min":273.75700000000001,"5min":273.75700000000001,"15min":273.75700000000001},"last":0.71699999999999997},{"interface":"front","average":{"1min":15.512,"5min":4.7480000000000002,"15min":2.1850000000000001},"min":{"1min":0.36199999999999999,"5min":0.35699999999999998,"15min":0.32600000000000001},"max":{"1min":274.27300000000002,"5min":274.27300000000002,"15min":274.27300000000002},"last":0.73999999999999999}]},{"osd":5,"last update":"Sat Jun 3 20:23:52 2023","interfaces":[{"interface":"back","average":{"1min":3.2109999999999999,"5min":2,"15min":2.3599999999999999},"min":{"1min":0.48499999999999999,"5min":0.33200000000000002,"15min":0.317},"max":{"1min":9.6020000000000003,"5min":17.751000000000001,"15min":273.78800000000001},"last":0.67000000000000004},{"interface":"front","average":{"1min":2.6320000000000001,"5min":1.6299999999999999,"15min":2.3180000000000001},"min":{"1min":0.38300000000000001,"5min":0.32600000000000001,"15min":0.32600000000000001},"max":{"1min":12.821999999999999,"5min":17.928000000000001,"15min":274.15800000000002},"last":0.48099999999999998}]},{"osd":6,"last update":"Sat Jun 3 20:23:52 2023","interfaces":[{"interface":"back","average":{"1min":2.8319999999999999,"5min":1.716,"15min":2.5049999999999999},"min":{"1min":0.39000000000000001,"5min":0.35199999999999998,"15min":0.34200000000000003},"max":{"1min":9.5809999999999995,"5min":14.331,"15min":274.03899999999999},"last":0.79900000000000004},{"interface":"front","average":{"1min":2.254,"5min":1.4910000000000001,"15min":2.4159999999999999},"min":{"1min":0.40799999999999997,"5min":0.35599999999999998,"15min":0.35099999999999998},"max":{"1min":12.867000000000001,"5min":12.867000000000001,"15min":274.06299999999999},"last":0.438}]},{"osd":7,"last update":"Sat Jun 3 20:23:52 2023","interfaces":[{"interface":"back","average":{"1min":3.0129999999999999,"5min":1.7969999999999999,"15min":2.5459999999999998},"min":{"1min":0.57199999999999995,"5min":0.376,"15min":0.317},"max":{"1min":9.6509999999999998,"5min":14.243,"15min":274.54000000000002},"last":0.70199999999999996},{"interface":"front","average":{"1min":2.4180000000000001,"5min":1.5289999999999999,"15min":2.5169999999999999},"min":{"1min":0.46300000000000002,"5min":0.38600000000000001,"15min":0.38600000000000001},"max":{"1min":12.845000000000001,"5min":12.845000000000001,"15min":274.12700000000001},"last":0.505}]}]},{"osd":3,"up_from":119,"seq":511101108557,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1792724,"kb_used_data":1266892,"kb_used_omap":2,"kb_used_meta":525821,"kb_avail":91952428,"statfs":{"total":95995035648,"available":94159286272,"internally_reserved":0,"allocated":1297297408,"data_stored":1296267111,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":538440757},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,1,3],"upper_bound":32},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 3 20:23:51 2023","interfaces":[{"interface":"back","average":{"1min":2.8580000000000001,"5min":2.722,"15min":2.2759999999999998},"min":{"1min":0.27900000000000003,"5min":0.214,"15min":0.192},"max":{"1min":31.449999999999999,"5min":76.379999999999995,"15min":97.164000000000001},"last":0.312},{"interface":"front","average":{"1min":2.6360000000000001,"5min":2.8780000000000001,"15min":2.3730000000000002},"min":{"1min":0.23499999999999999,"5min":0.19500000000000001,"15min":0.183},"max":{"1min":32.929000000000002,"5min":77.822000000000003,"15min":95.816000000000003},"last":0.35699999999999998}]},{"osd":1,"last update":"Sat Jun 3 20:23:51 2023","interfaces":[{"interface":"back","average":{"1min":1.0980000000000001,"5min":2.234,"15min":1.482},"min":{"1min":0.23799999999999999,"5min":0.22500000000000001,"15min":0.22500000000000001},"max":{"1min":6.649,"5min":117.047,"15min":117.047},"last":0.40100000000000002},{"interface":"front","average":{"1min":1.056,"5min":2.0630000000000002,"15min":1.4099999999999999},"min":{"1min":0.26200000000000001,"5min":0.20399999999999999,"15min":0.20399999999999999},"max":{"1min":3.843,"5min":116.505,"15min":116.505},"last":0.41299999999999998}]},{"osd":2,"last update":"Sat Jun 3 20:23:51 2023","interfaces":[{"interface":"back","average":{"1min":1.075,"5min":3.8730000000000002,"15min":2.3069999999999999},"min":{"1min":0.25700000000000001,"5min":0.25700000000000001,"15min":0.217},"max":{"1min":3.6349999999999998,"5min":117.584,"15min":117.584},"last":0.497},{"interface":"front","average":{"1min":1.659,"5min":3.9510000000000001,"15min":2.3559999999999999},"min":{"1min":0.35599999999999998,"5min":0.26600000000000001,"15min":0.23599999999999999},"max":{"1min":9.0860000000000003,"5min":117.01000000000001,"15min":117.01000000000001},"last":0.44700000000000001}]},{"osd":4,"last update":"Sat Jun 3 20:24:27 2023","interfaces":[{"interface":"back","average":{"1min":1.512,"5min":1.512,"15min":1.512},"min":{"1min":0.41199999999999998,"5min":0.41199999999999998,"15min":0.41199999999999998},"max":{"1min":6.8719999999999999,"5min":6.8719999999999999,"15min":6.8719999999999999},"last":5.1180000000000003},{"interface":"front","average":{"1min":1.7969999999999999,"5min":1.7969999999999999,"15min":1.7969999999999999},"min":{"1min":0.41299999999999998,"5min":0.41299999999999998,"15min":0.41299999999999998},"max":{"1min":6.2610000000000001,"5min":6.2610000000000001,"15min":6.2610000000000001},"last":0.51500000000000001}]},{"osd":5,"last update":"Sat Jun 3 20:23:51 2023","interfaces":[{"interface":"back","average":{"1min":3.0009999999999999,"5min":1.663,"15min":1.546},"min":{"1min":0.39300000000000002,"5min":0.35399999999999998,"15min":0.313},"max":{"1min":38.651000000000003,"5min":38.651000000000003,"15min":38.651000000000003},"last":0.55600000000000005},{"interface":"front","average":{"1min":3.169,"5min":1.7250000000000001,"15min":1.393},"min":{"1min":0.316,"5min":0.316,"15min":0.316},"max":{"1min":38.585999999999999,"5min":38.585999999999999,"15min":38.585999999999999},"last":0.67100000000000004}]},{"osd":6,"last update":"Sat Jun 3 20:23:51 2023","interfaces":[{"interface":"back","average":{"1min":0.73499999999999999,"5min":1.3009999999999999,"15min":1.4490000000000001},"min":{"1min":0.35899999999999999,"5min":0.34799999999999998,"15min":0.33500000000000002},"max":{"1min":1.363,"5min":9.0719999999999992,"15min":13.718},"last":4.367},{"interface":"front","average":{"1min":1.1180000000000001,"5min":1.4630000000000001,"15min":1.7},"min":{"1min":0.48099999999999998,"5min":0.39300000000000002,"15min":0.32500000000000001},"max":{"1min":6.5810000000000004,"5min":7.7510000000000003,"15min":13.734999999999999},"last":4.3890000000000002}]},{"osd":7,"last update":"Sat Jun 3 20:23:51 2023","interfaces":[{"interface":"back","average":{"1min":3.4649999999999999,"5min":1.9370000000000001,"15min":1.8009999999999999},"min":{"1min":0.39800000000000002,"5min":0.36599999999999999,"15min":0.35599999999999998},"max":{"1min":39.008000000000003,"5min":39.008000000000003,"15min":39.008000000000003},"last":1.637},{"interface":"front","average":{"1min":3.1749999999999998,"5min":1.794,"15min":1.919},"min":{"1min":0.50700000000000001,"5min":0.35699999999999998,"15min":0.34899999999999998},"max":{"1min":40.003999999999998,"5min":40.003999999999998,"15min":40.003999999999998},"last":2.306}]}]},{"osd":4,"up_from":250,"seq":1073741824014,"num_pgs":50,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":227468,"kb_used_data":153668,"kb_used_omap":10,"kb_used_meta":73717,"kb_avail":93517684,"statfs":{"total":95995035648,"available":95762108416,"internally_reserved":0,"allocated":157356032,"data_stored":156218006,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":10652,"internal_metadata":75486820},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,1,0,3,5],"upper_bound":128},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 3 20:24:27 2023","interfaces":[{"interface":"back","average":{"1min":1.829,"5min":1.829,"15min":1.829},"min":{"1min":0.26300000000000001,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":8.1649999999999991,"5min":8.1649999999999991,"15min":8.1649999999999991},"last":8.1649999999999991},{"interface":"front","average":{"1min":1.462,"5min":1.462,"15min":1.462},"min":{"1min":0.33100000000000002,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":4.7380000000000004,"5min":4.7380000000000004,"15min":4.7380000000000004},"last":0.629}]},{"osd":1,"last update":"Sat Jun 3 20:24:27 2023","interfaces":[{"interface":"back","average":{"1min":1.7070000000000001,"5min":1.7070000000000001,"15min":1.7070000000000001},"min":{"1min":0.38,"5min":0.38,"15min":0.38},"max":{"1min":4.7800000000000002,"5min":4.7800000000000002,"15min":4.7800000000000002},"last":3.907},{"interface":"front","average":{"1min":1.8819999999999999,"5min":1.8819999999999999,"15min":1.8819999999999999},"min":{"1min":0.39700000000000002,"5min":0.39700000000000002,"15min":0.39700000000000002},"max":{"1min":8.2560000000000002,"5min":8.2560000000000002,"15min":8.2560000000000002},"last":8.2560000000000002}]},{"osd":2,"last update":"Sat Jun 3 20:24:27 2023","interfaces":[{"interface":"back","average":{"1min":2.0779999999999998,"5min":2.0779999999999998,"15min":2.0779999999999998},"min":{"1min":0.35899999999999999,"5min":0.35899999999999999,"15min":0.35899999999999999},"max":{"1min":5.8109999999999999,"5min":5.8109999999999999,"15min":5.8109999999999999},"last":3.871},{"interface":"front","average":{"1min":2.6800000000000002,"5min":2.6800000000000002,"15min":2.6800000000000002},"min":{"1min":0.501,"5min":0.501,"15min":0.501},"max":{"1min":8.5749999999999993,"5min":8.5749999999999993,"15min":8.5749999999999993},"last":8.5749999999999993}]},{"osd":3,"last update":"Sat Jun 3 20:24:27 2023","interfaces":[{"interface":"back","average":{"1min":14.531000000000001,"5min":14.531000000000001,"15min":14.531000000000001},"min":{"1min":0.32500000000000001,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":217.578,"5min":217.578,"15min":217.578},"last":8.2919999999999998},{"interface":"front","average":{"1min":14.228,"5min":14.228,"15min":14.228},"min":{"1min":0.42499999999999999,"5min":0.42499999999999999,"15min":0.42499999999999999},"max":{"1min":218.55699999999999,"5min":218.55699999999999,"15min":218.55699999999999},"last":0.66500000000000004}]},{"osd":5,"last update":"Sat Jun 3 20:24:27 2023","interfaces":[{"interface":"back","average":{"1min":3.5059999999999998,"5min":3.5059999999999998,"15min":3.5059999999999998},"min":{"1min":0.44800000000000001,"5min":0.44800000000000001,"15min":0.44800000000000001},"max":{"1min":23.913,"5min":23.913,"15min":23.913},"last":0.71899999999999997},{"interface":"front","average":{"1min":2.2949999999999999,"5min":2.2949999999999999,"15min":2.2949999999999999},"min":{"1min":0.432,"5min":0.432,"15min":0.432},"max":{"1min":7.3029999999999999,"5min":7.3029999999999999,"15min":7.3029999999999999},"last":7.3029999999999999}]},{"osd":6,"last update":"Sat Jun 3 20:24:27 2023","interfaces":[{"interface":"back","average":{"1min":3.0009999999999999,"5min":3.0009999999999999,"15min":3.0009999999999999},"min":{"1min":0.47599999999999998,"5min":0.47599999999999998,"15min":0.47599999999999998},"max":{"1min":23.963000000000001,"5min":23.963000000000001,"15min":23.963000000000001},"last":3.8540000000000001},{"interface":"front","average":{"1min":1.399,"5min":1.399,"15min":1.399},"min":{"1min":0.46200000000000002,"5min":0.46200000000000002,"15min":0.46200000000000002},"max":{"1min":4.5380000000000003,"5min":4.5380000000000003,"15min":4.5380000000000003},"last":3.9700000000000002}]},{"osd":7,"last update":"Sat Jun 3 20:24:27 2023","interfaces":[{"interface":"back","average":{"1min":2.306,"5min":2.306,"15min":2.306},"min":{"1min":0.374,"5min":0.374,"15min":0.374},"max":{"1min":9.0440000000000005,"5min":9.0440000000000005,"15min":9.0440000000000005},"last":3.9319999999999999},{"interface":"front","average":{"1min":2.3559999999999999,"5min":2.3559999999999999,"15min":2.3559999999999999},"min":{"1min":0.45400000000000001,"5min":0.45400000000000001,"15min":0.45400000000000001},"max":{"1min":8.2159999999999993,"5min":8.2159999999999993,"15min":8.2159999999999993},"last":8.2159999999999993}]}]},{"osd":5,"up_from":36,"seq":154618823316,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":4340372,"kb_used_data":3506764,"kb_used_omap":0,"kb_used_meta":833536,"kb_avail":89404780,"statfs":{"total":95995035648,"available":91550494720,"internally_reserved":0,"allocated":3590926336,"data_stored":3590301529,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":853540864},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,0,2,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 3 20:24:08 2023","interfaces":[{"interface":"back","average":{"1min":1.2869999999999999,"5min":2.7879999999999998,"15min":2.3660000000000001},"min":{"1min":0.309,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":8.6140000000000008,"5min":126.461,"15min":126.461},"last":1.3560000000000001},{"interface":"front","average":{"1min":2.1230000000000002,"5min":3.0659999999999998,"15min":2.5649999999999999},"min":{"1min":0.29799999999999999,"5min":0.29799999999999999,"15min":0.27700000000000002},"max":{"1min":8.859,"5min":126.497,"15min":126.497},"last":0.97499999999999998}]},{"osd":1,"last update":"Sat Jun 3 20:24:08 2023","interfaces":[{"interface":"back","average":{"1min":3.3359999999999999,"5min":2.4409999999999998,"15min":1.5880000000000001},"min":{"1min":0.371,"5min":0.32900000000000001,"15min":0.28799999999999998},"max":{"1min":19.332000000000001,"5min":102.205,"15min":102.205},"last":0.72799999999999998},{"interface":"front","average":{"1min":2.9489999999999998,"5min":2.5009999999999999,"15min":1.5429999999999999},"min":{"1min":0.32900000000000001,"5min":0.32100000000000001,"15min":0.27600000000000002},"max":{"1min":19.530000000000001,"5min":102.265,"15min":102.265},"last":0.998}]},{"osd":2,"last update":"Sat Jun 3 20:24:08 2023","interfaces":[{"interface":"back","average":{"1min":2.7330000000000001,"5min":3.0350000000000001,"15min":3.5670000000000002},"min":{"1min":0.34499999999999997,"5min":0.34499999999999997,"15min":0.32500000000000001},"max":{"1min":12.884,"5min":69.697999999999993,"15min":216.881},"last":1.3859999999999999},{"interface":"front","average":{"1min":2.2669999999999999,"5min":3.2530000000000001,"15min":3.8500000000000001},"min":{"1min":0.34999999999999998,"5min":0.34999999999999998,"15min":0.314},"max":{"1min":8.9109999999999996,"5min":70.040999999999997,"15min":216.751},"last":8.6039999999999992}]},{"osd":3,"last update":"Sat Jun 3 20:23:44 2023","interfaces":[{"interface":"back","average":{"1min":2.431,"5min":1.643,"15min":3.359},"min":{"1min":0.45700000000000002,"5min":0.41899999999999998,"15min":0.36299999999999999},"max":{"1min":9.2390000000000008,"5min":9.2390000000000008,"15min":264.822},"last":0.76200000000000001},{"interface":"front","average":{"1min":1.337,"5min":1.784,"15min":3.153},"min":{"1min":0.377,"5min":0.37,"15min":0.35799999999999998},"max":{"1min":4.2640000000000002,"5min":39.537999999999997,"15min":264.75},"last":1.4079999999999999}]},{"osd":4,"last update":"Sat Jun 3 20:24:26 2023","interfaces":[{"interface":"back","average":{"1min":3.7509999999999999,"5min":3.7509999999999999,"15min":3.7509999999999999},"min":{"1min":0.61699999999999999,"5min":0.61699999999999999,"15min":0.61699999999999999},"max":{"1min":18.449000000000002,"5min":18.449000000000002,"15min":18.449000000000002},"last":0.83499999999999996},{"interface":"front","average":{"1min":3.0579999999999998,"5min":3.0579999999999998,"15min":3.0579999999999998},"min":{"1min":0.30199999999999999,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":26.478000000000002,"5min":26.478000000000002,"15min":26.478000000000002},"last":1.4339999999999999}]},{"osd":6,"last update":"Sat Jun 3 20:24:08 2023","interfaces":[{"interface":"back","average":{"1min":1.53,"5min":0.94699999999999995,"15min":1.3500000000000001},"min":{"1min":0.438,"5min":0.35199999999999998,"15min":0.29099999999999998},"max":{"1min":4.1109999999999998,"5min":4.1109999999999998,"15min":11.167},"last":0.78400000000000003},{"interface":"front","average":{"1min":1.9019999999999999,"5min":1.157,"15min":1.5049999999999999},"min":{"1min":0.55800000000000005,"5min":0.441,"15min":0.35799999999999998},"max":{"1min":5.4109999999999996,"5min":5.4109999999999996,"15min":12.140000000000001},"last":0.80500000000000005}]},{"osd":7,"last update":"Sat Jun 3 20:23:58 2023","interfaces":[{"interface":"back","average":{"1min":2.1110000000000002,"5min":1.236,"15min":1.51},"min":{"1min":0.39200000000000002,"5min":0.35999999999999999,"15min":0.35999999999999999},"max":{"1min":9.8510000000000009,"5min":9.8510000000000009,"15min":14.223000000000001},"last":0.85499999999999998},{"interface":"front","average":{"1min":1.327,"5min":0.89100000000000001,"15min":1.2070000000000001},"min":{"1min":0.40200000000000002,"5min":0.39700000000000002,"15min":0.28799999999999998},"max":{"1min":5.0140000000000002,"5min":5.0140000000000002,"15min":9.0039999999999996},"last":5.601}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6928,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4862,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":10652,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":4716,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4932,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":988479488,"data_stored":988479488,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":2037583872,"data_stored":2037579800,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":2985295872,"data_stored":2985295872,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":1288241152,"data_stored":1288241152,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":1048576,"data_stored":1048576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":3567714304,"data_stored":3567714304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":1967198208,"data_stored":1967194136,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-06-03T20:24:37.310 INFO:tasks.cephadm.ceph_manager.ceph:increase pool size by 10 2023-06-03T20:24:37.311 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph osd pool set unique_pool_8 pg_num 26 2023-06-03T20:24:37.323 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:24:37.361 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:24:37.364 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:24:37.364 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:24:37.447 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:24:37 smithi089 bash[75068]: audit 1685823877.1443233 mon.c (mon.2) 616 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T20:24:37.448 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:24:37 smithi089 bash[78452]: audit 1685823877.1443233 mon.c (mon.2) 616 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T20:24:37.465 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:24:37.498 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:24:37.502 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:24:37.502 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:24:37.584 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:24:37 smithi164 bash[59462]: audit 1685823877.1443233 mon.c (mon.2) 616 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T20:24:37.603 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:24:37.653 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:24:37.657 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:24:37.658 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:24:37.759 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:24:37.791 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:24:37.797 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:24:37.797 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:24:37.898 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:24:37.935 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:24:37.939 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:24:37.939 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:24:38.043 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:24:38.091 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:24:38.093 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:24:38.094 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:24:38.195 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:24:38.219 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:24:38.221 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:24:38.221 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:24:38.322 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:24:38.331 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:24:38 smithi089 bash[78452]: cluster 1685823877.0442524 mgr.y (mgr.34104) 1462 : cluster 0 pgmap v1485: 89 pgs: 2 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 6 active+undersized+degraded+remapped+backfill_wait, 75 active+clean; 4.7 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 510 op/s; 56642/230220 objects degraded (24.603%); 9644/230220 objects misplaced (4.189%); 805 KiB/s, 12 objects/s recovering 2023-06-03T20:24:38.331 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:24:38 smithi089 bash[78452]: cluster 1685823877.9444802 mon.a (mon.0) 1450 : cluster 3 Health check update: Degraded data redundancy: 56642/230220 objects degraded (24.603%), 13 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2023-06-03T20:24:38.331 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:24:38 smithi089 bash[75068]: cluster 1685823877.0442524 mgr.y (mgr.34104) 1462 : cluster 0 pgmap v1485: 89 pgs: 2 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 6 active+undersized+degraded+remapped+backfill_wait, 75 active+clean; 4.7 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 510 op/s; 56642/230220 objects degraded (24.603%); 9644/230220 objects misplaced (4.189%); 805 KiB/s, 12 objects/s recovering 2023-06-03T20:24:38.331 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:24:38 smithi089 bash[75068]: cluster 1685823877.9444802 mon.a (mon.0) 1450 : cluster 3 Health check update: Degraded data redundancy: 56642/230220 objects degraded (24.603%), 13 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2023-06-03T20:24:38.353 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:24:38.359 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:24:38.359 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:24:38.460 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:24:38.470 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:24:38 smithi164 bash[59462]: cluster 1685823877.0442524 mgr.y (mgr.34104) 1462 : cluster 0 pgmap v1485: 89 pgs: 2 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 6 active+undersized+degraded+remapped+backfill_wait, 75 active+clean; 4.7 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 510 op/s; 56642/230220 objects degraded (24.603%); 9644/230220 objects misplaced (4.189%); 805 KiB/s, 12 objects/s recovering 2023-06-03T20:24:38.470 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:24:38 smithi164 bash[59462]: cluster 1685823877.9444802 mon.a (mon.0) 1450 : cluster 3 Health check update: Degraded data redundancy: 56642/230220 objects degraded (24.603%), 13 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2023-06-03T20:24:38.490 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:24:38.500 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:24:38.500 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:24:38.601 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:24:38.638 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:24:38.641 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:24:38.641 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:24:38.742 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:24:38.769 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:24:38.777 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:24:38.778 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:24:38.879 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:24:38.908 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:24:38.914 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:24:38.915 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:24:39.016 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:24:39.070 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:24:39.076 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:24:39.078 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:24:39.179 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:24:39.211 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:24:39.215 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:24:39.216 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:24:39.317 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:24:39.344 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:24:39.346 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:24:39.347 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:24:39.448 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:24:39.473 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:24:39.475 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:24:39.476 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:24:39.577 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:24:39.614 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:24:39.621 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:24:39.622 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:24:39.723 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:24:39.758 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:24:39.767 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:24:39.767 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:24:39.868 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:24:39.913 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:24:39.919 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:24:39.920 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:24:40.020 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:24:40.061 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:24:40.063 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:24:40.064 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:24:40.165 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:24:40.226 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:24:40.234 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:24:40.235 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:24:40.337 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:24:40.493 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:24:40 smithi089 bash[78452]: cluster 1685823879.0451097 mgr.y (mgr.34104) 1463 : cluster 0 pgmap v1486: 89 pgs: 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 6 active+undersized+degraded+remapped+backfill_wait, 75 active+clean; 4.6 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 553 op/s; 55894/226188 objects degraded (24.711%); 9455/226188 objects misplaced (4.180%); 1.2 MiB/s, 19 objects/s recovering 2023-06-03T20:24:40.494 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:24:40 smithi089 bash[75068]: cluster 1685823879.0451097 mgr.y (mgr.34104) 1463 : cluster 0 pgmap v1486: 89 pgs: 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 6 active+undersized+degraded+remapped+backfill_wait, 75 active+clean; 4.6 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 553 op/s; 55894/226188 objects degraded (24.711%); 9455/226188 objects misplaced (4.180%); 1.2 MiB/s, 19 objects/s recovering 2023-06-03T20:24:40.526 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:24:40.528 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:24:40.530 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:24:40.585 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:24:40 smithi164 bash[59462]: cluster 1685823879.0451097 mgr.y (mgr.34104) 1463 : cluster 0 pgmap v1486: 89 pgs: 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 6 active+undersized+degraded+remapped+backfill_wait, 75 active+clean; 4.6 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 553 op/s; 55894/226188 objects degraded (24.711%); 9455/226188 objects misplaced (4.180%); 1.2 MiB/s, 19 objects/s recovering 2023-06-03T20:24:40.631 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:24:40.673 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:24:40.676 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:24:40.676 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:24:40.682 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph daemon osd.3 dump_blocked_ops 2023-06-03T20:24:40.777 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:24:40.823 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:24:40.826 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:24:40.827 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:24:40.928 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:24:40.973 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:24:40.979 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:24:40.980 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:24:41.081 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:24:41.123 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:24:41.125 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:24:41.127 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:24:41.228 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:24:41.267 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:24:41.272 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:24:41.275 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:24:41.377 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:24:41.411 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:24:41.415 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:24:41.416 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:24:41.518 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:24:41.553 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:24:41.561 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:24:41.562 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:24:41.664 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:24:41.701 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:24:41.702 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:24:41.703 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:24:41.804 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:24:41.850 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:24:41.851 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:24:41.851 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:24:41.952 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:24:41.999 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:24:42.001 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:24:42.002 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:24:42.103 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:24:42.141 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:24:42.147 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:24:42.148 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:24:42.249 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:24:42.297 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:24:42.299 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:24:42.300 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:24:42.401 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:24:42.445 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:24:42.451 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:24:42.452 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:24:42.529 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:24:42 smithi089 bash[75068]: cluster 1685823881.0459669 mgr.y (mgr.34104) 1464 : cluster 0 pgmap v1487: 89 pgs: 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 6 active+undersized+degraded+remapped+backfill_wait, 75 active+clean; 4.6 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 483 op/s; 55546/224643 objects degraded (24.726%); 9455/224643 objects misplaced (4.209%); 858 KiB/s, 13 objects/s recovering 2023-06-03T20:24:42.529 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:24:42 smithi089 bash[78452]: cluster 1685823881.0459669 mgr.y (mgr.34104) 1464 : cluster 0 pgmap v1487: 89 pgs: 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 6 active+undersized+degraded+remapped+backfill_wait, 75 active+clean; 4.6 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 483 op/s; 55546/224643 objects degraded (24.726%); 9455/224643 objects misplaced (4.209%); 858 KiB/s, 13 objects/s recovering 2023-06-03T20:24:42.553 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:24:42.584 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:24:42 smithi164 bash[59462]: cluster 1685823881.0459669 mgr.y (mgr.34104) 1464 : cluster 0 pgmap v1487: 89 pgs: 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 6 active+undersized+degraded+remapped+backfill_wait, 75 active+clean; 4.6 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 483 op/s; 55546/224643 objects degraded (24.726%); 9455/224643 objects misplaced (4.209%); 858 KiB/s, 13 objects/s recovering 2023-06-03T20:24:42.587 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:24:42.594 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:24:42.595 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:24:42.696 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:24:42.753 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:24:42.760 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:24:42.760 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:24:42.861 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:24:42.897 INFO:teuthology.orchestra.run.smithi089.stdout:osd.0: {} 2023-06-03T20:24:42.899 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:24:42.907 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:24:42.908 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:24:42.948 INFO:teuthology.orchestra.run.smithi089.stdout:osd.1: {} 2023-06-03T20:24:42.992 INFO:teuthology.orchestra.run.smithi089.stdout:osd.2: {} 2023-06-03T20:24:43.010 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:24:43.048 INFO:teuthology.orchestra.run.smithi089.stdout:osd.3: {} 2023-06-03T20:24:43.065 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:24:43.072 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:24:43.073 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:24:43.093 INFO:teuthology.orchestra.run.smithi089.stdout:osd.4: {} 2023-06-03T20:24:43.134 INFO:teuthology.orchestra.run.smithi089.stdout:osd.5: {} 2023-06-03T20:24:43.173 INFO:teuthology.orchestra.run.smithi089.stdout:osd.6: {} 2023-06-03T20:24:43.174 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:24:43.212 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:24:43.214 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:24:43.215 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:24:43 smithi164 bash[59462]: audit 1685823882.856156 mon.a (mon.0) 1451 : audit 0 from='client.? 172.21.15.89:0/689194969' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-03T20:24:43.215 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:24:43 smithi164 bash[59462]: cluster 1685823882.9455903 mon.a (mon.0) 1452 : cluster 3 Health check update: Degraded data redundancy: 55546/224643 objects degraded (24.726%), 13 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2023-06-03T20:24:43.216 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:24:43.233 INFO:teuthology.orchestra.run.smithi089.stdout:osd.7: {} 2023-06-03T20:24:43.238 INFO:teuthology.orchestra.run.smithi089.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-06-03T20:24:43.239 INFO:teuthology.orchestra.run.smithi089.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-06-03T20:24:43.239 INFO:teuthology.orchestra.run.smithi089.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-06-03T20:24:43.239 INFO:teuthology.orchestra.run.smithi089.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-06-03T20:24:43.240 INFO:teuthology.orchestra.run.smithi089.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-06-03T20:24:43.240 INFO:teuthology.orchestra.run.smithi089.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-06-03T20:24:43.240 INFO:teuthology.orchestra.run.smithi089.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-06-03T20:24:43.241 INFO:teuthology.orchestra.run.smithi089.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-06-03T20:24:43.278 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:24:43 smithi089 bash[78452]: audit 1685823882.856156 mon.a (mon.0) 1451 : audit 0 from='client.? 172.21.15.89:0/689194969' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-03T20:24:43.278 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:24:43 smithi089 bash[78452]: cluster 1685823882.9455903 mon.a (mon.0) 1452 : cluster 3 Health check update: Degraded data redundancy: 55546/224643 objects degraded (24.726%), 13 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2023-06-03T20:24:43.279 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:24:43 smithi089 bash[75068]: audit 1685823882.856156 mon.a (mon.0) 1451 : audit 0 from='client.? 172.21.15.89:0/689194969' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-03T20:24:43.279 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:24:43 smithi089 bash[75068]: cluster 1685823882.9455903 mon.a (mon.0) 1452 : cluster 3 Health check update: Degraded data redundancy: 55546/224643 objects degraded (24.726%), 13 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2023-06-03T20:24:43.318 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:24:43.356 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:24:43.364 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:24:43.365 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:24:43.466 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:24:43.517 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:24:43.525 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:24:43.526 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:24:43.627 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:24:43.667 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:24:43.669 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:24:43.670 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:24:43.771 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:24:43.808 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:24:43.809 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:24:43.810 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:24:43.911 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:24:43.954 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:24:43.956 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:24:43.958 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:24:44.058 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:24:44.092 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:24:44.098 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:24:44.099 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:24:44.200 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:24:44.230 INFO:teuthology.orchestra.run.smithi089.stderr:noscrub is unset 2023-06-03T20:24:44.244 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:24:44.251 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:24:44.251 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:24:44.352 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:24:44.376 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:24:44.378 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:24:44.379 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:24:44.480 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:24:44.493 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:24:44 smithi164 bash[59462]: cluster 1685823883.0468597 mgr.y (mgr.34104) 1465 : cluster 0 pgmap v1488: 89 pgs: 2 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 6 active+undersized+degraded+remapped+backfill_wait, 75 active+clean; 4.5 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 583 op/s; 55021/219567 objects degraded (25.059%); 9317/219567 objects misplaced (4.243%); 858 KiB/s, 13 objects/s recovering 2023-06-03T20:24:44.493 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:24:44 smithi164 bash[59462]: audit 1685823883.241039 mon.a (mon.0) 1453 : audit 1 from='client.? 172.21.15.89:0/3348868355' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-06-03T20:24:44.510 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:24:44.514 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:24:44.516 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:24:44.522 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:24:44 smithi089 bash[78452]: cluster 1685823883.0468597 mgr.y (mgr.34104) 1465 : cluster 0 pgmap v1488: 89 pgs: 2 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 6 active+undersized+degraded+remapped+backfill_wait, 75 active+clean; 4.5 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 583 op/s; 55021/219567 objects degraded (25.059%); 9317/219567 objects misplaced (4.243%); 858 KiB/s, 13 objects/s recovering 2023-06-03T20:24:44.522 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:24:44 smithi089 bash[78452]: audit 1685823883.241039 mon.a (mon.0) 1453 : audit 1 from='client.? 172.21.15.89:0/3348868355' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-06-03T20:24:44.529 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:24:44 smithi089 bash[75068]: cluster 1685823883.0468597 mgr.y (mgr.34104) 1465 : cluster 0 pgmap v1488: 89 pgs: 2 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 6 active+undersized+degraded+remapped+backfill_wait, 75 active+clean; 4.5 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 583 op/s; 55021/219567 objects degraded (25.059%); 9317/219567 objects misplaced (4.243%); 858 KiB/s, 13 objects/s recovering 2023-06-03T20:24:44.529 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:24:44 smithi089 bash[75068]: audit 1685823883.241039 mon.a (mon.0) 1453 : audit 1 from='client.? 172.21.15.89:0/3348868355' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-06-03T20:24:44.617 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:24:44.649 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:24:44.655 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:24:44.656 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:24:44.757 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:24:44.790 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:24:44.794 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:24:44.795 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:24:44.897 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:24:44.929 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:24:44.934 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:24:44.934 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:24:45.035 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:24:45.077 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:24:45.085 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:24:45.086 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:24:45.187 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:24:45.223 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:24:45.230 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:24:45 smithi089 bash[78452]: cluster 1685823884.2080247 mon.a (mon.0) 1454 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-06-03T20:24:45.230 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:24:45 smithi089 bash[78452]: cluster 1685823884.2136557 osd.5 (osd.5) 201 : cluster 3 failed to encode map e258 with expected crc 2023-06-03T20:24:45.230 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:24:45 smithi089 bash[78452]: audit 1685823884.2183826 mon.a (mon.0) 1455 : audit 1 from='client.? 172.21.15.89:0/3348868355' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-06-03T20:24:45.231 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:24:45 smithi089 bash[78452]: cluster 1685823884.2185726 mon.a (mon.0) 1456 : cluster 0 osdmap e258: 8 total, 8 up, 8 in 2023-06-03T20:24:45.231 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:24:45 smithi089 bash[78452]: cluster 1685823884.2362788 osd.6 (osd.6) 198 : cluster 3 failed to encode map e258 with expected crc 2023-06-03T20:24:45.231 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:24:45 smithi089 bash[75068]: cluster 1685823884.2080247 mon.a (mon.0) 1454 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-06-03T20:24:45.231 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:24:45 smithi089 bash[75068]: cluster 1685823884.2136557 osd.5 (osd.5) 201 : cluster 3 failed to encode map e258 with expected crc 2023-06-03T20:24:45.231 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:24:45 smithi089 bash[75068]: audit 1685823884.2183826 mon.a (mon.0) 1455 : audit 1 from='client.? 172.21.15.89:0/3348868355' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-06-03T20:24:45.231 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:24:45 smithi089 bash[75068]: cluster 1685823884.2185726 mon.a (mon.0) 1456 : cluster 0 osdmap e258: 8 total, 8 up, 8 in 2023-06-03T20:24:45.232 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:24:45 smithi089 bash[75068]: cluster 1685823884.2362788 osd.6 (osd.6) 198 : cluster 3 failed to encode map e258 with expected crc 2023-06-03T20:24:45.232 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:24:45.233 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:24:45.333 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:24:45.334 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:24:45 smithi164 bash[59462]: cluster 1685823884.2080247 mon.a (mon.0) 1454 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-06-03T20:24:45.334 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:24:45 smithi164 bash[59462]: cluster 1685823884.2136557 osd.5 (osd.5) 201 : cluster 3 failed to encode map e258 with expected crc 2023-06-03T20:24:45.335 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:24:45 smithi164 bash[59462]: audit 1685823884.2183826 mon.a (mon.0) 1455 : audit 1 from='client.? 172.21.15.89:0/3348868355' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-06-03T20:24:45.335 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:24:45 smithi164 bash[59462]: cluster 1685823884.2185726 mon.a (mon.0) 1456 : cluster 0 osdmap e258: 8 total, 8 up, 8 in 2023-06-03T20:24:45.335 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:24:45 smithi164 bash[59462]: cluster 1685823884.2362788 osd.6 (osd.6) 198 : cluster 3 failed to encode map e258 with expected crc 2023-06-03T20:24:45.366 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:24:45.369 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:24:45.369 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:24:45.470 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:24:45.476 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/mon.a/config 2023-06-03T20:24:45.509 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:24:45.515 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:24:45.515 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:24:45.616 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:24:45.644 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:24:45.652 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:24:45.654 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:24:45.755 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:24:45.781 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:24:45.787 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:24:45.788 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:24:45.889 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:24:45.936 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:24:45.943 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:24:45.944 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:24:46.045 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:24:46.095 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:24:46.105 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:24:46.106 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:24:46.206 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:24:46.247 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:24:46.254 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:24:46.255 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:24:46.356 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:24:46.398 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:24:46.404 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:24:46.405 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:24:46.506 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:24:46.515 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 20:24:46 smithi089 bash[71163]: ::ffff:172.21.15.164 - - [03/Jun/2023:20:24:46] "GET /metrics HTTP/1.1" 200 228929 "" "Prometheus/2.33.4" 2023-06-03T20:24:46.516 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:24:46 smithi089 bash[78452]: cluster 1685823885.047674 mgr.y (mgr.34104) 1466 : cluster 0 pgmap v1490: 89 pgs: 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 6 active+undersized+degraded+remapped+backfill_wait, 75 active+clean; 4.4 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 534 op/s; 54482/216663 objects degraded (25.146%); 9164/216663 objects misplaced (4.230%); 1.0 MiB/s, 16 objects/s recovering 2023-06-03T20:24:46.516 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:24:46 smithi089 bash[75068]: cluster 1685823885.047674 mgr.y (mgr.34104) 1466 : cluster 0 pgmap v1490: 89 pgs: 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 6 active+undersized+degraded+remapped+backfill_wait, 75 active+clean; 4.4 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 534 op/s; 54482/216663 objects degraded (25.146%); 9164/216663 objects misplaced (4.230%); 1.0 MiB/s, 16 objects/s recovering 2023-06-03T20:24:46.534 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:24:46.537 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:24:46.538 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:24:46.584 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:24:46 smithi164 bash[59462]: cluster 1685823885.047674 mgr.y (mgr.34104) 1466 : cluster 0 pgmap v1490: 89 pgs: 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 6 active+undersized+degraded+remapped+backfill_wait, 75 active+clean; 4.4 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 534 op/s; 54482/216663 objects degraded (25.146%); 9164/216663 objects misplaced (4.230%); 1.0 MiB/s, 16 objects/s recovering 2023-06-03T20:24:46.639 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:24:46.671 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:24:46.678 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:24:46.679 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:24:46.780 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:24:46.829 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:24:46.834 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:24:46.835 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:24:46.936 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:24:46.970 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:24:46.975 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:24:46.976 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:24:47.077 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:24:47.122 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:24:47.125 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:24:47.126 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:24:47.227 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:24:47.236 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-06-03T20:24:47.269 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:24:47.275 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:24:47.276 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:24:47.377 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:24:47.425 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:24:47.431 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:24:47.433 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:24:47.536 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:24:47.576 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:24:47.578 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:24:47.579 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:24:47.680 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:24:47.734 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:24:47.744 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:24:47.745 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:24:47.837 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/mon.a/config 2023-06-03T20:24:47.847 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:24:47.885 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:24:47.888 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:24:47.889 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:24:47.990 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:24:48.025 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:24:48.031 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:24:48.032 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:24:48.133 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:24:48.181 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:24:48.186 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:24:48.186 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:24:48.287 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:24:48.312 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:24:48.314 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:24:48.315 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:24:48.397 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:24:48 smithi089 bash[75068]: cluster 1685823887.0484598 mgr.y (mgr.34104) 1467 : cluster 0 pgmap v1491: 89 pgs: 2 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 6 active+undersized+degraded+remapped+backfill_wait, 75 active+clean; 4.4 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 530 op/s; 54482/214305 objects degraded (25.423%); 9009/214305 objects misplaced (4.204%); 1.0 MiB/s, 16 objects/s recovering 2023-06-03T20:24:48.398 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:24:48 smithi089 bash[75068]: cluster 1685823887.9468179 mon.a (mon.0) 1457 : cluster 3 Health check update: Degraded data redundancy: 54482/214305 objects degraded (25.423%), 13 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2023-06-03T20:24:48.416 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:24:48.441 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:24:48.443 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:24:48.444 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:24:48.545 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:24:48.556 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:24:48 smithi164 bash[59462]: cluster 1685823887.0484598 mgr.y (mgr.34104) 1467 : cluster 0 pgmap v1491: 89 pgs: 2 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 6 active+undersized+degraded+remapped+backfill_wait, 75 active+clean; 4.4 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 530 op/s; 54482/214305 objects degraded (25.423%); 9009/214305 objects misplaced (4.204%); 1.0 MiB/s, 16 objects/s recovering 2023-06-03T20:24:48.556 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:24:48 smithi164 bash[59462]: cluster 1685823887.9468179 mon.a (mon.0) 1457 : cluster 3 Health check update: Degraded data redundancy: 54482/214305 objects degraded (25.423%), 13 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2023-06-03T20:24:48.593 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:24:48.596 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:24:48.597 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:24:48.698 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:24:48.736 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:24:48.742 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:24:48.743 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:24:48.779 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:24:48 smithi089 bash[78452]: cluster 1685823887.0484598 mgr.y (mgr.34104) 1467 : cluster 0 pgmap v1491: 89 pgs: 2 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 6 active+undersized+degraded+remapped+backfill_wait, 75 active+clean; 4.4 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 530 op/s; 54482/214305 objects degraded (25.423%); 9009/214305 objects misplaced (4.204%); 1.0 MiB/s, 16 objects/s recovering 2023-06-03T20:24:48.779 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:24:48 smithi089 bash[78452]: cluster 1685823887.9468179 mon.a (mon.0) 1457 : cluster 3 Health check update: Degraded data redundancy: 54482/214305 objects degraded (25.423%), 13 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2023-06-03T20:24:48.844 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:24:48.879 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:24:48.881 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:24:48.882 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:24:48.983 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:24:49.017 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:24:49.021 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:24:49.022 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:24:49.123 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:24:49.162 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:24:49.165 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:24:49.166 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:24:49.266 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:24:49.309 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:24:49.313 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:24:49.313 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:24:49.369 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph osd unset nodeep-scrub 2023-06-03T20:24:49.414 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:24:49.448 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:24:49.451 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:24:49.452 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:24:49.554 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:24:49.620 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:24:49.625 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:24:49.626 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:24:49.727 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:24:49.757 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:24:49.765 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:24:49.765 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:24:49.866 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:24:49.909 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:24:49.910 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:24:49.910 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:24:50.011 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:24:50.052 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:24:50.060 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:24:50.061 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:24:50.162 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:24:50.193 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:24:50.199 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:24:50.200 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:24:50.301 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:24:50.339 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:24:50.340 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:24:50.340 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:24:50.442 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:24:50.495 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:24:50.502 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:24:50.503 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:24:50.529 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:24:50 smithi089 bash[78452]: cluster 1685823889.0495634 mgr.y (mgr.34104) 1468 : cluster 0 pgmap v1492: 89 pgs: 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 6 active+undersized+degraded+remapped+backfill_wait, 75 active+clean; 4.3 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 529 op/s; 53724/210288 objects degraded (25.548%); 8855/210288 objects misplaced (4.211%); 1.1 MiB/s, 17 objects/s recovering 2023-06-03T20:24:50.529 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:24:50 smithi089 bash[75068]: cluster 1685823889.0495634 mgr.y (mgr.34104) 1468 : cluster 0 pgmap v1492: 89 pgs: 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 6 active+undersized+degraded+remapped+backfill_wait, 75 active+clean; 4.3 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 529 op/s; 53724/210288 objects degraded (25.548%); 8855/210288 objects misplaced (4.211%); 1.1 MiB/s, 17 objects/s recovering 2023-06-03T20:24:50.584 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:24:50 smithi164 bash[59462]: cluster 1685823889.0495634 mgr.y (mgr.34104) 1468 : cluster 0 pgmap v1492: 89 pgs: 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 6 active+undersized+degraded+remapped+backfill_wait, 75 active+clean; 4.3 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 529 op/s; 53724/210288 objects degraded (25.548%); 8855/210288 objects misplaced (4.211%); 1.1 MiB/s, 17 objects/s recovering 2023-06-03T20:24:50.604 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:24:50.654 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:24:50.657 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:24:50.658 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:24:50.759 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:24:50.812 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:24:50.815 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:24:50.816 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:24:50.917 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:24:50.951 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:24:50.955 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:24:50.956 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:24:51.057 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:24:51.087 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:24:51.089 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:24:51.090 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:24:51.191 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:24:51.224 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:24:51.229 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:24:51.230 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:24:51.331 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:24:51.366 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:24:51.369 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:24:51.370 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:24:51.471 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:24:51.520 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:24:51.522 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:24:51.523 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:24:51.624 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:24:51.658 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:24:51.665 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:24:51.666 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:24:51.767 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:24:51.810 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:24:51.813 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:24:51.814 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:24:51.915 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:24:51.957 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:24:51.962 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:24:51.963 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:24:52.065 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:24:52.108 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:24:52.111 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:24:52.112 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:24:52.213 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:24:52.254 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:24:52.258 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:24:52.258 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:24:52.335 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:24:52 smithi164 bash[59462]: cluster 1685823891.0502853 mgr.y (mgr.34104) 1469 : cluster 0 pgmap v1493: 89 pgs: 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 6 active+undersized+degraded+remapped+backfill_wait, 75 active+clean; 4.2 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 525 op/s; 53397/208881 objects degraded (25.563%); 8855/208881 objects misplaced (4.239%); 1.1 MiB/s, 17 objects/s recovering 2023-06-03T20:24:52.335 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:24:52 smithi164 bash[59462]: audit 1685823892.144682 mon.c (mon.2) 617 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T20:24:52.360 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:24:52.388 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:24:52.393 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:24:52.394 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:24:52.495 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:24:52.504 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:24:52 smithi089 bash[75068]: cluster 1685823891.0502853 mgr.y (mgr.34104) 1469 : cluster 0 pgmap v1493: 89 pgs: 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 6 active+undersized+degraded+remapped+backfill_wait, 75 active+clean; 4.2 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 525 op/s; 53397/208881 objects degraded (25.563%); 8855/208881 objects misplaced (4.239%); 1.1 MiB/s, 17 objects/s recovering 2023-06-03T20:24:52.504 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:24:52 smithi089 bash[75068]: audit 1685823892.144682 mon.c (mon.2) 617 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T20:24:52.504 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:24:52 smithi089 bash[78452]: cluster 1685823891.0502853 mgr.y (mgr.34104) 1469 : cluster 0 pgmap v1493: 89 pgs: 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 6 active+undersized+degraded+remapped+backfill_wait, 75 active+clean; 4.2 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 525 op/s; 53397/208881 objects degraded (25.563%); 8855/208881 objects misplaced (4.239%); 1.1 MiB/s, 17 objects/s recovering 2023-06-03T20:24:52.505 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:24:52 smithi089 bash[78452]: audit 1685823892.144682 mon.c (mon.2) 617 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T20:24:52.546 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:24:52.547 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:24:52.547 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:24:52.649 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:24:52.683 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:24:52.689 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:24:52.690 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:24:52.707 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/mon.a/config 2023-06-03T20:24:52.791 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:24:52.825 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:24:52.830 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:24:52.831 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:24:52.932 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:24:52.964 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:24:52.969 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:24:52.970 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:24:53.071 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:24:53.101 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:24:53.113 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:24:53.114 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:24:53.215 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:24:53.256 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:24:53.258 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:24:53.259 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:24:53.270 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:24:53 smithi089 bash[75068]: cluster 1685823892.9478054 mon.a (mon.0) 1458 : cluster 3 Health check update: Degraded data redundancy: 53397/208881 objects degraded (25.563%), 13 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2023-06-03T20:24:53.270 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:24:53 smithi089 bash[78452]: cluster 1685823892.9478054 mon.a (mon.0) 1458 : cluster 3 Health check update: Degraded data redundancy: 53397/208881 objects degraded (25.563%), 13 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2023-06-03T20:24:53.334 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:24:53 smithi164 bash[59462]: cluster 1685823892.9478054 mon.a (mon.0) 1458 : cluster 3 Health check update: Degraded data redundancy: 53397/208881 objects degraded (25.563%), 13 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2023-06-03T20:24:53.360 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:24:53.412 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:24:53.421 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:24:53.422 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:24:53.523 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:24:53.551 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:24:53.553 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:24:53.554 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:24:53.654 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:24:53.687 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:24:53.688 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:24:53.688 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:24:53.789 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:24:53.823 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:24:53.827 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:24:53.828 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:24:53.930 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:24:53.987 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:24:53.988 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:24:53.989 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:24:54.090 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:24:54.144 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:24:54.151 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:24:54.152 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:24:54.253 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:24:54.285 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:24:54.287 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:24:54.288 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:24:54.389 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:24:54.416 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:24:54.422 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:24:54.423 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:24:54.524 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:24:54.529 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:24:54 smithi089 bash[75068]: cluster 1685823893.050913 mgr.y (mgr.34104) 1470 : cluster 0 pgmap v1494: 89 pgs: 2 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 6 active+undersized+degraded+remapped+backfill_wait, 75 active+clean; 4.1 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 521 op/s; 52893/203916 objects degraded (25.939%); 8706/203916 objects misplaced (4.269%); 1.1 MiB/s, 17 objects/s recovering 2023-06-03T20:24:54.530 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:24:54 smithi089 bash[78452]: cluster 1685823893.050913 mgr.y (mgr.34104) 1470 : cluster 0 pgmap v1494: 89 pgs: 2 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 6 active+undersized+degraded+remapped+backfill_wait, 75 active+clean; 4.1 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 521 op/s; 52893/203916 objects degraded (25.939%); 8706/203916 objects misplaced (4.269%); 1.1 MiB/s, 17 objects/s recovering 2023-06-03T20:24:54.553 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:24:54.560 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:24:54.561 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:24:54.586 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:24:54 smithi164 bash[59462]: cluster 1685823893.050913 mgr.y (mgr.34104) 1470 : cluster 0 pgmap v1494: 89 pgs: 2 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 6 active+undersized+degraded+remapped+backfill_wait, 75 active+clean; 4.1 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 521 op/s; 52893/203916 objects degraded (25.939%); 8706/203916 objects misplaced (4.269%); 1.1 MiB/s, 17 objects/s recovering 2023-06-03T20:24:54.662 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:24:54.694 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:24:54.698 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:24:54.698 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:24:54.799 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:24:54.839 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:24:54.847 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:24:54.848 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:24:54.950 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:24:54.980 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:24:54.982 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:24:54.983 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:24:55.084 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:24:55.119 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:24:55.123 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:24:55.124 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:24:55.225 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:24:55.277 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:24:55.283 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:24:55.284 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:24:55.385 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:24:55.424 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:24:55.430 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:24:55.431 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:24:55.532 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:24:55.574 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:24:55.581 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:24:55.582 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:24:55.683 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:24:55.721 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:24:55.723 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:24:55.723 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:24:55.824 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:24:55.863 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:24:55.867 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:24:55.868 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:24:55.969 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:24:56.000 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:24:56.002 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:24:56.003 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:24:56.104 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:24:56.136 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:24:56.142 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:24:56.143 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:24:56.244 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:24:56.293 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:24:56.297 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:24:56.298 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:24:56.372 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:24:56 smithi089 bash[78452]: cluster 1685823895.051687 mgr.y (mgr.34104) 1471 : cluster 0 pgmap v1495: 89 pgs: 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 6 active+undersized+degraded+remapped+backfill_wait, 75 active+clean; 4.1 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 524 op/s; 52318/201048 objects degraded (26.023%); 8540/201048 objects misplaced (4.248%); 1.0 MiB/s, 16 objects/s recovering 2023-06-03T20:24:56.373 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:24:56 smithi089 bash[75068]: cluster 1685823895.051687 mgr.y (mgr.34104) 1471 : cluster 0 pgmap v1495: 89 pgs: 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 6 active+undersized+degraded+remapped+backfill_wait, 75 active+clean; 4.1 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 524 op/s; 52318/201048 objects degraded (26.023%); 8540/201048 objects misplaced (4.248%); 1.0 MiB/s, 16 objects/s recovering 2023-06-03T20:24:56.373 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 20:24:56 smithi089 bash[71163]: ::ffff:172.21.15.164 - - [03/Jun/2023:20:24:56] "GET /metrics HTTP/1.1" 200 228946 "" "Prometheus/2.33.4" 2023-06-03T20:24:56.399 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:24:56.439 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:24:56.442 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:24:56.442 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:24:56.543 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:24:56.554 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:24:56 smithi164 bash[59462]: cluster 1685823895.051687 mgr.y (mgr.34104) 1471 : cluster 0 pgmap v1495: 89 pgs: 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 6 active+undersized+degraded+remapped+backfill_wait, 75 active+clean; 4.1 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 524 op/s; 52318/201048 objects degraded (26.023%); 8540/201048 objects misplaced (4.248%); 1.0 MiB/s, 16 objects/s recovering 2023-06-03T20:24:56.578 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:24:56.584 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:24:56.585 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:24:56.687 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:24:56.721 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:24:56.723 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:24:56.724 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:24:56.817 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/mon.a/config 2023-06-03T20:24:56.825 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:24:56.864 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:24:56.867 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:24:56.868 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:24:56.969 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:24:57.005 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:24:57.009 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:24:57.009 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:24:57.110 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:24:57.177 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:24:57.183 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:24:57.183 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:24:57.284 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:24:57.328 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:24:57.335 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:24:57.335 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:24:57.436 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:24:57.469 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:24:57.471 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:24:57.472 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:24:57.573 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:24:57.606 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:24:57.610 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:24:57.611 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:24:57.713 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:24:57.745 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:24:57.752 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:24:57.753 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:24:57.854 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:24:57.904 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:24:57.909 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:24:57.910 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:24:58.011 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:24:58.045 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:24:58.051 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:24:58.052 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:24:58.153 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:24:58.188 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:24:58.190 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:24:58.191 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:24:58.292 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:24:58.355 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:24:58.358 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:24:58.359 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:24:58.460 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:24:58.469 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:24:58 smithi089 bash[78452]: cluster 1685823897.0522575 mgr.y (mgr.34104) 1472 : cluster 0 pgmap v1496: 89 pgs: 2 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 6 active+undersized+degraded+remapped+backfill_wait, 75 active+clean; 4.0 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 497 op/s; 52318/198732 objects degraded (26.326%); 8389/198732 objects misplaced (4.221%); 938 KiB/s, 14 objects/s recovering 2023-06-03T20:24:58.469 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:24:58 smithi089 bash[78452]: cluster 1685823897.9489992 mon.a (mon.0) 1459 : cluster 3 Health check update: Degraded data redundancy: 52318/198732 objects degraded (26.326%), 13 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2023-06-03T20:24:58.470 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:24:58 smithi089 bash[75068]: cluster 1685823897.0522575 mgr.y (mgr.34104) 1472 : cluster 0 pgmap v1496: 89 pgs: 2 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 6 active+undersized+degraded+remapped+backfill_wait, 75 active+clean; 4.0 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 497 op/s; 52318/198732 objects degraded (26.326%); 8389/198732 objects misplaced (4.221%); 938 KiB/s, 14 objects/s recovering 2023-06-03T20:24:58.470 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:24:58 smithi089 bash[75068]: cluster 1685823897.9489992 mon.a (mon.0) 1459 : cluster 3 Health check update: Degraded data redundancy: 52318/198732 objects degraded (26.326%), 13 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2023-06-03T20:24:58.493 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:24:58.500 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:24:58.501 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:24:58.585 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:24:58 smithi164 bash[59462]: cluster 1685823897.0522575 mgr.y (mgr.34104) 1472 : cluster 0 pgmap v1496: 89 pgs: 2 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 6 active+undersized+degraded+remapped+backfill_wait, 75 active+clean; 4.0 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 497 op/s; 52318/198732 objects degraded (26.326%); 8389/198732 objects misplaced (4.221%); 938 KiB/s, 14 objects/s recovering 2023-06-03T20:24:58.585 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:24:58 smithi164 bash[59462]: cluster 1685823897.9489992 mon.a (mon.0) 1459 : cluster 3 Health check update: Degraded data redundancy: 52318/198732 objects degraded (26.326%), 13 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2023-06-03T20:24:58.602 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:24:58.663 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:24:58.667 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:24:58.667 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:24:58.768 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:24:58.817 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:24:58.822 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:24:58.823 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:24:58.924 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:24:58.962 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:24:58.967 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:24:58.967 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:24:59.068 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:24:59.108 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:24:59.113 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:24:59.114 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:24:59.215 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:24:59.260 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:24:59.263 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:24:59.264 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:24:59.301 INFO:teuthology.orchestra.run.smithi089.stderr:set pool 13 pg_num to 26 2023-06-03T20:24:59.334 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:24:59 smithi164 bash[59462]: audit 1685823899.260933 mon.a (mon.0) 1460 : audit 1 from='client.? 172.21.15.89:0/1579185385' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_8", "var": "pg_num", "val": "26"}]: dispatch 2023-06-03T20:24:59.365 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:24:59.392 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:24:59.394 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:24:59.394 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:24:59.495 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:24:59.504 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:24:59 smithi089 bash[75068]: audit 1685823899.260933 mon.a (mon.0) 1460 : audit 1 from='client.? 172.21.15.89:0/1579185385' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_8", "var": "pg_num", "val": "26"}]: dispatch 2023-06-03T20:24:59.505 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:24:59 smithi089 bash[78452]: audit 1685823899.260933 mon.a (mon.0) 1460 : audit 1 from='client.? 172.21.15.89:0/1579185385' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_8", "var": "pg_num", "val": "26"}]: dispatch 2023-06-03T20:24:59.521 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:24:59.525 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:24:59.526 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:24:59.627 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:24:59.701 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:24:59.707 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:24:59.708 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:24:59.809 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:24:59.852 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:24:59.856 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:24:59.856 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:24:59.958 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:25:00.006 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:25:00.009 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:25:00.010 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:25:00.111 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:25:00.151 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:25:00.155 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:25:00.160 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:25:00.261 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:25:00.292 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:25:00.299 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:25:00.300 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:25:00.401 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:25:00.444 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:25:00.447 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:25:00.448 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:25:00.529 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:25:00 smithi089 bash[78452]: cluster 1685823899.0530114 mgr.y (mgr.34104) 1473 : cluster 0 pgmap v1497: 89 pgs: 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 6 active+undersized+degraded+remapped+backfill_wait, 75 active+clean; 4.0 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 542 op/s; 51578/194784 objects degraded (26.480%); 8221/194784 objects misplaced (4.221%); 1.4 MiB/s, 22 objects/s recovering 2023-06-03T20:25:00.529 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:25:00 smithi089 bash[78452]: cluster 1685823899.2884848 osd.5 (osd.5) 202 : cluster 3 failed to encode map e259 with expected crc 2023-06-03T20:25:00.530 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:25:00 smithi089 bash[78452]: cluster 1685823899.290059 osd.7 (osd.7) 198 : cluster 3 failed to encode map e259 with expected crc 2023-06-03T20:25:00.530 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:25:00 smithi089 bash[78452]: audit 1685823899.2918925 mon.a (mon.0) 1461 : audit 1 from='client.? 172.21.15.89:0/1579185385' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_8", "var": "pg_num", "val": "26"}]': finished 2023-06-03T20:25:00.530 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:25:00 smithi089 bash[78452]: cluster 1685823899.2919538 mon.a (mon.0) 1462 : cluster 0 osdmap e259: 8 total, 8 up, 8 in 2023-06-03T20:25:00.530 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:25:00 smithi089 bash[75068]: cluster 1685823899.0530114 mgr.y (mgr.34104) 1473 : cluster 0 pgmap v1497: 89 pgs: 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 6 active+undersized+degraded+remapped+backfill_wait, 75 active+clean; 4.0 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 542 op/s; 51578/194784 objects degraded (26.480%); 8221/194784 objects misplaced (4.221%); 1.4 MiB/s, 22 objects/s recovering 2023-06-03T20:25:00.530 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:25:00 smithi089 bash[75068]: cluster 1685823899.2884848 osd.5 (osd.5) 202 : cluster 3 failed to encode map e259 with expected crc 2023-06-03T20:25:00.530 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:25:00 smithi089 bash[75068]: cluster 1685823899.290059 osd.7 (osd.7) 198 : cluster 3 failed to encode map e259 with expected crc 2023-06-03T20:25:00.531 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:25:00 smithi089 bash[75068]: audit 1685823899.2918925 mon.a (mon.0) 1461 : audit 1 from='client.? 172.21.15.89:0/1579185385' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_8", "var": "pg_num", "val": "26"}]': finished 2023-06-03T20:25:00.531 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:25:00 smithi089 bash[75068]: cluster 1685823899.2919538 mon.a (mon.0) 1462 : cluster 0 osdmap e259: 8 total, 8 up, 8 in 2023-06-03T20:25:00.548 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:25:00.557 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:25:00 smithi164 bash[59462]: cluster 1685823899.0530114 mgr.y (mgr.34104) 1473 : cluster 0 pgmap v1497: 89 pgs: 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 6 active+undersized+degraded+remapped+backfill_wait, 75 active+clean; 4.0 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 542 op/s; 51578/194784 objects degraded (26.480%); 8221/194784 objects misplaced (4.221%); 1.4 MiB/s, 22 objects/s recovering 2023-06-03T20:25:00.557 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:25:00 smithi164 bash[59462]: cluster 1685823899.2884848 osd.5 (osd.5) 202 : cluster 3 failed to encode map e259 with expected crc 2023-06-03T20:25:00.557 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:25:00 smithi164 bash[59462]: cluster 1685823899.290059 osd.7 (osd.7) 198 : cluster 3 failed to encode map e259 with expected crc 2023-06-03T20:25:00.558 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:25:00 smithi164 bash[59462]: audit 1685823899.2918925 mon.a (mon.0) 1461 : audit 1 from='client.? 172.21.15.89:0/1579185385' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_8", "var": "pg_num", "val": "26"}]': finished 2023-06-03T20:25:00.558 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:25:00 smithi164 bash[59462]: cluster 1685823899.2919538 mon.a (mon.0) 1462 : cluster 0 osdmap e259: 8 total, 8 up, 8 in 2023-06-03T20:25:00.580 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:25:00.586 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:25:00.586 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:25:00.687 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:25:00.724 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:25:00.727 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:25:00.728 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:25:00.829 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:25:00.872 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:25:00.874 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:25:00.875 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:25:00.976 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:25:01.024 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:25:01.028 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:25:01.028 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:25:01.129 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:25:01.159 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:25:01.168 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:25:01.169 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:25:01.270 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:25:01.302 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:25:01.305 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:25:01.306 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:25:01.308 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:25:01 smithi164 bash[59462]: cluster 1685823901.0536435 mgr.y (mgr.34104) 1474 : cluster 0 pgmap v1499: 89 pgs: 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 6 active+undersized+degraded+remapped+backfill_wait, 75 active+clean; 3.9 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 518 op/s; 51249/193335 objects degraded (26.508%); 8221/193335 objects misplaced (4.252%); 1.1 MiB/s, 17 objects/s recovering 2023-06-03T20:25:01.308 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:25:01 smithi164 bash[59462]: audit 1685823901.0542595 mon.c (mon.2) 618 : audit 1 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_8", "var": "pg_num_actual", "val": "26"}]: dispatch 2023-06-03T20:25:01.309 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:25:01 smithi164 bash[59462]: audit 1685823901.0554008 mon.a (mon.0) 1463 : audit 1 from='mgr.34104 ' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_8", "var": "pg_num_actual", "val": "26"}]: dispatch 2023-06-03T20:25:01.406 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:25:01.497 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:25:01.505 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:25:01.507 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:25:01.608 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:25:01.779 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:25:01 smithi089 bash[78452]: cluster 1685823901.0536435 mgr.y (mgr.34104) 1474 : cluster 0 pgmap v1499: 89 pgs: 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 6 active+undersized+degraded+remapped+backfill_wait, 75 active+clean; 3.9 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 518 op/s; 51249/193335 objects degraded (26.508%); 8221/193335 objects misplaced (4.252%); 1.1 MiB/s, 17 objects/s recovering 2023-06-03T20:25:01.779 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:25:01 smithi089 bash[78452]: audit 1685823901.0542595 mon.c (mon.2) 618 : audit 1 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_8", "var": "pg_num_actual", "val": "26"}]: dispatch 2023-06-03T20:25:01.779 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:25:01 smithi089 bash[78452]: audit 1685823901.0554008 mon.a (mon.0) 1463 : audit 1 from='mgr.34104 ' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_8", "var": "pg_num_actual", "val": "26"}]: dispatch 2023-06-03T20:25:01.780 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:25:01 smithi089 bash[75068]: cluster 1685823901.0536435 mgr.y (mgr.34104) 1474 : cluster 0 pgmap v1499: 89 pgs: 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 6 active+undersized+degraded+remapped+backfill_wait, 75 active+clean; 3.9 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 518 op/s; 51249/193335 objects degraded (26.508%); 8221/193335 objects misplaced (4.252%); 1.1 MiB/s, 17 objects/s recovering 2023-06-03T20:25:01.780 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:25:01 smithi089 bash[75068]: audit 1685823901.0542595 mon.c (mon.2) 618 : audit 1 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_8", "var": "pg_num_actual", "val": "26"}]: dispatch 2023-06-03T20:25:01.780 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:25:01 smithi089 bash[75068]: audit 1685823901.0554008 mon.a (mon.0) 1463 : audit 1 from='mgr.34104 ' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_8", "var": "pg_num_actual", "val": "26"}]: dispatch 2023-06-03T20:25:01.834 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:25:01.837 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:25:01.838 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:25:01.939 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:25:01.964 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:25:01.967 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:25:01.968 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:25:02.069 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:25:02.099 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:25:02.101 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:25:02.102 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:25:02.203 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:25:02.232 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:25:02.234 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:25:02.235 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:25:02.336 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:25:02.372 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:25:02.374 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:25:02.375 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:25:02.476 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:25:02.510 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:25:02.514 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:25:02.515 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:25:02.584 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:25:02 smithi164 bash[59462]: cluster 1685823901.2976623 osd.6 (osd.6) 199 : cluster 3 failed to encode map e260 with expected crc 2023-06-03T20:25:02.585 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:25:02 smithi164 bash[59462]: cluster 1685823901.29833 osd.5 (osd.5) 203 : cluster 3 failed to encode map e260 with expected crc 2023-06-03T20:25:02.585 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:25:02 smithi164 bash[59462]: audit 1685823901.3029215 mon.a (mon.0) 1464 : audit 1 from='mgr.34104 ' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_8", "var": "pg_num_actual", "val": "26"}]': finished 2023-06-03T20:25:02.585 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:25:02 smithi164 bash[59462]: cluster 1685823901.302987 mon.a (mon.0) 1465 : cluster 0 osdmap e260: 8 total, 8 up, 8 in 2023-06-03T20:25:02.616 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:25:02.646 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:25:02.654 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:25:02.655 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:25:02.713 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:25:02 smithi089 bash[78452]: cluster 1685823901.2976623 osd.6 (osd.6) 199 : cluster 3 failed to encode map e260 with expected crc 2023-06-03T20:25:02.713 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:25:02 smithi089 bash[78452]: cluster 1685823901.29833 osd.5 (osd.5) 203 : cluster 3 failed to encode map e260 with expected crc 2023-06-03T20:25:02.713 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:25:02 smithi089 bash[78452]: audit 1685823901.3029215 mon.a (mon.0) 1464 : audit 1 from='mgr.34104 ' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_8", "var": "pg_num_actual", "val": "26"}]': finished 2023-06-03T20:25:02.713 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:25:02 smithi089 bash[78452]: cluster 1685823901.302987 mon.a (mon.0) 1465 : cluster 0 osdmap e260: 8 total, 8 up, 8 in 2023-06-03T20:25:02.714 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:25:02 smithi089 bash[75068]: cluster 1685823901.2976623 osd.6 (osd.6) 199 : cluster 3 failed to encode map e260 with expected crc 2023-06-03T20:25:02.714 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:25:02 smithi089 bash[75068]: cluster 1685823901.29833 osd.5 (osd.5) 203 : cluster 3 failed to encode map e260 with expected crc 2023-06-03T20:25:02.714 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:25:02 smithi089 bash[75068]: audit 1685823901.3029215 mon.a (mon.0) 1464 : audit 1 from='mgr.34104 ' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_8", "var": "pg_num_actual", "val": "26"}]': finished 2023-06-03T20:25:02.714 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:25:02 smithi089 bash[75068]: cluster 1685823901.302987 mon.a (mon.0) 1465 : cluster 0 osdmap e260: 8 total, 8 up, 8 in 2023-06-03T20:25:02.757 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:25:02.788 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:25:02.791 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:25:02.792 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:25:02.893 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:25:02.915 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:25:02.917 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:25:02.918 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:25:03.019 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:25:03.046 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:25:03.048 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:25:03.048 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:25:03.149 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:25:03.174 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:25:03.176 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:25:03.177 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:25:03.278 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:25:03.312 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:25:03.314 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:25:03.315 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:25:03.416 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:25:03.450 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:25:03.453 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:25:03.454 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:25:03.555 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:25:03.580 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:25:03.583 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:25:03.584 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:25:03.585 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:25:03 smithi164 bash[59462]: cluster 1685823902.3110914 osd.5 (osd.5) 204 : cluster 3 failed to encode map e261 with expected crc 2023-06-03T20:25:03.585 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:25:03 smithi164 bash[59462]: cluster 1685823902.3121772 osd.6 (osd.6) 200 : cluster 3 failed to encode map e261 with expected crc 2023-06-03T20:25:03.585 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:25:03 smithi164 bash[59462]: cluster 1685823902.3128903 mon.a (mon.0) 1466 : cluster 0 osdmap e261: 8 total, 8 up, 8 in 2023-06-03T20:25:03.586 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:25:03 smithi164 bash[59462]: cluster 1685823902.313928 osd.5 (osd.5) 205 : cluster 3 failed to encode map e261 with expected crc 2023-06-03T20:25:03.586 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:25:03 smithi164 bash[59462]: cluster 1685823902.3150625 osd.3 (osd.3) 19 : cluster 1 13.2 continuing backfill to osd.4 from (245'2300,259'10474] MIN to 259'10474 2023-06-03T20:25:03.586 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:25:03 smithi164 bash[59462]: cluster 1685823902.3157797 osd.5 (osd.5) 206 : cluster 3 failed to encode map e261 with expected crc 2023-06-03T20:25:03.587 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:25:03 smithi164 bash[59462]: cluster 1685823902.315992 osd.3 (osd.3) 20 : cluster 1 13.6 continuing backfill to osd.4 from (245'2200,259'10338] MIN to 259'10338 2023-06-03T20:25:03.587 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:25:03 smithi164 bash[59462]: cluster 1685823902.3169627 osd.5 (osd.5) 207 : cluster 3 failed to encode map e261 with expected crc 2023-06-03T20:25:03.587 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:25:03 smithi164 bash[59462]: cluster 1685823902.3214319 osd.0 (osd.0) 14 : cluster 1 13.5 continuing backfill to osd.4 from (245'1100,259'10560] MIN to 259'10560 2023-06-03T20:25:03.587 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:25:03 smithi164 bash[59462]: cluster 1685823902.3237362 osd.7 (osd.7) 199 : cluster 3 failed to encode map e261 with expected crc 2023-06-03T20:25:03.587 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:25:03 smithi164 bash[59462]: cluster 1685823902.323891 osd.0 (osd.0) 15 : cluster 0 13.3 starting backfill to osd.4 from (245'1000,259'10408] MIN to 259'10409 2023-06-03T20:25:03.588 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:25:03 smithi164 bash[59462]: cluster 1685823902.3266892 osd.7 (osd.7) 200 : cluster 3 failed to encode map e261 with expected crc 2023-06-03T20:25:03.588 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:25:03 smithi164 bash[59462]: cluster 1685823902.332581 osd.7 (osd.7) 201 : cluster 1 13.7 continuing backfill to osd.4 from (245'2300,259'10670] MIN to 259'10670 2023-06-03T20:25:03.588 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:25:03 smithi164 bash[59462]: cluster 1685823902.3371148 osd.7 (osd.7) 202 : cluster 0 13.7 starting backfill to osd.6 from (245'2300,259'10668] MIN to 259'10670 2023-06-03T20:25:03.588 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:25:03 smithi164 bash[59462]: cluster 1685823902.9503546 mon.a (mon.0) 1467 : cluster 3 Health check update: Degraded data redundancy: 51249/193335 objects degraded (26.508%), 13 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2023-06-03T20:25:03.588 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:25:03 smithi164 bash[59462]: cluster 1685823902.9608912 mon.a (mon.0) 1468 : cluster 0 osdmap e262: 8 total, 8 up, 8 in 2023-06-03T20:25:03.588 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:25:03 smithi164 bash[59462]: cluster 1685823903.0542338 mgr.y (mgr.34104) 1475 : cluster 0 pgmap v1503: 99 pgs: 1 active+remapped+backfilling, 1 peering, 2 remapped+peering, 9 unknown, 3 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 5 active+undersized+degraded+remapped+backfill_wait, 74 active+clean; 3.6 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 32571/177309 objects degraded (18.370%); 8040/177309 objects misplaced (4.534%); 136 KiB/s, 2 objects/s recovering 2023-06-03T20:25:03.589 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:25:03 smithi089 bash[75068]: cluster 1685823902.3110914 osd.5 (osd.5) 204 : cluster 3 failed to encode map e261 with expected crc 2023-06-03T20:25:03.590 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:25:03 smithi089 bash[75068]: cluster 1685823902.3121772 osd.6 (osd.6) 200 : cluster 3 failed to encode map e261 with expected crc 2023-06-03T20:25:03.590 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:25:03 smithi089 bash[75068]: cluster 1685823902.3128903 mon.a (mon.0) 1466 : cluster 0 osdmap e261: 8 total, 8 up, 8 in 2023-06-03T20:25:03.590 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:25:03 smithi089 bash[75068]: cluster 1685823902.313928 osd.5 (osd.5) 205 : cluster 3 failed to encode map e261 with expected crc 2023-06-03T20:25:03.590 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:25:03 smithi089 bash[75068]: cluster 1685823902.3150625 osd.3 (osd.3) 19 : cluster 1 13.2 continuing backfill to osd.4 from (245'2300,259'10474] MIN to 259'10474 2023-06-03T20:25:03.591 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:25:03 smithi089 bash[75068]: cluster 1685823902.3157797 osd.5 (osd.5) 206 : cluster 3 failed to encode map e261 with expected crc 2023-06-03T20:25:03.591 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:25:03 smithi089 bash[75068]: cluster 1685823902.315992 osd.3 (osd.3) 20 : cluster 1 13.6 continuing backfill to osd.4 from (245'2200,259'10338] MIN to 259'10338 2023-06-03T20:25:03.591 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:25:03 smithi089 bash[75068]: cluster 1685823902.3169627 osd.5 (osd.5) 207 : cluster 3 failed to encode map e261 with expected crc 2023-06-03T20:25:03.591 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:25:03 smithi089 bash[75068]: cluster 1685823902.3214319 osd.0 (osd.0) 14 : cluster 1 13.5 continuing backfill to osd.4 from (245'1100,259'10560] MIN to 259'10560 2023-06-03T20:25:03.591 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:25:03 smithi089 bash[75068]: cluster 1685823902.3237362 osd.7 (osd.7) 199 : cluster 3 failed to encode map e261 with expected crc 2023-06-03T20:25:03.591 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:25:03 smithi089 bash[75068]: cluster 1685823902.323891 osd.0 (osd.0) 15 : cluster 0 13.3 starting backfill to osd.4 from (245'1000,259'10408] MIN to 259'10409 2023-06-03T20:25:03.591 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:25:03 smithi089 bash[75068]: cluster 1685823902.3266892 osd.7 (osd.7) 200 : cluster 3 failed to encode map e261 with expected crc 2023-06-03T20:25:03.591 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:25:03 smithi089 bash[75068]: cluster 1685823902.332581 osd.7 (osd.7) 201 : cluster 1 13.7 continuing backfill to osd.4 from (245'2300,259'10670] MIN to 259'10670 2023-06-03T20:25:03.592 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:25:03 smithi089 bash[75068]: cluster 1685823902.3371148 osd.7 (osd.7) 202 : cluster 0 13.7 starting backfill to osd.6 from (245'2300,259'10668] MIN to 259'10670 2023-06-03T20:25:03.592 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:25:03 smithi089 bash[75068]: cluster 1685823902.9503546 mon.a (mon.0) 1467 : cluster 3 Health check update: Degraded data redundancy: 51249/193335 objects degraded (26.508%), 13 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2023-06-03T20:25:03.592 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:25:03 smithi089 bash[75068]: cluster 1685823902.9608912 mon.a (mon.0) 1468 : cluster 0 osdmap e262: 8 total, 8 up, 8 in 2023-06-03T20:25:03.592 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:25:03 smithi089 bash[75068]: cluster 1685823903.0542338 mgr.y (mgr.34104) 1475 : cluster 0 pgmap v1503: 99 pgs: 1 active+remapped+backfilling, 1 peering, 2 remapped+peering, 9 unknown, 3 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 5 active+undersized+degraded+remapped+backfill_wait, 74 active+clean; 3.6 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 32571/177309 objects degraded (18.370%); 8040/177309 objects misplaced (4.534%); 136 KiB/s, 2 objects/s recovering 2023-06-03T20:25:03.592 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:25:03 smithi089 bash[78452]: cluster 1685823902.3110914 osd.5 (osd.5) 204 : cluster 3 failed to encode map e261 with expected crc 2023-06-03T20:25:03.593 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:25:03 smithi089 bash[78452]: cluster 1685823902.3121772 osd.6 (osd.6) 200 : cluster 3 failed to encode map e261 with expected crc 2023-06-03T20:25:03.593 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:25:03 smithi089 bash[78452]: cluster 1685823902.3128903 mon.a (mon.0) 1466 : cluster 0 osdmap e261: 8 total, 8 up, 8 in 2023-06-03T20:25:03.593 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:25:03 smithi089 bash[78452]: cluster 1685823902.313928 osd.5 (osd.5) 205 : cluster 3 failed to encode map e261 with expected crc 2023-06-03T20:25:03.593 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:25:03 smithi089 bash[78452]: cluster 1685823902.3150625 osd.3 (osd.3) 19 : cluster 1 13.2 continuing backfill to osd.4 from (245'2300,259'10474] MIN to 259'10474 2023-06-03T20:25:03.593 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:25:03 smithi089 bash[78452]: cluster 1685823902.3157797 osd.5 (osd.5) 206 : cluster 3 failed to encode map e261 with expected crc 2023-06-03T20:25:03.593 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:25:03 smithi089 bash[78452]: cluster 1685823902.315992 osd.3 (osd.3) 20 : cluster 1 13.6 continuing backfill to osd.4 from (245'2200,259'10338] MIN to 259'10338 2023-06-03T20:25:03.593 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:25:03 smithi089 bash[78452]: cluster 1685823902.3169627 osd.5 (osd.5) 207 : cluster 3 failed to encode map e261 with expected crc 2023-06-03T20:25:03.593 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:25:03 smithi089 bash[78452]: cluster 1685823902.3214319 osd.0 (osd.0) 14 : cluster 1 13.5 continuing backfill to osd.4 from (245'1100,259'10560] MIN to 259'10560 2023-06-03T20:25:03.594 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:25:03 smithi089 bash[78452]: cluster 1685823902.3237362 osd.7 (osd.7) 199 : cluster 3 failed to encode map e261 with expected crc 2023-06-03T20:25:03.594 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:25:03 smithi089 bash[78452]: cluster 1685823902.323891 osd.0 (osd.0) 15 : cluster 0 13.3 starting backfill to osd.4 from (245'1000,259'10408] MIN to 259'10409 2023-06-03T20:25:03.594 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:25:03 smithi089 bash[78452]: cluster 1685823902.3266892 osd.7 (osd.7) 200 : cluster 3 failed to encode map e261 with expected crc 2023-06-03T20:25:03.594 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:25:03 smithi089 bash[78452]: cluster 1685823902.332581 osd.7 (osd.7) 201 : cluster 1 13.7 continuing backfill to osd.4 from (245'2300,259'10670] MIN to 259'10670 2023-06-03T20:25:03.594 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:25:03 smithi089 bash[78452]: cluster 1685823902.3371148 osd.7 (osd.7) 202 : cluster 0 13.7 starting backfill to osd.6 from (245'2300,259'10668] MIN to 259'10670 2023-06-03T20:25:03.594 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:25:03 smithi089 bash[78452]: cluster 1685823902.9503546 mon.a (mon.0) 1467 : cluster 3 Health check update: Degraded data redundancy: 51249/193335 objects degraded (26.508%), 13 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2023-06-03T20:25:03.594 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:25:03 smithi089 bash[78452]: cluster 1685823902.9608912 mon.a (mon.0) 1468 : cluster 0 osdmap e262: 8 total, 8 up, 8 in 2023-06-03T20:25:03.595 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:25:03 smithi089 bash[78452]: cluster 1685823903.0542338 mgr.y (mgr.34104) 1475 : cluster 0 pgmap v1503: 99 pgs: 1 active+remapped+backfilling, 1 peering, 2 remapped+peering, 9 unknown, 3 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 5 active+undersized+degraded+remapped+backfill_wait, 74 active+clean; 3.6 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 32571/177309 objects degraded (18.370%); 8040/177309 objects misplaced (4.534%); 136 KiB/s, 2 objects/s recovering 2023-06-03T20:25:03.685 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:25:03.719 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:25:03.721 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:25:03.722 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:25:03.771 INFO:teuthology.orchestra.run.smithi089.stdout:ERROR: (22) Invalid argument 2023-06-03T20:25:03.771 INFO:teuthology.orchestra.run.smithi089.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-06-03T20:25:03.823 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:25:03.853 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:25:03.854 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:25:03.855 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:25:03.956 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:25:03.991 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:25:03.997 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:25:03.998 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:25:04.099 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:25:04.127 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:25:04.130 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:25:04.131 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:25:04.232 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:25:04.269 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:25:04.271 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:25:04.272 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:25:04.373 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:25:04.406 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:25:04.410 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:25:04.411 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:25:04.512 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:25:04.545 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:25:04.549 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:25:04.550 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:25:04.591 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:25:04 smithi164 bash[59462]: cluster 1685823902.9562552 osd.5 (osd.5) 208 : cluster 3 failed to encode map e262 with expected crc 2023-06-03T20:25:04.596 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:25:04 smithi164 bash[59462]: cluster 1685823902.9588509 osd.7 (osd.7) 203 : cluster 3 failed to encode map e262 with expected crc 2023-06-03T20:25:04.596 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:25:04 smithi164 bash[59462]: cluster 1685823902.9602122 osd.7 (osd.7) 204 : cluster 3 failed to encode map e262 with expected crc 2023-06-03T20:25:04.596 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:25:04 smithi164 bash[59462]: cluster 1685823903.9720387 mon.a (mon.0) 1469 : cluster 0 osdmap e263: 8 total, 8 up, 8 in 2023-06-03T20:25:04.652 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:25:04.681 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:25:04.684 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:25:04.685 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:25:04.739 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:25:04 smithi089 bash[78452]: cluster 1685823902.9562552 osd.5 (osd.5) 208 : cluster 3 failed to encode map e262 with expected crc 2023-06-03T20:25:04.739 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:25:04 smithi089 bash[78452]: cluster 1685823902.9588509 osd.7 (osd.7) 203 : cluster 3 failed to encode map e262 with expected crc 2023-06-03T20:25:04.739 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:25:04 smithi089 bash[78452]: cluster 1685823902.9602122 osd.7 (osd.7) 204 : cluster 3 failed to encode map e262 with expected crc 2023-06-03T20:25:04.739 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:25:04 smithi089 bash[78452]: cluster 1685823903.9720387 mon.a (mon.0) 1469 : cluster 0 osdmap e263: 8 total, 8 up, 8 in 2023-06-03T20:25:04.744 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:25:04 smithi089 bash[75068]: cluster 1685823902.9562552 osd.5 (osd.5) 208 : cluster 3 failed to encode map e262 with expected crc 2023-06-03T20:25:04.745 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:25:04 smithi089 bash[75068]: cluster 1685823902.9588509 osd.7 (osd.7) 203 : cluster 3 failed to encode map e262 with expected crc 2023-06-03T20:25:04.745 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:25:04 smithi089 bash[75068]: cluster 1685823902.9602122 osd.7 (osd.7) 204 : cluster 3 failed to encode map e262 with expected crc 2023-06-03T20:25:04.745 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:25:04 smithi089 bash[75068]: cluster 1685823903.9720387 mon.a (mon.0) 1469 : cluster 0 osdmap e263: 8 total, 8 up, 8 in 2023-06-03T20:25:04.786 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:25:04.818 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:25:04.819 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:25:04.820 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:25:04.921 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:25:04.953 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:25:04.957 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:25:04.957 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:25:05.058 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:25:05.086 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:25:05.089 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:25:05.090 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:25:05.190 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:25:05.228 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:25:05.234 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:25:05.234 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:25:05.324 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:25:05 smithi089 bash[75068]: cluster 1685823903.9674659 osd.7 (osd.7) 205 : cluster 3 failed to encode map e263 with expected crc 2023-06-03T20:25:05.325 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:25:05 smithi089 bash[75068]: cluster 1685823903.9683852 osd.5 (osd.5) 209 : cluster 3 failed to encode map e263 with expected crc 2023-06-03T20:25:05.325 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:25:05 smithi089 bash[75068]: cluster 1685823903.9698243 osd.7 (osd.7) 206 : cluster 3 failed to encode map e263 with expected crc 2023-06-03T20:25:05.325 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:25:05 smithi089 bash[75068]: cluster 1685823903.974038 osd.7 (osd.7) 207 : cluster 1 13.17 continuing backfill to osd.4 from (245'2300,259'10670] MIN to 259'10670 2023-06-03T20:25:05.325 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:25:05 smithi089 bash[75068]: cluster 1685823903.9745617 osd.3 (osd.3) 21 : cluster 1 13.12 continuing backfill to osd.4 from (245'2300,259'10474] MIN to 259'10474 2023-06-03T20:25:05.325 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:25:05 smithi089 bash[75068]: cluster 1685823903.975194 osd.7 (osd.7) 208 : cluster 0 13.17 starting backfill to osd.6 from (245'2300,259'10668] MIN to 259'10670 2023-06-03T20:25:05.325 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:25:05 smithi089 bash[75068]: cluster 1685823903.975732 osd.0 (osd.0) 16 : cluster 0 13.13 starting backfill to osd.4 from (245'1000,259'10408] MIN to 259'10409 2023-06-03T20:25:05.325 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:25:05 smithi089 bash[75068]: cluster 1685823903.977017 osd.0 (osd.0) 17 : cluster 1 13.15 continuing backfill to osd.4 from (245'1100,259'10560] MIN to 259'10560 2023-06-03T20:25:05.326 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:25:05 smithi089 bash[75068]: cluster 1685823904.0212097 osd.6 (osd.6) 201 : cluster 3 failed to encode map e263 with expected crc 2023-06-03T20:25:05.326 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:25:05 smithi089 bash[75068]: cluster 1685823904.1041074 osd.3 (osd.3) 22 : cluster 1 13.16 continuing backfill to osd.4 from (245'2200,259'10338] MIN to 259'10338 2023-06-03T20:25:05.326 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:25:05 smithi089 bash[75068]: cluster 1685823905.054866 mgr.y (mgr.34104) 1476 : cluster 0 pgmap v1505: 99 pgs: 1 active+remapped+backfilling, 1 peering, 3 remapped+peering, 5 unknown, 3 active+recovery_wait+degraded, 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 5 active+undersized+degraded+remapped+backfill_wait, 74 active+clean; 3.6 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 30174/175863 objects degraded (17.158%); 5951/175863 objects misplaced (3.384%); 2.4 MiB/s, 39 objects/s recovering 2023-06-03T20:25:05.336 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:25:05.365 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:25:05.367 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:25:05.368 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:25:05.436 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:25:05 smithi089 bash[78452]: cluster 1685823903.9674659 osd.7 (osd.7) 205 : cluster 3 failed to encode map e263 with expected crc 2023-06-03T20:25:05.436 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:25:05 smithi089 bash[78452]: cluster 1685823903.9683852 osd.5 (osd.5) 209 : cluster 3 failed to encode map e263 with expected crc 2023-06-03T20:25:05.436 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:25:05 smithi089 bash[78452]: cluster 1685823903.9698243 osd.7 (osd.7) 206 : cluster 3 failed to encode map e263 with expected crc 2023-06-03T20:25:05.436 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:25:05 smithi089 bash[78452]: cluster 1685823903.974038 osd.7 (osd.7) 207 : cluster 1 13.17 continuing backfill to osd.4 from (245'2300,259'10670] MIN to 259'10670 2023-06-03T20:25:05.436 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:25:05 smithi089 bash[78452]: cluster 1685823903.9745617 osd.3 (osd.3) 21 : cluster 1 13.12 continuing backfill to osd.4 from (245'2300,259'10474] MIN to 259'10474 2023-06-03T20:25:05.436 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:25:05 smithi089 bash[78452]: cluster 1685823903.975194 osd.7 (osd.7) 208 : cluster 0 13.17 starting backfill to osd.6 from (245'2300,259'10668] MIN to 259'10670 2023-06-03T20:25:05.437 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:25:05 smithi089 bash[78452]: cluster 1685823903.975732 osd.0 (osd.0) 16 : cluster 0 13.13 starting backfill to osd.4 from (245'1000,259'10408] MIN to 259'10409 2023-06-03T20:25:05.437 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:25:05 smithi089 bash[78452]: cluster 1685823903.977017 osd.0 (osd.0) 17 : cluster 1 13.15 continuing backfill to osd.4 from (245'1100,259'10560] MIN to 259'10560 2023-06-03T20:25:05.437 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:25:05 smithi089 bash[78452]: cluster 1685823904.0212097 osd.6 (osd.6) 201 : cluster 3 failed to encode map e263 with expected crc 2023-06-03T20:25:05.437 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:25:05 smithi089 bash[78452]: cluster 1685823904.1041074 osd.3 (osd.3) 22 : cluster 1 13.16 continuing backfill to osd.4 from (245'2200,259'10338] MIN to 259'10338 2023-06-03T20:25:05.437 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:25:05 smithi089 bash[78452]: cluster 1685823905.054866 mgr.y (mgr.34104) 1476 : cluster 0 pgmap v1505: 99 pgs: 1 active+remapped+backfilling, 1 peering, 3 remapped+peering, 5 unknown, 3 active+recovery_wait+degraded, 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 5 active+undersized+degraded+remapped+backfill_wait, 74 active+clean; 3.6 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 30174/175863 objects degraded (17.158%); 5951/175863 objects misplaced (3.384%); 2.4 MiB/s, 39 objects/s recovering 2023-06-03T20:25:05.469 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:25:05.477 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:25:05 smithi164 bash[59462]: cluster 1685823903.9674659 osd.7 (osd.7) 205 : cluster 3 failed to encode map e263 with expected crc 2023-06-03T20:25:05.477 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:25:05 smithi164 bash[59462]: cluster 1685823903.9683852 osd.5 (osd.5) 209 : cluster 3 failed to encode map e263 with expected crc 2023-06-03T20:25:05.478 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:25:05 smithi164 bash[59462]: cluster 1685823903.9698243 osd.7 (osd.7) 206 : cluster 3 failed to encode map e263 with expected crc 2023-06-03T20:25:05.479 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:25:05 smithi164 bash[59462]: cluster 1685823903.974038 osd.7 (osd.7) 207 : cluster 1 13.17 continuing backfill to osd.4 from (245'2300,259'10670] MIN to 259'10670 2023-06-03T20:25:05.479 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:25:05 smithi164 bash[59462]: cluster 1685823903.9745617 osd.3 (osd.3) 21 : cluster 1 13.12 continuing backfill to osd.4 from (245'2300,259'10474] MIN to 259'10474 2023-06-03T20:25:05.479 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:25:05 smithi164 bash[59462]: cluster 1685823903.975194 osd.7 (osd.7) 208 : cluster 0 13.17 starting backfill to osd.6 from (245'2300,259'10668] MIN to 259'10670 2023-06-03T20:25:05.479 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:25:05 smithi164 bash[59462]: cluster 1685823903.975732 osd.0 (osd.0) 16 : cluster 0 13.13 starting backfill to osd.4 from (245'1000,259'10408] MIN to 259'10409 2023-06-03T20:25:05.479 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:25:05 smithi164 bash[59462]: cluster 1685823903.977017 osd.0 (osd.0) 17 : cluster 1 13.15 continuing backfill to osd.4 from (245'1100,259'10560] MIN to 259'10560 2023-06-03T20:25:05.479 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:25:05 smithi164 bash[59462]: cluster 1685823904.0212097 osd.6 (osd.6) 201 : cluster 3 failed to encode map e263 with expected crc 2023-06-03T20:25:05.479 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:25:05 smithi164 bash[59462]: cluster 1685823904.1041074 osd.3 (osd.3) 22 : cluster 1 13.16 continuing backfill to osd.4 from (245'2200,259'10338] MIN to 259'10338 2023-06-03T20:25:05.480 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:25:05 smithi164 bash[59462]: cluster 1685823905.054866 mgr.y (mgr.34104) 1476 : cluster 0 pgmap v1505: 99 pgs: 1 active+remapped+backfilling, 1 peering, 3 remapped+peering, 5 unknown, 3 active+recovery_wait+degraded, 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 5 active+undersized+degraded+remapped+backfill_wait, 74 active+clean; 3.6 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 30174/175863 objects degraded (17.158%); 5951/175863 objects misplaced (3.384%); 2.4 MiB/s, 39 objects/s recovering 2023-06-03T20:25:05.503 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:25:05.506 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:25:05.506 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:25:05.608 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:25:05.637 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:25:05.643 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:25:05.644 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:25:05.744 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:25:05.779 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:25:05.781 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:25:05.782 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:25:05.883 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:25:05.923 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:25:05.926 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:25:05.926 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:25:06.027 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:25:06.067 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:25:06.072 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:25:06.074 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:25:06.176 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:25:06.209 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:25:06.218 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:25:06.219 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:25:06.320 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:25:06.347 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:25:06.351 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:25:06.352 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:25:06.453 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:25:06.493 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:25:06.495 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:25:06.495 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:25:06.597 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:25:06.635 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:25:06.638 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:25:06.639 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:25:06.740 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:25:06.748 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 20:25:06 smithi089 bash[71163]: ::ffff:172.21.15.164 - - [03/Jun/2023:20:25:06] "GET /metrics HTTP/1.1" 200 228946 "" "Prometheus/2.33.4" 2023-06-03T20:25:06.769 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:25:06.775 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:25:06.776 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:25:06.877 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:25:06.930 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:25:06.936 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:25:06.937 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:25:07.038 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:25:07.075 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:25:07.079 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:25:07.080 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:25:07.181 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:25:07.212 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:25:07.214 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:25:07.214 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:25:07.316 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:25:07.367 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:25:07.372 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:25:07.373 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:25:07.475 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:25:07.511 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:25:07.514 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:25:07.515 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:25:07.616 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:25:07.652 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:25:07.654 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:25:07.655 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:25:07.715 INFO:tasks.thrashosds.thrasher:in_osds: [0, 2, 3, 4, 5, 7, 1, 6] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 5, 6, 7, 4] 2023-06-03T20:25:07.716 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-06-03T20:25:07.716 INFO:tasks.thrashosds.thrasher:inject_pause on osd.4 2023-06-03T20:25:07.716 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2023-06-03T20:25:07.717 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2023-06-03T20:25:07.717 DEBUG:teuthology.orchestra.run.smithi164:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph daemon osd.4 config set filestore_inject_stall 3 2023-06-03T20:25:07.756 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:25:07.819 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:25:07.823 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:25:07.824 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:25:07.879 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph daemon osd.3 dump_historic_ops 2023-06-03T20:25:07.925 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:25:07.971 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:25:07.985 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:25:07.986 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:25:08.087 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:25:08.123 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:25:08.128 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:25:08.129 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:25:08.230 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:25:08.239 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:25:08 smithi089 bash[75068]: cluster 1685823907.055956 mgr.y (mgr.34104) 1477 : cluster 0 pgmap v1506: 99 pgs: 1 remapped+peering, 2 unknown, 7 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 5 active+undersized+degraded+remapped+backfill_wait, 76 active+clean; 3.7 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 44393/179982 objects degraded (24.665%); 5616/179982 objects misplaced (3.120%); 6.8 MiB/s, 110 objects/s recovering 2023-06-03T20:25:08.239 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:25:08 smithi089 bash[75068]: audit 1685823907.1451087 mon.c (mon.2) 619 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T20:25:08.239 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:25:08 smithi089 bash[75068]: cluster 1685823907.9567463 mon.a (mon.0) 1470 : cluster 3 Health check update: Degraded data redundancy: 44393/179982 objects degraded (24.665%), 18 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-06-03T20:25:08.240 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:25:08 smithi164 bash[59462]: cluster 1685823907.055956 mgr.y (mgr.34104) 1477 : cluster 0 pgmap v1506: 99 pgs: 1 remapped+peering, 2 unknown, 7 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 5 active+undersized+degraded+remapped+backfill_wait, 76 active+clean; 3.7 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 44393/179982 objects degraded (24.665%); 5616/179982 objects misplaced (3.120%); 6.8 MiB/s, 110 objects/s recovering 2023-06-03T20:25:08.240 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:25:08 smithi164 bash[59462]: audit 1685823907.1451087 mon.c (mon.2) 619 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T20:25:08.240 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:25:08 smithi164 bash[59462]: cluster 1685823907.9567463 mon.a (mon.0) 1470 : cluster 3 Health check update: Degraded data redundancy: 44393/179982 objects degraded (24.665%), 18 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-06-03T20:25:08.240 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:25:08 smithi089 bash[78452]: cluster 1685823907.055956 mgr.y (mgr.34104) 1477 : cluster 0 pgmap v1506: 99 pgs: 1 remapped+peering, 2 unknown, 7 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 5 active+undersized+degraded+remapped+backfill_wait, 76 active+clean; 3.7 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 44393/179982 objects degraded (24.665%); 5616/179982 objects misplaced (3.120%); 6.8 MiB/s, 110 objects/s recovering 2023-06-03T20:25:08.240 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:25:08 smithi089 bash[78452]: audit 1685823907.1451087 mon.c (mon.2) 619 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T20:25:08.240 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:25:08 smithi089 bash[78452]: cluster 1685823907.9567463 mon.a (mon.0) 1470 : cluster 3 Health check update: Degraded data redundancy: 44393/179982 objects degraded (24.665%), 18 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-06-03T20:25:08.273 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:25:08.274 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:25:08.274 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:25:08.375 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:25:08.410 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:25:08.411 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:25:08.412 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:25:08.510 INFO:teuthology.orchestra.run.smithi089.stdout:osd.0: {} 2023-06-03T20:25:08.513 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:25:08.549 INFO:teuthology.orchestra.run.smithi089.stdout:osd.1: {} 2023-06-03T20:25:08.552 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:25:08.556 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:25:08.557 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:25:08.581 INFO:teuthology.orchestra.run.smithi089.stdout:osd.2: {} 2023-06-03T20:25:08.614 INFO:teuthology.orchestra.run.smithi089.stdout:osd.3: {} 2023-06-03T20:25:08.646 INFO:teuthology.orchestra.run.smithi089.stdout:osd.4: {} 2023-06-03T20:25:08.658 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:25:08.682 INFO:teuthology.orchestra.run.smithi089.stdout:osd.5: {} 2023-06-03T20:25:08.694 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:25:08.697 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:25:08.697 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:25:08.708 INFO:teuthology.orchestra.run.smithi089.stdout:osd.6: {} 2023-06-03T20:25:08.743 INFO:teuthology.orchestra.run.smithi089.stdout:osd.7: {} 2023-06-03T20:25:08.754 INFO:teuthology.orchestra.run.smithi089.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-06-03T20:25:08.754 INFO:teuthology.orchestra.run.smithi089.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-06-03T20:25:08.754 INFO:teuthology.orchestra.run.smithi089.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-06-03T20:25:08.755 INFO:teuthology.orchestra.run.smithi089.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-06-03T20:25:08.755 INFO:teuthology.orchestra.run.smithi089.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-06-03T20:25:08.755 INFO:teuthology.orchestra.run.smithi089.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-06-03T20:25:08.755 INFO:teuthology.orchestra.run.smithi089.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-06-03T20:25:08.755 INFO:teuthology.orchestra.run.smithi089.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-06-03T20:25:08.799 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:25:08.829 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:25:08.838 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:25:08.839 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:25:08.940 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:25:08.969 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:25:08.975 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:25:08.977 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:25:09.078 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:25:09.123 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:25:09.126 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:25:09.127 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:25:09.228 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:25:09.260 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:25:09.272 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:25:09.273 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:25:09.374 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:25:09.404 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:25:09.405 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:25:09 smithi089 bash[75068]: audit 1685823908.469185 mon.c (mon.2) 620 : audit 0 from='client.? 172.21.15.89:0/10842243' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-03T20:25:09.405 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:25:09 smithi089 bash[78452]: audit 1685823908.469185 mon.c (mon.2) 620 : audit 0 from='client.? 172.21.15.89:0/10842243' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-03T20:25:09.406 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:25:09.406 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:25:09.507 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:25:09.547 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:25:09.548 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:25:09.549 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:25:09.584 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:25:09 smithi164 bash[59462]: audit 1685823908.469185 mon.c (mon.2) 620 : audit 0 from='client.? 172.21.15.89:0/10842243' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-03T20:25:09.650 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:25:09.691 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:25:09.695 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:25:09.696 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:25:09.797 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:25:09.838 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:25:09.847 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:25:09.848 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:25:09.949 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:25:09.980 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:25:09.987 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:25:09.987 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:25:10.088 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:25:10.134 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:25:10.135 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:25:10.135 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:25:10.236 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:25:10.244 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:25:10 smithi164 bash[59462]: cluster 1685823909.056875 mgr.y (mgr.34104) 1478 : cluster 0 pgmap v1507: 99 pgs: 6 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 2 active+recovery_wait, 7 active+recovery_wait+undersized+degraded+remapped, 6 active+undersized+degraded+remapped+backfill_wait, 76 active+clean; 3.7 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 34 MiB/s wr, 1.23k op/s; 49508/181722 objects degraded (27.244%); 7388/181722 objects misplaced (4.066%); 6.7 MiB/s, 108 objects/s recovering 2023-06-03T20:25:10.276 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:25:10.282 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:25:10.283 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:25:10.384 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:25:10.392 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:25:10 smithi089 bash[78452]: cluster 1685823909.056875 mgr.y (mgr.34104) 1478 : cluster 0 pgmap v1507: 99 pgs: 6 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 2 active+recovery_wait, 7 active+recovery_wait+undersized+degraded+remapped, 6 active+undersized+degraded+remapped+backfill_wait, 76 active+clean; 3.7 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 34 MiB/s wr, 1.23k op/s; 49508/181722 objects degraded (27.244%); 7388/181722 objects misplaced (4.066%); 6.7 MiB/s, 108 objects/s recovering 2023-06-03T20:25:10.393 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:25:10 smithi089 bash[75068]: cluster 1685823909.056875 mgr.y (mgr.34104) 1478 : cluster 0 pgmap v1507: 99 pgs: 6 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 2 active+recovery_wait, 7 active+recovery_wait+undersized+degraded+remapped, 6 active+undersized+degraded+remapped+backfill_wait, 76 active+clean; 3.7 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 34 MiB/s wr, 1.23k op/s; 49508/181722 objects degraded (27.244%); 7388/181722 objects misplaced (4.066%); 6.7 MiB/s, 108 objects/s recovering 2023-06-03T20:25:10.414 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:25:10.419 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:25:10.420 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:25:10.521 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:25:10.558 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:25:10.561 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:25:10.562 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:25:10.663 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:25:10.694 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:25:10.696 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:25:10.697 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:25:10.798 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:25:10.837 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:25:10.839 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:25:10.841 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:25:10.942 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:25:10.976 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:25:10.979 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:25:10.980 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:25:11.081 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:25:11.125 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:25:11.127 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:25:11.128 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:25:11.230 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:25:11.262 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:25:11.270 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:25:11.271 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:25:11.371 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:25:11.415 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:25:11.419 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:25:11.420 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:25:11.521 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:25:11.559 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:25:11.565 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:25:11.567 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:25:11.668 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:25:11.705 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:25:11.710 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:25:11.710 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:25:11.811 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:25:11.845 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:25:11.849 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:25:11.849 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:25:11.950 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:25:11.980 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:25:11.982 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:25:11.983 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:25:12.084 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:25:12.121 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:25:12.123 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:25:12.124 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:25:12.225 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:25:12.272 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:25:12.274 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:25:12.275 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:25:12.279 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:25:12 smithi089 bash[78452]: cluster 1685823911.0577476 mgr.y (mgr.34104) 1479 : cluster 0 pgmap v1508: 99 pgs: 6 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 2 active+recovery_wait, 7 active+recovery_wait+undersized+degraded+remapped, 6 active+undersized+degraded+remapped+backfill_wait, 76 active+clean; 3.7 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 56 MiB/s wr, 1.67k op/s; 49100/179898 objects degraded (27.293%); 7388/179898 objects misplaced (4.107%); 5.7 MiB/s, 91 objects/s recovering 2023-06-03T20:25:12.280 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:25:12 smithi089 bash[75068]: cluster 1685823911.0577476 mgr.y (mgr.34104) 1479 : cluster 0 pgmap v1508: 99 pgs: 6 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 2 active+recovery_wait, 7 active+recovery_wait+undersized+degraded+remapped, 6 active+undersized+degraded+remapped+backfill_wait, 76 active+clean; 3.7 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 56 MiB/s wr, 1.67k op/s; 49100/179898 objects degraded (27.293%); 7388/179898 objects misplaced (4.107%); 5.7 MiB/s, 91 objects/s recovering 2023-06-03T20:25:12.376 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:25:12.406 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:25:12.411 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:25:12.413 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:25:12.514 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:25:12.523 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:25:12 smithi164 bash[59462]: cluster 1685823911.0577476 mgr.y (mgr.34104) 1479 : cluster 0 pgmap v1508: 99 pgs: 6 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 2 active+recovery_wait, 7 active+recovery_wait+undersized+degraded+remapped, 6 active+undersized+degraded+remapped+backfill_wait, 76 active+clean; 3.7 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 56 MiB/s wr, 1.67k op/s; 49100/179898 objects degraded (27.293%); 7388/179898 objects misplaced (4.107%); 5.7 MiB/s, 91 objects/s recovering 2023-06-03T20:25:12.565 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:25:12.570 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:25:12.572 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:25:12.637 INFO:teuthology.orchestra.run.smithi164.stderr:Inferring config /var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/mon.b/config 2023-06-03T20:25:12.673 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:25:12.706 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:25:12.710 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:25:12.711 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:25:12.812 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:25:12.851 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:25:12.861 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:25:12.862 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:25:12.963 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:25:13.027 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:25:13.033 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:25:13.034 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:25:13.135 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:25:13.165 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:25:13.170 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:25:13.172 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:25:13.172 INFO:teuthology.orchestra.run.smithi089.stderr:nodeep-scrub is unset 2023-06-03T20:25:13.273 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:25:13.283 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:25:13 smithi164 bash[59462]: audit 1685823912.2853355 mon.b (mon.1) 63 : audit 1 from='client.? 172.21.15.89:0/3163537420' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-06-03T20:25:13.283 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:25:13 smithi164 bash[59462]: audit 1685823912.288312 mon.a (mon.0) 1471 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-06-03T20:25:13.283 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:25:13 smithi164 bash[59462]: cluster 1685823912.9583976 mon.a (mon.0) 1472 : cluster 3 Health check update: Degraded data redundancy: 49100/179898 objects degraded (27.293%), 20 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-06-03T20:25:13.303 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:25:13.305 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:25:13.306 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:25:13.407 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:25:13.457 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:25:13.463 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:25:13.465 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:25:13.529 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:25:13 smithi089 bash[78452]: audit 1685823912.2853355 mon.b (mon.1) 63 : audit 1 from='client.? 172.21.15.89:0/3163537420' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-06-03T20:25:13.529 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:25:13 smithi089 bash[78452]: audit 1685823912.288312 mon.a (mon.0) 1471 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-06-03T20:25:13.529 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:25:13 smithi089 bash[78452]: cluster 1685823912.9583976 mon.a (mon.0) 1472 : cluster 3 Health check update: Degraded data redundancy: 49100/179898 objects degraded (27.293%), 20 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-06-03T20:25:13.530 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:25:13 smithi089 bash[75068]: audit 1685823912.2853355 mon.b (mon.1) 63 : audit 1 from='client.? 172.21.15.89:0/3163537420' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-06-03T20:25:13.530 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:25:13 smithi089 bash[75068]: audit 1685823912.288312 mon.a (mon.0) 1471 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-06-03T20:25:13.530 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:25:13 smithi089 bash[75068]: cluster 1685823912.9583976 mon.a (mon.0) 1472 : cluster 3 Health check update: Degraded data redundancy: 49100/179898 objects degraded (27.293%), 20 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-06-03T20:25:13.544 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-06-03T20:25:13.567 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:25:13.624 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:25:13.630 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:25:13.630 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:25:13.731 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:25:13.775 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:25:13.780 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:25:13.781 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:25:13.882 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:25:13.922 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:25:13.924 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:25:13.925 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:25:14.025 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:25:14.069 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:25:14.072 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:25:14.073 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:25:14.174 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:25:14.213 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:25:14.219 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:25:14.220 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:25:14.320 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:25:14.362 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:25:14.364 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:25:14.365 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:25:14.466 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:25:14.475 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:25:14 smithi089 bash[75068]: cluster 1685823913.0586607 mgr.y (mgr.34104) 1480 : cluster 0 pgmap v1509: 99 pgs: 6 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait, 7 active+recovery_wait+undersized+degraded+remapped, 6 active+undersized+degraded+remapped+backfill_wait, 76 active+clean; 3.5 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 45 MiB/s wr, 1.56k op/s; 48015/173616 objects degraded (27.656%); 7185/173616 objects misplaced (4.138%); 7.3 MiB/s, 117 objects/s recovering 2023-06-03T20:25:14.475 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:25:14 smithi089 bash[75068]: cluster 1685823913.1373863 mon.a (mon.0) 1473 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-06-03T20:25:14.475 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:25:14 smithi089 bash[75068]: audit 1685823913.1529405 mon.a (mon.0) 1474 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-06-03T20:25:14.476 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:25:14 smithi089 bash[75068]: cluster 1685823913.1530335 mon.a (mon.0) 1475 : cluster 0 osdmap e264: 8 total, 8 up, 8 in 2023-06-03T20:25:14.476 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:25:14 smithi089 bash[75068]: cluster 1685823913.1593857 osd.7 (osd.7) 209 : cluster 3 failed to encode map e264 with expected crc 2023-06-03T20:25:14.476 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:25:14 smithi089 bash[78452]: cluster 1685823913.0586607 mgr.y (mgr.34104) 1480 : cluster 0 pgmap v1509: 99 pgs: 6 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait, 7 active+recovery_wait+undersized+degraded+remapped, 6 active+undersized+degraded+remapped+backfill_wait, 76 active+clean; 3.5 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 45 MiB/s wr, 1.56k op/s; 48015/173616 objects degraded (27.656%); 7185/173616 objects misplaced (4.138%); 7.3 MiB/s, 117 objects/s recovering 2023-06-03T20:25:14.476 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:25:14 smithi089 bash[78452]: cluster 1685823913.1373863 mon.a (mon.0) 1473 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-06-03T20:25:14.476 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:25:14 smithi089 bash[78452]: audit 1685823913.1529405 mon.a (mon.0) 1474 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-06-03T20:25:14.476 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:25:14 smithi089 bash[78452]: cluster 1685823913.1530335 mon.a (mon.0) 1475 : cluster 0 osdmap e264: 8 total, 8 up, 8 in 2023-06-03T20:25:14.477 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:25:14 smithi089 bash[78452]: cluster 1685823913.1593857 osd.7 (osd.7) 209 : cluster 3 failed to encode map e264 with expected crc 2023-06-03T20:25:14.503 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:25:14.506 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:25:14.507 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:25:14.584 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:25:14 smithi164 bash[59462]: cluster 1685823913.0586607 mgr.y (mgr.34104) 1480 : cluster 0 pgmap v1509: 99 pgs: 6 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait, 7 active+recovery_wait+undersized+degraded+remapped, 6 active+undersized+degraded+remapped+backfill_wait, 76 active+clean; 3.5 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 45 MiB/s wr, 1.56k op/s; 48015/173616 objects degraded (27.656%); 7185/173616 objects misplaced (4.138%); 7.3 MiB/s, 117 objects/s recovering 2023-06-03T20:25:14.585 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:25:14 smithi164 bash[59462]: cluster 1685823913.1373863 mon.a (mon.0) 1473 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-06-03T20:25:14.585 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:25:14 smithi164 bash[59462]: audit 1685823913.1529405 mon.a (mon.0) 1474 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-06-03T20:25:14.585 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:25:14 smithi164 bash[59462]: cluster 1685823913.1530335 mon.a (mon.0) 1475 : cluster 0 osdmap e264: 8 total, 8 up, 8 in 2023-06-03T20:25:14.585 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:25:14 smithi164 bash[59462]: cluster 1685823913.1593857 osd.7 (osd.7) 209 : cluster 3 failed to encode map e264 with expected crc 2023-06-03T20:25:14.607 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:25:14.643 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:25:14.646 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:25:14.646 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:25:14.747 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:25:14.778 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:25:14.783 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:25:14.783 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:25:14.884 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:25:14.921 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:25:14.923 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:25:14.924 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:25:15.025 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:25:15.073 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:25:15.078 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:25:15.079 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:25:15.180 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:25:15.216 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:25:15.225 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:25:15.226 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:25:15.327 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:25:15.361 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:25:15.363 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:25:15.364 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:25:15.465 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:25:15.497 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:25:15.500 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:25:15.501 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:25:15.602 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:25:15.662 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:25:15.667 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:25:15.668 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:25:15.693 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/mon.a/config 2023-06-03T20:25:15.768 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:25:15.800 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:25:15.803 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:25:15.804 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:25:15.905 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:25:15.935 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:25:15.940 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:25:15.940 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:25:16.042 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:25:16.069 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:25:16.072 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:25:16.073 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:25:16.173 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:25:16.208 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:25:16.211 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:25:16.212 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:25:16.313 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:25:16.320 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 20:25:16 smithi089 bash[71163]: ::ffff:172.21.15.164 - - [03/Jun/2023:20:25:16] "GET /metrics HTTP/1.1" 200 228987 "" "Prometheus/2.33.4" 2023-06-03T20:25:16.320 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:25:16 smithi089 bash[75068]: cluster 1685823915.0595105 mgr.y (mgr.34104) 1481 : cluster 0 pgmap v1511: 99 pgs: 6 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 2 active+recovery_wait, 7 active+recovery_wait+undersized+degraded+remapped, 6 active+undersized+degraded+remapped+backfill_wait, 76 active+clean; 3.5 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 45 MiB/s wr, 1.64k op/s; 47323/169863 objects degraded (27.860%); 6980/169863 objects misplaced (4.109%); 6.9 MiB/s, 110 objects/s recovering 2023-06-03T20:25:16.321 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:25:16 smithi089 bash[78452]: cluster 1685823915.0595105 mgr.y (mgr.34104) 1481 : cluster 0 pgmap v1511: 99 pgs: 6 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 2 active+recovery_wait, 7 active+recovery_wait+undersized+degraded+remapped, 6 active+undersized+degraded+remapped+backfill_wait, 76 active+clean; 3.5 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 45 MiB/s wr, 1.64k op/s; 47323/169863 objects degraded (27.860%); 6980/169863 objects misplaced (4.109%); 6.9 MiB/s, 110 objects/s recovering 2023-06-03T20:25:16.340 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:25:16.342 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:25:16.343 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:25:16.397 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:25:16 smithi164 bash[59462]: cluster 1685823915.0595105 mgr.y (mgr.34104) 1481 : cluster 0 pgmap v1511: 99 pgs: 6 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 2 active+recovery_wait, 7 active+recovery_wait+undersized+degraded+remapped, 6 active+undersized+degraded+remapped+backfill_wait, 76 active+clean; 3.5 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 45 MiB/s wr, 1.64k op/s; 47323/169863 objects degraded (27.860%); 6980/169863 objects misplaced (4.109%); 6.9 MiB/s, 110 objects/s recovering 2023-06-03T20:25:16.443 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:25:16.475 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:25:16.478 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:25:16.479 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:25:16.580 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:25:16.612 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:25:16.615 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:25:16.615 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:25:16.716 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:25:16.764 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:25:16.769 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:25:16.770 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:25:16.871 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:25:16.911 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:25:16.915 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:25:16.915 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:25:17.016 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:25:17.053 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:25:17.055 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:25:17.056 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:25:17.158 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:25:17.203 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:25:17.206 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:25:17.207 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:25:17.214 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph osd set noscrub 2023-06-03T20:25:17.308 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:25:17.347 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:25:17.351 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:25:17.351 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:25:17.452 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:25:17.489 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:25:17.495 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:25:17.496 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:25:17.597 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:25:17.649 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:25:17.652 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:25:17.653 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:25:17.754 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:25:17.784 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:25:17.786 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:25:17.787 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:25:17.888 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:25:17.931 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:25:17.939 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:25:17.940 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:25:18.041 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:25:18.076 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:25:18.077 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:25:18.078 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:25:18.155 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:25:18 smithi164 bash[59462]: cluster 1685823917.0603905 mgr.y (mgr.34104) 1482 : cluster 0 pgmap v1512: 99 pgs: 6 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait, 7 active+recovery_wait+undersized+degraded+remapped, 6 active+undersized+degraded+remapped+backfill_wait, 76 active+clean; 3.4 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 22 MiB/s wr, 1.16k op/s; 46889/166593 objects degraded (28.146%); 6787/166593 objects misplaced (4.074%); 6.6 MiB/s, 106 objects/s recovering 2023-06-03T20:25:18.155 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:25:18 smithi164 bash[59462]: cluster 1685823917.960337 mon.a (mon.0) 1476 : cluster 3 Health check update: Degraded data redundancy: 46889/166593 objects degraded (28.146%), 20 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-06-03T20:25:18.179 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:25:18.224 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:25:18.228 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:25:18.229 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:25:18.329 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:25:18.364 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:25:18.369 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:25:18.370 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:25:18.405 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:25:18 smithi089 bash[78452]: cluster 1685823917.0603905 mgr.y (mgr.34104) 1482 : cluster 0 pgmap v1512: 99 pgs: 6 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait, 7 active+recovery_wait+undersized+degraded+remapped, 6 active+undersized+degraded+remapped+backfill_wait, 76 active+clean; 3.4 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 22 MiB/s wr, 1.16k op/s; 46889/166593 objects degraded (28.146%); 6787/166593 objects misplaced (4.074%); 6.6 MiB/s, 106 objects/s recovering 2023-06-03T20:25:18.406 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:25:18 smithi089 bash[78452]: cluster 1685823917.960337 mon.a (mon.0) 1476 : cluster 3 Health check update: Degraded data redundancy: 46889/166593 objects degraded (28.146%), 20 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-06-03T20:25:18.406 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:25:18 smithi089 bash[75068]: cluster 1685823917.0603905 mgr.y (mgr.34104) 1482 : cluster 0 pgmap v1512: 99 pgs: 6 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait, 7 active+recovery_wait+undersized+degraded+remapped, 6 active+undersized+degraded+remapped+backfill_wait, 76 active+clean; 3.4 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 22 MiB/s wr, 1.16k op/s; 46889/166593 objects degraded (28.146%); 6787/166593 objects misplaced (4.074%); 6.6 MiB/s, 106 objects/s recovering 2023-06-03T20:25:18.406 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:25:18 smithi089 bash[75068]: cluster 1685823917.960337 mon.a (mon.0) 1476 : cluster 3 Health check update: Degraded data redundancy: 46889/166593 objects degraded (28.146%), 20 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-06-03T20:25:18.471 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:25:18.506 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:25:18.508 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:25:18.509 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:25:18.610 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:25:18.641 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:25:18.647 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:25:18.648 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:25:18.749 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:25:18.790 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/mon.a/config 2023-06-03T20:25:18.795 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:25:18.797 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:25:18.798 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:25:18.899 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:25:18.932 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:25:18.936 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:25:18.937 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:25:19.038 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:25:19.074 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:25:19.079 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:25:19.080 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:25:19.181 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:25:19.220 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:25:19.226 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:25:19.227 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:25:19.328 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:25:19.360 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:25:19.362 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:25:19.363 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:25:19.464 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:25:19.511 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:25:19.516 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:25:19.517 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:25:19.621 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:25:19.644 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:25:19.646 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:25:19.647 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:25:19.662 INFO:teuthology.orchestra.run.smithi164.stdout:{ 2023-06-03T20:25:19.663 INFO:teuthology.orchestra.run.smithi164.stdout: "success": "filestore_inject_stall = '3' (not observed, change may require restart) " 2023-06-03T20:25:19.663 INFO:teuthology.orchestra.run.smithi164.stdout:} 2023-06-03T20:25:19.748 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:25:19.777 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:25:19.783 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:25:19.784 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:25:19.885 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:25:19.929 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:25:19.931 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:25:19.932 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:25:20.033 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:25:20.069 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:25:20.074 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:25:20.076 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:25:20.174 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:25:20 smithi164 bash[59462]: cluster 1685823919.0613222 mgr.y (mgr.34104) 1483 : cluster 0 pgmap v1513: 99 pgs: 6 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 2 active+recovery_wait, 7 active+recovery_wait+undersized+degraded+remapped, 6 active+undersized+degraded+remapped+backfill_wait, 76 active+clean; 3.3 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 674 op/s; 45945/161481 objects degraded (28.452%); 6571/161481 objects misplaced (4.069%); 6.5 MiB/s, 103 objects/s recovering 2023-06-03T20:25:20.180 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:25:20.223 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:25:20.226 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:25:20.227 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:25:20.328 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:25:20.367 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:25:20.371 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:25:20.371 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:25:20.473 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:25:20.518 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:25:20.526 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:25:20.527 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:25:20.529 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:25:20 smithi089 bash[78452]: cluster 1685823919.0613222 mgr.y (mgr.34104) 1483 : cluster 0 pgmap v1513: 99 pgs: 6 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 2 active+recovery_wait, 7 active+recovery_wait+undersized+degraded+remapped, 6 active+undersized+degraded+remapped+backfill_wait, 76 active+clean; 3.3 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 674 op/s; 45945/161481 objects degraded (28.452%); 6571/161481 objects misplaced (4.069%); 6.5 MiB/s, 103 objects/s recovering 2023-06-03T20:25:20.529 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:25:20 smithi089 bash[75068]: cluster 1685823919.0613222 mgr.y (mgr.34104) 1483 : cluster 0 pgmap v1513: 99 pgs: 6 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 2 active+recovery_wait, 7 active+recovery_wait+undersized+degraded+remapped, 6 active+undersized+degraded+remapped+backfill_wait, 76 active+clean; 3.3 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 674 op/s; 45945/161481 objects degraded (28.452%); 6571/161481 objects misplaced (4.069%); 6.5 MiB/s, 103 objects/s recovering 2023-06-03T20:25:20.628 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:25:20.660 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:25:20.664 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:25:20.665 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:25:20.766 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:25:20.797 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:25:21.052 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:25:21.053 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:25:21.154 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:25:21.190 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:25:21.193 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:25:21.194 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:25:21.295 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:25:21.332 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:25:21.333 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:25:21.335 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:25:21.436 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:25:21.473 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:25:21.475 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:25:21.475 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:25:21.577 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:25:21.610 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:25:21.612 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:25:21.613 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:25:21.714 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:25:21.746 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:25:21.747 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:25:21.748 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:25:21.849 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:25:21.881 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:25:21.883 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:25:21.885 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:25:21.986 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:25:22.038 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:25:22.041 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:25:22.042 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:25:22.143 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:25:22.181 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:25:22.184 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:25:22.184 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:25:22.285 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:25:22.315 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:25:22.317 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:25:22.318 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:25:22.335 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:25:22 smithi164 bash[59462]: cluster 1685823921.06198 mgr.y (mgr.34104) 1484 : cluster 0 pgmap v1514: 99 pgs: 6 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 2 active+recovery_wait, 7 active+recovery_wait+undersized+degraded+remapped, 6 active+undersized+degraded+remapped+backfill_wait, 76 active+clean; 3.2 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 676 op/s; 45523/159606 objects degraded (28.522%); 6571/159606 objects misplaced (4.117%); 6.5 MiB/s, 103 objects/s recovering 2023-06-03T20:25:22.419 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:25:22.427 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:25:22 smithi089 bash[78452]: cluster 1685823921.06198 mgr.y (mgr.34104) 1484 : cluster 0 pgmap v1514: 99 pgs: 6 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 2 active+recovery_wait, 7 active+recovery_wait+undersized+degraded+remapped, 6 active+undersized+degraded+remapped+backfill_wait, 76 active+clean; 3.2 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 676 op/s; 45523/159606 objects degraded (28.522%); 6571/159606 objects misplaced (4.117%); 6.5 MiB/s, 103 objects/s recovering 2023-06-03T20:25:22.427 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:25:22 smithi089 bash[75068]: cluster 1685823921.06198 mgr.y (mgr.34104) 1484 : cluster 0 pgmap v1514: 99 pgs: 6 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 2 active+recovery_wait, 7 active+recovery_wait+undersized+degraded+remapped, 6 active+undersized+degraded+remapped+backfill_wait, 76 active+clean; 3.2 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 676 op/s; 45523/159606 objects degraded (28.522%); 6571/159606 objects misplaced (4.117%); 6.5 MiB/s, 103 objects/s recovering 2023-06-03T20:25:22.448 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:25:22.451 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:25:22.453 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:25:22.554 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:25:22.594 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:25:22.599 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:25:22.600 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:25:22.702 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:25:22.753 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:25:22.758 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:25:22.759 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:25:22.860 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:25:22.899 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:25:22.906 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:25:22.907 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:25:23.007 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:25:23.046 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:25:23.052 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:25:23.052 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:25:23.153 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:25:23.190 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:25:23.194 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:25:23.195 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:25:23.279 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:25:23 smithi089 bash[78452]: audit 1685823922.1600392 mon.a (mon.0) 1477 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:25:23.279 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:25:23 smithi089 bash[78452]: audit 1685823922.170755 mon.c (mon.2) 621 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T20:25:23.279 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:25:23 smithi089 bash[78452]: cluster 1685823922.9616807 mon.a (mon.0) 1478 : cluster 3 Health check update: Degraded data redundancy: 45523/159606 objects degraded (28.522%), 20 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-06-03T20:25:23.280 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:25:23 smithi089 bash[75068]: audit 1685823922.1600392 mon.a (mon.0) 1477 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:25:23.280 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:25:23 smithi089 bash[75068]: audit 1685823922.170755 mon.c (mon.2) 621 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T20:25:23.280 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:25:23 smithi089 bash[75068]: cluster 1685823922.9616807 mon.a (mon.0) 1478 : cluster 3 Health check update: Degraded data redundancy: 45523/159606 objects degraded (28.522%), 20 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-06-03T20:25:23.297 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:25:23.342 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:25:23.347 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:25:23.348 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:25:23.409 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/mon.a/config 2023-06-03T20:25:23.450 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:25:23.476 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:25:23.477 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:25:23.478 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:25:23.579 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:25:23.584 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:25:23 smithi164 bash[59462]: audit 1685823922.1600392 mon.a (mon.0) 1477 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:25:23.585 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:25:23 smithi164 bash[59462]: audit 1685823922.170755 mon.c (mon.2) 621 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T20:25:23.585 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:25:23 smithi164 bash[59462]: cluster 1685823922.9616807 mon.a (mon.0) 1478 : cluster 3 Health check update: Degraded data redundancy: 45523/159606 objects degraded (28.522%), 20 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-06-03T20:25:23.624 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:25:23.625 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:25:23.626 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:25:23.727 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:25:23.799 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:25:23.807 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:25:23.808 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:25:23.909 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:25:23.949 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:25:23.950 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:25:23.951 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:25:24.052 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:25:24.086 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:25:24.089 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:25:24.089 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:25:24.191 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:25:24.232 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:25:24.234 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:25:24.235 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:25:24.336 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:25:24.369 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:25:24.372 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:25:24.373 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:25:24.474 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:25:24.486 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:25:24 smithi164 bash[59462]: cluster 1685823923.0628028 mgr.y (mgr.34104) 1485 : cluster 0 pgmap v1515: 99 pgs: 6 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait, 7 active+recovery_wait+undersized+degraded+remapped, 6 active+undersized+degraded+remapped+backfill_wait, 76 active+clean; 3.1 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 670 op/s; 44471/153480 objects degraded (28.975%); 6380/153480 objects misplaced (4.157%); 6.3 MiB/s, 101 objects/s recovering 2023-06-03T20:25:24.486 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:25:24 smithi164 bash[59462]: audit 1685823923.5249834 mon.c (mon.2) 622 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-03T20:25:24.518 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:25:24.527 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:25:24.528 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:25:24.530 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:25:24 smithi089 bash[75068]: cluster 1685823923.0628028 mgr.y (mgr.34104) 1485 : cluster 0 pgmap v1515: 99 pgs: 6 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait, 7 active+recovery_wait+undersized+degraded+remapped, 6 active+undersized+degraded+remapped+backfill_wait, 76 active+clean; 3.1 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 670 op/s; 44471/153480 objects degraded (28.975%); 6380/153480 objects misplaced (4.157%); 6.3 MiB/s, 101 objects/s recovering 2023-06-03T20:25:24.530 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:25:24 smithi089 bash[75068]: audit 1685823923.5249834 mon.c (mon.2) 622 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-03T20:25:24.530 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:25:24 smithi089 bash[78452]: cluster 1685823923.0628028 mgr.y (mgr.34104) 1485 : cluster 0 pgmap v1515: 99 pgs: 6 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait, 7 active+recovery_wait+undersized+degraded+remapped, 6 active+undersized+degraded+remapped+backfill_wait, 76 active+clean; 3.1 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 670 op/s; 44471/153480 objects degraded (28.975%); 6380/153480 objects misplaced (4.157%); 6.3 MiB/s, 101 objects/s recovering 2023-06-03T20:25:24.530 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:25:24 smithi089 bash[78452]: audit 1685823923.5249834 mon.c (mon.2) 622 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-03T20:25:24.629 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:25:24.677 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:25:24.682 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:25:24.683 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:25:24.784 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:25:24.814 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:25:24.819 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:25:24.820 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:25:24.921 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:25:24.978 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:25:24.980 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:25:24.981 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:25:25.082 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:25:25.121 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:25:25.123 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:25:25.124 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:25:25.225 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:25:25.269 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:25:25.271 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:25:25.272 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:25:25.374 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:25:25.417 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:25:25.418 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:25:25.420 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:25:25.521 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:25:25.563 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:25:25.571 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:25:25.572 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:25:25.673 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:25:25.705 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:25:25.709 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:25:25.710 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:25:25.812 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:25:26.527 INFO:tasks.thrashosds.thrasher:in_osds: [0, 2, 3, 4, 5, 7, 1, 6] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 5, 6, 7, 4] 2023-06-03T20:25:26.528 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph osd reweight 0 1 2023-06-03T20:25:26.529 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:25:26 smithi089 bash[78452]: cluster 1685823925.0637343 mgr.y (mgr.34104) 1486 : cluster 0 pgmap v1516: 99 pgs: 6 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 2 active+recovery_wait, 7 active+recovery_wait+undersized+degraded+remapped, 6 active+undersized+degraded+remapped+backfill_wait, 76 active+clean; 3.1 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 657 op/s; 43797/150141 objects degraded (29.171%); 6163/150141 objects misplaced (4.105%); 5.9 MiB/s, 94 objects/s recovering 2023-06-03T20:25:26.530 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:25:26 smithi089 bash[75068]: cluster 1685823925.0637343 mgr.y (mgr.34104) 1486 : cluster 0 pgmap v1516: 99 pgs: 6 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 2 active+recovery_wait, 7 active+recovery_wait+undersized+degraded+remapped, 6 active+undersized+degraded+remapped+backfill_wait, 76 active+clean; 3.1 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 657 op/s; 43797/150141 objects degraded (29.171%); 6163/150141 objects misplaced (4.105%); 5.9 MiB/s, 94 objects/s recovering 2023-06-03T20:25:26.530 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 20:25:26 smithi089 bash[71163]: ::ffff:172.21.15.164 - - [03/Jun/2023:20:25:26] "GET /metrics HTTP/1.1" 200 228998 "" "Prometheus/2.33.4" 2023-06-03T20:25:26.579 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:25:26.585 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:25:26 smithi164 bash[59462]: cluster 1685823925.0637343 mgr.y (mgr.34104) 1486 : cluster 0 pgmap v1516: 99 pgs: 6 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 2 active+recovery_wait, 7 active+recovery_wait+undersized+degraded+remapped, 6 active+undersized+degraded+remapped+backfill_wait, 76 active+clean; 3.1 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 657 op/s; 43797/150141 objects degraded (29.171%); 6163/150141 objects misplaced (4.105%); 5.9 MiB/s, 94 objects/s recovering 2023-06-03T20:25:26.586 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:25:26.587 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:25:26.688 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:25:26.717 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:25:26.720 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:25:26.721 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:25:26.822 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:25:26.865 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:25:26.867 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:25:26.867 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:25:26.969 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:25:27.016 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:25:27.020 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:25:27.021 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:25:27.122 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:25:27.152 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:25:27.155 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:25:27.157 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:25:27.259 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:25:27.287 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:25:27.289 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:25:27.290 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:25:27.391 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:25:27.424 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:25:27.428 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:25:27.430 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:25:27.531 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:25:27.557 INFO:teuthology.orchestra.run.smithi089.stdout:{ 2023-06-03T20:25:27.557 INFO:teuthology.orchestra.run.smithi089.stdout: "size": 20, 2023-06-03T20:25:27.557 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 600, 2023-06-03T20:25:27.557 INFO:teuthology.orchestra.run.smithi089.stdout: "ops": [ 2023-06-03T20:25:27.557 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:25:27.558 INFO:teuthology.orchestra.run.smithi089.stdout: "description": "osd_op(client.36573.0:117008 13.2 13:4f78a186:::benchmark_data_smithi089_211581_object1483:head [delete] snapc 0=[] ondisk+write+known_if_redirected e250)", 2023-06-03T20:25:27.558 INFO:teuthology.orchestra.run.smithi089.stdout: "initiated_at": "2023-06-03T20:23:26.276859+0000", 2023-06-03T20:25:27.558 INFO:teuthology.orchestra.run.smithi089.stdout: "age": 121.271902637, 2023-06-03T20:25:27.558 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 2.4779910749999998, 2023-06-03T20:25:27.558 INFO:teuthology.orchestra.run.smithi089.stdout: "type_data": { 2023-06-03T20:25:27.558 INFO:teuthology.orchestra.run.smithi089.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-03T20:25:27.558 INFO:teuthology.orchestra.run.smithi089.stdout: "client_info": { 2023-06-03T20:25:27.558 INFO:teuthology.orchestra.run.smithi089.stdout: "client": "client.36573", 2023-06-03T20:25:27.559 INFO:teuthology.orchestra.run.smithi089.stdout: "client_addr": "172.21.15.89:0/616445942", 2023-06-03T20:25:27.559 INFO:teuthology.orchestra.run.smithi089.stdout: "tid": 117008 2023-06-03T20:25:27.559 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:25:27.559 INFO:teuthology.orchestra.run.smithi089.stdout: "events": [ 2023-06-03T20:25:27.559 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:25:27.559 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "initiated", 2023-06-03T20:25:27.559 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:23:26.276859+0000", 2023-06-03T20:25:27.560 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0 2023-06-03T20:25:27.560 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:25:27.560 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:25:27.560 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "throttled", 2023-06-03T20:25:27.560 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:23:26.276859+0000", 2023-06-03T20:25:27.560 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0 2023-06-03T20:25:27.560 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:25:27.560 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:25:27.561 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "header_read", 2023-06-03T20:25:27.561 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:23:26.276857+0000", 2023-06-03T20:25:27.561 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 4294967295.9999981 2023-06-03T20:25:27.561 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:25:27.561 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:25:27.561 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "all_read", 2023-06-03T20:25:27.561 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:23:26.276867+0000", 2023-06-03T20:25:27.562 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 9.7610000000000002e-06 2023-06-03T20:25:27.562 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:25:27.562 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:25:27.562 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "dispatched", 2023-06-03T20:25:27.562 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:23:26.276886+0000", 2023-06-03T20:25:27.562 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 1.8963000000000001e-05 2023-06-03T20:25:27.562 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:25:27.562 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:25:27.563 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "queued_for_pg", 2023-06-03T20:25:27.563 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:23:26.276895+0000", 2023-06-03T20:25:27.563 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 9.3249999999999997e-06 2023-06-03T20:25:27.563 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:25:27.563 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:25:27.563 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "reached_pg", 2023-06-03T20:25:27.563 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:23:26.276949+0000", 2023-06-03T20:25:27.563 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 5.4159000000000001e-05 2023-06-03T20:25:27.564 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:25:27.564 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:25:27.564 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "waiting for peered", 2023-06-03T20:25:27.564 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:23:26.276960+0000", 2023-06-03T20:25:27.564 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 1.1028e-05 2023-06-03T20:25:27.564 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:25:27.564 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:25:27.565 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "reached_pg", 2023-06-03T20:25:27.565 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:23:27.247082+0000", 2023-06-03T20:25:27.565 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.97012133099999998 2023-06-03T20:25:27.565 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:25:27.565 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:25:27.565 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "waiting for peered", 2023-06-03T20:25:27.566 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:23:27.247115+0000", 2023-06-03T20:25:27.566 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 3.3305e-05 2023-06-03T20:25:27.566 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:25:27.566 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:25:27.566 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "reached_pg", 2023-06-03T20:25:27.566 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:23:28.741832+0000", 2023-06-03T20:25:27.566 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 1.4947163809999999 2023-06-03T20:25:27.567 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:25:27.567 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:25:27.567 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "started", 2023-06-03T20:25:27.567 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:23:28.741938+0000", 2023-06-03T20:25:27.567 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.000106173 2023-06-03T20:25:27.567 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:25:27.567 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:25:27.567 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "waiting for subops from 4,5", 2023-06-03T20:25:27.568 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:23:28.742022+0000", 2023-06-03T20:25:27.568 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 8.4766999999999997e-05 2023-06-03T20:25:27.568 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:25:27.568 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:25:27.568 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:25:27.568 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:23:28.745790+0000", 2023-06-03T20:25:27.568 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.0037672740000000001 2023-06-03T20:25:27.568 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:25:27.569 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:25:27.569 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "op_commit", 2023-06-03T20:25:27.569 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:23:28.750021+0000", 2023-06-03T20:25:27.569 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.0042314600000000003 2023-06-03T20:25:27.569 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:25:27.569 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:25:27.569 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:25:27.570 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:23:28.754606+0000", 2023-06-03T20:25:27.570 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.0045850830000000002 2023-06-03T20:25:27.570 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:25:27.570 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:25:27.570 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "commit_sent", 2023-06-03T20:25:27.570 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:23:28.754701+0000", 2023-06-03T20:25:27.570 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 9.4728999999999994e-05 2023-06-03T20:25:27.570 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:25:27.571 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:25:27.571 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "done", 2023-06-03T20:25:27.571 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:23:28.754850+0000", 2023-06-03T20:25:27.571 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.00014934400000000001 2023-06-03T20:25:27.571 INFO:teuthology.orchestra.run.smithi089.stdout: } 2023-06-03T20:25:27.571 INFO:teuthology.orchestra.run.smithi089.stdout: ] 2023-06-03T20:25:27.571 INFO:teuthology.orchestra.run.smithi089.stdout: } 2023-06-03T20:25:27.572 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:25:27.572 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:25:27.572 INFO:teuthology.orchestra.run.smithi089.stdout: "description": "osd_op(client.36573.0:157125 13.b 13:d198e8c5:::benchmark_data_smithi089_211581_object41600:head [delete] snapc 0=[] ondisk+write+known_if_redirected e257)", 2023-06-03T20:25:27.572 INFO:teuthology.orchestra.run.smithi089.stdout: "initiated_at": "2023-06-03T20:24:39.363088+0000", 2023-06-03T20:25:27.573 INFO:teuthology.orchestra.run.smithi089.stdout: "age": 48.185673414, 2023-06-03T20:25:27.573 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.240032098, 2023-06-03T20:25:27.573 INFO:teuthology.orchestra.run.smithi089.stdout: "type_data": { 2023-06-03T20:25:27.573 INFO:teuthology.orchestra.run.smithi089.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-03T20:25:27.573 INFO:teuthology.orchestra.run.smithi089.stdout: "client_info": { 2023-06-03T20:25:27.573 INFO:teuthology.orchestra.run.smithi089.stdout: "client": "client.36573", 2023-06-03T20:25:27.573 INFO:teuthology.orchestra.run.smithi089.stdout: "client_addr": "172.21.15.89:0/616445942", 2023-06-03T20:25:27.573 INFO:teuthology.orchestra.run.smithi089.stdout: "tid": 157125 2023-06-03T20:25:27.574 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:25:27.574 INFO:teuthology.orchestra.run.smithi089.stdout: "events": [ 2023-06-03T20:25:27.574 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:25:27.574 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "initiated", 2023-06-03T20:25:27.574 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:24:39.363088+0000", 2023-06-03T20:25:27.574 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0 2023-06-03T20:25:27.574 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:25:27.574 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:25:27.575 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "throttled", 2023-06-03T20:25:27.575 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:24:39.363088+0000", 2023-06-03T20:25:27.575 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0 2023-06-03T20:25:27.575 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:25:27.575 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:25:27.575 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "header_read", 2023-06-03T20:25:27.576 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:24:39.363087+0000", 2023-06-03T20:25:27.576 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 4294967295.9999981 2023-06-03T20:25:27.576 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:25:27.576 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:25:27.576 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "all_read", 2023-06-03T20:25:27.576 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:24:39.363094+0000", 2023-06-03T20:25:27.576 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 7.391e-06 2023-06-03T20:25:27.577 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:25:27.577 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:25:27.577 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "dispatched", 2023-06-03T20:25:27.577 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:24:39.363105+0000", 2023-06-03T20:25:27.577 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 1.1538e-05 2023-06-03T20:25:27.577 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:25:27.577 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:25:27.577 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "queued_for_pg", 2023-06-03T20:25:27.578 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:24:39.363112+0000", 2023-06-03T20:25:27.578 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 6.7390000000000002e-06 2023-06-03T20:25:27.578 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:25:27.578 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:25:27.578 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "reached_pg", 2023-06-03T20:25:27.578 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:24:39.363162+0000", 2023-06-03T20:25:27.578 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 4.9858999999999998e-05 2023-06-03T20:25:27.579 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:25:27.579 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:25:27.579 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "started", 2023-06-03T20:25:27.579 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:24:39.363269+0000", 2023-06-03T20:25:27.579 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.00010676 2023-06-03T20:25:27.580 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:25:27.580 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:25:27.581 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "waiting for subops from 2,5", 2023-06-03T20:25:27.581 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:24:39.363963+0000", 2023-06-03T20:25:27.581 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.00069423 2023-06-03T20:25:27.581 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:25:27.582 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:25:27.582 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "op_commit", 2023-06-03T20:25:27.582 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:24:39.370231+0000", 2023-06-03T20:25:27.582 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.0062679930000000003 2023-06-03T20:25:27.583 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:25:27.583 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:25:27.583 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:25:27.585 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:24:39.372235+0000", 2023-06-03T20:25:27.586 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.002003767 2023-06-03T20:25:27.586 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:25:27.586 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:25:27.586 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:25:27.586 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:24:39.602939+0000", 2023-06-03T20:25:27.586 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.23070391700000001 2023-06-03T20:25:27.586 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:25:27.587 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:25:27.587 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "commit_sent", 2023-06-03T20:25:27.587 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:24:39.603013+0000", 2023-06-03T20:25:27.587 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 7.3998000000000005e-05 2023-06-03T20:25:27.587 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:25:27.587 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:25:27.587 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "done", 2023-06-03T20:25:27.588 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:24:39.603121+0000", 2023-06-03T20:25:27.588 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.000107837 2023-06-03T20:25:27.588 INFO:teuthology.orchestra.run.smithi089.stdout: } 2023-06-03T20:25:27.588 INFO:teuthology.orchestra.run.smithi089.stdout: ] 2023-06-03T20:25:27.588 INFO:teuthology.orchestra.run.smithi089.stdout: } 2023-06-03T20:25:27.588 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:25:27.588 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:25:27.589 INFO:teuthology.orchestra.run.smithi089.stdout: "description": "osd_op(client.36573.0:179036 13.b 13:de955b9e:::benchmark_data_smithi089_211581_object63511:head [delete] snapc 0=[] ondisk+write+known_if_redirected e264)", 2023-06-03T20:25:27.589 INFO:teuthology.orchestra.run.smithi089.stdout: "initiated_at": "2023-06-03T20:25:19.550873+0000", 2023-06-03T20:25:27.589 INFO:teuthology.orchestra.run.smithi089.stdout: "age": 7.9978891000000001, 2023-06-03T20:25:27.589 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.29629856500000001, 2023-06-03T20:25:27.589 INFO:teuthology.orchestra.run.smithi089.stdout: "type_data": { 2023-06-03T20:25:27.589 INFO:teuthology.orchestra.run.smithi089.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-03T20:25:27.589 INFO:teuthology.orchestra.run.smithi089.stdout: "client_info": { 2023-06-03T20:25:27.590 INFO:teuthology.orchestra.run.smithi089.stdout: "client": "client.36573", 2023-06-03T20:25:27.590 INFO:teuthology.orchestra.run.smithi089.stdout: "client_addr": "172.21.15.89:0/616445942", 2023-06-03T20:25:27.590 INFO:teuthology.orchestra.run.smithi089.stdout: "tid": 179036 2023-06-03T20:25:27.590 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:25:27.590 INFO:teuthology.orchestra.run.smithi089.stdout: "events": [ 2023-06-03T20:25:27.590 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:25:27.590 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "initiated", 2023-06-03T20:25:27.590 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:25:19.550873+0000", 2023-06-03T20:25:27.591 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0 2023-06-03T20:25:27.591 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:25:27.591 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:25:27.591 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "throttled", 2023-06-03T20:25:27.591 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:25:19.550873+0000", 2023-06-03T20:25:27.591 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0 2023-06-03T20:25:27.591 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:25:27.592 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:25:27.592 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "header_read", 2023-06-03T20:25:27.592 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:25:19.550872+0000", 2023-06-03T20:25:27.592 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 4294967295.999999 2023-06-03T20:25:27.592 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:25:27.592 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:25:27.592 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "all_read", 2023-06-03T20:25:27.593 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:25:19.550877+0000", 2023-06-03T20:25:27.593 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 4.6789999999999996e-06 2023-06-03T20:25:27.593 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:25:27.593 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:25:27.593 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "dispatched", 2023-06-03T20:25:27.593 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:25:19.550886+0000", 2023-06-03T20:25:27.593 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 9.3060000000000003e-06 2023-06-03T20:25:27.596 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:25:27.596 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:25:27.596 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "queued_for_pg", 2023-06-03T20:25:27.596 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:25:19.550892+0000", 2023-06-03T20:25:27.596 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 6.0959999999999997e-06 2023-06-03T20:25:27.597 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:25:27.597 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:25:27.597 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "reached_pg", 2023-06-03T20:25:27.597 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:25:19.550930+0000", 2023-06-03T20:25:27.597 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 3.7557e-05 2023-06-03T20:25:27.597 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:25:27.597 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:25:27.598 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "started", 2023-06-03T20:25:27.598 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:25:19.551015+0000", 2023-06-03T20:25:27.598 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 8.53e-05 2023-06-03T20:25:27.598 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:25:27.598 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:25:27.598 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "waiting for subops from 2,5", 2023-06-03T20:25:27.598 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:25:19.551090+0000", 2023-06-03T20:25:27.598 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 7.5228999999999994e-05 2023-06-03T20:25:27.599 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:25:27.599 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:25:27.599 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "op_commit", 2023-06-03T20:25:27.599 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:25:19.554513+0000", 2023-06-03T20:25:27.599 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.003422965 2023-06-03T20:25:27.599 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:25:27.599 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:25:27.599 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:25:27.600 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:25:19.556734+0000", 2023-06-03T20:25:27.600 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.0022206320000000002 2023-06-03T20:25:27.600 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:25:27.600 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:25:27.600 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:25:27.600 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:25:19.847056+0000", 2023-06-03T20:25:27.600 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.29032211499999999 2023-06-03T20:25:27.601 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:25:27.601 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:25:27.601 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "commit_sent", 2023-06-03T20:25:27.601 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:25:19.847129+0000", 2023-06-03T20:25:27.601 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 7.3175000000000003e-05 2023-06-03T20:25:27.601 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:25:27.601 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:25:27.602 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "done", 2023-06-03T20:25:27.602 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:25:19.847171+0000", 2023-06-03T20:25:27.602 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 4.2301e-05 2023-06-03T20:25:27.602 INFO:teuthology.orchestra.run.smithi089.stdout: } 2023-06-03T20:25:27.602 INFO:teuthology.orchestra.run.smithi089.stdout: ] 2023-06-03T20:25:27.602 INFO:teuthology.orchestra.run.smithi089.stdout: } 2023-06-03T20:25:27.602 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:25:27.602 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:25:27.603 INFO:teuthology.orchestra.run.smithi089.stdout: "description": "osd_op(client.36573.0:181633 13.6 13:610eb2b6:::benchmark_data_smithi089_211581_object66108:head [delete] snapc 0=[] ondisk+write+known_if_redirected e264)", 2023-06-03T20:25:27.603 INFO:teuthology.orchestra.run.smithi089.stdout: "initiated_at": "2023-06-03T20:25:23.848734+0000", 2023-06-03T20:25:27.603 INFO:teuthology.orchestra.run.smithi089.stdout: "age": 3.7000282220000003, 2023-06-03T20:25:27.603 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.27333703799999998, 2023-06-03T20:25:27.603 INFO:teuthology.orchestra.run.smithi089.stdout: "type_data": { 2023-06-03T20:25:27.603 INFO:teuthology.orchestra.run.smithi089.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-03T20:25:27.603 INFO:teuthology.orchestra.run.smithi089.stdout: "client_info": { 2023-06-03T20:25:27.603 INFO:teuthology.orchestra.run.smithi089.stdout: "client": "client.36573", 2023-06-03T20:25:27.604 INFO:teuthology.orchestra.run.smithi089.stdout: "client_addr": "172.21.15.89:0/616445942", 2023-06-03T20:25:27.604 INFO:teuthology.orchestra.run.smithi089.stdout: "tid": 181633 2023-06-03T20:25:27.604 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:25:27.604 INFO:teuthology.orchestra.run.smithi089.stdout: "events": [ 2023-06-03T20:25:27.604 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:25:27.604 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "initiated", 2023-06-03T20:25:27.604 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:25:23.848734+0000", 2023-06-03T20:25:27.605 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0 2023-06-03T20:25:27.605 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:25:27.605 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:25:27.605 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "throttled", 2023-06-03T20:25:27.606 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:25:23.848734+0000", 2023-06-03T20:25:27.606 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0 2023-06-03T20:25:27.606 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:25:27.606 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:25:27.606 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "header_read", 2023-06-03T20:25:27.606 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:25:23.848732+0000", 2023-06-03T20:25:27.607 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 4294967295.9999981 2023-06-03T20:25:27.607 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:25:27.607 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:25:27.607 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "all_read", 2023-06-03T20:25:27.607 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:25:23.848739+0000", 2023-06-03T20:25:27.607 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 7.6520000000000006e-06 2023-06-03T20:25:27.607 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:25:27.608 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:25:27.608 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "dispatched", 2023-06-03T20:25:27.608 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:25:23.848752+0000", 2023-06-03T20:25:27.608 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 1.2733e-05 2023-06-03T20:25:27.608 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:25:27.608 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:25:27.608 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "queued_for_pg", 2023-06-03T20:25:27.608 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:25:23.848760+0000", 2023-06-03T20:25:27.609 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 7.8390000000000007e-06 2023-06-03T20:25:27.609 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:25:27.609 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:25:27.609 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "reached_pg", 2023-06-03T20:25:27.609 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:25:23.848820+0000", 2023-06-03T20:25:27.609 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 5.9880999999999998e-05 2023-06-03T20:25:27.609 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:25:27.610 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:25:27.610 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "started", 2023-06-03T20:25:27.610 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:25:23.848955+0000", 2023-06-03T20:25:27.610 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.00013507800000000001 2023-06-03T20:25:27.610 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:25:27.610 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:25:27.610 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "waiting for subops from 4,5", 2023-06-03T20:25:27.610 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:25:23.849438+0000", 2023-06-03T20:25:27.611 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.00048347599999999999 2023-06-03T20:25:27.611 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:25:27.611 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:25:27.611 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "op_commit", 2023-06-03T20:25:27.611 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:25:23.853472+0000", 2023-06-03T20:25:27.611 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.0040334029999999996 2023-06-03T20:25:27.611 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:25:27.612 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:25:27.612 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:25:27.612 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:25:23.867575+0000", 2023-06-03T20:25:27.612 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.014103667 2023-06-03T20:25:27.612 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:25:27.612 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:25:27.612 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:25:27.612 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:25:24.121918+0000", 2023-06-03T20:25:27.613 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.25434207599999997 2023-06-03T20:25:27.613 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:25:27.613 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:25:27.613 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "commit_sent", 2023-06-03T20:25:27.613 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:25:24.121967+0000", 2023-06-03T20:25:27.613 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 4.9369000000000001e-05 2023-06-03T20:25:27.613 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:25:27.614 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:25:27.614 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "done", 2023-06-03T20:25:27.614 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:25:24.122071+0000", 2023-06-03T20:25:27.614 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.00010382000000000001 2023-06-03T20:25:27.614 INFO:teuthology.orchestra.run.smithi089.stdout: } 2023-06-03T20:25:27.614 INFO:teuthology.orchestra.run.smithi089.stdout: ] 2023-06-03T20:25:27.614 INFO:teuthology.orchestra.run.smithi089.stdout: } 2023-06-03T20:25:27.614 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:25:27.615 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:25:27.615 INFO:teuthology.orchestra.run.smithi089.stdout: "description": "osd_op(client.36573.0:183137 13.b 13:d4340b79:::benchmark_data_smithi089_211581_object67612:head [delete] snapc 0=[] ondisk+write+known_if_redirected e264)", 2023-06-03T20:25:27.615 INFO:teuthology.orchestra.run.smithi089.stdout: "initiated_at": "2023-06-03T20:25:26.391380+0000", 2023-06-03T20:25:27.615 INFO:teuthology.orchestra.run.smithi089.stdout: "age": 1.1573815240000001, 2023-06-03T20:25:27.615 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.096127405999999999, 2023-06-03T20:25:27.615 INFO:teuthology.orchestra.run.smithi089.stdout: "type_data": { 2023-06-03T20:25:27.615 INFO:teuthology.orchestra.run.smithi089.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-03T20:25:27.616 INFO:teuthology.orchestra.run.smithi089.stdout: "client_info": { 2023-06-03T20:25:27.616 INFO:teuthology.orchestra.run.smithi089.stdout: "client": "client.36573", 2023-06-03T20:25:27.616 INFO:teuthology.orchestra.run.smithi089.stdout: "client_addr": "172.21.15.89:0/616445942", 2023-06-03T20:25:27.617 INFO:teuthology.orchestra.run.smithi089.stdout: "tid": 183137 2023-06-03T20:25:27.617 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:25:27.617 INFO:teuthology.orchestra.run.smithi089.stdout: "events": [ 2023-06-03T20:25:27.617 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:25:27.617 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "initiated", 2023-06-03T20:25:27.617 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:25:26.391380+0000", 2023-06-03T20:25:27.617 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0 2023-06-03T20:25:27.618 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:25:27.618 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:25:27.618 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "throttled", 2023-06-03T20:25:27.618 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:25:26.391380+0000", 2023-06-03T20:25:27.618 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0 2023-06-03T20:25:27.618 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:25:27.618 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:25:27.619 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "header_read", 2023-06-03T20:25:27.619 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:25:26.391379+0000", 2023-06-03T20:25:27.619 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 4294967295.9999986 2023-06-03T20:25:27.619 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:25:27.619 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:25:27.619 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "all_read", 2023-06-03T20:25:27.619 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:25:26.391385+0000", 2023-06-03T20:25:27.619 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 6.1539999999999999e-06 2023-06-03T20:25:27.620 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:25:27.620 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:25:27.620 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "dispatched", 2023-06-03T20:25:27.620 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:25:26.391398+0000", 2023-06-03T20:25:27.620 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 1.2407e-05 2023-06-03T20:25:27.620 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:25:27.621 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:25:27.621 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "queued_for_pg", 2023-06-03T20:25:27.621 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:25:26.391404+0000", 2023-06-03T20:25:27.621 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 6.6109999999999999e-06 2023-06-03T20:25:27.621 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:25:27.621 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:25:27.621 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "reached_pg", 2023-06-03T20:25:27.621 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:25:26.391444+0000", 2023-06-03T20:25:27.622 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 4.0086999999999998e-05 2023-06-03T20:25:27.622 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:25:27.622 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:25:27.622 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "started", 2023-06-03T20:25:27.622 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:25:26.391531+0000", 2023-06-03T20:25:27.622 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 8.7070999999999994e-05 2023-06-03T20:25:27.622 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:25:27.622 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:25:27.623 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "waiting for subops from 2,5", 2023-06-03T20:25:27.623 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:25:26.391607+0000", 2023-06-03T20:25:27.623 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 7.5893e-05 2023-06-03T20:25:27.623 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:25:27.623 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:25:27.623 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "op_commit", 2023-06-03T20:25:27.623 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:25:26.395155+0000", 2023-06-03T20:25:27.624 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.0035480989999999999 2023-06-03T20:25:27.624 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:25:27.624 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:25:27.624 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:25:27.624 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:25:26.399531+0000", 2023-06-03T20:25:27.624 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.0043753409999999996 2023-06-03T20:25:27.624 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:25:27.625 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:25:27.625 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:25:27.625 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:25:26.487375+0000", 2023-06-03T20:25:27.625 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.087843798000000001 2023-06-03T20:25:27.625 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:25:27.625 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:25:27.625 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "commit_sent", 2023-06-03T20:25:27.625 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:25:26.487456+0000", 2023-06-03T20:25:27.626 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 8.1316000000000005e-05 2023-06-03T20:25:27.626 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:25:27.626 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:25:27.626 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "done", 2023-06-03T20:25:27.626 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:25:26.487508+0000", 2023-06-03T20:25:27.626 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 5.1940000000000001e-05 2023-06-03T20:25:27.626 INFO:teuthology.orchestra.run.smithi089.stdout: } 2023-06-03T20:25:27.627 INFO:teuthology.orchestra.run.smithi089.stdout: ] 2023-06-03T20:25:27.627 INFO:teuthology.orchestra.run.smithi089.stdout: } 2023-06-03T20:25:27.627 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:25:27.627 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:25:27.627 INFO:teuthology.orchestra.run.smithi089.stdout: "description": "osd_op(client.36573.0:183370 13.b 13:db967720:::benchmark_data_smithi089_211581_object67845:head [delete] snapc 0=[] ondisk+write+known_if_redirected e264)", 2023-06-03T20:25:27.627 INFO:teuthology.orchestra.run.smithi089.stdout: "initiated_at": "2023-06-03T20:25:26.991569+0000", 2023-06-03T20:25:27.627 INFO:teuthology.orchestra.run.smithi089.stdout: "age": 0.55719261099999995, 2023-06-03T20:25:27.628 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.092631150999999995, 2023-06-03T20:25:27.628 INFO:teuthology.orchestra.run.smithi089.stdout: "type_data": { 2023-06-03T20:25:27.628 INFO:teuthology.orchestra.run.smithi089.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-03T20:25:27.628 INFO:teuthology.orchestra.run.smithi089.stdout: "client_info": { 2023-06-03T20:25:27.628 INFO:teuthology.orchestra.run.smithi089.stdout: "client": "client.36573", 2023-06-03T20:25:27.628 INFO:teuthology.orchestra.run.smithi089.stdout: "client_addr": "172.21.15.89:0/616445942", 2023-06-03T20:25:27.628 INFO:teuthology.orchestra.run.smithi089.stdout: "tid": 183370 2023-06-03T20:25:27.629 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:25:27.629 INFO:teuthology.orchestra.run.smithi089.stdout: "events": [ 2023-06-03T20:25:27.629 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:25:27.629 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "initiated", 2023-06-03T20:25:27.629 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:25:26.991569+0000", 2023-06-03T20:25:27.629 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0 2023-06-03T20:25:27.629 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:25:27.629 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:25:27.630 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "throttled", 2023-06-03T20:25:27.630 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:25:26.991569+0000", 2023-06-03T20:25:27.630 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0 2023-06-03T20:25:27.630 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:25:27.630 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:25:27.630 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "header_read", 2023-06-03T20:25:27.630 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:25:26.991568+0000", 2023-06-03T20:25:27.630 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 4294967295.999999 2023-06-03T20:25:27.631 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:25:27.631 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:25:27.631 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "all_read", 2023-06-03T20:25:27.631 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:25:26.991574+0000", 2023-06-03T20:25:27.631 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 6.0739999999999998e-06 2023-06-03T20:25:27.631 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:25:27.631 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:25:27.631 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "dispatched", 2023-06-03T20:25:27.632 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:25:26.991585+0000", 2023-06-03T20:25:27.632 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 1.0678e-05 2023-06-03T20:25:27.632 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:25:27.632 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:25:27.632 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "queued_for_pg", 2023-06-03T20:25:27.632 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:25:26.991593+0000", 2023-06-03T20:25:27.632 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 7.7349999999999996e-06 2023-06-03T20:25:27.633 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:25:27.633 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:25:27.633 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "reached_pg", 2023-06-03T20:25:27.633 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:25:26.991646+0000", 2023-06-03T20:25:27.633 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 5.3247e-05 2023-06-03T20:25:27.633 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:25:27.633 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:25:27.633 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "started", 2023-06-03T20:25:27.634 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:25:26.991749+0000", 2023-06-03T20:25:27.634 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.000102958 2023-06-03T20:25:27.634 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:25:27.634 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:25:27.634 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "waiting for subops from 2,5", 2023-06-03T20:25:27.634 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:25:26.991846+0000", 2023-06-03T20:25:27.634 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 9.7499999999999998e-05 2023-06-03T20:25:27.634 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:25:27.635 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:25:27.635 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:25:27.635 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:25:27.083910+0000", 2023-06-03T20:25:27.635 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.092063435999999998 2023-06-03T20:25:27.635 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:25:27.635 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:25:27.635 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "op_commit", 2023-06-03T20:25:27.635 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:25:27.084090+0000", 2023-06-03T20:25:27.636 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.000180516 2023-06-03T20:25:27.636 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:25:27.636 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:25:27.636 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:25:27.636 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:25:27.084123+0000", 2023-06-03T20:25:27.636 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 3.2472999999999999e-05 2023-06-03T20:25:27.636 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:25:27.637 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:25:27.637 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "commit_sent", 2023-06-03T20:25:27.637 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:25:27.084161+0000", 2023-06-03T20:25:27.637 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 3.8464999999999998e-05 2023-06-03T20:25:27.637 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:25:27.637 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:25:27.637 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "done", 2023-06-03T20:25:27.637 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:25:27.084200+0000", 2023-06-03T20:25:27.638 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 3.9224999999999998e-05 2023-06-03T20:25:27.638 INFO:teuthology.orchestra.run.smithi089.stdout: } 2023-06-03T20:25:27.638 INFO:teuthology.orchestra.run.smithi089.stdout: ] 2023-06-03T20:25:27.638 INFO:teuthology.orchestra.run.smithi089.stdout: } 2023-06-03T20:25:27.638 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:25:27.638 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:25:27.638 INFO:teuthology.orchestra.run.smithi089.stdout: "description": "osd_op(client.36573.0:183371 13.b 13:d5766999:::benchmark_data_smithi089_211581_object67846:head [delete] snapc 0=[] ondisk+write+known_if_redirected e264)", 2023-06-03T20:25:27.639 INFO:teuthology.orchestra.run.smithi089.stdout: "initiated_at": "2023-06-03T20:25:26.991607+0000", 2023-06-03T20:25:27.639 INFO:teuthology.orchestra.run.smithi089.stdout: "age": 0.55715466499999999, 2023-06-03T20:25:27.639 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.092952520999999996, 2023-06-03T20:25:27.639 INFO:teuthology.orchestra.run.smithi089.stdout: "type_data": { 2023-06-03T20:25:27.639 INFO:teuthology.orchestra.run.smithi089.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-03T20:25:27.639 INFO:teuthology.orchestra.run.smithi089.stdout: "client_info": { 2023-06-03T20:25:27.639 INFO:teuthology.orchestra.run.smithi089.stdout: "client": "client.36573", 2023-06-03T20:25:27.639 INFO:teuthology.orchestra.run.smithi089.stdout: "client_addr": "172.21.15.89:0/616445942", 2023-06-03T20:25:27.640 INFO:teuthology.orchestra.run.smithi089.stdout: "tid": 183371 2023-06-03T20:25:27.640 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:25:27.640 INFO:teuthology.orchestra.run.smithi089.stdout: "events": [ 2023-06-03T20:25:27.640 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:25:27.640 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "initiated", 2023-06-03T20:25:27.640 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:25:26.991607+0000", 2023-06-03T20:25:27.640 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0 2023-06-03T20:25:27.641 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:25:27.641 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:25:27.641 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "throttled", 2023-06-03T20:25:27.641 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:25:26.991607+0000", 2023-06-03T20:25:27.641 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0 2023-06-03T20:25:27.641 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:25:27.641 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:25:27.641 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "header_read", 2023-06-03T20:25:27.642 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:25:26.991607+0000", 2023-06-03T20:25:27.642 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 4294967295.9999995 2023-06-03T20:25:27.642 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:25:27.642 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:25:27.642 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "all_read", 2023-06-03T20:25:27.642 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:25:26.991610+0000", 2023-06-03T20:25:27.642 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 3.0359999999999998e-06 2023-06-03T20:25:27.643 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:25:27.643 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:25:27.643 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "dispatched", 2023-06-03T20:25:27.643 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:25:26.991617+0000", 2023-06-03T20:25:27.643 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 7.0770000000000002e-06 2023-06-03T20:25:27.643 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:25:27.643 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:25:27.643 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "queued_for_pg", 2023-06-03T20:25:27.644 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:25:26.991622+0000", 2023-06-03T20:25:27.644 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 5.4569999999999996e-06 2023-06-03T20:25:27.644 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:25:27.644 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:25:27.644 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "reached_pg", 2023-06-03T20:25:27.645 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:25:26.996347+0000", 2023-06-03T20:25:27.645 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.0047245719999999998 2023-06-03T20:25:27.645 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:25:27.645 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:25:27.645 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "started", 2023-06-03T20:25:27.645 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:25:26.996461+0000", 2023-06-03T20:25:27.645 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.00011387699999999999 2023-06-03T20:25:27.646 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:25:27.646 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:25:27.646 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "waiting for subops from 2,5", 2023-06-03T20:25:27.646 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:25:26.996543+0000", 2023-06-03T20:25:27.646 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 8.2689999999999996e-05 2023-06-03T20:25:27.646 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:25:27.646 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:25:27.647 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:25:27.647 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:25:27.083949+0000", 2023-06-03T20:25:27.647 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.087405347999999994 2023-06-03T20:25:27.647 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:25:27.647 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:25:27.647 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:25:27.647 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:25:27.084478+0000", 2023-06-03T20:25:27.647 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.00052888700000000002 2023-06-03T20:25:27.648 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:25:27.648 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:25:27.648 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "op_commit", 2023-06-03T20:25:27.648 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:25:27.084486+0000", 2023-06-03T20:25:27.648 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 8.6889999999999993e-06 2023-06-03T20:25:27.649 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:25:27.649 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:25:27.649 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "commit_sent", 2023-06-03T20:25:27.649 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:25:27.084523+0000", 2023-06-03T20:25:27.649 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 3.6890000000000001e-05 2023-06-03T20:25:27.649 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:25:27.649 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:25:27.649 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "done", 2023-06-03T20:25:27.650 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:25:27.084560+0000", 2023-06-03T20:25:27.650 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 3.6474999999999999e-05 2023-06-03T20:25:27.650 INFO:teuthology.orchestra.run.smithi089.stdout: } 2023-06-03T20:25:27.650 INFO:teuthology.orchestra.run.smithi089.stdout: ] 2023-06-03T20:25:27.650 INFO:teuthology.orchestra.run.smithi089.stdout: } 2023-06-03T20:25:27.650 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:25:27.650 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:25:27.651 INFO:teuthology.orchestra.run.smithi089.stdout: "description": "osd_op(client.36573.0:183373 13.16 13:693b96b5:::benchmark_data_smithi089_211581_object67848:head [delete] snapc 0=[] ondisk+write+known_if_redirected e264)", 2023-06-03T20:25:27.651 INFO:teuthology.orchestra.run.smithi089.stdout: "initiated_at": "2023-06-03T20:25:26.994374+0000", 2023-06-03T20:25:27.651 INFO:teuthology.orchestra.run.smithi089.stdout: "age": 0.55438741199999997, 2023-06-03T20:25:27.651 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.095132218000000004, 2023-06-03T20:25:27.651 INFO:teuthology.orchestra.run.smithi089.stdout: "type_data": { 2023-06-03T20:25:27.651 INFO:teuthology.orchestra.run.smithi089.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-03T20:25:27.651 INFO:teuthology.orchestra.run.smithi089.stdout: "client_info": { 2023-06-03T20:25:27.652 INFO:teuthology.orchestra.run.smithi089.stdout: "client": "client.36573", 2023-06-03T20:25:27.652 INFO:teuthology.orchestra.run.smithi089.stdout: "client_addr": "172.21.15.89:0/616445942", 2023-06-03T20:25:27.652 INFO:teuthology.orchestra.run.smithi089.stdout: "tid": 183373 2023-06-03T20:25:27.652 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:25:27.652 INFO:teuthology.orchestra.run.smithi089.stdout: "events": [ 2023-06-03T20:25:27.653 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:25:27.653 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "initiated", 2023-06-03T20:25:27.653 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:25:26.994374+0000", 2023-06-03T20:25:27.653 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0 2023-06-03T20:25:27.653 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:25:27.653 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:25:27.653 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "throttled", 2023-06-03T20:25:27.654 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:25:26.994374+0000", 2023-06-03T20:25:27.654 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0 2023-06-03T20:25:27.654 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:25:27.654 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:25:27.654 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "header_read", 2023-06-03T20:25:27.654 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:25:26.994373+0000", 2023-06-03T20:25:27.654 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 4294967295.9999981 2023-06-03T20:25:27.654 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:25:27.655 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:25:27.655 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "all_read", 2023-06-03T20:25:27.655 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:25:26.994381+0000", 2023-06-03T20:25:27.655 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 8.4749999999999993e-06 2023-06-03T20:25:27.655 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:25:27.655 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:25:27.655 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "dispatched", 2023-06-03T20:25:27.656 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:25:26.994396+0000", 2023-06-03T20:25:27.656 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 1.5031000000000001e-05 2023-06-03T20:25:27.656 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:25:27.656 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:25:27.658 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "queued_for_pg", 2023-06-03T20:25:27.658 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:25:26.994406+0000", 2023-06-03T20:25:27.658 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 9.4839999999999993e-06 2023-06-03T20:25:27.659 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:25:27.659 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:25:27.659 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "reached_pg", 2023-06-03T20:25:27.659 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:25:26.995007+0000", 2023-06-03T20:25:27.659 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.00060142599999999996 2023-06-03T20:25:27.660 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:25:27.660 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:25:27.660 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "started", 2023-06-03T20:25:27.661 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:25:26.995129+0000", 2023-06-03T20:25:27.661 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.000121786 2023-06-03T20:25:27.662 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:25:27.662 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:25:27.662 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "waiting for subops from 4,5", 2023-06-03T20:25:27.662 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:25:26.995509+0000", 2023-06-03T20:25:27.662 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.00038038200000000002 2023-06-03T20:25:27.662 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:25:27.662 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:25:27.663 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:25:27.663 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:25:27.086421+0000", 2023-06-03T20:25:27.663 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.090911386999999996 2023-06-03T20:25:27.663 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:25:27.663 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:25:27.663 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:25:27.663 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:25:27.086455+0000", 2023-06-03T20:25:27.663 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 3.4013000000000001e-05 2023-06-03T20:25:27.664 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:25:27.664 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:25:27.664 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "op_commit", 2023-06-03T20:25:27.664 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:25:27.089349+0000", 2023-06-03T20:25:27.664 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.002894621 2023-06-03T20:25:27.664 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:25:27.664 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:25:27.665 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "commit_sent", 2023-06-03T20:25:27.665 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:25:27.089407+0000", 2023-06-03T20:25:27.665 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 5.7438999999999997e-05 2023-06-03T20:25:27.665 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:25:27.665 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:25:27.665 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "done", 2023-06-03T20:25:27.665 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:25:27.089507+0000", 2023-06-03T20:25:27.665 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 9.9908000000000006e-05 2023-06-03T20:25:27.666 INFO:teuthology.orchestra.run.smithi089.stdout: } 2023-06-03T20:25:27.666 INFO:teuthology.orchestra.run.smithi089.stdout: ] 2023-06-03T20:25:27.666 INFO:teuthology.orchestra.run.smithi089.stdout: } 2023-06-03T20:25:27.666 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:25:27.666 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:25:27.667 INFO:teuthology.orchestra.run.smithi089.stdout: "description": "osd_op(client.36573.0:183374 13.12 13:4ae0bc2e:::benchmark_data_smithi089_211581_object67849:head [delete] snapc 0=[] ondisk+write+known_if_redirected e264)", 2023-06-03T20:25:27.667 INFO:teuthology.orchestra.run.smithi089.stdout: "initiated_at": "2023-06-03T20:25:26.994416+0000", 2023-06-03T20:25:27.667 INFO:teuthology.orchestra.run.smithi089.stdout: "age": 0.55434570599999999, 2023-06-03T20:25:27.668 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.089840240000000002, 2023-06-03T20:25:27.668 INFO:teuthology.orchestra.run.smithi089.stdout: "type_data": { 2023-06-03T20:25:27.668 INFO:teuthology.orchestra.run.smithi089.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-03T20:25:27.668 INFO:teuthology.orchestra.run.smithi089.stdout: "client_info": { 2023-06-03T20:25:27.668 INFO:teuthology.orchestra.run.smithi089.stdout: "client": "client.36573", 2023-06-03T20:25:27.669 INFO:teuthology.orchestra.run.smithi089.stdout: "client_addr": "172.21.15.89:0/616445942", 2023-06-03T20:25:27.669 INFO:teuthology.orchestra.run.smithi089.stdout: "tid": 183374 2023-06-03T20:25:27.669 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:25:27.669 INFO:teuthology.orchestra.run.smithi089.stdout: "events": [ 2023-06-03T20:25:27.669 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:25:27.669 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "initiated", 2023-06-03T20:25:27.669 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:25:26.994416+0000", 2023-06-03T20:25:27.670 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0 2023-06-03T20:25:27.670 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:25:27.670 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:25:27.670 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "throttled", 2023-06-03T20:25:27.670 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:25:26.994416+0000", 2023-06-03T20:25:27.670 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0 2023-06-03T20:25:27.670 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:25:27.670 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:25:27.671 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "header_read", 2023-06-03T20:25:27.671 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:25:26.994416+0000", 2023-06-03T20:25:27.671 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 4294967295.9999995 2023-06-03T20:25:27.672 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:25:27.672 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:25:27.673 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "all_read", 2023-06-03T20:25:27.673 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:25:26.994419+0000", 2023-06-03T20:25:27.674 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 3.4410000000000002e-06 2023-06-03T20:25:27.674 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:25:27.675 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:25:27.675 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "dispatched", 2023-06-03T20:25:27.675 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:25:26.994426+0000", 2023-06-03T20:25:27.675 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 6.5440000000000004e-06 2023-06-03T20:25:27.675 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:25:27.675 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:25:27.676 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "queued_for_pg", 2023-06-03T20:25:27.676 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:25:26.994431+0000", 2023-06-03T20:25:27.676 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 5.5690000000000004e-06 2023-06-03T20:25:27.676 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:25:27.676 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:25:27.676 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "reached_pg", 2023-06-03T20:25:27.676 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:25:26.994478+0000", 2023-06-03T20:25:27.677 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 4.6400000000000003e-05 2023-06-03T20:25:27.677 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:25:27.677 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:25:27.677 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "started", 2023-06-03T20:25:27.677 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:25:26.994628+0000", 2023-06-03T20:25:27.677 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.000150476 2023-06-03T20:25:27.678 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:25:27.678 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:25:27.678 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "waiting for subops from 4,5", 2023-06-03T20:25:27.678 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:25:26.995054+0000", 2023-06-03T20:25:27.678 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.00042546000000000001 2023-06-03T20:25:27.678 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:25:27.678 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:25:27.679 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "op_commit", 2023-06-03T20:25:27.679 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:25:26.997486+0000", 2023-06-03T20:25:27.679 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.002432511 2023-06-03T20:25:27.679 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:25:27.679 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:25:27.679 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:25:27.679 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:25:26.997744+0000", 2023-06-03T20:25:27.679 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.00025754899999999998 2023-06-03T20:25:27.680 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:25:27.680 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:25:27.680 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:25:27.680 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:25:27.084096+0000", 2023-06-03T20:25:27.680 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.086352532999999995 2023-06-03T20:25:27.680 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:25:27.680 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:25:27.681 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "commit_sent", 2023-06-03T20:25:27.681 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:25:27.084158+0000", 2023-06-03T20:25:27.681 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 6.1884000000000006e-05 2023-06-03T20:25:27.681 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:25:27.681 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:25:27.681 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "done", 2023-06-03T20:25:27.681 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:25:27.084256+0000", 2023-06-03T20:25:27.681 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 9.8354000000000001e-05 2023-06-03T20:25:27.682 INFO:teuthology.orchestra.run.smithi089.stdout: } 2023-06-03T20:25:27.682 INFO:teuthology.orchestra.run.smithi089.stdout: ] 2023-06-03T20:25:27.682 INFO:teuthology.orchestra.run.smithi089.stdout: } 2023-06-03T20:25:27.682 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:25:27.682 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:25:27.682 INFO:teuthology.orchestra.run.smithi089.stdout: "description": "osd_op(client.36573.0:183375 13.16 13:6c7b52f0:::benchmark_data_smithi089_211581_object67850:head [delete] snapc 0=[] ondisk+write+known_if_redirected e264)", 2023-06-03T20:25:27.682 INFO:teuthology.orchestra.run.smithi089.stdout: "initiated_at": "2023-06-03T20:25:26.994445+0000", 2023-06-03T20:25:27.682 INFO:teuthology.orchestra.run.smithi089.stdout: "age": 0.554316433, 2023-06-03T20:25:27.683 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.095204868999999998, 2023-06-03T20:25:27.683 INFO:teuthology.orchestra.run.smithi089.stdout: "type_data": { 2023-06-03T20:25:27.683 INFO:teuthology.orchestra.run.smithi089.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-03T20:25:27.683 INFO:teuthology.orchestra.run.smithi089.stdout: "client_info": { 2023-06-03T20:25:27.683 INFO:teuthology.orchestra.run.smithi089.stdout: "client": "client.36573", 2023-06-03T20:25:27.683 INFO:teuthology.orchestra.run.smithi089.stdout: "client_addr": "172.21.15.89:0/616445942", 2023-06-03T20:25:27.683 INFO:teuthology.orchestra.run.smithi089.stdout: "tid": 183375 2023-06-03T20:25:27.684 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:25:27.684 INFO:teuthology.orchestra.run.smithi089.stdout: "events": [ 2023-06-03T20:25:27.684 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:25:27.684 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "initiated", 2023-06-03T20:25:27.684 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:25:26.994445+0000", 2023-06-03T20:25:27.684 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0 2023-06-03T20:25:27.684 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:25:27.684 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:25:27.685 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "throttled", 2023-06-03T20:25:27.685 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:25:26.994445+0000", 2023-06-03T20:25:27.685 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0 2023-06-03T20:25:27.685 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:25:27.685 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:25:27.685 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "header_read", 2023-06-03T20:25:27.685 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:25:26.994445+0000", 2023-06-03T20:25:27.686 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 4294967295.9999995 2023-06-03T20:25:27.686 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:25:27.686 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:25:27.686 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "all_read", 2023-06-03T20:25:27.686 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:25:26.994448+0000", 2023-06-03T20:25:27.686 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 3.2219999999999998e-06 2023-06-03T20:25:27.686 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:25:27.686 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:25:27.687 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "dispatched", 2023-06-03T20:25:27.687 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:25:26.994455+0000", 2023-06-03T20:25:27.687 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 6.584e-06 2023-06-03T20:25:27.687 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:25:27.687 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:25:27.687 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "queued_for_pg", 2023-06-03T20:25:27.687 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:25:26.994460+0000", 2023-06-03T20:25:27.688 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 5.2569999999999998e-06 2023-06-03T20:25:27.688 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:25:27.688 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:25:27.688 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "reached_pg", 2023-06-03T20:25:27.688 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:25:26.997643+0000", 2023-06-03T20:25:27.688 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.0031831329999999999 2023-06-03T20:25:27.688 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:25:27.688 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:25:27.689 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "started", 2023-06-03T20:25:27.689 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:25:26.997768+0000", 2023-06-03T20:25:27.689 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.000125325 2023-06-03T20:25:27.689 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:25:27.689 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:25:27.689 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "waiting for subops from 4,5", 2023-06-03T20:25:27.689 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:25:26.998123+0000", 2023-06-03T20:25:27.689 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.00035499700000000002 2023-06-03T20:25:27.690 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:25:27.690 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:25:27.690 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:25:27.690 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:25:27.086486+0000", 2023-06-03T20:25:27.691 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.088362274000000005 2023-06-03T20:25:27.691 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:25:27.691 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:25:27.691 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:25:27.691 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:25:27.086516+0000", 2023-06-03T20:25:27.691 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 3.061e-05 2023-06-03T20:25:27.691 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:25:27.691 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:25:27.692 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "op_commit", 2023-06-03T20:25:27.692 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:25:27.089517+0000", 2023-06-03T20:25:27.692 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.0030006099999999999 2023-06-03T20:25:27.692 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:25:27.692 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:25:27.692 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "commit_sent", 2023-06-03T20:25:27.692 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:25:27.089559+0000", 2023-06-03T20:25:27.693 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 4.2586999999999997e-05 2023-06-03T20:25:27.694 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:25:27.694 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:25:27.694 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "done", 2023-06-03T20:25:27.694 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:25:27.089650+0000", 2023-06-03T20:25:27.694 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 9.0866999999999996e-05 2023-06-03T20:25:27.694 INFO:teuthology.orchestra.run.smithi089.stdout: } 2023-06-03T20:25:27.694 INFO:teuthology.orchestra.run.smithi089.stdout: ] 2023-06-03T20:25:27.695 INFO:teuthology.orchestra.run.smithi089.stdout: } 2023-06-03T20:25:27.695 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:25:27.695 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:25:27.695 INFO:teuthology.orchestra.run.smithi089.stdout: "description": "osd_op(client.36573.0:183380 13.16 13:6d195f35:::benchmark_data_smithi089_211581_object67855:head [delete] snapc 0=[] ondisk+write+known_if_redirected e264)", 2023-06-03T20:25:27.695 INFO:teuthology.orchestra.run.smithi089.stdout: "initiated_at": "2023-06-03T20:25:26.998082+0000", 2023-06-03T20:25:27.695 INFO:teuthology.orchestra.run.smithi089.stdout: "age": 0.55068027900000005, 2023-06-03T20:25:27.695 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.103962818, 2023-06-03T20:25:27.695 INFO:teuthology.orchestra.run.smithi089.stdout: "type_data": { 2023-06-03T20:25:27.696 INFO:teuthology.orchestra.run.smithi089.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-03T20:25:27.696 INFO:teuthology.orchestra.run.smithi089.stdout: "client_info": { 2023-06-03T20:25:27.696 INFO:teuthology.orchestra.run.smithi089.stdout: "client": "client.36573", 2023-06-03T20:25:27.696 INFO:teuthology.orchestra.run.smithi089.stdout: "client_addr": "172.21.15.89:0/616445942", 2023-06-03T20:25:27.696 INFO:teuthology.orchestra.run.smithi089.stdout: "tid": 183380 2023-06-03T20:25:27.697 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:25:27.697 INFO:teuthology.orchestra.run.smithi089.stdout: "events": [ 2023-06-03T20:25:27.697 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:25:27.697 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "initiated", 2023-06-03T20:25:27.697 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:25:26.998082+0000", 2023-06-03T20:25:27.697 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0 2023-06-03T20:25:27.697 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:25:27.697 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:25:27.698 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "throttled", 2023-06-03T20:25:27.698 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:25:26.998082+0000", 2023-06-03T20:25:27.698 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0 2023-06-03T20:25:27.698 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:25:27.698 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:25:27.698 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "header_read", 2023-06-03T20:25:27.698 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:25:26.998080+0000", 2023-06-03T20:25:27.699 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 4294967295.9999986 2023-06-03T20:25:27.699 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:25:27.699 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:25:27.699 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "all_read", 2023-06-03T20:25:27.699 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:25:26.998086+0000", 2023-06-03T20:25:27.699 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 5.9429999999999997e-06 2023-06-03T20:25:27.699 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:25:27.699 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:25:27.700 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "dispatched", 2023-06-03T20:25:27.700 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:25:26.998097+0000", 2023-06-03T20:25:27.700 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 1.0964e-05 2023-06-03T20:25:27.700 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:25:27.700 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:25:27.700 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "queued_for_pg", 2023-06-03T20:25:27.700 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:25:26.998105+0000", 2023-06-03T20:25:27.701 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 7.7680000000000008e-06 2023-06-03T20:25:27.701 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:25:27.701 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:25:27.701 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "reached_pg", 2023-06-03T20:25:27.701 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:25:26.999771+0000", 2023-06-03T20:25:27.701 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.001666252 2023-06-03T20:25:27.702 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:25:27.702 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:25:27.702 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "started", 2023-06-03T20:25:27.702 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:25:27.083907+0000", 2023-06-03T20:25:27.702 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.084135671999999995 2023-06-03T20:25:27.702 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:25:27.703 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:25:27.703 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "waiting for subops from 4,5", 2023-06-03T20:25:27.703 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:25:27.084306+0000", 2023-06-03T20:25:27.703 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.000399279 2023-06-03T20:25:27.703 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:25:27.703 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:25:27.703 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:25:27.703 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:25:27.089032+0000", 2023-06-03T20:25:27.704 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.0047259800000000003 2023-06-03T20:25:27.704 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:25:27.704 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:25:27.704 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "op_commit", 2023-06-03T20:25:27.704 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:25:27.089657+0000", 2023-06-03T20:25:27.704 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.00062500499999999998 2023-06-03T20:25:27.704 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:25:27.705 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:25:27.705 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:25:27.705 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:25:27.101951+0000", 2023-06-03T20:25:27.705 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.012293759 2023-06-03T20:25:27.705 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:25:27.705 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:25:27.705 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "commit_sent", 2023-06-03T20:25:27.706 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:25:27.101984+0000", 2023-06-03T20:25:27.706 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 3.3430999999999999e-05 2023-06-03T20:25:27.706 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:25:27.706 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:25:27.706 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "done", 2023-06-03T20:25:27.706 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:25:27.102044+0000", 2023-06-03T20:25:27.706 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 6.0229000000000002e-05 2023-06-03T20:25:27.706 INFO:teuthology.orchestra.run.smithi089.stdout: } 2023-06-03T20:25:27.707 INFO:teuthology.orchestra.run.smithi089.stdout: ] 2023-06-03T20:25:27.707 INFO:teuthology.orchestra.run.smithi089.stdout: } 2023-06-03T20:25:27.707 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:25:27.707 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:25:27.707 INFO:teuthology.orchestra.run.smithi089.stdout: "description": "osd_op(client.36573.0:183382 13.16 13:6b52695b:::benchmark_data_smithi089_211581_object67857:head [delete] snapc 0=[] ondisk+write+known_if_redirected e264)", 2023-06-03T20:25:27.707 INFO:teuthology.orchestra.run.smithi089.stdout: "initiated_at": "2023-06-03T20:25:26.998502+0000", 2023-06-03T20:25:27.707 INFO:teuthology.orchestra.run.smithi089.stdout: "age": 0.55025951200000001, 2023-06-03T20:25:27.708 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.10630216100000001, 2023-06-03T20:25:27.708 INFO:teuthology.orchestra.run.smithi089.stdout: "type_data": { 2023-06-03T20:25:27.708 INFO:teuthology.orchestra.run.smithi089.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-03T20:25:27.708 INFO:teuthology.orchestra.run.smithi089.stdout: "client_info": { 2023-06-03T20:25:27.708 INFO:teuthology.orchestra.run.smithi089.stdout: "client": "client.36573", 2023-06-03T20:25:27.708 INFO:teuthology.orchestra.run.smithi089.stdout: "client_addr": "172.21.15.89:0/616445942", 2023-06-03T20:25:27.708 INFO:teuthology.orchestra.run.smithi089.stdout: "tid": 183382 2023-06-03T20:25:27.708 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:25:27.709 INFO:teuthology.orchestra.run.smithi089.stdout: "events": [ 2023-06-03T20:25:27.709 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:25:27.709 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "initiated", 2023-06-03T20:25:27.709 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:25:26.998502+0000", 2023-06-03T20:25:27.709 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0 2023-06-03T20:25:27.709 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:25:27.709 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:25:27.710 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "throttled", 2023-06-03T20:25:27.710 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:25:26.998502+0000", 2023-06-03T20:25:27.710 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0 2023-06-03T20:25:27.710 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:25:27.710 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:25:27.710 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "header_read", 2023-06-03T20:25:27.710 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:25:26.998501+0000", 2023-06-03T20:25:27.710 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 4294967295.999999 2023-06-03T20:25:27.711 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:25:27.711 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:25:27.711 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "all_read", 2023-06-03T20:25:27.711 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:25:26.998507+0000", 2023-06-03T20:25:27.711 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 5.383e-06 2023-06-03T20:25:27.711 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:25:27.711 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:25:27.711 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "dispatched", 2023-06-03T20:25:27.712 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:25:26.998518+0000", 2023-06-03T20:25:27.712 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 1.1623999999999999e-05 2023-06-03T20:25:27.712 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:25:27.712 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:25:27.712 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "queued_for_pg", 2023-06-03T20:25:27.712 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:25:26.998526+0000", 2023-06-03T20:25:27.713 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 7.6650000000000003e-06 2023-06-03T20:25:27.713 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:25:27.713 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:25:27.713 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "reached_pg", 2023-06-03T20:25:27.713 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:25:27.086544+0000", 2023-06-03T20:25:27.713 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.088018236999999999 2023-06-03T20:25:27.713 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:25:27.714 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:25:27.714 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "started", 2023-06-03T20:25:27.714 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:25:27.086669+0000", 2023-06-03T20:25:27.714 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.00012442200000000001 2023-06-03T20:25:27.714 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:25:27.714 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:25:27.714 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "waiting for subops from 4,5", 2023-06-03T20:25:27.714 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:25:27.087023+0000", 2023-06-03T20:25:27.715 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.00035456700000000002 2023-06-03T20:25:27.715 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:25:27.715 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:25:27.715 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "op_commit", 2023-06-03T20:25:27.715 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:25:27.089662+0000", 2023-06-03T20:25:27.715 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.0026388729999999999 2023-06-03T20:25:27.715 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:25:27.715 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:25:27.716 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:25:27.716 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:25:27.102091+0000", 2023-06-03T20:25:27.716 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.012428832000000001 2023-06-03T20:25:27.716 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:25:27.716 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:25:27.716 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:25:27.716 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:25:27.104608+0000", 2023-06-03T20:25:27.716 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.002516782 2023-06-03T20:25:27.717 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:25:27.717 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:25:27.717 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "commit_sent", 2023-06-03T20:25:27.717 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:25:27.104661+0000", 2023-06-03T20:25:27.717 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 5.3083e-05 2023-06-03T20:25:27.717 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:25:27.717 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:25:27.717 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "done", 2023-06-03T20:25:27.718 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:25:27.104805+0000", 2023-06-03T20:25:27.718 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.00014366499999999999 2023-06-03T20:25:27.718 INFO:teuthology.orchestra.run.smithi089.stdout: } 2023-06-03T20:25:27.718 INFO:teuthology.orchestra.run.smithi089.stdout: ] 2023-06-03T20:25:27.718 INFO:teuthology.orchestra.run.smithi089.stdout: } 2023-06-03T20:25:27.718 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:25:27.718 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:25:27.718 INFO:teuthology.orchestra.run.smithi089.stdout: "description": "osd_op(client.36573.0:183383 13.2 13:444194ce:::benchmark_data_smithi089_211581_object67858:head [delete] snapc 0=[] ondisk+write+known_if_redirected e264)", 2023-06-03T20:25:27.719 INFO:teuthology.orchestra.run.smithi089.stdout: "initiated_at": "2023-06-03T20:25:26.999168+0000", 2023-06-03T20:25:27.719 INFO:teuthology.orchestra.run.smithi089.stdout: "age": 0.54959364700000002, 2023-06-03T20:25:27.719 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.092544517000000007, 2023-06-03T20:25:27.719 INFO:teuthology.orchestra.run.smithi089.stdout: "type_data": { 2023-06-03T20:25:27.719 INFO:teuthology.orchestra.run.smithi089.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-03T20:25:27.719 INFO:teuthology.orchestra.run.smithi089.stdout: "client_info": { 2023-06-03T20:25:27.719 INFO:teuthology.orchestra.run.smithi089.stdout: "client": "client.36573", 2023-06-03T20:25:27.720 INFO:teuthology.orchestra.run.smithi089.stdout: "client_addr": "172.21.15.89:0/616445942", 2023-06-03T20:25:27.720 INFO:teuthology.orchestra.run.smithi089.stdout: "tid": 183383 2023-06-03T20:25:27.720 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:25:27.720 INFO:teuthology.orchestra.run.smithi089.stdout: "events": [ 2023-06-03T20:25:27.720 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:25:27.720 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "initiated", 2023-06-03T20:25:27.720 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:25:26.999168+0000", 2023-06-03T20:25:27.720 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0 2023-06-03T20:25:27.721 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:25:27.721 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:25:27.721 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "throttled", 2023-06-03T20:25:27.721 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:25:26.999168+0000", 2023-06-03T20:25:27.721 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0 2023-06-03T20:25:27.721 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:25:27.721 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:25:27.721 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "header_read", 2023-06-03T20:25:27.722 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:25:26.999167+0000", 2023-06-03T20:25:27.722 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 4294967295.999999 2023-06-03T20:25:27.722 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:25:27.722 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:25:27.723 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "all_read", 2023-06-03T20:25:27.723 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:25:26.999173+0000", 2023-06-03T20:25:27.723 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 5.2900000000000002e-06 2023-06-03T20:25:27.723 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:25:27.723 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:25:27.723 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "dispatched", 2023-06-03T20:25:27.723 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:25:26.999184+0000", 2023-06-03T20:25:27.723 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 1.1066999999999999e-05 2023-06-03T20:25:27.724 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:25:27.724 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:25:27.724 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "queued_for_pg", 2023-06-03T20:25:27.724 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:25:26.999191+0000", 2023-06-03T20:25:27.724 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 7.2400000000000001e-06 2023-06-03T20:25:27.724 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:25:27.724 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:25:27.724 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "reached_pg", 2023-06-03T20:25:27.725 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:25:26.999240+0000", 2023-06-03T20:25:27.725 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 4.8581000000000001e-05 2023-06-03T20:25:27.725 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:25:27.725 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:25:27.725 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "started", 2023-06-03T20:25:27.725 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:25:26.999365+0000", 2023-06-03T20:25:27.725 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.000124919 2023-06-03T20:25:27.726 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:25:27.726 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:25:27.726 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "waiting for subops from 4,5", 2023-06-03T20:25:27.726 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:25:26.999789+0000", 2023-06-03T20:25:27.726 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.00042429799999999998 2023-06-03T20:25:27.726 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:25:27.726 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:25:27.726 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:25:27.727 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:25:27.086443+0000", 2023-06-03T20:25:27.727 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.086654495999999998 2023-06-03T20:25:27.727 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:25:27.727 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:25:27.727 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:25:27.727 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:25:27.088927+0000", 2023-06-03T20:25:27.727 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.002484067 2023-06-03T20:25:27.727 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:25:27.728 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:25:27.728 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "op_commit", 2023-06-03T20:25:27.728 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:25:27.091475+0000", 2023-06-03T20:25:27.728 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.002547821 2023-06-03T20:25:27.728 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:25:27.728 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:25:27.728 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "commit_sent", 2023-06-03T20:25:27.728 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:25:27.091526+0000", 2023-06-03T20:25:27.729 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 5.1143000000000001e-05 2023-06-03T20:25:27.729 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:25:27.729 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:25:27.729 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "done", 2023-06-03T20:25:27.729 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:25:27.091713+0000", 2023-06-03T20:25:27.729 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.00018642499999999999 2023-06-03T20:25:27.729 INFO:teuthology.orchestra.run.smithi089.stdout: } 2023-06-03T20:25:27.729 INFO:teuthology.orchestra.run.smithi089.stdout: ] 2023-06-03T20:25:27.730 INFO:teuthology.orchestra.run.smithi089.stdout: } 2023-06-03T20:25:27.730 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:25:27.730 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:25:27.730 INFO:teuthology.orchestra.run.smithi089.stdout: "description": "osd_repop_reply(client.36573.0:183370 13.b e264/242 ondisk, result = 0)", 2023-06-03T20:25:27.730 INFO:teuthology.orchestra.run.smithi089.stdout: "initiated_at": "2023-06-03T20:25:27.001617+0000", 2023-06-03T20:25:27.730 INFO:teuthology.orchestra.run.smithi089.stdout: "age": 0.54714516099999999, 2023-06-03T20:25:27.730 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.082296795000000006, 2023-06-03T20:25:27.731 INFO:teuthology.orchestra.run.smithi089.stdout: "type_data": { 2023-06-03T20:25:27.731 INFO:teuthology.orchestra.run.smithi089.stdout: "flag_point": "started", 2023-06-03T20:25:27.731 INFO:teuthology.orchestra.run.smithi089.stdout: "events": [ 2023-06-03T20:25:27.731 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:25:27.731 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "initiated", 2023-06-03T20:25:27.731 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:25:27.001617+0000", 2023-06-03T20:25:27.731 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0 2023-06-03T20:25:27.731 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:25:27.732 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:25:27.732 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "throttled", 2023-06-03T20:25:27.732 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:25:27.001617+0000", 2023-06-03T20:25:27.732 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0 2023-06-03T20:25:27.732 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:25:27.732 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:25:27.732 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "header_read", 2023-06-03T20:25:27.732 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:25:27.001616+0000", 2023-06-03T20:25:27.733 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 4294967295.999999 2023-06-03T20:25:27.733 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:25:27.733 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:25:27.733 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "all_read", 2023-06-03T20:25:27.733 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:25:27.001623+0000", 2023-06-03T20:25:27.733 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 7.1629999999999999e-06 2023-06-03T20:25:27.733 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:25:27.734 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:25:27.734 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "dispatched", 2023-06-03T20:25:27.734 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:25:27.083841+0000", 2023-06-03T20:25:27.734 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.082217944000000001 2023-06-03T20:25:27.734 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:25:27.734 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:25:27.734 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "queued_for_pg", 2023-06-03T20:25:27.734 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:25:27.083855+0000", 2023-06-03T20:25:27.735 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 1.4297000000000001e-05 2023-06-03T20:25:27.735 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:25:27.735 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:25:27.735 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "reached_pg", 2023-06-03T20:25:27.735 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:25:27.083895+0000", 2023-06-03T20:25:27.735 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 4.0037000000000003e-05 2023-06-03T20:25:27.735 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:25:27.735 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:25:27.736 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "started", 2023-06-03T20:25:27.736 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:25:27.083903+0000", 2023-06-03T20:25:27.736 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 7.7770000000000002e-06 2023-06-03T20:25:27.736 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:25:27.736 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:25:27.736 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "done", 2023-06-03T20:25:27.736 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:25:27.083914+0000", 2023-06-03T20:25:27.736 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 1.0451e-05 2023-06-03T20:25:27.737 INFO:teuthology.orchestra.run.smithi089.stdout: } 2023-06-03T20:25:27.737 INFO:teuthology.orchestra.run.smithi089.stdout: ] 2023-06-03T20:25:27.737 INFO:teuthology.orchestra.run.smithi089.stdout: } 2023-06-03T20:25:27.737 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:25:27.737 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:25:27.737 INFO:teuthology.orchestra.run.smithi089.stdout: "description": "osd_repop_reply(client.36573.0:183373 13.16 e264/262 ondisk, result = 0)", 2023-06-03T20:25:27.737 INFO:teuthology.orchestra.run.smithi089.stdout: "initiated_at": "2023-06-03T20:25:27.001954+0000", 2023-06-03T20:25:27.737 INFO:teuthology.orchestra.run.smithi089.stdout: "age": 0.54680811799999995, 2023-06-03T20:25:27.738 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.084469677000000007, 2023-06-03T20:25:27.738 INFO:teuthology.orchestra.run.smithi089.stdout: "type_data": { 2023-06-03T20:25:27.738 INFO:teuthology.orchestra.run.smithi089.stdout: "flag_point": "started", 2023-06-03T20:25:27.738 INFO:teuthology.orchestra.run.smithi089.stdout: "events": [ 2023-06-03T20:25:27.738 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:25:27.738 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "initiated", 2023-06-03T20:25:27.738 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:25:27.001954+0000", 2023-06-03T20:25:27.739 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0 2023-06-03T20:25:27.739 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:25:27.739 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:25:27.739 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "throttled", 2023-06-03T20:25:27.739 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:25:27.001954+0000", 2023-06-03T20:25:27.739 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0 2023-06-03T20:25:27.740 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:25:27.740 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:25:27.740 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "header_read", 2023-06-03T20:25:27.740 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:25:27.001953+0000", 2023-06-03T20:25:27.740 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 4294967295.999999 2023-06-03T20:25:27.740 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:25:27.740 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:25:27.740 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "all_read", 2023-06-03T20:25:27.741 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:25:27.001961+0000", 2023-06-03T20:25:27.741 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 8.4959999999999996e-06 2023-06-03T20:25:27.741 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:25:27.741 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:25:27.741 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "dispatched", 2023-06-03T20:25:27.741 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:25:27.083984+0000", 2023-06-03T20:25:27.741 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.082022209999999998 2023-06-03T20:25:27.742 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:25:27.742 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:25:27.742 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "queued_for_pg", 2023-06-03T20:25:27.742 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:25:27.083993+0000", 2023-06-03T20:25:27.742 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 9.0070000000000003e-06 2023-06-03T20:25:27.742 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:25:27.742 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:25:27.742 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "reached_pg", 2023-06-03T20:25:27.743 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:25:27.086403+0000", 2023-06-03T20:25:27.743 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.002410073 2023-06-03T20:25:27.743 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:25:27.743 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:25:27.743 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "started", 2023-06-03T20:25:27.743 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:25:27.086413+0000", 2023-06-03T20:25:27.743 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 1.0468e-05 2023-06-03T20:25:27.743 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:25:27.744 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:25:27.744 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "done", 2023-06-03T20:25:27.744 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:25:27.086423+0000", 2023-06-03T20:25:27.744 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 1.0217000000000001e-05 2023-06-03T20:25:27.744 INFO:teuthology.orchestra.run.smithi089.stdout: } 2023-06-03T20:25:27.744 INFO:teuthology.orchestra.run.smithi089.stdout: ] 2023-06-03T20:25:27.744 INFO:teuthology.orchestra.run.smithi089.stdout: } 2023-06-03T20:25:27.744 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:25:27.745 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:25:27.745 INFO:teuthology.orchestra.run.smithi089.stdout: "description": "osd_op(client.36573.0:183436 13.b 13:d502a0c7:::benchmark_data_smithi089_211581_object67911:head [delete] snapc 0=[] ondisk+write+known_if_redirected e264)", 2023-06-03T20:25:27.745 INFO:teuthology.orchestra.run.smithi089.stdout: "initiated_at": "2023-06-03T20:25:27.110361+0000", 2023-06-03T20:25:27.745 INFO:teuthology.orchestra.run.smithi089.stdout: "age": 0.43840127200000001, 2023-06-03T20:25:27.745 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.093919170999999996, 2023-06-03T20:25:27.745 INFO:teuthology.orchestra.run.smithi089.stdout: "type_data": { 2023-06-03T20:25:27.745 INFO:teuthology.orchestra.run.smithi089.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-03T20:25:27.745 INFO:teuthology.orchestra.run.smithi089.stdout: "client_info": { 2023-06-03T20:25:27.746 INFO:teuthology.orchestra.run.smithi089.stdout: "client": "client.36573", 2023-06-03T20:25:27.746 INFO:teuthology.orchestra.run.smithi089.stdout: "client_addr": "172.21.15.89:0/616445942", 2023-06-03T20:25:27.746 INFO:teuthology.orchestra.run.smithi089.stdout: "tid": 183436 2023-06-03T20:25:27.746 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:25:27.746 INFO:teuthology.orchestra.run.smithi089.stdout: "events": [ 2023-06-03T20:25:27.746 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:25:27.746 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "initiated", 2023-06-03T20:25:27.747 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:25:27.110361+0000", 2023-06-03T20:25:27.747 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0 2023-06-03T20:25:27.747 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:25:27.747 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:25:27.747 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "throttled", 2023-06-03T20:25:27.747 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:25:27.110361+0000", 2023-06-03T20:25:27.747 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0 2023-06-03T20:25:27.747 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:25:27.748 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:25:27.748 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "header_read", 2023-06-03T20:25:27.748 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:25:27.110360+0000", 2023-06-03T20:25:27.748 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 4294967295.999999 2023-06-03T20:25:27.748 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:25:27.748 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:25:27.748 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "all_read", 2023-06-03T20:25:27.748 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:25:27.110365+0000", 2023-06-03T20:25:27.749 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 5.2390000000000002e-06 2023-06-03T20:25:27.749 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:25:27.749 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:25:27.749 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "dispatched", 2023-06-03T20:25:27.749 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:25:27.110372+0000", 2023-06-03T20:25:27.749 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 7.3640000000000001e-06 2023-06-03T20:25:27.749 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:25:27.749 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:25:27.750 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "queued_for_pg", 2023-06-03T20:25:27.750 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:25:27.110377+0000", 2023-06-03T20:25:27.750 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 5.0930000000000002e-06 2023-06-03T20:25:27.750 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:25:27.750 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:25:27.750 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "reached_pg", 2023-06-03T20:25:27.750 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:25:27.111434+0000", 2023-06-03T20:25:27.751 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.0010565329999999999 2023-06-03T20:25:27.751 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:25:27.751 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:25:27.751 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "started", 2023-06-03T20:25:27.751 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:25:27.111544+0000", 2023-06-03T20:25:27.751 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.000110073 2023-06-03T20:25:27.751 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:25:27.751 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:25:27.752 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "waiting for subops from 2,5", 2023-06-03T20:25:27.752 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:25:27.111658+0000", 2023-06-03T20:25:27.752 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.00011448200000000001 2023-06-03T20:25:27.752 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:25:27.752 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:25:27.752 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:25:27.752 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:25:27.115533+0000", 2023-06-03T20:25:27.752 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.0038750989999999999 2023-06-03T20:25:27.753 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:25:27.753 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:25:27.753 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "op_commit", 2023-06-03T20:25:27.753 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:25:27.115623+0000", 2023-06-03T20:25:27.753 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 8.9939999999999996e-05 2023-06-03T20:25:27.753 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:25:27.753 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:25:27.754 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:25:27.754 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:25:27.204171+0000", 2023-06-03T20:25:27.754 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.088547550000000003 2023-06-03T20:25:27.754 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:25:27.754 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:25:27.754 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "commit_sent", 2023-06-03T20:25:27.754 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:25:27.204224+0000", 2023-06-03T20:25:27.754 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 5.2850999999999999e-05 2023-06-03T20:25:27.755 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:25:27.755 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:25:27.755 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "done", 2023-06-03T20:25:27.755 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:25:27.204280+0000", 2023-06-03T20:25:27.755 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 5.5964000000000002e-05 2023-06-03T20:25:27.755 INFO:teuthology.orchestra.run.smithi089.stdout: } 2023-06-03T20:25:27.755 INFO:teuthology.orchestra.run.smithi089.stdout: ] 2023-06-03T20:25:27.755 INFO:teuthology.orchestra.run.smithi089.stdout: } 2023-06-03T20:25:27.756 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:25:27.756 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:25:27.756 INFO:teuthology.orchestra.run.smithi089.stdout: "description": "osd_op(client.36573.0:183437 13.b 13:dbdf3b14:::benchmark_data_smithi089_211581_object67912:head [delete] snapc 0=[] ondisk+write+known_if_redirected e264)", 2023-06-03T20:25:27.756 INFO:teuthology.orchestra.run.smithi089.stdout: "initiated_at": "2023-06-03T20:25:27.112056+0000", 2023-06-03T20:25:27.756 INFO:teuthology.orchestra.run.smithi089.stdout: "age": 0.43670566199999999, 2023-06-03T20:25:27.756 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.10300887, 2023-06-03T20:25:27.756 INFO:teuthology.orchestra.run.smithi089.stdout: "type_data": { 2023-06-03T20:25:27.757 INFO:teuthology.orchestra.run.smithi089.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-03T20:25:27.757 INFO:teuthology.orchestra.run.smithi089.stdout: "client_info": { 2023-06-03T20:25:27.757 INFO:teuthology.orchestra.run.smithi089.stdout: "client": "client.36573", 2023-06-03T20:25:27.757 INFO:teuthology.orchestra.run.smithi089.stdout: "client_addr": "172.21.15.89:0/616445942", 2023-06-03T20:25:27.757 INFO:teuthology.orchestra.run.smithi089.stdout: "tid": 183437 2023-06-03T20:25:27.757 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:25:27.757 INFO:teuthology.orchestra.run.smithi089.stdout: "events": [ 2023-06-03T20:25:27.757 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:25:27.758 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "initiated", 2023-06-03T20:25:27.758 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:25:27.112056+0000", 2023-06-03T20:25:27.758 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0 2023-06-03T20:25:27.758 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:25:27.758 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:25:27.758 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "throttled", 2023-06-03T20:25:27.758 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:25:27.112056+0000", 2023-06-03T20:25:27.758 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0 2023-06-03T20:25:27.759 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:25:27.759 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:25:27.759 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "header_read", 2023-06-03T20:25:27.759 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:25:27.112055+0000", 2023-06-03T20:25:27.759 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 4294967295.9999986 2023-06-03T20:25:27.759 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:25:27.759 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:25:27.759 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "all_read", 2023-06-03T20:25:27.760 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:25:27.112062+0000", 2023-06-03T20:25:27.760 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 7.4540000000000001e-06 2023-06-03T20:25:27.760 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:25:27.760 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:25:27.760 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "dispatched", 2023-06-03T20:25:27.760 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:25:27.112077+0000", 2023-06-03T20:25:27.760 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 1.4904e-05 2023-06-03T20:25:27.760 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:25:27.761 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:25:27.761 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "queued_for_pg", 2023-06-03T20:25:27.761 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:25:27.112087+0000", 2023-06-03T20:25:27.761 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 9.9760000000000007e-06 2023-06-03T20:25:27.761 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:25:27.761 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:25:27.761 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "reached_pg", 2023-06-03T20:25:27.762 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:25:27.199875+0000", 2023-06-03T20:25:27.762 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.087787666 2023-06-03T20:25:27.762 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:25:27.762 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:25:27.762 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "started", 2023-06-03T20:25:27.762 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:25:27.200002+0000", 2023-06-03T20:25:27.762 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.000127222 2023-06-03T20:25:27.762 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:25:27.763 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:25:27.763 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "waiting for subops from 2,5", 2023-06-03T20:25:27.763 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:25:27.200130+0000", 2023-06-03T20:25:27.763 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.00012812000000000001 2023-06-03T20:25:27.763 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:25:27.763 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:25:27.763 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "op_commit", 2023-06-03T20:25:27.763 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:25:27.204297+0000", 2023-06-03T20:25:27.764 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.0041663669999999998 2023-06-03T20:25:27.764 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:25:27.764 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:25:27.764 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:25:27.764 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:25:27.206478+0000", 2023-06-03T20:25:27.764 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.002181896 2023-06-03T20:25:27.764 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:25:27.764 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:25:27.765 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:25:27.765 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:25:27.214989+0000", 2023-06-03T20:25:27.765 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.0085102649999999995 2023-06-03T20:25:27.765 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:25:27.765 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:25:27.765 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "commit_sent", 2023-06-03T20:25:27.765 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:25:27.215030+0000", 2023-06-03T20:25:27.765 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 4.1547e-05 2023-06-03T20:25:27.766 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:25:27.766 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:25:27.766 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "done", 2023-06-03T20:25:27.766 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:25:27.215065+0000", 2023-06-03T20:25:27.766 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 3.4892999999999999e-05 2023-06-03T20:25:27.766 INFO:teuthology.orchestra.run.smithi089.stdout: } 2023-06-03T20:25:27.766 INFO:teuthology.orchestra.run.smithi089.stdout: ] 2023-06-03T20:25:27.767 INFO:teuthology.orchestra.run.smithi089.stdout: } 2023-06-03T20:25:27.767 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:25:27.767 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:25:27.767 INFO:teuthology.orchestra.run.smithi089.stdout: "description": "osd_op(client.36573.0:183439 13.6 13:63649f66:::benchmark_data_smithi089_211581_object67914:head [delete] snapc 0=[] ondisk+write+known_if_redirected e264)", 2023-06-03T20:25:27.767 INFO:teuthology.orchestra.run.smithi089.stdout: "initiated_at": "2023-06-03T20:25:27.115272+0000", 2023-06-03T20:25:27.767 INFO:teuthology.orchestra.run.smithi089.stdout: "age": 0.43348941499999999, 2023-06-03T20:25:27.767 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.090628708000000002, 2023-06-03T20:25:27.767 INFO:teuthology.orchestra.run.smithi089.stdout: "type_data": { 2023-06-03T20:25:27.768 INFO:teuthology.orchestra.run.smithi089.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-03T20:25:27.768 INFO:teuthology.orchestra.run.smithi089.stdout: "client_info": { 2023-06-03T20:25:27.768 INFO:teuthology.orchestra.run.smithi089.stdout: "client": "client.36573", 2023-06-03T20:25:27.768 INFO:teuthology.orchestra.run.smithi089.stdout: "client_addr": "172.21.15.89:0/616445942", 2023-06-03T20:25:27.768 INFO:teuthology.orchestra.run.smithi089.stdout: "tid": 183439 2023-06-03T20:25:27.768 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:25:27.768 INFO:teuthology.orchestra.run.smithi089.stdout: "events": [ 2023-06-03T20:25:27.768 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:25:27.769 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "initiated", 2023-06-03T20:25:27.769 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:25:27.115272+0000", 2023-06-03T20:25:27.769 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0 2023-06-03T20:25:27.769 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:25:27.769 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:25:27.769 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "throttled", 2023-06-03T20:25:27.769 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:25:27.115272+0000", 2023-06-03T20:25:27.769 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0 2023-06-03T20:25:27.770 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:25:27.770 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:25:27.770 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "header_read", 2023-06-03T20:25:27.770 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:25:27.115271+0000", 2023-06-03T20:25:27.770 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 4294967295.9999986 2023-06-03T20:25:27.770 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:25:27.771 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:25:27.771 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "all_read", 2023-06-03T20:25:27.771 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:25:27.115277+0000", 2023-06-03T20:25:27.771 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 5.4970000000000001e-06 2023-06-03T20:25:27.771 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:25:27.771 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:25:27.771 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "dispatched", 2023-06-03T20:25:27.772 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:25:27.115285+0000", 2023-06-03T20:25:27.772 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 8.2560000000000002e-06 2023-06-03T20:25:27.772 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:25:27.772 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:25:27.772 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "queued_for_pg", 2023-06-03T20:25:27.772 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:25:27.115290+0000", 2023-06-03T20:25:27.772 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 5.0919999999999997e-06 2023-06-03T20:25:27.773 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:25:27.773 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:25:27.773 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "reached_pg", 2023-06-03T20:25:27.773 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:25:27.115346+0000", 2023-06-03T20:25:27.773 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 5.5986999999999998e-05 2023-06-03T20:25:27.773 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:25:27.773 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:25:27.773 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "started", 2023-06-03T20:25:27.774 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:25:27.115488+0000", 2023-06-03T20:25:27.774 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.00014199699999999999 2023-06-03T20:25:27.774 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:25:27.774 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:25:27.774 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "waiting for subops from 4,5", 2023-06-03T20:25:27.774 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:25:27.199483+0000", 2023-06-03T20:25:27.774 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.083995285000000003 2023-06-03T20:25:27.774 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:25:27.775 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:25:27.775 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "op_commit", 2023-06-03T20:25:27.775 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:25:27.201936+0000", 2023-06-03T20:25:27.775 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.0024529790000000001 2023-06-03T20:25:27.775 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:25:27.775 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:25:27.775 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:25:27.775 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:25:27.205280+0000", 2023-06-03T20:25:27.776 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.0033434900000000002 2023-06-03T20:25:27.776 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:25:27.776 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:25:27.776 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:25:27.776 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:25:27.205739+0000", 2023-06-03T20:25:27.776 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.00045883499999999999 2023-06-03T20:25:27.776 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:25:27.776 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:25:27.777 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "commit_sent", 2023-06-03T20:25:27.777 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:25:27.205798+0000", 2023-06-03T20:25:27.777 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 5.9508e-05 2023-06-03T20:25:27.777 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:25:27.777 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:25:27.777 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "done", 2023-06-03T20:25:27.777 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:25:27.205901+0000", 2023-06-03T20:25:27.778 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.000103082 2023-06-03T20:25:27.778 INFO:teuthology.orchestra.run.smithi089.stdout: } 2023-06-03T20:25:27.778 INFO:teuthology.orchestra.run.smithi089.stdout: ] 2023-06-03T20:25:27.778 INFO:teuthology.orchestra.run.smithi089.stdout: } 2023-06-03T20:25:27.778 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:25:27.778 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:25:27.778 INFO:teuthology.orchestra.run.smithi089.stdout: "description": "osd_repop_reply(client.36573.0:183436 13.b e264/242 ondisk, result = 0)", 2023-06-03T20:25:27.778 INFO:teuthology.orchestra.run.smithi089.stdout: "initiated_at": "2023-06-03T20:25:27.116768+0000", 2023-06-03T20:25:27.779 INFO:teuthology.orchestra.run.smithi089.stdout: "age": 0.43199359599999998, 2023-06-03T20:25:27.779 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.087518433000000007, 2023-06-03T20:25:27.779 INFO:teuthology.orchestra.run.smithi089.stdout: "type_data": { 2023-06-03T20:25:27.779 INFO:teuthology.orchestra.run.smithi089.stdout: "flag_point": "started", 2023-06-03T20:25:27.779 INFO:teuthology.orchestra.run.smithi089.stdout: "events": [ 2023-06-03T20:25:27.779 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:25:27.779 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "initiated", 2023-06-03T20:25:27.779 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:25:27.116768+0000", 2023-06-03T20:25:27.780 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0 2023-06-03T20:25:27.780 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:25:27.780 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:25:27.780 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "throttled", 2023-06-03T20:25:27.780 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:25:27.116768+0000", 2023-06-03T20:25:27.780 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0 2023-06-03T20:25:27.780 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:25:27.780 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:25:27.781 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "header_read", 2023-06-03T20:25:27.781 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:25:27.116768+0000", 2023-06-03T20:25:27.781 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 4294967295.999999 2023-06-03T20:25:27.781 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:25:27.781 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:25:27.781 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "all_read", 2023-06-03T20:25:27.781 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:25:27.116776+0000", 2023-06-03T20:25:27.782 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 8.4589999999999998e-06 2023-06-03T20:25:27.782 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:25:27.782 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:25:27.782 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "dispatched", 2023-06-03T20:25:27.782 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:25:27.197565+0000", 2023-06-03T20:25:27.782 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.080788544000000004 2023-06-03T20:25:27.782 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:25:27.782 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:25:27.783 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "queued_for_pg", 2023-06-03T20:25:27.783 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:25:27.197598+0000", 2023-06-03T20:25:27.783 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 3.3624000000000002e-05 2023-06-03T20:25:27.783 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:25:27.783 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:25:27.783 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "reached_pg", 2023-06-03T20:25:27.783 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:25:27.204152+0000", 2023-06-03T20:25:27.783 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.0065535560000000003 2023-06-03T20:25:27.784 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:25:27.784 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:25:27.784 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "started", 2023-06-03T20:25:27.784 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:25:27.204161+0000", 2023-06-03T20:25:27.784 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 9.6330000000000008e-06 2023-06-03T20:25:27.784 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:25:27.784 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:25:27.784 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "done", 2023-06-03T20:25:27.785 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:25:27.204287+0000", 2023-06-03T20:25:27.785 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.00012538299999999999 2023-06-03T20:25:27.785 INFO:teuthology.orchestra.run.smithi089.stdout: } 2023-06-03T20:25:27.785 INFO:teuthology.orchestra.run.smithi089.stdout: ] 2023-06-03T20:25:27.785 INFO:teuthology.orchestra.run.smithi089.stdout: } 2023-06-03T20:25:27.785 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:25:27.785 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:25:27.785 INFO:teuthology.orchestra.run.smithi089.stdout: "description": "osd_op(client.36573.0:183499 13.b 13:dc549a92:::benchmark_data_smithi089_211581_object67974:head [delete] snapc 0=[] ondisk+write+known_if_redirected e264)", 2023-06-03T20:25:27.786 INFO:teuthology.orchestra.run.smithi089.stdout: "initiated_at": "2023-06-03T20:25:27.271246+0000", 2023-06-03T20:25:27.786 INFO:teuthology.orchestra.run.smithi089.stdout: "age": 0.27751570199999998, 2023-06-03T20:25:27.786 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.090646674999999996, 2023-06-03T20:25:27.786 INFO:teuthology.orchestra.run.smithi089.stdout: "type_data": { 2023-06-03T20:25:27.786 INFO:teuthology.orchestra.run.smithi089.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-03T20:25:27.786 INFO:teuthology.orchestra.run.smithi089.stdout: "client_info": { 2023-06-03T20:25:27.786 INFO:teuthology.orchestra.run.smithi089.stdout: "client": "client.36573", 2023-06-03T20:25:27.787 INFO:teuthology.orchestra.run.smithi089.stdout: "client_addr": "172.21.15.89:0/616445942", 2023-06-03T20:25:27.787 INFO:teuthology.orchestra.run.smithi089.stdout: "tid": 183499 2023-06-03T20:25:27.787 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:25:27.787 INFO:teuthology.orchestra.run.smithi089.stdout: "events": [ 2023-06-03T20:25:27.787 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:25:27.787 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "initiated", 2023-06-03T20:25:27.787 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:25:27.271246+0000", 2023-06-03T20:25:27.787 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0 2023-06-03T20:25:27.788 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:25:27.788 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:25:27.788 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "throttled", 2023-06-03T20:25:27.788 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:25:27.271246+0000", 2023-06-03T20:25:27.788 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0 2023-06-03T20:25:27.788 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:25:27.788 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:25:27.788 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "header_read", 2023-06-03T20:25:27.789 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:25:27.271245+0000", 2023-06-03T20:25:27.789 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 4294967295.999999 2023-06-03T20:25:27.789 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:25:27.789 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:25:27.789 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "all_read", 2023-06-03T20:25:27.789 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:25:27.271251+0000", 2023-06-03T20:25:27.789 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 5.463e-06 2023-06-03T20:25:27.789 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:25:27.790 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:25:27.790 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "dispatched", 2023-06-03T20:25:27.790 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:25:27.271262+0000", 2023-06-03T20:25:27.790 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 1.1289e-05 2023-06-03T20:25:27.790 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:25:27.790 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:25:27.790 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "queued_for_pg", 2023-06-03T20:25:27.790 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:25:27.271269+0000", 2023-06-03T20:25:27.791 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 7.4279999999999999e-06 2023-06-03T20:25:27.791 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:25:27.791 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:25:27.791 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "reached_pg", 2023-06-03T20:25:27.791 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:25:27.271311+0000", 2023-06-03T20:25:27.791 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 4.1891000000000003e-05 2023-06-03T20:25:27.791 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:25:27.791 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:25:27.792 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "started", 2023-06-03T20:25:27.792 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:25:27.271382+0000", 2023-06-03T20:25:27.792 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 7.0888000000000003e-05 2023-06-03T20:25:27.792 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:25:27.792 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:25:27.792 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "waiting for subops from 2,5", 2023-06-03T20:25:27.792 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:25:27.271447+0000", 2023-06-03T20:25:27.792 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 6.4684000000000006e-05 2023-06-03T20:25:27.793 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:25:27.793 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:25:27.793 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "op_commit", 2023-06-03T20:25:27.793 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:25:27.274513+0000", 2023-06-03T20:25:27.793 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.003066184 2023-06-03T20:25:27.793 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:25:27.793 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:25:27.793 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:25:27.794 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:25:27.275905+0000", 2023-06-03T20:25:27.794 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.0013919889999999999 2023-06-03T20:25:27.794 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:25:27.794 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:25:27.794 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:25:27.794 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:25:27.361802+0000", 2023-06-03T20:25:27.794 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.085897343000000001 2023-06-03T20:25:27.795 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:25:27.795 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:25:27.795 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "commit_sent", 2023-06-03T20:25:27.795 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:25:27.361853+0000", 2023-06-03T20:25:27.795 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 5.0810000000000003e-05 2023-06-03T20:25:27.795 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:25:27.795 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:25:27.795 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "done", 2023-06-03T20:25:27.796 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:25:27.361893+0000", 2023-06-03T20:25:27.796 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 3.9755999999999997e-05 2023-06-03T20:25:27.796 INFO:teuthology.orchestra.run.smithi089.stdout: } 2023-06-03T20:25:27.796 INFO:teuthology.orchestra.run.smithi089.stdout: ] 2023-06-03T20:25:27.796 INFO:teuthology.orchestra.run.smithi089.stdout: } 2023-06-03T20:25:27.796 INFO:teuthology.orchestra.run.smithi089.stdout: } 2023-06-03T20:25:27.796 INFO:teuthology.orchestra.run.smithi089.stdout: ] 2023-06-03T20:25:27.796 INFO:teuthology.orchestra.run.smithi089.stdout:} 2023-06-03T20:25:27.797 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:25:27.798 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:25:27.798 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:25:27.899 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:25:27.934 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:25:27.939 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:25:27.940 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:25:28.041 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:25:28.067 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:25:28.071 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:25:28.073 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:25:28.175 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:25:28.179 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:25:28 smithi089 bash[78452]: cluster 1685823927.0644586 mgr.y (mgr.34104) 1487 : cluster 0 pgmap v1517: 99 pgs: 6 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait, 7 active+recovery_wait+undersized+degraded+remapped, 6 active+undersized+degraded+remapped+backfill_wait, 76 active+clean; 3.0 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 623 op/s; 43429/147393 objects degraded (29.465%); 5980/147393 objects misplaced (4.057%); 7.3 MiB/s, 117 objects/s recovering 2023-06-03T20:25:28.179 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:25:28 smithi089 bash[78452]: cluster 1685823927.963209 mon.a (mon.0) 1479 : cluster 3 Health check update: Degraded data redundancy: 43429/147393 objects degraded (29.465%), 20 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-06-03T20:25:28.180 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:25:28 smithi089 bash[75068]: cluster 1685823927.0644586 mgr.y (mgr.34104) 1487 : cluster 0 pgmap v1517: 99 pgs: 6 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait, 7 active+recovery_wait+undersized+degraded+remapped, 6 active+undersized+degraded+remapped+backfill_wait, 76 active+clean; 3.0 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 623 op/s; 43429/147393 objects degraded (29.465%); 5980/147393 objects misplaced (4.057%); 7.3 MiB/s, 117 objects/s recovering 2023-06-03T20:25:28.180 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:25:28 smithi089 bash[75068]: cluster 1685823927.963209 mon.a (mon.0) 1479 : cluster 3 Health check update: Degraded data redundancy: 43429/147393 objects degraded (29.465%), 20 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-06-03T20:25:28.214 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:25:28.216 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:25:28.216 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:25:28.317 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:25:28.348 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:25:28.357 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:25:28.358 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:25:28.459 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:25:28.468 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:25:28 smithi164 bash[59462]: cluster 1685823927.0644586 mgr.y (mgr.34104) 1487 : cluster 0 pgmap v1517: 99 pgs: 6 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait, 7 active+recovery_wait+undersized+degraded+remapped, 6 active+undersized+degraded+remapped+backfill_wait, 76 active+clean; 3.0 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 623 op/s; 43429/147393 objects degraded (29.465%); 5980/147393 objects misplaced (4.057%); 7.3 MiB/s, 117 objects/s recovering 2023-06-03T20:25:28.468 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:25:28 smithi164 bash[59462]: cluster 1685823927.963209 mon.a (mon.0) 1479 : cluster 3 Health check update: Degraded data redundancy: 43429/147393 objects degraded (29.465%), 20 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-06-03T20:25:28.489 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:25:28.492 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:25:28.493 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:25:28.594 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:25:28.628 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:25:28.632 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:25:28.633 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:25:28.734 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:25:28.793 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:25:28.797 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:25:28.798 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:25:28.899 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:25:28.929 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:25:28.931 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:25:28.932 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:25:29.033 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:25:29.061 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:25:29.063 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:25:29.064 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:25:29.165 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:25:29.193 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:25:29.195 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:25:29.195 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:25:29.296 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:25:29.333 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:25:29.334 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:25:29.335 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:25:29.436 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:25:29.466 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:25:29.469 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:25:29.470 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:25:29.571 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:25:29.639 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:25:29.644 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:25:29.645 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:25:29.746 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:25:29.773 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:25:29.778 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:25:29.779 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:25:29.880 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:25:29.916 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:25:29.921 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:25:29.922 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:25:30.023 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:25:30.054 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:25:30.056 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:25:30.057 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:25:30.158 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:25:30.215 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:25:30.218 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:25:30.219 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:25:30.320 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:25:30.328 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:25:30 smithi089 bash[78452]: cluster 1685823929.0653532 mgr.y (mgr.34104) 1488 : cluster 0 pgmap v1518: 99 pgs: 6 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 2 active+recovery_wait, 7 active+recovery_wait+undersized+degraded+remapped, 6 active+undersized+degraded+remapped+backfill_wait, 76 active+clean; 2.9 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 645 op/s; 42660/143355 objects degraded (29.758%); 5805/143355 objects misplaced (4.049%); 5.7 MiB/s, 90 objects/s recovering 2023-06-03T20:25:30.329 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:25:30 smithi089 bash[75068]: cluster 1685823929.0653532 mgr.y (mgr.34104) 1488 : cluster 0 pgmap v1518: 99 pgs: 6 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 2 active+recovery_wait, 7 active+recovery_wait+undersized+degraded+remapped, 6 active+undersized+degraded+remapped+backfill_wait, 76 active+clean; 2.9 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 645 op/s; 42660/143355 objects degraded (29.758%); 5805/143355 objects misplaced (4.049%); 5.7 MiB/s, 90 objects/s recovering 2023-06-03T20:25:30.347 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:25:30.348 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:25:30.349 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:25:30.450 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:25:30.460 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:25:30 smithi164 bash[59462]: cluster 1685823929.0653532 mgr.y (mgr.34104) 1488 : cluster 0 pgmap v1518: 99 pgs: 6 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 2 active+recovery_wait, 7 active+recovery_wait+undersized+degraded+remapped, 6 active+undersized+degraded+remapped+backfill_wait, 76 active+clean; 2.9 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 645 op/s; 42660/143355 objects degraded (29.758%); 5805/143355 objects misplaced (4.049%); 5.7 MiB/s, 90 objects/s recovering 2023-06-03T20:25:30.480 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:25:30.483 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:25:30.483 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:25:30.584 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:25:30.614 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:25:30.619 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:25:30.619 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:25:30.720 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:25:30.755 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:25:30.758 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:25:30.759 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:25:30.860 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:25:30.910 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:25:30.914 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:25:30.914 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:25:31.015 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:25:31.053 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:25:31.058 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:25:31.059 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:25:31.160 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:25:31.187 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:25:31.189 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:25:31.189 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:25:31.290 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:25:31.335 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:25:31.339 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:25:31.340 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:25:31.440 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:25:31.482 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:25:31.486 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:25:31.487 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:25:31.587 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:25:31.612 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:25:31.614 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:25:31.615 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:25:31.716 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:25:31.745 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:25:31.751 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:25:31.752 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:25:31.853 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:25:31.892 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:25:31.897 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:25:31.897 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:25:31.898 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:25:31 smithi164 bash[59462]: audit 1685823930.628931 mon.a (mon.0) 1480 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:25:31.898 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:25:31 smithi164 bash[59462]: audit 1685823930.6368377 mon.a (mon.0) 1481 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:25:31.898 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:25:31 smithi164 bash[59462]: cluster 1685823931.066127 mgr.y (mgr.34104) 1489 : cluster 0 pgmap v1519: 99 pgs: 6 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 2 active+recovery_wait, 7 active+recovery_wait+undersized+degraded+remapped, 6 active+undersized+degraded+remapped+backfill_wait, 76 active+clean; 2.9 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 547 op/s; 42316/141756 objects degraded (29.851%); 5805/141756 objects misplaced (4.095%); 5.2 MiB/s, 83 objects/s recovering 2023-06-03T20:25:31.998 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:25:32.007 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:25:31 smithi089 bash[75068]: audit 1685823930.628931 mon.a (mon.0) 1480 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:25:32.007 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:25:31 smithi089 bash[75068]: audit 1685823930.6368377 mon.a (mon.0) 1481 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:25:32.008 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:25:31 smithi089 bash[75068]: cluster 1685823931.066127 mgr.y (mgr.34104) 1489 : cluster 0 pgmap v1519: 99 pgs: 6 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 2 active+recovery_wait, 7 active+recovery_wait+undersized+degraded+remapped, 6 active+undersized+degraded+remapped+backfill_wait, 76 active+clean; 2.9 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 547 op/s; 42316/141756 objects degraded (29.851%); 5805/141756 objects misplaced (4.095%); 5.2 MiB/s, 83 objects/s recovering 2023-06-03T20:25:32.008 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:25:31 smithi089 bash[78452]: audit 1685823930.628931 mon.a (mon.0) 1480 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:25:32.008 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:25:31 smithi089 bash[78452]: audit 1685823930.6368377 mon.a (mon.0) 1481 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:25:32.008 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:25:31 smithi089 bash[78452]: cluster 1685823931.066127 mgr.y (mgr.34104) 1489 : cluster 0 pgmap v1519: 99 pgs: 6 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 2 active+recovery_wait, 7 active+recovery_wait+undersized+degraded+remapped, 6 active+undersized+degraded+remapped+backfill_wait, 76 active+clean; 2.9 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 547 op/s; 42316/141756 objects degraded (29.851%); 5805/141756 objects misplaced (4.095%); 5.2 MiB/s, 83 objects/s recovering 2023-06-03T20:25:32.026 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:25:32.028 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:25:32.028 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:25:32.129 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:25:32.171 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:25:32.172 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:25:32.172 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:25:32.273 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:25:32.302 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:25:32.304 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:25:32.305 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:25:32.359 DEBUG:teuthology.orchestra.run.smithi164:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph daemon osd.5 dump_ops_in_flight 2023-06-03T20:25:32.406 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:25:32.437 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:25:32.445 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:25:32.446 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:25:32.547 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:25:32.578 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:25:32.585 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:25:32.585 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:25:32.686 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:25:32.718 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:25:32.724 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:25:32.725 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:25:32.826 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:25:32.854 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:25:32.858 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:25:32.858 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:25:32.959 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:25:32.997 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:25:33.001 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:25:33.002 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:25:33.103 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:25:33.148 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:25:33.153 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:25:33.153 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:25:33.249 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:25:33 smithi089 bash[78452]: cluster 1685823932.970899 mon.a (mon.0) 1482 : cluster 3 Health check update: Degraded data redundancy: 42316/141756 objects degraded (29.851%), 20 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-06-03T20:25:33.249 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:25:33 smithi089 bash[75068]: cluster 1685823932.970899 mon.a (mon.0) 1482 : cluster 3 Health check update: Degraded data redundancy: 42316/141756 objects degraded (29.851%), 20 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-06-03T20:25:33.254 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:25:33.317 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:25:33.321 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:25:33 smithi164 bash[59462]: cluster 1685823932.970899 mon.a (mon.0) 1482 : cluster 3 Health check update: Degraded data redundancy: 42316/141756 objects degraded (29.851%), 20 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-06-03T20:25:33.322 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:25:33.324 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:25:33.425 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:25:33.475 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:25:33.481 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:25:33.482 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:25:33.583 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:25:33.620 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:25:33.623 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:25:33.625 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:25:33.726 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:25:33.758 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:25:33.763 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:25:33.764 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:25:33.865 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:25:33.906 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:25:33.908 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:25:33.909 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:25:34.010 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:25:34.056 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:25:34.058 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:25:34.059 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:25:34.137 INFO:teuthology.orchestra.run.smithi089.stdout:osd.0: {} 2023-06-03T20:25:34.159 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:25:34.169 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:25:34 smithi164 bash[59462]: cluster 1685823933.067205 mgr.y (mgr.34104) 1490 : cluster 0 pgmap v1520: 99 pgs: 6 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait, 7 active+recovery_wait+undersized+degraded+remapped, 6 active+undersized+degraded+remapped+backfill_wait, 76 active+clean; 2.8 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 653 op/s; 41374/136065 objects degraded (30.408%); 5610/136065 objects misplaced (4.123%); 7.2 MiB/s, 114 objects/s recovering 2023-06-03T20:25:34.170 INFO:teuthology.orchestra.run.smithi089.stdout:osd.1: {} 2023-06-03T20:25:34.199 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:25:34.203 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:25:34.203 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:25:34.208 INFO:teuthology.orchestra.run.smithi089.stdout:osd.2: {} 2023-06-03T20:25:34.242 INFO:teuthology.orchestra.run.smithi089.stdout:osd.3: {} 2023-06-03T20:25:34.279 INFO:teuthology.orchestra.run.smithi089.stdout:osd.4: {} 2023-06-03T20:25:34.304 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:25:34.315 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:25:34 smithi089 bash[75068]: cluster 1685823933.067205 mgr.y (mgr.34104) 1490 : cluster 0 pgmap v1520: 99 pgs: 6 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait, 7 active+recovery_wait+undersized+degraded+remapped, 6 active+undersized+degraded+remapped+backfill_wait, 76 active+clean; 2.8 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 653 op/s; 41374/136065 objects degraded (30.408%); 5610/136065 objects misplaced (4.123%); 7.2 MiB/s, 114 objects/s recovering 2023-06-03T20:25:34.315 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:25:34 smithi089 bash[78452]: cluster 1685823933.067205 mgr.y (mgr.34104) 1490 : cluster 0 pgmap v1520: 99 pgs: 6 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait, 7 active+recovery_wait+undersized+degraded+remapped, 6 active+undersized+degraded+remapped+backfill_wait, 76 active+clean; 2.8 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 653 op/s; 41374/136065 objects degraded (30.408%); 5610/136065 objects misplaced (4.123%); 7.2 MiB/s, 114 objects/s recovering 2023-06-03T20:25:34.317 INFO:teuthology.orchestra.run.smithi089.stdout:osd.5: {} 2023-06-03T20:25:34.337 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:25:34.345 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:25:34.346 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:25:34.348 INFO:teuthology.orchestra.run.smithi089.stdout:osd.6: {} 2023-06-03T20:25:34.386 INFO:teuthology.orchestra.run.smithi089.stdout:osd.7: {} 2023-06-03T20:25:34.391 INFO:teuthology.orchestra.run.smithi089.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-06-03T20:25:34.391 INFO:teuthology.orchestra.run.smithi089.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-06-03T20:25:34.391 INFO:teuthology.orchestra.run.smithi089.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-06-03T20:25:34.391 INFO:teuthology.orchestra.run.smithi089.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-06-03T20:25:34.391 INFO:teuthology.orchestra.run.smithi089.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-06-03T20:25:34.392 INFO:teuthology.orchestra.run.smithi089.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-06-03T20:25:34.392 INFO:teuthology.orchestra.run.smithi089.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-06-03T20:25:34.392 INFO:teuthology.orchestra.run.smithi089.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-06-03T20:25:34.448 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:25:34.490 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:25:34.496 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:25:34.497 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:25:34.598 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:25:34.626 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:25:34.627 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:25:34.629 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:25:34.731 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:25:34.771 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:25:34.772 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:25:34.773 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:25:34.873 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:25:34.906 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:25:34.908 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:25:34.908 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:25:35.009 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:25:35.064 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:25:35.066 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:25:35.067 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:25:35.168 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:25:35.200 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:25:35.204 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:25:35.204 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:25:35.279 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:25:35 smithi089 bash[78452]: audit 1685823934.0979235 mon.c (mon.2) 623 : audit 0 from='client.? 172.21.15.89:0/3644378734' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-03T20:25:35.280 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:25:35 smithi089 bash[75068]: audit 1685823934.0979235 mon.c (mon.2) 623 : audit 0 from='client.? 172.21.15.89:0/3644378734' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-03T20:25:35.306 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:25:35.334 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:25:35 smithi164 bash[59462]: audit 1685823934.0979235 mon.c (mon.2) 623 : audit 0 from='client.? 172.21.15.89:0/3644378734' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-03T20:25:35.338 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:25:35.341 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:25:35.342 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:25:35.443 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:25:35.488 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:25:35.491 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:25:35.492 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:25:35.594 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:25:35.637 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:25:35.643 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:25:35.644 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:25:35.745 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:25:35.791 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:25:35.799 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:25:35.800 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:25:35.901 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:25:35.947 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:25:35.952 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:25:35.953 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:25:36.047 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:25:36 smithi164 bash[59462]: cluster 1685823935.0682707 mgr.y (mgr.34104) 1491 : cluster 0 pgmap v1521: 99 pgs: 6 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 2 active+recovery_wait, 7 active+recovery_wait+undersized+degraded+remapped, 6 active+undersized+degraded+remapped+backfill_wait, 76 active+clean; 2.7 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 578 op/s; 40695/132642 objects degraded (30.680%); 5392/132642 objects misplaced (4.065%); 5.5 MiB/s, 87 objects/s recovering 2023-06-03T20:25:36.055 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:25:36.097 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:25:36.104 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:25:36.105 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:25:36.207 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:25:36.238 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:25:36.241 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:25:36.242 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:25:36.343 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:25:36.351 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 20:25:36 smithi089 bash[71163]: ::ffff:172.21.15.164 - - [03/Jun/2023:20:25:36] "GET /metrics HTTP/1.1" 200 228998 "" "Prometheus/2.33.4" 2023-06-03T20:25:36.352 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:25:36 smithi089 bash[75068]: cluster 1685823935.0682707 mgr.y (mgr.34104) 1491 : cluster 0 pgmap v1521: 99 pgs: 6 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 2 active+recovery_wait, 7 active+recovery_wait+undersized+degraded+remapped, 6 active+undersized+degraded+remapped+backfill_wait, 76 active+clean; 2.7 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 578 op/s; 40695/132642 objects degraded (30.680%); 5392/132642 objects misplaced (4.065%); 5.5 MiB/s, 87 objects/s recovering 2023-06-03T20:25:36.352 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:25:36 smithi089 bash[78452]: cluster 1685823935.0682707 mgr.y (mgr.34104) 1491 : cluster 0 pgmap v1521: 99 pgs: 6 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 2 active+recovery_wait, 7 active+recovery_wait+undersized+degraded+remapped, 6 active+undersized+degraded+remapped+backfill_wait, 76 active+clean; 2.7 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 578 op/s; 40695/132642 objects degraded (30.680%); 5392/132642 objects misplaced (4.065%); 5.5 MiB/s, 87 objects/s recovering 2023-06-03T20:25:36.376 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:25:36.382 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:25:36.383 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:25:36.484 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:25:36.515 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:25:36.518 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:25:36.519 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:25:36.620 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:25:36.654 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:25:36.656 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:25:36.657 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:25:36.759 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:25:36.791 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:25:36.797 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:25:36.798 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:25:36.899 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:25:36.948 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:25:36.950 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:25:36.950 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:25:37.051 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:25:37.094 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:25:37.098 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:25:37.100 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:25:37.187 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/mon.a/config 2023-06-03T20:25:37.201 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:25:37.244 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:25:37.245 INFO:teuthology.orchestra.run.smithi164.stderr:Inferring config /var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/mon.b/config 2023-06-03T20:25:37.247 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:25:37.248 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:25:37.349 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:25:37.408 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:25:37.412 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:25:37.413 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:25:37.514 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:25:37.548 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:25:37.550 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:25:37.551 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:25:37.653 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:25:37.685 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:25:37.933 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:25:37.934 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:25:38.036 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:25:38.078 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:25:38.081 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:25:38.081 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:25:38.150 INFO:teuthology.orchestra.run.smithi089.stderr:noscrub is set 2023-06-03T20:25:38.182 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:25:38.241 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:25:38.247 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:25:38.248 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:25:38.334 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:25:38 smithi164 bash[59462]: cluster 1685823937.0691984 mgr.y (mgr.34104) 1492 : cluster 0 pgmap v1522: 99 pgs: 6 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait, 7 active+recovery_wait+undersized+degraded+remapped, 6 active+undersized+degraded+remapped+backfill_wait, 76 active+clean; 2.6 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 571 op/s; 40277/129567 objects degraded (31.086%); 5209/129567 objects misplaced (4.020%); 7.1 MiB/s, 113 objects/s recovering 2023-06-03T20:25:38.335 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:25:38 smithi164 bash[59462]: audit 1685823937.145914 mon.c (mon.2) 624 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T20:25:38.335 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:25:38 smithi164 bash[59462]: audit 1685823937.4619012 mon.a (mon.0) 1483 : audit 1 from='client.? 172.21.15.89:0/3664958386' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-06-03T20:25:38.335 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:25:38 smithi164 bash[59462]: cluster 1685823937.9720907 mon.a (mon.0) 1484 : cluster 3 Health check update: Degraded data redundancy: 40277/129567 objects degraded (31.086%), 20 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-06-03T20:25:38.348 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:25:38.386 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:25:38.387 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:25:38.388 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:25:38.489 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:25:38.497 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:25:38 smithi089 bash[75068]: cluster 1685823937.0691984 mgr.y (mgr.34104) 1492 : cluster 0 pgmap v1522: 99 pgs: 6 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait, 7 active+recovery_wait+undersized+degraded+remapped, 6 active+undersized+degraded+remapped+backfill_wait, 76 active+clean; 2.6 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 571 op/s; 40277/129567 objects degraded (31.086%); 5209/129567 objects misplaced (4.020%); 7.1 MiB/s, 113 objects/s recovering 2023-06-03T20:25:38.497 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:25:38 smithi089 bash[75068]: audit 1685823937.145914 mon.c (mon.2) 624 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T20:25:38.497 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:25:38 smithi089 bash[75068]: audit 1685823937.4619012 mon.a (mon.0) 1483 : audit 1 from='client.? 172.21.15.89:0/3664958386' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-06-03T20:25:38.497 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:25:38 smithi089 bash[75068]: cluster 1685823937.9720907 mon.a (mon.0) 1484 : cluster 3 Health check update: Degraded data redundancy: 40277/129567 objects degraded (31.086%), 20 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-06-03T20:25:38.498 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:25:38 smithi089 bash[78452]: cluster 1685823937.0691984 mgr.y (mgr.34104) 1492 : cluster 0 pgmap v1522: 99 pgs: 6 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait, 7 active+recovery_wait+undersized+degraded+remapped, 6 active+undersized+degraded+remapped+backfill_wait, 76 active+clean; 2.6 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 571 op/s; 40277/129567 objects degraded (31.086%); 5209/129567 objects misplaced (4.020%); 7.1 MiB/s, 113 objects/s recovering 2023-06-03T20:25:38.498 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:25:38 smithi089 bash[78452]: audit 1685823937.145914 mon.c (mon.2) 624 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T20:25:38.498 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:25:38 smithi089 bash[78452]: audit 1685823937.4619012 mon.a (mon.0) 1483 : audit 1 from='client.? 172.21.15.89:0/3664958386' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-06-03T20:25:38.498 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:25:38 smithi089 bash[78452]: cluster 1685823937.9720907 mon.a (mon.0) 1484 : cluster 3 Health check update: Degraded data redundancy: 40277/129567 objects degraded (31.086%), 20 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-06-03T20:25:38.524 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:25:38.530 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:25:38.531 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:25:38.632 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:25:38.663 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:25:38.674 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:25:38.675 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:25:38.714 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-06-03T20:25:38.776 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:25:38.828 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:25:38.831 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:25:38.832 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:25:38.933 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:25:38.961 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:25:38.968 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:25:38.969 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:25:39.070 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:25:39.107 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:25:39.111 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:25:39.112 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:25:39.213 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:25:39.222 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:25:39 smithi164 bash[59462]: cluster 1685823938.1244836 mon.a (mon.0) 1485 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-06-03T20:25:39.222 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:25:39 smithi164 bash[59462]: audit 1685823938.134094 mon.a (mon.0) 1486 : audit 1 from='client.? 172.21.15.89:0/3664958386' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-06-03T20:25:39.222 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:25:39 smithi164 bash[59462]: cluster 1685823938.134303 mon.a (mon.0) 1487 : cluster 0 osdmap e265: 8 total, 8 up, 8 in 2023-06-03T20:25:39.222 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:25:39 smithi164 bash[59462]: cluster 1685823938.1445115 osd.5 (osd.5) 210 : cluster 3 failed to encode map e265 with expected crc 2023-06-03T20:25:39.223 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:25:39 smithi164 bash[59462]: cluster 1685823938.1468275 osd.6 (osd.6) 202 : cluster 3 failed to encode map e265 with expected crc 2023-06-03T20:25:39.223 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:25:39 smithi164 bash[59462]: cluster 1685823938.1482584 osd.7 (osd.7) 210 : cluster 3 failed to encode map e265 with expected crc 2023-06-03T20:25:39.223 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:25:39 smithi164 bash[59462]: cluster 1685823938.1489482 osd.5 (osd.5) 211 : cluster 3 failed to encode map e265 with expected crc 2023-06-03T20:25:39.223 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:25:39 smithi164 bash[59462]: cluster 1685823938.1539087 osd.7 (osd.7) 211 : cluster 3 failed to encode map e265 with expected crc 2023-06-03T20:25:39.223 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:25:39 smithi164 bash[59462]: cluster 1685823938.160764 osd.5 (osd.5) 212 : cluster 3 failed to encode map e265 with expected crc 2023-06-03T20:25:39.223 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:25:39 smithi164 bash[59462]: cluster 1685823938.1653013 osd.7 (osd.7) 212 : cluster 3 failed to encode map e265 with expected crc 2023-06-03T20:25:39.223 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:25:39 smithi164 bash[59462]: audit 1685823938.731255 mon.a (mon.0) 1488 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:25:39.224 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:25:39 smithi164 bash[59462]: audit 1685823938.740546 mon.a (mon.0) 1489 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:25:39.243 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:25:39.249 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:25:39.250 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:25:39.351 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:25:39.359 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:25:39 smithi089 bash[75068]: cluster 1685823938.1244836 mon.a (mon.0) 1485 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-06-03T20:25:39.360 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:25:39 smithi089 bash[75068]: audit 1685823938.134094 mon.a (mon.0) 1486 : audit 1 from='client.? 172.21.15.89:0/3664958386' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-06-03T20:25:39.360 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:25:39 smithi089 bash[75068]: cluster 1685823938.134303 mon.a (mon.0) 1487 : cluster 0 osdmap e265: 8 total, 8 up, 8 in 2023-06-03T20:25:39.360 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:25:39 smithi089 bash[75068]: cluster 1685823938.1445115 osd.5 (osd.5) 210 : cluster 3 failed to encode map e265 with expected crc 2023-06-03T20:25:39.360 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:25:39 smithi089 bash[75068]: cluster 1685823938.1468275 osd.6 (osd.6) 202 : cluster 3 failed to encode map e265 with expected crc 2023-06-03T20:25:39.360 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:25:39 smithi089 bash[75068]: cluster 1685823938.1482584 osd.7 (osd.7) 210 : cluster 3 failed to encode map e265 with expected crc 2023-06-03T20:25:39.360 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:25:39 smithi089 bash[75068]: cluster 1685823938.1489482 osd.5 (osd.5) 211 : cluster 3 failed to encode map e265 with expected crc 2023-06-03T20:25:39.360 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:25:39 smithi089 bash[75068]: cluster 1685823938.1539087 osd.7 (osd.7) 211 : cluster 3 failed to encode map e265 with expected crc 2023-06-03T20:25:39.361 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:25:39 smithi089 bash[75068]: cluster 1685823938.160764 osd.5 (osd.5) 212 : cluster 3 failed to encode map e265 with expected crc 2023-06-03T20:25:39.361 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:25:39 smithi089 bash[75068]: cluster 1685823938.1653013 osd.7 (osd.7) 212 : cluster 3 failed to encode map e265 with expected crc 2023-06-03T20:25:39.361 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:25:39 smithi089 bash[75068]: audit 1685823938.731255 mon.a (mon.0) 1488 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:25:39.361 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:25:39 smithi089 bash[75068]: audit 1685823938.740546 mon.a (mon.0) 1489 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:25:39.361 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:25:39 smithi089 bash[78452]: cluster 1685823938.1244836 mon.a (mon.0) 1485 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-06-03T20:25:39.362 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:25:39 smithi089 bash[78452]: audit 1685823938.134094 mon.a (mon.0) 1486 : audit 1 from='client.? 172.21.15.89:0/3664958386' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-06-03T20:25:39.362 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:25:39 smithi089 bash[78452]: cluster 1685823938.134303 mon.a (mon.0) 1487 : cluster 0 osdmap e265: 8 total, 8 up, 8 in 2023-06-03T20:25:39.362 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:25:39 smithi089 bash[78452]: cluster 1685823938.1445115 osd.5 (osd.5) 210 : cluster 3 failed to encode map e265 with expected crc 2023-06-03T20:25:39.362 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:25:39 smithi089 bash[78452]: cluster 1685823938.1468275 osd.6 (osd.6) 202 : cluster 3 failed to encode map e265 with expected crc 2023-06-03T20:25:39.362 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:25:39 smithi089 bash[78452]: cluster 1685823938.1482584 osd.7 (osd.7) 210 : cluster 3 failed to encode map e265 with expected crc 2023-06-03T20:25:39.362 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:25:39 smithi089 bash[78452]: cluster 1685823938.1489482 osd.5 (osd.5) 211 : cluster 3 failed to encode map e265 with expected crc 2023-06-03T20:25:39.362 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:25:39 smithi089 bash[78452]: cluster 1685823938.1539087 osd.7 (osd.7) 211 : cluster 3 failed to encode map e265 with expected crc 2023-06-03T20:25:39.362 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:25:39 smithi089 bash[78452]: cluster 1685823938.160764 osd.5 (osd.5) 212 : cluster 3 failed to encode map e265 with expected crc 2023-06-03T20:25:39.363 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:25:39 smithi089 bash[78452]: cluster 1685823938.1653013 osd.7 (osd.7) 212 : cluster 3 failed to encode map e265 with expected crc 2023-06-03T20:25:39.363 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:25:39 smithi089 bash[78452]: audit 1685823938.731255 mon.a (mon.0) 1488 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:25:39.363 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:25:39 smithi089 bash[78452]: audit 1685823938.740546 mon.a (mon.0) 1489 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:25:39.382 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:25:39.387 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:25:39.387 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:25:39.489 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:25:39.525 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:25:39.527 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:25:39.529 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:25:39.630 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:25:39.673 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:25:39.678 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:25:39.680 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:25:39.781 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:25:39.813 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:25:39.815 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:25:39.816 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:25:39.917 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:25:39.958 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:25:39.961 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:25:39.962 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:25:40.063 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:25:40.094 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:25:40.096 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:25:40.097 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:25:40.198 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:25:40.208 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:25:40 smithi164 bash[59462]: cluster 1685823939.0701766 mgr.y (mgr.34104) 1493 : cluster 0 pgmap v1524: 99 pgs: 6 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 2 active+recovery_wait, 7 active+recovery_wait+undersized+degraded+remapped, 6 active+undersized+degraded+remapped+backfill_wait, 76 active+clean; 2.5 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 625 op/s; 39341/124584 objects degraded (31.578%); 5013/124584 objects misplaced (4.024%); 6.2 MiB/s, 99 objects/s recovering 2023-06-03T20:25:40.208 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:25:40 smithi164 bash[59462]: audit 1685823939.3750045 mon.c (mon.2) 625 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-03T20:25:40.209 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:25:40 smithi164 bash[59462]: audit 1685823939.37621 mon.c (mon.2) 626 : audit 1 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-03T20:25:40.209 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:25:40 smithi164 bash[59462]: audit 1685823939.385913 mon.a (mon.0) 1490 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:25:40.243 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:25:40.248 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:25:40.249 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:25:40.350 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:25:40.385 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:25:40.392 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:25:40.393 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:25:40.408 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:25:40 smithi089 bash[75068]: cluster 1685823939.0701766 mgr.y (mgr.34104) 1493 : cluster 0 pgmap v1524: 99 pgs: 6 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 2 active+recovery_wait, 7 active+recovery_wait+undersized+degraded+remapped, 6 active+undersized+degraded+remapped+backfill_wait, 76 active+clean; 2.5 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 625 op/s; 39341/124584 objects degraded (31.578%); 5013/124584 objects misplaced (4.024%); 6.2 MiB/s, 99 objects/s recovering 2023-06-03T20:25:40.408 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:25:40 smithi089 bash[75068]: audit 1685823939.3750045 mon.c (mon.2) 625 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-03T20:25:40.408 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:25:40 smithi089 bash[75068]: audit 1685823939.37621 mon.c (mon.2) 626 : audit 1 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-03T20:25:40.409 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:25:40 smithi089 bash[75068]: audit 1685823939.385913 mon.a (mon.0) 1490 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:25:40.409 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:25:40 smithi089 bash[78452]: cluster 1685823939.0701766 mgr.y (mgr.34104) 1493 : cluster 0 pgmap v1524: 99 pgs: 6 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 2 active+recovery_wait, 7 active+recovery_wait+undersized+degraded+remapped, 6 active+undersized+degraded+remapped+backfill_wait, 76 active+clean; 2.5 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 625 op/s; 39341/124584 objects degraded (31.578%); 5013/124584 objects misplaced (4.024%); 6.2 MiB/s, 99 objects/s recovering 2023-06-03T20:25:40.409 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:25:40 smithi089 bash[78452]: audit 1685823939.3750045 mon.c (mon.2) 625 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-03T20:25:40.409 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:25:40 smithi089 bash[78452]: audit 1685823939.37621 mon.c (mon.2) 626 : audit 1 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-03T20:25:40.409 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:25:40 smithi089 bash[78452]: audit 1685823939.385913 mon.a (mon.0) 1490 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:25:40.495 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:25:40.523 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:25:40.525 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:25:40.526 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:25:40.628 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:25:40.665 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:25:40.671 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:25:40.672 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:25:40.773 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:25:40.809 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:25:40.813 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:25:40.814 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:25:40.915 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:25:40.952 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:25:40.957 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:25:40.958 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:25:41.059 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:25:41.092 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:25:41.096 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:25:41.096 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:25:41.197 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:25:41.237 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:25:41.244 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:25:41.245 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:25:41.346 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:25:41.381 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:25:41.383 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:25:41.383 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:25:41.484 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:25:41.521 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:25:41.526 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:25:41.527 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:25:41.628 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:25:41.676 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:25:41.682 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:25:41.683 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:25:41.784 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:25:41.819 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:25:41.822 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:25:41.823 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:25:41.924 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:25:41.961 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:25:41.964 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:25:41.965 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:25:42.066 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:25:42.128 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:25:42.130 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:25:42.132 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:25:42.155 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:25:42 smithi089 bash[75068]: cluster 1685823941.0709302 mgr.y (mgr.34104) 1494 : cluster 0 pgmap v1525: 99 pgs: 6 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 2 active+recovery_wait, 7 active+recovery_wait+undersized+degraded+remapped, 6 active+undersized+degraded+remapped+backfill_wait, 76 active+clean; 2.5 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 626 op/s; 38957/122946 objects degraded (31.686%); 5013/122946 objects misplaced (4.077%); 6.2 MiB/s, 99 objects/s recovering 2023-06-03T20:25:42.232 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:25:42.272 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:25:42.273 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph osd set nodeep-scrub 2023-06-03T20:25:42.274 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:25:42.275 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:25:42.354 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:25:42 smithi164 bash[59462]: cluster 1685823941.0709302 mgr.y (mgr.34104) 1494 : cluster 0 pgmap v1525: 99 pgs: 6 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 2 active+recovery_wait, 7 active+recovery_wait+undersized+degraded+remapped, 6 active+undersized+degraded+remapped+backfill_wait, 76 active+clean; 2.5 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 626 op/s; 38957/122946 objects degraded (31.686%); 5013/122946 objects misplaced (4.077%); 6.2 MiB/s, 99 objects/s recovering 2023-06-03T20:25:42.377 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:25:42.430 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:25:42.433 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:25:42 smithi089 bash[78452]: cluster 1685823941.0709302 mgr.y (mgr.34104) 1494 : cluster 0 pgmap v1525: 99 pgs: 6 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 2 active+recovery_wait, 7 active+recovery_wait+undersized+degraded+remapped, 6 active+undersized+degraded+remapped+backfill_wait, 76 active+clean; 2.5 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 626 op/s; 38957/122946 objects degraded (31.686%); 5013/122946 objects misplaced (4.077%); 6.2 MiB/s, 99 objects/s recovering 2023-06-03T20:25:42.434 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:25:42.435 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:25:42.536 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:25:42.572 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:25:42.574 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:25:42.575 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:25:42.676 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:25:42.715 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:25:42.720 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:25:42.721 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:25:42.822 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:25:42.867 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:25:42.873 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:25:42.875 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:25:42.976 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:25:43.011 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:25:43.014 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:25:43.016 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:25:43.117 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:25:43.151 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:25:43.155 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:25:43.156 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:25:43.257 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:25:43.279 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:25:43 smithi089 bash[78452]: cluster 1685823942.9730535 mon.a (mon.0) 1491 : cluster 3 Health check update: Degraded data redundancy: 38957/122946 objects degraded (31.686%), 20 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-06-03T20:25:43.280 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:25:43 smithi089 bash[75068]: cluster 1685823942.9730535 mon.a (mon.0) 1491 : cluster 3 Health check update: Degraded data redundancy: 38957/122946 objects degraded (31.686%), 20 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-06-03T20:25:43.311 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:25:43.318 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:25:43.319 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:25:43.420 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:25:43.454 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:25:43.455 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:25:43.456 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:25:43.557 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:25:43.585 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:25:43 smithi164 bash[59462]: cluster 1685823942.9730535 mon.a (mon.0) 1491 : cluster 3 Health check update: Degraded data redundancy: 38957/122946 objects degraded (31.686%), 20 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-06-03T20:25:43.590 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:25:43.591 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:25:43.592 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:25:43.693 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:25:43.722 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:25:43.726 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:25:43.728 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:25:43.828 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:25:43.879 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:25:43.879 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:25:43.880 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:25:43.981 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:25:44.028 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:25:44.030 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:25:44.031 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:25:44.132 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:25:44.165 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:25:44.172 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:25:44.173 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:25:44.268 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:25:44 smithi089 bash[78452]: cluster 1685823943.0718226 mgr.y (mgr.34104) 1495 : cluster 0 pgmap v1526: 99 pgs: 6 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait, 7 active+recovery_wait+undersized+degraded+remapped, 6 active+undersized+degraded+remapped+backfill_wait, 76 active+clean; 2.4 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 637 op/s; 37988/116931 objects degraded (32.488%); 4833/116931 objects misplaced (4.133%); 6.3 MiB/s, 100 objects/s recovering 2023-06-03T20:25:44.269 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:25:44 smithi089 bash[75068]: cluster 1685823943.0718226 mgr.y (mgr.34104) 1495 : cluster 0 pgmap v1526: 99 pgs: 6 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait, 7 active+recovery_wait+undersized+degraded+remapped, 6 active+undersized+degraded+remapped+backfill_wait, 76 active+clean; 2.4 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 637 op/s; 37988/116931 objects degraded (32.488%); 4833/116931 objects misplaced (4.133%); 6.3 MiB/s, 100 objects/s recovering 2023-06-03T20:25:44.274 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:25:44.314 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:25:44.317 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:25:44.319 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:25:44.420 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:25:44.429 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:25:44 smithi164 bash[59462]: cluster 1685823943.0718226 mgr.y (mgr.34104) 1495 : cluster 0 pgmap v1526: 99 pgs: 6 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait, 7 active+recovery_wait+undersized+degraded+remapped, 6 active+undersized+degraded+remapped+backfill_wait, 76 active+clean; 2.4 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 637 op/s; 37988/116931 objects degraded (32.488%); 4833/116931 objects misplaced (4.133%); 6.3 MiB/s, 100 objects/s recovering 2023-06-03T20:25:44.457 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:25:44.463 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:25:44.464 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:25:44.564 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:25:44.598 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:25:44.601 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:25:44.602 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:25:44.703 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:25:44.747 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:25:44.751 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:25:44.752 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:25:44.854 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:25:44.889 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:25:44.894 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:25:44.895 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:25:44.996 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:25:45.053 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:25:45.059 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:25:45.060 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:25:45.161 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:25:45.162 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:25:45 smithi089 bash[75068]: audit 1685823944.8035896 mon.a (mon.0) 1492 : audit 1 from='client.? 172.21.15.89:0/3592016714' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-06-03T20:25:45.185 INFO:teuthology.orchestra.run.smithi089.stderr:reweighted osd.0 to 1 (10000) 2023-06-03T20:25:45.198 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:25:45.203 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:25:45.203 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:25:45.304 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:25:45.313 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:25:45 smithi164 bash[59462]: audit 1685823944.8035896 mon.a (mon.0) 1492 : audit 1 from='client.? 172.21.15.89:0/3592016714' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-06-03T20:25:45.345 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:25:45.349 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:25:45.350 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:25:45.451 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:25:45.459 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:25:45 smithi089 bash[78452]: audit 1685823944.8035896 mon.a (mon.0) 1492 : audit 1 from='client.? 172.21.15.89:0/3592016714' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-06-03T20:25:45.486 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:25:45.489 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:25:45.489 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:25:45.591 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:25:45.628 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:25:45.630 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:25:45.631 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:25:45.731 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:25:45.772 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:25:45.779 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:25:45.779 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:25:45.880 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:25:45.902 INFO:teuthology.orchestra.run.smithi164.stdout:ERROR: (22) Invalid argument 2023-06-03T20:25:45.903 INFO:teuthology.orchestra.run.smithi164.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-06-03T20:25:45.917 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:25:45.922 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:25:45.922 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:25:46.023 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:25:46.057 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:25:46.060 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:25:46.061 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:25:46.162 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:25:46.190 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:25:46.196 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:25:46.197 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:25:46.298 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:25:46.336 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:25:46.339 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:25:46.340 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:25:46.441 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:25:46.450 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 20:25:46 smithi089 bash[71163]: ::ffff:172.21.15.164 - - [03/Jun/2023:20:25:46] "GET /metrics HTTP/1.1" 200 228998 "" "Prometheus/2.33.4" 2023-06-03T20:25:46.451 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:25:46 smithi089 bash[78452]: cluster 1685823945.0730505 mgr.y (mgr.34104) 1496 : cluster 0 pgmap v1527: 99 pgs: 6 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 2 active+recovery_wait, 7 active+recovery_wait+undersized+degraded+remapped, 6 active+undersized+degraded+remapped+backfill_wait, 76 active+clean; 2.3 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 648 op/s; 37272/113178 objects degraded (32.932%); 4627/113178 objects misplaced (4.088%); 6.4 MiB/s, 101 objects/s recovering 2023-06-03T20:25:46.451 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:25:46 smithi089 bash[78452]: audit 1685823945.172172 mon.a (mon.0) 1493 : audit 1 from='client.? 172.21.15.89:0/3592016714' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2023-06-03T20:25:46.451 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:25:46 smithi089 bash[78452]: cluster 1685823945.1722736 mon.a (mon.0) 1494 : cluster 0 osdmap e266: 8 total, 8 up, 8 in 2023-06-03T20:25:46.452 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:25:46 smithi089 bash[78452]: cluster 1685823945.1728287 osd.7 (osd.7) 213 : cluster 3 failed to encode map e266 with expected crc 2023-06-03T20:25:46.452 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:25:46 smithi089 bash[78452]: cluster 1685823945.1760182 osd.7 (osd.7) 214 : cluster 3 failed to encode map e266 with expected crc 2023-06-03T20:25:46.452 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:25:46 smithi089 bash[78452]: cluster 1685823945.1769545 osd.5 (osd.5) 213 : cluster 3 failed to encode map e266 with expected crc 2023-06-03T20:25:46.452 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:25:46 smithi089 bash[78452]: cluster 1685823945.1788316 osd.5 (osd.5) 214 : cluster 3 failed to encode map e266 with expected crc 2023-06-03T20:25:46.453 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:25:46 smithi089 bash[78452]: cluster 1685823945.1802657 osd.5 (osd.5) 215 : cluster 3 failed to encode map e266 with expected crc 2023-06-03T20:25:46.453 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:25:46 smithi089 bash[78452]: cluster 1685823945.1812253 osd.6 (osd.6) 203 : cluster 3 failed to encode map e266 with expected crc 2023-06-03T20:25:46.453 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:25:46 smithi089 bash[75068]: cluster 1685823945.0730505 mgr.y (mgr.34104) 1496 : cluster 0 pgmap v1527: 99 pgs: 6 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 2 active+recovery_wait, 7 active+recovery_wait+undersized+degraded+remapped, 6 active+undersized+degraded+remapped+backfill_wait, 76 active+clean; 2.3 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 648 op/s; 37272/113178 objects degraded (32.932%); 4627/113178 objects misplaced (4.088%); 6.4 MiB/s, 101 objects/s recovering 2023-06-03T20:25:46.454 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:25:46 smithi089 bash[75068]: audit 1685823945.172172 mon.a (mon.0) 1493 : audit 1 from='client.? 172.21.15.89:0/3592016714' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2023-06-03T20:25:46.454 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:25:46 smithi089 bash[75068]: cluster 1685823945.1722736 mon.a (mon.0) 1494 : cluster 0 osdmap e266: 8 total, 8 up, 8 in 2023-06-03T20:25:46.454 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:25:46 smithi089 bash[75068]: cluster 1685823945.1728287 osd.7 (osd.7) 213 : cluster 3 failed to encode map e266 with expected crc 2023-06-03T20:25:46.454 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:25:46 smithi089 bash[75068]: cluster 1685823945.1760182 osd.7 (osd.7) 214 : cluster 3 failed to encode map e266 with expected crc 2023-06-03T20:25:46.455 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:25:46 smithi089 bash[75068]: cluster 1685823945.1769545 osd.5 (osd.5) 213 : cluster 3 failed to encode map e266 with expected crc 2023-06-03T20:25:46.455 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:25:46 smithi089 bash[75068]: cluster 1685823945.1788316 osd.5 (osd.5) 214 : cluster 3 failed to encode map e266 with expected crc 2023-06-03T20:25:46.455 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:25:46 smithi089 bash[75068]: cluster 1685823945.1802657 osd.5 (osd.5) 215 : cluster 3 failed to encode map e266 with expected crc 2023-06-03T20:25:46.456 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:25:46 smithi089 bash[75068]: cluster 1685823945.1812253 osd.6 (osd.6) 203 : cluster 3 failed to encode map e266 with expected crc 2023-06-03T20:25:46.473 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:25:46.479 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:25:46.479 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:25:46.580 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:25:46.584 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:25:46 smithi164 bash[59462]: cluster 1685823945.0730505 mgr.y (mgr.34104) 1496 : cluster 0 pgmap v1527: 99 pgs: 6 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 2 active+recovery_wait, 7 active+recovery_wait+undersized+degraded+remapped, 6 active+undersized+degraded+remapped+backfill_wait, 76 active+clean; 2.3 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 648 op/s; 37272/113178 objects degraded (32.932%); 4627/113178 objects misplaced (4.088%); 6.4 MiB/s, 101 objects/s recovering 2023-06-03T20:25:46.585 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:25:46 smithi164 bash[59462]: audit 1685823945.172172 mon.a (mon.0) 1493 : audit 1 from='client.? 172.21.15.89:0/3592016714' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2023-06-03T20:25:46.585 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:25:46 smithi164 bash[59462]: cluster 1685823945.1722736 mon.a (mon.0) 1494 : cluster 0 osdmap e266: 8 total, 8 up, 8 in 2023-06-03T20:25:46.585 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:25:46 smithi164 bash[59462]: cluster 1685823945.1728287 osd.7 (osd.7) 213 : cluster 3 failed to encode map e266 with expected crc 2023-06-03T20:25:46.585 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:25:46 smithi164 bash[59462]: cluster 1685823945.1760182 osd.7 (osd.7) 214 : cluster 3 failed to encode map e266 with expected crc 2023-06-03T20:25:46.585 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:25:46 smithi164 bash[59462]: cluster 1685823945.1769545 osd.5 (osd.5) 213 : cluster 3 failed to encode map e266 with expected crc 2023-06-03T20:25:46.585 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:25:46 smithi164 bash[59462]: cluster 1685823945.1788316 osd.5 (osd.5) 214 : cluster 3 failed to encode map e266 with expected crc 2023-06-03T20:25:46.585 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:25:46 smithi164 bash[59462]: cluster 1685823945.1802657 osd.5 (osd.5) 215 : cluster 3 failed to encode map e266 with expected crc 2023-06-03T20:25:46.586 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:25:46 smithi164 bash[59462]: cluster 1685823945.1812253 osd.6 (osd.6) 203 : cluster 3 failed to encode map e266 with expected crc 2023-06-03T20:25:46.608 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:25:46.611 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:25:46.611 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:25:46.691 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph osd reweight 2 1 2023-06-03T20:25:46.712 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:25:46.776 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:25:46.779 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:25:46.780 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:25:46.881 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:25:46.919 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:25:46.925 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:25:46.926 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:25:47.028 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:25:47.068 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:25:47.074 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:25:47.075 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:25:47.121 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/mon.a/config 2023-06-03T20:25:47.176 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:25:47.206 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:25:47.208 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:25:47.209 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:25:47.243 DEBUG:teuthology.orchestra.run.smithi164:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph daemon osd.5 dump_blocked_ops 2023-06-03T20:25:47.310 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:25:47.339 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:25:47.344 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:25:47.345 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:25:47.445 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:25:47.485 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:25:47.486 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:25:47.487 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:25:47.588 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:25:47.636 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:25:47.637 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:25:47.637 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:25:47.738 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:25:47.771 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:25:47.775 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:25:47.777 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:25:47.878 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:25:47.916 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:25:47.919 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:25:47.920 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:25:48.021 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:25:48.057 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:25:48.059 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:25:48.060 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:25:48.161 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:25:48.197 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:25:48.203 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:25:48.203 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:25:48.304 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:25:48.344 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:25:48.348 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:25:48.349 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:25:48.451 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:25:48.459 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:25:48 smithi164 bash[59462]: cluster 1685823947.0738394 mgr.y (mgr.34104) 1497 : cluster 0 pgmap v1529: 99 pgs: 6 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait, 7 active+recovery_wait+undersized+degraded+remapped, 6 active+undersized+degraded+remapped+backfill_wait, 76 active+clean; 2.2 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 607 op/s; 36867/110082 objects degraded (33.490%); 4429/110082 objects misplaced (4.023%); 7.0 MiB/s, 112 objects/s recovering 2023-06-03T20:25:48.459 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:25:48 smithi164 bash[59462]: cluster 1685823947.976808 mon.a (mon.0) 1495 : cluster 3 Health check update: Degraded data redundancy: 36867/110082 objects degraded (33.490%), 20 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-06-03T20:25:48.499 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:25:48.503 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:25:48.504 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:25:48.529 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:25:48 smithi089 bash[75068]: cluster 1685823947.0738394 mgr.y (mgr.34104) 1497 : cluster 0 pgmap v1529: 99 pgs: 6 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait, 7 active+recovery_wait+undersized+degraded+remapped, 6 active+undersized+degraded+remapped+backfill_wait, 76 active+clean; 2.2 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 607 op/s; 36867/110082 objects degraded (33.490%); 4429/110082 objects misplaced (4.023%); 7.0 MiB/s, 112 objects/s recovering 2023-06-03T20:25:48.529 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:25:48 smithi089 bash[75068]: cluster 1685823947.976808 mon.a (mon.0) 1495 : cluster 3 Health check update: Degraded data redundancy: 36867/110082 objects degraded (33.490%), 20 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-06-03T20:25:48.530 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:25:48 smithi089 bash[78452]: cluster 1685823947.0738394 mgr.y (mgr.34104) 1497 : cluster 0 pgmap v1529: 99 pgs: 6 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait, 7 active+recovery_wait+undersized+degraded+remapped, 6 active+undersized+degraded+remapped+backfill_wait, 76 active+clean; 2.2 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 607 op/s; 36867/110082 objects degraded (33.490%); 4429/110082 objects misplaced (4.023%); 7.0 MiB/s, 112 objects/s recovering 2023-06-03T20:25:48.530 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:25:48 smithi089 bash[78452]: cluster 1685823947.976808 mon.a (mon.0) 1495 : cluster 3 Health check update: Degraded data redundancy: 36867/110082 objects degraded (33.490%), 20 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-06-03T20:25:48.606 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:25:48.645 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:25:48.646 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:25:48.646 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:25:48.747 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:25:48.789 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:25:48.791 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:25:48.792 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:25:48.893 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:25:48.945 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:25:48.951 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:25:48.952 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:25:49.053 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:25:49.084 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:25:49.087 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:25:49.088 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:25:49.189 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:25:49.226 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/mon.a/config 2023-06-03T20:25:49.227 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:25:49.232 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:25:49.233 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:25:49.334 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:25:49.365 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:25:49.368 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:25:49.369 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:25:49.470 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:25:49.507 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:25:49.511 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:25:49.511 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:25:49.612 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:25:49.661 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:25:49.666 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:25:49.667 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:25:49.767 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:25:49.803 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:25:49.805 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:25:49.806 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:25:49.907 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:25:49.950 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:25:49.954 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:25:49.955 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:25:50.056 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:25:50.088 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:25:50.090 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:25:50.091 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:25:50.190 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:25:50 smithi164 bash[59462]: cluster 1685823949.0746844 mgr.y (mgr.34104) 1498 : cluster 0 pgmap v1530: 99 pgs: 6 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 2 active+recovery_wait, 7 active+recovery_wait+undersized+degraded+remapped, 6 active+undersized+degraded+remapped+backfill_wait, 76 active+clean; 2.1 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 651 op/s; 35958/105045 objects degraded (34.231%); 4229/105045 objects misplaced (4.026%); 6.2 MiB/s, 99 objects/s recovering 2023-06-03T20:25:50.192 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:25:50.223 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:25:50.231 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:25:50.232 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:25:50.333 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:25:50.343 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:25:50 smithi089 bash[75068]: cluster 1685823949.0746844 mgr.y (mgr.34104) 1498 : cluster 0 pgmap v1530: 99 pgs: 6 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 2 active+recovery_wait, 7 active+recovery_wait+undersized+degraded+remapped, 6 active+undersized+degraded+remapped+backfill_wait, 76 active+clean; 2.1 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 651 op/s; 35958/105045 objects degraded (34.231%); 4229/105045 objects misplaced (4.026%); 6.2 MiB/s, 99 objects/s recovering 2023-06-03T20:25:50.343 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:25:50 smithi089 bash[78452]: cluster 1685823949.0746844 mgr.y (mgr.34104) 1498 : cluster 0 pgmap v1530: 99 pgs: 6 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 2 active+recovery_wait, 7 active+recovery_wait+undersized+degraded+remapped, 6 active+undersized+degraded+remapped+backfill_wait, 76 active+clean; 2.1 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 651 op/s; 35958/105045 objects degraded (34.231%); 4229/105045 objects misplaced (4.026%); 6.2 MiB/s, 99 objects/s recovering 2023-06-03T20:25:50.366 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:25:50.368 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:25:50.369 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:25:50.470 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:25:50.505 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:25:50.511 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:25:50.511 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:25:50.612 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:25:50.644 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:25:50.656 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:25:50.657 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:25:50.757 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:25:50.792 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:25:50.794 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:25:50.795 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:25:50.896 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:25:50.938 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:25:50.943 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:25:50.944 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:25:51.045 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:25:51.073 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:25:51.076 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:25:51.077 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:25:51.178 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:25:51.206 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:25:51.208 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:25:51.209 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:25:51.309 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:25:51.356 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:25:51.370 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:25:51.371 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:25:51.472 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:25:51.500 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:25:51.503 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:25:51.504 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:25:51.604 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:25:51.632 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:25:51.634 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:25:51.634 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:25:51.736 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:25:51.777 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:25:51.784 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:25:51.784 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:25:51.885 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:25:51.920 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:25:51.930 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:25:51.930 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:25:52.032 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:25:52.087 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:25:52.089 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:25:52.090 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:25:52.143 INFO:teuthology.orchestra.run.smithi164.stderr:Inferring config /var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/mon.b/config 2023-06-03T20:25:52.192 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:25:52.222 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:25:52.228 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:25:52.229 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:25:52.329 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:25:52.335 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:25:52 smithi164 bash[59462]: cluster 1685823951.0754852 mgr.y (mgr.34104) 1499 : cluster 0 pgmap v1531: 99 pgs: 6 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 2 active+recovery_wait, 7 active+recovery_wait+undersized+degraded+remapped, 6 active+undersized+degraded+remapped+backfill_wait, 76 active+clean; 2.1 GiB data, 9.8 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 657 op/s; 35552/103209 objects degraded (34.447%); 4229/103209 objects misplaced (4.098%); 6.2 MiB/s, 99 objects/s recovering 2023-06-03T20:25:52.335 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:25:52 smithi164 bash[59462]: audit 1685823952.1505933 mon.a (mon.0) 1496 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:25:52.335 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:25:52 smithi164 bash[59462]: audit 1685823952.1519716 mon.c (mon.2) 627 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T20:25:52.365 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:25:52.368 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:25:52.369 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:25:52.470 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:25:52.477 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:25:52 smithi089 bash[78452]: cluster 1685823951.0754852 mgr.y (mgr.34104) 1499 : cluster 0 pgmap v1531: 99 pgs: 6 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 2 active+recovery_wait, 7 active+recovery_wait+undersized+degraded+remapped, 6 active+undersized+degraded+remapped+backfill_wait, 76 active+clean; 2.1 GiB data, 9.8 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 657 op/s; 35552/103209 objects degraded (34.447%); 4229/103209 objects misplaced (4.098%); 6.2 MiB/s, 99 objects/s recovering 2023-06-03T20:25:52.478 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:25:52 smithi089 bash[78452]: audit 1685823952.1505933 mon.a (mon.0) 1496 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:25:52.478 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:25:52 smithi089 bash[78452]: audit 1685823952.1519716 mon.c (mon.2) 627 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T20:25:52.478 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:25:52 smithi089 bash[75068]: cluster 1685823951.0754852 mgr.y (mgr.34104) 1499 : cluster 0 pgmap v1531: 99 pgs: 6 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 2 active+recovery_wait, 7 active+recovery_wait+undersized+degraded+remapped, 6 active+undersized+degraded+remapped+backfill_wait, 76 active+clean; 2.1 GiB data, 9.8 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 657 op/s; 35552/103209 objects degraded (34.447%); 4229/103209 objects misplaced (4.098%); 6.2 MiB/s, 99 objects/s recovering 2023-06-03T20:25:52.478 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:25:52 smithi089 bash[75068]: audit 1685823952.1505933 mon.a (mon.0) 1496 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:25:52.479 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:25:52 smithi089 bash[75068]: audit 1685823952.1519716 mon.c (mon.2) 627 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T20:25:52.498 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:25:52.500 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:25:52.501 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:25:52.602 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:25:52.655 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:25:52.658 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:25:52.659 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:25:52.761 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:25:52.802 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:25:52.805 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:25:52.806 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:25:52.835 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/mon.a/config 2023-06-03T20:25:52.907 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:25:52.955 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:25:52.957 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:25:52.959 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:25:53.060 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:25:53.091 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:25:53.093 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:25:53.094 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:25:53.194 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:25:53 smithi164 bash[59462]: cluster 1685823952.9781177 mon.a (mon.0) 1497 : cluster 3 Health check update: Degraded data redundancy: 35552/103209 objects degraded (34.447%), 20 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-06-03T20:25:53.194 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:25:53.223 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:25:53.227 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:25:53.227 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:25:53.328 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:25:53.361 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:25:53.362 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:25:53 smithi089 bash[78452]: cluster 1685823952.9781177 mon.a (mon.0) 1497 : cluster 3 Health check update: Degraded data redundancy: 35552/103209 objects degraded (34.447%), 20 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-06-03T20:25:53.362 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:25:53 smithi089 bash[75068]: cluster 1685823952.9781177 mon.a (mon.0) 1497 : cluster 3 Health check update: Degraded data redundancy: 35552/103209 objects degraded (34.447%), 20 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-06-03T20:25:53.366 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:25:53.367 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:25:53.468 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:25:53.506 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:25:53.512 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:25:53.513 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:25:53.614 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:25:53.665 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:25:53.674 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:25:53.675 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:25:53.776 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:25:53.808 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:25:53.811 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:25:53.813 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:25:53.914 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:25:53.949 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:25:53.954 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:25:53.955 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:25:54.056 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:25:54.101 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:25:54.106 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:25:54.106 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:25:54.196 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:25:54 smithi164 bash[59462]: cluster 1685823953.0761595 mgr.y (mgr.34104) 1500 : cluster 0 pgmap v1532: 99 pgs: 6 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait, 7 active+recovery_wait+undersized+degraded+remapped, 6 active+undersized+degraded+remapped+backfill_wait, 76 active+clean; 2.0 GiB data, 9.6 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 658 op/s; 34550/97182 objects degraded (35.552%); 4052/97182 objects misplaced (4.169%); 6.2 MiB/s, 99 objects/s recovering 2023-06-03T20:25:54.208 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:25:54.239 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:25:54.242 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:25:54.243 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:25:54.344 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:25:54.377 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:25:54.381 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:25:54.381 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:25:54.482 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:25:54.498 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:25:54 smithi089 bash[75068]: cluster 1685823953.0761595 mgr.y (mgr.34104) 1500 : cluster 0 pgmap v1532: 99 pgs: 6 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait, 7 active+recovery_wait+undersized+degraded+remapped, 6 active+undersized+degraded+remapped+backfill_wait, 76 active+clean; 2.0 GiB data, 9.6 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 658 op/s; 34550/97182 objects degraded (35.552%); 4052/97182 objects misplaced (4.169%); 6.2 MiB/s, 99 objects/s recovering 2023-06-03T20:25:54.499 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:25:54 smithi089 bash[78452]: cluster 1685823953.0761595 mgr.y (mgr.34104) 1500 : cluster 0 pgmap v1532: 99 pgs: 6 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait, 7 active+recovery_wait+undersized+degraded+remapped, 6 active+undersized+degraded+remapped+backfill_wait, 76 active+clean; 2.0 GiB data, 9.6 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 658 op/s; 34550/97182 objects degraded (35.552%); 4052/97182 objects misplaced (4.169%); 6.2 MiB/s, 99 objects/s recovering 2023-06-03T20:25:54.515 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:25:54.519 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:25:54.519 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:25:54.620 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:25:54.655 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:25:54.661 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:25:54.661 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:25:54.762 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:25:54.823 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:25:54.827 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:25:54.828 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:25:54.928 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:25:54.957 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:25:54.960 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:25:54.961 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:25:55.062 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:25:55.094 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:25:55.099 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:25:55.099 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:25:55.200 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:25:55.242 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:25:55.247 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:25:55.248 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:25:55.349 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:25:55.382 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:25:55.391 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:25:55.392 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:25:55.492 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:25:55.529 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:25:55.537 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:25:55.538 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:25:55.639 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:25:55.670 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:25:55.675 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:25:55.676 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:25:55.777 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:25:55.809 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:25:55.814 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:25:55.815 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:25:55.916 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:25:55.945 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:25:55.950 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:25:55.951 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:25:56.052 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:25:56.098 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:25:56.104 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:25:56.105 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:25:56.206 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:25:56.226 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:25:56 smithi089 bash[75068]: cluster 1685823955.077052 mgr.y (mgr.34104) 1501 : cluster 0 pgmap v1533: 99 pgs: 6 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 2 active+recovery_wait, 7 active+recovery_wait+undersized+degraded+remapped, 6 active+undersized+degraded+remapped+backfill_wait, 76 active+clean; 1.9 GiB data, 9.3 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 661 op/s; 33772/93339 objects degraded (36.182%); 3828/93339 objects misplaced (4.101%); 6.3 MiB/s, 100 objects/s recovering 2023-06-03T20:25:56.241 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:25:56.246 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:25:56.247 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:25:56.334 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:25:56 smithi164 bash[59462]: cluster 1685823955.077052 mgr.y (mgr.34104) 1501 : cluster 0 pgmap v1533: 99 pgs: 6 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 2 active+recovery_wait, 7 active+recovery_wait+undersized+degraded+remapped, 6 active+undersized+degraded+remapped+backfill_wait, 76 active+clean; 1.9 GiB data, 9.3 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 661 op/s; 33772/93339 objects degraded (36.182%); 3828/93339 objects misplaced (4.101%); 6.3 MiB/s, 100 objects/s recovering 2023-06-03T20:25:56.348 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:25:56.377 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:25:56.379 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:25:56.380 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:25:56.481 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:25:56.524 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:25:56.529 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 20:25:56 smithi089 bash[71163]: ::ffff:172.21.15.164 - - [03/Jun/2023:20:25:56] "GET /metrics HTTP/1.1" 200 229016 "" "Prometheus/2.33.4" 2023-06-03T20:25:56.529 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:25:56 smithi089 bash[78452]: cluster 1685823955.077052 mgr.y (mgr.34104) 1501 : cluster 0 pgmap v1533: 99 pgs: 6 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 2 active+recovery_wait, 7 active+recovery_wait+undersized+degraded+remapped, 6 active+undersized+degraded+remapped+backfill_wait, 76 active+clean; 1.9 GiB data, 9.3 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 661 op/s; 33772/93339 objects degraded (36.182%); 3828/93339 objects misplaced (4.101%); 6.3 MiB/s, 100 objects/s recovering 2023-06-03T20:25:56.529 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:25:56.530 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:25:56.631 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:25:56.662 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:25:56.664 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:25:56.665 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:25:56.766 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:25:56.804 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:25:56.811 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:25:56.812 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:25:56.912 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:25:56.943 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:25:56.945 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:25:56.946 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:25:57.047 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:25:57.101 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:25:57.107 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:25:57.107 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:25:57.208 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:25:57.245 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:25:57.249 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:25:57.249 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:25:57.350 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:25:57.387 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:25:57.389 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:25:57.389 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:25:57.491 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:25:57.524 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:25:57.527 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:25:57.528 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:25:57.629 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:25:57.672 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:25:57.674 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:25:57.675 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:25:57.775 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:25:57.818 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:25:57.823 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:25:57.824 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:25:57.925 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:25:57.960 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:25:57.962 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:25:57.963 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:25:58.064 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:25:58.097 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:25:58.099 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:25:58.100 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:25:58.201 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:25:58.234 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:25:58.238 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:25:58.238 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:25:58.339 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:25:58.422 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:25:58.425 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:25:58.426 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:25:58.493 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:25:58 smithi164 bash[59462]: cluster 1685823957.0777817 mgr.y (mgr.34104) 1502 : cluster 0 pgmap v1534: 99 pgs: 6 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait, 7 active+recovery_wait+undersized+degraded+remapped, 6 active+undersized+degraded+remapped+backfill_wait, 76 active+clean; 1.8 GiB data, 9.2 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 640 op/s; 33358/90282 objects degraded (36.949%); 3648/90282 objects misplaced (4.041%); 7.4 MiB/s, 119 objects/s recovering 2023-06-03T20:25:58.493 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:25:58 smithi164 bash[59462]: cluster 1685823957.9792876 mon.a (mon.0) 1498 : cluster 3 Health check update: Degraded data redundancy: 33358/90282 objects degraded (36.949%), 20 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-06-03T20:25:58.527 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:25:58.529 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:25:58 smithi089 bash[75068]: cluster 1685823957.0777817 mgr.y (mgr.34104) 1502 : cluster 0 pgmap v1534: 99 pgs: 6 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait, 7 active+recovery_wait+undersized+degraded+remapped, 6 active+undersized+degraded+remapped+backfill_wait, 76 active+clean; 1.8 GiB data, 9.2 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 640 op/s; 33358/90282 objects degraded (36.949%); 3648/90282 objects misplaced (4.041%); 7.4 MiB/s, 119 objects/s recovering 2023-06-03T20:25:58.529 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:25:58 smithi089 bash[75068]: cluster 1685823957.9792876 mon.a (mon.0) 1498 : cluster 3 Health check update: Degraded data redundancy: 33358/90282 objects degraded (36.949%), 20 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-06-03T20:25:58.530 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:25:58 smithi089 bash[78452]: cluster 1685823957.0777817 mgr.y (mgr.34104) 1502 : cluster 0 pgmap v1534: 99 pgs: 6 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait, 7 active+recovery_wait+undersized+degraded+remapped, 6 active+undersized+degraded+remapped+backfill_wait, 76 active+clean; 1.8 GiB data, 9.2 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 640 op/s; 33358/90282 objects degraded (36.949%); 3648/90282 objects misplaced (4.041%); 7.4 MiB/s, 119 objects/s recovering 2023-06-03T20:25:58.530 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:25:58 smithi089 bash[78452]: cluster 1685823957.9792876 mon.a (mon.0) 1498 : cluster 3 Health check update: Degraded data redundancy: 33358/90282 objects degraded (36.949%), 20 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-06-03T20:25:58.559 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:25:58.563 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:25:58.564 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:25:58.664 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:25:58.701 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:25:58.703 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:25:58.704 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:25:58.805 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:25:58.813 INFO:teuthology.orchestra.run.smithi164.stdout:ERROR: (22) Invalid argument 2023-06-03T20:25:58.814 INFO:teuthology.orchestra.run.smithi164.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-06-03T20:25:58.842 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:25:58.847 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:25:58.848 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:25:58.949 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:25:58.981 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:25:58.991 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:25:58.991 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:25:59.092 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:25:59.134 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:25:59.137 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:25:59.138 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:25:59.239 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:25:59.270 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:25:59.273 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:25:59.274 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:25:59.375 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:25:59.405 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:25:59.412 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:25:59.413 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:25:59.514 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:25:59.571 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:25:59.575 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:25:59.576 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:25:59.677 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:25:59.720 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:25:59.726 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:25:59.727 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:25:59.828 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:25:59.850 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:25:59.851 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:25:59.852 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:25:59.953 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:26:00.000 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:26:00.002 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:26:00.003 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:26:00.104 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:26:00.132 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:26:00.139 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:26:00.140 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:26:00.173 DEBUG:teuthology.orchestra.run.smithi164:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph daemon osd.5 dump_historic_ops 2023-06-03T20:26:00.241 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:26:00.277 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:26:00.279 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:26:00.280 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:26:00.334 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:26:00 smithi164 bash[59462]: cluster 1685823959.0787468 mgr.y (mgr.34104) 1503 : cluster 0 pgmap v1535: 99 pgs: 6 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 2 active+recovery_wait, 7 active+recovery_wait+undersized+degraded+remapped, 6 active+undersized+degraded+remapped+backfill_wait, 76 active+clean; 1.7 GiB data, 9.1 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 683 op/s; 32488/85455 objects degraded (38.018%); 3478/85455 objects misplaced (4.070%); 5.8 MiB/s, 93 objects/s recovering 2023-06-03T20:26:00.382 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:26:00.389 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:26:00 smithi089 bash[75068]: cluster 1685823959.0787468 mgr.y (mgr.34104) 1503 : cluster 0 pgmap v1535: 99 pgs: 6 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 2 active+recovery_wait, 7 active+recovery_wait+undersized+degraded+remapped, 6 active+undersized+degraded+remapped+backfill_wait, 76 active+clean; 1.7 GiB data, 9.1 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 683 op/s; 32488/85455 objects degraded (38.018%); 3478/85455 objects misplaced (4.070%); 5.8 MiB/s, 93 objects/s recovering 2023-06-03T20:26:00.390 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:26:00 smithi089 bash[78452]: cluster 1685823959.0787468 mgr.y (mgr.34104) 1503 : cluster 0 pgmap v1535: 99 pgs: 6 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 2 active+recovery_wait, 7 active+recovery_wait+undersized+degraded+remapped, 6 active+undersized+degraded+remapped+backfill_wait, 76 active+clean; 1.7 GiB data, 9.1 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 683 op/s; 32488/85455 objects degraded (38.018%); 3478/85455 objects misplaced (4.070%); 5.8 MiB/s, 93 objects/s recovering 2023-06-03T20:26:00.429 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:26:00.430 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:26:00.431 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:26:00.526 INFO:teuthology.orchestra.run.smithi089.stdout:osd.0: {} 2023-06-03T20:26:00.532 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:26:00.562 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:26:00.566 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:26:00.567 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:26:00.580 INFO:teuthology.orchestra.run.smithi089.stdout:osd.1: {} 2023-06-03T20:26:00.623 INFO:teuthology.orchestra.run.smithi089.stdout:osd.2: {} 2023-06-03T20:26:00.659 INFO:teuthology.orchestra.run.smithi089.stdout:osd.3: {} 2023-06-03T20:26:00.668 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:26:00.700 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:26:00.704 INFO:teuthology.orchestra.run.smithi089.stdout:osd.4: {} 2023-06-03T20:26:00.707 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:26:00.708 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:26:00.758 INFO:teuthology.orchestra.run.smithi089.stdout:osd.5: {} 2023-06-03T20:26:00.787 INFO:teuthology.orchestra.run.smithi089.stdout:osd.6: {} 2023-06-03T20:26:00.809 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:26:00.840 INFO:teuthology.orchestra.run.smithi089.stdout:osd.7: {} 2023-06-03T20:26:00.845 INFO:teuthology.orchestra.run.smithi089.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-06-03T20:26:00.845 INFO:teuthology.orchestra.run.smithi089.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-06-03T20:26:00.845 INFO:teuthology.orchestra.run.smithi089.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-06-03T20:26:00.845 INFO:teuthology.orchestra.run.smithi089.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-06-03T20:26:00.846 INFO:teuthology.orchestra.run.smithi089.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-06-03T20:26:00.846 INFO:teuthology.orchestra.run.smithi089.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-06-03T20:26:00.846 INFO:teuthology.orchestra.run.smithi089.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-06-03T20:26:00.846 INFO:teuthology.orchestra.run.smithi089.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-06-03T20:26:00.852 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:26:00.857 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:26:00.858 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:26:00.959 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:26:00.995 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:26:00.997 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:26:00.998 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:26:01.099 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:26:01.130 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:26:01.134 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:26:01.135 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:26:01.236 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:26:01.267 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:26:01.270 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:26:01.271 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:26:01.334 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:26:01 smithi164 bash[59462]: audit 1685823960.4887326 mon.a (mon.0) 1499 : audit 0 from='client.? 172.21.15.89:0/1865222766' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-03T20:26:01.372 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:26:01.416 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:26:01.421 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:26:01.421 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:26:01.522 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:26:01.528 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:26:01 smithi089 bash[75068]: audit 1685823960.4887326 mon.a (mon.0) 1499 : audit 0 from='client.? 172.21.15.89:0/1865222766' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-03T20:26:01.529 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:26:01 smithi089 bash[78452]: audit 1685823960.4887326 mon.a (mon.0) 1499 : audit 0 from='client.? 172.21.15.89:0/1865222766' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-03T20:26:01.561 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:26:01.564 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:26:01.565 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:26:01.666 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:26:01.697 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:26:01.700 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:26:01.701 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:26:01.802 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:26:01.846 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:26:01.854 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:26:01.856 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:26:01.958 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:26:01.997 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:26:02.000 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:26:02.000 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:26:02.101 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:26:02.156 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:26:02.161 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:26:02.162 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:26:02.219 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:26:02 smithi089 bash[75068]: cluster 1685823961.0795481 mgr.y (mgr.34104) 1504 : cluster 0 pgmap v1536: 99 pgs: 6 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 2 active+recovery_wait, 7 active+recovery_wait+undersized+degraded+remapped, 6 active+undersized+degraded+remapped+backfill_wait, 76 active+clean; 1.7 GiB data, 8.9 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 589 op/s; 32121/83826 objects degraded (38.319%); 3478/83826 objects misplaced (4.149%); 5.3 MiB/s, 85 objects/s recovering 2023-06-03T20:26:02.219 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:26:02 smithi089 bash[75068]: audit 1685823961.2890677 mon.b (mon.1) 64 : audit 1 from='client.? 172.21.15.89:0/3711430334' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-06-03T20:26:02.219 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:26:02 smithi089 bash[75068]: audit 1685823961.2923577 mon.a (mon.0) 1500 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-06-03T20:26:02.220 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:26:02 smithi089 bash[78452]: cluster 1685823961.0795481 mgr.y (mgr.34104) 1504 : cluster 0 pgmap v1536: 99 pgs: 6 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 2 active+recovery_wait, 7 active+recovery_wait+undersized+degraded+remapped, 6 active+undersized+degraded+remapped+backfill_wait, 76 active+clean; 1.7 GiB data, 8.9 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 589 op/s; 32121/83826 objects degraded (38.319%); 3478/83826 objects misplaced (4.149%); 5.3 MiB/s, 85 objects/s recovering 2023-06-03T20:26:02.220 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:26:02 smithi089 bash[78452]: audit 1685823961.2890677 mon.b (mon.1) 64 : audit 1 from='client.? 172.21.15.89:0/3711430334' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-06-03T20:26:02.220 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:26:02 smithi089 bash[78452]: audit 1685823961.2923577 mon.a (mon.0) 1500 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-06-03T20:26:02.239 INFO:teuthology.orchestra.run.smithi089.stderr:nodeep-scrub is set 2023-06-03T20:26:02.263 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:26:02.320 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:26:02.324 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:26:02.325 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:26:02.426 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:26:02.435 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:26:02 smithi164 bash[59462]: cluster 1685823961.0795481 mgr.y (mgr.34104) 1504 : cluster 0 pgmap v1536: 99 pgs: 6 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 2 active+recovery_wait, 7 active+recovery_wait+undersized+degraded+remapped, 6 active+undersized+degraded+remapped+backfill_wait, 76 active+clean; 1.7 GiB data, 8.9 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 589 op/s; 32121/83826 objects degraded (38.319%); 3478/83826 objects misplaced (4.149%); 5.3 MiB/s, 85 objects/s recovering 2023-06-03T20:26:02.435 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:26:02 smithi164 bash[59462]: audit 1685823961.2890677 mon.b (mon.1) 64 : audit 1 from='client.? 172.21.15.89:0/3711430334' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-06-03T20:26:02.435 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:26:02 smithi164 bash[59462]: audit 1685823961.2923577 mon.a (mon.0) 1500 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-06-03T20:26:02.458 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:26:02.461 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:26:02.462 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:26:02.563 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:26:02.599 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:26:02.602 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:26:02.603 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:26:02.704 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:26:02.731 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:26:02.734 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:26:02.735 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:26:02.836 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:26:02.872 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:26:02.873 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:26:02.874 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:26:02.974 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:26:03.003 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:26:03.005 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:26:03.006 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:26:03.107 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:26:03.154 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:26:03.160 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:26:03.161 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:26:03.262 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:26:03.315 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:26:03.318 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:26:03.319 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:26:03.420 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:26:03.431 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:26:03 smithi089 bash[78452]: cluster 1685823962.209882 mon.a (mon.0) 1501 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-06-03T20:26:03.431 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:26:03 smithi089 bash[78452]: audit 1685823962.2219124 mon.a (mon.0) 1502 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-06-03T20:26:03.431 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:26:03 smithi089 bash[78452]: cluster 1685823962.2219956 mon.a (mon.0) 1503 : cluster 0 osdmap e267: 8 total, 8 up, 8 in 2023-06-03T20:26:03.431 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:26:03 smithi089 bash[78452]: cluster 1685823962.225772 osd.5 (osd.5) 216 : cluster 3 failed to encode map e267 with expected crc 2023-06-03T20:26:03.431 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:26:03 smithi089 bash[78452]: cluster 1685823962.226348 osd.6 (osd.6) 204 : cluster 3 failed to encode map e267 with expected crc 2023-06-03T20:26:03.431 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:26:03 smithi089 bash[78452]: cluster 1685823962.2277958 osd.7 (osd.7) 215 : cluster 3 failed to encode map e267 with expected crc 2023-06-03T20:26:03.432 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:26:03 smithi089 bash[78452]: cluster 1685823962.228143 osd.5 (osd.5) 217 : cluster 3 failed to encode map e267 with expected crc 2023-06-03T20:26:03.432 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:26:03 smithi089 bash[78452]: cluster 1685823962.2307265 osd.7 (osd.7) 216 : cluster 3 failed to encode map e267 with expected crc 2023-06-03T20:26:03.432 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:26:03 smithi089 bash[78452]: cluster 1685823962.9803467 mon.a (mon.0) 1504 : cluster 3 Health check update: Degraded data redundancy: 32121/83826 objects degraded (38.319%), 20 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-06-03T20:26:03.432 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:26:03 smithi089 bash[75068]: cluster 1685823962.209882 mon.a (mon.0) 1501 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-06-03T20:26:03.432 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:26:03 smithi089 bash[75068]: audit 1685823962.2219124 mon.a (mon.0) 1502 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-06-03T20:26:03.432 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:26:03 smithi089 bash[75068]: cluster 1685823962.2219956 mon.a (mon.0) 1503 : cluster 0 osdmap e267: 8 total, 8 up, 8 in 2023-06-03T20:26:03.433 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:26:03 smithi089 bash[75068]: cluster 1685823962.225772 osd.5 (osd.5) 216 : cluster 3 failed to encode map e267 with expected crc 2023-06-03T20:26:03.433 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:26:03 smithi089 bash[75068]: cluster 1685823962.226348 osd.6 (osd.6) 204 : cluster 3 failed to encode map e267 with expected crc 2023-06-03T20:26:03.433 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:26:03 smithi089 bash[75068]: cluster 1685823962.2277958 osd.7 (osd.7) 215 : cluster 3 failed to encode map e267 with expected crc 2023-06-03T20:26:03.433 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:26:03 smithi089 bash[75068]: cluster 1685823962.228143 osd.5 (osd.5) 217 : cluster 3 failed to encode map e267 with expected crc 2023-06-03T20:26:03.433 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:26:03 smithi089 bash[75068]: cluster 1685823962.2307265 osd.7 (osd.7) 216 : cluster 3 failed to encode map e267 with expected crc 2023-06-03T20:26:03.433 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:26:03 smithi089 bash[75068]: cluster 1685823962.9803467 mon.a (mon.0) 1504 : cluster 3 Health check update: Degraded data redundancy: 32121/83826 objects degraded (38.319%), 20 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-06-03T20:26:03.453 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:26:03.459 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:26:03.460 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:26:03.561 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:26:03.570 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:26:03 smithi164 bash[59462]: cluster 1685823962.209882 mon.a (mon.0) 1501 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-06-03T20:26:03.570 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:26:03 smithi164 bash[59462]: audit 1685823962.2219124 mon.a (mon.0) 1502 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-06-03T20:26:03.570 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:26:03 smithi164 bash[59462]: cluster 1685823962.2219956 mon.a (mon.0) 1503 : cluster 0 osdmap e267: 8 total, 8 up, 8 in 2023-06-03T20:26:03.571 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:26:03 smithi164 bash[59462]: cluster 1685823962.225772 osd.5 (osd.5) 216 : cluster 3 failed to encode map e267 with expected crc 2023-06-03T20:26:03.571 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:26:03 smithi164 bash[59462]: cluster 1685823962.226348 osd.6 (osd.6) 204 : cluster 3 failed to encode map e267 with expected crc 2023-06-03T20:26:03.571 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:26:03 smithi164 bash[59462]: cluster 1685823962.2277958 osd.7 (osd.7) 215 : cluster 3 failed to encode map e267 with expected crc 2023-06-03T20:26:03.571 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:26:03 smithi164 bash[59462]: cluster 1685823962.228143 osd.5 (osd.5) 217 : cluster 3 failed to encode map e267 with expected crc 2023-06-03T20:26:03.571 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:26:03 smithi164 bash[59462]: cluster 1685823962.2307265 osd.7 (osd.7) 216 : cluster 3 failed to encode map e267 with expected crc 2023-06-03T20:26:03.571 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:26:03 smithi164 bash[59462]: cluster 1685823962.9803467 mon.a (mon.0) 1504 : cluster 3 Health check update: Degraded data redundancy: 32121/83826 objects degraded (38.319%), 20 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-06-03T20:26:03.612 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:26:03.615 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:26:03.616 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:26:03.717 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:26:03.744 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:26:03.745 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:26:03.746 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:26:03.848 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:26:03.885 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:26:03.891 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:26:03.891 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:26:03.992 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:26:04.025 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:26:04.027 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:26:04.028 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:26:04.129 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:26:04.160 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:26:04.166 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:26:04.167 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:26:04.222 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:26:04 smithi164 bash[59462]: cluster 1685823963.0802813 mgr.y (mgr.34104) 1505 : cluster 0 pgmap v1538: 99 pgs: 6 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait, 7 active+recovery_wait+undersized+degraded+remapped, 6 active+undersized+degraded+remapped+backfill_wait, 76 active+clean; 1.6 GiB data, 8.7 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 647 op/s; 31163/77757 objects degraded (40.077%); 3289/77757 objects misplaced (4.230%); 6.2 MiB/s, 99 objects/s recovering 2023-06-03T20:26:04.267 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:26:04.279 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:26:04 smithi089 bash[75068]: cluster 1685823963.0802813 mgr.y (mgr.34104) 1505 : cluster 0 pgmap v1538: 99 pgs: 6 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait, 7 active+recovery_wait+undersized+degraded+remapped, 6 active+undersized+degraded+remapped+backfill_wait, 76 active+clean; 1.6 GiB data, 8.7 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 647 op/s; 31163/77757 objects degraded (40.077%); 3289/77757 objects misplaced (4.230%); 6.2 MiB/s, 99 objects/s recovering 2023-06-03T20:26:04.279 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:26:04 smithi089 bash[78452]: cluster 1685823963.0802813 mgr.y (mgr.34104) 1505 : cluster 0 pgmap v1538: 99 pgs: 6 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait, 7 active+recovery_wait+undersized+degraded+remapped, 6 active+undersized+degraded+remapped+backfill_wait, 76 active+clean; 1.6 GiB data, 8.7 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 647 op/s; 31163/77757 objects degraded (40.077%); 3289/77757 objects misplaced (4.230%); 6.2 MiB/s, 99 objects/s recovering 2023-06-03T20:26:04.305 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:26:04.312 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:26:04.313 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:26:04.414 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:26:04.455 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:26:04.459 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:26:04.460 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:26:04.561 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:26:04.588 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:26:04.591 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:26:04.592 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:26:04.693 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:26:04.739 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:26:04.742 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:26:04.743 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:26:04.843 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:26:04.895 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:26:04.900 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:26:04.901 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:26:05.002 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:26:05.037 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:26:05.041 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:26:05.042 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:26:05.074 INFO:teuthology.orchestra.run.smithi164.stderr:Inferring config /var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/mon.b/config 2023-06-03T20:26:05.144 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:26:05.176 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:26:05.179 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:26:05.180 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:26:05.281 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:26:05.314 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:26:05.319 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:26:05.319 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:26:05.420 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:26:05.452 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:26:05.455 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:26:05.456 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:26:05.557 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:26:05.601 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:26:05.604 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:26:05.604 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:26:05.705 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:26:05.742 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:26:05.747 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:26:05.748 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:26:05.849 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:26:05.893 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:26:05.897 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:26:05.898 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:26:05.999 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:26:06.039 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:26:06.046 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:26:06.047 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:26:06.063 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-06-03T20:26:06.149 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:26:06.185 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:26:06.188 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:26:06.189 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:26:06.226 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:26:06 smithi164 bash[59462]: cluster 1685823965.0809925 mgr.y (mgr.34104) 1506 : cluster 0 pgmap v1539: 99 pgs: 6 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 2 active+recovery_wait, 7 active+recovery_wait+undersized+degraded+remapped, 6 active+undersized+degraded+remapped+backfill_wait, 76 active+clean; 1.5 GiB data, 8.4 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 634 op/s; 30483/74295 objects degraded (41.030%); 3094/74295 objects misplaced (4.164%); 6.3 MiB/s, 100 objects/s recovering 2023-06-03T20:26:06.279 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:26:06 smithi089 bash[75068]: cluster 1685823965.0809925 mgr.y (mgr.34104) 1506 : cluster 0 pgmap v1539: 99 pgs: 6 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 2 active+recovery_wait, 7 active+recovery_wait+undersized+degraded+remapped, 6 active+undersized+degraded+remapped+backfill_wait, 76 active+clean; 1.5 GiB data, 8.4 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 634 op/s; 30483/74295 objects degraded (41.030%); 3094/74295 objects misplaced (4.164%); 6.3 MiB/s, 100 objects/s recovering 2023-06-03T20:26:06.280 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 20:26:06 smithi089 bash[71163]: ::ffff:172.21.15.164 - - [03/Jun/2023:20:26:06] "GET /metrics HTTP/1.1" 200 229016 "" "Prometheus/2.33.4" 2023-06-03T20:26:06.280 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:26:06 smithi089 bash[78452]: cluster 1685823965.0809925 mgr.y (mgr.34104) 1506 : cluster 0 pgmap v1539: 99 pgs: 6 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 2 active+recovery_wait, 7 active+recovery_wait+undersized+degraded+remapped, 6 active+undersized+degraded+remapped+backfill_wait, 76 active+clean; 1.5 GiB data, 8.4 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 634 op/s; 30483/74295 objects degraded (41.030%); 3094/74295 objects misplaced (4.164%); 6.3 MiB/s, 100 objects/s recovering 2023-06-03T20:26:06.290 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:26:06.335 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:26:06.339 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:26:06.339 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:26:06.440 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:26:06.470 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:26:06.471 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:26:06.472 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:26:06.573 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:26:06.612 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:26:06.617 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:26:06.618 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:26:06.719 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:26:06.757 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:26:06.765 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:26:06.767 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:26:06.867 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:26:06.902 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:26:06.903 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:26:06.904 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:26:07.005 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:26:07.044 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:26:07.047 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:26:07.048 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:26:07.149 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:26:07.185 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:26:07.187 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:26:07.188 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:26:07.242 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:26:07 smithi089 bash[78452]: audit 1685823967.1464775 mon.c (mon.2) 628 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T20:26:07.243 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:26:07 smithi089 bash[75068]: audit 1685823967.1464775 mon.c (mon.2) 628 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T20:26:07.289 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:26:07.298 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:26:07 smithi164 bash[59462]: audit 1685823967.1464775 mon.c (mon.2) 628 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T20:26:07.335 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:26:07.338 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:26:07.339 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:26:07.439 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:26:07.475 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:26:07.477 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:26:07.477 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:26:07.578 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:26:07.611 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:26:07.614 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:26:07.615 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:26:07.716 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:26:07.749 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:26:07.754 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:26:07.754 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:26:07.788 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph osd unset noscrub 2023-06-03T20:26:07.856 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:26:07.895 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:26:07.898 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:26:07.899 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:26:08.000 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:26:08.033 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:26:08.040 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:26:08.041 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:26:08.142 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:26:08.174 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:26:08.177 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:26:08.178 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:26:08.266 INFO:teuthology.orchestra.run.smithi089.stderr:reweighted osd.2 to 1 (10000) 2023-06-03T20:26:08.269 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:26:08 smithi164 bash[59462]: cluster 1685823967.081801 mgr.y (mgr.34104) 1507 : cluster 0 pgmap v1540: 99 pgs: 6 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait, 7 active+recovery_wait+undersized+degraded+remapped, 6 active+undersized+degraded+remapped+backfill_wait, 76 active+clean; 1.4 GiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 634 op/s; 30076/71238 objects degraded (42.219%); 2887/71238 objects misplaced (4.053%); 6.2 MiB/s, 99 objects/s recovering 2023-06-03T20:26:08.270 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:26:08 smithi164 bash[59462]: audit 1685823967.5145636 mon.a (mon.0) 1505 : audit 1 from='client.? 172.21.15.89:0/1275576350' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-06-03T20:26:08.270 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:26:08 smithi164 bash[59462]: cluster 1685823967.9812894 mon.a (mon.0) 1506 : cluster 3 Health check update: Degraded data redundancy: 30076/71238 objects degraded (42.219%), 20 pgs degraded, 14 pgs undersized (PG_DEGRADED) 2023-06-03T20:26:08.279 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:26:08.316 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:26:08.325 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:26:08.326 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:26:08.428 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:26:08.475 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:26:08.479 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:26:08.479 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:26:08.500 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:26:08 smithi089 bash[78452]: cluster 1685823967.081801 mgr.y (mgr.34104) 1507 : cluster 0 pgmap v1540: 99 pgs: 6 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait, 7 active+recovery_wait+undersized+degraded+remapped, 6 active+undersized+degraded+remapped+backfill_wait, 76 active+clean; 1.4 GiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 634 op/s; 30076/71238 objects degraded (42.219%); 2887/71238 objects misplaced (4.053%); 6.2 MiB/s, 99 objects/s recovering 2023-06-03T20:26:08.501 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:26:08 smithi089 bash[78452]: audit 1685823967.5145636 mon.a (mon.0) 1505 : audit 1 from='client.? 172.21.15.89:0/1275576350' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-06-03T20:26:08.501 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:26:08 smithi089 bash[78452]: cluster 1685823967.9812894 mon.a (mon.0) 1506 : cluster 3 Health check update: Degraded data redundancy: 30076/71238 objects degraded (42.219%), 20 pgs degraded, 14 pgs undersized (PG_DEGRADED) 2023-06-03T20:26:08.502 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:26:08 smithi089 bash[75068]: cluster 1685823967.081801 mgr.y (mgr.34104) 1507 : cluster 0 pgmap v1540: 99 pgs: 6 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait, 7 active+recovery_wait+undersized+degraded+remapped, 6 active+undersized+degraded+remapped+backfill_wait, 76 active+clean; 1.4 GiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 634 op/s; 30076/71238 objects degraded (42.219%); 2887/71238 objects misplaced (4.053%); 6.2 MiB/s, 99 objects/s recovering 2023-06-03T20:26:08.502 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:26:08 smithi089 bash[75068]: audit 1685823967.5145636 mon.a (mon.0) 1505 : audit 1 from='client.? 172.21.15.89:0/1275576350' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-06-03T20:26:08.502 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:26:08 smithi089 bash[75068]: cluster 1685823967.9812894 mon.a (mon.0) 1506 : cluster 3 Health check update: Degraded data redundancy: 30076/71238 objects degraded (42.219%), 20 pgs degraded, 14 pgs undersized (PG_DEGRADED) 2023-06-03T20:26:08.581 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:26:08.620 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:26:08.625 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:26:08.625 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:26:08.726 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:26:08.761 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:26:08.762 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:26:08.763 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:26:08.864 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:26:08.913 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:26:08.914 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:26:08.915 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:26:09.016 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:26:09.051 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:26:09.054 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:26:09.054 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:26:09.155 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:26:09.190 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:26:09.195 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:26:09.196 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:26:09.296 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:26:09.305 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:26:09 smithi164 bash[59462]: cluster 1685823968.2404883 osd.5 (osd.5) 218 : cluster 3 failed to encode map e268 with expected crc 2023-06-03T20:26:09.306 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:26:09 smithi164 bash[59462]: cluster 1685823968.2412643 osd.6 (osd.6) 205 : cluster 3 failed to encode map e268 with expected crc 2023-06-03T20:26:09.306 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:26:09 smithi164 bash[59462]: audit 1685823968.2436035 mon.a (mon.0) 1507 : audit 1 from='client.? 172.21.15.89:0/1275576350' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2023-06-03T20:26:09.306 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:26:09 smithi164 bash[59462]: cluster 1685823968.2437046 mon.a (mon.0) 1508 : cluster 0 osdmap e268: 8 total, 8 up, 8 in 2023-06-03T20:26:09.327 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:26:09.332 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:26:09.333 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:26:09.434 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:26:09.442 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:26:09 smithi089 bash[78452]: cluster 1685823968.2404883 osd.5 (osd.5) 218 : cluster 3 failed to encode map e268 with expected crc 2023-06-03T20:26:09.442 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:26:09 smithi089 bash[78452]: cluster 1685823968.2412643 osd.6 (osd.6) 205 : cluster 3 failed to encode map e268 with expected crc 2023-06-03T20:26:09.443 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:26:09 smithi089 bash[78452]: audit 1685823968.2436035 mon.a (mon.0) 1507 : audit 1 from='client.? 172.21.15.89:0/1275576350' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2023-06-03T20:26:09.443 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:26:09 smithi089 bash[78452]: cluster 1685823968.2437046 mon.a (mon.0) 1508 : cluster 0 osdmap e268: 8 total, 8 up, 8 in 2023-06-03T20:26:09.443 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:26:09 smithi089 bash[75068]: cluster 1685823968.2404883 osd.5 (osd.5) 218 : cluster 3 failed to encode map e268 with expected crc 2023-06-03T20:26:09.443 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:26:09 smithi089 bash[75068]: cluster 1685823968.2412643 osd.6 (osd.6) 205 : cluster 3 failed to encode map e268 with expected crc 2023-06-03T20:26:09.443 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:26:09 smithi089 bash[75068]: audit 1685823968.2436035 mon.a (mon.0) 1507 : audit 1 from='client.? 172.21.15.89:0/1275576350' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2023-06-03T20:26:09.443 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:26:09 smithi089 bash[75068]: cluster 1685823968.2437046 mon.a (mon.0) 1508 : cluster 0 osdmap e268: 8 total, 8 up, 8 in 2023-06-03T20:26:09.468 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:26:09.474 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:26:09.475 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:26:09.576 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:26:09.603 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:26:09.605 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:26:09.606 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:26:09.707 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:26:09.744 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:26:09.748 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:26:09.749 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:26:09.850 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:26:09.878 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:26:09.885 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:26:09.886 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:26:09.987 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:26:10.019 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:26:10.027 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:26:10.028 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:26:10.129 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:26:10.163 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:26:10.166 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:26:10.168 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:26:10.171 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph osd reweight 3 1 2023-06-03T20:26:10.259 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:26:10 smithi089 bash[75068]: cluster 1685823969.082828 mgr.y (mgr.34104) 1508 : cluster 0 pgmap v1542: 99 pgs: 6 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 2 active+recovery_wait, 7 active+recovery_wait+undersized+degraded+remapped, 6 active+undersized+degraded+remapped+backfill_wait, 76 active+clean; 1.3 GiB data, 8.1 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 730 op/s; 29127/66288 objects degraded (43.940%); 2662/66288 objects misplaced (4.016%); 7.8 MiB/s, 124 objects/s recovering 2023-06-03T20:26:10.269 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:26:10.301 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:26:10.304 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:26:10.305 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:26:10.405 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:26:10.414 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:26:10 smithi164 bash[59462]: cluster 1685823969.082828 mgr.y (mgr.34104) 1508 : cluster 0 pgmap v1542: 99 pgs: 6 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 2 active+recovery_wait, 7 active+recovery_wait+undersized+degraded+remapped, 6 active+undersized+degraded+remapped+backfill_wait, 76 active+clean; 1.3 GiB data, 8.1 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 730 op/s; 29127/66288 objects degraded (43.940%); 2662/66288 objects misplaced (4.016%); 7.8 MiB/s, 124 objects/s recovering 2023-06-03T20:26:10.448 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:26:10.451 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:26:10.452 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:26:10.529 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:26:10 smithi089 bash[78452]: cluster 1685823969.082828 mgr.y (mgr.34104) 1508 : cluster 0 pgmap v1542: 99 pgs: 6 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 2 active+recovery_wait, 7 active+recovery_wait+undersized+degraded+remapped, 6 active+undersized+degraded+remapped+backfill_wait, 76 active+clean; 1.3 GiB data, 8.1 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 730 op/s; 29127/66288 objects degraded (43.940%); 2662/66288 objects misplaced (4.016%); 7.8 MiB/s, 124 objects/s recovering 2023-06-03T20:26:10.553 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:26:10.591 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:26:10.597 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:26:10.597 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:26:10.699 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:26:10.740 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:26:10.745 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:26:10.745 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:26:10.846 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:26:10.881 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:26:10.887 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:26:10.887 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:26:10.988 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:26:11.029 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:26:11.032 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:26:11.033 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:26:11.134 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:26:11.175 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:26:11.181 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:26:11.182 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:26:11.282 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:26:11.311 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:26:11.316 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:26:11.317 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:26:11.418 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:26:11.446 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:26:11.448 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:26:11.449 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:26:11.544 INFO:teuthology.orchestra.run.smithi164.stdout:{ 2023-06-03T20:26:11.545 INFO:teuthology.orchestra.run.smithi164.stdout: "size": 20, 2023-06-03T20:26:11.545 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 600, 2023-06-03T20:26:11.545 INFO:teuthology.orchestra.run.smithi164.stdout: "ops": [ 2023-06-03T20:26:11.545 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:26:11.546 INFO:teuthology.orchestra.run.smithi164.stdout: "description": "osd_repop(client.36573.0:120684 13.c e252/242 13:318461c5:::benchmark_data_smithi089_211581_object5159:head v 252'7681, mlcod=252'7676)", 2023-06-03T20:26:11.546 INFO:teuthology.orchestra.run.smithi164.stdout: "initiated_at": "2023-06-03T20:23:34.383131+0000", 2023-06-03T20:26:11.546 INFO:teuthology.orchestra.run.smithi164.stdout: "age": 157.14728192999999, 2023-06-03T20:26:11.546 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.062875651000000005, 2023-06-03T20:26:11.546 INFO:teuthology.orchestra.run.smithi164.stdout: "type_data": { 2023-06-03T20:26:11.546 INFO:teuthology.orchestra.run.smithi164.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-03T20:26:11.546 INFO:teuthology.orchestra.run.smithi164.stdout: "events": [ 2023-06-03T20:26:11.547 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:26:11.547 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "initiated", 2023-06-03T20:26:11.547 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:23:34.383131+0000", 2023-06-03T20:26:11.547 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0 2023-06-03T20:26:11.547 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:26:11.547 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:26:11.548 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "throttled", 2023-06-03T20:26:11.548 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:23:34.383131+0000", 2023-06-03T20:26:11.548 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 4294967295.999999 2023-06-03T20:26:11.548 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:26:11.548 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:26:11.548 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "header_read", 2023-06-03T20:26:11.549 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:23:34.383130+0000", 2023-06-03T20:26:11.549 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 9.1500000000000005e-06 2023-06-03T20:26:11.549 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:26:11.549 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:26:11.549 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "all_read", 2023-06-03T20:26:11.549 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:23:34.383139+0000", 2023-06-03T20:26:11.550 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 1.9833000000000001e-05 2023-06-03T20:26:11.550 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:26:11.550 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:26:11.550 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "dispatched", 2023-06-03T20:26:11.550 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:23:34.383159+0000", 2023-06-03T20:26:11.550 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 6.1140000000000002e-06 2023-06-03T20:26:11.551 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:26:11.551 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:26:11.551 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "queued_for_pg", 2023-06-03T20:26:11.551 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:23:34.383165+0000", 2023-06-03T20:26:11.551 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.0062322219999999999 2023-06-03T20:26:11.551 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:26:11.551 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:26:11.552 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "reached_pg", 2023-06-03T20:26:11.552 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:23:34.389397+0000", 2023-06-03T20:26:11.552 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 1.8162e-05 2023-06-03T20:26:11.552 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:26:11.552 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:26:11.552 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "started", 2023-06-03T20:26:11.552 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:23:34.389416+0000", 2023-06-03T20:26:11.552 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.052514661999999997 2023-06-03T20:26:11.553 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:26:11.553 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:26:11.553 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "commit_sent", 2023-06-03T20:26:11.553 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:23:34.441930+0000", 2023-06-03T20:26:11.553 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.0040764620000000003 2023-06-03T20:26:11.553 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:26:11.553 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:26:11.553 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "done", 2023-06-03T20:26:11.554 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:23:34.446007+0000", 2023-06-03T20:26:11.554 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.062875651000000005 2023-06-03T20:26:11.554 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:26:11.554 INFO:teuthology.orchestra.run.smithi164.stdout: ] 2023-06-03T20:26:11.554 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:26:11.554 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:26:11.554 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:26:11.554 INFO:teuthology.orchestra.run.smithi164.stdout: "description": "osd_repop(client.36573.0:126146 13.f e252/242 13:f697bd7f:::benchmark_data_smithi089_211581_object10621:head v 252'7970, mlcod=252'7964)", 2023-06-03T20:26:11.555 INFO:teuthology.orchestra.run.smithi164.stdout: "initiated_at": "2023-06-03T20:23:42.554774+0000", 2023-06-03T20:26:11.555 INFO:teuthology.orchestra.run.smithi164.stdout: "age": 148.97563890500001, 2023-06-03T20:26:11.555 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.060764682, 2023-06-03T20:26:11.555 INFO:teuthology.orchestra.run.smithi164.stdout: "type_data": { 2023-06-03T20:26:11.555 INFO:teuthology.orchestra.run.smithi164.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-03T20:26:11.555 INFO:teuthology.orchestra.run.smithi164.stdout: "events": [ 2023-06-03T20:26:11.555 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:26:11.556 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "initiated", 2023-06-03T20:26:11.556 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:23:42.554774+0000", 2023-06-03T20:26:11.556 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0 2023-06-03T20:26:11.556 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:26:11.556 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:26:11.556 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "throttled", 2023-06-03T20:26:11.556 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:23:42.554774+0000", 2023-06-03T20:26:11.556 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 4294967295.9999995 2023-06-03T20:26:11.557 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:26:11.557 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:26:11.557 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "header_read", 2023-06-03T20:26:11.557 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:23:42.554774+0000", 2023-06-03T20:26:11.557 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 4.1359999999999999e-06 2023-06-03T20:26:11.557 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:26:11.557 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:26:11.557 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "all_read", 2023-06-03T20:26:11.558 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:23:42.554778+0000", 2023-06-03T20:26:11.558 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 5.3090000000000003e-06 2023-06-03T20:26:11.558 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:26:11.558 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:26:11.558 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "dispatched", 2023-06-03T20:26:11.558 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:23:42.554783+0000", 2023-06-03T20:26:11.558 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 3.309e-06 2023-06-03T20:26:11.558 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:26:11.559 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:26:11.559 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "queued_for_pg", 2023-06-03T20:26:11.559 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:23:42.554787+0000", 2023-06-03T20:26:11.559 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.010180645 2023-06-03T20:26:11.559 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:26:11.559 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:26:11.559 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "reached_pg", 2023-06-03T20:26:11.559 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:23:42.564967+0000", 2023-06-03T20:26:11.560 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 1.4148000000000001e-05 2023-06-03T20:26:11.560 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:26:11.560 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:26:11.560 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "started", 2023-06-03T20:26:11.560 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:23:42.564981+0000", 2023-06-03T20:26:11.560 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.050535049999999998 2023-06-03T20:26:11.560 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:26:11.560 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:26:11.561 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "commit_sent", 2023-06-03T20:26:11.561 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:23:42.615516+0000", 2023-06-03T20:26:11.561 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 2.2325999999999999e-05 2023-06-03T20:26:11.561 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:26:11.561 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:26:11.561 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "done", 2023-06-03T20:26:11.561 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:23:42.615539+0000", 2023-06-03T20:26:11.561 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.060764682 2023-06-03T20:26:11.562 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:26:11.562 INFO:teuthology.orchestra.run.smithi164.stdout: ] 2023-06-03T20:26:11.562 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:26:11.562 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:26:11.562 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:26:11.562 INFO:teuthology.orchestra.run.smithi164.stdout: "description": "osd_repop(client.36573.0:145582 13.e e257/251 13:7649425e:::benchmark_data_smithi089_211581_object30057:head v 257'9043, mlcod=245'1906)", 2023-06-03T20:26:11.562 INFO:teuthology.orchestra.run.smithi164.stdout: "initiated_at": "2023-06-03T20:24:17.679174+0000", 2023-06-03T20:26:11.563 INFO:teuthology.orchestra.run.smithi164.stdout: "age": 113.851239342, 2023-06-03T20:26:11.563 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.060734807000000002, 2023-06-03T20:26:11.563 INFO:teuthology.orchestra.run.smithi164.stdout: "type_data": { 2023-06-03T20:26:11.563 INFO:teuthology.orchestra.run.smithi164.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-03T20:26:11.563 INFO:teuthology.orchestra.run.smithi164.stdout: "events": [ 2023-06-03T20:26:11.563 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:26:11.563 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "initiated", 2023-06-03T20:26:11.563 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:24:17.679174+0000", 2023-06-03T20:26:11.564 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0 2023-06-03T20:26:11.564 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:26:11.564 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:26:11.564 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "throttled", 2023-06-03T20:26:11.564 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:24:17.679174+0000", 2023-06-03T20:26:11.564 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 4294967296 2023-06-03T20:26:11.564 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:26:11.564 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:26:11.565 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "header_read", 2023-06-03T20:26:11.565 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:24:17.679173+0000", 2023-06-03T20:26:11.565 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 4.7539999999999997e-06 2023-06-03T20:26:11.565 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:26:11.565 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:26:11.565 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "all_read", 2023-06-03T20:26:11.565 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:24:17.679178+0000", 2023-06-03T20:26:11.566 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 5.378e-06 2023-06-03T20:26:11.566 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:26:11.566 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:26:11.566 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "dispatched", 2023-06-03T20:26:11.566 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:24:17.679184+0000", 2023-06-03T20:26:11.566 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 4.4220000000000002e-06 2023-06-03T20:26:11.566 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:26:11.566 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:26:11.567 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "queued_for_pg", 2023-06-03T20:26:11.567 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:24:17.679188+0000", 2023-06-03T20:26:11.567 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.017492583999999999 2023-06-03T20:26:11.567 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:26:11.567 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:26:11.567 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "reached_pg", 2023-06-03T20:26:11.567 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:24:17.696681+0000", 2023-06-03T20:26:11.567 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 2.6258000000000002e-05 2023-06-03T20:26:11.568 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:26:11.568 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:26:11.568 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "started", 2023-06-03T20:26:11.568 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:24:17.696707+0000", 2023-06-03T20:26:11.568 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.043118853999999998 2023-06-03T20:26:11.568 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:26:11.568 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:26:11.568 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "commit_sent", 2023-06-03T20:26:11.569 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:24:17.739826+0000", 2023-06-03T20:26:11.569 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 8.2789999999999998e-05 2023-06-03T20:26:11.569 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:26:11.569 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:26:11.569 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "done", 2023-06-03T20:26:11.569 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:24:17.739908+0000", 2023-06-03T20:26:11.569 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.060734807000000002 2023-06-03T20:26:11.570 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:26:11.570 INFO:teuthology.orchestra.run.smithi164.stdout: ] 2023-06-03T20:26:11.570 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:26:11.570 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:26:11.570 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:26:11.570 INFO:teuthology.orchestra.run.smithi164.stdout: "description": "osd_repop(client.36573.0:146920 13.f e257/255 13:f71c5bc5:::benchmark_data_smithi089_211581_object31395:head v 257'9337, mlcod=257'9334)", 2023-06-03T20:26:11.570 INFO:teuthology.orchestra.run.smithi164.stdout: "initiated_at": "2023-06-03T20:24:20.181724+0000", 2023-06-03T20:26:11.570 INFO:teuthology.orchestra.run.smithi164.stdout: "age": 111.34868872, 2023-06-03T20:26:11.571 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.063639554000000001, 2023-06-03T20:26:11.571 INFO:teuthology.orchestra.run.smithi164.stdout: "type_data": { 2023-06-03T20:26:11.571 INFO:teuthology.orchestra.run.smithi164.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-03T20:26:11.571 INFO:teuthology.orchestra.run.smithi164.stdout: "events": [ 2023-06-03T20:26:11.571 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:26:11.571 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "initiated", 2023-06-03T20:26:11.571 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:24:20.181724+0000", 2023-06-03T20:26:11.572 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0 2023-06-03T20:26:11.572 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:26:11.572 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:26:11.572 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "throttled", 2023-06-03T20:26:11.572 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:24:20.181724+0000", 2023-06-03T20:26:11.572 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 4294967295.9999995 2023-06-03T20:26:11.572 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:26:11.572 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:26:11.573 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "header_read", 2023-06-03T20:26:11.573 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:24:20.181724+0000", 2023-06-03T20:26:11.573 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 1.6084999999999999e-05 2023-06-03T20:26:11.573 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:26:11.573 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:26:11.573 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "all_read", 2023-06-03T20:26:11.573 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:24:20.181740+0000", 2023-06-03T20:26:11.573 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 7.9719999999999992e-06 2023-06-03T20:26:11.574 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:26:11.574 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:26:11.574 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "dispatched", 2023-06-03T20:26:11.574 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:24:20.181748+0000", 2023-06-03T20:26:11.574 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 4.5839999999999996e-06 2023-06-03T20:26:11.574 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:26:11.574 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:26:11.574 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "queued_for_pg", 2023-06-03T20:26:11.575 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:24:20.181752+0000", 2023-06-03T20:26:11.575 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.0058436920000000002 2023-06-03T20:26:11.575 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:26:11.575 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:26:11.575 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "reached_pg", 2023-06-03T20:26:11.575 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:24:20.187596+0000", 2023-06-03T20:26:11.575 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 2.5106999999999999e-05 2023-06-03T20:26:11.575 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:26:11.576 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:26:11.576 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "started", 2023-06-03T20:26:11.576 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:24:20.187621+0000", 2023-06-03T20:26:11.576 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.057630357 2023-06-03T20:26:11.576 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:26:11.576 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:26:11.576 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "commit_sent", 2023-06-03T20:26:11.577 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:24:20.245252+0000", 2023-06-03T20:26:11.577 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.000112225 2023-06-03T20:26:11.577 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:26:11.577 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:26:11.577 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "done", 2023-06-03T20:26:11.577 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:24:20.245364+0000", 2023-06-03T20:26:11.577 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.063639554000000001 2023-06-03T20:26:11.577 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:26:11.578 INFO:teuthology.orchestra.run.smithi164.stdout: ] 2023-06-03T20:26:11.578 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:26:11.578 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:26:11.578 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:26:11.578 INFO:teuthology.orchestra.run.smithi164.stdout: "description": "osd_repop(client.36573.0:147558 13.f e257/255 13:f807ede0:::benchmark_data_smithi089_211581_object32033:head v 257'9376, mlcod=257'9371)", 2023-06-03T20:26:11.578 INFO:teuthology.orchestra.run.smithi164.stdout: "initiated_at": "2023-06-03T20:24:21.478914+0000", 2023-06-03T20:26:11.578 INFO:teuthology.orchestra.run.smithi164.stdout: "age": 110.051499248, 2023-06-03T20:26:11.578 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.063243858999999999, 2023-06-03T20:26:11.579 INFO:teuthology.orchestra.run.smithi164.stdout: "type_data": { 2023-06-03T20:26:11.579 INFO:teuthology.orchestra.run.smithi164.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-03T20:26:11.579 INFO:teuthology.orchestra.run.smithi164.stdout: "events": [ 2023-06-03T20:26:11.579 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:26:11.579 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "initiated", 2023-06-03T20:26:11.579 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:24:21.478914+0000", 2023-06-03T20:26:11.579 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0 2023-06-03T20:26:11.579 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:26:11.580 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:26:11.580 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "throttled", 2023-06-03T20:26:11.580 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:24:21.478914+0000", 2023-06-03T20:26:11.580 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 4294967295.999999 2023-06-03T20:26:11.580 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:26:11.580 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:26:11.580 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "header_read", 2023-06-03T20:26:11.581 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:24:21.478913+0000", 2023-06-03T20:26:11.581 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 1.2228e-05 2023-06-03T20:26:11.581 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:26:11.581 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:26:11.581 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "all_read", 2023-06-03T20:26:11.581 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:24:21.478925+0000", 2023-06-03T20:26:11.581 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 1.3587999999999999e-05 2023-06-03T20:26:11.581 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:26:11.582 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:26:11.582 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "dispatched", 2023-06-03T20:26:11.582 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:24:21.478939+0000", 2023-06-03T20:26:11.582 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 6.8709999999999999e-06 2023-06-03T20:26:11.582 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:26:11.582 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:26:11.582 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "queued_for_pg", 2023-06-03T20:26:11.582 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:24:21.478945+0000", 2023-06-03T20:26:11.583 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.017864729999999999 2023-06-03T20:26:11.583 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:26:11.583 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:26:11.583 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "reached_pg", 2023-06-03T20:26:11.583 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:24:21.496810+0000", 2023-06-03T20:26:11.583 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 2.7838e-05 2023-06-03T20:26:11.583 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:26:11.583 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:26:11.584 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "started", 2023-06-03T20:26:11.584 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:24:21.496838+0000", 2023-06-03T20:26:11.584 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.045271490999999997 2023-06-03T20:26:11.584 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:26:11.584 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:26:11.584 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "commit_sent", 2023-06-03T20:26:11.584 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:24:21.542110+0000", 2023-06-03T20:26:11.585 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 4.8092e-05 2023-06-03T20:26:11.585 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:26:11.585 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:26:11.585 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "done", 2023-06-03T20:26:11.585 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:24:21.542158+0000", 2023-06-03T20:26:11.585 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.063243858999999999 2023-06-03T20:26:11.585 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:26:11.585 INFO:teuthology.orchestra.run.smithi164.stdout: ] 2023-06-03T20:26:11.586 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:26:11.586 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:26:11.586 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:26:11.586 INFO:teuthology.orchestra.run.smithi164.stdout: "description": "osd_repop(client.36573.0:147726 13.c e257/242 13:3eb2f3ba:::benchmark_data_smithi089_211581_object32201:head v 257'9380, mlcod=257'9377)", 2023-06-03T20:26:11.586 INFO:teuthology.orchestra.run.smithi164.stdout: "initiated_at": "2023-06-03T20:24:21.807217+0000", 2023-06-03T20:26:11.586 INFO:teuthology.orchestra.run.smithi164.stdout: "age": 109.7231963, 2023-06-03T20:26:11.586 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.067208406999999998, 2023-06-03T20:26:11.586 INFO:teuthology.orchestra.run.smithi164.stdout: "type_data": { 2023-06-03T20:26:11.587 INFO:teuthology.orchestra.run.smithi164.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-03T20:26:11.587 INFO:teuthology.orchestra.run.smithi164.stdout: "events": [ 2023-06-03T20:26:11.587 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:26:11.587 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "initiated", 2023-06-03T20:26:11.587 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:24:21.807217+0000", 2023-06-03T20:26:11.587 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0 2023-06-03T20:26:11.587 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:26:11.587 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:26:11.588 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "throttled", 2023-06-03T20:26:11.588 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:24:21.807217+0000", 2023-06-03T20:26:11.588 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 4294967295.999999 2023-06-03T20:26:11.588 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:26:11.588 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:26:11.588 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "header_read", 2023-06-03T20:26:11.588 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:24:21.807216+0000", 2023-06-03T20:26:11.588 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 9.8269999999999993e-06 2023-06-03T20:26:11.589 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:26:11.589 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:26:11.589 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "all_read", 2023-06-03T20:26:11.589 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:24:21.807226+0000", 2023-06-03T20:26:11.589 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 1.0468e-05 2023-06-03T20:26:11.589 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:26:11.589 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:26:11.589 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "dispatched", 2023-06-03T20:26:11.590 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:24:21.807236+0000", 2023-06-03T20:26:11.590 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 6.703e-06 2023-06-03T20:26:11.590 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:26:11.590 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:26:11.590 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "queued_for_pg", 2023-06-03T20:26:11.590 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:24:21.807243+0000", 2023-06-03T20:26:11.590 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.0038837429999999998 2023-06-03T20:26:11.591 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:26:11.591 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:26:11.591 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "reached_pg", 2023-06-03T20:26:11.591 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:24:21.811127+0000", 2023-06-03T20:26:11.591 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 2.1795999999999999e-05 2023-06-03T20:26:11.591 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:26:11.591 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:26:11.591 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "started", 2023-06-03T20:26:11.592 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:24:21.811149+0000", 2023-06-03T20:26:11.592 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.063059743000000001 2023-06-03T20:26:11.592 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:26:11.592 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:26:11.592 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "commit_sent", 2023-06-03T20:26:11.592 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:24:21.874208+0000", 2023-06-03T20:26:11.592 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.00021688100000000001 2023-06-03T20:26:11.592 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:26:11.593 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:26:11.593 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "done", 2023-06-03T20:26:11.593 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:24:21.874425+0000", 2023-06-03T20:26:11.593 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.067208406999999998 2023-06-03T20:26:11.593 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:26:11.593 INFO:teuthology.orchestra.run.smithi164.stdout: ] 2023-06-03T20:26:11.593 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:26:11.593 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:26:11.594 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:26:11.594 INFO:teuthology.orchestra.run.smithi164.stdout: "description": "osd_repop(client.36573.0:147727 13.c e257/242 13:3f710ead:::benchmark_data_smithi089_211581_object32202:head v 257'9381, mlcod=257'9377)", 2023-06-03T20:26:11.594 INFO:teuthology.orchestra.run.smithi164.stdout: "initiated_at": "2023-06-03T20:24:21.813718+0000", 2023-06-03T20:26:11.594 INFO:teuthology.orchestra.run.smithi164.stdout: "age": 109.716695401, 2023-06-03T20:26:11.594 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.060795643000000003, 2023-06-03T20:26:11.594 INFO:teuthology.orchestra.run.smithi164.stdout: "type_data": { 2023-06-03T20:26:11.594 INFO:teuthology.orchestra.run.smithi164.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-03T20:26:11.594 INFO:teuthology.orchestra.run.smithi164.stdout: "events": [ 2023-06-03T20:26:11.595 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:26:11.595 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "initiated", 2023-06-03T20:26:11.595 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:24:21.813718+0000", 2023-06-03T20:26:11.595 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0 2023-06-03T20:26:11.595 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:26:11.595 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:26:11.595 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "throttled", 2023-06-03T20:26:11.595 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:24:21.813718+0000", 2023-06-03T20:26:11.596 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 4294967295.9999986 2023-06-03T20:26:11.596 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:26:11.596 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:26:11.596 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "header_read", 2023-06-03T20:26:11.596 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:24:21.813716+0000", 2023-06-03T20:26:11.596 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 2.4386999999999999e-05 2023-06-03T20:26:11.596 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:26:11.597 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:26:11.597 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "all_read", 2023-06-03T20:26:11.597 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:24:21.813740+0000", 2023-06-03T20:26:11.597 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 1.7286e-05 2023-06-03T20:26:11.597 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:26:11.597 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:26:11.597 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "dispatched", 2023-06-03T20:26:11.597 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:24:21.813758+0000", 2023-06-03T20:26:11.598 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 7.2400000000000001e-06 2023-06-03T20:26:11.598 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:26:11.598 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:26:11.598 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "queued_for_pg", 2023-06-03T20:26:11.598 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:24:21.813765+0000", 2023-06-03T20:26:11.598 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.0081028309999999996 2023-06-03T20:26:11.598 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:26:11.598 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:26:11.599 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "reached_pg", 2023-06-03T20:26:11.599 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:24:21.821868+0000", 2023-06-03T20:26:11.599 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 2.6842000000000002e-05 2023-06-03T20:26:11.599 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:26:11.599 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:26:11.599 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "started", 2023-06-03T20:26:11.599 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:24:21.821895+0000", 2023-06-03T20:26:11.599 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.052535446 2023-06-03T20:26:11.600 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:26:11.600 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:26:11.600 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "commit_sent", 2023-06-03T20:26:11.600 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:24:21.874430+0000", 2023-06-03T20:26:11.600 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 8.3158999999999994e-05 2023-06-03T20:26:11.600 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:26:11.600 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:26:11.601 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "done", 2023-06-03T20:26:11.601 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:24:21.874513+0000", 2023-06-03T20:26:11.601 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.060795643000000003 2023-06-03T20:26:11.601 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:26:11.601 INFO:teuthology.orchestra.run.smithi164.stdout: ] 2023-06-03T20:26:11.601 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:26:11.601 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:26:11.601 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:26:11.602 INFO:teuthology.orchestra.run.smithi164.stdout: "description": "osd_repop(client.36573.0:147724 13.4 e257/242 13:2f77f775:::benchmark_data_smithi089_211581_object32199:head v 257'9213, mlcod=257'9211)", 2023-06-03T20:26:11.602 INFO:teuthology.orchestra.run.smithi164.stdout: "initiated_at": "2023-06-03T20:24:21.814420+0000", 2023-06-03T20:26:11.602 INFO:teuthology.orchestra.run.smithi164.stdout: "age": 109.71599351499999, 2023-06-03T20:26:11.602 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.063277366000000002, 2023-06-03T20:26:11.602 INFO:teuthology.orchestra.run.smithi164.stdout: "type_data": { 2023-06-03T20:26:11.602 INFO:teuthology.orchestra.run.smithi164.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-03T20:26:11.602 INFO:teuthology.orchestra.run.smithi164.stdout: "events": [ 2023-06-03T20:26:11.603 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:26:11.603 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "initiated", 2023-06-03T20:26:11.603 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:24:21.814420+0000", 2023-06-03T20:26:11.603 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0 2023-06-03T20:26:11.603 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:26:11.603 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:26:11.603 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "throttled", 2023-06-03T20:26:11.603 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:24:21.814420+0000", 2023-06-03T20:26:11.604 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 4294967295.999999 2023-06-03T20:26:11.604 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:26:11.604 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:26:11.604 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "header_read", 2023-06-03T20:26:11.604 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:24:21.814419+0000", 2023-06-03T20:26:11.604 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 8.0800000000000006e-06 2023-06-03T20:26:11.604 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:26:11.604 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:26:11.605 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "all_read", 2023-06-03T20:26:11.605 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:24:21.814427+0000", 2023-06-03T20:26:11.605 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 1.4399999999999999e-05 2023-06-03T20:26:11.605 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:26:11.605 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:26:11.605 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "dispatched", 2023-06-03T20:26:11.605 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:24:21.814441+0000", 2023-06-03T20:26:11.605 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 2.4834999999999999e-05 2023-06-03T20:26:11.606 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:26:11.606 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:26:11.606 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "queued_for_pg", 2023-06-03T20:26:11.606 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:24:21.814466+0000", 2023-06-03T20:26:11.606 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.0043573839999999997 2023-06-03T20:26:11.606 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:26:11.606 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:26:11.607 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "reached_pg", 2023-06-03T20:26:11.607 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:24:21.818823+0000", 2023-06-03T20:26:11.607 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 2.1736e-05 2023-06-03T20:26:11.607 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:26:11.607 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:26:11.607 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "started", 2023-06-03T20:26:11.607 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:24:21.818845+0000", 2023-06-03T20:26:11.607 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.058609266 2023-06-03T20:26:11.608 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:26:11.608 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:26:11.608 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "commit_sent", 2023-06-03T20:26:11.608 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:24:21.877454+0000", 2023-06-03T20:26:11.608 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.00024244800000000001 2023-06-03T20:26:11.608 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:26:11.608 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:26:11.608 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "done", 2023-06-03T20:26:11.609 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:24:21.877697+0000", 2023-06-03T20:26:11.609 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.063277366000000002 2023-06-03T20:26:11.609 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:26:11.609 INFO:teuthology.orchestra.run.smithi164.stdout: ] 2023-06-03T20:26:11.609 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:26:11.609 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:26:11.609 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:26:11.609 INFO:teuthology.orchestra.run.smithi164.stdout: "description": "osd_repop(client.36573.0:149490 13.6 e257/251 13:690f0f85:::benchmark_data_smithi089_211581_object33965:head v 257'9158, mlcod=257'9153)", 2023-06-03T20:26:11.610 INFO:teuthology.orchestra.run.smithi164.stdout: "initiated_at": "2023-06-03T20:24:25.180839+0000", 2023-06-03T20:26:11.610 INFO:teuthology.orchestra.run.smithi164.stdout: "age": 106.34957358600001, 2023-06-03T20:26:11.610 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.075161934999999999, 2023-06-03T20:26:11.610 INFO:teuthology.orchestra.run.smithi164.stdout: "type_data": { 2023-06-03T20:26:11.610 INFO:teuthology.orchestra.run.smithi164.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-03T20:26:11.610 INFO:teuthology.orchestra.run.smithi164.stdout: "events": [ 2023-06-03T20:26:11.610 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:26:11.610 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "initiated", 2023-06-03T20:26:11.611 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:24:25.180839+0000", 2023-06-03T20:26:11.611 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0 2023-06-03T20:26:11.611 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:26:11.611 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:26:11.611 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "throttled", 2023-06-03T20:26:11.611 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:24:25.180839+0000", 2023-06-03T20:26:11.611 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 4294967295.9999995 2023-06-03T20:26:11.611 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:26:11.612 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:26:11.612 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "header_read", 2023-06-03T20:26:11.612 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:24:25.180839+0000", 2023-06-03T20:26:11.612 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 6.9009999999999997e-06 2023-06-03T20:26:11.612 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:26:11.612 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:26:11.612 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "all_read", 2023-06-03T20:26:11.613 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:24:25.180846+0000", 2023-06-03T20:26:11.613 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 9.6809999999999993e-06 2023-06-03T20:26:11.613 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:26:11.613 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:26:11.613 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "dispatched", 2023-06-03T20:26:11.613 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:24:25.180855+0000", 2023-06-03T20:26:11.613 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 5.1510000000000003e-06 2023-06-03T20:26:11.613 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:26:11.614 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:26:11.614 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "queued_for_pg", 2023-06-03T20:26:11.614 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:24:25.180860+0000", 2023-06-03T20:26:11.614 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.029509850000000001 2023-06-03T20:26:11.614 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:26:11.614 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:26:11.614 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "reached_pg", 2023-06-03T20:26:11.614 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:24:25.210370+0000", 2023-06-03T20:26:11.615 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 5.7719000000000001e-05 2023-06-03T20:26:11.615 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:26:11.615 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:26:11.615 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "started", 2023-06-03T20:26:11.615 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:24:25.210428+0000", 2023-06-03T20:26:11.615 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.045531787999999997 2023-06-03T20:26:11.615 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:26:11.615 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:26:11.616 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "commit_sent", 2023-06-03T20:26:11.616 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:24:25.255960+0000", 2023-06-03T20:26:11.616 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 4.1520000000000002e-05 2023-06-03T20:26:11.616 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:26:11.616 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:26:11.616 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "done", 2023-06-03T20:26:11.616 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:24:25.256001+0000", 2023-06-03T20:26:11.616 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.075161934999999999 2023-06-03T20:26:11.617 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:26:11.617 INFO:teuthology.orchestra.run.smithi164.stdout: ] 2023-06-03T20:26:11.617 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:26:11.617 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:26:11.617 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:26:11.617 INFO:teuthology.orchestra.run.smithi164.stdout: "description": "osd_repop(client.36573.0:151376 13.7 e257/254 13:e920d967:::benchmark_data_smithi089_211581_object35851:head v 257'9581, mlcod=257'9578)", 2023-06-03T20:26:11.617 INFO:teuthology.orchestra.run.smithi164.stdout: "initiated_at": "2023-06-03T20:24:28.613854+0000", 2023-06-03T20:26:11.617 INFO:teuthology.orchestra.run.smithi164.stdout: "age": 102.916559487, 2023-06-03T20:26:11.618 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.064675888000000001, 2023-06-03T20:26:11.618 INFO:teuthology.orchestra.run.smithi164.stdout: "type_data": { 2023-06-03T20:26:11.618 INFO:teuthology.orchestra.run.smithi164.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-03T20:26:11.618 INFO:teuthology.orchestra.run.smithi164.stdout: "events": [ 2023-06-03T20:26:11.618 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:26:11.618 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "initiated", 2023-06-03T20:26:11.618 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:24:28.613854+0000", 2023-06-03T20:26:11.618 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0 2023-06-03T20:26:11.619 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:26:11.619 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:26:11.619 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "throttled", 2023-06-03T20:26:11.619 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:24:28.613854+0000", 2023-06-03T20:26:11.619 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 4294967295.9999995 2023-06-03T20:26:11.619 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:26:11.619 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:26:11.619 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "header_read", 2023-06-03T20:26:11.620 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:24:28.613853+0000", 2023-06-03T20:26:11.620 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 7.3830000000000003e-06 2023-06-03T20:26:11.620 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:26:11.620 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:26:11.620 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "all_read", 2023-06-03T20:26:11.620 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:24:28.613860+0000", 2023-06-03T20:26:11.620 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 1.4589999999999999e-05 2023-06-03T20:26:11.621 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:26:11.621 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:26:11.621 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "dispatched", 2023-06-03T20:26:11.621 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:24:28.613875+0000", 2023-06-03T20:26:11.621 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 6.0090000000000004e-06 2023-06-03T20:26:11.621 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:26:11.621 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:26:11.621 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "queued_for_pg", 2023-06-03T20:26:11.622 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:24:28.613881+0000", 2023-06-03T20:26:11.622 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.0079434880000000003 2023-06-03T20:26:11.622 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:26:11.622 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:26:11.622 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "reached_pg", 2023-06-03T20:26:11.622 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:24:28.621824+0000", 2023-06-03T20:26:11.622 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 2.4136999999999999e-05 2023-06-03T20:26:11.622 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:26:11.623 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:26:11.623 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "started", 2023-06-03T20:26:11.623 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:24:28.621848+0000", 2023-06-03T20:26:11.623 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.056644633 2023-06-03T20:26:11.623 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:26:11.623 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:26:11.623 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "commit_sent", 2023-06-03T20:26:11.623 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:24:28.678493+0000", 2023-06-03T20:26:11.624 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 3.6359999999999997e-05 2023-06-03T20:26:11.624 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:26:11.624 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:26:11.624 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "done", 2023-06-03T20:26:11.624 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:24:28.678529+0000", 2023-06-03T20:26:11.624 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.064675888000000001 2023-06-03T20:26:11.624 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:26:11.625 INFO:teuthology.orchestra.run.smithi164.stdout: ] 2023-06-03T20:26:11.625 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:26:11.625 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:26:11.625 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:26:11.625 INFO:teuthology.orchestra.run.smithi164.stdout: "description": "osd_repop(client.36573.0:151708 13.f e257/255 13:f1132b24:::benchmark_data_smithi089_211581_object36183:head v 257'9632, mlcod=257'9628)", 2023-06-03T20:26:11.625 INFO:teuthology.orchestra.run.smithi164.stdout: "initiated_at": "2023-06-03T20:24:29.197419+0000", 2023-06-03T20:26:11.625 INFO:teuthology.orchestra.run.smithi164.stdout: "age": 102.33299417000001, 2023-06-03T20:26:11.625 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.061092162999999998, 2023-06-03T20:26:11.626 INFO:teuthology.orchestra.run.smithi164.stdout: "type_data": { 2023-06-03T20:26:11.626 INFO:teuthology.orchestra.run.smithi164.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-03T20:26:11.626 INFO:teuthology.orchestra.run.smithi164.stdout: "events": [ 2023-06-03T20:26:11.626 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:26:11.626 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "initiated", 2023-06-03T20:26:11.626 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:24:29.197419+0000", 2023-06-03T20:26:11.626 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0 2023-06-03T20:26:11.626 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:26:11.627 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:26:11.627 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "throttled", 2023-06-03T20:26:11.627 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:24:29.197419+0000", 2023-06-03T20:26:11.627 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 4294967295.999999 2023-06-03T20:26:11.627 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:26:11.627 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:26:11.627 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "header_read", 2023-06-03T20:26:11.627 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:24:29.197418+0000", 2023-06-03T20:26:11.628 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 1.0485999999999999e-05 2023-06-03T20:26:11.628 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:26:11.628 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:26:11.628 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "all_read", 2023-06-03T20:26:11.628 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:24:29.197429+0000", 2023-06-03T20:26:11.628 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 1.1206000000000001e-05 2023-06-03T20:26:11.628 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:26:11.628 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:26:11.629 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "dispatched", 2023-06-03T20:26:11.629 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:24:29.197440+0000", 2023-06-03T20:26:11.629 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 5.9719999999999997e-06 2023-06-03T20:26:11.629 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:26:11.629 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:26:11.629 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "queued_for_pg", 2023-06-03T20:26:11.629 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:24:29.197446+0000", 2023-06-03T20:26:11.630 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.020514502 2023-06-03T20:26:11.630 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:26:11.630 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:26:11.630 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "reached_pg", 2023-06-03T20:26:11.630 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:24:29.217960+0000", 2023-06-03T20:26:11.630 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 3.3433000000000003e-05 2023-06-03T20:26:11.630 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:26:11.630 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:26:11.631 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "started", 2023-06-03T20:26:11.631 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:24:29.217994+0000", 2023-06-03T20:26:11.631 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.040407911999999997 2023-06-03T20:26:11.631 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:26:11.631 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:26:11.631 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "commit_sent", 2023-06-03T20:26:11.631 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:24:29.258402+0000", 2023-06-03T20:26:11.631 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.000109413 2023-06-03T20:26:11.632 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:26:11.632 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:26:11.632 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "done", 2023-06-03T20:26:11.632 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:24:29.258511+0000", 2023-06-03T20:26:11.632 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.061092162999999998 2023-06-03T20:26:11.632 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:26:11.632 INFO:teuthology.orchestra.run.smithi164.stdout: ] 2023-06-03T20:26:11.632 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:26:11.633 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:26:11.633 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:26:11.633 INFO:teuthology.orchestra.run.smithi164.stdout: "description": "osd_repop(client.36573.0:152103 13.7 e257/254 13:e651b7bd:::benchmark_data_smithi089_211581_object36578:head v 257'9624, mlcod=257'9621)", 2023-06-03T20:26:11.633 INFO:teuthology.orchestra.run.smithi164.stdout: "initiated_at": "2023-06-03T20:24:30.141771+0000", 2023-06-03T20:26:11.633 INFO:teuthology.orchestra.run.smithi164.stdout: "age": 101.388642226, 2023-06-03T20:26:11.633 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.061247203, 2023-06-03T20:26:11.633 INFO:teuthology.orchestra.run.smithi164.stdout: "type_data": { 2023-06-03T20:26:11.633 INFO:teuthology.orchestra.run.smithi164.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-03T20:26:11.634 INFO:teuthology.orchestra.run.smithi164.stdout: "events": [ 2023-06-03T20:26:11.634 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:26:11.634 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "initiated", 2023-06-03T20:26:11.634 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:24:30.141771+0000", 2023-06-03T20:26:11.634 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0 2023-06-03T20:26:11.634 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:26:11.634 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:26:11.635 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "throttled", 2023-06-03T20:26:11.635 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:24:30.141771+0000", 2023-06-03T20:26:11.635 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 4294967295.999999 2023-06-03T20:26:11.635 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:26:11.635 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:26:11.635 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "header_read", 2023-06-03T20:26:11.635 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:24:30.141770+0000", 2023-06-03T20:26:11.635 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 1.1353e-05 2023-06-03T20:26:11.636 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:26:11.636 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:26:11.636 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "all_read", 2023-06-03T20:26:11.636 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:24:30.141781+0000", 2023-06-03T20:26:11.636 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 2.5352000000000001e-05 2023-06-03T20:26:11.636 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:26:11.636 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:26:11.636 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "dispatched", 2023-06-03T20:26:11.637 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:24:30.141807+0000", 2023-06-03T20:26:11.637 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 7.2359999999999998e-06 2023-06-03T20:26:11.637 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:26:11.637 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:26:11.637 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "queued_for_pg", 2023-06-03T20:26:11.637 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:24:30.141814+0000", 2023-06-03T20:26:11.637 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.0055142580000000002 2023-06-03T20:26:11.637 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:26:11.638 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:26:11.638 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "reached_pg", 2023-06-03T20:26:11.638 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:24:30.147328+0000", 2023-06-03T20:26:11.638 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 2.6576000000000001e-05 2023-06-03T20:26:11.638 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:26:11.638 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:26:11.638 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "started", 2023-06-03T20:26:11.639 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:24:30.147355+0000", 2023-06-03T20:26:11.639 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.055632550000000003 2023-06-03T20:26:11.639 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:26:11.639 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:26:11.639 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "commit_sent", 2023-06-03T20:26:11.639 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:24:30.202987+0000", 2023-06-03T20:26:11.639 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 3.0732000000000003e-05 2023-06-03T20:26:11.639 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:26:11.640 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:26:11.640 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "done", 2023-06-03T20:26:11.640 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:24:30.203018+0000", 2023-06-03T20:26:11.640 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.061247203 2023-06-03T20:26:11.640 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:26:11.640 INFO:teuthology.orchestra.run.smithi164.stdout: ] 2023-06-03T20:26:11.640 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:26:11.640 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:26:11.641 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:26:11.641 INFO:teuthology.orchestra.run.smithi164.stdout: "description": "osd_repop(client.36573.0:152958 13.7 e257/254 13:e45a69a2:::benchmark_data_smithi089_211581_object37433:head v 257'9685, mlcod=257'9682)", 2023-06-03T20:26:11.641 INFO:teuthology.orchestra.run.smithi164.stdout: "initiated_at": "2023-06-03T20:24:31.725771+0000", 2023-06-03T20:26:11.641 INFO:teuthology.orchestra.run.smithi164.stdout: "age": 99.804642408999996, 2023-06-03T20:26:11.641 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.077545691999999999, 2023-06-03T20:26:11.641 INFO:teuthology.orchestra.run.smithi164.stdout: "type_data": { 2023-06-03T20:26:11.641 INFO:teuthology.orchestra.run.smithi164.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-03T20:26:11.641 INFO:teuthology.orchestra.run.smithi164.stdout: "events": [ 2023-06-03T20:26:11.642 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:26:11.642 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "initiated", 2023-06-03T20:26:11.642 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:24:31.725771+0000", 2023-06-03T20:26:11.642 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0 2023-06-03T20:26:11.642 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:26:11.642 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:26:11.642 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "throttled", 2023-06-03T20:26:11.642 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:24:31.725771+0000", 2023-06-03T20:26:11.643 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 4294967295.9999986 2023-06-03T20:26:11.643 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:26:11.643 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:26:11.643 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "header_read", 2023-06-03T20:26:11.643 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:24:31.725769+0000", 2023-06-03T20:26:11.643 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 1.4212999999999999e-05 2023-06-03T20:26:11.643 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:26:11.644 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:26:11.644 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "all_read", 2023-06-03T20:26:11.644 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:24:31.725784+0000", 2023-06-03T20:26:11.644 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 1.5141e-05 2023-06-03T20:26:11.644 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:26:11.644 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:26:11.644 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "dispatched", 2023-06-03T20:26:11.644 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:24:31.725799+0000", 2023-06-03T20:26:11.645 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 7.3030000000000002e-06 2023-06-03T20:26:11.645 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:26:11.645 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:26:11.645 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "queued_for_pg", 2023-06-03T20:26:11.645 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:24:31.725806+0000", 2023-06-03T20:26:11.645 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.0030421020000000001 2023-06-03T20:26:11.645 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:26:11.645 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:26:11.646 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "reached_pg", 2023-06-03T20:26:11.646 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:24:31.728848+0000", 2023-06-03T20:26:11.646 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 3.1983999999999997e-05 2023-06-03T20:26:11.646 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:26:11.646 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:26:11.646 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "started", 2023-06-03T20:26:11.646 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:24:31.728880+0000", 2023-06-03T20:26:11.647 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.074401386999999999 2023-06-03T20:26:11.647 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:26:11.647 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:26:11.647 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "commit_sent", 2023-06-03T20:26:11.647 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:24:31.803281+0000", 2023-06-03T20:26:11.647 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 3.4841000000000001e-05 2023-06-03T20:26:11.647 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:26:11.647 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:26:11.648 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "done", 2023-06-03T20:26:11.648 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:24:31.803316+0000", 2023-06-03T20:26:11.648 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.077545691999999999 2023-06-03T20:26:11.648 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:26:11.648 INFO:teuthology.orchestra.run.smithi164.stdout: ] 2023-06-03T20:26:11.648 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:26:11.648 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:26:11.649 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:26:11.649 INFO:teuthology.orchestra.run.smithi164.stdout: "description": "osd_repop(client.36573.0:152971 13.f e257/255 13:f183abf7:::benchmark_data_smithi089_211581_object37446:head v 257'9706, mlcod=257'9703)", 2023-06-03T20:26:11.649 INFO:teuthology.orchestra.run.smithi164.stdout: "initiated_at": "2023-06-03T20:24:31.732176+0000", 2023-06-03T20:26:11.649 INFO:teuthology.orchestra.run.smithi164.stdout: "age": 99.798237447000005, 2023-06-03T20:26:11.649 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.071167053999999993, 2023-06-03T20:26:11.649 INFO:teuthology.orchestra.run.smithi164.stdout: "type_data": { 2023-06-03T20:26:11.649 INFO:teuthology.orchestra.run.smithi164.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-03T20:26:11.649 INFO:teuthology.orchestra.run.smithi164.stdout: "events": [ 2023-06-03T20:26:11.650 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:26:11.650 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "initiated", 2023-06-03T20:26:11.650 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:24:31.732176+0000", 2023-06-03T20:26:11.650 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0 2023-06-03T20:26:11.650 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:26:11.650 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:26:11.650 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "throttled", 2023-06-03T20:26:11.650 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:24:31.732176+0000", 2023-06-03T20:26:11.651 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 4294967295.999999 2023-06-03T20:26:11.651 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:26:11.651 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:26:11.651 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "header_read", 2023-06-03T20:26:11.651 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:24:31.732175+0000", 2023-06-03T20:26:11.651 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 2.4059000000000001e-05 2023-06-03T20:26:11.651 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:26:11.651 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:26:11.652 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "all_read", 2023-06-03T20:26:11.652 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:24:31.732199+0000", 2023-06-03T20:26:11.652 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 1.4613000000000001e-05 2023-06-03T20:26:11.652 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:26:11.652 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:26:11.652 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "dispatched", 2023-06-03T20:26:11.652 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:24:31.732213+0000", 2023-06-03T20:26:11.652 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 6.3899999999999998e-06 2023-06-03T20:26:11.653 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:26:11.653 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:26:11.653 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "queued_for_pg", 2023-06-03T20:26:11.653 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:24:31.732220+0000", 2023-06-03T20:26:11.653 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.005565164 2023-06-03T20:26:11.653 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:26:11.653 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:26:11.653 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "reached_pg", 2023-06-03T20:26:11.654 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:24:31.737785+0000", 2023-06-03T20:26:11.654 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 2.3003e-05 2023-06-03T20:26:11.654 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:26:11.654 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:26:11.654 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "started", 2023-06-03T20:26:11.654 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:24:31.737808+0000", 2023-06-03T20:26:11.654 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.065511423999999999 2023-06-03T20:26:11.655 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:26:11.655 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:26:11.655 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "commit_sent", 2023-06-03T20:26:11.655 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:24:31.803319+0000", 2023-06-03T20:26:11.655 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 2.3353000000000002e-05 2023-06-03T20:26:11.655 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:26:11.655 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:26:11.655 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "done", 2023-06-03T20:26:11.656 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:24:31.803343+0000", 2023-06-03T20:26:11.656 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.071167053999999993 2023-06-03T20:26:11.656 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:26:11.656 INFO:teuthology.orchestra.run.smithi164.stdout: ] 2023-06-03T20:26:11.656 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:26:11.656 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:26:11.656 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:26:11.656 INFO:teuthology.orchestra.run.smithi164.stdout: "description": "osd_repop(client.36573.0:152974 13.f e257/255 13:fa3d1c05:::benchmark_data_smithi089_211581_object37449:head v 257'9707, mlcod=257'9703)", 2023-06-03T20:26:11.657 INFO:teuthology.orchestra.run.smithi164.stdout: "initiated_at": "2023-06-03T20:24:31.742332+0000", 2023-06-03T20:26:11.657 INFO:teuthology.orchestra.run.smithi164.stdout: "age": 99.788081023999993, 2023-06-03T20:26:11.657 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.061032139999999999, 2023-06-03T20:26:11.657 INFO:teuthology.orchestra.run.smithi164.stdout: "type_data": { 2023-06-03T20:26:11.657 INFO:teuthology.orchestra.run.smithi164.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-03T20:26:11.657 INFO:teuthology.orchestra.run.smithi164.stdout: "events": [ 2023-06-03T20:26:11.657 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:26:11.657 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "initiated", 2023-06-03T20:26:11.658 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:24:31.742332+0000", 2023-06-03T20:26:11.658 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0 2023-06-03T20:26:11.658 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:26:11.658 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:26:11.658 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "throttled", 2023-06-03T20:26:11.658 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:24:31.742332+0000", 2023-06-03T20:26:11.658 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 4294967295.999999 2023-06-03T20:26:11.658 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:26:11.659 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:26:11.659 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "header_read", 2023-06-03T20:26:11.659 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:24:31.742331+0000", 2023-06-03T20:26:11.659 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 1.4314999999999999e-05 2023-06-03T20:26:11.659 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:26:11.659 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:26:11.659 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "all_read", 2023-06-03T20:26:11.659 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:24:31.742345+0000", 2023-06-03T20:26:11.660 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 2.6630000000000001e-05 2023-06-03T20:26:11.660 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:26:11.660 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:26:11.660 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "dispatched", 2023-06-03T20:26:11.660 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:24:31.742372+0000", 2023-06-03T20:26:11.660 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 7.2119999999999997e-06 2023-06-03T20:26:11.660 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:26:11.660 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:26:11.661 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "queued_for_pg", 2023-06-03T20:26:11.661 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:24:31.742379+0000", 2023-06-03T20:26:11.661 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.026233309 2023-06-03T20:26:11.661 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:26:11.661 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:26:11.661 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "reached_pg", 2023-06-03T20:26:11.661 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:24:31.768613+0000", 2023-06-03T20:26:11.661 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 3.2981000000000002e-05 2023-06-03T20:26:11.662 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:26:11.662 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:26:11.662 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "started", 2023-06-03T20:26:11.662 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:24:31.768646+0000", 2023-06-03T20:26:11.662 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.034701296 2023-06-03T20:26:11.662 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:26:11.662 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:26:11.662 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "commit_sent", 2023-06-03T20:26:11.663 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:24:31.803347+0000", 2023-06-03T20:26:11.663 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 1.7237000000000001e-05 2023-06-03T20:26:11.663 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:26:11.663 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:26:11.663 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "done", 2023-06-03T20:26:11.663 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:24:31.803364+0000", 2023-06-03T20:26:11.663 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.061032139999999999 2023-06-03T20:26:11.664 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:26:11.664 INFO:teuthology.orchestra.run.smithi164.stdout: ] 2023-06-03T20:26:11.664 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:26:11.664 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:26:11.664 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:26:11.664 INFO:teuthology.orchestra.run.smithi164.stdout: "description": "osd_repop(client.36573.0:152979 13.f e257/255 13:f8731ba2:::benchmark_data_smithi089_211581_object37454:head v 257'9708, mlcod=257'9703)", 2023-06-03T20:26:11.664 INFO:teuthology.orchestra.run.smithi164.stdout: "initiated_at": "2023-06-03T20:24:31.751365+0000", 2023-06-03T20:26:11.664 INFO:teuthology.orchestra.run.smithi164.stdout: "age": 99.779047734000002, 2023-06-03T20:26:11.665 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.075673506000000001, 2023-06-03T20:26:11.665 INFO:teuthology.orchestra.run.smithi164.stdout: "type_data": { 2023-06-03T20:26:11.665 INFO:teuthology.orchestra.run.smithi164.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-03T20:26:11.665 INFO:teuthology.orchestra.run.smithi164.stdout: "events": [ 2023-06-03T20:26:11.665 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:26:11.665 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "initiated", 2023-06-03T20:26:11.665 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:24:31.751365+0000", 2023-06-03T20:26:11.666 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0 2023-06-03T20:26:11.666 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:26:11.666 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:26:11.666 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "throttled", 2023-06-03T20:26:11.666 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:24:31.751365+0000", 2023-06-03T20:26:11.666 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 4294967295.9999986 2023-06-03T20:26:11.666 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:26:11.666 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:26:11.667 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "header_read", 2023-06-03T20:26:11.667 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:24:31.751364+0000", 2023-06-03T20:26:11.667 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 1.3253e-05 2023-06-03T20:26:11.667 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:26:11.667 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:26:11.667 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "all_read", 2023-06-03T20:26:11.667 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:24:31.751377+0000", 2023-06-03T20:26:11.667 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 2.9513e-05 2023-06-03T20:26:11.668 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:26:11.668 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:26:11.668 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "dispatched", 2023-06-03T20:26:11.668 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:24:31.751407+0000", 2023-06-03T20:26:11.668 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 6.8129999999999998e-06 2023-06-03T20:26:11.668 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:26:11.668 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:26:11.669 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "queued_for_pg", 2023-06-03T20:26:11.669 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:24:31.751413+0000", 2023-06-03T20:26:11.669 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.022618528999999998 2023-06-03T20:26:11.669 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:26:11.669 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:26:11.669 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "reached_pg", 2023-06-03T20:26:11.669 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:24:31.774032+0000", 2023-06-03T20:26:11.669 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 2.5536000000000001e-05 2023-06-03T20:26:11.670 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:26:11.670 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:26:11.670 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "started", 2023-06-03T20:26:11.670 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:24:31.774057+0000", 2023-06-03T20:26:11.670 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.052932451999999998 2023-06-03T20:26:11.670 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:26:11.670 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:26:11.671 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "commit_sent", 2023-06-03T20:26:11.671 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:24:31.826990+0000", 2023-06-03T20:26:11.671 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 4.8875000000000002e-05 2023-06-03T20:26:11.671 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:26:11.671 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:26:11.671 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "done", 2023-06-03T20:26:11.671 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:24:31.827039+0000", 2023-06-03T20:26:11.671 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.075673506000000001 2023-06-03T20:26:11.672 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:26:11.672 INFO:teuthology.orchestra.run.smithi164.stdout: ] 2023-06-03T20:26:11.672 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:26:11.672 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:26:11.672 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:26:11.672 INFO:teuthology.orchestra.run.smithi164.stdout: "description": "osd_repop(client.36573.0:156510 13.9 e257/242 13:9ab9c7e8:::benchmark_data_smithi089_211581_object40985:head v 257'9904, mlcod=257'9901)", 2023-06-03T20:26:11.674 INFO:teuthology.orchestra.run.smithi164.stdout: "initiated_at": "2023-06-03T20:24:38.308971+0000", 2023-06-03T20:26:11.674 INFO:teuthology.orchestra.run.smithi164.stdout: "age": 93.221442349, 2023-06-03T20:26:11.674 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.062944448, 2023-06-03T20:26:11.674 INFO:teuthology.orchestra.run.smithi164.stdout: "type_data": { 2023-06-03T20:26:11.674 INFO:teuthology.orchestra.run.smithi164.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-03T20:26:11.674 INFO:teuthology.orchestra.run.smithi164.stdout: "events": [ 2023-06-03T20:26:11.674 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:26:11.675 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "initiated", 2023-06-03T20:26:11.675 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:24:38.308971+0000", 2023-06-03T20:26:11.675 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0 2023-06-03T20:26:11.675 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:26:11.675 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:26:11.675 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "throttled", 2023-06-03T20:26:11.675 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:24:38.308971+0000", 2023-06-03T20:26:11.676 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 4294967295.9999995 2023-06-03T20:26:11.676 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:26:11.676 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:26:11.676 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "header_read", 2023-06-03T20:26:11.676 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:24:38.308970+0000", 2023-06-03T20:26:11.676 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 7.1940000000000001e-06 2023-06-03T20:26:11.676 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:26:11.677 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:26:11.677 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "all_read", 2023-06-03T20:26:11.677 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:24:38.308977+0000", 2023-06-03T20:26:11.677 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 1.0047999999999999e-05 2023-06-03T20:26:11.677 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:26:11.677 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:26:11.677 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "dispatched", 2023-06-03T20:26:11.678 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:24:38.308987+0000", 2023-06-03T20:26:11.678 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 6.1210000000000003e-06 2023-06-03T20:26:11.678 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:26:11.678 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:26:11.678 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "queued_for_pg", 2023-06-03T20:26:11.678 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:24:38.308993+0000", 2023-06-03T20:26:11.678 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.003046769 2023-06-03T20:26:11.679 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:26:11.679 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:26:11.679 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "reached_pg", 2023-06-03T20:26:11.679 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:24:38.312040+0000", 2023-06-03T20:26:11.679 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 3.4465999999999998e-05 2023-06-03T20:26:11.679 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:26:11.679 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:26:11.680 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "started", 2023-06-03T20:26:11.680 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:24:38.312075+0000", 2023-06-03T20:26:11.680 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.059805166 2023-06-03T20:26:11.680 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:26:11.680 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:26:11.680 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "commit_sent", 2023-06-03T20:26:11.680 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:24:38.371880+0000", 2023-06-03T20:26:11.680 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 3.5264e-05 2023-06-03T20:26:11.681 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:26:11.681 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:26:11.681 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "done", 2023-06-03T20:26:11.681 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:24:38.371915+0000", 2023-06-03T20:26:11.681 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.062944448 2023-06-03T20:26:11.681 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:26:11.681 INFO:teuthology.orchestra.run.smithi164.stdout: ] 2023-06-03T20:26:11.682 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:26:11.682 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:26:11.682 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:26:11.682 INFO:teuthology.orchestra.run.smithi164.stdout: "description": "osd_repop(client.36573.0:172670 13.e e263/251 13:74c58e85:::benchmark_data_smithi089_211581_object57145:head v 263'10729, mlcod=246'3596)", 2023-06-03T20:26:11.682 INFO:teuthology.orchestra.run.smithi164.stdout: "initiated_at": "2023-06-03T20:25:10.179499+0000", 2023-06-03T20:26:11.682 INFO:teuthology.orchestra.run.smithi164.stdout: "age": 61.350913568000003, 2023-06-03T20:26:11.682 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.060342020000000003, 2023-06-03T20:26:11.683 INFO:teuthology.orchestra.run.smithi164.stdout: "type_data": { 2023-06-03T20:26:11.683 INFO:teuthology.orchestra.run.smithi164.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-03T20:26:11.683 INFO:teuthology.orchestra.run.smithi164.stdout: "events": [ 2023-06-03T20:26:11.683 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:26:11.683 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "initiated", 2023-06-03T20:26:11.683 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:25:10.179499+0000", 2023-06-03T20:26:11.683 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0 2023-06-03T20:26:11.684 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:26:11.684 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:26:11.684 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "throttled", 2023-06-03T20:26:11.684 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:25:10.179499+0000", 2023-06-03T20:26:11.684 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 4294967295.9999995 2023-06-03T20:26:11.684 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:26:11.684 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:26:11.684 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "header_read", 2023-06-03T20:26:11.685 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:25:10.179499+0000", 2023-06-03T20:26:11.685 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 5.2830000000000001e-06 2023-06-03T20:26:11.685 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:26:11.685 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:26:11.685 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "all_read", 2023-06-03T20:26:11.685 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:25:10.179504+0000", 2023-06-03T20:26:11.685 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 6.5209999999999999e-06 2023-06-03T20:26:11.686 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:26:11.686 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:26:11.686 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "dispatched", 2023-06-03T20:26:11.686 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:25:10.179511+0000", 2023-06-03T20:26:11.686 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 3.6229999999999999e-06 2023-06-03T20:26:11.686 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:26:11.686 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:26:11.687 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "queued_for_pg", 2023-06-03T20:26:11.687 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:25:10.179515+0000", 2023-06-03T20:26:11.687 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.0072800599999999997 2023-06-03T20:26:11.687 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:26:11.687 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:26:11.687 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "reached_pg", 2023-06-03T20:26:11.687 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:25:10.186795+0000", 2023-06-03T20:26:11.687 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 2.2135999999999999e-05 2023-06-03T20:26:11.688 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:26:11.688 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:26:11.688 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "started", 2023-06-03T20:26:11.688 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:25:10.186817+0000", 2023-06-03T20:26:11.688 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.052978250999999997 2023-06-03T20:26:11.688 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:26:11.688 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:26:11.689 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "commit_sent", 2023-06-03T20:26:11.689 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:25:10.239795+0000", 2023-06-03T20:26:11.689 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 4.6483000000000002e-05 2023-06-03T20:26:11.689 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:26:11.689 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:26:11.689 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "done", 2023-06-03T20:26:11.689 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:25:10.239841+0000", 2023-06-03T20:26:11.690 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.060342020000000003 2023-06-03T20:26:11.690 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:26:11.690 INFO:teuthology.orchestra.run.smithi164.stdout: ] 2023-06-03T20:26:11.690 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:26:11.690 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:26:11.690 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:26:11.690 INFO:teuthology.orchestra.run.smithi164.stdout: "description": "osd_repop(client.36573.0:184666 13.f e264/255 13:fef9730c:::benchmark_data_smithi089_211581_object69141:head v 264'11669, mlcod=264'11664)", 2023-06-03T20:26:11.690 INFO:teuthology.orchestra.run.smithi164.stdout: "initiated_at": "2023-06-03T20:25:29.263444+0000", 2023-06-03T20:26:11.691 INFO:teuthology.orchestra.run.smithi164.stdout: "age": 42.266969502999999, 2023-06-03T20:26:11.691 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.061756719000000002, 2023-06-03T20:26:11.691 INFO:teuthology.orchestra.run.smithi164.stdout: "type_data": { 2023-06-03T20:26:11.691 INFO:teuthology.orchestra.run.smithi164.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-03T20:26:11.691 INFO:teuthology.orchestra.run.smithi164.stdout: "events": [ 2023-06-03T20:26:11.691 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:26:11.691 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "initiated", 2023-06-03T20:26:11.692 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:25:29.263444+0000", 2023-06-03T20:26:11.692 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0 2023-06-03T20:26:11.692 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:26:11.692 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:26:11.692 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "throttled", 2023-06-03T20:26:11.692 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:25:29.263444+0000", 2023-06-03T20:26:11.692 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 4294967295.9999986 2023-06-03T20:26:11.693 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:26:11.693 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:26:11.693 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "header_read", 2023-06-03T20:26:11.693 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:25:29.263442+0000", 2023-06-03T20:26:11.693 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 1.3219e-05 2023-06-03T20:26:11.693 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:26:11.693 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:26:11.694 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "all_read", 2023-06-03T20:26:11.694 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:25:29.263455+0000", 2023-06-03T20:26:11.694 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 1.1749999999999999e-05 2023-06-03T20:26:11.694 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:26:11.694 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:26:11.694 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "dispatched", 2023-06-03T20:26:11.694 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:25:29.263467+0000", 2023-06-03T20:26:11.694 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 5.4970000000000001e-06 2023-06-03T20:26:11.695 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:26:11.695 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:26:11.695 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "queued_for_pg", 2023-06-03T20:26:11.695 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:25:29.263473+0000", 2023-06-03T20:26:11.695 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.0064721170000000003 2023-06-03T20:26:11.695 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:26:11.695 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:26:11.696 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "reached_pg", 2023-06-03T20:26:11.696 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:25:29.269945+0000", 2023-06-03T20:26:11.696 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 3.2950999999999997e-05 2023-06-03T20:26:11.696 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:26:11.696 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:26:11.696 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "started", 2023-06-03T20:26:11.696 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:25:29.269978+0000", 2023-06-03T20:26:11.697 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.055165666000000002 2023-06-03T20:26:11.697 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:26:11.697 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:26:11.697 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "commit_sent", 2023-06-03T20:26:11.697 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:25:29.325143+0000", 2023-06-03T20:26:11.697 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 5.6918e-05 2023-06-03T20:26:11.697 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:26:11.698 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:26:11.698 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "done", 2023-06-03T20:26:11.698 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:25:29.325200+0000", 2023-06-03T20:26:11.698 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.061756719000000002 2023-06-03T20:26:11.698 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:26:11.698 INFO:teuthology.orchestra.run.smithi164.stdout: ] 2023-06-03T20:26:11.698 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:26:11.699 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:26:11.699 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:26:11.699 INFO:teuthology.orchestra.run.smithi164.stdout: "description": "osd_repop(client.36573.0:205826 13.f e266/255 13:f2bcb237:::benchmark_data_smithi089_211581_object90301:head v 266'12965, mlcod=266'12961)", 2023-06-03T20:26:11.699 INFO:teuthology.orchestra.run.smithi164.stdout: "initiated_at": "2023-06-03T20:26:02.174525+0000", 2023-06-03T20:26:11.699 INFO:teuthology.orchestra.run.smithi164.stdout: "age": 9.3558875780000008, 2023-06-03T20:26:11.699 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.060141406000000001, 2023-06-03T20:26:11.699 INFO:teuthology.orchestra.run.smithi164.stdout: "type_data": { 2023-06-03T20:26:11.699 INFO:teuthology.orchestra.run.smithi164.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-03T20:26:11.700 INFO:teuthology.orchestra.run.smithi164.stdout: "events": [ 2023-06-03T20:26:11.700 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:26:11.700 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "initiated", 2023-06-03T20:26:11.700 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:26:02.174525+0000", 2023-06-03T20:26:11.700 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0 2023-06-03T20:26:11.700 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:26:11.700 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:26:11.701 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "throttled", 2023-06-03T20:26:11.701 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:26:02.174525+0000", 2023-06-03T20:26:11.701 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 4294967295.999999 2023-06-03T20:26:11.701 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:26:11.701 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:26:11.701 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "header_read", 2023-06-03T20:26:11.701 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:26:02.174525+0000", 2023-06-03T20:26:11.702 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 1.3716e-05 2023-06-03T20:26:11.702 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:26:11.702 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:26:11.702 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "all_read", 2023-06-03T20:26:11.702 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:26:02.174538+0000", 2023-06-03T20:26:11.702 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 1.1362000000000001e-05 2023-06-03T20:26:11.702 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:26:11.703 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:26:11.703 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "dispatched", 2023-06-03T20:26:11.703 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:26:02.174550+0000", 2023-06-03T20:26:11.703 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.000104039 2023-06-03T20:26:11.703 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:26:11.703 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:26:11.703 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "queued_for_pg", 2023-06-03T20:26:11.703 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:26:02.174654+0000", 2023-06-03T20:26:11.704 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.010596126000000001 2023-06-03T20:26:11.704 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:26:11.704 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:26:11.704 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "reached_pg", 2023-06-03T20:26:11.704 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:26:02.185250+0000", 2023-06-03T20:26:11.704 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 5.5470000000000003e-05 2023-06-03T20:26:11.704 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:26:11.705 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:26:11.705 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "started", 2023-06-03T20:26:11.705 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:26:02.185305+0000", 2023-06-03T20:26:11.705 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.049195160000000002 2023-06-03T20:26:11.705 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:26:11.705 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:26:11.705 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "commit_sent", 2023-06-03T20:26:11.706 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:26:02.234500+0000", 2023-06-03T20:26:11.706 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.00016646400000000001 2023-06-03T20:26:11.706 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:26:11.706 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:26:11.706 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "done", 2023-06-03T20:26:11.706 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:26:02.234667+0000", 2023-06-03T20:26:11.706 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.060141406000000001 2023-06-03T20:26:11.706 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:26:11.707 INFO:teuthology.orchestra.run.smithi164.stdout: ] 2023-06-03T20:26:11.707 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:26:11.707 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:26:11.707 INFO:teuthology.orchestra.run.smithi164.stdout: ] 2023-06-03T20:26:11.707 INFO:teuthology.orchestra.run.smithi164.stdout:} 2023-06-03T20:26:11.707 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:26:11.760 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:26:11.763 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:26:11.763 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:26:11.864 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:26:11.900 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:26:11.902 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:26:11.903 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:26:12.004 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:26:12.036 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:26:12.038 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:26:12.039 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:26:12.140 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:26:12.173 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:26:12.180 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:26:12.181 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:26:12.264 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:26:12 smithi164 bash[59462]: cluster 1685823971.0836177 mgr.y (mgr.34104) 1509 : cluster 0 pgmap v1543: 99 pgs: 6 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 2 active+recovery_wait, 7 active+recovery_wait+undersized+degraded+remapped, 6 active+undersized+degraded+remapped+backfill_wait, 76 active+clean; 1.3 GiB data, 8.0 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 613 op/s; 28729/64488 objects degraded (44.549%); 2662/64488 objects misplaced (4.128%); 4.4 MiB/s, 71 objects/s recovering 2023-06-03T20:26:12.282 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:26:12.325 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:26:12.329 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:26:12.330 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:26:12.430 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:26:12.472 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:26:12.476 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:26:12.476 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:26:12.529 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:26:12 smithi089 bash[75068]: cluster 1685823971.0836177 mgr.y (mgr.34104) 1509 : cluster 0 pgmap v1543: 99 pgs: 6 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 2 active+recovery_wait, 7 active+recovery_wait+undersized+degraded+remapped, 6 active+undersized+degraded+remapped+backfill_wait, 76 active+clean; 1.3 GiB data, 8.0 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 613 op/s; 28729/64488 objects degraded (44.549%); 2662/64488 objects misplaced (4.128%); 4.4 MiB/s, 71 objects/s recovering 2023-06-03T20:26:12.529 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:26:12 smithi089 bash[78452]: cluster 1685823971.0836177 mgr.y (mgr.34104) 1509 : cluster 0 pgmap v1543: 99 pgs: 6 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 2 active+recovery_wait, 7 active+recovery_wait+undersized+degraded+remapped, 6 active+undersized+degraded+remapped+backfill_wait, 76 active+clean; 1.3 GiB data, 8.0 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 613 op/s; 28729/64488 objects degraded (44.549%); 2662/64488 objects misplaced (4.128%); 4.4 MiB/s, 71 objects/s recovering 2023-06-03T20:26:12.578 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:26:12.619 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:26:12.623 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:26:12.624 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:26:12.716 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/mon.a/config 2023-06-03T20:26:12.717 DEBUG:teuthology.orchestra.run.smithi164:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph daemon osd.6 dump_ops_in_flight 2023-06-03T20:26:12.724 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:26:12.757 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:26:12.763 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:26:12.764 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:26:12.865 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:26:12.896 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:26:12.899 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:26:12.899 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:26:13.000 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:26:13.047 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:26:13.052 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:26:13.053 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:26:13.154 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:26:13.190 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:26:13.192 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:26:13.192 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:26:13.265 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:26:13 smithi089 bash[75068]: cluster 1685823972.982627 mon.a (mon.0) 1509 : cluster 3 Health check update: Degraded data redundancy: 28729/64488 objects degraded (44.549%), 20 pgs degraded, 14 pgs undersized (PG_DEGRADED) 2023-06-03T20:26:13.266 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:26:13 smithi164 bash[59462]: cluster 1685823972.982627 mon.a (mon.0) 1509 : cluster 3 Health check update: Degraded data redundancy: 28729/64488 objects degraded (44.549%), 20 pgs degraded, 14 pgs undersized (PG_DEGRADED) 2023-06-03T20:26:13.294 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:26:13.329 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:26:13.335 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:26:13.336 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:26:13.437 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:26:13.465 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:26:13.470 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:26:13.470 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:26:13.529 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:26:13 smithi089 bash[78452]: cluster 1685823972.982627 mon.a (mon.0) 1509 : cluster 3 Health check update: Degraded data redundancy: 28729/64488 objects degraded (44.549%), 20 pgs degraded, 14 pgs undersized (PG_DEGRADED) 2023-06-03T20:26:13.571 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:26:13.610 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:26:13.615 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:26:13.616 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:26:13.717 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:26:13.742 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/mon.a/config 2023-06-03T20:26:13.756 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:26:13.759 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:26:13.761 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:26:13.862 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:26:13.907 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:26:13.908 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:26:13.908 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:26:14.009 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:26:14.055 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:26:14.060 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:26:14.061 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:26:14.162 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:26:14.200 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:26:14.208 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:26:14.209 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:26:14.271 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:26:14 smithi164 bash[59462]: cluster 1685823973.0846994 mgr.y (mgr.34104) 1510 : cluster 0 pgmap v1544: 99 pgs: 6 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait, 7 active+recovery_wait+undersized+degraded+remapped, 6 active+undersized+degraded+remapped+backfill_wait, 76 active+clean; 1.2 GiB data, 7.7 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 645 op/s; 27702/58383 objects degraded (47.449%); 2474/58383 objects misplaced (4.238%); 6.4 MiB/s, 102 objects/s recovering 2023-06-03T20:26:14.310 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:26:14.318 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:26:14 smithi089 bash[78452]: cluster 1685823973.0846994 mgr.y (mgr.34104) 1510 : cluster 0 pgmap v1544: 99 pgs: 6 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait, 7 active+recovery_wait+undersized+degraded+remapped, 6 active+undersized+degraded+remapped+backfill_wait, 76 active+clean; 1.2 GiB data, 7.7 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 645 op/s; 27702/58383 objects degraded (47.449%); 2474/58383 objects misplaced (4.238%); 6.4 MiB/s, 102 objects/s recovering 2023-06-03T20:26:14.319 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:26:14 smithi089 bash[75068]: cluster 1685823973.0846994 mgr.y (mgr.34104) 1510 : cluster 0 pgmap v1544: 99 pgs: 6 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait, 7 active+recovery_wait+undersized+degraded+remapped, 6 active+undersized+degraded+remapped+backfill_wait, 76 active+clean; 1.2 GiB data, 7.7 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 645 op/s; 27702/58383 objects degraded (47.449%); 2474/58383 objects misplaced (4.238%); 6.4 MiB/s, 102 objects/s recovering 2023-06-03T20:26:14.335 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:26:14.338 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:26:14.338 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:26:14.439 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:26:14.477 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:26:14.482 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:26:14.483 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:26:14.584 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:26:14.640 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:26:14.645 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:26:14.646 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:26:14.747 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:26:14.764 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/mon.a/config 2023-06-03T20:26:14.779 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:26:14.785 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:26:14.786 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:26:14.887 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:26:14.915 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:26:14.917 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:26:14.918 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:26:15.019 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:26:15.047 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:26:15.052 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:26:15.053 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:26:15.154 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:26:15.195 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:26:15.197 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:26:15.198 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:26:15.299 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:26:15.346 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:26:15.350 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:26:15.351 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:26:15.452 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:26:15.481 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:26:15.483 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:26:15.484 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:26:15.585 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:26:15.615 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:26:15.618 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:26:15.619 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:26:15.720 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:26:15.746 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:26:15.748 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:26:15.749 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:26:15.850 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:26:15.876 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:26:15.879 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:26:15.880 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:26:15.982 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:26:16.015 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:26:16.017 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:26:16.018 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:26:16.119 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:26:16.156 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:26:16.158 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:26:16.159 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:26:16.260 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:26:16.297 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:26:16.303 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:26:16.304 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:26:16.405 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:26:16.436 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:26:16.444 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:26:16.445 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:26:16.507 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 20:26:16 smithi089 bash[71163]: ::ffff:172.21.15.164 - - [03/Jun/2023:20:26:16] "GET /metrics HTTP/1.1" 200 229010 "" "Prometheus/2.33.4" 2023-06-03T20:26:16.508 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:26:16 smithi089 bash[78452]: cluster 1685823975.085597 mgr.y (mgr.34104) 1511 : cluster 0 pgmap v1545: 99 pgs: 6 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 2 active+recovery_wait, 7 active+recovery_wait+undersized+degraded+remapped, 6 active+undersized+degraded+remapped+backfill_wait, 76 active+clean; 1.1 GiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 652 op/s; 26949/54711 objects degraded (49.257%); 2272/54711 objects misplaced (4.153%); 6.4 MiB/s, 102 objects/s recovering 2023-06-03T20:26:16.508 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:26:16 smithi089 bash[75068]: cluster 1685823975.085597 mgr.y (mgr.34104) 1511 : cluster 0 pgmap v1545: 99 pgs: 6 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 2 active+recovery_wait, 7 active+recovery_wait+undersized+degraded+remapped, 6 active+undersized+degraded+remapped+backfill_wait, 76 active+clean; 1.1 GiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 652 op/s; 26949/54711 objects degraded (49.257%); 2272/54711 objects misplaced (4.153%); 6.4 MiB/s, 102 objects/s recovering 2023-06-03T20:26:16.546 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:26:16.574 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:26:16.576 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:26:16.577 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:26:16.584 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:26:16 smithi164 bash[59462]: cluster 1685823975.085597 mgr.y (mgr.34104) 1511 : cluster 0 pgmap v1545: 99 pgs: 6 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 2 active+recovery_wait, 7 active+recovery_wait+undersized+degraded+remapped, 6 active+undersized+degraded+remapped+backfill_wait, 76 active+clean; 1.1 GiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 652 op/s; 26949/54711 objects degraded (49.257%); 2272/54711 objects misplaced (4.153%); 6.4 MiB/s, 102 objects/s recovering 2023-06-03T20:26:16.678 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:26:16.729 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:26:16.735 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:26:16.735 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:26:16.836 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:26:16.881 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:26:16.884 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:26:16.885 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:26:16.986 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:26:17.032 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:26:17.035 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:26:17.036 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:26:17.137 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:26:17.169 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:26:17.175 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:26:17.176 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:26:17.277 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:26:17.304 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:26:17.306 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:26:17.307 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:26:17.408 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:26:17.443 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:26:17.444 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:26:17.444 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:26:17.546 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:26:17.566 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:26:17.570 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:26:17.571 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:26:17.607 INFO:teuthology.orchestra.run.smithi164.stderr:Inferring config /var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/mon.b/config 2023-06-03T20:26:17.673 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:26:17.708 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:26:17.713 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:26:17.713 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:26:17.814 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:26:17.870 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:26:17.880 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:26:17.881 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:26:17.982 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:26:18.011 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:26:18.015 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:26:18.015 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:26:18.116 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:26:18.157 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:26:18.158 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:26:18.158 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:26:18.259 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:26:18.292 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:26:18.294 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:26:18.295 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:26:18.396 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:26:18.433 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:26:18.436 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:26:18.437 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:26:18.538 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:26:18.571 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:26:18.576 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:26:18.577 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:26:18.585 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:26:18 smithi164 bash[59462]: cluster 1685823977.0862746 mgr.y (mgr.34104) 1512 : cluster 0 pgmap v1546: 99 pgs: 6 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait, 7 active+recovery_wait+undersized+degraded+remapped, 6 active+undersized+degraded+remapped+backfill_wait, 76 active+clean; 1.1 GiB data, 7.4 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 648 op/s; 26540/51768 objects degraded (51.267%); 2088/51768 objects misplaced (4.033%); 6.4 MiB/s, 102 objects/s recovering 2023-06-03T20:26:18.585 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:26:18 smithi164 bash[59462]: cluster 1685823977.9840114 mon.a (mon.0) 1510 : cluster 3 Health check update: Degraded data redundancy: 26540/51768 objects degraded (51.267%), 20 pgs degraded, 14 pgs undersized (PG_DEGRADED) 2023-06-03T20:26:18.678 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:26:18.722 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:26:18.724 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:26:18.724 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:26:18.725 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:26:18 smithi089 bash[75068]: cluster 1685823977.0862746 mgr.y (mgr.34104) 1512 : cluster 0 pgmap v1546: 99 pgs: 6 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait, 7 active+recovery_wait+undersized+degraded+remapped, 6 active+undersized+degraded+remapped+backfill_wait, 76 active+clean; 1.1 GiB data, 7.4 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 648 op/s; 26540/51768 objects degraded (51.267%); 2088/51768 objects misplaced (4.033%); 6.4 MiB/s, 102 objects/s recovering 2023-06-03T20:26:18.725 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:26:18 smithi089 bash[75068]: cluster 1685823977.9840114 mon.a (mon.0) 1510 : cluster 3 Health check update: Degraded data redundancy: 26540/51768 objects degraded (51.267%), 20 pgs degraded, 14 pgs undersized (PG_DEGRADED) 2023-06-03T20:26:18.725 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:26:18 smithi089 bash[78452]: cluster 1685823977.0862746 mgr.y (mgr.34104) 1512 : cluster 0 pgmap v1546: 99 pgs: 6 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait, 7 active+recovery_wait+undersized+degraded+remapped, 6 active+undersized+degraded+remapped+backfill_wait, 76 active+clean; 1.1 GiB data, 7.4 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 648 op/s; 26540/51768 objects degraded (51.267%); 2088/51768 objects misplaced (4.033%); 6.4 MiB/s, 102 objects/s recovering 2023-06-03T20:26:18.725 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:26:18 smithi089 bash[78452]: cluster 1685823977.9840114 mon.a (mon.0) 1510 : cluster 3 Health check update: Degraded data redundancy: 26540/51768 objects degraded (51.267%), 20 pgs degraded, 14 pgs undersized (PG_DEGRADED) 2023-06-03T20:26:18.825 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:26:18.854 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:26:18.857 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:26:18.858 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:26:18.959 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:26:18.988 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:26:18.990 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:26:18.990 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:26:19.091 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:26:19.130 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:26:19.134 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:26:19.135 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:26:19.236 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:26:19.263 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:26:19.264 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:26:19.265 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:26:19.366 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:26:19.406 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:26:19.410 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:26:19.411 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:26:19.512 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:26:19.555 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:26:19.559 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:26:19.559 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:26:19.660 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:26:19.692 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:26:19.700 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:26:19.702 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:26:19.803 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:26:19.832 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:26:19.836 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:26:19.837 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:26:19.938 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:26:19.973 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:26:19.976 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:26:19.977 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:26:20.078 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:26:20.114 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:26:20.119 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:26:20.120 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:26:20.220 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:26:20.264 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:26:20.265 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:26:20.266 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:26:20.287 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:26:20 smithi164 bash[59462]: cluster 1685823979.0870879 mgr.y (mgr.34104) 1513 : cluster 0 pgmap v1547: 99 pgs: 6 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 2 active+recovery_wait, 7 active+recovery_wait+undersized+degraded+remapped, 6 active+undersized+degraded+remapped+backfill_wait, 76 active+clean; 982 MiB data, 7.2 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 642 op/s; 25643/47175 objects degraded (54.357%); 1896/47175 objects misplaced (4.019%); 6.6 MiB/s, 105 objects/s recovering 2023-06-03T20:26:20.366 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:26:20.395 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:26:20.398 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:26:20.399 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:26:20.500 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:26:20.501 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:26:20 smithi089 bash[75068]: cluster 1685823979.0870879 mgr.y (mgr.34104) 1513 : cluster 0 pgmap v1547: 99 pgs: 6 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 2 active+recovery_wait, 7 active+recovery_wait+undersized+degraded+remapped, 6 active+undersized+degraded+remapped+backfill_wait, 76 active+clean; 982 MiB data, 7.2 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 642 op/s; 25643/47175 objects degraded (54.357%); 1896/47175 objects misplaced (4.019%); 6.6 MiB/s, 105 objects/s recovering 2023-06-03T20:26:20.501 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:26:20 smithi089 bash[78452]: cluster 1685823979.0870879 mgr.y (mgr.34104) 1513 : cluster 0 pgmap v1547: 99 pgs: 6 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 2 active+recovery_wait, 7 active+recovery_wait+undersized+degraded+remapped, 6 active+undersized+degraded+remapped+backfill_wait, 76 active+clean; 982 MiB data, 7.2 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 642 op/s; 25643/47175 objects degraded (54.357%); 1896/47175 objects misplaced (4.019%); 6.6 MiB/s, 105 objects/s recovering 2023-06-03T20:26:20.540 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:26:20.546 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:26:20.547 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:26:20.648 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:26:20.683 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:26:20.687 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:26:20.688 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:26:20.789 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:26:20.832 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:26:20.836 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:26:20.838 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:26:20.939 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:26:20.970 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:26:20.974 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:26:20.975 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:26:21.076 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:26:21.115 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:26:21.119 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:26:21.119 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:26:21.220 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:26:21.254 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:26:21.256 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:26:21.257 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:26:21.358 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:26:21.403 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:26:21.409 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:26:21.410 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:26:21.511 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:26:21.555 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:26:21.558 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:26:21.559 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:26:21.660 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:26:21.706 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:26:21.709 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:26:21.710 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:26:21.810 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:26:21.848 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:26:21.851 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:26:21.851 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:26:21.952 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:26:21.987 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:26:21.994 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:26:21.995 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:26:22.096 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:26:22.130 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:26:22.135 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:26:22.136 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:26:22.238 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:26:22.284 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:26:22.287 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:26:22.288 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:26:22.389 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:26:22.429 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:26:22.434 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:26:22.434 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:26:22.535 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:26:22.546 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:26:22 smithi089 bash[75068]: cluster 1685823981.0877097 mgr.y (mgr.34104) 1514 : cluster 0 pgmap v1548: 99 pgs: 6 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 2 active+recovery_wait, 7 active+recovery_wait+undersized+degraded+remapped, 6 active+undersized+degraded+remapped+backfill_wait, 76 active+clean; 945 MiB data, 7.1 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 579 op/s; 25241/45429 objects degraded (55.561%); 1896/45429 objects misplaced (4.174%); 5.4 MiB/s, 86 objects/s recovering 2023-06-03T20:26:22.547 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:26:22 smithi089 bash[75068]: audit 1685823982.1470575 mon.c (mon.2) 629 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T20:26:22.547 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:26:22 smithi089 bash[78452]: cluster 1685823981.0877097 mgr.y (mgr.34104) 1514 : cluster 0 pgmap v1548: 99 pgs: 6 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 2 active+recovery_wait, 7 active+recovery_wait+undersized+degraded+remapped, 6 active+undersized+degraded+remapped+backfill_wait, 76 active+clean; 945 MiB data, 7.1 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 579 op/s; 25241/45429 objects degraded (55.561%); 1896/45429 objects misplaced (4.174%); 5.4 MiB/s, 86 objects/s recovering 2023-06-03T20:26:22.547 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:26:22 smithi089 bash[78452]: audit 1685823982.1470575 mon.c (mon.2) 629 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T20:26:22.567 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:26:22.675 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:26:22.676 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:26:22 smithi164 bash[59462]: cluster 1685823981.0877097 mgr.y (mgr.34104) 1514 : cluster 0 pgmap v1548: 99 pgs: 6 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 2 active+recovery_wait, 7 active+recovery_wait+undersized+degraded+remapped, 6 active+undersized+degraded+remapped+backfill_wait, 76 active+clean; 945 MiB data, 7.1 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 579 op/s; 25241/45429 objects degraded (55.561%); 1896/45429 objects misplaced (4.174%); 5.4 MiB/s, 86 objects/s recovering 2023-06-03T20:26:22.676 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:26:22 smithi164 bash[59462]: audit 1685823982.1470575 mon.c (mon.2) 629 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T20:26:22.677 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:26:22.780 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:26:22.824 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:26:22.829 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:26:22.830 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:26:22.930 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:26:22.961 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:26:22.964 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:26:22.964 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:26:23.065 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:26:23.095 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:26:23.101 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:26:23.102 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:26:23.203 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:26:23.238 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:26:23.246 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:26:23.247 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:26:23.348 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:26:23.387 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:26:23.390 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:26:23 smithi089 bash[75068]: cluster 1685823982.988827 mon.a (mon.0) 1511 : cluster 3 Health check update: Degraded data redundancy: 25241/45429 objects degraded (55.561%), 20 pgs degraded, 14 pgs undersized (PG_DEGRADED) 2023-06-03T20:26:23.390 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:26:23 smithi089 bash[78452]: cluster 1685823982.988827 mon.a (mon.0) 1511 : cluster 3 Health check update: Degraded data redundancy: 25241/45429 objects degraded (55.561%), 20 pgs degraded, 14 pgs undersized (PG_DEGRADED) 2023-06-03T20:26:23.391 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:26:23.392 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:26:23.493 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:26:23.502 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:26:23 smithi164 bash[59462]: cluster 1685823982.988827 mon.a (mon.0) 1511 : cluster 3 Health check update: Degraded data redundancy: 25241/45429 objects degraded (55.561%), 20 pgs degraded, 14 pgs undersized (PG_DEGRADED) 2023-06-03T20:26:23.528 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:26:23.533 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:26:23.534 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:26:23.635 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:26:23.678 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:26:23.680 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:26:23.681 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:26:23.781 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:26:23.829 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:26:23.831 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:26:23.831 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:26:23.932 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:26:23.969 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:26:23.972 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:26:23.973 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:26:24.073 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:26:24.116 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:26:24.118 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:26:24.119 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:26:24.119 INFO:teuthology.orchestra.run.smithi164.stdout:{ 2023-06-03T20:26:24.119 INFO:teuthology.orchestra.run.smithi164.stdout: "ops": [ 2023-06-03T20:26:24.119 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:26:24.120 INFO:teuthology.orchestra.run.smithi164.stdout: "description": "osd_repop(client.36573.0:219879 13.18 e268/260 13:195370dd:::benchmark_data_smithi089_211581_object104354:head v 268'12186, mlcod=247'6537)", 2023-06-03T20:26:24.120 INFO:teuthology.orchestra.run.smithi164.stdout: "initiated_at": "2023-06-03T20:26:24.104907+0000", 2023-06-03T20:26:24.120 INFO:teuthology.orchestra.run.smithi164.stdout: "age": 0.0062107250000000003, 2023-06-03T20:26:24.120 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.0062487000000000003, 2023-06-03T20:26:24.120 INFO:teuthology.orchestra.run.smithi164.stdout: "type_data": { 2023-06-03T20:26:24.120 INFO:teuthology.orchestra.run.smithi164.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-03T20:26:24.120 INFO:teuthology.orchestra.run.smithi164.stdout: "events": [ 2023-06-03T20:26:24.121 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:26:24.121 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "initiated", 2023-06-03T20:26:24.121 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:26:24.104907+0000", 2023-06-03T20:26:24.121 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0 2023-06-03T20:26:24.121 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:26:24.121 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:26:24.121 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "throttled", 2023-06-03T20:26:24.121 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:26:24.104907+0000", 2023-06-03T20:26:24.122 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 4294967295.9999986 2023-06-03T20:26:24.122 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:26:24.122 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:26:24.122 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "header_read", 2023-06-03T20:26:24.122 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:26:24.104905+0000", 2023-06-03T20:26:24.122 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 1.3101999999999999e-05 2023-06-03T20:26:24.122 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:26:24.122 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:26:24.123 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "all_read", 2023-06-03T20:26:24.123 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:26:24.104918+0000", 2023-06-03T20:26:24.123 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 3.8430000000000003e-05 2023-06-03T20:26:24.123 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:26:24.123 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:26:24.123 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "dispatched", 2023-06-03T20:26:24.123 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:26:24.104957+0000", 2023-06-03T20:26:24.123 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 8.5229999999999995e-06 2023-06-03T20:26:24.124 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:26:24.124 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:26:24.124 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "queued_for_pg", 2023-06-03T20:26:24.124 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:26:24.104965+0000", 2023-06-03T20:26:24.124 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.00016210000000000001 2023-06-03T20:26:24.124 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:26:24.124 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:26:24.125 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "reached_pg", 2023-06-03T20:26:24.125 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:26:24.105127+0000", 2023-06-03T20:26:24.125 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 5.4839000000000001e-05 2023-06-03T20:26:24.125 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:26:24.125 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:26:24.125 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "started", 2023-06-03T20:26:24.125 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:26:24.105182+0000", 2023-06-03T20:26:24.125 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.0059337870000000003 2023-06-03T20:26:24.126 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:26:24.126 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:26:24.126 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "commit_sent", 2023-06-03T20:26:24.126 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:26:24.111116+0000", 2023-06-03T20:26:24.126 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.0062091580000000002 2023-06-03T20:26:24.126 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:26:24.126 INFO:teuthology.orchestra.run.smithi164.stdout: ] 2023-06-03T20:26:24.126 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:26:24.127 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:26:24.127 INFO:teuthology.orchestra.run.smithi164.stdout: ], 2023-06-03T20:26:24.127 INFO:teuthology.orchestra.run.smithi164.stdout: "num_ops": 1 2023-06-03T20:26:24.127 INFO:teuthology.orchestra.run.smithi164.stdout:} 2023-06-03T20:26:24.219 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:26:24.254 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:26:24.256 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:26:24.256 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:26:24.334 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:26:24 smithi164 bash[59462]: cluster 1685823983.0885024 mgr.y (mgr.34104) 1515 : cluster 0 pgmap v1549: 99 pgs: 6 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait, 7 active+recovery_wait+undersized+degraded+remapped, 6 active+undersized+degraded+remapped+backfill_wait, 76 active+clean; 791 MiB data, 6.7 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 735 op/s; 23957/37995 objects degraded (63.053%); 1528/37995 objects misplaced (4.022%); 8.0 MiB/s, 127 objects/s recovering 2023-06-03T20:26:24.357 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:26:24.397 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:26:24.401 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:26:24.402 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:26:24.503 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:26:24.529 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:26:24 smithi089 bash[78452]: cluster 1685823983.0885024 mgr.y (mgr.34104) 1515 : cluster 0 pgmap v1549: 99 pgs: 6 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait, 7 active+recovery_wait+undersized+degraded+remapped, 6 active+undersized+degraded+remapped+backfill_wait, 76 active+clean; 791 MiB data, 6.7 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 735 op/s; 23957/37995 objects degraded (63.053%); 1528/37995 objects misplaced (4.022%); 8.0 MiB/s, 127 objects/s recovering 2023-06-03T20:26:24.529 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:26:24 smithi089 bash[75068]: cluster 1685823983.0885024 mgr.y (mgr.34104) 1515 : cluster 0 pgmap v1549: 99 pgs: 6 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait, 7 active+recovery_wait+undersized+degraded+remapped, 6 active+undersized+degraded+remapped+backfill_wait, 76 active+clean; 791 MiB data, 6.7 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 735 op/s; 23957/37995 objects degraded (63.053%); 1528/37995 objects misplaced (4.022%); 8.0 MiB/s, 127 objects/s recovering 2023-06-03T20:26:24.570 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:26:24.573 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:26:24.573 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:26:24.674 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:26:24.717 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:26:24.723 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:26:24.723 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:26:24.824 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:26:24.861 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:26:24.863 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:26:24.864 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:26:24.964 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:26:24.996 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:26:24.999 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:26:24.999 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:26:25.101 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:26:25.134 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:26:25.143 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:26:25.144 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:26:25.245 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:26:25.293 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:26:25.298 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:26:25.299 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:26:25.399 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:26:25.437 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:26:25.439 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:26:25.441 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:26:25.541 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:26:25.579 DEBUG:teuthology.orchestra.run.smithi164:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph daemon osd.6 dump_blocked_ops 2023-06-03T20:26:25.602 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:26:25.606 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:26:25.606 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:26:25.708 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:26:25.751 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:26:25.760 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:26:25.761 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:26:25.863 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:26:25.902 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:26:25.907 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:26:25.908 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:26:26.010 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:26:26.045 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:26:26.050 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:26:26.051 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:26:26.152 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:26:26.183 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:26:26.189 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:26:26.190 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:26:26.279 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 20:26:26 smithi089 bash[71163]: ::ffff:172.21.15.164 - - [03/Jun/2023:20:26:26] "GET /metrics HTTP/1.1" 200 229001 "" "Prometheus/2.33.4" 2023-06-03T20:26:26.291 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:26:26.297 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:26:26 smithi164 bash[59462]: cluster 1685823985.0892947 mgr.y (mgr.34104) 1516 : cluster 0 pgmap v1550: 99 pgs: 6 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait, 7 active+recovery_wait+undersized+degraded+remapped, 6 active+undersized+degraded+remapped+backfill_wait, 76 active+clean; 752 MiB data, 6.7 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 617 op/s; 23529/36162 objects degraded (65.066%); 1528/36162 objects misplaced (4.225%); 5.9 MiB/s, 94 objects/s recovering 2023-06-03T20:26:26.345 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:26:26.347 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:26:26.348 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:26:26.449 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:26:26.475 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:26:26.477 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:26:26.478 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:26:26.579 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:26:26.633 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:26:26.639 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:26:26.639 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:26:26.740 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:26:26.750 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:26:26 smithi089 bash[75068]: cluster 1685823985.0892947 mgr.y (mgr.34104) 1516 : cluster 0 pgmap v1550: 99 pgs: 6 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait, 7 active+recovery_wait+undersized+degraded+remapped, 6 active+undersized+degraded+remapped+backfill_wait, 76 active+clean; 752 MiB data, 6.7 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 617 op/s; 23529/36162 objects degraded (65.066%); 1528/36162 objects misplaced (4.225%); 5.9 MiB/s, 94 objects/s recovering 2023-06-03T20:26:26.750 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:26:26 smithi089 bash[78452]: cluster 1685823985.0892947 mgr.y (mgr.34104) 1516 : cluster 0 pgmap v1550: 99 pgs: 6 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait, 7 active+recovery_wait+undersized+degraded+remapped, 6 active+undersized+degraded+remapped+backfill_wait, 76 active+clean; 752 MiB data, 6.7 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 617 op/s; 23529/36162 objects degraded (65.066%); 1528/36162 objects misplaced (4.225%); 5.9 MiB/s, 94 objects/s recovering 2023-06-03T20:26:26.780 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:26:26.782 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:26:26.783 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:26:26.884 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:26:26.908 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:26:26.911 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:26:26.912 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:26:27.014 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:26:27.045 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:26:27.049 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:26:27.050 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:26:27.151 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:26:27.176 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:26:27.177 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:26:27.178 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:26:27.279 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:26:27.324 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:26:27.326 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:26:27.328 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:26:27.431 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:26:27.473 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:26:27.474 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:26:27.474 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:26:27.575 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:26:27.624 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:26:27.626 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:26:27.627 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:26:27.727 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:26:27.765 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:26:27.769 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:26:27.770 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:26:27.871 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:26:27.899 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:26:27.900 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:26:27.901 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:26:28.002 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:26:28.047 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:26:28.052 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:26:28.053 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:26:28.154 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:26:28.185 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:26:28.186 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:26:28.187 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:26:28.288 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:26:28.330 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:26:28.334 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:26:28.335 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:26:28.436 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:26:28.448 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:26:28 smithi089 bash[75068]: cluster 1685823987.090215 mgr.y (mgr.34104) 1517 : cluster 0 pgmap v1551: 99 pgs: 6 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait, 7 active+recovery_wait+undersized+degraded+remapped, 6 active+undersized+degraded+remapped+backfill_wait, 76 active+clean; 688 MiB data, 6.6 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 600 op/s; 23120/33087 objects degraded (69.876%); 1325/33087 objects misplaced (4.005%); 7.4 MiB/s, 118 objects/s recovering 2023-06-03T20:26:28.449 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:26:28 smithi089 bash[75068]: cluster 1685823987.9899948 mon.a (mon.0) 1512 : cluster 3 Health check update: Degraded data redundancy: 23120/33087 objects degraded (69.876%), 20 pgs degraded, 14 pgs undersized (PG_DEGRADED) 2023-06-03T20:26:28.449 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:26:28 smithi089 bash[78452]: cluster 1685823987.090215 mgr.y (mgr.34104) 1517 : cluster 0 pgmap v1551: 99 pgs: 6 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait, 7 active+recovery_wait+undersized+degraded+remapped, 6 active+undersized+degraded+remapped+backfill_wait, 76 active+clean; 688 MiB data, 6.6 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 600 op/s; 23120/33087 objects degraded (69.876%); 1325/33087 objects misplaced (4.005%); 7.4 MiB/s, 118 objects/s recovering 2023-06-03T20:26:28.449 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:26:28 smithi089 bash[78452]: cluster 1685823987.9899948 mon.a (mon.0) 1512 : cluster 3 Health check update: Degraded data redundancy: 23120/33087 objects degraded (69.876%), 20 pgs degraded, 14 pgs undersized (PG_DEGRADED) 2023-06-03T20:26:28.475 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:26:28.480 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:26:28.481 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:26:28.582 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:26:28.585 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:26:28 smithi164 bash[59462]: cluster 1685823987.090215 mgr.y (mgr.34104) 1517 : cluster 0 pgmap v1551: 99 pgs: 6 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait, 7 active+recovery_wait+undersized+degraded+remapped, 6 active+undersized+degraded+remapped+backfill_wait, 76 active+clean; 688 MiB data, 6.6 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 600 op/s; 23120/33087 objects degraded (69.876%); 1325/33087 objects misplaced (4.005%); 7.4 MiB/s, 118 objects/s recovering 2023-06-03T20:26:28.586 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:26:28 smithi164 bash[59462]: cluster 1685823987.9899948 mon.a (mon.0) 1512 : cluster 3 Health check update: Degraded data redundancy: 23120/33087 objects degraded (69.876%), 20 pgs degraded, 14 pgs undersized (PG_DEGRADED) 2023-06-03T20:26:28.631 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:26:28.635 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:26:28.636 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:26:28.737 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:26:28.783 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:26:28.785 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:26:28.786 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:26:28.887 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:26:28.919 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:26:28.921 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:26:28.922 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:26:29.023 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:26:29.067 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:26:29.071 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:26:29.072 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:26:29.170 INFO:teuthology.orchestra.run.smithi089.stdout:osd.0: {} 2023-06-03T20:26:29.173 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:26:29.202 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:26:29.205 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:26:29.206 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:26:29.216 INFO:teuthology.orchestra.run.smithi089.stdout:osd.1: {} 2023-06-03T20:26:29.267 INFO:teuthology.orchestra.run.smithi089.stdout:osd.2: {} 2023-06-03T20:26:29.307 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:26:29 smithi164 bash[59462]: audit 1685823989.1062179 mon.a (mon.0) 1513 : audit 1 from='client.? 172.21.15.89:0/1289129498' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-06-03T20:26:29.307 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:26:29 smithi164 bash[59462]: audit 1685823989.126434 mon.a (mon.0) 1514 : audit 0 from='client.? 172.21.15.89:0/843793914' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-03T20:26:29.308 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:26:29.309 INFO:teuthology.orchestra.run.smithi089.stdout:osd.3: {} 2023-06-03T20:26:29.329 INFO:teuthology.orchestra.run.smithi089.stderr:noscrub is unset 2023-06-03T20:26:29.346 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:26:29.353 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:26:29.354 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:26:29.362 INFO:teuthology.orchestra.run.smithi089.stdout:osd.4: {} 2023-06-03T20:26:29.412 INFO:teuthology.orchestra.run.smithi089.stdout:osd.5: {} 2023-06-03T20:26:29.455 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:26:29.457 INFO:teuthology.orchestra.run.smithi089.stdout:osd.6: {} 2023-06-03T20:26:29.465 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:26:29 smithi089 bash[78452]: audit 1685823989.1062179 mon.a (mon.0) 1513 : audit 1 from='client.? 172.21.15.89:0/1289129498' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-06-03T20:26:29.466 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:26:29 smithi089 bash[78452]: audit 1685823989.126434 mon.a (mon.0) 1514 : audit 0 from='client.? 172.21.15.89:0/843793914' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-03T20:26:29.466 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:26:29 smithi089 bash[75068]: audit 1685823989.1062179 mon.a (mon.0) 1513 : audit 1 from='client.? 172.21.15.89:0/1289129498' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-06-03T20:26:29.466 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:26:29 smithi089 bash[75068]: audit 1685823989.126434 mon.a (mon.0) 1514 : audit 0 from='client.? 172.21.15.89:0/843793914' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-03T20:26:29.486 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:26:29.488 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:26:29.489 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:26:29.501 INFO:teuthology.orchestra.run.smithi089.stdout:osd.7: {} 2023-06-03T20:26:29.507 INFO:teuthology.orchestra.run.smithi089.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-06-03T20:26:29.507 INFO:teuthology.orchestra.run.smithi089.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-06-03T20:26:29.508 INFO:teuthology.orchestra.run.smithi089.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-06-03T20:26:29.508 INFO:teuthology.orchestra.run.smithi089.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-06-03T20:26:29.508 INFO:teuthology.orchestra.run.smithi089.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-06-03T20:26:29.509 INFO:teuthology.orchestra.run.smithi089.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-06-03T20:26:29.509 INFO:teuthology.orchestra.run.smithi089.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-06-03T20:26:29.509 INFO:teuthology.orchestra.run.smithi089.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-06-03T20:26:29.590 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:26:29.625 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:26:29.633 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:26:29.634 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:26:29.735 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:26:29.765 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:26:29.768 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:26:29.769 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:26:29.870 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:26:29.918 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:26:29.923 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:26:29.924 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:26:30.025 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:26:30.060 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:26:30.062 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:26:30.063 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:26:30.165 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:26:30.209 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:26:30.210 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:26:30.210 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:26:30.307 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:26:30 smithi164 bash[59462]: cluster 1685823989.091462 mgr.y (mgr.34104) 1518 : cluster 0 pgmap v1552: 99 pgs: 6 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 2 active+recovery_wait, 7 active+recovery_wait+undersized+degraded+remapped, 6 active+undersized+degraded+remapped+backfill_wait, 76 active+clean; 586 MiB data, 6.3 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 654 op/s; 22197/28179 objects degraded (78.771%); 1132/28179 objects misplaced (4.017%); 6.0 MiB/s, 95 objects/s recovering 2023-06-03T20:26:30.308 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:26:30 smithi164 bash[59462]: cluster 1685823989.2982812 mon.a (mon.0) 1515 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-06-03T20:26:30.308 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:26:30 smithi164 bash[59462]: cluster 1685823989.3143122 osd.6 (osd.6) 206 : cluster 3 failed to encode map e269 with expected crc 2023-06-03T20:26:30.308 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:26:30 smithi164 bash[59462]: audit 1685823989.3167827 mon.a (mon.0) 1516 : audit 1 from='client.? 172.21.15.89:0/1289129498' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-06-03T20:26:30.308 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:26:30 smithi164 bash[59462]: cluster 1685823989.3176057 mon.a (mon.0) 1517 : cluster 0 osdmap e269: 8 total, 8 up, 8 in 2023-06-03T20:26:30.308 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:26:30 smithi164 bash[59462]: cluster 1685823989.3293319 osd.5 (osd.5) 219 : cluster 3 failed to encode map e269 with expected crc 2023-06-03T20:26:30.308 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:26:30 smithi164 bash[59462]: audit 1685823989.5070357 mon.a (mon.0) 1518 : audit 1 from='client.? 172.21.15.89:0/2110224101' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-06-03T20:26:30.311 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:26:30.331 INFO:teuthology.orchestra.run.smithi089.stderr:reweighted osd.3 to 1 (10000) 2023-06-03T20:26:30.360 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:26:30.361 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:26:30.362 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:26:30.463 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:26:30.476 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:26:30 smithi089 bash[78452]: cluster 1685823989.091462 mgr.y (mgr.34104) 1518 : cluster 0 pgmap v1552: 99 pgs: 6 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 2 active+recovery_wait, 7 active+recovery_wait+undersized+degraded+remapped, 6 active+undersized+degraded+remapped+backfill_wait, 76 active+clean; 586 MiB data, 6.3 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 654 op/s; 22197/28179 objects degraded (78.771%); 1132/28179 objects misplaced (4.017%); 6.0 MiB/s, 95 objects/s recovering 2023-06-03T20:26:30.476 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:26:30 smithi089 bash[78452]: cluster 1685823989.2982812 mon.a (mon.0) 1515 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-06-03T20:26:30.476 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:26:30 smithi089 bash[78452]: cluster 1685823989.3143122 osd.6 (osd.6) 206 : cluster 3 failed to encode map e269 with expected crc 2023-06-03T20:26:30.476 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:26:30 smithi089 bash[78452]: audit 1685823989.3167827 mon.a (mon.0) 1516 : audit 1 from='client.? 172.21.15.89:0/1289129498' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-06-03T20:26:30.477 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:26:30 smithi089 bash[78452]: cluster 1685823989.3176057 mon.a (mon.0) 1517 : cluster 0 osdmap e269: 8 total, 8 up, 8 in 2023-06-03T20:26:30.477 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:26:30 smithi089 bash[78452]: cluster 1685823989.3293319 osd.5 (osd.5) 219 : cluster 3 failed to encode map e269 with expected crc 2023-06-03T20:26:30.477 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:26:30 smithi089 bash[78452]: audit 1685823989.5070357 mon.a (mon.0) 1518 : audit 1 from='client.? 172.21.15.89:0/2110224101' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-06-03T20:26:30.477 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:26:30 smithi089 bash[75068]: cluster 1685823989.091462 mgr.y (mgr.34104) 1518 : cluster 0 pgmap v1552: 99 pgs: 6 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 2 active+recovery_wait, 7 active+recovery_wait+undersized+degraded+remapped, 6 active+undersized+degraded+remapped+backfill_wait, 76 active+clean; 586 MiB data, 6.3 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 654 op/s; 22197/28179 objects degraded (78.771%); 1132/28179 objects misplaced (4.017%); 6.0 MiB/s, 95 objects/s recovering 2023-06-03T20:26:30.480 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:26:30 smithi089 bash[75068]: cluster 1685823989.2982812 mon.a (mon.0) 1515 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-06-03T20:26:30.480 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:26:30 smithi089 bash[75068]: cluster 1685823989.3143122 osd.6 (osd.6) 206 : cluster 3 failed to encode map e269 with expected crc 2023-06-03T20:26:30.480 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:26:30 smithi089 bash[75068]: audit 1685823989.3167827 mon.a (mon.0) 1516 : audit 1 from='client.? 172.21.15.89:0/1289129498' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-06-03T20:26:30.480 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:26:30 smithi089 bash[75068]: cluster 1685823989.3176057 mon.a (mon.0) 1517 : cluster 0 osdmap e269: 8 total, 8 up, 8 in 2023-06-03T20:26:30.480 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:26:30 smithi089 bash[75068]: cluster 1685823989.3293319 osd.5 (osd.5) 219 : cluster 3 failed to encode map e269 with expected crc 2023-06-03T20:26:30.481 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:26:30 smithi089 bash[75068]: audit 1685823989.5070357 mon.a (mon.0) 1518 : audit 1 from='client.? 172.21.15.89:0/2110224101' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-06-03T20:26:30.493 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:26:30.495 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:26:30.496 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:26:30.513 INFO:teuthology.orchestra.run.smithi164.stderr:Inferring config /var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/mon.b/config 2023-06-03T20:26:30.597 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:26:30.627 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:26:30.629 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:26:30.630 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:26:30.731 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:26:30.771 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:26:30.777 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:26:30.778 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:26:30.879 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:26:30.914 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:26:30.919 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:26:30.919 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:26:31.020 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:26:31.054 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:26:31.055 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:26:31.056 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:26:31.157 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:26:31.200 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:26:31.206 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:26:31.207 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:26:31.308 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:26:31.354 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:26:31.355 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:26:31.356 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:26:31.457 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:26:31.484 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:26:31.486 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:26:31.486 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:26:31.587 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:26:31.592 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:26:31 smithi164 bash[59462]: audit 1685823990.3224447 mon.a (mon.0) 1519 : audit 1 from='client.? 172.21.15.89:0/2110224101' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2023-06-03T20:26:31.592 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:26:31 smithi164 bash[59462]: cluster 1685823990.3225245 mon.a (mon.0) 1520 : cluster 0 osdmap e270: 8 total, 8 up, 8 in 2023-06-03T20:26:31.592 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:26:31 smithi164 bash[59462]: cluster 1685823990.3413968 osd.7 (osd.7) 217 : cluster 3 failed to encode map e270 with expected crc 2023-06-03T20:26:31.592 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:26:31 smithi164 bash[59462]: cluster 1685823990.3416295 osd.5 (osd.5) 220 : cluster 3 failed to encode map e270 with expected crc 2023-06-03T20:26:31.593 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:26:31 smithi164 bash[59462]: cluster 1685823991.0922787 mgr.y (mgr.34104) 1519 : cluster 0 pgmap v1555: 99 pgs: 6 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 2 active+recovery_wait, 7 active+recovery_wait+undersized+degraded+remapped, 6 active+undersized+degraded+remapped+backfill_wait, 76 active+clean; 547 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 487 op/s; 21766/26289 objects degraded (82.795%); 1132/26289 objects misplaced (4.306%); 4.2 MiB/s, 66 objects/s recovering 2023-06-03T20:26:31.625 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:26:31.629 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:26:31.630 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:26:31.640 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:26:31 smithi089 bash[78452]: audit 1685823990.3224447 mon.a (mon.0) 1519 : audit 1 from='client.? 172.21.15.89:0/2110224101' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2023-06-03T20:26:31.640 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:26:31 smithi089 bash[78452]: cluster 1685823990.3225245 mon.a (mon.0) 1520 : cluster 0 osdmap e270: 8 total, 8 up, 8 in 2023-06-03T20:26:31.640 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:26:31 smithi089 bash[78452]: cluster 1685823990.3413968 osd.7 (osd.7) 217 : cluster 3 failed to encode map e270 with expected crc 2023-06-03T20:26:31.641 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:26:31 smithi089 bash[78452]: cluster 1685823990.3416295 osd.5 (osd.5) 220 : cluster 3 failed to encode map e270 with expected crc 2023-06-03T20:26:31.641 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:26:31 smithi089 bash[78452]: cluster 1685823991.0922787 mgr.y (mgr.34104) 1519 : cluster 0 pgmap v1555: 99 pgs: 6 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 2 active+recovery_wait, 7 active+recovery_wait+undersized+degraded+remapped, 6 active+undersized+degraded+remapped+backfill_wait, 76 active+clean; 547 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 487 op/s; 21766/26289 objects degraded (82.795%); 1132/26289 objects misplaced (4.306%); 4.2 MiB/s, 66 objects/s recovering 2023-06-03T20:26:31.641 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:26:31 smithi089 bash[75068]: audit 1685823990.3224447 mon.a (mon.0) 1519 : audit 1 from='client.? 172.21.15.89:0/2110224101' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2023-06-03T20:26:31.641 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:26:31 smithi089 bash[75068]: cluster 1685823990.3225245 mon.a (mon.0) 1520 : cluster 0 osdmap e270: 8 total, 8 up, 8 in 2023-06-03T20:26:31.641 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:26:31 smithi089 bash[75068]: cluster 1685823990.3413968 osd.7 (osd.7) 217 : cluster 3 failed to encode map e270 with expected crc 2023-06-03T20:26:31.641 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:26:31 smithi089 bash[75068]: cluster 1685823990.3416295 osd.5 (osd.5) 220 : cluster 3 failed to encode map e270 with expected crc 2023-06-03T20:26:31.642 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:26:31 smithi089 bash[75068]: cluster 1685823991.0922787 mgr.y (mgr.34104) 1519 : cluster 0 pgmap v1555: 99 pgs: 6 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 2 active+recovery_wait, 7 active+recovery_wait+undersized+degraded+remapped, 6 active+undersized+degraded+remapped+backfill_wait, 76 active+clean; 547 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 487 op/s; 21766/26289 objects degraded (82.795%); 1132/26289 objects misplaced (4.306%); 4.2 MiB/s, 66 objects/s recovering 2023-06-03T20:26:31.731 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:26:31.775 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:26:31.778 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:26:31.779 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:26:31.880 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:26:31.909 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:26:31.914 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:26:31.915 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:26:32.016 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:26:32.052 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:26:32.054 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:26:32.055 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:26:32.156 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:26:32.185 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:26:32.190 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:26:32.191 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:26:32.292 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:26:32.325 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:26:32.328 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:26:32.329 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:26:32.430 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:26:32.468 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:26:32.469 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:26:32.470 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:26:32.571 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:26:32.608 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:26:32.611 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:26:32.612 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:26:32.713 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:26:32.744 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:26:32.751 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:26:32.751 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:26:32.852 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:26:32.899 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:26:32.902 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:26:32.903 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:26:33.004 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:26:33.029 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph osd reweight 4 1 2023-06-03T20:26:33.033 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:26:33.042 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:26:33.043 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:26:33.085 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:26:33 smithi089 bash[78452]: cluster 1685823992.9911222 mon.a (mon.0) 1521 : cluster 3 Health check update: Degraded data redundancy: 21766/26289 objects degraded (82.795%), 20 pgs degraded, 14 pgs undersized (PG_DEGRADED) 2023-06-03T20:26:33.085 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:26:33 smithi089 bash[75068]: cluster 1685823992.9911222 mon.a (mon.0) 1521 : cluster 3 Health check update: Degraded data redundancy: 21766/26289 objects degraded (82.795%), 20 pgs degraded, 14 pgs undersized (PG_DEGRADED) 2023-06-03T20:26:33.144 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:26:33.186 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:26:33.188 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:26:33.189 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:26:33.289 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:26:33.328 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:26:33.335 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:26:33 smithi164 bash[59462]: cluster 1685823992.9911222 mon.a (mon.0) 1521 : cluster 3 Health check update: Degraded data redundancy: 21766/26289 objects degraded (82.795%), 20 pgs degraded, 14 pgs undersized (PG_DEGRADED) 2023-06-03T20:26:33.335 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:26:33.336 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:26:33.437 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:26:33.471 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:26:33.475 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:26:33.476 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:26:33.577 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:26:33.606 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:26:33.610 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:26:33.611 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:26:33.712 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:26:33.748 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:26:33.750 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:26:33.750 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:26:33.851 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:26:33.885 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:26:33.889 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:26:33.890 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:26:33.991 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:26:34.045 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:26:34.050 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:26:34.051 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:26:34.152 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:26:34.181 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:26:34.187 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:26:34.188 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:26:34.290 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:26:34.323 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:26:34.329 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:26:34 smithi089 bash[75068]: cluster 1685823993.093329 mgr.y (mgr.34104) 1520 : cluster 0 pgmap v1556: 99 pgs: 6 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait, 7 active+recovery_wait+undersized+degraded+remapped, 6 active+undersized+degraded+remapped+backfill_wait, 76 active+clean; 396 MiB data, 5.8 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 713 op/s; 20487/19035 objects degraded (107.628%); 731/19035 objects misplaced (3.840%); 8.1 MiB/s, 130 objects/s recovering 2023-06-03T20:26:34.329 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:26:34 smithi089 bash[78452]: cluster 1685823993.093329 mgr.y (mgr.34104) 1520 : cluster 0 pgmap v1556: 99 pgs: 6 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait, 7 active+recovery_wait+undersized+degraded+remapped, 6 active+undersized+degraded+remapped+backfill_wait, 76 active+clean; 396 MiB data, 5.8 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 713 op/s; 20487/19035 objects degraded (107.628%); 731/19035 objects misplaced (3.840%); 8.1 MiB/s, 130 objects/s recovering 2023-06-03T20:26:34.330 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:26:34.330 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:26:34.335 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:26:34 smithi164 bash[59462]: cluster 1685823993.093329 mgr.y (mgr.34104) 1520 : cluster 0 pgmap v1556: 99 pgs: 6 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait, 7 active+recovery_wait+undersized+degraded+remapped, 6 active+undersized+degraded+remapped+backfill_wait, 76 active+clean; 396 MiB data, 5.8 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 713 op/s; 20487/19035 objects degraded (107.628%); 731/19035 objects misplaced (3.840%); 8.1 MiB/s, 130 objects/s recovering 2023-06-03T20:26:34.431 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:26:34.467 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:26:34.472 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:26:34.473 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:26:34.574 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:26:34.605 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:26:34.611 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:26:34.612 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:26:34.714 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:26:34.746 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:26:34.749 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:26:34.749 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:26:34.834 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph osd unset nodeep-scrub 2023-06-03T20:26:34.850 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:26:34.889 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:26:34.896 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:26:34.896 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:26:34.997 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:26:35.010 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-06-03T20:26:35.031 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:26:35.035 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:26:35.036 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:26:35.137 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:26:35.179 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:26:35.185 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:26:35.186 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:26:35.287 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:26:35.352 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:26:35.353 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:26:35.353 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:26:35.454 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:26:35.495 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:26:35.497 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:26:35.497 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:26:35.598 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:26:35.646 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:26:35.651 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:26:35.652 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:26:35.753 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:26:35.791 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:26:35.794 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:26:35.795 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:26:35.896 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:26:35.940 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:26:35.946 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:26:35.947 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:26:36.048 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:26:36.088 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:26:36.090 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:26:36.091 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:26:36.192 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:26:36.208 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:26:36 smithi089 bash[78452]: cluster 1685823995.0939732 mgr.y (mgr.34104) 1521 : cluster 0 pgmap v1557: 99 pgs: 6 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait, 7 active+recovery_wait+undersized+degraded+remapped, 6 active+undersized+degraded+remapped+backfill_wait, 76 active+clean; 361 MiB data, 5.8 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 654 op/s; 20125/17379 objects degraded (115.801%); 731/17379 objects misplaced (4.206%); 4.9 MiB/s, 79 objects/s recovering 2023-06-03T20:26:36.209 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:26:36 smithi089 bash[75068]: cluster 1685823995.0939732 mgr.y (mgr.34104) 1521 : cluster 0 pgmap v1557: 99 pgs: 6 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait, 7 active+recovery_wait+undersized+degraded+remapped, 6 active+undersized+degraded+remapped+backfill_wait, 76 active+clean; 361 MiB data, 5.8 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 654 op/s; 20125/17379 objects degraded (115.801%); 731/17379 objects misplaced (4.206%); 4.9 MiB/s, 79 objects/s recovering 2023-06-03T20:26:36.220 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:26:36.225 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:26:36.227 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:26:36.328 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:26:36.335 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:26:36 smithi164 bash[59462]: cluster 1685823995.0939732 mgr.y (mgr.34104) 1521 : cluster 0 pgmap v1557: 99 pgs: 6 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait, 7 active+recovery_wait+undersized+degraded+remapped, 6 active+undersized+degraded+remapped+backfill_wait, 76 active+clean; 361 MiB data, 5.8 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 654 op/s; 20125/17379 objects degraded (115.801%); 731/17379 objects misplaced (4.206%); 4.9 MiB/s, 79 objects/s recovering 2023-06-03T20:26:36.365 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:26:36.369 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:26:36.370 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:26:36.471 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:26:36.505 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:26:36.510 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:26:36.510 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:26:36.529 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 20:26:36 smithi089 bash[71163]: ::ffff:172.21.15.164 - - [03/Jun/2023:20:26:36] "GET /metrics HTTP/1.1" 200 229001 "" "Prometheus/2.33.4" 2023-06-03T20:26:36.611 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:26:36.652 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:26:36.658 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:26:36.659 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:26:36.760 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:26:36.789 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:26:36.790 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:26:36.790 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:26:36.891 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:26:36.924 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:26:36.930 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:26:36.931 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:26:37.031 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:26:37.057 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:26:37.062 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:26:37.063 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:26:37.164 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:26:37.196 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:26:37.200 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:26:37.200 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:26:37.301 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:26:37.351 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:26:37.353 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:26:37.354 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:26:37.455 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:26:37.465 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:26:37 smithi089 bash[78452]: audit 1685823997.1472285 mon.c (mon.2) 630 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T20:26:37.465 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:26:37 smithi089 bash[75068]: audit 1685823997.1472285 mon.c (mon.2) 630 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T20:26:37.491 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:26:37.498 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:26:37.499 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:26:37.567 INFO:teuthology.orchestra.run.smithi164.stdout:ERROR: (22) Invalid argument 2023-06-03T20:26:37.568 INFO:teuthology.orchestra.run.smithi164.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-06-03T20:26:37.588 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:26:37 smithi164 bash[59462]: audit 1685823997.1472285 mon.c (mon.2) 630 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T20:26:37.600 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:26:37.635 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:26:37.642 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:26:37.642 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:26:37.743 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:26:37.778 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:26:37.783 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:26:37.783 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:26:37.884 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:26:37.920 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:26:37.923 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:26:37.923 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:26:37.939 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/mon.a/config 2023-06-03T20:26:38.025 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:26:38.056 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:26:38.059 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:26:38.060 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:26:38.161 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:26:38.196 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:26:38.197 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:26:38.198 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:26:38.279 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:26:38 smithi089 bash[75068]: cluster 1685823997.0948052 mgr.y (mgr.34104) 1522 : cluster 0 pgmap v1558: 99 pgs: 6 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait, 7 active+recovery_wait+undersized+degraded+remapped, 6 active+undersized+degraded+remapped+backfill_wait, 76 active+clean; 295 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 581 op/s; 19688/14217 objects degraded (138.482%); 519/14217 objects misplaced (3.651%); 7.4 MiB/s, 117 objects/s recovering 2023-06-03T20:26:38.279 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:26:38 smithi089 bash[75068]: cluster 1685823997.9921494 mon.a (mon.0) 1522 : cluster 3 Health check update: Degraded data redundancy: 19688/14217 objects degraded (138.482%), 20 pgs degraded, 14 pgs undersized (PG_DEGRADED) 2023-06-03T20:26:38.280 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:26:38 smithi089 bash[78452]: cluster 1685823997.0948052 mgr.y (mgr.34104) 1522 : cluster 0 pgmap v1558: 99 pgs: 6 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait, 7 active+recovery_wait+undersized+degraded+remapped, 6 active+undersized+degraded+remapped+backfill_wait, 76 active+clean; 295 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 581 op/s; 19688/14217 objects degraded (138.482%); 519/14217 objects misplaced (3.651%); 7.4 MiB/s, 117 objects/s recovering 2023-06-03T20:26:38.280 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:26:38 smithi089 bash[78452]: cluster 1685823997.9921494 mon.a (mon.0) 1522 : cluster 3 Health check update: Degraded data redundancy: 19688/14217 objects degraded (138.482%), 20 pgs degraded, 14 pgs undersized (PG_DEGRADED) 2023-06-03T20:26:38.299 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:26:38.335 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:26:38.337 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:26:38.338 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:26:38.439 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:26:38.452 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:26:38 smithi164 bash[59462]: cluster 1685823997.0948052 mgr.y (mgr.34104) 1522 : cluster 0 pgmap v1558: 99 pgs: 6 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait, 7 active+recovery_wait+undersized+degraded+remapped, 6 active+undersized+degraded+remapped+backfill_wait, 76 active+clean; 295 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 581 op/s; 19688/14217 objects degraded (138.482%); 519/14217 objects misplaced (3.651%); 7.4 MiB/s, 117 objects/s recovering 2023-06-03T20:26:38.453 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:26:38 smithi164 bash[59462]: cluster 1685823997.9921494 mon.a (mon.0) 1522 : cluster 3 Health check update: Degraded data redundancy: 19688/14217 objects degraded (138.482%), 20 pgs degraded, 14 pgs undersized (PG_DEGRADED) 2023-06-03T20:26:38.481 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:26:38.482 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:26:38.483 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:26:38.584 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:26:38.618 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:26:38.621 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:26:38.622 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:26:38.723 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:26:38.764 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:26:38.766 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:26:38.767 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:26:38.780 DEBUG:teuthology.orchestra.run.smithi164:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph daemon osd.6 dump_historic_ops 2023-06-03T20:26:38.869 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:26:38.908 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:26:38.911 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:26:38.912 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:26:38.973 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/mon.a/config 2023-06-03T20:26:38.978 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/mon.a/config 2023-06-03T20:26:39.013 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:26:39.051 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:26:39.055 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:26:39.055 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:26:39.156 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:26:39.197 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:26:39.202 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:26:39.202 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:26:39.303 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:26:39.357 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:26:39.362 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:26:39.363 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:26:39.464 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:26:39.493 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:26:39.496 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:26:39.497 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:26:39.598 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:26:39.652 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:26:39.657 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:26:39.657 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:26:39.758 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:26:39.795 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:26:39.802 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:26:39.803 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:26:39.904 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:26:39.947 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:26:39.955 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:26:39.956 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:26:40.058 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:26:40.106 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:26:40.110 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:26:40.111 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:26:40.178 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:26:40 smithi164 bash[59462]: cluster 1685823999.0957518 mgr.y (mgr.34104) 1523 : cluster 0 pgmap v1559: 99 pgs: 6 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 2 active+recovery_wait, 7 active+recovery_wait+undersized+degraded+remapped, 6 active+undersized+degraded+remapped+backfill_wait, 76 active+clean; 194 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 577 op/s; 18774/9345 objects degraded (200.899%); 305/9345 objects misplaced (3.264%); 6.8 MiB/s, 108 objects/s recovering 2023-06-03T20:26:40.179 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:26:40 smithi164 bash[59462]: audit 1685823999.4678075 mon.c (mon.2) 631 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-03T20:26:40.179 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:26:40 smithi164 bash[59462]: audit 1685823999.9038727 mon.a (mon.0) 1523 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:26:40.179 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:26:40 smithi164 bash[59462]: audit 1685823999.9130912 mon.a (mon.0) 1524 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:26:40.179 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:26:40 smithi164 bash[59462]: audit 1685823999.9461524 mon.a (mon.0) 1525 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:26:40.179 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:26:40 smithi164 bash[59462]: audit 1685823999.95713 mon.a (mon.0) 1526 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:26:40.212 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:26:40.270 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:26:40.272 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:26:40.273 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:26:40.375 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:26:40.397 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:26:40 smithi089 bash[78452]: cluster 1685823999.0957518 mgr.y (mgr.34104) 1523 : cluster 0 pgmap v1559: 99 pgs: 6 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 2 active+recovery_wait, 7 active+recovery_wait+undersized+degraded+remapped, 6 active+undersized+degraded+remapped+backfill_wait, 76 active+clean; 194 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 577 op/s; 18774/9345 objects degraded (200.899%); 305/9345 objects misplaced (3.264%); 6.8 MiB/s, 108 objects/s recovering 2023-06-03T20:26:40.397 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:26:40 smithi089 bash[78452]: audit 1685823999.4678075 mon.c (mon.2) 631 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-03T20:26:40.397 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:26:40 smithi089 bash[78452]: audit 1685823999.9038727 mon.a (mon.0) 1523 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:26:40.398 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:26:40 smithi089 bash[78452]: audit 1685823999.9130912 mon.a (mon.0) 1524 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:26:40.398 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:26:40 smithi089 bash[78452]: audit 1685823999.9461524 mon.a (mon.0) 1525 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:26:40.398 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:26:40 smithi089 bash[78452]: audit 1685823999.95713 mon.a (mon.0) 1526 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:26:40.398 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:26:40 smithi089 bash[75068]: cluster 1685823999.0957518 mgr.y (mgr.34104) 1523 : cluster 0 pgmap v1559: 99 pgs: 6 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 2 active+recovery_wait, 7 active+recovery_wait+undersized+degraded+remapped, 6 active+undersized+degraded+remapped+backfill_wait, 76 active+clean; 194 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 577 op/s; 18774/9345 objects degraded (200.899%); 305/9345 objects misplaced (3.264%); 6.8 MiB/s, 108 objects/s recovering 2023-06-03T20:26:40.398 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:26:40 smithi089 bash[75068]: audit 1685823999.4678075 mon.c (mon.2) 631 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-03T20:26:40.398 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:26:40 smithi089 bash[75068]: audit 1685823999.9038727 mon.a (mon.0) 1523 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:26:40.399 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:26:40 smithi089 bash[75068]: audit 1685823999.9130912 mon.a (mon.0) 1524 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:26:40.399 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:26:40 smithi089 bash[75068]: audit 1685823999.9461524 mon.a (mon.0) 1525 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:26:40.399 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:26:40 smithi089 bash[75068]: audit 1685823999.95713 mon.a (mon.0) 1526 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:26:40.413 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:26:40.416 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:26:40.417 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:26:40.518 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:26:40.553 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:26:40.563 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:26:40.563 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:26:40.665 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:26:40.709 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:26:40.715 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:26:40.715 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:26:40.816 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:26:40.853 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:26:40.857 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:26:40.858 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:26:40.959 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:26:40.997 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:26:40.999 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:26:41.000 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:26:41.100 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:26:41.126 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:26:41.130 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:26:41.131 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:26:41.232 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:26:41.276 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:26:41.279 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:26:41.280 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:26:41.381 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:26:41.411 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:26:41.414 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:26:41.415 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:26:41.516 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:26:41.562 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:26:41.566 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:26:41.567 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:26:41.659 INFO:tasks.radosbench.radosbench.0.smithi089.stdout:Removed 115522 objects 2023-06-03T20:26:41.659 INFO:tasks.radosbench.radosbench.0.smithi089.stdout:Clean up completed and total clean up time :197.039 2023-06-03T20:26:41.668 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:26:41.704 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:26:41.706 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:26:41.706 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:26:41.807 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:26:41.834 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:26:41.837 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:26:41.837 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:26:41.938 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:26:41.973 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:26:41.977 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:26:41.978 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:26:42.079 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:26:42.116 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:26:42.118 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:26:42.119 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:26:42.220 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:26:42.253 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:26:42.255 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:26:42.256 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:26:42.335 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:26:42 smithi164 bash[59462]: cluster 1685824001.0965145 mgr.y (mgr.34104) 1524 : cluster 0 pgmap v1560: 99 pgs: 6 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 2 active+recovery_wait, 7 active+recovery_wait+undersized+degraded+remapped, 6 active+undersized+degraded+remapped+backfill_wait, 76 active+clean; 154 MiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 582 op/s; 18335/7455 objects degraded (245.942%); 305/7455 objects misplaced (4.091%); 6.2 MiB/s, 98 objects/s recovering 2023-06-03T20:26:42.357 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:26:42.389 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:26:42.391 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:26:42.392 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:26:42.493 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:26:42.529 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:26:42 smithi089 bash[75068]: cluster 1685824001.0965145 mgr.y (mgr.34104) 1524 : cluster 0 pgmap v1560: 99 pgs: 6 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 2 active+recovery_wait, 7 active+recovery_wait+undersized+degraded+remapped, 6 active+undersized+degraded+remapped+backfill_wait, 76 active+clean; 154 MiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 582 op/s; 18335/7455 objects degraded (245.942%); 305/7455 objects misplaced (4.091%); 6.2 MiB/s, 98 objects/s recovering 2023-06-03T20:26:42.530 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:26:42 smithi089 bash[78452]: cluster 1685824001.0965145 mgr.y (mgr.34104) 1524 : cluster 0 pgmap v1560: 99 pgs: 6 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 2 active+recovery_wait, 7 active+recovery_wait+undersized+degraded+remapped, 6 active+undersized+degraded+remapped+backfill_wait, 76 active+clean; 154 MiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 582 op/s; 18335/7455 objects degraded (245.942%); 305/7455 objects misplaced (4.091%); 6.2 MiB/s, 98 objects/s recovering 2023-06-03T20:26:42.531 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:26:42.533 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:26:42.534 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:26:42.637 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:26:42.670 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:26:42.672 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:26:42.673 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:26:42.774 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:26:42.809 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:26:42.811 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:26:42.812 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:26:42.913 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:26:42.948 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:26:42.950 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:26:42.951 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:26:43.052 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:26:43.091 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:26:43.094 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:26:43.094 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:26:43.195 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:26:43.230 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:26:43.233 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:26:43.234 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:26:43.335 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:26:43.348 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:26:43 smithi089 bash[78452]: cluster 1685824002.9933906 mon.a (mon.0) 1527 : cluster 3 Health check update: Degraded data redundancy: 18335/7455 objects degraded (245.942%), 20 pgs degraded, 14 pgs undersized (PG_DEGRADED) 2023-06-03T20:26:43.348 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:26:43 smithi089 bash[75068]: cluster 1685824002.9933906 mon.a (mon.0) 1527 : cluster 3 Health check update: Degraded data redundancy: 18335/7455 objects degraded (245.942%), 20 pgs degraded, 14 pgs undersized (PG_DEGRADED) 2023-06-03T20:26:43.371 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:26:43.373 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:26:43.374 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:26:43.476 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:26:43.506 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:26:43.510 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:26:43.511 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:26:43.585 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:26:43 smithi164 bash[59462]: cluster 1685824002.9933906 mon.a (mon.0) 1527 : cluster 3 Health check update: Degraded data redundancy: 18335/7455 objects degraded (245.942%), 20 pgs degraded, 14 pgs undersized (PG_DEGRADED) 2023-06-03T20:26:43.611 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:26:43.642 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:26:43.642 INFO:teuthology.orchestra.run.smithi164.stderr:Inferring config /var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/mon.b/config 2023-06-03T20:26:43.643 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:26:43.644 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:26:43.745 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:26:43.779 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:26:43.781 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:26:43.784 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:26:43.884 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:26:43.911 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:26:43.914 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:26:43.914 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:26:44.015 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:26:44.043 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:26:44.045 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:26:44.046 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:26:44.147 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:26:44.184 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:26:44 smithi089 bash[75068]: cluster 1685824003.0976837 mgr.y (mgr.34104) 1525 : cluster 0 pgmap v1561: 99 pgs: 6 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+undersized+remapped, 6 active+undersized+degraded+remapped+backfill_wait, 76 active+clean; 31 MiB data, 5.0 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 704 op/s; 17278/1515 objects degraded (1140.462%); 39/1515 objects misplaced (2.574%); 7.9 MiB/s, 135 objects/s recovering 2023-06-03T20:26:44.184 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:26:44.187 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:26:44.187 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:26:44.288 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:26:44.319 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:26:44.322 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:26:44.323 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:26:44.424 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:26:44.435 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:26:44 smithi089 bash[78452]: cluster 1685824003.0976837 mgr.y (mgr.34104) 1525 : cluster 0 pgmap v1561: 99 pgs: 6 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+undersized+remapped, 6 active+undersized+degraded+remapped+backfill_wait, 76 active+clean; 31 MiB data, 5.0 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 704 op/s; 17278/1515 objects degraded (1140.462%); 39/1515 objects misplaced (2.574%); 7.9 MiB/s, 135 objects/s recovering 2023-06-03T20:26:44.456 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:26:44.459 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:26:44.459 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:26:44.560 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:26:44.585 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:26:44 smithi164 bash[59462]: cluster 1685824003.0976837 mgr.y (mgr.34104) 1525 : cluster 0 pgmap v1561: 99 pgs: 6 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+undersized+remapped, 6 active+undersized+degraded+remapped+backfill_wait, 76 active+clean; 31 MiB data, 5.0 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 704 op/s; 17278/1515 objects degraded (1140.462%); 39/1515 objects misplaced (2.574%); 7.9 MiB/s, 135 objects/s recovering 2023-06-03T20:26:44.594 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:26:44.596 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:26:44.597 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:26:44.698 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:26:44.727 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:26:44.729 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:26:44.730 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:26:44.831 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:26:44.862 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:26:44.865 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:26:44.865 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:26:44.966 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:26:44.996 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:26:44.998 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:26:44.999 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:26:45.100 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:26:45.155 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:26:45.158 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:26:45.159 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:26:45.260 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:26:45.300 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:26:45.302 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:26:45.302 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:26:45.403 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:26:45.443 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:26:45.445 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:26:45.446 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:26:45.547 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:26:45.577 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:26:45.579 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:26:45.579 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:26:45.680 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:26:45.712 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:26:45.715 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:26:45.716 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:26:45.817 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:26:45.848 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:26:45.850 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:26:45.851 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:26:45.952 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:26:45.983 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:26:45.984 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:26:45.985 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:26:46.086 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:26:46.118 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:26:46.120 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:26:46.121 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:26:46.222 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:26:46.229 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:26:46 smithi164 bash[59462]: cluster 1685824005.0990639 mgr.y (mgr.34104) 1526 : cluster 0 pgmap v1562: 99 pgs: 2 active+undersized+remapped+backfill_wait, 6 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+undersized+remapped, 4 active+undersized+degraded+remapped+backfill_wait, 76 active+clean; 15 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 526 op/s; 17108/750 objects degraded (2281.067%); 39/750 objects misplaced (5.200%); 5.3 MiB/s, 93 objects/s recovering 2023-06-03T20:26:46.230 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:26:46 smithi164 bash[59462]: audit 1685824005.9528294 mon.a (mon.0) 1528 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:26:46.230 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:26:46 smithi164 bash[59462]: audit 1685824005.9628103 mon.a (mon.0) 1529 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:26:46.247 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:26:46.250 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:26:46.251 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:26:46.351 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:26:46.378 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:26:46.380 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:26:46.381 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:26:46.470 INFO:tasks.cephadm.ceph_manager.ceph:removing pool_name unique_pool_8 2023-06-03T20:26:46.470 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph osd pool rm unique_pool_8 unique_pool_8 --yes-i-really-really-mean-it 2023-06-03T20:26:46.482 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:26:46.483 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 20:26:46 smithi089 bash[71163]: ::ffff:172.21.15.164 - - [03/Jun/2023:20:26:46] "GET /metrics HTTP/1.1" 200 229006 "" "Prometheus/2.33.4" 2023-06-03T20:26:46.485 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:26:46 smithi089 bash[75068]: cluster 1685824005.0990639 mgr.y (mgr.34104) 1526 : cluster 0 pgmap v1562: 99 pgs: 2 active+undersized+remapped+backfill_wait, 6 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+undersized+remapped, 4 active+undersized+degraded+remapped+backfill_wait, 76 active+clean; 15 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 526 op/s; 17108/750 objects degraded (2281.067%); 39/750 objects misplaced (5.200%); 5.3 MiB/s, 93 objects/s recovering 2023-06-03T20:26:46.485 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:26:46 smithi089 bash[75068]: audit 1685824005.9528294 mon.a (mon.0) 1528 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:26:46.485 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:26:46 smithi089 bash[75068]: audit 1685824005.9628103 mon.a (mon.0) 1529 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:26:46.485 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:26:46 smithi089 bash[78452]: cluster 1685824005.0990639 mgr.y (mgr.34104) 1526 : cluster 0 pgmap v1562: 99 pgs: 2 active+undersized+remapped+backfill_wait, 6 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+undersized+remapped, 4 active+undersized+degraded+remapped+backfill_wait, 76 active+clean; 15 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 526 op/s; 17108/750 objects degraded (2281.067%); 39/750 objects misplaced (5.200%); 5.3 MiB/s, 93 objects/s recovering 2023-06-03T20:26:46.485 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:26:46 smithi089 bash[78452]: audit 1685824005.9528294 mon.a (mon.0) 1528 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:26:46.485 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:26:46 smithi089 bash[78452]: audit 1685824005.9628103 mon.a (mon.0) 1529 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:26:46.510 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:26:46.512 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:26:46.513 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:26:46.614 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:26:46.648 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:26:46.651 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:26:46.651 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:26:46.752 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:26:46.777 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:26:46.780 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:26:46.781 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:26:46.882 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:26:46.918 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:26:46.922 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:26:46.922 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:26:47.023 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:26:47.047 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:26:47.049 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:26:47.050 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:26:47.152 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:26:47.178 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:26:47.180 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:26:47.181 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:26:47.281 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:26:47.311 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:26:47.314 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:26:47.315 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:26:47.416 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:26:47.451 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:26:47.453 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:26:47.454 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:26:47.555 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:26:47.588 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:26:47.589 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:26:47.590 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:26:47.691 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:26:47.723 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:26:47.725 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:26:47.726 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:26:47.828 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:26:47.857 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:26:47.859 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:26:47.859 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:26:47.960 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:26:47.993 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:26:47.996 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:26:47.997 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:26:48.097 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:26:48.122 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:26:48.125 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:26:48.125 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:26:48.226 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:26:48.250 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:26:48.254 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:26:48.255 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:26:48.356 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:26:48.374 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:26:48 smithi164 bash[59462]: cluster 1685824007.1004062 mgr.y (mgr.34104) 1527 : cluster 0 pgmap v1563: 99 pgs: 2 active+undersized+remapped+backfill_wait, 6 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+undersized+remapped, 4 active+undersized+degraded+remapped+backfill_wait, 76 active+clean; 15 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 480 op/s; 17108/750 objects degraded (2281.067%); 39/750 objects misplaced (5.200%); 5.3 MiB/s, 93 objects/s recovering 2023-06-03T20:26:48.374 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:26:48 smithi164 bash[59462]: cluster 1685824007.9952884 mon.a (mon.0) 1530 : cluster 3 Health check update: Degraded data redundancy: 17108/750 objects degraded (2281.067%), 15 pgs degraded, 14 pgs undersized (PG_DEGRADED) 2023-06-03T20:26:48.379 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:26:48.381 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:26:48.381 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:26:48.482 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:26:48.513 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:26:48.515 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:26:48.516 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:26:48.617 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:26:48.643 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:26:48.645 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:26:48.646 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:26:48.747 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:26:48.758 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:26:48 smithi089 bash[78452]: cluster 1685824007.1004062 mgr.y (mgr.34104) 1527 : cluster 0 pgmap v1563: 99 pgs: 2 active+undersized+remapped+backfill_wait, 6 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+undersized+remapped, 4 active+undersized+degraded+remapped+backfill_wait, 76 active+clean; 15 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 480 op/s; 17108/750 objects degraded (2281.067%); 39/750 objects misplaced (5.200%); 5.3 MiB/s, 93 objects/s recovering 2023-06-03T20:26:48.758 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:26:48 smithi089 bash[78452]: cluster 1685824007.9952884 mon.a (mon.0) 1530 : cluster 3 Health check update: Degraded data redundancy: 17108/750 objects degraded (2281.067%), 15 pgs degraded, 14 pgs undersized (PG_DEGRADED) 2023-06-03T20:26:48.759 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:26:48 smithi089 bash[75068]: cluster 1685824007.1004062 mgr.y (mgr.34104) 1527 : cluster 0 pgmap v1563: 99 pgs: 2 active+undersized+remapped+backfill_wait, 6 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+undersized+remapped, 4 active+undersized+degraded+remapped+backfill_wait, 76 active+clean; 15 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 480 op/s; 17108/750 objects degraded (2281.067%); 39/750 objects misplaced (5.200%); 5.3 MiB/s, 93 objects/s recovering 2023-06-03T20:26:48.759 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:26:48 smithi089 bash[75068]: cluster 1685824007.9952884 mon.a (mon.0) 1530 : cluster 3 Health check update: Degraded data redundancy: 17108/750 objects degraded (2281.067%), 15 pgs degraded, 14 pgs undersized (PG_DEGRADED) 2023-06-03T20:26:48.778 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:26:48.780 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:26:48.781 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:26:48.881 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:26:48.910 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:26:48.912 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:26:48.913 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:26:49.014 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:26:49.041 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:26:49.043 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:26:49.044 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:26:49.145 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:26:49.173 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:26:49.175 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:26:49.176 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:26:49.277 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:26:49.302 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:26:49.305 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:26:49.306 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:26:49.407 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:26:49.430 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:26:49.433 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:26:49.434 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:26:49.535 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:26:49.566 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:26:49.568 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:26:49.569 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:26:49.670 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:26:49.699 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:26:49.704 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:26:49.704 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:26:49.805 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:26:49.833 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:26:49.836 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:26:49.836 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:26:49.938 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:26:49.961 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:26:49.964 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:26:49.964 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:26:50.065 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:26:50.091 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:26:50.092 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:26:50.093 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:26:50.194 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:26:50.222 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:26:50.224 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:26:50.224 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:26:50.325 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:26:50.353 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:26:50.356 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:26:50.357 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:26:50.458 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:26:50.460 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:26:50 smithi164 bash[59462]: cluster 1685824009.1014228 mgr.y (mgr.34104) 1528 : cluster 0 pgmap v1564: 99 pgs: 2 active+undersized+remapped+backfill_wait, 6 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 2 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+undersized+remapped, 4 active+undersized+degraded+remapped+backfill_wait, 76 active+clean; 15 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 392 op/s; 16669/750 objects degraded (2222.533%); 39/750 objects misplaced (5.200%); 3.0 MiB/s, 93 objects/s recovering 2023-06-03T20:26:50.461 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:26:50 smithi164 bash[59462]: audit 1685824009.3663046 mon.a (mon.0) 1531 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:26:50.461 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:26:50 smithi164 bash[59462]: audit 1685824009.375031 mon.a (mon.0) 1532 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:26:50.461 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:26:50 smithi164 bash[59462]: audit 1685824010.1721935 mon.a (mon.0) 1533 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:26:50.461 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:26:50 smithi164 bash[59462]: audit 1685824010.1795454 mon.a (mon.0) 1534 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:26:50.461 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:26:50 smithi164 bash[59462]: audit 1685824010.1820126 mon.c (mon.2) 632 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-03T20:26:50.461 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:26:50 smithi164 bash[59462]: audit 1685824010.1827834 mon.c (mon.2) 633 : audit 1 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-03T20:26:50.461 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:26:50 smithi164 bash[59462]: audit 1685824010.1881468 mon.a (mon.0) 1535 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:26:50.487 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:26:50.490 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:26:50.490 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:26:50.591 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:26:50.603 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:26:50 smithi089 bash[78452]: cluster 1685824009.1014228 mgr.y (mgr.34104) 1528 : cluster 0 pgmap v1564: 99 pgs: 2 active+undersized+remapped+backfill_wait, 6 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 2 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+undersized+remapped, 4 active+undersized+degraded+remapped+backfill_wait, 76 active+clean; 15 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 392 op/s; 16669/750 objects degraded (2222.533%); 39/750 objects misplaced (5.200%); 3.0 MiB/s, 93 objects/s recovering 2023-06-03T20:26:50.604 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:26:50 smithi089 bash[78452]: audit 1685824009.3663046 mon.a (mon.0) 1531 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:26:50.604 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:26:50 smithi089 bash[78452]: audit 1685824009.375031 mon.a (mon.0) 1532 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:26:50.604 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:26:50 smithi089 bash[78452]: audit 1685824010.1721935 mon.a (mon.0) 1533 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:26:50.604 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:26:50 smithi089 bash[78452]: audit 1685824010.1795454 mon.a (mon.0) 1534 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:26:50.604 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:26:50 smithi089 bash[78452]: audit 1685824010.1820126 mon.c (mon.2) 632 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-03T20:26:50.604 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:26:50 smithi089 bash[78452]: audit 1685824010.1827834 mon.c (mon.2) 633 : audit 1 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-03T20:26:50.604 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:26:50 smithi089 bash[78452]: audit 1685824010.1881468 mon.a (mon.0) 1535 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:26:50.605 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:26:50 smithi089 bash[75068]: cluster 1685824009.1014228 mgr.y (mgr.34104) 1528 : cluster 0 pgmap v1564: 99 pgs: 2 active+undersized+remapped+backfill_wait, 6 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 2 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+undersized+remapped, 4 active+undersized+degraded+remapped+backfill_wait, 76 active+clean; 15 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 392 op/s; 16669/750 objects degraded (2222.533%); 39/750 objects misplaced (5.200%); 3.0 MiB/s, 93 objects/s recovering 2023-06-03T20:26:50.605 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:26:50 smithi089 bash[75068]: audit 1685824009.3663046 mon.a (mon.0) 1531 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:26:50.605 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:26:50 smithi089 bash[75068]: audit 1685824009.375031 mon.a (mon.0) 1532 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:26:50.605 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:26:50 smithi089 bash[75068]: audit 1685824010.1721935 mon.a (mon.0) 1533 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:26:50.606 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:26:50 smithi089 bash[75068]: audit 1685824010.1795454 mon.a (mon.0) 1534 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:26:50.606 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:26:50 smithi089 bash[75068]: audit 1685824010.1820126 mon.c (mon.2) 632 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-03T20:26:50.606 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:26:50 smithi089 bash[75068]: audit 1685824010.1827834 mon.c (mon.2) 633 : audit 1 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-03T20:26:50.606 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:26:50 smithi089 bash[75068]: audit 1685824010.1881468 mon.a (mon.0) 1535 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:26:50.626 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:26:50.630 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:26:50.631 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:26:50.724 INFO:teuthology.orchestra.run.smithi164.stdout:ERROR: (22) Invalid argument 2023-06-03T20:26:50.724 INFO:teuthology.orchestra.run.smithi164.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-06-03T20:26:50.732 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:26:50.763 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:26:50.766 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:26:50.767 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:26:50.867 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:26:50.899 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:26:50.904 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:26:50.905 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:26:51.006 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:26:51.042 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:26:51.044 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:26:51.045 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:26:51.146 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:26:51.178 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:26:51.180 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:26:51.181 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:26:51.212 INFO:teuthology.orchestra.run.smithi089.stderr:nodeep-scrub is unset 2023-06-03T20:26:51.245 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/mon.a/config 2023-06-03T20:26:51.282 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:26:51.289 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:26:51 smithi164 bash[59462]: audit 1685824011.1789942 mon.c (mon.2) 634 : audit 1 from='client.? 172.21.15.89:0/387618076' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-06-03T20:26:51.289 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:26:51 smithi164 bash[59462]: audit 1685824011.179491 mon.a (mon.0) 1536 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-06-03T20:26:51.308 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:26:51.311 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:26:51.311 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:26:51.412 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:26:51.437 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:26:51.440 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:26:51.440 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:26:51.529 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:26:51 smithi089 bash[78452]: audit 1685824011.1789942 mon.c (mon.2) 634 : audit 1 from='client.? 172.21.15.89:0/387618076' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-06-03T20:26:51.529 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:26:51 smithi089 bash[78452]: audit 1685824011.179491 mon.a (mon.0) 1536 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-06-03T20:26:51.529 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:26:51 smithi089 bash[75068]: audit 1685824011.1789942 mon.c (mon.2) 634 : audit 1 from='client.? 172.21.15.89:0/387618076' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-06-03T20:26:51.530 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:26:51 smithi089 bash[75068]: audit 1685824011.179491 mon.a (mon.0) 1536 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-06-03T20:26:51.541 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:26:51.569 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:26:51.572 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:26:51.573 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:26:51.674 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:26:51.699 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:26:51.702 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:26:51.702 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:26:51.803 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:26:51.831 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:26:51.832 INFO:teuthology.orchestra.run.smithi089.stdout:osd.0: {} 2023-06-03T20:26:51.833 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:26:51.834 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:26:51.854 INFO:teuthology.orchestra.run.smithi089.stdout:osd.1: {} 2023-06-03T20:26:51.874 INFO:teuthology.orchestra.run.smithi089.stdout:osd.2: {} 2023-06-03T20:26:51.896 INFO:teuthology.orchestra.run.smithi089.stdout:osd.3: {} 2023-06-03T20:26:51.915 INFO:teuthology.orchestra.run.smithi089.stdout:osd.4: {} 2023-06-03T20:26:51.935 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:26:51.937 INFO:teuthology.orchestra.run.smithi089.stdout:osd.5: {} 2023-06-03T20:26:51.956 INFO:teuthology.orchestra.run.smithi089.stdout:osd.6: {} 2023-06-03T20:26:51.971 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:26:51.973 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:26:51.974 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:26:51.975 INFO:teuthology.orchestra.run.smithi089.stdout:osd.7: {} 2023-06-03T20:26:51.977 INFO:teuthology.orchestra.run.smithi089.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-06-03T20:26:51.977 INFO:teuthology.orchestra.run.smithi089.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-06-03T20:26:51.977 INFO:teuthology.orchestra.run.smithi089.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-06-03T20:26:51.977 INFO:teuthology.orchestra.run.smithi089.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-06-03T20:26:51.978 INFO:teuthology.orchestra.run.smithi089.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-06-03T20:26:51.978 INFO:teuthology.orchestra.run.smithi089.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-06-03T20:26:51.978 INFO:teuthology.orchestra.run.smithi089.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-06-03T20:26:51.978 INFO:teuthology.orchestra.run.smithi089.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-06-03T20:26:52.074 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:26:52.104 DEBUG:teuthology.orchestra.run.smithi164:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph daemon osd.7 dump_ops_in_flight 2023-06-03T20:26:52.109 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:26:52.111 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:26:52.112 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:26:52.213 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:26:52.220 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:26:52 smithi089 bash[75068]: cluster 1685824011.1025457 mgr.y (mgr.34104) 1529 : cluster 0 pgmap v1565: 99 pgs: 2 active+undersized+remapped+backfill_wait, 6 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 2 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+undersized+remapped, 4 active+undersized+degraded+remapped+backfill_wait, 76 active+clean; 15 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 257 op/s; 16669/750 objects degraded (2222.533%); 39/750 objects misplaced (5.200%); 2.4 MiB/s, 84 objects/s recovering 2023-06-03T20:26:52.220 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:26:52 smithi089 bash[75068]: cluster 1685824011.1929078 mon.a (mon.0) 1537 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-06-03T20:26:52.221 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:26:52 smithi089 bash[75068]: audit 1685824011.2029436 mon.a (mon.0) 1538 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-06-03T20:26:52.221 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:26:52 smithi089 bash[75068]: cluster 1685824011.203165 mon.a (mon.0) 1539 : cluster 0 osdmap e271: 8 total, 8 up, 8 in 2023-06-03T20:26:52.221 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:26:52 smithi089 bash[75068]: cluster 1685824011.2114513 osd.7 (osd.7) 218 : cluster 3 failed to encode map e271 with expected crc 2023-06-03T20:26:52.221 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:26:52 smithi089 bash[75068]: audit 1685824011.8065746 mon.c (mon.2) 635 : audit 0 from='client.? 172.21.15.89:0/643271719' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-03T20:26:52.221 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:26:52 smithi089 bash[75068]: audit 1685824012.1516557 mon.a (mon.0) 1540 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:26:52.221 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:26:52 smithi089 bash[75068]: audit 1685824012.1526556 mon.c (mon.2) 636 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T20:26:52.222 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:26:52 smithi089 bash[78452]: cluster 1685824011.1025457 mgr.y (mgr.34104) 1529 : cluster 0 pgmap v1565: 99 pgs: 2 active+undersized+remapped+backfill_wait, 6 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 2 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+undersized+remapped, 4 active+undersized+degraded+remapped+backfill_wait, 76 active+clean; 15 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 257 op/s; 16669/750 objects degraded (2222.533%); 39/750 objects misplaced (5.200%); 2.4 MiB/s, 84 objects/s recovering 2023-06-03T20:26:52.222 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:26:52 smithi089 bash[78452]: cluster 1685824011.1929078 mon.a (mon.0) 1537 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-06-03T20:26:52.222 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:26:52 smithi089 bash[78452]: audit 1685824011.2029436 mon.a (mon.0) 1538 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-06-03T20:26:52.222 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:26:52 smithi089 bash[78452]: cluster 1685824011.203165 mon.a (mon.0) 1539 : cluster 0 osdmap e271: 8 total, 8 up, 8 in 2023-06-03T20:26:52.222 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:26:52 smithi089 bash[78452]: cluster 1685824011.2114513 osd.7 (osd.7) 218 : cluster 3 failed to encode map e271 with expected crc 2023-06-03T20:26:52.222 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:26:52 smithi089 bash[78452]: audit 1685824011.8065746 mon.c (mon.2) 635 : audit 0 from='client.? 172.21.15.89:0/643271719' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-03T20:26:52.222 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:26:52 smithi089 bash[78452]: audit 1685824012.1516557 mon.a (mon.0) 1540 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:26:52.223 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:26:52 smithi089 bash[78452]: audit 1685824012.1526556 mon.c (mon.2) 636 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T20:26:52.235 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:26:52.237 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:26:52.237 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:26:52.335 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:26:52 smithi164 bash[59462]: cluster 1685824011.1025457 mgr.y (mgr.34104) 1529 : cluster 0 pgmap v1565: 99 pgs: 2 active+undersized+remapped+backfill_wait, 6 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 2 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+undersized+remapped, 4 active+undersized+degraded+remapped+backfill_wait, 76 active+clean; 15 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 257 op/s; 16669/750 objects degraded (2222.533%); 39/750 objects misplaced (5.200%); 2.4 MiB/s, 84 objects/s recovering 2023-06-03T20:26:52.335 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:26:52 smithi164 bash[59462]: cluster 1685824011.1929078 mon.a (mon.0) 1537 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-06-03T20:26:52.336 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:26:52 smithi164 bash[59462]: audit 1685824011.2029436 mon.a (mon.0) 1538 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-06-03T20:26:52.336 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:26:52 smithi164 bash[59462]: cluster 1685824011.203165 mon.a (mon.0) 1539 : cluster 0 osdmap e271: 8 total, 8 up, 8 in 2023-06-03T20:26:52.336 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:26:52 smithi164 bash[59462]: cluster 1685824011.2114513 osd.7 (osd.7) 218 : cluster 3 failed to encode map e271 with expected crc 2023-06-03T20:26:52.337 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:26:52 smithi164 bash[59462]: audit 1685824011.8065746 mon.c (mon.2) 635 : audit 0 from='client.? 172.21.15.89:0/643271719' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-03T20:26:52.337 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:26:52 smithi164 bash[59462]: audit 1685824012.1516557 mon.a (mon.0) 1540 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:26:52.338 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:26:52 smithi164 bash[59462]: audit 1685824012.1526556 mon.c (mon.2) 636 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T20:26:52.338 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:26:52.366 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:26:52.368 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:26:52.368 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:26:52.469 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:26:52.503 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:26:52.505 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:26:52.506 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:26:52.607 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:26:52.640 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:26:52.643 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:26:52.643 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:26:52.744 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:26:52.776 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:26:52.779 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:26:52.780 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:26:52.880 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:26:52.904 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:26:52.906 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:26:52.907 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:26:53.008 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:26:53.042 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:26:53.045 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:26:53.045 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:26:53.146 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:26:53.176 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:26:53.179 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:26:53.180 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:26:53.218 INFO:teuthology.orchestra.run.smithi089.stderr:reweighted osd.4 to 1 (10000) 2023-06-03T20:26:53.281 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:26:53.313 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:26:53.316 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:26:53.317 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:26:53.334 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:26:53 smithi164 bash[59462]: audit 1685824012.217037 mon.a (mon.0) 1541 : audit 1 from='client.? 172.21.15.89:0/1243304136' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-06-03T20:26:53.335 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:26:53 smithi164 bash[59462]: cluster 1685824012.9969883 mon.a (mon.0) 1542 : cluster 3 Health check update: Degraded data redundancy: 16669/750 objects degraded (2222.533%), 15 pgs degraded, 14 pgs undersized (PG_DEGRADED) 2023-06-03T20:26:53.418 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:26:53.426 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:26:53 smithi089 bash[78452]: audit 1685824012.217037 mon.a (mon.0) 1541 : audit 1 from='client.? 172.21.15.89:0/1243304136' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-06-03T20:26:53.427 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:26:53 smithi089 bash[78452]: cluster 1685824012.9969883 mon.a (mon.0) 1542 : cluster 3 Health check update: Degraded data redundancy: 16669/750 objects degraded (2222.533%), 15 pgs degraded, 14 pgs undersized (PG_DEGRADED) 2023-06-03T20:26:53.427 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:26:53 smithi089 bash[75068]: audit 1685824012.217037 mon.a (mon.0) 1541 : audit 1 from='client.? 172.21.15.89:0/1243304136' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-06-03T20:26:53.427 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:26:53 smithi089 bash[75068]: cluster 1685824012.9969883 mon.a (mon.0) 1542 : cluster 3 Health check update: Degraded data redundancy: 16669/750 objects degraded (2222.533%), 15 pgs degraded, 14 pgs undersized (PG_DEGRADED) 2023-06-03T20:26:53.451 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:26:53.453 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:26:53.454 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:26:53.555 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:26:53.595 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:26:53.595 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:26:53.596 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:26:53.697 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:26:53.725 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:26:53.729 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:26:53.730 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:26:53.831 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:26:53.862 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:26:53.864 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:26:53.864 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:26:53.965 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:26:53.995 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:26:53.998 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:26:53.998 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:26:54.099 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:26:54.132 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:26:54.135 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:26:54.136 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:26:54.237 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:26:54.245 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:26:54 smithi089 bash[78452]: cluster 1685824013.1036954 mgr.y (mgr.34104) 1530 : cluster 0 pgmap v1567: 99 pgs: 3 active+undersized+remapped+backfill_wait, 6 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+undersized+remapped, 3 active+undersized+degraded+remapped+backfill_wait, 76 active+clean; 15 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 27 op/s; 15964/750 objects degraded (2128.533%); 39/750 objects misplaced (5.200%); 0 B/s, 114 objects/s recovering 2023-06-03T20:26:54.245 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:26:54 smithi089 bash[78452]: cluster 1685824013.207023 osd.7 (osd.7) 219 : cluster 3 failed to encode map e272 with expected crc 2023-06-03T20:26:54.246 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:26:54 smithi089 bash[78452]: audit 1685824013.2089467 mon.a (mon.0) 1543 : audit 1 from='client.? 172.21.15.89:0/1243304136' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2023-06-03T20:26:54.246 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:26:54 smithi089 bash[78452]: cluster 1685824013.2090685 mon.a (mon.0) 1544 : cluster 0 osdmap e272: 8 total, 8 up, 8 in 2023-06-03T20:26:54.246 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:26:54 smithi089 bash[78452]: cluster 1685824013.220844 osd.5 (osd.5) 221 : cluster 3 failed to encode map e272 with expected crc 2023-06-03T20:26:54.246 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:26:54 smithi089 bash[78452]: cluster 1685824013.2231684 osd.5 (osd.5) 222 : cluster 3 failed to encode map e272 with expected crc 2023-06-03T20:26:54.246 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:26:54 smithi089 bash[78452]: cluster 1685824013.7169516 osd.6 (osd.6) 207 : cluster 3 failed to encode map e272 with expected crc 2023-06-03T20:26:54.246 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:26:54 smithi089 bash[75068]: cluster 1685824013.1036954 mgr.y (mgr.34104) 1530 : cluster 0 pgmap v1567: 99 pgs: 3 active+undersized+remapped+backfill_wait, 6 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+undersized+remapped, 3 active+undersized+degraded+remapped+backfill_wait, 76 active+clean; 15 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 27 op/s; 15964/750 objects degraded (2128.533%); 39/750 objects misplaced (5.200%); 0 B/s, 114 objects/s recovering 2023-06-03T20:26:54.247 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:26:54 smithi089 bash[75068]: cluster 1685824013.207023 osd.7 (osd.7) 219 : cluster 3 failed to encode map e272 with expected crc 2023-06-03T20:26:54.247 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:26:54 smithi089 bash[75068]: audit 1685824013.2089467 mon.a (mon.0) 1543 : audit 1 from='client.? 172.21.15.89:0/1243304136' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2023-06-03T20:26:54.247 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:26:54 smithi089 bash[75068]: cluster 1685824013.2090685 mon.a (mon.0) 1544 : cluster 0 osdmap e272: 8 total, 8 up, 8 in 2023-06-03T20:26:54.247 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:26:54 smithi089 bash[75068]: cluster 1685824013.220844 osd.5 (osd.5) 221 : cluster 3 failed to encode map e272 with expected crc 2023-06-03T20:26:54.247 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:26:54 smithi089 bash[75068]: cluster 1685824013.2231684 osd.5 (osd.5) 222 : cluster 3 failed to encode map e272 with expected crc 2023-06-03T20:26:54.248 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:26:54 smithi089 bash[75068]: cluster 1685824013.7169516 osd.6 (osd.6) 207 : cluster 3 failed to encode map e272 with expected crc 2023-06-03T20:26:54.266 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:26:54.270 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:26:54.270 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:26:54.371 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:26:54.381 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:26:54 smithi164 bash[59462]: cluster 1685824013.1036954 mgr.y (mgr.34104) 1530 : cluster 0 pgmap v1567: 99 pgs: 3 active+undersized+remapped+backfill_wait, 6 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+undersized+remapped, 3 active+undersized+degraded+remapped+backfill_wait, 76 active+clean; 15 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 27 op/s; 15964/750 objects degraded (2128.533%); 39/750 objects misplaced (5.200%); 0 B/s, 114 objects/s recovering 2023-06-03T20:26:54.382 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:26:54 smithi164 bash[59462]: cluster 1685824013.207023 osd.7 (osd.7) 219 : cluster 3 failed to encode map e272 with expected crc 2023-06-03T20:26:54.382 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:26:54 smithi164 bash[59462]: audit 1685824013.2089467 mon.a (mon.0) 1543 : audit 1 from='client.? 172.21.15.89:0/1243304136' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2023-06-03T20:26:54.382 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:26:54 smithi164 bash[59462]: cluster 1685824013.2090685 mon.a (mon.0) 1544 : cluster 0 osdmap e272: 8 total, 8 up, 8 in 2023-06-03T20:26:54.383 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:26:54 smithi164 bash[59462]: cluster 1685824013.220844 osd.5 (osd.5) 221 : cluster 3 failed to encode map e272 with expected crc 2023-06-03T20:26:54.383 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:26:54 smithi164 bash[59462]: cluster 1685824013.2231684 osd.5 (osd.5) 222 : cluster 3 failed to encode map e272 with expected crc 2023-06-03T20:26:54.383 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:26:54 smithi164 bash[59462]: cluster 1685824013.7169516 osd.6 (osd.6) 207 : cluster 3 failed to encode map e272 with expected crc 2023-06-03T20:26:54.404 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:26:54.406 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:26:54.407 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:26:54.508 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:26:54.545 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:26:54.547 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:26:54.548 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:26:54.649 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:26:54.678 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:26:54.680 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:26:54.681 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:26:54.782 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:26:54.816 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:26:54.819 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:26:54.819 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:26:54.920 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:26:54.952 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:26:54.955 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:26:54.956 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:26:55.057 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:26:55.082 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:26:55.085 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:26:55.085 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:26:55.186 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:26:55.215 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:26:55.217 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:26:55.217 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:26:55.318 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:26:55.352 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:26:55.355 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:26:55.356 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:26:55.457 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:26:55.488 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:26:55.491 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:26:55.491 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:26:55.518 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph osd reweight 5 1 2023-06-03T20:26:55.592 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:26:55.628 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:26:55.631 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:26:55.632 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:26:55.733 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:26:55.764 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:26:55.768 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:26:55.769 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:26:55.870 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:26:55.902 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:26:55.904 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:26:55.904 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:26:56.005 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:26:56.037 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:26:56.039 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:26:56.040 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:26:56.141 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:26:56.172 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:26:56.176 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:26:56.177 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:26:56.216 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:26:56 smithi164 bash[59462]: cluster 1685824015.1042607 mgr.y (mgr.34104) 1531 : cluster 0 pgmap v1569: 99 pgs: 3 active+undersized+remapped+backfill_wait, 6 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+undersized+remapped, 3 active+undersized+degraded+remapped+backfill_wait, 76 active+clean; 15 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 15964/750 objects degraded (2128.533%); 39/750 objects misplaced (5.200%); 0 B/s, 142 objects/s recovering 2023-06-03T20:26:56.233 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:26:56 smithi089 bash[75068]: cluster 1685824015.1042607 mgr.y (mgr.34104) 1531 : cluster 0 pgmap v1569: 99 pgs: 3 active+undersized+remapped+backfill_wait, 6 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+undersized+remapped, 3 active+undersized+degraded+remapped+backfill_wait, 76 active+clean; 15 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 15964/750 objects degraded (2128.533%); 39/750 objects misplaced (5.200%); 0 B/s, 142 objects/s recovering 2023-06-03T20:26:56.233 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:26:56 smithi089 bash[78452]: cluster 1685824015.1042607 mgr.y (mgr.34104) 1531 : cluster 0 pgmap v1569: 99 pgs: 3 active+undersized+remapped+backfill_wait, 6 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+undersized+remapped, 3 active+undersized+degraded+remapped+backfill_wait, 76 active+clean; 15 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 15964/750 objects degraded (2128.533%); 39/750 objects misplaced (5.200%); 0 B/s, 142 objects/s recovering 2023-06-03T20:26:56.278 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:26:56.308 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:26:56.311 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:26:56.312 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:26:56.413 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:26:56.443 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:26:56.446 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:26:56.446 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:26:56.529 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 20:26:56 smithi089 bash[71163]: ::ffff:172.21.15.164 - - [03/Jun/2023:20:26:56] "GET /metrics HTTP/1.1" 200 228934 "" "Prometheus/2.33.4" 2023-06-03T20:26:56.541 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-06-03T20:26:56.548 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:26:56.619 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:26:56.623 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:26:56.623 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:26:56.657 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph osd set noscrub 2023-06-03T20:26:56.724 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:26:56.755 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:26:56.758 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:26:56.759 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:26:56.861 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:26:56.892 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:26:56.893 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:26:56.894 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:26:56.913 INFO:teuthology.orchestra.run.smithi164.stderr:Inferring config /var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/mon.b/config 2023-06-03T20:26:56.994 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:26:57.026 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:26:57.029 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:26:57.029 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:26:57.130 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:26:57.160 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:26:57.163 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:26:57.164 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:26:57.265 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:26:57.298 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:26:57.301 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:26:57.302 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:26:57.403 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:26:57.433 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:26:57.434 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:26:57.435 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:26:57.536 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:26:57.573 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:26:57.576 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:26:57.577 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:26:57.678 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:26:57.711 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:26:57.712 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:26:57.713 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:26:57.814 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:26:57.844 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:26:57.847 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:26:57.847 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:26:57.948 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:26:57.983 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:26:57.987 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:26:57.988 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:26:58.089 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:26:58.121 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:26:58.122 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:26:58.123 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:26:58.220 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:26:58 smithi089 bash[78452]: cluster 1685824017.105541 mgr.y (mgr.34104) 1532 : cluster 0 pgmap v1570: 99 pgs: 3 active+undersized+remapped+backfill_wait, 6 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+undersized+remapped, 3 active+undersized+degraded+remapped+backfill_wait, 76 active+clean; 15 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 15964/750 objects degraded (2128.533%); 39/750 objects misplaced (5.200%); 0 B/s, 88 objects/s recovering 2023-06-03T20:26:58.221 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:26:58 smithi089 bash[78452]: cluster 1685824017.998198 mon.a (mon.0) 1545 : cluster 3 Health check update: Degraded data redundancy: 15964/750 objects degraded (2128.533%), 14 pgs degraded, 14 pgs undersized (PG_DEGRADED) 2023-06-03T20:26:58.224 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:26:58.253 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:26:58.257 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:26:58.258 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:26:58.359 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:26:58.368 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:26:58 smithi164 bash[59462]: cluster 1685824017.105541 mgr.y (mgr.34104) 1532 : cluster 0 pgmap v1570: 99 pgs: 3 active+undersized+remapped+backfill_wait, 6 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+undersized+remapped, 3 active+undersized+degraded+remapped+backfill_wait, 76 active+clean; 15 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 15964/750 objects degraded (2128.533%); 39/750 objects misplaced (5.200%); 0 B/s, 88 objects/s recovering 2023-06-03T20:26:58.368 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:26:58 smithi164 bash[59462]: cluster 1685824017.998198 mon.a (mon.0) 1545 : cluster 3 Health check update: Degraded data redundancy: 15964/750 objects degraded (2128.533%), 14 pgs degraded, 14 pgs undersized (PG_DEGRADED) 2023-06-03T20:26:58.392 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:26:58.394 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:26:58.394 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:26:58.496 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:26:58.510 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:26:58 smithi089 bash[75068]: cluster 1685824017.105541 mgr.y (mgr.34104) 1532 : cluster 0 pgmap v1570: 99 pgs: 3 active+undersized+remapped+backfill_wait, 6 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+undersized+remapped, 3 active+undersized+degraded+remapped+backfill_wait, 76 active+clean; 15 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 15964/750 objects degraded (2128.533%); 39/750 objects misplaced (5.200%); 0 B/s, 88 objects/s recovering 2023-06-03T20:26:58.510 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:26:58 smithi089 bash[75068]: cluster 1685824017.998198 mon.a (mon.0) 1545 : cluster 3 Health check update: Degraded data redundancy: 15964/750 objects degraded (2128.533%), 14 pgs degraded, 14 pgs undersized (PG_DEGRADED) 2023-06-03T20:26:58.536 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:26:58.540 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:26:58.541 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:26:58.641 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:26:58.671 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:26:58.673 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:26:58.674 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:26:58.775 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:26:58.805 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:26:58.807 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:26:58.807 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:26:58.908 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:26:58.940 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:26:58.943 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:26:58.944 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:26:59.045 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:26:59.076 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:26:59.078 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:26:59.078 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:26:59.179 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:26:59.213 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:26:59.215 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:26:59.216 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:26:59.254 INFO:teuthology.orchestra.run.smithi089.stderr:pool 'unique_pool_8' removed 2023-06-03T20:26:59.317 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:26:59.328 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:26:59 smithi164 bash[59462]: audit 1685824019.093296 mon.c (mon.2) 637 : audit 1 from='client.? 172.21.15.89:0/2218516643' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_8", "pool2": "unique_pool_8", "yes_i_really_really_mean_it": true}]: dispatch 2023-06-03T20:26:59.329 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:26:59 smithi164 bash[59462]: audit 1685824019.0940406 mon.a (mon.0) 1546 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_8", "pool2": "unique_pool_8", "yes_i_really_really_mean_it": true}]: dispatch 2023-06-03T20:26:59.348 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:26:59.350 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:26:59.350 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:26:59.444 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:26:59 smithi089 bash[75068]: audit 1685824019.093296 mon.c (mon.2) 637 : audit 1 from='client.? 172.21.15.89:0/2218516643' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_8", "pool2": "unique_pool_8", "yes_i_really_really_mean_it": true}]: dispatch 2023-06-03T20:26:59.444 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:26:59 smithi089 bash[75068]: audit 1685824019.0940406 mon.a (mon.0) 1546 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_8", "pool2": "unique_pool_8", "yes_i_really_really_mean_it": true}]: dispatch 2023-06-03T20:26:59.444 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:26:59 smithi089 bash[78452]: audit 1685824019.093296 mon.c (mon.2) 637 : audit 1 from='client.? 172.21.15.89:0/2218516643' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_8", "pool2": "unique_pool_8", "yes_i_really_really_mean_it": true}]: dispatch 2023-06-03T20:26:59.444 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:26:59 smithi089 bash[78452]: audit 1685824019.0940406 mon.a (mon.0) 1546 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_8", "pool2": "unique_pool_8", "yes_i_really_really_mean_it": true}]: dispatch 2023-06-03T20:26:59.451 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:26:59.492 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:26:59.495 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:26:59.496 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:26:59.596 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:26:59.624 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:26:59.626 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:26:59.626 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:26:59.727 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:26:59.766 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:26:59.770 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:26:59.771 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:26:59.872 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:26:59.901 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:26:59.903 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:26:59.904 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:27:00.005 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:27:00.043 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:27:00.045 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:27:00.046 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:27:00.147 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:27:00.170 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:27:00.173 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:27:00.174 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:27:00.274 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:27:00.279 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:27:00 smithi089 bash[78452]: cluster 1685824019.1070507 mgr.y (mgr.34104) 1533 : cluster 0 pgmap v1571: 99 pgs: 3 active+undersized+remapped+backfill_wait, 6 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 2 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+undersized+remapped, 3 active+undersized+degraded+remapped+backfill_wait, 76 active+clean; 15 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 14524/750 objects degraded (1936.533%); 39/750 objects misplaced (5.200%); 0 B/s, 267 objects/s recovering 2023-06-03T20:27:00.279 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:27:00 smithi089 bash[78452]: cluster 1685824019.2278402 osd.7 (osd.7) 220 : cluster 3 failed to encode map e273 with expected crc 2023-06-03T20:27:00.279 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:27:00 smithi089 bash[78452]: audit 1685824019.2302577 mon.a (mon.0) 1547 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_8", "pool2": "unique_pool_8", "yes_i_really_really_mean_it": true}]': finished 2023-06-03T20:27:00.280 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:27:00 smithi089 bash[78452]: cluster 1685824019.2304907 mon.a (mon.0) 1548 : cluster 0 osdmap e273: 8 total, 8 up, 8 in 2023-06-03T20:27:00.280 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:27:00 smithi089 bash[78452]: cluster 1685824019.4629195 osd.5 (osd.5) 223 : cluster 3 failed to encode map e273 with expected crc 2023-06-03T20:27:00.280 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:27:00 smithi089 bash[78452]: cluster 1685824019.6632543 osd.6 (osd.6) 208 : cluster 3 failed to encode map e273 with expected crc 2023-06-03T20:27:00.280 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:27:00 smithi089 bash[75068]: cluster 1685824019.1070507 mgr.y (mgr.34104) 1533 : cluster 0 pgmap v1571: 99 pgs: 3 active+undersized+remapped+backfill_wait, 6 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 2 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+undersized+remapped, 3 active+undersized+degraded+remapped+backfill_wait, 76 active+clean; 15 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 14524/750 objects degraded (1936.533%); 39/750 objects misplaced (5.200%); 0 B/s, 267 objects/s recovering 2023-06-03T20:27:00.281 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:27:00 smithi089 bash[75068]: cluster 1685824019.2278402 osd.7 (osd.7) 220 : cluster 3 failed to encode map e273 with expected crc 2023-06-03T20:27:00.281 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:27:00 smithi089 bash[75068]: audit 1685824019.2302577 mon.a (mon.0) 1547 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_8", "pool2": "unique_pool_8", "yes_i_really_really_mean_it": true}]': finished 2023-06-03T20:27:00.281 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:27:00 smithi089 bash[75068]: cluster 1685824019.2304907 mon.a (mon.0) 1548 : cluster 0 osdmap e273: 8 total, 8 up, 8 in 2023-06-03T20:27:00.281 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:27:00 smithi089 bash[75068]: cluster 1685824019.4629195 osd.5 (osd.5) 223 : cluster 3 failed to encode map e273 with expected crc 2023-06-03T20:27:00.281 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:27:00 smithi089 bash[75068]: cluster 1685824019.6632543 osd.6 (osd.6) 208 : cluster 3 failed to encode map e273 with expected crc 2023-06-03T20:27:00.307 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:27:00.309 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:27:00.310 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:27:00.353 INFO:teuthology.task.full_sequential:In full_sequential, running task radosbench... 2023-06-03T20:27:00.354 INFO:tasks.radosbench:Beginning radosbench... 2023-06-03T20:27:00.354 INFO:tasks.cephadm.ceph_manager.ceph:creating pool_name unique_pool_9 2023-06-03T20:27:00.355 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph osd pool create unique_pool_9 16 2023-06-03T20:27:00.411 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:27:00.450 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:27:00.453 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:27:00.454 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:27:00.555 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:27:00.568 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:27:00 smithi164 bash[59462]: cluster 1685824019.1070507 mgr.y (mgr.34104) 1533 : cluster 0 pgmap v1571: 99 pgs: 3 active+undersized+remapped+backfill_wait, 6 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 2 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+undersized+remapped, 3 active+undersized+degraded+remapped+backfill_wait, 76 active+clean; 15 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 14524/750 objects degraded (1936.533%); 39/750 objects misplaced (5.200%); 0 B/s, 267 objects/s recovering 2023-06-03T20:27:00.568 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:27:00 smithi164 bash[59462]: cluster 1685824019.2278402 osd.7 (osd.7) 220 : cluster 3 failed to encode map e273 with expected crc 2023-06-03T20:27:00.569 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:27:00 smithi164 bash[59462]: audit 1685824019.2302577 mon.a (mon.0) 1547 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_8", "pool2": "unique_pool_8", "yes_i_really_really_mean_it": true}]': finished 2023-06-03T20:27:00.569 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:27:00 smithi164 bash[59462]: cluster 1685824019.2304907 mon.a (mon.0) 1548 : cluster 0 osdmap e273: 8 total, 8 up, 8 in 2023-06-03T20:27:00.569 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:27:00 smithi164 bash[59462]: cluster 1685824019.4629195 osd.5 (osd.5) 223 : cluster 3 failed to encode map e273 with expected crc 2023-06-03T20:27:00.569 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:27:00 smithi164 bash[59462]: cluster 1685824019.6632543 osd.6 (osd.6) 208 : cluster 3 failed to encode map e273 with expected crc 2023-06-03T20:27:00.590 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:27:00.593 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:27:00.594 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:27:00.696 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:27:00.732 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:27:00.736 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:27:00.737 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:27:00.822 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/mon.a/config 2023-06-03T20:27:00.823 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/mon.a/config 2023-06-03T20:27:00.838 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:27:00.867 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:27:00.871 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:27:00.871 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:27:00.972 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:27:01.011 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:27:01.013 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:27:01.014 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:27:01.115 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:27:01.151 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:27:01.154 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:27:01.155 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:27:01.256 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:27:01.283 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:27:01.285 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:27:01.286 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:27:01.387 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:27:01.401 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:27:01 smithi164 bash[59462]: cluster 1685824020.2295864 osd.7 (osd.7) 221 : cluster 3 failed to encode map e274 with expected crc 2023-06-03T20:27:01.401 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:27:01 smithi164 bash[59462]: cluster 1685824020.2320864 osd.7 (osd.7) 222 : cluster 3 failed to encode map e274 with expected crc 2023-06-03T20:27:01.401 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:27:01 smithi164 bash[59462]: cluster 1685824020.2343824 mon.a (mon.0) 1549 : cluster 0 osdmap e274: 8 total, 8 up, 8 in 2023-06-03T20:27:01.421 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:27:01.424 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:27:01.425 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:27:01.526 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:27:01.529 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:27:01 smithi089 bash[78452]: cluster 1685824020.2295864 osd.7 (osd.7) 221 : cluster 3 failed to encode map e274 with expected crc 2023-06-03T20:27:01.529 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:27:01 smithi089 bash[78452]: cluster 1685824020.2320864 osd.7 (osd.7) 222 : cluster 3 failed to encode map e274 with expected crc 2023-06-03T20:27:01.529 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:27:01 smithi089 bash[78452]: cluster 1685824020.2343824 mon.a (mon.0) 1549 : cluster 0 osdmap e274: 8 total, 8 up, 8 in 2023-06-03T20:27:01.529 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:27:01 smithi089 bash[75068]: cluster 1685824020.2295864 osd.7 (osd.7) 221 : cluster 3 failed to encode map e274 with expected crc 2023-06-03T20:27:01.530 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:27:01 smithi089 bash[75068]: cluster 1685824020.2320864 osd.7 (osd.7) 222 : cluster 3 failed to encode map e274 with expected crc 2023-06-03T20:27:01.530 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:27:01 smithi089 bash[75068]: cluster 1685824020.2343824 mon.a (mon.0) 1549 : cluster 0 osdmap e274: 8 total, 8 up, 8 in 2023-06-03T20:27:01.556 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:27:01.560 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:27:01.560 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:27:01.661 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:27:01.699 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:27:01.703 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:27:01.703 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:27:01.804 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:27:01.844 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:27:01.847 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:27:01.848 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:27:01.949 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:27:01.984 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:27:01.987 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:27:01.988 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:27:02.089 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:27:02.114 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:27:02.117 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:27:02.117 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:27:02.219 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:27:02.252 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:27:02.256 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:27:02.256 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:27:02.357 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:27:02.366 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:27:02 smithi089 bash[78452]: cluster 1685824021.1078467 mgr.y (mgr.34104) 1534 : cluster 0 pgmap v1574: 73 pgs: 2 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 70 active+clean; 582 KiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 7/72 objects degraded (9.722%) 2023-06-03T20:27:02.366 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:27:02 smithi089 bash[78452]: cluster 1685824021.241349 osd.7 (osd.7) 223 : cluster 3 failed to encode map e275 with expected crc 2023-06-03T20:27:02.367 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:27:02 smithi089 bash[78452]: cluster 1685824021.243599 mon.a (mon.0) 1550 : cluster 0 osdmap e275: 8 total, 8 up, 8 in 2023-06-03T20:27:02.367 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:27:02 smithi089 bash[75068]: cluster 1685824021.1078467 mgr.y (mgr.34104) 1534 : cluster 0 pgmap v1574: 73 pgs: 2 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 70 active+clean; 582 KiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 7/72 objects degraded (9.722%) 2023-06-03T20:27:02.367 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:27:02 smithi089 bash[75068]: cluster 1685824021.241349 osd.7 (osd.7) 223 : cluster 3 failed to encode map e275 with expected crc 2023-06-03T20:27:02.368 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:27:02 smithi089 bash[75068]: cluster 1685824021.243599 mon.a (mon.0) 1550 : cluster 0 osdmap e275: 8 total, 8 up, 8 in 2023-06-03T20:27:02.386 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:27:02.388 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:27:02.389 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:27:02.489 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:27:02.501 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:27:02 smithi164 bash[59462]: cluster 1685824021.1078467 mgr.y (mgr.34104) 1534 : cluster 0 pgmap v1574: 73 pgs: 2 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 70 active+clean; 582 KiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 7/72 objects degraded (9.722%) 2023-06-03T20:27:02.501 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:27:02 smithi164 bash[59462]: cluster 1685824021.241349 osd.7 (osd.7) 223 : cluster 3 failed to encode map e275 with expected crc 2023-06-03T20:27:02.501 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:27:02 smithi164 bash[59462]: cluster 1685824021.243599 mon.a (mon.0) 1550 : cluster 0 osdmap e275: 8 total, 8 up, 8 in 2023-06-03T20:27:02.523 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:27:02.525 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:27:02.526 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:27:02.627 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:27:02.659 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:27:02.661 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:27:02.662 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:27:02.763 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:27:02.797 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:27:02.799 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:27:02.800 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:27:02.865 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/mon.a/config 2023-06-03T20:27:02.902 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:27:02.929 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:27:02.932 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:27:02.933 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:27:03.034 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:27:03.066 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:27:03.067 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:27:03.068 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:27:03.169 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:27:03.202 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:27:03.205 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:27:03.205 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:27:03.307 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:27:03.337 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:27:03.339 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:27:03.340 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:27:03.442 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:27:03.474 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:27:03.476 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:27:03.477 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:27:03.529 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:27:03 smithi089 bash[78452]: cluster 1685824022.9997933 mon.a (mon.0) 1551 : cluster 3 Health check update: Degraded data redundancy: 7/72 objects degraded (9.722%), 3 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-06-03T20:27:03.529 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:27:03 smithi089 bash[75068]: cluster 1685824022.9997933 mon.a (mon.0) 1551 : cluster 3 Health check update: Degraded data redundancy: 7/72 objects degraded (9.722%), 3 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-06-03T20:27:03.578 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:27:03.584 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:27:03 smithi164 bash[59462]: cluster 1685824022.9997933 mon.a (mon.0) 1551 : cluster 3 Health check update: Degraded data redundancy: 7/72 objects degraded (9.722%), 3 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-06-03T20:27:03.607 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:27:03.609 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:27:03.610 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:27:03.711 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:27:03.752 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:27:03.754 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:27:03.755 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:27:03.856 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:27:03.885 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:27:03.887 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:27:03.887 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:27:03.988 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:27:04.027 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:27:04.029 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:27:04.030 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:27:04.131 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:27:04.161 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:27:04.163 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:27:04.164 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:27:04.265 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:27:04.295 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:27:04.297 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:27:04.298 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:27:04.399 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:27:04.416 INFO:teuthology.orchestra.run.smithi164.stdout:{ 2023-06-03T20:27:04.416 INFO:teuthology.orchestra.run.smithi164.stdout: "ops": [], 2023-06-03T20:27:04.416 INFO:teuthology.orchestra.run.smithi164.stdout: "num_ops": 0 2023-06-03T20:27:04.417 INFO:teuthology.orchestra.run.smithi164.stdout:} 2023-06-03T20:27:04.435 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:27:04.438 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:27:04.439 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:27:04.529 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:27:04 smithi089 bash[75068]: cluster 1685824023.1088953 mgr.y (mgr.34104) 1535 : cluster 0 pgmap v1576: 73 pgs: 73 active+clean; 582 KiB data, 5.0 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:27:04.529 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:27:04 smithi089 bash[75068]: cluster 1685824024.001901 mon.a (mon.0) 1552 : cluster 1 Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 7/72 objects degraded (9.722%), 3 pgs degraded, 1 pg undersized) 2023-06-03T20:27:04.529 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:27:04 smithi089 bash[75068]: cluster 1685824024.0019445 mon.a (mon.0) 1553 : cluster 1 Cluster is now healthy 2023-06-03T20:27:04.530 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:27:04 smithi089 bash[78452]: cluster 1685824023.1088953 mgr.y (mgr.34104) 1535 : cluster 0 pgmap v1576: 73 pgs: 73 active+clean; 582 KiB data, 5.0 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:27:04.530 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:27:04 smithi089 bash[78452]: cluster 1685824024.001901 mon.a (mon.0) 1552 : cluster 1 Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 7/72 objects degraded (9.722%), 3 pgs degraded, 1 pg undersized) 2023-06-03T20:27:04.530 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:27:04 smithi089 bash[78452]: cluster 1685824024.0019445 mon.a (mon.0) 1553 : cluster 1 Cluster is now healthy 2023-06-03T20:27:04.539 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:27:04.552 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:27:04 smithi164 bash[59462]: cluster 1685824023.1088953 mgr.y (mgr.34104) 1535 : cluster 0 pgmap v1576: 73 pgs: 73 active+clean; 582 KiB data, 5.0 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:27:04.553 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:27:04 smithi164 bash[59462]: cluster 1685824024.001901 mon.a (mon.0) 1552 : cluster 1 Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 7/72 objects degraded (9.722%), 3 pgs degraded, 1 pg undersized) 2023-06-03T20:27:04.553 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:27:04 smithi164 bash[59462]: cluster 1685824024.0019445 mon.a (mon.0) 1553 : cluster 1 Cluster is now healthy 2023-06-03T20:27:04.575 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:27:04.577 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:27:04.578 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:27:04.679 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:27:04.717 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:27:04.719 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:27:04.719 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:27:04.820 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:27:04.851 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:27:04.854 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:27:04.854 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:27:04.902 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/mon.a/config 2023-06-03T20:27:04.955 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:27:04.983 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:27:04.984 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:27:04.985 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:27:05.086 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:27:05.116 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:27:05.119 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:27:05.120 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:27:05.221 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:27:05.259 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:27:05.262 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:27:05.263 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:27:05.364 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:27:05.394 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:27:05.396 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:27:05.397 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:27:05.439 DEBUG:teuthology.orchestra.run.smithi164:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph daemon osd.7 dump_blocked_ops 2023-06-03T20:27:05.498 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:27:05.530 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:27:05.532 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:27:05.533 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:27:05.634 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:27:05.664 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:27:05.667 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:27:05.668 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:27:05.769 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:27:05.800 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:27:05.803 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:27:05.804 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:27:05.904 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:27:05.933 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:27:05.935 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:27:05.936 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:27:06.037 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:27:06.073 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:27:06.075 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:27:06.076 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:27:06.177 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:27:06.211 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:27:06.213 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:27:06.214 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:27:06.315 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:27:06.326 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:27:06 smithi164 bash[59462]: cluster 1685824025.1093028 mgr.y (mgr.34104) 1536 : cluster 0 pgmap v1577: 73 pgs: 73 active+clean; 582 KiB data, 5.0 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:27:06.351 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:27:06.355 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:27:06.356 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:27:06.456 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:27:06.494 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:27:06.497 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:27:06.498 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:27:06.529 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:27:06 smithi089 bash[75068]: cluster 1685824025.1093028 mgr.y (mgr.34104) 1536 : cluster 0 pgmap v1577: 73 pgs: 73 active+clean; 582 KiB data, 5.0 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:27:06.529 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:27:06 smithi089 bash[78452]: cluster 1685824025.1093028 mgr.y (mgr.34104) 1536 : cluster 0 pgmap v1577: 73 pgs: 73 active+clean; 582 KiB data, 5.0 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:27:06.530 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 20:27:06 smithi089 bash[71163]: ::ffff:172.21.15.164 - - [03/Jun/2023:20:27:06] "GET /metrics HTTP/1.1" 200 228934 "" "Prometheus/2.33.4" 2023-06-03T20:27:06.599 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:27:06.628 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:27:06.631 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:27:06.631 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:27:06.732 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:27:06.770 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:27:06.773 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:27:06.774 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:27:06.875 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:27:06.907 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:27:06.909 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:27:06.910 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:27:07.011 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:27:07.050 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:27:07.052 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:27:07.052 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:27:07.153 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:27:07.194 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:27:07.197 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:27:07.198 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:27:07.299 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:27:07.331 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:27:07.334 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:27:07.334 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:27:07.435 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:27:07.474 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:27:07.477 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:27:07.478 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:27:07.579 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:27:07.605 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:27:07.607 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:27:07.608 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:27:07.709 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:27:07.748 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:27:07.750 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:27:07.751 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:27:07.852 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:27:07.892 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:27:07.896 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:27:07.896 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:27:07.997 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:27:08.029 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:27:08.032 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:27:08.034 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:27:08.134 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:27:08.175 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:27:08.177 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:27:08.178 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:27:08.279 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:27:08.313 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:27:08.315 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:27:08.316 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:27:08.335 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:27:08 smithi164 bash[59462]: cluster 1685824027.109941 mgr.y (mgr.34104) 1537 : cluster 0 pgmap v1578: 73 pgs: 73 active+clean; 582 KiB data, 5.0 GiB used, 710 GiB / 715 GiB avail; 73 KiB/s, 0 objects/s recovering 2023-06-03T20:27:08.335 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:27:08 smithi164 bash[59462]: audit 1685824027.1525242 mon.a (mon.0) 1554 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:27:08.335 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:27:08 smithi164 bash[59462]: audit 1685824027.1564095 mon.c (mon.2) 638 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T20:27:08.417 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:27:08.430 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:27:08 smithi089 bash[78452]: cluster 1685824027.109941 mgr.y (mgr.34104) 1537 : cluster 0 pgmap v1578: 73 pgs: 73 active+clean; 582 KiB data, 5.0 GiB used, 710 GiB / 715 GiB avail; 73 KiB/s, 0 objects/s recovering 2023-06-03T20:27:08.430 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:27:08 smithi089 bash[78452]: audit 1685824027.1525242 mon.a (mon.0) 1554 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:27:08.430 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:27:08 smithi089 bash[78452]: audit 1685824027.1564095 mon.c (mon.2) 638 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T20:27:08.430 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:27:08 smithi089 bash[75068]: cluster 1685824027.109941 mgr.y (mgr.34104) 1537 : cluster 0 pgmap v1578: 73 pgs: 73 active+clean; 582 KiB data, 5.0 GiB used, 710 GiB / 715 GiB avail; 73 KiB/s, 0 objects/s recovering 2023-06-03T20:27:08.431 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:27:08 smithi089 bash[75068]: audit 1685824027.1525242 mon.a (mon.0) 1554 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:27:08.431 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:27:08 smithi089 bash[75068]: audit 1685824027.1564095 mon.c (mon.2) 638 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T20:27:08.448 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:27:08.450 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:27:08.451 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:27:08.551 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:27:08.579 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:27:08.582 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:27:08.583 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:27:08.684 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:27:08.713 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:27:08.715 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:27:08.716 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:27:08.817 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:27:08.859 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:27:08.861 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:27:08.862 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:27:08.963 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:27:08.991 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:27:08.993 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:27:08.993 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:27:09.094 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:27:09.127 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:27:09.129 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:27:09.130 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:27:09.231 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:27:09.264 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:27:09.267 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:27:09.268 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:27:09.369 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:27:09.396 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:27:09.399 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:27:09.400 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:27:09.500 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:27:09.533 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:27:09.536 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:27:09.536 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:27:09.637 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:27:09.671 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:27:09.674 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:27:09.674 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:27:09.775 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:27:09.799 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:27:09.801 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:27:09.801 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:27:09.902 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:27:09.940 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:27:09.943 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:27:09.944 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:27:10.045 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:27:10.074 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:27:10.076 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:27:10.077 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:27:10.178 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:27:10.206 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:27:10.208 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:27:10.208 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:27:10.254 INFO:teuthology.orchestra.run.smithi164.stderr:Inferring config /var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/mon.b/config 2023-06-03T20:27:10.289 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:27:10 smithi164 bash[59462]: cluster 1685824029.1105928 mgr.y (mgr.34104) 1538 : cluster 0 pgmap v1579: 73 pgs: 73 active+clean; 582 KiB data, 5.0 GiB used, 710 GiB / 715 GiB avail; 65 KiB/s, 0 objects/s recovering 2023-06-03T20:27:10.309 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:27:10.336 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:27:10.339 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:27:10.340 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:27:10.441 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:27:10.474 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:27:10.477 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:27:10.477 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:27:10.529 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:27:10 smithi089 bash[75068]: cluster 1685824029.1105928 mgr.y (mgr.34104) 1538 : cluster 0 pgmap v1579: 73 pgs: 73 active+clean; 582 KiB data, 5.0 GiB used, 710 GiB / 715 GiB avail; 65 KiB/s, 0 objects/s recovering 2023-06-03T20:27:10.529 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:27:10 smithi089 bash[78452]: cluster 1685824029.1105928 mgr.y (mgr.34104) 1538 : cluster 0 pgmap v1579: 73 pgs: 73 active+clean; 582 KiB data, 5.0 GiB used, 710 GiB / 715 GiB avail; 65 KiB/s, 0 objects/s recovering 2023-06-03T20:27:10.578 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:27:10.616 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:27:10.619 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:27:10.620 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:27:10.721 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:27:10.759 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:27:10.762 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:27:10.762 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:27:10.863 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:27:10.894 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:27:10.896 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:27:10.897 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:27:10.998 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:27:11.030 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:27:11.032 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:27:11.032 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:27:11.133 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:27:11.164 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:27:11.166 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:27:11.167 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:27:11.268 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:27:11.300 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:27:11.303 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:27:11.303 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:27:11.404 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:27:11.444 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:27:11.446 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:27:11.447 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:27:11.548 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:27:11.587 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:27:11.589 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:27:11.590 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:27:11.691 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:27:11.731 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:27:11.735 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:27:11.735 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:27:11.836 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:27:11.875 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:27:11.878 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:27:11.879 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:27:11.980 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:27:12.014 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:27:12.016 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:27:12.016 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:27:12.117 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:27:12.147 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:27:12.149 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:27:12.150 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:27:12.250 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:27:12.280 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:27:12.283 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:27:12.283 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:27:12.335 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:27:12 smithi164 bash[59462]: cluster 1685824031.1111073 mgr.y (mgr.34104) 1539 : cluster 0 pgmap v1580: 73 pgs: 73 active+clean; 582 KiB data, 5.0 GiB used, 710 GiB / 715 GiB avail; 58 KiB/s, 0 objects/s recovering 2023-06-03T20:27:12.384 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:27:12.415 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:27:12.417 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:27:12.418 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:27:12.519 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:27:12.529 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:27:12 smithi089 bash[78452]: cluster 1685824031.1111073 mgr.y (mgr.34104) 1539 : cluster 0 pgmap v1580: 73 pgs: 73 active+clean; 582 KiB data, 5.0 GiB used, 710 GiB / 715 GiB avail; 58 KiB/s, 0 objects/s recovering 2023-06-03T20:27:12.529 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:27:12 smithi089 bash[75068]: cluster 1685824031.1111073 mgr.y (mgr.34104) 1539 : cluster 0 pgmap v1580: 73 pgs: 73 active+clean; 582 KiB data, 5.0 GiB used, 710 GiB / 715 GiB avail; 58 KiB/s, 0 objects/s recovering 2023-06-03T20:27:12.553 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:27:12.556 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:27:12.556 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:27:12.657 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:27:12.685 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:27:12.687 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:27:12.687 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:27:12.788 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:27:12.821 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:27:12.824 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:27:12.824 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:27:12.926 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:27:12.961 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:27:12.963 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:27:12.964 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:27:13.065 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:27:13.095 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:27:13.098 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:27:13.099 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:27:13.200 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:27:13.228 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:27:13.232 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:27:13.233 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:27:13.334 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:27:13.374 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:27:13.377 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:27:13.378 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:27:13.479 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:27:13.513 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:27:13.516 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:27:13.517 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:27:13.618 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:27:13.655 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:27:13.658 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:27:13.658 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:27:13.760 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:27:13.796 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:27:13.799 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:27:13.800 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:27:13.901 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:27:13.937 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:27:13.940 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:27:13.941 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:27:14.042 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:27:14.067 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:27:14.070 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:27:14.071 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:27:14.172 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:27:14.179 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:27:13 smithi089 bash[78452]: audit 1685824032.9921293 mon.a (mon.0) 1555 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:27:14.179 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:27:13 smithi089 bash[78452]: cluster 1685824033.1121814 mgr.y (mgr.34104) 1540 : cluster 0 pgmap v1581: 73 pgs: 73 active+clean; 582 KiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 49 KiB/s, 0 objects/s recovering 2023-06-03T20:27:14.180 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:27:13 smithi089 bash[75068]: audit 1685824032.9921293 mon.a (mon.0) 1555 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:27:14.180 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:27:13 smithi089 bash[75068]: cluster 1685824033.1121814 mgr.y (mgr.34104) 1540 : cluster 0 pgmap v1581: 73 pgs: 73 active+clean; 582 KiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 49 KiB/s, 0 objects/s recovering 2023-06-03T20:27:14.196 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:27:14.199 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:27:14.199 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:27:14.300 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:27:14.334 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:27:13 smithi164 bash[59462]: audit 1685824032.9921293 mon.a (mon.0) 1555 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:27:14.334 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:27:13 smithi164 bash[59462]: cluster 1685824033.1121814 mgr.y (mgr.34104) 1540 : cluster 0 pgmap v1581: 73 pgs: 73 active+clean; 582 KiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 49 KiB/s, 0 objects/s recovering 2023-06-03T20:27:14.334 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:27:14.336 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:27:14.337 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:27:14.438 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:27:14.471 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:27:14.473 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:27:14.474 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:27:14.575 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:27:14.582 INFO:teuthology.orchestra.run.smithi164.stdout:{ 2023-06-03T20:27:14.583 INFO:teuthology.orchestra.run.smithi164.stdout: "ops": [], 2023-06-03T20:27:14.583 INFO:teuthology.orchestra.run.smithi164.stdout: "complaint_time": 30, 2023-06-03T20:27:14.583 INFO:teuthology.orchestra.run.smithi164.stdout: "num_blocked_ops": 0 2023-06-03T20:27:14.584 INFO:teuthology.orchestra.run.smithi164.stdout:} 2023-06-03T20:27:14.601 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:27:14.603 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:27:14.604 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:27:14.705 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:27:14.743 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:27:14.745 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:27:14.746 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:27:14.847 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:27:14.873 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:27:14.875 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:27:14.876 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:27:14.978 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:27:15.004 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:27:15.006 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:27:15.006 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:27:15.107 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:27:15.136 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:27:15.138 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:27:15.139 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:27:15.239 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:27:15.273 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:27:15.275 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:27:15.276 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:27:15.377 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:27:15.408 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:27:15.411 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:27:15.411 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:27:15.512 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:27:15.544 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:27:15.547 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:27:15.548 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:27:15.617 DEBUG:teuthology.orchestra.run.smithi164:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph daemon osd.7 dump_historic_ops 2023-06-03T20:27:15.648 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:27:15.683 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:27:15.686 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:27:15.687 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:27:15.788 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:27:15.823 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:27:15.826 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:27:15.826 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:27:15.927 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:27:15.961 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:27:15.964 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:27:15.965 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:27:16.066 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:27:16.103 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:27:16.106 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:27:16.107 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:27:16.208 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:27:16.238 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:27:16.240 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:27:16.241 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:27:16.335 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:27:16 smithi164 bash[59462]: cluster 1685824035.1126344 mgr.y (mgr.34104) 1541 : cluster 0 pgmap v1582: 73 pgs: 73 active+clean; 582 KiB data, 4.9 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:27:16.341 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:27:16.366 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:27:16.368 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:27:16.368 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:27:16.469 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:27:16.482 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 20:27:16 smithi089 bash[71163]: ::ffff:172.21.15.164 - - [03/Jun/2023:20:27:16] "GET /metrics HTTP/1.1" 200 226516 "" "Prometheus/2.33.4" 2023-06-03T20:27:16.482 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:27:16 smithi089 bash[78452]: cluster 1685824035.1126344 mgr.y (mgr.34104) 1541 : cluster 0 pgmap v1582: 73 pgs: 73 active+clean; 582 KiB data, 4.9 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:27:16.483 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:27:16 smithi089 bash[75068]: cluster 1685824035.1126344 mgr.y (mgr.34104) 1541 : cluster 0 pgmap v1582: 73 pgs: 73 active+clean; 582 KiB data, 4.9 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:27:16.503 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:27:16.505 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:27:16.506 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:27:16.607 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:27:16.635 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:27:16.637 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:27:16.638 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:27:16.739 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:27:16.767 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:27:16.770 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:27:16.770 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:27:16.871 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:27:16.906 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:27:16.909 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:27:16.910 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:27:17.011 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:27:17.038 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:27:17.040 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:27:17.041 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:27:17.142 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:27:17.174 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:27:17.176 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:27:17.177 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:27:17.278 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:27:17.300 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:27:17.302 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:27:17.302 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:27:17.403 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:27:17.439 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:27:17.441 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:27:17.442 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:27:17.543 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:27:17.573 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:27:17.575 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:27:17.576 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:27:17.677 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:27:17.705 INFO:teuthology.orchestra.run.smithi089.stdout:osd.0: {} 2023-06-03T20:27:17.714 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:27:17.717 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:27:17.718 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:27:17.733 INFO:teuthology.orchestra.run.smithi089.stdout:osd.1: {} 2023-06-03T20:27:17.755 INFO:teuthology.orchestra.run.smithi089.stdout:osd.2: {} 2023-06-03T20:27:17.779 INFO:teuthology.orchestra.run.smithi089.stdout:osd.3: {} 2023-06-03T20:27:17.804 INFO:teuthology.orchestra.run.smithi089.stdout:osd.4: {} 2023-06-03T20:27:17.819 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:27:17.833 INFO:teuthology.orchestra.run.smithi089.stdout:osd.5: {} 2023-06-03T20:27:17.845 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:27:17.848 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:27:17.848 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:27:17.862 INFO:teuthology.orchestra.run.smithi089.stdout:osd.6: {} 2023-06-03T20:27:17.886 INFO:teuthology.orchestra.run.smithi089.stdout:osd.7: {} 2023-06-03T20:27:17.888 INFO:teuthology.orchestra.run.smithi089.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-06-03T20:27:17.888 INFO:teuthology.orchestra.run.smithi089.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-06-03T20:27:17.889 INFO:teuthology.orchestra.run.smithi089.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-06-03T20:27:17.889 INFO:teuthology.orchestra.run.smithi089.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-06-03T20:27:17.889 INFO:teuthology.orchestra.run.smithi089.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-06-03T20:27:17.889 INFO:teuthology.orchestra.run.smithi089.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-06-03T20:27:17.890 INFO:teuthology.orchestra.run.smithi089.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-06-03T20:27:17.890 INFO:teuthology.orchestra.run.smithi089.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-06-03T20:27:17.949 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:27:17.978 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:27:17.981 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:27:17.981 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:27:18.082 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:27:18.118 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:27:18.121 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:27:18.122 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:27:18.190 INFO:teuthology.orchestra.run.smithi089.stderr:reweighted osd.5 to 1 (10000) 2023-06-03T20:27:18.223 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:27:18.248 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:27:18.250 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:27:18.251 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:27:18.351 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:27:18.385 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:27:18.387 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:27:18.388 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:27:18.439 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:27:18 smithi089 bash[78452]: cluster 1685824037.1133354 mgr.y (mgr.34104) 1542 : cluster 0 pgmap v1583: 73 pgs: 73 active+clean; 582 KiB data, 4.9 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:27:18.439 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:27:18 smithi089 bash[78452]: audit 1685824037.2545664 mon.a (mon.0) 1556 : audit 1 from='client.? 172.21.15.89:0/2725852605' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-06-03T20:27:18.439 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:27:18 smithi089 bash[78452]: audit 1685824037.6782086 mon.a (mon.0) 1557 : audit 0 from='client.? 172.21.15.89:0/1622223472' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-03T20:27:18.439 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:27:18 smithi089 bash[75068]: cluster 1685824037.1133354 mgr.y (mgr.34104) 1542 : cluster 0 pgmap v1583: 73 pgs: 73 active+clean; 582 KiB data, 4.9 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:27:18.440 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:27:18 smithi089 bash[75068]: audit 1685824037.2545664 mon.a (mon.0) 1556 : audit 1 from='client.? 172.21.15.89:0/2725852605' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-06-03T20:27:18.440 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:27:18 smithi089 bash[75068]: audit 1685824037.6782086 mon.a (mon.0) 1557 : audit 0 from='client.? 172.21.15.89:0/1622223472' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-03T20:27:18.489 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:27:18.522 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:27:18.527 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:27:18.527 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:27:18.585 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:27:18 smithi164 bash[59462]: cluster 1685824037.1133354 mgr.y (mgr.34104) 1542 : cluster 0 pgmap v1583: 73 pgs: 73 active+clean; 582 KiB data, 4.9 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:27:18.585 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:27:18 smithi164 bash[59462]: audit 1685824037.2545664 mon.a (mon.0) 1556 : audit 1 from='client.? 172.21.15.89:0/2725852605' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-06-03T20:27:18.585 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:27:18 smithi164 bash[59462]: audit 1685824037.6782086 mon.a (mon.0) 1557 : audit 0 from='client.? 172.21.15.89:0/1622223472' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-03T20:27:18.629 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:27:18.661 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:27:18.663 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:27:18.664 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:27:18.765 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:27:18.799 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:27:18.802 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:27:18.803 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:27:18.904 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:27:18.937 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:27:18.942 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:27:18.943 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:27:19.044 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:27:19.072 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:27:19.073 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:27:19.074 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:27:19.175 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:27:19.210 INFO:teuthology.orchestra.run.smithi089.stderr:noscrub is set 2023-06-03T20:27:19.215 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:27:19.218 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:27:19.219 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:27:19.320 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:27:19.331 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:27:19 smithi089 bash[78452]: audit 1685824038.1794994 mon.a (mon.0) 1558 : audit 1 from='client.? 172.21.15.89:0/2725852605' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2023-06-03T20:27:19.331 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:27:19 smithi089 bash[78452]: cluster 1685824038.179565 mon.a (mon.0) 1559 : cluster 0 osdmap e276: 8 total, 8 up, 8 in 2023-06-03T20:27:19.331 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:27:19 smithi089 bash[78452]: audit 1685824038.2391152 mon.a (mon.0) 1560 : audit 1 from='client.? 172.21.15.89:0/4217811767' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-06-03T20:27:19.332 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:27:19 smithi089 bash[78452]: cluster 1685824038.9260998 osd.5 (osd.5) 224 : cluster 3 failed to encode map e276 with expected crc 2023-06-03T20:27:19.332 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:27:19 smithi089 bash[78452]: cluster 1685824038.9284253 osd.5 (osd.5) 225 : cluster 3 failed to encode map e276 with expected crc 2023-06-03T20:27:19.332 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:27:19 smithi089 bash[75068]: audit 1685824038.1794994 mon.a (mon.0) 1558 : audit 1 from='client.? 172.21.15.89:0/2725852605' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2023-06-03T20:27:19.332 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:27:19 smithi089 bash[75068]: cluster 1685824038.179565 mon.a (mon.0) 1559 : cluster 0 osdmap e276: 8 total, 8 up, 8 in 2023-06-03T20:27:19.333 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:27:19 smithi089 bash[75068]: audit 1685824038.2391152 mon.a (mon.0) 1560 : audit 1 from='client.? 172.21.15.89:0/4217811767' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-06-03T20:27:19.333 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:27:19 smithi089 bash[75068]: cluster 1685824038.9260998 osd.5 (osd.5) 224 : cluster 3 failed to encode map e276 with expected crc 2023-06-03T20:27:19.333 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:27:19 smithi089 bash[75068]: cluster 1685824038.9284253 osd.5 (osd.5) 225 : cluster 3 failed to encode map e276 with expected crc 2023-06-03T20:27:19.349 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:27:19.351 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:27:19.351 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:27:19.452 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:27:19.480 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:27:19.482 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:27:19.482 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:27:19.583 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:27:19.585 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:27:19 smithi164 bash[59462]: audit 1685824038.1794994 mon.a (mon.0) 1558 : audit 1 from='client.? 172.21.15.89:0/2725852605' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2023-06-03T20:27:19.585 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:27:19 smithi164 bash[59462]: cluster 1685824038.179565 mon.a (mon.0) 1559 : cluster 0 osdmap e276: 8 total, 8 up, 8 in 2023-06-03T20:27:19.585 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:27:19 smithi164 bash[59462]: audit 1685824038.2391152 mon.a (mon.0) 1560 : audit 1 from='client.? 172.21.15.89:0/4217811767' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-06-03T20:27:19.585 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:27:19 smithi164 bash[59462]: cluster 1685824038.9260998 osd.5 (osd.5) 224 : cluster 3 failed to encode map e276 with expected crc 2023-06-03T20:27:19.585 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:27:19 smithi164 bash[59462]: cluster 1685824038.9284253 osd.5 (osd.5) 225 : cluster 3 failed to encode map e276 with expected crc 2023-06-03T20:27:19.627 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:27:19.630 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:27:19.631 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:27:19.731 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:27:19.759 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:27:19.762 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:27:19.762 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:27:19.863 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:27:19.903 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:27:19.907 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:27:19.908 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:27:20.009 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:27:20.043 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:27:20.046 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:27:20.047 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:27:20.148 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:27:20.184 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:27:20.187 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:27:20.187 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:27:20.288 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:27:20.313 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:27:20.315 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:27:20 smithi089 bash[78452]: cluster 1685824039.100468 osd.7 (osd.7) 224 : cluster 3 failed to encode map e276 with expected crc 2023-06-03T20:27:20.315 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:27:20 smithi089 bash[78452]: cluster 1685824039.1139326 mgr.y (mgr.34104) 1543 : cluster 0 pgmap v1585: 73 pgs: 73 active+clean; 582 KiB data, 4.9 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:27:20.315 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:27:20 smithi089 bash[78452]: cluster 1685824039.1888614 mon.a (mon.0) 1561 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-06-03T20:27:20.316 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:27:20 smithi089 bash[78452]: audit 1685824039.2015803 mon.a (mon.0) 1562 : audit 1 from='client.? 172.21.15.89:0/4217811767' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-06-03T20:27:20.316 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:27:20 smithi089 bash[78452]: cluster 1685824039.2016885 mon.a (mon.0) 1563 : cluster 0 osdmap e277: 8 total, 8 up, 8 in 2023-06-03T20:27:20.316 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:27:20 smithi089 bash[78452]: cluster 1685824039.3221965 osd.6 (osd.6) 209 : cluster 3 failed to encode map e276 with expected crc 2023-06-03T20:27:20.316 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:27:20 smithi089 bash[78452]: cluster 1685824039.327964 osd.6 (osd.6) 210 : cluster 3 failed to encode map e277 with expected crc 2023-06-03T20:27:20.316 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:27:20 smithi089 bash[75068]: cluster 1685824039.100468 osd.7 (osd.7) 224 : cluster 3 failed to encode map e276 with expected crc 2023-06-03T20:27:20.317 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:27:20 smithi089 bash[75068]: cluster 1685824039.1139326 mgr.y (mgr.34104) 1543 : cluster 0 pgmap v1585: 73 pgs: 73 active+clean; 582 KiB data, 4.9 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:27:20.317 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:27:20 smithi089 bash[75068]: cluster 1685824039.1888614 mon.a (mon.0) 1561 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-06-03T20:27:20.317 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:27:20 smithi089 bash[75068]: audit 1685824039.2015803 mon.a (mon.0) 1562 : audit 1 from='client.? 172.21.15.89:0/4217811767' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-06-03T20:27:20.317 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:27:20 smithi089 bash[75068]: cluster 1685824039.2016885 mon.a (mon.0) 1563 : cluster 0 osdmap e277: 8 total, 8 up, 8 in 2023-06-03T20:27:20.317 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:27:20 smithi089 bash[75068]: cluster 1685824039.3221965 osd.6 (osd.6) 209 : cluster 3 failed to encode map e276 with expected crc 2023-06-03T20:27:20.317 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:27:20 smithi089 bash[75068]: cluster 1685824039.327964 osd.6 (osd.6) 210 : cluster 3 failed to encode map e277 with expected crc 2023-06-03T20:27:20.318 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:27:20.318 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:27:20.363 INFO:teuthology.orchestra.run.smithi089.stderr:pool 'unique_pool_9' created 2023-06-03T20:27:20.419 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:27:20.427 INFO:teuthology.orchestra.run.smithi164.stderr:Inferring config /var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/mon.b/config 2023-06-03T20:27:20.431 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:27:20 smithi164 bash[59462]: cluster 1685824039.100468 osd.7 (osd.7) 224 : cluster 3 failed to encode map e276 with expected crc 2023-06-03T20:27:20.432 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:27:20 smithi164 bash[59462]: cluster 1685824039.1139326 mgr.y (mgr.34104) 1543 : cluster 0 pgmap v1585: 73 pgs: 73 active+clean; 582 KiB data, 4.9 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:27:20.432 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:27:20 smithi164 bash[59462]: cluster 1685824039.1888614 mon.a (mon.0) 1561 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-06-03T20:27:20.432 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:27:20 smithi164 bash[59462]: audit 1685824039.2015803 mon.a (mon.0) 1562 : audit 1 from='client.? 172.21.15.89:0/4217811767' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-06-03T20:27:20.432 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:27:20 smithi164 bash[59462]: cluster 1685824039.2016885 mon.a (mon.0) 1563 : cluster 0 osdmap e277: 8 total, 8 up, 8 in 2023-06-03T20:27:20.432 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:27:20 smithi164 bash[59462]: cluster 1685824039.3221965 osd.6 (osd.6) 209 : cluster 3 failed to encode map e276 with expected crc 2023-06-03T20:27:20.432 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:27:20 smithi164 bash[59462]: cluster 1685824039.327964 osd.6 (osd.6) 210 : cluster 3 failed to encode map e277 with expected crc 2023-06-03T20:27:20.450 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:27:20.453 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:27:20.454 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:27:20.555 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:27:20.584 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:27:20.586 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:27:20.587 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:27:20.688 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:27:20.718 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:27:20.720 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:27:20.721 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:27:20.822 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:27:20.853 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:27:20.856 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:27:20.856 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:27:20.957 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:27:20.988 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:27:20.990 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:27:20.991 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:27:21.092 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:27:21.121 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:27:21.123 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:27:21.124 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:27:21.225 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:27:21.254 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:27:21.256 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:27:21.257 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:27:21.358 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:27:21.369 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:27:21 smithi089 bash[78452]: cluster 1685824039.6529086 osd.7 (osd.7) 225 : cluster 3 failed to encode map e277 with expected crc 2023-06-03T20:27:21.369 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:27:21 smithi089 bash[78452]: cluster 1685824039.6529894 osd.5 (osd.5) 226 : cluster 3 failed to encode map e277 with expected crc 2023-06-03T20:27:21.369 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:27:21 smithi089 bash[78452]: audit 1685824039.8531973 mon.c (mon.2) 639 : audit 1 from='client.? 172.21.15.89:0/1100529423' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_9", "pg_num": 16}]: dispatch 2023-06-03T20:27:21.370 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:27:21 smithi089 bash[78452]: audit 1685824039.853745 mon.a (mon.0) 1564 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_9", "pg_num": 16}]: dispatch 2023-06-03T20:27:21.370 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:27:21 smithi089 bash[78452]: cluster 1685824040.346821 osd.5 (osd.5) 227 : cluster 3 failed to encode map e278 with expected crc 2023-06-03T20:27:21.370 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:27:21 smithi089 bash[78452]: cluster 1685824040.3505378 osd.7 (osd.7) 226 : cluster 3 failed to encode map e278 with expected crc 2023-06-03T20:27:21.370 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:27:21 smithi089 bash[78452]: audit 1685824040.3514464 mon.a (mon.0) 1565 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_9", "pg_num": 16}]': finished 2023-06-03T20:27:21.370 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:27:21 smithi089 bash[78452]: cluster 1685824040.3515508 mon.a (mon.0) 1566 : cluster 0 osdmap e278: 8 total, 8 up, 8 in 2023-06-03T20:27:21.370 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:27:21 smithi089 bash[78452]: cluster 1685824040.354285 osd.6 (osd.6) 211 : cluster 3 failed to encode map e278 with expected crc 2023-06-03T20:27:21.371 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:27:21 smithi089 bash[75068]: cluster 1685824039.6529086 osd.7 (osd.7) 225 : cluster 3 failed to encode map e277 with expected crc 2023-06-03T20:27:21.371 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:27:21 smithi089 bash[75068]: cluster 1685824039.6529894 osd.5 (osd.5) 226 : cluster 3 failed to encode map e277 with expected crc 2023-06-03T20:27:21.371 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:27:21 smithi089 bash[75068]: audit 1685824039.8531973 mon.c (mon.2) 639 : audit 1 from='client.? 172.21.15.89:0/1100529423' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_9", "pg_num": 16}]: dispatch 2023-06-03T20:27:21.371 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:27:21 smithi089 bash[75068]: audit 1685824039.853745 mon.a (mon.0) 1564 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_9", "pg_num": 16}]: dispatch 2023-06-03T20:27:21.371 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:27:21 smithi089 bash[75068]: cluster 1685824040.346821 osd.5 (osd.5) 227 : cluster 3 failed to encode map e278 with expected crc 2023-06-03T20:27:21.371 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:27:21 smithi089 bash[75068]: cluster 1685824040.3505378 osd.7 (osd.7) 226 : cluster 3 failed to encode map e278 with expected crc 2023-06-03T20:27:21.372 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:27:21 smithi089 bash[75068]: audit 1685824040.3514464 mon.a (mon.0) 1565 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_9", "pg_num": 16}]': finished 2023-06-03T20:27:21.372 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:27:21 smithi089 bash[75068]: cluster 1685824040.3515508 mon.a (mon.0) 1566 : cluster 0 osdmap e278: 8 total, 8 up, 8 in 2023-06-03T20:27:21.372 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:27:21 smithi089 bash[75068]: cluster 1685824040.354285 osd.6 (osd.6) 211 : cluster 3 failed to encode map e278 with expected crc 2023-06-03T20:27:21.391 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:27:21.393 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:27:21.394 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:27:21.494 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:27:21.523 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:27:21.525 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:27:21.525 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:27:21.585 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:27:21 smithi164 bash[59462]: cluster 1685824039.6529086 osd.7 (osd.7) 225 : cluster 3 failed to encode map e277 with expected crc 2023-06-03T20:27:21.585 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:27:21 smithi164 bash[59462]: cluster 1685824039.6529894 osd.5 (osd.5) 226 : cluster 3 failed to encode map e277 with expected crc 2023-06-03T20:27:21.585 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:27:21 smithi164 bash[59462]: audit 1685824039.8531973 mon.c (mon.2) 639 : audit 1 from='client.? 172.21.15.89:0/1100529423' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_9", "pg_num": 16}]: dispatch 2023-06-03T20:27:21.585 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:27:21 smithi164 bash[59462]: audit 1685824039.853745 mon.a (mon.0) 1564 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_9", "pg_num": 16}]: dispatch 2023-06-03T20:27:21.586 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:27:21 smithi164 bash[59462]: cluster 1685824040.346821 osd.5 (osd.5) 227 : cluster 3 failed to encode map e278 with expected crc 2023-06-03T20:27:21.586 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:27:21 smithi164 bash[59462]: cluster 1685824040.3505378 osd.7 (osd.7) 226 : cluster 3 failed to encode map e278 with expected crc 2023-06-03T20:27:21.586 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:27:21 smithi164 bash[59462]: audit 1685824040.3514464 mon.a (mon.0) 1565 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_9", "pg_num": 16}]': finished 2023-06-03T20:27:21.586 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:27:21 smithi164 bash[59462]: cluster 1685824040.3515508 mon.a (mon.0) 1566 : cluster 0 osdmap e278: 8 total, 8 up, 8 in 2023-06-03T20:27:21.586 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:27:21 smithi164 bash[59462]: cluster 1685824040.354285 osd.6 (osd.6) 211 : cluster 3 failed to encode map e278 with expected crc 2023-06-03T20:27:21.626 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:27:21.656 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:27:21.658 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:27:21.659 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:27:21.759 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:27:21.797 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:27:21.800 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:27:21.801 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:27:21.902 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:27:21.933 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:27:21.936 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:27:21.937 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:27:22.038 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:27:22.076 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:27:22.079 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:27:22.079 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:27:22.180 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:27:22.208 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:27:22.211 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:27:22.212 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:27:22.312 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:27:22.342 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:27:22.346 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:27:22.346 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:27:22.447 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:27:22.476 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:27:22.479 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:27:22.479 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:27:22.580 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:27:22.585 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:27:22 smithi164 bash[59462]: cluster 1685824041.1144495 mgr.y (mgr.34104) 1544 : cluster 0 pgmap v1588: 89 pgs: 16 unknown, 73 active+clean; 582 KiB data, 4.9 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:27:22.585 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:27:22 smithi164 bash[59462]: cluster 1685824041.3543062 osd.5 (osd.5) 228 : cluster 3 failed to encode map e279 with expected crc 2023-06-03T20:27:22.585 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:27:22 smithi164 bash[59462]: cluster 1685824041.357149 osd.6 (osd.6) 212 : cluster 3 failed to encode map e279 with expected crc 2023-06-03T20:27:22.585 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:27:22 smithi164 bash[59462]: cluster 1685824041.3574657 osd.7 (osd.7) 227 : cluster 3 failed to encode map e279 with expected crc 2023-06-03T20:27:22.586 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:27:22 smithi164 bash[59462]: cluster 1685824041.359023 mon.a (mon.0) 1567 : cluster 0 osdmap e279: 8 total, 8 up, 8 in 2023-06-03T20:27:22.586 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:27:22 smithi164 bash[59462]: audit 1685824042.1556942 mon.a (mon.0) 1568 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:27:22.586 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:27:22 smithi164 bash[59462]: audit 1685824042.1597714 mon.c (mon.2) 640 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T20:27:22.588 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:27:22 smithi089 bash[78452]: cluster 1685824041.1144495 mgr.y (mgr.34104) 1544 : cluster 0 pgmap v1588: 89 pgs: 16 unknown, 73 active+clean; 582 KiB data, 4.9 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:27:22.588 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:27:22 smithi089 bash[78452]: cluster 1685824041.3543062 osd.5 (osd.5) 228 : cluster 3 failed to encode map e279 with expected crc 2023-06-03T20:27:22.588 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:27:22 smithi089 bash[78452]: cluster 1685824041.357149 osd.6 (osd.6) 212 : cluster 3 failed to encode map e279 with expected crc 2023-06-03T20:27:22.589 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:27:22 smithi089 bash[78452]: cluster 1685824041.3574657 osd.7 (osd.7) 227 : cluster 3 failed to encode map e279 with expected crc 2023-06-03T20:27:22.589 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:27:22 smithi089 bash[78452]: cluster 1685824041.359023 mon.a (mon.0) 1567 : cluster 0 osdmap e279: 8 total, 8 up, 8 in 2023-06-03T20:27:22.589 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:27:22 smithi089 bash[78452]: audit 1685824042.1556942 mon.a (mon.0) 1568 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:27:22.589 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:27:22 smithi089 bash[78452]: audit 1685824042.1597714 mon.c (mon.2) 640 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T20:27:22.589 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:27:22 smithi089 bash[75068]: cluster 1685824041.1144495 mgr.y (mgr.34104) 1544 : cluster 0 pgmap v1588: 89 pgs: 16 unknown, 73 active+clean; 582 KiB data, 4.9 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:27:22.589 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:27:22 smithi089 bash[75068]: cluster 1685824041.3543062 osd.5 (osd.5) 228 : cluster 3 failed to encode map e279 with expected crc 2023-06-03T20:27:22.590 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:27:22 smithi089 bash[75068]: cluster 1685824041.357149 osd.6 (osd.6) 212 : cluster 3 failed to encode map e279 with expected crc 2023-06-03T20:27:22.590 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:27:22 smithi089 bash[75068]: cluster 1685824041.3574657 osd.7 (osd.7) 227 : cluster 3 failed to encode map e279 with expected crc 2023-06-03T20:27:22.590 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:27:22 smithi089 bash[75068]: cluster 1685824041.359023 mon.a (mon.0) 1567 : cluster 0 osdmap e279: 8 total, 8 up, 8 in 2023-06-03T20:27:22.590 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:27:22 smithi089 bash[75068]: audit 1685824042.1556942 mon.a (mon.0) 1568 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:27:22.590 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:27:22 smithi089 bash[75068]: audit 1685824042.1597714 mon.c (mon.2) 640 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T20:27:22.610 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:27:22.613 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:27:22.614 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:27:22.715 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:27:22.720 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-06-03T20:27:22.740 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:27:22.741 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:27:22.742 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:27:22.843 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:27:22.871 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:27:22.872 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:27:22.873 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:27:22.923 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph osd reweight 7 1 2023-06-03T20:27:22.974 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:27:23.003 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:27:23.006 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:27:23.007 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:27:23.108 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:27:23.142 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:27:23.145 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:27:23.146 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:27:23.246 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:27:23.280 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:27:23.282 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:27:23.283 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:27:23.384 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:27:23.386 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph osd pool application enable unique_pool_9 rados --yes-i-really-mean-it || true 2023-06-03T20:27:23.414 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:27:23.416 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:27:23.418 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:27:23.519 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:27:23.529 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:27:23 smithi089 bash[75068]: cluster 1685824042.3579955 mon.a (mon.0) 1569 : cluster 0 osdmap e280: 8 total, 8 up, 8 in 2023-06-03T20:27:23.529 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:27:23 smithi089 bash[75068]: cluster 1685824042.5568702 osd.5 (osd.5) 229 : cluster 3 failed to encode map e280 with expected crc 2023-06-03T20:27:23.529 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:27:23 smithi089 bash[75068]: cluster 1685824043.1150398 mgr.y (mgr.34104) 1545 : cluster 0 pgmap v1591: 89 pgs: 3 creating+peering, 86 active+clean; 582 KiB data, 4.9 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:27:23.530 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:27:23 smithi089 bash[78452]: cluster 1685824042.3579955 mon.a (mon.0) 1569 : cluster 0 osdmap e280: 8 total, 8 up, 8 in 2023-06-03T20:27:23.530 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:27:23 smithi089 bash[78452]: cluster 1685824042.5568702 osd.5 (osd.5) 229 : cluster 3 failed to encode map e280 with expected crc 2023-06-03T20:27:23.530 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:27:23 smithi089 bash[78452]: cluster 1685824043.1150398 mgr.y (mgr.34104) 1545 : cluster 0 pgmap v1591: 89 pgs: 3 creating+peering, 86 active+clean; 582 KiB data, 4.9 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:27:23.554 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:27:23.557 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:27:23.558 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:27:23.659 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:27:23.692 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:27:23.696 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:27:23.697 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:27:23.798 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:27:23.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:27:23 smithi164 bash[59462]: cluster 1685824042.3579955 mon.a (mon.0) 1569 : cluster 0 osdmap e280: 8 total, 8 up, 8 in 2023-06-03T20:27:23.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:27:23 smithi164 bash[59462]: cluster 1685824042.5568702 osd.5 (osd.5) 229 : cluster 3 failed to encode map e280 with expected crc 2023-06-03T20:27:23.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:27:23 smithi164 bash[59462]: cluster 1685824043.1150398 mgr.y (mgr.34104) 1545 : cluster 0 pgmap v1591: 89 pgs: 3 creating+peering, 86 active+clean; 582 KiB data, 4.9 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:27:23.836 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:27:23.838 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:27:23.840 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:27:23.941 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:27:24.128 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:27:24.131 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:27:24.131 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:27:24.232 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:27:24.272 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:27:24.275 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:27:24.276 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:27:24.377 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:27:24.411 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:27:24.414 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:27:24.415 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:27:24.516 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:27:24.557 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:27:24.560 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:27:24.561 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:27:24.662 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:27:24.674 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:27:24 smithi164 bash[59462]: cluster 1685824043.0360303 osd.7 (osd.7) 228 : cluster 3 failed to encode map e280 with expected crc 2023-06-03T20:27:24.675 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:27:24 smithi164 bash[59462]: cluster 1685824043.3241532 osd.6 (osd.6) 213 : cluster 3 failed to encode map e280 with expected crc 2023-06-03T20:27:24.702 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:27:24.704 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:27:24.705 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:27:24.760 INFO:teuthology.orchestra.run.smithi164.stdout:ERROR: (22) Invalid argument 2023-06-03T20:27:24.761 INFO:teuthology.orchestra.run.smithi164.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-06-03T20:27:24.779 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:27:24 smithi089 bash[78452]: cluster 1685824043.0360303 osd.7 (osd.7) 228 : cluster 3 failed to encode map e280 with expected crc 2023-06-03T20:27:24.779 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:27:24 smithi089 bash[78452]: cluster 1685824043.3241532 osd.6 (osd.6) 213 : cluster 3 failed to encode map e280 with expected crc 2023-06-03T20:27:24.780 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:27:24 smithi089 bash[75068]: cluster 1685824043.0360303 osd.7 (osd.7) 228 : cluster 3 failed to encode map e280 with expected crc 2023-06-03T20:27:24.780 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:27:24 smithi089 bash[75068]: cluster 1685824043.3241532 osd.6 (osd.6) 213 : cluster 3 failed to encode map e280 with expected crc 2023-06-03T20:27:24.806 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:27:24.847 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:27:24.851 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:27:24.851 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:27:24.952 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:27:24.993 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:27:24.996 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:27:24.996 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:27:25.097 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:27:25.131 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:27:25.134 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:27:25.134 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:27:25.202 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph osd set nodeep-scrub 2023-06-03T20:27:25.235 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:27:25.264 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:27:25.268 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:27:25.269 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:27:25.370 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:27:25.401 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:27:25.403 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:27:25.404 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:27:25.504 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:27:25.529 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:27:25 smithi089 bash[78452]: cluster 1685824045.115639 mgr.y (mgr.34104) 1546 : cluster 0 pgmap v1592: 89 pgs: 3 creating+peering, 86 active+clean; 582 KiB data, 4.9 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:27:25.530 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:27:25 smithi089 bash[75068]: cluster 1685824045.115639 mgr.y (mgr.34104) 1546 : cluster 0 pgmap v1592: 89 pgs: 3 creating+peering, 86 active+clean; 582 KiB data, 4.9 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:27:25.538 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:27:25.541 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:27:25 smithi164 bash[59462]: cluster 1685824045.115639 mgr.y (mgr.34104) 1546 : cluster 0 pgmap v1592: 89 pgs: 3 creating+peering, 86 active+clean; 582 KiB data, 4.9 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:27:25.542 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:27:25.542 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:27:25.643 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:27:25.675 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:27:25.676 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:27:25.677 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:27:25.718 DEBUG:teuthology.orchestra.run.smithi164:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph daemon osd.4 dump_ops_in_flight 2023-06-03T20:27:25.779 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:27:25.843 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:27:25.847 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:27:25.847 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:27:25.948 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:27:25.981 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:27:25.984 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:27:25.984 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:27:26.087 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:27:26.129 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:27:26.131 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:27:26.132 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:27:26.233 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:27:26.271 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:27:26.274 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:27:26.275 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:27:26.376 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:27:26.410 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:27:26.413 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:27:26.414 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:27:26.515 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:27:26.529 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 20:27:26 smithi089 bash[71163]: ::ffff:172.21.15.164 - - [03/Jun/2023:20:27:26] "GET /metrics HTTP/1.1" 200 228828 "" "Prometheus/2.33.4" 2023-06-03T20:27:26.555 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:27:26.558 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:27:26.558 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:27:26.659 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:27:26.692 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:27:26.695 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:27:26.696 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:27:26.798 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:27:26.833 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:27:26.836 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:27:26.836 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:27:26.937 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:27:26.972 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:27:26.974 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:27:26.975 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:27:27.075 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:27:27.113 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:27:27.116 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:27:27.116 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:27:27.217 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:27:27.249 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:27:27.252 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:27:27.252 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:27:27.353 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:27:27.388 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:27:27.391 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:27:27.391 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:27:27.492 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:27:27.530 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:27:27.532 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:27:27.533 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:27:27.551 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/mon.a/config 2023-06-03T20:27:27.553 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/mon.a/config 2023-06-03T20:27:27.553 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/mon.a/config 2023-06-03T20:27:27.634 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:27:27.668 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:27:27.672 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:27:27.673 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:27:27.773 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:27:27.807 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:27:27.809 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:27:27.810 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:27:27.911 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:27:27.949 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:27:27.951 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:27:27.952 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:27:28.053 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:27:28.083 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:27:28.086 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:27:28.086 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:27:28.187 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:27:28.220 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:27:28.223 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:27:28.224 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:27:28.325 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:27:28.333 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:27:28 smithi089 bash[78452]: cluster 1685824047.1164317 mgr.y (mgr.34104) 1547 : cluster 0 pgmap v1593: 89 pgs: 89 active+clean; 582 KiB data, 4.9 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:27:28.333 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:27:28 smithi089 bash[75068]: cluster 1685824047.1164317 mgr.y (mgr.34104) 1547 : cluster 0 pgmap v1593: 89 pgs: 89 active+clean; 582 KiB data, 4.9 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:27:28.354 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:27:28.357 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:27:28.358 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:27:28.458 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:27:28.471 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:27:28 smithi164 bash[59462]: cluster 1685824047.1164317 mgr.y (mgr.34104) 1547 : cluster 0 pgmap v1593: 89 pgs: 89 active+clean; 582 KiB data, 4.9 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:27:28.496 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:27:28.499 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:27:28.499 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:27:28.600 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:27:28.638 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:27:28.641 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:27:28.642 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:27:28.743 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:27:28.777 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:27:28.779 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:27:28.780 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:27:28.881 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:27:28.920 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:27:28.923 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:27:28.924 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:27:29.025 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:27:29.060 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:27:29.062 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:27:29.063 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:27:29.164 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:27:29.202 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:27:29.205 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:27:29.206 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:27:29.307 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:27:29.345 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:27:29.349 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:27:29.349 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:27:29.450 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:27:29.493 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:27:29.496 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:27:29.497 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:27:29.592 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/mon.a/config 2023-06-03T20:27:29.598 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:27:29.631 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:27:29.634 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:27:29.635 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:27:29.736 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:27:29.765 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:27:29.768 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:27:29.769 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:27:29.870 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:27:29.901 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:27:29.903 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:27:29.904 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:27:30.005 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:27:30.032 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:27:30.034 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:27:30.034 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:27:30.135 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:27:30.164 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:27:30.166 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:27:30.166 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:27:30.267 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:27:30.275 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:27:30 smithi089 bash[78452]: cluster 1685824049.1176088 mgr.y (mgr.34104) 1548 : cluster 0 pgmap v1594: 89 pgs: 89 active+clean; 582 KiB data, 4.9 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:27:30.276 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:27:30 smithi089 bash[75068]: cluster 1685824049.1176088 mgr.y (mgr.34104) 1548 : cluster 0 pgmap v1594: 89 pgs: 89 active+clean; 582 KiB data, 4.9 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:27:30.297 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:27:30.299 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:27:30.300 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:27:30.401 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:27:30.431 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:27:30.433 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:27:30.433 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:27:30.534 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:27:30.564 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:27:30.567 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:27:30.567 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:27:30.577 INFO:teuthology.orchestra.run.smithi164.stderr:Inferring config /var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/mon.b/config 2023-06-03T20:27:30.584 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:27:30 smithi164 bash[59462]: cluster 1685824049.1176088 mgr.y (mgr.34104) 1548 : cluster 0 pgmap v1594: 89 pgs: 89 active+clean; 582 KiB data, 4.9 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:27:30.668 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:27:30.697 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:27:30.699 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:27:30.700 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:27:30.801 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:27:30.826 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:27:30.828 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:27:30.829 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:27:30.930 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:27:30.963 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:27:30.965 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:27:30.965 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:27:31.066 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:27:31.102 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:27:31.106 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:27:31.106 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:27:31.207 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:27:31.241 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:27:31.242 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:27:31.243 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:27:31.344 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:27:31.378 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:27:31.380 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:27:31.381 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:27:31.482 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:27:31.515 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:27:31.518 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:27:31.519 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:27:31.619 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:27:31.652 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:27:31.655 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:27:31.656 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:27:31.756 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:27:31.795 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:27:31.799 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:27:31.799 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:27:31.900 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:27:31.934 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:27:31.936 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:27:31.937 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:27:32.038 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:27:32.070 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:27:32.072 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:27:32.072 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:27:32.173 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:27:32.203 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:27:32.206 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:27:32.206 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:27:32.307 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:27:32.328 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:27:32 smithi089 bash[75068]: cluster 1685824051.1183898 mgr.y (mgr.34104) 1549 : cluster 0 pgmap v1595: 89 pgs: 89 active+clean; 582 KiB data, 4.9 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:27:32.328 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:27:32 smithi089 bash[78452]: cluster 1685824051.1183898 mgr.y (mgr.34104) 1549 : cluster 0 pgmap v1595: 89 pgs: 89 active+clean; 582 KiB data, 4.9 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:27:32.334 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:27:32.336 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:27:32.337 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:27:32.438 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:27:32.467 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:27:32.469 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:27:32.470 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:27:32.512 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:27:32 smithi164 bash[59462]: cluster 1685824051.1183898 mgr.y (mgr.34104) 1549 : cluster 0 pgmap v1595: 89 pgs: 89 active+clean; 582 KiB data, 4.9 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:27:32.571 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:27:32.611 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:27:32.613 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:27:32.614 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:27:32.715 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:27:32.750 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:27:32.752 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:27:32.753 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:27:32.853 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:27:32.883 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:27:32.886 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:27:32.887 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:27:32.988 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:27:33.013 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:27:33.017 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:27:33.017 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:27:33.118 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:27:33.150 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:27:33.153 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:27:33.154 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:27:33.255 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:27:33.287 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:27:33.290 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:27:33.290 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:27:33.391 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:27:33.424 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:27:33.428 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:27:33.429 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:27:33.529 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:27:33.561 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:27:33.563 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:27:33.564 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:27:33.664 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:27:33.699 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:27:33.701 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:27:33.702 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:27:33.803 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:27:33.838 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:27:33.841 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:27:33.842 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:27:33.943 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:27:33.969 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:27:33.971 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:27:33.972 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:27:34.073 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:27:34.102 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:27:34.104 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:27:34 smithi089 bash[78452]: audit 1685824053.0264914 mon.a (mon.0) 1570 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:27:34.104 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:27:34 smithi089 bash[78452]: cluster 1685824053.1191316 mgr.y (mgr.34104) 1550 : cluster 0 pgmap v1596: 89 pgs: 89 active+clean; 582 KiB data, 4.9 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:27:34.105 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:27:34 smithi089 bash[75068]: audit 1685824053.0264914 mon.a (mon.0) 1570 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:27:34.105 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:27:34 smithi089 bash[75068]: cluster 1685824053.1191316 mgr.y (mgr.34104) 1550 : cluster 0 pgmap v1596: 89 pgs: 89 active+clean; 582 KiB data, 4.9 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:27:34.106 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:27:34.106 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:27:34.208 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:27:34.241 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:27:34.243 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:27:34.243 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:27:34.335 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:27:34 smithi164 bash[59462]: audit 1685824053.0264914 mon.a (mon.0) 1570 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:27:34.335 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:27:34 smithi164 bash[59462]: cluster 1685824053.1191316 mgr.y (mgr.34104) 1550 : cluster 0 pgmap v1596: 89 pgs: 89 active+clean; 582 KiB data, 4.9 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:27:34.344 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:27:34.378 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:27:34.381 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:27:34.382 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:27:34.482 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:27:34.516 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:27:34.519 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:27:34.520 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:27:34.621 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:27:34.659 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:27:34.661 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:27:34.662 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:27:34.762 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:27:34.795 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:27:34.798 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:27:34.798 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:27:34.899 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:27:34.942 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:27:34.945 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:27:34.946 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:27:35.046 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:27:35.074 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:27:35.076 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:27:35.077 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:27:35.178 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:27:35.204 INFO:teuthology.orchestra.run.smithi164.stdout:ERROR: (22) Invalid argument 2023-06-03T20:27:35.204 INFO:teuthology.orchestra.run.smithi164.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-06-03T20:27:35.212 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:27:35.215 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:27:35.216 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:27:35.317 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:27:35.345 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:27:35.347 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:27:35.348 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:27:35.450 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:27:35.476 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:27:35.480 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:27:35.480 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:27:35.581 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:27:35.620 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:27:35.622 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:27:35.623 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:27:35.723 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:27:35.759 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:27:35.761 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:27:35.762 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:27:35.863 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:27:35.895 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:27:35.897 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:27:35.897 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:27:35.999 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:27:36.034 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:27:36.036 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:27:36.037 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:27:36.138 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:27:36.164 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:27:36.166 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:27:36.167 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:27:36.245 DEBUG:teuthology.orchestra.run.smithi164:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph daemon osd.4 dump_blocked_ops 2023-06-03T20:27:36.268 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:27:36.296 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:27:36.299 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:27:36.299 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:27:36.335 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:27:36 smithi164 bash[59462]: cluster 1685824055.1196473 mgr.y (mgr.34104) 1551 : cluster 0 pgmap v1597: 89 pgs: 89 active+clean; 582 KiB data, 4.9 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:27:36.400 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:27:36.436 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:27:36.438 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:27:36.438 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:27:36.529 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 20:27:36 smithi089 bash[71163]: ::ffff:172.21.15.164 - - [03/Jun/2023:20:27:36] "GET /metrics HTTP/1.1" 200 228828 "" "Prometheus/2.33.4" 2023-06-03T20:27:36.530 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:27:36 smithi089 bash[75068]: cluster 1685824055.1196473 mgr.y (mgr.34104) 1551 : cluster 0 pgmap v1597: 89 pgs: 89 active+clean; 582 KiB data, 4.9 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:27:36.530 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:27:36 smithi089 bash[78452]: cluster 1685824055.1196473 mgr.y (mgr.34104) 1551 : cluster 0 pgmap v1597: 89 pgs: 89 active+clean; 582 KiB data, 4.9 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:27:36.539 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:27:36.576 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:27:36.579 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:27:36.580 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:27:36.681 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:27:36.713 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:27:36.716 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:27:36.717 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:27:36.818 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:27:36.847 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:27:36.849 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:27:36.850 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:27:36.951 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:27:36.979 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:27:36.981 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:27:36.982 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:27:37.083 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:27:37.118 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:27:37.121 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:27:37.121 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:27:37.222 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:27:37.258 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:27:37.261 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:27:37.262 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:27:37.363 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:27:37.391 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:27:37.392 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:27:37.393 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:27:37.494 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:27:37.505 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:27:37 smithi089 bash[78452]: audit 1685824057.1485317 mon.c (mon.2) 641 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T20:27:37.505 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:27:37 smithi089 bash[75068]: audit 1685824057.1485317 mon.c (mon.2) 641 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T20:27:37.526 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:27:37.529 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:27:37.530 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:27:37.585 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:27:37 smithi164 bash[59462]: audit 1685824057.1485317 mon.c (mon.2) 641 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T20:27:37.630 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:27:37.670 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:27:37.673 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:27:37.673 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:27:37.774 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:27:37.813 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:27:37.816 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:27:37.817 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:27:37.918 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:27:37.952 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:27:37.955 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:27:37.955 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:27:38.056 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:27:38.094 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:27:38.095 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:27:38.097 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:27:38.198 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:27:38.223 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:27:38.226 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:27:38.227 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:27:38.327 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:27:38.334 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:27:38 smithi164 bash[59462]: cluster 1685824057.1201787 mgr.y (mgr.34104) 1552 : cluster 0 pgmap v1598: 89 pgs: 89 active+clean; 582 KiB data, 4.9 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:27:38.363 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:27:38.367 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:27:38.367 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:27:38.468 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:27:38.503 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:27:38.505 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:27:38 smithi089 bash[75068]: cluster 1685824057.1201787 mgr.y (mgr.34104) 1552 : cluster 0 pgmap v1598: 89 pgs: 89 active+clean; 582 KiB data, 4.9 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:27:38.505 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:27:38 smithi089 bash[78452]: cluster 1685824057.1201787 mgr.y (mgr.34104) 1552 : cluster 0 pgmap v1598: 89 pgs: 89 active+clean; 582 KiB data, 4.9 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:27:38.506 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:27:38.506 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:27:38.607 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:27:38.650 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:27:38.653 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:27:38.653 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:27:38.754 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:27:38.793 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:27:38.796 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:27:38.797 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:27:38.898 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:27:38.935 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:27:38.938 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:27:38.938 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:27:39.039 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:27:39.078 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:27:39.081 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:27:39.082 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:27:39.183 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:27:39.215 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:27:39.219 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:27:39.220 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:27:39.320 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:27:39.354 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:27:39.356 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:27:39.356 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:27:39.457 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:27:39.487 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:27:39.490 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:27:39.491 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:27:39.592 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:27:39.631 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:27:39.634 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:27:39.635 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:27:39.736 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:27:39.778 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:27:39.782 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:27:39.782 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:27:39.883 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:27:39.923 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:27:39.926 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:27:39.927 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:27:40.028 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:27:40.061 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:27:40.063 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:27:40.064 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:27:40.165 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:27:40.205 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:27:40.208 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:27:40.209 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:27:40.310 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:27:40.322 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:27:40 smithi089 bash[78452]: cluster 1685824059.1212907 mgr.y (mgr.34104) 1553 : cluster 0 pgmap v1599: 89 pgs: 89 active+clean; 582 KiB data, 4.8 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:27:40.322 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:27:40 smithi089 bash[75068]: cluster 1685824059.1212907 mgr.y (mgr.34104) 1553 : cluster 0 pgmap v1599: 89 pgs: 89 active+clean; 582 KiB data, 4.8 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:27:40.346 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:27:40.348 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:27:40.349 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:27:40.450 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:27:40.462 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:27:40 smithi164 bash[59462]: cluster 1685824059.1212907 mgr.y (mgr.34104) 1553 : cluster 0 pgmap v1599: 89 pgs: 89 active+clean; 582 KiB data, 4.8 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:27:40.487 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:27:40.489 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:27:40.490 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:27:40.591 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:27:40.627 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:27:40.630 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:27:40.631 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:27:40.732 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:27:40.765 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:27:40.767 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:27:40.768 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:27:40.869 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:27:40.904 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:27:40.907 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:27:40.908 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:27:41.009 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:27:41.034 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:27:41.038 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:27:41.038 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:27:41.056 INFO:teuthology.orchestra.run.smithi164.stderr:Inferring config /var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/mon.b/config 2023-06-03T20:27:41.139 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:27:41.170 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:27:41.172 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:27:41.172 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:27:41.273 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:27:41.307 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:27:41.309 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:27:41.310 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:27:41.411 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:27:41.433 INFO:teuthology.orchestra.run.smithi089.stdout:osd.0: {} 2023-06-03T20:27:41.447 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:27:41.449 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:27:41.450 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:27:41.456 INFO:teuthology.orchestra.run.smithi089.stdout:osd.1: {} 2023-06-03T20:27:41.480 INFO:teuthology.orchestra.run.smithi089.stdout:osd.2: {} 2023-06-03T20:27:41.503 INFO:teuthology.orchestra.run.smithi089.stdout:osd.3: {} 2023-06-03T20:27:41.526 INFO:teuthology.orchestra.run.smithi089.stdout:osd.4: {} 2023-06-03T20:27:41.551 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:27:41.554 INFO:teuthology.orchestra.run.smithi089.stdout:osd.5: {} 2023-06-03T20:27:41.579 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:27:41.582 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:27:41.583 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:27:41.584 INFO:teuthology.orchestra.run.smithi089.stdout:osd.6: {} 2023-06-03T20:27:41.610 INFO:teuthology.orchestra.run.smithi089.stdout:osd.7: {} 2023-06-03T20:27:41.613 INFO:teuthology.orchestra.run.smithi089.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-06-03T20:27:41.613 INFO:teuthology.orchestra.run.smithi089.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-06-03T20:27:41.613 INFO:teuthology.orchestra.run.smithi089.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-06-03T20:27:41.613 INFO:teuthology.orchestra.run.smithi089.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-06-03T20:27:41.613 INFO:teuthology.orchestra.run.smithi089.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-06-03T20:27:41.613 INFO:teuthology.orchestra.run.smithi089.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-06-03T20:27:41.614 INFO:teuthology.orchestra.run.smithi089.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-06-03T20:27:41.614 INFO:teuthology.orchestra.run.smithi089.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-06-03T20:27:41.684 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:27:41.717 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:27:41.721 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:27:41.722 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:27:41.823 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:27:41.858 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:27:41.861 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:27:41.862 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:27:41.962 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:27:41.990 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:27:41.992 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:27:41.992 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:27:42.093 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:27:42.123 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:27:42.125 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:27:42.126 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:27:42.210 INFO:teuthology.orchestra.run.smithi089.stderr:enabled application 'rados' on pool 'unique_pool_9' 2023-06-03T20:27:42.210 INFO:teuthology.orchestra.run.smithi089.stderr:nodeep-scrub is set 2023-06-03T20:27:42.211 INFO:teuthology.orchestra.run.smithi089.stderr:reweighted osd.7 to 1 (10000) 2023-06-03T20:27:42.228 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:27:42.257 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:27:42.259 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:27:42.260 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:27:42.360 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:27:42.374 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:27:42 smithi089 bash[78452]: cluster 1685824061.1216793 mgr.y (mgr.34104) 1554 : cluster 0 pgmap v1600: 89 pgs: 89 active+clean; 582 KiB data, 4.8 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:27:42.374 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:27:42 smithi089 bash[78452]: audit 1685824061.4034963 mon.a (mon.0) 1571 : audit 0 from='client.? 172.21.15.89:0/3702660003' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-03T20:27:42.374 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:27:42 smithi089 bash[78452]: audit 1685824061.4125216 mon.a (mon.0) 1572 : audit 1 from='client.? 172.21.15.89:0/3683604691' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-06-03T20:27:42.374 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:27:42 smithi089 bash[78452]: audit 1685824061.4148934 mon.c (mon.2) 642 : audit 1 from='client.? 172.21.15.89:0/3043384551' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_9", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-06-03T20:27:42.374 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:27:42 smithi089 bash[78452]: audit 1685824061.415471 mon.a (mon.0) 1573 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_9", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-06-03T20:27:42.375 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:27:42 smithi089 bash[78452]: audit 1685824061.5796072 mon.c (mon.2) 643 : audit 1 from='client.? 172.21.15.89:0/2138027147' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-06-03T20:27:42.375 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:27:42 smithi089 bash[78452]: audit 1685824061.5800447 mon.a (mon.0) 1574 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-06-03T20:27:42.375 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:27:42 smithi089 bash[75068]: cluster 1685824061.1216793 mgr.y (mgr.34104) 1554 : cluster 0 pgmap v1600: 89 pgs: 89 active+clean; 582 KiB data, 4.8 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:27:42.375 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:27:42 smithi089 bash[75068]: audit 1685824061.4034963 mon.a (mon.0) 1571 : audit 0 from='client.? 172.21.15.89:0/3702660003' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-03T20:27:42.375 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:27:42 smithi089 bash[75068]: audit 1685824061.4125216 mon.a (mon.0) 1572 : audit 1 from='client.? 172.21.15.89:0/3683604691' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-06-03T20:27:42.375 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:27:42 smithi089 bash[75068]: audit 1685824061.4148934 mon.c (mon.2) 642 : audit 1 from='client.? 172.21.15.89:0/3043384551' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_9", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-06-03T20:27:42.376 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:27:42 smithi089 bash[75068]: audit 1685824061.415471 mon.a (mon.0) 1573 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_9", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-06-03T20:27:42.376 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:27:42 smithi089 bash[75068]: audit 1685824061.5796072 mon.c (mon.2) 643 : audit 1 from='client.? 172.21.15.89:0/2138027147' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-06-03T20:27:42.376 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:27:42 smithi089 bash[75068]: audit 1685824061.5800447 mon.a (mon.0) 1574 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-06-03T20:27:42.398 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:27:42.401 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:27:42.402 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:27:42.503 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:27:42.527 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:27:42 smithi164 bash[59462]: cluster 1685824061.1216793 mgr.y (mgr.34104) 1554 : cluster 0 pgmap v1600: 89 pgs: 89 active+clean; 582 KiB data, 4.8 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:27:42.528 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:27:42 smithi164 bash[59462]: audit 1685824061.4034963 mon.a (mon.0) 1571 : audit 0 from='client.? 172.21.15.89:0/3702660003' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-03T20:27:42.528 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:27:42 smithi164 bash[59462]: audit 1685824061.4125216 mon.a (mon.0) 1572 : audit 1 from='client.? 172.21.15.89:0/3683604691' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-06-03T20:27:42.528 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:27:42 smithi164 bash[59462]: audit 1685824061.4148934 mon.c (mon.2) 642 : audit 1 from='client.? 172.21.15.89:0/3043384551' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_9", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-06-03T20:27:42.528 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:27:42 smithi164 bash[59462]: audit 1685824061.415471 mon.a (mon.0) 1573 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_9", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-06-03T20:27:42.529 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:27:42 smithi164 bash[59462]: audit 1685824061.5796072 mon.c (mon.2) 643 : audit 1 from='client.? 172.21.15.89:0/2138027147' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-06-03T20:27:42.529 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:27:42 smithi164 bash[59462]: audit 1685824061.5800447 mon.a (mon.0) 1574 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-06-03T20:27:42.531 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:27:42.535 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:27:42.535 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:27:42.636 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:27:42.668 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:27:42.670 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:27:42.671 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:27:42.772 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:27:42.804 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:27:42.806 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:27:42.807 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:27:42.908 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:27:42.942 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:27:42.945 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:27:42.946 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:27:43.048 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:27:43.073 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:27:43.076 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:27:43.077 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:27:43.178 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:27:43.209 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:27:43.212 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:27:43.213 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:27:43.314 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:27:43.343 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:27:43.347 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:27:43.347 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:27:43.448 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:27:43.483 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:27:43.485 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:27:43.486 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:27:43.511 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:27:43 smithi089 bash[78452]: cluster 1685824062.19099 mon.a (mon.0) 1575 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-06-03T20:27:43.512 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:27:43 smithi089 bash[78452]: audit 1685824062.1960948 mon.a (mon.0) 1576 : audit 1 from='client.? 172.21.15.89:0/3683604691' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2023-06-03T20:27:43.512 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:27:43 smithi089 bash[78452]: audit 1685824062.196222 mon.a (mon.0) 1577 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_9", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-06-03T20:27:43.512 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:27:43 smithi089 bash[78452]: audit 1685824062.1963167 mon.a (mon.0) 1578 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-06-03T20:27:43.513 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:27:43 smithi089 bash[78452]: cluster 1685824062.1963918 mon.a (mon.0) 1579 : cluster 0 osdmap e281: 8 total, 8 up, 8 in 2023-06-03T20:27:43.513 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:27:43 smithi089 bash[75068]: cluster 1685824062.19099 mon.a (mon.0) 1575 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-06-03T20:27:43.513 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:27:43 smithi089 bash[75068]: audit 1685824062.1960948 mon.a (mon.0) 1576 : audit 1 from='client.? 172.21.15.89:0/3683604691' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2023-06-03T20:27:43.514 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:27:43 smithi089 bash[75068]: audit 1685824062.196222 mon.a (mon.0) 1577 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_9", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-06-03T20:27:43.514 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:27:43 smithi089 bash[75068]: audit 1685824062.1963167 mon.a (mon.0) 1578 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-06-03T20:27:43.514 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:27:43 smithi089 bash[75068]: cluster 1685824062.1963918 mon.a (mon.0) 1579 : cluster 0 osdmap e281: 8 total, 8 up, 8 in 2023-06-03T20:27:43.585 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:27:43 smithi164 bash[59462]: cluster 1685824062.19099 mon.a (mon.0) 1575 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-06-03T20:27:43.586 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:27:43 smithi164 bash[59462]: audit 1685824062.1960948 mon.a (mon.0) 1576 : audit 1 from='client.? 172.21.15.89:0/3683604691' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2023-06-03T20:27:43.586 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:27:43 smithi164 bash[59462]: audit 1685824062.196222 mon.a (mon.0) 1577 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_9", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-06-03T20:27:43.586 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:27:43 smithi164 bash[59462]: audit 1685824062.1963167 mon.a (mon.0) 1578 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-06-03T20:27:43.586 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:27:43 smithi164 bash[59462]: cluster 1685824062.1963918 mon.a (mon.0) 1579 : cluster 0 osdmap e281: 8 total, 8 up, 8 in 2023-06-03T20:27:43.587 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:27:43.621 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:27:43.624 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:27:43.625 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:27:43.726 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:27:43.757 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:27:43.760 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:27:43.761 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:27:43.861 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:27:43.891 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:27:43.893 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:27:43.894 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:27:43.995 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:27:44.024 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:27:44.026 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:27:44.026 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:27:44.127 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:27:44.153 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:27:44.156 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:27:44.156 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:27:44.257 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:27:44.289 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:27:44.290 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:27:44.291 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:27:44.392 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:27:44.403 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:27:44 smithi164 bash[59462]: cluster 1685824063.1227074 mgr.y (mgr.34104) 1555 : cluster 0 pgmap v1602: 89 pgs: 89 active+clean; 582 KiB data, 4.8 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:27:44.404 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:27:44 smithi164 bash[59462]: cluster 1685824063.1283906 osd.6 (osd.6) 214 : cluster 3 failed to encode map e281 with expected crc 2023-06-03T20:27:44.404 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:27:44 smithi164 bash[59462]: cluster 1685824063.1284382 osd.7 (osd.7) 229 : cluster 3 failed to encode map e281 with expected crc 2023-06-03T20:27:44.404 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:27:44 smithi164 bash[59462]: cluster 1685824063.1285257 osd.5 (osd.5) 230 : cluster 3 failed to encode map e281 with expected crc 2023-06-03T20:27:44.424 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:27:44.427 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:27:44.427 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:27:44.528 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:27:44.529 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:27:44 smithi089 bash[75068]: cluster 1685824063.1227074 mgr.y (mgr.34104) 1555 : cluster 0 pgmap v1602: 89 pgs: 89 active+clean; 582 KiB data, 4.8 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:27:44.529 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:27:44 smithi089 bash[75068]: cluster 1685824063.1283906 osd.6 (osd.6) 214 : cluster 3 failed to encode map e281 with expected crc 2023-06-03T20:27:44.529 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:27:44 smithi089 bash[75068]: cluster 1685824063.1284382 osd.7 (osd.7) 229 : cluster 3 failed to encode map e281 with expected crc 2023-06-03T20:27:44.529 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:27:44 smithi089 bash[75068]: cluster 1685824063.1285257 osd.5 (osd.5) 230 : cluster 3 failed to encode map e281 with expected crc 2023-06-03T20:27:44.530 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:27:44 smithi089 bash[78452]: cluster 1685824063.1227074 mgr.y (mgr.34104) 1555 : cluster 0 pgmap v1602: 89 pgs: 89 active+clean; 582 KiB data, 4.8 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:27:44.530 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:27:44 smithi089 bash[78452]: cluster 1685824063.1283906 osd.6 (osd.6) 214 : cluster 3 failed to encode map e281 with expected crc 2023-06-03T20:27:44.530 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:27:44 smithi089 bash[78452]: cluster 1685824063.1284382 osd.7 (osd.7) 229 : cluster 3 failed to encode map e281 with expected crc 2023-06-03T20:27:44.530 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:27:44 smithi089 bash[78452]: cluster 1685824063.1285257 osd.5 (osd.5) 230 : cluster 3 failed to encode map e281 with expected crc 2023-06-03T20:27:44.563 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:27:44.567 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:27:44.567 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:27:44.668 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:27:44.699 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:27:44.702 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:27:44.703 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:27:44.804 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:27:44.835 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:27:44.837 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:27:44.837 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:27:44.938 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:27:44.974 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:27:44.976 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:27:44.977 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:27:45.078 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:27:45.121 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:27:45.123 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:27:45.123 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:27:45.158 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-06-03T20:27:45.225 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:27:45.258 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:27:45.260 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:27:45.260 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:27:45.361 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:27:45.394 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:27:45.396 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:27:45.396 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:27:45.497 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:27:45.524 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:27:45.526 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:27:45.526 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:27:45.562 INFO:teuthology.orchestra.run.smithi164.stdout:{ 2023-06-03T20:27:45.562 INFO:teuthology.orchestra.run.smithi164.stdout: "ops": [], 2023-06-03T20:27:45.562 INFO:teuthology.orchestra.run.smithi164.stdout: "complaint_time": 30, 2023-06-03T20:27:45.562 INFO:teuthology.orchestra.run.smithi164.stdout: "num_blocked_ops": 0 2023-06-03T20:27:45.563 INFO:teuthology.orchestra.run.smithi164.stdout:} 2023-06-03T20:27:45.627 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:27:45.663 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:27:45.665 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:27:45.665 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:27:45.766 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:27:45.796 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:27:45.799 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:27:45.799 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:27:45.810 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph osd reweight 1 1 2023-06-03T20:27:45.901 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:27:45.939 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:27:45.942 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:27:45.943 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:27:46.043 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:27:46.077 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:27:46.080 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:27:46.081 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:27:46.182 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:27:46.220 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:27:46.223 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:27:46.224 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:27:46.325 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:27:46.338 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 20:27:46 smithi089 bash[71163]: ::ffff:172.21.15.164 - - [03/Jun/2023:20:27:46] "GET /metrics HTTP/1.1" 200 228798 "" "Prometheus/2.33.4" 2023-06-03T20:27:46.339 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:27:46 smithi089 bash[78452]: cluster 1685824065.1233003 mgr.y (mgr.34104) 1556 : cluster 0 pgmap v1603: 89 pgs: 89 active+clean; 582 KiB data, 4.8 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:27:46.339 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:27:46 smithi089 bash[75068]: cluster 1685824065.1233003 mgr.y (mgr.34104) 1556 : cluster 0 pgmap v1603: 89 pgs: 89 active+clean; 582 KiB data, 4.8 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:27:46.364 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:27:46.367 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:27:46.368 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:27:46.469 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:27:46.482 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:27:46 smithi164 bash[59462]: cluster 1685824065.1233003 mgr.y (mgr.34104) 1556 : cluster 0 pgmap v1603: 89 pgs: 89 active+clean; 582 KiB data, 4.8 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:27:46.501 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:27:46.504 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:27:46.505 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:27:46.561 DEBUG:teuthology.orchestra.run.smithi164:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph daemon osd.4 dump_historic_ops 2023-06-03T20:27:46.607 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:27:46.647 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:27:46.650 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:27:46.651 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:27:46.751 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:27:46.790 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:27:46.792 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:27:46.793 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:27:46.794 DEBUG:teuthology.orchestra.run.smithi089:> /bin/sh -c 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage rados --no-log-to-stderr --name client.0 -b 65536 --object-size 65536 -p unique_pool_9 bench 90 write' 2023-06-03T20:27:46.797 INFO:tasks.radosbench:joining radosbench (timing out after 3000s) 2023-06-03T20:27:46.797 INFO:teuthology.orchestra.run:waiting for 3000 2023-06-03T20:27:46.851 INFO:tasks.radosbench.radosbench.0.smithi089.stdout:hints = 1 2023-06-03T20:27:46.852 INFO:tasks.radosbench.radosbench.0.smithi089.stdout:Maintaining 16 concurrent writes of 65536 bytes to objects of size 65536 for up to 90 seconds or 0 objects 2023-06-03T20:27:46.852 INFO:tasks.radosbench.radosbench.0.smithi089.stdout:Object prefix: benchmark_data_smithi089_237281 2023-06-03T20:27:46.853 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-06-03T20:27:46.853 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 0 0 0 0 0 0 - 0 2023-06-03T20:27:46.894 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:27:46.937 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:27:46.942 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:27:46.943 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:27:47.044 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:27:47.079 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:27:47.083 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:27:47.083 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:27:47.184 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:27:47.216 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:27:47.219 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:27:47.219 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:27:47.320 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:27:47.356 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:27:47.358 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:27:47.359 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:27:47.440 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph osd unset noscrub 2023-06-03T20:27:47.460 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:27:47.496 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:27:47.498 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:27:47.499 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:27:47.599 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:27:47.647 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:27:47.651 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:27:47.651 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:27:47.752 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:27:47.786 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:27:47.789 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:27:47.790 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:27:47.852 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 1 16 2649 2633 164.554 164.562 0.00441614 0.00605037 2023-06-03T20:27:47.892 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:27:47.929 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:27:47.931 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:27:47.932 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:27:48.033 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:27:48.066 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:27:48.068 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:27:48.069 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:27:48.170 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:27:48.201 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:27:48.204 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:27:48.205 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:27:48.217 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:27:48 smithi164 bash[59462]: cluster 1685824067.1238241 mgr.y (mgr.34104) 1557 : cluster 0 pgmap v1604: 89 pgs: 89 active+clean; 2.4 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 186 KiB/s wr, 2 op/s 2023-06-03T20:27:48.305 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:27:48.341 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:27:48.344 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:27:48.345 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:27:48.446 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:27:48.479 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:27:48.482 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:27:48.483 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:27:48.529 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:27:48 smithi089 bash[78452]: cluster 1685824067.1238241 mgr.y (mgr.34104) 1557 : cluster 0 pgmap v1604: 89 pgs: 89 active+clean; 2.4 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 186 KiB/s wr, 2 op/s 2023-06-03T20:27:48.529 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:27:48 smithi089 bash[75068]: cluster 1685824067.1238241 mgr.y (mgr.34104) 1557 : cluster 0 pgmap v1604: 89 pgs: 89 active+clean; 2.4 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 186 KiB/s wr, 2 op/s 2023-06-03T20:27:48.584 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:27:48.615 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:27:48.617 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:27:48.618 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:27:48.719 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:27:48.751 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:27:48.753 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:27:48.754 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:27:48.852 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 2 16 5169 5153 161.019 157.5 0.0060346 0.00619648 2023-06-03T20:27:48.854 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:27:48.890 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:27:48.893 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:27:48.894 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:27:48.995 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:27:49.029 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:27:49.032 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:27:49.033 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:27:49.134 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:27:49.167 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:27:49.171 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:27:49.172 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:27:49.273 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:27:49.304 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:27:49.307 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:27:49.308 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:27:49.409 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:27:49.440 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:27:49.442 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:27:49.444 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:27:49.544 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:27:49.578 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:27:49.581 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:27:49.581 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:27:49.682 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:27:49.714 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:27:49.716 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:27:49.717 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:27:49.818 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:27:49.849 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:27:49.852 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 3 16 7658 7642 159.195 155.562 0.00445589 0.00627025 2023-06-03T20:27:49.853 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:27:49.853 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:27:49.954 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:27:49.977 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/mon.a/config 2023-06-03T20:27:49.977 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/mon.a/config 2023-06-03T20:27:49.986 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:27:49.990 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:27:49.991 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:27:50.092 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:27:50.128 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:27:50.132 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:27:50.133 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:27:50.221 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:27:50 smithi164 bash[59462]: cluster 1685824069.124564 mgr.y (mgr.34104) 1558 : cluster 0 pgmap v1605: 89 pgs: 89 active+clean; 109 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 11 MiB/s wr, 174 op/s 2023-06-03T20:27:50.233 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:27:50.270 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:27:50.272 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:27:50 smithi089 bash[78452]: cluster 1685824069.124564 mgr.y (mgr.34104) 1558 : cluster 0 pgmap v1605: 89 pgs: 89 active+clean; 109 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 11 MiB/s wr, 174 op/s 2023-06-03T20:27:50.273 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:27:50.274 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:27:50.274 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:27:50 smithi089 bash[75068]: cluster 1685824069.124564 mgr.y (mgr.34104) 1558 : cluster 0 pgmap v1605: 89 pgs: 89 active+clean; 109 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 11 MiB/s wr, 174 op/s 2023-06-03T20:27:50.375 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:27:50.418 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:27:50.421 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:27:50.422 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:27:50.522 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:27:50.557 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:27:50.560 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:27:50.560 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:27:50.661 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:27:50.697 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:27:50.700 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:27:50.701 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:27:50.801 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:27:50.837 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:27:50.841 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:27:50.842 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:27:50.852 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 4 16 9622 9606 150.081 122.75 0.0026694 0.00665348 2023-06-03T20:27:50.943 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:27:50.975 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:27:50.978 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:27:50.979 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:27:51.080 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:27:51.113 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:27:51.116 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:27:51.116 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:27:51.217 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:27:51.253 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:27:51.255 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:27:51.256 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:27:51.357 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:27:51.387 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:27:51.390 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:27:51.391 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:27:51.407 INFO:teuthology.orchestra.run.smithi164.stderr:Inferring config /var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/mon.b/config 2023-06-03T20:27:51.492 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:27:51.501 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:27:51 smithi164 bash[59462]: audit 1685824070.2507837 mon.c (mon.2) 644 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-03T20:27:51.528 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:27:51.529 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:27:51 smithi089 bash[78452]: audit 1685824070.2507837 mon.c (mon.2) 644 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-03T20:27:51.529 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:27:51 smithi089 bash[75068]: audit 1685824070.2507837 mon.c (mon.2) 644 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-03T20:27:51.531 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:27:51.532 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:27:51.633 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:27:51.675 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:27:51.678 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:27:51.679 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:27:51.780 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:27:51.813 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:27:51.816 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:27:51.816 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:27:51.852 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 5 16 11835 11819 147.725 138.312 0.00628439 0.00675869 2023-06-03T20:27:51.917 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:27:51.958 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:27:51.961 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:27:51.961 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:27:52.045 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/mon.a/config 2023-06-03T20:27:52.062 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:27:52.093 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:27:52.095 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:27:52.095 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:27:52.196 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:27:52.234 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:27:52.234 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:27:52 smithi089 bash[75068]: cluster 1685824071.125041 mgr.y (mgr.34104) 1559 : cluster 0 pgmap v1606: 89 pgs: 89 active+clean; 109 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 11 MiB/s wr, 174 op/s 2023-06-03T20:27:52.234 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:27:52 smithi089 bash[75068]: audit 1685824072.1491857 mon.c (mon.2) 645 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T20:27:52.235 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:27:52.236 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:27:52.335 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:27:52 smithi164 bash[59462]: cluster 1685824071.125041 mgr.y (mgr.34104) 1559 : cluster 0 pgmap v1606: 89 pgs: 89 active+clean; 109 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 11 MiB/s wr, 174 op/s 2023-06-03T20:27:52.335 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:27:52 smithi164 bash[59462]: audit 1685824072.1491857 mon.c (mon.2) 645 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T20:27:52.337 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:27:52.370 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:27:52.372 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:27:52.372 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:27:52.473 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:27:52.506 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:27:52.509 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:27:52 smithi089 bash[78452]: cluster 1685824071.125041 mgr.y (mgr.34104) 1559 : cluster 0 pgmap v1606: 89 pgs: 89 active+clean; 109 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 11 MiB/s wr, 174 op/s 2023-06-03T20:27:52.509 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:27:52 smithi089 bash[78452]: audit 1685824072.1491857 mon.c (mon.2) 645 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T20:27:52.510 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:27:52.510 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:27:52.611 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:27:52.646 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:27:52.653 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:27:52.654 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:27:52.755 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:27:52.784 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:27:52.786 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:27:52.787 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:27:52.852 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 6 16 13937 13921 144.998 131.375 0.00948613 0.0068896 2023-06-03T20:27:52.888 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:27:52.917 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:27:52.920 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:27:52.921 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:27:53.022 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:27:53.059 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:27:53.062 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:27:53.062 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:27:53.163 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:27:53.187 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:27:53.190 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:27:53.191 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:27:53.291 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:27:53.320 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:27:53.322 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:27:53.323 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:27:53.424 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:27:53.454 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:27:53.458 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:27:53.458 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:27:53.559 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:27:53.599 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:27:53.604 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:27:53.604 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:27:53.705 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:27:53.739 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:27:53.743 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:27:53.744 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:27:53.845 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:27:53.852 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 7 16 15971 15955 142.443 127.125 0.0259481 0.00701231 2023-06-03T20:27:53.879 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:27:53.881 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:27:53.882 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:27:53.983 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:27:54.013 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:27:54.016 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:27:54.017 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:27:54.118 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:27:54.151 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:27:54.155 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:27:54.155 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:27:54.257 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:27:54.267 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:27:54 smithi089 bash[78452]: cluster 1685824073.1258435 mgr.y (mgr.34104) 1560 : cluster 0 pgmap v1607: 89 pgs: 89 active+clean; 810 MiB data, 7.0 GiB used, 708 GiB / 715 GiB avail; 74 MiB/s wr, 1.18k op/s 2023-06-03T20:27:54.268 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:27:54 smithi089 bash[75068]: cluster 1685824073.1258435 mgr.y (mgr.34104) 1560 : cluster 0 pgmap v1607: 89 pgs: 89 active+clean; 810 MiB data, 7.0 GiB used, 708 GiB / 715 GiB avail; 74 MiB/s wr, 1.18k op/s 2023-06-03T20:27:54.287 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:27:54.295 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:27:54.296 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:27:54.334 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:27:54 smithi164 bash[59462]: cluster 1685824073.1258435 mgr.y (mgr.34104) 1560 : cluster 0 pgmap v1607: 89 pgs: 89 active+clean; 810 MiB data, 7.0 GiB used, 708 GiB / 715 GiB avail; 74 MiB/s wr, 1.18k op/s 2023-06-03T20:27:54.397 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:27:54.428 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:27:54.431 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:27:54.432 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:27:54.533 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:27:54.580 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:27:54.582 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:27:54.583 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:27:54.684 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:27:54.716 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:27:54.721 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:27:54.722 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:27:54.824 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:27:54.853 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 8 16 18010 17994 140.562 127.438 0.0110816 0.0071063 2023-06-03T20:27:54.853 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:27:54.856 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:27:54.857 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:27:54.958 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:27:55.000 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:27:55.003 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:27:55.004 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:27:55.105 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:27:55.138 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:27:55.143 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:27:55.143 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:27:55.245 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:27:55.305 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:27:55.308 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:27:55.309 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:27:55.410 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:27:55.449 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:27:55.455 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:27:55.455 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:27:55.556 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:27:55.595 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:27:55.601 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:27:55.605 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:27:55.706 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:27:55.737 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:27:55.742 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:27:55.742 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:27:55.843 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:27:55.853 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 9 16 19670 19654 136.47 103.75 0.0047505 0.00731787 2023-06-03T20:27:55.878 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:27:55.914 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:27:55.914 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:27:56.015 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:27:56.049 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:27:56.053 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:27:56.054 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:27:56.155 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:27:56.191 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:27:56 smithi089 bash[75068]: cluster 1685824075.1262648 mgr.y (mgr.34104) 1561 : cluster 0 pgmap v1608: 89 pgs: 89 active+clean; 810 MiB data, 7.0 GiB used, 708 GiB / 715 GiB avail; 67 MiB/s wr, 1.08k op/s 2023-06-03T20:27:56.191 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:27:56 smithi089 bash[75068]: audit 1685824075.999021 mon.a (mon.0) 1580 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:27:56.191 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:27:56 smithi089 bash[75068]: audit 1685824076.010611 mon.a (mon.0) 1581 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:27:56.192 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:27:56.193 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:27:56.194 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:27:56.295 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:27:56.332 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:27:56.335 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:27:56.336 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:27:56.437 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:27:56.446 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 20:27:56 smithi089 bash[71163]: ::ffff:172.21.15.164 - - [03/Jun/2023:20:27:56] "GET /metrics HTTP/1.1" 200 228983 "" "Prometheus/2.33.4" 2023-06-03T20:27:56.447 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:27:56 smithi089 bash[78452]: cluster 1685824075.1262648 mgr.y (mgr.34104) 1561 : cluster 0 pgmap v1608: 89 pgs: 89 active+clean; 810 MiB data, 7.0 GiB used, 708 GiB / 715 GiB avail; 67 MiB/s wr, 1.08k op/s 2023-06-03T20:27:56.447 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:27:56 smithi089 bash[78452]: audit 1685824075.999021 mon.a (mon.0) 1580 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:27:56.447 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:27:56 smithi089 bash[78452]: audit 1685824076.010611 mon.a (mon.0) 1581 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:27:56.473 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:27:56.476 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:27:56.477 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:27:56.494 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:27:56 smithi164 bash[59462]: cluster 1685824075.1262648 mgr.y (mgr.34104) 1561 : cluster 0 pgmap v1608: 89 pgs: 89 active+clean; 810 MiB data, 7.0 GiB used, 708 GiB / 715 GiB avail; 67 MiB/s wr, 1.08k op/s 2023-06-03T20:27:56.495 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:27:56 smithi164 bash[59462]: audit 1685824075.999021 mon.a (mon.0) 1580 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:27:56.495 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:27:56 smithi164 bash[59462]: audit 1685824076.010611 mon.a (mon.0) 1581 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:27:56.577 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:27:56.608 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:27:56.611 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:27:56.612 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:27:56.713 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:27:56.760 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:27:56.762 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:27:56.763 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:27:56.853 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 10 16 21174 21158 132.222 94 0.00429139 0.00755436 2023-06-03T20:27:56.864 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:27:56.900 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:27:56.904 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:27:56.904 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:27:57.005 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:27:57.034 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:27:57.035 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:27:57.036 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:27:57.137 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:27:57.171 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:27:57.173 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:27:57.174 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:27:57.275 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:27:57.307 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:27:57.310 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:27:57.311 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:27:57.412 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:27:57.444 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:27:57.446 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:27:57.446 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:27:57.547 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:27:57.591 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:27:57.593 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:27:57.594 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:27:57.695 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:27:57.727 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:27:57.733 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:27:57.733 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:27:57.834 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:27:57.844 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:27:57 smithi089 bash[78452]: audit 1685824076.6960802 mon.a (mon.0) 1582 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:27:57.844 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:27:57 smithi089 bash[78452]: audit 1685824076.709255 mon.a (mon.0) 1583 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:27:57.844 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:27:57 smithi089 bash[78452]: cluster 1685824077.1268284 mgr.y (mgr.34104) 1562 : cluster 0 pgmap v1609: 89 pgs: 89 active+clean; 961 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 80 MiB/s wr, 1.28k op/s 2023-06-03T20:27:57.844 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:27:57 smithi089 bash[78452]: audit 1685824077.4318964 mon.a (mon.0) 1584 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:27:57.844 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:27:57 smithi089 bash[78452]: audit 1685824077.4441397 mon.a (mon.0) 1585 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:27:57.845 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:27:57 smithi089 bash[75068]: audit 1685824076.6960802 mon.a (mon.0) 1582 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:27:57.845 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:27:57 smithi089 bash[75068]: audit 1685824076.709255 mon.a (mon.0) 1583 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:27:57.845 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:27:57 smithi089 bash[75068]: cluster 1685824077.1268284 mgr.y (mgr.34104) 1562 : cluster 0 pgmap v1609: 89 pgs: 89 active+clean; 961 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 80 MiB/s wr, 1.28k op/s 2023-06-03T20:27:57.845 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:27:57 smithi089 bash[75068]: audit 1685824077.4318964 mon.a (mon.0) 1584 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:27:57.845 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:27:57 smithi089 bash[75068]: audit 1685824077.4441397 mon.a (mon.0) 1585 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:27:57.853 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 11 16 22862 22846 129.792 105.5 0.0147667 0.0076953 2023-06-03T20:27:57.870 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:27:57.875 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:27:57.876 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:27:57.976 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:27:57.984 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:27:57 smithi164 bash[59462]: audit 1685824076.6960802 mon.a (mon.0) 1582 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:27:57.984 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:27:57 smithi164 bash[59462]: audit 1685824076.709255 mon.a (mon.0) 1583 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:27:57.984 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:27:57 smithi164 bash[59462]: cluster 1685824077.1268284 mgr.y (mgr.34104) 1562 : cluster 0 pgmap v1609: 89 pgs: 89 active+clean; 961 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 80 MiB/s wr, 1.28k op/s 2023-06-03T20:27:57.985 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:27:57 smithi164 bash[59462]: audit 1685824077.4318964 mon.a (mon.0) 1584 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:27:57.985 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:27:57 smithi164 bash[59462]: audit 1685824077.4441397 mon.a (mon.0) 1585 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:27:58.016 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:27:58.017 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:27:58.018 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:27:58.111 INFO:teuthology.orchestra.run.smithi164.stdout:{ 2023-06-03T20:27:58.111 INFO:teuthology.orchestra.run.smithi164.stdout: "size": 20, 2023-06-03T20:27:58.111 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 600, 2023-06-03T20:27:58.111 INFO:teuthology.orchestra.run.smithi164.stdout: "ops": [ 2023-06-03T20:27:58.111 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:27:58.112 INFO:teuthology.orchestra.run.smithi164.stdout: "description": "osd_op(client.36573.0:116993 13.d 13.f97d498d (undecoded) ondisk+retry+write+known_if_redirected e250)", 2023-06-03T20:27:58.112 INFO:teuthology.orchestra.run.smithi164.stdout: "initiated_at": "2023-06-03T20:23:26.244097+0000", 2023-06-03T20:27:58.112 INFO:teuthology.orchestra.run.smithi164.stdout: "age": 271.85519967200003, 2023-06-03T20:27:58.112 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 1.0120220230000001, 2023-06-03T20:27:58.112 INFO:teuthology.orchestra.run.smithi164.stdout: "type_data": { 2023-06-03T20:27:58.112 INFO:teuthology.orchestra.run.smithi164.stdout: "flag_point": "reached pg", 2023-06-03T20:27:58.112 INFO:teuthology.orchestra.run.smithi164.stdout: "client_info": { 2023-06-03T20:27:58.113 INFO:teuthology.orchestra.run.smithi164.stdout: "client": "client.36573", 2023-06-03T20:27:58.113 INFO:teuthology.orchestra.run.smithi164.stdout: "client_addr": "172.21.15.89:0/616445942", 2023-06-03T20:27:58.113 INFO:teuthology.orchestra.run.smithi164.stdout: "tid": 116993 2023-06-03T20:27:58.113 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:27:58.113 INFO:teuthology.orchestra.run.smithi164.stdout: "events": [ 2023-06-03T20:27:58.113 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:27:58.113 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "initiated", 2023-06-03T20:27:58.113 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:23:26.244097+0000", 2023-06-03T20:27:58.114 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0 2023-06-03T20:27:58.114 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:27:58.114 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:27:58.114 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "throttled", 2023-06-03T20:27:58.114 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:23:26.244097+0000", 2023-06-03T20:27:58.114 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0 2023-06-03T20:27:58.114 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:27:58.114 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:27:58.115 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "header_read", 2023-06-03T20:27:58.115 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:23:26.244095+0000", 2023-06-03T20:27:58.115 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 4294967295.9999981 2023-06-03T20:27:58.115 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:27:58.115 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:27:58.115 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "all_read", 2023-06-03T20:27:58.115 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:23:26.244103+0000", 2023-06-03T20:27:58.115 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 7.7959999999999996e-06 2023-06-03T20:27:58.116 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:27:58.116 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:27:58.116 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "dispatched", 2023-06-03T20:27:58.116 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:23:26.244116+0000", 2023-06-03T20:27:58.116 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 1.33e-05 2023-06-03T20:27:58.116 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:27:58.116 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:27:58.117 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "queued_for_pg", 2023-06-03T20:27:58.117 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:23:26.244128+0000", 2023-06-03T20:27:58.117 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 1.2063e-05 2023-06-03T20:27:58.117 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:27:58.117 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:27:58.117 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "reached_pg", 2023-06-03T20:27:58.117 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:23:26.250402+0000", 2023-06-03T20:27:58.117 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.0062744050000000003 2023-06-03T20:27:58.118 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:27:58.118 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:27:58.118 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "waiting for peered", 2023-06-03T20:27:58.118 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:23:26.250930+0000", 2023-06-03T20:27:58.118 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.00052798199999999997 2023-06-03T20:27:58.118 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:27:58.118 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:27:58.118 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "reached_pg", 2023-06-03T20:27:58.119 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:23:27.256111+0000", 2023-06-03T20:27:58.119 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 1.005180325 2023-06-03T20:27:58.119 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:27:58.119 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:27:58.119 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "done", 2023-06-03T20:27:58.119 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:23:27.256119+0000", 2023-06-03T20:27:58.119 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 7.9300000000000003e-06 2023-06-03T20:27:58.119 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:27:58.120 INFO:teuthology.orchestra.run.smithi164.stdout: ] 2023-06-03T20:27:58.120 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:27:58.120 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:27:58.120 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:27:58.120 INFO:teuthology.orchestra.run.smithi164.stdout: "description": "osd_op(client.36573.0:116995 13.d 13.521c468d (undecoded) ondisk+retry+write+known_if_redirected e250)", 2023-06-03T20:27:58.120 INFO:teuthology.orchestra.run.smithi164.stdout: "initiated_at": "2023-06-03T20:23:26.244146+0000", 2023-06-03T20:27:58.120 INFO:teuthology.orchestra.run.smithi164.stdout: "age": 271.85515031800003, 2023-06-03T20:27:58.120 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 1.0120016599999999, 2023-06-03T20:27:58.121 INFO:teuthology.orchestra.run.smithi164.stdout: "type_data": { 2023-06-03T20:27:58.121 INFO:teuthology.orchestra.run.smithi164.stdout: "flag_point": "reached pg", 2023-06-03T20:27:58.121 INFO:teuthology.orchestra.run.smithi164.stdout: "client_info": { 2023-06-03T20:27:58.121 INFO:teuthology.orchestra.run.smithi164.stdout: "client": "client.36573", 2023-06-03T20:27:58.121 INFO:teuthology.orchestra.run.smithi164.stdout: "client_addr": "172.21.15.89:0/616445942", 2023-06-03T20:27:58.121 INFO:teuthology.orchestra.run.smithi164.stdout: "tid": 116995 2023-06-03T20:27:58.121 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:27:58.121 INFO:teuthology.orchestra.run.smithi164.stdout: "events": [ 2023-06-03T20:27:58.122 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:27:58.122 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "initiated", 2023-06-03T20:27:58.122 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:23:26.244146+0000", 2023-06-03T20:27:58.122 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0 2023-06-03T20:27:58.122 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:27:58.122 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:27:58.122 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "throttled", 2023-06-03T20:27:58.122 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:23:26.244146+0000", 2023-06-03T20:27:58.123 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0 2023-06-03T20:27:58.123 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:27:58.123 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:27:58.123 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "header_read", 2023-06-03T20:27:58.123 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:23:26.244145+0000", 2023-06-03T20:27:58.123 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 4294967295.999999 2023-06-03T20:27:58.123 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:27:58.123 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:27:58.124 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "all_read", 2023-06-03T20:27:58.124 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:27:58.126 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:23:26.244150+0000", 2023-06-03T20:27:58.126 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 4.6190000000000002e-06 2023-06-03T20:27:58.126 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:27:58.127 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:27:58.127 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "dispatched", 2023-06-03T20:27:58.127 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:23:26.244158+0000", 2023-06-03T20:27:58.127 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 7.8990000000000001e-06 2023-06-03T20:27:58.127 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:27:58.128 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:27:58.128 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "queued_for_pg", 2023-06-03T20:27:58.128 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:23:26.244165+0000", 2023-06-03T20:27:58.128 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 7.622e-06 2023-06-03T20:27:58.129 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:27:58.129 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:27:58.129 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "reached_pg", 2023-06-03T20:27:58.129 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:23:26.250970+0000", 2023-06-03T20:27:58.130 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.0068049390000000003 2023-06-03T20:27:58.130 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:27:58.130 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:27:58.130 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "waiting for peered", 2023-06-03T20:27:58.131 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:23:26.250978+0000", 2023-06-03T20:27:58.131 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 7.5780000000000001e-06 2023-06-03T20:27:58.131 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:27:58.131 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:27:58.131 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "reached_pg", 2023-06-03T20:27:58.132 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:23:27.256139+0000", 2023-06-03T20:27:58.132 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 1.005161569 2023-06-03T20:27:58.132 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:27:58.132 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:27:58.133 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "done", 2023-06-03T20:27:58.133 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:23:27.256148+0000", 2023-06-03T20:27:58.133 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 8.2730000000000002e-06 2023-06-03T20:27:58.133 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:27:58.133 INFO:teuthology.orchestra.run.smithi164.stdout: ] 2023-06-03T20:27:58.134 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:27:58.134 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:27:58.134 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:27:58.134 INFO:teuthology.orchestra.run.smithi164.stdout: "description": "osd_op(client.36573.0:117000 13.d 13.30933f3d (undecoded) ondisk+write+known_if_redirected e250)", 2023-06-03T20:27:58.134 INFO:teuthology.orchestra.run.smithi164.stdout: "initiated_at": "2023-06-03T20:23:26.244171+0000", 2023-06-03T20:27:58.135 INFO:teuthology.orchestra.run.smithi164.stdout: "age": 271.85512566, 2023-06-03T20:27:58.135 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 1.0120043860000001, 2023-06-03T20:27:58.135 INFO:teuthology.orchestra.run.smithi164.stdout: "type_data": { 2023-06-03T20:27:58.135 INFO:teuthology.orchestra.run.smithi164.stdout: "flag_point": "reached pg", 2023-06-03T20:27:58.136 INFO:teuthology.orchestra.run.smithi164.stdout: "client_info": { 2023-06-03T20:27:58.136 INFO:teuthology.orchestra.run.smithi164.stdout: "client": "client.36573", 2023-06-03T20:27:58.136 INFO:teuthology.orchestra.run.smithi164.stdout: "client_addr": "172.21.15.89:0/616445942", 2023-06-03T20:27:58.136 INFO:teuthology.orchestra.run.smithi164.stdout: "tid": 117000 2023-06-03T20:27:58.137 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:27:58.137 INFO:teuthology.orchestra.run.smithi164.stdout: "events": [ 2023-06-03T20:27:58.137 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:27:58.137 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "initiated", 2023-06-03T20:27:58.137 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:23:26.244171+0000", 2023-06-03T20:27:58.137 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0 2023-06-03T20:27:58.138 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:27:58.138 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:27:58.138 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "throttled", 2023-06-03T20:27:58.138 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:23:26.244171+0000", 2023-06-03T20:27:58.138 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0 2023-06-03T20:27:58.138 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:27:58.138 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:27:58.138 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "header_read", 2023-06-03T20:27:58.139 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:23:26.244170+0000", 2023-06-03T20:27:58.139 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 4294967295.9999995 2023-06-03T20:27:58.139 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:27:58.139 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:27:58.139 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "all_read", 2023-06-03T20:27:58.139 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:23:26.244173+0000", 2023-06-03T20:27:58.139 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 2.9220000000000001e-06 2023-06-03T20:27:58.140 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:27:58.140 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:27:58.140 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "dispatched", 2023-06-03T20:27:58.140 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:23:26.244179+0000", 2023-06-03T20:27:58.140 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 6.2589999999999997e-06 2023-06-03T20:27:58.140 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:27:58.141 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:27:58.141 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "queued_for_pg", 2023-06-03T20:27:58.141 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:23:26.244185+0000", 2023-06-03T20:27:58.142 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 5.3249999999999998e-06 2023-06-03T20:27:58.142 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:27:58.142 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:27:58.142 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "reached_pg", 2023-06-03T20:27:58.142 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:23:26.251058+0000", 2023-06-03T20:27:58.143 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.0068736209999999999 2023-06-03T20:27:58.143 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:27:58.143 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:27:58.143 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "waiting for peered", 2023-06-03T20:27:58.143 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:23:26.251072+0000", 2023-06-03T20:27:58.144 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 1.3525e-05 2023-06-03T20:27:58.144 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:27:58.144 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:27:58.144 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "reached_pg", 2023-06-03T20:27:58.144 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:23:27.256168+0000", 2023-06-03T20:27:58.144 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 1.005096539 2023-06-03T20:27:58.145 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:27:58.145 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:27:58.145 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "done", 2023-06-03T20:27:58.145 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:23:27.256175+0000", 2023-06-03T20:27:58.145 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 6.7610000000000002e-06 2023-06-03T20:27:58.145 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:27:58.145 INFO:teuthology.orchestra.run.smithi164.stdout: ] 2023-06-03T20:27:58.146 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:27:58.146 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:27:58.146 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:27:58.146 INFO:teuthology.orchestra.run.smithi164.stdout: "description": "osd_op(client.36573.0:117001 13.5 13.1b6a2fb5 (undecoded) ondisk+write+known_if_redirected e250)", 2023-06-03T20:27:58.146 INFO:teuthology.orchestra.run.smithi164.stdout: "initiated_at": "2023-06-03T20:23:26.252966+0000", 2023-06-03T20:27:58.146 INFO:teuthology.orchestra.run.smithi164.stdout: "age": 271.84633065700001, 2023-06-03T20:27:58.146 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 1.0031182009999999, 2023-06-03T20:27:58.146 INFO:teuthology.orchestra.run.smithi164.stdout: "type_data": { 2023-06-03T20:27:58.147 INFO:teuthology.orchestra.run.smithi164.stdout: "flag_point": "reached pg", 2023-06-03T20:27:58.147 INFO:teuthology.orchestra.run.smithi164.stdout: "client_info": { 2023-06-03T20:27:58.147 INFO:teuthology.orchestra.run.smithi164.stdout: "client": "client.36573", 2023-06-03T20:27:58.147 INFO:teuthology.orchestra.run.smithi164.stdout: "client_addr": "172.21.15.89:0/616445942", 2023-06-03T20:27:58.147 INFO:teuthology.orchestra.run.smithi164.stdout: "tid": 117001 2023-06-03T20:27:58.147 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:27:58.147 INFO:teuthology.orchestra.run.smithi164.stdout: "events": [ 2023-06-03T20:27:58.148 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:27:58.148 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "initiated", 2023-06-03T20:27:58.148 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:23:26.252966+0000", 2023-06-03T20:27:58.148 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0 2023-06-03T20:27:58.148 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:27:58.148 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:27:58.148 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "throttled", 2023-06-03T20:27:58.149 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:23:26.252966+0000", 2023-06-03T20:27:58.149 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0 2023-06-03T20:27:58.149 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:27:58.149 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:27:58.149 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "header_read", 2023-06-03T20:27:58.149 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:23:26.252964+0000", 2023-06-03T20:27:58.149 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 4294967295.9999981 2023-06-03T20:27:58.149 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:27:58.150 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:27:58.150 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "all_read", 2023-06-03T20:27:58.150 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:23:26.252972+0000", 2023-06-03T20:27:58.150 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 7.7519999999999996e-06 2023-06-03T20:27:58.150 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:27:58.150 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:27:58.150 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "dispatched", 2023-06-03T20:27:58.151 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:23:26.252998+0000", 2023-06-03T20:27:58.151 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 2.6771000000000002e-05 2023-06-03T20:27:58.151 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:27:58.151 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:27:58.151 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "queued_for_pg", 2023-06-03T20:27:58.151 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:23:26.253021+0000", 2023-06-03T20:27:58.152 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 2.2121999999999999e-05 2023-06-03T20:27:58.152 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:27:58.152 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:27:58.152 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "reached_pg", 2023-06-03T20:27:58.152 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:23:26.253075+0000", 2023-06-03T20:27:58.152 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 5.4447000000000002e-05 2023-06-03T20:27:58.152 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:27:58.153 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:27:58.153 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "waiting for peered", 2023-06-03T20:27:58.153 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:23:26.253085+0000", 2023-06-03T20:27:58.153 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 1.0132000000000001e-05 2023-06-03T20:27:58.153 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:27:58.153 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:27:58.153 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "reached_pg", 2023-06-03T20:27:58.153 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:23:27.256068+0000", 2023-06-03T20:27:58.154 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 1.00298329 2023-06-03T20:27:58.154 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:27:58.154 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:27:58.154 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "done", 2023-06-03T20:27:58.154 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:23:27.256084+0000", 2023-06-03T20:27:58.154 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 1.5379999999999998e-05 2023-06-03T20:27:58.154 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:27:58.155 INFO:teuthology.orchestra.run.smithi164.stdout: ] 2023-06-03T20:27:58.155 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:27:58.155 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:27:58.155 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:27:58.155 INFO:teuthology.orchestra.run.smithi164.stdout: "description": "osd_op(client.36573.0:117002 13.d 13.a33661fd (undecoded) ondisk+write+known_if_redirected e250)", 2023-06-03T20:27:58.155 INFO:teuthology.orchestra.run.smithi164.stdout: "initiated_at": "2023-06-03T20:23:26.259696+0000", 2023-06-03T20:27:58.155 INFO:teuthology.orchestra.run.smithi164.stdout: "age": 271.83959998500001, 2023-06-03T20:27:58.155 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.99652494899999999, 2023-06-03T20:27:58.156 INFO:teuthology.orchestra.run.smithi164.stdout: "type_data": { 2023-06-03T20:27:58.156 INFO:teuthology.orchestra.run.smithi164.stdout: "flag_point": "reached pg", 2023-06-03T20:27:58.156 INFO:teuthology.orchestra.run.smithi164.stdout: "client_info": { 2023-06-03T20:27:58.156 INFO:teuthology.orchestra.run.smithi164.stdout: "client": "client.36573", 2023-06-03T20:27:58.156 INFO:teuthology.orchestra.run.smithi164.stdout: "client_addr": "172.21.15.89:0/616445942", 2023-06-03T20:27:58.156 INFO:teuthology.orchestra.run.smithi164.stdout: "tid": 117002 2023-06-03T20:27:58.157 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:27:58.157 INFO:teuthology.orchestra.run.smithi164.stdout: "events": [ 2023-06-03T20:27:58.157 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:27:58.157 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "initiated", 2023-06-03T20:27:58.158 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:23:26.259696+0000", 2023-06-03T20:27:58.158 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0 2023-06-03T20:27:58.158 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:27:58.158 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:27:58.159 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "throttled", 2023-06-03T20:27:58.159 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:23:26.259696+0000", 2023-06-03T20:27:58.159 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0 2023-06-03T20:27:58.160 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:27:58.160 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:27:58.160 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "header_read", 2023-06-03T20:27:58.160 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:23:26.259695+0000", 2023-06-03T20:27:58.161 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 4294967295.9999986 2023-06-03T20:27:58.161 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:27:58.161 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:27:58.161 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "all_read", 2023-06-03T20:27:58.162 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:23:26.259702+0000", 2023-06-03T20:27:58.162 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 6.6819999999999997e-06 2023-06-03T20:27:58.162 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:27:58.162 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:27:58.163 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "dispatched", 2023-06-03T20:27:58.163 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:23:26.259714+0000", 2023-06-03T20:27:58.163 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 1.2376e-05 2023-06-03T20:27:58.163 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:27:58.163 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:27:58.163 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "queued_for_pg", 2023-06-03T20:27:58.164 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:23:26.259730+0000", 2023-06-03T20:27:58.164 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 1.5736999999999999e-05 2023-06-03T20:27:58.164 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:27:58.164 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:27:58.164 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "reached_pg", 2023-06-03T20:27:58.164 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:23:26.259787+0000", 2023-06-03T20:27:58.164 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 5.7327999999999997e-05 2023-06-03T20:27:58.165 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:27:58.165 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:27:58.165 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "waiting for peered", 2023-06-03T20:27:58.165 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:23:26.259797+0000", 2023-06-03T20:27:58.165 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 1.0205e-05 2023-06-03T20:27:58.165 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:27:58.165 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:27:58.166 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "reached_pg", 2023-06-03T20:27:58.166 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:23:27.256215+0000", 2023-06-03T20:27:58.166 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.99641731499999997 2023-06-03T20:27:58.166 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:27:58.166 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:27:58.166 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "done", 2023-06-03T20:27:58.166 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:23:27.256221+0000", 2023-06-03T20:27:58.167 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 6.5459999999999997e-06 2023-06-03T20:27:58.167 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:27:58.167 INFO:teuthology.orchestra.run.smithi164.stdout: ] 2023-06-03T20:27:58.167 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:27:58.167 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:27:58.167 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:27:58.167 INFO:teuthology.orchestra.run.smithi164.stdout: "description": "osd_op(client.36573.0:117004 13.5 13.c88b9cf5 (undecoded) ondisk+write+known_if_redirected e250)", 2023-06-03T20:27:58.167 INFO:teuthology.orchestra.run.smithi164.stdout: "initiated_at": "2023-06-03T20:23:26.264887+0000", 2023-06-03T20:27:58.168 INFO:teuthology.orchestra.run.smithi164.stdout: "age": 271.83440883700001, 2023-06-03T20:27:58.168 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.99124461100000005, 2023-06-03T20:27:58.168 INFO:teuthology.orchestra.run.smithi164.stdout: "type_data": { 2023-06-03T20:27:58.168 INFO:teuthology.orchestra.run.smithi164.stdout: "flag_point": "reached pg", 2023-06-03T20:27:58.168 INFO:teuthology.orchestra.run.smithi164.stdout: "client_info": { 2023-06-03T20:27:58.168 INFO:teuthology.orchestra.run.smithi164.stdout: "client": "client.36573", 2023-06-03T20:27:58.168 INFO:teuthology.orchestra.run.smithi164.stdout: "client_addr": "172.21.15.89:0/616445942", 2023-06-03T20:27:58.169 INFO:teuthology.orchestra.run.smithi164.stdout: "tid": 117004 2023-06-03T20:27:58.169 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:27:58.169 INFO:teuthology.orchestra.run.smithi164.stdout: "events": [ 2023-06-03T20:27:58.169 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:27:58.169 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "initiated", 2023-06-03T20:27:58.169 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:23:26.264887+0000", 2023-06-03T20:27:58.169 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0 2023-06-03T20:27:58.169 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:27:58.170 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:27:58.170 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "throttled", 2023-06-03T20:27:58.170 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:23:26.264887+0000", 2023-06-03T20:27:58.170 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0 2023-06-03T20:27:58.170 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:27:58.170 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:27:58.170 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "header_read", 2023-06-03T20:27:58.171 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:23:26.264886+0000", 2023-06-03T20:27:58.171 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 4294967295.999999 2023-06-03T20:27:58.171 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:27:58.171 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:27:58.171 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "all_read", 2023-06-03T20:27:58.171 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:23:26.264893+0000", 2023-06-03T20:27:58.171 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 6.2589999999999997e-06 2023-06-03T20:27:58.172 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:27:58.172 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:27:58.172 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "dispatched", 2023-06-03T20:27:58.172 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:23:26.264907+0000", 2023-06-03T20:27:58.172 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 1.3891e-05 2023-06-03T20:27:58.173 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:27:58.173 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:27:58.173 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "queued_for_pg", 2023-06-03T20:27:58.173 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:23:26.264933+0000", 2023-06-03T20:27:58.173 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 2.6906e-05 2023-06-03T20:27:58.173 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:27:58.173 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:27:58.173 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "reached_pg", 2023-06-03T20:27:58.174 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:23:26.264990+0000", 2023-06-03T20:27:58.174 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 5.6393e-05 2023-06-03T20:27:58.174 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:27:58.174 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:27:58.174 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "waiting for peered", 2023-06-03T20:27:58.174 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:23:26.265002+0000", 2023-06-03T20:27:58.174 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 1.252e-05 2023-06-03T20:27:58.174 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:27:58.175 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:27:58.175 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "reached_pg", 2023-06-03T20:27:58.175 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:23:27.256123+0000", 2023-06-03T20:27:58.175 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.99112042600000005 2023-06-03T20:27:58.175 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:27:58.176 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:27:58.176 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "done", 2023-06-03T20:27:58.176 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:23:27.256132+0000", 2023-06-03T20:27:58.176 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 9.1959999999999997e-06 2023-06-03T20:27:58.176 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:27:58.176 INFO:teuthology.orchestra.run.smithi164.stdout: ] 2023-06-03T20:27:58.177 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:27:58.177 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:27:58.177 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:27:58.177 INFO:teuthology.orchestra.run.smithi164.stdout: "description": "osd_op(client.36573.0:117005 13.d 13.ae147efd (undecoded) ondisk+write+known_if_redirected e250)", 2023-06-03T20:27:58.177 INFO:teuthology.orchestra.run.smithi164.stdout: "initiated_at": "2023-06-03T20:23:26.264942+0000", 2023-06-03T20:27:58.177 INFO:teuthology.orchestra.run.smithi164.stdout: "age": 271.834354023, 2023-06-03T20:27:58.177 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.99130487599999995, 2023-06-03T20:27:58.177 INFO:teuthology.orchestra.run.smithi164.stdout: "type_data": { 2023-06-03T20:27:58.178 INFO:teuthology.orchestra.run.smithi164.stdout: "flag_point": "reached pg", 2023-06-03T20:27:58.178 INFO:teuthology.orchestra.run.smithi164.stdout: "client_info": { 2023-06-03T20:27:58.178 INFO:teuthology.orchestra.run.smithi164.stdout: "client": "client.36573", 2023-06-03T20:27:58.178 INFO:teuthology.orchestra.run.smithi164.stdout: "client_addr": "172.21.15.89:0/616445942", 2023-06-03T20:27:58.178 INFO:teuthology.orchestra.run.smithi164.stdout: "tid": 117005 2023-06-03T20:27:58.178 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:27:58.178 INFO:teuthology.orchestra.run.smithi164.stdout: "events": [ 2023-06-03T20:27:58.179 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:27:58.179 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "initiated", 2023-06-03T20:27:58.179 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:23:26.264942+0000", 2023-06-03T20:27:58.179 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0 2023-06-03T20:27:58.179 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:27:58.179 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:27:58.179 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "throttled", 2023-06-03T20:27:58.179 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:23:26.264942+0000", 2023-06-03T20:27:58.180 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0 2023-06-03T20:27:58.180 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:27:58.180 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:27:58.180 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "header_read", 2023-06-03T20:27:58.180 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:23:26.264942+0000", 2023-06-03T20:27:58.180 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 4294967295.9999995 2023-06-03T20:27:58.180 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:27:58.181 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:27:58.181 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "all_read", 2023-06-03T20:27:58.181 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:23:26.264945+0000", 2023-06-03T20:27:58.181 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 3.2430000000000001e-06 2023-06-03T20:27:58.181 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:27:58.181 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:27:58.182 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "dispatched", 2023-06-03T20:27:58.182 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:23:26.264952+0000", 2023-06-03T20:27:58.182 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 6.6900000000000003e-06 2023-06-03T20:27:58.182 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:27:58.182 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:27:58.182 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "queued_for_pg", 2023-06-03T20:27:58.182 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:23:26.264957+0000", 2023-06-03T20:27:58.182 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 5.0830000000000003e-06 2023-06-03T20:27:58.183 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:27:58.183 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:27:58.183 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "reached_pg", 2023-06-03T20:27:58.183 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:23:26.265021+0000", 2023-06-03T20:27:58.183 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 6.4377999999999999e-05 2023-06-03T20:27:58.183 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:27:58.183 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:27:58.184 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "waiting for peered", 2023-06-03T20:27:58.184 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:23:26.265026+0000", 2023-06-03T20:27:58.184 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 5.118e-06 2023-06-03T20:27:58.184 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:27:58.184 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:27:58.184 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "reached_pg", 2023-06-03T20:27:58.184 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:23:27.256238+0000", 2023-06-03T20:27:58.184 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.99121135999999999 2023-06-03T20:27:58.185 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:27:58.185 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:27:58.185 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "done", 2023-06-03T20:27:58.185 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:23:27.256247+0000", 2023-06-03T20:27:58.185 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 9.4450000000000002e-06 2023-06-03T20:27:58.185 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:27:58.185 INFO:teuthology.orchestra.run.smithi164.stdout: ] 2023-06-03T20:27:58.186 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:27:58.186 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:27:58.186 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:27:58.186 INFO:teuthology.orchestra.run.smithi164.stdout: "description": "osd_repop(client.36573.0:152112 13.1 e257/251 13:84a226e9:::benchmark_data_smithi089_211581_object36587:head v 257'9521, mlcod=245'102)", 2023-06-03T20:27:58.186 INFO:teuthology.orchestra.run.smithi164.stdout: "initiated_at": "2023-06-03T20:24:30.155003+0000", 2023-06-03T20:27:58.186 INFO:teuthology.orchestra.run.smithi164.stdout: "age": 207.94429364999999, 2023-06-03T20:27:58.186 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.127813703, 2023-06-03T20:27:58.186 INFO:teuthology.orchestra.run.smithi164.stdout: "type_data": { 2023-06-03T20:27:58.187 INFO:teuthology.orchestra.run.smithi164.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-03T20:27:58.187 INFO:teuthology.orchestra.run.smithi164.stdout: "events": [ 2023-06-03T20:27:58.187 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:27:58.187 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "initiated", 2023-06-03T20:27:58.187 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:24:30.155003+0000", 2023-06-03T20:27:58.187 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0 2023-06-03T20:27:58.187 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:27:58.188 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:27:58.188 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "throttled", 2023-06-03T20:27:58.188 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:24:30.155003+0000", 2023-06-03T20:27:58.188 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0 2023-06-03T20:27:58.188 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:27:58.188 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:27:58.188 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "header_read", 2023-06-03T20:27:58.188 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:24:30.155002+0000", 2023-06-03T20:27:58.189 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 4294967295.999999 2023-06-03T20:27:58.189 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:27:58.189 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:27:58.189 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "all_read", 2023-06-03T20:27:58.189 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:24:30.155010+0000", 2023-06-03T20:27:58.189 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 8.2919999999999996e-06 2023-06-03T20:27:58.189 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:27:58.189 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:27:58.190 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "dispatched", 2023-06-03T20:27:58.190 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:24:30.155021+0000", 2023-06-03T20:27:58.190 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 1.1355000000000001e-05 2023-06-03T20:27:58.190 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:27:58.190 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:27:58.190 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "queued_for_pg", 2023-06-03T20:27:58.190 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:24:30.155031+0000", 2023-06-03T20:27:58.191 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 1.0186999999999999e-05 2023-06-03T20:27:58.191 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:27:58.191 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:27:58.191 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "reached_pg", 2023-06-03T20:27:58.191 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:24:30.155107+0000", 2023-06-03T20:27:58.191 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 7.5517000000000002e-05 2023-06-03T20:27:58.191 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:27:58.191 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:27:58.192 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "started", 2023-06-03T20:27:58.192 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:24:30.155137+0000", 2023-06-03T20:27:58.192 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 3.0321999999999999e-05 2023-06-03T20:27:58.192 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:27:58.192 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:27:58.192 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "commit_sent", 2023-06-03T20:27:58.192 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:24:30.282771+0000", 2023-06-03T20:27:58.193 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.127633515 2023-06-03T20:27:58.193 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:27:58.193 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:27:58.193 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "done", 2023-06-03T20:27:58.193 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:24:30.282816+0000", 2023-06-03T20:27:58.193 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 4.5445999999999998e-05 2023-06-03T20:27:58.193 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:27:58.193 INFO:teuthology.orchestra.run.smithi164.stdout: ] 2023-06-03T20:27:58.194 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:27:58.194 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:27:58.194 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:27:58.194 INFO:teuthology.orchestra.run.smithi164.stdout: "description": "osd_repop(client.36573.0:179654 13.a e264/251 13:55e6ce98:::benchmark_data_smithi089_211581_object64129:head v 264'11157, mlcod=245'88)", 2023-06-03T20:27:58.194 INFO:teuthology.orchestra.run.smithi164.stdout: "initiated_at": "2023-06-03T20:25:20.702533+0000", 2023-06-03T20:27:58.194 INFO:teuthology.orchestra.run.smithi164.stdout: "age": 157.396762749, 2023-06-03T20:27:58.194 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.12643020999999999, 2023-06-03T20:27:58.194 INFO:teuthology.orchestra.run.smithi164.stdout: "type_data": { 2023-06-03T20:27:58.195 INFO:teuthology.orchestra.run.smithi164.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-03T20:27:58.195 INFO:teuthology.orchestra.run.smithi164.stdout: "events": [ 2023-06-03T20:27:58.195 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:27:58.195 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "initiated", 2023-06-03T20:27:58.195 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:25:20.702533+0000", 2023-06-03T20:27:58.195 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0 2023-06-03T20:27:58.195 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:27:58.195 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:27:58.196 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "throttled", 2023-06-03T20:27:58.196 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:25:20.702533+0000", 2023-06-03T20:27:58.196 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0 2023-06-03T20:27:58.196 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:27:58.196 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:27:58.196 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "header_read", 2023-06-03T20:27:58.196 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:25:20.702532+0000", 2023-06-03T20:27:58.197 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 4294967295.999999 2023-06-03T20:27:58.197 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:27:58.197 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:27:58.197 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "all_read", 2023-06-03T20:27:58.197 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:25:20.702543+0000", 2023-06-03T20:27:58.197 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 1.1002000000000001e-05 2023-06-03T20:27:58.197 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:27:58.197 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:27:58.198 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "dispatched", 2023-06-03T20:27:58.198 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:25:20.702556+0000", 2023-06-03T20:27:58.198 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 1.2862e-05 2023-06-03T20:27:58.198 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:27:58.198 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:27:58.198 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "queued_for_pg", 2023-06-03T20:27:58.198 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:25:20.702566+0000", 2023-06-03T20:27:58.199 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 9.3540000000000006e-06 2023-06-03T20:27:58.199 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:27:58.199 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:27:58.199 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "reached_pg", 2023-06-03T20:27:58.199 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:25:20.757519+0000", 2023-06-03T20:27:58.199 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.054953492 2023-06-03T20:27:58.199 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:27:58.199 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:27:58.200 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "started", 2023-06-03T20:27:58.200 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:25:20.757546+0000", 2023-06-03T20:27:58.200 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 2.7294e-05 2023-06-03T20:27:58.200 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:27:58.200 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:27:58.200 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "commit_sent", 2023-06-03T20:27:58.200 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:25:20.828927+0000", 2023-06-03T20:27:58.200 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.071380576000000001 2023-06-03T20:27:58.201 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:27:58.201 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:27:58.201 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "done", 2023-06-03T20:27:58.201 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:25:20.828964+0000", 2023-06-03T20:27:58.201 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 3.6677000000000001e-05 2023-06-03T20:27:58.201 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:27:58.201 INFO:teuthology.orchestra.run.smithi164.stdout: ] 2023-06-03T20:27:58.201 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:27:58.202 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:27:58.202 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:27:58.202 INFO:teuthology.orchestra.run.smithi164.stdout: "description": "osd_repop(client.36573.0:180038 13.a e264/251 13:53f2b68d:::benchmark_data_smithi089_211581_object64513:head v 264'11176, mlcod=245'88)", 2023-06-03T20:27:58.202 INFO:teuthology.orchestra.run.smithi164.stdout: "initiated_at": "2023-06-03T20:25:21.255989+0000", 2023-06-03T20:27:58.202 INFO:teuthology.orchestra.run.smithi164.stdout: "age": 156.84330721699999, 2023-06-03T20:27:58.202 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.124430602, 2023-06-03T20:27:58.202 INFO:teuthology.orchestra.run.smithi164.stdout: "type_data": { 2023-06-03T20:27:58.203 INFO:teuthology.orchestra.run.smithi164.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-03T20:27:58.203 INFO:teuthology.orchestra.run.smithi164.stdout: "events": [ 2023-06-03T20:27:58.203 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:27:58.203 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "initiated", 2023-06-03T20:27:58.203 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:25:21.255989+0000", 2023-06-03T20:27:58.203 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0 2023-06-03T20:27:58.203 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:27:58.203 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:27:58.204 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "throttled", 2023-06-03T20:27:58.204 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:25:21.255989+0000", 2023-06-03T20:27:58.204 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0 2023-06-03T20:27:58.204 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:27:58.204 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:27:58.204 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "header_read", 2023-06-03T20:27:58.204 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:25:21.255988+0000", 2023-06-03T20:27:58.205 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 4294967295.9999986 2023-06-03T20:27:58.205 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:27:58.205 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:27:58.205 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "all_read", 2023-06-03T20:27:58.205 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:25:21.255998+0000", 2023-06-03T20:27:58.205 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 1.0144e-05 2023-06-03T20:27:58.205 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:27:58.206 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:27:58.206 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "dispatched", 2023-06-03T20:27:58.206 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:25:21.256014+0000", 2023-06-03T20:27:58.206 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 1.577e-05 2023-06-03T20:27:58.206 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:27:58.206 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:27:58.206 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "queued_for_pg", 2023-06-03T20:27:58.206 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:25:21.256023+0000", 2023-06-03T20:27:58.207 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 8.8629999999999997e-06 2023-06-03T20:27:58.207 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:27:58.207 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:27:58.207 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "reached_pg", 2023-06-03T20:27:58.207 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:25:21.300334+0000", 2023-06-03T20:27:58.207 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.044311670999999997 2023-06-03T20:27:58.207 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:27:58.207 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:27:58.208 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "started", 2023-06-03T20:27:58.208 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:25:21.300541+0000", 2023-06-03T20:27:58.208 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.000207178 2023-06-03T20:27:58.208 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:27:58.208 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:27:58.208 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "commit_sent", 2023-06-03T20:27:58.208 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:25:21.380346+0000", 2023-06-03T20:27:58.209 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.079804458999999994 2023-06-03T20:27:58.209 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:27:58.209 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:27:58.209 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "done", 2023-06-03T20:27:58.209 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:25:21.380420+0000", 2023-06-03T20:27:58.209 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 7.3730000000000001e-05 2023-06-03T20:27:58.209 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:27:58.210 INFO:teuthology.orchestra.run.smithi164.stdout: ] 2023-06-03T20:27:58.210 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:27:58.210 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:27:58.210 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:27:58.210 INFO:teuthology.orchestra.run.smithi164.stdout: "description": "osd_repop(client.36573.0:180047 13.a e264/251 13:5b6f6de7:::benchmark_data_smithi089_211581_object64522:head v 264'11178, mlcod=245'88)", 2023-06-03T20:27:58.210 INFO:teuthology.orchestra.run.smithi164.stdout: "initiated_at": "2023-06-03T20:25:21.268860+0000", 2023-06-03T20:27:58.210 INFO:teuthology.orchestra.run.smithi164.stdout: "age": 156.83043642199999, 2023-06-03T20:27:58.210 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.14026773000000001, 2023-06-03T20:27:58.211 INFO:teuthology.orchestra.run.smithi164.stdout: "type_data": { 2023-06-03T20:27:58.211 INFO:teuthology.orchestra.run.smithi164.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-03T20:27:58.211 INFO:teuthology.orchestra.run.smithi164.stdout: "events": [ 2023-06-03T20:27:58.211 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:27:58.211 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "initiated", 2023-06-03T20:27:58.211 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:25:21.268860+0000", 2023-06-03T20:27:58.211 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0 2023-06-03T20:27:58.212 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:27:58.212 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:27:58.212 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "throttled", 2023-06-03T20:27:58.212 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:25:21.268860+0000", 2023-06-03T20:27:58.212 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0 2023-06-03T20:27:58.212 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:27:58.212 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:27:58.213 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "header_read", 2023-06-03T20:27:58.213 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:25:21.268858+0000", 2023-06-03T20:27:58.213 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 4294967295.9999986 2023-06-03T20:27:58.213 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:27:58.213 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:27:58.213 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "all_read", 2023-06-03T20:27:58.213 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:25:21.268873+0000", 2023-06-03T20:27:58.213 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 1.4331000000000001e-05 2023-06-03T20:27:58.214 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:27:58.214 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:27:58.214 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "dispatched", 2023-06-03T20:27:58.214 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:25:21.268889+0000", 2023-06-03T20:27:58.214 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 1.6447999999999999e-05 2023-06-03T20:27:58.214 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:27:58.214 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:27:58.214 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "queued_for_pg", 2023-06-03T20:27:58.215 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:25:21.268898+0000", 2023-06-03T20:27:58.215 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 8.5539999999999998e-06 2023-06-03T20:27:58.215 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:27:58.215 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:27:58.215 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "reached_pg", 2023-06-03T20:27:58.215 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:25:21.332446+0000", 2023-06-03T20:27:58.215 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.063547841999999993 2023-06-03T20:27:58.216 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:27:58.216 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:27:58.216 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "started", 2023-06-03T20:27:58.216 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:25:21.332469+0000", 2023-06-03T20:27:58.216 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 2.3136e-05 2023-06-03T20:27:58.216 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:27:58.216 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:27:58.216 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "commit_sent", 2023-06-03T20:27:58.217 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:25:21.409067+0000", 2023-06-03T20:27:58.217 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.076597928999999995 2023-06-03T20:27:58.217 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:27:58.217 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:27:58.217 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "done", 2023-06-03T20:27:58.217 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:25:21.409128+0000", 2023-06-03T20:27:58.218 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 6.0881999999999997e-05 2023-06-03T20:27:58.218 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:27:58.218 INFO:teuthology.orchestra.run.smithi164.stdout: ] 2023-06-03T20:27:58.218 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:27:58.218 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:27:58.218 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:27:58.218 INFO:teuthology.orchestra.run.smithi164.stdout: "description": "osd_repop(client.36573.0:180052 13.a e264/251 13:58df9d48:::benchmark_data_smithi089_211581_object64527:head v 264'11179, mlcod=245'88)", 2023-06-03T20:27:58.218 INFO:teuthology.orchestra.run.smithi164.stdout: "initiated_at": "2023-06-03T20:25:21.278394+0000", 2023-06-03T20:27:58.219 INFO:teuthology.orchestra.run.smithi164.stdout: "age": 156.820902096, 2023-06-03T20:27:58.219 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.13077418399999999, 2023-06-03T20:27:58.219 INFO:teuthology.orchestra.run.smithi164.stdout: "type_data": { 2023-06-03T20:27:58.219 INFO:teuthology.orchestra.run.smithi164.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-03T20:27:58.219 INFO:teuthology.orchestra.run.smithi164.stdout: "events": [ 2023-06-03T20:27:58.219 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:27:58.219 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "initiated", 2023-06-03T20:27:58.219 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:25:21.278394+0000", 2023-06-03T20:27:58.220 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0 2023-06-03T20:27:58.220 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:27:58.220 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:27:58.220 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "throttled", 2023-06-03T20:27:58.220 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:25:21.278394+0000", 2023-06-03T20:27:58.220 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0 2023-06-03T20:27:58.220 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:27:58.221 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:27:58.221 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "header_read", 2023-06-03T20:27:58.221 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:25:21.278393+0000", 2023-06-03T20:27:58.221 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 4294967295.9999986 2023-06-03T20:27:58.221 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:27:58.221 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:27:58.221 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "all_read", 2023-06-03T20:27:58.221 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:25:21.278407+0000", 2023-06-03T20:27:58.222 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 1.4619e-05 2023-06-03T20:27:58.222 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:27:58.222 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:27:58.222 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "dispatched", 2023-06-03T20:27:58.222 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:25:21.278429+0000", 2023-06-03T20:27:58.222 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 2.1954e-05 2023-06-03T20:27:58.222 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:27:58.222 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:27:58.223 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "queued_for_pg", 2023-06-03T20:27:58.223 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:25:21.278442+0000", 2023-06-03T20:27:58.223 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 1.2605e-05 2023-06-03T20:27:58.223 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:27:58.223 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:27:58.223 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "reached_pg", 2023-06-03T20:27:58.223 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:25:21.347505+0000", 2023-06-03T20:27:58.223 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.069062904999999994 2023-06-03T20:27:58.224 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:27:58.224 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:27:58.224 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "started", 2023-06-03T20:27:58.224 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:25:21.347560+0000", 2023-06-03T20:27:58.224 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 5.5789999999999999e-05 2023-06-03T20:27:58.224 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:27:58.224 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:27:58.224 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "commit_sent", 2023-06-03T20:27:58.225 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:25:21.409137+0000", 2023-06-03T20:27:58.225 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.061576946 2023-06-03T20:27:58.225 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:27:58.225 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:27:58.225 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "done", 2023-06-03T20:27:58.225 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:25:21.409168+0000", 2023-06-03T20:27:58.226 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 3.0907000000000001e-05 2023-06-03T20:27:58.226 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:27:58.226 INFO:teuthology.orchestra.run.smithi164.stdout: ] 2023-06-03T20:27:58.226 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:27:58.226 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:27:58.226 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:27:58.226 INFO:teuthology.orchestra.run.smithi164.stdout: "description": "osd_repop(client.36573.0:180055 13.a e264/251 13:568a5a4a:::benchmark_data_smithi089_211581_object64530:head v 264'11180, mlcod=245'88)", 2023-06-03T20:27:58.227 INFO:teuthology.orchestra.run.smithi164.stdout: "initiated_at": "2023-06-03T20:25:21.286202+0000", 2023-06-03T20:27:58.227 INFO:teuthology.orchestra.run.smithi164.stdout: "age": 156.81309377900001, 2023-06-03T20:27:58.227 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.137420608, 2023-06-03T20:27:58.227 INFO:teuthology.orchestra.run.smithi164.stdout: "type_data": { 2023-06-03T20:27:58.227 INFO:teuthology.orchestra.run.smithi164.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-03T20:27:58.227 INFO:teuthology.orchestra.run.smithi164.stdout: "events": [ 2023-06-03T20:27:58.227 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:27:58.227 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "initiated", 2023-06-03T20:27:58.228 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:25:21.286202+0000", 2023-06-03T20:27:58.228 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0 2023-06-03T20:27:58.228 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:27:58.228 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:27:58.228 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "throttled", 2023-06-03T20:27:58.229 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:25:21.286202+0000", 2023-06-03T20:27:58.229 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0 2023-06-03T20:27:58.229 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:27:58.229 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:27:58.230 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "header_read", 2023-06-03T20:27:58.230 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:25:21.286201+0000", 2023-06-03T20:27:58.230 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 4294967295.999999 2023-06-03T20:27:58.230 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:27:58.230 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:27:58.230 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "all_read", 2023-06-03T20:27:58.230 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:25:21.286210+0000", 2023-06-03T20:27:58.230 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 8.636e-06 2023-06-03T20:27:58.231 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:27:58.231 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:27:58.231 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "dispatched", 2023-06-03T20:27:58.231 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:25:21.286228+0000", 2023-06-03T20:27:58.231 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 1.7509000000000001e-05 2023-06-03T20:27:58.231 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:27:58.232 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:27:58.232 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "queued_for_pg", 2023-06-03T20:27:58.232 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:25:21.286235+0000", 2023-06-03T20:27:58.232 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 7.4850000000000003e-06 2023-06-03T20:27:58.232 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:27:58.232 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:27:58.232 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "reached_pg", 2023-06-03T20:27:58.233 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:25:21.361281+0000", 2023-06-03T20:27:58.233 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.075045710000000002 2023-06-03T20:27:58.233 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:27:58.233 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:27:58.233 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "started", 2023-06-03T20:27:58.233 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:25:21.361308+0000", 2023-06-03T20:27:58.233 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 2.6812e-05 2023-06-03T20:27:58.233 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:27:58.234 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:27:58.234 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "commit_sent", 2023-06-03T20:27:58.234 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:25:21.423577+0000", 2023-06-03T20:27:58.234 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.062269494000000002 2023-06-03T20:27:58.234 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:27:58.234 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:27:58.234 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "done", 2023-06-03T20:27:58.235 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:25:21.423623+0000", 2023-06-03T20:27:58.235 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 4.5955999999999998e-05 2023-06-03T20:27:58.235 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:27:58.235 INFO:teuthology.orchestra.run.smithi164.stdout: ] 2023-06-03T20:27:58.235 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:27:58.235 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:27:58.235 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:27:58.236 INFO:teuthology.orchestra.run.smithi164.stdout: "description": "osd_repop(client.36573.0:207724 13.a e267/251 13:56abe777:::benchmark_data_smithi089_211581_object92199:head v 267'12925, mlcod=245'88)", 2023-06-03T20:27:58.236 INFO:teuthology.orchestra.run.smithi164.stdout: "initiated_at": "2023-06-03T20:26:05.161153+0000", 2023-06-03T20:27:58.236 INFO:teuthology.orchestra.run.smithi164.stdout: "age": 112.938143677, 2023-06-03T20:27:58.237 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.14069462199999999, 2023-06-03T20:27:58.237 INFO:teuthology.orchestra.run.smithi164.stdout: "type_data": { 2023-06-03T20:27:58.237 INFO:teuthology.orchestra.run.smithi164.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-03T20:27:58.237 INFO:teuthology.orchestra.run.smithi164.stdout: "events": [ 2023-06-03T20:27:58.238 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:27:58.238 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "initiated", 2023-06-03T20:27:58.238 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:26:05.161153+0000", 2023-06-03T20:27:58.239 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0 2023-06-03T20:27:58.239 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:27:58.239 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:27:58.239 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "throttled", 2023-06-03T20:27:58.240 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:26:05.161153+0000", 2023-06-03T20:27:58.240 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0 2023-06-03T20:27:58.240 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:27:58.240 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:27:58.240 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "header_read", 2023-06-03T20:27:58.241 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:26:05.161150+0000", 2023-06-03T20:27:58.241 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 4294967295.9999976 2023-06-03T20:27:58.241 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:27:58.241 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:27:58.242 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "all_read", 2023-06-03T20:27:58.242 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:26:05.161163+0000", 2023-06-03T20:27:58.242 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 1.2646999999999999e-05 2023-06-03T20:27:58.242 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:27:58.243 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:27:58.243 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "dispatched", 2023-06-03T20:27:58.243 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:26:05.161176+0000", 2023-06-03T20:27:58.243 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 1.3471e-05 2023-06-03T20:27:58.243 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:27:58.243 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:27:58.244 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "queued_for_pg", 2023-06-03T20:27:58.244 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:26:05.161189+0000", 2023-06-03T20:27:58.244 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 1.2860000000000001e-05 2023-06-03T20:27:58.244 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:27:58.244 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:27:58.244 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "reached_pg", 2023-06-03T20:27:58.244 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:26:05.203972+0000", 2023-06-03T20:27:58.245 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.042783396000000001 2023-06-03T20:27:58.245 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:27:58.245 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:27:58.245 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "started", 2023-06-03T20:27:58.245 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:26:05.204002+0000", 2023-06-03T20:27:58.245 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 2.9784000000000001e-05 2023-06-03T20:27:58.245 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:27:58.245 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:27:58.246 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "commit_sent", 2023-06-03T20:27:58.246 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:26:05.301792+0000", 2023-06-03T20:27:58.246 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.097789475000000001 2023-06-03T20:27:58.246 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:27:58.246 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:27:58.246 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "done", 2023-06-03T20:27:58.246 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:26:05.301847+0000", 2023-06-03T20:27:58.247 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 5.5596000000000001e-05 2023-06-03T20:27:58.247 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:27:58.247 INFO:teuthology.orchestra.run.smithi164.stdout: ] 2023-06-03T20:27:58.247 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:27:58.247 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:27:58.247 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:27:58.247 INFO:teuthology.orchestra.run.smithi164.stdout: "description": "osd_repop(client.36573.0:207729 13.a e267/251 13:57a06b0d:::benchmark_data_smithi089_211581_object92204:head v 267'12926, mlcod=245'88)", 2023-06-03T20:27:58.247 INFO:teuthology.orchestra.run.smithi164.stdout: "initiated_at": "2023-06-03T20:26:05.174631+0000", 2023-06-03T20:27:58.248 INFO:teuthology.orchestra.run.smithi164.stdout: "age": 112.924665244, 2023-06-03T20:27:58.248 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.12724138099999999, 2023-06-03T20:27:58.248 INFO:teuthology.orchestra.run.smithi164.stdout: "type_data": { 2023-06-03T20:27:58.248 INFO:teuthology.orchestra.run.smithi164.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-03T20:27:58.248 INFO:teuthology.orchestra.run.smithi164.stdout: "events": [ 2023-06-03T20:27:58.248 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:27:58.248 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "initiated", 2023-06-03T20:27:58.249 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:26:05.174631+0000", 2023-06-03T20:27:58.249 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0 2023-06-03T20:27:58.249 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:27:58.249 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:27:58.249 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "throttled", 2023-06-03T20:27:58.250 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:26:05.174631+0000", 2023-06-03T20:27:58.250 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0 2023-06-03T20:27:58.250 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:27:58.250 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:27:58.250 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "header_read", 2023-06-03T20:27:58.251 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:26:05.174630+0000", 2023-06-03T20:27:58.251 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 4294967295.999999 2023-06-03T20:27:58.251 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:27:58.251 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:27:58.251 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "all_read", 2023-06-03T20:27:58.251 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:26:05.174639+0000", 2023-06-03T20:27:58.252 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 8.5539999999999998e-06 2023-06-03T20:27:58.252 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:27:58.252 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:27:58.252 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "dispatched", 2023-06-03T20:27:58.252 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:26:05.174649+0000", 2023-06-03T20:27:58.253 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 1.0752e-05 2023-06-03T20:27:58.253 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:27:58.253 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:27:58.253 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "queued_for_pg", 2023-06-03T20:27:58.253 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:26:05.174657+0000", 2023-06-03T20:27:58.253 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 7.277e-06 2023-06-03T20:27:58.254 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:27:58.254 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:27:58.254 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "reached_pg", 2023-06-03T20:27:58.254 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:26:05.217713+0000", 2023-06-03T20:27:58.254 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.043056523999999999 2023-06-03T20:27:58.254 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:27:58.254 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:27:58.254 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "started", 2023-06-03T20:27:58.255 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:26:05.217747+0000", 2023-06-03T20:27:58.255 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 3.3246e-05 2023-06-03T20:27:58.255 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:27:58.255 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:27:58.255 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "commit_sent", 2023-06-03T20:27:58.255 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:26:05.301852+0000", 2023-06-03T20:27:58.255 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.084105017000000004 2023-06-03T20:27:58.255 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:27:58.256 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:27:58.256 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "done", 2023-06-03T20:27:58.256 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:26:05.301872+0000", 2023-06-03T20:27:58.256 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 2.0818999999999998e-05 2023-06-03T20:27:58.256 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:27:58.256 INFO:teuthology.orchestra.run.smithi164.stdout: ] 2023-06-03T20:27:58.256 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:27:58.257 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:27:58.257 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:27:58.257 INFO:teuthology.orchestra.run.smithi164.stdout: "description": "osd_repop(client.36573.0:207736 13.a e267/251 13:50ef07ff:::benchmark_data_smithi089_211581_object92211:head v 267'12927, mlcod=245'88)", 2023-06-03T20:27:58.257 INFO:teuthology.orchestra.run.smithi164.stdout: "initiated_at": "2023-06-03T20:26:05.177325+0000", 2023-06-03T20:27:58.257 INFO:teuthology.orchestra.run.smithi164.stdout: "age": 112.921971318, 2023-06-03T20:27:58.257 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.12456791, 2023-06-03T20:27:58.257 INFO:teuthology.orchestra.run.smithi164.stdout: "type_data": { 2023-06-03T20:27:58.258 INFO:teuthology.orchestra.run.smithi164.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-03T20:27:58.258 INFO:teuthology.orchestra.run.smithi164.stdout: "events": [ 2023-06-03T20:27:58.258 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:27:58.258 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "initiated", 2023-06-03T20:27:58.258 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:26:05.177325+0000", 2023-06-03T20:27:58.258 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0 2023-06-03T20:27:58.258 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:27:58.258 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:27:58.259 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "throttled", 2023-06-03T20:27:58.259 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:26:05.177325+0000", 2023-06-03T20:27:58.259 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0 2023-06-03T20:27:58.259 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:27:58.259 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:27:58.259 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "header_read", 2023-06-03T20:27:58.259 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:26:05.177324+0000", 2023-06-03T20:27:58.259 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 4294967295.999999 2023-06-03T20:27:58.260 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:27:58.260 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:27:58.260 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "all_read", 2023-06-03T20:27:58.260 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:26:05.177352+0000", 2023-06-03T20:27:58.260 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 2.8189000000000001e-05 2023-06-03T20:27:58.260 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:27:58.261 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:27:58.261 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "dispatched", 2023-06-03T20:27:58.261 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:26:05.177365+0000", 2023-06-03T20:27:58.261 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 1.2631e-05 2023-06-03T20:27:58.261 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:27:58.261 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:27:58.261 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "queued_for_pg", 2023-06-03T20:27:58.261 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:26:05.177375+0000", 2023-06-03T20:27:58.262 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 1.0937999999999999e-05 2023-06-03T20:27:58.262 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:27:58.262 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:27:58.262 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "reached_pg", 2023-06-03T20:27:58.262 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:26:05.230633+0000", 2023-06-03T20:27:58.263 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.053257957000000002 2023-06-03T20:27:58.263 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:27:58.263 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:27:58.263 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "started", 2023-06-03T20:27:58.263 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:26:05.230662+0000", 2023-06-03T20:27:58.264 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 2.8198e-05 2023-06-03T20:27:58.264 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:27:58.264 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:27:58.264 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "commit_sent", 2023-06-03T20:27:58.265 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:26:05.301874+0000", 2023-06-03T20:27:58.265 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.071212701000000003 2023-06-03T20:27:58.265 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:27:58.265 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:27:58.265 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "done", 2023-06-03T20:27:58.266 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:26:05.301893+0000", 2023-06-03T20:27:58.266 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 1.8465999999999999e-05 2023-06-03T20:27:58.266 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:27:58.266 INFO:teuthology.orchestra.run.smithi164.stdout: ] 2023-06-03T20:27:58.267 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:27:58.267 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:27:58.267 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:27:58.267 INFO:teuthology.orchestra.run.smithi164.stdout: "description": "osd_repop(client.36573.0:218324 13.a e268/251 13:5ca03085:::benchmark_data_smithi089_211581_object102799:head v 268'13536, mlcod=245'88)", 2023-06-03T20:27:58.267 INFO:teuthology.orchestra.run.smithi164.stdout: "initiated_at": "2023-06-03T20:26:21.783874+0000", 2023-06-03T20:27:58.267 INFO:teuthology.orchestra.run.smithi164.stdout: "age": 96.315422710000007, 2023-06-03T20:27:58.268 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.147752249, 2023-06-03T20:27:58.268 INFO:teuthology.orchestra.run.smithi164.stdout: "type_data": { 2023-06-03T20:27:58.268 INFO:teuthology.orchestra.run.smithi164.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-03T20:27:58.268 INFO:teuthology.orchestra.run.smithi164.stdout: "events": [ 2023-06-03T20:27:58.268 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:27:58.268 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "initiated", 2023-06-03T20:27:58.268 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:26:21.783874+0000", 2023-06-03T20:27:58.269 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0 2023-06-03T20:27:58.269 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:27:58.269 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:27:58.269 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "throttled", 2023-06-03T20:27:58.269 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:26:21.783874+0000", 2023-06-03T20:27:58.269 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0 2023-06-03T20:27:58.269 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:27:58.269 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:27:58.270 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "header_read", 2023-06-03T20:27:58.270 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:26:21.783873+0000", 2023-06-03T20:27:58.270 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 4294967295.999999 2023-06-03T20:27:58.270 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:27:58.270 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:27:58.270 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "all_read", 2023-06-03T20:27:58.270 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:26:21.783882+0000", 2023-06-03T20:27:58.271 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 9.3270000000000007e-06 2023-06-03T20:27:58.271 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:27:58.271 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:27:58.271 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "dispatched", 2023-06-03T20:27:58.271 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:26:21.783893+0000", 2023-06-03T20:27:58.271 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 1.0659000000000001e-05 2023-06-03T20:27:58.271 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:27:58.272 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:27:58.272 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "queued_for_pg", 2023-06-03T20:27:58.272 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:26:21.783900+0000", 2023-06-03T20:27:58.272 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 7.3529999999999997e-06 2023-06-03T20:27:58.272 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:27:58.272 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:27:58.272 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "reached_pg", 2023-06-03T20:27:58.273 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:26:21.820819+0000", 2023-06-03T20:27:58.273 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.036918749000000001 2023-06-03T20:27:58.273 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:27:58.273 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:27:58.273 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "started", 2023-06-03T20:27:58.273 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:26:21.820848+0000", 2023-06-03T20:27:58.273 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 2.9346e-05 2023-06-03T20:27:58.274 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:27:58.274 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:27:58.274 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "commit_sent", 2023-06-03T20:27:58.274 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:26:21.931534+0000", 2023-06-03T20:27:58.274 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.110685772 2023-06-03T20:27:58.274 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:27:58.275 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:27:58.275 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "done", 2023-06-03T20:27:58.275 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:26:21.931626+0000", 2023-06-03T20:27:58.275 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 9.1971000000000004e-05 2023-06-03T20:27:58.275 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:27:58.276 INFO:teuthology.orchestra.run.smithi164.stdout: ] 2023-06-03T20:27:58.276 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:27:58.276 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:27:58.276 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:27:58.276 INFO:teuthology.orchestra.run.smithi164.stdout: "description": "osd_repop(client.36573.0:218327 13.a e268/251 13:544a4ad9:::benchmark_data_smithi089_211581_object102802:head v 268'13537, mlcod=245'88)", 2023-06-03T20:27:58.276 INFO:teuthology.orchestra.run.smithi164.stdout: "initiated_at": "2023-06-03T20:26:21.784481+0000", 2023-06-03T20:27:58.276 INFO:teuthology.orchestra.run.smithi164.stdout: "age": 96.314815629999998, 2023-06-03T20:27:58.277 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.14718578099999999, 2023-06-03T20:27:58.277 INFO:teuthology.orchestra.run.smithi164.stdout: "type_data": { 2023-06-03T20:27:58.277 INFO:teuthology.orchestra.run.smithi164.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-03T20:27:58.277 INFO:teuthology.orchestra.run.smithi164.stdout: "events": [ 2023-06-03T20:27:58.277 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:27:58.277 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "initiated", 2023-06-03T20:27:58.277 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:26:21.784481+0000", 2023-06-03T20:27:58.278 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0 2023-06-03T20:27:58.278 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:27:58.278 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:27:58.278 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "throttled", 2023-06-03T20:27:58.278 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:26:21.784481+0000", 2023-06-03T20:27:58.278 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0 2023-06-03T20:27:58.278 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:27:58.278 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:27:58.279 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "header_read", 2023-06-03T20:27:58.279 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:26:21.784479+0000", 2023-06-03T20:27:58.279 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 4294967295.9999986 2023-06-03T20:27:58.279 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:27:58.280 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:27:58.280 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "all_read", 2023-06-03T20:27:58.280 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:26:21.784487+0000", 2023-06-03T20:27:58.281 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 7.9219999999999997e-06 2023-06-03T20:27:58.281 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:27:58.281 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:27:58.281 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "dispatched", 2023-06-03T20:27:58.282 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:26:21.784499+0000", 2023-06-03T20:27:58.282 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 1.1426999999999999e-05 2023-06-03T20:27:58.282 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:27:58.282 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:27:58.283 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "queued_for_pg", 2023-06-03T20:27:58.283 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:26:21.784505+0000", 2023-06-03T20:27:58.283 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 6.1109999999999996e-06 2023-06-03T20:27:58.284 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:27:58.284 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:27:58.284 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "reached_pg", 2023-06-03T20:27:58.284 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:26:21.848045+0000", 2023-06-03T20:27:58.284 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.063540563999999994 2023-06-03T20:27:58.285 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:27:58.285 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:27:58.285 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "started", 2023-06-03T20:27:58.285 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:26:21.848116+0000", 2023-06-03T20:27:58.285 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 7.0838000000000002e-05 2023-06-03T20:27:58.285 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:27:58.285 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:27:58.285 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "commit_sent", 2023-06-03T20:27:58.286 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:26:21.931630+0000", 2023-06-03T20:27:58.286 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.083513872000000003 2023-06-03T20:27:58.286 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:27:58.286 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:27:58.286 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "done", 2023-06-03T20:27:58.286 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:26:21.931666+0000", 2023-06-03T20:27:58.286 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 3.6293e-05 2023-06-03T20:27:58.286 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:27:58.287 INFO:teuthology.orchestra.run.smithi164.stdout: ] 2023-06-03T20:27:58.287 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:27:58.287 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:27:58.287 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:27:58.287 INFO:teuthology.orchestra.run.smithi164.stdout: "description": "osd_repop(client.36573.0:218347 13.a e268/251 13:5c2fa7e5:::benchmark_data_smithi089_211581_object102822:head v 268'13538, mlcod=245'88)", 2023-06-03T20:27:58.287 INFO:teuthology.orchestra.run.smithi164.stdout: "initiated_at": "2023-06-03T20:26:21.810234+0000", 2023-06-03T20:27:58.287 INFO:teuthology.orchestra.run.smithi164.stdout: "age": 96.289062083999994, 2023-06-03T20:27:58.287 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.13582887599999999, 2023-06-03T20:27:58.288 INFO:teuthology.orchestra.run.smithi164.stdout: "type_data": { 2023-06-03T20:27:58.288 INFO:teuthology.orchestra.run.smithi164.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-03T20:27:58.288 INFO:teuthology.orchestra.run.smithi164.stdout: "events": [ 2023-06-03T20:27:58.288 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:27:58.288 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "initiated", 2023-06-03T20:27:58.288 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:26:21.810234+0000", 2023-06-03T20:27:58.288 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0 2023-06-03T20:27:58.289 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:27:58.289 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:27:58.289 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "throttled", 2023-06-03T20:27:58.289 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:26:21.810234+0000", 2023-06-03T20:27:58.290 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0 2023-06-03T20:27:58.290 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:27:58.290 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:27:58.290 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "header_read", 2023-06-03T20:27:58.290 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:26:21.810233+0000", 2023-06-03T20:27:58.290 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 4294967295.9999986 2023-06-03T20:27:58.290 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:27:58.291 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:27:58.291 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "all_read", 2023-06-03T20:27:58.291 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:26:21.810244+0000", 2023-06-03T20:27:58.291 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 1.147e-05 2023-06-03T20:27:58.291 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:27:58.291 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:27:58.291 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "dispatched", 2023-06-03T20:27:58.291 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:26:21.810259+0000", 2023-06-03T20:27:58.292 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 1.4949e-05 2023-06-03T20:27:58.292 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:27:58.292 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:27:58.292 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "queued_for_pg", 2023-06-03T20:27:58.293 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:26:21.810269+0000", 2023-06-03T20:27:58.293 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 1.009e-05 2023-06-03T20:27:58.293 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:27:58.293 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:27:58.294 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "reached_pg", 2023-06-03T20:27:58.294 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:26:21.868705+0000", 2023-06-03T20:27:58.294 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.058436090000000003 2023-06-03T20:27:58.294 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:27:58.294 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:27:58.294 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "started", 2023-06-03T20:27:58.294 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:26:21.868729+0000", 2023-06-03T20:27:58.295 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 2.4046e-05 2023-06-03T20:27:58.295 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:27:58.295 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:27:58.295 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "commit_sent", 2023-06-03T20:27:58.295 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:26:21.945976+0000", 2023-06-03T20:27:58.295 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.077246224000000002 2023-06-03T20:27:58.295 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:27:58.295 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:27:58.296 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "done", 2023-06-03T20:27:58.296 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:26:21.946063+0000", 2023-06-03T20:27:58.296 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 8.7411999999999996e-05 2023-06-03T20:27:58.296 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:27:58.296 INFO:teuthology.orchestra.run.smithi164.stdout: ] 2023-06-03T20:27:58.296 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:27:58.296 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:27:58.297 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:27:58.297 INFO:teuthology.orchestra.run.smithi164.stdout: "description": "osd_repop(client.36573.0:218350 13.a e268/251 13:5d1ee69d:::benchmark_data_smithi089_211581_object102825:head v 268'13539, mlcod=245'88)", 2023-06-03T20:27:58.297 INFO:teuthology.orchestra.run.smithi164.stdout: "initiated_at": "2023-06-03T20:26:21.816153+0000", 2023-06-03T20:27:58.297 INFO:teuthology.orchestra.run.smithi164.stdout: "age": 96.283143301999999, 2023-06-03T20:27:58.297 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.133045148, 2023-06-03T20:27:58.297 INFO:teuthology.orchestra.run.smithi164.stdout: "type_data": { 2023-06-03T20:27:58.297 INFO:teuthology.orchestra.run.smithi164.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-03T20:27:58.297 INFO:teuthology.orchestra.run.smithi164.stdout: "events": [ 2023-06-03T20:27:58.298 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:27:58.298 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "initiated", 2023-06-03T20:27:58.298 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:26:21.816153+0000", 2023-06-03T20:27:58.298 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0 2023-06-03T20:27:58.298 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:27:58.298 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:27:58.298 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "throttled", 2023-06-03T20:27:58.299 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:26:21.816153+0000", 2023-06-03T20:27:58.299 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0 2023-06-03T20:27:58.299 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:27:58.299 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:27:58.299 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "header_read", 2023-06-03T20:27:58.299 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:26:21.816153+0000", 2023-06-03T20:27:58.299 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 4294967295.9999995 2023-06-03T20:27:58.299 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:27:58.300 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:27:58.300 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "all_read", 2023-06-03T20:27:58.300 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:26:21.816157+0000", 2023-06-03T20:27:58.300 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 4.3000000000000003e-06 2023-06-03T20:27:58.300 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:27:58.300 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:27:58.300 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "dispatched", 2023-06-03T20:27:58.300 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:26:21.816163+0000", 2023-06-03T20:27:58.301 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 6.5599999999999999e-06 2023-06-03T20:27:58.301 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:27:58.301 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:27:58.301 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "queued_for_pg", 2023-06-03T20:27:58.301 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:26:21.816167+0000", 2023-06-03T20:27:58.301 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 3.5899999999999999e-06 2023-06-03T20:27:58.301 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:27:58.302 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:27:58.302 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "reached_pg", 2023-06-03T20:27:58.302 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:26:21.886394+0000", 2023-06-03T20:27:58.302 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.070226564000000005 2023-06-03T20:27:58.302 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:27:58.302 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:27:58.302 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "started", 2023-06-03T20:27:58.303 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:26:21.886447+0000", 2023-06-03T20:27:58.303 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 5.3446000000000003e-05 2023-06-03T20:27:58.303 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:27:58.303 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:27:58.303 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "commit_sent", 2023-06-03T20:27:58.303 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:26:21.949121+0000", 2023-06-03T20:27:58.303 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.062673752999999999 2023-06-03T20:27:58.303 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:27:58.304 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:27:58.304 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "done", 2023-06-03T20:27:58.304 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:26:21.949198+0000", 2023-06-03T20:27:58.304 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 7.7337999999999997e-05 2023-06-03T20:27:58.304 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:27:58.304 INFO:teuthology.orchestra.run.smithi164.stdout: ] 2023-06-03T20:27:58.304 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:27:58.305 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:27:58.305 INFO:teuthology.orchestra.run.smithi164.stdout: ] 2023-06-03T20:27:58.305 INFO:teuthology.orchestra.run.smithi164.stdout:} 2023-06-03T20:27:58.333 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:27:58.335 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:27:58.336 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:27:58.437 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:27:58.465 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:27:58.467 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:27:58.468 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:27:58.569 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:27:58.598 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:27:58.600 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:27:58.600 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:27:58.701 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:27:58.737 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:27:58.742 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:27:58.743 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:27:58.844 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:27:58.854 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 12 16 24464 24448 127.317 100.125 0.00535297 0.00784688 2023-06-03T20:27:58.876 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:27:58.878 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:27:58.879 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:27:58.979 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:27:59.016 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:27:59.019 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:27:59.020 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:27:59.121 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:27:59.149 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:27:59.151 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:27:59.151 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:27:59.252 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:27:59.273 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:27:59.275 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:27:59.276 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:27:59.376 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:27:59.408 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:27:59.412 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:27:59.413 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:27:59.513 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:27:59.519 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph daemon osd.0 dump_ops_in_flight 2023-06-03T20:27:59.560 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:27:59.565 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:27:59.566 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:27:59.667 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:27:59.709 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:27:59.714 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:27:59.715 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:27:59.816 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:27:59.854 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 13 16 25919 25903 124.516 90.9375 0.00771944 0.00802393 2023-06-03T20:27:59.868 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:27:59.871 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:27:59.872 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:27:59.973 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:28:00.003 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:28:00.004 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:28:00.005 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:28:00.106 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:28:00.137 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:28:00.139 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:28:00.140 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:28:00.241 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:28:00.267 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:28:00.270 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:28:00.271 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:28:00.372 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:28:00.380 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:28:00 smithi164 bash[59462]: cluster 1685824079.127613 mgr.y (mgr.34104) 1563 : cluster 0 pgmap v1610: 89 pgs: 89 active+clean; 1.3 GiB data, 8.7 GiB used, 706 GiB / 715 GiB avail; 114 MiB/s wr, 1.83k op/s 2023-06-03T20:28:00.406 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:28:00.409 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:28:00.409 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:28:00.508 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:28:00 smithi089 bash[75068]: cluster 1685824079.127613 mgr.y (mgr.34104) 1563 : cluster 0 pgmap v1610: 89 pgs: 89 active+clean; 1.3 GiB data, 8.7 GiB used, 706 GiB / 715 GiB avail; 114 MiB/s wr, 1.83k op/s 2023-06-03T20:28:00.509 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:28:00 smithi089 bash[78452]: cluster 1685824079.127613 mgr.y (mgr.34104) 1563 : cluster 0 pgmap v1610: 89 pgs: 89 active+clean; 1.3 GiB data, 8.7 GiB used, 706 GiB / 715 GiB avail; 114 MiB/s wr, 1.83k op/s 2023-06-03T20:28:00.510 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:28:00.542 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:28:00.544 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:28:00.545 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:28:00.645 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:28:00.677 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:28:00.680 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:28:00.680 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:28:00.781 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:28:00.820 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:28:00.822 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:28:00.823 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:28:00.854 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 14 16 27339 27323 121.961 88.75 0.0120601 0.00819131 2023-06-03T20:28:00.924 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:28:00.959 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:28:00.963 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:28:00.963 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:28:01.064 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:28:01.099 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:28:01.100 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:28:01.101 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:28:01.202 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:28:01.232 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:28:01.239 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:28:01.240 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:28:01.341 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:28:01.377 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:28:01.380 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:28:01.381 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:28:01.482 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:28:01.514 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:28:01.516 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:28:01.517 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:28:01.618 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:28:01.652 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:28:01.653 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:28:01.654 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:28:01.755 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:28:01.789 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:28:01.795 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:28:01.796 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:28:01.854 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 15 16 28969 28953 120.621 101.875 0.00695383 0.00828305 2023-06-03T20:28:01.897 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:28:01.930 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:28:01.935 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:28:01.936 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:28:02.037 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:28:02.069 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:28:02.073 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:28:02.074 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:28:02.175 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:28:02.204 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:28:02.208 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:28:02.209 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:28:02.310 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:28:02.343 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:28:02.347 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:28:02.348 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:28:02.449 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:28:02.458 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:28:02 smithi164 bash[59462]: cluster 1685824081.1280718 mgr.y (mgr.34104) 1564 : cluster 0 pgmap v1611: 89 pgs: 89 active+clean; 1.3 GiB data, 8.7 GiB used, 706 GiB / 715 GiB avail; 105 MiB/s wr, 1.69k op/s 2023-06-03T20:28:02.481 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:28:02.483 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:28:02.483 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:28:02.529 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:28:02 smithi089 bash[78452]: cluster 1685824081.1280718 mgr.y (mgr.34104) 1564 : cluster 0 pgmap v1611: 89 pgs: 89 active+clean; 1.3 GiB data, 8.7 GiB used, 706 GiB / 715 GiB avail; 105 MiB/s wr, 1.69k op/s 2023-06-03T20:28:02.529 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:28:02 smithi089 bash[75068]: cluster 1685824081.1280718 mgr.y (mgr.34104) 1564 : cluster 0 pgmap v1611: 89 pgs: 89 active+clean; 1.3 GiB data, 8.7 GiB used, 706 GiB / 715 GiB avail; 105 MiB/s wr, 1.69k op/s 2023-06-03T20:28:02.585 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:28:02.629 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:28:02.632 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:28:02.633 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:28:02.734 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:28:02.775 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:28:02.779 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:28:02.780 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:28:02.854 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 16 16 30530 30514 119.179 97.5625 0.00196665 0.00836255 2023-06-03T20:28:02.881 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:28:02.915 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:28:02.919 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:28:02.919 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:28:03.020 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:28:03.048 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:28:03.050 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:28:03.050 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:28:03.151 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:28:03.184 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:28:03.187 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:28:03.187 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:28:03.288 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:28:03.329 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:28:03.333 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:28:03.334 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:28:03.435 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:28:03.468 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:28:03.472 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:28:03.473 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:28:03.574 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:28:03.606 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:28:03.609 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:28:03.611 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:28:03.712 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:28:03.742 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:28:03.745 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:28:03.745 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:28:03.846 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:28:03.854 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 17 16 32206 32190 118.33 104.75 0.0110248 0.0084449 2023-06-03T20:28:03.884 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:28:03.887 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:28:03.887 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:28:03.988 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:28:04.018 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:28:04.024 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:28:04.025 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:28:04.126 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:28:04.155 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:28:04.158 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:28:04.159 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:28:04.260 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:28:04.270 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:28:04 smithi164 bash[59462]: cluster 1685824083.1288354 mgr.y (mgr.34104) 1565 : cluster 0 pgmap v1612: 89 pgs: 89 active+clean; 1.8 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 146 MiB/s wr, 2.34k op/s 2023-06-03T20:28:04.295 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:28:04.298 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:28:04.299 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:28:04.400 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:28:04.430 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:28:04.433 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:28:04.433 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:28:04.529 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:28:04 smithi089 bash[78452]: cluster 1685824083.1288354 mgr.y (mgr.34104) 1565 : cluster 0 pgmap v1612: 89 pgs: 89 active+clean; 1.8 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 146 MiB/s wr, 2.34k op/s 2023-06-03T20:28:04.529 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:28:04 smithi089 bash[75068]: cluster 1685824083.1288354 mgr.y (mgr.34104) 1565 : cluster 0 pgmap v1612: 89 pgs: 89 active+clean; 1.8 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 146 MiB/s wr, 2.34k op/s 2023-06-03T20:28:04.535 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:28:04.563 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:28:04.566 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:28:04.567 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:28:04.668 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:28:04.703 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:28:04.706 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:28:04.706 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:28:04.807 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:28:04.839 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:28:04.842 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:28:04.842 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:28:04.854 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 18 16 33630 33614 116.7 89 0.00778527 0.00856362 2023-06-03T20:28:04.943 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:28:04.972 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:28:04.974 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:28:04.975 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:28:05.076 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:28:05.110 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:28:05.113 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:28:05.114 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:28:05.215 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:28:05.254 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:28:05.256 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:28:05.257 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:28:05.358 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:28:05.391 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:28:05.393 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:28:05.394 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:28:05.495 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:28:05.521 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:28:05.524 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:28:05.524 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:28:05.625 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:28:05.649 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:28:05.651 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:28:05.652 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:28:05.753 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:28:05.791 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:28:05.798 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:28:05.798 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:28:05.854 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 19 16 34876 34860 114.656 77.875 0.00933395 0.00871646 2023-06-03T20:28:05.900 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:28:05.952 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:28:05.955 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:28:05.955 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:28:06.056 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:28:06.091 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:28:06.093 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:28:06.094 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:28:06.193 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:28:06 smithi164 bash[59462]: cluster 1685824085.1293914 mgr.y (mgr.34104) 1566 : cluster 0 pgmap v1613: 89 pgs: 89 active+clean; 1.8 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 88 MiB/s wr, 1.41k op/s 2023-06-03T20:28:06.195 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:28:06.227 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:28:06.229 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:28:06.230 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:28:06.331 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:28:06.366 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:28:06.370 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:28:06.371 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:28:06.472 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:28:06.489 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:28:06 smithi089 bash[75068]: cluster 1685824085.1293914 mgr.y (mgr.34104) 1566 : cluster 0 pgmap v1613: 89 pgs: 89 active+clean; 1.8 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 88 MiB/s wr, 1.41k op/s 2023-06-03T20:28:06.489 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 20:28:06 smithi089 bash[71163]: ::ffff:172.21.15.164 - - [03/Jun/2023:20:28:06] "GET /metrics HTTP/1.1" 200 228983 "" "Prometheus/2.33.4" 2023-06-03T20:28:06.490 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:28:06 smithi089 bash[78452]: cluster 1685824085.1293914 mgr.y (mgr.34104) 1566 : cluster 0 pgmap v1613: 89 pgs: 89 active+clean; 1.8 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 88 MiB/s wr, 1.41k op/s 2023-06-03T20:28:06.530 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:28:06.535 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:28:06.536 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:28:06.637 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:28:06.669 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:28:06.670 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:28:06.671 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:28:06.773 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:28:06.813 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:28:06.816 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:28:06.816 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:28:06.854 INFO:tasks.radosbench.radosbench.0.smithi089.stdout:2023-06-03T20:28:06.850273+0000 min lat: 0.00149899 max lat: 0.159284 avg lat: 0.00881584 2023-06-03T20:28:06.855 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-06-03T20:28:06.855 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 20 16 36292 36276 113.348 88.5 0.0128979 0.00881584 2023-06-03T20:28:06.917 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:28:06.952 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:28:06.954 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:28:06.955 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:28:07.056 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:28:07.096 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:28:07.099 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:28:07.100 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:28:07.201 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:28:07.204 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:28:07 smithi089 bash[75068]: audit 1685824087.1496224 mon.c (mon.2) 646 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T20:28:07.227 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:28:07.230 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:28:07.230 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:28:07.331 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:28:07.334 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:28:07 smithi164 bash[59462]: audit 1685824087.1496224 mon.c (mon.2) 646 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T20:28:07.367 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:28:07.371 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:28:07.372 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:28:07.473 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:28:07.510 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:28:07.513 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:28:07.514 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:28:07.603 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:28:07 smithi089 bash[78452]: audit 1685824087.1496224 mon.c (mon.2) 646 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T20:28:07.615 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:28:07.646 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:28:07.649 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:28:07.650 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:28:07.751 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:28:07.785 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:28:07.789 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:28:07.789 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:28:07.854 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 21 16 37736 37720 112.248 90.25 0.00704437 0.00890213 2023-06-03T20:28:07.890 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:28:07.922 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:28:07.924 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:28:07.925 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:28:08.027 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:28:08.057 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:28:08.059 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:28:08.059 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:28:08.160 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:28:08.185 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:28:08.187 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:28:08.187 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:28:08.288 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:28:08.326 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:28:08.329 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:28:08.329 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:28:08.430 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:28:08.440 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:28:08 smithi164 bash[59462]: cluster 1685824087.1300335 mgr.y (mgr.34104) 1567 : cluster 0 pgmap v1614: 89 pgs: 89 active+clean; 1.9 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 98 MiB/s wr, 1.56k op/s 2023-06-03T20:28:08.459 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:28:08.464 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:28:08.464 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:28:08.529 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:28:08 smithi089 bash[78452]: cluster 1685824087.1300335 mgr.y (mgr.34104) 1567 : cluster 0 pgmap v1614: 89 pgs: 89 active+clean; 1.9 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 98 MiB/s wr, 1.56k op/s 2023-06-03T20:28:08.529 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:28:08 smithi089 bash[75068]: cluster 1685824087.1300335 mgr.y (mgr.34104) 1567 : cluster 0 pgmap v1614: 89 pgs: 89 active+clean; 1.9 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 98 MiB/s wr, 1.56k op/s 2023-06-03T20:28:08.566 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:28:08.597 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:28:08.601 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:28:08.602 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:28:08.703 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:28:08.736 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:28:08.738 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:28:08.739 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:28:08.840 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:28:08.857 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 22 16 39069 39053 110.931 83.3125 0.00982257 0.00900835 2023-06-03T20:28:08.873 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:28:08.875 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:28:08.875 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:28:08.976 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:28:08.987 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/mon.a/config 2023-06-03T20:28:09.007 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:28:09.009 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:28:09.010 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:28:09.110 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:28:09.152 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:28:09.155 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:28:09.155 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:28:09.257 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:28:09.288 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:28:09.292 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:28:09.292 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:28:09.393 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:28:09.421 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:28:09.425 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:28:09.426 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:28:09.526 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:28:09.551 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:28:09.553 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:28:09.554 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:28:09.655 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:28:09.692 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:28:09.695 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:28:09.696 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:28:09.797 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:28:09.818 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:28:09.820 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:28:09.820 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:28:09.855 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 23 16 40211 40195 109.211 71.375 0.0111309 0.00915058 2023-06-03T20:28:09.922 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:28:09.956 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:28:09.958 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:28:09.959 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:28:10.059 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:28:10.088 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:28:10.091 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:28:10.092 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:28:10.193 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:28:10.226 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:28:10.229 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:28:10.230 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:28:10.331 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:28:10.361 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:28:10.363 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:28:10.364 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:28:10.464 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:28:10.477 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:28:10 smithi164 bash[59462]: cluster 1685824089.1309156 mgr.y (mgr.34104) 1568 : cluster 0 pgmap v1615: 89 pgs: 89 active+clean; 2.3 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 113 MiB/s wr, 1.81k op/s 2023-06-03T20:28:10.502 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:28:10.505 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:28:10.506 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:28:10.529 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:28:10 smithi089 bash[75068]: cluster 1685824089.1309156 mgr.y (mgr.34104) 1568 : cluster 0 pgmap v1615: 89 pgs: 89 active+clean; 2.3 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 113 MiB/s wr, 1.81k op/s 2023-06-03T20:28:10.529 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:28:10 smithi089 bash[78452]: cluster 1685824089.1309156 mgr.y (mgr.34104) 1568 : cluster 0 pgmap v1615: 89 pgs: 89 active+clean; 2.3 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 113 MiB/s wr, 1.81k op/s 2023-06-03T20:28:10.607 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:28:10.637 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:28:10.643 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:28:10.644 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:28:10.745 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:28:10.778 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:28:10.783 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:28:10.783 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:28:10.855 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 24 16 41281 41265 107.447 66.875 0.0047205 0.00930031 2023-06-03T20:28:10.884 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:28:10.918 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:28:10.921 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:28:10.922 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:28:11.023 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:28:11.059 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:28:11.062 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:28:11.063 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:28:11.164 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:28:11.192 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:28:11.195 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:28:11.196 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:28:11.297 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:28:11.334 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:28:11.336 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:28:11.337 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:28:11.438 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:28:11.474 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:28:11.476 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:28:11.477 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:28:11.578 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:28:11.616 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:28:11.618 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:28:11.619 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:28:11.720 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:28:11.754 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:28:11.757 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:28:11.758 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:28:11.855 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 25 16 42551 42535 106.324 79.375 0.00278803 0.00939941 2023-06-03T20:28:11.858 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:28:11.889 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:28:11.891 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:28:11.893 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:28:11.994 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:28:12.029 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:28:12.032 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:28:12.033 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:28:12.134 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:28:12.171 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:28:12.173 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:28:12.174 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:28:12.274 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:28:12.310 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:28:12.312 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:28:12.313 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:28:12.335 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:28:12 smithi164 bash[59462]: cluster 1685824091.1313522 mgr.y (mgr.34104) 1569 : cluster 0 pgmap v1616: 89 pgs: 89 active+clean; 2.3 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 79 MiB/s wr, 1.26k op/s 2023-06-03T20:28:12.335 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:28:12 smithi164 bash[59462]: audit 1685824091.6264844 mon.a (mon.0) 1586 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:28:12.335 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:28:12 smithi164 bash[59462]: audit 1685824091.637868 mon.a (mon.0) 1587 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:28:12.414 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:28:12.445 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:28:12.447 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:28:12.448 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:28:12.529 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:28:12 smithi089 bash[75068]: cluster 1685824091.1313522 mgr.y (mgr.34104) 1569 : cluster 0 pgmap v1616: 89 pgs: 89 active+clean; 2.3 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 79 MiB/s wr, 1.26k op/s 2023-06-03T20:28:12.529 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:28:12 smithi089 bash[75068]: audit 1685824091.6264844 mon.a (mon.0) 1586 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:28:12.530 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:28:12 smithi089 bash[75068]: audit 1685824091.637868 mon.a (mon.0) 1587 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:28:12.530 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:28:12 smithi089 bash[78452]: cluster 1685824091.1313522 mgr.y (mgr.34104) 1569 : cluster 0 pgmap v1616: 89 pgs: 89 active+clean; 2.3 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 79 MiB/s wr, 1.26k op/s 2023-06-03T20:28:12.530 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:28:12 smithi089 bash[78452]: audit 1685824091.6264844 mon.a (mon.0) 1586 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:28:12.530 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:28:12 smithi089 bash[78452]: audit 1685824091.637868 mon.a (mon.0) 1587 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:28:12.549 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:28:12.586 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:28:12.589 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:28:12.590 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:28:12.691 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:28:12.723 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:28:12.727 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:28:12.727 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:28:12.828 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:28:12.855 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 26 16 43909 43893 105.499 84.875 0.00278726 0.00946392 2023-06-03T20:28:12.861 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:28:12.864 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:28:12.865 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:28:12.965 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:28:12.993 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:28:12.999 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:28:13.000 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:28:13.101 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:28:13.135 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:28:13.140 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:28:13.141 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:28:13.242 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:28:13.269 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:28:13.273 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:28:13.274 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:28:13.375 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:28:13.401 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:28:13.402 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:28:13.403 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:28:13.504 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:28:13.539 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:28:13.547 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:28:13.548 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:28:13.648 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:28:13.683 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:28:13.686 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:28:13.687 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:28:13.788 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:28:13.825 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:28:13.829 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:28:13.830 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:28:13.855 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 27 16 45015 44999 104.151 69.125 0.0126073 0.00959559 2023-06-03T20:28:13.931 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:28:13.961 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:28:13.963 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:28:13.964 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:28:14.065 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:28:14.101 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:28:14.105 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:28:14.106 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:28:14.207 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:28:14.238 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:28:14.243 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:28:14.244 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:28:14.335 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:28:14 smithi164 bash[59462]: cluster 1685824093.1320858 mgr.y (mgr.34104) 1570 : cluster 0 pgmap v1617: 89 pgs: 89 active+clean; 2.6 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 111 MiB/s wr, 1.77k op/s 2023-06-03T20:28:14.345 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:28:14.379 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:28:14 smithi089 bash[78452]: cluster 1685824093.1320858 mgr.y (mgr.34104) 1570 : cluster 0 pgmap v1617: 89 pgs: 89 active+clean; 2.6 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 111 MiB/s wr, 1.77k op/s 2023-06-03T20:28:14.379 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:28:14 smithi089 bash[75068]: cluster 1685824093.1320858 mgr.y (mgr.34104) 1570 : cluster 0 pgmap v1617: 89 pgs: 89 active+clean; 2.6 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 111 MiB/s wr, 1.77k op/s 2023-06-03T20:28:14.380 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:28:14.386 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:28:14.387 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:28:14.488 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:28:14.514 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:28:14.515 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:28:14.516 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:28:14.616 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:28:14.653 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:28:14.656 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:28:14.657 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:28:14.758 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:28:14.794 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:28:14.797 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:28:14.798 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:28:14.855 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 28 16 46184 46168 103.041 73.0625 0.0145857 0.00969844 2023-06-03T20:28:14.899 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:28:14.931 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:28:14.935 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:28:14.936 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:28:15.037 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:28:15.069 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:28:15.071 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:28:15.072 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:28:15.173 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:28:15.206 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:28:15.210 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:28:15.211 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:28:15.313 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:28:15.347 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:28:15.349 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:28:15.349 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:28:15.450 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:28:15.483 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:28:15.490 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:28:15.490 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:28:15.591 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:28:15.623 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:28:15.625 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:28:15.626 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:28:15.727 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:28:15.757 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:28:15.762 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:28:15.763 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:28:15.855 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 29 16 47329 47313 101.955 71.5625 0.0126934 0.00980192 2023-06-03T20:28:15.864 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:28:15.898 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:28:15.903 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:28:15.904 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:28:16.005 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:28:16.060 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:28:16.066 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:28:16.067 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:28:16.168 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:28:16.204 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:28:16.206 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:28:16.207 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:28:16.207 INFO:teuthology.orchestra.run.smithi089.stderr:reweighted osd.1 to 1 (10000) 2023-06-03T20:28:16.279 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:28:16 smithi089 bash[75068]: cluster 1685824095.1331105 mgr.y (mgr.34104) 1571 : cluster 0 pgmap v1618: 89 pgs: 89 active+clean; 2.6 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 70 MiB/s wr, 1.12k op/s 2023-06-03T20:28:16.279 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:28:16 smithi089 bash[75068]: audit 1685824096.0404181 mon.c (mon.2) 647 : audit 1 from='client.? 172.21.15.89:0/2365235304' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-06-03T20:28:16.280 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:28:16 smithi089 bash[75068]: audit 1685824096.041123 mon.a (mon.0) 1588 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-06-03T20:28:16.280 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 20:28:16 smithi089 bash[71163]: ::ffff:172.21.15.164 - - [03/Jun/2023:20:28:16] "GET /metrics HTTP/1.1" 200 229075 "" "Prometheus/2.33.4" 2023-06-03T20:28:16.280 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:28:16 smithi089 bash[78452]: cluster 1685824095.1331105 mgr.y (mgr.34104) 1571 : cluster 0 pgmap v1618: 89 pgs: 89 active+clean; 2.6 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 70 MiB/s wr, 1.12k op/s 2023-06-03T20:28:16.280 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:28:16 smithi089 bash[78452]: audit 1685824096.0404181 mon.c (mon.2) 647 : audit 1 from='client.? 172.21.15.89:0/2365235304' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-06-03T20:28:16.281 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:28:16 smithi089 bash[78452]: audit 1685824096.041123 mon.a (mon.0) 1588 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-06-03T20:28:16.308 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:28:16.347 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:28:16.349 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:28:16.350 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:28:16.451 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:28:16.460 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:28:16 smithi164 bash[59462]: cluster 1685824095.1331105 mgr.y (mgr.34104) 1571 : cluster 0 pgmap v1618: 89 pgs: 89 active+clean; 2.6 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 70 MiB/s wr, 1.12k op/s 2023-06-03T20:28:16.460 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:28:16 smithi164 bash[59462]: audit 1685824096.0404181 mon.c (mon.2) 647 : audit 1 from='client.? 172.21.15.89:0/2365235304' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-06-03T20:28:16.460 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:28:16 smithi164 bash[59462]: audit 1685824096.041123 mon.a (mon.0) 1588 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-06-03T20:28:16.479 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:28:16.482 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:28:16.483 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:28:16.584 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:28:16.632 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:28:16.635 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:28:16.635 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:28:16.737 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:28:16.781 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:28:16.786 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:28:16.787 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:28:16.856 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 30 15 48452 48437 100.898 70.25 0.00440736 0.00990469 2023-06-03T20:28:16.887 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:28:16.919 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:28:16.921 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:28:16.921 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:28:17.023 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:28:17.056 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:28:17.060 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:28:17.060 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:28:17.161 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:28:17.194 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:28:17.200 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:28:17.201 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:28:17.302 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:28:17.318 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:28:17 smithi164 bash[59462]: cluster 1685824096.1904817 osd.5 (osd.5) 231 : cluster 3 failed to encode map e282 with expected crc 2023-06-03T20:28:17.318 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:28:17 smithi164 bash[59462]: cluster 1685824096.1905131 osd.6 (osd.6) 215 : cluster 3 failed to encode map e282 with expected crc 2023-06-03T20:28:17.318 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:28:17 smithi164 bash[59462]: audit 1685824096.1957188 mon.a (mon.0) 1589 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2023-06-03T20:28:17.319 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:28:17 smithi164 bash[59462]: cluster 1685824096.1958148 mon.a (mon.0) 1590 : cluster 0 osdmap e282: 8 total, 8 up, 8 in 2023-06-03T20:28:17.319 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:28:17 smithi164 bash[59462]: cluster 1685824096.2103224 osd.7 (osd.7) 230 : cluster 3 failed to encode map e282 with expected crc 2023-06-03T20:28:17.342 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:28:17.345 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:28:17.346 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:28:17.447 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:28:17.455 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:28:17 smithi089 bash[78452]: cluster 1685824096.1904817 osd.5 (osd.5) 231 : cluster 3 failed to encode map e282 with expected crc 2023-06-03T20:28:17.456 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:28:17 smithi089 bash[78452]: cluster 1685824096.1905131 osd.6 (osd.6) 215 : cluster 3 failed to encode map e282 with expected crc 2023-06-03T20:28:17.456 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:28:17 smithi089 bash[78452]: audit 1685824096.1957188 mon.a (mon.0) 1589 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2023-06-03T20:28:17.456 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:28:17 smithi089 bash[78452]: cluster 1685824096.1958148 mon.a (mon.0) 1590 : cluster 0 osdmap e282: 8 total, 8 up, 8 in 2023-06-03T20:28:17.456 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:28:17 smithi089 bash[78452]: cluster 1685824096.2103224 osd.7 (osd.7) 230 : cluster 3 failed to encode map e282 with expected crc 2023-06-03T20:28:17.456 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:28:17 smithi089 bash[75068]: cluster 1685824096.1904817 osd.5 (osd.5) 231 : cluster 3 failed to encode map e282 with expected crc 2023-06-03T20:28:17.457 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:28:17 smithi089 bash[75068]: cluster 1685824096.1905131 osd.6 (osd.6) 215 : cluster 3 failed to encode map e282 with expected crc 2023-06-03T20:28:17.457 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:28:17 smithi089 bash[75068]: audit 1685824096.1957188 mon.a (mon.0) 1589 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2023-06-03T20:28:17.457 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:28:17 smithi089 bash[75068]: cluster 1685824096.1958148 mon.a (mon.0) 1590 : cluster 0 osdmap e282: 8 total, 8 up, 8 in 2023-06-03T20:28:17.457 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:28:17 smithi089 bash[75068]: cluster 1685824096.2103224 osd.7 (osd.7) 230 : cluster 3 failed to encode map e282 with expected crc 2023-06-03T20:28:17.485 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:28:17.488 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:28:17.488 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:28:17.589 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:28:17.619 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:28:17.622 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:28:17.623 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:28:17.723 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:28:17.763 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:28:17.765 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:28:17.765 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:28:17.856 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 31 16 49619 49603 99.9938 72.875 0.00900183 0.00999228 2023-06-03T20:28:17.866 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:28:17.903 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:28:17.907 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:28:17.908 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:28:18.009 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:28:18.038 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:28:18.041 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:28:18.042 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:28:18.143 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:28:18.177 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:28:18.180 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:28:18.181 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:28:18.244 INFO:teuthology.orchestra.run.smithi089.stderr:noscrub is unset 2023-06-03T20:28:18.282 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:28:18.292 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:28:18 smithi164 bash[59462]: cluster 1685824097.1335962 mgr.y (mgr.34104) 1572 : cluster 0 pgmap v1620: 89 pgs: 89 active+clean; 2.7 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 82 MiB/s wr, 1.31k op/s 2023-06-03T20:28:18.293 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:28:18 smithi164 bash[59462]: audit 1685824097.9157195 mon.a (mon.0) 1591 : audit 1 from='client.? 172.21.15.89:0/2081186188' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-06-03T20:28:18.320 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:28:18.322 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:28:18.323 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:28:18.423 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:28:18.456 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:28:18.461 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:28:18.461 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:28:18.529 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:28:18 smithi089 bash[78452]: cluster 1685824097.1335962 mgr.y (mgr.34104) 1572 : cluster 0 pgmap v1620: 89 pgs: 89 active+clean; 2.7 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 82 MiB/s wr, 1.31k op/s 2023-06-03T20:28:18.530 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:28:18 smithi089 bash[78452]: audit 1685824097.9157195 mon.a (mon.0) 1591 : audit 1 from='client.? 172.21.15.89:0/2081186188' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-06-03T20:28:18.530 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:28:18 smithi089 bash[75068]: cluster 1685824097.1335962 mgr.y (mgr.34104) 1572 : cluster 0 pgmap v1620: 89 pgs: 89 active+clean; 2.7 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 82 MiB/s wr, 1.31k op/s 2023-06-03T20:28:18.530 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:28:18 smithi089 bash[75068]: audit 1685824097.9157195 mon.a (mon.0) 1591 : audit 1 from='client.? 172.21.15.89:0/2081186188' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-06-03T20:28:18.562 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:28:18.595 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:28:18.598 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:28:18.599 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:28:18.700 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:28:18.739 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:28:18.743 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:28:18.744 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:28:18.845 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:28:18.855 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 32 16 50812 50796 99.1988 74.5625 0.00856923 0.0100739 2023-06-03T20:28:18.871 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:28:18.876 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:28:18.877 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:28:18.977 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:28:19.010 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:28:19.015 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:28:19.015 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:28:19.091 INFO:teuthology.orchestra.run.smithi089.stdout:osd.0: {} 2023-06-03T20:28:19.116 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:28:19.146 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:28:19.149 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:28:19.150 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:28:19.150 INFO:teuthology.orchestra.run.smithi089.stdout:osd.1: {} 2023-06-03T20:28:19.189 INFO:teuthology.orchestra.run.smithi089.stdout:osd.2: {} 2023-06-03T20:28:19.222 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:28:19 smithi089 bash[78452]: cluster 1685824098.2113354 mon.a (mon.0) 1592 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-06-03T20:28:19.222 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:28:19 smithi089 bash[78452]: cluster 1685824098.2276657 osd.5 (osd.5) 232 : cluster 3 failed to encode map e283 with expected crc 2023-06-03T20:28:19.222 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:28:19 smithi089 bash[78452]: audit 1685824098.2301147 mon.a (mon.0) 1593 : audit 1 from='client.? 172.21.15.89:0/2081186188' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-06-03T20:28:19.222 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:28:19 smithi089 bash[78452]: cluster 1685824098.230291 osd.5 (osd.5) 233 : cluster 3 failed to encode map e283 with expected crc 2023-06-03T20:28:19.222 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:28:19 smithi089 bash[78452]: cluster 1685824098.2304778 osd.6 (osd.6) 216 : cluster 3 failed to encode map e283 with expected crc 2023-06-03T20:28:19.223 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:28:19 smithi089 bash[78452]: cluster 1685824098.2314284 mon.a (mon.0) 1594 : cluster 0 osdmap e283: 8 total, 8 up, 8 in 2023-06-03T20:28:19.223 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:28:19 smithi089 bash[78452]: cluster 1685824098.2321866 osd.6 (osd.6) 217 : cluster 3 failed to encode map e283 with expected crc 2023-06-03T20:28:19.223 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:28:19 smithi089 bash[78452]: cluster 1685824098.234266 osd.7 (osd.7) 231 : cluster 3 failed to encode map e283 with expected crc 2023-06-03T20:28:19.223 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:28:19 smithi089 bash[78452]: audit 1685824099.0171638 mon.a (mon.0) 1595 : audit 0 from='client.? 172.21.15.89:0/664468079' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-03T20:28:19.223 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:28:19 smithi089 bash[75068]: cluster 1685824098.2113354 mon.a (mon.0) 1592 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-06-03T20:28:19.223 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:28:19 smithi089 bash[75068]: cluster 1685824098.2276657 osd.5 (osd.5) 232 : cluster 3 failed to encode map e283 with expected crc 2023-06-03T20:28:19.224 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:28:19 smithi089 bash[75068]: audit 1685824098.2301147 mon.a (mon.0) 1593 : audit 1 from='client.? 172.21.15.89:0/2081186188' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-06-03T20:28:19.224 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:28:19 smithi089 bash[75068]: cluster 1685824098.230291 osd.5 (osd.5) 233 : cluster 3 failed to encode map e283 with expected crc 2023-06-03T20:28:19.224 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:28:19 smithi089 bash[75068]: cluster 1685824098.2304778 osd.6 (osd.6) 216 : cluster 3 failed to encode map e283 with expected crc 2023-06-03T20:28:19.224 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:28:19 smithi089 bash[75068]: cluster 1685824098.2314284 mon.a (mon.0) 1594 : cluster 0 osdmap e283: 8 total, 8 up, 8 in 2023-06-03T20:28:19.224 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:28:19 smithi089 bash[75068]: cluster 1685824098.2321866 osd.6 (osd.6) 217 : cluster 3 failed to encode map e283 with expected crc 2023-06-03T20:28:19.224 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:28:19 smithi089 bash[75068]: cluster 1685824098.234266 osd.7 (osd.7) 231 : cluster 3 failed to encode map e283 with expected crc 2023-06-03T20:28:19.224 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:28:19 smithi089 bash[75068]: audit 1685824099.0171638 mon.a (mon.0) 1595 : audit 0 from='client.? 172.21.15.89:0/664468079' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-03T20:28:19.238 INFO:teuthology.orchestra.run.smithi089.stdout:osd.3: {} 2023-06-03T20:28:19.251 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:28:19.260 INFO:teuthology.orchestra.run.smithi089.stdout:osd.4: {} 2023-06-03T20:28:19.281 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:28:19.283 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:28:19.284 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:28:19.294 INFO:teuthology.orchestra.run.smithi089.stdout:osd.5: {} 2023-06-03T20:28:19.328 INFO:teuthology.orchestra.run.smithi089.stdout:osd.6: {} 2023-06-03T20:28:19.335 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:28:19 smithi164 bash[59462]: cluster 1685824098.2113354 mon.a (mon.0) 1592 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-06-03T20:28:19.335 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:28:19 smithi164 bash[59462]: cluster 1685824098.2276657 osd.5 (osd.5) 232 : cluster 3 failed to encode map e283 with expected crc 2023-06-03T20:28:19.335 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:28:19 smithi164 bash[59462]: audit 1685824098.2301147 mon.a (mon.0) 1593 : audit 1 from='client.? 172.21.15.89:0/2081186188' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-06-03T20:28:19.335 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:28:19 smithi164 bash[59462]: cluster 1685824098.230291 osd.5 (osd.5) 233 : cluster 3 failed to encode map e283 with expected crc 2023-06-03T20:28:19.335 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:28:19 smithi164 bash[59462]: cluster 1685824098.2304778 osd.6 (osd.6) 216 : cluster 3 failed to encode map e283 with expected crc 2023-06-03T20:28:19.335 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:28:19 smithi164 bash[59462]: cluster 1685824098.2314284 mon.a (mon.0) 1594 : cluster 0 osdmap e283: 8 total, 8 up, 8 in 2023-06-03T20:28:19.336 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:28:19 smithi164 bash[59462]: cluster 1685824098.2321866 osd.6 (osd.6) 217 : cluster 3 failed to encode map e283 with expected crc 2023-06-03T20:28:19.336 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:28:19 smithi164 bash[59462]: cluster 1685824098.234266 osd.7 (osd.7) 231 : cluster 3 failed to encode map e283 with expected crc 2023-06-03T20:28:19.336 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:28:19 smithi164 bash[59462]: audit 1685824099.0171638 mon.a (mon.0) 1595 : audit 0 from='client.? 172.21.15.89:0/664468079' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-03T20:28:19.362 INFO:teuthology.orchestra.run.smithi089.stdout:osd.7: {} 2023-06-03T20:28:19.370 INFO:teuthology.orchestra.run.smithi089.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-06-03T20:28:19.370 INFO:teuthology.orchestra.run.smithi089.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-06-03T20:28:19.370 INFO:teuthology.orchestra.run.smithi089.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-06-03T20:28:19.370 INFO:teuthology.orchestra.run.smithi089.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-06-03T20:28:19.370 INFO:teuthology.orchestra.run.smithi089.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-06-03T20:28:19.370 INFO:teuthology.orchestra.run.smithi089.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-06-03T20:28:19.370 INFO:teuthology.orchestra.run.smithi089.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-06-03T20:28:19.371 INFO:teuthology.orchestra.run.smithi089.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-06-03T20:28:19.385 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:28:19.417 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:28:19.420 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:28:19.421 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:28:19.522 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:28:19.555 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:28:19.560 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:28:19.561 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:28:19.662 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:28:19.693 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:28:19.695 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:28:19.696 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:28:19.796 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:28:19.832 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:28:19.834 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:28:19.835 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:28:19.856 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 33 16 51970 51954 98.3858 72.375 0.00500601 0.010158 2023-06-03T20:28:19.936 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:28:19.968 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:28:19.971 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:28:19.972 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:28:20.073 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:28:20.112 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:28:20.115 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:28:20.115 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:28:20.216 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:28:20.248 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:28:20.250 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:28:20.251 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:28:20.335 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:28:20 smithi164 bash[59462]: cluster 1685824099.1340103 mgr.y (mgr.34104) 1573 : cluster 0 pgmap v1622: 89 pgs: 89 active+clean; 3.0 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 94 MiB/s wr, 1.50k op/s 2023-06-03T20:28:20.351 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:28:20.387 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:28:20.389 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:28:20.390 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:28:20.491 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:28:20.561 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:28:20 smithi089 bash[75068]: cluster 1685824099.1340103 mgr.y (mgr.34104) 1573 : cluster 0 pgmap v1622: 89 pgs: 89 active+clean; 3.0 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 94 MiB/s wr, 1.50k op/s 2023-06-03T20:28:20.561 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:28:20 smithi089 bash[78452]: cluster 1685824099.1340103 mgr.y (mgr.34104) 1573 : cluster 0 pgmap v1622: 89 pgs: 89 active+clean; 3.0 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 94 MiB/s wr, 1.50k op/s 2023-06-03T20:28:20.566 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:28:20.571 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:28:20.572 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:28:20.673 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:28:20.703 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:28:20.708 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:28:20.709 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:28:20.810 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:28:20.838 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:28:20.844 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:28:20.845 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:28:20.856 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 34 16 53111 53095 97.5894 71.3125 0.00591612 0.0102411 2023-06-03T20:28:20.945 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:28:20.983 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:28:20.987 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:28:20.988 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:28:21.089 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:28:21.131 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:28:21.133 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:28:21.134 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:28:21.235 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:28:21.267 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:28:21.271 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:28:21.272 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:28:21.373 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:28:21.408 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:28:21.410 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:28:21.410 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:28:21.511 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:28:21.553 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:28:21.556 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:28:21.557 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:28:21.657 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:28:21.691 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:28:21.695 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:28:21.695 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:28:21.796 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:28:21.840 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:28:21.843 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:28:21.844 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:28:21.856 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 35 16 53757 53741 95.9547 40.375 0.0129278 0.010415 2023-06-03T20:28:21.945 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:28:21.970 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:28:21.973 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:28:21.974 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:28:22.075 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:28:22.108 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:28:22.110 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:28:22.111 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:28:22.151 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph osd reweight 6 1 2023-06-03T20:28:22.212 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:28:22.246 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:28:22.249 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:28:22.250 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:28:22.351 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:28:22.387 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:28:22.393 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:28:22.394 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:28:22.494 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:28:22.525 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:28:22.527 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:28:22.527 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:28:22.585 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:28:22 smithi164 bash[59462]: cluster 1685824101.1343768 mgr.y (mgr.34104) 1574 : cluster 0 pgmap v1623: 89 pgs: 89 active+clean; 3.0 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 46 MiB/s wr, 729 op/s 2023-06-03T20:28:22.585 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:28:22 smithi164 bash[59462]: audit 1685824102.14941 mon.c (mon.2) 648 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T20:28:22.607 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:28:22 smithi089 bash[75068]: cluster 1685824101.1343768 mgr.y (mgr.34104) 1574 : cluster 0 pgmap v1623: 89 pgs: 89 active+clean; 3.0 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 46 MiB/s wr, 729 op/s 2023-06-03T20:28:22.607 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:28:22 smithi089 bash[75068]: audit 1685824102.14941 mon.c (mon.2) 648 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T20:28:22.607 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:28:22 smithi089 bash[78452]: cluster 1685824101.1343768 mgr.y (mgr.34104) 1574 : cluster 0 pgmap v1623: 89 pgs: 89 active+clean; 3.0 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 46 MiB/s wr, 729 op/s 2023-06-03T20:28:22.607 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:28:22 smithi089 bash[78452]: audit 1685824102.14941 mon.c (mon.2) 648 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T20:28:22.629 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:28:22.665 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:28:22.669 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:28:22.670 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:28:22.771 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:28:22.803 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:28:22.808 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:28:22.809 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:28:22.856 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 36 16 54756 54740 95.0235 62.4375 0.00899292 0.0105173 2023-06-03T20:28:22.910 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:28:22.943 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:28:22.946 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:28:22.946 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:28:23.049 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:28:23.082 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:28:23.085 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:28:23.086 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:28:23.186 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:28:23.213 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:28:23.215 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:28:23.216 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:28:23.316 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:28:23.356 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:28:23.359 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:28:23.360 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:28:23.461 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:28:23.465 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:28:23 smithi089 bash[75068]: audit 1685824102.9785712 mon.c (mon.2) 649 : audit 1 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.17", "id": [1, 2]}]: dispatch 2023-06-03T20:28:23.465 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:28:23 smithi089 bash[75068]: audit 1685824102.9790478 mon.c (mon.2) 650 : audit 1 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1b", "id": [1, 2]}]: dispatch 2023-06-03T20:28:23.466 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:28:23 smithi089 bash[75068]: audit 1685824102.979421 mon.a (mon.0) 1596 : audit 1 from='mgr.34104 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.17", "id": [1, 2]}]: dispatch 2023-06-03T20:28:23.466 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:28:23 smithi089 bash[75068]: audit 1685824102.9795275 mon.c (mon.2) 651 : audit 1 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1f", "id": [1, 2]}]: dispatch 2023-06-03T20:28:23.466 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:28:23 smithi089 bash[75068]: audit 1685824102.9809616 mon.a (mon.0) 1597 : audit 1 from='mgr.34104 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1b", "id": [1, 2]}]: dispatch 2023-06-03T20:28:23.466 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:28:23 smithi089 bash[75068]: audit 1685824102.98127 mon.a (mon.0) 1598 : audit 1 from='mgr.34104 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1f", "id": [1, 2]}]: dispatch 2023-06-03T20:28:23.466 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:28:23 smithi089 bash[78452]: audit 1685824102.9785712 mon.c (mon.2) 649 : audit 1 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.17", "id": [1, 2]}]: dispatch 2023-06-03T20:28:23.466 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:28:23 smithi089 bash[78452]: audit 1685824102.9790478 mon.c (mon.2) 650 : audit 1 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1b", "id": [1, 2]}]: dispatch 2023-06-03T20:28:23.467 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:28:23 smithi089 bash[78452]: audit 1685824102.979421 mon.a (mon.0) 1596 : audit 1 from='mgr.34104 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.17", "id": [1, 2]}]: dispatch 2023-06-03T20:28:23.467 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:28:23 smithi089 bash[78452]: audit 1685824102.9795275 mon.c (mon.2) 651 : audit 1 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1f", "id": [1, 2]}]: dispatch 2023-06-03T20:28:23.467 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:28:23 smithi089 bash[78452]: audit 1685824102.9809616 mon.a (mon.0) 1597 : audit 1 from='mgr.34104 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1b", "id": [1, 2]}]: dispatch 2023-06-03T20:28:23.467 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:28:23 smithi089 bash[78452]: audit 1685824102.98127 mon.a (mon.0) 1598 : audit 1 from='mgr.34104 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1f", "id": [1, 2]}]: dispatch 2023-06-03T20:28:23.507 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:28:23.509 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:28:23.510 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:28:23.585 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:28:23 smithi164 bash[59462]: audit 1685824102.9785712 mon.c (mon.2) 649 : audit 1 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.17", "id": [1, 2]}]: dispatch 2023-06-03T20:28:23.585 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:28:23 smithi164 bash[59462]: audit 1685824102.9790478 mon.c (mon.2) 650 : audit 1 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1b", "id": [1, 2]}]: dispatch 2023-06-03T20:28:23.586 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:28:23 smithi164 bash[59462]: audit 1685824102.979421 mon.a (mon.0) 1596 : audit 1 from='mgr.34104 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.17", "id": [1, 2]}]: dispatch 2023-06-03T20:28:23.586 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:28:23 smithi164 bash[59462]: audit 1685824102.9795275 mon.c (mon.2) 651 : audit 1 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1f", "id": [1, 2]}]: dispatch 2023-06-03T20:28:23.586 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:28:23 smithi164 bash[59462]: audit 1685824102.9809616 mon.a (mon.0) 1597 : audit 1 from='mgr.34104 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1b", "id": [1, 2]}]: dispatch 2023-06-03T20:28:23.586 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:28:23 smithi164 bash[59462]: audit 1685824102.98127 mon.a (mon.0) 1598 : audit 1 from='mgr.34104 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1f", "id": [1, 2]}]: dispatch 2023-06-03T20:28:23.611 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:28:23.655 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:28:23.661 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:28:23.661 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:28:23.762 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:28:23.799 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:28:23.801 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:28:23.802 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:28:23.856 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 37 16 55434 55418 93.6004 42.375 0.00732652 0.0106369 2023-06-03T20:28:23.903 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:28:23.925 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:28:23.927 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:28:23.927 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:28:24.028 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:28:24.070 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:28:24.074 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:28:24.075 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:28:24.176 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:28:24.209 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:28:24.211 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:28:24.211 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:28:24.255 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:28:24 smithi089 bash[75068]: cluster 1685824103.1351402 mgr.y (mgr.34104) 1575 : cluster 0 pgmap v1624: 89 pgs: 89 active+clean; 3.3 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 87 MiB/s wr, 1.39k op/s 2023-06-03T20:28:24.255 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:28:24 smithi089 bash[75068]: cluster 1685824103.2439787 osd.6 (osd.6) 218 : cluster 3 failed to encode map e284 with expected crc 2023-06-03T20:28:24.255 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:28:24 smithi089 bash[75068]: audit 1685824103.2484143 mon.a (mon.0) 1599 : audit 1 from='mgr.34104 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.17", "id": [1, 2]}]': finished 2023-06-03T20:28:24.256 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:28:24 smithi089 bash[75068]: audit 1685824103.2484906 mon.a (mon.0) 1600 : audit 1 from='mgr.34104 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1b", "id": [1, 2]}]': finished 2023-06-03T20:28:24.256 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:28:24 smithi089 bash[75068]: audit 1685824103.2485542 mon.a (mon.0) 1601 : audit 1 from='mgr.34104 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1f", "id": [1, 2]}]': finished 2023-06-03T20:28:24.256 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:28:24 smithi089 bash[75068]: cluster 1685824103.248614 mon.a (mon.0) 1602 : cluster 0 osdmap e284: 8 total, 8 up, 8 in 2023-06-03T20:28:24.256 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:28:24 smithi089 bash[78452]: cluster 1685824103.1351402 mgr.y (mgr.34104) 1575 : cluster 0 pgmap v1624: 89 pgs: 89 active+clean; 3.3 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 87 MiB/s wr, 1.39k op/s 2023-06-03T20:28:24.256 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:28:24 smithi089 bash[78452]: cluster 1685824103.2439787 osd.6 (osd.6) 218 : cluster 3 failed to encode map e284 with expected crc 2023-06-03T20:28:24.256 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:28:24 smithi089 bash[78452]: audit 1685824103.2484143 mon.a (mon.0) 1599 : audit 1 from='mgr.34104 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.17", "id": [1, 2]}]': finished 2023-06-03T20:28:24.257 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:28:24 smithi089 bash[78452]: audit 1685824103.2484906 mon.a (mon.0) 1600 : audit 1 from='mgr.34104 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1b", "id": [1, 2]}]': finished 2023-06-03T20:28:24.257 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:28:24 smithi089 bash[78452]: audit 1685824103.2485542 mon.a (mon.0) 1601 : audit 1 from='mgr.34104 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1f", "id": [1, 2]}]': finished 2023-06-03T20:28:24.257 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:28:24 smithi089 bash[78452]: cluster 1685824103.248614 mon.a (mon.0) 1602 : cluster 0 osdmap e284: 8 total, 8 up, 8 in 2023-06-03T20:28:24.312 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:28:24.342 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:28:24.344 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:28:24.345 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:28:24.446 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:28:24.467 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:28:24.469 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:28:24.470 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:28:24.571 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:28:24.580 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:28:24 smithi164 bash[59462]: cluster 1685824103.1351402 mgr.y (mgr.34104) 1575 : cluster 0 pgmap v1624: 89 pgs: 89 active+clean; 3.3 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 87 MiB/s wr, 1.39k op/s 2023-06-03T20:28:24.581 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:28:24 smithi164 bash[59462]: cluster 1685824103.2439787 osd.6 (osd.6) 218 : cluster 3 failed to encode map e284 with expected crc 2023-06-03T20:28:24.581 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:28:24 smithi164 bash[59462]: audit 1685824103.2484143 mon.a (mon.0) 1599 : audit 1 from='mgr.34104 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.17", "id": [1, 2]}]': finished 2023-06-03T20:28:24.581 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:28:24 smithi164 bash[59462]: audit 1685824103.2484906 mon.a (mon.0) 1600 : audit 1 from='mgr.34104 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1b", "id": [1, 2]}]': finished 2023-06-03T20:28:24.581 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:28:24 smithi164 bash[59462]: audit 1685824103.2485542 mon.a (mon.0) 1601 : audit 1 from='mgr.34104 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1f", "id": [1, 2]}]': finished 2023-06-03T20:28:24.581 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:28:24 smithi164 bash[59462]: cluster 1685824103.248614 mon.a (mon.0) 1602 : cluster 0 osdmap e284: 8 total, 8 up, 8 in 2023-06-03T20:28:24.608 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:28:24.611 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:28:24.612 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:28:24.713 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:28:24.749 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:28:24.754 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:28:24.755 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:28:24.856 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:28:24.857 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 38 16 56154 56138 92.3213 45 0.0227214 0.0108263 2023-06-03T20:28:24.896 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:28:24.897 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:28:24.897 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:28:24.998 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:28:25.046 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:28:25.051 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:28:25.051 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:28:25.152 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:28:25.185 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:28:25.191 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:28:25.192 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:28:25.267 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:28:25 smithi089 bash[75068]: cluster 1685824104.2546363 osd.6 (osd.6) 219 : cluster 3 failed to encode map e285 with expected crc 2023-06-03T20:28:25.293 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:28:25.331 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:28:25.335 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:28:25.336 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:28:25.437 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:28:25.466 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:28:25.471 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:28:25.471 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:28:25.529 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:28:25 smithi089 bash[75068]: cluster 1685824104.2571726 mon.a (mon.0) 1603 : cluster 0 osdmap e285: 8 total, 8 up, 8 in 2023-06-03T20:28:25.529 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:28:25 smithi089 bash[75068]: cluster 1685824104.258063 osd.5 (osd.5) 234 : cluster 3 failed to encode map e285 with expected crc 2023-06-03T20:28:25.530 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:28:25 smithi089 bash[75068]: cluster 1685824104.2661004 osd.7 (osd.7) 232 : cluster 3 failed to encode map e285 with expected crc 2023-06-03T20:28:25.530 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:28:25 smithi089 bash[78452]: cluster 1685824104.2546363 osd.6 (osd.6) 219 : cluster 3 failed to encode map e285 with expected crc 2023-06-03T20:28:25.530 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:28:25 smithi089 bash[78452]: cluster 1685824104.2571726 mon.a (mon.0) 1603 : cluster 0 osdmap e285: 8 total, 8 up, 8 in 2023-06-03T20:28:25.530 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:28:25 smithi089 bash[78452]: cluster 1685824104.258063 osd.5 (osd.5) 234 : cluster 3 failed to encode map e285 with expected crc 2023-06-03T20:28:25.530 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:28:25 smithi089 bash[78452]: cluster 1685824104.2661004 osd.7 (osd.7) 232 : cluster 3 failed to encode map e285 with expected crc 2023-06-03T20:28:25.572 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:28:25.580 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:28:25 smithi164 bash[59462]: cluster 1685824104.2546363 osd.6 (osd.6) 219 : cluster 3 failed to encode map e285 with expected crc 2023-06-03T20:28:25.581 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:28:25 smithi164 bash[59462]: cluster 1685824104.2571726 mon.a (mon.0) 1603 : cluster 0 osdmap e285: 8 total, 8 up, 8 in 2023-06-03T20:28:25.581 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:28:25 smithi164 bash[59462]: cluster 1685824104.258063 osd.5 (osd.5) 234 : cluster 3 failed to encode map e285 with expected crc 2023-06-03T20:28:25.581 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:28:25 smithi164 bash[59462]: cluster 1685824104.2661004 osd.7 (osd.7) 232 : cluster 3 failed to encode map e285 with expected crc 2023-06-03T20:28:25.602 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:28:25.608 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:28:25.608 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:28:25.709 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:28:25.738 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:28:25.741 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:28:25.742 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:28:25.843 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:28:25.856 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 39 16 57267 57251 91.7376 69.5625 0.010206 0.0108948 2023-06-03T20:28:25.879 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:28:25.882 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:28:25.882 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:28:25.983 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:28:26.013 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:28:26.017 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:28:26.018 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:28:26.119 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:28:26.150 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:28:26.155 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:28:26.155 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:28:26.256 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:28:26.268 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 20:28:26 smithi089 bash[71163]: ::ffff:172.21.15.164 - - [03/Jun/2023:20:28:26] "GET /metrics HTTP/1.1" 200 229114 "" "Prometheus/2.33.4" 2023-06-03T20:28:26.292 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:28:26.294 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:28:26.295 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:28:26.395 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:28:26.407 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:28:26 smithi164 bash[59462]: cluster 1685824105.1355946 mgr.y (mgr.34104) 1576 : cluster 0 pgmap v1627: 89 pgs: 89 active+clean; 3.3 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 48 MiB/s wr, 764 op/s 2023-06-03T20:28:26.430 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:28:26.435 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:28:26.436 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:28:26.529 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:28:26 smithi089 bash[75068]: cluster 1685824105.1355946 mgr.y (mgr.34104) 1576 : cluster 0 pgmap v1627: 89 pgs: 89 active+clean; 3.3 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 48 MiB/s wr, 764 op/s 2023-06-03T20:28:26.530 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:28:26 smithi089 bash[78452]: cluster 1685824105.1355946 mgr.y (mgr.34104) 1576 : cluster 0 pgmap v1627: 89 pgs: 89 active+clean; 3.3 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 48 MiB/s wr, 764 op/s 2023-06-03T20:28:26.537 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:28:26.570 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:28:26.574 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:28:26.575 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:28:26.676 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:28:26.721 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:28:26.728 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:28:26.729 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:28:26.830 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:28:26.857 INFO:tasks.radosbench.radosbench.0.smithi089.stdout:2023-06-03T20:28:26.852399+0000 min lat: 0.00149899 max lat: 0.286718 avg lat: 0.0109589 2023-06-03T20:28:26.858 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-06-03T20:28:26.858 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 40 16 58393 58377 91.2034 70.375 0.0194346 0.0109589 2023-06-03T20:28:26.866 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:28:26.870 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:28:26.871 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:28:26.919 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph osd unset nodeep-scrub 2023-06-03T20:28:26.972 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:28:27.011 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:28:27.013 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:28:27.014 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:28:27.038 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-06-03T20:28:27.115 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:28:27.147 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:28:27.150 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:28:27.150 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:28:27.251 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:28:27.286 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:28:27.290 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:28:27.290 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:28:27.391 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:28:27.423 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:28:27.430 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:28:27.430 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:28:27.532 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:28:27.563 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:28:27.566 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:28:27.567 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:28:27.668 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:28:27.700 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:28:27.703 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:28:27.703 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:28:27.804 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:28:27.836 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:28:27.839 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:28:27.839 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:28:27.856 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 41 16 59436 59420 90.5687 65.1875 0.0044102 0.0110355 2023-06-03T20:28:27.940 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:28:27.970 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:28:27.972 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:28:27.973 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:28:28.074 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:28:28.106 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:28:28.109 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:28:28.110 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:28:28.210 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:28:28.247 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:28:28.250 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:28:28.250 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:28:28.335 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:28:28 smithi164 bash[59462]: cluster 1685824107.136172 mgr.y (mgr.34104) 1577 : cluster 0 pgmap v1628: 89 pgs: 89 active+clean; 3.4 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 49 MiB/s wr, 786 op/s 2023-06-03T20:28:28.351 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:28:28.385 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:28:28.386 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:28:28.387 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:28:28.488 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:28:28.521 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:28:28.523 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:28:28.524 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:28:28.529 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:28:28 smithi089 bash[78452]: cluster 1685824107.136172 mgr.y (mgr.34104) 1577 : cluster 0 pgmap v1628: 89 pgs: 89 active+clean; 3.4 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 49 MiB/s wr, 786 op/s 2023-06-03T20:28:28.529 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:28:28 smithi089 bash[75068]: cluster 1685824107.136172 mgr.y (mgr.34104) 1577 : cluster 0 pgmap v1628: 89 pgs: 89 active+clean; 3.4 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 49 MiB/s wr, 786 op/s 2023-06-03T20:28:28.625 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:28:28.656 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:28:28.658 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:28:28.659 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:28:28.760 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:28:28.793 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:28:28.797 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:28:28.798 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:28:28.857 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 42 16 60541 60525 90.0565 69.0625 0.0155643 0.0110988 2023-06-03T20:28:28.899 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:28:28.929 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:28:28.936 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:28:28.937 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:28:29.037 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:28:29.073 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:28:29.077 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:28:29.079 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:28:29.180 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:28:29.221 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:28:29.222 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:28:29.223 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:28:29.324 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:28:29.355 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:28:29.359 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:28:29.360 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:28:29.461 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:28:29.490 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:28:29.493 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:28:29.494 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:28:29.595 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:28:29.632 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:28:29.637 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:28:29.638 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:28:29.739 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:28:29.758 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/mon.a/config 2023-06-03T20:28:29.767 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:28:29.769 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:28:29.770 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:28:29.857 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 43 16 61613 61597 89.5202 67 0.00791187 0.0111651 2023-06-03T20:28:29.871 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:28:29.900 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:28:29.903 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:28:29.904 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:28:30.005 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:28:30.035 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:28:30.037 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:28:30.038 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:28:30.138 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:28:30.168 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:28:30.173 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:28:30.174 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:28:30.275 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:28:30.310 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:28:30.312 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:28:30.313 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:28:30.414 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:28:30.446 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:28:30.451 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:28:30.452 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:28:30.529 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:28:30 smithi089 bash[75068]: cluster 1685824109.1369061 mgr.y (mgr.34104) 1578 : cluster 0 pgmap v1629: 89 pgs: 89 active+clean; 3.6 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 77 MiB/s wr, 1.24k op/s 2023-06-03T20:28:30.529 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:28:30 smithi089 bash[78452]: cluster 1685824109.1369061 mgr.y (mgr.34104) 1578 : cluster 0 pgmap v1629: 89 pgs: 89 active+clean; 3.6 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 77 MiB/s wr, 1.24k op/s 2023-06-03T20:28:30.554 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:28:30.564 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:28:30 smithi164 bash[59462]: cluster 1685824109.1369061 mgr.y (mgr.34104) 1578 : cluster 0 pgmap v1629: 89 pgs: 89 active+clean; 3.6 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 77 MiB/s wr, 1.24k op/s 2023-06-03T20:28:30.586 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:28:30.589 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:28:30.590 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:28:30.691 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:28:30.729 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:28:30.734 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:28:30.736 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:28:30.837 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:28:30.857 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 44 16 62671 62655 88.9883 66.125 0.0168467 0.0112312 2023-06-03T20:28:30.876 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:28:30.877 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:28:30.878 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:28:30.979 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:28:31.021 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:28:31.023 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:28:31.023 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:28:31.125 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:28:31.173 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:28:31.175 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:28:31.176 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:28:31.276 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:28:31.318 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:28:31.321 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:28:31.322 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:28:31.423 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:28:31.467 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:28:31.470 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:28:31.471 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:28:31.572 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:28:31.607 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:28:31.610 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:28:31.611 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:28:31.711 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:28:31.744 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:28:31.749 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:28:31.750 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:28:31.823 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/mon.a/config 2023-06-03T20:28:31.825 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/mon.a/config 2023-06-03T20:28:31.852 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:28:31.858 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 45 16 63611 63595 88.3151 58.75 0.00893199 0.0113174 2023-06-03T20:28:31.887 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:28:31.891 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:28:31.892 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:28:31.993 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:28:32.030 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:28:32.032 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:28:32.032 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:28:32.133 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:28:32.176 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:28:32.180 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:28:32.181 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:28:32.281 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:28:32.309 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:28:32.314 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:28:32.315 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:28:32.416 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:28:32.425 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:28:32 smithi089 bash[78452]: cluster 1685824111.137352 mgr.y (mgr.34104) 1579 : cluster 0 pgmap v1630: 89 pgs: 89 active+clean; 3.6 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 36 MiB/s wr, 575 op/s 2023-06-03T20:28:32.426 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:28:32 smithi089 bash[75068]: cluster 1685824111.137352 mgr.y (mgr.34104) 1579 : cluster 0 pgmap v1630: 89 pgs: 89 active+clean; 3.6 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 36 MiB/s wr, 575 op/s 2023-06-03T20:28:32.449 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:28:32.449 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:28:32.450 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:28:32.550 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:28:32.560 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:28:32 smithi164 bash[59462]: cluster 1685824111.137352 mgr.y (mgr.34104) 1579 : cluster 0 pgmap v1630: 89 pgs: 89 active+clean; 3.6 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 36 MiB/s wr, 575 op/s 2023-06-03T20:28:32.590 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:28:32.592 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:28:32.593 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:28:32.694 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:28:32.737 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:28:32.740 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:28:32.741 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:28:32.842 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:28:32.858 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 46 16 64417 64401 87.4902 50.375 0.0107988 0.0114242 2023-06-03T20:28:32.875 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:28:32.879 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:28:32.880 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:28:32.981 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:28:33.002 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:28:33.004 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:28:33.004 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:28:33.105 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:28:33.133 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:28:33.135 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:28:33.137 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:28:33.238 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:28:33.268 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:28:33.272 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:28:33.273 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:28:33.373 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:28:33.405 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:28:33.407 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:28:33.408 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:28:33.508 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:28:33.545 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:28:33.551 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:28:33.552 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:28:33.652 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:28:33.688 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:28:33.691 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:28:33.691 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:28:33.792 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:28:33.833 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:28:33.836 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:28:33.836 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:28:33.858 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 47 16 65462 65446 87.0183 65.3125 0.019325 0.0114861 2023-06-03T20:28:33.937 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:28:33.964 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:28:33.966 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:28:33.967 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:28:34.068 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:28:34.085 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:28:34 smithi164 bash[59462]: cluster 1685824113.1382027 mgr.y (mgr.34104) 1580 : cluster 0 pgmap v1631: 89 pgs: 89 active+clean; 3.9 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 61 MiB/s wr, 977 op/s 2023-06-03T20:28:34.100 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:28:34.103 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:28:34.104 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:28:34.205 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:28:34.238 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:28:34.241 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:28:34.242 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:28:34.343 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:28:34.385 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:28:34.394 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:28:34.394 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:28:34.475 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:28:34 smithi089 bash[75068]: cluster 1685824113.1382027 mgr.y (mgr.34104) 1580 : cluster 0 pgmap v1631: 89 pgs: 89 active+clean; 3.9 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 61 MiB/s wr, 977 op/s 2023-06-03T20:28:34.476 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:28:34 smithi089 bash[78452]: cluster 1685824113.1382027 mgr.y (mgr.34104) 1580 : cluster 0 pgmap v1631: 89 pgs: 89 active+clean; 3.9 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 61 MiB/s wr, 977 op/s 2023-06-03T20:28:34.495 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:28:34.532 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:28:34.537 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:28:34.537 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:28:34.638 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:28:34.687 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:28:34.690 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:28:34.691 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:28:34.791 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:28:34.820 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:28:34.822 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:28:34.823 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:28:34.856 INFO:teuthology.orchestra.run.smithi089.stdout:ERROR: (22) Invalid argument 2023-06-03T20:28:34.856 INFO:teuthology.orchestra.run.smithi089.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-06-03T20:28:34.858 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 48 16 66352 66336 86.3642 55.625 0.012447 0.0115728 2023-06-03T20:28:34.923 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:28:34.956 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:28:34.958 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:28:34.959 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:28:35.059 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:28:35.094 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:28:35.097 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:28:35.098 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:28:35.199 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:28:35.231 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:28:35.233 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:28:35.233 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:28:35.334 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:28:35.368 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:28:35.372 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:28:35.373 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:28:35.473 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:28:35.505 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:28:35.509 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:28:35.510 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:28:35.611 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:28:35.649 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:28:35.651 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:28:35.651 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:28:35.752 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:28:35.786 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:28:35.789 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:28:35.790 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:28:35.858 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 49 16 67255 67239 85.7533 56.4375 0.0143749 0.0116539 2023-06-03T20:28:35.890 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:28:35.918 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:28:35.920 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:28:35.924 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:28:36.025 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:28:36.058 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:28:36.061 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:28:36.061 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:28:36.162 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:28:36.203 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:28:36.207 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:28:36.207 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:28:36.308 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:28:36.335 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:28:36 smithi164 bash[59462]: cluster 1685824115.138651 mgr.y (mgr.34104) 1581 : cluster 0 pgmap v1632: 89 pgs: 89 active+clean; 3.9 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 56 MiB/s wr, 888 op/s 2023-06-03T20:28:36.349 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:28:36.353 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:28:36.354 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:28:36.456 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:28:36.464 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 20:28:36 smithi089 bash[71163]: ::ffff:172.21.15.164 - - [03/Jun/2023:20:28:36] "GET /metrics HTTP/1.1" 200 229114 "" "Prometheus/2.33.4" 2023-06-03T20:28:36.464 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:28:36 smithi089 bash[78452]: cluster 1685824115.138651 mgr.y (mgr.34104) 1581 : cluster 0 pgmap v1632: 89 pgs: 89 active+clean; 3.9 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 56 MiB/s wr, 888 op/s 2023-06-03T20:28:36.464 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:28:36 smithi089 bash[75068]: cluster 1685824115.138651 mgr.y (mgr.34104) 1581 : cluster 0 pgmap v1632: 89 pgs: 89 active+clean; 3.9 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 56 MiB/s wr, 888 op/s 2023-06-03T20:28:36.491 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:28:36.496 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:28:36.497 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:28:36.597 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:28:36.633 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:28:36.636 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:28:36.636 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:28:36.738 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:28:36.770 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:28:36.775 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:28:36.776 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:28:36.858 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 50 16 68281 68265 85.3207 64.125 0.0365126 0.0117141 2023-06-03T20:28:36.878 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:28:36.914 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:28:36.918 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:28:36.919 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:28:37.020 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:28:37.048 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:28:37.051 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:28:37.051 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:28:37.153 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:28:37.192 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:28:37.197 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:28:37.197 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:28:37.209 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:28:37 smithi164 bash[59462]: audit 1685824117.1494055 mon.c (mon.2) 652 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T20:28:37.298 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:28:37.329 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:28:37.332 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:28:37.333 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:28:37.434 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:28:37.442 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:28:37 smithi089 bash[78452]: audit 1685824117.1494055 mon.c (mon.2) 652 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T20:28:37.442 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:28:37 smithi089 bash[75068]: audit 1685824117.1494055 mon.c (mon.2) 652 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T20:28:37.474 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:28:37.481 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:28:37.481 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:28:37.582 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:28:37.611 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:28:37.614 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:28:37.615 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:28:37.716 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:28:37.750 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:28:37.755 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:28:37.756 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:28:37.857 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:28:37.858 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 51 16 69290 69274 84.8841 63.0625 0.0138784 0.0117748 2023-06-03T20:28:37.882 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:28:37.887 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:28:37.887 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:28:37.988 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:28:38.019 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:28:38.025 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:28:38.026 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:28:38.127 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:28:38.161 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:28:38.166 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:28:38.166 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:28:38.215 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:28:38 smithi164 bash[59462]: cluster 1685824117.139064 mgr.y (mgr.34104) 1582 : cluster 0 pgmap v1633: 89 pgs: 89 active+clean; 4.0 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 56 MiB/s wr, 900 op/s 2023-06-03T20:28:38.267 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:28:38.302 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:28:38.309 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:28:38.309 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:28:38.410 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:28:38.418 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:28:38 smithi089 bash[78452]: cluster 1685824117.139064 mgr.y (mgr.34104) 1582 : cluster 0 pgmap v1633: 89 pgs: 89 active+clean; 4.0 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 56 MiB/s wr, 900 op/s 2023-06-03T20:28:38.418 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:28:38 smithi089 bash[75068]: cluster 1685824117.139064 mgr.y (mgr.34104) 1582 : cluster 0 pgmap v1633: 89 pgs: 89 active+clean; 4.0 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 56 MiB/s wr, 900 op/s 2023-06-03T20:28:38.440 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:28:38.444 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:28:38.445 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:28:38.547 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:28:38.578 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:28:38.580 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:28:38.580 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:28:38.681 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:28:38.712 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:28:38.717 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:28:38.718 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:28:38.819 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:28:38.858 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 52 16 70293 70277 84.4572 62.6875 0.00456589 0.0118339 2023-06-03T20:28:38.862 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:28:38.869 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:28:38.869 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:28:38.970 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:28:39.003 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:28:39.008 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:28:39.008 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:28:39.109 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:28:39.140 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:28:39.143 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:28:39.144 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:28:39.245 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:28:39.273 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:28:39.275 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:28:39.275 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:28:39.376 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:28:39.413 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:28:39.419 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:28:39.420 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:28:39.521 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:28:39.558 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:28:39.560 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:28:39.561 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:28:39.662 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:28:39.698 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:28:39.700 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:28:39.701 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:28:39.802 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:28:39.834 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:28:39.839 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:28:39.840 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:28:39.858 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 53 16 71174 71158 83.9025 55.0625 0.0333195 0.0119132 2023-06-03T20:28:39.941 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:28:39.973 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:28:39.977 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:28:39.978 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:28:40.079 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:28:40.111 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:28:40.117 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:28:40.118 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:28:40.219 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:28:40.220 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:28:40 smithi164 bash[59462]: cluster 1685824119.139857 mgr.y (mgr.34104) 1583 : cluster 0 pgmap v1634: 89 pgs: 89 active+clean; 4.2 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 70 MiB/s wr, 1.12k op/s 2023-06-03T20:28:40.240 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:28:40 smithi089 bash[78452]: cluster 1685824119.139857 mgr.y (mgr.34104) 1583 : cluster 0 pgmap v1634: 89 pgs: 89 active+clean; 4.2 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 70 MiB/s wr, 1.12k op/s 2023-06-03T20:28:40.240 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:28:40 smithi089 bash[75068]: cluster 1685824119.139857 mgr.y (mgr.34104) 1583 : cluster 0 pgmap v1634: 89 pgs: 89 active+clean; 4.2 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 70 MiB/s wr, 1.12k op/s 2023-06-03T20:28:40.252 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:28:40.255 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:28:40.255 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:28:40.356 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:28:40.387 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:28:40.389 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:28:40.389 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:28:40.490 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:28:40.526 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:28:40.530 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:28:40.531 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:28:40.632 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:28:40.679 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:28:40.683 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:28:40.683 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:28:40.747 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph daemon osd.0 dump_blocked_ops 2023-06-03T20:28:40.784 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:28:40.838 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:28:40.841 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:28:40.841 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:28:40.858 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 54 16 72124 72108 83.4482 59.375 0.00543099 0.0119781 2023-06-03T20:28:40.942 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:28:40.977 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:28:40.981 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:28:40.981 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:28:41.082 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:28:41.120 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:28:41.123 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:28:41.124 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:28:41.225 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:28:41.248 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:28:41.250 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:28:41.250 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:28:41.351 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:28:41.394 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:28:41.400 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:28:41.400 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:28:41.501 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:28:41.537 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:28:41.539 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:28:41.540 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:28:41.641 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:28:41.680 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:28:41.682 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:28:41.683 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:28:41.784 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:28:41.819 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:28:41.823 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:28:41.823 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:28:41.859 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 55 16 72885 72869 82.7956 47.5625 0.0133565 0.0120652 2023-06-03T20:28:41.924 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:28:41.961 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:28:41.964 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:28:41.965 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:28:42.065 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:28:42.095 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:28:42.099 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:28:42.100 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:28:42.201 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:28:42.229 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:28:42.232 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:28:42.233 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:28:42.334 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:28:42.335 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:28:42 smithi164 bash[59462]: cluster 1685824121.1404915 mgr.y (mgr.34104) 1584 : cluster 0 pgmap v1635: 89 pgs: 89 active+clean; 4.2 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 51 MiB/s wr, 820 op/s 2023-06-03T20:28:42.370 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:28:42.376 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:28:42.376 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:28:42.467 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:28:42 smithi089 bash[75068]: cluster 1685824121.1404915 mgr.y (mgr.34104) 1584 : cluster 0 pgmap v1635: 89 pgs: 89 active+clean; 4.2 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 51 MiB/s wr, 820 op/s 2023-06-03T20:28:42.467 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:28:42 smithi089 bash[78452]: cluster 1685824121.1404915 mgr.y (mgr.34104) 1584 : cluster 0 pgmap v1635: 89 pgs: 89 active+clean; 4.2 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 51 MiB/s wr, 820 op/s 2023-06-03T20:28:42.478 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:28:42.513 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:28:42.515 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:28:42.516 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:28:42.617 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:28:42.648 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:28:42.652 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:28:42.652 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:28:42.753 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:28:42.787 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:28:42.790 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:28:42.791 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:28:42.858 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 56 16 73670 73654 82.1932 49.0625 0.0102586 0.0121608 2023-06-03T20:28:42.891 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:28:42.926 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:28:42.932 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:28:42.933 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:28:43.034 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:28:43.067 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:28:43.069 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:28:43.070 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:28:43.171 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:28:43.201 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:28:43.204 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:28:43.205 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:28:43.306 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:28:43.339 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:28:43.343 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:28:43.345 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:28:43.446 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:28:43.477 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:28:43.481 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:28:43.482 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:28:43.583 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:28:43.622 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:28:43.627 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:28:43.627 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:28:43.728 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:28:43.761 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:28:43.763 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:28:43.764 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:28:43.859 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 57 16 74482 74466 81.6415 50.75 0.0153442 0.0122431 2023-06-03T20:28:43.865 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:28:43.896 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:28:43.902 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:28:43.902 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:28:44.003 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:28:44.039 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:28:44.043 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:28:44.044 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:28:44.145 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:28:44.159 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:28:44 smithi089 bash[78452]: cluster 1685824123.1413984 mgr.y (mgr.34104) 1585 : cluster 0 pgmap v1636: 89 pgs: 89 active+clean; 4.5 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 75 MiB/s wr, 1.19k op/s 2023-06-03T20:28:44.160 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:28:44 smithi089 bash[75068]: cluster 1685824123.1413984 mgr.y (mgr.34104) 1585 : cluster 0 pgmap v1636: 89 pgs: 89 active+clean; 4.5 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 75 MiB/s wr, 1.19k op/s 2023-06-03T20:28:44.188 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:28:44.189 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:28:44.190 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:28:44.291 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:28:44.301 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:28:44 smithi164 bash[59462]: cluster 1685824123.1413984 mgr.y (mgr.34104) 1585 : cluster 0 pgmap v1636: 89 pgs: 89 active+clean; 4.5 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 75 MiB/s wr, 1.19k op/s 2023-06-03T20:28:44.325 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:28:44.328 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:28:44.328 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:28:44.430 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:28:44.457 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:28:44.461 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:28:44.461 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:28:44.562 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:28:44.609 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:28:44.615 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:28:44.616 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:28:44.716 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:28:44.748 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:28:44.750 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:28:44.751 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:28:44.852 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:28:44.859 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 58 16 75433 75417 81.2586 59.4375 0.0171112 0.0123007 2023-06-03T20:28:44.879 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:28:44.883 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:28:44.883 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:28:44.984 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:28:45.013 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:28:45.015 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:28:45.015 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:28:45.117 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:28:45.145 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:28:45.151 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:28:45.152 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:28:45.252 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:28:45.288 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:28:45.290 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:28:45.291 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:28:45.392 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:28:45.422 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:28:45.427 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:28:45.428 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:28:45.529 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:28:45.563 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:28:45.567 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:28:45.568 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:28:45.669 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:28:45.706 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:28:45.710 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:28:45.711 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:28:45.812 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:28:45.841 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:28:45.843 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:28:45.844 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:28:45.859 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 59 16 76261 76245 80.7584 51.75 0.0171426 0.012377 2023-06-03T20:28:45.945 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:28:45.977 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:28:45.982 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:28:45.983 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:28:46.084 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:28:46.114 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:28:46.126 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:28:46.127 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:28:46.228 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:28:46.262 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:28:46.267 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:28:46.267 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:28:46.335 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:28:46 smithi164 bash[59462]: cluster 1685824125.1418133 mgr.y (mgr.34104) 1586 : cluster 0 pgmap v1637: 89 pgs: 89 active+clean; 4.5 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 48 MiB/s wr, 772 op/s 2023-06-03T20:28:46.335 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:28:46 smithi164 bash[59462]: audit 1685824125.7247868 mon.a (mon.0) 1604 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:28:46.336 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:28:46 smithi164 bash[59462]: audit 1685824125.736376 mon.a (mon.0) 1605 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:28:46.336 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:28:46 smithi164 bash[59462]: audit 1685824125.7387762 mon.c (mon.2) 653 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-03T20:28:46.336 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:28:46 smithi164 bash[59462]: audit 1685824125.7399142 mon.c (mon.2) 654 : audit 1 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-03T20:28:46.336 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:28:46 smithi164 bash[59462]: audit 1685824125.7468677 mon.a (mon.0) 1606 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:28:46.368 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:28:46.408 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:28:46.410 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:28:46.411 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:28:46.483 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:28:46 smithi089 bash[75068]: cluster 1685824125.1418133 mgr.y (mgr.34104) 1586 : cluster 0 pgmap v1637: 89 pgs: 89 active+clean; 4.5 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 48 MiB/s wr, 772 op/s 2023-06-03T20:28:46.483 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:28:46 smithi089 bash[75068]: audit 1685824125.7247868 mon.a (mon.0) 1604 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:28:46.483 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:28:46 smithi089 bash[75068]: audit 1685824125.736376 mon.a (mon.0) 1605 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:28:46.483 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:28:46 smithi089 bash[75068]: audit 1685824125.7387762 mon.c (mon.2) 653 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-03T20:28:46.483 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:28:46 smithi089 bash[75068]: audit 1685824125.7399142 mon.c (mon.2) 654 : audit 1 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-03T20:28:46.484 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:28:46 smithi089 bash[75068]: audit 1685824125.7468677 mon.a (mon.0) 1606 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:28:46.484 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 20:28:46 smithi089 bash[71163]: ::ffff:172.21.15.164 - - [03/Jun/2023:20:28:46] "GET /metrics HTTP/1.1" 200 229132 "" "Prometheus/2.33.4" 2023-06-03T20:28:46.484 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:28:46 smithi089 bash[78452]: cluster 1685824125.1418133 mgr.y (mgr.34104) 1586 : cluster 0 pgmap v1637: 89 pgs: 89 active+clean; 4.5 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 48 MiB/s wr, 772 op/s 2023-06-03T20:28:46.484 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:28:46 smithi089 bash[78452]: audit 1685824125.7247868 mon.a (mon.0) 1604 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:28:46.485 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:28:46 smithi089 bash[78452]: audit 1685824125.736376 mon.a (mon.0) 1605 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:28:46.485 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:28:46 smithi089 bash[78452]: audit 1685824125.7387762 mon.c (mon.2) 653 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-03T20:28:46.485 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:28:46 smithi089 bash[78452]: audit 1685824125.7399142 mon.c (mon.2) 654 : audit 1 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-03T20:28:46.485 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:28:46 smithi089 bash[78452]: audit 1685824125.7468677 mon.a (mon.0) 1606 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:28:46.512 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:28:46.539 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:28:46.541 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:28:46.542 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:28:46.643 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:28:46.687 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:28:46.692 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:28:46.693 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:28:46.794 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:28:46.823 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:28:46.826 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:28:46.826 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:28:46.859 INFO:tasks.radosbench.radosbench.0.smithi089.stdout:2023-06-03T20:28:46.854883+0000 min lat: 0.00149899 max lat: 0.286718 avg lat: 0.0124296 2023-06-03T20:28:46.859 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-06-03T20:28:46.859 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 60 15 77216 77201 80.4081 59.75 0.00846895 0.0124296 2023-06-03T20:28:46.928 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:28:46.961 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:28:46.964 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:28:46.964 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:28:47.065 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:28:47.096 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:28:47.100 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:28:47.101 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:28:47.201 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:28:47.236 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:28:47.239 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:28:47.240 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:28:47.341 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:28:47.370 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:28:47.372 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:28:47.373 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:28:47.474 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:28:47.512 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:28:47.515 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:28:47.516 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:28:47.617 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:28:47.653 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:28:47.656 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:28:47.656 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:28:47.757 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:28:47.795 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:28:47.797 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:28:47.798 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:28:47.859 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 61 16 78028 78012 79.9208 50.6875 0.0146883 0.0125069 2023-06-03T20:28:47.899 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:28:47.945 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:28:47.950 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:28:47.950 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:28:48.051 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:28:48.089 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:28:48.097 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:28:48.097 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:28:48.198 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:28:48.232 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:28:48.235 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:28:48.236 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:28:48.335 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:28:48 smithi164 bash[59462]: cluster 1685824127.1423612 mgr.y (mgr.34104) 1587 : cluster 0 pgmap v1638: 89 pgs: 89 active+clean; 4.5 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 54 MiB/s wr, 860 op/s 2023-06-03T20:28:48.337 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:28:48.370 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:28:48.380 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:28:48.381 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:28:48.482 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:28:48.491 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:28:48 smithi089 bash[75068]: cluster 1685824127.1423612 mgr.y (mgr.34104) 1587 : cluster 0 pgmap v1638: 89 pgs: 89 active+clean; 4.5 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 54 MiB/s wr, 860 op/s 2023-06-03T20:28:48.492 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:28:48 smithi089 bash[78452]: cluster 1685824127.1423612 mgr.y (mgr.34104) 1587 : cluster 0 pgmap v1638: 89 pgs: 89 active+clean; 4.5 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 54 MiB/s wr, 860 op/s 2023-06-03T20:28:48.531 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:28:48.532 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:28:48.532 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:28:48.633 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:28:48.659 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:28:48.660 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:28:48.661 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:28:48.762 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:28:48.793 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:28:48.797 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:28:48.798 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:28:48.859 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 62 16 78921 78905 79.5319 55.8125 0.00714141 0.0125674 2023-06-03T20:28:48.899 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:28:48.933 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:28:48.934 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:28:48.935 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:28:49.036 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:28:49.067 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:28:49.069 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:28:49.070 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:28:49.171 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:28:49.203 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:28:49.207 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:28:49.207 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:28:49.308 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:28:49.340 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:28:49.343 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:28:49.343 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:28:49.444 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:28:49.484 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:28:49.488 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:28:49.489 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:28:49.590 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:28:49.625 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:28:49.631 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:28:49.632 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:28:49.733 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:28:49.766 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:28:49.771 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:28:49.772 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:28:49.859 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 63 16 79845 79829 79.1861 57.75 0.00787938 0.012621 2023-06-03T20:28:49.873 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:28:49.906 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:28:49.911 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:28:49.912 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:28:50.013 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:28:50.033 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:28:50.036 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:28:50.036 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:28:50.137 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:28:50.157 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:28:50.159 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:28:50.160 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:28:50.217 INFO:teuthology.orchestra.run.smithi089.stderr:reweighted osd.6 to 1 (10000) 2023-06-03T20:28:50.261 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:28:50.279 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:28:50 smithi089 bash[78452]: cluster 1685824129.1431472 mgr.y (mgr.34104) 1588 : cluster 0 pgmap v1639: 89 pgs: 89 active+clean; 4.7 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 65 MiB/s wr, 1.04k op/s 2023-06-03T20:28:50.279 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:28:50 smithi089 bash[78452]: audit 1685824130.1241126 mon.c (mon.2) 655 : audit 1 from='client.? 172.21.15.89:0/65802772' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-06-03T20:28:50.279 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:28:50 smithi089 bash[78452]: audit 1685824130.1244116 mon.a (mon.0) 1607 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-06-03T20:28:50.280 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:28:50 smithi089 bash[75068]: cluster 1685824129.1431472 mgr.y (mgr.34104) 1588 : cluster 0 pgmap v1639: 89 pgs: 89 active+clean; 4.7 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 65 MiB/s wr, 1.04k op/s 2023-06-03T20:28:50.280 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:28:50 smithi089 bash[75068]: audit 1685824130.1241126 mon.c (mon.2) 655 : audit 1 from='client.? 172.21.15.89:0/65802772' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-06-03T20:28:50.280 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:28:50 smithi089 bash[75068]: audit 1685824130.1244116 mon.a (mon.0) 1607 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-06-03T20:28:50.291 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:28:50.294 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:28:50.295 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:28:50.396 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:28:50.422 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:28:50.429 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:28:50.429 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:28:50.530 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:28:50.565 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:28:50.567 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:28:50.568 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:28:50.585 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:28:50 smithi164 bash[59462]: cluster 1685824129.1431472 mgr.y (mgr.34104) 1588 : cluster 0 pgmap v1639: 89 pgs: 89 active+clean; 4.7 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 65 MiB/s wr, 1.04k op/s 2023-06-03T20:28:50.585 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:28:50 smithi164 bash[59462]: audit 1685824130.1241126 mon.c (mon.2) 655 : audit 1 from='client.? 172.21.15.89:0/65802772' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-06-03T20:28:50.585 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:28:50 smithi164 bash[59462]: audit 1685824130.1244116 mon.a (mon.0) 1607 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-06-03T20:28:50.669 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:28:50.701 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:28:50.704 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:28:50.705 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:28:50.806 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:28:50.834 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:28:50.837 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:28:50.838 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:28:50.859 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 64 16 80215 80199 78.3101 23.125 0.00454921 0.0126681 2023-06-03T20:28:50.939 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:28:50.976 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:28:50.980 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:28:50.980 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:28:51.081 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:28:51.111 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:28:51.113 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:28:51.114 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:28:51.215 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:28:51.250 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:28:51.252 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:28:51.252 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:28:51.353 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:28:51.365 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:28:51 smithi089 bash[75068]: cluster 1685824130.2024431 osd.5 (osd.5) 235 : cluster 3 failed to encode map e286 with expected crc 2023-06-03T20:28:51.366 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:28:51 smithi089 bash[75068]: audit 1685824130.2066045 mon.a (mon.0) 1608 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2023-06-03T20:28:51.366 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:28:51 smithi089 bash[75068]: cluster 1685824130.2066507 mon.a (mon.0) 1609 : cluster 0 osdmap e286: 8 total, 8 up, 8 in 2023-06-03T20:28:51.366 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:28:51 smithi089 bash[78452]: cluster 1685824130.2024431 osd.5 (osd.5) 235 : cluster 3 failed to encode map e286 with expected crc 2023-06-03T20:28:51.366 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:28:51 smithi089 bash[78452]: audit 1685824130.2066045 mon.a (mon.0) 1608 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2023-06-03T20:28:51.366 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:28:51 smithi089 bash[78452]: cluster 1685824130.2066507 mon.a (mon.0) 1609 : cluster 0 osdmap e286: 8 total, 8 up, 8 in 2023-06-03T20:28:51.383 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:28:51.385 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:28:51.385 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:28:51.486 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:28:51.519 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:28:51.522 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:28:51.523 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:28:51.585 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:28:51 smithi164 bash[59462]: cluster 1685824130.2024431 osd.5 (osd.5) 235 : cluster 3 failed to encode map e286 with expected crc 2023-06-03T20:28:51.586 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:28:51 smithi164 bash[59462]: audit 1685824130.2066045 mon.a (mon.0) 1608 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2023-06-03T20:28:51.586 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:28:51 smithi164 bash[59462]: cluster 1685824130.2066507 mon.a (mon.0) 1609 : cluster 0 osdmap e286: 8 total, 8 up, 8 in 2023-06-03T20:28:51.623 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:28:51.655 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:28:51.658 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:28:51.659 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:28:51.760 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:28:51.797 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:28:51.799 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:28:51.800 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:28:51.860 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 65 16 80215 80199 77.1053 0 - 0.0126681 2023-06-03T20:28:51.900 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:28:51.941 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:28:51.944 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:28:51.945 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:28:52.046 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:28:52.084 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:28:52.086 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:28:52.087 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:28:52.188 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:28:52.231 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:28:52.234 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:28:52.235 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:28:52.335 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:28:52.343 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:28:52 smithi089 bash[78452]: cluster 1685824131.143496 mgr.y (mgr.34104) 1589 : cluster 0 pgmap v1641: 89 pgs: 89 active+clean; 4.7 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 55 MiB/s wr, 885 op/s 2023-06-03T20:28:52.343 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:28:52 smithi089 bash[78452]: cluster 1685824131.216678 osd.5 (osd.5) 236 : cluster 3 failed to encode map e287 with expected crc 2023-06-03T20:28:52.343 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:28:52 smithi089 bash[78452]: cluster 1685824131.2185652 osd.7 (osd.7) 233 : cluster 3 failed to encode map e287 with expected crc 2023-06-03T20:28:52.343 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:28:52 smithi089 bash[78452]: cluster 1685824131.2191226 osd.6 (osd.6) 220 : cluster 3 failed to encode map e287 with expected crc 2023-06-03T20:28:52.344 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:28:52 smithi089 bash[78452]: cluster 1685824131.2208822 osd.7 (osd.7) 234 : cluster 3 failed to encode map e287 with expected crc 2023-06-03T20:28:52.344 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:28:52 smithi089 bash[78452]: cluster 1685824131.2209082 mon.a (mon.0) 1610 : cluster 0 osdmap e287: 8 total, 8 up, 8 in 2023-06-03T20:28:52.344 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:28:52 smithi089 bash[78452]: cluster 1685824131.2234035 osd.6 (osd.6) 221 : cluster 3 failed to encode map e287 with expected crc 2023-06-03T20:28:52.344 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:28:52 smithi089 bash[78452]: cluster 1685824131.2259092 osd.6 (osd.6) 222 : cluster 3 failed to encode map e287 with expected crc 2023-06-03T20:28:52.344 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:28:52 smithi089 bash[78452]: audit 1685824132.151562 mon.c (mon.2) 656 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T20:28:52.344 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:28:52 smithi089 bash[75068]: cluster 1685824131.143496 mgr.y (mgr.34104) 1589 : cluster 0 pgmap v1641: 89 pgs: 89 active+clean; 4.7 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 55 MiB/s wr, 885 op/s 2023-06-03T20:28:52.345 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:28:52 smithi089 bash[75068]: cluster 1685824131.216678 osd.5 (osd.5) 236 : cluster 3 failed to encode map e287 with expected crc 2023-06-03T20:28:52.345 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:28:52 smithi089 bash[75068]: cluster 1685824131.2185652 osd.7 (osd.7) 233 : cluster 3 failed to encode map e287 with expected crc 2023-06-03T20:28:52.345 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:28:52 smithi089 bash[75068]: cluster 1685824131.2191226 osd.6 (osd.6) 220 : cluster 3 failed to encode map e287 with expected crc 2023-06-03T20:28:52.345 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:28:52 smithi089 bash[75068]: cluster 1685824131.2208822 osd.7 (osd.7) 234 : cluster 3 failed to encode map e287 with expected crc 2023-06-03T20:28:52.345 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:28:52 smithi089 bash[75068]: cluster 1685824131.2209082 mon.a (mon.0) 1610 : cluster 0 osdmap e287: 8 total, 8 up, 8 in 2023-06-03T20:28:52.345 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:28:52 smithi089 bash[75068]: cluster 1685824131.2234035 osd.6 (osd.6) 221 : cluster 3 failed to encode map e287 with expected crc 2023-06-03T20:28:52.345 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:28:52 smithi089 bash[75068]: cluster 1685824131.2259092 osd.6 (osd.6) 222 : cluster 3 failed to encode map e287 with expected crc 2023-06-03T20:28:52.346 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:28:52 smithi089 bash[75068]: audit 1685824132.151562 mon.c (mon.2) 656 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T20:28:52.359 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:28:52.361 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:28:52.362 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:28:52.463 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:28:52.494 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:28:52.497 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:28:52.497 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:28:52.585 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:28:52 smithi164 bash[59462]: cluster 1685824131.143496 mgr.y (mgr.34104) 1589 : cluster 0 pgmap v1641: 89 pgs: 89 active+clean; 4.7 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 55 MiB/s wr, 885 op/s 2023-06-03T20:28:52.585 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:28:52 smithi164 bash[59462]: cluster 1685824131.216678 osd.5 (osd.5) 236 : cluster 3 failed to encode map e287 with expected crc 2023-06-03T20:28:52.585 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:28:52 smithi164 bash[59462]: cluster 1685824131.2185652 osd.7 (osd.7) 233 : cluster 3 failed to encode map e287 with expected crc 2023-06-03T20:28:52.585 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:28:52 smithi164 bash[59462]: cluster 1685824131.2191226 osd.6 (osd.6) 220 : cluster 3 failed to encode map e287 with expected crc 2023-06-03T20:28:52.586 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:28:52 smithi164 bash[59462]: cluster 1685824131.2208822 osd.7 (osd.7) 234 : cluster 3 failed to encode map e287 with expected crc 2023-06-03T20:28:52.586 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:28:52 smithi164 bash[59462]: cluster 1685824131.2209082 mon.a (mon.0) 1610 : cluster 0 osdmap e287: 8 total, 8 up, 8 in 2023-06-03T20:28:52.586 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:28:52 smithi164 bash[59462]: cluster 1685824131.2234035 osd.6 (osd.6) 221 : cluster 3 failed to encode map e287 with expected crc 2023-06-03T20:28:52.587 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:28:52 smithi164 bash[59462]: cluster 1685824131.2259092 osd.6 (osd.6) 222 : cluster 3 failed to encode map e287 with expected crc 2023-06-03T20:28:52.587 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:28:52 smithi164 bash[59462]: audit 1685824132.151562 mon.c (mon.2) 656 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T20:28:52.598 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:28:52.636 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:28:52.638 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:28:52.639 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:28:52.697 INFO:tasks.cephadm.ceph_manager.ceph:waiting for recovery to complete 2023-06-03T20:28:52.698 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph pg dump --format=json 2023-06-03T20:28:52.739 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:28:52.793 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:28:52.799 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:28:52.800 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:28:52.860 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 66 16 80331 80315 76.0469 3.625 0.0217107 0.0131427 2023-06-03T20:28:52.901 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:28:52.954 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:28:52.958 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:28:52.958 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:28:53.059 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:28:53.089 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:28:53.092 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:28:53.093 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:28:53.165 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/mon.a/config 2023-06-03T20:28:53.193 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:28:53.232 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:28:53.235 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:28:53.236 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:28:53.335 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:28:53 smithi164 bash[59462]: cluster 1685824132.223265 mon.a (mon.0) 1611 : cluster 0 osdmap e288: 8 total, 8 up, 8 in 2023-06-03T20:28:53.335 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:28:53 smithi164 bash[59462]: cluster 1685824132.2499073 osd.5 (osd.5) 237 : cluster 3 failed to encode map e288 with expected crc 2023-06-03T20:28:53.336 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:28:53 smithi164 bash[59462]: cluster 1685824132.4964836 osd.6 (osd.6) 223 : cluster 3 failed to encode map e288 with expected crc 2023-06-03T20:28:53.337 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:28:53.373 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:28:53.380 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:28:53.381 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:28:53 smithi089 bash[75068]: cluster 1685824132.223265 mon.a (mon.0) 1611 : cluster 0 osdmap e288: 8 total, 8 up, 8 in 2023-06-03T20:28:53.381 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:28:53 smithi089 bash[75068]: cluster 1685824132.2499073 osd.5 (osd.5) 237 : cluster 3 failed to encode map e288 with expected crc 2023-06-03T20:28:53.381 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:28:53 smithi089 bash[75068]: cluster 1685824132.4964836 osd.6 (osd.6) 223 : cluster 3 failed to encode map e288 with expected crc 2023-06-03T20:28:53.382 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:28:53 smithi089 bash[78452]: cluster 1685824132.223265 mon.a (mon.0) 1611 : cluster 0 osdmap e288: 8 total, 8 up, 8 in 2023-06-03T20:28:53.382 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:28:53 smithi089 bash[78452]: cluster 1685824132.2499073 osd.5 (osd.5) 237 : cluster 3 failed to encode map e288 with expected crc 2023-06-03T20:28:53.382 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:28:53 smithi089 bash[78452]: cluster 1685824132.4964836 osd.6 (osd.6) 223 : cluster 3 failed to encode map e288 with expected crc 2023-06-03T20:28:53.383 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:28:53.448 INFO:teuthology.orchestra.run.smithi089.stdout:osd.0: {} 2023-06-03T20:28:53.484 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:28:53.508 INFO:teuthology.orchestra.run.smithi089.stdout:osd.1: {} 2023-06-03T20:28:53.516 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:28:53.518 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:28:53.519 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:28:53.549 INFO:teuthology.orchestra.run.smithi089.stdout:osd.2: {} 2023-06-03T20:28:53.592 INFO:teuthology.orchestra.run.smithi089.stdout:osd.3: {} 2023-06-03T20:28:53.619 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:28:53.641 INFO:teuthology.orchestra.run.smithi089.stdout:osd.4: {} 2023-06-03T20:28:53.656 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:28:53.663 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:28:53.664 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:28:53.684 INFO:teuthology.orchestra.run.smithi089.stdout:osd.5: {} 2023-06-03T20:28:53.713 INFO:teuthology.orchestra.run.smithi089.stdout:osd.6: {} 2023-06-03T20:28:53.756 INFO:teuthology.orchestra.run.smithi089.stdout:osd.7: {} 2023-06-03T20:28:53.762 INFO:teuthology.orchestra.run.smithi089.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-06-03T20:28:53.762 INFO:teuthology.orchestra.run.smithi089.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-06-03T20:28:53.763 INFO:teuthology.orchestra.run.smithi089.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-06-03T20:28:53.763 INFO:teuthology.orchestra.run.smithi089.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-06-03T20:28:53.763 INFO:teuthology.orchestra.run.smithi089.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-06-03T20:28:53.763 INFO:teuthology.orchestra.run.smithi089.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-06-03T20:28:53.763 INFO:teuthology.orchestra.run.smithi089.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-06-03T20:28:53.764 INFO:teuthology.orchestra.run.smithi089.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-06-03T20:28:53.765 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:28:53.801 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:28:53.807 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:28:53.807 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:28:53.860 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 67 16 80986 80970 75.5228 40.9375 0.0196166 0.0132347 2023-06-03T20:28:53.908 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:28:53.940 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:28:53.943 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:28:53.944 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:28:54.045 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:28:54.075 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:28:54.077 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:28:54.078 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:28:54.180 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:28:54.220 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:28:54.221 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:28:54.222 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:28:54.269 INFO:teuthology.orchestra.run.smithi089.stderr:nodeep-scrub is unset 2023-06-03T20:28:54.289 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:28:54 smithi089 bash[75068]: cluster 1685824132.6607509 osd.7 (osd.7) 235 : cluster 3 failed to encode map e288 with expected crc 2023-06-03T20:28:54.289 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:28:54 smithi089 bash[75068]: cluster 1685824133.145168 mgr.y (mgr.34104) 1590 : cluster 0 pgmap v1644: 89 pgs: 1 remapped+peering, 88 active+clean; 4.9 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 61 MiB/s wr, 980 op/s; 0 B/s, 0 objects/s recovering 2023-06-03T20:28:54.289 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:28:54 smithi089 bash[75068]: audit 1685824133.379823 mon.c (mon.2) 657 : audit 0 from='client.? 172.21.15.89:0/4090089307' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-03T20:28:54.289 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:28:54 smithi089 bash[75068]: audit 1685824133.7766874 mon.a (mon.0) 1612 : audit 1 from='client.? 172.21.15.89:0/1709841754' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-06-03T20:28:54.290 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:28:54 smithi089 bash[78452]: cluster 1685824132.6607509 osd.7 (osd.7) 235 : cluster 3 failed to encode map e288 with expected crc 2023-06-03T20:28:54.290 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:28:54 smithi089 bash[78452]: cluster 1685824133.145168 mgr.y (mgr.34104) 1590 : cluster 0 pgmap v1644: 89 pgs: 1 remapped+peering, 88 active+clean; 4.9 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 61 MiB/s wr, 980 op/s; 0 B/s, 0 objects/s recovering 2023-06-03T20:28:54.290 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:28:54 smithi089 bash[78452]: audit 1685824133.379823 mon.c (mon.2) 657 : audit 0 from='client.? 172.21.15.89:0/4090089307' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-03T20:28:54.290 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:28:54 smithi089 bash[78452]: audit 1685824133.7766874 mon.a (mon.0) 1612 : audit 1 from='client.? 172.21.15.89:0/1709841754' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-06-03T20:28:54.324 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:28:54.366 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:28:54.373 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:28:54.374 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:28:54.475 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:28:54.509 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:28:54.513 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:28:54.514 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:28:54.585 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:28:54 smithi164 bash[59462]: cluster 1685824132.6607509 osd.7 (osd.7) 235 : cluster 3 failed to encode map e288 with expected crc 2023-06-03T20:28:54.585 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:28:54 smithi164 bash[59462]: cluster 1685824133.145168 mgr.y (mgr.34104) 1590 : cluster 0 pgmap v1644: 89 pgs: 1 remapped+peering, 88 active+clean; 4.9 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 61 MiB/s wr, 980 op/s; 0 B/s, 0 objects/s recovering 2023-06-03T20:28:54.585 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:28:54 smithi164 bash[59462]: audit 1685824133.379823 mon.c (mon.2) 657 : audit 0 from='client.? 172.21.15.89:0/4090089307' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-03T20:28:54.586 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:28:54 smithi164 bash[59462]: audit 1685824133.7766874 mon.a (mon.0) 1612 : audit 1 from='client.? 172.21.15.89:0/1709841754' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-06-03T20:28:54.614 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:28:54.653 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:28:54.659 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:28:54.660 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:28:54.761 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:28:54.799 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:28:54.804 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:28:54.804 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:28:54.860 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 68 16 81761 81745 75.1245 48.4375 0.0159969 0.013305 2023-06-03T20:28:54.905 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:28:54.941 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:28:54.947 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:28:54.947 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:28:55.048 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:28:55.090 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:28:55.094 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:28:55.094 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:28:55.195 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:28:55.225 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:28:55.228 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:28:55.229 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:28:55.237 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:28:55 smithi089 bash[75068]: cluster 1685824134.2289517 mon.a (mon.0) 1613 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-06-03T20:28:55.237 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:28:55 smithi089 bash[75068]: cluster 1685824134.2289853 mon.a (mon.0) 1614 : cluster 1 Cluster is now healthy 2023-06-03T20:28:55.237 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:28:55 smithi089 bash[75068]: cluster 1685824134.25197 osd.6 (osd.6) 224 : cluster 3 failed to encode map e289 with expected crc 2023-06-03T20:28:55.237 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:28:55 smithi089 bash[75068]: cluster 1685824134.2533023 osd.7 (osd.7) 236 : cluster 3 failed to encode map e289 with expected crc 2023-06-03T20:28:55.330 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:28:55.368 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:28:55.371 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:28:55.372 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:28:55.473 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:28:55.501 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:28:55.507 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:28:55 smithi089 bash[78452]: cluster 1685824134.2289517 mon.a (mon.0) 1613 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-06-03T20:28:55.507 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:28:55 smithi089 bash[78452]: cluster 1685824134.2289853 mon.a (mon.0) 1614 : cluster 1 Cluster is now healthy 2023-06-03T20:28:55.507 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:28:55 smithi089 bash[78452]: cluster 1685824134.25197 osd.6 (osd.6) 224 : cluster 3 failed to encode map e289 with expected crc 2023-06-03T20:28:55.508 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:28:55 smithi089 bash[78452]: cluster 1685824134.2533023 osd.7 (osd.7) 236 : cluster 3 failed to encode map e289 with expected crc 2023-06-03T20:28:55.508 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:28:55 smithi089 bash[78452]: audit 1685824134.2547626 mon.a (mon.0) 1615 : audit 1 from='client.? 172.21.15.89:0/1709841754' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-06-03T20:28:55.508 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:28:55 smithi089 bash[78452]: cluster 1685824134.2549875 mon.a (mon.0) 1616 : cluster 0 osdmap e289: 8 total, 8 up, 8 in 2023-06-03T20:28:55.508 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:28:55 smithi089 bash[78452]: cluster 1685824134.2584674 osd.5 (osd.5) 238 : cluster 3 failed to encode map e289 with expected crc 2023-06-03T20:28:55.508 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:28:55 smithi089 bash[75068]: audit 1685824134.2547626 mon.a (mon.0) 1615 : audit 1 from='client.? 172.21.15.89:0/1709841754' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-06-03T20:28:55.509 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:28:55 smithi089 bash[75068]: cluster 1685824134.2549875 mon.a (mon.0) 1616 : cluster 0 osdmap e289: 8 total, 8 up, 8 in 2023-06-03T20:28:55.509 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:28:55 smithi089 bash[75068]: cluster 1685824134.2584674 osd.5 (osd.5) 238 : cluster 3 failed to encode map e289 with expected crc 2023-06-03T20:28:55.509 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:28:55.509 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:28:55.585 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:28:55 smithi164 bash[59462]: cluster 1685824134.2289517 mon.a (mon.0) 1613 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-06-03T20:28:55.585 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:28:55 smithi164 bash[59462]: cluster 1685824134.2289853 mon.a (mon.0) 1614 : cluster 1 Cluster is now healthy 2023-06-03T20:28:55.585 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:28:55 smithi164 bash[59462]: cluster 1685824134.25197 osd.6 (osd.6) 224 : cluster 3 failed to encode map e289 with expected crc 2023-06-03T20:28:55.585 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:28:55 smithi164 bash[59462]: cluster 1685824134.2533023 osd.7 (osd.7) 236 : cluster 3 failed to encode map e289 with expected crc 2023-06-03T20:28:55.586 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:28:55 smithi164 bash[59462]: audit 1685824134.2547626 mon.a (mon.0) 1615 : audit 1 from='client.? 172.21.15.89:0/1709841754' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-06-03T20:28:55.586 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:28:55 smithi164 bash[59462]: cluster 1685824134.2549875 mon.a (mon.0) 1616 : cluster 0 osdmap e289: 8 total, 8 up, 8 in 2023-06-03T20:28:55.586 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:28:55 smithi164 bash[59462]: cluster 1685824134.2584674 osd.5 (osd.5) 238 : cluster 3 failed to encode map e289 with expected crc 2023-06-03T20:28:55.610 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:28:55.638 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:28:55.642 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:28:55.643 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:28:55.744 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:28:55.784 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:28:55.787 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:28:55.788 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:28:55.860 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 69 16 82576 82560 74.7739 50.9375 0.0268179 0.0133679 2023-06-03T20:28:55.890 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:28:55.934 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:28:55.939 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:28:55.940 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:28:56.041 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:28:56.072 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:28:56.075 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:28:56.076 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:28:56.177 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:28:56.207 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:28:56.217 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:28:56.218 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:28:56.232 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 20:28:56 smithi089 bash[71163]: ::ffff:172.21.15.164 - - [03/Jun/2023:20:28:56] "GET /metrics HTTP/1.1" 200 229116 "" "Prometheus/2.33.4" 2023-06-03T20:28:56.318 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:28:56.326 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:28:56 smithi164 bash[59462]: cluster 1685824135.1455746 mgr.y (mgr.34104) 1591 : cluster 0 pgmap v1646: 89 pgs: 1 remapped+peering, 88 active+clean; 4.9 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 32 MiB/s wr, 517 op/s; 0 B/s, 0 objects/s recovering 2023-06-03T20:28:56.352 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:28:56.355 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:28:56.355 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:28:56.456 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:28:56.487 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:28:56.490 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:28:56.490 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:28:56 smithi089 bash[75068]: cluster 1685824135.1455746 mgr.y (mgr.34104) 1591 : cluster 0 pgmap v1646: 89 pgs: 1 remapped+peering, 88 active+clean; 4.9 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 32 MiB/s wr, 517 op/s; 0 B/s, 0 objects/s recovering 2023-06-03T20:28:56.491 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:28:56 smithi089 bash[78452]: cluster 1685824135.1455746 mgr.y (mgr.34104) 1591 : cluster 0 pgmap v1646: 89 pgs: 1 remapped+peering, 88 active+clean; 4.9 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 32 MiB/s wr, 517 op/s; 0 B/s, 0 objects/s recovering 2023-06-03T20:28:56.491 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:28:56.592 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:28:56.621 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:28:56.624 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:28:56.624 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:28:56.725 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:28:56.764 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:28:56.768 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:28:56.769 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:28:56.860 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 70 15 83397 83382 74.4395 51.375 0.0201075 0.0134278 2023-06-03T20:28:56.870 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:28:56.902 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:28:56.905 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:28:56.906 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:28:57.007 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:28:57.047 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:28:57.054 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:28:57.055 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:28:57.156 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:28:57.195 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:28:57.197 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:28:57.198 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:28:57.299 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:28:57.336 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:28:57.339 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:28:57.339 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:28:57.440 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:28:57.470 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:28:57.472 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:28:57.472 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:28:57.573 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:28:57.609 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:28:57.615 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:28:57.616 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:28:57.717 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:28:57.765 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:28:57.767 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:28:57.768 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:28:57.860 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 71 16 84196 84180 74.0935 49.875 0.00894715 0.0134904 2023-06-03T20:28:57.869 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:28:57.898 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:28:57.904 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:28:57.905 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:28:58.006 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:28:58.031 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:28:58.033 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:28:58.034 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:28:58.135 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:28:58.167 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:28:58.171 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:28:58.172 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:28:58.273 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:28:58.318 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:28:58.321 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:28:58.322 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:28:58.335 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:28:58 smithi164 bash[59462]: cluster 1685824137.1461048 mgr.y (mgr.34104) 1592 : cluster 0 pgmap v1647: 89 pgs: 1 active+recovering+remapped, 88 active+clean; 4.9 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 35 MiB/s wr, 553 op/s; 4977/243114 objects misplaced (2.047%); 949 KiB/s, 14 objects/s recovering 2023-06-03T20:28:58.423 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:28:58.455 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:28:58.459 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:28:58.460 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:28:58.529 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:28:58 smithi089 bash[78452]: cluster 1685824137.1461048 mgr.y (mgr.34104) 1592 : cluster 0 pgmap v1647: 89 pgs: 1 active+recovering+remapped, 88 active+clean; 4.9 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 35 MiB/s wr, 553 op/s; 4977/243114 objects misplaced (2.047%); 949 KiB/s, 14 objects/s recovering 2023-06-03T20:28:58.529 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:28:58 smithi089 bash[75068]: cluster 1685824137.1461048 mgr.y (mgr.34104) 1592 : cluster 0 pgmap v1647: 89 pgs: 1 active+recovering+remapped, 88 active+clean; 4.9 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 35 MiB/s wr, 553 op/s; 4977/243114 objects misplaced (2.047%); 949 KiB/s, 14 objects/s recovering 2023-06-03T20:28:58.561 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:28:58.596 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:28:58.599 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:28:58.600 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:28:58.702 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:28:58.756 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:28:58.760 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:28:58.761 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:28:58.860 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 72 16 85003 84987 73.7649 50.4375 0.060353 0.0135499 2023-06-03T20:28:58.863 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:28:58.890 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:28:58.901 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:28:58.901 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:28:59.002 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:28:59.035 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:28:59.043 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:28:59.043 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:28:59.144 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:28:59.190 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:28:59.197 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:28:59.198 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:28:59.299 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:28:59.327 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:28:59.329 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:28:59.329 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:28:59.430 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:28:59.465 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:28:59.471 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:28:59.471 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:28:59.572 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:28:59.610 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:28:59.615 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:28:59.615 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:28:59.716 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:28:59.749 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:28:59.753 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:28:59.754 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:28:59.855 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:28:59.860 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 73 16 85788 85772 73.4264 49.0625 0.00835077 0.013613 2023-06-03T20:28:59.885 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:28:59.892 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:28:59.893 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:28:59.994 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:29:00.026 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:29:00.031 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:29:00.032 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:29:00.133 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:29:00.161 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:29:00.167 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:29:00.168 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:29:00.269 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:29:00.298 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:29:00.300 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:29:00.300 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:29:00.402 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:29:00.432 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:29:00.435 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:29:00.436 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:29:00.529 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:29:00 smithi089 bash[78452]: cluster 1685824139.146794 mgr.y (mgr.34104) 1593 : cluster 0 pgmap v1648: 89 pgs: 1 active+recovering+remapped, 88 active+clean; 5.1 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 49 MiB/s wr, 784 op/s; 4977/251817 objects misplaced (1.976%); 719 KiB/s, 11 objects/s recovering 2023-06-03T20:29:00.529 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:29:00 smithi089 bash[75068]: cluster 1685824139.146794 mgr.y (mgr.34104) 1593 : cluster 0 pgmap v1648: 89 pgs: 1 active+recovering+remapped, 88 active+clean; 5.1 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 49 MiB/s wr, 784 op/s; 4977/251817 objects misplaced (1.976%); 719 KiB/s, 11 objects/s recovering 2023-06-03T20:29:00.537 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:29:00.545 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:29:00 smithi164 bash[59462]: cluster 1685824139.146794 mgr.y (mgr.34104) 1593 : cluster 0 pgmap v1648: 89 pgs: 1 active+recovering+remapped, 88 active+clean; 5.1 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 49 MiB/s wr, 784 op/s; 4977/251817 objects misplaced (1.976%); 719 KiB/s, 11 objects/s recovering 2023-06-03T20:29:00.569 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:29:00.572 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:29:00.572 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:29:00.645 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-06-03T20:29:00.674 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:29:00.682 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph osd set noscrub 2023-06-03T20:29:00.738 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:29:00.740 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:29:00.741 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:29:00.842 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:29:00.867 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 74 16 86543 86527 73.0718 47.1875 0.0112491 0.013677 2023-06-03T20:29:00.927 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:29:00.934 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:29:00.935 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:29:01.036 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:29:01.083 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:29:01.089 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:29:01.090 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:29:01.191 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:29:01.221 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:29:01.224 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:29:01.225 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:29:01.326 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:29:01.358 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:29:01.361 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:29:01.361 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:29:01.462 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:29:01.515 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:29:01.520 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:29:01.521 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:29:01.622 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:29:01.659 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:29:01.664 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:29:01.665 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:29:01.766 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:29:01.801 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:29:01.807 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:29:01.808 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:29:01.860 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 75 16 87307 87291 72.7341 47.75 0.00792003 0.0137414 2023-06-03T20:29:01.909 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:29:01.939 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:29:01.940 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:29:01.941 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:29:02.042 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:29:02.077 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:29:02.084 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:29:02.085 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:29:02.186 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:29:02.223 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:29:02.226 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:29:02.226 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:29:02.327 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:29:02.335 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:29:02 smithi089 bash[78452]: cluster 1685824141.1472144 mgr.y (mgr.34104) 1594 : cluster 0 pgmap v1649: 89 pgs: 1 active+recovering+remapped, 88 active+clean; 5.1 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 38 MiB/s wr, 604 op/s; 4977/251817 objects misplaced (1.976%); 638 KiB/s, 9 objects/s recovering 2023-06-03T20:29:02.336 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:29:02 smithi089 bash[75068]: cluster 1685824141.1472144 mgr.y (mgr.34104) 1594 : cluster 0 pgmap v1649: 89 pgs: 1 active+recovering+remapped, 88 active+clean; 5.1 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 38 MiB/s wr, 604 op/s; 4977/251817 objects misplaced (1.976%); 638 KiB/s, 9 objects/s recovering 2023-06-03T20:29:02.364 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:29:02.370 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:29:02.371 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:29:02.472 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:29:02.508 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:29:02.510 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:29:02.511 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:29:02.585 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:29:02 smithi164 bash[59462]: cluster 1685824141.1472144 mgr.y (mgr.34104) 1594 : cluster 0 pgmap v1649: 89 pgs: 1 active+recovering+remapped, 88 active+clean; 5.1 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 38 MiB/s wr, 604 op/s; 4977/251817 objects misplaced (1.976%); 638 KiB/s, 9 objects/s recovering 2023-06-03T20:29:02.612 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:29:02.655 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:29:02.658 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:29:02.658 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:29:02.759 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:29:02.794 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:29:02.796 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:29:02.798 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:29:02.861 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 76 16 87967 87951 72.3197 41.25 0.0045423 0.0137901 2023-06-03T20:29:02.899 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:29:02.929 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:29:02.931 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:29:02.931 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:29:03.032 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:29:03.065 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:29:03.067 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:29:03.068 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:29:03.169 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:29:03.196 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:29:03.198 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:29:03.198 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:29:03.299 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:29:03.332 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:29:03.337 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:29:03.338 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:29:03.438 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:29:03.480 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:29:03.483 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:29:03.484 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/mon.a/config 2023-06-03T20:29:03.484 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:29:03.585 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:29:03.624 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:29:03.630 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:29:03.631 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:29:03.732 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:29:03.771 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:29:03.778 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:29:03.779 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:29:03.861 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 77 16 88646 88630 71.9316 42.4375 0.0093007 0.0138962 2023-06-03T20:29:03.880 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:29:03.913 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:29:03.915 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:29:03.916 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:29:04.017 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:29:04.060 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:29:04.067 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:29:04.068 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:29:04.153 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:29:04 smithi089 bash[75068]: cluster 1685824143.147947 mgr.y (mgr.34104) 1595 : cluster 0 pgmap v1650: 89 pgs: 1 active+recovering+remapped, 88 active+clean; 5.4 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 47 MiB/s wr, 748 op/s; 4874/263292 objects misplaced (1.851%); 1.2 MiB/s, 19 objects/s recovering 2023-06-03T20:29:04.153 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:29:04 smithi089 bash[78452]: cluster 1685824143.147947 mgr.y (mgr.34104) 1595 : cluster 0 pgmap v1650: 89 pgs: 1 active+recovering+remapped, 88 active+clean; 5.4 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 47 MiB/s wr, 748 op/s; 4874/263292 objects misplaced (1.851%); 1.2 MiB/s, 19 objects/s recovering 2023-06-03T20:29:04.170 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:29:04.206 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:29:04.212 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:29:04.213 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:29:04.314 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:29:04.323 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:29:04 smithi164 bash[59462]: cluster 1685824143.147947 mgr.y (mgr.34104) 1595 : cluster 0 pgmap v1650: 89 pgs: 1 active+recovering+remapped, 88 active+clean; 5.4 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 47 MiB/s wr, 748 op/s; 4874/263292 objects misplaced (1.851%); 1.2 MiB/s, 19 objects/s recovering 2023-06-03T20:29:04.349 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:29:04.360 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:29:04.361 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:29:04.462 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:29:04.495 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:29:04.499 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:29:04.500 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:29:04.600 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:29:04.640 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:29:04.643 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:29:04.644 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:29:04.745 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:29:04.780 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:29:04.785 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:29:04.786 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:29:04.861 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 78 16 89408 89392 71.6199 47.625 0.0105901 0.0139566 2023-06-03T20:29:04.886 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:29:04.925 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:29:04.927 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:29:04.928 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:29:05.029 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:29:05.061 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:29:05.063 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:29:05.064 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:29:05.165 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:29:05.204 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:29:05.206 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:29:05.207 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:29:05.308 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:29:05.344 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:29:05.347 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:29:05.348 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:29:05.449 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:29:05.481 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:29:05.483 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:29:05.484 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:29:05.540 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/mon.a/config 2023-06-03T20:29:05.540 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/mon.a/config 2023-06-03T20:29:05.571 INFO:teuthology.orchestra.run.smithi089.stdout:{ 2023-06-03T20:29:05.572 INFO:teuthology.orchestra.run.smithi089.stdout: "ops": [], 2023-06-03T20:29:05.572 INFO:teuthology.orchestra.run.smithi089.stdout: "complaint_time": 30, 2023-06-03T20:29:05.572 INFO:teuthology.orchestra.run.smithi089.stdout: "num_blocked_ops": 0 2023-06-03T20:29:05.572 INFO:teuthology.orchestra.run.smithi089.stdout:} 2023-06-03T20:29:05.585 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:29:05.613 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:29:05.617 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:29:05.617 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:29:05.718 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:29:05.757 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:29:05.763 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:29:05.764 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:29:05.861 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 79 16 90144 90128 71.2956 46 0.012087 0.0140197 2023-06-03T20:29:05.864 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:29:05.897 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:29:05.904 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:29:05.904 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:29:06.005 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:29:06.040 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:29:06.042 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:29:06.043 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:29:06.144 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:29:06.176 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:29:06.179 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:29:06.180 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:29:06.212 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:29:06 smithi089 bash[78452]: cluster 1685824145.1483555 mgr.y (mgr.34104) 1596 : cluster 0 pgmap v1651: 89 pgs: 1 active+recovering+remapped, 88 active+clean; 5.4 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 43 MiB/s wr, 687 op/s; 4874/263292 objects misplaced (1.851%); 1.1 MiB/s, 17 objects/s recovering 2023-06-03T20:29:06.212 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:29:06 smithi089 bash[75068]: cluster 1685824145.1483555 mgr.y (mgr.34104) 1596 : cluster 0 pgmap v1651: 89 pgs: 1 active+recovering+remapped, 88 active+clean; 5.4 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 43 MiB/s wr, 687 op/s; 4874/263292 objects misplaced (1.851%); 1.1 MiB/s, 17 objects/s recovering 2023-06-03T20:29:06.281 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:29:06.289 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:29:06 smithi164 bash[59462]: cluster 1685824145.1483555 mgr.y (mgr.34104) 1596 : cluster 0 pgmap v1651: 89 pgs: 1 active+recovering+remapped, 88 active+clean; 5.4 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 43 MiB/s wr, 687 op/s; 4874/263292 objects misplaced (1.851%); 1.1 MiB/s, 17 objects/s recovering 2023-06-03T20:29:06.315 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:29:06.318 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:29:06.318 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:29:06.419 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:29:06.455 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:29:06.459 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:29:06.459 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:29:06.529 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 20:29:06 smithi089 bash[71163]: ::ffff:172.21.15.164 - - [03/Jun/2023:20:29:06] "GET /metrics HTTP/1.1" 200 229116 "" "Prometheus/2.33.4" 2023-06-03T20:29:06.560 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:29:06.596 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:29:06.598 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:29:06.599 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:29:06.700 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:29:06.738 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:29:06.742 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:29:06.743 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:29:06.844 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:29:06.861 INFO:tasks.radosbench.radosbench.0.smithi089.stdout:2023-06-03T20:29:06.856936+0000 min lat: 0.00149899 max lat: 2.45302 avg lat: 0.0140761 2023-06-03T20:29:06.861 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-06-03T20:29:06.861 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 80 16 90914 90898 71.0059 48.125 0.00657528 0.0140761 2023-06-03T20:29:06.877 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:29:06.879 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:29:06.880 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:29:06.981 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:29:07.017 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:29:07.023 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:29:07.023 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:29:07.124 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:29:07.168 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:29:07.170 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:29:07.170 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:29:07.271 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:29:07.305 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:29:07.309 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:29:07.310 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:29:07.410 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:29:07.443 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:29:07.447 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:29:07.448 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:29:07.549 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:29:07.577 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:29:07.580 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:29:07.581 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:29:07.682 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:29:07.713 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:29:07.720 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:29:07.721 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:29:07.821 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:29:07.860 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:29:07.861 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 81 16 91652 91636 70.6987 46.125 0.00711152 0.0141384 2023-06-03T20:29:07.863 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:29:07.864 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:29:07.965 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:29:07.995 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:29:08.006 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:29:08.006 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:29:08.107 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:29:08.160 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:29:08.166 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:29:08.167 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:29:08.268 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:29:08.300 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:29:08.302 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:29:08.302 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:29:08.335 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:29:08 smithi164 bash[59462]: cluster 1685824147.1489744 mgr.y (mgr.34104) 1597 : cluster 0 pgmap v1652: 89 pgs: 1 active+recovering+remapped, 88 active+clean; 5.4 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 41 MiB/s wr, 660 op/s; 4874/264603 objects misplaced (1.842%); 1024 KiB/s, 15 objects/s recovering 2023-06-03T20:29:08.335 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:29:08 smithi164 bash[59462]: audit 1685824147.1564407 mon.a (mon.0) 1617 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:29:08.335 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:29:08 smithi164 bash[59462]: audit 1685824147.1633606 mon.c (mon.2) 658 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T20:29:08.403 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:29:08.408 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:29:08 smithi089 bash[78452]: cluster 1685824147.1489744 mgr.y (mgr.34104) 1597 : cluster 0 pgmap v1652: 89 pgs: 1 active+recovering+remapped, 88 active+clean; 5.4 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 41 MiB/s wr, 660 op/s; 4874/264603 objects misplaced (1.842%); 1024 KiB/s, 15 objects/s recovering 2023-06-03T20:29:08.409 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:29:08 smithi089 bash[78452]: audit 1685824147.1564407 mon.a (mon.0) 1617 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:29:08.409 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:29:08 smithi089 bash[78452]: audit 1685824147.1633606 mon.c (mon.2) 658 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T20:29:08.409 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:29:08 smithi089 bash[75068]: cluster 1685824147.1489744 mgr.y (mgr.34104) 1597 : cluster 0 pgmap v1652: 89 pgs: 1 active+recovering+remapped, 88 active+clean; 5.4 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 41 MiB/s wr, 660 op/s; 4874/264603 objects misplaced (1.842%); 1024 KiB/s, 15 objects/s recovering 2023-06-03T20:29:08.409 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:29:08 smithi089 bash[75068]: audit 1685824147.1564407 mon.a (mon.0) 1617 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:29:08.410 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:29:08 smithi089 bash[75068]: audit 1685824147.1633606 mon.c (mon.2) 658 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T20:29:08.441 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:29:08.442 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:29:08.442 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:29:08.543 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:29:08.580 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:29:08.583 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:29:08.584 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:29:08.684 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:29:08.735 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:29:08.739 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:29:08.740 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:29:08.840 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:29:08.861 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 82 16 92334 92318 70.3563 42.625 0.0184561 0.0142057 2023-06-03T20:29:08.882 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:29:08.884 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:29:08.884 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:29:08.985 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:29:09.013 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:29:09.019 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:29:09.019 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:29:09.120 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:29:09.155 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:29:09.160 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:29:09.161 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:29:09.262 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:29:09.298 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:29:09.304 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:29:09.304 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:29:09.405 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:29:09.449 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:29:09.456 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:29:09.456 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:29:09.557 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:29:09.593 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:29:09.603 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:29:09.603 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:29:09.704 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:29:09.750 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:29:09.753 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:29:09.754 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:29:09.854 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:29:09.861 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 83 16 93125 93109 70.1042 49.4375 0.00683356 0.0142582 2023-06-03T20:29:09.886 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:29:09.891 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:29:09.892 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:29:09.993 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:29:10.026 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:29:10.028 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:29:10.029 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:29:10.130 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:29:10.165 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:29:10.170 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:29:10.170 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:29:10.271 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:29:10.292 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:29:10.294 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:29:10.294 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:29:10.298 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:29:10 smithi089 bash[75068]: cluster 1685824149.1497366 mgr.y (mgr.34104) 1598 : cluster 0 pgmap v1653: 89 pgs: 1 active+recovering+remapped, 88 active+clean; 5.6 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 54 MiB/s wr, 865 op/s; 4765/274266 objects misplaced (1.737%); 1.1 MiB/s, 17 objects/s recovering 2023-06-03T20:29:10.299 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:29:10 smithi089 bash[78452]: cluster 1685824149.1497366 mgr.y (mgr.34104) 1598 : cluster 0 pgmap v1653: 89 pgs: 1 active+recovering+remapped, 88 active+clean; 5.6 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 54 MiB/s wr, 865 op/s; 4765/274266 objects misplaced (1.737%); 1.1 MiB/s, 17 objects/s recovering 2023-06-03T20:29:10.396 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:29:10.404 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:29:10 smithi164 bash[59462]: cluster 1685824149.1497366 mgr.y (mgr.34104) 1598 : cluster 0 pgmap v1653: 89 pgs: 1 active+recovering+remapped, 88 active+clean; 5.6 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 54 MiB/s wr, 865 op/s; 4765/274266 objects misplaced (1.737%); 1.1 MiB/s, 17 objects/s recovering 2023-06-03T20:29:10.427 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:29:10.433 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:29:10.433 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:29:10.534 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:29:10.563 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:29:10.565 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:29:10.565 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:29:10.666 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:29:10.705 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:29:10.707 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:29:10.708 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:29:10.809 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:29:10.840 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:29:10.844 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:29:10.845 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:29:10.861 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 84 16 93764 93748 69.7451 39.9375 0.0124277 0.0143315 2023-06-03T20:29:10.946 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:29:10.973 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:29:10.975 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:29:10.976 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:29:11.041 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph daemon osd.0 dump_historic_ops 2023-06-03T20:29:11.078 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:29:11.106 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:29:11.110 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:29:11.111 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:29:11.212 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:29:11.262 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:29:11.267 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:29:11.268 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:29:11.368 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:29:11.398 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:29:11.400 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:29:11.401 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:29:11.502 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:29:11.533 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:29:11.536 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:29:11.538 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:29:11.638 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:29:11.674 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:29:11.677 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:29:11.677 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:29:11.778 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:29:11.815 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:29:11.819 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:29:11.820 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:29:11.861 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 85 16 94464 94448 69.4392 43.75 0.0135567 0.0143959 2023-06-03T20:29:11.921 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:29:11.956 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:29:11.963 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:29:11.963 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:29:12.064 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:29:12.102 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:29:12.104 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:29:12.105 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:29:12.206 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:29:12.212 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:29:12 smithi164 bash[59462]: cluster 1685824151.1501753 mgr.y (mgr.34104) 1599 : cluster 0 pgmap v1654: 89 pgs: 1 active+recovering+remapped, 88 active+clean; 5.6 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 39 MiB/s wr, 623 op/s; 4765/274266 objects misplaced (1.737%); 1.1 MiB/s, 17 objects/s recovering 2023-06-03T20:29:12.243 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:29:12.245 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:29:12.246 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:29:12.347 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:29:12.381 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:29:12.385 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:29:12.386 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:29:12.484 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:29:12 smithi089 bash[78452]: cluster 1685824151.1501753 mgr.y (mgr.34104) 1599 : cluster 0 pgmap v1654: 89 pgs: 1 active+recovering+remapped, 88 active+clean; 5.6 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 39 MiB/s wr, 623 op/s; 4765/274266 objects misplaced (1.737%); 1.1 MiB/s, 17 objects/s recovering 2023-06-03T20:29:12.485 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:29:12 smithi089 bash[75068]: cluster 1685824151.1501753 mgr.y (mgr.34104) 1599 : cluster 0 pgmap v1654: 89 pgs: 1 active+recovering+remapped, 88 active+clean; 5.6 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 39 MiB/s wr, 623 op/s; 4765/274266 objects misplaced (1.737%); 1.1 MiB/s, 17 objects/s recovering 2023-06-03T20:29:12.487 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:29:12.530 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:29:12.532 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:29:12.533 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:29:12.634 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:29:12.670 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:29:12.673 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:29:12.674 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:29:12.775 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:29:12.814 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:29:12.818 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:29:12.819 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:29:12.861 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 86 16 95237 95221 69.1935 48.3125 0.00895342 0.0144463 2023-06-03T20:29:12.919 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:29:12.950 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:29:12.953 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:29:12.954 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:29:13.054 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:29:13.099 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:29:13.105 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:29:13.105 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:29:13.206 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:29:13.251 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:29:13.255 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:29:13.255 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:29:13.356 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:29:13.395 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:29:13.400 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:29:13.401 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:29:13.502 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:29:13.543 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:29:13.547 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:29:13.547 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:29:13.648 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:29:13.682 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:29:13.684 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:29:13.685 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:29:13.786 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:29:13.823 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:29:13.825 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:29:13.825 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:29:13.862 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 87 16 95884 95868 68.8629 40.4375 0.0157069 0.0145153 2023-06-03T20:29:13.926 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:29:13.970 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:29:13.973 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:29:13.974 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:29:14.063 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:29:14 smithi164 bash[59462]: cluster 1685824153.150761 mgr.y (mgr.34104) 1600 : cluster 0 pgmap v1655: 89 pgs: 1 active+recovering+remapped, 88 active+clean; 5.8 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 58 MiB/s wr, 921 op/s; 4639/284991 objects misplaced (1.628%); 1.8 MiB/s, 28 objects/s recovering 2023-06-03T20:29:14.075 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:29:14.109 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:29:14.113 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:29:14.114 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:29:14.215 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:29:14.255 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:29:14.258 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:29:14.259 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:29:14.360 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:29:14.368 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:29:14 smithi089 bash[78452]: cluster 1685824153.150761 mgr.y (mgr.34104) 1600 : cluster 0 pgmap v1655: 89 pgs: 1 active+recovering+remapped, 88 active+clean; 5.8 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 58 MiB/s wr, 921 op/s; 4639/284991 objects misplaced (1.628%); 1.8 MiB/s, 28 objects/s recovering 2023-06-03T20:29:14.368 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:29:14 smithi089 bash[75068]: cluster 1685824153.150761 mgr.y (mgr.34104) 1600 : cluster 0 pgmap v1655: 89 pgs: 1 active+recovering+remapped, 88 active+clean; 5.8 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 58 MiB/s wr, 921 op/s; 4639/284991 objects misplaced (1.628%); 1.8 MiB/s, 28 objects/s recovering 2023-06-03T20:29:14.389 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:29:14.394 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:29:14.395 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:29:14.496 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:29:14.529 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:29:14.533 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:29:14.533 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:29:14.634 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:29:14.683 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:29:14.689 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:29:14.690 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:29:14.791 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:29:14.830 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:29:14.832 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:29:14.832 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:29:14.862 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 88 16 96629 96613 68.6095 46.5625 0.0236067 0.0145692 2023-06-03T20:29:14.933 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:29:14.963 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:29:14.965 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:29:14.965 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:29:15.066 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:29:15.098 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:29:15.103 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:29:15.104 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:29:15.205 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:29:15.236 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:29:15.239 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:29:15.239 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:29:15.340 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:29:15.373 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:29:15.379 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:29:15.380 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:29:15.480 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:29:15.513 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:29:15.517 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:29:15.517 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:29:15.618 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:29:15.660 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:29:15.664 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:29:15.665 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:29:15.766 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:29:15.825 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:29:15.827 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:29:15.828 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:29:15.866 INFO:tasks.radosbench.radosbench.0.smithi089.stdout: 89 16 97320 97304 68.3217 43.1875 0.0174409 0.0146297 2023-06-03T20:29:15.929 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:29:15.963 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:29:15.968 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:29:15.969 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:29:16.070 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:29:16.105 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:29:16.111 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:29:16.112 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:29:16.212 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:29:16.264 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:29:16.265 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:29:16.265 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:29:16.279 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 20:29:16 smithi089 bash[71163]: ::ffff:172.21.15.164 - - [03/Jun/2023:20:29:16] "GET /metrics HTTP/1.1" 200 229146 "" "Prometheus/2.33.4" 2023-06-03T20:29:16.279 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:29:16 smithi089 bash[78452]: cluster 1685824155.1512122 mgr.y (mgr.34104) 1601 : cluster 0 pgmap v1656: 89 pgs: 1 active+recovering+remapped, 88 active+clean; 5.8 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 38 MiB/s wr, 602 op/s; 4639/284991 objects misplaced (1.628%); 1.2 MiB/s, 19 objects/s recovering 2023-06-03T20:29:16.279 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:29:16 smithi089 bash[75068]: cluster 1685824155.1512122 mgr.y (mgr.34104) 1601 : cluster 0 pgmap v1656: 89 pgs: 1 active+recovering+remapped, 88 active+clean; 5.8 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 38 MiB/s wr, 602 op/s; 4639/284991 objects misplaced (1.628%); 1.2 MiB/s, 19 objects/s recovering 2023-06-03T20:29:16.335 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:29:16 smithi164 bash[59462]: cluster 1685824155.1512122 mgr.y (mgr.34104) 1601 : cluster 0 pgmap v1656: 89 pgs: 1 active+recovering+remapped, 88 active+clean; 5.8 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 38 MiB/s wr, 602 op/s; 4639/284991 objects misplaced (1.628%); 1.2 MiB/s, 19 objects/s recovering 2023-06-03T20:29:16.366 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:29:16.407 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:29:16.414 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:29:16.415 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:29:16.516 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:29:16.546 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:29:16.552 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:29:16.553 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:29:16.653 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:29:16.690 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:29:16.693 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:29:16.694 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:29:16.795 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:29:16.824 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:29:16.826 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:29:16.826 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:29:16.865 INFO:tasks.radosbench.radosbench.0.smithi089.stdout:Total time run: 90.008 2023-06-03T20:29:16.865 INFO:tasks.radosbench.radosbench.0.smithi089.stdout:Total writes made: 98013 2023-06-03T20:29:16.865 INFO:tasks.radosbench.radosbench.0.smithi089.stdout:Write size: 65536 2023-06-03T20:29:16.865 INFO:tasks.radosbench.radosbench.0.smithi089.stdout:Object size: 65536 2023-06-03T20:29:16.865 INFO:tasks.radosbench.radosbench.0.smithi089.stdout:Bandwidth (MB/sec): 68.0585 2023-06-03T20:29:16.866 INFO:tasks.radosbench.radosbench.0.smithi089.stdout:Stddev Bandwidth: 30.5301 2023-06-03T20:29:16.866 INFO:tasks.radosbench.radosbench.0.smithi089.stdout:Max bandwidth (MB/sec): 164.562 2023-06-03T20:29:16.866 INFO:tasks.radosbench.radosbench.0.smithi089.stdout:Min bandwidth (MB/sec): 0 2023-06-03T20:29:16.866 INFO:tasks.radosbench.radosbench.0.smithi089.stdout:Average IOPS: 1088 2023-06-03T20:29:16.866 INFO:tasks.radosbench.radosbench.0.smithi089.stdout:Stddev IOPS: 488.481 2023-06-03T20:29:16.866 INFO:tasks.radosbench.radosbench.0.smithi089.stdout:Max IOPS: 2633 2023-06-03T20:29:16.867 INFO:tasks.radosbench.radosbench.0.smithi089.stdout:Min IOPS: 0 2023-06-03T20:29:16.867 INFO:tasks.radosbench.radosbench.0.smithi089.stdout:Average Latency(s): 0.0146888 2023-06-03T20:29:16.867 INFO:tasks.radosbench.radosbench.0.smithi089.stdout:Stddev Latency(s): 0.0336149 2023-06-03T20:29:16.867 INFO:tasks.radosbench.radosbench.0.smithi089.stdout:Max latency(s): 2.45302 2023-06-03T20:29:16.867 INFO:tasks.radosbench.radosbench.0.smithi089.stdout:Min latency(s): 0.00149899 2023-06-03T20:29:16.893 INFO:tasks.radosbench.radosbench.0.smithi089.stdout:Cleaning up (deleting benchmark objects) 2023-06-03T20:29:16.927 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:29:16.963 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:29:16.965 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:29:16.965 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:29:17.068 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:29:17.097 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:29:17.103 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:29:17.104 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:29:17.205 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:29:17.245 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:29:17.247 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:29:17.248 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:29:17.349 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:29:17.382 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:29:17.388 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:29:17.389 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:29:17.490 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:29:17.524 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:29:17.529 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:29:17.530 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:29:17.631 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:29:17.660 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:29:17.663 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:29:17.663 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:29:17.764 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:29:17.802 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:29:17.807 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:29:17.808 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:29:17.908 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:29:17.939 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:29:17.942 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:29:17.942 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:29:18.043 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:29:18.075 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:29:18.078 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:29:18.079 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:29:18.180 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:29:18.216 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:29:18.220 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:29:18.220 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:29:18.295 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:29:18 smithi089 bash[78452]: cluster 1685824157.1517358 mgr.y (mgr.34104) 1602 : cluster 0 pgmap v1657: 89 pgs: 1 active+recovering+remapped, 88 active+clean; 5.8 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 40 MiB/s wr, 643 op/s; 4639/286464 objects misplaced (1.619%); 1.2 MiB/s, 19 objects/s recovering 2023-06-03T20:29:18.296 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:29:18 smithi089 bash[75068]: cluster 1685824157.1517358 mgr.y (mgr.34104) 1602 : cluster 0 pgmap v1657: 89 pgs: 1 active+recovering+remapped, 88 active+clean; 5.8 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 40 MiB/s wr, 643 op/s; 4639/286464 objects misplaced (1.619%); 1.2 MiB/s, 19 objects/s recovering 2023-06-03T20:29:18.321 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:29:18.359 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:29:18.363 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:29:18.363 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:29:18.464 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:29:18.497 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:29:18.502 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:29:18.502 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:29:18.585 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:29:18 smithi164 bash[59462]: cluster 1685824157.1517358 mgr.y (mgr.34104) 1602 : cluster 0 pgmap v1657: 89 pgs: 1 active+recovering+remapped, 88 active+clean; 5.8 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 40 MiB/s wr, 643 op/s; 4639/286464 objects misplaced (1.619%); 1.2 MiB/s, 19 objects/s recovering 2023-06-03T20:29:18.603 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:29:18.646 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:29:18.651 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:29:18.652 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:29:18.753 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:29:18.780 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:29:18.782 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:29:18.783 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:29:18.883 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:29:18.921 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:29:18.926 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:29:18.927 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:29:19.028 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:29:19.057 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:29:19.059 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:29:19.060 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:29:19.161 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:29:19.205 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:29:19.208 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:29:19.209 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:29:19.309 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:29:19.346 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:29:19.349 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:29:19.349 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:29:19.450 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:29:19.479 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:29:19.485 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:29:19.486 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:29:19.586 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:29:19.624 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:29:19.626 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:29:19.627 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:29:19.728 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:29:19.762 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:29:19.767 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:29:19.767 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:29:19.868 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:29:19.900 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:29:19.904 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:29:19.905 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:29:20.006 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:29:20.041 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:29:20.047 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:29:20.048 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:29:20.149 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:29:20.195 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:29:20.200 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:29:20.200 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:29:20.301 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:29:20.333 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:29:20.335 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:29:20 smithi164 bash[59462]: cluster 1685824159.1524458 mgr.y (mgr.34104) 1603 : cluster 0 pgmap v1658: 89 pgs: 1 active+recovering+remapped, 88 active+clean; 5.9 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 85 B/s rd, 51 MiB/s wr, 860 op/s; 4536/292332 objects misplaced (1.552%); 1.8 MiB/s, 28 objects/s recovering 2023-06-03T20:29:20.335 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:29:20.336 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:29:20.437 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:29:20.479 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:29:20.485 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:29:20.485 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:29:20.549 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:29:20 smithi089 bash[75068]: cluster 1685824159.1524458 mgr.y (mgr.34104) 1603 : cluster 0 pgmap v1658: 89 pgs: 1 active+recovering+remapped, 88 active+clean; 5.9 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 85 B/s rd, 51 MiB/s wr, 860 op/s; 4536/292332 objects misplaced (1.552%); 1.8 MiB/s, 28 objects/s recovering 2023-06-03T20:29:20.550 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:29:20 smithi089 bash[78452]: cluster 1685824159.1524458 mgr.y (mgr.34104) 1603 : cluster 0 pgmap v1658: 89 pgs: 1 active+recovering+remapped, 88 active+clean; 5.9 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 85 B/s rd, 51 MiB/s wr, 860 op/s; 4536/292332 objects misplaced (1.552%); 1.8 MiB/s, 28 objects/s recovering 2023-06-03T20:29:20.586 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:29:20.640 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:29:20.645 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:29:20.646 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:29:20.747 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:29:20.792 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:29:20.797 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:29:20.798 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:29:20.899 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:29:20.933 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:29:20.936 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:29:20.936 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:29:21.037 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:29:21.064 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:29:21.067 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:29:21.067 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:29:21.168 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:29:21.214 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:29:21.219 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:29:21.219 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:29:21.321 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:29:21.359 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:29:21.361 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:29:21.362 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:29:21.462 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:29:21.496 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:29:21.503 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:29:21.503 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:29:21.604 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:29:21.654 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:29:21.657 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:29:21.658 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:29:21.758 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:29:21.806 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:29:21.809 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:29:21.809 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:29:21.910 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:29:21.956 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:29:21.959 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:29:21.960 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:29:22.061 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/mon.a/config 2023-06-03T20:29:22.061 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:29:22.112 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:29:22.117 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:29:22.117 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:29:22.218 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:29:22.230 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:29:22 smithi089 bash[78452]: cluster 1685824161.1554327 mgr.y (mgr.34104) 1604 : cluster 0 pgmap v1659: 89 pgs: 1 active+recovering+remapped, 88 active+clean; 5.9 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 85 B/s rd, 34 MiB/s wr, 592 op/s; 4536/292332 objects misplaced (1.552%); 1.2 MiB/s, 19 objects/s recovering 2023-06-03T20:29:22.231 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:29:22 smithi089 bash[78452]: audit 1685824162.1508565 mon.c (mon.2) 659 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T20:29:22.231 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:29:22 smithi089 bash[75068]: cluster 1685824161.1554327 mgr.y (mgr.34104) 1604 : cluster 0 pgmap v1659: 89 pgs: 1 active+recovering+remapped, 88 active+clean; 5.9 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 85 B/s rd, 34 MiB/s wr, 592 op/s; 4536/292332 objects misplaced (1.552%); 1.2 MiB/s, 19 objects/s recovering 2023-06-03T20:29:22.231 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:29:22 smithi089 bash[75068]: audit 1685824162.1508565 mon.c (mon.2) 659 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T20:29:22.244 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:29:22.249 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:29:22.249 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:29:22.350 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:29:22.387 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:29:22.391 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:29:22.392 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:29:22.493 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:29:22.535 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:29:22.538 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:29:22.539 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:29:22.585 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:29:22 smithi164 bash[59462]: cluster 1685824161.1554327 mgr.y (mgr.34104) 1604 : cluster 0 pgmap v1659: 89 pgs: 1 active+recovering+remapped, 88 active+clean; 5.9 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 85 B/s rd, 34 MiB/s wr, 592 op/s; 4536/292332 objects misplaced (1.552%); 1.2 MiB/s, 19 objects/s recovering 2023-06-03T20:29:22.585 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:29:22 smithi164 bash[59462]: audit 1685824162.1508565 mon.c (mon.2) 659 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T20:29:22.640 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:29:22.667 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:29:22.671 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:29:22.672 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:29:22.773 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:29:22.811 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:29:22.814 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:29:22.814 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:29:22.915 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:29:22.953 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:29:22.959 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:29:22.959 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:29:23.060 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:29:23.101 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:29:23.103 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:29:23.104 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:29:23.205 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:29:23.237 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:29:23.240 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:29:23.241 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:29:23.341 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:29:23.396 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:29:23.399 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:29:23.400 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:29:23.475 INFO:teuthology.orchestra.run.smithi089.stdout: 2023-06-03T20:29:23.477 INFO:teuthology.orchestra.run.smithi089.stderr:dumped all 2023-06-03T20:29:23.500 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:29:23.541 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:29:23.547 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:29:23.548 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:29:23.648 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:29:23.678 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:29:23.684 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:29:23.685 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:29:23.786 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:29:23.818 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:29:23.822 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:29:23.823 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:29:23.924 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:29:23.956 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:29:23.958 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:29:23.959 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:29:24.060 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:29:24.063 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:29:24 smithi089 bash[75068]: cluster 1685824163.156245 mgr.y (mgr.34104) 1605 : cluster 0 pgmap v1660: 89 pgs: 1 active+recovering+remapped, 88 active+clean; 5.7 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 85 B/s rd, 34 MiB/s wr, 887 op/s; 4428/281985 objects misplaced (1.570%); 1.8 MiB/s, 28 objects/s recovering 2023-06-03T20:29:24.096 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:29:24.101 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:29:24.102 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:29:24.202 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:29:24.231 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:29:24.234 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:29:24.234 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:29:24.335 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:29:24 smithi164 bash[59462]: cluster 1685824163.156245 mgr.y (mgr.34104) 1605 : cluster 0 pgmap v1660: 89 pgs: 1 active+recovering+remapped, 88 active+clean; 5.7 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 85 B/s rd, 34 MiB/s wr, 887 op/s; 4428/281985 objects misplaced (1.570%); 1.8 MiB/s, 28 objects/s recovering 2023-06-03T20:29:24.336 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:29:24.385 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:29:24.387 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:29:24.388 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:29:24.490 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:29:24.521 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:29:24.525 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:29:24.526 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:29:24.529 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:29:24 smithi089 bash[78452]: cluster 1685824163.156245 mgr.y (mgr.34104) 1605 : cluster 0 pgmap v1660: 89 pgs: 1 active+recovering+remapped, 88 active+clean; 5.7 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 85 B/s rd, 34 MiB/s wr, 887 op/s; 4428/281985 objects misplaced (1.570%); 1.8 MiB/s, 28 objects/s recovering 2023-06-03T20:29:24.627 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:29:24.674 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:29:24.679 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:29:24.680 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:29:24.781 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:29:24.828 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:29:24.833 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:29:24.834 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:29:24.935 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:29:24.972 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:29:24.974 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:29:24.975 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:29:25.066 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:29:25 smithi089 bash[75068]: audit 1685824163.4534438 mgr.y (mgr.34104) 1606 : audit 0 from='client.36831 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-06-03T20:29:25.066 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:29:25 smithi089 bash[78452]: audit 1685824163.4534438 mgr.y (mgr.34104) 1606 : audit 0 from='client.36831 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-06-03T20:29:25.075 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:29:25.119 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:29:25.121 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:29:25.122 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:29:25.223 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:29:25.248 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:29:25.259 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:29:25.260 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:29:25.335 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:29:25 smithi164 bash[59462]: audit 1685824163.4534438 mgr.y (mgr.34104) 1606 : audit 0 from='client.36831 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-06-03T20:29:25.361 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:29:25.400 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:29:25.403 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:29:25.403 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:29:25.504 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:29:25.541 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:29:25.544 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:29:25.546 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:29:25.646 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:29:25.676 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:29:25.679 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:29:25.679 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:29:25.780 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:29:25.831 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:29:25.835 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:29:25.836 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:29:25.937 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:29:25.975 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:29:25.979 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:29:25.980 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:29:26.071 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:29:26 smithi164 bash[59462]: cluster 1685824165.1566484 mgr.y (mgr.34104) 1607 : cluster 0 pgmap v1661: 89 pgs: 1 active+recovering+remapped, 88 active+clean; 5.7 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 85 B/s rd, 16 MiB/s wr, 590 op/s; 4428/281985 objects misplaced (1.570%); 1.1 MiB/s, 17 objects/s recovering 2023-06-03T20:29:26.081 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:29:26.112 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:29:26.113 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:29:26.114 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:29:26.215 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:29:26.225 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:29:26 smithi089 bash[75068]: cluster 1685824165.1566484 mgr.y (mgr.34104) 1607 : cluster 0 pgmap v1661: 89 pgs: 1 active+recovering+remapped, 88 active+clean; 5.7 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 85 B/s rd, 16 MiB/s wr, 590 op/s; 4428/281985 objects misplaced (1.570%); 1.1 MiB/s, 17 objects/s recovering 2023-06-03T20:29:26.225 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:29:26 smithi089 bash[78452]: cluster 1685824165.1566484 mgr.y (mgr.34104) 1607 : cluster 0 pgmap v1661: 89 pgs: 1 active+recovering+remapped, 88 active+clean; 5.7 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 85 B/s rd, 16 MiB/s wr, 590 op/s; 4428/281985 objects misplaced (1.570%); 1.1 MiB/s, 17 objects/s recovering 2023-06-03T20:29:26.251 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:29:26.257 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:29:26.258 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:29:26.359 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:29:26.401 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:29:26.406 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:29:26.407 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:29:26.508 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:29:26.529 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 20:29:26 smithi089 bash[71163]: ::ffff:172.21.15.164 - - [03/Jun/2023:20:29:26] "GET /metrics HTTP/1.1" 200 229147 "" "Prometheus/2.33.4" 2023-06-03T20:29:26.545 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:29:26.549 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:29:26.549 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:29:26.650 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:29:26.690 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:29:26.700 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:29:26.701 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:29:26.802 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:29:26.831 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:29:26.834 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:29:26.834 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:29:26.935 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:29:26.969 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:29:26.974 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:29:26.974 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:29:27.075 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:29:27.111 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:29:27.112 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:29:27.113 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:29:27.214 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:29:27.241 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:29:27.243 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:29:27.244 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:29:27.345 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:29:27.376 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:29:27.378 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:29:27.378 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:29:27.479 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:29:27.512 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:29:27.517 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:29:27.518 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:29:27.619 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:29:27.653 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:29:27.655 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:29:27.656 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:29:27.757 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:29:27.799 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:29:27.802 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:29:27.802 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:29:27.903 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:29:27.945 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:29:27.947 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:29:27.948 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:29:28.049 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:29:28.077 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:29:28.082 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:29:28.083 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:29:28.184 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:29:28.245 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:29:28.249 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:29:28.250 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:29:28.335 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:29:28 smithi164 bash[59462]: cluster 1685824167.1572468 mgr.y (mgr.34104) 1608 : cluster 0 pgmap v1662: 89 pgs: 1 active+recovering+remapped, 88 active+clean; 5.7 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 85 B/s rd, 16 MiB/s wr, 627 op/s; 4428/280629 objects misplaced (1.578%); 1.1 MiB/s, 17 objects/s recovering 2023-06-03T20:29:28.351 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:29:28.386 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:29:28.390 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:29:28.390 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:29:28.491 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:29:28.522 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:29:28.524 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:29:28.524 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:29:28.529 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:29:28 smithi089 bash[78452]: cluster 1685824167.1572468 mgr.y (mgr.34104) 1608 : cluster 0 pgmap v1662: 89 pgs: 1 active+recovering+remapped, 88 active+clean; 5.7 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 85 B/s rd, 16 MiB/s wr, 627 op/s; 4428/280629 objects misplaced (1.578%); 1.1 MiB/s, 17 objects/s recovering 2023-06-03T20:29:28.529 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:29:28 smithi089 bash[75068]: cluster 1685824167.1572468 mgr.y (mgr.34104) 1608 : cluster 0 pgmap v1662: 89 pgs: 1 active+recovering+remapped, 88 active+clean; 5.7 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 85 B/s rd, 16 MiB/s wr, 627 op/s; 4428/280629 objects misplaced (1.578%); 1.1 MiB/s, 17 objects/s recovering 2023-06-03T20:29:28.595 INFO:teuthology.orchestra.run.smithi089.stdout:{"pg_ready":true,"pg_map":{"version":1660,"stamp":"2023-06-03T20:29:23.155769+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":6159013798,"num_objects":93995,"num_object_clones":0,"num_object_copies":281985,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":4428,"num_objects_unfound":0,"num_objects_dirty":93995,"num_whiteouts":0,"num_read":309,"num_read_kb":513,"num_write":102318,"num_write_kb":6275279,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":667,"num_bytes_recovered":43587928,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":102252,"ondisk_log_size":102252,"up":267,"acting":267,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":268,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":23323680,"kb_used_data":18217632,"kb_used_omap":30,"kb_used_meta":5105633,"kb_avail":726637536,"statfs":{"total":767960285184,"available":744076836864,"internally_reserved":0,"allocated":18654855168,"data_stored":18647180573,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":31502,"internal_metadata":5228168434},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[2,4,8,5,3,1],"upper_bound":64},"perf_stat":{"commit_latency_ms":5,"apply_latency_ms":5,"commit_latency_ns":5000000,"apply_latency_ns":5000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":168558616,"num_objects":2573,"num_object_clones":0,"num_object_copies":7719,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":-337,"num_objects_unfound":0,"num_objects_dirty":2573,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":10659,"num_write_kb":423361,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":337,"num_bytes_recovered":22085632,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.005833"},"pg_stats":[{"pgid":"3.1a","version":"60'1","reported_seq":215,"reported_epoch":288,"state":"active+clean","last_fresh":"2023-06-03T20:28:52.252247+0000","last_change":"2023-06-03T20:23:28.242261+0000","last_active":"2023-06-03T20:28:52.252247+0000","last_peered":"2023-06-03T20:28:52.252247+0000","last_clean":"2023-06-03T20:28:52.252247+0000","last_became_active":"2023-06-03T20:23:28.240045+0000","last_became_peered":"2023-06-03T20:23:28.240045+0000","last_unstale":"2023-06-03T20:28:52.252247+0000","last_undegraded":"2023-06-03T20:28:52.252247+0000","last_fullsized":"2023-06-03T20:28:52.252247+0000","mapping_epoch":251,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":252,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T05:54:33.215385+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":76,"reported_epoch":288,"state":"active+clean","last_fresh":"2023-06-03T20:28:52.223305+0000","last_change":"2023-06-03T20:23:28.283021+0000","last_active":"2023-06-03T20:28:52.223305+0000","last_peered":"2023-06-03T20:28:52.223305+0000","last_clean":"2023-06-03T20:28:52.223305+0000","last_became_active":"2023-06-03T20:23:28.282428+0000","last_became_peered":"2023-06-03T20:23:28.282428+0000","last_unstale":"2023-06-03T20:28:52.223305+0000","last_undegraded":"2023-06-03T20:28:52.223305+0000","last_fullsized":"2023-06-03T20:28:52.223305+0000","mapping_epoch":251,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":252,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T05:01:33.115332+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":76,"reported_epoch":288,"state":"active+clean","last_fresh":"2023-06-03T20:28:52.223632+0000","last_change":"2023-06-03T20:23:28.253821+0000","last_active":"2023-06-03T20:28:52.223632+0000","last_peered":"2023-06-03T20:28:52.223632+0000","last_clean":"2023-06-03T20:28:52.223632+0000","last_became_active":"2023-06-03T20:23:28.253448+0000","last_became_peered":"2023-06-03T20:23:28.253448+0000","last_unstale":"2023-06-03T20:28:52.223632+0000","last_undegraded":"2023-06-03T20:28:52.223632+0000","last_fullsized":"2023-06-03T20:28:52.223632+0000","mapping_epoch":251,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":252,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T23:40:02.264778+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":259,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-06-03T20:28:54.248449+0000","last_change":"2023-06-03T19:56:54.507653+0000","last_active":"2023-06-03T20:28:54.248449+0000","last_peered":"2023-06-03T20:28:54.248449+0000","last_clean":"2023-06-03T20:28:54.248449+0000","last_became_active":"2023-06-03T19:56:54.507350+0000","last_became_peered":"2023-06-03T19:56:54.507350+0000","last_unstale":"2023-06-03T20:28:54.248449+0000","last_undegraded":"2023-06-03T20:28:54.248449+0000","last_fullsized":"2023-06-03T20:28:54.248449+0000","mapping_epoch":119,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T22:33:20.065844+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":299,"reported_epoch":288,"state":"active+clean","last_fresh":"2023-06-03T20:28:52.671516+0000","last_change":"2023-06-03T20:21:01.256894+0000","last_active":"2023-06-03T20:28:52.671516+0000","last_peered":"2023-06-03T20:28:52.671516+0000","last_clean":"2023-06-03T20:28:52.671516+0000","last_became_active":"2023-06-03T20:21:01.255903+0000","last_became_peered":"2023-06-03T20:21:01.255903+0000","last_unstale":"2023-06-03T20:28:52.671516+0000","last_undegraded":"2023-06-03T20:28:52.671516+0000","last_fullsized":"2023-06-03T20:28:52.671516+0000","mapping_epoch":235,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":236,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T20:35:27.810860+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":53,"reported_epoch":288,"state":"active+clean","last_fresh":"2023-06-03T20:28:52.502278+0000","last_change":"2023-06-03T20:28:24.269355+0000","last_active":"2023-06-03T20:28:52.502278+0000","last_peered":"2023-06-03T20:28:52.502278+0000","last_clean":"2023-06-03T20:28:52.502278+0000","last_became_active":"2023-06-03T20:28:24.268859+0000","last_became_peered":"2023-06-03T20:28:24.268859+0000","last_unstale":"2023-06-03T20:28:52.502278+0000","last_undegraded":"2023-06-03T20:28:52.502278+0000","last_fullsized":"2023-06-03T20:28:52.502278+0000","mapping_epoch":284,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":285,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,2],"acting":[6,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":254,"reported_epoch":288,"state":"active+clean","last_fresh":"2023-06-03T20:28:52.223968+0000","last_change":"2023-06-03T19:56:54.509490+0000","last_active":"2023-06-03T20:28:52.223968+0000","last_peered":"2023-06-03T20:28:52.223968+0000","last_clean":"2023-06-03T20:28:52.223968+0000","last_became_active":"2023-06-03T19:56:54.509145+0000","last_became_peered":"2023-06-03T19:56:54.509145+0000","last_unstale":"2023-06-03T20:28:52.223968+0000","last_undegraded":"2023-06-03T20:28:52.223968+0000","last_fullsized":"2023-06-03T20:28:52.223968+0000","mapping_epoch":119,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T07:12:03.964110+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":319,"reported_epoch":288,"state":"active+clean","last_fresh":"2023-06-03T20:28:52.671505+0000","last_change":"2023-06-03T20:28:51.233277+0000","last_active":"2023-06-03T20:28:52.671505+0000","last_peered":"2023-06-03T20:28:52.671505+0000","last_clean":"2023-06-03T20:28:52.671505+0000","last_became_active":"2023-06-03T20:28:51.232919+0000","last_became_peered":"2023-06-03T20:28:51.232919+0000","last_unstale":"2023-06-03T20:28:52.671505+0000","last_undegraded":"2023-06-03T20:28:52.671505+0000","last_fullsized":"2023-06-03T20:28:52.671505+0000","mapping_epoch":286,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":287,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T02:06:02.736415+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":221,"reported_epoch":288,"state":"active+clean","last_fresh":"2023-06-03T20:28:52.664551+0000","last_change":"2023-06-03T20:21:01.258441+0000","last_active":"2023-06-03T20:28:52.664551+0000","last_peered":"2023-06-03T20:28:52.664551+0000","last_clean":"2023-06-03T20:28:52.664551+0000","last_became_active":"2023-06-03T20:21:01.257779+0000","last_became_peered":"2023-06-03T20:21:01.257779+0000","last_unstale":"2023-06-03T20:28:52.664551+0000","last_undegraded":"2023-06-03T20:28:52.664551+0000","last_fullsized":"2023-06-03T20:28:52.664551+0000","mapping_epoch":235,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":236,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,1],"acting":[7,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":264,"reported_epoch":288,"state":"active+clean","last_fresh":"2023-06-03T20:28:52.671923+0000","last_change":"2023-06-03T20:24:02.267331+0000","last_active":"2023-06-03T20:28:52.671923+0000","last_peered":"2023-06-03T20:28:52.671923+0000","last_clean":"2023-06-03T20:28:52.671923+0000","last_became_active":"2023-06-03T20:24:02.266997+0000","last_became_peered":"2023-06-03T20:24:02.266997+0000","last_unstale":"2023-06-03T20:28:52.671923+0000","last_undegraded":"2023-06-03T20:28:52.671923+0000","last_fullsized":"2023-06-03T20:28:52.671923+0000","mapping_epoch":254,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":255,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T22:09:18.993656+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"60'3","reported_seq":286,"reported_epoch":288,"state":"active+clean","last_fresh":"2023-06-03T20:28:52.664569+0000","last_change":"2023-06-03T20:23:27.261246+0000","last_active":"2023-06-03T20:28:52.664569+0000","last_peered":"2023-06-03T20:28:52.664569+0000","last_clean":"2023-06-03T20:28:52.664569+0000","last_became_active":"2023-06-03T20:23:27.259201+0000","last_became_peered":"2023-06-03T20:23:27.259201+0000","last_unstale":"2023-06-03T20:28:52.664569+0000","last_undegraded":"2023-06-03T20:28:52.664569+0000","last_fullsized":"2023-06-03T20:28:52.664569+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":76,"reported_epoch":288,"state":"active+clean","last_fresh":"2023-06-03T20:28:52.223806+0000","last_change":"2023-06-03T20:28:51.235184+0000","last_active":"2023-06-03T20:28:52.223806+0000","last_peered":"2023-06-03T20:28:52.223806+0000","last_clean":"2023-06-03T20:28:52.223806+0000","last_became_active":"2023-06-03T20:28:51.234830+0000","last_became_peered":"2023-06-03T20:28:51.234830+0000","last_unstale":"2023-06-03T20:28:52.223806+0000","last_undegraded":"2023-06-03T20:28:52.223806+0000","last_fullsized":"2023-06-03T20:28:52.223806+0000","mapping_epoch":286,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":287,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T04:40:02.479523+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"77'23","reported_seq":222,"reported_epoch":288,"state":"active+clean","last_fresh":"2023-06-03T20:28:52.252755+0000","last_change":"2023-06-03T20:24:02.280274+0000","last_active":"2023-06-03T20:28:52.252755+0000","last_peered":"2023-06-03T20:28:52.252755+0000","last_clean":"2023-06-03T20:28:52.252755+0000","last_became_active":"2023-06-03T20:24:02.264914+0000","last_became_peered":"2023-06-03T20:24:02.264914+0000","last_unstale":"2023-06-03T20:28:52.252755+0000","last_undegraded":"2023-06-03T20:28:52.252755+0000","last_fullsized":"2023-06-03T20:28:52.252755+0000","mapping_epoch":254,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":255,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":23,"log_dups_size":0,"ondisk_log_size":23,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T07:00:35.888220+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4716,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":23,"num_write_kb":20,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4716,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":198,"reported_epoch":288,"state":"active+clean","last_fresh":"2023-06-03T20:28:52.252214+0000","last_change":"2023-06-03T20:24:02.254850+0000","last_active":"2023-06-03T20:28:52.252214+0000","last_peered":"2023-06-03T20:28:52.252214+0000","last_clean":"2023-06-03T20:28:52.252214+0000","last_became_active":"2023-06-03T20:24:02.254409+0000","last_became_peered":"2023-06-03T20:24:02.254409+0000","last_unstale":"2023-06-03T20:28:52.252214+0000","last_undegraded":"2023-06-03T20:28:52.252214+0000","last_fullsized":"2023-06-03T20:28:52.252214+0000","mapping_epoch":254,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":255,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T19:49:53.393121+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"60'2","reported_seq":288,"reported_epoch":288,"state":"active+clean","last_fresh":"2023-06-03T20:28:52.664132+0000","last_change":"2023-06-03T20:23:27.258749+0000","last_active":"2023-06-03T20:28:52.664132+0000","last_peered":"2023-06-03T20:28:52.664132+0000","last_clean":"2023-06-03T20:28:52.664132+0000","last_became_active":"2023-06-03T20:23:27.256899+0000","last_became_peered":"2023-06-03T20:23:27.256899+0000","last_unstale":"2023-06-03T20:28:52.664132+0000","last_undegraded":"2023-06-03T20:28:52.664132+0000","last_fullsized":"2023-06-03T20:28:52.664132+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":283,"reported_epoch":288,"state":"active+clean","last_fresh":"2023-06-03T20:28:52.672993+0000","last_change":"2023-06-03T20:21:01.256612+0000","last_active":"2023-06-03T20:28:52.672993+0000","last_peered":"2023-06-03T20:28:52.672993+0000","last_clean":"2023-06-03T20:28:52.672993+0000","last_became_active":"2023-06-03T20:21:01.256243+0000","last_became_peered":"2023-06-03T20:21:01.256243+0000","last_unstale":"2023-06-03T20:28:52.672993+0000","last_undegraded":"2023-06-03T20:28:52.672993+0000","last_fullsized":"2023-06-03T20:28:52.672993+0000","mapping_epoch":235,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":236,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T05:24:47.054342+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":255,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-06-03T20:28:54.247936+0000","last_change":"2023-06-03T19:56:54.507479+0000","last_active":"2023-06-03T20:28:54.247936+0000","last_peered":"2023-06-03T20:28:54.247936+0000","last_clean":"2023-06-03T20:28:54.247936+0000","last_became_active":"2023-06-03T19:56:54.507149+0000","last_became_peered":"2023-06-03T19:56:54.507149+0000","last_unstale":"2023-06-03T20:28:54.247936+0000","last_undegraded":"2023-06-03T20:28:54.247936+0000","last_fullsized":"2023-06-03T20:28:54.247936+0000","mapping_epoch":119,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T20:22:36.676317+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":155,"reported_epoch":288,"state":"active+clean","last_fresh":"2023-06-03T20:28:52.664736+0000","last_change":"2023-06-03T20:17:22.832844+0000","last_active":"2023-06-03T20:28:52.664736+0000","last_peered":"2023-06-03T20:28:52.664736+0000","last_clean":"2023-06-03T20:28:52.664736+0000","last_became_active":"2023-06-03T20:17:22.832473+0000","last_became_peered":"2023-06-03T20:17:22.832473+0000","last_unstale":"2023-06-03T20:28:52.664736+0000","last_undegraded":"2023-06-03T20:28:52.664736+0000","last_fullsized":"2023-06-03T20:28:52.664736+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,3],"acting":[7,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":288,"reported_epoch":288,"state":"active+clean","last_fresh":"2023-06-03T20:28:52.663897+0000","last_change":"2023-06-03T20:24:02.267531+0000","last_active":"2023-06-03T20:28:52.663897+0000","last_peered":"2023-06-03T20:28:52.663897+0000","last_clean":"2023-06-03T20:28:52.663897+0000","last_became_active":"2023-06-03T20:24:02.267159+0000","last_became_peered":"2023-06-03T20:24:02.267159+0000","last_unstale":"2023-06-03T20:28:52.663897+0000","last_undegraded":"2023-06-03T20:28:52.663897+0000","last_fullsized":"2023-06-03T20:28:52.663897+0000","mapping_epoch":254,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":255,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":255,"reported_epoch":288,"state":"active+clean","last_fresh":"2023-06-03T20:28:52.222715+0000","last_change":"2023-06-03T19:56:54.510289+0000","last_active":"2023-06-03T20:28:52.222715+0000","last_peered":"2023-06-03T20:28:52.222715+0000","last_clean":"2023-06-03T20:28:52.222715+0000","last_became_active":"2023-06-03T19:56:54.509905+0000","last_became_peered":"2023-06-03T19:56:54.509905+0000","last_unstale":"2023-06-03T20:28:52.222715+0000","last_undegraded":"2023-06-03T20:28:52.222715+0000","last_fullsized":"2023-06-03T20:28:52.222715+0000","mapping_epoch":119,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T02:54:26.792401+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"60'8","reported_seq":54,"reported_epoch":288,"state":"active+clean","last_fresh":"2023-06-03T20:28:52.501614+0000","last_change":"2023-06-03T20:26:59.682623+0000","last_active":"2023-06-03T20:28:52.501614+0000","last_peered":"2023-06-03T20:28:52.501614+0000","last_clean":"2023-06-03T20:28:52.501614+0000","last_became_active":"2023-06-03T20:24:02.260105+0000","last_became_peered":"2023-06-03T20:24:02.260105+0000","last_unstale":"2023-06-03T20:28:52.501614+0000","last_undegraded":"2023-06-03T20:28:52.501614+0000","last_fullsized":"2023-06-03T20:28:52.501614+0000","mapping_epoch":254,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":273,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":292,"reported_epoch":288,"state":"active+clean","last_fresh":"2023-06-03T20:28:52.674773+0000","last_change":"2023-06-03T20:28:24.281308+0000","last_active":"2023-06-03T20:28:52.674773+0000","last_peered":"2023-06-03T20:28:52.674773+0000","last_clean":"2023-06-03T20:28:52.674773+0000","last_became_active":"2023-06-03T20:28:24.280981+0000","last_became_peered":"2023-06-03T20:28:24.280981+0000","last_unstale":"2023-06-03T20:28:52.674773+0000","last_undegraded":"2023-06-03T20:28:52.674773+0000","last_fullsized":"2023-06-03T20:28:52.674773+0000","mapping_epoch":284,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":285,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T02:44:14.890147+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"14.2","version":"289'6260","reported_seq":6279,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-06-03T20:29:22.939067+0000","last_change":"2023-06-03T20:27:21.371612+0000","last_active":"2023-06-03T20:29:22.939067+0000","last_peered":"2023-06-03T20:29:22.939067+0000","last_clean":"2023-06-03T20:29:22.939067+0000","last_became_active":"2023-06-03T20:27:21.371344+0000","last_became_peered":"2023-06-03T20:27:21.371344+0000","last_unstale":"2023-06-03T20:29:22.939067+0000","last_undegraded":"2023-06-03T20:29:22.939067+0000","last_fullsized":"2023-06-03T20:29:22.939067+0000","mapping_epoch":278,"log_start":"0'0","ondisk_log_start":"0'0","created":278,"last_epoch_clean":279,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T20:27:20.339366+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T20:27:20.339366+0000","last_clean_scrub_stamp":"2023-06-03T20:27:20.339366+0000","objects_scrubbed":0,"log_size":6260,"log_dups_size":0,"ondisk_log_size":6260,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":378142720,"num_objects":5770,"num_object_clones":0,"num_object_copies":17310,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5770,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6260,"num_write_kb":384960,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,3],"acting":[6,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.f","version":"60'1","reported_seq":287,"reported_epoch":288,"state":"active+clean","last_fresh":"2023-06-03T20:28:52.664817+0000","last_change":"2023-06-03T20:23:27.263667+0000","last_active":"2023-06-03T20:28:52.664817+0000","last_peered":"2023-06-03T20:28:52.664817+0000","last_clean":"2023-06-03T20:28:52.664817+0000","last_became_active":"2023-06-03T20:23:27.263110+0000","last_became_peered":"2023-06-03T20:23:27.263110+0000","last_unstale":"2023-06-03T20:28:52.664817+0000","last_undegraded":"2023-06-03T20:28:52.664817+0000","last_fullsized":"2023-06-03T20:28:52.664817+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":248,"reported_epoch":288,"state":"active+clean","last_fresh":"2023-06-03T20:28:52.664153+0000","last_change":"2023-06-03T20:28:51.231222+0000","last_active":"2023-06-03T20:28:52.664153+0000","last_peered":"2023-06-03T20:28:52.664153+0000","last_clean":"2023-06-03T20:28:52.664153+0000","last_became_active":"2023-06-03T20:28:51.230848+0000","last_became_peered":"2023-06-03T20:28:51.230848+0000","last_unstale":"2023-06-03T20:28:52.664153+0000","last_undegraded":"2023-06-03T20:28:52.664153+0000","last_fullsized":"2023-06-03T20:28:52.664153+0000","mapping_epoch":286,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":287,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"14.3","version":"289'6414","reported_seq":6434,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-06-03T20:29:22.752128+0000","last_change":"2023-06-03T20:27:21.374055+0000","last_active":"2023-06-03T20:29:22.752128+0000","last_peered":"2023-06-03T20:29:22.752128+0000","last_clean":"2023-06-03T20:29:22.752128+0000","last_became_active":"2023-06-03T20:27:21.373728+0000","last_became_peered":"2023-06-03T20:27:21.373728+0000","last_unstale":"2023-06-03T20:29:22.752128+0000","last_undegraded":"2023-06-03T20:29:22.752128+0000","last_fullsized":"2023-06-03T20:29:22.752128+0000","mapping_epoch":278,"log_start":"0'0","ondisk_log_start":"0'0","created":278,"last_epoch_clean":279,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T20:27:20.339366+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T20:27:20.339366+0000","last_clean_scrub_stamp":"2023-06-03T20:27:20.339366+0000","objects_scrubbed":0,"log_size":6414,"log_dups_size":0,"ondisk_log_size":6414,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T05:15:33.026018+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":388366336,"num_objects":5926,"num_object_clones":0,"num_object_copies":17778,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5926,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6414,"num_write_kb":394880,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.e","version":"60'2","reported_seq":316,"reported_epoch":288,"state":"active+clean","last_fresh":"2023-06-03T20:28:52.664470+0000","last_change":"2023-06-03T20:23:28.606751+0000","last_active":"2023-06-03T20:28:52.664470+0000","last_peered":"2023-06-03T20:28:52.664470+0000","last_clean":"2023-06-03T20:28:52.664470+0000","last_became_active":"2023-06-03T20:23:28.605968+0000","last_became_peered":"2023-06-03T20:23:28.605968+0000","last_unstale":"2023-06-03T20:28:52.664470+0000","last_undegraded":"2023-06-03T20:28:52.664470+0000","last_fullsized":"2023-06-03T20:28:52.664470+0000","mapping_epoch":251,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":252,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":227,"reported_epoch":288,"state":"active+clean","last_fresh":"2023-06-03T20:28:52.252682+0000","last_change":"2023-06-03T20:23:28.249694+0000","last_active":"2023-06-03T20:28:52.252682+0000","last_peered":"2023-06-03T20:28:52.252682+0000","last_clean":"2023-06-03T20:28:52.252682+0000","last_became_active":"2023-06-03T20:23:28.249388+0000","last_became_peered":"2023-06-03T20:23:28.249388+0000","last_unstale":"2023-06-03T20:28:52.252682+0000","last_undegraded":"2023-06-03T20:28:52.252682+0000","last_fullsized":"2023-06-03T20:28:52.252682+0000","mapping_epoch":251,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":252,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T05:44:01.835586+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"14.0","version":"289'6254","reported_seq":6274,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-06-03T20:29:22.226461+0000","last_change":"2023-06-03T20:27:21.378346+0000","last_active":"2023-06-03T20:29:22.226461+0000","last_peered":"2023-06-03T20:29:22.226461+0000","last_clean":"2023-06-03T20:29:22.226461+0000","last_became_active":"2023-06-03T20:27:21.377969+0000","last_became_peered":"2023-06-03T20:27:21.377969+0000","last_unstale":"2023-06-03T20:29:22.226461+0000","last_undegraded":"2023-06-03T20:29:22.226461+0000","last_fullsized":"2023-06-03T20:29:22.226461+0000","mapping_epoch":278,"log_start":"0'0","ondisk_log_start":"0'0","created":278,"last_epoch_clean":279,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T20:27:20.339366+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T20:27:20.339366+0000","last_clean_scrub_stamp":"2023-06-03T20:27:20.339366+0000","objects_scrubbed":0,"log_size":6254,"log_dups_size":0,"ondisk_log_size":6254,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T01:32:49.543336+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":382074880,"num_objects":5830,"num_object_clones":0,"num_object_copies":17490,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5830,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6254,"num_write_kb":386688,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,5],"acting":[2,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.d","version":"60'1","reported_seq":265,"reported_epoch":288,"state":"active+clean","last_fresh":"2023-06-03T20:28:52.664348+0000","last_change":"2023-06-03T20:26:59.686037+0000","last_active":"2023-06-03T20:28:52.664348+0000","last_peered":"2023-06-03T20:28:52.664348+0000","last_clean":"2023-06-03T20:28:52.664348+0000","last_became_active":"2023-06-03T20:24:02.268002+0000","last_became_peered":"2023-06-03T20:24:02.268002+0000","last_unstale":"2023-06-03T20:28:52.664348+0000","last_undegraded":"2023-06-03T20:28:52.664348+0000","last_fullsized":"2023-06-03T20:28:52.664348+0000","mapping_epoch":254,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":273,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":140,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":12,"reported_epoch":288,"state":"active+clean","last_fresh":"2023-06-03T20:28:52.501419+0000","last_change":"2023-06-03T20:28:51.230833+0000","last_active":"2023-06-03T20:28:52.501419+0000","last_peered":"2023-06-03T20:28:52.501419+0000","last_clean":"2023-06-03T20:28:52.501419+0000","last_became_active":"2023-06-03T20:28:51.230489+0000","last_became_peered":"2023-06-03T20:28:51.230489+0000","last_unstale":"2023-06-03T20:28:52.501419+0000","last_undegraded":"2023-06-03T20:28:52.501419+0000","last_fullsized":"2023-06-03T20:28:52.501419+0000","mapping_epoch":286,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":287,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"14.1","version":"289'6637","reported_seq":6657,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-06-03T20:29:22.624954+0000","last_change":"2023-06-03T20:27:21.372641+0000","last_active":"2023-06-03T20:29:22.624954+0000","last_peered":"2023-06-03T20:29:22.624954+0000","last_clean":"2023-06-03T20:29:22.624954+0000","last_became_active":"2023-06-03T20:27:21.372158+0000","last_became_peered":"2023-06-03T20:27:21.372158+0000","last_unstale":"2023-06-03T20:29:22.624954+0000","last_undegraded":"2023-06-03T20:29:22.624954+0000","last_fullsized":"2023-06-03T20:29:22.624954+0000","mapping_epoch":278,"log_start":"0'0","ondisk_log_start":"0'0","created":278,"last_epoch_clean":279,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T20:27:20.339366+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T20:27:20.339366+0000","last_clean_scrub_stamp":"2023-06-03T20:27:20.339366+0000","objects_scrubbed":0,"log_size":6637,"log_dups_size":0,"ondisk_log_size":6637,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T07:52:41.694374+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":400097280,"num_objects":6105,"num_object_clones":0,"num_object_copies":18315,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6105,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6637,"num_write_kb":407744,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":281,"reported_epoch":288,"state":"active+clean","last_fresh":"2023-06-03T20:28:52.673137+0000","last_change":"2023-06-03T20:24:02.261252+0000","last_active":"2023-06-03T20:28:52.673137+0000","last_peered":"2023-06-03T20:28:52.673137+0000","last_clean":"2023-06-03T20:28:52.673137+0000","last_became_active":"2023-06-03T20:24:02.260824+0000","last_became_peered":"2023-06-03T20:24:02.260824+0000","last_unstale":"2023-06-03T20:28:52.673137+0000","last_undegraded":"2023-06-03T20:28:52.673137+0000","last_fullsized":"2023-06-03T20:28:52.673137+0000","mapping_epoch":254,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":255,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T20:00:53.584753+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":318,"reported_epoch":288,"state":"active+clean","last_fresh":"2023-06-03T20:28:52.224333+0000","last_change":"2023-06-03T20:23:28.283360+0000","last_active":"2023-06-03T20:28:52.224333+0000","last_peered":"2023-06-03T20:28:52.224333+0000","last_clean":"2023-06-03T20:28:52.224333+0000","last_became_active":"2023-06-03T20:23:28.283035+0000","last_became_peered":"2023-06-03T20:23:28.283035+0000","last_unstale":"2023-06-03T20:28:52.224333+0000","last_undegraded":"2023-06-03T20:28:52.224333+0000","last_fullsized":"2023-06-03T20:28:52.224333+0000","mapping_epoch":251,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":252,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T00:22:59.446515+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"14.6","version":"289'6418","reported_seq":6438,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-06-03T20:29:22.768815+0000","last_change":"2023-06-03T20:27:21.375836+0000","last_active":"2023-06-03T20:29:22.768815+0000","last_peered":"2023-06-03T20:29:22.768815+0000","last_clean":"2023-06-03T20:29:22.768815+0000","last_became_active":"2023-06-03T20:27:21.375522+0000","last_became_peered":"2023-06-03T20:27:21.375522+0000","last_unstale":"2023-06-03T20:29:22.768815+0000","last_undegraded":"2023-06-03T20:29:22.768815+0000","last_fullsized":"2023-06-03T20:29:22.768815+0000","mapping_epoch":278,"log_start":"0'0","ondisk_log_start":"0'0","created":278,"last_epoch_clean":279,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T20:27:20.339366+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T20:27:20.339366+0000","last_clean_scrub_stamp":"2023-06-03T20:27:20.339366+0000","objects_scrubbed":0,"log_size":6418,"log_dups_size":0,"ondisk_log_size":6418,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T21:02:49.138563+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":384172032,"num_objects":5862,"num_object_clones":0,"num_object_copies":17586,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5862,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6418,"num_write_kb":392960,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,1],"acting":[0,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.b","version":"60'2","reported_seq":282,"reported_epoch":288,"state":"active+clean","last_fresh":"2023-06-03T20:28:52.672750+0000","last_change":"2023-06-03T20:23:27.254179+0000","last_active":"2023-06-03T20:28:52.672750+0000","last_peered":"2023-06-03T20:28:52.672750+0000","last_clean":"2023-06-03T20:28:52.672750+0000","last_became_active":"2023-06-03T20:23:27.253791+0000","last_became_peered":"2023-06-03T20:23:27.253791+0000","last_unstale":"2023-06-03T20:28:52.672750+0000","last_undegraded":"2023-06-03T20:28:52.672750+0000","last_fullsized":"2023-06-03T20:28:52.672750+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T00:31:27.336760+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":203,"reported_epoch":288,"state":"active+clean","last_fresh":"2023-06-03T20:28:52.252526+0000","last_change":"2023-06-03T20:24:02.255551+0000","last_active":"2023-06-03T20:28:52.252526+0000","last_peered":"2023-06-03T20:28:52.252526+0000","last_clean":"2023-06-03T20:28:52.252526+0000","last_became_active":"2023-06-03T20:24:02.255209+0000","last_became_peered":"2023-06-03T20:24:02.255209+0000","last_unstale":"2023-06-03T20:28:52.252526+0000","last_undegraded":"2023-06-03T20:28:52.252526+0000","last_fullsized":"2023-06-03T20:28:52.252526+0000","mapping_epoch":254,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":255,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T20:05:02.741217+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"14.7","version":"289'6353","reported_seq":6373,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-06-03T20:29:21.260499+0000","last_change":"2023-06-03T20:27:21.364098+0000","last_active":"2023-06-03T20:29:21.260499+0000","last_peered":"2023-06-03T20:29:21.260499+0000","last_clean":"2023-06-03T20:29:21.260499+0000","last_became_active":"2023-06-03T20:27:21.363548+0000","last_became_peered":"2023-06-03T20:27:21.363548+0000","last_unstale":"2023-06-03T20:29:21.260499+0000","last_undegraded":"2023-06-03T20:29:21.260499+0000","last_fullsized":"2023-06-03T20:29:21.260499+0000","mapping_epoch":278,"log_start":"0'0","ondisk_log_start":"0'0","created":278,"last_epoch_clean":279,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T20:27:20.339366+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T20:27:20.339366+0000","last_clean_scrub_stamp":"2023-06-03T20:27:20.339366+0000","objects_scrubbed":0,"log_size":6353,"log_dups_size":0,"ondisk_log_size":6353,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T06:05:40.316567+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":391446528,"num_objects":5973,"num_object_clones":0,"num_object_copies":17919,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5973,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6353,"num_write_kb":394432,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,0],"acting":[4,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":44,"reported_epoch":288,"state":"active+clean","last_fresh":"2023-06-03T20:28:52.501869+0000","last_change":"2023-06-03T20:24:02.254172+0000","last_active":"2023-06-03T20:28:52.501869+0000","last_peered":"2023-06-03T20:28:52.501869+0000","last_clean":"2023-06-03T20:28:52.501869+0000","last_became_active":"2023-06-03T20:24:02.253797+0000","last_became_peered":"2023-06-03T20:24:02.253797+0000","last_unstale":"2023-06-03T20:28:52.501869+0000","last_undegraded":"2023-06-03T20:28:52.501869+0000","last_fullsized":"2023-06-03T20:28:52.501869+0000","mapping_epoch":254,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":255,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":208,"reported_epoch":288,"state":"active+clean","last_fresh":"2023-06-03T20:28:52.253327+0000","last_change":"2023-06-03T20:23:28.291250+0000","last_active":"2023-06-03T20:28:52.253327+0000","last_peered":"2023-06-03T20:28:52.253327+0000","last_clean":"2023-06-03T20:28:52.253327+0000","last_became_active":"2023-06-03T20:23:28.290120+0000","last_became_peered":"2023-06-03T20:23:28.290120+0000","last_unstale":"2023-06-03T20:28:52.253327+0000","last_undegraded":"2023-06-03T20:28:52.253327+0000","last_fullsized":"2023-06-03T20:28:52.253327+0000","mapping_epoch":251,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":252,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T22:37:56.033327+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"14.4","version":"289'6399","reported_seq":6418,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-06-03T20:29:23.008692+0000","last_change":"2023-06-03T20:27:21.366337+0000","last_active":"2023-06-03T20:29:23.008692+0000","last_peered":"2023-06-03T20:29:23.008692+0000","last_clean":"2023-06-03T20:29:23.008692+0000","last_became_active":"2023-06-03T20:27:21.365886+0000","last_became_peered":"2023-06-03T20:27:21.365886+0000","last_unstale":"2023-06-03T20:29:23.008692+0000","last_undegraded":"2023-06-03T20:29:23.008692+0000","last_fullsized":"2023-06-03T20:29:23.008692+0000","mapping_epoch":278,"log_start":"0'0","ondisk_log_start":"0'0","created":278,"last_epoch_clean":279,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T20:27:20.339366+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T20:27:20.339366+0000","last_clean_scrub_stamp":"2023-06-03T20:27:20.339366+0000","objects_scrubbed":0,"log_size":6399,"log_dups_size":0,"ondisk_log_size":6399,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":380043264,"num_objects":5799,"num_object_clones":0,"num_object_copies":17397,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5799,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6399,"num_write_kb":390336,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,3],"acting":[6,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":193,"reported_epoch":288,"state":"active+clean","last_fresh":"2023-06-03T20:28:52.252478+0000","last_change":"2023-06-03T20:23:27.276693+0000","last_active":"2023-06-03T20:28:52.252478+0000","last_peered":"2023-06-03T20:28:52.252478+0000","last_clean":"2023-06-03T20:28:52.252478+0000","last_became_active":"2023-06-03T20:23:27.276305+0000","last_became_peered":"2023-06-03T20:23:27.276305+0000","last_unstale":"2023-06-03T20:28:52.252478+0000","last_undegraded":"2023-06-03T20:28:52.252478+0000","last_fullsized":"2023-06-03T20:28:52.252478+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T20:34:50.944936+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":198,"reported_epoch":288,"state":"active+clean","last_fresh":"2023-06-03T20:28:52.252828+0000","last_change":"2023-06-03T20:28:51.231829+0000","last_active":"2023-06-03T20:28:52.252828+0000","last_peered":"2023-06-03T20:28:52.252828+0000","last_clean":"2023-06-03T20:28:52.252828+0000","last_became_active":"2023-06-03T20:28:51.231452+0000","last_became_peered":"2023-06-03T20:28:51.231452+0000","last_unstale":"2023-06-03T20:28:52.252828+0000","last_undegraded":"2023-06-03T20:28:52.252828+0000","last_fullsized":"2023-06-03T20:28:52.252828+0000","mapping_epoch":286,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":287,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T07:00:21.647058+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"14.5","version":"289'6474","reported_seq":6494,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-06-03T20:29:22.709854+0000","last_change":"2023-06-03T20:27:21.371070+0000","last_active":"2023-06-03T20:29:22.709854+0000","last_peered":"2023-06-03T20:29:22.709854+0000","last_clean":"2023-06-03T20:29:22.709854+0000","last_became_active":"2023-06-03T20:27:21.370790+0000","last_became_peered":"2023-06-03T20:27:21.370790+0000","last_unstale":"2023-06-03T20:29:22.709854+0000","last_undegraded":"2023-06-03T20:29:22.709854+0000","last_fullsized":"2023-06-03T20:29:22.709854+0000","mapping_epoch":278,"log_start":"0'0","ondisk_log_start":"0'0","created":278,"last_epoch_clean":279,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T20:27:20.339366+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T20:27:20.339366+0000","last_clean_scrub_stamp":"2023-06-03T20:27:20.339366+0000","objects_scrubbed":0,"log_size":6474,"log_dups_size":0,"ondisk_log_size":6474,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T21:04:21.657715+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":388235264,"num_objects":5924,"num_object_clones":0,"num_object_copies":17772,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5924,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6474,"num_write_kb":396736,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":292,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-06-03T20:28:54.259789+0000","last_change":"2023-06-03T20:21:01.256958+0000","last_active":"2023-06-03T20:28:54.259789+0000","last_peered":"2023-06-03T20:28:54.259789+0000","last_clean":"2023-06-03T20:28:54.259789+0000","last_became_active":"2023-06-03T20:21:01.255452+0000","last_became_peered":"2023-06-03T20:21:01.255452+0000","last_unstale":"2023-06-03T20:28:54.259789+0000","last_undegraded":"2023-06-03T20:28:54.259789+0000","last_fullsized":"2023-06-03T20:28:54.259789+0000","mapping_epoch":235,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":236,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T01:56:59.777763+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":78,"reported_epoch":288,"state":"active+clean","last_fresh":"2023-06-03T20:28:52.225085+0000","last_change":"2023-06-03T20:23:28.250884+0000","last_active":"2023-06-03T20:28:52.225085+0000","last_peered":"2023-06-03T20:28:52.225085+0000","last_clean":"2023-06-03T20:28:52.225085+0000","last_became_active":"2023-06-03T20:23:28.243793+0000","last_became_peered":"2023-06-03T20:23:28.243793+0000","last_unstale":"2023-06-03T20:28:52.225085+0000","last_undegraded":"2023-06-03T20:28:52.225085+0000","last_fullsized":"2023-06-03T20:28:52.225085+0000","mapping_epoch":251,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":252,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T03:12:27.946002+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"14.8","version":"289'6406","reported_seq":6426,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-06-03T20:29:22.740687+0000","last_change":"2023-06-03T20:27:21.376040+0000","last_active":"2023-06-03T20:29:22.740687+0000","last_peered":"2023-06-03T20:29:22.740687+0000","last_clean":"2023-06-03T20:29:22.740687+0000","last_became_active":"2023-06-03T20:27:21.375709+0000","last_became_peered":"2023-06-03T20:27:21.375709+0000","last_unstale":"2023-06-03T20:29:22.740687+0000","last_undegraded":"2023-06-03T20:29:22.740687+0000","last_fullsized":"2023-06-03T20:29:22.740687+0000","mapping_epoch":278,"log_start":"0'0","ondisk_log_start":"0'0","created":278,"last_epoch_clean":279,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T20:27:20.339366+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T20:27:20.339366+0000","last_clean_scrub_stamp":"2023-06-03T20:27:20.339366+0000","objects_scrubbed":0,"log_size":6406,"log_dups_size":0,"ondisk_log_size":6406,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T05:50:28.157584+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":385089536,"num_objects":5876,"num_object_clones":0,"num_object_copies":17628,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5876,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6406,"num_write_kb":393024,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":68,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-06-03T20:28:54.266339+0000","last_change":"2023-06-03T20:21:01.258182+0000","last_active":"2023-06-03T20:28:54.266339+0000","last_peered":"2023-06-03T20:28:54.266339+0000","last_clean":"2023-06-03T20:28:54.266339+0000","last_became_active":"2023-06-03T20:21:01.254049+0000","last_became_peered":"2023-06-03T20:21:01.254049+0000","last_unstale":"2023-06-03T20:28:54.266339+0000","last_undegraded":"2023-06-03T20:28:54.266339+0000","last_fullsized":"2023-06-03T20:28:54.266339+0000","mapping_epoch":235,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":236,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:32:20.692043+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:32:20.692043+0000","last_clean_scrub_stamp":"2023-06-03T19:32:20.692043+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T05:16:30.078473+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":176,"reported_epoch":288,"state":"active+clean","last_fresh":"2023-06-03T20:28:52.672326+0000","last_change":"2023-06-03T20:17:22.834746+0000","last_active":"2023-06-03T20:28:52.672326+0000","last_peered":"2023-06-03T20:28:52.672326+0000","last_clean":"2023-06-03T20:28:52.672326+0000","last_became_active":"2023-06-03T20:17:22.834448+0000","last_became_peered":"2023-06-03T20:17:22.834448+0000","last_unstale":"2023-06-03T20:28:52.672326+0000","last_undegraded":"2023-06-03T20:28:52.672326+0000","last_fullsized":"2023-06-03T20:28:52.672326+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T21:16:31.167063+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,2],"acting":[3,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":79,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-06-03T20:28:54.267723+0000","last_change":"2023-06-03T20:23:28.254434+0000","last_active":"2023-06-03T20:28:54.267723+0000","last_peered":"2023-06-03T20:28:54.267723+0000","last_clean":"2023-06-03T20:28:54.267723+0000","last_became_active":"2023-06-03T20:23:28.254012+0000","last_became_peered":"2023-06-03T20:23:28.254012+0000","last_unstale":"2023-06-03T20:28:54.267723+0000","last_undegraded":"2023-06-03T20:28:54.267723+0000","last_fullsized":"2023-06-03T20:28:54.267723+0000","mapping_epoch":251,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":252,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T06:22:09.540298+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"14.e","version":"289'6360","reported_seq":6379,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-06-03T20:29:21.296701+0000","last_change":"2023-06-03T20:27:21.368301+0000","last_active":"2023-06-03T20:29:21.296701+0000","last_peered":"2023-06-03T20:29:21.296701+0000","last_clean":"2023-06-03T20:29:21.296701+0000","last_became_active":"2023-06-03T20:27:21.367955+0000","last_became_peered":"2023-06-03T20:27:21.367955+0000","last_unstale":"2023-06-03T20:29:21.296701+0000","last_undegraded":"2023-06-03T20:29:21.296701+0000","last_fullsized":"2023-06-03T20:29:21.296701+0000","mapping_epoch":278,"log_start":"0'0","ondisk_log_start":"0'0","created":278,"last_epoch_clean":279,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T20:27:20.339366+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T20:27:20.339366+0000","last_clean_scrub_stamp":"2023-06-03T20:27:20.339366+0000","objects_scrubbed":0,"log_size":6360,"log_dups_size":0,"ondisk_log_size":6360,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T22:35:27.855608+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":389021696,"num_objects":5936,"num_object_clones":0,"num_object_copies":17808,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5936,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6360,"num_write_kb":393472,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":77,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-06-03T20:28:54.267892+0000","last_change":"2023-06-03T20:28:51.234855+0000","last_active":"2023-06-03T20:28:54.267892+0000","last_peered":"2023-06-03T20:28:54.267892+0000","last_clean":"2023-06-03T20:28:54.267892+0000","last_became_active":"2023-06-03T20:28:51.234514+0000","last_became_peered":"2023-06-03T20:28:51.234514+0000","last_unstale":"2023-06-03T20:28:54.267892+0000","last_undegraded":"2023-06-03T20:28:54.267892+0000","last_fullsized":"2023-06-03T20:28:54.267892+0000","mapping_epoch":286,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":287,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:32:20.692043+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:32:20.692043+0000","last_clean_scrub_stamp":"2023-06-03T19:32:20.692043+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T03:00:56.480804+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":173,"reported_epoch":288,"state":"active+clean","last_fresh":"2023-06-03T20:28:52.253067+0000","last_change":"2023-06-03T20:24:02.258311+0000","last_active":"2023-06-03T20:28:52.253067+0000","last_peered":"2023-06-03T20:28:52.253067+0000","last_clean":"2023-06-03T20:28:52.253067+0000","last_became_active":"2023-06-03T20:24:02.257676+0000","last_became_peered":"2023-06-03T20:24:02.257676+0000","last_unstale":"2023-06-03T20:28:52.253067+0000","last_undegraded":"2023-06-03T20:28:52.253067+0000","last_fullsized":"2023-06-03T20:28:52.253067+0000","mapping_epoch":254,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":255,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T01:47:46.383038+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":68,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-06-03T20:28:54.266409+0000","last_change":"2023-06-03T20:21:01.259331+0000","last_active":"2023-06-03T20:28:54.266409+0000","last_peered":"2023-06-03T20:28:54.266409+0000","last_clean":"2023-06-03T20:28:54.266409+0000","last_became_active":"2023-06-03T20:21:01.258836+0000","last_became_peered":"2023-06-03T20:21:01.258836+0000","last_unstale":"2023-06-03T20:28:54.266409+0000","last_undegraded":"2023-06-03T20:28:54.266409+0000","last_fullsized":"2023-06-03T20:28:54.266409+0000","mapping_epoch":235,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":236,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T01:29:19.471600+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"14.d","version":"289'6414","reported_seq":7714,"reported_epoch":289,"state":"active+recovering+remapped","last_fresh":"2023-06-03T20:29:22.210127+0000","last_change":"2023-06-03T20:28:52.754748+0000","last_active":"2023-06-03T20:29:22.210127+0000","last_peered":"2023-06-03T20:29:22.210127+0000","last_clean":"2023-06-03T20:28:49.972400+0000","last_became_active":"2023-06-03T20:28:52.639960+0000","last_became_peered":"2023-06-03T20:28:52.639960+0000","last_unstale":"2023-06-03T20:29:22.210127+0000","last_undegraded":"2023-06-03T20:29:22.210127+0000","last_fullsized":"2023-06-03T20:29:22.210127+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":278,"last_epoch_clean":279,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T20:27:20.339366+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T20:27:20.339366+0000","last_clean_scrub_stamp":"2023-06-03T20:27:20.339366+0000","objects_scrubbed":0,"log_size":6414,"log_dups_size":0,"ondisk_log_size":6414,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T08:00:38.609409+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":389087256,"num_objects":5938,"num_object_clones":0,"num_object_copies":17814,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":4428,"num_objects_unfound":0,"num_objects_dirty":5938,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":6414,"num_write_kb":395201,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":638,"num_bytes_recovered":41811968,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,4],"acting":[2,5,4],"avail_no_missing":["2","4","5"],"object_location_counts":[{"shards":"2,4,5","objects":1510}],"blocked_by":[],"up_primary":6,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":274,"reported_epoch":288,"state":"active+clean","last_fresh":"2023-06-03T20:28:52.228003+0000","last_change":"2023-06-03T19:56:54.504859+0000","last_active":"2023-06-03T20:28:52.228003+0000","last_peered":"2023-06-03T20:28:52.228003+0000","last_clean":"2023-06-03T20:28:52.228003+0000","last_became_active":"2023-06-03T19:56:54.504455+0000","last_became_peered":"2023-06-03T19:56:54.504455+0000","last_unstale":"2023-06-03T20:28:52.228003+0000","last_undegraded":"2023-06-03T20:28:52.228003+0000","last_fullsized":"2023-06-03T20:28:52.228003+0000","mapping_epoch":119,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:32:20.692043+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:32:20.692043+0000","last_clean_scrub_stamp":"2023-06-03T19:32:20.692043+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T06:25:45.522100+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":77,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-06-03T20:28:54.262235+0000","last_change":"2023-06-03T20:24:02.256677+0000","last_active":"2023-06-03T20:28:54.262235+0000","last_peered":"2023-06-03T20:28:54.262235+0000","last_clean":"2023-06-03T20:28:54.262235+0000","last_became_active":"2023-06-03T20:24:02.256216+0000","last_became_peered":"2023-06-03T20:24:02.256216+0000","last_unstale":"2023-06-03T20:28:54.262235+0000","last_undegraded":"2023-06-03T20:28:54.262235+0000","last_fullsized":"2023-06-03T20:28:54.262235+0000","mapping_epoch":254,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":255,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T02:32:45.186204+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":67,"reported_epoch":288,"state":"active+clean","last_fresh":"2023-06-03T20:28:52.223970+0000","last_change":"2023-06-03T20:21:01.261473+0000","last_active":"2023-06-03T20:28:52.223970+0000","last_peered":"2023-06-03T20:28:52.223970+0000","last_clean":"2023-06-03T20:28:52.223970+0000","last_became_active":"2023-06-03T20:21:01.261098+0000","last_became_peered":"2023-06-03T20:21:01.261098+0000","last_unstale":"2023-06-03T20:28:52.223970+0000","last_undegraded":"2023-06-03T20:28:52.223970+0000","last_fullsized":"2023-06-03T20:28:52.223970+0000","mapping_epoch":235,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":236,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T03:56:22.986549+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"14.f","version":"289'6339","reported_seq":6358,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-06-03T20:29:23.084739+0000","last_change":"2023-06-03T20:27:21.374763+0000","last_active":"2023-06-03T20:29:23.084739+0000","last_peered":"2023-06-03T20:29:23.084739+0000","last_clean":"2023-06-03T20:29:23.084739+0000","last_became_active":"2023-06-03T20:27:21.374440+0000","last_became_peered":"2023-06-03T20:27:21.374440+0000","last_unstale":"2023-06-03T20:29:23.084739+0000","last_undegraded":"2023-06-03T20:29:23.084739+0000","last_fullsized":"2023-06-03T20:29:23.084739+0000","mapping_epoch":278,"log_start":"0'0","ondisk_log_start":"0'0","created":278,"last_epoch_clean":279,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T20:27:20.339366+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T20:27:20.339366+0000","last_clean_scrub_stamp":"2023-06-03T20:27:20.339366+0000","objects_scrubbed":0,"log_size":6339,"log_dups_size":0,"ondisk_log_size":6339,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":382140416,"num_objects":5831,"num_object_clones":0,"num_object_copies":17493,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5831,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6339,"num_write_kb":389440,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,1],"acting":[7,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":159,"reported_epoch":288,"state":"active+clean","last_fresh":"2023-06-03T20:28:52.229226+0000","last_change":"2023-06-03T20:17:22.834467+0000","last_active":"2023-06-03T20:28:52.229226+0000","last_peered":"2023-06-03T20:28:52.229226+0000","last_clean":"2023-06-03T20:28:52.229226+0000","last_became_active":"2023-06-03T20:17:22.834147+0000","last_became_peered":"2023-06-03T20:17:22.834147+0000","last_unstale":"2023-06-03T20:28:52.229226+0000","last_undegraded":"2023-06-03T20:28:52.229226+0000","last_fullsized":"2023-06-03T20:28:52.229226+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:32:20.692043+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:32:20.692043+0000","last_clean_scrub_stamp":"2023-06-03T19:32:20.692043+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T04:36:29.567051+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,7],"acting":[2,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"72'139","reported_seq":561,"reported_epoch":288,"state":"active+clean","last_fresh":"2023-06-03T20:28:52.663952+0000","last_change":"2023-06-03T20:27:01.248641+0000","last_active":"2023-06-03T20:28:52.663952+0000","last_peered":"2023-06-03T20:28:52.663952+0000","last_clean":"2023-06-03T20:28:52.663952+0000","last_became_active":"2023-06-03T20:27:01.248063+0000","last_became_peered":"2023-06-03T20:27:01.248063+0000","last_unstale":"2023-06-03T20:28:52.663952+0000","last_undegraded":"2023-06-03T20:28:52.663952+0000","last_fullsized":"2023-06-03T20:28:52.663952+0000","mapping_epoch":274,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":275,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:27:28.100808+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:27:28.100808+0000","last_clean_scrub_stamp":"2023-06-03T19:27:28.100808+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":277,"reported_epoch":288,"state":"active+clean","last_fresh":"2023-06-03T20:28:52.672022+0000","last_change":"2023-06-03T20:24:02.263360+0000","last_active":"2023-06-03T20:28:52.672022+0000","last_peered":"2023-06-03T20:28:52.672022+0000","last_clean":"2023-06-03T20:28:52.672022+0000","last_became_active":"2023-06-03T20:24:02.262954+0000","last_became_peered":"2023-06-03T20:24:02.262954+0000","last_unstale":"2023-06-03T20:28:52.672022+0000","last_undegraded":"2023-06-03T20:28:52.672022+0000","last_fullsized":"2023-06-03T20:28:52.672022+0000","mapping_epoch":254,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":255,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T19:38:46.084731+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":12,"reported_epoch":288,"state":"active+clean","last_fresh":"2023-06-03T20:28:52.502070+0000","last_change":"2023-06-03T20:28:51.234600+0000","last_active":"2023-06-03T20:28:52.502070+0000","last_peered":"2023-06-03T20:28:52.502070+0000","last_clean":"2023-06-03T20:28:52.502070+0000","last_became_active":"2023-06-03T20:28:51.234227+0000","last_became_peered":"2023-06-03T20:28:51.234227+0000","last_unstale":"2023-06-03T20:28:52.502070+0000","last_undegraded":"2023-06-03T20:28:52.502070+0000","last_fullsized":"2023-06-03T20:28:52.502070+0000","mapping_epoch":286,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":287,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"14.c","version":"289'6418","reported_seq":6438,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-06-03T20:29:22.742570+0000","last_change":"2023-06-03T20:27:21.375660+0000","last_active":"2023-06-03T20:29:22.742570+0000","last_peered":"2023-06-03T20:29:22.742570+0000","last_clean":"2023-06-03T20:29:22.742570+0000","last_became_active":"2023-06-03T20:27:21.375311+0000","last_became_peered":"2023-06-03T20:27:21.375311+0000","last_unstale":"2023-06-03T20:29:22.742570+0000","last_undegraded":"2023-06-03T20:29:22.742570+0000","last_fullsized":"2023-06-03T20:29:22.742570+0000","mapping_epoch":278,"log_start":"0'0","ondisk_log_start":"0'0","created":278,"last_epoch_clean":279,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T20:27:20.339366+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T20:27:20.339366+0000","last_clean_scrub_stamp":"2023-06-03T20:27:20.339366+0000","objects_scrubbed":0,"log_size":6418,"log_dups_size":0,"ondisk_log_size":6418,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T04:43:56.400215+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":380895232,"num_objects":5812,"num_object_clones":0,"num_object_copies":17436,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5812,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6418,"num_write_kb":391360,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,2],"acting":[0,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":281,"reported_epoch":288,"state":"active+clean","last_fresh":"2023-06-03T20:28:52.663766+0000","last_change":"2023-06-03T20:21:01.258113+0000","last_active":"2023-06-03T20:28:52.663766+0000","last_peered":"2023-06-03T20:28:52.663766+0000","last_clean":"2023-06-03T20:28:52.663766+0000","last_became_active":"2023-06-03T20:21:01.257488+0000","last_became_peered":"2023-06-03T20:21:01.257488+0000","last_unstale":"2023-06-03T20:28:52.663766+0000","last_undegraded":"2023-06-03T20:28:52.663766+0000","last_fullsized":"2023-06-03T20:28:52.663766+0000","mapping_epoch":235,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":236,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:32:20.692043+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:32:20.692043+0000","last_clean_scrub_stamp":"2023-06-03T19:32:20.692043+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":287,"reported_epoch":288,"state":"active+clean","last_fresh":"2023-06-03T20:28:52.222326+0000","last_change":"2023-06-03T20:23:27.255164+0000","last_active":"2023-06-03T20:28:52.222326+0000","last_peered":"2023-06-03T20:28:52.222326+0000","last_clean":"2023-06-03T20:28:52.222326+0000","last_became_active":"2023-06-03T20:23:27.253932+0000","last_became_peered":"2023-06-03T20:23:27.253932+0000","last_unstale":"2023-06-03T20:28:52.222326+0000","last_undegraded":"2023-06-03T20:28:52.222326+0000","last_fullsized":"2023-06-03T20:28:52.222326+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T06:42:02.303554+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":290,"reported_epoch":288,"state":"active+clean","last_fresh":"2023-06-03T20:28:52.222634+0000","last_change":"2023-06-03T20:21:01.250266+0000","last_active":"2023-06-03T20:28:52.222634+0000","last_peered":"2023-06-03T20:28:52.222634+0000","last_clean":"2023-06-03T20:28:52.222634+0000","last_became_active":"2023-06-03T20:21:01.249839+0000","last_became_peered":"2023-06-03T20:21:01.249839+0000","last_unstale":"2023-06-03T20:28:52.222634+0000","last_undegraded":"2023-06-03T20:28:52.222634+0000","last_fullsized":"2023-06-03T20:28:52.222634+0000","mapping_epoch":235,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":236,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T05:13:26.584607+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"14.9","version":"289'6385","reported_seq":6405,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-06-03T20:29:22.625371+0000","last_change":"2023-06-03T20:27:21.372404+0000","last_active":"2023-06-03T20:29:22.625371+0000","last_peered":"2023-06-03T20:29:22.625371+0000","last_clean":"2023-06-03T20:29:22.625371+0000","last_became_active":"2023-06-03T20:27:21.371829+0000","last_became_peered":"2023-06-03T20:27:21.371829+0000","last_unstale":"2023-06-03T20:29:22.625371+0000","last_undegraded":"2023-06-03T20:29:22.625371+0000","last_fullsized":"2023-06-03T20:29:22.625371+0000","mapping_epoch":278,"log_start":"0'0","ondisk_log_start":"0'0","created":278,"last_epoch_clean":279,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T20:27:20.339366+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T20:27:20.339366+0000","last_clean_scrub_stamp":"2023-06-03T20:27:20.339366+0000","objects_scrubbed":0,"log_size":6385,"log_dups_size":0,"ondisk_log_size":6385,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T02:20:57.246181+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":383844352,"num_objects":5857,"num_object_clones":0,"num_object_copies":17571,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5857,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6385,"num_write_kb":391744,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,4],"acting":[3,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":290,"reported_epoch":288,"state":"active+clean","last_fresh":"2023-06-03T20:28:52.664463+0000","last_change":"2023-06-03T20:23:27.261511+0000","last_active":"2023-06-03T20:28:52.664463+0000","last_peered":"2023-06-03T20:28:52.664463+0000","last_clean":"2023-06-03T20:28:52.664463+0000","last_became_active":"2023-06-03T20:23:27.259622+0000","last_became_peered":"2023-06-03T20:23:27.259622+0000","last_unstale":"2023-06-03T20:28:52.664463+0000","last_undegraded":"2023-06-03T20:28:52.664463+0000","last_fullsized":"2023-06-03T20:28:52.664463+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:32:20.692043+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:32:20.692043+0000","last_clean_scrub_stamp":"2023-06-03T19:32:20.692043+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":68,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-06-03T20:28:54.266269+0000","last_change":"2023-06-03T20:21:01.252859+0000","last_active":"2023-06-03T20:28:54.266269+0000","last_peered":"2023-06-03T20:28:54.266269+0000","last_clean":"2023-06-03T20:28:54.266269+0000","last_became_active":"2023-06-03T20:21:01.251222+0000","last_became_peered":"2023-06-03T20:21:01.251222+0000","last_unstale":"2023-06-03T20:28:54.266269+0000","last_undegraded":"2023-06-03T20:28:54.266269+0000","last_fullsized":"2023-06-03T20:28:54.266269+0000","mapping_epoch":235,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":236,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T05:19:58.100226+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":243,"reported_epoch":288,"state":"active+clean","last_fresh":"2023-06-03T20:28:52.224698+0000","last_change":"2023-06-03T19:47:06.733532+0000","last_active":"2023-06-03T20:28:52.224698+0000","last_peered":"2023-06-03T20:28:52.224698+0000","last_clean":"2023-06-03T20:28:52.224698+0000","last_became_active":"2023-06-03T19:47:06.733131+0000","last_became_peered":"2023-06-03T19:47:06.733131+0000","last_unstale":"2023-06-03T20:28:52.224698+0000","last_undegraded":"2023-06-03T20:28:52.224698+0000","last_fullsized":"2023-06-03T20:28:52.224698+0000","mapping_epoch":86,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":87,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T23:46:03.788646+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"14.a","version":"289'6295","reported_seq":6314,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-06-03T20:29:22.561207+0000","last_change":"2023-06-03T20:27:21.378317+0000","last_active":"2023-06-03T20:29:22.561207+0000","last_peered":"2023-06-03T20:29:22.561207+0000","last_clean":"2023-06-03T20:29:22.561207+0000","last_became_active":"2023-06-03T20:27:21.377947+0000","last_became_peered":"2023-06-03T20:27:21.377947+0000","last_unstale":"2023-06-03T20:29:22.561207+0000","last_undegraded":"2023-06-03T20:29:22.561207+0000","last_fullsized":"2023-06-03T20:29:22.561207+0000","mapping_epoch":278,"log_start":"0'0","ondisk_log_start":"0'0","created":278,"last_epoch_clean":279,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T20:27:20.339366+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T20:27:20.339366+0000","last_clean_scrub_stamp":"2023-06-03T20:27:20.339366+0000","objects_scrubbed":0,"log_size":6295,"log_dups_size":0,"ondisk_log_size":6295,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T23:39:02.595450+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":379387904,"num_objects":5789,"num_object_clones":0,"num_object_copies":17367,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5789,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6295,"num_write_kb":386688,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,3],"acting":[1,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.7","version":"60'2","reported_seq":253,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-06-03T20:28:54.260051+0000","last_change":"2023-06-03T19:56:54.510148+0000","last_active":"2023-06-03T20:28:54.260051+0000","last_peered":"2023-06-03T20:28:54.260051+0000","last_clean":"2023-06-03T20:28:54.260051+0000","last_became_active":"2023-06-03T19:56:54.509771+0000","last_became_peered":"2023-06-03T19:56:54.509771+0000","last_unstale":"2023-06-03T20:28:54.260051+0000","last_undegraded":"2023-06-03T20:28:54.260051+0000","last_fullsized":"2023-06-03T20:28:54.260051+0000","mapping_epoch":119,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T00:52:42.807432+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":85,"reported_epoch":288,"state":"active+clean","last_fresh":"2023-06-03T20:28:52.223969+0000","last_change":"2023-06-03T20:24:02.257142+0000","last_active":"2023-06-03T20:28:52.223969+0000","last_peered":"2023-06-03T20:28:52.223969+0000","last_clean":"2023-06-03T20:28:52.223969+0000","last_became_active":"2023-06-03T20:24:02.256483+0000","last_became_peered":"2023-06-03T20:24:02.256483+0000","last_unstale":"2023-06-03T20:28:52.223969+0000","last_undegraded":"2023-06-03T20:28:52.223969+0000","last_fullsized":"2023-06-03T20:28:52.223969+0000","mapping_epoch":254,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":255,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:32:20.692043+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:32:20.692043+0000","last_clean_scrub_stamp":"2023-06-03T19:32:20.692043+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T00:53:37.197795+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":251,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-06-03T20:28:54.259707+0000","last_change":"2023-06-03T19:56:54.507670+0000","last_active":"2023-06-03T20:28:54.259707+0000","last_peered":"2023-06-03T20:28:54.259707+0000","last_clean":"2023-06-03T20:28:54.259707+0000","last_became_active":"2023-06-03T19:56:54.507293+0000","last_became_peered":"2023-06-03T19:56:54.507293+0000","last_unstale":"2023-06-03T20:28:54.259707+0000","last_undegraded":"2023-06-03T20:28:54.259707+0000","last_fullsized":"2023-06-03T20:28:54.259707+0000","mapping_epoch":119,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T01:49:19.966054+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"14.b","version":"289'6231","reported_seq":6251,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-06-03T20:29:22.752137+0000","last_change":"2023-06-03T20:27:21.375306+0000","last_active":"2023-06-03T20:29:22.752137+0000","last_peered":"2023-06-03T20:29:22.752137+0000","last_clean":"2023-06-03T20:29:22.752137+0000","last_became_active":"2023-06-03T20:27:21.374975+0000","last_became_peered":"2023-06-03T20:27:21.374975+0000","last_unstale":"2023-06-03T20:29:22.752137+0000","last_undegraded":"2023-06-03T20:29:22.752137+0000","last_fullsized":"2023-06-03T20:29:22.752137+0000","mapping_epoch":278,"log_start":"0'0","ondisk_log_start":"0'0","created":278,"last_epoch_clean":279,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T20:27:20.339366+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T20:27:20.339366+0000","last_clean_scrub_stamp":"2023-06-03T20:27:20.339366+0000","objects_scrubbed":0,"log_size":6231,"log_dups_size":0,"ondisk_log_size":6231,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T02:22:27.778843+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":376373248,"num_objects":5743,"num_object_clones":0,"num_object_copies":17229,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5743,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6231,"num_write_kb":383168,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,1],"acting":[0,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":44,"reported_epoch":288,"state":"active+clean","last_fresh":"2023-06-03T20:28:52.502313+0000","last_change":"2023-06-03T20:24:02.319230+0000","last_active":"2023-06-03T20:28:52.502313+0000","last_peered":"2023-06-03T20:28:52.502313+0000","last_clean":"2023-06-03T20:28:52.502313+0000","last_became_active":"2023-06-03T20:24:02.318926+0000","last_became_peered":"2023-06-03T20:24:02.318926+0000","last_unstale":"2023-06-03T20:28:52.502313+0000","last_undegraded":"2023-06-03T20:28:52.502313+0000","last_fullsized":"2023-06-03T20:28:52.502313+0000","mapping_epoch":254,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":255,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:32:20.692043+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:32:20.692043+0000","last_clean_scrub_stamp":"2023-06-03T19:32:20.692043+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"60'1","reported_seq":318,"reported_epoch":288,"state":"active+clean","last_fresh":"2023-06-03T20:28:52.222535+0000","last_change":"2023-06-03T20:23:28.239664+0000","last_active":"2023-06-03T20:28:52.222535+0000","last_peered":"2023-06-03T20:28:52.222535+0000","last_clean":"2023-06-03T20:28:52.222535+0000","last_became_active":"2023-06-03T20:23:28.239252+0000","last_became_peered":"2023-06-03T20:23:28.239252+0000","last_unstale":"2023-06-03T20:28:52.222535+0000","last_undegraded":"2023-06-03T20:28:52.222535+0000","last_fullsized":"2023-06-03T20:28:52.222535+0000","mapping_epoch":251,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":252,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T06:52:20.862520+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":171,"reported_epoch":288,"state":"active+clean","last_fresh":"2023-06-03T20:28:52.252162+0000","last_change":"2023-06-03T20:24:02.259850+0000","last_active":"2023-06-03T20:28:52.252162+0000","last_peered":"2023-06-03T20:28:52.252162+0000","last_clean":"2023-06-03T20:28:52.252162+0000","last_became_active":"2023-06-03T20:24:02.258764+0000","last_became_peered":"2023-06-03T20:24:02.258764+0000","last_unstale":"2023-06-03T20:28:52.252162+0000","last_undegraded":"2023-06-03T20:28:52.252162+0000","last_fullsized":"2023-06-03T20:28:52.252162+0000","mapping_epoch":254,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":255,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T21:54:11.487473+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"60'1","reported_seq":272,"reported_epoch":288,"state":"active+clean","last_fresh":"2023-06-03T20:28:52.224527+0000","last_change":"2023-06-03T20:23:27.254853+0000","last_active":"2023-06-03T20:28:52.224527+0000","last_peered":"2023-06-03T20:28:52.224527+0000","last_clean":"2023-06-03T20:28:52.224527+0000","last_became_active":"2023-06-03T20:23:27.254410+0000","last_became_peered":"2023-06-03T20:23:27.254410+0000","last_unstale":"2023-06-03T20:28:52.224527+0000","last_undegraded":"2023-06-03T20:28:52.224527+0000","last_fullsized":"2023-06-03T20:28:52.224527+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T22:46:19.423939+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":302,"reported_epoch":288,"state":"active+clean","last_fresh":"2023-06-03T20:28:52.228635+0000","last_change":"2023-06-03T20:21:01.257656+0000","last_active":"2023-06-03T20:28:52.228635+0000","last_peered":"2023-06-03T20:28:52.228635+0000","last_clean":"2023-06-03T20:28:52.228635+0000","last_became_active":"2023-06-03T20:21:01.257107+0000","last_became_peered":"2023-06-03T20:21:01.257107+0000","last_unstale":"2023-06-03T20:28:52.228635+0000","last_undegraded":"2023-06-03T20:28:52.228635+0000","last_fullsized":"2023-06-03T20:28:52.228635+0000","mapping_epoch":235,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":236,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T06:08:18.954539+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"60'2","reported_seq":63,"reported_epoch":288,"state":"active+clean","last_fresh":"2023-06-03T20:28:52.252396+0000","last_change":"2023-06-03T20:23:28.243048+0000","last_active":"2023-06-03T20:28:52.252396+0000","last_peered":"2023-06-03T20:28:52.252396+0000","last_clean":"2023-06-03T20:28:52.252396+0000","last_became_active":"2023-06-03T20:23:28.241128+0000","last_became_peered":"2023-06-03T20:23:28.241128+0000","last_unstale":"2023-06-03T20:28:52.252396+0000","last_undegraded":"2023-06-03T20:28:52.252396+0000","last_fullsized":"2023-06-03T20:28:52.252396+0000","mapping_epoch":251,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":252,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T05:14:54.701329+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,1],"acting":[4,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":226,"reported_epoch":288,"state":"active+clean","last_fresh":"2023-06-03T20:28:52.252634+0000","last_change":"2023-06-03T20:28:24.276513+0000","last_active":"2023-06-03T20:28:52.252634+0000","last_peered":"2023-06-03T20:28:52.252634+0000","last_clean":"2023-06-03T20:28:52.252634+0000","last_became_active":"2023-06-03T20:28:24.276244+0000","last_became_peered":"2023-06-03T20:28:24.276244+0000","last_unstale":"2023-06-03T20:28:52.252634+0000","last_undegraded":"2023-06-03T20:28:52.252634+0000","last_fullsized":"2023-06-03T20:28:52.252634+0000","mapping_epoch":284,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":285,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T06:17:57.159168+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[4,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"60'2","reported_seq":76,"reported_epoch":288,"state":"active+clean","last_fresh":"2023-06-03T20:28:52.252920+0000","last_change":"2023-06-03T20:28:51.237342+0000","last_active":"2023-06-03T20:28:52.252920+0000","last_peered":"2023-06-03T20:28:52.252920+0000","last_clean":"2023-06-03T20:28:52.252920+0000","last_became_active":"2023-06-03T20:28:51.233969+0000","last_became_peered":"2023-06-03T20:28:51.233969+0000","last_unstale":"2023-06-03T20:28:52.252920+0000","last_undegraded":"2023-06-03T20:28:52.252920+0000","last_fullsized":"2023-06-03T20:28:52.252920+0000","mapping_epoch":286,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":287,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T07:30:09.503545+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":210,"reported_epoch":288,"state":"active+clean","last_fresh":"2023-06-03T20:28:52.252115+0000","last_change":"2023-06-03T20:23:27.254468+0000","last_active":"2023-06-03T20:28:52.252115+0000","last_peered":"2023-06-03T20:28:52.252115+0000","last_clean":"2023-06-03T20:28:52.252115+0000","last_became_active":"2023-06-03T20:23:27.253859+0000","last_became_peered":"2023-06-03T20:23:27.253859+0000","last_unstale":"2023-06-03T20:28:52.252115+0000","last_undegraded":"2023-06-03T20:28:52.252115+0000","last_fullsized":"2023-06-03T20:28:52.252115+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T20:15:25.066125+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"60'1","reported_seq":307,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-06-03T20:28:54.259766+0000","last_change":"2023-06-03T20:28:51.234748+0000","last_active":"2023-06-03T20:28:54.259766+0000","last_peered":"2023-06-03T20:28:54.259766+0000","last_clean":"2023-06-03T20:28:54.259766+0000","last_became_active":"2023-06-03T20:28:51.234341+0000","last_became_peered":"2023-06-03T20:28:51.234341+0000","last_unstale":"2023-06-03T20:28:54.259766+0000","last_undegraded":"2023-06-03T20:28:54.259766+0000","last_fullsized":"2023-06-03T20:28:54.259766+0000","mapping_epoch":286,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":287,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T06:08:49.901424+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":290,"reported_epoch":288,"state":"active+clean","last_fresh":"2023-06-03T20:28:52.671830+0000","last_change":"2023-06-03T19:56:54.503416+0000","last_active":"2023-06-03T20:28:52.671830+0000","last_peered":"2023-06-03T20:28:52.671830+0000","last_clean":"2023-06-03T20:28:52.671830+0000","last_became_active":"2023-06-03T19:56:54.503018+0000","last_became_peered":"2023-06-03T19:56:54.503018+0000","last_unstale":"2023-06-03T20:28:52.671830+0000","last_undegraded":"2023-06-03T20:28:52.671830+0000","last_fullsized":"2023-06-03T20:28:52.671830+0000","mapping_epoch":119,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T20:14:00.330616+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"60'4","reported_seq":261,"reported_epoch":288,"state":"active+clean","last_fresh":"2023-06-03T20:28:52.223027+0000","last_change":"2023-06-03T19:56:11.748116+0000","last_active":"2023-06-03T20:28:52.223027+0000","last_peered":"2023-06-03T20:28:52.223027+0000","last_clean":"2023-06-03T20:28:52.223027+0000","last_became_active":"2023-06-03T19:56:11.747636+0000","last_became_peered":"2023-06-03T19:56:11.747636+0000","last_unstale":"2023-06-03T20:28:52.223027+0000","last_undegraded":"2023-06-03T20:28:52.223027+0000","last_fullsized":"2023-06-03T20:28:52.223027+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":115,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T06:57:48.052055+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":210,"reported_epoch":288,"state":"active+clean","last_fresh":"2023-06-03T20:28:52.251985+0000","last_change":"2023-06-03T20:28:51.231879+0000","last_active":"2023-06-03T20:28:52.251985+0000","last_peered":"2023-06-03T20:28:52.251985+0000","last_clean":"2023-06-03T20:28:52.251985+0000","last_became_active":"2023-06-03T20:28:51.231451+0000","last_became_peered":"2023-06-03T20:28:51.231451+0000","last_unstale":"2023-06-03T20:28:52.251985+0000","last_undegraded":"2023-06-03T20:28:52.251985+0000","last_fullsized":"2023-06-03T20:28:52.251985+0000","mapping_epoch":286,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":287,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T05:47:15.344575+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":14,"num_pg":16,"stat_sum":{"num_bytes":6158417944,"num_objects":93971,"num_object_clones":0,"num_object_copies":281913,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":4428,"num_objects_unfound":0,"num_objects_dirty":93971,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":102057,"num_write_kb":6272833,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":638,"num_bytes_recovered":41811968,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":18579537920,"data_stored":18579521632,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":102057,"ondisk_log_size":102057,"up":48,"acting":48,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5486,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":62,"num_write_kb":30,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":23,"num_bytes_recovered":4856,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16458,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":30319,"internal_metadata":0},"log_size":56,"ondisk_log_size":56,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2969600,"data_stored":2951840,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":3,"num_store_stats":5},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":48,"seq":206158430908,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2084340,"kb_used_data":1129388,"kb_used_omap":0,"kb_used_meta":954944,"kb_avail":91660812,"statfs":{"total":95995035648,"available":93860671488,"internally_reserved":0,"allocated":1156493312,"data_stored":1155832098,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":977862656},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 3 20:28:29 2023","interfaces":[{"interface":"back","average":{"1min":1.998,"5min":1.6870000000000001,"15min":1.7230000000000001},"min":{"1min":0.63300000000000001,"5min":0.26400000000000001,"15min":0.25800000000000001},"max":{"1min":4.9409999999999998,"5min":7.3920000000000003,"15min":73.810000000000002},"last":0.318},{"interface":"front","average":{"1min":1.0469999999999999,"5min":1.492,"15min":1.6120000000000001},"min":{"1min":0.29999999999999999,"5min":0.27100000000000002,"15min":0.26700000000000002},"max":{"1min":2.1659999999999999,"5min":6.9109999999999996,"15min":72.878},"last":0.92600000000000005}]},{"osd":1,"last update":"Sat Jun 3 20:29:13 2023","interfaces":[{"interface":"back","average":{"1min":1.3720000000000001,"5min":2.7229999999999999,"15min":1.579},"min":{"1min":0.33500000000000002,"5min":0.313,"15min":0.308},"max":{"1min":4.2000000000000002,"5min":100.845,"15min":100.845},"last":0.71799999999999997},{"interface":"front","average":{"1min":1.4239999999999999,"5min":2.6739999999999999,"15min":1.6180000000000001},"min":{"1min":0.378,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":6.048,"5min":100.86499999999999,"15min":100.86499999999999},"last":0.77900000000000003}]},{"osd":2,"last update":"Sat Jun 3 20:29:13 2023","interfaces":[{"interface":"back","average":{"1min":1.534,"5min":3.4630000000000001,"15min":3.4079999999999999},"min":{"1min":0.32100000000000001,"5min":0.32100000000000001,"15min":0.29499999999999998},"max":{"1min":5.9669999999999996,"5min":101.185,"15min":216.93700000000001},"last":0.36799999999999999},{"interface":"front","average":{"1min":1.5349999999999999,"5min":3.488,"15min":3.5019999999999998},"min":{"1min":0.45900000000000002,"5min":0.32400000000000001,"15min":0.28699999999999998},"max":{"1min":5.266,"5min":100.61799999999999,"15min":218.119},"last":1.9299999999999999}]},{"osd":3,"last update":"Sat Jun 3 20:29:13 2023","interfaces":[{"interface":"back","average":{"1min":1.409,"5min":1.623,"15min":2.9830000000000001},"min":{"1min":0.40200000000000002,"5min":0.372,"15min":0.371},"max":{"1min":3.556,"5min":7.452,"15min":265.40499999999997},"last":0.42599999999999999},{"interface":"front","average":{"1min":1.5029999999999999,"5min":1.659,"15min":2.9470000000000001},"min":{"1min":0.38900000000000001,"5min":0.35399999999999998,"15min":0.31900000000000001},"max":{"1min":8.2360000000000007,"5min":11.656000000000001,"15min":265.43599999999998},"last":2.9329999999999998}]},{"osd":4,"last update":"Sat Jun 3 20:28:33 2023","interfaces":[{"interface":"back","average":{"1min":0.91100000000000003,"5min":1.9970000000000001,"15min":2.1869999999999998},"min":{"1min":0.38400000000000001,"5min":0.38400000000000001,"15min":0.38400000000000001},"max":{"1min":1.877,"5min":8.6010000000000009,"15min":8.6010000000000009},"last":0.54500000000000004},{"interface":"front","average":{"1min":0.81100000000000005,"5min":1.9239999999999999,"15min":1.956},"min":{"1min":0.40600000000000003,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":1.7669999999999999,"5min":9.8460000000000001,"15min":9.8460000000000001},"last":0.47499999999999998}]},{"osd":5,"last update":"Sat Jun 3 20:29:13 2023","interfaces":[{"interface":"back","average":{"1min":0.97499999999999998,"5min":1.913,"15min":1.284},"min":{"1min":0.33800000000000002,"5min":0.317,"15min":0.317},"max":{"1min":3.4929999999999999,"5min":17.696000000000002,"15min":17.696000000000002},"last":0.61299999999999999},{"interface":"front","average":{"1min":0.91800000000000004,"5min":1.8169999999999999,"15min":1.3140000000000001},"min":{"1min":0.36299999999999999,"5min":0.33600000000000002,"15min":0.29399999999999998},"max":{"1min":2.6539999999999999,"5min":8.7279999999999998,"15min":9.3710000000000004},"last":0.51200000000000001}]},{"osd":6,"last update":"Sat Jun 3 20:29:13 2023","interfaces":[{"interface":"back","average":{"1min":0.81699999999999995,"5min":1.554,"15min":1.1850000000000001},"min":{"1min":0.42599999999999999,"5min":0.36399999999999999,"15min":0.34799999999999998},"max":{"1min":1.849,"5min":9.7200000000000006,"15min":9.7200000000000006},"last":0.74099999999999999},{"interface":"front","average":{"1min":0.92700000000000005,"5min":1.5269999999999999,"15min":1.1639999999999999},"min":{"1min":0.40000000000000002,"5min":0.34300000000000003,"15min":0.34300000000000003},"max":{"1min":3.77,"5min":9.923,"15min":9.923},"last":4.3170000000000002}]}]},{"osd":6,"up_from":42,"seq":180388627141,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1809896,"kb_used_data":1218080,"kb_used_omap":0,"kb_used_meta":591808,"kb_avail":91935256,"statfs":{"total":95995035648,"available":94141702144,"internally_reserved":0,"allocated":1247313920,"data_stored":1246664557,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":606011392},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,1,0,0,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 3 20:28:43 2023","interfaces":[{"interface":"back","average":{"1min":1.9299999999999999,"5min":1.0289999999999999,"15min":1.264},"min":{"1min":0.36799999999999999,"5min":0.254,"15min":0.254},"max":{"1min":6.5730000000000004,"5min":6.5730000000000004,"15min":56.259},"last":5.5490000000000004},{"interface":"front","average":{"1min":2.198,"5min":1.1599999999999999,"15min":1.393},"min":{"1min":0.315,"5min":0.315,"15min":0.28299999999999997},"max":{"1min":6.6879999999999997,"5min":6.6879999999999997,"15min":56.234000000000002},"last":6.3079999999999998}]},{"osd":1,"last update":"Sat Jun 3 20:29:16 2023","interfaces":[{"interface":"back","average":{"1min":1.496,"5min":1.113,"15min":1.0209999999999999},"min":{"1min":0.371,"5min":0.30299999999999999,"15min":0.29199999999999998},"max":{"1min":6.7910000000000004,"5min":6.7910000000000004,"15min":30.736000000000001},"last":0.64300000000000002},{"interface":"front","average":{"1min":2.4359999999999999,"5min":1.3979999999999999,"15min":1.1859999999999999},"min":{"1min":0.34200000000000003,"5min":0.33700000000000002,"15min":0.32700000000000001},"max":{"1min":8.5869999999999997,"5min":8.5869999999999997,"15min":30.788},"last":3.181}]},{"osd":2,"last update":"Sat Jun 3 20:29:16 2023","interfaces":[{"interface":"back","average":{"1min":1.8260000000000001,"5min":1.2110000000000001,"15min":2.5070000000000001},"min":{"1min":0.34899999999999998,"5min":0.33500000000000002,"15min":0.315},"max":{"1min":8.1799999999999997,"5min":8.1799999999999997,"15min":93.558999999999997},"last":0.374},{"interface":"front","average":{"1min":2.044,"5min":1.375,"15min":2.3999999999999999},"min":{"1min":0.378,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":7.0860000000000003,"5min":7.319,"15min":94.888000000000005},"last":0.54100000000000004}]},{"osd":3,"last update":"Sat Jun 3 20:28:59 2023","interfaces":[{"interface":"back","average":{"1min":1.905,"5min":1.272,"15min":1.681},"min":{"1min":0.36899999999999999,"5min":0.36899999999999999,"15min":0.34100000000000003},"max":{"1min":4.8140000000000001,"5min":4.9459999999999997,"15min":58.615000000000002},"last":0.48099999999999998},{"interface":"front","average":{"1min":1.1910000000000001,"5min":1.135,"15min":1.6459999999999999},"min":{"1min":0.40100000000000002,"5min":0.40100000000000002,"15min":0.32500000000000001},"max":{"1min":4.4660000000000002,"5min":7.9459999999999997,"15min":58.546999999999997},"last":0.59799999999999998}]},{"osd":4,"last update":"Sat Jun 3 20:28:35 2023","interfaces":[{"interface":"back","average":{"1min":1.425,"5min":1.7290000000000001,"15min":1.8200000000000001},"min":{"1min":0.36699999999999999,"5min":0.33000000000000002,"15min":0.33000000000000002},"max":{"1min":4.8360000000000003,"5min":10.574999999999999,"15min":10.574999999999999},"last":3.2450000000000001},{"interface":"front","average":{"1min":0.97399999999999998,"5min":1.522,"15min":1.365},"min":{"1min":0.57799999999999996,"5min":0.34499999999999997,"15min":0.34499999999999997},"max":{"1min":3.3090000000000002,"5min":12.622999999999999,"15min":12.622999999999999},"last":3.6539999999999999}]},{"osd":5,"last update":"Sat Jun 3 20:28:29 2023","interfaces":[{"interface":"back","average":{"1min":0.94799999999999995,"5min":1.5129999999999999,"15min":1.022},"min":{"1min":0.45300000000000001,"5min":0.29899999999999999,"15min":0.27700000000000002},"max":{"1min":2.3740000000000001,"5min":10.625999999999999,"15min":10.625999999999999},"last":0.71999999999999997},{"interface":"front","average":{"1min":1.099,"5min":1.748,"15min":1.2390000000000001},"min":{"1min":0.45100000000000001,"5min":0.40300000000000002,"15min":0.309},"max":{"1min":2.2450000000000001,"5min":10.856,"15min":10.856},"last":3.2210000000000001}]},{"osd":7,"last update":"Sat Jun 3 20:29:02 2023","interfaces":[{"interface":"back","average":{"1min":0.85599999999999998,"5min":1.5660000000000001,"15min":1.2310000000000001},"min":{"1min":0.45200000000000001,"5min":0.44,"15min":0.315},"max":{"1min":3.5720000000000001,"5min":7.069,"15min":9.0640000000000001},"last":0.68000000000000005},{"interface":"front","average":{"1min":1.0449999999999999,"5min":1.6160000000000001,"15min":1.194},"min":{"1min":0.46899999999999997,"5min":0.39300000000000002,"15min":0.371},"max":{"1min":4.6079999999999997,"5min":10.52,"15min":10.52},"last":3.6000000000000001}]}]},{"osd":1,"up_from":103,"seq":442381631939,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3467940,"kb_used_data":3017820,"kb_used_omap":4,"kb_used_meta":450043,"kb_avail":90277212,"statfs":{"total":95995035648,"available":92443865088,"internally_reserved":0,"allocated":3090247680,"data_stored":3089136210,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":460844942},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1],"upper_bound":8},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 3 20:28:22 2023","interfaces":[{"interface":"back","average":{"1min":1.6870000000000001,"5min":1.139,"15min":2.048},"min":{"1min":0.23599999999999999,"5min":0.214,"15min":0.17100000000000001},"max":{"1min":14.005000000000001,"5min":14.005000000000001,"15min":119.94499999999999},"last":1.137},{"interface":"front","average":{"1min":1.506,"5min":1.165,"15min":2.0899999999999999},"min":{"1min":0.182,"5min":0.182,"15min":0.182},"max":{"1min":14.148,"5min":14.148,"15min":120.04300000000001},"last":2.2269999999999999}]},{"osd":2,"last update":"Sat Jun 3 20:28:22 2023","interfaces":[{"interface":"back","average":{"1min":0.65000000000000002,"5min":1.256,"15min":2.6749999999999998},"min":{"1min":0.217,"5min":0.217,"15min":0.217},"max":{"1min":1.3839999999999999,"5min":10.957000000000001,"15min":219.72300000000001},"last":0.56699999999999995},{"interface":"front","average":{"1min":0.73099999999999998,"5min":1.486,"15min":2.8079999999999998},"min":{"1min":0.251,"5min":0.23999999999999999,"15min":0.22500000000000001},"max":{"1min":2.141,"5min":10.438000000000001,"15min":219.82900000000001},"last":0.246}]},{"osd":3,"last update":"Sat Jun 3 20:28:57 2023","interfaces":[{"interface":"back","average":{"1min":2.1219999999999999,"5min":2.3519999999999999,"15min":2.0939999999999999},"min":{"1min":0.255,"5min":0.255,"15min":0.23400000000000001},"max":{"1min":14.302,"5min":95.525000000000006,"15min":96.864999999999995},"last":0.27800000000000002},{"interface":"front","average":{"1min":2.2320000000000002,"5min":2.4239999999999999,"15min":2.181},"min":{"1min":0.223,"5min":0.223,"15min":0.223},"max":{"1min":14.331,"5min":94.507000000000005,"15min":96.679000000000002},"last":1.4139999999999999}]},{"osd":4,"last update":"Sat Jun 3 20:28:39 2023","interfaces":[{"interface":"back","average":{"1min":1.333,"5min":1.72,"15min":1.7370000000000001},"min":{"1min":0.34699999999999998,"5min":0.34000000000000002,"15min":0.34000000000000002},"max":{"1min":3.9769999999999999,"5min":11.114000000000001,"15min":11.114000000000001},"last":0.35499999999999998},{"interface":"front","average":{"1min":0.998,"5min":1.409,"15min":1.4370000000000001},"min":{"1min":0.44,"5min":0.33400000000000002,"15min":0.33400000000000002},"max":{"1min":3.5529999999999999,"5min":11.247999999999999,"15min":11.247999999999999},"last":0.626}]},{"osd":5,"last update":"Sat Jun 3 20:28:22 2023","interfaces":[{"interface":"back","average":{"1min":1.1679999999999999,"5min":1.897,"15min":1.1419999999999999},"min":{"1min":0.44500000000000001,"5min":0.36299999999999999,"15min":0.30299999999999999},"max":{"1min":4.0359999999999996,"5min":11.246,"15min":11.246},"last":0.38900000000000001},{"interface":"front","average":{"1min":0.86199999999999999,"5min":1.7290000000000001,"15min":1.1299999999999999},"min":{"1min":0.38600000000000001,"5min":0.377,"15min":0.372},"max":{"1min":1.6679999999999999,"5min":10.922000000000001,"15min":10.922000000000001},"last":0.65600000000000003}]},{"osd":6,"last update":"Sat Jun 3 20:28:22 2023","interfaces":[{"interface":"back","average":{"1min":0.91600000000000004,"5min":1.117,"15min":0.98499999999999999},"min":{"1min":0.41899999999999998,"5min":0.38700000000000001,"15min":0.33900000000000002},"max":{"1min":1.95,"5min":10.002000000000001,"15min":10.002000000000001},"last":5.5899999999999999},{"interface":"front","average":{"1min":0.94699999999999995,"5min":1.4279999999999999,"15min":1.121},"min":{"1min":0.42099999999999999,"5min":0.36899999999999999,"15min":0.36899999999999999},"max":{"1min":2.5449999999999999,"5min":11.067,"15min":11.067},"last":5.609}]},{"osd":7,"last update":"Sat Jun 3 20:28:22 2023","interfaces":[{"interface":"back","average":{"1min":0.92600000000000005,"5min":1.9570000000000001,"15min":1.403},"min":{"1min":0.38300000000000001,"5min":0.38300000000000001,"15min":0.372},"max":{"1min":2.5070000000000001,"5min":10.538,"15min":10.538},"last":1.3819999999999999},{"interface":"front","average":{"1min":1.204,"5min":1.7450000000000001,"15min":1.1990000000000001},"min":{"1min":0.439,"5min":0.39200000000000002,"15min":0.34000000000000002},"max":{"1min":4.0060000000000002,"5min":14.025,"15min":14.025},"last":0.496}]}]},{"osd":0,"up_from":86,"seq":369367187965,"num_pgs":34,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3740708,"kb_used_data":3011676,"kb_used_omap":5,"kb_used_meta":729018,"kb_avail":90004444,"statfs":{"total":95995035648,"available":92164550656,"internally_reserved":0,"allocated":3083956224,"data_stored":3082787210,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":746515066},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,5,2,1,1],"upper_bound":64},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Jun 3 20:29:11 2023","interfaces":[{"interface":"back","average":{"1min":2.0379999999999998,"5min":2.4460000000000002,"15min":2.2200000000000002},"min":{"1min":0.215,"5min":0.215,"15min":0.185},"max":{"1min":5.7060000000000004,"5min":156.43600000000001,"15min":243.03200000000001},"last":6.3559999999999999},{"interface":"front","average":{"1min":3.6720000000000002,"5min":3.1099999999999999,"15min":2.5830000000000002},"min":{"1min":0.188,"5min":0.188,"15min":0.188},"max":{"1min":14.98,"5min":157.489,"15min":242.78399999999999},"last":9.593}]},{"osd":2,"last update":"Sat Jun 3 20:29:11 2023","interfaces":[{"interface":"back","average":{"1min":1.3460000000000001,"5min":1.804,"15min":1.4950000000000001},"min":{"1min":0.219,"5min":0.219,"15min":0.218},"max":{"1min":6.274,"5min":36.515000000000001,"15min":36.515000000000001},"last":1.6899999999999999},{"interface":"front","average":{"1min":2.5899999999999999,"5min":1.9299999999999999,"15min":2.6280000000000001},"min":{"1min":0.315,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":11.563000000000001,"5min":31.164000000000001,"15min":243.30099999999999},"last":0.32300000000000001}]},{"osd":3,"last update":"Sat Jun 3 20:28:50 2023","interfaces":[{"interface":"back","average":{"1min":1.214,"5min":4.4349999999999996,"15min":3.7429999999999999},"min":{"1min":0.26600000000000001,"5min":0.26600000000000001,"15min":0.248},"max":{"1min":6.9180000000000001,"5min":259.68400000000003,"15min":259.68400000000003},"last":0.68600000000000005},{"interface":"front","average":{"1min":1.6679999999999999,"5min":4.6369999999999996,"15min":3.024},"min":{"1min":0.27000000000000002,"5min":0.253,"15min":0.23799999999999999},"max":{"1min":7.1440000000000001,"5min":261.02699999999999,"15min":261.02699999999999},"last":0.35799999999999998}]},{"osd":4,"last update":"Sat Jun 3 20:28:36 2023","interfaces":[{"interface":"back","average":{"1min":2.5049999999999999,"5min":1.4550000000000001,"15min":1.5820000000000001},"min":{"1min":0.36399999999999999,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":11.49,"5min":11.49,"15min":11.49},"last":1.496},{"interface":"front","average":{"1min":1.0089999999999999,"5min":1.1639999999999999,"15min":1.629},"min":{"1min":0.34399999999999997,"5min":0.34399999999999997,"15min":0.34399999999999997},"max":{"1min":3.3679999999999999,"5min":7.21,"15min":7.21},"last":0.44}]},{"osd":5,"last update":"Sat Jun 3 20:28:26 2023","interfaces":[{"interface":"back","average":{"1min":2.3929999999999998,"5min":1.6639999999999999,"15min":2.3159999999999998},"min":{"1min":0.64200000000000002,"5min":0.34999999999999998,"15min":0.29299999999999998},"max":{"1min":11.445,"5min":20.238,"15min":243.15899999999999},"last":1.53},{"interface":"front","average":{"1min":1.0640000000000001,"5min":1.325,"15min":2.1230000000000002},"min":{"1min":0.441,"5min":0.33400000000000002,"15min":0.32300000000000001},"max":{"1min":4.5060000000000002,"5min":6.4429999999999996,"15min":243.32599999999999},"last":0.40899999999999997}]},{"osd":6,"last update":"Sat Jun 3 20:28:26 2023","interfaces":[{"interface":"back","average":{"1min":0.86699999999999999,"5min":0.95499999999999996,"15min":1.982},"min":{"1min":0.36799999999999999,"5min":0.36799999999999999,"15min":0.35399999999999998},"max":{"1min":1.786,"5min":4.9610000000000003,"15min":243.25},"last":0.51300000000000001},{"interface":"front","average":{"1min":1.083,"5min":1.0820000000000001,"15min":1.974},"min":{"1min":0.47999999999999998,"5min":0.379,"15min":0.379},"max":{"1min":4.657,"5min":6.8419999999999996,"15min":242.84399999999999},"last":0.58999999999999997}]},{"osd":7,"last update":"Sat Jun 3 20:28:26 2023","interfaces":[{"interface":"back","average":{"1min":0.94399999999999995,"5min":1.5049999999999999,"15min":2.4380000000000002},"min":{"1min":0.48799999999999999,"5min":0.40799999999999997,"15min":0.35499999999999998},"max":{"1min":1.8120000000000001,"5min":8.5289999999999999,"15min":242.93600000000001},"last":0.54000000000000004},{"interface":"front","average":{"1min":1.125,"5min":1.4930000000000001,"15min":2.3460000000000001},"min":{"1min":0.52400000000000002,"5min":0.46600000000000003,"15min":0.40000000000000002},"max":{"1min":4.3639999999999999,"5min":8.5009999999999994,"15min":243.21299999999999},"last":0.65300000000000002}]}]},{"osd":2,"up_from":114,"seq":489626272144,"num_pgs":28,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3646712,"kb_used_data":3029104,"kb_used_omap":7,"kb_used_meta":617528,"kb_avail":90098440,"statfs":{"total":95995035648,"available":92260802560,"internally_reserved":0,"allocated":3101802496,"data_stored":3100732577,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8111,"internal_metadata":632348753},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1,2,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 3 20:28:58 2023","interfaces":[{"interface":"back","average":{"1min":5.5750000000000002,"5min":1.798,"15min":2.7309999999999999},"min":{"1min":0.20399999999999999,"5min":0.193,"15min":0.17799999999999999},"max":{"1min":77.055000000000007,"5min":77.055000000000007,"15min":117.57899999999999},"last":0.58699999999999997},{"interface":"front","average":{"1min":6.0679999999999996,"5min":2.0099999999999998,"15min":2.6379999999999999},"min":{"1min":0.27900000000000003,"5min":0.20999999999999999,"15min":0.20399999999999999},"max":{"1min":76.453000000000003,"5min":76.453000000000003,"15min":117.38500000000001},"last":5.6150000000000002}]},{"osd":1,"last update":"Sat Jun 3 20:28:58 2023","interfaces":[{"interface":"back","average":{"1min":1.4059999999999999,"5min":3.0049999999999999,"15min":1.7050000000000001},"min":{"1min":0.246,"5min":0.222,"15min":0.222},"max":{"1min":4.9359999999999999,"5min":116.127,"15min":116.127},"last":1.8340000000000001},{"interface":"front","average":{"1min":1.431,"5min":3.1230000000000002,"15min":2.0430000000000001},"min":{"1min":0.39900000000000002,"5min":0.26700000000000002,"15min":0.25700000000000001},"max":{"1min":4.8460000000000001,"5min":116.461,"15min":116.461},"last":0.50800000000000001}]},{"osd":3,"last update":"Sat Jun 3 20:28:58 2023","interfaces":[{"interface":"back","average":{"1min":1.1419999999999999,"5min":1.6659999999999999,"15min":2.3940000000000001},"min":{"1min":0.22700000000000001,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":6.1699999999999999,"5min":36.695,"15min":103.714},"last":0.309},{"interface":"front","average":{"1min":1.371,"5min":1.9470000000000001,"15min":2.5059999999999998},"min":{"1min":0.28599999999999998,"5min":0.27100000000000002,"15min":0.23000000000000001},"max":{"1min":3.915,"5min":37.088999999999999,"15min":104.05200000000001},"last":0.60599999999999998}]},{"osd":4,"last update":"Sat Jun 3 20:28:34 2023","interfaces":[{"interface":"back","average":{"1min":0.81699999999999995,"5min":1.907,"15min":2.915},"min":{"1min":0.32700000000000001,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":2.194,"5min":16.561,"15min":16.561},"last":0.443},{"interface":"front","average":{"1min":0.90600000000000003,"5min":1.601,"15min":1.855},"min":{"1min":0.35099999999999998,"5min":0.35099999999999998,"15min":0.35099999999999998},"max":{"1min":1.7290000000000001,"5min":9.9339999999999993,"15min":9.9339999999999993},"last":0.65300000000000002}]},{"osd":5,"last update":"Sat Jun 3 20:28:58 2023","interfaces":[{"interface":"back","average":{"1min":0.82799999999999996,"5min":1.6990000000000001,"15min":1.617},"min":{"1min":0.39500000000000002,"5min":0.378,"15min":0.32100000000000001},"max":{"1min":1.607,"5min":17.236000000000001,"15min":17.751000000000001},"last":0.69299999999999995},{"interface":"front","average":{"1min":1.2629999999999999,"5min":1.643,"15min":1.4670000000000001},"min":{"1min":0.36299999999999999,"5min":0.316,"15min":0.316},"max":{"1min":3.8010000000000002,"5min":9.2219999999999995,"15min":17.928000000000001},"last":2.0760000000000001}]},{"osd":6,"last update":"Sat Jun 3 20:28:58 2023","interfaces":[{"interface":"back","average":{"1min":1.3400000000000001,"5min":1.371,"15min":1.4339999999999999},"min":{"1min":0.45200000000000001,"5min":0.377,"15min":0.35199999999999998},"max":{"1min":4.476,"5min":5.9160000000000004,"15min":14.331},"last":0.622},{"interface":"front","average":{"1min":0.92700000000000005,"5min":1.3069999999999999,"15min":1.306},"min":{"1min":0.39300000000000002,"5min":0.36699999999999999,"15min":0.35099999999999998},"max":{"1min":2.0230000000000001,"5min":5.4370000000000003,"15min":12.867000000000001},"last":2.048}]},{"osd":7,"last update":"Sat Jun 3 20:28:58 2023","interfaces":[{"interface":"back","average":{"1min":1.248,"5min":1.7649999999999999,"15min":1.6759999999999999},"min":{"1min":0.45500000000000002,"5min":0.44400000000000001,"15min":0.34799999999999998},"max":{"1min":3.77,"5min":10.529999999999999,"15min":14.243},"last":0.72199999999999998},{"interface":"front","average":{"1min":1.5149999999999999,"5min":1.7370000000000001,"15min":1.577},"min":{"1min":0.45300000000000001,"5min":0.45300000000000001,"15min":0.38600000000000001},"max":{"1min":3.714,"5min":8.5690000000000008,"15min":12.845000000000001},"last":2.0070000000000001}]}]},{"osd":3,"up_from":119,"seq":511101108616,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3238776,"kb_used_data":2645040,"kb_used_omap":2,"kb_used_meta":593533,"kb_avail":90506376,"statfs":{"total":95995035648,"available":92678529024,"internally_reserved":0,"allocated":2708520960,"data_stored":2707389314,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":607777845},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1],"upper_bound":8},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 3 20:28:58 2023","interfaces":[{"interface":"back","average":{"1min":9.6539999999999999,"5min":2.9510000000000001,"15min":2.7109999999999999},"min":{"1min":0.19600000000000001,"5min":0.19600000000000001,"15min":0.19600000000000001},"max":{"1min":168.33099999999999,"5min":168.33099999999999,"15min":168.33099999999999},"last":0.56899999999999995},{"interface":"front","average":{"1min":10.372,"5min":3.1179999999999999,"15min":2.782},"min":{"1min":0.26500000000000001,"5min":0.222,"15min":0.183},"max":{"1min":169.25800000000001,"5min":169.25800000000001,"15min":169.25800000000001},"last":2.5169999999999999}]},{"osd":1,"last update":"Sat Jun 3 20:28:58 2023","interfaces":[{"interface":"back","average":{"1min":1.6040000000000001,"5min":1.1890000000000001,"15min":1.512},"min":{"1min":0.245,"5min":0.22900000000000001,"15min":0.22500000000000001},"max":{"1min":4.1950000000000003,"5min":20.992000000000001,"15min":117.047},"last":0.38300000000000001},{"interface":"front","average":{"1min":2.198,"5min":1.3,"15min":1.55},"min":{"1min":0.36399999999999999,"5min":0.25800000000000001,"15min":0.20399999999999999},"max":{"1min":7.008,"5min":21.07,"15min":116.505},"last":0.32600000000000001}]},{"osd":2,"last update":"Sat Jun 3 20:28:58 2023","interfaces":[{"interface":"back","average":{"1min":1.73,"5min":2.117,"15min":2.5960000000000001},"min":{"1min":0.26600000000000001,"5min":0.25600000000000001,"15min":0.22900000000000001},"max":{"1min":5.3460000000000001,"5min":88.757999999999996,"15min":117.584},"last":2.0539999999999998},{"interface":"front","average":{"1min":2.0310000000000001,"5min":2.125,"15min":2.6280000000000001},"min":{"1min":0.28399999999999997,"5min":0.26900000000000002,"15min":0.23599999999999999},"max":{"1min":8.5069999999999997,"5min":88.106999999999999,"15min":117.01000000000001},"last":0.35199999999999998}]},{"osd":4,"last update":"Sat Jun 3 20:28:42 2023","interfaces":[{"interface":"back","average":{"1min":1.1879999999999999,"5min":1.141,"15min":1.3879999999999999},"min":{"1min":0.38,"5min":0.34599999999999997,"15min":0.34599999999999997},"max":{"1min":4.4139999999999997,"5min":6.8719999999999999,"15min":6.8719999999999999},"last":4.0389999999999997},{"interface":"front","average":{"1min":1.5580000000000001,"5min":1.655,"15min":1.7490000000000001},"min":{"1min":0.33300000000000002,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":5.9260000000000002,"5min":10.773999999999999,"15min":10.773999999999999},"last":0.54100000000000004}]},{"osd":5,"last update":"Sat Jun 3 20:28:58 2023","interfaces":[{"interface":"back","average":{"1min":1.716,"5min":1.46,"15min":1.6140000000000001},"min":{"1min":0.36299999999999999,"5min":0.36299999999999999,"15min":0.313},"max":{"1min":5.4480000000000004,"5min":7.6429999999999998,"15min":38.651000000000003},"last":4.3140000000000001},{"interface":"front","average":{"1min":1.5960000000000001,"5min":1.2509999999999999,"15min":1.391},"min":{"1min":0.376,"5min":0.34499999999999997,"15min":0.316},"max":{"1min":5.383,"5min":5.383,"15min":38.585999999999999},"last":4.4640000000000004}]},{"osd":6,"last update":"Sat Jun 3 20:28:58 2023","interfaces":[{"interface":"back","average":{"1min":1.4399999999999999,"5min":1.141,"15min":1.2929999999999999},"min":{"1min":0.44400000000000001,"5min":0.36899999999999999,"15min":0.33500000000000002},"max":{"1min":5.4189999999999996,"5min":5.4189999999999996,"15min":9.0719999999999992},"last":0.85999999999999999},{"interface":"front","average":{"1min":1.359,"5min":1.1870000000000001,"15min":1.446},"min":{"1min":0.38400000000000001,"5min":0.36699999999999999,"15min":0.32500000000000001},"max":{"1min":4.6150000000000002,"5min":5.1600000000000001,"15min":8.7279999999999998},"last":0.87}]},{"osd":7,"last update":"Sat Jun 3 20:28:58 2023","interfaces":[{"interface":"back","average":{"1min":1.492,"5min":1.4410000000000001,"15min":1.6839999999999999},"min":{"1min":0.47399999999999998,"5min":0.45000000000000001,"15min":0.35599999999999998},"max":{"1min":4.6429999999999998,"5min":6.0700000000000003,"15min":39.008000000000003},"last":0.65600000000000003},{"interface":"front","average":{"1min":1.671,"5min":1.9379999999999999,"15min":1.9159999999999999},"min":{"1min":0.38700000000000001,"5min":0.38700000000000001,"15min":0.34899999999999998},"max":{"1min":5.4660000000000002,"5min":12.831,"15min":40.003999999999998},"last":0.59599999999999997}]}]},{"osd":4,"up_from":250,"seq":1073741824072,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2517040,"kb_used_data":2291560,"kb_used_omap":10,"kb_used_meta":225461,"kb_avail":91228112,"statfs":{"total":95995035648,"available":93417586688,"internally_reserved":0,"allocated":2346557440,"data_stored":2345317252,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":10652,"internal_metadata":230872676},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,2],"upper_bound":4},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 3 20:28:37 2023","interfaces":[{"interface":"back","average":{"1min":0.96299999999999997,"5min":1.532,"15min":1.73},"min":{"1min":0.27600000000000002,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":4.2539999999999996,"5min":12.022,"15min":12.022},"last":2.226},{"interface":"front","average":{"1min":1.298,"5min":1.6160000000000001,"15min":1.5129999999999999},"min":{"1min":0.30399999999999999,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":4.0679999999999996,"5min":7.9340000000000002,"15min":7.9340000000000002},"last":0.90700000000000003}]},{"osd":1,"last update":"Sat Jun 3 20:28:37 2023","interfaces":[{"interface":"back","average":{"1min":1.2330000000000001,"5min":1.5589999999999999,"15min":1.657},"min":{"1min":0.36499999999999999,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":4.9219999999999997,"5min":8.3010000000000002,"15min":8.3010000000000002},"last":0.63300000000000001},{"interface":"front","average":{"1min":0.90600000000000003,"5min":1.6000000000000001,"15min":1.788},"min":{"1min":0.371,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":2.048,"5min":11.994,"15min":11.994},"last":1.101}]},{"osd":2,"last update":"Sat Jun 3 20:28:37 2023","interfaces":[{"interface":"back","average":{"1min":1.3879999999999999,"5min":2.5230000000000001,"15min":2.226},"min":{"1min":0.45600000000000002,"5min":0.35899999999999999,"15min":0.35899999999999999},"max":{"1min":6.1239999999999997,"5min":78.605000000000004,"15min":78.605000000000004},"last":3.4460000000000002},{"interface":"front","average":{"1min":1.6020000000000001,"5min":2.7949999999999999,"15min":2.718},"min":{"1min":0.42999999999999999,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":5.3460000000000001,"5min":73.569000000000003,"15min":73.569000000000003},"last":1.133}]},{"osd":3,"last update":"Sat Jun 3 20:28:37 2023","interfaces":[{"interface":"back","average":{"1min":2.742,"5min":4.4880000000000004,"15min":11.183},"min":{"1min":0.42199999999999999,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":33.667999999999999,"5min":217.578,"15min":217.578},"last":1.165},{"interface":"front","average":{"1min":3.1869999999999998,"5min":4.617,"15min":11.023999999999999},"min":{"1min":0.36199999999999999,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":34.094000000000001,"5min":218.55699999999999,"15min":218.55699999999999},"last":0.85199999999999998}]},{"osd":5,"last update":"Sat Jun 3 20:28:37 2023","interfaces":[{"interface":"back","average":{"1min":0.73899999999999999,"5min":2.194,"15min":3.0680000000000001},"min":{"1min":0.35099999999999998,"5min":0.35099999999999998,"15min":0.35099999999999998},"max":{"1min":2.395,"5min":23.913,"15min":23.913},"last":0.93899999999999995},{"interface":"front","average":{"1min":1.024,"5min":1.8899999999999999,"15min":2.1600000000000001},"min":{"1min":0.42599999999999999,"5min":0.34499999999999997,"15min":0.34499999999999997},"max":{"1min":4.8739999999999997,"5min":11.093999999999999,"15min":11.093999999999999},"last":1.0509999999999999}]},{"osd":6,"last update":"Sat Jun 3 20:28:37 2023","interfaces":[{"interface":"back","average":{"1min":0.80200000000000005,"5min":1.982,"15min":2.661},"min":{"1min":0.40999999999999998,"5min":0.40400000000000003,"15min":0.40400000000000003},"max":{"1min":2.569,"5min":23.963000000000001,"15min":23.963000000000001},"last":1.3240000000000001},{"interface":"front","average":{"1min":1.2470000000000001,"5min":1.595,"15min":1.464},"min":{"1min":0.42099999999999999,"5min":0.40799999999999997,"15min":0.40799999999999997},"max":{"1min":4.9610000000000003,"5min":8.4190000000000005,"15min":8.4190000000000005},"last":1.1990000000000001}]},{"osd":7,"last update":"Sat Jun 3 20:28:37 2023","interfaces":[{"interface":"back","average":{"1min":1.2849999999999999,"5min":2.1619999999999999,"15min":2.258},"min":{"1min":0.501,"5min":0.374,"15min":0.374},"max":{"1min":4.9820000000000002,"5min":10.747999999999999,"15min":10.747999999999999},"last":0.67300000000000004},{"interface":"front","average":{"1min":1.0680000000000001,"5min":1.9450000000000001,"15min":2.2189999999999999},"min":{"1min":0.46500000000000002,"5min":0.34699999999999998,"15min":0.34699999999999998},"max":{"1min":4.202,"5min":12.154,"15min":12.154},"last":1.6759999999999999}]}]},{"osd":5,"up_from":36,"seq":154618823375,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2818268,"kb_used_data":1874964,"kb_used_omap":0,"kb_used_meta":943296,"kb_avail":90926884,"statfs":{"total":95995035648,"available":93109129216,"internally_reserved":0,"allocated":1919963136,"data_stored":1919321355,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":965935104},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,1,0,1],"upper_bound":16},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 3 20:29:21 2023","interfaces":[{"interface":"back","average":{"1min":2.415,"5min":2.4430000000000001,"15min":2.448},"min":{"1min":0.30399999999999999,"5min":0.27100000000000002,"15min":0.25700000000000001},"max":{"1min":32.518999999999998,"5min":104.325,"15min":126.461},"last":0.96299999999999997},{"interface":"front","average":{"1min":2.786,"5min":2.6000000000000001,"15min":2.6309999999999998},"min":{"1min":0.34399999999999997,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":32.545000000000002,"5min":104.46599999999999,"15min":126.497},"last":0.38}]},{"osd":1,"last update":"Sat Jun 3 20:29:21 2023","interfaces":[{"interface":"back","average":{"1min":1.417,"5min":2.4020000000000001,"15min":1.9399999999999999},"min":{"1min":0.32100000000000001,"5min":0.29099999999999998,"15min":0.28799999999999998},"max":{"1min":10,"5min":55.515000000000001,"15min":102.205},"last":0.98299999999999998},{"interface":"front","average":{"1min":1.597,"5min":2.3599999999999999,"15min":1.887},"min":{"1min":0.32800000000000001,"5min":0.30499999999999999,"15min":0.27600000000000002},"max":{"1min":6.4589999999999996,"5min":55.194000000000003,"15min":102.265},"last":0.43099999999999999}]},{"osd":2,"last update":"Sat Jun 3 20:29:21 2023","interfaces":[{"interface":"back","average":{"1min":1.21,"5min":5.319,"15min":4.3209999999999997},"min":{"1min":0.40300000000000002,"5min":0.36199999999999999,"15min":0.33400000000000002},"max":{"1min":5.3159999999999998,"5min":236.40000000000001,"15min":236.40000000000001},"last":0.91000000000000003},{"interface":"front","average":{"1min":2.1899999999999999,"5min":5.7480000000000002,"15min":4.6310000000000002},"min":{"1min":0.442,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":5.931,"5min":237.86099999999999,"15min":237.86099999999999},"last":0.80500000000000005}]},{"osd":3,"last update":"Sat Jun 3 20:28:56 2023","interfaces":[{"interface":"back","average":{"1min":2.605,"5min":1.923,"15min":3.3860000000000001},"min":{"1min":0.46600000000000003,"5min":0.34399999999999997,"15min":0.34399999999999997},"max":{"1min":35.725999999999999,"5min":35.725999999999999,"15min":264.822},"last":1.03},{"interface":"front","average":{"1min":2.5720000000000001,"5min":1.649,"15min":3.2669999999999999},"min":{"1min":0.44600000000000001,"5min":0.39200000000000002,"15min":0.37},"max":{"1min":35.762999999999998,"5min":35.762999999999998,"15min":264.75},"last":1.0409999999999999}]},{"osd":4,"last update":"Sat Jun 3 20:28:35 2023","interfaces":[{"interface":"back","average":{"1min":0.79400000000000004,"5min":2.4940000000000002,"15min":3.3319999999999999},"min":{"1min":0.34699999999999998,"5min":0.34699999999999998,"15min":0.34699999999999998},"max":{"1min":3.6499999999999999,"5min":21.786000000000001,"15min":21.786000000000001},"last":1.3220000000000001},{"interface":"front","average":{"1min":0.82699999999999996,"5min":1.855,"15min":2.657},"min":{"1min":0.41499999999999998,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":1.8759999999999999,"5min":26.478000000000002,"15min":26.478000000000002},"last":3.6680000000000001}]},{"osd":6,"last update":"Sat Jun 3 20:29:21 2023","interfaces":[{"interface":"back","average":{"1min":0.76300000000000001,"5min":1.5760000000000001,"15min":1.2270000000000001},"min":{"1min":0.44,"5min":0.35499999999999998,"15min":0.29099999999999998},"max":{"1min":1.246,"5min":7.5140000000000002,"15min":9.7029999999999994},"last":1.0900000000000001},{"interface":"front","average":{"1min":0.85599999999999998,"5min":1.5580000000000001,"15min":1.304},"min":{"1min":0.53700000000000003,"5min":0.37,"15min":0.35799999999999998},"max":{"1min":1.395,"5min":6.5309999999999997,"15min":6.5309999999999997},"last":1.117}]},{"osd":7,"last update":"Sat Jun 3 20:29:10 2023","interfaces":[{"interface":"back","average":{"1min":0.83499999999999996,"5min":2.0899999999999999,"15min":1.5469999999999999},"min":{"1min":0.47999999999999998,"5min":0.47099999999999997,"15min":0.35999999999999999},"max":{"1min":1.421,"5min":15.516,"15min":15.516},"last":1.0600000000000001},{"interface":"front","average":{"1min":0.90900000000000003,"5min":1.8999999999999999,"15min":1.327},"min":{"1min":0.49199999999999999,"5min":0.47399999999999998,"15min":0.373},"max":{"1min":3.9420000000000002,"5min":9.6519999999999992,"15min":9.6519999999999992},"last":1.006}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6928,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4862,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":10652,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4786,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4932,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":14,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":3071475712,"data_stored":3071475712,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":14,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":3079208960,"data_stored":3079208960,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":14,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":3091861504,"data_stored":3091857432,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":14,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":2694316032,"data_stored":2694316032,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":14,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":2338263040,"data_stored":2338258968,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":14,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":1914834944,"data_stored":1914830872,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":14,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":1240010752,"data_stored":1240006680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":14,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":1149566976,"data_stored":1149566976,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-06-03T20:29:28.603 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph pg dump --format=json 2023-06-03T20:29:28.625 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:29:28.661 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:29:28.664 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:29:28.665 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:29:28.766 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:29:28.818 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:29:28.829 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:29:28.830 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:29:28.931 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:29:28.968 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:29:28.971 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:29:28.972 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:29:29.073 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:29:29.121 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:29:29.123 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:29:29.124 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:29:29.224 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:29:29.262 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:29:29.264 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:29:29.265 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:29:29.366 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:29:29.409 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:29:29.416 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:29:29.417 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:29:29.518 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:29:29.530 INFO:teuthology.orchestra.run.smithi089.stdout:osd.0: {} 2023-06-03T20:29:29.547 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:29:29.550 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:29:29.550 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:29:29.585 INFO:teuthology.orchestra.run.smithi089.stdout:osd.1: {} 2023-06-03T20:29:29.621 INFO:teuthology.orchestra.run.smithi089.stdout:osd.2: {} 2023-06-03T20:29:29.651 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:29:29.666 INFO:teuthology.orchestra.run.smithi089.stdout:osd.3: {} 2023-06-03T20:29:29.679 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:29:29.684 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:29:29.684 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:29:29.705 INFO:teuthology.orchestra.run.smithi089.stdout:osd.4: {} 2023-06-03T20:29:29.737 INFO:teuthology.orchestra.run.smithi089.stdout:osd.5: {} 2023-06-03T20:29:29.774 INFO:teuthology.orchestra.run.smithi089.stdout:osd.6: {} 2023-06-03T20:29:29.785 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:29:29.817 INFO:teuthology.orchestra.run.smithi089.stdout:osd.7: {} 2023-06-03T20:29:29.820 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:29:29.823 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:29:29.823 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:29:29.825 INFO:teuthology.orchestra.run.smithi089.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-06-03T20:29:29.825 INFO:teuthology.orchestra.run.smithi089.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-06-03T20:29:29.825 INFO:teuthology.orchestra.run.smithi089.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-06-03T20:29:29.825 INFO:teuthology.orchestra.run.smithi089.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-06-03T20:29:29.826 INFO:teuthology.orchestra.run.smithi089.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-06-03T20:29:29.826 INFO:teuthology.orchestra.run.smithi089.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-06-03T20:29:29.826 INFO:teuthology.orchestra.run.smithi089.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-06-03T20:29:29.826 INFO:teuthology.orchestra.run.smithi089.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-06-03T20:29:29.924 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:29:29.958 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:29:29.964 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:29:29.964 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:29:30.065 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:29:30.105 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:29:30.109 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:29:30.110 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:29:30.211 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:29:30.237 INFO:teuthology.orchestra.run.smithi089.stderr:noscrub is set 2023-06-03T20:29:30.250 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:29:30.254 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:29:30.255 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:29:30.357 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:29:30.387 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:29:30.389 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:29:30.390 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:29:30.491 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:29:30.499 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:29:30 smithi164 bash[59462]: cluster 1685824169.1579213 mgr.y (mgr.34104) 1609 : cluster 0 pgmap v1663: 89 pgs: 1 active+recovering+remapped, 88 active+clean; 5.5 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 85 B/s rd, 13 MiB/s wr, 835 op/s; 4316/271656 objects misplaced (1.589%); 1.7 MiB/s, 26 objects/s recovering 2023-06-03T20:29:30.499 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:29:30 smithi164 bash[59462]: audit 1685824169.483503 mon.c (mon.2) 660 : audit 0 from='client.? 172.21.15.89:0/3828791676' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-03T20:29:30.499 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:29:30 smithi164 bash[59462]: audit 1685824169.9301817 mon.c (mon.2) 661 : audit 1 from='client.? 172.21.15.89:0/2095810987' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-06-03T20:29:30.499 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:29:30 smithi164 bash[59462]: audit 1685824169.931385 mon.a (mon.0) 1618 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-06-03T20:29:30.528 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:29:30.529 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:29:30 smithi089 bash[75068]: cluster 1685824169.1579213 mgr.y (mgr.34104) 1609 : cluster 0 pgmap v1663: 89 pgs: 1 active+recovering+remapped, 88 active+clean; 5.5 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 85 B/s rd, 13 MiB/s wr, 835 op/s; 4316/271656 objects misplaced (1.589%); 1.7 MiB/s, 26 objects/s recovering 2023-06-03T20:29:30.529 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:29:30 smithi089 bash[75068]: audit 1685824169.483503 mon.c (mon.2) 660 : audit 0 from='client.? 172.21.15.89:0/3828791676' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-03T20:29:30.529 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:29:30 smithi089 bash[75068]: audit 1685824169.9301817 mon.c (mon.2) 661 : audit 1 from='client.? 172.21.15.89:0/2095810987' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-06-03T20:29:30.529 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:29:30 smithi089 bash[75068]: audit 1685824169.931385 mon.a (mon.0) 1618 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-06-03T20:29:30.530 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:29:30 smithi089 bash[78452]: cluster 1685824169.1579213 mgr.y (mgr.34104) 1609 : cluster 0 pgmap v1663: 89 pgs: 1 active+recovering+remapped, 88 active+clean; 5.5 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 85 B/s rd, 13 MiB/s wr, 835 op/s; 4316/271656 objects misplaced (1.589%); 1.7 MiB/s, 26 objects/s recovering 2023-06-03T20:29:30.530 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:29:30 smithi089 bash[78452]: audit 1685824169.483503 mon.c (mon.2) 660 : audit 0 from='client.? 172.21.15.89:0/3828791676' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-03T20:29:30.530 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:29:30 smithi089 bash[78452]: audit 1685824169.9301817 mon.c (mon.2) 661 : audit 1 from='client.? 172.21.15.89:0/2095810987' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-06-03T20:29:30.530 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:29:30 smithi089 bash[78452]: audit 1685824169.931385 mon.a (mon.0) 1618 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-06-03T20:29:30.533 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:29:30.534 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:29:30.635 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:29:30.666 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:29:30.671 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:29:30.672 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:29:30.773 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:29:30.803 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:29:30.806 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:29:30.806 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:29:30.907 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:29:30.938 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:29:30.940 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:29:30.941 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:29:31.042 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:29:31.070 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:29:31.075 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:29:31.075 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:29:31.176 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:29:31.213 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:29:31.215 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:29:31.216 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:29:31.247 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:29:31 smithi089 bash[75068]: cluster 1685824170.2141697 mon.a (mon.0) 1619 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-06-03T20:29:31.316 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:29:31.329 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:29:31 smithi164 bash[59462]: cluster 1685824170.2141697 mon.a (mon.0) 1619 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-06-03T20:29:31.329 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:29:31 smithi164 bash[59462]: cluster 1685824170.2205245 osd.5 (osd.5) 239 : cluster 3 failed to encode map e290 with expected crc 2023-06-03T20:29:31.330 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:29:31 smithi164 bash[59462]: audit 1685824170.2227647 mon.a (mon.0) 1620 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-06-03T20:29:31.330 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:29:31 smithi164 bash[59462]: cluster 1685824170.2228143 mon.a (mon.0) 1621 : cluster 0 osdmap e290: 8 total, 8 up, 8 in 2023-06-03T20:29:31.330 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:29:31 smithi164 bash[59462]: cluster 1685824170.2451773 osd.7 (osd.7) 237 : cluster 3 failed to encode map e290 with expected crc 2023-06-03T20:29:31.330 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:29:31 smithi164 bash[59462]: cluster 1685824170.2456186 osd.6 (osd.6) 225 : cluster 3 failed to encode map e290 with expected crc 2023-06-03T20:29:31.350 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:29:31.354 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:29:31.354 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:29:31.455 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:29:31.493 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:29:31.498 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:29:31.499 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:29:31.499 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:29:31 smithi089 bash[75068]: cluster 1685824170.2205245 osd.5 (osd.5) 239 : cluster 3 failed to encode map e290 with expected crc 2023-06-03T20:29:31.499 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:29:31 smithi089 bash[75068]: audit 1685824170.2227647 mon.a (mon.0) 1620 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-06-03T20:29:31.500 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:29:31 smithi089 bash[75068]: cluster 1685824170.2228143 mon.a (mon.0) 1621 : cluster 0 osdmap e290: 8 total, 8 up, 8 in 2023-06-03T20:29:31.500 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:29:31 smithi089 bash[75068]: cluster 1685824170.2451773 osd.7 (osd.7) 237 : cluster 3 failed to encode map e290 with expected crc 2023-06-03T20:29:31.500 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:29:31 smithi089 bash[75068]: cluster 1685824170.2456186 osd.6 (osd.6) 225 : cluster 3 failed to encode map e290 with expected crc 2023-06-03T20:29:31.500 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:29:31 smithi089 bash[78452]: cluster 1685824170.2141697 mon.a (mon.0) 1619 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-06-03T20:29:31.500 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:29:31 smithi089 bash[78452]: cluster 1685824170.2205245 osd.5 (osd.5) 239 : cluster 3 failed to encode map e290 with expected crc 2023-06-03T20:29:31.500 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:29:31 smithi089 bash[78452]: audit 1685824170.2227647 mon.a (mon.0) 1620 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-06-03T20:29:31.501 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:29:31 smithi089 bash[78452]: cluster 1685824170.2228143 mon.a (mon.0) 1621 : cluster 0 osdmap e290: 8 total, 8 up, 8 in 2023-06-03T20:29:31.501 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:29:31 smithi089 bash[78452]: cluster 1685824170.2451773 osd.7 (osd.7) 237 : cluster 3 failed to encode map e290 with expected crc 2023-06-03T20:29:31.501 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:29:31 smithi089 bash[78452]: cluster 1685824170.2456186 osd.6 (osd.6) 225 : cluster 3 failed to encode map e290 with expected crc 2023-06-03T20:29:31.600 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:29:31.627 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:29:31.631 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:29:31.632 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:29:31.733 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:29:31.764 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:29:31.767 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:29:31.768 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:29:31.868 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:29:31.900 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:29:31.904 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:29:31.905 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:29:32.006 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:29:32.047 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:29:32.051 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:29:32.053 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:29:32.153 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:29:32.239 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:29:32.240 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:29:32.241 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:29:32.342 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:29:32.384 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:29:32.387 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:29:32.388 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:29:32.489 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:29:32.517 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:29:32.522 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:29:32 smithi164 bash[59462]: cluster 1685824171.1583233 mgr.y (mgr.34104) 1610 : cluster 0 pgmap v1665: 89 pgs: 1 active+recovering+remapped, 88 active+clean; 5.5 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 320 KiB/s wr, 698 op/s; 4316/271656 objects misplaced (1.589%); 1.4 MiB/s, 21 objects/s recovering 2023-06-03T20:29:32.523 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:29:32.523 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:29:32.529 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:29:32 smithi089 bash[78452]: cluster 1685824171.1583233 mgr.y (mgr.34104) 1610 : cluster 0 pgmap v1665: 89 pgs: 1 active+recovering+remapped, 88 active+clean; 5.5 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 320 KiB/s wr, 698 op/s; 4316/271656 objects misplaced (1.589%); 1.4 MiB/s, 21 objects/s recovering 2023-06-03T20:29:32.529 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:29:32 smithi089 bash[75068]: cluster 1685824171.1583233 mgr.y (mgr.34104) 1610 : cluster 0 pgmap v1665: 89 pgs: 1 active+recovering+remapped, 88 active+clean; 5.5 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 320 KiB/s wr, 698 op/s; 4316/271656 objects misplaced (1.589%); 1.4 MiB/s, 21 objects/s recovering 2023-06-03T20:29:32.624 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:29:32.651 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:29:32.656 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:29:32.657 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:29:32.757 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:29:32.793 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:29:32.796 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:29:32.797 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:29:32.898 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:29:32.935 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:29:32.938 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:29:32.939 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:29:33.040 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:29:33.080 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:29:33.083 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:29:33.084 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:29:33.185 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:29:33.210 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:29:33.214 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:29:33.215 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:29:33.316 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:29:33.369 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:29:33.371 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:29:33.372 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:29:33.473 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:29:33.504 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:29:33.507 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:29:33.508 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:29:33.609 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:29:33.645 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:29:33.651 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:29:33.652 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:29:33.753 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:29:33.788 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:29:33.790 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:29:33.791 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:29:33.892 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:29:33.919 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:29:33.922 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:29:33.923 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:29:34.023 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:29:34.066 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:29:34.071 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:29:34.071 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:29:34.144 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:29:34 smithi089 bash[78452]: cluster 1685824173.1591117 mgr.y (mgr.34104) 1611 : cluster 0 pgmap v1666: 89 pgs: 1 active+recovering+remapped, 88 active+clean; 5.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 689 op/s; 4219/261306 objects misplaced (1.615%); 1.3 MiB/s, 20 objects/s recovering 2023-06-03T20:29:34.144 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:29:34 smithi089 bash[75068]: cluster 1685824173.1591117 mgr.y (mgr.34104) 1611 : cluster 0 pgmap v1666: 89 pgs: 1 active+recovering+remapped, 88 active+clean; 5.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 689 op/s; 4219/261306 objects misplaced (1.615%); 1.3 MiB/s, 20 objects/s recovering 2023-06-03T20:29:34.173 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:29:34.206 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:29:34.207 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:29:34.208 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:29:34.309 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:29:34.319 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:29:34 smithi164 bash[59462]: cluster 1685824173.1591117 mgr.y (mgr.34104) 1611 : cluster 0 pgmap v1666: 89 pgs: 1 active+recovering+remapped, 88 active+clean; 5.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 689 op/s; 4219/261306 objects misplaced (1.615%); 1.3 MiB/s, 20 objects/s recovering 2023-06-03T20:29:34.338 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:29:34.343 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:29:34.344 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:29:34.445 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:29:34.476 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:29:34.478 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:29:34.478 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:29:34.579 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:29:34.614 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:29:34.619 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:29:34.619 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:29:34.720 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:29:34.755 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:29:34.760 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:29:34.761 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:29:34.862 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:29:34.905 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:29:34.908 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:29:34.909 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:29:35.010 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:29:35.049 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:29:35.053 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:29:35.054 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:29:35.155 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:29:35.187 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:29:35.193 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:29:35.194 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:29:35.295 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:29:35.330 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:29:35.334 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:29:35.335 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:29:35.436 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:29:35.465 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:29:35.467 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:29:35.468 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:29:35.569 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:29:35.589 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:29:35.591 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:29:35.591 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:29:35.692 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:29:35.727 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:29:35.731 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:29:35.732 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:29:35.833 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:29:35.869 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:29:35.873 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:29:35.874 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:29:35.975 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:29:36.004 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:29:36.006 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:29:36.007 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:29:36.108 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:29:36.140 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:29:36.142 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:29:36.143 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:29:36.244 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:29:36.290 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:29:36.293 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:29:36.294 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:29:36.394 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:29:36.427 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:29:36.433 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:29:36.434 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:29:36.529 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 20:29:36 smithi089 bash[71163]: ::ffff:172.21.15.164 - - [03/Jun/2023:20:29:36] "GET /metrics HTTP/1.1" 200 229147 "" "Prometheus/2.33.4" 2023-06-03T20:29:36.529 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:29:36 smithi089 bash[78452]: cluster 1685824175.159536 mgr.y (mgr.34104) 1612 : cluster 0 pgmap v1667: 89 pgs: 1 active+recovering+remapped, 88 active+clean; 5.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 689 op/s; 4219/261306 objects misplaced (1.615%); 1.3 MiB/s, 20 objects/s recovering 2023-06-03T20:29:36.530 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:29:36 smithi089 bash[75068]: cluster 1685824175.159536 mgr.y (mgr.34104) 1612 : cluster 0 pgmap v1667: 89 pgs: 1 active+recovering+remapped, 88 active+clean; 5.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 689 op/s; 4219/261306 objects misplaced (1.615%); 1.3 MiB/s, 20 objects/s recovering 2023-06-03T20:29:36.534 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:29:36.572 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:29:36.578 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:29:36.578 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:29:36.585 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:29:36 smithi164 bash[59462]: cluster 1685824175.159536 mgr.y (mgr.34104) 1612 : cluster 0 pgmap v1667: 89 pgs: 1 active+recovering+remapped, 88 active+clean; 5.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 689 op/s; 4219/261306 objects misplaced (1.615%); 1.3 MiB/s, 20 objects/s recovering 2023-06-03T20:29:36.586 INFO:teuthology.orchestra.run.smithi089.stdout:ERROR: (22) Invalid argument 2023-06-03T20:29:36.586 INFO:teuthology.orchestra.run.smithi089.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-06-03T20:29:36.680 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:29:36.708 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:29:36.711 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:29:36.711 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:29:36.812 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:29:36.849 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:29:36.849 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:29:36.850 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:29:36.898 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph osd set nodeep-scrub 2023-06-03T20:29:36.951 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:29:36.984 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:29:36.986 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:29:36.987 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:29:37.088 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:29:37.120 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:29:37.128 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:29:37.129 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:29:37.146 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-06-03T20:29:37.230 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:29:37.262 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:29:37.265 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:29:37.266 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:29:37.367 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:29:37.429 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:29:37.433 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:29:37.433 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:29:37.534 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:29:37.564 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:29:37.566 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:29:37.567 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:29:37.668 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:29:37.709 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:29:37.710 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:29:37.711 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:29:37.812 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:29:37.843 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:29:37.846 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:29:37.847 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:29:37.948 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:29:37.978 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:29:37.980 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:29:37.981 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:29:38.082 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:29:38.124 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:29:38.127 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:29:38.127 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:29:38.228 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:29:38.265 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:29:38.269 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:29:38.270 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:29:38.372 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:29:38.380 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:29:38 smithi089 bash[78452]: audit 1685824177.1593585 mon.a (mon.0) 1622 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:29:38.381 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:29:38 smithi089 bash[78452]: cluster 1685824177.1600404 mgr.y (mgr.34104) 1613 : cluster 0 pgmap v1668: 89 pgs: 1 active+recovering+remapped, 88 active+clean; 5.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 687 op/s; 4219/259986 objects misplaced (1.623%); 1.3 MiB/s, 20 objects/s recovering 2023-06-03T20:29:38.381 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:29:38 smithi089 bash[78452]: audit 1685824177.166648 mon.c (mon.2) 662 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T20:29:38.381 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:29:38 smithi089 bash[75068]: audit 1685824177.1593585 mon.a (mon.0) 1622 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:29:38.382 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:29:38 smithi089 bash[75068]: cluster 1685824177.1600404 mgr.y (mgr.34104) 1613 : cluster 0 pgmap v1668: 89 pgs: 1 active+recovering+remapped, 88 active+clean; 5.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 687 op/s; 4219/259986 objects misplaced (1.623%); 1.3 MiB/s, 20 objects/s recovering 2023-06-03T20:29:38.382 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:29:38 smithi089 bash[75068]: audit 1685824177.166648 mon.c (mon.2) 662 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T20:29:38.405 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:29:38.411 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph daemon osd.1 dump_ops_in_flight 2023-06-03T20:29:38.415 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:29:38.416 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:29:38.517 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:29:38.526 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:29:38 smithi164 bash[59462]: audit 1685824177.1593585 mon.a (mon.0) 1622 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:29:38.526 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:29:38 smithi164 bash[59462]: cluster 1685824177.1600404 mgr.y (mgr.34104) 1613 : cluster 0 pgmap v1668: 89 pgs: 1 active+recovering+remapped, 88 active+clean; 5.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 687 op/s; 4219/259986 objects misplaced (1.623%); 1.3 MiB/s, 20 objects/s recovering 2023-06-03T20:29:38.526 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:29:38 smithi164 bash[59462]: audit 1685824177.166648 mon.c (mon.2) 662 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T20:29:38.549 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:29:38.553 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:29:38.554 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:29:38.655 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:29:38.715 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:29:38.720 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:29:38.721 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:29:38.822 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:29:38.866 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:29:38.867 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:29:38.868 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:29:38.877 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/mon.a/config 2023-06-03T20:29:38.970 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:29:39.006 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:29:39.008 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:29:39.009 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:29:39.110 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:29:39.150 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:29:39.157 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:29:39.158 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:29:39.259 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:29:39.288 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:29:39.290 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:29:39.291 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:29:39.392 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:29:39.427 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:29:39.433 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:29:39.434 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:29:39.535 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:29:39.578 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:29:39.583 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:29:39.584 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:29:39.685 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:29:39.713 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:29:39.715 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:29:39.716 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:29:39.817 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:29:39.860 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:29:39.863 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:29:39.863 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:29:39.964 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:29:39.998 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:29:40.003 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:29:40.003 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:29:40.104 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:29:40.137 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:29:40.142 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:29:40.143 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:29:40.244 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:29:40.253 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:29:40 smithi164 bash[59462]: cluster 1685824179.160823 mgr.y (mgr.34104) 1614 : cluster 0 pgmap v1669: 89 pgs: 1 active+recovering+remapped, 88 active+clean; 5.1 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 685 op/s; 4102/251082 objects misplaced (1.634%); 1.3 MiB/s, 21 objects/s recovering 2023-06-03T20:29:40.273 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:29:40.275 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:29:40.276 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:29:40.377 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:29:40.412 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:29:40.417 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:29:40.418 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:29:40.519 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:29:40.529 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:29:40 smithi089 bash[78452]: cluster 1685824179.160823 mgr.y (mgr.34104) 1614 : cluster 0 pgmap v1669: 89 pgs: 1 active+recovering+remapped, 88 active+clean; 5.1 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 685 op/s; 4102/251082 objects misplaced (1.634%); 1.3 MiB/s, 21 objects/s recovering 2023-06-03T20:29:40.529 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:29:40 smithi089 bash[75068]: cluster 1685824179.160823 mgr.y (mgr.34104) 1614 : cluster 0 pgmap v1669: 89 pgs: 1 active+recovering+remapped, 88 active+clean; 5.1 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 685 op/s; 4102/251082 objects misplaced (1.634%); 1.3 MiB/s, 21 objects/s recovering 2023-06-03T20:29:40.554 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:29:40.562 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:29:40.563 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:29:40.664 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:29:40.694 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:29:40.696 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:29:40.696 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:29:40.797 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:29:40.828 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:29:40.834 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:29:40.835 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:29:40.936 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:29:40.973 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:29:40.977 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:29:40.978 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:29:41.079 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:29:41.121 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:29:41.123 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:29:41.123 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:29:41.224 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:29:41.251 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:29:41.255 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:29:41.255 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:29:41.356 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:29:41.391 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:29:41.394 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:29:41.394 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:29:41.495 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:29:41.549 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:29:41.550 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:29:41.550 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:29:41.651 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:29:41.687 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:29:41.691 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:29:41.691 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:29:41.792 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:29:41.824 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/mon.a/config 2023-06-03T20:29:41.828 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/mon.a/config 2023-06-03T20:29:41.839 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:29:41.846 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:29:41.847 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:29:41.948 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:29:41.981 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:29:41.984 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:29:41.984 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:29:42.085 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:29:42.115 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:29:42.125 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:29:42.125 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:29:42.181 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:29:42 smithi089 bash[75068]: cluster 1685824181.1614006 mgr.y (mgr.34104) 1615 : cluster 0 pgmap v1670: 89 pgs: 1 active+recovering+remapped, 88 active+clean; 5.1 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 626 op/s; 4102/251082 objects misplaced (1.634%); 1.2 MiB/s, 19 objects/s recovering 2023-06-03T20:29:42.181 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:29:42 smithi089 bash[78452]: cluster 1685824181.1614006 mgr.y (mgr.34104) 1615 : cluster 0 pgmap v1670: 89 pgs: 1 active+recovering+remapped, 88 active+clean; 5.1 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 626 op/s; 4102/251082 objects misplaced (1.634%); 1.2 MiB/s, 19 objects/s recovering 2023-06-03T20:29:42.226 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:29:42.234 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:29:42 smithi164 bash[59462]: cluster 1685824181.1614006 mgr.y (mgr.34104) 1615 : cluster 0 pgmap v1670: 89 pgs: 1 active+recovering+remapped, 88 active+clean; 5.1 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 626 op/s; 4102/251082 objects misplaced (1.634%); 1.2 MiB/s, 19 objects/s recovering 2023-06-03T20:29:42.260 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:29:42.264 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:29:42.264 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:29:42.365 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:29:42.404 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:29:42.414 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:29:42.415 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:29:42.516 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:29:42.569 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:29:42.572 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:29:42.573 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:29:42.673 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:29:42.715 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:29:42.719 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:29:42.720 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:29:42.821 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:29:42.858 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:29:42.863 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:29:42.864 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:29:42.965 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:29:42.999 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:29:43.001 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:29:43.002 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:29:43.102 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:29:43.141 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:29:43.143 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:29:43.143 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:29:43.244 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:29:43.280 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:29:43.285 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:29:43.285 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:29:43.386 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:29:43.417 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:29:43.424 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:29:43.425 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:29:43.525 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:29:43.548 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:29:43.549 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:29:43.550 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:29:43.651 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:29:43.685 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:29:43.688 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:29:43.688 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:29:43.789 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:29:43.825 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:29:43.826 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:29:43.826 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:29:43.927 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:29:43.970 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:29:43.975 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:29:43.976 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:29:44.077 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:29:44.109 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:29:44.115 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:29:44.115 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:29:44.216 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:29:44.249 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:29:44.254 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:29:44.255 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:29:44.335 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:29:44 smithi164 bash[59462]: cluster 1685824183.1621723 mgr.y (mgr.34104) 1616 : cluster 0 pgmap v1671: 89 pgs: 1 active+recovering+remapped, 88 active+clean; 4.9 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 852 op/s; 4007/240975 objects misplaced (1.663%); 1.6 MiB/s, 25 objects/s recovering 2023-06-03T20:29:44.357 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:29:44.385 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:29:44.391 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:29:44.392 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:29:44.493 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:29:44.529 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:29:44 smithi089 bash[75068]: cluster 1685824183.1621723 mgr.y (mgr.34104) 1616 : cluster 0 pgmap v1671: 89 pgs: 1 active+recovering+remapped, 88 active+clean; 4.9 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 852 op/s; 4007/240975 objects misplaced (1.663%); 1.6 MiB/s, 25 objects/s recovering 2023-06-03T20:29:44.530 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:29:44 smithi089 bash[78452]: cluster 1685824183.1621723 mgr.y (mgr.34104) 1616 : cluster 0 pgmap v1671: 89 pgs: 1 active+recovering+remapped, 88 active+clean; 4.9 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 852 op/s; 4007/240975 objects misplaced (1.663%); 1.6 MiB/s, 25 objects/s recovering 2023-06-03T20:29:44.530 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:29:44.531 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:29:44.531 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:29:44.632 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:29:44.668 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:29:44.669 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:29:44.670 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:29:44.771 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:29:44.806 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:29:44.810 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:29:44.811 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:29:44.912 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:29:44.955 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:29:44.958 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:29:44.959 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:29:45.060 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:29:45.092 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:29:45.097 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:29:45.098 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:29:45.199 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:29:45.232 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:29:45.239 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:29:45.240 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:29:45.341 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:29:45.383 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:29:45.389 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:29:45.389 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:29:45.471 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/mon.a/config 2023-06-03T20:29:45.490 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:29:45.537 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:29:45.539 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:29:45.540 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:29:45.641 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:29:45.676 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:29:45.679 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:29:45.680 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:29:45.781 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:29:45.817 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:29:45.823 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:29:45.824 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:29:45.925 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:29:45.959 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:29:45.963 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:29:45.964 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:29:46.065 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:29:46.098 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:29:46.104 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:29:46.105 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:29:46.206 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:29:46.239 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:29:46.241 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:29:46.241 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:29:46.342 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:29:46.386 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:29:46.391 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:29:46.392 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:29:46.493 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:29:46.500 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:29:46 smithi164 bash[59462]: cluster 1685824185.1626081 mgr.y (mgr.34104) 1617 : cluster 0 pgmap v1672: 89 pgs: 1 active+recovering+remapped, 88 active+clean; 4.9 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 564 op/s; 4007/240975 objects misplaced (1.663%); 1.1 MiB/s, 17 objects/s recovering 2023-06-03T20:29:46.527 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:29:46.529 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:29:46 smithi089 bash[75068]: cluster 1685824185.1626081 mgr.y (mgr.34104) 1617 : cluster 0 pgmap v1672: 89 pgs: 1 active+recovering+remapped, 88 active+clean; 4.9 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 564 op/s; 4007/240975 objects misplaced (1.663%); 1.1 MiB/s, 17 objects/s recovering 2023-06-03T20:29:46.529 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 20:29:46 smithi089 bash[71163]: ::ffff:172.21.15.164 - - [03/Jun/2023:20:29:46] "GET /metrics HTTP/1.1" 200 229150 "" "Prometheus/2.33.4" 2023-06-03T20:29:46.530 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:29:46 smithi089 bash[78452]: cluster 1685824185.1626081 mgr.y (mgr.34104) 1617 : cluster 0 pgmap v1672: 89 pgs: 1 active+recovering+remapped, 88 active+clean; 4.9 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 564 op/s; 4007/240975 objects misplaced (1.663%); 1.1 MiB/s, 17 objects/s recovering 2023-06-03T20:29:46.531 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:29:46.532 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:29:46.633 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:29:46.675 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:29:46.679 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:29:46.679 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:29:46.780 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:29:46.812 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:29:46.814 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:29:46.815 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:29:46.916 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:29:46.962 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:29:46.966 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:29:46.967 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:29:47.068 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:29:47.094 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:29:47.100 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:29:47.101 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:29:47.201 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:29:47.229 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:29:47.230 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:29:47 smithi089 bash[75068]: audit 1685824186.2479875 mon.c (mon.2) 663 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-03T20:29:47.231 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:29:47.232 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:29:47.333 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:29:47.335 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:29:47 smithi164 bash[59462]: audit 1685824186.2479875 mon.c (mon.2) 663 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-03T20:29:47.382 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:29:47.389 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:29:47.390 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:29:47.491 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:29:47.500 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:29:47 smithi089 bash[78452]: audit 1685824186.2479875 mon.c (mon.2) 663 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-03T20:29:47.528 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:29:47.533 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:29:47.534 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:29:47.635 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:29:47.673 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:29:47.677 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:29:47.679 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:29:47.780 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:29:47.809 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:29:47.811 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:29:47.812 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:29:47.913 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:29:47.937 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:29:47.938 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:29:47.939 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:29:48.040 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:29:48.080 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:29:48.083 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:29:48.084 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:29:48.185 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:29:48.219 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:29:48.221 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:29:48.222 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:29:48.231 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:29:48 smithi089 bash[75068]: cluster 1685824187.1631305 mgr.y (mgr.34104) 1618 : cluster 0 pgmap v1673: 89 pgs: 1 active+recovering+remapped, 88 active+clean; 4.9 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 599 op/s; 4007/239712 objects misplaced (1.672%); 1.1 MiB/s, 17 objects/s recovering 2023-06-03T20:29:48.323 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:29:48.366 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:29:48.371 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:29:48.372 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:29:48.473 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:29:48.481 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:29:48 smithi164 bash[59462]: cluster 1685824187.1631305 mgr.y (mgr.34104) 1618 : cluster 0 pgmap v1673: 89 pgs: 1 active+recovering+remapped, 88 active+clean; 4.9 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 599 op/s; 4007/239712 objects misplaced (1.672%); 1.1 MiB/s, 17 objects/s recovering 2023-06-03T20:29:48.508 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:29:48.515 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:29:48.515 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:29:48.532 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:29:48 smithi089 bash[78452]: cluster 1685824187.1631305 mgr.y (mgr.34104) 1618 : cluster 0 pgmap v1673: 89 pgs: 1 active+recovering+remapped, 88 active+clean; 4.9 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 599 op/s; 4007/239712 objects misplaced (1.672%); 1.1 MiB/s, 17 objects/s recovering 2023-06-03T20:29:48.616 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:29:48.646 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:29:48.648 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:29:48.649 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:29:48.750 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:29:48.778 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:29:48.785 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:29:48.785 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:29:48.886 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:29:48.929 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:29:48.931 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:29:48.931 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:29:49.032 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:29:49.074 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:29:49.076 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:29:49.076 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:29:49.177 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:29:49.216 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:29:49.219 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:29:49.220 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:29:49.321 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:29:49.351 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:29:49.355 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:29:49.356 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:29:49.457 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:29:49.494 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:29:49.499 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:29:49.500 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:29:49.601 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:29:49.629 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:29:49.632 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:29:49.633 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:29:49.734 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:29:49.768 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:29:49.771 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:29:49.772 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:29:49.873 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:29:49.898 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:29:49.899 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:29:49.900 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:29:50.001 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:29:50.033 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:29:50.039 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:29:50.040 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:29:50.141 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:29:50.179 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:29:50.181 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:29:50.182 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:29:50.234 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:29:50 smithi164 bash[59462]: cluster 1685824189.1647484 mgr.y (mgr.34104) 1619 : cluster 0 pgmap v1674: 89 pgs: 1 active+recovering+remapped, 88 active+clean; 4.7 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 811 op/s; 3890/230754 objects misplaced (1.686%); 1.7 MiB/s, 27 objects/s recovering 2023-06-03T20:29:50.283 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:29:50.315 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:29:50.320 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:29:50.320 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:29:50.421 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:29:50.450 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:29:50.456 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:29:50.457 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:29:50.529 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:29:50 smithi089 bash[78452]: cluster 1685824189.1647484 mgr.y (mgr.34104) 1619 : cluster 0 pgmap v1674: 89 pgs: 1 active+recovering+remapped, 88 active+clean; 4.7 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 811 op/s; 3890/230754 objects misplaced (1.686%); 1.7 MiB/s, 27 objects/s recovering 2023-06-03T20:29:50.529 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:29:50 smithi089 bash[75068]: cluster 1685824189.1647484 mgr.y (mgr.34104) 1619 : cluster 0 pgmap v1674: 89 pgs: 1 active+recovering+remapped, 88 active+clean; 4.7 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 811 op/s; 3890/230754 objects misplaced (1.686%); 1.7 MiB/s, 27 objects/s recovering 2023-06-03T20:29:50.558 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:29:50.594 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:29:50.598 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:29:50.598 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:29:50.699 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:29:50.748 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:29:50.753 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:29:50.754 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:29:50.855 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:29:50.888 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:29:50.894 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:29:50.895 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:29:50.996 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:29:51.031 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:29:51.034 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:29:51.034 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:29:51.135 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:29:51.173 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:29:51.177 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:29:51.177 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:29:51.278 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:29:51.304 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:29:51.306 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:29:51.307 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:29:51.408 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:29:51.439 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:29:51.445 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:29:51.446 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:29:51.547 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:29:51.574 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:29:51.577 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:29:51.577 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:29:51.678 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:29:51.717 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:29:51.724 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:29:51.725 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:29:51.826 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:29:51.866 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:29:51.872 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:29:51.872 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:29:51.973 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:29:52.004 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:29:52.009 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:29:52.010 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:29:52.111 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:29:52.150 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:29:52.153 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:29:52.154 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:29:52.255 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:29:52.279 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:29:52 smithi089 bash[78452]: cluster 1685824191.1689086 mgr.y (mgr.34104) 1620 : cluster 0 pgmap v1675: 89 pgs: 1 active+recovering+remapped, 88 active+clean; 4.7 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 564 op/s; 3890/230754 objects misplaced (1.686%); 1.1 MiB/s, 17 objects/s recovering 2023-06-03T20:29:52.279 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:29:52 smithi089 bash[78452]: audit 1685824192.1547666 mon.c (mon.2) 664 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T20:29:52.280 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:29:52 smithi089 bash[75068]: cluster 1685824191.1689086 mgr.y (mgr.34104) 1620 : cluster 0 pgmap v1675: 89 pgs: 1 active+recovering+remapped, 88 active+clean; 4.7 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 564 op/s; 3890/230754 objects misplaced (1.686%); 1.1 MiB/s, 17 objects/s recovering 2023-06-03T20:29:52.280 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:29:52 smithi089 bash[75068]: audit 1685824192.1547666 mon.c (mon.2) 664 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T20:29:52.302 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:29:52.309 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:29:52.310 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:29:52.412 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:29:52.448 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:29:52.453 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:29:52.453 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:29:52.554 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:29:52.563 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:29:52 smithi164 bash[59462]: cluster 1685824191.1689086 mgr.y (mgr.34104) 1620 : cluster 0 pgmap v1675: 89 pgs: 1 active+recovering+remapped, 88 active+clean; 4.7 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 564 op/s; 3890/230754 objects misplaced (1.686%); 1.1 MiB/s, 17 objects/s recovering 2023-06-03T20:29:52.563 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:29:52 smithi164 bash[59462]: audit 1685824192.1547666 mon.c (mon.2) 664 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T20:29:52.586 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:29:52.588 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:29:52.589 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:29:52.690 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:29:52.730 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:29:52.736 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:29:52.736 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:29:52.837 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:29:52.869 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:29:52.871 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:29:52.872 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:29:52.973 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:29:53.010 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:29:53.013 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:29:53.014 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:29:53.115 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:29:53.146 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:29:53.153 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:29:53.153 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:29:53.254 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:29:53.302 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:29:53.308 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:29:53.309 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:29:53.409 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:29:53.439 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:29:53.442 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:29:53.442 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:29:53.543 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:29:53.589 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:29:53.594 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:29:53.595 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:29:53.696 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:29:53.728 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:29:53.737 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:29:53.738 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:29:53.839 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:29:53.871 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:29:53.875 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:29:53.875 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:29:53.976 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:29:53.984 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:29:53 smithi089 bash[78452]: audit 1685824192.8676457 mon.a (mon.0) 1623 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:29:53.985 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:29:53 smithi089 bash[78452]: audit 1685824192.8743455 mon.a (mon.0) 1624 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:29:53.985 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:29:53 smithi089 bash[78452]: cluster 1685824193.1696956 mgr.y (mgr.34104) 1621 : cluster 0 pgmap v1676: 89 pgs: 1 active+recovering+remapped, 88 active+clean; 4.5 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 822 op/s; 3765/221436 objects misplaced (1.700%); 1.8 MiB/s, 28 objects/s recovering 2023-06-03T20:29:53.985 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:29:53 smithi089 bash[75068]: audit 1685824192.8676457 mon.a (mon.0) 1623 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:29:53.986 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:29:53 smithi089 bash[75068]: audit 1685824192.8743455 mon.a (mon.0) 1624 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:29:53.986 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:29:53 smithi089 bash[75068]: cluster 1685824193.1696956 mgr.y (mgr.34104) 1621 : cluster 0 pgmap v1676: 89 pgs: 1 active+recovering+remapped, 88 active+clean; 4.5 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 822 op/s; 3765/221436 objects misplaced (1.700%); 1.8 MiB/s, 28 objects/s recovering 2023-06-03T20:29:54.022 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:29:54.027 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:29:54.028 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:29:54.085 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:29:53 smithi164 bash[59462]: audit 1685824192.8676457 mon.a (mon.0) 1623 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:29:54.085 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:29:53 smithi164 bash[59462]: audit 1685824192.8743455 mon.a (mon.0) 1624 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:29:54.085 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:29:53 smithi164 bash[59462]: cluster 1685824193.1696956 mgr.y (mgr.34104) 1621 : cluster 0 pgmap v1676: 89 pgs: 1 active+recovering+remapped, 88 active+clean; 4.5 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 822 op/s; 3765/221436 objects misplaced (1.700%); 1.8 MiB/s, 28 objects/s recovering 2023-06-03T20:29:54.128 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:29:54.164 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:29:54.169 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:29:54.170 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:29:54.271 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:29:54.308 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:29:54.311 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:29:54.311 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:29:54.412 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:29:54.450 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:29:54.454 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:29:54.454 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:29:54.555 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:29:54.598 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:29:54.603 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:29:54.604 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:29:54.704 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:29:54.735 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:29:54.739 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:29:54.740 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:29:54.840 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:29:54.872 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:29:54.874 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:29:54.875 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:29:54.976 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:29:55.016 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:29:55.019 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:29:55.020 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:29:55.121 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:29:55.157 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:29:55.162 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:29:55.163 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:29:55.264 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:29:55.298 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:29:55.300 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:29:55.301 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:29:55.402 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:29:55.435 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:29:55.437 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:29:55.437 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:29:55.538 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:29:55.589 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:29:55.596 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:29:55.596 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:29:55.697 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:29:55.729 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:29:55.732 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:29:55.732 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:29:55.833 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:29:55.869 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:29:55.872 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:29:55.873 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:29:55.973 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:29:56.009 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:29:56.014 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:29:56.014 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:29:56.115 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:29:56.173 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:29:56.179 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:29:56.180 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:29:56.281 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:29:56.311 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:29:56.316 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:29:56.317 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:29:56.419 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:29:56.429 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:29:56 smithi089 bash[75068]: cluster 1685824195.1701624 mgr.y (mgr.34104) 1622 : cluster 0 pgmap v1677: 89 pgs: 1 active+recovering+remapped, 88 active+clean; 4.5 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 542 op/s; 3765/221436 objects misplaced (1.700%); 1.3 MiB/s, 20 objects/s recovering 2023-06-03T20:29:56.430 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 20:29:56 smithi089 bash[71163]: ::ffff:172.21.15.164 - - [03/Jun/2023:20:29:56] "GET /metrics HTTP/1.1" 200 229137 "" "Prometheus/2.33.4" 2023-06-03T20:29:56.431 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:29:56 smithi089 bash[78452]: cluster 1685824195.1701624 mgr.y (mgr.34104) 1622 : cluster 0 pgmap v1677: 89 pgs: 1 active+recovering+remapped, 88 active+clean; 4.5 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 542 op/s; 3765/221436 objects misplaced (1.700%); 1.3 MiB/s, 20 objects/s recovering 2023-06-03T20:29:56.449 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:29:56.454 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:29:56.454 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:29:56.555 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:29:56.565 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:29:56 smithi164 bash[59462]: cluster 1685824195.1701624 mgr.y (mgr.34104) 1622 : cluster 0 pgmap v1677: 89 pgs: 1 active+recovering+remapped, 88 active+clean; 4.5 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 542 op/s; 3765/221436 objects misplaced (1.700%); 1.3 MiB/s, 20 objects/s recovering 2023-06-03T20:29:56.584 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:29:56.586 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:29:56.586 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:29:56.687 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:29:56.736 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:29:56.739 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:29:56.740 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:29:56.841 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:29:56.878 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:29:56.882 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:29:56.882 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:29:56.894 INFO:teuthology.orchestra.run.smithi089.stdout: 2023-06-03T20:29:56.900 INFO:teuthology.orchestra.run.smithi089.stderr:dumped all 2023-06-03T20:29:56.984 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:29:57.016 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:29:57.023 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:29:57.023 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:29:57.124 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:29:57.154 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:29:57.166 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:29:57.166 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:29:57.267 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:29:57.300 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:29:57.303 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:29:57.304 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:29:57.405 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:29:57.445 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:29:57.447 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:29:57.448 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:29:57.549 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:29:57.578 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:29:57.583 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:29:57.584 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:29:57.685 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:29:57.714 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:29:57.717 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:29:57.718 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:29:57.818 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:29:57.862 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:29:57.865 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:29:57.866 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:29:57.967 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:29:58.014 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:29:58.016 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:29:58.017 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:29:58.118 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:29:58.148 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:29:58.153 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:29:58.154 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:29:58.255 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:29:58.290 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:29:58.293 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:29:58.294 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:29:58.395 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:29:58.432 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:29:58.434 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:29:58.435 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:29:58.529 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:29:58 smithi089 bash[75068]: audit 1685824196.8757384 mgr.y (mgr.34104) 1623 : audit 0 from='client.36843 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-06-03T20:29:58.530 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:29:58 smithi089 bash[75068]: cluster 1685824197.1706634 mgr.y (mgr.34104) 1624 : cluster 0 pgmap v1678: 89 pgs: 1 active+recovering+remapped, 88 active+clean; 4.5 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 575 op/s; 3765/220254 objects misplaced (1.709%); 1.3 MiB/s, 20 objects/s recovering 2023-06-03T20:29:58.530 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:29:58 smithi089 bash[78452]: audit 1685824196.8757384 mgr.y (mgr.34104) 1623 : audit 0 from='client.36843 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-06-03T20:29:58.530 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:29:58 smithi089 bash[78452]: cluster 1685824197.1706634 mgr.y (mgr.34104) 1624 : cluster 0 pgmap v1678: 89 pgs: 1 active+recovering+remapped, 88 active+clean; 4.5 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 575 op/s; 3765/220254 objects misplaced (1.709%); 1.3 MiB/s, 20 objects/s recovering 2023-06-03T20:29:58.536 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:29:58.543 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:29:58 smithi164 bash[59462]: audit 1685824196.8757384 mgr.y (mgr.34104) 1623 : audit 0 from='client.36843 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-06-03T20:29:58.543 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:29:58 smithi164 bash[59462]: cluster 1685824197.1706634 mgr.y (mgr.34104) 1624 : cluster 0 pgmap v1678: 89 pgs: 1 active+recovering+remapped, 88 active+clean; 4.5 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 575 op/s; 3765/220254 objects misplaced (1.709%); 1.3 MiB/s, 20 objects/s recovering 2023-06-03T20:29:58.579 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:29:58.583 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:29:58.584 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:29:58.685 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:29:58.717 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:29:58.722 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:29:58.723 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:29:58.824 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:29:58.853 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:29:58.859 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:29:58.860 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:29:58.960 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:29:58.989 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:29:58.993 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:29:58.993 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:29:59.094 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:29:59.132 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:29:59.134 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:29:59.134 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:29:59.235 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:29:59.269 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:29:59.273 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:29:59.274 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:29:59.375 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:29:59.412 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:29:59.416 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:29:59.417 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:29:59.518 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:29:59.557 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:29:59.566 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:29:59.567 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:29:59.667 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:29:59.710 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:29:59.715 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:29:59.715 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:29:59.816 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:29:59.844 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:29:59.847 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:29:59.847 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:29:59.948 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:29:59.976 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:29:59.983 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:29:59.983 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:30:00.084 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:30:00.124 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:30:00.128 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:30:00.129 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:30:00.229 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:30:00.263 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:30:00.266 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:30:00.267 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:30:00.368 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:30:00.379 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:30:00 smithi164 bash[59462]: cluster 1685824199.1713452 mgr.y (mgr.34104) 1625 : cluster 0 pgmap v1679: 89 pgs: 1 active+recovering+remapped, 88 active+clean; 4.3 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 765 op/s; 3632/212118 objects misplaced (1.712%); 2.0 MiB/s, 31 objects/s recovering 2023-06-03T20:30:00.380 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:30:00 smithi164 bash[59462]: cluster 1685824200.0000796 mon.a (mon.0) 1625 : cluster 3 Health detail: HEALTH_WARN noscrub flag(s) set 2023-06-03T20:30:00.380 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:30:00 smithi164 bash[59462]: cluster 1685824200.0001116 mon.a (mon.0) 1626 : cluster 3 [WRN] OSDMAP_FLAGS: noscrub flag(s) set 2023-06-03T20:30:00.405 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:30:00.408 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:30:00.408 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:30:00.509 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:30:00.529 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:30:00 smithi089 bash[78452]: cluster 1685824199.1713452 mgr.y (mgr.34104) 1625 : cluster 0 pgmap v1679: 89 pgs: 1 active+recovering+remapped, 88 active+clean; 4.3 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 765 op/s; 3632/212118 objects misplaced (1.712%); 2.0 MiB/s, 31 objects/s recovering 2023-06-03T20:30:00.529 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:30:00 smithi089 bash[78452]: cluster 1685824200.0000796 mon.a (mon.0) 1625 : cluster 3 Health detail: HEALTH_WARN noscrub flag(s) set 2023-06-03T20:30:00.529 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:30:00 smithi089 bash[78452]: cluster 1685824200.0001116 mon.a (mon.0) 1626 : cluster 3 [WRN] OSDMAP_FLAGS: noscrub flag(s) set 2023-06-03T20:30:00.530 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:30:00 smithi089 bash[75068]: cluster 1685824199.1713452 mgr.y (mgr.34104) 1625 : cluster 0 pgmap v1679: 89 pgs: 1 active+recovering+remapped, 88 active+clean; 4.3 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 765 op/s; 3632/212118 objects misplaced (1.712%); 2.0 MiB/s, 31 objects/s recovering 2023-06-03T20:30:00.530 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:30:00 smithi089 bash[75068]: cluster 1685824200.0000796 mon.a (mon.0) 1625 : cluster 3 Health detail: HEALTH_WARN noscrub flag(s) set 2023-06-03T20:30:00.530 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:30:00 smithi089 bash[75068]: cluster 1685824200.0001116 mon.a (mon.0) 1626 : cluster 3 [WRN] OSDMAP_FLAGS: noscrub flag(s) set 2023-06-03T20:30:00.536 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:30:00.541 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:30:00.541 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:30:00.642 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:30:00.683 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:30:00.685 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:30:00.686 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:30:00.786 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:30:00.818 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:30:00.819 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:30:00.820 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:30:00.920 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:30:00.958 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:30:00.961 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:30:00.962 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:30:01.063 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:30:01.104 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:30:01.107 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:30:01.108 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:30:01.209 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:30:01.247 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:30:01.252 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:30:01.252 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:30:01.353 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:30:01.385 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:30:01.389 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:30:01.390 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:30:01.491 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:30:01.530 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:30:01.532 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:30:01.532 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:30:01.633 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:30:01.665 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:30:01.670 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:30:01.671 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:30:01.772 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:30:01.812 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:30:01.814 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:30:01.815 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:30:01.915 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:30:01.953 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:30:01.956 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:30:01.956 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:30:02.057 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:30:02.082 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:30:02.084 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:30:02.085 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:30:02.186 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:30:02.222 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:30:02.226 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:30:02.227 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:30:02.255 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:30:02 smithi164 bash[59462]: cluster 1685824201.1717713 mgr.y (mgr.34104) 1626 : cluster 0 pgmap v1680: 89 pgs: 1 active+recovering+remapped, 88 active+clean; 4.3 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 517 op/s; 3632/212118 objects misplaced (1.712%); 1.3 MiB/s, 21 objects/s recovering 2023-06-03T20:30:02.328 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:30:02.362 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:30:02.364 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:30:02.364 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:30:02.388 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:30:02 smithi089 bash[78452]: cluster 1685824201.1717713 mgr.y (mgr.34104) 1626 : cluster 0 pgmap v1680: 89 pgs: 1 active+recovering+remapped, 88 active+clean; 4.3 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 517 op/s; 3632/212118 objects misplaced (1.712%); 1.3 MiB/s, 21 objects/s recovering 2023-06-03T20:30:02.465 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:30:02.496 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:30:02.502 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:30:02.503 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:30:02.604 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:30:02.636 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:30:02.639 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:30:02 smithi089 bash[75068]: cluster 1685824201.1717713 mgr.y (mgr.34104) 1626 : cluster 0 pgmap v1680: 89 pgs: 1 active+recovering+remapped, 88 active+clean; 4.3 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 517 op/s; 3632/212118 objects misplaced (1.712%); 1.3 MiB/s, 21 objects/s recovering 2023-06-03T20:30:02.639 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:30:02.640 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:30:02.741 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:30:02.771 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:30:02.775 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:30:02.776 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:30:02.877 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:30:02.905 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:30:02.906 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:30:02.907 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:30:03.008 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:30:03.049 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:30:03.053 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:30:03.054 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:30:03.155 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:30:03.196 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:30:03.201 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:30:03.202 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:30:03.277 INFO:teuthology.orchestra.run.smithi089.stdout:{"pg_ready":true,"pg_map":{"version":1677,"stamp":"2023-06-03T20:29:55.169924+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":4836300710,"num_objects":73812,"num_object_clones":0,"num_object_copies":221436,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":3765,"num_objects_unfound":0,"num_objects_dirty":73812,"num_whiteouts":0,"num_read":309,"num_read_kb":513,"num_write":122501,"num_write_kb":6275279,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1330,"num_bytes_recovered":87038296,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":122435,"ondisk_log_size":122435,"up":267,"acting":267,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":268,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":19477748,"kb_used_data":14318068,"kb_used_omap":30,"kb_used_meta":5158881,"kb_avail":730483468,"statfs":{"total":767960285184,"available":748015071232,"internally_reserved":0,"allocated":14661701632,"data_stored":14654005573,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":31502,"internal_metadata":5282694386},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":7,"apply_latency_ms":7,"commit_latency_ns":7000000,"apply_latency_ns":7000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-426835968,"num_objects":-6513,"num_object_clones":0,"num_object_copies":-19539,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":-242,"num_objects_unfound":0,"num_objects_dirty":-6513,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6513,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":242,"num_bytes_recovered":15859712,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.008333"},"pg_stats":[{"pgid":"3.1a","version":"60'1","reported_seq":216,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-06-03T20:28:54.248793+0000","last_change":"2023-06-03T20:23:28.242261+0000","last_active":"2023-06-03T20:28:54.248793+0000","last_peered":"2023-06-03T20:28:54.248793+0000","last_clean":"2023-06-03T20:28:54.248793+0000","last_became_active":"2023-06-03T20:23:28.240045+0000","last_became_peered":"2023-06-03T20:23:28.240045+0000","last_unstale":"2023-06-03T20:28:54.248793+0000","last_undegraded":"2023-06-03T20:28:54.248793+0000","last_fullsized":"2023-06-03T20:28:54.248793+0000","mapping_epoch":251,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":252,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T05:54:33.215385+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":77,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-06-03T20:28:54.266463+0000","last_change":"2023-06-03T20:23:28.283021+0000","last_active":"2023-06-03T20:28:54.266463+0000","last_peered":"2023-06-03T20:28:54.266463+0000","last_clean":"2023-06-03T20:28:54.266463+0000","last_became_active":"2023-06-03T20:23:28.282428+0000","last_became_peered":"2023-06-03T20:23:28.282428+0000","last_unstale":"2023-06-03T20:28:54.266463+0000","last_undegraded":"2023-06-03T20:28:54.266463+0000","last_fullsized":"2023-06-03T20:28:54.266463+0000","mapping_epoch":251,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":252,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T05:01:33.115332+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":77,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-06-03T20:28:54.266338+0000","last_change":"2023-06-03T20:23:28.253821+0000","last_active":"2023-06-03T20:28:54.266338+0000","last_peered":"2023-06-03T20:28:54.266338+0000","last_clean":"2023-06-03T20:28:54.266338+0000","last_became_active":"2023-06-03T20:23:28.253448+0000","last_became_peered":"2023-06-03T20:23:28.253448+0000","last_unstale":"2023-06-03T20:28:54.266338+0000","last_undegraded":"2023-06-03T20:28:54.266338+0000","last_fullsized":"2023-06-03T20:28:54.266338+0000","mapping_epoch":251,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":252,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T23:40:02.264778+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":260,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-06-03T20:29:12.741539+0000","last_change":"2023-06-03T19:56:54.507653+0000","last_active":"2023-06-03T20:29:12.741539+0000","last_peered":"2023-06-03T20:29:12.741539+0000","last_clean":"2023-06-03T20:29:12.741539+0000","last_became_active":"2023-06-03T19:56:54.507350+0000","last_became_peered":"2023-06-03T19:56:54.507350+0000","last_unstale":"2023-06-03T20:29:12.741539+0000","last_undegraded":"2023-06-03T20:29:12.741539+0000","last_fullsized":"2023-06-03T20:29:12.741539+0000","mapping_epoch":119,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T22:33:20.065844+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":300,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-06-03T20:28:54.259860+0000","last_change":"2023-06-03T20:21:01.256894+0000","last_active":"2023-06-03T20:28:54.259860+0000","last_peered":"2023-06-03T20:28:54.259860+0000","last_clean":"2023-06-03T20:28:54.259860+0000","last_became_active":"2023-06-03T20:21:01.255903+0000","last_became_peered":"2023-06-03T20:21:01.255903+0000","last_unstale":"2023-06-03T20:28:54.259860+0000","last_undegraded":"2023-06-03T20:28:54.259860+0000","last_fullsized":"2023-06-03T20:28:54.259860+0000","mapping_epoch":235,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":236,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T20:35:27.810860+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":54,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-06-03T20:28:54.256325+0000","last_change":"2023-06-03T20:28:24.269355+0000","last_active":"2023-06-03T20:28:54.256325+0000","last_peered":"2023-06-03T20:28:54.256325+0000","last_clean":"2023-06-03T20:28:54.256325+0000","last_became_active":"2023-06-03T20:28:24.268859+0000","last_became_peered":"2023-06-03T20:28:24.268859+0000","last_unstale":"2023-06-03T20:28:54.256325+0000","last_undegraded":"2023-06-03T20:28:54.256325+0000","last_fullsized":"2023-06-03T20:28:54.256325+0000","mapping_epoch":284,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":285,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,2],"acting":[6,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":255,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-06-03T20:28:54.248945+0000","last_change":"2023-06-03T19:56:54.509490+0000","last_active":"2023-06-03T20:28:54.248945+0000","last_peered":"2023-06-03T20:28:54.248945+0000","last_clean":"2023-06-03T20:28:54.248945+0000","last_became_active":"2023-06-03T19:56:54.509145+0000","last_became_peered":"2023-06-03T19:56:54.509145+0000","last_unstale":"2023-06-03T20:28:54.248945+0000","last_undegraded":"2023-06-03T20:28:54.248945+0000","last_fullsized":"2023-06-03T20:28:54.248945+0000","mapping_epoch":119,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T07:12:03.964110+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":321,"reported_epoch":290,"state":"active+clean","last_fresh":"2023-06-03T20:29:30.229002+0000","last_change":"2023-06-03T20:28:51.233277+0000","last_active":"2023-06-03T20:29:30.229002+0000","last_peered":"2023-06-03T20:29:30.229002+0000","last_clean":"2023-06-03T20:29:30.229002+0000","last_became_active":"2023-06-03T20:28:51.232919+0000","last_became_peered":"2023-06-03T20:28:51.232919+0000","last_unstale":"2023-06-03T20:29:30.229002+0000","last_undegraded":"2023-06-03T20:29:30.229002+0000","last_fullsized":"2023-06-03T20:29:30.229002+0000","mapping_epoch":286,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":287,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T02:06:02.736415+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":222,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-06-03T20:28:54.257414+0000","last_change":"2023-06-03T20:21:01.258441+0000","last_active":"2023-06-03T20:28:54.257414+0000","last_peered":"2023-06-03T20:28:54.257414+0000","last_clean":"2023-06-03T20:28:54.257414+0000","last_became_active":"2023-06-03T20:21:01.257779+0000","last_became_peered":"2023-06-03T20:21:01.257779+0000","last_unstale":"2023-06-03T20:28:54.257414+0000","last_undegraded":"2023-06-03T20:28:54.257414+0000","last_fullsized":"2023-06-03T20:28:54.257414+0000","mapping_epoch":235,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":236,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,1],"acting":[7,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":266,"reported_epoch":290,"state":"active+clean","last_fresh":"2023-06-03T20:29:30.229256+0000","last_change":"2023-06-03T20:24:02.267331+0000","last_active":"2023-06-03T20:29:30.229256+0000","last_peered":"2023-06-03T20:29:30.229256+0000","last_clean":"2023-06-03T20:29:30.229256+0000","last_became_active":"2023-06-03T20:24:02.266997+0000","last_became_peered":"2023-06-03T20:24:02.266997+0000","last_unstale":"2023-06-03T20:29:30.229256+0000","last_undegraded":"2023-06-03T20:29:30.229256+0000","last_fullsized":"2023-06-03T20:29:30.229256+0000","mapping_epoch":254,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":255,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T22:09:18.993656+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"60'3","reported_seq":287,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-06-03T20:28:54.258000+0000","last_change":"2023-06-03T20:23:27.261246+0000","last_active":"2023-06-03T20:28:54.258000+0000","last_peered":"2023-06-03T20:28:54.258000+0000","last_clean":"2023-06-03T20:28:54.258000+0000","last_became_active":"2023-06-03T20:23:27.259201+0000","last_became_peered":"2023-06-03T20:23:27.259201+0000","last_unstale":"2023-06-03T20:28:54.258000+0000","last_undegraded":"2023-06-03T20:28:54.258000+0000","last_fullsized":"2023-06-03T20:28:54.258000+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":77,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-06-03T20:28:54.267787+0000","last_change":"2023-06-03T20:28:51.235184+0000","last_active":"2023-06-03T20:28:54.267787+0000","last_peered":"2023-06-03T20:28:54.267787+0000","last_clean":"2023-06-03T20:28:54.267787+0000","last_became_active":"2023-06-03T20:28:51.234830+0000","last_became_peered":"2023-06-03T20:28:51.234830+0000","last_unstale":"2023-06-03T20:28:54.267787+0000","last_undegraded":"2023-06-03T20:28:54.267787+0000","last_fullsized":"2023-06-03T20:28:54.267787+0000","mapping_epoch":286,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":287,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T04:40:02.479523+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"77'23","reported_seq":223,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-06-03T20:28:54.249406+0000","last_change":"2023-06-03T20:24:02.280274+0000","last_active":"2023-06-03T20:28:54.249406+0000","last_peered":"2023-06-03T20:28:54.249406+0000","last_clean":"2023-06-03T20:28:54.249406+0000","last_became_active":"2023-06-03T20:24:02.264914+0000","last_became_peered":"2023-06-03T20:24:02.264914+0000","last_unstale":"2023-06-03T20:28:54.249406+0000","last_undegraded":"2023-06-03T20:28:54.249406+0000","last_fullsized":"2023-06-03T20:28:54.249406+0000","mapping_epoch":254,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":255,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":23,"log_dups_size":0,"ondisk_log_size":23,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T07:00:35.888220+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4716,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":23,"num_write_kb":20,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4716,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":199,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-06-03T20:28:54.248772+0000","last_change":"2023-06-03T20:24:02.254850+0000","last_active":"2023-06-03T20:28:54.248772+0000","last_peered":"2023-06-03T20:28:54.248772+0000","last_clean":"2023-06-03T20:28:54.248772+0000","last_became_active":"2023-06-03T20:24:02.254409+0000","last_became_peered":"2023-06-03T20:24:02.254409+0000","last_unstale":"2023-06-03T20:28:54.248772+0000","last_undegraded":"2023-06-03T20:28:54.248772+0000","last_fullsized":"2023-06-03T20:28:54.248772+0000","mapping_epoch":254,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":255,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T19:49:53.393121+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"60'2","reported_seq":289,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-06-03T20:28:54.257567+0000","last_change":"2023-06-03T20:23:27.258749+0000","last_active":"2023-06-03T20:28:54.257567+0000","last_peered":"2023-06-03T20:28:54.257567+0000","last_clean":"2023-06-03T20:28:54.257567+0000","last_became_active":"2023-06-03T20:23:27.256899+0000","last_became_peered":"2023-06-03T20:23:27.256899+0000","last_unstale":"2023-06-03T20:28:54.257567+0000","last_undegraded":"2023-06-03T20:28:54.257567+0000","last_fullsized":"2023-06-03T20:28:54.257567+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":285,"reported_epoch":290,"state":"active+clean","last_fresh":"2023-06-03T20:29:30.230703+0000","last_change":"2023-06-03T20:21:01.256612+0000","last_active":"2023-06-03T20:29:30.230703+0000","last_peered":"2023-06-03T20:29:30.230703+0000","last_clean":"2023-06-03T20:29:30.230703+0000","last_became_active":"2023-06-03T20:21:01.256243+0000","last_became_peered":"2023-06-03T20:21:01.256243+0000","last_unstale":"2023-06-03T20:29:30.230703+0000","last_undegraded":"2023-06-03T20:29:30.230703+0000","last_fullsized":"2023-06-03T20:29:30.230703+0000","mapping_epoch":235,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":236,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T05:24:47.054342+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":256,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-06-03T20:29:07.730720+0000","last_change":"2023-06-03T19:56:54.507479+0000","last_active":"2023-06-03T20:29:07.730720+0000","last_peered":"2023-06-03T20:29:07.730720+0000","last_clean":"2023-06-03T20:29:07.730720+0000","last_became_active":"2023-06-03T19:56:54.507149+0000","last_became_peered":"2023-06-03T19:56:54.507149+0000","last_unstale":"2023-06-03T20:29:07.730720+0000","last_undegraded":"2023-06-03T20:29:07.730720+0000","last_fullsized":"2023-06-03T20:29:07.730720+0000","mapping_epoch":119,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T20:22:36.676317+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":156,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-06-03T20:28:54.258134+0000","last_change":"2023-06-03T20:17:22.832844+0000","last_active":"2023-06-03T20:28:54.258134+0000","last_peered":"2023-06-03T20:28:54.258134+0000","last_clean":"2023-06-03T20:28:54.258134+0000","last_became_active":"2023-06-03T20:17:22.832473+0000","last_became_peered":"2023-06-03T20:17:22.832473+0000","last_unstale":"2023-06-03T20:28:54.258134+0000","last_undegraded":"2023-06-03T20:28:54.258134+0000","last_fullsized":"2023-06-03T20:28:54.258134+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,3],"acting":[7,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":289,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-06-03T20:28:54.257257+0000","last_change":"2023-06-03T20:24:02.267531+0000","last_active":"2023-06-03T20:28:54.257257+0000","last_peered":"2023-06-03T20:28:54.257257+0000","last_clean":"2023-06-03T20:28:54.257257+0000","last_became_active":"2023-06-03T20:24:02.267159+0000","last_became_peered":"2023-06-03T20:24:02.267159+0000","last_unstale":"2023-06-03T20:28:54.257257+0000","last_undegraded":"2023-06-03T20:28:54.257257+0000","last_fullsized":"2023-06-03T20:28:54.257257+0000","mapping_epoch":254,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":255,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":256,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-06-03T20:28:54.248622+0000","last_change":"2023-06-03T19:56:54.510289+0000","last_active":"2023-06-03T20:28:54.248622+0000","last_peered":"2023-06-03T20:28:54.248622+0000","last_clean":"2023-06-03T20:28:54.248622+0000","last_became_active":"2023-06-03T19:56:54.509905+0000","last_became_peered":"2023-06-03T19:56:54.509905+0000","last_unstale":"2023-06-03T20:28:54.248622+0000","last_undegraded":"2023-06-03T20:28:54.248622+0000","last_fullsized":"2023-06-03T20:28:54.248622+0000","mapping_epoch":119,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T02:54:26.792401+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"60'8","reported_seq":55,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-06-03T20:28:54.255732+0000","last_change":"2023-06-03T20:26:59.682623+0000","last_active":"2023-06-03T20:28:54.255732+0000","last_peered":"2023-06-03T20:28:54.255732+0000","last_clean":"2023-06-03T20:28:54.255732+0000","last_became_active":"2023-06-03T20:24:02.260105+0000","last_became_peered":"2023-06-03T20:24:02.260105+0000","last_unstale":"2023-06-03T20:28:54.255732+0000","last_undegraded":"2023-06-03T20:28:54.255732+0000","last_fullsized":"2023-06-03T20:28:54.255732+0000","mapping_epoch":254,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":273,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":293,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-06-03T20:28:54.260128+0000","last_change":"2023-06-03T20:28:24.281308+0000","last_active":"2023-06-03T20:28:54.260128+0000","last_peered":"2023-06-03T20:28:54.260128+0000","last_clean":"2023-06-03T20:28:54.260128+0000","last_became_active":"2023-06-03T20:28:24.280981+0000","last_became_peered":"2023-06-03T20:28:24.280981+0000","last_unstale":"2023-06-03T20:28:54.260128+0000","last_undegraded":"2023-06-03T20:28:54.260128+0000","last_fullsized":"2023-06-03T20:28:54.260128+0000","mapping_epoch":284,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":285,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T02:44:14.890147+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"14.2","version":"290'7451","reported_seq":7471,"reported_epoch":290,"state":"active+clean","last_fresh":"2023-06-03T20:29:52.785164+0000","last_change":"2023-06-03T20:27:21.371612+0000","last_active":"2023-06-03T20:29:52.785164+0000","last_peered":"2023-06-03T20:29:52.785164+0000","last_clean":"2023-06-03T20:29:52.785164+0000","last_became_active":"2023-06-03T20:27:21.371344+0000","last_became_peered":"2023-06-03T20:27:21.371344+0000","last_unstale":"2023-06-03T20:29:52.785164+0000","last_undegraded":"2023-06-03T20:29:52.785164+0000","last_fullsized":"2023-06-03T20:29:52.785164+0000","mapping_epoch":278,"log_start":"0'0","ondisk_log_start":"0'0","created":278,"last_epoch_clean":279,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T20:27:20.339366+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T20:27:20.339366+0000","last_clean_scrub_stamp":"2023-06-03T20:27:20.339366+0000","objects_scrubbed":0,"log_size":7451,"log_dups_size":0,"ondisk_log_size":7451,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":300089344,"num_objects":4579,"num_object_clones":0,"num_object_copies":13737,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4579,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7451,"num_write_kb":384960,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,3],"acting":[6,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.f","version":"60'1","reported_seq":288,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-06-03T20:28:54.257555+0000","last_change":"2023-06-03T20:23:27.263667+0000","last_active":"2023-06-03T20:28:54.257555+0000","last_peered":"2023-06-03T20:28:54.257555+0000","last_clean":"2023-06-03T20:28:54.257555+0000","last_became_active":"2023-06-03T20:23:27.263110+0000","last_became_peered":"2023-06-03T20:23:27.263110+0000","last_unstale":"2023-06-03T20:28:54.257555+0000","last_undegraded":"2023-06-03T20:28:54.257555+0000","last_fullsized":"2023-06-03T20:28:54.257555+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":249,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-06-03T20:28:54.257799+0000","last_change":"2023-06-03T20:28:51.231222+0000","last_active":"2023-06-03T20:28:54.257799+0000","last_peered":"2023-06-03T20:28:54.257799+0000","last_clean":"2023-06-03T20:28:54.257799+0000","last_became_active":"2023-06-03T20:28:51.230848+0000","last_became_peered":"2023-06-03T20:28:51.230848+0000","last_unstale":"2023-06-03T20:28:54.257799+0000","last_undegraded":"2023-06-03T20:28:54.257799+0000","last_fullsized":"2023-06-03T20:28:54.257799+0000","mapping_epoch":286,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":287,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"14.3","version":"290'7675","reported_seq":7697,"reported_epoch":290,"state":"active+clean","last_fresh":"2023-06-03T20:29:52.763788+0000","last_change":"2023-06-03T20:27:21.374055+0000","last_active":"2023-06-03T20:29:52.763788+0000","last_peered":"2023-06-03T20:29:52.763788+0000","last_clean":"2023-06-03T20:29:52.763788+0000","last_became_active":"2023-06-03T20:27:21.373728+0000","last_became_peered":"2023-06-03T20:27:21.373728+0000","last_unstale":"2023-06-03T20:29:52.763788+0000","last_undegraded":"2023-06-03T20:29:52.763788+0000","last_fullsized":"2023-06-03T20:29:52.763788+0000","mapping_epoch":278,"log_start":"0'0","ondisk_log_start":"0'0","created":278,"last_epoch_clean":279,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T20:27:20.339366+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T20:27:20.339366+0000","last_clean_scrub_stamp":"2023-06-03T20:27:20.339366+0000","objects_scrubbed":0,"log_size":7675,"log_dups_size":0,"ondisk_log_size":7675,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T05:15:33.026018+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":305725440,"num_objects":4665,"num_object_clones":0,"num_object_copies":13995,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4665,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7675,"num_write_kb":394880,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.e","version":"60'2","reported_seq":317,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-06-03T20:28:54.257690+0000","last_change":"2023-06-03T20:23:28.606751+0000","last_active":"2023-06-03T20:28:54.257690+0000","last_peered":"2023-06-03T20:28:54.257690+0000","last_clean":"2023-06-03T20:28:54.257690+0000","last_became_active":"2023-06-03T20:23:28.605968+0000","last_became_peered":"2023-06-03T20:23:28.605968+0000","last_unstale":"2023-06-03T20:28:54.257690+0000","last_undegraded":"2023-06-03T20:28:54.257690+0000","last_fullsized":"2023-06-03T20:28:54.257690+0000","mapping_epoch":251,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":252,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":228,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-06-03T20:28:54.249049+0000","last_change":"2023-06-03T20:23:28.249694+0000","last_active":"2023-06-03T20:28:54.249049+0000","last_peered":"2023-06-03T20:28:54.249049+0000","last_clean":"2023-06-03T20:28:54.249049+0000","last_became_active":"2023-06-03T20:23:28.249388+0000","last_became_peered":"2023-06-03T20:23:28.249388+0000","last_unstale":"2023-06-03T20:28:54.249049+0000","last_undegraded":"2023-06-03T20:28:54.249049+0000","last_fullsized":"2023-06-03T20:28:54.249049+0000","mapping_epoch":251,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":252,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T05:44:01.835586+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"14.0","version":"290'7527","reported_seq":7549,"reported_epoch":290,"state":"active+clean","last_fresh":"2023-06-03T20:29:52.357936+0000","last_change":"2023-06-03T20:27:21.378346+0000","last_active":"2023-06-03T20:29:52.357936+0000","last_peered":"2023-06-03T20:29:52.357936+0000","last_clean":"2023-06-03T20:29:52.357936+0000","last_became_active":"2023-06-03T20:27:21.377969+0000","last_became_peered":"2023-06-03T20:27:21.377969+0000","last_unstale":"2023-06-03T20:29:52.357936+0000","last_undegraded":"2023-06-03T20:29:52.357936+0000","last_fullsized":"2023-06-03T20:29:52.357936+0000","mapping_epoch":278,"log_start":"0'0","ondisk_log_start":"0'0","created":278,"last_epoch_clean":279,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T20:27:20.339366+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T20:27:20.339366+0000","last_clean_scrub_stamp":"2023-06-03T20:27:20.339366+0000","objects_scrubbed":0,"log_size":7527,"log_dups_size":0,"ondisk_log_size":7527,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T01:32:49.543336+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":298647552,"num_objects":4557,"num_object_clones":0,"num_object_copies":13671,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4557,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7527,"num_write_kb":386688,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,5],"acting":[2,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.d","version":"60'1","reported_seq":266,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-06-03T20:28:54.257825+0000","last_change":"2023-06-03T20:26:59.686037+0000","last_active":"2023-06-03T20:28:54.257825+0000","last_peered":"2023-06-03T20:28:54.257825+0000","last_clean":"2023-06-03T20:28:54.257825+0000","last_became_active":"2023-06-03T20:24:02.268002+0000","last_became_peered":"2023-06-03T20:24:02.268002+0000","last_unstale":"2023-06-03T20:28:54.257825+0000","last_undegraded":"2023-06-03T20:28:54.257825+0000","last_fullsized":"2023-06-03T20:28:54.257825+0000","mapping_epoch":254,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":273,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":140,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":13,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-06-03T20:28:54.256730+0000","last_change":"2023-06-03T20:28:51.230833+0000","last_active":"2023-06-03T20:28:54.256730+0000","last_peered":"2023-06-03T20:28:54.256730+0000","last_clean":"2023-06-03T20:28:54.256730+0000","last_became_active":"2023-06-03T20:28:51.230489+0000","last_became_peered":"2023-06-03T20:28:51.230489+0000","last_unstale":"2023-06-03T20:28:54.256730+0000","last_undegraded":"2023-06-03T20:28:54.256730+0000","last_fullsized":"2023-06-03T20:28:54.256730+0000","mapping_epoch":286,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":287,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"14.1","version":"290'7931","reported_seq":7953,"reported_epoch":290,"state":"active+clean","last_fresh":"2023-06-03T20:29:52.701145+0000","last_change":"2023-06-03T20:27:21.372641+0000","last_active":"2023-06-03T20:29:52.701145+0000","last_peered":"2023-06-03T20:29:52.701145+0000","last_clean":"2023-06-03T20:29:52.701145+0000","last_became_active":"2023-06-03T20:27:21.372158+0000","last_became_peered":"2023-06-03T20:27:21.372158+0000","last_unstale":"2023-06-03T20:29:52.701145+0000","last_undegraded":"2023-06-03T20:29:52.701145+0000","last_fullsized":"2023-06-03T20:29:52.701145+0000","mapping_epoch":278,"log_start":"0'0","ondisk_log_start":"0'0","created":278,"last_epoch_clean":279,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T20:27:20.339366+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T20:27:20.339366+0000","last_clean_scrub_stamp":"2023-06-03T20:27:20.339366+0000","objects_scrubbed":0,"log_size":7931,"log_dups_size":0,"ondisk_log_size":7931,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T07:52:41.694374+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":315293696,"num_objects":4811,"num_object_clones":0,"num_object_copies":14433,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4811,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7931,"num_write_kb":407744,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":282,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-06-03T20:28:54.260467+0000","last_change":"2023-06-03T20:24:02.261252+0000","last_active":"2023-06-03T20:28:54.260467+0000","last_peered":"2023-06-03T20:28:54.260467+0000","last_clean":"2023-06-03T20:28:54.260467+0000","last_became_active":"2023-06-03T20:24:02.260824+0000","last_became_peered":"2023-06-03T20:24:02.260824+0000","last_unstale":"2023-06-03T20:28:54.260467+0000","last_undegraded":"2023-06-03T20:28:54.260467+0000","last_fullsized":"2023-06-03T20:28:54.260467+0000","mapping_epoch":254,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":255,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T20:00:53.584753+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":320,"reported_epoch":290,"state":"active+clean","last_fresh":"2023-06-03T20:29:30.243839+0000","last_change":"2023-06-03T20:23:28.283360+0000","last_active":"2023-06-03T20:29:30.243839+0000","last_peered":"2023-06-03T20:29:30.243839+0000","last_clean":"2023-06-03T20:29:30.243839+0000","last_became_active":"2023-06-03T20:23:28.283035+0000","last_became_peered":"2023-06-03T20:23:28.283035+0000","last_unstale":"2023-06-03T20:29:30.243839+0000","last_undegraded":"2023-06-03T20:29:30.243839+0000","last_fullsized":"2023-06-03T20:29:30.243839+0000","mapping_epoch":251,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":252,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T00:22:59.446515+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"14.6","version":"290'7688","reported_seq":7710,"reported_epoch":290,"state":"active+clean","last_fresh":"2023-06-03T20:29:52.713816+0000","last_change":"2023-06-03T20:27:21.375836+0000","last_active":"2023-06-03T20:29:52.713816+0000","last_peered":"2023-06-03T20:29:52.713816+0000","last_clean":"2023-06-03T20:29:52.713816+0000","last_became_active":"2023-06-03T20:27:21.375522+0000","last_became_peered":"2023-06-03T20:27:21.375522+0000","last_unstale":"2023-06-03T20:29:52.713816+0000","last_undegraded":"2023-06-03T20:29:52.713816+0000","last_fullsized":"2023-06-03T20:29:52.713816+0000","mapping_epoch":278,"log_start":"0'0","ondisk_log_start":"0'0","created":278,"last_epoch_clean":279,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T20:27:20.339366+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T20:27:20.339366+0000","last_clean_scrub_stamp":"2023-06-03T20:27:20.339366+0000","objects_scrubbed":0,"log_size":7688,"log_dups_size":0,"ondisk_log_size":7688,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T21:02:49.138563+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":300941312,"num_objects":4592,"num_object_clones":0,"num_object_copies":13776,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4592,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7688,"num_write_kb":392960,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,1],"acting":[0,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.b","version":"60'2","reported_seq":283,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-06-03T20:28:54.260739+0000","last_change":"2023-06-03T20:23:27.254179+0000","last_active":"2023-06-03T20:28:54.260739+0000","last_peered":"2023-06-03T20:28:54.260739+0000","last_clean":"2023-06-03T20:28:54.260739+0000","last_became_active":"2023-06-03T20:23:27.253791+0000","last_became_peered":"2023-06-03T20:23:27.253791+0000","last_unstale":"2023-06-03T20:28:54.260739+0000","last_undegraded":"2023-06-03T20:28:54.260739+0000","last_fullsized":"2023-06-03T20:28:54.260739+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T00:31:27.336760+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":204,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-06-03T20:28:54.249272+0000","last_change":"2023-06-03T20:24:02.255551+0000","last_active":"2023-06-03T20:28:54.249272+0000","last_peered":"2023-06-03T20:28:54.249272+0000","last_clean":"2023-06-03T20:28:54.249272+0000","last_became_active":"2023-06-03T20:24:02.255209+0000","last_became_peered":"2023-06-03T20:24:02.255209+0000","last_unstale":"2023-06-03T20:28:54.249272+0000","last_undegraded":"2023-06-03T20:28:54.249272+0000","last_fullsized":"2023-06-03T20:28:54.249272+0000","mapping_epoch":254,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":255,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T20:05:02.741217+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"14.7","version":"290'7663","reported_seq":7685,"reported_epoch":290,"state":"active+clean","last_fresh":"2023-06-03T20:29:51.225713+0000","last_change":"2023-06-03T20:27:21.364098+0000","last_active":"2023-06-03T20:29:51.225713+0000","last_peered":"2023-06-03T20:29:51.225713+0000","last_clean":"2023-06-03T20:29:51.225713+0000","last_became_active":"2023-06-03T20:27:21.363548+0000","last_became_peered":"2023-06-03T20:27:21.363548+0000","last_unstale":"2023-06-03T20:29:51.225713+0000","last_undegraded":"2023-06-03T20:29:51.225713+0000","last_fullsized":"2023-06-03T20:29:51.225713+0000","mapping_epoch":278,"log_start":"0'0","ondisk_log_start":"0'0","created":278,"last_epoch_clean":279,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T20:27:20.339366+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T20:27:20.339366+0000","last_clean_scrub_stamp":"2023-06-03T20:27:20.339366+0000","objects_scrubbed":0,"log_size":7663,"log_dups_size":0,"ondisk_log_size":7663,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T06:05:40.316567+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":305594368,"num_objects":4663,"num_object_clones":0,"num_object_copies":13989,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4663,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7663,"num_write_kb":394432,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,0],"acting":[4,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":45,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-06-03T20:28:54.256959+0000","last_change":"2023-06-03T20:24:02.254172+0000","last_active":"2023-06-03T20:28:54.256959+0000","last_peered":"2023-06-03T20:28:54.256959+0000","last_clean":"2023-06-03T20:28:54.256959+0000","last_became_active":"2023-06-03T20:24:02.253797+0000","last_became_peered":"2023-06-03T20:24:02.253797+0000","last_unstale":"2023-06-03T20:28:54.256959+0000","last_undegraded":"2023-06-03T20:28:54.256959+0000","last_fullsized":"2023-06-03T20:28:54.256959+0000","mapping_epoch":254,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":255,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":209,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-06-03T20:28:54.249368+0000","last_change":"2023-06-03T20:23:28.291250+0000","last_active":"2023-06-03T20:28:54.249368+0000","last_peered":"2023-06-03T20:28:54.249368+0000","last_clean":"2023-06-03T20:28:54.249368+0000","last_became_active":"2023-06-03T20:23:28.290120+0000","last_became_peered":"2023-06-03T20:23:28.290120+0000","last_unstale":"2023-06-03T20:28:54.249368+0000","last_undegraded":"2023-06-03T20:28:54.249368+0000","last_fullsized":"2023-06-03T20:28:54.249368+0000","mapping_epoch":251,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":252,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T22:37:56.033327+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"14.4","version":"290'7586","reported_seq":7606,"reported_epoch":290,"state":"active+clean","last_fresh":"2023-06-03T20:29:52.770241+0000","last_change":"2023-06-03T20:27:21.366337+0000","last_active":"2023-06-03T20:29:52.770241+0000","last_peered":"2023-06-03T20:29:52.770241+0000","last_clean":"2023-06-03T20:29:52.770241+0000","last_became_active":"2023-06-03T20:27:21.365886+0000","last_became_peered":"2023-06-03T20:27:21.365886+0000","last_unstale":"2023-06-03T20:29:52.770241+0000","last_undegraded":"2023-06-03T20:29:52.770241+0000","last_fullsized":"2023-06-03T20:29:52.770241+0000","mapping_epoch":278,"log_start":"0'0","ondisk_log_start":"0'0","created":278,"last_epoch_clean":279,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T20:27:20.339366+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T20:27:20.339366+0000","last_clean_scrub_stamp":"2023-06-03T20:27:20.339366+0000","objects_scrubbed":0,"log_size":7586,"log_dups_size":0,"ondisk_log_size":7586,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":302252032,"num_objects":4612,"num_object_clones":0,"num_object_copies":13836,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4612,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7586,"num_write_kb":390336,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,3],"acting":[6,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":194,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-06-03T20:28:54.248956+0000","last_change":"2023-06-03T20:23:27.276693+0000","last_active":"2023-06-03T20:28:54.248956+0000","last_peered":"2023-06-03T20:28:54.248956+0000","last_clean":"2023-06-03T20:28:54.248956+0000","last_became_active":"2023-06-03T20:23:27.276305+0000","last_became_peered":"2023-06-03T20:23:27.276305+0000","last_unstale":"2023-06-03T20:28:54.248956+0000","last_undegraded":"2023-06-03T20:28:54.248956+0000","last_fullsized":"2023-06-03T20:28:54.248956+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T20:34:50.944936+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":199,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-06-03T20:28:54.250839+0000","last_change":"2023-06-03T20:28:51.231829+0000","last_active":"2023-06-03T20:28:54.250839+0000","last_peered":"2023-06-03T20:28:54.250839+0000","last_clean":"2023-06-03T20:28:54.250839+0000","last_became_active":"2023-06-03T20:28:51.231452+0000","last_became_peered":"2023-06-03T20:28:51.231452+0000","last_unstale":"2023-06-03T20:28:54.250839+0000","last_undegraded":"2023-06-03T20:28:54.250839+0000","last_fullsized":"2023-06-03T20:28:54.250839+0000","mapping_epoch":286,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":287,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T07:00:21.647058+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"14.5","version":"290'7743","reported_seq":7765,"reported_epoch":290,"state":"active+clean","last_fresh":"2023-06-03T20:29:52.696622+0000","last_change":"2023-06-03T20:27:21.371070+0000","last_active":"2023-06-03T20:29:52.696622+0000","last_peered":"2023-06-03T20:29:52.696622+0000","last_clean":"2023-06-03T20:29:52.696622+0000","last_became_active":"2023-06-03T20:27:21.370790+0000","last_became_peered":"2023-06-03T20:27:21.370790+0000","last_unstale":"2023-06-03T20:29:52.696622+0000","last_undegraded":"2023-06-03T20:29:52.696622+0000","last_fullsized":"2023-06-03T20:29:52.696622+0000","mapping_epoch":278,"log_start":"0'0","ondisk_log_start":"0'0","created":278,"last_epoch_clean":279,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T20:27:20.339366+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T20:27:20.339366+0000","last_clean_scrub_stamp":"2023-06-03T20:27:20.339366+0000","objects_scrubbed":0,"log_size":7743,"log_dups_size":0,"ondisk_log_size":7743,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T21:04:21.657715+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":305070080,"num_objects":4655,"num_object_clones":0,"num_object_copies":13965,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4655,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7743,"num_write_kb":396736,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":293,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-06-03T20:29:22.687500+0000","last_change":"2023-06-03T20:21:01.256958+0000","last_active":"2023-06-03T20:29:22.687500+0000","last_peered":"2023-06-03T20:29:22.687500+0000","last_clean":"2023-06-03T20:29:22.687500+0000","last_became_active":"2023-06-03T20:21:01.255452+0000","last_became_peered":"2023-06-03T20:21:01.255452+0000","last_unstale":"2023-06-03T20:29:22.687500+0000","last_undegraded":"2023-06-03T20:29:22.687500+0000","last_fullsized":"2023-06-03T20:29:22.687500+0000","mapping_epoch":235,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":236,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T01:56:59.777763+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":79,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-06-03T20:28:54.266069+0000","last_change":"2023-06-03T20:23:28.250884+0000","last_active":"2023-06-03T20:28:54.266069+0000","last_peered":"2023-06-03T20:28:54.266069+0000","last_clean":"2023-06-03T20:28:54.266069+0000","last_became_active":"2023-06-03T20:23:28.243793+0000","last_became_peered":"2023-06-03T20:23:28.243793+0000","last_unstale":"2023-06-03T20:28:54.266069+0000","last_undegraded":"2023-06-03T20:28:54.266069+0000","last_fullsized":"2023-06-03T20:28:54.266069+0000","mapping_epoch":251,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":252,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T03:12:27.946002+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"14.8","version":"290'7670","reported_seq":7692,"reported_epoch":290,"state":"active+clean","last_fresh":"2023-06-03T20:29:52.724602+0000","last_change":"2023-06-03T20:27:21.376040+0000","last_active":"2023-06-03T20:29:52.724602+0000","last_peered":"2023-06-03T20:29:52.724602+0000","last_clean":"2023-06-03T20:29:52.724602+0000","last_became_active":"2023-06-03T20:27:21.375709+0000","last_became_peered":"2023-06-03T20:27:21.375709+0000","last_unstale":"2023-06-03T20:29:52.724602+0000","last_undegraded":"2023-06-03T20:29:52.724602+0000","last_fullsized":"2023-06-03T20:29:52.724602+0000","mapping_epoch":278,"log_start":"0'0","ondisk_log_start":"0'0","created":278,"last_epoch_clean":279,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T20:27:20.339366+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T20:27:20.339366+0000","last_clean_scrub_stamp":"2023-06-03T20:27:20.339366+0000","objects_scrubbed":0,"log_size":7670,"log_dups_size":0,"ondisk_log_size":7670,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T05:50:28.157584+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":302252032,"num_objects":4612,"num_object_clones":0,"num_object_copies":13836,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4612,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7670,"num_write_kb":393024,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":69,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-06-03T20:29:22.581860+0000","last_change":"2023-06-03T20:21:01.258182+0000","last_active":"2023-06-03T20:29:22.581860+0000","last_peered":"2023-06-03T20:29:22.581860+0000","last_clean":"2023-06-03T20:29:22.581860+0000","last_became_active":"2023-06-03T20:21:01.254049+0000","last_became_peered":"2023-06-03T20:21:01.254049+0000","last_unstale":"2023-06-03T20:29:22.581860+0000","last_undegraded":"2023-06-03T20:29:22.581860+0000","last_fullsized":"2023-06-03T20:29:22.581860+0000","mapping_epoch":235,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":236,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:32:20.692043+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:32:20.692043+0000","last_clean_scrub_stamp":"2023-06-03T19:32:20.692043+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T05:16:30.078473+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":177,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-06-03T20:28:54.259760+0000","last_change":"2023-06-03T20:17:22.834746+0000","last_active":"2023-06-03T20:28:54.259760+0000","last_peered":"2023-06-03T20:28:54.259760+0000","last_clean":"2023-06-03T20:28:54.259760+0000","last_became_active":"2023-06-03T20:17:22.834448+0000","last_became_peered":"2023-06-03T20:17:22.834448+0000","last_unstale":"2023-06-03T20:28:54.259760+0000","last_undegraded":"2023-06-03T20:28:54.259760+0000","last_fullsized":"2023-06-03T20:28:54.259760+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T21:16:31.167063+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,2],"acting":[3,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":80,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-06-03T20:29:22.581800+0000","last_change":"2023-06-03T20:23:28.254434+0000","last_active":"2023-06-03T20:29:22.581800+0000","last_peered":"2023-06-03T20:29:22.581800+0000","last_clean":"2023-06-03T20:29:22.581800+0000","last_became_active":"2023-06-03T20:23:28.254012+0000","last_became_peered":"2023-06-03T20:23:28.254012+0000","last_unstale":"2023-06-03T20:29:22.581800+0000","last_undegraded":"2023-06-03T20:29:22.581800+0000","last_fullsized":"2023-06-03T20:29:22.581800+0000","mapping_epoch":251,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":252,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T06:22:09.540298+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"14.e","version":"290'7599","reported_seq":7621,"reported_epoch":290,"state":"active+clean","last_fresh":"2023-06-03T20:29:51.288457+0000","last_change":"2023-06-03T20:27:21.368301+0000","last_active":"2023-06-03T20:29:51.288457+0000","last_peered":"2023-06-03T20:29:51.288457+0000","last_clean":"2023-06-03T20:29:51.288457+0000","last_became_active":"2023-06-03T20:27:21.367955+0000","last_became_peered":"2023-06-03T20:27:21.367955+0000","last_unstale":"2023-06-03T20:29:51.288457+0000","last_undegraded":"2023-06-03T20:29:51.288457+0000","last_fullsized":"2023-06-03T20:29:51.288457+0000","mapping_epoch":278,"log_start":"0'0","ondisk_log_start":"0'0","created":278,"last_epoch_clean":279,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T20:27:20.339366+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T20:27:20.339366+0000","last_clean_scrub_stamp":"2023-06-03T20:27:20.339366+0000","objects_scrubbed":0,"log_size":7599,"log_dups_size":0,"ondisk_log_size":7599,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T22:35:27.855608+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":307822592,"num_objects":4697,"num_object_clones":0,"num_object_copies":14091,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4697,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7599,"num_write_kb":393472,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":78,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-06-03T20:29:22.581827+0000","last_change":"2023-06-03T20:28:51.234855+0000","last_active":"2023-06-03T20:29:22.581827+0000","last_peered":"2023-06-03T20:29:22.581827+0000","last_clean":"2023-06-03T20:29:22.581827+0000","last_became_active":"2023-06-03T20:28:51.234514+0000","last_became_peered":"2023-06-03T20:28:51.234514+0000","last_unstale":"2023-06-03T20:29:22.581827+0000","last_undegraded":"2023-06-03T20:29:22.581827+0000","last_fullsized":"2023-06-03T20:29:22.581827+0000","mapping_epoch":286,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":287,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:32:20.692043+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:32:20.692043+0000","last_clean_scrub_stamp":"2023-06-03T19:32:20.692043+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T03:00:56.480804+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":175,"reported_epoch":290,"state":"active+clean","last_fresh":"2023-06-03T20:29:30.219097+0000","last_change":"2023-06-03T20:24:02.258311+0000","last_active":"2023-06-03T20:29:30.219097+0000","last_peered":"2023-06-03T20:29:30.219097+0000","last_clean":"2023-06-03T20:29:30.219097+0000","last_became_active":"2023-06-03T20:24:02.257676+0000","last_became_peered":"2023-06-03T20:24:02.257676+0000","last_unstale":"2023-06-03T20:29:30.219097+0000","last_undegraded":"2023-06-03T20:29:30.219097+0000","last_fullsized":"2023-06-03T20:29:30.219097+0000","mapping_epoch":254,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":255,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T01:47:46.383038+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":69,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-06-03T20:29:22.581876+0000","last_change":"2023-06-03T20:21:01.259331+0000","last_active":"2023-06-03T20:29:22.581876+0000","last_peered":"2023-06-03T20:29:22.581876+0000","last_clean":"2023-06-03T20:29:22.581876+0000","last_became_active":"2023-06-03T20:21:01.258836+0000","last_became_peered":"2023-06-03T20:21:01.258836+0000","last_unstale":"2023-06-03T20:29:22.581876+0000","last_undegraded":"2023-06-03T20:29:22.581876+0000","last_fullsized":"2023-06-03T20:29:22.581876+0000","mapping_epoch":235,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":236,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T01:29:19.471600+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"14.d","version":"290'7758","reported_seq":10389,"reported_epoch":290,"state":"active+recovering+remapped","last_fresh":"2023-06-03T20:29:52.395476+0000","last_change":"2023-06-03T20:28:52.754748+0000","last_active":"2023-06-03T20:29:52.395476+0000","last_peered":"2023-06-03T20:29:52.395476+0000","last_clean":"2023-06-03T20:28:49.972400+0000","last_became_active":"2023-06-03T20:28:52.639960+0000","last_became_peered":"2023-06-03T20:28:52.639960+0000","last_unstale":"2023-06-03T20:29:52.395476+0000","last_undegraded":"2023-06-03T20:29:52.395476+0000","last_fullsized":"2023-06-03T20:29:52.395476+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":278,"last_epoch_clean":279,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T20:27:20.339366+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T20:27:20.339366+0000","last_clean_scrub_stamp":"2023-06-03T20:27:20.339366+0000","objects_scrubbed":0,"log_size":7758,"log_dups_size":0,"ondisk_log_size":7758,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T08:00:38.609409+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":301006872,"num_objects":4594,"num_object_clones":0,"num_object_copies":13782,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":3765,"num_objects_unfound":0,"num_objects_dirty":4594,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":7758,"num_write_kb":395201,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1301,"num_bytes_recovered":85262336,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,4],"acting":[2,5,4],"avail_no_missing":["2","4","5"],"object_location_counts":[{"shards":"2,4,5","objects":829}],"blocked_by":[],"up_primary":6,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":276,"reported_epoch":290,"state":"active+clean","last_fresh":"2023-06-03T20:29:30.242260+0000","last_change":"2023-06-03T19:56:54.504859+0000","last_active":"2023-06-03T20:29:30.242260+0000","last_peered":"2023-06-03T20:29:30.242260+0000","last_clean":"2023-06-03T20:29:30.242260+0000","last_became_active":"2023-06-03T19:56:54.504455+0000","last_became_peered":"2023-06-03T19:56:54.504455+0000","last_unstale":"2023-06-03T20:29:30.242260+0000","last_undegraded":"2023-06-03T20:29:30.242260+0000","last_fullsized":"2023-06-03T20:29:30.242260+0000","mapping_epoch":119,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:32:20.692043+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:32:20.692043+0000","last_clean_scrub_stamp":"2023-06-03T19:32:20.692043+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T06:25:45.522100+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":78,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-06-03T20:29:07.565149+0000","last_change":"2023-06-03T20:24:02.256677+0000","last_active":"2023-06-03T20:29:07.565149+0000","last_peered":"2023-06-03T20:29:07.565149+0000","last_clean":"2023-06-03T20:29:07.565149+0000","last_became_active":"2023-06-03T20:24:02.256216+0000","last_became_peered":"2023-06-03T20:24:02.256216+0000","last_unstale":"2023-06-03T20:29:07.565149+0000","last_undegraded":"2023-06-03T20:29:07.565149+0000","last_fullsized":"2023-06-03T20:29:07.565149+0000","mapping_epoch":254,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":255,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T02:32:45.186204+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":69,"reported_epoch":290,"state":"active+clean","last_fresh":"2023-06-03T20:29:30.239459+0000","last_change":"2023-06-03T20:21:01.261473+0000","last_active":"2023-06-03T20:29:30.239459+0000","last_peered":"2023-06-03T20:29:30.239459+0000","last_clean":"2023-06-03T20:29:30.239459+0000","last_became_active":"2023-06-03T20:21:01.261098+0000","last_became_peered":"2023-06-03T20:21:01.261098+0000","last_unstale":"2023-06-03T20:29:30.239459+0000","last_undegraded":"2023-06-03T20:29:30.239459+0000","last_fullsized":"2023-06-03T20:29:30.239459+0000","mapping_epoch":235,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":236,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T03:56:22.986549+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"14.f","version":"290'7596","reported_seq":7616,"reported_epoch":290,"state":"active+clean","last_fresh":"2023-06-03T20:29:52.785990+0000","last_change":"2023-06-03T20:27:21.374763+0000","last_active":"2023-06-03T20:29:52.785990+0000","last_peered":"2023-06-03T20:29:52.785990+0000","last_clean":"2023-06-03T20:29:52.785990+0000","last_became_active":"2023-06-03T20:27:21.374440+0000","last_became_peered":"2023-06-03T20:27:21.374440+0000","last_unstale":"2023-06-03T20:29:52.785990+0000","last_undegraded":"2023-06-03T20:29:52.785990+0000","last_fullsized":"2023-06-03T20:29:52.785990+0000","mapping_epoch":278,"log_start":"0'0","ondisk_log_start":"0'0","created":278,"last_epoch_clean":279,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T20:27:20.339366+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T20:27:20.339366+0000","last_clean_scrub_stamp":"2023-06-03T20:27:20.339366+0000","objects_scrubbed":0,"log_size":7596,"log_dups_size":0,"ondisk_log_size":7596,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":299761664,"num_objects":4574,"num_object_clones":0,"num_object_copies":13722,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4574,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7596,"num_write_kb":389440,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,1],"acting":[7,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":160,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-06-03T20:28:54.257194+0000","last_change":"2023-06-03T20:17:22.834467+0000","last_active":"2023-06-03T20:28:54.257194+0000","last_peered":"2023-06-03T20:28:54.257194+0000","last_clean":"2023-06-03T20:28:54.257194+0000","last_became_active":"2023-06-03T20:17:22.834147+0000","last_became_peered":"2023-06-03T20:17:22.834147+0000","last_unstale":"2023-06-03T20:28:54.257194+0000","last_undegraded":"2023-06-03T20:28:54.257194+0000","last_fullsized":"2023-06-03T20:28:54.257194+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:32:20.692043+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:32:20.692043+0000","last_clean_scrub_stamp":"2023-06-03T19:32:20.692043+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T04:36:29.567051+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,7],"acting":[2,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"72'139","reported_seq":562,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-06-03T20:28:54.257198+0000","last_change":"2023-06-03T20:27:01.248641+0000","last_active":"2023-06-03T20:28:54.257198+0000","last_peered":"2023-06-03T20:28:54.257198+0000","last_clean":"2023-06-03T20:28:54.257198+0000","last_became_active":"2023-06-03T20:27:01.248063+0000","last_became_peered":"2023-06-03T20:27:01.248063+0000","last_unstale":"2023-06-03T20:28:54.257198+0000","last_undegraded":"2023-06-03T20:28:54.257198+0000","last_fullsized":"2023-06-03T20:28:54.257198+0000","mapping_epoch":274,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":275,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:27:28.100808+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:27:28.100808+0000","last_clean_scrub_stamp":"2023-06-03T19:27:28.100808+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":278,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-06-03T20:28:54.263424+0000","last_change":"2023-06-03T20:24:02.263360+0000","last_active":"2023-06-03T20:28:54.263424+0000","last_peered":"2023-06-03T20:28:54.263424+0000","last_clean":"2023-06-03T20:28:54.263424+0000","last_became_active":"2023-06-03T20:24:02.262954+0000","last_became_peered":"2023-06-03T20:24:02.262954+0000","last_unstale":"2023-06-03T20:28:54.263424+0000","last_undegraded":"2023-06-03T20:28:54.263424+0000","last_fullsized":"2023-06-03T20:28:54.263424+0000","mapping_epoch":254,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":255,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T19:38:46.084731+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":13,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-06-03T20:28:54.256320+0000","last_change":"2023-06-03T20:28:51.234600+0000","last_active":"2023-06-03T20:28:54.256320+0000","last_peered":"2023-06-03T20:28:54.256320+0000","last_clean":"2023-06-03T20:28:54.256320+0000","last_became_active":"2023-06-03T20:28:51.234227+0000","last_became_peered":"2023-06-03T20:28:51.234227+0000","last_unstale":"2023-06-03T20:28:54.256320+0000","last_undegraded":"2023-06-03T20:28:54.256320+0000","last_fullsized":"2023-06-03T20:28:54.256320+0000","mapping_epoch":286,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":287,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"14.c","version":"290'7683","reported_seq":7705,"reported_epoch":290,"state":"active+clean","last_fresh":"2023-06-03T20:29:52.762461+0000","last_change":"2023-06-03T20:27:21.375660+0000","last_active":"2023-06-03T20:29:52.762461+0000","last_peered":"2023-06-03T20:29:52.762461+0000","last_clean":"2023-06-03T20:29:52.762461+0000","last_became_active":"2023-06-03T20:27:21.375311+0000","last_became_peered":"2023-06-03T20:27:21.375311+0000","last_unstale":"2023-06-03T20:29:52.762461+0000","last_undegraded":"2023-06-03T20:29:52.762461+0000","last_fullsized":"2023-06-03T20:29:52.762461+0000","mapping_epoch":278,"log_start":"0'0","ondisk_log_start":"0'0","created":278,"last_epoch_clean":279,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T20:27:20.339366+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T20:27:20.339366+0000","last_clean_scrub_stamp":"2023-06-03T20:27:20.339366+0000","objects_scrubbed":0,"log_size":7683,"log_dups_size":0,"ondisk_log_size":7683,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T04:43:56.400215+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":297992192,"num_objects":4547,"num_object_clones":0,"num_object_copies":13641,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4547,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7683,"num_write_kb":391360,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,2],"acting":[0,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":282,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-06-03T20:28:54.257159+0000","last_change":"2023-06-03T20:21:01.258113+0000","last_active":"2023-06-03T20:28:54.257159+0000","last_peered":"2023-06-03T20:28:54.257159+0000","last_clean":"2023-06-03T20:28:54.257159+0000","last_became_active":"2023-06-03T20:21:01.257488+0000","last_became_peered":"2023-06-03T20:21:01.257488+0000","last_unstale":"2023-06-03T20:28:54.257159+0000","last_undegraded":"2023-06-03T20:28:54.257159+0000","last_fullsized":"2023-06-03T20:28:54.257159+0000","mapping_epoch":235,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":236,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:32:20.692043+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:32:20.692043+0000","last_clean_scrub_stamp":"2023-06-03T19:32:20.692043+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":288,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-06-03T20:28:54.247769+0000","last_change":"2023-06-03T20:23:27.255164+0000","last_active":"2023-06-03T20:28:54.247769+0000","last_peered":"2023-06-03T20:28:54.247769+0000","last_clean":"2023-06-03T20:28:54.247769+0000","last_became_active":"2023-06-03T20:23:27.253932+0000","last_became_peered":"2023-06-03T20:23:27.253932+0000","last_unstale":"2023-06-03T20:28:54.247769+0000","last_undegraded":"2023-06-03T20:28:54.247769+0000","last_fullsized":"2023-06-03T20:28:54.247769+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T06:42:02.303554+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":291,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-06-03T20:28:54.248513+0000","last_change":"2023-06-03T20:21:01.250266+0000","last_active":"2023-06-03T20:28:54.248513+0000","last_peered":"2023-06-03T20:28:54.248513+0000","last_clean":"2023-06-03T20:28:54.248513+0000","last_became_active":"2023-06-03T20:21:01.249839+0000","last_became_peered":"2023-06-03T20:21:01.249839+0000","last_unstale":"2023-06-03T20:28:54.248513+0000","last_undegraded":"2023-06-03T20:28:54.248513+0000","last_fullsized":"2023-06-03T20:28:54.248513+0000","mapping_epoch":235,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":236,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T05:13:26.584607+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"14.9","version":"290'7650","reported_seq":7672,"reported_epoch":290,"state":"active+clean","last_fresh":"2023-06-03T20:29:52.710209+0000","last_change":"2023-06-03T20:27:21.372404+0000","last_active":"2023-06-03T20:29:52.710209+0000","last_peered":"2023-06-03T20:29:52.710209+0000","last_clean":"2023-06-03T20:29:52.710209+0000","last_became_active":"2023-06-03T20:27:21.371829+0000","last_became_peered":"2023-06-03T20:27:21.371829+0000","last_unstale":"2023-06-03T20:29:52.710209+0000","last_undegraded":"2023-06-03T20:29:52.710209+0000","last_fullsized":"2023-06-03T20:29:52.710209+0000","mapping_epoch":278,"log_start":"0'0","ondisk_log_start":"0'0","created":278,"last_epoch_clean":279,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T20:27:20.339366+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T20:27:20.339366+0000","last_clean_scrub_stamp":"2023-06-03T20:27:20.339366+0000","objects_scrubbed":0,"log_size":7650,"log_dups_size":0,"ondisk_log_size":7650,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T02:20:57.246181+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":300941312,"num_objects":4592,"num_object_clones":0,"num_object_copies":13776,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4592,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7650,"num_write_kb":391744,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,4],"acting":[3,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":291,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-06-03T20:28:54.257909+0000","last_change":"2023-06-03T20:23:27.261511+0000","last_active":"2023-06-03T20:28:54.257909+0000","last_peered":"2023-06-03T20:28:54.257909+0000","last_clean":"2023-06-03T20:28:54.257909+0000","last_became_active":"2023-06-03T20:23:27.259622+0000","last_became_peered":"2023-06-03T20:23:27.259622+0000","last_unstale":"2023-06-03T20:28:54.257909+0000","last_undegraded":"2023-06-03T20:28:54.257909+0000","last_fullsized":"2023-06-03T20:28:54.257909+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:32:20.692043+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:32:20.692043+0000","last_clean_scrub_stamp":"2023-06-03T19:32:20.692043+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":69,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-06-03T20:29:22.581844+0000","last_change":"2023-06-03T20:21:01.252859+0000","last_active":"2023-06-03T20:29:22.581844+0000","last_peered":"2023-06-03T20:29:22.581844+0000","last_clean":"2023-06-03T20:29:22.581844+0000","last_became_active":"2023-06-03T20:21:01.251222+0000","last_became_peered":"2023-06-03T20:21:01.251222+0000","last_unstale":"2023-06-03T20:29:22.581844+0000","last_undegraded":"2023-06-03T20:29:22.581844+0000","last_fullsized":"2023-06-03T20:29:22.581844+0000","mapping_epoch":235,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":236,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T05:19:58.100226+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":244,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-06-03T20:28:54.259671+0000","last_change":"2023-06-03T19:47:06.733532+0000","last_active":"2023-06-03T20:28:54.259671+0000","last_peered":"2023-06-03T20:28:54.259671+0000","last_clean":"2023-06-03T20:28:54.259671+0000","last_became_active":"2023-06-03T19:47:06.733131+0000","last_became_peered":"2023-06-03T19:47:06.733131+0000","last_unstale":"2023-06-03T20:28:54.259671+0000","last_undegraded":"2023-06-03T20:28:54.259671+0000","last_fullsized":"2023-06-03T20:28:54.259671+0000","mapping_epoch":86,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":87,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T23:46:03.788646+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"14.a","version":"290'7548","reported_seq":7567,"reported_epoch":290,"state":"active+clean","last_fresh":"2023-06-03T20:29:52.613866+0000","last_change":"2023-06-03T20:27:21.378317+0000","last_active":"2023-06-03T20:29:52.613866+0000","last_peered":"2023-06-03T20:29:52.613866+0000","last_clean":"2023-06-03T20:29:52.613866+0000","last_became_active":"2023-06-03T20:27:21.377947+0000","last_became_peered":"2023-06-03T20:27:21.377947+0000","last_unstale":"2023-06-03T20:29:52.613866+0000","last_undegraded":"2023-06-03T20:29:52.613866+0000","last_fullsized":"2023-06-03T20:29:52.613866+0000","mapping_epoch":278,"log_start":"0'0","ondisk_log_start":"0'0","created":278,"last_epoch_clean":279,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T20:27:20.339366+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T20:27:20.339366+0000","last_clean_scrub_stamp":"2023-06-03T20:27:20.339366+0000","objects_scrubbed":0,"log_size":7548,"log_dups_size":0,"ondisk_log_size":7548,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T23:39:02.595450+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":297271296,"num_objects":4536,"num_object_clones":0,"num_object_copies":13608,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4536,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7548,"num_write_kb":386688,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,3],"acting":[1,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.7","version":"60'2","reported_seq":254,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-06-03T20:29:22.687631+0000","last_change":"2023-06-03T19:56:54.510148+0000","last_active":"2023-06-03T20:29:22.687631+0000","last_peered":"2023-06-03T20:29:22.687631+0000","last_clean":"2023-06-03T20:29:22.687631+0000","last_became_active":"2023-06-03T19:56:54.509771+0000","last_became_peered":"2023-06-03T19:56:54.509771+0000","last_unstale":"2023-06-03T20:29:22.687631+0000","last_undegraded":"2023-06-03T20:29:22.687631+0000","last_fullsized":"2023-06-03T20:29:22.687631+0000","mapping_epoch":119,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T00:52:42.807432+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":87,"reported_epoch":290,"state":"active+clean","last_fresh":"2023-06-03T20:29:30.239080+0000","last_change":"2023-06-03T20:24:02.257142+0000","last_active":"2023-06-03T20:29:30.239080+0000","last_peered":"2023-06-03T20:29:30.239080+0000","last_clean":"2023-06-03T20:29:30.239080+0000","last_became_active":"2023-06-03T20:24:02.256483+0000","last_became_peered":"2023-06-03T20:24:02.256483+0000","last_unstale":"2023-06-03T20:29:30.239080+0000","last_undegraded":"2023-06-03T20:29:30.239080+0000","last_fullsized":"2023-06-03T20:29:30.239080+0000","mapping_epoch":254,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":255,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:32:20.692043+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:32:20.692043+0000","last_clean_scrub_stamp":"2023-06-03T19:32:20.692043+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T00:53:37.197795+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":252,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-06-03T20:29:02.664077+0000","last_change":"2023-06-03T19:56:54.507670+0000","last_active":"2023-06-03T20:29:02.664077+0000","last_peered":"2023-06-03T20:29:02.664077+0000","last_clean":"2023-06-03T20:29:02.664077+0000","last_became_active":"2023-06-03T19:56:54.507293+0000","last_became_peered":"2023-06-03T19:56:54.507293+0000","last_unstale":"2023-06-03T20:29:02.664077+0000","last_undegraded":"2023-06-03T20:29:02.664077+0000","last_fullsized":"2023-06-03T20:29:02.664077+0000","mapping_epoch":119,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T01:49:19.966054+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"14.b","version":"290'7472","reported_seq":7494,"reported_epoch":290,"state":"active+clean","last_fresh":"2023-06-03T20:29:52.762354+0000","last_change":"2023-06-03T20:27:21.375306+0000","last_active":"2023-06-03T20:29:52.762354+0000","last_peered":"2023-06-03T20:29:52.762354+0000","last_clean":"2023-06-03T20:29:52.762354+0000","last_became_active":"2023-06-03T20:27:21.374975+0000","last_became_peered":"2023-06-03T20:27:21.374975+0000","last_unstale":"2023-06-03T20:29:52.762354+0000","last_undegraded":"2023-06-03T20:29:52.762354+0000","last_fullsized":"2023-06-03T20:29:52.762354+0000","mapping_epoch":278,"log_start":"0'0","ondisk_log_start":"0'0","created":278,"last_epoch_clean":279,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T20:27:20.339366+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T20:27:20.339366+0000","last_clean_scrub_stamp":"2023-06-03T20:27:20.339366+0000","objects_scrubbed":0,"log_size":7472,"log_dups_size":0,"ondisk_log_size":7472,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T02:22:27.778843+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":295043072,"num_objects":4502,"num_object_clones":0,"num_object_copies":13506,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4502,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7472,"num_write_kb":383168,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,1],"acting":[0,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":45,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-06-03T20:28:54.256422+0000","last_change":"2023-06-03T20:24:02.319230+0000","last_active":"2023-06-03T20:28:54.256422+0000","last_peered":"2023-06-03T20:28:54.256422+0000","last_clean":"2023-06-03T20:28:54.256422+0000","last_became_active":"2023-06-03T20:24:02.318926+0000","last_became_peered":"2023-06-03T20:24:02.318926+0000","last_unstale":"2023-06-03T20:28:54.256422+0000","last_undegraded":"2023-06-03T20:28:54.256422+0000","last_fullsized":"2023-06-03T20:28:54.256422+0000","mapping_epoch":254,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":255,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:32:20.692043+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:32:20.692043+0000","last_clean_scrub_stamp":"2023-06-03T19:32:20.692043+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"60'1","reported_seq":319,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-06-03T20:28:54.248373+0000","last_change":"2023-06-03T20:23:28.239664+0000","last_active":"2023-06-03T20:28:54.248373+0000","last_peered":"2023-06-03T20:28:54.248373+0000","last_clean":"2023-06-03T20:28:54.248373+0000","last_became_active":"2023-06-03T20:23:28.239252+0000","last_became_peered":"2023-06-03T20:23:28.239252+0000","last_unstale":"2023-06-03T20:28:54.248373+0000","last_undegraded":"2023-06-03T20:28:54.248373+0000","last_fullsized":"2023-06-03T20:28:54.248373+0000","mapping_epoch":251,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":252,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T06:52:20.862520+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":173,"reported_epoch":290,"state":"active+clean","last_fresh":"2023-06-03T20:29:30.218746+0000","last_change":"2023-06-03T20:24:02.259850+0000","last_active":"2023-06-03T20:29:30.218746+0000","last_peered":"2023-06-03T20:29:30.218746+0000","last_clean":"2023-06-03T20:29:30.218746+0000","last_became_active":"2023-06-03T20:24:02.258764+0000","last_became_peered":"2023-06-03T20:24:02.258764+0000","last_unstale":"2023-06-03T20:29:30.218746+0000","last_undegraded":"2023-06-03T20:29:30.218746+0000","last_fullsized":"2023-06-03T20:29:30.218746+0000","mapping_epoch":254,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":255,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T21:54:11.487473+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"60'1","reported_seq":274,"reported_epoch":290,"state":"active+clean","last_fresh":"2023-06-03T20:29:30.243884+0000","last_change":"2023-06-03T20:23:27.254853+0000","last_active":"2023-06-03T20:29:30.243884+0000","last_peered":"2023-06-03T20:29:30.243884+0000","last_clean":"2023-06-03T20:29:30.243884+0000","last_became_active":"2023-06-03T20:23:27.254410+0000","last_became_peered":"2023-06-03T20:23:27.254410+0000","last_unstale":"2023-06-03T20:29:30.243884+0000","last_undegraded":"2023-06-03T20:29:30.243884+0000","last_fullsized":"2023-06-03T20:29:30.243884+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T22:46:19.423939+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":303,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-06-03T20:28:54.256764+0000","last_change":"2023-06-03T20:21:01.257656+0000","last_active":"2023-06-03T20:28:54.256764+0000","last_peered":"2023-06-03T20:28:54.256764+0000","last_clean":"2023-06-03T20:28:54.256764+0000","last_became_active":"2023-06-03T20:21:01.257107+0000","last_became_peered":"2023-06-03T20:21:01.257107+0000","last_unstale":"2023-06-03T20:28:54.256764+0000","last_undegraded":"2023-06-03T20:28:54.256764+0000","last_fullsized":"2023-06-03T20:28:54.256764+0000","mapping_epoch":235,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":236,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T06:08:18.954539+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"60'2","reported_seq":64,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-06-03T20:28:54.249175+0000","last_change":"2023-06-03T20:23:28.243048+0000","last_active":"2023-06-03T20:28:54.249175+0000","last_peered":"2023-06-03T20:28:54.249175+0000","last_clean":"2023-06-03T20:28:54.249175+0000","last_became_active":"2023-06-03T20:23:28.241128+0000","last_became_peered":"2023-06-03T20:23:28.241128+0000","last_unstale":"2023-06-03T20:28:54.249175+0000","last_undegraded":"2023-06-03T20:28:54.249175+0000","last_fullsized":"2023-06-03T20:28:54.249175+0000","mapping_epoch":251,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":252,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T05:14:54.701329+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,1],"acting":[4,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":227,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-06-03T20:28:54.248916+0000","last_change":"2023-06-03T20:28:24.276513+0000","last_active":"2023-06-03T20:28:54.248916+0000","last_peered":"2023-06-03T20:28:54.248916+0000","last_clean":"2023-06-03T20:28:54.248916+0000","last_became_active":"2023-06-03T20:28:24.276244+0000","last_became_peered":"2023-06-03T20:28:24.276244+0000","last_unstale":"2023-06-03T20:28:54.248916+0000","last_undegraded":"2023-06-03T20:28:54.248916+0000","last_fullsized":"2023-06-03T20:28:54.248916+0000","mapping_epoch":284,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":285,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T06:17:57.159168+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[4,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"60'2","reported_seq":77,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-06-03T20:28:54.249145+0000","last_change":"2023-06-03T20:28:51.237342+0000","last_active":"2023-06-03T20:28:54.249145+0000","last_peered":"2023-06-03T20:28:54.249145+0000","last_clean":"2023-06-03T20:28:54.249145+0000","last_became_active":"2023-06-03T20:28:51.233969+0000","last_became_peered":"2023-06-03T20:28:51.233969+0000","last_unstale":"2023-06-03T20:28:54.249145+0000","last_undegraded":"2023-06-03T20:28:54.249145+0000","last_fullsized":"2023-06-03T20:28:54.249145+0000","mapping_epoch":286,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":287,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T07:30:09.503545+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":211,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-06-03T20:28:54.248709+0000","last_change":"2023-06-03T20:23:27.254468+0000","last_active":"2023-06-03T20:28:54.248709+0000","last_peered":"2023-06-03T20:28:54.248709+0000","last_clean":"2023-06-03T20:28:54.248709+0000","last_became_active":"2023-06-03T20:23:27.253859+0000","last_became_peered":"2023-06-03T20:23:27.253859+0000","last_unstale":"2023-06-03T20:28:54.248709+0000","last_undegraded":"2023-06-03T20:28:54.248709+0000","last_fullsized":"2023-06-03T20:28:54.248709+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T20:15:25.066125+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"60'1","reported_seq":308,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-06-03T20:29:02.664228+0000","last_change":"2023-06-03T20:28:51.234748+0000","last_active":"2023-06-03T20:29:02.664228+0000","last_peered":"2023-06-03T20:29:02.664228+0000","last_clean":"2023-06-03T20:29:02.664228+0000","last_became_active":"2023-06-03T20:28:51.234341+0000","last_became_peered":"2023-06-03T20:28:51.234341+0000","last_unstale":"2023-06-03T20:29:02.664228+0000","last_undegraded":"2023-06-03T20:29:02.664228+0000","last_fullsized":"2023-06-03T20:29:02.664228+0000","mapping_epoch":286,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":287,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T06:08:49.901424+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":291,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-06-03T20:28:54.268560+0000","last_change":"2023-06-03T19:56:54.503416+0000","last_active":"2023-06-03T20:28:54.268560+0000","last_peered":"2023-06-03T20:28:54.268560+0000","last_clean":"2023-06-03T20:28:54.268560+0000","last_became_active":"2023-06-03T19:56:54.503018+0000","last_became_peered":"2023-06-03T19:56:54.503018+0000","last_unstale":"2023-06-03T20:28:54.268560+0000","last_undegraded":"2023-06-03T20:28:54.268560+0000","last_fullsized":"2023-06-03T20:28:54.268560+0000","mapping_epoch":119,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T20:14:00.330616+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"60'4","reported_seq":262,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-06-03T20:28:54.248831+0000","last_change":"2023-06-03T19:56:11.748116+0000","last_active":"2023-06-03T20:28:54.248831+0000","last_peered":"2023-06-03T20:28:54.248831+0000","last_clean":"2023-06-03T20:28:54.248831+0000","last_became_active":"2023-06-03T19:56:11.747636+0000","last_became_peered":"2023-06-03T19:56:11.747636+0000","last_unstale":"2023-06-03T20:28:54.248831+0000","last_undegraded":"2023-06-03T20:28:54.248831+0000","last_fullsized":"2023-06-03T20:28:54.248831+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":115,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T06:57:48.052055+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":211,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-06-03T20:28:54.248591+0000","last_change":"2023-06-03T20:28:51.231879+0000","last_active":"2023-06-03T20:28:54.248591+0000","last_peered":"2023-06-03T20:28:54.248591+0000","last_clean":"2023-06-03T20:28:54.248591+0000","last_became_active":"2023-06-03T20:28:51.231451+0000","last_became_peered":"2023-06-03T20:28:51.231451+0000","last_unstale":"2023-06-03T20:28:54.248591+0000","last_undegraded":"2023-06-03T20:28:54.248591+0000","last_fullsized":"2023-06-03T20:28:54.248591+0000","mapping_epoch":286,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":287,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T05:47:15.344575+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":14,"num_pg":16,"stat_sum":{"num_bytes":4835704856,"num_objects":73788,"num_object_clones":0,"num_object_copies":221364,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":3765,"num_objects_unfound":0,"num_objects_dirty":73788,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":122240,"num_write_kb":6272833,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1301,"num_bytes_recovered":85262336,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":14579875840,"data_stored":14579859552,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":122240,"ondisk_log_size":122240,"up":48,"acting":48,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5486,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":62,"num_write_kb":30,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":23,"num_bytes_recovered":4856,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16458,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":30319,"internal_metadata":0},"log_size":56,"ondisk_log_size":56,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2969600,"data_stored":2951840,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":3,"num_store_stats":5},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":48,"seq":206158430914,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1843708,"kb_used_data":890356,"kb_used_omap":0,"kb_used_meta":953344,"kb_avail":91901444,"statfs":{"total":95995035648,"available":94107078656,"internally_reserved":0,"allocated":911724544,"data_stored":911062982,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":976224256},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 3 20:29:29 2023","interfaces":[{"interface":"back","average":{"1min":1.425,"5min":1.698,"15min":1.665},"min":{"1min":0.312,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":5.0129999999999999,"5min":7.3920000000000003,"15min":73.810000000000002},"last":0.41399999999999998},{"interface":"front","average":{"1min":0.72199999999999998,"5min":1.27,"15min":1.548},"min":{"1min":0.34000000000000002,"5min":0.29999999999999999,"15min":0.26700000000000002},"max":{"1min":1.7869999999999999,"5min":6.5460000000000003,"15min":72.878},"last":0.69799999999999995}]},{"osd":1,"last update":"Sat Jun 3 20:29:13 2023","interfaces":[{"interface":"back","average":{"1min":1.3720000000000001,"5min":2.7229999999999999,"15min":1.579},"min":{"1min":0.33500000000000002,"5min":0.313,"15min":0.308},"max":{"1min":4.2000000000000002,"5min":100.845,"15min":100.845},"last":0.315},{"interface":"front","average":{"1min":1.4239999999999999,"5min":2.6739999999999999,"15min":1.6180000000000001},"min":{"1min":0.378,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":6.048,"5min":100.86499999999999,"15min":100.86499999999999},"last":0.74099999999999999}]},{"osd":2,"last update":"Sat Jun 3 20:29:13 2023","interfaces":[{"interface":"back","average":{"1min":1.534,"5min":3.4630000000000001,"15min":3.4079999999999999},"min":{"1min":0.32100000000000001,"5min":0.32100000000000001,"15min":0.29499999999999998},"max":{"1min":5.9669999999999996,"5min":101.185,"15min":216.93700000000001},"last":0.81999999999999995},{"interface":"front","average":{"1min":1.5349999999999999,"5min":3.488,"15min":3.5019999999999998},"min":{"1min":0.45900000000000002,"5min":0.32400000000000001,"15min":0.28699999999999998},"max":{"1min":5.266,"5min":100.61799999999999,"15min":218.119},"last":1.004}]},{"osd":3,"last update":"Sat Jun 3 20:29:13 2023","interfaces":[{"interface":"back","average":{"1min":1.409,"5min":1.623,"15min":2.9830000000000001},"min":{"1min":0.40200000000000002,"5min":0.372,"15min":0.371},"max":{"1min":3.556,"5min":7.452,"15min":265.40499999999997},"last":0.51200000000000001},{"interface":"front","average":{"1min":1.5029999999999999,"5min":1.659,"15min":2.9470000000000001},"min":{"1min":0.38900000000000001,"5min":0.35399999999999998,"15min":0.31900000000000001},"max":{"1min":8.2360000000000007,"5min":11.656000000000001,"15min":265.43599999999998},"last":0.433}]},{"osd":4,"last update":"Sat Jun 3 20:29:35 2023","interfaces":[{"interface":"back","average":{"1min":0.79900000000000004,"5min":1.7010000000000001,"15min":2.0880000000000001},"min":{"1min":0.46400000000000002,"5min":0.38400000000000001,"15min":0.38400000000000001},"max":{"1min":2.7360000000000002,"5min":8.6010000000000009,"15min":8.6010000000000009},"last":0.48799999999999999},{"interface":"front","average":{"1min":1.1140000000000001,"5min":1.752,"15min":1.899},"min":{"1min":0.47499999999999998,"5min":0.315,"15min":0.27600000000000002},"max":{"1min":3.742,"5min":9.8460000000000001,"15min":9.8460000000000001},"last":0.54600000000000004}]},{"osd":5,"last update":"Sat Jun 3 20:29:13 2023","interfaces":[{"interface":"back","average":{"1min":0.97499999999999998,"5min":1.913,"15min":1.284},"min":{"1min":0.33800000000000002,"5min":0.317,"15min":0.317},"max":{"1min":3.4929999999999999,"5min":17.696000000000002,"15min":17.696000000000002},"last":1.401},{"interface":"front","average":{"1min":0.91800000000000004,"5min":1.8169999999999999,"15min":1.3140000000000001},"min":{"1min":0.36299999999999999,"5min":0.33600000000000002,"15min":0.29399999999999998},"max":{"1min":2.6539999999999999,"5min":8.7279999999999998,"15min":9.3710000000000004},"last":1.0449999999999999}]},{"osd":6,"last update":"Sat Jun 3 20:29:13 2023","interfaces":[{"interface":"back","average":{"1min":0.81699999999999995,"5min":1.554,"15min":1.1850000000000001},"min":{"1min":0.42599999999999999,"5min":0.36399999999999999,"15min":0.34799999999999998},"max":{"1min":1.849,"5min":9.7200000000000006,"15min":9.7200000000000006},"last":1.597},{"interface":"front","average":{"1min":0.92700000000000005,"5min":1.5269999999999999,"15min":1.1639999999999999},"min":{"1min":0.40000000000000002,"5min":0.34300000000000003,"15min":0.34300000000000003},"max":{"1min":3.77,"5min":9.923,"15min":9.923},"last":0.60599999999999998}]}]},{"osd":6,"up_from":42,"seq":180388627147,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1561264,"kb_used_data":960104,"kb_used_omap":0,"kb_used_meta":601024,"kb_avail":92183888,"statfs":{"total":95995035648,"available":94396301312,"internally_reserved":0,"allocated":983146496,"data_stored":982496785,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":615448576},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 3 20:29:45 2023","interfaces":[{"interface":"back","average":{"1min":3.266,"5min":1.534,"15min":1.3859999999999999},"min":{"1min":0.30599999999999999,"5min":0.254,"15min":0.254},"max":{"1min":22.181999999999999,"5min":22.181999999999999,"15min":56.259},"last":5.9800000000000004},{"interface":"front","average":{"1min":4.032,"5min":1.827,"15min":1.5529999999999999},"min":{"1min":0.34999999999999998,"5min":0.315,"15min":0.28299999999999997},"max":{"1min":22.166,"5min":22.166,"15min":56.234000000000002},"last":6.391}]},{"osd":1,"last update":"Sat Jun 3 20:29:16 2023","interfaces":[{"interface":"back","average":{"1min":1.496,"5min":1.113,"15min":1.0209999999999999},"min":{"1min":0.371,"5min":0.30299999999999999,"15min":0.29199999999999998},"max":{"1min":6.7910000000000004,"5min":6.7910000000000004,"15min":30.736000000000001},"last":5.5679999999999996},{"interface":"front","average":{"1min":2.4359999999999999,"5min":1.3979999999999999,"15min":1.1859999999999999},"min":{"1min":0.34200000000000003,"5min":0.33700000000000002,"15min":0.32700000000000001},"max":{"1min":8.5869999999999997,"5min":8.5869999999999997,"15min":30.788},"last":6.1189999999999998}]},{"osd":2,"last update":"Sat Jun 3 20:29:16 2023","interfaces":[{"interface":"back","average":{"1min":1.8260000000000001,"5min":1.2110000000000001,"15min":2.5070000000000001},"min":{"1min":0.34899999999999998,"5min":0.33500000000000002,"15min":0.315},"max":{"1min":8.1799999999999997,"5min":8.1799999999999997,"15min":93.558999999999997},"last":7.2199999999999998},{"interface":"front","average":{"1min":2.044,"5min":1.375,"15min":2.3999999999999999},"min":{"1min":0.378,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":7.0860000000000003,"5min":7.319,"15min":94.888000000000005},"last":5.9470000000000001}]},{"osd":3,"last update":"Sat Jun 3 20:28:59 2023","interfaces":[{"interface":"back","average":{"1min":1.905,"5min":1.272,"15min":1.681},"min":{"1min":0.36899999999999999,"5min":0.36899999999999999,"15min":0.34100000000000003},"max":{"1min":4.8140000000000001,"5min":4.9459999999999997,"15min":58.615000000000002},"last":6.5860000000000003},{"interface":"front","average":{"1min":1.1910000000000001,"5min":1.135,"15min":1.6459999999999999},"min":{"1min":0.40100000000000002,"5min":0.40100000000000002,"15min":0.32500000000000001},"max":{"1min":4.4660000000000002,"5min":7.9459999999999997,"15min":58.546999999999997},"last":6.0810000000000004}]},{"osd":4,"last update":"Sat Jun 3 20:29:38 2023","interfaces":[{"interface":"back","average":{"1min":1.8080000000000001,"5min":1.7170000000000001,"15min":1.8160000000000001},"min":{"1min":0.40999999999999998,"5min":0.33000000000000002,"15min":0.33000000000000002},"max":{"1min":8.0429999999999993,"5min":10.574999999999999,"15min":10.574999999999999},"last":6.6980000000000004},{"interface":"front","average":{"1min":1.3,"5min":1.5249999999999999,"15min":1.3660000000000001},"min":{"1min":0.40699999999999997,"5min":0.35799999999999998,"15min":0.34499999999999997},"max":{"1min":6.3799999999999999,"5min":12.622999999999999,"15min":12.622999999999999},"last":6.6710000000000003}]},{"osd":5,"last update":"Sat Jun 3 20:29:30 2023","interfaces":[{"interface":"back","average":{"1min":0.90800000000000003,"5min":1.3009999999999999,"15min":1.024},"min":{"1min":0.38600000000000001,"5min":0.38600000000000001,"15min":0.27700000000000002},"max":{"1min":3.125,"5min":7.7290000000000001,"15min":10.625999999999999},"last":6.0549999999999997},{"interface":"front","average":{"1min":2.2189999999999999,"5min":1.7689999999999999,"15min":1.2989999999999999},"min":{"1min":0.443,"5min":0.40300000000000002,"15min":0.309},"max":{"1min":8.0199999999999996,"5min":8.2880000000000003,"15min":10.856},"last":6.7320000000000002}]},{"osd":7,"last update":"Sat Jun 3 20:29:02 2023","interfaces":[{"interface":"back","average":{"1min":0.85599999999999998,"5min":1.5660000000000001,"15min":1.2310000000000001},"min":{"1min":0.45200000000000001,"5min":0.44,"15min":0.315},"max":{"1min":3.5720000000000001,"5min":7.069,"15min":9.0640000000000001},"last":6.6189999999999998},{"interface":"front","average":{"1min":1.0449999999999999,"5min":1.6160000000000001,"15min":1.194},"min":{"1min":0.46899999999999997,"5min":0.39300000000000002,"15min":0.371},"max":{"1min":4.6079999999999997,"5min":10.52,"15min":10.52},"last":6.6280000000000001}]}]},{"osd":1,"up_from":103,"seq":442381631945,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2834608,"kb_used_data":2374632,"kb_used_omap":4,"kb_used_meta":459963,"kb_avail":90910544,"statfs":{"total":95995035648,"available":93092397056,"internally_reserved":0,"allocated":2431623168,"data_stored":2430507614,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":471003022},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 3 20:29:22 2023","interfaces":[{"interface":"back","average":{"1min":7.133,"5min":2.4100000000000001,"15min":2.4590000000000001},"min":{"1min":0.19800000000000001,"5min":0.19800000000000001,"15min":0.17100000000000001},"max":{"1min":97.388000000000005,"5min":97.388000000000005,"15min":119.94499999999999},"last":2.198},{"interface":"front","average":{"1min":7.8259999999999996,"5min":2.5569999999999999,"15min":2.5329999999999999},"min":{"1min":0.35599999999999998,"5min":0.182,"15min":0.182},"max":{"1min":97.352000000000004,"5min":97.352000000000004,"15min":120.04300000000001},"last":2.2549999999999999}]},{"osd":2,"last update":"Sat Jun 3 20:29:22 2023","interfaces":[{"interface":"back","average":{"1min":2.0169999999999999,"5min":1.379,"15min":2.782},"min":{"1min":0.20599999999999999,"5min":0.20599999999999999,"15min":0.20599999999999999},"max":{"1min":7.4169999999999998,"5min":10.957000000000001,"15min":219.72300000000001},"last":0.66600000000000004},{"interface":"front","average":{"1min":1.923,"5min":1.6040000000000001,"15min":2.8639999999999999},"min":{"1min":0.246,"5min":0.23999999999999999,"15min":0.22500000000000001},"max":{"1min":4.7919999999999998,"5min":10.438000000000001,"15min":219.82900000000001},"last":4.133}]},{"osd":3,"last update":"Sat Jun 3 20:28:57 2023","interfaces":[{"interface":"back","average":{"1min":2.1219999999999999,"5min":2.3519999999999999,"15min":2.0939999999999999},"min":{"1min":0.255,"5min":0.255,"15min":0.23400000000000001},"max":{"1min":14.302,"5min":95.525000000000006,"15min":96.864999999999995},"last":3.512},{"interface":"front","average":{"1min":2.2320000000000002,"5min":2.4239999999999999,"15min":2.181},"min":{"1min":0.223,"5min":0.223,"15min":0.223},"max":{"1min":14.331,"5min":94.507000000000005,"15min":96.679000000000002},"last":2.3660000000000001}]},{"osd":4,"last update":"Sat Jun 3 20:29:40 2023","interfaces":[{"interface":"back","average":{"1min":1.831,"5min":1.7370000000000001,"15min":1.7430000000000001},"min":{"1min":0.35399999999999998,"5min":0.34000000000000002,"15min":0.34000000000000002},"max":{"1min":6.8220000000000001,"5min":11.114000000000001,"15min":11.114000000000001},"last":0.73899999999999999},{"interface":"front","average":{"1min":2.1739999999999999,"5min":1.554,"15min":1.4850000000000001},"min":{"1min":0.36499999999999999,"5min":0.33400000000000002,"15min":0.33400000000000002},"max":{"1min":7.4610000000000003,"5min":11.247999999999999,"15min":11.247999999999999},"last":0.751}]},{"osd":5,"last update":"Sat Jun 3 20:29:22 2023","interfaces":[{"interface":"back","average":{"1min":1.617,"5min":1.9059999999999999,"15min":1.2170000000000001},"min":{"1min":0.38900000000000001,"5min":0.38900000000000001,"15min":0.30299999999999999},"max":{"1min":8.8330000000000002,"5min":11.246,"15min":11.246},"last":0.64800000000000002},{"interface":"front","average":{"1min":2.1309999999999998,"5min":1.845,"15min":1.224},"min":{"1min":0.47299999999999998,"5min":0.377,"15min":0.372},"max":{"1min":7.5170000000000003,"5min":10.922000000000001,"15min":10.922000000000001},"last":0.71799999999999997}]},{"osd":6,"last update":"Sat Jun 3 20:29:22 2023","interfaces":[{"interface":"back","average":{"1min":2.2200000000000002,"5min":1.331,"15min":1.079},"min":{"1min":0.441,"5min":0.38700000000000001,"15min":0.34100000000000003},"max":{"1min":7.4900000000000002,"5min":10.002000000000001,"15min":10.002000000000001},"last":0.69199999999999995},{"interface":"front","average":{"1min":1.702,"5min":1.4850000000000001,"15min":1.1839999999999999},"min":{"1min":0.375,"5min":0.36899999999999999,"15min":0.36899999999999999},"max":{"1min":5.609,"5min":11.067,"15min":11.067},"last":4.0640000000000001}]},{"osd":7,"last update":"Sat Jun 3 20:29:22 2023","interfaces":[{"interface":"back","average":{"1min":1.4890000000000001,"5min":1.909,"15min":1.415},"min":{"1min":0.40699999999999997,"5min":0.38300000000000001,"15min":0.372},"max":{"1min":4.5700000000000003,"5min":10.538,"15min":10.538},"last":4.1070000000000002},{"interface":"front","average":{"1min":1.2709999999999999,"5min":1.6140000000000001,"15min":1.222},"min":{"1min":0.41599999999999998,"5min":0.41599999999999998,"15min":0.34000000000000002},"max":{"1min":5.9329999999999998,"5min":14.025,"15min":14.025},"last":0.61299999999999999}]}]},{"osd":0,"up_from":86,"seq":369367187971,"num_pgs":34,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3100592,"kb_used_data":2361448,"kb_used_omap":5,"kb_used_meta":738938,"kb_avail":90644560,"statfs":{"total":95995035648,"available":92820029440,"internally_reserved":0,"allocated":2418122752,"data_stored":2416949654,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":756673146},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Jun 3 20:29:11 2023","interfaces":[{"interface":"back","average":{"1min":2.0379999999999998,"5min":2.4460000000000002,"15min":2.2200000000000002},"min":{"1min":0.215,"5min":0.215,"15min":0.185},"max":{"1min":5.7060000000000004,"5min":156.43600000000001,"15min":243.03200000000001},"last":11.707000000000001},{"interface":"front","average":{"1min":3.6720000000000002,"5min":3.1099999999999999,"15min":2.5830000000000002},"min":{"1min":0.188,"5min":0.188,"15min":0.188},"max":{"1min":14.98,"5min":157.489,"15min":242.78399999999999},"last":1.681}]},{"osd":2,"last update":"Sat Jun 3 20:29:11 2023","interfaces":[{"interface":"back","average":{"1min":1.3460000000000001,"5min":1.804,"15min":1.4950000000000001},"min":{"1min":0.219,"5min":0.219,"15min":0.218},"max":{"1min":6.274,"5min":36.515000000000001,"15min":36.515000000000001},"last":0.35299999999999998},{"interface":"front","average":{"1min":2.5899999999999999,"5min":1.9299999999999999,"15min":2.6280000000000001},"min":{"1min":0.315,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":11.563000000000001,"5min":31.164000000000001,"15min":243.30099999999999},"last":0.70699999999999996}]},{"osd":3,"last update":"Sat Jun 3 20:29:52 2023","interfaces":[{"interface":"back","average":{"1min":2.1640000000000001,"5min":4.6680000000000001,"15min":3.7080000000000002},"min":{"1min":0.24399999999999999,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":6.3259999999999996,"5min":259.68400000000003,"15min":259.68400000000003},"last":0.46500000000000002},{"interface":"front","average":{"1min":2.0129999999999999,"5min":4.8159999999999998,"15min":3.004},"min":{"1min":0.35799999999999998,"5min":0.27000000000000002,"15min":0.23799999999999999},"max":{"1min":6.5819999999999999,"5min":261.02699999999999,"15min":261.02699999999999},"last":0.65000000000000002}]},{"osd":4,"last update":"Sat Jun 3 20:29:38 2023","interfaces":[{"interface":"back","average":{"1min":2.8069999999999999,"5min":1.6870000000000001,"15min":1.659},"min":{"1min":0.34899999999999998,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":11.021000000000001,"5min":11.49,"15min":11.49},"last":0.98099999999999998},{"interface":"front","average":{"1min":1.5549999999999999,"5min":1.1020000000000001,"15min":1.6080000000000001},"min":{"1min":0.34100000000000003,"5min":0.34100000000000003,"15min":0.34100000000000003},"max":{"1min":6.3600000000000003,"5min":6.3600000000000003,"15min":7.21},"last":0.442}]},{"osd":5,"last update":"Sat Jun 3 20:29:30 2023","interfaces":[{"interface":"back","average":{"1min":2.5230000000000001,"5min":1.853,"15min":2.2879999999999998},"min":{"1min":0.33600000000000002,"5min":0.33600000000000002,"15min":0.29299999999999998},"max":{"1min":10.981,"5min":20.238,"15min":243.15899999999999},"last":1.01},{"interface":"front","average":{"1min":1.7190000000000001,"5min":1.395,"15min":2.0960000000000001},"min":{"1min":0.40899999999999997,"5min":0.40500000000000003,"15min":0.33400000000000002},"max":{"1min":6.4139999999999997,"5min":6.4429999999999996,"15min":243.32599999999999},"last":3.0630000000000002}]},{"osd":6,"last update":"Sat Jun 3 20:29:30 2023","interfaces":[{"interface":"back","average":{"1min":1.524,"5min":1.0409999999999999,"15min":1.954},"min":{"1min":0.44700000000000001,"5min":0.36799999999999999,"15min":0.36299999999999999},"max":{"1min":6.3879999999999999,"5min":6.3879999999999999,"15min":243.25},"last":0.38800000000000001},{"interface":"front","average":{"1min":1.357,"5min":1.0760000000000001,"15min":1.9219999999999999},"min":{"1min":0.52200000000000002,"5min":0.379,"15min":0.379},"max":{"1min":4.8719999999999999,"5min":4.8719999999999999,"15min":242.84399999999999},"last":11.634}]},{"osd":7,"last update":"Sat Jun 3 20:29:30 2023","interfaces":[{"interface":"back","average":{"1min":1.7729999999999999,"5min":1.623,"15min":2.3969999999999998},"min":{"1min":0.47799999999999998,"5min":0.41999999999999998,"15min":0.35499999999999998},"max":{"1min":6.4450000000000003,"5min":8.5289999999999999,"15min":242.93600000000001},"last":0.41999999999999998},{"interface":"front","average":{"1min":1.579,"5min":1.5069999999999999,"15min":2.302},"min":{"1min":0.499,"5min":0.46600000000000003,"15min":0.40000000000000002},"max":{"1min":4.8390000000000004,"5min":8.5009999999999994,"15min":243.21299999999999},"last":11.68}]}]},{"osd":2,"up_from":114,"seq":489626272150,"num_pgs":28,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2996548,"kb_used_data":2372220,"kb_used_omap":7,"kb_used_meta":624248,"kb_avail":90748604,"statfs":{"total":95995035648,"available":92926570496,"internally_reserved":0,"allocated":2429153280,"data_stored":2428079277,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8111,"internal_metadata":639230033},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 3 20:28:58 2023","interfaces":[{"interface":"back","average":{"1min":5.5750000000000002,"5min":1.798,"15min":2.7309999999999999},"min":{"1min":0.20399999999999999,"5min":0.193,"15min":0.17799999999999999},"max":{"1min":77.055000000000007,"5min":77.055000000000007,"15min":117.57899999999999},"last":0.70999999999999996},{"interface":"front","average":{"1min":6.0679999999999996,"5min":2.0099999999999998,"15min":2.6379999999999999},"min":{"1min":0.27900000000000003,"5min":0.20999999999999999,"15min":0.20399999999999999},"max":{"1min":76.453000000000003,"5min":76.453000000000003,"15min":117.38500000000001},"last":0.81799999999999995}]},{"osd":1,"last update":"Sat Jun 3 20:28:58 2023","interfaces":[{"interface":"back","average":{"1min":1.4059999999999999,"5min":3.0049999999999999,"15min":1.7050000000000001},"min":{"1min":0.246,"5min":0.222,"15min":0.222},"max":{"1min":4.9359999999999999,"5min":116.127,"15min":116.127},"last":0.58499999999999996},{"interface":"front","average":{"1min":1.431,"5min":3.1230000000000002,"15min":2.0430000000000001},"min":{"1min":0.39900000000000002,"5min":0.26700000000000002,"15min":0.25700000000000001},"max":{"1min":4.8460000000000001,"5min":116.461,"15min":116.461},"last":0.251}]},{"osd":3,"last update":"Sat Jun 3 20:28:58 2023","interfaces":[{"interface":"back","average":{"1min":1.1419999999999999,"5min":1.6659999999999999,"15min":2.3940000000000001},"min":{"1min":0.22700000000000001,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":6.1699999999999999,"5min":36.695,"15min":103.714},"last":0.39100000000000001},{"interface":"front","average":{"1min":1.371,"5min":1.9470000000000001,"15min":2.5059999999999998},"min":{"1min":0.28599999999999998,"5min":0.27100000000000002,"15min":0.23000000000000001},"max":{"1min":3.915,"5min":37.088999999999999,"15min":104.05200000000001},"last":2.2650000000000001}]},{"osd":4,"last update":"Sat Jun 3 20:29:38 2023","interfaces":[{"interface":"back","average":{"1min":1.4099999999999999,"5min":1.5049999999999999,"15min":2.7810000000000001},"min":{"1min":0.40600000000000003,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":7.6719999999999997,"5min":10.664999999999999,"15min":16.561},"last":0.77400000000000002},{"interface":"front","average":{"1min":1.0549999999999999,"5min":1.4159999999999999,"15min":1.7929999999999999},"min":{"1min":0.46899999999999997,"5min":0.35099999999999998,"15min":0.35099999999999998},"max":{"1min":2.8420000000000001,"5min":6.0759999999999996,"15min":9.9339999999999993},"last":0.55400000000000005}]},{"osd":5,"last update":"Sat Jun 3 20:28:58 2023","interfaces":[{"interface":"back","average":{"1min":0.82799999999999996,"5min":1.6990000000000001,"15min":1.617},"min":{"1min":0.39500000000000002,"5min":0.378,"15min":0.32100000000000001},"max":{"1min":1.607,"5min":17.236000000000001,"15min":17.751000000000001},"last":0.73699999999999999},{"interface":"front","average":{"1min":1.2629999999999999,"5min":1.643,"15min":1.4670000000000001},"min":{"1min":0.36299999999999999,"5min":0.316,"15min":0.316},"max":{"1min":3.8010000000000002,"5min":9.2219999999999995,"15min":17.928000000000001},"last":0.67900000000000005}]},{"osd":6,"last update":"Sat Jun 3 20:28:58 2023","interfaces":[{"interface":"back","average":{"1min":1.3400000000000001,"5min":1.371,"15min":1.4339999999999999},"min":{"1min":0.45200000000000001,"5min":0.377,"15min":0.35199999999999998},"max":{"1min":4.476,"5min":5.9160000000000004,"15min":14.331},"last":0.79000000000000004},{"interface":"front","average":{"1min":0.92700000000000005,"5min":1.3069999999999999,"15min":1.306},"min":{"1min":0.39300000000000002,"5min":0.36699999999999999,"15min":0.35099999999999998},"max":{"1min":2.0230000000000001,"5min":5.4370000000000003,"15min":12.867000000000001},"last":0.629}]},{"osd":7,"last update":"Sat Jun 3 20:28:58 2023","interfaces":[{"interface":"back","average":{"1min":1.248,"5min":1.7649999999999999,"15min":1.6759999999999999},"min":{"1min":0.45500000000000002,"5min":0.44400000000000001,"15min":0.34799999999999998},"max":{"1min":3.77,"5min":10.529999999999999,"15min":14.243},"last":0.76500000000000001},{"interface":"front","average":{"1min":1.5149999999999999,"5min":1.7370000000000001,"15min":1.577},"min":{"1min":0.45300000000000001,"5min":0.45300000000000001,"15min":0.38600000000000001},"max":{"1min":3.714,"5min":8.5690000000000008,"15min":12.845000000000001},"last":0.65200000000000002}]}]},{"osd":3,"up_from":119,"seq":511101108622,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2689732,"kb_used_data":2091068,"kb_used_omap":2,"kb_used_meta":598525,"kb_avail":91055420,"statfs":{"total":95995035648,"available":93240750080,"internally_reserved":0,"allocated":2141253632,"data_stored":2140117902,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":612889653},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 3 20:28:58 2023","interfaces":[{"interface":"back","average":{"1min":9.6539999999999999,"5min":2.9510000000000001,"15min":2.7109999999999999},"min":{"1min":0.19600000000000001,"5min":0.19600000000000001,"15min":0.19600000000000001},"max":{"1min":168.33099999999999,"5min":168.33099999999999,"15min":168.33099999999999},"last":0.42799999999999999},{"interface":"front","average":{"1min":10.372,"5min":3.1179999999999999,"15min":2.782},"min":{"1min":0.26500000000000001,"5min":0.222,"15min":0.183},"max":{"1min":169.25800000000001,"5min":169.25800000000001,"15min":169.25800000000001},"last":0.38100000000000001}]},{"osd":1,"last update":"Sat Jun 3 20:28:58 2023","interfaces":[{"interface":"back","average":{"1min":1.6040000000000001,"5min":1.1890000000000001,"15min":1.512},"min":{"1min":0.245,"5min":0.22900000000000001,"15min":0.22500000000000001},"max":{"1min":4.1950000000000003,"5min":20.992000000000001,"15min":117.047},"last":2.0070000000000001},{"interface":"front","average":{"1min":2.198,"5min":1.3,"15min":1.55},"min":{"1min":0.36399999999999999,"5min":0.25800000000000001,"15min":0.20399999999999999},"max":{"1min":7.008,"5min":21.07,"15min":116.505},"last":0.57199999999999995}]},{"osd":2,"last update":"Sat Jun 3 20:28:58 2023","interfaces":[{"interface":"back","average":{"1min":1.73,"5min":2.117,"15min":2.5960000000000001},"min":{"1min":0.26600000000000001,"5min":0.25600000000000001,"15min":0.22900000000000001},"max":{"1min":5.3460000000000001,"5min":88.757999999999996,"15min":117.584},"last":8.2829999999999995},{"interface":"front","average":{"1min":2.0310000000000001,"5min":2.125,"15min":2.6280000000000001},"min":{"1min":0.28399999999999997,"5min":0.26900000000000002,"15min":0.23599999999999999},"max":{"1min":8.5069999999999997,"5min":88.106999999999999,"15min":117.01000000000001},"last":4.016}]},{"osd":4,"last update":"Sat Jun 3 20:29:44 2023","interfaces":[{"interface":"back","average":{"1min":1.8049999999999999,"5min":1.2,"15min":1.4079999999999999},"min":{"1min":0.39600000000000002,"5min":0.34599999999999997,"15min":0.34599999999999997},"max":{"1min":5.7229999999999999,"5min":6.3150000000000004,"15min":6.8719999999999999},"last":2.089},{"interface":"front","average":{"1min":1.623,"5min":1.6200000000000001,"15min":1.738},"min":{"1min":0.46999999999999997,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":5.2110000000000003,"5min":10.773999999999999,"15min":10.773999999999999},"last":0.34599999999999997}]},{"osd":5,"last update":"Sat Jun 3 20:28:58 2023","interfaces":[{"interface":"back","average":{"1min":1.716,"5min":1.46,"15min":1.6140000000000001},"min":{"1min":0.36299999999999999,"5min":0.36299999999999999,"15min":0.313},"max":{"1min":5.4480000000000004,"5min":7.6429999999999998,"15min":38.651000000000003},"last":0.46300000000000002},{"interface":"front","average":{"1min":1.5960000000000001,"5min":1.2509999999999999,"15min":1.391},"min":{"1min":0.376,"5min":0.34499999999999997,"15min":0.316},"max":{"1min":5.383,"5min":5.383,"15min":38.585999999999999},"last":0.44800000000000001}]},{"osd":6,"last update":"Sat Jun 3 20:28:58 2023","interfaces":[{"interface":"back","average":{"1min":1.4399999999999999,"5min":1.141,"15min":1.2929999999999999},"min":{"1min":0.44400000000000001,"5min":0.36899999999999999,"15min":0.33500000000000002},"max":{"1min":5.4189999999999996,"5min":5.4189999999999996,"15min":9.0719999999999992},"last":0.41099999999999998},{"interface":"front","average":{"1min":1.359,"5min":1.1870000000000001,"15min":1.446},"min":{"1min":0.38400000000000001,"5min":0.36699999999999999,"15min":0.32500000000000001},"max":{"1min":4.6150000000000002,"5min":5.1600000000000001,"15min":8.7279999999999998},"last":2.04}]},{"osd":7,"last update":"Sat Jun 3 20:28:58 2023","interfaces":[{"interface":"back","average":{"1min":1.492,"5min":1.4410000000000001,"15min":1.6839999999999999},"min":{"1min":0.47399999999999998,"5min":0.45000000000000001,"15min":0.35599999999999998},"max":{"1min":4.6429999999999998,"5min":6.0700000000000003,"15min":39.008000000000003},"last":2.0659999999999998},{"interface":"front","average":{"1min":1.671,"5min":1.9379999999999999,"15min":1.9159999999999999},"min":{"1min":0.38700000000000001,"5min":0.38700000000000001,"15min":0.34899999999999998},"max":{"1min":5.4660000000000002,"5min":12.831,"15min":40.003999999999998},"last":0.39500000000000002}]}]},{"osd":4,"up_from":250,"seq":1073741824078,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2020540,"kb_used_data":1793844,"kb_used_omap":10,"kb_used_meta":226613,"kb_avail":91724612,"statfs":{"total":95995035648,"available":93926002688,"internally_reserved":0,"allocated":1836896256,"data_stored":1835651984,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":10652,"internal_metadata":232052324},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 3 20:29:38 2023","interfaces":[{"interface":"back","average":{"1min":7.1109999999999998,"5min":2.589,"15min":2.0819999999999999},"min":{"1min":0.28299999999999997,"5min":0.27600000000000002,"15min":0.26300000000000001},"max":{"1min":145.755,"5min":145.755,"15min":145.755},"last":2.1629999999999998},{"interface":"front","average":{"1min":7.7670000000000003,"5min":2.8769999999999998,"15min":1.9330000000000001},"min":{"1min":0.30599999999999999,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":145.71799999999999,"5min":145.71799999999999,"15min":145.71799999999999},"last":0.45000000000000001}]},{"osd":1,"last update":"Sat Jun 3 20:29:38 2023","interfaces":[{"interface":"back","average":{"1min":1.7769999999999999,"5min":1.573,"15min":1.6619999999999999},"min":{"1min":0.36199999999999999,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":9.7769999999999992,"5min":9.7769999999999992,"15min":9.7769999999999992},"last":2.5659999999999998},{"interface":"front","average":{"1min":1.3540000000000001,"5min":1.4950000000000001,"15min":1.7529999999999999},"min":{"1min":0.35499999999999998,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":6.8010000000000002,"5min":11.994,"15min":11.994},"last":3.9039999999999999}]},{"osd":2,"last update":"Sat Jun 3 20:29:38 2023","interfaces":[{"interface":"back","average":{"1min":1.661,"5min":2.4399999999999999,"15min":2.198},"min":{"1min":0.497,"5min":0.36299999999999999,"15min":0.35899999999999999},"max":{"1min":3.8340000000000001,"5min":78.605000000000004,"15min":78.605000000000004},"last":0.52800000000000002},{"interface":"front","average":{"1min":1.744,"5min":2.6080000000000001,"15min":2.6560000000000001},"min":{"1min":0.39100000000000001,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":6.7160000000000002,"5min":73.569000000000003,"15min":73.569000000000003},"last":0.504}]},{"osd":3,"last update":"Sat Jun 3 20:29:38 2023","interfaces":[{"interface":"back","average":{"1min":2.5499999999999998,"5min":2.0920000000000001,"15min":10.384},"min":{"1min":0.379,"5min":0.379,"15min":0.32500000000000001},"max":{"1min":25.007000000000001,"5min":33.667999999999999,"15min":217.578},"last":4.0469999999999997},{"interface":"front","average":{"1min":2.194,"5min":2.21,"15min":10.222},"min":{"1min":0.39300000000000002,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":25.358000000000001,"5min":34.094000000000001,"15min":218.55699999999999},"last":2.3610000000000002}]},{"osd":5,"last update":"Sat Jun 3 20:29:38 2023","interfaces":[{"interface":"back","average":{"1min":0.88800000000000001,"5min":1.6699999999999999,"15min":2.8940000000000001},"min":{"1min":0.36399999999999999,"5min":0.35099999999999998,"15min":0.35099999999999998},"max":{"1min":2.883,"5min":11.284000000000001,"15min":23.913},"last":1.0229999999999999},{"interface":"front","average":{"1min":1.3049999999999999,"5min":1.6919999999999999,"15min":2.0939999999999999},"min":{"1min":0.38800000000000001,"5min":0.34499999999999997,"15min":0.34499999999999997},"max":{"1min":8.2650000000000006,"5min":11.093999999999999,"15min":11.093999999999999},"last":0.94699999999999995}]},{"osd":6,"last update":"Sat Jun 3 20:29:38 2023","interfaces":[{"interface":"back","average":{"1min":1.177,"5min":1.617,"15min":2.5390000000000001},"min":{"1min":0.437,"5min":0.40400000000000003,"15min":0.40400000000000003},"max":{"1min":3.6789999999999998,"5min":10.952,"15min":23.963000000000001},"last":0.97399999999999998},{"interface":"front","average":{"1min":1.4379999999999999,"5min":1.603,"15min":1.4670000000000001},"min":{"1min":0.45800000000000002,"5min":0.40799999999999997,"15min":0.40799999999999997},"max":{"1min":6.8630000000000004,"5min":8.4190000000000005,"15min":8.4190000000000005},"last":0.996}]},{"osd":7,"last update":"Sat Jun 3 20:29:38 2023","interfaces":[{"interface":"back","average":{"1min":1.133,"5min":1.927,"15min":2.1789999999999998},"min":{"1min":0.40999999999999998,"5min":0.39800000000000002,"15min":0.374},"max":{"1min":3.1749999999999998,"5min":10.747999999999999,"15min":10.747999999999999},"last":1.1930000000000001},{"interface":"front","average":{"1min":1.2549999999999999,"5min":1.7250000000000001,"15min":2.145},"min":{"1min":0.433,"5min":0.34699999999999998,"15min":0.34699999999999998},"max":{"1min":6.7699999999999996,"5min":12.154,"15min":12.154},"last":0.76900000000000002}]}]},{"osd":5,"up_from":36,"seq":154618823381,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2430756,"kb_used_data":1474396,"kb_used_omap":0,"kb_used_meta":956224,"kb_avail":91314396,"statfs":{"total":95995035648,"available":93505941504,"internally_reserved":0,"allocated":1509781504,"data_stored":1509139375,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":979173376},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 3 20:29:21 2023","interfaces":[{"interface":"back","average":{"1min":2.415,"5min":2.4430000000000001,"15min":2.448},"min":{"1min":0.30399999999999999,"5min":0.27100000000000002,"15min":0.25700000000000001},"max":{"1min":32.518999999999998,"5min":104.325,"15min":126.461},"last":1.456},{"interface":"front","average":{"1min":2.786,"5min":2.6000000000000001,"15min":2.6309999999999998},"min":{"1min":0.34399999999999997,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":32.545000000000002,"5min":104.46599999999999,"15min":126.497},"last":1.4430000000000001}]},{"osd":1,"last update":"Sat Jun 3 20:29:21 2023","interfaces":[{"interface":"back","average":{"1min":1.417,"5min":2.4020000000000001,"15min":1.9399999999999999},"min":{"1min":0.32100000000000001,"5min":0.29099999999999998,"15min":0.28799999999999998},"max":{"1min":10,"5min":55.515000000000001,"15min":102.205},"last":0.68400000000000005},{"interface":"front","average":{"1min":1.597,"5min":2.3599999999999999,"15min":1.887},"min":{"1min":0.32800000000000001,"5min":0.30499999999999999,"15min":0.27600000000000002},"max":{"1min":6.4589999999999996,"5min":55.194000000000003,"15min":102.265},"last":0.65700000000000003}]},{"osd":2,"last update":"Sat Jun 3 20:29:21 2023","interfaces":[{"interface":"back","average":{"1min":1.21,"5min":5.319,"15min":4.3209999999999997},"min":{"1min":0.40300000000000002,"5min":0.36199999999999999,"15min":0.33400000000000002},"max":{"1min":5.3159999999999998,"5min":236.40000000000001,"15min":236.40000000000001},"last":0.624},{"interface":"front","average":{"1min":2.1899999999999999,"5min":5.7480000000000002,"15min":4.6310000000000002},"min":{"1min":0.442,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":5.931,"5min":237.86099999999999,"15min":237.86099999999999},"last":0.747}]},{"osd":3,"last update":"Sat Jun 3 20:28:56 2023","interfaces":[{"interface":"back","average":{"1min":2.605,"5min":1.923,"15min":3.3860000000000001},"min":{"1min":0.46600000000000003,"5min":0.34399999999999997,"15min":0.34399999999999997},"max":{"1min":35.725999999999999,"5min":35.725999999999999,"15min":264.822},"last":0.93500000000000005},{"interface":"front","average":{"1min":2.5720000000000001,"5min":1.649,"15min":3.2669999999999999},"min":{"1min":0.44600000000000001,"5min":0.39200000000000002,"15min":0.37},"max":{"1min":35.762999999999998,"5min":35.762999999999998,"15min":264.75},"last":0.81799999999999995}]},{"osd":4,"last update":"Sat Jun 3 20:29:36 2023","interfaces":[{"interface":"back","average":{"1min":1.083,"5min":1.9610000000000001,"15min":3.1539999999999999},"min":{"1min":0.35599999999999998,"5min":0.34699999999999998,"15min":0.34699999999999998},"max":{"1min":3.8069999999999999,"5min":21.786000000000001,"15min":21.786000000000001},"last":1.0229999999999999},{"interface":"front","average":{"1min":1.3320000000000001,"5min":1.51,"15min":2.5419999999999998},"min":{"1min":0.44900000000000001,"5min":0.36799999999999999,"15min":0.30199999999999999},"max":{"1min":3.6680000000000001,"5min":9.7010000000000005,"15min":26.478000000000002},"last":1.089}]},{"osd":6,"last update":"Sat Jun 3 20:29:21 2023","interfaces":[{"interface":"back","average":{"1min":0.76300000000000001,"5min":1.5760000000000001,"15min":1.2270000000000001},"min":{"1min":0.44,"5min":0.35499999999999998,"15min":0.29099999999999998},"max":{"1min":1.246,"5min":7.5140000000000002,"15min":9.7029999999999994},"last":1.2749999999999999},{"interface":"front","average":{"1min":0.85599999999999998,"5min":1.5580000000000001,"15min":1.304},"min":{"1min":0.53700000000000003,"5min":0.37,"15min":0.35799999999999998},"max":{"1min":1.395,"5min":6.5309999999999997,"15min":6.5309999999999997},"last":1.123}]},{"osd":7,"last update":"Sat Jun 3 20:29:10 2023","interfaces":[{"interface":"back","average":{"1min":0.83499999999999996,"5min":2.0899999999999999,"15min":1.5469999999999999},"min":{"1min":0.47999999999999998,"5min":0.47099999999999997,"15min":0.35999999999999999},"max":{"1min":1.421,"5min":15.516,"15min":15.516},"last":1.1859999999999999},{"interface":"front","average":{"1min":0.90900000000000003,"5min":1.8999999999999999,"15min":1.327},"min":{"1min":0.49199999999999999,"5min":0.47399999999999998,"15min":0.373},"max":{"1min":3.9420000000000002,"5min":9.6519999999999992,"15min":9.6519999999999992},"last":1.1679999999999999}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6928,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4862,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":10652,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4786,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4932,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":14,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":2405957632,"data_stored":2405957632,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":14,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":2418868224,"data_stored":2418868224,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":14,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":2419789824,"data_stored":2419785752,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":14,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":2121990144,"data_stored":2121990144,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":14,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":1831079936,"data_stored":1831075864,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":14,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":1496911872,"data_stored":1496907800,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":14,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":978784256,"data_stored":978780184,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":14,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":906493952,"data_stored":906493952,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-06-03T20:30:03.285 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph status --format=json 2023-06-03T20:30:03.303 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:30:03.373 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:30:03.380 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:30:03.381 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:30:03.482 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:30:03.518 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:30:03.521 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:30:03.522 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:30:03.623 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:30:03.656 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:30:03.661 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:30:03.662 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:30:03.763 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:30:03.806 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:30:03.811 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:30:03.812 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:30:03.913 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:30:03.960 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:30:03.963 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:30:03.964 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:30:04.065 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:30:04.097 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:30:04.105 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:30:04.105 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:30:04 smithi089 bash[75068]: cluster 1685824203.1725018 mgr.y (mgr.34104) 1627 : cluster 0 pgmap v1681: 89 pgs: 1 active+recovering+remapped, 88 active+clean; 4.1 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 774 op/s; 3489/202851 objects misplaced (1.720%); 2.1 MiB/s, 33 objects/s recovering 2023-06-03T20:30:04.105 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:30:04 smithi089 bash[78452]: cluster 1685824203.1725018 mgr.y (mgr.34104) 1627 : cluster 0 pgmap v1681: 89 pgs: 1 active+recovering+remapped, 88 active+clean; 4.1 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 774 op/s; 3489/202851 objects misplaced (1.720%); 2.1 MiB/s, 33 objects/s recovering 2023-06-03T20:30:04.106 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:30:04.207 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:30:04.216 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:30:04 smithi164 bash[59462]: cluster 1685824203.1725018 mgr.y (mgr.34104) 1627 : cluster 0 pgmap v1681: 89 pgs: 1 active+recovering+remapped, 88 active+clean; 4.1 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 774 op/s; 3489/202851 objects misplaced (1.720%); 2.1 MiB/s, 33 objects/s recovering 2023-06-03T20:30:04.269 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:30:04.271 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:30:04.272 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:30:04.372 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:30:04.402 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:30:04.403 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:30:04.404 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:30:04.505 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:30:04.545 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:30:04.551 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:30:04.551 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:30:04.652 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:30:04.687 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:30:04.690 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:30:04.691 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:30:04.792 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:30:04.830 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:30:04.833 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:30:04.833 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:30:04.934 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:30:04.973 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:30:04.976 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:30:04.977 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:30:05.078 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:30:05.103 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:30:05.106 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:30:05.107 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:30:05.208 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:30:05.263 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:30:05.273 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:30:05.274 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:30:05.374 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:30:05.416 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:30:05.448 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:30:05.449 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:30:05.550 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:30:05.584 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:30:05.586 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:30:05.587 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:30:05.688 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:30:05.714 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:30:05.722 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:30:05.723 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:30:05.824 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:30:05.857 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:30:05.860 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:30:05.861 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:30:05.962 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:30:06.004 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:30:06.011 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:30:06.012 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:30:06.113 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:30:06.144 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:30:06.146 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:30:06.147 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:30:06.247 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:30:06.286 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:30:06.287 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:30:06.288 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:30:06.369 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 20:30:06 smithi089 bash[71163]: ::ffff:172.21.15.164 - - [03/Jun/2023:20:30:06] "GET /metrics HTTP/1.1" 200 229137 "" "Prometheus/2.33.4" 2023-06-03T20:30:06.370 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:30:06 smithi089 bash[75068]: cluster 1685824205.1730287 mgr.y (mgr.34104) 1628 : cluster 0 pgmap v1682: 89 pgs: 1 active+recovering+remapped, 88 active+clean; 4.1 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 516 op/s; 3489/202851 objects misplaced (1.720%); 1.4 MiB/s, 22 objects/s recovering 2023-06-03T20:30:06.371 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:30:06 smithi089 bash[75068]: audit 1685824206.0137687 mon.a (mon.0) 1627 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:30:06.371 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:30:06 smithi089 bash[75068]: audit 1685824206.0258615 mon.a (mon.0) 1628 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:30:06.371 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:30:06 smithi089 bash[78452]: cluster 1685824205.1730287 mgr.y (mgr.34104) 1628 : cluster 0 pgmap v1682: 89 pgs: 1 active+recovering+remapped, 88 active+clean; 4.1 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 516 op/s; 3489/202851 objects misplaced (1.720%); 1.4 MiB/s, 22 objects/s recovering 2023-06-03T20:30:06.371 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:30:06 smithi089 bash[78452]: audit 1685824206.0137687 mon.a (mon.0) 1627 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:30:06.371 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:30:06 smithi089 bash[78452]: audit 1685824206.0258615 mon.a (mon.0) 1628 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:30:06.389 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:30:06.429 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:30:06.442 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:30:06.443 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:30:06.543 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:30:06.551 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:30:06 smithi164 bash[59462]: cluster 1685824205.1730287 mgr.y (mgr.34104) 1628 : cluster 0 pgmap v1682: 89 pgs: 1 active+recovering+remapped, 88 active+clean; 4.1 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 516 op/s; 3489/202851 objects misplaced (1.720%); 1.4 MiB/s, 22 objects/s recovering 2023-06-03T20:30:06.551 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:30:06 smithi164 bash[59462]: audit 1685824206.0137687 mon.a (mon.0) 1627 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:30:06.552 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:30:06 smithi164 bash[59462]: audit 1685824206.0258615 mon.a (mon.0) 1628 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:30:06.573 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:30:06.579 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:30:06.580 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:30:06.680 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:30:06.728 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:30:06.733 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:30:06.734 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:30:06.835 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:30:06.870 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:30:06.875 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:30:06.876 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:30:06.977 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:30:07.009 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:30:07.013 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:30:07.014 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:30:07.115 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:30:07.157 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:30:07.162 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:30:07.162 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:30:07.247 INFO:teuthology.orchestra.run.smithi089.stdout:osd.0: {} 2023-06-03T20:30:07.263 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:30:07.277 INFO:teuthology.orchestra.run.smithi089.stderr:nodeep-scrub is set 2023-06-03T20:30:07.296 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:30:07.298 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:30:07.299 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:30:07.300 INFO:teuthology.orchestra.run.smithi089.stdout:osd.1: {} 2023-06-03T20:30:07.340 INFO:teuthology.orchestra.run.smithi089.stdout:osd.2: {} 2023-06-03T20:30:07.388 INFO:teuthology.orchestra.run.smithi089.stdout:osd.3: {} 2023-06-03T20:30:07.400 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:30:07.410 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:30:07 smithi164 bash[59462]: audit 1685824206.712078 mon.a (mon.0) 1629 : audit 1 from='client.? 172.21.15.89:0/3334109258' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-06-03T20:30:07.411 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:30:07 smithi164 bash[59462]: audit 1685824206.7600436 mon.c (mon.2) 665 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-03T20:30:07.411 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:30:07 smithi164 bash[59462]: audit 1685824206.775828 mon.c (mon.2) 666 : audit 1 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-03T20:30:07.411 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:30:07 smithi164 bash[59462]: audit 1685824206.7833698 mon.a (mon.0) 1630 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:30:07.411 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:30:07 smithi164 bash[59462]: audit 1685824207.1548786 mon.c (mon.2) 667 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T20:30:07.411 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:30:07 smithi164 bash[59462]: audit 1685824207.158149 mon.a (mon.0) 1631 : audit 0 from='client.? 172.21.15.89:0/2830457371' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-03T20:30:07.428 INFO:teuthology.orchestra.run.smithi089.stdout:osd.4: {} 2023-06-03T20:30:07.446 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:30:07.450 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:30:07.451 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:30:07.463 INFO:teuthology.orchestra.run.smithi089.stdout:osd.5: {} 2023-06-03T20:30:07.491 INFO:teuthology.orchestra.run.smithi089.stdout:osd.6: {} 2023-06-03T20:30:07.523 INFO:teuthology.orchestra.run.smithi089.stdout:osd.7: {} 2023-06-03T20:30:07.526 INFO:teuthology.orchestra.run.smithi089.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-06-03T20:30:07.527 INFO:teuthology.orchestra.run.smithi089.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-06-03T20:30:07.527 INFO:teuthology.orchestra.run.smithi089.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-06-03T20:30:07.527 INFO:teuthology.orchestra.run.smithi089.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-06-03T20:30:07.527 INFO:teuthology.orchestra.run.smithi089.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-06-03T20:30:07.527 INFO:teuthology.orchestra.run.smithi089.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-06-03T20:30:07.527 INFO:teuthology.orchestra.run.smithi089.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-06-03T20:30:07.527 INFO:teuthology.orchestra.run.smithi089.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-06-03T20:30:07.529 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:30:07 smithi089 bash[75068]: audit 1685824206.712078 mon.a (mon.0) 1629 : audit 1 from='client.? 172.21.15.89:0/3334109258' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-06-03T20:30:07.529 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:30:07 smithi089 bash[75068]: audit 1685824206.7600436 mon.c (mon.2) 665 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-03T20:30:07.529 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:30:07 smithi089 bash[75068]: audit 1685824206.775828 mon.c (mon.2) 666 : audit 1 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-03T20:30:07.530 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:30:07 smithi089 bash[75068]: audit 1685824206.7833698 mon.a (mon.0) 1630 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:30:07.530 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:30:07 smithi089 bash[75068]: audit 1685824207.1548786 mon.c (mon.2) 667 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T20:30:07.530 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:30:07 smithi089 bash[75068]: audit 1685824207.158149 mon.a (mon.0) 1631 : audit 0 from='client.? 172.21.15.89:0/2830457371' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-03T20:30:07.530 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:30:07 smithi089 bash[78452]: audit 1685824206.712078 mon.a (mon.0) 1629 : audit 1 from='client.? 172.21.15.89:0/3334109258' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-06-03T20:30:07.530 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:30:07 smithi089 bash[78452]: audit 1685824206.7600436 mon.c (mon.2) 665 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-03T20:30:07.531 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:30:07 smithi089 bash[78452]: audit 1685824206.775828 mon.c (mon.2) 666 : audit 1 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-03T20:30:07.531 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:30:07 smithi089 bash[78452]: audit 1685824206.7833698 mon.a (mon.0) 1630 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:30:07.531 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:30:07 smithi089 bash[78452]: audit 1685824207.1548786 mon.c (mon.2) 667 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T20:30:07.531 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:30:07 smithi089 bash[78452]: audit 1685824207.158149 mon.a (mon.0) 1631 : audit 0 from='client.? 172.21.15.89:0/2830457371' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-03T20:30:07.551 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:30:07.589 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:30:07.592 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:30:07.592 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:30:07.693 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:30:07.726 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:30:07.730 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:30:07.731 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:30:07.832 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:30:07.864 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:30:07.867 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:30:07.867 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:30:07.968 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:30:08.022 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:30:08.027 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:30:08.027 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:30:08.128 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:30:08.169 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:30:08.176 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:30:08.176 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:30:08.277 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:30:08.305 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:30:08.313 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:30:08.314 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:30:08.414 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:30:08.445 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:30:08.453 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:30:08.454 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:30:08.455 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:30:08 smithi089 bash[75068]: cluster 1685824207.1737168 mgr.y (mgr.34104) 1629 : cluster 0 pgmap v1683: 89 pgs: 1 active+recovering+remapped, 88 active+clean; 4.1 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 546 op/s; 3489/201762 objects misplaced (1.729%); 1.4 MiB/s, 22 objects/s recovering 2023-06-03T20:30:08.455 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:30:08 smithi089 bash[75068]: cluster 1685824207.236849 mon.a (mon.0) 1632 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-06-03T20:30:08.455 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:30:08 smithi089 bash[75068]: cluster 1685824207.2563515 osd.6 (osd.6) 226 : cluster 3 failed to encode map e291 with expected crc 2023-06-03T20:30:08.455 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:30:08 smithi089 bash[75068]: audit 1685824207.2591565 mon.a (mon.0) 1633 : audit 1 from='client.? 172.21.15.89:0/3334109258' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-06-03T20:30:08.455 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:30:08 smithi089 bash[75068]: cluster 1685824207.2592127 mon.a (mon.0) 1634 : cluster 0 osdmap e291: 8 total, 8 up, 8 in 2023-06-03T20:30:08.456 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:30:08 smithi089 bash[75068]: cluster 1685824207.2691076 osd.5 (osd.5) 240 : cluster 3 failed to encode map e291 with expected crc 2023-06-03T20:30:08.456 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:30:08 smithi089 bash[75068]: cluster 1685824207.278883 osd.7 (osd.7) 238 : cluster 3 failed to encode map e291 with expected crc 2023-06-03T20:30:08.456 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:30:08 smithi089 bash[78452]: cluster 1685824207.1737168 mgr.y (mgr.34104) 1629 : cluster 0 pgmap v1683: 89 pgs: 1 active+recovering+remapped, 88 active+clean; 4.1 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 546 op/s; 3489/201762 objects misplaced (1.729%); 1.4 MiB/s, 22 objects/s recovering 2023-06-03T20:30:08.456 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:30:08 smithi089 bash[78452]: cluster 1685824207.236849 mon.a (mon.0) 1632 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-06-03T20:30:08.456 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:30:08 smithi089 bash[78452]: cluster 1685824207.2563515 osd.6 (osd.6) 226 : cluster 3 failed to encode map e291 with expected crc 2023-06-03T20:30:08.456 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:30:08 smithi089 bash[78452]: audit 1685824207.2591565 mon.a (mon.0) 1633 : audit 1 from='client.? 172.21.15.89:0/3334109258' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-06-03T20:30:08.457 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:30:08 smithi089 bash[78452]: cluster 1685824207.2592127 mon.a (mon.0) 1634 : cluster 0 osdmap e291: 8 total, 8 up, 8 in 2023-06-03T20:30:08.457 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:30:08 smithi089 bash[78452]: cluster 1685824207.2691076 osd.5 (osd.5) 240 : cluster 3 failed to encode map e291 with expected crc 2023-06-03T20:30:08.457 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:30:08 smithi089 bash[78452]: cluster 1685824207.278883 osd.7 (osd.7) 238 : cluster 3 failed to encode map e291 with expected crc 2023-06-03T20:30:08.556 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:30:08.585 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:30:08 smithi164 bash[59462]: cluster 1685824207.1737168 mgr.y (mgr.34104) 1629 : cluster 0 pgmap v1683: 89 pgs: 1 active+recovering+remapped, 88 active+clean; 4.1 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 546 op/s; 3489/201762 objects misplaced (1.729%); 1.4 MiB/s, 22 objects/s recovering 2023-06-03T20:30:08.585 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:30:08 smithi164 bash[59462]: cluster 1685824207.236849 mon.a (mon.0) 1632 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-06-03T20:30:08.586 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:30:08 smithi164 bash[59462]: cluster 1685824207.2563515 osd.6 (osd.6) 226 : cluster 3 failed to encode map e291 with expected crc 2023-06-03T20:30:08.586 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:30:08 smithi164 bash[59462]: audit 1685824207.2591565 mon.a (mon.0) 1633 : audit 1 from='client.? 172.21.15.89:0/3334109258' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-06-03T20:30:08.586 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:30:08 smithi164 bash[59462]: cluster 1685824207.2592127 mon.a (mon.0) 1634 : cluster 0 osdmap e291: 8 total, 8 up, 8 in 2023-06-03T20:30:08.586 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:30:08 smithi164 bash[59462]: cluster 1685824207.2691076 osd.5 (osd.5) 240 : cluster 3 failed to encode map e291 with expected crc 2023-06-03T20:30:08.586 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:30:08 smithi164 bash[59462]: cluster 1685824207.278883 osd.7 (osd.7) 238 : cluster 3 failed to encode map e291 with expected crc 2023-06-03T20:30:08.595 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:30:08.601 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:30:08.601 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:30:08.702 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:30:08.740 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:30:08.743 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:30:08.743 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:30:08.844 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:30:08.876 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:30:08.878 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:30:08.879 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:30:08.979 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:30:09.011 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:30:09.019 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:30:09.020 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:30:09.121 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:30:09.151 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:30:09.155 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:30:09.156 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:30:09.257 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:30:09.286 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:30:09.289 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:30:09.290 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:30:09.391 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:30:09.428 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:30:09.434 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:30:09.434 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:30:09.535 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:30:09.567 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:30:09.571 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:30:09.572 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:30:09.673 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:30:09.705 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:30:09.713 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:30:09.714 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:30:09.815 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:30:09.850 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:30:09.856 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:30:09.857 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:30:09.958 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:30:09.981 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:30:09.983 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:30:09.984 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:30:10.085 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:30:10.119 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:30:10.122 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:30:10.123 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:30:10.223 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:30:10.260 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:30:10.263 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:30:10.263 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:30:10.364 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:30:10.372 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:30:10 smithi089 bash[75068]: cluster 1685824209.1744242 mgr.y (mgr.34104) 1630 : cluster 0 pgmap v1685: 89 pgs: 1 active+recovering+remapped, 88 active+clean; 4.0 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 588 op/s; 3377/194451 objects misplaced (1.737%); 1.6 MiB/s, 25 objects/s recovering 2023-06-03T20:30:10.373 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:30:10 smithi089 bash[78452]: cluster 1685824209.1744242 mgr.y (mgr.34104) 1630 : cluster 0 pgmap v1685: 89 pgs: 1 active+recovering+remapped, 88 active+clean; 4.0 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 588 op/s; 3377/194451 objects misplaced (1.737%); 1.6 MiB/s, 25 objects/s recovering 2023-06-03T20:30:10.404 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:30:10.409 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:30:10.410 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:30:10.510 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:30:10.561 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:30:10.563 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:30:10.564 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:30:10.585 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:30:10 smithi164 bash[59462]: cluster 1685824209.1744242 mgr.y (mgr.34104) 1630 : cluster 0 pgmap v1685: 89 pgs: 1 active+recovering+remapped, 88 active+clean; 4.0 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 588 op/s; 3377/194451 objects misplaced (1.737%); 1.6 MiB/s, 25 objects/s recovering 2023-06-03T20:30:10.666 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:30:10.696 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:30:10.698 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:30:10.699 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:30:10.799 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:30:10.840 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:30:10.842 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:30:10.842 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:30:10.943 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:30:10.983 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:30:10.987 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:30:10.987 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:30:11.088 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:30:11.127 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:30:11.136 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:30:11.137 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:30:11.238 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:30:11.281 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:30:11.287 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:30:11.287 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:30:11.388 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:30:11.417 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:30:11.419 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:30:11.419 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:30:11.520 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:30:11.564 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:30:11.567 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:30:11.568 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:30:11.669 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:30:11.700 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:30:11.703 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:30:11.704 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:30:11.805 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:30:11.813 INFO:teuthology.orchestra.run.smithi089.stdout:{ 2023-06-03T20:30:11.813 INFO:teuthology.orchestra.run.smithi089.stdout: "ops": [ 2023-06-03T20:30:11.814 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:30:11.814 INFO:teuthology.orchestra.run.smithi089.stdout: "description": "osd_repop(client.46739.0:133669 14.3 e291/278 14:c5140b49:::benchmark_data_smithi089_237281_object35653:head v 291'8405, mlcod=291'8403)", 2023-06-03T20:30:11.814 INFO:teuthology.orchestra.run.smithi089.stdout: "initiated_at": "2023-06-03T20:30:11.799656+0000", 2023-06-03T20:30:11.814 INFO:teuthology.orchestra.run.smithi089.stdout: "age": 0.0083491499999999996, 2023-06-03T20:30:11.814 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.0083799319999999997, 2023-06-03T20:30:11.814 INFO:teuthology.orchestra.run.smithi089.stdout: "type_data": { 2023-06-03T20:30:11.814 INFO:teuthology.orchestra.run.smithi089.stdout: "flag_point": "started", 2023-06-03T20:30:11.814 INFO:teuthology.orchestra.run.smithi089.stdout: "events": [ 2023-06-03T20:30:11.815 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:30:11.815 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "initiated", 2023-06-03T20:30:11.815 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:30:11.799656+0000", 2023-06-03T20:30:11.815 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0 2023-06-03T20:30:11.815 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:30:11.815 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:30:11.815 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "throttled", 2023-06-03T20:30:11.815 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:30:11.799656+0000", 2023-06-03T20:30:11.816 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0 2023-06-03T20:30:11.816 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:30:11.816 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:30:11.816 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "header_read", 2023-06-03T20:30:11.816 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:30:11.799655+0000", 2023-06-03T20:30:11.816 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 4294967295.9999986 2023-06-03T20:30:11.816 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:30:11.817 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:30:11.817 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "all_read", 2023-06-03T20:30:11.817 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:30:11.799665+0000", 2023-06-03T20:30:11.817 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 1.0601999999999999e-05 2023-06-03T20:30:11.817 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:30:11.817 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:30:11.817 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "dispatched", 2023-06-03T20:30:11.817 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:30:11.799687+0000", 2023-06-03T20:30:11.818 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 2.1923e-05 2023-06-03T20:30:11.818 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:30:11.818 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:30:11.818 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "queued_for_pg", 2023-06-03T20:30:11.818 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:30:11.799697+0000", 2023-06-03T20:30:11.818 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 9.9240000000000002e-06 2023-06-03T20:30:11.818 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:30:11.818 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:30:11.819 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "reached_pg", 2023-06-03T20:30:11.819 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:30:11.804616+0000", 2023-06-03T20:30:11.819 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.0049186550000000001 2023-06-03T20:30:11.819 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:30:11.819 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:30:11.819 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "started", 2023-06-03T20:30:11.819 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:30:11.804665+0000", 2023-06-03T20:30:11.819 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 4.8959999999999999e-05 2023-06-03T20:30:11.820 INFO:teuthology.orchestra.run.smithi089.stdout: } 2023-06-03T20:30:11.820 INFO:teuthology.orchestra.run.smithi089.stdout: ] 2023-06-03T20:30:11.820 INFO:teuthology.orchestra.run.smithi089.stdout: } 2023-06-03T20:30:11.820 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:30:11.820 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:30:11.820 INFO:teuthology.orchestra.run.smithi089.stdout: "description": "osd_repop(client.46739.0:133667 14.1 e291/278 14:806409e2:::benchmark_data_smithi089_237281_object35651:head v 291'8633, mlcod=291'8631)", 2023-06-03T20:30:11.820 INFO:teuthology.orchestra.run.smithi089.stdout: "initiated_at": "2023-06-03T20:30:11.803296+0000", 2023-06-03T20:30:11.820 INFO:teuthology.orchestra.run.smithi089.stdout: "age": 0.0047090370000000001, 2023-06-03T20:30:11.821 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.0048081390000000003, 2023-06-03T20:30:11.821 INFO:teuthology.orchestra.run.smithi089.stdout: "type_data": { 2023-06-03T20:30:11.821 INFO:teuthology.orchestra.run.smithi089.stdout: "flag_point": "started", 2023-06-03T20:30:11.821 INFO:teuthology.orchestra.run.smithi089.stdout: "events": [ 2023-06-03T20:30:11.821 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:30:11.821 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "initiated", 2023-06-03T20:30:11.821 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:30:11.803296+0000", 2023-06-03T20:30:11.822 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0 2023-06-03T20:30:11.822 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:30:11.822 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:30:11.822 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "throttled", 2023-06-03T20:30:11.822 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:30:11.803296+0000", 2023-06-03T20:30:11.822 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0 2023-06-03T20:30:11.822 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:30:11.822 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:30:11.823 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "header_read", 2023-06-03T20:30:11.823 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:30:11.803295+0000", 2023-06-03T20:30:11.823 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 4294967295.999999 2023-06-03T20:30:11.823 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:30:11.823 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:30:11.823 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "all_read", 2023-06-03T20:30:11.823 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:30:11.803303+0000", 2023-06-03T20:30:11.823 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 7.9359999999999999e-06 2023-06-03T20:30:11.824 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:30:11.824 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:30:11.824 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "dispatched", 2023-06-03T20:30:11.824 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:30:11.803320+0000", 2023-06-03T20:30:11.824 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 1.6398000000000001e-05 2023-06-03T20:30:11.824 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:30:11.824 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:30:11.824 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "queued_for_pg", 2023-06-03T20:30:11.825 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:30:11.803327+0000", 2023-06-03T20:30:11.825 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 7.6090000000000003e-06 2023-06-03T20:30:11.825 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:30:11.825 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:30:11.825 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "reached_pg", 2023-06-03T20:30:11.825 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:30:11.803511+0000", 2023-06-03T20:30:11.825 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.000184069 2023-06-03T20:30:11.825 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:30:11.826 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:30:11.826 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "started", 2023-06-03T20:30:11.826 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:30:11.803567+0000", 2023-06-03T20:30:11.826 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 5.5986999999999998e-05 2023-06-03T20:30:11.826 INFO:teuthology.orchestra.run.smithi089.stdout: } 2023-06-03T20:30:11.826 INFO:teuthology.orchestra.run.smithi089.stdout: ] 2023-06-03T20:30:11.826 INFO:teuthology.orchestra.run.smithi089.stdout: } 2023-06-03T20:30:11.826 INFO:teuthology.orchestra.run.smithi089.stdout: } 2023-06-03T20:30:11.827 INFO:teuthology.orchestra.run.smithi089.stdout: ], 2023-06-03T20:30:11.827 INFO:teuthology.orchestra.run.smithi089.stdout: "num_ops": 2 2023-06-03T20:30:11.827 INFO:teuthology.orchestra.run.smithi089.stdout:} 2023-06-03T20:30:11.837 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:30:11.840 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:30:11.840 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:30:11.942 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:30:11.977 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:30:11.983 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:30:11.983 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:30:12.084 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:30:12.123 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:30:12.125 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:30:12.126 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:30:12.227 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:30:12.257 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:30:12 smithi089 bash[78452]: cluster 1685824211.1748555 mgr.y (mgr.34104) 1631 : cluster 0 pgmap v1686: 89 pgs: 1 active+recovering+remapped, 88 active+clean; 4.0 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 588 op/s; 3377/194451 objects misplaced (1.737%); 1.6 MiB/s, 25 objects/s recovering 2023-06-03T20:30:12.270 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:30:12.272 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:30:12.273 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:30:12.374 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:30:12.384 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:30:12 smithi164 bash[59462]: cluster 1685824211.1748555 mgr.y (mgr.34104) 1631 : cluster 0 pgmap v1686: 89 pgs: 1 active+recovering+remapped, 88 active+clean; 4.0 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 588 op/s; 3377/194451 objects misplaced (1.737%); 1.6 MiB/s, 25 objects/s recovering 2023-06-03T20:30:12.404 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:30:12.406 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:30:12.407 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:30:12.507 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:30:12.516 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:30:12 smithi089 bash[75068]: cluster 1685824211.1748555 mgr.y (mgr.34104) 1631 : cluster 0 pgmap v1686: 89 pgs: 1 active+recovering+remapped, 88 active+clean; 4.0 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 588 op/s; 3377/194451 objects misplaced (1.737%); 1.6 MiB/s, 25 objects/s recovering 2023-06-03T20:30:12.544 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:30:12.546 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:30:12.546 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:30:12.647 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:30:12.687 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:30:12.693 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:30:12.694 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:30:12.795 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:30:12.827 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:30:12.830 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:30:12.830 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:30:12.931 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:30:12.964 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:30:12.966 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:30:12.967 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:30:13.067 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:30:13.108 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:30:13.114 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:30:13.114 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:30:13.215 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:30:13.263 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:30:13.266 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:30:13.267 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:30:13.297 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph daemon osd.1 dump_blocked_ops 2023-06-03T20:30:13.369 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:30:13.477 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:30:13.479 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:30:13.479 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:30:13.580 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:30:13.625 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:30:13.636 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:30:13.636 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:30:13.737 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:30:13.754 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/mon.a/config 2023-06-03T20:30:13.766 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:30:13.768 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:30:13.769 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:30:13.860 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph osd unset noscrub 2023-06-03T20:30:13.870 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:30:13.905 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:30:13.908 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:30:13.908 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:30:14.009 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:30:14.021 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-06-03T20:30:14.049 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:30:14.052 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:30:14.053 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:30:14.154 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:30:14.168 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:30:14 smithi089 bash[75068]: cluster 1685824213.1773362 mgr.y (mgr.34104) 1632 : cluster 0 pgmap v1687: 89 pgs: 1 active+recovering+remapped, 88 active+clean; 3.8 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 588 op/s; 3248/185202 objects misplaced (1.754%); 1.5 MiB/s, 24 objects/s recovering 2023-06-03T20:30:14.168 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:30:14 smithi089 bash[78452]: cluster 1685824213.1773362 mgr.y (mgr.34104) 1632 : cluster 0 pgmap v1687: 89 pgs: 1 active+recovering+remapped, 88 active+clean; 3.8 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 588 op/s; 3248/185202 objects misplaced (1.754%); 1.5 MiB/s, 24 objects/s recovering 2023-06-03T20:30:14.191 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:30:14.193 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:30:14.194 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:30:14.295 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:30:14.335 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:30:14 smithi164 bash[59462]: cluster 1685824213.1773362 mgr.y (mgr.34104) 1632 : cluster 0 pgmap v1687: 89 pgs: 1 active+recovering+remapped, 88 active+clean; 3.8 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 588 op/s; 3248/185202 objects misplaced (1.754%); 1.5 MiB/s, 24 objects/s recovering 2023-06-03T20:30:14.339 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:30:14.340 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:30:14.340 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:30:14.441 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:30:14.479 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:30:14.482 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:30:14.482 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:30:14.583 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:30:14.626 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:30:14.631 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:30:14.632 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:30:14.733 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:30:14.764 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:30:14.766 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:30:14.767 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:30:14.868 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:30:14.899 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:30:14.906 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:30:14.906 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:30:15.007 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:30:15.052 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:30:15.055 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:30:15.056 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:30:15.157 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:30:15.194 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:30:15.198 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:30:15.199 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:30:15.300 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:30:15.328 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:30:15.330 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:30:15.330 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:30:15.431 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:30:15.480 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:30:15.481 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:30:15.482 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:30:15.582 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:30:15.618 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:30:15.620 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:30:15.621 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:30:15.721 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:30:15.774 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:30:15.777 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:30:15.778 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:30:15.879 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:30:15.906 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:30:15.908 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:30:15.909 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:30:16.009 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:30:16.037 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:30:16.043 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:30:16.043 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:30:16.144 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:30:16.176 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:30:16.182 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:30:16.183 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:30:16.284 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:30:16.292 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:30:16 smithi164 bash[59462]: cluster 1685824215.1777563 mgr.y (mgr.34104) 1633 : cluster 0 pgmap v1688: 89 pgs: 1 active+recovering+remapped, 88 active+clean; 3.8 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 588 op/s; 3248/185202 objects misplaced (1.754%); 1.5 MiB/s, 24 objects/s recovering 2023-06-03T20:30:16.309 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:30:16.316 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:30:16.317 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:30:16.418 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:30:16.457 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:30:16.462 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:30:16.463 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:30:16.530 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:30:16 smithi089 bash[75068]: cluster 1685824215.1777563 mgr.y (mgr.34104) 1633 : cluster 0 pgmap v1688: 89 pgs: 1 active+recovering+remapped, 88 active+clean; 3.8 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 588 op/s; 3248/185202 objects misplaced (1.754%); 1.5 MiB/s, 24 objects/s recovering 2023-06-03T20:30:16.530 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 20:30:16 smithi089 bash[71163]: ::ffff:172.21.15.164 - - [03/Jun/2023:20:30:16] "GET /metrics HTTP/1.1" 200 229142 "" "Prometheus/2.33.4" 2023-06-03T20:30:16.531 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:30:16 smithi089 bash[78452]: cluster 1685824215.1777563 mgr.y (mgr.34104) 1633 : cluster 0 pgmap v1688: 89 pgs: 1 active+recovering+remapped, 88 active+clean; 3.8 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 588 op/s; 3248/185202 objects misplaced (1.754%); 1.5 MiB/s, 24 objects/s recovering 2023-06-03T20:30:16.565 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:30:16.592 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:30:16.601 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:30:16.602 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:30:16.703 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:30:16.733 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:30:16.735 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:30:16.736 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:30:16.837 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:30:16.867 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:30:16.870 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:30:16.870 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:30:16.971 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:30:16.999 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:30:17.001 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:30:17.001 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:30:17.102 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:30:17.138 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:30:17.143 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:30:17.144 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:30:17.245 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:30:17.290 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:30:17.296 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:30:17.296 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:30:17.397 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:30:17.427 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:30:17.430 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:30:17.431 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:30:17.532 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:30:17.575 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:30:17.580 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:30:17.581 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:30:17.682 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:30:17.738 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:30:17.743 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:30:17.743 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:30:17.844 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:30:17.885 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:30:17.889 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:30:17.889 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:30:17.990 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:30:18.022 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:30:18.024 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:30:18.025 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:30:18.126 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:30:18.162 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:30:18.168 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:30:18.169 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:30:18.215 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/mon.a/config 2023-06-03T20:30:18.216 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/mon.a/config 2023-06-03T20:30:18.220 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/mon.a/config 2023-06-03T20:30:18.244 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:30:18 smithi164 bash[59462]: cluster 1685824217.1782656 mgr.y (mgr.34104) 1634 : cluster 0 pgmap v1689: 89 pgs: 1 active+recovering+remapped, 88 active+clean; 3.7 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 590 op/s; 3248/184050 objects misplaced (1.765%); 1.5 MiB/s, 24 objects/s recovering 2023-06-03T20:30:18.270 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:30:18.311 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:30:18.313 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:30:18.314 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:30:18.414 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:30:18.422 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:30:18 smithi089 bash[78452]: cluster 1685824217.1782656 mgr.y (mgr.34104) 1634 : cluster 0 pgmap v1689: 89 pgs: 1 active+recovering+remapped, 88 active+clean; 3.7 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 590 op/s; 3248/184050 objects misplaced (1.765%); 1.5 MiB/s, 24 objects/s recovering 2023-06-03T20:30:18.423 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:30:18 smithi089 bash[75068]: cluster 1685824217.1782656 mgr.y (mgr.34104) 1634 : cluster 0 pgmap v1689: 89 pgs: 1 active+recovering+remapped, 88 active+clean; 3.7 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 590 op/s; 3248/184050 objects misplaced (1.765%); 1.5 MiB/s, 24 objects/s recovering 2023-06-03T20:30:18.444 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:30:18.447 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:30:18.448 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:30:18.548 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:30:18.587 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:30:18.590 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:30:18.590 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:30:18.691 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:30:18.731 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:30:18.736 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:30:18.741 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:30:18.842 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:30:18.877 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:30:18.880 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:30:18.881 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:30:18.982 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:30:19.006 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:30:19.009 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:30:19.009 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:30:19.110 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:30:19.151 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:30:19.154 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:30:19.155 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:30:19.256 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:30:19.297 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:30:19.300 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:30:19.300 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:30:19.402 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:30:19.440 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:30:19.447 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:30:19.447 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:30:19.548 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:30:19.579 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:30:19.590 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:30:19.591 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:30:19.692 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:30:19.722 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:30:19.725 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:30:19.726 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:30:19.827 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:30:19.876 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:30:19.883 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:30:19.885 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:30:19.985 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:30:20.021 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:30:20.026 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:30:20.027 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:30:20.128 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:30:20.177 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:30:20.182 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:30:20.183 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:30:20.247 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:30:20 smithi164 bash[59462]: cluster 1685824219.1789703 mgr.y (mgr.34104) 1635 : cluster 0 pgmap v1690: 89 pgs: 1 active+recovering+remapped, 88 active+clean; 3.6 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 725 op/s; 3113/175809 objects misplaced (1.771%); 2.0 MiB/s, 31 objects/s recovering 2023-06-03T20:30:20.284 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:30:20.328 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:30:20.330 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:30:20.331 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:30:20.432 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:30:20.441 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:30:20 smithi089 bash[75068]: cluster 1685824219.1789703 mgr.y (mgr.34104) 1635 : cluster 0 pgmap v1690: 89 pgs: 1 active+recovering+remapped, 88 active+clean; 3.6 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 725 op/s; 3113/175809 objects misplaced (1.771%); 2.0 MiB/s, 31 objects/s recovering 2023-06-03T20:30:20.442 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:30:20 smithi089 bash[78452]: cluster 1685824219.1789703 mgr.y (mgr.34104) 1635 : cluster 0 pgmap v1690: 89 pgs: 1 active+recovering+remapped, 88 active+clean; 3.6 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 725 op/s; 3113/175809 objects misplaced (1.771%); 2.0 MiB/s, 31 objects/s recovering 2023-06-03T20:30:20.473 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:30:20.479 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:30:20.479 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:30:20.580 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:30:20.620 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:30:20.625 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:30:20.626 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:30:20.727 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:30:20.766 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:30:20.771 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:30:20.772 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:30:20.873 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:30:20.934 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:30:20.939 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:30:20.940 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:30:21.041 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:30:21.070 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:30:21.075 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:30:21.076 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:30:21.177 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:30:21.206 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:30:21.213 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:30:21.213 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:30:21.314 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:30:21.352 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:30:21.355 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:30:21.355 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:30:21.456 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:30:21.499 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:30:21.502 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:30:21.503 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:30:21.604 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:30:21.644 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:30:21.651 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:30:21.652 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:30:21.753 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:30:21.788 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:30:21.791 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:30:21.791 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:30:21.892 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:30:21.925 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:30:21.927 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:30:21.928 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:30:22.028 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:30:22.078 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:30:22.083 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:30:22.083 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:30:22.184 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:30:22.214 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:30:22.219 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:30:22.219 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:30:22.251 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:30:22 smithi089 bash[75068]: cluster 1685824221.1794267 mgr.y (mgr.34104) 1636 : cluster 0 pgmap v1691: 89 pgs: 1 active+recovering+remapped, 88 active+clean; 3.6 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 517 op/s; 3113/175809 objects misplaced (1.771%); 1.4 MiB/s, 21 objects/s recovering 2023-06-03T20:30:22.251 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:30:22 smithi089 bash[75068]: audit 1685824222.1557128 mon.c (mon.2) 668 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T20:30:22.251 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:30:22 smithi089 bash[78452]: cluster 1685824221.1794267 mgr.y (mgr.34104) 1636 : cluster 0 pgmap v1691: 89 pgs: 1 active+recovering+remapped, 88 active+clean; 3.6 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 517 op/s; 3113/175809 objects misplaced (1.771%); 1.4 MiB/s, 21 objects/s recovering 2023-06-03T20:30:22.251 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:30:22 smithi089 bash[78452]: audit 1685824222.1557128 mon.c (mon.2) 668 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T20:30:22.320 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:30:22.348 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:30:22.352 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:30:22.353 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:30:22.453 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:30:22.484 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:30:22.489 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:30:22.490 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:30:22.585 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:30:22 smithi164 bash[59462]: cluster 1685824221.1794267 mgr.y (mgr.34104) 1636 : cluster 0 pgmap v1691: 89 pgs: 1 active+recovering+remapped, 88 active+clean; 3.6 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 517 op/s; 3113/175809 objects misplaced (1.771%); 1.4 MiB/s, 21 objects/s recovering 2023-06-03T20:30:22.585 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:30:22 smithi164 bash[59462]: audit 1685824222.1557128 mon.c (mon.2) 668 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T20:30:22.591 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:30:22.617 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:30:22.623 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:30:22.624 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:30:22.652 INFO:teuthology.orchestra.run.smithi089.stdout: 2023-06-03T20:30:22.652 INFO:teuthology.orchestra.run.smithi089.stdout:{"fsid":"f1db3478-0243-11ee-9b1d-001a4aab830c","health":{"status":"HEALTH_WARN","checks":{"OSDMAP_FLAGS":{"severity":"HEALTH_WARN","summary":{"message":"noscrub,nodeep-scrub flag(s) set","count":20},"muted":false}},"mutes":[]},"election_epoch":36,"quorum":[0,1,2],"quorum_names":["a","b","c"],"quorum_age":2727,"monmap":{"epoch":4,"min_mon_release_name":"reef","num_mons":3},"osdmap":{"epoch":291,"num_osds":8,"num_up_osds":8,"osd_up_since":1685823806,"num_in_osds":8,"osd_in_since":1685823841,"num_remapped_pgs":1},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":88},{"state_name":"active+recovering+remapped","count":1}],"num_pgs":89,"num_pools":5,"num_objects":58603,"data_bytes":3839563686,"bytes_used":17002209280,"bytes_avail":750958075904,"bytes_total":767960285184,"misplaced_objects":3113,"misplaced_total":175809,"misplaced_ratio":0.017706715810908429,"recovering_objects_per_sec":21,"recovering_bytes_per_sec":1441162,"recovering_keys_per_sec":0,"num_objects_recovered":264,"num_bytes_recovered":17301504,"num_keys_recovered":0,"write_bytes_sec":0,"read_op_per_sec":0,"write_op_per_sec":517},"fsmap":{"epoch":7,"id":1,"up":1,"in":1,"max":1,"by_rank":[{"filesystem_id":1,"rank":0,"name":"foo.smithi089.giuxuj","status":"up:active","gid":14646}],"up:standby":1},"mgrmap":{"available":true,"num_standbys":1,"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"services":{"dashboard":"https://172.21.15.89:8443/","prometheus":"http://172.21.15.89:9283/"}},"servicemap":{"epoch":39,"modified":"2023-06-03T20:27:53.125660+0000","services":{}},"progress_events":{"16787ee9-eb5a-44b2-97e1-37974eb2bde6":{"message":"Global Recovery Event (0s)\n [............................] ","progress":0,"add_to_ceph_s":true},"2c7b9f96-2368-4bf9-83ac-8c6c53d71110":{"message":"Upgrade to 18.0.0-3532-g8e24bcf4 (14m)\n [============................] (remaining: 17m)","progress":0.44999998807907104,"add_to_ceph_s":true}}} 2023-06-03T20:30:22.726 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:30:22.775 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:30:22.776 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:30:22.776 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:30:22.877 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:30:22.917 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:30:22.924 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:30:22.925 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:30:23.026 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:30:23.061 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:30:23.062 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:30:23.063 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:30:23.163 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:30:23.229 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:30:23.231 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:30:23.232 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:30:23.333 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:30:23.336 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:30:23 smithi164 bash[59462]: audit 1685824222.6434827 mon.b (mon.1) 65 : audit 0 from='client.? 172.21.15.89:0/2205768363' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-06-03T20:30:23.370 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:30:23.372 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:30:23.373 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:30:23.473 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:30:23.482 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:30:23 smithi089 bash[75068]: audit 1685824222.6434827 mon.b (mon.1) 65 : audit 0 from='client.? 172.21.15.89:0/2205768363' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-06-03T20:30:23.482 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:30:23 smithi089 bash[78452]: audit 1685824222.6434827 mon.b (mon.1) 65 : audit 0 from='client.? 172.21.15.89:0/2205768363' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-06-03T20:30:23.502 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:30:23.508 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:30:23.509 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:30:23.609 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:30:23.647 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:30:23.654 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:30:23.655 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:30:23.755 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:30:23.793 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:30:23.795 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:30:23.796 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:30:23.896 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:30:23.929 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:30:23.933 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:30:23.934 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:30:24.034 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:30:24.074 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:30:24.079 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:30:24.080 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:30:24.181 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:30:24.224 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:30:24.231 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:30:24.231 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:30:24.252 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:30:24 smithi089 bash[75068]: cluster 1685824223.1800656 mgr.y (mgr.34104) 1637 : cluster 0 pgmap v1692: 89 pgs: 1 active+recovering+remapped, 88 active+clean; 3.5 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 672 op/s; 2975/170244 objects misplaced (1.747%); 2.1 MiB/s, 33 objects/s recovering 2023-06-03T20:30:24.332 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:30:24.377 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:30:24.379 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:30:24.380 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:30:24.481 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:30:24.521 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:30:24.527 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:30:24 smithi089 bash[78452]: cluster 1685824223.1800656 mgr.y (mgr.34104) 1637 : cluster 0 pgmap v1692: 89 pgs: 1 active+recovering+remapped, 88 active+clean; 3.5 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 672 op/s; 2975/170244 objects misplaced (1.747%); 2.1 MiB/s, 33 objects/s recovering 2023-06-03T20:30:24.527 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:30:24.528 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:30:24.585 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:30:24 smithi164 bash[59462]: cluster 1685824223.1800656 mgr.y (mgr.34104) 1637 : cluster 0 pgmap v1692: 89 pgs: 1 active+recovering+remapped, 88 active+clean; 3.5 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 672 op/s; 2975/170244 objects misplaced (1.747%); 2.1 MiB/s, 33 objects/s recovering 2023-06-03T20:30:24.629 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:30:24.657 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:30:24.659 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:30:24.659 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:30:24.760 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:30:24.807 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:30:24.811 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:30:24.811 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:30:24.912 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:30:24.944 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:30:24.945 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:30:24.946 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:30:25.047 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:30:25.078 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:30:25.084 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:30:25.085 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:30:25.186 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:30:25.217 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:30:25.224 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:30:25.224 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:30:25.325 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:30:25.355 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:30:25.357 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:30:25.358 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:30:25.459 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:30:25.498 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:30:25.503 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:30:25.504 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:30:25.605 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:30:25.634 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:30:25.636 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:30:25.636 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:30:25.737 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:30:25.769 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:30:25.775 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:30:25.776 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:30:25.877 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:30:25.919 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:30:25.923 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:30:25.924 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:30:26.025 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:30:26.081 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:30:26.087 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:30:26.088 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:30:26.189 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:30:26.232 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:30:26.237 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:30:26.237 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:30:26.338 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:30:26.372 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:30:26.379 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:30:26.380 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:30:26.481 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:30:26.521 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:30:26.527 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:30:26 smithi164 bash[59462]: cluster 1685824225.1805117 mgr.y (mgr.34104) 1638 : cluster 0 pgmap v1693: 89 pgs: 1 active+recovering+remapped, 88 active+clean; 3.4 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 508 op/s; 2975/166893 objects misplaced (1.783%); 1.4 MiB/s, 22 objects/s recovering 2023-06-03T20:30:26.528 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:30:26.528 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:30:26.529 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:30:26 smithi089 bash[75068]: cluster 1685824225.1805117 mgr.y (mgr.34104) 1638 : cluster 0 pgmap v1693: 89 pgs: 1 active+recovering+remapped, 88 active+clean; 3.4 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 508 op/s; 2975/166893 objects misplaced (1.783%); 1.4 MiB/s, 22 objects/s recovering 2023-06-03T20:30:26.529 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 20:30:26 smithi089 bash[71163]: ::ffff:172.21.15.164 - - [03/Jun/2023:20:30:26] "GET /metrics HTTP/1.1" 200 229167 "" "Prometheus/2.33.4" 2023-06-03T20:30:26.530 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:30:26 smithi089 bash[78452]: cluster 1685824225.1805117 mgr.y (mgr.34104) 1638 : cluster 0 pgmap v1693: 89 pgs: 1 active+recovering+remapped, 88 active+clean; 3.4 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 508 op/s; 2975/166893 objects misplaced (1.783%); 1.4 MiB/s, 22 objects/s recovering 2023-06-03T20:30:26.629 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:30:26.673 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:30:26.676 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:30:26.677 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:30:26.778 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:30:26.813 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:30:26.818 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:30:26.819 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:30:26.919 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:30:26.953 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:30:26.955 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:30:26.955 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:30:27.056 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:30:27.093 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:30:27.097 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:30:27.098 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:30:27.199 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:30:27.227 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:30:27.231 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:30:27.232 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:30:27.332 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:30:27.380 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:30:27.383 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:30:27.383 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:30:27.484 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:30:27.521 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:30:27.523 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:30:27.523 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:30:27.624 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:30:27.656 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:30:27.659 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:30:27.660 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:30:27.760 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:30:27.794 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:30:27.797 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:30:27.797 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:30:27.898 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:30:27.931 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:30:27.939 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:30:27.940 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:30:28.041 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:30:28.091 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:30:28.095 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:30:28.095 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:30:28.171 INFO:tasks.cephadm.ceph_manager.ceph:making progress, resetting timeout 2023-06-03T20:30:28.171 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph pg dump --format=json 2023-06-03T20:30:28.196 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:30:28.231 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:30:28.233 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:30:28.234 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:30:28.335 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:30:28.336 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:30:28 smithi164 bash[59462]: cluster 1685824227.1810327 mgr.y (mgr.34104) 1639 : cluster 0 pgmap v1694: 89 pgs: 1 active+recovering+remapped, 88 active+clean; 3.4 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 542 op/s; 2975/165675 objects misplaced (1.796%); 1.4 MiB/s, 22 objects/s recovering 2023-06-03T20:30:28.361 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:30:28.365 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:30:28.365 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:30:28.466 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:30:28.505 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:30:28.507 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:30:28.508 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:30:28.529 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:30:28 smithi089 bash[75068]: cluster 1685824227.1810327 mgr.y (mgr.34104) 1639 : cluster 0 pgmap v1694: 89 pgs: 1 active+recovering+remapped, 88 active+clean; 3.4 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 542 op/s; 2975/165675 objects misplaced (1.796%); 1.4 MiB/s, 22 objects/s recovering 2023-06-03T20:30:28.529 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:30:28 smithi089 bash[78452]: cluster 1685824227.1810327 mgr.y (mgr.34104) 1639 : cluster 0 pgmap v1694: 89 pgs: 1 active+recovering+remapped, 88 active+clean; 3.4 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 542 op/s; 2975/165675 objects misplaced (1.796%); 1.4 MiB/s, 22 objects/s recovering 2023-06-03T20:30:28.609 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:30:28.654 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:30:28.661 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:30:28.662 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:30:28.763 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:30:28.797 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:30:28.804 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:30:28.805 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:30:28.906 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:30:28.934 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:30:28.939 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:30:28.939 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:30:29.040 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:30:29.079 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:30:29.085 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:30:29.086 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:30:29.187 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:30:29.219 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:30:29.221 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:30:29.222 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:30:29.323 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:30:29.371 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:30:29.375 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:30:29.376 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:30:29.477 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:30:29.512 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:30:29.515 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:30:29.515 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:30:29.616 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:30:29.654 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:30:29.659 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:30:29.660 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:30:29.761 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:30:29.794 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:30:29.798 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:30:29.798 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:30:29.899 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:30:29.942 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:30:29.947 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:30:29.948 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:30:30.049 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:30:30.084 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:30:30.091 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:30:30.092 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:30:30.193 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:30:30.226 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:30:30.231 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:30:30.231 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:30:30.332 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:30:30.335 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:30:30 smithi164 bash[59462]: cluster 1685824229.1817112 mgr.y (mgr.34104) 1640 : cluster 0 pgmap v1695: 89 pgs: 1 active+recovering+remapped, 88 active+clean; 3.2 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 734 op/s; 2837/157605 objects misplaced (1.800%); 2.1 MiB/s, 34 objects/s recovering 2023-06-03T20:30:30.367 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:30:30.371 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:30:30.372 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:30:30.472 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:30:30.505 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:30:30.509 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:30:30.510 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:30:30.529 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:30:30 smithi089 bash[75068]: cluster 1685824229.1817112 mgr.y (mgr.34104) 1640 : cluster 0 pgmap v1695: 89 pgs: 1 active+recovering+remapped, 88 active+clean; 3.2 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 734 op/s; 2837/157605 objects misplaced (1.800%); 2.1 MiB/s, 34 objects/s recovering 2023-06-03T20:30:30.529 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:30:30 smithi089 bash[78452]: cluster 1685824229.1817112 mgr.y (mgr.34104) 1640 : cluster 0 pgmap v1695: 89 pgs: 1 active+recovering+remapped, 88 active+clean; 3.2 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 734 op/s; 2837/157605 objects misplaced (1.800%); 2.1 MiB/s, 34 objects/s recovering 2023-06-03T20:30:30.611 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:30:30.646 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:30:30.650 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:30:30.650 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:30:30.751 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:30:30.791 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:30:30.793 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:30:30.794 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:30:30.895 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:30:30.931 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:30:30.935 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:30:30.935 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:30:31.036 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:30:31.074 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:30:31.078 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:30:31.079 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:30:31.180 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:30:31.210 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:30:31.216 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:30:31.217 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:30:31.317 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:30:31.357 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:30:31.365 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:30:31.366 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:30:31.467 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:30:31.505 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:30:31.509 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:30:31.510 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:30:31.611 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:30:31.647 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:30:31.650 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:30:31.651 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:30:31.752 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:30:31.782 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:30:31.788 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:30:31.789 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:30:31.889 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:30:31.943 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:30:31.946 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:30:31.947 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:30:32.048 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:30:32.089 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:30:32.093 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:30:32.093 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:30:32.194 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:30:32.223 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:30:32.225 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:30:32.226 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:30:32.277 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:30:32 smithi164 bash[59462]: cluster 1685824231.1820774 mgr.y (mgr.34104) 1641 : cluster 0 pgmap v1696: 89 pgs: 1 active+recovering+remapped, 88 active+clean; 3.2 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 505 op/s; 2837/157605 objects misplaced (1.800%); 1.4 MiB/s, 22 objects/s recovering 2023-06-03T20:30:32.326 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:30:32.335 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:30:32 smithi089 bash[75068]: cluster 1685824231.1820774 mgr.y (mgr.34104) 1641 : cluster 0 pgmap v1696: 89 pgs: 1 active+recovering+remapped, 88 active+clean; 3.2 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 505 op/s; 2837/157605 objects misplaced (1.800%); 1.4 MiB/s, 22 objects/s recovering 2023-06-03T20:30:32.336 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:30:32 smithi089 bash[78452]: cluster 1685824231.1820774 mgr.y (mgr.34104) 1641 : cluster 0 pgmap v1696: 89 pgs: 1 active+recovering+remapped, 88 active+clean; 3.2 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 505 op/s; 2837/157605 objects misplaced (1.800%); 1.4 MiB/s, 22 objects/s recovering 2023-06-03T20:30:32.364 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:30:32.371 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:30:32.372 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:30:32.473 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:30:32.502 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:30:32.506 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:30:32.507 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:30:32.608 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:30:32.645 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:30:32.654 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:30:32.654 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:30:32.755 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:30:32.803 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:30:32.807 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:30:32.808 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:30:32.909 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:30:32.955 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:30:32.956 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:30:32.957 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:30:33.043 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/mon.a/config 2023-06-03T20:30:33.058 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:30:33.098 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:30:33.100 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:30:33.101 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:30:33.202 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:30:33.234 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:30:33.237 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:30:33.237 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:30:33.338 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:30:33.386 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:30:33.391 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:30:33.392 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:30:33.492 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:30:33.536 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:30:33.540 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:30:33.541 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:30:33.641 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:30:33.676 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:30:33.681 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:30:33.682 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:30:33.783 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:30:33.825 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:30:33.828 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:30:33.829 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:30:33.929 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:30:33.979 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:30:33.980 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:30:33.980 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:30:34.081 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:30:34.089 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:30:34 smithi089 bash[75068]: cluster 1685824233.1827672 mgr.y (mgr.34104) 1642 : cluster 0 pgmap v1697: 89 pgs: 1 active+recovering+remapped, 88 active+clean; 3.1 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 653 op/s; 2698/152283 objects misplaced (1.772%); 2.2 MiB/s, 34 objects/s recovering 2023-06-03T20:30:34.122 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:30:34.128 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:30:34.129 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:30:34.230 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:30:34.279 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:30:34.283 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:30:34.284 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:30:34.385 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:30:34.393 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:30:34 smithi164 bash[59462]: cluster 1685824233.1827672 mgr.y (mgr.34104) 1642 : cluster 0 pgmap v1697: 89 pgs: 1 active+recovering+remapped, 88 active+clean; 3.1 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 653 op/s; 2698/152283 objects misplaced (1.772%); 2.2 MiB/s, 34 objects/s recovering 2023-06-03T20:30:34.416 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:30:34.419 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:30:34.420 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:30:34.521 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:30:34.529 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:30:34 smithi089 bash[78452]: cluster 1685824233.1827672 mgr.y (mgr.34104) 1642 : cluster 0 pgmap v1697: 89 pgs: 1 active+recovering+remapped, 88 active+clean; 3.1 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 653 op/s; 2698/152283 objects misplaced (1.772%); 2.2 MiB/s, 34 objects/s recovering 2023-06-03T20:30:34.556 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:30:34.561 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:30:34.562 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:30:34.662 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:30:34.696 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:30:34.699 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:30:34.700 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:30:34.801 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:30:34.838 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:30:34.839 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:30:34.839 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:30:34.940 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:30:34.976 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:30:34.979 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:30:34.980 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:30:35.081 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:30:35.120 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:30:35.125 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:30:35.126 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:30:35.227 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:30:35.261 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:30:35.264 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:30:35.265 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:30:35.366 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:30:35.404 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:30:35.410 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:30:35.411 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:30:35.511 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:30:35.545 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:30:35.553 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:30:35.554 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:30:35.655 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:30:35.695 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:30:35.699 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:30:35.700 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:30:35.801 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:30:35.831 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:30:35.839 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:30:35.840 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:30:35.941 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:30:35.976 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:30:35.983 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:30:35.984 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:30:36.085 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:30:36.113 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:30:36.115 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:30:36.116 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:30:36.217 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:30:36.233 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 20:30:36 smithi089 bash[71163]: ::ffff:172.21.15.164 - - [03/Jun/2023:20:30:36] "GET /metrics HTTP/1.1" 200 229167 "" "Prometheus/2.33.4" 2023-06-03T20:30:36.247 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:30:36.249 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:30:36.251 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:30:36.352 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:30:36.390 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:30:36.394 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:30:36.395 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:30:36.496 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:30:36.506 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:30:36 smithi164 bash[59462]: cluster 1685824235.1833022 mgr.y (mgr.34104) 1643 : cluster 0 pgmap v1698: 89 pgs: 1 active+recovering+remapped, 88 active+clean; 3.0 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 590 op/s; 2698/148974 objects misplaced (1.811%); 1.4 MiB/s, 23 objects/s recovering 2023-06-03T20:30:36.529 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:30:36 smithi089 bash[75068]: cluster 1685824235.1833022 mgr.y (mgr.34104) 1643 : cluster 0 pgmap v1698: 89 pgs: 1 active+recovering+remapped, 88 active+clean; 3.0 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 590 op/s; 2698/148974 objects misplaced (1.811%); 1.4 MiB/s, 23 objects/s recovering 2023-06-03T20:30:36.530 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:30:36 smithi089 bash[78452]: cluster 1685824235.1833022 mgr.y (mgr.34104) 1643 : cluster 0 pgmap v1698: 89 pgs: 1 active+recovering+remapped, 88 active+clean; 3.0 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 590 op/s; 2698/148974 objects misplaced (1.811%); 1.4 MiB/s, 23 objects/s recovering 2023-06-03T20:30:36.536 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:30:36.542 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:30:36.543 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:30:36.644 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:30:36.675 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:30:36.681 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:30:36.682 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:30:36.783 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:30:36.823 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:30:36.831 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:30:36.832 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:30:36.933 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:30:36.970 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:30:36.972 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:30:36.973 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:30:36.982 INFO:teuthology.orchestra.run.smithi089.stdout:osd.0: {} 2023-06-03T20:30:37.027 INFO:teuthology.orchestra.run.smithi089.stdout:osd.1: {} 2023-06-03T20:30:37.065 INFO:teuthology.orchestra.run.smithi089.stdout:osd.2: {} 2023-06-03T20:30:37.074 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:30:37.107 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:30:37.109 INFO:teuthology.orchestra.run.smithi089.stdout:osd.3: {} 2023-06-03T20:30:37.110 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:30:37.110 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:30:37.153 INFO:teuthology.orchestra.run.smithi089.stdout:osd.4: {} 2023-06-03T20:30:37.197 INFO:teuthology.orchestra.run.smithi089.stdout:osd.5: {} 2023-06-03T20:30:37.211 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:30:37.244 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:30:37.248 INFO:teuthology.orchestra.run.smithi089.stdout:osd.6: {} 2023-06-03T20:30:37.249 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:30:37.250 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:30:37.293 INFO:teuthology.orchestra.run.smithi089.stdout:osd.7: {} 2023-06-03T20:30:37.296 INFO:teuthology.orchestra.run.smithi089.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-06-03T20:30:37.296 INFO:teuthology.orchestra.run.smithi089.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-06-03T20:30:37.296 INFO:teuthology.orchestra.run.smithi089.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-06-03T20:30:37.296 INFO:teuthology.orchestra.run.smithi089.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-06-03T20:30:37.296 INFO:teuthology.orchestra.run.smithi089.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-06-03T20:30:37.297 INFO:teuthology.orchestra.run.smithi089.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-06-03T20:30:37.297 INFO:teuthology.orchestra.run.smithi089.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-06-03T20:30:37.297 INFO:teuthology.orchestra.run.smithi089.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-06-03T20:30:37.351 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:30:37.384 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:30:37.387 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:30:37.387 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:30:37.481 INFO:teuthology.orchestra.run.smithi089.stdout:ERROR: (22) Invalid argument 2023-06-03T20:30:37.481 INFO:teuthology.orchestra.run.smithi089.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-06-03T20:30:37.488 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:30:37.495 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:30:37 smithi164 bash[59462]: audit 1685824236.9259014 mon.a (mon.0) 1635 : audit 0 from='client.? 172.21.15.89:0/1022040635' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-03T20:30:37.496 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:30:37 smithi164 bash[59462]: audit 1685824237.1561403 mon.c (mon.2) 669 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T20:30:37.520 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:30:37 smithi089 bash[78452]: audit 1685824236.9259014 mon.a (mon.0) 1635 : audit 0 from='client.? 172.21.15.89:0/1022040635' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-03T20:30:37.521 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:30:37 smithi089 bash[78452]: audit 1685824237.1561403 mon.c (mon.2) 669 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T20:30:37.522 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:30:37 smithi089 bash[75068]: audit 1685824236.9259014 mon.a (mon.0) 1635 : audit 0 from='client.? 172.21.15.89:0/1022040635' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-03T20:30:37.522 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:30:37 smithi089 bash[75068]: audit 1685824237.1561403 mon.c (mon.2) 669 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T20:30:37.524 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:30:37.527 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:30:37.528 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:30:37.629 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:30:37.658 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:30:37.662 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:30:37.662 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:30:37.763 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:30:37.804 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:30:37.811 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:30:37.811 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:30:37.912 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:30:37.943 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:30:37.945 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:30:37.945 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:30:38.046 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:30:38.091 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:30:38.095 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:30:38.095 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:30:38.196 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:30:38.234 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:30:38.238 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:30:38.239 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:30:38.294 INFO:teuthology.orchestra.run.smithi089.stderr:noscrub is unset 2023-06-03T20:30:38.340 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:30:38.349 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:30:38 smithi089 bash[75068]: cluster 1685824237.1838238 mgr.y (mgr.34104) 1644 : cluster 0 pgmap v1699: 89 pgs: 1 active+recovering+remapped, 88 active+clean; 3.0 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 528 op/s; 2698/147861 objects misplaced (1.825%); 1.4 MiB/s, 23 objects/s recovering 2023-06-03T20:30:38.349 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:30:38 smithi089 bash[75068]: audit 1685824237.8225708 mon.a (mon.0) 1636 : audit 1 from='client.? 172.21.15.89:0/2341624450' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-06-03T20:30:38.350 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:30:38 smithi089 bash[78452]: cluster 1685824237.1838238 mgr.y (mgr.34104) 1644 : cluster 0 pgmap v1699: 89 pgs: 1 active+recovering+remapped, 88 active+clean; 3.0 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 528 op/s; 2698/147861 objects misplaced (1.825%); 1.4 MiB/s, 23 objects/s recovering 2023-06-03T20:30:38.350 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:30:38 smithi089 bash[78452]: audit 1685824237.8225708 mon.a (mon.0) 1636 : audit 1 from='client.? 172.21.15.89:0/2341624450' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-06-03T20:30:38.376 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:30:38.378 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:30:38.379 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:30:38.480 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:30:38.509 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:30:38.516 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:30:38.517 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:30:38.585 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:30:38 smithi164 bash[59462]: cluster 1685824237.1838238 mgr.y (mgr.34104) 1644 : cluster 0 pgmap v1699: 89 pgs: 1 active+recovering+remapped, 88 active+clean; 3.0 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 528 op/s; 2698/147861 objects misplaced (1.825%); 1.4 MiB/s, 23 objects/s recovering 2023-06-03T20:30:38.585 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:30:38 smithi164 bash[59462]: audit 1685824237.8225708 mon.a (mon.0) 1636 : audit 1 from='client.? 172.21.15.89:0/2341624450' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-06-03T20:30:38.618 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:30:38.647 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:30:38.648 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:30:38.649 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:30:38.750 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:30:38.781 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:30:38.786 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:30:38.787 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:30:38.888 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:30:38.917 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:30:38.919 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:30:38.920 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:30:39.020 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:30:39.069 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:30:39.075 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:30:39.076 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:30:39.176 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:30:39.220 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:30:39.224 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:30:39.224 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:30:39.325 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:30:39.334 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:30:39 smithi089 bash[78452]: cluster 1685824238.2668142 mon.a (mon.0) 1637 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-06-03T20:30:39.335 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:30:39 smithi089 bash[78452]: cluster 1685824238.2759373 osd.6 (osd.6) 227 : cluster 3 failed to encode map e292 with expected crc 2023-06-03T20:30:39.335 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:30:39 smithi089 bash[78452]: cluster 1685824238.2781012 osd.5 (osd.5) 241 : cluster 3 failed to encode map e292 with expected crc 2023-06-03T20:30:39.335 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:30:39 smithi089 bash[78452]: audit 1685824238.281607 mon.a (mon.0) 1638 : audit 1 from='client.? 172.21.15.89:0/2341624450' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-06-03T20:30:39.335 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:30:39 smithi089 bash[78452]: cluster 1685824238.2816682 mon.a (mon.0) 1639 : cluster 0 osdmap e292: 8 total, 8 up, 8 in 2023-06-03T20:30:39.335 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:30:39 smithi089 bash[75068]: cluster 1685824238.2668142 mon.a (mon.0) 1637 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-06-03T20:30:39.336 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:30:39 smithi089 bash[75068]: cluster 1685824238.2759373 osd.6 (osd.6) 227 : cluster 3 failed to encode map e292 with expected crc 2023-06-03T20:30:39.336 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:30:39 smithi089 bash[75068]: cluster 1685824238.2781012 osd.5 (osd.5) 241 : cluster 3 failed to encode map e292 with expected crc 2023-06-03T20:30:39.336 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:30:39 smithi089 bash[75068]: audit 1685824238.281607 mon.a (mon.0) 1638 : audit 1 from='client.? 172.21.15.89:0/2341624450' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-06-03T20:30:39.336 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:30:39 smithi089 bash[75068]: cluster 1685824238.2816682 mon.a (mon.0) 1639 : cluster 0 osdmap e292: 8 total, 8 up, 8 in 2023-06-03T20:30:39.367 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:30:39.370 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:30:39.371 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:30:39.472 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:30:39.498 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:30:39.506 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:30:39.507 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:30:39.585 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:30:39 smithi164 bash[59462]: cluster 1685824238.2668142 mon.a (mon.0) 1637 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-06-03T20:30:39.585 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:30:39 smithi164 bash[59462]: cluster 1685824238.2759373 osd.6 (osd.6) 227 : cluster 3 failed to encode map e292 with expected crc 2023-06-03T20:30:39.586 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:30:39 smithi164 bash[59462]: cluster 1685824238.2781012 osd.5 (osd.5) 241 : cluster 3 failed to encode map e292 with expected crc 2023-06-03T20:30:39.586 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:30:39 smithi164 bash[59462]: audit 1685824238.281607 mon.a (mon.0) 1638 : audit 1 from='client.? 172.21.15.89:0/2341624450' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-06-03T20:30:39.586 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:30:39 smithi164 bash[59462]: cluster 1685824238.2816682 mon.a (mon.0) 1639 : cluster 0 osdmap e292: 8 total, 8 up, 8 in 2023-06-03T20:30:39.607 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:30:39.644 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:30:39.651 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:30:39.652 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:30:39.753 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:30:39.793 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:30:39.797 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:30:39.798 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:30:39.898 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:30:39.944 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:30:39.950 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:30:39.951 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:30:40.051 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:30:40.093 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:30:40.099 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:30:40.100 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:30:40.200 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:30:40.237 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:30:40.243 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:30:40.244 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:30:40.345 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:30:40.382 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:30:40.387 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:30:40.388 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:30:40.489 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:30:40.521 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:30:40.529 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:30:40.530 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:30:40.585 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:30:40 smithi164 bash[59462]: cluster 1685824239.1844115 mgr.y (mgr.34104) 1645 : cluster 0 pgmap v1701: 89 pgs: 1 active+recovering+remapped, 88 active+clean; 2.9 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 581 op/s; 2550/140163 objects misplaced (1.819%); 1.8 MiB/s, 28 objects/s recovering 2023-06-03T20:30:40.631 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:30:40.661 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:30:40.663 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:30:40.664 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:30:40.764 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:30:40.773 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:30:40 smithi089 bash[78452]: cluster 1685824239.1844115 mgr.y (mgr.34104) 1645 : cluster 0 pgmap v1701: 89 pgs: 1 active+recovering+remapped, 88 active+clean; 2.9 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 581 op/s; 2550/140163 objects misplaced (1.819%); 1.8 MiB/s, 28 objects/s recovering 2023-06-03T20:30:40.773 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:30:40 smithi089 bash[75068]: cluster 1685824239.1844115 mgr.y (mgr.34104) 1645 : cluster 0 pgmap v1701: 89 pgs: 1 active+recovering+remapped, 88 active+clean; 2.9 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 581 op/s; 2550/140163 objects misplaced (1.819%); 1.8 MiB/s, 28 objects/s recovering 2023-06-03T20:30:40.798 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:30:40.804 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:30:40.805 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:30:40.905 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:30:40.934 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:30:40.936 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:30:40.936 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:30:41.037 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:30:41.071 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:30:41.076 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:30:41.077 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:30:41.178 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:30:41.216 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:30:41.219 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:30:41.220 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:30:41.320 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:30:41.353 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:30:41.360 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:30:41.361 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:30:41.461 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:30:41.497 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:30:41.502 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:30:41.503 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:30:41.604 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:30:41.629 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:30:41.631 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:30:41.631 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:30:41.732 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:30:41.785 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:30:41.792 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:30:41.793 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:30:41.894 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:30:41.928 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:30:41.934 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:30:41.935 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:30:42.036 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:30:42.065 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:30:42.071 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:30:42.071 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:30:42.172 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:30:42.209 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:30:42.211 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:30:42.211 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:30:42.312 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:30:42.322 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:30:42 smithi164 bash[59462]: cluster 1685824241.184856 mgr.y (mgr.34104) 1646 : cluster 0 pgmap v1702: 89 pgs: 1 active+recovering+remapped, 88 active+clean; 2.9 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 581 op/s; 2550/140163 objects misplaced (1.819%); 1.8 MiB/s, 28 objects/s recovering 2023-06-03T20:30:42.342 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:30:42.347 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:30:42.348 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:30:42.389 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph daemon osd.1 dump_historic_ops 2023-06-03T20:30:42.449 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:30:42.488 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:30:42 smithi089 bash[75068]: cluster 1685824241.184856 mgr.y (mgr.34104) 1646 : cluster 0 pgmap v1702: 89 pgs: 1 active+recovering+remapped, 88 active+clean; 2.9 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 581 op/s; 2550/140163 objects misplaced (1.819%); 1.8 MiB/s, 28 objects/s recovering 2023-06-03T20:30:42.489 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:30:42 smithi089 bash[78452]: cluster 1685824241.184856 mgr.y (mgr.34104) 1646 : cluster 0 pgmap v1702: 89 pgs: 1 active+recovering+remapped, 88 active+clean; 2.9 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 581 op/s; 2550/140163 objects misplaced (1.819%); 1.8 MiB/s, 28 objects/s recovering 2023-06-03T20:30:42.497 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:30:42.505 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:30:42.505 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:30:42.606 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:30:42.633 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:30:42.637 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:30:42.638 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:30:42.739 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:30:42.767 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:30:42.771 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:30:42.772 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:30:42.873 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:30:42.901 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:30:42.908 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:30:42.908 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:30:43.009 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:30:43.049 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:30:43.055 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:30:43.055 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:30:43.156 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:30:43.191 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:30:43.201 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:30:43.201 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:30:43.302 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:30:43.350 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:30:43.357 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:30:43.357 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:30:43.458 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:30:43.503 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:30:43.508 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:30:43.509 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:30:43.610 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:30:43.656 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:30:43.658 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:30:43.658 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:30:43.760 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:30:43.806 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:30:43.808 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:30:43.809 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:30:43.910 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:30:43.938 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:30:43.943 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:30:43.944 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:30:44.045 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:30:44.077 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:30:44.080 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:30:44.080 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:30:44.181 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:30:44.193 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:30:44 smithi164 bash[59462]: cluster 1685824243.1855657 mgr.y (mgr.34104) 1647 : cluster 0 pgmap v1703: 89 pgs: 1 active+recovering+remapped, 88 active+clean; 2.7 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 593 op/s; 2415/134466 objects misplaced (1.796%); 1.8 MiB/s, 28 objects/s recovering 2023-06-03T20:30:44.212 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:30:44.214 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:30:44.215 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:30:44.233 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-06-03T20:30:44.279 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:30:44 smithi089 bash[75068]: cluster 1685824243.1855657 mgr.y (mgr.34104) 1647 : cluster 0 pgmap v1703: 89 pgs: 1 active+recovering+remapped, 88 active+clean; 2.7 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 593 op/s; 2415/134466 objects misplaced (1.796%); 1.8 MiB/s, 28 objects/s recovering 2023-06-03T20:30:44.280 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:30:44 smithi089 bash[78452]: cluster 1685824243.1855657 mgr.y (mgr.34104) 1647 : cluster 0 pgmap v1703: 89 pgs: 1 active+recovering+remapped, 88 active+clean; 2.7 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 593 op/s; 2415/134466 objects misplaced (1.796%); 1.8 MiB/s, 28 objects/s recovering 2023-06-03T20:30:44.316 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:30:44.349 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:30:44.355 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:30:44.356 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:30:44.457 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:30:44.490 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph osd unset nodeep-scrub 2023-06-03T20:30:44.493 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:30:44.499 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:30:44.499 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:30:44.600 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:30:44.653 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:30:44.660 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:30:44.660 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:30:44.761 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:30:44.812 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:30:44.817 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:30:44.817 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:30:44.918 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:30:44.954 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:30:44.958 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:30:44.959 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:30:45.060 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:30:45.102 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:30:45.104 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:30:45.105 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:30:45.206 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:30:45.234 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:30:45.237 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:30:45.238 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:30:45.338 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:30:45.383 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:30:45.388 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:30:45.388 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:30:45.434 INFO:teuthology.orchestra.run.smithi089.stdout: 2023-06-03T20:30:45.451 INFO:teuthology.orchestra.run.smithi089.stderr:dumped all 2023-06-03T20:30:45.489 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:30:45.533 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:30:45.539 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:30:45.540 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:30:45.641 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:30:45.676 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:30:45.678 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:30:45.679 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:30:45.780 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:30:45.815 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:30:45.821 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:30:45.821 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:30:45.922 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:30:45.963 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:30:45.966 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:30:45.967 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:30:46.068 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:30:46.097 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:30:46.104 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:30:46.105 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:30:46.206 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:30:46.252 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:30:46.257 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:30:46.258 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:30:46.335 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:30:46 smithi164 bash[59462]: cluster 1685824245.1861048 mgr.y (mgr.34104) 1648 : cluster 0 pgmap v1704: 89 pgs: 1 active+recovering+remapped, 88 active+clean; 2.7 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 597 op/s; 2415/131040 objects misplaced (1.843%); 1.8 MiB/s, 28 objects/s recovering 2023-06-03T20:30:46.360 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:30:46.389 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:30:46.392 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:30:46.393 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:30:46.494 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:30:46.521 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:30:46.526 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:30:46.527 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:30:46.565 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 20:30:46 smithi089 bash[71163]: ::ffff:172.21.15.164 - - [03/Jun/2023:20:30:46] "GET /metrics HTTP/1.1" 200 229139 "" "Prometheus/2.33.4" 2023-06-03T20:30:46.565 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:30:46 smithi089 bash[75068]: cluster 1685824245.1861048 mgr.y (mgr.34104) 1648 : cluster 0 pgmap v1704: 89 pgs: 1 active+recovering+remapped, 88 active+clean; 2.7 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 597 op/s; 2415/131040 objects misplaced (1.843%); 1.8 MiB/s, 28 objects/s recovering 2023-06-03T20:30:46.628 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:30:46.667 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:30:46.669 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:30:46.669 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:30:46.770 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:30:46.801 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:30:46.803 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:30:46.803 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:30:46.904 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:30:46.913 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:30:46 smithi089 bash[78452]: cluster 1685824245.1861048 mgr.y (mgr.34104) 1648 : cluster 0 pgmap v1704: 89 pgs: 1 active+recovering+remapped, 88 active+clean; 2.7 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 597 op/s; 2415/131040 objects misplaced (1.843%); 1.8 MiB/s, 28 objects/s recovering 2023-06-03T20:30:46.931 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:30:46.935 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:30:46.935 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:30:47.036 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:30:47.072 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:30:47.075 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:30:47.075 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:30:47.176 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:30:47.214 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:30:47.219 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:30:47.219 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:30:47.244 INFO:teuthology.orchestra.run.smithi089.stdout:{"pg_ready":true,"pg_map":{"version":1704,"stamp":"2023-06-03T20:30:45.185775+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":2861569958,"num_objects":43680,"num_object_clones":0,"num_object_copies":131040,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":2415,"num_objects_unfound":0,"num_objects_dirty":43680,"num_whiteouts":0,"num_read":309,"num_read_kb":513,"num_write":152633,"num_write_kb":6275279,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2680,"num_bytes_recovered":175511896,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139867,"ondisk_log_size":139867,"up":267,"acting":267,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":268,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":13760968,"kb_used_data":8541384,"kb_used_omap":30,"kb_used_meta":5219297,"kb_avail":736200248,"statfs":{"total":767960285184,"available":753869053952,"internally_reserved":0,"allocated":8746377216,"data_stored":8738656909,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":31502,"internal_metadata":5344560370},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":8,"apply_latency_ms":8,"commit_latency_ns":8000000,"apply_latency_ns":8000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-391774208,"num_objects":-5978,"num_object_clones":0,"num_object_copies":-17934,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":-283,"num_objects_unfound":0,"num_objects_dirty":-5978,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5978,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":283,"num_bytes_recovered":18546688,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.002813"},"pg_stats":[{"pgid":"3.1a","version":"60'1","reported_seq":218,"reported_epoch":291,"state":"active+clean","last_fresh":"2023-06-03T20:30:07.275997+0000","last_change":"2023-06-03T20:23:28.242261+0000","last_active":"2023-06-03T20:30:07.275997+0000","last_peered":"2023-06-03T20:30:07.275997+0000","last_clean":"2023-06-03T20:30:07.275997+0000","last_became_active":"2023-06-03T20:23:28.240045+0000","last_became_peered":"2023-06-03T20:23:28.240045+0000","last_unstale":"2023-06-03T20:30:07.275997+0000","last_undegraded":"2023-06-03T20:30:07.275997+0000","last_fullsized":"2023-06-03T20:30:07.275997+0000","mapping_epoch":251,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":252,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T05:54:33.215385+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":79,"reported_epoch":291,"state":"active+clean","last_fresh":"2023-06-03T20:30:07.285226+0000","last_change":"2023-06-03T20:23:28.283021+0000","last_active":"2023-06-03T20:30:07.285226+0000","last_peered":"2023-06-03T20:30:07.285226+0000","last_clean":"2023-06-03T20:30:07.285226+0000","last_became_active":"2023-06-03T20:23:28.282428+0000","last_became_peered":"2023-06-03T20:23:28.282428+0000","last_unstale":"2023-06-03T20:30:07.285226+0000","last_undegraded":"2023-06-03T20:30:07.285226+0000","last_fullsized":"2023-06-03T20:30:07.285226+0000","mapping_epoch":251,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":252,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T05:01:33.115332+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":79,"reported_epoch":291,"state":"active+clean","last_fresh":"2023-06-03T20:30:07.284001+0000","last_change":"2023-06-03T20:23:28.253821+0000","last_active":"2023-06-03T20:30:07.284001+0000","last_peered":"2023-06-03T20:30:07.284001+0000","last_clean":"2023-06-03T20:30:07.284001+0000","last_became_active":"2023-06-03T20:23:28.253448+0000","last_became_peered":"2023-06-03T20:23:28.253448+0000","last_unstale":"2023-06-03T20:30:07.284001+0000","last_undegraded":"2023-06-03T20:30:07.284001+0000","last_fullsized":"2023-06-03T20:30:07.284001+0000","mapping_epoch":251,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":252,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T23:40:02.264778+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":262,"reported_epoch":291,"state":"active+clean","last_fresh":"2023-06-03T20:30:07.270919+0000","last_change":"2023-06-03T19:56:54.507653+0000","last_active":"2023-06-03T20:30:07.270919+0000","last_peered":"2023-06-03T20:30:07.270919+0000","last_clean":"2023-06-03T20:30:07.270919+0000","last_became_active":"2023-06-03T19:56:54.507350+0000","last_became_peered":"2023-06-03T19:56:54.507350+0000","last_unstale":"2023-06-03T20:30:07.270919+0000","last_undegraded":"2023-06-03T20:30:07.270919+0000","last_fullsized":"2023-06-03T20:30:07.270919+0000","mapping_epoch":119,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T22:33:20.065844+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":302,"reported_epoch":291,"state":"active+clean","last_fresh":"2023-06-03T20:30:07.264906+0000","last_change":"2023-06-03T20:21:01.256894+0000","last_active":"2023-06-03T20:30:07.264906+0000","last_peered":"2023-06-03T20:30:07.264906+0000","last_clean":"2023-06-03T20:30:07.264906+0000","last_became_active":"2023-06-03T20:21:01.255903+0000","last_became_peered":"2023-06-03T20:21:01.255903+0000","last_unstale":"2023-06-03T20:30:07.264906+0000","last_undegraded":"2023-06-03T20:30:07.264906+0000","last_fullsized":"2023-06-03T20:30:07.264906+0000","mapping_epoch":235,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":236,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T20:35:27.810860+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":56,"reported_epoch":291,"state":"active+clean","last_fresh":"2023-06-03T20:30:07.274783+0000","last_change":"2023-06-03T20:28:24.269355+0000","last_active":"2023-06-03T20:30:07.274783+0000","last_peered":"2023-06-03T20:30:07.274783+0000","last_clean":"2023-06-03T20:30:07.274783+0000","last_became_active":"2023-06-03T20:28:24.268859+0000","last_became_peered":"2023-06-03T20:28:24.268859+0000","last_unstale":"2023-06-03T20:30:07.274783+0000","last_undegraded":"2023-06-03T20:30:07.274783+0000","last_fullsized":"2023-06-03T20:30:07.274783+0000","mapping_epoch":284,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":285,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,2],"acting":[6,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":258,"reported_epoch":291,"state":"active+clean","last_fresh":"2023-06-03T20:30:38.299124+0000","last_change":"2023-06-03T19:56:54.509490+0000","last_active":"2023-06-03T20:30:38.299124+0000","last_peered":"2023-06-03T20:30:38.299124+0000","last_clean":"2023-06-03T20:30:38.299124+0000","last_became_active":"2023-06-03T19:56:54.509145+0000","last_became_peered":"2023-06-03T19:56:54.509145+0000","last_unstale":"2023-06-03T20:30:38.299124+0000","last_undegraded":"2023-06-03T20:30:38.299124+0000","last_fullsized":"2023-06-03T20:30:38.299124+0000","mapping_epoch":119,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T07:12:03.964110+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":323,"reported_epoch":291,"state":"active+clean","last_fresh":"2023-06-03T20:30:07.264971+0000","last_change":"2023-06-03T20:28:51.233277+0000","last_active":"2023-06-03T20:30:07.264971+0000","last_peered":"2023-06-03T20:30:07.264971+0000","last_clean":"2023-06-03T20:30:07.264971+0000","last_became_active":"2023-06-03T20:28:51.232919+0000","last_became_peered":"2023-06-03T20:28:51.232919+0000","last_unstale":"2023-06-03T20:30:07.264971+0000","last_undegraded":"2023-06-03T20:30:07.264971+0000","last_fullsized":"2023-06-03T20:30:07.264971+0000","mapping_epoch":286,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":287,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T02:06:02.736415+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":224,"reported_epoch":291,"state":"active+clean","last_fresh":"2023-06-03T20:30:07.283105+0000","last_change":"2023-06-03T20:21:01.258441+0000","last_active":"2023-06-03T20:30:07.283105+0000","last_peered":"2023-06-03T20:30:07.283105+0000","last_clean":"2023-06-03T20:30:07.283105+0000","last_became_active":"2023-06-03T20:21:01.257779+0000","last_became_peered":"2023-06-03T20:21:01.257779+0000","last_unstale":"2023-06-03T20:30:07.283105+0000","last_undegraded":"2023-06-03T20:30:07.283105+0000","last_fullsized":"2023-06-03T20:30:07.283105+0000","mapping_epoch":235,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":236,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,1],"acting":[7,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":268,"reported_epoch":291,"state":"active+clean","last_fresh":"2023-06-03T20:30:07.276808+0000","last_change":"2023-06-03T20:24:02.267331+0000","last_active":"2023-06-03T20:30:07.276808+0000","last_peered":"2023-06-03T20:30:07.276808+0000","last_clean":"2023-06-03T20:30:07.276808+0000","last_became_active":"2023-06-03T20:24:02.266997+0000","last_became_peered":"2023-06-03T20:24:02.266997+0000","last_unstale":"2023-06-03T20:30:07.276808+0000","last_undegraded":"2023-06-03T20:30:07.276808+0000","last_fullsized":"2023-06-03T20:30:07.276808+0000","mapping_epoch":254,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":255,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T22:09:18.993656+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"60'3","reported_seq":289,"reported_epoch":291,"state":"active+clean","last_fresh":"2023-06-03T20:30:07.284206+0000","last_change":"2023-06-03T20:23:27.261246+0000","last_active":"2023-06-03T20:30:07.284206+0000","last_peered":"2023-06-03T20:30:07.284206+0000","last_clean":"2023-06-03T20:30:07.284206+0000","last_became_active":"2023-06-03T20:23:27.259201+0000","last_became_peered":"2023-06-03T20:23:27.259201+0000","last_unstale":"2023-06-03T20:30:07.284206+0000","last_undegraded":"2023-06-03T20:30:07.284206+0000","last_fullsized":"2023-06-03T20:30:07.284206+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":80,"reported_epoch":291,"state":"active+clean","last_fresh":"2023-06-03T20:30:22.800932+0000","last_change":"2023-06-03T20:28:51.235184+0000","last_active":"2023-06-03T20:30:22.800932+0000","last_peered":"2023-06-03T20:30:22.800932+0000","last_clean":"2023-06-03T20:30:22.800932+0000","last_became_active":"2023-06-03T20:28:51.234830+0000","last_became_peered":"2023-06-03T20:28:51.234830+0000","last_unstale":"2023-06-03T20:30:22.800932+0000","last_undegraded":"2023-06-03T20:30:22.800932+0000","last_fullsized":"2023-06-03T20:30:22.800932+0000","mapping_epoch":286,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":287,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T04:40:02.479523+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"77'23","reported_seq":226,"reported_epoch":291,"state":"active+clean","last_fresh":"2023-06-03T20:30:31.403248+0000","last_change":"2023-06-03T20:24:02.280274+0000","last_active":"2023-06-03T20:30:31.403248+0000","last_peered":"2023-06-03T20:30:31.403248+0000","last_clean":"2023-06-03T20:30:31.403248+0000","last_became_active":"2023-06-03T20:24:02.264914+0000","last_became_peered":"2023-06-03T20:24:02.264914+0000","last_unstale":"2023-06-03T20:30:31.403248+0000","last_undegraded":"2023-06-03T20:30:31.403248+0000","last_fullsized":"2023-06-03T20:30:31.403248+0000","mapping_epoch":254,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":255,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":23,"log_dups_size":0,"ondisk_log_size":23,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T07:00:35.888220+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4716,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":23,"num_write_kb":20,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4716,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":202,"reported_epoch":291,"state":"active+clean","last_fresh":"2023-06-03T20:30:31.403157+0000","last_change":"2023-06-03T20:24:02.254850+0000","last_active":"2023-06-03T20:30:31.403157+0000","last_peered":"2023-06-03T20:30:31.403157+0000","last_clean":"2023-06-03T20:30:31.403157+0000","last_became_active":"2023-06-03T20:24:02.254409+0000","last_became_peered":"2023-06-03T20:24:02.254409+0000","last_unstale":"2023-06-03T20:30:31.403157+0000","last_undegraded":"2023-06-03T20:30:31.403157+0000","last_fullsized":"2023-06-03T20:30:31.403157+0000","mapping_epoch":254,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":255,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T19:49:53.393121+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"60'2","reported_seq":291,"reported_epoch":291,"state":"active+clean","last_fresh":"2023-06-03T20:30:07.283665+0000","last_change":"2023-06-03T20:23:27.258749+0000","last_active":"2023-06-03T20:30:07.283665+0000","last_peered":"2023-06-03T20:30:07.283665+0000","last_clean":"2023-06-03T20:30:07.283665+0000","last_became_active":"2023-06-03T20:23:27.256899+0000","last_became_peered":"2023-06-03T20:23:27.256899+0000","last_unstale":"2023-06-03T20:30:07.283665+0000","last_undegraded":"2023-06-03T20:30:07.283665+0000","last_fullsized":"2023-06-03T20:30:07.283665+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":287,"reported_epoch":291,"state":"active+clean","last_fresh":"2023-06-03T20:30:07.265795+0000","last_change":"2023-06-03T20:21:01.256612+0000","last_active":"2023-06-03T20:30:07.265795+0000","last_peered":"2023-06-03T20:30:07.265795+0000","last_clean":"2023-06-03T20:30:07.265795+0000","last_became_active":"2023-06-03T20:21:01.256243+0000","last_became_peered":"2023-06-03T20:21:01.256243+0000","last_unstale":"2023-06-03T20:30:07.265795+0000","last_undegraded":"2023-06-03T20:30:07.265795+0000","last_fullsized":"2023-06-03T20:30:07.265795+0000","mapping_epoch":235,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":236,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T05:24:47.054342+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":258,"reported_epoch":291,"state":"active+clean","last_fresh":"2023-06-03T20:30:07.270615+0000","last_change":"2023-06-03T19:56:54.507479+0000","last_active":"2023-06-03T20:30:07.270615+0000","last_peered":"2023-06-03T20:30:07.270615+0000","last_clean":"2023-06-03T20:30:07.270615+0000","last_became_active":"2023-06-03T19:56:54.507149+0000","last_became_peered":"2023-06-03T19:56:54.507149+0000","last_unstale":"2023-06-03T20:30:07.270615+0000","last_undegraded":"2023-06-03T20:30:07.270615+0000","last_fullsized":"2023-06-03T20:30:07.270615+0000","mapping_epoch":119,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T20:22:36.676317+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":158,"reported_epoch":291,"state":"active+clean","last_fresh":"2023-06-03T20:30:07.284292+0000","last_change":"2023-06-03T20:17:22.832844+0000","last_active":"2023-06-03T20:30:07.284292+0000","last_peered":"2023-06-03T20:30:07.284292+0000","last_clean":"2023-06-03T20:30:07.284292+0000","last_became_active":"2023-06-03T20:17:22.832473+0000","last_became_peered":"2023-06-03T20:17:22.832473+0000","last_unstale":"2023-06-03T20:30:07.284292+0000","last_undegraded":"2023-06-03T20:30:07.284292+0000","last_fullsized":"2023-06-03T20:30:07.284292+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,3],"acting":[7,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":291,"reported_epoch":291,"state":"active+clean","last_fresh":"2023-06-03T20:30:07.283383+0000","last_change":"2023-06-03T20:24:02.267531+0000","last_active":"2023-06-03T20:30:07.283383+0000","last_peered":"2023-06-03T20:30:07.283383+0000","last_clean":"2023-06-03T20:30:07.283383+0000","last_became_active":"2023-06-03T20:24:02.267159+0000","last_became_peered":"2023-06-03T20:24:02.267159+0000","last_unstale":"2023-06-03T20:30:07.283383+0000","last_undegraded":"2023-06-03T20:30:07.283383+0000","last_fullsized":"2023-06-03T20:30:07.283383+0000","mapping_epoch":254,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":255,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":259,"reported_epoch":291,"state":"active+clean","last_fresh":"2023-06-03T20:30:38.299098+0000","last_change":"2023-06-03T19:56:54.510289+0000","last_active":"2023-06-03T20:30:38.299098+0000","last_peered":"2023-06-03T20:30:38.299098+0000","last_clean":"2023-06-03T20:30:38.299098+0000","last_became_active":"2023-06-03T19:56:54.509905+0000","last_became_peered":"2023-06-03T19:56:54.509905+0000","last_unstale":"2023-06-03T20:30:38.299098+0000","last_undegraded":"2023-06-03T20:30:38.299098+0000","last_fullsized":"2023-06-03T20:30:38.299098+0000","mapping_epoch":119,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T02:54:26.792401+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"60'8","reported_seq":57,"reported_epoch":291,"state":"active+clean","last_fresh":"2023-06-03T20:30:07.270150+0000","last_change":"2023-06-03T20:26:59.682623+0000","last_active":"2023-06-03T20:30:07.270150+0000","last_peered":"2023-06-03T20:30:07.270150+0000","last_clean":"2023-06-03T20:30:07.270150+0000","last_became_active":"2023-06-03T20:24:02.260105+0000","last_became_peered":"2023-06-03T20:24:02.260105+0000","last_unstale":"2023-06-03T20:30:07.270150+0000","last_undegraded":"2023-06-03T20:30:07.270150+0000","last_fullsized":"2023-06-03T20:30:07.270150+0000","mapping_epoch":254,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":273,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":296,"reported_epoch":292,"state":"active+clean","last_fresh":"2023-06-03T20:30:38.308161+0000","last_change":"2023-06-03T20:28:24.281308+0000","last_active":"2023-06-03T20:30:38.308161+0000","last_peered":"2023-06-03T20:30:38.308161+0000","last_clean":"2023-06-03T20:30:38.308161+0000","last_became_active":"2023-06-03T20:28:24.280981+0000","last_became_peered":"2023-06-03T20:28:24.280981+0000","last_unstale":"2023-06-03T20:30:38.308161+0000","last_undegraded":"2023-06-03T20:30:38.308161+0000","last_fullsized":"2023-06-03T20:30:38.308161+0000","mapping_epoch":284,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":285,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T02:44:14.890147+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"14.2","version":"292'9339","reported_seq":9361,"reported_epoch":292,"state":"active+clean","last_fresh":"2023-06-03T20:30:43.094978+0000","last_change":"2023-06-03T20:27:21.371612+0000","last_active":"2023-06-03T20:30:43.094978+0000","last_peered":"2023-06-03T20:30:43.094978+0000","last_clean":"2023-06-03T20:30:43.094978+0000","last_became_active":"2023-06-03T20:27:21.371344+0000","last_became_peered":"2023-06-03T20:27:21.371344+0000","last_unstale":"2023-06-03T20:30:43.094978+0000","last_undegraded":"2023-06-03T20:30:43.094978+0000","last_fullsized":"2023-06-03T20:30:43.094978+0000","mapping_epoch":278,"log_start":"0'0","ondisk_log_start":"0'0","created":278,"last_epoch_clean":279,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T20:27:20.339366+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T20:27:20.339366+0000","last_clean_scrub_stamp":"2023-06-03T20:27:20.339366+0000","objects_scrubbed":0,"log_size":9339,"log_dups_size":0,"ondisk_log_size":9339,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":176357376,"num_objects":2691,"num_object_clones":0,"num_object_copies":8073,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2691,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9339,"num_write_kb":384960,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,3],"acting":[6,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.f","version":"60'1","reported_seq":290,"reported_epoch":291,"state":"active+clean","last_fresh":"2023-06-03T20:30:07.283319+0000","last_change":"2023-06-03T20:23:27.263667+0000","last_active":"2023-06-03T20:30:07.283319+0000","last_peered":"2023-06-03T20:30:07.283319+0000","last_clean":"2023-06-03T20:30:07.283319+0000","last_became_active":"2023-06-03T20:23:27.263110+0000","last_became_peered":"2023-06-03T20:23:27.263110+0000","last_unstale":"2023-06-03T20:30:07.283319+0000","last_undegraded":"2023-06-03T20:30:07.283319+0000","last_fullsized":"2023-06-03T20:30:07.283319+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":251,"reported_epoch":291,"state":"active+clean","last_fresh":"2023-06-03T20:30:07.283324+0000","last_change":"2023-06-03T20:28:51.231222+0000","last_active":"2023-06-03T20:30:07.283324+0000","last_peered":"2023-06-03T20:30:07.283324+0000","last_clean":"2023-06-03T20:30:07.283324+0000","last_became_active":"2023-06-03T20:28:51.230848+0000","last_became_peered":"2023-06-03T20:28:51.230848+0000","last_unstale":"2023-06-03T20:30:07.283324+0000","last_undegraded":"2023-06-03T20:30:07.283324+0000","last_fullsized":"2023-06-03T20:30:07.283324+0000","mapping_epoch":286,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":287,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"14.3","version":"292'9590","reported_seq":9613,"reported_epoch":292,"state":"active+clean","last_fresh":"2023-06-03T20:30:43.287786+0000","last_change":"2023-06-03T20:27:21.374055+0000","last_active":"2023-06-03T20:30:43.287786+0000","last_peered":"2023-06-03T20:30:43.287786+0000","last_clean":"2023-06-03T20:30:43.287786+0000","last_became_active":"2023-06-03T20:27:21.373728+0000","last_became_peered":"2023-06-03T20:27:21.373728+0000","last_unstale":"2023-06-03T20:30:43.287786+0000","last_undegraded":"2023-06-03T20:30:43.287786+0000","last_fullsized":"2023-06-03T20:30:43.287786+0000","mapping_epoch":278,"log_start":"281'700","ondisk_log_start":"281'700","created":278,"last_epoch_clean":279,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T20:27:20.339366+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T20:27:20.339366+0000","last_clean_scrub_stamp":"2023-06-03T20:27:20.339366+0000","objects_scrubbed":0,"log_size":8890,"log_dups_size":0,"ondisk_log_size":8890,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T05:15:33.026018+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":180224000,"num_objects":2750,"num_object_clones":0,"num_object_copies":8250,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2750,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9590,"num_write_kb":394880,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.e","version":"60'2","reported_seq":319,"reported_epoch":291,"state":"active+clean","last_fresh":"2023-06-03T20:30:07.283120+0000","last_change":"2023-06-03T20:23:28.606751+0000","last_active":"2023-06-03T20:30:07.283120+0000","last_peered":"2023-06-03T20:30:07.283120+0000","last_clean":"2023-06-03T20:30:07.283120+0000","last_became_active":"2023-06-03T20:23:28.605968+0000","last_became_peered":"2023-06-03T20:23:28.605968+0000","last_unstale":"2023-06-03T20:30:07.283120+0000","last_undegraded":"2023-06-03T20:30:07.283120+0000","last_fullsized":"2023-06-03T20:30:07.283120+0000","mapping_epoch":251,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":252,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":231,"reported_epoch":291,"state":"active+clean","last_fresh":"2023-06-03T20:30:07.280779+0000","last_change":"2023-06-03T20:23:28.249694+0000","last_active":"2023-06-03T20:30:07.280779+0000","last_peered":"2023-06-03T20:30:07.280779+0000","last_clean":"2023-06-03T20:30:07.280779+0000","last_became_active":"2023-06-03T20:23:28.249388+0000","last_became_peered":"2023-06-03T20:23:28.249388+0000","last_unstale":"2023-06-03T20:30:07.280779+0000","last_undegraded":"2023-06-03T20:30:07.280779+0000","last_fullsized":"2023-06-03T20:30:07.280779+0000","mapping_epoch":251,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":252,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T05:44:01.835586+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"14.0","version":"292'9415","reported_seq":9439,"reported_epoch":292,"state":"active+clean","last_fresh":"2023-06-03T20:30:42.657294+0000","last_change":"2023-06-03T20:27:21.378346+0000","last_active":"2023-06-03T20:30:42.657294+0000","last_peered":"2023-06-03T20:30:42.657294+0000","last_clean":"2023-06-03T20:30:42.657294+0000","last_became_active":"2023-06-03T20:27:21.377969+0000","last_became_peered":"2023-06-03T20:27:21.377969+0000","last_unstale":"2023-06-03T20:30:42.657294+0000","last_undegraded":"2023-06-03T20:30:42.657294+0000","last_fullsized":"2023-06-03T20:30:42.657294+0000","mapping_epoch":278,"log_start":"0'0","ondisk_log_start":"0'0","created":278,"last_epoch_clean":279,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T20:27:20.339366+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T20:27:20.339366+0000","last_clean_scrub_stamp":"2023-06-03T20:27:20.339366+0000","objects_scrubbed":0,"log_size":9415,"log_dups_size":0,"ondisk_log_size":9415,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T01:32:49.543336+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":174915584,"num_objects":2669,"num_object_clones":0,"num_object_copies":8007,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2669,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9415,"num_write_kb":386688,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,5],"acting":[2,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.d","version":"60'1","reported_seq":268,"reported_epoch":291,"state":"active+clean","last_fresh":"2023-06-03T20:30:07.284069+0000","last_change":"2023-06-03T20:26:59.686037+0000","last_active":"2023-06-03T20:30:07.284069+0000","last_peered":"2023-06-03T20:30:07.284069+0000","last_clean":"2023-06-03T20:30:07.284069+0000","last_became_active":"2023-06-03T20:24:02.268002+0000","last_became_peered":"2023-06-03T20:24:02.268002+0000","last_unstale":"2023-06-03T20:30:07.284069+0000","last_undegraded":"2023-06-03T20:30:07.284069+0000","last_fullsized":"2023-06-03T20:30:07.284069+0000","mapping_epoch":254,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":273,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":140,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":15,"reported_epoch":291,"state":"active+clean","last_fresh":"2023-06-03T20:30:07.271467+0000","last_change":"2023-06-03T20:28:51.230833+0000","last_active":"2023-06-03T20:30:07.271467+0000","last_peered":"2023-06-03T20:30:07.271467+0000","last_clean":"2023-06-03T20:30:07.271467+0000","last_became_active":"2023-06-03T20:28:51.230489+0000","last_became_peered":"2023-06-03T20:28:51.230489+0000","last_unstale":"2023-06-03T20:30:07.271467+0000","last_undegraded":"2023-06-03T20:30:07.271467+0000","last_fullsized":"2023-06-03T20:30:07.271467+0000","mapping_epoch":286,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":287,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"14.1","version":"292'9882","reported_seq":9906,"reported_epoch":292,"state":"active+clean","last_fresh":"2023-06-03T20:30:42.858336+0000","last_change":"2023-06-03T20:27:21.372641+0000","last_active":"2023-06-03T20:30:42.858336+0000","last_peered":"2023-06-03T20:30:42.858336+0000","last_clean":"2023-06-03T20:30:42.858336+0000","last_became_active":"2023-06-03T20:27:21.372158+0000","last_became_peered":"2023-06-03T20:27:21.372158+0000","last_unstale":"2023-06-03T20:30:42.858336+0000","last_undegraded":"2023-06-03T20:30:42.858336+0000","last_fullsized":"2023-06-03T20:30:42.858336+0000","mapping_epoch":278,"log_start":"281'1700","ondisk_log_start":"281'1700","created":278,"last_epoch_clean":279,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T20:27:20.339366+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T20:27:20.339366+0000","last_clean_scrub_stamp":"2023-06-03T20:27:20.339366+0000","objects_scrubbed":0,"log_size":8182,"log_dups_size":0,"ondisk_log_size":8182,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T07:52:41.694374+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":187432960,"num_objects":2860,"num_object_clones":0,"num_object_copies":8580,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2860,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9882,"num_write_kb":407744,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":284,"reported_epoch":291,"state":"active+clean","last_fresh":"2023-06-03T20:30:07.265946+0000","last_change":"2023-06-03T20:24:02.261252+0000","last_active":"2023-06-03T20:30:07.265946+0000","last_peered":"2023-06-03T20:30:07.265946+0000","last_clean":"2023-06-03T20:30:07.265946+0000","last_became_active":"2023-06-03T20:24:02.260824+0000","last_became_peered":"2023-06-03T20:24:02.260824+0000","last_unstale":"2023-06-03T20:30:07.265946+0000","last_undegraded":"2023-06-03T20:30:07.265946+0000","last_fullsized":"2023-06-03T20:30:07.265946+0000","mapping_epoch":254,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":255,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T20:00:53.584753+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":322,"reported_epoch":291,"state":"active+clean","last_fresh":"2023-06-03T20:30:07.275881+0000","last_change":"2023-06-03T20:23:28.283360+0000","last_active":"2023-06-03T20:30:07.275881+0000","last_peered":"2023-06-03T20:30:07.275881+0000","last_clean":"2023-06-03T20:30:07.275881+0000","last_became_active":"2023-06-03T20:23:28.283035+0000","last_became_peered":"2023-06-03T20:23:28.283035+0000","last_unstale":"2023-06-03T20:30:07.275881+0000","last_undegraded":"2023-06-03T20:30:07.275881+0000","last_fullsized":"2023-06-03T20:30:07.275881+0000","mapping_epoch":251,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":252,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T00:22:59.446515+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"14.6","version":"292'9569","reported_seq":9593,"reported_epoch":292,"state":"active+clean","last_fresh":"2023-06-03T20:30:43.186627+0000","last_change":"2023-06-03T20:27:21.375836+0000","last_active":"2023-06-03T20:30:43.186627+0000","last_peered":"2023-06-03T20:30:43.186627+0000","last_clean":"2023-06-03T20:30:43.186627+0000","last_became_active":"2023-06-03T20:27:21.375522+0000","last_became_peered":"2023-06-03T20:27:21.375522+0000","last_unstale":"2023-06-03T20:30:43.186627+0000","last_undegraded":"2023-06-03T20:30:43.186627+0000","last_fullsized":"2023-06-03T20:30:43.186627+0000","mapping_epoch":278,"log_start":"281'700","ondisk_log_start":"281'700","created":278,"last_epoch_clean":279,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T20:27:20.339366+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T20:27:20.339366+0000","last_clean_scrub_stamp":"2023-06-03T20:27:20.339366+0000","objects_scrubbed":0,"log_size":8869,"log_dups_size":0,"ondisk_log_size":8869,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T21:02:49.138563+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":177668096,"num_objects":2711,"num_object_clones":0,"num_object_copies":8133,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2711,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9569,"num_write_kb":392960,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,1],"acting":[0,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.b","version":"60'2","reported_seq":286,"reported_epoch":291,"state":"active+clean","last_fresh":"2023-06-03T20:30:37.896780+0000","last_change":"2023-06-03T20:23:27.254179+0000","last_active":"2023-06-03T20:30:37.896780+0000","last_peered":"2023-06-03T20:30:37.896780+0000","last_clean":"2023-06-03T20:30:37.896780+0000","last_became_active":"2023-06-03T20:23:27.253791+0000","last_became_peered":"2023-06-03T20:23:27.253791+0000","last_unstale":"2023-06-03T20:30:37.896780+0000","last_undegraded":"2023-06-03T20:30:37.896780+0000","last_fullsized":"2023-06-03T20:30:37.896780+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T00:31:27.336760+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":207,"reported_epoch":291,"state":"active+clean","last_fresh":"2023-06-03T20:30:16.388677+0000","last_change":"2023-06-03T20:24:02.255551+0000","last_active":"2023-06-03T20:30:16.388677+0000","last_peered":"2023-06-03T20:30:16.388677+0000","last_clean":"2023-06-03T20:30:16.388677+0000","last_became_active":"2023-06-03T20:24:02.255209+0000","last_became_peered":"2023-06-03T20:24:02.255209+0000","last_unstale":"2023-06-03T20:30:16.388677+0000","last_undegraded":"2023-06-03T20:30:16.388677+0000","last_fullsized":"2023-06-03T20:30:16.388677+0000","mapping_epoch":254,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":255,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T20:05:02.741217+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"14.7","version":"292'9572","reported_seq":9595,"reported_epoch":292,"state":"active+clean","last_fresh":"2023-06-03T20:30:41.355597+0000","last_change":"2023-06-03T20:27:21.364098+0000","last_active":"2023-06-03T20:30:41.355597+0000","last_peered":"2023-06-03T20:30:41.355597+0000","last_clean":"2023-06-03T20:30:41.355597+0000","last_became_active":"2023-06-03T20:27:21.363548+0000","last_became_peered":"2023-06-03T20:27:21.363548+0000","last_unstale":"2023-06-03T20:30:41.355597+0000","last_undegraded":"2023-06-03T20:30:41.355597+0000","last_fullsized":"2023-06-03T20:30:41.355597+0000","mapping_epoch":278,"log_start":"281'1800","ondisk_log_start":"281'1800","created":278,"last_epoch_clean":279,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T20:27:20.339366+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T20:27:20.339366+0000","last_clean_scrub_stamp":"2023-06-03T20:27:20.339366+0000","objects_scrubbed":0,"log_size":7772,"log_dups_size":0,"ondisk_log_size":7772,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T06:05:40.316567+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":180486144,"num_objects":2754,"num_object_clones":0,"num_object_copies":8262,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2754,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9572,"num_write_kb":394432,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,0],"acting":[4,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":47,"reported_epoch":291,"state":"active+clean","last_fresh":"2023-06-03T20:30:07.271683+0000","last_change":"2023-06-03T20:24:02.254172+0000","last_active":"2023-06-03T20:30:07.271683+0000","last_peered":"2023-06-03T20:30:07.271683+0000","last_clean":"2023-06-03T20:30:07.271683+0000","last_became_active":"2023-06-03T20:24:02.253797+0000","last_became_peered":"2023-06-03T20:24:02.253797+0000","last_unstale":"2023-06-03T20:30:07.271683+0000","last_undegraded":"2023-06-03T20:30:07.271683+0000","last_fullsized":"2023-06-03T20:30:07.271683+0000","mapping_epoch":254,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":255,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":212,"reported_epoch":291,"state":"active+clean","last_fresh":"2023-06-03T20:30:16.388718+0000","last_change":"2023-06-03T20:23:28.291250+0000","last_active":"2023-06-03T20:30:16.388718+0000","last_peered":"2023-06-03T20:30:16.388718+0000","last_clean":"2023-06-03T20:30:16.388718+0000","last_became_active":"2023-06-03T20:23:28.290120+0000","last_became_peered":"2023-06-03T20:23:28.290120+0000","last_unstale":"2023-06-03T20:30:16.388718+0000","last_undegraded":"2023-06-03T20:30:16.388718+0000","last_fullsized":"2023-06-03T20:30:16.388718+0000","mapping_epoch":251,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":252,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T22:37:56.033327+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"14.4","version":"292'9484","reported_seq":9506,"reported_epoch":292,"state":"active+clean","last_fresh":"2023-06-03T20:30:43.017546+0000","last_change":"2023-06-03T20:27:21.366337+0000","last_active":"2023-06-03T20:30:43.017546+0000","last_peered":"2023-06-03T20:30:43.017546+0000","last_clean":"2023-06-03T20:30:43.017546+0000","last_became_active":"2023-06-03T20:27:21.365886+0000","last_became_peered":"2023-06-03T20:27:21.365886+0000","last_unstale":"2023-06-03T20:30:43.017546+0000","last_undegraded":"2023-06-03T20:30:43.017546+0000","last_fullsized":"2023-06-03T20:30:43.017546+0000","mapping_epoch":278,"log_start":"0'0","ondisk_log_start":"0'0","created":278,"last_epoch_clean":279,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T20:27:20.339366+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T20:27:20.339366+0000","last_clean_scrub_stamp":"2023-06-03T20:27:20.339366+0000","objects_scrubbed":0,"log_size":9484,"log_dups_size":0,"ondisk_log_size":9484,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":177864704,"num_objects":2714,"num_object_clones":0,"num_object_copies":8142,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2714,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9484,"num_write_kb":390336,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,3],"acting":[6,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":197,"reported_epoch":291,"state":"active+clean","last_fresh":"2023-06-03T20:30:07.280777+0000","last_change":"2023-06-03T20:23:27.276693+0000","last_active":"2023-06-03T20:30:07.280777+0000","last_peered":"2023-06-03T20:30:07.280777+0000","last_clean":"2023-06-03T20:30:07.280777+0000","last_became_active":"2023-06-03T20:23:27.276305+0000","last_became_peered":"2023-06-03T20:23:27.276305+0000","last_unstale":"2023-06-03T20:30:07.280777+0000","last_undegraded":"2023-06-03T20:30:07.280777+0000","last_fullsized":"2023-06-03T20:30:07.280777+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T20:34:50.944936+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":202,"reported_epoch":291,"state":"active+clean","last_fresh":"2023-06-03T20:30:16.392582+0000","last_change":"2023-06-03T20:28:51.231829+0000","last_active":"2023-06-03T20:30:16.392582+0000","last_peered":"2023-06-03T20:30:16.392582+0000","last_clean":"2023-06-03T20:30:16.392582+0000","last_became_active":"2023-06-03T20:28:51.231452+0000","last_became_peered":"2023-06-03T20:28:51.231452+0000","last_unstale":"2023-06-03T20:30:16.392582+0000","last_undegraded":"2023-06-03T20:30:16.392582+0000","last_fullsized":"2023-06-03T20:30:16.392582+0000","mapping_epoch":286,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":287,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T07:00:21.647058+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"14.5","version":"292'9663","reported_seq":9687,"reported_epoch":292,"state":"active+clean","last_fresh":"2023-06-03T20:30:43.198638+0000","last_change":"2023-06-03T20:27:21.371070+0000","last_active":"2023-06-03T20:30:43.198638+0000","last_peered":"2023-06-03T20:30:43.198638+0000","last_clean":"2023-06-03T20:30:43.198638+0000","last_became_active":"2023-06-03T20:27:21.370790+0000","last_became_peered":"2023-06-03T20:27:21.370790+0000","last_unstale":"2023-06-03T20:30:43.198638+0000","last_undegraded":"2023-06-03T20:30:43.198638+0000","last_fullsized":"2023-06-03T20:30:43.198638+0000","mapping_epoch":278,"log_start":"281'800","ondisk_log_start":"281'800","created":278,"last_epoch_clean":279,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T20:27:20.339366+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T20:27:20.339366+0000","last_clean_scrub_stamp":"2023-06-03T20:27:20.339366+0000","objects_scrubbed":0,"log_size":8863,"log_dups_size":0,"ondisk_log_size":8863,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T21:04:21.657715+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":179240960,"num_objects":2735,"num_object_clones":0,"num_object_copies":8205,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2735,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9663,"num_write_kb":396736,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":295,"reported_epoch":291,"state":"active+clean","last_fresh":"2023-06-03T20:30:07.264837+0000","last_change":"2023-06-03T20:21:01.256958+0000","last_active":"2023-06-03T20:30:07.264837+0000","last_peered":"2023-06-03T20:30:07.264837+0000","last_clean":"2023-06-03T20:30:07.264837+0000","last_became_active":"2023-06-03T20:21:01.255452+0000","last_became_peered":"2023-06-03T20:21:01.255452+0000","last_unstale":"2023-06-03T20:30:07.264837+0000","last_undegraded":"2023-06-03T20:30:07.264837+0000","last_fullsized":"2023-06-03T20:30:07.264837+0000","mapping_epoch":235,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":236,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T01:56:59.777763+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":82,"reported_epoch":291,"state":"active+clean","last_fresh":"2023-06-03T20:30:07.744885+0000","last_change":"2023-06-03T20:23:28.250884+0000","last_active":"2023-06-03T20:30:07.744885+0000","last_peered":"2023-06-03T20:30:07.744885+0000","last_clean":"2023-06-03T20:30:07.744885+0000","last_became_active":"2023-06-03T20:23:28.243793+0000","last_became_peered":"2023-06-03T20:23:28.243793+0000","last_unstale":"2023-06-03T20:30:07.744885+0000","last_undegraded":"2023-06-03T20:30:07.744885+0000","last_fullsized":"2023-06-03T20:30:07.744885+0000","mapping_epoch":251,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":252,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T03:12:27.946002+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"14.8","version":"292'9538","reported_seq":9562,"reported_epoch":292,"state":"active+clean","last_fresh":"2023-06-03T20:30:43.287191+0000","last_change":"2023-06-03T20:27:21.376040+0000","last_active":"2023-06-03T20:30:43.287191+0000","last_peered":"2023-06-03T20:30:43.287191+0000","last_clean":"2023-06-03T20:30:43.287191+0000","last_became_active":"2023-06-03T20:27:21.375709+0000","last_became_peered":"2023-06-03T20:27:21.375709+0000","last_unstale":"2023-06-03T20:30:43.287191+0000","last_undegraded":"2023-06-03T20:30:43.287191+0000","last_fullsized":"2023-06-03T20:30:43.287191+0000","mapping_epoch":278,"log_start":"281'700","ondisk_log_start":"281'700","created":278,"last_epoch_clean":279,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T20:27:20.339366+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T20:27:20.339366+0000","last_clean_scrub_stamp":"2023-06-03T20:27:20.339366+0000","objects_scrubbed":0,"log_size":8838,"log_dups_size":0,"ondisk_log_size":8838,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T05:50:28.157584+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":179830784,"num_objects":2744,"num_object_clones":0,"num_object_copies":8232,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2744,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9538,"num_write_kb":393024,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":71,"reported_epoch":291,"state":"active+clean","last_fresh":"2023-06-03T20:30:07.285481+0000","last_change":"2023-06-03T20:21:01.258182+0000","last_active":"2023-06-03T20:30:07.285481+0000","last_peered":"2023-06-03T20:30:07.285481+0000","last_clean":"2023-06-03T20:30:07.285481+0000","last_became_active":"2023-06-03T20:21:01.254049+0000","last_became_peered":"2023-06-03T20:21:01.254049+0000","last_unstale":"2023-06-03T20:30:07.285481+0000","last_undegraded":"2023-06-03T20:30:07.285481+0000","last_fullsized":"2023-06-03T20:30:07.285481+0000","mapping_epoch":235,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":236,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:32:20.692043+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:32:20.692043+0000","last_clean_scrub_stamp":"2023-06-03T19:32:20.692043+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T05:16:30.078473+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":180,"reported_epoch":291,"state":"active+clean","last_fresh":"2023-06-03T20:30:07.265095+0000","last_change":"2023-06-03T20:17:22.834746+0000","last_active":"2023-06-03T20:30:07.265095+0000","last_peered":"2023-06-03T20:30:07.265095+0000","last_clean":"2023-06-03T20:30:07.265095+0000","last_became_active":"2023-06-03T20:17:22.834448+0000","last_became_peered":"2023-06-03T20:17:22.834448+0000","last_unstale":"2023-06-03T20:30:07.265095+0000","last_undegraded":"2023-06-03T20:30:07.265095+0000","last_fullsized":"2023-06-03T20:30:07.265095+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T21:16:31.167063+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,2],"acting":[3,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":82,"reported_epoch":291,"state":"active+clean","last_fresh":"2023-06-03T20:30:07.285193+0000","last_change":"2023-06-03T20:23:28.254434+0000","last_active":"2023-06-03T20:30:07.285193+0000","last_peered":"2023-06-03T20:30:07.285193+0000","last_clean":"2023-06-03T20:30:07.285193+0000","last_became_active":"2023-06-03T20:23:28.254012+0000","last_became_peered":"2023-06-03T20:23:28.254012+0000","last_unstale":"2023-06-03T20:30:07.285193+0000","last_undegraded":"2023-06-03T20:30:07.285193+0000","last_fullsized":"2023-06-03T20:30:07.285193+0000","mapping_epoch":251,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":252,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T06:22:09.540298+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"14.e","version":"292'9526","reported_seq":9550,"reported_epoch":292,"state":"active+clean","last_fresh":"2023-06-03T20:30:41.350953+0000","last_change":"2023-06-03T20:27:21.368301+0000","last_active":"2023-06-03T20:30:41.350953+0000","last_peered":"2023-06-03T20:30:41.350953+0000","last_clean":"2023-06-03T20:30:41.350953+0000","last_became_active":"2023-06-03T20:27:21.367955+0000","last_became_peered":"2023-06-03T20:27:21.367955+0000","last_unstale":"2023-06-03T20:30:41.350953+0000","last_undegraded":"2023-06-03T20:30:41.350953+0000","last_fullsized":"2023-06-03T20:30:41.350953+0000","mapping_epoch":278,"log_start":"281'1800","ondisk_log_start":"281'1800","created":278,"last_epoch_clean":279,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T20:27:20.339366+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T20:27:20.339366+0000","last_clean_scrub_stamp":"2023-06-03T20:27:20.339366+0000","objects_scrubbed":0,"log_size":7726,"log_dups_size":0,"ondisk_log_size":7726,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T22:35:27.855608+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":181534720,"num_objects":2770,"num_object_clones":0,"num_object_copies":8310,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2770,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9526,"num_write_kb":393472,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":80,"reported_epoch":291,"state":"active+clean","last_fresh":"2023-06-03T20:30:07.285473+0000","last_change":"2023-06-03T20:28:51.234855+0000","last_active":"2023-06-03T20:30:07.285473+0000","last_peered":"2023-06-03T20:30:07.285473+0000","last_clean":"2023-06-03T20:30:07.285473+0000","last_became_active":"2023-06-03T20:28:51.234514+0000","last_became_peered":"2023-06-03T20:28:51.234514+0000","last_unstale":"2023-06-03T20:30:07.285473+0000","last_undegraded":"2023-06-03T20:30:07.285473+0000","last_fullsized":"2023-06-03T20:30:07.285473+0000","mapping_epoch":286,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":287,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:32:20.692043+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:32:20.692043+0000","last_clean_scrub_stamp":"2023-06-03T19:32:20.692043+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T03:00:56.480804+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":177,"reported_epoch":291,"state":"active+clean","last_fresh":"2023-06-03T20:30:07.276932+0000","last_change":"2023-06-03T20:24:02.258311+0000","last_active":"2023-06-03T20:30:07.276932+0000","last_peered":"2023-06-03T20:30:07.276932+0000","last_clean":"2023-06-03T20:30:07.276932+0000","last_became_active":"2023-06-03T20:24:02.257676+0000","last_became_peered":"2023-06-03T20:24:02.257676+0000","last_unstale":"2023-06-03T20:30:07.276932+0000","last_undegraded":"2023-06-03T20:30:07.276932+0000","last_fullsized":"2023-06-03T20:30:07.276932+0000","mapping_epoch":254,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":255,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T01:47:46.383038+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":71,"reported_epoch":291,"state":"active+clean","last_fresh":"2023-06-03T20:30:07.285545+0000","last_change":"2023-06-03T20:21:01.259331+0000","last_active":"2023-06-03T20:30:07.285545+0000","last_peered":"2023-06-03T20:30:07.285545+0000","last_clean":"2023-06-03T20:30:07.285545+0000","last_became_active":"2023-06-03T20:21:01.258836+0000","last_became_peered":"2023-06-03T20:21:01.258836+0000","last_unstale":"2023-06-03T20:30:07.285545+0000","last_undegraded":"2023-06-03T20:30:07.285545+0000","last_fullsized":"2023-06-03T20:30:07.285545+0000","mapping_epoch":235,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":236,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T01:29:19.471600+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"14.d","version":"292'9622","reported_seq":14953,"reported_epoch":292,"state":"active+recovering+remapped","last_fresh":"2023-06-03T20:30:42.687428+0000","last_change":"2023-06-03T20:28:52.754748+0000","last_active":"2023-06-03T20:30:42.687428+0000","last_peered":"2023-06-03T20:30:42.687428+0000","last_clean":"2023-06-03T20:28:49.972400+0000","last_became_active":"2023-06-03T20:28:52.639960+0000","last_became_peered":"2023-06-03T20:28:52.639960+0000","last_unstale":"2023-06-03T20:30:42.687428+0000","last_undegraded":"2023-06-03T20:30:42.687428+0000","last_fullsized":"2023-06-03T20:30:42.687428+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":278,"last_epoch_clean":279,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T20:27:20.339366+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T20:27:20.339366+0000","last_clean_scrub_stamp":"2023-06-03T20:27:20.339366+0000","objects_scrubbed":0,"log_size":9622,"log_dups_size":0,"ondisk_log_size":9622,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T08:00:38.609409+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":178847768,"num_objects":2730,"num_object_clones":0,"num_object_copies":8190,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":2415,"num_objects_unfound":0,"num_objects_dirty":2730,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":9622,"num_write_kb":395201,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2651,"num_bytes_recovered":173735936,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,4],"acting":[2,5,4],"avail_no_missing":["2","4","5"],"object_location_counts":[{"shards":"2,4,5","objects":315}],"blocked_by":[],"up_primary":6,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":278,"reported_epoch":291,"state":"active+clean","last_fresh":"2023-06-03T20:30:07.290750+0000","last_change":"2023-06-03T19:56:54.504859+0000","last_active":"2023-06-03T20:30:07.290750+0000","last_peered":"2023-06-03T20:30:07.290750+0000","last_clean":"2023-06-03T20:30:07.290750+0000","last_became_active":"2023-06-03T19:56:54.504455+0000","last_became_peered":"2023-06-03T19:56:54.504455+0000","last_unstale":"2023-06-03T20:30:07.290750+0000","last_undegraded":"2023-06-03T20:30:07.290750+0000","last_fullsized":"2023-06-03T20:30:07.290750+0000","mapping_epoch":119,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:32:20.692043+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:32:20.692043+0000","last_clean_scrub_stamp":"2023-06-03T19:32:20.692043+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T06:25:45.522100+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":80,"reported_epoch":291,"state":"active+clean","last_fresh":"2023-06-03T20:30:07.283762+0000","last_change":"2023-06-03T20:24:02.256677+0000","last_active":"2023-06-03T20:30:07.283762+0000","last_peered":"2023-06-03T20:30:07.283762+0000","last_clean":"2023-06-03T20:30:07.283762+0000","last_became_active":"2023-06-03T20:24:02.256216+0000","last_became_peered":"2023-06-03T20:24:02.256216+0000","last_unstale":"2023-06-03T20:30:07.283762+0000","last_undegraded":"2023-06-03T20:30:07.283762+0000","last_fullsized":"2023-06-03T20:30:07.283762+0000","mapping_epoch":254,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":255,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T02:32:45.186204+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":71,"reported_epoch":291,"state":"active+clean","last_fresh":"2023-06-03T20:30:07.285682+0000","last_change":"2023-06-03T20:21:01.261473+0000","last_active":"2023-06-03T20:30:07.285682+0000","last_peered":"2023-06-03T20:30:07.285682+0000","last_clean":"2023-06-03T20:30:07.285682+0000","last_became_active":"2023-06-03T20:21:01.261098+0000","last_became_peered":"2023-06-03T20:21:01.261098+0000","last_unstale":"2023-06-03T20:30:07.285682+0000","last_undegraded":"2023-06-03T20:30:07.285682+0000","last_fullsized":"2023-06-03T20:30:07.285682+0000","mapping_epoch":235,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":236,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T03:56:22.986549+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"14.f","version":"292'9402","reported_seq":9424,"reported_epoch":292,"state":"active+clean","last_fresh":"2023-06-03T20:30:43.114058+0000","last_change":"2023-06-03T20:27:21.374763+0000","last_active":"2023-06-03T20:30:43.114058+0000","last_peered":"2023-06-03T20:30:43.114058+0000","last_clean":"2023-06-03T20:30:43.114058+0000","last_became_active":"2023-06-03T20:27:21.374440+0000","last_became_peered":"2023-06-03T20:27:21.374440+0000","last_unstale":"2023-06-03T20:30:43.114058+0000","last_undegraded":"2023-06-03T20:30:43.114058+0000","last_fullsized":"2023-06-03T20:30:43.114058+0000","mapping_epoch":278,"log_start":"281'300","ondisk_log_start":"281'300","created":278,"last_epoch_clean":279,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T20:27:20.339366+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T20:27:20.339366+0000","last_clean_scrub_stamp":"2023-06-03T20:27:20.339366+0000","objects_scrubbed":0,"log_size":9102,"log_dups_size":0,"ondisk_log_size":9102,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":181403648,"num_objects":2768,"num_object_clones":0,"num_object_copies":8304,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2768,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9402,"num_write_kb":389440,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,1],"acting":[7,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":163,"reported_epoch":291,"state":"active+clean","last_fresh":"2023-06-03T20:30:07.293992+0000","last_change":"2023-06-03T20:17:22.834467+0000","last_active":"2023-06-03T20:30:07.293992+0000","last_peered":"2023-06-03T20:30:07.293992+0000","last_clean":"2023-06-03T20:30:07.293992+0000","last_became_active":"2023-06-03T20:17:22.834147+0000","last_became_peered":"2023-06-03T20:17:22.834147+0000","last_unstale":"2023-06-03T20:30:07.293992+0000","last_undegraded":"2023-06-03T20:30:07.293992+0000","last_fullsized":"2023-06-03T20:30:07.293992+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:32:20.692043+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:32:20.692043+0000","last_clean_scrub_stamp":"2023-06-03T19:32:20.692043+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T04:36:29.567051+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,7],"acting":[2,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"72'139","reported_seq":564,"reported_epoch":291,"state":"active+clean","last_fresh":"2023-06-03T20:30:07.283134+0000","last_change":"2023-06-03T20:27:01.248641+0000","last_active":"2023-06-03T20:30:07.283134+0000","last_peered":"2023-06-03T20:30:07.283134+0000","last_clean":"2023-06-03T20:30:07.283134+0000","last_became_active":"2023-06-03T20:27:01.248063+0000","last_became_peered":"2023-06-03T20:27:01.248063+0000","last_unstale":"2023-06-03T20:30:07.283134+0000","last_undegraded":"2023-06-03T20:30:07.283134+0000","last_fullsized":"2023-06-03T20:30:07.283134+0000","mapping_epoch":274,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":275,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:27:28.100808+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:27:28.100808+0000","last_clean_scrub_stamp":"2023-06-03T19:27:28.100808+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":281,"reported_epoch":291,"state":"active+clean","last_fresh":"2023-06-03T20:30:12.858530+0000","last_change":"2023-06-03T20:24:02.263360+0000","last_active":"2023-06-03T20:30:12.858530+0000","last_peered":"2023-06-03T20:30:12.858530+0000","last_clean":"2023-06-03T20:30:12.858530+0000","last_became_active":"2023-06-03T20:24:02.262954+0000","last_became_peered":"2023-06-03T20:24:02.262954+0000","last_unstale":"2023-06-03T20:30:12.858530+0000","last_undegraded":"2023-06-03T20:30:12.858530+0000","last_fullsized":"2023-06-03T20:30:12.858530+0000","mapping_epoch":254,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":255,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T19:38:46.084731+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":15,"reported_epoch":291,"state":"active+clean","last_fresh":"2023-06-03T20:30:07.275411+0000","last_change":"2023-06-03T20:28:51.234600+0000","last_active":"2023-06-03T20:30:07.275411+0000","last_peered":"2023-06-03T20:30:07.275411+0000","last_clean":"2023-06-03T20:30:07.275411+0000","last_became_active":"2023-06-03T20:28:51.234227+0000","last_became_peered":"2023-06-03T20:28:51.234227+0000","last_unstale":"2023-06-03T20:30:07.275411+0000","last_undegraded":"2023-06-03T20:30:07.275411+0000","last_fullsized":"2023-06-03T20:30:07.275411+0000","mapping_epoch":286,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":287,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"14.c","version":"292'9578","reported_seq":9602,"reported_epoch":292,"state":"active+clean","last_fresh":"2023-06-03T20:30:43.293433+0000","last_change":"2023-06-03T20:27:21.375660+0000","last_active":"2023-06-03T20:30:43.293433+0000","last_peered":"2023-06-03T20:30:43.293433+0000","last_clean":"2023-06-03T20:30:43.293433+0000","last_became_active":"2023-06-03T20:27:21.375311+0000","last_became_peered":"2023-06-03T20:27:21.375311+0000","last_unstale":"2023-06-03T20:30:43.293433+0000","last_undegraded":"2023-06-03T20:30:43.293433+0000","last_fullsized":"2023-06-03T20:30:43.293433+0000","mapping_epoch":278,"log_start":"281'700","ondisk_log_start":"281'700","created":278,"last_epoch_clean":279,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T20:27:20.339366+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T20:27:20.339366+0000","last_clean_scrub_stamp":"2023-06-03T20:27:20.339366+0000","objects_scrubbed":0,"log_size":8878,"log_dups_size":0,"ondisk_log_size":8878,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T04:43:56.400215+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":173801472,"num_objects":2652,"num_object_clones":0,"num_object_copies":7956,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2652,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9578,"num_write_kb":391360,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,2],"acting":[0,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":284,"reported_epoch":291,"state":"active+clean","last_fresh":"2023-06-03T20:30:07.283105+0000","last_change":"2023-06-03T20:21:01.258113+0000","last_active":"2023-06-03T20:30:07.283105+0000","last_peered":"2023-06-03T20:30:07.283105+0000","last_clean":"2023-06-03T20:30:07.283105+0000","last_became_active":"2023-06-03T20:21:01.257488+0000","last_became_peered":"2023-06-03T20:21:01.257488+0000","last_unstale":"2023-06-03T20:30:07.283105+0000","last_undegraded":"2023-06-03T20:30:07.283105+0000","last_fullsized":"2023-06-03T20:30:07.283105+0000","mapping_epoch":235,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":236,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:32:20.692043+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:32:20.692043+0000","last_clean_scrub_stamp":"2023-06-03T19:32:20.692043+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":290,"reported_epoch":291,"state":"active+clean","last_fresh":"2023-06-03T20:30:07.270484+0000","last_change":"2023-06-03T20:23:27.255164+0000","last_active":"2023-06-03T20:30:07.270484+0000","last_peered":"2023-06-03T20:30:07.270484+0000","last_clean":"2023-06-03T20:30:07.270484+0000","last_became_active":"2023-06-03T20:23:27.253932+0000","last_became_peered":"2023-06-03T20:23:27.253932+0000","last_unstale":"2023-06-03T20:30:07.270484+0000","last_undegraded":"2023-06-03T20:30:07.270484+0000","last_fullsized":"2023-06-03T20:30:07.270484+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T06:42:02.303554+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":294,"reported_epoch":291,"state":"active+clean","last_fresh":"2023-06-03T20:30:28.218972+0000","last_change":"2023-06-03T20:21:01.250266+0000","last_active":"2023-06-03T20:30:28.218972+0000","last_peered":"2023-06-03T20:30:28.218972+0000","last_clean":"2023-06-03T20:30:28.218972+0000","last_became_active":"2023-06-03T20:21:01.249839+0000","last_became_peered":"2023-06-03T20:21:01.249839+0000","last_unstale":"2023-06-03T20:30:28.218972+0000","last_undegraded":"2023-06-03T20:30:28.218972+0000","last_fullsized":"2023-06-03T20:30:28.218972+0000","mapping_epoch":235,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":236,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T05:13:26.584607+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"14.9","version":"292'9499","reported_seq":9523,"reported_epoch":292,"state":"active+clean","last_fresh":"2023-06-03T20:30:42.882340+0000","last_change":"2023-06-03T20:27:21.372404+0000","last_active":"2023-06-03T20:30:42.882340+0000","last_peered":"2023-06-03T20:30:42.882340+0000","last_clean":"2023-06-03T20:30:42.882340+0000","last_became_active":"2023-06-03T20:27:21.371829+0000","last_became_peered":"2023-06-03T20:27:21.371829+0000","last_unstale":"2023-06-03T20:30:42.882340+0000","last_undegraded":"2023-06-03T20:30:42.882340+0000","last_fullsized":"2023-06-03T20:30:42.882340+0000","mapping_epoch":278,"log_start":"281'1300","ondisk_log_start":"281'1300","created":278,"last_epoch_clean":279,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T20:27:20.339366+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T20:27:20.339366+0000","last_clean_scrub_stamp":"2023-06-03T20:27:20.339366+0000","objects_scrubbed":0,"log_size":8199,"log_dups_size":0,"ondisk_log_size":8199,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T02:20:57.246181+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":179765248,"num_objects":2743,"num_object_clones":0,"num_object_copies":8229,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2743,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9499,"num_write_kb":391744,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,4],"acting":[3,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":293,"reported_epoch":291,"state":"active+clean","last_fresh":"2023-06-03T20:30:07.284148+0000","last_change":"2023-06-03T20:23:27.261511+0000","last_active":"2023-06-03T20:30:07.284148+0000","last_peered":"2023-06-03T20:30:07.284148+0000","last_clean":"2023-06-03T20:30:07.284148+0000","last_became_active":"2023-06-03T20:23:27.259622+0000","last_became_peered":"2023-06-03T20:23:27.259622+0000","last_unstale":"2023-06-03T20:30:07.284148+0000","last_undegraded":"2023-06-03T20:30:07.284148+0000","last_fullsized":"2023-06-03T20:30:07.284148+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:32:20.692043+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:32:20.692043+0000","last_clean_scrub_stamp":"2023-06-03T19:32:20.692043+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":71,"reported_epoch":291,"state":"active+clean","last_fresh":"2023-06-03T20:30:07.285408+0000","last_change":"2023-06-03T20:21:01.252859+0000","last_active":"2023-06-03T20:30:07.285408+0000","last_peered":"2023-06-03T20:30:07.285408+0000","last_clean":"2023-06-03T20:30:07.285408+0000","last_became_active":"2023-06-03T20:21:01.251222+0000","last_became_peered":"2023-06-03T20:21:01.251222+0000","last_unstale":"2023-06-03T20:30:07.285408+0000","last_undegraded":"2023-06-03T20:30:07.285408+0000","last_fullsized":"2023-06-03T20:30:07.285408+0000","mapping_epoch":235,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":236,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T05:19:58.100226+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":246,"reported_epoch":291,"state":"active+clean","last_fresh":"2023-06-03T20:30:07.276028+0000","last_change":"2023-06-03T19:47:06.733532+0000","last_active":"2023-06-03T20:30:07.276028+0000","last_peered":"2023-06-03T20:30:07.276028+0000","last_clean":"2023-06-03T20:30:07.276028+0000","last_became_active":"2023-06-03T19:47:06.733131+0000","last_became_peered":"2023-06-03T19:47:06.733131+0000","last_unstale":"2023-06-03T20:30:07.276028+0000","last_undegraded":"2023-06-03T20:30:07.276028+0000","last_fullsized":"2023-06-03T20:30:07.276028+0000","mapping_epoch":86,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":87,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T23:46:03.788646+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"14.a","version":"292'9387","reported_seq":9411,"reported_epoch":292,"state":"active+clean","last_fresh":"2023-06-03T20:30:42.788875+0000","last_change":"2023-06-03T20:27:21.378317+0000","last_active":"2023-06-03T20:30:42.788875+0000","last_peered":"2023-06-03T20:30:42.788875+0000","last_clean":"2023-06-03T20:30:42.788875+0000","last_became_active":"2023-06-03T20:27:21.377947+0000","last_became_peered":"2023-06-03T20:27:21.377947+0000","last_unstale":"2023-06-03T20:30:42.788875+0000","last_undegraded":"2023-06-03T20:30:42.788875+0000","last_fullsized":"2023-06-03T20:30:42.788875+0000","mapping_epoch":278,"log_start":"281'1800","ondisk_log_start":"281'1800","created":278,"last_epoch_clean":279,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T20:27:20.339366+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T20:27:20.339366+0000","last_clean_scrub_stamp":"2023-06-03T20:27:20.339366+0000","objects_scrubbed":0,"log_size":7587,"log_dups_size":0,"ondisk_log_size":7587,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T23:39:02.595450+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":176750592,"num_objects":2697,"num_object_clones":0,"num_object_copies":8091,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2697,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9387,"num_write_kb":386688,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,3],"acting":[1,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.7","version":"60'2","reported_seq":256,"reported_epoch":291,"state":"active+clean","last_fresh":"2023-06-03T20:30:07.265600+0000","last_change":"2023-06-03T19:56:54.510148+0000","last_active":"2023-06-03T20:30:07.265600+0000","last_peered":"2023-06-03T20:30:07.265600+0000","last_clean":"2023-06-03T20:30:07.265600+0000","last_became_active":"2023-06-03T19:56:54.509771+0000","last_became_peered":"2023-06-03T19:56:54.509771+0000","last_unstale":"2023-06-03T20:30:07.265600+0000","last_undegraded":"2023-06-03T20:30:07.265600+0000","last_fullsized":"2023-06-03T20:30:07.265600+0000","mapping_epoch":119,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T00:52:42.807432+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":89,"reported_epoch":291,"state":"active+clean","last_fresh":"2023-06-03T20:30:07.285916+0000","last_change":"2023-06-03T20:24:02.257142+0000","last_active":"2023-06-03T20:30:07.285916+0000","last_peered":"2023-06-03T20:30:07.285916+0000","last_clean":"2023-06-03T20:30:07.285916+0000","last_became_active":"2023-06-03T20:24:02.256483+0000","last_became_peered":"2023-06-03T20:24:02.256483+0000","last_unstale":"2023-06-03T20:30:07.285916+0000","last_undegraded":"2023-06-03T20:30:07.285916+0000","last_fullsized":"2023-06-03T20:30:07.285916+0000","mapping_epoch":254,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":255,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:32:20.692043+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:32:20.692043+0000","last_clean_scrub_stamp":"2023-06-03T19:32:20.692043+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T00:53:37.197795+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":254,"reported_epoch":291,"state":"active+clean","last_fresh":"2023-06-03T20:30:07.264745+0000","last_change":"2023-06-03T19:56:54.507670+0000","last_active":"2023-06-03T20:30:07.264745+0000","last_peered":"2023-06-03T20:30:07.264745+0000","last_clean":"2023-06-03T20:30:07.264745+0000","last_became_active":"2023-06-03T19:56:54.507293+0000","last_became_peered":"2023-06-03T19:56:54.507293+0000","last_unstale":"2023-06-03T20:30:07.264745+0000","last_undegraded":"2023-06-03T20:30:07.264745+0000","last_fullsized":"2023-06-03T20:30:07.264745+0000","mapping_epoch":119,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T01:49:19.966054+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"14.b","version":"292'9306","reported_seq":9330,"reported_epoch":292,"state":"active+clean","last_fresh":"2023-06-03T20:30:43.185370+0000","last_change":"2023-06-03T20:27:21.375306+0000","last_active":"2023-06-03T20:30:43.185370+0000","last_peered":"2023-06-03T20:30:43.185370+0000","last_clean":"2023-06-03T20:30:43.185370+0000","last_became_active":"2023-06-03T20:27:21.374975+0000","last_became_peered":"2023-06-03T20:27:21.374975+0000","last_unstale":"2023-06-03T20:30:43.185370+0000","last_undegraded":"2023-06-03T20:30:43.185370+0000","last_fullsized":"2023-06-03T20:30:43.185370+0000","mapping_epoch":278,"log_start":"281'400","ondisk_log_start":"281'400","created":278,"last_epoch_clean":279,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T20:27:20.339366+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T20:27:20.339366+0000","last_clean_scrub_stamp":"2023-06-03T20:27:20.339366+0000","objects_scrubbed":0,"log_size":8906,"log_dups_size":0,"ondisk_log_size":8906,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T02:22:27.778843+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":174850048,"num_objects":2668,"num_object_clones":0,"num_object_copies":8004,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2668,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9306,"num_write_kb":383168,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,1],"acting":[0,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":47,"reported_epoch":291,"state":"active+clean","last_fresh":"2023-06-03T20:30:07.275114+0000","last_change":"2023-06-03T20:24:02.319230+0000","last_active":"2023-06-03T20:30:07.275114+0000","last_peered":"2023-06-03T20:30:07.275114+0000","last_clean":"2023-06-03T20:30:07.275114+0000","last_became_active":"2023-06-03T20:24:02.318926+0000","last_became_peered":"2023-06-03T20:24:02.318926+0000","last_unstale":"2023-06-03T20:30:07.275114+0000","last_undegraded":"2023-06-03T20:30:07.275114+0000","last_fullsized":"2023-06-03T20:30:07.275114+0000","mapping_epoch":254,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":255,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:32:20.692043+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:32:20.692043+0000","last_clean_scrub_stamp":"2023-06-03T19:32:20.692043+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"60'1","reported_seq":322,"reported_epoch":291,"state":"active+clean","last_fresh":"2023-06-03T20:30:28.218936+0000","last_change":"2023-06-03T20:23:28.239664+0000","last_active":"2023-06-03T20:30:28.218936+0000","last_peered":"2023-06-03T20:30:28.218936+0000","last_clean":"2023-06-03T20:30:28.218936+0000","last_became_active":"2023-06-03T20:23:28.239252+0000","last_became_peered":"2023-06-03T20:23:28.239252+0000","last_unstale":"2023-06-03T20:30:28.218936+0000","last_undegraded":"2023-06-03T20:30:28.218936+0000","last_fullsized":"2023-06-03T20:30:28.218936+0000","mapping_epoch":251,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":252,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T06:52:20.862520+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":175,"reported_epoch":291,"state":"active+clean","last_fresh":"2023-06-03T20:30:07.280652+0000","last_change":"2023-06-03T20:24:02.259850+0000","last_active":"2023-06-03T20:30:07.280652+0000","last_peered":"2023-06-03T20:30:07.280652+0000","last_clean":"2023-06-03T20:30:07.280652+0000","last_became_active":"2023-06-03T20:24:02.258764+0000","last_became_peered":"2023-06-03T20:24:02.258764+0000","last_unstale":"2023-06-03T20:30:07.280652+0000","last_undegraded":"2023-06-03T20:30:07.280652+0000","last_fullsized":"2023-06-03T20:30:07.280652+0000","mapping_epoch":254,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":255,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T21:54:11.487473+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"60'1","reported_seq":276,"reported_epoch":291,"state":"active+clean","last_fresh":"2023-06-03T20:30:07.275958+0000","last_change":"2023-06-03T20:23:27.254853+0000","last_active":"2023-06-03T20:30:07.275958+0000","last_peered":"2023-06-03T20:30:07.275958+0000","last_clean":"2023-06-03T20:30:07.275958+0000","last_became_active":"2023-06-03T20:23:27.254410+0000","last_became_peered":"2023-06-03T20:23:27.254410+0000","last_unstale":"2023-06-03T20:30:07.275958+0000","last_undegraded":"2023-06-03T20:30:07.275958+0000","last_fullsized":"2023-06-03T20:30:07.275958+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T22:46:19.423939+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":305,"reported_epoch":291,"state":"active+clean","last_fresh":"2023-06-03T20:30:07.293782+0000","last_change":"2023-06-03T20:21:01.257656+0000","last_active":"2023-06-03T20:30:07.293782+0000","last_peered":"2023-06-03T20:30:07.293782+0000","last_clean":"2023-06-03T20:30:07.293782+0000","last_became_active":"2023-06-03T20:21:01.257107+0000","last_became_peered":"2023-06-03T20:21:01.257107+0000","last_unstale":"2023-06-03T20:30:07.293782+0000","last_undegraded":"2023-06-03T20:30:07.293782+0000","last_fullsized":"2023-06-03T20:30:07.293782+0000","mapping_epoch":235,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":236,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T06:08:18.954539+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"60'2","reported_seq":66,"reported_epoch":291,"state":"active+clean","last_fresh":"2023-06-03T20:30:07.275999+0000","last_change":"2023-06-03T20:23:28.243048+0000","last_active":"2023-06-03T20:30:07.275999+0000","last_peered":"2023-06-03T20:30:07.275999+0000","last_clean":"2023-06-03T20:30:07.275999+0000","last_became_active":"2023-06-03T20:23:28.241128+0000","last_became_peered":"2023-06-03T20:23:28.241128+0000","last_unstale":"2023-06-03T20:30:07.275999+0000","last_undegraded":"2023-06-03T20:30:07.275999+0000","last_fullsized":"2023-06-03T20:30:07.275999+0000","mapping_epoch":251,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":252,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T05:14:54.701329+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,1],"acting":[4,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":229,"reported_epoch":291,"state":"active+clean","last_fresh":"2023-06-03T20:30:07.276732+0000","last_change":"2023-06-03T20:28:24.276513+0000","last_active":"2023-06-03T20:30:07.276732+0000","last_peered":"2023-06-03T20:30:07.276732+0000","last_clean":"2023-06-03T20:30:07.276732+0000","last_became_active":"2023-06-03T20:28:24.276244+0000","last_became_peered":"2023-06-03T20:28:24.276244+0000","last_unstale":"2023-06-03T20:30:07.276732+0000","last_undegraded":"2023-06-03T20:30:07.276732+0000","last_fullsized":"2023-06-03T20:30:07.276732+0000","mapping_epoch":284,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":285,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T06:17:57.159168+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[4,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"60'2","reported_seq":79,"reported_epoch":291,"state":"active+clean","last_fresh":"2023-06-03T20:30:07.277979+0000","last_change":"2023-06-03T20:28:51.237342+0000","last_active":"2023-06-03T20:30:07.277979+0000","last_peered":"2023-06-03T20:30:07.277979+0000","last_clean":"2023-06-03T20:30:07.277979+0000","last_became_active":"2023-06-03T20:28:51.233969+0000","last_became_peered":"2023-06-03T20:28:51.233969+0000","last_unstale":"2023-06-03T20:30:07.277979+0000","last_undegraded":"2023-06-03T20:30:07.277979+0000","last_fullsized":"2023-06-03T20:30:07.277979+0000","mapping_epoch":286,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":287,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T07:30:09.503545+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":213,"reported_epoch":291,"state":"active+clean","last_fresh":"2023-06-03T20:30:07.275873+0000","last_change":"2023-06-03T20:23:27.254468+0000","last_active":"2023-06-03T20:30:07.275873+0000","last_peered":"2023-06-03T20:30:07.275873+0000","last_clean":"2023-06-03T20:30:07.275873+0000","last_became_active":"2023-06-03T20:23:27.253859+0000","last_became_peered":"2023-06-03T20:23:27.253859+0000","last_unstale":"2023-06-03T20:30:07.275873+0000","last_undegraded":"2023-06-03T20:30:07.275873+0000","last_fullsized":"2023-06-03T20:30:07.275873+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T20:15:25.066125+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"60'1","reported_seq":310,"reported_epoch":291,"state":"active+clean","last_fresh":"2023-06-03T20:30:07.265265+0000","last_change":"2023-06-03T20:28:51.234748+0000","last_active":"2023-06-03T20:30:07.265265+0000","last_peered":"2023-06-03T20:30:07.265265+0000","last_clean":"2023-06-03T20:30:07.265265+0000","last_became_active":"2023-06-03T20:28:51.234341+0000","last_became_peered":"2023-06-03T20:28:51.234341+0000","last_unstale":"2023-06-03T20:30:07.265265+0000","last_undegraded":"2023-06-03T20:30:07.265265+0000","last_fullsized":"2023-06-03T20:30:07.265265+0000","mapping_epoch":286,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":287,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T06:08:49.901424+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":293,"reported_epoch":291,"state":"active+clean","last_fresh":"2023-06-03T20:30:07.276731+0000","last_change":"2023-06-03T19:56:54.503416+0000","last_active":"2023-06-03T20:30:07.276731+0000","last_peered":"2023-06-03T20:30:07.276731+0000","last_clean":"2023-06-03T20:30:07.276731+0000","last_became_active":"2023-06-03T19:56:54.503018+0000","last_became_peered":"2023-06-03T19:56:54.503018+0000","last_unstale":"2023-06-03T20:30:07.276731+0000","last_undegraded":"2023-06-03T20:30:07.276731+0000","last_fullsized":"2023-06-03T20:30:07.276731+0000","mapping_epoch":119,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T20:14:00.330616+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"60'4","reported_seq":264,"reported_epoch":291,"state":"active+clean","last_fresh":"2023-06-03T20:30:07.285671+0000","last_change":"2023-06-03T19:56:11.748116+0000","last_active":"2023-06-03T20:30:07.285671+0000","last_peered":"2023-06-03T20:30:07.285671+0000","last_clean":"2023-06-03T20:30:07.285671+0000","last_became_active":"2023-06-03T19:56:11.747636+0000","last_became_peered":"2023-06-03T19:56:11.747636+0000","last_unstale":"2023-06-03T20:30:07.285671+0000","last_undegraded":"2023-06-03T20:30:07.285671+0000","last_fullsized":"2023-06-03T20:30:07.285671+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":115,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T06:57:48.052055+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":213,"reported_epoch":291,"state":"active+clean","last_fresh":"2023-06-03T20:30:07.275802+0000","last_change":"2023-06-03T20:28:51.231879+0000","last_active":"2023-06-03T20:30:07.275802+0000","last_peered":"2023-06-03T20:30:07.275802+0000","last_clean":"2023-06-03T20:30:07.275802+0000","last_became_active":"2023-06-03T20:28:51.231451+0000","last_became_peered":"2023-06-03T20:28:51.231451+0000","last_unstale":"2023-06-03T20:30:07.275802+0000","last_undegraded":"2023-06-03T20:30:07.275802+0000","last_fullsized":"2023-06-03T20:30:07.275802+0000","mapping_epoch":286,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":287,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T05:47:15.344575+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":14,"num_pg":16,"stat_sum":{"num_bytes":2860974104,"num_objects":43656,"num_object_clones":0,"num_object_copies":130968,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":2415,"num_objects_unfound":0,"num_objects_dirty":43656,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":152372,"num_write_kb":6272833,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2651,"num_bytes_recovered":173735936,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":8667938816,"data_stored":8667922528,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139672,"ondisk_log_size":139672,"up":48,"acting":48,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5486,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":62,"num_write_kb":30,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":23,"num_bytes_recovered":4856,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16458,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":30319,"internal_metadata":0},"log_size":56,"ondisk_log_size":56,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2969600,"data_stored":2951840,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":3,"num_store_stats":5},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":48,"seq":206158430924,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1497804,"kb_used_data":532484,"kb_used_omap":0,"kb_used_meta":965312,"kb_avail":92247348,"statfs":{"total":95995035648,"available":94461284352,"internally_reserved":0,"allocated":545263616,"data_stored":544601358,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":988479488},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 3 20:30:33 2023","interfaces":[{"interface":"back","average":{"1min":1.502,"5min":1.573,"15min":1.603},"min":{"1min":0.28999999999999998,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":4.3840000000000003,"5min":5.5540000000000003,"15min":73.810000000000002},"last":3.1190000000000002},{"interface":"front","average":{"1min":2.3919999999999999,"5min":1.3220000000000001,"15min":1.534},"min":{"1min":0.32100000000000001,"5min":0.29999999999999999,"15min":0.26700000000000002},"max":{"1min":18.901,"5min":18.901,"15min":72.878},"last":3.3650000000000002}]},{"osd":1,"last update":"Sat Jun 3 20:30:14 2023","interfaces":[{"interface":"back","average":{"1min":0.872,"5min":1.264,"15min":1.5640000000000001},"min":{"1min":0.27000000000000002,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":2.9239999999999999,"5min":5.5999999999999996,"15min":100.845},"last":2.9359999999999999},{"interface":"front","average":{"1min":1.4950000000000001,"5min":1.3400000000000001,"15min":1.6539999999999999},"min":{"1min":0.30099999999999999,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":4.173,"5min":6.048,"15min":100.86499999999999},"last":3.0459999999999998}]},{"osd":2,"last update":"Sat Jun 3 20:30:14 2023","interfaces":[{"interface":"back","average":{"1min":0.874,"5min":1.8959999999999999,"15min":3.1339999999999999},"min":{"1min":0.318,"5min":0.318,"15min":0.29499999999999998},"max":{"1min":2.5979999999999999,"5min":36.250999999999998,"15min":216.93700000000001},"last":2.5489999999999999},{"interface":"front","average":{"1min":1.639,"5min":1.966,"15min":3.25},"min":{"1min":0.33200000000000002,"5min":0.32400000000000001,"15min":0.315},"max":{"1min":8.1500000000000004,"5min":35.563000000000002,"15min":218.119},"last":3.3039999999999998}]},{"osd":3,"last update":"Sat Jun 3 20:30:14 2023","interfaces":[{"interface":"back","average":{"1min":1.0529999999999999,"5min":1.431,"15min":1.929},"min":{"1min":0.379,"5min":0.372,"15min":0.371},"max":{"1min":4.1100000000000003,"5min":6.9269999999999996,"15min":74.039000000000001},"last":0.55600000000000005},{"interface":"front","average":{"1min":1.5600000000000001,"5min":1.5109999999999999,"15min":1.9139999999999999},"min":{"1min":0.433,"5min":0.38200000000000001,"15min":0.35399999999999998},"max":{"1min":4.4649999999999999,"5min":11.656000000000001,"15min":73.001999999999995},"last":2.9660000000000002}]},{"osd":4,"last update":"Sat Jun 3 20:30:38 2023","interfaces":[{"interface":"back","average":{"1min":1.6060000000000001,"5min":1.3520000000000001,"15min":2.0430000000000001},"min":{"1min":0.30399999999999999,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":7.718,"5min":7.8419999999999996,"15min":8.6010000000000009},"last":0.61899999999999999},{"interface":"front","average":{"1min":1.3759999999999999,"5min":1.4430000000000001,"15min":1.859},"min":{"1min":0.35599999999999998,"5min":0.315,"15min":0.27600000000000002},"max":{"1min":8.9459999999999997,"5min":9.8460000000000001,"15min":9.8460000000000001},"last":0.74399999999999999}]},{"osd":5,"last update":"Sat Jun 3 20:30:14 2023","interfaces":[{"interface":"back","average":{"1min":0.80700000000000005,"5min":1.601,"15min":1.2410000000000001},"min":{"1min":0.40999999999999998,"5min":0.317,"15min":0.317},"max":{"1min":3.702,"5min":17.696000000000002,"15min":17.696000000000002},"last":0.72399999999999998},{"interface":"front","average":{"1min":0.72499999999999998,"5min":1.325,"15min":1.254},"min":{"1min":0.41199999999999998,"5min":0.33600000000000002,"15min":0.29399999999999998},"max":{"1min":1.524,"5min":7.3620000000000001,"15min":9.3710000000000004},"last":0.89900000000000002}]},{"osd":6,"last update":"Sat Jun 3 20:30:14 2023","interfaces":[{"interface":"back","average":{"1min":1.119,"5min":1.244,"15min":1.1499999999999999},"min":{"1min":0.44,"5min":0.36399999999999999,"15min":0.34799999999999998},"max":{"1min":5.3040000000000003,"5min":7.5250000000000004,"15min":9.7200000000000006},"last":1.1779999999999999},{"interface":"front","average":{"1min":1.069,"5min":1.496,"15min":1.115},"min":{"1min":0.39800000000000002,"5min":0.36799999999999999,"15min":0.34300000000000003},"max":{"1min":5.7380000000000004,"5min":9.923,"15min":9.923},"last":1.0600000000000001}]}]},{"osd":6,"up_from":42,"seq":180388627157,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1195008,"kb_used_data":597688,"kb_used_omap":0,"kb_used_meta":597312,"kb_avail":92550144,"statfs":{"total":95995035648,"available":94771347456,"internally_reserved":0,"allocated":612032512,"data_stored":611382105,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":611647488},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 3 20:29:45 2023","interfaces":[{"interface":"back","average":{"1min":3.266,"5min":1.534,"15min":1.3859999999999999},"min":{"1min":0.30599999999999999,"5min":0.254,"15min":0.254},"max":{"1min":22.181999999999999,"5min":22.181999999999999,"15min":56.259},"last":0.65100000000000002},{"interface":"front","average":{"1min":4.032,"5min":1.827,"15min":1.5529999999999999},"min":{"1min":0.34999999999999998,"5min":0.315,"15min":0.28299999999999997},"max":{"1min":22.166,"5min":22.166,"15min":56.234000000000002},"last":0.67000000000000004}]},{"osd":1,"last update":"Sat Jun 3 20:30:19 2023","interfaces":[{"interface":"back","average":{"1min":1.3640000000000001,"5min":1.129,"15min":1.0620000000000001},"min":{"1min":0.32900000000000001,"5min":0.30299999999999999,"15min":0.29199999999999998},"max":{"1min":5.5679999999999996,"5min":6.7910000000000004,"15min":30.736000000000001},"last":3.371},{"interface":"front","average":{"1min":1.911,"5min":1.6479999999999999,"15min":1.2210000000000001},"min":{"1min":0.379,"5min":0.34200000000000003,"15min":0.32700000000000001},"max":{"1min":6.2380000000000004,"5min":8.5869999999999997,"15min":30.788},"last":3.3580000000000001}]},{"osd":2,"last update":"Sat Jun 3 20:30:19 2023","interfaces":[{"interface":"back","average":{"1min":3.0529999999999999,"5min":1.5940000000000001,"15min":2.331},"min":{"1min":0.34899999999999998,"5min":0.34300000000000003,"15min":0.315},"max":{"1min":32.881,"5min":32.881,"15min":93.558999999999997},"last":0.69299999999999995},{"interface":"front","average":{"1min":2.7829999999999999,"5min":1.498,"15min":2.242},"min":{"1min":0.373,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":33.103000000000002,"5min":33.103000000000002,"15min":94.888000000000005},"last":0.77200000000000002}]},{"osd":3,"last update":"Sat Jun 3 20:30:01 2023","interfaces":[{"interface":"back","average":{"1min":2.633,"5min":1.5760000000000001,"15min":1.7330000000000001},"min":{"1min":0.36299999999999999,"5min":0.36299999999999999,"15min":0.34899999999999998},"max":{"1min":30.777000000000001,"5min":30.777000000000001,"15min":58.615000000000002},"last":1.6140000000000001},{"interface":"front","average":{"1min":3.9279999999999999,"5min":1.5780000000000001,"15min":1.8109999999999999},"min":{"1min":0.41399999999999998,"5min":0.40100000000000002,"15min":0.32500000000000001},"max":{"1min":30.074999999999999,"5min":30.074999999999999,"15min":58.546999999999997},"last":1.472}]},{"osd":4,"last update":"Sat Jun 3 20:30:40 2023","interfaces":[{"interface":"back","average":{"1min":1.1699999999999999,"5min":1.6830000000000001,"15min":1.77},"min":{"1min":0.33000000000000002,"5min":0.33000000000000002,"15min":0.33000000000000002},"max":{"1min":6.6980000000000004,"5min":10.574999999999999,"15min":10.574999999999999},"last":0.80800000000000005},{"interface":"front","average":{"1min":1.0940000000000001,"5min":1.3029999999999999,"15min":1.353},"min":{"1min":0.432,"5min":0.35799999999999998,"15min":0.34499999999999997},"max":{"1min":6.6710000000000003,"5min":12.622999999999999,"15min":12.622999999999999},"last":6.5590000000000002}]},{"osd":5,"last update":"Sat Jun 3 20:30:31 2023","interfaces":[{"interface":"back","average":{"1min":1.3100000000000001,"5min":1.167,"15min":1.056},"min":{"1min":0.36299999999999999,"5min":0.36299999999999999,"15min":0.29899999999999999},"max":{"1min":7.5830000000000002,"5min":7.5830000000000002,"15min":10.625999999999999},"last":0.82999999999999996},{"interface":"front","average":{"1min":1.3,"5min":1.722,"15min":1.2589999999999999},"min":{"1min":0.40999999999999998,"5min":0.40999999999999998,"15min":0.32200000000000001},"max":{"1min":6.7320000000000002,"5min":8.2880000000000003,"15min":10.856},"last":0.78400000000000003}]},{"osd":7,"last update":"Sat Jun 3 20:30:06 2023","interfaces":[{"interface":"back","average":{"1min":1.0489999999999999,"5min":1.292,"15min":1.1739999999999999},"min":{"1min":0.41299999999999998,"5min":0.41299999999999998,"15min":0.315},"max":{"1min":6.6189999999999998,"5min":7.069,"15min":7.069},"last":0.85299999999999998},{"interface":"front","average":{"1min":1.161,"5min":1.361,"15min":1.1879999999999999},"min":{"1min":0.45900000000000002,"5min":0.42599999999999999,"15min":0.38},"max":{"1min":6.6280000000000001,"5min":7.9630000000000001,"15min":10.52},"last":0.74099999999999999}]}]},{"osd":1,"up_from":103,"seq":442381631955,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1882180,"kb_used_data":1413116,"kb_used_omap":4,"kb_used_meta":469051,"kb_avail":91862972,"statfs":{"total":95995035648,"available":94067683328,"internally_reserved":0,"allocated":1447030784,"data_stored":1445910798,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":480309134},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 3 20:30:24 2023","interfaces":[{"interface":"back","average":{"1min":12.113,"5min":4.5309999999999997,"15min":3.2160000000000002},"min":{"1min":0.26900000000000002,"5min":0.19800000000000001,"15min":0.17100000000000001},"max":{"1min":99.870000000000005,"5min":99.870000000000005,"15min":119.94499999999999},"last":2.2570000000000001},{"interface":"front","average":{"1min":12.196999999999999,"5min":4.6740000000000004,"15min":3.2890000000000001},"min":{"1min":0.29699999999999999,"5min":0.182,"15min":0.182},"max":{"1min":99.897999999999996,"5min":99.897999999999996,"15min":120.04300000000001},"last":2.6150000000000002}]},{"osd":2,"last update":"Sat Jun 3 20:30:24 2023","interfaces":[{"interface":"back","average":{"1min":2.5059999999999998,"5min":1.5660000000000001,"15min":2.1890000000000001},"min":{"1min":0.23799999999999999,"5min":0.20599999999999999,"15min":0.20599999999999999},"max":{"1min":11.009,"5min":11.009,"15min":77.719999999999999},"last":2.4990000000000001},{"interface":"front","average":{"1min":2.702,"5min":1.716,"15min":2.2989999999999999},"min":{"1min":0.32100000000000001,"5min":0.246,"15min":0.22700000000000001},"max":{"1min":10.096,"5min":10.096,"15min":78.007999999999996},"last":0.47299999999999998}]},{"osd":3,"last update":"Sat Jun 3 20:29:57 2023","interfaces":[{"interface":"back","average":{"1min":5.6589999999999998,"5min":3.165,"15min":2.3919999999999999},"min":{"1min":0.27800000000000002,"5min":0.255,"15min":0.23400000000000001},"max":{"1min":58.119,"5min":95.525000000000006,"15min":96.864999999999995},"last":0.27300000000000002},{"interface":"front","average":{"1min":6.1429999999999998,"5min":3.29,"15min":2.5289999999999999},"min":{"1min":0.34699999999999998,"5min":0.223,"15min":0.223},"max":{"1min":58.087000000000003,"5min":94.507000000000005,"15min":96.679000000000002},"last":0.44500000000000001}]},{"osd":4,"last update":"Sat Jun 3 20:30:41 2023","interfaces":[{"interface":"back","average":{"1min":2.0670000000000002,"5min":1.802,"15min":1.764},"min":{"1min":0.33100000000000002,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":6.1959999999999997,"5min":11.114000000000001,"15min":11.114000000000001},"last":0.33300000000000002},{"interface":"front","average":{"1min":1.9710000000000001,"5min":1.623,"15min":1.52},"min":{"1min":0.35199999999999998,"5min":0.35199999999999998,"15min":0.33400000000000002},"max":{"1min":6.125,"5min":7.556,"15min":11.247999999999999},"last":2.5630000000000002}]},{"osd":5,"last update":"Sat Jun 3 20:30:24 2023","interfaces":[{"interface":"back","average":{"1min":2.3570000000000002,"5min":1.802,"15min":1.3400000000000001},"min":{"1min":0.35799999999999998,"5min":0.35799999999999998,"15min":0.30299999999999999},"max":{"1min":6.8700000000000001,"5min":11.246,"15min":11.246},"last":0.38200000000000001},{"interface":"front","average":{"1min":2.0030000000000001,"5min":1.643,"15min":1.3100000000000001},"min":{"1min":0.35799999999999998,"5min":0.35799999999999998,"15min":0.35799999999999998},"max":{"1min":7.0890000000000004,"5min":8.7840000000000007,"15min":10.922000000000001},"last":2.589}]},{"osd":6,"last update":"Sat Jun 3 20:30:24 2023","interfaces":[{"interface":"back","average":{"1min":2.2879999999999998,"5min":1.5289999999999999,"15min":1.171},"min":{"1min":0.37,"5min":0.37,"15min":0.36799999999999999},"max":{"1min":7.1159999999999997,"5min":7.6139999999999999,"15min":10.002000000000001},"last":2.5339999999999998},{"interface":"front","average":{"1min":1.889,"5min":1.4350000000000001,"15min":1.2589999999999999},"min":{"1min":0.42099999999999999,"5min":0.375,"15min":0.36899999999999999},"max":{"1min":4.8129999999999997,"5min":6.5300000000000002,"15min":11.067},"last":0.5}]},{"osd":7,"last update":"Sat Jun 3 20:30:24 2023","interfaces":[{"interface":"back","average":{"1min":1.899,"5min":1.5740000000000001,"15min":1.4610000000000001},"min":{"1min":0.44700000000000001,"5min":0.38300000000000001,"15min":0.372},"max":{"1min":4.8479999999999999,"5min":9.1170000000000009,"15min":10.538},"last":0.53900000000000003},{"interface":"front","average":{"1min":2.0720000000000001,"5min":1.627,"15min":1.2649999999999999},"min":{"1min":0.41899999999999998,"5min":0.41599999999999998,"15min":0.34000000000000002},"max":{"1min":6.1989999999999998,"5min":14.025,"15min":14.025},"last":0.40999999999999998}]}]},{"osd":0,"up_from":86,"seq":369367187981,"num_pgs":34,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2144900,"kb_used_data":1396860,"kb_used_omap":5,"kb_used_meta":748026,"kb_avail":91600252,"statfs":{"total":95995035648,"available":93798658048,"internally_reserved":0,"allocated":1430384640,"data_stored":1429207110,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":765979258},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Jun 3 20:30:13 2023","interfaces":[{"interface":"back","average":{"1min":3.9100000000000001,"5min":3.1019999999999999,"15min":2.3969999999999998},"min":{"1min":0.19500000000000001,"5min":0.19500000000000001,"15min":0.185},"max":{"1min":11.707000000000001,"5min":156.43600000000001,"15min":243.03200000000001},"last":4.1559999999999997},{"interface":"front","average":{"1min":3.4790000000000001,"5min":3.6699999999999999,"15min":2.6909999999999998},"min":{"1min":0.23300000000000001,"5min":0.188,"15min":0.188},"max":{"1min":9.593,"5min":157.489,"15min":242.78399999999999},"last":0.745}]},{"osd":2,"last update":"Sat Jun 3 20:30:13 2023","interfaces":[{"interface":"back","average":{"1min":2.5219999999999998,"5min":1.5880000000000001,"15min":1.514},"min":{"1min":0.25600000000000001,"5min":0.219,"15min":0.218},"max":{"1min":8.6799999999999997,"5min":18.425000000000001,"15min":36.515000000000001},"last":0.57099999999999995},{"interface":"front","average":{"1min":3.5590000000000002,"5min":2.048,"15min":2.7250000000000001},"min":{"1min":0.30599999999999999,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":16.687999999999999,"5min":17.466000000000001,"15min":243.30099999999999},"last":0.42299999999999999}]},{"osd":3,"last update":"Sat Jun 3 20:29:52 2023","interfaces":[{"interface":"back","average":{"1min":2.1640000000000001,"5min":4.6680000000000001,"15min":3.7080000000000002},"min":{"1min":0.24399999999999999,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":6.3259999999999996,"5min":259.68400000000003,"15min":259.68400000000003},"last":11.906000000000001},{"interface":"front","average":{"1min":2.0129999999999999,"5min":4.8159999999999998,"15min":3.004},"min":{"1min":0.35799999999999998,"5min":0.27000000000000002,"15min":0.23799999999999999},"max":{"1min":6.5819999999999999,"5min":261.02699999999999,"15min":261.02699999999999},"last":4.2939999999999996}]},{"osd":4,"last update":"Sat Jun 3 20:30:38 2023","interfaces":[{"interface":"back","average":{"1min":2.3580000000000001,"5min":1.8979999999999999,"15min":1.7070000000000001},"min":{"1min":0.34200000000000003,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":7.1980000000000004,"5min":11.49,"15min":11.49},"last":0.77600000000000002},{"interface":"front","average":{"1min":0.94899999999999995,"5min":1.0349999999999999,"15min":1.548},"min":{"1min":0.38,"5min":0.34100000000000003,"15min":0.34100000000000003},"max":{"1min":3.1219999999999999,"5min":6.3600000000000003,"15min":7.21},"last":0.60899999999999999}]},{"osd":5,"last update":"Sat Jun 3 20:30:32 2023","interfaces":[{"interface":"back","average":{"1min":2.052,"5min":2.0539999999999998,"15min":2.3050000000000002},"min":{"1min":0.34599999999999997,"5min":0.33600000000000002,"15min":0.29299999999999998},"max":{"1min":7.2560000000000002,"5min":20.238,"15min":243.15899999999999},"last":0.80200000000000005},{"interface":"front","average":{"1min":1.2709999999999999,"5min":1.278,"15min":2.1040000000000001},"min":{"1min":0.434,"5min":0.40500000000000003,"15min":0.33400000000000002},"max":{"1min":4.5270000000000001,"5min":6.4429999999999996,"15min":243.32599999999999},"last":1.851}]},{"osd":6,"last update":"Sat Jun 3 20:30:32 2023","interfaces":[{"interface":"back","average":{"1min":1.373,"5min":1.0720000000000001,"15min":1.954},"min":{"1min":0.36499999999999999,"5min":0.36499999999999999,"15min":0.36299999999999999},"max":{"1min":7.8869999999999996,"5min":7.8869999999999996,"15min":243.25},"last":0.63900000000000001},{"interface":"front","average":{"1min":1.95,"5min":1.2270000000000001,"15min":1.974},"min":{"1min":0.41399999999999998,"5min":0.41399999999999998,"15min":0.379},"max":{"1min":11.634,"5min":11.634,"15min":242.84399999999999},"last":0.45900000000000002}]},{"osd":7,"last update":"Sat Jun 3 20:30:32 2023","interfaces":[{"interface":"back","average":{"1min":1.2050000000000001,"5min":1.3779999999999999,"15min":2.2320000000000002},"min":{"1min":0.39600000000000002,"5min":0.39600000000000002,"15min":0.35499999999999998},"max":{"1min":4.452,"5min":7.173,"15min":242.93600000000001},"last":0.66400000000000003},{"interface":"front","average":{"1min":1.9079999999999999,"5min":1.421,"15min":2.2069999999999999},"min":{"1min":0.44,"5min":0.44,"15min":0.40000000000000002},"max":{"1min":11.68,"5min":11.68,"15min":243.21299999999999},"last":0.69599999999999995}]}]},{"osd":2,"up_from":114,"seq":489626272160,"num_pgs":28,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2035096,"kb_used_data":1412880,"kb_used_omap":7,"kb_used_meta":622072,"kb_avail":91710056,"statfs":{"total":95995035648,"available":93911097344,"internally_reserved":0,"allocated":1446789120,"data_stored":1445710685,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8111,"internal_metadata":637001809},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 3 20:30:00 2023","interfaces":[{"interface":"back","average":{"1min":1.7729999999999999,"5min":1.976,"15min":2.7410000000000001},"min":{"1min":0.20999999999999999,"5min":0.193,"15min":0.193},"max":{"1min":6.798,"5min":77.055000000000007,"15min":117.57899999999999},"last":0.88300000000000001},{"interface":"front","average":{"1min":2.5600000000000001,"5min":2.2829999999999999,"15min":2.6789999999999998},"min":{"1min":0.20699999999999999,"5min":0.20699999999999999,"15min":0.20399999999999999},"max":{"1min":11.786,"5min":76.453000000000003,"15min":117.38500000000001},"last":1.012}]},{"osd":1,"last update":"Sat Jun 3 20:30:00 2023","interfaces":[{"interface":"back","average":{"1min":2.468,"5min":2.0270000000000001,"15min":1.7989999999999999},"min":{"1min":0.193,"5min":0.193,"15min":0.193},"max":{"1min":11.845000000000001,"5min":33.149999999999999,"15min":116.127},"last":0.36699999999999999},{"interface":"front","average":{"1min":2.141,"5min":1.8460000000000001,"15min":2.1030000000000002},"min":{"1min":0.251,"5min":0.251,"15min":0.251},"max":{"1min":8.2940000000000005,"5min":33.651000000000003,"15min":116.461},"last":0.81799999999999995}]},{"osd":3,"last update":"Sat Jun 3 20:30:00 2023","interfaces":[{"interface":"back","average":{"1min":1.8360000000000001,"5min":1.722,"15min":2.3359999999999999},"min":{"1min":0.309,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":6.452,"5min":36.695,"15min":103.714},"last":0.90600000000000003},{"interface":"front","average":{"1min":2.7160000000000002,"5min":1.8620000000000001,"15min":2.512},"min":{"1min":0.34100000000000003,"5min":0.28599999999999998,"15min":0.23000000000000001},"max":{"1min":7.625,"5min":37.088999999999999,"15min":104.05200000000001},"last":3.5110000000000001}]},{"osd":4,"last update":"Sat Jun 3 20:30:39 2023","interfaces":[{"interface":"back","average":{"1min":2.617,"5min":1.661,"15min":2.7280000000000002},"min":{"1min":0.34799999999999998,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":8.9100000000000001,"5min":10.664999999999999,"15min":16.561},"last":0.78700000000000003},{"interface":"front","average":{"1min":2.7890000000000001,"5min":1.615,"15min":1.847},"min":{"1min":0.55400000000000005,"5min":0.35099999999999998,"15min":0.35099999999999998},"max":{"1min":6.7400000000000002,"5min":6.7400000000000002,"15min":9.9339999999999993},"last":0.92700000000000005}]},{"osd":5,"last update":"Sat Jun 3 20:30:00 2023","interfaces":[{"interface":"back","average":{"1min":1.4390000000000001,"5min":1.5920000000000001,"15min":1.6140000000000001},"min":{"1min":0.45100000000000001,"5min":0.378,"15min":0.32100000000000001},"max":{"1min":4.29,"5min":17.236000000000001,"15min":17.751000000000001},"last":0.95299999999999996},{"interface":"front","average":{"1min":1.732,"5min":1.6240000000000001,"15min":1.4730000000000001},"min":{"1min":0.32500000000000001,"5min":0.316,"15min":0.316},"max":{"1min":7.2549999999999999,"5min":9.2219999999999995,"15min":17.928000000000001},"last":1.0469999999999999}]},{"osd":6,"last update":"Sat Jun 3 20:30:00 2023","interfaces":[{"interface":"back","average":{"1min":1.75,"5min":1.502,"15min":1.4510000000000001},"min":{"1min":0.46600000000000003,"5min":0.377,"15min":0.35199999999999998},"max":{"1min":6.3760000000000003,"5min":6.3760000000000003,"15min":14.331},"last":2.8559999999999999},{"interface":"front","average":{"1min":1.974,"5min":1.419,"15min":1.3420000000000001},"min":{"1min":0.35699999999999998,"5min":0.35699999999999998,"15min":0.35599999999999998},"max":{"1min":9.3010000000000002,"5min":9.3010000000000002,"15min":12.867000000000001},"last":11.943}]},{"osd":7,"last update":"Sat Jun 3 20:30:00 2023","interfaces":[{"interface":"back","average":{"1min":1.6259999999999999,"5min":1.6910000000000001,"15min":1.694},"min":{"1min":0.47999999999999998,"5min":0.44400000000000001,"15min":0.34799999999999998},"max":{"1min":5.282,"5min":10.529999999999999,"15min":14.243},"last":3.4359999999999999},{"interface":"front","average":{"1min":1.5429999999999999,"5min":1.615,"15min":1.5669999999999999},"min":{"1min":0.44700000000000001,"5min":0.44700000000000001,"15min":0.38600000000000001},"max":{"1min":7.3220000000000001,"5min":8.5690000000000008,"15min":12.845000000000001},"last":3.4790000000000001}]}]},{"osd":3,"up_from":119,"seq":511101108632,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1850072,"kb_used_data":1247440,"kb_used_omap":2,"kb_used_meta":602557,"kb_avail":91895080,"statfs":{"total":95995035648,"available":94100561920,"internally_reserved":0,"allocated":1277378560,"data_stored":1276238398,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":617018421},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 3 20:30:03 2023","interfaces":[{"interface":"back","average":{"1min":7.3840000000000003,"5min":3.7730000000000001,"15min":3.0880000000000001},"min":{"1min":0.27300000000000002,"5min":0.19600000000000001,"15min":0.19600000000000001},"max":{"1min":96.629000000000005,"5min":168.33099999999999,"15min":168.33099999999999},"last":0.318},{"interface":"front","average":{"1min":8.2739999999999991,"5min":4.165,"15min":3.2330000000000001},"min":{"1min":0.23200000000000001,"5min":0.222,"15min":0.183},"max":{"1min":98.703000000000003,"5min":169.25800000000001,"15min":169.25800000000001},"last":2.2549999999999999}]},{"osd":1,"last update":"Sat Jun 3 20:30:03 2023","interfaces":[{"interface":"back","average":{"1min":1.7,"5min":1.2949999999999999,"15min":1.5529999999999999},"min":{"1min":0.36699999999999999,"5min":0.22900000000000001,"15min":0.22500000000000001},"max":{"1min":5.5650000000000004,"5min":20.992000000000001,"15min":117.047},"last":0.38700000000000001},{"interface":"front","average":{"1min":1.5660000000000001,"5min":1.4299999999999999,"15min":1.5509999999999999},"min":{"1min":0.25600000000000001,"5min":0.25600000000000001,"15min":0.20399999999999999},"max":{"1min":13.356,"5min":21.07,"15min":116.505},"last":0.35299999999999998}]},{"osd":2,"last update":"Sat Jun 3 20:30:03 2023","interfaces":[{"interface":"back","average":{"1min":2.5920000000000001,"5min":1.4279999999999999,"15min":2.6160000000000001},"min":{"1min":0.255,"5min":0.255,"15min":0.255},"max":{"1min":8.8510000000000009,"5min":8.8510000000000009,"15min":117.584},"last":0.97899999999999998},{"interface":"front","average":{"1min":2.048,"5min":1.349,"15min":2.665},"min":{"1min":0.22600000000000001,"5min":0.22600000000000001,"15min":0.22600000000000001},"max":{"1min":5.1029999999999998,"5min":8.5069999999999997,"15min":117.01000000000001},"last":0.41999999999999998}]},{"osd":4,"last update":"Sat Jun 3 20:29:44 2023","interfaces":[{"interface":"back","average":{"1min":1.8049999999999999,"5min":1.2,"15min":1.4079999999999999},"min":{"1min":0.39600000000000002,"5min":0.34599999999999997,"15min":0.34599999999999997},"max":{"1min":5.7229999999999999,"5min":6.3150000000000004,"15min":6.8719999999999999},"last":0.47499999999999998},{"interface":"front","average":{"1min":1.623,"5min":1.6200000000000001,"15min":1.738},"min":{"1min":0.46999999999999997,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":5.2110000000000003,"5min":10.773999999999999,"15min":10.773999999999999},"last":0.56000000000000005}]},{"osd":5,"last update":"Sat Jun 3 20:30:03 2023","interfaces":[{"interface":"back","average":{"1min":1.319,"5min":1.3480000000000001,"15min":1.5489999999999999},"min":{"1min":0.41199999999999998,"5min":0.36299999999999999,"15min":0.313},"max":{"1min":5.1399999999999997,"5min":7.6429999999999998,"15min":38.651000000000003},"last":0.58099999999999996},{"interface":"front","average":{"1min":1.806,"5min":1.347,"15min":1.4330000000000001},"min":{"1min":0.39200000000000002,"5min":0.34499999999999997,"15min":0.316},"max":{"1min":5.9640000000000004,"5min":5.9640000000000004,"15min":38.585999999999999},"last":1.032}]},{"osd":6,"last update":"Sat Jun 3 20:30:03 2023","interfaces":[{"interface":"back","average":{"1min":1.8100000000000001,"5min":1.2929999999999999,"15min":1.296},"min":{"1min":0.36499999999999999,"5min":0.36499999999999999,"15min":0.33500000000000002},"max":{"1min":5.8869999999999996,"5min":5.8869999999999996,"15min":9.0719999999999992},"last":1.01},{"interface":"front","average":{"1min":1.5589999999999999,"5min":1.2569999999999999,"15min":1.452},"min":{"1min":0.41999999999999998,"5min":0.36699999999999999,"15min":0.36699999999999999},"max":{"1min":5.6660000000000004,"5min":5.6660000000000004,"15min":8.7279999999999998},"last":0.442}]},{"osd":7,"last update":"Sat Jun 3 20:30:03 2023","interfaces":[{"interface":"back","average":{"1min":1.8260000000000001,"5min":1.4970000000000001,"15min":1.7090000000000001},"min":{"1min":0.51400000000000001,"5min":0.45000000000000001,"15min":0.36599999999999999},"max":{"1min":5.6239999999999997,"5min":6.0700000000000003,"15min":39.008000000000003},"last":0.49199999999999999},{"interface":"front","average":{"1min":1.609,"5min":1.704,"15min":1.849},"min":{"1min":0.39500000000000002,"5min":0.38700000000000001,"15min":0.34899999999999998},"max":{"1min":7.4960000000000004,"5min":12.831,"15min":40.003999999999998},"last":0.61399999999999999}]}]},{"osd":4,"up_from":250,"seq":1073741824088,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1320528,"kb_used_data":1073544,"kb_used_omap":10,"kb_used_meta":246965,"kb_avail":92424624,"statfs":{"total":95995035648,"available":94642814976,"internally_reserved":0,"allocated":1099309056,"data_stored":1098060352,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":10652,"internal_metadata":252892772},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 3 20:30:39 2023","interfaces":[{"interface":"back","average":{"1min":1.7969999999999999,"5min":2.5070000000000001,"15min":2.0800000000000001},"min":{"1min":0.30099999999999999,"5min":0.27600000000000002,"15min":0.26300000000000001},"max":{"1min":8.3770000000000007,"5min":145.755,"15min":145.755},"last":3.9820000000000002},{"interface":"front","average":{"1min":1.2350000000000001,"5min":2.5950000000000002,"15min":1.9179999999999999},"min":{"1min":0.35099999999999998,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":5.6269999999999998,"5min":145.71799999999999,"15min":145.71799999999999},"last":1.4850000000000001}]},{"osd":1,"last update":"Sat Jun 3 20:30:39 2023","interfaces":[{"interface":"back","average":{"1min":2.1110000000000002,"5min":1.49,"15min":1.6890000000000001},"min":{"1min":0.29799999999999999,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":5.6319999999999997,"5min":9.7769999999999992,"15min":9.7769999999999992},"last":4.0709999999999997},{"interface":"front","average":{"1min":1.2230000000000001,"5min":1.302,"15min":1.7090000000000001},"min":{"1min":0.29599999999999999,"5min":0.29599999999999999,"15min":0.29599999999999999},"max":{"1min":3.9039999999999999,"5min":11.994,"15min":11.994},"last":0.67500000000000004}]},{"osd":2,"last update":"Sat Jun 3 20:30:39 2023","interfaces":[{"interface":"back","average":{"1min":5.2439999999999998,"5min":2.927,"15min":2.4089999999999998},"min":{"1min":0.40300000000000002,"5min":0.36299999999999999,"15min":0.35899999999999999},"max":{"1min":53.454999999999998,"5min":78.605000000000004,"15min":78.605000000000004},"last":1.794},{"interface":"front","average":{"1min":5.3460000000000001,"5min":2.9580000000000002,"15min":2.8330000000000002},"min":{"1min":0.371,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":53.902999999999999,"5min":73.569000000000003,"15min":73.569000000000003},"last":5.2610000000000001}]},{"osd":3,"last update":"Sat Jun 3 20:30:39 2023","interfaces":[{"interface":"back","average":{"1min":4.96,"5min":2.6539999999999999,"15min":9.7460000000000004},"min":{"1min":0.39200000000000002,"5min":0.379,"15min":0.32500000000000001},"max":{"1min":59.765999999999998,"5min":59.765999999999998,"15min":217.578},"last":1.708},{"interface":"front","average":{"1min":5.351,"5min":2.702,"15min":9.6300000000000008},"min":{"1min":0.44900000000000001,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":60.597000000000001,"5min":60.597000000000001,"15min":218.55699999999999},"last":4.1100000000000003}]},{"osd":5,"last update":"Sat Jun 3 20:30:39 2023","interfaces":[{"interface":"back","average":{"1min":0.70399999999999996,"5min":1.145,"15min":2.7069999999999999},"min":{"1min":0.38400000000000001,"5min":0.35099999999999998,"15min":0.35099999999999998},"max":{"1min":1.448,"5min":8.4760000000000009,"15min":23.913},"last":0.54100000000000004},{"interface":"front","average":{"1min":0.98899999999999999,"5min":1.1830000000000001,"15min":2.0070000000000001},"min":{"1min":0.34000000000000002,"5min":0.34000000000000002,"15min":0.34000000000000002},"max":{"1min":3.9289999999999998,"5min":8.2650000000000006,"15min":11.093999999999999},"last":0.42899999999999999}]},{"osd":6,"last update":"Sat Jun 3 20:30:39 2023","interfaces":[{"interface":"back","average":{"1min":0.75900000000000001,"5min":1.2070000000000001,"15min":2.3900000000000001},"min":{"1min":0.40300000000000002,"5min":0.40300000000000002,"15min":0.40300000000000002},"max":{"1min":1.482,"5min":9.3239999999999998,"15min":23.963000000000001},"last":0.58099999999999996},{"interface":"front","average":{"1min":1.0840000000000001,"5min":1.2649999999999999,"15min":1.446},"min":{"1min":0.38,"5min":0.38,"15min":0.38},"max":{"1min":4.0039999999999996,"5min":6.8630000000000004,"15min":8.4190000000000005},"last":0.55900000000000005}]},{"osd":7,"last update":"Sat Jun 3 20:30:39 2023","interfaces":[{"interface":"back","average":{"1min":1.123,"5min":1.4199999999999999,"15min":2.1000000000000001},"min":{"1min":0.39300000000000002,"5min":0.39300000000000002,"15min":0.374},"max":{"1min":4.032,"5min":10.496,"15min":10.747999999999999},"last":0.47699999999999998},{"interface":"front","average":{"1min":0.79100000000000004,"5min":1.294,"15min":2.0409999999999999},"min":{"1min":0.52600000000000002,"5min":0.34699999999999998,"15min":0.34699999999999998},"max":{"1min":0.998,"5min":12.154,"15min":12.154},"last":0.85499999999999998}]}]},{"osd":5,"up_from":36,"seq":154618823391,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1835380,"kb_used_data":867372,"kb_used_omap":0,"kb_used_meta":968000,"kb_avail":91909772,"statfs":{"total":95995035648,"available":94115606528,"internally_reserved":0,"allocated":888188928,"data_stored":887546103,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":991232000},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 3 20:30:22 2023","interfaces":[{"interface":"back","average":{"1min":10.137,"5min":3.952,"15min":3.0150000000000001},"min":{"1min":0.312,"5min":0.27100000000000002,"15min":0.25700000000000001},"max":{"1min":176.88300000000001,"5min":176.88300000000001,"15min":176.88300000000001},"last":0.40400000000000003},{"interface":"front","average":{"1min":10.225,"5min":4.2469999999999999,"15min":3.1760000000000002},"min":{"1min":0.37,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":176.96199999999999,"5min":176.96199999999999,"15min":176.96199999999999},"last":0.433}]},{"osd":1,"last update":"Sat Jun 3 20:30:22 2023","interfaces":[{"interface":"back","average":{"1min":2.0139999999999998,"5min":2.359,"15min":1.976},"min":{"1min":0.29899999999999999,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":5.2969999999999997,"5min":55.515000000000001,"15min":102.205},"last":0.81999999999999995},{"interface":"front","average":{"1min":1.6100000000000001,"5min":2.2160000000000002,"15min":1.9239999999999999},"min":{"1min":0.316,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":7.7539999999999996,"5min":55.194000000000003,"15min":102.265},"last":0.49299999999999999}]},{"osd":2,"last update":"Sat Jun 3 20:30:22 2023","interfaces":[{"interface":"back","average":{"1min":2.149,"5min":2.7040000000000002,"15min":3.4569999999999999},"min":{"1min":0.35699999999999998,"5min":0.35699999999999998,"15min":0.33400000000000002},"max":{"1min":9.6199999999999992,"5min":104.199,"15min":236.40000000000001},"last":0.53600000000000003},{"interface":"front","average":{"1min":2.3090000000000002,"5min":3.1909999999999998,"15min":3.7360000000000002},"min":{"1min":0.308,"5min":0.308,"15min":0.308},"max":{"1min":7.577,"5min":104.33499999999999,"15min":237.86099999999999},"last":0.75900000000000001}]},{"osd":3,"last update":"Sat Jun 3 20:29:57 2023","interfaces":[{"interface":"back","average":{"1min":1.798,"5min":1.8440000000000001,"15min":2.3839999999999999},"min":{"1min":0.47799999999999998,"5min":0.34399999999999997,"15min":0.34399999999999997},"max":{"1min":7.0919999999999996,"5min":35.725999999999999,"15min":86.486999999999995},"last":2.923},{"interface":"front","average":{"1min":1.645,"5min":1.6519999999999999,"15min":2.2639999999999998},"min":{"1min":0.434,"5min":0.39200000000000002,"15min":0.37},"max":{"1min":7.0540000000000003,"5min":35.762999999999998,"15min":86.512},"last":2.9089999999999998}]},{"osd":4,"last update":"Sat Jun 3 20:30:38 2023","interfaces":[{"interface":"back","average":{"1min":1.1160000000000001,"5min":1.3839999999999999,"15min":2.9780000000000002},"min":{"1min":0.32700000000000001,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":3.9079999999999999,"5min":7.6669999999999998,"15min":21.786000000000001},"last":0.60599999999999998},{"interface":"front","average":{"1min":1.5329999999999999,"5min":1.298,"15min":2.4399999999999999},"min":{"1min":0.38300000000000001,"5min":0.36799999999999999,"15min":0.30199999999999999},"max":{"1min":11.880000000000001,"5min":11.880000000000001,"15min":26.478000000000002},"last":0.77200000000000002}]},{"osd":6,"last update":"Sat Jun 3 20:30:22 2023","interfaces":[{"interface":"back","average":{"1min":0.996,"5min":1.369,"15min":1.1539999999999999},"min":{"1min":0.39200000000000002,"5min":0.35499999999999998,"15min":0.35199999999999998},"max":{"1min":3.0649999999999999,"5min":7.5140000000000002,"15min":7.5140000000000002},"last":0.65300000000000002},{"interface":"front","average":{"1min":1.129,"5min":1.3740000000000001,"15min":1.256},"min":{"1min":0.42299999999999999,"5min":0.37,"15min":0.37},"max":{"1min":3.234,"5min":6.5309999999999997,"15min":6.5309999999999997},"last":0.74199999999999999}]},{"osd":7,"last update":"Sat Jun 3 20:30:15 2023","interfaces":[{"interface":"back","average":{"1min":1.2669999999999999,"5min":1.5609999999999999,"15min":1.506},"min":{"1min":0.442,"5min":0.442,"15min":0.35999999999999999},"max":{"1min":3.1549999999999998,"5min":8.6549999999999994,"15min":15.516},"last":0.79100000000000004},{"interface":"front","average":{"1min":1.319,"5min":1.6220000000000001,"15min":1.3149999999999999},"min":{"1min":0.38400000000000001,"5min":0.38400000000000001,"15min":0.373},"max":{"1min":5.0350000000000001,"5min":9.6519999999999992,"15min":9.6519999999999992},"last":0.64400000000000002}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6928,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4862,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":10652,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4786,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4932,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":14,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":1415839744,"data_stored":1415839744,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":14,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":1440284672,"data_stored":1440284672,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":14,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":1433014272,"data_stored":1433010200,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":14,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":1261502464,"data_stored":1261502464,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":14,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":1088753664,"data_stored":1088749592,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":14,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":884805632,"data_stored":884801560,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":14,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":605097984,"data_stored":605093912,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":14,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":538640384,"data_stored":538640384,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-06-03T20:30:47.321 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:30:47.329 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:30:47 smithi164 bash[59462]: audit 1685824245.4118378 mgr.y (mgr.34104) 1649 : audit 0 from='client.46820 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-06-03T20:30:47.362 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:30:47.366 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:30:47.367 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:30:47.468 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:30:47.509 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:30:47.511 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:30:47.512 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:30:47.513 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:30:47 smithi089 bash[78452]: audit 1685824245.4118378 mgr.y (mgr.34104) 1649 : audit 0 from='client.46820 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-06-03T20:30:47.513 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:30:47 smithi089 bash[75068]: audit 1685824245.4118378 mgr.y (mgr.34104) 1649 : audit 0 from='client.46820 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-06-03T20:30:47.613 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:30:47.639 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:30:47.643 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:30:47.644 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:30:47.745 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:30:47.795 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:30:47.799 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:30:47.799 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:30:47.900 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:30:47.936 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:30:47.942 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:30:47.943 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:30:48.044 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:30:48.079 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:30:48.084 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:30:48.085 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:30:48.186 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:30:48.222 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:30:48.224 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:30:48.225 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:30:48.269 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:30:48 smithi089 bash[75068]: cluster 1685824247.186616 mgr.y (mgr.34104) 1650 : cluster 0 pgmap v1705: 89 pgs: 1 active+recovering+remapped, 88 active+clean; 2.6 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 592 op/s; 2415/130074 objects misplaced (1.857%); 1.8 MiB/s, 28 objects/s recovering 2023-06-03T20:30:48.270 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:30:48 smithi089 bash[78452]: cluster 1685824247.186616 mgr.y (mgr.34104) 1650 : cluster 0 pgmap v1705: 89 pgs: 1 active+recovering+remapped, 88 active+clean; 2.6 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 592 op/s; 2415/130074 objects misplaced (1.857%); 1.8 MiB/s, 28 objects/s recovering 2023-06-03T20:30:48.325 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:30:48.334 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:30:48 smithi164 bash[59462]: cluster 1685824247.186616 mgr.y (mgr.34104) 1650 : cluster 0 pgmap v1705: 89 pgs: 1 active+recovering+remapped, 88 active+clean; 2.6 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 592 op/s; 2415/130074 objects misplaced (1.857%); 1.8 MiB/s, 28 objects/s recovering 2023-06-03T20:30:48.362 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:30:48.365 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:30:48.365 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:30:48.467 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:30:48.497 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:30:48.499 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:30:48.500 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:30:48.600 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:30:48.638 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:30:48.642 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:30:48.643 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:30:48.743 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:30:48.757 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/mon.a/config 2023-06-03T20:30:48.774 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:30:48.777 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:30:48.778 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:30:48.879 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:30:48.906 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:30:48.912 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:30:48.912 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:30:49.013 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:30:49.044 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:30:49.046 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:30:49.046 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:30:49.147 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:30:49.185 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:30:49.189 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:30:49.189 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:30:49.290 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:30:49.318 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:30:49.323 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:30:49.323 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:30:49.424 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:30:49.456 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:30:49.458 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:30:49.459 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:30:49.560 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:30:49.584 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:30:49.588 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:30:49.589 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:30:49.690 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:30:49.723 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:30:49.725 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:30:49.726 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:30:49.784 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/mon.a/config 2023-06-03T20:30:49.828 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:30:49.875 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:30:49.883 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:30:49.884 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:30:49.985 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:30:50.017 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:30:50.019 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:30:50.020 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:30:50.121 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:30:50.154 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:30:50.159 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:30:50.160 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:30:50.261 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:30:50.294 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph pg dump --format=json 2023-06-03T20:30:50.299 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:30:50.306 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:30:50.307 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:30:50.408 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:30:50.438 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:30:50.444 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:30:50.445 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:30:50.529 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:30:50 smithi089 bash[75068]: cluster 1685824249.1873367 mgr.y (mgr.34104) 1651 : cluster 0 pgmap v1706: 89 pgs: 1 active+recovering+remapped, 88 active+clean; 2.5 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 648 op/s; 2264/122052 objects misplaced (1.855%); 1.6 MiB/s, 26 objects/s recovering 2023-06-03T20:30:50.529 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:30:50 smithi089 bash[78452]: cluster 1685824249.1873367 mgr.y (mgr.34104) 1651 : cluster 0 pgmap v1706: 89 pgs: 1 active+recovering+remapped, 88 active+clean; 2.5 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 648 op/s; 2264/122052 objects misplaced (1.855%); 1.6 MiB/s, 26 objects/s recovering 2023-06-03T20:30:50.546 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:30:50.552 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:30:50 smithi164 bash[59462]: cluster 1685824249.1873367 mgr.y (mgr.34104) 1651 : cluster 0 pgmap v1706: 89 pgs: 1 active+recovering+remapped, 88 active+clean; 2.5 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 648 op/s; 2264/122052 objects misplaced (1.855%); 1.6 MiB/s, 26 objects/s recovering 2023-06-03T20:30:50.580 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:30:50.584 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:30:50.584 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:30:50.685 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:30:50.710 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:30:50.712 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:30:50.712 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:30:50.813 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:30:50.818 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/mon.a/config 2023-06-03T20:30:50.848 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:30:50.851 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:30:50.852 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:30:50.953 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:30:51.001 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:30:51.001 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:30:51.002 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:30:51.102 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:30:51.142 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:30:51.144 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:30:51.144 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:30:51.245 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:30:51.276 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:30:51.277 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:30:51.278 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:30:51.379 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:30:51.413 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:30:51.419 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:30:51.419 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:30:51.520 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:30:51.562 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:30:51.569 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:30:51.570 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:30:51.670 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:30:51.713 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:30:51.718 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:30:51.719 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:30:51.820 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:30:51.857 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:30:51.859 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:30:51.859 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:30:51.960 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:30:52.009 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:30:52.014 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:30:52.015 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:30:52.115 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:30:52.143 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:30:52.145 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:30:52.145 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:30:52.246 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:30:52.286 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:30:52.291 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:30:52.292 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:30:52.392 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:30:52.427 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:30:52.434 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:30:52.435 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:30:52.529 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:30:52 smithi089 bash[75068]: cluster 1685824251.1877904 mgr.y (mgr.34104) 1652 : cluster 0 pgmap v1707: 89 pgs: 1 active+recovering+remapped, 88 active+clean; 2.5 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 502 op/s; 2264/122052 objects misplaced (1.855%); 1.5 MiB/s, 23 objects/s recovering 2023-06-03T20:30:52.529 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:30:52 smithi089 bash[75068]: audit 1685824252.1571043 mon.c (mon.2) 670 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T20:30:52.530 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:30:52 smithi089 bash[78452]: cluster 1685824251.1877904 mgr.y (mgr.34104) 1652 : cluster 0 pgmap v1707: 89 pgs: 1 active+recovering+remapped, 88 active+clean; 2.5 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 502 op/s; 2264/122052 objects misplaced (1.855%); 1.5 MiB/s, 23 objects/s recovering 2023-06-03T20:30:52.530 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:30:52 smithi089 bash[78452]: audit 1685824252.1571043 mon.c (mon.2) 670 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T20:30:52.537 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:30:52.576 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:30:52.582 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:30:52.583 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:30:52.585 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:30:52 smithi164 bash[59462]: cluster 1685824251.1877904 mgr.y (mgr.34104) 1652 : cluster 0 pgmap v1707: 89 pgs: 1 active+recovering+remapped, 88 active+clean; 2.5 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 502 op/s; 2264/122052 objects misplaced (1.855%); 1.5 MiB/s, 23 objects/s recovering 2023-06-03T20:30:52.585 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:30:52 smithi164 bash[59462]: audit 1685824252.1571043 mon.c (mon.2) 670 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T20:30:52.684 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:30:52.718 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:30:52.724 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:30:52.724 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:30:52.825 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:30:52.865 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:30:52.870 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:30:52.870 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:30:52.971 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:30:53.008 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:30:53.016 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:30:53.017 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:30:53.118 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:30:53.142 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:30:53.143 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:30:53.144 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:30:53.245 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:30:53.282 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:30:53.287 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:30:53.287 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:30:53.388 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:30:53.421 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:30:53.427 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:30:53.428 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:30:53.529 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:30:53.568 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:30:53.570 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:30:53.570 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:30:53.671 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:30:53.700 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:30:53.705 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:30:53.706 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:30:53.807 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:30:53.840 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:30:53.842 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:30:53.843 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:30:53.944 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:30:53.980 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:30:53.987 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:30:53.988 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:30:54.089 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:30:54.125 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:30:54.141 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:30:54.141 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:30:54.242 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:30:54.251 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:30:54 smithi089 bash[78452]: cluster 1685824253.1884644 mgr.y (mgr.34104) 1653 : cluster 0 pgmap v1708: 89 pgs: 1 active+recovering+remapped, 88 active+clean; 2.4 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 652 op/s; 2099/116667 objects misplaced (1.799%); 2.3 MiB/s, 37 objects/s recovering 2023-06-03T20:30:54.251 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:30:54 smithi089 bash[75068]: cluster 1685824253.1884644 mgr.y (mgr.34104) 1653 : cluster 0 pgmap v1708: 89 pgs: 1 active+recovering+remapped, 88 active+clean; 2.4 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 652 op/s; 2099/116667 objects misplaced (1.799%); 2.3 MiB/s, 37 objects/s recovering 2023-06-03T20:30:54.290 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:30:54.299 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:30:54.299 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:30:54.335 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:30:54 smithi164 bash[59462]: cluster 1685824253.1884644 mgr.y (mgr.34104) 1653 : cluster 0 pgmap v1708: 89 pgs: 1 active+recovering+remapped, 88 active+clean; 2.4 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 652 op/s; 2099/116667 objects misplaced (1.799%); 2.3 MiB/s, 37 objects/s recovering 2023-06-03T20:30:54.401 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:30:54.455 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:30:54.457 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:30:54.458 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:30:54.559 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:30:54.603 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:30:54.609 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:30:54.609 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:30:54.710 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:30:54.745 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:30:54.748 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:30:54.748 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:30:54.849 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:30:54.878 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:30:54.883 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:30:54.885 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:30:54.933 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/mon.a/config 2023-06-03T20:30:54.987 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:30:55.039 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:30:55.042 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:30:55.043 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:30:55.144 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:30:55.191 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:30:55.195 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:30:55.196 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:30:55.297 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:30:55.325 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:30:55.331 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:30:55.331 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:30:55.432 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:30:55.470 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:30:55.475 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:30:55.476 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:30:55.577 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:30:55.611 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:30:55.614 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:30:55.614 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:30:55.715 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:30:55.750 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:30:55.757 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:30:55.758 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:30:55.859 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:30:55.889 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:30:55.894 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:30:55.894 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:30:55.995 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:30:56.021 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:30:56.022 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:30:56.023 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:30:56.124 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:30:56.152 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:30:56.157 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:30:56.158 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:30:56.257 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 20:30:56 smithi089 bash[71163]: ::ffff:172.21.15.164 - - [03/Jun/2023:20:30:56] "GET /metrics HTTP/1.1" 200 229145 "" "Prometheus/2.33.4" 2023-06-03T20:30:56.259 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:30:56.289 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:30:56.294 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:30:56.295 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:30:56.395 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:30:56.432 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:30:56.435 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:30:56.436 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:30:56.529 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:30:56 smithi089 bash[78452]: cluster 1685824255.1889632 mgr.y (mgr.34104) 1654 : cluster 0 pgmap v1709: 89 pgs: 1 active+recovering+remapped, 88 active+clean; 2.3 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 582 op/s; 2099/113493 objects misplaced (1.849%); 1.6 MiB/s, 26 objects/s recovering 2023-06-03T20:30:56.529 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:30:56 smithi089 bash[75068]: cluster 1685824255.1889632 mgr.y (mgr.34104) 1654 : cluster 0 pgmap v1709: 89 pgs: 1 active+recovering+remapped, 88 active+clean; 2.3 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 582 op/s; 2099/113493 objects misplaced (1.849%); 1.6 MiB/s, 26 objects/s recovering 2023-06-03T20:30:56.537 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:30:56.568 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:30:56.571 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:30:56.572 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:30:56.585 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:30:56 smithi164 bash[59462]: cluster 1685824255.1889632 mgr.y (mgr.34104) 1654 : cluster 0 pgmap v1709: 89 pgs: 1 active+recovering+remapped, 88 active+clean; 2.3 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 582 op/s; 2099/113493 objects misplaced (1.849%); 1.6 MiB/s, 26 objects/s recovering 2023-06-03T20:30:56.673 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:30:56.711 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:30:56.715 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:30:56.716 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:30:56.817 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:30:56.850 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:30:56.853 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:30:56.853 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:30:56.954 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:30:57.009 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:30:57.011 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:30:57.012 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:30:57.112 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:30:57.153 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:30:57.160 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:30:57.161 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:30:57.261 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:30:57.296 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:30:57.301 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:30:57.301 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:30:57.402 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:30:57.434 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:30:57.440 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:30:57.440 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:30:57.541 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:30:57.572 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:30:57.576 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:30:57.577 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:30:57.677 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:30:57.718 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:30:57.724 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:30:57.725 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:30:57.826 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:30:57.860 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:30:57.863 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:30:57.864 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:30:57.965 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:30:57.992 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:30:57.995 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:30:57.995 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:30:58.096 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:30:58.124 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:30:58.126 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:30:58.126 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:30:58.227 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:30:58.274 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:30:58.276 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:30:58.276 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:30:58.377 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:30:58.408 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:30:58.412 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:30:58.412 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:30:58.513 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:30:58.520 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:30:58 smithi089 bash[78452]: cluster 1685824257.1894476 mgr.y (mgr.34104) 1655 : cluster 0 pgmap v1710: 89 pgs: 1 active+recovering+remapped, 88 active+clean; 2.3 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 512 op/s; 2099/112593 objects misplaced (1.864%); 1.6 MiB/s, 26 objects/s recovering 2023-06-03T20:30:58.521 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:30:58 smithi089 bash[75068]: cluster 1685824257.1894476 mgr.y (mgr.34104) 1655 : cluster 0 pgmap v1710: 89 pgs: 1 active+recovering+remapped, 88 active+clean; 2.3 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 512 op/s; 2099/112593 objects misplaced (1.864%); 1.6 MiB/s, 26 objects/s recovering 2023-06-03T20:30:58.553 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:30:58.557 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:30:58.557 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:30:58.585 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:30:58 smithi164 bash[59462]: cluster 1685824257.1894476 mgr.y (mgr.34104) 1655 : cluster 0 pgmap v1710: 89 pgs: 1 active+recovering+remapped, 88 active+clean; 2.3 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 512 op/s; 2099/112593 objects misplaced (1.864%); 1.6 MiB/s, 26 objects/s recovering 2023-06-03T20:30:58.658 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:30:58.700 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:30:58.706 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:30:58.706 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:30:58.807 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:30:58.849 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:30:58.857 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:30:58.857 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:30:58.958 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:30:58.986 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:30:58.991 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:30:58.992 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:30:59.093 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:30:59.127 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:30:59.141 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:30:59.142 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:30:59.243 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:30:59.289 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:30:59.291 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:30:59.292 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:30:59.393 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:30:59.432 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:30:59.435 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:30:59.436 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:30:59.536 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:30:59.573 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:30:59.579 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:30:59.579 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:30:59.680 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:30:59.719 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:30:59.725 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:30:59.725 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:30:59.826 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:30:59.870 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:30:59.874 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:30:59.874 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:30:59.975 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:31:00.010 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:31:00.015 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:31:00.016 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:31:00.117 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:31:00.156 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:31:00.163 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:31:00.164 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:31:00.262 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:31:00 smithi089 bash[75068]: cluster 1685824259.1901252 mgr.y (mgr.34104) 1656 : cluster 0 pgmap v1711: 89 pgs: 1 active+recovering+remapped, 88 active+clean; 2.1 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 696 op/s; 1924/104985 objects misplaced (1.833%); 2.6 MiB/s, 40 objects/s recovering 2023-06-03T20:31:00.263 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:31:00 smithi164 bash[59462]: cluster 1685824259.1901252 mgr.y (mgr.34104) 1656 : cluster 0 pgmap v1711: 89 pgs: 1 active+recovering+remapped, 88 active+clean; 2.1 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 696 op/s; 1924/104985 objects misplaced (1.833%); 2.6 MiB/s, 40 objects/s recovering 2023-06-03T20:31:00.266 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:31:00.311 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:31:00.315 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:31:00.316 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:31:00.417 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:31:00.462 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:31:00.463 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:31:00.463 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:31:00.529 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:31:00 smithi089 bash[78452]: cluster 1685824259.1901252 mgr.y (mgr.34104) 1656 : cluster 0 pgmap v1711: 89 pgs: 1 active+recovering+remapped, 88 active+clean; 2.1 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 696 op/s; 1924/104985 objects misplaced (1.833%); 2.6 MiB/s, 40 objects/s recovering 2023-06-03T20:31:00.565 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:31:00.599 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:31:00.605 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:31:00.606 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:31:00.707 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:31:00.734 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:31:00.735 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:31:00.736 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:31:00.837 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:31:00.876 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:31:00.879 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:31:00.880 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:31:00.981 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:31:01.024 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:31:01.026 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:31:01.027 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:31:01.128 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:31:01.254 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:31:01.256 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:31:01.257 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:31:01.358 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:31:01.409 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:31:01.411 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:31:01.412 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:31:01.512 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:31:01.546 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:31:01.557 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:31:01.558 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:31:01.659 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:31:01.692 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:31:01.696 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:31:01.696 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:31:01.797 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:31:01.831 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:31:01.832 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:31:01.833 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:31:01.934 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:31:01.970 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:31:01.972 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:31:01.973 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:31:02.074 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:31:02.114 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:31:02.118 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:31:02.119 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:31:02.220 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:31:02.252 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:31:02.254 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:31:02.255 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:31:02.335 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:31:02 smithi164 bash[59462]: cluster 1685824261.190547 mgr.y (mgr.34104) 1657 : cluster 0 pgmap v1712: 89 pgs: 1 active+recovering+remapped, 88 active+clean; 2.1 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 473 op/s; 1924/104985 objects misplaced (1.833%); 1.8 MiB/s, 28 objects/s recovering 2023-06-03T20:31:02.356 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:31:02.388 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:31:02.390 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:31:02.391 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:31:02.492 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:31:02.521 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:31:02.523 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:31:02.524 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:31:02.529 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:31:02 smithi089 bash[78452]: cluster 1685824261.190547 mgr.y (mgr.34104) 1657 : cluster 0 pgmap v1712: 89 pgs: 1 active+recovering+remapped, 88 active+clean; 2.1 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 473 op/s; 1924/104985 objects misplaced (1.833%); 1.8 MiB/s, 28 objects/s recovering 2023-06-03T20:31:02.529 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:31:02 smithi089 bash[75068]: cluster 1685824261.190547 mgr.y (mgr.34104) 1657 : cluster 0 pgmap v1712: 89 pgs: 1 active+recovering+remapped, 88 active+clean; 2.1 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 473 op/s; 1924/104985 objects misplaced (1.833%); 1.8 MiB/s, 28 objects/s recovering 2023-06-03T20:31:02.624 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:31:02.658 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:31:02.663 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:31:02.663 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:31:02.764 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:31:02.803 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:31:02.810 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:31:02.811 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:31:02.912 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:31:02.947 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:31:02.951 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:31:02.952 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:31:03.053 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:31:03.087 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:31:03.095 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:31:03.096 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:31:03.197 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:31:03.238 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:31:03.242 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:31:03.243 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:31:03.343 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:31:03.375 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:31:03.379 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:31:03.380 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:31:03.481 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:31:03.518 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:31:03.526 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:31:03.526 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:31:03.627 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:31:03.662 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:31:03.664 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:31:03.665 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:31:03.766 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:31:03.805 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:31:03.808 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:31:03.809 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:31:03.910 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:31:03.957 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:31:03.963 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:31:03.964 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:31:04.065 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:31:04.096 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:31:04.099 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:31:04.100 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:31:04.200 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:31:04.233 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:31:04.238 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:31:04.239 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:31:04.335 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:31:04 smithi164 bash[59462]: cluster 1685824263.1912322 mgr.y (mgr.34104) 1658 : cluster 0 pgmap v1713: 89 pgs: 1 active+recovering+remapped, 88 active+clean; 2.0 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 623 op/s; 1730/99600 objects misplaced (1.737%); 2.8 MiB/s, 44 objects/s recovering 2023-06-03T20:31:04.340 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:31:04.385 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:31:04.391 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:31:04.392 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:31:04.493 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:31:04.529 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:31:04 smithi089 bash[78452]: cluster 1685824263.1912322 mgr.y (mgr.34104) 1658 : cluster 0 pgmap v1713: 89 pgs: 1 active+recovering+remapped, 88 active+clean; 2.0 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 623 op/s; 1730/99600 objects misplaced (1.737%); 2.8 MiB/s, 44 objects/s recovering 2023-06-03T20:31:04.529 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:31:04 smithi089 bash[75068]: cluster 1685824263.1912322 mgr.y (mgr.34104) 1658 : cluster 0 pgmap v1713: 89 pgs: 1 active+recovering+remapped, 88 active+clean; 2.0 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 623 op/s; 1730/99600 objects misplaced (1.737%); 2.8 MiB/s, 44 objects/s recovering 2023-06-03T20:31:04.534 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:31:04.543 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:31:04.544 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:31:04.645 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:31:04.679 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:31:04.684 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:31:04.684 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:31:04.785 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:31:04.825 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:31:04.837 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:31:04.838 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:31:04.938 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:31:04.982 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:31:04.983 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:31:04.984 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:31:05.085 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:31:05.120 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:31:05.123 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:31:05.125 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:31:05.226 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:31:05.260 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:31:05.269 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:31:05.270 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:31:05.371 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:31:05.400 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:31:05.406 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:31:05.407 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:31:05.508 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:31:05.555 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:31:05.559 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:31:05.560 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:31:05.660 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:31:05.687 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:31:05.694 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:31:05.694 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:31:05.795 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:31:05.837 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:31:05.844 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:31:05.845 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:31:05.946 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:31:05.977 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:31:05.980 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:31:05.981 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:31:06.082 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:31:06.122 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:31:06.125 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:31:06.126 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:31:06.226 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:31:06.232 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 20:31:06 smithi089 bash[71163]: ::ffff:172.21.15.164 - - [03/Jun/2023:20:31:06] "GET /metrics HTTP/1.1" 200 229145 "" "Prometheus/2.33.4" 2023-06-03T20:31:06.256 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:31:06 smithi164 bash[59462]: cluster 1685824265.1916597 mgr.y (mgr.34104) 1659 : cluster 0 pgmap v1714: 89 pgs: 1 active+recovering+remapped, 88 active+clean; 2.0 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 562 op/s; 1730/96423 objects misplaced (1.794%); 1.9 MiB/s, 30 objects/s recovering 2023-06-03T20:31:06.266 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:31:06.271 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:31:06.271 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:31:06.372 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:31:06.404 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:31:06.406 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:31:06.406 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:31:06.507 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:31:06.515 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:31:06 smithi089 bash[75068]: cluster 1685824265.1916597 mgr.y (mgr.34104) 1659 : cluster 0 pgmap v1714: 89 pgs: 1 active+recovering+remapped, 88 active+clean; 2.0 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 562 op/s; 1730/96423 objects misplaced (1.794%); 1.9 MiB/s, 30 objects/s recovering 2023-06-03T20:31:06.516 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:31:06 smithi089 bash[78452]: cluster 1685824265.1916597 mgr.y (mgr.34104) 1659 : cluster 0 pgmap v1714: 89 pgs: 1 active+recovering+remapped, 88 active+clean; 2.0 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 562 op/s; 1730/96423 objects misplaced (1.794%); 1.9 MiB/s, 30 objects/s recovering 2023-06-03T20:31:06.536 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:31:06.539 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:31:06.540 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:31:06.641 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:31:06.684 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:31:06.685 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:31:06.686 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:31:06.787 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:31:06.838 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:31:06.841 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:31:06.842 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:31:06.943 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:31:06.992 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:31:06.994 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:31:06.995 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:31:07.096 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:31:07.127 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:31:07.131 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:31:07.133 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:31:07.234 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:31:07.276 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:31:07 smithi164 bash[59462]: audit 1685824266.8569489 mon.c (mon.2) 671 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-03T20:31:07.276 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:31:07 smithi164 bash[59462]: audit 1685824267.1568277 mon.c (mon.2) 672 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T20:31:07.305 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:31:07.308 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:31:07.309 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:31:07.410 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:31:07.452 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:31:07.454 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:31:07.455 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:31:07.529 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:31:07 smithi089 bash[75068]: audit 1685824266.8569489 mon.c (mon.2) 671 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-03T20:31:07.529 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:31:07 smithi089 bash[75068]: audit 1685824267.1568277 mon.c (mon.2) 672 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T20:31:07.530 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:31:07 smithi089 bash[78452]: audit 1685824266.8569489 mon.c (mon.2) 671 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-03T20:31:07.530 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:31:07 smithi089 bash[78452]: audit 1685824267.1568277 mon.c (mon.2) 672 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T20:31:07.555 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:31:07.593 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:31:07.595 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:31:07.595 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:31:07.696 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:31:07.740 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:31:07.744 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:31:07.745 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:31:07.846 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:31:07.877 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:31:07.879 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:31:07.880 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:31:07.981 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:31:08.025 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:31:08.031 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:31:08.032 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:31:08.133 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:31:08.179 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:31:08.190 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:31:08.190 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:31:08.277 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:31:08 smithi164 bash[59462]: cluster 1685824267.192156 mgr.y (mgr.34104) 1660 : cluster 0 pgmap v1715: 89 pgs: 1 active+recovering+remapped, 88 active+clean; 1.9 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 504 op/s; 1730/95343 objects misplaced (1.815%); 1.9 MiB/s, 30 objects/s recovering 2023-06-03T20:31:08.291 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:31:08.325 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:31:08.332 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:31:08.332 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:31:08.433 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:31:08.471 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:31:08.475 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:31:08.475 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:31:08.529 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:31:08 smithi089 bash[75068]: cluster 1685824267.192156 mgr.y (mgr.34104) 1660 : cluster 0 pgmap v1715: 89 pgs: 1 active+recovering+remapped, 88 active+clean; 1.9 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 504 op/s; 1730/95343 objects misplaced (1.815%); 1.9 MiB/s, 30 objects/s recovering 2023-06-03T20:31:08.529 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:31:08 smithi089 bash[78452]: cluster 1685824267.192156 mgr.y (mgr.34104) 1660 : cluster 0 pgmap v1715: 89 pgs: 1 active+recovering+remapped, 88 active+clean; 1.9 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 504 op/s; 1730/95343 objects misplaced (1.815%); 1.9 MiB/s, 30 objects/s recovering 2023-06-03T20:31:08.576 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:31:08.609 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:31:08.611 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:31:08.611 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:31:08.712 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:31:08.743 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:31:08.746 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:31:08.747 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:31:08.848 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:31:08.881 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:31:08.889 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:31:08.890 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:31:08.990 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:31:09.034 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:31:09.035 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:31:09.035 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:31:09.136 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:31:09.202 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:31:09.209 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:31:09.209 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:31:09.310 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:31:09.342 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:31:09.357 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:31:09.357 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:31:09.458 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:31:09.505 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:31:09.508 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:31:09.509 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:31:09.610 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:31:09.639 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:31:09.646 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:31:09.647 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:31:09.748 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:31:09.789 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:31:09.793 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:31:09.794 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:31:09.895 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:31:09.933 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:31:09.935 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:31:09.936 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:31:10.037 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:31:10.068 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:31:10.070 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:31:10.071 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:31:10.172 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:31:10.198 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:31:10.199 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:31:10.200 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:31:10.301 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:31:10.335 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:31:10.337 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:31:10.338 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:31:10.439 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:31:10.448 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:31:10 smithi089 bash[78452]: cluster 1685824269.1928813 mgr.y (mgr.34104) 1661 : cluster 0 pgmap v1716: 89 pgs: 1 active+recovering+remapped, 88 active+clean; 1.8 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 689 op/s; 1554/87774 objects misplaced (1.770%); 2.8 MiB/s, 45 objects/s recovering 2023-06-03T20:31:10.449 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:31:10 smithi089 bash[75068]: cluster 1685824269.1928813 mgr.y (mgr.34104) 1661 : cluster 0 pgmap v1716: 89 pgs: 1 active+recovering+remapped, 88 active+clean; 1.8 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 689 op/s; 1554/87774 objects misplaced (1.770%); 2.8 MiB/s, 45 objects/s recovering 2023-06-03T20:31:10.478 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:31:10.490 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:31:10.491 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:31:10.585 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:31:10 smithi164 bash[59462]: cluster 1685824269.1928813 mgr.y (mgr.34104) 1661 : cluster 0 pgmap v1716: 89 pgs: 1 active+recovering+remapped, 88 active+clean; 1.8 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 689 op/s; 1554/87774 objects misplaced (1.770%); 2.8 MiB/s, 45 objects/s recovering 2023-06-03T20:31:10.592 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:31:10.629 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:31:10.632 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:31:10.632 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:31:10.733 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:31:10.785 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:31:10.793 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:31:10.793 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:31:10.894 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:31:10.940 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:31:10.942 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:31:10.943 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:31:11.044 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:31:11.076 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:31:11.083 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:31:11.084 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:31:11.185 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:31:11.223 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:31:11.228 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:31:11.229 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:31:11.330 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:31:11.363 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:31:11.367 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:31:11.368 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:31:11.469 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:31:11.509 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:31:11.511 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:31:11.512 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:31:11.613 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:31:11.633 INFO:teuthology.orchestra.run.smithi089.stdout:ERROR: (22) Invalid argument 2023-06-03T20:31:11.633 INFO:teuthology.orchestra.run.smithi089.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-06-03T20:31:11.648 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:31:11.651 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:31:11.651 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:31:11.752 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:31:11.789 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:31:11.791 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:31:11.792 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:31:11.893 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:31:11.933 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:31:11.939 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:31:11.940 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:31:12.041 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:31:12.080 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:31:12.087 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:31:12.088 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:31:12.189 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:31:12.228 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:31:12.231 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:31:12.232 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:31:12.333 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:31:12.370 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:31:12.377 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:31:12.378 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:31:12.479 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:31:12.498 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:31:12 smithi089 bash[75068]: cluster 1685824271.193335 mgr.y (mgr.34104) 1662 : cluster 0 pgmap v1717: 89 pgs: 1 active+recovering+remapped, 88 active+clean; 1.8 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 477 op/s; 1554/87774 objects misplaced (1.770%); 1.9 MiB/s, 30 objects/s recovering 2023-06-03T20:31:12.498 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:31:12 smithi089 bash[78452]: cluster 1685824271.193335 mgr.y (mgr.34104) 1662 : cluster 0 pgmap v1717: 89 pgs: 1 active+recovering+remapped, 88 active+clean; 1.8 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 477 op/s; 1554/87774 objects misplaced (1.770%); 1.9 MiB/s, 30 objects/s recovering 2023-06-03T20:31:12.523 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:31:12.527 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:31:12.528 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:31:12.587 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:31:12 smithi164 bash[59462]: cluster 1685824271.193335 mgr.y (mgr.34104) 1662 : cluster 0 pgmap v1717: 89 pgs: 1 active+recovering+remapped, 88 active+clean; 1.8 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 477 op/s; 1554/87774 objects misplaced (1.770%); 1.9 MiB/s, 30 objects/s recovering 2023-06-03T20:31:12.629 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:31:12.664 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:31:12.669 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:31:12.670 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:31:12.770 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:31:12.785 INFO:teuthology.orchestra.run.smithi089.stdout:osd.0: {} 2023-06-03T20:31:12.799 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:31:12.801 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:31:12.802 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:31:12.810 INFO:teuthology.orchestra.run.smithi089.stdout:osd.1: {} 2023-06-03T20:31:12.878 INFO:teuthology.orchestra.run.smithi089.stdout:osd.2: {} 2023-06-03T20:31:12.903 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:31:12.924 INFO:teuthology.orchestra.run.smithi089.stdout:osd.3: {} 2023-06-03T20:31:12.955 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:31:12.959 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:31:12.960 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:31:12.973 INFO:teuthology.orchestra.run.smithi089.stdout:osd.4: {} 2023-06-03T20:31:13.026 INFO:teuthology.orchestra.run.smithi089.stdout:osd.5: {} 2023-06-03T20:31:13.062 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:31:13.069 INFO:teuthology.orchestra.run.smithi089.stdout:osd.6: {} 2023-06-03T20:31:13.093 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:31:13.096 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:31:13.096 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:31:13.102 INFO:teuthology.orchestra.run.smithi089.stdout:osd.7: {} 2023-06-03T20:31:13.106 INFO:teuthology.orchestra.run.smithi089.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-06-03T20:31:13.106 INFO:teuthology.orchestra.run.smithi089.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-06-03T20:31:13.106 INFO:teuthology.orchestra.run.smithi089.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-06-03T20:31:13.107 INFO:teuthology.orchestra.run.smithi089.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-06-03T20:31:13.107 INFO:teuthology.orchestra.run.smithi089.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-06-03T20:31:13.107 INFO:teuthology.orchestra.run.smithi089.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-06-03T20:31:13.107 INFO:teuthology.orchestra.run.smithi089.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-06-03T20:31:13.107 INFO:teuthology.orchestra.run.smithi089.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-06-03T20:31:13.197 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:31:13.248 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:31:13.274 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:31:13.275 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:31:13.335 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:31:13 smithi164 bash[59462]: audit 1685824272.7392638 mon.b (mon.1) 66 : audit 0 from='client.? 172.21.15.89:0/4103408265' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-03T20:31:13.377 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:31:13.426 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:31:13.430 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:31:13.430 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:31:13 smithi089 bash[75068]: audit 1685824272.7392638 mon.b (mon.1) 66 : audit 0 from='client.? 172.21.15.89:0/4103408265' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-03T20:31:13.431 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:31:13 smithi089 bash[78452]: audit 1685824272.7392638 mon.b (mon.1) 66 : audit 0 from='client.? 172.21.15.89:0/4103408265' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-03T20:31:13.431 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:31:13.532 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:31:13.567 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:31:13.572 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:31:13.573 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:31:13.674 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:31:13.723 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:31:13.726 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:31:13.727 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:31:13.828 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:31:13.928 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:31:13.932 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:31:13.932 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:31:14.033 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:31:14.059 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:31:14.067 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:31:14.068 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:31:14.169 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:31:14.201 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:31:14.208 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:31:14.208 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:31:14.309 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:31:14.310 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:31:14 smithi164 bash[59462]: cluster 1685824273.19406 mgr.y (mgr.34104) 1663 : cluster 0 pgmap v1718: 89 pgs: 1 active+recovering+remapped, 88 active+clean; 1.7 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 622 op/s; 1379/82581 objects misplaced (1.670%); 2.8 MiB/s, 45 objects/s recovering 2023-06-03T20:31:14.310 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:31:14 smithi164 bash[59462]: audit 1685824273.4316273 mon.a (mon.0) 1640 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:31:14.311 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:31:14 smithi164 bash[59462]: audit 1685824273.4464338 mon.a (mon.0) 1641 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:31:14.311 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:31:14 smithi164 bash[59462]: audit 1685824273.540529 mon.a (mon.0) 1642 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:31:14.311 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:31:14 smithi164 bash[59462]: audit 1685824273.5573437 mon.a (mon.0) 1643 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:31:14.311 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:31:14 smithi164 bash[59462]: audit 1685824274.195693 mon.c (mon.2) 673 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-03T20:31:14.311 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:31:14 smithi164 bash[59462]: audit 1685824274.196973 mon.c (mon.2) 674 : audit 1 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-03T20:31:14.311 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:31:14 smithi164 bash[59462]: audit 1685824274.2032573 mon.a (mon.0) 1644 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:31:14.341 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:31:14.348 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:31:14.349 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:31:14.450 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:31:14.480 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:31:14.483 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:31:14.483 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:31:14.584 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:31:14.613 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:31:14.619 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:31:14.620 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:31:14.721 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:31:14.729 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:31:14 smithi089 bash[78452]: cluster 1685824273.19406 mgr.y (mgr.34104) 1663 : cluster 0 pgmap v1718: 89 pgs: 1 active+recovering+remapped, 88 active+clean; 1.7 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 622 op/s; 1379/82581 objects misplaced (1.670%); 2.8 MiB/s, 45 objects/s recovering 2023-06-03T20:31:14.730 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:31:14 smithi089 bash[78452]: audit 1685824273.4316273 mon.a (mon.0) 1640 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:31:14.730 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:31:14 smithi089 bash[78452]: audit 1685824273.4464338 mon.a (mon.0) 1641 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:31:14.730 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:31:14 smithi089 bash[78452]: audit 1685824273.540529 mon.a (mon.0) 1642 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:31:14.730 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:31:14 smithi089 bash[78452]: audit 1685824273.5573437 mon.a (mon.0) 1643 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:31:14.730 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:31:14 smithi089 bash[78452]: audit 1685824274.195693 mon.c (mon.2) 673 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-03T20:31:14.730 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:31:14 smithi089 bash[78452]: audit 1685824274.196973 mon.c (mon.2) 674 : audit 1 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-03T20:31:14.730 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:31:14 smithi089 bash[78452]: audit 1685824274.2032573 mon.a (mon.0) 1644 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:31:14.731 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:31:14 smithi089 bash[75068]: cluster 1685824273.19406 mgr.y (mgr.34104) 1663 : cluster 0 pgmap v1718: 89 pgs: 1 active+recovering+remapped, 88 active+clean; 1.7 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 622 op/s; 1379/82581 objects misplaced (1.670%); 2.8 MiB/s, 45 objects/s recovering 2023-06-03T20:31:14.731 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:31:14 smithi089 bash[75068]: audit 1685824273.4316273 mon.a (mon.0) 1640 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:31:14.731 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:31:14 smithi089 bash[75068]: audit 1685824273.4464338 mon.a (mon.0) 1641 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:31:14.731 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:31:14 smithi089 bash[75068]: audit 1685824273.540529 mon.a (mon.0) 1642 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:31:14.731 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:31:14 smithi089 bash[75068]: audit 1685824273.5573437 mon.a (mon.0) 1643 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:31:14.732 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:31:14 smithi089 bash[75068]: audit 1685824274.195693 mon.c (mon.2) 673 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-03T20:31:14.732 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:31:14 smithi089 bash[75068]: audit 1685824274.196973 mon.c (mon.2) 674 : audit 1 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-03T20:31:14.732 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:31:14 smithi089 bash[75068]: audit 1685824274.2032573 mon.a (mon.0) 1644 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:31:14.760 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:31:14.763 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:31:14.764 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:31:14.864 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:31:14.900 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:31:14.902 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:31:14.902 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:31:15.003 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:31:15.048 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:31:15.051 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:31:15.052 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:31:15.153 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:31:15.191 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:31:15.192 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:31:15.192 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:31:15.293 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:31:15.317 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:31:15.375 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:31:15.376 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:31:15.476 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:31:15.509 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:31:15.515 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:31:15.517 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:31:15.618 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:31:15.648 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:31:15.650 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:31:15.651 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:31:15.752 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:31:15.782 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:31:15.787 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:31:15.787 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:31:15.888 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:31:15.911 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:31:15.914 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:31:15.914 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:31:16.015 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:31:16.053 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:31:16.055 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:31:16.056 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:31:16.157 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:31:16.186 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:31:16.191 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:31:16.192 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:31:16.293 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:31:16.301 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 20:31:16 smithi089 bash[71163]: ::ffff:172.21.15.164 - - [03/Jun/2023:20:31:16] "GET /metrics HTTP/1.1" 200 229114 "" "Prometheus/2.33.4" 2023-06-03T20:31:16.336 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:31:16.339 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:31:16.340 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:31:16.343 INFO:teuthology.orchestra.run.smithi089.stderr:nodeep-scrub is unset 2023-06-03T20:31:16.441 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:31:16.454 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:31:16 smithi164 bash[59462]: cluster 1685824275.1945806 mgr.y (mgr.34104) 1664 : cluster 0 pgmap v1719: 89 pgs: 1 active+recovering+remapped, 88 active+clean; 1.6 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 558 op/s; 1379/79491 objects misplaced (1.735%); 1.8 MiB/s, 29 objects/s recovering 2023-06-03T20:31:16.454 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:31:16 smithi164 bash[59462]: audit 1685824276.1699207 mon.c (mon.2) 675 : audit 1 from='client.? 172.21.15.89:0/1713192137' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-06-03T20:31:16.454 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:31:16 smithi164 bash[59462]: audit 1685824276.1703196 mon.a (mon.0) 1645 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-06-03T20:31:16.477 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:31:16.479 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:31:16.480 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:31:16.581 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:31:16.615 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:31:16.618 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:31:16.619 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:31:16.720 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:31:16.748 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:31:16.749 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:31:16.750 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:31:16.779 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:31:16 smithi089 bash[75068]: cluster 1685824275.1945806 mgr.y (mgr.34104) 1664 : cluster 0 pgmap v1719: 89 pgs: 1 active+recovering+remapped, 88 active+clean; 1.6 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 558 op/s; 1379/79491 objects misplaced (1.735%); 1.8 MiB/s, 29 objects/s recovering 2023-06-03T20:31:16.779 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:31:16 smithi089 bash[75068]: audit 1685824276.1699207 mon.c (mon.2) 675 : audit 1 from='client.? 172.21.15.89:0/1713192137' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-06-03T20:31:16.780 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:31:16 smithi089 bash[75068]: audit 1685824276.1703196 mon.a (mon.0) 1645 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-06-03T20:31:16.780 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:31:16 smithi089 bash[78452]: cluster 1685824275.1945806 mgr.y (mgr.34104) 1664 : cluster 0 pgmap v1719: 89 pgs: 1 active+recovering+remapped, 88 active+clean; 1.6 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 558 op/s; 1379/79491 objects misplaced (1.735%); 1.8 MiB/s, 29 objects/s recovering 2023-06-03T20:31:16.780 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:31:16 smithi089 bash[78452]: audit 1685824276.1699207 mon.c (mon.2) 675 : audit 1 from='client.? 172.21.15.89:0/1713192137' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-06-03T20:31:16.780 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:31:16 smithi089 bash[78452]: audit 1685824276.1703196 mon.a (mon.0) 1645 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-06-03T20:31:16.851 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:31:16.887 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:31:16.891 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:31:16.892 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:31:16.993 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:31:17.025 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:31:17.028 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:31:17.028 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:31:17.111 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph daemon osd.2 dump_ops_in_flight 2023-06-03T20:31:17.129 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:31:17.160 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:31:17.162 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:31:17.162 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:31:17.263 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:31:17.307 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:31:17.313 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:31:17.313 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:31:17.317 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:31:17 smithi164 bash[59462]: cluster 1685824276.3057513 mon.a (mon.0) 1646 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-06-03T20:31:17.317 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:31:17 smithi164 bash[59462]: cluster 1685824276.305786 mon.a (mon.0) 1647 : cluster 1 Cluster is now healthy 2023-06-03T20:31:17.317 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:31:17 smithi164 bash[59462]: audit 1685824276.32618 mon.a (mon.0) 1648 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-06-03T20:31:17.317 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:31:17 smithi164 bash[59462]: cluster 1685824276.3264666 mon.a (mon.0) 1649 : cluster 0 osdmap e293: 8 total, 8 up, 8 in 2023-06-03T20:31:17.318 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:31:17 smithi164 bash[59462]: cluster 1685824276.3342388 osd.5 (osd.5) 242 : cluster 3 failed to encode map e293 with expected crc 2023-06-03T20:31:17.318 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:31:17 smithi164 bash[59462]: cluster 1685824276.3370605 osd.6 (osd.6) 228 : cluster 3 failed to encode map e293 with expected crc 2023-06-03T20:31:17.318 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:31:17 smithi164 bash[59462]: cluster 1685824276.3372157 osd.5 (osd.5) 243 : cluster 3 failed to encode map e293 with expected crc 2023-06-03T20:31:17.318 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:31:17 smithi164 bash[59462]: cluster 1685824276.339128 osd.6 (osd.6) 229 : cluster 3 failed to encode map e293 with expected crc 2023-06-03T20:31:17.414 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:31:17.422 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:31:17 smithi089 bash[75068]: cluster 1685824276.3057513 mon.a (mon.0) 1646 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-06-03T20:31:17.422 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:31:17 smithi089 bash[75068]: cluster 1685824276.305786 mon.a (mon.0) 1647 : cluster 1 Cluster is now healthy 2023-06-03T20:31:17.422 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:31:17 smithi089 bash[75068]: audit 1685824276.32618 mon.a (mon.0) 1648 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-06-03T20:31:17.423 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:31:17 smithi089 bash[75068]: cluster 1685824276.3264666 mon.a (mon.0) 1649 : cluster 0 osdmap e293: 8 total, 8 up, 8 in 2023-06-03T20:31:17.423 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:31:17 smithi089 bash[75068]: cluster 1685824276.3342388 osd.5 (osd.5) 242 : cluster 3 failed to encode map e293 with expected crc 2023-06-03T20:31:17.423 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:31:17 smithi089 bash[75068]: cluster 1685824276.3370605 osd.6 (osd.6) 228 : cluster 3 failed to encode map e293 with expected crc 2023-06-03T20:31:17.423 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:31:17 smithi089 bash[75068]: cluster 1685824276.3372157 osd.5 (osd.5) 243 : cluster 3 failed to encode map e293 with expected crc 2023-06-03T20:31:17.423 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:31:17 smithi089 bash[75068]: cluster 1685824276.339128 osd.6 (osd.6) 229 : cluster 3 failed to encode map e293 with expected crc 2023-06-03T20:31:17.427 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:31:17 smithi089 bash[78452]: cluster 1685824276.3057513 mon.a (mon.0) 1646 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-06-03T20:31:17.428 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:31:17 smithi089 bash[78452]: cluster 1685824276.305786 mon.a (mon.0) 1647 : cluster 1 Cluster is now healthy 2023-06-03T20:31:17.428 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:31:17 smithi089 bash[78452]: audit 1685824276.32618 mon.a (mon.0) 1648 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-06-03T20:31:17.428 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:31:17 smithi089 bash[78452]: cluster 1685824276.3264666 mon.a (mon.0) 1649 : cluster 0 osdmap e293: 8 total, 8 up, 8 in 2023-06-03T20:31:17.428 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:31:17 smithi089 bash[78452]: cluster 1685824276.3342388 osd.5 (osd.5) 242 : cluster 3 failed to encode map e293 with expected crc 2023-06-03T20:31:17.428 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:31:17 smithi089 bash[78452]: cluster 1685824276.3370605 osd.6 (osd.6) 228 : cluster 3 failed to encode map e293 with expected crc 2023-06-03T20:31:17.428 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:31:17 smithi089 bash[78452]: cluster 1685824276.3372157 osd.5 (osd.5) 243 : cluster 3 failed to encode map e293 with expected crc 2023-06-03T20:31:17.428 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:31:17 smithi089 bash[78452]: cluster 1685824276.339128 osd.6 (osd.6) 229 : cluster 3 failed to encode map e293 with expected crc 2023-06-03T20:31:17.457 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:31:17.458 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:31:17.458 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:31:17.559 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:31:17.591 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:31:17.595 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:31:17.596 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:31:17.697 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:31:17.731 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:31:17.733 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:31:17.734 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:31:17.836 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:31:17.880 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:31:17.883 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:31:17.883 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:31:17.984 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:31:18.013 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:31:18.015 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:31:18.016 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:31:18.117 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:31:18.153 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:31:18.156 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:31:18.157 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:31:18.258 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:31:18.293 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:31:18.297 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:31:18.298 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:31:18.399 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:31:18.433 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:31:18.439 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:31:18.440 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:31:18.541 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:31:18.551 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:31:18 smithi164 bash[59462]: cluster 1685824277.1951091 mgr.y (mgr.34104) 1665 : cluster 0 pgmap v1721: 89 pgs: 1 active+recovering+remapped, 88 active+clean; 1.6 GiB data, 9.9 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 561 op/s; 1379/78486 objects misplaced (1.757%); 2.2 MiB/s, 35 objects/s recovering 2023-06-03T20:31:18.571 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:31:18.575 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:31:18.576 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:31:18.677 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:31:18.720 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:31:18.726 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:31:18.727 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:31:18.779 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:31:18 smithi089 bash[75068]: cluster 1685824277.1951091 mgr.y (mgr.34104) 1665 : cluster 0 pgmap v1721: 89 pgs: 1 active+recovering+remapped, 88 active+clean; 1.6 GiB data, 9.9 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 561 op/s; 1379/78486 objects misplaced (1.757%); 2.2 MiB/s, 35 objects/s recovering 2023-06-03T20:31:18.780 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:31:18 smithi089 bash[78452]: cluster 1685824277.1951091 mgr.y (mgr.34104) 1665 : cluster 0 pgmap v1721: 89 pgs: 1 active+recovering+remapped, 88 active+clean; 1.6 GiB data, 9.9 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 561 op/s; 1379/78486 objects misplaced (1.757%); 2.2 MiB/s, 35 objects/s recovering 2023-06-03T20:31:18.828 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:31:18.860 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:31:18.868 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:31:18.869 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:31:18.970 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:31:19.002 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:31:19.005 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:31:19.006 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:31:19.107 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:31:19.165 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:31:19.166 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:31:19.167 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:31:19.267 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:31:19.296 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:31:19.298 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:31:19.299 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:31:19.400 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:31:19.450 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:31:19.456 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:31:19.457 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:31:19.558 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:31:19.623 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:31:19.627 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:31:19.628 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:31:19.729 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:31:19.775 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:31:19.779 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:31:19.779 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:31:19.816 INFO:teuthology.orchestra.run.smithi089.stdout: 2023-06-03T20:31:19.821 INFO:teuthology.orchestra.run.smithi089.stderr:dumped all 2023-06-03T20:31:19.881 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:31:19.918 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:31:19.923 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:31:19.924 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:31:20.025 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:31:20.061 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:31:20.064 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:31:20.065 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:31:20.166 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:31:20.209 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:31:20.213 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:31:20.214 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:31:20.315 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:31:20.333 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:31:20 smithi089 bash[75068]: cluster 1685824279.195836 mgr.y (mgr.34104) 1666 : cluster 0 pgmap v1722: 89 pgs: 1 active+recovering+remapped, 88 active+clean; 1.5 GiB data, 9.5 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 533 op/s; 1189/71763 objects misplaced (1.657%); 2.3 MiB/s, 36 objects/s recovering 2023-06-03T20:31:20.348 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:31:20.352 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:31:20.352 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:31:20.454 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:31:20.486 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:31:20.490 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:31:20.490 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:31:20.586 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:31:20 smithi164 bash[59462]: cluster 1685824279.195836 mgr.y (mgr.34104) 1666 : cluster 0 pgmap v1722: 89 pgs: 1 active+recovering+remapped, 88 active+clean; 1.5 GiB data, 9.5 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 533 op/s; 1189/71763 objects misplaced (1.657%); 2.3 MiB/s, 36 objects/s recovering 2023-06-03T20:31:20.592 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:31:20.625 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:31:20.629 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:31:20.630 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:31:20.731 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:31:20.769 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:31:20.773 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:31:20.773 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:31:20.779 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:31:20 smithi089 bash[78452]: cluster 1685824279.195836 mgr.y (mgr.34104) 1666 : cluster 0 pgmap v1722: 89 pgs: 1 active+recovering+remapped, 88 active+clean; 1.5 GiB data, 9.5 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 533 op/s; 1189/71763 objects misplaced (1.657%); 2.3 MiB/s, 36 objects/s recovering 2023-06-03T20:31:20.875 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:31:20.903 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:31:20.912 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:31:20.913 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:31:21.014 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:31:21.045 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:31:21.050 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:31:21.051 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:31:21.153 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:31:21.190 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:31:21.193 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:31:21.194 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:31:21.260 INFO:teuthology.orchestra.run.smithi089.stdout:{"pg_ready":true,"pg_map":{"version":1722,"stamp":"2023-06-03T20:31:19.195336+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":1566644134,"num_objects":23921,"num_object_clones":0,"num_object_copies":71763,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":1189,"num_objects_unfound":0,"num_objects_dirty":23921,"num_whiteouts":0,"num_read":309,"num_read_kb":513,"num_write":172392,"num_write_kb":6275279,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3906,"num_bytes_recovered":255859032,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":141626,"ondisk_log_size":141626,"up":267,"acting":267,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":268,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":10010576,"kb_used_data":4758864,"kb_used_omap":30,"kb_used_meta":5251297,"kb_avail":739950640,"statfs":{"total":767960285184,"available":757709455360,"internally_reserved":0,"allocated":4873076736,"data_stored":4865339049,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":31502,"internal_metadata":5377328370},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,4,5,5,1,1],"upper_bound":128},"perf_stat":{"commit_latency_ms":6,"apply_latency_ms":6,"commit_latency_ns":6000000,"apply_latency_ns":6000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-349765632,"num_objects":-5337,"num_object_clones":0,"num_object_copies":-16011,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":-365,"num_objects_unfound":0,"num_objects_dirty":-5337,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5337,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":365,"num_bytes_recovered":23920640,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.002959"},"pg_stats":[{"pgid":"3.1a","version":"60'1","reported_seq":220,"reported_epoch":292,"state":"active+clean","last_fresh":"2023-06-03T20:31:01.426038+0000","last_change":"2023-06-03T20:23:28.242261+0000","last_active":"2023-06-03T20:31:01.426038+0000","last_peered":"2023-06-03T20:31:01.426038+0000","last_clean":"2023-06-03T20:31:01.426038+0000","last_became_active":"2023-06-03T20:23:28.240045+0000","last_became_peered":"2023-06-03T20:23:28.240045+0000","last_unstale":"2023-06-03T20:31:01.426038+0000","last_undegraded":"2023-06-03T20:31:01.426038+0000","last_fullsized":"2023-06-03T20:31:01.426038+0000","mapping_epoch":251,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":252,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T05:54:33.215385+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":81,"reported_epoch":292,"state":"active+clean","last_fresh":"2023-06-03T20:31:02.854664+0000","last_change":"2023-06-03T20:23:28.283021+0000","last_active":"2023-06-03T20:31:02.854664+0000","last_peered":"2023-06-03T20:31:02.854664+0000","last_clean":"2023-06-03T20:31:02.854664+0000","last_became_active":"2023-06-03T20:23:28.282428+0000","last_became_peered":"2023-06-03T20:23:28.282428+0000","last_unstale":"2023-06-03T20:31:02.854664+0000","last_undegraded":"2023-06-03T20:31:02.854664+0000","last_fullsized":"2023-06-03T20:31:02.854664+0000","mapping_epoch":251,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":252,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T05:01:33.115332+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":81,"reported_epoch":292,"state":"active+clean","last_fresh":"2023-06-03T20:30:47.844345+0000","last_change":"2023-06-03T20:23:28.253821+0000","last_active":"2023-06-03T20:30:47.844345+0000","last_peered":"2023-06-03T20:30:47.844345+0000","last_clean":"2023-06-03T20:30:47.844345+0000","last_became_active":"2023-06-03T20:23:28.253448+0000","last_became_peered":"2023-06-03T20:23:28.253448+0000","last_unstale":"2023-06-03T20:30:47.844345+0000","last_undegraded":"2023-06-03T20:30:47.844345+0000","last_fullsized":"2023-06-03T20:30:47.844345+0000","mapping_epoch":251,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":252,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T23:40:02.264778+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":264,"reported_epoch":292,"state":"active+clean","last_fresh":"2023-06-03T20:31:13.493086+0000","last_change":"2023-06-03T19:56:54.507653+0000","last_active":"2023-06-03T20:31:13.493086+0000","last_peered":"2023-06-03T20:31:13.493086+0000","last_clean":"2023-06-03T20:31:13.493086+0000","last_became_active":"2023-06-03T19:56:54.507350+0000","last_became_peered":"2023-06-03T19:56:54.507350+0000","last_unstale":"2023-06-03T20:31:13.493086+0000","last_undegraded":"2023-06-03T20:31:13.493086+0000","last_fullsized":"2023-06-03T20:31:13.493086+0000","mapping_epoch":119,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T22:33:20.065844+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":304,"reported_epoch":292,"state":"active+clean","last_fresh":"2023-06-03T20:30:47.935523+0000","last_change":"2023-06-03T20:21:01.256894+0000","last_active":"2023-06-03T20:30:47.935523+0000","last_peered":"2023-06-03T20:30:47.935523+0000","last_clean":"2023-06-03T20:30:47.935523+0000","last_became_active":"2023-06-03T20:21:01.255903+0000","last_became_peered":"2023-06-03T20:21:01.255903+0000","last_unstale":"2023-06-03T20:30:47.935523+0000","last_undegraded":"2023-06-03T20:30:47.935523+0000","last_fullsized":"2023-06-03T20:30:47.935523+0000","mapping_epoch":235,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":236,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T20:35:27.810860+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":57,"reported_epoch":292,"state":"active+clean","last_fresh":"2023-06-03T20:30:38.288542+0000","last_change":"2023-06-03T20:28:24.269355+0000","last_active":"2023-06-03T20:30:38.288542+0000","last_peered":"2023-06-03T20:30:38.288542+0000","last_clean":"2023-06-03T20:30:38.288542+0000","last_became_active":"2023-06-03T20:28:24.268859+0000","last_became_peered":"2023-06-03T20:28:24.268859+0000","last_unstale":"2023-06-03T20:30:38.288542+0000","last_undegraded":"2023-06-03T20:30:38.288542+0000","last_fullsized":"2023-06-03T20:30:38.288542+0000","mapping_epoch":284,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":285,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,2],"acting":[6,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":259,"reported_epoch":292,"state":"active+clean","last_fresh":"2023-06-03T20:30:38.338221+0000","last_change":"2023-06-03T19:56:54.509490+0000","last_active":"2023-06-03T20:30:38.338221+0000","last_peered":"2023-06-03T20:30:38.338221+0000","last_clean":"2023-06-03T20:30:38.338221+0000","last_became_active":"2023-06-03T19:56:54.509145+0000","last_became_peered":"2023-06-03T19:56:54.509145+0000","last_unstale":"2023-06-03T20:30:38.338221+0000","last_undegraded":"2023-06-03T20:30:38.338221+0000","last_fullsized":"2023-06-03T20:30:38.338221+0000","mapping_epoch":119,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T07:12:03.964110+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":324,"reported_epoch":292,"state":"active+clean","last_fresh":"2023-06-03T20:30:38.305901+0000","last_change":"2023-06-03T20:28:51.233277+0000","last_active":"2023-06-03T20:30:38.305901+0000","last_peered":"2023-06-03T20:30:38.305901+0000","last_clean":"2023-06-03T20:30:38.305901+0000","last_became_active":"2023-06-03T20:28:51.232919+0000","last_became_peered":"2023-06-03T20:28:51.232919+0000","last_unstale":"2023-06-03T20:30:38.305901+0000","last_undegraded":"2023-06-03T20:30:38.305901+0000","last_fullsized":"2023-06-03T20:30:38.305901+0000","mapping_epoch":286,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":287,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T02:06:02.736415+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":225,"reported_epoch":292,"state":"active+clean","last_fresh":"2023-06-03T20:30:38.306008+0000","last_change":"2023-06-03T20:21:01.258441+0000","last_active":"2023-06-03T20:30:38.306008+0000","last_peered":"2023-06-03T20:30:38.306008+0000","last_clean":"2023-06-03T20:30:38.306008+0000","last_became_active":"2023-06-03T20:21:01.257779+0000","last_became_peered":"2023-06-03T20:21:01.257779+0000","last_unstale":"2023-06-03T20:30:38.306008+0000","last_undegraded":"2023-06-03T20:30:38.306008+0000","last_fullsized":"2023-06-03T20:30:38.306008+0000","mapping_epoch":235,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":236,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,1],"acting":[7,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":269,"reported_epoch":292,"state":"active+clean","last_fresh":"2023-06-03T20:30:38.308366+0000","last_change":"2023-06-03T20:24:02.267331+0000","last_active":"2023-06-03T20:30:38.308366+0000","last_peered":"2023-06-03T20:30:38.308366+0000","last_clean":"2023-06-03T20:30:38.308366+0000","last_became_active":"2023-06-03T20:24:02.266997+0000","last_became_peered":"2023-06-03T20:24:02.266997+0000","last_unstale":"2023-06-03T20:30:38.308366+0000","last_undegraded":"2023-06-03T20:30:38.308366+0000","last_fullsized":"2023-06-03T20:30:38.308366+0000","mapping_epoch":254,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":255,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T22:09:18.993656+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"60'3","reported_seq":290,"reported_epoch":292,"state":"active+clean","last_fresh":"2023-06-03T20:30:38.306357+0000","last_change":"2023-06-03T20:23:27.261246+0000","last_active":"2023-06-03T20:30:38.306357+0000","last_peered":"2023-06-03T20:30:38.306357+0000","last_clean":"2023-06-03T20:30:38.306357+0000","last_became_active":"2023-06-03T20:23:27.259201+0000","last_became_peered":"2023-06-03T20:23:27.259201+0000","last_unstale":"2023-06-03T20:30:38.306357+0000","last_undegraded":"2023-06-03T20:30:38.306357+0000","last_fullsized":"2023-06-03T20:30:38.306357+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":81,"reported_epoch":292,"state":"active+clean","last_fresh":"2023-06-03T20:30:38.298519+0000","last_change":"2023-06-03T20:28:51.235184+0000","last_active":"2023-06-03T20:30:38.298519+0000","last_peered":"2023-06-03T20:30:38.298519+0000","last_clean":"2023-06-03T20:30:38.298519+0000","last_became_active":"2023-06-03T20:28:51.234830+0000","last_became_peered":"2023-06-03T20:28:51.234830+0000","last_unstale":"2023-06-03T20:30:38.298519+0000","last_undegraded":"2023-06-03T20:30:38.298519+0000","last_fullsized":"2023-06-03T20:30:38.298519+0000","mapping_epoch":286,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":287,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T04:40:02.479523+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"77'23","reported_seq":227,"reported_epoch":292,"state":"active+clean","last_fresh":"2023-06-03T20:30:38.288860+0000","last_change":"2023-06-03T20:24:02.280274+0000","last_active":"2023-06-03T20:30:38.288860+0000","last_peered":"2023-06-03T20:30:38.288860+0000","last_clean":"2023-06-03T20:30:38.288860+0000","last_became_active":"2023-06-03T20:24:02.264914+0000","last_became_peered":"2023-06-03T20:24:02.264914+0000","last_unstale":"2023-06-03T20:30:38.288860+0000","last_undegraded":"2023-06-03T20:30:38.288860+0000","last_fullsized":"2023-06-03T20:30:38.288860+0000","mapping_epoch":254,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":255,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":23,"log_dups_size":0,"ondisk_log_size":23,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T07:00:35.888220+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4716,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":23,"num_write_kb":20,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4716,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":203,"reported_epoch":292,"state":"active+clean","last_fresh":"2023-06-03T20:30:38.288455+0000","last_change":"2023-06-03T20:24:02.254850+0000","last_active":"2023-06-03T20:30:38.288455+0000","last_peered":"2023-06-03T20:30:38.288455+0000","last_clean":"2023-06-03T20:30:38.288455+0000","last_became_active":"2023-06-03T20:24:02.254409+0000","last_became_peered":"2023-06-03T20:24:02.254409+0000","last_unstale":"2023-06-03T20:30:38.288455+0000","last_undegraded":"2023-06-03T20:30:38.288455+0000","last_fullsized":"2023-06-03T20:30:38.288455+0000","mapping_epoch":254,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":255,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T19:49:53.393121+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"60'2","reported_seq":292,"reported_epoch":292,"state":"active+clean","last_fresh":"2023-06-03T20:30:38.306201+0000","last_change":"2023-06-03T20:23:27.258749+0000","last_active":"2023-06-03T20:30:38.306201+0000","last_peered":"2023-06-03T20:30:38.306201+0000","last_clean":"2023-06-03T20:30:38.306201+0000","last_became_active":"2023-06-03T20:23:27.256899+0000","last_became_peered":"2023-06-03T20:23:27.256899+0000","last_unstale":"2023-06-03T20:30:38.306201+0000","last_undegraded":"2023-06-03T20:30:38.306201+0000","last_fullsized":"2023-06-03T20:30:38.306201+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":288,"reported_epoch":292,"state":"active+clean","last_fresh":"2023-06-03T20:30:38.307050+0000","last_change":"2023-06-03T20:21:01.256612+0000","last_active":"2023-06-03T20:30:38.307050+0000","last_peered":"2023-06-03T20:30:38.307050+0000","last_clean":"2023-06-03T20:30:38.307050+0000","last_became_active":"2023-06-03T20:21:01.256243+0000","last_became_peered":"2023-06-03T20:21:01.256243+0000","last_unstale":"2023-06-03T20:30:38.307050+0000","last_undegraded":"2023-06-03T20:30:38.307050+0000","last_fullsized":"2023-06-03T20:30:38.307050+0000","mapping_epoch":235,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":236,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T05:24:47.054342+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":259,"reported_epoch":292,"state":"active+clean","last_fresh":"2023-06-03T20:30:38.312395+0000","last_change":"2023-06-03T19:56:54.507479+0000","last_active":"2023-06-03T20:30:38.312395+0000","last_peered":"2023-06-03T20:30:38.312395+0000","last_clean":"2023-06-03T20:30:38.312395+0000","last_became_active":"2023-06-03T19:56:54.507149+0000","last_became_peered":"2023-06-03T19:56:54.507149+0000","last_unstale":"2023-06-03T20:30:38.312395+0000","last_undegraded":"2023-06-03T20:30:38.312395+0000","last_fullsized":"2023-06-03T20:30:38.312395+0000","mapping_epoch":119,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T20:22:36.676317+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":159,"reported_epoch":292,"state":"active+clean","last_fresh":"2023-06-03T20:30:38.306500+0000","last_change":"2023-06-03T20:17:22.832844+0000","last_active":"2023-06-03T20:30:38.306500+0000","last_peered":"2023-06-03T20:30:38.306500+0000","last_clean":"2023-06-03T20:30:38.306500+0000","last_became_active":"2023-06-03T20:17:22.832473+0000","last_became_peered":"2023-06-03T20:17:22.832473+0000","last_unstale":"2023-06-03T20:30:38.306500+0000","last_undegraded":"2023-06-03T20:30:38.306500+0000","last_fullsized":"2023-06-03T20:30:38.306500+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,3],"acting":[7,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":292,"reported_epoch":292,"state":"active+clean","last_fresh":"2023-06-03T20:30:38.305889+0000","last_change":"2023-06-03T20:24:02.267531+0000","last_active":"2023-06-03T20:30:38.305889+0000","last_peered":"2023-06-03T20:30:38.305889+0000","last_clean":"2023-06-03T20:30:38.305889+0000","last_became_active":"2023-06-03T20:24:02.267159+0000","last_became_peered":"2023-06-03T20:24:02.267159+0000","last_unstale":"2023-06-03T20:30:38.305889+0000","last_undegraded":"2023-06-03T20:30:38.305889+0000","last_fullsized":"2023-06-03T20:30:38.305889+0000","mapping_epoch":254,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":255,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":260,"reported_epoch":292,"state":"active+clean","last_fresh":"2023-06-03T20:30:38.322998+0000","last_change":"2023-06-03T19:56:54.510289+0000","last_active":"2023-06-03T20:30:38.322998+0000","last_peered":"2023-06-03T20:30:38.322998+0000","last_clean":"2023-06-03T20:30:38.322998+0000","last_became_active":"2023-06-03T19:56:54.509905+0000","last_became_peered":"2023-06-03T19:56:54.509905+0000","last_unstale":"2023-06-03T20:30:38.322998+0000","last_undegraded":"2023-06-03T20:30:38.322998+0000","last_fullsized":"2023-06-03T20:30:38.322998+0000","mapping_epoch":119,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T02:54:26.792401+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"60'8","reported_seq":58,"reported_epoch":292,"state":"active+clean","last_fresh":"2023-06-03T20:30:38.288154+0000","last_change":"2023-06-03T20:26:59.682623+0000","last_active":"2023-06-03T20:30:38.288154+0000","last_peered":"2023-06-03T20:30:38.288154+0000","last_clean":"2023-06-03T20:30:38.288154+0000","last_became_active":"2023-06-03T20:24:02.260105+0000","last_became_peered":"2023-06-03T20:24:02.260105+0000","last_unstale":"2023-06-03T20:30:38.288154+0000","last_undegraded":"2023-06-03T20:30:38.288154+0000","last_fullsized":"2023-06-03T20:30:38.288154+0000","mapping_epoch":254,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":273,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":297,"reported_epoch":292,"state":"active+clean","last_fresh":"2023-06-03T20:30:42.935110+0000","last_change":"2023-06-03T20:28:24.281308+0000","last_active":"2023-06-03T20:30:42.935110+0000","last_peered":"2023-06-03T20:30:42.935110+0000","last_clean":"2023-06-03T20:30:42.935110+0000","last_became_active":"2023-06-03T20:28:24.280981+0000","last_became_peered":"2023-06-03T20:28:24.280981+0000","last_unstale":"2023-06-03T20:30:42.935110+0000","last_undegraded":"2023-06-03T20:30:42.935110+0000","last_fullsized":"2023-06-03T20:30:42.935110+0000","mapping_epoch":284,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":285,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T02:44:14.890147+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"14.2","version":"293'10547","reported_seq":10570,"reported_epoch":293,"state":"active+clean","last_fresh":"2023-06-03T20:31:17.960120+0000","last_change":"2023-06-03T20:27:21.371612+0000","last_active":"2023-06-03T20:31:17.960120+0000","last_peered":"2023-06-03T20:31:17.960120+0000","last_clean":"2023-06-03T20:31:17.960120+0000","last_became_active":"2023-06-03T20:27:21.371344+0000","last_became_peered":"2023-06-03T20:27:21.371344+0000","last_unstale":"2023-06-03T20:31:17.960120+0000","last_undegraded":"2023-06-03T20:31:17.960120+0000","last_fullsized":"2023-06-03T20:31:17.960120+0000","mapping_epoch":278,"log_start":"281'800","ondisk_log_start":"281'800","created":278,"last_epoch_clean":279,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T20:27:20.339366+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T20:27:20.339366+0000","last_clean_scrub_stamp":"2023-06-03T20:27:20.339366+0000","objects_scrubbed":0,"log_size":9747,"log_dups_size":0,"ondisk_log_size":9747,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":97189888,"num_objects":1483,"num_object_clones":0,"num_object_copies":4449,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1483,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10547,"num_write_kb":384960,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,3],"acting":[6,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.f","version":"60'1","reported_seq":291,"reported_epoch":292,"state":"active+clean","last_fresh":"2023-06-03T20:30:38.306689+0000","last_change":"2023-06-03T20:23:27.263667+0000","last_active":"2023-06-03T20:30:38.306689+0000","last_peered":"2023-06-03T20:30:38.306689+0000","last_clean":"2023-06-03T20:30:38.306689+0000","last_became_active":"2023-06-03T20:23:27.263110+0000","last_became_peered":"2023-06-03T20:23:27.263110+0000","last_unstale":"2023-06-03T20:30:38.306689+0000","last_undegraded":"2023-06-03T20:30:38.306689+0000","last_fullsized":"2023-06-03T20:30:38.306689+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":252,"reported_epoch":292,"state":"active+clean","last_fresh":"2023-06-03T20:30:38.305875+0000","last_change":"2023-06-03T20:28:51.231222+0000","last_active":"2023-06-03T20:30:38.305875+0000","last_peered":"2023-06-03T20:30:38.305875+0000","last_clean":"2023-06-03T20:30:38.305875+0000","last_became_active":"2023-06-03T20:28:51.230848+0000","last_became_peered":"2023-06-03T20:28:51.230848+0000","last_unstale":"2023-06-03T20:30:38.305875+0000","last_undegraded":"2023-06-03T20:30:38.305875+0000","last_fullsized":"2023-06-03T20:30:38.305875+0000","mapping_epoch":286,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":287,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"14.3","version":"293'10841","reported_seq":10866,"reported_epoch":293,"state":"active+clean","last_fresh":"2023-06-03T20:31:18.444488+0000","last_change":"2023-06-03T20:27:21.374055+0000","last_active":"2023-06-03T20:31:18.444488+0000","last_peered":"2023-06-03T20:31:18.444488+0000","last_clean":"2023-06-03T20:31:18.444488+0000","last_became_active":"2023-06-03T20:27:21.373728+0000","last_became_peered":"2023-06-03T20:27:21.373728+0000","last_unstale":"2023-06-03T20:31:18.444488+0000","last_undegraded":"2023-06-03T20:31:18.444488+0000","last_fullsized":"2023-06-03T20:31:18.444488+0000","mapping_epoch":278,"log_start":"281'2000","ondisk_log_start":"281'2000","created":278,"last_epoch_clean":279,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T20:27:20.339366+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T20:27:20.339366+0000","last_clean_scrub_stamp":"2023-06-03T20:27:20.339366+0000","objects_scrubbed":0,"log_size":8841,"log_dups_size":0,"ondisk_log_size":8841,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T05:15:33.026018+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":98238464,"num_objects":1499,"num_object_clones":0,"num_object_copies":4497,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1499,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10841,"num_write_kb":394880,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.e","version":"60'2","reported_seq":320,"reported_epoch":292,"state":"active+clean","last_fresh":"2023-06-03T20:30:38.306134+0000","last_change":"2023-06-03T20:23:28.606751+0000","last_active":"2023-06-03T20:30:38.306134+0000","last_peered":"2023-06-03T20:30:38.306134+0000","last_clean":"2023-06-03T20:30:38.306134+0000","last_became_active":"2023-06-03T20:23:28.605968+0000","last_became_peered":"2023-06-03T20:23:28.605968+0000","last_unstale":"2023-06-03T20:30:38.306134+0000","last_undegraded":"2023-06-03T20:30:38.306134+0000","last_fullsized":"2023-06-03T20:30:38.306134+0000","mapping_epoch":251,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":252,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":232,"reported_epoch":292,"state":"active+clean","last_fresh":"2023-06-03T20:30:38.289225+0000","last_change":"2023-06-03T20:23:28.249694+0000","last_active":"2023-06-03T20:30:38.289225+0000","last_peered":"2023-06-03T20:30:38.289225+0000","last_clean":"2023-06-03T20:30:38.289225+0000","last_became_active":"2023-06-03T20:23:28.249388+0000","last_became_peered":"2023-06-03T20:23:28.249388+0000","last_unstale":"2023-06-03T20:30:38.289225+0000","last_undegraded":"2023-06-03T20:30:38.289225+0000","last_fullsized":"2023-06-03T20:30:38.289225+0000","mapping_epoch":251,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":252,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T05:44:01.835586+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"14.0","version":"293'10629","reported_seq":10654,"reported_epoch":293,"state":"active+clean","last_fresh":"2023-06-03T20:31:17.792951+0000","last_change":"2023-06-03T20:27:21.378346+0000","last_active":"2023-06-03T20:31:17.792951+0000","last_peered":"2023-06-03T20:31:17.792951+0000","last_clean":"2023-06-03T20:31:17.792951+0000","last_became_active":"2023-06-03T20:27:21.377969+0000","last_became_peered":"2023-06-03T20:27:21.377969+0000","last_unstale":"2023-06-03T20:31:17.792951+0000","last_undegraded":"2023-06-03T20:31:17.792951+0000","last_fullsized":"2023-06-03T20:31:17.792951+0000","mapping_epoch":278,"log_start":"281'600","ondisk_log_start":"281'600","created":278,"last_epoch_clean":279,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T20:27:20.339366+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T20:27:20.339366+0000","last_clean_scrub_stamp":"2023-06-03T20:27:20.339366+0000","objects_scrubbed":0,"log_size":10029,"log_dups_size":0,"ondisk_log_size":10029,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T01:32:49.543336+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":95354880,"num_objects":1455,"num_object_clones":0,"num_object_copies":4365,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1455,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10629,"num_write_kb":386688,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,5],"acting":[2,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.d","version":"60'1","reported_seq":269,"reported_epoch":292,"state":"active+clean","last_fresh":"2023-06-03T20:30:38.306058+0000","last_change":"2023-06-03T20:26:59.686037+0000","last_active":"2023-06-03T20:30:38.306058+0000","last_peered":"2023-06-03T20:30:38.306058+0000","last_clean":"2023-06-03T20:30:38.306058+0000","last_became_active":"2023-06-03T20:24:02.268002+0000","last_became_peered":"2023-06-03T20:24:02.268002+0000","last_unstale":"2023-06-03T20:30:38.306058+0000","last_undegraded":"2023-06-03T20:30:38.306058+0000","last_fullsized":"2023-06-03T20:30:38.306058+0000","mapping_epoch":254,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":273,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":140,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":16,"reported_epoch":292,"state":"active+clean","last_fresh":"2023-06-03T20:30:38.288262+0000","last_change":"2023-06-03T20:28:51.230833+0000","last_active":"2023-06-03T20:30:38.288262+0000","last_peered":"2023-06-03T20:30:38.288262+0000","last_clean":"2023-06-03T20:30:38.288262+0000","last_became_active":"2023-06-03T20:28:51.230489+0000","last_became_peered":"2023-06-03T20:28:51.230489+0000","last_unstale":"2023-06-03T20:30:38.288262+0000","last_undegraded":"2023-06-03T20:30:38.288262+0000","last_fullsized":"2023-06-03T20:30:38.288262+0000","mapping_epoch":286,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":287,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"14.1","version":"293'11206","reported_seq":11231,"reported_epoch":293,"state":"active+clean","last_fresh":"2023-06-03T20:31:18.001389+0000","last_change":"2023-06-03T20:27:21.372641+0000","last_active":"2023-06-03T20:31:18.001389+0000","last_peered":"2023-06-03T20:31:18.001389+0000","last_clean":"2023-06-03T20:31:18.001389+0000","last_became_active":"2023-06-03T20:27:21.372158+0000","last_became_peered":"2023-06-03T20:27:21.372158+0000","last_unstale":"2023-06-03T20:31:18.001389+0000","last_undegraded":"2023-06-03T20:31:18.001389+0000","last_fullsized":"2023-06-03T20:31:18.001389+0000","mapping_epoch":278,"log_start":"281'3000","ondisk_log_start":"281'3000","created":278,"last_epoch_clean":279,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T20:27:20.339366+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T20:27:20.339366+0000","last_clean_scrub_stamp":"2023-06-03T20:27:20.339366+0000","objects_scrubbed":0,"log_size":8206,"log_dups_size":0,"ondisk_log_size":8206,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T07:52:41.694374+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":100663296,"num_objects":1536,"num_object_clones":0,"num_object_copies":4608,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1536,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11206,"num_write_kb":407744,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":286,"reported_epoch":292,"state":"active+clean","last_fresh":"2023-06-03T20:30:52.971799+0000","last_change":"2023-06-03T20:24:02.261252+0000","last_active":"2023-06-03T20:30:52.971799+0000","last_peered":"2023-06-03T20:30:52.971799+0000","last_clean":"2023-06-03T20:30:52.971799+0000","last_became_active":"2023-06-03T20:24:02.260824+0000","last_became_peered":"2023-06-03T20:24:02.260824+0000","last_unstale":"2023-06-03T20:30:52.971799+0000","last_undegraded":"2023-06-03T20:30:52.971799+0000","last_fullsized":"2023-06-03T20:30:52.971799+0000","mapping_epoch":254,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":255,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T20:00:53.584753+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":323,"reported_epoch":292,"state":"active+clean","last_fresh":"2023-06-03T20:30:38.315781+0000","last_change":"2023-06-03T20:23:28.283360+0000","last_active":"2023-06-03T20:30:38.315781+0000","last_peered":"2023-06-03T20:30:38.315781+0000","last_clean":"2023-06-03T20:30:38.315781+0000","last_became_active":"2023-06-03T20:23:28.283035+0000","last_became_peered":"2023-06-03T20:23:28.283035+0000","last_unstale":"2023-06-03T20:30:38.315781+0000","last_undegraded":"2023-06-03T20:30:38.315781+0000","last_fullsized":"2023-06-03T20:30:38.315781+0000","mapping_epoch":251,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":252,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T00:22:59.446515+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"14.6","version":"293'10807","reported_seq":10832,"reported_epoch":293,"state":"active+clean","last_fresh":"2023-06-03T20:31:18.495958+0000","last_change":"2023-06-03T20:27:21.375836+0000","last_active":"2023-06-03T20:31:18.495958+0000","last_peered":"2023-06-03T20:31:18.495958+0000","last_clean":"2023-06-03T20:31:18.495958+0000","last_became_active":"2023-06-03T20:27:21.375522+0000","last_became_peered":"2023-06-03T20:27:21.375522+0000","last_unstale":"2023-06-03T20:31:18.495958+0000","last_undegraded":"2023-06-03T20:31:18.495958+0000","last_fullsized":"2023-06-03T20:31:18.495958+0000","mapping_epoch":278,"log_start":"281'1900","ondisk_log_start":"281'1900","created":278,"last_epoch_clean":279,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T20:27:20.339366+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T20:27:20.339366+0000","last_clean_scrub_stamp":"2023-06-03T20:27:20.339366+0000","objects_scrubbed":0,"log_size":8907,"log_dups_size":0,"ondisk_log_size":8907,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T21:02:49.138563+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":96534528,"num_objects":1473,"num_object_clones":0,"num_object_copies":4419,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1473,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10807,"num_write_kb":392960,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,1],"acting":[0,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.b","version":"60'2","reported_seq":287,"reported_epoch":292,"state":"active+clean","last_fresh":"2023-06-03T20:30:38.317531+0000","last_change":"2023-06-03T20:23:27.254179+0000","last_active":"2023-06-03T20:30:38.317531+0000","last_peered":"2023-06-03T20:30:38.317531+0000","last_clean":"2023-06-03T20:30:38.317531+0000","last_became_active":"2023-06-03T20:23:27.253791+0000","last_became_peered":"2023-06-03T20:23:27.253791+0000","last_unstale":"2023-06-03T20:30:38.317531+0000","last_undegraded":"2023-06-03T20:30:38.317531+0000","last_fullsized":"2023-06-03T20:30:38.317531+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T00:31:27.336760+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":208,"reported_epoch":292,"state":"active+clean","last_fresh":"2023-06-03T20:30:38.288700+0000","last_change":"2023-06-03T20:24:02.255551+0000","last_active":"2023-06-03T20:30:38.288700+0000","last_peered":"2023-06-03T20:30:38.288700+0000","last_clean":"2023-06-03T20:30:38.288700+0000","last_became_active":"2023-06-03T20:24:02.255209+0000","last_became_peered":"2023-06-03T20:24:02.255209+0000","last_unstale":"2023-06-03T20:30:38.288700+0000","last_undegraded":"2023-06-03T20:30:38.288700+0000","last_fullsized":"2023-06-03T20:30:38.288700+0000","mapping_epoch":254,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":255,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T20:05:02.741217+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"14.7","version":"293'10759","reported_seq":10784,"reported_epoch":293,"state":"active+clean","last_fresh":"2023-06-03T20:31:16.351813+0000","last_change":"2023-06-03T20:27:21.364098+0000","last_active":"2023-06-03T20:31:16.351813+0000","last_peered":"2023-06-03T20:31:16.351813+0000","last_clean":"2023-06-03T20:31:16.351813+0000","last_became_active":"2023-06-03T20:27:21.363548+0000","last_became_peered":"2023-06-03T20:27:21.363548+0000","last_unstale":"2023-06-03T20:31:16.351813+0000","last_undegraded":"2023-06-03T20:31:16.351813+0000","last_fullsized":"2023-06-03T20:31:16.351813+0000","mapping_epoch":278,"log_start":"281'3000","ondisk_log_start":"281'3000","created":278,"last_epoch_clean":279,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T20:27:20.339366+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T20:27:20.339366+0000","last_clean_scrub_stamp":"2023-06-03T20:27:20.339366+0000","objects_scrubbed":0,"log_size":7759,"log_dups_size":0,"ondisk_log_size":7759,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T06:05:40.316567+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":102694912,"num_objects":1567,"num_object_clones":0,"num_object_copies":4701,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1567,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10759,"num_write_kb":394432,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,0],"acting":[4,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":48,"reported_epoch":292,"state":"active+clean","last_fresh":"2023-06-03T20:30:38.291217+0000","last_change":"2023-06-03T20:24:02.254172+0000","last_active":"2023-06-03T20:30:38.291217+0000","last_peered":"2023-06-03T20:30:38.291217+0000","last_clean":"2023-06-03T20:30:38.291217+0000","last_became_active":"2023-06-03T20:24:02.253797+0000","last_became_peered":"2023-06-03T20:24:02.253797+0000","last_unstale":"2023-06-03T20:30:38.291217+0000","last_undegraded":"2023-06-03T20:30:38.291217+0000","last_fullsized":"2023-06-03T20:30:38.291217+0000","mapping_epoch":254,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":255,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":213,"reported_epoch":292,"state":"active+clean","last_fresh":"2023-06-03T20:30:38.289279+0000","last_change":"2023-06-03T20:23:28.291250+0000","last_active":"2023-06-03T20:30:38.289279+0000","last_peered":"2023-06-03T20:30:38.289279+0000","last_clean":"2023-06-03T20:30:38.289279+0000","last_became_active":"2023-06-03T20:23:28.290120+0000","last_became_peered":"2023-06-03T20:23:28.290120+0000","last_unstale":"2023-06-03T20:30:38.289279+0000","last_undegraded":"2023-06-03T20:30:38.289279+0000","last_fullsized":"2023-06-03T20:30:38.289279+0000","mapping_epoch":251,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":252,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T22:37:56.033327+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"14.4","version":"293'10729","reported_seq":10752,"reported_epoch":293,"state":"active+clean","last_fresh":"2023-06-03T20:31:17.992736+0000","last_change":"2023-06-03T20:27:21.366337+0000","last_active":"2023-06-03T20:31:17.992736+0000","last_peered":"2023-06-03T20:31:17.992736+0000","last_clean":"2023-06-03T20:31:17.992736+0000","last_became_active":"2023-06-03T20:27:21.365886+0000","last_became_peered":"2023-06-03T20:27:21.365886+0000","last_unstale":"2023-06-03T20:31:17.992736+0000","last_undegraded":"2023-06-03T20:31:17.992736+0000","last_fullsized":"2023-06-03T20:31:17.992736+0000","mapping_epoch":278,"log_start":"281'1000","ondisk_log_start":"281'1000","created":278,"last_epoch_clean":279,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T20:27:20.339366+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T20:27:20.339366+0000","last_clean_scrub_stamp":"2023-06-03T20:27:20.339366+0000","objects_scrubbed":0,"log_size":9729,"log_dups_size":0,"ondisk_log_size":9729,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":96272384,"num_objects":1469,"num_object_clones":0,"num_object_copies":4407,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1469,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10729,"num_write_kb":390336,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,3],"acting":[6,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":198,"reported_epoch":292,"state":"active+clean","last_fresh":"2023-06-03T20:30:38.289112+0000","last_change":"2023-06-03T20:23:27.276693+0000","last_active":"2023-06-03T20:30:38.289112+0000","last_peered":"2023-06-03T20:30:38.289112+0000","last_clean":"2023-06-03T20:30:38.289112+0000","last_became_active":"2023-06-03T20:23:27.276305+0000","last_became_peered":"2023-06-03T20:23:27.276305+0000","last_unstale":"2023-06-03T20:30:38.289112+0000","last_undegraded":"2023-06-03T20:30:38.289112+0000","last_fullsized":"2023-06-03T20:30:38.289112+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T20:34:50.944936+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":203,"reported_epoch":292,"state":"active+clean","last_fresh":"2023-06-03T20:30:38.288845+0000","last_change":"2023-06-03T20:28:51.231829+0000","last_active":"2023-06-03T20:30:38.288845+0000","last_peered":"2023-06-03T20:30:38.288845+0000","last_clean":"2023-06-03T20:30:38.288845+0000","last_became_active":"2023-06-03T20:28:51.231452+0000","last_became_peered":"2023-06-03T20:28:51.231452+0000","last_unstale":"2023-06-03T20:30:38.288845+0000","last_undegraded":"2023-06-03T20:30:38.288845+0000","last_fullsized":"2023-06-03T20:30:38.288845+0000","mapping_epoch":286,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":287,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T07:00:21.647058+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"14.5","version":"293'10901","reported_seq":10926,"reported_epoch":293,"state":"active+clean","last_fresh":"2023-06-03T20:31:18.485630+0000","last_change":"2023-06-03T20:27:21.371070+0000","last_active":"2023-06-03T20:31:18.485630+0000","last_peered":"2023-06-03T20:31:18.485630+0000","last_clean":"2023-06-03T20:31:18.485630+0000","last_became_active":"2023-06-03T20:27:21.370790+0000","last_became_peered":"2023-06-03T20:27:21.370790+0000","last_unstale":"2023-06-03T20:31:18.485630+0000","last_undegraded":"2023-06-03T20:31:18.485630+0000","last_fullsized":"2023-06-03T20:31:18.485630+0000","mapping_epoch":278,"log_start":"281'2000","ondisk_log_start":"281'2000","created":278,"last_epoch_clean":279,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T20:27:20.339366+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T20:27:20.339366+0000","last_clean_scrub_stamp":"2023-06-03T20:27:20.339366+0000","objects_scrubbed":0,"log_size":8901,"log_dups_size":0,"ondisk_log_size":8901,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T21:04:21.657715+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":98107392,"num_objects":1497,"num_object_clones":0,"num_object_copies":4491,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1497,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10901,"num_write_kb":396736,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":296,"reported_epoch":292,"state":"active+clean","last_fresh":"2023-06-03T20:30:38.305958+0000","last_change":"2023-06-03T20:21:01.256958+0000","last_active":"2023-06-03T20:30:38.305958+0000","last_peered":"2023-06-03T20:30:38.305958+0000","last_clean":"2023-06-03T20:30:38.305958+0000","last_became_active":"2023-06-03T20:21:01.255452+0000","last_became_peered":"2023-06-03T20:21:01.255452+0000","last_unstale":"2023-06-03T20:30:38.305958+0000","last_undegraded":"2023-06-03T20:30:38.305958+0000","last_fullsized":"2023-06-03T20:30:38.305958+0000","mapping_epoch":235,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":236,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T01:56:59.777763+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":83,"reported_epoch":292,"state":"active+clean","last_fresh":"2023-06-03T20:30:38.306298+0000","last_change":"2023-06-03T20:23:28.250884+0000","last_active":"2023-06-03T20:30:38.306298+0000","last_peered":"2023-06-03T20:30:38.306298+0000","last_clean":"2023-06-03T20:30:38.306298+0000","last_became_active":"2023-06-03T20:23:28.243793+0000","last_became_peered":"2023-06-03T20:23:28.243793+0000","last_unstale":"2023-06-03T20:30:38.306298+0000","last_undegraded":"2023-06-03T20:30:38.306298+0000","last_fullsized":"2023-06-03T20:30:38.306298+0000","mapping_epoch":251,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":252,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T03:12:27.946002+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"14.8","version":"293'10752","reported_seq":10775,"reported_epoch":293,"state":"active+clean","last_fresh":"2023-06-03T20:31:18.493824+0000","last_change":"2023-06-03T20:27:21.376040+0000","last_active":"2023-06-03T20:31:18.493824+0000","last_peered":"2023-06-03T20:31:18.493824+0000","last_clean":"2023-06-03T20:31:18.493824+0000","last_became_active":"2023-06-03T20:27:21.375709+0000","last_became_peered":"2023-06-03T20:27:21.375709+0000","last_unstale":"2023-06-03T20:31:18.493824+0000","last_undegraded":"2023-06-03T20:31:18.493824+0000","last_fullsized":"2023-06-03T20:31:18.493824+0000","mapping_epoch":278,"log_start":"281'1900","ondisk_log_start":"281'1900","created":278,"last_epoch_clean":279,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T20:27:20.339366+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T20:27:20.339366+0000","last_clean_scrub_stamp":"2023-06-03T20:27:20.339366+0000","objects_scrubbed":0,"log_size":8852,"log_dups_size":0,"ondisk_log_size":8852,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T05:50:28.157584+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":100270080,"num_objects":1530,"num_object_clones":0,"num_object_copies":4590,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1530,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10752,"num_write_kb":393024,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":72,"reported_epoch":292,"state":"active+clean","last_fresh":"2023-06-03T20:30:38.298664+0000","last_change":"2023-06-03T20:21:01.258182+0000","last_active":"2023-06-03T20:30:38.298664+0000","last_peered":"2023-06-03T20:30:38.298664+0000","last_clean":"2023-06-03T20:30:38.298664+0000","last_became_active":"2023-06-03T20:21:01.254049+0000","last_became_peered":"2023-06-03T20:21:01.254049+0000","last_unstale":"2023-06-03T20:30:38.298664+0000","last_undegraded":"2023-06-03T20:30:38.298664+0000","last_fullsized":"2023-06-03T20:30:38.298664+0000","mapping_epoch":235,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":236,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:32:20.692043+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:32:20.692043+0000","last_clean_scrub_stamp":"2023-06-03T19:32:20.692043+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T05:16:30.078473+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":181,"reported_epoch":292,"state":"active+clean","last_fresh":"2023-06-03T20:30:38.306637+0000","last_change":"2023-06-03T20:17:22.834746+0000","last_active":"2023-06-03T20:30:38.306637+0000","last_peered":"2023-06-03T20:30:38.306637+0000","last_clean":"2023-06-03T20:30:38.306637+0000","last_became_active":"2023-06-03T20:17:22.834448+0000","last_became_peered":"2023-06-03T20:17:22.834448+0000","last_unstale":"2023-06-03T20:30:38.306637+0000","last_undegraded":"2023-06-03T20:30:38.306637+0000","last_fullsized":"2023-06-03T20:30:38.306637+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T21:16:31.167063+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,2],"acting":[3,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":83,"reported_epoch":292,"state":"active+clean","last_fresh":"2023-06-03T20:30:38.298450+0000","last_change":"2023-06-03T20:23:28.254434+0000","last_active":"2023-06-03T20:30:38.298450+0000","last_peered":"2023-06-03T20:30:38.298450+0000","last_clean":"2023-06-03T20:30:38.298450+0000","last_became_active":"2023-06-03T20:23:28.254012+0000","last_became_peered":"2023-06-03T20:23:28.254012+0000","last_unstale":"2023-06-03T20:30:38.298450+0000","last_undegraded":"2023-06-03T20:30:38.298450+0000","last_fullsized":"2023-06-03T20:30:38.298450+0000","mapping_epoch":251,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":252,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T06:22:09.540298+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"14.e","version":"293'10713","reported_seq":10738,"reported_epoch":293,"state":"active+clean","last_fresh":"2023-06-03T20:31:16.359029+0000","last_change":"2023-06-03T20:27:21.368301+0000","last_active":"2023-06-03T20:31:16.359029+0000","last_peered":"2023-06-03T20:31:16.359029+0000","last_clean":"2023-06-03T20:31:16.359029+0000","last_became_active":"2023-06-03T20:27:21.367955+0000","last_became_peered":"2023-06-03T20:27:21.367955+0000","last_unstale":"2023-06-03T20:31:16.359029+0000","last_undegraded":"2023-06-03T20:31:16.359029+0000","last_fullsized":"2023-06-03T20:31:16.359029+0000","mapping_epoch":278,"log_start":"281'3000","ondisk_log_start":"281'3000","created":278,"last_epoch_clean":279,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T20:27:20.339366+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T20:27:20.339366+0000","last_clean_scrub_stamp":"2023-06-03T20:27:20.339366+0000","objects_scrubbed":0,"log_size":7713,"log_dups_size":0,"ondisk_log_size":7713,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T22:35:27.855608+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":103743488,"num_objects":1583,"num_object_clones":0,"num_object_copies":4749,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1583,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10713,"num_write_kb":393472,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":81,"reported_epoch":292,"state":"active+clean","last_fresh":"2023-06-03T20:30:38.298585+0000","last_change":"2023-06-03T20:28:51.234855+0000","last_active":"2023-06-03T20:30:38.298585+0000","last_peered":"2023-06-03T20:30:38.298585+0000","last_clean":"2023-06-03T20:30:38.298585+0000","last_became_active":"2023-06-03T20:28:51.234514+0000","last_became_peered":"2023-06-03T20:28:51.234514+0000","last_unstale":"2023-06-03T20:30:38.298585+0000","last_undegraded":"2023-06-03T20:30:38.298585+0000","last_fullsized":"2023-06-03T20:30:38.298585+0000","mapping_epoch":286,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":287,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:32:20.692043+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:32:20.692043+0000","last_clean_scrub_stamp":"2023-06-03T19:32:20.692043+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T03:00:56.480804+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":178,"reported_epoch":292,"state":"active+clean","last_fresh":"2023-06-03T20:30:38.288844+0000","last_change":"2023-06-03T20:24:02.258311+0000","last_active":"2023-06-03T20:30:38.288844+0000","last_peered":"2023-06-03T20:30:38.288844+0000","last_clean":"2023-06-03T20:30:38.288844+0000","last_became_active":"2023-06-03T20:24:02.257676+0000","last_became_peered":"2023-06-03T20:24:02.257676+0000","last_unstale":"2023-06-03T20:30:38.288844+0000","last_undegraded":"2023-06-03T20:30:38.288844+0000","last_fullsized":"2023-06-03T20:30:38.288844+0000","mapping_epoch":254,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":255,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T01:47:46.383038+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":72,"reported_epoch":292,"state":"active+clean","last_fresh":"2023-06-03T20:30:38.298747+0000","last_change":"2023-06-03T20:21:01.259331+0000","last_active":"2023-06-03T20:30:38.298747+0000","last_peered":"2023-06-03T20:30:38.298747+0000","last_clean":"2023-06-03T20:30:38.298747+0000","last_became_active":"2023-06-03T20:21:01.258836+0000","last_became_peered":"2023-06-03T20:21:01.258836+0000","last_unstale":"2023-06-03T20:30:38.298747+0000","last_undegraded":"2023-06-03T20:30:38.298747+0000","last_fullsized":"2023-06-03T20:30:38.298747+0000","mapping_epoch":235,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":236,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T01:29:19.471600+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"14.d","version":"293'10876","reported_seq":18663,"reported_epoch":293,"state":"active+recovering+remapped","last_fresh":"2023-06-03T20:31:17.847527+0000","last_change":"2023-06-03T20:28:52.754748+0000","last_active":"2023-06-03T20:31:17.847527+0000","last_peered":"2023-06-03T20:31:17.847527+0000","last_clean":"2023-06-03T20:28:49.972400+0000","last_became_active":"2023-06-03T20:28:52.639960+0000","last_became_peered":"2023-06-03T20:28:52.639960+0000","last_unstale":"2023-06-03T20:31:17.847527+0000","last_undegraded":"2023-06-03T20:31:17.847527+0000","last_fullsized":"2023-06-03T20:31:17.847527+0000","mapping_epoch":287,"log_start":"281'800","ondisk_log_start":"281'800","created":278,"last_epoch_clean":279,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T20:27:20.339366+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T20:27:20.339366+0000","last_clean_scrub_stamp":"2023-06-03T20:27:20.339366+0000","objects_scrubbed":0,"log_size":10076,"log_dups_size":0,"ondisk_log_size":10076,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T08:00:38.609409+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":96665624,"num_objects":1476,"num_object_clones":0,"num_object_copies":4428,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":1189,"num_objects_unfound":0,"num_objects_dirty":1476,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":10876,"num_write_kb":395201,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3877,"num_bytes_recovered":254083072,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,4],"acting":[2,5,4],"avail_no_missing":["2","4","5"],"object_location_counts":[{"shards":"2,4,5","objects":287}],"blocked_by":[],"up_primary":6,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":279,"reported_epoch":292,"state":"active+clean","last_fresh":"2023-06-03T20:30:38.303432+0000","last_change":"2023-06-03T19:56:54.504859+0000","last_active":"2023-06-03T20:30:38.303432+0000","last_peered":"2023-06-03T20:30:38.303432+0000","last_clean":"2023-06-03T20:30:38.303432+0000","last_became_active":"2023-06-03T19:56:54.504455+0000","last_became_peered":"2023-06-03T19:56:54.504455+0000","last_unstale":"2023-06-03T20:30:38.303432+0000","last_undegraded":"2023-06-03T20:30:38.303432+0000","last_fullsized":"2023-06-03T20:30:38.303432+0000","mapping_epoch":119,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:32:20.692043+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:32:20.692043+0000","last_clean_scrub_stamp":"2023-06-03T19:32:20.692043+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T06:25:45.522100+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":82,"reported_epoch":292,"state":"active+clean","last_fresh":"2023-06-03T20:31:07.856948+0000","last_change":"2023-06-03T20:24:02.256677+0000","last_active":"2023-06-03T20:31:07.856948+0000","last_peered":"2023-06-03T20:31:07.856948+0000","last_clean":"2023-06-03T20:31:07.856948+0000","last_became_active":"2023-06-03T20:24:02.256216+0000","last_became_peered":"2023-06-03T20:24:02.256216+0000","last_unstale":"2023-06-03T20:31:07.856948+0000","last_undegraded":"2023-06-03T20:31:07.856948+0000","last_fullsized":"2023-06-03T20:31:07.856948+0000","mapping_epoch":254,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":255,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T02:32:45.186204+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":72,"reported_epoch":292,"state":"active+clean","last_fresh":"2023-06-03T20:30:38.306220+0000","last_change":"2023-06-03T20:21:01.261473+0000","last_active":"2023-06-03T20:30:38.306220+0000","last_peered":"2023-06-03T20:30:38.306220+0000","last_clean":"2023-06-03T20:30:38.306220+0000","last_became_active":"2023-06-03T20:21:01.261098+0000","last_became_peered":"2023-06-03T20:21:01.261098+0000","last_unstale":"2023-06-03T20:30:38.306220+0000","last_undegraded":"2023-06-03T20:30:38.306220+0000","last_fullsized":"2023-06-03T20:30:38.306220+0000","mapping_epoch":235,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":236,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T03:56:22.986549+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"14.f","version":"293'10683","reported_seq":10706,"reported_epoch":293,"state":"active+clean","last_fresh":"2023-06-03T20:31:18.012620+0000","last_change":"2023-06-03T20:27:21.374763+0000","last_active":"2023-06-03T20:31:18.012620+0000","last_peered":"2023-06-03T20:31:18.012620+0000","last_clean":"2023-06-03T20:31:18.012620+0000","last_became_active":"2023-06-03T20:27:21.374440+0000","last_became_peered":"2023-06-03T20:27:21.374440+0000","last_unstale":"2023-06-03T20:31:18.012620+0000","last_undegraded":"2023-06-03T20:31:18.012620+0000","last_fullsized":"2023-06-03T20:31:18.012620+0000","mapping_epoch":278,"log_start":"281'1500","ondisk_log_start":"281'1500","created":278,"last_epoch_clean":279,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T20:27:20.339366+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T20:27:20.339366+0000","last_clean_scrub_stamp":"2023-06-03T20:27:20.339366+0000","objects_scrubbed":0,"log_size":9183,"log_dups_size":0,"ondisk_log_size":9183,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":97452032,"num_objects":1487,"num_object_clones":0,"num_object_copies":4461,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1487,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10683,"num_write_kb":389440,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,1],"acting":[7,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":164,"reported_epoch":292,"state":"active+clean","last_fresh":"2023-06-03T20:30:38.300549+0000","last_change":"2023-06-03T20:17:22.834467+0000","last_active":"2023-06-03T20:30:38.300549+0000","last_peered":"2023-06-03T20:30:38.300549+0000","last_clean":"2023-06-03T20:30:38.300549+0000","last_became_active":"2023-06-03T20:17:22.834147+0000","last_became_peered":"2023-06-03T20:17:22.834147+0000","last_unstale":"2023-06-03T20:30:38.300549+0000","last_undegraded":"2023-06-03T20:30:38.300549+0000","last_fullsized":"2023-06-03T20:30:38.300549+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:32:20.692043+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:32:20.692043+0000","last_clean_scrub_stamp":"2023-06-03T19:32:20.692043+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T04:36:29.567051+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,7],"acting":[2,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"72'139","reported_seq":565,"reported_epoch":292,"state":"active+clean","last_fresh":"2023-06-03T20:30:38.305706+0000","last_change":"2023-06-03T20:27:01.248641+0000","last_active":"2023-06-03T20:30:38.305706+0000","last_peered":"2023-06-03T20:30:38.305706+0000","last_clean":"2023-06-03T20:30:38.305706+0000","last_became_active":"2023-06-03T20:27:01.248063+0000","last_became_peered":"2023-06-03T20:27:01.248063+0000","last_unstale":"2023-06-03T20:30:38.305706+0000","last_undegraded":"2023-06-03T20:30:38.305706+0000","last_fullsized":"2023-06-03T20:30:38.305706+0000","mapping_epoch":274,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":275,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:27:28.100808+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:27:28.100808+0000","last_clean_scrub_stamp":"2023-06-03T19:27:28.100808+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":282,"reported_epoch":292,"state":"active+clean","last_fresh":"2023-06-03T20:30:38.308944+0000","last_change":"2023-06-03T20:24:02.263360+0000","last_active":"2023-06-03T20:30:38.308944+0000","last_peered":"2023-06-03T20:30:38.308944+0000","last_clean":"2023-06-03T20:30:38.308944+0000","last_became_active":"2023-06-03T20:24:02.262954+0000","last_became_peered":"2023-06-03T20:24:02.262954+0000","last_unstale":"2023-06-03T20:30:38.308944+0000","last_undegraded":"2023-06-03T20:30:38.308944+0000","last_fullsized":"2023-06-03T20:30:38.308944+0000","mapping_epoch":254,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":255,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T19:38:46.084731+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":16,"reported_epoch":292,"state":"active+clean","last_fresh":"2023-06-03T20:30:38.288408+0000","last_change":"2023-06-03T20:28:51.234600+0000","last_active":"2023-06-03T20:30:38.288408+0000","last_peered":"2023-06-03T20:30:38.288408+0000","last_clean":"2023-06-03T20:30:38.288408+0000","last_became_active":"2023-06-03T20:28:51.234227+0000","last_became_peered":"2023-06-03T20:28:51.234227+0000","last_unstale":"2023-06-03T20:30:38.288408+0000","last_undegraded":"2023-06-03T20:30:38.288408+0000","last_fullsized":"2023-06-03T20:30:38.288408+0000","mapping_epoch":286,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":287,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"14.c","version":"293'10785","reported_seq":10810,"reported_epoch":293,"state":"active+clean","last_fresh":"2023-06-03T20:31:18.409086+0000","last_change":"2023-06-03T20:27:21.375660+0000","last_active":"2023-06-03T20:31:18.409086+0000","last_peered":"2023-06-03T20:31:18.409086+0000","last_clean":"2023-06-03T20:31:18.409086+0000","last_became_active":"2023-06-03T20:27:21.375311+0000","last_became_peered":"2023-06-03T20:27:21.375311+0000","last_unstale":"2023-06-03T20:31:18.409086+0000","last_undegraded":"2023-06-03T20:31:18.409086+0000","last_fullsized":"2023-06-03T20:31:18.409086+0000","mapping_epoch":278,"log_start":"281'1900","ondisk_log_start":"281'1900","created":278,"last_epoch_clean":279,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T20:27:20.339366+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T20:27:20.339366+0000","last_clean_scrub_stamp":"2023-06-03T20:27:20.339366+0000","objects_scrubbed":0,"log_size":8885,"log_dups_size":0,"ondisk_log_size":8885,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T04:43:56.400215+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":94699520,"num_objects":1445,"num_object_clones":0,"num_object_copies":4335,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1445,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10785,"num_write_kb":391360,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,2],"acting":[0,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":285,"reported_epoch":292,"state":"active+clean","last_fresh":"2023-06-03T20:30:38.305646+0000","last_change":"2023-06-03T20:21:01.258113+0000","last_active":"2023-06-03T20:30:38.305646+0000","last_peered":"2023-06-03T20:30:38.305646+0000","last_clean":"2023-06-03T20:30:38.305646+0000","last_became_active":"2023-06-03T20:21:01.257488+0000","last_became_peered":"2023-06-03T20:21:01.257488+0000","last_unstale":"2023-06-03T20:30:38.305646+0000","last_undegraded":"2023-06-03T20:30:38.305646+0000","last_fullsized":"2023-06-03T20:30:38.305646+0000","mapping_epoch":235,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":236,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:32:20.692043+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:32:20.692043+0000","last_clean_scrub_stamp":"2023-06-03T19:32:20.692043+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":292,"reported_epoch":292,"state":"active+clean","last_fresh":"2023-06-03T20:30:53.356634+0000","last_change":"2023-06-03T20:23:27.255164+0000","last_active":"2023-06-03T20:30:53.356634+0000","last_peered":"2023-06-03T20:30:53.356634+0000","last_clean":"2023-06-03T20:30:53.356634+0000","last_became_active":"2023-06-03T20:23:27.253932+0000","last_became_peered":"2023-06-03T20:23:27.253932+0000","last_unstale":"2023-06-03T20:30:53.356634+0000","last_undegraded":"2023-06-03T20:30:53.356634+0000","last_fullsized":"2023-06-03T20:30:53.356634+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T06:42:02.303554+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":295,"reported_epoch":292,"state":"active+clean","last_fresh":"2023-06-03T20:30:38.322957+0000","last_change":"2023-06-03T20:21:01.250266+0000","last_active":"2023-06-03T20:30:38.322957+0000","last_peered":"2023-06-03T20:30:38.322957+0000","last_clean":"2023-06-03T20:30:38.322957+0000","last_became_active":"2023-06-03T20:21:01.249839+0000","last_became_peered":"2023-06-03T20:21:01.249839+0000","last_unstale":"2023-06-03T20:30:38.322957+0000","last_undegraded":"2023-06-03T20:30:38.322957+0000","last_fullsized":"2023-06-03T20:30:38.322957+0000","mapping_epoch":235,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":236,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T05:13:26.584607+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"14.9","version":"293'10783","reported_seq":10808,"reported_epoch":293,"state":"active+clean","last_fresh":"2023-06-03T20:31:18.237917+0000","last_change":"2023-06-03T20:27:21.372404+0000","last_active":"2023-06-03T20:31:18.237917+0000","last_peered":"2023-06-03T20:31:18.237917+0000","last_clean":"2023-06-03T20:31:18.237917+0000","last_became_active":"2023-06-03T20:27:21.371829+0000","last_became_peered":"2023-06-03T20:27:21.371829+0000","last_unstale":"2023-06-03T20:31:18.237917+0000","last_undegraded":"2023-06-03T20:31:18.237917+0000","last_fullsized":"2023-06-03T20:31:18.237917+0000","mapping_epoch":278,"log_start":"281'2600","ondisk_log_start":"281'2600","created":278,"last_epoch_clean":279,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T20:27:20.339366+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T20:27:20.339366+0000","last_clean_scrub_stamp":"2023-06-03T20:27:20.339366+0000","objects_scrubbed":0,"log_size":8183,"log_dups_size":0,"ondisk_log_size":8183,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T02:20:57.246181+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":95617024,"num_objects":1459,"num_object_clones":0,"num_object_copies":4377,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1459,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10783,"num_write_kb":391744,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,4],"acting":[3,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":294,"reported_epoch":292,"state":"active+clean","last_fresh":"2023-06-03T20:30:38.306178+0000","last_change":"2023-06-03T20:23:27.261511+0000","last_active":"2023-06-03T20:30:38.306178+0000","last_peered":"2023-06-03T20:30:38.306178+0000","last_clean":"2023-06-03T20:30:38.306178+0000","last_became_active":"2023-06-03T20:23:27.259622+0000","last_became_peered":"2023-06-03T20:23:27.259622+0000","last_unstale":"2023-06-03T20:30:38.306178+0000","last_undegraded":"2023-06-03T20:30:38.306178+0000","last_fullsized":"2023-06-03T20:30:38.306178+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:32:20.692043+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:32:20.692043+0000","last_clean_scrub_stamp":"2023-06-03T19:32:20.692043+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":72,"reported_epoch":292,"state":"active+clean","last_fresh":"2023-06-03T20:30:38.298584+0000","last_change":"2023-06-03T20:21:01.252859+0000","last_active":"2023-06-03T20:30:38.298584+0000","last_peered":"2023-06-03T20:30:38.298584+0000","last_clean":"2023-06-03T20:30:38.298584+0000","last_became_active":"2023-06-03T20:21:01.251222+0000","last_became_peered":"2023-06-03T20:21:01.251222+0000","last_unstale":"2023-06-03T20:30:38.298584+0000","last_undegraded":"2023-06-03T20:30:38.298584+0000","last_fullsized":"2023-06-03T20:30:38.298584+0000","mapping_epoch":235,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":236,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T05:19:58.100226+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":248,"reported_epoch":292,"state":"active+clean","last_fresh":"2023-06-03T20:30:53.364790+0000","last_change":"2023-06-03T19:47:06.733532+0000","last_active":"2023-06-03T20:30:53.364790+0000","last_peered":"2023-06-03T20:30:53.364790+0000","last_clean":"2023-06-03T20:30:53.364790+0000","last_became_active":"2023-06-03T19:47:06.733131+0000","last_became_peered":"2023-06-03T19:47:06.733131+0000","last_unstale":"2023-06-03T20:30:53.364790+0000","last_undegraded":"2023-06-03T20:30:53.364790+0000","last_fullsized":"2023-06-03T20:30:53.364790+0000","mapping_epoch":86,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":87,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T23:46:03.788646+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"14.a","version":"293'10556","reported_seq":10579,"reported_epoch":293,"state":"active+clean","last_fresh":"2023-06-03T20:31:17.876678+0000","last_change":"2023-06-03T20:27:21.378317+0000","last_active":"2023-06-03T20:31:17.876678+0000","last_peered":"2023-06-03T20:31:17.876678+0000","last_clean":"2023-06-03T20:31:17.876678+0000","last_became_active":"2023-06-03T20:27:21.377947+0000","last_became_peered":"2023-06-03T20:27:21.377947+0000","last_unstale":"2023-06-03T20:31:17.876678+0000","last_undegraded":"2023-06-03T20:31:17.876678+0000","last_fullsized":"2023-06-03T20:31:17.876678+0000","mapping_epoch":278,"log_start":"282'3000","ondisk_log_start":"282'3000","created":278,"last_epoch_clean":279,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T20:27:20.339366+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T20:27:20.339366+0000","last_clean_scrub_stamp":"2023-06-03T20:27:20.339366+0000","objects_scrubbed":0,"log_size":7556,"log_dups_size":0,"ondisk_log_size":7556,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T23:39:02.595450+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":100139008,"num_objects":1528,"num_object_clones":0,"num_object_copies":4584,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1528,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10556,"num_write_kb":386688,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,3],"acting":[1,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.7","version":"60'2","reported_seq":257,"reported_epoch":292,"state":"active+clean","last_fresh":"2023-06-03T20:30:38.308152+0000","last_change":"2023-06-03T19:56:54.510148+0000","last_active":"2023-06-03T20:30:38.308152+0000","last_peered":"2023-06-03T20:30:38.308152+0000","last_clean":"2023-06-03T20:30:38.308152+0000","last_became_active":"2023-06-03T19:56:54.509771+0000","last_became_peered":"2023-06-03T19:56:54.509771+0000","last_unstale":"2023-06-03T20:30:38.308152+0000","last_undegraded":"2023-06-03T20:30:38.308152+0000","last_fullsized":"2023-06-03T20:30:38.308152+0000","mapping_epoch":119,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T00:52:42.807432+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":90,"reported_epoch":292,"state":"active+clean","last_fresh":"2023-06-03T20:30:38.299881+0000","last_change":"2023-06-03T20:24:02.257142+0000","last_active":"2023-06-03T20:30:38.299881+0000","last_peered":"2023-06-03T20:30:38.299881+0000","last_clean":"2023-06-03T20:30:38.299881+0000","last_became_active":"2023-06-03T20:24:02.256483+0000","last_became_peered":"2023-06-03T20:24:02.256483+0000","last_unstale":"2023-06-03T20:30:38.299881+0000","last_undegraded":"2023-06-03T20:30:38.299881+0000","last_fullsized":"2023-06-03T20:30:38.299881+0000","mapping_epoch":254,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":255,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:32:20.692043+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:32:20.692043+0000","last_clean_scrub_stamp":"2023-06-03T19:32:20.692043+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T00:53:37.197795+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":256,"reported_epoch":292,"state":"active+clean","last_fresh":"2023-06-03T20:31:02.983064+0000","last_change":"2023-06-03T19:56:54.507670+0000","last_active":"2023-06-03T20:31:02.983064+0000","last_peered":"2023-06-03T20:31:02.983064+0000","last_clean":"2023-06-03T20:31:02.983064+0000","last_became_active":"2023-06-03T19:56:54.507293+0000","last_became_peered":"2023-06-03T19:56:54.507293+0000","last_unstale":"2023-06-03T20:31:02.983064+0000","last_undegraded":"2023-06-03T20:31:02.983064+0000","last_fullsized":"2023-06-03T20:31:02.983064+0000","mapping_epoch":119,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T01:49:19.966054+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"14.b","version":"293'10564","reported_seq":10588,"reported_epoch":293,"state":"active+clean","last_fresh":"2023-06-03T20:31:18.492584+0000","last_change":"2023-06-03T20:27:21.375306+0000","last_active":"2023-06-03T20:31:18.492584+0000","last_peered":"2023-06-03T20:31:18.492584+0000","last_clean":"2023-06-03T20:31:18.492584+0000","last_became_active":"2023-06-03T20:27:21.374975+0000","last_became_peered":"2023-06-03T20:27:21.374975+0000","last_unstale":"2023-06-03T20:31:18.492584+0000","last_undegraded":"2023-06-03T20:31:18.492584+0000","last_fullsized":"2023-06-03T20:31:18.492584+0000","mapping_epoch":278,"log_start":"281'1700","ondisk_log_start":"281'1700","created":278,"last_epoch_clean":279,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T20:27:20.339366+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T20:27:20.339366+0000","last_clean_scrub_stamp":"2023-06-03T20:27:20.339366+0000","objects_scrubbed":0,"log_size":8864,"log_dups_size":0,"ondisk_log_size":8864,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T02:22:27.778843+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":92405760,"num_objects":1410,"num_object_clones":0,"num_object_copies":4230,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1410,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10564,"num_write_kb":383168,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,1],"acting":[0,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":48,"reported_epoch":292,"state":"active+clean","last_fresh":"2023-06-03T20:30:38.288621+0000","last_change":"2023-06-03T20:24:02.319230+0000","last_active":"2023-06-03T20:30:38.288621+0000","last_peered":"2023-06-03T20:30:38.288621+0000","last_clean":"2023-06-03T20:30:38.288621+0000","last_became_active":"2023-06-03T20:24:02.318926+0000","last_became_peered":"2023-06-03T20:24:02.318926+0000","last_unstale":"2023-06-03T20:30:38.288621+0000","last_undegraded":"2023-06-03T20:30:38.288621+0000","last_fullsized":"2023-06-03T20:30:38.288621+0000","mapping_epoch":254,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":255,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:32:20.692043+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:32:20.692043+0000","last_clean_scrub_stamp":"2023-06-03T19:32:20.692043+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"60'1","reported_seq":323,"reported_epoch":292,"state":"active+clean","last_fresh":"2023-06-03T20:30:38.322787+0000","last_change":"2023-06-03T20:23:28.239664+0000","last_active":"2023-06-03T20:30:38.322787+0000","last_peered":"2023-06-03T20:30:38.322787+0000","last_clean":"2023-06-03T20:30:38.322787+0000","last_became_active":"2023-06-03T20:23:28.239252+0000","last_became_peered":"2023-06-03T20:23:28.239252+0000","last_unstale":"2023-06-03T20:30:38.322787+0000","last_undegraded":"2023-06-03T20:30:38.322787+0000","last_fullsized":"2023-06-03T20:30:38.322787+0000","mapping_epoch":251,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":252,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T06:52:20.862520+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":176,"reported_epoch":292,"state":"active+clean","last_fresh":"2023-06-03T20:30:38.289804+0000","last_change":"2023-06-03T20:24:02.259850+0000","last_active":"2023-06-03T20:30:38.289804+0000","last_peered":"2023-06-03T20:30:38.289804+0000","last_clean":"2023-06-03T20:30:38.289804+0000","last_became_active":"2023-06-03T20:24:02.258764+0000","last_became_peered":"2023-06-03T20:24:02.258764+0000","last_unstale":"2023-06-03T20:30:38.289804+0000","last_undegraded":"2023-06-03T20:30:38.289804+0000","last_fullsized":"2023-06-03T20:30:38.289804+0000","mapping_epoch":254,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":255,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T21:54:11.487473+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"60'1","reported_seq":277,"reported_epoch":292,"state":"active+clean","last_fresh":"2023-06-03T20:30:38.316127+0000","last_change":"2023-06-03T20:23:27.254853+0000","last_active":"2023-06-03T20:30:38.316127+0000","last_peered":"2023-06-03T20:30:38.316127+0000","last_clean":"2023-06-03T20:30:38.316127+0000","last_became_active":"2023-06-03T20:23:27.254410+0000","last_became_peered":"2023-06-03T20:23:27.254410+0000","last_unstale":"2023-06-03T20:30:38.316127+0000","last_undegraded":"2023-06-03T20:30:38.316127+0000","last_fullsized":"2023-06-03T20:30:38.316127+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T22:46:19.423939+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":307,"reported_epoch":292,"state":"active+clean","last_fresh":"2023-06-03T20:31:12.842648+0000","last_change":"2023-06-03T20:21:01.257656+0000","last_active":"2023-06-03T20:31:12.842648+0000","last_peered":"2023-06-03T20:31:12.842648+0000","last_clean":"2023-06-03T20:31:12.842648+0000","last_became_active":"2023-06-03T20:21:01.257107+0000","last_became_peered":"2023-06-03T20:21:01.257107+0000","last_unstale":"2023-06-03T20:31:12.842648+0000","last_undegraded":"2023-06-03T20:31:12.842648+0000","last_fullsized":"2023-06-03T20:31:12.842648+0000","mapping_epoch":235,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":236,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T06:08:18.954539+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"60'2","reported_seq":68,"reported_epoch":292,"state":"active+clean","last_fresh":"2023-06-03T20:31:01.426089+0000","last_change":"2023-06-03T20:23:28.243048+0000","last_active":"2023-06-03T20:31:01.426089+0000","last_peered":"2023-06-03T20:31:01.426089+0000","last_clean":"2023-06-03T20:31:01.426089+0000","last_became_active":"2023-06-03T20:23:28.241128+0000","last_became_peered":"2023-06-03T20:23:28.241128+0000","last_unstale":"2023-06-03T20:31:01.426089+0000","last_undegraded":"2023-06-03T20:31:01.426089+0000","last_fullsized":"2023-06-03T20:31:01.426089+0000","mapping_epoch":251,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":252,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T05:14:54.701329+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,1],"acting":[4,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":231,"reported_epoch":292,"state":"active+clean","last_fresh":"2023-06-03T20:31:01.426071+0000","last_change":"2023-06-03T20:28:24.276513+0000","last_active":"2023-06-03T20:31:01.426071+0000","last_peered":"2023-06-03T20:31:01.426071+0000","last_clean":"2023-06-03T20:31:01.426071+0000","last_became_active":"2023-06-03T20:28:24.276244+0000","last_became_peered":"2023-06-03T20:28:24.276244+0000","last_unstale":"2023-06-03T20:31:01.426071+0000","last_undegraded":"2023-06-03T20:31:01.426071+0000","last_fullsized":"2023-06-03T20:31:01.426071+0000","mapping_epoch":284,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":285,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T06:17:57.159168+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[4,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"60'2","reported_seq":81,"reported_epoch":292,"state":"active+clean","last_fresh":"2023-06-03T20:31:01.426115+0000","last_change":"2023-06-03T20:28:51.237342+0000","last_active":"2023-06-03T20:31:01.426115+0000","last_peered":"2023-06-03T20:31:01.426115+0000","last_clean":"2023-06-03T20:31:01.426115+0000","last_became_active":"2023-06-03T20:28:51.233969+0000","last_became_peered":"2023-06-03T20:28:51.233969+0000","last_unstale":"2023-06-03T20:31:01.426115+0000","last_undegraded":"2023-06-03T20:31:01.426115+0000","last_fullsized":"2023-06-03T20:31:01.426115+0000","mapping_epoch":286,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":287,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T07:30:09.503545+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":215,"reported_epoch":292,"state":"active+clean","last_fresh":"2023-06-03T20:31:01.426019+0000","last_change":"2023-06-03T20:23:27.254468+0000","last_active":"2023-06-03T20:31:01.426019+0000","last_peered":"2023-06-03T20:31:01.426019+0000","last_clean":"2023-06-03T20:31:01.426019+0000","last_became_active":"2023-06-03T20:23:27.253859+0000","last_became_peered":"2023-06-03T20:23:27.253859+0000","last_unstale":"2023-06-03T20:31:01.426019+0000","last_undegraded":"2023-06-03T20:31:01.426019+0000","last_fullsized":"2023-06-03T20:31:01.426019+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T20:15:25.066125+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"60'1","reported_seq":312,"reported_epoch":292,"state":"active+clean","last_fresh":"2023-06-03T20:31:02.983273+0000","last_change":"2023-06-03T20:28:51.234748+0000","last_active":"2023-06-03T20:31:02.983273+0000","last_peered":"2023-06-03T20:31:02.983273+0000","last_clean":"2023-06-03T20:31:02.983273+0000","last_became_active":"2023-06-03T20:28:51.234341+0000","last_became_peered":"2023-06-03T20:28:51.234341+0000","last_unstale":"2023-06-03T20:31:02.983273+0000","last_undegraded":"2023-06-03T20:31:02.983273+0000","last_fullsized":"2023-06-03T20:31:02.983273+0000","mapping_epoch":286,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":287,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T06:08:49.901424+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":295,"reported_epoch":292,"state":"active+clean","last_fresh":"2023-06-03T20:30:52.971724+0000","last_change":"2023-06-03T19:56:54.503416+0000","last_active":"2023-06-03T20:30:52.971724+0000","last_peered":"2023-06-03T20:30:52.971724+0000","last_clean":"2023-06-03T20:30:52.971724+0000","last_became_active":"2023-06-03T19:56:54.503018+0000","last_became_peered":"2023-06-03T19:56:54.503018+0000","last_unstale":"2023-06-03T20:30:52.971724+0000","last_undegraded":"2023-06-03T20:30:52.971724+0000","last_fullsized":"2023-06-03T20:30:52.971724+0000","mapping_epoch":119,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T20:14:00.330616+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"60'4","reported_seq":266,"reported_epoch":292,"state":"active+clean","last_fresh":"2023-06-03T20:30:53.393540+0000","last_change":"2023-06-03T19:56:11.748116+0000","last_active":"2023-06-03T20:30:53.393540+0000","last_peered":"2023-06-03T20:30:53.393540+0000","last_clean":"2023-06-03T20:30:53.393540+0000","last_became_active":"2023-06-03T19:56:11.747636+0000","last_became_peered":"2023-06-03T19:56:11.747636+0000","last_unstale":"2023-06-03T20:30:53.393540+0000","last_undegraded":"2023-06-03T20:30:53.393540+0000","last_fullsized":"2023-06-03T20:30:53.393540+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":115,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T06:57:48.052055+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":215,"reported_epoch":292,"state":"active+clean","last_fresh":"2023-06-03T20:30:46.410920+0000","last_change":"2023-06-03T20:28:51.231879+0000","last_active":"2023-06-03T20:30:46.410920+0000","last_peered":"2023-06-03T20:30:46.410920+0000","last_clean":"2023-06-03T20:30:46.410920+0000","last_became_active":"2023-06-03T20:28:51.231451+0000","last_became_peered":"2023-06-03T20:28:51.231451+0000","last_unstale":"2023-06-03T20:30:46.410920+0000","last_undegraded":"2023-06-03T20:30:46.410920+0000","last_fullsized":"2023-06-03T20:30:46.410920+0000","mapping_epoch":286,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":287,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T05:47:15.344575+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":14,"num_pg":16,"stat_sum":{"num_bytes":1566048280,"num_objects":23897,"num_object_clones":0,"num_object_copies":71691,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":1189,"num_objects_unfound":0,"num_objects_dirty":23897,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":172131,"num_write_kb":6272833,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3877,"num_bytes_recovered":254083072,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":4781457408,"data_stored":4781441120,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":141431,"ondisk_log_size":141431,"up":48,"acting":48,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5486,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":62,"num_write_kb":30,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":23,"num_bytes_recovered":4856,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16458,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":30319,"internal_metadata":0},"log_size":56,"ondisk_log_size":56,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2969600,"data_stored":2951840,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":3,"num_store_stats":5},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":48,"seq":206158430931,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1258196,"kb_used_data":290188,"kb_used_omap":0,"kb_used_meta":968000,"kb_avail":92486956,"statfs":{"total":95995035648,"available":94706642944,"internally_reserved":0,"allocated":297152512,"data_stored":296489906,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":991232000},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1,1],"upper_bound":16},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 3 20:30:33 2023","interfaces":[{"interface":"back","average":{"1min":1.502,"5min":1.573,"15min":1.603},"min":{"1min":0.28999999999999998,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":4.3840000000000003,"5min":5.5540000000000003,"15min":73.810000000000002},"last":0.46100000000000002},{"interface":"front","average":{"1min":2.3919999999999999,"5min":1.3220000000000001,"15min":1.534},"min":{"1min":0.32100000000000001,"5min":0.29999999999999999,"15min":0.26700000000000002},"max":{"1min":18.901,"5min":18.901,"15min":72.878},"last":0.53800000000000003}]},{"osd":1,"last update":"Sat Jun 3 20:31:17 2023","interfaces":[{"interface":"back","average":{"1min":1.609,"5min":1.2450000000000001,"15min":1.593},"min":{"1min":0.28899999999999998,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":4.0650000000000004,"5min":5.5999999999999996,"15min":100.845},"last":1.4650000000000001},{"interface":"front","average":{"1min":2.194,"5min":1.45,"15min":1.7370000000000001},"min":{"1min":0.30399999999999999,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":7.6559999999999997,"5min":7.6559999999999997,"15min":100.86499999999999},"last":0.39000000000000001}]},{"osd":2,"last update":"Sat Jun 3 20:31:17 2023","interfaces":[{"interface":"back","average":{"1min":2.774,"5min":1.631,"15min":2.5459999999999998},"min":{"1min":0.28899999999999998,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":16.809999999999999,"5min":16.809999999999999,"15min":101.185},"last":0.40999999999999998},{"interface":"front","average":{"1min":2.7749999999999999,"5min":1.7010000000000001,"15min":2.6749999999999998},"min":{"1min":0.36799999999999999,"5min":0.33200000000000002,"15min":0.315},"max":{"1min":16.587,"5min":16.587,"15min":100.61799999999999},"last":1.905}]},{"osd":3,"last update":"Sat Jun 3 20:31:17 2023","interfaces":[{"interface":"back","average":{"1min":1.488,"5min":1.341,"15min":1.843},"min":{"1min":0.34999999999999998,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":6.3150000000000004,"5min":6.3150000000000004,"15min":74.039000000000001},"last":0.35999999999999999},{"interface":"front","average":{"1min":2.1000000000000001,"5min":1.5720000000000001,"15min":1.9299999999999999},"min":{"1min":0.39400000000000002,"5min":0.38900000000000001,"15min":0.35399999999999998},"max":{"1min":7.7480000000000002,"5min":8.2360000000000007,"15min":73.001999999999995},"last":0.48199999999999998}]},{"osd":4,"last update":"Sat Jun 3 20:30:38 2023","interfaces":[{"interface":"back","average":{"1min":1.6060000000000001,"5min":1.3520000000000001,"15min":2.0430000000000001},"min":{"1min":0.30399999999999999,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":7.718,"5min":7.8419999999999996,"15min":8.6010000000000009},"last":1.431},{"interface":"front","average":{"1min":1.3759999999999999,"5min":1.4430000000000001,"15min":1.859},"min":{"1min":0.35599999999999998,"5min":0.315,"15min":0.27600000000000002},"max":{"1min":8.9459999999999997,"5min":9.8460000000000001,"15min":9.8460000000000001},"last":5.6360000000000001}]},{"osd":5,"last update":"Sat Jun 3 20:31:17 2023","interfaces":[{"interface":"back","average":{"1min":1.609,"5min":1.3380000000000001,"15min":1.2629999999999999},"min":{"1min":0.38100000000000001,"5min":0.317,"15min":0.317},"max":{"1min":9.1609999999999996,"5min":9.1609999999999996,"15min":17.696000000000002},"last":1.508},{"interface":"front","average":{"1min":1.861,"5min":1.3,"15min":1.298},"min":{"1min":0.36499999999999999,"5min":0.33600000000000002,"15min":0.29399999999999998},"max":{"1min":7.7990000000000004,"5min":7.7990000000000004,"15min":9.3710000000000004},"last":0.44800000000000001}]},{"osd":6,"last update":"Sat Jun 3 20:31:17 2023","interfaces":[{"interface":"back","average":{"1min":1.363,"5min":1.1910000000000001,"15min":1.139},"min":{"1min":0.439,"5min":0.36399999999999999,"15min":0.36399999999999999},"max":{"1min":7.7750000000000004,"5min":7.7750000000000004,"15min":9.7200000000000006},"last":0.51200000000000001},{"interface":"front","average":{"1min":1.7250000000000001,"5min":1.3109999999999999,"15min":1.107},"min":{"1min":0.39100000000000001,"5min":0.36799999999999999,"15min":0.34300000000000003},"max":{"1min":8.8770000000000007,"5min":9.923,"15min":9.923},"last":5.6829999999999998}]}]},{"osd":6,"up_from":42,"seq":180388627164,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":979528,"kb_used_data":367744,"kb_used_omap":0,"kb_used_meta":611712,"kb_avail":92765624,"statfs":{"total":95995035648,"available":94991998976,"internally_reserved":0,"allocated":376569856,"data_stored":375919101,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":626393088},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 3 20:30:46 2023","interfaces":[{"interface":"back","average":{"1min":3.9860000000000002,"5min":2.1030000000000002,"15min":1.5820000000000001},"min":{"1min":0.36699999999999999,"5min":0.254,"15min":0.254},"max":{"1min":31.547999999999998,"5min":31.547999999999998,"15min":56.259},"last":0.38400000000000001},{"interface":"front","average":{"1min":4.1840000000000002,"5min":2.496,"15min":1.7210000000000001},"min":{"1min":0.38600000000000001,"5min":0.315,"15min":0.28299999999999997},"max":{"1min":31.631,"5min":31.631,"15min":56.234000000000002},"last":0.36099999999999999}]},{"osd":1,"last update":"Sat Jun 3 20:30:19 2023","interfaces":[{"interface":"back","average":{"1min":1.3640000000000001,"5min":1.129,"15min":1.0620000000000001},"min":{"1min":0.32900000000000001,"5min":0.30299999999999999,"15min":0.29199999999999998},"max":{"1min":5.5679999999999996,"5min":6.7910000000000004,"15min":30.736000000000001},"last":0.32900000000000001},{"interface":"front","average":{"1min":1.911,"5min":1.6479999999999999,"15min":1.2210000000000001},"min":{"1min":0.379,"5min":0.34200000000000003,"15min":0.32700000000000001},"max":{"1min":6.2380000000000004,"5min":8.5869999999999997,"15min":30.788},"last":0.40600000000000003}]},{"osd":2,"last update":"Sat Jun 3 20:30:19 2023","interfaces":[{"interface":"back","average":{"1min":3.0529999999999999,"5min":1.5940000000000001,"15min":2.331},"min":{"1min":0.34899999999999998,"5min":0.34300000000000003,"15min":0.315},"max":{"1min":32.881,"5min":32.881,"15min":93.558999999999997},"last":2.8050000000000002},{"interface":"front","average":{"1min":2.7829999999999999,"5min":1.498,"15min":2.242},"min":{"1min":0.373,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":33.103000000000002,"5min":33.103000000000002,"15min":94.888000000000005},"last":0.52900000000000003}]},{"osd":3,"last update":"Sat Jun 3 20:31:04 2023","interfaces":[{"interface":"back","average":{"1min":1.411,"5min":1.5720000000000001,"15min":1.698},"min":{"1min":0.35799999999999998,"5min":0.35799999999999998,"15min":0.35799999999999998},"max":{"1min":8.6669999999999998,"5min":30.777000000000001,"15min":58.615000000000002},"last":3.02},{"interface":"front","average":{"1min":2.0760000000000001,"5min":1.758,"15min":1.8169999999999999},"min":{"1min":0.39100000000000001,"5min":0.39100000000000001,"15min":0.39100000000000001},"max":{"1min":8.8550000000000004,"5min":30.074999999999999,"15min":58.546999999999997},"last":2.004}]},{"osd":4,"last update":"Sat Jun 3 20:30:40 2023","interfaces":[{"interface":"back","average":{"1min":1.1699999999999999,"5min":1.6830000000000001,"15min":1.77},"min":{"1min":0.33000000000000002,"5min":0.33000000000000002,"15min":0.33000000000000002},"max":{"1min":6.6980000000000004,"5min":10.574999999999999,"15min":10.574999999999999},"last":0.89200000000000002},{"interface":"front","average":{"1min":1.0940000000000001,"5min":1.3029999999999999,"15min":1.353},"min":{"1min":0.432,"5min":0.35799999999999998,"15min":0.34499999999999997},"max":{"1min":6.6710000000000003,"5min":12.622999999999999,"15min":12.622999999999999},"last":1.125}]},{"osd":5,"last update":"Sat Jun 3 20:30:31 2023","interfaces":[{"interface":"back","average":{"1min":1.3100000000000001,"5min":1.167,"15min":1.056},"min":{"1min":0.36299999999999999,"5min":0.36299999999999999,"15min":0.29899999999999999},"max":{"1min":7.5830000000000002,"5min":7.5830000000000002,"15min":10.625999999999999},"last":1.1459999999999999},{"interface":"front","average":{"1min":1.3,"5min":1.722,"15min":1.2589999999999999},"min":{"1min":0.40999999999999998,"5min":0.40999999999999998,"15min":0.32200000000000001},"max":{"1min":6.7320000000000002,"5min":8.2880000000000003,"15min":10.856},"last":0.81599999999999995}]},{"osd":7,"last update":"Sat Jun 3 20:31:07 2023","interfaces":[{"interface":"back","average":{"1min":1.6859999999999999,"5min":1.2689999999999999,"15min":1.22},"min":{"1min":0.42399999999999999,"5min":0.41299999999999998,"15min":0.37},"max":{"1min":9.0519999999999996,"5min":9.0519999999999996,"15min":9.0519999999999996},"last":0.95499999999999996},{"interface":"front","average":{"1min":0.95599999999999996,"5min":1.131,"15min":1.145},"min":{"1min":0.43099999999999999,"5min":0.42599999999999999,"15min":0.38},"max":{"1min":7.2850000000000001,"5min":7.2850000000000001,"15min":10.52},"last":1.044}]}]},{"osd":1,"up_from":103,"seq":442381631962,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1240656,"kb_used_data":777544,"kb_used_omap":4,"kb_used_meta":463099,"kb_avail":92504496,"statfs":{"total":95995035648,"available":94724603904,"internally_reserved":0,"allocated":796205056,"data_stored":795080986,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":474214286},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,1],"upper_bound":16},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 3 20:30:24 2023","interfaces":[{"interface":"back","average":{"1min":12.113,"5min":4.5309999999999997,"15min":3.2160000000000002},"min":{"1min":0.26900000000000002,"5min":0.19800000000000001,"15min":0.17100000000000001},"max":{"1min":99.870000000000005,"5min":99.870000000000005,"15min":119.94499999999999},"last":0.19800000000000001},{"interface":"front","average":{"1min":12.196999999999999,"5min":4.6740000000000004,"15min":3.2890000000000001},"min":{"1min":0.29699999999999999,"5min":0.182,"15min":0.182},"max":{"1min":99.897999999999996,"5min":99.897999999999996,"15min":120.04300000000001},"last":0.36699999999999999}]},{"osd":2,"last update":"Sat Jun 3 20:30:24 2023","interfaces":[{"interface":"back","average":{"1min":2.5059999999999998,"5min":1.5660000000000001,"15min":2.1890000000000001},"min":{"1min":0.23799999999999999,"5min":0.20599999999999999,"15min":0.20599999999999999},"max":{"1min":11.009,"5min":11.009,"15min":77.719999999999999},"last":2.899},{"interface":"front","average":{"1min":2.702,"5min":1.716,"15min":2.2989999999999999},"min":{"1min":0.32100000000000001,"5min":0.246,"15min":0.22700000000000001},"max":{"1min":10.096,"5min":10.096,"15min":78.007999999999996},"last":3.5390000000000001}]},{"osd":3,"last update":"Sat Jun 3 20:31:00 2023","interfaces":[{"interface":"back","average":{"1min":2.6200000000000001,"5min":3.2429999999999999,"15min":2.339},"min":{"1min":0.27300000000000002,"5min":0.255,"15min":0.23400000000000001},"max":{"1min":9.3889999999999993,"5min":95.525000000000006,"15min":96.864999999999995},"last":3.6099999999999999},{"interface":"front","average":{"1min":2.3220000000000001,"5min":3.3119999999999998,"15min":2.464},"min":{"1min":0.32400000000000001,"5min":0.223,"15min":0.223},"max":{"1min":8.7349999999999994,"5min":94.507000000000005,"15min":96.679000000000002},"last":3.5790000000000002}]},{"osd":4,"last update":"Sat Jun 3 20:30:41 2023","interfaces":[{"interface":"back","average":{"1min":2.0670000000000002,"5min":1.802,"15min":1.764},"min":{"1min":0.33100000000000002,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":6.1959999999999997,"5min":11.114000000000001,"15min":11.114000000000001},"last":0.54500000000000004},{"interface":"front","average":{"1min":1.9710000000000001,"5min":1.623,"15min":1.52},"min":{"1min":0.35199999999999998,"5min":0.35199999999999998,"15min":0.33400000000000002},"max":{"1min":6.125,"5min":7.556,"15min":11.247999999999999},"last":2.8260000000000001}]},{"osd":5,"last update":"Sat Jun 3 20:30:24 2023","interfaces":[{"interface":"back","average":{"1min":2.3570000000000002,"5min":1.802,"15min":1.3400000000000001},"min":{"1min":0.35799999999999998,"5min":0.35799999999999998,"15min":0.30299999999999999},"max":{"1min":6.8700000000000001,"5min":11.246,"15min":11.246},"last":0.73399999999999999},{"interface":"front","average":{"1min":2.0030000000000001,"5min":1.643,"15min":1.3100000000000001},"min":{"1min":0.35799999999999998,"5min":0.35799999999999998,"15min":0.35799999999999998},"max":{"1min":7.0890000000000004,"5min":8.7840000000000007,"15min":10.922000000000001},"last":2.9239999999999999}]},{"osd":6,"last update":"Sat Jun 3 20:30:24 2023","interfaces":[{"interface":"back","average":{"1min":2.2879999999999998,"5min":1.5289999999999999,"15min":1.171},"min":{"1min":0.37,"5min":0.37,"15min":0.36799999999999999},"max":{"1min":7.1159999999999997,"5min":7.6139999999999999,"15min":10.002000000000001},"last":2.8719999999999999},{"interface":"front","average":{"1min":1.889,"5min":1.4350000000000001,"15min":1.2589999999999999},"min":{"1min":0.42099999999999999,"5min":0.375,"15min":0.36899999999999999},"max":{"1min":4.8129999999999997,"5min":6.5300000000000002,"15min":11.067},"last":3.645}]},{"osd":7,"last update":"Sat Jun 3 20:30:24 2023","interfaces":[{"interface":"back","average":{"1min":1.899,"5min":1.5740000000000001,"15min":1.4610000000000001},"min":{"1min":0.44700000000000001,"5min":0.38300000000000001,"15min":0.372},"max":{"1min":4.8479999999999999,"5min":9.1170000000000009,"15min":10.538},"last":3.677},{"interface":"front","average":{"1min":2.0720000000000001,"5min":1.627,"15min":1.2649999999999999},"min":{"1min":0.41899999999999998,"5min":0.41599999999999998,"15min":0.34000000000000002},"max":{"1min":6.1989999999999998,"5min":14.025,"15min":14.025},"last":0.59399999999999997}]}]},{"osd":0,"up_from":86,"seq":369367187988,"num_pgs":34,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1528144,"kb_used_data":769544,"kb_used_omap":5,"kb_used_meta":758394,"kb_avail":92217008,"statfs":{"total":95995035648,"available":94430216192,"internally_reserved":0,"allocated":788013056,"data_stored":786831442,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":776596090},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,2,2,1],"upper_bound":64},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Jun 3 20:31:15 2023","interfaces":[{"interface":"back","average":{"1min":1.8740000000000001,"5min":3.2730000000000001,"15min":2.476},"min":{"1min":0.24199999999999999,"5min":0.19500000000000001,"15min":0.185},"max":{"1min":8.5899999999999999,"5min":156.43600000000001,"15min":243.03200000000001},"last":0.34799999999999998},{"interface":"front","average":{"1min":3.0590000000000002,"5min":4.1139999999999999,"15min":2.851},"min":{"1min":0.41299999999999998,"5min":0.188,"15min":0.188},"max":{"1min":10.605,"5min":157.489,"15min":242.78399999999999},"last":2.2869999999999999}]},{"osd":2,"last update":"Sat Jun 3 20:31:15 2023","interfaces":[{"interface":"back","average":{"1min":1.26,"5min":1.284,"15min":1.542},"min":{"1min":0.223,"5min":0.219,"15min":0.218},"max":{"1min":4.4950000000000001,"5min":8.6799999999999997,"15min":36.515000000000001},"last":7.5780000000000003},{"interface":"front","average":{"1min":5.1390000000000002,"5min":2.6030000000000002,"15min":2.968},"min":{"1min":0.24099999999999999,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":61.795999999999999,"5min":61.795999999999999,"15min":243.30099999999999},"last":7.6200000000000001}]},{"osd":3,"last update":"Sat Jun 3 20:30:56 2023","interfaces":[{"interface":"back","average":{"1min":2.4980000000000002,"5min":4.0659999999999998,"15min":3.5590000000000002},"min":{"1min":0.39100000000000001,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":11.906000000000001,"5min":259.68400000000003,"15min":259.68400000000003},"last":7.4279999999999999},{"interface":"front","average":{"1min":2.1099999999999999,"5min":4.0640000000000001,"15min":2.7829999999999999},"min":{"1min":0.245,"5min":0.245,"15min":0.23799999999999999},"max":{"1min":8.6679999999999993,"5min":261.02699999999999,"15min":261.02699999999999},"last":0.38500000000000001}]},{"osd":4,"last update":"Sat Jun 3 20:30:38 2023","interfaces":[{"interface":"back","average":{"1min":2.3580000000000001,"5min":1.8979999999999999,"15min":1.7070000000000001},"min":{"1min":0.34200000000000003,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":7.1980000000000004,"5min":11.49,"15min":11.49},"last":2.3239999999999998},{"interface":"front","average":{"1min":0.94899999999999995,"5min":1.0349999999999999,"15min":1.548},"min":{"1min":0.38,"5min":0.34100000000000003,"15min":0.34100000000000003},"max":{"1min":3.1219999999999999,"5min":6.3600000000000003,"15min":7.21},"last":7.5060000000000002}]},{"osd":5,"last update":"Sat Jun 3 20:30:32 2023","interfaces":[{"interface":"back","average":{"1min":2.052,"5min":2.0539999999999998,"15min":2.3050000000000002},"min":{"1min":0.34599999999999997,"5min":0.33600000000000002,"15min":0.29299999999999998},"max":{"1min":7.2560000000000002,"5min":20.238,"15min":243.15899999999999},"last":2.351},{"interface":"front","average":{"1min":1.2709999999999999,"5min":1.278,"15min":2.1040000000000001},"min":{"1min":0.434,"5min":0.40500000000000003,"15min":0.33400000000000002},"max":{"1min":4.5270000000000001,"5min":6.4429999999999996,"15min":243.32599999999999},"last":7.4779999999999998}]},{"osd":6,"last update":"Sat Jun 3 20:30:32 2023","interfaces":[{"interface":"back","average":{"1min":1.373,"5min":1.0720000000000001,"15min":1.954},"min":{"1min":0.36499999999999999,"5min":0.36499999999999999,"15min":0.36299999999999999},"max":{"1min":7.8869999999999996,"5min":7.8869999999999996,"15min":243.25},"last":7.5549999999999997},{"interface":"front","average":{"1min":1.95,"5min":1.2270000000000001,"15min":1.974},"min":{"1min":0.41399999999999998,"5min":0.41399999999999998,"15min":0.379},"max":{"1min":11.634,"5min":11.634,"15min":242.84399999999999},"last":0.55900000000000005}]},{"osd":7,"last update":"Sat Jun 3 20:30:32 2023","interfaces":[{"interface":"back","average":{"1min":1.2050000000000001,"5min":1.3779999999999999,"15min":2.2320000000000002},"min":{"1min":0.39600000000000002,"5min":0.39600000000000002,"15min":0.35499999999999998},"max":{"1min":4.452,"5min":7.173,"15min":242.93600000000001},"last":7.5300000000000002},{"interface":"front","average":{"1min":1.9079999999999999,"5min":1.421,"15min":2.2069999999999999},"min":{"1min":0.44,"5min":0.44,"15min":0.40000000000000002},"max":{"1min":11.68,"5min":11.68,"15min":243.21299999999999},"last":0.58699999999999997}]}]},{"osd":2,"up_from":114,"seq":489626272167,"num_pgs":28,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1408292,"kb_used_data":779164,"kb_used_omap":7,"kb_used_meta":629112,"kb_avail":92336860,"statfs":{"total":95995035648,"available":94552944640,"internally_reserved":0,"allocated":797863936,"data_stored":796781417,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8111,"internal_metadata":644210769},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,1,0,0,1],"upper_bound":128},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 3 20:31:03 2023","interfaces":[{"interface":"back","average":{"1min":3.0870000000000002,"5min":2.415,"15min":2.444},"min":{"1min":0.182,"5min":0.182,"15min":0.182},"max":{"1min":14.602,"5min":77.055000000000007,"15min":117.57899999999999},"last":1.073},{"interface":"front","average":{"1min":2.9870000000000001,"5min":2.661,"15min":2.3839999999999999},"min":{"1min":0.28199999999999997,"5min":0.20699999999999999,"15min":0.20399999999999999},"max":{"1min":7.6699999999999999,"5min":76.453000000000003,"15min":117.38500000000001},"last":6.5439999999999996}]},{"osd":1,"last update":"Sat Jun 3 20:31:03 2023","interfaces":[{"interface":"back","average":{"1min":3.1360000000000001,"5min":2.3650000000000002,"15min":1.9670000000000001},"min":{"1min":0.36699999999999999,"5min":0.193,"15min":0.193},"max":{"1min":10.378,"5min":33.149999999999999,"15min":116.127},"last":8.6910000000000007},{"interface":"front","average":{"1min":2.121,"5min":2.0350000000000001,"15min":2.1869999999999998},"min":{"1min":0.33400000000000002,"5min":0.251,"15min":0.251},"max":{"1min":7.6289999999999996,"5min":33.651000000000003,"15min":116.461},"last":8.0630000000000006}]},{"osd":3,"last update":"Sat Jun 3 20:31:03 2023","interfaces":[{"interface":"back","average":{"1min":3.863,"5min":1.8919999999999999,"15min":2.161},"min":{"1min":0.40500000000000003,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":12.231,"5min":12.231,"15min":74.703000000000003},"last":13.186999999999999},{"interface":"front","average":{"1min":3.1520000000000001,"5min":1.8720000000000001,"15min":2.2490000000000001},"min":{"1min":0.39600000000000002,"5min":0.28599999999999998,"15min":0.23000000000000001},"max":{"1min":14.355,"5min":14.355,"15min":37.088999999999999},"last":3.9279999999999999}]},{"osd":4,"last update":"Sat Jun 3 20:30:39 2023","interfaces":[{"interface":"back","average":{"1min":2.617,"5min":1.661,"15min":2.7280000000000002},"min":{"1min":0.34799999999999998,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":8.9100000000000001,"5min":10.664999999999999,"15min":16.561},"last":3.8300000000000001},{"interface":"front","average":{"1min":2.7890000000000001,"5min":1.615,"15min":1.847},"min":{"1min":0.55400000000000005,"5min":0.35099999999999998,"15min":0.35099999999999998},"max":{"1min":6.7400000000000002,"5min":6.7400000000000002,"15min":9.9339999999999993},"last":1.2210000000000001}]},{"osd":5,"last update":"Sat Jun 3 20:31:03 2023","interfaces":[{"interface":"back","average":{"1min":2.3359999999999999,"5min":1.468,"15min":1.712},"min":{"1min":0.35899999999999999,"5min":0.35899999999999999,"15min":0.32100000000000001},"max":{"1min":6.665,"5min":6.665,"15min":17.751000000000001},"last":1.2430000000000001},{"interface":"front","average":{"1min":2.6070000000000002,"5min":1.7669999999999999,"15min":1.5860000000000001},"min":{"1min":0.57399999999999995,"5min":0.32500000000000001,"15min":0.316},"max":{"1min":10.281000000000001,"5min":10.281000000000001,"15min":17.928000000000001},"last":6.6299999999999999}]},{"osd":6,"last update":"Sat Jun 3 20:31:03 2023","interfaces":[{"interface":"back","average":{"1min":2.7570000000000001,"5min":1.6659999999999999,"15min":1.5700000000000001},"min":{"1min":0.33900000000000002,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":6.8179999999999996,"5min":6.8179999999999996,"15min":14.331},"last":1.2649999999999999},{"interface":"front","average":{"1min":2.7829999999999999,"5min":1.645,"15min":1.4650000000000001},"min":{"1min":0.51800000000000002,"5min":0.35699999999999998,"15min":0.35599999999999998},"max":{"1min":11.943,"5min":11.943,"15min":12.867000000000001},"last":6.601}]},{"osd":7,"last update":"Sat Jun 3 20:31:03 2023","interfaces":[{"interface":"back","average":{"1min":2.6389999999999998,"5min":1.6850000000000001,"15min":1.7569999999999999},"min":{"1min":0.38300000000000001,"5min":0.38300000000000001,"15min":0.376},"max":{"1min":6.7640000000000002,"5min":6.7640000000000002,"15min":14.243},"last":1.3520000000000001},{"interface":"front","average":{"1min":2.6619999999999999,"5min":1.6719999999999999,"15min":1.627},"min":{"1min":0.57399999999999995,"5min":0.44700000000000001,"15min":0.38600000000000001},"max":{"1min":10.233000000000001,"5min":10.233000000000001,"15min":12.845000000000001},"last":6.6669999999999998}]}]},{"osd":3,"up_from":119,"seq":511101108639,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1293348,"kb_used_data":685148,"kb_used_omap":2,"kb_used_meta":608125,"kb_avail":92451804,"statfs":{"total":95995035648,"available":94670647296,"internally_reserved":0,"allocated":701591552,"data_stored":700447306,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":622720053},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,3],"upper_bound":32},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 3 20:31:05 2023","interfaces":[{"interface":"back","average":{"1min":3.504,"5min":4.343,"15min":3.202},"min":{"1min":0.183,"5min":0.183,"15min":0.183},"max":{"1min":15.346,"5min":168.33099999999999,"15min":168.33099999999999},"last":3.4769999999999999},{"interface":"front","average":{"1min":4.718,"5min":4.9649999999999999,"15min":3.4039999999999999},"min":{"1min":0.251,"5min":0.23200000000000001,"15min":0.183},"max":{"1min":18.370000000000001,"5min":169.25800000000001,"15min":169.25800000000001},"last":5.7610000000000001}]},{"osd":1,"last update":"Sat Jun 3 20:31:05 2023","interfaces":[{"interface":"back","average":{"1min":2.601,"5min":1.6699999999999999,"15min":1.651},"min":{"1min":0.33500000000000002,"5min":0.245,"15min":0.22500000000000001},"max":{"1min":9.4529999999999994,"5min":20.992000000000001,"15min":117.047},"last":6.7530000000000001},{"interface":"front","average":{"1min":3.0680000000000001,"5min":1.883,"15min":1.6659999999999999},"min":{"1min":0.221,"5min":0.221,"15min":0.20399999999999999},"max":{"1min":15.263,"5min":21.07,"15min":116.505},"last":1.7450000000000001}]},{"osd":2,"last update":"Sat Jun 3 20:31:05 2023","interfaces":[{"interface":"back","average":{"1min":5.2839999999999998,"5min":2.3130000000000002,"15min":2.7570000000000001},"min":{"1min":0.248,"5min":0.248,"15min":0.248},"max":{"1min":18.916,"5min":18.916,"15min":117.584},"last":2.4020000000000001},{"interface":"front","average":{"1min":2.5720000000000001,"5min":1.641,"15min":2.6269999999999998},"min":{"1min":0.24099999999999999,"5min":0.22600000000000001,"15min":0.22600000000000001},"max":{"1min":11.228,"5min":11.228,"15min":117.01000000000001},"last":11.526}]},{"osd":4,"last update":"Sat Jun 3 20:30:45 2023","interfaces":[{"interface":"back","average":{"1min":1.8640000000000001,"5min":1.4259999999999999,"15min":1.431},"min":{"1min":0.39500000000000002,"5min":0.36699999999999999,"15min":0.34599999999999997},"max":{"1min":8.0449999999999999,"5min":8.0449999999999999,"15min":8.0449999999999999},"last":7.0709999999999997},{"interface":"front","average":{"1min":1.5209999999999999,"5min":1.5920000000000001,"15min":1.7190000000000001},"min":{"1min":0.34200000000000003,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":4.883,"5min":10.773999999999999,"15min":10.773999999999999},"last":3.452}]},{"osd":5,"last update":"Sat Jun 3 20:31:05 2023","interfaces":[{"interface":"back","average":{"1min":1.4590000000000001,"5min":1.3280000000000001,"15min":1.492},"min":{"1min":0.39400000000000002,"5min":0.36299999999999999,"15min":0.313},"max":{"1min":6.7140000000000004,"5min":6.7140000000000004,"15min":38.651000000000003},"last":3.4100000000000001},{"interface":"front","average":{"1min":2.4580000000000002,"5min":1.615,"15min":1.516},"min":{"1min":0.41299999999999998,"5min":0.376,"15min":0.316},"max":{"1min":9.4909999999999997,"5min":9.4909999999999997,"15min":38.585999999999999},"last":2.476}]},{"osd":6,"last update":"Sat Jun 3 20:31:05 2023","interfaces":[{"interface":"back","average":{"1min":2.3279999999999998,"5min":1.5269999999999999,"15min":1.341},"min":{"1min":0.49199999999999999,"5min":0.36499999999999999,"15min":0.34799999999999998},"max":{"1min":9.5199999999999996,"5min":9.5199999999999996,"15min":9.5199999999999996},"last":2.5019999999999998},{"interface":"front","average":{"1min":2.0619999999999998,"5min":1.4330000000000001,"15min":1.446},"min":{"1min":0.41699999999999998,"5min":0.36699999999999999,"15min":0.36699999999999999},"max":{"1min":8.0790000000000006,"5min":8.0790000000000006,"15min":8.7279999999999998},"last":6.9119999999999999}]},{"osd":7,"last update":"Sat Jun 3 20:31:05 2023","interfaces":[{"interface":"back","average":{"1min":2.145,"5min":1.5329999999999999,"15min":1.716},"min":{"1min":0.44400000000000001,"5min":0.44400000000000001,"15min":0.36599999999999999},"max":{"1min":7.9809999999999999,"5min":7.9809999999999999,"15min":39.008000000000003},"last":6.9450000000000003},{"interface":"front","average":{"1min":1.748,"5min":1.5660000000000001,"15min":1.7450000000000001},"min":{"1min":0.39100000000000001,"5min":0.38700000000000001,"15min":0.34899999999999998},"max":{"1min":6.742,"5min":7.4960000000000004,"15min":40.003999999999998},"last":3.5390000000000001}]}]},{"osd":4,"up_from":250,"seq":1073741824095,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":848912,"kb_used_data":600648,"kb_used_omap":10,"kb_used_meta":248245,"kb_avail":92896240,"statfs":{"total":95995035648,"available":95125749760,"internally_reserved":0,"allocated":615063552,"data_stored":613814848,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":10652,"internal_metadata":254203492},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,2],"upper_bound":8},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 3 20:30:39 2023","interfaces":[{"interface":"back","average":{"1min":1.7969999999999999,"5min":2.5070000000000001,"15min":2.0800000000000001},"min":{"1min":0.30099999999999999,"5min":0.27600000000000002,"15min":0.26300000000000001},"max":{"1min":8.3770000000000007,"5min":145.755,"15min":145.755},"last":3.254},{"interface":"front","average":{"1min":1.2350000000000001,"5min":2.5950000000000002,"15min":1.9179999999999999},"min":{"1min":0.35099999999999998,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":5.6269999999999998,"5min":145.71799999999999,"15min":145.71799999999999},"last":1.3300000000000001}]},{"osd":1,"last update":"Sat Jun 3 20:30:39 2023","interfaces":[{"interface":"back","average":{"1min":2.1110000000000002,"5min":1.49,"15min":1.6890000000000001},"min":{"1min":0.29799999999999999,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":5.6319999999999997,"5min":9.7769999999999992,"15min":9.7769999999999992},"last":1.22},{"interface":"front","average":{"1min":1.2230000000000001,"5min":1.302,"15min":1.7090000000000001},"min":{"1min":0.29599999999999999,"5min":0.29599999999999999,"15min":0.29599999999999999},"max":{"1min":3.9039999999999999,"5min":11.994,"15min":11.994},"last":0.65900000000000003}]},{"osd":2,"last update":"Sat Jun 3 20:30:39 2023","interfaces":[{"interface":"back","average":{"1min":5.2439999999999998,"5min":2.927,"15min":2.4089999999999998},"min":{"1min":0.40300000000000002,"5min":0.36299999999999999,"15min":0.35899999999999999},"max":{"1min":53.454999999999998,"5min":78.605000000000004,"15min":78.605000000000004},"last":0.80100000000000005},{"interface":"front","average":{"1min":5.3460000000000001,"5min":2.9580000000000002,"15min":2.8330000000000002},"min":{"1min":0.371,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":53.902999999999999,"5min":73.569000000000003,"15min":73.569000000000003},"last":0.498}]},{"osd":3,"last update":"Sat Jun 3 20:30:39 2023","interfaces":[{"interface":"back","average":{"1min":4.96,"5min":2.6539999999999999,"15min":9.7460000000000004},"min":{"1min":0.39200000000000002,"5min":0.379,"15min":0.32500000000000001},"max":{"1min":59.765999999999998,"5min":59.765999999999998,"15min":217.578},"last":1.1890000000000001},{"interface":"front","average":{"1min":5.351,"5min":2.702,"15min":9.6300000000000008},"min":{"1min":0.44900000000000001,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":60.597000000000001,"5min":60.597000000000001,"15min":218.55699999999999},"last":1.425}]},{"osd":5,"last update":"Sat Jun 3 20:30:39 2023","interfaces":[{"interface":"back","average":{"1min":0.70399999999999996,"5min":1.145,"15min":2.7069999999999999},"min":{"1min":0.38400000000000001,"5min":0.35099999999999998,"15min":0.35099999999999998},"max":{"1min":1.448,"5min":8.4760000000000009,"15min":23.913},"last":1.3680000000000001},{"interface":"front","average":{"1min":0.98899999999999999,"5min":1.1830000000000001,"15min":2.0070000000000001},"min":{"1min":0.34000000000000002,"5min":0.34000000000000002,"15min":0.34000000000000002},"max":{"1min":3.9289999999999998,"5min":8.2650000000000006,"15min":11.093999999999999},"last":0.59099999999999997}]},{"osd":6,"last update":"Sat Jun 3 20:30:39 2023","interfaces":[{"interface":"back","average":{"1min":0.75900000000000001,"5min":1.2070000000000001,"15min":2.3900000000000001},"min":{"1min":0.40300000000000002,"5min":0.40300000000000002,"15min":0.40300000000000002},"max":{"1min":1.482,"5min":9.3239999999999998,"15min":23.963000000000001},"last":1.399},{"interface":"front","average":{"1min":1.0840000000000001,"5min":1.2649999999999999,"15min":1.446},"min":{"1min":0.38,"5min":0.38,"15min":0.38},"max":{"1min":4.0039999999999996,"5min":6.8630000000000004,"15min":8.4190000000000005},"last":0.77200000000000002}]},{"osd":7,"last update":"Sat Jun 3 20:30:39 2023","interfaces":[{"interface":"back","average":{"1min":1.123,"5min":1.4199999999999999,"15min":2.1000000000000001},"min":{"1min":0.39300000000000002,"5min":0.39300000000000002,"15min":0.374},"max":{"1min":4.032,"5min":10.496,"15min":10.747999999999999},"last":1.268},{"interface":"front","average":{"1min":0.79100000000000004,"5min":1.294,"15min":2.0409999999999999},"min":{"1min":0.52600000000000002,"5min":0.34699999999999998,"15min":0.34699999999999998},"max":{"1min":0.998,"5min":12.154,"15min":12.154},"last":0.71799999999999997}]}]},{"osd":5,"up_from":36,"seq":154618823398,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1453500,"kb_used_data":488884,"kb_used_omap":0,"kb_used_meta":964608,"kb_avail":92291652,"statfs":{"total":95995035648,"available":94506651648,"internally_reserved":0,"allocated":500617216,"data_stored":499974043,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":987758592},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1],"upper_bound":8},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 3 20:30:22 2023","interfaces":[{"interface":"back","average":{"1min":10.137,"5min":3.952,"15min":3.0150000000000001},"min":{"1min":0.312,"5min":0.27100000000000002,"15min":0.25700000000000001},"max":{"1min":176.88300000000001,"5min":176.88300000000001,"15min":176.88300000000001},"last":0.58199999999999996},{"interface":"front","average":{"1min":10.225,"5min":4.2469999999999999,"15min":3.1760000000000002},"min":{"1min":0.37,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":176.96199999999999,"5min":176.96199999999999,"15min":176.96199999999999},"last":0.59199999999999997}]},{"osd":1,"last update":"Sat Jun 3 20:30:22 2023","interfaces":[{"interface":"back","average":{"1min":2.0139999999999998,"5min":2.359,"15min":1.976},"min":{"1min":0.29899999999999999,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":5.2969999999999997,"5min":55.515000000000001,"15min":102.205},"last":0.60899999999999999},{"interface":"front","average":{"1min":1.6100000000000001,"5min":2.2160000000000002,"15min":1.9239999999999999},"min":{"1min":0.316,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":7.7539999999999996,"5min":55.194000000000003,"15min":102.265},"last":0.46400000000000002}]},{"osd":2,"last update":"Sat Jun 3 20:30:22 2023","interfaces":[{"interface":"back","average":{"1min":2.149,"5min":2.7040000000000002,"15min":3.4569999999999999},"min":{"1min":0.35699999999999998,"5min":0.35699999999999998,"15min":0.33400000000000002},"max":{"1min":9.6199999999999992,"5min":104.199,"15min":236.40000000000001},"last":0.53800000000000003},{"interface":"front","average":{"1min":2.3090000000000002,"5min":3.1909999999999998,"15min":3.7360000000000002},"min":{"1min":0.308,"5min":0.308,"15min":0.308},"max":{"1min":7.577,"5min":104.33499999999999,"15min":237.86099999999999},"last":8.4469999999999992}]},{"osd":3,"last update":"Sat Jun 3 20:30:59 2023","interfaces":[{"interface":"back","average":{"1min":1.4750000000000001,"5min":1.752,"15min":2.1520000000000001},"min":{"1min":0.36799999999999999,"5min":0.34399999999999997,"15min":0.34399999999999997},"max":{"1min":3.601,"5min":35.725999999999999,"15min":86.486999999999995},"last":0.39700000000000002},{"interface":"front","average":{"1min":1.903,"5min":1.601,"15min":2.1240000000000001},"min":{"1min":0.45300000000000001,"5min":0.39200000000000002,"15min":0.37},"max":{"1min":5.0739999999999998,"5min":35.762999999999998,"15min":86.512},"last":0.40799999999999997}]},{"osd":4,"last update":"Sat Jun 3 20:30:38 2023","interfaces":[{"interface":"back","average":{"1min":1.1160000000000001,"5min":1.3839999999999999,"15min":2.9780000000000002},"min":{"1min":0.32700000000000001,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":3.9079999999999999,"5min":7.6669999999999998,"15min":21.786000000000001},"last":0.34100000000000003},{"interface":"front","average":{"1min":1.5329999999999999,"5min":1.298,"15min":2.4399999999999999},"min":{"1min":0.38300000000000001,"5min":0.36799999999999999,"15min":0.30199999999999999},"max":{"1min":11.880000000000001,"5min":11.880000000000001,"15min":26.478000000000002},"last":0.622}]},{"osd":6,"last update":"Sat Jun 3 20:30:22 2023","interfaces":[{"interface":"back","average":{"1min":0.996,"5min":1.369,"15min":1.1539999999999999},"min":{"1min":0.39200000000000002,"5min":0.35499999999999998,"15min":0.35199999999999998},"max":{"1min":3.0649999999999999,"5min":7.5140000000000002,"15min":7.5140000000000002},"last":0.371},{"interface":"front","average":{"1min":1.129,"5min":1.3740000000000001,"15min":1.256},"min":{"1min":0.42299999999999999,"5min":0.37,"15min":0.37},"max":{"1min":3.234,"5min":6.5309999999999997,"15min":6.5309999999999997},"last":0.42899999999999999}]},{"osd":7,"last update":"Sat Jun 3 20:30:15 2023","interfaces":[{"interface":"back","average":{"1min":1.2669999999999999,"5min":1.5609999999999999,"15min":1.506},"min":{"1min":0.442,"5min":0.442,"15min":0.35999999999999999},"max":{"1min":3.1549999999999998,"5min":8.6549999999999994,"15min":15.516},"last":0.45200000000000001},{"interface":"front","average":{"1min":1.319,"5min":1.6220000000000001,"15min":1.3149999999999999},"min":{"1min":0.38400000000000001,"5min":0.38400000000000001,"15min":0.373},"max":{"1min":5.0350000000000001,"5min":9.6519999999999992,"15min":9.6519999999999992},"last":0.97199999999999998}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6928,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4862,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":10652,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4786,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4932,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":14,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":772472832,"data_stored":772472832,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":14,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":788201472,"data_stored":788201472,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":14,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":781848576,"data_stored":781844504,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":14,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":685244416,"data_stored":685244416,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":14,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":606015488,"data_stored":606011416,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":14,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":485756928,"data_stored":485752856,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":14,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":369430528,"data_stored":369426456,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":14,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":292487168,"data_stored":292487168,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-06-03T20:31:21.267 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph status --format=json 2023-06-03T20:31:21.295 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:31:21.327 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:31:21.335 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:31:21.336 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:31:21.437 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:31:21.446 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:31:21 smithi089 bash[75068]: audit 1685824279.78594 mgr.y (mgr.34104) 1667 : audit 0 from='client.46832 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-06-03T20:31:21.447 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:31:21 smithi089 bash[78452]: audit 1685824279.78594 mgr.y (mgr.34104) 1667 : audit 0 from='client.46832 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-06-03T20:31:21.476 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:31:21.485 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:31:21.486 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:31:21.495 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-06-03T20:31:21.533 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph osd set noscrub 2023-06-03T20:31:21.585 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:31:21 smithi164 bash[59462]: audit 1685824279.78594 mgr.y (mgr.34104) 1667 : audit 0 from='client.46832 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-06-03T20:31:21.588 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:31:21.668 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:31:21.670 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:31:21.672 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:31:21.773 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:31:21.834 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:31:21.843 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:31:21.844 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:31:21.945 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:31:21.975 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:31:21.977 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:31:21.977 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:31:22.078 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:31:22.116 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:31:22.120 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:31:22.121 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:31:22.221 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:31:22.272 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:31:22.279 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:31:22.280 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:31:22.351 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:31:22 smithi089 bash[78452]: cluster 1685824281.1962657 mgr.y (mgr.34104) 1668 : cluster 0 pgmap v1723: 89 pgs: 1 active+recovering+remapped, 88 active+clean; 1.5 GiB data, 9.5 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 533 op/s; 1189/71763 objects misplaced (1.657%); 2.3 MiB/s, 36 objects/s recovering 2023-06-03T20:31:22.351 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:31:22 smithi089 bash[78452]: audit 1685824282.1658354 mon.a (mon.0) 1650 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:31:22.352 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:31:22 smithi089 bash[75068]: cluster 1685824281.1962657 mgr.y (mgr.34104) 1668 : cluster 0 pgmap v1723: 89 pgs: 1 active+recovering+remapped, 88 active+clean; 1.5 GiB data, 9.5 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 533 op/s; 1189/71763 objects misplaced (1.657%); 2.3 MiB/s, 36 objects/s recovering 2023-06-03T20:31:22.352 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:31:22 smithi089 bash[75068]: audit 1685824282.1658354 mon.a (mon.0) 1650 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:31:22.381 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:31:22.421 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:31:22.428 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:31:22.428 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:31:22.529 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:31:22.558 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:31:22.561 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:31:22.562 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:31:22.663 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:31:22.672 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:31:22 smithi089 bash[75068]: audit 1685824282.1688728 mon.c (mon.2) 676 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T20:31:22.672 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:31:22 smithi089 bash[78452]: audit 1685824282.1688728 mon.c (mon.2) 676 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T20:31:22.696 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:31:22.699 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:31:22.701 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:31:22.802 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:31:22.831 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:31:22.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:31:22 smithi164 bash[59462]: cluster 1685824281.1962657 mgr.y (mgr.34104) 1668 : cluster 0 pgmap v1723: 89 pgs: 1 active+recovering+remapped, 88 active+clean; 1.5 GiB data, 9.5 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 533 op/s; 1189/71763 objects misplaced (1.657%); 2.3 MiB/s, 36 objects/s recovering 2023-06-03T20:31:22.836 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:31:22 smithi164 bash[59462]: audit 1685824282.1658354 mon.a (mon.0) 1650 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:31:22.836 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:31:22 smithi164 bash[59462]: audit 1685824282.1688728 mon.c (mon.2) 676 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T20:31:22.836 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:31:22.837 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:31:22.938 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:31:22.979 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:31:22.983 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:31:22.983 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:31:23.084 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:31:23.120 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:31:23.125 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:31:23.126 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:31:23.227 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:31:23.267 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:31:23.270 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:31:23.271 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:31:23.372 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:31:23.404 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:31:23.409 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:31:23.410 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:31:23.512 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:31:23.545 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:31:23.551 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:31:23.552 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:31:23.653 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:31:23.681 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:31:23.683 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:31:23.684 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:31:23.785 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:31:23.798 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/mon.a/config 2023-06-03T20:31:23.814 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:31:23.817 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:31:23.817 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:31:23.918 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:31:23.947 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:31:23.953 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:31:23.954 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:31:24.055 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:31:24.089 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:31:24.093 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:31:24.094 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:31:24.195 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:31:24.204 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:31:24 smithi089 bash[78452]: cluster 1685824283.196947 mgr.y (mgr.34104) 1669 : cluster 0 pgmap v1724: 89 pgs: 1 active+recovering+remapped, 88 active+clean; 1.4 GiB data, 9.2 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 496 op/s; 976/67674 objects misplaced (1.442%); 2.5 MiB/s, 40 objects/s recovering 2023-06-03T20:31:24.204 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:31:24 smithi089 bash[75068]: cluster 1685824283.196947 mgr.y (mgr.34104) 1669 : cluster 0 pgmap v1724: 89 pgs: 1 active+recovering+remapped, 88 active+clean; 1.4 GiB data, 9.2 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 496 op/s; 976/67674 objects misplaced (1.442%); 2.5 MiB/s, 40 objects/s recovering 2023-06-03T20:31:24.247 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:31:24.249 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:31:24.250 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:31:24.335 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:31:24 smithi164 bash[59462]: cluster 1685824283.196947 mgr.y (mgr.34104) 1669 : cluster 0 pgmap v1724: 89 pgs: 1 active+recovering+remapped, 88 active+clean; 1.4 GiB data, 9.2 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 496 op/s; 976/67674 objects misplaced (1.442%); 2.5 MiB/s, 40 objects/s recovering 2023-06-03T20:31:24.351 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:31:24.380 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:31:24.384 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:31:24.385 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:31:24.486 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:31:24.526 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:31:24.531 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:31:24.531 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:31:24.632 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:31:24.674 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:31:24.678 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:31:24.679 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:31:24.780 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:31:24.815 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:31:24.819 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:31:24.820 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:31:24.921 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:31:24.954 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:31:24.956 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:31:24.956 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:31:25.057 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:31:25.098 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:31:25.100 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:31:25.100 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:31:25.201 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:31:25.242 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:31:25.244 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:31:25.244 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:31:25.345 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:31:25.393 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:31:25.397 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:31:25.398 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:31:25.499 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:31:25.535 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:31:25.539 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:31:25.540 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:31:25.641 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:31:25.673 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:31:25.678 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:31:25.679 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:31:25.780 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:31:25.814 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:31:25.819 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:31:25.820 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:31:25.853 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/mon.a/config 2023-06-03T20:31:25.853 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/mon.a/config 2023-06-03T20:31:25.859 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/mon.a/config 2023-06-03T20:31:25.921 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:31:25.965 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:31:25.971 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:31:25.971 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:31:26.072 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:31:26.107 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:31:26.110 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:31:26.110 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:31:26.211 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:31:26.255 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:31:26.259 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:31:26.260 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:31:26.335 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:31:26 smithi164 bash[59462]: cluster 1685824285.1975033 mgr.y (mgr.34104) 1670 : cluster 0 pgmap v1725: 89 pgs: 1 active+recovering+remapped, 88 active+clean; 1.3 GiB data, 9.1 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 546 op/s; 976/63090 objects misplaced (1.547%); 2.5 MiB/s, 40 objects/s recovering 2023-06-03T20:31:26.361 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:31:26.383 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:31:26.385 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:31:26.386 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:31:26.487 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:31:26.498 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:31:26 smithi089 bash[75068]: cluster 1685824285.1975033 mgr.y (mgr.34104) 1670 : cluster 0 pgmap v1725: 89 pgs: 1 active+recovering+remapped, 88 active+clean; 1.3 GiB data, 9.1 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 546 op/s; 976/63090 objects misplaced (1.547%); 2.5 MiB/s, 40 objects/s recovering 2023-06-03T20:31:26.499 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:31:26 smithi089 bash[78452]: cluster 1685824285.1975033 mgr.y (mgr.34104) 1670 : cluster 0 pgmap v1725: 89 pgs: 1 active+recovering+remapped, 88 active+clean; 1.3 GiB data, 9.1 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 546 op/s; 976/63090 objects misplaced (1.547%); 2.5 MiB/s, 40 objects/s recovering 2023-06-03T20:31:26.499 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 20:31:26 smithi089 bash[71163]: ::ffff:172.21.15.164 - - [03/Jun/2023:20:31:26] "GET /metrics HTTP/1.1" 200 229108 "" "Prometheus/2.33.4" 2023-06-03T20:31:26.513 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:31:26.526 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:31:26.527 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:31:26.629 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:31:26.660 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:31:26.662 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:31:26.662 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:31:26.763 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:31:26.796 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:31:26.799 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:31:26.800 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:31:26.900 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:31:26.944 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:31:26.949 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:31:26.950 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:31:27.050 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:31:27.075 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:31:27.078 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:31:27.079 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:31:27.179 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:31:27.218 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:31:27.221 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:31:27.221 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:31:27.322 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:31:27.357 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:31:27.358 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:31:27.359 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:31:27.460 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:31:27.490 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:31:27.494 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:31:27.494 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:31:27.595 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:31:27.636 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:31:27.642 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:31:27.643 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:31:27.744 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:31:27.772 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:31:27.775 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:31:27.775 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:31:27.876 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:31:27.908 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:31:27.915 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:31:27.917 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:31:28.018 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:31:28.038 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:31:28.040 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:31:28.040 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:31:28.141 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:31:28.176 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:31:28.181 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:31:28.181 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:31:28.282 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:31:28.317 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:31:28.320 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:31:28.321 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:31:28.422 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:31:28.452 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:31:28.455 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:31:28.455 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:31:28.532 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:31:28 smithi089 bash[75068]: cluster 1685824287.198066 mgr.y (mgr.34104) 1671 : cluster 0 pgmap v1726: 89 pgs: 1 active+recovering+remapped, 88 active+clean; 1.3 GiB data, 9.0 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 535 op/s; 976/62025 objects misplaced (1.574%); 2.3 MiB/s, 37 objects/s recovering 2023-06-03T20:31:28.533 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:31:28 smithi089 bash[78452]: cluster 1685824287.198066 mgr.y (mgr.34104) 1671 : cluster 0 pgmap v1726: 89 pgs: 1 active+recovering+remapped, 88 active+clean; 1.3 GiB data, 9.0 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 535 op/s; 976/62025 objects misplaced (1.574%); 2.3 MiB/s, 37 objects/s recovering 2023-06-03T20:31:28.556 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:31:28.568 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:31:28 smithi164 bash[59462]: cluster 1685824287.198066 mgr.y (mgr.34104) 1671 : cluster 0 pgmap v1726: 89 pgs: 1 active+recovering+remapped, 88 active+clean; 1.3 GiB data, 9.0 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 535 op/s; 976/62025 objects misplaced (1.574%); 2.3 MiB/s, 37 objects/s recovering 2023-06-03T20:31:28.594 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:31:28.602 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:31:28.603 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:31:28.704 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:31:28.733 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:31:28.735 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:31:28.735 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:31:28.836 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:31:28.870 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:31:28.872 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:31:28.873 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:31:28.974 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:31:29.005 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:31:29.010 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:31:29.011 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:31:29.112 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:31:29.144 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:31:29.147 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:31:29.148 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:31:29.249 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:31:29.288 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:31:29.291 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:31:29.291 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:31:29.392 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:31:29.427 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:31:29.434 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:31:29.435 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:31:29.535 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:31:29.570 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:31:29.575 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:31:29.576 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:31:29.677 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:31:29.717 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:31:29.722 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:31:29.723 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:31:29.823 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:31:29.870 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:31:29.875 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:31:29.876 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:31:29.977 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:31:30.015 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:31:30.017 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:31:30.018 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:31:30.119 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:31:30.150 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:31:30.160 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:31:30.160 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:31:30.261 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:31:30.267 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:31:30 smithi164 bash[59462]: cluster 1685824289.1987693 mgr.y (mgr.34104) 1672 : cluster 0 pgmap v1727: 89 pgs: 1 active+recovering+remapped, 88 active+clean; 1.1 GiB data, 8.5 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 666 op/s; 711/54471 objects misplaced (1.305%); 2.7 MiB/s, 55 objects/s recovering 2023-06-03T20:31:30.268 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:31:30 smithi089 bash[75068]: cluster 1685824289.1987693 mgr.y (mgr.34104) 1672 : cluster 0 pgmap v1727: 89 pgs: 1 active+recovering+remapped, 88 active+clean; 1.1 GiB data, 8.5 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 666 op/s; 711/54471 objects misplaced (1.305%); 2.7 MiB/s, 55 objects/s recovering 2023-06-03T20:31:30.268 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:31:30 smithi089 bash[78452]: cluster 1685824289.1987693 mgr.y (mgr.34104) 1672 : cluster 0 pgmap v1727: 89 pgs: 1 active+recovering+remapped, 88 active+clean; 1.1 GiB data, 8.5 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 666 op/s; 711/54471 objects misplaced (1.305%); 2.7 MiB/s, 55 objects/s recovering 2023-06-03T20:31:30.312 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:31:30.316 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:31:30.316 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:31:30.417 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:31:30.449 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:31:30.455 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:31:30.456 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:31:30.556 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:31:30.610 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:31:30.612 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:31:30.613 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:31:30.715 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:31:30.760 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:31:30.764 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:31:30.765 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:31:30.866 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:31:30.895 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:31:30.897 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:31:30.898 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:31:30.999 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:31:31.041 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:31:31.045 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:31:31.046 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:31:31.147 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:31:31.173 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:31:31.175 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:31:31.176 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:31:31.277 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:31:31.320 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:31:31.321 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:31:31.322 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:31:31.423 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:31:31.454 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:31:31.459 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:31:31.461 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:31:31.562 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:31:31.593 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:31:31.595 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:31:31.596 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:31:31.697 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:31:31.737 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:31:31.738 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:31:31.739 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:31:31.840 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:31:31.882 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:31:31.887 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:31:31.888 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:31:31.988 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:31:32.026 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:31:32.028 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:31:32.029 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:31:32.130 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:31:32.161 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:31:32.167 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:31:32.168 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:31:32.268 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:31:32.307 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:31:32.310 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:31:32.311 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:31:32.412 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:31:32.420 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:31:32 smithi089 bash[75068]: cluster 1685824291.199205 mgr.y (mgr.34104) 1673 : cluster 0 pgmap v1728: 89 pgs: 1 active+recovering+remapped, 88 active+clean; 1.1 GiB data, 8.5 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 480 op/s; 711/54471 objects misplaced (1.305%); 1.7 MiB/s, 39 objects/s recovering 2023-06-03T20:31:32.421 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:31:32 smithi089 bash[78452]: cluster 1685824291.199205 mgr.y (mgr.34104) 1673 : cluster 0 pgmap v1728: 89 pgs: 1 active+recovering+remapped, 88 active+clean; 1.1 GiB data, 8.5 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 480 op/s; 711/54471 objects misplaced (1.305%); 1.7 MiB/s, 39 objects/s recovering 2023-06-03T20:31:32.450 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:31:32.457 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:31:32.457 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:31:32.560 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:31:32.585 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:31:32 smithi164 bash[59462]: cluster 1685824291.199205 mgr.y (mgr.34104) 1673 : cluster 0 pgmap v1728: 89 pgs: 1 active+recovering+remapped, 88 active+clean; 1.1 GiB data, 8.5 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 480 op/s; 711/54471 objects misplaced (1.305%); 1.7 MiB/s, 39 objects/s recovering 2023-06-03T20:31:32.602 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:31:32.609 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:31:32.609 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:31:32.710 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:31:32.752 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:31:32.754 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:31:32.755 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:31:32.856 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:31:32.901 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:31:32.905 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:31:32.906 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:31:33.006 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:31:33.034 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:31:33.037 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:31:33.038 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:31:33.138 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:31:33.174 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:31:33.176 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:31:33.177 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:31:33.278 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:31:33.313 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:31:33.314 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:31:33.314 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:31:33.415 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:31:33.445 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:31:33.447 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:31:33.448 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:31:33.549 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:31:33.589 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:31:33.593 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:31:33.594 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:31:33.695 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:31:33.727 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:31:33.735 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:31:33.736 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:31:33.836 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:31:33.885 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:31:33.894 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:31:33.895 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:31:33.996 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:31:34.022 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:31:34.023 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:31:34.024 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:31:34.096 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:31:34 smithi089 bash[75068]: cluster 1685824293.1998618 mgr.y (mgr.34104) 1674 : cluster 0 pgmap v1729: 89 pgs: 1 active+recovering+remapped, 88 active+clean; 1.0 GiB data, 8.1 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 610 op/s; 388/49791 objects misplaced (0.779%); 1.7 MiB/s, 66 objects/s recovering 2023-06-03T20:31:34.125 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:31:34.153 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:31:34.155 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:31:34.155 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:31:34.256 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:31:34.288 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:31:34.290 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:31:34.290 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:31:34.391 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:31:34.405 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:31:34 smithi164 bash[59462]: cluster 1685824293.1998618 mgr.y (mgr.34104) 1674 : cluster 0 pgmap v1729: 89 pgs: 1 active+recovering+remapped, 88 active+clean; 1.0 GiB data, 8.1 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 610 op/s; 388/49791 objects misplaced (0.779%); 1.7 MiB/s, 66 objects/s recovering 2023-06-03T20:31:34.424 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:31:34.428 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:31:34.428 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:31:34.529 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:31:34.530 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:31:34 smithi089 bash[78452]: cluster 1685824293.1998618 mgr.y (mgr.34104) 1674 : cluster 0 pgmap v1729: 89 pgs: 1 active+recovering+remapped, 88 active+clean; 1.0 GiB data, 8.1 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 610 op/s; 388/49791 objects misplaced (0.779%); 1.7 MiB/s, 66 objects/s recovering 2023-06-03T20:31:34.562 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:31:34.565 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:31:34.565 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:31:34.666 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:31:34.705 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:31:34.710 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:31:34.711 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:31:34.812 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:31:34.842 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:31:34.849 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:31:34.850 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:31:34.951 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:31:34.985 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:31:34.985 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:31:34.986 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:31:35.087 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:31:35.126 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:31:35.131 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:31:35.132 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:31:35.233 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:31:35.261 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:31:35.266 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:31:35.266 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:31:35.367 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:31:35.397 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:31:35.403 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:31:35.403 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:31:35.504 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:31:35.542 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:31:35.545 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:31:35.545 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:31:35.646 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:31:35.676 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:31:35.680 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:31:35.680 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:31:35.781 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:31:35.818 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:31:35.828 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:31:35.829 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:31:35.930 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:31:35.963 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:31:35.965 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:31:35.966 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:31:36.067 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:31:36.108 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:31:36.113 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:31:36.114 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:31:36.214 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:31:36.256 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:31:36.259 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:31:36.260 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:31:36.263 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:31:36 smithi164 bash[59462]: cluster 1685824295.2004035 mgr.y (mgr.34104) 1675 : cluster 0 pgmap v1730: 89 pgs: 1 active+recovering+remapped, 88 active+clean; 938 MiB data, 7.9 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 627 op/s; 388/45063 objects misplaced (0.861%); 613 KiB/s, 48 objects/s recovering 2023-06-03T20:31:36.360 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:31:36.397 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:31:36.399 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:31:36.400 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:31:36.501 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:31:36.533 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:31:36.539 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:31:36 smithi089 bash[75068]: cluster 1685824295.2004035 mgr.y (mgr.34104) 1675 : cluster 0 pgmap v1730: 89 pgs: 1 active+recovering+remapped, 88 active+clean; 938 MiB data, 7.9 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 627 op/s; 388/45063 objects misplaced (0.861%); 613 KiB/s, 48 objects/s recovering 2023-06-03T20:31:36.539 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 20:31:36 smithi089 bash[71163]: ::ffff:172.21.15.164 - - [03/Jun/2023:20:31:36] "GET /metrics HTTP/1.1" 200 229108 "" "Prometheus/2.33.4" 2023-06-03T20:31:36.540 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:31:36 smithi089 bash[78452]: cluster 1685824295.2004035 mgr.y (mgr.34104) 1675 : cluster 0 pgmap v1730: 89 pgs: 1 active+recovering+remapped, 88 active+clean; 938 MiB data, 7.9 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 627 op/s; 388/45063 objects misplaced (0.861%); 613 KiB/s, 48 objects/s recovering 2023-06-03T20:31:36.540 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:31:36.541 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:31:36.642 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:31:36.679 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:31:36.685 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:31:36.686 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:31:36.786 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:31:36.830 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:31:36.836 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:31:36.837 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:31:36.937 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:31:36.969 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:31:36.971 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:31:36.971 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:31:37.072 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:31:37.113 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:31:37.119 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:31:37.120 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:31:37.221 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:31:37.252 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:31:37.256 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:31:37.256 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:31:37.357 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:31:37.366 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:31:37 smithi089 bash[78452]: audit 1685824297.160426 mon.c (mon.2) 677 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T20:31:37.366 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:31:37 smithi089 bash[75068]: audit 1685824297.160426 mon.c (mon.2) 677 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T20:31:37.394 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:31:37.399 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:31:37.400 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:31:37.501 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:31:37.531 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:31:37.534 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:31:37.535 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:31:37.585 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:31:37 smithi164 bash[59462]: audit 1685824297.160426 mon.c (mon.2) 677 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T20:31:37.636 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:31:37.668 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:31:37.673 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:31:37.674 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:31:37.774 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:31:37.829 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:31:37.834 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:31:37.835 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:31:37.936 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:31:37.967 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:31:37.971 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:31:37.971 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:31:38.072 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:31:38.109 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:31:38.115 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:31:38.116 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:31:38.217 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:31:38.253 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:31:38.254 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:31:38.255 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:31:38.276 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:31:38 smithi089 bash[75068]: cluster 1685824297.2010124 mgr.y (mgr.34104) 1676 : cluster 0 pgmap v1731: 89 pgs: 1 active+recovering+remapped, 88 active+clean; 915 MiB data, 7.8 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 531 op/s; 388/43953 objects misplaced (0.883%); 613 KiB/s, 48 objects/s recovering 2023-06-03T20:31:38.276 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:31:38 smithi089 bash[78452]: cluster 1685824297.2010124 mgr.y (mgr.34104) 1676 : cluster 0 pgmap v1731: 89 pgs: 1 active+recovering+remapped, 88 active+clean; 915 MiB data, 7.8 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 531 op/s; 388/43953 objects misplaced (0.883%); 613 KiB/s, 48 objects/s recovering 2023-06-03T20:31:38.356 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:31:38.385 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:31:38.389 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:31:38.390 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:31:38.490 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:31:38.498 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:31:38 smithi164 bash[59462]: cluster 1685824297.2010124 mgr.y (mgr.34104) 1676 : cluster 0 pgmap v1731: 89 pgs: 1 active+recovering+remapped, 88 active+clean; 915 MiB data, 7.8 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 531 op/s; 388/43953 objects misplaced (0.883%); 613 KiB/s, 48 objects/s recovering 2023-06-03T20:31:38.526 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:31:38.531 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:31:38.532 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:31:38.633 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:31:38.666 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:31:38.668 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:31:38.668 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:31:38.770 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:31:38.800 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:31:38.803 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:31:38.803 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:31:38.905 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:31:38.941 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:31:38.945 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:31:38.946 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:31:39.046 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:31:39.088 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:31:39.092 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:31:39.093 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:31:39.194 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:31:39.230 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:31:39.231 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:31:39.232 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:31:39.333 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:31:39.365 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:31:39.366 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:31:39.366 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:31:39.467 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:31:39.493 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:31:39.494 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:31:39.495 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:31:39.596 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:31:39.627 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:31:39.630 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:31:39.630 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:31:39.731 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:31:39.763 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:31:39.765 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:31:39.766 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:31:39.867 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:31:39.906 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:31:39.909 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:31:39.910 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:31:40.011 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:31:40.045 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:31:40.048 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:31:40.049 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:31:40.149 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:31:40.183 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:31:40.185 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:31:40.185 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:31:40.286 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:31:40.314 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:31:40.316 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:31:40.316 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:31:40.417 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:31:40.427 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:31:40 smithi089 bash[75068]: cluster 1685824299.201742 mgr.y (mgr.34104) 1677 : cluster 0 pgmap v1732: 89 pgs: 1 active+recovering+remapped, 88 active+clean; 757 MiB data, 7.4 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 712 op/s; 71/36360 objects misplaced (0.195%); 613 KiB/s, 75 objects/s recovering 2023-06-03T20:31:40.428 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:31:40 smithi089 bash[75068]: cluster 1685824299.2784088 osd.6 (osd.6) 230 : cluster 3 failed to encode map e294 with expected crc 2023-06-03T20:31:40.428 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:31:40 smithi089 bash[75068]: cluster 1685824299.280888 mon.a (mon.0) 1651 : cluster 0 osdmap e294: 8 total, 8 up, 8 in 2023-06-03T20:31:40.428 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:31:40 smithi089 bash[78452]: cluster 1685824299.201742 mgr.y (mgr.34104) 1677 : cluster 0 pgmap v1732: 89 pgs: 1 active+recovering+remapped, 88 active+clean; 757 MiB data, 7.4 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 712 op/s; 71/36360 objects misplaced (0.195%); 613 KiB/s, 75 objects/s recovering 2023-06-03T20:31:40.429 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:31:40 smithi089 bash[78452]: cluster 1685824299.2784088 osd.6 (osd.6) 230 : cluster 3 failed to encode map e294 with expected crc 2023-06-03T20:31:40.429 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:31:40 smithi089 bash[78452]: cluster 1685824299.280888 mon.a (mon.0) 1651 : cluster 0 osdmap e294: 8 total, 8 up, 8 in 2023-06-03T20:31:40.456 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:31:40.458 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:31:40.459 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:31:40.560 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:31:40.585 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:31:40 smithi164 bash[59462]: cluster 1685824299.201742 mgr.y (mgr.34104) 1677 : cluster 0 pgmap v1732: 89 pgs: 1 active+recovering+remapped, 88 active+clean; 757 MiB data, 7.4 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 712 op/s; 71/36360 objects misplaced (0.195%); 613 KiB/s, 75 objects/s recovering 2023-06-03T20:31:40.586 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:31:40 smithi164 bash[59462]: cluster 1685824299.2784088 osd.6 (osd.6) 230 : cluster 3 failed to encode map e294 with expected crc 2023-06-03T20:31:40.586 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:31:40 smithi164 bash[59462]: cluster 1685824299.280888 mon.a (mon.0) 1651 : cluster 0 osdmap e294: 8 total, 8 up, 8 in 2023-06-03T20:31:40.616 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:31:40.621 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:31:40.622 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:31:40.723 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:31:40.752 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:31:40.756 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:31:40.756 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:31:40.857 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:31:40.883 INFO:teuthology.orchestra.run.smithi089.stdout:{ 2023-06-03T20:31:40.883 INFO:teuthology.orchestra.run.smithi089.stdout: "ops": [ 2023-06-03T20:31:40.883 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:31:40.883 INFO:teuthology.orchestra.run.smithi089.stdout: "description": "osd_op(client.46739.0:185131 14.0 14:0fe9d17a:::benchmark_data_smithi089_237281_object87115:head [delete] snapc 0=[] ondisk+write+known_if_redirected e295)", 2023-06-03T20:31:40.883 INFO:teuthology.orchestra.run.smithi089.stdout: "initiated_at": "2023-06-03T20:31:40.817858+0000", 2023-06-03T20:31:40.884 INFO:teuthology.orchestra.run.smithi089.stdout: "age": 0.058990612999999997, 2023-06-03T20:31:40.884 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.059023196999999999, 2023-06-03T20:31:40.884 INFO:teuthology.orchestra.run.smithi089.stdout: "type_data": { 2023-06-03T20:31:40.884 INFO:teuthology.orchestra.run.smithi089.stdout: "flag_point": "waiting for sub ops", 2023-06-03T20:31:40.884 INFO:teuthology.orchestra.run.smithi089.stdout: "client_info": { 2023-06-03T20:31:40.884 INFO:teuthology.orchestra.run.smithi089.stdout: "client": "client.46739", 2023-06-03T20:31:40.885 INFO:teuthology.orchestra.run.smithi089.stdout: "client_addr": "172.21.15.89:0/2479974613", 2023-06-03T20:31:40.885 INFO:teuthology.orchestra.run.smithi089.stdout: "tid": 185131 2023-06-03T20:31:40.885 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:31:40.885 INFO:teuthology.orchestra.run.smithi089.stdout: "events": [ 2023-06-03T20:31:40.885 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:31:40.886 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "initiated", 2023-06-03T20:31:40.886 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:31:40.817858+0000", 2023-06-03T20:31:40.886 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0 2023-06-03T20:31:40.886 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:31:40.886 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:31:40.886 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "throttled", 2023-06-03T20:31:40.886 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:31:40.817858+0000", 2023-06-03T20:31:40.887 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0 2023-06-03T20:31:40.887 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:31:40.887 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:31:40.887 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "header_read", 2023-06-03T20:31:40.887 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:31:40.817857+0000", 2023-06-03T20:31:40.887 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 4294967295.9999981 2023-06-03T20:31:40.887 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:31:40.888 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:31:40.888 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "all_read", 2023-06-03T20:31:40.888 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:31:40.817863+0000", 2023-06-03T20:31:40.888 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 6.7100000000000001e-06 2023-06-03T20:31:40.888 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:31:40.888 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:31:40.888 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "dispatched", 2023-06-03T20:31:40.889 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:31:40.817877+0000", 2023-06-03T20:31:40.889 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 1.3223e-05 2023-06-03T20:31:40.889 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:31:40.889 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:31:40.889 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "queued_for_pg", 2023-06-03T20:31:40.889 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:31:40.817885+0000", 2023-06-03T20:31:40.890 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 8.0760000000000003e-06 2023-06-03T20:31:40.890 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:31:40.890 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:31:40.890 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "reached_pg", 2023-06-03T20:31:40.890 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:31:40.826259+0000", 2023-06-03T20:31:40.890 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.0083739850000000005 2023-06-03T20:31:40.890 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:31:40.890 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:31:40.891 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "started", 2023-06-03T20:31:40.891 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:31:40.826373+0000", 2023-06-03T20:31:40.891 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.000114574 2023-06-03T20:31:40.891 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:31:40.891 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:31:40.891 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "waiting for subops from 0,5", 2023-06-03T20:31:40.891 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:31:40.826484+0000", 2023-06-03T20:31:40.892 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.000110568 2023-06-03T20:31:40.892 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:31:40.892 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:31:40.892 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:31:40.892 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:31:40.840806+0000", 2023-06-03T20:31:40.892 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.014322441999999999 2023-06-03T20:31:40.892 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:31:40.892 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:31:40.893 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "op_commit", 2023-06-03T20:31:40.893 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:31:40.853658+0000", 2023-06-03T20:31:40.893 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.012851583999999999 2023-06-03T20:31:40.893 INFO:teuthology.orchestra.run.smithi089.stdout: } 2023-06-03T20:31:40.893 INFO:teuthology.orchestra.run.smithi089.stdout: ] 2023-06-03T20:31:40.893 INFO:teuthology.orchestra.run.smithi089.stdout: } 2023-06-03T20:31:40.893 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:31:40.893 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:31:40.894 INFO:teuthology.orchestra.run.smithi089.stdout: "description": "osd_repop(client.46739.0:185150 14.c e295/278 14:3d1d931c:::benchmark_data_smithi089_237281_object87134:head v 295'11545, mlcod=295'11542)", 2023-06-03T20:31:40.894 INFO:teuthology.orchestra.run.smithi089.stdout: "initiated_at": "2023-06-03T20:31:40.873745+0000", 2023-06-03T20:31:40.894 INFO:teuthology.orchestra.run.smithi089.stdout: "age": 0.003104253, 2023-06-03T20:31:40.894 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.0032314290000000001, 2023-06-03T20:31:40.894 INFO:teuthology.orchestra.run.smithi089.stdout: "type_data": { 2023-06-03T20:31:40.894 INFO:teuthology.orchestra.run.smithi089.stdout: "flag_point": "started", 2023-06-03T20:31:40.894 INFO:teuthology.orchestra.run.smithi089.stdout: "events": [ 2023-06-03T20:31:40.895 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:31:40.895 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "initiated", 2023-06-03T20:31:40.895 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:31:40.873745+0000", 2023-06-03T20:31:40.895 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0 2023-06-03T20:31:40.895 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:31:40.895 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:31:40.896 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "throttled", 2023-06-03T20:31:40.896 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:31:40.873745+0000", 2023-06-03T20:31:40.896 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0 2023-06-03T20:31:40.896 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:31:40.896 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:31:40.896 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "header_read", 2023-06-03T20:31:40.896 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:31:40.873744+0000", 2023-06-03T20:31:40.897 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 4294967295.9999995 2023-06-03T20:31:40.897 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:31:40.897 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:31:40.897 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "all_read", 2023-06-03T20:31:40.897 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:31:40.873750+0000", 2023-06-03T20:31:40.897 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 5.4720000000000003e-06 2023-06-03T20:31:40.897 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:31:40.898 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:31:40.898 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "dispatched", 2023-06-03T20:31:40.898 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:31:40.873758+0000", 2023-06-03T20:31:40.898 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 7.7400000000000004e-06 2023-06-03T20:31:40.898 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:31:40.898 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:31:40.898 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "queued_for_pg", 2023-06-03T20:31:40.899 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:31:40.873763+0000", 2023-06-03T20:31:40.899 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 4.853e-06 2023-06-03T20:31:40.899 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:31:40.899 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:31:40.899 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "reached_pg", 2023-06-03T20:31:40.899 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:31:40.874005+0000", 2023-06-03T20:31:40.900 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.00024268799999999999 2023-06-03T20:31:40.900 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:31:40.901 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:31:40.901 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "started", 2023-06-03T20:31:40.901 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:31:40.874061+0000", 2023-06-03T20:31:40.901 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 5.5454000000000001e-05 2023-06-03T20:31:40.901 INFO:teuthology.orchestra.run.smithi089.stdout: } 2023-06-03T20:31:40.901 INFO:teuthology.orchestra.run.smithi089.stdout: ] 2023-06-03T20:31:40.902 INFO:teuthology.orchestra.run.smithi089.stdout: } 2023-06-03T20:31:40.902 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:31:40.902 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:31:40.902 INFO:teuthology.orchestra.run.smithi089.stdout: "description": "osd_repop(client.46739.0:185160 14.1 e295/278 14:85f60550:::benchmark_data_smithi089_237281_object87144:head v 295'12051, mlcod=295'12049)", 2023-06-03T20:31:40.902 INFO:teuthology.orchestra.run.smithi089.stdout: "initiated_at": "2023-06-03T20:31:40.876034+0000", 2023-06-03T20:31:40.902 INFO:teuthology.orchestra.run.smithi089.stdout: "age": 0.00081475700000000005, 2023-06-03T20:31:40.902 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.001003831, 2023-06-03T20:31:40.902 INFO:teuthology.orchestra.run.smithi089.stdout: "type_data": { 2023-06-03T20:31:40.903 INFO:teuthology.orchestra.run.smithi089.stdout: "flag_point": "started", 2023-06-03T20:31:40.903 INFO:teuthology.orchestra.run.smithi089.stdout: "events": [ 2023-06-03T20:31:40.903 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:31:40.903 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "initiated", 2023-06-03T20:31:40.903 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:31:40.876034+0000", 2023-06-03T20:31:40.903 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0 2023-06-03T20:31:40.903 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:31:40.904 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:31:40.904 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "throttled", 2023-06-03T20:31:40.904 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:31:40.876034+0000", 2023-06-03T20:31:40.904 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0 2023-06-03T20:31:40.904 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:31:40.904 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:31:40.904 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "header_read", 2023-06-03T20:31:40.905 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:31:40.876033+0000", 2023-06-03T20:31:40.905 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 4294967295.999999 2023-06-03T20:31:40.905 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:31:40.905 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:31:40.905 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "all_read", 2023-06-03T20:31:40.905 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:31:40.876041+0000", 2023-06-03T20:31:40.905 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 8.1880000000000002e-06 2023-06-03T20:31:40.905 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:31:40.906 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:31:40.906 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "dispatched", 2023-06-03T20:31:40.906 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:31:40.876058+0000", 2023-06-03T20:31:40.906 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 1.6801999999999999e-05 2023-06-03T20:31:40.906 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:31:40.906 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:31:40.906 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "queued_for_pg", 2023-06-03T20:31:40.907 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:31:40.876068+0000", 2023-06-03T20:31:40.907 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 9.9289999999999993e-06 2023-06-03T20:31:40.907 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:31:40.907 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:31:40.907 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "reached_pg", 2023-06-03T20:31:40.907 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:31:40.876122+0000", 2023-06-03T20:31:40.907 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 5.4174e-05 2023-06-03T20:31:40.907 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:31:40.908 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:31:40.908 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "started", 2023-06-03T20:31:40.908 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:31:40.876149+0000", 2023-06-03T20:31:40.908 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 2.6449999999999999e-05 2023-06-03T20:31:40.908 INFO:teuthology.orchestra.run.smithi089.stdout: } 2023-06-03T20:31:40.908 INFO:teuthology.orchestra.run.smithi089.stdout: ] 2023-06-03T20:31:40.908 INFO:teuthology.orchestra.run.smithi089.stdout: } 2023-06-03T20:31:40.908 INFO:teuthology.orchestra.run.smithi089.stdout: } 2023-06-03T20:31:40.909 INFO:teuthology.orchestra.run.smithi089.stdout: ], 2023-06-03T20:31:40.909 INFO:teuthology.orchestra.run.smithi089.stdout: "num_ops": 3 2023-06-03T20:31:40.909 INFO:teuthology.orchestra.run.smithi089.stdout:} 2023-06-03T20:31:40.910 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:31:40.910 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:31:40.910 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:31:41.011 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:31:41.051 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:31:41.052 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:31:41.053 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:31:41.154 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:31:41.188 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:31:41.193 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:31:41.194 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:31:41.295 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:31:41.299 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:31:41 smithi164 bash[59462]: cluster 1685824300.276047 osd.7 (osd.7) 239 : cluster 3 failed to encode map e295 with expected crc 2023-06-03T20:31:41.300 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:31:41 smithi164 bash[59462]: cluster 1685824300.2799475 osd.6 (osd.6) 231 : cluster 3 failed to encode map e295 with expected crc 2023-06-03T20:31:41.300 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:31:41 smithi164 bash[59462]: cluster 1685824300.2826138 mon.a (mon.0) 1652 : cluster 0 osdmap e295: 8 total, 8 up, 8 in 2023-06-03T20:31:41.330 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:31:41.336 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:31:41.337 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:31:41.438 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:31:41.469 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:31:41.471 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:31:41.471 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:31:41.529 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:31:41 smithi089 bash[75068]: cluster 1685824300.276047 osd.7 (osd.7) 239 : cluster 3 failed to encode map e295 with expected crc 2023-06-03T20:31:41.529 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:31:41 smithi089 bash[75068]: cluster 1685824300.2799475 osd.6 (osd.6) 231 : cluster 3 failed to encode map e295 with expected crc 2023-06-03T20:31:41.530 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:31:41 smithi089 bash[75068]: cluster 1685824300.2826138 mon.a (mon.0) 1652 : cluster 0 osdmap e295: 8 total, 8 up, 8 in 2023-06-03T20:31:41.530 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:31:41 smithi089 bash[78452]: cluster 1685824300.276047 osd.7 (osd.7) 239 : cluster 3 failed to encode map e295 with expected crc 2023-06-03T20:31:41.530 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:31:41 smithi089 bash[78452]: cluster 1685824300.2799475 osd.6 (osd.6) 231 : cluster 3 failed to encode map e295 with expected crc 2023-06-03T20:31:41.530 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:31:41 smithi089 bash[78452]: cluster 1685824300.2826138 mon.a (mon.0) 1652 : cluster 0 osdmap e295: 8 total, 8 up, 8 in 2023-06-03T20:31:41.573 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:31:41.613 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:31:41.617 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:31:41.618 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:31:41.719 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:31:41.755 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:31:41.757 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:31:41.758 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:31:41.859 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:31:41.890 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:31:41.893 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:31:41.893 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:31:41.994 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:31:42.046 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:31:42.051 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:31:42.052 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:31:42.152 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:31:42.196 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:31:42.199 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:31:42.200 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:31:42.301 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:31:42.330 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:31:42.332 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:31:42.333 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:31:42.434 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:31:42.447 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:31:42 smithi164 bash[59462]: cluster 1685824301.2021716 mgr.y (mgr.34104) 1678 : cluster 0 pgmap v1735: 89 pgs: 1 active+recovering+remapped, 88 active+clean; 757 MiB data, 7.4 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 559 op/s; 71/36360 objects misplaced (0.195%); 0 B/s, 39 objects/s recovering 2023-06-03T20:31:42.472 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:31:42.474 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:31:42.475 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:31:42.576 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:31:42.614 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:31:42.622 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:31:42.623 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:31:42.724 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:31:42.733 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:31:42 smithi089 bash[78452]: cluster 1685824301.2021716 mgr.y (mgr.34104) 1678 : cluster 0 pgmap v1735: 89 pgs: 1 active+recovering+remapped, 88 active+clean; 757 MiB data, 7.4 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 559 op/s; 71/36360 objects misplaced (0.195%); 0 B/s, 39 objects/s recovering 2023-06-03T20:31:42.734 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:31:42 smithi089 bash[75068]: cluster 1685824301.2021716 mgr.y (mgr.34104) 1678 : cluster 0 pgmap v1735: 89 pgs: 1 active+recovering+remapped, 88 active+clean; 757 MiB data, 7.4 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 559 op/s; 71/36360 objects misplaced (0.195%); 0 B/s, 39 objects/s recovering 2023-06-03T20:31:42.758 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:31:42.765 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:31:42.765 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:31:42.866 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:31:42.895 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:31:42.896 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:31:42.897 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:31:42.998 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:31:43.029 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:31:43.031 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:31:43.032 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:31:43.132 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:31:43.163 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:31:43.167 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:31:43.168 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:31:43.268 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:31:43.299 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:31:43.302 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:31:43.303 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:31:43.404 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:31:43.446 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:31:43.455 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:31:43.456 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:31:43.557 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:31:43.596 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:31:43.599 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:31:43.600 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:31:43.701 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:31:43.741 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:31:43.746 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:31:43.746 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:31:43.847 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:31:43.875 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:31:43.884 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:31:43.884 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:31:43.985 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:31:44.021 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:31:44.022 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:31:44.023 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:31:44.124 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:31:44.169 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:31:44.171 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:31:44.172 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:31:44.273 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:31:44.279 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:31:44 smithi089 bash[78452]: cluster 1685824303.202822 mgr.y (mgr.34104) 1679 : cluster 0 pgmap v1736: 89 pgs: 89 active+clean; 673 MiB data, 7.1 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 529 op/s; 0 B/s, 48 objects/s recovering 2023-06-03T20:31:44.279 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:31:44 smithi089 bash[75068]: cluster 1685824303.202822 mgr.y (mgr.34104) 1679 : cluster 0 pgmap v1736: 89 pgs: 89 active+clean; 673 MiB data, 7.1 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 529 op/s; 0 B/s, 48 objects/s recovering 2023-06-03T20:31:44.302 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:31:44.310 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:31:44.311 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:31:44.412 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:31:44.443 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:31:44.450 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:31:44.450 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:31:44.551 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:31:44.558 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:31:44 smithi164 bash[59462]: cluster 1685824303.202822 mgr.y (mgr.34104) 1679 : cluster 0 pgmap v1736: 89 pgs: 89 active+clean; 673 MiB data, 7.1 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 529 op/s; 0 B/s, 48 objects/s recovering 2023-06-03T20:31:44.586 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:31:44.591 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:31:44.592 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:31:44.693 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:31:44.729 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:31:44.731 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:31:44.732 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:31:44.833 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:31:44.870 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:31:44.875 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:31:44.876 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:31:44.977 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:31:45.008 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:31:45.014 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:31:45.014 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:31:45.115 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:31:45.160 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:31:45.163 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:31:45.164 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:31:45.265 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:31:45.295 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:31:45.300 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:31:45.301 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:31:45.402 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:31:45.433 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:31:45.436 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:31:45.437 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:31:45.538 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:31:45.578 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:31:45.582 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:31:45.583 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:31:45.685 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:31:45.721 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:31:45.725 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:31:45.725 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:31:45.826 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:31:45.865 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:31:45.868 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:31:45.868 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:31:45.969 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:31:45.977 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph daemon osd.2 dump_blocked_ops 2023-06-03T20:31:46.005 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:31:46.006 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:31:46.007 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:31:46.108 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:31:46.144 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:31:46.149 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:31:46.150 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:31:46.233 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 20:31:46 smithi089 bash[71163]: ::ffff:172.21.15.164 - - [03/Jun/2023:20:31:46] "GET /metrics HTTP/1.1" 200 229096 "" "Prometheus/2.33.4" 2023-06-03T20:31:46.251 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:31:46.280 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:31:46.282 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:31:46.283 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:31:46.384 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:31:46.876 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:31:46 smithi164 bash[59462]: cluster 1685824305.2033868 mgr.y (mgr.34104) 1680 : cluster 0 pgmap v1737: 89 pgs: 89 active+clean; 593 MiB data, 6.9 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 643 op/s; 0 B/s, 48 objects/s recovering 2023-06-03T20:31:46.877 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:31:46 smithi089 bash[75068]: cluster 1685824305.2033868 mgr.y (mgr.34104) 1680 : cluster 0 pgmap v1737: 89 pgs: 89 active+clean; 593 MiB data, 6.9 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 643 op/s; 0 B/s, 48 objects/s recovering 2023-06-03T20:31:46.877 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:31:46 smithi089 bash[78452]: cluster 1685824305.2033868 mgr.y (mgr.34104) 1680 : cluster 0 pgmap v1737: 89 pgs: 89 active+clean; 593 MiB data, 6.9 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 643 op/s; 0 B/s, 48 objects/s recovering 2023-06-03T20:31:46.927 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:31:46.934 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:31:46.935 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:31:47.036 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:31:47.069 INFO:teuthology.orchestra.run.smithi089.stdout:osd.0: {} 2023-06-03T20:31:47.089 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:31:47.095 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:31:47.095 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:31:47.112 INFO:teuthology.orchestra.run.smithi089.stdout:osd.1: {} 2023-06-03T20:31:47.161 INFO:teuthology.orchestra.run.smithi089.stdout:osd.2: {} 2023-06-03T20:31:47.197 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:31:47.207 INFO:teuthology.orchestra.run.smithi089.stdout:osd.3: {} 2023-06-03T20:31:47.231 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:31:47.233 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:31:47.234 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:31:47.253 INFO:teuthology.orchestra.run.smithi089.stdout:osd.4: {} 2023-06-03T20:31:47.293 INFO:teuthology.orchestra.run.smithi089.stdout:osd.5: {} 2023-06-03T20:31:47.333 INFO:teuthology.orchestra.run.smithi089.stdout:osd.6: {} 2023-06-03T20:31:47.335 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:31:47.376 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:31:47.377 INFO:teuthology.orchestra.run.smithi089.stdout:osd.7: {} 2023-06-03T20:31:47.382 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:31:47.383 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:31:47.383 INFO:teuthology.orchestra.run.smithi089.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-06-03T20:31:47.384 INFO:teuthology.orchestra.run.smithi089.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-06-03T20:31:47.384 INFO:teuthology.orchestra.run.smithi089.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-06-03T20:31:47.384 INFO:teuthology.orchestra.run.smithi089.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-06-03T20:31:47.385 INFO:teuthology.orchestra.run.smithi089.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-06-03T20:31:47.385 INFO:teuthology.orchestra.run.smithi089.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-06-03T20:31:47.385 INFO:teuthology.orchestra.run.smithi089.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-06-03T20:31:47.386 INFO:teuthology.orchestra.run.smithi089.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-06-03T20:31:47.399 INFO:teuthology.orchestra.run.smithi089.stdout: 2023-06-03T20:31:47.399 INFO:teuthology.orchestra.run.smithi089.stdout:{"fsid":"f1db3478-0243-11ee-9b1d-001a4aab830c","health":{"status":"HEALTH_OK","checks":{},"mutes":[]},"election_epoch":36,"quorum":[0,1,2],"quorum_names":["a","b","c"],"quorum_age":2812,"monmap":{"epoch":4,"min_mon_release_name":"reef","num_mons":3},"osdmap":{"epoch":295,"num_osds":8,"num_up_osds":8,"osd_up_since":1685823806,"num_in_osds":8,"osd_in_since":1685823841,"num_remapped_pgs":0},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":89}],"num_pgs":89,"num_pools":5,"num_objects":9114,"data_bytes":596252582,"bytes_used":7349592064,"bytes_avail":760610693120,"bytes_total":767960285184,"recovering_objects_per_sec":8,"recovering_bytes_per_sec":0,"recovering_keys_per_sec":0,"num_objects_recovered":71,"num_bytes_recovered":0,"num_keys_recovered":0,"write_bytes_sec":0,"read_op_per_sec":0,"write_op_per_sec":375},"fsmap":{"epoch":7,"id":1,"up":1,"in":1,"max":1,"by_rank":[{"filesystem_id":1,"rank":0,"name":"foo.smithi089.giuxuj","status":"up:active","gid":14646}],"up:standby":1},"mgrmap":{"available":true,"num_standbys":1,"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"services":{"dashboard":"https://172.21.15.89:8443/","prometheus":"http://172.21.15.89:9283/"}},"servicemap":{"epoch":40,"modified":"2023-06-03T20:30:49.187120+0000","services":{}},"progress_events":{"16787ee9-eb5a-44b2-97e1-37974eb2bde6":{"message":"Global Recovery Event (0s)\n [............................] ","progress":0,"add_to_ceph_s":true},"2c7b9f96-2368-4bf9-83ac-8c6c53d71110":{"message":"Upgrade to 18.0.0-3532-g8e24bcf4 (14m)\n [============................] (remaining: 17m)","progress":0.44999998807907104,"add_to_ceph_s":true}}} 2023-06-03T20:31:47.484 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:31:47.523 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:31:47.525 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:31:47.525 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:31:47.585 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:31:47 smithi164 bash[59462]: audit 1685824307.019876 mon.a (mon.0) 1653 : audit 0 from='client.? 172.21.15.89:0/1901185969' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-03T20:31:47.624 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:31:47 smithi089 bash[78452]: audit 1685824307.019876 mon.a (mon.0) 1653 : audit 0 from='client.? 172.21.15.89:0/1901185969' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-03T20:31:47.624 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:31:47 smithi089 bash[75068]: audit 1685824307.019876 mon.a (mon.0) 1653 : audit 0 from='client.? 172.21.15.89:0/1901185969' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-03T20:31:47.627 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:31:47.661 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:31:47.665 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:31:47.666 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:31:47.767 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:31:47.803 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:31:47.807 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:31:47.808 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:31:47.908 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:31:47.939 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:31:47.942 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:31:47.943 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:31:48.044 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:31:48.075 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:31:48.080 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:31:48.080 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:31:48.181 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:31:48.203 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:31:48.205 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:31:48.206 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:31:48.290 INFO:teuthology.orchestra.run.smithi089.stderr:noscrub is set 2023-06-03T20:31:48.307 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:31:48.316 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:31:48 smithi164 bash[59462]: cluster 1685824307.2039042 mgr.y (mgr.34104) 1681 : cluster 0 pgmap v1738: 89 pgs: 89 active+clean; 569 MiB data, 6.8 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 375 op/s; 0 B/s, 8 objects/s recovering 2023-06-03T20:31:48.316 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:31:48 smithi164 bash[59462]: audit 1685824307.2724783 mon.a (mon.0) 1654 : audit 1 from='client.? 172.21.15.89:0/1504953376' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-06-03T20:31:48.316 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:31:48 smithi164 bash[59462]: audit 1685824307.3939734 mon.a (mon.0) 1655 : audit 0 from='client.? 172.21.15.89:0/3099639284' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-06-03T20:31:48.317 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:31:48 smithi164 bash[59462]: cluster 1685824308.2646644 mon.a (mon.0) 1656 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-06-03T20:31:48.344 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:31:48.347 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:31:48.348 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:31:48.449 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:31:48.482 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:31:48.490 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:31:48.490 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:31:48.591 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:31:48.618 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:31:48.620 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:31:48.620 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:31:48.720 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:31:48 smithi089 bash[75068]: cluster 1685824307.2039042 mgr.y (mgr.34104) 1681 : cluster 0 pgmap v1738: 89 pgs: 89 active+clean; 569 MiB data, 6.8 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 375 op/s; 0 B/s, 8 objects/s recovering 2023-06-03T20:31:48.720 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:31:48 smithi089 bash[75068]: audit 1685824307.2724783 mon.a (mon.0) 1654 : audit 1 from='client.? 172.21.15.89:0/1504953376' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-06-03T20:31:48.721 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:31:48 smithi089 bash[75068]: audit 1685824307.3939734 mon.a (mon.0) 1655 : audit 0 from='client.? 172.21.15.89:0/3099639284' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-06-03T20:31:48.721 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:31:48 smithi089 bash[75068]: cluster 1685824308.2646644 mon.a (mon.0) 1656 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-06-03T20:31:48.721 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:31:48 smithi089 bash[78452]: cluster 1685824307.2039042 mgr.y (mgr.34104) 1681 : cluster 0 pgmap v1738: 89 pgs: 89 active+clean; 569 MiB data, 6.8 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 375 op/s; 0 B/s, 8 objects/s recovering 2023-06-03T20:31:48.721 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:31:48 smithi089 bash[78452]: audit 1685824307.2724783 mon.a (mon.0) 1654 : audit 1 from='client.? 172.21.15.89:0/1504953376' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-06-03T20:31:48.721 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:31:48 smithi089 bash[78452]: audit 1685824307.3939734 mon.a (mon.0) 1655 : audit 0 from='client.? 172.21.15.89:0/3099639284' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-06-03T20:31:48.721 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:31:48 smithi089 bash[78452]: cluster 1685824308.2646644 mon.a (mon.0) 1656 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-06-03T20:31:48.722 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:31:48.746 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:31:48.751 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:31:48.751 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:31:48.852 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:31:48.890 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:31:48.893 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:31:48.893 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:31:48.994 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:31:49.030 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:31:49.036 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:31:49.037 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:31:49.137 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:31:49.185 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:31:49.188 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:31:49.190 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:31:49.291 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:31:49.326 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:31:49.327 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:31:49.328 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:31:49.429 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:31:49.461 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:31:49.461 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:31:49.462 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:31:49.529 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:31:49 smithi089 bash[78452]: cluster 1685824308.2640579 osd.6 (osd.6) 232 : cluster 3 failed to encode map e296 with expected crc 2023-06-03T20:31:49.529 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:31:49 smithi089 bash[78452]: cluster 1685824308.2769046 osd.5 (osd.5) 244 : cluster 3 failed to encode map e296 with expected crc 2023-06-03T20:31:49.529 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:31:49 smithi089 bash[78452]: audit 1685824308.2769578 mon.a (mon.0) 1657 : audit 1 from='client.? 172.21.15.89:0/1504953376' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-06-03T20:31:49.530 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:31:49 smithi089 bash[78452]: cluster 1685824308.2771885 mon.a (mon.0) 1658 : cluster 0 osdmap e296: 8 total, 8 up, 8 in 2023-06-03T20:31:49.530 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:31:49 smithi089 bash[78452]: cluster 1685824308.278158 osd.7 (osd.7) 240 : cluster 3 failed to encode map e296 with expected crc 2023-06-03T20:31:49.530 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:31:49 smithi089 bash[75068]: cluster 1685824308.2640579 osd.6 (osd.6) 232 : cluster 3 failed to encode map e296 with expected crc 2023-06-03T20:31:49.530 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:31:49 smithi089 bash[75068]: cluster 1685824308.2769046 osd.5 (osd.5) 244 : cluster 3 failed to encode map e296 with expected crc 2023-06-03T20:31:49.530 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:31:49 smithi089 bash[75068]: audit 1685824308.2769578 mon.a (mon.0) 1657 : audit 1 from='client.? 172.21.15.89:0/1504953376' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-06-03T20:31:49.531 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:31:49 smithi089 bash[75068]: cluster 1685824308.2771885 mon.a (mon.0) 1658 : cluster 0 osdmap e296: 8 total, 8 up, 8 in 2023-06-03T20:31:49.531 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:31:49 smithi089 bash[75068]: cluster 1685824308.278158 osd.7 (osd.7) 240 : cluster 3 failed to encode map e296 with expected crc 2023-06-03T20:31:49.562 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:31:49.585 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:31:49 smithi164 bash[59462]: cluster 1685824308.2640579 osd.6 (osd.6) 232 : cluster 3 failed to encode map e296 with expected crc 2023-06-03T20:31:49.585 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:31:49 smithi164 bash[59462]: cluster 1685824308.2769046 osd.5 (osd.5) 244 : cluster 3 failed to encode map e296 with expected crc 2023-06-03T20:31:49.585 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:31:49 smithi164 bash[59462]: audit 1685824308.2769578 mon.a (mon.0) 1657 : audit 1 from='client.? 172.21.15.89:0/1504953376' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-06-03T20:31:49.586 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:31:49 smithi164 bash[59462]: cluster 1685824308.2771885 mon.a (mon.0) 1658 : cluster 0 osdmap e296: 8 total, 8 up, 8 in 2023-06-03T20:31:49.586 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:31:49 smithi164 bash[59462]: cluster 1685824308.278158 osd.7 (osd.7) 240 : cluster 3 failed to encode map e296 with expected crc 2023-06-03T20:31:49.587 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:31:49.590 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:31:49.591 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:31:49.691 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:31:49.733 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:31:49.737 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:31:49.738 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:31:49.839 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:31:49.878 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:31:49.881 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:31:49.882 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:31:49.983 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:31:50.016 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:31:50.021 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:31:50.022 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:31:50.123 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:31:50.163 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:31:50.168 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:31:50.168 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:31:50.269 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:31:50.308 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:31:50.311 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:31:50.313 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:31:50.414 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:31:50.453 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:31:50.456 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:31:50.457 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:31:50.529 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:31:50 smithi089 bash[78452]: cluster 1685824309.2091627 mgr.y (mgr.34104) 1682 : cluster 0 pgmap v1740: 89 pgs: 89 active+clean; 398 MiB data, 6.3 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 643 op/s; 0 B/s, 7 objects/s recovering 2023-06-03T20:31:50.529 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:31:50 smithi089 bash[75068]: cluster 1685824309.2091627 mgr.y (mgr.34104) 1682 : cluster 0 pgmap v1740: 89 pgs: 89 active+clean; 398 MiB data, 6.3 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 643 op/s; 0 B/s, 7 objects/s recovering 2023-06-03T20:31:50.558 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:31:50.566 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:31:50 smithi164 bash[59462]: cluster 1685824309.2091627 mgr.y (mgr.34104) 1682 : cluster 0 pgmap v1740: 89 pgs: 89 active+clean; 398 MiB data, 6.3 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 643 op/s; 0 B/s, 7 objects/s recovering 2023-06-03T20:31:50.598 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:31:50.603 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:31:50.604 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:31:50.705 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:31:50.748 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:31:50.753 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:31:50.754 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:31:50.855 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:31:50.892 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:31:50.899 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:31:50.900 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:31:51.001 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:31:51.045 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:31:51.048 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:31:51.049 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:31:51.150 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:31:51.179 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:31:51.181 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:31:51.182 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:31:51.283 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:31:51.326 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:31:51.329 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:31:51.330 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:31:51.431 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:31:51.478 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:31:51.481 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:31:51.482 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:31:51.582 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:31:51.624 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:31:51.627 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:31:51.627 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:31:51.728 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:31:51.756 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:31:51.761 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:31:51.761 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:31:51.839 INFO:tasks.cephadm.ceph_manager.ceph:making progress, resetting timeout 2023-06-03T20:31:51.840 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph pg dump --format=json 2023-06-03T20:31:51.862 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:31:51.943 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:31:51.950 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:31:51.951 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:31:52.051 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:31:52.099 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:31:52.103 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:31:52.103 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:31:52.204 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:31:52.233 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:31:52.238 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:31:52.238 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:31:52.339 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:31:52.374 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:31:52.378 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:31:52.379 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:31:52.480 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:31:52.489 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:31:52 smithi089 bash[75068]: cluster 1685824311.2096064 mgr.y (mgr.34104) 1683 : cluster 0 pgmap v1741: 89 pgs: 89 active+clean; 398 MiB data, 6.3 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 573 op/s; 0 B/s, 7 objects/s recovering 2023-06-03T20:31:52.489 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:31:52 smithi089 bash[75068]: audit 1685824312.1661384 mon.a (mon.0) 1659 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:31:52.489 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:31:52 smithi089 bash[75068]: audit 1685824312.1734433 mon.c (mon.2) 678 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T20:31:52.490 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:31:52 smithi089 bash[78452]: cluster 1685824311.2096064 mgr.y (mgr.34104) 1683 : cluster 0 pgmap v1741: 89 pgs: 89 active+clean; 398 MiB data, 6.3 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 573 op/s; 0 B/s, 7 objects/s recovering 2023-06-03T20:31:52.490 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:31:52 smithi089 bash[78452]: audit 1685824312.1661384 mon.a (mon.0) 1659 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:31:52.490 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:31:52 smithi089 bash[78452]: audit 1685824312.1734433 mon.c (mon.2) 678 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T20:31:52.520 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:31:52.524 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:31:52.525 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:31:52.585 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:31:52 smithi164 bash[59462]: cluster 1685824311.2096064 mgr.y (mgr.34104) 1683 : cluster 0 pgmap v1741: 89 pgs: 89 active+clean; 398 MiB data, 6.3 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 573 op/s; 0 B/s, 7 objects/s recovering 2023-06-03T20:31:52.586 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:31:52 smithi164 bash[59462]: audit 1685824312.1661384 mon.a (mon.0) 1659 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:31:52.586 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:31:52 smithi164 bash[59462]: audit 1685824312.1734433 mon.c (mon.2) 678 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T20:31:52.626 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:31:52.653 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:31:52.655 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:31:52.656 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:31:52.757 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:31:52.782 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:31:52.790 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:31:52.791 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:31:52.892 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:31:52.926 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:31:52.930 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:31:52.930 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:31:53.031 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:31:53.060 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:31:53.062 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:31:53.063 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:31:53.164 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:31:53.196 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:31:53.200 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:31:53.201 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:31:53.302 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:31:53.343 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:31:53.346 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:31:53.346 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:31:53.447 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:31:53.482 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:31:53.488 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:31:53.489 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:31:53.590 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:31:53.620 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:31:53.625 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:31:53.625 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:31:53.679 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-06-03T20:31:53.726 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:31:53.778 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:31:53.785 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:31:53.786 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:31:53.853 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph osd set nodeep-scrub 2023-06-03T20:31:53.888 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:31:53.914 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:31:53.920 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:31:53.921 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:31:54.022 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:31:54.073 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:31:54.078 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:31:54.079 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:31:54.180 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:31:54.222 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:31:54.227 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:31:54.228 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:31:54.329 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:31:54.335 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:31:54 smithi164 bash[59462]: audit 1685824313.0820522 mon.a (mon.0) 1660 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:31:54.335 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:31:54 smithi164 bash[59462]: cluster 1685824313.2102304 mgr.y (mgr.34104) 1684 : cluster 0 pgmap v1742: 89 pgs: 89 active+clean; 296 MiB data, 5.9 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 602 op/s 2023-06-03T20:31:54.339 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:31:54 smithi089 bash[75068]: audit 1685824313.0820522 mon.a (mon.0) 1660 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:31:54.339 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:31:54 smithi089 bash[75068]: cluster 1685824313.2102304 mgr.y (mgr.34104) 1684 : cluster 0 pgmap v1742: 89 pgs: 89 active+clean; 296 MiB data, 5.9 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 602 op/s 2023-06-03T20:31:54.339 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:31:54 smithi089 bash[78452]: audit 1685824313.0820522 mon.a (mon.0) 1660 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:31:54.340 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:31:54 smithi089 bash[78452]: cluster 1685824313.2102304 mgr.y (mgr.34104) 1684 : cluster 0 pgmap v1742: 89 pgs: 89 active+clean; 296 MiB data, 5.9 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 602 op/s 2023-06-03T20:31:54.363 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:31:54.368 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:31:54.369 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:31:54.369 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/mon.a/config 2023-06-03T20:31:54.470 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:31:54.501 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:31:54.504 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:31:54.504 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:31:54.605 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:31:54.653 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:31:54.658 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:31:54.659 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:31:54.760 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:31:54.788 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:31:54.796 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:31:54.796 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:31:54.897 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:31:54.950 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:31:54.957 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:31:54.957 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:31:55.058 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:31:55.108 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:31:55.109 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:31:55.109 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:31:55.210 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:31:55.253 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:31:55.254 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:31:55.254 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:31:55.355 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:31:55.387 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:31:55.393 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:31:55.394 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:31:55.495 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:31:55.519 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:31:55.526 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:31:55.526 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:31:55.628 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:31:55.656 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:31:55.658 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:31:55.658 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:31:55.759 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:31:55.803 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:31:55.807 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:31:55.808 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:31:55.909 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:31:55.960 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:31:55.967 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:31:55.968 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:31:56.069 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:31:56.104 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:31:56.107 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:31:56.108 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:31:56.209 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:31:56.257 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:31:56.260 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:31:56.260 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:31:56.361 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:31:56.369 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 20:31:56 smithi089 bash[71163]: ::ffff:172.21.15.164 - - [03/Jun/2023:20:31:56] "GET /metrics HTTP/1.1" 200 229091 "" "Prometheus/2.33.4" 2023-06-03T20:31:56.370 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:31:56 smithi089 bash[78452]: cluster 1685824315.2108526 mgr.y (mgr.34104) 1685 : cluster 0 pgmap v1743: 89 pgs: 89 active+clean; 192 MiB data, 5.7 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 641 op/s 2023-06-03T20:31:56.370 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:31:56 smithi089 bash[75068]: cluster 1685824315.2108526 mgr.y (mgr.34104) 1685 : cluster 0 pgmap v1743: 89 pgs: 89 active+clean; 192 MiB data, 5.7 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 641 op/s 2023-06-03T20:31:56.403 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:31:56.408 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:31:56.409 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:31:56.432 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/mon.a/config 2023-06-03T20:31:56.510 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:31:56.552 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:31:56.555 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:31:56.556 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:31:56.557 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:31:56 smithi164 bash[59462]: cluster 1685824315.2108526 mgr.y (mgr.34104) 1685 : cluster 0 pgmap v1743: 89 pgs: 89 active+clean; 192 MiB data, 5.7 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 641 op/s 2023-06-03T20:31:56.657 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:31:56.700 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:31:56.705 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:31:56.705 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:31:56.806 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:31:56.857 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:31:56.861 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:31:56.862 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:31:56.963 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:31:56.993 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:31:56.999 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:31:57.000 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:31:57.101 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:31:57.137 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:31:57.143 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:31:57.143 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:31:57.244 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:31:57.275 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:31:57.277 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:31:57.278 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:31:57.379 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:31:57.423 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:31:57.428 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:31:57.429 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:31:57.531 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:31:57.556 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:31:57.562 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:31:57.563 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:31:57.664 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:31:57.709 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:31:57.711 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:31:57.712 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:31:57.813 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:31:57.839 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:31:57.849 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:31:57.849 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:31:57.869 INFO:tasks.radosbench.radosbench.0.smithi089.stdout:Removed 98013 objects 2023-06-03T20:31:57.869 INFO:tasks.radosbench.radosbench.0.smithi089.stdout:Clean up completed and total clean up time :160.976 2023-06-03T20:31:57.951 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:31:57.982 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:31:57.985 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:31:57.985 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:31:58.086 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:31:58.116 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:31:58.119 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:31:58.119 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:31:58.220 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:31:58.253 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:31:58.254 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:31:58.255 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:31:58.356 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:31:58.367 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:31:58 smithi089 bash[75068]: cluster 1685824317.2113671 mgr.y (mgr.34104) 1686 : cluster 0 pgmap v1744: 89 pgs: 89 active+clean; 164 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 647 op/s 2023-06-03T20:31:58.367 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:31:58 smithi089 bash[78452]: cluster 1685824317.2113671 mgr.y (mgr.34104) 1686 : cluster 0 pgmap v1744: 89 pgs: 89 active+clean; 164 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 647 op/s 2023-06-03T20:31:58.386 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:31:58.389 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:31:58.389 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:31:58.490 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:31:58.528 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:31:58.529 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:31:58 smithi164 bash[59462]: cluster 1685824317.2113671 mgr.y (mgr.34104) 1686 : cluster 0 pgmap v1744: 89 pgs: 89 active+clean; 164 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 647 op/s 2023-06-03T20:31:58.530 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:31:58.530 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:31:58.631 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:31:58.668 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:31:58.673 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:31:58.674 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:31:58.775 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:31:58.800 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:31:58.803 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:31:58.804 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:31:58.905 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:31:58.942 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:31:58.945 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:31:58.946 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:31:59.047 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:31:59.079 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:31:59.081 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:31:59.081 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:31:59.182 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:31:59.214 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:31:59.216 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:31:59.216 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:31:59.317 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:31:59.348 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:31:59.350 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:31:59.350 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:31:59.451 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:31:59.491 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:31:59.493 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:31:59.494 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:31:59.595 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:31:59.630 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:31:59.633 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:31:59.634 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:31:59.735 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:31:59.765 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:31:59.768 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:31:59.769 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:31:59.872 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:31:59.911 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:31:59.915 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:31:59.915 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:32:00.016 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:32:00.054 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:32:00.056 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:32:00.057 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:32:00.158 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:32:00.188 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:32:00.190 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:32:00.190 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:32:00.291 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:32:00.326 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:32:00.329 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:32:00.330 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:32:00.431 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:32:00.471 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:32:00.474 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:32:00.475 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:32:00.529 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:32:00 smithi089 bash[75068]: cluster 1685824319.2120008 mgr.y (mgr.34104) 1687 : cluster 0 pgmap v1745: 89 pgs: 89 active+clean; 8.1 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 711 op/s 2023-06-03T20:32:00.529 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:32:00 smithi089 bash[78452]: cluster 1685824319.2120008 mgr.y (mgr.34104) 1687 : cluster 0 pgmap v1745: 89 pgs: 89 active+clean; 8.1 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 711 op/s 2023-06-03T20:32:00.576 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:32:00.585 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:32:00 smithi164 bash[59462]: cluster 1685824319.2120008 mgr.y (mgr.34104) 1687 : cluster 0 pgmap v1745: 89 pgs: 89 active+clean; 8.1 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 711 op/s 2023-06-03T20:32:00.615 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:32:00.618 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:32:00.619 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:32:00.687 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/mon.a/config 2023-06-03T20:32:00.688 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/mon.a/config 2023-06-03T20:32:00.721 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:32:00.748 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:32:00.750 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:32:00.751 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:32:00.851 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:32:00.883 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:32:00.885 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:32:00.886 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:32:00.987 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:32:01.010 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:32:01.012 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:32:01.013 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:32:01.113 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:32:01.153 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:32:01.156 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:32:01.157 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:32:01.258 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:32:01.284 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:32:01.286 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:32:01.287 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:32:01.388 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:32:01.420 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:32:01.422 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:32:01.422 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:32:01.523 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:32:01.561 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:32:01.562 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:32:01.562 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:32:01.663 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:32:01.694 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:32:01.696 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:32:01.698 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:32:01.799 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:32:01.838 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:32:01.840 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:32:01.841 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:32:01.942 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:32:01.974 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:32:01.976 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:32:01.976 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:32:02.078 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:32:02.111 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:32:02.114 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:32:02.115 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:32:02.216 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:32:02.255 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:32:02.259 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:32:02.259 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:32:02.360 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:32:02.372 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:32:02 smithi164 bash[59462]: cluster 1685824321.2126942 mgr.y (mgr.34104) 1688 : cluster 0 pgmap v1746: 89 pgs: 89 active+clean; 8.1 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 519 op/s 2023-06-03T20:32:02.400 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:32:02.402 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:32:02.403 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:32:02.504 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:32:02.536 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:32:02.539 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:32:02.540 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:32:02.641 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:32:02.680 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:32:02.682 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:32:02.683 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:32:02.779 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:32:02 smithi089 bash[75068]: cluster 1685824321.2126942 mgr.y (mgr.34104) 1688 : cluster 0 pgmap v1746: 89 pgs: 89 active+clean; 8.1 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 519 op/s 2023-06-03T20:32:02.780 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:32:02 smithi089 bash[78452]: cluster 1685824321.2126942 mgr.y (mgr.34104) 1688 : cluster 0 pgmap v1746: 89 pgs: 89 active+clean; 8.1 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 519 op/s 2023-06-03T20:32:02.784 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:32:02.818 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:32:02.820 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:32:02.821 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:32:02.922 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:32:02.956 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:32:02.959 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:32:02.959 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:32:03.055 INFO:tasks.cephadm.ceph_manager.ceph:removing pool_name unique_pool_9 2023-06-03T20:32:03.056 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph osd pool rm unique_pool_9 unique_pool_9 --yes-i-really-really-mean-it 2023-06-03T20:32:03.060 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:32:03.124 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:32:03.126 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:32:03.127 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:32:03.228 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:32:03.263 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:32:03.264 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:32:03.265 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:32:03.365 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:32:03.396 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:32:03.397 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:32:03.398 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:32:03.499 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:32:03.537 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:32:03.539 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:32:03.540 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:32:03.641 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:32:03.677 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:32:03.680 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:32:03.680 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:32:03.781 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:32:03.815 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:32:03.818 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:32:03.818 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:32:03.919 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:32:03.953 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:32:03.955 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:32:03.956 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:32:04.057 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:32:04.100 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:32:04.103 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:32:04.104 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:32:04.205 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:32:04.241 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:32:04.245 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:32:04.245 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:32:04.279 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:32:04 smithi089 bash[78452]: cluster 1685824323.2132971 mgr.y (mgr.34104) 1689 : cluster 0 pgmap v1747: 89 pgs: 89 active+clean; 582 KiB data, 5.1 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 529 op/s 2023-06-03T20:32:04.279 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:32:04 smithi089 bash[75068]: cluster 1685824323.2132971 mgr.y (mgr.34104) 1689 : cluster 0 pgmap v1747: 89 pgs: 89 active+clean; 582 KiB data, 5.1 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 529 op/s 2023-06-03T20:32:04.335 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:32:04 smithi164 bash[59462]: cluster 1685824323.2132971 mgr.y (mgr.34104) 1689 : cluster 0 pgmap v1747: 89 pgs: 89 active+clean; 582 KiB data, 5.1 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 529 op/s 2023-06-03T20:32:04.347 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:32:04.383 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:32:04.385 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:32:04.386 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:32:04.487 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:32:04.515 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:32:04.518 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:32:04.518 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:32:04.619 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:32:04.659 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:32:04.661 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:32:04.662 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:32:04.763 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:32:04.796 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:32:04.799 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:32:04.799 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:32:04.900 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:32:04.935 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:32:04.937 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:32:04.937 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:32:05.038 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:32:05.073 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:32:05.075 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:32:05.075 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:32:05.176 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:32:05.217 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:32:05.220 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:32:05.221 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:32:05.321 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:32:05.356 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:32:05.359 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:32:05.359 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:32:05.460 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:32:05.496 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:32:05.500 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:32:05.501 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:32:05.602 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:32:05.629 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:32:05.631 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:32:05.631 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:32:05.732 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:32:05.771 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:32:05.773 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:32:05.774 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:32:05.875 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:32:05.904 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:32:05.905 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:32:05.905 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:32:06.006 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:32:06.032 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:32:06.035 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:32:06.036 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:32:06.136 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:32:06.172 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:32:06.175 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:32:06.176 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:32:06.277 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:32:06.303 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:32:06.307 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:32:06.308 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:32:06.409 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:32:06.437 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:32:06.439 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:32:06.439 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:32:06.529 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 20:32:06 smithi089 bash[71163]: ::ffff:172.21.15.164 - - [03/Jun/2023:20:32:06] "GET /metrics HTTP/1.1" 200 229091 "" "Prometheus/2.33.4" 2023-06-03T20:32:06.530 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:32:06 smithi089 bash[75068]: cluster 1685824325.2139401 mgr.y (mgr.34104) 1690 : cluster 0 pgmap v1748: 89 pgs: 89 active+clean; 582 KiB data, 5.1 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 394 op/s 2023-06-03T20:32:06.530 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:32:06 smithi089 bash[78452]: cluster 1685824325.2139401 mgr.y (mgr.34104) 1690 : cluster 0 pgmap v1748: 89 pgs: 89 active+clean; 582 KiB data, 5.1 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 394 op/s 2023-06-03T20:32:06.540 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:32:06.571 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:32:06.573 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:32:06.574 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:32:06.585 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:32:06 smithi164 bash[59462]: cluster 1685824325.2139401 mgr.y (mgr.34104) 1690 : cluster 0 pgmap v1748: 89 pgs: 89 active+clean; 582 KiB data, 5.1 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 394 op/s 2023-06-03T20:32:06.676 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:32:06.712 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:32:06.715 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:32:06.716 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:32:06.817 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:32:06.852 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:32:06.855 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:32:06.856 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:32:06.957 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:32:06.985 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:32:06.988 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:32:06.988 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:32:07.089 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:32:07.127 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:32:07.130 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:32:07.131 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:32:07.157 INFO:teuthology.orchestra.run.smithi089.stdout:ERROR: (22) Invalid argument 2023-06-03T20:32:07.157 INFO:teuthology.orchestra.run.smithi089.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-06-03T20:32:07.231 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:32:07.270 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:32:07.274 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:32:07.275 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:32:07.376 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:32:07.389 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:32:07 smithi164 bash[59462]: audit 1685824327.1608746 mon.c (mon.2) 679 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T20:32:07.417 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:32:07.420 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:32:07.421 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:32:07.522 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:32:07.558 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:32:07.561 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:32:07.562 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:32:07.664 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:32:07.673 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:32:07 smithi089 bash[75068]: audit 1685824327.1608746 mon.c (mon.2) 679 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T20:32:07.674 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:32:07 smithi089 bash[78452]: audit 1685824327.1608746 mon.c (mon.2) 679 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T20:32:07.694 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:32:07.697 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:32:07.698 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:32:07.800 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:32:07.833 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:32:07.836 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:32:07.837 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:32:07.840 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/mon.a/config 2023-06-03T20:32:07.938 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:32:07.966 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:32:07.967 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:32:07.968 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:32:08.069 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:32:08.098 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:32:08.100 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:32:08.101 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:32:08.202 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:32:08.242 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:32:08.245 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:32:08.245 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:32:08.346 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:32:08.385 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:32:08.387 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:32:08.388 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:32:08.489 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:32:08.516 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:32:08.518 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:32:08.518 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:32:08.585 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:32:08 smithi164 bash[59462]: cluster 1685824327.214634 mgr.y (mgr.34104) 1691 : cluster 0 pgmap v1749: 89 pgs: 89 active+clean; 582 KiB data, 5.1 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 254 op/s 2023-06-03T20:32:08.619 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:32:08.631 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:32:08 smithi089 bash[75068]: cluster 1685824327.214634 mgr.y (mgr.34104) 1691 : cluster 0 pgmap v1749: 89 pgs: 89 active+clean; 582 KiB data, 5.1 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 254 op/s 2023-06-03T20:32:08.631 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:32:08 smithi089 bash[78452]: cluster 1685824327.214634 mgr.y (mgr.34104) 1691 : cluster 0 pgmap v1749: 89 pgs: 89 active+clean; 582 KiB data, 5.1 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 254 op/s 2023-06-03T20:32:08.646 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:32:08.648 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:32:08.649 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:32:08.749 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:32:08.788 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:32:08.791 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:32:08.791 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:32:08.892 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:32:08.922 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:32:08.925 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:32:08.926 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:32:09.028 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:32:09.071 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:32:09.074 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:32:09.075 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:32:09.176 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:32:09.205 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:32:09.207 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:32:09.208 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:32:09.309 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:32:09.344 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:32:09.347 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:32:09.348 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:32:09.449 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:32:09.483 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:32:09.486 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:32:09.486 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:32:09.587 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:32:09.625 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:32:09.628 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:32:09.629 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:32:09.730 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:32:09.759 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:32:09.761 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:32:09.762 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:32:09.863 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:32:09.889 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:32:09.891 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:32:09.892 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:32:09.993 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:32:10.023 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:32:10.026 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:32:10.026 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:32:10.127 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:32:10.167 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:32:10.169 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:32:10.170 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:32:10.271 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:32:10.305 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:32:10.308 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:32:10.309 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:32:10.410 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:32:10.437 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:32:10.439 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:32:10.440 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:32:10.541 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:32:10.551 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:32:10 smithi089 bash[78452]: cluster 1685824329.2155886 mgr.y (mgr.34104) 1692 : cluster 0 pgmap v1750: 89 pgs: 89 active+clean; 582 KiB data, 5.1 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 217 op/s 2023-06-03T20:32:10.552 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:32:10 smithi089 bash[75068]: cluster 1685824329.2155886 mgr.y (mgr.34104) 1692 : cluster 0 pgmap v1750: 89 pgs: 89 active+clean; 582 KiB data, 5.1 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 217 op/s 2023-06-03T20:32:10.569 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:32:10.571 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:32:10.572 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:32:10.585 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:32:10 smithi164 bash[59462]: cluster 1685824329.2155886 mgr.y (mgr.34104) 1692 : cluster 0 pgmap v1750: 89 pgs: 89 active+clean; 582 KiB data, 5.1 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 217 op/s 2023-06-03T20:32:10.673 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:32:10.706 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:32:10.709 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:32:10.709 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:32:10.810 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:32:10.846 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:32:10.850 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:32:10.850 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:32:10.951 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:32:10.990 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:32:10.992 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:32:10.993 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:32:11.094 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:32:11.125 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:32:11.128 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:32:11.129 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:32:11.229 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:32:11.265 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph daemon osd.2 dump_historic_ops 2023-06-03T20:32:11.265 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:32:11.268 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:32:11.268 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:32:11.369 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:32:11.407 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:32:11.410 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:32:11.411 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:32:11.512 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:32:11.536 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:32:11.538 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:32:11.539 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:32:11.640 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:32:11.677 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:32:11.680 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:32:11.680 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:32:11.781 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:32:11.804 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:32:11.806 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:32:11.807 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:32:11.907 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:32:11.947 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:32:11.950 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:32:11.951 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:32:12.052 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:32:12.062 INFO:teuthology.orchestra.run.smithi089.stdout: 2023-06-03T20:32:12.063 INFO:teuthology.orchestra.run.smithi089.stderr:dumped all 2023-06-03T20:32:12.093 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:32:12.095 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:32:12.096 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:32:12.197 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:32:12.233 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:32:12.236 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:32:12.237 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:32:12.338 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:32:12.364 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:32:12.366 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:32:12.367 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:32:12.468 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:32:12.476 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:32:12 smithi089 bash[75068]: cluster 1685824331.2164297 mgr.y (mgr.34104) 1693 : cluster 0 pgmap v1751: 89 pgs: 89 active+clean; 582 KiB data, 5.1 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 10 op/s 2023-06-03T20:32:12.477 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:32:12 smithi089 bash[78452]: cluster 1685824331.2164297 mgr.y (mgr.34104) 1693 : cluster 0 pgmap v1751: 89 pgs: 89 active+clean; 582 KiB data, 5.1 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 10 op/s 2023-06-03T20:32:12.494 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:32:12.497 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:32:12.497 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:32:12.585 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:32:12 smithi164 bash[59462]: cluster 1685824331.2164297 mgr.y (mgr.34104) 1693 : cluster 0 pgmap v1751: 89 pgs: 89 active+clean; 582 KiB data, 5.1 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 10 op/s 2023-06-03T20:32:12.598 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:32:12.636 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:32:12.640 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:32:12.641 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:32:12.742 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:32:12.779 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:32:12.783 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:32:12.784 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:32:12.885 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:32:12.919 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:32:12.922 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:32:12.923 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:32:13.023 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:32:13.052 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:32:13.054 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:32:13.055 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:32:13.156 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:32:13.180 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:32:13.183 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:32:13.184 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:32:13.285 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:32:13.320 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:32:13.322 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:32:13.323 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:32:13.424 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:32:13.460 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:32:13.462 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:32:13.463 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:32:13.529 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:32:13 smithi089 bash[78452]: audit 1685824332.0370786 mgr.y (mgr.34104) 1694 : audit 0 from='client.36921 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-06-03T20:32:13.529 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:32:13 smithi089 bash[75068]: audit 1685824332.0370786 mgr.y (mgr.34104) 1694 : audit 0 from='client.36921 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-06-03T20:32:13.564 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:32:13.576 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:32:13 smithi164 bash[59462]: audit 1685824332.0370786 mgr.y (mgr.34104) 1694 : audit 0 from='client.36921 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-06-03T20:32:13.599 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:32:13.602 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:32:13.602 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:32:13.703 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:32:13.736 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:32:13.738 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:32:13.739 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:32:13.840 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:32:13.873 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:32:13.875 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:32:13.876 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:32:13.977 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:32:14.014 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:32:14.018 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:32:14.019 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:32:14.120 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:32:14.149 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:32:14.152 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:32:14.153 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:32:14.254 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:32:14.285 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:32:14.287 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:32:14.288 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:32:14.389 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:32:14.416 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:32:14.417 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:32:14.418 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:32:14.519 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:32:14.543 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:32:14.545 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:32:14.546 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:32:14.585 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:32:14 smithi164 bash[59462]: cluster 1685824333.2172909 mgr.y (mgr.34104) 1695 : cluster 0 pgmap v1752: 89 pgs: 89 active+clean; 582 KiB data, 5.1 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 10 op/s 2023-06-03T20:32:14.647 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:32:14.671 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:32:14.673 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:32:14.674 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:32:14.775 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:32:14.779 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:32:14 smithi089 bash[75068]: cluster 1685824333.2172909 mgr.y (mgr.34104) 1695 : cluster 0 pgmap v1752: 89 pgs: 89 active+clean; 582 KiB data, 5.1 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 10 op/s 2023-06-03T20:32:14.779 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:32:14 smithi089 bash[78452]: cluster 1685824333.2172909 mgr.y (mgr.34104) 1695 : cluster 0 pgmap v1752: 89 pgs: 89 active+clean; 582 KiB data, 5.1 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 10 op/s 2023-06-03T20:32:14.818 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:32:14.821 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:32:14.821 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:32:14.922 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:32:14.946 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:32:14.948 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:32:14.948 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:32:15.050 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:32:15.090 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:32:15.093 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:32:15.093 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:32:15.194 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:32:15.219 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:32:15.223 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:32:15.223 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:32:15.324 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:32:15.335 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:32:15 smithi164 bash[59462]: audit 1685824334.2898095 mon.c (mon.2) 680 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-03T20:32:15.349 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:32:15.352 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:32:15.353 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:32:15.454 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:32:15.491 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:32:15.494 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:32:15.494 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:32:15.540 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:32:15 smithi089 bash[78452]: audit 1685824334.2898095 mon.c (mon.2) 680 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-03T20:32:15.540 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:32:15 smithi089 bash[75068]: audit 1685824334.2898095 mon.c (mon.2) 680 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-03T20:32:15.595 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:32:15.627 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:32:15.629 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:32:15.630 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:32:15.730 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:32:15.769 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:32:15.773 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:32:15.774 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:32:15.875 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:32:15.906 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:32:15.908 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:32:15.909 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:32:16.010 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:32:16.053 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:32:16.055 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:32:16.056 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:32:16.129 INFO:teuthology.orchestra.run.smithi089.stdout:{"pg_ready":true,"pg_map":{"version":1751,"stamp":"2023-06-03T20:32:11.215902+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":595854,"num_objects":24,"num_object_clones":0,"num_object_copies":72,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":24,"num_whiteouts":0,"num_read":309,"num_read_kb":513,"num_write":196289,"num_write_kb":6275279,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5095,"num_bytes_recovered":277354840,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":141071,"ondisk_log_size":141071,"up":267,"acting":267,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":267,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":5359288,"kb_used_data":26232,"kb_used_omap":30,"kb_used_meta":5332961,"kb_avail":744601928,"statfs":{"total":767960285184,"available":762472374272,"internally_reserved":0,"allocated":26861568,"data_stored":19085985,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":31502,"internal_metadata":5460952306},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":5,"apply_latency_ms":5,"commit_latency_ns":5000000,"apply_latency_ns":5000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-7929856,"num_objects":-121,"num_object_clones":0,"num_object_copies":-363,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":-121,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":121,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.004334"},"pg_stats":[{"pgid":"3.1a","version":"60'1","reported_seq":223,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-06-03T20:31:40.290145+0000","last_change":"2023-06-03T20:23:28.242261+0000","last_active":"2023-06-03T20:31:40.290145+0000","last_peered":"2023-06-03T20:31:40.290145+0000","last_clean":"2023-06-03T20:31:40.290145+0000","last_became_active":"2023-06-03T20:23:28.240045+0000","last_became_peered":"2023-06-03T20:23:28.240045+0000","last_unstale":"2023-06-03T20:31:40.290145+0000","last_undegraded":"2023-06-03T20:31:40.290145+0000","last_fullsized":"2023-06-03T20:31:40.290145+0000","mapping_epoch":251,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":252,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T05:54:33.215385+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":84,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-06-03T20:31:40.413469+0000","last_change":"2023-06-03T20:23:28.283021+0000","last_active":"2023-06-03T20:31:40.413469+0000","last_peered":"2023-06-03T20:31:40.413469+0000","last_clean":"2023-06-03T20:31:40.413469+0000","last_became_active":"2023-06-03T20:23:28.282428+0000","last_became_peered":"2023-06-03T20:23:28.282428+0000","last_unstale":"2023-06-03T20:31:40.413469+0000","last_undegraded":"2023-06-03T20:31:40.413469+0000","last_fullsized":"2023-06-03T20:31:40.413469+0000","mapping_epoch":251,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":252,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T05:01:33.115332+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":84,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-06-03T20:31:40.412595+0000","last_change":"2023-06-03T20:23:28.253821+0000","last_active":"2023-06-03T20:31:40.412595+0000","last_peered":"2023-06-03T20:31:40.412595+0000","last_clean":"2023-06-03T20:31:40.412595+0000","last_became_active":"2023-06-03T20:23:28.253448+0000","last_became_peered":"2023-06-03T20:23:28.253448+0000","last_unstale":"2023-06-03T20:31:40.412595+0000","last_undegraded":"2023-06-03T20:31:40.412595+0000","last_fullsized":"2023-06-03T20:31:40.412595+0000","mapping_epoch":251,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":252,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T23:40:02.264778+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":267,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-06-03T20:31:40.415674+0000","last_change":"2023-06-03T19:56:54.507653+0000","last_active":"2023-06-03T20:31:40.415674+0000","last_peered":"2023-06-03T20:31:40.415674+0000","last_clean":"2023-06-03T20:31:40.415674+0000","last_became_active":"2023-06-03T19:56:54.507350+0000","last_became_peered":"2023-06-03T19:56:54.507350+0000","last_unstale":"2023-06-03T20:31:40.415674+0000","last_undegraded":"2023-06-03T20:31:40.415674+0000","last_fullsized":"2023-06-03T20:31:40.415674+0000","mapping_epoch":119,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T22:33:20.065844+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":307,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-06-03T20:31:40.280884+0000","last_change":"2023-06-03T20:21:01.256894+0000","last_active":"2023-06-03T20:31:40.280884+0000","last_peered":"2023-06-03T20:31:40.280884+0000","last_clean":"2023-06-03T20:31:40.280884+0000","last_became_active":"2023-06-03T20:21:01.255903+0000","last_became_peered":"2023-06-03T20:21:01.255903+0000","last_unstale":"2023-06-03T20:31:40.280884+0000","last_undegraded":"2023-06-03T20:31:40.280884+0000","last_fullsized":"2023-06-03T20:31:40.280884+0000","mapping_epoch":235,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":236,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T20:35:27.810860+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":60,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-06-03T20:31:40.286748+0000","last_change":"2023-06-03T20:28:24.269355+0000","last_active":"2023-06-03T20:31:40.286748+0000","last_peered":"2023-06-03T20:31:40.286748+0000","last_clean":"2023-06-03T20:31:40.286748+0000","last_became_active":"2023-06-03T20:28:24.268859+0000","last_became_peered":"2023-06-03T20:28:24.268859+0000","last_unstale":"2023-06-03T20:31:40.286748+0000","last_undegraded":"2023-06-03T20:31:40.286748+0000","last_fullsized":"2023-06-03T20:31:40.286748+0000","mapping_epoch":284,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":285,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,2],"acting":[6,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":262,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-06-03T20:31:40.432783+0000","last_change":"2023-06-03T19:56:54.509490+0000","last_active":"2023-06-03T20:31:40.432783+0000","last_peered":"2023-06-03T20:31:40.432783+0000","last_clean":"2023-06-03T20:31:40.432783+0000","last_became_active":"2023-06-03T19:56:54.509145+0000","last_became_peered":"2023-06-03T19:56:54.509145+0000","last_unstale":"2023-06-03T20:31:40.432783+0000","last_undegraded":"2023-06-03T20:31:40.432783+0000","last_fullsized":"2023-06-03T20:31:40.432783+0000","mapping_epoch":119,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T07:12:03.964110+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":328,"reported_epoch":296,"state":"active+clean","last_fresh":"2023-06-03T20:31:48.283253+0000","last_change":"2023-06-03T20:28:51.233277+0000","last_active":"2023-06-03T20:31:48.283253+0000","last_peered":"2023-06-03T20:31:48.283253+0000","last_clean":"2023-06-03T20:31:48.283253+0000","last_became_active":"2023-06-03T20:28:51.232919+0000","last_became_peered":"2023-06-03T20:28:51.232919+0000","last_unstale":"2023-06-03T20:31:48.283253+0000","last_undegraded":"2023-06-03T20:31:48.283253+0000","last_fullsized":"2023-06-03T20:31:48.283253+0000","mapping_epoch":286,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":287,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T02:06:02.736415+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":228,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-06-03T20:31:40.282397+0000","last_change":"2023-06-03T20:21:01.258441+0000","last_active":"2023-06-03T20:31:40.282397+0000","last_peered":"2023-06-03T20:31:40.282397+0000","last_clean":"2023-06-03T20:31:40.282397+0000","last_became_active":"2023-06-03T20:21:01.257779+0000","last_became_peered":"2023-06-03T20:21:01.257779+0000","last_unstale":"2023-06-03T20:31:40.282397+0000","last_undegraded":"2023-06-03T20:31:40.282397+0000","last_fullsized":"2023-06-03T20:31:40.282397+0000","mapping_epoch":235,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":236,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,1],"acting":[7,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":273,"reported_epoch":296,"state":"active+clean","last_fresh":"2023-06-03T20:31:48.284418+0000","last_change":"2023-06-03T20:24:02.267331+0000","last_active":"2023-06-03T20:31:48.284418+0000","last_peered":"2023-06-03T20:31:48.284418+0000","last_clean":"2023-06-03T20:31:48.284418+0000","last_became_active":"2023-06-03T20:24:02.266997+0000","last_became_peered":"2023-06-03T20:24:02.266997+0000","last_unstale":"2023-06-03T20:31:48.284418+0000","last_undegraded":"2023-06-03T20:31:48.284418+0000","last_fullsized":"2023-06-03T20:31:48.284418+0000","mapping_epoch":254,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":255,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T22:09:18.993656+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"60'3","reported_seq":293,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-06-03T20:31:40.282881+0000","last_change":"2023-06-03T20:23:27.261246+0000","last_active":"2023-06-03T20:31:40.282881+0000","last_peered":"2023-06-03T20:31:40.282881+0000","last_clean":"2023-06-03T20:31:40.282881+0000","last_became_active":"2023-06-03T20:23:27.259201+0000","last_became_peered":"2023-06-03T20:23:27.259201+0000","last_unstale":"2023-06-03T20:31:40.282881+0000","last_undegraded":"2023-06-03T20:31:40.282881+0000","last_fullsized":"2023-06-03T20:31:40.282881+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":84,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-06-03T20:31:40.413464+0000","last_change":"2023-06-03T20:28:51.235184+0000","last_active":"2023-06-03T20:31:40.413464+0000","last_peered":"2023-06-03T20:31:40.413464+0000","last_clean":"2023-06-03T20:31:40.413464+0000","last_became_active":"2023-06-03T20:28:51.234830+0000","last_became_peered":"2023-06-03T20:28:51.234830+0000","last_unstale":"2023-06-03T20:31:40.413464+0000","last_undegraded":"2023-06-03T20:31:40.413464+0000","last_fullsized":"2023-06-03T20:31:40.413464+0000","mapping_epoch":286,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":287,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T04:40:02.479523+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"77'23","reported_seq":230,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-06-03T20:31:40.290416+0000","last_change":"2023-06-03T20:24:02.280274+0000","last_active":"2023-06-03T20:31:40.290416+0000","last_peered":"2023-06-03T20:31:40.290416+0000","last_clean":"2023-06-03T20:31:40.290416+0000","last_became_active":"2023-06-03T20:24:02.264914+0000","last_became_peered":"2023-06-03T20:24:02.264914+0000","last_unstale":"2023-06-03T20:31:40.290416+0000","last_undegraded":"2023-06-03T20:31:40.290416+0000","last_fullsized":"2023-06-03T20:31:40.290416+0000","mapping_epoch":254,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":255,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":23,"log_dups_size":0,"ondisk_log_size":23,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T07:00:35.888220+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4716,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":23,"num_write_kb":20,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4716,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":206,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-06-03T20:31:40.289848+0000","last_change":"2023-06-03T20:24:02.254850+0000","last_active":"2023-06-03T20:31:40.289848+0000","last_peered":"2023-06-03T20:31:40.289848+0000","last_clean":"2023-06-03T20:31:40.289848+0000","last_became_active":"2023-06-03T20:24:02.254409+0000","last_became_peered":"2023-06-03T20:24:02.254409+0000","last_unstale":"2023-06-03T20:31:40.289848+0000","last_undegraded":"2023-06-03T20:31:40.289848+0000","last_fullsized":"2023-06-03T20:31:40.289848+0000","mapping_epoch":254,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":255,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T19:49:53.393121+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"60'2","reported_seq":295,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-06-03T20:31:40.282477+0000","last_change":"2023-06-03T20:23:27.258749+0000","last_active":"2023-06-03T20:31:40.282477+0000","last_peered":"2023-06-03T20:31:40.282477+0000","last_clean":"2023-06-03T20:31:40.282477+0000","last_became_active":"2023-06-03T20:23:27.256899+0000","last_became_peered":"2023-06-03T20:23:27.256899+0000","last_unstale":"2023-06-03T20:31:40.282477+0000","last_undegraded":"2023-06-03T20:31:40.282477+0000","last_fullsized":"2023-06-03T20:31:40.282477+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":292,"reported_epoch":296,"state":"active+clean","last_fresh":"2023-06-03T20:31:48.284475+0000","last_change":"2023-06-03T20:21:01.256612+0000","last_active":"2023-06-03T20:31:48.284475+0000","last_peered":"2023-06-03T20:31:48.284475+0000","last_clean":"2023-06-03T20:31:48.284475+0000","last_became_active":"2023-06-03T20:21:01.256243+0000","last_became_peered":"2023-06-03T20:21:01.256243+0000","last_unstale":"2023-06-03T20:31:48.284475+0000","last_undegraded":"2023-06-03T20:31:48.284475+0000","last_fullsized":"2023-06-03T20:31:48.284475+0000","mapping_epoch":235,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":236,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T05:24:47.054342+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":263,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-06-03T20:31:40.414867+0000","last_change":"2023-06-03T19:56:54.507479+0000","last_active":"2023-06-03T20:31:40.414867+0000","last_peered":"2023-06-03T20:31:40.414867+0000","last_clean":"2023-06-03T20:31:40.414867+0000","last_became_active":"2023-06-03T19:56:54.507149+0000","last_became_peered":"2023-06-03T19:56:54.507149+0000","last_unstale":"2023-06-03T20:31:40.414867+0000","last_undegraded":"2023-06-03T20:31:40.414867+0000","last_fullsized":"2023-06-03T20:31:40.414867+0000","mapping_epoch":119,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T20:22:36.676317+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":162,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-06-03T20:31:40.283040+0000","last_change":"2023-06-03T20:17:22.832844+0000","last_active":"2023-06-03T20:31:40.283040+0000","last_peered":"2023-06-03T20:31:40.283040+0000","last_clean":"2023-06-03T20:31:40.283040+0000","last_became_active":"2023-06-03T20:17:22.832473+0000","last_became_peered":"2023-06-03T20:17:22.832473+0000","last_unstale":"2023-06-03T20:31:40.283040+0000","last_undegraded":"2023-06-03T20:31:40.283040+0000","last_fullsized":"2023-06-03T20:31:40.283040+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,3],"acting":[7,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":295,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-06-03T20:31:40.281874+0000","last_change":"2023-06-03T20:24:02.267531+0000","last_active":"2023-06-03T20:31:40.281874+0000","last_peered":"2023-06-03T20:31:40.281874+0000","last_clean":"2023-06-03T20:31:40.281874+0000","last_became_active":"2023-06-03T20:24:02.267159+0000","last_became_peered":"2023-06-03T20:24:02.267159+0000","last_unstale":"2023-06-03T20:31:40.281874+0000","last_undegraded":"2023-06-03T20:31:40.281874+0000","last_fullsized":"2023-06-03T20:31:40.281874+0000","mapping_epoch":254,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":255,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":263,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-06-03T20:31:40.426609+0000","last_change":"2023-06-03T19:56:54.510289+0000","last_active":"2023-06-03T20:31:40.426609+0000","last_peered":"2023-06-03T20:31:40.426609+0000","last_clean":"2023-06-03T20:31:40.426609+0000","last_became_active":"2023-06-03T19:56:54.509905+0000","last_became_peered":"2023-06-03T19:56:54.509905+0000","last_unstale":"2023-06-03T20:31:40.426609+0000","last_undegraded":"2023-06-03T20:31:40.426609+0000","last_fullsized":"2023-06-03T20:31:40.426609+0000","mapping_epoch":119,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T02:54:26.792401+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"60'8","reported_seq":61,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-06-03T20:31:40.286428+0000","last_change":"2023-06-03T20:26:59.682623+0000","last_active":"2023-06-03T20:31:40.286428+0000","last_peered":"2023-06-03T20:31:40.286428+0000","last_clean":"2023-06-03T20:31:40.286428+0000","last_became_active":"2023-06-03T20:24:02.260105+0000","last_became_peered":"2023-06-03T20:24:02.260105+0000","last_unstale":"2023-06-03T20:31:40.286428+0000","last_undegraded":"2023-06-03T20:31:40.286428+0000","last_fullsized":"2023-06-03T20:31:40.286428+0000","mapping_epoch":254,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":273,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":300,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-06-03T20:31:40.281716+0000","last_change":"2023-06-03T20:28:24.281308+0000","last_active":"2023-06-03T20:31:40.281716+0000","last_peered":"2023-06-03T20:31:40.281716+0000","last_clean":"2023-06-03T20:31:40.281716+0000","last_became_active":"2023-06-03T20:28:24.280981+0000","last_became_peered":"2023-06-03T20:28:24.280981+0000","last_unstale":"2023-06-03T20:31:40.281716+0000","last_undegraded":"2023-06-03T20:31:40.281716+0000","last_fullsized":"2023-06-03T20:31:40.281716+0000","mapping_epoch":284,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":285,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T02:44:14.890147+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"14.2","version":"296'12030","reported_seq":12056,"reported_epoch":296,"state":"active+clean","last_fresh":"2023-06-03T20:31:57.774796+0000","last_change":"2023-06-03T20:27:21.371612+0000","last_active":"2023-06-03T20:31:57.774796+0000","last_peered":"2023-06-03T20:31:57.774796+0000","last_clean":"2023-06-03T20:31:57.774796+0000","last_became_active":"2023-06-03T20:27:21.371344+0000","last_became_peered":"2023-06-03T20:27:21.371344+0000","last_unstale":"2023-06-03T20:31:57.774796+0000","last_undegraded":"2023-06-03T20:31:57.774796+0000","last_fullsized":"2023-06-03T20:31:57.774796+0000","mapping_epoch":278,"log_start":"281'2300","ondisk_log_start":"281'2300","created":278,"last_epoch_clean":279,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T20:27:20.339366+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T20:27:20.339366+0000","last_clean_scrub_stamp":"2023-06-03T20:27:20.339366+0000","objects_scrubbed":0,"log_size":9730,"log_dups_size":0,"ondisk_log_size":9730,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12030,"num_write_kb":384960,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,3],"acting":[6,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.f","version":"60'1","reported_seq":294,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-06-03T20:31:40.282741+0000","last_change":"2023-06-03T20:23:27.263667+0000","last_active":"2023-06-03T20:31:40.282741+0000","last_peered":"2023-06-03T20:31:40.282741+0000","last_clean":"2023-06-03T20:31:40.282741+0000","last_became_active":"2023-06-03T20:23:27.263110+0000","last_became_peered":"2023-06-03T20:23:27.263110+0000","last_unstale":"2023-06-03T20:31:40.282741+0000","last_undegraded":"2023-06-03T20:31:40.282741+0000","last_fullsized":"2023-06-03T20:31:40.282741+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":255,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-06-03T20:31:40.282077+0000","last_change":"2023-06-03T20:28:51.231222+0000","last_active":"2023-06-03T20:31:40.282077+0000","last_peered":"2023-06-03T20:31:40.282077+0000","last_clean":"2023-06-03T20:31:40.282077+0000","last_became_active":"2023-06-03T20:28:51.230848+0000","last_became_peered":"2023-06-03T20:28:51.230848+0000","last_unstale":"2023-06-03T20:31:40.282077+0000","last_undegraded":"2023-06-03T20:31:40.282077+0000","last_fullsized":"2023-06-03T20:31:40.282077+0000","mapping_epoch":286,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":287,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"14.3","version":"296'12340","reported_seq":12369,"reported_epoch":296,"state":"active+clean","last_fresh":"2023-06-03T20:31:57.819434+0000","last_change":"2023-06-03T20:27:21.374055+0000","last_active":"2023-06-03T20:31:57.819434+0000","last_peered":"2023-06-03T20:31:57.819434+0000","last_clean":"2023-06-03T20:31:57.819434+0000","last_became_active":"2023-06-03T20:27:21.373728+0000","last_became_peered":"2023-06-03T20:27:21.373728+0000","last_unstale":"2023-06-03T20:31:57.819434+0000","last_undegraded":"2023-06-03T20:31:57.819434+0000","last_fullsized":"2023-06-03T20:31:57.819434+0000","mapping_epoch":278,"log_start":"285'3500","ondisk_log_start":"285'3500","created":278,"last_epoch_clean":279,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T20:27:20.339366+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T20:27:20.339366+0000","last_clean_scrub_stamp":"2023-06-03T20:27:20.339366+0000","objects_scrubbed":0,"log_size":8840,"log_dups_size":0,"ondisk_log_size":8840,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T05:15:33.026018+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12340,"num_write_kb":394880,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.e","version":"60'2","reported_seq":323,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-06-03T20:31:40.282421+0000","last_change":"2023-06-03T20:23:28.606751+0000","last_active":"2023-06-03T20:31:40.282421+0000","last_peered":"2023-06-03T20:31:40.282421+0000","last_clean":"2023-06-03T20:31:40.282421+0000","last_became_active":"2023-06-03T20:23:28.605968+0000","last_became_peered":"2023-06-03T20:23:28.605968+0000","last_unstale":"2023-06-03T20:31:40.282421+0000","last_undegraded":"2023-06-03T20:31:40.282421+0000","last_fullsized":"2023-06-03T20:31:40.282421+0000","mapping_epoch":251,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":252,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":235,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-06-03T20:31:40.290142+0000","last_change":"2023-06-03T20:23:28.249694+0000","last_active":"2023-06-03T20:31:40.290142+0000","last_peered":"2023-06-03T20:31:40.290142+0000","last_clean":"2023-06-03T20:31:40.290142+0000","last_became_active":"2023-06-03T20:23:28.249388+0000","last_became_peered":"2023-06-03T20:23:28.249388+0000","last_unstale":"2023-06-03T20:31:40.290142+0000","last_undegraded":"2023-06-03T20:31:40.290142+0000","last_fullsized":"2023-06-03T20:31:40.290142+0000","mapping_epoch":251,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":252,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T05:44:01.835586+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"14.0","version":"296'12084","reported_seq":12113,"reported_epoch":296,"state":"active+clean","last_fresh":"2023-06-03T20:31:57.830498+0000","last_change":"2023-06-03T20:27:21.378346+0000","last_active":"2023-06-03T20:31:57.830498+0000","last_peered":"2023-06-03T20:31:57.830498+0000","last_clean":"2023-06-03T20:31:57.830498+0000","last_became_active":"2023-06-03T20:27:21.377969+0000","last_became_peered":"2023-06-03T20:27:21.377969+0000","last_unstale":"2023-06-03T20:31:57.830498+0000","last_undegraded":"2023-06-03T20:31:57.830498+0000","last_fullsized":"2023-06-03T20:31:57.830498+0000","mapping_epoch":278,"log_start":"281'2000","ondisk_log_start":"281'2000","created":278,"last_epoch_clean":279,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T20:27:20.339366+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T20:27:20.339366+0000","last_clean_scrub_stamp":"2023-06-03T20:27:20.339366+0000","objects_scrubbed":0,"log_size":10084,"log_dups_size":0,"ondisk_log_size":10084,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T01:32:49.543336+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12084,"num_write_kb":386688,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,5],"acting":[2,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.d","version":"60'1","reported_seq":272,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-06-03T20:31:40.282678+0000","last_change":"2023-06-03T20:26:59.686037+0000","last_active":"2023-06-03T20:31:40.282678+0000","last_peered":"2023-06-03T20:31:40.282678+0000","last_clean":"2023-06-03T20:31:40.282678+0000","last_became_active":"2023-06-03T20:24:02.268002+0000","last_became_peered":"2023-06-03T20:24:02.268002+0000","last_unstale":"2023-06-03T20:31:40.282678+0000","last_undegraded":"2023-06-03T20:31:40.282678+0000","last_fullsized":"2023-06-03T20:31:40.282678+0000","mapping_epoch":254,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":273,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":140,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":19,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-06-03T20:31:40.286468+0000","last_change":"2023-06-03T20:28:51.230833+0000","last_active":"2023-06-03T20:31:40.286468+0000","last_peered":"2023-06-03T20:31:40.286468+0000","last_clean":"2023-06-03T20:31:40.286468+0000","last_became_active":"2023-06-03T20:28:51.230489+0000","last_became_peered":"2023-06-03T20:28:51.230489+0000","last_unstale":"2023-06-03T20:31:40.286468+0000","last_undegraded":"2023-06-03T20:31:40.286468+0000","last_fullsized":"2023-06-03T20:31:40.286468+0000","mapping_epoch":286,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":287,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"14.1","version":"296'12742","reported_seq":12771,"reported_epoch":296,"state":"active+clean","last_fresh":"2023-06-03T20:31:57.828356+0000","last_change":"2023-06-03T20:27:21.372641+0000","last_active":"2023-06-03T20:31:57.828356+0000","last_peered":"2023-06-03T20:31:57.828356+0000","last_clean":"2023-06-03T20:31:57.828356+0000","last_became_active":"2023-06-03T20:27:21.372158+0000","last_became_peered":"2023-06-03T20:27:21.372158+0000","last_unstale":"2023-06-03T20:31:57.828356+0000","last_undegraded":"2023-06-03T20:31:57.828356+0000","last_fullsized":"2023-06-03T20:31:57.828356+0000","mapping_epoch":278,"log_start":"285'4600","ondisk_log_start":"285'4600","created":278,"last_epoch_clean":279,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T20:27:20.339366+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T20:27:20.339366+0000","last_clean_scrub_stamp":"2023-06-03T20:27:20.339366+0000","objects_scrubbed":0,"log_size":8142,"log_dups_size":0,"ondisk_log_size":8142,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T07:52:41.694374+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12742,"num_write_kb":407744,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":289,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-06-03T20:31:40.281669+0000","last_change":"2023-06-03T20:24:02.261252+0000","last_active":"2023-06-03T20:31:40.281669+0000","last_peered":"2023-06-03T20:31:40.281669+0000","last_clean":"2023-06-03T20:31:40.281669+0000","last_became_active":"2023-06-03T20:24:02.260824+0000","last_became_peered":"2023-06-03T20:24:02.260824+0000","last_unstale":"2023-06-03T20:31:40.281669+0000","last_undegraded":"2023-06-03T20:31:40.281669+0000","last_fullsized":"2023-06-03T20:31:40.281669+0000","mapping_epoch":254,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":255,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T20:00:53.584753+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":327,"reported_epoch":296,"state":"active+clean","last_fresh":"2023-06-03T20:31:48.298129+0000","last_change":"2023-06-03T20:23:28.283360+0000","last_active":"2023-06-03T20:31:48.298129+0000","last_peered":"2023-06-03T20:31:48.298129+0000","last_clean":"2023-06-03T20:31:48.298129+0000","last_became_active":"2023-06-03T20:23:28.283035+0000","last_became_peered":"2023-06-03T20:23:28.283035+0000","last_unstale":"2023-06-03T20:31:48.298129+0000","last_undegraded":"2023-06-03T20:31:48.298129+0000","last_fullsized":"2023-06-03T20:31:48.298129+0000","mapping_epoch":251,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":252,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T00:22:59.446515+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"14.6","version":"296'12280","reported_seq":12309,"reported_epoch":296,"state":"active+clean","last_fresh":"2023-06-03T20:31:57.853233+0000","last_change":"2023-06-03T20:27:21.375836+0000","last_active":"2023-06-03T20:31:57.853233+0000","last_peered":"2023-06-03T20:31:57.853233+0000","last_clean":"2023-06-03T20:31:57.853233+0000","last_became_active":"2023-06-03T20:27:21.375522+0000","last_became_peered":"2023-06-03T20:27:21.375522+0000","last_unstale":"2023-06-03T20:31:57.853233+0000","last_undegraded":"2023-06-03T20:31:57.853233+0000","last_fullsized":"2023-06-03T20:31:57.853233+0000","mapping_epoch":278,"log_start":"283'3400","ondisk_log_start":"283'3400","created":278,"last_epoch_clean":279,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T20:27:20.339366+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T20:27:20.339366+0000","last_clean_scrub_stamp":"2023-06-03T20:27:20.339366+0000","objects_scrubbed":0,"log_size":8880,"log_dups_size":0,"ondisk_log_size":8880,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T21:02:49.138563+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12280,"num_write_kb":392960,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,1],"acting":[0,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.b","version":"60'2","reported_seq":290,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-06-03T20:31:40.281208+0000","last_change":"2023-06-03T20:23:27.254179+0000","last_active":"2023-06-03T20:31:40.281208+0000","last_peered":"2023-06-03T20:31:40.281208+0000","last_clean":"2023-06-03T20:31:40.281208+0000","last_became_active":"2023-06-03T20:23:27.253791+0000","last_became_peered":"2023-06-03T20:23:27.253791+0000","last_unstale":"2023-06-03T20:31:40.281208+0000","last_undegraded":"2023-06-03T20:31:40.281208+0000","last_fullsized":"2023-06-03T20:31:40.281208+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T00:31:27.336760+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":211,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-06-03T20:31:40.290057+0000","last_change":"2023-06-03T20:24:02.255551+0000","last_active":"2023-06-03T20:31:40.290057+0000","last_peered":"2023-06-03T20:31:40.290057+0000","last_clean":"2023-06-03T20:31:40.290057+0000","last_became_active":"2023-06-03T20:24:02.255209+0000","last_became_peered":"2023-06-03T20:24:02.255209+0000","last_unstale":"2023-06-03T20:31:40.290057+0000","last_undegraded":"2023-06-03T20:31:40.290057+0000","last_fullsized":"2023-06-03T20:31:40.290057+0000","mapping_epoch":254,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":255,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T20:05:02.741217+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"14.7","version":"296'12326","reported_seq":12355,"reported_epoch":296,"state":"active+clean","last_fresh":"2023-06-03T20:31:57.816191+0000","last_change":"2023-06-03T20:27:21.364098+0000","last_active":"2023-06-03T20:31:57.816191+0000","last_peered":"2023-06-03T20:31:57.816191+0000","last_clean":"2023-06-03T20:31:57.816191+0000","last_became_active":"2023-06-03T20:27:21.363548+0000","last_became_peered":"2023-06-03T20:27:21.363548+0000","last_unstale":"2023-06-03T20:31:57.816191+0000","last_undegraded":"2023-06-03T20:31:57.816191+0000","last_fullsized":"2023-06-03T20:31:57.816191+0000","mapping_epoch":278,"log_start":"285'4600","ondisk_log_start":"285'4600","created":278,"last_epoch_clean":279,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T20:27:20.339366+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T20:27:20.339366+0000","last_clean_scrub_stamp":"2023-06-03T20:27:20.339366+0000","objects_scrubbed":0,"log_size":7726,"log_dups_size":0,"ondisk_log_size":7726,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T06:05:40.316567+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12326,"num_write_kb":394432,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,0],"acting":[4,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":51,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-06-03T20:31:40.286795+0000","last_change":"2023-06-03T20:24:02.254172+0000","last_active":"2023-06-03T20:31:40.286795+0000","last_peered":"2023-06-03T20:31:40.286795+0000","last_clean":"2023-06-03T20:31:40.286795+0000","last_became_active":"2023-06-03T20:24:02.253797+0000","last_became_peered":"2023-06-03T20:24:02.253797+0000","last_unstale":"2023-06-03T20:31:40.286795+0000","last_undegraded":"2023-06-03T20:31:40.286795+0000","last_fullsized":"2023-06-03T20:31:40.286795+0000","mapping_epoch":254,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":255,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":216,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-06-03T20:31:40.290399+0000","last_change":"2023-06-03T20:23:28.291250+0000","last_active":"2023-06-03T20:31:40.290399+0000","last_peered":"2023-06-03T20:31:40.290399+0000","last_clean":"2023-06-03T20:31:40.290399+0000","last_became_active":"2023-06-03T20:23:28.290120+0000","last_became_peered":"2023-06-03T20:23:28.290120+0000","last_unstale":"2023-06-03T20:31:40.290399+0000","last_undegraded":"2023-06-03T20:31:40.290399+0000","last_fullsized":"2023-06-03T20:31:40.290399+0000","mapping_epoch":251,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":252,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T22:37:56.033327+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"14.4","version":"296'12198","reported_seq":12224,"reported_epoch":296,"state":"active+clean","last_fresh":"2023-06-03T20:31:57.836020+0000","last_change":"2023-06-03T20:27:21.366337+0000","last_active":"2023-06-03T20:31:57.836020+0000","last_peered":"2023-06-03T20:31:57.836020+0000","last_clean":"2023-06-03T20:31:57.836020+0000","last_became_active":"2023-06-03T20:27:21.365886+0000","last_became_peered":"2023-06-03T20:27:21.365886+0000","last_unstale":"2023-06-03T20:31:57.836020+0000","last_undegraded":"2023-06-03T20:31:57.836020+0000","last_fullsized":"2023-06-03T20:31:57.836020+0000","mapping_epoch":278,"log_start":"281'2500","ondisk_log_start":"281'2500","created":278,"last_epoch_clean":279,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T20:27:20.339366+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T20:27:20.339366+0000","last_clean_scrub_stamp":"2023-06-03T20:27:20.339366+0000","objects_scrubbed":0,"log_size":9698,"log_dups_size":0,"ondisk_log_size":9698,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12198,"num_write_kb":390336,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,3],"acting":[6,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":201,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-06-03T20:31:40.289676+0000","last_change":"2023-06-03T20:23:27.276693+0000","last_active":"2023-06-03T20:31:40.289676+0000","last_peered":"2023-06-03T20:31:40.289676+0000","last_clean":"2023-06-03T20:31:40.289676+0000","last_became_active":"2023-06-03T20:23:27.276305+0000","last_became_peered":"2023-06-03T20:23:27.276305+0000","last_unstale":"2023-06-03T20:31:40.289676+0000","last_undegraded":"2023-06-03T20:31:40.289676+0000","last_fullsized":"2023-06-03T20:31:40.289676+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T20:34:50.944936+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":206,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-06-03T20:31:40.290158+0000","last_change":"2023-06-03T20:28:51.231829+0000","last_active":"2023-06-03T20:31:40.290158+0000","last_peered":"2023-06-03T20:31:40.290158+0000","last_clean":"2023-06-03T20:31:40.290158+0000","last_became_active":"2023-06-03T20:28:51.231452+0000","last_became_peered":"2023-06-03T20:28:51.231452+0000","last_unstale":"2023-06-03T20:31:40.290158+0000","last_undegraded":"2023-06-03T20:31:40.290158+0000","last_fullsized":"2023-06-03T20:31:40.290158+0000","mapping_epoch":286,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":287,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T07:00:21.647058+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"14.5","version":"296'12398","reported_seq":12427,"reported_epoch":296,"state":"active+clean","last_fresh":"2023-06-03T20:31:57.755513+0000","last_change":"2023-06-03T20:27:21.371070+0000","last_active":"2023-06-03T20:31:57.755513+0000","last_peered":"2023-06-03T20:31:57.755513+0000","last_clean":"2023-06-03T20:31:57.755513+0000","last_became_active":"2023-06-03T20:27:21.370790+0000","last_became_peered":"2023-06-03T20:27:21.370790+0000","last_unstale":"2023-06-03T20:31:57.755513+0000","last_undegraded":"2023-06-03T20:31:57.755513+0000","last_fullsized":"2023-06-03T20:31:57.755513+0000","mapping_epoch":278,"log_start":"284'3500","ondisk_log_start":"284'3500","created":278,"last_epoch_clean":279,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T20:27:20.339366+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T20:27:20.339366+0000","last_clean_scrub_stamp":"2023-06-03T20:27:20.339366+0000","objects_scrubbed":0,"log_size":8898,"log_dups_size":0,"ondisk_log_size":8898,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T21:04:21.657715+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12398,"num_write_kb":396736,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":300,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-06-03T20:31:40.280786+0000","last_change":"2023-06-03T20:21:01.256958+0000","last_active":"2023-06-03T20:31:40.280786+0000","last_peered":"2023-06-03T20:31:40.280786+0000","last_clean":"2023-06-03T20:31:40.280786+0000","last_became_active":"2023-06-03T20:21:01.255452+0000","last_became_peered":"2023-06-03T20:21:01.255452+0000","last_unstale":"2023-06-03T20:31:40.280786+0000","last_undegraded":"2023-06-03T20:31:40.280786+0000","last_fullsized":"2023-06-03T20:31:40.280786+0000","mapping_epoch":235,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":236,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T01:56:59.777763+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":86,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-06-03T20:31:40.415640+0000","last_change":"2023-06-03T20:23:28.250884+0000","last_active":"2023-06-03T20:31:40.415640+0000","last_peered":"2023-06-03T20:31:40.415640+0000","last_clean":"2023-06-03T20:31:40.415640+0000","last_became_active":"2023-06-03T20:23:28.243793+0000","last_became_peered":"2023-06-03T20:23:28.243793+0000","last_unstale":"2023-06-03T20:31:40.415640+0000","last_undegraded":"2023-06-03T20:31:40.415640+0000","last_fullsized":"2023-06-03T20:31:40.415640+0000","mapping_epoch":251,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":252,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T03:12:27.946002+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"14.8","version":"296'12282","reported_seq":12311,"reported_epoch":296,"state":"active+clean","last_fresh":"2023-06-03T20:31:57.864163+0000","last_change":"2023-06-03T20:27:21.376040+0000","last_active":"2023-06-03T20:31:57.864163+0000","last_peered":"2023-06-03T20:31:57.864163+0000","last_clean":"2023-06-03T20:31:57.864163+0000","last_became_active":"2023-06-03T20:27:21.375709+0000","last_became_peered":"2023-06-03T20:27:21.375709+0000","last_unstale":"2023-06-03T20:31:57.864163+0000","last_undegraded":"2023-06-03T20:31:57.864163+0000","last_fullsized":"2023-06-03T20:31:57.864163+0000","mapping_epoch":278,"log_start":"283'3400","ondisk_log_start":"283'3400","created":278,"last_epoch_clean":279,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T20:27:20.339366+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T20:27:20.339366+0000","last_clean_scrub_stamp":"2023-06-03T20:27:20.339366+0000","objects_scrubbed":0,"log_size":8882,"log_dups_size":0,"ondisk_log_size":8882,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T05:50:28.157584+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12282,"num_write_kb":393024,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":76,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-06-03T20:31:40.413254+0000","last_change":"2023-06-03T20:21:01.258182+0000","last_active":"2023-06-03T20:31:40.413254+0000","last_peered":"2023-06-03T20:31:40.413254+0000","last_clean":"2023-06-03T20:31:40.413254+0000","last_became_active":"2023-06-03T20:21:01.254049+0000","last_became_peered":"2023-06-03T20:21:01.254049+0000","last_unstale":"2023-06-03T20:31:40.413254+0000","last_undegraded":"2023-06-03T20:31:40.413254+0000","last_fullsized":"2023-06-03T20:31:40.413254+0000","mapping_epoch":235,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":236,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:32:20.692043+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:32:20.692043+0000","last_clean_scrub_stamp":"2023-06-03T19:32:20.692043+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T05:16:30.078473+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":185,"reported_epoch":296,"state":"active+clean","last_fresh":"2023-06-03T20:31:48.289906+0000","last_change":"2023-06-03T20:17:22.834746+0000","last_active":"2023-06-03T20:31:48.289906+0000","last_peered":"2023-06-03T20:31:48.289906+0000","last_clean":"2023-06-03T20:31:48.289906+0000","last_became_active":"2023-06-03T20:17:22.834448+0000","last_became_peered":"2023-06-03T20:17:22.834448+0000","last_unstale":"2023-06-03T20:31:48.289906+0000","last_undegraded":"2023-06-03T20:31:48.289906+0000","last_fullsized":"2023-06-03T20:31:48.289906+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T21:16:31.167063+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,2],"acting":[3,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":87,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-06-03T20:31:40.413272+0000","last_change":"2023-06-03T20:23:28.254434+0000","last_active":"2023-06-03T20:31:40.413272+0000","last_peered":"2023-06-03T20:31:40.413272+0000","last_clean":"2023-06-03T20:31:40.413272+0000","last_became_active":"2023-06-03T20:23:28.254012+0000","last_became_peered":"2023-06-03T20:23:28.254012+0000","last_unstale":"2023-06-03T20:31:40.413272+0000","last_undegraded":"2023-06-03T20:31:40.413272+0000","last_fullsized":"2023-06-03T20:31:40.413272+0000","mapping_epoch":251,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":252,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T06:22:09.540298+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"14.e","version":"296'12296","reported_seq":12325,"reported_epoch":296,"state":"active+clean","last_fresh":"2023-06-03T20:31:57.851104+0000","last_change":"2023-06-03T20:27:21.368301+0000","last_active":"2023-06-03T20:31:57.851104+0000","last_peered":"2023-06-03T20:31:57.851104+0000","last_clean":"2023-06-03T20:31:57.851104+0000","last_became_active":"2023-06-03T20:27:21.367955+0000","last_became_peered":"2023-06-03T20:27:21.367955+0000","last_unstale":"2023-06-03T20:31:57.851104+0000","last_undegraded":"2023-06-03T20:31:57.851104+0000","last_fullsized":"2023-06-03T20:31:57.851104+0000","mapping_epoch":278,"log_start":"285'4600","ondisk_log_start":"285'4600","created":278,"last_epoch_clean":279,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T20:27:20.339366+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T20:27:20.339366+0000","last_clean_scrub_stamp":"2023-06-03T20:27:20.339366+0000","objects_scrubbed":0,"log_size":7696,"log_dups_size":0,"ondisk_log_size":7696,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T22:35:27.855608+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12296,"num_write_kb":393472,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":85,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-06-03T20:31:40.413783+0000","last_change":"2023-06-03T20:28:51.234855+0000","last_active":"2023-06-03T20:31:40.413783+0000","last_peered":"2023-06-03T20:31:40.413783+0000","last_clean":"2023-06-03T20:31:40.413783+0000","last_became_active":"2023-06-03T20:28:51.234514+0000","last_became_peered":"2023-06-03T20:28:51.234514+0000","last_unstale":"2023-06-03T20:31:40.413783+0000","last_undegraded":"2023-06-03T20:31:40.413783+0000","last_fullsized":"2023-06-03T20:31:40.413783+0000","mapping_epoch":286,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":287,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:32:20.692043+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:32:20.692043+0000","last_clean_scrub_stamp":"2023-06-03T19:32:20.692043+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T03:00:56.480804+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":182,"reported_epoch":296,"state":"active+clean","last_fresh":"2023-06-03T20:31:48.296068+0000","last_change":"2023-06-03T20:24:02.258311+0000","last_active":"2023-06-03T20:31:48.296068+0000","last_peered":"2023-06-03T20:31:48.296068+0000","last_clean":"2023-06-03T20:31:48.296068+0000","last_became_active":"2023-06-03T20:24:02.257676+0000","last_became_peered":"2023-06-03T20:24:02.257676+0000","last_unstale":"2023-06-03T20:31:48.296068+0000","last_undegraded":"2023-06-03T20:31:48.296068+0000","last_fullsized":"2023-06-03T20:31:48.296068+0000","mapping_epoch":254,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":255,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T01:47:46.383038+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":76,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-06-03T20:31:40.413346+0000","last_change":"2023-06-03T20:21:01.259331+0000","last_active":"2023-06-03T20:31:40.413346+0000","last_peered":"2023-06-03T20:31:40.413346+0000","last_clean":"2023-06-03T20:31:40.413346+0000","last_became_active":"2023-06-03T20:21:01.258836+0000","last_became_peered":"2023-06-03T20:21:01.258836+0000","last_unstale":"2023-06-03T20:31:40.413346+0000","last_undegraded":"2023-06-03T20:31:40.413346+0000","last_fullsized":"2023-06-03T20:31:40.413346+0000","mapping_epoch":235,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":236,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T01:29:19.471600+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"14.d","version":"296'12352","reported_seq":22531,"reported_epoch":296,"state":"active+clean","last_fresh":"2023-06-03T20:31:57.829661+0000","last_change":"2023-06-03T20:31:40.405580+0000","last_active":"2023-06-03T20:31:57.829661+0000","last_peered":"2023-06-03T20:31:57.829661+0000","last_clean":"2023-06-03T20:31:57.829661+0000","last_became_active":"2023-06-03T20:31:40.291187+0000","last_became_peered":"2023-06-03T20:31:40.291187+0000","last_unstale":"2023-06-03T20:31:57.829661+0000","last_undegraded":"2023-06-03T20:31:57.829661+0000","last_fullsized":"2023-06-03T20:31:57.829661+0000","mapping_epoch":294,"log_start":"281'2652","ondisk_log_start":"281'2652","created":278,"last_epoch_clean":295,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T20:27:20.339366+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T20:27:20.339366+0000","last_clean_scrub_stamp":"2023-06-03T20:27:20.339366+0000","objects_scrubbed":0,"log_size":9700,"log_dups_size":0,"ondisk_log_size":9700,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":12352,"num_write_kb":395201,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5066,"num_bytes_recovered":275578880,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,4],"acting":[6,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":283,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-06-03T20:31:40.416286+0000","last_change":"2023-06-03T19:56:54.504859+0000","last_active":"2023-06-03T20:31:40.416286+0000","last_peered":"2023-06-03T20:31:40.416286+0000","last_clean":"2023-06-03T20:31:40.416286+0000","last_became_active":"2023-06-03T19:56:54.504455+0000","last_became_peered":"2023-06-03T19:56:54.504455+0000","last_unstale":"2023-06-03T20:31:40.416286+0000","last_undegraded":"2023-06-03T20:31:40.416286+0000","last_fullsized":"2023-06-03T20:31:40.416286+0000","mapping_epoch":119,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:32:20.692043+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:32:20.692043+0000","last_clean_scrub_stamp":"2023-06-03T19:32:20.692043+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T06:25:45.522100+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":85,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-06-03T20:31:40.412850+0000","last_change":"2023-06-03T20:24:02.256677+0000","last_active":"2023-06-03T20:31:40.412850+0000","last_peered":"2023-06-03T20:31:40.412850+0000","last_clean":"2023-06-03T20:31:40.412850+0000","last_became_active":"2023-06-03T20:24:02.256216+0000","last_became_peered":"2023-06-03T20:24:02.256216+0000","last_unstale":"2023-06-03T20:31:40.412850+0000","last_undegraded":"2023-06-03T20:31:40.412850+0000","last_fullsized":"2023-06-03T20:31:40.412850+0000","mapping_epoch":254,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":255,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T02:32:45.186204+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":76,"reported_epoch":296,"state":"active+clean","last_fresh":"2023-06-03T20:31:48.281425+0000","last_change":"2023-06-03T20:21:01.261473+0000","last_active":"2023-06-03T20:31:48.281425+0000","last_peered":"2023-06-03T20:31:48.281425+0000","last_clean":"2023-06-03T20:31:48.281425+0000","last_became_active":"2023-06-03T20:21:01.261098+0000","last_became_peered":"2023-06-03T20:21:01.261098+0000","last_unstale":"2023-06-03T20:31:48.281425+0000","last_undegraded":"2023-06-03T20:31:48.281425+0000","last_fullsized":"2023-06-03T20:31:48.281425+0000","mapping_epoch":235,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":236,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T03:56:22.986549+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"14.f","version":"296'12170","reported_seq":12196,"reported_epoch":296,"state":"active+clean","last_fresh":"2023-06-03T20:31:57.851714+0000","last_change":"2023-06-03T20:27:21.374763+0000","last_active":"2023-06-03T20:31:57.851714+0000","last_peered":"2023-06-03T20:31:57.851714+0000","last_clean":"2023-06-03T20:31:57.851714+0000","last_became_active":"2023-06-03T20:27:21.374440+0000","last_became_peered":"2023-06-03T20:27:21.374440+0000","last_unstale":"2023-06-03T20:31:57.851714+0000","last_undegraded":"2023-06-03T20:31:57.851714+0000","last_fullsized":"2023-06-03T20:31:57.851714+0000","mapping_epoch":278,"log_start":"282'3000","ondisk_log_start":"282'3000","created":278,"last_epoch_clean":279,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T20:27:20.339366+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T20:27:20.339366+0000","last_clean_scrub_stamp":"2023-06-03T20:27:20.339366+0000","objects_scrubbed":0,"log_size":9170,"log_dups_size":0,"ondisk_log_size":9170,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12170,"num_write_kb":389440,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,1],"acting":[7,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":168,"reported_epoch":296,"state":"active+clean","last_fresh":"2023-06-03T20:31:48.300769+0000","last_change":"2023-06-03T20:17:22.834467+0000","last_active":"2023-06-03T20:31:48.300769+0000","last_peered":"2023-06-03T20:31:48.300769+0000","last_clean":"2023-06-03T20:31:48.300769+0000","last_became_active":"2023-06-03T20:17:22.834147+0000","last_became_peered":"2023-06-03T20:17:22.834147+0000","last_unstale":"2023-06-03T20:31:48.300769+0000","last_undegraded":"2023-06-03T20:31:48.300769+0000","last_fullsized":"2023-06-03T20:31:48.300769+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:32:20.692043+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:32:20.692043+0000","last_clean_scrub_stamp":"2023-06-03T19:32:20.692043+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T04:36:29.567051+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,7],"acting":[2,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"72'139","reported_seq":568,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-06-03T20:31:40.281714+0000","last_change":"2023-06-03T20:27:01.248641+0000","last_active":"2023-06-03T20:31:40.281714+0000","last_peered":"2023-06-03T20:31:40.281714+0000","last_clean":"2023-06-03T20:31:40.281714+0000","last_became_active":"2023-06-03T20:27:01.248063+0000","last_became_peered":"2023-06-03T20:27:01.248063+0000","last_unstale":"2023-06-03T20:31:40.281714+0000","last_undegraded":"2023-06-03T20:31:40.281714+0000","last_fullsized":"2023-06-03T20:31:40.281714+0000","mapping_epoch":274,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":275,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:27:28.100808+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:27:28.100808+0000","last_clean_scrub_stamp":"2023-06-03T19:27:28.100808+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":285,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-06-03T20:31:40.281166+0000","last_change":"2023-06-03T20:24:02.263360+0000","last_active":"2023-06-03T20:31:40.281166+0000","last_peered":"2023-06-03T20:31:40.281166+0000","last_clean":"2023-06-03T20:31:40.281166+0000","last_became_active":"2023-06-03T20:24:02.262954+0000","last_became_peered":"2023-06-03T20:24:02.262954+0000","last_unstale":"2023-06-03T20:31:40.281166+0000","last_undegraded":"2023-06-03T20:31:40.281166+0000","last_fullsized":"2023-06-03T20:31:40.281166+0000","mapping_epoch":254,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":255,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T19:38:46.084731+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":19,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-06-03T20:31:40.286579+0000","last_change":"2023-06-03T20:28:51.234600+0000","last_active":"2023-06-03T20:31:40.286579+0000","last_peered":"2023-06-03T20:31:40.286579+0000","last_clean":"2023-06-03T20:31:40.286579+0000","last_became_active":"2023-06-03T20:28:51.234227+0000","last_became_peered":"2023-06-03T20:28:51.234227+0000","last_unstale":"2023-06-03T20:31:40.286579+0000","last_undegraded":"2023-06-03T20:31:40.286579+0000","last_fullsized":"2023-06-03T20:31:40.286579+0000","mapping_epoch":286,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":287,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"14.c","version":"296'12230","reported_seq":12259,"reported_epoch":296,"state":"active+clean","last_fresh":"2023-06-03T20:31:57.862396+0000","last_change":"2023-06-03T20:27:21.375660+0000","last_active":"2023-06-03T20:31:57.862396+0000","last_peered":"2023-06-03T20:31:57.862396+0000","last_clean":"2023-06-03T20:31:57.862396+0000","last_became_active":"2023-06-03T20:27:21.375311+0000","last_became_peered":"2023-06-03T20:27:21.375311+0000","last_unstale":"2023-06-03T20:31:57.862396+0000","last_undegraded":"2023-06-03T20:31:57.862396+0000","last_fullsized":"2023-06-03T20:31:57.862396+0000","mapping_epoch":278,"log_start":"283'3400","ondisk_log_start":"283'3400","created":278,"last_epoch_clean":279,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T20:27:20.339366+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T20:27:20.339366+0000","last_clean_scrub_stamp":"2023-06-03T20:27:20.339366+0000","objects_scrubbed":0,"log_size":8830,"log_dups_size":0,"ondisk_log_size":8830,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T04:43:56.400215+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12230,"num_write_kb":391360,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,2],"acting":[0,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":288,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-06-03T20:31:40.281630+0000","last_change":"2023-06-03T20:21:01.258113+0000","last_active":"2023-06-03T20:31:40.281630+0000","last_peered":"2023-06-03T20:31:40.281630+0000","last_clean":"2023-06-03T20:31:40.281630+0000","last_became_active":"2023-06-03T20:21:01.257488+0000","last_became_peered":"2023-06-03T20:21:01.257488+0000","last_unstale":"2023-06-03T20:31:40.281630+0000","last_undegraded":"2023-06-03T20:31:40.281630+0000","last_fullsized":"2023-06-03T20:31:40.281630+0000","mapping_epoch":235,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":236,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:32:20.692043+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:32:20.692043+0000","last_clean_scrub_stamp":"2023-06-03T19:32:20.692043+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":295,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-06-03T20:31:40.414478+0000","last_change":"2023-06-03T20:23:27.255164+0000","last_active":"2023-06-03T20:31:40.414478+0000","last_peered":"2023-06-03T20:31:40.414478+0000","last_clean":"2023-06-03T20:31:40.414478+0000","last_became_active":"2023-06-03T20:23:27.253932+0000","last_became_peered":"2023-06-03T20:23:27.253932+0000","last_unstale":"2023-06-03T20:31:40.414478+0000","last_undegraded":"2023-06-03T20:31:40.414478+0000","last_fullsized":"2023-06-03T20:31:40.414478+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T06:42:02.303554+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":298,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-06-03T20:31:40.415857+0000","last_change":"2023-06-03T20:21:01.250266+0000","last_active":"2023-06-03T20:31:40.415857+0000","last_peered":"2023-06-03T20:31:40.415857+0000","last_clean":"2023-06-03T20:31:40.415857+0000","last_became_active":"2023-06-03T20:21:01.249839+0000","last_became_peered":"2023-06-03T20:21:01.249839+0000","last_unstale":"2023-06-03T20:31:40.415857+0000","last_undegraded":"2023-06-03T20:31:40.415857+0000","last_fullsized":"2023-06-03T20:31:40.415857+0000","mapping_epoch":235,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":236,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T05:13:26.584607+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"14.9","version":"296'12242","reported_seq":12271,"reported_epoch":296,"state":"active+clean","last_fresh":"2023-06-03T20:31:57.806430+0000","last_change":"2023-06-03T20:27:21.372404+0000","last_active":"2023-06-03T20:31:57.806430+0000","last_peered":"2023-06-03T20:31:57.806430+0000","last_clean":"2023-06-03T20:31:57.806430+0000","last_became_active":"2023-06-03T20:27:21.371829+0000","last_became_peered":"2023-06-03T20:27:21.371829+0000","last_unstale":"2023-06-03T20:31:57.806430+0000","last_undegraded":"2023-06-03T20:31:57.806430+0000","last_fullsized":"2023-06-03T20:31:57.806430+0000","mapping_epoch":278,"log_start":"285'4100","ondisk_log_start":"285'4100","created":278,"last_epoch_clean":279,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T20:27:20.339366+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T20:27:20.339366+0000","last_clean_scrub_stamp":"2023-06-03T20:27:20.339366+0000","objects_scrubbed":0,"log_size":8142,"log_dups_size":0,"ondisk_log_size":8142,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T02:20:57.246181+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12242,"num_write_kb":391744,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,4],"acting":[3,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":297,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-06-03T20:31:40.282732+0000","last_change":"2023-06-03T20:23:27.261511+0000","last_active":"2023-06-03T20:31:40.282732+0000","last_peered":"2023-06-03T20:31:40.282732+0000","last_clean":"2023-06-03T20:31:40.282732+0000","last_became_active":"2023-06-03T20:23:27.259622+0000","last_became_peered":"2023-06-03T20:23:27.259622+0000","last_unstale":"2023-06-03T20:31:40.282732+0000","last_undegraded":"2023-06-03T20:31:40.282732+0000","last_fullsized":"2023-06-03T20:31:40.282732+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:32:20.692043+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:32:20.692043+0000","last_clean_scrub_stamp":"2023-06-03T19:32:20.692043+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":76,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-06-03T20:31:40.413152+0000","last_change":"2023-06-03T20:21:01.252859+0000","last_active":"2023-06-03T20:31:40.413152+0000","last_peered":"2023-06-03T20:31:40.413152+0000","last_clean":"2023-06-03T20:31:40.413152+0000","last_became_active":"2023-06-03T20:21:01.251222+0000","last_became_peered":"2023-06-03T20:21:01.251222+0000","last_unstale":"2023-06-03T20:31:40.413152+0000","last_undegraded":"2023-06-03T20:31:40.413152+0000","last_fullsized":"2023-06-03T20:31:40.413152+0000","mapping_epoch":235,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":236,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T05:19:58.100226+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":251,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-06-03T20:31:40.421178+0000","last_change":"2023-06-03T19:47:06.733532+0000","last_active":"2023-06-03T20:31:40.421178+0000","last_peered":"2023-06-03T20:31:40.421178+0000","last_clean":"2023-06-03T20:31:40.421178+0000","last_became_active":"2023-06-03T19:47:06.733131+0000","last_became_peered":"2023-06-03T19:47:06.733131+0000","last_unstale":"2023-06-03T20:31:40.421178+0000","last_undegraded":"2023-06-03T20:31:40.421178+0000","last_fullsized":"2023-06-03T20:31:40.421178+0000","mapping_epoch":86,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":87,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T23:46:03.788646+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"14.a","version":"296'12084","reported_seq":12113,"reported_epoch":296,"state":"active+clean","last_fresh":"2023-06-03T20:31:57.789569+0000","last_change":"2023-06-03T20:27:21.378317+0000","last_active":"2023-06-03T20:31:57.789569+0000","last_peered":"2023-06-03T20:31:57.789569+0000","last_clean":"2023-06-03T20:31:57.789569+0000","last_became_active":"2023-06-03T20:27:21.377947+0000","last_became_peered":"2023-06-03T20:27:21.377947+0000","last_unstale":"2023-06-03T20:31:57.789569+0000","last_undegraded":"2023-06-03T20:31:57.789569+0000","last_fullsized":"2023-06-03T20:31:57.789569+0000","mapping_epoch":278,"log_start":"285'4500","ondisk_log_start":"285'4500","created":278,"last_epoch_clean":279,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T20:27:20.339366+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T20:27:20.339366+0000","last_clean_scrub_stamp":"2023-06-03T20:27:20.339366+0000","objects_scrubbed":0,"log_size":7584,"log_dups_size":0,"ondisk_log_size":7584,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T23:39:02.595450+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12084,"num_write_kb":386688,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,3],"acting":[1,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.7","version":"60'2","reported_seq":261,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-06-03T20:31:40.281773+0000","last_change":"2023-06-03T19:56:54.510148+0000","last_active":"2023-06-03T20:31:40.281773+0000","last_peered":"2023-06-03T20:31:40.281773+0000","last_clean":"2023-06-03T20:31:40.281773+0000","last_became_active":"2023-06-03T19:56:54.509771+0000","last_became_peered":"2023-06-03T19:56:54.509771+0000","last_unstale":"2023-06-03T20:31:40.281773+0000","last_undegraded":"2023-06-03T20:31:40.281773+0000","last_fullsized":"2023-06-03T20:31:40.281773+0000","mapping_epoch":119,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T00:52:42.807432+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":94,"reported_epoch":296,"state":"active+clean","last_fresh":"2023-06-03T20:31:48.276678+0000","last_change":"2023-06-03T20:24:02.257142+0000","last_active":"2023-06-03T20:31:48.276678+0000","last_peered":"2023-06-03T20:31:48.276678+0000","last_clean":"2023-06-03T20:31:48.276678+0000","last_became_active":"2023-06-03T20:24:02.256483+0000","last_became_peered":"2023-06-03T20:24:02.256483+0000","last_unstale":"2023-06-03T20:31:48.276678+0000","last_undegraded":"2023-06-03T20:31:48.276678+0000","last_fullsized":"2023-06-03T20:31:48.276678+0000","mapping_epoch":254,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":255,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:32:20.692043+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:32:20.692043+0000","last_clean_scrub_stamp":"2023-06-03T19:32:20.692043+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T00:53:37.197795+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":259,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-06-03T20:31:40.280779+0000","last_change":"2023-06-03T19:56:54.507670+0000","last_active":"2023-06-03T20:31:40.280779+0000","last_peered":"2023-06-03T20:31:40.280779+0000","last_clean":"2023-06-03T20:31:40.280779+0000","last_became_active":"2023-06-03T19:56:54.507293+0000","last_became_peered":"2023-06-03T19:56:54.507293+0000","last_unstale":"2023-06-03T20:31:40.280779+0000","last_undegraded":"2023-06-03T20:31:40.280779+0000","last_fullsized":"2023-06-03T20:31:40.280779+0000","mapping_epoch":119,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T01:49:19.966054+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"14.b","version":"296'11974","reported_seq":12003,"reported_epoch":296,"state":"active+clean","last_fresh":"2023-06-03T20:31:57.785503+0000","last_change":"2023-06-03T20:27:21.375306+0000","last_active":"2023-06-03T20:31:57.785503+0000","last_peered":"2023-06-03T20:31:57.785503+0000","last_clean":"2023-06-03T20:31:57.785503+0000","last_became_active":"2023-06-03T20:27:21.374975+0000","last_became_peered":"2023-06-03T20:27:21.374975+0000","last_unstale":"2023-06-03T20:31:57.785503+0000","last_undegraded":"2023-06-03T20:31:57.785503+0000","last_fullsized":"2023-06-03T20:31:57.785503+0000","mapping_epoch":278,"log_start":"283'3100","ondisk_log_start":"283'3100","created":278,"last_epoch_clean":279,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T20:27:20.339366+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T20:27:20.339366+0000","last_clean_scrub_stamp":"2023-06-03T20:27:20.339366+0000","objects_scrubbed":0,"log_size":8874,"log_dups_size":0,"ondisk_log_size":8874,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T02:22:27.778843+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11974,"num_write_kb":383168,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,1],"acting":[0,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":51,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-06-03T20:31:40.286889+0000","last_change":"2023-06-03T20:24:02.319230+0000","last_active":"2023-06-03T20:31:40.286889+0000","last_peered":"2023-06-03T20:31:40.286889+0000","last_clean":"2023-06-03T20:31:40.286889+0000","last_became_active":"2023-06-03T20:24:02.318926+0000","last_became_peered":"2023-06-03T20:24:02.318926+0000","last_unstale":"2023-06-03T20:31:40.286889+0000","last_undegraded":"2023-06-03T20:31:40.286889+0000","last_fullsized":"2023-06-03T20:31:40.286889+0000","mapping_epoch":254,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":255,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:32:20.692043+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:32:20.692043+0000","last_clean_scrub_stamp":"2023-06-03T19:32:20.692043+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"60'1","reported_seq":326,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-06-03T20:31:40.415478+0000","last_change":"2023-06-03T20:23:28.239664+0000","last_active":"2023-06-03T20:31:40.415478+0000","last_peered":"2023-06-03T20:31:40.415478+0000","last_clean":"2023-06-03T20:31:40.415478+0000","last_became_active":"2023-06-03T20:23:28.239252+0000","last_became_peered":"2023-06-03T20:23:28.239252+0000","last_unstale":"2023-06-03T20:31:40.415478+0000","last_undegraded":"2023-06-03T20:31:40.415478+0000","last_fullsized":"2023-06-03T20:31:40.415478+0000","mapping_epoch":251,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":252,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T06:52:20.862520+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":180,"reported_epoch":296,"state":"active+clean","last_fresh":"2023-06-03T20:31:48.296376+0000","last_change":"2023-06-03T20:24:02.259850+0000","last_active":"2023-06-03T20:31:48.296376+0000","last_peered":"2023-06-03T20:31:48.296376+0000","last_clean":"2023-06-03T20:31:48.296376+0000","last_became_active":"2023-06-03T20:24:02.258764+0000","last_became_peered":"2023-06-03T20:24:02.258764+0000","last_unstale":"2023-06-03T20:31:48.296376+0000","last_undegraded":"2023-06-03T20:31:48.296376+0000","last_fullsized":"2023-06-03T20:31:48.296376+0000","mapping_epoch":254,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":255,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T21:54:11.487473+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"60'1","reported_seq":281,"reported_epoch":296,"state":"active+clean","last_fresh":"2023-06-03T20:31:48.298201+0000","last_change":"2023-06-03T20:23:27.254853+0000","last_active":"2023-06-03T20:31:48.298201+0000","last_peered":"2023-06-03T20:31:48.298201+0000","last_clean":"2023-06-03T20:31:48.298201+0000","last_became_active":"2023-06-03T20:23:27.254410+0000","last_became_peered":"2023-06-03T20:23:27.254410+0000","last_unstale":"2023-06-03T20:31:48.298201+0000","last_undegraded":"2023-06-03T20:31:48.298201+0000","last_fullsized":"2023-06-03T20:31:48.298201+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T22:46:19.423939+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":310,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-06-03T20:31:40.416593+0000","last_change":"2023-06-03T20:21:01.257656+0000","last_active":"2023-06-03T20:31:40.416593+0000","last_peered":"2023-06-03T20:31:40.416593+0000","last_clean":"2023-06-03T20:31:40.416593+0000","last_became_active":"2023-06-03T20:21:01.257107+0000","last_became_peered":"2023-06-03T20:21:01.257107+0000","last_unstale":"2023-06-03T20:31:40.416593+0000","last_undegraded":"2023-06-03T20:31:40.416593+0000","last_fullsized":"2023-06-03T20:31:40.416593+0000","mapping_epoch":235,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":236,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T06:08:18.954539+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"60'2","reported_seq":71,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-06-03T20:31:40.289930+0000","last_change":"2023-06-03T20:23:28.243048+0000","last_active":"2023-06-03T20:31:40.289930+0000","last_peered":"2023-06-03T20:31:40.289930+0000","last_clean":"2023-06-03T20:31:40.289930+0000","last_became_active":"2023-06-03T20:23:28.241128+0000","last_became_peered":"2023-06-03T20:23:28.241128+0000","last_unstale":"2023-06-03T20:31:40.289930+0000","last_undegraded":"2023-06-03T20:31:40.289930+0000","last_fullsized":"2023-06-03T20:31:40.289930+0000","mapping_epoch":251,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":252,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T05:14:54.701329+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,1],"acting":[4,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":234,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-06-03T20:31:40.290032+0000","last_change":"2023-06-03T20:28:24.276513+0000","last_active":"2023-06-03T20:31:40.290032+0000","last_peered":"2023-06-03T20:31:40.290032+0000","last_clean":"2023-06-03T20:31:40.290032+0000","last_became_active":"2023-06-03T20:28:24.276244+0000","last_became_peered":"2023-06-03T20:28:24.276244+0000","last_unstale":"2023-06-03T20:31:40.290032+0000","last_undegraded":"2023-06-03T20:31:40.290032+0000","last_fullsized":"2023-06-03T20:31:40.290032+0000","mapping_epoch":284,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":285,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T06:17:57.159168+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[4,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"60'2","reported_seq":84,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-06-03T20:31:40.290163+0000","last_change":"2023-06-03T20:28:51.237342+0000","last_active":"2023-06-03T20:31:40.290163+0000","last_peered":"2023-06-03T20:31:40.290163+0000","last_clean":"2023-06-03T20:31:40.290163+0000","last_became_active":"2023-06-03T20:28:51.233969+0000","last_became_peered":"2023-06-03T20:28:51.233969+0000","last_unstale":"2023-06-03T20:31:40.290163+0000","last_undegraded":"2023-06-03T20:31:40.290163+0000","last_fullsized":"2023-06-03T20:31:40.290163+0000","mapping_epoch":286,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":287,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T07:30:09.503545+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":218,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-06-03T20:31:40.289148+0000","last_change":"2023-06-03T20:23:27.254468+0000","last_active":"2023-06-03T20:31:40.289148+0000","last_peered":"2023-06-03T20:31:40.289148+0000","last_clean":"2023-06-03T20:31:40.289148+0000","last_became_active":"2023-06-03T20:23:27.253859+0000","last_became_peered":"2023-06-03T20:23:27.253859+0000","last_unstale":"2023-06-03T20:31:40.289148+0000","last_undegraded":"2023-06-03T20:31:40.289148+0000","last_fullsized":"2023-06-03T20:31:40.289148+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T20:15:25.066125+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"60'1","reported_seq":315,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-06-03T20:31:40.281497+0000","last_change":"2023-06-03T20:28:51.234748+0000","last_active":"2023-06-03T20:31:40.281497+0000","last_peered":"2023-06-03T20:31:40.281497+0000","last_clean":"2023-06-03T20:31:40.281497+0000","last_became_active":"2023-06-03T20:28:51.234341+0000","last_became_peered":"2023-06-03T20:28:51.234341+0000","last_unstale":"2023-06-03T20:31:40.281497+0000","last_undegraded":"2023-06-03T20:31:40.281497+0000","last_fullsized":"2023-06-03T20:31:40.281497+0000","mapping_epoch":286,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":287,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T06:08:49.901424+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":298,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-06-03T20:31:40.281535+0000","last_change":"2023-06-03T19:56:54.503416+0000","last_active":"2023-06-03T20:31:40.281535+0000","last_peered":"2023-06-03T20:31:40.281535+0000","last_clean":"2023-06-03T20:31:40.281535+0000","last_became_active":"2023-06-03T19:56:54.503018+0000","last_became_peered":"2023-06-03T19:56:54.503018+0000","last_unstale":"2023-06-03T20:31:40.281535+0000","last_undegraded":"2023-06-03T20:31:40.281535+0000","last_fullsized":"2023-06-03T20:31:40.281535+0000","mapping_epoch":119,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T20:14:00.330616+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"60'4","reported_seq":269,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-06-03T20:31:40.432594+0000","last_change":"2023-06-03T19:56:11.748116+0000","last_active":"2023-06-03T20:31:40.432594+0000","last_peered":"2023-06-03T20:31:40.432594+0000","last_clean":"2023-06-03T20:31:40.432594+0000","last_became_active":"2023-06-03T19:56:11.747636+0000","last_became_peered":"2023-06-03T19:56:11.747636+0000","last_unstale":"2023-06-03T20:31:40.432594+0000","last_undegraded":"2023-06-03T20:31:40.432594+0000","last_fullsized":"2023-06-03T20:31:40.432594+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":115,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T06:57:48.052055+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":218,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-06-03T20:31:40.289148+0000","last_change":"2023-06-03T20:28:51.231879+0000","last_active":"2023-06-03T20:31:40.289148+0000","last_peered":"2023-06-03T20:31:40.289148+0000","last_clean":"2023-06-03T20:31:40.289148+0000","last_became_active":"2023-06-03T20:28:51.231451+0000","last_became_peered":"2023-06-03T20:28:51.231451+0000","last_unstale":"2023-06-03T20:31:40.289148+0000","last_undegraded":"2023-06-03T20:31:40.289148+0000","last_fullsized":"2023-06-03T20:31:40.289148+0000","mapping_epoch":286,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":287,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T05:47:15.344575+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":14,"num_pg":16,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":196028,"num_write_kb":6272833,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5066,"num_bytes_recovered":275578880,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":140876,"ondisk_log_size":140876,"up":48,"acting":48,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5486,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":62,"num_write_kb":30,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":23,"num_bytes_recovered":4856,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16458,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":30319,"internal_metadata":0},"log_size":56,"ondisk_log_size":56,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2969600,"data_stored":2951840,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":3,"num_store_stats":5},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":48,"seq":206158430941,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":974572,"kb_used_data":3300,"kb_used_omap":0,"kb_used_meta":971264,"kb_avail":92770580,"statfs":{"total":95995035648,"available":94997073920,"internally_reserved":0,"allocated":3379200,"data_stored":2715451,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":994574336},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 3 20:31:33 2023","interfaces":[{"interface":"back","average":{"1min":4.8049999999999997,"5min":2.226,"15min":1.8480000000000001},"min":{"1min":0.26800000000000002,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":59.356000000000002,"5min":59.356000000000002,"15min":73.810000000000002},"last":0.61599999999999999},{"interface":"front","average":{"1min":4.5469999999999997,"5min":2.0059999999999998,"15min":1.7729999999999999},"min":{"1min":0.34599999999999997,"5min":0.29999999999999999,"15min":0.26700000000000002},"max":{"1min":58.725999999999999,"5min":58.725999999999999,"15min":72.878},"last":0.78600000000000003}]},{"osd":1,"last update":"Sat Jun 3 20:31:17 2023","interfaces":[{"interface":"back","average":{"1min":1.609,"5min":1.2450000000000001,"15min":1.593},"min":{"1min":0.28899999999999998,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":4.0650000000000004,"5min":5.5999999999999996,"15min":100.845},"last":0.82799999999999996},{"interface":"front","average":{"1min":2.194,"5min":1.45,"15min":1.7370000000000001},"min":{"1min":0.30399999999999999,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":7.6559999999999997,"5min":7.6559999999999997,"15min":100.86499999999999},"last":0.84399999999999997}]},{"osd":2,"last update":"Sat Jun 3 20:31:17 2023","interfaces":[{"interface":"back","average":{"1min":2.774,"5min":1.631,"15min":2.5459999999999998},"min":{"1min":0.28899999999999998,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":16.809999999999999,"5min":16.809999999999999,"15min":101.185},"last":0.89300000000000002},{"interface":"front","average":{"1min":2.7749999999999999,"5min":1.7010000000000001,"15min":2.6749999999999998},"min":{"1min":0.36799999999999999,"5min":0.33200000000000002,"15min":0.315},"max":{"1min":16.587,"5min":16.587,"15min":100.61799999999999},"last":1.0389999999999999}]},{"osd":3,"last update":"Sat Jun 3 20:31:17 2023","interfaces":[{"interface":"back","average":{"1min":1.488,"5min":1.341,"15min":1.843},"min":{"1min":0.34999999999999998,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":6.3150000000000004,"5min":6.3150000000000004,"15min":74.039000000000001},"last":0.752},{"interface":"front","average":{"1min":2.1000000000000001,"5min":1.5720000000000001,"15min":1.9299999999999999},"min":{"1min":0.39400000000000002,"5min":0.38900000000000001,"15min":0.35399999999999998},"max":{"1min":7.7480000000000002,"5min":8.2360000000000007,"15min":73.001999999999995},"last":0.71999999999999997}]},{"osd":4,"last update":"Sat Jun 3 20:31:39 2023","interfaces":[{"interface":"back","average":{"1min":1.2569999999999999,"5min":1.1259999999999999,"15min":1.974},"min":{"1min":0.45500000000000002,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":5.9800000000000004,"5min":7.718,"15min":8.6010000000000009},"last":1.131},{"interface":"front","average":{"1min":1.6970000000000001,"5min":1.254,"15min":1.841},"min":{"1min":0.51700000000000002,"5min":0.35599999999999998,"15min":0.27600000000000002},"max":{"1min":7.6310000000000002,"5min":8.9459999999999997,"15min":9.8460000000000001},"last":1.024}]},{"osd":5,"last update":"Sat Jun 3 20:31:17 2023","interfaces":[{"interface":"back","average":{"1min":1.609,"5min":1.3380000000000001,"15min":1.2629999999999999},"min":{"1min":0.38100000000000001,"5min":0.317,"15min":0.317},"max":{"1min":9.1609999999999996,"5min":9.1609999999999996,"15min":17.696000000000002},"last":0.95299999999999996},{"interface":"front","average":{"1min":1.861,"5min":1.3,"15min":1.298},"min":{"1min":0.36499999999999999,"5min":0.33600000000000002,"15min":0.29399999999999998},"max":{"1min":7.7990000000000004,"5min":7.7990000000000004,"15min":9.3710000000000004},"last":1.167}]},{"osd":6,"last update":"Sat Jun 3 20:31:17 2023","interfaces":[{"interface":"back","average":{"1min":1.363,"5min":1.1910000000000001,"15min":1.139},"min":{"1min":0.439,"5min":0.36399999999999999,"15min":0.36399999999999999},"max":{"1min":7.7750000000000004,"5min":7.7750000000000004,"15min":9.7200000000000006},"last":1.093},{"interface":"front","average":{"1min":1.7250000000000001,"5min":1.3109999999999999,"15min":1.107},"min":{"1min":0.39100000000000001,"5min":0.36799999999999999,"15min":0.34300000000000003},"max":{"1min":8.8770000000000007,"5min":9.923,"15min":9.923},"last":0.92500000000000004}]}]},{"osd":6,"up_from":42,"seq":180388627174,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":621532,"kb_used_data":3284,"kb_used_omap":0,"kb_used_meta":618240,"kb_avail":93123620,"statfs":{"total":95995035648,"available":95358586880,"internally_reserved":0,"allocated":3362816,"data_stored":2714990,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":633077760},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 3 20:31:46 2023","interfaces":[{"interface":"back","average":{"1min":9.0449999999999999,"5min":3.8149999999999999,"15min":2.129},"min":{"1min":0.26300000000000001,"5min":0.26300000000000001,"15min":0.254},"max":{"1min":109.90900000000001,"5min":109.90900000000001,"15min":109.90900000000001},"last":0.80000000000000004},{"interface":"front","average":{"1min":9.0939999999999994,"5min":4.0659999999999998,"15min":2.258},"min":{"1min":0.36099999999999999,"5min":0.315,"15min":0.315},"max":{"1min":109.86499999999999,"5min":109.86499999999999,"15min":109.86499999999999},"last":0.71599999999999997}]},{"osd":1,"last update":"Sat Jun 3 20:31:21 2023","interfaces":[{"interface":"back","average":{"1min":7.9180000000000001,"5min":2.5459999999999998,"15min":1.552},"min":{"1min":0.29999999999999999,"5min":0.29999999999999999,"15min":0.29199999999999998},"max":{"1min":107.90600000000001,"5min":107.90600000000001,"15min":107.90600000000001},"last":1.0069999999999999},{"interface":"front","average":{"1min":7.9820000000000002,"5min":2.923,"15min":1.6719999999999999},"min":{"1min":0.34699999999999998,"5min":0.34200000000000003,"15min":0.32700000000000001},"max":{"1min":107.79900000000001,"5min":107.79900000000001,"15min":107.79900000000001},"last":0.85199999999999998}]},{"osd":2,"last update":"Sat Jun 3 20:31:21 2023","interfaces":[{"interface":"back","average":{"1min":8.4429999999999996,"5min":3.008,"15min":2.8079999999999998},"min":{"1min":0.437,"5min":0.34899999999999998,"15min":0.33500000000000002},"max":{"1min":107.61199999999999,"5min":107.61199999999999,"15min":107.61199999999999},"last":0.98099999999999998},{"interface":"front","average":{"1min":9.375,"5min":3.1549999999999998,"15min":2.8090000000000002},"min":{"1min":0.35699999999999998,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":107.83,"5min":107.83,"15min":107.83},"last":1.1200000000000001}]},{"osd":3,"last update":"Sat Jun 3 20:32:05 2023","interfaces":[{"interface":"back","average":{"1min":8.0389999999999997,"5min":2.9740000000000002,"15min":2.1640000000000001},"min":{"1min":0.35499999999999998,"5min":0.35499999999999998,"15min":0.35499999999999998},"max":{"1min":108.254,"5min":108.254,"15min":108.254},"last":1.038},{"interface":"front","average":{"1min":7.2599999999999998,"5min":3.073,"15min":2.2429999999999999},"min":{"1min":0.443,"5min":0.39100000000000001,"15min":0.39100000000000001},"max":{"1min":107.67700000000001,"5min":107.67700000000001,"15min":107.67700000000001},"last":0.92800000000000005}]},{"osd":4,"last update":"Sat Jun 3 20:31:42 2023","interfaces":[{"interface":"back","average":{"1min":1.51,"5min":1.357,"15min":1.746},"min":{"1min":0.45400000000000001,"5min":0.33000000000000002,"15min":0.33000000000000002},"max":{"1min":7.2640000000000002,"5min":8.0429999999999993,"15min":10.574999999999999},"last":1.081},{"interface":"front","average":{"1min":1.4139999999999999,"5min":1.127,"15min":1.3620000000000001},"min":{"1min":0.42299999999999999,"5min":0.40699999999999997,"15min":0.34499999999999997},"max":{"1min":6.5590000000000002,"5min":6.6710000000000003,"15min":12.622999999999999},"last":1.155}]},{"osd":5,"last update":"Sat Jun 3 20:31:34 2023","interfaces":[{"interface":"back","average":{"1min":1.8460000000000001,"5min":1.1679999999999999,"15min":1.113},"min":{"1min":0.41199999999999998,"5min":0.36299999999999999,"15min":0.29899999999999999},"max":{"1min":9.0229999999999997,"5min":9.0229999999999997,"15min":10.625999999999999},"last":1.2789999999999999},{"interface":"front","average":{"1min":1.4099999999999999,"5min":1.518,"15min":1.2589999999999999},"min":{"1min":0.46100000000000002,"5min":0.40999999999999998,"15min":0.35299999999999998},"max":{"1min":7.3879999999999999,"5min":8.0199999999999996,"15min":10.856},"last":0.95699999999999996}]},{"osd":7,"last update":"Sat Jun 3 20:31:07 2023","interfaces":[{"interface":"back","average":{"1min":1.6859999999999999,"5min":1.2689999999999999,"15min":1.22},"min":{"1min":0.42399999999999999,"5min":0.41299999999999998,"15min":0.37},"max":{"1min":9.0519999999999996,"5min":9.0519999999999996,"15min":9.0519999999999996},"last":1.2210000000000001},{"interface":"front","average":{"1min":0.95599999999999996,"5min":1.131,"15min":1.145},"min":{"1min":0.43099999999999999,"5min":0.42599999999999999,"15min":0.38},"max":{"1min":7.2850000000000001,"5min":7.2850000000000001,"15min":10.52},"last":1.2050000000000001}]}]},{"osd":1,"up_from":103,"seq":442381631972,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":482348,"kb_used_data":3236,"kb_used_omap":4,"kb_used_meta":479099,"kb_avail":93262804,"statfs":{"total":95995035648,"available":95501111296,"internally_reserved":0,"allocated":3313664,"data_stored":2184715,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":490598286},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 3 20:31:26 2023","interfaces":[{"interface":"back","average":{"1min":9.4160000000000004,"5min":6.2089999999999996,"15min":3.778},"min":{"1min":0.19800000000000001,"5min":0.19800000000000001,"15min":0.17100000000000001},"max":{"1min":131.83600000000001,"5min":131.83600000000001,"15min":131.83600000000001},"last":0.52900000000000003},{"interface":"front","average":{"1min":8.9960000000000004,"5min":6.2270000000000003,"15min":3.8050000000000002},"min":{"1min":0.33300000000000002,"5min":0.182,"15min":0.182},"max":{"1min":131.49100000000001,"5min":131.49100000000001,"15min":131.49100000000001},"last":0.72799999999999998}]},{"osd":2,"last update":"Sat Jun 3 20:31:26 2023","interfaces":[{"interface":"back","average":{"1min":10.074,"5min":3.2160000000000002,"15min":2.7999999999999998},"min":{"1min":0.26100000000000001,"5min":0.20599999999999999,"15min":0.20599999999999999},"max":{"1min":131.858,"5min":131.858,"15min":131.858},"last":0.496},{"interface":"front","average":{"1min":9.2829999999999995,"5min":3.0710000000000002,"15min":2.8479999999999999},"min":{"1min":0.28999999999999998,"5min":0.246,"15min":0.22700000000000001},"max":{"1min":131.63,"5min":131.63,"15min":131.63},"last":0.45500000000000002}]},{"osd":3,"last update":"Sat Jun 3 20:32:01 2023","interfaces":[{"interface":"back","average":{"1min":9.8569999999999993,"5min":4.2450000000000001,"15min":2.9590000000000001},"min":{"1min":0.42499999999999999,"5min":0.255,"15min":0.23400000000000001},"max":{"1min":131.65199999999999,"5min":131.65199999999999,"15min":131.65199999999999},"last":0.55600000000000005},{"interface":"front","average":{"1min":9.516,"5min":4.1929999999999996,"15min":3.056},"min":{"1min":0.33300000000000002,"5min":0.223,"15min":0.223},"max":{"1min":131.60599999999999,"5min":131.60599999999999,"15min":131.60599999999999},"last":0.76500000000000001}]},{"osd":4,"last update":"Sat Jun 3 20:31:41 2023","interfaces":[{"interface":"back","average":{"1min":1.7889999999999999,"5min":1.5980000000000001,"15min":1.7669999999999999},"min":{"1min":0.34300000000000003,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":6.1980000000000004,"5min":6.8220000000000001,"15min":11.114000000000001},"last":0.84899999999999998},{"interface":"front","average":{"1min":2.2029999999999998,"5min":1.7090000000000001,"15min":1.5700000000000001},"min":{"1min":0.39300000000000002,"5min":0.35199999999999998,"15min":0.33400000000000002},"max":{"1min":9.1270000000000007,"5min":9.1270000000000007,"15min":11.247999999999999},"last":0.876}]},{"osd":5,"last update":"Sat Jun 3 20:31:26 2023","interfaces":[{"interface":"back","average":{"1min":1.9370000000000001,"5min":1.696,"15min":1.4259999999999999},"min":{"1min":0.38200000000000001,"5min":0.35799999999999998,"15min":0.30299999999999999},"max":{"1min":6.5350000000000001,"5min":8.8889999999999993,"15min":11.246},"last":0.70699999999999996},{"interface":"front","average":{"1min":8.843,"5min":3.032,"15min":1.8400000000000001},"min":{"1min":0.42599999999999999,"5min":0.35799999999999998,"15min":0.35799999999999998},"max":{"1min":131.25299999999999,"5min":131.25299999999999,"15min":131.25299999999999},"last":0.995}]},{"osd":6,"last update":"Sat Jun 3 20:31:26 2023","interfaces":[{"interface":"back","average":{"1min":2.4260000000000002,"5min":1.7370000000000001,"15min":1.232},"min":{"1min":0.44900000000000001,"5min":0.37,"15min":0.37},"max":{"1min":9.1980000000000004,"5min":9.1980000000000004,"15min":10.002000000000001},"last":0.94699999999999995},{"interface":"front","average":{"1min":1.9319999999999999,"5min":1.4690000000000001,"15min":1.2949999999999999},"min":{"1min":0.40100000000000002,"5min":0.375,"15min":0.36899999999999999},"max":{"1min":8.6720000000000006,"5min":8.6720000000000006,"15min":11.067},"last":0.97799999999999998}]},{"osd":7,"last update":"Sat Jun 3 20:31:26 2023","interfaces":[{"interface":"back","average":{"1min":2.0249999999999999,"5min":1.506,"15min":1.478},"min":{"1min":0.377,"5min":0.377,"15min":0.372},"max":{"1min":8.7010000000000005,"5min":8.7010000000000005,"15min":10.538},"last":0.92000000000000004},{"interface":"front","average":{"1min":2.2069999999999999,"5min":1.5900000000000001,"15min":1.3520000000000001},"min":{"1min":0.40999999999999998,"5min":0.40999999999999998,"15min":0.34000000000000002},"max":{"1min":7.5739999999999998,"5min":7.5739999999999998,"15min":14.025},"last":0.90100000000000002}]}]},{"osd":0,"up_from":86,"seq":369367187999,"num_pgs":34,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":778164,"kb_used_data":3884,"kb_used_omap":5,"kb_used_meta":774266,"kb_avail":92966988,"statfs":{"total":95995035648,"available":95198195712,"internally_reserved":0,"allocated":3977216,"data_stored":2783288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":792849018},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Jun 3 20:31:15 2023","interfaces":[{"interface":"back","average":{"1min":1.8740000000000001,"5min":3.2730000000000001,"15min":2.476},"min":{"1min":0.24199999999999999,"5min":0.19500000000000001,"15min":0.185},"max":{"1min":8.5899999999999999,"5min":156.43600000000001,"15min":243.03200000000001},"last":0.995},{"interface":"front","average":{"1min":3.0590000000000002,"5min":4.1139999999999999,"15min":2.851},"min":{"1min":0.41299999999999998,"5min":0.188,"15min":0.188},"max":{"1min":10.605,"5min":157.489,"15min":242.78399999999999},"last":0.498}]},{"osd":2,"last update":"Sat Jun 3 20:31:15 2023","interfaces":[{"interface":"back","average":{"1min":1.26,"5min":1.284,"15min":1.542},"min":{"1min":0.223,"5min":0.219,"15min":0.218},"max":{"1min":4.4950000000000001,"5min":8.6799999999999997,"15min":36.515000000000001},"last":0.69299999999999995},{"interface":"front","average":{"1min":5.1390000000000002,"5min":2.6030000000000002,"15min":2.968},"min":{"1min":0.24099999999999999,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":61.795999999999999,"5min":61.795999999999999,"15min":243.30099999999999},"last":0.92000000000000004}]},{"osd":3,"last update":"Sat Jun 3 20:31:59 2023","interfaces":[{"interface":"back","average":{"1min":9.2759999999999998,"5min":3.161,"15min":4.0330000000000004},"min":{"1min":0.46800000000000003,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":152.923,"5min":152.923,"15min":259.68400000000003},"last":0.54600000000000004},{"interface":"front","average":{"1min":5.7880000000000003,"5min":2.476,"15min":3.0209999999999999},"min":{"1min":0.245,"5min":0.245,"15min":0.23799999999999999},"max":{"1min":61.957999999999998,"5min":61.957999999999998,"15min":261.02699999999999},"last":0.78600000000000003}]},{"osd":4,"last update":"Sat Jun 3 20:31:42 2023","interfaces":[{"interface":"back","average":{"1min":2.395,"5min":2.1829999999999998,"15min":1.7569999999999999},"min":{"1min":0.35899999999999999,"5min":0.34200000000000003,"15min":0.32400000000000001},"max":{"1min":5.694,"5min":11.49,"15min":11.49},"last":0.90100000000000002},{"interface":"front","average":{"1min":1.4319999999999999,"5min":1.1659999999999999,"15min":1.5189999999999999},"min":{"1min":0.39400000000000002,"5min":0.34100000000000003,"15min":0.34100000000000003},"max":{"1min":7.5060000000000002,"5min":7.5060000000000002,"15min":7.5060000000000002},"last":1.083}]},{"osd":5,"last update":"Sat Jun 3 20:31:34 2023","interfaces":[{"interface":"back","average":{"1min":2.6829999999999998,"5min":2.3069999999999999,"15min":2.4289999999999998},"min":{"1min":0.45900000000000002,"5min":0.33600000000000002,"15min":0.29299999999999998},"max":{"1min":5.6900000000000004,"5min":20.238,"15min":243.15899999999999},"last":0.95399999999999996},{"interface":"front","average":{"1min":1.3160000000000001,"5min":1.2889999999999999,"15min":2.1339999999999999},"min":{"1min":0.36199999999999999,"5min":0.36199999999999999,"15min":0.33400000000000002},"max":{"1min":7.4779999999999998,"5min":7.4779999999999998,"15min":243.32599999999999},"last":1.0229999999999999}]},{"osd":6,"last update":"Sat Jun 3 20:31:34 2023","interfaces":[{"interface":"back","average":{"1min":1.3740000000000001,"5min":1.2,"15min":1.9890000000000001},"min":{"1min":0.42399999999999999,"5min":0.36499999999999999,"15min":0.36299999999999999},"max":{"1min":7.5549999999999997,"5min":7.8869999999999996,"15min":243.25},"last":1.1850000000000001},{"interface":"front","average":{"1min":1.5640000000000001,"5min":1.3740000000000001,"15min":2.0219999999999998},"min":{"1min":0.441,"5min":0.41399999999999998,"15min":0.379},"max":{"1min":8.6170000000000009,"5min":11.634,"15min":242.84399999999999},"last":1.0580000000000001}]},{"osd":7,"last update":"Sat Jun 3 20:31:34 2023","interfaces":[{"interface":"back","average":{"1min":1.3919999999999999,"5min":1.274,"15min":2.2599999999999998},"min":{"1min":0.42599999999999999,"5min":0.39600000000000002,"15min":0.35499999999999998},"max":{"1min":7.5300000000000002,"5min":7.5300000000000002,"15min":242.93600000000001},"last":1.244},{"interface":"front","average":{"1min":1.6819999999999999,"5min":1.4750000000000001,"15min":2.2549999999999999},"min":{"1min":0.48799999999999999,"5min":0.44,"15min":0.40000000000000002},"max":{"1min":8.6440000000000001,"5min":11.68,"15min":243.21299999999999},"last":1.2250000000000001}]}]},{"osd":2,"up_from":114,"seq":489626272177,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":648836,"kb_used_data":3196,"kb_used_omap":7,"kb_used_meta":645624,"kb_avail":93096316,"statfs":{"total":95995035648,"available":95330627584,"internally_reserved":0,"allocated":3272704,"data_stored":2181943,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8111,"internal_metadata":661119057},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":5,"apply_latency_ms":5,"commit_latency_ns":5000000,"apply_latency_ns":5000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 3 20:32:05 2023","interfaces":[{"interface":"back","average":{"1min":1.647,"5min":2.5630000000000002,"15min":2.508},"min":{"1min":0.27800000000000002,"5min":0.182,"15min":0.182},"max":{"1min":7.367,"5min":77.055000000000007,"15min":117.57899999999999},"last":0.61399999999999999},{"interface":"front","average":{"1min":2.411,"5min":2.9449999999999998,"15min":2.4969999999999999},"min":{"1min":0.20200000000000001,"5min":0.20200000000000001,"15min":0.20200000000000001},"max":{"1min":8.9969999999999999,"5min":76.453000000000003,"15min":117.38500000000001},"last":0.68999999999999995}]},{"osd":1,"last update":"Sat Jun 3 20:32:05 2023","interfaces":[{"interface":"back","average":{"1min":2.3279999999999998,"5min":2.1059999999999999,"15min":2.0779999999999998},"min":{"1min":0.24099999999999999,"5min":0.193,"15min":0.193},"max":{"1min":8.9589999999999996,"5min":11.845000000000001,"15min":116.127},"last":0.80200000000000005},{"interface":"front","average":{"1min":3.0259999999999998,"5min":2.0110000000000001,"15min":2.3399999999999999},"min":{"1min":0.28799999999999998,"5min":0.251,"15min":0.251},"max":{"1min":9.6449999999999996,"5min":10.124000000000001,"15min":116.461},"last":0.53600000000000003}]},{"osd":3,"last update":"Sat Jun 3 20:32:05 2023","interfaces":[{"interface":"back","average":{"1min":2.7759999999999998,"5min":2.109,"15min":2.3010000000000002},"min":{"1min":0.32100000000000001,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":13.186999999999999,"5min":13.186999999999999,"15min":74.703000000000003},"last":0.40000000000000002},{"interface":"front","average":{"1min":3.0710000000000002,"5min":2.2040000000000002,"15min":2.3959999999999999},"min":{"1min":0.35399999999999998,"5min":0.28599999999999998,"15min":0.23000000000000001},"max":{"1min":8.2629999999999999,"5min":14.355,"15min":37.088999999999999},"last":0.65700000000000003}]},{"osd":4,"last update":"Sat Jun 3 20:31:40 2023","interfaces":[{"interface":"back","average":{"1min":2.0649999999999999,"5min":1.5600000000000001,"15min":2.637},"min":{"1min":0.39300000000000002,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":8.0839999999999996,"5min":8.9100000000000001,"15min":16.561},"last":0.96699999999999997},{"interface":"front","average":{"1min":1.5800000000000001,"5min":1.4810000000000001,"15min":1.8200000000000001},"min":{"1min":0.377,"5min":0.35099999999999998,"15min":0.35099999999999998},"max":{"1min":5.375,"5min":6.7400000000000002,"15min":9.9339999999999993},"last":0.78300000000000003}]},{"osd":5,"last update":"Sat Jun 3 20:32:05 2023","interfaces":[{"interface":"back","average":{"1min":1.5780000000000001,"5min":1.4419999999999999,"15min":1.7649999999999999},"min":{"1min":0.51500000000000001,"5min":0.35899999999999999,"15min":0.32100000000000001},"max":{"1min":7.2160000000000002,"5min":7.2160000000000002,"15min":17.751000000000001},"last":0.755},{"interface":"front","average":{"1min":2.2530000000000001,"5min":1.7769999999999999,"15min":1.679},"min":{"1min":0.44700000000000001,"5min":0.32500000000000001,"15min":0.316},"max":{"1min":6.6299999999999999,"5min":10.281000000000001,"15min":17.928000000000001},"last":0.997}]},{"osd":6,"last update":"Sat Jun 3 20:32:05 2023","interfaces":[{"interface":"back","average":{"1min":1.7490000000000001,"5min":1.7310000000000001,"15min":1.6299999999999999},"min":{"1min":0.49399999999999999,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":7.2939999999999996,"5min":7.2939999999999996,"15min":14.331},"last":0.82699999999999996},{"interface":"front","average":{"1min":2.6110000000000002,"5min":1.871,"15min":1.5780000000000001},"min":{"1min":0.47999999999999998,"5min":0.35699999999999998,"15min":0.35599999999999998},"max":{"1min":10.493,"5min":11.943,"15min":12.867000000000001},"last":0.92600000000000005}]},{"osd":7,"last update":"Sat Jun 3 20:32:05 2023","interfaces":[{"interface":"back","average":{"1min":2.0870000000000002,"5min":1.7410000000000001,"15min":1.8049999999999999},"min":{"1min":0.42099999999999999,"5min":0.38300000000000001,"15min":0.376},"max":{"1min":7.5739999999999998,"5min":7.5739999999999998,"15min":14.243},"last":0.90200000000000002},{"interface":"front","average":{"1min":2.1520000000000001,"5min":1.788,"15min":1.675},"min":{"1min":0.503,"5min":0.44700000000000001,"15min":0.38600000000000001},"max":{"1min":6.6669999999999998,"5min":10.233000000000001,"15min":12.845000000000001},"last":1.0580000000000001}]}]},{"osd":3,"up_from":119,"seq":511101108649,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":622340,"kb_used_data":3260,"kb_used_omap":2,"kb_used_meta":619069,"kb_avail":93122812,"statfs":{"total":95995035648,"available":95357759488,"internally_reserved":0,"allocated":3338240,"data_stored":2185416,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":633926709},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 3 20:32:07 2023","interfaces":[{"interface":"back","average":{"1min":1.468,"5min":4.5389999999999997,"15min":3.2690000000000001},"min":{"1min":0.41499999999999998,"5min":0.183,"15min":0.183},"max":{"1min":3.887,"5min":168.33099999999999,"15min":168.33099999999999},"last":0.60599999999999998},{"interface":"front","average":{"1min":2.2109999999999999,"5min":5.2910000000000004,"15min":3.52},"min":{"1min":0.214,"5min":0.214,"15min":0.183},"max":{"1min":9.5640000000000001,"5min":169.25800000000001,"15min":169.25800000000001},"last":0.55500000000000005}]},{"osd":1,"last update":"Sat Jun 3 20:32:07 2023","interfaces":[{"interface":"back","average":{"1min":1.966,"5min":1.911,"15min":1.7410000000000001},"min":{"1min":0.34200000000000003,"5min":0.245,"15min":0.22500000000000001},"max":{"1min":6.7530000000000001,"5min":20.992000000000001,"15min":117.047},"last":0.51300000000000001},{"interface":"front","average":{"1min":2.488,"5min":2.2210000000000001,"15min":1.7929999999999999},"min":{"1min":0.311,"5min":0.221,"15min":0.20399999999999999},"max":{"1min":9.4160000000000004,"5min":21.07,"15min":116.505},"last":0.48699999999999999}]},{"osd":2,"last update":"Sat Jun 3 20:32:07 2023","interfaces":[{"interface":"back","average":{"1min":1.425,"5min":2.4089999999999998,"15min":2.8149999999999999},"min":{"1min":0.27500000000000002,"5min":0.248,"15min":0.248},"max":{"1min":5.3860000000000001,"5min":18.916,"15min":117.584},"last":0.47099999999999997},{"interface":"front","average":{"1min":2.7280000000000002,"5min":2.0209999999999999,"15min":2.77},"min":{"1min":0.40699999999999997,"5min":0.22600000000000001,"15min":0.22600000000000001},"max":{"1min":11.526,"5min":11.526,"15min":117.01000000000001},"last":0.57799999999999996}]},{"osd":4,"last update":"Sat Jun 3 20:31:46 2023","interfaces":[{"interface":"back","average":{"1min":1.75,"5min":1.498,"15min":1.4470000000000001},"min":{"1min":0.38600000000000001,"5min":0.38,"15min":0.34599999999999997},"max":{"1min":7.0709999999999997,"5min":8.0449999999999999,"15min":8.0449999999999999},"last":0.746},{"interface":"front","average":{"1min":1.643,"5min":1.4470000000000001,"15min":1.7090000000000001},"min":{"1min":0.34699999999999998,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":6.6699999999999999,"5min":6.6699999999999999,"15min":10.773999999999999},"last":0.92800000000000005}]},{"osd":5,"last update":"Sat Jun 3 20:32:07 2023","interfaces":[{"interface":"back","average":{"1min":1.9299999999999999,"5min":1.4930000000000001,"15min":1.5680000000000001},"min":{"1min":0.38200000000000001,"5min":0.36299999999999999,"15min":0.313},"max":{"1min":7.0049999999999999,"5min":7.0049999999999999,"15min":38.651000000000003},"last":1.1279999999999999},{"interface":"front","average":{"1min":1.8620000000000001,"5min":1.7809999999999999,"15min":1.585},"min":{"1min":0.432,"5min":0.376,"15min":0.316},"max":{"1min":5.5830000000000002,"5min":9.4909999999999997,"15min":38.585999999999999},"last":0.91200000000000003}]},{"osd":6,"last update":"Sat Jun 3 20:32:07 2023","interfaces":[{"interface":"back","average":{"1min":1.3540000000000001,"5min":1.6100000000000001,"15min":1.371},"min":{"1min":0.45800000000000002,"5min":0.36499999999999999,"15min":0.34799999999999998},"max":{"1min":5.149,"5min":9.5199999999999996,"15min":9.5199999999999996},"last":0.88100000000000001},{"interface":"front","average":{"1min":2.117,"5min":1.6299999999999999,"15min":1.5249999999999999},"min":{"1min":0.50700000000000001,"5min":0.36699999999999999,"15min":0.36699999999999999},"max":{"1min":6.9119999999999999,"5min":8.0790000000000006,"15min":8.7279999999999998},"last":1.149}]},{"osd":7,"last update":"Sat Jun 3 20:32:07 2023","interfaces":[{"interface":"back","average":{"1min":1.9990000000000001,"5min":1.6970000000000001,"15min":1.7869999999999999},"min":{"1min":0.45000000000000001,"5min":0.44400000000000001,"15min":0.36599999999999999},"max":{"1min":6.9450000000000003,"5min":7.9809999999999999,"15min":39.008000000000003},"last":1.0760000000000001},{"interface":"front","average":{"1min":1.6279999999999999,"5min":1.5409999999999999,"15min":1.7949999999999999},"min":{"1min":0.40300000000000002,"5min":0.38700000000000001,"15min":0.34899999999999998},"max":{"1min":4.8239999999999998,"5min":7.4960000000000004,"15min":40.003999999999998},"last":1.014}]}]},{"osd":4,"up_from":250,"seq":1073741824105,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":262328,"kb_used_data":3376,"kb_used_omap":10,"kb_used_meta":258933,"kb_avail":93482824,"statfs":{"total":95995035648,"available":95726411776,"internally_reserved":0,"allocated":3457024,"data_stored":2199722,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":10652,"internal_metadata":265148004},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 3 20:31:42 2023","interfaces":[{"interface":"back","average":{"1min":5.4390000000000001,"5min":3.2839999999999998,"15min":2.3199999999999998},"min":{"1min":0.311,"5min":0.27600000000000002,"15min":0.26300000000000001},"max":{"1min":62.296999999999997,"5min":145.755,"15min":145.755},"last":1.137},{"interface":"front","average":{"1min":5.7750000000000004,"5min":3.3809999999999998,"15min":2.206},"min":{"1min":0.32300000000000001,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":62.450000000000003,"5min":145.71799999999999,"15min":145.71799999999999},"last":0.79000000000000004}]},{"osd":1,"last update":"Sat Jun 3 20:31:42 2023","interfaces":[{"interface":"back","average":{"1min":5.641,"5min":2.3130000000000002,"15min":1.9510000000000001},"min":{"1min":0.29399999999999998,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":62.539000000000001,"5min":62.539000000000001,"15min":62.539000000000001},"last":0.70399999999999996},{"interface":"front","average":{"1min":5.4740000000000002,"5min":2.0259999999999998,"15min":1.948},"min":{"1min":0.33800000000000002,"5min":0.29599999999999999,"15min":0.29599999999999999},"max":{"1min":62.604999999999997,"5min":62.604999999999997,"15min":62.604999999999997},"last":1.296}]},{"osd":2,"last update":"Sat Jun 3 20:31:42 2023","interfaces":[{"interface":"back","average":{"1min":10.465,"5min":3.9409999999999998,"15min":2.968},"min":{"1min":0.34000000000000002,"5min":0.34000000000000002,"15min":0.34000000000000002},"max":{"1min":78.941999999999993,"5min":78.941999999999993,"15min":78.941999999999993},"last":1.1970000000000001},{"interface":"front","average":{"1min":9.952,"5min":3.9649999999999999,"15min":3.3180000000000001},"min":{"1min":0.28999999999999998,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":80.069000000000003,"5min":80.069000000000003,"15min":80.069000000000003},"last":0.73899999999999999}]},{"osd":3,"last update":"Sat Jun 3 20:31:42 2023","interfaces":[{"interface":"back","average":{"1min":6.29,"5min":3.5510000000000002,"15min":9.1969999999999992},"min":{"1min":0.38400000000000001,"5min":0.379,"15min":0.32500000000000001},"max":{"1min":63.005000000000003,"5min":63.005000000000003,"15min":217.578},"last":1.3740000000000001},{"interface":"front","average":{"1min":6.0940000000000003,"5min":3.5430000000000001,"15min":9.0879999999999992},"min":{"1min":0.40899999999999997,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":62.569000000000003,"5min":62.569000000000003,"15min":218.55699999999999},"last":1.3540000000000001}]},{"osd":5,"last update":"Sat Jun 3 20:31:42 2023","interfaces":[{"interface":"back","average":{"1min":0.624,"5min":0.78500000000000003,"15min":2.5150000000000001},"min":{"1min":0.371,"5min":0.35099999999999998,"15min":0.35099999999999998},"max":{"1min":1.3680000000000001,"5min":3.0259999999999998,"15min":23.913},"last":1.256},{"interface":"front","average":{"1min":0.88100000000000001,"5min":1.046,"15min":1.9119999999999999},"min":{"1min":0.42599999999999999,"5min":0.34000000000000002,"15min":0.34000000000000002},"max":{"1min":3.0190000000000001,"5min":8.2650000000000006,"15min":11.093999999999999},"last":1.3180000000000001}]},{"osd":6,"last update":"Sat Jun 3 20:31:42 2023","interfaces":[{"interface":"back","average":{"1min":0.93999999999999995,"5min":0.94399999999999995,"15min":2.2519999999999998},"min":{"1min":0.36099999999999999,"5min":0.36099999999999999,"15min":0.36099999999999999},"max":{"1min":3.8799999999999999,"5min":3.8799999999999999,"15min":23.963000000000001},"last":1.429},{"interface":"front","average":{"1min":1.1180000000000001,"5min":1.1850000000000001,"15min":1.427},"min":{"1min":0.40300000000000002,"5min":0.38,"15min":0.38},"max":{"1min":3.278,"5min":6.8630000000000004,"15min":8.4190000000000005},"last":1.393}]},{"osd":7,"last update":"Sat Jun 3 20:31:42 2023","interfaces":[{"interface":"back","average":{"1min":1.25,"5min":1.165,"15min":2.0299999999999998},"min":{"1min":0.372,"5min":0.372,"15min":0.372},"max":{"1min":5.0410000000000004,"5min":5.0410000000000004,"15min":10.747999999999999},"last":1.492},{"interface":"front","average":{"1min":0.76100000000000001,"5min":1.034,"15min":1.9350000000000001},"min":{"1min":0.40799999999999997,"5min":0.40799999999999997,"15min":0.34699999999999998},"max":{"1min":2.2559999999999998,"5min":6.7699999999999996,"15min":12.154},"last":1.4590000000000001}]}]},{"osd":5,"up_from":36,"seq":154618823408,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":969168,"kb_used_data":2696,"kb_used_omap":0,"kb_used_meta":966464,"kb_avail":92775984,"statfs":{"total":95995035648,"available":95002607616,"internally_reserved":0,"allocated":2760704,"data_stored":2120460,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":989659136},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 3 20:31:24 2023","interfaces":[{"interface":"back","average":{"1min":1.71,"5min":4.0229999999999997,"15min":3.0609999999999999},"min":{"1min":0.312,"5min":0.30399999999999999,"15min":0.25700000000000001},"max":{"1min":13.555,"5min":176.88300000000001,"15min":176.88300000000001},"last":0.83299999999999996},{"interface":"front","average":{"1min":1.8640000000000001,"5min":4.2460000000000004,"15min":3.234},"min":{"1min":0.42599999999999999,"5min":0.33800000000000002,"15min":0.26500000000000001},"max":{"1min":13.589,"5min":176.96199999999999,"15min":176.96199999999999},"last":1.0620000000000001}]},{"osd":1,"last update":"Sat Jun 3 20:31:24 2023","interfaces":[{"interface":"back","average":{"1min":0.80800000000000005,"5min":2.1720000000000002,"15min":1.9370000000000001},"min":{"1min":0.28899999999999998,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":2.9470000000000001,"5min":55.515000000000001,"15min":102.205},"last":0.67800000000000005},{"interface":"front","average":{"1min":1.095,"5min":2.02,"15min":1.95},"min":{"1min":0.34499999999999997,"5min":0.316,"15min":0.30499999999999999},"max":{"1min":6.5810000000000004,"5min":55.194000000000003,"15min":102.265},"last":0.94599999999999995}]},{"osd":2,"last update":"Sat Jun 3 20:31:24 2023","interfaces":[{"interface":"back","average":{"1min":1.25,"5min":2.4849999999999999,"15min":3.4569999999999999},"min":{"1min":0.374,"5min":0.35699999999999998,"15min":0.34499999999999997},"max":{"1min":6.819,"5min":104.199,"15min":236.40000000000001},"last":0.56799999999999995},{"interface":"front","average":{"1min":1.696,"5min":3.0880000000000001,"15min":3.7799999999999998},"min":{"1min":0.39600000000000002,"5min":0.308,"15min":0.308},"max":{"1min":8.4469999999999992,"5min":104.33499999999999,"15min":237.86099999999999},"last":0.85399999999999998}]},{"osd":3,"last update":"Sat Jun 3 20:32:01 2023","interfaces":[{"interface":"back","average":{"1min":1.1499999999999999,"5min":1.5940000000000001,"15min":2.1720000000000002},"min":{"1min":0.39700000000000002,"5min":0.36799999999999999,"15min":0.34399999999999997},"max":{"1min":4.327,"5min":35.725999999999999,"15min":86.486999999999995},"last":1.1140000000000001},{"interface":"front","average":{"1min":1.3620000000000001,"5min":1.696,"15min":2.1600000000000001},"min":{"1min":0.40799999999999997,"5min":0.40799999999999997,"15min":0.37},"max":{"1min":5.6210000000000004,"5min":35.762999999999998,"15min":86.512},"last":0.99399999999999999}]},{"osd":4,"last update":"Sat Jun 3 20:31:39 2023","interfaces":[{"interface":"back","average":{"1min":0.81499999999999995,"5min":1.024,"15min":2.782},"min":{"1min":0.34100000000000003,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":3.3599999999999999,"5min":6.5670000000000002,"15min":21.786000000000001},"last":0.97399999999999998},{"interface":"front","average":{"1min":0.89700000000000002,"5min":1.1619999999999999,"15min":2.2959999999999998},"min":{"1min":0.34000000000000002,"5min":0.34000000000000002,"15min":0.30199999999999999},"max":{"1min":3.0310000000000001,"5min":11.880000000000001,"15min":26.478000000000002},"last":1.089}]},{"osd":6,"last update":"Sat Jun 3 20:31:24 2023","interfaces":[{"interface":"back","average":{"1min":0.99099999999999999,"5min":1.0660000000000001,"15min":1.133},"min":{"1min":0.371,"5min":0.371,"15min":0.35199999999999998},"max":{"1min":3.702,"5min":6.1130000000000004,"15min":7.5140000000000002},"last":0.89000000000000001},{"interface":"front","average":{"1min":1.0229999999999999,"5min":1.1439999999999999,"15min":1.22},"min":{"1min":0.42899999999999999,"5min":0.41499999999999998,"15min":0.37},"max":{"1min":6.4260000000000002,"5min":6.5309999999999997,"15min":6.5309999999999997},"last":1.0309999999999999}]},{"osd":7,"last update":"Sat Jun 3 20:31:20 2023","interfaces":[{"interface":"back","average":{"1min":0.82899999999999996,"5min":1.216,"15min":1.4239999999999999},"min":{"1min":0.45200000000000001,"5min":0.442,"15min":0.35999999999999999},"max":{"1min":3.1429999999999998,"5min":8.6549999999999994,"15min":15.516},"last":1.169},{"interface":"front","average":{"1min":0.85999999999999999,"5min":1.1319999999999999,"15min":1.25},"min":{"1min":0.38700000000000001,"5min":0.38400000000000001,"15min":0.373},"max":{"1min":3.9500000000000002,"5min":5.7009999999999996,"15min":9.6519999999999992},"last":0.93300000000000005}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6928,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4862,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":10652,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4786,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4932,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":14,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":14,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":14,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":14,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":14,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":14,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":14,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":14,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-06-03T20:32:16.158 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:32:16.197 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:32:16.200 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:32:16.201 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:32:16.302 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:32:16.330 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:32:16.332 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:32:16.333 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:32:16.434 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:32:16.469 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:32:16.472 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:32:16.473 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:32:16.574 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:32:16.585 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:32:16 smithi164 bash[59462]: cluster 1685824335.2178357 mgr.y (mgr.34104) 1696 : cluster 0 pgmap v1753: 89 pgs: 89 active+clean; 582 KiB data, 5.1 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:32:16.606 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:32:16.609 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:32:16.609 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:32:16.710 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:32:16.748 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:32:16.750 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:32:16.751 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:32:16.779 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:32:16 smithi089 bash[75068]: cluster 1685824335.2178357 mgr.y (mgr.34104) 1696 : cluster 0 pgmap v1753: 89 pgs: 89 active+clean; 582 KiB data, 5.1 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:32:16.780 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 20:32:16 smithi089 bash[71163]: ::ffff:172.21.15.164 - - [03/Jun/2023:20:32:16] "GET /metrics HTTP/1.1" 200 229016 "" "Prometheus/2.33.4" 2023-06-03T20:32:16.780 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:32:16 smithi089 bash[78452]: cluster 1685824335.2178357 mgr.y (mgr.34104) 1696 : cluster 0 pgmap v1753: 89 pgs: 89 active+clean; 582 KiB data, 5.1 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:32:16.851 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:32:16.881 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:32:16.883 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:32:16.884 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:32:16.984 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:32:17.014 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:32:17.018 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:32:17.018 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:32:17.119 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:32:17.155 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:32:17.158 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:32:17.159 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:32:17.260 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:32:17.298 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:32:17.300 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:32:17.301 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:32:17.402 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:32:17.436 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:32:17.439 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:32:17.440 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:32:17.540 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:32:17.575 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:32:17.578 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:32:17.579 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:32:17.679 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:32:17.726 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:32:17.729 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:32:17.729 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:32:17.830 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:32:17.864 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:32:17.866 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:32:17.867 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:32:17.968 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:32:18.005 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:32:18.008 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:32:18.008 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:32:18.109 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:32:18.136 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:32:18.138 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:32:18.138 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:32:18.239 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:32:18.273 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:32:18.276 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:32:18.277 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:32:18.378 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:32:18.419 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:32:18.422 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:32:18.422 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:32:18.523 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:32:18.529 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:32:18 smithi089 bash[78452]: cluster 1685824337.2186255 mgr.y (mgr.34104) 1697 : cluster 0 pgmap v1754: 89 pgs: 89 active+clean; 582 KiB data, 5.1 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:32:18.529 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:32:18 smithi089 bash[75068]: cluster 1685824337.2186255 mgr.y (mgr.34104) 1697 : cluster 0 pgmap v1754: 89 pgs: 89 active+clean; 582 KiB data, 5.1 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:32:18.552 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:32:18.554 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:32:18.554 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:32:18.585 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:32:18 smithi164 bash[59462]: cluster 1685824337.2186255 mgr.y (mgr.34104) 1697 : cluster 0 pgmap v1754: 89 pgs: 89 active+clean; 582 KiB data, 5.1 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:32:18.655 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:32:18.698 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:32:18.700 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:32:18.700 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:32:18.801 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:32:18.836 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:32:18.838 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:32:18.838 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:32:18.939 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:32:18.979 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:32:18.983 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:32:18.984 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:32:19.085 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:32:19.121 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:32:19.124 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:32:19.124 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:32:19.142 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph pg dump --format=json 2023-06-03T20:32:19.225 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:32:19.251 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:32:19.254 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:32:19.255 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:32:19.356 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:32:19.379 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:32:19.381 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:32:19.382 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:32:19.483 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:32:19.510 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:32:19.512 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:32:19.513 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:32:19.614 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:32:19.643 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:32:19.644 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:32:19.645 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:32:19.746 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:32:19.776 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:32:19.778 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:32:19.779 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:32:19.867 INFO:teuthology.orchestra.run.smithi089.stdout:osd.0: {} 2023-06-03T20:32:19.879 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:32:19.889 INFO:teuthology.orchestra.run.smithi089.stdout:osd.1: {} 2023-06-03T20:32:19.905 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:32:19.909 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:32:19.909 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:32:19.910 INFO:teuthology.orchestra.run.smithi089.stdout:osd.2: {} 2023-06-03T20:32:19.932 INFO:teuthology.orchestra.run.smithi089.stdout:osd.3: {} 2023-06-03T20:32:19.954 INFO:teuthology.orchestra.run.smithi089.stdout:osd.4: {} 2023-06-03T20:32:19.975 INFO:teuthology.orchestra.run.smithi089.stdout:osd.5: {} 2023-06-03T20:32:19.996 INFO:teuthology.orchestra.run.smithi089.stdout:osd.6: {} 2023-06-03T20:32:20.010 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:32:20.017 INFO:teuthology.orchestra.run.smithi089.stdout:osd.7: {} 2023-06-03T20:32:20.019 INFO:teuthology.orchestra.run.smithi089.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-06-03T20:32:20.019 INFO:teuthology.orchestra.run.smithi089.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-06-03T20:32:20.019 INFO:teuthology.orchestra.run.smithi089.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-06-03T20:32:20.019 INFO:teuthology.orchestra.run.smithi089.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-06-03T20:32:20.019 INFO:teuthology.orchestra.run.smithi089.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-06-03T20:32:20.019 INFO:teuthology.orchestra.run.smithi089.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-06-03T20:32:20.020 INFO:teuthology.orchestra.run.smithi089.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-06-03T20:32:20.020 INFO:teuthology.orchestra.run.smithi089.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-06-03T20:32:20.041 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:32:20.044 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:32:20.045 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:32:20.146 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:32:20.179 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:32:20.181 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:32:20.182 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:32:20.283 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:32:20.318 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:32:20.361 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:32:20.362 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:32:20.463 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:32:20.469 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:32:20 smithi089 bash[78452]: cluster 1685824339.219568 mgr.y (mgr.34104) 1698 : cluster 0 pgmap v1755: 89 pgs: 89 active+clean; 582 KiB data, 5.1 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:32:20.470 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:32:20 smithi089 bash[78452]: audit 1685824339.8384337 mon.a (mon.0) 1661 : audit 0 from='client.? 172.21.15.89:0/4059280570' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-03T20:32:20.470 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:32:20 smithi089 bash[75068]: cluster 1685824339.219568 mgr.y (mgr.34104) 1698 : cluster 0 pgmap v1755: 89 pgs: 89 active+clean; 582 KiB data, 5.1 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:32:20.470 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:32:20 smithi089 bash[75068]: audit 1685824339.8384337 mon.a (mon.0) 1661 : audit 0 from='client.? 172.21.15.89:0/4059280570' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-03T20:32:20.492 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:32:20.494 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:32:20.494 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:32:20.586 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:32:20 smithi164 bash[59462]: cluster 1685824339.219568 mgr.y (mgr.34104) 1698 : cluster 0 pgmap v1755: 89 pgs: 89 active+clean; 582 KiB data, 5.1 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:32:20.586 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:32:20 smithi164 bash[59462]: audit 1685824339.8384337 mon.a (mon.0) 1661 : audit 0 from='client.? 172.21.15.89:0/4059280570' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-03T20:32:20.595 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:32:20.623 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:32:20.625 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:32:20.625 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:32:20.726 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:32:20.752 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:32:20.754 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:32:20.755 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:32:20.856 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:32:20.885 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:32:20.888 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:32:20.888 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:32:20.989 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:32:21.018 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:32:21.021 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:32:21.021 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:32:21.122 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:32:21.154 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:32:21.156 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:32:21.156 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:32:21.257 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:32:21.296 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:32:21.299 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:32:21.300 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:32:21.327 INFO:teuthology.orchestra.run.smithi089.stderr:nodeep-scrub is set 2023-06-03T20:32:21.383 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:32:21 smithi089 bash[75068]: audit 1685824340.3956535 mon.a (mon.0) 1662 : audit 1 from='client.? 172.21.15.89:0/4225522444' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-06-03T20:32:21.384 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:32:21 smithi089 bash[75068]: audit 1685824340.7347062 mon.a (mon.0) 1663 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:32:21.384 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:32:21 smithi089 bash[75068]: audit 1685824340.747585 mon.a (mon.0) 1664 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:32:21.384 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:32:21 smithi089 bash[78452]: audit 1685824340.3956535 mon.a (mon.0) 1662 : audit 1 from='client.? 172.21.15.89:0/4225522444' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-06-03T20:32:21.384 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:32:21 smithi089 bash[78452]: audit 1685824340.7347062 mon.a (mon.0) 1663 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:32:21.384 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:32:21 smithi089 bash[78452]: audit 1685824340.747585 mon.a (mon.0) 1664 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:32:21.401 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:32:21.435 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:32:21.438 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:32:21.439 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:32:21.540 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:32:21.553 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:32:21 smithi164 bash[59462]: audit 1685824340.3956535 mon.a (mon.0) 1662 : audit 1 from='client.? 172.21.15.89:0/4225522444' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-06-03T20:32:21.554 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:32:21 smithi164 bash[59462]: audit 1685824340.7347062 mon.a (mon.0) 1663 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:32:21.554 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:32:21 smithi164 bash[59462]: audit 1685824340.747585 mon.a (mon.0) 1664 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:32:21.580 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:32:21.583 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:32:21.584 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:32:21.685 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:32:21.725 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:32:21.729 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:32:21.730 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:32:21.830 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:32:21.869 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:32:21.872 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:32:21.872 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:32:21.973 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:32:22.011 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:32:22.014 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:32:22.014 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:32:22.115 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:32:22.145 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:32:22.147 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:32:22.147 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:32:22.248 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:32:22.277 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:32:22.279 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:32:22.280 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:32:22.381 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:32:22.394 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:32:22 smithi089 bash[75068]: cluster 1685824341.2201364 mgr.y (mgr.34104) 1699 : cluster 0 pgmap v1756: 89 pgs: 89 active+clean; 582 KiB data, 5.1 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:32:22.394 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:32:22 smithi089 bash[75068]: cluster 1685824341.3082972 mon.a (mon.0) 1665 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-06-03T20:32:22.394 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:32:22 smithi089 bash[75068]: cluster 1685824341.3138595 osd.7 (osd.7) 241 : cluster 3 failed to encode map e297 with expected crc 2023-06-03T20:32:22.395 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:32:22 smithi089 bash[75068]: cluster 1685824341.3146174 osd.5 (osd.5) 245 : cluster 3 failed to encode map e297 with expected crc 2023-06-03T20:32:22.395 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:32:22 smithi089 bash[75068]: audit 1685824341.3186915 mon.a (mon.0) 1666 : audit 1 from='client.? 172.21.15.89:0/4225522444' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-06-03T20:32:22.395 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:32:22 smithi089 bash[75068]: cluster 1685824341.3187668 mon.a (mon.0) 1667 : cluster 0 osdmap e297: 8 total, 8 up, 8 in 2023-06-03T20:32:22.395 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:32:22 smithi089 bash[75068]: audit 1685824342.1100078 mon.c (mon.2) 681 : audit 1 from='client.? 172.21.15.89:0/292785894' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_9", "pool2": "unique_pool_9", "yes_i_really_really_mean_it": true}]: dispatch 2023-06-03T20:32:22.395 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:32:22 smithi089 bash[75068]: audit 1685824342.1104763 mon.a (mon.0) 1668 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_9", "pool2": "unique_pool_9", "yes_i_really_really_mean_it": true}]: dispatch 2023-06-03T20:32:22.395 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:32:22 smithi089 bash[75068]: audit 1685824342.1611774 mon.c (mon.2) 682 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T20:32:22.396 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:32:22 smithi089 bash[78452]: cluster 1685824341.2201364 mgr.y (mgr.34104) 1699 : cluster 0 pgmap v1756: 89 pgs: 89 active+clean; 582 KiB data, 5.1 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:32:22.396 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:32:22 smithi089 bash[78452]: cluster 1685824341.3082972 mon.a (mon.0) 1665 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-06-03T20:32:22.396 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:32:22 smithi089 bash[78452]: cluster 1685824341.3138595 osd.7 (osd.7) 241 : cluster 3 failed to encode map e297 with expected crc 2023-06-03T20:32:22.396 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:32:22 smithi089 bash[78452]: cluster 1685824341.3146174 osd.5 (osd.5) 245 : cluster 3 failed to encode map e297 with expected crc 2023-06-03T20:32:22.396 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:32:22 smithi089 bash[78452]: audit 1685824341.3186915 mon.a (mon.0) 1666 : audit 1 from='client.? 172.21.15.89:0/4225522444' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-06-03T20:32:22.396 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:32:22 smithi089 bash[78452]: cluster 1685824341.3187668 mon.a (mon.0) 1667 : cluster 0 osdmap e297: 8 total, 8 up, 8 in 2023-06-03T20:32:22.397 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:32:22 smithi089 bash[78452]: audit 1685824342.1100078 mon.c (mon.2) 681 : audit 1 from='client.? 172.21.15.89:0/292785894' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_9", "pool2": "unique_pool_9", "yes_i_really_really_mean_it": true}]: dispatch 2023-06-03T20:32:22.397 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:32:22 smithi089 bash[78452]: audit 1685824342.1104763 mon.a (mon.0) 1668 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_9", "pool2": "unique_pool_9", "yes_i_really_really_mean_it": true}]: dispatch 2023-06-03T20:32:22.397 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:32:22 smithi089 bash[78452]: audit 1685824342.1611774 mon.c (mon.2) 682 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T20:32:22.414 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:32:22.416 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:32:22.417 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:32:22.518 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:32:22.548 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:32:22.550 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:32:22.551 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:32:22.586 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:32:22 smithi164 bash[59462]: cluster 1685824341.2201364 mgr.y (mgr.34104) 1699 : cluster 0 pgmap v1756: 89 pgs: 89 active+clean; 582 KiB data, 5.1 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:32:22.586 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:32:22 smithi164 bash[59462]: cluster 1685824341.3082972 mon.a (mon.0) 1665 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-06-03T20:32:22.586 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:32:22 smithi164 bash[59462]: cluster 1685824341.3138595 osd.7 (osd.7) 241 : cluster 3 failed to encode map e297 with expected crc 2023-06-03T20:32:22.586 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:32:22 smithi164 bash[59462]: cluster 1685824341.3146174 osd.5 (osd.5) 245 : cluster 3 failed to encode map e297 with expected crc 2023-06-03T20:32:22.586 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:32:22 smithi164 bash[59462]: audit 1685824341.3186915 mon.a (mon.0) 1666 : audit 1 from='client.? 172.21.15.89:0/4225522444' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-06-03T20:32:22.587 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:32:22 smithi164 bash[59462]: cluster 1685824341.3187668 mon.a (mon.0) 1667 : cluster 0 osdmap e297: 8 total, 8 up, 8 in 2023-06-03T20:32:22.587 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:32:22 smithi164 bash[59462]: audit 1685824342.1100078 mon.c (mon.2) 681 : audit 1 from='client.? 172.21.15.89:0/292785894' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_9", "pool2": "unique_pool_9", "yes_i_really_really_mean_it": true}]: dispatch 2023-06-03T20:32:22.587 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:32:22 smithi164 bash[59462]: audit 1685824342.1104763 mon.a (mon.0) 1668 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_9", "pool2": "unique_pool_9", "yes_i_really_really_mean_it": true}]: dispatch 2023-06-03T20:32:22.587 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:32:22 smithi164 bash[59462]: audit 1685824342.1611774 mon.c (mon.2) 682 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T20:32:22.652 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:32:22.688 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:32:22.691 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:32:22.692 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:32:22.770 INFO:teuthology.orchestra.run.smithi089.stderr:pool 'unique_pool_9' removed 2023-06-03T20:32:22.793 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:32:22.821 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:32:22.825 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:32:22.826 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:32:22.927 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:32:22.969 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:32:22.973 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:32:22.974 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:32:23.075 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:32:23.113 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:32:23.117 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:32:23.118 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:32:23.218 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:32:23.246 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:32:23.247 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:32:23.248 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:32:23.349 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:32:23.390 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:32:23.393 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:32:23.393 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:32:23.494 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:32:23.524 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:32:23.527 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:32:23.528 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:32:23.629 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:32:23.669 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:32:23.672 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:32:23.673 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:32:23.759 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:32:23 smithi089 bash[78452]: audit 1685824342.7549279 mon.a (mon.0) 1669 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_9", "pool2": "unique_pool_9", "yes_i_really_really_mean_it": true}]': finished 2023-06-03T20:32:23.759 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:32:23 smithi089 bash[78452]: cluster 1685824342.7550545 mon.a (mon.0) 1670 : cluster 0 osdmap e298: 8 total, 8 up, 8 in 2023-06-03T20:32:23.760 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:32:23 smithi089 bash[78452]: cluster 1685824342.8799658 osd.5 (osd.5) 246 : cluster 3 failed to encode map e298 with expected crc 2023-06-03T20:32:23.760 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:32:23 smithi089 bash[78452]: cluster 1685824342.88035 osd.7 (osd.7) 242 : cluster 3 failed to encode map e298 with expected crc 2023-06-03T20:32:23.760 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:32:23 smithi089 bash[78452]: cluster 1685824343.2209425 mgr.y (mgr.34104) 1700 : cluster 0 pgmap v1759: 73 pgs: 73 active+clean; 582 KiB data, 5.1 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:32:23.760 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:32:23 smithi089 bash[75068]: audit 1685824342.7549279 mon.a (mon.0) 1669 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_9", "pool2": "unique_pool_9", "yes_i_really_really_mean_it": true}]': finished 2023-06-03T20:32:23.761 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:32:23 smithi089 bash[75068]: cluster 1685824342.7550545 mon.a (mon.0) 1670 : cluster 0 osdmap e298: 8 total, 8 up, 8 in 2023-06-03T20:32:23.761 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:32:23 smithi089 bash[75068]: cluster 1685824342.8799658 osd.5 (osd.5) 246 : cluster 3 failed to encode map e298 with expected crc 2023-06-03T20:32:23.761 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:32:23 smithi089 bash[75068]: cluster 1685824342.88035 osd.7 (osd.7) 242 : cluster 3 failed to encode map e298 with expected crc 2023-06-03T20:32:23.761 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:32:23 smithi089 bash[75068]: cluster 1685824343.2209425 mgr.y (mgr.34104) 1700 : cluster 0 pgmap v1759: 73 pgs: 73 active+clean; 582 KiB data, 5.1 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:32:23.774 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:32:23.813 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:32:23.815 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:32:23.816 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:32:23.917 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:32:23.951 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:32:23.953 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:32:23.954 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:32:24.055 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:32:24.069 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:32:23 smithi164 bash[59462]: audit 1685824342.7549279 mon.a (mon.0) 1669 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_9", "pool2": "unique_pool_9", "yes_i_really_really_mean_it": true}]': finished 2023-06-03T20:32:24.069 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:32:23 smithi164 bash[59462]: cluster 1685824342.7550545 mon.a (mon.0) 1670 : cluster 0 osdmap e298: 8 total, 8 up, 8 in 2023-06-03T20:32:24.069 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:32:23 smithi164 bash[59462]: cluster 1685824342.8799658 osd.5 (osd.5) 246 : cluster 3 failed to encode map e298 with expected crc 2023-06-03T20:32:24.069 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:32:23 smithi164 bash[59462]: cluster 1685824342.88035 osd.7 (osd.7) 242 : cluster 3 failed to encode map e298 with expected crc 2023-06-03T20:32:24.070 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:32:23 smithi164 bash[59462]: cluster 1685824343.2209425 mgr.y (mgr.34104) 1700 : cluster 0 pgmap v1759: 73 pgs: 73 active+clean; 582 KiB data, 5.1 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:32:24.095 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:32:24.098 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:32:24.100 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:32:24.145 INFO:teuthology.task.sequential:In sequential, running task print... 2023-06-03T20:32:24.146 INFO:teuthology.task.print:**** done end radosbench.yaml 2023-06-03T20:32:24.146 INFO:teuthology.task.sequential:In sequential, running task workunit... 2023-06-03T20:32:24.152 INFO:tasks.workunit:Pulling workunits from ref pacific 2023-06-03T20:32:24.153 INFO:tasks.workunit:Making a separate scratch dir for every client... 2023-06-03T20:32:24.154 DEBUG:teuthology.orchestra.run.smithi089:> stat -- /home/ubuntu/cephtest/mnt.0 2023-06-03T20:32:24.165 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:32:24.165 INFO:teuthology.orchestra.run.smithi089.stderr:stat: cannot stat '/home/ubuntu/cephtest/mnt.0': No such file or directory 2023-06-03T20:32:24.166 DEBUG:teuthology.orchestra.run.smithi089:> mkdir -- /home/ubuntu/cephtest/mnt.0 2023-06-03T20:32:24.200 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:32:24.240 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:32:24.241 INFO:tasks.workunit:Created dir /home/ubuntu/cephtest/mnt.0 2023-06-03T20:32:24.242 DEBUG:teuthology.orchestra.run.smithi089:> cd -- /home/ubuntu/cephtest/mnt.0 && mkdir -- client.0 2023-06-03T20:32:24.243 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-06-03T20:32:24.244 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:32:24.245 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:32:24.295 INFO:tasks.workunit:timeout=3h 2023-06-03T20:32:24.296 INFO:tasks.workunit:cleanup=True 2023-06-03T20:32:24.296 DEBUG:teuthology.orchestra.run.smithi089:> rm -rf /home/ubuntu/cephtest/clone.client.0 && git clone --depth 1 --branch pacific https://github.com/ceph/ceph.git /home/ubuntu/cephtest/clone.client.0 && cd /home/ubuntu/cephtest/clone.client.0 2023-06-03T20:32:24.346 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:32:24.377 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:32:24.379 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:32:24.379 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:32:24.414 INFO:tasks.workunit.client.0.smithi089.stderr:Cloning into '/home/ubuntu/cephtest/clone.client.0'... 2023-06-03T20:32:24.480 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:32:24.511 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:32:24.514 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:32:24.515 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:32:24.570 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/mon.a/config 2023-06-03T20:32:24.616 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:32:24.642 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:32:24.644 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:32:24.645 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:32:24.746 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:32:24.778 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:32:24.780 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:32:24.781 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:32:24.882 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:32:24.890 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:32:24 smithi089 bash[78452]: cluster 1685824342.8800874 osd.6 (osd.6) 233 : cluster 3 failed to encode map e298 with expected crc 2023-06-03T20:32:24.891 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:32:24 smithi089 bash[75068]: cluster 1685824342.8800874 osd.6 (osd.6) 233 : cluster 3 failed to encode map e298 with expected crc 2023-06-03T20:32:24.907 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:32:24.909 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:32:24.909 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:32:25.010 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:32:25.038 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:32:25.041 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:32:25.041 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:32:25.085 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:32:24 smithi164 bash[59462]: cluster 1685824342.8800874 osd.6 (osd.6) 233 : cluster 3 failed to encode map e298 with expected crc 2023-06-03T20:32:25.142 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:32:25.201 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:32:25.203 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:32:25.204 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:32:25.298 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph osd unset noscrub 2023-06-03T20:32:25.304 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:32:25.402 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:32:25.404 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:32:25.405 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:32:25.506 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:32:25.541 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:32:25.544 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:32:25.544 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:32:25.646 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:32:25.687 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:32:25.690 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:32:25.690 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:32:25.791 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:32:25.805 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:32:25 smithi164 bash[59462]: cluster 1685824345.221694 mgr.y (mgr.34104) 1701 : cluster 0 pgmap v1760: 73 pgs: 73 active+clean; 582 KiB data, 5.1 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:32:25.832 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:32:25.835 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:32:25.836 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:32:25.937 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:32:25.976 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:32:25.978 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:32:25.979 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:32:26.029 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:32:25 smithi089 bash[75068]: cluster 1685824345.221694 mgr.y (mgr.34104) 1701 : cluster 0 pgmap v1760: 73 pgs: 73 active+clean; 582 KiB data, 5.1 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:32:26.029 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:32:25 smithi089 bash[78452]: cluster 1685824345.221694 mgr.y (mgr.34104) 1701 : cluster 0 pgmap v1760: 73 pgs: 73 active+clean; 582 KiB data, 5.1 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:32:26.080 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:32:26.121 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:32:26.124 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:32:26.124 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:32:26.225 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:32:26.252 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:32:26.255 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:32:26.256 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:32:26.357 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:32:26.398 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:32:26.400 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:32:26.401 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:32:26.501 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:32:26.523 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 20:32:26 smithi089 bash[71163]: ::ffff:172.21.15.164 - - [03/Jun/2023:20:32:26] "GET /metrics HTTP/1.1" 200 228947 "" "Prometheus/2.33.4" 2023-06-03T20:32:26.542 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:32:26.545 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:32:26.545 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:32:26.647 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:32:26.693 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:32:26.697 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:32:26.697 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:32:26.779 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:32:26 smithi089 bash[78452]: audit 1685824345.756669 mon.a (mon.0) 1671 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:32:26.779 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:32:26 smithi089 bash[78452]: audit 1685824345.772454 mon.a (mon.0) 1672 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:32:26.779 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:32:26 smithi089 bash[78452]: audit 1685824346.2476337 mon.c (mon.2) 683 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-03T20:32:26.780 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:32:26 smithi089 bash[78452]: audit 1685824346.2488399 mon.c (mon.2) 684 : audit 1 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-03T20:32:26.780 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:32:26 smithi089 bash[78452]: audit 1685824346.2576876 mon.a (mon.0) 1673 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:32:26.780 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:32:26 smithi089 bash[75068]: audit 1685824345.756669 mon.a (mon.0) 1671 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:32:26.780 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:32:26 smithi089 bash[75068]: audit 1685824345.772454 mon.a (mon.0) 1672 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:32:26.780 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:32:26 smithi089 bash[75068]: audit 1685824346.2476337 mon.c (mon.2) 683 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-03T20:32:26.781 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:32:26 smithi089 bash[75068]: audit 1685824346.2488399 mon.c (mon.2) 684 : audit 1 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-03T20:32:26.781 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:32:26 smithi089 bash[75068]: audit 1685824346.2576876 mon.a (mon.0) 1673 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:32:26.798 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:32:26.836 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:32:26.838 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:32:26.839 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:32:26.939 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:32:26.969 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:32:26.972 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:32:26.972 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:32:27.055 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/mon.a/config 2023-06-03T20:32:27.074 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:32:27.085 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:32:26 smithi164 bash[59462]: audit 1685824345.756669 mon.a (mon.0) 1671 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:32:27.086 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:32:26 smithi164 bash[59462]: audit 1685824345.772454 mon.a (mon.0) 1672 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:32:27.086 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:32:26 smithi164 bash[59462]: audit 1685824346.2476337 mon.c (mon.2) 683 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-03T20:32:27.086 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:32:26 smithi164 bash[59462]: audit 1685824346.2488399 mon.c (mon.2) 684 : audit 1 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-03T20:32:27.086 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:32:26 smithi164 bash[59462]: audit 1685824346.2576876 mon.a (mon.0) 1673 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:32:27.115 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:32:27.118 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:32:27.119 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:32:27.220 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:32:27.255 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:32:27.258 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:32:27.259 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:32:27.360 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:32:27.392 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:32:27.394 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:32:27.395 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:32:27.496 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:32:27.534 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:32:27.536 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:32:27.537 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:32:27.638 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:32:27.666 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:32:27.668 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:32:27.668 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:32:27.769 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:32:27.806 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:32:27.810 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:32:27.811 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:32:27.912 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:32:27.952 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:32:27.954 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:32:27.955 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:32:28.029 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:32:27 smithi089 bash[75068]: cluster 1685824347.2224672 mgr.y (mgr.34104) 1702 : cluster 0 pgmap v1761: 73 pgs: 73 active+clean; 582 KiB data, 5.1 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:32:28.029 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:32:27 smithi089 bash[78452]: cluster 1685824347.2224672 mgr.y (mgr.34104) 1702 : cluster 0 pgmap v1761: 73 pgs: 73 active+clean; 582 KiB data, 5.1 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:32:28.056 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:32:28.074 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:32:27 smithi164 bash[59462]: cluster 1685824347.2224672 mgr.y (mgr.34104) 1702 : cluster 0 pgmap v1761: 73 pgs: 73 active+clean; 582 KiB data, 5.1 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:32:28.102 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:32:28.107 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:32:28.108 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:32:28.209 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:32:28.250 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:32:28.253 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:32:28.254 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:32:28.355 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:32:28.392 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:32:28.395 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:32:28.396 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:32:28.497 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:32:28.537 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:32:28.539 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:32:28.540 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:32:28.640 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:32:28.678 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:32:28.681 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:32:28.682 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:32:28.783 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:32:28.809 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:32:28.811 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:32:28.813 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:32:28.914 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:32:28.953 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:32:28.956 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:32:28.956 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:32:29.057 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:32:29.088 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:32:29.091 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:32:29.092 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:32:29.092 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/mon.a/config 2023-06-03T20:32:29.193 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:32:29.230 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:32:29.232 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:32:29.233 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:32:29.334 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:32:29.368 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:32:29.370 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:32:29.371 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:32:29.472 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:32:29.508 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:32:29.512 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:32:29.513 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:32:29.613 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:32:29.648 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:32:29.651 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:32:29.652 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:32:29.753 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:32:29.794 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:32:29.797 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:32:29.797 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:32:29.898 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:32:29.924 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:32:29.926 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:32:29.927 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:32:30.028 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:32:30.067 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:32:30.069 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:32:30.069 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:32:30.171 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:32:30.203 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:32:30.206 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:32:30.206 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:32:30.307 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:32:30.335 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:32:30 smithi164 bash[59462]: cluster 1685824349.2234657 mgr.y (mgr.34104) 1703 : cluster 0 pgmap v1762: 73 pgs: 73 active+clean; 582 KiB data, 5.1 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:32:30.336 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:32:30.338 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:32:30.339 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:32:30.440 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:32:30.481 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:32:30.484 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:32:30.484 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:32:30.529 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:32:30 smithi089 bash[78452]: cluster 1685824349.2234657 mgr.y (mgr.34104) 1703 : cluster 0 pgmap v1762: 73 pgs: 73 active+clean; 582 KiB data, 5.1 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:32:30.529 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:32:30 smithi089 bash[75068]: cluster 1685824349.2234657 mgr.y (mgr.34104) 1703 : cluster 0 pgmap v1762: 73 pgs: 73 active+clean; 582 KiB data, 5.1 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:32:30.585 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:32:30.622 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:32:30.625 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:32:30.626 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:32:30.727 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:32:30.757 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:32:30.759 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:32:30.760 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:32:30.861 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:32:30.895 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:32:30.898 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:32:30.899 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:32:31.000 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:32:31.034 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:32:31.037 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:32:31.038 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:32:31.135 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/mon.a/config 2023-06-03T20:32:31.138 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:32:31.168 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:32:31.171 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:32:31.172 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:32:31.273 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:32:31.306 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:32:31.308 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:32:31.308 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:32:31.409 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:32:31.432 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:32:31.435 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:32:31.435 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:32:31.536 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:32:31.562 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:32:31.567 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:32:31.567 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:32:31.668 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:32:31.704 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:32:31.707 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:32:31.708 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:32:31.808 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:32:31.848 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:32:31.850 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:32:31.851 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:32:31.951 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:32:31.990 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:32:31.993 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:32:31.994 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:32:32.095 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:32:32.121 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:32:32.126 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:32:32.126 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:32:32.227 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:32:32.252 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:32:32.254 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:32:32.255 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:32:32.356 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:32:32.381 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:32:32.383 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:32:32.384 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:32:32.484 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:32:32.492 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:32:32 smithi089 bash[78452]: cluster 1685824351.2239492 mgr.y (mgr.34104) 1704 : cluster 0 pgmap v1763: 73 pgs: 73 active+clean; 582 KiB data, 5.1 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:32:32.492 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:32:32 smithi089 bash[75068]: cluster 1685824351.2239492 mgr.y (mgr.34104) 1704 : cluster 0 pgmap v1763: 73 pgs: 73 active+clean; 582 KiB data, 5.1 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:32:32.509 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:32:32.511 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:32:32.512 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:32:32.585 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:32:32 smithi164 bash[59462]: cluster 1685824351.2239492 mgr.y (mgr.34104) 1704 : cluster 0 pgmap v1763: 73 pgs: 73 active+clean; 582 KiB data, 5.1 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:32:32.613 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:32:32.638 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:32:32.640 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:32:32.640 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:32:32.741 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:32:32.782 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:32:32.784 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:32:32.785 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:32:32.886 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:32:32.923 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:32:32.927 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:32:32.927 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:32:33.028 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:32:33.053 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:32:33.055 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:32:33.056 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:32:33.156 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:32:33.193 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:32:33.196 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:32:33.197 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:32:33.297 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:32:33.323 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:32:33.326 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:32:33.327 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:32:33.428 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:32:33.453 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:32:33.455 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:32:33.456 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:32:33.557 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:32:33.593 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:32:33.596 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:32:33.597 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:32:33.698 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:32:33.724 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:32:33.726 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:32:33.727 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:32:33.829 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:32:33.861 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:32:33.864 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:32:33.864 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:32:33.965 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:32:34.001 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:32:34.004 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:32:34.004 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:32:34.105 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:32:34.132 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:32:34.134 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:32:34.134 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:32:34.236 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:32:34.262 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:32:34.264 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:32:34.264 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:32:34.335 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:32:34 smithi164 bash[59462]: cluster 1685824353.2249312 mgr.y (mgr.34104) 1705 : cluster 0 pgmap v1764: 73 pgs: 73 active+clean; 582 KiB data, 5.1 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:32:34.365 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:32:34.400 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:32:34.403 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:32:34.404 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:32:34.504 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:32:34.517 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:32:34 smithi089 bash[75068]: cluster 1685824353.2249312 mgr.y (mgr.34104) 1705 : cluster 0 pgmap v1764: 73 pgs: 73 active+clean; 582 KiB data, 5.1 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:32:34.518 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:32:34 smithi089 bash[78452]: cluster 1685824353.2249312 mgr.y (mgr.34104) 1705 : cluster 0 pgmap v1764: 73 pgs: 73 active+clean; 582 KiB data, 5.1 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:32:34.532 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:32:34.534 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:32:34.535 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:32:34.636 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:32:34.675 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:32:34.678 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:32:34.679 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:32:34.764 DEBUG:teuthology.orchestra.run.smithi089:> cd -- /home/ubuntu/cephtest/clone.client.0/qa/workunits && if test -e Makefile ; then make ; fi && find -executable -type f -printf '%P\0' >/home/ubuntu/cephtest/workunits.list.client.0 2023-06-03T20:32:34.780 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:32:34.820 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:32:34.822 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:32:34.823 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:32:34.924 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:32:34.947 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:32:34.949 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:32:34.950 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:32:35.009 INFO:tasks.workunit.client.0.smithi089.stdout:for d in direct_io fs ; do ( cd $d ; make all ) ; done 2023-06-03T20:32:35.050 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:32:35.093 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:32:35.096 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:32:35.096 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:32:35.198 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:32:35.224 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:32:35.226 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:32:35.227 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:32:35.286 INFO:tasks.workunit.client.0.smithi089.stdout:make[1]: Entering directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/direct_io' 2023-06-03T20:32:35.286 INFO:tasks.workunit.client.0.smithi089.stdout:cc -Wall -Wextra -D_GNU_SOURCE direct_io_test.c -o direct_io_test 2023-06-03T20:32:35.328 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:32:35.352 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:32:35.354 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:32:35.354 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:32:35.455 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:32:35.480 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:32:35.482 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:32:35.483 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:32:35.583 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:32:35.608 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:32:35.610 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:32:35.610 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:32:35.711 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:32:35.740 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:32:35.741 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:32:35.742 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:32:35.843 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:32:35.878 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:32:35.881 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:32:35.882 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:32:35.983 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:32:36.012 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:32:36.014 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:32:36.015 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:32:36.116 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:32:36.154 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:32:36.157 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:32:36.157 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:32:36.258 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:32:36.279 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 20:32:36 smithi089 bash[71163]: ::ffff:172.21.15.164 - - [03/Jun/2023:20:32:36] "GET /metrics HTTP/1.1" 200 228947 "" "Prometheus/2.33.4" 2023-06-03T20:32:36.296 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:32:36.300 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:32:36.301 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:32:36.402 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:32:36.414 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:32:36 smithi164 bash[59462]: cluster 1685824355.225407 mgr.y (mgr.34104) 1706 : cluster 0 pgmap v1765: 73 pgs: 73 active+clean; 582 KiB data, 5.1 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:32:36.439 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:32:36.441 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:32:36.442 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:32:36.543 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:32:36.578 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:32:36.581 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:32:36.582 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:32:36.682 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:32:36.695 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:32:36 smithi089 bash[75068]: cluster 1685824355.225407 mgr.y (mgr.34104) 1706 : cluster 0 pgmap v1765: 73 pgs: 73 active+clean; 582 KiB data, 5.1 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:32:36.695 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:32:36 smithi089 bash[78452]: cluster 1685824355.225407 mgr.y (mgr.34104) 1706 : cluster 0 pgmap v1765: 73 pgs: 73 active+clean; 582 KiB data, 5.1 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:32:36.712 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:32:36.715 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:32:36.716 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:32:36.816 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:32:36.856 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:32:36.859 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:32:36.859 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:32:36.960 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:32:36.997 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:32:37.001 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:32:37.002 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:32:37.103 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:32:37.140 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:32:37.143 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:32:37.144 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:32:37.245 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:32:37.267 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:32:37.269 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:32:37.270 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:32:37.304 INFO:teuthology.orchestra.run.smithi089.stdout:{ 2023-06-03T20:32:37.305 INFO:teuthology.orchestra.run.smithi089.stdout: "size": 20, 2023-06-03T20:32:37.305 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 600, 2023-06-03T20:32:37.305 INFO:teuthology.orchestra.run.smithi089.stdout: "ops": [ 2023-06-03T20:32:37.305 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:32:37.305 INFO:teuthology.orchestra.run.smithi089.stdout: "description": "osd_op(client.36573.0:116987 13.1 13:8a1d207d:::benchmark_data_smithi089_211581_object1462:head [delete] snapc 0=[] ondisk+write+known_if_redirected e249)", 2023-06-03T20:32:37.305 INFO:teuthology.orchestra.run.smithi089.stdout: "initiated_at": "2023-06-03T20:23:26.217981+0000", 2023-06-03T20:32:37.305 INFO:teuthology.orchestra.run.smithi089.stdout: "age": 551.079306449, 2023-06-03T20:32:37.306 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 2.5776251869999998, 2023-06-03T20:32:37.306 INFO:teuthology.orchestra.run.smithi089.stdout: "type_data": { 2023-06-03T20:32:37.306 INFO:teuthology.orchestra.run.smithi089.stdout: "flag_point": "reached pg", 2023-06-03T20:32:37.306 INFO:teuthology.orchestra.run.smithi089.stdout: "client_info": { 2023-06-03T20:32:37.306 INFO:teuthology.orchestra.run.smithi089.stdout: "client": "client.36573", 2023-06-03T20:32:37.306 INFO:teuthology.orchestra.run.smithi089.stdout: "client_addr": "172.21.15.89:0/616445942", 2023-06-03T20:32:37.306 INFO:teuthology.orchestra.run.smithi089.stdout: "tid": 116987 2023-06-03T20:32:37.307 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:32:37.307 INFO:teuthology.orchestra.run.smithi089.stdout: "events": [ 2023-06-03T20:32:37.307 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:32:37.307 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "initiated", 2023-06-03T20:32:37.307 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:23:26.217981+0000", 2023-06-03T20:32:37.307 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0 2023-06-03T20:32:37.307 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:32:37.308 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:32:37.308 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "throttled", 2023-06-03T20:32:37.308 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:23:26.217981+0000", 2023-06-03T20:32:37.308 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0 2023-06-03T20:32:37.308 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:32:37.308 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:32:37.308 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "header_read", 2023-06-03T20:32:37.309 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:23:26.217981+0000", 2023-06-03T20:32:37.309 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 4294967295.9999995 2023-06-03T20:32:37.309 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:32:37.309 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:32:37.309 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "all_read", 2023-06-03T20:32:37.310 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:23:26.217983+0000", 2023-06-03T20:32:37.310 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 2.7530000000000002e-06 2023-06-03T20:32:37.310 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:32:37.310 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:32:37.311 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "dispatched", 2023-06-03T20:32:37.311 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:23:26.217989+0000", 2023-06-03T20:32:37.311 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 5.7080000000000002e-06 2023-06-03T20:32:37.311 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:32:37.311 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:32:37.311 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "queued_for_pg", 2023-06-03T20:32:37.311 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:23:26.217994+0000", 2023-06-03T20:32:37.312 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 4.8069999999999999e-06 2023-06-03T20:32:37.312 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:32:37.312 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:32:37.312 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "reached_pg", 2023-06-03T20:32:37.313 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:23:26.223352+0000", 2023-06-03T20:32:37.313 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.0053583119999999996 2023-06-03T20:32:37.313 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:32:37.313 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:32:37.313 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "started", 2023-06-03T20:32:37.313 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:23:26.223459+0000", 2023-06-03T20:32:37.313 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.000106609 2023-06-03T20:32:37.314 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:32:37.314 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:32:37.314 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "waiting for subops from 7", 2023-06-03T20:32:37.314 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:23:26.223556+0000", 2023-06-03T20:32:37.314 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 9.7079999999999999e-05 2023-06-03T20:32:37.314 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:32:37.314 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:32:37.314 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "op_commit", 2023-06-03T20:32:37.315 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:23:26.234737+0000", 2023-06-03T20:32:37.315 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.011181538 2023-06-03T20:32:37.315 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:32:37.315 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:32:37.315 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "reached_pg", 2023-06-03T20:32:37.315 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:23:26.249228+0000", 2023-06-03T20:32:37.316 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.014490493 2023-06-03T20:32:37.316 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:32:37.316 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:32:37.316 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "waiting for peered", 2023-06-03T20:32:37.316 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:23:26.249246+0000", 2023-06-03T20:32:37.316 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 1.8165999999999999e-05 2023-06-03T20:32:37.316 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:32:37.317 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:32:37.317 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "reached_pg", 2023-06-03T20:32:37.317 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:23:27.232819+0000", 2023-06-03T20:32:37.317 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.98357342599999997 2023-06-03T20:32:37.317 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:32:37.317 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:32:37.317 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "waiting for peered", 2023-06-03T20:32:37.317 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:23:27.232836+0000", 2023-06-03T20:32:37.318 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 1.6183000000000001e-05 2023-06-03T20:32:37.318 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:32:37.318 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:32:37.318 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "reached_pg", 2023-06-03T20:32:37.318 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:23:28.775787+0000", 2023-06-03T20:32:37.318 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 1.5429516219999999 2023-06-03T20:32:37.318 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:32:37.319 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:32:37.319 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "waiting for missing object", 2023-06-03T20:32:37.319 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:23:28.776107+0000", 2023-06-03T20:32:37.319 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.00032023300000000001 2023-06-03T20:32:37.319 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:32:37.319 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:32:37.319 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "reached_pg", 2023-06-03T20:32:37.319 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:23:28.795538+0000", 2023-06-03T20:32:37.320 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.019430551000000001 2023-06-03T20:32:37.320 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:32:37.320 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:32:37.320 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "done", 2023-06-03T20:32:37.320 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:23:28.795606+0000", 2023-06-03T20:32:37.320 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 6.8267000000000002e-05 2023-06-03T20:32:37.320 INFO:teuthology.orchestra.run.smithi089.stdout: } 2023-06-03T20:32:37.321 INFO:teuthology.orchestra.run.smithi089.stdout: ] 2023-06-03T20:32:37.321 INFO:teuthology.orchestra.run.smithi089.stdout: } 2023-06-03T20:32:37.321 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:32:37.321 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:32:37.321 INFO:teuthology.orchestra.run.smithi089.stdout: "description": "osd_op(client.36573.0:116990 13.1 13:829eb31f:::benchmark_data_smithi089_211581_object1465:head [delete] snapc 0=[] ondisk+write+known_if_redirected e249)", 2023-06-03T20:32:37.321 INFO:teuthology.orchestra.run.smithi089.stdout: "initiated_at": "2023-06-03T20:23:26.225309+0000", 2023-06-03T20:32:37.321 INFO:teuthology.orchestra.run.smithi089.stdout: "age": 551.07197807099999, 2023-06-03T20:32:37.321 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 2.606919397, 2023-06-03T20:32:37.322 INFO:teuthology.orchestra.run.smithi089.stdout: "type_data": { 2023-06-03T20:32:37.322 INFO:teuthology.orchestra.run.smithi089.stdout: "flag_point": "reached pg", 2023-06-03T20:32:37.322 INFO:teuthology.orchestra.run.smithi089.stdout: "client_info": { 2023-06-03T20:32:37.322 INFO:teuthology.orchestra.run.smithi089.stdout: "client": "client.36573", 2023-06-03T20:32:37.323 INFO:teuthology.orchestra.run.smithi089.stdout: "client_addr": "172.21.15.89:0/616445942", 2023-06-03T20:32:37.323 INFO:teuthology.orchestra.run.smithi089.stdout: "tid": 116990 2023-06-03T20:32:37.323 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:32:37.323 INFO:teuthology.orchestra.run.smithi089.stdout: "events": [ 2023-06-03T20:32:37.324 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:32:37.324 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "initiated", 2023-06-03T20:32:37.324 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:23:26.225309+0000", 2023-06-03T20:32:37.324 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0 2023-06-03T20:32:37.324 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:32:37.324 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:32:37.324 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "throttled", 2023-06-03T20:32:37.325 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:23:26.225309+0000", 2023-06-03T20:32:37.325 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0 2023-06-03T20:32:37.325 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:32:37.325 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:32:37.325 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "header_read", 2023-06-03T20:32:37.325 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:23:26.225308+0000", 2023-06-03T20:32:37.325 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 4294967295.9999986 2023-06-03T20:32:37.325 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:32:37.326 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:32:37.326 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "all_read", 2023-06-03T20:32:37.326 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:23:26.225315+0000", 2023-06-03T20:32:37.326 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 7.1210000000000001e-06 2023-06-03T20:32:37.326 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:32:37.326 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:32:37.326 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "dispatched", 2023-06-03T20:32:37.326 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:23:26.225397+0000", 2023-06-03T20:32:37.327 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 8.1778000000000002e-05 2023-06-03T20:32:37.327 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:32:37.327 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:32:37.327 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "queued_for_pg", 2023-06-03T20:32:37.327 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:23:26.225407+0000", 2023-06-03T20:32:37.327 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 9.4469999999999995e-06 2023-06-03T20:32:37.328 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:32:37.328 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:32:37.328 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "reached_pg", 2023-06-03T20:32:37.328 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:23:26.230556+0000", 2023-06-03T20:32:37.328 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.0051495719999999998 2023-06-03T20:32:37.328 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:32:37.328 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:32:37.328 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "started", 2023-06-03T20:32:37.329 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:23:26.230692+0000", 2023-06-03T20:32:37.329 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.000136303 2023-06-03T20:32:37.329 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:32:37.329 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:32:37.329 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "waiting for subops from 7", 2023-06-03T20:32:37.329 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:23:26.230812+0000", 2023-06-03T20:32:37.329 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.00011954 2023-06-03T20:32:37.330 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:32:37.330 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:32:37.330 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "op_commit", 2023-06-03T20:32:37.330 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:23:26.235727+0000", 2023-06-03T20:32:37.330 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.0049148960000000002 2023-06-03T20:32:37.330 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:32:37.330 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:32:37.330 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "reached_pg", 2023-06-03T20:32:37.331 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:23:26.249280+0000", 2023-06-03T20:32:37.331 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.013553107 2023-06-03T20:32:37.331 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:32:37.331 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:32:37.331 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "waiting for peered", 2023-06-03T20:32:37.331 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:23:26.249289+0000", 2023-06-03T20:32:37.331 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 8.8699999999999998e-06 2023-06-03T20:32:37.332 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:32:37.332 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:32:37.332 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "reached_pg", 2023-06-03T20:32:37.332 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:23:27.232918+0000", 2023-06-03T20:32:37.332 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.98362897900000001 2023-06-03T20:32:37.332 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:32:37.332 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:32:37.333 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "waiting for peered", 2023-06-03T20:32:37.333 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:23:27.232929+0000", 2023-06-03T20:32:37.333 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 1.1433e-05 2023-06-03T20:32:37.333 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:32:37.333 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:32:37.333 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "reached_pg", 2023-06-03T20:32:37.333 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:23:28.786760+0000", 2023-06-03T20:32:37.334 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 1.553830993 2023-06-03T20:32:37.334 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:32:37.334 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:32:37.334 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "waiting for missing object", 2023-06-03T20:32:37.334 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:23:28.786929+0000", 2023-06-03T20:32:37.334 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.000168524 2023-06-03T20:32:37.334 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:32:37.334 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:32:37.335 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "reached_pg", 2023-06-03T20:32:37.335 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:23:28.832161+0000", 2023-06-03T20:32:37.335 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.045231905000000003 2023-06-03T20:32:37.335 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:32:37.335 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:32:37.335 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "done", 2023-06-03T20:32:37.335 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:23:28.832229+0000", 2023-06-03T20:32:37.335 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 6.8143000000000001e-05 2023-06-03T20:32:37.336 INFO:teuthology.orchestra.run.smithi089.stdout: } 2023-06-03T20:32:37.336 INFO:teuthology.orchestra.run.smithi089.stdout: ] 2023-06-03T20:32:37.336 INFO:teuthology.orchestra.run.smithi089.stdout: } 2023-06-03T20:32:37.336 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:32:37.336 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:32:37.336 INFO:teuthology.orchestra.run.smithi089.stdout: "description": "osd_op(client.36573.0:178983 13.a 13:5df9ca14:::benchmark_data_smithi089_211581_object63458:head [delete] snapc 0=[] ondisk+write+known_if_redirected e264)", 2023-06-03T20:32:37.336 INFO:teuthology.orchestra.run.smithi089.stdout: "initiated_at": "2023-06-03T20:25:19.455374+0000", 2023-06-03T20:32:37.336 INFO:teuthology.orchestra.run.smithi089.stdout: "age": 437.841913342, 2023-06-03T20:32:37.337 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.39042197099999998, 2023-06-03T20:32:37.337 INFO:teuthology.orchestra.run.smithi089.stdout: "type_data": { 2023-06-03T20:32:37.337 INFO:teuthology.orchestra.run.smithi089.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-03T20:32:37.337 INFO:teuthology.orchestra.run.smithi089.stdout: "client_info": { 2023-06-03T20:32:37.337 INFO:teuthology.orchestra.run.smithi089.stdout: "client": "client.36573", 2023-06-03T20:32:37.337 INFO:teuthology.orchestra.run.smithi089.stdout: "client_addr": "172.21.15.89:0/616445942", 2023-06-03T20:32:37.337 INFO:teuthology.orchestra.run.smithi089.stdout: "tid": 178983 2023-06-03T20:32:37.338 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:32:37.338 INFO:teuthology.orchestra.run.smithi089.stdout: "events": [ 2023-06-03T20:32:37.338 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:32:37.338 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "initiated", 2023-06-03T20:32:37.338 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:25:19.455374+0000", 2023-06-03T20:32:37.338 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0 2023-06-03T20:32:37.338 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:32:37.339 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:32:37.339 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "throttled", 2023-06-03T20:32:37.339 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:25:19.455374+0000", 2023-06-03T20:32:37.339 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0 2023-06-03T20:32:37.339 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:32:37.339 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:32:37.339 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "header_read", 2023-06-03T20:32:37.339 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:25:19.455373+0000", 2023-06-03T20:32:37.340 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 4294967295.9999986 2023-06-03T20:32:37.340 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:32:37.340 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:32:37.340 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "all_read", 2023-06-03T20:32:37.340 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:25:19.455380+0000", 2023-06-03T20:32:37.340 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 7.3350000000000001e-06 2023-06-03T20:32:37.340 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:32:37.340 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:32:37.341 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "dispatched", 2023-06-03T20:32:37.341 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:25:19.455393+0000", 2023-06-03T20:32:37.341 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 1.2673e-05 2023-06-03T20:32:37.341 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:32:37.341 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:32:37.341 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "queued_for_pg", 2023-06-03T20:32:37.341 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:25:19.455402+0000", 2023-06-03T20:32:37.342 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 8.986e-06 2023-06-03T20:32:37.342 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:32:37.342 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:32:37.342 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "reached_pg", 2023-06-03T20:32:37.342 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:25:19.473653+0000", 2023-06-03T20:32:37.342 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.018251414000000001 2023-06-03T20:32:37.343 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:32:37.343 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:32:37.343 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "started", 2023-06-03T20:32:37.343 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:25:19.474062+0000", 2023-06-03T20:32:37.343 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.00040880400000000003 2023-06-03T20:32:37.343 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:32:37.343 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:32:37.344 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "waiting for subops from 1,4", 2023-06-03T20:32:37.344 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:25:19.474234+0000", 2023-06-03T20:32:37.344 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.000171779 2023-06-03T20:32:37.344 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:32:37.344 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:32:37.344 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "op_commit", 2023-06-03T20:32:37.344 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:25:19.486850+0000", 2023-06-03T20:32:37.345 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.012615648 2023-06-03T20:32:37.345 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:32:37.345 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:32:37.345 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:32:37.345 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:25:19.486907+0000", 2023-06-03T20:32:37.345 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 5.7371999999999999e-05 2023-06-03T20:32:37.345 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:32:37.345 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:32:37.346 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:32:37.346 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:25:19.841716+0000", 2023-06-03T20:32:37.346 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.35480877599999999 2023-06-03T20:32:37.346 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:32:37.346 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:32:37.346 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "commit_sent", 2023-06-03T20:32:37.346 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:25:19.841776+0000", 2023-06-03T20:32:37.346 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 6.0600000000000003e-05 2023-06-03T20:32:37.347 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:32:37.347 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:32:37.347 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "done", 2023-06-03T20:32:37.347 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:25:19.845796+0000", 2023-06-03T20:32:37.347 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.0040199149999999998 2023-06-03T20:32:37.347 INFO:teuthology.orchestra.run.smithi089.stdout: } 2023-06-03T20:32:37.347 INFO:teuthology.orchestra.run.smithi089.stdout: ] 2023-06-03T20:32:37.348 INFO:teuthology.orchestra.run.smithi089.stdout: } 2023-06-03T20:32:37.348 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:32:37.348 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:32:37.348 INFO:teuthology.orchestra.run.smithi089.stdout: "description": "osd_op(client.36573.0:178991 13.a 13:58131c32:::benchmark_data_smithi089_211581_object63466:head [delete] snapc 0=[] ondisk+write+known_if_redirected e264)", 2023-06-03T20:32:37.349 INFO:teuthology.orchestra.run.smithi089.stdout: "initiated_at": "2023-06-03T20:25:19.466480+0000", 2023-06-03T20:32:37.349 INFO:teuthology.orchestra.run.smithi089.stdout: "age": 437.83080745199999, 2023-06-03T20:32:37.349 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.38674060199999999, 2023-06-03T20:32:37.349 INFO:teuthology.orchestra.run.smithi089.stdout: "type_data": { 2023-06-03T20:32:37.349 INFO:teuthology.orchestra.run.smithi089.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-03T20:32:37.349 INFO:teuthology.orchestra.run.smithi089.stdout: "client_info": { 2023-06-03T20:32:37.349 INFO:teuthology.orchestra.run.smithi089.stdout: "client": "client.36573", 2023-06-03T20:32:37.349 INFO:teuthology.orchestra.run.smithi089.stdout: "client_addr": "172.21.15.89:0/616445942", 2023-06-03T20:32:37.350 INFO:teuthology.orchestra.run.smithi089.stdout: "tid": 178991 2023-06-03T20:32:37.350 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:32:37.350 INFO:teuthology.orchestra.run.smithi089.stdout: "events": [ 2023-06-03T20:32:37.350 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:32:37.350 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "initiated", 2023-06-03T20:32:37.350 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:25:19.466480+0000", 2023-06-03T20:32:37.351 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0 2023-06-03T20:32:37.351 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:32:37.351 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:32:37.352 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "throttled", 2023-06-03T20:32:37.352 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:25:19.466480+0000", 2023-06-03T20:32:37.352 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0 2023-06-03T20:32:37.352 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:32:37.352 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:32:37.353 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "header_read", 2023-06-03T20:32:37.353 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:25:19.466478+0000", 2023-06-03T20:32:37.353 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 4294967295.9999986 2023-06-03T20:32:37.353 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:32:37.353 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:32:37.353 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "all_read", 2023-06-03T20:32:37.353 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:25:19.466485+0000", 2023-06-03T20:32:37.353 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 6.9650000000000002e-06 2023-06-03T20:32:37.354 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:32:37.354 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:32:37.354 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "dispatched", 2023-06-03T20:32:37.354 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:25:19.466497+0000", 2023-06-03T20:32:37.354 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 1.1929999999999999e-05 2023-06-03T20:32:37.354 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:32:37.354 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:32:37.354 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "queued_for_pg", 2023-06-03T20:32:37.355 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:25:19.466506+0000", 2023-06-03T20:32:37.355 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 8.4659999999999999e-06 2023-06-03T20:32:37.355 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:32:37.355 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:32:37.355 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "reached_pg", 2023-06-03T20:32:37.355 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:25:19.479185+0000", 2023-06-03T20:32:37.355 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.012679671999999999 2023-06-03T20:32:37.356 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:32:37.356 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:32:37.356 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "started", 2023-06-03T20:32:37.356 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:25:19.479333+0000", 2023-06-03T20:32:37.356 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.00014733199999999999 2023-06-03T20:32:37.356 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:32:37.356 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:32:37.356 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "waiting for subops from 1,4", 2023-06-03T20:32:37.357 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:25:19.479435+0000", 2023-06-03T20:32:37.357 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.000102464 2023-06-03T20:32:37.357 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:32:37.357 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:32:37.357 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:32:37.357 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:25:19.486951+0000", 2023-06-03T20:32:37.357 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.0075158079999999997 2023-06-03T20:32:37.358 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:32:37.358 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:32:37.358 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "op_commit", 2023-06-03T20:32:37.358 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:25:19.486964+0000", 2023-06-03T20:32:37.358 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 1.2479e-05 2023-06-03T20:32:37.358 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:32:37.358 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:32:37.358 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:32:37.359 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:25:19.845939+0000", 2023-06-03T20:32:37.359 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.35897562999999999 2023-06-03T20:32:37.359 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:32:37.359 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:32:37.359 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "commit_sent", 2023-06-03T20:32:37.359 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:25:19.846065+0000", 2023-06-03T20:32:37.359 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.00012628800000000001 2023-06-03T20:32:37.360 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:32:37.360 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:32:37.360 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "done", 2023-06-03T20:32:37.360 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:25:19.853221+0000", 2023-06-03T20:32:37.360 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.0071552330000000004 2023-06-03T20:32:37.360 INFO:teuthology.orchestra.run.smithi089.stdout: } 2023-06-03T20:32:37.360 INFO:teuthology.orchestra.run.smithi089.stdout: ] 2023-06-03T20:32:37.360 INFO:teuthology.orchestra.run.smithi089.stdout: } 2023-06-03T20:32:37.361 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:32:37.361 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:32:37.361 INFO:teuthology.orchestra.run.smithi089.stdout: "description": "osd_op(client.46739.0:87943 14.d 14:b33778b4:::benchmark_data_smithi089_237281_object87942:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e289)", 2023-06-03T20:32:37.361 INFO:teuthology.orchestra.run.smithi089.stdout: "initiated_at": "2023-06-03T20:29:02.662631+0000", 2023-06-03T20:32:37.361 INFO:teuthology.orchestra.run.smithi089.stdout: "age": 214.634656904, 2023-06-03T20:32:37.361 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.30713804900000002, 2023-06-03T20:32:37.361 INFO:teuthology.orchestra.run.smithi089.stdout: "type_data": { 2023-06-03T20:32:37.362 INFO:teuthology.orchestra.run.smithi089.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-03T20:32:37.362 INFO:teuthology.orchestra.run.smithi089.stdout: "client_info": { 2023-06-03T20:32:37.362 INFO:teuthology.orchestra.run.smithi089.stdout: "client": "client.46739", 2023-06-03T20:32:37.362 INFO:teuthology.orchestra.run.smithi089.stdout: "client_addr": "172.21.15.89:0/2479974613", 2023-06-03T20:32:37.362 INFO:teuthology.orchestra.run.smithi089.stdout: "tid": 87943 2023-06-03T20:32:37.362 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:32:37.362 INFO:teuthology.orchestra.run.smithi089.stdout: "events": [ 2023-06-03T20:32:37.362 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:32:37.363 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "initiated", 2023-06-03T20:32:37.363 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:29:02.662631+0000", 2023-06-03T20:32:37.363 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0 2023-06-03T20:32:37.363 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:32:37.363 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:32:37.363 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "throttled", 2023-06-03T20:32:37.363 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:29:02.662631+0000", 2023-06-03T20:32:37.363 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0 2023-06-03T20:32:37.364 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:32:37.364 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:32:37.364 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "header_read", 2023-06-03T20:32:37.364 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:29:02.662629+0000", 2023-06-03T20:32:37.364 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 4294967295.9999981 2023-06-03T20:32:37.364 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:32:37.364 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:32:37.364 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "all_read", 2023-06-03T20:32:37.365 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:29:02.662666+0000", 2023-06-03T20:32:37.365 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 3.7072e-05 2023-06-03T20:32:37.365 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:32:37.365 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:32:37.365 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "dispatched", 2023-06-03T20:32:37.365 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:29:02.662676+0000", 2023-06-03T20:32:37.365 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 1.0148e-05 2023-06-03T20:32:37.366 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:32:37.366 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:32:37.366 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "queued_for_pg", 2023-06-03T20:32:37.366 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:29:02.662683+0000", 2023-06-03T20:32:37.366 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 6.64e-06 2023-06-03T20:32:37.366 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:32:37.366 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:32:37.367 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "reached_pg", 2023-06-03T20:32:37.367 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:29:02.662720+0000", 2023-06-03T20:32:37.367 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 3.7515000000000003e-05 2023-06-03T20:32:37.367 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:32:37.367 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:32:37.367 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "started", 2023-06-03T20:32:37.367 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:29:02.662890+0000", 2023-06-03T20:32:37.367 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.00017013099999999999 2023-06-03T20:32:37.368 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:32:37.368 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:32:37.368 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "waiting for subops from 4,5,6", 2023-06-03T20:32:37.368 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:29:02.663055+0000", 2023-06-03T20:32:37.368 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.00016487500000000001 2023-06-03T20:32:37.368 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:32:37.368 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:32:37.368 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:32:37.369 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:29:02.670263+0000", 2023-06-03T20:32:37.369 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.007207738 2023-06-03T20:32:37.369 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:32:37.369 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:32:37.369 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:32:37.369 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:29:02.670302+0000", 2023-06-03T20:32:37.369 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 3.9551999999999997e-05 2023-06-03T20:32:37.369 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:32:37.370 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:32:37.370 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:32:37.370 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:29:02.963722+0000", 2023-06-03T20:32:37.370 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.29341955800000002 2023-06-03T20:32:37.370 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:32:37.370 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:32:37.370 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "op_commit", 2023-06-03T20:32:37.371 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:29:02.963734+0000", 2023-06-03T20:32:37.371 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 1.2288e-05 2023-06-03T20:32:37.371 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:32:37.371 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:32:37.371 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "commit_sent", 2023-06-03T20:32:37.371 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:29:02.963781+0000", 2023-06-03T20:32:37.371 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 4.7131e-05 2023-06-03T20:32:37.371 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:32:37.372 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:32:37.372 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "done", 2023-06-03T20:32:37.372 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:29:02.969769+0000", 2023-06-03T20:32:37.372 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.0059872830000000004 2023-06-03T20:32:37.372 INFO:teuthology.orchestra.run.smithi089.stdout: } 2023-06-03T20:32:37.372 INFO:teuthology.orchestra.run.smithi089.stdout: ] 2023-06-03T20:32:37.372 INFO:teuthology.orchestra.run.smithi089.stdout: } 2023-06-03T20:32:37.372 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:32:37.373 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:32:37.373 INFO:teuthology.orchestra.run.smithi089.stdout: "description": "MOSDPGPushReply(14.d 289/287 [PushReplyOp(14:be1bc837:::benchmark_data_smithi089_237281_object3248:head)])", 2023-06-03T20:32:37.373 INFO:teuthology.orchestra.run.smithi089.stdout: "initiated_at": "2023-06-03T20:29:02.663332+0000", 2023-06-03T20:32:37.373 INFO:teuthology.orchestra.run.smithi089.stdout: "age": 214.63395580700001, 2023-06-03T20:32:37.373 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.300335295, 2023-06-03T20:32:37.373 INFO:teuthology.orchestra.run.smithi089.stdout: "type_data": { 2023-06-03T20:32:37.374 INFO:teuthology.orchestra.run.smithi089.stdout: "flag_point": "reached pg", 2023-06-03T20:32:37.374 INFO:teuthology.orchestra.run.smithi089.stdout: "events": [ 2023-06-03T20:32:37.374 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:32:37.374 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "initiated", 2023-06-03T20:32:37.374 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:29:02.663332+0000", 2023-06-03T20:32:37.374 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0 2023-06-03T20:32:37.374 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:32:37.374 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:32:37.375 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "throttled", 2023-06-03T20:32:37.375 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:29:02.663332+0000", 2023-06-03T20:32:37.375 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0 2023-06-03T20:32:37.375 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:32:37.375 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:32:37.375 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "header_read", 2023-06-03T20:32:37.375 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:29:02.663331+0000", 2023-06-03T20:32:37.376 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 4294967295.9999995 2023-06-03T20:32:37.376 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:32:37.376 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:32:37.376 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "all_read", 2023-06-03T20:32:37.376 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:29:02.663337+0000", 2023-06-03T20:32:37.376 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 5.6420000000000004e-06 2023-06-03T20:32:37.377 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:32:37.377 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:32:37.377 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "dispatched", 2023-06-03T20:32:37.377 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:29:02.663346+0000", 2023-06-03T20:32:37.377 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 9.2960000000000004e-06 2023-06-03T20:32:37.377 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:32:37.377 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:32:37.378 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "queued_for_pg", 2023-06-03T20:32:37.378 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:29:02.663353+0000", 2023-06-03T20:32:37.378 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 7.2860000000000002e-06 2023-06-03T20:32:37.378 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:32:37.378 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:32:37.378 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "reached_pg", 2023-06-03T20:32:37.378 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:29:02.670329+0000", 2023-06-03T20:32:37.379 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.0069760389999999999 2023-06-03T20:32:37.379 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:32:37.379 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:32:37.379 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "done", 2023-06-03T20:32:37.379 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:29:02.963667+0000", 2023-06-03T20:32:37.379 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.293337656 2023-06-03T20:32:37.379 INFO:teuthology.orchestra.run.smithi089.stdout: } 2023-06-03T20:32:37.379 INFO:teuthology.orchestra.run.smithi089.stdout: ] 2023-06-03T20:32:37.380 INFO:teuthology.orchestra.run.smithi089.stdout: } 2023-06-03T20:32:37.380 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:32:37.380 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:32:37.380 INFO:teuthology.orchestra.run.smithi089.stdout: "description": "osd_repop(client.46739.0:87939 14.6 e289/278 14:61b791d3:::benchmark_data_smithi089_237281_object87938:head v 289'5518, mlcod=289'5515)", 2023-06-03T20:32:37.380 INFO:teuthology.orchestra.run.smithi089.stdout: "initiated_at": "2023-06-03T20:29:02.663638+0000", 2023-06-03T20:32:37.380 INFO:teuthology.orchestra.run.smithi089.stdout: "age": 214.63364997400001, 2023-06-03T20:32:37.380 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.29686519700000003, 2023-06-03T20:32:37.381 INFO:teuthology.orchestra.run.smithi089.stdout: "type_data": { 2023-06-03T20:32:37.381 INFO:teuthology.orchestra.run.smithi089.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-03T20:32:37.381 INFO:teuthology.orchestra.run.smithi089.stdout: "events": [ 2023-06-03T20:32:37.381 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:32:37.381 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "initiated", 2023-06-03T20:32:37.381 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:29:02.663638+0000", 2023-06-03T20:32:37.381 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0 2023-06-03T20:32:37.381 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:32:37.382 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:32:37.382 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "throttled", 2023-06-03T20:32:37.382 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:29:02.663638+0000", 2023-06-03T20:32:37.382 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0 2023-06-03T20:32:37.382 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:32:37.382 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:32:37.382 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "header_read", 2023-06-03T20:32:37.382 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:29:02.663637+0000", 2023-06-03T20:32:37.383 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 4294967295.999999 2023-06-03T20:32:37.383 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:32:37.383 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:32:37.383 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "all_read", 2023-06-03T20:32:37.383 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:29:02.663663+0000", 2023-06-03T20:32:37.383 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 2.6497000000000001e-05 2023-06-03T20:32:37.383 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:32:37.384 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:32:37.384 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "dispatched", 2023-06-03T20:32:37.384 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:29:02.663675+0000", 2023-06-03T20:32:37.384 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 1.1712000000000001e-05 2023-06-03T20:32:37.384 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:32:37.384 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:32:37.384 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "queued_for_pg", 2023-06-03T20:32:37.384 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:29:02.663683+0000", 2023-06-03T20:32:37.385 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 8.1980000000000001e-06 2023-06-03T20:32:37.385 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:32:37.385 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:32:37.385 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "reached_pg", 2023-06-03T20:32:37.385 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:29:02.664988+0000", 2023-06-03T20:32:37.385 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.0013043969999999999 2023-06-03T20:32:37.385 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:32:37.385 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:32:37.386 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "started", 2023-06-03T20:32:37.386 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:29:02.665002+0000", 2023-06-03T20:32:37.386 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 1.4372000000000001e-05 2023-06-03T20:32:37.386 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:32:37.386 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:32:37.386 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "commit_sent", 2023-06-03T20:32:37.386 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:29:02.673920+0000", 2023-06-03T20:32:37.387 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.0089181729999999997 2023-06-03T20:32:37.387 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:32:37.387 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:32:37.387 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "done", 2023-06-03T20:32:37.387 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:29:02.960503+0000", 2023-06-03T20:32:37.387 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.28658263499999997 2023-06-03T20:32:37.387 INFO:teuthology.orchestra.run.smithi089.stdout: } 2023-06-03T20:32:37.387 INFO:teuthology.orchestra.run.smithi089.stdout: ] 2023-06-03T20:32:37.388 INFO:teuthology.orchestra.run.smithi089.stdout: } 2023-06-03T20:32:37.388 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:32:37.388 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:32:37.388 INFO:teuthology.orchestra.run.smithi089.stdout: "description": "osd_op(client.46739.0:87946 14.0 14:0337fe92:::benchmark_data_smithi089_237281_object87945:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e289)", 2023-06-03T20:32:37.388 INFO:teuthology.orchestra.run.smithi089.stdout: "initiated_at": "2023-06-03T20:29:02.665540+0000", 2023-06-03T20:32:37.388 INFO:teuthology.orchestra.run.smithi089.stdout: "age": 214.63174756500001, 2023-06-03T20:32:37.388 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.29777079899999997, 2023-06-03T20:32:37.389 INFO:teuthology.orchestra.run.smithi089.stdout: "type_data": { 2023-06-03T20:32:37.389 INFO:teuthology.orchestra.run.smithi089.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-03T20:32:37.389 INFO:teuthology.orchestra.run.smithi089.stdout: "client_info": { 2023-06-03T20:32:37.389 INFO:teuthology.orchestra.run.smithi089.stdout: "client": "client.46739", 2023-06-03T20:32:37.389 INFO:teuthology.orchestra.run.smithi089.stdout: "client_addr": "172.21.15.89:0/2479974613", 2023-06-03T20:32:37.389 INFO:teuthology.orchestra.run.smithi089.stdout: "tid": 87946 2023-06-03T20:32:37.389 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:32:37.389 INFO:teuthology.orchestra.run.smithi089.stdout: "events": [ 2023-06-03T20:32:37.390 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:32:37.390 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "initiated", 2023-06-03T20:32:37.390 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:29:02.665540+0000", 2023-06-03T20:32:37.390 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0 2023-06-03T20:32:37.390 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:32:37.390 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:32:37.390 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "throttled", 2023-06-03T20:32:37.390 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:29:02.665540+0000", 2023-06-03T20:32:37.391 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0 2023-06-03T20:32:37.391 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:32:37.391 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:32:37.391 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "header_read", 2023-06-03T20:32:37.391 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:29:02.665538+0000", 2023-06-03T20:32:37.392 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 4294967295.9999986 2023-06-03T20:32:37.392 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:32:37.392 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:32:37.392 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "all_read", 2023-06-03T20:32:37.393 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:29:02.665562+0000", 2023-06-03T20:32:37.393 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 2.3768e-05 2023-06-03T20:32:37.393 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:32:37.393 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:32:37.393 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "dispatched", 2023-06-03T20:32:37.393 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:29:02.665574+0000", 2023-06-03T20:32:37.394 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 1.1863e-05 2023-06-03T20:32:37.394 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:32:37.394 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:32:37.394 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "queued_for_pg", 2023-06-03T20:32:37.395 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:29:02.665582+0000", 2023-06-03T20:32:37.395 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 7.807e-06 2023-06-03T20:32:37.395 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:32:37.395 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:32:37.395 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "reached_pg", 2023-06-03T20:32:37.395 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:29:02.665626+0000", 2023-06-03T20:32:37.395 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 4.3621000000000001e-05 2023-06-03T20:32:37.396 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:32:37.396 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:32:37.396 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "started", 2023-06-03T20:32:37.396 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:29:02.665740+0000", 2023-06-03T20:32:37.396 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.00011466799999999999 2023-06-03T20:32:37.396 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:32:37.396 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:32:37.396 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "waiting for subops from 0,5", 2023-06-03T20:32:37.397 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:29:02.665835+0000", 2023-06-03T20:32:37.397 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 9.4642999999999995e-05 2023-06-03T20:32:37.397 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:32:37.397 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:32:37.397 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "op_commit", 2023-06-03T20:32:37.397 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:29:02.668533+0000", 2023-06-03T20:32:37.397 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.0026986039999999998 2023-06-03T20:32:37.397 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:32:37.398 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:32:37.398 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:32:37.398 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:29:02.671638+0000", 2023-06-03T20:32:37.398 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.0031041369999999999 2023-06-03T20:32:37.398 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:32:37.398 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:32:37.398 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:32:37.399 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:29:02.963210+0000", 2023-06-03T20:32:37.399 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.29157273299999997 2023-06-03T20:32:37.399 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:32:37.399 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:32:37.399 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "commit_sent", 2023-06-03T20:32:37.399 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:29:02.963266+0000", 2023-06-03T20:32:37.399 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 5.6070999999999999e-05 2023-06-03T20:32:37.400 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:32:37.400 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:32:37.400 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "done", 2023-06-03T20:32:37.400 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:29:02.963311+0000", 2023-06-03T20:32:37.400 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 4.4406000000000001e-05 2023-06-03T20:32:37.400 INFO:teuthology.orchestra.run.smithi089.stdout: } 2023-06-03T20:32:37.400 INFO:teuthology.orchestra.run.smithi089.stdout: ] 2023-06-03T20:32:37.401 INFO:teuthology.orchestra.run.smithi089.stdout: } 2023-06-03T20:32:37.401 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:32:37.401 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:32:37.401 INFO:teuthology.orchestra.run.smithi089.stdout: "description": "osd_repop_reply(client.46739.0:87943 14.d e289/287 ondisk, result = 0)", 2023-06-03T20:32:37.401 INFO:teuthology.orchestra.run.smithi089.stdout: "initiated_at": "2023-06-03T20:29:02.670784+0000", 2023-06-03T20:32:37.401 INFO:teuthology.orchestra.run.smithi089.stdout: "age": 214.62650333600001, 2023-06-03T20:32:37.401 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.29899203899999999, 2023-06-03T20:32:37.402 INFO:teuthology.orchestra.run.smithi089.stdout: "type_data": { 2023-06-03T20:32:37.402 INFO:teuthology.orchestra.run.smithi089.stdout: "flag_point": "started", 2023-06-03T20:32:37.402 INFO:teuthology.orchestra.run.smithi089.stdout: "events": [ 2023-06-03T20:32:37.402 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:32:37.402 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "initiated", 2023-06-03T20:32:37.402 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:29:02.670784+0000", 2023-06-03T20:32:37.402 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0 2023-06-03T20:32:37.402 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:32:37.403 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:32:37.403 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "throttled", 2023-06-03T20:32:37.403 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:29:02.670784+0000", 2023-06-03T20:32:37.403 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0 2023-06-03T20:32:37.403 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:32:37.403 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:32:37.403 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "header_read", 2023-06-03T20:32:37.404 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:29:02.670783+0000", 2023-06-03T20:32:37.404 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 4294967295.999999 2023-06-03T20:32:37.404 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:32:37.404 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:32:37.404 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "all_read", 2023-06-03T20:32:37.404 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:29:02.670790+0000", 2023-06-03T20:32:37.404 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 6.5200000000000003e-06 2023-06-03T20:32:37.404 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:32:37.405 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:32:37.405 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "dispatched", 2023-06-03T20:32:37.405 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:29:02.670800+0000", 2023-06-03T20:32:37.405 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 1.0278000000000001e-05 2023-06-03T20:32:37.405 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:32:37.405 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:32:37.405 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "queued_for_pg", 2023-06-03T20:32:37.405 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:29:02.670807+0000", 2023-06-03T20:32:37.406 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 7.0269999999999998e-06 2023-06-03T20:32:37.406 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:32:37.406 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:32:37.406 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "reached_pg", 2023-06-03T20:32:37.406 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:29:02.963704+0000", 2023-06-03T20:32:37.406 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.29289658000000002 2023-06-03T20:32:37.406 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:32:37.406 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:32:37.407 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "started", 2023-06-03T20:32:37.407 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:29:02.963714+0000", 2023-06-03T20:32:37.407 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 1.0212e-05 2023-06-03T20:32:37.407 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:32:37.407 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:32:37.407 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "done", 2023-06-03T20:32:37.407 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:29:02.969776+0000", 2023-06-03T20:32:37.408 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.0060623450000000002 2023-06-03T20:32:37.408 INFO:teuthology.orchestra.run.smithi089.stdout: } 2023-06-03T20:32:37.408 INFO:teuthology.orchestra.run.smithi089.stdout: ] 2023-06-03T20:32:37.408 INFO:teuthology.orchestra.run.smithi089.stdout: } 2023-06-03T20:32:37.408 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:32:37.408 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:32:37.408 INFO:teuthology.orchestra.run.smithi089.stdout: "description": "osd_op(client.46739.0:87955 14.d 14:b6000507:::benchmark_data_smithi089_237281_object87954:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e289)", 2023-06-03T20:32:37.408 INFO:teuthology.orchestra.run.smithi089.stdout: "initiated_at": "2023-06-03T20:29:02.676412+0000", 2023-06-03T20:32:37.409 INFO:teuthology.orchestra.run.smithi089.stdout: "age": 214.620875635, 2023-06-03T20:32:37.409 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.323104689, 2023-06-03T20:32:37.409 INFO:teuthology.orchestra.run.smithi089.stdout: "type_data": { 2023-06-03T20:32:37.409 INFO:teuthology.orchestra.run.smithi089.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-03T20:32:37.409 INFO:teuthology.orchestra.run.smithi089.stdout: "client_info": { 2023-06-03T20:32:37.409 INFO:teuthology.orchestra.run.smithi089.stdout: "client": "client.46739", 2023-06-03T20:32:37.409 INFO:teuthology.orchestra.run.smithi089.stdout: "client_addr": "172.21.15.89:0/2479974613", 2023-06-03T20:32:37.409 INFO:teuthology.orchestra.run.smithi089.stdout: "tid": 87955 2023-06-03T20:32:37.410 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:32:37.410 INFO:teuthology.orchestra.run.smithi089.stdout: "events": [ 2023-06-03T20:32:37.410 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:32:37.410 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "initiated", 2023-06-03T20:32:37.410 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:29:02.676412+0000", 2023-06-03T20:32:37.410 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0 2023-06-03T20:32:37.410 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:32:37.410 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:32:37.411 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "throttled", 2023-06-03T20:32:37.411 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:29:02.676412+0000", 2023-06-03T20:32:37.411 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0 2023-06-03T20:32:37.411 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:32:37.411 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:32:37.411 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "header_read", 2023-06-03T20:32:37.411 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:29:02.676410+0000", 2023-06-03T20:32:37.412 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 4294967295.9999976 2023-06-03T20:32:37.412 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:32:37.412 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:32:37.412 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "all_read", 2023-06-03T20:32:37.412 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:29:02.676435+0000", 2023-06-03T20:32:37.412 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 2.5273999999999999e-05 2023-06-03T20:32:37.412 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:32:37.413 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:32:37.413 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "dispatched", 2023-06-03T20:32:37.413 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:29:02.960295+0000", 2023-06-03T20:32:37.413 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.28385954299999999 2023-06-03T20:32:37.413 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:32:37.413 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:32:37.413 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "queued_for_pg", 2023-06-03T20:32:37.413 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:29:02.960308+0000", 2023-06-03T20:32:37.414 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 1.3494e-05 2023-06-03T20:32:37.414 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:32:37.414 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:32:37.414 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "reached_pg", 2023-06-03T20:32:37.414 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:29:02.969826+0000", 2023-06-03T20:32:37.414 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.0095175390000000002 2023-06-03T20:32:37.414 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:32:37.415 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:32:37.415 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "started", 2023-06-03T20:32:37.415 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:29:02.970019+0000", 2023-06-03T20:32:37.415 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.00019311099999999999 2023-06-03T20:32:37.415 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:32:37.415 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:32:37.415 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "waiting for subops from 4,5,6", 2023-06-03T20:32:37.416 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:29:02.970205+0000", 2023-06-03T20:32:37.416 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.000186143 2023-06-03T20:32:37.416 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:32:37.416 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:32:37.416 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "op_commit", 2023-06-03T20:32:37.416 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:29:02.976236+0000", 2023-06-03T20:32:37.416 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.0060307240000000003 2023-06-03T20:32:37.416 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:32:37.417 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:32:37.417 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:32:37.417 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:29:02.996004+0000", 2023-06-03T20:32:37.417 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.01976849 2023-06-03T20:32:37.417 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:32:37.417 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:32:37.417 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:32:37.418 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:29:02.996094+0000", 2023-06-03T20:32:37.418 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 8.9702000000000003e-05 2023-06-03T20:32:37.418 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:32:37.418 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:32:37.418 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:32:37.418 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:29:02.996130+0000", 2023-06-03T20:32:37.418 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 3.5902000000000003e-05 2023-06-03T20:32:37.418 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:32:37.419 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:32:37.419 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "commit_sent", 2023-06-03T20:32:37.419 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:29:02.996184+0000", 2023-06-03T20:32:37.419 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 5.3857000000000003e-05 2023-06-03T20:32:37.419 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:32:37.420 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:32:37.420 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "done", 2023-06-03T20:32:37.420 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:29:02.999517+0000", 2023-06-03T20:32:37.420 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.0033330980000000001 2023-06-03T20:32:37.420 INFO:teuthology.orchestra.run.smithi089.stdout: } 2023-06-03T20:32:37.420 INFO:teuthology.orchestra.run.smithi089.stdout: ] 2023-06-03T20:32:37.421 INFO:teuthology.orchestra.run.smithi089.stdout: } 2023-06-03T20:32:37.421 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:32:37.421 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:32:37.421 INFO:teuthology.orchestra.run.smithi089.stdout: "description": "osd_repop_reply(client.46739.0:87946 14.0 e289/278 ondisk, result = 0)", 2023-06-03T20:32:37.422 INFO:teuthology.orchestra.run.smithi089.stdout: "initiated_at": "2023-06-03T20:29:02.684698+0000", 2023-06-03T20:32:37.422 INFO:teuthology.orchestra.run.smithi089.stdout: "age": 214.612589566, 2023-06-03T20:32:37.422 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.278621326, 2023-06-03T20:32:37.422 INFO:teuthology.orchestra.run.smithi089.stdout: "type_data": { 2023-06-03T20:32:37.423 INFO:teuthology.orchestra.run.smithi089.stdout: "flag_point": "started", 2023-06-03T20:32:37.423 INFO:teuthology.orchestra.run.smithi089.stdout: "events": [ 2023-06-03T20:32:37.423 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:32:37.423 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "initiated", 2023-06-03T20:32:37.423 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:29:02.684698+0000", 2023-06-03T20:32:37.423 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0 2023-06-03T20:32:37.423 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:32:37.424 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:32:37.424 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "throttled", 2023-06-03T20:32:37.424 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:29:02.684698+0000", 2023-06-03T20:32:37.424 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0 2023-06-03T20:32:37.424 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:32:37.424 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:32:37.424 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "header_read", 2023-06-03T20:32:37.424 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:29:02.684697+0000", 2023-06-03T20:32:37.425 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 4294967295.999999 2023-06-03T20:32:37.425 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:32:37.425 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:32:37.425 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "all_read", 2023-06-03T20:32:37.425 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:29:02.684711+0000", 2023-06-03T20:32:37.425 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 1.3799e-05 2023-06-03T20:32:37.425 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:32:37.426 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:32:37.426 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "dispatched", 2023-06-03T20:32:37.426 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:29:02.960227+0000", 2023-06-03T20:32:37.426 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.27551685799999998 2023-06-03T20:32:37.426 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:32:37.426 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:32:37.426 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "queued_for_pg", 2023-06-03T20:32:37.426 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:29:02.960244+0000", 2023-06-03T20:32:37.427 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 1.6920999999999999e-05 2023-06-03T20:32:37.427 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:32:37.427 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:32:37.427 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "reached_pg", 2023-06-03T20:32:37.427 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:29:02.963191+0000", 2023-06-03T20:32:37.427 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.002946438 2023-06-03T20:32:37.427 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:32:37.428 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:32:37.428 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "started", 2023-06-03T20:32:37.428 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:29:02.963203+0000", 2023-06-03T20:32:37.428 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 1.1864e-05 2023-06-03T20:32:37.428 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:32:37.428 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:32:37.429 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "done", 2023-06-03T20:32:37.429 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:29:02.963319+0000", 2023-06-03T20:32:37.429 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.000116607 2023-06-03T20:32:37.429 INFO:teuthology.orchestra.run.smithi089.stdout: } 2023-06-03T20:32:37.429 INFO:teuthology.orchestra.run.smithi089.stdout: ] 2023-06-03T20:32:37.429 INFO:teuthology.orchestra.run.smithi089.stdout: } 2023-06-03T20:32:37.429 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:32:37.429 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:32:37.430 INFO:teuthology.orchestra.run.smithi089.stdout: "description": "osd_op(client.46739.0:100926 14.d 14:b737fdee:::benchmark_data_smithi089_237281_object2910:head [delete] snapc 0=[] ondisk+write+known_if_redirected e289)", 2023-06-03T20:32:37.430 INFO:teuthology.orchestra.run.smithi089.stdout: "initiated_at": "2023-06-03T20:29:20.748581+0000", 2023-06-03T20:32:37.430 INFO:teuthology.orchestra.run.smithi089.stdout: "age": 196.548706971, 2023-06-03T20:32:37.430 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.24510954099999999, 2023-06-03T20:32:37.430 INFO:teuthology.orchestra.run.smithi089.stdout: "type_data": { 2023-06-03T20:32:37.430 INFO:teuthology.orchestra.run.smithi089.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-03T20:32:37.431 INFO:teuthology.orchestra.run.smithi089.stdout: "client_info": { 2023-06-03T20:32:37.431 INFO:teuthology.orchestra.run.smithi089.stdout: "client": "client.46739", 2023-06-03T20:32:37.431 INFO:teuthology.orchestra.run.smithi089.stdout: "client_addr": "172.21.15.89:0/2479974613", 2023-06-03T20:32:37.431 INFO:teuthology.orchestra.run.smithi089.stdout: "tid": 100926 2023-06-03T20:32:37.432 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:32:37.432 INFO:teuthology.orchestra.run.smithi089.stdout: "events": [ 2023-06-03T20:32:37.432 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:32:37.432 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "initiated", 2023-06-03T20:32:37.432 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:29:20.748581+0000", 2023-06-03T20:32:37.433 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0 2023-06-03T20:32:37.433 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:32:37.433 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:32:37.433 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "throttled", 2023-06-03T20:32:37.433 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:29:20.748581+0000", 2023-06-03T20:32:37.433 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0 2023-06-03T20:32:37.433 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:32:37.434 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:32:37.434 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "header_read", 2023-06-03T20:32:37.434 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:29:20.748579+0000", 2023-06-03T20:32:37.434 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 4294967295.9999981 2023-06-03T20:32:37.435 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:32:37.435 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:32:37.435 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "all_read", 2023-06-03T20:32:37.435 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:29:20.748588+0000", 2023-06-03T20:32:37.435 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 9.1940000000000004e-06 2023-06-03T20:32:37.435 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:32:37.435 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:32:37.436 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "dispatched", 2023-06-03T20:32:37.436 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:29:20.748602+0000", 2023-06-03T20:32:37.436 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 1.3586e-05 2023-06-03T20:32:37.436 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:32:37.436 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:32:37.436 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "queued_for_pg", 2023-06-03T20:32:37.436 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:29:20.748611+0000", 2023-06-03T20:32:37.437 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 9.7920000000000004e-06 2023-06-03T20:32:37.437 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:32:37.437 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:32:37.437 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "reached_pg", 2023-06-03T20:32:37.437 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:29:20.774399+0000", 2023-06-03T20:32:37.437 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.025788017 2023-06-03T20:32:37.437 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:32:37.438 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:32:37.438 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "started", 2023-06-03T20:32:37.438 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:29:20.774536+0000", 2023-06-03T20:32:37.438 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.00013619399999999999 2023-06-03T20:32:37.438 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:32:37.438 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:32:37.438 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "waiting for subops from 4,5,6", 2023-06-03T20:32:37.438 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:29:20.774658+0000", 2023-06-03T20:32:37.439 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.00012221399999999999 2023-06-03T20:32:37.439 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:32:37.439 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:32:37.439 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:32:37.439 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:29:20.989854+0000", 2023-06-03T20:32:37.439 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.215196099 2023-06-03T20:32:37.439 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:32:37.440 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:32:37.440 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:32:37.440 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:29:20.989924+0000", 2023-06-03T20:32:37.440 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 6.9998000000000003e-05 2023-06-03T20:32:37.440 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:32:37.440 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:32:37.440 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "op_commit", 2023-06-03T20:32:37.441 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:29:20.989931+0000", 2023-06-03T20:32:37.441 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 7.5390000000000002e-06 2023-06-03T20:32:37.441 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:32:37.441 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:32:37.441 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:32:37.441 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:29:20.990011+0000", 2023-06-03T20:32:37.441 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 7.9857000000000005e-05 2023-06-03T20:32:37.441 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:32:37.442 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:32:37.442 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "commit_sent", 2023-06-03T20:32:37.442 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:29:20.990116+0000", 2023-06-03T20:32:37.442 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.00010428400000000001 2023-06-03T20:32:37.442 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:32:37.442 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:32:37.442 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "done", 2023-06-03T20:32:37.443 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:29:20.993690+0000", 2023-06-03T20:32:37.443 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.003574621 2023-06-03T20:32:37.443 INFO:teuthology.orchestra.run.smithi089.stdout: } 2023-06-03T20:32:37.443 INFO:teuthology.orchestra.run.smithi089.stdout: ] 2023-06-03T20:32:37.443 INFO:teuthology.orchestra.run.smithi089.stdout: } 2023-06-03T20:32:37.443 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:32:37.443 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:32:37.444 INFO:teuthology.orchestra.run.smithi089.stdout: "description": "MOSDPGPushReply(14.d 289/287 [PushReplyOp(14:b97d6e8e:::benchmark_data_smithi089_237281_object9453:head)])", 2023-06-03T20:32:37.444 INFO:teuthology.orchestra.run.smithi089.stdout: "initiated_at": "2023-06-03T20:29:20.756956+0000", 2023-06-03T20:32:37.444 INFO:teuthology.orchestra.run.smithi089.stdout: "age": 196.54033179800001, 2023-06-03T20:32:37.444 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.232790476, 2023-06-03T20:32:37.444 INFO:teuthology.orchestra.run.smithi089.stdout: "type_data": { 2023-06-03T20:32:37.444 INFO:teuthology.orchestra.run.smithi089.stdout: "flag_point": "reached pg", 2023-06-03T20:32:37.444 INFO:teuthology.orchestra.run.smithi089.stdout: "events": [ 2023-06-03T20:32:37.444 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:32:37.445 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "initiated", 2023-06-03T20:32:37.445 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:29:20.756956+0000", 2023-06-03T20:32:37.445 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0 2023-06-03T20:32:37.445 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:32:37.445 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:32:37.445 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "throttled", 2023-06-03T20:32:37.445 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:29:20.756956+0000", 2023-06-03T20:32:37.445 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0 2023-06-03T20:32:37.446 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:32:37.446 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:32:37.446 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "header_read", 2023-06-03T20:32:37.446 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:29:20.756956+0000", 2023-06-03T20:32:37.446 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 4294967296 2023-06-03T20:32:37.446 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:32:37.446 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:32:37.447 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "all_read", 2023-06-03T20:32:37.447 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:29:20.756959+0000", 2023-06-03T20:32:37.447 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 3.6780000000000002e-06 2023-06-03T20:32:37.447 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:32:37.447 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:32:37.447 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "dispatched", 2023-06-03T20:32:37.447 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:29:20.756967+0000", 2023-06-03T20:32:37.448 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 7.5490000000000001e-06 2023-06-03T20:32:37.448 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:32:37.448 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:32:37.448 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "queued_for_pg", 2023-06-03T20:32:37.448 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:29:20.756972+0000", 2023-06-03T20:32:37.448 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 5.1229999999999999e-06 2023-06-03T20:32:37.448 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:32:37.448 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:32:37.449 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "reached_pg", 2023-06-03T20:32:37.449 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:29:20.784763+0000", 2023-06-03T20:32:37.449 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.027791204999999999 2023-06-03T20:32:37.449 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:32:37.449 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:32:37.449 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "done", 2023-06-03T20:32:37.449 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:29:20.989746+0000", 2023-06-03T20:32:37.449 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.20498309100000001 2023-06-03T20:32:37.450 INFO:teuthology.orchestra.run.smithi089.stdout: } 2023-06-03T20:32:37.450 INFO:teuthology.orchestra.run.smithi089.stdout: ] 2023-06-03T20:32:37.450 INFO:teuthology.orchestra.run.smithi089.stdout: } 2023-06-03T20:32:37.450 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:32:37.450 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:32:37.450 INFO:teuthology.orchestra.run.smithi089.stdout: "description": "osd_op(client.46739.0:172172 14.0 14:0bdc6b8b:::benchmark_data_smithi089_237281_object74156:head [delete] snapc 0=[] ondisk+write+known_if_redirected e293)", 2023-06-03T20:32:37.450 INFO:teuthology.orchestra.run.smithi089.stdout: "initiated_at": "2023-06-03T20:31:17.963519+0000", 2023-06-03T20:32:37.451 INFO:teuthology.orchestra.run.smithi089.stdout: "age": 79.333768884999998, 2023-06-03T20:32:37.451 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.28548722900000001, 2023-06-03T20:32:37.451 INFO:teuthology.orchestra.run.smithi089.stdout: "type_data": { 2023-06-03T20:32:37.451 INFO:teuthology.orchestra.run.smithi089.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-03T20:32:37.451 INFO:teuthology.orchestra.run.smithi089.stdout: "client_info": { 2023-06-03T20:32:37.451 INFO:teuthology.orchestra.run.smithi089.stdout: "client": "client.46739", 2023-06-03T20:32:37.451 INFO:teuthology.orchestra.run.smithi089.stdout: "client_addr": "172.21.15.89:0/2479974613", 2023-06-03T20:32:37.451 INFO:teuthology.orchestra.run.smithi089.stdout: "tid": 172172 2023-06-03T20:32:37.452 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:32:37.452 INFO:teuthology.orchestra.run.smithi089.stdout: "events": [ 2023-06-03T20:32:37.452 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:32:37.452 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "initiated", 2023-06-03T20:32:37.452 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:31:17.963519+0000", 2023-06-03T20:32:37.452 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0 2023-06-03T20:32:37.452 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:32:37.453 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:32:37.453 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "throttled", 2023-06-03T20:32:37.453 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:31:17.963519+0000", 2023-06-03T20:32:37.453 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0 2023-06-03T20:32:37.453 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:32:37.453 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:32:37.453 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "header_read", 2023-06-03T20:32:37.453 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:31:17.963517+0000", 2023-06-03T20:32:37.454 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 4294967295.9999986 2023-06-03T20:32:37.454 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:32:37.454 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:32:37.454 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "all_read", 2023-06-03T20:32:37.454 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:31:17.963523+0000", 2023-06-03T20:32:37.454 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 6.2600000000000002e-06 2023-06-03T20:32:37.454 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:32:37.454 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:32:37.455 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "dispatched", 2023-06-03T20:32:37.455 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:31:17.963534+0000", 2023-06-03T20:32:37.455 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 1.0613e-05 2023-06-03T20:32:37.455 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:32:37.455 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:32:37.455 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "queued_for_pg", 2023-06-03T20:32:37.455 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:31:17.963541+0000", 2023-06-03T20:32:37.456 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 7.2590000000000003e-06 2023-06-03T20:32:37.456 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:32:37.456 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:32:37.456 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "reached_pg", 2023-06-03T20:32:37.456 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:31:17.978954+0000", 2023-06-03T20:32:37.456 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.015412492999999999 2023-06-03T20:32:37.456 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:32:37.456 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:32:37.457 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "started", 2023-06-03T20:32:37.457 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:31:17.979083+0000", 2023-06-03T20:32:37.457 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.00012913000000000001 2023-06-03T20:32:37.457 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:32:37.457 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:32:37.457 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "waiting for subops from 0,5", 2023-06-03T20:32:37.457 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:31:17.979201+0000", 2023-06-03T20:32:37.458 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.00011824099999999999 2023-06-03T20:32:37.458 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:32:37.458 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:32:37.458 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "op_commit", 2023-06-03T20:32:37.458 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:31:17.985463+0000", 2023-06-03T20:32:37.458 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.0062619559999999999 2023-06-03T20:32:37.458 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:32:37.458 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:32:37.459 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:32:37.459 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:31:17.991154+0000", 2023-06-03T20:32:37.459 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.0056914670000000004 2023-06-03T20:32:37.459 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:32:37.459 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:32:37.459 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:32:37.459 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:31:18.248908+0000", 2023-06-03T20:32:37.460 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.25775378999999998 2023-06-03T20:32:37.460 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:32:37.460 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:32:37.460 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "commit_sent", 2023-06-03T20:32:37.460 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:31:18.248963+0000", 2023-06-03T20:32:37.460 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 5.4568000000000003e-05 2023-06-03T20:32:37.460 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:32:37.460 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:32:37.461 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "done", 2023-06-03T20:32:37.461 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:31:18.249006+0000", 2023-06-03T20:32:37.461 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 4.3118000000000003e-05 2023-06-03T20:32:37.461 INFO:teuthology.orchestra.run.smithi089.stdout: } 2023-06-03T20:32:37.461 INFO:teuthology.orchestra.run.smithi089.stdout: ] 2023-06-03T20:32:37.461 INFO:teuthology.orchestra.run.smithi089.stdout: } 2023-06-03T20:32:37.461 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:32:37.462 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:32:37.462 INFO:teuthology.orchestra.run.smithi089.stdout: "description": "osd_op(client.46739.0:172183 14.0 14:098d2d59:::benchmark_data_smithi089_237281_object74167:head [delete] snapc 0=[] ondisk+write+known_if_redirected e293)", 2023-06-03T20:32:37.462 INFO:teuthology.orchestra.run.smithi089.stdout: "initiated_at": "2023-06-03T20:31:17.985359+0000", 2023-06-03T20:32:37.462 INFO:teuthology.orchestra.run.smithi089.stdout: "age": 79.311929028999998, 2023-06-03T20:32:37.462 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.26376130800000003, 2023-06-03T20:32:37.462 INFO:teuthology.orchestra.run.smithi089.stdout: "type_data": { 2023-06-03T20:32:37.462 INFO:teuthology.orchestra.run.smithi089.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-03T20:32:37.462 INFO:teuthology.orchestra.run.smithi089.stdout: "client_info": { 2023-06-03T20:32:37.463 INFO:teuthology.orchestra.run.smithi089.stdout: "client": "client.46739", 2023-06-03T20:32:37.463 INFO:teuthology.orchestra.run.smithi089.stdout: "client_addr": "172.21.15.89:0/2479974613", 2023-06-03T20:32:37.463 INFO:teuthology.orchestra.run.smithi089.stdout: "tid": 172183 2023-06-03T20:32:37.463 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:32:37.463 INFO:teuthology.orchestra.run.smithi089.stdout: "events": [ 2023-06-03T20:32:37.463 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:32:37.463 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "initiated", 2023-06-03T20:32:37.463 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:31:17.985359+0000", 2023-06-03T20:32:37.464 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0 2023-06-03T20:32:37.464 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:32:37.464 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:32:37.464 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "throttled", 2023-06-03T20:32:37.464 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:31:17.985359+0000", 2023-06-03T20:32:37.464 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0 2023-06-03T20:32:37.464 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:32:37.465 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:32:37.465 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "header_read", 2023-06-03T20:32:37.465 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:31:17.985357+0000", 2023-06-03T20:32:37.465 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 4294967295.9999986 2023-06-03T20:32:37.465 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:32:37.465 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:32:37.465 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "all_read", 2023-06-03T20:32:37.465 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:31:17.985364+0000", 2023-06-03T20:32:37.466 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 6.4790000000000001e-06 2023-06-03T20:32:37.466 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:32:37.466 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:32:37.466 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "dispatched", 2023-06-03T20:32:37.466 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:31:17.985375+0000", 2023-06-03T20:32:37.466 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 1.1100999999999999e-05 2023-06-03T20:32:37.466 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:32:37.466 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:32:37.467 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "queued_for_pg", 2023-06-03T20:32:37.467 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:31:17.985383+0000", 2023-06-03T20:32:37.467 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 8.0209999999999999e-06 2023-06-03T20:32:37.467 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:32:37.467 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:32:37.467 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "reached_pg", 2023-06-03T20:32:37.467 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:31:17.985483+0000", 2023-06-03T20:32:37.468 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.000100079 2023-06-03T20:32:37.468 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:32:37.468 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:32:37.468 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "started", 2023-06-03T20:32:37.468 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:31:17.985603+0000", 2023-06-03T20:32:37.468 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.00011999500000000001 2023-06-03T20:32:37.468 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:32:37.469 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:32:37.469 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "waiting for subops from 0,5", 2023-06-03T20:32:37.469 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:31:17.985726+0000", 2023-06-03T20:32:37.469 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.00012360600000000001 2023-06-03T20:32:37.469 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:32:37.469 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:32:37.469 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:32:37.469 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:31:17.995631+0000", 2023-06-03T20:32:37.470 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.0099047500000000004 2023-06-03T20:32:37.470 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:32:37.470 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:32:37.470 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "op_commit", 2023-06-03T20:32:37.470 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:31:17.995656+0000", 2023-06-03T20:32:37.470 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 2.5307e-05 2023-06-03T20:32:37.470 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:32:37.471 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:32:37.471 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:32:37.471 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:31:18.249044+0000", 2023-06-03T20:32:37.471 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.25338780500000002 2023-06-03T20:32:37.471 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:32:37.471 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:32:37.471 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "commit_sent", 2023-06-03T20:32:37.471 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:31:18.249082+0000", 2023-06-03T20:32:37.472 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 3.7889000000000003e-05 2023-06-03T20:32:37.472 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:32:37.472 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:32:37.472 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "done", 2023-06-03T20:32:37.472 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:31:18.249120+0000", 2023-06-03T20:32:37.472 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 3.7638000000000001e-05 2023-06-03T20:32:37.472 INFO:teuthology.orchestra.run.smithi089.stdout: } 2023-06-03T20:32:37.473 INFO:teuthology.orchestra.run.smithi089.stdout: ] 2023-06-03T20:32:37.473 INFO:teuthology.orchestra.run.smithi089.stdout: } 2023-06-03T20:32:37.473 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:32:37.473 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:32:37.473 INFO:teuthology.orchestra.run.smithi089.stdout: "description": "osd_op(client.46739.0:172191 14.0 14:054c8c24:::benchmark_data_smithi089_237281_object74175:head [delete] snapc 0=[] ondisk+write+known_if_redirected e293)", 2023-06-03T20:32:37.473 INFO:teuthology.orchestra.run.smithi089.stdout: "initiated_at": "2023-06-03T20:31:18.001316+0000", 2023-06-03T20:32:37.473 INFO:teuthology.orchestra.run.smithi089.stdout: "age": 79.295971985999998, 2023-06-03T20:32:37.473 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.24792256400000001, 2023-06-03T20:32:37.474 INFO:teuthology.orchestra.run.smithi089.stdout: "type_data": { 2023-06-03T20:32:37.474 INFO:teuthology.orchestra.run.smithi089.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-03T20:32:37.474 INFO:teuthology.orchestra.run.smithi089.stdout: "client_info": { 2023-06-03T20:32:37.474 INFO:teuthology.orchestra.run.smithi089.stdout: "client": "client.46739", 2023-06-03T20:32:37.474 INFO:teuthology.orchestra.run.smithi089.stdout: "client_addr": "172.21.15.89:0/2479974613", 2023-06-03T20:32:37.474 INFO:teuthology.orchestra.run.smithi089.stdout: "tid": 172191 2023-06-03T20:32:37.474 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:32:37.474 INFO:teuthology.orchestra.run.smithi089.stdout: "events": [ 2023-06-03T20:32:37.475 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:32:37.475 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "initiated", 2023-06-03T20:32:37.475 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:31:18.001316+0000", 2023-06-03T20:32:37.475 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0 2023-06-03T20:32:37.475 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:32:37.475 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:32:37.475 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "throttled", 2023-06-03T20:32:37.476 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:31:18.001316+0000", 2023-06-03T20:32:37.476 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0 2023-06-03T20:32:37.476 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:32:37.476 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:32:37.476 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "header_read", 2023-06-03T20:32:37.476 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:31:18.001314+0000", 2023-06-03T20:32:37.476 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 4294967295.9999981 2023-06-03T20:32:37.476 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:32:37.477 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:32:37.477 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "all_read", 2023-06-03T20:32:37.477 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:31:18.001321+0000", 2023-06-03T20:32:37.477 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 7.17e-06 2023-06-03T20:32:37.477 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:32:37.477 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:32:37.477 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "dispatched", 2023-06-03T20:32:37.477 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:31:18.001334+0000", 2023-06-03T20:32:37.478 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 1.2771e-05 2023-06-03T20:32:37.478 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:32:37.478 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:32:37.478 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "queued_for_pg", 2023-06-03T20:32:37.478 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:31:18.001342+0000", 2023-06-03T20:32:37.478 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 8.2679999999999994e-06 2023-06-03T20:32:37.478 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:32:37.478 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:32:37.479 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "reached_pg", 2023-06-03T20:32:37.479 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:31:18.001396+0000", 2023-06-03T20:32:37.479 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 5.3718000000000003e-05 2023-06-03T20:32:37.479 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:32:37.479 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:32:37.479 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "started", 2023-06-03T20:32:37.479 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:31:18.001498+0000", 2023-06-03T20:32:37.480 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.000102407 2023-06-03T20:32:37.480 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:32:37.480 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:32:37.480 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "waiting for subops from 0,5", 2023-06-03T20:32:37.480 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:31:18.001584+0000", 2023-06-03T20:32:37.480 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 8.6259000000000002e-05 2023-06-03T20:32:37.480 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:32:37.480 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:32:37.481 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:32:37.481 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:31:18.237990+0000", 2023-06-03T20:32:37.481 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.23640583800000001 2023-06-03T20:32:37.481 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:32:37.481 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:32:37.481 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "op_commit", 2023-06-03T20:32:37.481 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:31:18.237997+0000", 2023-06-03T20:32:37.481 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 7.1330000000000001e-06 2023-06-03T20:32:37.482 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:32:37.482 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:32:37.482 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:32:37.482 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:31:18.249159+0000", 2023-06-03T20:32:37.482 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.011161614 2023-06-03T20:32:37.482 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:32:37.482 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:32:37.482 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "commit_sent", 2023-06-03T20:32:37.483 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:31:18.249200+0000", 2023-06-03T20:32:37.483 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 4.0896999999999998e-05 2023-06-03T20:32:37.483 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:32:37.483 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:32:37.483 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "done", 2023-06-03T20:32:37.483 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:31:18.249238+0000", 2023-06-03T20:32:37.483 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 3.8279999999999999e-05 2023-06-03T20:32:37.484 INFO:teuthology.orchestra.run.smithi089.stdout: } 2023-06-03T20:32:37.484 INFO:teuthology.orchestra.run.smithi089.stdout: ] 2023-06-03T20:32:37.484 INFO:teuthology.orchestra.run.smithi089.stdout: } 2023-06-03T20:32:37.484 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:32:37.484 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:32:37.484 INFO:teuthology.orchestra.run.smithi089.stdout: "description": "osd_op(client.46739.0:172193 14.0 14:0a87f78c:::benchmark_data_smithi089_237281_object74177:head [delete] snapc 0=[] ondisk+write+known_if_redirected e293)", 2023-06-03T20:32:37.484 INFO:teuthology.orchestra.run.smithi089.stdout: "initiated_at": "2023-06-03T20:31:18.001479+0000", 2023-06-03T20:32:37.485 INFO:teuthology.orchestra.run.smithi089.stdout: "age": 79.295808586000007, 2023-06-03T20:32:37.485 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.25353188300000001, 2023-06-03T20:32:37.485 INFO:teuthology.orchestra.run.smithi089.stdout: "type_data": { 2023-06-03T20:32:37.485 INFO:teuthology.orchestra.run.smithi089.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-03T20:32:37.485 INFO:teuthology.orchestra.run.smithi089.stdout: "client_info": { 2023-06-03T20:32:37.485 INFO:teuthology.orchestra.run.smithi089.stdout: "client": "client.46739", 2023-06-03T20:32:37.485 INFO:teuthology.orchestra.run.smithi089.stdout: "client_addr": "172.21.15.89:0/2479974613", 2023-06-03T20:32:37.485 INFO:teuthology.orchestra.run.smithi089.stdout: "tid": 172193 2023-06-03T20:32:37.486 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:32:37.486 INFO:teuthology.orchestra.run.smithi089.stdout: "events": [ 2023-06-03T20:32:37.486 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:32:37.486 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "initiated", 2023-06-03T20:32:37.486 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:31:18.001479+0000", 2023-06-03T20:32:37.486 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0 2023-06-03T20:32:37.486 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:32:37.486 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:32:37.487 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "throttled", 2023-06-03T20:32:37.487 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:31:18.001479+0000", 2023-06-03T20:32:37.487 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0 2023-06-03T20:32:37.487 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:32:37.487 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:32:37.487 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "header_read", 2023-06-03T20:32:37.487 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:31:18.001478+0000", 2023-06-03T20:32:37.488 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 4294967295.9999995 2023-06-03T20:32:37.488 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:32:37.488 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:32:37.488 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "all_read", 2023-06-03T20:32:37.488 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:31:18.001482+0000", 2023-06-03T20:32:37.488 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 3.9589999999999997e-06 2023-06-03T20:32:37.488 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:32:37.488 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:32:37.489 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "dispatched", 2023-06-03T20:32:37.489 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:31:18.001490+0000", 2023-06-03T20:32:37.489 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 7.3579999999999997e-06 2023-06-03T20:32:37.489 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:32:37.489 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:32:37.489 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "queued_for_pg", 2023-06-03T20:32:37.489 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:31:18.001494+0000", 2023-06-03T20:32:37.490 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 4.2520000000000001e-06 2023-06-03T20:32:37.490 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:32:37.490 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:32:37.490 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "reached_pg", 2023-06-03T20:32:37.490 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:31:18.005709+0000", 2023-06-03T20:32:37.490 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.0042152190000000001 2023-06-03T20:32:37.490 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:32:37.490 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:32:37.491 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "started", 2023-06-03T20:32:37.491 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:31:18.005791+0000", 2023-06-03T20:32:37.491 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 8.2204000000000001e-05 2023-06-03T20:32:37.491 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:32:37.491 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:32:37.491 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "waiting for subops from 0,5", 2023-06-03T20:32:37.491 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:31:18.005906+0000", 2023-06-03T20:32:37.491 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.000114762 2023-06-03T20:32:37.492 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:32:37.492 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:32:37.492 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "op_commit", 2023-06-03T20:32:37.492 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:31:18.238116+0000", 2023-06-03T20:32:37.492 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.232209741 2023-06-03T20:32:37.492 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:32:37.492 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:32:37.493 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:32:37.493 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:31:18.238298+0000", 2023-06-03T20:32:37.493 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.000182015 2023-06-03T20:32:37.493 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:32:37.493 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:32:37.493 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:32:37.493 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:31:18.254928+0000", 2023-06-03T20:32:37.494 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.016630419 2023-06-03T20:32:37.494 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:32:37.494 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:32:37.494 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "commit_sent", 2023-06-03T20:32:37.494 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:31:18.254974+0000", 2023-06-03T20:32:37.494 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 4.5705999999999999e-05 2023-06-03T20:32:37.494 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:32:37.494 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:32:37.495 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "done", 2023-06-03T20:32:37.495 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:31:18.255011+0000", 2023-06-03T20:32:37.495 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 3.6872000000000002e-05 2023-06-03T20:32:37.495 INFO:teuthology.orchestra.run.smithi089.stdout: } 2023-06-03T20:32:37.495 INFO:teuthology.orchestra.run.smithi089.stdout: ] 2023-06-03T20:32:37.495 INFO:teuthology.orchestra.run.smithi089.stdout: } 2023-06-03T20:32:37.495 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:32:37.495 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:32:37.496 INFO:teuthology.orchestra.run.smithi089.stdout: "description": "osd_repop_reply(client.46739.0:172191 14.0 e293/278 ondisk, result = 0)", 2023-06-03T20:32:37.496 INFO:teuthology.orchestra.run.smithi089.stdout: "initiated_at": "2023-06-03T20:31:18.012790+0000", 2023-06-03T20:32:37.496 INFO:teuthology.orchestra.run.smithi089.stdout: "age": 79.284497880000004, 2023-06-03T20:32:37.496 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.225208453, 2023-06-03T20:32:37.496 INFO:teuthology.orchestra.run.smithi089.stdout: "type_data": { 2023-06-03T20:32:37.496 INFO:teuthology.orchestra.run.smithi089.stdout: "flag_point": "started", 2023-06-03T20:32:37.496 INFO:teuthology.orchestra.run.smithi089.stdout: "events": [ 2023-06-03T20:32:37.497 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:32:37.497 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "initiated", 2023-06-03T20:32:37.497 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:31:18.012790+0000", 2023-06-03T20:32:37.497 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0 2023-06-03T20:32:37.497 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:32:37.497 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:32:37.497 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "throttled", 2023-06-03T20:32:37.497 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:31:18.012790+0000", 2023-06-03T20:32:37.498 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0 2023-06-03T20:32:37.498 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:32:37.498 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:32:37.498 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "header_read", 2023-06-03T20:32:37.498 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:31:18.012789+0000", 2023-06-03T20:32:37.498 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 4294967295.999999 2023-06-03T20:32:37.498 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:32:37.498 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:32:37.499 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "all_read", 2023-06-03T20:32:37.499 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:31:18.012795+0000", 2023-06-03T20:32:37.499 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 6.032e-06 2023-06-03T20:32:37.499 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:32:37.499 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:32:37.499 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "dispatched", 2023-06-03T20:32:37.499 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:31:18.232156+0000", 2023-06-03T20:32:37.500 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.219361264 2023-06-03T20:32:37.500 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:32:37.500 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:32:37.500 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "queued_for_pg", 2023-06-03T20:32:37.500 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:31:18.232167+0000", 2023-06-03T20:32:37.500 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 1.0891e-05 2023-06-03T20:32:37.500 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:32:37.500 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:32:37.501 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "reached_pg", 2023-06-03T20:32:37.501 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:31:18.237977+0000", 2023-06-03T20:32:37.501 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.0058104790000000003 2023-06-03T20:32:37.501 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:32:37.501 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:32:37.501 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "started", 2023-06-03T20:32:37.501 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:31:18.237984+0000", 2023-06-03T20:32:37.501 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 6.888e-06 2023-06-03T20:32:37.502 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:32:37.502 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:32:37.502 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "done", 2023-06-03T20:32:37.502 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:31:18.237998+0000", 2023-06-03T20:32:37.502 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 1.3916e-05 2023-06-03T20:32:37.502 INFO:teuthology.orchestra.run.smithi089.stdout: } 2023-06-03T20:32:37.502 INFO:teuthology.orchestra.run.smithi089.stdout: ] 2023-06-03T20:32:37.502 INFO:teuthology.orchestra.run.smithi089.stdout: } 2023-06-03T20:32:37.503 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:32:37.503 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:32:37.503 INFO:teuthology.orchestra.run.smithi089.stdout: "description": "osd_op(client.46739.0:177009 14.0 14:0ddadc32:::benchmark_data_smithi089_237281_object78993:head [delete] snapc 0=[] ondisk+write+known_if_redirected e293)", 2023-06-03T20:32:37.503 INFO:teuthology.orchestra.run.smithi089.stdout: "initiated_at": "2023-06-03T20:31:26.251670+0000", 2023-06-03T20:32:37.503 INFO:teuthology.orchestra.run.smithi089.stdout: "age": 71.045617710000002, 2023-06-03T20:32:37.503 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.26966959400000001, 2023-06-03T20:32:37.503 INFO:teuthology.orchestra.run.smithi089.stdout: "type_data": { 2023-06-03T20:32:37.504 INFO:teuthology.orchestra.run.smithi089.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-03T20:32:37.504 INFO:teuthology.orchestra.run.smithi089.stdout: "client_info": { 2023-06-03T20:32:37.504 INFO:teuthology.orchestra.run.smithi089.stdout: "client": "client.46739", 2023-06-03T20:32:37.504 INFO:teuthology.orchestra.run.smithi089.stdout: "client_addr": "172.21.15.89:0/2479974613", 2023-06-03T20:32:37.504 INFO:teuthology.orchestra.run.smithi089.stdout: "tid": 177009 2023-06-03T20:32:37.504 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:32:37.504 INFO:teuthology.orchestra.run.smithi089.stdout: "events": [ 2023-06-03T20:32:37.504 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:32:37.505 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "initiated", 2023-06-03T20:32:37.505 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:31:26.251670+0000", 2023-06-03T20:32:37.505 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0 2023-06-03T20:32:37.505 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:32:37.505 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:32:37.505 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "throttled", 2023-06-03T20:32:37.505 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:31:26.251670+0000", 2023-06-03T20:32:37.506 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0 2023-06-03T20:32:37.506 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:32:37.506 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:32:37.506 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "header_read", 2023-06-03T20:32:37.506 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:31:26.251668+0000", 2023-06-03T20:32:37.506 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 4294967295.9999981 2023-06-03T20:32:37.506 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:32:37.506 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:32:37.507 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "all_read", 2023-06-03T20:32:37.507 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:31:26.251677+0000", 2023-06-03T20:32:37.507 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 9.4900000000000006e-06 2023-06-03T20:32:37.507 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:32:37.507 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:32:37.507 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "dispatched", 2023-06-03T20:32:37.507 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:31:26.251698+0000", 2023-06-03T20:32:37.507 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 2.0353e-05 2023-06-03T20:32:37.508 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:32:37.508 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:32:37.508 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "queued_for_pg", 2023-06-03T20:32:37.508 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:31:26.251708+0000", 2023-06-03T20:32:37.508 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 1.0258000000000001e-05 2023-06-03T20:32:37.508 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:32:37.508 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:32:37.508 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "reached_pg", 2023-06-03T20:32:37.509 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:31:26.251856+0000", 2023-06-03T20:32:37.509 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.00014767299999999999 2023-06-03T20:32:37.509 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:32:37.509 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:32:37.509 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "started", 2023-06-03T20:32:37.509 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:31:26.252168+0000", 2023-06-03T20:32:37.509 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.000312744 2023-06-03T20:32:37.510 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:32:37.510 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:32:37.510 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "waiting for subops from 0,5", 2023-06-03T20:32:37.510 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:31:26.252519+0000", 2023-06-03T20:32:37.510 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.000351126 2023-06-03T20:32:37.510 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:32:37.510 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:32:37.510 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "op_commit", 2023-06-03T20:32:37.511 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:31:26.261415+0000", 2023-06-03T20:32:37.511 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.0088958079999999998 2023-06-03T20:32:37.511 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:32:37.511 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:32:37.511 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:32:37.511 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:31:26.264756+0000", 2023-06-03T20:32:37.511 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.0033407549999999999 2023-06-03T20:32:37.511 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:32:37.512 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:32:37.512 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:32:37.512 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:31:26.521256+0000", 2023-06-03T20:32:37.512 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.25650035999999998 2023-06-03T20:32:37.512 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:32:37.512 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:32:37.512 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "commit_sent", 2023-06-03T20:32:37.513 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:31:26.521300+0000", 2023-06-03T20:32:37.513 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 4.3730000000000003e-05 2023-06-03T20:32:37.513 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:32:37.513 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:32:37.513 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "done", 2023-06-03T20:32:37.513 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:31:26.521339+0000", 2023-06-03T20:32:37.513 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 3.9357999999999998e-05 2023-06-03T20:32:37.513 INFO:teuthology.orchestra.run.smithi089.stdout: } 2023-06-03T20:32:37.514 INFO:teuthology.orchestra.run.smithi089.stdout: ] 2023-06-03T20:32:37.514 INFO:teuthology.orchestra.run.smithi089.stdout: } 2023-06-03T20:32:37.514 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:32:37.514 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:32:37.514 INFO:teuthology.orchestra.run.smithi089.stdout: "description": "osd_op(client.46739.0:177022 14.0 14:0c31eeb2:::benchmark_data_smithi089_237281_object79006:head [delete] snapc 0=[] ondisk+write+known_if_redirected e293)", 2023-06-03T20:32:37.514 INFO:teuthology.orchestra.run.smithi089.stdout: "initiated_at": "2023-06-03T20:31:26.272662+0000", 2023-06-03T20:32:37.514 INFO:teuthology.orchestra.run.smithi089.stdout: "age": 71.024625520000001, 2023-06-03T20:32:37.514 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.25181078899999998, 2023-06-03T20:32:37.515 INFO:teuthology.orchestra.run.smithi089.stdout: "type_data": { 2023-06-03T20:32:37.515 INFO:teuthology.orchestra.run.smithi089.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-03T20:32:37.515 INFO:teuthology.orchestra.run.smithi089.stdout: "client_info": { 2023-06-03T20:32:37.515 INFO:teuthology.orchestra.run.smithi089.stdout: "client": "client.46739", 2023-06-03T20:32:37.515 INFO:teuthology.orchestra.run.smithi089.stdout: "client_addr": "172.21.15.89:0/2479974613", 2023-06-03T20:32:37.515 INFO:teuthology.orchestra.run.smithi089.stdout: "tid": 177022 2023-06-03T20:32:37.515 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:32:37.516 INFO:teuthology.orchestra.run.smithi089.stdout: "events": [ 2023-06-03T20:32:37.516 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:32:37.516 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "initiated", 2023-06-03T20:32:37.516 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:31:26.272662+0000", 2023-06-03T20:32:37.516 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0 2023-06-03T20:32:37.516 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:32:37.516 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:32:37.516 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "throttled", 2023-06-03T20:32:37.517 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:31:26.272662+0000", 2023-06-03T20:32:37.517 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0 2023-06-03T20:32:37.517 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:32:37.517 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:32:37.517 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "header_read", 2023-06-03T20:32:37.517 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:31:26.272661+0000", 2023-06-03T20:32:37.517 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 4294967295.999999 2023-06-03T20:32:37.518 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:32:37.518 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:32:37.518 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "all_read", 2023-06-03T20:32:37.518 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:31:26.272666+0000", 2023-06-03T20:32:37.518 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 5.3639999999999998e-06 2023-06-03T20:32:37.518 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:32:37.518 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:32:37.518 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "dispatched", 2023-06-03T20:32:37.519 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:31:26.272677+0000", 2023-06-03T20:32:37.519 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 1.0869e-05 2023-06-03T20:32:37.519 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:32:37.519 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:32:37.519 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "queued_for_pg", 2023-06-03T20:32:37.519 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:31:26.272685+0000", 2023-06-03T20:32:37.519 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 7.6620000000000005e-06 2023-06-03T20:32:37.519 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:32:37.520 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:32:37.520 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "reached_pg", 2023-06-03T20:32:37.520 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:31:26.272778+0000", 2023-06-03T20:32:37.520 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 9.2880000000000005e-05 2023-06-03T20:32:37.520 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:32:37.520 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:32:37.520 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "started", 2023-06-03T20:32:37.521 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:31:26.272926+0000", 2023-06-03T20:32:37.521 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.00014867600000000001 2023-06-03T20:32:37.521 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:32:37.521 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:32:37.521 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "waiting for subops from 0,5", 2023-06-03T20:32:37.521 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:31:26.273034+0000", 2023-06-03T20:32:37.521 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.000107552 2023-06-03T20:32:37.521 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:32:37.522 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:32:37.522 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "op_commit", 2023-06-03T20:32:37.522 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:31:26.277434+0000", 2023-06-03T20:32:37.522 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.004399755 2023-06-03T20:32:37.522 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:32:37.522 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:32:37.522 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:32:37.522 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:31:26.280611+0000", 2023-06-03T20:32:37.523 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.0031772189999999998 2023-06-03T20:32:37.523 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:32:37.523 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:32:37.523 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:32:37.523 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:31:26.524388+0000", 2023-06-03T20:32:37.523 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.243777463 2023-06-03T20:32:37.523 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:32:37.523 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:32:37.524 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "commit_sent", 2023-06-03T20:32:37.524 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:31:26.524435+0000", 2023-06-03T20:32:37.524 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 4.6669999999999999e-05 2023-06-03T20:32:37.524 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:32:37.524 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:32:37.524 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "done", 2023-06-03T20:32:37.524 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:31:26.524473+0000", 2023-06-03T20:32:37.525 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 3.7738999999999999e-05 2023-06-03T20:32:37.525 INFO:teuthology.orchestra.run.smithi089.stdout: } 2023-06-03T20:32:37.525 INFO:teuthology.orchestra.run.smithi089.stdout: ] 2023-06-03T20:32:37.525 INFO:teuthology.orchestra.run.smithi089.stdout: } 2023-06-03T20:32:37.525 INFO:teuthology.orchestra.run.smithi089.stdout: } 2023-06-03T20:32:37.525 INFO:teuthology.orchestra.run.smithi089.stdout: ] 2023-06-03T20:32:37.525 INFO:teuthology.orchestra.run.smithi089.stdout:} 2023-06-03T20:32:37.526 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:32:37.549 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:32:37.552 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:32:37.553 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:32:37.585 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:32:37 smithi164 bash[59462]: audit 1685824357.1613996 mon.c (mon.2) 685 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T20:32:37.653 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:32:37.695 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:32:37.696 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:32:37 smithi089 bash[75068]: audit 1685824357.1613996 mon.c (mon.2) 685 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T20:32:37.698 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:32:37.699 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:32:37.703 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:32:37 smithi089 bash[78452]: audit 1685824357.1613996 mon.c (mon.2) 685 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T20:32:37.800 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:32:37.824 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:32:37.826 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:32:37.826 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:32:37.927 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:32:37.967 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:32:37.969 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:32:37.970 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:32:38.071 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:32:38.109 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:32:38.112 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:32:38.113 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:32:38.213 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:32:38.252 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:32:38.255 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:32:38.256 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:32:38.357 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:32:38.391 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:32:38.394 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:32:38.395 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:32:38.495 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:32:38.509 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:32:38 smithi164 bash[59462]: cluster 1685824357.225799 mgr.y (mgr.34104) 1707 : cluster 0 pgmap v1766: 73 pgs: 73 active+clean; 582 KiB data, 5.1 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:32:38.531 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:32:38.534 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:32:38.535 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:32:38.636 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:32:38.680 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:32:38.683 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:32:38.684 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:32:38.779 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:32:38 smithi089 bash[78452]: cluster 1685824357.225799 mgr.y (mgr.34104) 1707 : cluster 0 pgmap v1766: 73 pgs: 73 active+clean; 582 KiB data, 5.1 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:32:38.780 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:32:38 smithi089 bash[75068]: cluster 1685824357.225799 mgr.y (mgr.34104) 1707 : cluster 0 pgmap v1766: 73 pgs: 73 active+clean; 582 KiB data, 5.1 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:32:38.785 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:32:38.818 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:32:38.820 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:32:38.821 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:32:38.922 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:32:38.954 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:32:38.956 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:32:38.957 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:32:39.058 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:32:39.097 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:32:39.100 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:32:39.101 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:32:39.202 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:32:39.242 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:32:39.245 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:32:39.245 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:32:39.346 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:32:39.388 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:32:39.392 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:32:39.393 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:32:39.494 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:32:39.534 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:32:39.536 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:32:39.537 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:32:39.638 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:32:39.670 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:32:39.674 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:32:39.674 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:32:39.775 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:32:39.813 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:32:39.817 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:32:39.817 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:32:39.918 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:32:39.948 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:32:39.949 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:32:39.950 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:32:40.051 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:32:40.080 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:32:40.083 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:32:40.083 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:32:40.184 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:32:40.223 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:32:40.226 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:32:40.227 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:32:40.328 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:32:40.367 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:32:40.370 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:32:40.371 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:32:40.472 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:32:40.506 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:32:40.509 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:32:40.510 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:32:40.585 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:32:40 smithi164 bash[59462]: cluster 1685824359.2263064 mgr.y (mgr.34104) 1708 : cluster 0 pgmap v1767: 73 pgs: 73 active+clean; 582 KiB data, 5.1 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:32:40.611 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:32:40.622 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:32:40 smithi089 bash[75068]: cluster 1685824359.2263064 mgr.y (mgr.34104) 1708 : cluster 0 pgmap v1767: 73 pgs: 73 active+clean; 582 KiB data, 5.1 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:32:40.622 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:32:40 smithi089 bash[78452]: cluster 1685824359.2263064 mgr.y (mgr.34104) 1708 : cluster 0 pgmap v1767: 73 pgs: 73 active+clean; 582 KiB data, 5.1 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:32:40.644 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:32:40.647 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:32:40.648 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:32:40.748 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:32:40.790 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:32:40.793 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:32:40.793 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:32:40.894 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:32:40.927 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:32:40.929 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:32:40.930 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:32:41.031 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:32:41.056 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:32:41.058 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:32:41.059 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:32:41.160 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:32:41.200 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:32:41.203 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:32:41.203 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:32:41.304 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:32:41.331 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:32:41.333 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:32:41.334 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:32:41.435 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:32:41.439 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph daemon osd.3 dump_ops_in_flight 2023-06-03T20:32:41.458 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:32:41.461 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:32:41.461 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:32:41.562 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:32:41.600 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:32:41.602 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:32:41.602 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:32:41.703 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:32:41.746 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:32:41.748 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:32:41.749 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:32:41.850 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:32:41.879 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:32:41.881 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:32:41.882 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:32:41.983 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:32:42.015 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:32:42.017 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:32:42.018 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:32:42.118 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:32:42.157 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:32:42.160 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:32:42.161 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:32:42.262 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:32:42.287 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:32:42.288 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:32:42.289 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:32:42.390 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:32:42.401 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:32:42 smithi164 bash[59462]: cluster 1685824361.2267907 mgr.y (mgr.34104) 1709 : cluster 0 pgmap v1768: 73 pgs: 73 active+clean; 582 KiB data, 5.1 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:32:42.424 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:32:42.428 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:32:42.429 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:32:42.529 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:32:42 smithi089 bash[75068]: cluster 1685824361.2267907 mgr.y (mgr.34104) 1709 : cluster 0 pgmap v1768: 73 pgs: 73 active+clean; 582 KiB data, 5.1 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:32:42.529 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:32:42 smithi089 bash[78452]: cluster 1685824361.2267907 mgr.y (mgr.34104) 1709 : cluster 0 pgmap v1768: 73 pgs: 73 active+clean; 582 KiB data, 5.1 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:32:42.530 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:32:42.569 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:32:42.571 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:32:42.572 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:32:42.673 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:32:42.702 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:32:42.705 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:32:42.705 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:32:42.806 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:32:42.845 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:32:42.848 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:32:42.849 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:32:42.949 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:32:42.988 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:32:42.990 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:32:42.991 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:32:43.092 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:32:43.127 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:32:43.130 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:32:43.130 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:32:43.231 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:32:43.263 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:32:43.265 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:32:43.266 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:32:43.367 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:32:43.396 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:32:43.399 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:32:43.400 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:32:43.501 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:32:43.535 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:32:43.538 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:32:43.539 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:32:43.640 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:32:43.673 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:32:43.675 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:32:43.676 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:32:43.777 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:32:43.814 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:32:43.816 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:32:43.817 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:32:43.918 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:32:43.959 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:32:43.962 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:32:43.962 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:32:44.063 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:32:44.103 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:32:44.106 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:32:44.107 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:32:44.208 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:32:44.219 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:32:44 smithi164 bash[59462]: cluster 1685824363.2277064 mgr.y (mgr.34104) 1710 : cluster 0 pgmap v1769: 73 pgs: 73 active+clean; 582 KiB data, 5.1 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:32:44.244 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:32:44.247 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:32:44.248 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:32:44.349 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:32:44.386 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:32:44.389 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:32:44.390 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:32:44.491 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:32:44.529 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:32:44.529 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:32:44 smithi089 bash[78452]: cluster 1685824363.2277064 mgr.y (mgr.34104) 1710 : cluster 0 pgmap v1769: 73 pgs: 73 active+clean; 582 KiB data, 5.1 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:32:44.530 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:32:44 smithi089 bash[75068]: cluster 1685824363.2277064 mgr.y (mgr.34104) 1710 : cluster 0 pgmap v1769: 73 pgs: 73 active+clean; 582 KiB data, 5.1 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:32:44.532 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:32:44.533 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:32:44.634 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:32:44.668 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:32:44.672 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:32:44.672 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:32:44.773 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:32:44.812 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:32:44.815 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:32:44.816 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:32:44.916 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:32:44.938 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:32:44.940 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:32:44.941 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:32:45.042 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:32:45.069 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:32:45.071 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:32:45.071 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:32:45.088 INFO:teuthology.orchestra.run.smithi089.stdout: 2023-06-03T20:32:45.092 INFO:teuthology.orchestra.run.smithi089.stderr:dumped all 2023-06-03T20:32:45.173 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:32:45.201 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:32:45.203 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:32:45.203 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:32:45.304 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:32:45.333 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:32:45.337 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:32:45.337 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:32:45.438 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:32:45.475 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:32:45.478 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:32:45.478 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:32:45.579 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:32:45.619 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:32:45.621 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:32:45.622 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:32:45.723 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:32:45.759 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:32:45.762 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:32:45.763 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:32:45.864 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:32:45.908 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:32:45.910 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:32:45.911 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:32:46.012 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:32:46.049 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:32:46.053 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:32:46.053 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:32:46.154 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:32:46.191 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:32:46.194 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:32:46.195 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:32:46.289 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 20:32:46 smithi089 bash[71163]: ::ffff:172.21.15.164 - - [03/Jun/2023:20:32:46] "GET /metrics HTTP/1.1" 200 226545 "" "Prometheus/2.33.4" 2023-06-03T20:32:46.296 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:32:46.322 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:32:46.325 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:32:46.325 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:32:46.331 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:32:46 smithi089 bash[75068]: audit 1685824365.072059 mgr.y (mgr.34104) 1711 : audit 0 from='client.46871 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-06-03T20:32:46.331 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:32:46 smithi089 bash[75068]: cluster 1685824365.2285004 mgr.y (mgr.34104) 1712 : cluster 0 pgmap v1770: 73 pgs: 73 active+clean; 582 KiB data, 5.1 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:32:46.331 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:32:46 smithi089 bash[78452]: audit 1685824365.072059 mgr.y (mgr.34104) 1711 : audit 0 from='client.46871 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-06-03T20:32:46.331 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:32:46 smithi089 bash[78452]: cluster 1685824365.2285004 mgr.y (mgr.34104) 1712 : cluster 0 pgmap v1770: 73 pgs: 73 active+clean; 582 KiB data, 5.1 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:32:46.426 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:32:46.453 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:32:46.455 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:32:46.455 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:32:46.556 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:32:46.570 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:32:46 smithi164 bash[59462]: audit 1685824365.072059 mgr.y (mgr.34104) 1711 : audit 0 from='client.46871 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-06-03T20:32:46.570 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:32:46 smithi164 bash[59462]: cluster 1685824365.2285004 mgr.y (mgr.34104) 1712 : cluster 0 pgmap v1770: 73 pgs: 73 active+clean; 582 KiB data, 5.1 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:32:46.596 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:32:46.600 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:32:46.600 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:32:46.702 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:32:46.738 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:32:46.742 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:32:46.743 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:32:46.843 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:32:46.885 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:32:46.887 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:32:46.888 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:32:46.989 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:32:47.020 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:32:47.022 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:32:47.023 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:32:47.124 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:32:47.140 INFO:teuthology.orchestra.run.smithi089.stdout:osd.0: {} 2023-06-03T20:32:47.161 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:32:47.161 INFO:teuthology.orchestra.run.smithi089.stdout:osd.1: {} 2023-06-03T20:32:47.164 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:32:47.164 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:32:47.182 INFO:teuthology.orchestra.run.smithi089.stdout:osd.2: {} 2023-06-03T20:32:47.204 INFO:teuthology.orchestra.run.smithi089.stdout:osd.3: {} 2023-06-03T20:32:47.225 INFO:teuthology.orchestra.run.smithi089.stdout:osd.4: {} 2023-06-03T20:32:47.247 INFO:teuthology.orchestra.run.smithi089.stdout:osd.5: {} 2023-06-03T20:32:47.266 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:32:47.268 INFO:teuthology.orchestra.run.smithi089.stdout:osd.6: {} 2023-06-03T20:32:47.289 INFO:teuthology.orchestra.run.smithi089.stdout:osd.7: {} 2023-06-03T20:32:47.290 INFO:teuthology.orchestra.run.smithi089.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-06-03T20:32:47.290 INFO:teuthology.orchestra.run.smithi089.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-06-03T20:32:47.290 INFO:teuthology.orchestra.run.smithi089.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-06-03T20:32:47.290 INFO:teuthology.orchestra.run.smithi089.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-06-03T20:32:47.291 INFO:teuthology.orchestra.run.smithi089.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-06-03T20:32:47.291 INFO:teuthology.orchestra.run.smithi089.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-06-03T20:32:47.291 INFO:teuthology.orchestra.run.smithi089.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-06-03T20:32:47.291 INFO:teuthology.orchestra.run.smithi089.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-06-03T20:32:47.301 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:32:47.303 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:32:47.304 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:32:47.405 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:32:47.416 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:32:47 smithi164 bash[59462]: audit 1685824367.1152234 mon.c (mon.2) 686 : audit 0 from='client.? 172.21.15.89:0/59769536' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-03T20:32:47.439 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:32:47.442 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:32:47.443 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:32:47.544 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:32:47.578 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:32:47.582 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:32:47.582 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:32:47.619 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:32:47 smithi089 bash[75068]: audit 1685824367.1152234 mon.c (mon.2) 686 : audit 0 from='client.? 172.21.15.89:0/59769536' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-03T20:32:47.620 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:32:47 smithi089 bash[78452]: audit 1685824367.1152234 mon.c (mon.2) 686 : audit 0 from='client.? 172.21.15.89:0/59769536' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-03T20:32:47.683 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:32:47.718 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:32:47.721 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:32:47.721 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:32:47.822 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:32:47.859 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:32:47.862 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:32:47.863 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:32:47.964 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:32:47.989 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:32:47.991 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:32:47.991 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:32:48.092 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:32:48.127 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:32:48.130 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:32:48.131 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:32:48.132 INFO:teuthology.orchestra.run.smithi089.stdout:{"pg_ready":true,"pg_map":{"version":1769,"stamp":"2023-06-03T20:32:43.227029+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":595854,"num_objects":24,"num_object_clones":0,"num_object_copies":72,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":24,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":261,"num_write_kb":2446,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":29,"num_bytes_recovered":1775960,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":195,"ondisk_log_size":195,"up":219,"acting":219,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":219,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":5377516,"kb_used_data":26412,"kb_used_omap":30,"kb_used_meta":5351009,"kb_avail":744583700,"statfs":{"total":767960285184,"available":762453708800,"internally_reserved":0,"allocated":27045888,"data_stored":19212801,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":31502,"internal_metadata":5479433458},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.003387"},"pg_stats":[{"pgid":"3.1a","version":"60'1","reported_seq":225,"reported_epoch":297,"state":"active+clean","last_fresh":"2023-06-03T20:32:21.780902+0000","last_change":"2023-06-03T20:23:28.242261+0000","last_active":"2023-06-03T20:32:21.780902+0000","last_peered":"2023-06-03T20:32:21.780902+0000","last_clean":"2023-06-03T20:32:21.780902+0000","last_became_active":"2023-06-03T20:23:28.240045+0000","last_became_peered":"2023-06-03T20:23:28.240045+0000","last_unstale":"2023-06-03T20:32:21.780902+0000","last_undegraded":"2023-06-03T20:32:21.780902+0000","last_fullsized":"2023-06-03T20:32:21.780902+0000","mapping_epoch":251,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":252,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T05:54:33.215385+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":86,"reported_epoch":297,"state":"active+clean","last_fresh":"2023-06-03T20:32:21.422986+0000","last_change":"2023-06-03T20:23:28.283021+0000","last_active":"2023-06-03T20:32:21.422986+0000","last_peered":"2023-06-03T20:32:21.422986+0000","last_clean":"2023-06-03T20:32:21.422986+0000","last_became_active":"2023-06-03T20:23:28.282428+0000","last_became_peered":"2023-06-03T20:23:28.282428+0000","last_unstale":"2023-06-03T20:32:21.422986+0000","last_undegraded":"2023-06-03T20:32:21.422986+0000","last_fullsized":"2023-06-03T20:32:21.422986+0000","mapping_epoch":251,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":252,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T05:01:33.115332+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":86,"reported_epoch":297,"state":"active+clean","last_fresh":"2023-06-03T20:32:21.422528+0000","last_change":"2023-06-03T20:23:28.253821+0000","last_active":"2023-06-03T20:32:21.422528+0000","last_peered":"2023-06-03T20:32:21.422528+0000","last_clean":"2023-06-03T20:32:21.422528+0000","last_became_active":"2023-06-03T20:23:28.253448+0000","last_became_peered":"2023-06-03T20:23:28.253448+0000","last_unstale":"2023-06-03T20:32:21.422528+0000","last_undegraded":"2023-06-03T20:32:21.422528+0000","last_fullsized":"2023-06-03T20:32:21.422528+0000","mapping_epoch":251,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":252,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T23:40:02.264778+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":269,"reported_epoch":297,"state":"active+clean","last_fresh":"2023-06-03T20:32:22.073205+0000","last_change":"2023-06-03T19:56:54.507653+0000","last_active":"2023-06-03T20:32:22.073205+0000","last_peered":"2023-06-03T20:32:22.073205+0000","last_clean":"2023-06-03T20:32:22.073205+0000","last_became_active":"2023-06-03T19:56:54.507350+0000","last_became_peered":"2023-06-03T19:56:54.507350+0000","last_unstale":"2023-06-03T20:32:22.073205+0000","last_undegraded":"2023-06-03T20:32:22.073205+0000","last_fullsized":"2023-06-03T20:32:22.073205+0000","mapping_epoch":119,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T22:33:20.065844+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":309,"reported_epoch":297,"state":"active+clean","last_fresh":"2023-06-03T20:32:21.411745+0000","last_change":"2023-06-03T20:21:01.256894+0000","last_active":"2023-06-03T20:32:21.411745+0000","last_peered":"2023-06-03T20:32:21.411745+0000","last_clean":"2023-06-03T20:32:21.411745+0000","last_became_active":"2023-06-03T20:21:01.255903+0000","last_became_peered":"2023-06-03T20:21:01.255903+0000","last_unstale":"2023-06-03T20:32:21.411745+0000","last_undegraded":"2023-06-03T20:32:21.411745+0000","last_fullsized":"2023-06-03T20:32:21.411745+0000","mapping_epoch":235,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":236,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T20:35:27.810860+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":62,"reported_epoch":297,"state":"active+clean","last_fresh":"2023-06-03T20:32:22.071221+0000","last_change":"2023-06-03T20:28:24.269355+0000","last_active":"2023-06-03T20:32:22.071221+0000","last_peered":"2023-06-03T20:32:22.071221+0000","last_clean":"2023-06-03T20:32:22.071221+0000","last_became_active":"2023-06-03T20:28:24.268859+0000","last_became_peered":"2023-06-03T20:28:24.268859+0000","last_unstale":"2023-06-03T20:32:22.071221+0000","last_undegraded":"2023-06-03T20:32:22.071221+0000","last_fullsized":"2023-06-03T20:32:22.071221+0000","mapping_epoch":284,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":285,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,2],"acting":[6,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":264,"reported_epoch":297,"state":"active+clean","last_fresh":"2023-06-03T20:32:22.073395+0000","last_change":"2023-06-03T19:56:54.509490+0000","last_active":"2023-06-03T20:32:22.073395+0000","last_peered":"2023-06-03T20:32:22.073395+0000","last_clean":"2023-06-03T20:32:22.073395+0000","last_became_active":"2023-06-03T19:56:54.509145+0000","last_became_peered":"2023-06-03T19:56:54.509145+0000","last_unstale":"2023-06-03T20:32:22.073395+0000","last_undegraded":"2023-06-03T20:32:22.073395+0000","last_fullsized":"2023-06-03T20:32:22.073395+0000","mapping_epoch":119,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T07:12:03.964110+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":330,"reported_epoch":297,"state":"active+clean","last_fresh":"2023-06-03T20:32:21.411845+0000","last_change":"2023-06-03T20:28:51.233277+0000","last_active":"2023-06-03T20:32:21.411845+0000","last_peered":"2023-06-03T20:32:21.411845+0000","last_clean":"2023-06-03T20:32:21.411845+0000","last_became_active":"2023-06-03T20:28:51.232919+0000","last_became_peered":"2023-06-03T20:28:51.232919+0000","last_unstale":"2023-06-03T20:32:21.411845+0000","last_undegraded":"2023-06-03T20:32:21.411845+0000","last_fullsized":"2023-06-03T20:32:21.411845+0000","mapping_epoch":286,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":287,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T02:06:02.736415+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":230,"reported_epoch":297,"state":"active+clean","last_fresh":"2023-06-03T20:32:21.318293+0000","last_change":"2023-06-03T20:21:01.258441+0000","last_active":"2023-06-03T20:32:21.318293+0000","last_peered":"2023-06-03T20:32:21.318293+0000","last_clean":"2023-06-03T20:32:21.318293+0000","last_became_active":"2023-06-03T20:21:01.257779+0000","last_became_peered":"2023-06-03T20:21:01.257779+0000","last_unstale":"2023-06-03T20:32:21.318293+0000","last_undegraded":"2023-06-03T20:32:21.318293+0000","last_fullsized":"2023-06-03T20:32:21.318293+0000","mapping_epoch":235,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":236,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,1],"acting":[7,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":275,"reported_epoch":297,"state":"active+clean","last_fresh":"2023-06-03T20:32:21.412182+0000","last_change":"2023-06-03T20:24:02.267331+0000","last_active":"2023-06-03T20:32:21.412182+0000","last_peered":"2023-06-03T20:32:21.412182+0000","last_clean":"2023-06-03T20:32:21.412182+0000","last_became_active":"2023-06-03T20:24:02.266997+0000","last_became_peered":"2023-06-03T20:24:02.266997+0000","last_unstale":"2023-06-03T20:32:21.412182+0000","last_undegraded":"2023-06-03T20:32:21.412182+0000","last_fullsized":"2023-06-03T20:32:21.412182+0000","mapping_epoch":254,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":255,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T22:09:18.993656+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"60'3","reported_seq":295,"reported_epoch":297,"state":"active+clean","last_fresh":"2023-06-03T20:32:21.319275+0000","last_change":"2023-06-03T20:23:27.261246+0000","last_active":"2023-06-03T20:32:21.319275+0000","last_peered":"2023-06-03T20:32:21.319275+0000","last_clean":"2023-06-03T20:32:21.319275+0000","last_became_active":"2023-06-03T20:23:27.259201+0000","last_became_peered":"2023-06-03T20:23:27.259201+0000","last_unstale":"2023-06-03T20:32:21.319275+0000","last_undegraded":"2023-06-03T20:32:21.319275+0000","last_fullsized":"2023-06-03T20:32:21.319275+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":87,"reported_epoch":298,"state":"active+clean","last_fresh":"2023-06-03T20:32:22.910648+0000","last_change":"2023-06-03T20:28:51.235184+0000","last_active":"2023-06-03T20:32:22.910648+0000","last_peered":"2023-06-03T20:32:22.910648+0000","last_clean":"2023-06-03T20:32:22.910648+0000","last_became_active":"2023-06-03T20:28:51.234830+0000","last_became_peered":"2023-06-03T20:28:51.234830+0000","last_unstale":"2023-06-03T20:32:22.910648+0000","last_undegraded":"2023-06-03T20:32:22.910648+0000","last_fullsized":"2023-06-03T20:32:22.910648+0000","mapping_epoch":286,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":287,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T04:40:02.479523+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"77'23","reported_seq":232,"reported_epoch":297,"state":"active+clean","last_fresh":"2023-06-03T20:32:21.781347+0000","last_change":"2023-06-03T20:24:02.280274+0000","last_active":"2023-06-03T20:32:21.781347+0000","last_peered":"2023-06-03T20:32:21.781347+0000","last_clean":"2023-06-03T20:32:21.781347+0000","last_became_active":"2023-06-03T20:24:02.264914+0000","last_became_peered":"2023-06-03T20:24:02.264914+0000","last_unstale":"2023-06-03T20:32:21.781347+0000","last_undegraded":"2023-06-03T20:32:21.781347+0000","last_fullsized":"2023-06-03T20:32:21.781347+0000","mapping_epoch":254,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":255,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":23,"log_dups_size":0,"ondisk_log_size":23,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T07:00:35.888220+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4716,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":23,"num_write_kb":20,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4716,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":208,"reported_epoch":297,"state":"active+clean","last_fresh":"2023-06-03T20:32:21.780884+0000","last_change":"2023-06-03T20:24:02.254850+0000","last_active":"2023-06-03T20:32:21.780884+0000","last_peered":"2023-06-03T20:32:21.780884+0000","last_clean":"2023-06-03T20:32:21.780884+0000","last_became_active":"2023-06-03T20:24:02.254409+0000","last_became_peered":"2023-06-03T20:24:02.254409+0000","last_unstale":"2023-06-03T20:32:21.780884+0000","last_undegraded":"2023-06-03T20:32:21.780884+0000","last_fullsized":"2023-06-03T20:32:21.780884+0000","mapping_epoch":254,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":255,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T19:49:53.393121+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"60'2","reported_seq":297,"reported_epoch":297,"state":"active+clean","last_fresh":"2023-06-03T20:32:21.318686+0000","last_change":"2023-06-03T20:23:27.258749+0000","last_active":"2023-06-03T20:32:21.318686+0000","last_peered":"2023-06-03T20:32:21.318686+0000","last_clean":"2023-06-03T20:32:21.318686+0000","last_became_active":"2023-06-03T20:23:27.256899+0000","last_became_peered":"2023-06-03T20:23:27.256899+0000","last_unstale":"2023-06-03T20:32:21.318686+0000","last_undegraded":"2023-06-03T20:32:21.318686+0000","last_fullsized":"2023-06-03T20:32:21.318686+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":294,"reported_epoch":297,"state":"active+clean","last_fresh":"2023-06-03T20:32:21.411604+0000","last_change":"2023-06-03T20:21:01.256612+0000","last_active":"2023-06-03T20:32:21.411604+0000","last_peered":"2023-06-03T20:32:21.411604+0000","last_clean":"2023-06-03T20:32:21.411604+0000","last_became_active":"2023-06-03T20:21:01.256243+0000","last_became_peered":"2023-06-03T20:21:01.256243+0000","last_unstale":"2023-06-03T20:32:21.411604+0000","last_undegraded":"2023-06-03T20:32:21.411604+0000","last_fullsized":"2023-06-03T20:32:21.411604+0000","mapping_epoch":235,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":236,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T05:24:47.054342+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":265,"reported_epoch":297,"state":"active+clean","last_fresh":"2023-06-03T20:32:22.072765+0000","last_change":"2023-06-03T19:56:54.507479+0000","last_active":"2023-06-03T20:32:22.072765+0000","last_peered":"2023-06-03T20:32:22.072765+0000","last_clean":"2023-06-03T20:32:22.072765+0000","last_became_active":"2023-06-03T19:56:54.507149+0000","last_became_peered":"2023-06-03T19:56:54.507149+0000","last_unstale":"2023-06-03T20:32:22.072765+0000","last_undegraded":"2023-06-03T20:32:22.072765+0000","last_fullsized":"2023-06-03T20:32:22.072765+0000","mapping_epoch":119,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T20:22:36.676317+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":164,"reported_epoch":297,"state":"active+clean","last_fresh":"2023-06-03T20:32:21.319352+0000","last_change":"2023-06-03T20:17:22.832844+0000","last_active":"2023-06-03T20:32:21.319352+0000","last_peered":"2023-06-03T20:32:21.319352+0000","last_clean":"2023-06-03T20:32:21.319352+0000","last_became_active":"2023-06-03T20:17:22.832473+0000","last_became_peered":"2023-06-03T20:17:22.832473+0000","last_unstale":"2023-06-03T20:32:21.319352+0000","last_undegraded":"2023-06-03T20:32:21.319352+0000","last_fullsized":"2023-06-03T20:32:21.319352+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,3],"acting":[7,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":297,"reported_epoch":297,"state":"active+clean","last_fresh":"2023-06-03T20:32:21.318102+0000","last_change":"2023-06-03T20:24:02.267531+0000","last_active":"2023-06-03T20:32:21.318102+0000","last_peered":"2023-06-03T20:32:21.318102+0000","last_clean":"2023-06-03T20:32:21.318102+0000","last_became_active":"2023-06-03T20:24:02.267159+0000","last_became_peered":"2023-06-03T20:24:02.267159+0000","last_unstale":"2023-06-03T20:32:21.318102+0000","last_undegraded":"2023-06-03T20:32:21.318102+0000","last_fullsized":"2023-06-03T20:32:21.318102+0000","mapping_epoch":254,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":255,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":265,"reported_epoch":297,"state":"active+clean","last_fresh":"2023-06-03T20:32:22.073331+0000","last_change":"2023-06-03T19:56:54.510289+0000","last_active":"2023-06-03T20:32:22.073331+0000","last_peered":"2023-06-03T20:32:22.073331+0000","last_clean":"2023-06-03T20:32:22.073331+0000","last_became_active":"2023-06-03T19:56:54.509905+0000","last_became_peered":"2023-06-03T19:56:54.509905+0000","last_unstale":"2023-06-03T20:32:22.073331+0000","last_undegraded":"2023-06-03T20:32:22.073331+0000","last_fullsized":"2023-06-03T20:32:22.073331+0000","mapping_epoch":119,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T02:54:26.792401+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"60'8","reported_seq":63,"reported_epoch":297,"state":"active+clean","last_fresh":"2023-06-03T20:32:22.070647+0000","last_change":"2023-06-03T20:26:59.682623+0000","last_active":"2023-06-03T20:32:22.070647+0000","last_peered":"2023-06-03T20:32:22.070647+0000","last_clean":"2023-06-03T20:32:22.070647+0000","last_became_active":"2023-06-03T20:24:02.260105+0000","last_became_peered":"2023-06-03T20:24:02.260105+0000","last_unstale":"2023-06-03T20:32:22.070647+0000","last_undegraded":"2023-06-03T20:32:22.070647+0000","last_fullsized":"2023-06-03T20:32:22.070647+0000","mapping_epoch":254,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":273,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":302,"reported_epoch":297,"state":"active+clean","last_fresh":"2023-06-03T20:32:21.412759+0000","last_change":"2023-06-03T20:28:24.281308+0000","last_active":"2023-06-03T20:32:21.412759+0000","last_peered":"2023-06-03T20:32:21.412759+0000","last_clean":"2023-06-03T20:32:21.412759+0000","last_became_active":"2023-06-03T20:28:24.280981+0000","last_became_peered":"2023-06-03T20:28:24.280981+0000","last_unstale":"2023-06-03T20:32:21.412759+0000","last_undegraded":"2023-06-03T20:32:21.412759+0000","last_fullsized":"2023-06-03T20:32:21.412759+0000","mapping_epoch":284,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":285,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T02:44:14.890147+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.f","version":"60'1","reported_seq":296,"reported_epoch":297,"state":"active+clean","last_fresh":"2023-06-03T20:32:21.318576+0000","last_change":"2023-06-03T20:23:27.263667+0000","last_active":"2023-06-03T20:32:21.318576+0000","last_peered":"2023-06-03T20:32:21.318576+0000","last_clean":"2023-06-03T20:32:21.318576+0000","last_became_active":"2023-06-03T20:23:27.263110+0000","last_became_peered":"2023-06-03T20:23:27.263110+0000","last_unstale":"2023-06-03T20:32:21.318576+0000","last_undegraded":"2023-06-03T20:32:21.318576+0000","last_fullsized":"2023-06-03T20:32:21.318576+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":257,"reported_epoch":297,"state":"active+clean","last_fresh":"2023-06-03T20:32:21.318673+0000","last_change":"2023-06-03T20:28:51.231222+0000","last_active":"2023-06-03T20:32:21.318673+0000","last_peered":"2023-06-03T20:32:21.318673+0000","last_clean":"2023-06-03T20:32:21.318673+0000","last_became_active":"2023-06-03T20:28:51.230848+0000","last_became_peered":"2023-06-03T20:28:51.230848+0000","last_unstale":"2023-06-03T20:32:21.318673+0000","last_undegraded":"2023-06-03T20:32:21.318673+0000","last_fullsized":"2023-06-03T20:32:21.318673+0000","mapping_epoch":286,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":287,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.e","version":"60'2","reported_seq":325,"reported_epoch":297,"state":"active+clean","last_fresh":"2023-06-03T20:32:21.318477+0000","last_change":"2023-06-03T20:23:28.606751+0000","last_active":"2023-06-03T20:32:21.318477+0000","last_peered":"2023-06-03T20:32:21.318477+0000","last_clean":"2023-06-03T20:32:21.318477+0000","last_became_active":"2023-06-03T20:23:28.605968+0000","last_became_peered":"2023-06-03T20:23:28.605968+0000","last_unstale":"2023-06-03T20:32:21.318477+0000","last_undegraded":"2023-06-03T20:32:21.318477+0000","last_fullsized":"2023-06-03T20:32:21.318477+0000","mapping_epoch":251,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":252,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":238,"reported_epoch":297,"state":"active+clean","last_fresh":"2023-06-03T20:32:21.780985+0000","last_change":"2023-06-03T20:23:28.249694+0000","last_active":"2023-06-03T20:32:21.780985+0000","last_peered":"2023-06-03T20:32:21.780985+0000","last_clean":"2023-06-03T20:32:21.780985+0000","last_became_active":"2023-06-03T20:23:28.249388+0000","last_became_peered":"2023-06-03T20:23:28.249388+0000","last_unstale":"2023-06-03T20:32:21.780985+0000","last_undegraded":"2023-06-03T20:32:21.780985+0000","last_fullsized":"2023-06-03T20:32:21.780985+0000","mapping_epoch":251,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":252,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T05:44:01.835586+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.d","version":"60'1","reported_seq":274,"reported_epoch":297,"state":"active+clean","last_fresh":"2023-06-03T20:32:21.319132+0000","last_change":"2023-06-03T20:26:59.686037+0000","last_active":"2023-06-03T20:32:21.319132+0000","last_peered":"2023-06-03T20:32:21.319132+0000","last_clean":"2023-06-03T20:32:21.319132+0000","last_became_active":"2023-06-03T20:24:02.268002+0000","last_became_peered":"2023-06-03T20:24:02.268002+0000","last_unstale":"2023-06-03T20:32:21.319132+0000","last_undegraded":"2023-06-03T20:32:21.319132+0000","last_fullsized":"2023-06-03T20:32:21.319132+0000","mapping_epoch":254,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":273,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":140,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":21,"reported_epoch":297,"state":"active+clean","last_fresh":"2023-06-03T20:32:22.070511+0000","last_change":"2023-06-03T20:28:51.230833+0000","last_active":"2023-06-03T20:32:22.070511+0000","last_peered":"2023-06-03T20:32:22.070511+0000","last_clean":"2023-06-03T20:32:22.070511+0000","last_became_active":"2023-06-03T20:28:51.230489+0000","last_became_peered":"2023-06-03T20:28:51.230489+0000","last_unstale":"2023-06-03T20:32:22.070511+0000","last_undegraded":"2023-06-03T20:32:22.070511+0000","last_fullsized":"2023-06-03T20:32:22.070511+0000","mapping_epoch":286,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":287,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":291,"reported_epoch":297,"state":"active+clean","last_fresh":"2023-06-03T20:32:21.412420+0000","last_change":"2023-06-03T20:24:02.261252+0000","last_active":"2023-06-03T20:32:21.412420+0000","last_peered":"2023-06-03T20:32:21.412420+0000","last_clean":"2023-06-03T20:32:21.412420+0000","last_became_active":"2023-06-03T20:24:02.260824+0000","last_became_peered":"2023-06-03T20:24:02.260824+0000","last_unstale":"2023-06-03T20:32:21.412420+0000","last_undegraded":"2023-06-03T20:32:21.412420+0000","last_fullsized":"2023-06-03T20:32:21.412420+0000","mapping_epoch":254,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":255,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T20:00:53.584753+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":329,"reported_epoch":297,"state":"active+clean","last_fresh":"2023-06-03T20:32:22.072908+0000","last_change":"2023-06-03T20:23:28.283360+0000","last_active":"2023-06-03T20:32:22.072908+0000","last_peered":"2023-06-03T20:32:22.072908+0000","last_clean":"2023-06-03T20:32:22.072908+0000","last_became_active":"2023-06-03T20:23:28.283035+0000","last_became_peered":"2023-06-03T20:23:28.283035+0000","last_unstale":"2023-06-03T20:32:22.072908+0000","last_undegraded":"2023-06-03T20:32:22.072908+0000","last_fullsized":"2023-06-03T20:32:22.072908+0000","mapping_epoch":251,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":252,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T00:22:59.446515+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.b","version":"60'2","reported_seq":292,"reported_epoch":297,"state":"active+clean","last_fresh":"2023-06-03T20:32:21.412190+0000","last_change":"2023-06-03T20:23:27.254179+0000","last_active":"2023-06-03T20:32:21.412190+0000","last_peered":"2023-06-03T20:32:21.412190+0000","last_clean":"2023-06-03T20:32:21.412190+0000","last_became_active":"2023-06-03T20:23:27.253791+0000","last_became_peered":"2023-06-03T20:23:27.253791+0000","last_unstale":"2023-06-03T20:32:21.412190+0000","last_undegraded":"2023-06-03T20:32:21.412190+0000","last_fullsized":"2023-06-03T20:32:21.412190+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T00:31:27.336760+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":214,"reported_epoch":298,"state":"active+clean","last_fresh":"2023-06-03T20:32:22.750701+0000","last_change":"2023-06-03T20:24:02.255551+0000","last_active":"2023-06-03T20:32:22.750701+0000","last_peered":"2023-06-03T20:32:22.750701+0000","last_clean":"2023-06-03T20:32:22.750701+0000","last_became_active":"2023-06-03T20:24:02.255209+0000","last_became_peered":"2023-06-03T20:24:02.255209+0000","last_unstale":"2023-06-03T20:32:22.750701+0000","last_undegraded":"2023-06-03T20:32:22.750701+0000","last_fullsized":"2023-06-03T20:32:22.750701+0000","mapping_epoch":254,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":255,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T20:05:02.741217+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":53,"reported_epoch":297,"state":"active+clean","last_fresh":"2023-06-03T20:32:22.070908+0000","last_change":"2023-06-03T20:24:02.254172+0000","last_active":"2023-06-03T20:32:22.070908+0000","last_peered":"2023-06-03T20:32:22.070908+0000","last_clean":"2023-06-03T20:32:22.070908+0000","last_became_active":"2023-06-03T20:24:02.253797+0000","last_became_peered":"2023-06-03T20:24:02.253797+0000","last_unstale":"2023-06-03T20:32:22.070908+0000","last_undegraded":"2023-06-03T20:32:22.070908+0000","last_fullsized":"2023-06-03T20:32:22.070908+0000","mapping_epoch":254,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":255,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":219,"reported_epoch":298,"state":"active+clean","last_fresh":"2023-06-03T20:32:22.752713+0000","last_change":"2023-06-03T20:23:28.291250+0000","last_active":"2023-06-03T20:32:22.752713+0000","last_peered":"2023-06-03T20:32:22.752713+0000","last_clean":"2023-06-03T20:32:22.752713+0000","last_became_active":"2023-06-03T20:23:28.290120+0000","last_became_peered":"2023-06-03T20:23:28.290120+0000","last_unstale":"2023-06-03T20:32:22.752713+0000","last_undegraded":"2023-06-03T20:32:22.752713+0000","last_fullsized":"2023-06-03T20:32:22.752713+0000","mapping_epoch":251,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":252,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T22:37:56.033327+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":204,"reported_epoch":297,"state":"active+clean","last_fresh":"2023-06-03T20:32:21.780862+0000","last_change":"2023-06-03T20:23:27.276693+0000","last_active":"2023-06-03T20:32:21.780862+0000","last_peered":"2023-06-03T20:32:21.780862+0000","last_clean":"2023-06-03T20:32:21.780862+0000","last_became_active":"2023-06-03T20:23:27.276305+0000","last_became_peered":"2023-06-03T20:23:27.276305+0000","last_unstale":"2023-06-03T20:32:21.780862+0000","last_undegraded":"2023-06-03T20:32:21.780862+0000","last_fullsized":"2023-06-03T20:32:21.780862+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T20:34:50.944936+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":209,"reported_epoch":298,"state":"active+clean","last_fresh":"2023-06-03T20:32:22.751185+0000","last_change":"2023-06-03T20:28:51.231829+0000","last_active":"2023-06-03T20:32:22.751185+0000","last_peered":"2023-06-03T20:32:22.751185+0000","last_clean":"2023-06-03T20:32:22.751185+0000","last_became_active":"2023-06-03T20:28:51.231452+0000","last_became_peered":"2023-06-03T20:28:51.231452+0000","last_unstale":"2023-06-03T20:32:22.751185+0000","last_undegraded":"2023-06-03T20:32:22.751185+0000","last_fullsized":"2023-06-03T20:32:22.751185+0000","mapping_epoch":286,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":287,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T07:00:21.647058+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":302,"reported_epoch":297,"state":"active+clean","last_fresh":"2023-06-03T20:32:21.411612+0000","last_change":"2023-06-03T20:21:01.256958+0000","last_active":"2023-06-03T20:32:21.411612+0000","last_peered":"2023-06-03T20:32:21.411612+0000","last_clean":"2023-06-03T20:32:21.411612+0000","last_became_active":"2023-06-03T20:21:01.255452+0000","last_became_peered":"2023-06-03T20:21:01.255452+0000","last_unstale":"2023-06-03T20:32:21.411612+0000","last_undegraded":"2023-06-03T20:32:21.411612+0000","last_fullsized":"2023-06-03T20:32:21.411612+0000","mapping_epoch":235,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":236,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T01:56:59.777763+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":89,"reported_epoch":298,"state":"active+clean","last_fresh":"2023-06-03T20:32:22.887013+0000","last_change":"2023-06-03T20:23:28.250884+0000","last_active":"2023-06-03T20:32:22.887013+0000","last_peered":"2023-06-03T20:32:22.887013+0000","last_clean":"2023-06-03T20:32:22.887013+0000","last_became_active":"2023-06-03T20:23:28.243793+0000","last_became_peered":"2023-06-03T20:23:28.243793+0000","last_unstale":"2023-06-03T20:32:22.887013+0000","last_undegraded":"2023-06-03T20:32:22.887013+0000","last_fullsized":"2023-06-03T20:32:22.887013+0000","mapping_epoch":251,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":252,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T03:12:27.946002+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":78,"reported_epoch":297,"state":"active+clean","last_fresh":"2023-06-03T20:32:21.422223+0000","last_change":"2023-06-03T20:21:01.258182+0000","last_active":"2023-06-03T20:32:21.422223+0000","last_peered":"2023-06-03T20:32:21.422223+0000","last_clean":"2023-06-03T20:32:21.422223+0000","last_became_active":"2023-06-03T20:21:01.254049+0000","last_became_peered":"2023-06-03T20:21:01.254049+0000","last_unstale":"2023-06-03T20:32:21.422223+0000","last_undegraded":"2023-06-03T20:32:21.422223+0000","last_fullsized":"2023-06-03T20:32:21.422223+0000","mapping_epoch":235,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":236,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:32:20.692043+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:32:20.692043+0000","last_clean_scrub_stamp":"2023-06-03T19:32:20.692043+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T05:16:30.078473+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":187,"reported_epoch":297,"state":"active+clean","last_fresh":"2023-06-03T20:32:21.412488+0000","last_change":"2023-06-03T20:17:22.834746+0000","last_active":"2023-06-03T20:32:21.412488+0000","last_peered":"2023-06-03T20:32:21.412488+0000","last_clean":"2023-06-03T20:32:21.412488+0000","last_became_active":"2023-06-03T20:17:22.834448+0000","last_became_peered":"2023-06-03T20:17:22.834448+0000","last_unstale":"2023-06-03T20:32:21.412488+0000","last_undegraded":"2023-06-03T20:32:21.412488+0000","last_fullsized":"2023-06-03T20:32:21.412488+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T21:16:31.167063+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,2],"acting":[3,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":89,"reported_epoch":297,"state":"active+clean","last_fresh":"2023-06-03T20:32:21.422705+0000","last_change":"2023-06-03T20:23:28.254434+0000","last_active":"2023-06-03T20:32:21.422705+0000","last_peered":"2023-06-03T20:32:21.422705+0000","last_clean":"2023-06-03T20:32:21.422705+0000","last_became_active":"2023-06-03T20:23:28.254012+0000","last_became_peered":"2023-06-03T20:23:28.254012+0000","last_unstale":"2023-06-03T20:32:21.422705+0000","last_undegraded":"2023-06-03T20:32:21.422705+0000","last_fullsized":"2023-06-03T20:32:21.422705+0000","mapping_epoch":251,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":252,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T06:22:09.540298+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":87,"reported_epoch":297,"state":"active+clean","last_fresh":"2023-06-03T20:32:21.422864+0000","last_change":"2023-06-03T20:28:51.234855+0000","last_active":"2023-06-03T20:32:21.422864+0000","last_peered":"2023-06-03T20:32:21.422864+0000","last_clean":"2023-06-03T20:32:21.422864+0000","last_became_active":"2023-06-03T20:28:51.234514+0000","last_became_peered":"2023-06-03T20:28:51.234514+0000","last_unstale":"2023-06-03T20:32:21.422864+0000","last_undegraded":"2023-06-03T20:32:21.422864+0000","last_fullsized":"2023-06-03T20:32:21.422864+0000","mapping_epoch":286,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":287,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:32:20.692043+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:32:20.692043+0000","last_clean_scrub_stamp":"2023-06-03T19:32:20.692043+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T03:00:56.480804+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":184,"reported_epoch":297,"state":"active+clean","last_fresh":"2023-06-03T20:32:21.781572+0000","last_change":"2023-06-03T20:24:02.258311+0000","last_active":"2023-06-03T20:32:21.781572+0000","last_peered":"2023-06-03T20:32:21.781572+0000","last_clean":"2023-06-03T20:32:21.781572+0000","last_became_active":"2023-06-03T20:24:02.257676+0000","last_became_peered":"2023-06-03T20:24:02.257676+0000","last_unstale":"2023-06-03T20:32:21.781572+0000","last_undegraded":"2023-06-03T20:32:21.781572+0000","last_fullsized":"2023-06-03T20:32:21.781572+0000","mapping_epoch":254,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":255,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T01:47:46.383038+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":78,"reported_epoch":297,"state":"active+clean","last_fresh":"2023-06-03T20:32:21.423067+0000","last_change":"2023-06-03T20:21:01.259331+0000","last_active":"2023-06-03T20:32:21.423067+0000","last_peered":"2023-06-03T20:32:21.423067+0000","last_clean":"2023-06-03T20:32:21.423067+0000","last_became_active":"2023-06-03T20:21:01.258836+0000","last_became_peered":"2023-06-03T20:21:01.258836+0000","last_unstale":"2023-06-03T20:32:21.423067+0000","last_undegraded":"2023-06-03T20:32:21.423067+0000","last_fullsized":"2023-06-03T20:32:21.423067+0000","mapping_epoch":235,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":236,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T01:29:19.471600+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":285,"reported_epoch":297,"state":"active+clean","last_fresh":"2023-06-03T20:32:21.554488+0000","last_change":"2023-06-03T19:56:54.504859+0000","last_active":"2023-06-03T20:32:21.554488+0000","last_peered":"2023-06-03T20:32:21.554488+0000","last_clean":"2023-06-03T20:32:21.554488+0000","last_became_active":"2023-06-03T19:56:54.504455+0000","last_became_peered":"2023-06-03T19:56:54.504455+0000","last_unstale":"2023-06-03T20:32:21.554488+0000","last_undegraded":"2023-06-03T20:32:21.554488+0000","last_fullsized":"2023-06-03T20:32:21.554488+0000","mapping_epoch":119,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:32:20.692043+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:32:20.692043+0000","last_clean_scrub_stamp":"2023-06-03T19:32:20.692043+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T06:25:45.522100+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":87,"reported_epoch":297,"state":"active+clean","last_fresh":"2023-06-03T20:32:21.422661+0000","last_change":"2023-06-03T20:24:02.256677+0000","last_active":"2023-06-03T20:32:21.422661+0000","last_peered":"2023-06-03T20:32:21.422661+0000","last_clean":"2023-06-03T20:32:21.422661+0000","last_became_active":"2023-06-03T20:24:02.256216+0000","last_became_peered":"2023-06-03T20:24:02.256216+0000","last_unstale":"2023-06-03T20:32:21.422661+0000","last_undegraded":"2023-06-03T20:32:21.422661+0000","last_fullsized":"2023-06-03T20:32:21.422661+0000","mapping_epoch":254,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":255,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T02:32:45.186204+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":78,"reported_epoch":297,"state":"active+clean","last_fresh":"2023-06-03T20:32:21.422254+0000","last_change":"2023-06-03T20:21:01.261473+0000","last_active":"2023-06-03T20:32:21.422254+0000","last_peered":"2023-06-03T20:32:21.422254+0000","last_clean":"2023-06-03T20:32:21.422254+0000","last_became_active":"2023-06-03T20:21:01.261098+0000","last_became_peered":"2023-06-03T20:21:01.261098+0000","last_unstale":"2023-06-03T20:32:21.422254+0000","last_undegraded":"2023-06-03T20:32:21.422254+0000","last_fullsized":"2023-06-03T20:32:21.422254+0000","mapping_epoch":235,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":236,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T03:56:22.986549+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":170,"reported_epoch":297,"state":"active+clean","last_fresh":"2023-06-03T20:32:21.554924+0000","last_change":"2023-06-03T20:17:22.834467+0000","last_active":"2023-06-03T20:32:21.554924+0000","last_peered":"2023-06-03T20:32:21.554924+0000","last_clean":"2023-06-03T20:32:21.554924+0000","last_became_active":"2023-06-03T20:17:22.834147+0000","last_became_peered":"2023-06-03T20:17:22.834147+0000","last_unstale":"2023-06-03T20:32:21.554924+0000","last_undegraded":"2023-06-03T20:32:21.554924+0000","last_fullsized":"2023-06-03T20:32:21.554924+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:32:20.692043+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:32:20.692043+0000","last_clean_scrub_stamp":"2023-06-03T19:32:20.692043+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T04:36:29.567051+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,7],"acting":[2,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"72'139","reported_seq":570,"reported_epoch":297,"state":"active+clean","last_fresh":"2023-06-03T20:32:21.318309+0000","last_change":"2023-06-03T20:27:01.248641+0000","last_active":"2023-06-03T20:32:21.318309+0000","last_peered":"2023-06-03T20:32:21.318309+0000","last_clean":"2023-06-03T20:32:21.318309+0000","last_became_active":"2023-06-03T20:27:01.248063+0000","last_became_peered":"2023-06-03T20:27:01.248063+0000","last_unstale":"2023-06-03T20:32:21.318309+0000","last_undegraded":"2023-06-03T20:32:21.318309+0000","last_fullsized":"2023-06-03T20:32:21.318309+0000","mapping_epoch":274,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":275,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:27:28.100808+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:27:28.100808+0000","last_clean_scrub_stamp":"2023-06-03T19:27:28.100808+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":288,"reported_epoch":297,"state":"active+clean","last_fresh":"2023-06-03T20:32:21.412148+0000","last_change":"2023-06-03T20:24:02.263360+0000","last_active":"2023-06-03T20:32:21.412148+0000","last_peered":"2023-06-03T20:32:21.412148+0000","last_clean":"2023-06-03T20:32:21.412148+0000","last_became_active":"2023-06-03T20:24:02.262954+0000","last_became_peered":"2023-06-03T20:24:02.262954+0000","last_unstale":"2023-06-03T20:32:21.412148+0000","last_undegraded":"2023-06-03T20:32:21.412148+0000","last_fullsized":"2023-06-03T20:32:21.412148+0000","mapping_epoch":254,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":255,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T19:38:46.084731+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":21,"reported_epoch":297,"state":"active+clean","last_fresh":"2023-06-03T20:32:22.071060+0000","last_change":"2023-06-03T20:28:51.234600+0000","last_active":"2023-06-03T20:32:22.071060+0000","last_peered":"2023-06-03T20:32:22.071060+0000","last_clean":"2023-06-03T20:32:22.071060+0000","last_became_active":"2023-06-03T20:28:51.234227+0000","last_became_peered":"2023-06-03T20:28:51.234227+0000","last_unstale":"2023-06-03T20:32:22.071060+0000","last_undegraded":"2023-06-03T20:32:22.071060+0000","last_fullsized":"2023-06-03T20:32:22.071060+0000","mapping_epoch":286,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":287,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":290,"reported_epoch":297,"state":"active+clean","last_fresh":"2023-06-03T20:32:21.317823+0000","last_change":"2023-06-03T20:21:01.258113+0000","last_active":"2023-06-03T20:32:21.317823+0000","last_peered":"2023-06-03T20:32:21.317823+0000","last_clean":"2023-06-03T20:32:21.317823+0000","last_became_active":"2023-06-03T20:21:01.257488+0000","last_became_peered":"2023-06-03T20:21:01.257488+0000","last_unstale":"2023-06-03T20:32:21.317823+0000","last_undegraded":"2023-06-03T20:32:21.317823+0000","last_fullsized":"2023-06-03T20:32:21.317823+0000","mapping_epoch":235,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":236,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:32:20.692043+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:32:20.692043+0000","last_clean_scrub_stamp":"2023-06-03T19:32:20.692043+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":297,"reported_epoch":297,"state":"active+clean","last_fresh":"2023-06-03T20:32:22.073156+0000","last_change":"2023-06-03T20:23:27.255164+0000","last_active":"2023-06-03T20:32:22.073156+0000","last_peered":"2023-06-03T20:32:22.073156+0000","last_clean":"2023-06-03T20:32:22.073156+0000","last_became_active":"2023-06-03T20:23:27.253932+0000","last_became_peered":"2023-06-03T20:23:27.253932+0000","last_unstale":"2023-06-03T20:32:22.073156+0000","last_undegraded":"2023-06-03T20:32:22.073156+0000","last_fullsized":"2023-06-03T20:32:22.073156+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T06:42:02.303554+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":301,"reported_epoch":298,"state":"active+clean","last_fresh":"2023-06-03T20:32:22.893361+0000","last_change":"2023-06-03T20:21:01.250266+0000","last_active":"2023-06-03T20:32:22.893361+0000","last_peered":"2023-06-03T20:32:22.893361+0000","last_clean":"2023-06-03T20:32:22.893361+0000","last_became_active":"2023-06-03T20:21:01.249839+0000","last_became_peered":"2023-06-03T20:21:01.249839+0000","last_unstale":"2023-06-03T20:32:22.893361+0000","last_undegraded":"2023-06-03T20:32:22.893361+0000","last_fullsized":"2023-06-03T20:32:22.893361+0000","mapping_epoch":235,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":236,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T05:13:26.584607+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":299,"reported_epoch":297,"state":"active+clean","last_fresh":"2023-06-03T20:32:21.319225+0000","last_change":"2023-06-03T20:23:27.261511+0000","last_active":"2023-06-03T20:32:21.319225+0000","last_peered":"2023-06-03T20:32:21.319225+0000","last_clean":"2023-06-03T20:32:21.319225+0000","last_became_active":"2023-06-03T20:23:27.259622+0000","last_became_peered":"2023-06-03T20:23:27.259622+0000","last_unstale":"2023-06-03T20:32:21.319225+0000","last_undegraded":"2023-06-03T20:32:21.319225+0000","last_fullsized":"2023-06-03T20:32:21.319225+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:32:20.692043+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:32:20.692043+0000","last_clean_scrub_stamp":"2023-06-03T19:32:20.692043+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":78,"reported_epoch":297,"state":"active+clean","last_fresh":"2023-06-03T20:32:21.422227+0000","last_change":"2023-06-03T20:21:01.252859+0000","last_active":"2023-06-03T20:32:21.422227+0000","last_peered":"2023-06-03T20:32:21.422227+0000","last_clean":"2023-06-03T20:32:21.422227+0000","last_became_active":"2023-06-03T20:21:01.251222+0000","last_became_peered":"2023-06-03T20:21:01.251222+0000","last_unstale":"2023-06-03T20:32:21.422227+0000","last_undegraded":"2023-06-03T20:32:21.422227+0000","last_fullsized":"2023-06-03T20:32:21.422227+0000","mapping_epoch":235,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":236,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T05:19:58.100226+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":253,"reported_epoch":297,"state":"active+clean","last_fresh":"2023-06-03T20:32:22.073050+0000","last_change":"2023-06-03T19:47:06.733532+0000","last_active":"2023-06-03T20:32:22.073050+0000","last_peered":"2023-06-03T20:32:22.073050+0000","last_clean":"2023-06-03T20:32:22.073050+0000","last_became_active":"2023-06-03T19:47:06.733131+0000","last_became_peered":"2023-06-03T19:47:06.733131+0000","last_unstale":"2023-06-03T20:32:22.073050+0000","last_undegraded":"2023-06-03T20:32:22.073050+0000","last_fullsized":"2023-06-03T20:32:22.073050+0000","mapping_epoch":86,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":87,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T23:46:03.788646+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.7","version":"60'2","reported_seq":263,"reported_epoch":297,"state":"active+clean","last_fresh":"2023-06-03T20:32:21.412757+0000","last_change":"2023-06-03T19:56:54.510148+0000","last_active":"2023-06-03T20:32:21.412757+0000","last_peered":"2023-06-03T20:32:21.412757+0000","last_clean":"2023-06-03T20:32:21.412757+0000","last_became_active":"2023-06-03T19:56:54.509771+0000","last_became_peered":"2023-06-03T19:56:54.509771+0000","last_unstale":"2023-06-03T20:32:21.412757+0000","last_undegraded":"2023-06-03T20:32:21.412757+0000","last_fullsized":"2023-06-03T20:32:21.412757+0000","mapping_epoch":119,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T00:52:42.807432+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":96,"reported_epoch":297,"state":"active+clean","last_fresh":"2023-06-03T20:32:21.423248+0000","last_change":"2023-06-03T20:24:02.257142+0000","last_active":"2023-06-03T20:32:21.423248+0000","last_peered":"2023-06-03T20:32:21.423248+0000","last_clean":"2023-06-03T20:32:21.423248+0000","last_became_active":"2023-06-03T20:24:02.256483+0000","last_became_peered":"2023-06-03T20:24:02.256483+0000","last_unstale":"2023-06-03T20:32:21.423248+0000","last_undegraded":"2023-06-03T20:32:21.423248+0000","last_fullsized":"2023-06-03T20:32:21.423248+0000","mapping_epoch":254,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":255,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:32:20.692043+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:32:20.692043+0000","last_clean_scrub_stamp":"2023-06-03T19:32:20.692043+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T00:53:37.197795+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":261,"reported_epoch":297,"state":"active+clean","last_fresh":"2023-06-03T20:32:21.411818+0000","last_change":"2023-06-03T19:56:54.507670+0000","last_active":"2023-06-03T20:32:21.411818+0000","last_peered":"2023-06-03T20:32:21.411818+0000","last_clean":"2023-06-03T20:32:21.411818+0000","last_became_active":"2023-06-03T19:56:54.507293+0000","last_became_peered":"2023-06-03T19:56:54.507293+0000","last_unstale":"2023-06-03T20:32:21.411818+0000","last_undegraded":"2023-06-03T20:32:21.411818+0000","last_fullsized":"2023-06-03T20:32:21.411818+0000","mapping_epoch":119,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T01:49:19.966054+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":53,"reported_epoch":297,"state":"active+clean","last_fresh":"2023-06-03T20:32:22.071313+0000","last_change":"2023-06-03T20:24:02.319230+0000","last_active":"2023-06-03T20:32:22.071313+0000","last_peered":"2023-06-03T20:32:22.071313+0000","last_clean":"2023-06-03T20:32:22.071313+0000","last_became_active":"2023-06-03T20:24:02.318926+0000","last_became_peered":"2023-06-03T20:24:02.318926+0000","last_unstale":"2023-06-03T20:32:22.071313+0000","last_undegraded":"2023-06-03T20:32:22.071313+0000","last_fullsized":"2023-06-03T20:32:22.071313+0000","mapping_epoch":254,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":255,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:32:20.692043+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:32:20.692043+0000","last_clean_scrub_stamp":"2023-06-03T19:32:20.692043+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"60'1","reported_seq":329,"reported_epoch":298,"state":"active+clean","last_fresh":"2023-06-03T20:32:22.893226+0000","last_change":"2023-06-03T20:23:28.239664+0000","last_active":"2023-06-03T20:32:22.893226+0000","last_peered":"2023-06-03T20:32:22.893226+0000","last_clean":"2023-06-03T20:32:22.893226+0000","last_became_active":"2023-06-03T20:23:28.239252+0000","last_became_peered":"2023-06-03T20:23:28.239252+0000","last_unstale":"2023-06-03T20:32:22.893226+0000","last_undegraded":"2023-06-03T20:32:22.893226+0000","last_fullsized":"2023-06-03T20:32:22.893226+0000","mapping_epoch":251,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":252,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T06:52:20.862520+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":182,"reported_epoch":297,"state":"active+clean","last_fresh":"2023-06-03T20:32:21.780623+0000","last_change":"2023-06-03T20:24:02.259850+0000","last_active":"2023-06-03T20:32:21.780623+0000","last_peered":"2023-06-03T20:32:21.780623+0000","last_clean":"2023-06-03T20:32:21.780623+0000","last_became_active":"2023-06-03T20:24:02.258764+0000","last_became_peered":"2023-06-03T20:24:02.258764+0000","last_unstale":"2023-06-03T20:32:21.780623+0000","last_undegraded":"2023-06-03T20:32:21.780623+0000","last_fullsized":"2023-06-03T20:32:21.780623+0000","mapping_epoch":254,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":255,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T21:54:11.487473+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"60'1","reported_seq":283,"reported_epoch":297,"state":"active+clean","last_fresh":"2023-06-03T20:32:22.072961+0000","last_change":"2023-06-03T20:23:27.254853+0000","last_active":"2023-06-03T20:32:22.072961+0000","last_peered":"2023-06-03T20:32:22.072961+0000","last_clean":"2023-06-03T20:32:22.072961+0000","last_became_active":"2023-06-03T20:23:27.254410+0000","last_became_peered":"2023-06-03T20:23:27.254410+0000","last_unstale":"2023-06-03T20:32:22.072961+0000","last_undegraded":"2023-06-03T20:32:22.072961+0000","last_fullsized":"2023-06-03T20:32:22.072961+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T22:46:19.423939+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":312,"reported_epoch":297,"state":"active+clean","last_fresh":"2023-06-03T20:32:21.554866+0000","last_change":"2023-06-03T20:21:01.257656+0000","last_active":"2023-06-03T20:32:21.554866+0000","last_peered":"2023-06-03T20:32:21.554866+0000","last_clean":"2023-06-03T20:32:21.554866+0000","last_became_active":"2023-06-03T20:21:01.257107+0000","last_became_peered":"2023-06-03T20:21:01.257107+0000","last_unstale":"2023-06-03T20:32:21.554866+0000","last_undegraded":"2023-06-03T20:32:21.554866+0000","last_fullsized":"2023-06-03T20:32:21.554866+0000","mapping_epoch":235,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":236,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T06:08:18.954539+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"60'2","reported_seq":73,"reported_epoch":297,"state":"active+clean","last_fresh":"2023-06-03T20:32:21.781029+0000","last_change":"2023-06-03T20:23:28.243048+0000","last_active":"2023-06-03T20:32:21.781029+0000","last_peered":"2023-06-03T20:32:21.781029+0000","last_clean":"2023-06-03T20:32:21.781029+0000","last_became_active":"2023-06-03T20:23:28.241128+0000","last_became_peered":"2023-06-03T20:23:28.241128+0000","last_unstale":"2023-06-03T20:32:21.781029+0000","last_undegraded":"2023-06-03T20:32:21.781029+0000","last_fullsized":"2023-06-03T20:32:21.781029+0000","mapping_epoch":251,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":252,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T05:14:54.701329+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,1],"acting":[4,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":236,"reported_epoch":297,"state":"active+clean","last_fresh":"2023-06-03T20:32:21.781145+0000","last_change":"2023-06-03T20:28:24.276513+0000","last_active":"2023-06-03T20:32:21.781145+0000","last_peered":"2023-06-03T20:32:21.781145+0000","last_clean":"2023-06-03T20:32:21.781145+0000","last_became_active":"2023-06-03T20:28:24.276244+0000","last_became_peered":"2023-06-03T20:28:24.276244+0000","last_unstale":"2023-06-03T20:32:21.781145+0000","last_undegraded":"2023-06-03T20:32:21.781145+0000","last_fullsized":"2023-06-03T20:32:21.781145+0000","mapping_epoch":284,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":285,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T06:17:57.159168+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[4,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"60'2","reported_seq":86,"reported_epoch":297,"state":"active+clean","last_fresh":"2023-06-03T20:32:21.781641+0000","last_change":"2023-06-03T20:28:51.237342+0000","last_active":"2023-06-03T20:32:21.781641+0000","last_peered":"2023-06-03T20:32:21.781641+0000","last_clean":"2023-06-03T20:32:21.781641+0000","last_became_active":"2023-06-03T20:28:51.233969+0000","last_became_peered":"2023-06-03T20:28:51.233969+0000","last_unstale":"2023-06-03T20:32:21.781641+0000","last_undegraded":"2023-06-03T20:32:21.781641+0000","last_fullsized":"2023-06-03T20:32:21.781641+0000","mapping_epoch":286,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":287,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T07:30:09.503545+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":220,"reported_epoch":297,"state":"active+clean","last_fresh":"2023-06-03T20:32:21.780662+0000","last_change":"2023-06-03T20:23:27.254468+0000","last_active":"2023-06-03T20:32:21.780662+0000","last_peered":"2023-06-03T20:32:21.780662+0000","last_clean":"2023-06-03T20:32:21.780662+0000","last_became_active":"2023-06-03T20:23:27.253859+0000","last_became_peered":"2023-06-03T20:23:27.253859+0000","last_unstale":"2023-06-03T20:32:21.780662+0000","last_undegraded":"2023-06-03T20:32:21.780662+0000","last_fullsized":"2023-06-03T20:32:21.780662+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T20:15:25.066125+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"60'1","reported_seq":317,"reported_epoch":297,"state":"active+clean","last_fresh":"2023-06-03T20:32:21.412649+0000","last_change":"2023-06-03T20:28:51.234748+0000","last_active":"2023-06-03T20:32:21.412649+0000","last_peered":"2023-06-03T20:32:21.412649+0000","last_clean":"2023-06-03T20:32:21.412649+0000","last_became_active":"2023-06-03T20:28:51.234341+0000","last_became_peered":"2023-06-03T20:28:51.234341+0000","last_unstale":"2023-06-03T20:32:21.412649+0000","last_undegraded":"2023-06-03T20:32:21.412649+0000","last_fullsized":"2023-06-03T20:32:21.412649+0000","mapping_epoch":286,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":287,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T06:08:49.901424+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":300,"reported_epoch":297,"state":"active+clean","last_fresh":"2023-06-03T20:32:21.412049+0000","last_change":"2023-06-03T19:56:54.503416+0000","last_active":"2023-06-03T20:32:21.412049+0000","last_peered":"2023-06-03T20:32:21.412049+0000","last_clean":"2023-06-03T20:32:21.412049+0000","last_became_active":"2023-06-03T19:56:54.503018+0000","last_became_peered":"2023-06-03T19:56:54.503018+0000","last_unstale":"2023-06-03T20:32:21.412049+0000","last_undegraded":"2023-06-03T20:32:21.412049+0000","last_fullsized":"2023-06-03T20:32:21.412049+0000","mapping_epoch":119,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T20:14:00.330616+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"60'4","reported_seq":271,"reported_epoch":297,"state":"active+clean","last_fresh":"2023-06-03T20:32:22.073286+0000","last_change":"2023-06-03T19:56:11.748116+0000","last_active":"2023-06-03T20:32:22.073286+0000","last_peered":"2023-06-03T20:32:22.073286+0000","last_clean":"2023-06-03T20:32:22.073286+0000","last_became_active":"2023-06-03T19:56:11.747636+0000","last_became_peered":"2023-06-03T19:56:11.747636+0000","last_unstale":"2023-06-03T20:32:22.073286+0000","last_undegraded":"2023-06-03T20:32:22.073286+0000","last_fullsized":"2023-06-03T20:32:22.073286+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":115,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T06:57:48.052055+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":220,"reported_epoch":297,"state":"active+clean","last_fresh":"2023-06-03T20:32:21.780483+0000","last_change":"2023-06-03T20:28:51.231879+0000","last_active":"2023-06-03T20:32:21.780483+0000","last_peered":"2023-06-03T20:32:21.780483+0000","last_clean":"2023-06-03T20:32:21.780483+0000","last_became_active":"2023-06-03T20:28:51.231451+0000","last_became_peered":"2023-06-03T20:28:51.231451+0000","last_unstale":"2023-06-03T20:32:21.780483+0000","last_undegraded":"2023-06-03T20:32:21.780483+0000","last_fullsized":"2023-06-03T20:32:21.780483+0000","mapping_epoch":286,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":287,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T05:47:15.344575+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5486,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":62,"num_write_kb":30,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":23,"num_bytes_recovered":4856,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16458,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":30319,"internal_metadata":0},"log_size":56,"ondisk_log_size":56,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2969600,"data_stored":2951840,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":3,"num_store_stats":5},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":48,"seq":206158430948,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":975808,"kb_used_data":3320,"kb_used_omap":0,"kb_used_meta":972480,"kb_avail":92769344,"statfs":{"total":95995035648,"available":94995808256,"internally_reserved":0,"allocated":3399680,"data_stored":2731073,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":995819520},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 3 20:32:38 2023","interfaces":[{"interface":"back","average":{"1min":1.0169999999999999,"5min":2.149,"15min":1.8640000000000001},"min":{"1min":0.39100000000000001,"5min":0.26800000000000002,"15min":0.26400000000000001},"max":{"1min":2.4569999999999999,"5min":59.356000000000002,"15min":73.810000000000002},"last":0.77500000000000002},{"interface":"front","average":{"1min":0.83099999999999996,"5min":1.907,"15min":1.772},"min":{"1min":0.45800000000000002,"5min":0.29999999999999999,"15min":0.26700000000000002},"max":{"1min":1.2490000000000001,"5min":58.725999999999999,"15min":72.878},"last":0.93200000000000005}]},{"osd":1,"last update":"Sat Jun 3 20:32:22 2023","interfaces":[{"interface":"back","average":{"1min":0.84999999999999998,"5min":1.119,"15min":1.595},"min":{"1min":0.30299999999999999,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":1.9039999999999999,"5min":4.2000000000000002,"15min":100.845},"last":0.66200000000000003},{"interface":"front","average":{"1min":1.361,"5min":1.4590000000000001,"15min":1.7729999999999999},"min":{"1min":0.38900000000000001,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":4.0789999999999997,"5min":7.6559999999999997,"15min":100.86499999999999},"last":0.71299999999999997}]},{"osd":2,"last update":"Sat Jun 3 20:32:22 2023","interfaces":[{"interface":"back","average":{"1min":1.1719999999999999,"5min":1.4630000000000001,"15min":2.5670000000000002},"min":{"1min":0.34300000000000003,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":4.484,"5min":16.809999999999999,"15min":101.185},"last":0.88700000000000001},{"interface":"front","average":{"1min":1.972,"5min":1.7809999999999999,"15min":2.7519999999999998},"min":{"1min":0.56000000000000005,"5min":0.33200000000000002,"15min":0.315},"max":{"1min":7.2850000000000001,"5min":16.587,"15min":100.61799999999999},"last":1.022}]},{"osd":3,"last update":"Sat Jun 3 20:32:22 2023","interfaces":[{"interface":"back","average":{"1min":1.3160000000000001,"5min":1.232,"15min":1.8779999999999999},"min":{"1min":0.372,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":4.774,"5min":6.3150000000000004,"15min":74.039000000000001},"last":0.98599999999999999},{"interface":"front","average":{"1min":1.7390000000000001,"5min":1.5680000000000001,"15min":1.986},"min":{"1min":0.45300000000000001,"5min":0.38900000000000001,"15min":0.35399999999999998},"max":{"1min":5.3920000000000003,"5min":8.2360000000000007,"15min":73.001999999999995},"last":0.84899999999999998}]},{"osd":4,"last update":"Sat Jun 3 20:32:42 2023","interfaces":[{"interface":"back","average":{"1min":1.0469999999999999,"5min":1.1240000000000001,"15min":1.8919999999999999},"min":{"1min":0.64600000000000002,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":1.351,"5min":7.718,"15min":8.6010000000000009},"last":1.1579999999999999},{"interface":"front","average":{"1min":1.121,"5min":1.2230000000000001,"15min":1.784},"min":{"1min":0.442,"5min":0.35599999999999998,"15min":0.27600000000000002},"max":{"1min":3.9169999999999998,"5min":8.9459999999999997,"15min":9.8460000000000001},"last":1.137}]},{"osd":5,"last update":"Sat Jun 3 20:32:22 2023","interfaces":[{"interface":"back","average":{"1min":1.2749999999999999,"5min":1.0900000000000001,"15min":1.298},"min":{"1min":0.40699999999999997,"5min":0.317,"15min":0.317},"max":{"1min":4.1799999999999997,"5min":9.1609999999999996,"15min":17.696000000000002},"last":1.077},{"interface":"front","average":{"1min":1.105,"5min":1.095,"15min":1.3140000000000001},"min":{"1min":0.51200000000000001,"5min":0.33600000000000002,"15min":0.29399999999999998},"max":{"1min":4.032,"5min":7.7990000000000004,"15min":9.3710000000000004},"last":1.1020000000000001}]},{"osd":6,"last update":"Sat Jun 3 20:32:22 2023","interfaces":[{"interface":"back","average":{"1min":1.3120000000000001,"5min":1.1040000000000001,"15min":1.1659999999999999},"min":{"1min":0.35899999999999999,"5min":0.35899999999999999,"15min":0.35899999999999999},"max":{"1min":4.1680000000000001,"5min":7.7750000000000004,"15min":9.7200000000000006},"last":1.216},{"interface":"front","average":{"1min":1.681,"5min":1.2390000000000001,"15min":1.1659999999999999},"min":{"1min":0.54400000000000004,"5min":0.377,"15min":0.34300000000000003},"max":{"1min":5.5940000000000003,"5min":8.8770000000000007,"15min":9.923},"last":1.0429999999999999}]}]},{"osd":6,"up_from":42,"seq":180388627181,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":623152,"kb_used_data":3304,"kb_used_omap":0,"kb_used_meta":619840,"kb_avail":93122000,"statfs":{"total":95995035648,"available":95356928000,"internally_reserved":0,"allocated":3383296,"data_stored":2730612,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":634716160},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 3 20:31:46 2023","interfaces":[{"interface":"back","average":{"1min":9.0449999999999999,"5min":3.8149999999999999,"15min":2.129},"min":{"1min":0.26300000000000001,"5min":0.26300000000000001,"15min":0.254},"max":{"1min":109.90900000000001,"5min":109.90900000000001,"15min":109.90900000000001},"last":1.1299999999999999},{"interface":"front","average":{"1min":9.0939999999999994,"5min":4.0659999999999998,"15min":2.258},"min":{"1min":0.36099999999999999,"5min":0.315,"15min":0.315},"max":{"1min":109.86499999999999,"5min":109.86499999999999,"15min":109.86499999999999},"last":0.84699999999999998}]},{"osd":1,"last update":"Sat Jun 3 20:32:22 2023","interfaces":[{"interface":"back","average":{"1min":1.1659999999999999,"5min":2.589,"15min":1.5720000000000001},"min":{"1min":0.312,"5min":0.29999999999999999,"15min":0.29199999999999998},"max":{"1min":4.1470000000000002,"5min":107.90600000000001,"15min":107.90600000000001},"last":0.98999999999999999},{"interface":"front","average":{"1min":1.3879999999999999,"5min":2.9550000000000001,"15min":1.71},"min":{"1min":0.36599999999999999,"5min":0.34200000000000003,"15min":0.32700000000000001},"max":{"1min":4.2670000000000003,"5min":107.79900000000001,"15min":107.79900000000001},"last":1.044}]},{"osd":2,"last update":"Sat Jun 3 20:32:22 2023","interfaces":[{"interface":"back","average":{"1min":2.0619999999999998,"5min":3.2679999999999998,"15min":2.887},"min":{"1min":0.38,"5min":0.34899999999999998,"15min":0.33500000000000002},"max":{"1min":15.206,"5min":107.61199999999999,"15min":107.61199999999999},"last":0.92000000000000004},{"interface":"front","average":{"1min":2.0030000000000001,"5min":3.4100000000000001,"15min":2.891},"min":{"1min":0.46500000000000002,"5min":0.35699999999999998,"15min":0.29399999999999998},"max":{"1min":15.890000000000001,"5min":107.83,"15min":107.83},"last":0.81899999999999995}]},{"osd":3,"last update":"Sat Jun 3 20:32:05 2023","interfaces":[{"interface":"back","average":{"1min":8.0389999999999997,"5min":2.9740000000000002,"15min":2.1640000000000001},"min":{"1min":0.35499999999999998,"5min":0.35499999999999998,"15min":0.35499999999999998},"max":{"1min":108.254,"5min":108.254,"15min":108.254},"last":0.76200000000000001},{"interface":"front","average":{"1min":7.2599999999999998,"5min":3.073,"15min":2.2429999999999999},"min":{"1min":0.443,"5min":0.39100000000000001,"15min":0.39100000000000001},"max":{"1min":107.67700000000001,"5min":107.67700000000001,"15min":107.67700000000001},"last":1.0669999999999999}]},{"osd":4,"last update":"Sat Jun 3 20:31:42 2023","interfaces":[{"interface":"back","average":{"1min":1.51,"5min":1.357,"15min":1.746},"min":{"1min":0.45400000000000001,"5min":0.33000000000000002,"15min":0.33000000000000002},"max":{"1min":7.2640000000000002,"5min":8.0429999999999993,"15min":10.574999999999999},"last":0.70599999999999996},{"interface":"front","average":{"1min":1.4139999999999999,"5min":1.127,"15min":1.3620000000000001},"min":{"1min":0.42299999999999999,"5min":0.40699999999999997,"15min":0.34499999999999997},"max":{"1min":6.5590000000000002,"5min":6.6710000000000003,"15min":12.622999999999999},"last":1.0109999999999999}]},{"osd":5,"last update":"Sat Jun 3 20:32:38 2023","interfaces":[{"interface":"back","average":{"1min":1.056,"5min":1.2130000000000001,"15min":1.1200000000000001},"min":{"1min":0.49299999999999999,"5min":0.36299999999999999,"15min":0.29899999999999999},"max":{"1min":1.4219999999999999,"5min":9.0229999999999997,"15min":10.625999999999999},"last":1.181},{"interface":"front","average":{"1min":0.93000000000000005,"5min":1.391,"15min":1.26},"min":{"1min":0.45700000000000002,"5min":0.40999999999999998,"15min":0.35299999999999998},"max":{"1min":1.3460000000000001,"5min":8.0199999999999996,"15min":10.856},"last":0.96499999999999997}]},{"osd":7,"last update":"Sat Jun 3 20:32:08 2023","interfaces":[{"interface":"back","average":{"1min":0.83199999999999996,"5min":1.083,"15min":1.212},"min":{"1min":0.442,"5min":0.41299999999999998,"15min":0.37},"max":{"1min":1.4299999999999999,"5min":9.0519999999999996,"15min":9.0519999999999996},"last":1.264},{"interface":"front","average":{"1min":0.998,"5min":1.04,"15min":1.153},"min":{"1min":0.46800000000000003,"5min":0.43099999999999999,"15min":0.38},"max":{"1min":2.819,"5min":7.2850000000000001,"15min":10.52},"last":1.24}]}]},{"osd":1,"up_from":103,"seq":442381631979,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":485316,"kb_used_data":3260,"kb_used_omap":4,"kb_used_meta":482043,"kb_avail":93259836,"statfs":{"total":95995035648,"available":95498072064,"internally_reserved":0,"allocated":3338240,"data_stored":2200705,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":493612942},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 3 20:32:27 2023","interfaces":[{"interface":"back","average":{"1min":1.389,"5min":6.3470000000000004,"15min":3.8340000000000001},"min":{"1min":0.20499999999999999,"5min":0.19800000000000001,"15min":0.17100000000000001},"max":{"1min":6.7329999999999997,"5min":131.83600000000001,"15min":131.83600000000001},"last":0.45400000000000001},{"interface":"front","average":{"1min":1.5660000000000001,"5min":6.4180000000000001,"15min":3.8700000000000001},"min":{"1min":0.26300000000000001,"5min":0.182,"15min":0.182},"max":{"1min":8.2699999999999996,"5min":131.49100000000001,"15min":131.49100000000001},"last":0.496}]},{"osd":2,"last update":"Sat Jun 3 20:32:27 2023","interfaces":[{"interface":"back","average":{"1min":1.4890000000000001,"5min":3.347,"15min":2.8620000000000001},"min":{"1min":0.251,"5min":0.20599999999999999,"15min":0.20599999999999999},"max":{"1min":7.9370000000000003,"5min":131.858,"15min":131.858},"last":0.54100000000000004},{"interface":"front","average":{"1min":2.2200000000000002,"5min":3.371,"15min":2.9590000000000001},"min":{"1min":0.42599999999999999,"5min":0.246,"15min":0.22700000000000001},"max":{"1min":7.109,"5min":131.63,"15min":131.63},"last":0.47899999999999998}]},{"osd":3,"last update":"Sat Jun 3 20:32:01 2023","interfaces":[{"interface":"back","average":{"1min":9.8569999999999993,"5min":4.2450000000000001,"15min":2.9590000000000001},"min":{"1min":0.42499999999999999,"5min":0.255,"15min":0.23400000000000001},"max":{"1min":131.65199999999999,"5min":131.65199999999999,"15min":131.65199999999999},"last":0.56799999999999995},{"interface":"front","average":{"1min":9.516,"5min":4.1929999999999996,"15min":3.056},"min":{"1min":0.33300000000000002,"5min":0.223,"15min":0.223},"max":{"1min":131.60599999999999,"5min":131.60599999999999,"15min":131.60599999999999},"last":0.59199999999999997}]},{"osd":4,"last update":"Sat Jun 3 20:32:42 2023","interfaces":[{"interface":"back","average":{"1min":1.012,"5min":1.6060000000000001,"15min":1.718},"min":{"1min":0.376,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":2.3839999999999999,"5min":6.8220000000000001,"15min":11.114000000000001},"last":0.95399999999999996},{"interface":"front","average":{"1min":1.2,"5min":1.7090000000000001,"15min":1.5529999999999999},"min":{"1min":0.38900000000000001,"5min":0.35199999999999998,"15min":0.33400000000000002},"max":{"1min":4.2320000000000002,"5min":9.1270000000000007,"15min":11.247999999999999},"last":1.0289999999999999}]},{"osd":5,"last update":"Sat Jun 3 20:32:27 2023","interfaces":[{"interface":"back","average":{"1min":1.135,"5min":1.6419999999999999,"15min":1.4450000000000001},"min":{"1min":0.34300000000000003,"5min":0.34300000000000003,"15min":0.30299999999999999},"max":{"1min":4.992,"5min":8.8330000000000002,"15min":11.246},"last":1.048},{"interface":"front","average":{"1min":1.3799999999999999,"5min":3.0430000000000001,"15min":1.873},"min":{"1min":0.35999999999999999,"5min":0.35799999999999998,"15min":0.35799999999999998},"max":{"1min":4.3120000000000003,"5min":131.25299999999999,"15min":131.25299999999999},"last":1.101}]},{"osd":6,"last update":"Sat Jun 3 20:32:27 2023","interfaces":[{"interface":"back","average":{"1min":1.974,"5min":1.964,"15min":1.3089999999999999},"min":{"1min":0.36399999999999999,"5min":0.36399999999999999,"15min":0.36399999999999999},"max":{"1min":8.9860000000000007,"5min":9.1980000000000004,"15min":10.002000000000001},"last":1.071},{"interface":"front","average":{"1min":2.4460000000000002,"5min":1.7829999999999999,"15min":1.4019999999999999},"min":{"1min":0.70199999999999996,"5min":0.375,"15min":0.36899999999999999},"max":{"1min":8.7889999999999997,"5min":8.7889999999999997,"15min":11.067},"last":0.98199999999999998}]},{"osd":7,"last update":"Sat Jun 3 20:32:27 2023","interfaces":[{"interface":"back","average":{"1min":2.02,"5min":1.671,"15min":1.5589999999999999},"min":{"1min":0.63300000000000001,"5min":0.377,"15min":0.372},"max":{"1min":5.7619999999999996,"5min":8.7010000000000005,"15min":10.538},"last":0.93400000000000005},{"interface":"front","average":{"1min":1.7629999999999999,"5min":1.7030000000000001,"15min":1.415},"min":{"1min":0.45800000000000002,"5min":0.40999999999999998,"15min":0.34000000000000002},"max":{"1min":5.101,"5min":7.5739999999999998,"15min":14.025},"last":1.016}]}]},{"osd":0,"up_from":86,"seq":369367188005,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":781196,"kb_used_data":3908,"kb_used_omap":5,"kb_used_meta":777274,"kb_avail":92963956,"statfs":{"total":95995035648,"available":95195090944,"internally_reserved":0,"allocated":4001792,"data_stored":2799278,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":795929210},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Jun 3 20:32:17 2023","interfaces":[{"interface":"back","average":{"1min":11.513999999999999,"5min":5.4489999999999998,"15min":3.206},"min":{"1min":0.34399999999999997,"5min":0.19500000000000001,"15min":0.185},"max":{"1min":153.53999999999999,"5min":156.43600000000001,"15min":243.03200000000001},"last":0.58899999999999997},{"interface":"front","average":{"1min":10.923,"5min":6.117,"15min":3.5419999999999998},"min":{"1min":0.33100000000000002,"5min":0.188,"15min":0.188},"max":{"1min":153.22999999999999,"5min":157.489,"15min":242.78399999999999},"last":0.63}]},{"osd":2,"last update":"Sat Jun 3 20:32:17 2023","interfaces":[{"interface":"back","average":{"1min":10.669,"5min":3.2850000000000001,"15min":2.2170000000000001},"min":{"1min":0.34899999999999998,"5min":0.219,"15min":0.218},"max":{"1min":153.01300000000001,"5min":153.01300000000001,"15min":153.01300000000001},"last":0.56599999999999995},{"interface":"front","average":{"1min":12.307,"5min":4.8879999999999999,"15min":3.7490000000000001},"min":{"1min":0.40000000000000002,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":153.482,"5min":153.482,"15min":243.30099999999999},"last":0.68899999999999995}]},{"osd":3,"last update":"Sat Jun 3 20:31:59 2023","interfaces":[{"interface":"back","average":{"1min":9.2759999999999998,"5min":3.161,"15min":4.0330000000000004},"min":{"1min":0.46800000000000003,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":152.923,"5min":152.923,"15min":259.68400000000003},"last":0.61799999999999999},{"interface":"front","average":{"1min":5.7880000000000003,"5min":2.476,"15min":3.0209999999999999},"min":{"1min":0.245,"5min":0.245,"15min":0.23799999999999999},"max":{"1min":61.957999999999998,"5min":61.957999999999998,"15min":261.02699999999999},"last":0.72299999999999998}]},{"osd":4,"last update":"Sat Jun 3 20:31:42 2023","interfaces":[{"interface":"back","average":{"1min":2.395,"5min":2.1829999999999998,"15min":1.7569999999999999},"min":{"1min":0.35899999999999999,"5min":0.34200000000000003,"15min":0.32400000000000001},"max":{"1min":5.694,"5min":11.49,"15min":11.49},"last":0.90500000000000003},{"interface":"front","average":{"1min":1.4319999999999999,"5min":1.1659999999999999,"15min":1.5189999999999999},"min":{"1min":0.39400000000000002,"5min":0.34100000000000003,"15min":0.34100000000000003},"max":{"1min":7.5060000000000002,"5min":7.5060000000000002,"15min":7.5060000000000002},"last":1.1040000000000001}]},{"osd":5,"last update":"Sat Jun 3 20:32:34 2023","interfaces":[{"interface":"back","average":{"1min":1.228,"5min":2.1749999999999998,"15min":2.4540000000000002},"min":{"1min":0.51600000000000001,"5min":0.33600000000000002,"15min":0.29299999999999998},"max":{"1min":4.7649999999999997,"5min":11.445,"15min":243.15899999999999},"last":1.004},{"interface":"front","average":{"1min":9.5090000000000003,"5min":2.9750000000000001,"15min":2.7109999999999999},"min":{"1min":0.56799999999999995,"5min":0.36199999999999999,"15min":0.33400000000000002},"max":{"1min":152.732,"5min":152.732,"15min":243.32599999999999},"last":1.056}]},{"osd":6,"last update":"Sat Jun 3 20:32:34 2023","interfaces":[{"interface":"back","average":{"1min":9.7149999999999999,"5min":2.9700000000000002,"15min":2.577},"min":{"1min":0.60599999999999998,"5min":0.36499999999999999,"15min":0.36299999999999999},"max":{"1min":152.98699999999999,"5min":152.98699999999999,"15min":243.25},"last":1.2030000000000001},{"interface":"front","average":{"1min":10.984,"5min":3.387,"15min":2.6899999999999999},"min":{"1min":0.64800000000000002,"5min":0.41399999999999998,"15min":0.379},"max":{"1min":153.44399999999999,"5min":153.44399999999999,"15min":242.84399999999999},"last":1.274}]},{"osd":7,"last update":"Sat Jun 3 20:32:34 2023","interfaces":[{"interface":"back","average":{"1min":9.7989999999999995,"5min":3.0219999999999998,"15min":2.8530000000000002},"min":{"1min":0.56799999999999995,"5min":0.39600000000000002,"15min":0.35499999999999998},"max":{"1min":152.958,"5min":152.958,"15min":242.93600000000001},"last":1.397},{"interface":"front","average":{"1min":2.512,"5min":1.7609999999999999,"15min":2.3559999999999999},"min":{"1min":0.51200000000000001,"5min":0.44,"15min":0.40000000000000002},"max":{"1min":16.472000000000001,"5min":16.472000000000001,"15min":243.21299999999999},"last":1.3620000000000001}]}]},{"osd":2,"up_from":114,"seq":489626272184,"num_pgs":20,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":651420,"kb_used_data":3220,"kb_used_omap":7,"kb_used_meta":648184,"kb_avail":93093732,"statfs":{"total":95995035648,"available":95327981568,"internally_reserved":0,"allocated":3297280,"data_stored":2197933,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8111,"internal_metadata":663740497},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 3 20:32:05 2023","interfaces":[{"interface":"back","average":{"1min":1.647,"5min":2.5630000000000002,"15min":2.508},"min":{"1min":0.27800000000000002,"5min":0.182,"15min":0.182},"max":{"1min":7.367,"5min":77.055000000000007,"15min":117.57899999999999},"last":0.48899999999999999},{"interface":"front","average":{"1min":2.411,"5min":2.9449999999999998,"15min":2.4969999999999999},"min":{"1min":0.20200000000000001,"5min":0.20200000000000001,"15min":0.20200000000000001},"max":{"1min":8.9969999999999999,"5min":76.453000000000003,"15min":117.38500000000001},"last":0.64600000000000002}]},{"osd":1,"last update":"Sat Jun 3 20:32:05 2023","interfaces":[{"interface":"back","average":{"1min":2.3279999999999998,"5min":2.1059999999999999,"15min":2.0779999999999998},"min":{"1min":0.24099999999999999,"5min":0.193,"15min":0.193},"max":{"1min":8.9589999999999996,"5min":11.845000000000001,"15min":116.127},"last":0.88600000000000001},{"interface":"front","average":{"1min":3.0259999999999998,"5min":2.0110000000000001,"15min":2.3399999999999999},"min":{"1min":0.28799999999999998,"5min":0.251,"15min":0.251},"max":{"1min":9.6449999999999996,"5min":10.124000000000001,"15min":116.461},"last":0.46400000000000002}]},{"osd":3,"last update":"Sat Jun 3 20:32:05 2023","interfaces":[{"interface":"back","average":{"1min":2.7759999999999998,"5min":2.109,"15min":2.3010000000000002},"min":{"1min":0.32100000000000001,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":13.186999999999999,"5min":13.186999999999999,"15min":74.703000000000003},"last":0.624},{"interface":"front","average":{"1min":3.0710000000000002,"5min":2.2040000000000002,"15min":2.3959999999999999},"min":{"1min":0.35399999999999998,"5min":0.28599999999999998,"15min":0.23000000000000001},"max":{"1min":8.2629999999999999,"5min":14.355,"15min":37.088999999999999},"last":0.41099999999999998}]},{"osd":4,"last update":"Sat Jun 3 20:31:40 2023","interfaces":[{"interface":"back","average":{"1min":2.0649999999999999,"5min":1.5600000000000001,"15min":2.637},"min":{"1min":0.39300000000000002,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":8.0839999999999996,"5min":8.9100000000000001,"15min":16.561},"last":0.72099999999999997},{"interface":"front","average":{"1min":1.5800000000000001,"5min":1.4810000000000001,"15min":1.8200000000000001},"min":{"1min":0.377,"5min":0.35099999999999998,"15min":0.35099999999999998},"max":{"1min":5.375,"5min":6.7400000000000002,"15min":9.9339999999999993},"last":0.70399999999999996}]},{"osd":5,"last update":"Sat Jun 3 20:32:05 2023","interfaces":[{"interface":"back","average":{"1min":1.5780000000000001,"5min":1.4419999999999999,"15min":1.7649999999999999},"min":{"1min":0.51500000000000001,"5min":0.35899999999999999,"15min":0.32100000000000001},"max":{"1min":7.2160000000000002,"5min":7.2160000000000002,"15min":17.751000000000001},"last":0.58399999999999996},{"interface":"front","average":{"1min":2.2530000000000001,"5min":1.7769999999999999,"15min":1.679},"min":{"1min":0.44700000000000001,"5min":0.32500000000000001,"15min":0.316},"max":{"1min":6.6299999999999999,"5min":10.281000000000001,"15min":17.928000000000001},"last":0.75700000000000001}]},{"osd":6,"last update":"Sat Jun 3 20:32:05 2023","interfaces":[{"interface":"back","average":{"1min":1.7490000000000001,"5min":1.7310000000000001,"15min":1.6299999999999999},"min":{"1min":0.49399999999999999,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":7.2939999999999996,"5min":7.2939999999999996,"15min":14.331},"last":0.78000000000000003},{"interface":"front","average":{"1min":2.6110000000000002,"5min":1.871,"15min":1.5780000000000001},"min":{"1min":0.47999999999999998,"5min":0.35699999999999998,"15min":0.35599999999999998},"max":{"1min":10.493,"5min":11.943,"15min":12.867000000000001},"last":1.0149999999999999}]},{"osd":7,"last update":"Sat Jun 3 20:32:05 2023","interfaces":[{"interface":"back","average":{"1min":2.0870000000000002,"5min":1.7410000000000001,"15min":1.8049999999999999},"min":{"1min":0.42099999999999999,"5min":0.38300000000000001,"15min":0.376},"max":{"1min":7.5739999999999998,"5min":7.5739999999999998,"15min":14.243},"last":0.80800000000000005},{"interface":"front","average":{"1min":2.1520000000000001,"5min":1.788,"15min":1.675},"min":{"1min":0.503,"5min":0.44700000000000001,"15min":0.38600000000000001},"max":{"1min":6.6669999999999998,"5min":10.233000000000001,"15min":12.845000000000001},"last":1.244}]}]},{"osd":3,"up_from":119,"seq":511101108655,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":624988,"kb_used_data":3284,"kb_used_omap":2,"kb_used_meta":621693,"kb_avail":93120164,"statfs":{"total":95995035648,"available":95355047936,"internally_reserved":0,"allocated":3362816,"data_stored":2201406,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":636613685},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 3 20:32:07 2023","interfaces":[{"interface":"back","average":{"1min":1.468,"5min":4.5389999999999997,"15min":3.2690000000000001},"min":{"1min":0.41499999999999998,"5min":0.183,"15min":0.183},"max":{"1min":3.887,"5min":168.33099999999999,"15min":168.33099999999999},"last":0.63300000000000001},{"interface":"front","average":{"1min":2.2109999999999999,"5min":5.2910000000000004,"15min":3.52},"min":{"1min":0.214,"5min":0.214,"15min":0.183},"max":{"1min":9.5640000000000001,"5min":169.25800000000001,"15min":169.25800000000001},"last":0.61199999999999999}]},{"osd":1,"last update":"Sat Jun 3 20:32:07 2023","interfaces":[{"interface":"back","average":{"1min":1.966,"5min":1.911,"15min":1.7410000000000001},"min":{"1min":0.34200000000000003,"5min":0.245,"15min":0.22500000000000001},"max":{"1min":6.7530000000000001,"5min":20.992000000000001,"15min":117.047},"last":0.52500000000000002},{"interface":"front","average":{"1min":2.488,"5min":2.2210000000000001,"15min":1.7929999999999999},"min":{"1min":0.311,"5min":0.221,"15min":0.20399999999999999},"max":{"1min":9.4160000000000004,"5min":21.07,"15min":116.505},"last":0.57599999999999996}]},{"osd":2,"last update":"Sat Jun 3 20:32:07 2023","interfaces":[{"interface":"back","average":{"1min":1.425,"5min":2.4089999999999998,"15min":2.8149999999999999},"min":{"1min":0.27500000000000002,"5min":0.248,"15min":0.248},"max":{"1min":5.3860000000000001,"5min":18.916,"15min":117.584},"last":0.65400000000000003},{"interface":"front","average":{"1min":2.7280000000000002,"5min":2.0209999999999999,"15min":2.77},"min":{"1min":0.40699999999999997,"5min":0.22600000000000001,"15min":0.22600000000000001},"max":{"1min":11.526,"5min":11.526,"15min":117.01000000000001},"last":0.66900000000000004}]},{"osd":4,"last update":"Sat Jun 3 20:31:46 2023","interfaces":[{"interface":"back","average":{"1min":1.75,"5min":1.498,"15min":1.4470000000000001},"min":{"1min":0.38600000000000001,"5min":0.38,"15min":0.34599999999999997},"max":{"1min":7.0709999999999997,"5min":8.0449999999999999,"15min":8.0449999999999999},"last":1.075},{"interface":"front","average":{"1min":1.643,"5min":1.4470000000000001,"15min":1.7090000000000001},"min":{"1min":0.34699999999999998,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":6.6699999999999999,"5min":6.6699999999999999,"15min":10.773999999999999},"last":0.82499999999999996}]},{"osd":5,"last update":"Sat Jun 3 20:32:07 2023","interfaces":[{"interface":"back","average":{"1min":1.9299999999999999,"5min":1.4930000000000001,"15min":1.5680000000000001},"min":{"1min":0.38200000000000001,"5min":0.36299999999999999,"15min":0.313},"max":{"1min":7.0049999999999999,"5min":7.0049999999999999,"15min":38.651000000000003},"last":1.008},{"interface":"front","average":{"1min":1.8620000000000001,"5min":1.7809999999999999,"15min":1.585},"min":{"1min":0.432,"5min":0.376,"15min":0.316},"max":{"1min":5.5830000000000002,"5min":9.4909999999999997,"15min":38.585999999999999},"last":0.88800000000000001}]},{"osd":6,"last update":"Sat Jun 3 20:32:07 2023","interfaces":[{"interface":"back","average":{"1min":1.3540000000000001,"5min":1.6100000000000001,"15min":1.371},"min":{"1min":0.45800000000000002,"5min":0.36499999999999999,"15min":0.34799999999999998},"max":{"1min":5.149,"5min":9.5199999999999996,"15min":9.5199999999999996},"last":0.85299999999999998},{"interface":"front","average":{"1min":2.117,"5min":1.6299999999999999,"15min":1.5249999999999999},"min":{"1min":0.50700000000000001,"5min":0.36699999999999999,"15min":0.36699999999999999},"max":{"1min":6.9119999999999999,"5min":8.0790000000000006,"15min":8.7279999999999998},"last":1.1339999999999999}]},{"osd":7,"last update":"Sat Jun 3 20:32:07 2023","interfaces":[{"interface":"back","average":{"1min":1.9990000000000001,"5min":1.6970000000000001,"15min":1.7869999999999999},"min":{"1min":0.45000000000000001,"5min":0.44400000000000001,"15min":0.36599999999999999},"max":{"1min":6.9450000000000003,"5min":7.9809999999999999,"15min":39.008000000000003},"last":1.1859999999999999},{"interface":"front","average":{"1min":1.6279999999999999,"5min":1.5409999999999999,"15min":1.7949999999999999},"min":{"1min":0.40300000000000002,"5min":0.38700000000000001,"15min":0.34899999999999998},"max":{"1min":4.8239999999999998,"5min":7.4960000000000004,"15min":40.003999999999998},"last":0.875}]}]},{"osd":4,"up_from":250,"seq":1073741824112,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":264592,"kb_used_data":3400,"kb_used_omap":10,"kb_used_meta":261173,"kb_avail":93480560,"statfs":{"total":95995035648,"available":95724093440,"internally_reserved":0,"allocated":3481600,"data_stored":2215712,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":10652,"internal_metadata":267441764},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 3 20:31:42 2023","interfaces":[{"interface":"back","average":{"1min":5.4390000000000001,"5min":3.2839999999999998,"15min":2.3199999999999998},"min":{"1min":0.311,"5min":0.27600000000000002,"15min":0.26300000000000001},"max":{"1min":62.296999999999997,"5min":145.755,"15min":145.755},"last":1.0329999999999999},{"interface":"front","average":{"1min":5.7750000000000004,"5min":3.3809999999999998,"15min":2.206},"min":{"1min":0.32300000000000001,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":62.450000000000003,"5min":145.71799999999999,"15min":145.71799999999999},"last":0.95499999999999996}]},{"osd":1,"last update":"Sat Jun 3 20:31:42 2023","interfaces":[{"interface":"back","average":{"1min":5.641,"5min":2.3130000000000002,"15min":1.9510000000000001},"min":{"1min":0.29399999999999998,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":62.539000000000001,"5min":62.539000000000001,"15min":62.539000000000001},"last":0.69199999999999995},{"interface":"front","average":{"1min":5.4740000000000002,"5min":2.0259999999999998,"15min":1.948},"min":{"1min":0.33800000000000002,"5min":0.29599999999999999,"15min":0.29599999999999999},"max":{"1min":62.604999999999997,"5min":62.604999999999997,"15min":62.604999999999997},"last":1.1080000000000001}]},{"osd":2,"last update":"Sat Jun 3 20:31:42 2023","interfaces":[{"interface":"back","average":{"1min":10.465,"5min":3.9409999999999998,"15min":2.968},"min":{"1min":0.34000000000000002,"5min":0.34000000000000002,"15min":0.34000000000000002},"max":{"1min":78.941999999999993,"5min":78.941999999999993,"15min":78.941999999999993},"last":0.78400000000000003},{"interface":"front","average":{"1min":9.952,"5min":3.9649999999999999,"15min":3.3180000000000001},"min":{"1min":0.28999999999999998,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":80.069000000000003,"5min":80.069000000000003,"15min":80.069000000000003},"last":0.754}]},{"osd":3,"last update":"Sat Jun 3 20:31:42 2023","interfaces":[{"interface":"back","average":{"1min":6.29,"5min":3.5510000000000002,"15min":9.1969999999999992},"min":{"1min":0.38400000000000001,"5min":0.379,"15min":0.32500000000000001},"max":{"1min":63.005000000000003,"5min":63.005000000000003,"15min":217.578},"last":1.155},{"interface":"front","average":{"1min":6.0940000000000003,"5min":3.5430000000000001,"15min":9.0879999999999992},"min":{"1min":0.40899999999999997,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":62.569000000000003,"5min":62.569000000000003,"15min":218.55699999999999},"last":1.0680000000000001}]},{"osd":5,"last update":"Sat Jun 3 20:31:42 2023","interfaces":[{"interface":"back","average":{"1min":0.624,"5min":0.78500000000000003,"15min":2.5150000000000001},"min":{"1min":0.371,"5min":0.35099999999999998,"15min":0.35099999999999998},"max":{"1min":1.3680000000000001,"5min":3.0259999999999998,"15min":23.913},"last":0.82799999999999996},{"interface":"front","average":{"1min":0.88100000000000001,"5min":1.046,"15min":1.9119999999999999},"min":{"1min":0.42599999999999999,"5min":0.34000000000000002,"15min":0.34000000000000002},"max":{"1min":3.0190000000000001,"5min":8.2650000000000006,"15min":11.093999999999999},"last":0.876}]},{"osd":6,"last update":"Sat Jun 3 20:31:42 2023","interfaces":[{"interface":"back","average":{"1min":0.93999999999999995,"5min":0.94399999999999995,"15min":2.2519999999999998},"min":{"1min":0.36099999999999999,"5min":0.36099999999999999,"15min":0.36099999999999999},"max":{"1min":3.8799999999999999,"5min":3.8799999999999999,"15min":23.963000000000001},"last":1.129},{"interface":"front","average":{"1min":1.1180000000000001,"5min":1.1850000000000001,"15min":1.427},"min":{"1min":0.40300000000000002,"5min":0.38,"15min":0.38},"max":{"1min":3.278,"5min":6.8630000000000004,"15min":8.4190000000000005},"last":1.0860000000000001}]},{"osd":7,"last update":"Sat Jun 3 20:31:42 2023","interfaces":[{"interface":"back","average":{"1min":1.25,"5min":1.165,"15min":2.0299999999999998},"min":{"1min":0.372,"5min":0.372,"15min":0.372},"max":{"1min":5.0410000000000004,"5min":5.0410000000000004,"15min":10.747999999999999},"last":1.01},{"interface":"front","average":{"1min":0.76100000000000001,"5min":1.034,"15min":1.9350000000000001},"min":{"1min":0.40799999999999997,"5min":0.40799999999999997,"15min":0.34699999999999998},"max":{"1min":2.2559999999999998,"5min":6.7699999999999996,"15min":12.154},"last":1.194}]}]},{"osd":5,"up_from":36,"seq":154618823415,"num_pgs":21,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":971044,"kb_used_data":2716,"kb_used_omap":0,"kb_used_meta":968320,"kb_avail":92774108,"statfs":{"total":95995035648,"available":95000686592,"internally_reserved":0,"allocated":2781184,"data_stored":2136082,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":991559680},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 3 20:32:26 2023","interfaces":[{"interface":"back","average":{"1min":1.6859999999999999,"5min":4.2110000000000003,"15min":3.1190000000000002},"min":{"1min":0.49099999999999999,"5min":0.30399999999999999,"15min":0.25700000000000001},"max":{"1min":6.9109999999999996,"5min":176.88300000000001,"15min":176.88300000000001},"last":0.87},{"interface":"front","average":{"1min":1.6519999999999999,"5min":4.3449999999999998,"15min":3.2850000000000001},"min":{"1min":0.53100000000000003,"5min":0.34399999999999997,"15min":0.26500000000000001},"max":{"1min":6.931,"5min":176.96199999999999,"15min":176.96199999999999},"last":1.024}]},{"osd":1,"last update":"Sat Jun 3 20:32:26 2023","interfaces":[{"interface":"back","average":{"1min":0.879,"5min":1.9550000000000001,"15min":1.944},"min":{"1min":0.35599999999999998,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":2.4580000000000002,"5min":55.515000000000001,"15min":102.205},"last":0.79500000000000004},{"interface":"front","average":{"1min":1.0149999999999999,"5min":1.9870000000000001,"15min":1.9610000000000001},"min":{"1min":0.34300000000000003,"5min":0.316,"15min":0.30499999999999999},"max":{"1min":2.8679999999999999,"5min":55.194000000000003,"15min":102.265},"last":0.91300000000000003}]},{"osd":2,"last update":"Sat Jun 3 20:32:26 2023","interfaces":[{"interface":"back","average":{"1min":1.331,"5min":2.5880000000000001,"15min":3.492},"min":{"1min":0.32300000000000001,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":3.2269999999999999,"5min":104.199,"15min":236.40000000000001},"last":1.3220000000000001},{"interface":"front","average":{"1min":1.5820000000000001,"5min":2.9430000000000001,"15min":3.8300000000000001},"min":{"1min":0.41599999999999998,"5min":0.308,"15min":0.308},"max":{"1min":5.4790000000000001,"5min":104.33499999999999,"15min":237.86099999999999},"last":1.1819999999999999}]},{"osd":3,"last update":"Sat Jun 3 20:32:01 2023","interfaces":[{"interface":"back","average":{"1min":1.1499999999999999,"5min":1.5940000000000001,"15min":2.1720000000000002},"min":{"1min":0.39700000000000002,"5min":0.36799999999999999,"15min":0.34399999999999997},"max":{"1min":4.327,"5min":35.725999999999999,"15min":86.486999999999995},"last":1.093},{"interface":"front","average":{"1min":1.3620000000000001,"5min":1.696,"15min":2.1600000000000001},"min":{"1min":0.40799999999999997,"5min":0.40799999999999997,"15min":0.37},"max":{"1min":5.6210000000000004,"5min":35.762999999999998,"15min":86.512},"last":1.4910000000000001}]},{"osd":4,"last update":"Sat Jun 3 20:32:41 2023","interfaces":[{"interface":"back","average":{"1min":0.99199999999999999,"5min":0.95999999999999996,"15min":2.5990000000000002},"min":{"1min":0.38100000000000001,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":1.448,"5min":3.9079999999999999,"15min":21.786000000000001},"last":1.625},{"interface":"front","average":{"1min":1.1140000000000001,"5min":1.1399999999999999,"15min":2.1659999999999999},"min":{"1min":0.45500000000000002,"5min":0.34000000000000002,"15min":0.30199999999999999},"max":{"1min":1.7509999999999999,"5min":11.880000000000001,"15min":26.478000000000002},"last":1.603}]},{"osd":6,"last update":"Sat Jun 3 20:32:26 2023","interfaces":[{"interface":"back","average":{"1min":0.94299999999999995,"5min":0.92100000000000004,"15min":1.139},"min":{"1min":0.39000000000000001,"5min":0.371,"15min":0.35199999999999998},"max":{"1min":1.349,"5min":3.702,"15min":7.5140000000000002},"last":1.294},{"interface":"front","average":{"1min":1.097,"5min":1.018,"15min":1.228},"min":{"1min":0.47599999999999998,"5min":0.41499999999999998,"15min":0.37},"max":{"1min":2.4910000000000001,"5min":6.4260000000000002,"15min":6.5309999999999997},"last":1.4530000000000001}]},{"osd":7,"last update":"Sat Jun 3 20:32:23 2023","interfaces":[{"interface":"back","average":{"1min":1.161,"5min":1.032,"15min":1.4390000000000001},"min":{"1min":0.52100000000000002,"5min":0.442,"15min":0.35999999999999999},"max":{"1min":2.5169999999999999,"5min":3.1549999999999998,"15min":15.516},"last":1.379},{"interface":"front","average":{"1min":1.089,"5min":1.0369999999999999,"15min":1.262},"min":{"1min":0.40000000000000002,"5min":0.38400000000000001,"15min":0.373},"max":{"1min":3.1280000000000001,"5min":5.0350000000000001,"15min":9.6519999999999992},"last":1.6799999999999999}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6928,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4862,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":10652,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4786,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4932,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-06-03T20:32:48.138 INFO:tasks.cephadm.ceph_manager.ceph:recovered! 2023-06-03T20:32:48.138 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-06-03T20:32:48.138 INFO:tasks.thrashosds.thrasher:inject_pause on osd.0 2023-06-03T20:32:48.138 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2023-06-03T20:32:48.139 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2023-06-03T20:32:48.139 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph daemon osd.0 config set filestore_inject_stall 3 2023-06-03T20:32:48.231 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:32:48.263 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:32:48.266 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:32:48.267 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:32:48.368 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:32:48.376 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:32:48 smithi164 bash[59462]: cluster 1685824367.2292361 mgr.y (mgr.34104) 1713 : cluster 0 pgmap v1771: 73 pgs: 73 active+clean; 582 KiB data, 5.1 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:32:48.396 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:32:48.399 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:32:48.399 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:32:48.500 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:32:48.539 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:32:48.542 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:32:48.543 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:32:48.644 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:32:48.657 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:32:48 smithi089 bash[78452]: cluster 1685824367.2292361 mgr.y (mgr.34104) 1713 : cluster 0 pgmap v1771: 73 pgs: 73 active+clean; 582 KiB data, 5.1 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:32:48.657 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:32:48 smithi089 bash[75068]: cluster 1685824367.2292361 mgr.y (mgr.34104) 1713 : cluster 0 pgmap v1771: 73 pgs: 73 active+clean; 582 KiB data, 5.1 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:32:48.675 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:32:48.681 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:32:48.682 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:32:48.783 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:32:49.060 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:32:49.064 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:32:49.065 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:32:49.166 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:32:49.193 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:32:49.196 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:32:49.196 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:32:49.237 INFO:tasks.workunit.client.0.smithi089.stdout:cc -Wall -Wextra -D_GNU_SOURCE test_sync_io.c -o test_sync_io 2023-06-03T20:32:49.297 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:32:49.334 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:32:49.336 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:32:49.337 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:32:49.438 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:32:49.468 INFO:tasks.workunit.client.0.smithi089.stdout:cc -Wall -Wextra -D_GNU_SOURCE test_short_dio_read.c -o test_short_dio_read 2023-06-03T20:32:49.469 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:32:49.473 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:32:49.473 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:32:49.528 INFO:tasks.workunit.client.0.smithi089.stdout:make[1]: Leaving directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/direct_io' 2023-06-03T20:32:49.530 INFO:tasks.workunit.client.0.smithi089.stdout:make[1]: Entering directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/fs' 2023-06-03T20:32:49.530 INFO:tasks.workunit.client.0.smithi089.stdout:cc -Wall -Wextra -D_GNU_SOURCE test_o_trunc.c -o test_o_trunc 2023-06-03T20:32:49.575 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:32:49.585 INFO:tasks.workunit.client.0.smithi089.stdout:make[1]: Leaving directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/fs' 2023-06-03T20:32:49.591 DEBUG:teuthology.orchestra.run.smithi089:> set -ex 2023-06-03T20:32:49.591 DEBUG:teuthology.orchestra.run.smithi089:> dd if=/home/ubuntu/cephtest/workunits.list.client.0 of=/dev/stdout 2023-06-03T20:32:49.609 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:32:49.612 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:32:49.613 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:32:49.651 INFO:tasks.workunit:Running workunits matching cls/test_cls_rbd.sh on client.0... 2023-06-03T20:32:49.652 INFO:tasks.workunit:Running workunit cls/test_cls_rbd.sh... 2023-06-03T20:32:49.653 DEBUG:teuthology.orchestra.run.smithi089:workunit test cls/test_cls_rbd.sh> mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=pacific TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 CEPH_MNT=/home/ubuntu/cephtest/mnt.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/cls/test_cls_rbd.sh 2023-06-03T20:32:49.713 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:32:49.768 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:32:49.771 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:32:49.772 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:32:49.773 INFO:tasks.workunit.client.0.smithi089.stdout:Running main() from gmock_main.cc 2023-06-03T20:32:49.774 INFO:tasks.workunit.client.0.smithi089.stdout:[==========] Running 70 tests from 1 test suite. 2023-06-03T20:32:49.774 INFO:tasks.workunit.client.0.smithi089.stdout:[----------] Global test environment set-up. 2023-06-03T20:32:49.774 INFO:tasks.workunit.client.0.smithi089.stdout:[----------] 70 tests from TestClsRbd 2023-06-03T20:32:49.873 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:32:49.906 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:32:49.910 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:32:49.910 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:32:50.011 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:32:50.044 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:32:50.047 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:32:50.048 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:32:50.149 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:32:50.184 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:32:50.187 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:32:50.188 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:32:50.289 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:32:50.326 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:32:50.330 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:32:50.330 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:32:50.333 INFO:teuthology.orchestra.run.smithi089.stderr:noscrub is unset 2023-06-03T20:32:50.431 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:32:50.472 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:32:50.475 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:32:50.476 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:32:50.577 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:32:50.585 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:32:50 smithi164 bash[59462]: cluster 1685824369.229944 mgr.y (mgr.34104) 1714 : cluster 0 pgmap v1772: 73 pgs: 73 active+clean; 582 KiB data, 5.1 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:32:50.585 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:32:50 smithi164 bash[59462]: audit 1685824370.1334403 mon.b (mon.1) 67 : audit 1 from='client.? 172.21.15.89:0/3924461392' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-06-03T20:32:50.586 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:32:50 smithi164 bash[59462]: audit 1685824370.2422705 mon.a (mon.0) 1674 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-06-03T20:32:50.614 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:32:50.617 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:32:50.618 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:32:50.719 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:32:50.758 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:32:50.761 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:32:50.762 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:32:50.779 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:32:50 smithi089 bash[75068]: cluster 1685824369.229944 mgr.y (mgr.34104) 1714 : cluster 0 pgmap v1772: 73 pgs: 73 active+clean; 582 KiB data, 5.1 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:32:50.780 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:32:50 smithi089 bash[75068]: audit 1685824370.1334403 mon.b (mon.1) 67 : audit 1 from='client.? 172.21.15.89:0/3924461392' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-06-03T20:32:50.780 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:32:50 smithi089 bash[75068]: audit 1685824370.2422705 mon.a (mon.0) 1674 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-06-03T20:32:50.780 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:32:50 smithi089 bash[78452]: cluster 1685824369.229944 mgr.y (mgr.34104) 1714 : cluster 0 pgmap v1772: 73 pgs: 73 active+clean; 582 KiB data, 5.1 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:32:50.780 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:32:50 smithi089 bash[78452]: audit 1685824370.1334403 mon.b (mon.1) 67 : audit 1 from='client.? 172.21.15.89:0/3924461392' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-06-03T20:32:50.780 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:32:50 smithi089 bash[78452]: audit 1685824370.2422705 mon.a (mon.0) 1674 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-06-03T20:32:50.863 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:32:50.902 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:32:50.906 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:32:50.906 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:32:51.008 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:32:51.042 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:32:51.044 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:32:51.045 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:32:51.146 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:32:51.176 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:32:51.180 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:32:51.180 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:32:51.281 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:32:51.324 INFO:tasks.workunit.client.0.smithi089.stdout:[ RUN ] TestClsRbd.get_all_features 2023-06-03T20:32:51.332 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:32:51.342 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:32:51.343 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:32:51.348 INFO:tasks.workunit.client.0.smithi089.stdout:[ OK ] TestClsRbd.get_all_features (24 ms) 2023-06-03T20:32:51.348 INFO:tasks.workunit.client.0.smithi089.stdout:[ RUN ] TestClsRbd.copyup 2023-06-03T20:32:51.444 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:32:51.455 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:32:51 smithi089 bash[78452]: cluster 1685824370.2994542 mon.a (mon.0) 1675 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-06-03T20:32:51.455 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:32:51 smithi089 bash[78452]: cluster 1685824370.308991 osd.5 (osd.5) 247 : cluster 3 failed to encode map e299 with expected crc 2023-06-03T20:32:51.456 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:32:51 smithi089 bash[78452]: audit 1685824370.313939 mon.a (mon.0) 1676 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-06-03T20:32:51.456 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:32:51 smithi089 bash[78452]: cluster 1685824370.3140345 mon.a (mon.0) 1677 : cluster 0 osdmap e299: 8 total, 8 up, 8 in 2023-06-03T20:32:51.456 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:32:51 smithi089 bash[78452]: cluster 1685824370.3142111 osd.6 (osd.6) 234 : cluster 3 failed to encode map e299 with expected crc 2023-06-03T20:32:51.456 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:32:51 smithi089 bash[78452]: cluster 1685824370.3158455 osd.6 (osd.6) 235 : cluster 3 failed to encode map e299 with expected crc 2023-06-03T20:32:51.456 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:32:51 smithi089 bash[78452]: audit 1685824370.3160837 mon.a (mon.0) 1678 : audit 1 from='client.? 172.21.15.89:0/3709264146' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi089-258064-1","app": "rados","yes_i_really_mean_it": true}]: dispatch 2023-06-03T20:32:51.456 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:32:51 smithi089 bash[78452]: cluster 1685824370.316637 osd.7 (osd.7) 243 : cluster 3 failed to encode map e299 with expected crc 2023-06-03T20:32:51.456 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:32:51 smithi089 bash[78452]: cluster 1685824370.318611 osd.6 (osd.6) 236 : cluster 3 failed to encode map e299 with expected crc 2023-06-03T20:32:51.457 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:32:51 smithi089 bash[75068]: cluster 1685824370.2994542 mon.a (mon.0) 1675 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-06-03T20:32:51.457 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:32:51 smithi089 bash[75068]: cluster 1685824370.308991 osd.5 (osd.5) 247 : cluster 3 failed to encode map e299 with expected crc 2023-06-03T20:32:51.457 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:32:51 smithi089 bash[75068]: audit 1685824370.313939 mon.a (mon.0) 1676 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-06-03T20:32:51.457 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:32:51 smithi089 bash[75068]: cluster 1685824370.3140345 mon.a (mon.0) 1677 : cluster 0 osdmap e299: 8 total, 8 up, 8 in 2023-06-03T20:32:51.457 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:32:51 smithi089 bash[75068]: cluster 1685824370.3142111 osd.6 (osd.6) 234 : cluster 3 failed to encode map e299 with expected crc 2023-06-03T20:32:51.457 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:32:51 smithi089 bash[75068]: cluster 1685824370.3158455 osd.6 (osd.6) 235 : cluster 3 failed to encode map e299 with expected crc 2023-06-03T20:32:51.458 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:32:51 smithi089 bash[75068]: audit 1685824370.3160837 mon.a (mon.0) 1678 : audit 1 from='client.? 172.21.15.89:0/3709264146' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi089-258064-1","app": "rados","yes_i_really_mean_it": true}]: dispatch 2023-06-03T20:32:51.458 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:32:51 smithi089 bash[75068]: cluster 1685824370.316637 osd.7 (osd.7) 243 : cluster 3 failed to encode map e299 with expected crc 2023-06-03T20:32:51.458 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:32:51 smithi089 bash[75068]: cluster 1685824370.318611 osd.6 (osd.6) 236 : cluster 3 failed to encode map e299 with expected crc 2023-06-03T20:32:51.477 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:32:51.479 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:32:51.480 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:32:51.491 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-06-03T20:32:51.581 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:32:51.585 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:32:51 smithi164 bash[59462]: cluster 1685824370.2994542 mon.a (mon.0) 1675 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-06-03T20:32:51.585 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:32:51 smithi164 bash[59462]: cluster 1685824370.308991 osd.5 (osd.5) 247 : cluster 3 failed to encode map e299 with expected crc 2023-06-03T20:32:51.586 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:32:51 smithi164 bash[59462]: audit 1685824370.313939 mon.a (mon.0) 1676 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-06-03T20:32:51.586 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:32:51 smithi164 bash[59462]: cluster 1685824370.3140345 mon.a (mon.0) 1677 : cluster 0 osdmap e299: 8 total, 8 up, 8 in 2023-06-03T20:32:51.586 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:32:51 smithi164 bash[59462]: cluster 1685824370.3142111 osd.6 (osd.6) 234 : cluster 3 failed to encode map e299 with expected crc 2023-06-03T20:32:51.586 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:32:51 smithi164 bash[59462]: cluster 1685824370.3158455 osd.6 (osd.6) 235 : cluster 3 failed to encode map e299 with expected crc 2023-06-03T20:32:51.586 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:32:51 smithi164 bash[59462]: audit 1685824370.3160837 mon.a (mon.0) 1678 : audit 1 from='client.? 172.21.15.89:0/3709264146' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi089-258064-1","app": "rados","yes_i_really_mean_it": true}]: dispatch 2023-06-03T20:32:51.586 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:32:51 smithi164 bash[59462]: cluster 1685824370.316637 osd.7 (osd.7) 243 : cluster 3 failed to encode map e299 with expected crc 2023-06-03T20:32:51.586 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:32:51 smithi164 bash[59462]: cluster 1685824370.318611 osd.6 (osd.6) 236 : cluster 3 failed to encode map e299 with expected crc 2023-06-03T20:32:51.616 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:32:51.618 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:32:51.620 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:32:51.656 INFO:tasks.workunit.client.0.smithi089.stdout:[ OK ] TestClsRbd.copyup (308 ms) 2023-06-03T20:32:51.657 INFO:tasks.workunit.client.0.smithi089.stdout:[ RUN ] TestClsRbd.sparse_copyup 2023-06-03T20:32:51.691 INFO:tasks.workunit.client.0.smithi089.stdout:[ OK ] TestClsRbd.sparse_copyup (35 ms) 2023-06-03T20:32:51.692 INFO:tasks.workunit.client.0.smithi089.stdout:[ RUN ] TestClsRbd.get_and_set_id 2023-06-03T20:32:51.707 INFO:tasks.workunit.client.0.smithi089.stdout:[ OK ] TestClsRbd.get_and_set_id (15 ms) 2023-06-03T20:32:51.707 INFO:tasks.workunit.client.0.smithi089.stdout:[ RUN ] TestClsRbd.add_remove_child 2023-06-03T20:32:51.721 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:32:51.727 INFO:tasks.workunit.client.0.smithi089.stdout:[ OK ] TestClsRbd.add_remove_child (20 ms) 2023-06-03T20:32:51.727 INFO:tasks.workunit.client.0.smithi089.stdout:[ RUN ] TestClsRbd.directory_methods 2023-06-03T20:32:51.751 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:32:51.754 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:32:51.754 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:32:51.776 INFO:tasks.workunit.client.0.smithi089.stdout:[ OK ] TestClsRbd.directory_methods (49 ms) 2023-06-03T20:32:51.776 INFO:tasks.workunit.client.0.smithi089.stdout:[ RUN ] TestClsRbd.create 2023-06-03T20:32:51.810 INFO:tasks.workunit.client.0.smithi089.stdout:[ OK ] TestClsRbd.create (34 ms) 2023-06-03T20:32:51.810 INFO:tasks.workunit.client.0.smithi089.stdout:[ RUN ] TestClsRbd.get_features 2023-06-03T20:32:51.813 INFO:tasks.workunit.client.0.smithi089.stdout:[ OK ] TestClsRbd.get_features (4 ms) 2023-06-03T20:32:51.813 INFO:tasks.workunit.client.0.smithi089.stdout:[ RUN ] TestClsRbd.get_object_prefix 2023-06-03T20:32:51.816 INFO:tasks.workunit.client.0.smithi089.stdout:[ OK ] TestClsRbd.get_object_prefix (3 ms) 2023-06-03T20:32:51.816 INFO:tasks.workunit.client.0.smithi089.stdout:[ RUN ] TestClsRbd.get_create_timestamp 2023-06-03T20:32:51.828 INFO:tasks.workunit.client.0.smithi089.stdout:[ OK ] TestClsRbd.get_create_timestamp (11 ms) 2023-06-03T20:32:51.828 INFO:tasks.workunit.client.0.smithi089.stdout:[ RUN ] TestClsRbd.get_access_timestamp 2023-06-03T20:32:51.831 INFO:tasks.workunit.client.0.smithi089.stdout:[ OK ] TestClsRbd.get_access_timestamp (3 ms) 2023-06-03T20:32:51.831 INFO:tasks.workunit.client.0.smithi089.stdout:[ RUN ] TestClsRbd.get_modify_timestamp 2023-06-03T20:32:51.833 INFO:tasks.workunit.client.0.smithi089.stdout:[ OK ] TestClsRbd.get_modify_timestamp (3 ms) 2023-06-03T20:32:51.834 INFO:tasks.workunit.client.0.smithi089.stdout:[ RUN ] TestClsRbd.get_data_pool 2023-06-03T20:32:51.840 INFO:tasks.workunit.client.0.smithi089.stdout:[ OK ] TestClsRbd.get_data_pool (6 ms) 2023-06-03T20:32:51.840 INFO:tasks.workunit.client.0.smithi089.stdout:[ RUN ] TestClsRbd.get_size 2023-06-03T20:32:51.850 INFO:tasks.workunit.client.0.smithi089.stdout:[ OK ] TestClsRbd.get_size (11 ms) 2023-06-03T20:32:51.850 INFO:tasks.workunit.client.0.smithi089.stdout:[ RUN ] TestClsRbd.set_size 2023-06-03T20:32:51.856 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:32:51.860 INFO:tasks.workunit.client.0.smithi089.stdout:[ OK ] TestClsRbd.set_size (9 ms) 2023-06-03T20:32:51.860 INFO:tasks.workunit.client.0.smithi089.stdout:[ RUN ] TestClsRbd.protection_status 2023-06-03T20:32:51.889 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:32:51.893 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:32:51.894 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:32:51.900 INFO:tasks.workunit.client.0.smithi089.stdout:[ OK ] TestClsRbd.protection_status (41 ms) 2023-06-03T20:32:51.900 INFO:tasks.workunit.client.0.smithi089.stdout:[ RUN ] TestClsRbd.snapshot_limits 2023-06-03T20:32:51.915 INFO:tasks.workunit.client.0.smithi089.stdout:[ OK ] TestClsRbd.snapshot_limits (15 ms) 2023-06-03T20:32:51.915 INFO:tasks.workunit.client.0.smithi089.stdout:[ RUN ] TestClsRbd.parents_v1 2023-06-03T20:32:51.995 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:32:51.998 INFO:tasks.workunit.client.0.smithi089.stdout:[ OK ] TestClsRbd.parents_v1 (82 ms) 2023-06-03T20:32:51.999 INFO:tasks.workunit.client.0.smithi089.stdout:[ RUN ] TestClsRbd.parents_v2 2023-06-03T20:32:52.031 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:32:52.035 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:32:52.036 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:32:52.056 INFO:tasks.workunit.client.0.smithi089.stdout:[ OK ] TestClsRbd.parents_v2 (58 ms) 2023-06-03T20:32:52.056 INFO:tasks.workunit.client.0.smithi089.stdout:[ RUN ] TestClsRbd.snapshots 2023-06-03T20:32:52.093 INFO:tasks.workunit.client.0.smithi089.stdout:[ OK ] TestClsRbd.snapshots (37 ms) 2023-06-03T20:32:52.093 INFO:tasks.workunit.client.0.smithi089.stdout:[ RUN ] TestClsRbd.snapid_race 2023-06-03T20:32:52.105 INFO:tasks.workunit.client.0.smithi089.stdout:[ OK ] TestClsRbd.snapid_race (12 ms) 2023-06-03T20:32:52.105 INFO:tasks.workunit.client.0.smithi089.stdout:[ RUN ] TestClsRbd.stripingv2 2023-06-03T20:32:52.124 INFO:tasks.workunit.client.0.smithi089.stdout:[ OK ] TestClsRbd.stripingv2 (19 ms) 2023-06-03T20:32:52.124 INFO:tasks.workunit.client.0.smithi089.stdout:[ RUN ] TestClsRbd.object_map_save 2023-06-03T20:32:52.130 INFO:tasks.workunit.client.0.smithi089.stdout:[ OK ] TestClsRbd.object_map_save (7 ms) 2023-06-03T20:32:52.130 INFO:tasks.workunit.client.0.smithi089.stdout:[ RUN ] TestClsRbd.object_map_resize 2023-06-03T20:32:52.137 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:32:52.147 INFO:tasks.workunit.client.0.smithi089.stdout:[ OK ] TestClsRbd.object_map_resize (16 ms) 2023-06-03T20:32:52.147 INFO:tasks.workunit.client.0.smithi089.stdout:[ RUN ] TestClsRbd.object_map_update 2023-06-03T20:32:52.161 INFO:tasks.workunit.client.0.smithi089.stdout:[ OK ] TestClsRbd.object_map_update (13 ms) 2023-06-03T20:32:52.161 INFO:tasks.workunit.client.0.smithi089.stdout:[ RUN ] TestClsRbd.object_map_load_enoent 2023-06-03T20:32:52.161 INFO:tasks.workunit.client.0.smithi089.stdout:[ OK ] TestClsRbd.object_map_load_enoent (1 ms) 2023-06-03T20:32:52.162 INFO:tasks.workunit.client.0.smithi089.stdout:[ RUN ] TestClsRbd.object_map_snap_add 2023-06-03T20:32:52.172 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:32:52.174 INFO:tasks.workunit.client.0.smithi089.stdout:[ OK ] TestClsRbd.object_map_snap_add (12 ms) 2023-06-03T20:32:52.174 INFO:tasks.workunit.client.0.smithi089.stdout:[ RUN ] TestClsRbd.object_map_snap_remove 2023-06-03T20:32:52.175 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:32:52.175 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:32:52.184 INFO:tasks.workunit.client.0.smithi089.stdout:[ OK ] TestClsRbd.object_map_snap_remove (11 ms) 2023-06-03T20:32:52.184 INFO:tasks.workunit.client.0.smithi089.stdout:[ RUN ] TestClsRbd.flags 2023-06-03T20:32:52.194 INFO:tasks.workunit.client.0.smithi089.stdout:[ OK ] TestClsRbd.flags (9 ms) 2023-06-03T20:32:52.194 INFO:tasks.workunit.client.0.smithi089.stdout:[ RUN ] TestClsRbd.metadata 2023-06-03T20:32:52.243 INFO:tasks.workunit.client.0.smithi089.stdout:[ OK ] TestClsRbd.metadata (49 ms) 2023-06-03T20:32:52.244 INFO:tasks.workunit.client.0.smithi089.stdout:[ RUN ] TestClsRbd.set_features 2023-06-03T20:32:52.257 INFO:tasks.workunit.client.0.smithi089.stdout:[ OK ] TestClsRbd.set_features (14 ms) 2023-06-03T20:32:52.257 INFO:tasks.workunit.client.0.smithi089.stdout:[ RUN ] TestClsRbd.mirror 2023-06-03T20:32:52.276 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:32:52.313 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:32:52.316 INFO:tasks.workunit.client.0.smithi089.stdout:[ OK ] TestClsRbd.mirror (60 ms) 2023-06-03T20:32:52.316 INFO:tasks.workunit.client.0.smithi089.stdout:[ RUN ] TestClsRbd.mirror_image 2023-06-03T20:32:52.317 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:32:52.318 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:32:52.350 INFO:tasks.workunit.client.0.smithi089.stdout:[ OK ] TestClsRbd.mirror_image (33 ms) 2023-06-03T20:32:52.350 INFO:tasks.workunit.client.0.smithi089.stdout:[ RUN ] TestClsRbd.mirror_image_status 2023-06-03T20:32:52.418 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:32:52.453 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:32:52.455 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:32:52.457 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:32:52.558 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:32:52.585 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:32:52 smithi164 bash[59462]: cluster 1685824371.2304764 mgr.y (mgr.34104) 1715 : cluster 0 pgmap v1774: 105 pgs: 32 unknown, 73 active+clean; 582 KiB data, 5.1 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:32:52.586 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:32:52 smithi164 bash[59462]: cluster 1685824371.3143318 osd.5 (osd.5) 248 : cluster 3 failed to encode map e300 with expected crc 2023-06-03T20:32:52.586 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:32:52 smithi164 bash[59462]: cluster 1685824371.316506 osd.7 (osd.7) 244 : cluster 3 failed to encode map e300 with expected crc 2023-06-03T20:32:52.586 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:32:52 smithi164 bash[59462]: cluster 1685824371.316506 osd.6 (osd.6) 237 : cluster 3 failed to encode map e300 with expected crc 2023-06-03T20:32:52.586 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:32:52 smithi164 bash[59462]: audit 1685824371.3189251 mon.a (mon.0) 1679 : audit 1 from='client.? 172.21.15.89:0/3709264146' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi089-258064-1","app": "rados","yes_i_really_mean_it": true}]': finished 2023-06-03T20:32:52.586 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:32:52 smithi164 bash[59462]: cluster 1685824371.3191426 mon.a (mon.0) 1680 : cluster 0 osdmap e300: 8 total, 8 up, 8 in 2023-06-03T20:32:52.587 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:32:52 smithi164 bash[59462]: audit 1685824372.1625485 mon.c (mon.2) 687 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T20:32:52.590 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:32:52.593 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:32:52 smithi089 bash[75068]: cluster 1685824371.2304764 mgr.y (mgr.34104) 1715 : cluster 0 pgmap v1774: 105 pgs: 32 unknown, 73 active+clean; 582 KiB data, 5.1 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:32:52.593 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:32:52 smithi089 bash[75068]: cluster 1685824371.3143318 osd.5 (osd.5) 248 : cluster 3 failed to encode map e300 with expected crc 2023-06-03T20:32:52.594 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:32:52 smithi089 bash[75068]: cluster 1685824371.316506 osd.7 (osd.7) 244 : cluster 3 failed to encode map e300 with expected crc 2023-06-03T20:32:52.594 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:32:52 smithi089 bash[75068]: cluster 1685824371.316506 osd.6 (osd.6) 237 : cluster 3 failed to encode map e300 with expected crc 2023-06-03T20:32:52.594 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:32:52 smithi089 bash[75068]: audit 1685824371.3189251 mon.a (mon.0) 1679 : audit 1 from='client.? 172.21.15.89:0/3709264146' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi089-258064-1","app": "rados","yes_i_really_mean_it": true}]': finished 2023-06-03T20:32:52.594 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:32:52 smithi089 bash[75068]: cluster 1685824371.3191426 mon.a (mon.0) 1680 : cluster 0 osdmap e300: 8 total, 8 up, 8 in 2023-06-03T20:32:52.594 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:32:52 smithi089 bash[75068]: audit 1685824372.1625485 mon.c (mon.2) 687 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T20:32:52.595 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:32:52 smithi089 bash[78452]: cluster 1685824371.2304764 mgr.y (mgr.34104) 1715 : cluster 0 pgmap v1774: 105 pgs: 32 unknown, 73 active+clean; 582 KiB data, 5.1 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:32:52.595 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:32:52 smithi089 bash[78452]: cluster 1685824371.3143318 osd.5 (osd.5) 248 : cluster 3 failed to encode map e300 with expected crc 2023-06-03T20:32:52.595 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:32:52 smithi089 bash[78452]: cluster 1685824371.316506 osd.7 (osd.7) 244 : cluster 3 failed to encode map e300 with expected crc 2023-06-03T20:32:52.596 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:32:52 smithi089 bash[78452]: cluster 1685824371.316506 osd.6 (osd.6) 237 : cluster 3 failed to encode map e300 with expected crc 2023-06-03T20:32:52.596 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:32:52 smithi089 bash[78452]: audit 1685824371.3189251 mon.a (mon.0) 1679 : audit 1 from='client.? 172.21.15.89:0/3709264146' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi089-258064-1","app": "rados","yes_i_really_mean_it": true}]': finished 2023-06-03T20:32:52.596 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:32:52 smithi089 bash[78452]: cluster 1685824371.3191426 mon.a (mon.0) 1680 : cluster 0 osdmap e300: 8 total, 8 up, 8 in 2023-06-03T20:32:52.597 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:32:52 smithi089 bash[78452]: audit 1685824372.1625485 mon.c (mon.2) 687 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T20:32:52.598 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:32:52.598 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:32:52.700 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:32:52.737 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:32:52.739 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:32:52.740 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:32:52.841 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:32:52.877 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:32:52.880 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:32:52.880 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:32:52.981 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:32:53.014 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:32:53.017 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:32:53.018 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:32:53.072 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/mon.a/config 2023-06-03T20:32:53.119 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:32:53.152 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:32:53.154 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:32:53.155 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:32:53.256 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:32:53.286 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:32:53.289 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:32:53.290 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:32:53.391 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:32:53.401 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:32:53 smithi089 bash[78452]: cluster 1685824372.316083 osd.6 (osd.6) 238 : cluster 3 failed to encode map e301 with expected crc 2023-06-03T20:32:53.401 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:32:53 smithi089 bash[78452]: cluster 1685824372.316818 osd.5 (osd.5) 249 : cluster 3 failed to encode map e301 with expected crc 2023-06-03T20:32:53.402 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:32:53 smithi089 bash[78452]: cluster 1685824372.322203 mon.a (mon.0) 1681 : cluster 0 osdmap e301: 8 total, 8 up, 8 in 2023-06-03T20:32:53.402 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:32:53 smithi089 bash[75068]: cluster 1685824372.316083 osd.6 (osd.6) 238 : cluster 3 failed to encode map e301 with expected crc 2023-06-03T20:32:53.402 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:32:53 smithi089 bash[75068]: cluster 1685824372.316818 osd.5 (osd.5) 249 : cluster 3 failed to encode map e301 with expected crc 2023-06-03T20:32:53.402 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:32:53 smithi089 bash[75068]: cluster 1685824372.322203 mon.a (mon.0) 1681 : cluster 0 osdmap e301: 8 total, 8 up, 8 in 2023-06-03T20:32:53.425 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:32:53.427 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:32:53.428 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:32:53.529 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:32:53.558 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:32:53.560 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:32:53.561 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:32:53.563 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph osd unset nodeep-scrub 2023-06-03T20:32:53.585 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:32:53 smithi164 bash[59462]: cluster 1685824372.316083 osd.6 (osd.6) 238 : cluster 3 failed to encode map e301 with expected crc 2023-06-03T20:32:53.585 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:32:53 smithi164 bash[59462]: cluster 1685824372.316818 osd.5 (osd.5) 249 : cluster 3 failed to encode map e301 with expected crc 2023-06-03T20:32:53.586 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:32:53 smithi164 bash[59462]: cluster 1685824372.322203 mon.a (mon.0) 1681 : cluster 0 osdmap e301: 8 total, 8 up, 8 in 2023-06-03T20:32:53.662 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:32:53.695 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:32:53.698 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:32:53.698 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:32:53.799 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:32:53.830 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:32:53.833 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:32:53.834 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:32:53.935 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:32:53.983 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:32:53.986 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:32:53.988 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:32:54.089 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:32:54.094 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/mon.a/config 2023-06-03T20:32:54.132 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:32:54.134 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:32:54.136 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:32:54.238 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:32:54.268 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:32:54.270 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:32:54.271 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:32:54.372 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:32:54.404 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:32:54.406 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:32:54.407 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:32:54.508 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:32:54.544 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:32:54.547 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:32:54.548 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:32:54.585 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:32:54 smithi164 bash[59462]: cluster 1685824373.2313874 mgr.y (mgr.34104) 1716 : cluster 0 pgmap v1777: 105 pgs: 5 unknown, 100 active+clean; 582 KiB data, 5.1 GiB used, 710 GiB / 715 GiB avail; 1.4 MiB/s rd, 707 KiB/s wr, 118 op/s 2023-06-03T20:32:54.612 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:32:54 smithi089 bash[75068]: cluster 1685824373.2313874 mgr.y (mgr.34104) 1716 : cluster 0 pgmap v1777: 105 pgs: 5 unknown, 100 active+clean; 582 KiB data, 5.1 GiB used, 710 GiB / 715 GiB avail; 1.4 MiB/s rd, 707 KiB/s wr, 118 op/s 2023-06-03T20:32:54.649 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:32:54.686 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:32:54.689 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:32:54.690 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:32:54.791 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:32:54.824 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:32:54.826 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:32:54.827 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:32:54.928 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:32:54.961 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:32:54.964 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:32:54.964 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:32:55.029 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:32:54 smithi089 bash[78452]: cluster 1685824373.2313874 mgr.y (mgr.34104) 1716 : cluster 0 pgmap v1777: 105 pgs: 5 unknown, 100 active+clean; 582 KiB data, 5.1 GiB used, 710 GiB / 715 GiB avail; 1.4 MiB/s rd, 707 KiB/s wr, 118 op/s 2023-06-03T20:32:55.065 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:32:55.096 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:32:55.100 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:32:55.100 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:32:55.201 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:32:55.233 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:32:55.235 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:32:55.236 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:32:55.336 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:32:55.373 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:32:55.376 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:32:55.377 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:32:55.478 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:32:55.511 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:32:55.515 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:32:55.515 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:32:55.616 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:32:55.647 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:32:55.650 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:32:55.650 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:32:55.751 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:32:55.781 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:32:55.785 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:32:55.786 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:32:55.887 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:32:55.922 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:32:55.925 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:32:55.926 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:32:56.027 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:32:56.060 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:32:56.062 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:32:56.062 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:32:56.141 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/mon.a/config 2023-06-03T20:32:56.163 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:32:56.205 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:32:56.207 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:32:56.208 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:32:56.309 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:32:56.319 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 20:32:56 smithi089 bash[71163]: ::ffff:172.21.15.164 - - [03/Jun/2023:20:32:56] "GET /metrics HTTP/1.1" 200 228902 "" "Prometheus/2.33.4" 2023-06-03T20:32:56.340 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:32:56.343 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:32:56.343 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:32:56.444 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:32:56.474 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:32:56.477 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:32:56.477 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:32:56.578 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:32:56.585 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:32:56 smithi164 bash[59462]: cluster 1685824375.2320507 mgr.y (mgr.34104) 1717 : cluster 0 pgmap v1778: 105 pgs: 105 active+clean; 582 KiB data, 5.1 GiB used, 710 GiB / 715 GiB avail; 1.9 MiB/s rd, 1017 KiB/s wr, 897 op/s 2023-06-03T20:32:56.588 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:32:56 smithi089 bash[78452]: cluster 1685824375.2320507 mgr.y (mgr.34104) 1717 : cluster 0 pgmap v1778: 105 pgs: 105 active+clean; 582 KiB data, 5.1 GiB used, 710 GiB / 715 GiB avail; 1.9 MiB/s rd, 1017 KiB/s wr, 897 op/s 2023-06-03T20:32:56.589 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:32:56 smithi089 bash[75068]: cluster 1685824375.2320507 mgr.y (mgr.34104) 1717 : cluster 0 pgmap v1778: 105 pgs: 105 active+clean; 582 KiB data, 5.1 GiB used, 710 GiB / 715 GiB avail; 1.9 MiB/s rd, 1017 KiB/s wr, 897 op/s 2023-06-03T20:32:56.609 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:32:56.614 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:32:56.615 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:32:56.701 INFO:tasks.workunit.client.0.smithi089.stdout:[ OK ] TestClsRbd.mirror_image_status (4352 ms) 2023-06-03T20:32:56.702 INFO:tasks.workunit.client.0.smithi089.stdout:[ RUN ] TestClsRbd.mirror_image_map 2023-06-03T20:32:56.715 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:32:56.744 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:32:56.746 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:32:56.747 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:32:56.834 INFO:tasks.workunit.client.0.smithi089.stdout:[ OK ] TestClsRbd.mirror_image_map (133 ms) 2023-06-03T20:32:56.835 INFO:tasks.workunit.client.0.smithi089.stdout:[ RUN ] TestClsRbd.mirror_instances 2023-06-03T20:32:56.846 INFO:tasks.workunit.client.0.smithi089.stdout:[ OK ] TestClsRbd.mirror_instances (12 ms) 2023-06-03T20:32:56.847 INFO:tasks.workunit.client.0.smithi089.stdout:[ RUN ] TestClsRbd.mirror_snapshot 2023-06-03T20:32:56.848 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:32:56.854 INFO:tasks.workunit.client.0.smithi089.stdout:/build/ceph-16.2.13-257-gd8c5d349/src/test/cls_rbd/test_cls_rbd.cc:2278: Failure 2023-06-03T20:32:56.854 INFO:tasks.workunit.client.0.smithi089.stdout:Expected equality of these values: 2023-06-03T20:32:56.855 INFO:tasks.workunit.client.0.smithi089.stdout: -85 2023-06-03T20:32:56.855 INFO:tasks.workunit.client.0.smithi089.stdout: mirror_image_snapshot_unlink_peer(&ioctx, oid, 1, "peer2") 2023-06-03T20:32:56.855 INFO:tasks.workunit.client.0.smithi089.stdout: Which is: 0 2023-06-03T20:32:56.855 INFO:tasks.workunit.client.0.smithi089.stdout:[ FAILED ] TestClsRbd.mirror_snapshot (8 ms) 2023-06-03T20:32:56.855 INFO:tasks.workunit.client.0.smithi089.stdout:[ RUN ] TestClsRbd.group_dir_list 2023-06-03T20:32:56.860 INFO:tasks.workunit.client.0.smithi089.stdout:[ OK ] TestClsRbd.group_dir_list (5 ms) 2023-06-03T20:32:56.860 INFO:tasks.workunit.client.0.smithi089.stdout:[ RUN ] TestClsRbd.group_dir_add 2023-06-03T20:32:56.864 INFO:tasks.workunit.client.0.smithi089.stdout:[ OK ] TestClsRbd.group_dir_add (5 ms) 2023-06-03T20:32:56.865 INFO:tasks.workunit.client.0.smithi089.stdout:[ RUN ] TestClsRbd.dir_add_already_existing 2023-06-03T20:32:56.869 INFO:tasks.workunit.client.0.smithi089.stdout:[ OK ] TestClsRbd.dir_add_already_existing (5 ms) 2023-06-03T20:32:56.870 INFO:tasks.workunit.client.0.smithi089.stdout:[ RUN ] TestClsRbd.group_dir_rename 2023-06-03T20:32:56.872 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:32:56.874 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:32:56.875 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:32:56.879 INFO:tasks.workunit.client.0.smithi089.stdout:[ OK ] TestClsRbd.group_dir_rename (10 ms) 2023-06-03T20:32:56.880 INFO:tasks.workunit.client.0.smithi089.stdout:[ RUN ] TestClsRbd.group_dir_remove 2023-06-03T20:32:56.885 INFO:tasks.workunit.client.0.smithi089.stdout:[ OK ] TestClsRbd.group_dir_remove (6 ms) 2023-06-03T20:32:56.886 INFO:tasks.workunit.client.0.smithi089.stdout:[ RUN ] TestClsRbd.group_dir_remove_missing 2023-06-03T20:32:56.893 INFO:tasks.workunit.client.0.smithi089.stdout:[ OK ] TestClsRbd.group_dir_remove_missing (7 ms) 2023-06-03T20:32:56.893 INFO:tasks.workunit.client.0.smithi089.stdout:[ RUN ] TestClsRbd.group_image_add 2023-06-03T20:32:56.896 INFO:tasks.workunit.client.0.smithi089.stdout:[ OK ] TestClsRbd.group_image_add (4 ms) 2023-06-03T20:32:56.896 INFO:tasks.workunit.client.0.smithi089.stdout:[ RUN ] TestClsRbd.group_image_remove 2023-06-03T20:32:56.904 INFO:tasks.workunit.client.0.smithi089.stdout:[ OK ] TestClsRbd.group_image_remove (7 ms) 2023-06-03T20:32:56.904 INFO:tasks.workunit.client.0.smithi089.stdout:[ RUN ] TestClsRbd.group_image_list 2023-06-03T20:32:56.908 INFO:tasks.workunit.client.0.smithi089.stdout:[ OK ] TestClsRbd.group_image_list (4 ms) 2023-06-03T20:32:56.908 INFO:tasks.workunit.client.0.smithi089.stdout:[ RUN ] TestClsRbd.group_image_clean 2023-06-03T20:32:56.915 INFO:tasks.workunit.client.0.smithi089.stdout:[ OK ] TestClsRbd.group_image_clean (7 ms) 2023-06-03T20:32:56.915 INFO:tasks.workunit.client.0.smithi089.stdout:[ RUN ] TestClsRbd.image_group_add 2023-06-03T20:32:56.920 INFO:tasks.workunit.client.0.smithi089.stdout:[ OK ] TestClsRbd.image_group_add (5 ms) 2023-06-03T20:32:56.920 INFO:tasks.workunit.client.0.smithi089.stdout:[ RUN ] TestClsRbd.image_group_remove 2023-06-03T20:32:56.926 INFO:tasks.workunit.client.0.smithi089.stdout:[ OK ] TestClsRbd.image_group_remove (7 ms) 2023-06-03T20:32:56.926 INFO:tasks.workunit.client.0.smithi089.stdout:[ RUN ] TestClsRbd.image_group_get 2023-06-03T20:32:56.930 INFO:tasks.workunit.client.0.smithi089.stdout:[ OK ] TestClsRbd.image_group_get (4 ms) 2023-06-03T20:32:56.931 INFO:tasks.workunit.client.0.smithi089.stdout:[ RUN ] TestClsRbd.group_snap_set_empty_name 2023-06-03T20:32:56.934 INFO:tasks.workunit.client.0.smithi089.stdout:[ OK ] TestClsRbd.group_snap_set_empty_name (3 ms) 2023-06-03T20:32:56.934 INFO:tasks.workunit.client.0.smithi089.stdout:[ RUN ] TestClsRbd.group_snap_set_empty_id 2023-06-03T20:32:56.936 INFO:tasks.workunit.client.0.smithi089.stdout:[ OK ] TestClsRbd.group_snap_set_empty_id (3 ms) 2023-06-03T20:32:56.937 INFO:tasks.workunit.client.0.smithi089.stdout:[ RUN ] TestClsRbd.group_snap_set_duplicate_id 2023-06-03T20:32:56.940 INFO:tasks.workunit.client.0.smithi089.stdout:[ OK ] TestClsRbd.group_snap_set_duplicate_id (4 ms) 2023-06-03T20:32:56.940 INFO:tasks.workunit.client.0.smithi089.stdout:[ RUN ] TestClsRbd.group_snap_set_duplicate_name 2023-06-03T20:32:56.946 INFO:tasks.workunit.client.0.smithi089.stdout:[ OK ] TestClsRbd.group_snap_set_duplicate_name (5 ms) 2023-06-03T20:32:56.946 INFO:tasks.workunit.client.0.smithi089.stdout:[ RUN ] TestClsRbd.group_snap_set 2023-06-03T20:32:56.949 INFO:tasks.workunit.client.0.smithi089.stdout:[ OK ] TestClsRbd.group_snap_set (4 ms) 2023-06-03T20:32:56.950 INFO:tasks.workunit.client.0.smithi089.stdout:[ RUN ] TestClsRbd.group_snap_list 2023-06-03T20:32:56.955 INFO:tasks.workunit.client.0.smithi089.stdout:[ OK ] TestClsRbd.group_snap_list (6 ms) 2023-06-03T20:32:56.956 INFO:tasks.workunit.client.0.smithi089.stdout:[ RUN ] TestClsRbd.group_snap_list_max_return 2023-06-03T20:32:56.976 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:32:56.979 INFO:tasks.workunit.client.0.smithi089.stdout:[ OK ] TestClsRbd.group_snap_list_max_return (24 ms) 2023-06-03T20:32:56.979 INFO:tasks.workunit.client.0.smithi089.stdout:[ RUN ] TestClsRbd.group_snap_list_max_read 2023-06-03T20:32:57.012 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:32:57.015 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:32:57.016 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:32:57.116 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:32:57.151 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:32:57.154 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:32:57.154 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:32:57.255 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:32:57.280 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:32:57.283 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:32:57.283 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:32:57.367 INFO:tasks.workunit.client.0.smithi089.stdout:[ OK ] TestClsRbd.group_snap_list_max_read (387 ms) 2023-06-03T20:32:57.367 INFO:tasks.workunit.client.0.smithi089.stdout:[ RUN ] TestClsRbd.group_snap_remove 2023-06-03T20:32:57.373 INFO:tasks.workunit.client.0.smithi089.stdout:[ OK ] TestClsRbd.group_snap_remove (6 ms) 2023-06-03T20:32:57.373 INFO:tasks.workunit.client.0.smithi089.stdout:[ RUN ] TestClsRbd.group_snap_get_by_id 2023-06-03T20:32:57.376 INFO:tasks.workunit.client.0.smithi089.stdout:[ OK ] TestClsRbd.group_snap_get_by_id (4 ms) 2023-06-03T20:32:57.377 INFO:tasks.workunit.client.0.smithi089.stdout:[ RUN ] TestClsRbd.trash_methods 2023-06-03T20:32:57.384 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:32:57.391 INFO:tasks.workunit.client.0.smithi089.stdout:[ OK ] TestClsRbd.trash_methods (14 ms) 2023-06-03T20:32:57.391 INFO:tasks.workunit.client.0.smithi089.stdout:[ RUN ] TestClsRbd.op_features 2023-06-03T20:32:57.402 INFO:tasks.workunit.client.0.smithi089.stdout:[ OK ] TestClsRbd.op_features (12 ms) 2023-06-03T20:32:57.402 INFO:tasks.workunit.client.0.smithi089.stdout:[ RUN ] TestClsRbd.clone_parent 2023-06-03T20:32:57.417 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:32:57.419 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:32:57.420 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:32:57.428 INFO:tasks.workunit.client.0.smithi089.stdout:[ OK ] TestClsRbd.clone_parent (25 ms) 2023-06-03T20:32:57.428 INFO:tasks.workunit.client.0.smithi089.stdout:[ RUN ] TestClsRbd.clone_parent_ns 2023-06-03T20:32:57.447 INFO:tasks.workunit.client.0.smithi089.stdout:[ OK ] TestClsRbd.clone_parent_ns (20 ms) 2023-06-03T20:32:57.448 INFO:tasks.workunit.client.0.smithi089.stdout:[ RUN ] TestClsRbd.clone_child 2023-06-03T20:32:57.465 INFO:tasks.workunit.client.0.smithi089.stdout:[ OK ] TestClsRbd.clone_child (18 ms) 2023-06-03T20:32:57.465 INFO:tasks.workunit.client.0.smithi089.stdout:[ RUN ] TestClsRbd.namespace_methods 2023-06-03T20:32:57.480 INFO:tasks.workunit.client.0.smithi089.stdout:[ OK ] TestClsRbd.namespace_methods (14 ms) 2023-06-03T20:32:57.480 INFO:tasks.workunit.client.0.smithi089.stdout:[ RUN ] TestClsRbd.migration 2023-06-03T20:32:57.492 INFO:tasks.workunit.client.0.smithi089.stdout:[ OK ] TestClsRbd.migration (13 ms) 2023-06-03T20:32:57.493 INFO:tasks.workunit.client.0.smithi089.stdout:[ RUN ] TestClsRbd.migration_v1 2023-06-03T20:32:57.507 INFO:tasks.workunit.client.0.smithi089.stdout:[ OK ] TestClsRbd.migration_v1 (14 ms) 2023-06-03T20:32:57.507 INFO:tasks.workunit.client.0.smithi089.stdout:[ RUN ] TestClsRbd.assert_snapc_seq 2023-06-03T20:32:57.521 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:32:57.558 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:32:57.560 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:32:57.561 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:32:57.662 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:32:57.688 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:32:57.690 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:32:57.690 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:32:57.791 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:32:57.829 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:32:57.832 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:32:57.833 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:32:57.933 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:32:57.961 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:32:57.964 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:32:57.965 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:32:58.065 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:32:58.097 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:32:58.099 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:32:58.100 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:32:58.179 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/mon.a/config 2023-06-03T20:32:58.200 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:32:58.236 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:32:58.240 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:32:58.240 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:32:58.341 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:32:58.375 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:32:58.377 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:32:58.378 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:32:58.479 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:32:58.487 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:32:58 smithi089 bash[78452]: cluster 1685824377.2328851 mgr.y (mgr.34104) 1718 : cluster 0 pgmap v1779: 105 pgs: 105 active+clean; 582 KiB data, 5.1 GiB used, 710 GiB / 715 GiB avail; 1.6 MiB/s rd, 883 KiB/s wr, 782 op/s 2023-06-03T20:32:58.488 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:32:58 smithi089 bash[75068]: cluster 1685824377.2328851 mgr.y (mgr.34104) 1718 : cluster 0 pgmap v1779: 105 pgs: 105 active+clean; 582 KiB data, 5.1 GiB used, 710 GiB / 715 GiB avail; 1.6 MiB/s rd, 883 KiB/s wr, 782 op/s 2023-06-03T20:32:58.504 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3 2023-06-03T20:32:58.507 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:32:58.508 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.3' 2023-06-03T20:32:58.609 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:32:58.622 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:32:58 smithi164 bash[59462]: cluster 1685824377.2328851 mgr.y (mgr.34104) 1718 : cluster 0 pgmap v1779: 105 pgs: 105 active+clean; 582 KiB data, 5.1 GiB used, 710 GiB / 715 GiB avail; 1.6 MiB/s rd, 883 KiB/s wr, 782 op/s 2023-06-03T20:32:58.645 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:32:58.648 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:32:58.649 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:32:58.750 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:32:58.788 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:32:58.791 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:32:58.792 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:32:58.893 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:32:58.923 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0 2023-06-03T20:32:58.925 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:32:58.925 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.0' 2023-06-03T20:32:59.026 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:32:59.066 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:32:59.069 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:32:59.070 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:32:59.171 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:32:59.205 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2 2023-06-03T20:32:59.209 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:32:59.210 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.2' 2023-06-03T20:32:59.311 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:32:59.350 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:32:59.353 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:32:59.354 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:32:59.364 INFO:tasks.workunit.client.0.smithi089.stdout:[ OK ] TestClsRbd.assert_snapc_seq (1856 ms) 2023-06-03T20:32:59.364 INFO:tasks.workunit.client.0.smithi089.stdout:[ RUN ] TestClsRbd.sparsify 2023-06-03T20:32:59.426 INFO:tasks.workunit.client.0.smithi089.stdout:[ OK ] TestClsRbd.sparsify (63 ms) 2023-06-03T20:32:59.454 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:32:59.490 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:32:59.492 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:32:59.493 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:32:59.529 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:32:59 smithi089 bash[75068]: cluster 1685824378.3333783 osd.5 (osd.5) 250 : cluster 3 failed to encode map e302 with expected crc 2023-06-03T20:32:59.529 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:32:59 smithi089 bash[75068]: cluster 1685824378.3384073 mon.a (mon.0) 1682 : cluster 0 osdmap e302: 8 total, 8 up, 8 in 2023-06-03T20:32:59.530 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:32:59 smithi089 bash[75068]: cluster 1685824378.6149242 osd.6 (osd.6) 239 : cluster 3 failed to encode map e302 with expected crc 2023-06-03T20:32:59.530 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:32:59 smithi089 bash[78452]: cluster 1685824378.3333783 osd.5 (osd.5) 250 : cluster 3 failed to encode map e302 with expected crc 2023-06-03T20:32:59.530 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:32:59 smithi089 bash[78452]: cluster 1685824378.3384073 mon.a (mon.0) 1682 : cluster 0 osdmap e302: 8 total, 8 up, 8 in 2023-06-03T20:32:59.530 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:32:59 smithi089 bash[78452]: cluster 1685824378.6149242 osd.6 (osd.6) 239 : cluster 3 failed to encode map e302 with expected crc 2023-06-03T20:32:59.585 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:32:59 smithi164 bash[59462]: cluster 1685824378.3333783 osd.5 (osd.5) 250 : cluster 3 failed to encode map e302 with expected crc 2023-06-03T20:32:59.586 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:32:59 smithi164 bash[59462]: cluster 1685824378.3384073 mon.a (mon.0) 1682 : cluster 0 osdmap e302: 8 total, 8 up, 8 in 2023-06-03T20:32:59.586 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:32:59 smithi164 bash[59462]: cluster 1685824378.6149242 osd.6 (osd.6) 239 : cluster 3 failed to encode map e302 with expected crc 2023-06-03T20:32:59.594 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:32:59.632 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:32:59.635 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:32:59.636 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:32:59.737 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:32:59.772 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:32:59.775 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:32:59.776 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:32:59.877 DEBUG:teuthology.orchestra.run.smithi089:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:33:00.106 INFO:teuthology.orchestra.run.smithi089.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1 2023-06-03T20:33:00.107 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:33:00.108 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi089 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.1' 2023-06-03T20:33:00.209 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:33:00.244 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:33:00.246 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:33:00.247 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:33:00.348 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:33:00.367 INFO:tasks.workunit.client.0.smithi089.stdout:[----------] 70 tests from TestClsRbd (8101 ms total) 2023-06-03T20:33:00.367 INFO:tasks.workunit.client.0.smithi089.stdout: 2023-06-03T20:33:00.367 INFO:tasks.workunit.client.0.smithi089.stdout:[----------] Global test environment tear-down 2023-06-03T20:33:00.367 INFO:tasks.workunit.client.0.smithi089.stdout:[==========] 70 tests from 1 test suite ran. (10593 ms total) 2023-06-03T20:33:00.367 INFO:tasks.workunit.client.0.smithi089.stdout:[ PASSED ] 69 tests. 2023-06-03T20:33:00.368 INFO:tasks.workunit.client.0.smithi089.stdout:[ FAILED ] 1 test, listed below: 2023-06-03T20:33:00.368 INFO:tasks.workunit.client.0.smithi089.stdout:[ FAILED ] TestClsRbd.mirror_snapshot 2023-06-03T20:33:00.368 INFO:tasks.workunit.client.0.smithi089.stdout: 2023-06-03T20:33:00.368 INFO:tasks.workunit.client.0.smithi089.stdout: 1 FAILED TEST 2023-06-03T20:33:00.369 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:33:00.369 INFO:tasks.workunit:Stopping ['cls/test_cls_rbd.sh'] on client.0... 2023-06-03T20:33:00.370 DEBUG:teuthology.orchestra.run.smithi089:> sudo rm -rf -- /home/ubuntu/cephtest/workunits.list.client.0 /home/ubuntu/cephtest/clone.client.0 2023-06-03T20:33:00.383 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4 2023-06-03T20:33:00.385 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:33:00.386 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.4' 2023-06-03T20:33:00.487 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:33:00.501 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:33:00 smithi164 bash[59462]: cluster 1685824378.615587 osd.7 (osd.7) 245 : cluster 3 failed to encode map e302 with expected crc 2023-06-03T20:33:00.501 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:33:00 smithi164 bash[59462]: cluster 1685824379.2333798 mgr.y (mgr.34104) 1719 : cluster 0 pgmap v1781: 105 pgs: 105 active+clean; 582 KiB data, 5.1 GiB used, 710 GiB / 715 GiB avail; 1.6 MiB/s rd, 806 KiB/s wr, 812 op/s 2023-06-03T20:33:00.501 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:33:00 smithi164 bash[59462]: cluster 1685824379.349491 osd.6 (osd.6) 240 : cluster 3 failed to encode map e303 with expected crc 2023-06-03T20:33:00.501 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:33:00 smithi164 bash[59462]: cluster 1685824379.3583179 mon.a (mon.0) 1683 : cluster 0 osdmap e303: 8 total, 8 up, 8 in 2023-06-03T20:33:00.524 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5 2023-06-03T20:33:00.526 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:33:00.527 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.5' 2023-06-03T20:33:00.628 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:33:00.667 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6 2023-06-03T20:33:00.670 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:33:00.670 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.6' 2023-06-03T20:33:00.771 DEBUG:teuthology.orchestra.run.smithi164:> sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:33:00.774 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:33:00 smithi089 bash[75068]: cluster 1685824378.615587 osd.7 (osd.7) 245 : cluster 3 failed to encode map e302 with expected crc 2023-06-03T20:33:00.774 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:33:00 smithi089 bash[75068]: cluster 1685824379.2333798 mgr.y (mgr.34104) 1719 : cluster 0 pgmap v1781: 105 pgs: 105 active+clean; 582 KiB data, 5.1 GiB used, 710 GiB / 715 GiB avail; 1.6 MiB/s rd, 806 KiB/s wr, 812 op/s 2023-06-03T20:33:00.775 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:33:00 smithi089 bash[75068]: cluster 1685824379.349491 osd.6 (osd.6) 240 : cluster 3 failed to encode map e303 with expected crc 2023-06-03T20:33:00.775 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:33:00 smithi089 bash[75068]: cluster 1685824379.3583179 mon.a (mon.0) 1683 : cluster 0 osdmap e303: 8 total, 8 up, 8 in 2023-06-03T20:33:00.775 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:33:00 smithi089 bash[78452]: cluster 1685824378.615587 osd.7 (osd.7) 245 : cluster 3 failed to encode map e302 with expected crc 2023-06-03T20:33:00.775 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:33:00 smithi089 bash[78452]: cluster 1685824379.2333798 mgr.y (mgr.34104) 1719 : cluster 0 pgmap v1781: 105 pgs: 105 active+clean; 582 KiB data, 5.1 GiB used, 710 GiB / 715 GiB avail; 1.6 MiB/s rd, 806 KiB/s wr, 812 op/s 2023-06-03T20:33:00.775 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:33:00 smithi089 bash[78452]: cluster 1685824379.349491 osd.6 (osd.6) 240 : cluster 3 failed to encode map e303 with expected crc 2023-06-03T20:33:00.775 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:33:00 smithi089 bash[78452]: cluster 1685824379.3583179 mon.a (mon.0) 1683 : cluster 0 osdmap e303: 8 total, 8 up, 8 in 2023-06-03T20:33:00.798 INFO:tasks.thrashosds:joining thrashosds 2023-06-03T20:33:00.809 INFO:teuthology.orchestra.run.smithi164.stderr:Error response from daemon: Cannot kill container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7: No such container: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7 2023-06-03T20:33:00.812 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:33:00.813 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi164 with status 1: 'sudo docker kill -s 1 ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd.7' 2023-06-03T20:33:01.627 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:33:01 smithi089 bash[75068]: cluster 1685824379.9986975 osd.7 (osd.7) 246 : cluster 3 failed to encode map e303 with expected crc 2023-06-03T20:33:01.627 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:33:01 smithi089 bash[75068]: cluster 1685824380.1859393 osd.5 (osd.5) 251 : cluster 3 failed to encode map e303 with expected crc 2023-06-03T20:33:01.627 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:33:01 smithi089 bash[75068]: cluster 1685824380.3557458 osd.5 (osd.5) 252 : cluster 3 failed to encode map e304 with expected crc 2023-06-03T20:33:01.628 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:33:01 smithi089 bash[75068]: cluster 1685824380.3606818 mon.a (mon.0) 1684 : cluster 0 osdmap e304: 8 total, 8 up, 8 in 2023-06-03T20:33:01.628 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:33:01 smithi089 bash[75068]: cluster 1685824380.7426066 osd.6 (osd.6) 241 : cluster 3 failed to encode map e304 with expected crc 2023-06-03T20:33:01.628 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:33:01 smithi089 bash[78452]: cluster 1685824379.9986975 osd.7 (osd.7) 246 : cluster 3 failed to encode map e303 with expected crc 2023-06-03T20:33:01.628 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:33:01 smithi089 bash[78452]: cluster 1685824380.1859393 osd.5 (osd.5) 251 : cluster 3 failed to encode map e303 with expected crc 2023-06-03T20:33:01.628 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:33:01 smithi089 bash[78452]: cluster 1685824380.3557458 osd.5 (osd.5) 252 : cluster 3 failed to encode map e304 with expected crc 2023-06-03T20:33:01.629 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:33:01 smithi089 bash[78452]: cluster 1685824380.3606818 mon.a (mon.0) 1684 : cluster 0 osdmap e304: 8 total, 8 up, 8 in 2023-06-03T20:33:01.629 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:33:01 smithi089 bash[78452]: cluster 1685824380.7426066 osd.6 (osd.6) 241 : cluster 3 failed to encode map e304 with expected crc 2023-06-03T20:33:01.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:33:01 smithi164 bash[59462]: cluster 1685824379.9986975 osd.7 (osd.7) 246 : cluster 3 failed to encode map e303 with expected crc 2023-06-03T20:33:01.836 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:33:01 smithi164 bash[59462]: cluster 1685824380.1859393 osd.5 (osd.5) 251 : cluster 3 failed to encode map e303 with expected crc 2023-06-03T20:33:01.836 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:33:01 smithi164 bash[59462]: cluster 1685824380.3557458 osd.5 (osd.5) 252 : cluster 3 failed to encode map e304 with expected crc 2023-06-03T20:33:01.836 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:33:01 smithi164 bash[59462]: cluster 1685824380.3606818 mon.a (mon.0) 1684 : cluster 0 osdmap e304: 8 total, 8 up, 8 in 2023-06-03T20:33:01.836 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:33:01 smithi164 bash[59462]: cluster 1685824380.7426066 osd.6 (osd.6) 241 : cluster 3 failed to encode map e304 with expected crc 2023-06-03T20:33:02.673 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:33:02 smithi089 bash[78452]: cluster 1685824381.2337236 mgr.y (mgr.34104) 1720 : cluster 0 pgmap v1784: 73 pgs: 73 active+clean; 582 KiB data, 5.1 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:33:02.674 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:33:02 smithi089 bash[75068]: cluster 1685824381.2337236 mgr.y (mgr.34104) 1720 : cluster 0 pgmap v1784: 73 pgs: 73 active+clean; 582 KiB data, 5.1 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:33:02.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:33:02 smithi164 bash[59462]: cluster 1685824381.2337236 mgr.y (mgr.34104) 1720 : cluster 0 pgmap v1784: 73 pgs: 73 active+clean; 582 KiB data, 5.1 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:33:03.779 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:33:03 smithi089 bash[78452]: cluster 1685824381.7752724 osd.7 (osd.7) 247 : cluster 3 failed to encode map e304 with expected crc 2023-06-03T20:33:03.780 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:33:03 smithi089 bash[78452]: cluster 1685824381.7779431 osd.7 (osd.7) 248 : cluster 3 failed to encode map e304 with expected crc 2023-06-03T20:33:03.780 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:33:03 smithi089 bash[78452]: cluster 1685824381.7805364 osd.7 (osd.7) 249 : cluster 3 failed to encode map e304 with expected crc 2023-06-03T20:33:03.780 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:33:03 smithi089 bash[78452]: audit 1685824383.1205196 mon.a (mon.0) 1685 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:33:03.780 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:33:03 smithi089 bash[75068]: cluster 1685824381.7752724 osd.7 (osd.7) 247 : cluster 3 failed to encode map e304 with expected crc 2023-06-03T20:33:03.780 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:33:03 smithi089 bash[75068]: cluster 1685824381.7779431 osd.7 (osd.7) 248 : cluster 3 failed to encode map e304 with expected crc 2023-06-03T20:33:03.781 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:33:03 smithi089 bash[75068]: cluster 1685824381.7805364 osd.7 (osd.7) 249 : cluster 3 failed to encode map e304 with expected crc 2023-06-03T20:33:03.781 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:33:03 smithi089 bash[75068]: audit 1685824383.1205196 mon.a (mon.0) 1685 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:33:03.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:33:03 smithi164 bash[59462]: cluster 1685824381.7752724 osd.7 (osd.7) 247 : cluster 3 failed to encode map e304 with expected crc 2023-06-03T20:33:03.836 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:33:03 smithi164 bash[59462]: cluster 1685824381.7779431 osd.7 (osd.7) 248 : cluster 3 failed to encode map e304 with expected crc 2023-06-03T20:33:03.836 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:33:03 smithi164 bash[59462]: cluster 1685824381.7805364 osd.7 (osd.7) 249 : cluster 3 failed to encode map e304 with expected crc 2023-06-03T20:33:03.836 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:33:03 smithi164 bash[59462]: audit 1685824383.1205196 mon.a (mon.0) 1685 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:33:04.529 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:33:04 smithi089 bash[78452]: cluster 1685824383.2346184 mgr.y (mgr.34104) 1721 : cluster 0 pgmap v1785: 73 pgs: 73 active+clean; 582 KiB data, 5.1 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:33:04.530 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:33:04 smithi089 bash[75068]: cluster 1685824383.2346184 mgr.y (mgr.34104) 1721 : cluster 0 pgmap v1785: 73 pgs: 73 active+clean; 582 KiB data, 5.1 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:33:04.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:33:04 smithi164 bash[59462]: cluster 1685824383.2346184 mgr.y (mgr.34104) 1721 : cluster 0 pgmap v1785: 73 pgs: 73 active+clean; 582 KiB data, 5.1 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:33:06.488 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 20:33:06 smithi089 bash[71163]: ::ffff:172.21.15.164 - - [03/Jun/2023:20:33:06] "GET /metrics HTTP/1.1" 200 228902 "" "Prometheus/2.33.4" 2023-06-03T20:33:06.488 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:33:06 smithi089 bash[75068]: cluster 1685824385.2351613 mgr.y (mgr.34104) 1722 : cluster 0 pgmap v1786: 73 pgs: 73 active+clean; 582 KiB data, 5.1 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:33:06.779 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:33:06 smithi089 bash[78452]: cluster 1685824385.2351613 mgr.y (mgr.34104) 1722 : cluster 0 pgmap v1786: 73 pgs: 73 active+clean; 582 KiB data, 5.1 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:33:06.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:33:06 smithi164 bash[59462]: cluster 1685824385.2351613 mgr.y (mgr.34104) 1722 : cluster 0 pgmap v1786: 73 pgs: 73 active+clean; 582 KiB data, 5.1 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:33:07.562 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:33:07 smithi089 bash[75068]: audit 1685824387.1626847 mon.c (mon.2) 688 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T20:33:07.562 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:33:07 smithi089 bash[78452]: audit 1685824387.1626847 mon.c (mon.2) 688 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T20:33:07.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:33:07 smithi164 bash[59462]: audit 1685824387.1626847 mon.c (mon.2) 688 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T20:33:07.862 INFO:teuthology.orchestra.run.smithi089.stdout:ERROR: (22) Invalid argument 2023-06-03T20:33:07.862 INFO:teuthology.orchestra.run.smithi089.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-06-03T20:33:08.699 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:33:08 smithi089 bash[75068]: cluster 1685824387.23587 mgr.y (mgr.34104) 1723 : cluster 0 pgmap v1787: 73 pgs: 73 active+clean; 582 KiB data, 5.1 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:33:08.699 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:33:08 smithi089 bash[78452]: cluster 1685824387.23587 mgr.y (mgr.34104) 1723 : cluster 0 pgmap v1787: 73 pgs: 73 active+clean; 582 KiB data, 5.1 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:33:08.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:33:08 smithi164 bash[59462]: cluster 1685824387.23587 mgr.y (mgr.34104) 1723 : cluster 0 pgmap v1787: 73 pgs: 73 active+clean; 582 KiB data, 5.1 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:33:09.896 INFO:teuthology.orchestra.run.smithi089.stdout:{ 2023-06-03T20:33:09.896 INFO:teuthology.orchestra.run.smithi089.stdout: "success": "" 2023-06-03T20:33:09.897 INFO:teuthology.orchestra.run.smithi089.stdout:} 2023-06-03T20:33:10.779 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:33:10 smithi089 bash[78452]: cluster 1685824389.2364655 mgr.y (mgr.34104) 1724 : cluster 0 pgmap v1788: 73 pgs: 73 active+clean; 582 KiB data, 5.1 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:33:10.779 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:33:10 smithi089 bash[75068]: cluster 1685824389.2364655 mgr.y (mgr.34104) 1724 : cluster 0 pgmap v1788: 73 pgs: 73 active+clean; 582 KiB data, 5.1 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:33:10.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:33:10 smithi164 bash[59462]: cluster 1685824389.2364655 mgr.y (mgr.34104) 1724 : cluster 0 pgmap v1788: 73 pgs: 73 active+clean; 582 KiB data, 5.1 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:33:11.307 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph daemon osd.3 dump_blocked_ops 2023-06-03T20:33:11.476 INFO:teuthology.orchestra.run.smithi089.stdout:osd.0: {} 2023-06-03T20:33:11.497 INFO:teuthology.orchestra.run.smithi089.stdout:osd.1: {} 2023-06-03T20:33:11.520 INFO:teuthology.orchestra.run.smithi089.stdout:osd.2: {} 2023-06-03T20:33:11.541 INFO:teuthology.orchestra.run.smithi089.stdout:osd.3: {} 2023-06-03T20:33:11.564 INFO:teuthology.orchestra.run.smithi089.stdout:osd.4: {} 2023-06-03T20:33:11.585 INFO:teuthology.orchestra.run.smithi089.stdout:osd.5: {} 2023-06-03T20:33:11.613 INFO:teuthology.orchestra.run.smithi089.stdout:osd.6: {} 2023-06-03T20:33:11.640 INFO:teuthology.orchestra.run.smithi089.stdout:osd.7: {} 2023-06-03T20:33:11.642 INFO:teuthology.orchestra.run.smithi089.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-06-03T20:33:11.642 INFO:teuthology.orchestra.run.smithi089.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-06-03T20:33:11.642 INFO:teuthology.orchestra.run.smithi089.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-06-03T20:33:11.642 INFO:teuthology.orchestra.run.smithi089.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-06-03T20:33:11.643 INFO:teuthology.orchestra.run.smithi089.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-06-03T20:33:11.643 INFO:teuthology.orchestra.run.smithi089.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-06-03T20:33:11.643 INFO:teuthology.orchestra.run.smithi089.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-06-03T20:33:11.643 INFO:teuthology.orchestra.run.smithi089.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-06-03T20:33:12.416 INFO:teuthology.orchestra.run.smithi089.stderr:nodeep-scrub is unset 2023-06-03T20:33:12.779 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:33:12 smithi089 bash[75068]: cluster 1685824391.2371783 mgr.y (mgr.34104) 1725 : cluster 0 pgmap v1789: 73 pgs: 73 active+clean; 582 KiB data, 5.1 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:33:12.779 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:33:12 smithi089 bash[75068]: audit 1685824391.4506183 mon.c (mon.2) 689 : audit 0 from='client.? 172.21.15.89:0/802208725' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-03T20:33:12.780 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:33:12 smithi089 bash[75068]: audit 1685824392.2470088 mon.c (mon.2) 690 : audit 1 from='client.? 172.21.15.89:0/3988911905' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-06-03T20:33:12.780 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:33:12 smithi089 bash[75068]: audit 1685824392.247438 mon.a (mon.0) 1686 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-06-03T20:33:12.780 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:33:12 smithi089 bash[78452]: cluster 1685824391.2371783 mgr.y (mgr.34104) 1725 : cluster 0 pgmap v1789: 73 pgs: 73 active+clean; 582 KiB data, 5.1 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:33:12.780 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:33:12 smithi089 bash[78452]: audit 1685824391.4506183 mon.c (mon.2) 689 : audit 0 from='client.? 172.21.15.89:0/802208725' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-03T20:33:12.780 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:33:12 smithi089 bash[78452]: audit 1685824392.2470088 mon.c (mon.2) 690 : audit 1 from='client.? 172.21.15.89:0/3988911905' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-06-03T20:33:12.780 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:33:12 smithi089 bash[78452]: audit 1685824392.247438 mon.a (mon.0) 1686 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-06-03T20:33:12.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:33:12 smithi164 bash[59462]: cluster 1685824391.2371783 mgr.y (mgr.34104) 1725 : cluster 0 pgmap v1789: 73 pgs: 73 active+clean; 582 KiB data, 5.1 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:33:12.836 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:33:12 smithi164 bash[59462]: audit 1685824391.4506183 mon.c (mon.2) 689 : audit 0 from='client.? 172.21.15.89:0/802208725' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-06-03T20:33:12.836 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:33:12 smithi164 bash[59462]: audit 1685824392.2470088 mon.c (mon.2) 690 : audit 1 from='client.? 172.21.15.89:0/3988911905' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-06-03T20:33:12.836 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:33:12 smithi164 bash[59462]: audit 1685824392.247438 mon.a (mon.0) 1686 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-06-03T20:33:13.779 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:33:13 smithi089 bash[75068]: cluster 1685824392.3904436 mon.a (mon.0) 1687 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-06-03T20:33:13.779 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:33:13 smithi089 bash[75068]: cluster 1685824392.3905141 mon.a (mon.0) 1688 : cluster 1 Cluster is now healthy 2023-06-03T20:33:13.780 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:33:13 smithi089 bash[75068]: cluster 1685824392.402913 osd.5 (osd.5) 253 : cluster 3 failed to encode map e305 with expected crc 2023-06-03T20:33:13.780 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:33:13 smithi089 bash[75068]: audit 1685824392.404557 mon.a (mon.0) 1689 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-06-03T20:33:13.780 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:33:13 smithi089 bash[75068]: cluster 1685824392.40461 mon.a (mon.0) 1690 : cluster 0 osdmap e305: 8 total, 8 up, 8 in 2023-06-03T20:33:13.780 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:33:13 smithi089 bash[75068]: cluster 1685824393.2380378 mgr.y (mgr.34104) 1726 : cluster 0 pgmap v1791: 73 pgs: 73 active+clean; 582 KiB data, 5.1 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:33:13.780 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:33:13 smithi089 bash[78452]: cluster 1685824392.3904436 mon.a (mon.0) 1687 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-06-03T20:33:13.781 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:33:13 smithi089 bash[78452]: cluster 1685824392.3905141 mon.a (mon.0) 1688 : cluster 1 Cluster is now healthy 2023-06-03T20:33:13.781 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:33:13 smithi089 bash[78452]: cluster 1685824392.402913 osd.5 (osd.5) 253 : cluster 3 failed to encode map e305 with expected crc 2023-06-03T20:33:13.781 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:33:13 smithi089 bash[78452]: audit 1685824392.404557 mon.a (mon.0) 1689 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-06-03T20:33:13.781 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:33:13 smithi089 bash[78452]: cluster 1685824392.40461 mon.a (mon.0) 1690 : cluster 0 osdmap e305: 8 total, 8 up, 8 in 2023-06-03T20:33:13.781 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:33:13 smithi089 bash[78452]: cluster 1685824393.2380378 mgr.y (mgr.34104) 1726 : cluster 0 pgmap v1791: 73 pgs: 73 active+clean; 582 KiB data, 5.1 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:33:13.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:33:13 smithi164 bash[59462]: cluster 1685824392.3904436 mon.a (mon.0) 1687 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-06-03T20:33:13.836 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:33:13 smithi164 bash[59462]: cluster 1685824392.3905141 mon.a (mon.0) 1688 : cluster 1 Cluster is now healthy 2023-06-03T20:33:13.836 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:33:13 smithi164 bash[59462]: cluster 1685824392.402913 osd.5 (osd.5) 253 : cluster 3 failed to encode map e305 with expected crc 2023-06-03T20:33:13.836 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:33:13 smithi164 bash[59462]: audit 1685824392.404557 mon.a (mon.0) 1689 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-06-03T20:33:13.836 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:33:13 smithi164 bash[59462]: cluster 1685824392.40461 mon.a (mon.0) 1690 : cluster 0 osdmap e305: 8 total, 8 up, 8 in 2023-06-03T20:33:13.836 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:33:13 smithi164 bash[59462]: cluster 1685824393.2380378 mgr.y (mgr.34104) 1726 : cluster 0 pgmap v1791: 73 pgs: 73 active+clean; 582 KiB data, 5.1 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:33:14.779 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:33:14 smithi089 bash[75068]: cluster 1685824393.0454996 osd.7 (osd.7) 250 : cluster 3 failed to encode map e305 with expected crc 2023-06-03T20:33:14.780 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:33:14 smithi089 bash[75068]: cluster 1685824393.6448355 osd.6 (osd.6) 242 : cluster 3 failed to encode map e305 with expected crc 2023-06-03T20:33:14.780 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:33:14 smithi089 bash[75068]: cluster 1685824393.6476858 osd.6 (osd.6) 243 : cluster 3 failed to encode map e305 with expected crc 2023-06-03T20:33:14.780 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:33:14 smithi089 bash[78452]: cluster 1685824393.0454996 osd.7 (osd.7) 250 : cluster 3 failed to encode map e305 with expected crc 2023-06-03T20:33:14.781 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:33:14 smithi089 bash[78452]: cluster 1685824393.6448355 osd.6 (osd.6) 242 : cluster 3 failed to encode map e305 with expected crc 2023-06-03T20:33:14.781 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:33:14 smithi089 bash[78452]: cluster 1685824393.6476858 osd.6 (osd.6) 243 : cluster 3 failed to encode map e305 with expected crc 2023-06-03T20:33:14.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:33:14 smithi164 bash[59462]: cluster 1685824393.0454996 osd.7 (osd.7) 250 : cluster 3 failed to encode map e305 with expected crc 2023-06-03T20:33:14.836 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:33:14 smithi164 bash[59462]: cluster 1685824393.6448355 osd.6 (osd.6) 242 : cluster 3 failed to encode map e305 with expected crc 2023-06-03T20:33:14.836 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:33:14 smithi164 bash[59462]: cluster 1685824393.6476858 osd.6 (osd.6) 243 : cluster 3 failed to encode map e305 with expected crc 2023-06-03T20:33:15.593 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph osd unset noscrub 2023-06-03T20:33:15.779 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:33:15 smithi089 bash[75068]: cluster 1685824395.2387338 mgr.y (mgr.34104) 1727 : cluster 0 pgmap v1792: 73 pgs: 73 active+clean; 582 KiB data, 5.1 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:33:15.779 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:33:15 smithi089 bash[78452]: cluster 1685824395.2387338 mgr.y (mgr.34104) 1727 : cluster 0 pgmap v1792: 73 pgs: 73 active+clean; 582 KiB data, 5.1 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:33:15.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:33:15 smithi164 bash[59462]: cluster 1685824395.2387338 mgr.y (mgr.34104) 1727 : cluster 0 pgmap v1792: 73 pgs: 73 active+clean; 582 KiB data, 5.1 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:33:16.417 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/mon.a/config 2023-06-03T20:33:16.529 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 20:33:16 smithi089 bash[71163]: ::ffff:172.21.15.164 - - [03/Jun/2023:20:33:16] "GET /metrics HTTP/1.1" 200 227006 "" "Prometheus/2.33.4" 2023-06-03T20:33:16.672 INFO:tasks.thrashosds.thrasher:fixing pg num pool device_health_metrics 2023-06-03T20:33:16.673 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph osd pool set device_health_metrics pgp_num 21 2023-06-03T20:33:18.529 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:33:18 smithi089 bash[75068]: cluster 1685824397.239184 mgr.y (mgr.34104) 1728 : cluster 0 pgmap v1793: 73 pgs: 73 active+clean; 582 KiB data, 5.1 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:33:18.529 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:33:18 smithi089 bash[78452]: cluster 1685824397.239184 mgr.y (mgr.34104) 1728 : cluster 0 pgmap v1793: 73 pgs: 73 active+clean; 582 KiB data, 5.1 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:33:18.585 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:33:18 smithi164 bash[59462]: cluster 1685824397.239184 mgr.y (mgr.34104) 1728 : cluster 0 pgmap v1793: 73 pgs: 73 active+clean; 582 KiB data, 5.1 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:33:19.569 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/mon.a/config 2023-06-03T20:33:20.512 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/mon.a/config 2023-06-03T20:33:20.585 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:33:20 smithi164 bash[59462]: cluster 1685824399.240162 mgr.y (mgr.34104) 1729 : cluster 0 pgmap v1794: 73 pgs: 73 active+clean; 582 KiB data, 5.1 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:33:20.779 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:33:20 smithi089 bash[75068]: cluster 1685824399.240162 mgr.y (mgr.34104) 1729 : cluster 0 pgmap v1794: 73 pgs: 73 active+clean; 582 KiB data, 5.1 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:33:20.779 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:33:20 smithi089 bash[78452]: cluster 1685824399.240162 mgr.y (mgr.34104) 1729 : cluster 0 pgmap v1794: 73 pgs: 73 active+clean; 582 KiB data, 5.1 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:33:22.480 INFO:teuthology.orchestra.run.smithi089.stdout:{ 2023-06-03T20:33:22.481 INFO:teuthology.orchestra.run.smithi089.stdout: "ops": [], 2023-06-03T20:33:22.481 INFO:teuthology.orchestra.run.smithi089.stdout: "complaint_time": 30, 2023-06-03T20:33:22.481 INFO:teuthology.orchestra.run.smithi089.stdout: "num_blocked_ops": 0 2023-06-03T20:33:22.481 INFO:teuthology.orchestra.run.smithi089.stdout:} 2023-06-03T20:33:22.500 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:33:22 smithi089 bash[75068]: cluster 1685824401.2407625 mgr.y (mgr.34104) 1730 : cluster 0 pgmap v1795: 73 pgs: 73 active+clean; 582 KiB data, 5.1 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:33:22.500 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:33:22 smithi089 bash[75068]: audit 1685824402.1698096 mon.a (mon.0) 1691 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:33:22.500 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:33:22 smithi089 bash[75068]: audit 1685824402.1713734 mon.c (mon.2) 691 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T20:33:22.501 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:33:22 smithi089 bash[78452]: cluster 1685824401.2407625 mgr.y (mgr.34104) 1730 : cluster 0 pgmap v1795: 73 pgs: 73 active+clean; 582 KiB data, 5.1 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:33:22.501 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:33:22 smithi089 bash[78452]: audit 1685824402.1698096 mon.a (mon.0) 1691 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:33:22.501 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:33:22 smithi089 bash[78452]: audit 1685824402.1713734 mon.c (mon.2) 691 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T20:33:22.585 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:33:22 smithi164 bash[59462]: cluster 1685824401.2407625 mgr.y (mgr.34104) 1730 : cluster 0 pgmap v1795: 73 pgs: 73 active+clean; 582 KiB data, 5.1 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:33:22.586 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:33:22 smithi164 bash[59462]: audit 1685824402.1698096 mon.a (mon.0) 1691 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:33:22.586 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:33:22 smithi164 bash[59462]: audit 1685824402.1713734 mon.c (mon.2) 691 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T20:33:24.450 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:33:24 smithi089 bash[78452]: cluster 1685824403.2417402 mgr.y (mgr.34104) 1731 : cluster 0 pgmap v1796: 73 pgs: 73 active+clean; 582 KiB data, 5.1 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:33:24.451 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:33:24 smithi089 bash[75068]: cluster 1685824403.2417402 mgr.y (mgr.34104) 1731 : cluster 0 pgmap v1796: 73 pgs: 73 active+clean; 582 KiB data, 5.1 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:33:24.585 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:33:24 smithi164 bash[59462]: cluster 1685824403.2417402 mgr.y (mgr.34104) 1731 : cluster 0 pgmap v1796: 73 pgs: 73 active+clean; 582 KiB data, 5.1 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:33:24.621 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph daemon osd.3 dump_historic_ops 2023-06-03T20:33:26.502 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 20:33:26 smithi089 bash[71163]: ::ffff:172.21.15.164 - - [03/Jun/2023:20:33:26] "GET /metrics HTTP/1.1" 200 226974 "" "Prometheus/2.33.4" 2023-06-03T20:33:26.503 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:33:26 smithi089 bash[75068]: cluster 1685824405.2425878 mgr.y (mgr.34104) 1732 : cluster 0 pgmap v1797: 73 pgs: 73 active+clean; 582 KiB data, 5.1 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:33:26.503 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:33:26 smithi089 bash[78452]: cluster 1685824405.2425878 mgr.y (mgr.34104) 1732 : cluster 0 pgmap v1797: 73 pgs: 73 active+clean; 582 KiB data, 5.1 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:33:26.585 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:33:26 smithi164 bash[59462]: cluster 1685824405.2425878 mgr.y (mgr.34104) 1732 : cluster 0 pgmap v1797: 73 pgs: 73 active+clean; 582 KiB data, 5.1 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:33:26.916 INFO:teuthology.orchestra.run.smithi089.stderr:Error ENOENT: unrecognized pool 'device_health_metrics' 2023-06-03T20:33:27.322 INFO:teuthology.orchestra.run.smithi089.stderr:noscrub is unset 2023-06-03T20:33:27.378 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:33:27 smithi089 bash[78452]: audit 1685824406.3403099 mon.c (mon.2) 692 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-03T20:33:27.378 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:33:27 smithi089 bash[78452]: audit 1685824406.9096987 mon.a (mon.0) 1692 : audit 1 from='client.? 172.21.15.89:0/2237121969' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "device_health_metrics", "var": "pgp_num", "val": "21"}]: dispatch 2023-06-03T20:33:27.378 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:33:27 smithi089 bash[78452]: audit 1685824406.9248998 mon.c (mon.2) 693 : audit 1 from='client.? 172.21.15.89:0/2182834240' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-06-03T20:33:27.378 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:33:27 smithi089 bash[78452]: audit 1685824406.9252844 mon.a (mon.0) 1693 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-06-03T20:33:27.379 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:33:27 smithi089 bash[75068]: audit 1685824406.3403099 mon.c (mon.2) 692 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-03T20:33:27.379 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:33:27 smithi089 bash[75068]: audit 1685824406.9096987 mon.a (mon.0) 1692 : audit 1 from='client.? 172.21.15.89:0/2237121969' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "device_health_metrics", "var": "pgp_num", "val": "21"}]: dispatch 2023-06-03T20:33:27.379 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:33:27 smithi089 bash[75068]: audit 1685824406.9248998 mon.c (mon.2) 693 : audit 1 from='client.? 172.21.15.89:0/2182834240' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-06-03T20:33:27.379 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:33:27 smithi089 bash[75068]: audit 1685824406.9252844 mon.a (mon.0) 1693 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-06-03T20:33:27.585 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:33:27 smithi164 bash[59462]: audit 1685824406.3403099 mon.c (mon.2) 692 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-03T20:33:27.586 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:33:27 smithi164 bash[59462]: audit 1685824406.9096987 mon.a (mon.0) 1692 : audit 1 from='client.? 172.21.15.89:0/2237121969' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "device_health_metrics", "var": "pgp_num", "val": "21"}]: dispatch 2023-06-03T20:33:27.586 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:33:27 smithi164 bash[59462]: audit 1685824406.9248998 mon.c (mon.2) 693 : audit 1 from='client.? 172.21.15.89:0/2182834240' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-06-03T20:33:27.586 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:33:27 smithi164 bash[59462]: audit 1685824406.9252844 mon.a (mon.0) 1693 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-06-03T20:33:28.418 DEBUG:teuthology.orchestra.run:got remote process result: 2 2023-06-03T20:33:28.419 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph -- tell 'mon.*' injectargs --mon_osd_down_out_interval=600 2023-06-03T20:33:28.585 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:33:28 smithi164 bash[59462]: cluster 1685824407.2433736 mgr.y (mgr.34104) 1733 : cluster 0 pgmap v1798: 73 pgs: 73 active+clean; 582 KiB data, 5.1 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:33:28.586 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:33:28 smithi164 bash[59462]: audit 1685824407.3111403 mon.a (mon.0) 1694 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-06-03T20:33:28.586 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:33:28 smithi164 bash[59462]: cluster 1685824407.3112552 mon.a (mon.0) 1695 : cluster 0 osdmap e306: 8 total, 8 up, 8 in 2023-06-03T20:33:28.586 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:33:28 smithi164 bash[59462]: cluster 1685824407.3608983 osd.5 (osd.5) 254 : cluster 3 failed to encode map e306 with expected crc 2023-06-03T20:33:28.586 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:33:28 smithi164 bash[59462]: cluster 1685824407.3611472 osd.7 (osd.7) 251 : cluster 3 failed to encode map e306 with expected crc 2023-06-03T20:33:28.586 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:33:28 smithi164 bash[59462]: cluster 1685824407.361232 osd.6 (osd.6) 244 : cluster 3 failed to encode map e306 with expected crc 2023-06-03T20:33:28.779 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:33:28 smithi089 bash[78452]: cluster 1685824407.2433736 mgr.y (mgr.34104) 1733 : cluster 0 pgmap v1798: 73 pgs: 73 active+clean; 582 KiB data, 5.1 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:33:28.779 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:33:28 smithi089 bash[78452]: audit 1685824407.3111403 mon.a (mon.0) 1694 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-06-03T20:33:28.779 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:33:28 smithi089 bash[78452]: cluster 1685824407.3112552 mon.a (mon.0) 1695 : cluster 0 osdmap e306: 8 total, 8 up, 8 in 2023-06-03T20:33:28.780 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:33:28 smithi089 bash[78452]: cluster 1685824407.3608983 osd.5 (osd.5) 254 : cluster 3 failed to encode map e306 with expected crc 2023-06-03T20:33:28.780 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:33:28 smithi089 bash[78452]: cluster 1685824407.3611472 osd.7 (osd.7) 251 : cluster 3 failed to encode map e306 with expected crc 2023-06-03T20:33:28.780 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:33:28 smithi089 bash[78452]: cluster 1685824407.361232 osd.6 (osd.6) 244 : cluster 3 failed to encode map e306 with expected crc 2023-06-03T20:33:28.780 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:33:28 smithi089 bash[75068]: cluster 1685824407.2433736 mgr.y (mgr.34104) 1733 : cluster 0 pgmap v1798: 73 pgs: 73 active+clean; 582 KiB data, 5.1 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:33:28.780 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:33:28 smithi089 bash[75068]: audit 1685824407.3111403 mon.a (mon.0) 1694 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-06-03T20:33:28.780 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:33:28 smithi089 bash[75068]: cluster 1685824407.3112552 mon.a (mon.0) 1695 : cluster 0 osdmap e306: 8 total, 8 up, 8 in 2023-06-03T20:33:28.781 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:33:28 smithi089 bash[75068]: cluster 1685824407.3608983 osd.5 (osd.5) 254 : cluster 3 failed to encode map e306 with expected crc 2023-06-03T20:33:28.781 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:33:28 smithi089 bash[75068]: cluster 1685824407.3611472 osd.7 (osd.7) 251 : cluster 3 failed to encode map e306 with expected crc 2023-06-03T20:33:28.781 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:33:28 smithi089 bash[75068]: cluster 1685824407.361232 osd.6 (osd.6) 244 : cluster 3 failed to encode map e306 with expected crc 2023-06-03T20:33:29.104 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph osd unset nodeep-scrub 2023-06-03T20:33:30.585 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:33:30 smithi164 bash[59462]: cluster 1685824409.244273 mgr.y (mgr.34104) 1734 : cluster 0 pgmap v1800: 73 pgs: 73 active+clean; 582 KiB data, 5.1 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:33:30.779 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:33:30 smithi089 bash[78452]: cluster 1685824409.244273 mgr.y (mgr.34104) 1734 : cluster 0 pgmap v1800: 73 pgs: 73 active+clean; 582 KiB data, 5.1 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:33:30.780 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:33:30 smithi089 bash[75068]: cluster 1685824409.244273 mgr.y (mgr.34104) 1734 : cluster 0 pgmap v1800: 73 pgs: 73 active+clean; 582 KiB data, 5.1 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:33:31.636 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/mon.a/config 2023-06-03T20:33:32.585 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:33:32 smithi164 bash[59462]: cluster 1685824411.2450733 mgr.y (mgr.34104) 1735 : cluster 0 pgmap v1801: 73 pgs: 73 active+clean; 582 KiB data, 5.1 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:33:32.779 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:33:32 smithi089 bash[78452]: cluster 1685824411.2450733 mgr.y (mgr.34104) 1735 : cluster 0 pgmap v1801: 73 pgs: 73 active+clean; 582 KiB data, 5.1 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:33:32.779 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:33:32 smithi089 bash[75068]: cluster 1685824411.2450733 mgr.y (mgr.34104) 1735 : cluster 0 pgmap v1801: 73 pgs: 73 active+clean; 582 KiB data, 5.1 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:33:33.684 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/mon.a/config 2023-06-03T20:33:33.685 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/mon.a/config 2023-06-03T20:33:34.085 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:33:33 smithi164 bash[59462]: audit 1685824412.7850463 mon.a (mon.0) 1696 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:33:34.086 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:33:33 smithi164 bash[59462]: audit 1685824412.795103 mon.a (mon.0) 1697 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:33:34.086 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:33:33 smithi164 bash[59462]: audit 1685824412.892678 mon.a (mon.0) 1698 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:33:34.086 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:33:33 smithi164 bash[59462]: audit 1685824412.9024093 mon.a (mon.0) 1699 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:33:34.086 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:33:33 smithi164 bash[59462]: cluster 1685824413.2457898 mgr.y (mgr.34104) 1736 : cluster 0 pgmap v1802: 73 pgs: 73 active+clean; 582 KiB data, 5.1 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:33:34.086 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:33:33 smithi164 bash[59462]: audit 1685824413.375434 mon.c (mon.2) 694 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-03T20:33:34.086 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:33:33 smithi164 bash[59462]: audit 1685824413.3769417 mon.c (mon.2) 695 : audit 1 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-03T20:33:34.087 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:33:33 smithi164 bash[59462]: audit 1685824413.3847513 mon.a (mon.0) 1700 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:33:34.279 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:33:33 smithi089 bash[78452]: audit 1685824412.7850463 mon.a (mon.0) 1696 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:33:34.279 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:33:33 smithi089 bash[78452]: audit 1685824412.795103 mon.a (mon.0) 1697 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:33:34.279 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:33:33 smithi089 bash[78452]: audit 1685824412.892678 mon.a (mon.0) 1698 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:33:34.280 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:33:33 smithi089 bash[78452]: audit 1685824412.9024093 mon.a (mon.0) 1699 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:33:34.280 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:33:33 smithi089 bash[78452]: cluster 1685824413.2457898 mgr.y (mgr.34104) 1736 : cluster 0 pgmap v1802: 73 pgs: 73 active+clean; 582 KiB data, 5.1 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:33:34.280 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:33:33 smithi089 bash[78452]: audit 1685824413.375434 mon.c (mon.2) 694 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-03T20:33:34.280 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:33:33 smithi089 bash[78452]: audit 1685824413.3769417 mon.c (mon.2) 695 : audit 1 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-03T20:33:34.280 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:33:33 smithi089 bash[78452]: audit 1685824413.3847513 mon.a (mon.0) 1700 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:33:34.280 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:33:33 smithi089 bash[75068]: audit 1685824412.7850463 mon.a (mon.0) 1696 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:33:34.281 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:33:33 smithi089 bash[75068]: audit 1685824412.795103 mon.a (mon.0) 1697 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:33:34.281 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:33:33 smithi089 bash[75068]: audit 1685824412.892678 mon.a (mon.0) 1698 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:33:34.281 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:33:33 smithi089 bash[75068]: audit 1685824412.9024093 mon.a (mon.0) 1699 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:33:34.281 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:33:33 smithi089 bash[75068]: cluster 1685824413.2457898 mgr.y (mgr.34104) 1736 : cluster 0 pgmap v1802: 73 pgs: 73 active+clean; 582 KiB data, 5.1 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:33:34.281 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:33:33 smithi089 bash[75068]: audit 1685824413.375434 mon.c (mon.2) 694 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-03T20:33:34.282 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:33:33 smithi089 bash[75068]: audit 1685824413.3769417 mon.c (mon.2) 695 : audit 1 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-03T20:33:34.282 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:33:33 smithi089 bash[75068]: audit 1685824413.3847513 mon.a (mon.0) 1700 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:33:36.529 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:33:36 smithi089 bash[75068]: cluster 1685824415.2464478 mgr.y (mgr.34104) 1737 : cluster 0 pgmap v1803: 73 pgs: 73 active+clean; 582 KiB data, 5.1 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:33:36.529 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 20:33:36 smithi089 bash[71163]: ::ffff:172.21.15.164 - - [03/Jun/2023:20:33:36] "GET /metrics HTTP/1.1" 200 226974 "" "Prometheus/2.33.4" 2023-06-03T20:33:36.530 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:33:36 smithi089 bash[78452]: cluster 1685824415.2464478 mgr.y (mgr.34104) 1737 : cluster 0 pgmap v1803: 73 pgs: 73 active+clean; 582 KiB data, 5.1 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:33:36.585 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:33:36 smithi164 bash[59462]: cluster 1685824415.2464478 mgr.y (mgr.34104) 1737 : cluster 0 pgmap v1803: 73 pgs: 73 active+clean; 582 KiB data, 5.1 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:33:36.980 INFO:teuthology.orchestra.run.smithi089.stdout:{ 2023-06-03T20:33:36.981 INFO:teuthology.orchestra.run.smithi089.stdout: "size": 20, 2023-06-03T20:33:36.981 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 600, 2023-06-03T20:33:36.981 INFO:teuthology.orchestra.run.smithi089.stdout: "ops": [ 2023-06-03T20:33:36.981 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:33:36.981 INFO:teuthology.orchestra.run.smithi089.stdout: "description": "osd_op(client.36573.0:157125 13.b 13:d198e8c5:::benchmark_data_smithi089_211581_object41600:head [delete] snapc 0=[] ondisk+write+known_if_redirected e257)", 2023-06-03T20:33:36.982 INFO:teuthology.orchestra.run.smithi089.stdout: "initiated_at": "2023-06-03T20:24:39.363088+0000", 2023-06-03T20:33:36.982 INFO:teuthology.orchestra.run.smithi089.stdout: "age": 537.61046062000003, 2023-06-03T20:33:36.982 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.240032098, 2023-06-03T20:33:36.982 INFO:teuthology.orchestra.run.smithi089.stdout: "type_data": { 2023-06-03T20:33:36.982 INFO:teuthology.orchestra.run.smithi089.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-03T20:33:36.982 INFO:teuthology.orchestra.run.smithi089.stdout: "client_info": { 2023-06-03T20:33:36.982 INFO:teuthology.orchestra.run.smithi089.stdout: "client": "client.36573", 2023-06-03T20:33:36.982 INFO:teuthology.orchestra.run.smithi089.stdout: "client_addr": "172.21.15.89:0/616445942", 2023-06-03T20:33:36.983 INFO:teuthology.orchestra.run.smithi089.stdout: "tid": 157125 2023-06-03T20:33:36.983 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:33:36.983 INFO:teuthology.orchestra.run.smithi089.stdout: "events": [ 2023-06-03T20:33:36.983 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:33:36.983 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "initiated", 2023-06-03T20:33:36.983 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:24:39.363088+0000", 2023-06-03T20:33:36.983 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0 2023-06-03T20:33:36.984 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:33:36.984 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:33:36.984 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "throttled", 2023-06-03T20:33:36.984 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:24:39.363088+0000", 2023-06-03T20:33:36.984 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0 2023-06-03T20:33:36.984 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:33:36.984 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:33:36.984 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "header_read", 2023-06-03T20:33:36.985 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:24:39.363087+0000", 2023-06-03T20:33:36.985 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 4294967295.9999981 2023-06-03T20:33:36.985 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:33:36.985 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:33:36.985 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "all_read", 2023-06-03T20:33:36.985 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:24:39.363094+0000", 2023-06-03T20:33:36.985 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 7.391e-06 2023-06-03T20:33:36.986 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:33:36.986 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:33:36.986 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "dispatched", 2023-06-03T20:33:36.986 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:24:39.363105+0000", 2023-06-03T20:33:36.986 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 1.1538e-05 2023-06-03T20:33:36.986 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:33:36.987 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:33:36.987 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "queued_for_pg", 2023-06-03T20:33:36.987 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:24:39.363112+0000", 2023-06-03T20:33:36.987 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 6.7390000000000002e-06 2023-06-03T20:33:36.987 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:33:36.987 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:33:36.987 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "reached_pg", 2023-06-03T20:33:36.988 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:24:39.363162+0000", 2023-06-03T20:33:36.988 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 4.9858999999999998e-05 2023-06-03T20:33:36.988 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:33:36.988 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:33:36.988 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "started", 2023-06-03T20:33:36.988 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:24:39.363269+0000", 2023-06-03T20:33:36.988 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.00010676 2023-06-03T20:33:36.989 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:33:36.989 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:33:36.989 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "waiting for subops from 2,5", 2023-06-03T20:33:36.989 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:24:39.363963+0000", 2023-06-03T20:33:36.989 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.00069423 2023-06-03T20:33:36.989 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:33:36.989 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:33:36.989 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "op_commit", 2023-06-03T20:33:36.990 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:24:39.370231+0000", 2023-06-03T20:33:36.990 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.0062679930000000003 2023-06-03T20:33:36.990 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:33:36.990 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:33:36.990 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:33:36.990 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:24:39.372235+0000", 2023-06-03T20:33:36.990 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.002003767 2023-06-03T20:33:36.991 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:33:36.991 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:33:36.991 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:33:36.991 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:24:39.602939+0000", 2023-06-03T20:33:36.991 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.23070391700000001 2023-06-03T20:33:36.991 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:33:36.991 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:33:36.991 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "commit_sent", 2023-06-03T20:33:36.992 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:24:39.603013+0000", 2023-06-03T20:33:36.992 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 7.3998000000000005e-05 2023-06-03T20:33:36.992 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:33:36.992 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:33:36.992 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "done", 2023-06-03T20:33:36.992 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:24:39.603121+0000", 2023-06-03T20:33:36.992 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.000107837 2023-06-03T20:33:36.992 INFO:teuthology.orchestra.run.smithi089.stdout: } 2023-06-03T20:33:36.993 INFO:teuthology.orchestra.run.smithi089.stdout: ] 2023-06-03T20:33:36.993 INFO:teuthology.orchestra.run.smithi089.stdout: } 2023-06-03T20:33:36.993 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:33:36.993 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:33:36.993 INFO:teuthology.orchestra.run.smithi089.stdout: "description": "osd_op(client.36573.0:179036 13.b 13:de955b9e:::benchmark_data_smithi089_211581_object63511:head [delete] snapc 0=[] ondisk+write+known_if_redirected e264)", 2023-06-03T20:33:36.994 INFO:teuthology.orchestra.run.smithi089.stdout: "initiated_at": "2023-06-03T20:25:19.550873+0000", 2023-06-03T20:33:36.994 INFO:teuthology.orchestra.run.smithi089.stdout: "age": 497.42267630600003, 2023-06-03T20:33:36.994 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.29629856500000001, 2023-06-03T20:33:36.994 INFO:teuthology.orchestra.run.smithi089.stdout: "type_data": { 2023-06-03T20:33:36.994 INFO:teuthology.orchestra.run.smithi089.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-03T20:33:36.994 INFO:teuthology.orchestra.run.smithi089.stdout: "client_info": { 2023-06-03T20:33:36.994 INFO:teuthology.orchestra.run.smithi089.stdout: "client": "client.36573", 2023-06-03T20:33:36.994 INFO:teuthology.orchestra.run.smithi089.stdout: "client_addr": "172.21.15.89:0/616445942", 2023-06-03T20:33:36.995 INFO:teuthology.orchestra.run.smithi089.stdout: "tid": 179036 2023-06-03T20:33:36.995 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:33:36.995 INFO:teuthology.orchestra.run.smithi089.stdout: "events": [ 2023-06-03T20:33:36.995 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:33:36.995 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "initiated", 2023-06-03T20:33:36.995 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:25:19.550873+0000", 2023-06-03T20:33:36.995 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0 2023-06-03T20:33:36.996 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:33:36.996 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:33:36.996 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "throttled", 2023-06-03T20:33:36.996 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:25:19.550873+0000", 2023-06-03T20:33:36.996 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0 2023-06-03T20:33:36.996 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:33:36.996 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:33:36.996 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "header_read", 2023-06-03T20:33:36.997 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:25:19.550872+0000", 2023-06-03T20:33:36.997 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 4294967295.999999 2023-06-03T20:33:36.997 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:33:36.997 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:33:36.997 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "all_read", 2023-06-03T20:33:36.997 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:25:19.550877+0000", 2023-06-03T20:33:36.997 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 4.6789999999999996e-06 2023-06-03T20:33:36.998 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:33:36.998 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:33:36.998 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "dispatched", 2023-06-03T20:33:36.998 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:25:19.550886+0000", 2023-06-03T20:33:36.998 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 9.3060000000000003e-06 2023-06-03T20:33:36.998 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:33:36.999 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:33:36.999 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "queued_for_pg", 2023-06-03T20:33:36.999 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:25:19.550892+0000", 2023-06-03T20:33:36.999 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 6.0959999999999997e-06 2023-06-03T20:33:36.999 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:33:36.999 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:33:36.999 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "reached_pg", 2023-06-03T20:33:36.999 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:25:19.550930+0000", 2023-06-03T20:33:37.000 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 3.7557e-05 2023-06-03T20:33:37.000 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:33:37.000 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:33:37.000 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "started", 2023-06-03T20:33:37.000 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:25:19.551015+0000", 2023-06-03T20:33:37.000 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 8.53e-05 2023-06-03T20:33:37.000 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:33:37.001 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:33:37.001 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "waiting for subops from 2,5", 2023-06-03T20:33:37.001 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:25:19.551090+0000", 2023-06-03T20:33:37.001 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 7.5228999999999994e-05 2023-06-03T20:33:37.001 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:33:37.001 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:33:37.001 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "op_commit", 2023-06-03T20:33:37.001 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:25:19.554513+0000", 2023-06-03T20:33:37.002 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.003422965 2023-06-03T20:33:37.002 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:33:37.002 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:33:37.002 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:33:37.002 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:25:19.556734+0000", 2023-06-03T20:33:37.002 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.0022206320000000002 2023-06-03T20:33:37.002 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:33:37.002 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:33:37.003 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:33:37.003 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:25:19.847056+0000", 2023-06-03T20:33:37.003 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.29032211499999999 2023-06-03T20:33:37.003 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:33:37.003 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:33:37.003 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "commit_sent", 2023-06-03T20:33:37.003 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:25:19.847129+0000", 2023-06-03T20:33:37.003 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 7.3175000000000003e-05 2023-06-03T20:33:37.004 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:33:37.004 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:33:37.004 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "done", 2023-06-03T20:33:37.004 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:25:19.847171+0000", 2023-06-03T20:33:37.004 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 4.2301e-05 2023-06-03T20:33:37.004 INFO:teuthology.orchestra.run.smithi089.stdout: } 2023-06-03T20:33:37.004 INFO:teuthology.orchestra.run.smithi089.stdout: ] 2023-06-03T20:33:37.005 INFO:teuthology.orchestra.run.smithi089.stdout: } 2023-06-03T20:33:37.005 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:33:37.005 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:33:37.005 INFO:teuthology.orchestra.run.smithi089.stdout: "description": "osd_op(client.36573.0:181633 13.6 13:610eb2b6:::benchmark_data_smithi089_211581_object66108:head [delete] snapc 0=[] ondisk+write+known_if_redirected e264)", 2023-06-03T20:33:37.005 INFO:teuthology.orchestra.run.smithi089.stdout: "initiated_at": "2023-06-03T20:25:23.848734+0000", 2023-06-03T20:33:37.005 INFO:teuthology.orchestra.run.smithi089.stdout: "age": 493.12481542799998, 2023-06-03T20:33:37.005 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.27333703799999998, 2023-06-03T20:33:37.005 INFO:teuthology.orchestra.run.smithi089.stdout: "type_data": { 2023-06-03T20:33:37.006 INFO:teuthology.orchestra.run.smithi089.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-03T20:33:37.006 INFO:teuthology.orchestra.run.smithi089.stdout: "client_info": { 2023-06-03T20:33:37.006 INFO:teuthology.orchestra.run.smithi089.stdout: "client": "client.36573", 2023-06-03T20:33:37.006 INFO:teuthology.orchestra.run.smithi089.stdout: "client_addr": "172.21.15.89:0/616445942", 2023-06-03T20:33:37.006 INFO:teuthology.orchestra.run.smithi089.stdout: "tid": 181633 2023-06-03T20:33:37.006 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:33:37.006 INFO:teuthology.orchestra.run.smithi089.stdout: "events": [ 2023-06-03T20:33:37.006 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:33:37.007 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "initiated", 2023-06-03T20:33:37.007 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:25:23.848734+0000", 2023-06-03T20:33:37.007 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0 2023-06-03T20:33:37.007 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:33:37.007 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:33:37.007 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "throttled", 2023-06-03T20:33:37.007 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:25:23.848734+0000", 2023-06-03T20:33:37.008 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0 2023-06-03T20:33:37.008 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:33:37.008 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:33:37.008 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "header_read", 2023-06-03T20:33:37.008 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:25:23.848732+0000", 2023-06-03T20:33:37.008 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 4294967295.9999981 2023-06-03T20:33:37.008 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:33:37.008 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:33:37.009 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "all_read", 2023-06-03T20:33:37.009 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:25:23.848739+0000", 2023-06-03T20:33:37.009 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 7.6520000000000006e-06 2023-06-03T20:33:37.009 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:33:37.009 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:33:37.009 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "dispatched", 2023-06-03T20:33:37.009 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:25:23.848752+0000", 2023-06-03T20:33:37.010 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 1.2733e-05 2023-06-03T20:33:37.010 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:33:37.010 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:33:37.010 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "queued_for_pg", 2023-06-03T20:33:37.010 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:25:23.848760+0000", 2023-06-03T20:33:37.010 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 7.8390000000000007e-06 2023-06-03T20:33:37.010 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:33:37.010 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:33:37.011 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "reached_pg", 2023-06-03T20:33:37.011 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:25:23.848820+0000", 2023-06-03T20:33:37.011 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 5.9880999999999998e-05 2023-06-03T20:33:37.011 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:33:37.011 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:33:37.011 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "started", 2023-06-03T20:33:37.011 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:25:23.848955+0000", 2023-06-03T20:33:37.012 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.00013507800000000001 2023-06-03T20:33:37.012 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:33:37.012 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:33:37.012 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "waiting for subops from 4,5", 2023-06-03T20:33:37.012 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:25:23.849438+0000", 2023-06-03T20:33:37.012 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.00048347599999999999 2023-06-03T20:33:37.012 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:33:37.013 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:33:37.013 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "op_commit", 2023-06-03T20:33:37.013 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:25:23.853472+0000", 2023-06-03T20:33:37.013 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.0040334029999999996 2023-06-03T20:33:37.013 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:33:37.013 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:33:37.013 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:33:37.013 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:25:23.867575+0000", 2023-06-03T20:33:37.014 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.014103667 2023-06-03T20:33:37.014 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:33:37.014 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:33:37.014 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:33:37.014 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:25:24.121918+0000", 2023-06-03T20:33:37.014 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.25434207599999997 2023-06-03T20:33:37.014 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:33:37.014 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:33:37.015 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "commit_sent", 2023-06-03T20:33:37.015 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:25:24.121967+0000", 2023-06-03T20:33:37.015 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 4.9369000000000001e-05 2023-06-03T20:33:37.015 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:33:37.015 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:33:37.015 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "done", 2023-06-03T20:33:37.015 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:25:24.122071+0000", 2023-06-03T20:33:37.016 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.00010382000000000001 2023-06-03T20:33:37.016 INFO:teuthology.orchestra.run.smithi089.stdout: } 2023-06-03T20:33:37.016 INFO:teuthology.orchestra.run.smithi089.stdout: ] 2023-06-03T20:33:37.016 INFO:teuthology.orchestra.run.smithi089.stdout: } 2023-06-03T20:33:37.016 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:33:37.016 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:33:37.016 INFO:teuthology.orchestra.run.smithi089.stdout: "description": "osd_op(client.36573.0:185432 13.b 13:d044e2d5:::benchmark_data_smithi089_211581_object69907:head [delete] snapc 0=[] ondisk+write+known_if_redirected e264)", 2023-06-03T20:33:37.017 INFO:teuthology.orchestra.run.smithi089.stdout: "initiated_at": "2023-06-03T20:25:30.526392+0000", 2023-06-03T20:33:37.017 INFO:teuthology.orchestra.run.smithi089.stdout: "age": 486.447157146, 2023-06-03T20:33:37.017 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.25877160300000002, 2023-06-03T20:33:37.017 INFO:teuthology.orchestra.run.smithi089.stdout: "type_data": { 2023-06-03T20:33:37.017 INFO:teuthology.orchestra.run.smithi089.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-03T20:33:37.017 INFO:teuthology.orchestra.run.smithi089.stdout: "client_info": { 2023-06-03T20:33:37.017 INFO:teuthology.orchestra.run.smithi089.stdout: "client": "client.36573", 2023-06-03T20:33:37.017 INFO:teuthology.orchestra.run.smithi089.stdout: "client_addr": "172.21.15.89:0/616445942", 2023-06-03T20:33:37.018 INFO:teuthology.orchestra.run.smithi089.stdout: "tid": 185432 2023-06-03T20:33:37.018 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:33:37.018 INFO:teuthology.orchestra.run.smithi089.stdout: "events": [ 2023-06-03T20:33:37.018 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:33:37.018 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "initiated", 2023-06-03T20:33:37.018 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:25:30.526392+0000", 2023-06-03T20:33:37.018 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0 2023-06-03T20:33:37.019 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:33:37.019 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:33:37.019 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "throttled", 2023-06-03T20:33:37.019 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:25:30.526392+0000", 2023-06-03T20:33:37.019 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0 2023-06-03T20:33:37.019 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:33:37.019 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:33:37.019 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "header_read", 2023-06-03T20:33:37.020 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:25:30.526391+0000", 2023-06-03T20:33:37.020 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 4294967295.9999986 2023-06-03T20:33:37.020 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:33:37.020 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:33:37.020 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "all_read", 2023-06-03T20:33:37.020 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:25:30.526397+0000", 2023-06-03T20:33:37.020 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 6.1290000000000001e-06 2023-06-03T20:33:37.021 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:33:37.021 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:33:37.021 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "dispatched", 2023-06-03T20:33:37.021 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:25:30.526409+0000", 2023-06-03T20:33:37.021 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 1.2086000000000001e-05 2023-06-03T20:33:37.021 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:33:37.021 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:33:37.021 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "queued_for_pg", 2023-06-03T20:33:37.022 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:25:30.526417+0000", 2023-06-03T20:33:37.022 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 7.8350000000000004e-06 2023-06-03T20:33:37.022 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:33:37.022 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:33:37.022 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "reached_pg", 2023-06-03T20:33:37.022 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:25:30.526480+0000", 2023-06-03T20:33:37.022 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 6.2848000000000005e-05 2023-06-03T20:33:37.022 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:33:37.023 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:33:37.023 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "started", 2023-06-03T20:33:37.023 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:25:30.526709+0000", 2023-06-03T20:33:37.023 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.00022962300000000001 2023-06-03T20:33:37.023 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:33:37.023 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:33:37.023 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "waiting for subops from 2,5", 2023-06-03T20:33:37.024 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:25:30.526961+0000", 2023-06-03T20:33:37.024 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.00025146899999999999 2023-06-03T20:33:37.024 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:33:37.024 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:33:37.024 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "op_commit", 2023-06-03T20:33:37.024 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:25:30.531276+0000", 2023-06-03T20:33:37.024 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.0043157129999999997 2023-06-03T20:33:37.024 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:33:37.025 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:33:37.025 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:33:37.025 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:25:30.535783+0000", 2023-06-03T20:33:37.025 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.0045065929999999997 2023-06-03T20:33:37.025 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:33:37.025 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:33:37.025 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:33:37.025 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:25:30.785056+0000", 2023-06-03T20:33:37.026 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.249272673 2023-06-03T20:33:37.026 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:33:37.026 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:33:37.026 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "commit_sent", 2023-06-03T20:33:37.026 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:25:30.785113+0000", 2023-06-03T20:33:37.026 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 5.6830000000000003e-05 2023-06-03T20:33:37.026 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:33:37.027 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:33:37.027 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "done", 2023-06-03T20:33:37.027 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:25:30.785164+0000", 2023-06-03T20:33:37.027 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 5.1011999999999999e-05 2023-06-03T20:33:37.027 INFO:teuthology.orchestra.run.smithi089.stdout: } 2023-06-03T20:33:37.027 INFO:teuthology.orchestra.run.smithi089.stdout: ] 2023-06-03T20:33:37.027 INFO:teuthology.orchestra.run.smithi089.stdout: } 2023-06-03T20:33:37.027 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:33:37.028 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:33:37.028 INFO:teuthology.orchestra.run.smithi089.stdout: "description": "osd_op(client.36573.0:185434 13.b 13:de403742:::benchmark_data_smithi089_211581_object69909:head [delete] snapc 0=[] ondisk+write+known_if_redirected e264)", 2023-06-03T20:33:37.028 INFO:teuthology.orchestra.run.smithi089.stdout: "initiated_at": "2023-06-03T20:25:30.531571+0000", 2023-06-03T20:33:37.028 INFO:teuthology.orchestra.run.smithi089.stdout: "age": 486.441977839, 2023-06-03T20:33:37.028 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.25371148900000001, 2023-06-03T20:33:37.028 INFO:teuthology.orchestra.run.smithi089.stdout: "type_data": { 2023-06-03T20:33:37.028 INFO:teuthology.orchestra.run.smithi089.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-03T20:33:37.028 INFO:teuthology.orchestra.run.smithi089.stdout: "client_info": { 2023-06-03T20:33:37.029 INFO:teuthology.orchestra.run.smithi089.stdout: "client": "client.36573", 2023-06-03T20:33:37.029 INFO:teuthology.orchestra.run.smithi089.stdout: "client_addr": "172.21.15.89:0/616445942", 2023-06-03T20:33:37.029 INFO:teuthology.orchestra.run.smithi089.stdout: "tid": 185434 2023-06-03T20:33:37.029 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:33:37.029 INFO:teuthology.orchestra.run.smithi089.stdout: "events": [ 2023-06-03T20:33:37.029 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:33:37.029 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "initiated", 2023-06-03T20:33:37.030 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:25:30.531571+0000", 2023-06-03T20:33:37.030 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0 2023-06-03T20:33:37.030 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:33:37.030 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:33:37.030 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "throttled", 2023-06-03T20:33:37.030 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:25:30.531571+0000", 2023-06-03T20:33:37.030 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0 2023-06-03T20:33:37.030 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:33:37.031 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:33:37.031 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "header_read", 2023-06-03T20:33:37.031 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:25:30.531570+0000", 2023-06-03T20:33:37.031 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 4294967295.9999986 2023-06-03T20:33:37.031 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:33:37.031 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:33:37.031 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "all_read", 2023-06-03T20:33:37.032 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:25:30.531577+0000", 2023-06-03T20:33:37.032 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 7.1280000000000002e-06 2023-06-03T20:33:37.032 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:33:37.032 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:33:37.032 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "dispatched", 2023-06-03T20:33:37.032 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:25:30.531590+0000", 2023-06-03T20:33:37.032 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 1.3570000000000001e-05 2023-06-03T20:33:37.032 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:33:37.033 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:33:37.033 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "queued_for_pg", 2023-06-03T20:33:37.033 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:25:30.531599+0000", 2023-06-03T20:33:37.033 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 8.9479999999999997e-06 2023-06-03T20:33:37.033 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:33:37.033 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:33:37.033 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "reached_pg", 2023-06-03T20:33:37.034 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:25:30.531643+0000", 2023-06-03T20:33:37.034 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 4.3850999999999997e-05 2023-06-03T20:33:37.034 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:33:37.034 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:33:37.034 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "started", 2023-06-03T20:33:37.034 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:25:30.531756+0000", 2023-06-03T20:33:37.034 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.00011237499999999999 2023-06-03T20:33:37.034 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:33:37.035 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:33:37.035 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "waiting for subops from 2,5", 2023-06-03T20:33:37.035 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:25:30.531877+0000", 2023-06-03T20:33:37.035 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.00012150900000000001 2023-06-03T20:33:37.035 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:33:37.035 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:33:37.035 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:33:37.035 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:25:30.535811+0000", 2023-06-03T20:33:37.036 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.0039342480000000004 2023-06-03T20:33:37.036 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:33:37.036 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:33:37.036 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "op_commit", 2023-06-03T20:33:37.036 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:25:30.535888+0000", 2023-06-03T20:33:37.036 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 7.6997999999999997e-05 2023-06-03T20:33:37.036 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:33:37.037 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:33:37.037 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:33:37.037 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:25:30.785206+0000", 2023-06-03T20:33:37.037 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.24931789700000001 2023-06-03T20:33:37.037 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:33:37.037 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:33:37.037 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "commit_sent", 2023-06-03T20:33:37.037 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:25:30.785246+0000", 2023-06-03T20:33:37.038 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 4.0105999999999999e-05 2023-06-03T20:33:37.038 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:33:37.038 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:33:37.038 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "done", 2023-06-03T20:33:37.038 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:25:30.785283+0000", 2023-06-03T20:33:37.038 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 3.6372999999999999e-05 2023-06-03T20:33:37.038 INFO:teuthology.orchestra.run.smithi089.stdout: } 2023-06-03T20:33:37.038 INFO:teuthology.orchestra.run.smithi089.stdout: ] 2023-06-03T20:33:37.039 INFO:teuthology.orchestra.run.smithi089.stdout: } 2023-06-03T20:33:37.039 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:33:37.039 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:33:37.039 INFO:teuthology.orchestra.run.smithi089.stdout: "description": "osd_op(client.36573.0:214432 13.b 13:d05a50fb:::benchmark_data_smithi089_211581_object98907:head [delete] snapc 0=[] ondisk+write+known_if_redirected e268)", 2023-06-03T20:33:37.039 INFO:teuthology.orchestra.run.smithi089.stdout: "initiated_at": "2023-06-03T20:26:15.412347+0000", 2023-06-03T20:33:37.039 INFO:teuthology.orchestra.run.smithi089.stdout: "age": 441.56120206600002, 2023-06-03T20:33:37.039 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.212433231, 2023-06-03T20:33:37.039 INFO:teuthology.orchestra.run.smithi089.stdout: "type_data": { 2023-06-03T20:33:37.040 INFO:teuthology.orchestra.run.smithi089.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-03T20:33:37.040 INFO:teuthology.orchestra.run.smithi089.stdout: "client_info": { 2023-06-03T20:33:37.040 INFO:teuthology.orchestra.run.smithi089.stdout: "client": "client.36573", 2023-06-03T20:33:37.040 INFO:teuthology.orchestra.run.smithi089.stdout: "client_addr": "172.21.15.89:0/616445942", 2023-06-03T20:33:37.040 INFO:teuthology.orchestra.run.smithi089.stdout: "tid": 214432 2023-06-03T20:33:37.040 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:33:37.040 INFO:teuthology.orchestra.run.smithi089.stdout: "events": [ 2023-06-03T20:33:37.041 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:33:37.041 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "initiated", 2023-06-03T20:33:37.041 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:26:15.412347+0000", 2023-06-03T20:33:37.041 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0 2023-06-03T20:33:37.041 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:33:37.041 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:33:37.041 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "throttled", 2023-06-03T20:33:37.041 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:26:15.412347+0000", 2023-06-03T20:33:37.042 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0 2023-06-03T20:33:37.042 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:33:37.042 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:33:37.042 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "header_read", 2023-06-03T20:33:37.042 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:26:15.412346+0000", 2023-06-03T20:33:37.042 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 4294967295.9999995 2023-06-03T20:33:37.042 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:33:37.042 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:33:37.043 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "all_read", 2023-06-03T20:33:37.043 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:26:15.412350+0000", 2023-06-03T20:33:37.043 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 3.7479999999999999e-06 2023-06-03T20:33:37.043 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:33:37.043 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:33:37.043 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "dispatched", 2023-06-03T20:33:37.043 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:26:15.412358+0000", 2023-06-03T20:33:37.043 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 7.8650000000000001e-06 2023-06-03T20:33:37.044 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:33:37.044 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:33:37.044 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "queued_for_pg", 2023-06-03T20:33:37.044 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:26:15.412364+0000", 2023-06-03T20:33:37.044 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 5.541e-06 2023-06-03T20:33:37.044 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:33:37.044 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:33:37.045 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "reached_pg", 2023-06-03T20:33:37.045 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:26:15.412405+0000", 2023-06-03T20:33:37.045 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 4.1185e-05 2023-06-03T20:33:37.045 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:33:37.045 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:33:37.045 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "started", 2023-06-03T20:33:37.045 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:26:15.412492+0000", 2023-06-03T20:33:37.045 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 8.7509000000000005e-05 2023-06-03T20:33:37.046 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:33:37.046 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:33:37.046 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "waiting for subops from 2,5", 2023-06-03T20:33:37.046 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:26:15.412566+0000", 2023-06-03T20:33:37.046 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 7.3998999999999994e-05 2023-06-03T20:33:37.046 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:33:37.046 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:33:37.046 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "op_commit", 2023-06-03T20:33:37.047 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:26:15.416366+0000", 2023-06-03T20:33:37.047 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.0038000239999999999 2023-06-03T20:33:37.047 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:33:37.047 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:33:37.047 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:33:37.047 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:26:15.417574+0000", 2023-06-03T20:33:37.047 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.0012073030000000001 2023-06-03T20:33:37.047 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:33:37.048 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:33:37.048 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:33:37.048 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:26:15.624633+0000", 2023-06-03T20:33:37.048 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.20705957899999999 2023-06-03T20:33:37.048 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:33:37.048 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:33:37.048 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "commit_sent", 2023-06-03T20:33:37.049 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:26:15.624720+0000", 2023-06-03T20:33:37.049 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 8.6680000000000004e-05 2023-06-03T20:33:37.049 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:33:37.049 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:33:37.049 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "done", 2023-06-03T20:33:37.049 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:26:15.624780+0000", 2023-06-03T20:33:37.049 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 6.0406999999999999e-05 2023-06-03T20:33:37.049 INFO:teuthology.orchestra.run.smithi089.stdout: } 2023-06-03T20:33:37.050 INFO:teuthology.orchestra.run.smithi089.stdout: ] 2023-06-03T20:33:37.050 INFO:teuthology.orchestra.run.smithi089.stdout: } 2023-06-03T20:33:37.050 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:33:37.050 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:33:37.050 INFO:teuthology.orchestra.run.smithi089.stdout: "description": "osd_op(client.46739.0:40709 14.1 14:895ae406:::benchmark_data_smithi089_237281_object40708:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e281)", 2023-06-03T20:33:37.050 INFO:teuthology.orchestra.run.smithi089.stdout: "initiated_at": "2023-06-03T20:28:10.249167+0000", 2023-06-03T20:33:37.050 INFO:teuthology.orchestra.run.smithi089.stdout: "age": 326.72438210799999, 2023-06-03T20:33:37.051 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.249608987, 2023-06-03T20:33:37.051 INFO:teuthology.orchestra.run.smithi089.stdout: "type_data": { 2023-06-03T20:33:37.051 INFO:teuthology.orchestra.run.smithi089.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-03T20:33:37.051 INFO:teuthology.orchestra.run.smithi089.stdout: "client_info": { 2023-06-03T20:33:37.051 INFO:teuthology.orchestra.run.smithi089.stdout: "client": "client.46739", 2023-06-03T20:33:37.051 INFO:teuthology.orchestra.run.smithi089.stdout: "client_addr": "172.21.15.89:0/2479974613", 2023-06-03T20:33:37.051 INFO:teuthology.orchestra.run.smithi089.stdout: "tid": 40709 2023-06-03T20:33:37.051 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:33:37.052 INFO:teuthology.orchestra.run.smithi089.stdout: "events": [ 2023-06-03T20:33:37.052 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:33:37.052 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "initiated", 2023-06-03T20:33:37.052 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:28:10.249167+0000", 2023-06-03T20:33:37.052 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0 2023-06-03T20:33:37.052 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:33:37.052 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:33:37.053 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "throttled", 2023-06-03T20:33:37.053 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:28:10.249167+0000", 2023-06-03T20:33:37.053 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0 2023-06-03T20:33:37.053 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:33:37.053 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:33:37.053 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "header_read", 2023-06-03T20:33:37.053 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:28:10.249165+0000", 2023-06-03T20:33:37.053 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 4294967295.9999986 2023-06-03T20:33:37.054 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:33:37.054 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:33:37.054 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "all_read", 2023-06-03T20:33:37.054 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:28:10.249296+0000", 2023-06-03T20:33:37.054 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.00013057300000000001 2023-06-03T20:33:37.054 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:33:37.054 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:33:37.054 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "dispatched", 2023-06-03T20:33:37.055 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:28:10.249309+0000", 2023-06-03T20:33:37.055 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 1.2789000000000001e-05 2023-06-03T20:33:37.055 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:33:37.055 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:33:37.055 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "queued_for_pg", 2023-06-03T20:33:37.055 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:28:10.249317+0000", 2023-06-03T20:33:37.055 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 8.5949999999999999e-06 2023-06-03T20:33:37.056 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:33:37.056 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:33:37.056 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "reached_pg", 2023-06-03T20:33:37.056 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:28:10.249362+0000", 2023-06-03T20:33:37.056 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 4.4168999999999997e-05 2023-06-03T20:33:37.056 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:33:37.056 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:33:37.056 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "started", 2023-06-03T20:33:37.057 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:28:10.249518+0000", 2023-06-03T20:33:37.057 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.000156186 2023-06-03T20:33:37.057 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:33:37.057 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:33:37.057 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "waiting for subops from 1,2", 2023-06-03T20:33:37.057 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:28:10.249656+0000", 2023-06-03T20:33:37.057 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.00013856599999999999 2023-06-03T20:33:37.058 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:33:37.058 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:33:37.058 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "op_commit", 2023-06-03T20:33:37.058 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:28:10.255642+0000", 2023-06-03T20:33:37.058 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.0059859370000000002 2023-06-03T20:33:37.058 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:33:37.058 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:33:37.059 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:33:37.059 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:28:10.256796+0000", 2023-06-03T20:33:37.059 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.001153888 2023-06-03T20:33:37.059 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:33:37.059 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:33:37.059 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:33:37.059 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:28:10.498292+0000", 2023-06-03T20:33:37.059 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.241495723 2023-06-03T20:33:37.060 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:33:37.060 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:33:37.060 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "commit_sent", 2023-06-03T20:33:37.060 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:28:10.498411+0000", 2023-06-03T20:33:37.060 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.00011948 2023-06-03T20:33:37.060 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:33:37.060 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:33:37.060 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "done", 2023-06-03T20:33:37.061 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:28:10.498776+0000", 2023-06-03T20:33:37.061 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.00036460000000000003 2023-06-03T20:33:37.061 INFO:teuthology.orchestra.run.smithi089.stdout: } 2023-06-03T20:33:37.061 INFO:teuthology.orchestra.run.smithi089.stdout: ] 2023-06-03T20:33:37.061 INFO:teuthology.orchestra.run.smithi089.stdout: } 2023-06-03T20:33:37.061 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:33:37.061 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:33:37.062 INFO:teuthology.orchestra.run.smithi089.stdout: "description": "osd_op(client.46739.0:40710 14.1 14:82cff33e:::benchmark_data_smithi089_237281_object40709:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e281)", 2023-06-03T20:33:37.062 INFO:teuthology.orchestra.run.smithi089.stdout: "initiated_at": "2023-06-03T20:28:10.249330+0000", 2023-06-03T20:33:37.062 INFO:teuthology.orchestra.run.smithi089.stdout: "age": 326.72421940300001, 2023-06-03T20:33:37.062 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.249714143, 2023-06-03T20:33:37.062 INFO:teuthology.orchestra.run.smithi089.stdout: "type_data": { 2023-06-03T20:33:37.062 INFO:teuthology.orchestra.run.smithi089.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-03T20:33:37.062 INFO:teuthology.orchestra.run.smithi089.stdout: "client_info": { 2023-06-03T20:33:37.062 INFO:teuthology.orchestra.run.smithi089.stdout: "client": "client.46739", 2023-06-03T20:33:37.063 INFO:teuthology.orchestra.run.smithi089.stdout: "client_addr": "172.21.15.89:0/2479974613", 2023-06-03T20:33:37.063 INFO:teuthology.orchestra.run.smithi089.stdout: "tid": 40710 2023-06-03T20:33:37.063 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:33:37.063 INFO:teuthology.orchestra.run.smithi089.stdout: "events": [ 2023-06-03T20:33:37.063 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:33:37.063 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "initiated", 2023-06-03T20:33:37.063 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:28:10.249330+0000", 2023-06-03T20:33:37.063 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0 2023-06-03T20:33:37.064 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:33:37.064 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:33:37.064 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "throttled", 2023-06-03T20:33:37.064 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:28:10.249330+0000", 2023-06-03T20:33:37.064 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0 2023-06-03T20:33:37.064 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:33:37.064 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:33:37.065 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "header_read", 2023-06-03T20:33:37.065 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:28:10.249329+0000", 2023-06-03T20:33:37.065 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 4294967295.9999995 2023-06-03T20:33:37.065 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:33:37.065 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:33:37.065 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "all_read", 2023-06-03T20:33:37.065 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:28:10.249351+0000", 2023-06-03T20:33:37.065 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 2.232e-05 2023-06-03T20:33:37.066 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:33:37.066 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:33:37.066 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "dispatched", 2023-06-03T20:33:37.066 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:28:10.249359+0000", 2023-06-03T20:33:37.066 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 7.2289999999999997e-06 2023-06-03T20:33:37.066 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:33:37.066 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:33:37.066 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "queued_for_pg", 2023-06-03T20:33:37.067 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:28:10.249364+0000", 2023-06-03T20:33:37.067 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 5.6509999999999998e-06 2023-06-03T20:33:37.067 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:33:37.067 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:33:37.067 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "reached_pg", 2023-06-03T20:33:37.067 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:28:10.251271+0000", 2023-06-03T20:33:37.067 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.001906385 2023-06-03T20:33:37.068 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:33:37.068 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:33:37.068 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "started", 2023-06-03T20:33:37.068 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:28:10.251423+0000", 2023-06-03T20:33:37.068 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.000152814 2023-06-03T20:33:37.068 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:33:37.068 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:33:37.068 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "waiting for subops from 1,2", 2023-06-03T20:33:37.069 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:28:10.251574+0000", 2023-06-03T20:33:37.069 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.000150185 2023-06-03T20:33:37.069 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:33:37.069 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:33:37.069 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "op_commit", 2023-06-03T20:33:37.069 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:28:10.255647+0000", 2023-06-03T20:33:37.069 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.004073686 2023-06-03T20:33:37.069 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:33:37.070 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:33:37.070 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:33:37.070 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:28:10.256826+0000", 2023-06-03T20:33:37.070 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.0011791219999999999 2023-06-03T20:33:37.070 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:33:37.070 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:33:37.070 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:33:37.070 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:28:10.498869+0000", 2023-06-03T20:33:37.071 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.24204261799999999 2023-06-03T20:33:37.071 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:33:37.071 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:33:37.071 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "commit_sent", 2023-06-03T20:33:37.071 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:28:10.498971+0000", 2023-06-03T20:33:37.071 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.000101803 2023-06-03T20:33:37.071 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:33:37.072 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:33:37.072 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "done", 2023-06-03T20:33:37.072 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:28:10.499044+0000", 2023-06-03T20:33:37.072 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 7.2965000000000004e-05 2023-06-03T20:33:37.072 INFO:teuthology.orchestra.run.smithi089.stdout: } 2023-06-03T20:33:37.072 INFO:teuthology.orchestra.run.smithi089.stdout: ] 2023-06-03T20:33:37.072 INFO:teuthology.orchestra.run.smithi089.stdout: } 2023-06-03T20:33:37.072 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:33:37.073 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:33:37.073 INFO:teuthology.orchestra.run.smithi089.stdout: "description": "osd_op(client.46739.0:40711 14.1 14:8afd6546:::benchmark_data_smithi089_237281_object40710:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e281)", 2023-06-03T20:33:37.073 INFO:teuthology.orchestra.run.smithi089.stdout: "initiated_at": "2023-06-03T20:28:10.249369+0000", 2023-06-03T20:33:37.073 INFO:teuthology.orchestra.run.smithi089.stdout: "age": 326.724180031, 2023-06-03T20:33:37.073 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.25268299, 2023-06-03T20:33:37.073 INFO:teuthology.orchestra.run.smithi089.stdout: "type_data": { 2023-06-03T20:33:37.073 INFO:teuthology.orchestra.run.smithi089.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-03T20:33:37.073 INFO:teuthology.orchestra.run.smithi089.stdout: "client_info": { 2023-06-03T20:33:37.074 INFO:teuthology.orchestra.run.smithi089.stdout: "client": "client.46739", 2023-06-03T20:33:37.074 INFO:teuthology.orchestra.run.smithi089.stdout: "client_addr": "172.21.15.89:0/2479974613", 2023-06-03T20:33:37.074 INFO:teuthology.orchestra.run.smithi089.stdout: "tid": 40711 2023-06-03T20:33:37.074 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:33:37.074 INFO:teuthology.orchestra.run.smithi089.stdout: "events": [ 2023-06-03T20:33:37.074 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:33:37.074 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "initiated", 2023-06-03T20:33:37.074 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:28:10.249369+0000", 2023-06-03T20:33:37.075 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0 2023-06-03T20:33:37.075 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:33:37.075 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:33:37.075 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "throttled", 2023-06-03T20:33:37.075 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:28:10.249369+0000", 2023-06-03T20:33:37.075 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0 2023-06-03T20:33:37.075 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:33:37.076 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:33:37.076 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "header_read", 2023-06-03T20:33:37.076 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:28:10.249369+0000", 2023-06-03T20:33:37.076 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 4294967295.9999995 2023-06-03T20:33:37.076 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:33:37.076 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:33:37.076 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "all_read", 2023-06-03T20:33:37.076 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:28:10.249387+0000", 2023-06-03T20:33:37.077 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 1.8590999999999999e-05 2023-06-03T20:33:37.077 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:33:37.077 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:33:37.077 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "dispatched", 2023-06-03T20:33:37.077 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:28:10.249393+0000", 2023-06-03T20:33:37.077 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 6.1199999999999999e-06 2023-06-03T20:33:37.077 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:33:37.077 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:33:37.078 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "queued_for_pg", 2023-06-03T20:33:37.078 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:28:10.249398+0000", 2023-06-03T20:33:37.078 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 4.8180000000000003e-06 2023-06-03T20:33:37.078 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:33:37.078 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:33:37.078 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "reached_pg", 2023-06-03T20:33:37.078 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:28:10.253271+0000", 2023-06-03T20:33:37.078 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.003872768 2023-06-03T20:33:37.079 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:33:37.079 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:33:37.079 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "started", 2023-06-03T20:33:37.079 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:28:10.253408+0000", 2023-06-03T20:33:37.079 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.00013754300000000001 2023-06-03T20:33:37.079 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:33:37.079 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:33:37.080 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "waiting for subops from 1,2", 2023-06-03T20:33:37.080 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:28:10.253555+0000", 2023-06-03T20:33:37.080 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.00014670800000000001 2023-06-03T20:33:37.080 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:33:37.080 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:33:37.080 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "op_commit", 2023-06-03T20:33:37.080 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:28:10.255792+0000", 2023-06-03T20:33:37.080 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.0022371750000000001 2023-06-03T20:33:37.081 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:33:37.081 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:33:37.081 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:33:37.081 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:28:10.256854+0000", 2023-06-03T20:33:37.081 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.0010615259999999999 2023-06-03T20:33:37.081 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:33:37.081 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:33:37.081 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:33:37.082 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:28:10.501964+0000", 2023-06-03T20:33:37.082 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.245109943 2023-06-03T20:33:37.082 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:33:37.082 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:33:37.082 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "commit_sent", 2023-06-03T20:33:37.082 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:28:10.502013+0000", 2023-06-03T20:33:37.082 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 4.9347e-05 2023-06-03T20:33:37.083 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:33:37.083 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:33:37.083 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "done", 2023-06-03T20:33:37.083 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:28:10.502052+0000", 2023-06-03T20:33:37.083 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 3.8877000000000001e-05 2023-06-03T20:33:37.083 INFO:teuthology.orchestra.run.smithi089.stdout: } 2023-06-03T20:33:37.083 INFO:teuthology.orchestra.run.smithi089.stdout: ] 2023-06-03T20:33:37.083 INFO:teuthology.orchestra.run.smithi089.stdout: } 2023-06-03T20:33:37.084 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:33:37.084 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:33:37.084 INFO:teuthology.orchestra.run.smithi089.stdout: "description": "osd_op(client.46739.0:40735 14.1 14:89bbe23b:::benchmark_data_smithi089_237281_object40734:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e281)", 2023-06-03T20:33:37.084 INFO:teuthology.orchestra.run.smithi089.stdout: "initiated_at": "2023-06-03T20:28:10.264582+0000", 2023-06-03T20:33:37.084 INFO:teuthology.orchestra.run.smithi089.stdout: "age": 326.70896691399997, 2023-06-03T20:33:37.084 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.237591156, 2023-06-03T20:33:37.084 INFO:teuthology.orchestra.run.smithi089.stdout: "type_data": { 2023-06-03T20:33:37.084 INFO:teuthology.orchestra.run.smithi089.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-03T20:33:37.085 INFO:teuthology.orchestra.run.smithi089.stdout: "client_info": { 2023-06-03T20:33:37.085 INFO:teuthology.orchestra.run.smithi089.stdout: "client": "client.46739", 2023-06-03T20:33:37.085 INFO:teuthology.orchestra.run.smithi089.stdout: "client_addr": "172.21.15.89:0/2479974613", 2023-06-03T20:33:37.085 INFO:teuthology.orchestra.run.smithi089.stdout: "tid": 40735 2023-06-03T20:33:37.085 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:33:37.085 INFO:teuthology.orchestra.run.smithi089.stdout: "events": [ 2023-06-03T20:33:37.085 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:33:37.086 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "initiated", 2023-06-03T20:33:37.086 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:28:10.264582+0000", 2023-06-03T20:33:37.086 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0 2023-06-03T20:33:37.086 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:33:37.086 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:33:37.086 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "throttled", 2023-06-03T20:33:37.086 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:28:10.264582+0000", 2023-06-03T20:33:37.086 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0 2023-06-03T20:33:37.087 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:33:37.087 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:33:37.087 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "header_read", 2023-06-03T20:33:37.087 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:28:10.264580+0000", 2023-06-03T20:33:37.087 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 4294967295.9999981 2023-06-03T20:33:37.087 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:33:37.087 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:33:37.087 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "all_read", 2023-06-03T20:33:37.088 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:28:10.264607+0000", 2023-06-03T20:33:37.088 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 2.6469e-05 2023-06-03T20:33:37.088 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:33:37.088 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:33:37.088 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "dispatched", 2023-06-03T20:33:37.088 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:28:10.264618+0000", 2023-06-03T20:33:37.088 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 1.1728e-05 2023-06-03T20:33:37.088 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:33:37.089 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:33:37.089 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "queued_for_pg", 2023-06-03T20:33:37.089 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:28:10.264624+0000", 2023-06-03T20:33:37.089 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 5.5140000000000001e-06 2023-06-03T20:33:37.089 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:33:37.089 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:33:37.089 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "reached_pg", 2023-06-03T20:33:37.090 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:28:10.264675+0000", 2023-06-03T20:33:37.090 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 5.0933999999999997e-05 2023-06-03T20:33:37.090 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:33:37.090 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:33:37.090 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "started", 2023-06-03T20:33:37.090 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:28:10.264815+0000", 2023-06-03T20:33:37.090 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.000140388 2023-06-03T20:33:37.090 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:33:37.091 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:33:37.091 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "waiting for subops from 1,2", 2023-06-03T20:33:37.091 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:28:10.264905+0000", 2023-06-03T20:33:37.091 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 9.0222000000000005e-05 2023-06-03T20:33:37.091 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:33:37.091 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:33:37.091 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "op_commit", 2023-06-03T20:33:37.091 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:28:10.266343+0000", 2023-06-03T20:33:37.092 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.0014378539999999999 2023-06-03T20:33:37.092 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:33:37.092 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:33:37.092 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:33:37.092 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:28:10.266544+0000", 2023-06-03T20:33:37.092 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.00020106099999999999 2023-06-03T20:33:37.092 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:33:37.092 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:33:37.093 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:33:37.093 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:28:10.502091+0000", 2023-06-03T20:33:37.093 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.235546594 2023-06-03T20:33:37.093 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:33:37.093 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:33:37.093 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "commit_sent", 2023-06-03T20:33:37.093 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:28:10.502137+0000", 2023-06-03T20:33:37.094 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 4.5973999999999997e-05 2023-06-03T20:33:37.094 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:33:37.094 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:33:37.094 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "done", 2023-06-03T20:33:37.094 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:28:10.502173+0000", 2023-06-03T20:33:37.094 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 3.6384000000000003e-05 2023-06-03T20:33:37.094 INFO:teuthology.orchestra.run.smithi089.stdout: } 2023-06-03T20:33:37.094 INFO:teuthology.orchestra.run.smithi089.stdout: ] 2023-06-03T20:33:37.095 INFO:teuthology.orchestra.run.smithi089.stdout: } 2023-06-03T20:33:37.095 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:33:37.095 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:33:37.095 INFO:teuthology.orchestra.run.smithi089.stdout: "description": "osd_op(client.46739.0:171063 14.1 14:8686ed59:::benchmark_data_smithi089_237281_object73047:head [delete] snapc 0=[] ondisk+write+known_if_redirected e292)", 2023-06-03T20:33:37.095 INFO:teuthology.orchestra.run.smithi089.stdout: "initiated_at": "2023-06-03T20:31:15.703303+0000", 2023-06-03T20:33:37.095 INFO:teuthology.orchestra.run.smithi089.stdout: "age": 141.27024629900001, 2023-06-03T20:33:37.095 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.22534325899999999, 2023-06-03T20:33:37.095 INFO:teuthology.orchestra.run.smithi089.stdout: "type_data": { 2023-06-03T20:33:37.096 INFO:teuthology.orchestra.run.smithi089.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-03T20:33:37.096 INFO:teuthology.orchestra.run.smithi089.stdout: "client_info": { 2023-06-03T20:33:37.096 INFO:teuthology.orchestra.run.smithi089.stdout: "client": "client.46739", 2023-06-03T20:33:37.096 INFO:teuthology.orchestra.run.smithi089.stdout: "client_addr": "172.21.15.89:0/2479974613", 2023-06-03T20:33:37.096 INFO:teuthology.orchestra.run.smithi089.stdout: "tid": 171063 2023-06-03T20:33:37.096 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:33:37.096 INFO:teuthology.orchestra.run.smithi089.stdout: "events": [ 2023-06-03T20:33:37.097 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:33:37.097 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "initiated", 2023-06-03T20:33:37.097 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:31:15.703303+0000", 2023-06-03T20:33:37.097 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0 2023-06-03T20:33:37.097 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:33:37.097 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:33:37.097 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "throttled", 2023-06-03T20:33:37.097 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:31:15.703303+0000", 2023-06-03T20:33:37.098 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0 2023-06-03T20:33:37.098 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:33:37.098 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:33:37.098 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "header_read", 2023-06-03T20:33:37.098 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:31:15.703302+0000", 2023-06-03T20:33:37.098 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 4294967295.999999 2023-06-03T20:33:37.098 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:33:37.098 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:33:37.099 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "all_read", 2023-06-03T20:33:37.099 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:31:15.703306+0000", 2023-06-03T20:33:37.099 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 4.6e-06 2023-06-03T20:33:37.099 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:33:37.099 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:33:37.099 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "dispatched", 2023-06-03T20:33:37.099 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:31:15.703316+0000", 2023-06-03T20:33:37.100 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 9.8539999999999992e-06 2023-06-03T20:33:37.100 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:33:37.100 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:33:37.100 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "queued_for_pg", 2023-06-03T20:33:37.100 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:31:15.703323+0000", 2023-06-03T20:33:37.100 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 6.6019999999999996e-06 2023-06-03T20:33:37.100 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:33:37.100 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:33:37.101 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "reached_pg", 2023-06-03T20:33:37.101 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:31:15.703355+0000", 2023-06-03T20:33:37.101 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 3.1767999999999998e-05 2023-06-03T20:33:37.101 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:33:37.101 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:33:37.101 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "started", 2023-06-03T20:33:37.101 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:31:15.703432+0000", 2023-06-03T20:33:37.101 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 7.7233999999999999e-05 2023-06-03T20:33:37.102 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:33:37.102 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:33:37.102 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "waiting for subops from 1,2", 2023-06-03T20:33:37.102 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:31:15.703499+0000", 2023-06-03T20:33:37.102 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 6.7471000000000005e-05 2023-06-03T20:33:37.102 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:33:37.102 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:33:37.102 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "op_commit", 2023-06-03T20:33:37.103 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:31:15.707844+0000", 2023-06-03T20:33:37.103 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.0043445710000000002 2023-06-03T20:33:37.103 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:33:37.103 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:33:37.103 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:33:37.103 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:31:15.709681+0000", 2023-06-03T20:33:37.103 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.001836708 2023-06-03T20:33:37.103 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:33:37.104 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:33:37.104 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:33:37.104 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:31:15.928427+0000", 2023-06-03T20:33:37.104 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.218746099 2023-06-03T20:33:37.104 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:33:37.104 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:33:37.104 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "commit_sent", 2023-06-03T20:33:37.105 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:31:15.928554+0000", 2023-06-03T20:33:37.105 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.00012743699999999999 2023-06-03T20:33:37.105 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:33:37.105 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:33:37.105 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "done", 2023-06-03T20:33:37.105 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:31:15.928646+0000", 2023-06-03T20:33:37.105 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 9.1917999999999996e-05 2023-06-03T20:33:37.105 INFO:teuthology.orchestra.run.smithi089.stdout: } 2023-06-03T20:33:37.106 INFO:teuthology.orchestra.run.smithi089.stdout: ] 2023-06-03T20:33:37.106 INFO:teuthology.orchestra.run.smithi089.stdout: } 2023-06-03T20:33:37.106 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:33:37.106 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:33:37.106 INFO:teuthology.orchestra.run.smithi089.stdout: "description": "osd_op(client.46739.0:171073 14.1 14:887353a3:::benchmark_data_smithi089_237281_object73057:head [delete] snapc 0=[] ondisk+write+known_if_redirected e292)", 2023-06-03T20:33:37.106 INFO:teuthology.orchestra.run.smithi089.stdout: "initiated_at": "2023-06-03T20:31:15.720237+0000", 2023-06-03T20:33:37.106 INFO:teuthology.orchestra.run.smithi089.stdout: "age": 141.25331258, 2023-06-03T20:33:37.107 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.20857588899999999, 2023-06-03T20:33:37.107 INFO:teuthology.orchestra.run.smithi089.stdout: "type_data": { 2023-06-03T20:33:37.107 INFO:teuthology.orchestra.run.smithi089.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-03T20:33:37.107 INFO:teuthology.orchestra.run.smithi089.stdout: "client_info": { 2023-06-03T20:33:37.107 INFO:teuthology.orchestra.run.smithi089.stdout: "client": "client.46739", 2023-06-03T20:33:37.107 INFO:teuthology.orchestra.run.smithi089.stdout: "client_addr": "172.21.15.89:0/2479974613", 2023-06-03T20:33:37.107 INFO:teuthology.orchestra.run.smithi089.stdout: "tid": 171073 2023-06-03T20:33:37.107 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:33:37.108 INFO:teuthology.orchestra.run.smithi089.stdout: "events": [ 2023-06-03T20:33:37.108 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:33:37.108 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "initiated", 2023-06-03T20:33:37.108 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:31:15.720237+0000", 2023-06-03T20:33:37.108 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0 2023-06-03T20:33:37.108 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:33:37.108 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:33:37.108 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "throttled", 2023-06-03T20:33:37.109 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:31:15.720237+0000", 2023-06-03T20:33:37.109 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0 2023-06-03T20:33:37.109 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:33:37.109 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:33:37.109 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "header_read", 2023-06-03T20:33:37.109 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:31:15.720235+0000", 2023-06-03T20:33:37.109 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 4294967295.9999981 2023-06-03T20:33:37.109 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:33:37.110 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:33:37.110 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "all_read", 2023-06-03T20:33:37.110 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:31:15.720243+0000", 2023-06-03T20:33:37.110 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 8.259e-06 2023-06-03T20:33:37.110 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:33:37.110 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:33:37.110 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "dispatched", 2023-06-03T20:33:37.111 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:31:15.720257+0000", 2023-06-03T20:33:37.111 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 1.3453e-05 2023-06-03T20:33:37.111 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:33:37.111 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:33:37.111 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "queued_for_pg", 2023-06-03T20:33:37.111 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:31:15.720266+0000", 2023-06-03T20:33:37.111 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 9.4089999999999992e-06 2023-06-03T20:33:37.111 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:33:37.112 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:33:37.112 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "reached_pg", 2023-06-03T20:33:37.112 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:31:15.720308+0000", 2023-06-03T20:33:37.112 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 4.2512000000000002e-05 2023-06-03T20:33:37.112 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:33:37.112 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:33:37.112 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "started", 2023-06-03T20:33:37.112 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:31:15.720431+0000", 2023-06-03T20:33:37.113 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.00012283699999999999 2023-06-03T20:33:37.113 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:33:37.113 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:33:37.113 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "waiting for subops from 1,2", 2023-06-03T20:33:37.113 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:31:15.720540+0000", 2023-06-03T20:33:37.113 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.000108826 2023-06-03T20:33:37.113 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:33:37.114 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:33:37.114 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:33:37.114 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:31:15.731995+0000", 2023-06-03T20:33:37.114 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.011454799 2023-06-03T20:33:37.114 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:33:37.114 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:33:37.114 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "op_commit", 2023-06-03T20:33:37.114 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:31:15.928321+0000", 2023-06-03T20:33:37.115 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.196326421 2023-06-03T20:33:37.115 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:33:37.115 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:33:37.115 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:33:37.115 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:31:15.928732+0000", 2023-06-03T20:33:37.115 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.00041093500000000002 2023-06-03T20:33:37.115 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:33:37.115 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:33:37.116 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "commit_sent", 2023-06-03T20:33:37.116 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:31:15.928776+0000", 2023-06-03T20:33:37.116 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 4.4192e-05 2023-06-03T20:33:37.116 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:33:37.116 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:33:37.116 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "done", 2023-06-03T20:33:37.116 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:31:15.928812+0000", 2023-06-03T20:33:37.116 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 3.5945999999999998e-05 2023-06-03T20:33:37.117 INFO:teuthology.orchestra.run.smithi089.stdout: } 2023-06-03T20:33:37.117 INFO:teuthology.orchestra.run.smithi089.stdout: ] 2023-06-03T20:33:37.117 INFO:teuthology.orchestra.run.smithi089.stdout: } 2023-06-03T20:33:37.117 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:33:37.117 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:33:37.117 INFO:teuthology.orchestra.run.smithi089.stdout: "description": "osd_op(client.46739.0:171489 14.1 14:83de7efe:::benchmark_data_smithi089_237281_object73473:head [delete] snapc 0=[] ondisk+write+known_if_redirected e293)", 2023-06-03T20:33:37.117 INFO:teuthology.orchestra.run.smithi089.stdout: "initiated_at": "2023-06-03T20:31:16.592313+0000", 2023-06-03T20:33:37.117 INFO:teuthology.orchestra.run.smithi089.stdout: "age": 140.38123658399999, 2023-06-03T20:33:37.118 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.21269164099999999, 2023-06-03T20:33:37.118 INFO:teuthology.orchestra.run.smithi089.stdout: "type_data": { 2023-06-03T20:33:37.118 INFO:teuthology.orchestra.run.smithi089.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-03T20:33:37.118 INFO:teuthology.orchestra.run.smithi089.stdout: "client_info": { 2023-06-03T20:33:37.118 INFO:teuthology.orchestra.run.smithi089.stdout: "client": "client.46739", 2023-06-03T20:33:37.118 INFO:teuthology.orchestra.run.smithi089.stdout: "client_addr": "172.21.15.89:0/2479974613", 2023-06-03T20:33:37.118 INFO:teuthology.orchestra.run.smithi089.stdout: "tid": 171489 2023-06-03T20:33:37.119 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:33:37.119 INFO:teuthology.orchestra.run.smithi089.stdout: "events": [ 2023-06-03T20:33:37.119 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:33:37.119 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "initiated", 2023-06-03T20:33:37.119 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:31:16.592313+0000", 2023-06-03T20:33:37.119 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0 2023-06-03T20:33:37.119 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:33:37.119 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:33:37.120 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "throttled", 2023-06-03T20:33:37.120 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:31:16.592313+0000", 2023-06-03T20:33:37.120 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0 2023-06-03T20:33:37.120 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:33:37.120 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:33:37.120 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "header_read", 2023-06-03T20:33:37.120 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:31:16.592312+0000", 2023-06-03T20:33:37.120 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 4294967295.999999 2023-06-03T20:33:37.121 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:33:37.121 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:33:37.121 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "all_read", 2023-06-03T20:33:37.121 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:31:16.592317+0000", 2023-06-03T20:33:37.121 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 5.5060000000000003e-06 2023-06-03T20:33:37.121 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:33:37.121 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:33:37.121 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "dispatched", 2023-06-03T20:33:37.122 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:31:16.592327+0000", 2023-06-03T20:33:37.122 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 1.0156000000000001e-05 2023-06-03T20:33:37.122 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:33:37.122 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:33:37.122 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "queued_for_pg", 2023-06-03T20:33:37.122 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:31:16.592335+0000", 2023-06-03T20:33:37.122 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 7.6790000000000005e-06 2023-06-03T20:33:37.122 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:33:37.123 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:33:37.123 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "reached_pg", 2023-06-03T20:33:37.123 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:31:16.595642+0000", 2023-06-03T20:33:37.123 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.003307581 2023-06-03T20:33:37.123 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:33:37.123 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:33:37.123 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "started", 2023-06-03T20:33:37.124 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:31:16.595770+0000", 2023-06-03T20:33:37.124 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.00012769699999999999 2023-06-03T20:33:37.124 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:33:37.124 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:33:37.124 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "waiting for subops from 1,2", 2023-06-03T20:33:37.124 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:31:16.595884+0000", 2023-06-03T20:33:37.124 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.000114343 2023-06-03T20:33:37.124 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:33:37.125 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:33:37.125 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:33:37.125 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:31:16.600735+0000", 2023-06-03T20:33:37.125 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.0048502470000000002 2023-06-03T20:33:37.125 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:33:37.125 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:33:37.125 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "op_commit", 2023-06-03T20:33:37.125 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:31:16.600745+0000", 2023-06-03T20:33:37.126 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 1.0094e-05 2023-06-03T20:33:37.126 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:33:37.126 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:33:37.126 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:33:37.126 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:31:16.804804+0000", 2023-06-03T20:33:37.126 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.20405913000000001 2023-06-03T20:33:37.126 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:33:37.126 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:33:37.127 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "commit_sent", 2023-06-03T20:33:37.127 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:31:16.804961+0000", 2023-06-03T20:33:37.127 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.000157124 2023-06-03T20:33:37.127 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:33:37.127 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:33:37.127 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "done", 2023-06-03T20:33:37.127 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:31:16.805004+0000", 2023-06-03T20:33:37.127 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 4.3075999999999999e-05 2023-06-03T20:33:37.128 INFO:teuthology.orchestra.run.smithi089.stdout: } 2023-06-03T20:33:37.128 INFO:teuthology.orchestra.run.smithi089.stdout: ] 2023-06-03T20:33:37.128 INFO:teuthology.orchestra.run.smithi089.stdout: } 2023-06-03T20:33:37.128 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:33:37.128 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:33:37.128 INFO:teuthology.orchestra.run.smithi089.stdout: "description": "osd_op(client.46739.0:172195 14.9 14:90f6ee8b:::benchmark_data_smithi089_237281_object74179:head [delete] snapc 0=[] ondisk+write+known_if_redirected e293)", 2023-06-03T20:33:37.128 INFO:teuthology.orchestra.run.smithi089.stdout: "initiated_at": "2023-06-03T20:31:18.005897+0000", 2023-06-03T20:33:37.129 INFO:teuthology.orchestra.run.smithi089.stdout: "age": 138.967651953, 2023-06-03T20:33:37.129 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.23216778099999999, 2023-06-03T20:33:37.129 INFO:teuthology.orchestra.run.smithi089.stdout: "type_data": { 2023-06-03T20:33:37.129 INFO:teuthology.orchestra.run.smithi089.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-03T20:33:37.129 INFO:teuthology.orchestra.run.smithi089.stdout: "client_info": { 2023-06-03T20:33:37.129 INFO:teuthology.orchestra.run.smithi089.stdout: "client": "client.46739", 2023-06-03T20:33:37.129 INFO:teuthology.orchestra.run.smithi089.stdout: "client_addr": "172.21.15.89:0/2479974613", 2023-06-03T20:33:37.129 INFO:teuthology.orchestra.run.smithi089.stdout: "tid": 172195 2023-06-03T20:33:37.130 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:33:37.130 INFO:teuthology.orchestra.run.smithi089.stdout: "events": [ 2023-06-03T20:33:37.130 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:33:37.130 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "initiated", 2023-06-03T20:33:37.130 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:31:18.005897+0000", 2023-06-03T20:33:37.130 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0 2023-06-03T20:33:37.130 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:33:37.130 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:33:37.131 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "throttled", 2023-06-03T20:33:37.131 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:31:18.005897+0000", 2023-06-03T20:33:37.131 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0 2023-06-03T20:33:37.131 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:33:37.131 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:33:37.131 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "header_read", 2023-06-03T20:33:37.131 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:31:18.005895+0000", 2023-06-03T20:33:37.131 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 4294967295.9999986 2023-06-03T20:33:37.132 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:33:37.132 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:33:37.132 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "all_read", 2023-06-03T20:33:37.132 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:31:18.005903+0000", 2023-06-03T20:33:37.132 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 7.6520000000000006e-06 2023-06-03T20:33:37.132 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:33:37.132 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:33:37.133 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "dispatched", 2023-06-03T20:33:37.133 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:31:18.005916+0000", 2023-06-03T20:33:37.133 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 1.2785e-05 2023-06-03T20:33:37.133 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:33:37.133 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:33:37.133 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "queued_for_pg", 2023-06-03T20:33:37.133 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:31:18.005926+0000", 2023-06-03T20:33:37.133 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 9.6139999999999998e-06 2023-06-03T20:33:37.134 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:33:37.134 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:33:37.134 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "reached_pg", 2023-06-03T20:33:37.134 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:31:18.005977+0000", 2023-06-03T20:33:37.134 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 5.1675000000000002e-05 2023-06-03T20:33:37.134 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:33:37.134 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:33:37.134 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "started", 2023-06-03T20:33:37.135 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:31:18.006090+0000", 2023-06-03T20:33:37.135 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.00011291200000000001 2023-06-03T20:33:37.135 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:33:37.135 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:33:37.135 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "waiting for subops from 4,5", 2023-06-03T20:33:37.135 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:31:18.006169+0000", 2023-06-03T20:33:37.135 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 7.9054999999999994e-05 2023-06-03T20:33:37.135 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:33:37.136 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:33:37.136 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "op_commit", 2023-06-03T20:33:37.136 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:31:18.012418+0000", 2023-06-03T20:33:37.136 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.0062484289999999998 2023-06-03T20:33:37.136 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:33:37.136 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:33:37.136 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:33:37.137 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:31:18.237630+0000", 2023-06-03T20:33:37.137 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.225212144 2023-06-03T20:33:37.137 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:33:37.137 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:33:37.137 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:33:37.137 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:31:18.237779+0000", 2023-06-03T20:33:37.137 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.00014905300000000001 2023-06-03T20:33:37.137 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:33:37.138 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:33:37.138 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "commit_sent", 2023-06-03T20:33:37.138 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:31:18.237914+0000", 2023-06-03T20:33:37.138 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.000135504 2023-06-03T20:33:37.138 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:33:37.138 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:33:37.138 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "done", 2023-06-03T20:33:37.138 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:31:18.238065+0000", 2023-06-03T20:33:37.139 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.000150607 2023-06-03T20:33:37.139 INFO:teuthology.orchestra.run.smithi089.stdout: } 2023-06-03T20:33:37.139 INFO:teuthology.orchestra.run.smithi089.stdout: ] 2023-06-03T20:33:37.139 INFO:teuthology.orchestra.run.smithi089.stdout: } 2023-06-03T20:33:37.139 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:33:37.139 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:33:37.139 INFO:teuthology.orchestra.run.smithi089.stdout: "description": "osd_op(client.46739.0:172196 14.9 14:95d2269f:::benchmark_data_smithi089_237281_object74180:head [delete] snapc 0=[] ondisk+write+known_if_redirected e293)", 2023-06-03T20:33:37.139 INFO:teuthology.orchestra.run.smithi089.stdout: "initiated_at": "2023-06-03T20:31:18.006014+0000", 2023-06-03T20:33:37.140 INFO:teuthology.orchestra.run.smithi089.stdout: "age": 138.967535309, 2023-06-03T20:33:37.140 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.232714386, 2023-06-03T20:33:37.140 INFO:teuthology.orchestra.run.smithi089.stdout: "type_data": { 2023-06-03T20:33:37.140 INFO:teuthology.orchestra.run.smithi089.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-03T20:33:37.140 INFO:teuthology.orchestra.run.smithi089.stdout: "client_info": { 2023-06-03T20:33:37.140 INFO:teuthology.orchestra.run.smithi089.stdout: "client": "client.46739", 2023-06-03T20:33:37.140 INFO:teuthology.orchestra.run.smithi089.stdout: "client_addr": "172.21.15.89:0/2479974613", 2023-06-03T20:33:37.141 INFO:teuthology.orchestra.run.smithi089.stdout: "tid": 172196 2023-06-03T20:33:37.141 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:33:37.141 INFO:teuthology.orchestra.run.smithi089.stdout: "events": [ 2023-06-03T20:33:37.141 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:33:37.141 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "initiated", 2023-06-03T20:33:37.141 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:31:18.006014+0000", 2023-06-03T20:33:37.141 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0 2023-06-03T20:33:37.141 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:33:37.142 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:33:37.142 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "throttled", 2023-06-03T20:33:37.142 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:31:18.006014+0000", 2023-06-03T20:33:37.142 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0 2023-06-03T20:33:37.142 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:33:37.142 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:33:37.142 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "header_read", 2023-06-03T20:33:37.142 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:31:18.006013+0000", 2023-06-03T20:33:37.143 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 4294967295.999999 2023-06-03T20:33:37.143 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:33:37.143 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:33:37.143 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "all_read", 2023-06-03T20:33:37.143 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:31:18.006018+0000", 2023-06-03T20:33:37.143 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 4.6140000000000002e-06 2023-06-03T20:33:37.143 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:33:37.143 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:33:37.144 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "dispatched", 2023-06-03T20:33:37.144 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:31:18.006027+0000", 2023-06-03T20:33:37.144 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 9.4010000000000003e-06 2023-06-03T20:33:37.144 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:33:37.144 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:33:37.144 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "queued_for_pg", 2023-06-03T20:33:37.144 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:31:18.006033+0000", 2023-06-03T20:33:37.145 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 5.6219999999999997e-06 2023-06-03T20:33:37.145 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:33:37.145 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:33:37.145 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "reached_pg", 2023-06-03T20:33:37.145 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:31:18.009337+0000", 2023-06-03T20:33:37.145 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.0033042729999999999 2023-06-03T20:33:37.145 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:33:37.145 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:33:37.146 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "started", 2023-06-03T20:33:37.146 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:31:18.009415+0000", 2023-06-03T20:33:37.146 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 7.7805000000000005e-05 2023-06-03T20:33:37.146 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:33:37.146 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:33:37.146 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "waiting for subops from 4,5", 2023-06-03T20:33:37.146 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:31:18.009486+0000", 2023-06-03T20:33:37.146 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 7.1204000000000005e-05 2023-06-03T20:33:37.147 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:33:37.147 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:33:37.147 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "op_commit", 2023-06-03T20:33:37.147 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:31:18.012615+0000", 2023-06-03T20:33:37.147 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.003129106 2023-06-03T20:33:37.147 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:33:37.147 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:33:37.147 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:33:37.148 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:31:18.238238+0000", 2023-06-03T20:33:37.148 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.22562290600000001 2023-06-03T20:33:37.148 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:33:37.148 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:33:37.148 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:33:37.148 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:31:18.238485+0000", 2023-06-03T20:33:37.148 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.00024691099999999999 2023-06-03T20:33:37.148 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:33:37.149 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:33:37.149 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "commit_sent", 2023-06-03T20:33:37.149 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:31:18.238611+0000", 2023-06-03T20:33:37.149 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.000126379 2023-06-03T20:33:37.149 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:33:37.149 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:33:37.149 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "done", 2023-06-03T20:33:37.149 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:31:18.238728+0000", 2023-06-03T20:33:37.150 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.00011703499999999999 2023-06-03T20:33:37.150 INFO:teuthology.orchestra.run.smithi089.stdout: } 2023-06-03T20:33:37.150 INFO:teuthology.orchestra.run.smithi089.stdout: ] 2023-06-03T20:33:37.150 INFO:teuthology.orchestra.run.smithi089.stdout: } 2023-06-03T20:33:37.150 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:33:37.150 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:33:37.150 INFO:teuthology.orchestra.run.smithi089.stdout: "description": "osd_op(client.46739.0:172198 14.1 14:8876f4f0:::benchmark_data_smithi089_237281_object74182:head [delete] snapc 0=[] ondisk+write+known_if_redirected e293)", 2023-06-03T20:33:37.151 INFO:teuthology.orchestra.run.smithi089.stdout: "initiated_at": "2023-06-03T20:31:18.013080+0000", 2023-06-03T20:33:37.151 INFO:teuthology.orchestra.run.smithi089.stdout: "age": 138.960469222, 2023-06-03T20:33:37.151 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.23060697999999999, 2023-06-03T20:33:37.151 INFO:teuthology.orchestra.run.smithi089.stdout: "type_data": { 2023-06-03T20:33:37.151 INFO:teuthology.orchestra.run.smithi089.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-03T20:33:37.151 INFO:teuthology.orchestra.run.smithi089.stdout: "client_info": { 2023-06-03T20:33:37.151 INFO:teuthology.orchestra.run.smithi089.stdout: "client": "client.46739", 2023-06-03T20:33:37.151 INFO:teuthology.orchestra.run.smithi089.stdout: "client_addr": "172.21.15.89:0/2479974613", 2023-06-03T20:33:37.152 INFO:teuthology.orchestra.run.smithi089.stdout: "tid": 172198 2023-06-03T20:33:37.152 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:33:37.152 INFO:teuthology.orchestra.run.smithi089.stdout: "events": [ 2023-06-03T20:33:37.152 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:33:37.152 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "initiated", 2023-06-03T20:33:37.152 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:31:18.013080+0000", 2023-06-03T20:33:37.152 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0 2023-06-03T20:33:37.152 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:33:37.153 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:33:37.153 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "throttled", 2023-06-03T20:33:37.153 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:31:18.013080+0000", 2023-06-03T20:33:37.153 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0 2023-06-03T20:33:37.153 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:33:37.153 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:33:37.153 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "header_read", 2023-06-03T20:33:37.153 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:31:18.013079+0000", 2023-06-03T20:33:37.154 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 4294967295.999999 2023-06-03T20:33:37.154 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:33:37.154 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:33:37.154 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "all_read", 2023-06-03T20:33:37.154 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:31:18.013084+0000", 2023-06-03T20:33:37.154 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 4.9010000000000002e-06 2023-06-03T20:33:37.154 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:33:37.155 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:33:37.155 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "dispatched", 2023-06-03T20:33:37.155 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:31:18.013093+0000", 2023-06-03T20:33:37.155 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 9.5529999999999998e-06 2023-06-03T20:33:37.155 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:33:37.155 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:33:37.155 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "queued_for_pg", 2023-06-03T20:33:37.155 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:31:18.013100+0000", 2023-06-03T20:33:37.156 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 6.4690000000000002e-06 2023-06-03T20:33:37.156 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:33:37.156 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:33:37.156 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "reached_pg", 2023-06-03T20:33:37.156 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:31:18.013128+0000", 2023-06-03T20:33:37.156 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 2.8177000000000001e-05 2023-06-03T20:33:37.156 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:33:37.156 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:33:37.157 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "started", 2023-06-03T20:33:37.157 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:31:18.013197+0000", 2023-06-03T20:33:37.157 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 6.9247999999999998e-05 2023-06-03T20:33:37.157 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:33:37.157 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:33:37.157 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "waiting for subops from 1,2", 2023-06-03T20:33:37.157 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:31:18.013261+0000", 2023-06-03T20:33:37.158 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 6.3868999999999994e-05 2023-06-03T20:33:37.158 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:33:37.158 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:33:37.158 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:33:37.158 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:31:18.238425+0000", 2023-06-03T20:33:37.158 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.22516365899999999 2023-06-03T20:33:37.158 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:33:37.158 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:33:37.159 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "op_commit", 2023-06-03T20:33:37.159 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:31:18.238773+0000", 2023-06-03T20:33:37.159 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.00034873199999999998 2023-06-03T20:33:37.159 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:33:37.159 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:33:37.159 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:33:37.159 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:31:18.243604+0000", 2023-06-03T20:33:37.159 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.0048307560000000003 2023-06-03T20:33:37.160 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:33:37.160 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:33:37.160 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "commit_sent", 2023-06-03T20:33:37.160 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:31:18.243648+0000", 2023-06-03T20:33:37.160 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 4.3776000000000002e-05 2023-06-03T20:33:37.160 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:33:37.160 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:33:37.160 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "done", 2023-06-03T20:33:37.161 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:31:18.243687+0000", 2023-06-03T20:33:37.161 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 3.8884000000000002e-05 2023-06-03T20:33:37.161 INFO:teuthology.orchestra.run.smithi089.stdout: } 2023-06-03T20:33:37.161 INFO:teuthology.orchestra.run.smithi089.stdout: ] 2023-06-03T20:33:37.161 INFO:teuthology.orchestra.run.smithi089.stdout: } 2023-06-03T20:33:37.161 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:33:37.161 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:33:37.161 INFO:teuthology.orchestra.run.smithi089.stdout: "description": "osd_repop_reply(client.46739.0:172195 14.9 e293/278 ondisk, result = 0)", 2023-06-03T20:33:37.162 INFO:teuthology.orchestra.run.smithi089.stdout: "initiated_at": "2023-06-03T20:31:18.015363+0000", 2023-06-03T20:33:37.162 INFO:teuthology.orchestra.run.smithi089.stdout: "age": 138.95818605400001, 2023-06-03T20:33:37.162 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.22274127599999999, 2023-06-03T20:33:37.162 INFO:teuthology.orchestra.run.smithi089.stdout: "type_data": { 2023-06-03T20:33:37.162 INFO:teuthology.orchestra.run.smithi089.stdout: "flag_point": "started", 2023-06-03T20:33:37.162 INFO:teuthology.orchestra.run.smithi089.stdout: "events": [ 2023-06-03T20:33:37.162 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:33:37.163 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "initiated", 2023-06-03T20:33:37.163 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:31:18.015363+0000", 2023-06-03T20:33:37.163 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0 2023-06-03T20:33:37.163 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:33:37.163 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:33:37.163 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "throttled", 2023-06-03T20:33:37.163 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:31:18.015363+0000", 2023-06-03T20:33:37.163 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0 2023-06-03T20:33:37.164 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:33:37.164 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:33:37.164 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "header_read", 2023-06-03T20:33:37.164 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:31:18.015362+0000", 2023-06-03T20:33:37.164 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 4294967295.999999 2023-06-03T20:33:37.164 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:33:37.164 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:33:37.164 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "all_read", 2023-06-03T20:33:37.165 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:31:18.015367+0000", 2023-06-03T20:33:37.165 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 4.5190000000000003e-06 2023-06-03T20:33:37.165 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:33:37.165 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:33:37.165 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "dispatched", 2023-06-03T20:33:37.165 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:31:18.232173+0000", 2023-06-03T20:33:37.165 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.21680618700000001 2023-06-03T20:33:37.165 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:33:37.166 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:33:37.166 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "queued_for_pg", 2023-06-03T20:33:37.166 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:31:18.232202+0000", 2023-06-03T20:33:37.166 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 2.9278999999999999e-05 2023-06-03T20:33:37.166 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:33:37.166 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:33:37.166 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "reached_pg", 2023-06-03T20:33:37.167 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:31:18.237740+0000", 2023-06-03T20:33:37.167 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.0055376100000000001 2023-06-03T20:33:37.167 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:33:37.167 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:33:37.167 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "started", 2023-06-03T20:33:37.167 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:31:18.237760+0000", 2023-06-03T20:33:37.167 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 2.0523999999999999e-05 2023-06-03T20:33:37.167 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:33:37.168 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:33:37.168 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "done", 2023-06-03T20:33:37.168 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:31:18.238104+0000", 2023-06-03T20:33:37.168 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.000343913 2023-06-03T20:33:37.168 INFO:teuthology.orchestra.run.smithi089.stdout: } 2023-06-03T20:33:37.168 INFO:teuthology.orchestra.run.smithi089.stdout: ] 2023-06-03T20:33:37.168 INFO:teuthology.orchestra.run.smithi089.stdout: } 2023-06-03T20:33:37.168 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:33:37.169 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:33:37.169 INFO:teuthology.orchestra.run.smithi089.stdout: "description": "osd_op(client.46739.0:172200 14.9 14:9b8437a0:::benchmark_data_smithi089_237281_object74184:head [delete] snapc 0=[] ondisk+write+known_if_redirected e293)", 2023-06-03T20:33:37.169 INFO:teuthology.orchestra.run.smithi089.stdout: "initiated_at": "2023-06-03T20:31:18.015975+0000", 2023-06-03T20:33:37.169 INFO:teuthology.orchestra.run.smithi089.stdout: "age": 138.95757376899999, 2023-06-03T20:33:37.169 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.24867015000000001, 2023-06-03T20:33:37.169 INFO:teuthology.orchestra.run.smithi089.stdout: "type_data": { 2023-06-03T20:33:37.169 INFO:teuthology.orchestra.run.smithi089.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-03T20:33:37.169 INFO:teuthology.orchestra.run.smithi089.stdout: "client_info": { 2023-06-03T20:33:37.170 INFO:teuthology.orchestra.run.smithi089.stdout: "client": "client.46739", 2023-06-03T20:33:37.170 INFO:teuthology.orchestra.run.smithi089.stdout: "client_addr": "172.21.15.89:0/2479974613", 2023-06-03T20:33:37.170 INFO:teuthology.orchestra.run.smithi089.stdout: "tid": 172200 2023-06-03T20:33:37.170 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:33:37.170 INFO:teuthology.orchestra.run.smithi089.stdout: "events": [ 2023-06-03T20:33:37.170 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:33:37.170 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "initiated", 2023-06-03T20:33:37.171 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:31:18.015975+0000", 2023-06-03T20:33:37.171 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0 2023-06-03T20:33:37.171 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:33:37.171 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:33:37.171 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "throttled", 2023-06-03T20:33:37.171 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:31:18.015975+0000", 2023-06-03T20:33:37.171 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0 2023-06-03T20:33:37.171 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:33:37.172 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:33:37.172 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "header_read", 2023-06-03T20:33:37.172 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:31:18.015975+0000", 2023-06-03T20:33:37.172 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 4294967295.9999995 2023-06-03T20:33:37.172 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:33:37.172 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:33:37.172 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "all_read", 2023-06-03T20:33:37.172 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:31:18.015978+0000", 2023-06-03T20:33:37.173 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 3.6200000000000001e-06 2023-06-03T20:33:37.173 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:33:37.173 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:33:37.173 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "dispatched", 2023-06-03T20:33:37.173 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:31:18.231690+0000", 2023-06-03T20:33:37.173 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.215711879 2023-06-03T20:33:37.173 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:33:37.174 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:33:37.174 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "queued_for_pg", 2023-06-03T20:33:37.174 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:31:18.231703+0000", 2023-06-03T20:33:37.174 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 1.2514000000000001e-05 2023-06-03T20:33:37.174 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:33:37.174 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:33:37.174 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "reached_pg", 2023-06-03T20:33:37.174 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:31:18.238821+0000", 2023-06-03T20:33:37.175 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.0071179269999999996 2023-06-03T20:33:37.175 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:33:37.175 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:33:37.175 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "started", 2023-06-03T20:33:37.175 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:31:18.238995+0000", 2023-06-03T20:33:37.175 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.00017459100000000001 2023-06-03T20:33:37.175 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:33:37.175 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:33:37.176 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "waiting for subops from 4,5", 2023-06-03T20:33:37.176 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:31:18.239133+0000", 2023-06-03T20:33:37.176 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.00013809599999999999 2023-06-03T20:33:37.176 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:33:37.176 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:33:37.176 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:33:37.176 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:31:18.254580+0000", 2023-06-03T20:33:37.176 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.015447012 2023-06-03T20:33:37.177 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:33:37.177 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:33:37.177 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:33:37.177 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:31:18.254719+0000", 2023-06-03T20:33:37.177 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.000138653 2023-06-03T20:33:37.177 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:33:37.177 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:33:37.178 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "op_commit", 2023-06-03T20:33:37.178 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:31:18.264555+0000", 2023-06-03T20:33:37.178 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.0098364149999999994 2023-06-03T20:33:37.178 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:33:37.178 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:33:37.178 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "commit_sent", 2023-06-03T20:33:37.178 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:31:18.264604+0000", 2023-06-03T20:33:37.178 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 4.8412000000000003e-05 2023-06-03T20:33:37.179 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:33:37.179 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:33:37.179 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "done", 2023-06-03T20:33:37.179 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:31:18.264645+0000", 2023-06-03T20:33:37.179 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 4.1575000000000001e-05 2023-06-03T20:33:37.179 INFO:teuthology.orchestra.run.smithi089.stdout: } 2023-06-03T20:33:37.179 INFO:teuthology.orchestra.run.smithi089.stdout: ] 2023-06-03T20:33:37.179 INFO:teuthology.orchestra.run.smithi089.stdout: } 2023-06-03T20:33:37.180 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:33:37.180 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:33:37.180 INFO:teuthology.orchestra.run.smithi089.stdout: "description": "osd_repop_reply(client.46739.0:172198 14.1 e293/278 ondisk, result = 0)", 2023-06-03T20:33:37.180 INFO:teuthology.orchestra.run.smithi089.stdout: "initiated_at": "2023-06-03T20:31:18.016416+0000", 2023-06-03T20:33:37.180 INFO:teuthology.orchestra.run.smithi089.stdout: "age": 138.957132922, 2023-06-03T20:33:37.180 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.222012237, 2023-06-03T20:33:37.180 INFO:teuthology.orchestra.run.smithi089.stdout: "type_data": { 2023-06-03T20:33:37.181 INFO:teuthology.orchestra.run.smithi089.stdout: "flag_point": "started", 2023-06-03T20:33:37.181 INFO:teuthology.orchestra.run.smithi089.stdout: "events": [ 2023-06-03T20:33:37.181 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:33:37.181 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "initiated", 2023-06-03T20:33:37.181 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:31:18.016416+0000", 2023-06-03T20:33:37.181 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0 2023-06-03T20:33:37.181 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:33:37.181 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:33:37.182 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "throttled", 2023-06-03T20:33:37.182 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:31:18.016416+0000", 2023-06-03T20:33:37.182 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0 2023-06-03T20:33:37.182 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:33:37.182 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:33:37.182 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "header_read", 2023-06-03T20:33:37.182 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:31:18.016415+0000", 2023-06-03T20:33:37.182 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 4294967295.999999 2023-06-03T20:33:37.183 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:33:37.183 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:33:37.183 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "all_read", 2023-06-03T20:33:37.183 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:31:18.016419+0000", 2023-06-03T20:33:37.183 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 3.822e-06 2023-06-03T20:33:37.183 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:33:37.183 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:33:37.183 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "dispatched", 2023-06-03T20:33:37.184 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:31:18.231565+0000", 2023-06-03T20:33:37.184 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.215146116 2023-06-03T20:33:37.184 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:33:37.184 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:33:37.184 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "queued_for_pg", 2023-06-03T20:33:37.184 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:31:18.231580+0000", 2023-06-03T20:33:37.184 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 1.4946000000000001e-05 2023-06-03T20:33:37.185 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:33:37.185 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:33:37.185 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "reached_pg", 2023-06-03T20:33:37.185 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:31:18.238410+0000", 2023-06-03T20:33:37.185 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 0.0068301100000000003 2023-06-03T20:33:37.185 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:33:37.185 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:33:37.185 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "started", 2023-06-03T20:33:37.186 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:31:18.238419+0000", 2023-06-03T20:33:37.186 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 8.1280000000000008e-06 2023-06-03T20:33:37.186 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2023-06-03T20:33:37.186 INFO:teuthology.orchestra.run.smithi089.stdout: { 2023-06-03T20:33:37.186 INFO:teuthology.orchestra.run.smithi089.stdout: "event": "done", 2023-06-03T20:33:37.186 INFO:teuthology.orchestra.run.smithi089.stdout: "time": "2023-06-03T20:31:18.238428+0000", 2023-06-03T20:33:37.186 INFO:teuthology.orchestra.run.smithi089.stdout: "duration": 9.8670000000000006e-06 2023-06-03T20:33:37.186 INFO:teuthology.orchestra.run.smithi089.stdout: } 2023-06-03T20:33:37.187 INFO:teuthology.orchestra.run.smithi089.stdout: ] 2023-06-03T20:33:37.187 INFO:teuthology.orchestra.run.smithi089.stdout: } 2023-06-03T20:33:37.187 INFO:teuthology.orchestra.run.smithi089.stdout: } 2023-06-03T20:33:37.187 INFO:teuthology.orchestra.run.smithi089.stdout: ] 2023-06-03T20:33:37.187 INFO:teuthology.orchestra.run.smithi089.stdout:} 2023-06-03T20:33:37.585 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:33:37 smithi164 bash[59462]: audit 1685824417.164077 mon.c (mon.2) 696 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T20:33:37.779 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:33:37 smithi089 bash[75068]: audit 1685824417.164077 mon.c (mon.2) 696 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T20:33:37.779 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:33:37 smithi089 bash[78452]: audit 1685824417.164077 mon.c (mon.2) 696 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T20:33:38.474 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:33:38 smithi089 bash[78452]: cluster 1685824417.2471685 mgr.y (mgr.34104) 1738 : cluster 0 pgmap v1804: 73 pgs: 73 active+clean; 582 KiB data, 5.1 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:33:38.474 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:33:38 smithi089 bash[75068]: cluster 1685824417.2471685 mgr.y (mgr.34104) 1738 : cluster 0 pgmap v1804: 73 pgs: 73 active+clean; 582 KiB data, 5.1 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:33:38.585 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:33:38 smithi164 bash[59462]: cluster 1685824417.2471685 mgr.y (mgr.34104) 1738 : cluster 0 pgmap v1804: 73 pgs: 73 active+clean; 582 KiB data, 5.1 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:33:39.928 DEBUG:teuthology.orchestra.run.smithi164:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph daemon osd.5 dump_ops_in_flight 2023-06-03T20:33:40.585 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:33:40 smithi164 bash[59462]: cluster 1685824419.2480013 mgr.y (mgr.34104) 1739 : cluster 0 pgmap v1805: 73 pgs: 73 active+clean; 582 KiB data, 5.1 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:33:40.779 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:33:40 smithi089 bash[75068]: cluster 1685824419.2480013 mgr.y (mgr.34104) 1739 : cluster 0 pgmap v1805: 73 pgs: 73 active+clean; 582 KiB data, 5.1 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:33:40.780 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:33:40 smithi089 bash[78452]: cluster 1685824419.2480013 mgr.y (mgr.34104) 1739 : cluster 0 pgmap v1805: 73 pgs: 73 active+clean; 582 KiB data, 5.1 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:33:41.782 INFO:teuthology.orchestra.run.smithi089.stdout:mon.a: {} 2023-06-03T20:33:41.801 INFO:teuthology.orchestra.run.smithi089.stdout:mon.b: {} 2023-06-03T20:33:41.820 INFO:teuthology.orchestra.run.smithi089.stdout:mon.c: {} 2023-06-03T20:33:41.822 INFO:teuthology.orchestra.run.smithi089.stderr:mon.a: mon_osd_down_out_interval = '600' (not observed, change may require restart) 2023-06-03T20:33:41.822 INFO:teuthology.orchestra.run.smithi089.stderr:mon.b: mon_osd_down_out_interval = '600' (not observed, change may require restart) 2023-06-03T20:33:41.822 INFO:teuthology.orchestra.run.smithi089.stderr:mon.c: mon_osd_down_out_interval = '600' (not observed, change may require restart) 2023-06-03T20:33:42.337 INFO:teuthology.orchestra.run.smithi089.stderr:nodeep-scrub is unset 2023-06-03T20:33:42.740 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:33:42 smithi089 bash[78452]: cluster 1685824421.2487524 mgr.y (mgr.34104) 1740 : cluster 0 pgmap v1806: 73 pgs: 73 active+clean; 582 KiB data, 5.1 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:33:42.740 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:33:42 smithi089 bash[78452]: audit 1685824421.764396 mon.a (mon.0) 1701 : audit 0 from='client.? 172.21.15.89:0/2443547188' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-06-03T20:33:42.740 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:33:42 smithi089 bash[78452]: audit 1685824422.2077394 mon.c (mon.2) 697 : audit 1 from='client.? 172.21.15.89:0/900958346' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-06-03T20:33:42.741 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:33:42 smithi089 bash[78452]: audit 1685824422.2084508 mon.a (mon.0) 1702 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-06-03T20:33:42.741 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:33:42 smithi089 bash[75068]: cluster 1685824421.2487524 mgr.y (mgr.34104) 1740 : cluster 0 pgmap v1806: 73 pgs: 73 active+clean; 582 KiB data, 5.1 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:33:42.741 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:33:42 smithi089 bash[75068]: audit 1685824421.764396 mon.a (mon.0) 1701 : audit 0 from='client.? 172.21.15.89:0/2443547188' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-06-03T20:33:42.741 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:33:42 smithi089 bash[75068]: audit 1685824422.2077394 mon.c (mon.2) 697 : audit 1 from='client.? 172.21.15.89:0/900958346' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-06-03T20:33:42.741 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:33:42 smithi089 bash[75068]: audit 1685824422.2084508 mon.a (mon.0) 1702 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-06-03T20:33:42.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:33:42 smithi164 bash[59462]: cluster 1685824421.2487524 mgr.y (mgr.34104) 1740 : cluster 0 pgmap v1806: 73 pgs: 73 active+clean; 582 KiB data, 5.1 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:33:42.836 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:33:42 smithi164 bash[59462]: audit 1685824421.764396 mon.a (mon.0) 1701 : audit 0 from='client.? 172.21.15.89:0/2443547188' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-06-03T20:33:42.836 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:33:42 smithi164 bash[59462]: audit 1685824422.2077394 mon.c (mon.2) 697 : audit 1 from='client.? 172.21.15.89:0/900958346' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-06-03T20:33:42.836 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:33:42 smithi164 bash[59462]: audit 1685824422.2084508 mon.a (mon.0) 1702 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-06-03T20:33:43.146 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph osd reweight 0 1 2023-06-03T20:33:43.529 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:33:43 smithi089 bash[75068]: cluster 1685824422.3201256 osd.5 (osd.5) 255 : cluster 3 failed to encode map e307 with expected crc 2023-06-03T20:33:43.530 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:33:43 smithi089 bash[75068]: audit 1685824422.323713 mon.a (mon.0) 1703 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-06-03T20:33:43.530 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:33:43 smithi089 bash[75068]: cluster 1685824422.3237653 mon.a (mon.0) 1704 : cluster 0 osdmap e307: 8 total, 8 up, 8 in 2023-06-03T20:33:43.530 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:33:43 smithi089 bash[78452]: cluster 1685824422.3201256 osd.5 (osd.5) 255 : cluster 3 failed to encode map e307 with expected crc 2023-06-03T20:33:43.530 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:33:43 smithi089 bash[78452]: audit 1685824422.323713 mon.a (mon.0) 1703 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-06-03T20:33:43.530 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:33:43 smithi089 bash[78452]: cluster 1685824422.3237653 mon.a (mon.0) 1704 : cluster 0 osdmap e307: 8 total, 8 up, 8 in 2023-06-03T20:33:43.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:33:43 smithi164 bash[59462]: cluster 1685824422.3201256 osd.5 (osd.5) 255 : cluster 3 failed to encode map e307 with expected crc 2023-06-03T20:33:43.836 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:33:43 smithi164 bash[59462]: audit 1685824422.323713 mon.a (mon.0) 1703 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-06-03T20:33:43.836 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:33:43 smithi164 bash[59462]: cluster 1685824422.3237653 mon.a (mon.0) 1704 : cluster 0 osdmap e307: 8 total, 8 up, 8 in 2023-06-03T20:33:44.744 INFO:teuthology.orchestra.run.smithi164.stderr:Inferring config /var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/mon.b/config 2023-06-03T20:33:44.779 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:33:44 smithi089 bash[75068]: cluster 1685824423.2494285 mgr.y (mgr.34104) 1741 : cluster 0 pgmap v1808: 73 pgs: 73 active+clean; 582 KiB data, 5.1 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:33:44.779 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:33:44 smithi089 bash[75068]: cluster 1685824423.5042164 osd.7 (osd.7) 252 : cluster 3 failed to encode map e307 with expected crc 2023-06-03T20:33:44.780 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:33:44 smithi089 bash[78452]: cluster 1685824423.2494285 mgr.y (mgr.34104) 1741 : cluster 0 pgmap v1808: 73 pgs: 73 active+clean; 582 KiB data, 5.1 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:33:44.780 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:33:44 smithi089 bash[78452]: cluster 1685824423.5042164 osd.7 (osd.7) 252 : cluster 3 failed to encode map e307 with expected crc 2023-06-03T20:33:44.784 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:33:44 smithi164 bash[59462]: cluster 1685824423.2494285 mgr.y (mgr.34104) 1741 : cluster 0 pgmap v1808: 73 pgs: 73 active+clean; 582 KiB data, 5.1 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:33:44.784 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:33:44 smithi164 bash[59462]: cluster 1685824423.5042164 osd.7 (osd.7) 252 : cluster 3 failed to encode map e307 with expected crc 2023-06-03T20:33:46.529 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:33:46 smithi089 bash[78452]: cluster 1685824425.2502987 mgr.y (mgr.34104) 1742 : cluster 0 pgmap v1809: 73 pgs: 73 active+clean; 582 KiB data, 5.1 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:33:46.529 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:33:46 smithi089 bash[75068]: cluster 1685824425.2502987 mgr.y (mgr.34104) 1742 : cluster 0 pgmap v1809: 73 pgs: 73 active+clean; 582 KiB data, 5.1 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:33:46.530 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 20:33:46 smithi089 bash[71163]: ::ffff:172.21.15.164 - - [03/Jun/2023:20:33:46] "GET /metrics HTTP/1.1" 200 226955 "" "Prometheus/2.33.4" 2023-06-03T20:33:46.585 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:33:46 smithi164 bash[59462]: cluster 1685824425.2502987 mgr.y (mgr.34104) 1742 : cluster 0 pgmap v1809: 73 pgs: 73 active+clean; 582 KiB data, 5.1 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:33:47.950 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/mon.a/config 2023-06-03T20:33:48.734 INFO:teuthology.orchestra.run.smithi164.stdout:{ 2023-06-03T20:33:48.734 INFO:teuthology.orchestra.run.smithi164.stdout: "ops": [], 2023-06-03T20:33:48.734 INFO:teuthology.orchestra.run.smithi164.stdout: "num_ops": 0 2023-06-03T20:33:48.734 INFO:teuthology.orchestra.run.smithi164.stdout:} 2023-06-03T20:33:48.761 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:33:48 smithi164 bash[59462]: cluster 1685824427.2512743 mgr.y (mgr.34104) 1743 : cluster 0 pgmap v1810: 73 pgs: 73 active+clean; 582 KiB data, 5.1 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:33:48.779 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:33:48 smithi089 bash[75068]: cluster 1685824427.2512743 mgr.y (mgr.34104) 1743 : cluster 0 pgmap v1810: 73 pgs: 73 active+clean; 582 KiB data, 5.1 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:33:48.779 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:33:48 smithi089 bash[78452]: cluster 1685824427.2512743 mgr.y (mgr.34104) 1743 : cluster 0 pgmap v1810: 73 pgs: 73 active+clean; 582 KiB data, 5.1 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:33:49.637 DEBUG:teuthology.orchestra.run.smithi164:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph daemon osd.5 dump_blocked_ops 2023-06-03T20:33:50.779 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:33:50 smithi089 bash[75068]: cluster 1685824429.2522743 mgr.y (mgr.34104) 1744 : cluster 0 pgmap v1811: 73 pgs: 73 active+clean; 582 KiB data, 5.1 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:33:50.779 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:33:50 smithi089 bash[78452]: cluster 1685824429.2522743 mgr.y (mgr.34104) 1744 : cluster 0 pgmap v1811: 73 pgs: 73 active+clean; 582 KiB data, 5.1 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:33:50.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:33:50 smithi164 bash[59462]: cluster 1685824429.2522743 mgr.y (mgr.34104) 1744 : cluster 0 pgmap v1811: 73 pgs: 73 active+clean; 582 KiB data, 5.1 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:33:52.368 INFO:teuthology.orchestra.run.smithi089.stderr:reweighted osd.0 to 1 (10000) 2023-06-03T20:33:52.779 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:33:52 smithi089 bash[75068]: cluster 1685824431.2530613 mgr.y (mgr.34104) 1745 : cluster 0 pgmap v1812: 73 pgs: 73 active+clean; 582 KiB data, 5.1 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:33:52.779 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:33:52 smithi089 bash[75068]: audit 1685824431.9874492 mon.a (mon.0) 1705 : audit 1 from='client.? 172.21.15.89:0/2411878065' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-06-03T20:33:52.779 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:33:52 smithi089 bash[75068]: audit 1685824432.1649113 mon.c (mon.2) 698 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T20:33:52.780 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:33:52 smithi089 bash[78452]: cluster 1685824431.2530613 mgr.y (mgr.34104) 1745 : cluster 0 pgmap v1812: 73 pgs: 73 active+clean; 582 KiB data, 5.1 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:33:52.780 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:33:52 smithi089 bash[78452]: audit 1685824431.9874492 mon.a (mon.0) 1705 : audit 1 from='client.? 172.21.15.89:0/2411878065' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-06-03T20:33:52.780 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:33:52 smithi089 bash[78452]: audit 1685824432.1649113 mon.c (mon.2) 698 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T20:33:52.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:33:52 smithi164 bash[59462]: cluster 1685824431.2530613 mgr.y (mgr.34104) 1745 : cluster 0 pgmap v1812: 73 pgs: 73 active+clean; 582 KiB data, 5.1 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:33:52.836 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:33:52 smithi164 bash[59462]: audit 1685824431.9874492 mon.a (mon.0) 1705 : audit 1 from='client.? 172.21.15.89:0/2411878065' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-06-03T20:33:52.836 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:33:52 smithi164 bash[59462]: audit 1685824432.1649113 mon.c (mon.2) 698 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T20:33:53.339 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph osd primary-affinity 0 1 2023-06-03T20:33:53.779 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:33:53 smithi089 bash[78452]: cluster 1685824432.3520944 osd.6 (osd.6) 245 : cluster 3 failed to encode map e308 with expected crc 2023-06-03T20:33:53.779 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:33:53 smithi089 bash[78452]: audit 1685824432.358057 mon.a (mon.0) 1706 : audit 1 from='client.? 172.21.15.89:0/2411878065' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2023-06-03T20:33:53.780 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:33:53 smithi089 bash[78452]: cluster 1685824432.3581598 mon.a (mon.0) 1707 : cluster 0 osdmap e308: 8 total, 8 up, 8 in 2023-06-03T20:33:53.780 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:33:53 smithi089 bash[75068]: cluster 1685824432.3520944 osd.6 (osd.6) 245 : cluster 3 failed to encode map e308 with expected crc 2023-06-03T20:33:53.780 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:33:53 smithi089 bash[75068]: audit 1685824432.358057 mon.a (mon.0) 1706 : audit 1 from='client.? 172.21.15.89:0/2411878065' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2023-06-03T20:33:53.780 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:33:53 smithi089 bash[75068]: cluster 1685824432.3581598 mon.a (mon.0) 1707 : cluster 0 osdmap e308: 8 total, 8 up, 8 in 2023-06-03T20:33:53.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:33:53 smithi164 bash[59462]: cluster 1685824432.3520944 osd.6 (osd.6) 245 : cluster 3 failed to encode map e308 with expected crc 2023-06-03T20:33:53.836 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:33:53 smithi164 bash[59462]: audit 1685824432.358057 mon.a (mon.0) 1706 : audit 1 from='client.? 172.21.15.89:0/2411878065' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2023-06-03T20:33:53.836 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:33:53 smithi164 bash[59462]: cluster 1685824432.3581598 mon.a (mon.0) 1707 : cluster 0 osdmap e308: 8 total, 8 up, 8 in 2023-06-03T20:33:54.450 INFO:teuthology.orchestra.run.smithi164.stderr:Inferring config /var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/mon.b/config 2023-06-03T20:33:54.779 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:33:54 smithi089 bash[75068]: cluster 1685824433.1849513 osd.7 (osd.7) 253 : cluster 3 failed to encode map e308 with expected crc 2023-06-03T20:33:54.780 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:33:54 smithi089 bash[75068]: cluster 1685824433.2072263 osd.5 (osd.5) 256 : cluster 3 failed to encode map e308 with expected crc 2023-06-03T20:33:54.780 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:33:54 smithi089 bash[75068]: cluster 1685824433.2539027 mgr.y (mgr.34104) 1746 : cluster 0 pgmap v1814: 73 pgs: 73 active+clean; 582 KiB data, 5.1 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:33:54.780 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:33:54 smithi089 bash[78452]: cluster 1685824433.1849513 osd.7 (osd.7) 253 : cluster 3 failed to encode map e308 with expected crc 2023-06-03T20:33:54.780 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:33:54 smithi089 bash[78452]: cluster 1685824433.2072263 osd.5 (osd.5) 256 : cluster 3 failed to encode map e308 with expected crc 2023-06-03T20:33:54.781 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:33:54 smithi089 bash[78452]: cluster 1685824433.2539027 mgr.y (mgr.34104) 1746 : cluster 0 pgmap v1814: 73 pgs: 73 active+clean; 582 KiB data, 5.1 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:33:54.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:33:54 smithi164 bash[59462]: cluster 1685824433.1849513 osd.7 (osd.7) 253 : cluster 3 failed to encode map e308 with expected crc 2023-06-03T20:33:54.836 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:33:54 smithi164 bash[59462]: cluster 1685824433.2072263 osd.5 (osd.5) 256 : cluster 3 failed to encode map e308 with expected crc 2023-06-03T20:33:54.836 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:33:54 smithi164 bash[59462]: cluster 1685824433.2539027 mgr.y (mgr.34104) 1746 : cluster 0 pgmap v1814: 73 pgs: 73 active+clean; 582 KiB data, 5.1 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:33:56.529 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 20:33:56 smithi089 bash[71163]: ::ffff:172.21.15.164 - - [03/Jun/2023:20:33:56] "GET /metrics HTTP/1.1" 200 226958 "" "Prometheus/2.33.4" 2023-06-03T20:33:56.529 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:33:56 smithi089 bash[78452]: cluster 1685824435.254731 mgr.y (mgr.34104) 1747 : cluster 0 pgmap v1815: 73 pgs: 73 active+clean; 582 KiB data, 5.1 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:33:56.530 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:33:56 smithi089 bash[75068]: cluster 1685824435.254731 mgr.y (mgr.34104) 1747 : cluster 0 pgmap v1815: 73 pgs: 73 active+clean; 582 KiB data, 5.1 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:33:56.585 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:33:56 smithi164 bash[59462]: cluster 1685824435.254731 mgr.y (mgr.34104) 1747 : cluster 0 pgmap v1815: 73 pgs: 73 active+clean; 582 KiB data, 5.1 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:33:58.150 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/mon.a/config 2023-06-03T20:33:58.172 INFO:teuthology.orchestra.run.smithi164.stdout:{ 2023-06-03T20:33:58.173 INFO:teuthology.orchestra.run.smithi164.stdout: "ops": [], 2023-06-03T20:33:58.173 INFO:teuthology.orchestra.run.smithi164.stdout: "complaint_time": 30, 2023-06-03T20:33:58.173 INFO:teuthology.orchestra.run.smithi164.stdout: "num_blocked_ops": 0 2023-06-03T20:33:58.173 INFO:teuthology.orchestra.run.smithi164.stdout:} 2023-06-03T20:33:58.585 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:33:58 smithi164 bash[59462]: cluster 1685824437.255497 mgr.y (mgr.34104) 1748 : cluster 0 pgmap v1816: 73 pgs: 73 active+clean; 582 KiB data, 5.1 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:33:58.779 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:33:58 smithi089 bash[75068]: cluster 1685824437.255497 mgr.y (mgr.34104) 1748 : cluster 0 pgmap v1816: 73 pgs: 73 active+clean; 582 KiB data, 5.1 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:33:58.779 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:33:58 smithi089 bash[78452]: cluster 1685824437.255497 mgr.y (mgr.34104) 1748 : cluster 0 pgmap v1816: 73 pgs: 73 active+clean; 582 KiB data, 5.1 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:33:59.096 DEBUG:teuthology.orchestra.run.smithi164:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph daemon osd.5 dump_historic_ops 2023-06-03T20:34:00.636 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:34:00 smithi089 bash[75068]: cluster 1685824439.256464 mgr.y (mgr.34104) 1749 : cluster 0 pgmap v1817: 73 pgs: 73 active+clean; 582 KiB data, 5.1 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:34:00.637 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:34:00 smithi089 bash[78452]: cluster 1685824439.256464 mgr.y (mgr.34104) 1749 : cluster 0 pgmap v1817: 73 pgs: 73 active+clean; 582 KiB data, 5.1 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:34:00.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:34:00 smithi164 bash[59462]: cluster 1685824439.256464 mgr.y (mgr.34104) 1749 : cluster 0 pgmap v1817: 73 pgs: 73 active+clean; 582 KiB data, 5.1 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:34:02.399 INFO:teuthology.orchestra.run.smithi089.stderr:set osd.0 primary-affinity to 1 (8655362) 2023-06-03T20:34:02.779 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:34:02 smithi089 bash[78452]: cluster 1685824441.2572913 mgr.y (mgr.34104) 1750 : cluster 0 pgmap v1818: 73 pgs: 73 active+clean; 582 KiB data, 5.1 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:34:02.780 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:34:02 smithi089 bash[78452]: audit 1685824442.2861767 mon.a (mon.0) 1708 : audit 1 from='client.? 172.21.15.89:0/651372910' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 0, "weight": 1.0}]: dispatch 2023-06-03T20:34:02.780 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:34:02 smithi089 bash[75068]: cluster 1685824441.2572913 mgr.y (mgr.34104) 1750 : cluster 0 pgmap v1818: 73 pgs: 73 active+clean; 582 KiB data, 5.1 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:34:02.780 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:34:02 smithi089 bash[75068]: audit 1685824442.2861767 mon.a (mon.0) 1708 : audit 1 from='client.? 172.21.15.89:0/651372910' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 0, "weight": 1.0}]: dispatch 2023-06-03T20:34:02.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:34:02 smithi164 bash[59462]: cluster 1685824441.2572913 mgr.y (mgr.34104) 1750 : cluster 0 pgmap v1818: 73 pgs: 73 active+clean; 582 KiB data, 5.1 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:34:02.836 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:34:02 smithi164 bash[59462]: audit 1685824442.2861767 mon.a (mon.0) 1708 : audit 1 from='client.? 172.21.15.89:0/651372910' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 0, "weight": 1.0}]: dispatch 2023-06-03T20:34:03.374 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph osd reweight 1 1 2023-06-03T20:34:03.779 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:34:03 smithi089 bash[75068]: audit 1685824442.3877401 mon.a (mon.0) 1709 : audit 1 from='client.? 172.21.15.89:0/651372910' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 0, "weight": 1.0}]': finished 2023-06-03T20:34:03.780 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:34:03 smithi089 bash[75068]: cluster 1685824442.387838 mon.a (mon.0) 1710 : cluster 0 osdmap e309: 8 total, 8 up, 8 in 2023-06-03T20:34:03.780 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:34:03 smithi089 bash[78452]: audit 1685824442.3877401 mon.a (mon.0) 1709 : audit 1 from='client.? 172.21.15.89:0/651372910' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 0, "weight": 1.0}]': finished 2023-06-03T20:34:03.780 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:34:03 smithi089 bash[78452]: cluster 1685824442.387838 mon.a (mon.0) 1710 : cluster 0 osdmap e309: 8 total, 8 up, 8 in 2023-06-03T20:34:03.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:34:03 smithi164 bash[59462]: audit 1685824442.3877401 mon.a (mon.0) 1709 : audit 1 from='client.? 172.21.15.89:0/651372910' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 0, "weight": 1.0}]': finished 2023-06-03T20:34:03.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:34:03 smithi164 bash[59462]: cluster 1685824442.387838 mon.a (mon.0) 1710 : cluster 0 osdmap e309: 8 total, 8 up, 8 in 2023-06-03T20:34:03.925 INFO:teuthology.orchestra.run.smithi164.stderr:Inferring config /var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/mon.b/config 2023-06-03T20:34:04.779 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:34:04 smithi089 bash[75068]: cluster 1685824443.2430384 osd.7 (osd.7) 254 : cluster 3 failed to encode map e309 with expected crc 2023-06-03T20:34:04.779 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:34:04 smithi089 bash[75068]: cluster 1685824443.2581193 mgr.y (mgr.34104) 1751 : cluster 0 pgmap v1820: 73 pgs: 73 active+clean; 582 KiB data, 5.1 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:34:04.780 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:34:04 smithi089 bash[75068]: cluster 1685824443.469639 osd.5 (osd.5) 257 : cluster 3 failed to encode map e309 with expected crc 2023-06-03T20:34:04.780 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:34:04 smithi089 bash[75068]: cluster 1685824443.4697225 osd.6 (osd.6) 246 : cluster 3 failed to encode map e309 with expected crc 2023-06-03T20:34:04.780 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:34:04 smithi089 bash[78452]: cluster 1685824443.2430384 osd.7 (osd.7) 254 : cluster 3 failed to encode map e309 with expected crc 2023-06-03T20:34:04.780 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:34:04 smithi089 bash[78452]: cluster 1685824443.2581193 mgr.y (mgr.34104) 1751 : cluster 0 pgmap v1820: 73 pgs: 73 active+clean; 582 KiB data, 5.1 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:34:04.780 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:34:04 smithi089 bash[78452]: cluster 1685824443.469639 osd.5 (osd.5) 257 : cluster 3 failed to encode map e309 with expected crc 2023-06-03T20:34:04.780 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:34:04 smithi089 bash[78452]: cluster 1685824443.4697225 osd.6 (osd.6) 246 : cluster 3 failed to encode map e309 with expected crc 2023-06-03T20:34:04.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:34:04 smithi164 bash[59462]: cluster 1685824443.2430384 osd.7 (osd.7) 254 : cluster 3 failed to encode map e309 with expected crc 2023-06-03T20:34:04.836 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:34:04 smithi164 bash[59462]: cluster 1685824443.2581193 mgr.y (mgr.34104) 1751 : cluster 0 pgmap v1820: 73 pgs: 73 active+clean; 582 KiB data, 5.1 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:34:04.836 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:34:04 smithi164 bash[59462]: cluster 1685824443.469639 osd.5 (osd.5) 257 : cluster 3 failed to encode map e309 with expected crc 2023-06-03T20:34:04.836 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:34:04 smithi164 bash[59462]: cluster 1685824443.4697225 osd.6 (osd.6) 246 : cluster 3 failed to encode map e309 with expected crc 2023-06-03T20:34:06.529 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:34:06 smithi089 bash[75068]: cluster 1685824445.2589247 mgr.y (mgr.34104) 1752 : cluster 0 pgmap v1821: 73 pgs: 73 active+clean; 582 KiB data, 5.1 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:34:06.529 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 20:34:06 smithi089 bash[71163]: ::ffff:172.21.15.164 - - [03/Jun/2023:20:34:06] "GET /metrics HTTP/1.1" 200 226958 "" "Prometheus/2.33.4" 2023-06-03T20:34:06.530 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:34:06 smithi089 bash[78452]: cluster 1685824445.2589247 mgr.y (mgr.34104) 1752 : cluster 0 pgmap v1821: 73 pgs: 73 active+clean; 582 KiB data, 5.1 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:34:06.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:34:06 smithi164 bash[59462]: cluster 1685824445.2589247 mgr.y (mgr.34104) 1752 : cluster 0 pgmap v1821: 73 pgs: 73 active+clean; 582 KiB data, 5.1 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:34:07.602 INFO:teuthology.orchestra.run.smithi164.stdout:{ 2023-06-03T20:34:07.602 INFO:teuthology.orchestra.run.smithi164.stdout: "size": 20, 2023-06-03T20:34:07.602 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 600, 2023-06-03T20:34:07.602 INFO:teuthology.orchestra.run.smithi164.stdout: "ops": [ 2023-06-03T20:34:07.602 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:07.602 INFO:teuthology.orchestra.run.smithi164.stdout: "description": "osd_repop(client.36573.0:145582 13.e e257/251 13:7649425e:::benchmark_data_smithi089_211581_object30057:head v 257'9043, mlcod=245'1906)", 2023-06-03T20:34:07.603 INFO:teuthology.orchestra.run.smithi164.stdout: "initiated_at": "2023-06-03T20:24:17.679174+0000", 2023-06-03T20:34:07.603 INFO:teuthology.orchestra.run.smithi164.stdout: "age": 589.91111049100004, 2023-06-03T20:34:07.603 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.060734807000000002, 2023-06-03T20:34:07.603 INFO:teuthology.orchestra.run.smithi164.stdout: "type_data": { 2023-06-03T20:34:07.603 INFO:teuthology.orchestra.run.smithi164.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-03T20:34:07.603 INFO:teuthology.orchestra.run.smithi164.stdout: "events": [ 2023-06-03T20:34:07.603 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:07.604 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "initiated", 2023-06-03T20:34:07.604 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:24:17.679174+0000", 2023-06-03T20:34:07.604 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0 2023-06-03T20:34:07.604 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:07.604 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:07.604 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "throttled", 2023-06-03T20:34:07.604 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:24:17.679174+0000", 2023-06-03T20:34:07.605 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 4294967296 2023-06-03T20:34:07.605 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:07.605 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:07.605 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "header_read", 2023-06-03T20:34:07.605 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:24:17.679173+0000", 2023-06-03T20:34:07.605 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 4.7539999999999997e-06 2023-06-03T20:34:07.605 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:07.605 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:07.606 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "all_read", 2023-06-03T20:34:07.606 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:24:17.679178+0000", 2023-06-03T20:34:07.606 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 5.378e-06 2023-06-03T20:34:07.606 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:07.606 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:07.606 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "dispatched", 2023-06-03T20:34:07.606 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:24:17.679184+0000", 2023-06-03T20:34:07.606 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 4.4220000000000002e-06 2023-06-03T20:34:07.607 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:07.607 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:07.607 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "queued_for_pg", 2023-06-03T20:34:07.607 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:24:17.679188+0000", 2023-06-03T20:34:07.607 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.017492583999999999 2023-06-03T20:34:07.607 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:07.607 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:07.607 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "reached_pg", 2023-06-03T20:34:07.608 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:24:17.696681+0000", 2023-06-03T20:34:07.608 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 2.6258000000000002e-05 2023-06-03T20:34:07.608 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:07.608 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:07.608 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "started", 2023-06-03T20:34:07.608 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:24:17.696707+0000", 2023-06-03T20:34:07.608 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.043118853999999998 2023-06-03T20:34:07.609 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:07.609 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:07.609 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "commit_sent", 2023-06-03T20:34:07.609 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:24:17.739826+0000", 2023-06-03T20:34:07.609 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 8.2789999999999998e-05 2023-06-03T20:34:07.609 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:07.609 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:07.609 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "done", 2023-06-03T20:34:07.610 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:24:17.739908+0000", 2023-06-03T20:34:07.610 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.060734807000000002 2023-06-03T20:34:07.610 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:34:07.610 INFO:teuthology.orchestra.run.smithi164.stdout: ] 2023-06-03T20:34:07.610 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:34:07.610 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:07.610 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:07.610 INFO:teuthology.orchestra.run.smithi164.stdout: "description": "osd_repop(client.36573.0:146920 13.f e257/255 13:f71c5bc5:::benchmark_data_smithi089_211581_object31395:head v 257'9337, mlcod=257'9334)", 2023-06-03T20:34:07.611 INFO:teuthology.orchestra.run.smithi164.stdout: "initiated_at": "2023-06-03T20:24:20.181724+0000", 2023-06-03T20:34:07.611 INFO:teuthology.orchestra.run.smithi164.stdout: "age": 587.40855986899999, 2023-06-03T20:34:07.611 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.063639554000000001, 2023-06-03T20:34:07.611 INFO:teuthology.orchestra.run.smithi164.stdout: "type_data": { 2023-06-03T20:34:07.611 INFO:teuthology.orchestra.run.smithi164.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-03T20:34:07.611 INFO:teuthology.orchestra.run.smithi164.stdout: "events": [ 2023-06-03T20:34:07.611 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:07.611 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "initiated", 2023-06-03T20:34:07.612 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:24:20.181724+0000", 2023-06-03T20:34:07.612 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0 2023-06-03T20:34:07.612 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:07.612 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:07.612 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "throttled", 2023-06-03T20:34:07.612 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:24:20.181724+0000", 2023-06-03T20:34:07.612 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 4294967295.9999995 2023-06-03T20:34:07.612 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:07.613 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:07.613 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "header_read", 2023-06-03T20:34:07.613 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:24:20.181724+0000", 2023-06-03T20:34:07.613 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 1.6084999999999999e-05 2023-06-03T20:34:07.613 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:07.613 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:07.613 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "all_read", 2023-06-03T20:34:07.614 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:24:20.181740+0000", 2023-06-03T20:34:07.614 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 7.9719999999999992e-06 2023-06-03T20:34:07.614 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:07.614 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:07.614 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "dispatched", 2023-06-03T20:34:07.614 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:24:20.181748+0000", 2023-06-03T20:34:07.614 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 4.5839999999999996e-06 2023-06-03T20:34:07.614 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:07.615 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:07.615 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "queued_for_pg", 2023-06-03T20:34:07.615 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:24:20.181752+0000", 2023-06-03T20:34:07.615 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.0058436920000000002 2023-06-03T20:34:07.615 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:07.615 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:07.615 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "reached_pg", 2023-06-03T20:34:07.615 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:24:20.187596+0000", 2023-06-03T20:34:07.616 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 2.5106999999999999e-05 2023-06-03T20:34:07.616 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:07.616 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:07.616 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "started", 2023-06-03T20:34:07.616 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:24:20.187621+0000", 2023-06-03T20:34:07.616 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.057630357 2023-06-03T20:34:07.616 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:07.616 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:07.617 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "commit_sent", 2023-06-03T20:34:07.617 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:24:20.245252+0000", 2023-06-03T20:34:07.617 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.000112225 2023-06-03T20:34:07.617 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:07.617 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:07.617 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "done", 2023-06-03T20:34:07.617 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:24:20.245364+0000", 2023-06-03T20:34:07.617 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.063639554000000001 2023-06-03T20:34:07.618 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:34:07.618 INFO:teuthology.orchestra.run.smithi164.stdout: ] 2023-06-03T20:34:07.618 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:34:07.618 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:07.618 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:07.618 INFO:teuthology.orchestra.run.smithi164.stdout: "description": "osd_repop(client.36573.0:147558 13.f e257/255 13:f807ede0:::benchmark_data_smithi089_211581_object32033:head v 257'9376, mlcod=257'9371)", 2023-06-03T20:34:07.618 INFO:teuthology.orchestra.run.smithi164.stdout: "initiated_at": "2023-06-03T20:24:21.478914+0000", 2023-06-03T20:34:07.618 INFO:teuthology.orchestra.run.smithi164.stdout: "age": 586.11137039699997, 2023-06-03T20:34:07.619 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.063243858999999999, 2023-06-03T20:34:07.619 INFO:teuthology.orchestra.run.smithi164.stdout: "type_data": { 2023-06-03T20:34:07.619 INFO:teuthology.orchestra.run.smithi164.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-03T20:34:07.619 INFO:teuthology.orchestra.run.smithi164.stdout: "events": [ 2023-06-03T20:34:07.619 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:07.619 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "initiated", 2023-06-03T20:34:07.619 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:24:21.478914+0000", 2023-06-03T20:34:07.620 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0 2023-06-03T20:34:07.620 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:07.620 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:07.620 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "throttled", 2023-06-03T20:34:07.620 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:24:21.478914+0000", 2023-06-03T20:34:07.620 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 4294967295.999999 2023-06-03T20:34:07.620 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:07.620 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:07.621 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "header_read", 2023-06-03T20:34:07.621 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:24:21.478913+0000", 2023-06-03T20:34:07.621 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 1.2228e-05 2023-06-03T20:34:07.621 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:07.621 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:07.621 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "all_read", 2023-06-03T20:34:07.621 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:24:21.478925+0000", 2023-06-03T20:34:07.621 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 1.3587999999999999e-05 2023-06-03T20:34:07.622 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:07.622 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:07.622 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "dispatched", 2023-06-03T20:34:07.622 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:24:21.478939+0000", 2023-06-03T20:34:07.622 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 6.8709999999999999e-06 2023-06-03T20:34:07.622 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:07.622 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:07.622 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "queued_for_pg", 2023-06-03T20:34:07.623 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:24:21.478945+0000", 2023-06-03T20:34:07.623 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.017864729999999999 2023-06-03T20:34:07.623 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:07.623 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:07.623 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "reached_pg", 2023-06-03T20:34:07.623 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:24:21.496810+0000", 2023-06-03T20:34:07.623 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 2.7838e-05 2023-06-03T20:34:07.623 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:07.624 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:07.624 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "started", 2023-06-03T20:34:07.624 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:24:21.496838+0000", 2023-06-03T20:34:07.624 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.045271490999999997 2023-06-03T20:34:07.624 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:07.624 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:07.624 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "commit_sent", 2023-06-03T20:34:07.625 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:24:21.542110+0000", 2023-06-03T20:34:07.625 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 4.8092e-05 2023-06-03T20:34:07.625 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:07.625 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:07.625 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "done", 2023-06-03T20:34:07.625 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:24:21.542158+0000", 2023-06-03T20:34:07.625 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.063243858999999999 2023-06-03T20:34:07.625 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:34:07.626 INFO:teuthology.orchestra.run.smithi164.stdout: ] 2023-06-03T20:34:07.626 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:34:07.626 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:07.626 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:07.626 INFO:teuthology.orchestra.run.smithi164.stdout: "description": "osd_repop(client.36573.0:147726 13.c e257/242 13:3eb2f3ba:::benchmark_data_smithi089_211581_object32201:head v 257'9380, mlcod=257'9377)", 2023-06-03T20:34:07.626 INFO:teuthology.orchestra.run.smithi164.stdout: "initiated_at": "2023-06-03T20:24:21.807217+0000", 2023-06-03T20:34:07.626 INFO:teuthology.orchestra.run.smithi164.stdout: "age": 585.78306744899999, 2023-06-03T20:34:07.626 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.067208406999999998, 2023-06-03T20:34:07.627 INFO:teuthology.orchestra.run.smithi164.stdout: "type_data": { 2023-06-03T20:34:07.627 INFO:teuthology.orchestra.run.smithi164.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-03T20:34:07.627 INFO:teuthology.orchestra.run.smithi164.stdout: "events": [ 2023-06-03T20:34:07.627 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:07.627 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "initiated", 2023-06-03T20:34:07.627 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:24:21.807217+0000", 2023-06-03T20:34:07.627 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0 2023-06-03T20:34:07.627 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:07.628 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:07.628 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "throttled", 2023-06-03T20:34:07.628 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:24:21.807217+0000", 2023-06-03T20:34:07.628 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 4294967295.999999 2023-06-03T20:34:07.628 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:07.628 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:07.628 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "header_read", 2023-06-03T20:34:07.629 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:24:21.807216+0000", 2023-06-03T20:34:07.629 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 9.8269999999999993e-06 2023-06-03T20:34:07.629 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:07.629 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:07.629 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "all_read", 2023-06-03T20:34:07.629 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:24:21.807226+0000", 2023-06-03T20:34:07.629 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 1.0468e-05 2023-06-03T20:34:07.629 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:07.630 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:07.630 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "dispatched", 2023-06-03T20:34:07.630 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:24:21.807236+0000", 2023-06-03T20:34:07.630 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 6.703e-06 2023-06-03T20:34:07.630 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:07.630 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:07.630 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "queued_for_pg", 2023-06-03T20:34:07.630 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:24:21.807243+0000", 2023-06-03T20:34:07.631 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.0038837429999999998 2023-06-03T20:34:07.631 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:07.631 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:07.631 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "reached_pg", 2023-06-03T20:34:07.631 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:24:21.811127+0000", 2023-06-03T20:34:07.631 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 2.1795999999999999e-05 2023-06-03T20:34:07.631 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:07.631 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:07.632 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "started", 2023-06-03T20:34:07.632 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:24:21.811149+0000", 2023-06-03T20:34:07.632 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.063059743000000001 2023-06-03T20:34:07.632 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:07.632 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:07.632 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "commit_sent", 2023-06-03T20:34:07.632 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:24:21.874208+0000", 2023-06-03T20:34:07.632 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.00021688100000000001 2023-06-03T20:34:07.633 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:07.633 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:07.633 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "done", 2023-06-03T20:34:07.633 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:24:21.874425+0000", 2023-06-03T20:34:07.633 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.067208406999999998 2023-06-03T20:34:07.633 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:34:07.633 INFO:teuthology.orchestra.run.smithi164.stdout: ] 2023-06-03T20:34:07.633 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:34:07.634 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:07.634 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:07.634 INFO:teuthology.orchestra.run.smithi164.stdout: "description": "osd_repop(client.36573.0:147727 13.c e257/242 13:3f710ead:::benchmark_data_smithi089_211581_object32202:head v 257'9381, mlcod=257'9377)", 2023-06-03T20:34:07.634 INFO:teuthology.orchestra.run.smithi164.stdout: "initiated_at": "2023-06-03T20:24:21.813718+0000", 2023-06-03T20:34:07.634 INFO:teuthology.orchestra.run.smithi164.stdout: "age": 585.77656654999998, 2023-06-03T20:34:07.634 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.060795643000000003, 2023-06-03T20:34:07.634 INFO:teuthology.orchestra.run.smithi164.stdout: "type_data": { 2023-06-03T20:34:07.635 INFO:teuthology.orchestra.run.smithi164.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-03T20:34:07.635 INFO:teuthology.orchestra.run.smithi164.stdout: "events": [ 2023-06-03T20:34:07.635 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:07.635 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "initiated", 2023-06-03T20:34:07.635 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:24:21.813718+0000", 2023-06-03T20:34:07.635 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0 2023-06-03T20:34:07.635 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:07.635 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:07.636 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "throttled", 2023-06-03T20:34:07.636 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:24:21.813718+0000", 2023-06-03T20:34:07.636 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 4294967295.9999986 2023-06-03T20:34:07.636 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:07.636 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:07.636 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "header_read", 2023-06-03T20:34:07.636 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:24:21.813716+0000", 2023-06-03T20:34:07.636 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 2.4386999999999999e-05 2023-06-03T20:34:07.637 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:07.637 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:07.637 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "all_read", 2023-06-03T20:34:07.637 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:24:21.813740+0000", 2023-06-03T20:34:07.637 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 1.7286e-05 2023-06-03T20:34:07.637 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:07.637 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:07.637 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "dispatched", 2023-06-03T20:34:07.638 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:24:21.813758+0000", 2023-06-03T20:34:07.638 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 7.2400000000000001e-06 2023-06-03T20:34:07.638 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:07.638 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:07.638 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "queued_for_pg", 2023-06-03T20:34:07.638 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:24:21.813765+0000", 2023-06-03T20:34:07.638 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.0081028309999999996 2023-06-03T20:34:07.639 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:07.639 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:07.639 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "reached_pg", 2023-06-03T20:34:07.639 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:24:21.821868+0000", 2023-06-03T20:34:07.639 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 2.6842000000000002e-05 2023-06-03T20:34:07.639 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:07.639 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:07.639 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "started", 2023-06-03T20:34:07.640 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:24:21.821895+0000", 2023-06-03T20:34:07.640 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.052535446 2023-06-03T20:34:07.640 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:07.640 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:07.640 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "commit_sent", 2023-06-03T20:34:07.640 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:24:21.874430+0000", 2023-06-03T20:34:07.640 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 8.3158999999999994e-05 2023-06-03T20:34:07.640 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:07.641 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:07.641 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "done", 2023-06-03T20:34:07.641 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:24:21.874513+0000", 2023-06-03T20:34:07.641 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.060795643000000003 2023-06-03T20:34:07.641 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:34:07.641 INFO:teuthology.orchestra.run.smithi164.stdout: ] 2023-06-03T20:34:07.641 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:34:07.641 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:07.642 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:07.642 INFO:teuthology.orchestra.run.smithi164.stdout: "description": "osd_repop(client.36573.0:147724 13.4 e257/242 13:2f77f775:::benchmark_data_smithi089_211581_object32199:head v 257'9213, mlcod=257'9211)", 2023-06-03T20:34:07.642 INFO:teuthology.orchestra.run.smithi164.stdout: "initiated_at": "2023-06-03T20:24:21.814420+0000", 2023-06-03T20:34:07.642 INFO:teuthology.orchestra.run.smithi164.stdout: "age": 585.77586466399998, 2023-06-03T20:34:07.642 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.063277366000000002, 2023-06-03T20:34:07.642 INFO:teuthology.orchestra.run.smithi164.stdout: "type_data": { 2023-06-03T20:34:07.642 INFO:teuthology.orchestra.run.smithi164.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-03T20:34:07.642 INFO:teuthology.orchestra.run.smithi164.stdout: "events": [ 2023-06-03T20:34:07.643 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:07.643 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "initiated", 2023-06-03T20:34:07.643 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:24:21.814420+0000", 2023-06-03T20:34:07.643 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0 2023-06-03T20:34:07.643 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:07.643 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:07.643 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "throttled", 2023-06-03T20:34:07.643 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:24:21.814420+0000", 2023-06-03T20:34:07.644 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 4294967295.999999 2023-06-03T20:34:07.644 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:07.644 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:07.644 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "header_read", 2023-06-03T20:34:07.644 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:24:21.814419+0000", 2023-06-03T20:34:07.644 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 8.0800000000000006e-06 2023-06-03T20:34:07.644 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:07.645 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:07.645 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "all_read", 2023-06-03T20:34:07.645 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:24:21.814427+0000", 2023-06-03T20:34:07.645 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 1.4399999999999999e-05 2023-06-03T20:34:07.645 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:07.645 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:07.645 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "dispatched", 2023-06-03T20:34:07.645 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:24:21.814441+0000", 2023-06-03T20:34:07.646 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 2.4834999999999999e-05 2023-06-03T20:34:07.646 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:07.646 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:07.646 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "queued_for_pg", 2023-06-03T20:34:07.646 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:24:21.814466+0000", 2023-06-03T20:34:07.646 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.0043573839999999997 2023-06-03T20:34:07.646 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:07.646 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:07.647 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "reached_pg", 2023-06-03T20:34:07.647 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:24:21.818823+0000", 2023-06-03T20:34:07.647 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 2.1736e-05 2023-06-03T20:34:07.647 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:07.647 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:07.647 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "started", 2023-06-03T20:34:07.647 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:24:21.818845+0000", 2023-06-03T20:34:07.647 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.058609266 2023-06-03T20:34:07.648 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:07.648 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:07.648 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "commit_sent", 2023-06-03T20:34:07.648 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:24:21.877454+0000", 2023-06-03T20:34:07.648 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.00024244800000000001 2023-06-03T20:34:07.648 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:07.648 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:07.648 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "done", 2023-06-03T20:34:07.649 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:24:21.877697+0000", 2023-06-03T20:34:07.649 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.063277366000000002 2023-06-03T20:34:07.649 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:34:07.649 INFO:teuthology.orchestra.run.smithi164.stdout: ] 2023-06-03T20:34:07.649 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:34:07.649 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:07.649 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:07.650 INFO:teuthology.orchestra.run.smithi164.stdout: "description": "osd_repop(client.36573.0:149490 13.6 e257/251 13:690f0f85:::benchmark_data_smithi089_211581_object33965:head v 257'9158, mlcod=257'9153)", 2023-06-03T20:34:07.650 INFO:teuthology.orchestra.run.smithi164.stdout: "initiated_at": "2023-06-03T20:24:25.180839+0000", 2023-06-03T20:34:07.650 INFO:teuthology.orchestra.run.smithi164.stdout: "age": 582.40944473499997, 2023-06-03T20:34:07.650 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.075161934999999999, 2023-06-03T20:34:07.650 INFO:teuthology.orchestra.run.smithi164.stdout: "type_data": { 2023-06-03T20:34:07.650 INFO:teuthology.orchestra.run.smithi164.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-03T20:34:07.650 INFO:teuthology.orchestra.run.smithi164.stdout: "events": [ 2023-06-03T20:34:07.650 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:07.651 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "initiated", 2023-06-03T20:34:07.651 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:24:25.180839+0000", 2023-06-03T20:34:07.651 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0 2023-06-03T20:34:07.651 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:07.651 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:07.651 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "throttled", 2023-06-03T20:34:07.651 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:24:25.180839+0000", 2023-06-03T20:34:07.651 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 4294967295.9999995 2023-06-03T20:34:07.652 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:07.652 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:07.652 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "header_read", 2023-06-03T20:34:07.652 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:24:25.180839+0000", 2023-06-03T20:34:07.652 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 6.9009999999999997e-06 2023-06-03T20:34:07.652 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:07.652 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:07.652 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "all_read", 2023-06-03T20:34:07.653 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:24:25.180846+0000", 2023-06-03T20:34:07.653 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 9.6809999999999993e-06 2023-06-03T20:34:07.653 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:07.653 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:07.653 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "dispatched", 2023-06-03T20:34:07.653 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:24:25.180855+0000", 2023-06-03T20:34:07.653 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 5.1510000000000003e-06 2023-06-03T20:34:07.654 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:07.654 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:07.654 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "queued_for_pg", 2023-06-03T20:34:07.654 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:24:25.180860+0000", 2023-06-03T20:34:07.654 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.029509850000000001 2023-06-03T20:34:07.654 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:07.654 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:07.654 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "reached_pg", 2023-06-03T20:34:07.655 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:24:25.210370+0000", 2023-06-03T20:34:07.655 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 5.7719000000000001e-05 2023-06-03T20:34:07.655 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:07.655 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:07.655 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "started", 2023-06-03T20:34:07.655 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:24:25.210428+0000", 2023-06-03T20:34:07.655 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.045531787999999997 2023-06-03T20:34:07.655 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:07.656 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:07.656 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "commit_sent", 2023-06-03T20:34:07.656 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:24:25.255960+0000", 2023-06-03T20:34:07.656 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 4.1520000000000002e-05 2023-06-03T20:34:07.656 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:07.656 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:07.656 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "done", 2023-06-03T20:34:07.656 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:24:25.256001+0000", 2023-06-03T20:34:07.657 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.075161934999999999 2023-06-03T20:34:07.657 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:34:07.657 INFO:teuthology.orchestra.run.smithi164.stdout: ] 2023-06-03T20:34:07.657 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:34:07.657 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:07.657 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:07.657 INFO:teuthology.orchestra.run.smithi164.stdout: "description": "osd_repop(client.36573.0:151376 13.7 e257/254 13:e920d967:::benchmark_data_smithi089_211581_object35851:head v 257'9581, mlcod=257'9578)", 2023-06-03T20:34:07.657 INFO:teuthology.orchestra.run.smithi164.stdout: "initiated_at": "2023-06-03T20:24:28.613854+0000", 2023-06-03T20:34:07.658 INFO:teuthology.orchestra.run.smithi164.stdout: "age": 578.97643063600003, 2023-06-03T20:34:07.658 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.064675888000000001, 2023-06-03T20:34:07.658 INFO:teuthology.orchestra.run.smithi164.stdout: "type_data": { 2023-06-03T20:34:07.658 INFO:teuthology.orchestra.run.smithi164.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-03T20:34:07.658 INFO:teuthology.orchestra.run.smithi164.stdout: "events": [ 2023-06-03T20:34:07.658 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:07.658 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "initiated", 2023-06-03T20:34:07.659 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:24:28.613854+0000", 2023-06-03T20:34:07.659 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0 2023-06-03T20:34:07.659 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:07.659 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:07.659 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "throttled", 2023-06-03T20:34:07.659 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:24:28.613854+0000", 2023-06-03T20:34:07.659 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 4294967295.9999995 2023-06-03T20:34:07.659 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:07.660 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:07.660 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "header_read", 2023-06-03T20:34:07.660 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:24:28.613853+0000", 2023-06-03T20:34:07.660 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 7.3830000000000003e-06 2023-06-03T20:34:07.660 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:07.660 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:07.660 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "all_read", 2023-06-03T20:34:07.660 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:24:28.613860+0000", 2023-06-03T20:34:07.661 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 1.4589999999999999e-05 2023-06-03T20:34:07.661 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:07.661 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:07.661 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "dispatched", 2023-06-03T20:34:07.661 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:24:28.613875+0000", 2023-06-03T20:34:07.661 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 6.0090000000000004e-06 2023-06-03T20:34:07.661 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:07.661 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:07.662 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "queued_for_pg", 2023-06-03T20:34:07.662 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:24:28.613881+0000", 2023-06-03T20:34:07.662 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.0079434880000000003 2023-06-03T20:34:07.662 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:07.662 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:07.662 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "reached_pg", 2023-06-03T20:34:07.662 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:24:28.621824+0000", 2023-06-03T20:34:07.663 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 2.4136999999999999e-05 2023-06-03T20:34:07.663 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:07.663 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:07.663 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "started", 2023-06-03T20:34:07.663 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:24:28.621848+0000", 2023-06-03T20:34:07.663 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.056644633 2023-06-03T20:34:07.663 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:07.663 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:07.664 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "commit_sent", 2023-06-03T20:34:07.664 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:24:28.678493+0000", 2023-06-03T20:34:07.664 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 3.6359999999999997e-05 2023-06-03T20:34:07.664 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:07.664 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:07.664 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "done", 2023-06-03T20:34:07.664 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:24:28.678529+0000", 2023-06-03T20:34:07.664 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.064675888000000001 2023-06-03T20:34:07.665 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:34:07.665 INFO:teuthology.orchestra.run.smithi164.stdout: ] 2023-06-03T20:34:07.665 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:34:07.665 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:07.665 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:07.665 INFO:teuthology.orchestra.run.smithi164.stdout: "description": "osd_repop(client.36573.0:151708 13.f e257/255 13:f1132b24:::benchmark_data_smithi089_211581_object36183:head v 257'9632, mlcod=257'9628)", 2023-06-03T20:34:07.665 INFO:teuthology.orchestra.run.smithi164.stdout: "initiated_at": "2023-06-03T20:24:29.197419+0000", 2023-06-03T20:34:07.665 INFO:teuthology.orchestra.run.smithi164.stdout: "age": 578.39286531899995, 2023-06-03T20:34:07.666 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.061092162999999998, 2023-06-03T20:34:07.666 INFO:teuthology.orchestra.run.smithi164.stdout: "type_data": { 2023-06-03T20:34:07.666 INFO:teuthology.orchestra.run.smithi164.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-03T20:34:07.666 INFO:teuthology.orchestra.run.smithi164.stdout: "events": [ 2023-06-03T20:34:07.666 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:07.666 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "initiated", 2023-06-03T20:34:07.666 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:24:29.197419+0000", 2023-06-03T20:34:07.666 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0 2023-06-03T20:34:07.667 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:07.667 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:07.667 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "throttled", 2023-06-03T20:34:07.667 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:24:29.197419+0000", 2023-06-03T20:34:07.667 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 4294967295.999999 2023-06-03T20:34:07.667 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:07.667 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:07.667 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "header_read", 2023-06-03T20:34:07.668 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:24:29.197418+0000", 2023-06-03T20:34:07.668 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 1.0485999999999999e-05 2023-06-03T20:34:07.668 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:07.668 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:07.668 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "all_read", 2023-06-03T20:34:07.668 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:24:29.197429+0000", 2023-06-03T20:34:07.668 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 1.1206000000000001e-05 2023-06-03T20:34:07.669 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:07.669 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:07.669 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "dispatched", 2023-06-03T20:34:07.669 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:24:29.197440+0000", 2023-06-03T20:34:07.669 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 5.9719999999999997e-06 2023-06-03T20:34:07.669 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:07.669 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:07.669 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "queued_for_pg", 2023-06-03T20:34:07.670 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:24:29.197446+0000", 2023-06-03T20:34:07.670 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.020514502 2023-06-03T20:34:07.670 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:07.670 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:07.670 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "reached_pg", 2023-06-03T20:34:07.670 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:24:29.217960+0000", 2023-06-03T20:34:07.670 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 3.3433000000000003e-05 2023-06-03T20:34:07.670 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:07.671 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:07.671 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "started", 2023-06-03T20:34:07.671 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:24:29.217994+0000", 2023-06-03T20:34:07.671 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.040407911999999997 2023-06-03T20:34:07.671 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:07.671 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:07.671 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "commit_sent", 2023-06-03T20:34:07.671 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:24:29.258402+0000", 2023-06-03T20:34:07.672 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.000109413 2023-06-03T20:34:07.672 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:07.672 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:07.672 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "done", 2023-06-03T20:34:07.672 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:24:29.258511+0000", 2023-06-03T20:34:07.672 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.061092162999999998 2023-06-03T20:34:07.672 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:34:07.672 INFO:teuthology.orchestra.run.smithi164.stdout: ] 2023-06-03T20:34:07.673 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:34:07.673 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:07.673 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:07.673 INFO:teuthology.orchestra.run.smithi164.stdout: "description": "osd_repop(client.36573.0:152103 13.7 e257/254 13:e651b7bd:::benchmark_data_smithi089_211581_object36578:head v 257'9624, mlcod=257'9621)", 2023-06-03T20:34:07.673 INFO:teuthology.orchestra.run.smithi164.stdout: "initiated_at": "2023-06-03T20:24:30.141771+0000", 2023-06-03T20:34:07.673 INFO:teuthology.orchestra.run.smithi164.stdout: "age": 577.44851337499995, 2023-06-03T20:34:07.673 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.061247203, 2023-06-03T20:34:07.673 INFO:teuthology.orchestra.run.smithi164.stdout: "type_data": { 2023-06-03T20:34:07.674 INFO:teuthology.orchestra.run.smithi164.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-03T20:34:07.674 INFO:teuthology.orchestra.run.smithi164.stdout: "events": [ 2023-06-03T20:34:07.674 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:07.674 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "initiated", 2023-06-03T20:34:07.674 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:24:30.141771+0000", 2023-06-03T20:34:07.674 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0 2023-06-03T20:34:07.674 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:07.674 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:07.675 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "throttled", 2023-06-03T20:34:07.675 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:24:30.141771+0000", 2023-06-03T20:34:07.675 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 4294967295.999999 2023-06-03T20:34:07.675 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:07.675 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:07.675 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "header_read", 2023-06-03T20:34:07.675 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:24:30.141770+0000", 2023-06-03T20:34:07.676 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 1.1353e-05 2023-06-03T20:34:07.676 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:07.676 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:07.676 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "all_read", 2023-06-03T20:34:07.676 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:24:30.141781+0000", 2023-06-03T20:34:07.676 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 2.5352000000000001e-05 2023-06-03T20:34:07.676 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:07.676 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:07.677 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "dispatched", 2023-06-03T20:34:07.677 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:24:30.141807+0000", 2023-06-03T20:34:07.677 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 7.2359999999999998e-06 2023-06-03T20:34:07.677 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:07.677 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:07.677 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "queued_for_pg", 2023-06-03T20:34:07.677 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:24:30.141814+0000", 2023-06-03T20:34:07.677 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.0055142580000000002 2023-06-03T20:34:07.678 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:07.678 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:07.678 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "reached_pg", 2023-06-03T20:34:07.678 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:24:30.147328+0000", 2023-06-03T20:34:07.678 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 2.6576000000000001e-05 2023-06-03T20:34:07.678 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:07.678 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:07.678 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "started", 2023-06-03T20:34:07.679 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:24:30.147355+0000", 2023-06-03T20:34:07.679 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.055632550000000003 2023-06-03T20:34:07.679 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:07.679 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:07.679 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "commit_sent", 2023-06-03T20:34:07.679 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:24:30.202987+0000", 2023-06-03T20:34:07.679 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 3.0732000000000003e-05 2023-06-03T20:34:07.680 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:07.680 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:07.680 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "done", 2023-06-03T20:34:07.680 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:24:30.203018+0000", 2023-06-03T20:34:07.680 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.061247203 2023-06-03T20:34:07.680 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:34:07.680 INFO:teuthology.orchestra.run.smithi164.stdout: ] 2023-06-03T20:34:07.680 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:34:07.681 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:07.681 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:07.681 INFO:teuthology.orchestra.run.smithi164.stdout: "description": "osd_repop(client.36573.0:152958 13.7 e257/254 13:e45a69a2:::benchmark_data_smithi089_211581_object37433:head v 257'9685, mlcod=257'9682)", 2023-06-03T20:34:07.681 INFO:teuthology.orchestra.run.smithi164.stdout: "initiated_at": "2023-06-03T20:24:31.725771+0000", 2023-06-03T20:34:07.681 INFO:teuthology.orchestra.run.smithi164.stdout: "age": 575.864513558, 2023-06-03T20:34:07.681 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.077545691999999999, 2023-06-03T20:34:07.681 INFO:teuthology.orchestra.run.smithi164.stdout: "type_data": { 2023-06-03T20:34:07.681 INFO:teuthology.orchestra.run.smithi164.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-03T20:34:07.682 INFO:teuthology.orchestra.run.smithi164.stdout: "events": [ 2023-06-03T20:34:07.682 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:07.682 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "initiated", 2023-06-03T20:34:07.682 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:24:31.725771+0000", 2023-06-03T20:34:07.682 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0 2023-06-03T20:34:07.682 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:07.682 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:07.682 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "throttled", 2023-06-03T20:34:07.683 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:24:31.725771+0000", 2023-06-03T20:34:07.683 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 4294967295.9999986 2023-06-03T20:34:07.683 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:07.683 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:07.683 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "header_read", 2023-06-03T20:34:07.683 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:24:31.725769+0000", 2023-06-03T20:34:07.683 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 1.4212999999999999e-05 2023-06-03T20:34:07.683 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:07.684 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:07.684 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "all_read", 2023-06-03T20:34:07.684 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:24:31.725784+0000", 2023-06-03T20:34:07.684 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 1.5141e-05 2023-06-03T20:34:07.684 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:07.684 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:07.684 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "dispatched", 2023-06-03T20:34:07.684 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:24:31.725799+0000", 2023-06-03T20:34:07.685 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 7.3030000000000002e-06 2023-06-03T20:34:07.685 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:07.685 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:07.685 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "queued_for_pg", 2023-06-03T20:34:07.685 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:24:31.725806+0000", 2023-06-03T20:34:07.685 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.0030421020000000001 2023-06-03T20:34:07.685 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:07.686 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:07.686 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "reached_pg", 2023-06-03T20:34:07.686 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:24:31.728848+0000", 2023-06-03T20:34:07.686 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 3.1983999999999997e-05 2023-06-03T20:34:07.686 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:07.686 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:07.686 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "started", 2023-06-03T20:34:07.686 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:24:31.728880+0000", 2023-06-03T20:34:07.687 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.074401386999999999 2023-06-03T20:34:07.687 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:07.687 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:07.687 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "commit_sent", 2023-06-03T20:34:07.687 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:24:31.803281+0000", 2023-06-03T20:34:07.687 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 3.4841000000000001e-05 2023-06-03T20:34:07.687 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:07.687 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:07.688 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "done", 2023-06-03T20:34:07.688 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:24:31.803316+0000", 2023-06-03T20:34:07.688 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.077545691999999999 2023-06-03T20:34:07.688 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:34:07.688 INFO:teuthology.orchestra.run.smithi164.stdout: ] 2023-06-03T20:34:07.688 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:34:07.688 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:07.688 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:07.689 INFO:teuthology.orchestra.run.smithi164.stdout: "description": "osd_repop(client.36573.0:152971 13.f e257/255 13:f183abf7:::benchmark_data_smithi089_211581_object37446:head v 257'9706, mlcod=257'9703)", 2023-06-03T20:34:07.689 INFO:teuthology.orchestra.run.smithi164.stdout: "initiated_at": "2023-06-03T20:24:31.732176+0000", 2023-06-03T20:34:07.689 INFO:teuthology.orchestra.run.smithi164.stdout: "age": 575.85810859599997, 2023-06-03T20:34:07.689 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.071167053999999993, 2023-06-03T20:34:07.689 INFO:teuthology.orchestra.run.smithi164.stdout: "type_data": { 2023-06-03T20:34:07.689 INFO:teuthology.orchestra.run.smithi164.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-03T20:34:07.689 INFO:teuthology.orchestra.run.smithi164.stdout: "events": [ 2023-06-03T20:34:07.689 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:07.690 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "initiated", 2023-06-03T20:34:07.690 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:24:31.732176+0000", 2023-06-03T20:34:07.690 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0 2023-06-03T20:34:07.690 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:07.690 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:07.690 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "throttled", 2023-06-03T20:34:07.690 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:24:31.732176+0000", 2023-06-03T20:34:07.690 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 4294967295.999999 2023-06-03T20:34:07.691 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:07.691 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:07.691 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "header_read", 2023-06-03T20:34:07.691 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:24:31.732175+0000", 2023-06-03T20:34:07.691 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 2.4059000000000001e-05 2023-06-03T20:34:07.691 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:07.691 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:07.691 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "all_read", 2023-06-03T20:34:07.692 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:24:31.732199+0000", 2023-06-03T20:34:07.692 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 1.4613000000000001e-05 2023-06-03T20:34:07.692 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:07.692 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:07.692 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "dispatched", 2023-06-03T20:34:07.692 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:24:31.732213+0000", 2023-06-03T20:34:07.692 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 6.3899999999999998e-06 2023-06-03T20:34:07.693 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:07.693 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:07.693 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "queued_for_pg", 2023-06-03T20:34:07.693 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:24:31.732220+0000", 2023-06-03T20:34:07.693 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.005565164 2023-06-03T20:34:07.693 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:07.693 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:07.693 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "reached_pg", 2023-06-03T20:34:07.694 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:24:31.737785+0000", 2023-06-03T20:34:07.694 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 2.3003e-05 2023-06-03T20:34:07.694 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:07.694 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:07.694 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "started", 2023-06-03T20:34:07.694 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:24:31.737808+0000", 2023-06-03T20:34:07.694 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.065511423999999999 2023-06-03T20:34:07.694 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:07.695 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:07.695 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "commit_sent", 2023-06-03T20:34:07.695 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:24:31.803319+0000", 2023-06-03T20:34:07.695 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 2.3353000000000002e-05 2023-06-03T20:34:07.695 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:07.695 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:07.695 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "done", 2023-06-03T20:34:07.695 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:24:31.803343+0000", 2023-06-03T20:34:07.696 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.071167053999999993 2023-06-03T20:34:07.696 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:34:07.696 INFO:teuthology.orchestra.run.smithi164.stdout: ] 2023-06-03T20:34:07.696 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:34:07.696 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:07.696 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:07.696 INFO:teuthology.orchestra.run.smithi164.stdout: "description": "osd_repop(client.36573.0:152974 13.f e257/255 13:fa3d1c05:::benchmark_data_smithi089_211581_object37449:head v 257'9707, mlcod=257'9703)", 2023-06-03T20:34:07.696 INFO:teuthology.orchestra.run.smithi164.stdout: "initiated_at": "2023-06-03T20:24:31.742332+0000", 2023-06-03T20:34:07.697 INFO:teuthology.orchestra.run.smithi164.stdout: "age": 575.84795217299995, 2023-06-03T20:34:07.697 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.061032139999999999, 2023-06-03T20:34:07.697 INFO:teuthology.orchestra.run.smithi164.stdout: "type_data": { 2023-06-03T20:34:07.697 INFO:teuthology.orchestra.run.smithi164.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-03T20:34:07.697 INFO:teuthology.orchestra.run.smithi164.stdout: "events": [ 2023-06-03T20:34:07.697 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:07.697 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "initiated", 2023-06-03T20:34:07.697 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:24:31.742332+0000", 2023-06-03T20:34:07.698 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0 2023-06-03T20:34:07.698 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:07.698 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:07.698 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "throttled", 2023-06-03T20:34:07.698 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:24:31.742332+0000", 2023-06-03T20:34:07.698 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 4294967295.999999 2023-06-03T20:34:07.698 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:07.698 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:07.699 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "header_read", 2023-06-03T20:34:07.699 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:24:31.742331+0000", 2023-06-03T20:34:07.699 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 1.4314999999999999e-05 2023-06-03T20:34:07.699 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:07.699 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:07.699 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "all_read", 2023-06-03T20:34:07.699 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:24:31.742345+0000", 2023-06-03T20:34:07.699 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 2.6630000000000001e-05 2023-06-03T20:34:07.700 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:07.700 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:07.700 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "dispatched", 2023-06-03T20:34:07.700 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:24:31.742372+0000", 2023-06-03T20:34:07.700 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 7.2119999999999997e-06 2023-06-03T20:34:07.700 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:07.700 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:07.701 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "queued_for_pg", 2023-06-03T20:34:07.701 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:24:31.742379+0000", 2023-06-03T20:34:07.701 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.026233309 2023-06-03T20:34:07.701 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:07.701 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:07.701 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "reached_pg", 2023-06-03T20:34:07.701 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:24:31.768613+0000", 2023-06-03T20:34:07.701 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 3.2981000000000002e-05 2023-06-03T20:34:07.702 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:07.702 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:07.702 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "started", 2023-06-03T20:34:07.702 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:24:31.768646+0000", 2023-06-03T20:34:07.702 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.034701296 2023-06-03T20:34:07.702 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:07.702 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:07.702 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "commit_sent", 2023-06-03T20:34:07.703 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:24:31.803347+0000", 2023-06-03T20:34:07.703 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 1.7237000000000001e-05 2023-06-03T20:34:07.703 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:07.703 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:07.703 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "done", 2023-06-03T20:34:07.703 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:24:31.803364+0000", 2023-06-03T20:34:07.703 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.061032139999999999 2023-06-03T20:34:07.703 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:34:07.704 INFO:teuthology.orchestra.run.smithi164.stdout: ] 2023-06-03T20:34:07.704 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:34:07.704 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:07.704 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:07.704 INFO:teuthology.orchestra.run.smithi164.stdout: "description": "osd_repop(client.36573.0:152979 13.f e257/255 13:f8731ba2:::benchmark_data_smithi089_211581_object37454:head v 257'9708, mlcod=257'9703)", 2023-06-03T20:34:07.704 INFO:teuthology.orchestra.run.smithi164.stdout: "initiated_at": "2023-06-03T20:24:31.751365+0000", 2023-06-03T20:34:07.704 INFO:teuthology.orchestra.run.smithi164.stdout: "age": 575.83891888300002, 2023-06-03T20:34:07.705 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.075673506000000001, 2023-06-03T20:34:07.705 INFO:teuthology.orchestra.run.smithi164.stdout: "type_data": { 2023-06-03T20:34:07.705 INFO:teuthology.orchestra.run.smithi164.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-03T20:34:07.705 INFO:teuthology.orchestra.run.smithi164.stdout: "events": [ 2023-06-03T20:34:07.705 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:07.705 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "initiated", 2023-06-03T20:34:07.705 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:24:31.751365+0000", 2023-06-03T20:34:07.705 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0 2023-06-03T20:34:07.706 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:07.706 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:07.706 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "throttled", 2023-06-03T20:34:07.706 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:24:31.751365+0000", 2023-06-03T20:34:07.706 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 4294967295.9999986 2023-06-03T20:34:07.706 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:07.706 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:07.706 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "header_read", 2023-06-03T20:34:07.707 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:24:31.751364+0000", 2023-06-03T20:34:07.707 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 1.3253e-05 2023-06-03T20:34:07.707 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:07.707 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:07.707 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "all_read", 2023-06-03T20:34:07.707 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:24:31.751377+0000", 2023-06-03T20:34:07.707 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 2.9513e-05 2023-06-03T20:34:07.707 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:07.708 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:07.708 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "dispatched", 2023-06-03T20:34:07.708 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:24:31.751407+0000", 2023-06-03T20:34:07.708 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 6.8129999999999998e-06 2023-06-03T20:34:07.708 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:07.708 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:07.708 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "queued_for_pg", 2023-06-03T20:34:07.708 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:24:31.751413+0000", 2023-06-03T20:34:07.709 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.022618528999999998 2023-06-03T20:34:07.709 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:07.709 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:07.709 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "reached_pg", 2023-06-03T20:34:07.709 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:24:31.774032+0000", 2023-06-03T20:34:07.709 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 2.5536000000000001e-05 2023-06-03T20:34:07.709 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:07.709 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:07.710 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "started", 2023-06-03T20:34:07.710 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:24:31.774057+0000", 2023-06-03T20:34:07.710 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.052932451999999998 2023-06-03T20:34:07.710 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:07.710 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:07.710 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "commit_sent", 2023-06-03T20:34:07.710 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:24:31.826990+0000", 2023-06-03T20:34:07.710 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 4.8875000000000002e-05 2023-06-03T20:34:07.711 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:07.711 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:07.711 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "done", 2023-06-03T20:34:07.711 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:24:31.827039+0000", 2023-06-03T20:34:07.711 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.075673506000000001 2023-06-03T20:34:07.711 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:34:07.711 INFO:teuthology.orchestra.run.smithi164.stdout: ] 2023-06-03T20:34:07.712 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:34:07.712 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:07.712 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:07.712 INFO:teuthology.orchestra.run.smithi164.stdout: "description": "osd_repop(client.36573.0:156510 13.9 e257/242 13:9ab9c7e8:::benchmark_data_smithi089_211581_object40985:head v 257'9904, mlcod=257'9901)", 2023-06-03T20:34:07.712 INFO:teuthology.orchestra.run.smithi164.stdout: "initiated_at": "2023-06-03T20:24:38.308971+0000", 2023-06-03T20:34:07.712 INFO:teuthology.orchestra.run.smithi164.stdout: "age": 569.28131349800003, 2023-06-03T20:34:07.712 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.062944448, 2023-06-03T20:34:07.712 INFO:teuthology.orchestra.run.smithi164.stdout: "type_data": { 2023-06-03T20:34:07.713 INFO:teuthology.orchestra.run.smithi164.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-03T20:34:07.713 INFO:teuthology.orchestra.run.smithi164.stdout: "events": [ 2023-06-03T20:34:07.713 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:07.713 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "initiated", 2023-06-03T20:34:07.713 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:24:38.308971+0000", 2023-06-03T20:34:07.713 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0 2023-06-03T20:34:07.713 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:07.713 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:07.714 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "throttled", 2023-06-03T20:34:07.714 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:24:38.308971+0000", 2023-06-03T20:34:07.714 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 4294967295.9999995 2023-06-03T20:34:07.714 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:07.714 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:07.714 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "header_read", 2023-06-03T20:34:07.714 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:24:38.308970+0000", 2023-06-03T20:34:07.714 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 7.1940000000000001e-06 2023-06-03T20:34:07.715 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:07.715 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:07.715 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "all_read", 2023-06-03T20:34:07.715 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:24:38.308977+0000", 2023-06-03T20:34:07.715 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 1.0047999999999999e-05 2023-06-03T20:34:07.715 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:07.715 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:07.715 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "dispatched", 2023-06-03T20:34:07.716 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:24:38.308987+0000", 2023-06-03T20:34:07.716 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 6.1210000000000003e-06 2023-06-03T20:34:07.716 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:07.716 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:07.716 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "queued_for_pg", 2023-06-03T20:34:07.716 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:24:38.308993+0000", 2023-06-03T20:34:07.716 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.003046769 2023-06-03T20:34:07.716 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:07.717 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:07.717 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "reached_pg", 2023-06-03T20:34:07.717 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:24:38.312040+0000", 2023-06-03T20:34:07.717 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 3.4465999999999998e-05 2023-06-03T20:34:07.717 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:07.717 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:07.717 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "started", 2023-06-03T20:34:07.717 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:24:38.312075+0000", 2023-06-03T20:34:07.718 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.059805166 2023-06-03T20:34:07.718 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:07.718 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:07.718 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "commit_sent", 2023-06-03T20:34:07.718 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:24:38.371880+0000", 2023-06-03T20:34:07.718 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 3.5264e-05 2023-06-03T20:34:07.718 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:07.718 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:07.719 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "done", 2023-06-03T20:34:07.719 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:24:38.371915+0000", 2023-06-03T20:34:07.719 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.062944448 2023-06-03T20:34:07.719 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:34:07.719 INFO:teuthology.orchestra.run.smithi164.stdout: ] 2023-06-03T20:34:07.719 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:34:07.719 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:07.720 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:07.720 INFO:teuthology.orchestra.run.smithi164.stdout: "description": "osd_repop(client.36573.0:172670 13.e e263/251 13:74c58e85:::benchmark_data_smithi089_211581_object57145:head v 263'10729, mlcod=246'3596)", 2023-06-03T20:34:07.720 INFO:teuthology.orchestra.run.smithi164.stdout: "initiated_at": "2023-06-03T20:25:10.179499+0000", 2023-06-03T20:34:07.720 INFO:teuthology.orchestra.run.smithi164.stdout: "age": 537.41078471699996, 2023-06-03T20:34:07.720 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.060342020000000003, 2023-06-03T20:34:07.720 INFO:teuthology.orchestra.run.smithi164.stdout: "type_data": { 2023-06-03T20:34:07.720 INFO:teuthology.orchestra.run.smithi164.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-03T20:34:07.720 INFO:teuthology.orchestra.run.smithi164.stdout: "events": [ 2023-06-03T20:34:07.721 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:07.721 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "initiated", 2023-06-03T20:34:07.721 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:25:10.179499+0000", 2023-06-03T20:34:07.721 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0 2023-06-03T20:34:07.721 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:07.721 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:07.721 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "throttled", 2023-06-03T20:34:07.721 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:25:10.179499+0000", 2023-06-03T20:34:07.722 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 4294967295.9999995 2023-06-03T20:34:07.722 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:07.722 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:07.722 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "header_read", 2023-06-03T20:34:07.722 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:25:10.179499+0000", 2023-06-03T20:34:07.722 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 5.2830000000000001e-06 2023-06-03T20:34:07.722 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:07.722 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:07.723 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "all_read", 2023-06-03T20:34:07.723 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:25:10.179504+0000", 2023-06-03T20:34:07.723 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 6.5209999999999999e-06 2023-06-03T20:34:07.723 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:07.723 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:07.723 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "dispatched", 2023-06-03T20:34:07.723 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:25:10.179511+0000", 2023-06-03T20:34:07.723 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 3.6229999999999999e-06 2023-06-03T20:34:07.724 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:07.724 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:07.724 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "queued_for_pg", 2023-06-03T20:34:07.724 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:25:10.179515+0000", 2023-06-03T20:34:07.724 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.0072800599999999997 2023-06-03T20:34:07.724 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:07.724 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:07.724 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "reached_pg", 2023-06-03T20:34:07.725 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:25:10.186795+0000", 2023-06-03T20:34:07.725 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 2.2135999999999999e-05 2023-06-03T20:34:07.725 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:07.725 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:07.725 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "started", 2023-06-03T20:34:07.725 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:25:10.186817+0000", 2023-06-03T20:34:07.725 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.052978250999999997 2023-06-03T20:34:07.726 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:07.726 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:07.726 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "commit_sent", 2023-06-03T20:34:07.726 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:25:10.239795+0000", 2023-06-03T20:34:07.726 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 4.6483000000000002e-05 2023-06-03T20:34:07.726 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:07.726 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:07.726 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "done", 2023-06-03T20:34:07.727 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:25:10.239841+0000", 2023-06-03T20:34:07.727 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.060342020000000003 2023-06-03T20:34:07.727 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:34:07.727 INFO:teuthology.orchestra.run.smithi164.stdout: ] 2023-06-03T20:34:07.727 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:34:07.727 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:07.727 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:07.727 INFO:teuthology.orchestra.run.smithi164.stdout: "description": "osd_repop(client.36573.0:184666 13.f e264/255 13:fef9730c:::benchmark_data_smithi089_211581_object69141:head v 264'11669, mlcod=264'11664)", 2023-06-03T20:34:07.728 INFO:teuthology.orchestra.run.smithi164.stdout: "initiated_at": "2023-06-03T20:25:29.263444+0000", 2023-06-03T20:34:07.728 INFO:teuthology.orchestra.run.smithi164.stdout: "age": 518.32684065199999, 2023-06-03T20:34:07.728 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.061756719000000002, 2023-06-03T20:34:07.728 INFO:teuthology.orchestra.run.smithi164.stdout: "type_data": { 2023-06-03T20:34:07.728 INFO:teuthology.orchestra.run.smithi164.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-03T20:34:07.728 INFO:teuthology.orchestra.run.smithi164.stdout: "events": [ 2023-06-03T20:34:07.728 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:07.728 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "initiated", 2023-06-03T20:34:07.729 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:25:29.263444+0000", 2023-06-03T20:34:07.729 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0 2023-06-03T20:34:07.729 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:07.729 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:07.729 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "throttled", 2023-06-03T20:34:07.729 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:25:29.263444+0000", 2023-06-03T20:34:07.729 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 4294967295.9999986 2023-06-03T20:34:07.729 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:07.730 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:07.730 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "header_read", 2023-06-03T20:34:07.730 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:25:29.263442+0000", 2023-06-03T20:34:07.730 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 1.3219e-05 2023-06-03T20:34:07.730 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:07.730 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:07.730 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "all_read", 2023-06-03T20:34:07.731 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:25:29.263455+0000", 2023-06-03T20:34:07.731 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 1.1749999999999999e-05 2023-06-03T20:34:07.731 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:07.731 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:07.731 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "dispatched", 2023-06-03T20:34:07.731 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:25:29.263467+0000", 2023-06-03T20:34:07.731 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 5.4970000000000001e-06 2023-06-03T20:34:07.731 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:07.732 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:07.732 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "queued_for_pg", 2023-06-03T20:34:07.732 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:25:29.263473+0000", 2023-06-03T20:34:07.732 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.0064721170000000003 2023-06-03T20:34:07.732 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:07.732 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:07.732 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "reached_pg", 2023-06-03T20:34:07.732 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:25:29.269945+0000", 2023-06-03T20:34:07.733 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 3.2950999999999997e-05 2023-06-03T20:34:07.733 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:07.733 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:07.733 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "started", 2023-06-03T20:34:07.733 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:25:29.269978+0000", 2023-06-03T20:34:07.733 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.055165666000000002 2023-06-03T20:34:07.733 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:07.733 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:07.734 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "commit_sent", 2023-06-03T20:34:07.734 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:25:29.325143+0000", 2023-06-03T20:34:07.734 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 5.6918e-05 2023-06-03T20:34:07.734 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:07.734 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:07.734 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "done", 2023-06-03T20:34:07.734 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:25:29.325200+0000", 2023-06-03T20:34:07.734 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.061756719000000002 2023-06-03T20:34:07.735 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:34:07.735 INFO:teuthology.orchestra.run.smithi164.stdout: ] 2023-06-03T20:34:07.735 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:34:07.735 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:07.735 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:07.735 INFO:teuthology.orchestra.run.smithi164.stdout: "description": "osd_repop(client.36573.0:218924 13.f e268/255 13:f7979af5:::benchmark_data_smithi089_211581_object103399:head v 268'13802, mlcod=268'13800)", 2023-06-03T20:34:07.735 INFO:teuthology.orchestra.run.smithi164.stdout: "initiated_at": "2023-06-03T20:26:22.644636+0000", 2023-06-03T20:34:07.736 INFO:teuthology.orchestra.run.smithi164.stdout: "age": 464.94564809899998, 2023-06-03T20:34:07.736 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.063161644000000003, 2023-06-03T20:34:07.736 INFO:teuthology.orchestra.run.smithi164.stdout: "type_data": { 2023-06-03T20:34:07.736 INFO:teuthology.orchestra.run.smithi164.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-03T20:34:07.736 INFO:teuthology.orchestra.run.smithi164.stdout: "events": [ 2023-06-03T20:34:07.736 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:07.736 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "initiated", 2023-06-03T20:34:07.736 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:26:22.644636+0000", 2023-06-03T20:34:07.737 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0 2023-06-03T20:34:07.737 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:07.737 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:07.737 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "throttled", 2023-06-03T20:34:07.737 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:26:22.644636+0000", 2023-06-03T20:34:07.737 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 4294967295.9999986 2023-06-03T20:34:07.737 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:07.737 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:07.738 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "header_read", 2023-06-03T20:34:07.738 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:26:22.644635+0000", 2023-06-03T20:34:07.738 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 1.6198e-05 2023-06-03T20:34:07.738 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:07.738 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:07.738 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "all_read", 2023-06-03T20:34:07.738 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:26:22.644651+0000", 2023-06-03T20:34:07.738 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 2.3485e-05 2023-06-03T20:34:07.739 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:07.739 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:07.739 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "dispatched", 2023-06-03T20:34:07.739 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:26:22.644674+0000", 2023-06-03T20:34:07.739 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 8.0749999999999998e-06 2023-06-03T20:34:07.739 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:07.739 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:07.739 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "queued_for_pg", 2023-06-03T20:34:07.740 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:26:22.644683+0000", 2023-06-03T20:34:07.740 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.001665311 2023-06-03T20:34:07.740 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:07.740 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:07.740 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "reached_pg", 2023-06-03T20:34:07.740 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:26:22.646348+0000", 2023-06-03T20:34:07.740 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.0080263620000000004 2023-06-03T20:34:07.740 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:07.741 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:07.741 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "started", 2023-06-03T20:34:07.741 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:26:22.654374+0000", 2023-06-03T20:34:07.741 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.053375072000000003 2023-06-03T20:34:07.741 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:07.741 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:07.741 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "commit_sent", 2023-06-03T20:34:07.742 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:26:22.707749+0000", 2023-06-03T20:34:07.742 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 4.8399999999999997e-05 2023-06-03T20:34:07.742 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:07.742 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:07.742 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "done", 2023-06-03T20:34:07.742 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:26:22.707798+0000", 2023-06-03T20:34:07.742 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.063161644000000003 2023-06-03T20:34:07.742 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:34:07.743 INFO:teuthology.orchestra.run.smithi164.stdout: ] 2023-06-03T20:34:07.743 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:34:07.743 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:34:07.743 INFO:teuthology.orchestra.run.smithi164.stdout: ] 2023-06-03T20:34:07.743 INFO:teuthology.orchestra.run.smithi164.stdout:} 2023-06-03T20:34:07.779 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:34:07 smithi089 bash[78452]: audit 1685824447.1649535 mon.c (mon.2) 699 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T20:34:07.779 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:34:07 smithi089 bash[78452]: cluster 1685824447.2596357 mgr.y (mgr.34104) 1753 : cluster 0 pgmap v1822: 73 pgs: 73 active+clean; 582 KiB data, 5.1 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:34:07.779 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:34:07 smithi089 bash[75068]: audit 1685824447.1649535 mon.c (mon.2) 699 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T20:34:07.780 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:34:07 smithi089 bash[75068]: cluster 1685824447.2596357 mgr.y (mgr.34104) 1753 : cluster 0 pgmap v1822: 73 pgs: 73 active+clean; 582 KiB data, 5.1 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:34:07.788 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:34:07 smithi164 bash[59462]: audit 1685824447.1649535 mon.c (mon.2) 699 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T20:34:07.788 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:34:07 smithi164 bash[59462]: cluster 1685824447.2596357 mgr.y (mgr.34104) 1753 : cluster 0 pgmap v1822: 73 pgs: 73 active+clean; 582 KiB data, 5.1 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:34:08.197 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/mon.a/config 2023-06-03T20:34:08.479 DEBUG:teuthology.orchestra.run.smithi164:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph daemon osd.6 dump_ops_in_flight 2023-06-03T20:34:10.585 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:34:10 smithi164 bash[59462]: cluster 1685824449.2603927 mgr.y (mgr.34104) 1754 : cluster 0 pgmap v1823: 73 pgs: 73 active+clean; 582 KiB data, 5.1 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:34:10.779 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:34:10 smithi089 bash[75068]: cluster 1685824449.2603927 mgr.y (mgr.34104) 1754 : cluster 0 pgmap v1823: 73 pgs: 73 active+clean; 582 KiB data, 5.1 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:34:10.779 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:34:10 smithi089 bash[78452]: cluster 1685824449.2603927 mgr.y (mgr.34104) 1754 : cluster 0 pgmap v1823: 73 pgs: 73 active+clean; 582 KiB data, 5.1 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:34:12.339 INFO:teuthology.orchestra.run.smithi089.stderr:reweighted osd.1 to 1 (10000) 2023-06-03T20:34:12.779 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:34:12 smithi089 bash[75068]: cluster 1685824451.261092 mgr.y (mgr.34104) 1755 : cluster 0 pgmap v1824: 73 pgs: 73 active+clean; 582 KiB data, 5.1 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:34:12.779 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:34:12 smithi089 bash[75068]: audit 1685824452.1674616 mon.c (mon.2) 700 : audit 1 from='client.? 172.21.15.89:0/117392457' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-06-03T20:34:12.779 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:34:12 smithi089 bash[75068]: audit 1685824452.1682725 mon.a (mon.0) 1711 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-06-03T20:34:12.780 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:34:12 smithi089 bash[78452]: cluster 1685824451.261092 mgr.y (mgr.34104) 1755 : cluster 0 pgmap v1824: 73 pgs: 73 active+clean; 582 KiB data, 5.1 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:34:12.780 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:34:12 smithi089 bash[78452]: audit 1685824452.1674616 mon.c (mon.2) 700 : audit 1 from='client.? 172.21.15.89:0/117392457' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-06-03T20:34:12.780 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:34:12 smithi089 bash[78452]: audit 1685824452.1682725 mon.a (mon.0) 1711 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-06-03T20:34:12.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:34:12 smithi164 bash[59462]: cluster 1685824451.261092 mgr.y (mgr.34104) 1755 : cluster 0 pgmap v1824: 73 pgs: 73 active+clean; 582 KiB data, 5.1 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:34:12.836 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:34:12 smithi164 bash[59462]: audit 1685824452.1674616 mon.c (mon.2) 700 : audit 1 from='client.? 172.21.15.89:0/117392457' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-06-03T20:34:12.836 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:34:12 smithi164 bash[59462]: audit 1685824452.1682725 mon.a (mon.0) 1711 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-06-03T20:34:13.280 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph osd primary-affinity 1 1 2023-06-03T20:34:13.297 INFO:teuthology.orchestra.run.smithi164.stderr:Inferring config /var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/mon.b/config 2023-06-03T20:34:13.529 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:34:13 smithi089 bash[78452]: cluster 1685824452.3204193 osd.6 (osd.6) 247 : cluster 3 failed to encode map e310 with expected crc 2023-06-03T20:34:13.530 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:34:13 smithi089 bash[78452]: audit 1685824452.3281124 mon.a (mon.0) 1712 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2023-06-03T20:34:13.530 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:34:13 smithi089 bash[78452]: cluster 1685824452.3282514 mon.a (mon.0) 1713 : cluster 0 osdmap e310: 8 total, 8 up, 8 in 2023-06-03T20:34:13.531 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:34:13 smithi089 bash[75068]: cluster 1685824452.3204193 osd.6 (osd.6) 247 : cluster 3 failed to encode map e310 with expected crc 2023-06-03T20:34:13.531 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:34:13 smithi089 bash[75068]: audit 1685824452.3281124 mon.a (mon.0) 1712 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2023-06-03T20:34:13.531 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:34:13 smithi089 bash[75068]: cluster 1685824452.3282514 mon.a (mon.0) 1713 : cluster 0 osdmap e310: 8 total, 8 up, 8 in 2023-06-03T20:34:13.585 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:34:13 smithi164 bash[59462]: cluster 1685824452.3204193 osd.6 (osd.6) 247 : cluster 3 failed to encode map e310 with expected crc 2023-06-03T20:34:13.586 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:34:13 smithi164 bash[59462]: audit 1685824452.3281124 mon.a (mon.0) 1712 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2023-06-03T20:34:13.586 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:34:13 smithi164 bash[59462]: cluster 1685824452.3282514 mon.a (mon.0) 1713 : cluster 0 osdmap e310: 8 total, 8 up, 8 in 2023-06-03T20:34:14.596 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:34:14 smithi164 bash[59462]: cluster 1685824453.2618887 mgr.y (mgr.34104) 1756 : cluster 0 pgmap v1826: 73 pgs: 73 active+clean; 582 KiB data, 5.1 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:34:14.779 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:34:14 smithi089 bash[78452]: cluster 1685824453.2618887 mgr.y (mgr.34104) 1756 : cluster 0 pgmap v1826: 73 pgs: 73 active+clean; 582 KiB data, 5.1 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:34:14.779 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:34:14 smithi089 bash[75068]: cluster 1685824453.2618887 mgr.y (mgr.34104) 1756 : cluster 0 pgmap v1826: 73 pgs: 73 active+clean; 582 KiB data, 5.1 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:34:16.529 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:34:16 smithi089 bash[75068]: cluster 1685824455.2627633 mgr.y (mgr.34104) 1757 : cluster 0 pgmap v1827: 73 pgs: 73 active+clean; 582 KiB data, 5.1 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:34:16.530 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 20:34:16 smithi089 bash[71163]: ::ffff:172.21.15.164 - - [03/Jun/2023:20:34:16] "GET /metrics HTTP/1.1" 200 226944 "" "Prometheus/2.33.4" 2023-06-03T20:34:16.530 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:34:16 smithi089 bash[78452]: cluster 1685824455.2627633 mgr.y (mgr.34104) 1757 : cluster 0 pgmap v1827: 73 pgs: 73 active+clean; 582 KiB data, 5.1 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:34:16.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:34:16 smithi164 bash[59462]: cluster 1685824455.2627633 mgr.y (mgr.34104) 1757 : cluster 0 pgmap v1827: 73 pgs: 73 active+clean; 582 KiB data, 5.1 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:34:17.084 INFO:teuthology.orchestra.run.smithi164.stdout:{ 2023-06-03T20:34:17.084 INFO:teuthology.orchestra.run.smithi164.stdout: "ops": [], 2023-06-03T20:34:17.084 INFO:teuthology.orchestra.run.smithi164.stdout: "num_ops": 0 2023-06-03T20:34:17.085 INFO:teuthology.orchestra.run.smithi164.stdout:} 2023-06-03T20:34:18.017 DEBUG:teuthology.orchestra.run.smithi164:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph daemon osd.6 dump_blocked_ops 2023-06-03T20:34:18.099 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/mon.a/config 2023-06-03T20:34:18.530 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:34:18 smithi089 bash[78452]: cluster 1685824457.263663 mgr.y (mgr.34104) 1758 : cluster 0 pgmap v1828: 73 pgs: 73 active+clean; 582 KiB data, 5.1 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:34:18.531 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:34:18 smithi089 bash[75068]: cluster 1685824457.263663 mgr.y (mgr.34104) 1758 : cluster 0 pgmap v1828: 73 pgs: 73 active+clean; 582 KiB data, 5.1 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:34:18.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:34:18 smithi164 bash[59462]: cluster 1685824457.263663 mgr.y (mgr.34104) 1758 : cluster 0 pgmap v1828: 73 pgs: 73 active+clean; 582 KiB data, 5.1 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:34:20.732 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:34:20 smithi089 bash[75068]: cluster 1685824459.2646728 mgr.y (mgr.34104) 1759 : cluster 0 pgmap v1829: 73 pgs: 73 active+clean; 582 KiB data, 5.1 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:34:20.732 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:34:20 smithi089 bash[78452]: cluster 1685824459.2646728 mgr.y (mgr.34104) 1759 : cluster 0 pgmap v1829: 73 pgs: 73 active+clean; 582 KiB data, 5.1 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:34:20.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:34:20 smithi164 bash[59462]: cluster 1685824459.2646728 mgr.y (mgr.34104) 1759 : cluster 0 pgmap v1829: 73 pgs: 73 active+clean; 582 KiB data, 5.1 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:34:22.779 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:34:22 smithi089 bash[75068]: cluster 1685824461.2654145 mgr.y (mgr.34104) 1760 : cluster 0 pgmap v1830: 73 pgs: 73 active+clean; 582 KiB data, 5.1 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:34:22.779 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:34:22 smithi089 bash[75068]: audit 1685824462.1641061 mon.c (mon.2) 701 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T20:34:22.780 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:34:22 smithi089 bash[78452]: cluster 1685824461.2654145 mgr.y (mgr.34104) 1760 : cluster 0 pgmap v1830: 73 pgs: 73 active+clean; 582 KiB data, 5.1 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:34:22.780 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:34:22 smithi089 bash[78452]: audit 1685824462.1641061 mon.c (mon.2) 701 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T20:34:22.834 INFO:teuthology.orchestra.run.smithi164.stderr:Inferring config /var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/mon.b/config 2023-06-03T20:34:22.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:34:22 smithi164 bash[59462]: cluster 1685824461.2654145 mgr.y (mgr.34104) 1760 : cluster 0 pgmap v1830: 73 pgs: 73 active+clean; 582 KiB data, 5.1 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:34:22.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:34:22 smithi164 bash[59462]: audit 1685824462.1641061 mon.c (mon.2) 701 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T20:34:23.379 INFO:teuthology.orchestra.run.smithi089.stderr:set osd.1 primary-affinity to 1 (8655362) 2023-06-03T20:34:23.779 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:34:23 smithi089 bash[75068]: audit 1685824462.4903529 mon.c (mon.2) 702 : audit 1 from='client.? 172.21.15.89:0/3425763898' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 1, "weight": 1.0}]: dispatch 2023-06-03T20:34:23.779 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:34:23 smithi089 bash[75068]: audit 1685824462.4910378 mon.a (mon.0) 1714 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 1, "weight": 1.0}]: dispatch 2023-06-03T20:34:23.780 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:34:23 smithi089 bash[78452]: audit 1685824462.4903529 mon.c (mon.2) 702 : audit 1 from='client.? 172.21.15.89:0/3425763898' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 1, "weight": 1.0}]: dispatch 2023-06-03T20:34:23.780 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:34:23 smithi089 bash[78452]: audit 1685824462.4910378 mon.a (mon.0) 1714 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 1, "weight": 1.0}]: dispatch 2023-06-03T20:34:23.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:34:23 smithi164 bash[59462]: audit 1685824462.4903529 mon.c (mon.2) 702 : audit 1 from='client.? 172.21.15.89:0/3425763898' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 1, "weight": 1.0}]: dispatch 2023-06-03T20:34:23.836 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:34:23 smithi164 bash[59462]: audit 1685824462.4910378 mon.a (mon.0) 1714 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 1, "weight": 1.0}]: dispatch 2023-06-03T20:34:24.369 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph osd reweight 2 1 2023-06-03T20:34:24.552 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:34:24 smithi164 bash[59462]: cluster 1685824463.2663767 mgr.y (mgr.34104) 1761 : cluster 0 pgmap v1831: 73 pgs: 73 active+clean; 582 KiB data, 5.1 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:34:24.553 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:34:24 smithi164 bash[59462]: audit 1685824463.3675256 mon.a (mon.0) 1715 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 1, "weight": 1.0}]': finished 2023-06-03T20:34:24.553 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:34:24 smithi164 bash[59462]: cluster 1685824463.367655 mon.a (mon.0) 1716 : cluster 0 osdmap e311: 8 total, 8 up, 8 in 2023-06-03T20:34:24.553 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:34:24 smithi164 bash[59462]: cluster 1685824463.496108 osd.7 (osd.7) 255 : cluster 3 failed to encode map e311 with expected crc 2023-06-03T20:34:24.553 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:34:24 smithi164 bash[59462]: cluster 1685824463.6385832 osd.5 (osd.5) 258 : cluster 3 failed to encode map e311 with expected crc 2023-06-03T20:34:24.553 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:34:24 smithi164 bash[59462]: cluster 1685824463.6386628 osd.6 (osd.6) 248 : cluster 3 failed to encode map e311 with expected crc 2023-06-03T20:34:24.779 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:34:24 smithi089 bash[78452]: cluster 1685824463.2663767 mgr.y (mgr.34104) 1761 : cluster 0 pgmap v1831: 73 pgs: 73 active+clean; 582 KiB data, 5.1 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:34:24.779 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:34:24 smithi089 bash[78452]: audit 1685824463.3675256 mon.a (mon.0) 1715 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 1, "weight": 1.0}]': finished 2023-06-03T20:34:24.780 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:34:24 smithi089 bash[78452]: cluster 1685824463.367655 mon.a (mon.0) 1716 : cluster 0 osdmap e311: 8 total, 8 up, 8 in 2023-06-03T20:34:24.780 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:34:24 smithi089 bash[78452]: cluster 1685824463.496108 osd.7 (osd.7) 255 : cluster 3 failed to encode map e311 with expected crc 2023-06-03T20:34:24.780 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:34:24 smithi089 bash[78452]: cluster 1685824463.6385832 osd.5 (osd.5) 258 : cluster 3 failed to encode map e311 with expected crc 2023-06-03T20:34:24.780 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:34:24 smithi089 bash[78452]: cluster 1685824463.6386628 osd.6 (osd.6) 248 : cluster 3 failed to encode map e311 with expected crc 2023-06-03T20:34:24.780 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:34:24 smithi089 bash[75068]: cluster 1685824463.2663767 mgr.y (mgr.34104) 1761 : cluster 0 pgmap v1831: 73 pgs: 73 active+clean; 582 KiB data, 5.1 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:34:24.781 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:34:24 smithi089 bash[75068]: audit 1685824463.3675256 mon.a (mon.0) 1715 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 1, "weight": 1.0}]': finished 2023-06-03T20:34:24.781 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:34:24 smithi089 bash[75068]: cluster 1685824463.367655 mon.a (mon.0) 1716 : cluster 0 osdmap e311: 8 total, 8 up, 8 in 2023-06-03T20:34:24.781 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:34:24 smithi089 bash[75068]: cluster 1685824463.496108 osd.7 (osd.7) 255 : cluster 3 failed to encode map e311 with expected crc 2023-06-03T20:34:24.781 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:34:24 smithi089 bash[75068]: cluster 1685824463.6385832 osd.5 (osd.5) 258 : cluster 3 failed to encode map e311 with expected crc 2023-06-03T20:34:24.781 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:34:24 smithi089 bash[75068]: cluster 1685824463.6386628 osd.6 (osd.6) 248 : cluster 3 failed to encode map e311 with expected crc 2023-06-03T20:34:26.529 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:34:26 smithi089 bash[75068]: cluster 1685824465.2671561 mgr.y (mgr.34104) 1762 : cluster 0 pgmap v1833: 73 pgs: 73 active+clean; 582 KiB data, 5.1 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:34:26.529 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 20:34:26 smithi089 bash[71163]: ::ffff:172.21.15.164 - - [03/Jun/2023:20:34:26] "GET /metrics HTTP/1.1" 200 226951 "" "Prometheus/2.33.4" 2023-06-03T20:34:26.530 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:34:26 smithi089 bash[78452]: cluster 1685824465.2671561 mgr.y (mgr.34104) 1762 : cluster 0 pgmap v1833: 73 pgs: 73 active+clean; 582 KiB data, 5.1 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:34:26.629 INFO:teuthology.orchestra.run.smithi164.stdout:{ 2023-06-03T20:34:26.629 INFO:teuthology.orchestra.run.smithi164.stdout: "ops": [], 2023-06-03T20:34:26.629 INFO:teuthology.orchestra.run.smithi164.stdout: "complaint_time": 30, 2023-06-03T20:34:26.629 INFO:teuthology.orchestra.run.smithi164.stdout: "num_blocked_ops": 0 2023-06-03T20:34:26.630 INFO:teuthology.orchestra.run.smithi164.stdout:} 2023-06-03T20:34:26.655 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:34:26 smithi164 bash[59462]: cluster 1685824465.2671561 mgr.y (mgr.34104) 1762 : cluster 0 pgmap v1833: 73 pgs: 73 active+clean; 582 KiB data, 5.1 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:34:27.525 DEBUG:teuthology.orchestra.run.smithi164:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph daemon osd.6 dump_historic_ops 2023-06-03T20:34:28.779 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:34:28 smithi089 bash[75068]: cluster 1685824467.267978 mgr.y (mgr.34104) 1763 : cluster 0 pgmap v1834: 73 pgs: 73 active+clean; 582 KiB data, 5.1 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:34:28.779 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:34:28 smithi089 bash[78452]: cluster 1685824467.267978 mgr.y (mgr.34104) 1763 : cluster 0 pgmap v1834: 73 pgs: 73 active+clean; 582 KiB data, 5.1 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:34:28.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:34:28 smithi164 bash[59462]: cluster 1685824467.267978 mgr.y (mgr.34104) 1763 : cluster 0 pgmap v1834: 73 pgs: 73 active+clean; 582 KiB data, 5.1 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:34:29.182 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/mon.a/config 2023-06-03T20:34:30.667 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:34:30 smithi089 bash[75068]: cluster 1685824469.2690225 mgr.y (mgr.34104) 1764 : cluster 0 pgmap v1835: 73 pgs: 73 active+clean; 582 KiB data, 5.1 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:34:30.668 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:34:30 smithi089 bash[78452]: cluster 1685824469.2690225 mgr.y (mgr.34104) 1764 : cluster 0 pgmap v1835: 73 pgs: 73 active+clean; 582 KiB data, 5.1 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:34:30.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:34:30 smithi164 bash[59462]: cluster 1685824469.2690225 mgr.y (mgr.34104) 1764 : cluster 0 pgmap v1835: 73 pgs: 73 active+clean; 582 KiB data, 5.1 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:34:32.336 INFO:teuthology.orchestra.run.smithi164.stderr:Inferring config /var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/mon.b/config 2023-06-03T20:34:32.779 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:34:32 smithi089 bash[75068]: cluster 1685824471.2696745 mgr.y (mgr.34104) 1765 : cluster 0 pgmap v1836: 73 pgs: 73 active+clean; 582 KiB data, 5.1 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:34:32.779 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:34:32 smithi089 bash[78452]: cluster 1685824471.2696745 mgr.y (mgr.34104) 1765 : cluster 0 pgmap v1836: 73 pgs: 73 active+clean; 582 KiB data, 5.1 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:34:32.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:34:32 smithi164 bash[59462]: cluster 1685824471.2696745 mgr.y (mgr.34104) 1765 : cluster 0 pgmap v1836: 73 pgs: 73 active+clean; 582 KiB data, 5.1 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:34:34.175 INFO:teuthology.orchestra.run.smithi089.stderr:reweighted osd.2 to 1 (10000) 2023-06-03T20:34:34.335 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:34:34 smithi164 bash[59462]: audit 1685824473.243408 mon.c (mon.2) 703 : audit 1 from='client.? 172.21.15.89:0/2427725966' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-06-03T20:34:34.336 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:34:34 smithi164 bash[59462]: audit 1685824473.244219 mon.a (mon.0) 1717 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-06-03T20:34:34.336 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:34:34 smithi164 bash[59462]: cluster 1685824473.2706032 mgr.y (mgr.34104) 1766 : cluster 0 pgmap v1837: 73 pgs: 73 active+clean; 582 KiB data, 5.1 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:34:34.336 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:34:34 smithi164 bash[59462]: audit 1685824473.4682102 mon.c (mon.2) 704 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-03T20:34:34.529 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:34:34 smithi089 bash[78452]: audit 1685824473.243408 mon.c (mon.2) 703 : audit 1 from='client.? 172.21.15.89:0/2427725966' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-06-03T20:34:34.529 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:34:34 smithi089 bash[78452]: audit 1685824473.244219 mon.a (mon.0) 1717 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-06-03T20:34:34.530 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:34:34 smithi089 bash[78452]: cluster 1685824473.2706032 mgr.y (mgr.34104) 1766 : cluster 0 pgmap v1837: 73 pgs: 73 active+clean; 582 KiB data, 5.1 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:34:34.530 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:34:34 smithi089 bash[78452]: audit 1685824473.4682102 mon.c (mon.2) 704 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-03T20:34:34.530 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:34:34 smithi089 bash[75068]: audit 1685824473.243408 mon.c (mon.2) 703 : audit 1 from='client.? 172.21.15.89:0/2427725966' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-06-03T20:34:34.530 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:34:34 smithi089 bash[75068]: audit 1685824473.244219 mon.a (mon.0) 1717 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-06-03T20:34:34.530 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:34:34 smithi089 bash[75068]: cluster 1685824473.2706032 mgr.y (mgr.34104) 1766 : cluster 0 pgmap v1837: 73 pgs: 73 active+clean; 582 KiB data, 5.1 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:34:34.531 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:34:34 smithi089 bash[75068]: audit 1685824473.4682102 mon.c (mon.2) 704 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-03T20:34:35.209 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph osd primary-affinity 2 1 2023-06-03T20:34:35.529 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:34:35 smithi089 bash[78452]: cluster 1685824474.1537988 osd.6 (osd.6) 249 : cluster 3 failed to encode map e312 with expected crc 2023-06-03T20:34:35.530 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:34:35 smithi089 bash[78452]: audit 1685824474.1631725 mon.a (mon.0) 1718 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2023-06-03T20:34:35.530 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:34:35 smithi089 bash[78452]: cluster 1685824474.1632478 mon.a (mon.0) 1719 : cluster 0 osdmap e312: 8 total, 8 up, 8 in 2023-06-03T20:34:35.530 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:34:35 smithi089 bash[75068]: cluster 1685824474.1537988 osd.6 (osd.6) 249 : cluster 3 failed to encode map e312 with expected crc 2023-06-03T20:34:35.530 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:34:35 smithi089 bash[75068]: audit 1685824474.1631725 mon.a (mon.0) 1718 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2023-06-03T20:34:35.530 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:34:35 smithi089 bash[75068]: cluster 1685824474.1632478 mon.a (mon.0) 1719 : cluster 0 osdmap e312: 8 total, 8 up, 8 in 2023-06-03T20:34:35.585 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:34:35 smithi164 bash[59462]: cluster 1685824474.1537988 osd.6 (osd.6) 249 : cluster 3 failed to encode map e312 with expected crc 2023-06-03T20:34:35.586 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:34:35 smithi164 bash[59462]: audit 1685824474.1631725 mon.a (mon.0) 1718 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2023-06-03T20:34:35.586 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:34:35 smithi164 bash[59462]: cluster 1685824474.1632478 mon.a (mon.0) 1719 : cluster 0 osdmap e312: 8 total, 8 up, 8 in 2023-06-03T20:34:36.260 INFO:teuthology.orchestra.run.smithi164.stdout:{ 2023-06-03T20:34:36.260 INFO:teuthology.orchestra.run.smithi164.stdout: "size": 20, 2023-06-03T20:34:36.260 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 600, 2023-06-03T20:34:36.261 INFO:teuthology.orchestra.run.smithi164.stdout: "ops": [ 2023-06-03T20:34:36.261 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:36.261 INFO:teuthology.orchestra.run.smithi164.stdout: "description": "osd_op(client.46739.0:138097 14.2 14:41c0cb6a:::benchmark_data_smithi089_237281_object40081:head [delete] snapc 0=[] ondisk+write+known_if_redirected e291)", 2023-06-03T20:34:36.261 INFO:teuthology.orchestra.run.smithi164.stdout: "initiated_at": "2023-06-03T20:30:18.887317+0000", 2023-06-03T20:34:36.261 INFO:teuthology.orchestra.run.smithi164.stdout: "age": 257.35838719999998, 2023-06-03T20:34:36.261 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.18266654900000001, 2023-06-03T20:34:36.261 INFO:teuthology.orchestra.run.smithi164.stdout: "type_data": { 2023-06-03T20:34:36.261 INFO:teuthology.orchestra.run.smithi164.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-03T20:34:36.262 INFO:teuthology.orchestra.run.smithi164.stdout: "client_info": { 2023-06-03T20:34:36.262 INFO:teuthology.orchestra.run.smithi164.stdout: "client": "client.46739", 2023-06-03T20:34:36.262 INFO:teuthology.orchestra.run.smithi164.stdout: "client_addr": "172.21.15.89:0/2479974613", 2023-06-03T20:34:36.262 INFO:teuthology.orchestra.run.smithi164.stdout: "tid": 138097 2023-06-03T20:34:36.262 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:36.262 INFO:teuthology.orchestra.run.smithi164.stdout: "events": [ 2023-06-03T20:34:36.263 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:36.263 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "initiated", 2023-06-03T20:34:36.263 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:30:18.887317+0000", 2023-06-03T20:34:36.264 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0 2023-06-03T20:34:36.264 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:36.264 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:36.264 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "throttled", 2023-06-03T20:34:36.265 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:30:18.887317+0000", 2023-06-03T20:34:36.265 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 4294967295.9999981 2023-06-03T20:34:36.265 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:36.266 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:36.266 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "header_read", 2023-06-03T20:34:36.266 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:30:18.887316+0000", 2023-06-03T20:34:36.266 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 8.8089999999999999e-06 2023-06-03T20:34:36.267 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:36.267 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:36.267 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "all_read", 2023-06-03T20:34:36.268 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:30:18.887325+0000", 2023-06-03T20:34:36.268 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 1.3913999999999999e-05 2023-06-03T20:34:36.268 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:36.269 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:36.269 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "dispatched", 2023-06-03T20:34:36.269 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:30:18.887338+0000", 2023-06-03T20:34:36.270 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 7.0049999999999998e-06 2023-06-03T20:34:36.270 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:36.270 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:36.271 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "queued_for_pg", 2023-06-03T20:34:36.271 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:30:18.887345+0000", 2023-06-03T20:34:36.271 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 4.3037000000000001e-05 2023-06-03T20:34:36.271 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:36.271 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:36.272 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "reached_pg", 2023-06-03T20:34:36.272 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:30:18.887388+0000", 2023-06-03T20:34:36.272 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 9.2540000000000005e-05 2023-06-03T20:34:36.272 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:36.273 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:36.273 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "started", 2023-06-03T20:34:36.273 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:30:18.887481+0000", 2023-06-03T20:34:36.273 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 8.4548000000000004e-05 2023-06-03T20:34:36.274 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:36.274 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:36.274 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "waiting for subops from 1,3", 2023-06-03T20:34:36.275 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:30:18.887566+0000", 2023-06-03T20:34:36.275 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.0039463010000000001 2023-06-03T20:34:36.275 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:36.275 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:36.275 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "op_commit", 2023-06-03T20:34:36.275 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:30:18.891512+0000", 2023-06-03T20:34:36.275 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.0045368819999999999 2023-06-03T20:34:36.276 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:36.276 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:36.276 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:34:36.276 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:30:18.896049+0000", 2023-06-03T20:34:36.276 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.17382581499999999 2023-06-03T20:34:36.276 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:36.277 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:36.277 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:34:36.277 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:30:19.069875+0000", 2023-06-03T20:34:36.277 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 5.5167999999999998e-05 2023-06-03T20:34:36.277 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:36.277 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:36.277 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "commit_sent", 2023-06-03T20:34:36.277 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:30:19.069930+0000", 2023-06-03T20:34:36.278 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 5.4200999999999998e-05 2023-06-03T20:34:36.278 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:36.278 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:36.278 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "done", 2023-06-03T20:34:36.278 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:30:19.069984+0000", 2023-06-03T20:34:36.278 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.18266654900000001 2023-06-03T20:34:36.278 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:34:36.279 INFO:teuthology.orchestra.run.smithi164.stdout: ] 2023-06-03T20:34:36.279 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:34:36.279 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:36.279 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:36.279 INFO:teuthology.orchestra.run.smithi164.stdout: "description": "osd_op(client.46739.0:171079 14.2 14:4757bd47:::benchmark_data_smithi089_237281_object73063:head [delete] snapc 0=[] ondisk+write+known_if_redirected e292)", 2023-06-03T20:34:36.279 INFO:teuthology.orchestra.run.smithi164.stdout: "initiated_at": "2023-06-03T20:31:15.732550+0000", 2023-06-03T20:34:36.279 INFO:teuthology.orchestra.run.smithi164.stdout: "age": 200.51315502599999, 2023-06-03T20:34:36.280 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.208597644, 2023-06-03T20:34:36.280 INFO:teuthology.orchestra.run.smithi164.stdout: "type_data": { 2023-06-03T20:34:36.280 INFO:teuthology.orchestra.run.smithi164.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-03T20:34:36.280 INFO:teuthology.orchestra.run.smithi164.stdout: "client_info": { 2023-06-03T20:34:36.280 INFO:teuthology.orchestra.run.smithi164.stdout: "client": "client.46739", 2023-06-03T20:34:36.280 INFO:teuthology.orchestra.run.smithi164.stdout: "client_addr": "172.21.15.89:0/2479974613", 2023-06-03T20:34:36.280 INFO:teuthology.orchestra.run.smithi164.stdout: "tid": 171079 2023-06-03T20:34:36.281 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:36.281 INFO:teuthology.orchestra.run.smithi164.stdout: "events": [ 2023-06-03T20:34:36.281 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:36.281 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "initiated", 2023-06-03T20:34:36.281 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:31:15.732550+0000", 2023-06-03T20:34:36.281 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0 2023-06-03T20:34:36.281 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:36.282 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:36.282 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "throttled", 2023-06-03T20:34:36.282 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:31:15.732550+0000", 2023-06-03T20:34:36.282 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 4294967295.9999981 2023-06-03T20:34:36.282 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:36.282 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:36.282 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "header_read", 2023-06-03T20:34:36.283 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:31:15.732548+0000", 2023-06-03T20:34:36.283 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 1.4596000000000001e-05 2023-06-03T20:34:36.283 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:36.283 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:36.283 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "all_read", 2023-06-03T20:34:36.283 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:31:15.732562+0000", 2023-06-03T20:34:36.283 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 1.9743000000000001e-05 2023-06-03T20:34:36.283 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:36.284 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:36.284 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "dispatched", 2023-06-03T20:34:36.284 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:31:15.732582+0000", 2023-06-03T20:34:36.284 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 9.9979999999999998e-06 2023-06-03T20:34:36.284 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:36.284 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:36.284 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "queued_for_pg", 2023-06-03T20:34:36.285 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:31:15.732592+0000", 2023-06-03T20:34:36.285 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 6.1217999999999996e-05 2023-06-03T20:34:36.285 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:36.285 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:36.285 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "reached_pg", 2023-06-03T20:34:36.285 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:31:15.732653+0000", 2023-06-03T20:34:36.285 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.00016114699999999999 2023-06-03T20:34:36.286 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:36.286 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:36.286 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "started", 2023-06-03T20:34:36.286 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:31:15.732814+0000", 2023-06-03T20:34:36.286 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.00018993999999999999 2023-06-03T20:34:36.286 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:36.286 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:36.287 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "waiting for subops from 1,3", 2023-06-03T20:34:36.287 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:31:15.733004+0000", 2023-06-03T20:34:36.287 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.006227771 2023-06-03T20:34:36.287 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:36.287 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:36.287 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "op_commit", 2023-06-03T20:34:36.287 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:31:15.739232+0000", 2023-06-03T20:34:36.288 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.003143065 2023-06-03T20:34:36.288 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:36.288 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:36.288 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:34:36.288 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:31:15.742375+0000", 2023-06-03T20:34:36.288 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.19866339699999999 2023-06-03T20:34:36.288 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:36.289 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:36.289 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:34:36.289 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:31:15.941038+0000", 2023-06-03T20:34:36.289 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 5.8767999999999997e-05 2023-06-03T20:34:36.289 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:36.289 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:36.290 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "commit_sent", 2023-06-03T20:34:36.290 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:31:15.941097+0000", 2023-06-03T20:34:36.290 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 5.0003000000000002e-05 2023-06-03T20:34:36.290 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:36.290 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:36.290 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "done", 2023-06-03T20:34:36.290 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:31:15.941147+0000", 2023-06-03T20:34:36.290 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.208597644 2023-06-03T20:34:36.291 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:34:36.291 INFO:teuthology.orchestra.run.smithi164.stdout: ] 2023-06-03T20:34:36.291 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:34:36.291 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:36.291 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:36.291 INFO:teuthology.orchestra.run.smithi164.stdout: "description": "osd_op(client.46739.0:171084 14.4 14:20549f97:::benchmark_data_smithi089_237281_object73068:head [delete] snapc 0=[] ondisk+write+known_if_redirected e292)", 2023-06-03T20:34:36.291 INFO:teuthology.orchestra.run.smithi164.stdout: "initiated_at": "2023-06-03T20:31:15.734087+0000", 2023-06-03T20:34:36.291 INFO:teuthology.orchestra.run.smithi164.stdout: "age": 200.51161791999999, 2023-06-03T20:34:36.292 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.206932172, 2023-06-03T20:34:36.292 INFO:teuthology.orchestra.run.smithi164.stdout: "type_data": { 2023-06-03T20:34:36.292 INFO:teuthology.orchestra.run.smithi164.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-03T20:34:36.292 INFO:teuthology.orchestra.run.smithi164.stdout: "client_info": { 2023-06-03T20:34:36.292 INFO:teuthology.orchestra.run.smithi164.stdout: "client": "client.46739", 2023-06-03T20:34:36.292 INFO:teuthology.orchestra.run.smithi164.stdout: "client_addr": "172.21.15.89:0/2479974613", 2023-06-03T20:34:36.292 INFO:teuthology.orchestra.run.smithi164.stdout: "tid": 171084 2023-06-03T20:34:36.293 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:36.293 INFO:teuthology.orchestra.run.smithi164.stdout: "events": [ 2023-06-03T20:34:36.293 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:36.293 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "initiated", 2023-06-03T20:34:36.293 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:31:15.734087+0000", 2023-06-03T20:34:36.293 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0 2023-06-03T20:34:36.294 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:36.294 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:36.294 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "throttled", 2023-06-03T20:34:36.294 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:31:15.734087+0000", 2023-06-03T20:34:36.294 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 4294967295.999999 2023-06-03T20:34:36.294 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:36.294 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:36.295 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "header_read", 2023-06-03T20:34:36.295 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:31:15.734086+0000", 2023-06-03T20:34:36.295 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 6.635e-06 2023-06-03T20:34:36.295 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:36.295 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:36.295 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "all_read", 2023-06-03T20:34:36.295 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:31:15.734092+0000", 2023-06-03T20:34:36.295 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 1.1002000000000001e-05 2023-06-03T20:34:36.296 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:36.296 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:36.296 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "dispatched", 2023-06-03T20:34:36.296 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:31:15.734103+0000", 2023-06-03T20:34:36.296 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 6.8020000000000003e-06 2023-06-03T20:34:36.296 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:36.296 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:36.297 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "queued_for_pg", 2023-06-03T20:34:36.297 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:31:15.734110+0000", 2023-06-03T20:34:36.297 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.000107812 2023-06-03T20:34:36.297 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:36.297 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:36.297 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "reached_pg", 2023-06-03T20:34:36.297 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:31:15.734218+0000", 2023-06-03T20:34:36.298 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.000283944 2023-06-03T20:34:36.298 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:36.298 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:36.298 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "started", 2023-06-03T20:34:36.298 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:31:15.734502+0000", 2023-06-03T20:34:36.298 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.00032039699999999999 2023-06-03T20:34:36.298 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:36.298 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:36.299 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "waiting for subops from 3,7", 2023-06-03T20:34:36.299 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:31:15.734822+0000", 2023-06-03T20:34:36.299 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.0077447499999999999 2023-06-03T20:34:36.299 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:36.299 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:36.300 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "op_commit", 2023-06-03T20:34:36.300 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:31:15.742567+0000", 2023-06-03T20:34:36.300 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.0083015439999999992 2023-06-03T20:34:36.300 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:36.300 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:36.301 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:34:36.301 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:31:15.750868+0000", 2023-06-03T20:34:36.301 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.190014405 2023-06-03T20:34:36.301 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:36.301 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:36.301 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:34:36.301 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:31:15.940883+0000", 2023-06-03T20:34:36.302 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 7.5286999999999999e-05 2023-06-03T20:34:36.302 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:36.302 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:36.302 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "commit_sent", 2023-06-03T20:34:36.302 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:31:15.940958+0000", 2023-06-03T20:34:36.302 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 6.0649000000000002e-05 2023-06-03T20:34:36.302 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:36.303 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:36.303 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "done", 2023-06-03T20:34:36.303 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:31:15.941019+0000", 2023-06-03T20:34:36.303 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.206932172 2023-06-03T20:34:36.304 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:34:36.304 INFO:teuthology.orchestra.run.smithi164.stdout: ] 2023-06-03T20:34:36.304 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:34:36.305 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:36.305 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:36.305 INFO:teuthology.orchestra.run.smithi164.stdout: "description": "osd_op(client.46739.0:177879 14.2 14:47f7b10b:::benchmark_data_smithi089_237281_object79863:head [delete] snapc 0=[] ondisk+write+known_if_redirected e293)", 2023-06-03T20:34:36.305 INFO:teuthology.orchestra.run.smithi164.stdout: "initiated_at": "2023-06-03T20:31:27.895287+0000", 2023-06-03T20:34:36.305 INFO:teuthology.orchestra.run.smithi164.stdout: "age": 188.35041787, 2023-06-03T20:34:36.305 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.16922973299999999, 2023-06-03T20:34:36.305 INFO:teuthology.orchestra.run.smithi164.stdout: "type_data": { 2023-06-03T20:34:36.306 INFO:teuthology.orchestra.run.smithi164.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-03T20:34:36.306 INFO:teuthology.orchestra.run.smithi164.stdout: "client_info": { 2023-06-03T20:34:36.306 INFO:teuthology.orchestra.run.smithi164.stdout: "client": "client.46739", 2023-06-03T20:34:36.306 INFO:teuthology.orchestra.run.smithi164.stdout: "client_addr": "172.21.15.89:0/2479974613", 2023-06-03T20:34:36.306 INFO:teuthology.orchestra.run.smithi164.stdout: "tid": 177879 2023-06-03T20:34:36.306 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:36.306 INFO:teuthology.orchestra.run.smithi164.stdout: "events": [ 2023-06-03T20:34:36.306 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:36.307 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "initiated", 2023-06-03T20:34:36.307 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:31:27.895287+0000", 2023-06-03T20:34:36.307 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0 2023-06-03T20:34:36.307 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:36.307 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:36.307 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "throttled", 2023-06-03T20:34:36.307 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:31:27.895287+0000", 2023-06-03T20:34:36.308 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 4294967295.9999981 2023-06-03T20:34:36.308 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:36.308 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:36.308 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "header_read", 2023-06-03T20:34:36.308 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:31:27.895285+0000", 2023-06-03T20:34:36.308 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 9.4190000000000008e-06 2023-06-03T20:34:36.308 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:36.309 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:36.309 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "all_read", 2023-06-03T20:34:36.309 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:31:27.895294+0000", 2023-06-03T20:34:36.309 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 1.9334000000000002e-05 2023-06-03T20:34:36.309 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:36.309 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:36.309 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "dispatched", 2023-06-03T20:34:36.309 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:31:27.895313+0000", 2023-06-03T20:34:36.310 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 1.7567999999999998e-05 2023-06-03T20:34:36.310 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:36.310 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:36.310 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "queued_for_pg", 2023-06-03T20:34:36.310 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:31:27.895331+0000", 2023-06-03T20:34:36.310 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 7.8617999999999998e-05 2023-06-03T20:34:36.310 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:36.310 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:36.311 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "reached_pg", 2023-06-03T20:34:36.311 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:31:27.895410+0000", 2023-06-03T20:34:36.311 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.00015970200000000001 2023-06-03T20:34:36.311 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:36.311 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:36.311 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "started", 2023-06-03T20:34:36.311 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:31:27.895569+0000", 2023-06-03T20:34:36.312 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.000193709 2023-06-03T20:34:36.312 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:36.312 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:36.312 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "waiting for subops from 1,3", 2023-06-03T20:34:36.312 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:31:27.895763+0000", 2023-06-03T20:34:36.313 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.0056047730000000004 2023-06-03T20:34:36.313 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:36.313 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:36.313 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "op_commit", 2023-06-03T20:34:36.313 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:31:27.901368+0000", 2023-06-03T20:34:36.314 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.0096641309999999994 2023-06-03T20:34:36.314 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:36.314 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:36.314 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:34:36.314 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:31:27.911032+0000", 2023-06-03T20:34:36.314 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.15327903100000001 2023-06-03T20:34:36.315 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:36.315 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:36.315 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:34:36.315 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:31:28.064311+0000", 2023-06-03T20:34:36.316 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.00011322100000000001 2023-06-03T20:34:36.316 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:36.316 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:36.316 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "commit_sent", 2023-06-03T20:34:36.316 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:31:28.064424+0000", 2023-06-03T20:34:36.317 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 9.2199000000000003e-05 2023-06-03T20:34:36.317 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:36.317 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:36.317 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "done", 2023-06-03T20:34:36.317 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:31:28.064516+0000", 2023-06-03T20:34:36.317 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.16922973299999999 2023-06-03T20:34:36.317 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:34:36.317 INFO:teuthology.orchestra.run.smithi164.stdout: ] 2023-06-03T20:34:36.318 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:34:36.318 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:36.318 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:36.318 INFO:teuthology.orchestra.run.smithi164.stdout: "description": "osd_op(client.46739.0:184673 14.d 14:b93db841:::benchmark_data_smithi089_237281_object86657:head [delete] snapc 0=[] ondisk+write+known_if_redirected e294)", 2023-06-03T20:34:36.318 INFO:teuthology.orchestra.run.smithi164.stdout: "initiated_at": "2023-06-03T20:31:39.333522+0000", 2023-06-03T20:34:36.318 INFO:teuthology.orchestra.run.smithi164.stdout: "age": 176.912182282, 2023-06-03T20:34:36.318 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 1.0724160730000001, 2023-06-03T20:34:36.318 INFO:teuthology.orchestra.run.smithi164.stdout: "type_data": { 2023-06-03T20:34:36.319 INFO:teuthology.orchestra.run.smithi164.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-03T20:34:36.319 INFO:teuthology.orchestra.run.smithi164.stdout: "client_info": { 2023-06-03T20:34:36.319 INFO:teuthology.orchestra.run.smithi164.stdout: "client": "client.46739", 2023-06-03T20:34:36.319 INFO:teuthology.orchestra.run.smithi164.stdout: "client_addr": "172.21.15.89:0/2479974613", 2023-06-03T20:34:36.319 INFO:teuthology.orchestra.run.smithi164.stdout: "tid": 184673 2023-06-03T20:34:36.319 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:36.319 INFO:teuthology.orchestra.run.smithi164.stdout: "events": [ 2023-06-03T20:34:36.320 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:36.320 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "initiated", 2023-06-03T20:34:36.320 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:31:39.333522+0000", 2023-06-03T20:34:36.320 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0 2023-06-03T20:34:36.320 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:36.320 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:36.321 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "throttled", 2023-06-03T20:34:36.321 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:31:39.333522+0000", 2023-06-03T20:34:36.321 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 4294967295.9999981 2023-06-03T20:34:36.321 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:36.321 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:36.321 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "header_read", 2023-06-03T20:34:36.321 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:31:39.333520+0000", 2023-06-03T20:34:36.321 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 1.117e-05 2023-06-03T20:34:36.322 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:36.322 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:36.322 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "all_read", 2023-06-03T20:34:36.322 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:31:39.333531+0000", 2023-06-03T20:34:36.322 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 2.1863999999999999e-05 2023-06-03T20:34:36.322 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:36.322 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:36.323 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "dispatched", 2023-06-03T20:34:36.323 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:31:39.333553+0000", 2023-06-03T20:34:36.323 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 1.029e-05 2023-06-03T20:34:36.323 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:36.323 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:36.323 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "queued_for_pg", 2023-06-03T20:34:36.323 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:31:39.333564+0000", 2023-06-03T20:34:36.324 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 8.4283999999999995e-05 2023-06-03T20:34:36.324 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:36.324 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:36.324 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "reached_pg", 2023-06-03T20:34:36.324 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:31:39.333648+0000", 2023-06-03T20:34:36.324 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 1.3988000000000001e-05 2023-06-03T20:34:36.325 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:36.325 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:36.325 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "waiting for peered", 2023-06-03T20:34:36.325 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:31:39.333662+0000", 2023-06-03T20:34:36.325 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.99378995400000003 2023-06-03T20:34:36.325 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:36.325 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:36.326 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "reached_pg", 2023-06-03T20:34:36.326 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:31:40.327452+0000", 2023-06-03T20:34:36.326 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.000119078 2023-06-03T20:34:36.326 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:36.326 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:36.326 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "started", 2023-06-03T20:34:36.326 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:31:40.327571+0000", 2023-06-03T20:34:36.326 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.000762841 2023-06-03T20:34:36.327 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:36.327 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:36.327 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "waiting for subops from 4,5", 2023-06-03T20:34:36.327 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:31:40.328334+0000", 2023-06-03T20:34:36.327 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.013366233 2023-06-03T20:34:36.327 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:36.327 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:36.327 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "op_commit", 2023-06-03T20:34:36.328 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:31:40.341700+0000", 2023-06-03T20:34:36.328 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.064009910000000003 2023-06-03T20:34:36.328 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:36.328 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:36.328 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:34:36.328 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:31:40.405710+0000", 2023-06-03T20:34:36.328 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.000108461 2023-06-03T20:34:36.329 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:36.329 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:36.329 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:34:36.329 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:31:40.405818+0000", 2023-06-03T20:34:36.329 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 8.0412000000000002e-05 2023-06-03T20:34:36.329 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:36.329 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:36.330 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "commit_sent", 2023-06-03T20:34:36.330 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:31:40.405899+0000", 2023-06-03T20:34:36.330 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 3.9552999999999999e-05 2023-06-03T20:34:36.330 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:36.330 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:36.330 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "done", 2023-06-03T20:34:36.330 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:31:40.405938+0000", 2023-06-03T20:34:36.330 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 1.0724160730000001 2023-06-03T20:34:36.331 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:34:36.331 INFO:teuthology.orchestra.run.smithi164.stdout: ] 2023-06-03T20:34:36.331 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:34:36.331 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:36.331 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:36.331 INFO:teuthology.orchestra.run.smithi164.stdout: "description": "osd_op(client.46739.0:184684 14.d 14:bbd65054:::benchmark_data_smithi089_237281_object86668:head [delete] snapc 0=[] ondisk+write+known_if_redirected e294)", 2023-06-03T20:34:36.331 INFO:teuthology.orchestra.run.smithi164.stdout: "initiated_at": "2023-06-03T20:31:39.357604+0000", 2023-06-03T20:34:36.331 INFO:teuthology.orchestra.run.smithi164.stdout: "age": 176.88810073499999, 2023-06-03T20:34:36.332 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 1.0484886840000001, 2023-06-03T20:34:36.332 INFO:teuthology.orchestra.run.smithi164.stdout: "type_data": { 2023-06-03T20:34:36.332 INFO:teuthology.orchestra.run.smithi164.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-03T20:34:36.332 INFO:teuthology.orchestra.run.smithi164.stdout: "client_info": { 2023-06-03T20:34:36.332 INFO:teuthology.orchestra.run.smithi164.stdout: "client": "client.46739", 2023-06-03T20:34:36.332 INFO:teuthology.orchestra.run.smithi164.stdout: "client_addr": "172.21.15.89:0/2479974613", 2023-06-03T20:34:36.333 INFO:teuthology.orchestra.run.smithi164.stdout: "tid": 184684 2023-06-03T20:34:36.333 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:36.333 INFO:teuthology.orchestra.run.smithi164.stdout: "events": [ 2023-06-03T20:34:36.333 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:36.333 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "initiated", 2023-06-03T20:34:36.334 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:31:39.357604+0000", 2023-06-03T20:34:36.334 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0 2023-06-03T20:34:36.334 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:36.334 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:36.334 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "throttled", 2023-06-03T20:34:36.334 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:31:39.357604+0000", 2023-06-03T20:34:36.334 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 4294967295.9999981 2023-06-03T20:34:36.335 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:36.335 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:36.335 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "header_read", 2023-06-03T20:34:36.335 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:31:39.357602+0000", 2023-06-03T20:34:36.335 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 1.0624e-05 2023-06-03T20:34:36.335 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:36.335 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:36.336 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "all_read", 2023-06-03T20:34:36.336 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:31:39.357613+0000", 2023-06-03T20:34:36.336 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 1.9853000000000001e-05 2023-06-03T20:34:36.336 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:36.336 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:36.336 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "dispatched", 2023-06-03T20:34:36.336 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:31:39.357633+0000", 2023-06-03T20:34:36.337 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 8.0520000000000002e-06 2023-06-03T20:34:36.337 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:36.337 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:36.337 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "queued_for_pg", 2023-06-03T20:34:36.337 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:31:39.357641+0000", 2023-06-03T20:34:36.337 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 5.7924999999999999e-05 2023-06-03T20:34:36.337 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:36.337 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:36.338 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "reached_pg", 2023-06-03T20:34:36.338 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:31:39.357699+0000", 2023-06-03T20:34:36.338 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 8.9299999999999992e-06 2023-06-03T20:34:36.338 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:36.338 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:36.338 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "waiting for peered", 2023-06-03T20:34:36.338 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:31:39.357707+0000", 2023-06-03T20:34:36.338 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.97630063199999995 2023-06-03T20:34:36.339 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:36.339 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:36.339 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "reached_pg", 2023-06-03T20:34:36.339 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:31:40.334008+0000", 2023-06-03T20:34:36.339 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 9.0859000000000005e-05 2023-06-03T20:34:36.339 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:36.339 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:36.340 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "started", 2023-06-03T20:34:36.340 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:31:40.334099+0000", 2023-06-03T20:34:36.340 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 9.4481000000000006e-05 2023-06-03T20:34:36.340 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:36.340 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:36.340 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "waiting for subops from 4,5", 2023-06-03T20:34:36.340 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:31:40.334193+0000", 2023-06-03T20:34:36.341 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.016518851000000001 2023-06-03T20:34:36.341 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:36.341 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:36.341 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "op_commit", 2023-06-03T20:34:36.341 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:31:40.350712+0000", 2023-06-03T20:34:36.341 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.055285494999999997 2023-06-03T20:34:36.341 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:36.341 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:36.342 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:34:36.342 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:31:40.405998+0000", 2023-06-03T20:34:36.342 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 3.5976000000000002e-05 2023-06-03T20:34:36.342 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:36.342 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:36.342 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:34:36.343 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:31:40.406034+0000", 2023-06-03T20:34:36.343 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 3.2830999999999998e-05 2023-06-03T20:34:36.343 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:36.343 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:36.343 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "commit_sent", 2023-06-03T20:34:36.343 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:31:40.406067+0000", 2023-06-03T20:34:36.343 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 2.5928999999999999e-05 2023-06-03T20:34:36.343 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:36.344 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:36.344 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "done", 2023-06-03T20:34:36.344 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:31:40.406093+0000", 2023-06-03T20:34:36.344 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 1.0484886840000001 2023-06-03T20:34:36.344 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:34:36.344 INFO:teuthology.orchestra.run.smithi164.stdout: ] 2023-06-03T20:34:36.344 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:34:36.345 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:36.345 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:36.345 INFO:teuthology.orchestra.run.smithi164.stdout: "description": "osd_op(client.46739.0:184690 14.d 14:bbf919b9:::benchmark_data_smithi089_237281_object86674:head [delete] snapc 0=[] ondisk+write+known_if_redirected e294)", 2023-06-03T20:34:36.345 INFO:teuthology.orchestra.run.smithi164.stdout: "initiated_at": "2023-06-03T20:31:39.364961+0000", 2023-06-03T20:34:36.345 INFO:teuthology.orchestra.run.smithi164.stdout: "age": 176.88074329400001, 2023-06-03T20:34:36.345 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 1.041342376, 2023-06-03T20:34:36.345 INFO:teuthology.orchestra.run.smithi164.stdout: "type_data": { 2023-06-03T20:34:36.345 INFO:teuthology.orchestra.run.smithi164.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-03T20:34:36.346 INFO:teuthology.orchestra.run.smithi164.stdout: "client_info": { 2023-06-03T20:34:36.346 INFO:teuthology.orchestra.run.smithi164.stdout: "client": "client.46739", 2023-06-03T20:34:36.346 INFO:teuthology.orchestra.run.smithi164.stdout: "client_addr": "172.21.15.89:0/2479974613", 2023-06-03T20:34:36.346 INFO:teuthology.orchestra.run.smithi164.stdout: "tid": 184690 2023-06-03T20:34:36.346 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:36.346 INFO:teuthology.orchestra.run.smithi164.stdout: "events": [ 2023-06-03T20:34:36.346 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:36.346 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "initiated", 2023-06-03T20:34:36.347 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:31:39.364961+0000", 2023-06-03T20:34:36.347 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0 2023-06-03T20:34:36.347 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:36.347 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:36.347 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "throttled", 2023-06-03T20:34:36.347 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:31:39.364961+0000", 2023-06-03T20:34:36.347 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 4294967295.9999986 2023-06-03T20:34:36.348 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:36.348 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:36.348 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "header_read", 2023-06-03T20:34:36.348 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:31:39.364960+0000", 2023-06-03T20:34:36.348 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 7.8159999999999994e-06 2023-06-03T20:34:36.348 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:36.348 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:36.349 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "all_read", 2023-06-03T20:34:36.349 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:31:39.364968+0000", 2023-06-03T20:34:36.349 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 1.4044e-05 2023-06-03T20:34:36.349 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:36.349 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:36.349 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "dispatched", 2023-06-03T20:34:36.349 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:31:39.364982+0000", 2023-06-03T20:34:36.350 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 6.2829999999999998e-06 2023-06-03T20:34:36.350 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:36.350 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:36.350 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "queued_for_pg", 2023-06-03T20:34:36.350 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:31:39.364988+0000", 2023-06-03T20:34:36.350 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 8.6360000000000007e-05 2023-06-03T20:34:36.350 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:36.351 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:36.351 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "reached_pg", 2023-06-03T20:34:36.351 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:31:39.365074+0000", 2023-06-03T20:34:36.351 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 1.5364999999999999e-05 2023-06-03T20:34:36.351 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:36.351 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:36.351 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "waiting for peered", 2023-06-03T20:34:36.352 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:31:39.365090+0000", 2023-06-03T20:34:36.352 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.97279540499999995 2023-06-03T20:34:36.352 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:36.352 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:36.352 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "reached_pg", 2023-06-03T20:34:36.352 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:31:40.337885+0000", 2023-06-03T20:34:36.352 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 8.1658999999999999e-05 2023-06-03T20:34:36.352 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:36.353 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:36.353 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "started", 2023-06-03T20:34:36.353 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:31:40.337967+0000", 2023-06-03T20:34:36.353 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 9.3189999999999994e-05 2023-06-03T20:34:36.353 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:36.353 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:36.353 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "waiting for subops from 4,5", 2023-06-03T20:34:36.354 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:31:40.338060+0000", 2023-06-03T20:34:36.354 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.028312192 2023-06-03T20:34:36.354 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:36.354 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:36.354 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "op_commit", 2023-06-03T20:34:36.354 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:31:40.366372+0000", 2023-06-03T20:34:36.354 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.039819891000000003 2023-06-03T20:34:36.355 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:36.355 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:36.355 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:34:36.355 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:31:40.406192+0000", 2023-06-03T20:34:36.355 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 3.7401999999999999e-05 2023-06-03T20:34:36.355 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:36.355 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:36.355 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:34:36.356 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:31:40.406229+0000", 2023-06-03T20:34:36.356 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 3.4578e-05 2023-06-03T20:34:36.356 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:36.356 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:36.356 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "commit_sent", 2023-06-03T20:34:36.356 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:31:40.406264+0000", 2023-06-03T20:34:36.356 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 3.9629999999999998e-05 2023-06-03T20:34:36.356 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:36.357 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:36.357 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "done", 2023-06-03T20:34:36.357 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:31:40.406304+0000", 2023-06-03T20:34:36.357 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 1.041342376 2023-06-03T20:34:36.357 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:34:36.357 INFO:teuthology.orchestra.run.smithi164.stdout: ] 2023-06-03T20:34:36.357 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:34:36.358 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:36.358 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:36.358 INFO:teuthology.orchestra.run.smithi164.stdout: "description": "osd_op(client.46739.0:184717 14.d 14:bba62e92:::benchmark_data_smithi089_237281_object86701:head [delete] snapc 0=[] ondisk+write+known_if_redirected e294)", 2023-06-03T20:34:36.358 INFO:teuthology.orchestra.run.smithi164.stdout: "initiated_at": "2023-06-03T20:31:39.398581+0000", 2023-06-03T20:34:36.358 INFO:teuthology.orchestra.run.smithi164.stdout: "age": 176.84712386699999, 2023-06-03T20:34:36.358 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 1.007914998, 2023-06-03T20:34:36.358 INFO:teuthology.orchestra.run.smithi164.stdout: "type_data": { 2023-06-03T20:34:36.358 INFO:teuthology.orchestra.run.smithi164.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-03T20:34:36.359 INFO:teuthology.orchestra.run.smithi164.stdout: "client_info": { 2023-06-03T20:34:36.359 INFO:teuthology.orchestra.run.smithi164.stdout: "client": "client.46739", 2023-06-03T20:34:36.359 INFO:teuthology.orchestra.run.smithi164.stdout: "client_addr": "172.21.15.89:0/2479974613", 2023-06-03T20:34:36.359 INFO:teuthology.orchestra.run.smithi164.stdout: "tid": 184717 2023-06-03T20:34:36.359 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:36.359 INFO:teuthology.orchestra.run.smithi164.stdout: "events": [ 2023-06-03T20:34:36.359 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:36.360 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "initiated", 2023-06-03T20:34:36.360 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:31:39.398581+0000", 2023-06-03T20:34:36.360 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0 2023-06-03T20:34:36.360 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:36.360 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:36.360 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "throttled", 2023-06-03T20:34:36.360 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:31:39.398581+0000", 2023-06-03T20:34:36.360 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 4294967295.9999981 2023-06-03T20:34:36.361 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:36.361 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:36.361 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "header_read", 2023-06-03T20:34:36.361 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:31:39.398579+0000", 2023-06-03T20:34:36.361 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 1.1316e-05 2023-06-03T20:34:36.361 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:36.361 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:36.362 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "all_read", 2023-06-03T20:34:36.362 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:31:39.398590+0000", 2023-06-03T20:34:36.362 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 2.5429e-05 2023-06-03T20:34:36.362 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:36.362 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:36.362 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "dispatched", 2023-06-03T20:34:36.362 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:31:39.398615+0000", 2023-06-03T20:34:36.362 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 1.2098e-05 2023-06-03T20:34:36.363 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:36.363 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:36.363 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "queued_for_pg", 2023-06-03T20:34:36.363 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:31:39.398627+0000", 2023-06-03T20:34:36.363 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.000104067 2023-06-03T20:34:36.363 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:36.363 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:36.363 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "reached_pg", 2023-06-03T20:34:36.364 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:31:39.398732+0000", 2023-06-03T20:34:36.364 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 1.7714999999999999e-05 2023-06-03T20:34:36.364 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:36.364 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:36.364 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "waiting for peered", 2023-06-03T20:34:36.364 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:31:39.398749+0000", 2023-06-03T20:34:36.364 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.94302888500000004 2023-06-03T20:34:36.365 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:36.365 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:36.365 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "reached_pg", 2023-06-03T20:34:36.365 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:31:40.341778+0000", 2023-06-03T20:34:36.365 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.000182103 2023-06-03T20:34:36.365 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:36.365 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:36.366 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "started", 2023-06-03T20:34:36.366 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:31:40.341960+0000", 2023-06-03T20:34:36.366 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.00022882999999999999 2023-06-03T20:34:36.366 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:36.366 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:36.366 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "waiting for subops from 4,5", 2023-06-03T20:34:36.366 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:31:40.342189+0000", 2023-06-03T20:34:36.366 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.029904258999999999 2023-06-03T20:34:36.367 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:36.367 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:36.367 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "op_commit", 2023-06-03T20:34:36.367 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:31:40.372093+0000", 2023-06-03T20:34:36.367 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.034257124 2023-06-03T20:34:36.367 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:36.367 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:36.367 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:34:36.368 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:31:40.406350+0000", 2023-06-03T20:34:36.368 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 4.1161000000000001e-05 2023-06-03T20:34:36.368 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:36.368 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:36.368 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:34:36.368 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:31:40.406392+0000", 2023-06-03T20:34:36.368 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 6.4131e-05 2023-06-03T20:34:36.368 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:36.369 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:36.369 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "commit_sent", 2023-06-03T20:34:36.369 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:31:40.406456+0000", 2023-06-03T20:34:36.369 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 3.9984000000000002e-05 2023-06-03T20:34:36.369 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:36.369 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:36.369 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "done", 2023-06-03T20:34:36.370 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:31:40.406496+0000", 2023-06-03T20:34:36.370 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 1.007914998 2023-06-03T20:34:36.370 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:34:36.370 INFO:teuthology.orchestra.run.smithi164.stdout: ] 2023-06-03T20:34:36.370 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:34:36.370 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:36.370 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:36.370 INFO:teuthology.orchestra.run.smithi164.stdout: "description": "osd_op(client.46739.0:184719 14.d 14:bdbe1bcd:::benchmark_data_smithi089_237281_object86703:head [delete] snapc 0=[] ondisk+write+known_if_redirected e294)", 2023-06-03T20:34:36.371 INFO:teuthology.orchestra.run.smithi164.stdout: "initiated_at": "2023-06-03T20:31:39.404265+0000", 2023-06-03T20:34:36.371 INFO:teuthology.orchestra.run.smithi164.stdout: "age": 176.84143916100001, 2023-06-03T20:34:36.371 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 1.0024367270000001, 2023-06-03T20:34:36.371 INFO:teuthology.orchestra.run.smithi164.stdout: "type_data": { 2023-06-03T20:34:36.371 INFO:teuthology.orchestra.run.smithi164.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-03T20:34:36.371 INFO:teuthology.orchestra.run.smithi164.stdout: "client_info": { 2023-06-03T20:34:36.371 INFO:teuthology.orchestra.run.smithi164.stdout: "client": "client.46739", 2023-06-03T20:34:36.371 INFO:teuthology.orchestra.run.smithi164.stdout: "client_addr": "172.21.15.89:0/2479974613", 2023-06-03T20:34:36.372 INFO:teuthology.orchestra.run.smithi164.stdout: "tid": 184719 2023-06-03T20:34:36.372 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:36.372 INFO:teuthology.orchestra.run.smithi164.stdout: "events": [ 2023-06-03T20:34:36.372 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:36.372 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "initiated", 2023-06-03T20:34:36.372 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:31:39.404265+0000", 2023-06-03T20:34:36.372 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0 2023-06-03T20:34:36.373 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:36.373 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:36.373 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "throttled", 2023-06-03T20:34:36.373 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:31:39.404265+0000", 2023-06-03T20:34:36.373 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 4294967295.9999981 2023-06-03T20:34:36.373 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:36.373 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:36.373 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "header_read", 2023-06-03T20:34:36.374 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:31:39.404264+0000", 2023-06-03T20:34:36.374 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 1.0183999999999999e-05 2023-06-03T20:34:36.374 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:36.374 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:36.374 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "all_read", 2023-06-03T20:34:36.374 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:31:39.404274+0000", 2023-06-03T20:34:36.374 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 1.6169999999999999e-05 2023-06-03T20:34:36.375 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:36.375 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:36.375 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "dispatched", 2023-06-03T20:34:36.375 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:31:39.404290+0000", 2023-06-03T20:34:36.375 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 7.43e-06 2023-06-03T20:34:36.375 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:36.375 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:36.375 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "queued_for_pg", 2023-06-03T20:34:36.376 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:31:39.404297+0000", 2023-06-03T20:34:36.376 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 5.698e-05 2023-06-03T20:34:36.376 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:36.376 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:36.376 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "reached_pg", 2023-06-03T20:34:36.376 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:31:39.404354+0000", 2023-06-03T20:34:36.376 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 9.1579999999999994e-06 2023-06-03T20:34:36.377 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:36.377 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:36.377 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "waiting for peered", 2023-06-03T20:34:36.377 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:31:39.404363+0000", 2023-06-03T20:34:36.377 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.94211646500000001 2023-06-03T20:34:36.377 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:36.377 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:36.378 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "reached_pg", 2023-06-03T20:34:36.378 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:31:40.346480+0000", 2023-06-03T20:34:36.378 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.000144733 2023-06-03T20:34:36.378 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:36.378 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:36.378 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "started", 2023-06-03T20:34:36.378 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:31:40.346625+0000", 2023-06-03T20:34:36.378 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.000130553 2023-06-03T20:34:36.379 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:36.379 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:36.379 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "waiting for subops from 4,5", 2023-06-03T20:34:36.379 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:31:40.346755+0000", 2023-06-03T20:34:36.379 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.036284016000000002 2023-06-03T20:34:36.379 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:36.379 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:36.379 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "op_commit", 2023-06-03T20:34:36.380 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:31:40.383039+0000", 2023-06-03T20:34:36.380 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.023503919000000002 2023-06-03T20:34:36.380 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:36.380 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:36.380 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:34:36.380 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:31:40.406543+0000", 2023-06-03T20:34:36.380 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.000100648 2023-06-03T20:34:36.380 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:36.381 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:36.381 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:34:36.381 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:31:40.406644+0000", 2023-06-03T20:34:36.381 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 3.2641000000000001e-05 2023-06-03T20:34:36.381 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:36.381 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:36.381 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "commit_sent", 2023-06-03T20:34:36.382 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:31:40.406676+0000", 2023-06-03T20:34:36.382 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 2.5734000000000001e-05 2023-06-03T20:34:36.382 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:36.382 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:36.382 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "done", 2023-06-03T20:34:36.382 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:31:40.406702+0000", 2023-06-03T20:34:36.382 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 1.0024367270000001 2023-06-03T20:34:36.382 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:34:36.383 INFO:teuthology.orchestra.run.smithi164.stdout: ] 2023-06-03T20:34:36.383 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:34:36.383 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:36.383 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:36.383 INFO:teuthology.orchestra.run.smithi164.stdout: "description": "osd_op(client.46739.0:184756 14.d 14:bc34151a:::benchmark_data_smithi089_237281_object86740:head [delete] snapc 0=[] ondisk+write+known_if_redirected e294)", 2023-06-03T20:34:36.383 INFO:teuthology.orchestra.run.smithi164.stdout: "initiated_at": "2023-06-03T20:31:39.456017+0000", 2023-06-03T20:34:36.383 INFO:teuthology.orchestra.run.smithi164.stdout: "age": 176.78968747299999, 2023-06-03T20:34:36.384 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.95078445199999995, 2023-06-03T20:34:36.384 INFO:teuthology.orchestra.run.smithi164.stdout: "type_data": { 2023-06-03T20:34:36.384 INFO:teuthology.orchestra.run.smithi164.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-03T20:34:36.384 INFO:teuthology.orchestra.run.smithi164.stdout: "client_info": { 2023-06-03T20:34:36.384 INFO:teuthology.orchestra.run.smithi164.stdout: "client": "client.46739", 2023-06-03T20:34:36.384 INFO:teuthology.orchestra.run.smithi164.stdout: "client_addr": "172.21.15.89:0/2479974613", 2023-06-03T20:34:36.384 INFO:teuthology.orchestra.run.smithi164.stdout: "tid": 184756 2023-06-03T20:34:36.385 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:36.385 INFO:teuthology.orchestra.run.smithi164.stdout: "events": [ 2023-06-03T20:34:36.385 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:36.385 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "initiated", 2023-06-03T20:34:36.386 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:31:39.456017+0000", 2023-06-03T20:34:36.386 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0 2023-06-03T20:34:36.386 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:36.386 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:36.386 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "throttled", 2023-06-03T20:34:36.386 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:31:39.456017+0000", 2023-06-03T20:34:36.386 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 4294967295.9999995 2023-06-03T20:34:36.387 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:36.387 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:36.387 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "header_read", 2023-06-03T20:34:36.387 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:31:39.456016+0000", 2023-06-03T20:34:36.387 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 4.2509999999999996e-06 2023-06-03T20:34:36.387 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:36.387 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:36.387 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "all_read", 2023-06-03T20:34:36.388 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:31:39.456021+0000", 2023-06-03T20:34:36.388 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 6.6540000000000002e-06 2023-06-03T20:34:36.388 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:36.388 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:36.388 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "dispatched", 2023-06-03T20:34:36.388 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:31:39.456027+0000", 2023-06-03T20:34:36.389 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 3.8519999999999997e-06 2023-06-03T20:34:36.389 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:36.389 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:36.389 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "queued_for_pg", 2023-06-03T20:34:36.389 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:31:39.456031+0000", 2023-06-03T20:34:36.390 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 6.2978999999999995e-05 2023-06-03T20:34:36.390 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:36.390 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:36.390 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "reached_pg", 2023-06-03T20:34:36.390 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:31:39.456094+0000", 2023-06-03T20:34:36.390 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 1.2840000000000001e-05 2023-06-03T20:34:36.390 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:36.390 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:36.391 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "waiting for peered", 2023-06-03T20:34:36.391 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:31:39.456107+0000", 2023-06-03T20:34:36.391 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.89466720300000002 2023-06-03T20:34:36.391 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:36.391 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:36.391 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "reached_pg", 2023-06-03T20:34:36.391 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:31:40.350774+0000", 2023-06-03T20:34:36.391 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.000178147 2023-06-03T20:34:36.392 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:36.392 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:36.392 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "started", 2023-06-03T20:34:36.392 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:31:40.350952+0000", 2023-06-03T20:34:36.392 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.00025212999999999999 2023-06-03T20:34:36.392 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:36.392 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:36.393 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "waiting for subops from 4,5", 2023-06-03T20:34:36.393 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:31:40.351204+0000", 2023-06-03T20:34:36.393 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.031853318999999998 2023-06-03T20:34:36.393 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:36.393 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:36.393 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "op_commit", 2023-06-03T20:34:36.393 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:31:40.383058+0000", 2023-06-03T20:34:36.394 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.023521233999999999 2023-06-03T20:34:36.394 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:36.394 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:36.394 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:34:36.394 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:31:40.406579+0000", 2023-06-03T20:34:36.394 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.00016198800000000001 2023-06-03T20:34:36.394 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:36.395 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:36.395 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:34:36.395 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:31:40.406741+0000", 2023-06-03T20:34:36.395 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 3.3325999999999999e-05 2023-06-03T20:34:36.395 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:36.395 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:36.395 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "commit_sent", 2023-06-03T20:34:36.396 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:31:40.406774+0000", 2023-06-03T20:34:36.396 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 2.7223e-05 2023-06-03T20:34:36.396 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:36.396 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:36.396 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "done", 2023-06-03T20:34:36.396 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:31:40.406802+0000", 2023-06-03T20:34:36.396 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.95078445199999995 2023-06-03T20:34:36.396 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:34:36.397 INFO:teuthology.orchestra.run.smithi164.stdout: ] 2023-06-03T20:34:36.397 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:34:36.397 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:36.397 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:36.397 INFO:teuthology.orchestra.run.smithi164.stdout: "description": "osd_op(client.46739.0:184757 14.d 14:b18ca65f:::benchmark_data_smithi089_237281_object86741:head [delete] snapc 0=[] ondisk+write+known_if_redirected e294)", 2023-06-03T20:34:36.397 INFO:teuthology.orchestra.run.smithi164.stdout: "initiated_at": "2023-06-03T20:31:39.461823+0000", 2023-06-03T20:34:36.397 INFO:teuthology.orchestra.run.smithi164.stdout: "age": 176.78388170700001, 2023-06-03T20:34:36.397 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.94557367699999995, 2023-06-03T20:34:36.398 INFO:teuthology.orchestra.run.smithi164.stdout: "type_data": { 2023-06-03T20:34:36.398 INFO:teuthology.orchestra.run.smithi164.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-03T20:34:36.398 INFO:teuthology.orchestra.run.smithi164.stdout: "client_info": { 2023-06-03T20:34:36.398 INFO:teuthology.orchestra.run.smithi164.stdout: "client": "client.46739", 2023-06-03T20:34:36.398 INFO:teuthology.orchestra.run.smithi164.stdout: "client_addr": "172.21.15.89:0/2479974613", 2023-06-03T20:34:36.398 INFO:teuthology.orchestra.run.smithi164.stdout: "tid": 184757 2023-06-03T20:34:36.398 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:36.399 INFO:teuthology.orchestra.run.smithi164.stdout: "events": [ 2023-06-03T20:34:36.399 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:36.399 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "initiated", 2023-06-03T20:34:36.399 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:31:39.461823+0000", 2023-06-03T20:34:36.399 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0 2023-06-03T20:34:36.399 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:36.399 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:36.400 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "throttled", 2023-06-03T20:34:36.400 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:31:39.461823+0000", 2023-06-03T20:34:36.400 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 4294967295.999999 2023-06-03T20:34:36.400 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:36.400 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:36.400 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "header_read", 2023-06-03T20:34:36.400 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:31:39.461822+0000", 2023-06-03T20:34:36.401 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 6.8569999999999997e-06 2023-06-03T20:34:36.401 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:36.401 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:36.401 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "all_read", 2023-06-03T20:34:36.401 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:31:39.461829+0000", 2023-06-03T20:34:36.401 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 1.3287e-05 2023-06-03T20:34:36.401 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:36.402 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:36.402 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "dispatched", 2023-06-03T20:34:36.402 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:31:39.461842+0000", 2023-06-03T20:34:36.402 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 6.4010000000000002e-06 2023-06-03T20:34:36.402 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:36.403 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:36.403 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "queued_for_pg", 2023-06-03T20:34:36.403 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:31:39.461848+0000", 2023-06-03T20:34:36.403 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 7.1734000000000001e-05 2023-06-03T20:34:36.404 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:36.404 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:36.404 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "reached_pg", 2023-06-03T20:34:36.405 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:31:39.461920+0000", 2023-06-03T20:34:36.405 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 1.3142000000000001e-05 2023-06-03T20:34:36.405 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:36.405 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:36.406 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "waiting for peered", 2023-06-03T20:34:36.406 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:31:39.461933+0000", 2023-06-03T20:34:36.406 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.89360882200000002 2023-06-03T20:34:36.407 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:36.407 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:36.407 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "reached_pg", 2023-06-03T20:34:36.408 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:31:40.355542+0000", 2023-06-03T20:34:36.408 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.00011459900000000001 2023-06-03T20:34:36.408 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:36.408 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:36.409 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "started", 2023-06-03T20:34:36.409 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:31:40.355657+0000", 2023-06-03T20:34:36.409 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.00014334999999999999 2023-06-03T20:34:36.410 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:36.410 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:36.410 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "waiting for subops from 4,5", 2023-06-03T20:34:36.410 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:31:40.355800+0000", 2023-06-03T20:34:36.410 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.027275326999999999 2023-06-03T20:34:36.411 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:36.411 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:36.411 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "op_commit", 2023-06-03T20:34:36.411 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:31:40.383075+0000", 2023-06-03T20:34:36.412 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.023536896000000002 2023-06-03T20:34:36.412 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:36.412 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:36.412 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:34:36.413 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:31:40.406612+0000", 2023-06-03T20:34:36.413 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.00068781099999999998 2023-06-03T20:34:36.413 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:36.413 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:36.414 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:34:36.414 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:31:40.407300+0000", 2023-06-03T20:34:36.414 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 5.0138e-05 2023-06-03T20:34:36.414 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:36.414 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:36.414 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "commit_sent", 2023-06-03T20:34:36.414 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:31:40.407350+0000", 2023-06-03T20:34:36.415 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 4.6474999999999998e-05 2023-06-03T20:34:36.415 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:36.415 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:36.415 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "done", 2023-06-03T20:34:36.415 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:31:40.407397+0000", 2023-06-03T20:34:36.415 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.94557367699999995 2023-06-03T20:34:36.415 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:34:36.415 INFO:teuthology.orchestra.run.smithi164.stdout: ] 2023-06-03T20:34:36.416 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:34:36.416 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:36.416 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:36.416 INFO:teuthology.orchestra.run.smithi164.stdout: "description": "osd_op(client.46739.0:184768 14.d 14:bd165c93:::benchmark_data_smithi089_237281_object86752:head [delete] snapc 0=[] ondisk+write+known_if_redirected e294)", 2023-06-03T20:34:36.416 INFO:teuthology.orchestra.run.smithi164.stdout: "initiated_at": "2023-06-03T20:31:39.476798+0000", 2023-06-03T20:34:36.416 INFO:teuthology.orchestra.run.smithi164.stdout: "age": 176.76890673, 2023-06-03T20:34:36.416 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.93080934000000004, 2023-06-03T20:34:36.417 INFO:teuthology.orchestra.run.smithi164.stdout: "type_data": { 2023-06-03T20:34:36.417 INFO:teuthology.orchestra.run.smithi164.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-03T20:34:36.417 INFO:teuthology.orchestra.run.smithi164.stdout: "client_info": { 2023-06-03T20:34:36.417 INFO:teuthology.orchestra.run.smithi164.stdout: "client": "client.46739", 2023-06-03T20:34:36.417 INFO:teuthology.orchestra.run.smithi164.stdout: "client_addr": "172.21.15.89:0/2479974613", 2023-06-03T20:34:36.417 INFO:teuthology.orchestra.run.smithi164.stdout: "tid": 184768 2023-06-03T20:34:36.417 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:36.418 INFO:teuthology.orchestra.run.smithi164.stdout: "events": [ 2023-06-03T20:34:36.418 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:36.418 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "initiated", 2023-06-03T20:34:36.418 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:31:39.476798+0000", 2023-06-03T20:34:36.418 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0 2023-06-03T20:34:36.418 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:36.418 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:36.419 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "throttled", 2023-06-03T20:34:36.419 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:31:39.476798+0000", 2023-06-03T20:34:36.419 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 4294967295.9999976 2023-06-03T20:34:36.419 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:36.419 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:36.419 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "header_read", 2023-06-03T20:34:36.419 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:31:39.476796+0000", 2023-06-03T20:34:36.420 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 1.6376e-05 2023-06-03T20:34:36.420 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:36.420 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:36.420 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "all_read", 2023-06-03T20:34:36.420 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:31:39.476812+0000", 2023-06-03T20:34:36.420 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 1.7751e-05 2023-06-03T20:34:36.421 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:36.421 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:36.421 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "dispatched", 2023-06-03T20:34:36.421 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:31:39.476830+0000", 2023-06-03T20:34:36.421 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 8.9369999999999993e-06 2023-06-03T20:34:36.422 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:36.422 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:36.422 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "queued_for_pg", 2023-06-03T20:34:36.422 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:31:39.476839+0000", 2023-06-03T20:34:36.422 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 8.0377000000000006e-05 2023-06-03T20:34:36.422 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:36.422 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:36.423 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "reached_pg", 2023-06-03T20:34:36.423 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:31:39.476919+0000", 2023-06-03T20:34:36.423 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 1.6109000000000001e-05 2023-06-03T20:34:36.423 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:36.424 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:36.424 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "waiting for peered", 2023-06-03T20:34:36.424 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:31:39.476935+0000", 2023-06-03T20:34:36.425 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.88312541099999997 2023-06-03T20:34:36.425 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:36.425 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:36.425 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "reached_pg", 2023-06-03T20:34:36.425 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:31:40.360061+0000", 2023-06-03T20:34:36.425 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.000177875 2023-06-03T20:34:36.426 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:36.426 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:36.426 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "started", 2023-06-03T20:34:36.426 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:31:40.360238+0000", 2023-06-03T20:34:36.426 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.000220979 2023-06-03T20:34:36.426 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:36.426 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:36.426 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "waiting for subops from 4,5", 2023-06-03T20:34:36.427 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:31:40.360459+0000", 2023-06-03T20:34:36.427 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.022632449999999998 2023-06-03T20:34:36.427 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:36.427 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:36.427 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "op_commit", 2023-06-03T20:34:36.427 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:31:40.383092+0000", 2023-06-03T20:34:36.427 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.023749908 2023-06-03T20:34:36.428 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:36.428 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:36.428 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:34:36.428 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:31:40.406842+0000", 2023-06-03T20:34:36.428 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.00061361 2023-06-03T20:34:36.428 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:36.429 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:36.429 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:34:36.429 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:31:40.407455+0000", 2023-06-03T20:34:36.429 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.000102469 2023-06-03T20:34:36.430 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:36.430 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:36.430 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "commit_sent", 2023-06-03T20:34:36.430 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:31:40.407558+0000", 2023-06-03T20:34:36.431 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 4.9393e-05 2023-06-03T20:34:36.431 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:36.431 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:36.431 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "done", 2023-06-03T20:34:36.432 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:31:40.407607+0000", 2023-06-03T20:34:36.432 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.93080934000000004 2023-06-03T20:34:36.432 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:34:36.433 INFO:teuthology.orchestra.run.smithi164.stdout: ] 2023-06-03T20:34:36.433 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:34:36.433 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:36.433 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:36.434 INFO:teuthology.orchestra.run.smithi164.stdout: "description": "osd_op(client.46739.0:184782 14.d 14:b8daef26:::benchmark_data_smithi089_237281_object86766:head [delete] snapc 0=[] ondisk+write+known_if_redirected e294)", 2023-06-03T20:34:36.434 INFO:teuthology.orchestra.run.smithi164.stdout: "initiated_at": "2023-06-03T20:31:39.504861+0000", 2023-06-03T20:34:36.434 INFO:teuthology.orchestra.run.smithi164.stdout: "age": 176.74084335399999, 2023-06-03T20:34:36.434 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.90295458299999998, 2023-06-03T20:34:36.435 INFO:teuthology.orchestra.run.smithi164.stdout: "type_data": { 2023-06-03T20:34:36.435 INFO:teuthology.orchestra.run.smithi164.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-03T20:34:36.435 INFO:teuthology.orchestra.run.smithi164.stdout: "client_info": { 2023-06-03T20:34:36.435 INFO:teuthology.orchestra.run.smithi164.stdout: "client": "client.46739", 2023-06-03T20:34:36.436 INFO:teuthology.orchestra.run.smithi164.stdout: "client_addr": "172.21.15.89:0/2479974613", 2023-06-03T20:34:36.436 INFO:teuthology.orchestra.run.smithi164.stdout: "tid": 184782 2023-06-03T20:34:36.436 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:36.436 INFO:teuthology.orchestra.run.smithi164.stdout: "events": [ 2023-06-03T20:34:36.437 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:36.437 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "initiated", 2023-06-03T20:34:36.437 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:31:39.504861+0000", 2023-06-03T20:34:36.437 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0 2023-06-03T20:34:36.438 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:36.438 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:36.438 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "throttled", 2023-06-03T20:34:36.439 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:31:39.504861+0000", 2023-06-03T20:34:36.439 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 4294967295.999999 2023-06-03T20:34:36.439 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:36.439 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:36.440 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "header_read", 2023-06-03T20:34:36.440 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:31:39.504860+0000", 2023-06-03T20:34:36.440 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 5.5169999999999999e-06 2023-06-03T20:34:36.441 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:36.441 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:36.441 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "all_read", 2023-06-03T20:34:36.441 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:31:39.504866+0000", 2023-06-03T20:34:36.442 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 9.5969999999999998e-06 2023-06-03T20:34:36.442 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:36.442 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:36.442 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "dispatched", 2023-06-03T20:34:36.443 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:31:39.504875+0000", 2023-06-03T20:34:36.443 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 4.5050000000000001e-06 2023-06-03T20:34:36.443 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:36.444 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:36.444 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "queued_for_pg", 2023-06-03T20:34:36.444 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:31:39.504880+0000", 2023-06-03T20:34:36.444 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 9.0552999999999999e-05 2023-06-03T20:34:36.444 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:36.444 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:36.444 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "reached_pg", 2023-06-03T20:34:36.445 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:31:39.504970+0000", 2023-06-03T20:34:36.445 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 1.6271999999999999e-05 2023-06-03T20:34:36.445 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:36.445 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:36.445 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "waiting for peered", 2023-06-03T20:34:36.445 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:31:39.504987+0000", 2023-06-03T20:34:36.445 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.86146252400000001 2023-06-03T20:34:36.446 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:36.446 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:36.446 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "reached_pg", 2023-06-03T20:34:36.446 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:31:40.366449+0000", 2023-06-03T20:34:36.446 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.00029169299999999999 2023-06-03T20:34:36.446 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:36.446 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:36.446 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "started", 2023-06-03T20:34:36.447 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:31:40.366741+0000", 2023-06-03T20:34:36.447 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.00024312499999999999 2023-06-03T20:34:36.447 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:36.447 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:36.447 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "waiting for subops from 4,5", 2023-06-03T20:34:36.447 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:31:40.366984+0000", 2023-06-03T20:34:36.447 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.024896932 2023-06-03T20:34:36.448 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:36.448 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:36.448 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "op_commit", 2023-06-03T20:34:36.448 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:31:40.391881+0000", 2023-06-03T20:34:36.448 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.01499554 2023-06-03T20:34:36.448 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:36.448 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:36.449 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:34:36.449 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:31:40.406876+0000", 2023-06-03T20:34:36.449 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.00078612199999999999 2023-06-03T20:34:36.449 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:36.449 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:36.449 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:34:36.449 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:31:40.407662+0000", 2023-06-03T20:34:36.449 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.000104069 2023-06-03T20:34:36.450 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:36.450 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:36.450 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "commit_sent", 2023-06-03T20:34:36.450 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:31:40.407767+0000", 2023-06-03T20:34:36.450 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 4.9264000000000001e-05 2023-06-03T20:34:36.450 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:36.450 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:36.451 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "done", 2023-06-03T20:34:36.451 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:31:40.407816+0000", 2023-06-03T20:34:36.451 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.90295458299999998 2023-06-03T20:34:36.451 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:34:36.451 INFO:teuthology.orchestra.run.smithi164.stdout: ] 2023-06-03T20:34:36.451 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:34:36.452 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:36.452 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:36.452 INFO:teuthology.orchestra.run.smithi164.stdout: "description": "osd_op(client.46739.0:184790 14.d 14:bee63cc5:::benchmark_data_smithi089_237281_object86774:head [delete] snapc 0=[] ondisk+write+known_if_redirected e294)", 2023-06-03T20:34:36.453 INFO:teuthology.orchestra.run.smithi164.stdout: "initiated_at": "2023-06-03T20:31:39.517298+0000", 2023-06-03T20:34:36.453 INFO:teuthology.orchestra.run.smithi164.stdout: "age": 176.728406484, 2023-06-03T20:34:36.453 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.89071872100000005, 2023-06-03T20:34:36.453 INFO:teuthology.orchestra.run.smithi164.stdout: "type_data": { 2023-06-03T20:34:36.454 INFO:teuthology.orchestra.run.smithi164.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-03T20:34:36.454 INFO:teuthology.orchestra.run.smithi164.stdout: "client_info": { 2023-06-03T20:34:36.454 INFO:teuthology.orchestra.run.smithi164.stdout: "client": "client.46739", 2023-06-03T20:34:36.454 INFO:teuthology.orchestra.run.smithi164.stdout: "client_addr": "172.21.15.89:0/2479974613", 2023-06-03T20:34:36.454 INFO:teuthology.orchestra.run.smithi164.stdout: "tid": 184790 2023-06-03T20:34:36.455 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:36.455 INFO:teuthology.orchestra.run.smithi164.stdout: "events": [ 2023-06-03T20:34:36.455 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:36.455 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "initiated", 2023-06-03T20:34:36.455 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:31:39.517298+0000", 2023-06-03T20:34:36.456 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0 2023-06-03T20:34:36.456 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:36.456 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:36.456 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "throttled", 2023-06-03T20:34:36.456 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:31:39.517298+0000", 2023-06-03T20:34:36.456 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 4294967295.9999981 2023-06-03T20:34:36.457 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:36.457 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:36.457 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "header_read", 2023-06-03T20:34:36.457 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:31:39.517296+0000", 2023-06-03T20:34:36.457 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 1.1348e-05 2023-06-03T20:34:36.457 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:36.457 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:36.457 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "all_read", 2023-06-03T20:34:36.458 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:31:39.517307+0000", 2023-06-03T20:34:36.458 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 2.2425e-05 2023-06-03T20:34:36.458 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:36.458 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:36.458 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "dispatched", 2023-06-03T20:34:36.458 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:31:39.517330+0000", 2023-06-03T20:34:36.458 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 1.4577e-05 2023-06-03T20:34:36.459 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:36.459 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:36.459 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "queued_for_pg", 2023-06-03T20:34:36.459 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:31:39.517344+0000", 2023-06-03T20:34:36.459 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 7.5809000000000006e-05 2023-06-03T20:34:36.459 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:36.459 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:36.459 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "reached_pg", 2023-06-03T20:34:36.460 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:31:39.517420+0000", 2023-06-03T20:34:36.460 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 1.0233e-05 2023-06-03T20:34:36.460 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:36.460 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:36.460 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "waiting for peered", 2023-06-03T20:34:36.460 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:31:39.517431+0000", 2023-06-03T20:34:36.460 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.85472747999999998 2023-06-03T20:34:36.461 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:36.461 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:36.461 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "reached_pg", 2023-06-03T20:34:36.461 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:31:40.372158+0000", 2023-06-03T20:34:36.461 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.000158381 2023-06-03T20:34:36.461 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:36.461 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:36.462 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "started", 2023-06-03T20:34:36.462 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:31:40.372316+0000", 2023-06-03T20:34:36.462 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.00021036699999999999 2023-06-03T20:34:36.462 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:36.462 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:36.462 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "waiting for subops from 4,5", 2023-06-03T20:34:36.463 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:31:40.372527+0000", 2023-06-03T20:34:36.463 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.019358785 2023-06-03T20:34:36.463 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:36.464 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:36.464 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "op_commit", 2023-06-03T20:34:36.464 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:31:40.391886+0000", 2023-06-03T20:34:36.464 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.015124287 2023-06-03T20:34:36.465 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:36.465 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:36.465 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:34:36.465 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:31:40.407010+0000", 2023-06-03T20:34:36.466 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.00085344000000000001 2023-06-03T20:34:36.466 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:36.466 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:36.467 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:34:36.467 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:31:40.407863+0000", 2023-06-03T20:34:36.467 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.000106996 2023-06-03T20:34:36.468 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:36.468 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:36.468 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "commit_sent", 2023-06-03T20:34:36.468 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:31:40.407970+0000", 2023-06-03T20:34:36.469 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 4.6558999999999999e-05 2023-06-03T20:34:36.469 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:36.469 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:36.470 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "done", 2023-06-03T20:34:36.470 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:31:40.408017+0000", 2023-06-03T20:34:36.470 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.89071872100000005 2023-06-03T20:34:36.470 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:34:36.471 INFO:teuthology.orchestra.run.smithi164.stdout: ] 2023-06-03T20:34:36.471 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:34:36.471 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:36.471 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:36.471 INFO:teuthology.orchestra.run.smithi164.stdout: "description": "osd_op(client.46739.0:184799 14.d 14:b7f25d3a:::benchmark_data_smithi089_237281_object86783:head [delete] snapc 0=[] ondisk+write+known_if_redirected e294)", 2023-06-03T20:34:36.472 INFO:teuthology.orchestra.run.smithi164.stdout: "initiated_at": "2023-06-03T20:31:39.534185+0000", 2023-06-03T20:34:36.472 INFO:teuthology.orchestra.run.smithi164.stdout: "age": 176.71151950699999, 2023-06-03T20:34:36.472 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.87398139600000002, 2023-06-03T20:34:36.472 INFO:teuthology.orchestra.run.smithi164.stdout: "type_data": { 2023-06-03T20:34:36.472 INFO:teuthology.orchestra.run.smithi164.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-03T20:34:36.472 INFO:teuthology.orchestra.run.smithi164.stdout: "client_info": { 2023-06-03T20:34:36.472 INFO:teuthology.orchestra.run.smithi164.stdout: "client": "client.46739", 2023-06-03T20:34:36.473 INFO:teuthology.orchestra.run.smithi164.stdout: "client_addr": "172.21.15.89:0/2479974613", 2023-06-03T20:34:36.473 INFO:teuthology.orchestra.run.smithi164.stdout: "tid": 184799 2023-06-03T20:34:36.473 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:36.473 INFO:teuthology.orchestra.run.smithi164.stdout: "events": [ 2023-06-03T20:34:36.473 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:36.473 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "initiated", 2023-06-03T20:34:36.473 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:31:39.534185+0000", 2023-06-03T20:34:36.474 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0 2023-06-03T20:34:36.474 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:36.474 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:36.474 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "throttled", 2023-06-03T20:34:36.474 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:31:39.534185+0000", 2023-06-03T20:34:36.474 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 4294967295.9999986 2023-06-03T20:34:36.474 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:36.474 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:36.475 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "header_read", 2023-06-03T20:34:36.475 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:31:39.534184+0000", 2023-06-03T20:34:36.475 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 7.1999999999999997e-06 2023-06-03T20:34:36.475 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:36.475 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:36.475 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "all_read", 2023-06-03T20:34:36.475 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:31:39.534191+0000", 2023-06-03T20:34:36.476 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 1.1583000000000001e-05 2023-06-03T20:34:36.476 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:36.476 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:36.476 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "dispatched", 2023-06-03T20:34:36.476 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:31:39.534203+0000", 2023-06-03T20:34:36.476 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 5.4569999999999996e-06 2023-06-03T20:34:36.476 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:36.476 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:36.477 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "queued_for_pg", 2023-06-03T20:34:36.477 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:31:39.534208+0000", 2023-06-03T20:34:36.477 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 7.3701999999999993e-05 2023-06-03T20:34:36.477 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:36.477 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:36.477 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "reached_pg", 2023-06-03T20:34:36.477 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:31:39.534282+0000", 2023-06-03T20:34:36.478 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 1.0175e-05 2023-06-03T20:34:36.478 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:36.478 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:36.478 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "waiting for peered", 2023-06-03T20:34:36.478 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:31:39.534292+0000", 2023-06-03T20:34:36.478 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.84289912199999995 2023-06-03T20:34:36.478 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:36.478 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:36.479 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "reached_pg", 2023-06-03T20:34:36.479 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:31:40.377191+0000", 2023-06-03T20:34:36.479 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.000155997 2023-06-03T20:34:36.479 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:36.479 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:36.479 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "started", 2023-06-03T20:34:36.479 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:31:40.377347+0000", 2023-06-03T20:34:36.479 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.000189075 2023-06-03T20:34:36.480 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:36.480 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:36.480 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "waiting for subops from 4,5", 2023-06-03T20:34:36.480 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:31:40.377536+0000", 2023-06-03T20:34:36.480 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.023347152999999999 2023-06-03T20:34:36.480 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:36.480 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:36.480 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "op_commit", 2023-06-03T20:34:36.481 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:31:40.400883+0000", 2023-06-03T20:34:36.481 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.0061886179999999999 2023-06-03T20:34:36.481 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:36.481 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:36.481 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:34:36.481 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:31:40.407072+0000", 2023-06-03T20:34:36.481 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.00099120700000000003 2023-06-03T20:34:36.482 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:36.482 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:36.482 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:34:36.482 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:31:40.408063+0000", 2023-06-03T20:34:36.482 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 6.4975999999999996e-05 2023-06-03T20:34:36.482 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:36.482 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:36.482 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "commit_sent", 2023-06-03T20:34:36.483 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:31:40.408128+0000", 2023-06-03T20:34:36.483 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 3.8472e-05 2023-06-03T20:34:36.483 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:36.483 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:36.483 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "done", 2023-06-03T20:34:36.483 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:31:40.408166+0000", 2023-06-03T20:34:36.483 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.87398139600000002 2023-06-03T20:34:36.483 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:34:36.484 INFO:teuthology.orchestra.run.smithi164.stdout: ] 2023-06-03T20:34:36.484 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:34:36.484 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:36.484 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:36.484 INFO:teuthology.orchestra.run.smithi164.stdout: "description": "osd_op(client.46739.0:184806 14.d 14:b852dd89:::benchmark_data_smithi089_237281_object86790:head [delete] snapc 0=[] ondisk+write+known_if_redirected e294)", 2023-06-03T20:34:36.484 INFO:teuthology.orchestra.run.smithi164.stdout: "initiated_at": "2023-06-03T20:31:39.544644+0000", 2023-06-03T20:34:36.484 INFO:teuthology.orchestra.run.smithi164.stdout: "age": 176.701060966, 2023-06-03T20:34:36.484 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.86960870400000001, 2023-06-03T20:34:36.485 INFO:teuthology.orchestra.run.smithi164.stdout: "type_data": { 2023-06-03T20:34:36.485 INFO:teuthology.orchestra.run.smithi164.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-03T20:34:36.485 INFO:teuthology.orchestra.run.smithi164.stdout: "client_info": { 2023-06-03T20:34:36.485 INFO:teuthology.orchestra.run.smithi164.stdout: "client": "client.46739", 2023-06-03T20:34:36.485 INFO:teuthology.orchestra.run.smithi164.stdout: "client_addr": "172.21.15.89:0/2479974613", 2023-06-03T20:34:36.485 INFO:teuthology.orchestra.run.smithi164.stdout: "tid": 184806 2023-06-03T20:34:36.485 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:36.486 INFO:teuthology.orchestra.run.smithi164.stdout: "events": [ 2023-06-03T20:34:36.486 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:36.486 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "initiated", 2023-06-03T20:34:36.486 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:31:39.544644+0000", 2023-06-03T20:34:36.486 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0 2023-06-03T20:34:36.486 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:36.486 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:36.486 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "throttled", 2023-06-03T20:34:36.487 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:31:39.544644+0000", 2023-06-03T20:34:36.487 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 4294967295.9999976 2023-06-03T20:34:36.487 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:36.487 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:36.487 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "header_read", 2023-06-03T20:34:36.487 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:31:39.544641+0000", 2023-06-03T20:34:36.487 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 1.241e-05 2023-06-03T20:34:36.487 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:36.488 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:36.488 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "all_read", 2023-06-03T20:34:36.488 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:31:39.544654+0000", 2023-06-03T20:34:36.488 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 2.2937000000000001e-05 2023-06-03T20:34:36.488 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:36.488 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:36.488 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "dispatched", 2023-06-03T20:34:36.489 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:31:39.544677+0000", 2023-06-03T20:34:36.489 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 1.0899e-05 2023-06-03T20:34:36.489 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:36.489 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:36.489 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "queued_for_pg", 2023-06-03T20:34:36.489 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:31:39.544687+0000", 2023-06-03T20:34:36.489 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 8.6823999999999994e-05 2023-06-03T20:34:36.489 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:36.490 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:36.490 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "reached_pg", 2023-06-03T20:34:36.490 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:31:39.544774+0000", 2023-06-03T20:34:36.490 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 1.4389000000000001e-05 2023-06-03T20:34:36.490 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:36.490 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:36.490 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "waiting for peered", 2023-06-03T20:34:36.490 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:31:39.544789+0000", 2023-06-03T20:34:36.491 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.83838628699999995 2023-06-03T20:34:36.491 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:36.491 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:36.491 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "reached_pg", 2023-06-03T20:34:36.491 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:31:40.383175+0000", 2023-06-03T20:34:36.491 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.00017178699999999999 2023-06-03T20:34:36.491 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:36.491 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:36.492 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "started", 2023-06-03T20:34:36.492 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:31:40.383347+0000", 2023-06-03T20:34:36.492 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.000102612 2023-06-03T20:34:36.492 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:36.492 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:36.492 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "waiting for subops from 4,5", 2023-06-03T20:34:36.492 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:31:40.383449+0000", 2023-06-03T20:34:36.493 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.017444418 2023-06-03T20:34:36.493 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:36.493 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:36.493 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "op_commit", 2023-06-03T20:34:36.493 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:31:40.400894+0000", 2023-06-03T20:34:36.493 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.0073107509999999999 2023-06-03T20:34:36.493 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:36.493 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:36.494 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:34:36.494 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:31:40.408205+0000", 2023-06-03T20:34:36.494 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.0059126919999999998 2023-06-03T20:34:36.494 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:36.494 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:36.494 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:34:36.494 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:31:40.414117+0000", 2023-06-03T20:34:36.494 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 7.2067e-05 2023-06-03T20:34:36.495 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:36.495 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:36.495 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "commit_sent", 2023-06-03T20:34:36.495 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:31:40.414189+0000", 2023-06-03T20:34:36.495 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 6.3015000000000006e-05 2023-06-03T20:34:36.495 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:36.495 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:36.495 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "done", 2023-06-03T20:34:36.496 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:31:40.414252+0000", 2023-06-03T20:34:36.496 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.86960870400000001 2023-06-03T20:34:36.496 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:34:36.496 INFO:teuthology.orchestra.run.smithi164.stdout: ] 2023-06-03T20:34:36.496 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:34:36.496 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:36.496 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:36.497 INFO:teuthology.orchestra.run.smithi164.stdout: "description": "osd_op(client.46739.0:184807 14.d 14:b86f37ca:::benchmark_data_smithi089_237281_object86791:head [delete] snapc 0=[] ondisk+write+known_if_redirected e294)", 2023-06-03T20:34:36.497 INFO:teuthology.orchestra.run.smithi164.stdout: "initiated_at": "2023-06-03T20:31:39.544749+0000", 2023-06-03T20:34:36.497 INFO:teuthology.orchestra.run.smithi164.stdout: "age": 176.700955341, 2023-06-03T20:34:36.497 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.86976470299999997, 2023-06-03T20:34:36.497 INFO:teuthology.orchestra.run.smithi164.stdout: "type_data": { 2023-06-03T20:34:36.497 INFO:teuthology.orchestra.run.smithi164.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-03T20:34:36.497 INFO:teuthology.orchestra.run.smithi164.stdout: "client_info": { 2023-06-03T20:34:36.497 INFO:teuthology.orchestra.run.smithi164.stdout: "client": "client.46739", 2023-06-03T20:34:36.498 INFO:teuthology.orchestra.run.smithi164.stdout: "client_addr": "172.21.15.89:0/2479974613", 2023-06-03T20:34:36.498 INFO:teuthology.orchestra.run.smithi164.stdout: "tid": 184807 2023-06-03T20:34:36.498 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:36.498 INFO:teuthology.orchestra.run.smithi164.stdout: "events": [ 2023-06-03T20:34:36.498 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:36.498 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "initiated", 2023-06-03T20:34:36.498 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:31:39.544749+0000", 2023-06-03T20:34:36.498 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0 2023-06-03T20:34:36.499 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:36.499 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:36.499 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "throttled", 2023-06-03T20:34:36.499 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:31:39.544749+0000", 2023-06-03T20:34:36.499 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 4294967295.999999 2023-06-03T20:34:36.499 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:36.499 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:36.499 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "header_read", 2023-06-03T20:34:36.500 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:31:39.544748+0000", 2023-06-03T20:34:36.500 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 1.1615e-05 2023-06-03T20:34:36.500 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:36.500 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:36.500 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "all_read", 2023-06-03T20:34:36.500 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:31:39.544760+0000", 2023-06-03T20:34:36.500 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 1.5480000000000001e-05 2023-06-03T20:34:36.501 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:36.501 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:36.501 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "dispatched", 2023-06-03T20:34:36.501 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:31:39.544775+0000", 2023-06-03T20:34:36.501 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 7.554e-06 2023-06-03T20:34:36.501 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:36.501 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:36.501 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "queued_for_pg", 2023-06-03T20:34:36.502 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:31:39.544783+0000", 2023-06-03T20:34:36.502 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.00040318999999999999 2023-06-03T20:34:36.502 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:36.502 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:36.502 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "reached_pg", 2023-06-03T20:34:36.502 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:31:39.545186+0000", 2023-06-03T20:34:36.502 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 8.8000000000000004e-06 2023-06-03T20:34:36.502 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:36.503 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:36.503 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "waiting for peered", 2023-06-03T20:34:36.503 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:31:39.545195+0000", 2023-06-03T20:34:36.503 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.84223235600000002 2023-06-03T20:34:36.503 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:36.503 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:36.503 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "reached_pg", 2023-06-03T20:34:36.504 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:31:40.387427+0000", 2023-06-03T20:34:36.504 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.00014051600000000001 2023-06-03T20:34:36.504 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:36.504 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:36.504 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "started", 2023-06-03T20:34:36.504 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:31:40.387568+0000", 2023-06-03T20:34:36.504 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.00018044600000000001 2023-06-03T20:34:36.504 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:36.505 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:36.505 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "waiting for subops from 4,5", 2023-06-03T20:34:36.505 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:31:40.387748+0000", 2023-06-03T20:34:36.505 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.017556697 2023-06-03T20:34:36.505 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:36.505 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:36.505 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "op_commit", 2023-06-03T20:34:36.505 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:31:40.405305+0000", 2023-06-03T20:34:36.506 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.0029312140000000001 2023-06-03T20:34:36.506 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:36.506 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:36.506 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:34:36.506 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:31:40.408236+0000", 2023-06-03T20:34:36.506 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.0061304419999999998 2023-06-03T20:34:36.506 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:36.507 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:36.507 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:34:36.507 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:31:40.414367+0000", 2023-06-03T20:34:36.507 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 8.6572999999999999e-05 2023-06-03T20:34:36.507 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:36.507 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:36.507 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "commit_sent", 2023-06-03T20:34:36.507 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:31:40.414453+0000", 2023-06-03T20:34:36.508 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 6.0844000000000002e-05 2023-06-03T20:34:36.508 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:36.508 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:36.508 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "done", 2023-06-03T20:34:36.508 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:31:40.414514+0000", 2023-06-03T20:34:36.508 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.86976470299999997 2023-06-03T20:34:36.508 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:34:36.508 INFO:teuthology.orchestra.run.smithi164.stdout: ] 2023-06-03T20:34:36.509 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:34:36.509 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:36.509 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:36.509 INFO:teuthology.orchestra.run.smithi164.stdout: "description": "osd_op(client.46739.0:184816 14.d 14:b822ae9f:::benchmark_data_smithi089_237281_object86800:head [delete] snapc 0=[] ondisk+write+known_if_redirected e294)", 2023-06-03T20:34:36.509 INFO:teuthology.orchestra.run.smithi164.stdout: "initiated_at": "2023-06-03T20:31:39.560255+0000", 2023-06-03T20:34:36.509 INFO:teuthology.orchestra.run.smithi164.stdout: "age": 176.68544940199999, 2023-06-03T20:34:36.509 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.85444147699999995, 2023-06-03T20:34:36.509 INFO:teuthology.orchestra.run.smithi164.stdout: "type_data": { 2023-06-03T20:34:36.510 INFO:teuthology.orchestra.run.smithi164.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-03T20:34:36.510 INFO:teuthology.orchestra.run.smithi164.stdout: "client_info": { 2023-06-03T20:34:36.510 INFO:teuthology.orchestra.run.smithi164.stdout: "client": "client.46739", 2023-06-03T20:34:36.510 INFO:teuthology.orchestra.run.smithi164.stdout: "client_addr": "172.21.15.89:0/2479974613", 2023-06-03T20:34:36.510 INFO:teuthology.orchestra.run.smithi164.stdout: "tid": 184816 2023-06-03T20:34:36.510 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:36.510 INFO:teuthology.orchestra.run.smithi164.stdout: "events": [ 2023-06-03T20:34:36.511 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:36.511 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "initiated", 2023-06-03T20:34:36.511 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:31:39.560255+0000", 2023-06-03T20:34:36.511 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0 2023-06-03T20:34:36.511 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:36.511 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:36.511 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "throttled", 2023-06-03T20:34:36.511 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:31:39.560255+0000", 2023-06-03T20:34:36.512 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 4294967295.9999981 2023-06-03T20:34:36.512 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:36.512 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:36.512 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "header_read", 2023-06-03T20:34:36.512 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:31:39.560253+0000", 2023-06-03T20:34:36.512 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 8.0150000000000003e-06 2023-06-03T20:34:36.512 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:36.512 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:36.513 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "all_read", 2023-06-03T20:34:36.513 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:31:39.560262+0000", 2023-06-03T20:34:36.513 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 1.6742999999999999e-05 2023-06-03T20:34:36.513 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:36.513 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:36.513 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "dispatched", 2023-06-03T20:34:36.513 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:31:39.560278+0000", 2023-06-03T20:34:36.514 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 1.0316000000000001e-05 2023-06-03T20:34:36.514 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:36.514 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:36.514 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "queued_for_pg", 2023-06-03T20:34:36.514 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:31:39.560289+0000", 2023-06-03T20:34:36.514 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 7.6033999999999997e-05 2023-06-03T20:34:36.514 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:36.514 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:36.515 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "reached_pg", 2023-06-03T20:34:36.515 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:31:39.560365+0000", 2023-06-03T20:34:36.515 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 1.1962e-05 2023-06-03T20:34:36.515 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:36.515 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:36.515 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "waiting for peered", 2023-06-03T20:34:36.515 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:31:39.560377+0000", 2023-06-03T20:34:36.515 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.83156191499999998 2023-06-03T20:34:36.516 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:36.516 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:36.516 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "reached_pg", 2023-06-03T20:34:36.516 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:31:40.391938+0000", 2023-06-03T20:34:36.516 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.00016462099999999999 2023-06-03T20:34:36.516 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:36.516 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:36.516 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "started", 2023-06-03T20:34:36.517 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:31:40.392103+0000", 2023-06-03T20:34:36.517 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.00015437299999999999 2023-06-03T20:34:36.517 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:36.517 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:36.517 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "waiting for subops from 4,5", 2023-06-03T20:34:36.517 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:31:40.392257+0000", 2023-06-03T20:34:36.517 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.013058581999999999 2023-06-03T20:34:36.517 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:36.518 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:36.518 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "op_commit", 2023-06-03T20:34:36.518 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:31:40.405316+0000", 2023-06-03T20:34:36.518 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.002951221 2023-06-03T20:34:36.518 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:36.518 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:36.518 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:34:36.519 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:31:40.408267+0000", 2023-06-03T20:34:36.519 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.0063332429999999997 2023-06-03T20:34:36.519 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:36.519 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:36.519 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:34:36.519 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:31:40.414601+0000", 2023-06-03T20:34:36.519 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 6.4640000000000005e-05 2023-06-03T20:34:36.519 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:36.520 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:36.520 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "commit_sent", 2023-06-03T20:34:36.520 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:31:40.414665+0000", 2023-06-03T20:34:36.520 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 3.1486000000000003e-05 2023-06-03T20:34:36.520 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:36.520 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:36.520 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "done", 2023-06-03T20:34:36.520 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:31:40.414697+0000", 2023-06-03T20:34:36.521 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.85444147699999995 2023-06-03T20:34:36.521 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:34:36.521 INFO:teuthology.orchestra.run.smithi164.stdout: ] 2023-06-03T20:34:36.521 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:34:36.521 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:36.521 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:36.521 INFO:teuthology.orchestra.run.smithi164.stdout: "description": "osd_op(client.46739.0:184820 14.d 14:b7203e3b:::benchmark_data_smithi089_237281_object86804:head [delete] snapc 0=[] ondisk+write+known_if_redirected e294)", 2023-06-03T20:34:36.521 INFO:teuthology.orchestra.run.smithi164.stdout: "initiated_at": "2023-06-03T20:31:39.579968+0000", 2023-06-03T20:34:36.522 INFO:teuthology.orchestra.run.smithi164.stdout: "age": 176.665736993, 2023-06-03T20:34:36.522 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.83490239099999997, 2023-06-03T20:34:36.522 INFO:teuthology.orchestra.run.smithi164.stdout: "type_data": { 2023-06-03T20:34:36.522 INFO:teuthology.orchestra.run.smithi164.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-03T20:34:36.522 INFO:teuthology.orchestra.run.smithi164.stdout: "client_info": { 2023-06-03T20:34:36.522 INFO:teuthology.orchestra.run.smithi164.stdout: "client": "client.46739", 2023-06-03T20:34:36.522 INFO:teuthology.orchestra.run.smithi164.stdout: "client_addr": "172.21.15.89:0/2479974613", 2023-06-03T20:34:36.523 INFO:teuthology.orchestra.run.smithi164.stdout: "tid": 184820 2023-06-03T20:34:36.523 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:36.523 INFO:teuthology.orchestra.run.smithi164.stdout: "events": [ 2023-06-03T20:34:36.523 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:36.523 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "initiated", 2023-06-03T20:34:36.523 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:31:39.579968+0000", 2023-06-03T20:34:36.523 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0 2023-06-03T20:34:36.523 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:36.524 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:36.524 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "throttled", 2023-06-03T20:34:36.524 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:31:39.579968+0000", 2023-06-03T20:34:36.524 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 4294967295.9999981 2023-06-03T20:34:36.524 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:36.524 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:36.525 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "header_read", 2023-06-03T20:34:36.525 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:31:39.579966+0000", 2023-06-03T20:34:36.525 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 9.9869999999999994e-06 2023-06-03T20:34:36.525 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:36.525 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:36.525 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "all_read", 2023-06-03T20:34:36.525 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:31:39.579976+0000", 2023-06-03T20:34:36.525 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 1.6949999999999999e-05 2023-06-03T20:34:36.526 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:36.526 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:36.526 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "dispatched", 2023-06-03T20:34:36.526 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:31:39.579993+0000", 2023-06-03T20:34:36.526 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 9.8050000000000001e-06 2023-06-03T20:34:36.526 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:36.526 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:36.526 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "queued_for_pg", 2023-06-03T20:34:36.527 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:31:39.580003+0000", 2023-06-03T20:34:36.527 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.00012369200000000001 2023-06-03T20:34:36.527 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:36.527 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:36.527 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "reached_pg", 2023-06-03T20:34:36.527 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:31:39.580126+0000", 2023-06-03T20:34:36.527 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 1.8567999999999999e-05 2023-06-03T20:34:36.527 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:36.528 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:36.528 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "waiting for peered", 2023-06-03T20:34:36.528 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:31:39.580145+0000", 2023-06-03T20:34:36.528 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.81628704299999999 2023-06-03T20:34:36.528 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:36.528 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:36.528 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "reached_pg", 2023-06-03T20:34:36.529 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:31:40.396432+0000", 2023-06-03T20:34:36.529 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.000143385 2023-06-03T20:34:36.529 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:36.529 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:36.529 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "started", 2023-06-03T20:34:36.529 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:31:40.396575+0000", 2023-06-03T20:34:36.529 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.00016804400000000001 2023-06-03T20:34:36.529 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:36.530 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:36.530 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "waiting for subops from 4,5", 2023-06-03T20:34:36.530 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:31:40.396743+0000", 2023-06-03T20:34:36.530 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.0089067929999999997 2023-06-03T20:34:36.530 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:36.530 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:36.530 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "op_commit", 2023-06-03T20:34:36.530 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:31:40.405650+0000", 2023-06-03T20:34:36.531 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.0056183370000000002 2023-06-03T20:34:36.531 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:36.531 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:36.531 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:34:36.531 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:31:40.411268+0000", 2023-06-03T20:34:36.531 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.0034994029999999999 2023-06-03T20:34:36.531 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:36.531 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:36.532 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:34:36.532 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:31:40.414768+0000", 2023-06-03T20:34:36.532 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 5.7021999999999997e-05 2023-06-03T20:34:36.532 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:36.532 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:36.532 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "commit_sent", 2023-06-03T20:34:36.532 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:31:40.414825+0000", 2023-06-03T20:34:36.533 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 4.5082e-05 2023-06-03T20:34:36.533 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:36.533 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:36.533 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "done", 2023-06-03T20:34:36.533 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:31:40.414870+0000", 2023-06-03T20:34:36.533 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.83490239099999997 2023-06-03T20:34:36.533 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:34:36.533 INFO:teuthology.orchestra.run.smithi164.stdout: ] 2023-06-03T20:34:36.534 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:34:36.534 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:36.534 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:36.534 INFO:teuthology.orchestra.run.smithi164.stdout: "description": "osd_op(client.46739.0:184886 14.d 14:b2fb3c75:::benchmark_data_smithi089_237281_object86870:head [delete] snapc 0=[] ondisk+write+known_if_redirected e294)", 2023-06-03T20:34:36.534 INFO:teuthology.orchestra.run.smithi164.stdout: "initiated_at": "2023-06-03T20:31:40.008440+0000", 2023-06-03T20:34:36.534 INFO:teuthology.orchestra.run.smithi164.stdout: "age": 176.23726491100001, 2023-06-03T20:34:36.534 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.40658934800000002, 2023-06-03T20:34:36.534 INFO:teuthology.orchestra.run.smithi164.stdout: "type_data": { 2023-06-03T20:34:36.535 INFO:teuthology.orchestra.run.smithi164.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-03T20:34:36.535 INFO:teuthology.orchestra.run.smithi164.stdout: "client_info": { 2023-06-03T20:34:36.535 INFO:teuthology.orchestra.run.smithi164.stdout: "client": "client.46739", 2023-06-03T20:34:36.535 INFO:teuthology.orchestra.run.smithi164.stdout: "client_addr": "172.21.15.89:0/2479974613", 2023-06-03T20:34:36.535 INFO:teuthology.orchestra.run.smithi164.stdout: "tid": 184886 2023-06-03T20:34:36.535 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:36.535 INFO:teuthology.orchestra.run.smithi164.stdout: "events": [ 2023-06-03T20:34:36.536 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:36.536 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "initiated", 2023-06-03T20:34:36.536 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:31:40.008440+0000", 2023-06-03T20:34:36.536 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0 2023-06-03T20:34:36.536 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:36.536 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:36.536 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "throttled", 2023-06-03T20:34:36.536 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:31:40.008440+0000", 2023-06-03T20:34:36.537 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 4294967295.9999967 2023-06-03T20:34:36.537 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:36.537 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:36.537 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "header_read", 2023-06-03T20:34:36.537 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:31:40.008436+0000", 2023-06-03T20:34:36.537 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 1.8238999999999999e-05 2023-06-03T20:34:36.537 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:36.537 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:36.538 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "all_read", 2023-06-03T20:34:36.538 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:31:40.008455+0000", 2023-06-03T20:34:36.538 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 4.3590000000000001e-05 2023-06-03T20:34:36.538 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:36.538 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:36.538 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "dispatched", 2023-06-03T20:34:36.538 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:31:40.008498+0000", 2023-06-03T20:34:36.539 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 2.3056000000000001e-05 2023-06-03T20:34:36.539 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:36.539 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:36.539 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "queued_for_pg", 2023-06-03T20:34:36.539 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:31:40.008521+0000", 2023-06-03T20:34:36.539 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.000180304 2023-06-03T20:34:36.539 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:36.539 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:36.540 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "reached_pg", 2023-06-03T20:34:36.540 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:31:40.008702+0000", 2023-06-03T20:34:36.540 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 1.7374e-05 2023-06-03T20:34:36.540 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:36.540 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:36.540 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "waiting for peered", 2023-06-03T20:34:36.540 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:31:40.008719+0000", 2023-06-03T20:34:36.540 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.39222180000000001 2023-06-03T20:34:36.541 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:36.541 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:36.541 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "reached_pg", 2023-06-03T20:34:36.541 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:31:40.400941+0000", 2023-06-03T20:34:36.541 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 9.8692999999999999e-05 2023-06-03T20:34:36.541 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:36.541 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:36.541 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "started", 2023-06-03T20:34:36.542 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:31:40.401039+0000", 2023-06-03T20:34:36.542 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.000116301 2023-06-03T20:34:36.542 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:36.542 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:36.542 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "waiting for subops from 4,5", 2023-06-03T20:34:36.542 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:31:40.401156+0000", 2023-06-03T20:34:36.542 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.0048011729999999997 2023-06-03T20:34:36.543 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:36.543 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:36.543 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "op_commit", 2023-06-03T20:34:36.543 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:31:40.405957+0000", 2023-06-03T20:34:36.543 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.0055197229999999998 2023-06-03T20:34:36.543 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:36.543 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:36.543 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:34:36.544 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:31:40.411477+0000", 2023-06-03T20:34:36.544 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.0034494410000000001 2023-06-03T20:34:36.544 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:36.544 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:36.544 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:34:36.544 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:31:40.414926+0000", 2023-06-03T20:34:36.544 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 5.5713e-05 2023-06-03T20:34:36.544 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:36.545 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:36.545 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "commit_sent", 2023-06-03T20:34:36.545 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:31:40.414982+0000", 2023-06-03T20:34:36.545 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 4.7216999999999999e-05 2023-06-03T20:34:36.545 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:34:36.545 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:34:36.545 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "done", 2023-06-03T20:34:36.546 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:31:40.415029+0000", 2023-06-03T20:34:36.546 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.40658934800000002 2023-06-03T20:34:36.546 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:34:36.546 INFO:teuthology.orchestra.run.smithi164.stdout: ] 2023-06-03T20:34:36.546 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:34:36.546 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:34:36.546 INFO:teuthology.orchestra.run.smithi164.stdout: ] 2023-06-03T20:34:36.546 INFO:teuthology.orchestra.run.smithi164.stdout:} 2023-06-03T20:34:36.547 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:34:36 smithi089 bash[75068]: cluster 1685824475.2712781 mgr.y (mgr.34104) 1767 : cluster 0 pgmap v1839: 73 pgs: 73 active+clean; 582 KiB data, 5.1 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:34:36.547 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:34:36 smithi164 bash[59462]: cluster 1685824475.2712781 mgr.y (mgr.34104) 1767 : cluster 0 pgmap v1839: 73 pgs: 73 active+clean; 582 KiB data, 5.1 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:34:36.548 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 20:34:36 smithi089 bash[71163]: ::ffff:172.21.15.164 - - [03/Jun/2023:20:34:36] "GET /metrics HTTP/1.1" 200 226951 "" "Prometheus/2.33.4" 2023-06-03T20:34:36.548 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:34:36 smithi089 bash[78452]: cluster 1685824475.2712781 mgr.y (mgr.34104) 1767 : cluster 0 pgmap v1839: 73 pgs: 73 active+clean; 582 KiB data, 5.1 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:34:37.237 DEBUG:teuthology.orchestra.run.smithi164:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph daemon osd.7 dump_ops_in_flight 2023-06-03T20:34:37.529 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:34:37 smithi089 bash[78452]: audit 1685824477.165563 mon.c (mon.2) 705 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T20:34:37.529 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:34:37 smithi089 bash[75068]: audit 1685824477.165563 mon.c (mon.2) 705 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T20:34:37.585 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:34:37 smithi164 bash[59462]: audit 1685824477.165563 mon.c (mon.2) 705 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T20:34:38.529 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:34:38 smithi089 bash[75068]: cluster 1685824477.271969 mgr.y (mgr.34104) 1768 : cluster 0 pgmap v1840: 73 pgs: 73 active+clean; 582 KiB data, 5.1 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:34:38.529 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:34:38 smithi089 bash[78452]: cluster 1685824477.271969 mgr.y (mgr.34104) 1768 : cluster 0 pgmap v1840: 73 pgs: 73 active+clean; 582 KiB data, 5.1 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:34:38.585 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:34:38 smithi164 bash[59462]: cluster 1685824477.271969 mgr.y (mgr.34104) 1768 : cluster 0 pgmap v1840: 73 pgs: 73 active+clean; 582 KiB data, 5.1 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:34:39.792 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/mon.a/config 2023-06-03T20:34:40.586 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:34:40 smithi164 bash[59462]: cluster 1685824479.272831 mgr.y (mgr.34104) 1769 : cluster 0 pgmap v1841: 73 pgs: 73 active+clean; 582 KiB data, 5.1 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:34:40.586 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:34:40 smithi164 bash[59462]: audit 1685824479.9639044 mon.a (mon.0) 1720 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:34:40.586 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:34:40 smithi164 bash[59462]: audit 1685824479.9759026 mon.a (mon.0) 1721 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:34:40.586 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:34:40 smithi164 bash[59462]: audit 1685824480.1155193 mon.a (mon.0) 1722 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:34:40.587 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:34:40 smithi164 bash[59462]: audit 1685824480.124241 mon.a (mon.0) 1723 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:34:40.779 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:34:40 smithi089 bash[78452]: cluster 1685824479.272831 mgr.y (mgr.34104) 1769 : cluster 0 pgmap v1841: 73 pgs: 73 active+clean; 582 KiB data, 5.1 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:34:40.779 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:34:40 smithi089 bash[78452]: audit 1685824479.9639044 mon.a (mon.0) 1720 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:34:40.779 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:34:40 smithi089 bash[78452]: audit 1685824479.9759026 mon.a (mon.0) 1721 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:34:40.780 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:34:40 smithi089 bash[78452]: audit 1685824480.1155193 mon.a (mon.0) 1722 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:34:40.780 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:34:40 smithi089 bash[78452]: audit 1685824480.124241 mon.a (mon.0) 1723 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:34:40.780 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:34:40 smithi089 bash[75068]: cluster 1685824479.272831 mgr.y (mgr.34104) 1769 : cluster 0 pgmap v1841: 73 pgs: 73 active+clean; 582 KiB data, 5.1 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:34:40.780 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:34:40 smithi089 bash[75068]: audit 1685824479.9639044 mon.a (mon.0) 1720 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:34:40.780 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:34:40 smithi089 bash[75068]: audit 1685824479.9759026 mon.a (mon.0) 1721 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:34:40.781 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:34:40 smithi089 bash[75068]: audit 1685824480.1155193 mon.a (mon.0) 1722 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:34:40.781 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:34:40 smithi089 bash[75068]: audit 1685824480.124241 mon.a (mon.0) 1723 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:34:41.452 INFO:teuthology.orchestra.run.smithi164.stderr:Inferring config /var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/mon.b/config 2023-06-03T20:34:41.529 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:34:41 smithi089 bash[75068]: audit 1685824480.6028636 mon.c (mon.2) 706 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-03T20:34:41.530 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:34:41 smithi089 bash[75068]: audit 1685824480.6043427 mon.c (mon.2) 707 : audit 1 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-03T20:34:41.530 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:34:41 smithi089 bash[75068]: audit 1685824480.6141388 mon.a (mon.0) 1724 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:34:41.530 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:34:41 smithi089 bash[78452]: audit 1685824480.6028636 mon.c (mon.2) 706 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-03T20:34:41.531 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:34:41 smithi089 bash[78452]: audit 1685824480.6043427 mon.c (mon.2) 707 : audit 1 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-03T20:34:41.531 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:34:41 smithi089 bash[78452]: audit 1685824480.6141388 mon.a (mon.0) 1724 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:34:41.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:34:41 smithi164 bash[59462]: audit 1685824480.6028636 mon.c (mon.2) 706 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-03T20:34:41.836 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:34:41 smithi164 bash[59462]: audit 1685824480.6043427 mon.c (mon.2) 707 : audit 1 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-03T20:34:41.836 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:34:41 smithi164 bash[59462]: audit 1685824480.6141388 mon.a (mon.0) 1724 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:34:42.643 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:34:42 smithi164 bash[59462]: cluster 1685824481.2735226 mgr.y (mgr.34104) 1770 : cluster 0 pgmap v1842: 73 pgs: 73 active+clean; 582 KiB data, 5.1 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:34:42.779 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:34:42 smithi089 bash[78452]: cluster 1685824481.2735226 mgr.y (mgr.34104) 1770 : cluster 0 pgmap v1842: 73 pgs: 73 active+clean; 582 KiB data, 5.1 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:34:42.779 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:34:42 smithi089 bash[75068]: cluster 1685824481.2735226 mgr.y (mgr.34104) 1770 : cluster 0 pgmap v1842: 73 pgs: 73 active+clean; 582 KiB data, 5.1 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:34:44.169 INFO:teuthology.orchestra.run.smithi089.stderr:set osd.2 primary-affinity to 1 (8655362) 2023-06-03T20:34:44.529 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:34:44 smithi089 bash[78452]: cluster 1685824483.274503 mgr.y (mgr.34104) 1771 : cluster 0 pgmap v1843: 73 pgs: 73 active+clean; 582 KiB data, 5.1 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:34:44.529 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:34:44 smithi089 bash[78452]: audit 1685824484.0632176 mon.c (mon.2) 708 : audit 1 from='client.? 172.21.15.89:0/2486053502' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 2, "weight": 1.0}]: dispatch 2023-06-03T20:34:44.529 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:34:44 smithi089 bash[78452]: audit 1685824484.0638273 mon.a (mon.0) 1725 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 2, "weight": 1.0}]: dispatch 2023-06-03T20:34:44.529 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:34:44 smithi089 bash[75068]: cluster 1685824483.274503 mgr.y (mgr.34104) 1771 : cluster 0 pgmap v1843: 73 pgs: 73 active+clean; 582 KiB data, 5.1 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:34:44.530 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:34:44 smithi089 bash[75068]: audit 1685824484.0632176 mon.c (mon.2) 708 : audit 1 from='client.? 172.21.15.89:0/2486053502' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 2, "weight": 1.0}]: dispatch 2023-06-03T20:34:44.530 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:34:44 smithi089 bash[75068]: audit 1685824484.0638273 mon.a (mon.0) 1725 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 2, "weight": 1.0}]: dispatch 2023-06-03T20:34:44.585 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:34:44 smithi164 bash[59462]: cluster 1685824483.274503 mgr.y (mgr.34104) 1771 : cluster 0 pgmap v1843: 73 pgs: 73 active+clean; 582 KiB data, 5.1 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:34:44.586 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:34:44 smithi164 bash[59462]: audit 1685824484.0632176 mon.c (mon.2) 708 : audit 1 from='client.? 172.21.15.89:0/2486053502' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 2, "weight": 1.0}]: dispatch 2023-06-03T20:34:44.586 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:34:44 smithi164 bash[59462]: audit 1685824484.0638273 mon.a (mon.0) 1725 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 2, "weight": 1.0}]: dispatch 2023-06-03T20:34:44.922 INFO:teuthology.orchestra.run.smithi164.stdout:{ 2023-06-03T20:34:44.922 INFO:teuthology.orchestra.run.smithi164.stdout: "ops": [], 2023-06-03T20:34:44.922 INFO:teuthology.orchestra.run.smithi164.stdout: "num_ops": 0 2023-06-03T20:34:44.923 INFO:teuthology.orchestra.run.smithi164.stdout:} 2023-06-03T20:34:45.097 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph osd reweight 3 1 2023-06-03T20:34:45.335 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:34:45 smithi164 bash[59462]: audit 1685824484.1570816 mon.a (mon.0) 1726 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 2, "weight": 1.0}]': finished 2023-06-03T20:34:45.336 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:34:45 smithi164 bash[59462]: cluster 1685824484.157181 mon.a (mon.0) 1727 : cluster 0 osdmap e313: 8 total, 8 up, 8 in 2023-06-03T20:34:45.336 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:34:45 smithi164 bash[59462]: cluster 1685824484.3135364 osd.7 (osd.7) 256 : cluster 3 failed to encode map e313 with expected crc 2023-06-03T20:34:45.336 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:34:45 smithi164 bash[59462]: cluster 1685824484.3137164 osd.5 (osd.5) 259 : cluster 3 failed to encode map e313 with expected crc 2023-06-03T20:34:45.336 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:34:45 smithi164 bash[59462]: cluster 1685824484.3138776 osd.6 (osd.6) 250 : cluster 3 failed to encode map e313 with expected crc 2023-06-03T20:34:45.529 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:34:45 smithi089 bash[78452]: audit 1685824484.1570816 mon.a (mon.0) 1726 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 2, "weight": 1.0}]': finished 2023-06-03T20:34:45.530 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:34:45 smithi089 bash[78452]: cluster 1685824484.157181 mon.a (mon.0) 1727 : cluster 0 osdmap e313: 8 total, 8 up, 8 in 2023-06-03T20:34:45.530 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:34:45 smithi089 bash[78452]: cluster 1685824484.3135364 osd.7 (osd.7) 256 : cluster 3 failed to encode map e313 with expected crc 2023-06-03T20:34:45.530 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:34:45 smithi089 bash[78452]: cluster 1685824484.3137164 osd.5 (osd.5) 259 : cluster 3 failed to encode map e313 with expected crc 2023-06-03T20:34:45.530 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:34:45 smithi089 bash[78452]: cluster 1685824484.3138776 osd.6 (osd.6) 250 : cluster 3 failed to encode map e313 with expected crc 2023-06-03T20:34:45.531 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:34:45 smithi089 bash[75068]: audit 1685824484.1570816 mon.a (mon.0) 1726 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 2, "weight": 1.0}]': finished 2023-06-03T20:34:45.531 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:34:45 smithi089 bash[75068]: cluster 1685824484.157181 mon.a (mon.0) 1727 : cluster 0 osdmap e313: 8 total, 8 up, 8 in 2023-06-03T20:34:45.531 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:34:45 smithi089 bash[75068]: cluster 1685824484.3135364 osd.7 (osd.7) 256 : cluster 3 failed to encode map e313 with expected crc 2023-06-03T20:34:45.532 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:34:45 smithi089 bash[75068]: cluster 1685824484.3137164 osd.5 (osd.5) 259 : cluster 3 failed to encode map e313 with expected crc 2023-06-03T20:34:45.532 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:34:45 smithi089 bash[75068]: cluster 1685824484.3138776 osd.6 (osd.6) 250 : cluster 3 failed to encode map e313 with expected crc 2023-06-03T20:34:45.885 DEBUG:teuthology.orchestra.run.smithi164:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph daemon osd.7 dump_blocked_ops 2023-06-03T20:34:46.529 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 20:34:46 smithi089 bash[71163]: ::ffff:172.21.15.164 - - [03/Jun/2023:20:34:46] "GET /metrics HTTP/1.1" 200 226932 "" "Prometheus/2.33.4" 2023-06-03T20:34:46.602 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:34:46 smithi089 bash[78452]: cluster 1685824485.2752807 mgr.y (mgr.34104) 1772 : cluster 0 pgmap v1845: 73 pgs: 73 active+clean; 582 KiB data, 5.1 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:34:46.603 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:34:46 smithi089 bash[75068]: cluster 1685824485.2752807 mgr.y (mgr.34104) 1772 : cluster 0 pgmap v1845: 73 pgs: 73 active+clean; 582 KiB data, 5.1 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:34:46.603 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:34:46 smithi164 bash[59462]: cluster 1685824485.2752807 mgr.y (mgr.34104) 1772 : cluster 0 pgmap v1845: 73 pgs: 73 active+clean; 582 KiB data, 5.1 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:34:48.779 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:34:48 smithi089 bash[78452]: cluster 1685824487.2761052 mgr.y (mgr.34104) 1773 : cluster 0 pgmap v1846: 73 pgs: 73 active+clean; 582 KiB data, 5.1 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:34:48.779 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:34:48 smithi089 bash[75068]: cluster 1685824487.2761052 mgr.y (mgr.34104) 1773 : cluster 0 pgmap v1846: 73 pgs: 73 active+clean; 582 KiB data, 5.1 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:34:48.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:34:48 smithi164 bash[59462]: cluster 1685824487.2761052 mgr.y (mgr.34104) 1773 : cluster 0 pgmap v1846: 73 pgs: 73 active+clean; 582 KiB data, 5.1 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:34:49.920 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/mon.a/config 2023-06-03T20:34:50.695 INFO:teuthology.orchestra.run.smithi164.stderr:Inferring config /var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/mon.b/config 2023-06-03T20:34:50.736 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:34:50 smithi164 bash[59462]: cluster 1685824489.2771556 mgr.y (mgr.34104) 1774 : cluster 0 pgmap v1847: 73 pgs: 73 active+clean; 582 KiB data, 5.1 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:34:50.779 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:34:50 smithi089 bash[75068]: cluster 1685824489.2771556 mgr.y (mgr.34104) 1774 : cluster 0 pgmap v1847: 73 pgs: 73 active+clean; 582 KiB data, 5.1 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:34:50.780 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:34:50 smithi089 bash[78452]: cluster 1685824489.2771556 mgr.y (mgr.34104) 1774 : cluster 0 pgmap v1847: 73 pgs: 73 active+clean; 582 KiB data, 5.1 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:34:52.585 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:34:52 smithi164 bash[59462]: cluster 1685824491.277907 mgr.y (mgr.34104) 1775 : cluster 0 pgmap v1848: 73 pgs: 73 active+clean; 582 KiB data, 5.1 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:34:52.585 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:34:52 smithi164 bash[59462]: audit 1685824492.1661558 mon.c (mon.2) 709 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T20:34:52.779 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:34:52 smithi089 bash[78452]: cluster 1685824491.277907 mgr.y (mgr.34104) 1775 : cluster 0 pgmap v1848: 73 pgs: 73 active+clean; 582 KiB data, 5.1 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:34:52.779 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:34:52 smithi089 bash[78452]: audit 1685824492.1661558 mon.c (mon.2) 709 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T20:34:52.780 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:34:52 smithi089 bash[75068]: cluster 1685824491.277907 mgr.y (mgr.34104) 1775 : cluster 0 pgmap v1848: 73 pgs: 73 active+clean; 582 KiB data, 5.1 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:34:52.780 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:34:52 smithi089 bash[75068]: audit 1685824492.1661558 mon.c (mon.2) 709 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T20:34:54.345 INFO:teuthology.orchestra.run.smithi164.stdout:{ 2023-06-03T20:34:54.346 INFO:teuthology.orchestra.run.smithi164.stdout: "ops": [], 2023-06-03T20:34:54.346 INFO:teuthology.orchestra.run.smithi164.stdout: "complaint_time": 30, 2023-06-03T20:34:54.346 INFO:teuthology.orchestra.run.smithi164.stdout: "num_blocked_ops": 0 2023-06-03T20:34:54.346 INFO:teuthology.orchestra.run.smithi164.stdout:} 2023-06-03T20:34:54.372 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:34:54 smithi164 bash[59462]: cluster 1685824493.2787747 mgr.y (mgr.34104) 1776 : cluster 0 pgmap v1849: 73 pgs: 73 active+clean; 582 KiB data, 5.1 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:34:54.529 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:34:54 smithi089 bash[75068]: cluster 1685824493.2787747 mgr.y (mgr.34104) 1776 : cluster 0 pgmap v1849: 73 pgs: 73 active+clean; 582 KiB data, 5.1 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:34:54.529 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:34:54 smithi089 bash[78452]: cluster 1685824493.2787747 mgr.y (mgr.34104) 1776 : cluster 0 pgmap v1849: 73 pgs: 73 active+clean; 582 KiB data, 5.1 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:34:55.176 DEBUG:teuthology.orchestra.run.smithi164:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph daemon osd.7 dump_historic_ops 2023-06-03T20:34:55.191 INFO:teuthology.orchestra.run.smithi089.stderr:reweighted osd.3 to 1 (10000) 2023-06-03T20:34:55.529 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:34:55 smithi089 bash[75068]: audit 1685824494.231847 mon.a (mon.0) 1728 : audit 1 from='client.? 172.21.15.89:0/1027163132' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-06-03T20:34:55.530 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:34:55 smithi089 bash[78452]: audit 1685824494.231847 mon.a (mon.0) 1728 : audit 1 from='client.? 172.21.15.89:0/1027163132' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-06-03T20:34:55.585 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:34:55 smithi164 bash[59462]: audit 1685824494.231847 mon.a (mon.0) 1728 : audit 1 from='client.? 172.21.15.89:0/1027163132' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-06-03T20:34:56.073 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph osd primary-affinity 3 1 2023-06-03T20:34:56.529 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 20:34:56 smithi089 bash[71163]: ::ffff:172.21.15.164 - - [03/Jun/2023:20:34:56] "GET /metrics HTTP/1.1" 200 226942 "" "Prometheus/2.33.4" 2023-06-03T20:34:56.529 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:34:56 smithi089 bash[75068]: audit 1685824495.1776338 mon.a (mon.0) 1729 : audit 1 from='client.? 172.21.15.89:0/1027163132' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2023-06-03T20:34:56.530 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:34:56 smithi089 bash[75068]: cluster 1685824495.1777253 mon.a (mon.0) 1730 : cluster 0 osdmap e314: 8 total, 8 up, 8 in 2023-06-03T20:34:56.530 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:34:56 smithi089 bash[75068]: cluster 1685824495.2794259 mgr.y (mgr.34104) 1777 : cluster 0 pgmap v1851: 73 pgs: 73 active+clean; 582 KiB data, 5.1 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:34:56.530 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:34:56 smithi089 bash[75068]: cluster 1685824495.307228 osd.7 (osd.7) 257 : cluster 3 failed to encode map e314 with expected crc 2023-06-03T20:34:56.530 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:34:56 smithi089 bash[78452]: audit 1685824495.1776338 mon.a (mon.0) 1729 : audit 1 from='client.? 172.21.15.89:0/1027163132' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2023-06-03T20:34:56.530 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:34:56 smithi089 bash[78452]: cluster 1685824495.1777253 mon.a (mon.0) 1730 : cluster 0 osdmap e314: 8 total, 8 up, 8 in 2023-06-03T20:34:56.531 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:34:56 smithi089 bash[78452]: cluster 1685824495.2794259 mgr.y (mgr.34104) 1777 : cluster 0 pgmap v1851: 73 pgs: 73 active+clean; 582 KiB data, 5.1 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:34:56.531 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:34:56 smithi089 bash[78452]: cluster 1685824495.307228 osd.7 (osd.7) 257 : cluster 3 failed to encode map e314 with expected crc 2023-06-03T20:34:56.585 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:34:56 smithi164 bash[59462]: audit 1685824495.1776338 mon.a (mon.0) 1729 : audit 1 from='client.? 172.21.15.89:0/1027163132' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2023-06-03T20:34:56.586 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:34:56 smithi164 bash[59462]: cluster 1685824495.1777253 mon.a (mon.0) 1730 : cluster 0 osdmap e314: 8 total, 8 up, 8 in 2023-06-03T20:34:56.586 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:34:56 smithi164 bash[59462]: cluster 1685824495.2794259 mgr.y (mgr.34104) 1777 : cluster 0 pgmap v1851: 73 pgs: 73 active+clean; 582 KiB data, 5.1 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:34:56.586 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:34:56 smithi164 bash[59462]: cluster 1685824495.307228 osd.7 (osd.7) 257 : cluster 3 failed to encode map e314 with expected crc 2023-06-03T20:34:57.529 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:34:57 smithi089 bash[75068]: cluster 1685824496.4189448 osd.5 (osd.5) 260 : cluster 3 failed to encode map e314 with expected crc 2023-06-03T20:34:57.529 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:34:57 smithi089 bash[75068]: cluster 1685824496.4193387 osd.6 (osd.6) 251 : cluster 3 failed to encode map e314 with expected crc 2023-06-03T20:34:57.530 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:34:57 smithi089 bash[78452]: cluster 1685824496.4189448 osd.5 (osd.5) 260 : cluster 3 failed to encode map e314 with expected crc 2023-06-03T20:34:57.530 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:34:57 smithi089 bash[78452]: cluster 1685824496.4193387 osd.6 (osd.6) 251 : cluster 3 failed to encode map e314 with expected crc 2023-06-03T20:34:57.585 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:34:57 smithi164 bash[59462]: cluster 1685824496.4189448 osd.5 (osd.5) 260 : cluster 3 failed to encode map e314 with expected crc 2023-06-03T20:34:57.586 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:34:57 smithi164 bash[59462]: cluster 1685824496.4193387 osd.6 (osd.6) 251 : cluster 3 failed to encode map e314 with expected crc 2023-06-03T20:34:58.529 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:34:58 smithi089 bash[78452]: cluster 1685824497.2801917 mgr.y (mgr.34104) 1778 : cluster 0 pgmap v1852: 73 pgs: 73 active+clean; 582 KiB data, 5.1 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:34:58.529 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:34:58 smithi089 bash[75068]: cluster 1685824497.2801917 mgr.y (mgr.34104) 1778 : cluster 0 pgmap v1852: 73 pgs: 73 active+clean; 582 KiB data, 5.1 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:34:58.585 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:34:58 smithi164 bash[59462]: cluster 1685824497.2801917 mgr.y (mgr.34104) 1778 : cluster 0 pgmap v1852: 73 pgs: 73 active+clean; 582 KiB data, 5.1 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:34:59.990 INFO:teuthology.orchestra.run.smithi164.stderr:Inferring config /var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/mon.b/config 2023-06-03T20:35:00.779 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:35:00 smithi089 bash[78452]: cluster 1685824499.2811437 mgr.y (mgr.34104) 1779 : cluster 0 pgmap v1853: 73 pgs: 73 active+clean; 582 KiB data, 5.1 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:35:00.779 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:35:00 smithi089 bash[75068]: cluster 1685824499.2811437 mgr.y (mgr.34104) 1779 : cluster 0 pgmap v1853: 73 pgs: 73 active+clean; 582 KiB data, 5.1 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:35:00.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:35:00 smithi164 bash[59462]: cluster 1685824499.2811437 mgr.y (mgr.34104) 1779 : cluster 0 pgmap v1853: 73 pgs: 73 active+clean; 582 KiB data, 5.1 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:35:00.885 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/mon.a/config 2023-06-03T20:35:02.529 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:35:02 smithi089 bash[78452]: cluster 1685824501.2818887 mgr.y (mgr.34104) 1780 : cluster 0 pgmap v1854: 73 pgs: 73 active+clean; 582 KiB data, 5.1 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:35:02.529 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:35:02 smithi089 bash[75068]: cluster 1685824501.2818887 mgr.y (mgr.34104) 1780 : cluster 0 pgmap v1854: 73 pgs: 73 active+clean; 582 KiB data, 5.1 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:35:02.842 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:35:02 smithi164 bash[59462]: cluster 1685824501.2818887 mgr.y (mgr.34104) 1780 : cluster 0 pgmap v1854: 73 pgs: 73 active+clean; 582 KiB data, 5.1 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:35:03.868 INFO:teuthology.orchestra.run.smithi164.stdout:{ 2023-06-03T20:35:03.869 INFO:teuthology.orchestra.run.smithi164.stdout: "size": 20, 2023-06-03T20:35:03.869 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 600, 2023-06-03T20:35:03.869 INFO:teuthology.orchestra.run.smithi164.stdout: "ops": [ 2023-06-03T20:35:03.869 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:35:03.869 INFO:teuthology.orchestra.run.smithi164.stdout: "description": "osd_op(client.36573.0:179042 13.0 13:050b037b:::benchmark_data_smithi089_211581_object63517:head [delete] snapc 0=[] ondisk+write+known_if_redirected e264)", 2023-06-03T20:35:03.869 INFO:teuthology.orchestra.run.smithi164.stdout: "initiated_at": "2023-06-03T20:25:19.574243+0000", 2023-06-03T20:35:03.870 INFO:teuthology.orchestra.run.smithi164.stdout: "age": 584.28203593600006, 2023-06-03T20:35:03.870 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.27047456199999997, 2023-06-03T20:35:03.870 INFO:teuthology.orchestra.run.smithi164.stdout: "type_data": { 2023-06-03T20:35:03.870 INFO:teuthology.orchestra.run.smithi164.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-03T20:35:03.870 INFO:teuthology.orchestra.run.smithi164.stdout: "client_info": { 2023-06-03T20:35:03.870 INFO:teuthology.orchestra.run.smithi164.stdout: "client": "client.36573", 2023-06-03T20:35:03.870 INFO:teuthology.orchestra.run.smithi164.stdout: "client_addr": "172.21.15.89:0/616445942", 2023-06-03T20:35:03.871 INFO:teuthology.orchestra.run.smithi164.stdout: "tid": 179042 2023-06-03T20:35:03.871 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:35:03.871 INFO:teuthology.orchestra.run.smithi164.stdout: "events": [ 2023-06-03T20:35:03.871 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:35:03.871 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "initiated", 2023-06-03T20:35:03.871 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:25:19.574243+0000", 2023-06-03T20:35:03.871 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0 2023-06-03T20:35:03.871 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:35:03.872 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:35:03.872 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "throttled", 2023-06-03T20:35:03.872 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:25:19.574243+0000", 2023-06-03T20:35:03.872 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 4294967295.9999967 2023-06-03T20:35:03.872 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:35:03.872 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:35:03.872 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "header_read", 2023-06-03T20:35:03.873 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:25:19.574239+0000", 2023-06-03T20:35:03.873 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 1.6515e-05 2023-06-03T20:35:03.873 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:35:03.873 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:35:03.873 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "all_read", 2023-06-03T20:35:03.873 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:25:19.574256+0000", 2023-06-03T20:35:03.873 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 2.4369e-05 2023-06-03T20:35:03.873 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:35:03.874 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:35:03.874 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "dispatched", 2023-06-03T20:35:03.874 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:25:19.574280+0000", 2023-06-03T20:35:03.874 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 1.2588e-05 2023-06-03T20:35:03.874 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:35:03.874 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:35:03.874 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "queued_for_pg", 2023-06-03T20:35:03.874 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:25:19.574293+0000", 2023-06-03T20:35:03.875 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 6.8497999999999994e-05 2023-06-03T20:35:03.875 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:35:03.875 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:35:03.875 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "reached_pg", 2023-06-03T20:35:03.875 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:25:19.574361+0000", 2023-06-03T20:35:03.875 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.00017331000000000001 2023-06-03T20:35:03.875 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:35:03.876 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:35:03.876 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "started", 2023-06-03T20:35:03.876 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:25:19.574535+0000", 2023-06-03T20:35:03.876 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.00059500299999999998 2023-06-03T20:35:03.876 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:35:03.876 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:35:03.876 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "waiting for subops from 1,2", 2023-06-03T20:35:03.876 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:25:19.575130+0000", 2023-06-03T20:35:03.877 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.0029263829999999998 2023-06-03T20:35:03.877 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:35:03.877 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:35:03.877 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:35:03.877 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:25:19.578056+0000", 2023-06-03T20:35:03.877 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 2.6979999999999999e-05 2023-06-03T20:35:03.877 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:35:03.878 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:35:03.878 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "op_commit", 2023-06-03T20:35:03.878 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:25:19.578083+0000", 2023-06-03T20:35:03.878 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.26652616000000001 2023-06-03T20:35:03.878 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:35:03.878 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:35:03.878 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:35:03.878 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:25:19.844609+0000", 2023-06-03T20:35:03.879 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 6.648e-05 2023-06-03T20:35:03.879 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:35:03.879 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:35:03.879 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "commit_sent", 2023-06-03T20:35:03.879 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:25:19.844676+0000", 2023-06-03T20:35:03.879 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 4.1692e-05 2023-06-03T20:35:03.879 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:35:03.879 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:35:03.880 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "done", 2023-06-03T20:35:03.880 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:25:19.844717+0000", 2023-06-03T20:35:03.880 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.27047456199999997 2023-06-03T20:35:03.880 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:35:03.880 INFO:teuthology.orchestra.run.smithi164.stdout: ] 2023-06-03T20:35:03.880 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:35:03.880 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:35:03.881 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:35:03.881 INFO:teuthology.orchestra.run.smithi164.stdout: "description": "osd_op(client.36573.0:209329 13.e 13:79588904:::benchmark_data_smithi089_211581_object93804:head [delete] snapc 0=[] ondisk+write+known_if_redirected e267)", 2023-06-03T20:35:03.881 INFO:teuthology.orchestra.run.smithi164.stdout: "initiated_at": "2023-06-03T20:26:07.562932+0000", 2023-06-03T20:35:03.881 INFO:teuthology.orchestra.run.smithi164.stdout: "age": 536.29334700300001, 2023-06-03T20:35:03.881 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.27694666299999998, 2023-06-03T20:35:03.881 INFO:teuthology.orchestra.run.smithi164.stdout: "type_data": { 2023-06-03T20:35:03.881 INFO:teuthology.orchestra.run.smithi164.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-03T20:35:03.881 INFO:teuthology.orchestra.run.smithi164.stdout: "client_info": { 2023-06-03T20:35:03.882 INFO:teuthology.orchestra.run.smithi164.stdout: "client": "client.36573", 2023-06-03T20:35:03.882 INFO:teuthology.orchestra.run.smithi164.stdout: "client_addr": "172.21.15.89:0/616445942", 2023-06-03T20:35:03.882 INFO:teuthology.orchestra.run.smithi164.stdout: "tid": 209329 2023-06-03T20:35:03.882 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:35:03.882 INFO:teuthology.orchestra.run.smithi164.stdout: "events": [ 2023-06-03T20:35:03.882 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:35:03.882 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "initiated", 2023-06-03T20:35:03.882 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:26:07.562932+0000", 2023-06-03T20:35:03.883 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0 2023-06-03T20:35:03.883 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:35:03.883 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:35:03.883 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "throttled", 2023-06-03T20:35:03.883 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:26:07.562932+0000", 2023-06-03T20:35:03.883 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 4294967295.9999981 2023-06-03T20:35:03.883 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:35:03.884 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:35:03.884 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "header_read", 2023-06-03T20:35:03.884 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:26:07.562930+0000", 2023-06-03T20:35:03.884 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 8.4829999999999999e-06 2023-06-03T20:35:03.884 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:35:03.884 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:35:03.884 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "all_read", 2023-06-03T20:35:03.884 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:26:07.562938+0000", 2023-06-03T20:35:03.885 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 1.4404e-05 2023-06-03T20:35:03.885 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:35:03.885 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:35:03.885 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "dispatched", 2023-06-03T20:35:03.885 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:26:07.562953+0000", 2023-06-03T20:35:03.885 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 6.691e-06 2023-06-03T20:35:03.885 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:35:03.885 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:35:03.886 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "queued_for_pg", 2023-06-03T20:35:03.886 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:26:07.562960+0000", 2023-06-03T20:35:03.886 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.018504244999999999 2023-06-03T20:35:03.886 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:35:03.886 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:35:03.886 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "reached_pg", 2023-06-03T20:35:03.886 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:26:07.581464+0000", 2023-06-03T20:35:03.886 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.000130693 2023-06-03T20:35:03.887 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:35:03.887 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:35:03.887 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "waiting for rw locks", 2023-06-03T20:35:03.887 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:26:07.581595+0000", 2023-06-03T20:35:03.887 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.224686576 2023-06-03T20:35:03.887 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:35:03.887 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:35:03.888 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "reached_pg", 2023-06-03T20:35:03.888 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:26:07.806281+0000", 2023-06-03T20:35:03.888 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.00015249600000000001 2023-06-03T20:35:03.888 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:35:03.888 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:35:03.888 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "started", 2023-06-03T20:35:03.888 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:26:07.806434+0000", 2023-06-03T20:35:03.888 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.000228619 2023-06-03T20:35:03.889 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:35:03.889 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:35:03.889 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "waiting for subops from 4,5", 2023-06-03T20:35:03.889 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:26:07.806662+0000", 2023-06-03T20:35:03.889 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.018756116 2023-06-03T20:35:03.889 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:35:03.889 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:35:03.889 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "op_commit", 2023-06-03T20:35:03.890 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:26:07.825418+0000", 2023-06-03T20:35:03.890 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.0077602460000000002 2023-06-03T20:35:03.890 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:35:03.890 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:35:03.890 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:35:03.890 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:26:07.833179+0000", 2023-06-03T20:35:03.890 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.0041320870000000004 2023-06-03T20:35:03.891 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:35:03.891 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:35:03.891 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:35:03.891 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:26:07.837311+0000", 2023-06-03T20:35:03.891 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.00016003999999999999 2023-06-03T20:35:03.891 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:35:03.891 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:35:03.891 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "commit_sent", 2023-06-03T20:35:03.892 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:26:07.837471+0000", 2023-06-03T20:35:03.892 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.0024077040000000001 2023-06-03T20:35:03.892 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:35:03.892 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:35:03.892 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "done", 2023-06-03T20:35:03.892 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:26:07.839878+0000", 2023-06-03T20:35:03.892 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.27694666299999998 2023-06-03T20:35:03.893 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:35:03.893 INFO:teuthology.orchestra.run.smithi164.stdout: ] 2023-06-03T20:35:03.893 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:35:03.893 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:35:03.893 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:35:03.893 INFO:teuthology.orchestra.run.smithi164.stdout: "description": "osd_op(client.46739.0:24969 14.f 14:f22d6744:::benchmark_data_smithi089_237281_object24968:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e281)", 2023-06-03T20:35:03.893 INFO:teuthology.orchestra.run.smithi164.stdout: "initiated_at": "2023-06-03T20:27:59.132834+0000", 2023-06-03T20:35:03.893 INFO:teuthology.orchestra.run.smithi164.stdout: "age": 424.72344450600002, 2023-06-03T20:35:03.894 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.147327913, 2023-06-03T20:35:03.894 INFO:teuthology.orchestra.run.smithi164.stdout: "type_data": { 2023-06-03T20:35:03.894 INFO:teuthology.orchestra.run.smithi164.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-03T20:35:03.894 INFO:teuthology.orchestra.run.smithi164.stdout: "client_info": { 2023-06-03T20:35:03.894 INFO:teuthology.orchestra.run.smithi164.stdout: "client": "client.46739", 2023-06-03T20:35:03.894 INFO:teuthology.orchestra.run.smithi164.stdout: "client_addr": "172.21.15.89:0/2479974613", 2023-06-03T20:35:03.894 INFO:teuthology.orchestra.run.smithi164.stdout: "tid": 24969 2023-06-03T20:35:03.895 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:35:03.895 INFO:teuthology.orchestra.run.smithi164.stdout: "events": [ 2023-06-03T20:35:03.895 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:35:03.895 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "initiated", 2023-06-03T20:35:03.895 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:27:59.132834+0000", 2023-06-03T20:35:03.895 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0 2023-06-03T20:35:03.895 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:35:03.895 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:35:03.896 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "throttled", 2023-06-03T20:35:03.896 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:27:59.132834+0000", 2023-06-03T20:35:03.896 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 4294967295.9999971 2023-06-03T20:35:03.896 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:35:03.896 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:35:03.896 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "header_read", 2023-06-03T20:35:03.896 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:27:59.132831+0000", 2023-06-03T20:35:03.897 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 6.5178000000000005e-05 2023-06-03T20:35:03.897 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:35:03.897 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:35:03.897 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "all_read", 2023-06-03T20:35:03.897 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:27:59.132896+0000", 2023-06-03T20:35:03.897 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 3.294e-05 2023-06-03T20:35:03.897 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:35:03.897 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:35:03.898 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "dispatched", 2023-06-03T20:35:03.898 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:27:59.132929+0000", 2023-06-03T20:35:03.898 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 1.3261000000000001e-05 2023-06-03T20:35:03.898 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:35:03.898 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:35:03.898 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "queued_for_pg", 2023-06-03T20:35:03.898 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:27:59.132943+0000", 2023-06-03T20:35:03.898 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.00015296299999999999 2023-06-03T20:35:03.899 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:35:03.899 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:35:03.899 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "reached_pg", 2023-06-03T20:35:03.899 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:27:59.133096+0000", 2023-06-03T20:35:03.899 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.00023025500000000001 2023-06-03T20:35:03.899 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:35:03.899 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:35:03.899 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "started", 2023-06-03T20:35:03.900 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:27:59.133326+0000", 2023-06-03T20:35:03.900 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.00027385400000000002 2023-06-03T20:35:03.900 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:35:03.900 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:35:03.900 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "waiting for subops from 1,3", 2023-06-03T20:35:03.900 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:27:59.133600+0000", 2023-06-03T20:35:03.900 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.0026968980000000001 2023-06-03T20:35:03.901 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:35:03.901 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:35:03.901 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "op_commit", 2023-06-03T20:35:03.901 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:27:59.136297+0000", 2023-06-03T20:35:03.901 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.14167355100000001 2023-06-03T20:35:03.901 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:35:03.901 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:35:03.901 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:35:03.902 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:27:59.277970+0000", 2023-06-03T20:35:03.902 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.0019668329999999999 2023-06-03T20:35:03.902 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:35:03.902 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:35:03.902 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:35:03.902 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:27:59.279937+0000", 2023-06-03T20:35:03.902 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.00013572000000000001 2023-06-03T20:35:03.902 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:35:03.903 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:35:03.903 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "commit_sent", 2023-06-03T20:35:03.903 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:27:59.280073+0000", 2023-06-03T20:35:03.903 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 8.9468000000000005e-05 2023-06-03T20:35:03.903 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:35:03.903 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:35:03.903 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "done", 2023-06-03T20:35:03.904 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:27:59.280162+0000", 2023-06-03T20:35:03.904 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.147327913 2023-06-03T20:35:03.904 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:35:03.904 INFO:teuthology.orchestra.run.smithi164.stdout: ] 2023-06-03T20:35:03.904 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:35:03.904 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:35:03.904 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:35:03.904 INFO:teuthology.orchestra.run.smithi164.stdout: "description": "osd_op(client.46739.0:24970 14.f 14:f39260e7:::benchmark_data_smithi089_237281_object24969:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e281)", 2023-06-03T20:35:03.905 INFO:teuthology.orchestra.run.smithi164.stdout: "initiated_at": "2023-06-03T20:27:59.132965+0000", 2023-06-03T20:35:03.905 INFO:teuthology.orchestra.run.smithi164.stdout: "age": 424.723313942, 2023-06-03T20:35:03.905 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.147497084, 2023-06-03T20:35:03.905 INFO:teuthology.orchestra.run.smithi164.stdout: "type_data": { 2023-06-03T20:35:03.905 INFO:teuthology.orchestra.run.smithi164.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-03T20:35:03.905 INFO:teuthology.orchestra.run.smithi164.stdout: "client_info": { 2023-06-03T20:35:03.905 INFO:teuthology.orchestra.run.smithi164.stdout: "client": "client.46739", 2023-06-03T20:35:03.906 INFO:teuthology.orchestra.run.smithi164.stdout: "client_addr": "172.21.15.89:0/2479974613", 2023-06-03T20:35:03.906 INFO:teuthology.orchestra.run.smithi164.stdout: "tid": 24970 2023-06-03T20:35:03.906 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:35:03.906 INFO:teuthology.orchestra.run.smithi164.stdout: "events": [ 2023-06-03T20:35:03.906 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:35:03.906 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "initiated", 2023-06-03T20:35:03.906 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:27:59.132965+0000", 2023-06-03T20:35:03.906 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0 2023-06-03T20:35:03.907 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:35:03.907 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:35:03.907 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "throttled", 2023-06-03T20:35:03.907 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:27:59.132965+0000", 2023-06-03T20:35:03.907 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 4294967295.999999 2023-06-03T20:35:03.907 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:35:03.907 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:35:03.907 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "header_read", 2023-06-03T20:35:03.908 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:27:59.132964+0000", 2023-06-03T20:35:03.908 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 4.4230000000000002e-05 2023-06-03T20:35:03.908 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:35:03.908 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:35:03.908 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "all_read", 2023-06-03T20:35:03.908 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:27:59.133008+0000", 2023-06-03T20:35:03.908 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 1.6076999999999999e-05 2023-06-03T20:35:03.909 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:35:03.909 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:35:03.909 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "dispatched", 2023-06-03T20:35:03.909 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:27:59.133024+0000", 2023-06-03T20:35:03.909 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 7.5870000000000004e-06 2023-06-03T20:35:03.909 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:35:03.909 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:35:03.909 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "queued_for_pg", 2023-06-03T20:35:03.910 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:27:59.133032+0000", 2023-06-03T20:35:03.910 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.0019720470000000002 2023-06-03T20:35:03.910 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:35:03.910 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:35:03.910 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "reached_pg", 2023-06-03T20:35:03.910 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:27:59.135004+0000", 2023-06-03T20:35:03.910 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.00020113100000000001 2023-06-03T20:35:03.910 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:35:03.911 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:35:03.911 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "started", 2023-06-03T20:35:03.911 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:27:59.135205+0000", 2023-06-03T20:35:03.911 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.00017030800000000001 2023-06-03T20:35:03.911 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:35:03.911 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:35:03.911 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "waiting for subops from 1,3", 2023-06-03T20:35:03.912 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:27:59.135375+0000", 2023-06-03T20:35:03.912 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.001089189 2023-06-03T20:35:03.912 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:35:03.912 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:35:03.912 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "op_commit", 2023-06-03T20:35:03.912 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:27:59.136464+0000", 2023-06-03T20:35:03.912 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.141691442 2023-06-03T20:35:03.912 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:35:03.913 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:35:03.913 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:35:03.913 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:27:59.278156+0000", 2023-06-03T20:35:03.913 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.0021012919999999998 2023-06-03T20:35:03.913 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:35:03.913 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:35:03.913 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:35:03.914 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:27:59.280257+0000", 2023-06-03T20:35:03.914 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.00011642 2023-06-03T20:35:03.914 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:35:03.914 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:35:03.914 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "commit_sent", 2023-06-03T20:35:03.914 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:27:59.280373+0000", 2023-06-03T20:35:03.914 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 8.8424000000000006e-05 2023-06-03T20:35:03.914 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:35:03.915 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:35:03.915 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "done", 2023-06-03T20:35:03.915 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:27:59.280462+0000", 2023-06-03T20:35:03.915 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.147497084 2023-06-03T20:35:03.915 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:35:03.915 INFO:teuthology.orchestra.run.smithi164.stdout: ] 2023-06-03T20:35:03.915 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:35:03.915 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:35:03.916 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:35:03.916 INFO:teuthology.orchestra.run.smithi164.stdout: "description": "osd_op(client.46739.0:33436 14.f 14:fb1ec335:::benchmark_data_smithi089_237281_object33435:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e281)", 2023-06-03T20:35:03.916 INFO:teuthology.orchestra.run.smithi164.stdout: "initiated_at": "2023-06-03T20:28:04.609463+0000", 2023-06-03T20:35:03.916 INFO:teuthology.orchestra.run.smithi164.stdout: "age": 419.24681545999999, 2023-06-03T20:35:03.916 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.14904371299999999, 2023-06-03T20:35:03.916 INFO:teuthology.orchestra.run.smithi164.stdout: "type_data": { 2023-06-03T20:35:03.916 INFO:teuthology.orchestra.run.smithi164.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-03T20:35:03.917 INFO:teuthology.orchestra.run.smithi164.stdout: "client_info": { 2023-06-03T20:35:03.917 INFO:teuthology.orchestra.run.smithi164.stdout: "client": "client.46739", 2023-06-03T20:35:03.917 INFO:teuthology.orchestra.run.smithi164.stdout: "client_addr": "172.21.15.89:0/2479974613", 2023-06-03T20:35:03.917 INFO:teuthology.orchestra.run.smithi164.stdout: "tid": 33436 2023-06-03T20:35:03.917 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:35:03.917 INFO:teuthology.orchestra.run.smithi164.stdout: "events": [ 2023-06-03T20:35:03.917 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:35:03.917 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "initiated", 2023-06-03T20:35:03.918 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:28:04.609463+0000", 2023-06-03T20:35:03.918 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0 2023-06-03T20:35:03.918 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:35:03.918 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:35:03.918 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "throttled", 2023-06-03T20:35:03.918 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:28:04.609463+0000", 2023-06-03T20:35:03.918 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 4294967295.9999981 2023-06-03T20:35:03.919 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:35:03.919 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:35:03.919 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "header_read", 2023-06-03T20:35:03.919 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:28:04.609461+0000", 2023-06-03T20:35:03.919 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.000103699 2023-06-03T20:35:03.919 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:35:03.919 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:35:03.919 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "all_read", 2023-06-03T20:35:03.920 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:28:04.609565+0000", 2023-06-03T20:35:03.920 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 1.3132999999999999e-05 2023-06-03T20:35:03.920 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:35:03.920 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:35:03.920 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "dispatched", 2023-06-03T20:35:03.920 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:28:04.609578+0000", 2023-06-03T20:35:03.920 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 5.7880000000000003e-06 2023-06-03T20:35:03.920 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:35:03.921 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:35:03.921 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "queued_for_pg", 2023-06-03T20:35:03.921 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:28:04.609584+0000", 2023-06-03T20:35:03.921 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 6.4024999999999998e-05 2023-06-03T20:35:03.921 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:35:03.921 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:35:03.921 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "reached_pg", 2023-06-03T20:35:03.922 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:28:04.609648+0000", 2023-06-03T20:35:03.922 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.00020809600000000001 2023-06-03T20:35:03.922 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:35:03.922 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:35:03.922 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "started", 2023-06-03T20:35:03.922 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:28:04.609856+0000", 2023-06-03T20:35:03.922 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.00015433299999999999 2023-06-03T20:35:03.922 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:35:03.923 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:35:03.923 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "waiting for subops from 1,3", 2023-06-03T20:35:03.923 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:28:04.610010+0000", 2023-06-03T20:35:03.923 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.001717583 2023-06-03T20:35:03.923 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:35:03.923 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:35:03.923 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:35:03.924 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:28:04.611728+0000", 2023-06-03T20:35:03.924 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.000104706 2023-06-03T20:35:03.924 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:35:03.924 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:35:03.924 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "op_commit", 2023-06-03T20:35:03.924 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:28:04.611833+0000", 2023-06-03T20:35:03.924 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.14639500899999999 2023-06-03T20:35:03.924 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:35:03.925 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:35:03.925 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:35:03.925 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:28:04.758228+0000", 2023-06-03T20:35:03.925 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.00014817399999999999 2023-06-03T20:35:03.925 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:35:03.925 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:35:03.925 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "commit_sent", 2023-06-03T20:35:03.925 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:28:04.758376+0000", 2023-06-03T20:35:03.926 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.00013121199999999999 2023-06-03T20:35:03.926 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:35:03.926 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:35:03.926 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "done", 2023-06-03T20:35:03.926 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:28:04.758507+0000", 2023-06-03T20:35:03.926 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.14904371299999999 2023-06-03T20:35:03.926 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:35:03.926 INFO:teuthology.orchestra.run.smithi164.stdout: ] 2023-06-03T20:35:03.927 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:35:03.927 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:35:03.927 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:35:03.927 INFO:teuthology.orchestra.run.smithi164.stdout: "description": "osd_op(client.46739.0:33451 14.f 14:fc5de881:::benchmark_data_smithi089_237281_object33450:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e281)", 2023-06-03T20:35:03.927 INFO:teuthology.orchestra.run.smithi164.stdout: "initiated_at": "2023-06-03T20:28:04.616895+0000", 2023-06-03T20:35:03.927 INFO:teuthology.orchestra.run.smithi164.stdout: "age": 419.23938346400001, 2023-06-03T20:35:03.927 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.143399095, 2023-06-03T20:35:03.928 INFO:teuthology.orchestra.run.smithi164.stdout: "type_data": { 2023-06-03T20:35:03.928 INFO:teuthology.orchestra.run.smithi164.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-03T20:35:03.928 INFO:teuthology.orchestra.run.smithi164.stdout: "client_info": { 2023-06-03T20:35:03.928 INFO:teuthology.orchestra.run.smithi164.stdout: "client": "client.46739", 2023-06-03T20:35:03.928 INFO:teuthology.orchestra.run.smithi164.stdout: "client_addr": "172.21.15.89:0/2479974613", 2023-06-03T20:35:03.928 INFO:teuthology.orchestra.run.smithi164.stdout: "tid": 33451 2023-06-03T20:35:03.928 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:35:03.928 INFO:teuthology.orchestra.run.smithi164.stdout: "events": [ 2023-06-03T20:35:03.929 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:35:03.929 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "initiated", 2023-06-03T20:35:03.929 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:28:04.616895+0000", 2023-06-03T20:35:03.929 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0 2023-06-03T20:35:03.929 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:35:03.929 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:35:03.929 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "throttled", 2023-06-03T20:35:03.929 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:28:04.616895+0000", 2023-06-03T20:35:03.930 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 4294967295.9999976 2023-06-03T20:35:03.930 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:35:03.930 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:35:03.930 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "header_read", 2023-06-03T20:35:03.930 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:28:04.616893+0000", 2023-06-03T20:35:03.930 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 4.2107999999999997e-05 2023-06-03T20:35:03.930 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:35:03.930 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:35:03.931 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "all_read", 2023-06-03T20:35:03.931 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:28:04.616935+0000", 2023-06-03T20:35:03.931 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 1.9488e-05 2023-06-03T20:35:03.931 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:35:03.931 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:35:03.931 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "dispatched", 2023-06-03T20:35:03.931 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:28:04.616954+0000", 2023-06-03T20:35:03.931 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 7.5159999999999997e-06 2023-06-03T20:35:03.932 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:35:03.932 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:35:03.932 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "queued_for_pg", 2023-06-03T20:35:03.932 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:28:04.616962+0000", 2023-06-03T20:35:03.932 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 6.5659000000000003e-05 2023-06-03T20:35:03.932 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:35:03.932 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:35:03.933 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "reached_pg", 2023-06-03T20:35:03.933 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:28:04.617027+0000", 2023-06-03T20:35:03.933 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.00015551000000000001 2023-06-03T20:35:03.933 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:35:03.933 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:35:03.933 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "started", 2023-06-03T20:35:03.933 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:28:04.617183+0000", 2023-06-03T20:35:03.933 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.00018781399999999999 2023-06-03T20:35:03.934 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:35:03.934 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:35:03.934 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "waiting for subops from 1,3", 2023-06-03T20:35:03.934 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:28:04.617371+0000", 2023-06-03T20:35:03.934 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.0014337200000000001 2023-06-03T20:35:03.934 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:35:03.934 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:35:03.934 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "op_commit", 2023-06-03T20:35:03.935 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:28:04.618804+0000", 2023-06-03T20:35:03.935 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.002080681 2023-06-03T20:35:03.935 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:35:03.935 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:35:03.935 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:35:03.935 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:28:04.620885+0000", 2023-06-03T20:35:03.935 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.13905435399999999 2023-06-03T20:35:03.935 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:35:03.936 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:35:03.936 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:35:03.936 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:28:04.759940+0000", 2023-06-03T20:35:03.936 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.00013550199999999999 2023-06-03T20:35:03.936 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:35:03.936 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:35:03.936 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "commit_sent", 2023-06-03T20:35:03.937 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:28:04.760075+0000", 2023-06-03T20:35:03.937 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.00021935600000000001 2023-06-03T20:35:03.937 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:35:03.937 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:35:03.937 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "done", 2023-06-03T20:35:03.937 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:28:04.760294+0000", 2023-06-03T20:35:03.937 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.143399095 2023-06-03T20:35:03.937 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:35:03.938 INFO:teuthology.orchestra.run.smithi164.stdout: ] 2023-06-03T20:35:03.938 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:35:03.938 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:35:03.938 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:35:03.938 INFO:teuthology.orchestra.run.smithi164.stdout: "description": "osd_op(client.46739.0:104850 14.f 14:f5d8f815:::benchmark_data_smithi089_237281_object6834:head [delete] snapc 0=[] ondisk+write+known_if_redirected e289)", 2023-06-03T20:35:03.938 INFO:teuthology.orchestra.run.smithi164.stdout: "initiated_at": "2023-06-03T20:29:26.510289+0000", 2023-06-03T20:35:03.938 INFO:teuthology.orchestra.run.smithi164.stdout: "age": 337.34598980800001, 2023-06-03T20:35:03.939 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.15454975600000001, 2023-06-03T20:35:03.939 INFO:teuthology.orchestra.run.smithi164.stdout: "type_data": { 2023-06-03T20:35:03.939 INFO:teuthology.orchestra.run.smithi164.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-03T20:35:03.939 INFO:teuthology.orchestra.run.smithi164.stdout: "client_info": { 2023-06-03T20:35:03.939 INFO:teuthology.orchestra.run.smithi164.stdout: "client": "client.46739", 2023-06-03T20:35:03.939 INFO:teuthology.orchestra.run.smithi164.stdout: "client_addr": "172.21.15.89:0/2479974613", 2023-06-03T20:35:03.939 INFO:teuthology.orchestra.run.smithi164.stdout: "tid": 104850 2023-06-03T20:35:03.939 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:35:03.940 INFO:teuthology.orchestra.run.smithi164.stdout: "events": [ 2023-06-03T20:35:03.940 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:35:03.940 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "initiated", 2023-06-03T20:35:03.940 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:29:26.510289+0000", 2023-06-03T20:35:03.940 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0 2023-06-03T20:35:03.940 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:35:03.940 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:35:03.940 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "throttled", 2023-06-03T20:35:03.941 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:29:26.510289+0000", 2023-06-03T20:35:03.941 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 4294967295.9999981 2023-06-03T20:35:03.941 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:35:03.941 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:35:03.941 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "header_read", 2023-06-03T20:35:03.941 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:29:26.510287+0000", 2023-06-03T20:35:03.941 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 1.0525e-05 2023-06-03T20:35:03.941 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:35:03.942 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:35:03.942 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "all_read", 2023-06-03T20:35:03.942 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:29:26.510297+0000", 2023-06-03T20:35:03.942 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 2.0052999999999999e-05 2023-06-03T20:35:03.942 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:35:03.942 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:35:03.942 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "dispatched", 2023-06-03T20:35:03.943 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:29:26.510318+0000", 2023-06-03T20:35:03.943 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 9.0250000000000008e-06 2023-06-03T20:35:03.943 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:35:03.943 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:35:03.943 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "queued_for_pg", 2023-06-03T20:35:03.943 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:29:26.510327+0000", 2023-06-03T20:35:03.943 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.0054459520000000004 2023-06-03T20:35:03.944 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:35:03.944 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:35:03.944 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "reached_pg", 2023-06-03T20:35:03.944 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:29:26.515772+0000", 2023-06-03T20:35:03.944 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.00015307 2023-06-03T20:35:03.944 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:35:03.944 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:35:03.944 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "started", 2023-06-03T20:35:03.945 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:29:26.515926+0000", 2023-06-03T20:35:03.945 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.00016083100000000001 2023-06-03T20:35:03.945 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:35:03.945 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:35:03.945 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "waiting for subops from 1,3", 2023-06-03T20:35:03.945 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:29:26.516086+0000", 2023-06-03T20:35:03.945 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.0036087329999999998 2023-06-03T20:35:03.945 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:35:03.946 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:35:03.946 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "op_commit", 2023-06-03T20:35:03.946 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:29:26.519695+0000", 2023-06-03T20:35:03.946 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.0046606729999999997 2023-06-03T20:35:03.946 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:35:03.946 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:35:03.946 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:35:03.946 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:29:26.524356+0000", 2023-06-03T20:35:03.947 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.14036491500000001 2023-06-03T20:35:03.947 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:35:03.947 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:35:03.947 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:35:03.947 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:29:26.664721+0000", 2023-06-03T20:35:03.947 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 6.3592999999999999e-05 2023-06-03T20:35:03.947 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:35:03.948 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:35:03.948 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "commit_sent", 2023-06-03T20:35:03.948 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:29:26.664784+0000", 2023-06-03T20:35:03.948 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 5.4384e-05 2023-06-03T20:35:03.948 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:35:03.948 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:35:03.948 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "done", 2023-06-03T20:35:03.948 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:29:26.664839+0000", 2023-06-03T20:35:03.949 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.15454975600000001 2023-06-03T20:35:03.949 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:35:03.949 INFO:teuthology.orchestra.run.smithi164.stdout: ] 2023-06-03T20:35:03.949 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:35:03.949 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:35:03.949 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:35:03.949 INFO:teuthology.orchestra.run.smithi164.stdout: "description": "osd_op(client.46739.0:138091 14.f 14:fee360bb:::benchmark_data_smithi089_237281_object40075:head [delete] snapc 0=[] ondisk+write+known_if_redirected e291)", 2023-06-03T20:35:03.950 INFO:teuthology.orchestra.run.smithi164.stdout: "initiated_at": "2023-06-03T20:30:18.875116+0000", 2023-06-03T20:35:03.950 INFO:teuthology.orchestra.run.smithi164.stdout: "age": 284.981163004, 2023-06-03T20:35:03.950 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.171107805, 2023-06-03T20:35:03.950 INFO:teuthology.orchestra.run.smithi164.stdout: "type_data": { 2023-06-03T20:35:03.950 INFO:teuthology.orchestra.run.smithi164.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-03T20:35:03.950 INFO:teuthology.orchestra.run.smithi164.stdout: "client_info": { 2023-06-03T20:35:03.950 INFO:teuthology.orchestra.run.smithi164.stdout: "client": "client.46739", 2023-06-03T20:35:03.950 INFO:teuthology.orchestra.run.smithi164.stdout: "client_addr": "172.21.15.89:0/2479974613", 2023-06-03T20:35:03.951 INFO:teuthology.orchestra.run.smithi164.stdout: "tid": 138091 2023-06-03T20:35:03.951 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:35:03.951 INFO:teuthology.orchestra.run.smithi164.stdout: "events": [ 2023-06-03T20:35:03.951 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:35:03.951 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "initiated", 2023-06-03T20:35:03.951 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:30:18.875116+0000", 2023-06-03T20:35:03.951 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0 2023-06-03T20:35:03.951 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:35:03.952 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:35:03.952 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "throttled", 2023-06-03T20:35:03.952 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:30:18.875116+0000", 2023-06-03T20:35:03.952 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 4294967295.9999943 2023-06-03T20:35:03.952 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:35:03.952 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:35:03.952 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "header_read", 2023-06-03T20:35:03.953 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:30:18.875110+0000", 2023-06-03T20:35:03.953 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 1.3582e-05 2023-06-03T20:35:03.953 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:35:03.953 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:35:03.953 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "all_read", 2023-06-03T20:35:03.953 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:30:18.875124+0000", 2023-06-03T20:35:03.953 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 2.5848000000000001e-05 2023-06-03T20:35:03.953 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:35:03.954 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:35:03.954 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "dispatched", 2023-06-03T20:35:03.954 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:30:18.875149+0000", 2023-06-03T20:35:03.954 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 7.0530000000000001e-06 2023-06-03T20:35:03.954 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:35:03.954 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:35:03.954 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "queued_for_pg", 2023-06-03T20:35:03.954 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:30:18.875157+0000", 2023-06-03T20:35:03.955 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 5.7871000000000002e-05 2023-06-03T20:35:03.955 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:35:03.955 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:35:03.955 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "reached_pg", 2023-06-03T20:35:03.955 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:30:18.875214+0000", 2023-06-03T20:35:03.955 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.00012566300000000001 2023-06-03T20:35:03.955 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:35:03.955 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:35:03.956 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "started", 2023-06-03T20:35:03.956 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:30:18.875340+0000", 2023-06-03T20:35:03.956 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.00013453200000000001 2023-06-03T20:35:03.956 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:35:03.956 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:35:03.956 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "waiting for subops from 1,3", 2023-06-03T20:35:03.956 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:30:18.875475+0000", 2023-06-03T20:35:03.957 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.0054103220000000004 2023-06-03T20:35:03.957 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:35:03.957 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:35:03.957 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "op_commit", 2023-06-03T20:35:03.957 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:30:18.880885+0000", 2023-06-03T20:35:03.957 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.00036923499999999998 2023-06-03T20:35:03.957 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:35:03.957 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:35:03.958 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:35:03.958 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:30:18.881254+0000", 2023-06-03T20:35:03.958 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.16489546499999999 2023-06-03T20:35:03.958 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:35:03.958 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:35:03.958 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:35:03.958 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:30:19.046150+0000", 2023-06-03T20:35:03.958 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 4.1975999999999999e-05 2023-06-03T20:35:03.959 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:35:03.959 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:35:03.959 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "commit_sent", 2023-06-03T20:35:03.959 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:30:19.046192+0000", 2023-06-03T20:35:03.959 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 3.1928999999999999e-05 2023-06-03T20:35:03.959 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:35:03.959 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:35:03.960 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "done", 2023-06-03T20:35:03.960 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:30:19.046224+0000", 2023-06-03T20:35:03.960 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.171107805 2023-06-03T20:35:03.960 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:35:03.960 INFO:teuthology.orchestra.run.smithi164.stdout: ] 2023-06-03T20:35:03.960 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:35:03.960 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:35:03.960 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:35:03.961 INFO:teuthology.orchestra.run.smithi164.stdout: "description": "osd_op(client.46739.0:138096 14.f 14:ff56730b:::benchmark_data_smithi089_237281_object40080:head [delete] snapc 0=[] ondisk+write+known_if_redirected e291)", 2023-06-03T20:35:03.961 INFO:teuthology.orchestra.run.smithi164.stdout: "initiated_at": "2023-06-03T20:30:18.885920+0000", 2023-06-03T20:35:03.961 INFO:teuthology.orchestra.run.smithi164.stdout: "age": 284.97035893700001, 2023-06-03T20:35:03.961 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.160410791, 2023-06-03T20:35:03.961 INFO:teuthology.orchestra.run.smithi164.stdout: "type_data": { 2023-06-03T20:35:03.961 INFO:teuthology.orchestra.run.smithi164.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-03T20:35:03.961 INFO:teuthology.orchestra.run.smithi164.stdout: "client_info": { 2023-06-03T20:35:03.961 INFO:teuthology.orchestra.run.smithi164.stdout: "client": "client.46739", 2023-06-03T20:35:03.962 INFO:teuthology.orchestra.run.smithi164.stdout: "client_addr": "172.21.15.89:0/2479974613", 2023-06-03T20:35:03.962 INFO:teuthology.orchestra.run.smithi164.stdout: "tid": 138096 2023-06-03T20:35:03.962 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:35:03.962 INFO:teuthology.orchestra.run.smithi164.stdout: "events": [ 2023-06-03T20:35:03.962 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:35:03.962 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "initiated", 2023-06-03T20:35:03.962 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:30:18.885920+0000", 2023-06-03T20:35:03.963 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0 2023-06-03T20:35:03.963 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:35:03.963 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:35:03.963 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "throttled", 2023-06-03T20:35:03.963 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:30:18.885920+0000", 2023-06-03T20:35:03.963 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 4294967295.9999981 2023-06-03T20:35:03.963 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:35:03.963 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:35:03.964 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "header_read", 2023-06-03T20:35:03.964 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:30:18.885918+0000", 2023-06-03T20:35:03.964 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 1.0587999999999999e-05 2023-06-03T20:35:03.964 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:35:03.964 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:35:03.964 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "all_read", 2023-06-03T20:35:03.964 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:30:18.885928+0000", 2023-06-03T20:35:03.964 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 1.6424999999999999e-05 2023-06-03T20:35:03.965 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:35:03.965 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:35:03.965 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "dispatched", 2023-06-03T20:35:03.965 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:30:18.885945+0000", 2023-06-03T20:35:03.965 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 7.7980000000000006e-06 2023-06-03T20:35:03.965 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:35:03.965 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:35:03.966 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "queued_for_pg", 2023-06-03T20:35:03.966 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:30:18.885953+0000", 2023-06-03T20:35:03.966 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 6.5099999999999997e-05 2023-06-03T20:35:03.966 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:35:03.966 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:35:03.966 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "reached_pg", 2023-06-03T20:35:03.966 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:30:18.886018+0000", 2023-06-03T20:35:03.966 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.000186505 2023-06-03T20:35:03.967 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:35:03.967 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:35:03.967 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "started", 2023-06-03T20:35:03.967 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:30:18.886204+0000", 2023-06-03T20:35:03.967 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.00019960600000000001 2023-06-03T20:35:03.967 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:35:03.967 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:35:03.967 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "waiting for subops from 1,3", 2023-06-03T20:35:03.968 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:30:18.886404+0000", 2023-06-03T20:35:03.968 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.00507245 2023-06-03T20:35:03.968 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:35:03.968 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:35:03.968 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "op_commit", 2023-06-03T20:35:03.968 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:30:18.891476+0000", 2023-06-03T20:35:03.968 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.0045675300000000002 2023-06-03T20:35:03.969 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:35:03.969 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:35:03.969 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:35:03.969 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:30:18.896044+0000", 2023-06-03T20:35:03.969 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.150230171 2023-06-03T20:35:03.969 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:35:03.969 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:35:03.969 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:35:03.970 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:30:19.046274+0000", 2023-06-03T20:35:03.970 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 3.0791999999999999e-05 2023-06-03T20:35:03.970 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:35:03.970 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:35:03.970 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "commit_sent", 2023-06-03T20:35:03.970 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:30:19.046305+0000", 2023-06-03T20:35:03.970 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 2.5851999999999999e-05 2023-06-03T20:35:03.970 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:35:03.971 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:35:03.971 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "done", 2023-06-03T20:35:03.971 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:30:19.046331+0000", 2023-06-03T20:35:03.971 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.160410791 2023-06-03T20:35:03.971 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:35:03.971 INFO:teuthology.orchestra.run.smithi164.stdout: ] 2023-06-03T20:35:03.971 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:35:03.972 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:35:03.972 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:35:03.972 INFO:teuthology.orchestra.run.smithi164.stdout: "description": "osd_op(client.46739.0:171083 14.f 14:f002525e:::benchmark_data_smithi089_237281_object73067:head [delete] snapc 0=[] ondisk+write+known_if_redirected e292)", 2023-06-03T20:35:03.972 INFO:teuthology.orchestra.run.smithi164.stdout: "initiated_at": "2023-06-03T20:31:15.733877+0000", 2023-06-03T20:35:03.972 INFO:teuthology.orchestra.run.smithi164.stdout: "age": 228.12240165399999, 2023-06-03T20:35:03.972 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.20588669600000001, 2023-06-03T20:35:03.972 INFO:teuthology.orchestra.run.smithi164.stdout: "type_data": { 2023-06-03T20:35:03.972 INFO:teuthology.orchestra.run.smithi164.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-03T20:35:03.973 INFO:teuthology.orchestra.run.smithi164.stdout: "client_info": { 2023-06-03T20:35:03.973 INFO:teuthology.orchestra.run.smithi164.stdout: "client": "client.46739", 2023-06-03T20:35:03.973 INFO:teuthology.orchestra.run.smithi164.stdout: "client_addr": "172.21.15.89:0/2479974613", 2023-06-03T20:35:03.973 INFO:teuthology.orchestra.run.smithi164.stdout: "tid": 171083 2023-06-03T20:35:03.973 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:35:03.973 INFO:teuthology.orchestra.run.smithi164.stdout: "events": [ 2023-06-03T20:35:03.973 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:35:03.973 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "initiated", 2023-06-03T20:35:03.974 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:31:15.733877+0000", 2023-06-03T20:35:03.974 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0 2023-06-03T20:35:03.974 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:35:03.974 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:35:03.974 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "throttled", 2023-06-03T20:35:03.974 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:31:15.733877+0000", 2023-06-03T20:35:03.974 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 4294967295.9999976 2023-06-03T20:35:03.975 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:35:03.975 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:35:03.975 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "header_read", 2023-06-03T20:35:03.975 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:31:15.733875+0000", 2023-06-03T20:35:03.975 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 1.2082000000000001e-05 2023-06-03T20:35:03.975 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:35:03.975 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:35:03.975 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "all_read", 2023-06-03T20:35:03.976 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:31:15.733887+0000", 2023-06-03T20:35:03.976 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 2.6322999999999999e-05 2023-06-03T20:35:03.976 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:35:03.976 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:35:03.976 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "dispatched", 2023-06-03T20:35:03.976 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:31:15.733913+0000", 2023-06-03T20:35:03.976 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 1.1768e-05 2023-06-03T20:35:03.976 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:35:03.977 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:35:03.977 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "queued_for_pg", 2023-06-03T20:35:03.977 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:31:15.733925+0000", 2023-06-03T20:35:03.977 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 9.365e-05 2023-06-03T20:35:03.977 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:35:03.977 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:35:03.977 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "reached_pg", 2023-06-03T20:35:03.977 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:31:15.734019+0000", 2023-06-03T20:35:03.978 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.00027610599999999998 2023-06-03T20:35:03.978 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:35:03.978 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:35:03.978 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "started", 2023-06-03T20:35:03.978 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:31:15.734295+0000", 2023-06-03T20:35:03.978 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.00033604000000000001 2023-06-03T20:35:03.978 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:35:03.979 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:35:03.979 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "waiting for subops from 1,3", 2023-06-03T20:35:03.979 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:31:15.734631+0000", 2023-06-03T20:35:03.979 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.012887641 2023-06-03T20:35:03.979 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:35:03.979 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:35:03.979 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:35:03.979 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:31:15.747518+0000", 2023-06-03T20:35:03.980 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 1.1238e-05 2023-06-03T20:35:03.980 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:35:03.980 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:35:03.980 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "op_commit", 2023-06-03T20:35:03.980 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:31:15.747530+0000", 2023-06-03T20:35:03.980 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.19207464699999999 2023-06-03T20:35:03.980 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:35:03.980 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:35:03.981 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:35:03.981 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:31:15.939604+0000", 2023-06-03T20:35:03.981 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 7.9754999999999998e-05 2023-06-03T20:35:03.981 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:35:03.981 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:35:03.981 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "commit_sent", 2023-06-03T20:35:03.981 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:31:15.939684+0000", 2023-06-03T20:35:03.981 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 7.9730999999999999e-05 2023-06-03T20:35:03.982 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:35:03.982 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:35:03.982 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "done", 2023-06-03T20:35:03.982 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:31:15.939764+0000", 2023-06-03T20:35:03.982 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.20588669600000001 2023-06-03T20:35:03.982 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:35:03.982 INFO:teuthology.orchestra.run.smithi164.stdout: ] 2023-06-03T20:35:03.983 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:35:03.983 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:35:03.983 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:35:03.983 INFO:teuthology.orchestra.run.smithi164.stdout: "description": "osd_op(client.46739.0:171085 14.f 14:fb54310e:::benchmark_data_smithi089_237281_object73069:head [delete] snapc 0=[] ondisk+write+known_if_redirected e292)", 2023-06-03T20:35:03.983 INFO:teuthology.orchestra.run.smithi164.stdout: "initiated_at": "2023-06-03T20:31:15.737090+0000", 2023-06-03T20:35:03.983 INFO:teuthology.orchestra.run.smithi164.stdout: "age": 228.11918893199999, 2023-06-03T20:35:03.983 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.20287834099999999, 2023-06-03T20:35:03.983 INFO:teuthology.orchestra.run.smithi164.stdout: "type_data": { 2023-06-03T20:35:03.984 INFO:teuthology.orchestra.run.smithi164.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-03T20:35:03.984 INFO:teuthology.orchestra.run.smithi164.stdout: "client_info": { 2023-06-03T20:35:03.984 INFO:teuthology.orchestra.run.smithi164.stdout: "client": "client.46739", 2023-06-03T20:35:03.984 INFO:teuthology.orchestra.run.smithi164.stdout: "client_addr": "172.21.15.89:0/2479974613", 2023-06-03T20:35:03.984 INFO:teuthology.orchestra.run.smithi164.stdout: "tid": 171085 2023-06-03T20:35:03.984 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:35:03.984 INFO:teuthology.orchestra.run.smithi164.stdout: "events": [ 2023-06-03T20:35:03.984 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:35:03.985 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "initiated", 2023-06-03T20:35:03.985 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:31:15.737090+0000", 2023-06-03T20:35:03.985 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0 2023-06-03T20:35:03.985 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:35:03.985 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:35:03.985 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "throttled", 2023-06-03T20:35:03.985 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:31:15.737090+0000", 2023-06-03T20:35:03.986 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 4294967295.9999971 2023-06-03T20:35:03.986 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:35:03.986 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:35:03.986 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "header_read", 2023-06-03T20:35:03.986 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:31:15.737087+0000", 2023-06-03T20:35:03.986 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 1.5279000000000001e-05 2023-06-03T20:35:03.986 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:35:03.986 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:35:03.987 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "all_read", 2023-06-03T20:35:03.987 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:31:15.737102+0000", 2023-06-03T20:35:03.987 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.00010485299999999999 2023-06-03T20:35:03.987 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:35:03.987 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:35:03.987 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "dispatched", 2023-06-03T20:35:03.987 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:31:15.737207+0000", 2023-06-03T20:35:03.987 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 2.2538999999999999e-05 2023-06-03T20:35:03.988 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:35:03.988 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:35:03.988 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "queued_for_pg", 2023-06-03T20:35:03.988 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:31:15.737230+0000", 2023-06-03T20:35:03.988 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.0048668579999999999 2023-06-03T20:35:03.988 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:35:03.988 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:35:03.988 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "reached_pg", 2023-06-03T20:35:03.989 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:31:15.742096+0000", 2023-06-03T20:35:03.989 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.000133824 2023-06-03T20:35:03.989 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:35:03.989 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:35:03.989 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "started", 2023-06-03T20:35:03.989 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:31:15.742230+0000", 2023-06-03T20:35:03.989 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.00016191799999999999 2023-06-03T20:35:03.990 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:35:03.990 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:35:03.990 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "waiting for subops from 1,3", 2023-06-03T20:35:03.990 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:31:15.742392+0000", 2023-06-03T20:35:03.990 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.010108228 2023-06-03T20:35:03.990 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:35:03.990 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:35:03.990 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "op_commit", 2023-06-03T20:35:03.991 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:31:15.752500+0000", 2023-06-03T20:35:03.991 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.00084675099999999997 2023-06-03T20:35:03.991 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:35:03.991 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:35:03.991 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:35:03.991 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:31:15.753347+0000", 2023-06-03T20:35:03.991 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.18647906 2023-06-03T20:35:03.991 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:35:03.992 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:35:03.992 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:35:03.992 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:31:15.939826+0000", 2023-06-03T20:35:03.992 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 7.9139000000000002e-05 2023-06-03T20:35:03.992 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:35:03.992 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:35:03.992 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "commit_sent", 2023-06-03T20:35:03.993 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:31:15.939905+0000", 2023-06-03T20:35:03.993 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 6.2788999999999998e-05 2023-06-03T20:35:03.993 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:35:03.993 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:35:03.993 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "done", 2023-06-03T20:35:03.993 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:31:15.939968+0000", 2023-06-03T20:35:03.993 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.20287834099999999 2023-06-03T20:35:03.993 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:35:03.994 INFO:teuthology.orchestra.run.smithi164.stdout: ] 2023-06-03T20:35:03.994 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:35:03.994 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:35:03.994 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:35:03.994 INFO:teuthology.orchestra.run.smithi164.stdout: "description": "osd_op(client.46739.0:171087 14.f 14:fe3e739c:::benchmark_data_smithi089_237281_object73071:head [delete] snapc 0=[] ondisk+write+known_if_redirected e292)", 2023-06-03T20:35:03.994 INFO:teuthology.orchestra.run.smithi164.stdout: "initiated_at": "2023-06-03T20:31:15.737304+0000", 2023-06-03T20:35:03.994 INFO:teuthology.orchestra.run.smithi164.stdout: "age": 228.11897442200001, 2023-06-03T20:35:03.995 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.203710589, 2023-06-03T20:35:03.995 INFO:teuthology.orchestra.run.smithi164.stdout: "type_data": { 2023-06-03T20:35:03.995 INFO:teuthology.orchestra.run.smithi164.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-03T20:35:03.995 INFO:teuthology.orchestra.run.smithi164.stdout: "client_info": { 2023-06-03T20:35:03.995 INFO:teuthology.orchestra.run.smithi164.stdout: "client": "client.46739", 2023-06-03T20:35:03.995 INFO:teuthology.orchestra.run.smithi164.stdout: "client_addr": "172.21.15.89:0/2479974613", 2023-06-03T20:35:03.995 INFO:teuthology.orchestra.run.smithi164.stdout: "tid": 171087 2023-06-03T20:35:03.995 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:35:03.996 INFO:teuthology.orchestra.run.smithi164.stdout: "events": [ 2023-06-03T20:35:03.996 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:35:03.996 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "initiated", 2023-06-03T20:35:03.996 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:31:15.737304+0000", 2023-06-03T20:35:03.996 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0 2023-06-03T20:35:03.996 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:35:03.996 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:35:03.996 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "throttled", 2023-06-03T20:35:03.997 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:31:15.737304+0000", 2023-06-03T20:35:03.997 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 4294967295.9999981 2023-06-03T20:35:03.997 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:35:03.997 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:35:03.997 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "header_read", 2023-06-03T20:35:03.997 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:31:15.737303+0000", 2023-06-03T20:35:03.997 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 1.0557000000000001e-05 2023-06-03T20:35:03.998 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:35:03.998 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:35:03.998 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "all_read", 2023-06-03T20:35:03.998 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:31:15.737313+0000", 2023-06-03T20:35:03.998 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 2.6727999999999999e-05 2023-06-03T20:35:03.998 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:35:03.998 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:35:03.998 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "dispatched", 2023-06-03T20:35:03.999 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:31:15.737340+0000", 2023-06-03T20:35:03.999 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 1.6745e-05 2023-06-03T20:35:03.999 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:35:03.999 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:35:03.999 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "queued_for_pg", 2023-06-03T20:35:03.999 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:31:15.737357+0000", 2023-06-03T20:35:03.999 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.010212268 2023-06-03T20:35:03.999 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:35:04.000 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:35:04.000 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "reached_pg", 2023-06-03T20:35:04.000 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:31:15.747569+0000", 2023-06-03T20:35:04.000 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.00013854500000000001 2023-06-03T20:35:04.000 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:35:04.000 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:35:04.000 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "started", 2023-06-03T20:35:04.000 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:31:15.747707+0000", 2023-06-03T20:35:04.001 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.00015935400000000001 2023-06-03T20:35:04.001 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:35:04.001 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:35:04.001 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "waiting for subops from 1,3", 2023-06-03T20:35:04.001 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:31:15.747867+0000", 2023-06-03T20:35:04.001 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.0047902379999999996 2023-06-03T20:35:04.001 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:35:04.002 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:35:04.002 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "op_commit", 2023-06-03T20:35:04.002 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:31:15.752657+0000", 2023-06-03T20:35:04.002 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.00087111299999999997 2023-06-03T20:35:04.002 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:35:04.002 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:35:04.002 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:35:04.002 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:31:15.753528+0000", 2023-06-03T20:35:04.003 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.187350605 2023-06-03T20:35:04.003 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:35:04.003 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:35:04.003 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:35:04.003 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:31:15.940879+0000", 2023-06-03T20:35:04.003 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 7.8170999999999994e-05 2023-06-03T20:35:04.003 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:35:04.003 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:35:04.004 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "commit_sent", 2023-06-03T20:35:04.004 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:31:15.940957+0000", 2023-06-03T20:35:04.004 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 5.7955999999999999e-05 2023-06-03T20:35:04.004 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:35:04.004 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:35:04.004 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "done", 2023-06-03T20:35:04.004 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:31:15.941015+0000", 2023-06-03T20:35:04.004 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.203710589 2023-06-03T20:35:04.005 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:35:04.005 INFO:teuthology.orchestra.run.smithi164.stdout: ] 2023-06-03T20:35:04.005 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:35:04.005 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:35:04.005 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:35:04.005 INFO:teuthology.orchestra.run.smithi164.stdout: "description": "osd_op(client.46739.0:177885 14.f 14:fbf0f041:::benchmark_data_smithi089_237281_object79869:head [delete] snapc 0=[] ondisk+write+known_if_redirected e293)", 2023-06-03T20:35:04.005 INFO:teuthology.orchestra.run.smithi164.stdout: "initiated_at": "2023-06-03T20:31:27.907049+0000", 2023-06-03T20:35:04.005 INFO:teuthology.orchestra.run.smithi164.stdout: "age": 215.94922970799999, 2023-06-03T20:35:04.006 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.160373817, 2023-06-03T20:35:04.006 INFO:teuthology.orchestra.run.smithi164.stdout: "type_data": { 2023-06-03T20:35:04.006 INFO:teuthology.orchestra.run.smithi164.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-03T20:35:04.006 INFO:teuthology.orchestra.run.smithi164.stdout: "client_info": { 2023-06-03T20:35:04.006 INFO:teuthology.orchestra.run.smithi164.stdout: "client": "client.46739", 2023-06-03T20:35:04.006 INFO:teuthology.orchestra.run.smithi164.stdout: "client_addr": "172.21.15.89:0/2479974613", 2023-06-03T20:35:04.006 INFO:teuthology.orchestra.run.smithi164.stdout: "tid": 177885 2023-06-03T20:35:04.007 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:35:04.007 INFO:teuthology.orchestra.run.smithi164.stdout: "events": [ 2023-06-03T20:35:04.007 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:35:04.007 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "initiated", 2023-06-03T20:35:04.007 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:31:27.907049+0000", 2023-06-03T20:35:04.007 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0 2023-06-03T20:35:04.007 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:35:04.007 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:35:04.008 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "throttled", 2023-06-03T20:35:04.008 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:31:27.907049+0000", 2023-06-03T20:35:04.008 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 4294967295.9999981 2023-06-03T20:35:04.008 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:35:04.008 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:35:04.008 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "header_read", 2023-06-03T20:35:04.008 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:31:27.907047+0000", 2023-06-03T20:35:04.008 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 8.3729999999999993e-06 2023-06-03T20:35:04.009 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:35:04.009 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:35:04.009 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "all_read", 2023-06-03T20:35:04.009 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:31:27.907055+0000", 2023-06-03T20:35:04.009 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 1.6163000000000001e-05 2023-06-03T20:35:04.009 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:35:04.009 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:35:04.009 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "dispatched", 2023-06-03T20:35:04.010 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:31:27.907071+0000", 2023-06-03T20:35:04.010 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 1.0137e-05 2023-06-03T20:35:04.010 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:35:04.010 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:35:04.010 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "queued_for_pg", 2023-06-03T20:35:04.010 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:31:27.907082+0000", 2023-06-03T20:35:04.010 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 5.8579000000000003e-05 2023-06-03T20:35:04.011 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:35:04.011 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:35:04.011 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "reached_pg", 2023-06-03T20:35:04.011 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:31:27.907140+0000", 2023-06-03T20:35:04.011 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.000118509 2023-06-03T20:35:04.011 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:35:04.011 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:35:04.011 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "started", 2023-06-03T20:35:04.012 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:31:27.907259+0000", 2023-06-03T20:35:04.012 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.000125752 2023-06-03T20:35:04.012 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:35:04.012 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:35:04.012 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "waiting for subops from 1,3", 2023-06-03T20:35:04.012 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:31:27.907384+0000", 2023-06-03T20:35:04.012 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.0043370609999999997 2023-06-03T20:35:04.012 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:35:04.013 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:35:04.013 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "op_commit", 2023-06-03T20:35:04.013 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:31:27.911721+0000", 2023-06-03T20:35:04.013 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.00018203699999999999 2023-06-03T20:35:04.013 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:35:04.013 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:35:04.013 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:35:04.013 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:31:27.911904+0000", 2023-06-03T20:35:04.014 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.15539281199999999 2023-06-03T20:35:04.014 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:35:04.014 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:35:04.014 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:35:04.014 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:31:28.067296+0000", 2023-06-03T20:35:04.014 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 7.1285000000000006e-05 2023-06-03T20:35:04.014 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:35:04.015 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:35:04.015 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "commit_sent", 2023-06-03T20:35:04.015 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:31:28.067368+0000", 2023-06-03T20:35:04.015 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 5.5222000000000001e-05 2023-06-03T20:35:04.015 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:35:04.015 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:35:04.015 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "done", 2023-06-03T20:35:04.015 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:31:28.067423+0000", 2023-06-03T20:35:04.016 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.160373817 2023-06-03T20:35:04.016 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:35:04.016 INFO:teuthology.orchestra.run.smithi164.stdout: ] 2023-06-03T20:35:04.016 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:35:04.016 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:35:04.016 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:35:04.016 INFO:teuthology.orchestra.run.smithi164.stdout: "description": "osd_op(client.46739.0:177893 14.f 14:fca33255:::benchmark_data_smithi089_237281_object79877:head [delete] snapc 0=[] ondisk+write+known_if_redirected e293)", 2023-06-03T20:35:04.016 INFO:teuthology.orchestra.run.smithi164.stdout: "initiated_at": "2023-06-03T20:31:27.919252+0000", 2023-06-03T20:35:04.017 INFO:teuthology.orchestra.run.smithi164.stdout: "age": 215.93702703700001, 2023-06-03T20:35:04.017 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.148311158, 2023-06-03T20:35:04.017 INFO:teuthology.orchestra.run.smithi164.stdout: "type_data": { 2023-06-03T20:35:04.017 INFO:teuthology.orchestra.run.smithi164.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-03T20:35:04.017 INFO:teuthology.orchestra.run.smithi164.stdout: "client_info": { 2023-06-03T20:35:04.017 INFO:teuthology.orchestra.run.smithi164.stdout: "client": "client.46739", 2023-06-03T20:35:04.017 INFO:teuthology.orchestra.run.smithi164.stdout: "client_addr": "172.21.15.89:0/2479974613", 2023-06-03T20:35:04.018 INFO:teuthology.orchestra.run.smithi164.stdout: "tid": 177893 2023-06-03T20:35:04.018 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:35:04.018 INFO:teuthology.orchestra.run.smithi164.stdout: "events": [ 2023-06-03T20:35:04.018 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:35:04.018 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "initiated", 2023-06-03T20:35:04.018 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:31:27.919252+0000", 2023-06-03T20:35:04.018 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0 2023-06-03T20:35:04.018 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:35:04.019 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:35:04.019 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "throttled", 2023-06-03T20:35:04.019 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:31:27.919252+0000", 2023-06-03T20:35:04.019 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 4294967295.9999981 2023-06-03T20:35:04.019 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:35:04.019 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:35:04.019 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "header_read", 2023-06-03T20:35:04.019 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:31:27.919250+0000", 2023-06-03T20:35:04.020 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 1.1428e-05 2023-06-03T20:35:04.020 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:35:04.020 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:35:04.020 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "all_read", 2023-06-03T20:35:04.020 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:31:27.919261+0000", 2023-06-03T20:35:04.020 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 2.2826000000000001e-05 2023-06-03T20:35:04.020 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:35:04.020 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:35:04.021 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "dispatched", 2023-06-03T20:35:04.021 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:31:27.919284+0000", 2023-06-03T20:35:04.021 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 1.0496000000000001e-05 2023-06-03T20:35:04.021 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:35:04.021 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:35:04.021 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "queued_for_pg", 2023-06-03T20:35:04.021 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:31:27.919294+0000", 2023-06-03T20:35:04.022 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 4.7874000000000003e-05 2023-06-03T20:35:04.022 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:35:04.022 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:35:04.022 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "reached_pg", 2023-06-03T20:35:04.022 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:31:27.919342+0000", 2023-06-03T20:35:04.022 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.00010741700000000001 2023-06-03T20:35:04.022 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:35:04.022 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:35:04.023 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "started", 2023-06-03T20:35:04.023 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:31:27.919450+0000", 2023-06-03T20:35:04.023 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.000108081 2023-06-03T20:35:04.023 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:35:04.023 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:35:04.023 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "waiting for subops from 1,3", 2023-06-03T20:35:04.023 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:31:27.919558+0000", 2023-06-03T20:35:04.023 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.0050719750000000003 2023-06-03T20:35:04.024 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:35:04.024 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:35:04.024 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "op_commit", 2023-06-03T20:35:04.024 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:31:27.924630+0000", 2023-06-03T20:35:04.024 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.00021790700000000001 2023-06-03T20:35:04.024 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:35:04.024 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:35:04.025 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:35:04.025 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:31:27.924848+0000", 2023-06-03T20:35:04.025 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.14262338599999999 2023-06-03T20:35:04.025 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:35:04.025 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:35:04.025 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:35:04.025 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:31:28.067471+0000", 2023-06-03T20:35:04.025 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 4.2518999999999997e-05 2023-06-03T20:35:04.026 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:35:04.026 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:35:04.026 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "commit_sent", 2023-06-03T20:35:04.026 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:31:28.067514+0000", 2023-06-03T20:35:04.026 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 4.9348000000000003e-05 2023-06-03T20:35:04.026 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:35:04.026 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:35:04.026 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "done", 2023-06-03T20:35:04.027 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:31:28.067563+0000", 2023-06-03T20:35:04.027 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.148311158 2023-06-03T20:35:04.027 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:35:04.027 INFO:teuthology.orchestra.run.smithi164.stdout: ] 2023-06-03T20:35:04.027 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:35:04.027 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:35:04.027 INFO:teuthology.orchestra.run.smithi164.stdout: ] 2023-06-03T20:35:04.027 INFO:teuthology.orchestra.run.smithi164.stdout:} 2023-06-03T20:35:04.335 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:35:04 smithi164 bash[59462]: cluster 1685824503.282876 mgr.y (mgr.34104) 1781 : cluster 0 pgmap v1855: 73 pgs: 73 active+clean; 582 KiB data, 5.1 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:35:04.529 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:35:04 smithi089 bash[78452]: cluster 1685824503.282876 mgr.y (mgr.34104) 1781 : cluster 0 pgmap v1855: 73 pgs: 73 active+clean; 582 KiB data, 5.1 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:35:04.529 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:35:04 smithi089 bash[75068]: cluster 1685824503.282876 mgr.y (mgr.34104) 1781 : cluster 0 pgmap v1855: 73 pgs: 73 active+clean; 582 KiB data, 5.1 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:35:04.760 DEBUG:teuthology.orchestra.run.smithi164:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph daemon osd.4 dump_ops_in_flight 2023-06-03T20:35:05.197 INFO:teuthology.orchestra.run.smithi089.stderr:set osd.3 primary-affinity to 1 (8655362) 2023-06-03T20:35:05.529 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:35:05 smithi089 bash[78452]: audit 1685824505.0630248 mon.c (mon.2) 710 : audit 1 from='client.? 172.21.15.89:0/220616026' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 3, "weight": 1.0}]: dispatch 2023-06-03T20:35:05.529 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:35:05 smithi089 bash[78452]: audit 1685824505.0638177 mon.a (mon.0) 1731 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 3, "weight": 1.0}]: dispatch 2023-06-03T20:35:05.530 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:35:05 smithi089 bash[75068]: audit 1685824505.0630248 mon.c (mon.2) 710 : audit 1 from='client.? 172.21.15.89:0/220616026' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 3, "weight": 1.0}]: dispatch 2023-06-03T20:35:05.530 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:35:05 smithi089 bash[75068]: audit 1685824505.0638177 mon.a (mon.0) 1731 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 3, "weight": 1.0}]: dispatch 2023-06-03T20:35:05.585 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:35:05 smithi164 bash[59462]: audit 1685824505.0630248 mon.c (mon.2) 710 : audit 1 from='client.? 172.21.15.89:0/220616026' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 3, "weight": 1.0}]: dispatch 2023-06-03T20:35:05.585 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:35:05 smithi164 bash[59462]: audit 1685824505.0638177 mon.a (mon.0) 1731 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 3, "weight": 1.0}]: dispatch 2023-06-03T20:35:06.132 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph osd reweight 5 1 2023-06-03T20:35:06.529 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 20:35:06 smithi089 bash[71163]: ::ffff:172.21.15.164 - - [03/Jun/2023:20:35:06] "GET /metrics HTTP/1.1" 200 226942 "" "Prometheus/2.33.4" 2023-06-03T20:35:06.529 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:35:06 smithi089 bash[75068]: cluster 1685824505.1798358 osd.5 (osd.5) 261 : cluster 3 failed to encode map e315 with expected crc 2023-06-03T20:35:06.530 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:35:06 smithi089 bash[75068]: audit 1685824505.1860695 mon.a (mon.0) 1732 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 3, "weight": 1.0}]': finished 2023-06-03T20:35:06.530 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:35:06 smithi089 bash[75068]: cluster 1685824505.18624 mon.a (mon.0) 1733 : cluster 0 osdmap e315: 8 total, 8 up, 8 in 2023-06-03T20:35:06.530 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:35:06 smithi089 bash[75068]: cluster 1685824505.2836156 mgr.y (mgr.34104) 1782 : cluster 0 pgmap v1857: 73 pgs: 73 active+clean; 582 KiB data, 5.1 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:35:06.530 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:35:06 smithi089 bash[75068]: cluster 1685824505.8193 osd.6 (osd.6) 252 : cluster 3 failed to encode map e315 with expected crc 2023-06-03T20:35:06.530 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:35:06 smithi089 bash[78452]: cluster 1685824505.1798358 osd.5 (osd.5) 261 : cluster 3 failed to encode map e315 with expected crc 2023-06-03T20:35:06.531 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:35:06 smithi089 bash[78452]: audit 1685824505.1860695 mon.a (mon.0) 1732 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 3, "weight": 1.0}]': finished 2023-06-03T20:35:06.531 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:35:06 smithi089 bash[78452]: cluster 1685824505.18624 mon.a (mon.0) 1733 : cluster 0 osdmap e315: 8 total, 8 up, 8 in 2023-06-03T20:35:06.531 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:35:06 smithi089 bash[78452]: cluster 1685824505.2836156 mgr.y (mgr.34104) 1782 : cluster 0 pgmap v1857: 73 pgs: 73 active+clean; 582 KiB data, 5.1 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:35:06.531 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:35:06 smithi089 bash[78452]: cluster 1685824505.8193 osd.6 (osd.6) 252 : cluster 3 failed to encode map e315 with expected crc 2023-06-03T20:35:06.585 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:35:06 smithi164 bash[59462]: cluster 1685824505.1798358 osd.5 (osd.5) 261 : cluster 3 failed to encode map e315 with expected crc 2023-06-03T20:35:06.586 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:35:06 smithi164 bash[59462]: audit 1685824505.1860695 mon.a (mon.0) 1732 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 3, "weight": 1.0}]': finished 2023-06-03T20:35:06.586 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:35:06 smithi164 bash[59462]: cluster 1685824505.18624 mon.a (mon.0) 1733 : cluster 0 osdmap e315: 8 total, 8 up, 8 in 2023-06-03T20:35:06.586 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:35:06 smithi164 bash[59462]: cluster 1685824505.2836156 mgr.y (mgr.34104) 1782 : cluster 0 pgmap v1857: 73 pgs: 73 active+clean; 582 KiB data, 5.1 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:35:06.586 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:35:06 smithi164 bash[59462]: cluster 1685824505.8193 osd.6 (osd.6) 252 : cluster 3 failed to encode map e315 with expected crc 2023-06-03T20:35:07.529 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:35:07 smithi089 bash[78452]: cluster 1685824505.8189514 osd.7 (osd.7) 258 : cluster 3 failed to encode map e315 with expected crc 2023-06-03T20:35:07.529 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:35:07 smithi089 bash[78452]: audit 1685824507.1663551 mon.c (mon.2) 711 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T20:35:07.529 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:35:07 smithi089 bash[75068]: cluster 1685824505.8189514 osd.7 (osd.7) 258 : cluster 3 failed to encode map e315 with expected crc 2023-06-03T20:35:07.530 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:35:07 smithi089 bash[75068]: audit 1685824507.1663551 mon.c (mon.2) 711 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T20:35:07.585 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:35:07 smithi164 bash[59462]: cluster 1685824505.8189514 osd.7 (osd.7) 258 : cluster 3 failed to encode map e315 with expected crc 2023-06-03T20:35:07.585 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:35:07 smithi164 bash[59462]: audit 1685824507.1663551 mon.c (mon.2) 711 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T20:35:08.529 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:35:08 smithi089 bash[75068]: cluster 1685824507.2843993 mgr.y (mgr.34104) 1783 : cluster 0 pgmap v1858: 73 pgs: 73 active+clean; 582 KiB data, 5.1 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:35:08.529 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:35:08 smithi089 bash[78452]: cluster 1685824507.2843993 mgr.y (mgr.34104) 1783 : cluster 0 pgmap v1858: 73 pgs: 73 active+clean; 582 KiB data, 5.1 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:35:08.585 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:35:08 smithi164 bash[59462]: cluster 1685824507.2843993 mgr.y (mgr.34104) 1783 : cluster 0 pgmap v1858: 73 pgs: 73 active+clean; 582 KiB data, 5.1 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:35:09.578 INFO:teuthology.orchestra.run.smithi164.stderr:Inferring config /var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/mon.b/config 2023-06-03T20:35:10.779 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:35:10 smithi089 bash[78452]: cluster 1685824509.2853806 mgr.y (mgr.34104) 1784 : cluster 0 pgmap v1859: 73 pgs: 73 active+clean; 582 KiB data, 5.1 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:35:10.779 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:35:10 smithi089 bash[75068]: cluster 1685824509.2853806 mgr.y (mgr.34104) 1784 : cluster 0 pgmap v1859: 73 pgs: 73 active+clean; 582 KiB data, 5.1 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:35:10.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:35:10 smithi164 bash[59462]: cluster 1685824509.2853806 mgr.y (mgr.34104) 1784 : cluster 0 pgmap v1859: 73 pgs: 73 active+clean; 582 KiB data, 5.1 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:35:10.948 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/mon.a/config 2023-06-03T20:35:12.643 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:35:12 smithi089 bash[75068]: cluster 1685824511.2860806 mgr.y (mgr.34104) 1785 : cluster 0 pgmap v1860: 73 pgs: 73 active+clean; 582 KiB data, 5.1 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:35:12.643 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:35:12 smithi089 bash[78452]: cluster 1685824511.2860806 mgr.y (mgr.34104) 1785 : cluster 0 pgmap v1860: 73 pgs: 73 active+clean; 582 KiB data, 5.1 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:35:12.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:35:12 smithi164 bash[59462]: cluster 1685824511.2860806 mgr.y (mgr.34104) 1785 : cluster 0 pgmap v1860: 73 pgs: 73 active+clean; 582 KiB data, 5.1 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:35:13.342 INFO:teuthology.orchestra.run.smithi164.stdout:{ 2023-06-03T20:35:13.342 INFO:teuthology.orchestra.run.smithi164.stdout: "ops": [], 2023-06-03T20:35:13.343 INFO:teuthology.orchestra.run.smithi164.stdout: "num_ops": 0 2023-06-03T20:35:13.343 INFO:teuthology.orchestra.run.smithi164.stdout:} 2023-06-03T20:35:14.213 DEBUG:teuthology.orchestra.run.smithi164:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph daemon osd.4 dump_blocked_ops 2023-06-03T20:35:14.529 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:35:14 smithi089 bash[75068]: cluster 1685824513.2871091 mgr.y (mgr.34104) 1786 : cluster 0 pgmap v1861: 73 pgs: 73 active+clean; 582 KiB data, 5.1 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:35:14.529 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:35:14 smithi089 bash[78452]: cluster 1685824513.2871091 mgr.y (mgr.34104) 1786 : cluster 0 pgmap v1861: 73 pgs: 73 active+clean; 582 KiB data, 5.1 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:35:14.585 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:35:14 smithi164 bash[59462]: cluster 1685824513.2871091 mgr.y (mgr.34104) 1786 : cluster 0 pgmap v1861: 73 pgs: 73 active+clean; 582 KiB data, 5.1 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:35:15.191 INFO:teuthology.orchestra.run.smithi089.stderr:reweighted osd.5 to 1 (10000) 2023-06-03T20:35:15.529 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:35:15 smithi089 bash[75068]: audit 1685824515.0711405 mon.c (mon.2) 712 : audit 1 from='client.? 172.21.15.89:0/3976589196' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-06-03T20:35:15.529 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:35:15 smithi089 bash[75068]: audit 1685824515.071899 mon.a (mon.0) 1734 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-06-03T20:35:15.529 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:35:15 smithi089 bash[78452]: audit 1685824515.0711405 mon.c (mon.2) 712 : audit 1 from='client.? 172.21.15.89:0/3976589196' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-06-03T20:35:15.529 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:35:15 smithi089 bash[78452]: audit 1685824515.071899 mon.a (mon.0) 1734 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-06-03T20:35:15.585 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:35:15 smithi164 bash[59462]: audit 1685824515.0711405 mon.c (mon.2) 712 : audit 1 from='client.? 172.21.15.89:0/3976589196' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-06-03T20:35:15.586 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:35:15 smithi164 bash[59462]: audit 1685824515.071899 mon.a (mon.0) 1734 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-06-03T20:35:16.153 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph osd primary-affinity 5 1 2023-06-03T20:35:16.529 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 20:35:16 smithi089 bash[71163]: ::ffff:172.21.15.164 - - [03/Jun/2023:20:35:16] "GET /metrics HTTP/1.1" 200 226924 "" "Prometheus/2.33.4" 2023-06-03T20:35:16.529 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:35:16 smithi089 bash[75068]: audit 1685824515.18094 mon.a (mon.0) 1735 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2023-06-03T20:35:16.530 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:35:16 smithi089 bash[75068]: cluster 1685824515.181037 mon.a (mon.0) 1736 : cluster 0 osdmap e316: 8 total, 8 up, 8 in 2023-06-03T20:35:16.530 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:35:16 smithi089 bash[75068]: cluster 1685824515.2254124 osd.5 (osd.5) 262 : cluster 3 failed to encode map e316 with expected crc 2023-06-03T20:35:16.530 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:35:16 smithi089 bash[75068]: cluster 1685824515.2276292 osd.5 (osd.5) 263 : cluster 3 failed to encode map e316 with expected crc 2023-06-03T20:35:16.530 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:35:16 smithi089 bash[75068]: cluster 1685824515.2875192 mgr.y (mgr.34104) 1787 : cluster 0 pgmap v1863: 73 pgs: 73 active+clean; 582 KiB data, 5.1 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:35:16.530 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:35:16 smithi089 bash[75068]: cluster 1685824515.419375 osd.7 (osd.7) 259 : cluster 3 failed to encode map e316 with expected crc 2023-06-03T20:35:16.530 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:35:16 smithi089 bash[75068]: cluster 1685824515.7230442 osd.6 (osd.6) 253 : cluster 3 failed to encode map e316 with expected crc 2023-06-03T20:35:16.531 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:35:16 smithi089 bash[78452]: audit 1685824515.18094 mon.a (mon.0) 1735 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2023-06-03T20:35:16.531 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:35:16 smithi089 bash[78452]: cluster 1685824515.181037 mon.a (mon.0) 1736 : cluster 0 osdmap e316: 8 total, 8 up, 8 in 2023-06-03T20:35:16.531 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:35:16 smithi089 bash[78452]: cluster 1685824515.2254124 osd.5 (osd.5) 262 : cluster 3 failed to encode map e316 with expected crc 2023-06-03T20:35:16.531 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:35:16 smithi089 bash[78452]: cluster 1685824515.2276292 osd.5 (osd.5) 263 : cluster 3 failed to encode map e316 with expected crc 2023-06-03T20:35:16.531 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:35:16 smithi089 bash[78452]: cluster 1685824515.2875192 mgr.y (mgr.34104) 1787 : cluster 0 pgmap v1863: 73 pgs: 73 active+clean; 582 KiB data, 5.1 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:35:16.531 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:35:16 smithi089 bash[78452]: cluster 1685824515.419375 osd.7 (osd.7) 259 : cluster 3 failed to encode map e316 with expected crc 2023-06-03T20:35:16.531 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:35:16 smithi089 bash[78452]: cluster 1685824515.7230442 osd.6 (osd.6) 253 : cluster 3 failed to encode map e316 with expected crc 2023-06-03T20:35:16.585 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:35:16 smithi164 bash[59462]: audit 1685824515.18094 mon.a (mon.0) 1735 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2023-06-03T20:35:16.586 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:35:16 smithi164 bash[59462]: cluster 1685824515.181037 mon.a (mon.0) 1736 : cluster 0 osdmap e316: 8 total, 8 up, 8 in 2023-06-03T20:35:16.586 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:35:16 smithi164 bash[59462]: cluster 1685824515.2254124 osd.5 (osd.5) 262 : cluster 3 failed to encode map e316 with expected crc 2023-06-03T20:35:16.586 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:35:16 smithi164 bash[59462]: cluster 1685824515.2276292 osd.5 (osd.5) 263 : cluster 3 failed to encode map e316 with expected crc 2023-06-03T20:35:16.586 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:35:16 smithi164 bash[59462]: cluster 1685824515.2875192 mgr.y (mgr.34104) 1787 : cluster 0 pgmap v1863: 73 pgs: 73 active+clean; 582 KiB data, 5.1 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:35:16.586 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:35:16 smithi164 bash[59462]: cluster 1685824515.419375 osd.7 (osd.7) 259 : cluster 3 failed to encode map e316 with expected crc 2023-06-03T20:35:16.586 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:35:16 smithi164 bash[59462]: cluster 1685824515.7230442 osd.6 (osd.6) 253 : cluster 3 failed to encode map e316 with expected crc 2023-06-03T20:35:18.779 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:35:18 smithi089 bash[75068]: cluster 1685824517.288263 mgr.y (mgr.34104) 1788 : cluster 0 pgmap v1864: 73 pgs: 73 active+clean; 582 KiB data, 5.1 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:35:18.779 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:35:18 smithi089 bash[78452]: cluster 1685824517.288263 mgr.y (mgr.34104) 1788 : cluster 0 pgmap v1864: 73 pgs: 73 active+clean; 582 KiB data, 5.1 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:35:18.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:35:18 smithi164 bash[59462]: cluster 1685824517.288263 mgr.y (mgr.34104) 1788 : cluster 0 pgmap v1864: 73 pgs: 73 active+clean; 582 KiB data, 5.1 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:35:19.032 INFO:teuthology.orchestra.run.smithi164.stderr:Inferring config /var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/mon.b/config 2023-06-03T20:35:20.585 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:35:20 smithi164 bash[59462]: cluster 1685824519.2892625 mgr.y (mgr.34104) 1789 : cluster 0 pgmap v1865: 73 pgs: 73 active+clean; 582 KiB data, 5.1 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:35:20.779 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:35:20 smithi089 bash[75068]: cluster 1685824519.2892625 mgr.y (mgr.34104) 1789 : cluster 0 pgmap v1865: 73 pgs: 73 active+clean; 582 KiB data, 5.1 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:35:20.779 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:35:20 smithi089 bash[78452]: cluster 1685824519.2892625 mgr.y (mgr.34104) 1789 : cluster 0 pgmap v1865: 73 pgs: 73 active+clean; 582 KiB data, 5.1 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:35:20.965 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/mon.a/config 2023-06-03T20:35:22.620 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:35:22 smithi089 bash[75068]: cluster 1685824521.290019 mgr.y (mgr.34104) 1790 : cluster 0 pgmap v1866: 73 pgs: 73 active+clean; 582 KiB data, 5.1 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:35:22.620 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:35:22 smithi089 bash[75068]: audit 1685824522.1668887 mon.c (mon.2) 713 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T20:35:22.620 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:35:22 smithi089 bash[78452]: cluster 1685824521.290019 mgr.y (mgr.34104) 1790 : cluster 0 pgmap v1866: 73 pgs: 73 active+clean; 582 KiB data, 5.1 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:35:22.621 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:35:22 smithi089 bash[78452]: audit 1685824522.1668887 mon.c (mon.2) 713 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T20:35:22.695 INFO:teuthology.orchestra.run.smithi164.stdout:{ 2023-06-03T20:35:22.695 INFO:teuthology.orchestra.run.smithi164.stdout: "ops": [], 2023-06-03T20:35:22.695 INFO:teuthology.orchestra.run.smithi164.stdout: "complaint_time": 30, 2023-06-03T20:35:22.695 INFO:teuthology.orchestra.run.smithi164.stdout: "num_blocked_ops": 0 2023-06-03T20:35:22.696 INFO:teuthology.orchestra.run.smithi164.stdout:} 2023-06-03T20:35:22.741 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:35:22 smithi164 bash[59462]: cluster 1685824521.290019 mgr.y (mgr.34104) 1790 : cluster 0 pgmap v1866: 73 pgs: 73 active+clean; 582 KiB data, 5.1 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:35:22.741 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:35:22 smithi164 bash[59462]: audit 1685824522.1668887 mon.c (mon.2) 713 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T20:35:23.630 DEBUG:teuthology.orchestra.run.smithi164:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph daemon osd.4 dump_historic_ops 2023-06-03T20:35:24.529 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:35:24 smithi089 bash[75068]: cluster 1685824523.2909453 mgr.y (mgr.34104) 1791 : cluster 0 pgmap v1867: 73 pgs: 73 active+clean; 582 KiB data, 5.1 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:35:24.529 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:35:24 smithi089 bash[78452]: cluster 1685824523.2909453 mgr.y (mgr.34104) 1791 : cluster 0 pgmap v1867: 73 pgs: 73 active+clean; 582 KiB data, 5.1 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:35:24.585 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:35:24 smithi164 bash[59462]: cluster 1685824523.2909453 mgr.y (mgr.34104) 1791 : cluster 0 pgmap v1867: 73 pgs: 73 active+clean; 582 KiB data, 5.1 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:35:25.196 INFO:teuthology.orchestra.run.smithi089.stderr:set osd.5 primary-affinity to 1 (8655362) 2023-06-03T20:35:25.529 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:35:25 smithi089 bash[78452]: audit 1685824525.0808647 mon.a (mon.0) 1737 : audit 1 from='client.? 172.21.15.89:0/283653565' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 5, "weight": 1.0}]: dispatch 2023-06-03T20:35:25.529 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:35:25 smithi089 bash[75068]: audit 1685824525.0808647 mon.a (mon.0) 1737 : audit 1 from='client.? 172.21.15.89:0/283653565' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 5, "weight": 1.0}]: dispatch 2023-06-03T20:35:25.585 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:35:25 smithi164 bash[59462]: audit 1685824525.0808647 mon.a (mon.0) 1737 : audit 1 from='client.? 172.21.15.89:0/283653565' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 5, "weight": 1.0}]: dispatch 2023-06-03T20:35:26.200 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph osd reweight 6 1 2023-06-03T20:35:26.529 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 20:35:26 smithi089 bash[71163]: ::ffff:172.21.15.164 - - [03/Jun/2023:20:35:26] "GET /metrics HTTP/1.1" 200 226935 "" "Prometheus/2.33.4" 2023-06-03T20:35:26.529 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:35:26 smithi089 bash[75068]: cluster 1685824525.184392 osd.5 (osd.5) 264 : cluster 3 failed to encode map e317 with expected crc 2023-06-03T20:35:26.529 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:35:26 smithi089 bash[75068]: cluster 1685824525.1865156 osd.5 (osd.5) 265 : cluster 3 failed to encode map e317 with expected crc 2023-06-03T20:35:26.530 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:35:26 smithi089 bash[75068]: audit 1685824525.1865427 mon.a (mon.0) 1738 : audit 1 from='client.? 172.21.15.89:0/283653565' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 5, "weight": 1.0}]': finished 2023-06-03T20:35:26.530 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:35:26 smithi089 bash[75068]: cluster 1685824525.1865919 mon.a (mon.0) 1739 : cluster 0 osdmap e317: 8 total, 8 up, 8 in 2023-06-03T20:35:26.530 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:35:26 smithi089 bash[75068]: cluster 1685824525.2917213 mgr.y (mgr.34104) 1792 : cluster 0 pgmap v1869: 73 pgs: 73 active+clean; 582 KiB data, 5.1 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:35:26.530 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:35:26 smithi089 bash[78452]: cluster 1685824525.184392 osd.5 (osd.5) 264 : cluster 3 failed to encode map e317 with expected crc 2023-06-03T20:35:26.530 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:35:26 smithi089 bash[78452]: cluster 1685824525.1865156 osd.5 (osd.5) 265 : cluster 3 failed to encode map e317 with expected crc 2023-06-03T20:35:26.530 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:35:26 smithi089 bash[78452]: audit 1685824525.1865427 mon.a (mon.0) 1738 : audit 1 from='client.? 172.21.15.89:0/283653565' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 5, "weight": 1.0}]': finished 2023-06-03T20:35:26.531 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:35:26 smithi089 bash[78452]: cluster 1685824525.1865919 mon.a (mon.0) 1739 : cluster 0 osdmap e317: 8 total, 8 up, 8 in 2023-06-03T20:35:26.531 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:35:26 smithi089 bash[78452]: cluster 1685824525.2917213 mgr.y (mgr.34104) 1792 : cluster 0 pgmap v1869: 73 pgs: 73 active+clean; 582 KiB data, 5.1 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:35:26.585 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:35:26 smithi164 bash[59462]: cluster 1685824525.184392 osd.5 (osd.5) 264 : cluster 3 failed to encode map e317 with expected crc 2023-06-03T20:35:26.586 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:35:26 smithi164 bash[59462]: cluster 1685824525.1865156 osd.5 (osd.5) 265 : cluster 3 failed to encode map e317 with expected crc 2023-06-03T20:35:26.586 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:35:26 smithi164 bash[59462]: audit 1685824525.1865427 mon.a (mon.0) 1738 : audit 1 from='client.? 172.21.15.89:0/283653565' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 5, "weight": 1.0}]': finished 2023-06-03T20:35:26.586 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:35:26 smithi164 bash[59462]: cluster 1685824525.1865919 mon.a (mon.0) 1739 : cluster 0 osdmap e317: 8 total, 8 up, 8 in 2023-06-03T20:35:26.586 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:35:26 smithi164 bash[59462]: cluster 1685824525.2917213 mgr.y (mgr.34104) 1792 : cluster 0 pgmap v1869: 73 pgs: 73 active+clean; 582 KiB data, 5.1 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:35:27.529 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:35:27 smithi089 bash[78452]: cluster 1685824526.1876166 osd.6 (osd.6) 254 : cluster 3 failed to encode map e318 with expected crc 2023-06-03T20:35:27.529 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:35:27 smithi089 bash[78452]: cluster 1685824526.1905148 osd.5 (osd.5) 266 : cluster 3 failed to encode map e318 with expected crc 2023-06-03T20:35:27.529 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:35:27 smithi089 bash[78452]: cluster 1685824526.1932921 osd.5 (osd.5) 267 : cluster 3 failed to encode map e318 with expected crc 2023-06-03T20:35:27.530 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:35:27 smithi089 bash[78452]: cluster 1685824526.1960626 mon.a (mon.0) 1740 : cluster 0 osdmap e318: 8 total, 8 up, 8 in 2023-06-03T20:35:27.530 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:35:27 smithi089 bash[78452]: cluster 1685824526.3850932 osd.7 (osd.7) 260 : cluster 3 failed to encode map e318 with expected crc 2023-06-03T20:35:27.530 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:35:27 smithi089 bash[75068]: cluster 1685824526.1876166 osd.6 (osd.6) 254 : cluster 3 failed to encode map e318 with expected crc 2023-06-03T20:35:27.530 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:35:27 smithi089 bash[75068]: cluster 1685824526.1905148 osd.5 (osd.5) 266 : cluster 3 failed to encode map e318 with expected crc 2023-06-03T20:35:27.530 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:35:27 smithi089 bash[75068]: cluster 1685824526.1932921 osd.5 (osd.5) 267 : cluster 3 failed to encode map e318 with expected crc 2023-06-03T20:35:27.531 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:35:27 smithi089 bash[75068]: cluster 1685824526.1960626 mon.a (mon.0) 1740 : cluster 0 osdmap e318: 8 total, 8 up, 8 in 2023-06-03T20:35:27.531 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:35:27 smithi089 bash[75068]: cluster 1685824526.3850932 osd.7 (osd.7) 260 : cluster 3 failed to encode map e318 with expected crc 2023-06-03T20:35:27.585 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:35:27 smithi164 bash[59462]: cluster 1685824526.1876166 osd.6 (osd.6) 254 : cluster 3 failed to encode map e318 with expected crc 2023-06-03T20:35:27.586 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:35:27 smithi164 bash[59462]: cluster 1685824526.1905148 osd.5 (osd.5) 266 : cluster 3 failed to encode map e318 with expected crc 2023-06-03T20:35:27.586 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:35:27 smithi164 bash[59462]: cluster 1685824526.1932921 osd.5 (osd.5) 267 : cluster 3 failed to encode map e318 with expected crc 2023-06-03T20:35:27.586 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:35:27 smithi164 bash[59462]: cluster 1685824526.1960626 mon.a (mon.0) 1740 : cluster 0 osdmap e318: 8 total, 8 up, 8 in 2023-06-03T20:35:27.586 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:35:27 smithi164 bash[59462]: cluster 1685824526.3850932 osd.7 (osd.7) 260 : cluster 3 failed to encode map e318 with expected crc 2023-06-03T20:35:28.439 INFO:teuthology.orchestra.run.smithi164.stderr:Inferring config /var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/mon.b/config 2023-06-03T20:35:28.481 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:35:28 smithi164 bash[59462]: cluster 1685824527.292604 mgr.y (mgr.34104) 1793 : cluster 0 pgmap v1871: 73 pgs: 73 active+clean; 582 KiB data, 5.1 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:35:28.529 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:35:28 smithi089 bash[75068]: cluster 1685824527.292604 mgr.y (mgr.34104) 1793 : cluster 0 pgmap v1871: 73 pgs: 73 active+clean; 582 KiB data, 5.1 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:35:28.529 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:35:28 smithi089 bash[78452]: cluster 1685824527.292604 mgr.y (mgr.34104) 1793 : cluster 0 pgmap v1871: 73 pgs: 73 active+clean; 582 KiB data, 5.1 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:35:30.585 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:35:30 smithi164 bash[59462]: cluster 1685824529.2935846 mgr.y (mgr.34104) 1794 : cluster 0 pgmap v1872: 73 pgs: 73 active+clean; 582 KiB data, 5.1 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:35:30.779 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:35:30 smithi089 bash[78452]: cluster 1685824529.2935846 mgr.y (mgr.34104) 1794 : cluster 0 pgmap v1872: 73 pgs: 73 active+clean; 582 KiB data, 5.1 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:35:30.779 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:35:30 smithi089 bash[75068]: cluster 1685824529.2935846 mgr.y (mgr.34104) 1794 : cluster 0 pgmap v1872: 73 pgs: 73 active+clean; 582 KiB data, 5.1 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:35:31.004 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/mon.a/config 2023-06-03T20:35:32.337 INFO:teuthology.orchestra.run.smithi164.stdout:{ 2023-06-03T20:35:32.337 INFO:teuthology.orchestra.run.smithi164.stdout: "size": 20, 2023-06-03T20:35:32.337 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 600, 2023-06-03T20:35:32.337 INFO:teuthology.orchestra.run.smithi164.stdout: "ops": [ 2023-06-03T20:35:32.338 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:35:32.338 INFO:teuthology.orchestra.run.smithi164.stdout: "description": "osd_op(client.46739.0:87958 14.e 14:72e738fb:::benchmark_data_smithi089_237281_object87957:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e289)", 2023-06-03T20:35:32.338 INFO:teuthology.orchestra.run.smithi164.stdout: "initiated_at": "2023-06-03T20:29:02.675877+0000", 2023-06-03T20:35:32.338 INFO:teuthology.orchestra.run.smithi164.stdout: "age": 389.649470044, 2023-06-03T20:35:32.338 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.29321554999999999, 2023-06-03T20:35:32.338 INFO:teuthology.orchestra.run.smithi164.stdout: "type_data": { 2023-06-03T20:35:32.338 INFO:teuthology.orchestra.run.smithi164.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-03T20:35:32.339 INFO:teuthology.orchestra.run.smithi164.stdout: "client_info": { 2023-06-03T20:35:32.339 INFO:teuthology.orchestra.run.smithi164.stdout: "client": "client.46739", 2023-06-03T20:35:32.339 INFO:teuthology.orchestra.run.smithi164.stdout: "client_addr": "172.21.15.89:0/2479974613", 2023-06-03T20:35:32.339 INFO:teuthology.orchestra.run.smithi164.stdout: "tid": 87958 2023-06-03T20:35:32.339 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:35:32.339 INFO:teuthology.orchestra.run.smithi164.stdout: "events": [ 2023-06-03T20:35:32.339 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:35:32.339 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "initiated", 2023-06-03T20:35:32.340 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:29:02.675877+0000", 2023-06-03T20:35:32.340 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0 2023-06-03T20:35:32.340 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:35:32.340 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:35:32.340 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "throttled", 2023-06-03T20:35:32.340 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:29:02.675877+0000", 2023-06-03T20:35:32.341 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0 2023-06-03T20:35:32.341 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:35:32.341 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:35:32.341 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "header_read", 2023-06-03T20:35:32.341 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:29:02.675875+0000", 2023-06-03T20:35:32.341 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 4294967295.9999986 2023-06-03T20:35:32.341 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:35:32.341 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:35:32.342 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "all_read", 2023-06-03T20:35:32.342 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:29:02.675934+0000", 2023-06-03T20:35:32.342 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 5.8669999999999999e-05 2023-06-03T20:35:32.342 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:35:32.342 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:35:32.342 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "dispatched", 2023-06-03T20:35:32.342 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:29:02.675945+0000", 2023-06-03T20:35:32.343 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 1.1620000000000001e-05 2023-06-03T20:35:32.343 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:35:32.343 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:35:32.343 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "queued_for_pg", 2023-06-03T20:35:32.343 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:29:02.675951+0000", 2023-06-03T20:35:32.343 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 6.0129999999999999e-06 2023-06-03T20:35:32.343 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:35:32.343 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:35:32.344 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "reached_pg", 2023-06-03T20:35:32.344 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:29:02.675999+0000", 2023-06-03T20:35:32.344 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 4.7679000000000002e-05 2023-06-03T20:35:32.344 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:35:32.344 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:35:32.344 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "started", 2023-06-03T20:35:32.345 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:29:02.676144+0000", 2023-06-03T20:35:32.345 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.000144437 2023-06-03T20:35:32.345 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:35:32.345 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:35:32.345 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "waiting for subops from 2,6", 2023-06-03T20:35:32.345 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:29:02.676269+0000", 2023-06-03T20:35:32.345 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.00012550199999999999 2023-06-03T20:35:32.346 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:35:32.346 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:35:32.346 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "op_commit", 2023-06-03T20:35:32.346 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:29:02.679250+0000", 2023-06-03T20:35:32.346 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.0029812139999999998 2023-06-03T20:35:32.346 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:35:32.346 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:35:32.347 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:35:32.347 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:29:02.680235+0000", 2023-06-03T20:35:32.347 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.0009849399999999999 2023-06-03T20:35:32.347 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:35:32.347 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:35:32.347 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:35:32.348 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:29:02.968824+0000", 2023-06-03T20:35:32.348 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.288588438 2023-06-03T20:35:32.348 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:35:32.348 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:35:32.348 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "commit_sent", 2023-06-03T20:35:32.348 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:29:02.968977+0000", 2023-06-03T20:35:32.348 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.00015328100000000001 2023-06-03T20:35:32.349 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:35:32.349 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:35:32.349 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "done", 2023-06-03T20:35:32.349 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:29:02.969092+0000", 2023-06-03T20:35:32.349 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.000115121 2023-06-03T20:35:32.349 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:35:32.349 INFO:teuthology.orchestra.run.smithi164.stdout: ] 2023-06-03T20:35:32.349 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:35:32.350 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:35:32.350 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:35:32.350 INFO:teuthology.orchestra.run.smithi164.stdout: "description": "osd_op(client.46739.0:100958 14.e 14:7a30f7a0:::benchmark_data_smithi089_237281_object2942:head [delete] snapc 0=[] ondisk+write+known_if_redirected e289)", 2023-06-03T20:35:32.350 INFO:teuthology.orchestra.run.smithi164.stdout: "initiated_at": "2023-06-03T20:29:20.786885+0000", 2023-06-03T20:35:32.350 INFO:teuthology.orchestra.run.smithi164.stdout: "age": 371.53846157599997, 2023-06-03T20:35:32.350 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.22647610400000001, 2023-06-03T20:35:32.351 INFO:teuthology.orchestra.run.smithi164.stdout: "type_data": { 2023-06-03T20:35:32.351 INFO:teuthology.orchestra.run.smithi164.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-03T20:35:32.351 INFO:teuthology.orchestra.run.smithi164.stdout: "client_info": { 2023-06-03T20:35:32.351 INFO:teuthology.orchestra.run.smithi164.stdout: "client": "client.46739", 2023-06-03T20:35:32.351 INFO:teuthology.orchestra.run.smithi164.stdout: "client_addr": "172.21.15.89:0/2479974613", 2023-06-03T20:35:32.352 INFO:teuthology.orchestra.run.smithi164.stdout: "tid": 100958 2023-06-03T20:35:32.352 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:35:32.352 INFO:teuthology.orchestra.run.smithi164.stdout: "events": [ 2023-06-03T20:35:32.352 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:35:32.352 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "initiated", 2023-06-03T20:35:32.352 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:29:20.786885+0000", 2023-06-03T20:35:32.352 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0 2023-06-03T20:35:32.353 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:35:32.353 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:35:32.353 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "throttled", 2023-06-03T20:35:32.353 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:29:20.786885+0000", 2023-06-03T20:35:32.353 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0 2023-06-03T20:35:32.353 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:35:32.353 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:35:32.354 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "header_read", 2023-06-03T20:35:32.354 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:29:20.786884+0000", 2023-06-03T20:35:32.354 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 4294967295.999999 2023-06-03T20:35:32.354 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:35:32.354 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:35:32.354 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "all_read", 2023-06-03T20:35:32.354 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:29:20.786890+0000", 2023-06-03T20:35:32.354 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 5.4360000000000001e-06 2023-06-03T20:35:32.355 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:35:32.355 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:35:32.355 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "dispatched", 2023-06-03T20:35:32.355 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:29:20.786901+0000", 2023-06-03T20:35:32.355 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 1.1199000000000001e-05 2023-06-03T20:35:32.355 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:35:32.355 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:35:32.355 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "queued_for_pg", 2023-06-03T20:35:32.356 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:29:20.786908+0000", 2023-06-03T20:35:32.356 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 7.7000000000000008e-06 2023-06-03T20:35:32.356 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:35:32.356 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:35:32.356 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "reached_pg", 2023-06-03T20:35:32.356 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:29:20.786977+0000", 2023-06-03T20:35:32.356 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 6.8435999999999993e-05 2023-06-03T20:35:32.357 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:35:32.357 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:35:32.357 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "started", 2023-06-03T20:35:32.357 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:29:20.787142+0000", 2023-06-03T20:35:32.357 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.00016469299999999999 2023-06-03T20:35:32.358 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:35:32.358 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:35:32.358 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "waiting for subops from 2,6", 2023-06-03T20:35:32.358 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:29:20.787267+0000", 2023-06-03T20:35:32.358 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.00012563599999999999 2023-06-03T20:35:32.358 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:35:32.358 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:35:32.359 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "op_commit", 2023-06-03T20:35:32.359 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:29:20.794617+0000", 2023-06-03T20:35:32.359 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.00734981 2023-06-03T20:35:32.359 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:35:32.359 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:35:32.359 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:35:32.359 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:29:20.796435+0000", 2023-06-03T20:35:32.359 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.001818102 2023-06-03T20:35:32.360 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:35:32.360 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:35:32.360 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:35:32.360 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:29:21.013256+0000", 2023-06-03T20:35:32.360 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.21682094199999999 2023-06-03T20:35:32.360 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:35:32.360 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:35:32.361 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "commit_sent", 2023-06-03T20:35:32.361 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:29:21.013304+0000", 2023-06-03T20:35:32.361 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 4.7484000000000001e-05 2023-06-03T20:35:32.361 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:35:32.361 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:35:32.361 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "done", 2023-06-03T20:35:32.361 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:29:21.013361+0000", 2023-06-03T20:35:32.362 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 5.7540000000000001e-05 2023-06-03T20:35:32.362 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:35:32.362 INFO:teuthology.orchestra.run.smithi164.stdout: ] 2023-06-03T20:35:32.362 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:35:32.362 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:35:32.362 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:35:32.362 INFO:teuthology.orchestra.run.smithi164.stdout: "description": "osd_op(client.46739.0:100959 14.e 14:762f310d:::benchmark_data_smithi089_237281_object2943:head [delete] snapc 0=[] ondisk+write+known_if_redirected e289)", 2023-06-03T20:35:32.362 INFO:teuthology.orchestra.run.smithi164.stdout: "initiated_at": "2023-06-03T20:29:20.789783+0000", 2023-06-03T20:35:32.363 INFO:teuthology.orchestra.run.smithi164.stdout: "age": 371.53556370299998, 2023-06-03T20:35:32.363 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.22376321399999999, 2023-06-03T20:35:32.363 INFO:teuthology.orchestra.run.smithi164.stdout: "type_data": { 2023-06-03T20:35:32.363 INFO:teuthology.orchestra.run.smithi164.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-03T20:35:32.363 INFO:teuthology.orchestra.run.smithi164.stdout: "client_info": { 2023-06-03T20:35:32.363 INFO:teuthology.orchestra.run.smithi164.stdout: "client": "client.46739", 2023-06-03T20:35:32.363 INFO:teuthology.orchestra.run.smithi164.stdout: "client_addr": "172.21.15.89:0/2479974613", 2023-06-03T20:35:32.363 INFO:teuthology.orchestra.run.smithi164.stdout: "tid": 100959 2023-06-03T20:35:32.364 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:35:32.364 INFO:teuthology.orchestra.run.smithi164.stdout: "events": [ 2023-06-03T20:35:32.364 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:35:32.364 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "initiated", 2023-06-03T20:35:32.364 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:29:20.789783+0000", 2023-06-03T20:35:32.364 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0 2023-06-03T20:35:32.365 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:35:32.365 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:35:32.365 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "throttled", 2023-06-03T20:35:32.365 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:29:20.789783+0000", 2023-06-03T20:35:32.365 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0 2023-06-03T20:35:32.365 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:35:32.365 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:35:32.365 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "header_read", 2023-06-03T20:35:32.366 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:29:20.789781+0000", 2023-06-03T20:35:32.366 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 4294967295.9999981 2023-06-03T20:35:32.366 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:35:32.366 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:35:32.366 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "all_read", 2023-06-03T20:35:32.366 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:29:20.789791+0000", 2023-06-03T20:35:32.366 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 9.8800000000000003e-06 2023-06-03T20:35:32.366 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:35:32.367 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:35:32.367 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "dispatched", 2023-06-03T20:35:32.367 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:29:20.789813+0000", 2023-06-03T20:35:32.367 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 2.1688e-05 2023-06-03T20:35:32.367 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:35:32.367 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:35:32.367 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "queued_for_pg", 2023-06-03T20:35:32.367 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:29:20.789826+0000", 2023-06-03T20:35:32.368 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 1.3763e-05 2023-06-03T20:35:32.368 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:35:32.368 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:35:32.368 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "reached_pg", 2023-06-03T20:35:32.368 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:29:20.790940+0000", 2023-06-03T20:35:32.369 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.001113525 2023-06-03T20:35:32.369 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:35:32.369 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:35:32.369 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "started", 2023-06-03T20:35:32.369 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:29:20.791136+0000", 2023-06-03T20:35:32.369 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.00019586000000000001 2023-06-03T20:35:32.369 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:35:32.369 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:35:32.370 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "waiting for subops from 2,6", 2023-06-03T20:35:32.370 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:29:20.791302+0000", 2023-06-03T20:35:32.370 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.00016651400000000001 2023-06-03T20:35:32.370 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:35:32.370 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:35:32.370 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "op_commit", 2023-06-03T20:35:32.370 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:29:20.794797+0000", 2023-06-03T20:35:32.371 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.0034947089999999999 2023-06-03T20:35:32.371 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:35:32.371 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:35:32.371 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:35:32.371 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:29:20.796534+0000", 2023-06-03T20:35:32.371 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.001736881 2023-06-03T20:35:32.371 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:35:32.371 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:35:32.372 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:35:32.372 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:29:21.013428+0000", 2023-06-03T20:35:32.372 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.216893792 2023-06-03T20:35:32.372 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:35:32.372 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:35:32.372 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "commit_sent", 2023-06-03T20:35:32.372 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:29:21.013503+0000", 2023-06-03T20:35:32.372 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 7.5240000000000005e-05 2023-06-03T20:35:32.373 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:35:32.373 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:35:32.373 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "done", 2023-06-03T20:35:32.373 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:29:21.013546+0000", 2023-06-03T20:35:32.373 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 4.3254000000000003e-05 2023-06-03T20:35:32.373 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:35:32.373 INFO:teuthology.orchestra.run.smithi164.stdout: ] 2023-06-03T20:35:32.374 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:35:32.374 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:35:32.374 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:35:32.374 INFO:teuthology.orchestra.run.smithi164.stdout: "description": "osd_op(client.46739.0:143770 14.e 14:705fda41:::benchmark_data_smithi089_237281_object45754:head [delete] snapc 0=[] ondisk+write+known_if_redirected e291)", 2023-06-03T20:35:32.374 INFO:teuthology.orchestra.run.smithi164.stdout: "initiated_at": "2023-06-03T20:30:28.276204+0000", 2023-06-03T20:35:32.374 INFO:teuthology.orchestra.run.smithi164.stdout: "age": 304.049142313, 2023-06-03T20:35:32.374 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.18917615800000001, 2023-06-03T20:35:32.375 INFO:teuthology.orchestra.run.smithi164.stdout: "type_data": { 2023-06-03T20:35:32.375 INFO:teuthology.orchestra.run.smithi164.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-03T20:35:32.375 INFO:teuthology.orchestra.run.smithi164.stdout: "client_info": { 2023-06-03T20:35:32.375 INFO:teuthology.orchestra.run.smithi164.stdout: "client": "client.46739", 2023-06-03T20:35:32.375 INFO:teuthology.orchestra.run.smithi164.stdout: "client_addr": "172.21.15.89:0/2479974613", 2023-06-03T20:35:32.375 INFO:teuthology.orchestra.run.smithi164.stdout: "tid": 143770 2023-06-03T20:35:32.375 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:35:32.376 INFO:teuthology.orchestra.run.smithi164.stdout: "events": [ 2023-06-03T20:35:32.376 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:35:32.376 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "initiated", 2023-06-03T20:35:32.376 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:30:28.276204+0000", 2023-06-03T20:35:32.376 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0 2023-06-03T20:35:32.376 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:35:32.376 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:35:32.376 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "throttled", 2023-06-03T20:35:32.377 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:30:28.276204+0000", 2023-06-03T20:35:32.377 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0 2023-06-03T20:35:32.377 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:35:32.377 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:35:32.377 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "header_read", 2023-06-03T20:35:32.377 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:30:28.276203+0000", 2023-06-03T20:35:32.377 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 4294967295.9999986 2023-06-03T20:35:32.378 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:35:32.378 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:35:32.378 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "all_read", 2023-06-03T20:35:32.378 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:30:28.276210+0000", 2023-06-03T20:35:32.378 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 6.939e-06 2023-06-03T20:35:32.378 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:35:32.379 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:35:32.379 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "dispatched", 2023-06-03T20:35:32.379 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:30:28.276229+0000", 2023-06-03T20:35:32.379 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 1.8785000000000001e-05 2023-06-03T20:35:32.379 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:35:32.379 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:35:32.379 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "queued_for_pg", 2023-06-03T20:35:32.380 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:30:28.276242+0000", 2023-06-03T20:35:32.380 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 1.3104e-05 2023-06-03T20:35:32.380 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:35:32.380 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:35:32.380 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "reached_pg", 2023-06-03T20:35:32.380 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:30:28.281044+0000", 2023-06-03T20:35:32.380 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.0048026919999999999 2023-06-03T20:35:32.380 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:35:32.381 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:35:32.381 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "started", 2023-06-03T20:35:32.381 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:30:28.281212+0000", 2023-06-03T20:35:32.381 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.00016731 2023-06-03T20:35:32.381 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:35:32.381 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:35:32.381 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "waiting for subops from 2,6", 2023-06-03T20:35:32.382 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:30:28.281326+0000", 2023-06-03T20:35:32.382 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.000114757 2023-06-03T20:35:32.382 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:35:32.382 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:35:32.382 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:35:32.382 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:30:28.292185+0000", 2023-06-03T20:35:32.382 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.010858825000000001 2023-06-03T20:35:32.383 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:35:32.383 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:35:32.383 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "op_commit", 2023-06-03T20:35:32.383 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:30:28.292307+0000", 2023-06-03T20:35:32.383 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.00012218 2023-06-03T20:35:32.383 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:35:32.383 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:35:32.383 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:35:32.384 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:30:28.465149+0000", 2023-06-03T20:35:32.384 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.17284202500000001 2023-06-03T20:35:32.384 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:35:32.384 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:35:32.384 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "commit_sent", 2023-06-03T20:35:32.385 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:30:28.465282+0000", 2023-06-03T20:35:32.385 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.00013227700000000001 2023-06-03T20:35:32.385 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:35:32.386 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:35:32.386 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "done", 2023-06-03T20:35:32.386 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:30:28.465380+0000", 2023-06-03T20:35:32.386 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 9.8621000000000003e-05 2023-06-03T20:35:32.387 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:35:32.387 INFO:teuthology.orchestra.run.smithi164.stdout: ] 2023-06-03T20:35:32.387 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:35:32.387 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:35:32.388 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:35:32.388 INFO:teuthology.orchestra.run.smithi164.stdout: "description": "osd_op(client.46739.0:143771 14.e 14:719cda19:::benchmark_data_smithi089_237281_object45755:head [delete] snapc 0=[] ondisk+write+known_if_redirected e291)", 2023-06-03T20:35:32.388 INFO:teuthology.orchestra.run.smithi164.stdout: "initiated_at": "2023-06-03T20:30:28.277391+0000", 2023-06-03T20:35:32.389 INFO:teuthology.orchestra.run.smithi164.stdout: "age": 304.04795523899998, 2023-06-03T20:35:32.389 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.188256277, 2023-06-03T20:35:32.389 INFO:teuthology.orchestra.run.smithi164.stdout: "type_data": { 2023-06-03T20:35:32.389 INFO:teuthology.orchestra.run.smithi164.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-03T20:35:32.389 INFO:teuthology.orchestra.run.smithi164.stdout: "client_info": { 2023-06-03T20:35:32.389 INFO:teuthology.orchestra.run.smithi164.stdout: "client": "client.46739", 2023-06-03T20:35:32.390 INFO:teuthology.orchestra.run.smithi164.stdout: "client_addr": "172.21.15.89:0/2479974613", 2023-06-03T20:35:32.390 INFO:teuthology.orchestra.run.smithi164.stdout: "tid": 143771 2023-06-03T20:35:32.390 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:35:32.390 INFO:teuthology.orchestra.run.smithi164.stdout: "events": [ 2023-06-03T20:35:32.390 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:35:32.390 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "initiated", 2023-06-03T20:35:32.390 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:30:28.277391+0000", 2023-06-03T20:35:32.391 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0 2023-06-03T20:35:32.391 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:35:32.391 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:35:32.391 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "throttled", 2023-06-03T20:35:32.391 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:30:28.277391+0000", 2023-06-03T20:35:32.391 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0 2023-06-03T20:35:32.392 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:35:32.392 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:35:32.392 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "header_read", 2023-06-03T20:35:32.392 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:30:28.277390+0000", 2023-06-03T20:35:32.392 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 4294967295.999999 2023-06-03T20:35:32.392 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:35:32.392 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:35:32.393 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "all_read", 2023-06-03T20:35:32.393 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:30:28.277396+0000", 2023-06-03T20:35:32.393 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 5.6049999999999997e-06 2023-06-03T20:35:32.393 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:35:32.393 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:35:32.393 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "dispatched", 2023-06-03T20:35:32.393 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:30:28.277410+0000", 2023-06-03T20:35:32.394 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 1.3835999999999999e-05 2023-06-03T20:35:32.394 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:35:32.394 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:35:32.394 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "queued_for_pg", 2023-06-03T20:35:32.394 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:30:28.277419+0000", 2023-06-03T20:35:32.394 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 9.6709999999999994e-06 2023-06-03T20:35:32.394 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:35:32.394 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:35:32.395 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "reached_pg", 2023-06-03T20:35:32.395 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:30:28.286940+0000", 2023-06-03T20:35:32.395 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.0095208010000000006 2023-06-03T20:35:32.395 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:35:32.395 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:35:32.395 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "started", 2023-06-03T20:35:32.395 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:30:28.287136+0000", 2023-06-03T20:35:32.395 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.00019624599999999999 2023-06-03T20:35:32.396 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:35:32.396 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:35:32.396 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "waiting for subops from 2,6", 2023-06-03T20:35:32.396 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:30:28.287330+0000", 2023-06-03T20:35:32.396 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.00019387000000000001 2023-06-03T20:35:32.396 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:35:32.396 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:35:32.397 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "op_commit", 2023-06-03T20:35:32.397 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:30:28.292316+0000", 2023-06-03T20:35:32.397 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.0049854280000000001 2023-06-03T20:35:32.397 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:35:32.397 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:35:32.397 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:35:32.397 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:30:28.292353+0000", 2023-06-03T20:35:32.397 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 3.7567000000000001e-05 2023-06-03T20:35:32.398 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:35:32.398 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:35:32.398 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:35:32.398 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:30:28.465484+0000", 2023-06-03T20:35:32.398 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.17313029599999999 2023-06-03T20:35:32.398 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:35:32.398 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:35:32.398 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "commit_sent", 2023-06-03T20:35:32.399 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:30:28.465573+0000", 2023-06-03T20:35:32.399 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 8.9790999999999995e-05 2023-06-03T20:35:32.399 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:35:32.399 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:35:32.399 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "done", 2023-06-03T20:35:32.399 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:30:28.465648+0000", 2023-06-03T20:35:32.400 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 7.4179999999999998e-05 2023-06-03T20:35:32.400 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:35:32.400 INFO:teuthology.orchestra.run.smithi164.stdout: ] 2023-06-03T20:35:32.400 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:35:32.400 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:35:32.400 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:35:32.400 INFO:teuthology.orchestra.run.smithi164.stdout: "description": "osd_op(client.46739.0:171483 14.e 14:7eb15089:::benchmark_data_smithi089_237281_object73467:head [delete] snapc 0=[] ondisk+write+known_if_redirected e293)", 2023-06-03T20:35:32.401 INFO:teuthology.orchestra.run.smithi164.stdout: "initiated_at": "2023-06-03T20:31:16.582442+0000", 2023-06-03T20:35:32.401 INFO:teuthology.orchestra.run.smithi164.stdout: "age": 255.742904458, 2023-06-03T20:35:32.401 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.21443590900000001, 2023-06-03T20:35:32.401 INFO:teuthology.orchestra.run.smithi164.stdout: "type_data": { 2023-06-03T20:35:32.401 INFO:teuthology.orchestra.run.smithi164.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-03T20:35:32.401 INFO:teuthology.orchestra.run.smithi164.stdout: "client_info": { 2023-06-03T20:35:32.401 INFO:teuthology.orchestra.run.smithi164.stdout: "client": "client.46739", 2023-06-03T20:35:32.401 INFO:teuthology.orchestra.run.smithi164.stdout: "client_addr": "172.21.15.89:0/2479974613", 2023-06-03T20:35:32.402 INFO:teuthology.orchestra.run.smithi164.stdout: "tid": 171483 2023-06-03T20:35:32.402 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:35:32.402 INFO:teuthology.orchestra.run.smithi164.stdout: "events": [ 2023-06-03T20:35:32.402 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:35:32.402 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "initiated", 2023-06-03T20:35:32.402 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:31:16.582442+0000", 2023-06-03T20:35:32.402 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0 2023-06-03T20:35:32.403 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:35:32.403 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:35:32.403 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "throttled", 2023-06-03T20:35:32.403 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:31:16.582442+0000", 2023-06-03T20:35:32.403 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0 2023-06-03T20:35:32.403 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:35:32.403 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:35:32.403 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "header_read", 2023-06-03T20:35:32.404 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:31:16.582440+0000", 2023-06-03T20:35:32.404 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 4294967295.9999981 2023-06-03T20:35:32.404 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:35:32.404 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:35:32.405 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "all_read", 2023-06-03T20:35:32.405 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:31:16.582451+0000", 2023-06-03T20:35:32.405 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 1.1283e-05 2023-06-03T20:35:32.405 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:35:32.405 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:35:32.405 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "dispatched", 2023-06-03T20:35:32.406 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:31:16.582472+0000", 2023-06-03T20:35:32.406 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 2.1114000000000001e-05 2023-06-03T20:35:32.406 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:35:32.406 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:35:32.406 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "queued_for_pg", 2023-06-03T20:35:32.406 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:31:16.582485+0000", 2023-06-03T20:35:32.406 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 1.2181000000000001e-05 2023-06-03T20:35:32.406 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:35:32.407 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:35:32.407 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "reached_pg", 2023-06-03T20:35:32.407 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:31:16.584392+0000", 2023-06-03T20:35:32.407 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.0019072780000000001 2023-06-03T20:35:32.407 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:35:32.407 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:35:32.407 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "started", 2023-06-03T20:35:32.407 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:31:16.584548+0000", 2023-06-03T20:35:32.408 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.00015569600000000001 2023-06-03T20:35:32.408 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:35:32.408 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:35:32.408 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "waiting for subops from 2,6", 2023-06-03T20:35:32.408 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:31:16.584694+0000", 2023-06-03T20:35:32.408 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.00014630099999999999 2023-06-03T20:35:32.408 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:35:32.409 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:35:32.409 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "op_commit", 2023-06-03T20:35:32.409 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:31:16.590048+0000", 2023-06-03T20:35:32.409 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.0053537230000000003 2023-06-03T20:35:32.409 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:35:32.409 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:35:32.409 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:35:32.410 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:31:16.592060+0000", 2023-06-03T20:35:32.410 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.002012771 2023-06-03T20:35:32.410 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:35:32.410 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:35:32.410 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:35:32.410 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:31:16.796710+0000", 2023-06-03T20:35:32.411 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.204650158 2023-06-03T20:35:32.411 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:35:32.411 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:35:32.411 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "commit_sent", 2023-06-03T20:35:32.411 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:31:16.796797+0000", 2023-06-03T20:35:32.411 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 8.6872000000000004e-05 2023-06-03T20:35:32.411 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:35:32.412 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:35:32.412 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "done", 2023-06-03T20:35:32.412 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:31:16.796878+0000", 2023-06-03T20:35:32.412 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 8.0641000000000003e-05 2023-06-03T20:35:32.412 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:35:32.412 INFO:teuthology.orchestra.run.smithi164.stdout: ] 2023-06-03T20:35:32.412 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:35:32.413 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:35:32.413 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:35:32.413 INFO:teuthology.orchestra.run.smithi164.stdout: "description": "osd_op(client.46739.0:171502 14.e 14:73b636c4:::benchmark_data_smithi089_237281_object73486:head [delete] snapc 0=[] ondisk+write+known_if_redirected e293)", 2023-06-03T20:35:32.413 INFO:teuthology.orchestra.run.smithi164.stdout: "initiated_at": "2023-06-03T20:31:16.615709+0000", 2023-06-03T20:35:32.413 INFO:teuthology.orchestra.run.smithi164.stdout: "age": 255.70963739800001, 2023-06-03T20:35:32.413 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.18945182999999999, 2023-06-03T20:35:32.413 INFO:teuthology.orchestra.run.smithi164.stdout: "type_data": { 2023-06-03T20:35:32.413 INFO:teuthology.orchestra.run.smithi164.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-03T20:35:32.414 INFO:teuthology.orchestra.run.smithi164.stdout: "client_info": { 2023-06-03T20:35:32.414 INFO:teuthology.orchestra.run.smithi164.stdout: "client": "client.46739", 2023-06-03T20:35:32.414 INFO:teuthology.orchestra.run.smithi164.stdout: "client_addr": "172.21.15.89:0/2479974613", 2023-06-03T20:35:32.414 INFO:teuthology.orchestra.run.smithi164.stdout: "tid": 171502 2023-06-03T20:35:32.414 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:35:32.414 INFO:teuthology.orchestra.run.smithi164.stdout: "events": [ 2023-06-03T20:35:32.414 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:35:32.415 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "initiated", 2023-06-03T20:35:32.415 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:31:16.615709+0000", 2023-06-03T20:35:32.415 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0 2023-06-03T20:35:32.415 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:35:32.415 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:35:32.415 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "throttled", 2023-06-03T20:35:32.415 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:31:16.615709+0000", 2023-06-03T20:35:32.415 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0 2023-06-03T20:35:32.416 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:35:32.416 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:35:32.416 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "header_read", 2023-06-03T20:35:32.416 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:31:16.615708+0000", 2023-06-03T20:35:32.416 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 4294967295.9999986 2023-06-03T20:35:32.416 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:35:32.416 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:35:32.417 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "all_read", 2023-06-03T20:35:32.417 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:31:16.615719+0000", 2023-06-03T20:35:32.417 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 1.1032999999999999e-05 2023-06-03T20:35:32.417 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:35:32.417 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:35:32.417 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "dispatched", 2023-06-03T20:35:32.417 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:31:16.615734+0000", 2023-06-03T20:35:32.417 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 1.5528000000000001e-05 2023-06-03T20:35:32.418 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:35:32.418 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:35:32.418 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "queued_for_pg", 2023-06-03T20:35:32.418 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:31:16.615744+0000", 2023-06-03T20:35:32.418 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 9.7070000000000004e-06 2023-06-03T20:35:32.418 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:35:32.418 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:35:32.418 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "reached_pg", 2023-06-03T20:35:32.419 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:31:16.615805+0000", 2023-06-03T20:35:32.419 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 6.0863999999999999e-05 2023-06-03T20:35:32.419 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:35:32.419 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:35:32.419 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "started", 2023-06-03T20:35:32.419 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:31:16.615954+0000", 2023-06-03T20:35:32.419 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.000149565 2023-06-03T20:35:32.419 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:35:32.420 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:35:32.420 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "waiting for subops from 2,6", 2023-06-03T20:35:32.420 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:31:16.616100+0000", 2023-06-03T20:35:32.420 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.00014590999999999999 2023-06-03T20:35:32.420 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:35:32.421 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:35:32.421 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "op_commit", 2023-06-03T20:35:32.421 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:31:16.620311+0000", 2023-06-03T20:35:32.421 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.0042101630000000003 2023-06-03T20:35:32.421 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:35:32.421 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:35:32.421 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:35:32.422 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:31:16.620575+0000", 2023-06-03T20:35:32.422 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.000264646 2023-06-03T20:35:32.422 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:35:32.422 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:35:32.422 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:35:32.422 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:31:16.804867+0000", 2023-06-03T20:35:32.422 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.18429216700000001 2023-06-03T20:35:32.423 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:35:32.423 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:35:32.423 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "commit_sent", 2023-06-03T20:35:32.423 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:31:16.805016+0000", 2023-06-03T20:35:32.423 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.00014905699999999999 2023-06-03T20:35:32.423 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:35:32.423 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:35:32.424 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "done", 2023-06-03T20:35:32.424 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:31:16.805161+0000", 2023-06-03T20:35:32.424 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.00014455199999999999 2023-06-03T20:35:32.424 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:35:32.424 INFO:teuthology.orchestra.run.smithi164.stdout: ] 2023-06-03T20:35:32.424 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:35:32.424 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:35:32.425 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:35:32.425 INFO:teuthology.orchestra.run.smithi164.stdout: "description": "osd_op(client.46739.0:177016 14.7 14:eff8a93a:::benchmark_data_smithi089_237281_object79000:head [delete] snapc 0=[] ondisk+write+known_if_redirected e293)", 2023-06-03T20:35:32.425 INFO:teuthology.orchestra.run.smithi164.stdout: "initiated_at": "2023-06-03T20:31:26.258946+0000", 2023-06-03T20:35:32.425 INFO:teuthology.orchestra.run.smithi164.stdout: "age": 246.06640028999999, 2023-06-03T20:35:32.425 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.258915966, 2023-06-03T20:35:32.425 INFO:teuthology.orchestra.run.smithi164.stdout: "type_data": { 2023-06-03T20:35:32.425 INFO:teuthology.orchestra.run.smithi164.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-03T20:35:32.426 INFO:teuthology.orchestra.run.smithi164.stdout: "client_info": { 2023-06-03T20:35:32.426 INFO:teuthology.orchestra.run.smithi164.stdout: "client": "client.46739", 2023-06-03T20:35:32.426 INFO:teuthology.orchestra.run.smithi164.stdout: "client_addr": "172.21.15.89:0/2479974613", 2023-06-03T20:35:32.426 INFO:teuthology.orchestra.run.smithi164.stdout: "tid": 177016 2023-06-03T20:35:32.426 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:35:32.427 INFO:teuthology.orchestra.run.smithi164.stdout: "events": [ 2023-06-03T20:35:32.427 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:35:32.427 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "initiated", 2023-06-03T20:35:32.427 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:31:26.258946+0000", 2023-06-03T20:35:32.427 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0 2023-06-03T20:35:32.427 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:35:32.427 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:35:32.428 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "throttled", 2023-06-03T20:35:32.428 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:31:26.258946+0000", 2023-06-03T20:35:32.428 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0 2023-06-03T20:35:32.428 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:35:32.429 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:35:32.429 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "header_read", 2023-06-03T20:35:32.429 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:31:26.258945+0000", 2023-06-03T20:35:32.430 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 4294967295.9999981 2023-06-03T20:35:32.430 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:35:32.430 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:35:32.430 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "all_read", 2023-06-03T20:35:32.430 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:31:26.258952+0000", 2023-06-03T20:35:32.430 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 7.6590000000000007e-06 2023-06-03T20:35:32.430 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:35:32.431 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:35:32.431 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "dispatched", 2023-06-03T20:35:32.431 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:31:26.258969+0000", 2023-06-03T20:35:32.431 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 1.6946999999999999e-05 2023-06-03T20:35:32.431 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:35:32.431 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:35:32.432 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "queued_for_pg", 2023-06-03T20:35:32.432 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:31:26.258978+0000", 2023-06-03T20:35:32.432 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 8.3720000000000005e-06 2023-06-03T20:35:32.432 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:35:32.432 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:35:32.432 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "reached_pg", 2023-06-03T20:35:32.432 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:31:26.259028+0000", 2023-06-03T20:35:32.433 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 5.0139999999999998e-05 2023-06-03T20:35:32.433 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:35:32.433 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:35:32.433 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "started", 2023-06-03T20:35:32.433 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:31:26.259183+0000", 2023-06-03T20:35:32.433 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.00015537999999999999 2023-06-03T20:35:32.433 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:35:32.434 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:35:32.434 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "waiting for subops from 0,5", 2023-06-03T20:35:32.434 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:31:26.259337+0000", 2023-06-03T20:35:32.434 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.00015425599999999999 2023-06-03T20:35:32.434 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:35:32.434 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:35:32.434 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "op_commit", 2023-06-03T20:35:32.434 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:31:26.263149+0000", 2023-06-03T20:35:32.435 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.003811381 2023-06-03T20:35:32.435 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:35:32.435 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:35:32.435 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:35:32.435 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:31:26.265926+0000", 2023-06-03T20:35:32.435 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.002777725 2023-06-03T20:35:32.435 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:35:32.436 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:35:32.436 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:35:32.436 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:31:26.517769+0000", 2023-06-03T20:35:32.436 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.25184256700000002 2023-06-03T20:35:32.436 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:35:32.436 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:35:32.436 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "commit_sent", 2023-06-03T20:35:32.437 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:31:26.517824+0000", 2023-06-03T20:35:32.437 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 5.4604000000000001e-05 2023-06-03T20:35:32.437 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:35:32.437 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:35:32.437 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "done", 2023-06-03T20:35:32.437 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:31:26.517862+0000", 2023-06-03T20:35:32.437 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 3.8618000000000002e-05 2023-06-03T20:35:32.437 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:35:32.438 INFO:teuthology.orchestra.run.smithi164.stdout: ] 2023-06-03T20:35:32.438 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:35:32.438 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:35:32.438 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:35:32.438 INFO:teuthology.orchestra.run.smithi164.stdout: "description": "osd_op(client.46739.0:177021 14.7 14:ea1c2cce:::benchmark_data_smithi089_237281_object79005:head [delete] snapc 0=[] ondisk+write+known_if_redirected e293)", 2023-06-03T20:35:32.438 INFO:teuthology.orchestra.run.smithi164.stdout: "initiated_at": "2023-06-03T20:31:26.268896+0000", 2023-06-03T20:35:32.438 INFO:teuthology.orchestra.run.smithi164.stdout: "age": 246.05645036300001, 2023-06-03T20:35:32.439 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.252183351, 2023-06-03T20:35:32.439 INFO:teuthology.orchestra.run.smithi164.stdout: "type_data": { 2023-06-03T20:35:32.439 INFO:teuthology.orchestra.run.smithi164.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-03T20:35:32.439 INFO:teuthology.orchestra.run.smithi164.stdout: "client_info": { 2023-06-03T20:35:32.439 INFO:teuthology.orchestra.run.smithi164.stdout: "client": "client.46739", 2023-06-03T20:35:32.439 INFO:teuthology.orchestra.run.smithi164.stdout: "client_addr": "172.21.15.89:0/2479974613", 2023-06-03T20:35:32.439 INFO:teuthology.orchestra.run.smithi164.stdout: "tid": 177021 2023-06-03T20:35:32.439 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:35:32.440 INFO:teuthology.orchestra.run.smithi164.stdout: "events": [ 2023-06-03T20:35:32.440 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:35:32.440 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "initiated", 2023-06-03T20:35:32.440 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:31:26.268896+0000", 2023-06-03T20:35:32.440 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0 2023-06-03T20:35:32.440 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:35:32.440 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:35:32.441 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "throttled", 2023-06-03T20:35:32.441 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:31:26.268896+0000", 2023-06-03T20:35:32.441 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0 2023-06-03T20:35:32.441 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:35:32.441 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:35:32.441 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "header_read", 2023-06-03T20:35:32.442 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:31:26.268895+0000", 2023-06-03T20:35:32.442 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 4294967295.999999 2023-06-03T20:35:32.442 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:35:32.442 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:35:32.442 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "all_read", 2023-06-03T20:35:32.442 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:31:26.268901+0000", 2023-06-03T20:35:32.442 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 5.4399999999999996e-06 2023-06-03T20:35:32.443 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:35:32.443 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:35:32.443 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "dispatched", 2023-06-03T20:35:32.443 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:31:26.268912+0000", 2023-06-03T20:35:32.443 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 1.1311000000000001e-05 2023-06-03T20:35:32.443 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:35:32.443 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:35:32.443 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "queued_for_pg", 2023-06-03T20:35:32.444 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:31:26.268919+0000", 2023-06-03T20:35:32.444 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 6.9210000000000003e-06 2023-06-03T20:35:32.444 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:35:32.444 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:35:32.444 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "reached_pg", 2023-06-03T20:35:32.444 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:31:26.268959+0000", 2023-06-03T20:35:32.444 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 4.0604e-05 2023-06-03T20:35:32.445 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:35:32.445 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:35:32.445 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "started", 2023-06-03T20:35:32.445 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:31:26.269062+0000", 2023-06-03T20:35:32.445 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.000102969 2023-06-03T20:35:32.445 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:35:32.445 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:35:32.445 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "waiting for subops from 0,5", 2023-06-03T20:35:32.446 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:31:26.269151+0000", 2023-06-03T20:35:32.446 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 8.8590000000000004e-05 2023-06-03T20:35:32.446 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:35:32.446 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:35:32.446 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:35:32.446 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:31:26.273418+0000", 2023-06-03T20:35:32.446 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.0042674790000000002 2023-06-03T20:35:32.446 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:35:32.447 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:35:32.447 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "op_commit", 2023-06-03T20:35:32.447 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:31:26.273430+0000", 2023-06-03T20:35:32.447 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 1.1681e-05 2023-06-03T20:35:32.447 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:35:32.447 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:35:32.447 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:35:32.448 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:31:26.520896+0000", 2023-06-03T20:35:32.448 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.24746584999999999 2023-06-03T20:35:32.448 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:35:32.448 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:35:32.448 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "commit_sent", 2023-06-03T20:35:32.448 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:31:26.521024+0000", 2023-06-03T20:35:32.448 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.00012844199999999999 2023-06-03T20:35:32.449 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:35:32.449 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:35:32.449 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "done", 2023-06-03T20:35:32.449 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:31:26.521080+0000", 2023-06-03T20:35:32.449 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 5.5127999999999998e-05 2023-06-03T20:35:32.449 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:35:32.449 INFO:teuthology.orchestra.run.smithi164.stdout: ] 2023-06-03T20:35:32.449 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:35:32.450 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:35:32.450 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:35:32.450 INFO:teuthology.orchestra.run.smithi164.stdout: "description": "osd_op(client.46739.0:177033 14.7 14:e9adbbeb:::benchmark_data_smithi089_237281_object79017:head [delete] snapc 0=[] ondisk+write+known_if_redirected e293)", 2023-06-03T20:35:32.450 INFO:teuthology.orchestra.run.smithi164.stdout: "initiated_at": "2023-06-03T20:31:26.286174+0000", 2023-06-03T20:35:32.450 INFO:teuthology.orchestra.run.smithi164.stdout: "age": 246.03917207500001, 2023-06-03T20:35:32.450 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.23505214799999999, 2023-06-03T20:35:32.450 INFO:teuthology.orchestra.run.smithi164.stdout: "type_data": { 2023-06-03T20:35:32.451 INFO:teuthology.orchestra.run.smithi164.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-03T20:35:32.451 INFO:teuthology.orchestra.run.smithi164.stdout: "client_info": { 2023-06-03T20:35:32.451 INFO:teuthology.orchestra.run.smithi164.stdout: "client": "client.46739", 2023-06-03T20:35:32.451 INFO:teuthology.orchestra.run.smithi164.stdout: "client_addr": "172.21.15.89:0/2479974613", 2023-06-03T20:35:32.451 INFO:teuthology.orchestra.run.smithi164.stdout: "tid": 177033 2023-06-03T20:35:32.451 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:35:32.451 INFO:teuthology.orchestra.run.smithi164.stdout: "events": [ 2023-06-03T20:35:32.452 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:35:32.452 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "initiated", 2023-06-03T20:35:32.452 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:31:26.286174+0000", 2023-06-03T20:35:32.452 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0 2023-06-03T20:35:32.452 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:35:32.452 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:35:32.452 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "throttled", 2023-06-03T20:35:32.453 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:31:26.286174+0000", 2023-06-03T20:35:32.453 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0 2023-06-03T20:35:32.453 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:35:32.453 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:35:32.453 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "header_read", 2023-06-03T20:35:32.453 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:31:26.286173+0000", 2023-06-03T20:35:32.453 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 4294967295.9999986 2023-06-03T20:35:32.454 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:35:32.454 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:35:32.454 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "all_read", 2023-06-03T20:35:32.454 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:31:26.286181+0000", 2023-06-03T20:35:32.454 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 7.5140000000000004e-06 2023-06-03T20:35:32.454 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:35:32.454 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:35:32.454 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "dispatched", 2023-06-03T20:35:32.455 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:31:26.286195+0000", 2023-06-03T20:35:32.455 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 1.4857e-05 2023-06-03T20:35:32.455 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:35:32.455 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:35:32.455 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "queued_for_pg", 2023-06-03T20:35:32.455 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:31:26.286207+0000", 2023-06-03T20:35:32.455 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 1.131e-05 2023-06-03T20:35:32.456 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:35:32.456 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:35:32.456 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "reached_pg", 2023-06-03T20:35:32.456 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:31:26.286277+0000", 2023-06-03T20:35:32.456 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 6.9726000000000003e-05 2023-06-03T20:35:32.456 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:35:32.456 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:35:32.456 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "started", 2023-06-03T20:35:32.457 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:31:26.286406+0000", 2023-06-03T20:35:32.457 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.00012905499999999999 2023-06-03T20:35:32.457 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:35:32.457 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:35:32.457 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "waiting for subops from 0,5", 2023-06-03T20:35:32.457 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:31:26.286512+0000", 2023-06-03T20:35:32.458 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.000106081 2023-06-03T20:35:32.458 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:35:32.458 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:35:32.458 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "op_commit", 2023-06-03T20:35:32.458 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:31:26.290136+0000", 2023-06-03T20:35:32.458 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.0036246080000000001 2023-06-03T20:35:32.458 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:35:32.458 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:35:32.459 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:35:32.459 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:31:26.290303+0000", 2023-06-03T20:35:32.459 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.00016672900000000001 2023-06-03T20:35:32.459 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:35:32.459 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:35:32.459 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:35:32.459 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:31:26.521133+0000", 2023-06-03T20:35:32.459 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.23083008199999999 2023-06-03T20:35:32.460 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:35:32.460 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:35:32.460 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "commit_sent", 2023-06-03T20:35:32.460 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:31:26.521185+0000", 2023-06-03T20:35:32.460 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 5.1492000000000001e-05 2023-06-03T20:35:32.460 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:35:32.460 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:35:32.461 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "done", 2023-06-03T20:35:32.461 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:31:26.521227+0000", 2023-06-03T20:35:32.461 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 4.2061000000000003e-05 2023-06-03T20:35:32.461 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:35:32.461 INFO:teuthology.orchestra.run.smithi164.stdout: ] 2023-06-03T20:35:32.461 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:35:32.461 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:35:32.461 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:35:32.462 INFO:teuthology.orchestra.run.smithi164.stdout: "description": "osd_op(client.46739.0:184386 14.7 14:ecc662f4:::benchmark_data_smithi089_237281_object86370:head [delete] snapc 0=[] ondisk+write+known_if_redirected e293)", 2023-06-03T20:35:32.462 INFO:teuthology.orchestra.run.smithi164.stdout: "initiated_at": "2023-06-03T20:31:38.755331+0000", 2023-06-03T20:35:32.462 INFO:teuthology.orchestra.run.smithi164.stdout: "age": 233.57001513899999, 2023-06-03T20:35:32.462 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.18573309399999999, 2023-06-03T20:35:32.462 INFO:teuthology.orchestra.run.smithi164.stdout: "type_data": { 2023-06-03T20:35:32.462 INFO:teuthology.orchestra.run.smithi164.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-03T20:35:32.463 INFO:teuthology.orchestra.run.smithi164.stdout: "client_info": { 2023-06-03T20:35:32.463 INFO:teuthology.orchestra.run.smithi164.stdout: "client": "client.46739", 2023-06-03T20:35:32.463 INFO:teuthology.orchestra.run.smithi164.stdout: "client_addr": "172.21.15.89:0/2479974613", 2023-06-03T20:35:32.463 INFO:teuthology.orchestra.run.smithi164.stdout: "tid": 184386 2023-06-03T20:35:32.463 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:35:32.463 INFO:teuthology.orchestra.run.smithi164.stdout: "events": [ 2023-06-03T20:35:32.463 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:35:32.464 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "initiated", 2023-06-03T20:35:32.464 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:31:38.755331+0000", 2023-06-03T20:35:32.464 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0 2023-06-03T20:35:32.464 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:35:32.464 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:35:32.464 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "throttled", 2023-06-03T20:35:32.464 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:31:38.755331+0000", 2023-06-03T20:35:32.465 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0 2023-06-03T20:35:32.465 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:35:32.465 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:35:32.465 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "header_read", 2023-06-03T20:35:32.465 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:31:38.755331+0000", 2023-06-03T20:35:32.465 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 4294967295.9999995 2023-06-03T20:35:32.465 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:35:32.465 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:35:32.466 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "all_read", 2023-06-03T20:35:32.466 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:31:38.755336+0000", 2023-06-03T20:35:32.466 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 4.8019999999999999e-06 2023-06-03T20:35:32.466 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:35:32.466 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:35:32.466 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "dispatched", 2023-06-03T20:35:32.466 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:31:38.755344+0000", 2023-06-03T20:35:32.467 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 8.7830000000000004e-06 2023-06-03T20:35:32.467 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:35:32.467 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:35:32.467 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "queued_for_pg", 2023-06-03T20:35:32.467 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:31:38.755350+0000", 2023-06-03T20:35:32.467 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 5.5659999999999997e-06 2023-06-03T20:35:32.467 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:35:32.467 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:35:32.468 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "reached_pg", 2023-06-03T20:35:32.468 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:31:38.759988+0000", 2023-06-03T20:35:32.468 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.0046380090000000002 2023-06-03T20:35:32.468 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:35:32.468 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:35:32.468 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "started", 2023-06-03T20:35:32.468 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:31:38.760097+0000", 2023-06-03T20:35:32.468 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.000108773 2023-06-03T20:35:32.469 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:35:32.469 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:35:32.469 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "waiting for subops from 0,5", 2023-06-03T20:35:32.469 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:31:38.760187+0000", 2023-06-03T20:35:32.469 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 9.0588999999999996e-05 2023-06-03T20:35:32.469 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:35:32.469 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:35:32.469 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:35:32.470 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:31:38.764296+0000", 2023-06-03T20:35:32.470 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.0041090780000000004 2023-06-03T20:35:32.470 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:35:32.470 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:35:32.470 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "op_commit", 2023-06-03T20:35:32.470 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:31:38.767193+0000", 2023-06-03T20:35:32.470 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.0028962620000000001 2023-06-03T20:35:32.471 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:35:32.471 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:35:32.471 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:35:32.471 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:31:38.940996+0000", 2023-06-03T20:35:32.471 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.17380322400000001 2023-06-03T20:35:32.471 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:35:32.471 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:35:32.471 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "commit_sent", 2023-06-03T20:35:32.472 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:31:38.941033+0000", 2023-06-03T20:35:32.472 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 3.7358999999999999e-05 2023-06-03T20:35:32.472 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:35:32.472 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:35:32.472 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "done", 2023-06-03T20:35:32.472 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:31:38.941065+0000", 2023-06-03T20:35:32.473 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 3.1319000000000003e-05 2023-06-03T20:35:32.473 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:35:32.473 INFO:teuthology.orchestra.run.smithi164.stdout: ] 2023-06-03T20:35:32.473 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:35:32.473 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:35:32.473 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:35:32.473 INFO:teuthology.orchestra.run.smithi164.stdout: "description": "osd_op(client.46739.0:184396 14.7 14:e8ffd6a7:::benchmark_data_smithi089_237281_object86380:head [delete] snapc 0=[] ondisk+write+known_if_redirected e293)", 2023-06-03T20:35:32.474 INFO:teuthology.orchestra.run.smithi164.stdout: "initiated_at": "2023-06-03T20:31:38.768140+0000", 2023-06-03T20:35:32.474 INFO:teuthology.orchestra.run.smithi164.stdout: "age": 233.557206476, 2023-06-03T20:35:32.474 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.17312044700000001, 2023-06-03T20:35:32.474 INFO:teuthology.orchestra.run.smithi164.stdout: "type_data": { 2023-06-03T20:35:32.474 INFO:teuthology.orchestra.run.smithi164.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-03T20:35:32.474 INFO:teuthology.orchestra.run.smithi164.stdout: "client_info": { 2023-06-03T20:35:32.474 INFO:teuthology.orchestra.run.smithi164.stdout: "client": "client.46739", 2023-06-03T20:35:32.474 INFO:teuthology.orchestra.run.smithi164.stdout: "client_addr": "172.21.15.89:0/2479974613", 2023-06-03T20:35:32.475 INFO:teuthology.orchestra.run.smithi164.stdout: "tid": 184396 2023-06-03T20:35:32.475 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:35:32.475 INFO:teuthology.orchestra.run.smithi164.stdout: "events": [ 2023-06-03T20:35:32.475 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:35:32.475 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "initiated", 2023-06-03T20:35:32.475 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:31:38.768140+0000", 2023-06-03T20:35:32.475 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0 2023-06-03T20:35:32.476 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:35:32.476 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:35:32.476 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "throttled", 2023-06-03T20:35:32.476 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:31:38.768140+0000", 2023-06-03T20:35:32.476 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0 2023-06-03T20:35:32.476 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:35:32.476 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:35:32.477 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "header_read", 2023-06-03T20:35:32.477 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:31:38.768138+0000", 2023-06-03T20:35:32.477 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 4294967295.9999981 2023-06-03T20:35:32.477 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:35:32.477 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:35:32.477 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "all_read", 2023-06-03T20:35:32.477 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:31:38.768145+0000", 2023-06-03T20:35:32.478 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 7.1899999999999998e-06 2023-06-03T20:35:32.478 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:35:32.478 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:35:32.478 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "dispatched", 2023-06-03T20:35:32.478 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:31:38.768157+0000", 2023-06-03T20:35:32.478 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 1.1809e-05 2023-06-03T20:35:32.478 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:35:32.478 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:35:32.479 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "queued_for_pg", 2023-06-03T20:35:32.479 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:31:38.768165+0000", 2023-06-03T20:35:32.479 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 7.537e-06 2023-06-03T20:35:32.479 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:35:32.479 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:35:32.479 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "reached_pg", 2023-06-03T20:35:32.479 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:31:38.768213+0000", 2023-06-03T20:35:32.479 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 4.8306999999999997e-05 2023-06-03T20:35:32.480 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:35:32.480 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:35:32.480 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "started", 2023-06-03T20:35:32.480 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:31:38.768321+0000", 2023-06-03T20:35:32.480 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.000108293 2023-06-03T20:35:32.480 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:35:32.480 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:35:32.480 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "waiting for subops from 0,5", 2023-06-03T20:35:32.481 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:31:38.768409+0000", 2023-06-03T20:35:32.481 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 8.7635999999999999e-05 2023-06-03T20:35:32.481 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:35:32.481 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:35:32.481 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "op_commit", 2023-06-03T20:35:32.481 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:31:38.772155+0000", 2023-06-03T20:35:32.481 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.003745777 2023-06-03T20:35:32.482 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:35:32.482 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:35:32.482 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:35:32.482 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:31:38.772727+0000", 2023-06-03T20:35:32.482 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.00057210599999999996 2023-06-03T20:35:32.482 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:35:32.482 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:35:32.483 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:35:32.483 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:31:38.941134+0000", 2023-06-03T20:35:32.483 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.16840755900000001 2023-06-03T20:35:32.483 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:35:32.483 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:35:32.483 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "commit_sent", 2023-06-03T20:35:32.483 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:31:38.941206+0000", 2023-06-03T20:35:32.484 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 7.1684e-05 2023-06-03T20:35:32.484 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:35:32.484 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:35:32.484 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "done", 2023-06-03T20:35:32.484 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:31:38.941261+0000", 2023-06-03T20:35:32.484 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 5.4373999999999998e-05 2023-06-03T20:35:32.484 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:35:32.485 INFO:teuthology.orchestra.run.smithi164.stdout: ] 2023-06-03T20:35:32.485 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:35:32.485 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:35:32.485 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:35:32.485 INFO:teuthology.orchestra.run.smithi164.stdout: "description": "osd_op(client.46739.0:184398 14.7 14:edf3a630:::benchmark_data_smithi089_237281_object86382:head [delete] snapc 0=[] ondisk+write+known_if_redirected e293)", 2023-06-03T20:35:32.485 INFO:teuthology.orchestra.run.smithi164.stdout: "initiated_at": "2023-06-03T20:31:38.776341+0000", 2023-06-03T20:35:32.485 INFO:teuthology.orchestra.run.smithi164.stdout: "age": 233.54900537, 2023-06-03T20:35:32.485 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.17104580799999999, 2023-06-03T20:35:32.486 INFO:teuthology.orchestra.run.smithi164.stdout: "type_data": { 2023-06-03T20:35:32.486 INFO:teuthology.orchestra.run.smithi164.stdout: "flag_point": "commit sent; apply or cleanup", 2023-06-03T20:35:32.486 INFO:teuthology.orchestra.run.smithi164.stdout: "client_info": { 2023-06-03T20:35:32.486 INFO:teuthology.orchestra.run.smithi164.stdout: "client": "client.46739", 2023-06-03T20:35:32.486 INFO:teuthology.orchestra.run.smithi164.stdout: "client_addr": "172.21.15.89:0/2479974613", 2023-06-03T20:35:32.486 INFO:teuthology.orchestra.run.smithi164.stdout: "tid": 184398 2023-06-03T20:35:32.486 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:35:32.486 INFO:teuthology.orchestra.run.smithi164.stdout: "events": [ 2023-06-03T20:35:32.487 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:35:32.487 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "initiated", 2023-06-03T20:35:32.487 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:31:38.776341+0000", 2023-06-03T20:35:32.487 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0 2023-06-03T20:35:32.487 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:35:32.487 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:35:32.487 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "throttled", 2023-06-03T20:35:32.487 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:31:38.776341+0000", 2023-06-03T20:35:32.488 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0 2023-06-03T20:35:32.488 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:35:32.488 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:35:32.488 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "header_read", 2023-06-03T20:35:32.488 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:31:38.776340+0000", 2023-06-03T20:35:32.488 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 4294967295.9999986 2023-06-03T20:35:32.489 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:35:32.489 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:35:32.489 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "all_read", 2023-06-03T20:35:32.489 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:31:38.776346+0000", 2023-06-03T20:35:32.489 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 6.9160000000000004e-06 2023-06-03T20:35:32.489 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:35:32.489 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:35:32.489 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "dispatched", 2023-06-03T20:35:32.490 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:31:38.776362+0000", 2023-06-03T20:35:32.490 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 1.5896000000000002e-05 2023-06-03T20:35:32.490 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:35:32.490 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:35:32.490 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "queued_for_pg", 2023-06-03T20:35:32.490 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:31:38.776372+0000", 2023-06-03T20:35:32.490 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 9.4390000000000006e-06 2023-06-03T20:35:32.490 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:35:32.491 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:35:32.491 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "reached_pg", 2023-06-03T20:35:32.491 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:31:38.776419+0000", 2023-06-03T20:35:32.491 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 4.685e-05 2023-06-03T20:35:32.491 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:35:32.491 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:35:32.491 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "started", 2023-06-03T20:35:32.492 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:31:38.776532+0000", 2023-06-03T20:35:32.492 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.000113532 2023-06-03T20:35:32.492 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:35:32.492 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:35:32.492 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "waiting for subops from 0,5", 2023-06-03T20:35:32.493 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:31:38.776637+0000", 2023-06-03T20:35:32.493 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.000104635 2023-06-03T20:35:32.493 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:35:32.493 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:35:32.493 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "op_commit", 2023-06-03T20:35:32.493 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:31:38.785009+0000", 2023-06-03T20:35:32.494 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.0083723719999999995 2023-06-03T20:35:32.494 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:35:32.494 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:35:32.494 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:35:32.494 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:31:38.785070+0000", 2023-06-03T20:35:32.494 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 6.0408000000000002e-05 2023-06-03T20:35:32.494 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:35:32.495 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:35:32.495 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "sub_op_commit_rec", 2023-06-03T20:35:32.495 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:31:38.947256+0000", 2023-06-03T20:35:32.495 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 0.162186576 2023-06-03T20:35:32.495 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:35:32.495 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:35:32.495 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "commit_sent", 2023-06-03T20:35:32.495 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:31:38.947326+0000", 2023-06-03T20:35:32.496 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 6.9665999999999993e-05 2023-06-03T20:35:32.496 INFO:teuthology.orchestra.run.smithi164.stdout: }, 2023-06-03T20:35:32.496 INFO:teuthology.orchestra.run.smithi164.stdout: { 2023-06-03T20:35:32.496 INFO:teuthology.orchestra.run.smithi164.stdout: "event": "done", 2023-06-03T20:35:32.496 INFO:teuthology.orchestra.run.smithi164.stdout: "time": "2023-06-03T20:31:38.947387+0000", 2023-06-03T20:35:32.496 INFO:teuthology.orchestra.run.smithi164.stdout: "duration": 6.1158e-05 2023-06-03T20:35:32.496 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:35:32.497 INFO:teuthology.orchestra.run.smithi164.stdout: ] 2023-06-03T20:35:32.497 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:35:32.497 INFO:teuthology.orchestra.run.smithi164.stdout: } 2023-06-03T20:35:32.497 INFO:teuthology.orchestra.run.smithi164.stdout: ] 2023-06-03T20:35:32.497 INFO:teuthology.orchestra.run.smithi164.stdout:} 2023-06-03T20:35:32.722 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:35:32 smithi089 bash[75068]: cluster 1685824531.2940876 mgr.y (mgr.34104) 1795 : cluster 0 pgmap v1873: 73 pgs: 73 active+clean; 582 KiB data, 5.1 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:35:32.723 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:35:32 smithi089 bash[78452]: cluster 1685824531.2940876 mgr.y (mgr.34104) 1795 : cluster 0 pgmap v1873: 73 pgs: 73 active+clean; 582 KiB data, 5.1 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:35:32.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:35:32 smithi164 bash[59462]: cluster 1685824531.2940876 mgr.y (mgr.34104) 1795 : cluster 0 pgmap v1873: 73 pgs: 73 active+clean; 582 KiB data, 5.1 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:35:34.529 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:35:34 smithi089 bash[75068]: cluster 1685824533.2949238 mgr.y (mgr.34104) 1796 : cluster 0 pgmap v1874: 73 pgs: 73 active+clean; 582 KiB data, 5.1 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:35:34.529 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:35:34 smithi089 bash[78452]: cluster 1685824533.2949238 mgr.y (mgr.34104) 1796 : cluster 0 pgmap v1874: 73 pgs: 73 active+clean; 582 KiB data, 5.1 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:35:34.585 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:35:34 smithi164 bash[59462]: cluster 1685824533.2949238 mgr.y (mgr.34104) 1796 : cluster 0 pgmap v1874: 73 pgs: 73 active+clean; 582 KiB data, 5.1 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:35:35.196 INFO:teuthology.orchestra.run.smithi089.stderr:reweighted osd.6 to 1 (10000) 2023-06-03T20:35:35.529 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:35:35 smithi089 bash[75068]: audit 1685824535.1325939 mon.a (mon.0) 1741 : audit 1 from='client.? 172.21.15.89:0/880945653' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-06-03T20:35:35.529 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:35:35 smithi089 bash[78452]: audit 1685824535.1325939 mon.a (mon.0) 1741 : audit 1 from='client.? 172.21.15.89:0/880945653' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-06-03T20:35:35.585 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:35:35 smithi164 bash[59462]: audit 1685824535.1325939 mon.a (mon.0) 1741 : audit 1 from='client.? 172.21.15.89:0/880945653' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-06-03T20:35:36.146 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph osd primary-affinity 6 1 2023-06-03T20:35:36.529 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 20:35:36 smithi089 bash[71163]: ::ffff:172.21.15.164 - - [03/Jun/2023:20:35:36] "GET /metrics HTTP/1.1" 200 226935 "" "Prometheus/2.33.4" 2023-06-03T20:35:36.529 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:35:36 smithi089 bash[78452]: audit 1685824535.1865017 mon.a (mon.0) 1742 : audit 1 from='client.? 172.21.15.89:0/880945653' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2023-06-03T20:35:36.530 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:35:36 smithi089 bash[78452]: cluster 1685824535.186582 mon.a (mon.0) 1743 : cluster 0 osdmap e319: 8 total, 8 up, 8 in 2023-06-03T20:35:36.530 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:35:36 smithi089 bash[78452]: cluster 1685824535.295555 mgr.y (mgr.34104) 1797 : cluster 0 pgmap v1876: 73 pgs: 73 active+clean; 582 KiB data, 5.1 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:35:36.530 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:35:36 smithi089 bash[78452]: cluster 1685824535.4378688 osd.7 (osd.7) 261 : cluster 3 failed to encode map e319 with expected crc 2023-06-03T20:35:36.530 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:35:36 smithi089 bash[78452]: cluster 1685824535.594988 osd.6 (osd.6) 255 : cluster 3 failed to encode map e319 with expected crc 2023-06-03T20:35:36.530 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:35:36 smithi089 bash[78452]: cluster 1685824535.5955007 osd.5 (osd.5) 268 : cluster 3 failed to encode map e319 with expected crc 2023-06-03T20:35:36.530 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:35:36 smithi089 bash[75068]: audit 1685824535.1865017 mon.a (mon.0) 1742 : audit 1 from='client.? 172.21.15.89:0/880945653' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2023-06-03T20:35:36.531 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:35:36 smithi089 bash[75068]: cluster 1685824535.186582 mon.a (mon.0) 1743 : cluster 0 osdmap e319: 8 total, 8 up, 8 in 2023-06-03T20:35:36.531 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:35:36 smithi089 bash[75068]: cluster 1685824535.295555 mgr.y (mgr.34104) 1797 : cluster 0 pgmap v1876: 73 pgs: 73 active+clean; 582 KiB data, 5.1 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:35:36.531 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:35:36 smithi089 bash[75068]: cluster 1685824535.4378688 osd.7 (osd.7) 261 : cluster 3 failed to encode map e319 with expected crc 2023-06-03T20:35:36.531 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:35:36 smithi089 bash[75068]: cluster 1685824535.594988 osd.6 (osd.6) 255 : cluster 3 failed to encode map e319 with expected crc 2023-06-03T20:35:36.531 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:35:36 smithi089 bash[75068]: cluster 1685824535.5955007 osd.5 (osd.5) 268 : cluster 3 failed to encode map e319 with expected crc 2023-06-03T20:35:36.585 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:35:36 smithi164 bash[59462]: audit 1685824535.1865017 mon.a (mon.0) 1742 : audit 1 from='client.? 172.21.15.89:0/880945653' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2023-06-03T20:35:36.586 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:35:36 smithi164 bash[59462]: cluster 1685824535.186582 mon.a (mon.0) 1743 : cluster 0 osdmap e319: 8 total, 8 up, 8 in 2023-06-03T20:35:36.586 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:35:36 smithi164 bash[59462]: cluster 1685824535.295555 mgr.y (mgr.34104) 1797 : cluster 0 pgmap v1876: 73 pgs: 73 active+clean; 582 KiB data, 5.1 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:35:36.586 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:35:36 smithi164 bash[59462]: cluster 1685824535.4378688 osd.7 (osd.7) 261 : cluster 3 failed to encode map e319 with expected crc 2023-06-03T20:35:36.586 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:35:36 smithi164 bash[59462]: cluster 1685824535.594988 osd.6 (osd.6) 255 : cluster 3 failed to encode map e319 with expected crc 2023-06-03T20:35:36.586 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:35:36 smithi164 bash[59462]: cluster 1685824535.5955007 osd.5 (osd.5) 268 : cluster 3 failed to encode map e319 with expected crc 2023-06-03T20:35:37.529 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:35:37 smithi089 bash[75068]: audit 1685824537.167212 mon.c (mon.2) 714 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T20:35:37.529 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:35:37 smithi089 bash[78452]: audit 1685824537.167212 mon.c (mon.2) 714 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T20:35:37.585 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:35:37 smithi164 bash[59462]: audit 1685824537.167212 mon.c (mon.2) 714 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T20:35:38.529 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:35:38 smithi089 bash[78452]: cluster 1685824537.2963285 mgr.y (mgr.34104) 1798 : cluster 0 pgmap v1877: 73 pgs: 73 active+clean; 582 KiB data, 5.1 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:35:38.529 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:35:38 smithi089 bash[75068]: cluster 1685824537.2963285 mgr.y (mgr.34104) 1798 : cluster 0 pgmap v1877: 73 pgs: 73 active+clean; 582 KiB data, 5.1 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:35:38.585 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:35:38 smithi164 bash[59462]: cluster 1685824537.2963285 mgr.y (mgr.34104) 1798 : cluster 0 pgmap v1877: 73 pgs: 73 active+clean; 582 KiB data, 5.1 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:35:40.779 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:35:40 smithi089 bash[78452]: cluster 1685824539.2973943 mgr.y (mgr.34104) 1799 : cluster 0 pgmap v1878: 73 pgs: 73 active+clean; 582 KiB data, 5.1 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:35:40.779 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:35:40 smithi089 bash[75068]: cluster 1685824539.2973943 mgr.y (mgr.34104) 1799 : cluster 0 pgmap v1878: 73 pgs: 73 active+clean; 582 KiB data, 5.1 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:35:40.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:35:40 smithi164 bash[59462]: cluster 1685824539.2973943 mgr.y (mgr.34104) 1799 : cluster 0 pgmap v1878: 73 pgs: 73 active+clean; 582 KiB data, 5.1 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:35:40.968 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/mon.a/config 2023-06-03T20:35:41.782 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:35:41 smithi089 bash[75068]: audit 1685824540.7000265 mon.c (mon.2) 715 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-03T20:35:41.782 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:35:41 smithi089 bash[78452]: audit 1685824540.7000265 mon.c (mon.2) 715 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-03T20:35:41.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:35:41 smithi164 bash[59462]: audit 1685824540.7000265 mon.c (mon.2) 715 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-03T20:35:42.718 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:35:42 smithi089 bash[75068]: cluster 1685824541.2981584 mgr.y (mgr.34104) 1800 : cluster 0 pgmap v1879: 73 pgs: 73 active+clean; 582 KiB data, 5.1 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:35:42.719 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:35:42 smithi089 bash[78452]: cluster 1685824541.2981584 mgr.y (mgr.34104) 1800 : cluster 0 pgmap v1879: 73 pgs: 73 active+clean; 582 KiB data, 5.1 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:35:42.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:35:42 smithi164 bash[59462]: cluster 1685824541.2981584 mgr.y (mgr.34104) 1800 : cluster 0 pgmap v1879: 73 pgs: 73 active+clean; 582 KiB data, 5.1 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:35:44.529 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:35:44 smithi089 bash[78452]: cluster 1685824543.2991445 mgr.y (mgr.34104) 1801 : cluster 0 pgmap v1880: 73 pgs: 73 active+clean; 582 KiB data, 5.1 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:35:44.530 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:35:44 smithi089 bash[75068]: cluster 1685824543.2991445 mgr.y (mgr.34104) 1801 : cluster 0 pgmap v1880: 73 pgs: 73 active+clean; 582 KiB data, 5.1 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:35:44.585 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:35:44 smithi164 bash[59462]: cluster 1685824543.2991445 mgr.y (mgr.34104) 1801 : cluster 0 pgmap v1880: 73 pgs: 73 active+clean; 582 KiB data, 5.1 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:35:45.207 INFO:teuthology.orchestra.run.smithi089.stderr:set osd.6 primary-affinity to 1 (8655362) 2023-06-03T20:35:45.529 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:35:45 smithi089 bash[75068]: audit 1685824545.1583667 mon.a (mon.0) 1744 : audit 1 from='client.? 172.21.15.89:0/1025681105' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 6, "weight": 1.0}]: dispatch 2023-06-03T20:35:45.529 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:35:45 smithi089 bash[78452]: audit 1685824545.1583667 mon.a (mon.0) 1744 : audit 1 from='client.? 172.21.15.89:0/1025681105' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 6, "weight": 1.0}]: dispatch 2023-06-03T20:35:45.585 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:35:45 smithi164 bash[59462]: audit 1685824545.1583667 mon.a (mon.0) 1744 : audit 1 from='client.? 172.21.15.89:0/1025681105' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 6, "weight": 1.0}]: dispatch 2023-06-03T20:35:46.166 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph osd reweight 7 1 2023-06-03T20:35:46.529 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 20:35:46 smithi089 bash[71163]: ::ffff:172.21.15.164 - - [03/Jun/2023:20:35:46] "GET /metrics HTTP/1.1" 200 226927 "" "Prometheus/2.33.4" 2023-06-03T20:35:46.529 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:35:46 smithi089 bash[78452]: cluster 1685824545.191156 osd.5 (osd.5) 269 : cluster 3 failed to encode map e320 with expected crc 2023-06-03T20:35:46.529 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:35:46 smithi089 bash[78452]: audit 1685824545.1972754 mon.a (mon.0) 1745 : audit 1 from='client.? 172.21.15.89:0/1025681105' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 6, "weight": 1.0}]': finished 2023-06-03T20:35:46.529 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:35:46 smithi089 bash[78452]: cluster 1685824545.1974134 mon.a (mon.0) 1746 : cluster 0 osdmap e320: 8 total, 8 up, 8 in 2023-06-03T20:35:46.530 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:35:46 smithi089 bash[78452]: cluster 1685824545.2997983 mgr.y (mgr.34104) 1802 : cluster 0 pgmap v1882: 73 pgs: 73 active+clean; 582 KiB data, 5.1 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:35:46.530 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:35:46 smithi089 bash[78452]: cluster 1685824545.4075391 osd.7 (osd.7) 262 : cluster 3 failed to encode map e320 with expected crc 2023-06-03T20:35:46.530 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:35:46 smithi089 bash[78452]: cluster 1685824545.6763473 osd.6 (osd.6) 256 : cluster 3 failed to encode map e320 with expected crc 2023-06-03T20:35:46.530 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:35:46 smithi089 bash[75068]: cluster 1685824545.191156 osd.5 (osd.5) 269 : cluster 3 failed to encode map e320 with expected crc 2023-06-03T20:35:46.530 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:35:46 smithi089 bash[75068]: audit 1685824545.1972754 mon.a (mon.0) 1745 : audit 1 from='client.? 172.21.15.89:0/1025681105' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 6, "weight": 1.0}]': finished 2023-06-03T20:35:46.530 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:35:46 smithi089 bash[75068]: cluster 1685824545.1974134 mon.a (mon.0) 1746 : cluster 0 osdmap e320: 8 total, 8 up, 8 in 2023-06-03T20:35:46.531 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:35:46 smithi089 bash[75068]: cluster 1685824545.2997983 mgr.y (mgr.34104) 1802 : cluster 0 pgmap v1882: 73 pgs: 73 active+clean; 582 KiB data, 5.1 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:35:46.531 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:35:46 smithi089 bash[75068]: cluster 1685824545.4075391 osd.7 (osd.7) 262 : cluster 3 failed to encode map e320 with expected crc 2023-06-03T20:35:46.531 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:35:46 smithi089 bash[75068]: cluster 1685824545.6763473 osd.6 (osd.6) 256 : cluster 3 failed to encode map e320 with expected crc 2023-06-03T20:35:46.585 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:35:46 smithi164 bash[59462]: cluster 1685824545.191156 osd.5 (osd.5) 269 : cluster 3 failed to encode map e320 with expected crc 2023-06-03T20:35:46.585 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:35:46 smithi164 bash[59462]: audit 1685824545.1972754 mon.a (mon.0) 1745 : audit 1 from='client.? 172.21.15.89:0/1025681105' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 6, "weight": 1.0}]': finished 2023-06-03T20:35:46.586 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:35:46 smithi164 bash[59462]: cluster 1685824545.1974134 mon.a (mon.0) 1746 : cluster 0 osdmap e320: 8 total, 8 up, 8 in 2023-06-03T20:35:46.586 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:35:46 smithi164 bash[59462]: cluster 1685824545.2997983 mgr.y (mgr.34104) 1802 : cluster 0 pgmap v1882: 73 pgs: 73 active+clean; 582 KiB data, 5.1 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:35:46.586 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:35:46 smithi164 bash[59462]: cluster 1685824545.4075391 osd.7 (osd.7) 262 : cluster 3 failed to encode map e320 with expected crc 2023-06-03T20:35:46.586 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:35:46 smithi164 bash[59462]: cluster 1685824545.6763473 osd.6 (osd.6) 256 : cluster 3 failed to encode map e320 with expected crc 2023-06-03T20:35:47.529 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:35:47 smithi089 bash[78452]: audit 1685824546.2069256 mon.a (mon.0) 1747 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:35:47.529 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:35:47 smithi089 bash[78452]: audit 1685824546.216813 mon.a (mon.0) 1748 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:35:47.529 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:35:47 smithi089 bash[78452]: audit 1685824547.1712067 mon.a (mon.0) 1749 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:35:47.530 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:35:47 smithi089 bash[78452]: audit 1685824547.1808665 mon.a (mon.0) 1750 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:35:47.530 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:35:47 smithi089 bash[75068]: audit 1685824546.2069256 mon.a (mon.0) 1747 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:35:47.530 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:35:47 smithi089 bash[75068]: audit 1685824546.216813 mon.a (mon.0) 1748 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:35:47.530 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:35:47 smithi089 bash[75068]: audit 1685824547.1712067 mon.a (mon.0) 1749 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:35:47.530 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:35:47 smithi089 bash[75068]: audit 1685824547.1808665 mon.a (mon.0) 1750 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:35:47.585 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:35:47 smithi164 bash[59462]: audit 1685824546.2069256 mon.a (mon.0) 1747 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:35:47.586 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:35:47 smithi164 bash[59462]: audit 1685824546.216813 mon.a (mon.0) 1748 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:35:47.586 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:35:47 smithi164 bash[59462]: audit 1685824547.1712067 mon.a (mon.0) 1749 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:35:47.586 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:35:47 smithi164 bash[59462]: audit 1685824547.1808665 mon.a (mon.0) 1750 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:35:48.530 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:35:48 smithi089 bash[78452]: cluster 1685824547.3005528 mgr.y (mgr.34104) 1803 : cluster 0 pgmap v1883: 73 pgs: 73 active+clean; 582 KiB data, 5.1 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:35:48.530 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:35:48 smithi089 bash[78452]: audit 1685824547.6595 mon.c (mon.2) 716 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-03T20:35:48.531 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:35:48 smithi089 bash[78452]: audit 1685824547.6615021 mon.c (mon.2) 717 : audit 1 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-03T20:35:48.531 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:35:48 smithi089 bash[78452]: audit 1685824547.6736813 mon.a (mon.0) 1751 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:35:48.531 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:35:48 smithi089 bash[75068]: cluster 1685824547.3005528 mgr.y (mgr.34104) 1803 : cluster 0 pgmap v1883: 73 pgs: 73 active+clean; 582 KiB data, 5.1 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:35:48.532 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:35:48 smithi089 bash[75068]: audit 1685824547.6595 mon.c (mon.2) 716 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-03T20:35:48.532 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:35:48 smithi089 bash[75068]: audit 1685824547.6615021 mon.c (mon.2) 717 : audit 1 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-03T20:35:48.532 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:35:48 smithi089 bash[75068]: audit 1685824547.6736813 mon.a (mon.0) 1751 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:35:48.585 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:35:48 smithi164 bash[59462]: cluster 1685824547.3005528 mgr.y (mgr.34104) 1803 : cluster 0 pgmap v1883: 73 pgs: 73 active+clean; 582 KiB data, 5.1 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:35:48.586 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:35:48 smithi164 bash[59462]: audit 1685824547.6595 mon.c (mon.2) 716 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-03T20:35:48.586 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:35:48 smithi164 bash[59462]: audit 1685824547.6615021 mon.c (mon.2) 717 : audit 1 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-03T20:35:48.586 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:35:48 smithi164 bash[59462]: audit 1685824547.6736813 mon.a (mon.0) 1751 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:35:50.779 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:35:50 smithi089 bash[78452]: cluster 1685824549.3015835 mgr.y (mgr.34104) 1804 : cluster 0 pgmap v1884: 73 pgs: 73 active+clean; 582 KiB data, 5.1 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:35:50.779 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:35:50 smithi089 bash[75068]: cluster 1685824549.3015835 mgr.y (mgr.34104) 1804 : cluster 0 pgmap v1884: 73 pgs: 73 active+clean; 582 KiB data, 5.1 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:35:50.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:35:50 smithi164 bash[59462]: cluster 1685824549.3015835 mgr.y (mgr.34104) 1804 : cluster 0 pgmap v1884: 73 pgs: 73 active+clean; 582 KiB data, 5.1 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:35:50.986 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/mon.a/config 2023-06-03T20:35:52.628 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:35:52 smithi089 bash[78452]: cluster 1685824551.3023934 mgr.y (mgr.34104) 1805 : cluster 0 pgmap v1885: 73 pgs: 73 active+clean; 582 KiB data, 5.1 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:35:52.628 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:35:52 smithi089 bash[78452]: audit 1685824552.1666136 mon.c (mon.2) 718 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T20:35:52.628 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:35:52 smithi089 bash[75068]: cluster 1685824551.3023934 mgr.y (mgr.34104) 1805 : cluster 0 pgmap v1885: 73 pgs: 73 active+clean; 582 KiB data, 5.1 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:35:52.628 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:35:52 smithi089 bash[75068]: audit 1685824552.1666136 mon.c (mon.2) 718 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T20:35:52.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:35:52 smithi164 bash[59462]: cluster 1685824551.3023934 mgr.y (mgr.34104) 1805 : cluster 0 pgmap v1885: 73 pgs: 73 active+clean; 582 KiB data, 5.1 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:35:52.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:35:52 smithi164 bash[59462]: audit 1685824552.1666136 mon.c (mon.2) 718 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T20:35:54.529 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:35:54 smithi089 bash[78452]: cluster 1685824553.3034158 mgr.y (mgr.34104) 1806 : cluster 0 pgmap v1886: 73 pgs: 73 active+clean; 582 KiB data, 5.1 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:35:54.529 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:35:54 smithi089 bash[75068]: cluster 1685824553.3034158 mgr.y (mgr.34104) 1806 : cluster 0 pgmap v1886: 73 pgs: 73 active+clean; 582 KiB data, 5.1 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:35:54.585 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:35:54 smithi164 bash[59462]: cluster 1685824553.3034158 mgr.y (mgr.34104) 1806 : cluster 0 pgmap v1886: 73 pgs: 73 active+clean; 582 KiB data, 5.1 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:35:55.205 INFO:teuthology.orchestra.run.smithi089.stderr:reweighted osd.7 to 1 (10000) 2023-06-03T20:35:55.529 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:35:55 smithi089 bash[75068]: audit 1685824555.0083067 mon.c (mon.2) 719 : audit 1 from='client.? 172.21.15.89:0/1939817511' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-06-03T20:35:55.529 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:35:55 smithi089 bash[75068]: audit 1685824555.0089068 mon.a (mon.0) 1752 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-06-03T20:35:55.529 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:35:55 smithi089 bash[78452]: audit 1685824555.0083067 mon.c (mon.2) 719 : audit 1 from='client.? 172.21.15.89:0/1939817511' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-06-03T20:35:55.530 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:35:55 smithi089 bash[78452]: audit 1685824555.0089068 mon.a (mon.0) 1752 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-06-03T20:35:55.585 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:35:55 smithi164 bash[59462]: audit 1685824555.0083067 mon.c (mon.2) 719 : audit 1 from='client.? 172.21.15.89:0/1939817511' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-06-03T20:35:55.586 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:35:55 smithi164 bash[59462]: audit 1685824555.0089068 mon.a (mon.0) 1752 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-06-03T20:35:56.170 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph osd primary-affinity 7 1 2023-06-03T20:35:56.529 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 20:35:56 smithi089 bash[71163]: ::ffff:172.21.15.164 - - [03/Jun/2023:20:35:56] "GET /metrics HTTP/1.1" 200 226927 "" "Prometheus/2.33.4" 2023-06-03T20:35:56.530 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:35:56 smithi089 bash[78452]: cluster 1685824555.1900704 osd.7 (osd.7) 263 : cluster 3 failed to encode map e321 with expected crc 2023-06-03T20:35:56.530 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:35:56 smithi089 bash[78452]: audit 1685824555.195702 mon.a (mon.0) 1753 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2023-06-03T20:35:56.530 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:35:56 smithi089 bash[78452]: cluster 1685824555.1957998 mon.a (mon.0) 1754 : cluster 0 osdmap e321: 8 total, 8 up, 8 in 2023-06-03T20:35:56.530 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:35:56 smithi089 bash[78452]: cluster 1685824555.3041248 mgr.y (mgr.34104) 1807 : cluster 0 pgmap v1888: 73 pgs: 73 active+clean; 582 KiB data, 5.1 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:35:56.530 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:35:56 smithi089 bash[78452]: cluster 1685824555.3354523 osd.5 (osd.5) 270 : cluster 3 failed to encode map e321 with expected crc 2023-06-03T20:35:56.530 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:35:56 smithi089 bash[78452]: cluster 1685824555.597391 osd.6 (osd.6) 257 : cluster 3 failed to encode map e321 with expected crc 2023-06-03T20:35:56.531 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:35:56 smithi089 bash[75068]: cluster 1685824555.1900704 osd.7 (osd.7) 263 : cluster 3 failed to encode map e321 with expected crc 2023-06-03T20:35:56.531 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:35:56 smithi089 bash[75068]: audit 1685824555.195702 mon.a (mon.0) 1753 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2023-06-03T20:35:56.531 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:35:56 smithi089 bash[75068]: cluster 1685824555.1957998 mon.a (mon.0) 1754 : cluster 0 osdmap e321: 8 total, 8 up, 8 in 2023-06-03T20:35:56.531 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:35:56 smithi089 bash[75068]: cluster 1685824555.3041248 mgr.y (mgr.34104) 1807 : cluster 0 pgmap v1888: 73 pgs: 73 active+clean; 582 KiB data, 5.1 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:35:56.531 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:35:56 smithi089 bash[75068]: cluster 1685824555.3354523 osd.5 (osd.5) 270 : cluster 3 failed to encode map e321 with expected crc 2023-06-03T20:35:56.531 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:35:56 smithi089 bash[75068]: cluster 1685824555.597391 osd.6 (osd.6) 257 : cluster 3 failed to encode map e321 with expected crc 2023-06-03T20:35:56.585 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:35:56 smithi164 bash[59462]: cluster 1685824555.1900704 osd.7 (osd.7) 263 : cluster 3 failed to encode map e321 with expected crc 2023-06-03T20:35:56.586 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:35:56 smithi164 bash[59462]: audit 1685824555.195702 mon.a (mon.0) 1753 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2023-06-03T20:35:56.586 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:35:56 smithi164 bash[59462]: cluster 1685824555.1957998 mon.a (mon.0) 1754 : cluster 0 osdmap e321: 8 total, 8 up, 8 in 2023-06-03T20:35:56.586 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:35:56 smithi164 bash[59462]: cluster 1685824555.3041248 mgr.y (mgr.34104) 1807 : cluster 0 pgmap v1888: 73 pgs: 73 active+clean; 582 KiB data, 5.1 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:35:56.587 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:35:56 smithi164 bash[59462]: cluster 1685824555.3354523 osd.5 (osd.5) 270 : cluster 3 failed to encode map e321 with expected crc 2023-06-03T20:35:56.587 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:35:56 smithi164 bash[59462]: cluster 1685824555.597391 osd.6 (osd.6) 257 : cluster 3 failed to encode map e321 with expected crc 2023-06-03T20:35:58.779 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:35:58 smithi089 bash[78452]: cluster 1685824557.3050017 mgr.y (mgr.34104) 1808 : cluster 0 pgmap v1889: 73 pgs: 73 active+clean; 582 KiB data, 5.1 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:35:58.779 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:35:58 smithi089 bash[75068]: cluster 1685824557.3050017 mgr.y (mgr.34104) 1808 : cluster 0 pgmap v1889: 73 pgs: 73 active+clean; 582 KiB data, 5.1 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:35:58.836 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:35:58 smithi164 bash[59462]: cluster 1685824557.3050017 mgr.y (mgr.34104) 1808 : cluster 0 pgmap v1889: 73 pgs: 73 active+clean; 582 KiB data, 5.1 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:36:00.779 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:36:00 smithi089 bash[78452]: cluster 1685824559.3060467 mgr.y (mgr.34104) 1809 : cluster 0 pgmap v1890: 73 pgs: 73 active+clean; 582 KiB data, 5.1 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:36:00.779 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:36:00 smithi089 bash[75068]: cluster 1685824559.3060467 mgr.y (mgr.34104) 1809 : cluster 0 pgmap v1890: 73 pgs: 73 active+clean; 582 KiB data, 5.1 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:36:00.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:36:00 smithi164 bash[59462]: cluster 1685824559.3060467 mgr.y (mgr.34104) 1809 : cluster 0 pgmap v1890: 73 pgs: 73 active+clean; 582 KiB data, 5.1 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:36:00.986 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/mon.a/config 2023-06-03T20:36:02.667 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:36:02 smithi089 bash[75068]: cluster 1685824561.3067784 mgr.y (mgr.34104) 1810 : cluster 0 pgmap v1891: 73 pgs: 73 active+clean; 582 KiB data, 5.1 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:36:02.668 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:36:02 smithi089 bash[78452]: cluster 1685824561.3067784 mgr.y (mgr.34104) 1810 : cluster 0 pgmap v1891: 73 pgs: 73 active+clean; 582 KiB data, 5.1 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:36:02.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:36:02 smithi164 bash[59462]: cluster 1685824561.3067784 mgr.y (mgr.34104) 1810 : cluster 0 pgmap v1891: 73 pgs: 73 active+clean; 582 KiB data, 5.1 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:36:04.529 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:36:04 smithi089 bash[78452]: cluster 1685824563.3076892 mgr.y (mgr.34104) 1811 : cluster 0 pgmap v1892: 73 pgs: 73 active+clean; 582 KiB data, 5.1 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:36:04.529 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:36:04 smithi089 bash[75068]: cluster 1685824563.3076892 mgr.y (mgr.34104) 1811 : cluster 0 pgmap v1892: 73 pgs: 73 active+clean; 582 KiB data, 5.1 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:36:04.585 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:36:04 smithi164 bash[59462]: cluster 1685824563.3076892 mgr.y (mgr.34104) 1811 : cluster 0 pgmap v1892: 73 pgs: 73 active+clean; 582 KiB data, 5.1 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:36:05.207 INFO:teuthology.orchestra.run.smithi089.stderr:set osd.7 primary-affinity to 1 (8655362) 2023-06-03T20:36:05.529 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:36:05 smithi089 bash[78452]: audit 1685824565.1384447 mon.c (mon.2) 720 : audit 1 from='client.? 172.21.15.89:0/1205257677' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 7, "weight": 1.0}]: dispatch 2023-06-03T20:36:05.529 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:36:05 smithi089 bash[78452]: audit 1685824565.1390355 mon.a (mon.0) 1755 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 7, "weight": 1.0}]: dispatch 2023-06-03T20:36:05.530 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:36:05 smithi089 bash[75068]: audit 1685824565.1384447 mon.c (mon.2) 720 : audit 1 from='client.? 172.21.15.89:0/1205257677' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 7, "weight": 1.0}]: dispatch 2023-06-03T20:36:05.530 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:36:05 smithi089 bash[75068]: audit 1685824565.1390355 mon.a (mon.0) 1755 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 7, "weight": 1.0}]: dispatch 2023-06-03T20:36:05.586 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:36:05 smithi164 bash[59462]: audit 1685824565.1384447 mon.c (mon.2) 720 : audit 1 from='client.? 172.21.15.89:0/1205257677' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 7, "weight": 1.0}]: dispatch 2023-06-03T20:36:05.586 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:36:05 smithi164 bash[59462]: audit 1685824565.1390355 mon.a (mon.0) 1755 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 7, "weight": 1.0}]: dispatch 2023-06-03T20:36:06.144 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph osd reweight 4 1 2023-06-03T20:36:06.529 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 20:36:06 smithi089 bash[71163]: ::ffff:172.21.15.164 - - [03/Jun/2023:20:36:06] "GET /metrics HTTP/1.1" 200 226927 "" "Prometheus/2.33.4" 2023-06-03T20:36:06.529 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:36:06 smithi089 bash[78452]: cluster 1685824565.1905568 osd.5 (osd.5) 271 : cluster 3 failed to encode map e322 with expected crc 2023-06-03T20:36:06.530 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:36:06 smithi089 bash[78452]: audit 1685824565.1971 mon.a (mon.0) 1756 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 7, "weight": 1.0}]': finished 2023-06-03T20:36:06.530 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:36:06 smithi089 bash[78452]: cluster 1685824565.197194 mon.a (mon.0) 1757 : cluster 0 osdmap e322: 8 total, 8 up, 8 in 2023-06-03T20:36:06.530 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:36:06 smithi089 bash[78452]: cluster 1685824565.2329798 osd.6 (osd.6) 258 : cluster 3 failed to encode map e322 with expected crc 2023-06-03T20:36:06.530 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:36:06 smithi089 bash[78452]: cluster 1685824565.2331474 osd.7 (osd.7) 264 : cluster 3 failed to encode map e322 with expected crc 2023-06-03T20:36:06.530 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:36:06 smithi089 bash[78452]: cluster 1685824565.3083413 mgr.y (mgr.34104) 1812 : cluster 0 pgmap v1894: 73 pgs: 73 active+clean; 582 KiB data, 5.1 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:36:06.530 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:36:06 smithi089 bash[75068]: cluster 1685824565.1905568 osd.5 (osd.5) 271 : cluster 3 failed to encode map e322 with expected crc 2023-06-03T20:36:06.530 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:36:06 smithi089 bash[75068]: audit 1685824565.1971 mon.a (mon.0) 1756 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 7, "weight": 1.0}]': finished 2023-06-03T20:36:06.531 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:36:06 smithi089 bash[75068]: cluster 1685824565.197194 mon.a (mon.0) 1757 : cluster 0 osdmap e322: 8 total, 8 up, 8 in 2023-06-03T20:36:06.531 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:36:06 smithi089 bash[75068]: cluster 1685824565.2329798 osd.6 (osd.6) 258 : cluster 3 failed to encode map e322 with expected crc 2023-06-03T20:36:06.531 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:36:06 smithi089 bash[75068]: cluster 1685824565.2331474 osd.7 (osd.7) 264 : cluster 3 failed to encode map e322 with expected crc 2023-06-03T20:36:06.531 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:36:06 smithi089 bash[75068]: cluster 1685824565.3083413 mgr.y (mgr.34104) 1812 : cluster 0 pgmap v1894: 73 pgs: 73 active+clean; 582 KiB data, 5.1 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:36:06.585 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:36:06 smithi164 bash[59462]: cluster 1685824565.1905568 osd.5 (osd.5) 271 : cluster 3 failed to encode map e322 with expected crc 2023-06-03T20:36:06.586 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:36:06 smithi164 bash[59462]: audit 1685824565.1971 mon.a (mon.0) 1756 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 7, "weight": 1.0}]': finished 2023-06-03T20:36:06.586 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:36:06 smithi164 bash[59462]: cluster 1685824565.197194 mon.a (mon.0) 1757 : cluster 0 osdmap e322: 8 total, 8 up, 8 in 2023-06-03T20:36:06.587 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:36:06 smithi164 bash[59462]: cluster 1685824565.2329798 osd.6 (osd.6) 258 : cluster 3 failed to encode map e322 with expected crc 2023-06-03T20:36:06.587 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:36:06 smithi164 bash[59462]: cluster 1685824565.2331474 osd.7 (osd.7) 264 : cluster 3 failed to encode map e322 with expected crc 2023-06-03T20:36:06.587 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:36:06 smithi164 bash[59462]: cluster 1685824565.3083413 mgr.y (mgr.34104) 1812 : cluster 0 pgmap v1894: 73 pgs: 73 active+clean; 582 KiB data, 5.1 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:36:07.529 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:36:07 smithi089 bash[75068]: audit 1685824567.1679888 mon.c (mon.2) 721 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T20:36:07.529 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:36:07 smithi089 bash[78452]: audit 1685824567.1679888 mon.c (mon.2) 721 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T20:36:07.585 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:36:07 smithi164 bash[59462]: audit 1685824567.1679888 mon.c (mon.2) 721 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T20:36:08.529 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:36:08 smithi089 bash[75068]: cluster 1685824567.309156 mgr.y (mgr.34104) 1813 : cluster 0 pgmap v1895: 73 pgs: 73 active+clean; 582 KiB data, 5.1 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:36:08.529 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:36:08 smithi089 bash[78452]: cluster 1685824567.309156 mgr.y (mgr.34104) 1813 : cluster 0 pgmap v1895: 73 pgs: 73 active+clean; 582 KiB data, 5.1 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:36:08.585 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:36:08 smithi164 bash[59462]: cluster 1685824567.309156 mgr.y (mgr.34104) 1813 : cluster 0 pgmap v1895: 73 pgs: 73 active+clean; 582 KiB data, 5.1 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:36:10.779 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:36:10 smithi089 bash[78452]: cluster 1685824569.3103132 mgr.y (mgr.34104) 1814 : cluster 0 pgmap v1896: 73 pgs: 73 active+clean; 582 KiB data, 5.1 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:36:10.779 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:36:10 smithi089 bash[75068]: cluster 1685824569.3103132 mgr.y (mgr.34104) 1814 : cluster 0 pgmap v1896: 73 pgs: 73 active+clean; 582 KiB data, 5.1 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:36:10.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:36:10 smithi164 bash[59462]: cluster 1685824569.3103132 mgr.y (mgr.34104) 1814 : cluster 0 pgmap v1896: 73 pgs: 73 active+clean; 582 KiB data, 5.1 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:36:10.963 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/mon.a/config 2023-06-03T20:36:12.642 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:36:12 smithi089 bash[78452]: cluster 1685824571.3111775 mgr.y (mgr.34104) 1815 : cluster 0 pgmap v1897: 73 pgs: 73 active+clean; 582 KiB data, 5.1 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:36:12.642 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:36:12 smithi089 bash[75068]: cluster 1685824571.3111775 mgr.y (mgr.34104) 1815 : cluster 0 pgmap v1897: 73 pgs: 73 active+clean; 582 KiB data, 5.1 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:36:12.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:36:12 smithi164 bash[59462]: cluster 1685824571.3111775 mgr.y (mgr.34104) 1815 : cluster 0 pgmap v1897: 73 pgs: 73 active+clean; 582 KiB data, 5.1 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:36:14.529 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:36:14 smithi089 bash[75068]: cluster 1685824573.3122394 mgr.y (mgr.34104) 1816 : cluster 0 pgmap v1898: 73 pgs: 73 active+clean; 582 KiB data, 5.1 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:36:14.529 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:36:14 smithi089 bash[78452]: cluster 1685824573.3122394 mgr.y (mgr.34104) 1816 : cluster 0 pgmap v1898: 73 pgs: 73 active+clean; 582 KiB data, 5.1 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:36:14.585 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:36:14 smithi164 bash[59462]: cluster 1685824573.3122394 mgr.y (mgr.34104) 1816 : cluster 0 pgmap v1898: 73 pgs: 73 active+clean; 582 KiB data, 5.1 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:36:15.208 INFO:teuthology.orchestra.run.smithi089.stderr:reweighted osd.4 to 1 (10000) 2023-06-03T20:36:15.529 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:36:15 smithi089 bash[75068]: audit 1685824575.0691457 mon.c (mon.2) 722 : audit 1 from='client.? 172.21.15.89:0/4099598099' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-06-03T20:36:15.529 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:36:15 smithi089 bash[75068]: audit 1685824575.0697103 mon.a (mon.0) 1758 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-06-03T20:36:15.530 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:36:15 smithi089 bash[78452]: audit 1685824575.0691457 mon.c (mon.2) 722 : audit 1 from='client.? 172.21.15.89:0/4099598099' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-06-03T20:36:15.530 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:36:15 smithi089 bash[78452]: audit 1685824575.0697103 mon.a (mon.0) 1758 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-06-03T20:36:15.585 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:36:15 smithi164 bash[59462]: audit 1685824575.0691457 mon.c (mon.2) 722 : audit 1 from='client.? 172.21.15.89:0/4099598099' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-06-03T20:36:15.586 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:36:15 smithi164 bash[59462]: audit 1685824575.0697103 mon.a (mon.0) 1758 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-06-03T20:36:16.158 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph osd primary-affinity 4 1 2023-06-03T20:36:16.529 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 20:36:16 smithi089 bash[71163]: ::ffff:172.21.15.164 - - [03/Jun/2023:20:36:16] "GET /metrics HTTP/1.1" 200 226914 "" "Prometheus/2.33.4" 2023-06-03T20:36:16.529 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:36:16 smithi089 bash[78452]: audit 1685824575.196679 mon.a (mon.0) 1759 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2023-06-03T20:36:16.529 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:36:16 smithi089 bash[78452]: cluster 1685824575.196845 mon.a (mon.0) 1760 : cluster 0 osdmap e323: 8 total, 8 up, 8 in 2023-06-03T20:36:16.530 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:36:16 smithi089 bash[78452]: cluster 1685824575.312941 mgr.y (mgr.34104) 1817 : cluster 0 pgmap v1900: 73 pgs: 73 active+clean; 582 KiB data, 5.1 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:36:16.530 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:36:16 smithi089 bash[75068]: audit 1685824575.196679 mon.a (mon.0) 1759 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2023-06-03T20:36:16.530 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:36:16 smithi089 bash[75068]: cluster 1685824575.196845 mon.a (mon.0) 1760 : cluster 0 osdmap e323: 8 total, 8 up, 8 in 2023-06-03T20:36:16.530 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:36:16 smithi089 bash[75068]: cluster 1685824575.312941 mgr.y (mgr.34104) 1817 : cluster 0 pgmap v1900: 73 pgs: 73 active+clean; 582 KiB data, 5.1 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:36:16.585 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:36:16 smithi164 bash[59462]: audit 1685824575.196679 mon.a (mon.0) 1759 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2023-06-03T20:36:16.586 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:36:16 smithi164 bash[59462]: cluster 1685824575.196845 mon.a (mon.0) 1760 : cluster 0 osdmap e323: 8 total, 8 up, 8 in 2023-06-03T20:36:16.586 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:36:16 smithi164 bash[59462]: cluster 1685824575.312941 mgr.y (mgr.34104) 1817 : cluster 0 pgmap v1900: 73 pgs: 73 active+clean; 582 KiB data, 5.1 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:36:17.529 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:36:17 smithi089 bash[75068]: cluster 1685824575.7301872 osd.7 (osd.7) 265 : cluster 3 failed to encode map e323 with expected crc 2023-06-03T20:36:17.529 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:36:17 smithi089 bash[75068]: cluster 1685824576.3053317 osd.6 (osd.6) 259 : cluster 3 failed to encode map e323 with expected crc 2023-06-03T20:36:17.529 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:36:17 smithi089 bash[75068]: cluster 1685824576.305635 osd.5 (osd.5) 272 : cluster 3 failed to encode map e323 with expected crc 2023-06-03T20:36:17.530 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:36:17 smithi089 bash[78452]: cluster 1685824575.7301872 osd.7 (osd.7) 265 : cluster 3 failed to encode map e323 with expected crc 2023-06-03T20:36:17.530 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:36:17 smithi089 bash[78452]: cluster 1685824576.3053317 osd.6 (osd.6) 259 : cluster 3 failed to encode map e323 with expected crc 2023-06-03T20:36:17.530 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:36:17 smithi089 bash[78452]: cluster 1685824576.305635 osd.5 (osd.5) 272 : cluster 3 failed to encode map e323 with expected crc 2023-06-03T20:36:17.589 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:36:17 smithi164 bash[59462]: cluster 1685824575.7301872 osd.7 (osd.7) 265 : cluster 3 failed to encode map e323 with expected crc 2023-06-03T20:36:17.589 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:36:17 smithi164 bash[59462]: cluster 1685824576.3053317 osd.6 (osd.6) 259 : cluster 3 failed to encode map e323 with expected crc 2023-06-03T20:36:17.589 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:36:17 smithi164 bash[59462]: cluster 1685824576.305635 osd.5 (osd.5) 272 : cluster 3 failed to encode map e323 with expected crc 2023-06-03T20:36:18.529 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:36:18 smithi089 bash[75068]: cluster 1685824577.3136652 mgr.y (mgr.34104) 1818 : cluster 0 pgmap v1901: 73 pgs: 73 active+clean; 582 KiB data, 5.1 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:36:18.530 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:36:18 smithi089 bash[78452]: cluster 1685824577.3136652 mgr.y (mgr.34104) 1818 : cluster 0 pgmap v1901: 73 pgs: 73 active+clean; 582 KiB data, 5.1 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:36:18.585 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:36:18 smithi164 bash[59462]: cluster 1685824577.3136652 mgr.y (mgr.34104) 1818 : cluster 0 pgmap v1901: 73 pgs: 73 active+clean; 582 KiB data, 5.1 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:36:20.779 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:36:20 smithi089 bash[78452]: cluster 1685824579.3145216 mgr.y (mgr.34104) 1819 : cluster 0 pgmap v1902: 73 pgs: 73 active+clean; 582 KiB data, 5.1 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:36:20.779 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:36:20 smithi089 bash[75068]: cluster 1685824579.3145216 mgr.y (mgr.34104) 1819 : cluster 0 pgmap v1902: 73 pgs: 73 active+clean; 582 KiB data, 5.1 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:36:20.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:36:20 smithi164 bash[59462]: cluster 1685824579.3145216 mgr.y (mgr.34104) 1819 : cluster 0 pgmap v1902: 73 pgs: 73 active+clean; 582 KiB data, 5.1 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:36:20.978 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/mon.a/config 2023-06-03T20:36:22.715 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:36:22 smithi089 bash[78452]: cluster 1685824581.3152807 mgr.y (mgr.34104) 1820 : cluster 0 pgmap v1903: 73 pgs: 73 active+clean; 582 KiB data, 5.1 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:36:22.715 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:36:22 smithi089 bash[78452]: audit 1685824582.1682763 mon.c (mon.2) 723 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T20:36:22.716 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:36:22 smithi089 bash[75068]: cluster 1685824581.3152807 mgr.y (mgr.34104) 1820 : cluster 0 pgmap v1903: 73 pgs: 73 active+clean; 582 KiB data, 5.1 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:36:22.716 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:36:22 smithi089 bash[75068]: audit 1685824582.1682763 mon.c (mon.2) 723 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T20:36:22.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:36:22 smithi164 bash[59462]: cluster 1685824581.3152807 mgr.y (mgr.34104) 1820 : cluster 0 pgmap v1903: 73 pgs: 73 active+clean; 582 KiB data, 5.1 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:36:22.836 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:36:22 smithi164 bash[59462]: audit 1685824582.1682763 mon.c (mon.2) 723 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T20:36:24.529 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:36:24 smithi089 bash[78452]: cluster 1685824583.316206 mgr.y (mgr.34104) 1821 : cluster 0 pgmap v1904: 73 pgs: 73 active+clean; 582 KiB data, 5.1 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:36:24.529 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:36:24 smithi089 bash[75068]: cluster 1685824583.316206 mgr.y (mgr.34104) 1821 : cluster 0 pgmap v1904: 73 pgs: 73 active+clean; 582 KiB data, 5.1 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:36:24.585 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:36:24 smithi164 bash[59462]: cluster 1685824583.316206 mgr.y (mgr.34104) 1821 : cluster 0 pgmap v1904: 73 pgs: 73 active+clean; 582 KiB data, 5.1 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:36:25.212 INFO:teuthology.orchestra.run.smithi089.stderr:set osd.4 primary-affinity to 1 (8655362) 2023-06-03T20:36:25.529 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:36:25 smithi089 bash[75068]: audit 1685824585.0902436 mon.a (mon.0) 1761 : audit 1 from='client.? 172.21.15.89:0/420044385' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 4, "weight": 1.0}]: dispatch 2023-06-03T20:36:25.529 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:36:25 smithi089 bash[78452]: audit 1685824585.0902436 mon.a (mon.0) 1761 : audit 1 from='client.? 172.21.15.89:0/420044385' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 4, "weight": 1.0}]: dispatch 2023-06-03T20:36:25.585 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:36:25 smithi164 bash[59462]: audit 1685824585.0902436 mon.a (mon.0) 1761 : audit 1 from='client.? 172.21.15.89:0/420044385' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 4, "weight": 1.0}]: dispatch 2023-06-03T20:36:26.153 INFO:tasks.thrashosds.thrasher:joining the do_sighup greenlet 2023-06-03T20:36:26.154 INFO:tasks.thrashosds.thrasher:joining the do_optrack_toggle greenlet 2023-06-03T20:36:26.154 INFO:tasks.thrashosds.thrasher:joining the do_dump_ops greenlet 2023-06-03T20:36:26.154 INFO:tasks.thrashosds.thrasher:joining the do_noscrub_toggle greenlet 2023-06-03T20:36:26.155 INFO:tasks.cephadm.ceph_manager.ceph:waiting for all up 2023-06-03T20:36:26.155 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph osd dump --format=json 2023-06-03T20:36:26.529 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 20:36:26 smithi089 bash[71163]: ::ffff:172.21.15.164 - - [03/Jun/2023:20:36:26] "GET /metrics HTTP/1.1" 200 226939 "" "Prometheus/2.33.4" 2023-06-03T20:36:26.529 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:36:26 smithi089 bash[75068]: cluster 1685824585.1983435 osd.5 (osd.5) 273 : cluster 3 failed to encode map e324 with expected crc 2023-06-03T20:36:26.529 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:36:26 smithi089 bash[75068]: audit 1685824585.20333 mon.a (mon.0) 1762 : audit 1 from='client.? 172.21.15.89:0/420044385' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 4, "weight": 1.0}]': finished 2023-06-03T20:36:26.530 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:36:26 smithi089 bash[75068]: cluster 1685824585.203397 mon.a (mon.0) 1763 : cluster 0 osdmap e324: 8 total, 8 up, 8 in 2023-06-03T20:36:26.530 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:36:26 smithi089 bash[75068]: cluster 1685824585.3169768 mgr.y (mgr.34104) 1822 : cluster 0 pgmap v1906: 73 pgs: 73 active+clean; 582 KiB data, 5.1 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:36:26.530 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:36:26 smithi089 bash[78452]: cluster 1685824585.1983435 osd.5 (osd.5) 273 : cluster 3 failed to encode map e324 with expected crc 2023-06-03T20:36:26.530 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:36:26 smithi089 bash[78452]: audit 1685824585.20333 mon.a (mon.0) 1762 : audit 1 from='client.? 172.21.15.89:0/420044385' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 4, "weight": 1.0}]': finished 2023-06-03T20:36:26.530 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:36:26 smithi089 bash[78452]: cluster 1685824585.203397 mon.a (mon.0) 1763 : cluster 0 osdmap e324: 8 total, 8 up, 8 in 2023-06-03T20:36:26.531 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:36:26 smithi089 bash[78452]: cluster 1685824585.3169768 mgr.y (mgr.34104) 1822 : cluster 0 pgmap v1906: 73 pgs: 73 active+clean; 582 KiB data, 5.1 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:36:26.585 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:36:26 smithi164 bash[59462]: cluster 1685824585.1983435 osd.5 (osd.5) 273 : cluster 3 failed to encode map e324 with expected crc 2023-06-03T20:36:26.586 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:36:26 smithi164 bash[59462]: audit 1685824585.20333 mon.a (mon.0) 1762 : audit 1 from='client.? 172.21.15.89:0/420044385' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 4, "weight": 1.0}]': finished 2023-06-03T20:36:26.586 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:36:26 smithi164 bash[59462]: cluster 1685824585.203397 mon.a (mon.0) 1763 : cluster 0 osdmap e324: 8 total, 8 up, 8 in 2023-06-03T20:36:26.586 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:36:26 smithi164 bash[59462]: cluster 1685824585.3169768 mgr.y (mgr.34104) 1822 : cluster 0 pgmap v1906: 73 pgs: 73 active+clean; 582 KiB data, 5.1 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:36:27.529 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:36:27 smithi089 bash[78452]: cluster 1685824586.5040777 osd.6 (osd.6) 260 : cluster 3 failed to encode map e324 with expected crc 2023-06-03T20:36:27.529 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:36:27 smithi089 bash[75068]: cluster 1685824586.5040777 osd.6 (osd.6) 260 : cluster 3 failed to encode map e324 with expected crc 2023-06-03T20:36:27.585 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:36:27 smithi164 bash[59462]: cluster 1685824586.5040777 osd.6 (osd.6) 260 : cluster 3 failed to encode map e324 with expected crc 2023-06-03T20:36:28.529 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:36:28 smithi089 bash[75068]: cluster 1685824586.5410318 osd.7 (osd.7) 266 : cluster 3 failed to encode map e324 with expected crc 2023-06-03T20:36:28.529 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:36:28 smithi089 bash[75068]: cluster 1685824587.3177347 mgr.y (mgr.34104) 1823 : cluster 0 pgmap v1907: 73 pgs: 73 active+clean; 582 KiB data, 5.1 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:36:28.529 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:36:28 smithi089 bash[78452]: cluster 1685824586.5410318 osd.7 (osd.7) 266 : cluster 3 failed to encode map e324 with expected crc 2023-06-03T20:36:28.530 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:36:28 smithi089 bash[78452]: cluster 1685824587.3177347 mgr.y (mgr.34104) 1823 : cluster 0 pgmap v1907: 73 pgs: 73 active+clean; 582 KiB data, 5.1 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:36:28.585 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:36:28 smithi164 bash[59462]: cluster 1685824586.5410318 osd.7 (osd.7) 266 : cluster 3 failed to encode map e324 with expected crc 2023-06-03T20:36:28.585 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:36:28 smithi164 bash[59462]: cluster 1685824587.3177347 mgr.y (mgr.34104) 1823 : cluster 0 pgmap v1907: 73 pgs: 73 active+clean; 582 KiB data, 5.1 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:36:30.779 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:36:30 smithi089 bash[75068]: cluster 1685824589.31875 mgr.y (mgr.34104) 1824 : cluster 0 pgmap v1908: 73 pgs: 73 active+clean; 582 KiB data, 5.1 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:36:30.779 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:36:30 smithi089 bash[78452]: cluster 1685824589.31875 mgr.y (mgr.34104) 1824 : cluster 0 pgmap v1908: 73 pgs: 73 active+clean; 582 KiB data, 5.1 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:36:30.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:36:30 smithi164 bash[59462]: cluster 1685824589.31875 mgr.y (mgr.34104) 1824 : cluster 0 pgmap v1908: 73 pgs: 73 active+clean; 582 KiB data, 5.1 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:36:30.977 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/mon.a/config 2023-06-03T20:36:32.529 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:36:32 smithi089 bash[75068]: cluster 1685824591.319555 mgr.y (mgr.34104) 1825 : cluster 0 pgmap v1909: 73 pgs: 73 active+clean; 582 KiB data, 5.2 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:36:32.529 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:36:32 smithi089 bash[78452]: cluster 1685824591.319555 mgr.y (mgr.34104) 1825 : cluster 0 pgmap v1909: 73 pgs: 73 active+clean; 582 KiB data, 5.2 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:36:32.836 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:36:32 smithi164 bash[59462]: cluster 1685824591.319555 mgr.y (mgr.34104) 1825 : cluster 0 pgmap v1909: 73 pgs: 73 active+clean; 582 KiB data, 5.2 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:36:34.529 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:36:34 smithi089 bash[75068]: cluster 1685824593.3205643 mgr.y (mgr.34104) 1826 : cluster 0 pgmap v1910: 73 pgs: 73 active+clean; 582 KiB data, 5.2 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:36:34.529 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:36:34 smithi089 bash[78452]: cluster 1685824593.3205643 mgr.y (mgr.34104) 1826 : cluster 0 pgmap v1910: 73 pgs: 73 active+clean; 582 KiB data, 5.2 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:36:34.585 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:36:34 smithi164 bash[59462]: cluster 1685824593.3205643 mgr.y (mgr.34104) 1826 : cluster 0 pgmap v1910: 73 pgs: 73 active+clean; 582 KiB data, 5.2 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:36:35.059 INFO:teuthology.orchestra.run.smithi089.stdout: 2023-06-03T20:36:35.060 INFO:teuthology.orchestra.run.smithi089.stdout:{"epoch":324,"fsid":"f1db3478-0243-11ee-9b1d-001a4aab830c","created":"2023-06-03T19:22:54.447184+0000","modified":"2023-06-03T20:36:25.190868+0000","last_up_change":"2023-06-03T20:23:26.213117+0000","last_in_change":"2023-06-03T20:24:01.219178+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":24,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":15,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"luminous","require_osd_release":"pacific","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2023-06-03T19:27:28.100610+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"70","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{},"mgr_devicehealth":{}},"read_balance":{"score_acting":7.8899998664855957,"score_stable":7.8899998664855957,"optimal_score":0.37999999523162842,"raw_score_acting":3,"raw_score_stable":3,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2023-06-03T19:32:20.641846+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"55","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_acting":2,"score_stable":2,"optimal_score":1,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":3,"pool_name":"cephfs.foo.meta","create_time":"2023-06-03T19:34:15.554658+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"59","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"foo"}},"read_balance":{"score_acting":1.5,"score_stable":1.5,"optimal_score":1,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":4,"pool_name":"cephfs.foo.data","create_time":"2023-06-03T19:34:16.201294+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"60","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"cephfs":{"data":"foo"}},"read_balance":{"score_acting":2.25,"score_stable":2.25,"optimal_score":1,"raw_score_acting":2.25,"raw_score_stable":2.25,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"86063bfe-126e-46e7-93ed-c73813af165b","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":8,"last_clean_end":82,"up_from":86,"up_thru":299,"down_at":83,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:6802","nonce":2920103307},{"type":"v1","addr":"172.21.15.89:6803","nonce":2920103307}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:6804","nonce":2920103307},{"type":"v1","addr":"172.21.15.89:6805","nonce":2920103307}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:6808","nonce":2920103307},{"type":"v1","addr":"172.21.15.89:6809","nonce":2920103307}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:6806","nonce":2920103307},{"type":"v1","addr":"172.21.15.89:6807","nonce":2920103307}]},"public_addr":"172.21.15.89:6803/2920103307","cluster_addr":"172.21.15.89:6805/2920103307","heartbeat_back_addr":"172.21.15.89:6809/2920103307","heartbeat_front_addr":"172.21.15.89:6807/2920103307","state":["exists","up"]},{"osd":1,"uuid":"04783a81-6b8c-4657-8446-00efc872b313","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":13,"last_clean_end":99,"up_from":103,"up_thru":299,"down_at":100,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:6810","nonce":2092956116},{"type":"v1","addr":"172.21.15.89:6811","nonce":2092956116}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:6812","nonce":2092956116},{"type":"v1","addr":"172.21.15.89:6813","nonce":2092956116}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:6816","nonce":2092956116},{"type":"v1","addr":"172.21.15.89:6817","nonce":2092956116}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:6814","nonce":2092956116},{"type":"v1","addr":"172.21.15.89:6815","nonce":2092956116}]},"public_addr":"172.21.15.89:6811/2092956116","cluster_addr":"172.21.15.89:6813/2092956116","heartbeat_back_addr":"172.21.15.89:6817/2092956116","heartbeat_front_addr":"172.21.15.89:6815/2092956116","state":["exists","up"]},{"osd":2,"uuid":"d2d2c641-392a-444f-81bc-84eef6340f30","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":18,"last_clean_end":110,"up_from":114,"up_thru":299,"down_at":111,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:6818","nonce":1439643284},{"type":"v1","addr":"172.21.15.89:6819","nonce":1439643284}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:6820","nonce":1439643284},{"type":"v1","addr":"172.21.15.89:6821","nonce":1439643284}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:6824","nonce":1439643284},{"type":"v1","addr":"172.21.15.89:6825","nonce":1439643284}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:6822","nonce":1439643284},{"type":"v1","addr":"172.21.15.89:6823","nonce":1439643284}]},"public_addr":"172.21.15.89:6819/1439643284","cluster_addr":"172.21.15.89:6821/1439643284","heartbeat_back_addr":"172.21.15.89:6825/1439643284","heartbeat_front_addr":"172.21.15.89:6823/1439643284","state":["exists","up"]},{"osd":3,"uuid":"b38408eb-3acf-4a41-a314-72e4cd37b1ef","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":25,"last_clean_end":115,"up_from":119,"up_thru":299,"down_at":116,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:6826","nonce":1558463544},{"type":"v1","addr":"172.21.15.89:6827","nonce":1558463544}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:6828","nonce":1558463544},{"type":"v1","addr":"172.21.15.89:6829","nonce":1558463544}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:6832","nonce":1558463544},{"type":"v1","addr":"172.21.15.89:6833","nonce":1558463544}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:6830","nonce":1558463544},{"type":"v1","addr":"172.21.15.89:6831","nonce":1558463544}]},"public_addr":"172.21.15.89:6827/1558463544","cluster_addr":"172.21.15.89:6829/1558463544","heartbeat_back_addr":"172.21.15.89:6833/1558463544","heartbeat_front_addr":"172.21.15.89:6831/1558463544","state":["exists","up"]},{"osd":4,"uuid":"a94be785-9f54-4ab8-962a-9cf974a5b5e3","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":124,"last_clean_end":194,"up_from":250,"up_thru":299,"down_at":195,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.164:6800","nonce":3945899737},{"type":"v1","addr":"172.21.15.164:6801","nonce":3945899737}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.164:6802","nonce":3945899737},{"type":"v1","addr":"172.21.15.164:6803","nonce":3945899737}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.164:6806","nonce":3945899737},{"type":"v1","addr":"172.21.15.164:6807","nonce":3945899737}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.164:6804","nonce":3945899737},{"type":"v1","addr":"172.21.15.164:6805","nonce":3945899737}]},"public_addr":"172.21.15.164:6801/3945899737","cluster_addr":"172.21.15.164:6803/3945899737","heartbeat_back_addr":"172.21.15.164:6807/3945899737","heartbeat_front_addr":"172.21.15.164:6805/3945899737","state":["exists","up"]},{"osd":5,"uuid":"b8a5e8a1-23cc-442b-90aa-19bd0036dd3c","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":36,"up_thru":317,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.164:6808","nonce":3123006405},{"type":"v1","addr":"172.21.15.164:6809","nonce":3123006405}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.164:6810","nonce":3123006405},{"type":"v1","addr":"172.21.15.164:6811","nonce":3123006405}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.164:6814","nonce":3123006405},{"type":"v1","addr":"172.21.15.164:6815","nonce":3123006405}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.164:6812","nonce":3123006405},{"type":"v1","addr":"172.21.15.164:6813","nonce":3123006405}]},"public_addr":"172.21.15.164:6809/3123006405","cluster_addr":"172.21.15.164:6811/3123006405","heartbeat_back_addr":"172.21.15.164:6815/3123006405","heartbeat_front_addr":"172.21.15.164:6813/3123006405","state":["exists","up"]},{"osd":6,"uuid":"5c5edd3b-d92c-4f46-94ff-2d5828b2e732","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":42,"up_thru":299,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.164:6816","nonce":712403290},{"type":"v1","addr":"172.21.15.164:6817","nonce":712403290}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.164:6818","nonce":712403290},{"type":"v1","addr":"172.21.15.164:6819","nonce":712403290}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.164:6822","nonce":712403290},{"type":"v1","addr":"172.21.15.164:6823","nonce":712403290}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.164:6820","nonce":712403290},{"type":"v1","addr":"172.21.15.164:6821","nonce":712403290}]},"public_addr":"172.21.15.164:6817/712403290","cluster_addr":"172.21.15.164:6819/712403290","heartbeat_back_addr":"172.21.15.164:6823/712403290","heartbeat_front_addr":"172.21.15.164:6821/712403290","state":["exists","up"]},{"osd":7,"uuid":"bdff9290-e6c4-407e-9113-eb39696c0e27","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":48,"up_thru":299,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.164:6824","nonce":711997991},{"type":"v1","addr":"172.21.15.164:6825","nonce":711997991}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.164:6826","nonce":711997991},{"type":"v1","addr":"172.21.15.164:6827","nonce":711997991}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.164:6830","nonce":711997991},{"type":"v1","addr":"172.21.15.164:6831","nonce":711997991}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.164:6828","nonce":711997991},{"type":"v1","addr":"172.21.15.164:6829","nonce":711997991}]},"public_addr":"172.21.15.164:6825/711997991","cluster_addr":"172.21.15.164:6827/711997991","heartbeat_back_addr":"172.21.15.164:6831/711997991","heartbeat_front_addr":"172.21.15.164:6829/711997991","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-06-03T19:25:52.155714+0000","dead_epoch":82},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-06-03T19:26:38.554458+0000","dead_epoch":99},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-06-03T19:27:25.383237+0000","dead_epoch":110},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-06-03T19:28:13.272585+0000","dead_epoch":115},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-06-03T19:28:56.685710+0000","dead_epoch":194},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-06-03T19:29:41.572533+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-06-03T19:30:26.631983+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-06-03T19:31:12.018414+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[{"pgid":"4.17","mappings":[{"from":1,"to":2}]},{"pgid":"4.1b","mappings":[{"from":1,"to":2}]},{"pgid":"4.1d","mappings":[{"from":3,"to":5}]},{"pgid":"4.1f","mappings":[{"from":1,"to":2}]}],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.89:0/2890034071":"2023-06-04T19:41:21.936058+0000","172.21.15.89:6801/3918707857":"2023-06-04T19:41:21.936058+0000","172.21.15.89:6800/3918707857":"2023-06-04T19:41:21.936058+0000","172.21.15.89:0/101502033":"2023-06-04T19:41:21.936058+0000","172.21.15.89:0/3543817158":"2023-06-04T19:41:21.936058+0000","172.21.15.164:0/2793695939":"2023-06-04T19:40:01.102939+0000","172.21.15.89:0/3944267183":"2023-06-04T19:23:55.626873+0000","172.21.15.89:0/3044409358":"2023-06-04T19:41:21.936058+0000","172.21.15.89:0/811967393":"2023-06-04T19:23:23.903333+0000","172.21.15.89:6800/2256305550":"2023-06-04T19:31:48.303240+0000","172.21.15.164:6835/9946422":"2023-06-04T19:40:01.102939+0000","172.21.15.89:0/2023636358":"2023-06-04T19:37:38.185918+0000","172.21.15.164:0/3249546020":"2023-06-04T19:40:01.102939+0000","172.21.15.164:0/1307416622":"2023-06-04T19:40:01.102939+0000","172.21.15.89:6801/2009133143":"2023-06-04T19:37:38.185918+0000","172.21.15.164:6834/9946422":"2023-06-04T19:40:01.102939+0000","172.21.15.89:6800/1487566136":"2023-06-04T19:23:23.903333+0000","172.21.15.89:0/1198702890":"2023-06-04T19:23:23.903333+0000","172.21.15.89:6801/1487566136":"2023-06-04T19:23:23.903333+0000","172.21.15.89:0/274274920":"2023-06-04T19:23:55.626873+0000","172.21.15.89:0/2392321634":"2023-06-04T19:31:48.303240+0000","172.21.15.89:0/1591228500":"2023-06-04T19:37:38.185918+0000","172.21.15.89:0/949193420":"2023-06-04T19:41:21.936058+0000","172.21.15.89:6800/2009133143":"2023-06-04T19:37:38.185918+0000","172.21.15.164:0/3011990159":"2023-06-04T19:40:01.102939+0000","172.21.15.89:6801/2256305550":"2023-06-04T19:31:48.303240+0000","172.21.15.89:6800/4116589411":"2023-06-04T19:23:55.626873+0000","172.21.15.89:0/3513444952":"2023-06-04T19:41:21.936058+0000","172.21.15.89:6801/4116589411":"2023-06-04T19:23:55.626873+0000","172.21.15.89:0/470853232":"2023-06-04T19:31:48.303240+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[{"pool":15,"snaps":[{"begin":2,"length":1}]}],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2023-06-03T20:36:35.529 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:36:35 smithi089 bash[75068]: audit 1685824595.0516224 mon.a (mon.0) 1764 : audit 0 from='client.? 172.21.15.89:0/2098585891' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-06-03T20:36:35.529 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:36:35 smithi089 bash[78452]: audit 1685824595.0516224 mon.a (mon.0) 1764 : audit 0 from='client.? 172.21.15.89:0/2098585891' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-06-03T20:36:35.585 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:36:35 smithi164 bash[59462]: audit 1685824595.0516224 mon.a (mon.0) 1764 : audit 0 from='client.? 172.21.15.89:0/2098585891' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-06-03T20:36:36.055 INFO:tasks.cephadm.ceph_manager.ceph:all up! 2023-06-03T20:36:36.056 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph osd dump --format=json 2023-06-03T20:36:36.529 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:36:36 smithi089 bash[75068]: cluster 1685824595.3213542 mgr.y (mgr.34104) 1827 : cluster 0 pgmap v1911: 73 pgs: 73 active+clean; 582 KiB data, 5.2 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:36:36.529 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 20:36:36 smithi089 bash[71163]: ::ffff:172.21.15.164 - - [03/Jun/2023:20:36:36] "GET /metrics HTTP/1.1" 200 226939 "" "Prometheus/2.33.4" 2023-06-03T20:36:36.530 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:36:36 smithi089 bash[78452]: cluster 1685824595.3213542 mgr.y (mgr.34104) 1827 : cluster 0 pgmap v1911: 73 pgs: 73 active+clean; 582 KiB data, 5.2 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:36:36.585 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:36:36 smithi164 bash[59462]: cluster 1685824595.3213542 mgr.y (mgr.34104) 1827 : cluster 0 pgmap v1911: 73 pgs: 73 active+clean; 582 KiB data, 5.2 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:36:37.529 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:36:37 smithi089 bash[75068]: audit 1685824597.1685565 mon.c (mon.2) 724 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T20:36:37.529 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:36:37 smithi089 bash[78452]: audit 1685824597.1685565 mon.c (mon.2) 724 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T20:36:37.585 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:36:37 smithi164 bash[59462]: audit 1685824597.1685565 mon.c (mon.2) 724 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T20:36:38.529 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:36:38 smithi089 bash[75068]: cluster 1685824597.3221018 mgr.y (mgr.34104) 1828 : cluster 0 pgmap v1912: 73 pgs: 73 active+clean; 582 KiB data, 5.2 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:36:38.529 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:36:38 smithi089 bash[78452]: cluster 1685824597.3221018 mgr.y (mgr.34104) 1828 : cluster 0 pgmap v1912: 73 pgs: 73 active+clean; 582 KiB data, 5.2 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:36:38.585 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:36:38 smithi164 bash[59462]: cluster 1685824597.3221018 mgr.y (mgr.34104) 1828 : cluster 0 pgmap v1912: 73 pgs: 73 active+clean; 582 KiB data, 5.2 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:36:40.779 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:36:40 smithi089 bash[75068]: cluster 1685824599.3234897 mgr.y (mgr.34104) 1829 : cluster 0 pgmap v1913: 73 pgs: 73 active+clean; 582 KiB data, 5.2 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:36:40.780 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:36:40 smithi089 bash[78452]: cluster 1685824599.3234897 mgr.y (mgr.34104) 1829 : cluster 0 pgmap v1913: 73 pgs: 73 active+clean; 582 KiB data, 5.2 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:36:40.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:36:40 smithi164 bash[59462]: cluster 1685824599.3234897 mgr.y (mgr.34104) 1829 : cluster 0 pgmap v1913: 73 pgs: 73 active+clean; 582 KiB data, 5.2 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:36:40.875 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/mon.a/config 2023-06-03T20:36:42.528 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:36:42 smithi089 bash[75068]: cluster 1685824601.3242326 mgr.y (mgr.34104) 1830 : cluster 0 pgmap v1914: 73 pgs: 73 active+clean; 582 KiB data, 5.2 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:36:42.528 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:36:42 smithi089 bash[78452]: cluster 1685824601.3242326 mgr.y (mgr.34104) 1830 : cluster 0 pgmap v1914: 73 pgs: 73 active+clean; 582 KiB data, 5.2 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:36:42.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:36:42 smithi164 bash[59462]: cluster 1685824601.3242326 mgr.y (mgr.34104) 1830 : cluster 0 pgmap v1914: 73 pgs: 73 active+clean; 582 KiB data, 5.2 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:36:44.507 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:36:44 smithi089 bash[75068]: cluster 1685824603.3250813 mgr.y (mgr.34104) 1831 : cluster 0 pgmap v1915: 73 pgs: 73 active+clean; 582 KiB data, 5.2 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:36:44.507 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:36:44 smithi089 bash[78452]: cluster 1685824603.3250813 mgr.y (mgr.34104) 1831 : cluster 0 pgmap v1915: 73 pgs: 73 active+clean; 582 KiB data, 5.2 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:36:44.585 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:36:44 smithi164 bash[59462]: cluster 1685824603.3250813 mgr.y (mgr.34104) 1831 : cluster 0 pgmap v1915: 73 pgs: 73 active+clean; 582 KiB data, 5.2 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:36:44.894 INFO:teuthology.orchestra.run.smithi089.stdout: 2023-06-03T20:36:44.894 INFO:teuthology.orchestra.run.smithi089.stdout:{"epoch":324,"fsid":"f1db3478-0243-11ee-9b1d-001a4aab830c","created":"2023-06-03T19:22:54.447184+0000","modified":"2023-06-03T20:36:25.190868+0000","last_up_change":"2023-06-03T20:23:26.213117+0000","last_in_change":"2023-06-03T20:24:01.219178+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":24,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":15,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"luminous","require_osd_release":"pacific","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2023-06-03T19:27:28.100610+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"70","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{},"mgr_devicehealth":{}},"read_balance":{"score_acting":7.8899998664855957,"score_stable":7.8899998664855957,"optimal_score":0.37999999523162842,"raw_score_acting":3,"raw_score_stable":3,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2023-06-03T19:32:20.641846+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"55","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_acting":2,"score_stable":2,"optimal_score":1,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":3,"pool_name":"cephfs.foo.meta","create_time":"2023-06-03T19:34:15.554658+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"59","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"foo"}},"read_balance":{"score_acting":1.5,"score_stable":1.5,"optimal_score":1,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":4,"pool_name":"cephfs.foo.data","create_time":"2023-06-03T19:34:16.201294+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"60","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"cephfs":{"data":"foo"}},"read_balance":{"score_acting":2.25,"score_stable":2.25,"optimal_score":1,"raw_score_acting":2.25,"raw_score_stable":2.25,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"86063bfe-126e-46e7-93ed-c73813af165b","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":8,"last_clean_end":82,"up_from":86,"up_thru":299,"down_at":83,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:6802","nonce":2920103307},{"type":"v1","addr":"172.21.15.89:6803","nonce":2920103307}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:6804","nonce":2920103307},{"type":"v1","addr":"172.21.15.89:6805","nonce":2920103307}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:6808","nonce":2920103307},{"type":"v1","addr":"172.21.15.89:6809","nonce":2920103307}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:6806","nonce":2920103307},{"type":"v1","addr":"172.21.15.89:6807","nonce":2920103307}]},"public_addr":"172.21.15.89:6803/2920103307","cluster_addr":"172.21.15.89:6805/2920103307","heartbeat_back_addr":"172.21.15.89:6809/2920103307","heartbeat_front_addr":"172.21.15.89:6807/2920103307","state":["exists","up"]},{"osd":1,"uuid":"04783a81-6b8c-4657-8446-00efc872b313","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":13,"last_clean_end":99,"up_from":103,"up_thru":299,"down_at":100,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:6810","nonce":2092956116},{"type":"v1","addr":"172.21.15.89:6811","nonce":2092956116}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:6812","nonce":2092956116},{"type":"v1","addr":"172.21.15.89:6813","nonce":2092956116}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:6816","nonce":2092956116},{"type":"v1","addr":"172.21.15.89:6817","nonce":2092956116}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:6814","nonce":2092956116},{"type":"v1","addr":"172.21.15.89:6815","nonce":2092956116}]},"public_addr":"172.21.15.89:6811/2092956116","cluster_addr":"172.21.15.89:6813/2092956116","heartbeat_back_addr":"172.21.15.89:6817/2092956116","heartbeat_front_addr":"172.21.15.89:6815/2092956116","state":["exists","up"]},{"osd":2,"uuid":"d2d2c641-392a-444f-81bc-84eef6340f30","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":18,"last_clean_end":110,"up_from":114,"up_thru":299,"down_at":111,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:6818","nonce":1439643284},{"type":"v1","addr":"172.21.15.89:6819","nonce":1439643284}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:6820","nonce":1439643284},{"type":"v1","addr":"172.21.15.89:6821","nonce":1439643284}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:6824","nonce":1439643284},{"type":"v1","addr":"172.21.15.89:6825","nonce":1439643284}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:6822","nonce":1439643284},{"type":"v1","addr":"172.21.15.89:6823","nonce":1439643284}]},"public_addr":"172.21.15.89:6819/1439643284","cluster_addr":"172.21.15.89:6821/1439643284","heartbeat_back_addr":"172.21.15.89:6825/1439643284","heartbeat_front_addr":"172.21.15.89:6823/1439643284","state":["exists","up"]},{"osd":3,"uuid":"b38408eb-3acf-4a41-a314-72e4cd37b1ef","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":25,"last_clean_end":115,"up_from":119,"up_thru":299,"down_at":116,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:6826","nonce":1558463544},{"type":"v1","addr":"172.21.15.89:6827","nonce":1558463544}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:6828","nonce":1558463544},{"type":"v1","addr":"172.21.15.89:6829","nonce":1558463544}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:6832","nonce":1558463544},{"type":"v1","addr":"172.21.15.89:6833","nonce":1558463544}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:6830","nonce":1558463544},{"type":"v1","addr":"172.21.15.89:6831","nonce":1558463544}]},"public_addr":"172.21.15.89:6827/1558463544","cluster_addr":"172.21.15.89:6829/1558463544","heartbeat_back_addr":"172.21.15.89:6833/1558463544","heartbeat_front_addr":"172.21.15.89:6831/1558463544","state":["exists","up"]},{"osd":4,"uuid":"a94be785-9f54-4ab8-962a-9cf974a5b5e3","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":124,"last_clean_end":194,"up_from":250,"up_thru":299,"down_at":195,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.164:6800","nonce":3945899737},{"type":"v1","addr":"172.21.15.164:6801","nonce":3945899737}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.164:6802","nonce":3945899737},{"type":"v1","addr":"172.21.15.164:6803","nonce":3945899737}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.164:6806","nonce":3945899737},{"type":"v1","addr":"172.21.15.164:6807","nonce":3945899737}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.164:6804","nonce":3945899737},{"type":"v1","addr":"172.21.15.164:6805","nonce":3945899737}]},"public_addr":"172.21.15.164:6801/3945899737","cluster_addr":"172.21.15.164:6803/3945899737","heartbeat_back_addr":"172.21.15.164:6807/3945899737","heartbeat_front_addr":"172.21.15.164:6805/3945899737","state":["exists","up"]},{"osd":5,"uuid":"b8a5e8a1-23cc-442b-90aa-19bd0036dd3c","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":36,"up_thru":317,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.164:6808","nonce":3123006405},{"type":"v1","addr":"172.21.15.164:6809","nonce":3123006405}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.164:6810","nonce":3123006405},{"type":"v1","addr":"172.21.15.164:6811","nonce":3123006405}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.164:6814","nonce":3123006405},{"type":"v1","addr":"172.21.15.164:6815","nonce":3123006405}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.164:6812","nonce":3123006405},{"type":"v1","addr":"172.21.15.164:6813","nonce":3123006405}]},"public_addr":"172.21.15.164:6809/3123006405","cluster_addr":"172.21.15.164:6811/3123006405","heartbeat_back_addr":"172.21.15.164:6815/3123006405","heartbeat_front_addr":"172.21.15.164:6813/3123006405","state":["exists","up"]},{"osd":6,"uuid":"5c5edd3b-d92c-4f46-94ff-2d5828b2e732","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":42,"up_thru":299,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.164:6816","nonce":712403290},{"type":"v1","addr":"172.21.15.164:6817","nonce":712403290}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.164:6818","nonce":712403290},{"type":"v1","addr":"172.21.15.164:6819","nonce":712403290}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.164:6822","nonce":712403290},{"type":"v1","addr":"172.21.15.164:6823","nonce":712403290}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.164:6820","nonce":712403290},{"type":"v1","addr":"172.21.15.164:6821","nonce":712403290}]},"public_addr":"172.21.15.164:6817/712403290","cluster_addr":"172.21.15.164:6819/712403290","heartbeat_back_addr":"172.21.15.164:6823/712403290","heartbeat_front_addr":"172.21.15.164:6821/712403290","state":["exists","up"]},{"osd":7,"uuid":"bdff9290-e6c4-407e-9113-eb39696c0e27","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":48,"up_thru":299,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.164:6824","nonce":711997991},{"type":"v1","addr":"172.21.15.164:6825","nonce":711997991}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.164:6826","nonce":711997991},{"type":"v1","addr":"172.21.15.164:6827","nonce":711997991}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.164:6830","nonce":711997991},{"type":"v1","addr":"172.21.15.164:6831","nonce":711997991}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.164:6828","nonce":711997991},{"type":"v1","addr":"172.21.15.164:6829","nonce":711997991}]},"public_addr":"172.21.15.164:6825/711997991","cluster_addr":"172.21.15.164:6827/711997991","heartbeat_back_addr":"172.21.15.164:6831/711997991","heartbeat_front_addr":"172.21.15.164:6829/711997991","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-06-03T19:25:52.155714+0000","dead_epoch":82},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-06-03T19:26:38.554458+0000","dead_epoch":99},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-06-03T19:27:25.383237+0000","dead_epoch":110},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-06-03T19:28:13.272585+0000","dead_epoch":115},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-06-03T19:28:56.685710+0000","dead_epoch":194},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-06-03T19:29:41.572533+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-06-03T19:30:26.631983+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-06-03T19:31:12.018414+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[{"pgid":"4.17","mappings":[{"from":1,"to":2}]},{"pgid":"4.1b","mappings":[{"from":1,"to":2}]},{"pgid":"4.1d","mappings":[{"from":3,"to":5}]},{"pgid":"4.1f","mappings":[{"from":1,"to":2}]}],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.89:0/3543817158":"2023-06-04T19:41:21.936058+0000","172.21.15.164:0/2793695939":"2023-06-04T19:40:01.102939+0000","172.21.15.89:0/3944267183":"2023-06-04T19:23:55.626873+0000","172.21.15.89:0/2890034071":"2023-06-04T19:41:21.936058+0000","172.21.15.89:6801/3918707857":"2023-06-04T19:41:21.936058+0000","172.21.15.89:6800/3918707857":"2023-06-04T19:41:21.936058+0000","172.21.15.89:6800/2256305550":"2023-06-04T19:31:48.303240+0000","172.21.15.89:0/2023636358":"2023-06-04T19:37:38.185918+0000","172.21.15.89:0/470853232":"2023-06-04T19:31:48.303240+0000","172.21.15.164:6835/9946422":"2023-06-04T19:40:01.102939+0000","172.21.15.89:0/811967393":"2023-06-04T19:23:23.903333+0000","172.21.15.89:0/3044409358":"2023-06-04T19:41:21.936058+0000","172.21.15.164:0/3249546020":"2023-06-04T19:40:01.102939+0000","172.21.15.89:0/1198702890":"2023-06-04T19:23:23.903333+0000","172.21.15.164:0/1307416622":"2023-06-04T19:40:01.102939+0000","172.21.15.89:6801/2009133143":"2023-06-04T19:37:38.185918+0000","172.21.15.164:6834/9946422":"2023-06-04T19:40:01.102939+0000","172.21.15.89:6800/1487566136":"2023-06-04T19:23:23.903333+0000","172.21.15.89:6801/1487566136":"2023-06-04T19:23:23.903333+0000","172.21.15.89:0/274274920":"2023-06-04T19:23:55.626873+0000","172.21.15.89:0/2392321634":"2023-06-04T19:31:48.303240+0000","172.21.15.89:0/101502033":"2023-06-04T19:41:21.936058+0000","172.21.15.164:0/3011990159":"2023-06-04T19:40:01.102939+0000","172.21.15.89:6801/2256305550":"2023-06-04T19:31:48.303240+0000","172.21.15.89:6800/4116589411":"2023-06-04T19:23:55.626873+0000","172.21.15.89:0/1591228500":"2023-06-04T19:37:38.185918+0000","172.21.15.89:0/949193420":"2023-06-04T19:41:21.936058+0000","172.21.15.89:6800/2009133143":"2023-06-04T19:37:38.185918+0000","172.21.15.89:0/3513444952":"2023-06-04T19:41:21.936058+0000","172.21.15.89:6801/4116589411":"2023-06-04T19:23:55.626873+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[{"pool":15,"snaps":[{"begin":2,"length":1}]}],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2023-06-03T20:36:45.529 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:36:45 smithi089 bash[75068]: audit 1685824604.885826 mon.c (mon.2) 725 : audit 0 from='client.? 172.21.15.89:0/1627217383' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-06-03T20:36:45.529 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:36:45 smithi089 bash[78452]: audit 1685824604.885826 mon.c (mon.2) 725 : audit 0 from='client.? 172.21.15.89:0/1627217383' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-06-03T20:36:45.585 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:36:45 smithi164 bash[59462]: audit 1685824604.885826 mon.c (mon.2) 725 : audit 0 from='client.? 172.21.15.89:0/1627217383' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-06-03T20:36:45.817 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph tell osd.0 flush_pg_stats 2023-06-03T20:36:45.818 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph tell osd.1 flush_pg_stats 2023-06-03T20:36:45.819 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph tell osd.2 flush_pg_stats 2023-06-03T20:36:45.819 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph tell osd.3 flush_pg_stats 2023-06-03T20:36:45.819 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph tell osd.4 flush_pg_stats 2023-06-03T20:36:45.820 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph tell osd.5 flush_pg_stats 2023-06-03T20:36:45.821 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph tell osd.6 flush_pg_stats 2023-06-03T20:36:45.821 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph tell osd.7 flush_pg_stats 2023-06-03T20:36:46.529 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:36:46 smithi089 bash[75068]: cluster 1685824605.32582 mgr.y (mgr.34104) 1832 : cluster 0 pgmap v1916: 73 pgs: 73 active+clean; 582 KiB data, 5.2 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:36:46.529 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 20:36:46 smithi089 bash[71163]: ::ffff:172.21.15.164 - - [03/Jun/2023:20:36:46] "GET /metrics HTTP/1.1" 200 226927 "" "Prometheus/2.33.4" 2023-06-03T20:36:46.530 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:36:46 smithi089 bash[78452]: cluster 1685824605.32582 mgr.y (mgr.34104) 1832 : cluster 0 pgmap v1916: 73 pgs: 73 active+clean; 582 KiB data, 5.2 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:36:46.585 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:36:46 smithi164 bash[59462]: cluster 1685824605.32582 mgr.y (mgr.34104) 1832 : cluster 0 pgmap v1916: 73 pgs: 73 active+clean; 582 KiB data, 5.2 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:36:48.779 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:36:48 smithi089 bash[75068]: cluster 1685824607.3265717 mgr.y (mgr.34104) 1833 : cluster 0 pgmap v1917: 73 pgs: 73 active+clean; 582 KiB data, 5.2 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:36:48.779 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:36:48 smithi089 bash[75068]: audit 1685824607.7565873 mon.c (mon.2) 726 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-03T20:36:48.779 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:36:48 smithi089 bash[75068]: audit 1685824608.1372745 mon.a (mon.0) 1765 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:36:48.779 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:36:48 smithi089 bash[75068]: audit 1685824608.1430786 mon.a (mon.0) 1766 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:36:48.779 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:36:48 smithi089 bash[75068]: audit 1685824608.1498451 mon.a (mon.0) 1767 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:36:48.780 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:36:48 smithi089 bash[75068]: audit 1685824608.162107 mon.a (mon.0) 1768 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:36:48.780 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:36:48 smithi089 bash[78452]: cluster 1685824607.3265717 mgr.y (mgr.34104) 1833 : cluster 0 pgmap v1917: 73 pgs: 73 active+clean; 582 KiB data, 5.2 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:36:48.780 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:36:48 smithi089 bash[78452]: audit 1685824607.7565873 mon.c (mon.2) 726 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-03T20:36:48.780 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:36:48 smithi089 bash[78452]: audit 1685824608.1372745 mon.a (mon.0) 1765 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:36:48.781 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:36:48 smithi089 bash[78452]: audit 1685824608.1430786 mon.a (mon.0) 1766 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:36:48.781 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:36:48 smithi089 bash[78452]: audit 1685824608.1498451 mon.a (mon.0) 1767 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:36:48.781 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:36:48 smithi089 bash[78452]: audit 1685824608.162107 mon.a (mon.0) 1768 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:36:48.836 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:36:48 smithi164 bash[59462]: cluster 1685824607.3265717 mgr.y (mgr.34104) 1833 : cluster 0 pgmap v1917: 73 pgs: 73 active+clean; 582 KiB data, 5.2 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:36:48.836 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:36:48 smithi164 bash[59462]: audit 1685824607.7565873 mon.c (mon.2) 726 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-03T20:36:48.836 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:36:48 smithi164 bash[59462]: audit 1685824608.1372745 mon.a (mon.0) 1765 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:36:48.836 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:36:48 smithi164 bash[59462]: audit 1685824608.1430786 mon.a (mon.0) 1766 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:36:48.836 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:36:48 smithi164 bash[59462]: audit 1685824608.1498451 mon.a (mon.0) 1767 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:36:48.837 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:36:48 smithi164 bash[59462]: audit 1685824608.162107 mon.a (mon.0) 1768 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:36:50.742 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/mon.a/config 2023-06-03T20:36:50.743 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/mon.a/config 2023-06-03T20:36:50.744 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/mon.a/config 2023-06-03T20:36:50.748 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/mon.a/config 2023-06-03T20:36:50.749 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/mon.a/config 2023-06-03T20:36:50.749 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/mon.a/config 2023-06-03T20:36:50.750 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/mon.a/config 2023-06-03T20:36:50.751 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/mon.a/config 2023-06-03T20:36:50.780 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:36:50 smithi089 bash[75068]: cluster 1685824609.3275044 mgr.y (mgr.34104) 1834 : cluster 0 pgmap v1918: 73 pgs: 73 active+clean; 582 KiB data, 5.2 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:36:50.781 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:36:50 smithi089 bash[78452]: cluster 1685824609.3275044 mgr.y (mgr.34104) 1834 : cluster 0 pgmap v1918: 73 pgs: 73 active+clean; 582 KiB data, 5.2 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:36:50.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:36:50 smithi164 bash[59462]: cluster 1685824609.3275044 mgr.y (mgr.34104) 1834 : cluster 0 pgmap v1918: 73 pgs: 73 active+clean; 582 KiB data, 5.2 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:36:52.778 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:36:52 smithi089 bash[75068]: cluster 1685824611.3279917 mgr.y (mgr.34104) 1835 : cluster 0 pgmap v1919: 73 pgs: 73 active+clean; 582 KiB data, 5.2 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:36:52.779 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:36:52 smithi089 bash[75068]: audit 1685824612.1682405 mon.c (mon.2) 727 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T20:36:52.779 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:36:52 smithi089 bash[78452]: cluster 1685824611.3279917 mgr.y (mgr.34104) 1835 : cluster 0 pgmap v1919: 73 pgs: 73 active+clean; 582 KiB data, 5.2 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:36:52.779 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:36:52 smithi089 bash[78452]: audit 1685824612.1682405 mon.c (mon.2) 727 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T20:36:52.836 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:36:52 smithi164 bash[59462]: cluster 1685824611.3279917 mgr.y (mgr.34104) 1835 : cluster 0 pgmap v1919: 73 pgs: 73 active+clean; 582 KiB data, 5.2 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:36:52.836 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:36:52 smithi164 bash[59462]: audit 1685824612.1682405 mon.c (mon.2) 727 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T20:36:54.226 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:36:54 smithi089 bash[75068]: cluster 1685824613.3287528 mgr.y (mgr.34104) 1836 : cluster 0 pgmap v1920: 73 pgs: 73 active+clean; 582 KiB data, 5.2 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:36:54.226 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:36:54 smithi089 bash[75068]: audit 1685824613.89864 mon.a (mon.0) 1769 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:36:54.226 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:36:54 smithi089 bash[75068]: audit 1685824613.9102445 mon.a (mon.0) 1770 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:36:54.529 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:36:54 smithi089 bash[78452]: cluster 1685824613.3287528 mgr.y (mgr.34104) 1836 : cluster 0 pgmap v1920: 73 pgs: 73 active+clean; 582 KiB data, 5.2 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:36:54.529 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:36:54 smithi089 bash[78452]: audit 1685824613.89864 mon.a (mon.0) 1769 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:36:54.529 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:36:54 smithi089 bash[78452]: audit 1685824613.9102445 mon.a (mon.0) 1770 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:36:54.585 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:36:54 smithi164 bash[59462]: cluster 1685824613.3287528 mgr.y (mgr.34104) 1836 : cluster 0 pgmap v1920: 73 pgs: 73 active+clean; 582 KiB data, 5.2 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:36:54.585 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:36:54 smithi164 bash[59462]: audit 1685824613.89864 mon.a (mon.0) 1769 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:36:54.586 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:36:54 smithi164 bash[59462]: audit 1685824613.9102445 mon.a (mon.0) 1770 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:36:55.779 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:36:55 smithi089 bash[75068]: audit 1685824614.5688872 mon.a (mon.0) 1771 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:36:55.779 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:36:55 smithi089 bash[75068]: audit 1685824614.5758703 mon.a (mon.0) 1772 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:36:55.779 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:36:55 smithi089 bash[75068]: audit 1685824615.0449731 mon.c (mon.2) 728 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-03T20:36:55.779 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:36:55 smithi089 bash[75068]: audit 1685824615.0463881 mon.c (mon.2) 729 : audit 1 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-03T20:36:55.780 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:36:55 smithi089 bash[75068]: audit 1685824615.0530784 mon.a (mon.0) 1773 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:36:55.780 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:36:55 smithi089 bash[75068]: cluster 1685824615.3293412 mgr.y (mgr.34104) 1837 : cluster 0 pgmap v1921: 73 pgs: 73 active+clean; 582 KiB data, 5.2 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:36:55.780 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:36:55 smithi089 bash[78452]: audit 1685824614.5688872 mon.a (mon.0) 1771 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:36:55.780 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:36:55 smithi089 bash[78452]: audit 1685824614.5758703 mon.a (mon.0) 1772 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:36:55.780 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:36:55 smithi089 bash[78452]: audit 1685824615.0449731 mon.c (mon.2) 728 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-03T20:36:55.780 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:36:55 smithi089 bash[78452]: audit 1685824615.0463881 mon.c (mon.2) 729 : audit 1 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-03T20:36:55.781 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:36:55 smithi089 bash[78452]: audit 1685824615.0530784 mon.a (mon.0) 1773 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:36:55.781 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:36:55 smithi089 bash[78452]: cluster 1685824615.3293412 mgr.y (mgr.34104) 1837 : cluster 0 pgmap v1921: 73 pgs: 73 active+clean; 582 KiB data, 5.2 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:36:55.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:36:55 smithi164 bash[59462]: audit 1685824614.5688872 mon.a (mon.0) 1771 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:36:55.836 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:36:55 smithi164 bash[59462]: audit 1685824614.5758703 mon.a (mon.0) 1772 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:36:55.836 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:36:55 smithi164 bash[59462]: audit 1685824615.0449731 mon.c (mon.2) 728 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-03T20:36:55.836 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:36:55 smithi164 bash[59462]: audit 1685824615.0463881 mon.c (mon.2) 729 : audit 1 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-03T20:36:55.836 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:36:55 smithi164 bash[59462]: audit 1685824615.0530784 mon.a (mon.0) 1773 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:36:55.836 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:36:55 smithi164 bash[59462]: cluster 1685824615.3293412 mgr.y (mgr.34104) 1837 : cluster 0 pgmap v1921: 73 pgs: 73 active+clean; 582 KiB data, 5.2 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:36:56.360 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 20:36:56 smithi089 bash[71163]: ::ffff:172.21.15.164 - - [03/Jun/2023:20:36:56] "GET /metrics HTTP/1.1" 200 226928 "" "Prometheus/2.33.4" 2023-06-03T20:36:58.529 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:36:58 smithi089 bash[78452]: cluster 1685824617.3301 mgr.y (mgr.34104) 1838 : cluster 0 pgmap v1922: 73 pgs: 73 active+clean; 582 KiB data, 5.2 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:36:58.529 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:36:58 smithi089 bash[75068]: cluster 1685824617.3301 mgr.y (mgr.34104) 1838 : cluster 0 pgmap v1922: 73 pgs: 73 active+clean; 582 KiB data, 5.2 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:36:58.836 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:36:58 smithi164 bash[59462]: cluster 1685824617.3301 mgr.y (mgr.34104) 1838 : cluster 0 pgmap v1922: 73 pgs: 73 active+clean; 582 KiB data, 5.2 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:37:00.778 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:37:00 smithi089 bash[78452]: cluster 1685824619.3306599 mgr.y (mgr.34104) 1839 : cluster 0 pgmap v1923: 73 pgs: 73 active+clean; 582 KiB data, 5.2 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:37:00.779 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:37:00 smithi089 bash[75068]: cluster 1685824619.3306599 mgr.y (mgr.34104) 1839 : cluster 0 pgmap v1923: 73 pgs: 73 active+clean; 582 KiB data, 5.2 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:37:00.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:37:00 smithi164 bash[59462]: cluster 1685824619.3306599 mgr.y (mgr.34104) 1839 : cluster 0 pgmap v1923: 73 pgs: 73 active+clean; 582 KiB data, 5.2 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:37:02.779 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:37:02 smithi089 bash[78452]: cluster 1685824621.3311057 mgr.y (mgr.34104) 1840 : cluster 0 pgmap v1924: 73 pgs: 73 active+clean; 582 KiB data, 5.2 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:37:02.779 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:37:02 smithi089 bash[75068]: cluster 1685824621.3311057 mgr.y (mgr.34104) 1840 : cluster 0 pgmap v1924: 73 pgs: 73 active+clean; 582 KiB data, 5.2 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:37:02.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:37:02 smithi164 bash[59462]: cluster 1685824621.3311057 mgr.y (mgr.34104) 1840 : cluster 0 pgmap v1924: 73 pgs: 73 active+clean; 582 KiB data, 5.2 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:37:04.529 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:37:04 smithi089 bash[75068]: cluster 1685824623.3317413 mgr.y (mgr.34104) 1841 : cluster 0 pgmap v1925: 73 pgs: 73 active+clean; 582 KiB data, 5.2 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:37:04.529 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:37:04 smithi089 bash[78452]: cluster 1685824623.3317413 mgr.y (mgr.34104) 1841 : cluster 0 pgmap v1925: 73 pgs: 73 active+clean; 582 KiB data, 5.2 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:37:04.585 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:37:04 smithi164 bash[59462]: cluster 1685824623.3317413 mgr.y (mgr.34104) 1841 : cluster 0 pgmap v1925: 73 pgs: 73 active+clean; 582 KiB data, 5.2 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:37:06.528 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 20:37:06 smithi089 bash[71163]: ::ffff:172.21.15.164 - - [03/Jun/2023:20:37:06] "GET /metrics HTTP/1.1" 200 226928 "" "Prometheus/2.33.4" 2023-06-03T20:37:06.529 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:37:06 smithi089 bash[75068]: cluster 1685824625.3322968 mgr.y (mgr.34104) 1842 : cluster 0 pgmap v1926: 73 pgs: 73 active+clean; 582 KiB data, 5.2 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:37:06.529 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:37:06 smithi089 bash[78452]: cluster 1685824625.3322968 mgr.y (mgr.34104) 1842 : cluster 0 pgmap v1926: 73 pgs: 73 active+clean; 582 KiB data, 5.2 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:37:06.829 INFO:teuthology.orchestra.run.smithi089.stdout:180388627234 2023-06-03T20:37:06.830 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph osd last-stat-seq osd.6 2023-06-03T20:37:06.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:37:06 smithi164 bash[59462]: cluster 1685824625.3322968 mgr.y (mgr.34104) 1842 : cluster 0 pgmap v1926: 73 pgs: 73 active+clean; 582 KiB data, 5.2 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:37:07.779 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:37:07 smithi089 bash[75068]: audit 1685824627.1694205 mon.c (mon.2) 730 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T20:37:07.779 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:37:07 smithi089 bash[78452]: audit 1685824627.1694205 mon.c (mon.2) 730 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T20:37:07.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:37:07 smithi164 bash[59462]: audit 1685824627.1694205 mon.c (mon.2) 730 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T20:37:08.779 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:37:08 smithi089 bash[75068]: cluster 1685824627.3329961 mgr.y (mgr.34104) 1843 : cluster 0 pgmap v1927: 73 pgs: 73 active+clean; 582 KiB data, 5.2 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:37:08.779 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:37:08 smithi089 bash[78452]: cluster 1685824627.3329961 mgr.y (mgr.34104) 1843 : cluster 0 pgmap v1927: 73 pgs: 73 active+clean; 582 KiB data, 5.2 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:37:08.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:37:08 smithi164 bash[59462]: cluster 1685824627.3329961 mgr.y (mgr.34104) 1843 : cluster 0 pgmap v1927: 73 pgs: 73 active+clean; 582 KiB data, 5.2 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:37:09.067 INFO:teuthology.orchestra.run.smithi089.stdout:154618823468 2023-06-03T20:37:09.067 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph osd last-stat-seq osd.5 2023-06-03T20:37:10.529 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:37:10 smithi089 bash[78452]: cluster 1685824629.3335373 mgr.y (mgr.34104) 1844 : cluster 0 pgmap v1928: 73 pgs: 73 active+clean; 582 KiB data, 5.2 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:37:10.529 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:37:10 smithi089 bash[75068]: cluster 1685824629.3335373 mgr.y (mgr.34104) 1844 : cluster 0 pgmap v1928: 73 pgs: 73 active+clean; 582 KiB data, 5.2 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:37:10.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:37:10 smithi164 bash[59462]: cluster 1685824629.3335373 mgr.y (mgr.34104) 1844 : cluster 0 pgmap v1928: 73 pgs: 73 active+clean; 582 KiB data, 5.2 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:37:12.075 INFO:teuthology.orchestra.run.smithi089.stdout:442381632033 2023-06-03T20:37:12.075 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph osd last-stat-seq osd.1 2023-06-03T20:37:12.528 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:37:12 smithi089 bash[78452]: cluster 1685824631.3341556 mgr.y (mgr.34104) 1845 : cluster 0 pgmap v1929: 73 pgs: 73 active+clean; 582 KiB data, 5.2 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:37:12.529 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:37:12 smithi089 bash[75068]: cluster 1685824631.3341556 mgr.y (mgr.34104) 1845 : cluster 0 pgmap v1929: 73 pgs: 73 active+clean; 582 KiB data, 5.2 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:37:12.733 INFO:teuthology.orchestra.run.smithi089.stdout:1073741824166 2023-06-03T20:37:12.733 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph osd last-stat-seq osd.4 2023-06-03T20:37:12.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:37:12 smithi164 bash[59462]: cluster 1685824631.3341556 mgr.y (mgr.34104) 1845 : cluster 0 pgmap v1929: 73 pgs: 73 active+clean; 582 KiB data, 5.2 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:37:12.847 INFO:teuthology.orchestra.run.smithi089.stdout:489626272238 2023-06-03T20:37:12.848 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph osd last-stat-seq osd.2 2023-06-03T20:37:14.529 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:37:14 smithi089 bash[78452]: cluster 1685824633.3347619 mgr.y (mgr.34104) 1846 : cluster 0 pgmap v1930: 73 pgs: 73 active+clean; 582 KiB data, 5.2 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:37:14.529 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:37:14 smithi089 bash[75068]: cluster 1685824633.3347619 mgr.y (mgr.34104) 1846 : cluster 0 pgmap v1930: 73 pgs: 73 active+clean; 582 KiB data, 5.2 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:37:14.539 INFO:teuthology.orchestra.run.smithi089.stdout:206158431002 2023-06-03T20:37:14.539 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph osd last-stat-seq osd.7 2023-06-03T20:37:14.585 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:37:14 smithi164 bash[59462]: cluster 1685824633.3347619 mgr.y (mgr.34104) 1846 : cluster 0 pgmap v1930: 73 pgs: 73 active+clean; 582 KiB data, 5.2 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:37:14.712 INFO:teuthology.orchestra.run.smithi089.stdout:369367188060 2023-06-03T20:37:14.712 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph osd last-stat-seq osd.0 2023-06-03T20:37:14.731 INFO:teuthology.orchestra.run.smithi089.stdout:511101108710 2023-06-03T20:37:14.731 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph osd last-stat-seq osd.3 2023-06-03T20:37:16.529 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:37:16 smithi089 bash[75068]: cluster 1685824635.3354118 mgr.y (mgr.34104) 1847 : cluster 0 pgmap v1931: 73 pgs: 73 active+clean; 582 KiB data, 5.2 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:37:16.529 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 20:37:16 smithi089 bash[71163]: ::ffff:172.21.15.164 - - [03/Jun/2023:20:37:16] "GET /metrics HTTP/1.1" 200 226908 "" "Prometheus/2.33.4" 2023-06-03T20:37:16.529 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:37:16 smithi089 bash[78452]: cluster 1685824635.3354118 mgr.y (mgr.34104) 1847 : cluster 0 pgmap v1931: 73 pgs: 73 active+clean; 582 KiB data, 5.2 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:37:16.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:37:16 smithi164 bash[59462]: cluster 1685824635.3354118 mgr.y (mgr.34104) 1847 : cluster 0 pgmap v1931: 73 pgs: 73 active+clean; 582 KiB data, 5.2 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:37:16.866 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/mon.a/config 2023-06-03T20:37:16.869 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/mon.a/config 2023-06-03T20:37:16.870 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/mon.a/config 2023-06-03T20:37:16.870 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/mon.a/config 2023-06-03T20:37:16.874 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/mon.a/config 2023-06-03T20:37:18.779 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:37:18 smithi089 bash[75068]: cluster 1685824637.335884 mgr.y (mgr.34104) 1848 : cluster 0 pgmap v1932: 73 pgs: 73 active+clean; 582 KiB data, 5.2 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:37:18.779 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:37:18 smithi089 bash[78452]: cluster 1685824637.335884 mgr.y (mgr.34104) 1848 : cluster 0 pgmap v1932: 73 pgs: 73 active+clean; 582 KiB data, 5.2 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:37:18.836 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:37:18 smithi164 bash[59462]: cluster 1685824637.335884 mgr.y (mgr.34104) 1848 : cluster 0 pgmap v1932: 73 pgs: 73 active+clean; 582 KiB data, 5.2 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:37:18.907 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/mon.a/config 2023-06-03T20:37:18.907 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/mon.a/config 2023-06-03T20:37:18.909 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/mon.a/config 2023-06-03T20:37:20.779 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:37:20 smithi089 bash[78452]: cluster 1685824639.33677 mgr.y (mgr.34104) 1849 : cluster 0 pgmap v1933: 73 pgs: 73 active+clean; 582 KiB data, 5.2 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:37:20.779 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:37:20 smithi089 bash[75068]: cluster 1685824639.33677 mgr.y (mgr.34104) 1849 : cluster 0 pgmap v1933: 73 pgs: 73 active+clean; 582 KiB data, 5.2 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:37:20.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:37:20 smithi164 bash[59462]: cluster 1685824639.33677 mgr.y (mgr.34104) 1849 : cluster 0 pgmap v1933: 73 pgs: 73 active+clean; 582 KiB data, 5.2 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:37:22.830 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:37:22 smithi089 bash[75068]: cluster 1685824641.337499 mgr.y (mgr.34104) 1850 : cluster 0 pgmap v1934: 73 pgs: 73 active+clean; 582 KiB data, 5.2 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:37:22.830 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:37:22 smithi089 bash[75068]: audit 1685824642.1699188 mon.c (mon.2) 731 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T20:37:22.830 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:37:22 smithi089 bash[78452]: cluster 1685824641.337499 mgr.y (mgr.34104) 1850 : cluster 0 pgmap v1934: 73 pgs: 73 active+clean; 582 KiB data, 5.2 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:37:22.830 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:37:22 smithi089 bash[78452]: audit 1685824642.1699188 mon.c (mon.2) 731 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T20:37:22.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:37:22 smithi164 bash[59462]: cluster 1685824641.337499 mgr.y (mgr.34104) 1850 : cluster 0 pgmap v1934: 73 pgs: 73 active+clean; 582 KiB data, 5.2 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:37:22.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:37:22 smithi164 bash[59462]: audit 1685824642.1699188 mon.c (mon.2) 731 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T20:37:24.529 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:37:24 smithi089 bash[75068]: cluster 1685824643.3381724 mgr.y (mgr.34104) 1851 : cluster 0 pgmap v1935: 73 pgs: 73 active+clean; 582 KiB data, 5.2 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:37:24.529 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:37:24 smithi089 bash[78452]: cluster 1685824643.3381724 mgr.y (mgr.34104) 1851 : cluster 0 pgmap v1935: 73 pgs: 73 active+clean; 582 KiB data, 5.2 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:37:24.585 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:37:24 smithi164 bash[59462]: cluster 1685824643.3381724 mgr.y (mgr.34104) 1851 : cluster 0 pgmap v1935: 73 pgs: 73 active+clean; 582 KiB data, 5.2 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:37:26.596 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 20:37:26 smithi089 bash[71163]: ::ffff:172.21.15.164 - - [03/Jun/2023:20:37:26] "GET /metrics HTTP/1.1" 200 226906 "" "Prometheus/2.33.4" 2023-06-03T20:37:26.596 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:37:26 smithi089 bash[78452]: cluster 1685824645.338714 mgr.y (mgr.34104) 1852 : cluster 0 pgmap v1936: 73 pgs: 73 active+clean; 582 KiB data, 5.2 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:37:26.597 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:37:26 smithi089 bash[75068]: cluster 1685824645.338714 mgr.y (mgr.34104) 1852 : cluster 0 pgmap v1936: 73 pgs: 73 active+clean; 582 KiB data, 5.2 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:37:26.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:37:26 smithi164 bash[59462]: cluster 1685824645.338714 mgr.y (mgr.34104) 1852 : cluster 0 pgmap v1936: 73 pgs: 73 active+clean; 582 KiB data, 5.2 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:37:28.666 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:37:28 smithi089 bash[75068]: cluster 1685824647.3391833 mgr.y (mgr.34104) 1853 : cluster 0 pgmap v1937: 73 pgs: 73 active+clean; 582 KiB data, 5.2 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:37:28.667 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:37:28 smithi089 bash[78452]: cluster 1685824647.3391833 mgr.y (mgr.34104) 1853 : cluster 0 pgmap v1937: 73 pgs: 73 active+clean; 582 KiB data, 5.2 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:37:28.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:37:28 smithi164 bash[59462]: cluster 1685824647.3391833 mgr.y (mgr.34104) 1853 : cluster 0 pgmap v1937: 73 pgs: 73 active+clean; 582 KiB data, 5.2 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:37:30.758 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:37:30 smithi089 bash[75068]: cluster 1685824649.3398447 mgr.y (mgr.34104) 1854 : cluster 0 pgmap v1938: 73 pgs: 73 active+clean; 582 KiB data, 5.2 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:37:30.758 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:37:30 smithi089 bash[78452]: cluster 1685824649.3398447 mgr.y (mgr.34104) 1854 : cluster 0 pgmap v1938: 73 pgs: 73 active+clean; 582 KiB data, 5.2 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:37:30.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:37:30 smithi164 bash[59462]: cluster 1685824649.3398447 mgr.y (mgr.34104) 1854 : cluster 0 pgmap v1938: 73 pgs: 73 active+clean; 582 KiB data, 5.2 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:37:31.796 INFO:teuthology.orchestra.run.smithi089.stdout:1073741824170 2023-06-03T20:37:32.717 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:37:32 smithi089 bash[75068]: cluster 1685824651.3403902 mgr.y (mgr.34104) 1855 : cluster 0 pgmap v1939: 73 pgs: 73 active+clean; 582 KiB data, 5.2 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:37:32.718 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:37:32 smithi089 bash[75068]: audit 1685824651.7914479 mon.a (mon.0) 1774 : audit 0 from='client.? 172.21.15.89:0/641437508' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2023-06-03T20:37:32.718 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:37:32 smithi089 bash[78452]: cluster 1685824651.3403902 mgr.y (mgr.34104) 1855 : cluster 0 pgmap v1939: 73 pgs: 73 active+clean; 582 KiB data, 5.2 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:37:32.719 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:37:32 smithi089 bash[78452]: audit 1685824651.7914479 mon.a (mon.0) 1774 : audit 0 from='client.? 172.21.15.89:0/641437508' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2023-06-03T20:37:32.836 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:37:32 smithi164 bash[59462]: cluster 1685824651.3403902 mgr.y (mgr.34104) 1855 : cluster 0 pgmap v1939: 73 pgs: 73 active+clean; 582 KiB data, 5.2 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:37:32.836 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:37:32 smithi164 bash[59462]: audit 1685824651.7914479 mon.a (mon.0) 1774 : audit 0 from='client.? 172.21.15.89:0/641437508' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2023-06-03T20:37:33.123 INFO:teuthology.orchestra.run.smithi089.stdout:154618823473 2023-06-03T20:37:33.529 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:37:33 smithi089 bash[75068]: audit 1685824653.118769 mon.c (mon.2) 732 : audit 0 from='client.? 172.21.15.89:0/611443438' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2023-06-03T20:37:33.529 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:37:33 smithi089 bash[78452]: audit 1685824653.118769 mon.c (mon.2) 732 : audit 0 from='client.? 172.21.15.89:0/611443438' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2023-06-03T20:37:33.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:37:33 smithi164 bash[59462]: audit 1685824653.118769 mon.c (mon.2) 732 : audit 0 from='client.? 172.21.15.89:0/611443438' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2023-06-03T20:37:34.529 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:37:34 smithi089 bash[78452]: cluster 1685824653.3409462 mgr.y (mgr.34104) 1856 : cluster 0 pgmap v1940: 73 pgs: 73 active+clean; 582 KiB data, 5.2 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:37:34.529 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:37:34 smithi089 bash[75068]: cluster 1685824653.3409462 mgr.y (mgr.34104) 1856 : cluster 0 pgmap v1940: 73 pgs: 73 active+clean; 582 KiB data, 5.2 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:37:34.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:37:34 smithi164 bash[59462]: cluster 1685824653.3409462 mgr.y (mgr.34104) 1856 : cluster 0 pgmap v1940: 73 pgs: 73 active+clean; 582 KiB data, 5.2 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:37:35.159 INFO:tasks.cephadm.ceph_manager.ceph:need seq 1073741824166 got 1073741824170 for osd.4 2023-06-03T20:37:35.159 DEBUG:teuthology.parallel:result is None 2023-06-03T20:37:36.382 INFO:tasks.cephadm.ceph_manager.ceph:need seq 154618823468 got 154618823473 for osd.5 2023-06-03T20:37:36.382 DEBUG:teuthology.parallel:result is None 2023-06-03T20:37:36.529 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:37:36 smithi089 bash[75068]: cluster 1685824655.3416784 mgr.y (mgr.34104) 1857 : cluster 0 pgmap v1941: 73 pgs: 73 active+clean; 582 KiB data, 5.2 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:37:36.529 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:37:36 smithi089 bash[78452]: cluster 1685824655.3416784 mgr.y (mgr.34104) 1857 : cluster 0 pgmap v1941: 73 pgs: 73 active+clean; 582 KiB data, 5.2 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:37:36.530 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 20:37:36 smithi089 bash[71163]: ::ffff:172.21.15.164 - - [03/Jun/2023:20:37:36] "GET /metrics HTTP/1.1" 200 226906 "" "Prometheus/2.33.4" 2023-06-03T20:37:36.714 INFO:teuthology.orchestra.run.smithi089.stdout:442381632038 2023-06-03T20:37:36.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:37:36 smithi164 bash[59462]: cluster 1685824655.3416784 mgr.y (mgr.34104) 1857 : cluster 0 pgmap v1941: 73 pgs: 73 active+clean; 582 KiB data, 5.2 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:37:37.342 INFO:teuthology.orchestra.run.smithi089.stdout:206158431007 2023-06-03T20:37:37.612 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:37:37 smithi089 bash[75068]: audit 1685824656.7095232 mon.c (mon.2) 733 : audit 0 from='client.? 172.21.15.89:0/919155412' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2023-06-03T20:37:37.612 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:37:37 smithi089 bash[75068]: audit 1685824657.1700869 mon.c (mon.2) 734 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T20:37:37.612 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:37:37 smithi089 bash[75068]: audit 1685824657.3381915 mon.a (mon.0) 1775 : audit 0 from='client.? 172.21.15.89:0/769500344' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2023-06-03T20:37:37.613 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:37:37 smithi089 bash[78452]: audit 1685824656.7095232 mon.c (mon.2) 733 : audit 0 from='client.? 172.21.15.89:0/919155412' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2023-06-03T20:37:37.613 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:37:37 smithi089 bash[78452]: audit 1685824657.1700869 mon.c (mon.2) 734 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T20:37:37.613 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:37:37 smithi089 bash[78452]: audit 1685824657.3381915 mon.a (mon.0) 1775 : audit 0 from='client.? 172.21.15.89:0/769500344' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2023-06-03T20:37:37.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:37:37 smithi164 bash[59462]: audit 1685824656.7095232 mon.c (mon.2) 733 : audit 0 from='client.? 172.21.15.89:0/919155412' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2023-06-03T20:37:37.836 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:37:37 smithi164 bash[59462]: audit 1685824657.1700869 mon.c (mon.2) 734 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T20:37:37.836 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:37:37 smithi164 bash[59462]: audit 1685824657.3381915 mon.a (mon.0) 1775 : audit 0 from='client.? 172.21.15.89:0/769500344' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2023-06-03T20:37:37.993 INFO:teuthology.orchestra.run.smithi089.stdout:489626272243 2023-06-03T20:37:38.233 INFO:teuthology.orchestra.run.smithi089.stdout:180388627240 2023-06-03T20:37:38.406 INFO:teuthology.orchestra.run.smithi089.stdout:369367188065 2023-06-03T20:37:38.672 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:37:38 smithi089 bash[75068]: cluster 1685824657.3423147 mgr.y (mgr.34104) 1858 : cluster 0 pgmap v1942: 73 pgs: 73 active+clean; 582 KiB data, 5.2 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:37:38.672 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:37:38 smithi089 bash[75068]: audit 1685824657.989468 mon.a (mon.0) 1776 : audit 0 from='client.? 172.21.15.89:0/2923256646' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2023-06-03T20:37:38.672 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:37:38 smithi089 bash[75068]: audit 1685824658.2292092 mon.c (mon.2) 735 : audit 0 from='client.? 172.21.15.89:0/1644559351' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2023-06-03T20:37:38.673 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:37:38 smithi089 bash[75068]: audit 1685824658.401855 mon.c (mon.2) 736 : audit 0 from='client.? 172.21.15.89:0/481739464' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2023-06-03T20:37:38.673 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:37:38 smithi089 bash[78452]: cluster 1685824657.3423147 mgr.y (mgr.34104) 1858 : cluster 0 pgmap v1942: 73 pgs: 73 active+clean; 582 KiB data, 5.2 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:37:38.673 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:37:38 smithi089 bash[78452]: audit 1685824657.989468 mon.a (mon.0) 1776 : audit 0 from='client.? 172.21.15.89:0/2923256646' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2023-06-03T20:37:38.674 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:37:38 smithi089 bash[78452]: audit 1685824658.2292092 mon.c (mon.2) 735 : audit 0 from='client.? 172.21.15.89:0/1644559351' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2023-06-03T20:37:38.674 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:37:38 smithi089 bash[78452]: audit 1685824658.401855 mon.c (mon.2) 736 : audit 0 from='client.? 172.21.15.89:0/481739464' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2023-06-03T20:37:38.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:37:38 smithi164 bash[59462]: cluster 1685824657.3423147 mgr.y (mgr.34104) 1858 : cluster 0 pgmap v1942: 73 pgs: 73 active+clean; 582 KiB data, 5.2 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:37:38.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:37:38 smithi164 bash[59462]: audit 1685824657.989468 mon.a (mon.0) 1776 : audit 0 from='client.? 172.21.15.89:0/2923256646' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2023-06-03T20:37:38.836 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:37:38 smithi164 bash[59462]: audit 1685824658.2292092 mon.c (mon.2) 735 : audit 0 from='client.? 172.21.15.89:0/1644559351' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2023-06-03T20:37:38.836 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:37:38 smithi164 bash[59462]: audit 1685824658.401855 mon.c (mon.2) 736 : audit 0 from='client.? 172.21.15.89:0/481739464' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2023-06-03T20:37:39.185 INFO:teuthology.orchestra.run.smithi089.stdout:511101108715 2023-06-03T20:37:39.779 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:37:39 smithi089 bash[75068]: audit 1685824659.180789 mon.a (mon.0) 1777 : audit 0 from='client.? 172.21.15.89:0/3849298739' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2023-06-03T20:37:39.779 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:37:39 smithi089 bash[78452]: audit 1685824659.180789 mon.a (mon.0) 1777 : audit 0 from='client.? 172.21.15.89:0/3849298739' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2023-06-03T20:37:39.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:37:39 smithi164 bash[59462]: audit 1685824659.180789 mon.a (mon.0) 1777 : audit 0 from='client.? 172.21.15.89:0/3849298739' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2023-06-03T20:37:40.423 INFO:tasks.cephadm.ceph_manager.ceph:need seq 442381632033 got 442381632038 for osd.1 2023-06-03T20:37:40.424 DEBUG:teuthology.parallel:result is None 2023-06-03T20:37:40.779 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:37:40 smithi089 bash[78452]: cluster 1685824659.3430817 mgr.y (mgr.34104) 1859 : cluster 0 pgmap v1943: 73 pgs: 73 active+clean; 582 KiB data, 5.2 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:37:40.779 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:37:40 smithi089 bash[75068]: cluster 1685824659.3430817 mgr.y (mgr.34104) 1859 : cluster 0 pgmap v1943: 73 pgs: 73 active+clean; 582 KiB data, 5.2 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:37:40.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:37:40 smithi164 bash[59462]: cluster 1685824659.3430817 mgr.y (mgr.34104) 1859 : cluster 0 pgmap v1943: 73 pgs: 73 active+clean; 582 KiB data, 5.2 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:37:41.369 INFO:tasks.cephadm.ceph_manager.ceph:need seq 206158431002 got 206158431007 for osd.7 2023-06-03T20:37:41.370 DEBUG:teuthology.parallel:result is None 2023-06-03T20:37:41.652 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:37:41 smithi089 bash[78452]: cluster 1685824661.3435194 mgr.y (mgr.34104) 1860 : cluster 0 pgmap v1944: 73 pgs: 73 active+clean; 582 KiB data, 5.2 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:37:41.653 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:37:41 smithi089 bash[75068]: cluster 1685824661.3435194 mgr.y (mgr.34104) 1860 : cluster 0 pgmap v1944: 73 pgs: 73 active+clean; 582 KiB data, 5.2 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:37:41.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:37:41 smithi164 bash[59462]: cluster 1685824661.3435194 mgr.y (mgr.34104) 1860 : cluster 0 pgmap v1944: 73 pgs: 73 active+clean; 582 KiB data, 5.2 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:37:42.237 INFO:tasks.cephadm.ceph_manager.ceph:need seq 489626272238 got 489626272243 for osd.2 2023-06-03T20:37:42.238 DEBUG:teuthology.parallel:result is None 2023-06-03T20:37:43.209 INFO:tasks.cephadm.ceph_manager.ceph:need seq 369367188060 got 369367188065 for osd.0 2023-06-03T20:37:43.209 DEBUG:teuthology.parallel:result is None 2023-06-03T20:37:43.342 INFO:tasks.cephadm.ceph_manager.ceph:need seq 180388627234 got 180388627240 for osd.6 2023-06-03T20:37:43.342 DEBUG:teuthology.parallel:result is None 2023-06-03T20:37:43.352 INFO:tasks.cephadm.ceph_manager.ceph:need seq 511101108710 got 511101108715 for osd.3 2023-06-03T20:37:43.353 DEBUG:teuthology.parallel:result is None 2023-06-03T20:37:43.353 INFO:tasks.cephadm.ceph_manager.ceph:waiting for recovery to complete 2023-06-03T20:37:43.354 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph pg dump --format=json 2023-06-03T20:37:44.529 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:37:44 smithi089 bash[78452]: cluster 1685824663.3441806 mgr.y (mgr.34104) 1861 : cluster 0 pgmap v1945: 73 pgs: 73 active+clean; 582 KiB data, 5.2 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:37:44.529 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:37:44 smithi089 bash[75068]: cluster 1685824663.3441806 mgr.y (mgr.34104) 1861 : cluster 0 pgmap v1945: 73 pgs: 73 active+clean; 582 KiB data, 5.2 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:37:44.585 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:37:44 smithi164 bash[59462]: cluster 1685824663.3441806 mgr.y (mgr.34104) 1861 : cluster 0 pgmap v1945: 73 pgs: 73 active+clean; 582 KiB data, 5.2 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:37:46.529 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 20:37:46 smithi089 bash[71163]: ::ffff:172.21.15.164 - - [03/Jun/2023:20:37:46] "GET /metrics HTTP/1.1" 200 226904 "" "Prometheus/2.33.4" 2023-06-03T20:37:46.529 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:37:46 smithi089 bash[78452]: cluster 1685824665.3450205 mgr.y (mgr.34104) 1862 : cluster 0 pgmap v1946: 73 pgs: 73 active+clean; 582 KiB data, 5.2 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:37:46.529 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:37:46 smithi089 bash[75068]: cluster 1685824665.3450205 mgr.y (mgr.34104) 1862 : cluster 0 pgmap v1946: 73 pgs: 73 active+clean; 582 KiB data, 5.2 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:37:46.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:37:46 smithi164 bash[59462]: cluster 1685824665.3450205 mgr.y (mgr.34104) 1862 : cluster 0 pgmap v1946: 73 pgs: 73 active+clean; 582 KiB data, 5.2 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:37:48.217 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/mon.a/config 2023-06-03T20:37:48.529 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:37:48 smithi089 bash[75068]: cluster 1685824667.3458767 mgr.y (mgr.34104) 1863 : cluster 0 pgmap v1947: 73 pgs: 73 active+clean; 582 KiB data, 5.2 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:37:48.529 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:37:48 smithi089 bash[78452]: cluster 1685824667.3458767 mgr.y (mgr.34104) 1863 : cluster 0 pgmap v1947: 73 pgs: 73 active+clean; 582 KiB data, 5.2 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:37:48.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:37:48 smithi164 bash[59462]: cluster 1685824667.3458767 mgr.y (mgr.34104) 1863 : cluster 0 pgmap v1947: 73 pgs: 73 active+clean; 582 KiB data, 5.2 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:37:50.779 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:37:50 smithi089 bash[75068]: cluster 1685824669.3468063 mgr.y (mgr.34104) 1864 : cluster 0 pgmap v1948: 73 pgs: 73 active+clean; 582 KiB data, 5.2 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:37:50.779 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:37:50 smithi089 bash[78452]: cluster 1685824669.3468063 mgr.y (mgr.34104) 1864 : cluster 0 pgmap v1948: 73 pgs: 73 active+clean; 582 KiB data, 5.2 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:37:50.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:37:50 smithi164 bash[59462]: cluster 1685824669.3468063 mgr.y (mgr.34104) 1864 : cluster 0 pgmap v1948: 73 pgs: 73 active+clean; 582 KiB data, 5.2 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:37:52.474 INFO:teuthology.orchestra.run.smithi089.stdout: 2023-06-03T20:37:52.477 INFO:teuthology.orchestra.run.smithi089.stderr:dumped all 2023-06-03T20:37:52.779 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:37:52 smithi089 bash[75068]: cluster 1685824671.3475406 mgr.y (mgr.34104) 1865 : cluster 0 pgmap v1949: 73 pgs: 73 active+clean; 582 KiB data, 5.2 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:37:52.779 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:37:52 smithi089 bash[75068]: audit 1685824672.1699395 mon.c (mon.2) 737 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T20:37:52.779 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:37:52 smithi089 bash[78452]: cluster 1685824671.3475406 mgr.y (mgr.34104) 1865 : cluster 0 pgmap v1949: 73 pgs: 73 active+clean; 582 KiB data, 5.2 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:37:52.780 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:37:52 smithi089 bash[78452]: audit 1685824672.1699395 mon.c (mon.2) 737 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T20:37:52.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:37:52 smithi164 bash[59462]: cluster 1685824671.3475406 mgr.y (mgr.34104) 1865 : cluster 0 pgmap v1949: 73 pgs: 73 active+clean; 582 KiB data, 5.2 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:37:52.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:37:52 smithi164 bash[59462]: audit 1685824672.1699395 mon.c (mon.2) 737 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-06-03T20:37:53.475 INFO:teuthology.orchestra.run.smithi089.stdout:{"pg_ready":true,"pg_map":{"version":1949,"stamp":"2023-06-03T20:37:51.347106+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":595854,"num_objects":24,"num_object_clones":0,"num_object_copies":72,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":24,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":261,"num_write_kb":2446,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":27,"num_bytes_recovered":1775960,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":195,"ondisk_log_size":195,"up":219,"acting":219,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":219,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":5400236,"kb_used_data":28524,"kb_used_omap":30,"kb_used_meta":5371617,"kb_avail":744560980,"statfs":{"total":767960285184,"available":762430443520,"internally_reserved":0,"allocated":29208576,"data_stored":20830601,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":31502,"internal_metadata":5500536050},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.004500"},"pg_stats":[{"pgid":"3.1a","version":"60'1","reported_seq":253,"reported_epoch":323,"state":"active+clean","last_fresh":"2023-06-03T20:36:15.510110+0000","last_change":"2023-06-03T20:23:28.242261+0000","last_active":"2023-06-03T20:36:15.510110+0000","last_peered":"2023-06-03T20:36:15.510110+0000","last_clean":"2023-06-03T20:36:15.510110+0000","last_became_active":"2023-06-03T20:23:28.240045+0000","last_became_peered":"2023-06-03T20:23:28.240045+0000","last_unstale":"2023-06-03T20:36:15.510110+0000","last_undegraded":"2023-06-03T20:36:15.510110+0000","last_fullsized":"2023-06-03T20:36:15.510110+0000","mapping_epoch":251,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":252,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T05:54:33.215385+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":60,"reported_epoch":323,"state":"active+clean","last_fresh":"2023-06-03T20:36:16.311167+0000","last_change":"2023-06-03T20:35:26.203516+0000","last_active":"2023-06-03T20:36:16.311167+0000","last_peered":"2023-06-03T20:36:16.311167+0000","last_clean":"2023-06-03T20:36:16.311167+0000","last_became_active":"2023-06-03T20:35:26.203123+0000","last_became_peered":"2023-06-03T20:35:26.203123+0000","last_unstale":"2023-06-03T20:36:16.311167+0000","last_undegraded":"2023-06-03T20:36:16.311167+0000","last_fullsized":"2023-06-03T20:36:16.311167+0000","mapping_epoch":317,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,4],"acting":[5,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":114,"reported_epoch":323,"state":"active+clean","last_fresh":"2023-06-03T20:36:16.310781+0000","last_change":"2023-06-03T20:23:28.253821+0000","last_active":"2023-06-03T20:36:16.310781+0000","last_peered":"2023-06-03T20:36:16.310781+0000","last_clean":"2023-06-03T20:36:16.310781+0000","last_became_active":"2023-06-03T20:23:28.253448+0000","last_became_peered":"2023-06-03T20:23:28.253448+0000","last_unstale":"2023-06-03T20:36:16.310781+0000","last_undegraded":"2023-06-03T20:36:16.310781+0000","last_fullsized":"2023-06-03T20:36:16.310781+0000","mapping_epoch":251,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":252,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T23:40:02.264778+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":298,"reported_epoch":324,"state":"active+clean","last_fresh":"2023-06-03T20:36:25.444382+0000","last_change":"2023-06-03T19:56:54.507653+0000","last_active":"2023-06-03T20:36:25.444382+0000","last_peered":"2023-06-03T20:36:25.444382+0000","last_clean":"2023-06-03T20:36:25.444382+0000","last_became_active":"2023-06-03T19:56:54.507350+0000","last_became_peered":"2023-06-03T19:56:54.507350+0000","last_unstale":"2023-06-03T20:36:25.444382+0000","last_undegraded":"2023-06-03T20:36:25.444382+0000","last_fullsized":"2023-06-03T20:36:25.444382+0000","mapping_epoch":119,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T22:33:20.065844+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":337,"reported_epoch":323,"state":"active+clean","last_fresh":"2023-06-03T20:36:16.309990+0000","last_change":"2023-06-03T20:21:01.256894+0000","last_active":"2023-06-03T20:36:16.309990+0000","last_peered":"2023-06-03T20:36:16.309990+0000","last_clean":"2023-06-03T20:36:16.309990+0000","last_became_active":"2023-06-03T20:21:01.255903+0000","last_became_peered":"2023-06-03T20:21:01.255903+0000","last_unstale":"2023-06-03T20:36:16.309990+0000","last_undegraded":"2023-06-03T20:36:16.309990+0000","last_fullsized":"2023-06-03T20:36:16.309990+0000","mapping_epoch":235,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":236,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T20:35:27.810860+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":88,"reported_epoch":323,"state":"active+clean","last_fresh":"2023-06-03T20:36:16.309457+0000","last_change":"2023-06-03T20:28:24.269355+0000","last_active":"2023-06-03T20:36:16.309457+0000","last_peered":"2023-06-03T20:36:16.309457+0000","last_clean":"2023-06-03T20:36:16.309457+0000","last_became_active":"2023-06-03T20:28:24.268859+0000","last_became_peered":"2023-06-03T20:28:24.268859+0000","last_unstale":"2023-06-03T20:36:16.309457+0000","last_undegraded":"2023-06-03T20:36:16.309457+0000","last_fullsized":"2023-06-03T20:36:16.309457+0000","mapping_epoch":284,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":285,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,2],"acting":[6,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":292,"reported_epoch":323,"state":"active+clean","last_fresh":"2023-06-03T20:36:15.199336+0000","last_change":"2023-06-03T19:56:54.509490+0000","last_active":"2023-06-03T20:36:15.199336+0000","last_peered":"2023-06-03T20:36:15.199336+0000","last_clean":"2023-06-03T20:36:15.199336+0000","last_became_active":"2023-06-03T19:56:54.509145+0000","last_became_peered":"2023-06-03T19:56:54.509145+0000","last_unstale":"2023-06-03T20:36:15.199336+0000","last_undegraded":"2023-06-03T20:36:15.199336+0000","last_fullsized":"2023-06-03T20:36:15.199336+0000","mapping_epoch":119,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T07:12:03.964110+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":358,"reported_epoch":324,"state":"active+clean","last_fresh":"2023-06-03T20:36:26.545916+0000","last_change":"2023-06-03T20:28:51.233277+0000","last_active":"2023-06-03T20:36:26.545916+0000","last_peered":"2023-06-03T20:36:26.545916+0000","last_clean":"2023-06-03T20:36:26.545916+0000","last_became_active":"2023-06-03T20:28:51.232919+0000","last_became_peered":"2023-06-03T20:28:51.232919+0000","last_unstale":"2023-06-03T20:36:26.545916+0000","last_undegraded":"2023-06-03T20:36:26.545916+0000","last_fullsized":"2023-06-03T20:36:26.545916+0000","mapping_epoch":286,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":287,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T02:06:02.736415+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":214,"reported_epoch":323,"state":"active+clean","last_fresh":"2023-06-03T20:36:16.311268+0000","last_change":"2023-06-03T20:35:26.391963+0000","last_active":"2023-06-03T20:36:16.311268+0000","last_peered":"2023-06-03T20:36:16.311268+0000","last_clean":"2023-06-03T20:36:16.311268+0000","last_became_active":"2023-06-03T20:35:26.390925+0000","last_became_peered":"2023-06-03T20:35:26.390925+0000","last_unstale":"2023-06-03T20:36:16.311268+0000","last_undegraded":"2023-06-03T20:36:16.311268+0000","last_fullsized":"2023-06-03T20:36:16.311268+0000","mapping_epoch":317,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":303,"reported_epoch":324,"state":"active+clean","last_fresh":"2023-06-03T20:36:26.546440+0000","last_change":"2023-06-03T20:24:02.267331+0000","last_active":"2023-06-03T20:36:26.546440+0000","last_peered":"2023-06-03T20:36:26.546440+0000","last_clean":"2023-06-03T20:36:26.546440+0000","last_became_active":"2023-06-03T20:24:02.266997+0000","last_became_peered":"2023-06-03T20:24:02.266997+0000","last_unstale":"2023-06-03T20:36:26.546440+0000","last_undegraded":"2023-06-03T20:36:26.546440+0000","last_fullsized":"2023-06-03T20:36:26.546440+0000","mapping_epoch":254,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":255,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T22:09:18.993656+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"60'3","reported_seq":321,"reported_epoch":323,"state":"active+clean","last_fresh":"2023-06-03T20:36:15.736071+0000","last_change":"2023-06-03T20:23:27.261246+0000","last_active":"2023-06-03T20:36:15.736071+0000","last_peered":"2023-06-03T20:36:15.736071+0000","last_clean":"2023-06-03T20:36:15.736071+0000","last_became_active":"2023-06-03T20:23:27.259201+0000","last_became_peered":"2023-06-03T20:23:27.259201+0000","last_unstale":"2023-06-03T20:36:15.736071+0000","last_undegraded":"2023-06-03T20:36:15.736071+0000","last_fullsized":"2023-06-03T20:36:15.736071+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":115,"reported_epoch":324,"state":"active+clean","last_fresh":"2023-06-03T20:36:26.547283+0000","last_change":"2023-06-03T20:28:51.235184+0000","last_active":"2023-06-03T20:36:26.547283+0000","last_peered":"2023-06-03T20:36:26.547283+0000","last_clean":"2023-06-03T20:36:26.547283+0000","last_became_active":"2023-06-03T20:28:51.234830+0000","last_became_peered":"2023-06-03T20:28:51.234830+0000","last_unstale":"2023-06-03T20:36:26.547283+0000","last_undegraded":"2023-06-03T20:36:26.547283+0000","last_fullsized":"2023-06-03T20:36:26.547283+0000","mapping_epoch":286,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":287,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T04:40:02.479523+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"77'23","reported_seq":261,"reported_epoch":324,"state":"active+clean","last_fresh":"2023-06-03T20:36:25.200352+0000","last_change":"2023-06-03T20:24:02.280274+0000","last_active":"2023-06-03T20:36:25.200352+0000","last_peered":"2023-06-03T20:36:25.200352+0000","last_clean":"2023-06-03T20:36:25.200352+0000","last_became_active":"2023-06-03T20:24:02.264914+0000","last_became_peered":"2023-06-03T20:24:02.264914+0000","last_unstale":"2023-06-03T20:36:25.200352+0000","last_undegraded":"2023-06-03T20:36:25.200352+0000","last_fullsized":"2023-06-03T20:36:25.200352+0000","mapping_epoch":254,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":255,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":23,"log_dups_size":0,"ondisk_log_size":23,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T07:00:35.888220+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4716,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":23,"num_write_kb":20,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4716,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":237,"reported_epoch":324,"state":"active+clean","last_fresh":"2023-06-03T20:36:25.199950+0000","last_change":"2023-06-03T20:24:02.254850+0000","last_active":"2023-06-03T20:36:25.199950+0000","last_peered":"2023-06-03T20:36:25.199950+0000","last_clean":"2023-06-03T20:36:25.199950+0000","last_became_active":"2023-06-03T20:24:02.254409+0000","last_became_peered":"2023-06-03T20:24:02.254409+0000","last_unstale":"2023-06-03T20:36:25.199950+0000","last_undegraded":"2023-06-03T20:36:25.199950+0000","last_fullsized":"2023-06-03T20:36:25.199950+0000","mapping_epoch":254,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":255,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T19:49:53.393121+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"60'2","reported_seq":323,"reported_epoch":323,"state":"active+clean","last_fresh":"2023-06-03T20:36:15.735143+0000","last_change":"2023-06-03T20:23:27.258749+0000","last_active":"2023-06-03T20:36:15.735143+0000","last_peered":"2023-06-03T20:36:15.735143+0000","last_clean":"2023-06-03T20:36:15.735143+0000","last_became_active":"2023-06-03T20:23:27.256899+0000","last_became_peered":"2023-06-03T20:23:27.256899+0000","last_unstale":"2023-06-03T20:36:15.735143+0000","last_undegraded":"2023-06-03T20:36:15.735143+0000","last_fullsized":"2023-06-03T20:36:15.735143+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":322,"reported_epoch":324,"state":"active+clean","last_fresh":"2023-06-03T20:36:26.546533+0000","last_change":"2023-06-03T20:21:01.256612+0000","last_active":"2023-06-03T20:36:26.546533+0000","last_peered":"2023-06-03T20:36:26.546533+0000","last_clean":"2023-06-03T20:36:26.546533+0000","last_became_active":"2023-06-03T20:21:01.256243+0000","last_became_peered":"2023-06-03T20:21:01.256243+0000","last_unstale":"2023-06-03T20:36:26.546533+0000","last_undegraded":"2023-06-03T20:36:26.546533+0000","last_fullsized":"2023-06-03T20:36:26.546533+0000","mapping_epoch":235,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":236,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T05:24:47.054342+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":293,"reported_epoch":323,"state":"active+clean","last_fresh":"2023-06-03T20:36:15.197612+0000","last_change":"2023-06-03T19:56:54.507479+0000","last_active":"2023-06-03T20:36:15.197612+0000","last_peered":"2023-06-03T20:36:15.197612+0000","last_clean":"2023-06-03T20:36:15.197612+0000","last_became_active":"2023-06-03T19:56:54.507149+0000","last_became_peered":"2023-06-03T19:56:54.507149+0000","last_unstale":"2023-06-03T20:36:15.197612+0000","last_undegraded":"2023-06-03T20:36:15.197612+0000","last_fullsized":"2023-06-03T20:36:15.197612+0000","mapping_epoch":119,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T20:22:36.676317+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":204,"reported_epoch":323,"state":"active+clean","last_fresh":"2023-06-03T20:36:16.311500+0000","last_change":"2023-06-03T20:35:26.592533+0000","last_active":"2023-06-03T20:36:16.311500+0000","last_peered":"2023-06-03T20:36:16.311500+0000","last_clean":"2023-06-03T20:36:16.311500+0000","last_became_active":"2023-06-03T20:35:26.591612+0000","last_became_peered":"2023-06-03T20:35:26.591612+0000","last_unstale":"2023-06-03T20:36:16.311500+0000","last_undegraded":"2023-06-03T20:36:16.311500+0000","last_fullsized":"2023-06-03T20:36:16.311500+0000","mapping_epoch":317,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":323,"reported_epoch":323,"state":"active+clean","last_fresh":"2023-06-03T20:36:15.735467+0000","last_change":"2023-06-03T20:24:02.267531+0000","last_active":"2023-06-03T20:36:15.735467+0000","last_peered":"2023-06-03T20:36:15.735467+0000","last_clean":"2023-06-03T20:36:15.735467+0000","last_became_active":"2023-06-03T20:24:02.267159+0000","last_became_peered":"2023-06-03T20:24:02.267159+0000","last_unstale":"2023-06-03T20:36:15.735467+0000","last_undegraded":"2023-06-03T20:36:15.735467+0000","last_fullsized":"2023-06-03T20:36:15.735467+0000","mapping_epoch":254,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":255,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":293,"reported_epoch":323,"state":"active+clean","last_fresh":"2023-06-03T20:36:15.198845+0000","last_change":"2023-06-03T19:56:54.510289+0000","last_active":"2023-06-03T20:36:15.198845+0000","last_peered":"2023-06-03T20:36:15.198845+0000","last_clean":"2023-06-03T20:36:15.198845+0000","last_became_active":"2023-06-03T19:56:54.509905+0000","last_became_peered":"2023-06-03T19:56:54.509905+0000","last_unstale":"2023-06-03T20:36:15.198845+0000","last_undegraded":"2023-06-03T20:36:15.198845+0000","last_fullsized":"2023-06-03T20:36:15.198845+0000","mapping_epoch":119,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T02:54:26.792401+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"60'8","reported_seq":89,"reported_epoch":323,"state":"active+clean","last_fresh":"2023-06-03T20:36:16.309953+0000","last_change":"2023-06-03T20:26:59.682623+0000","last_active":"2023-06-03T20:36:16.309953+0000","last_peered":"2023-06-03T20:36:16.309953+0000","last_clean":"2023-06-03T20:36:16.309953+0000","last_became_active":"2023-06-03T20:24:02.260105+0000","last_became_peered":"2023-06-03T20:24:02.260105+0000","last_unstale":"2023-06-03T20:36:16.309953+0000","last_undegraded":"2023-06-03T20:36:16.309953+0000","last_fullsized":"2023-06-03T20:36:16.309953+0000","mapping_epoch":254,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":273,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":330,"reported_epoch":323,"state":"active+clean","last_fresh":"2023-06-03T20:36:16.310378+0000","last_change":"2023-06-03T20:28:24.281308+0000","last_active":"2023-06-03T20:36:16.310378+0000","last_peered":"2023-06-03T20:36:16.310378+0000","last_clean":"2023-06-03T20:36:16.310378+0000","last_became_active":"2023-06-03T20:28:24.280981+0000","last_became_peered":"2023-06-03T20:28:24.280981+0000","last_unstale":"2023-06-03T20:36:16.310378+0000","last_undegraded":"2023-06-03T20:36:16.310378+0000","last_fullsized":"2023-06-03T20:36:16.310378+0000","mapping_epoch":284,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":285,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T02:44:14.890147+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.f","version":"60'1","reported_seq":322,"reported_epoch":323,"state":"active+clean","last_fresh":"2023-06-03T20:36:15.735420+0000","last_change":"2023-06-03T20:23:27.263667+0000","last_active":"2023-06-03T20:36:15.735420+0000","last_peered":"2023-06-03T20:36:15.735420+0000","last_clean":"2023-06-03T20:36:15.735420+0000","last_became_active":"2023-06-03T20:23:27.263110+0000","last_became_peered":"2023-06-03T20:23:27.263110+0000","last_unstale":"2023-06-03T20:36:15.735420+0000","last_undegraded":"2023-06-03T20:36:15.735420+0000","last_fullsized":"2023-06-03T20:36:15.735420+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":255,"reported_epoch":323,"state":"active+clean","last_fresh":"2023-06-03T20:36:16.310100+0000","last_change":"2023-06-03T20:35:26.391598+0000","last_active":"2023-06-03T20:36:16.310100+0000","last_peered":"2023-06-03T20:36:16.310100+0000","last_clean":"2023-06-03T20:36:16.310100+0000","last_became_active":"2023-06-03T20:35:26.391038+0000","last_became_peered":"2023-06-03T20:35:26.391038+0000","last_unstale":"2023-06-03T20:36:16.310100+0000","last_undegraded":"2023-06-03T20:36:16.310100+0000","last_fullsized":"2023-06-03T20:36:16.310100+0000","mapping_epoch":317,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.e","version":"60'2","reported_seq":351,"reported_epoch":323,"state":"active+clean","last_fresh":"2023-06-03T20:36:15.736060+0000","last_change":"2023-06-03T20:23:28.606751+0000","last_active":"2023-06-03T20:36:15.736060+0000","last_peered":"2023-06-03T20:36:15.736060+0000","last_clean":"2023-06-03T20:36:15.736060+0000","last_became_active":"2023-06-03T20:23:28.605968+0000","last_became_peered":"2023-06-03T20:23:28.605968+0000","last_unstale":"2023-06-03T20:36:15.736060+0000","last_undegraded":"2023-06-03T20:36:15.736060+0000","last_fullsized":"2023-06-03T20:36:15.736060+0000","mapping_epoch":251,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":252,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":266,"reported_epoch":324,"state":"active+clean","last_fresh":"2023-06-03T20:36:25.200011+0000","last_change":"2023-06-03T20:23:28.249694+0000","last_active":"2023-06-03T20:36:25.200011+0000","last_peered":"2023-06-03T20:36:25.200011+0000","last_clean":"2023-06-03T20:36:25.200011+0000","last_became_active":"2023-06-03T20:23:28.249388+0000","last_became_peered":"2023-06-03T20:23:28.249388+0000","last_unstale":"2023-06-03T20:36:25.200011+0000","last_undegraded":"2023-06-03T20:36:25.200011+0000","last_fullsized":"2023-06-03T20:36:25.200011+0000","mapping_epoch":251,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":252,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T05:44:01.835586+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.d","version":"60'1","reported_seq":300,"reported_epoch":323,"state":"active+clean","last_fresh":"2023-06-03T20:36:15.735967+0000","last_change":"2023-06-03T20:26:59.686037+0000","last_active":"2023-06-03T20:36:15.735967+0000","last_peered":"2023-06-03T20:36:15.735967+0000","last_clean":"2023-06-03T20:36:15.735967+0000","last_became_active":"2023-06-03T20:24:02.268002+0000","last_became_peered":"2023-06-03T20:24:02.268002+0000","last_unstale":"2023-06-03T20:36:15.735967+0000","last_undegraded":"2023-06-03T20:36:15.735967+0000","last_fullsized":"2023-06-03T20:36:15.735967+0000","mapping_epoch":254,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":273,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":140,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":47,"reported_epoch":323,"state":"active+clean","last_fresh":"2023-06-03T20:36:16.310081+0000","last_change":"2023-06-03T20:28:51.230833+0000","last_active":"2023-06-03T20:36:16.310081+0000","last_peered":"2023-06-03T20:36:16.310081+0000","last_clean":"2023-06-03T20:36:16.310081+0000","last_became_active":"2023-06-03T20:28:51.230489+0000","last_became_peered":"2023-06-03T20:28:51.230489+0000","last_unstale":"2023-06-03T20:36:16.310081+0000","last_undegraded":"2023-06-03T20:36:16.310081+0000","last_fullsized":"2023-06-03T20:36:16.310081+0000","mapping_epoch":286,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":287,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":253,"reported_epoch":323,"state":"active+clean","last_fresh":"2023-06-03T20:36:16.311181+0000","last_change":"2023-06-03T20:35:26.592536+0000","last_active":"2023-06-03T20:36:16.311181+0000","last_peered":"2023-06-03T20:36:16.311181+0000","last_clean":"2023-06-03T20:36:16.311181+0000","last_became_active":"2023-06-03T20:35:26.591580+0000","last_became_peered":"2023-06-03T20:35:26.591580+0000","last_unstale":"2023-06-03T20:36:16.311181+0000","last_undegraded":"2023-06-03T20:36:16.311181+0000","last_fullsized":"2023-06-03T20:36:16.311181+0000","mapping_epoch":317,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":357,"reported_epoch":324,"state":"active+clean","last_fresh":"2023-06-03T20:36:25.444317+0000","last_change":"2023-06-03T20:23:28.283360+0000","last_active":"2023-06-03T20:36:25.444317+0000","last_peered":"2023-06-03T20:36:25.444317+0000","last_clean":"2023-06-03T20:36:25.444317+0000","last_became_active":"2023-06-03T20:23:28.283035+0000","last_became_peered":"2023-06-03T20:23:28.283035+0000","last_unstale":"2023-06-03T20:36:25.444317+0000","last_undegraded":"2023-06-03T20:36:25.444317+0000","last_fullsized":"2023-06-03T20:36:25.444317+0000","mapping_epoch":251,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":252,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T00:22:59.446515+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.b","version":"60'2","reported_seq":321,"reported_epoch":324,"state":"active+clean","last_fresh":"2023-06-03T20:36:26.546141+0000","last_change":"2023-06-03T20:23:27.254179+0000","last_active":"2023-06-03T20:36:26.546141+0000","last_peered":"2023-06-03T20:36:26.546141+0000","last_clean":"2023-06-03T20:36:26.546141+0000","last_became_active":"2023-06-03T20:23:27.253791+0000","last_became_peered":"2023-06-03T20:23:27.253791+0000","last_unstale":"2023-06-03T20:36:26.546141+0000","last_undegraded":"2023-06-03T20:36:26.546141+0000","last_fullsized":"2023-06-03T20:36:26.546141+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T00:31:27.336760+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":242,"reported_epoch":324,"state":"active+clean","last_fresh":"2023-06-03T20:36:25.200243+0000","last_change":"2023-06-03T20:24:02.255551+0000","last_active":"2023-06-03T20:36:25.200243+0000","last_peered":"2023-06-03T20:36:25.200243+0000","last_clean":"2023-06-03T20:36:25.200243+0000","last_became_active":"2023-06-03T20:24:02.255209+0000","last_became_peered":"2023-06-03T20:24:02.255209+0000","last_unstale":"2023-06-03T20:36:25.200243+0000","last_undegraded":"2023-06-03T20:36:25.200243+0000","last_fullsized":"2023-06-03T20:36:25.200243+0000","mapping_epoch":254,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":255,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T20:05:02.741217+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":79,"reported_epoch":323,"state":"active+clean","last_fresh":"2023-06-03T20:36:16.310521+0000","last_change":"2023-06-03T20:24:02.254172+0000","last_active":"2023-06-03T20:36:16.310521+0000","last_peered":"2023-06-03T20:36:16.310521+0000","last_clean":"2023-06-03T20:36:16.310521+0000","last_became_active":"2023-06-03T20:24:02.253797+0000","last_became_peered":"2023-06-03T20:24:02.253797+0000","last_unstale":"2023-06-03T20:36:16.310521+0000","last_undegraded":"2023-06-03T20:36:16.310521+0000","last_fullsized":"2023-06-03T20:36:16.310521+0000","mapping_epoch":254,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":255,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":247,"reported_epoch":324,"state":"active+clean","last_fresh":"2023-06-03T20:36:25.200494+0000","last_change":"2023-06-03T20:23:28.291250+0000","last_active":"2023-06-03T20:36:25.200494+0000","last_peered":"2023-06-03T20:36:25.200494+0000","last_clean":"2023-06-03T20:36:25.200494+0000","last_became_active":"2023-06-03T20:23:28.290120+0000","last_became_peered":"2023-06-03T20:23:28.290120+0000","last_unstale":"2023-06-03T20:36:25.200494+0000","last_undegraded":"2023-06-03T20:36:25.200494+0000","last_fullsized":"2023-06-03T20:36:25.200494+0000","mapping_epoch":251,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":252,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T22:37:56.033327+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":232,"reported_epoch":324,"state":"active+clean","last_fresh":"2023-06-03T20:36:25.199846+0000","last_change":"2023-06-03T20:23:27.276693+0000","last_active":"2023-06-03T20:36:25.199846+0000","last_peered":"2023-06-03T20:36:25.199846+0000","last_clean":"2023-06-03T20:36:25.199846+0000","last_became_active":"2023-06-03T20:23:27.276305+0000","last_became_peered":"2023-06-03T20:23:27.276305+0000","last_unstale":"2023-06-03T20:36:25.199846+0000","last_undegraded":"2023-06-03T20:36:25.199846+0000","last_fullsized":"2023-06-03T20:36:25.199846+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T20:34:50.944936+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":237,"reported_epoch":324,"state":"active+clean","last_fresh":"2023-06-03T20:36:25.200666+0000","last_change":"2023-06-03T20:28:51.231829+0000","last_active":"2023-06-03T20:36:25.200666+0000","last_peered":"2023-06-03T20:36:25.200666+0000","last_clean":"2023-06-03T20:36:25.200666+0000","last_became_active":"2023-06-03T20:28:51.231452+0000","last_became_peered":"2023-06-03T20:28:51.231452+0000","last_unstale":"2023-06-03T20:36:25.200666+0000","last_undegraded":"2023-06-03T20:36:25.200666+0000","last_fullsized":"2023-06-03T20:36:25.200666+0000","mapping_epoch":286,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":287,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T07:00:21.647058+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":330,"reported_epoch":323,"state":"active+clean","last_fresh":"2023-06-03T20:36:16.309812+0000","last_change":"2023-06-03T20:21:01.256958+0000","last_active":"2023-06-03T20:36:16.309812+0000","last_peered":"2023-06-03T20:36:16.309812+0000","last_clean":"2023-06-03T20:36:16.309812+0000","last_became_active":"2023-06-03T20:21:01.255452+0000","last_became_peered":"2023-06-03T20:21:01.255452+0000","last_unstale":"2023-06-03T20:36:16.309812+0000","last_undegraded":"2023-06-03T20:36:16.309812+0000","last_fullsized":"2023-06-03T20:36:16.309812+0000","mapping_epoch":235,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":236,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T01:56:59.777763+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":117,"reported_epoch":324,"state":"active+clean","last_fresh":"2023-06-03T20:36:26.547834+0000","last_change":"2023-06-03T20:23:28.250884+0000","last_active":"2023-06-03T20:36:26.547834+0000","last_peered":"2023-06-03T20:36:26.547834+0000","last_clean":"2023-06-03T20:36:26.547834+0000","last_became_active":"2023-06-03T20:23:28.243793+0000","last_became_peered":"2023-06-03T20:23:28.243793+0000","last_unstale":"2023-06-03T20:36:26.547834+0000","last_undegraded":"2023-06-03T20:36:26.547834+0000","last_fullsized":"2023-06-03T20:36:26.547834+0000","mapping_epoch":251,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":252,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T03:12:27.946002+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":107,"reported_epoch":324,"state":"active+clean","last_fresh":"2023-06-03T20:36:26.547657+0000","last_change":"2023-06-03T20:21:01.258182+0000","last_active":"2023-06-03T20:36:26.547657+0000","last_peered":"2023-06-03T20:36:26.547657+0000","last_clean":"2023-06-03T20:36:26.547657+0000","last_became_active":"2023-06-03T20:21:01.254049+0000","last_became_peered":"2023-06-03T20:21:01.254049+0000","last_unstale":"2023-06-03T20:36:26.547657+0000","last_undegraded":"2023-06-03T20:36:26.547657+0000","last_fullsized":"2023-06-03T20:36:26.547657+0000","mapping_epoch":235,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":236,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:32:20.692043+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:32:20.692043+0000","last_clean_scrub_stamp":"2023-06-03T19:32:20.692043+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T05:16:30.078473+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":220,"reported_epoch":323,"state":"active+clean","last_fresh":"2023-06-03T20:36:16.311374+0000","last_change":"2023-06-03T20:35:26.592542+0000","last_active":"2023-06-03T20:36:16.311374+0000","last_peered":"2023-06-03T20:36:16.311374+0000","last_clean":"2023-06-03T20:36:16.311374+0000","last_became_active":"2023-06-03T20:35:26.591632+0000","last_became_peered":"2023-06-03T20:35:26.591632+0000","last_unstale":"2023-06-03T20:36:16.311374+0000","last_undegraded":"2023-06-03T20:36:16.311374+0000","last_fullsized":"2023-06-03T20:36:16.311374+0000","mapping_epoch":317,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":118,"reported_epoch":324,"state":"active+clean","last_fresh":"2023-06-03T20:36:26.547103+0000","last_change":"2023-06-03T20:23:28.254434+0000","last_active":"2023-06-03T20:36:26.547103+0000","last_peered":"2023-06-03T20:36:26.547103+0000","last_clean":"2023-06-03T20:36:26.547103+0000","last_became_active":"2023-06-03T20:23:28.254012+0000","last_became_peered":"2023-06-03T20:23:28.254012+0000","last_unstale":"2023-06-03T20:36:26.547103+0000","last_undegraded":"2023-06-03T20:36:26.547103+0000","last_fullsized":"2023-06-03T20:36:26.547103+0000","mapping_epoch":251,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":252,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T06:22:09.540298+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":227,"reported_epoch":323,"state":"active+clean","last_fresh":"2023-06-03T20:36:16.310964+0000","last_change":"2023-06-03T20:35:26.203105+0000","last_active":"2023-06-03T20:36:16.310964+0000","last_peered":"2023-06-03T20:36:16.310964+0000","last_clean":"2023-06-03T20:36:16.310964+0000","last_became_active":"2023-06-03T20:35:26.202634+0000","last_became_peered":"2023-06-03T20:35:26.202634+0000","last_unstale":"2023-06-03T20:36:16.310964+0000","last_undegraded":"2023-06-03T20:36:16.310964+0000","last_fullsized":"2023-06-03T20:36:16.310964+0000","mapping_epoch":317,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:32:20.692043+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:32:20.692043+0000","last_clean_scrub_stamp":"2023-06-03T19:32:20.692043+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":212,"reported_epoch":323,"state":"active+clean","last_fresh":"2023-06-03T20:36:15.510565+0000","last_change":"2023-06-03T20:24:02.258311+0000","last_active":"2023-06-03T20:36:15.510565+0000","last_peered":"2023-06-03T20:36:15.510565+0000","last_clean":"2023-06-03T20:36:15.510565+0000","last_became_active":"2023-06-03T20:24:02.257676+0000","last_became_peered":"2023-06-03T20:24:02.257676+0000","last_unstale":"2023-06-03T20:36:15.510565+0000","last_undegraded":"2023-06-03T20:36:15.510565+0000","last_fullsized":"2023-06-03T20:36:15.510565+0000","mapping_epoch":254,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":255,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T01:47:46.383038+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":107,"reported_epoch":324,"state":"active+clean","last_fresh":"2023-06-03T20:36:26.547834+0000","last_change":"2023-06-03T20:21:01.259331+0000","last_active":"2023-06-03T20:36:26.547834+0000","last_peered":"2023-06-03T20:36:26.547834+0000","last_clean":"2023-06-03T20:36:26.547834+0000","last_became_active":"2023-06-03T20:21:01.258836+0000","last_became_peered":"2023-06-03T20:21:01.258836+0000","last_unstale":"2023-06-03T20:36:26.547834+0000","last_undegraded":"2023-06-03T20:36:26.547834+0000","last_fullsized":"2023-06-03T20:36:26.547834+0000","mapping_epoch":235,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":236,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T01:29:19.471600+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":313,"reported_epoch":323,"state":"active+clean","last_fresh":"2023-06-03T20:36:15.194505+0000","last_change":"2023-06-03T19:56:54.504859+0000","last_active":"2023-06-03T20:36:15.194505+0000","last_peered":"2023-06-03T20:36:15.194505+0000","last_clean":"2023-06-03T20:36:15.194505+0000","last_became_active":"2023-06-03T19:56:54.504455+0000","last_became_peered":"2023-06-03T19:56:54.504455+0000","last_unstale":"2023-06-03T20:36:15.194505+0000","last_undegraded":"2023-06-03T20:36:15.194505+0000","last_fullsized":"2023-06-03T20:36:15.194505+0000","mapping_epoch":119,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:32:20.692043+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:32:20.692043+0000","last_clean_scrub_stamp":"2023-06-03T19:32:20.692043+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T06:25:45.522100+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":116,"reported_epoch":324,"state":"active+clean","last_fresh":"2023-06-03T20:36:26.546653+0000","last_change":"2023-06-03T20:24:02.256677+0000","last_active":"2023-06-03T20:36:26.546653+0000","last_peered":"2023-06-03T20:36:26.546653+0000","last_clean":"2023-06-03T20:36:26.546653+0000","last_became_active":"2023-06-03T20:24:02.256216+0000","last_became_peered":"2023-06-03T20:24:02.256216+0000","last_unstale":"2023-06-03T20:36:26.546653+0000","last_undegraded":"2023-06-03T20:36:26.546653+0000","last_fullsized":"2023-06-03T20:36:26.546653+0000","mapping_epoch":254,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":255,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T02:32:45.186204+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":106,"reported_epoch":323,"state":"active+clean","last_fresh":"2023-06-03T20:36:16.312549+0000","last_change":"2023-06-03T20:21:01.261473+0000","last_active":"2023-06-03T20:36:16.312549+0000","last_peered":"2023-06-03T20:36:16.312549+0000","last_clean":"2023-06-03T20:36:16.312549+0000","last_became_active":"2023-06-03T20:21:01.261098+0000","last_became_peered":"2023-06-03T20:21:01.261098+0000","last_unstale":"2023-06-03T20:36:16.312549+0000","last_undegraded":"2023-06-03T20:36:16.312549+0000","last_fullsized":"2023-06-03T20:36:16.312549+0000","mapping_epoch":235,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":236,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T03:56:22.986549+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":209,"reported_epoch":323,"state":"active+clean","last_fresh":"2023-06-03T20:36:16.311275+0000","last_change":"2023-06-03T20:35:26.391589+0000","last_active":"2023-06-03T20:36:16.311275+0000","last_peered":"2023-06-03T20:36:16.311275+0000","last_clean":"2023-06-03T20:36:16.311275+0000","last_became_active":"2023-06-03T20:35:26.390813+0000","last_became_peered":"2023-06-03T20:35:26.390813+0000","last_unstale":"2023-06-03T20:36:16.311275+0000","last_undegraded":"2023-06-03T20:36:16.311275+0000","last_fullsized":"2023-06-03T20:36:16.311275+0000","mapping_epoch":317,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:32:20.692043+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:32:20.692043+0000","last_clean_scrub_stamp":"2023-06-03T19:32:20.692043+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"72'139","reported_seq":596,"reported_epoch":323,"state":"active+clean","last_fresh":"2023-06-03T20:36:15.735251+0000","last_change":"2023-06-03T20:27:01.248641+0000","last_active":"2023-06-03T20:36:15.735251+0000","last_peered":"2023-06-03T20:36:15.735251+0000","last_clean":"2023-06-03T20:36:15.735251+0000","last_became_active":"2023-06-03T20:27:01.248063+0000","last_became_peered":"2023-06-03T20:27:01.248063+0000","last_unstale":"2023-06-03T20:36:15.735251+0000","last_undegraded":"2023-06-03T20:36:15.735251+0000","last_fullsized":"2023-06-03T20:36:15.735251+0000","mapping_epoch":274,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":275,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:27:28.100808+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:27:28.100808+0000","last_clean_scrub_stamp":"2023-06-03T19:27:28.100808+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":316,"reported_epoch":324,"state":"active+clean","last_fresh":"2023-06-03T20:36:26.545437+0000","last_change":"2023-06-03T20:24:02.263360+0000","last_active":"2023-06-03T20:36:26.545437+0000","last_peered":"2023-06-03T20:36:26.545437+0000","last_clean":"2023-06-03T20:36:26.545437+0000","last_became_active":"2023-06-03T20:24:02.262954+0000","last_became_peered":"2023-06-03T20:24:02.262954+0000","last_unstale":"2023-06-03T20:36:26.545437+0000","last_undegraded":"2023-06-03T20:36:26.545437+0000","last_fullsized":"2023-06-03T20:36:26.545437+0000","mapping_epoch":254,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":255,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T19:38:46.084731+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":47,"reported_epoch":323,"state":"active+clean","last_fresh":"2023-06-03T20:36:16.310086+0000","last_change":"2023-06-03T20:28:51.234600+0000","last_active":"2023-06-03T20:36:16.310086+0000","last_peered":"2023-06-03T20:36:16.310086+0000","last_clean":"2023-06-03T20:36:16.310086+0000","last_became_active":"2023-06-03T20:28:51.234227+0000","last_became_peered":"2023-06-03T20:28:51.234227+0000","last_unstale":"2023-06-03T20:36:16.310086+0000","last_undegraded":"2023-06-03T20:36:16.310086+0000","last_fullsized":"2023-06-03T20:36:16.310086+0000","mapping_epoch":286,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":287,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":316,"reported_epoch":323,"state":"active+clean","last_fresh":"2023-06-03T20:36:15.735240+0000","last_change":"2023-06-03T20:21:01.258113+0000","last_active":"2023-06-03T20:36:15.735240+0000","last_peered":"2023-06-03T20:36:15.735240+0000","last_clean":"2023-06-03T20:36:15.735240+0000","last_became_active":"2023-06-03T20:21:01.257488+0000","last_became_peered":"2023-06-03T20:21:01.257488+0000","last_unstale":"2023-06-03T20:36:15.735240+0000","last_undegraded":"2023-06-03T20:36:15.735240+0000","last_fullsized":"2023-06-03T20:36:15.735240+0000","mapping_epoch":235,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":236,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:32:20.692043+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:32:20.692043+0000","last_clean_scrub_stamp":"2023-06-03T19:32:20.692043+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":326,"reported_epoch":324,"state":"active+clean","last_fresh":"2023-06-03T20:36:25.444164+0000","last_change":"2023-06-03T20:23:27.255164+0000","last_active":"2023-06-03T20:36:25.444164+0000","last_peered":"2023-06-03T20:36:25.444164+0000","last_clean":"2023-06-03T20:36:25.444164+0000","last_became_active":"2023-06-03T20:23:27.253932+0000","last_became_peered":"2023-06-03T20:23:27.253932+0000","last_unstale":"2023-06-03T20:36:25.444164+0000","last_undegraded":"2023-06-03T20:36:25.444164+0000","last_fullsized":"2023-06-03T20:36:25.444164+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T06:42:02.303554+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":329,"reported_epoch":324,"state":"active+clean","last_fresh":"2023-06-03T20:36:25.444659+0000","last_change":"2023-06-03T20:21:01.250266+0000","last_active":"2023-06-03T20:36:25.444659+0000","last_peered":"2023-06-03T20:36:25.444659+0000","last_clean":"2023-06-03T20:36:25.444659+0000","last_became_active":"2023-06-03T20:21:01.249839+0000","last_became_peered":"2023-06-03T20:21:01.249839+0000","last_unstale":"2023-06-03T20:36:25.444659+0000","last_undegraded":"2023-06-03T20:36:25.444659+0000","last_fullsized":"2023-06-03T20:36:25.444659+0000","mapping_epoch":235,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":236,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T05:13:26.584607+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":325,"reported_epoch":323,"state":"active+clean","last_fresh":"2023-06-03T20:36:15.735164+0000","last_change":"2023-06-03T20:23:27.261511+0000","last_active":"2023-06-03T20:36:15.735164+0000","last_peered":"2023-06-03T20:36:15.735164+0000","last_clean":"2023-06-03T20:36:15.735164+0000","last_became_active":"2023-06-03T20:23:27.259622+0000","last_became_peered":"2023-06-03T20:23:27.259622+0000","last_unstale":"2023-06-03T20:36:15.735164+0000","last_undegraded":"2023-06-03T20:36:15.735164+0000","last_fullsized":"2023-06-03T20:36:15.735164+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:32:20.692043+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:32:20.692043+0000","last_clean_scrub_stamp":"2023-06-03T19:32:20.692043+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":107,"reported_epoch":324,"state":"active+clean","last_fresh":"2023-06-03T20:36:26.547662+0000","last_change":"2023-06-03T20:21:01.252859+0000","last_active":"2023-06-03T20:36:26.547662+0000","last_peered":"2023-06-03T20:36:26.547662+0000","last_clean":"2023-06-03T20:36:26.547662+0000","last_became_active":"2023-06-03T20:21:01.251222+0000","last_became_peered":"2023-06-03T20:21:01.251222+0000","last_unstale":"2023-06-03T20:36:26.547662+0000","last_undegraded":"2023-06-03T20:36:26.547662+0000","last_fullsized":"2023-06-03T20:36:26.547662+0000","mapping_epoch":235,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":236,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T05:19:58.100226+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":282,"reported_epoch":324,"state":"active+clean","last_fresh":"2023-06-03T20:36:25.444441+0000","last_change":"2023-06-03T19:47:06.733532+0000","last_active":"2023-06-03T20:36:25.444441+0000","last_peered":"2023-06-03T20:36:25.444441+0000","last_clean":"2023-06-03T20:36:25.444441+0000","last_became_active":"2023-06-03T19:47:06.733131+0000","last_became_peered":"2023-06-03T19:47:06.733131+0000","last_unstale":"2023-06-03T20:36:25.444441+0000","last_undegraded":"2023-06-03T20:36:25.444441+0000","last_fullsized":"2023-06-03T20:36:25.444441+0000","mapping_epoch":86,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":87,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T23:46:03.788646+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.7","version":"60'2","reported_seq":291,"reported_epoch":323,"state":"active+clean","last_fresh":"2023-06-03T20:36:16.310239+0000","last_change":"2023-06-03T19:56:54.510148+0000","last_active":"2023-06-03T20:36:16.310239+0000","last_peered":"2023-06-03T20:36:16.310239+0000","last_clean":"2023-06-03T20:36:16.310239+0000","last_became_active":"2023-06-03T19:56:54.509771+0000","last_became_peered":"2023-06-03T19:56:54.509771+0000","last_unstale":"2023-06-03T20:36:16.310239+0000","last_undegraded":"2023-06-03T20:36:16.310239+0000","last_fullsized":"2023-06-03T20:36:16.310239+0000","mapping_epoch":119,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T00:52:42.807432+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":124,"reported_epoch":323,"state":"active+clean","last_fresh":"2023-06-03T20:36:16.312159+0000","last_change":"2023-06-03T20:24:02.257142+0000","last_active":"2023-06-03T20:36:16.312159+0000","last_peered":"2023-06-03T20:36:16.312159+0000","last_clean":"2023-06-03T20:36:16.312159+0000","last_became_active":"2023-06-03T20:24:02.256483+0000","last_became_peered":"2023-06-03T20:24:02.256483+0000","last_unstale":"2023-06-03T20:36:16.312159+0000","last_undegraded":"2023-06-03T20:36:16.312159+0000","last_fullsized":"2023-06-03T20:36:16.312159+0000","mapping_epoch":254,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":255,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:32:20.692043+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:32:20.692043+0000","last_clean_scrub_stamp":"2023-06-03T19:32:20.692043+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T00:53:37.197795+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":290,"reported_epoch":324,"state":"active+clean","last_fresh":"2023-06-03T20:36:26.545539+0000","last_change":"2023-06-03T19:56:54.507670+0000","last_active":"2023-06-03T20:36:26.545539+0000","last_peered":"2023-06-03T20:36:26.545539+0000","last_clean":"2023-06-03T20:36:26.545539+0000","last_became_active":"2023-06-03T19:56:54.507293+0000","last_became_peered":"2023-06-03T19:56:54.507293+0000","last_unstale":"2023-06-03T20:36:26.545539+0000","last_undegraded":"2023-06-03T20:36:26.545539+0000","last_fullsized":"2023-06-03T20:36:26.545539+0000","mapping_epoch":119,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T01:49:19.966054+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":79,"reported_epoch":323,"state":"active+clean","last_fresh":"2023-06-03T20:36:16.310480+0000","last_change":"2023-06-03T20:24:02.319230+0000","last_active":"2023-06-03T20:36:16.310480+0000","last_peered":"2023-06-03T20:36:16.310480+0000","last_clean":"2023-06-03T20:36:16.310480+0000","last_became_active":"2023-06-03T20:24:02.318926+0000","last_became_peered":"2023-06-03T20:24:02.318926+0000","last_unstale":"2023-06-03T20:36:16.310480+0000","last_undegraded":"2023-06-03T20:36:16.310480+0000","last_fullsized":"2023-06-03T20:36:16.310480+0000","mapping_epoch":254,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":255,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:32:20.692043+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:32:20.692043+0000","last_clean_scrub_stamp":"2023-06-03T19:32:20.692043+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"60'1","reported_seq":357,"reported_epoch":324,"state":"active+clean","last_fresh":"2023-06-03T20:36:25.444404+0000","last_change":"2023-06-03T20:23:28.239664+0000","last_active":"2023-06-03T20:36:25.444404+0000","last_peered":"2023-06-03T20:36:25.444404+0000","last_clean":"2023-06-03T20:36:25.444404+0000","last_became_active":"2023-06-03T20:23:28.239252+0000","last_became_peered":"2023-06-03T20:23:28.239252+0000","last_unstale":"2023-06-03T20:36:25.444404+0000","last_undegraded":"2023-06-03T20:36:25.444404+0000","last_fullsized":"2023-06-03T20:36:25.444404+0000","mapping_epoch":251,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":252,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T06:52:20.862520+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":210,"reported_epoch":323,"state":"active+clean","last_fresh":"2023-06-03T20:36:15.510224+0000","last_change":"2023-06-03T20:24:02.259850+0000","last_active":"2023-06-03T20:36:15.510224+0000","last_peered":"2023-06-03T20:36:15.510224+0000","last_clean":"2023-06-03T20:36:15.510224+0000","last_became_active":"2023-06-03T20:24:02.258764+0000","last_became_peered":"2023-06-03T20:24:02.258764+0000","last_unstale":"2023-06-03T20:36:15.510224+0000","last_undegraded":"2023-06-03T20:36:15.510224+0000","last_fullsized":"2023-06-03T20:36:15.510224+0000","mapping_epoch":254,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":255,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T21:54:11.487473+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"60'1","reported_seq":311,"reported_epoch":324,"state":"active+clean","last_fresh":"2023-06-03T20:36:25.444388+0000","last_change":"2023-06-03T20:23:27.254853+0000","last_active":"2023-06-03T20:36:25.444388+0000","last_peered":"2023-06-03T20:36:25.444388+0000","last_clean":"2023-06-03T20:36:25.444388+0000","last_became_active":"2023-06-03T20:23:27.254410+0000","last_became_peered":"2023-06-03T20:23:27.254410+0000","last_unstale":"2023-06-03T20:36:25.444388+0000","last_undegraded":"2023-06-03T20:36:25.444388+0000","last_fullsized":"2023-06-03T20:36:25.444388+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T22:46:19.423939+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":340,"reported_epoch":323,"state":"active+clean","last_fresh":"2023-06-03T20:36:15.195645+0000","last_change":"2023-06-03T20:21:01.257656+0000","last_active":"2023-06-03T20:36:15.195645+0000","last_peered":"2023-06-03T20:36:15.195645+0000","last_clean":"2023-06-03T20:36:15.195645+0000","last_became_active":"2023-06-03T20:21:01.257107+0000","last_became_peered":"2023-06-03T20:21:01.257107+0000","last_unstale":"2023-06-03T20:36:15.195645+0000","last_undegraded":"2023-06-03T20:36:15.195645+0000","last_fullsized":"2023-06-03T20:36:15.195645+0000","mapping_epoch":235,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":236,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T06:08:18.954539+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"60'2","reported_seq":257,"reported_epoch":323,"state":"active+clean","last_fresh":"2023-06-03T20:36:16.311222+0000","last_change":"2023-06-03T20:35:26.203752+0000","last_active":"2023-06-03T20:36:16.311222+0000","last_peered":"2023-06-03T20:36:16.311222+0000","last_clean":"2023-06-03T20:36:16.311222+0000","last_became_active":"2023-06-03T20:35:26.203286+0000","last_became_peered":"2023-06-03T20:35:26.203286+0000","last_unstale":"2023-06-03T20:36:16.311222+0000","last_undegraded":"2023-06-03T20:36:16.311222+0000","last_fullsized":"2023-06-03T20:36:16.311222+0000","mapping_epoch":317,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":264,"reported_epoch":323,"state":"active+clean","last_fresh":"2023-06-03T20:36:15.510253+0000","last_change":"2023-06-03T20:28:24.276513+0000","last_active":"2023-06-03T20:36:15.510253+0000","last_peered":"2023-06-03T20:36:15.510253+0000","last_clean":"2023-06-03T20:36:15.510253+0000","last_became_active":"2023-06-03T20:28:24.276244+0000","last_became_peered":"2023-06-03T20:28:24.276244+0000","last_unstale":"2023-06-03T20:36:15.510253+0000","last_undegraded":"2023-06-03T20:36:15.510253+0000","last_fullsized":"2023-06-03T20:36:15.510253+0000","mapping_epoch":284,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":285,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T06:17:57.159168+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[4,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"60'2","reported_seq":229,"reported_epoch":323,"state":"active+clean","last_fresh":"2023-06-03T20:36:16.311284+0000","last_change":"2023-06-03T20:35:26.204044+0000","last_active":"2023-06-03T20:36:16.311284+0000","last_peered":"2023-06-03T20:36:16.311284+0000","last_clean":"2023-06-03T20:36:16.311284+0000","last_became_active":"2023-06-03T20:35:26.203591+0000","last_became_peered":"2023-06-03T20:35:26.203591+0000","last_unstale":"2023-06-03T20:36:16.311284+0000","last_undegraded":"2023-06-03T20:36:16.311284+0000","last_fullsized":"2023-06-03T20:36:16.311284+0000","mapping_epoch":317,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":248,"reported_epoch":323,"state":"active+clean","last_fresh":"2023-06-03T20:36:15.509616+0000","last_change":"2023-06-03T20:23:27.254468+0000","last_active":"2023-06-03T20:36:15.509616+0000","last_peered":"2023-06-03T20:36:15.509616+0000","last_clean":"2023-06-03T20:36:15.509616+0000","last_became_active":"2023-06-03T20:23:27.253859+0000","last_became_peered":"2023-06-03T20:23:27.253859+0000","last_unstale":"2023-06-03T20:36:15.509616+0000","last_undegraded":"2023-06-03T20:36:15.509616+0000","last_fullsized":"2023-06-03T20:36:15.509616+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T20:15:25.066125+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"60'1","reported_seq":346,"reported_epoch":324,"state":"active+clean","last_fresh":"2023-06-03T20:36:26.546976+0000","last_change":"2023-06-03T20:28:51.234748+0000","last_active":"2023-06-03T20:36:26.546976+0000","last_peered":"2023-06-03T20:36:26.546976+0000","last_clean":"2023-06-03T20:36:26.546976+0000","last_became_active":"2023-06-03T20:28:51.234341+0000","last_became_peered":"2023-06-03T20:28:51.234341+0000","last_unstale":"2023-06-03T20:36:26.546976+0000","last_undegraded":"2023-06-03T20:36:26.546976+0000","last_fullsized":"2023-06-03T20:36:26.546976+0000","mapping_epoch":286,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":287,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T06:08:49.901424+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":328,"reported_epoch":323,"state":"active+clean","last_fresh":"2023-06-03T20:36:16.309895+0000","last_change":"2023-06-03T19:56:54.503416+0000","last_active":"2023-06-03T20:36:16.309895+0000","last_peered":"2023-06-03T20:36:16.309895+0000","last_clean":"2023-06-03T20:36:16.309895+0000","last_became_active":"2023-06-03T19:56:54.503018+0000","last_became_peered":"2023-06-03T19:56:54.503018+0000","last_unstale":"2023-06-03T20:36:16.309895+0000","last_undegraded":"2023-06-03T20:36:16.309895+0000","last_fullsized":"2023-06-03T20:36:16.309895+0000","mapping_epoch":119,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T20:14:00.330616+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"60'4","reported_seq":300,"reported_epoch":324,"state":"active+clean","last_fresh":"2023-06-03T20:36:25.444697+0000","last_change":"2023-06-03T19:56:11.748116+0000","last_active":"2023-06-03T20:36:25.444697+0000","last_peered":"2023-06-03T20:36:25.444697+0000","last_clean":"2023-06-03T20:36:25.444697+0000","last_became_active":"2023-06-03T19:56:11.747636+0000","last_became_peered":"2023-06-03T19:56:11.747636+0000","last_unstale":"2023-06-03T20:36:25.444697+0000","last_undegraded":"2023-06-03T20:36:25.444697+0000","last_fullsized":"2023-06-03T20:36:25.444697+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":115,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T06:57:48.052055+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":248,"reported_epoch":323,"state":"active+clean","last_fresh":"2023-06-03T20:36:15.509709+0000","last_change":"2023-06-03T20:28:51.231879+0000","last_active":"2023-06-03T20:36:15.509709+0000","last_peered":"2023-06-03T20:36:15.509709+0000","last_clean":"2023-06-03T20:36:15.509709+0000","last_became_active":"2023-06-03T20:28:51.231451+0000","last_became_peered":"2023-06-03T20:28:51.231451+0000","last_unstale":"2023-06-03T20:36:15.509709+0000","last_undegraded":"2023-06-03T20:36:15.509709+0000","last_fullsized":"2023-06-03T20:36:15.509709+0000","mapping_epoch":286,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":287,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T05:47:15.344575+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5486,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":62,"num_write_kb":30,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":21,"num_bytes_recovered":4856,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16458,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":30319,"internal_metadata":0},"log_size":56,"ondisk_log_size":56,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2969600,"data_stored":2951840,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":3,"num_store_stats":5},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":48,"seq":206158431010,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":976660,"kb_used_data":3532,"kb_used_omap":0,"kb_used_meta":973120,"kb_avail":92768492,"statfs":{"total":95995035648,"available":94994935808,"internally_reserved":0,"allocated":3616768,"data_stored":2927630,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":996474880},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 3 20:37:46 2023","interfaces":[{"interface":"back","average":{"1min":0.78400000000000003,"5min":0.85499999999999998,"15min":1.516},"min":{"1min":0.54000000000000004,"5min":0.52000000000000002,"15min":0.26400000000000001},"max":{"1min":1.157,"5min":1.2929999999999999,"15min":59.356000000000002},"last":0.72799999999999998},{"interface":"front","average":{"1min":0.90300000000000002,"5min":0.96199999999999997,"15min":1.464},"min":{"1min":0.54100000000000004,"5min":0.54100000000000004,"15min":0.26700000000000002},"max":{"1min":1.274,"5min":1.423,"15min":58.725999999999999},"last":1.117}]},{"osd":1,"last update":"Sat Jun 3 20:37:32 2023","interfaces":[{"interface":"back","average":{"1min":0.93400000000000005,"5min":0.90800000000000003,"15min":1.6539999999999999},"min":{"1min":0.624,"5min":0.45800000000000002,"15min":0.27000000000000002},"max":{"1min":1.248,"5min":1.4750000000000001,"15min":100.845},"last":1.004},{"interface":"front","average":{"1min":1.0329999999999999,"5min":0.98499999999999999,"15min":1.79},"min":{"1min":0.75800000000000001,"5min":0.50900000000000001,"15min":0.29899999999999999},"max":{"1min":1.206,"5min":1.5129999999999999,"15min":100.86499999999999},"last":1.034}]},{"osd":2,"last update":"Sat Jun 3 20:37:32 2023","interfaces":[{"interface":"back","average":{"1min":0.94199999999999995,"5min":0.94699999999999995,"15min":2.0499999999999998},"min":{"1min":0.68600000000000005,"5min":0.57999999999999996,"15min":0.28899999999999998},"max":{"1min":1.1819999999999999,"5min":1.7589999999999999,"15min":101.185},"last":0.83899999999999997},{"interface":"front","average":{"1min":0.96899999999999997,"5min":0.97699999999999998,"15min":2.1019999999999999},"min":{"1min":0.64600000000000002,"5min":0.502,"15min":0.32400000000000001},"max":{"1min":1.1799999999999999,"5min":1.464,"15min":100.61799999999999},"last":1.196}]},{"osd":3,"last update":"Sat Jun 3 20:37:32 2023","interfaces":[{"interface":"back","average":{"1min":0.997,"5min":0.98199999999999998,"15min":1.363},"min":{"1min":0.80100000000000005,"5min":0.59499999999999997,"15min":0.34999999999999998},"max":{"1min":1.222,"5min":1.6890000000000001,"15min":8.9120000000000008},"last":0.92400000000000004},{"interface":"front","average":{"1min":1.1040000000000001,"5min":1.075,"15min":1.431},"min":{"1min":0.72799999999999998,"5min":0.624,"15min":0.35399999999999998},"max":{"1min":1.349,"5min":1.5629999999999999,"15min":11.656000000000001},"last":0.94999999999999996}]},{"osd":4,"last update":"Sat Jun 3 20:36:55 2023","interfaces":[{"interface":"back","average":{"1min":1.129,"5min":1.091,"15min":1.5780000000000001},"min":{"1min":0.66200000000000003,"5min":0.53800000000000003,"15min":0.30399999999999999},"max":{"1min":1.546,"5min":1.6479999999999999,"15min":8.6010000000000009},"last":0.873},{"interface":"front","average":{"1min":1.085,"5min":1.056,"15min":1.536},"min":{"1min":0.76500000000000001,"5min":0.442,"15min":0.27600000000000002},"max":{"1min":1.349,"5min":3.9169999999999998,"15min":9.8460000000000001},"last":1.4710000000000001}]},{"osd":5,"last update":"Sat Jun 3 20:37:32 2023","interfaces":[{"interface":"back","average":{"1min":1.03,"5min":1.04,"15min":1.407},"min":{"1min":0.69699999999999995,"5min":0.51600000000000001,"15min":0.317},"max":{"1min":1.276,"5min":1.5980000000000001,"15min":17.696000000000002},"last":1.0820000000000001},{"interface":"front","average":{"1min":1.1659999999999999,"5min":1.1599999999999999,"15min":1.427},"min":{"1min":0.90200000000000002,"5min":0.625,"15min":0.29399999999999998},"max":{"1min":1.3720000000000001,"5min":1.6719999999999999,"15min":9.3710000000000004},"last":1.2430000000000001}]},{"osd":6,"last update":"Sat Jun 3 20:37:32 2023","interfaces":[{"interface":"back","average":{"1min":1.2330000000000001,"5min":1.1859999999999999,"15min":1.278},"min":{"1min":1.0029999999999999,"5min":0.45300000000000001,"15min":0.35899999999999999},"max":{"1min":1.4239999999999999,"5min":1.704,"15min":9.7200000000000006},"last":1.3460000000000001},{"interface":"front","average":{"1min":1.0189999999999999,"5min":1.0149999999999999,"15min":1.244},"min":{"1min":0.81799999999999995,"5min":0.48199999999999998,"15min":0.34300000000000003},"max":{"1min":1.242,"5min":1.522,"15min":9.923},"last":1.421}]}]},{"osd":6,"up_from":42,"seq":180388627243,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":624004,"kb_used_data":3516,"kb_used_omap":0,"kb_used_meta":620480,"kb_avail":93121148,"statfs":{"total":95995035648,"available":95356055552,"internally_reserved":0,"allocated":3600384,"data_stored":2927169,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":635371520},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 3 20:36:55 2023","interfaces":[{"interface":"back","average":{"1min":0.878,"5min":1.014,"15min":1.899},"min":{"1min":0.59999999999999998,"5min":0.495,"15min":0.254},"max":{"1min":1.367,"5min":3.7000000000000002,"15min":109.90900000000001},"last":0.89400000000000002},{"interface":"front","average":{"1min":0.97199999999999998,"5min":0.99399999999999999,"15min":2.0089999999999999},"min":{"1min":0.69399999999999995,"5min":0.53900000000000003,"15min":0.315},"max":{"1min":1.389,"5min":3.6589999999999998,"15min":109.86499999999999},"last":0.91300000000000003}]},{"osd":1,"last update":"Sat Jun 3 20:37:34 2023","interfaces":[{"interface":"back","average":{"1min":0.91200000000000003,"5min":0.99099999999999999,"15min":1.6439999999999999},"min":{"1min":0.55800000000000005,"5min":0.48199999999999998,"15min":0.29199999999999998},"max":{"1min":1.2969999999999999,"5min":2.3340000000000001,"15min":107.90600000000001},"last":0.70599999999999996},{"interface":"front","average":{"1min":0.96799999999999997,"5min":1.0009999999999999,"15min":1.7729999999999999},"min":{"1min":0.67800000000000005,"5min":0.627,"15min":0.32700000000000001},"max":{"1min":1.444,"5min":1.444,"15min":107.79900000000001},"last":1.077}]},{"osd":2,"last update":"Sat Jun 3 20:37:34 2023","interfaces":[{"interface":"back","average":{"1min":0.89900000000000002,"5min":0.91400000000000003,"15min":1.8140000000000001},"min":{"1min":0.66000000000000003,"5min":0.52000000000000002,"15min":0.33500000000000002},"max":{"1min":1.1990000000000001,"5min":1.3129999999999999,"15min":107.61199999999999},"last":0.85399999999999998},{"interface":"front","average":{"1min":0.94999999999999996,"5min":1.0009999999999999,"15min":1.905},"min":{"1min":0.62,"5min":0.62,"15min":0.29399999999999998},"max":{"1min":1.2529999999999999,"5min":2.2999999999999998,"15min":107.83},"last":1.05}]},{"osd":3,"last update":"Sat Jun 3 20:37:15 2023","interfaces":[{"interface":"back","average":{"1min":1.0389999999999999,"5min":1,"15min":1.8200000000000001},"min":{"1min":0.88100000000000001,"5min":0.64700000000000002,"15min":0.35499999999999998},"max":{"1min":1.381,"5min":1.397,"15min":108.254},"last":1.1399999999999999},{"interface":"front","average":{"1min":0.97699999999999998,"5min":1.054,"15min":1.861},"min":{"1min":0.59999999999999998,"5min":0.59999999999999998,"15min":0.39100000000000001},"max":{"1min":1.2330000000000001,"5min":2.1930000000000001,"15min":107.67700000000001},"last":0.81699999999999995}]},{"osd":4,"last update":"Sat Jun 3 20:36:48 2023","interfaces":[{"interface":"back","average":{"1min":1.048,"5min":0.96899999999999997,"15min":1.4470000000000001},"min":{"1min":0.72299999999999998,"5min":0.50800000000000001,"15min":0.33000000000000002},"max":{"1min":1.4510000000000001,"5min":1.488,"15min":10.574999999999999},"last":1.1830000000000001},{"interface":"front","average":{"1min":1.1339999999999999,"5min":1.1100000000000001,"15min":1.3029999999999999},"min":{"1min":0.77800000000000002,"5min":0.43099999999999999,"15min":0.34499999999999997},"max":{"1min":1.448,"5min":5.6639999999999997,"15min":12.622999999999999},"last":1.002}]},{"osd":5,"last update":"Sat Jun 3 20:36:48 2023","interfaces":[{"interface":"back","average":{"1min":1.23,"5min":1.1240000000000001,"15min":1.2470000000000001},"min":{"1min":0.505,"5min":0.49299999999999999,"15min":0.29899999999999999},"max":{"1min":2.2599999999999998,"5min":2.2599999999999998,"15min":10.625999999999999},"last":0.97499999999999998},{"interface":"front","average":{"1min":1.1359999999999999,"5min":1.056,"15min":1.371},"min":{"1min":0.55500000000000005,"5min":0.45700000000000002,"15min":0.40300000000000002},"max":{"1min":1.3400000000000001,"5min":1.51,"15min":10.856},"last":1.2649999999999999}]},{"osd":7,"last update":"Sat Jun 3 20:37:15 2023","interfaces":[{"interface":"back","average":{"1min":1.1910000000000001,"5min":1.2050000000000001,"15min":1.3340000000000001},"min":{"1min":0.66900000000000004,"5min":0.498,"15min":0.378},"max":{"1min":1.403,"5min":2.4020000000000001,"15min":9.0519999999999996},"last":1.1279999999999999},{"interface":"front","average":{"1min":1.196,"5min":1.1679999999999999,"15min":1.2769999999999999},"min":{"1min":0.81899999999999995,"5min":0.67800000000000005,"15min":0.38},"max":{"1min":1.504,"5min":1.73,"15min":10.52},"last":1.2350000000000001}]}]},{"osd":1,"up_from":103,"seq":442381632041,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":490608,"kb_used_data":3560,"kb_used_omap":4,"kb_used_meta":487035,"kb_avail":93254544,"statfs":{"total":95995035648,"available":95492653056,"internally_reserved":0,"allocated":3645440,"data_stored":2406778,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":498724750},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 3 20:37:35 2023","interfaces":[{"interface":"back","average":{"1min":0.57099999999999995,"5min":0.65300000000000002,"15min":2.7130000000000001},"min":{"1min":0.36799999999999999,"5min":0.248,"15min":0.17100000000000001},"max":{"1min":0.84699999999999998,"5min":2.04,"15min":131.83600000000001},"last":0.83399999999999996},{"interface":"front","average":{"1min":0.626,"5min":0.75900000000000001,"15min":2.7589999999999999},"min":{"1min":0.33900000000000002,"5min":0.33900000000000002,"15min":0.182},"max":{"1min":0.92900000000000005,"5min":1.1870000000000001,"15min":131.49100000000001},"last":1.0089999999999999}]},{"osd":2,"last update":"Sat Jun 3 20:37:35 2023","interfaces":[{"interface":"back","average":{"1min":0.56399999999999995,"5min":0.70499999999999996,"15min":1.851},"min":{"1min":0.373,"5min":0.34100000000000003,"15min":0.20599999999999999},"max":{"1min":0.76200000000000001,"5min":1.353,"15min":131.858},"last":1.1699999999999999},{"interface":"front","average":{"1min":0.55100000000000005,"5min":0.69699999999999995,"15min":1.964},"min":{"1min":0.30599999999999999,"5min":0.27600000000000002,"15min":0.23499999999999999},"max":{"1min":1.0620000000000001,"5min":1.2070000000000001,"15min":131.63},"last":0.93200000000000005}]},{"osd":3,"last update":"Sat Jun 3 20:37:09 2023","interfaces":[{"interface":"back","average":{"1min":0.81200000000000006,"5min":0.75900000000000001,"15min":2.4470000000000001},"min":{"1min":0.39000000000000001,"5min":0.313,"15min":0.23400000000000001},"max":{"1min":1.139,"5min":1.216,"15min":131.65199999999999},"last":1.2250000000000001},{"interface":"front","average":{"1min":0.85699999999999998,"5min":0.83899999999999997,"15min":2.528},"min":{"1min":0.47799999999999998,"5min":0.34699999999999998,"15min":0.223},"max":{"1min":1.2929999999999999,"5min":1.371,"15min":131.60599999999999},"last":1.1499999999999999}]},{"osd":4,"last update":"Sat Jun 3 20:36:48 2023","interfaces":[{"interface":"back","average":{"1min":0.93600000000000005,"5min":0.99299999999999999,"15min":1.516},"min":{"1min":0.77700000000000002,"5min":0.376,"15min":0.33100000000000002},"max":{"1min":1.276,"5min":2.3839999999999999,"15min":11.114000000000001},"last":1.206},{"interface":"front","average":{"1min":1.0669999999999999,"5min":1.107,"15min":1.4550000000000001},"min":{"1min":0.86499999999999999,"5min":0.38900000000000001,"15min":0.33400000000000002},"max":{"1min":1.3080000000000001,"5min":4.2320000000000002,"15min":11.247999999999999},"last":1.274}]},{"osd":5,"last update":"Sat Jun 3 20:37:35 2023","interfaces":[{"interface":"back","average":{"1min":0.92500000000000004,"5min":1.0289999999999999,"15min":1.542},"min":{"1min":0.65000000000000002,"5min":0.56899999999999995,"15min":0.30599999999999999},"max":{"1min":1.417,"5min":2.1320000000000001,"15min":11.246},"last":1.0940000000000001},{"interface":"front","average":{"1min":0.91900000000000004,"5min":1.077,"15min":1.9630000000000001},"min":{"1min":0.72799999999999998,"5min":0.72799999999999998,"15min":0.35799999999999998},"max":{"1min":1.1990000000000001,"5min":1.7190000000000001,"15min":131.25299999999999},"last":1.4279999999999999}]},{"osd":6,"last update":"Sat Jun 3 20:37:35 2023","interfaces":[{"interface":"back","average":{"1min":0.89600000000000002,"5min":1.038,"15min":1.381},"min":{"1min":0.61899999999999999,"5min":0.57599999999999996,"15min":0.36399999999999999},"max":{"1min":1.403,"5min":1.589,"15min":10.002000000000001},"last":1.343},{"interface":"front","average":{"1min":1.0149999999999999,"5min":1.101,"15min":1.4730000000000001},"min":{"1min":0.60199999999999998,"5min":0.60199999999999998,"15min":0.36899999999999999},"max":{"1min":1.3899999999999999,"5min":1.6379999999999999,"15min":11.067},"last":1.464}]},{"osd":7,"last update":"Sat Jun 3 20:37:35 2023","interfaces":[{"interface":"back","average":{"1min":1.0129999999999999,"5min":1.077,"15min":1.6180000000000001},"min":{"1min":0.70599999999999996,"5min":0.65500000000000003,"15min":0.377},"max":{"1min":1.4079999999999999,"5min":1.546,"15min":10.538},"last":1.5449999999999999},{"interface":"front","average":{"1min":0.98099999999999998,"5min":1.081,"15min":1.518},"min":{"1min":0.63700000000000001,"5min":0.60099999999999998,"15min":0.378},"max":{"1min":1.421,"5min":2.5550000000000002,"15min":14.025},"last":1.5069999999999999}]}]},{"osd":0,"up_from":86,"seq":369367188068,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":776300,"kb_used_data":4196,"kb_used_omap":5,"kb_used_meta":772090,"kb_avail":92968852,"statfs":{"total":95995035648,"available":95200104448,"internally_reserved":0,"allocated":4296704,"data_stored":3004247,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":790620794},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Jun 3 20:37:32 2023","interfaces":[{"interface":"back","average":{"1min":0.56999999999999995,"5min":0.67900000000000005,"15min":3.1579999999999999},"min":{"1min":0.35199999999999998,"5min":0.246,"15min":0.185},"max":{"1min":1.0700000000000001,"5min":1.1439999999999999,"15min":243.03200000000001},"last":1.069},{"interface":"front","average":{"1min":0.65600000000000003,"5min":0.66700000000000004,"15min":3.419},"min":{"1min":0.32100000000000001,"5min":0.32100000000000001,"15min":0.188},"max":{"1min":0.95299999999999996,"5min":1.1799999999999999,"15min":242.78399999999999},"last":0.58499999999999996}]},{"osd":2,"last update":"Sat Jun 3 20:37:32 2023","interfaces":[{"interface":"back","average":{"1min":0.68500000000000005,"5min":0.71599999999999997,"15min":1.9890000000000001},"min":{"1min":0.38400000000000001,"5min":0.36099999999999999,"15min":0.219},"max":{"1min":1.0149999999999999,"5min":1.294,"15min":153.01300000000001},"last":0.56000000000000005},{"interface":"front","average":{"1min":0.72399999999999998,"5min":0.76900000000000002,"15min":3.4249999999999998},"min":{"1min":0.44700000000000001,"5min":0.32700000000000001,"15min":0.24099999999999999},"max":{"1min":1.204,"5min":1.359,"15min":243.30099999999999},"last":0.73999999999999999}]},{"osd":3,"last update":"Sat Jun 3 20:37:07 2023","interfaces":[{"interface":"back","average":{"1min":0.81299999999999994,"5min":0.78700000000000003,"15min":3.5390000000000001},"min":{"1min":0.40300000000000002,"5min":0.40300000000000002,"15min":0.24399999999999999},"max":{"1min":1.1659999999999999,"5min":1.4239999999999999,"15min":259.68400000000003},"last":0.69599999999999995},{"interface":"front","average":{"1min":0.84999999999999998,"5min":0.82699999999999996,"15min":2.6070000000000002},"min":{"1min":0.48899999999999999,"5min":0.34799999999999998,"15min":0.23799999999999999},"max":{"1min":1.254,"5min":1.254,"15min":261.02699999999999},"last":1.1679999999999999}]},{"osd":4,"last update":"Sat Jun 3 20:36:49 2023","interfaces":[{"interface":"back","average":{"1min":1.0900000000000001,"5min":1.0840000000000001,"15min":1.569},"min":{"1min":0.71099999999999997,"5min":0.443,"15min":0.32400000000000001},"max":{"1min":1.383,"5min":4.9889999999999999,"15min":11.49},"last":0.95599999999999996},{"interface":"front","average":{"1min":1.1160000000000001,"5min":1.1160000000000001,"15min":1.2709999999999999},"min":{"1min":0.78700000000000003,"5min":0.57299999999999995,"15min":0.34100000000000003},"max":{"1min":1.841,"5min":4.806,"15min":7.5060000000000002},"last":0.84499999999999997}]},{"osd":5,"last update":"Sat Jun 3 20:37:39 2023","interfaces":[{"interface":"back","average":{"1min":0.91200000000000003,"5min":0.98499999999999999,"15min":2.3460000000000001},"min":{"1min":0.63500000000000001,"5min":0.54000000000000004,"15min":0.29299999999999998},"max":{"1min":1.3080000000000001,"5min":1.55,"15min":243.15899999999999},"last":0.82799999999999996},{"interface":"front","average":{"1min":0.99299999999999999,"5min":1.0669999999999999,"15min":2.6360000000000001},"min":{"1min":0.71199999999999997,"5min":0.60999999999999999,"15min":0.33400000000000002},"max":{"1min":1.4430000000000001,"5min":2.6400000000000001,"15min":243.32599999999999},"last":0.91400000000000003}]},{"osd":6,"last update":"Sat Jun 3 20:37:39 2023","interfaces":[{"interface":"back","average":{"1min":1.123,"5min":1.133,"15min":2.512},"min":{"1min":0.81999999999999995,"5min":0.70899999999999996,"15min":0.36499999999999999},"max":{"1min":1.4419999999999999,"5min":1.8919999999999999,"15min":243.25},"last":1.0429999999999999},{"interface":"front","average":{"1min":1.0409999999999999,"5min":1.139,"15min":2.6579999999999999},"min":{"1min":0.63100000000000001,"5min":0.63100000000000001,"15min":0.379},"max":{"1min":1.4099999999999999,"5min":1.716,"15min":242.84399999999999},"last":1.286}]},{"osd":7,"last update":"Sat Jun 3 20:37:39 2023","interfaces":[{"interface":"back","average":{"1min":1.151,"5min":1.165,"15min":2.7240000000000002},"min":{"1min":0.73599999999999999,"5min":0.55800000000000005,"15min":0.374},"max":{"1min":1.4970000000000001,"5min":1.9430000000000001,"15min":242.93600000000001},"last":1.107},{"interface":"front","average":{"1min":1.0669999999999999,"5min":1.177,"15min":2.2749999999999999},"min":{"1min":0.67700000000000005,"5min":0.56499999999999995,"15min":0.435},"max":{"1min":1.427,"5min":1.694,"15min":243.21299999999999},"last":1.363}]}]},{"osd":2,"up_from":114,"seq":489626272246,"num_pgs":20,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":656516,"kb_used_data":3516,"kb_used_omap":7,"kb_used_meta":652984,"kb_avail":93088636,"statfs":{"total":95995035648,"available":95322763264,"internally_reserved":0,"allocated":3600384,"data_stored":2403970,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8111,"internal_metadata":668655697},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 3 20:37:10 2023","interfaces":[{"interface":"back","average":{"1min":0.623,"5min":0.68200000000000005,"15min":1.9650000000000001},"min":{"1min":0.36299999999999999,"5min":0.253,"15min":0.182},"max":{"1min":0.95199999999999996,"5min":1.224,"15min":117.57899999999999},"last":0.76900000000000002},{"interface":"front","average":{"1min":0.71899999999999997,"5min":0.73899999999999999,"15min":2.0939999999999999},"min":{"1min":0.437,"5min":0.35499999999999998,"15min":0.20200000000000001},"max":{"1min":1.0649999999999999,"5min":1.1679999999999999,"15min":117.38500000000001},"last":0.68100000000000005}]},{"osd":1,"last update":"Sat Jun 3 20:37:10 2023","interfaces":[{"interface":"back","average":{"1min":0.71899999999999997,"5min":0.73799999999999999,"15min":2.0289999999999999},"min":{"1min":0.47199999999999998,"5min":0.40200000000000002,"15min":0.193},"max":{"1min":0.96999999999999997,"5min":1.5860000000000001,"15min":116.127},"last":0.85999999999999999},{"interface":"front","average":{"1min":0.69099999999999995,"5min":0.72099999999999997,"15min":2.0870000000000002},"min":{"1min":0.32400000000000001,"5min":0.32400000000000001,"15min":0.251},"max":{"1min":1.1299999999999999,"5min":1.3999999999999999,"15min":116.461},"last":0.89500000000000002}]},{"osd":3,"last update":"Sat Jun 3 20:37:10 2023","interfaces":[{"interface":"back","average":{"1min":0.755,"5min":0.75800000000000001,"15min":1.681},"min":{"1min":0.42199999999999999,"5min":0.41099999999999998,"15min":0.22700000000000001},"max":{"1min":1.1599999999999999,"5min":1.484,"15min":36.695},"last":1.002},{"interface":"front","average":{"1min":0.74199999999999999,"5min":0.79000000000000004,"15min":1.8100000000000001},"min":{"1min":0.45500000000000002,"5min":0.40500000000000003,"15min":0.27100000000000002},"max":{"1min":1.0389999999999999,"5min":1.298,"15min":37.088999999999999},"last":0.73499999999999999}]},{"osd":4,"last update":"Sat Jun 3 20:36:50 2023","interfaces":[{"interface":"back","average":{"1min":1.0349999999999999,"5min":1.038,"15min":1.8440000000000001},"min":{"1min":0.81200000000000006,"5min":0.58799999999999997,"15min":0.32700000000000001},"max":{"1min":1.6080000000000001,"5min":5.2210000000000001,"15min":16.561},"last":1.127},{"interface":"front","average":{"1min":1.1279999999999999,"5min":1.107,"15min":1.528},"min":{"1min":0.80200000000000005,"5min":0.46800000000000003,"15min":0.35099999999999998},"max":{"1min":1.6910000000000001,"5min":7.3230000000000004,"15min":9.9339999999999993},"last":1.159}]},{"osd":5,"last update":"Sat Jun 3 20:37:10 2023","interfaces":[{"interface":"back","average":{"1min":1.131,"5min":1.099,"15min":1.649},"min":{"1min":0.75600000000000001,"5min":0.58399999999999996,"15min":0.33200000000000002},"max":{"1min":1.837,"5min":1.837,"15min":17.751000000000001},"last":0.92300000000000004},{"interface":"front","average":{"1min":1.0840000000000001,"5min":1.081,"15min":1.669},"min":{"1min":0.79900000000000004,"5min":0.59399999999999997,"15min":0.316},"max":{"1min":1.452,"5min":1.663,"15min":17.928000000000001},"last":1.089}]},{"osd":6,"last update":"Sat Jun 3 20:37:10 2023","interfaces":[{"interface":"back","average":{"1min":1.1599999999999999,"5min":1.159,"15min":1.5069999999999999},"min":{"1min":0.623,"5min":0.623,"15min":0.33900000000000002},"max":{"1min":1.8999999999999999,"5min":1.8999999999999999,"15min":9.5809999999999995},"last":1.2430000000000001},{"interface":"front","average":{"1min":1.1499999999999999,"5min":1.1200000000000001,"15min":1.5329999999999999},"min":{"1min":0.73099999999999998,"5min":0.67100000000000004,"15min":0.35699999999999998},"max":{"1min":1.621,"5min":1.9770000000000001,"15min":12.867000000000001},"last":1.2749999999999999}]},{"osd":7,"last update":"Sat Jun 3 20:37:10 2023","interfaces":[{"interface":"back","average":{"1min":1.21,"5min":1.208,"15min":1.681},"min":{"1min":0.67300000000000004,"5min":0.67300000000000004,"15min":0.376},"max":{"1min":1.784,"5min":1.9950000000000001,"15min":10.529999999999999},"last":1.3200000000000001},{"interface":"front","average":{"1min":1.2250000000000001,"5min":1.1990000000000001,"15min":1.6559999999999999},"min":{"1min":0.94399999999999995,"5min":0.629,"15min":0.44700000000000001},"max":{"1min":1.595,"5min":2.0310000000000001,"15min":12.845000000000001},"last":1.345}]}]},{"osd":3,"up_from":119,"seq":511101108718,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":624256,"kb_used_data":3576,"kb_used_omap":2,"kb_used_meta":620669,"kb_avail":93120896,"statfs":{"total":95995035648,"available":95355797504,"internally_reserved":0,"allocated":3661824,"data_stored":2406383,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":635565109},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 3 20:37:17 2023","interfaces":[{"interface":"back","average":{"1min":0.57299999999999995,"5min":0.63200000000000001,"15min":2.278},"min":{"1min":0.32200000000000001,"5min":0.30299999999999999,"15min":0.183},"max":{"1min":0.76200000000000001,"5min":1.1279999999999999,"15min":168.33099999999999},"last":0.80900000000000005},{"interface":"front","average":{"1min":0.64100000000000001,"5min":0.72199999999999998,"15min":2.524},"min":{"1min":0.443,"5min":0.38700000000000001,"15min":0.19500000000000001},"max":{"1min":0.86599999999999999,"5min":1.446,"15min":169.25800000000001},"last":0.73599999999999999}]},{"osd":1,"last update":"Sat Jun 3 20:37:17 2023","interfaces":[{"interface":"back","average":{"1min":0.70499999999999996,"5min":0.73199999999999998,"15min":1.53},"min":{"1min":0.42699999999999999,"5min":0.34799999999999998,"15min":0.22900000000000001},"max":{"1min":0.99199999999999999,"5min":1.2789999999999999,"15min":117.047},"last":1.0609999999999999},{"interface":"front","average":{"1min":0.72299999999999998,"5min":0.77900000000000003,"15min":1.647},"min":{"1min":0.46200000000000002,"5min":0.38900000000000001,"15min":0.20399999999999999},"max":{"1min":0.91300000000000003,"5min":1.1890000000000001,"15min":116.505},"last":0.96599999999999997}]},{"osd":2,"last update":"Sat Jun 3 20:37:17 2023","interfaces":[{"interface":"back","average":{"1min":0.68100000000000005,"5min":0.73899999999999999,"15min":2.1619999999999999},"min":{"1min":0.35199999999999998,"5min":0.33100000000000002,"15min":0.248},"max":{"1min":0.95399999999999996,"5min":1.4470000000000001,"15min":117.584},"last":0.872},{"interface":"front","average":{"1min":0.71499999999999997,"5min":0.76000000000000001,"15min":2.085},"min":{"1min":0.34100000000000003,"5min":0.34100000000000003,"15min":0.22600000000000001},"max":{"1min":0.92700000000000005,"5min":1.4930000000000001,"15min":117.01000000000001},"last":1.236}]},{"osd":4,"last update":"Sat Jun 3 20:36:55 2023","interfaces":[{"interface":"back","average":{"1min":0.98699999999999999,"5min":1.0369999999999999,"15min":1.2889999999999999},"min":{"1min":0.78500000000000003,"5min":0.58899999999999997,"15min":0.34599999999999997},"max":{"1min":1.1719999999999999,"5min":3.6589999999999998,"15min":8.0449999999999999},"last":0.91300000000000003},{"interface":"front","average":{"1min":1.141,"5min":1.0840000000000001,"15min":1.4710000000000001},"min":{"1min":0.69299999999999995,"5min":0.624,"15min":0.33300000000000002},"max":{"1min":3.2799999999999998,"5min":4.2160000000000002,"15min":10.773999999999999},"last":1.194}]},{"osd":5,"last update":"Sat Jun 3 20:37:17 2023","interfaces":[{"interface":"back","average":{"1min":1.0289999999999999,"5min":1.0600000000000001,"15min":1.411},"min":{"1min":0.63100000000000001,"5min":0.63100000000000001,"15min":0.35599999999999998},"max":{"1min":1.2909999999999999,"5min":1.7350000000000001,"15min":38.651000000000003},"last":1.2609999999999999},{"interface":"front","average":{"1min":1.123,"5min":1.0880000000000001,"15min":1.458},"min":{"1min":0.68500000000000005,"5min":0.61599999999999999,"15min":0.316},"max":{"1min":3.0430000000000001,"5min":3.0430000000000001,"15min":38.585999999999999},"last":1.105}]},{"osd":6,"last update":"Sat Jun 3 20:37:17 2023","interfaces":[{"interface":"back","average":{"1min":1.2050000000000001,"5min":1.1279999999999999,"15min":1.2390000000000001},"min":{"1min":0.78300000000000003,"5min":0.53000000000000003,"15min":0.35899999999999999},"max":{"1min":3.7389999999999999,"5min":3.7389999999999999,"15min":9.5199999999999996},"last":1.3560000000000001},{"interface":"front","average":{"1min":1.2010000000000001,"5min":1.123,"15min":1.298},"min":{"1min":0.86899999999999999,"5min":0.68200000000000005,"15min":0.36699999999999999},"max":{"1min":3.6579999999999999,"5min":3.6579999999999999,"15min":8.0790000000000006},"last":1.161}]},{"osd":7,"last update":"Sat Jun 3 20:37:17 2023","interfaces":[{"interface":"back","average":{"1min":1.1479999999999999,"5min":1.133,"15min":1.5589999999999999},"min":{"1min":0.73499999999999999,"5min":0.63300000000000001,"15min":0.38900000000000001},"max":{"1min":2.7269999999999999,"5min":2.7269999999999999,"15min":39.008000000000003},"last":1.3109999999999999},{"interface":"front","average":{"1min":1.238,"5min":1.175,"15min":1.6459999999999999},"min":{"1min":0.73599999999999999,"5min":0.73599999999999999,"15min":0.38700000000000001},"max":{"1min":3.7799999999999998,"5min":3.7799999999999998,"15min":40.003999999999998},"last":1.3300000000000001}]}]},{"osd":4,"up_from":250,"seq":1073741824174,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":265532,"kb_used_data":3700,"kb_used_omap":10,"kb_used_meta":261813,"kb_avail":93479620,"statfs":{"total":95995035648,"available":95723130880,"internally_reserved":0,"allocated":3788800,"data_stored":2421785,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":10652,"internal_metadata":268097124},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 3 20:36:57 2023","interfaces":[{"interface":"back","average":{"1min":1.0580000000000001,"5min":0.96299999999999997,"15min":2.032},"min":{"1min":0.504,"5min":0.309,"15min":0.26300000000000001},"max":{"1min":1.3420000000000001,"5min":1.9770000000000001,"15min":145.755},"last":0.98299999999999998},{"interface":"front","average":{"1min":0.94099999999999995,"5min":0.90100000000000002,"15min":2.0190000000000001},"min":{"1min":0.58499999999999996,"5min":0.45500000000000002,"15min":0.30399999999999999},"max":{"1min":1.365,"5min":1.696,"15min":145.71799999999999},"last":1.0249999999999999}]},{"osd":1,"last update":"Sat Jun 3 20:36:57 2023","interfaces":[{"interface":"back","average":{"1min":0.92900000000000005,"5min":0.96199999999999997,"15min":1.7030000000000001},"min":{"1min":0.57499999999999996,"5min":0.55700000000000005,"15min":0.29399999999999998},"max":{"1min":1.252,"5min":2.141,"15min":62.539000000000001},"last":0.95599999999999996},{"interface":"front","average":{"1min":1.034,"5min":1.014,"15min":1.659},"min":{"1min":0.71399999999999997,"5min":0.29699999999999999,"15min":0.29599999999999999},"max":{"1min":1.4419999999999999,"5min":5.4749999999999996,"15min":62.604999999999997},"last":1.117}]},{"osd":2,"last update":"Sat Jun 3 20:36:57 2023","interfaces":[{"interface":"back","average":{"1min":0.96299999999999997,"5min":0.93700000000000006,"15min":2.5880000000000001},"min":{"1min":0.73099999999999998,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":1.294,"5min":1.532,"15min":78.941999999999993},"last":1.2490000000000001},{"interface":"front","average":{"1min":1.02,"5min":1.018,"15min":2.7650000000000001},"min":{"1min":0.66100000000000003,"5min":0.41199999999999998,"15min":0.28999999999999998},"max":{"1min":1.421,"5min":3.6589999999999998,"15min":80.069000000000003},"last":0.76000000000000001}]},{"osd":3,"last update":"Sat Jun 3 20:36:57 2023","interfaces":[{"interface":"back","average":{"1min":1.056,"5min":1.0660000000000001,"15min":4.7089999999999996},"min":{"1min":0.70299999999999996,"5min":0.38800000000000001,"15min":0.32500000000000001},"max":{"1min":1.518,"5min":2.266,"15min":217.578},"last":0.85499999999999998},{"interface":"front","average":{"1min":1.002,"5min":1.0820000000000001,"15min":4.7060000000000004},"min":{"1min":0.71999999999999997,"5min":0.58299999999999996,"15min":0.36199999999999999},"max":{"1min":1.284,"5min":4.6890000000000001,"15min":218.55699999999999},"last":1.1759999999999999}]},{"osd":5,"last update":"Sat Jun 3 20:36:57 2023","interfaces":[{"interface":"back","average":{"1min":1,"5min":0.98899999999999999,"15min":1.6759999999999999},"min":{"1min":0.751,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":1.385,"5min":2.0979999999999999,"15min":23.913},"last":1.0960000000000001},{"interface":"front","average":{"1min":1.085,"5min":1.0229999999999999,"15min":1.4890000000000001},"min":{"1min":0.73499999999999999,"5min":0.46600000000000003,"15min":0.34000000000000002},"max":{"1min":1.4910000000000001,"5min":1.4910000000000001,"15min":11.093999999999999},"last":1.1479999999999999}]},{"osd":6,"last update":"Sat Jun 3 20:36:57 2023","interfaces":[{"interface":"back","average":{"1min":1.087,"5min":1.077,"15min":1.611},"min":{"1min":0.72499999999999998,"5min":0.372,"15min":0.36099999999999999},"max":{"1min":1.482,"5min":2.141,"15min":23.963000000000001},"last":0.90900000000000003},{"interface":"front","average":{"1min":1.1140000000000001,"5min":1.1100000000000001,"15min":1.331},"min":{"1min":0.86299999999999999,"5min":0.41099999999999998,"15min":0.38},"max":{"1min":1.3939999999999999,"5min":1.613,"15min":8.4190000000000005},"last":1.306}]},{"osd":7,"last update":"Sat Jun 3 20:36:57 2023","interfaces":[{"interface":"back","average":{"1min":1.212,"5min":1.135,"15min":1.6399999999999999},"min":{"1min":0.85499999999999998,"5min":0.44800000000000001,"15min":0.372},"max":{"1min":1.5629999999999999,"5min":1.5629999999999999,"15min":10.747999999999999},"last":1.0620000000000001},{"interface":"front","average":{"1min":1.234,"5min":1.175,"15min":1.5409999999999999},"min":{"1min":0.88600000000000001,"5min":0.35999999999999999,"15min":0.34699999999999998},"max":{"1min":1.7769999999999999,"5min":1.7769999999999999,"15min":12.154},"last":1.214}]}]},{"osd":5,"up_from":36,"seq":154618823477,"num_pgs":21,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":986360,"kb_used_data":2928,"kb_used_omap":0,"kb_used_meta":983424,"kb_avail":92758792,"statfs":{"total":95995035648,"available":94985003008,"internally_reserved":0,"allocated":2998272,"data_stored":2332639,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":1007026176},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 3 20:37:36 2023","interfaces":[{"interface":"back","average":{"1min":0.86199999999999999,"5min":0.95099999999999996,"15min":2.1549999999999998},"min":{"1min":0.57699999999999996,"5min":0.44,"15min":0.25700000000000001},"max":{"1min":1.1679999999999999,"5min":2.1379999999999999,"15min":176.88300000000001},"last":0.999},{"interface":"front","average":{"1min":0.88200000000000001,"5min":0.97799999999999998,"15min":2.3079999999999998},"min":{"1min":0.64900000000000002,"5min":0.53200000000000003,"15min":0.26500000000000001},"max":{"1min":1.1970000000000001,"5min":2.1150000000000002,"15min":176.96199999999999},"last":0.80400000000000005}]},{"osd":1,"last update":"Sat Jun 3 20:37:36 2023","interfaces":[{"interface":"back","average":{"1min":0.86199999999999999,"5min":0.88800000000000001,"15min":2.012},"min":{"1min":0.67200000000000004,"5min":0.50700000000000001,"15min":0.28899999999999998},"max":{"1min":1.1539999999999999,"5min":1.288,"15min":102.205},"last":1.026},{"interface":"front","average":{"1min":0.84499999999999997,"5min":0.92000000000000004,"15min":2.0249999999999999},"min":{"1min":0.63900000000000001,"5min":0.57699999999999996,"15min":0.30499999999999999},"max":{"1min":1.1160000000000001,"5min":2.4849999999999999,"15min":102.265},"last":0.93500000000000005}]},{"osd":2,"last update":"Sat Jun 3 20:37:36 2023","interfaces":[{"interface":"back","average":{"1min":0.86499999999999999,"5min":0.93799999999999994,"15min":2.8959999999999999},"min":{"1min":0.58299999999999996,"5min":0.56299999999999994,"15min":0.32300000000000001},"max":{"1min":1.244,"5min":1.3819999999999999,"15min":236.40000000000001},"last":0.82299999999999995},{"interface":"front","average":{"1min":0.93200000000000005,"5min":0.96299999999999997,"15min":3.1339999999999999},"min":{"1min":0.64200000000000002,"5min":0.56699999999999995,"15min":0.308},"max":{"1min":1.212,"5min":1.3520000000000001,"15min":237.86099999999999},"last":0.86299999999999999}]},{"osd":3,"last update":"Sat Jun 3 20:37:14 2023","interfaces":[{"interface":"back","average":{"1min":0.999,"5min":1.0369999999999999,"15min":1.5449999999999999},"min":{"1min":0.59499999999999997,"5min":0.59499999999999997,"15min":0.34399999999999997},"max":{"1min":1.343,"5min":2.0840000000000001,"15min":35.725999999999999},"last":0.72199999999999998},{"interface":"front","average":{"1min":1.133,"5min":1.105,"15min":1.4139999999999999},"min":{"1min":0.74399999999999999,"5min":0.69299999999999995,"15min":0.377},"max":{"1min":1.4670000000000001,"5min":1.552,"15min":35.762999999999998},"last":1.0680000000000001}]},{"osd":4,"last update":"Sat Jun 3 20:36:49 2023","interfaces":[{"interface":"back","average":{"1min":0.999,"5min":1.016,"15min":1.871},"min":{"1min":0.69699999999999995,"5min":0.38100000000000001,"15min":0.32700000000000001},"max":{"1min":1.446,"5min":2.1760000000000002,"15min":21.786000000000001},"last":0.84799999999999998},{"interface":"front","average":{"1min":1.1599999999999999,"5min":1.0800000000000001,"15min":1.637},"min":{"1min":0.81499999999999995,"5min":0.45500000000000002,"15min":0.30199999999999999},"max":{"1min":1.5549999999999999,"5min":1.7509999999999999,"15min":26.478000000000002},"last":0.89800000000000002}]},{"osd":6,"last update":"Sat Jun 3 20:37:36 2023","interfaces":[{"interface":"back","average":{"1min":1.0609999999999999,"5min":1.0640000000000001,"15min":1.2450000000000001},"min":{"1min":0.84299999999999997,"5min":0.53200000000000003,"15min":0.35499999999999998},"max":{"1min":1.1919999999999999,"5min":1.6160000000000001,"15min":7.5140000000000002},"last":0.89100000000000001},{"interface":"front","average":{"1min":1.2,"5min":1.1699999999999999,"15min":1.337},"min":{"1min":1.012,"5min":0.64100000000000001,"15min":0.37},"max":{"1min":1.385,"5min":1.8899999999999999,"15min":6.5309999999999997},"last":0.92000000000000004}]},{"osd":7,"last update":"Sat Jun 3 20:37:29 2023","interfaces":[{"interface":"back","average":{"1min":1.2050000000000001,"5min":1.2110000000000001,"15min":1.548},"min":{"1min":0.91600000000000004,"5min":0.66900000000000004,"15min":0.39200000000000002},"max":{"1min":1.3620000000000001,"5min":2.2549999999999999,"15min":15.516},"last":0.96499999999999997},{"interface":"front","average":{"1min":1.1890000000000001,"5min":1.177,"15min":1.371},"min":{"1min":0.98799999999999999,"5min":0.58499999999999996,"15min":0.38400000000000001},"max":{"1min":1.6279999999999999,"5min":1.736,"15min":9.6519999999999992},"last":1.1080000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6928,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4862,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":10652,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4786,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4932,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-06-03T20:37:53.482 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid f1db3478-0243-11ee-9b1d-001a4aab830c -- ceph pg dump --format=json 2023-06-03T20:37:54.529 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:37:54 smithi089 bash[78452]: audit 1685824672.4543903 mgr.y (mgr.34104) 1866 : audit 0 from='client.47141 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-06-03T20:37:54.529 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:37:54 smithi089 bash[78452]: cluster 1685824673.348487 mgr.y (mgr.34104) 1867 : cluster 0 pgmap v1950: 73 pgs: 73 active+clean; 582 KiB data, 5.2 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:37:54.530 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:37:54 smithi089 bash[75068]: audit 1685824672.4543903 mgr.y (mgr.34104) 1866 : audit 0 from='client.47141 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-06-03T20:37:54.530 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:37:54 smithi089 bash[75068]: cluster 1685824673.348487 mgr.y (mgr.34104) 1867 : cluster 0 pgmap v1950: 73 pgs: 73 active+clean; 582 KiB data, 5.2 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:37:54.585 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:37:54 smithi164 bash[59462]: audit 1685824672.4543903 mgr.y (mgr.34104) 1866 : audit 0 from='client.47141 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-06-03T20:37:54.585 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:37:54 smithi164 bash[59462]: cluster 1685824673.348487 mgr.y (mgr.34104) 1867 : cluster 0 pgmap v1950: 73 pgs: 73 active+clean; 582 KiB data, 5.2 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:37:55.529 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:37:55 smithi089 bash[78452]: audit 1685824675.1199229 mon.c (mon.2) 738 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-03T20:37:55.529 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:37:55 smithi089 bash[75068]: audit 1685824675.1199229 mon.c (mon.2) 738 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-03T20:37:55.585 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:37:55 smithi164 bash[59462]: audit 1685824675.1199229 mon.c (mon.2) 738 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-06-03T20:37:56.529 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:37:56 smithi089 bash[78452]: cluster 1685824675.3491635 mgr.y (mgr.34104) 1868 : cluster 0 pgmap v1951: 73 pgs: 73 active+clean; 582 KiB data, 5.2 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:37:56.529 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:37:56 smithi089 bash[75068]: cluster 1685824675.3491635 mgr.y (mgr.34104) 1868 : cluster 0 pgmap v1951: 73 pgs: 73 active+clean; 582 KiB data, 5.2 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:37:56.529 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 20:37:56 smithi089 bash[71163]: ::ffff:172.21.15.164 - - [03/Jun/2023:20:37:56] "GET /metrics HTTP/1.1" 200 226902 "" "Prometheus/2.33.4" 2023-06-03T20:37:56.585 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:37:56 smithi164 bash[59462]: cluster 1685824675.3491635 mgr.y (mgr.34104) 1868 : cluster 0 pgmap v1951: 73 pgs: 73 active+clean; 582 KiB data, 5.2 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:37:58.298 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/mon.a/config 2023-06-03T20:37:58.779 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:37:58 smithi089 bash[78452]: cluster 1685824677.3498137 mgr.y (mgr.34104) 1869 : cluster 0 pgmap v1952: 73 pgs: 73 active+clean; 582 KiB data, 5.2 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:37:58.779 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:37:58 smithi089 bash[75068]: cluster 1685824677.3498137 mgr.y (mgr.34104) 1869 : cluster 0 pgmap v1952: 73 pgs: 73 active+clean; 582 KiB data, 5.2 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:37:58.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:37:58 smithi164 bash[59462]: cluster 1685824677.3498137 mgr.y (mgr.34104) 1869 : cluster 0 pgmap v1952: 73 pgs: 73 active+clean; 582 KiB data, 5.2 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:38:00.417 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:38:00 smithi089 bash[75068]: cluster 1685824679.3506632 mgr.y (mgr.34104) 1870 : cluster 0 pgmap v1953: 73 pgs: 73 active+clean; 582 KiB data, 5.2 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:38:00.528 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:38:00 smithi089 bash[78452]: cluster 1685824679.3506632 mgr.y (mgr.34104) 1870 : cluster 0 pgmap v1953: 73 pgs: 73 active+clean; 582 KiB data, 5.2 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:38:00.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:38:00 smithi164 bash[59462]: cluster 1685824679.3506632 mgr.y (mgr.34104) 1870 : cluster 0 pgmap v1953: 73 pgs: 73 active+clean; 582 KiB data, 5.2 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:38:01.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:38:01 smithi164 bash[59462]: audit 1685824680.5618477 mon.a (mon.0) 1778 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:38:01.835 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:38:01 smithi164 bash[59462]: audit 1685824680.5714898 mon.a (mon.0) 1779 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:38:01.836 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:38:01 smithi164 bash[59462]: cluster 1685824681.3514807 mgr.y (mgr.34104) 1871 : cluster 0 pgmap v1954: 73 pgs: 73 active+clean; 582 KiB data, 5.2 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:38:02.029 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:38:01 smithi089 bash[78452]: audit 1685824680.5618477 mon.a (mon.0) 1778 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:38:02.029 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:38:01 smithi089 bash[78452]: audit 1685824680.5714898 mon.a (mon.0) 1779 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:38:02.029 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:38:01 smithi089 bash[78452]: cluster 1685824681.3514807 mgr.y (mgr.34104) 1871 : cluster 0 pgmap v1954: 73 pgs: 73 active+clean; 582 KiB data, 5.2 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:38:02.029 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:38:01 smithi089 bash[75068]: audit 1685824680.5618477 mon.a (mon.0) 1778 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:38:02.030 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:38:01 smithi089 bash[75068]: audit 1685824680.5714898 mon.a (mon.0) 1779 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:38:02.030 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:38:01 smithi089 bash[75068]: cluster 1685824681.3514807 mgr.y (mgr.34104) 1871 : cluster 0 pgmap v1954: 73 pgs: 73 active+clean; 582 KiB data, 5.2 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:38:02.779 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:38:02 smithi089 bash[78452]: audit 1685824681.6165202 mon.a (mon.0) 1780 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:38:02.779 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:38:02 smithi089 bash[78452]: audit 1685824681.6292963 mon.a (mon.0) 1781 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:38:02.779 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:38:02 smithi089 bash[78452]: audit 1685824682.099853 mon.c (mon.2) 739 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-03T20:38:02.779 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:38:02 smithi089 bash[78452]: audit 1685824682.1008642 mon.c (mon.2) 740 : audit 1 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-03T20:38:02.780 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:38:02 smithi089 bash[78452]: audit 1685824682.107022 mon.a (mon.0) 1782 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:38:02.780 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:38:02 smithi089 bash[75068]: audit 1685824681.6165202 mon.a (mon.0) 1780 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:38:02.780 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:38:02 smithi089 bash[75068]: audit 1685824681.6292963 mon.a (mon.0) 1781 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:38:02.780 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:38:02 smithi089 bash[75068]: audit 1685824682.099853 mon.c (mon.2) 739 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-03T20:38:02.780 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:38:02 smithi089 bash[75068]: audit 1685824682.1008642 mon.c (mon.2) 740 : audit 1 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-03T20:38:02.780 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:38:02 smithi089 bash[75068]: audit 1685824682.107022 mon.a (mon.0) 1782 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:38:02.855 INFO:teuthology.orchestra.run.smithi089.stdout: 2023-06-03T20:38:02.857 INFO:teuthology.orchestra.run.smithi089.stderr:dumped all 2023-06-03T20:38:03.085 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:38:02 smithi164 bash[59462]: audit 1685824681.6165202 mon.a (mon.0) 1780 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:38:03.085 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:38:02 smithi164 bash[59462]: audit 1685824681.6292963 mon.a (mon.0) 1781 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:38:03.086 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:38:02 smithi164 bash[59462]: audit 1685824682.099853 mon.c (mon.2) 739 : audit 0 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-06-03T20:38:03.086 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:38:02 smithi164 bash[59462]: audit 1685824682.1008642 mon.c (mon.2) 740 : audit 1 from='mgr.34104 172.21.15.89:0/949803768' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-06-03T20:38:03.086 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:38:02 smithi164 bash[59462]: audit 1685824682.107022 mon.a (mon.0) 1782 : audit 1 from='mgr.34104 ' entity='mgr.y' 2023-06-03T20:38:03.848 INFO:teuthology.orchestra.run.smithi089.stdout:{"pg_ready":true,"pg_map":{"version":1954,"stamp":"2023-06-03T20:38:01.351034+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":595854,"num_objects":24,"num_object_clones":0,"num_object_copies":72,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":24,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":261,"num_write_kb":2446,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":27,"num_bytes_recovered":1775960,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":195,"ondisk_log_size":195,"up":219,"acting":219,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":219,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":5400236,"kb_used_data":28524,"kb_used_omap":30,"kb_used_meta":5371617,"kb_avail":744560980,"statfs":{"total":767960285184,"available":762430443520,"internally_reserved":0,"allocated":29208576,"data_stored":20830601,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":31502,"internal_metadata":5500536050},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.004854"},"pg_stats":[{"pgid":"3.1a","version":"60'1","reported_seq":253,"reported_epoch":323,"state":"active+clean","last_fresh":"2023-06-03T20:36:15.510110+0000","last_change":"2023-06-03T20:23:28.242261+0000","last_active":"2023-06-03T20:36:15.510110+0000","last_peered":"2023-06-03T20:36:15.510110+0000","last_clean":"2023-06-03T20:36:15.510110+0000","last_became_active":"2023-06-03T20:23:28.240045+0000","last_became_peered":"2023-06-03T20:23:28.240045+0000","last_unstale":"2023-06-03T20:36:15.510110+0000","last_undegraded":"2023-06-03T20:36:15.510110+0000","last_fullsized":"2023-06-03T20:36:15.510110+0000","mapping_epoch":251,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":252,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T05:54:33.215385+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":60,"reported_epoch":323,"state":"active+clean","last_fresh":"2023-06-03T20:36:16.311167+0000","last_change":"2023-06-03T20:35:26.203516+0000","last_active":"2023-06-03T20:36:16.311167+0000","last_peered":"2023-06-03T20:36:16.311167+0000","last_clean":"2023-06-03T20:36:16.311167+0000","last_became_active":"2023-06-03T20:35:26.203123+0000","last_became_peered":"2023-06-03T20:35:26.203123+0000","last_unstale":"2023-06-03T20:36:16.311167+0000","last_undegraded":"2023-06-03T20:36:16.311167+0000","last_fullsized":"2023-06-03T20:36:16.311167+0000","mapping_epoch":317,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,4],"acting":[5,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":114,"reported_epoch":323,"state":"active+clean","last_fresh":"2023-06-03T20:36:16.310781+0000","last_change":"2023-06-03T20:23:28.253821+0000","last_active":"2023-06-03T20:36:16.310781+0000","last_peered":"2023-06-03T20:36:16.310781+0000","last_clean":"2023-06-03T20:36:16.310781+0000","last_became_active":"2023-06-03T20:23:28.253448+0000","last_became_peered":"2023-06-03T20:23:28.253448+0000","last_unstale":"2023-06-03T20:36:16.310781+0000","last_undegraded":"2023-06-03T20:36:16.310781+0000","last_fullsized":"2023-06-03T20:36:16.310781+0000","mapping_epoch":251,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":252,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T23:40:02.264778+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":298,"reported_epoch":324,"state":"active+clean","last_fresh":"2023-06-03T20:36:25.444382+0000","last_change":"2023-06-03T19:56:54.507653+0000","last_active":"2023-06-03T20:36:25.444382+0000","last_peered":"2023-06-03T20:36:25.444382+0000","last_clean":"2023-06-03T20:36:25.444382+0000","last_became_active":"2023-06-03T19:56:54.507350+0000","last_became_peered":"2023-06-03T19:56:54.507350+0000","last_unstale":"2023-06-03T20:36:25.444382+0000","last_undegraded":"2023-06-03T20:36:25.444382+0000","last_fullsized":"2023-06-03T20:36:25.444382+0000","mapping_epoch":119,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T22:33:20.065844+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":337,"reported_epoch":323,"state":"active+clean","last_fresh":"2023-06-03T20:36:16.309990+0000","last_change":"2023-06-03T20:21:01.256894+0000","last_active":"2023-06-03T20:36:16.309990+0000","last_peered":"2023-06-03T20:36:16.309990+0000","last_clean":"2023-06-03T20:36:16.309990+0000","last_became_active":"2023-06-03T20:21:01.255903+0000","last_became_peered":"2023-06-03T20:21:01.255903+0000","last_unstale":"2023-06-03T20:36:16.309990+0000","last_undegraded":"2023-06-03T20:36:16.309990+0000","last_fullsized":"2023-06-03T20:36:16.309990+0000","mapping_epoch":235,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":236,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T20:35:27.810860+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":88,"reported_epoch":323,"state":"active+clean","last_fresh":"2023-06-03T20:36:16.309457+0000","last_change":"2023-06-03T20:28:24.269355+0000","last_active":"2023-06-03T20:36:16.309457+0000","last_peered":"2023-06-03T20:36:16.309457+0000","last_clean":"2023-06-03T20:36:16.309457+0000","last_became_active":"2023-06-03T20:28:24.268859+0000","last_became_peered":"2023-06-03T20:28:24.268859+0000","last_unstale":"2023-06-03T20:36:16.309457+0000","last_undegraded":"2023-06-03T20:36:16.309457+0000","last_fullsized":"2023-06-03T20:36:16.309457+0000","mapping_epoch":284,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":285,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,2],"acting":[6,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":293,"reported_epoch":324,"state":"active+clean","last_fresh":"2023-06-03T20:36:25.444859+0000","last_change":"2023-06-03T19:56:54.509490+0000","last_active":"2023-06-03T20:36:25.444859+0000","last_peered":"2023-06-03T20:36:25.444859+0000","last_clean":"2023-06-03T20:36:25.444859+0000","last_became_active":"2023-06-03T19:56:54.509145+0000","last_became_peered":"2023-06-03T19:56:54.509145+0000","last_unstale":"2023-06-03T20:36:25.444859+0000","last_undegraded":"2023-06-03T20:36:25.444859+0000","last_fullsized":"2023-06-03T20:36:25.444859+0000","mapping_epoch":119,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T07:12:03.964110+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":358,"reported_epoch":324,"state":"active+clean","last_fresh":"2023-06-03T20:36:26.545916+0000","last_change":"2023-06-03T20:28:51.233277+0000","last_active":"2023-06-03T20:36:26.545916+0000","last_peered":"2023-06-03T20:36:26.545916+0000","last_clean":"2023-06-03T20:36:26.545916+0000","last_became_active":"2023-06-03T20:28:51.232919+0000","last_became_peered":"2023-06-03T20:28:51.232919+0000","last_unstale":"2023-06-03T20:36:26.545916+0000","last_undegraded":"2023-06-03T20:36:26.545916+0000","last_fullsized":"2023-06-03T20:36:26.545916+0000","mapping_epoch":286,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":287,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T02:06:02.736415+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":214,"reported_epoch":323,"state":"active+clean","last_fresh":"2023-06-03T20:36:16.311268+0000","last_change":"2023-06-03T20:35:26.391963+0000","last_active":"2023-06-03T20:36:16.311268+0000","last_peered":"2023-06-03T20:36:16.311268+0000","last_clean":"2023-06-03T20:36:16.311268+0000","last_became_active":"2023-06-03T20:35:26.390925+0000","last_became_peered":"2023-06-03T20:35:26.390925+0000","last_unstale":"2023-06-03T20:36:16.311268+0000","last_undegraded":"2023-06-03T20:36:16.311268+0000","last_fullsized":"2023-06-03T20:36:16.311268+0000","mapping_epoch":317,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":303,"reported_epoch":324,"state":"active+clean","last_fresh":"2023-06-03T20:36:26.546440+0000","last_change":"2023-06-03T20:24:02.267331+0000","last_active":"2023-06-03T20:36:26.546440+0000","last_peered":"2023-06-03T20:36:26.546440+0000","last_clean":"2023-06-03T20:36:26.546440+0000","last_became_active":"2023-06-03T20:24:02.266997+0000","last_became_peered":"2023-06-03T20:24:02.266997+0000","last_unstale":"2023-06-03T20:36:26.546440+0000","last_undegraded":"2023-06-03T20:36:26.546440+0000","last_fullsized":"2023-06-03T20:36:26.546440+0000","mapping_epoch":254,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":255,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T22:09:18.993656+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"60'3","reported_seq":321,"reported_epoch":323,"state":"active+clean","last_fresh":"2023-06-03T20:36:15.736071+0000","last_change":"2023-06-03T20:23:27.261246+0000","last_active":"2023-06-03T20:36:15.736071+0000","last_peered":"2023-06-03T20:36:15.736071+0000","last_clean":"2023-06-03T20:36:15.736071+0000","last_became_active":"2023-06-03T20:23:27.259201+0000","last_became_peered":"2023-06-03T20:23:27.259201+0000","last_unstale":"2023-06-03T20:36:15.736071+0000","last_undegraded":"2023-06-03T20:36:15.736071+0000","last_fullsized":"2023-06-03T20:36:15.736071+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":115,"reported_epoch":324,"state":"active+clean","last_fresh":"2023-06-03T20:36:26.547283+0000","last_change":"2023-06-03T20:28:51.235184+0000","last_active":"2023-06-03T20:36:26.547283+0000","last_peered":"2023-06-03T20:36:26.547283+0000","last_clean":"2023-06-03T20:36:26.547283+0000","last_became_active":"2023-06-03T20:28:51.234830+0000","last_became_peered":"2023-06-03T20:28:51.234830+0000","last_unstale":"2023-06-03T20:36:26.547283+0000","last_undegraded":"2023-06-03T20:36:26.547283+0000","last_fullsized":"2023-06-03T20:36:26.547283+0000","mapping_epoch":286,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":287,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T04:40:02.479523+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"77'23","reported_seq":261,"reported_epoch":324,"state":"active+clean","last_fresh":"2023-06-03T20:36:25.200352+0000","last_change":"2023-06-03T20:24:02.280274+0000","last_active":"2023-06-03T20:36:25.200352+0000","last_peered":"2023-06-03T20:36:25.200352+0000","last_clean":"2023-06-03T20:36:25.200352+0000","last_became_active":"2023-06-03T20:24:02.264914+0000","last_became_peered":"2023-06-03T20:24:02.264914+0000","last_unstale":"2023-06-03T20:36:25.200352+0000","last_undegraded":"2023-06-03T20:36:25.200352+0000","last_fullsized":"2023-06-03T20:36:25.200352+0000","mapping_epoch":254,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":255,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":23,"log_dups_size":0,"ondisk_log_size":23,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T07:00:35.888220+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4716,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":23,"num_write_kb":20,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4716,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":237,"reported_epoch":324,"state":"active+clean","last_fresh":"2023-06-03T20:36:25.199950+0000","last_change":"2023-06-03T20:24:02.254850+0000","last_active":"2023-06-03T20:36:25.199950+0000","last_peered":"2023-06-03T20:36:25.199950+0000","last_clean":"2023-06-03T20:36:25.199950+0000","last_became_active":"2023-06-03T20:24:02.254409+0000","last_became_peered":"2023-06-03T20:24:02.254409+0000","last_unstale":"2023-06-03T20:36:25.199950+0000","last_undegraded":"2023-06-03T20:36:25.199950+0000","last_fullsized":"2023-06-03T20:36:25.199950+0000","mapping_epoch":254,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":255,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T19:49:53.393121+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"60'2","reported_seq":323,"reported_epoch":323,"state":"active+clean","last_fresh":"2023-06-03T20:36:15.735143+0000","last_change":"2023-06-03T20:23:27.258749+0000","last_active":"2023-06-03T20:36:15.735143+0000","last_peered":"2023-06-03T20:36:15.735143+0000","last_clean":"2023-06-03T20:36:15.735143+0000","last_became_active":"2023-06-03T20:23:27.256899+0000","last_became_peered":"2023-06-03T20:23:27.256899+0000","last_unstale":"2023-06-03T20:36:15.735143+0000","last_undegraded":"2023-06-03T20:36:15.735143+0000","last_fullsized":"2023-06-03T20:36:15.735143+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":322,"reported_epoch":324,"state":"active+clean","last_fresh":"2023-06-03T20:36:26.546533+0000","last_change":"2023-06-03T20:21:01.256612+0000","last_active":"2023-06-03T20:36:26.546533+0000","last_peered":"2023-06-03T20:36:26.546533+0000","last_clean":"2023-06-03T20:36:26.546533+0000","last_became_active":"2023-06-03T20:21:01.256243+0000","last_became_peered":"2023-06-03T20:21:01.256243+0000","last_unstale":"2023-06-03T20:36:26.546533+0000","last_undegraded":"2023-06-03T20:36:26.546533+0000","last_fullsized":"2023-06-03T20:36:26.546533+0000","mapping_epoch":235,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":236,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T05:24:47.054342+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":293,"reported_epoch":323,"state":"active+clean","last_fresh":"2023-06-03T20:36:15.197612+0000","last_change":"2023-06-03T19:56:54.507479+0000","last_active":"2023-06-03T20:36:15.197612+0000","last_peered":"2023-06-03T20:36:15.197612+0000","last_clean":"2023-06-03T20:36:15.197612+0000","last_became_active":"2023-06-03T19:56:54.507149+0000","last_became_peered":"2023-06-03T19:56:54.507149+0000","last_unstale":"2023-06-03T20:36:15.197612+0000","last_undegraded":"2023-06-03T20:36:15.197612+0000","last_fullsized":"2023-06-03T20:36:15.197612+0000","mapping_epoch":119,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T20:22:36.676317+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":204,"reported_epoch":323,"state":"active+clean","last_fresh":"2023-06-03T20:36:16.311500+0000","last_change":"2023-06-03T20:35:26.592533+0000","last_active":"2023-06-03T20:36:16.311500+0000","last_peered":"2023-06-03T20:36:16.311500+0000","last_clean":"2023-06-03T20:36:16.311500+0000","last_became_active":"2023-06-03T20:35:26.591612+0000","last_became_peered":"2023-06-03T20:35:26.591612+0000","last_unstale":"2023-06-03T20:36:16.311500+0000","last_undegraded":"2023-06-03T20:36:16.311500+0000","last_fullsized":"2023-06-03T20:36:16.311500+0000","mapping_epoch":317,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":323,"reported_epoch":323,"state":"active+clean","last_fresh":"2023-06-03T20:36:15.735467+0000","last_change":"2023-06-03T20:24:02.267531+0000","last_active":"2023-06-03T20:36:15.735467+0000","last_peered":"2023-06-03T20:36:15.735467+0000","last_clean":"2023-06-03T20:36:15.735467+0000","last_became_active":"2023-06-03T20:24:02.267159+0000","last_became_peered":"2023-06-03T20:24:02.267159+0000","last_unstale":"2023-06-03T20:36:15.735467+0000","last_undegraded":"2023-06-03T20:36:15.735467+0000","last_fullsized":"2023-06-03T20:36:15.735467+0000","mapping_epoch":254,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":255,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":294,"reported_epoch":324,"state":"active+clean","last_fresh":"2023-06-03T20:36:25.444583+0000","last_change":"2023-06-03T19:56:54.510289+0000","last_active":"2023-06-03T20:36:25.444583+0000","last_peered":"2023-06-03T20:36:25.444583+0000","last_clean":"2023-06-03T20:36:25.444583+0000","last_became_active":"2023-06-03T19:56:54.509905+0000","last_became_peered":"2023-06-03T19:56:54.509905+0000","last_unstale":"2023-06-03T20:36:25.444583+0000","last_undegraded":"2023-06-03T20:36:25.444583+0000","last_fullsized":"2023-06-03T20:36:25.444583+0000","mapping_epoch":119,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T02:54:26.792401+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"60'8","reported_seq":89,"reported_epoch":323,"state":"active+clean","last_fresh":"2023-06-03T20:36:16.309953+0000","last_change":"2023-06-03T20:26:59.682623+0000","last_active":"2023-06-03T20:36:16.309953+0000","last_peered":"2023-06-03T20:36:16.309953+0000","last_clean":"2023-06-03T20:36:16.309953+0000","last_became_active":"2023-06-03T20:24:02.260105+0000","last_became_peered":"2023-06-03T20:24:02.260105+0000","last_unstale":"2023-06-03T20:36:16.309953+0000","last_undegraded":"2023-06-03T20:36:16.309953+0000","last_fullsized":"2023-06-03T20:36:16.309953+0000","mapping_epoch":254,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":273,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":331,"reported_epoch":324,"state":"active+clean","last_fresh":"2023-06-03T20:36:26.547073+0000","last_change":"2023-06-03T20:28:24.281308+0000","last_active":"2023-06-03T20:36:26.547073+0000","last_peered":"2023-06-03T20:36:26.547073+0000","last_clean":"2023-06-03T20:36:26.547073+0000","last_became_active":"2023-06-03T20:28:24.280981+0000","last_became_peered":"2023-06-03T20:28:24.280981+0000","last_unstale":"2023-06-03T20:36:26.547073+0000","last_undegraded":"2023-06-03T20:36:26.547073+0000","last_fullsized":"2023-06-03T20:36:26.547073+0000","mapping_epoch":284,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":285,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T02:44:14.890147+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.f","version":"60'1","reported_seq":322,"reported_epoch":323,"state":"active+clean","last_fresh":"2023-06-03T20:36:15.735420+0000","last_change":"2023-06-03T20:23:27.263667+0000","last_active":"2023-06-03T20:36:15.735420+0000","last_peered":"2023-06-03T20:36:15.735420+0000","last_clean":"2023-06-03T20:36:15.735420+0000","last_became_active":"2023-06-03T20:23:27.263110+0000","last_became_peered":"2023-06-03T20:23:27.263110+0000","last_unstale":"2023-06-03T20:36:15.735420+0000","last_undegraded":"2023-06-03T20:36:15.735420+0000","last_fullsized":"2023-06-03T20:36:15.735420+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":255,"reported_epoch":323,"state":"active+clean","last_fresh":"2023-06-03T20:36:16.310100+0000","last_change":"2023-06-03T20:35:26.391598+0000","last_active":"2023-06-03T20:36:16.310100+0000","last_peered":"2023-06-03T20:36:16.310100+0000","last_clean":"2023-06-03T20:36:16.310100+0000","last_became_active":"2023-06-03T20:35:26.391038+0000","last_became_peered":"2023-06-03T20:35:26.391038+0000","last_unstale":"2023-06-03T20:36:16.310100+0000","last_undegraded":"2023-06-03T20:36:16.310100+0000","last_fullsized":"2023-06-03T20:36:16.310100+0000","mapping_epoch":317,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.e","version":"60'2","reported_seq":351,"reported_epoch":323,"state":"active+clean","last_fresh":"2023-06-03T20:36:15.736060+0000","last_change":"2023-06-03T20:23:28.606751+0000","last_active":"2023-06-03T20:36:15.736060+0000","last_peered":"2023-06-03T20:36:15.736060+0000","last_clean":"2023-06-03T20:36:15.736060+0000","last_became_active":"2023-06-03T20:23:28.605968+0000","last_became_peered":"2023-06-03T20:23:28.605968+0000","last_unstale":"2023-06-03T20:36:15.736060+0000","last_undegraded":"2023-06-03T20:36:15.736060+0000","last_fullsized":"2023-06-03T20:36:15.736060+0000","mapping_epoch":251,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":252,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":266,"reported_epoch":324,"state":"active+clean","last_fresh":"2023-06-03T20:36:25.200011+0000","last_change":"2023-06-03T20:23:28.249694+0000","last_active":"2023-06-03T20:36:25.200011+0000","last_peered":"2023-06-03T20:36:25.200011+0000","last_clean":"2023-06-03T20:36:25.200011+0000","last_became_active":"2023-06-03T20:23:28.249388+0000","last_became_peered":"2023-06-03T20:23:28.249388+0000","last_unstale":"2023-06-03T20:36:25.200011+0000","last_undegraded":"2023-06-03T20:36:25.200011+0000","last_fullsized":"2023-06-03T20:36:25.200011+0000","mapping_epoch":251,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":252,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T05:44:01.835586+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.d","version":"60'1","reported_seq":300,"reported_epoch":323,"state":"active+clean","last_fresh":"2023-06-03T20:36:15.735967+0000","last_change":"2023-06-03T20:26:59.686037+0000","last_active":"2023-06-03T20:36:15.735967+0000","last_peered":"2023-06-03T20:36:15.735967+0000","last_clean":"2023-06-03T20:36:15.735967+0000","last_became_active":"2023-06-03T20:24:02.268002+0000","last_became_peered":"2023-06-03T20:24:02.268002+0000","last_unstale":"2023-06-03T20:36:15.735967+0000","last_undegraded":"2023-06-03T20:36:15.735967+0000","last_fullsized":"2023-06-03T20:36:15.735967+0000","mapping_epoch":254,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":273,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":140,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":47,"reported_epoch":323,"state":"active+clean","last_fresh":"2023-06-03T20:36:16.310081+0000","last_change":"2023-06-03T20:28:51.230833+0000","last_active":"2023-06-03T20:36:16.310081+0000","last_peered":"2023-06-03T20:36:16.310081+0000","last_clean":"2023-06-03T20:36:16.310081+0000","last_became_active":"2023-06-03T20:28:51.230489+0000","last_became_peered":"2023-06-03T20:28:51.230489+0000","last_unstale":"2023-06-03T20:36:16.310081+0000","last_undegraded":"2023-06-03T20:36:16.310081+0000","last_fullsized":"2023-06-03T20:36:16.310081+0000","mapping_epoch":286,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":287,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":253,"reported_epoch":323,"state":"active+clean","last_fresh":"2023-06-03T20:36:16.311181+0000","last_change":"2023-06-03T20:35:26.592536+0000","last_active":"2023-06-03T20:36:16.311181+0000","last_peered":"2023-06-03T20:36:16.311181+0000","last_clean":"2023-06-03T20:36:16.311181+0000","last_became_active":"2023-06-03T20:35:26.591580+0000","last_became_peered":"2023-06-03T20:35:26.591580+0000","last_unstale":"2023-06-03T20:36:16.311181+0000","last_undegraded":"2023-06-03T20:36:16.311181+0000","last_fullsized":"2023-06-03T20:36:16.311181+0000","mapping_epoch":317,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":357,"reported_epoch":324,"state":"active+clean","last_fresh":"2023-06-03T20:36:25.444317+0000","last_change":"2023-06-03T20:23:28.283360+0000","last_active":"2023-06-03T20:36:25.444317+0000","last_peered":"2023-06-03T20:36:25.444317+0000","last_clean":"2023-06-03T20:36:25.444317+0000","last_became_active":"2023-06-03T20:23:28.283035+0000","last_became_peered":"2023-06-03T20:23:28.283035+0000","last_unstale":"2023-06-03T20:36:25.444317+0000","last_undegraded":"2023-06-03T20:36:25.444317+0000","last_fullsized":"2023-06-03T20:36:25.444317+0000","mapping_epoch":251,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":252,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T00:22:59.446515+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.b","version":"60'2","reported_seq":321,"reported_epoch":324,"state":"active+clean","last_fresh":"2023-06-03T20:36:26.546141+0000","last_change":"2023-06-03T20:23:27.254179+0000","last_active":"2023-06-03T20:36:26.546141+0000","last_peered":"2023-06-03T20:36:26.546141+0000","last_clean":"2023-06-03T20:36:26.546141+0000","last_became_active":"2023-06-03T20:23:27.253791+0000","last_became_peered":"2023-06-03T20:23:27.253791+0000","last_unstale":"2023-06-03T20:36:26.546141+0000","last_undegraded":"2023-06-03T20:36:26.546141+0000","last_fullsized":"2023-06-03T20:36:26.546141+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T00:31:27.336760+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":242,"reported_epoch":324,"state":"active+clean","last_fresh":"2023-06-03T20:36:25.200243+0000","last_change":"2023-06-03T20:24:02.255551+0000","last_active":"2023-06-03T20:36:25.200243+0000","last_peered":"2023-06-03T20:36:25.200243+0000","last_clean":"2023-06-03T20:36:25.200243+0000","last_became_active":"2023-06-03T20:24:02.255209+0000","last_became_peered":"2023-06-03T20:24:02.255209+0000","last_unstale":"2023-06-03T20:36:25.200243+0000","last_undegraded":"2023-06-03T20:36:25.200243+0000","last_fullsized":"2023-06-03T20:36:25.200243+0000","mapping_epoch":254,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":255,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T20:05:02.741217+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":79,"reported_epoch":323,"state":"active+clean","last_fresh":"2023-06-03T20:36:16.310521+0000","last_change":"2023-06-03T20:24:02.254172+0000","last_active":"2023-06-03T20:36:16.310521+0000","last_peered":"2023-06-03T20:36:16.310521+0000","last_clean":"2023-06-03T20:36:16.310521+0000","last_became_active":"2023-06-03T20:24:02.253797+0000","last_became_peered":"2023-06-03T20:24:02.253797+0000","last_unstale":"2023-06-03T20:36:16.310521+0000","last_undegraded":"2023-06-03T20:36:16.310521+0000","last_fullsized":"2023-06-03T20:36:16.310521+0000","mapping_epoch":254,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":255,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":247,"reported_epoch":324,"state":"active+clean","last_fresh":"2023-06-03T20:36:25.200494+0000","last_change":"2023-06-03T20:23:28.291250+0000","last_active":"2023-06-03T20:36:25.200494+0000","last_peered":"2023-06-03T20:36:25.200494+0000","last_clean":"2023-06-03T20:36:25.200494+0000","last_became_active":"2023-06-03T20:23:28.290120+0000","last_became_peered":"2023-06-03T20:23:28.290120+0000","last_unstale":"2023-06-03T20:36:25.200494+0000","last_undegraded":"2023-06-03T20:36:25.200494+0000","last_fullsized":"2023-06-03T20:36:25.200494+0000","mapping_epoch":251,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":252,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T22:37:56.033327+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":232,"reported_epoch":324,"state":"active+clean","last_fresh":"2023-06-03T20:36:25.199846+0000","last_change":"2023-06-03T20:23:27.276693+0000","last_active":"2023-06-03T20:36:25.199846+0000","last_peered":"2023-06-03T20:36:25.199846+0000","last_clean":"2023-06-03T20:36:25.199846+0000","last_became_active":"2023-06-03T20:23:27.276305+0000","last_became_peered":"2023-06-03T20:23:27.276305+0000","last_unstale":"2023-06-03T20:36:25.199846+0000","last_undegraded":"2023-06-03T20:36:25.199846+0000","last_fullsized":"2023-06-03T20:36:25.199846+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T20:34:50.944936+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":237,"reported_epoch":324,"state":"active+clean","last_fresh":"2023-06-03T20:36:25.200666+0000","last_change":"2023-06-03T20:28:51.231829+0000","last_active":"2023-06-03T20:36:25.200666+0000","last_peered":"2023-06-03T20:36:25.200666+0000","last_clean":"2023-06-03T20:36:25.200666+0000","last_became_active":"2023-06-03T20:28:51.231452+0000","last_became_peered":"2023-06-03T20:28:51.231452+0000","last_unstale":"2023-06-03T20:36:25.200666+0000","last_undegraded":"2023-06-03T20:36:25.200666+0000","last_fullsized":"2023-06-03T20:36:25.200666+0000","mapping_epoch":286,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":287,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T07:00:21.647058+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":330,"reported_epoch":323,"state":"active+clean","last_fresh":"2023-06-03T20:36:16.309812+0000","last_change":"2023-06-03T20:21:01.256958+0000","last_active":"2023-06-03T20:36:16.309812+0000","last_peered":"2023-06-03T20:36:16.309812+0000","last_clean":"2023-06-03T20:36:16.309812+0000","last_became_active":"2023-06-03T20:21:01.255452+0000","last_became_peered":"2023-06-03T20:21:01.255452+0000","last_unstale":"2023-06-03T20:36:16.309812+0000","last_undegraded":"2023-06-03T20:36:16.309812+0000","last_fullsized":"2023-06-03T20:36:16.309812+0000","mapping_epoch":235,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":236,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T01:56:59.777763+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":117,"reported_epoch":324,"state":"active+clean","last_fresh":"2023-06-03T20:36:26.547834+0000","last_change":"2023-06-03T20:23:28.250884+0000","last_active":"2023-06-03T20:36:26.547834+0000","last_peered":"2023-06-03T20:36:26.547834+0000","last_clean":"2023-06-03T20:36:26.547834+0000","last_became_active":"2023-06-03T20:23:28.243793+0000","last_became_peered":"2023-06-03T20:23:28.243793+0000","last_unstale":"2023-06-03T20:36:26.547834+0000","last_undegraded":"2023-06-03T20:36:26.547834+0000","last_fullsized":"2023-06-03T20:36:26.547834+0000","mapping_epoch":251,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":252,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T03:12:27.946002+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":107,"reported_epoch":324,"state":"active+clean","last_fresh":"2023-06-03T20:36:26.547657+0000","last_change":"2023-06-03T20:21:01.258182+0000","last_active":"2023-06-03T20:36:26.547657+0000","last_peered":"2023-06-03T20:36:26.547657+0000","last_clean":"2023-06-03T20:36:26.547657+0000","last_became_active":"2023-06-03T20:21:01.254049+0000","last_became_peered":"2023-06-03T20:21:01.254049+0000","last_unstale":"2023-06-03T20:36:26.547657+0000","last_undegraded":"2023-06-03T20:36:26.547657+0000","last_fullsized":"2023-06-03T20:36:26.547657+0000","mapping_epoch":235,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":236,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:32:20.692043+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:32:20.692043+0000","last_clean_scrub_stamp":"2023-06-03T19:32:20.692043+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T05:16:30.078473+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":220,"reported_epoch":323,"state":"active+clean","last_fresh":"2023-06-03T20:36:16.311374+0000","last_change":"2023-06-03T20:35:26.592542+0000","last_active":"2023-06-03T20:36:16.311374+0000","last_peered":"2023-06-03T20:36:16.311374+0000","last_clean":"2023-06-03T20:36:16.311374+0000","last_became_active":"2023-06-03T20:35:26.591632+0000","last_became_peered":"2023-06-03T20:35:26.591632+0000","last_unstale":"2023-06-03T20:36:16.311374+0000","last_undegraded":"2023-06-03T20:36:16.311374+0000","last_fullsized":"2023-06-03T20:36:16.311374+0000","mapping_epoch":317,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":118,"reported_epoch":324,"state":"active+clean","last_fresh":"2023-06-03T20:36:26.547103+0000","last_change":"2023-06-03T20:23:28.254434+0000","last_active":"2023-06-03T20:36:26.547103+0000","last_peered":"2023-06-03T20:36:26.547103+0000","last_clean":"2023-06-03T20:36:26.547103+0000","last_became_active":"2023-06-03T20:23:28.254012+0000","last_became_peered":"2023-06-03T20:23:28.254012+0000","last_unstale":"2023-06-03T20:36:26.547103+0000","last_undegraded":"2023-06-03T20:36:26.547103+0000","last_fullsized":"2023-06-03T20:36:26.547103+0000","mapping_epoch":251,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":252,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T06:22:09.540298+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":227,"reported_epoch":323,"state":"active+clean","last_fresh":"2023-06-03T20:36:16.310964+0000","last_change":"2023-06-03T20:35:26.203105+0000","last_active":"2023-06-03T20:36:16.310964+0000","last_peered":"2023-06-03T20:36:16.310964+0000","last_clean":"2023-06-03T20:36:16.310964+0000","last_became_active":"2023-06-03T20:35:26.202634+0000","last_became_peered":"2023-06-03T20:35:26.202634+0000","last_unstale":"2023-06-03T20:36:16.310964+0000","last_undegraded":"2023-06-03T20:36:16.310964+0000","last_fullsized":"2023-06-03T20:36:16.310964+0000","mapping_epoch":317,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:32:20.692043+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:32:20.692043+0000","last_clean_scrub_stamp":"2023-06-03T19:32:20.692043+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":212,"reported_epoch":323,"state":"active+clean","last_fresh":"2023-06-03T20:36:15.510565+0000","last_change":"2023-06-03T20:24:02.258311+0000","last_active":"2023-06-03T20:36:15.510565+0000","last_peered":"2023-06-03T20:36:15.510565+0000","last_clean":"2023-06-03T20:36:15.510565+0000","last_became_active":"2023-06-03T20:24:02.257676+0000","last_became_peered":"2023-06-03T20:24:02.257676+0000","last_unstale":"2023-06-03T20:36:15.510565+0000","last_undegraded":"2023-06-03T20:36:15.510565+0000","last_fullsized":"2023-06-03T20:36:15.510565+0000","mapping_epoch":254,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":255,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T01:47:46.383038+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":107,"reported_epoch":324,"state":"active+clean","last_fresh":"2023-06-03T20:36:26.547834+0000","last_change":"2023-06-03T20:21:01.259331+0000","last_active":"2023-06-03T20:36:26.547834+0000","last_peered":"2023-06-03T20:36:26.547834+0000","last_clean":"2023-06-03T20:36:26.547834+0000","last_became_active":"2023-06-03T20:21:01.258836+0000","last_became_peered":"2023-06-03T20:21:01.258836+0000","last_unstale":"2023-06-03T20:36:26.547834+0000","last_undegraded":"2023-06-03T20:36:26.547834+0000","last_fullsized":"2023-06-03T20:36:26.547834+0000","mapping_epoch":235,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":236,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T01:29:19.471600+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":314,"reported_epoch":324,"state":"active+clean","last_fresh":"2023-06-03T20:36:26.310674+0000","last_change":"2023-06-03T19:56:54.504859+0000","last_active":"2023-06-03T20:36:26.310674+0000","last_peered":"2023-06-03T20:36:26.310674+0000","last_clean":"2023-06-03T20:36:26.310674+0000","last_became_active":"2023-06-03T19:56:54.504455+0000","last_became_peered":"2023-06-03T19:56:54.504455+0000","last_unstale":"2023-06-03T20:36:26.310674+0000","last_undegraded":"2023-06-03T20:36:26.310674+0000","last_fullsized":"2023-06-03T20:36:26.310674+0000","mapping_epoch":119,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:32:20.692043+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:32:20.692043+0000","last_clean_scrub_stamp":"2023-06-03T19:32:20.692043+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T06:25:45.522100+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":116,"reported_epoch":324,"state":"active+clean","last_fresh":"2023-06-03T20:36:26.546653+0000","last_change":"2023-06-03T20:24:02.256677+0000","last_active":"2023-06-03T20:36:26.546653+0000","last_peered":"2023-06-03T20:36:26.546653+0000","last_clean":"2023-06-03T20:36:26.546653+0000","last_became_active":"2023-06-03T20:24:02.256216+0000","last_became_peered":"2023-06-03T20:24:02.256216+0000","last_unstale":"2023-06-03T20:36:26.546653+0000","last_undegraded":"2023-06-03T20:36:26.546653+0000","last_fullsized":"2023-06-03T20:36:26.546653+0000","mapping_epoch":254,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":255,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T02:32:45.186204+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":106,"reported_epoch":323,"state":"active+clean","last_fresh":"2023-06-03T20:36:16.312549+0000","last_change":"2023-06-03T20:21:01.261473+0000","last_active":"2023-06-03T20:36:16.312549+0000","last_peered":"2023-06-03T20:36:16.312549+0000","last_clean":"2023-06-03T20:36:16.312549+0000","last_became_active":"2023-06-03T20:21:01.261098+0000","last_became_peered":"2023-06-03T20:21:01.261098+0000","last_unstale":"2023-06-03T20:36:16.312549+0000","last_undegraded":"2023-06-03T20:36:16.312549+0000","last_fullsized":"2023-06-03T20:36:16.312549+0000","mapping_epoch":235,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":236,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T03:56:22.986549+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":209,"reported_epoch":323,"state":"active+clean","last_fresh":"2023-06-03T20:36:16.311275+0000","last_change":"2023-06-03T20:35:26.391589+0000","last_active":"2023-06-03T20:36:16.311275+0000","last_peered":"2023-06-03T20:36:16.311275+0000","last_clean":"2023-06-03T20:36:16.311275+0000","last_became_active":"2023-06-03T20:35:26.390813+0000","last_became_peered":"2023-06-03T20:35:26.390813+0000","last_unstale":"2023-06-03T20:36:16.311275+0000","last_undegraded":"2023-06-03T20:36:16.311275+0000","last_fullsized":"2023-06-03T20:36:16.311275+0000","mapping_epoch":317,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:32:20.692043+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:32:20.692043+0000","last_clean_scrub_stamp":"2023-06-03T19:32:20.692043+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"72'139","reported_seq":596,"reported_epoch":323,"state":"active+clean","last_fresh":"2023-06-03T20:36:15.735251+0000","last_change":"2023-06-03T20:27:01.248641+0000","last_active":"2023-06-03T20:36:15.735251+0000","last_peered":"2023-06-03T20:36:15.735251+0000","last_clean":"2023-06-03T20:36:15.735251+0000","last_became_active":"2023-06-03T20:27:01.248063+0000","last_became_peered":"2023-06-03T20:27:01.248063+0000","last_unstale":"2023-06-03T20:36:15.735251+0000","last_undegraded":"2023-06-03T20:36:15.735251+0000","last_fullsized":"2023-06-03T20:36:15.735251+0000","mapping_epoch":274,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":275,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:27:28.100808+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:27:28.100808+0000","last_clean_scrub_stamp":"2023-06-03T19:27:28.100808+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":316,"reported_epoch":324,"state":"active+clean","last_fresh":"2023-06-03T20:36:26.545437+0000","last_change":"2023-06-03T20:24:02.263360+0000","last_active":"2023-06-03T20:36:26.545437+0000","last_peered":"2023-06-03T20:36:26.545437+0000","last_clean":"2023-06-03T20:36:26.545437+0000","last_became_active":"2023-06-03T20:24:02.262954+0000","last_became_peered":"2023-06-03T20:24:02.262954+0000","last_unstale":"2023-06-03T20:36:26.545437+0000","last_undegraded":"2023-06-03T20:36:26.545437+0000","last_fullsized":"2023-06-03T20:36:26.545437+0000","mapping_epoch":254,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":255,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T19:38:46.084731+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":47,"reported_epoch":323,"state":"active+clean","last_fresh":"2023-06-03T20:36:16.310086+0000","last_change":"2023-06-03T20:28:51.234600+0000","last_active":"2023-06-03T20:36:16.310086+0000","last_peered":"2023-06-03T20:36:16.310086+0000","last_clean":"2023-06-03T20:36:16.310086+0000","last_became_active":"2023-06-03T20:28:51.234227+0000","last_became_peered":"2023-06-03T20:28:51.234227+0000","last_unstale":"2023-06-03T20:36:16.310086+0000","last_undegraded":"2023-06-03T20:36:16.310086+0000","last_fullsized":"2023-06-03T20:36:16.310086+0000","mapping_epoch":286,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":287,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":316,"reported_epoch":323,"state":"active+clean","last_fresh":"2023-06-03T20:36:15.735240+0000","last_change":"2023-06-03T20:21:01.258113+0000","last_active":"2023-06-03T20:36:15.735240+0000","last_peered":"2023-06-03T20:36:15.735240+0000","last_clean":"2023-06-03T20:36:15.735240+0000","last_became_active":"2023-06-03T20:21:01.257488+0000","last_became_peered":"2023-06-03T20:21:01.257488+0000","last_unstale":"2023-06-03T20:36:15.735240+0000","last_undegraded":"2023-06-03T20:36:15.735240+0000","last_fullsized":"2023-06-03T20:36:15.735240+0000","mapping_epoch":235,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":236,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:32:20.692043+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:32:20.692043+0000","last_clean_scrub_stamp":"2023-06-03T19:32:20.692043+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":326,"reported_epoch":324,"state":"active+clean","last_fresh":"2023-06-03T20:36:25.444164+0000","last_change":"2023-06-03T20:23:27.255164+0000","last_active":"2023-06-03T20:36:25.444164+0000","last_peered":"2023-06-03T20:36:25.444164+0000","last_clean":"2023-06-03T20:36:25.444164+0000","last_became_active":"2023-06-03T20:23:27.253932+0000","last_became_peered":"2023-06-03T20:23:27.253932+0000","last_unstale":"2023-06-03T20:36:25.444164+0000","last_undegraded":"2023-06-03T20:36:25.444164+0000","last_fullsized":"2023-06-03T20:36:25.444164+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T06:42:02.303554+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":329,"reported_epoch":324,"state":"active+clean","last_fresh":"2023-06-03T20:36:25.444659+0000","last_change":"2023-06-03T20:21:01.250266+0000","last_active":"2023-06-03T20:36:25.444659+0000","last_peered":"2023-06-03T20:36:25.444659+0000","last_clean":"2023-06-03T20:36:25.444659+0000","last_became_active":"2023-06-03T20:21:01.249839+0000","last_became_peered":"2023-06-03T20:21:01.249839+0000","last_unstale":"2023-06-03T20:36:25.444659+0000","last_undegraded":"2023-06-03T20:36:25.444659+0000","last_fullsized":"2023-06-03T20:36:25.444659+0000","mapping_epoch":235,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":236,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T05:13:26.584607+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":325,"reported_epoch":323,"state":"active+clean","last_fresh":"2023-06-03T20:36:15.735164+0000","last_change":"2023-06-03T20:23:27.261511+0000","last_active":"2023-06-03T20:36:15.735164+0000","last_peered":"2023-06-03T20:36:15.735164+0000","last_clean":"2023-06-03T20:36:15.735164+0000","last_became_active":"2023-06-03T20:23:27.259622+0000","last_became_peered":"2023-06-03T20:23:27.259622+0000","last_unstale":"2023-06-03T20:36:15.735164+0000","last_undegraded":"2023-06-03T20:36:15.735164+0000","last_fullsized":"2023-06-03T20:36:15.735164+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:32:20.692043+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:32:20.692043+0000","last_clean_scrub_stamp":"2023-06-03T19:32:20.692043+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":107,"reported_epoch":324,"state":"active+clean","last_fresh":"2023-06-03T20:36:26.547662+0000","last_change":"2023-06-03T20:21:01.252859+0000","last_active":"2023-06-03T20:36:26.547662+0000","last_peered":"2023-06-03T20:36:26.547662+0000","last_clean":"2023-06-03T20:36:26.547662+0000","last_became_active":"2023-06-03T20:21:01.251222+0000","last_became_peered":"2023-06-03T20:21:01.251222+0000","last_unstale":"2023-06-03T20:36:26.547662+0000","last_undegraded":"2023-06-03T20:36:26.547662+0000","last_fullsized":"2023-06-03T20:36:26.547662+0000","mapping_epoch":235,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":236,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T05:19:58.100226+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":282,"reported_epoch":324,"state":"active+clean","last_fresh":"2023-06-03T20:36:25.444441+0000","last_change":"2023-06-03T19:47:06.733532+0000","last_active":"2023-06-03T20:36:25.444441+0000","last_peered":"2023-06-03T20:36:25.444441+0000","last_clean":"2023-06-03T20:36:25.444441+0000","last_became_active":"2023-06-03T19:47:06.733131+0000","last_became_peered":"2023-06-03T19:47:06.733131+0000","last_unstale":"2023-06-03T20:36:25.444441+0000","last_undegraded":"2023-06-03T20:36:25.444441+0000","last_fullsized":"2023-06-03T20:36:25.444441+0000","mapping_epoch":86,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":87,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T23:46:03.788646+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.7","version":"60'2","reported_seq":291,"reported_epoch":323,"state":"active+clean","last_fresh":"2023-06-03T20:36:16.310239+0000","last_change":"2023-06-03T19:56:54.510148+0000","last_active":"2023-06-03T20:36:16.310239+0000","last_peered":"2023-06-03T20:36:16.310239+0000","last_clean":"2023-06-03T20:36:16.310239+0000","last_became_active":"2023-06-03T19:56:54.509771+0000","last_became_peered":"2023-06-03T19:56:54.509771+0000","last_unstale":"2023-06-03T20:36:16.310239+0000","last_undegraded":"2023-06-03T20:36:16.310239+0000","last_fullsized":"2023-06-03T20:36:16.310239+0000","mapping_epoch":119,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T00:52:42.807432+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":124,"reported_epoch":323,"state":"active+clean","last_fresh":"2023-06-03T20:36:16.312159+0000","last_change":"2023-06-03T20:24:02.257142+0000","last_active":"2023-06-03T20:36:16.312159+0000","last_peered":"2023-06-03T20:36:16.312159+0000","last_clean":"2023-06-03T20:36:16.312159+0000","last_became_active":"2023-06-03T20:24:02.256483+0000","last_became_peered":"2023-06-03T20:24:02.256483+0000","last_unstale":"2023-06-03T20:36:16.312159+0000","last_undegraded":"2023-06-03T20:36:16.312159+0000","last_fullsized":"2023-06-03T20:36:16.312159+0000","mapping_epoch":254,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":255,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:32:20.692043+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:32:20.692043+0000","last_clean_scrub_stamp":"2023-06-03T19:32:20.692043+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T00:53:37.197795+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":290,"reported_epoch":324,"state":"active+clean","last_fresh":"2023-06-03T20:36:26.545539+0000","last_change":"2023-06-03T19:56:54.507670+0000","last_active":"2023-06-03T20:36:26.545539+0000","last_peered":"2023-06-03T20:36:26.545539+0000","last_clean":"2023-06-03T20:36:26.545539+0000","last_became_active":"2023-06-03T19:56:54.507293+0000","last_became_peered":"2023-06-03T19:56:54.507293+0000","last_unstale":"2023-06-03T20:36:26.545539+0000","last_undegraded":"2023-06-03T20:36:26.545539+0000","last_fullsized":"2023-06-03T20:36:26.545539+0000","mapping_epoch":119,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T01:49:19.966054+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":79,"reported_epoch":323,"state":"active+clean","last_fresh":"2023-06-03T20:36:16.310480+0000","last_change":"2023-06-03T20:24:02.319230+0000","last_active":"2023-06-03T20:36:16.310480+0000","last_peered":"2023-06-03T20:36:16.310480+0000","last_clean":"2023-06-03T20:36:16.310480+0000","last_became_active":"2023-06-03T20:24:02.318926+0000","last_became_peered":"2023-06-03T20:24:02.318926+0000","last_unstale":"2023-06-03T20:36:16.310480+0000","last_undegraded":"2023-06-03T20:36:16.310480+0000","last_fullsized":"2023-06-03T20:36:16.310480+0000","mapping_epoch":254,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":255,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:32:20.692043+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:32:20.692043+0000","last_clean_scrub_stamp":"2023-06-03T19:32:20.692043+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"60'1","reported_seq":357,"reported_epoch":324,"state":"active+clean","last_fresh":"2023-06-03T20:36:25.444404+0000","last_change":"2023-06-03T20:23:28.239664+0000","last_active":"2023-06-03T20:36:25.444404+0000","last_peered":"2023-06-03T20:36:25.444404+0000","last_clean":"2023-06-03T20:36:25.444404+0000","last_became_active":"2023-06-03T20:23:28.239252+0000","last_became_peered":"2023-06-03T20:23:28.239252+0000","last_unstale":"2023-06-03T20:36:25.444404+0000","last_undegraded":"2023-06-03T20:36:25.444404+0000","last_fullsized":"2023-06-03T20:36:25.444404+0000","mapping_epoch":251,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":252,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T06:52:20.862520+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":210,"reported_epoch":323,"state":"active+clean","last_fresh":"2023-06-03T20:36:15.510224+0000","last_change":"2023-06-03T20:24:02.259850+0000","last_active":"2023-06-03T20:36:15.510224+0000","last_peered":"2023-06-03T20:36:15.510224+0000","last_clean":"2023-06-03T20:36:15.510224+0000","last_became_active":"2023-06-03T20:24:02.258764+0000","last_became_peered":"2023-06-03T20:24:02.258764+0000","last_unstale":"2023-06-03T20:36:15.510224+0000","last_undegraded":"2023-06-03T20:36:15.510224+0000","last_fullsized":"2023-06-03T20:36:15.510224+0000","mapping_epoch":254,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":255,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T21:54:11.487473+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"60'1","reported_seq":311,"reported_epoch":324,"state":"active+clean","last_fresh":"2023-06-03T20:36:25.444388+0000","last_change":"2023-06-03T20:23:27.254853+0000","last_active":"2023-06-03T20:36:25.444388+0000","last_peered":"2023-06-03T20:36:25.444388+0000","last_clean":"2023-06-03T20:36:25.444388+0000","last_became_active":"2023-06-03T20:23:27.254410+0000","last_became_peered":"2023-06-03T20:23:27.254410+0000","last_unstale":"2023-06-03T20:36:25.444388+0000","last_undegraded":"2023-06-03T20:36:25.444388+0000","last_fullsized":"2023-06-03T20:36:25.444388+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T22:46:19.423939+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":340,"reported_epoch":323,"state":"active+clean","last_fresh":"2023-06-03T20:36:15.195645+0000","last_change":"2023-06-03T20:21:01.257656+0000","last_active":"2023-06-03T20:36:15.195645+0000","last_peered":"2023-06-03T20:36:15.195645+0000","last_clean":"2023-06-03T20:36:15.195645+0000","last_became_active":"2023-06-03T20:21:01.257107+0000","last_became_peered":"2023-06-03T20:21:01.257107+0000","last_unstale":"2023-06-03T20:36:15.195645+0000","last_undegraded":"2023-06-03T20:36:15.195645+0000","last_fullsized":"2023-06-03T20:36:15.195645+0000","mapping_epoch":235,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":236,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T06:08:18.954539+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"60'2","reported_seq":257,"reported_epoch":323,"state":"active+clean","last_fresh":"2023-06-03T20:36:16.311222+0000","last_change":"2023-06-03T20:35:26.203752+0000","last_active":"2023-06-03T20:36:16.311222+0000","last_peered":"2023-06-03T20:36:16.311222+0000","last_clean":"2023-06-03T20:36:16.311222+0000","last_became_active":"2023-06-03T20:35:26.203286+0000","last_became_peered":"2023-06-03T20:35:26.203286+0000","last_unstale":"2023-06-03T20:36:16.311222+0000","last_undegraded":"2023-06-03T20:36:16.311222+0000","last_fullsized":"2023-06-03T20:36:16.311222+0000","mapping_epoch":317,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":264,"reported_epoch":323,"state":"active+clean","last_fresh":"2023-06-03T20:36:15.510253+0000","last_change":"2023-06-03T20:28:24.276513+0000","last_active":"2023-06-03T20:36:15.510253+0000","last_peered":"2023-06-03T20:36:15.510253+0000","last_clean":"2023-06-03T20:36:15.510253+0000","last_became_active":"2023-06-03T20:28:24.276244+0000","last_became_peered":"2023-06-03T20:28:24.276244+0000","last_unstale":"2023-06-03T20:36:15.510253+0000","last_undegraded":"2023-06-03T20:36:15.510253+0000","last_fullsized":"2023-06-03T20:36:15.510253+0000","mapping_epoch":284,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":285,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T06:17:57.159168+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[4,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"60'2","reported_seq":229,"reported_epoch":323,"state":"active+clean","last_fresh":"2023-06-03T20:36:16.311284+0000","last_change":"2023-06-03T20:35:26.204044+0000","last_active":"2023-06-03T20:36:16.311284+0000","last_peered":"2023-06-03T20:36:16.311284+0000","last_clean":"2023-06-03T20:36:16.311284+0000","last_became_active":"2023-06-03T20:35:26.203591+0000","last_became_peered":"2023-06-03T20:35:26.203591+0000","last_unstale":"2023-06-03T20:36:16.311284+0000","last_undegraded":"2023-06-03T20:36:16.311284+0000","last_fullsized":"2023-06-03T20:36:16.311284+0000","mapping_epoch":317,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":248,"reported_epoch":323,"state":"active+clean","last_fresh":"2023-06-03T20:36:15.509616+0000","last_change":"2023-06-03T20:23:27.254468+0000","last_active":"2023-06-03T20:36:15.509616+0000","last_peered":"2023-06-03T20:36:15.509616+0000","last_clean":"2023-06-03T20:36:15.509616+0000","last_became_active":"2023-06-03T20:23:27.253859+0000","last_became_peered":"2023-06-03T20:23:27.253859+0000","last_unstale":"2023-06-03T20:36:15.509616+0000","last_undegraded":"2023-06-03T20:36:15.509616+0000","last_fullsized":"2023-06-03T20:36:15.509616+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T20:15:25.066125+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"60'1","reported_seq":346,"reported_epoch":324,"state":"active+clean","last_fresh":"2023-06-03T20:36:26.546976+0000","last_change":"2023-06-03T20:28:51.234748+0000","last_active":"2023-06-03T20:36:26.546976+0000","last_peered":"2023-06-03T20:36:26.546976+0000","last_clean":"2023-06-03T20:36:26.546976+0000","last_became_active":"2023-06-03T20:28:51.234341+0000","last_became_peered":"2023-06-03T20:28:51.234341+0000","last_unstale":"2023-06-03T20:36:26.546976+0000","last_undegraded":"2023-06-03T20:36:26.546976+0000","last_fullsized":"2023-06-03T20:36:26.546976+0000","mapping_epoch":286,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":287,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T06:08:49.901424+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":328,"reported_epoch":323,"state":"active+clean","last_fresh":"2023-06-03T20:36:16.309895+0000","last_change":"2023-06-03T19:56:54.503416+0000","last_active":"2023-06-03T20:36:16.309895+0000","last_peered":"2023-06-03T20:36:16.309895+0000","last_clean":"2023-06-03T20:36:16.309895+0000","last_became_active":"2023-06-03T19:56:54.503018+0000","last_became_peered":"2023-06-03T19:56:54.503018+0000","last_unstale":"2023-06-03T20:36:16.309895+0000","last_undegraded":"2023-06-03T20:36:16.309895+0000","last_fullsized":"2023-06-03T20:36:16.309895+0000","mapping_epoch":119,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-04T20:14:00.330616+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"60'4","reported_seq":300,"reported_epoch":324,"state":"active+clean","last_fresh":"2023-06-03T20:36:25.444697+0000","last_change":"2023-06-03T19:56:11.748116+0000","last_active":"2023-06-03T20:36:25.444697+0000","last_peered":"2023-06-03T20:36:25.444697+0000","last_clean":"2023-06-03T20:36:25.444697+0000","last_became_active":"2023-06-03T19:56:11.747636+0000","last_became_peered":"2023-06-03T19:56:11.747636+0000","last_unstale":"2023-06-03T20:36:25.444697+0000","last_undegraded":"2023-06-03T20:36:25.444697+0000","last_fullsized":"2023-06-03T20:36:25.444697+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":115,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:15.767991+0000","last_clean_scrub_stamp":"2023-06-03T19:34:15.767991+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T06:57:48.052055+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":248,"reported_epoch":323,"state":"active+clean","last_fresh":"2023-06-03T20:36:15.509709+0000","last_change":"2023-06-03T20:28:51.231879+0000","last_active":"2023-06-03T20:36:15.509709+0000","last_peered":"2023-06-03T20:36:15.509709+0000","last_clean":"2023-06-03T20:36:15.509709+0000","last_became_active":"2023-06-03T20:28:51.231451+0000","last_became_peered":"2023-06-03T20:28:51.231451+0000","last_unstale":"2023-06-03T20:36:15.509709+0000","last_undegraded":"2023-06-03T20:36:15.509709+0000","last_fullsized":"2023-06-03T20:36:15.509709+0000","mapping_epoch":286,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":287,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-06-03T19:34:16.776254+0000","last_clean_scrub_stamp":"2023-06-03T19:34:16.776254+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-06-05T05:47:15.344575+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5486,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":62,"num_write_kb":30,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":21,"num_bytes_recovered":4856,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16458,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":30319,"internal_metadata":0},"log_size":56,"ondisk_log_size":56,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2969600,"data_stored":2951840,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":3,"num_store_stats":5},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":48,"seq":206158431012,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":976660,"kb_used_data":3532,"kb_used_omap":0,"kb_used_meta":973120,"kb_avail":92768492,"statfs":{"total":95995035648,"available":94994935808,"internally_reserved":0,"allocated":3616768,"data_stored":2927630,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":996474880},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 3 20:37:46 2023","interfaces":[{"interface":"back","average":{"1min":0.78400000000000003,"5min":0.85499999999999998,"15min":1.516},"min":{"1min":0.54000000000000004,"5min":0.52000000000000002,"15min":0.26400000000000001},"max":{"1min":1.157,"5min":1.2929999999999999,"15min":59.356000000000002},"last":0.72299999999999998},{"interface":"front","average":{"1min":0.90300000000000002,"5min":0.96199999999999997,"15min":1.464},"min":{"1min":0.54100000000000004,"5min":0.54100000000000004,"15min":0.26700000000000002},"max":{"1min":1.274,"5min":1.423,"15min":58.725999999999999},"last":1.0369999999999999}]},{"osd":1,"last update":"Sat Jun 3 20:37:32 2023","interfaces":[{"interface":"back","average":{"1min":0.93400000000000005,"5min":0.90800000000000003,"15min":1.6539999999999999},"min":{"1min":0.624,"5min":0.45800000000000002,"15min":0.27000000000000002},"max":{"1min":1.248,"5min":1.4750000000000001,"15min":100.845},"last":0.47599999999999998},{"interface":"front","average":{"1min":1.0329999999999999,"5min":0.98499999999999999,"15min":1.79},"min":{"1min":0.75800000000000001,"5min":0.50900000000000001,"15min":0.29899999999999999},"max":{"1min":1.206,"5min":1.5129999999999999,"15min":100.86499999999999},"last":0.74099999999999999}]},{"osd":2,"last update":"Sat Jun 3 20:37:32 2023","interfaces":[{"interface":"back","average":{"1min":0.94199999999999995,"5min":0.94699999999999995,"15min":2.0499999999999998},"min":{"1min":0.68600000000000005,"5min":0.57999999999999996,"15min":0.28899999999999998},"max":{"1min":1.1819999999999999,"5min":1.7589999999999999,"15min":101.185},"last":0.56899999999999995},{"interface":"front","average":{"1min":0.96899999999999997,"5min":0.97699999999999998,"15min":2.1019999999999999},"min":{"1min":0.64600000000000002,"5min":0.502,"15min":0.32400000000000001},"max":{"1min":1.1799999999999999,"5min":1.464,"15min":100.61799999999999},"last":0.97399999999999998}]},{"osd":3,"last update":"Sat Jun 3 20:37:32 2023","interfaces":[{"interface":"back","average":{"1min":0.997,"5min":0.98199999999999998,"15min":1.363},"min":{"1min":0.80100000000000005,"5min":0.59499999999999997,"15min":0.34999999999999998},"max":{"1min":1.222,"5min":1.6890000000000001,"15min":8.9120000000000008},"last":0.64400000000000002},{"interface":"front","average":{"1min":1.1040000000000001,"5min":1.075,"15min":1.431},"min":{"1min":0.72799999999999998,"5min":0.624,"15min":0.35399999999999998},"max":{"1min":1.349,"5min":1.5629999999999999,"15min":11.656000000000001},"last":0.83099999999999996}]},{"osd":4,"last update":"Sat Jun 3 20:37:56 2023","interfaces":[{"interface":"back","average":{"1min":1.0680000000000001,"5min":1.0960000000000001,"15min":1.4970000000000001},"min":{"1min":0.53200000000000003,"5min":0.53200000000000003,"15min":0.30399999999999999},"max":{"1min":1.4019999999999999,"5min":1.6479999999999999,"15min":8.6010000000000009},"last":1.167},{"interface":"front","average":{"1min":1.0149999999999999,"5min":1.0349999999999999,"15min":1.472},"min":{"1min":0.66100000000000003,"5min":0.57199999999999995,"15min":0.27600000000000002},"max":{"1min":1.4710000000000001,"5min":1.4710000000000001,"15min":9.8460000000000001},"last":0.66100000000000003}]},{"osd":5,"last update":"Sat Jun 3 20:37:32 2023","interfaces":[{"interface":"back","average":{"1min":1.03,"5min":1.04,"15min":1.407},"min":{"1min":0.69699999999999995,"5min":0.51600000000000001,"15min":0.317},"max":{"1min":1.276,"5min":1.5980000000000001,"15min":17.696000000000002},"last":0.69399999999999995},{"interface":"front","average":{"1min":1.1659999999999999,"5min":1.1599999999999999,"15min":1.427},"min":{"1min":0.90200000000000002,"5min":0.625,"15min":0.29399999999999998},"max":{"1min":1.3720000000000001,"5min":1.6719999999999999,"15min":9.3710000000000004},"last":0.90500000000000003}]},{"osd":6,"last update":"Sat Jun 3 20:37:32 2023","interfaces":[{"interface":"back","average":{"1min":1.2330000000000001,"5min":1.1859999999999999,"15min":1.278},"min":{"1min":1.0029999999999999,"5min":0.45300000000000001,"15min":0.35899999999999999},"max":{"1min":1.4239999999999999,"5min":1.704,"15min":9.7200000000000006},"last":1.1060000000000001},{"interface":"front","average":{"1min":1.0189999999999999,"5min":1.0149999999999999,"15min":1.244},"min":{"1min":0.81799999999999995,"5min":0.48199999999999998,"15min":0.34300000000000003},"max":{"1min":1.242,"5min":1.522,"15min":9.923},"last":0.55600000000000005}]}]},{"osd":6,"up_from":42,"seq":180388627245,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":624004,"kb_used_data":3516,"kb_used_omap":0,"kb_used_meta":620480,"kb_avail":93121148,"statfs":{"total":95995035648,"available":95356055552,"internally_reserved":0,"allocated":3600384,"data_stored":2927169,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":635371520},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 3 20:36:55 2023","interfaces":[{"interface":"back","average":{"1min":0.878,"5min":1.014,"15min":1.899},"min":{"1min":0.59999999999999998,"5min":0.495,"15min":0.254},"max":{"1min":1.367,"5min":3.7000000000000002,"15min":109.90900000000001},"last":2.2919999999999998},{"interface":"front","average":{"1min":0.97199999999999998,"5min":0.99399999999999999,"15min":2.0089999999999999},"min":{"1min":0.69399999999999995,"5min":0.53900000000000003,"15min":0.315},"max":{"1min":1.389,"5min":3.6589999999999998,"15min":109.86499999999999},"last":0.65800000000000003}]},{"osd":1,"last update":"Sat Jun 3 20:37:34 2023","interfaces":[{"interface":"back","average":{"1min":0.91200000000000003,"5min":0.99099999999999999,"15min":1.6439999999999999},"min":{"1min":0.55800000000000005,"5min":0.48199999999999998,"15min":0.29199999999999998},"max":{"1min":1.2969999999999999,"5min":2.3340000000000001,"15min":107.90600000000001},"last":0.86499999999999999},{"interface":"front","average":{"1min":0.96799999999999997,"5min":1.0009999999999999,"15min":1.7729999999999999},"min":{"1min":0.67800000000000005,"5min":0.627,"15min":0.32700000000000001},"max":{"1min":1.444,"5min":1.444,"15min":107.79900000000001},"last":0.70499999999999996}]},{"osd":2,"last update":"Sat Jun 3 20:37:34 2023","interfaces":[{"interface":"back","average":{"1min":0.89900000000000002,"5min":0.91400000000000003,"15min":1.8140000000000001},"min":{"1min":0.66000000000000003,"5min":0.52000000000000002,"15min":0.33500000000000002},"max":{"1min":1.1990000000000001,"5min":1.3129999999999999,"15min":107.61199999999999},"last":0.91500000000000004},{"interface":"front","average":{"1min":0.94999999999999996,"5min":1.0009999999999999,"15min":1.905},"min":{"1min":0.62,"5min":0.62,"15min":0.29399999999999998},"max":{"1min":1.2529999999999999,"5min":2.2999999999999998,"15min":107.83},"last":0.93899999999999995}]},{"osd":3,"last update":"Sat Jun 3 20:37:15 2023","interfaces":[{"interface":"back","average":{"1min":1.0389999999999999,"5min":1,"15min":1.8200000000000001},"min":{"1min":0.88100000000000001,"5min":0.64700000000000002,"15min":0.35499999999999998},"max":{"1min":1.381,"5min":1.397,"15min":108.254},"last":0.98399999999999999},{"interface":"front","average":{"1min":0.97699999999999998,"5min":1.054,"15min":1.861},"min":{"1min":0.59999999999999998,"5min":0.59999999999999998,"15min":0.39100000000000001},"max":{"1min":1.2330000000000001,"5min":2.1930000000000001,"15min":107.67700000000001},"last":0.74199999999999999}]},{"osd":4,"last update":"Sat Jun 3 20:37:50 2023","interfaces":[{"interface":"back","average":{"1min":1.079,"5min":0.995,"15min":1.395},"min":{"1min":0.747,"5min":0.50800000000000001,"15min":0.33000000000000002},"max":{"1min":1.3360000000000001,"5min":1.488,"15min":10.574999999999999},"last":0.89000000000000001},{"interface":"front","average":{"1min":1.0649999999999999,"5min":1.0660000000000001,"15min":1.288},"min":{"1min":0.71799999999999997,"5min":0.52900000000000003,"15min":0.34499999999999997},"max":{"1min":1.4990000000000001,"5min":1.621,"15min":12.622999999999999},"last":0.80300000000000005}]},{"osd":5,"last update":"Sat Jun 3 20:37:50 2023","interfaces":[{"interface":"back","average":{"1min":1.0960000000000001,"5min":1.1319999999999999,"15min":1.264},"min":{"1min":0.66600000000000004,"5min":0.505,"15min":0.29899999999999999},"max":{"1min":1.3540000000000001,"5min":2.2599999999999998,"15min":10.625999999999999},"last":0.60599999999999998},{"interface":"front","average":{"1min":1.1739999999999999,"5min":1.105,"15min":1.393},"min":{"1min":0.78800000000000003,"5min":0.52100000000000002,"15min":0.40300000000000002},"max":{"1min":1.4199999999999999,"5min":1.51,"15min":10.856},"last":0.58699999999999997}]},{"osd":7,"last update":"Sat Jun 3 20:37:15 2023","interfaces":[{"interface":"back","average":{"1min":1.1910000000000001,"5min":1.2050000000000001,"15min":1.3340000000000001},"min":{"1min":0.66900000000000004,"5min":0.498,"15min":0.378},"max":{"1min":1.403,"5min":2.4020000000000001,"15min":9.0519999999999996},"last":0.68200000000000005},{"interface":"front","average":{"1min":1.196,"5min":1.1679999999999999,"15min":1.2769999999999999},"min":{"1min":0.81899999999999995,"5min":0.67800000000000005,"15min":0.38},"max":{"1min":1.504,"5min":1.73,"15min":10.52},"last":0.55600000000000005}]}]},{"osd":1,"up_from":103,"seq":442381632043,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":490608,"kb_used_data":3560,"kb_used_omap":4,"kb_used_meta":487035,"kb_avail":93254544,"statfs":{"total":95995035648,"available":95492653056,"internally_reserved":0,"allocated":3645440,"data_stored":2406778,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":498724750},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 3 20:37:35 2023","interfaces":[{"interface":"back","average":{"1min":0.57099999999999995,"5min":0.65300000000000002,"15min":2.7130000000000001},"min":{"1min":0.36799999999999999,"5min":0.248,"15min":0.17100000000000001},"max":{"1min":0.84699999999999998,"5min":2.04,"15min":131.83600000000001},"last":1.0029999999999999},{"interface":"front","average":{"1min":0.626,"5min":0.75900000000000001,"15min":2.7589999999999999},"min":{"1min":0.33900000000000002,"5min":0.33900000000000002,"15min":0.182},"max":{"1min":0.92900000000000005,"5min":1.1870000000000001,"15min":131.49100000000001},"last":1.054}]},{"osd":2,"last update":"Sat Jun 3 20:37:35 2023","interfaces":[{"interface":"back","average":{"1min":0.56399999999999995,"5min":0.70499999999999996,"15min":1.851},"min":{"1min":0.373,"5min":0.34100000000000003,"15min":0.20599999999999999},"max":{"1min":0.76200000000000001,"5min":1.353,"15min":131.858},"last":0.83499999999999996},{"interface":"front","average":{"1min":0.55100000000000005,"5min":0.69699999999999995,"15min":1.964},"min":{"1min":0.30599999999999999,"5min":0.27600000000000002,"15min":0.23499999999999999},"max":{"1min":1.0620000000000001,"5min":1.2070000000000001,"15min":131.63},"last":1.2190000000000001}]},{"osd":3,"last update":"Sat Jun 3 20:37:09 2023","interfaces":[{"interface":"back","average":{"1min":0.81200000000000006,"5min":0.75900000000000001,"15min":2.4470000000000001},"min":{"1min":0.39000000000000001,"5min":0.313,"15min":0.23400000000000001},"max":{"1min":1.139,"5min":1.216,"15min":131.65199999999999},"last":0.95599999999999996},{"interface":"front","average":{"1min":0.85699999999999998,"5min":0.83899999999999997,"15min":2.528},"min":{"1min":0.47799999999999998,"5min":0.34699999999999998,"15min":0.223},"max":{"1min":1.2929999999999999,"5min":1.371,"15min":131.60599999999999},"last":1.097}]},{"osd":4,"last update":"Sat Jun 3 20:37:51 2023","interfaces":[{"interface":"back","average":{"1min":0.86699999999999999,"5min":0.96399999999999997,"15min":1.4570000000000001},"min":{"1min":0.60599999999999998,"5min":0.60599999999999998,"15min":0.33100000000000002},"max":{"1min":1.206,"5min":1.4510000000000001,"15min":11.114000000000001},"last":1.244},{"interface":"front","average":{"1min":0.92300000000000004,"5min":1.0509999999999999,"15min":1.4199999999999999},"min":{"1min":0.69999999999999996,"5min":0.69999999999999996,"15min":0.33400000000000002},"max":{"1min":1.387,"5min":1.6140000000000001,"15min":11.247999999999999},"last":1.333}]},{"osd":5,"last update":"Sat Jun 3 20:37:35 2023","interfaces":[{"interface":"back","average":{"1min":0.92500000000000004,"5min":1.0289999999999999,"15min":1.542},"min":{"1min":0.65000000000000002,"5min":0.56899999999999995,"15min":0.30599999999999999},"max":{"1min":1.417,"5min":2.1320000000000001,"15min":11.246},"last":1.1359999999999999},{"interface":"front","average":{"1min":0.91900000000000004,"5min":1.077,"15min":1.9630000000000001},"min":{"1min":0.72799999999999998,"5min":0.72799999999999998,"15min":0.35799999999999998},"max":{"1min":1.1990000000000001,"5min":1.7190000000000001,"15min":131.25299999999999},"last":1.1719999999999999}]},{"osd":6,"last update":"Sat Jun 3 20:37:35 2023","interfaces":[{"interface":"back","average":{"1min":0.89600000000000002,"5min":1.038,"15min":1.381},"min":{"1min":0.61899999999999999,"5min":0.57599999999999996,"15min":0.36399999999999999},"max":{"1min":1.403,"5min":1.589,"15min":10.002000000000001},"last":1.4319999999999999},{"interface":"front","average":{"1min":1.0149999999999999,"5min":1.101,"15min":1.4730000000000001},"min":{"1min":0.60199999999999998,"5min":0.60199999999999998,"15min":0.36899999999999999},"max":{"1min":1.3899999999999999,"5min":1.6379999999999999,"15min":11.067},"last":1.397}]},{"osd":7,"last update":"Sat Jun 3 20:37:35 2023","interfaces":[{"interface":"back","average":{"1min":1.0129999999999999,"5min":1.077,"15min":1.6180000000000001},"min":{"1min":0.70599999999999996,"5min":0.65500000000000003,"15min":0.377},"max":{"1min":1.4079999999999999,"5min":1.546,"15min":10.538},"last":1.3009999999999999},{"interface":"front","average":{"1min":0.98099999999999998,"5min":1.081,"15min":1.518},"min":{"1min":0.63700000000000001,"5min":0.60099999999999998,"15min":0.378},"max":{"1min":1.421,"5min":2.5550000000000002,"15min":14.025},"last":1.198}]}]},{"osd":0,"up_from":86,"seq":369367188070,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":776300,"kb_used_data":4196,"kb_used_omap":5,"kb_used_meta":772090,"kb_avail":92968852,"statfs":{"total":95995035648,"available":95200104448,"internally_reserved":0,"allocated":4296704,"data_stored":3004247,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":790620794},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Jun 3 20:37:32 2023","interfaces":[{"interface":"back","average":{"1min":0.56999999999999995,"5min":0.67900000000000005,"15min":3.1579999999999999},"min":{"1min":0.35199999999999998,"5min":0.246,"15min":0.185},"max":{"1min":1.0700000000000001,"5min":1.1439999999999999,"15min":243.03200000000001},"last":0.75800000000000001},{"interface":"front","average":{"1min":0.65600000000000003,"5min":0.66700000000000004,"15min":3.419},"min":{"1min":0.32100000000000001,"5min":0.32100000000000001,"15min":0.188},"max":{"1min":0.95299999999999996,"5min":1.1799999999999999,"15min":242.78399999999999},"last":1.232}]},{"osd":2,"last update":"Sat Jun 3 20:37:32 2023","interfaces":[{"interface":"back","average":{"1min":0.68500000000000005,"5min":0.71599999999999997,"15min":1.9890000000000001},"min":{"1min":0.38400000000000001,"5min":0.36099999999999999,"15min":0.219},"max":{"1min":1.0149999999999999,"5min":1.294,"15min":153.01300000000001},"last":0.69099999999999995},{"interface":"front","average":{"1min":0.72399999999999998,"5min":0.76900000000000002,"15min":3.4249999999999998},"min":{"1min":0.44700000000000001,"5min":0.32700000000000001,"15min":0.24099999999999999},"max":{"1min":1.204,"5min":1.359,"15min":243.30099999999999},"last":1.1599999999999999}]},{"osd":3,"last update":"Sat Jun 3 20:37:07 2023","interfaces":[{"interface":"back","average":{"1min":0.81299999999999994,"5min":0.78700000000000003,"15min":3.5390000000000001},"min":{"1min":0.40300000000000002,"5min":0.40300000000000002,"15min":0.24399999999999999},"max":{"1min":1.1659999999999999,"5min":1.4239999999999999,"15min":259.68400000000003},"last":0.79000000000000004},{"interface":"front","average":{"1min":0.84999999999999998,"5min":0.82699999999999996,"15min":2.6070000000000002},"min":{"1min":0.48899999999999999,"5min":0.34799999999999998,"15min":0.23799999999999999},"max":{"1min":1.254,"5min":1.254,"15min":261.02699999999999},"last":0.85299999999999998}]},{"osd":4,"last update":"Sat Jun 3 20:37:50 2023","interfaces":[{"interface":"back","average":{"1min":0.88900000000000001,"5min":1.004,"15min":1.5189999999999999},"min":{"1min":0.55200000000000005,"5min":0.55200000000000005,"15min":0.32400000000000001},"max":{"1min":1.3169999999999999,"5min":1.6659999999999999,"15min":11.49},"last":1.075},{"interface":"front","average":{"1min":1.0429999999999999,"5min":1.103,"15min":1.216},"min":{"1min":0.65600000000000003,"5min":0.65600000000000003,"15min":0.34100000000000003},"max":{"1min":1.4239999999999999,"5min":1.9159999999999999,"15min":7.5060000000000002},"last":1.3240000000000001}]},{"osd":5,"last update":"Sat Jun 3 20:37:39 2023","interfaces":[{"interface":"back","average":{"1min":0.91200000000000003,"5min":0.98499999999999999,"15min":2.3460000000000001},"min":{"1min":0.63500000000000001,"5min":0.54000000000000004,"15min":0.29299999999999998},"max":{"1min":1.3080000000000001,"5min":1.55,"15min":243.15899999999999},"last":0.98999999999999999},{"interface":"front","average":{"1min":0.99299999999999999,"5min":1.0669999999999999,"15min":2.6360000000000001},"min":{"1min":0.71199999999999997,"5min":0.60999999999999999,"15min":0.33400000000000002},"max":{"1min":1.4430000000000001,"5min":2.6400000000000001,"15min":243.32599999999999},"last":1.262}]},{"osd":6,"last update":"Sat Jun 3 20:37:39 2023","interfaces":[{"interface":"back","average":{"1min":1.123,"5min":1.133,"15min":2.512},"min":{"1min":0.81999999999999995,"5min":0.70899999999999996,"15min":0.36499999999999999},"max":{"1min":1.4419999999999999,"5min":1.8919999999999999,"15min":243.25},"last":0.94699999999999995},{"interface":"front","average":{"1min":1.0409999999999999,"5min":1.139,"15min":2.6579999999999999},"min":{"1min":0.63100000000000001,"5min":0.63100000000000001,"15min":0.379},"max":{"1min":1.4099999999999999,"5min":1.716,"15min":242.84399999999999},"last":1.29}]},{"osd":7,"last update":"Sat Jun 3 20:37:39 2023","interfaces":[{"interface":"back","average":{"1min":1.151,"5min":1.165,"15min":2.7240000000000002},"min":{"1min":0.73599999999999999,"5min":0.55800000000000005,"15min":0.374},"max":{"1min":1.4970000000000001,"5min":1.9430000000000001,"15min":242.93600000000001},"last":1.0940000000000001},{"interface":"front","average":{"1min":1.0669999999999999,"5min":1.177,"15min":2.2749999999999999},"min":{"1min":0.67700000000000005,"5min":0.56499999999999995,"15min":0.435},"max":{"1min":1.427,"5min":1.694,"15min":243.21299999999999},"last":1.3460000000000001}]}]},{"osd":2,"up_from":114,"seq":489626272248,"num_pgs":20,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":656516,"kb_used_data":3516,"kb_used_omap":7,"kb_used_meta":652984,"kb_avail":93088636,"statfs":{"total":95995035648,"available":95322763264,"internally_reserved":0,"allocated":3600384,"data_stored":2403970,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8111,"internal_metadata":668655697},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 3 20:37:10 2023","interfaces":[{"interface":"back","average":{"1min":0.623,"5min":0.68200000000000005,"15min":1.9650000000000001},"min":{"1min":0.36299999999999999,"5min":0.253,"15min":0.182},"max":{"1min":0.95199999999999996,"5min":1.224,"15min":117.57899999999999},"last":0.76000000000000001},{"interface":"front","average":{"1min":0.71899999999999997,"5min":0.73899999999999999,"15min":2.0939999999999999},"min":{"1min":0.437,"5min":0.35499999999999998,"15min":0.20200000000000001},"max":{"1min":1.0649999999999999,"5min":1.1679999999999999,"15min":117.38500000000001},"last":0.87}]},{"osd":1,"last update":"Sat Jun 3 20:37:10 2023","interfaces":[{"interface":"back","average":{"1min":0.71899999999999997,"5min":0.73799999999999999,"15min":2.0289999999999999},"min":{"1min":0.47199999999999998,"5min":0.40200000000000002,"15min":0.193},"max":{"1min":0.96999999999999997,"5min":1.5860000000000001,"15min":116.127},"last":0.80200000000000005},{"interface":"front","average":{"1min":0.69099999999999995,"5min":0.72099999999999997,"15min":2.0870000000000002},"min":{"1min":0.32400000000000001,"5min":0.32400000000000001,"15min":0.251},"max":{"1min":1.1299999999999999,"5min":1.3999999999999999,"15min":116.461},"last":0.83399999999999996}]},{"osd":3,"last update":"Sat Jun 3 20:37:10 2023","interfaces":[{"interface":"back","average":{"1min":0.755,"5min":0.75800000000000001,"15min":1.681},"min":{"1min":0.42199999999999999,"5min":0.41099999999999998,"15min":0.22700000000000001},"max":{"1min":1.1599999999999999,"5min":1.484,"15min":36.695},"last":1.012},{"interface":"front","average":{"1min":0.74199999999999999,"5min":0.79000000000000004,"15min":1.8100000000000001},"min":{"1min":0.45500000000000002,"5min":0.40500000000000003,"15min":0.27100000000000002},"max":{"1min":1.0389999999999999,"5min":1.298,"15min":37.088999999999999},"last":0.71099999999999997}]},{"osd":4,"last update":"Sat Jun 3 20:37:54 2023","interfaces":[{"interface":"back","average":{"1min":0.90500000000000003,"5min":0.97999999999999998,"15min":1.6759999999999999},"min":{"1min":0.60899999999999999,"5min":0.60899999999999999,"15min":0.32700000000000001},"max":{"1min":1.345,"5min":1.766,"15min":16.561},"last":0.92900000000000005},{"interface":"front","average":{"1min":1.044,"5min":1.0760000000000001,"15min":1.466},"min":{"1min":0.78600000000000003,"5min":0.59399999999999997,"15min":0.35099999999999998},"max":{"1min":1.403,"5min":1.6910000000000001,"15min":9.9339999999999993},"last":1.403}]},{"osd":5,"last update":"Sat Jun 3 20:37:10 2023","interfaces":[{"interface":"back","average":{"1min":1.131,"5min":1.099,"15min":1.649},"min":{"1min":0.75600000000000001,"5min":0.58399999999999996,"15min":0.33200000000000002},"max":{"1min":1.837,"5min":1.837,"15min":17.751000000000001},"last":1.3100000000000001},{"interface":"front","average":{"1min":1.0840000000000001,"5min":1.081,"15min":1.669},"min":{"1min":0.79900000000000004,"5min":0.59399999999999997,"15min":0.316},"max":{"1min":1.452,"5min":1.663,"15min":17.928000000000001},"last":1.5289999999999999}]},{"osd":6,"last update":"Sat Jun 3 20:37:10 2023","interfaces":[{"interface":"back","average":{"1min":1.1599999999999999,"5min":1.159,"15min":1.5069999999999999},"min":{"1min":0.623,"5min":0.623,"15min":0.33900000000000002},"max":{"1min":1.8999999999999999,"5min":1.8999999999999999,"15min":9.5809999999999995},"last":1.498},{"interface":"front","average":{"1min":1.1499999999999999,"5min":1.1200000000000001,"15min":1.5329999999999999},"min":{"1min":0.73099999999999998,"5min":0.67100000000000004,"15min":0.35699999999999998},"max":{"1min":1.621,"5min":1.9770000000000001,"15min":12.867000000000001},"last":1.3380000000000001}]},{"osd":7,"last update":"Sat Jun 3 20:37:10 2023","interfaces":[{"interface":"back","average":{"1min":1.21,"5min":1.208,"15min":1.681},"min":{"1min":0.67300000000000004,"5min":0.67300000000000004,"15min":0.376},"max":{"1min":1.784,"5min":1.9950000000000001,"15min":10.529999999999999},"last":1.579},{"interface":"front","average":{"1min":1.2250000000000001,"5min":1.1990000000000001,"15min":1.6559999999999999},"min":{"1min":0.94399999999999995,"5min":0.629,"15min":0.44700000000000001},"max":{"1min":1.595,"5min":2.0310000000000001,"15min":12.845000000000001},"last":1.476}]}]},{"osd":3,"up_from":119,"seq":511101108720,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":624256,"kb_used_data":3576,"kb_used_omap":2,"kb_used_meta":620669,"kb_avail":93120896,"statfs":{"total":95995035648,"available":95355797504,"internally_reserved":0,"allocated":3661824,"data_stored":2406383,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":635565109},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 3 20:37:17 2023","interfaces":[{"interface":"back","average":{"1min":0.57299999999999995,"5min":0.63200000000000001,"15min":2.278},"min":{"1min":0.32200000000000001,"5min":0.30299999999999999,"15min":0.183},"max":{"1min":0.76200000000000001,"5min":1.1279999999999999,"15min":168.33099999999999},"last":0.65800000000000003},{"interface":"front","average":{"1min":0.64100000000000001,"5min":0.72199999999999998,"15min":2.524},"min":{"1min":0.443,"5min":0.38700000000000001,"15min":0.19500000000000001},"max":{"1min":0.86599999999999999,"5min":1.446,"15min":169.25800000000001},"last":1.036}]},{"osd":1,"last update":"Sat Jun 3 20:37:17 2023","interfaces":[{"interface":"back","average":{"1min":0.70499999999999996,"5min":0.73199999999999998,"15min":1.53},"min":{"1min":0.42699999999999999,"5min":0.34799999999999998,"15min":0.22900000000000001},"max":{"1min":0.99199999999999999,"5min":1.2789999999999999,"15min":117.047},"last":0.79200000000000004},{"interface":"front","average":{"1min":0.72299999999999998,"5min":0.77900000000000003,"15min":1.647},"min":{"1min":0.46200000000000002,"5min":0.38900000000000001,"15min":0.20399999999999999},"max":{"1min":0.91300000000000003,"5min":1.1890000000000001,"15min":116.505},"last":0.81999999999999995}]},{"osd":2,"last update":"Sat Jun 3 20:37:17 2023","interfaces":[{"interface":"back","average":{"1min":0.68100000000000005,"5min":0.73899999999999999,"15min":2.1619999999999999},"min":{"1min":0.35199999999999998,"5min":0.33100000000000002,"15min":0.248},"max":{"1min":0.95399999999999996,"5min":1.4470000000000001,"15min":117.584},"last":0.753},{"interface":"front","average":{"1min":0.71499999999999997,"5min":0.76000000000000001,"15min":2.085},"min":{"1min":0.34100000000000003,"5min":0.34100000000000003,"15min":0.22600000000000001},"max":{"1min":0.92700000000000005,"5min":1.4930000000000001,"15min":117.01000000000001},"last":0.70099999999999996}]},{"osd":4,"last update":"Sat Jun 3 20:37:57 2023","interfaces":[{"interface":"back","average":{"1min":0.83799999999999997,"5min":0.98299999999999998,"15min":1.244},"min":{"1min":0.66900000000000004,"5min":0.66900000000000004,"15min":0.34599999999999997},"max":{"1min":1.079,"5min":1.704,"15min":8.0449999999999999},"last":0.871},{"interface":"front","average":{"1min":0.94699999999999995,"5min":1.0349999999999999,"15min":1.415},"min":{"1min":0.65300000000000002,"5min":0.624,"15min":0.33300000000000002},"max":{"1min":1.262,"5min":3.2799999999999998,"15min":10.773999999999999},"last":0.94499999999999995}]},{"osd":5,"last update":"Sat Jun 3 20:37:17 2023","interfaces":[{"interface":"back","average":{"1min":1.0289999999999999,"5min":1.0600000000000001,"15min":1.411},"min":{"1min":0.63100000000000001,"5min":0.63100000000000001,"15min":0.35599999999999998},"max":{"1min":1.2909999999999999,"5min":1.7350000000000001,"15min":38.651000000000003},"last":1.0169999999999999},{"interface":"front","average":{"1min":1.123,"5min":1.0880000000000001,"15min":1.458},"min":{"1min":0.68500000000000005,"5min":0.61599999999999999,"15min":0.316},"max":{"1min":3.0430000000000001,"5min":3.0430000000000001,"15min":38.585999999999999},"last":1.1910000000000001}]},{"osd":6,"last update":"Sat Jun 3 20:37:17 2023","interfaces":[{"interface":"back","average":{"1min":1.2050000000000001,"5min":1.1279999999999999,"15min":1.2390000000000001},"min":{"1min":0.78300000000000003,"5min":0.53000000000000003,"15min":0.35899999999999999},"max":{"1min":3.7389999999999999,"5min":3.7389999999999999,"15min":9.5199999999999996},"last":1.2749999999999999},{"interface":"front","average":{"1min":1.2010000000000001,"5min":1.123,"15min":1.298},"min":{"1min":0.86899999999999999,"5min":0.68200000000000005,"15min":0.36699999999999999},"max":{"1min":3.6579999999999999,"5min":3.6579999999999999,"15min":8.0790000000000006},"last":1.077}]},{"osd":7,"last update":"Sat Jun 3 20:37:17 2023","interfaces":[{"interface":"back","average":{"1min":1.1479999999999999,"5min":1.133,"15min":1.5589999999999999},"min":{"1min":0.73499999999999999,"5min":0.63300000000000001,"15min":0.38900000000000001},"max":{"1min":2.7269999999999999,"5min":2.7269999999999999,"15min":39.008000000000003},"last":1.161},{"interface":"front","average":{"1min":1.238,"5min":1.175,"15min":1.6459999999999999},"min":{"1min":0.73599999999999999,"5min":0.73599999999999999,"15min":0.38700000000000001},"max":{"1min":3.7799999999999998,"5min":3.7799999999999998,"15min":40.003999999999998},"last":1.1140000000000001}]}]},{"osd":4,"up_from":250,"seq":1073741824176,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":265532,"kb_used_data":3700,"kb_used_omap":10,"kb_used_meta":261813,"kb_avail":93479620,"statfs":{"total":95995035648,"available":95723130880,"internally_reserved":0,"allocated":3788800,"data_stored":2421785,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":10652,"internal_metadata":268097124},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 3 20:36:57 2023","interfaces":[{"interface":"back","average":{"1min":1.0580000000000001,"5min":0.96299999999999997,"15min":2.032},"min":{"1min":0.504,"5min":0.309,"15min":0.26300000000000001},"max":{"1min":1.3420000000000001,"5min":1.9770000000000001,"15min":145.755},"last":0.877},{"interface":"front","average":{"1min":0.94099999999999995,"5min":0.90100000000000002,"15min":2.0190000000000001},"min":{"1min":0.58499999999999996,"5min":0.45500000000000002,"15min":0.30399999999999999},"max":{"1min":1.365,"5min":1.696,"15min":145.71799999999999},"last":0.84199999999999997}]},{"osd":1,"last update":"Sat Jun 3 20:36:57 2023","interfaces":[{"interface":"back","average":{"1min":0.92900000000000005,"5min":0.96199999999999997,"15min":1.7030000000000001},"min":{"1min":0.57499999999999996,"5min":0.55700000000000005,"15min":0.29399999999999998},"max":{"1min":1.252,"5min":2.141,"15min":62.539000000000001},"last":1.004},{"interface":"front","average":{"1min":1.034,"5min":1.014,"15min":1.659},"min":{"1min":0.71399999999999997,"5min":0.29699999999999999,"15min":0.29599999999999999},"max":{"1min":1.4419999999999999,"5min":5.4749999999999996,"15min":62.604999999999997},"last":0.74199999999999999}]},{"osd":2,"last update":"Sat Jun 3 20:36:57 2023","interfaces":[{"interface":"back","average":{"1min":0.96299999999999997,"5min":0.93700000000000006,"15min":2.5880000000000001},"min":{"1min":0.73099999999999998,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":1.294,"5min":1.532,"15min":78.941999999999993},"last":1.097},{"interface":"front","average":{"1min":1.02,"5min":1.018,"15min":2.7650000000000001},"min":{"1min":0.66100000000000003,"5min":0.41199999999999998,"15min":0.28999999999999998},"max":{"1min":1.421,"5min":3.6589999999999998,"15min":80.069000000000003},"last":1.0389999999999999}]},{"osd":3,"last update":"Sat Jun 3 20:36:57 2023","interfaces":[{"interface":"back","average":{"1min":1.056,"5min":1.0660000000000001,"15min":4.7089999999999996},"min":{"1min":0.70299999999999996,"5min":0.38800000000000001,"15min":0.32500000000000001},"max":{"1min":1.518,"5min":2.266,"15min":217.578},"last":1.2110000000000001},{"interface":"front","average":{"1min":1.002,"5min":1.0820000000000001,"15min":4.7060000000000004},"min":{"1min":0.71999999999999997,"5min":0.58299999999999996,"15min":0.36199999999999999},"max":{"1min":1.284,"5min":4.6890000000000001,"15min":218.55699999999999},"last":1.129}]},{"osd":5,"last update":"Sat Jun 3 20:36:57 2023","interfaces":[{"interface":"back","average":{"1min":1,"5min":0.98899999999999999,"15min":1.6759999999999999},"min":{"1min":0.751,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":1.385,"5min":2.0979999999999999,"15min":23.913},"last":0.96099999999999997},{"interface":"front","average":{"1min":1.085,"5min":1.0229999999999999,"15min":1.4890000000000001},"min":{"1min":0.73499999999999999,"5min":0.46600000000000003,"15min":0.34000000000000002},"max":{"1min":1.4910000000000001,"5min":1.4910000000000001,"15min":11.093999999999999},"last":0.81000000000000005}]},{"osd":6,"last update":"Sat Jun 3 20:36:57 2023","interfaces":[{"interface":"back","average":{"1min":1.087,"5min":1.077,"15min":1.611},"min":{"1min":0.72499999999999998,"5min":0.372,"15min":0.36099999999999999},"max":{"1min":1.482,"5min":2.141,"15min":23.963000000000001},"last":1.0649999999999999},{"interface":"front","average":{"1min":1.1140000000000001,"5min":1.1100000000000001,"15min":1.331},"min":{"1min":0.86299999999999999,"5min":0.41099999999999998,"15min":0.38},"max":{"1min":1.3939999999999999,"5min":1.613,"15min":8.4190000000000005},"last":0.92800000000000005}]},{"osd":7,"last update":"Sat Jun 3 20:36:57 2023","interfaces":[{"interface":"back","average":{"1min":1.212,"5min":1.135,"15min":1.6399999999999999},"min":{"1min":0.85499999999999998,"5min":0.44800000000000001,"15min":0.372},"max":{"1min":1.5629999999999999,"5min":1.5629999999999999,"15min":10.747999999999999},"last":1.1759999999999999},{"interface":"front","average":{"1min":1.234,"5min":1.175,"15min":1.5409999999999999},"min":{"1min":0.88600000000000001,"5min":0.35999999999999999,"15min":0.34699999999999998},"max":{"1min":1.7769999999999999,"5min":1.7769999999999999,"15min":12.154},"last":1.153}]}]},{"osd":5,"up_from":36,"seq":154618823479,"num_pgs":21,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":986360,"kb_used_data":2928,"kb_used_omap":0,"kb_used_meta":983424,"kb_avail":92758792,"statfs":{"total":95995035648,"available":94985003008,"internally_reserved":0,"allocated":2998272,"data_stored":2332639,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":1007026176},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Jun 3 20:37:36 2023","interfaces":[{"interface":"back","average":{"1min":0.86199999999999999,"5min":0.95099999999999996,"15min":2.1549999999999998},"min":{"1min":0.57699999999999996,"5min":0.44,"15min":0.25700000000000001},"max":{"1min":1.1679999999999999,"5min":2.1379999999999999,"15min":176.88300000000001},"last":1.25},{"interface":"front","average":{"1min":0.88200000000000001,"5min":0.97799999999999998,"15min":2.3079999999999998},"min":{"1min":0.64900000000000002,"5min":0.53200000000000003,"15min":0.26500000000000001},"max":{"1min":1.1970000000000001,"5min":2.1150000000000002,"15min":176.96199999999999},"last":1.1060000000000001}]},{"osd":1,"last update":"Sat Jun 3 20:37:36 2023","interfaces":[{"interface":"back","average":{"1min":0.86199999999999999,"5min":0.88800000000000001,"15min":2.012},"min":{"1min":0.67200000000000004,"5min":0.50700000000000001,"15min":0.28899999999999998},"max":{"1min":1.1539999999999999,"5min":1.288,"15min":102.205},"last":1.4670000000000001},{"interface":"front","average":{"1min":0.84499999999999997,"5min":0.92000000000000004,"15min":2.0249999999999999},"min":{"1min":0.63900000000000001,"5min":0.57699999999999996,"15min":0.30499999999999999},"max":{"1min":1.1160000000000001,"5min":2.4849999999999999,"15min":102.265},"last":0.91500000000000004}]},{"osd":2,"last update":"Sat Jun 3 20:37:36 2023","interfaces":[{"interface":"back","average":{"1min":0.86499999999999999,"5min":0.93799999999999994,"15min":2.8959999999999999},"min":{"1min":0.58299999999999996,"5min":0.56299999999999994,"15min":0.32300000000000001},"max":{"1min":1.244,"5min":1.3819999999999999,"15min":236.40000000000001},"last":1.173},{"interface":"front","average":{"1min":0.93200000000000005,"5min":0.96299999999999997,"15min":3.1339999999999999},"min":{"1min":0.64200000000000002,"5min":0.56699999999999995,"15min":0.308},"max":{"1min":1.212,"5min":1.3520000000000001,"15min":237.86099999999999},"last":1.2170000000000001}]},{"osd":3,"last update":"Sat Jun 3 20:37:14 2023","interfaces":[{"interface":"back","average":{"1min":0.999,"5min":1.0369999999999999,"15min":1.5449999999999999},"min":{"1min":0.59499999999999997,"5min":0.59499999999999997,"15min":0.34399999999999997},"max":{"1min":1.343,"5min":2.0840000000000001,"15min":35.725999999999999},"last":1.3080000000000001},{"interface":"front","average":{"1min":1.133,"5min":1.105,"15min":1.4139999999999999},"min":{"1min":0.74399999999999999,"5min":0.69299999999999995,"15min":0.377},"max":{"1min":1.4670000000000001,"5min":1.552,"15min":35.762999999999998},"last":0.88300000000000001}]},{"osd":4,"last update":"Sat Jun 3 20:37:51 2023","interfaces":[{"interface":"back","average":{"1min":0.96799999999999997,"5min":1.0109999999999999,"15min":1.6850000000000001},"min":{"1min":0.71399999999999997,"5min":0.56299999999999994,"15min":0.32700000000000001},"max":{"1min":1.2170000000000001,"5min":2.1760000000000002,"15min":21.786000000000001},"last":0.98399999999999999},{"interface":"front","average":{"1min":1.0269999999999999,"5min":1.0629999999999999,"15min":1.5009999999999999},"min":{"1min":0.65400000000000003,"5min":0.61299999999999999,"15min":0.30199999999999999},"max":{"1min":1.345,"5min":1.603,"15min":26.478000000000002},"last":0.73699999999999999}]},{"osd":6,"last update":"Sat Jun 3 20:37:36 2023","interfaces":[{"interface":"back","average":{"1min":1.0609999999999999,"5min":1.0640000000000001,"15min":1.2450000000000001},"min":{"1min":0.84299999999999997,"5min":0.53200000000000003,"15min":0.35499999999999998},"max":{"1min":1.1919999999999999,"5min":1.6160000000000001,"15min":7.5140000000000002},"last":1.1870000000000001},{"interface":"front","average":{"1min":1.2,"5min":1.1699999999999999,"15min":1.337},"min":{"1min":1.012,"5min":0.64100000000000001,"15min":0.37},"max":{"1min":1.385,"5min":1.8899999999999999,"15min":6.5309999999999997},"last":1.393}]},{"osd":7,"last update":"Sat Jun 3 20:37:29 2023","interfaces":[{"interface":"back","average":{"1min":1.2050000000000001,"5min":1.2110000000000001,"15min":1.548},"min":{"1min":0.91600000000000004,"5min":0.66900000000000004,"15min":0.39200000000000002},"max":{"1min":1.3620000000000001,"5min":2.2549999999999999,"15min":15.516},"last":1.546},{"interface":"front","average":{"1min":1.1890000000000001,"5min":1.177,"15min":1.371},"min":{"1min":0.98799999999999999,"5min":0.58499999999999996,"15min":0.38400000000000001},"max":{"1min":1.6279999999999999,"5min":1.736,"15min":9.6519999999999992},"last":1.0840000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6928,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4862,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":10652,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4786,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4932,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-06-03T20:38:03.854 INFO:tasks.cephadm.ceph_manager.ceph:recovered! 2023-06-03T20:38:03.854 ERROR:teuthology.run_tasks:Saw exception from tasks. Traceback (most recent call last): File "/home/teuthworker/src/git.ceph.com_teuthology_62814aba234085f597115b9ba9bdc56af117de9e/teuthology/run_tasks.py", line 105, in run_tasks manager = run_one_task(taskname, ctx=ctx, config=config) File "/home/teuthworker/src/git.ceph.com_teuthology_62814aba234085f597115b9ba9bdc56af117de9e/teuthology/run_tasks.py", line 84, in run_one_task return task(**kwargs) File "/home/teuthworker/src/git.ceph.com_teuthology_62814aba234085f597115b9ba9bdc56af117de9e/teuthology/task/parallel.py", line 56, in task p.spawn(_run_spawned, ctx, confg, taskname) File "/home/teuthworker/src/git.ceph.com_teuthology_62814aba234085f597115b9ba9bdc56af117de9e/teuthology/parallel.py", line 84, in __exit__ for result in self: File "/home/teuthworker/src/git.ceph.com_teuthology_62814aba234085f597115b9ba9bdc56af117de9e/teuthology/parallel.py", line 98, in __next__ resurrect_traceback(result) File "/home/teuthworker/src/git.ceph.com_teuthology_62814aba234085f597115b9ba9bdc56af117de9e/teuthology/parallel.py", line 30, in resurrect_traceback raise exc.exc_info[1] File "/home/teuthworker/src/git.ceph.com_teuthology_62814aba234085f597115b9ba9bdc56af117de9e/teuthology/parallel.py", line 23, in capture_traceback return func(*args, **kwargs) File "/home/teuthworker/src/git.ceph.com_teuthology_62814aba234085f597115b9ba9bdc56af117de9e/teuthology/task/parallel.py", line 64, in _run_spawned mgr = run_tasks.run_one_task(taskname, ctx=ctx, config=config) File "/home/teuthworker/src/git.ceph.com_teuthology_62814aba234085f597115b9ba9bdc56af117de9e/teuthology/run_tasks.py", line 84, in run_one_task return task(**kwargs) File "/home/teuthworker/src/git.ceph.com_teuthology_62814aba234085f597115b9ba9bdc56af117de9e/teuthology/task/sequential.py", line 47, in task mgr = run_tasks.run_one_task(taskname, ctx=ctx, config=confg) File "/home/teuthworker/src/git.ceph.com_teuthology_62814aba234085f597115b9ba9bdc56af117de9e/teuthology/run_tasks.py", line 84, in run_one_task return task(**kwargs) File "/home/teuthworker/src/github.com_ceph_ceph_dbb22f29210f43ee95e67f3b727f470d3faecd40/qa/tasks/workunit.py", line 129, in task p.spawn(_run_tests, ctx, refspec, role, tests, File "/home/teuthworker/src/git.ceph.com_teuthology_62814aba234085f597115b9ba9bdc56af117de9e/teuthology/parallel.py", line 84, in __exit__ for result in self: File "/home/teuthworker/src/git.ceph.com_teuthology_62814aba234085f597115b9ba9bdc56af117de9e/teuthology/parallel.py", line 98, in __next__ resurrect_traceback(result) File "/home/teuthworker/src/git.ceph.com_teuthology_62814aba234085f597115b9ba9bdc56af117de9e/teuthology/parallel.py", line 30, in resurrect_traceback raise exc.exc_info[1] File "/home/teuthworker/src/git.ceph.com_teuthology_62814aba234085f597115b9ba9bdc56af117de9e/teuthology/parallel.py", line 23, in capture_traceback return func(*args, **kwargs) File "/home/teuthworker/src/github.com_ceph_ceph_dbb22f29210f43ee95e67f3b727f470d3faecd40/qa/tasks/workunit.py", line 424, in _run_tests remote.run( File "/home/teuthworker/src/git.ceph.com_teuthology_62814aba234085f597115b9ba9bdc56af117de9e/teuthology/orchestra/remote.py", line 525, in run r = self._runner(client=self.ssh, name=self.shortname, **kwargs) File "/home/teuthworker/src/git.ceph.com_teuthology_62814aba234085f597115b9ba9bdc56af117de9e/teuthology/orchestra/run.py", line 455, in run r.wait() File "/home/teuthworker/src/git.ceph.com_teuthology_62814aba234085f597115b9ba9bdc56af117de9e/teuthology/orchestra/run.py", line 161, in wait self._raise_for_status() File "/home/teuthworker/src/git.ceph.com_teuthology_62814aba234085f597115b9ba9bdc56af117de9e/teuthology/orchestra/run.py", line 181, in _raise_for_status raise CommandFailedError( teuthology.exceptions.CommandFailedError: Command failed (workunit test cls/test_cls_rbd.sh) on smithi089 with status 1: 'mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=pacific TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 CEPH_MNT=/home/ubuntu/cephtest/mnt.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/cls/test_cls_rbd.sh' 2023-06-03T20:38:03.927 ERROR:teuthology.run_tasks: Sentry event: https://sentry.ceph.com/organizations/ceph/?query=0b7d7cf0c2b848d082d6a003769122dc Traceback (most recent call last): File "/home/teuthworker/src/git.ceph.com_teuthology_62814aba234085f597115b9ba9bdc56af117de9e/teuthology/run_tasks.py", line 105, in run_tasks manager = run_one_task(taskname, ctx=ctx, config=config) File "/home/teuthworker/src/git.ceph.com_teuthology_62814aba234085f597115b9ba9bdc56af117de9e/teuthology/run_tasks.py", line 84, in run_one_task return task(**kwargs) File "/home/teuthworker/src/git.ceph.com_teuthology_62814aba234085f597115b9ba9bdc56af117de9e/teuthology/task/parallel.py", line 56, in task p.spawn(_run_spawned, ctx, confg, taskname) File "/home/teuthworker/src/git.ceph.com_teuthology_62814aba234085f597115b9ba9bdc56af117de9e/teuthology/parallel.py", line 84, in __exit__ for result in self: File "/home/teuthworker/src/git.ceph.com_teuthology_62814aba234085f597115b9ba9bdc56af117de9e/teuthology/parallel.py", line 98, in __next__ resurrect_traceback(result) File "/home/teuthworker/src/git.ceph.com_teuthology_62814aba234085f597115b9ba9bdc56af117de9e/teuthology/parallel.py", line 30, in resurrect_traceback raise exc.exc_info[1] File "/home/teuthworker/src/git.ceph.com_teuthology_62814aba234085f597115b9ba9bdc56af117de9e/teuthology/parallel.py", line 23, in capture_traceback return func(*args, **kwargs) File "/home/teuthworker/src/git.ceph.com_teuthology_62814aba234085f597115b9ba9bdc56af117de9e/teuthology/task/parallel.py", line 64, in _run_spawned mgr = run_tasks.run_one_task(taskname, ctx=ctx, config=config) File "/home/teuthworker/src/git.ceph.com_teuthology_62814aba234085f597115b9ba9bdc56af117de9e/teuthology/run_tasks.py", line 84, in run_one_task return task(**kwargs) File "/home/teuthworker/src/git.ceph.com_teuthology_62814aba234085f597115b9ba9bdc56af117de9e/teuthology/task/sequential.py", line 47, in task mgr = run_tasks.run_one_task(taskname, ctx=ctx, config=confg) File "/home/teuthworker/src/git.ceph.com_teuthology_62814aba234085f597115b9ba9bdc56af117de9e/teuthology/run_tasks.py", line 84, in run_one_task return task(**kwargs) File "/home/teuthworker/src/github.com_ceph_ceph_dbb22f29210f43ee95e67f3b727f470d3faecd40/qa/tasks/workunit.py", line 129, in task p.spawn(_run_tests, ctx, refspec, role, tests, File "/home/teuthworker/src/git.ceph.com_teuthology_62814aba234085f597115b9ba9bdc56af117de9e/teuthology/parallel.py", line 84, in __exit__ for result in self: File "/home/teuthworker/src/git.ceph.com_teuthology_62814aba234085f597115b9ba9bdc56af117de9e/teuthology/parallel.py", line 98, in __next__ resurrect_traceback(result) File "/home/teuthworker/src/git.ceph.com_teuthology_62814aba234085f597115b9ba9bdc56af117de9e/teuthology/parallel.py", line 30, in resurrect_traceback raise exc.exc_info[1] File "/home/teuthworker/src/git.ceph.com_teuthology_62814aba234085f597115b9ba9bdc56af117de9e/teuthology/parallel.py", line 23, in capture_traceback return func(*args, **kwargs) File "/home/teuthworker/src/github.com_ceph_ceph_dbb22f29210f43ee95e67f3b727f470d3faecd40/qa/tasks/workunit.py", line 424, in _run_tests remote.run( File "/home/teuthworker/src/git.ceph.com_teuthology_62814aba234085f597115b9ba9bdc56af117de9e/teuthology/orchestra/remote.py", line 525, in run r = self._runner(client=self.ssh, name=self.shortname, **kwargs) File "/home/teuthworker/src/git.ceph.com_teuthology_62814aba234085f597115b9ba9bdc56af117de9e/teuthology/orchestra/run.py", line 455, in run r.wait() File "/home/teuthworker/src/git.ceph.com_teuthology_62814aba234085f597115b9ba9bdc56af117de9e/teuthology/orchestra/run.py", line 161, in wait self._raise_for_status() File "/home/teuthworker/src/git.ceph.com_teuthology_62814aba234085f597115b9ba9bdc56af117de9e/teuthology/orchestra/run.py", line 181, in _raise_for_status raise CommandFailedError( teuthology.exceptions.CommandFailedError: Command failed (workunit test cls/test_cls_rbd.sh) on smithi089 with status 1: 'mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=pacific TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 CEPH_MNT=/home/ubuntu/cephtest/mnt.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/cls/test_cls_rbd.sh' 2023-06-03T20:38:03.930 DEBUG:teuthology.run_tasks:Unwinding manager cephadm 2023-06-03T20:38:03.948 INFO:tasks.cephadm:Teardown begin 2023-06-03T20:38:03.949 DEBUG:teuthology.orchestra.run.smithi089:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2023-06-03T20:38:03.980 DEBUG:teuthology.orchestra.run.smithi164:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2023-06-03T20:38:04.010 INFO:tasks.cephadm:Cleaning up testdir ceph.* files... 2023-06-03T20:38:04.011 DEBUG:teuthology.orchestra.run.smithi089:> rm -f /home/ubuntu/cephtest/seed.ceph.conf /home/ubuntu/cephtest/ceph.pub 2023-06-03T20:38:04.030 DEBUG:teuthology.orchestra.run.smithi164:> rm -f /home/ubuntu/cephtest/seed.ceph.conf /home/ubuntu/cephtest/ceph.pub 2023-06-03T20:38:04.059 INFO:tasks.cephadm:Stopping all daemons... 2023-06-03T20:38:04.060 INFO:tasks.cephadm.mon.a:Stopping mon.a... 2023-06-03T20:38:04.060 DEBUG:teuthology.orchestra.run.smithi089:> sudo systemctl stop ceph-f1db3478-0243-11ee-9b1d-001a4aab830c@mon.a 2023-06-03T20:38:04.109 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:38:04 smithi089 systemd[1]: Stopping Ceph mon.a for f1db3478-0243-11ee-9b1d-001a4aab830c... 2023-06-03T20:38:04.335 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:38:04 smithi164 bash[59462]: audit 1685824682.8389096 mgr.y (mgr.34104) 1872 : audit 0 from='client.37218 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-06-03T20:38:04.336 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:38:04 smithi164 bash[59462]: cluster 1685824683.3524683 mgr.y (mgr.34104) 1873 : cluster 0 pgmap v1955: 73 pgs: 73 active+clean; 582 KiB data, 5.2 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:38:04.492 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:38:04 smithi089 bash[78452]: audit 1685824682.8389096 mgr.y (mgr.34104) 1872 : audit 0 from='client.37218 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-06-03T20:38:04.492 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:38:04 smithi089 bash[78452]: cluster 1685824683.3524683 mgr.y (mgr.34104) 1873 : cluster 0 pgmap v1955: 73 pgs: 73 active+clean; 582 KiB data, 5.2 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:38:04.492 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:38:04 smithi089 bash[75068]: audit 1685824682.8389096 mgr.y (mgr.34104) 1872 : audit 0 from='client.37218 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-06-03T20:38:04.493 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:38:04 smithi089 bash[75068]: cluster 1685824683.3524683 mgr.y (mgr.34104) 1873 : cluster 0 pgmap v1955: 73 pgs: 73 active+clean; 582 KiB data, 5.2 GiB used, 710 GiB / 715 GiB avail 2023-06-03T20:38:04.493 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:38:04 smithi089 bash[75068]: debug 2023-06-03T20:38:04.292+0000 7f0ff3cdd700 -1 received signal: Terminated from /sbin/docker-init -- /usr/bin/ceph-mon -n mon.a -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-stderr=true (PID: 1) UID: 0 2023-06-03T20:38:04.493 INFO:journalctl@ceph.mon.a.smithi089.stdout:Jun 03 20:38:04 smithi089 bash[75068]: debug 2023-06-03T20:38:04.292+0000 7f0ff3cdd700 -1 mon.a@0(leader) e4 *** Got Signal Terminated *** 2023-06-03T20:38:05.614 DEBUG:teuthology.orchestra.run.smithi089:> sudo pkill -f 'journalctl -f -n 0 -u ceph-f1db3478-0243-11ee-9b1d-001a4aab830c@mon.a.service' 2023-06-03T20:38:05.719 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-06-03T20:38:05.719 INFO:tasks.cephadm.mon.a:Stopped mon.a 2023-06-03T20:38:05.720 INFO:tasks.cephadm.mon.b:Stopping mon.c... 2023-06-03T20:38:05.720 DEBUG:teuthology.orchestra.run.smithi089:> sudo systemctl stop ceph-f1db3478-0243-11ee-9b1d-001a4aab830c@mon.c 2023-06-03T20:38:05.967 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:38:05 smithi089 systemd[1]: Stopping Ceph mon.c for f1db3478-0243-11ee-9b1d-001a4aab830c... 2023-06-03T20:38:06.235 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:38:05 smithi089 bash[78452]: debug 2023-06-03T20:38:05.960+0000 7f9a18445700 -1 received signal: Terminated from /sbin/docker-init -- /usr/bin/ceph-mon -n mon.c -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-stderr=true (PID: 1) UID: 0 2023-06-03T20:38:06.284 INFO:journalctl@ceph.mon.c.smithi089.stdout:Jun 03 20:38:05 smithi089 bash[78452]: debug 2023-06-03T20:38:05.960+0000 7f9a18445700 -1 mon.c@2(peon) e4 *** Got Signal Terminated *** 2023-06-03T20:38:06.285 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 20:38:05 smithi089 bash[71163]: [03/Jun/2023:20:38:05] ENGINE Bus STOPPING 2023-06-03T20:38:06.286 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 20:38:05 smithi089 bash[71163]: [03/Jun/2023:20:38:05] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2023-06-03T20:38:06.287 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 20:38:05 smithi089 bash[71163]: [03/Jun/2023:20:38:05] ENGINE Bus STOPPED 2023-06-03T20:38:06.287 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 20:38:05 smithi089 bash[71163]: [03/Jun/2023:20:38:05] ENGINE Bus STARTING 2023-06-03T20:38:06.288 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 20:38:06 smithi089 bash[71163]: [03/Jun/2023:20:38:06] ENGINE Serving on http://:::9283 2023-06-03T20:38:06.289 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 20:38:06 smithi089 bash[71163]: [03/Jun/2023:20:38:06] ENGINE Bus STARTED 2023-06-03T20:38:06.289 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 20:38:06 smithi089 bash[71163]: ::ffff:172.21.15.164 - - [03/Jun/2023:20:38:06] "GET /metrics HTTP/1.1" 200 226902 "" "Prometheus/2.33.4" 2023-06-03T20:38:07.262 DEBUG:teuthology.orchestra.run.smithi089:> sudo pkill -f 'journalctl -f -n 0 -u ceph-f1db3478-0243-11ee-9b1d-001a4aab830c@mon.c.service' 2023-06-03T20:38:07.297 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-06-03T20:38:07.298 INFO:tasks.cephadm.mon.b:Stopped mon.c 2023-06-03T20:38:07.298 INFO:tasks.cephadm.mon.b:Stopping mon.b... 2023-06-03T20:38:07.298 DEBUG:teuthology.orchestra.run.smithi164:> sudo systemctl stop ceph-f1db3478-0243-11ee-9b1d-001a4aab830c@mon.b 2023-06-03T20:38:07.585 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:38:07 smithi164 systemd[1]: Stopping Ceph mon.b for f1db3478-0243-11ee-9b1d-001a4aab830c... 2023-06-03T20:38:07.586 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:38:07 smithi164 bash[59462]: debug 2023-06-03T20:38:07.457+0000 7fdabea04700 -1 received signal: Terminated from /sbin/docker-init -- /usr/bin/ceph-mon -n mon.b -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-stderr=true (PID: 1) UID: 0 2023-06-03T20:38:07.586 INFO:journalctl@ceph.mon.b.smithi164.stdout:Jun 03 20:38:07 smithi164 bash[59462]: debug 2023-06-03T20:38:07.457+0000 7fdabea04700 -1 mon.b@1(peon) e4 *** Got Signal Terminated *** 2023-06-03T20:38:08.682 DEBUG:teuthology.orchestra.run.smithi164:> sudo pkill -f 'journalctl -f -n 0 -u ceph-f1db3478-0243-11ee-9b1d-001a4aab830c@mon.b.service' 2023-06-03T20:38:08.727 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-06-03T20:38:08.728 INFO:tasks.cephadm.mon.b:Stopped mon.b 2023-06-03T20:38:08.728 INFO:tasks.cephadm.mgr.y:Stopping mgr.y... 2023-06-03T20:38:08.729 DEBUG:teuthology.orchestra.run.smithi089:> sudo systemctl stop ceph-f1db3478-0243-11ee-9b1d-001a4aab830c@mgr.y 2023-06-03T20:38:08.994 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 20:38:08 smithi089 systemd[1]: Stopping Ceph mgr.y for f1db3478-0243-11ee-9b1d-001a4aab830c... 2023-06-03T20:38:08.994 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 20:38:08 smithi089 bash[71163]: debug 2023-06-03T20:38:08.916+0000 7fe4c2ec1700 -1 received signal: Terminated from /sbin/docker-init -- /usr/bin/ceph-mgr -n mgr.y -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug (PID: 1) UID: 0 2023-06-03T20:38:08.995 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Jun 03 20:38:08 smithi089 bash[71163]: debug 2023-06-03T20:38:08.916+0000 7fe4c2ec1700 -1 mgr handle_mgr_signal *** Got signal Terminated *** 2023-06-03T20:38:10.028 DEBUG:teuthology.orchestra.run.smithi089:> sudo pkill -f 'journalctl -f -n 0 -u ceph-f1db3478-0243-11ee-9b1d-001a4aab830c@mgr.y.service' 2023-06-03T20:38:10.063 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-06-03T20:38:10.064 INFO:tasks.cephadm.mgr.y:Stopped mgr.y 2023-06-03T20:38:10.064 INFO:tasks.cephadm.mgr.x:Stopping mgr.x... 2023-06-03T20:38:10.065 DEBUG:teuthology.orchestra.run.smithi164:> sudo systemctl stop ceph-f1db3478-0243-11ee-9b1d-001a4aab830c@mgr.x 2023-06-03T20:38:10.329 INFO:journalctl@ceph.mgr.x.smithi164.stdout:Jun 03 20:38:10 smithi164 systemd[1]: Stopping Ceph mgr.x for f1db3478-0243-11ee-9b1d-001a4aab830c... 2023-06-03T20:38:11.362 DEBUG:teuthology.orchestra.run.smithi164:> sudo pkill -f 'journalctl -f -n 0 -u ceph-f1db3478-0243-11ee-9b1d-001a4aab830c@mgr.x.service' 2023-06-03T20:38:11.392 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-06-03T20:38:11.393 INFO:tasks.cephadm.mgr.x:Stopped mgr.x 2023-06-03T20:38:11.393 INFO:tasks.cephadm.osd.0:Stopping osd.0... 2023-06-03T20:38:11.394 DEBUG:teuthology.orchestra.run.smithi089:> sudo systemctl stop ceph-f1db3478-0243-11ee-9b1d-001a4aab830c@osd.0 2023-06-03T20:38:11.779 INFO:journalctl@ceph.osd.0.smithi089.stdout:Jun 03 20:38:11 smithi089 systemd[1]: Stopping Ceph osd.0 for f1db3478-0243-11ee-9b1d-001a4aab830c... 2023-06-03T20:38:11.780 INFO:journalctl@ceph.osd.0.smithi089.stdout:Jun 03 20:38:11 smithi089 bash[85044]: debug 2023-06-03T20:38:11.604+0000 7f48a635c700 -1 received signal: Terminated from /sbin/docker-init -- /usr/bin/ceph-osd -n osd.0 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug (PID: 1) UID: 0 2023-06-03T20:38:11.780 INFO:journalctl@ceph.osd.0.smithi089.stdout:Jun 03 20:38:11 smithi089 bash[85044]: debug 2023-06-03T20:38:11.604+0000 7f48a635c700 -1 osd.0 324 *** Got signal Terminated *** 2023-06-03T20:38:11.780 INFO:journalctl@ceph.osd.0.smithi089.stdout:Jun 03 20:38:11 smithi089 bash[85044]: debug 2023-06-03T20:38:11.604+0000 7f48a635c700 -1 osd.0 324 *** Immediate shutdown (osd_fast_shutdown=true) *** 2023-06-03T20:38:18.779 INFO:journalctl@ceph.osd.0.smithi089.stdout:Jun 03 20:38:18 smithi089 bash[271023]: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd-0 2023-06-03T20:38:21.281 DEBUG:teuthology.orchestra.run.smithi089:> sudo pkill -f 'journalctl -f -n 0 -u ceph-f1db3478-0243-11ee-9b1d-001a4aab830c@osd.0.service' 2023-06-03T20:38:21.315 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-06-03T20:38:21.315 INFO:tasks.cephadm.osd.0:Stopped osd.0 2023-06-03T20:38:21.316 INFO:tasks.cephadm.osd.1:Stopping osd.1... 2023-06-03T20:38:21.316 DEBUG:teuthology.orchestra.run.smithi089:> sudo systemctl stop ceph-f1db3478-0243-11ee-9b1d-001a4aab830c@osd.1 2023-06-03T20:38:21.529 INFO:journalctl@ceph.osd.1.smithi089.stdout:Jun 03 20:38:21 smithi089 systemd[1]: Stopping Ceph osd.1 for f1db3478-0243-11ee-9b1d-001a4aab830c... 2023-06-03T20:38:22.029 INFO:journalctl@ceph.osd.1.smithi089.stdout:Jun 03 20:38:21 smithi089 bash[92089]: debug 2023-06-03T20:38:21.536+0000 7f4467ddd700 -1 received signal: Terminated from /sbin/docker-init -- /usr/bin/ceph-osd -n osd.1 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug (PID: 1) UID: 0 2023-06-03T20:38:22.029 INFO:journalctl@ceph.osd.1.smithi089.stdout:Jun 03 20:38:21 smithi089 bash[92089]: debug 2023-06-03T20:38:21.536+0000 7f4467ddd700 -1 osd.1 324 *** Got signal Terminated *** 2023-06-03T20:38:22.030 INFO:journalctl@ceph.osd.1.smithi089.stdout:Jun 03 20:38:21 smithi089 bash[92089]: debug 2023-06-03T20:38:21.536+0000 7f4467ddd700 -1 osd.1 324 *** Immediate shutdown (osd_fast_shutdown=true) *** 2023-06-03T20:38:28.528 INFO:journalctl@ceph.osd.1.smithi089.stdout:Jun 03 20:38:28 smithi089 bash[271248]: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd-1 2023-06-03T20:38:30.623 DEBUG:teuthology.orchestra.run.smithi089:> sudo pkill -f 'journalctl -f -n 0 -u ceph-f1db3478-0243-11ee-9b1d-001a4aab830c@osd.1.service' 2023-06-03T20:38:30.657 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-06-03T20:38:30.657 INFO:tasks.cephadm.osd.1:Stopped osd.1 2023-06-03T20:38:30.658 INFO:tasks.cephadm.osd.2:Stopping osd.2... 2023-06-03T20:38:30.658 DEBUG:teuthology.orchestra.run.smithi089:> sudo systemctl stop ceph-f1db3478-0243-11ee-9b1d-001a4aab830c@osd.2 2023-06-03T20:38:30.895 INFO:journalctl@ceph.osd.2.smithi089.stdout:Jun 03 20:38:30 smithi089 systemd[1]: Stopping Ceph osd.2 for f1db3478-0243-11ee-9b1d-001a4aab830c... 2023-06-03T20:38:31.279 INFO:journalctl@ceph.osd.2.smithi089.stdout:Jun 03 20:38:30 smithi089 bash[98494]: debug 2023-06-03T20:38:30.888+0000 7f63ccbad700 -1 received signal: Terminated from /sbin/docker-init -- /usr/bin/ceph-osd -n osd.2 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug (PID: 1) UID: 0 2023-06-03T20:38:31.279 INFO:journalctl@ceph.osd.2.smithi089.stdout:Jun 03 20:38:30 smithi089 bash[98494]: debug 2023-06-03T20:38:30.888+0000 7f63ccbad700 -1 osd.2 324 *** Got signal Terminated *** 2023-06-03T20:38:31.279 INFO:journalctl@ceph.osd.2.smithi089.stdout:Jun 03 20:38:30 smithi089 bash[98494]: debug 2023-06-03T20:38:30.888+0000 7f63ccbad700 -1 osd.2 324 *** Immediate shutdown (osd_fast_shutdown=true) *** 2023-06-03T20:38:36.835 INFO:journalctl@ceph.prometheus.a.smithi164.stdout:Jun 03 20:38:36 smithi164 bash[55188]: ts=2023-06-03T20:38:36.428Z caller=refresh.go:98 level=error component="discovery manager scrape" discovery=http msg="Unable to refresh target groups" err="Get \"http://172.21.15.89:8765/sd/prometheus/sd-config?service=node-exporter\": dial tcp 172.21.15.89:8765: connect: connection refused" 2023-06-03T20:38:36.836 INFO:journalctl@ceph.prometheus.a.smithi164.stdout:Jun 03 20:38:36 smithi164 bash[55188]: ts=2023-06-03T20:38:36.432Z caller=refresh.go:98 level=error component="discovery manager scrape" discovery=http msg="Unable to refresh target groups" err="Get \"http://172.21.15.89:8765/sd/prometheus/sd-config?service=ceph-exporter\": dial tcp 172.21.15.89:8765: connect: connection refused" 2023-06-03T20:38:36.836 INFO:journalctl@ceph.prometheus.a.smithi164.stdout:Jun 03 20:38:36 smithi164 bash[55188]: ts=2023-06-03T20:38:36.428Z caller=refresh.go:98 level=error component="discovery manager notify" discovery=http msg="Unable to refresh target groups" err="Get \"http://172.21.15.89:8765/sd/prometheus/sd-config?service=alertmanager\": dial tcp 172.21.15.89:8765: connect: connection refused" 2023-06-03T20:38:36.837 INFO:journalctl@ceph.prometheus.a.smithi164.stdout:Jun 03 20:38:36 smithi164 bash[55188]: ts=2023-06-03T20:38:36.432Z caller=refresh.go:98 level=error component="discovery manager scrape" discovery=http msg="Unable to refresh target groups" err="Get \"http://172.21.15.89:8765/sd/prometheus/sd-config?service=mgr-prometheus\": dial tcp 172.21.15.89:8765: connect: connection refused" 2023-06-03T20:38:38.028 INFO:journalctl@ceph.osd.2.smithi089.stdout:Jun 03 20:38:37 smithi089 bash[271467]: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd-2 2023-06-03T20:38:38.335 INFO:journalctl@ceph.osd.7.smithi164.stdout:Jun 03 20:38:38 smithi164 bash[43424]: debug 2023-06-03T20:38:38.012+0000 7f5bbd1c4700 -1 osd.7 324 heartbeat_check: no reply from 172.21.15.89:6806 osd.0 since back 2023-06-03T20:38:13.843087+0000 front 2023-06-03T20:38:13.842982+0000 (oldest deadline 2023-06-03T20:38:37.942624+0000) 2023-06-03T20:38:38.835 INFO:journalctl@ceph.osd.4.smithi164.stdout:Jun 03 20:38:38 smithi164 bash[130758]: debug 2023-06-03T20:38:38.496+0000 7fd6c7535700 -1 osd.4 324 heartbeat_check: no reply from 172.21.15.89:6806 osd.0 since back 2023-06-03T20:38:11.958176+0000 front 2023-06-03T20:38:11.958143+0000 (oldest deadline 2023-06-03T20:38:37.857969+0000) 2023-06-03T20:38:39.335 INFO:journalctl@ceph.osd.7.smithi164.stdout:Jun 03 20:38:39 smithi164 bash[43424]: debug 2023-06-03T20:38:39.000+0000 7f5bbd1c4700 -1 osd.7 324 heartbeat_check: no reply from 172.21.15.89:6806 osd.0 since back 2023-06-03T20:38:13.843087+0000 front 2023-06-03T20:38:13.842982+0000 (oldest deadline 2023-06-03T20:38:37.942624+0000) 2023-06-03T20:38:39.835 INFO:journalctl@ceph.osd.4.smithi164.stdout:Jun 03 20:38:39 smithi164 bash[130758]: debug 2023-06-03T20:38:39.472+0000 7fd6c7535700 -1 osd.4 324 heartbeat_check: no reply from 172.21.15.89:6806 osd.0 since back 2023-06-03T20:38:11.958176+0000 front 2023-06-03T20:38:11.958143+0000 (oldest deadline 2023-06-03T20:38:37.857969+0000) 2023-06-03T20:38:40.007 DEBUG:teuthology.orchestra.run.smithi089:> sudo pkill -f 'journalctl -f -n 0 -u ceph-f1db3478-0243-11ee-9b1d-001a4aab830c@osd.2.service' 2023-06-03T20:38:40.039 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-06-03T20:38:40.039 INFO:tasks.cephadm.osd.2:Stopped osd.2 2023-06-03T20:38:40.039 INFO:tasks.cephadm.osd.3:Stopping osd.3... 2023-06-03T20:38:40.040 DEBUG:teuthology.orchestra.run.smithi089:> sudo systemctl stop ceph-f1db3478-0243-11ee-9b1d-001a4aab830c@osd.3 2023-06-03T20:38:40.279 INFO:journalctl@ceph.osd.3.smithi089.stdout:Jun 03 20:38:40 smithi089 systemd[1]: Stopping Ceph osd.3 for f1db3478-0243-11ee-9b1d-001a4aab830c... 2023-06-03T20:38:40.335 INFO:journalctl@ceph.osd.7.smithi164.stdout:Jun 03 20:38:39 smithi164 bash[43424]: debug 2023-06-03T20:38:39.952+0000 7f5bbd1c4700 -1 osd.7 324 heartbeat_check: no reply from 172.21.15.89:6806 osd.0 since back 2023-06-03T20:38:13.843087+0000 front 2023-06-03T20:38:13.842982+0000 (oldest deadline 2023-06-03T20:38:37.942624+0000) 2023-06-03T20:38:40.779 INFO:journalctl@ceph.osd.3.smithi089.stdout:Jun 03 20:38:40 smithi089 bash[101590]: debug 2023-06-03T20:38:40.287+0000 7f5a3fbc1700 -1 received signal: Terminated from /sbin/docker-init -- /usr/bin/ceph-osd -n osd.3 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug (PID: 1) UID: 0 2023-06-03T20:38:40.779 INFO:journalctl@ceph.osd.3.smithi089.stdout:Jun 03 20:38:40 smithi089 bash[101590]: debug 2023-06-03T20:38:40.287+0000 7f5a3fbc1700 -1 osd.3 324 *** Got signal Terminated *** 2023-06-03T20:38:40.779 INFO:journalctl@ceph.osd.3.smithi089.stdout:Jun 03 20:38:40 smithi089 bash[101590]: debug 2023-06-03T20:38:40.287+0000 7f5a3fbc1700 -1 osd.3 324 *** Immediate shutdown (osd_fast_shutdown=true) *** 2023-06-03T20:38:40.835 INFO:journalctl@ceph.osd.4.smithi164.stdout:Jun 03 20:38:40 smithi164 bash[130758]: debug 2023-06-03T20:38:40.472+0000 7fd6c7535700 -1 osd.4 324 heartbeat_check: no reply from 172.21.15.89:6806 osd.0 since back 2023-06-03T20:38:11.958176+0000 front 2023-06-03T20:38:11.958143+0000 (oldest deadline 2023-06-03T20:38:37.857969+0000) 2023-06-03T20:38:40.836 INFO:journalctl@ceph.osd.6.smithi164.stdout:Jun 03 20:38:40 smithi164 bash[39156]: debug 2023-06-03T20:38:40.640+0000 7f06476b1700 -1 osd.6 324 heartbeat_check: no reply from 172.21.15.89:6806 osd.0 since back 2023-06-03T20:38:11.707882+0000 front 2023-06-03T20:38:11.707850+0000 (oldest deadline 2023-06-03T20:38:39.908620+0000) 2023-06-03T20:38:41.335 INFO:journalctl@ceph.osd.7.smithi164.stdout:Jun 03 20:38:40 smithi164 bash[43424]: debug 2023-06-03T20:38:40.920+0000 7f5bbd1c4700 -1 osd.7 324 heartbeat_check: no reply from 172.21.15.89:6806 osd.0 since back 2023-06-03T20:38:13.843087+0000 front 2023-06-03T20:38:13.842982+0000 (oldest deadline 2023-06-03T20:38:37.942624+0000) 2023-06-03T20:38:41.836 INFO:journalctl@ceph.osd.6.smithi164.stdout:Jun 03 20:38:41 smithi164 bash[39156]: debug 2023-06-03T20:38:41.632+0000 7f06476b1700 -1 osd.6 324 heartbeat_check: no reply from 172.21.15.89:6806 osd.0 since back 2023-06-03T20:38:11.707882+0000 front 2023-06-03T20:38:11.707850+0000 (oldest deadline 2023-06-03T20:38:39.908620+0000) 2023-06-03T20:38:41.836 INFO:journalctl@ceph.osd.4.smithi164.stdout:Jun 03 20:38:41 smithi164 bash[130758]: debug 2023-06-03T20:38:41.472+0000 7fd6c7535700 -1 osd.4 324 heartbeat_check: no reply from 172.21.15.89:6806 osd.0 since back 2023-06-03T20:38:11.958176+0000 front 2023-06-03T20:38:11.958143+0000 (oldest deadline 2023-06-03T20:38:37.857969+0000) 2023-06-03T20:38:42.335 INFO:journalctl@ceph.osd.7.smithi164.stdout:Jun 03 20:38:41 smithi164 bash[43424]: debug 2023-06-03T20:38:41.952+0000 7f5bbd1c4700 -1 osd.7 324 heartbeat_check: no reply from 172.21.15.89:6806 osd.0 since back 2023-06-03T20:38:13.843087+0000 front 2023-06-03T20:38:13.842982+0000 (oldest deadline 2023-06-03T20:38:37.942624+0000) 2023-06-03T20:38:42.597 INFO:journalctl@ceph.osd.4.smithi164.stdout:Jun 03 20:38:42 smithi164 bash[130758]: debug 2023-06-03T20:38:42.504+0000 7fd6c7535700 -1 osd.4 324 heartbeat_check: no reply from 172.21.15.89:6806 osd.0 since back 2023-06-03T20:38:11.958176+0000 front 2023-06-03T20:38:11.958143+0000 (oldest deadline 2023-06-03T20:38:37.857969+0000) 2023-06-03T20:38:42.597 INFO:journalctl@ceph.osd.5.smithi164.stdout:Jun 03 20:38:42 smithi164 bash[34916]: debug 2023-06-03T20:38:42.328+0000 7fa6fa8f6700 -1 osd.5 324 heartbeat_check: no reply from 172.21.15.89:6806 osd.0 since back 2023-06-03T20:38:12.866069+0000 front 2023-06-03T20:38:12.866084+0000 (oldest deadline 2023-06-03T20:38:41.666721+0000) 2023-06-03T20:38:42.779 INFO:journalctl@ceph.osd.3.smithi089.stdout:Jun 03 20:38:42 smithi089 bash[101590]: debug 2023-06-03T20:38:42.491+0000 7f5a3a9b4700 -1 osd.3 324 heartbeat_check: no reply from 172.21.15.89:6806 osd.0 since back 2023-06-03T20:38:16.490171+0000 front 2023-06-03T20:38:16.490253+0000 (oldest deadline 2023-06-03T20:38:42.291338+0000) 2023-06-03T20:38:42.918 INFO:journalctl@ceph.osd.6.smithi164.stdout:Jun 03 20:38:42 smithi164 bash[39156]: debug 2023-06-03T20:38:42.584+0000 7f06476b1700 -1 osd.6 324 heartbeat_check: no reply from 172.21.15.89:6806 osd.0 since back 2023-06-03T20:38:11.707882+0000 front 2023-06-03T20:38:11.707850+0000 (oldest deadline 2023-06-03T20:38:39.908620+0000) 2023-06-03T20:38:43.335 INFO:journalctl@ceph.osd.7.smithi164.stdout:Jun 03 20:38:42 smithi164 bash[43424]: debug 2023-06-03T20:38:42.908+0000 7f5bbd1c4700 -1 osd.7 324 heartbeat_check: no reply from 172.21.15.89:6806 osd.0 since back 2023-06-03T20:38:13.843087+0000 front 2023-06-03T20:38:13.842982+0000 (oldest deadline 2023-06-03T20:38:37.942624+0000) 2023-06-03T20:38:43.627 INFO:journalctl@ceph.osd.5.smithi164.stdout:Jun 03 20:38:43 smithi164 bash[34916]: debug 2023-06-03T20:38:43.348+0000 7fa6fa8f6700 -1 osd.5 324 heartbeat_check: no reply from 172.21.15.89:6806 osd.0 since back 2023-06-03T20:38:12.866069+0000 front 2023-06-03T20:38:12.866084+0000 (oldest deadline 2023-06-03T20:38:41.666721+0000) 2023-06-03T20:38:43.628 INFO:journalctl@ceph.osd.4.smithi164.stdout:Jun 03 20:38:43 smithi164 bash[130758]: debug 2023-06-03T20:38:43.464+0000 7fd6c7535700 -1 osd.4 324 heartbeat_check: no reply from 172.21.15.89:6806 osd.0 since back 2023-06-03T20:38:11.958176+0000 front 2023-06-03T20:38:11.958143+0000 (oldest deadline 2023-06-03T20:38:37.857969+0000) 2023-06-03T20:38:43.779 INFO:journalctl@ceph.osd.3.smithi089.stdout:Jun 03 20:38:43 smithi089 bash[101590]: debug 2023-06-03T20:38:43.491+0000 7f5a3a9b4700 -1 osd.3 324 heartbeat_check: no reply from 172.21.15.89:6806 osd.0 since back 2023-06-03T20:38:16.490171+0000 front 2023-06-03T20:38:16.490253+0000 (oldest deadline 2023-06-03T20:38:42.291338+0000) 2023-06-03T20:38:43.893 INFO:journalctl@ceph.osd.6.smithi164.stdout:Jun 03 20:38:43 smithi164 bash[39156]: debug 2023-06-03T20:38:43.616+0000 7f06476b1700 -1 osd.6 324 heartbeat_check: no reply from 172.21.15.89:6806 osd.0 since back 2023-06-03T20:38:11.707882+0000 front 2023-06-03T20:38:11.707850+0000 (oldest deadline 2023-06-03T20:38:39.908620+0000) 2023-06-03T20:38:44.335 INFO:journalctl@ceph.osd.7.smithi164.stdout:Jun 03 20:38:43 smithi164 bash[43424]: debug 2023-06-03T20:38:43.884+0000 7f5bbd1c4700 -1 osd.7 324 heartbeat_check: no reply from 172.21.15.89:6806 osd.0 since back 2023-06-03T20:38:13.843087+0000 front 2023-06-03T20:38:13.842982+0000 (oldest deadline 2023-06-03T20:38:37.942624+0000) 2023-06-03T20:38:44.673 INFO:journalctl@ceph.osd.5.smithi164.stdout:Jun 03 20:38:44 smithi164 bash[34916]: debug 2023-06-03T20:38:44.388+0000 7fa6fa8f6700 -1 osd.5 324 heartbeat_check: no reply from 172.21.15.89:6806 osd.0 since back 2023-06-03T20:38:12.866069+0000 front 2023-06-03T20:38:12.866084+0000 (oldest deadline 2023-06-03T20:38:41.666721+0000) 2023-06-03T20:38:44.674 INFO:journalctl@ceph.osd.4.smithi164.stdout:Jun 03 20:38:44 smithi164 bash[130758]: debug 2023-06-03T20:38:44.452+0000 7fd6c7535700 -1 osd.4 324 heartbeat_check: no reply from 172.21.15.89:6806 osd.0 since back 2023-06-03T20:38:11.958176+0000 front 2023-06-03T20:38:11.958143+0000 (oldest deadline 2023-06-03T20:38:37.857969+0000) 2023-06-03T20:38:44.778 INFO:journalctl@ceph.osd.3.smithi089.stdout:Jun 03 20:38:44 smithi089 bash[101590]: debug 2023-06-03T20:38:44.467+0000 7f5a3a9b4700 -1 osd.3 324 heartbeat_check: no reply from 172.21.15.89:6806 osd.0 since back 2023-06-03T20:38:16.490171+0000 front 2023-06-03T20:38:16.490253+0000 (oldest deadline 2023-06-03T20:38:42.291338+0000) 2023-06-03T20:38:45.085 INFO:journalctl@ceph.osd.6.smithi164.stdout:Jun 03 20:38:44 smithi164 bash[39156]: debug 2023-06-03T20:38:44.660+0000 7f06476b1700 -1 osd.6 324 heartbeat_check: no reply from 172.21.15.89:6806 osd.0 since back 2023-06-03T20:38:11.707882+0000 front 2023-06-03T20:38:11.707850+0000 (oldest deadline 2023-06-03T20:38:39.908620+0000) 2023-06-03T20:38:45.086 INFO:journalctl@ceph.osd.7.smithi164.stdout:Jun 03 20:38:44 smithi164 bash[43424]: debug 2023-06-03T20:38:44.888+0000 7f5bbd1c4700 -1 osd.7 324 heartbeat_check: no reply from 172.21.15.89:6806 osd.0 since back 2023-06-03T20:38:13.843087+0000 front 2023-06-03T20:38:13.842982+0000 (oldest deadline 2023-06-03T20:38:37.942624+0000) 2023-06-03T20:38:45.721 INFO:journalctl@ceph.osd.5.smithi164.stdout:Jun 03 20:38:45 smithi164 bash[34916]: debug 2023-06-03T20:38:45.368+0000 7fa6fa8f6700 -1 osd.5 324 heartbeat_check: no reply from 172.21.15.89:6806 osd.0 since back 2023-06-03T20:38:12.866069+0000 front 2023-06-03T20:38:12.866084+0000 (oldest deadline 2023-06-03T20:38:41.666721+0000) 2023-06-03T20:38:45.721 INFO:journalctl@ceph.osd.4.smithi164.stdout:Jun 03 20:38:45 smithi164 bash[130758]: debug 2023-06-03T20:38:45.504+0000 7fd6c7535700 -1 osd.4 324 heartbeat_check: no reply from 172.21.15.89:6806 osd.0 since back 2023-06-03T20:38:11.958176+0000 front 2023-06-03T20:38:11.958143+0000 (oldest deadline 2023-06-03T20:38:37.857969+0000) 2023-06-03T20:38:46.085 INFO:journalctl@ceph.osd.6.smithi164.stdout:Jun 03 20:38:45 smithi164 bash[39156]: debug 2023-06-03T20:38:45.712+0000 7f06476b1700 -1 osd.6 324 heartbeat_check: no reply from 172.21.15.89:6806 osd.0 since back 2023-06-03T20:38:11.707882+0000 front 2023-06-03T20:38:11.707850+0000 (oldest deadline 2023-06-03T20:38:39.908620+0000) 2023-06-03T20:38:46.086 INFO:journalctl@ceph.osd.7.smithi164.stdout:Jun 03 20:38:45 smithi164 bash[43424]: debug 2023-06-03T20:38:45.900+0000 7f5bbd1c4700 -1 osd.7 324 heartbeat_check: no reply from 172.21.15.89:6806 osd.0 since back 2023-06-03T20:38:13.843087+0000 front 2023-06-03T20:38:13.842982+0000 (oldest deadline 2023-06-03T20:38:37.942624+0000) 2023-06-03T20:38:46.760 INFO:journalctl@ceph.osd.5.smithi164.stdout:Jun 03 20:38:46 smithi164 bash[34916]: debug 2023-06-03T20:38:46.356+0000 7fa6fa8f6700 -1 osd.5 324 heartbeat_check: no reply from 172.21.15.89:6806 osd.0 since back 2023-06-03T20:38:12.866069+0000 front 2023-06-03T20:38:12.866084+0000 (oldest deadline 2023-06-03T20:38:41.666721+0000) 2023-06-03T20:38:46.760 INFO:journalctl@ceph.osd.4.smithi164.stdout:Jun 03 20:38:46 smithi164 bash[130758]: debug 2023-06-03T20:38:46.496+0000 7fd6c7535700 -1 osd.4 324 heartbeat_check: no reply from 172.21.15.89:6806 osd.0 since back 2023-06-03T20:38:11.958176+0000 front 2023-06-03T20:38:11.958143+0000 (oldest deadline 2023-06-03T20:38:37.857969+0000) 2023-06-03T20:38:47.085 INFO:journalctl@ceph.osd.6.smithi164.stdout:Jun 03 20:38:46 smithi164 bash[39156]: debug 2023-06-03T20:38:46.748+0000 7f06476b1700 -1 osd.6 324 heartbeat_check: no reply from 172.21.15.89:6806 osd.0 since back 2023-06-03T20:38:11.707882+0000 front 2023-06-03T20:38:11.707850+0000 (oldest deadline 2023-06-03T20:38:39.908620+0000) 2023-06-03T20:38:47.086 INFO:journalctl@ceph.osd.7.smithi164.stdout:Jun 03 20:38:46 smithi164 bash[43424]: debug 2023-06-03T20:38:46.864+0000 7f5bbd1c4700 -1 osd.7 324 heartbeat_check: no reply from 172.21.15.89:6806 osd.0 since back 2023-06-03T20:38:13.843087+0000 front 2023-06-03T20:38:13.842982+0000 (oldest deadline 2023-06-03T20:38:37.942624+0000) 2023-06-03T20:38:47.279 INFO:journalctl@ceph.osd.3.smithi089.stdout:Jun 03 20:38:46 smithi089 bash[271687]: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd-3 2023-06-03T20:38:47.722 INFO:journalctl@ceph.osd.5.smithi164.stdout:Jun 03 20:38:47 smithi164 bash[34916]: debug 2023-06-03T20:38:47.364+0000 7fa6fa8f6700 -1 osd.5 324 heartbeat_check: no reply from 172.21.15.89:6806 osd.0 since back 2023-06-03T20:38:12.866069+0000 front 2023-06-03T20:38:12.866084+0000 (oldest deadline 2023-06-03T20:38:41.666721+0000) 2023-06-03T20:38:47.722 INFO:journalctl@ceph.osd.4.smithi164.stdout:Jun 03 20:38:47 smithi164 bash[130758]: debug 2023-06-03T20:38:47.512+0000 7fd6c7535700 -1 osd.4 324 heartbeat_check: no reply from 172.21.15.89:6806 osd.0 since back 2023-06-03T20:38:11.958176+0000 front 2023-06-03T20:38:11.958143+0000 (oldest deadline 2023-06-03T20:38:37.857969+0000) 2023-06-03T20:38:48.085 INFO:journalctl@ceph.osd.6.smithi164.stdout:Jun 03 20:38:47 smithi164 bash[39156]: debug 2023-06-03T20:38:47.708+0000 7f06476b1700 -1 osd.6 324 heartbeat_check: no reply from 172.21.15.89:6806 osd.0 since back 2023-06-03T20:38:11.707882+0000 front 2023-06-03T20:38:11.707850+0000 (oldest deadline 2023-06-03T20:38:39.908620+0000) 2023-06-03T20:38:48.086 INFO:journalctl@ceph.osd.6.smithi164.stdout:Jun 03 20:38:47 smithi164 bash[39156]: debug 2023-06-03T20:38:47.708+0000 7f06476b1700 -1 osd.6 324 heartbeat_check: no reply from 172.21.15.89:6814 osd.1 since back 2023-06-03T20:38:23.410387+0000 front 2023-06-03T20:38:23.410208+0000 (oldest deadline 2023-06-03T20:38:47.510018+0000) 2023-06-03T20:38:48.086 INFO:journalctl@ceph.osd.7.smithi164.stdout:Jun 03 20:38:47 smithi164 bash[43424]: debug 2023-06-03T20:38:47.848+0000 7f5bbd1c4700 -1 osd.7 324 heartbeat_check: no reply from 172.21.15.89:6806 osd.0 since back 2023-06-03T20:38:13.843087+0000 front 2023-06-03T20:38:13.842982+0000 (oldest deadline 2023-06-03T20:38:37.942624+0000) 2023-06-03T20:38:48.086 INFO:journalctl@ceph.osd.7.smithi164.stdout:Jun 03 20:38:47 smithi164 bash[43424]: debug 2023-06-03T20:38:47.848+0000 7f5bbd1c4700 -1 osd.7 324 heartbeat_check: no reply from 172.21.15.89:6814 osd.1 since back 2023-06-03T20:38:23.145022+0000 front 2023-06-03T20:38:23.144779+0000 (oldest deadline 2023-06-03T20:38:47.844517+0000) 2023-06-03T20:38:48.755 INFO:journalctl@ceph.osd.5.smithi164.stdout:Jun 03 20:38:48 smithi164 bash[34916]: debug 2023-06-03T20:38:48.408+0000 7fa6fa8f6700 -1 osd.5 324 heartbeat_check: no reply from 172.21.15.89:6806 osd.0 since back 2023-06-03T20:38:12.866069+0000 front 2023-06-03T20:38:12.866084+0000 (oldest deadline 2023-06-03T20:38:41.666721+0000) 2023-06-03T20:38:48.756 INFO:journalctl@ceph.osd.4.smithi164.stdout:Jun 03 20:38:48 smithi164 bash[130758]: debug 2023-06-03T20:38:48.516+0000 7fd6c7535700 -1 osd.4 324 heartbeat_check: no reply from 172.21.15.89:6806 osd.0 since back 2023-06-03T20:38:11.958176+0000 front 2023-06-03T20:38:11.958143+0000 (oldest deadline 2023-06-03T20:38:37.857969+0000) 2023-06-03T20:38:48.756 INFO:journalctl@ceph.osd.4.smithi164.stdout:Jun 03 20:38:48 smithi164 bash[130758]: debug 2023-06-03T20:38:48.516+0000 7fd6c7535700 -1 osd.4 324 heartbeat_check: no reply from 172.21.15.89:6814 osd.1 since back 2023-06-03T20:38:24.260154+0000 front 2023-06-03T20:38:24.260363+0000 (oldest deadline 2023-06-03T20:38:47.760124+0000) 2023-06-03T20:38:49.085 INFO:journalctl@ceph.osd.6.smithi164.stdout:Jun 03 20:38:48 smithi164 bash[39156]: debug 2023-06-03T20:38:48.744+0000 7f06476b1700 -1 osd.6 324 heartbeat_check: no reply from 172.21.15.89:6806 osd.0 since back 2023-06-03T20:38:11.707882+0000 front 2023-06-03T20:38:11.707850+0000 (oldest deadline 2023-06-03T20:38:39.908620+0000) 2023-06-03T20:38:49.086 INFO:journalctl@ceph.osd.6.smithi164.stdout:Jun 03 20:38:48 smithi164 bash[39156]: debug 2023-06-03T20:38:48.744+0000 7f06476b1700 -1 osd.6 324 heartbeat_check: no reply from 172.21.15.89:6814 osd.1 since back 2023-06-03T20:38:23.410387+0000 front 2023-06-03T20:38:23.410208+0000 (oldest deadline 2023-06-03T20:38:47.510018+0000) 2023-06-03T20:38:49.086 INFO:journalctl@ceph.osd.7.smithi164.stdout:Jun 03 20:38:48 smithi164 bash[43424]: debug 2023-06-03T20:38:48.848+0000 7f5bbd1c4700 -1 osd.7 324 heartbeat_check: no reply from 172.21.15.89:6806 osd.0 since back 2023-06-03T20:38:13.843087+0000 front 2023-06-03T20:38:13.842982+0000 (oldest deadline 2023-06-03T20:38:37.942624+0000) 2023-06-03T20:38:49.086 INFO:journalctl@ceph.osd.7.smithi164.stdout:Jun 03 20:38:48 smithi164 bash[43424]: debug 2023-06-03T20:38:48.848+0000 7f5bbd1c4700 -1 osd.7 324 heartbeat_check: no reply from 172.21.15.89:6814 osd.1 since back 2023-06-03T20:38:23.145022+0000 front 2023-06-03T20:38:23.144779+0000 (oldest deadline 2023-06-03T20:38:47.844517+0000) 2023-06-03T20:38:49.228 DEBUG:teuthology.orchestra.run.smithi089:> sudo pkill -f 'journalctl -f -n 0 -u ceph-f1db3478-0243-11ee-9b1d-001a4aab830c@osd.3.service' 2023-06-03T20:38:49.260 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-06-03T20:38:49.260 INFO:tasks.cephadm.osd.3:Stopped osd.3 2023-06-03T20:38:49.261 INFO:tasks.cephadm.osd.4:Stopping osd.4... 2023-06-03T20:38:49.261 DEBUG:teuthology.orchestra.run.smithi164:> sudo systemctl stop ceph-f1db3478-0243-11ee-9b1d-001a4aab830c@osd.4 2023-06-03T20:38:49.585 INFO:journalctl@ceph.osd.5.smithi164.stdout:Jun 03 20:38:49 smithi164 bash[34916]: debug 2023-06-03T20:38:49.432+0000 7fa6fa8f6700 -1 osd.5 324 heartbeat_check: no reply from 172.21.15.89:6806 osd.0 since back 2023-06-03T20:38:12.866069+0000 front 2023-06-03T20:38:12.866084+0000 (oldest deadline 2023-06-03T20:38:41.666721+0000) 2023-06-03T20:38:49.586 INFO:journalctl@ceph.osd.5.smithi164.stdout:Jun 03 20:38:49 smithi164 bash[34916]: debug 2023-06-03T20:38:49.432+0000 7fa6fa8f6700 -1 osd.5 324 heartbeat_check: no reply from 172.21.15.89:6814 osd.1 since back 2023-06-03T20:38:25.768372+0000 front 2023-06-03T20:38:25.768328+0000 (oldest deadline 2023-06-03T20:38:49.268137+0000) 2023-06-03T20:38:49.586 INFO:journalctl@ceph.osd.4.smithi164.stdout:Jun 03 20:38:49 smithi164 systemd[1]: Stopping Ceph osd.4 for f1db3478-0243-11ee-9b1d-001a4aab830c... 2023-06-03T20:38:49.586 INFO:journalctl@ceph.osd.4.smithi164.stdout:Jun 03 20:38:49 smithi164 bash[130758]: debug 2023-06-03T20:38:49.504+0000 7fd6c7535700 -1 osd.4 324 heartbeat_check: no reply from 172.21.15.89:6806 osd.0 since back 2023-06-03T20:38:11.958176+0000 front 2023-06-03T20:38:11.958143+0000 (oldest deadline 2023-06-03T20:38:37.857969+0000) 2023-06-03T20:38:49.586 INFO:journalctl@ceph.osd.4.smithi164.stdout:Jun 03 20:38:49 smithi164 bash[130758]: debug 2023-06-03T20:38:49.504+0000 7fd6c7535700 -1 osd.4 324 heartbeat_check: no reply from 172.21.15.89:6814 osd.1 since back 2023-06-03T20:38:24.260154+0000 front 2023-06-03T20:38:24.260363+0000 (oldest deadline 2023-06-03T20:38:47.760124+0000) 2023-06-03T20:38:49.587 INFO:journalctl@ceph.osd.4.smithi164.stdout:Jun 03 20:38:49 smithi164 bash[130758]: debug 2023-06-03T20:38:49.508+0000 7fd6ccf43700 -1 received signal: Terminated from /sbin/docker-init -- /usr/bin/ceph-osd -n osd.4 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug (PID: 1) UID: 0 2023-06-03T20:38:49.587 INFO:journalctl@ceph.osd.4.smithi164.stdout:Jun 03 20:38:49 smithi164 bash[130758]: debug 2023-06-03T20:38:49.508+0000 7fd6ccf43700 -1 osd.4 324 *** Got signal Terminated *** 2023-06-03T20:38:49.587 INFO:journalctl@ceph.osd.4.smithi164.stdout:Jun 03 20:38:49 smithi164 bash[130758]: debug 2023-06-03T20:38:49.508+0000 7fd6ccf43700 -1 osd.4 324 *** Immediate shutdown (osd_fast_shutdown=true) *** 2023-06-03T20:38:50.085 INFO:journalctl@ceph.osd.6.smithi164.stdout:Jun 03 20:38:49 smithi164 bash[39156]: debug 2023-06-03T20:38:49.704+0000 7f06476b1700 -1 osd.6 324 heartbeat_check: no reply from 172.21.15.89:6806 osd.0 since back 2023-06-03T20:38:11.707882+0000 front 2023-06-03T20:38:11.707850+0000 (oldest deadline 2023-06-03T20:38:39.908620+0000) 2023-06-03T20:38:50.086 INFO:journalctl@ceph.osd.6.smithi164.stdout:Jun 03 20:38:49 smithi164 bash[39156]: debug 2023-06-03T20:38:49.704+0000 7f06476b1700 -1 osd.6 324 heartbeat_check: no reply from 172.21.15.89:6814 osd.1 since back 2023-06-03T20:38:23.410387+0000 front 2023-06-03T20:38:23.410208+0000 (oldest deadline 2023-06-03T20:38:47.510018+0000) 2023-06-03T20:38:50.086 INFO:journalctl@ceph.osd.7.smithi164.stdout:Jun 03 20:38:49 smithi164 bash[43424]: debug 2023-06-03T20:38:49.888+0000 7f5bbd1c4700 -1 osd.7 324 heartbeat_check: no reply from 172.21.15.89:6806 osd.0 since back 2023-06-03T20:38:13.843087+0000 front 2023-06-03T20:38:13.842982+0000 (oldest deadline 2023-06-03T20:38:37.942624+0000) 2023-06-03T20:38:50.087 INFO:journalctl@ceph.osd.7.smithi164.stdout:Jun 03 20:38:49 smithi164 bash[43424]: debug 2023-06-03T20:38:49.888+0000 7f5bbd1c4700 -1 osd.7 324 heartbeat_check: no reply from 172.21.15.89:6814 osd.1 since back 2023-06-03T20:38:23.145022+0000 front 2023-06-03T20:38:23.144779+0000 (oldest deadline 2023-06-03T20:38:47.844517+0000) 2023-06-03T20:38:50.712 INFO:journalctl@ceph.osd.5.smithi164.stdout:Jun 03 20:38:50 smithi164 bash[34916]: debug 2023-06-03T20:38:50.412+0000 7fa6fa8f6700 -1 osd.5 324 heartbeat_check: no reply from 172.21.15.89:6806 osd.0 since back 2023-06-03T20:38:12.866069+0000 front 2023-06-03T20:38:12.866084+0000 (oldest deadline 2023-06-03T20:38:41.666721+0000) 2023-06-03T20:38:50.713 INFO:journalctl@ceph.osd.5.smithi164.stdout:Jun 03 20:38:50 smithi164 bash[34916]: debug 2023-06-03T20:38:50.412+0000 7fa6fa8f6700 -1 osd.5 324 heartbeat_check: no reply from 172.21.15.89:6814 osd.1 since back 2023-06-03T20:38:25.768372+0000 front 2023-06-03T20:38:25.768328+0000 (oldest deadline 2023-06-03T20:38:49.268137+0000) 2023-06-03T20:38:50.713 INFO:journalctl@ceph.osd.4.smithi164.stdout:Jun 03 20:38:50 smithi164 bash[130758]: debug 2023-06-03T20:38:50.508+0000 7fd6c7535700 -1 osd.4 324 heartbeat_check: no reply from 172.21.15.89:6806 osd.0 since back 2023-06-03T20:38:11.958176+0000 front 2023-06-03T20:38:11.958143+0000 (oldest deadline 2023-06-03T20:38:37.857969+0000) 2023-06-03T20:38:50.714 INFO:journalctl@ceph.osd.4.smithi164.stdout:Jun 03 20:38:50 smithi164 bash[130758]: debug 2023-06-03T20:38:50.508+0000 7fd6c7535700 -1 osd.4 324 heartbeat_check: no reply from 172.21.15.89:6814 osd.1 since back 2023-06-03T20:38:24.260154+0000 front 2023-06-03T20:38:24.260363+0000 (oldest deadline 2023-06-03T20:38:47.760124+0000) 2023-06-03T20:38:51.085 INFO:journalctl@ceph.osd.6.smithi164.stdout:Jun 03 20:38:50 smithi164 bash[39156]: debug 2023-06-03T20:38:50.700+0000 7f06476b1700 -1 osd.6 324 heartbeat_check: no reply from 172.21.15.89:6806 osd.0 since back 2023-06-03T20:38:11.707882+0000 front 2023-06-03T20:38:11.707850+0000 (oldest deadline 2023-06-03T20:38:39.908620+0000) 2023-06-03T20:38:51.086 INFO:journalctl@ceph.osd.6.smithi164.stdout:Jun 03 20:38:50 smithi164 bash[39156]: debug 2023-06-03T20:38:50.700+0000 7f06476b1700 -1 osd.6 324 heartbeat_check: no reply from 172.21.15.89:6814 osd.1 since back 2023-06-03T20:38:23.410387+0000 front 2023-06-03T20:38:23.410208+0000 (oldest deadline 2023-06-03T20:38:47.510018+0000) 2023-06-03T20:38:51.086 INFO:journalctl@ceph.osd.7.smithi164.stdout:Jun 03 20:38:50 smithi164 bash[43424]: debug 2023-06-03T20:38:50.848+0000 7f5bbd1c4700 -1 osd.7 324 heartbeat_check: no reply from 172.21.15.89:6806 osd.0 since back 2023-06-03T20:38:13.843087+0000 front 2023-06-03T20:38:13.842982+0000 (oldest deadline 2023-06-03T20:38:37.942624+0000) 2023-06-03T20:38:51.087 INFO:journalctl@ceph.osd.7.smithi164.stdout:Jun 03 20:38:50 smithi164 bash[43424]: debug 2023-06-03T20:38:50.848+0000 7f5bbd1c4700 -1 osd.7 324 heartbeat_check: no reply from 172.21.15.89:6814 osd.1 since back 2023-06-03T20:38:23.145022+0000 front 2023-06-03T20:38:23.144779+0000 (oldest deadline 2023-06-03T20:38:47.844517+0000) 2023-06-03T20:38:51.703 INFO:journalctl@ceph.osd.4.smithi164.stdout:Jun 03 20:38:51 smithi164 bash[130758]: debug 2023-06-03T20:38:51.556+0000 7fd6c7535700 -1 osd.4 324 heartbeat_check: no reply from 172.21.15.89:6806 osd.0 since back 2023-06-03T20:38:11.958176+0000 front 2023-06-03T20:38:11.958143+0000 (oldest deadline 2023-06-03T20:38:37.857969+0000) 2023-06-03T20:38:51.704 INFO:journalctl@ceph.osd.4.smithi164.stdout:Jun 03 20:38:51 smithi164 bash[130758]: debug 2023-06-03T20:38:51.556+0000 7fd6c7535700 -1 osd.4 324 heartbeat_check: no reply from 172.21.15.89:6814 osd.1 since back 2023-06-03T20:38:24.260154+0000 front 2023-06-03T20:38:24.260363+0000 (oldest deadline 2023-06-03T20:38:47.760124+0000) 2023-06-03T20:38:51.704 INFO:journalctl@ceph.osd.5.smithi164.stdout:Jun 03 20:38:51 smithi164 bash[34916]: debug 2023-06-03T20:38:51.388+0000 7fa6fa8f6700 -1 osd.5 324 heartbeat_check: no reply from 172.21.15.89:6806 osd.0 since back 2023-06-03T20:38:12.866069+0000 front 2023-06-03T20:38:12.866084+0000 (oldest deadline 2023-06-03T20:38:41.666721+0000) 2023-06-03T20:38:51.704 INFO:journalctl@ceph.osd.5.smithi164.stdout:Jun 03 20:38:51 smithi164 bash[34916]: debug 2023-06-03T20:38:51.388+0000 7fa6fa8f6700 -1 osd.5 324 heartbeat_check: no reply from 172.21.15.89:6814 osd.1 since back 2023-06-03T20:38:25.768372+0000 front 2023-06-03T20:38:25.768328+0000 (oldest deadline 2023-06-03T20:38:49.268137+0000) 2023-06-03T20:38:52.086 INFO:journalctl@ceph.osd.6.smithi164.stdout:Jun 03 20:38:51 smithi164 bash[39156]: debug 2023-06-03T20:38:51.692+0000 7f06476b1700 -1 osd.6 324 heartbeat_check: no reply from 172.21.15.89:6806 osd.0 since back 2023-06-03T20:38:11.707882+0000 front 2023-06-03T20:38:11.707850+0000 (oldest deadline 2023-06-03T20:38:39.908620+0000) 2023-06-03T20:38:52.087 INFO:journalctl@ceph.osd.6.smithi164.stdout:Jun 03 20:38:51 smithi164 bash[39156]: debug 2023-06-03T20:38:51.692+0000 7f06476b1700 -1 osd.6 324 heartbeat_check: no reply from 172.21.15.89:6814 osd.1 since back 2023-06-03T20:38:23.410387+0000 front 2023-06-03T20:38:23.410208+0000 (oldest deadline 2023-06-03T20:38:47.510018+0000) 2023-06-03T20:38:52.087 INFO:journalctl@ceph.osd.7.smithi164.stdout:Jun 03 20:38:51 smithi164 bash[43424]: debug 2023-06-03T20:38:51.832+0000 7f5bbd1c4700 -1 osd.7 324 heartbeat_check: no reply from 172.21.15.89:6806 osd.0 since back 2023-06-03T20:38:13.843087+0000 front 2023-06-03T20:38:13.842982+0000 (oldest deadline 2023-06-03T20:38:37.942624+0000) 2023-06-03T20:38:52.088 INFO:journalctl@ceph.osd.7.smithi164.stdout:Jun 03 20:38:51 smithi164 bash[43424]: debug 2023-06-03T20:38:51.832+0000 7f5bbd1c4700 -1 osd.7 324 heartbeat_check: no reply from 172.21.15.89:6814 osd.1 since back 2023-06-03T20:38:23.145022+0000 front 2023-06-03T20:38:23.144779+0000 (oldest deadline 2023-06-03T20:38:47.844517+0000) 2023-06-03T20:38:52.677 INFO:journalctl@ceph.osd.5.smithi164.stdout:Jun 03 20:38:52 smithi164 bash[34916]: debug 2023-06-03T20:38:52.356+0000 7fa6fa8f6700 -1 osd.5 324 heartbeat_check: no reply from 172.21.15.89:6806 osd.0 since back 2023-06-03T20:38:12.866069+0000 front 2023-06-03T20:38:12.866084+0000 (oldest deadline 2023-06-03T20:38:41.666721+0000) 2023-06-03T20:38:52.678 INFO:journalctl@ceph.osd.5.smithi164.stdout:Jun 03 20:38:52 smithi164 bash[34916]: debug 2023-06-03T20:38:52.356+0000 7fa6fa8f6700 -1 osd.5 324 heartbeat_check: no reply from 172.21.15.89:6814 osd.1 since back 2023-06-03T20:38:25.768372+0000 front 2023-06-03T20:38:25.768328+0000 (oldest deadline 2023-06-03T20:38:49.268137+0000) 2023-06-03T20:38:52.678 INFO:journalctl@ceph.osd.4.smithi164.stdout:Jun 03 20:38:52 smithi164 bash[130758]: debug 2023-06-03T20:38:52.508+0000 7fd6c7535700 -1 osd.4 324 heartbeat_check: no reply from 172.21.15.89:6806 osd.0 since back 2023-06-03T20:38:11.958176+0000 front 2023-06-03T20:38:11.958143+0000 (oldest deadline 2023-06-03T20:38:37.857969+0000) 2023-06-03T20:38:52.678 INFO:journalctl@ceph.osd.4.smithi164.stdout:Jun 03 20:38:52 smithi164 bash[130758]: debug 2023-06-03T20:38:52.508+0000 7fd6c7535700 -1 osd.4 324 heartbeat_check: no reply from 172.21.15.89:6814 osd.1 since back 2023-06-03T20:38:24.260154+0000 front 2023-06-03T20:38:24.260363+0000 (oldest deadline 2023-06-03T20:38:47.760124+0000) 2023-06-03T20:38:53.085 INFO:journalctl@ceph.osd.6.smithi164.stdout:Jun 03 20:38:52 smithi164 bash[39156]: debug 2023-06-03T20:38:52.664+0000 7f06476b1700 -1 osd.6 324 heartbeat_check: no reply from 172.21.15.89:6806 osd.0 since back 2023-06-03T20:38:11.707882+0000 front 2023-06-03T20:38:11.707850+0000 (oldest deadline 2023-06-03T20:38:39.908620+0000) 2023-06-03T20:38:53.085 INFO:journalctl@ceph.osd.6.smithi164.stdout:Jun 03 20:38:52 smithi164 bash[39156]: debug 2023-06-03T20:38:52.664+0000 7f06476b1700 -1 osd.6 324 heartbeat_check: no reply from 172.21.15.89:6814 osd.1 since back 2023-06-03T20:38:23.410387+0000 front 2023-06-03T20:38:23.410208+0000 (oldest deadline 2023-06-03T20:38:47.510018+0000) 2023-06-03T20:38:53.086 INFO:journalctl@ceph.osd.7.smithi164.stdout:Jun 03 20:38:52 smithi164 bash[43424]: debug 2023-06-03T20:38:52.848+0000 7f5bbd1c4700 -1 osd.7 324 heartbeat_check: no reply from 172.21.15.89:6806 osd.0 since back 2023-06-03T20:38:13.843087+0000 front 2023-06-03T20:38:13.842982+0000 (oldest deadline 2023-06-03T20:38:37.942624+0000) 2023-06-03T20:38:53.086 INFO:journalctl@ceph.osd.7.smithi164.stdout:Jun 03 20:38:52 smithi164 bash[43424]: debug 2023-06-03T20:38:52.848+0000 7f5bbd1c4700 -1 osd.7 324 heartbeat_check: no reply from 172.21.15.89:6814 osd.1 since back 2023-06-03T20:38:23.145022+0000 front 2023-06-03T20:38:23.144779+0000 (oldest deadline 2023-06-03T20:38:47.844517+0000) 2023-06-03T20:38:53.699 INFO:journalctl@ceph.osd.5.smithi164.stdout:Jun 03 20:38:53 smithi164 bash[34916]: debug 2023-06-03T20:38:53.340+0000 7fa6fa8f6700 -1 osd.5 324 heartbeat_check: no reply from 172.21.15.89:6806 osd.0 since back 2023-06-03T20:38:12.866069+0000 front 2023-06-03T20:38:12.866084+0000 (oldest deadline 2023-06-03T20:38:41.666721+0000) 2023-06-03T20:38:53.700 INFO:journalctl@ceph.osd.5.smithi164.stdout:Jun 03 20:38:53 smithi164 bash[34916]: debug 2023-06-03T20:38:53.340+0000 7fa6fa8f6700 -1 osd.5 324 heartbeat_check: no reply from 172.21.15.89:6814 osd.1 since back 2023-06-03T20:38:25.768372+0000 front 2023-06-03T20:38:25.768328+0000 (oldest deadline 2023-06-03T20:38:49.268137+0000) 2023-06-03T20:38:53.700 INFO:journalctl@ceph.osd.4.smithi164.stdout:Jun 03 20:38:53 smithi164 bash[130758]: debug 2023-06-03T20:38:53.556+0000 7fd6c7535700 -1 osd.4 324 heartbeat_check: no reply from 172.21.15.89:6806 osd.0 since back 2023-06-03T20:38:11.958176+0000 front 2023-06-03T20:38:11.958143+0000 (oldest deadline 2023-06-03T20:38:37.857969+0000) 2023-06-03T20:38:53.701 INFO:journalctl@ceph.osd.4.smithi164.stdout:Jun 03 20:38:53 smithi164 bash[130758]: debug 2023-06-03T20:38:53.556+0000 7fd6c7535700 -1 osd.4 324 heartbeat_check: no reply from 172.21.15.89:6814 osd.1 since back 2023-06-03T20:38:24.260154+0000 front 2023-06-03T20:38:24.260363+0000 (oldest deadline 2023-06-03T20:38:47.760124+0000) 2023-06-03T20:38:54.085 INFO:journalctl@ceph.osd.6.smithi164.stdout:Jun 03 20:38:53 smithi164 bash[39156]: debug 2023-06-03T20:38:53.688+0000 7f06476b1700 -1 osd.6 324 heartbeat_check: no reply from 172.21.15.89:6806 osd.0 since back 2023-06-03T20:38:11.707882+0000 front 2023-06-03T20:38:11.707850+0000 (oldest deadline 2023-06-03T20:38:39.908620+0000) 2023-06-03T20:38:54.085 INFO:journalctl@ceph.osd.6.smithi164.stdout:Jun 03 20:38:53 smithi164 bash[39156]: debug 2023-06-03T20:38:53.688+0000 7f06476b1700 -1 osd.6 324 heartbeat_check: no reply from 172.21.15.89:6814 osd.1 since back 2023-06-03T20:38:23.410387+0000 front 2023-06-03T20:38:23.410208+0000 (oldest deadline 2023-06-03T20:38:47.510018+0000) 2023-06-03T20:38:54.086 INFO:journalctl@ceph.osd.7.smithi164.stdout:Jun 03 20:38:53 smithi164 bash[43424]: debug 2023-06-03T20:38:53.872+0000 7f5bbd1c4700 -1 osd.7 324 heartbeat_check: no reply from 172.21.15.89:6806 osd.0 since back 2023-06-03T20:38:13.843087+0000 front 2023-06-03T20:38:13.842982+0000 (oldest deadline 2023-06-03T20:38:37.942624+0000) 2023-06-03T20:38:54.086 INFO:journalctl@ceph.osd.7.smithi164.stdout:Jun 03 20:38:53 smithi164 bash[43424]: debug 2023-06-03T20:38:53.872+0000 7f5bbd1c4700 -1 osd.7 324 heartbeat_check: no reply from 172.21.15.89:6814 osd.1 since back 2023-06-03T20:38:23.145022+0000 front 2023-06-03T20:38:23.144779+0000 (oldest deadline 2023-06-03T20:38:47.844517+0000) 2023-06-03T20:38:54.713 INFO:journalctl@ceph.osd.5.smithi164.stdout:Jun 03 20:38:54 smithi164 bash[34916]: debug 2023-06-03T20:38:54.340+0000 7fa6fa8f6700 -1 osd.5 324 heartbeat_check: no reply from 172.21.15.89:6806 osd.0 since back 2023-06-03T20:38:12.866069+0000 front 2023-06-03T20:38:12.866084+0000 (oldest deadline 2023-06-03T20:38:41.666721+0000) 2023-06-03T20:38:54.714 INFO:journalctl@ceph.osd.5.smithi164.stdout:Jun 03 20:38:54 smithi164 bash[34916]: debug 2023-06-03T20:38:54.340+0000 7fa6fa8f6700 -1 osd.5 324 heartbeat_check: no reply from 172.21.15.89:6814 osd.1 since back 2023-06-03T20:38:25.768372+0000 front 2023-06-03T20:38:25.768328+0000 (oldest deadline 2023-06-03T20:38:49.268137+0000) 2023-06-03T20:38:55.085 INFO:journalctl@ceph.osd.7.smithi164.stdout:Jun 03 20:38:54 smithi164 bash[43424]: debug 2023-06-03T20:38:54.828+0000 7f5bbd1c4700 -1 osd.7 324 heartbeat_check: no reply from 172.21.15.89:6806 osd.0 since back 2023-06-03T20:38:13.843087+0000 front 2023-06-03T20:38:13.842982+0000 (oldest deadline 2023-06-03T20:38:37.942624+0000) 2023-06-03T20:38:55.085 INFO:journalctl@ceph.osd.7.smithi164.stdout:Jun 03 20:38:54 smithi164 bash[43424]: debug 2023-06-03T20:38:54.828+0000 7f5bbd1c4700 -1 osd.7 324 heartbeat_check: no reply from 172.21.15.89:6814 osd.1 since back 2023-06-03T20:38:23.145022+0000 front 2023-06-03T20:38:23.144779+0000 (oldest deadline 2023-06-03T20:38:47.844517+0000) 2023-06-03T20:38:55.086 INFO:journalctl@ceph.osd.6.smithi164.stdout:Jun 03 20:38:54 smithi164 bash[39156]: debug 2023-06-03T20:38:54.704+0000 7f06476b1700 -1 osd.6 324 heartbeat_check: no reply from 172.21.15.89:6806 osd.0 since back 2023-06-03T20:38:11.707882+0000 front 2023-06-03T20:38:11.707850+0000 (oldest deadline 2023-06-03T20:38:39.908620+0000) 2023-06-03T20:38:55.086 INFO:journalctl@ceph.osd.6.smithi164.stdout:Jun 03 20:38:54 smithi164 bash[39156]: debug 2023-06-03T20:38:54.704+0000 7f06476b1700 -1 osd.6 324 heartbeat_check: no reply from 172.21.15.89:6814 osd.1 since back 2023-06-03T20:38:23.410387+0000 front 2023-06-03T20:38:23.410208+0000 (oldest deadline 2023-06-03T20:38:47.510018+0000) 2023-06-03T20:38:55.585 INFO:journalctl@ceph.osd.5.smithi164.stdout:Jun 03 20:38:55 smithi164 bash[34916]: debug 2023-06-03T20:38:55.376+0000 7fa6fa8f6700 -1 osd.5 324 heartbeat_check: no reply from 172.21.15.89:6806 osd.0 since back 2023-06-03T20:38:12.866069+0000 front 2023-06-03T20:38:12.866084+0000 (oldest deadline 2023-06-03T20:38:41.666721+0000) 2023-06-03T20:38:55.585 INFO:journalctl@ceph.osd.5.smithi164.stdout:Jun 03 20:38:55 smithi164 bash[34916]: debug 2023-06-03T20:38:55.376+0000 7fa6fa8f6700 -1 osd.5 324 heartbeat_check: no reply from 172.21.15.89:6814 osd.1 since back 2023-06-03T20:38:25.768372+0000 front 2023-06-03T20:38:25.768328+0000 (oldest deadline 2023-06-03T20:38:49.268137+0000) 2023-06-03T20:38:56.085 INFO:journalctl@ceph.osd.6.smithi164.stdout:Jun 03 20:38:55 smithi164 bash[39156]: debug 2023-06-03T20:38:55.700+0000 7f06476b1700 -1 osd.6 324 heartbeat_check: no reply from 172.21.15.89:6806 osd.0 since back 2023-06-03T20:38:11.707882+0000 front 2023-06-03T20:38:11.707850+0000 (oldest deadline 2023-06-03T20:38:39.908620+0000) 2023-06-03T20:38:56.085 INFO:journalctl@ceph.osd.6.smithi164.stdout:Jun 03 20:38:55 smithi164 bash[39156]: debug 2023-06-03T20:38:55.700+0000 7f06476b1700 -1 osd.6 324 heartbeat_check: no reply from 172.21.15.89:6814 osd.1 since back 2023-06-03T20:38:23.410387+0000 front 2023-06-03T20:38:23.410208+0000 (oldest deadline 2023-06-03T20:38:47.510018+0000) 2023-06-03T20:38:56.086 INFO:journalctl@ceph.osd.7.smithi164.stdout:Jun 03 20:38:55 smithi164 bash[43424]: debug 2023-06-03T20:38:55.856+0000 7f5bbd1c4700 -1 osd.7 324 heartbeat_check: no reply from 172.21.15.89:6806 osd.0 since back 2023-06-03T20:38:13.843087+0000 front 2023-06-03T20:38:13.842982+0000 (oldest deadline 2023-06-03T20:38:37.942624+0000) 2023-06-03T20:38:56.087 INFO:journalctl@ceph.osd.7.smithi164.stdout:Jun 03 20:38:55 smithi164 bash[43424]: debug 2023-06-03T20:38:55.856+0000 7f5bbd1c4700 -1 osd.7 324 heartbeat_check: no reply from 172.21.15.89:6814 osd.1 since back 2023-06-03T20:38:23.145022+0000 front 2023-06-03T20:38:23.144779+0000 (oldest deadline 2023-06-03T20:38:47.844517+0000) 2023-06-03T20:38:56.406 INFO:journalctl@ceph.osd.4.smithi164.stdout:Jun 03 20:38:56 smithi164 bash[164277]: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd-4 2023-06-03T20:38:56.585 INFO:journalctl@ceph.osd.5.smithi164.stdout:Jun 03 20:38:56 smithi164 bash[34916]: debug 2023-06-03T20:38:56.396+0000 7fa6fa8f6700 -1 osd.5 324 heartbeat_check: no reply from 172.21.15.89:6806 osd.0 since back 2023-06-03T20:38:12.866069+0000 front 2023-06-03T20:38:12.866084+0000 (oldest deadline 2023-06-03T20:38:41.666721+0000) 2023-06-03T20:38:56.585 INFO:journalctl@ceph.osd.5.smithi164.stdout:Jun 03 20:38:56 smithi164 bash[34916]: debug 2023-06-03T20:38:56.396+0000 7fa6fa8f6700 -1 osd.5 324 heartbeat_check: no reply from 172.21.15.89:6814 osd.1 since back 2023-06-03T20:38:25.768372+0000 front 2023-06-03T20:38:25.768328+0000 (oldest deadline 2023-06-03T20:38:49.268137+0000) 2023-06-03T20:38:57.085 INFO:journalctl@ceph.osd.6.smithi164.stdout:Jun 03 20:38:56 smithi164 bash[39156]: debug 2023-06-03T20:38:56.712+0000 7f06476b1700 -1 osd.6 324 heartbeat_check: no reply from 172.21.15.89:6806 osd.0 since back 2023-06-03T20:38:11.707882+0000 front 2023-06-03T20:38:11.707850+0000 (oldest deadline 2023-06-03T20:38:39.908620+0000) 2023-06-03T20:38:57.085 INFO:journalctl@ceph.osd.6.smithi164.stdout:Jun 03 20:38:56 smithi164 bash[39156]: debug 2023-06-03T20:38:56.712+0000 7f06476b1700 -1 osd.6 324 heartbeat_check: no reply from 172.21.15.89:6814 osd.1 since back 2023-06-03T20:38:23.410387+0000 front 2023-06-03T20:38:23.410208+0000 (oldest deadline 2023-06-03T20:38:47.510018+0000) 2023-06-03T20:38:57.086 INFO:journalctl@ceph.osd.7.smithi164.stdout:Jun 03 20:38:56 smithi164 bash[43424]: debug 2023-06-03T20:38:56.896+0000 7f5bbd1c4700 -1 osd.7 324 heartbeat_check: no reply from 172.21.15.89:6806 osd.0 since back 2023-06-03T20:38:13.843087+0000 front 2023-06-03T20:38:13.842982+0000 (oldest deadline 2023-06-03T20:38:37.942624+0000) 2023-06-03T20:38:57.086 INFO:journalctl@ceph.osd.7.smithi164.stdout:Jun 03 20:38:56 smithi164 bash[43424]: debug 2023-06-03T20:38:56.896+0000 7f5bbd1c4700 -1 osd.7 324 heartbeat_check: no reply from 172.21.15.89:6814 osd.1 since back 2023-06-03T20:38:23.145022+0000 front 2023-06-03T20:38:23.144779+0000 (oldest deadline 2023-06-03T20:38:47.844517+0000) 2023-06-03T20:38:57.757 INFO:journalctl@ceph.osd.5.smithi164.stdout:Jun 03 20:38:57 smithi164 bash[34916]: debug 2023-06-03T20:38:57.376+0000 7fa6fa8f6700 -1 osd.5 324 heartbeat_check: no reply from 172.21.15.89:6806 osd.0 since back 2023-06-03T20:38:12.866069+0000 front 2023-06-03T20:38:12.866084+0000 (oldest deadline 2023-06-03T20:38:41.666721+0000) 2023-06-03T20:38:57.757 INFO:journalctl@ceph.osd.5.smithi164.stdout:Jun 03 20:38:57 smithi164 bash[34916]: debug 2023-06-03T20:38:57.376+0000 7fa6fa8f6700 -1 osd.5 324 heartbeat_check: no reply from 172.21.15.89:6814 osd.1 since back 2023-06-03T20:38:25.768372+0000 front 2023-06-03T20:38:25.768328+0000 (oldest deadline 2023-06-03T20:38:49.268137+0000) 2023-06-03T20:38:58.085 INFO:journalctl@ceph.osd.7.smithi164.stdout:Jun 03 20:38:57 smithi164 bash[43424]: debug 2023-06-03T20:38:57.940+0000 7f5bbd1c4700 -1 osd.7 324 heartbeat_check: no reply from 172.21.15.89:6806 osd.0 since back 2023-06-03T20:38:13.843087+0000 front 2023-06-03T20:38:13.842982+0000 (oldest deadline 2023-06-03T20:38:37.942624+0000) 2023-06-03T20:38:58.085 INFO:journalctl@ceph.osd.7.smithi164.stdout:Jun 03 20:38:57 smithi164 bash[43424]: debug 2023-06-03T20:38:57.940+0000 7f5bbd1c4700 -1 osd.7 324 heartbeat_check: no reply from 172.21.15.89:6814 osd.1 since back 2023-06-03T20:38:23.145022+0000 front 2023-06-03T20:38:23.144779+0000 (oldest deadline 2023-06-03T20:38:47.844517+0000) 2023-06-03T20:38:58.086 INFO:journalctl@ceph.osd.6.smithi164.stdout:Jun 03 20:38:57 smithi164 bash[39156]: debug 2023-06-03T20:38:57.744+0000 7f06476b1700 -1 osd.6 324 heartbeat_check: no reply from 172.21.15.89:6806 osd.0 since back 2023-06-03T20:38:11.707882+0000 front 2023-06-03T20:38:11.707850+0000 (oldest deadline 2023-06-03T20:38:39.908620+0000) 2023-06-03T20:38:58.086 INFO:journalctl@ceph.osd.6.smithi164.stdout:Jun 03 20:38:57 smithi164 bash[39156]: debug 2023-06-03T20:38:57.744+0000 7f06476b1700 -1 osd.6 324 heartbeat_check: no reply from 172.21.15.89:6814 osd.1 since back 2023-06-03T20:38:23.410387+0000 front 2023-06-03T20:38:23.410208+0000 (oldest deadline 2023-06-03T20:38:47.510018+0000) 2023-06-03T20:38:58.585 INFO:journalctl@ceph.osd.5.smithi164.stdout:Jun 03 20:38:58 smithi164 bash[34916]: debug 2023-06-03T20:38:58.396+0000 7fa6fa8f6700 -1 osd.5 324 heartbeat_check: no reply from 172.21.15.89:6806 osd.0 since back 2023-06-03T20:38:12.866069+0000 front 2023-06-03T20:38:12.866084+0000 (oldest deadline 2023-06-03T20:38:41.666721+0000) 2023-06-03T20:38:58.585 INFO:journalctl@ceph.osd.5.smithi164.stdout:Jun 03 20:38:58 smithi164 bash[34916]: debug 2023-06-03T20:38:58.396+0000 7fa6fa8f6700 -1 osd.5 324 heartbeat_check: no reply from 172.21.15.89:6814 osd.1 since back 2023-06-03T20:38:25.768372+0000 front 2023-06-03T20:38:25.768328+0000 (oldest deadline 2023-06-03T20:38:49.268137+0000) 2023-06-03T20:38:59.076 INFO:journalctl@ceph.osd.6.smithi164.stdout:Jun 03 20:38:58 smithi164 bash[39156]: debug 2023-06-03T20:38:58.764+0000 7f06476b1700 -1 osd.6 324 heartbeat_check: no reply from 172.21.15.89:6806 osd.0 since back 2023-06-03T20:38:11.707882+0000 front 2023-06-03T20:38:11.707850+0000 (oldest deadline 2023-06-03T20:38:39.908620+0000) 2023-06-03T20:38:59.077 INFO:journalctl@ceph.osd.6.smithi164.stdout:Jun 03 20:38:58 smithi164 bash[39156]: debug 2023-06-03T20:38:58.764+0000 7f06476b1700 -1 osd.6 324 heartbeat_check: no reply from 172.21.15.89:6814 osd.1 since back 2023-06-03T20:38:23.410387+0000 front 2023-06-03T20:38:23.410208+0000 (oldest deadline 2023-06-03T20:38:47.510018+0000) 2023-06-03T20:38:59.077 INFO:journalctl@ceph.osd.7.smithi164.stdout:Jun 03 20:38:58 smithi164 bash[43424]: debug 2023-06-03T20:38:58.968+0000 7f5bbd1c4700 -1 osd.7 324 heartbeat_check: no reply from 172.21.15.89:6806 osd.0 since back 2023-06-03T20:38:13.843087+0000 front 2023-06-03T20:38:13.842982+0000 (oldest deadline 2023-06-03T20:38:37.942624+0000) 2023-06-03T20:38:59.077 INFO:journalctl@ceph.osd.7.smithi164.stdout:Jun 03 20:38:58 smithi164 bash[43424]: debug 2023-06-03T20:38:58.968+0000 7f5bbd1c4700 -1 osd.7 324 heartbeat_check: no reply from 172.21.15.89:6814 osd.1 since back 2023-06-03T20:38:23.145022+0000 front 2023-06-03T20:38:23.144779+0000 (oldest deadline 2023-06-03T20:38:47.844517+0000) 2023-06-03T20:38:59.078 INFO:journalctl@ceph.osd.7.smithi164.stdout:Jun 03 20:38:58 smithi164 bash[43424]: debug 2023-06-03T20:38:58.968+0000 7f5bbd1c4700 -1 osd.7 324 heartbeat_check: no reply from 172.21.15.89:6822 osd.2 since back 2023-06-03T20:38:33.046905+0000 front 2023-06-03T20:38:33.047150+0000 (oldest deadline 2023-06-03T20:38:58.847539+0000) 2023-06-03T20:38:59.079 DEBUG:teuthology.orchestra.run.smithi164:> sudo pkill -f 'journalctl -f -n 0 -u ceph-f1db3478-0243-11ee-9b1d-001a4aab830c@osd.4.service' 2023-06-03T20:38:59.111 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-06-03T20:38:59.112 INFO:tasks.cephadm.osd.4:Stopped osd.4 2023-06-03T20:38:59.112 INFO:tasks.cephadm.osd.5:Stopping osd.5... 2023-06-03T20:38:59.113 DEBUG:teuthology.orchestra.run.smithi164:> sudo systemctl stop ceph-f1db3478-0243-11ee-9b1d-001a4aab830c@osd.5 2023-06-03T20:38:59.335 INFO:journalctl@ceph.osd.5.smithi164.stdout:Jun 03 20:38:59 smithi164 systemd[1]: Stopping Ceph osd.5 for f1db3478-0243-11ee-9b1d-001a4aab830c... 2023-06-03T20:38:59.771 INFO:journalctl@ceph.osd.5.smithi164.stdout:Jun 03 20:38:59 smithi164 bash[34916]: debug 2023-06-03T20:38:59.352+0000 7fa70017d700 -1 received signal: Terminated from /sbin/docker-init -- /usr/bin/ceph-osd -n osd.5 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug (PID: 1) UID: 0 2023-06-03T20:38:59.771 INFO:journalctl@ceph.osd.5.smithi164.stdout:Jun 03 20:38:59 smithi164 bash[34916]: debug 2023-06-03T20:38:59.352+0000 7fa70017d700 -1 osd.5 324 *** Got signal Terminated *** 2023-06-03T20:38:59.772 INFO:journalctl@ceph.osd.5.smithi164.stdout:Jun 03 20:38:59 smithi164 bash[34916]: debug 2023-06-03T20:38:59.352+0000 7fa70017d700 -1 osd.5 324 *** Immediate shutdown (osd_fast_shutdown=true) *** 2023-06-03T20:38:59.772 INFO:journalctl@ceph.osd.5.smithi164.stdout:Jun 03 20:38:59 smithi164 bash[34916]: debug 2023-06-03T20:38:59.368+0000 7fa6fa8f6700 -1 osd.5 324 heartbeat_check: no reply from 172.21.15.89:6806 osd.0 since back 2023-06-03T20:38:12.866069+0000 front 2023-06-03T20:38:12.866084+0000 (oldest deadline 2023-06-03T20:38:41.666721+0000) 2023-06-03T20:38:59.772 INFO:journalctl@ceph.osd.5.smithi164.stdout:Jun 03 20:38:59 smithi164 bash[34916]: debug 2023-06-03T20:38:59.368+0000 7fa6fa8f6700 -1 osd.5 324 heartbeat_check: no reply from 172.21.15.89:6814 osd.1 since back 2023-06-03T20:38:25.768372+0000 front 2023-06-03T20:38:25.768328+0000 (oldest deadline 2023-06-03T20:38:49.268137+0000) 2023-06-03T20:39:00.085 INFO:journalctl@ceph.osd.6.smithi164.stdout:Jun 03 20:38:59 smithi164 bash[39156]: debug 2023-06-03T20:38:59.760+0000 7f06476b1700 -1 osd.6 324 heartbeat_check: no reply from 172.21.15.89:6806 osd.0 since back 2023-06-03T20:38:11.707882+0000 front 2023-06-03T20:38:11.707850+0000 (oldest deadline 2023-06-03T20:38:39.908620+0000) 2023-06-03T20:39:00.085 INFO:journalctl@ceph.osd.6.smithi164.stdout:Jun 03 20:38:59 smithi164 bash[39156]: debug 2023-06-03T20:38:59.760+0000 7f06476b1700 -1 osd.6 324 heartbeat_check: no reply from 172.21.15.89:6814 osd.1 since back 2023-06-03T20:38:23.410387+0000 front 2023-06-03T20:38:23.410208+0000 (oldest deadline 2023-06-03T20:38:47.510018+0000) 2023-06-03T20:39:00.086 INFO:journalctl@ceph.osd.6.smithi164.stdout:Jun 03 20:38:59 smithi164 bash[39156]: debug 2023-06-03T20:38:59.760+0000 7f06476b1700 -1 osd.6 324 heartbeat_check: no reply from 172.21.15.89:6822 osd.2 since back 2023-06-03T20:38:35.012927+0000 front 2023-06-03T20:38:35.012734+0000 (oldest deadline 2023-06-03T20:38:59.712773+0000) 2023-06-03T20:39:00.086 INFO:journalctl@ceph.osd.7.smithi164.stdout:Jun 03 20:38:59 smithi164 bash[43424]: debug 2023-06-03T20:38:59.960+0000 7f5bbd1c4700 -1 osd.7 324 heartbeat_check: no reply from 172.21.15.89:6806 osd.0 since back 2023-06-03T20:38:13.843087+0000 front 2023-06-03T20:38:13.842982+0000 (oldest deadline 2023-06-03T20:38:37.942624+0000) 2023-06-03T20:39:00.086 INFO:journalctl@ceph.osd.7.smithi164.stdout:Jun 03 20:38:59 smithi164 bash[43424]: debug 2023-06-03T20:38:59.960+0000 7f5bbd1c4700 -1 osd.7 324 heartbeat_check: no reply from 172.21.15.89:6814 osd.1 since back 2023-06-03T20:38:23.145022+0000 front 2023-06-03T20:38:23.144779+0000 (oldest deadline 2023-06-03T20:38:47.844517+0000) 2023-06-03T20:39:00.086 INFO:journalctl@ceph.osd.7.smithi164.stdout:Jun 03 20:38:59 smithi164 bash[43424]: debug 2023-06-03T20:38:59.960+0000 7f5bbd1c4700 -1 osd.7 324 heartbeat_check: no reply from 172.21.15.89:6822 osd.2 since back 2023-06-03T20:38:33.046905+0000 front 2023-06-03T20:38:33.047150+0000 (oldest deadline 2023-06-03T20:38:58.847539+0000) 2023-06-03T20:39:00.796 INFO:journalctl@ceph.osd.5.smithi164.stdout:Jun 03 20:39:00 smithi164 bash[34916]: debug 2023-06-03T20:39:00.344+0000 7fa6fa8f6700 -1 osd.5 324 heartbeat_check: no reply from 172.21.15.89:6806 osd.0 since back 2023-06-03T20:38:12.866069+0000 front 2023-06-03T20:38:12.866084+0000 (oldest deadline 2023-06-03T20:38:41.666721+0000) 2023-06-03T20:39:00.796 INFO:journalctl@ceph.osd.5.smithi164.stdout:Jun 03 20:39:00 smithi164 bash[34916]: debug 2023-06-03T20:39:00.344+0000 7fa6fa8f6700 -1 osd.5 324 heartbeat_check: no reply from 172.21.15.89:6814 osd.1 since back 2023-06-03T20:38:25.768372+0000 front 2023-06-03T20:38:25.768328+0000 (oldest deadline 2023-06-03T20:38:49.268137+0000) 2023-06-03T20:39:01.085 INFO:journalctl@ceph.osd.7.smithi164.stdout:Jun 03 20:39:00 smithi164 bash[43424]: debug 2023-06-03T20:39:00.920+0000 7f5bbd1c4700 -1 osd.7 324 heartbeat_check: no reply from 172.21.15.89:6806 osd.0 since back 2023-06-03T20:38:13.843087+0000 front 2023-06-03T20:38:13.842982+0000 (oldest deadline 2023-06-03T20:38:37.942624+0000) 2023-06-03T20:39:01.085 INFO:journalctl@ceph.osd.7.smithi164.stdout:Jun 03 20:39:00 smithi164 bash[43424]: debug 2023-06-03T20:39:00.920+0000 7f5bbd1c4700 -1 osd.7 324 heartbeat_check: no reply from 172.21.15.89:6814 osd.1 since back 2023-06-03T20:38:23.145022+0000 front 2023-06-03T20:38:23.144779+0000 (oldest deadline 2023-06-03T20:38:47.844517+0000) 2023-06-03T20:39:01.086 INFO:journalctl@ceph.osd.7.smithi164.stdout:Jun 03 20:39:00 smithi164 bash[43424]: debug 2023-06-03T20:39:00.920+0000 7f5bbd1c4700 -1 osd.7 324 heartbeat_check: no reply from 172.21.15.89:6822 osd.2 since back 2023-06-03T20:38:33.046905+0000 front 2023-06-03T20:38:33.047150+0000 (oldest deadline 2023-06-03T20:38:58.847539+0000) 2023-06-03T20:39:01.086 INFO:journalctl@ceph.osd.6.smithi164.stdout:Jun 03 20:39:00 smithi164 bash[39156]: debug 2023-06-03T20:39:00.784+0000 7f06476b1700 -1 osd.6 324 heartbeat_check: no reply from 172.21.15.89:6806 osd.0 since back 2023-06-03T20:38:11.707882+0000 front 2023-06-03T20:38:11.707850+0000 (oldest deadline 2023-06-03T20:38:39.908620+0000) 2023-06-03T20:39:01.086 INFO:journalctl@ceph.osd.6.smithi164.stdout:Jun 03 20:39:00 smithi164 bash[39156]: debug 2023-06-03T20:39:00.784+0000 7f06476b1700 -1 osd.6 324 heartbeat_check: no reply from 172.21.15.89:6814 osd.1 since back 2023-06-03T20:38:23.410387+0000 front 2023-06-03T20:38:23.410208+0000 (oldest deadline 2023-06-03T20:38:47.510018+0000) 2023-06-03T20:39:01.086 INFO:journalctl@ceph.osd.6.smithi164.stdout:Jun 03 20:39:00 smithi164 bash[39156]: debug 2023-06-03T20:39:00.784+0000 7f06476b1700 -1 osd.6 324 heartbeat_check: no reply from 172.21.15.89:6822 osd.2 since back 2023-06-03T20:38:35.012927+0000 front 2023-06-03T20:38:35.012734+0000 (oldest deadline 2023-06-03T20:38:59.712773+0000) 2023-06-03T20:39:01.809 INFO:journalctl@ceph.osd.5.smithi164.stdout:Jun 03 20:39:01 smithi164 bash[34916]: debug 2023-06-03T20:39:01.368+0000 7fa6fa8f6700 -1 osd.5 324 heartbeat_check: no reply from 172.21.15.89:6806 osd.0 since back 2023-06-03T20:38:12.866069+0000 front 2023-06-03T20:38:12.866084+0000 (oldest deadline 2023-06-03T20:38:41.666721+0000) 2023-06-03T20:39:01.809 INFO:journalctl@ceph.osd.5.smithi164.stdout:Jun 03 20:39:01 smithi164 bash[34916]: debug 2023-06-03T20:39:01.368+0000 7fa6fa8f6700 -1 osd.5 324 heartbeat_check: no reply from 172.21.15.89:6814 osd.1 since back 2023-06-03T20:38:25.768372+0000 front 2023-06-03T20:38:25.768328+0000 (oldest deadline 2023-06-03T20:38:49.268137+0000) 2023-06-03T20:39:02.085 INFO:journalctl@ceph.osd.6.smithi164.stdout:Jun 03 20:39:01 smithi164 bash[39156]: debug 2023-06-03T20:39:01.796+0000 7f06476b1700 -1 osd.6 324 heartbeat_check: no reply from 172.21.15.89:6806 osd.0 since back 2023-06-03T20:38:11.707882+0000 front 2023-06-03T20:38:11.707850+0000 (oldest deadline 2023-06-03T20:38:39.908620+0000) 2023-06-03T20:39:02.086 INFO:journalctl@ceph.osd.6.smithi164.stdout:Jun 03 20:39:01 smithi164 bash[39156]: debug 2023-06-03T20:39:01.796+0000 7f06476b1700 -1 osd.6 324 heartbeat_check: no reply from 172.21.15.89:6814 osd.1 since back 2023-06-03T20:38:23.410387+0000 front 2023-06-03T20:38:23.410208+0000 (oldest deadline 2023-06-03T20:38:47.510018+0000) 2023-06-03T20:39:02.086 INFO:journalctl@ceph.osd.6.smithi164.stdout:Jun 03 20:39:01 smithi164 bash[39156]: debug 2023-06-03T20:39:01.796+0000 7f06476b1700 -1 osd.6 324 heartbeat_check: no reply from 172.21.15.89:6822 osd.2 since back 2023-06-03T20:38:35.012927+0000 front 2023-06-03T20:38:35.012734+0000 (oldest deadline 2023-06-03T20:38:59.712773+0000) 2023-06-03T20:39:02.086 INFO:journalctl@ceph.osd.7.smithi164.stdout:Jun 03 20:39:01 smithi164 bash[43424]: debug 2023-06-03T20:39:01.932+0000 7f5bbd1c4700 -1 osd.7 324 heartbeat_check: no reply from 172.21.15.89:6806 osd.0 since back 2023-06-03T20:38:13.843087+0000 front 2023-06-03T20:38:13.842982+0000 (oldest deadline 2023-06-03T20:38:37.942624+0000) 2023-06-03T20:39:02.087 INFO:journalctl@ceph.osd.7.smithi164.stdout:Jun 03 20:39:01 smithi164 bash[43424]: debug 2023-06-03T20:39:01.932+0000 7f5bbd1c4700 -1 osd.7 324 heartbeat_check: no reply from 172.21.15.89:6814 osd.1 since back 2023-06-03T20:38:23.145022+0000 front 2023-06-03T20:38:23.144779+0000 (oldest deadline 2023-06-03T20:38:47.844517+0000) 2023-06-03T20:39:02.087 INFO:journalctl@ceph.osd.7.smithi164.stdout:Jun 03 20:39:01 smithi164 bash[43424]: debug 2023-06-03T20:39:01.932+0000 7f5bbd1c4700 -1 osd.7 324 heartbeat_check: no reply from 172.21.15.89:6822 osd.2 since back 2023-06-03T20:38:33.046905+0000 front 2023-06-03T20:38:33.047150+0000 (oldest deadline 2023-06-03T20:38:58.847539+0000) 2023-06-03T20:39:02.760 INFO:journalctl@ceph.osd.5.smithi164.stdout:Jun 03 20:39:02 smithi164 bash[34916]: debug 2023-06-03T20:39:02.356+0000 7fa6fa8f6700 -1 osd.5 324 heartbeat_check: no reply from 172.21.15.89:6806 osd.0 since back 2023-06-03T20:38:12.866069+0000 front 2023-06-03T20:38:12.866084+0000 (oldest deadline 2023-06-03T20:38:41.666721+0000) 2023-06-03T20:39:02.761 INFO:journalctl@ceph.osd.5.smithi164.stdout:Jun 03 20:39:02 smithi164 bash[34916]: debug 2023-06-03T20:39:02.356+0000 7fa6fa8f6700 -1 osd.5 324 heartbeat_check: no reply from 172.21.15.89:6814 osd.1 since back 2023-06-03T20:38:25.768372+0000 front 2023-06-03T20:38:25.768328+0000 (oldest deadline 2023-06-03T20:38:49.268137+0000) 2023-06-03T20:39:02.761 INFO:journalctl@ceph.osd.5.smithi164.stdout:Jun 03 20:39:02 smithi164 bash[34916]: debug 2023-06-03T20:39:02.356+0000 7fa6fa8f6700 -1 osd.5 324 heartbeat_check: no reply from 172.21.15.89:6822 osd.2 since back 2023-06-03T20:38:35.670318+0000 front 2023-06-03T20:38:35.670223+0000 (oldest deadline 2023-06-03T20:39:02.070474+0000) 2023-06-03T20:39:03.085 INFO:journalctl@ceph.osd.6.smithi164.stdout:Jun 03 20:39:02 smithi164 bash[39156]: debug 2023-06-03T20:39:02.752+0000 7f06476b1700 -1 osd.6 324 heartbeat_check: no reply from 172.21.15.89:6806 osd.0 since back 2023-06-03T20:38:11.707882+0000 front 2023-06-03T20:38:11.707850+0000 (oldest deadline 2023-06-03T20:38:39.908620+0000) 2023-06-03T20:39:03.086 INFO:journalctl@ceph.osd.6.smithi164.stdout:Jun 03 20:39:02 smithi164 bash[39156]: debug 2023-06-03T20:39:02.752+0000 7f06476b1700 -1 osd.6 324 heartbeat_check: no reply from 172.21.15.89:6814 osd.1 since back 2023-06-03T20:38:23.410387+0000 front 2023-06-03T20:38:23.410208+0000 (oldest deadline 2023-06-03T20:38:47.510018+0000) 2023-06-03T20:39:03.086 INFO:journalctl@ceph.osd.6.smithi164.stdout:Jun 03 20:39:02 smithi164 bash[39156]: debug 2023-06-03T20:39:02.752+0000 7f06476b1700 -1 osd.6 324 heartbeat_check: no reply from 172.21.15.89:6822 osd.2 since back 2023-06-03T20:38:35.012927+0000 front 2023-06-03T20:38:35.012734+0000 (oldest deadline 2023-06-03T20:38:59.712773+0000) 2023-06-03T20:39:03.086 INFO:journalctl@ceph.osd.7.smithi164.stdout:Jun 03 20:39:02 smithi164 bash[43424]: debug 2023-06-03T20:39:02.976+0000 7f5bbd1c4700 -1 osd.7 324 heartbeat_check: no reply from 172.21.15.89:6806 osd.0 since back 2023-06-03T20:38:13.843087+0000 front 2023-06-03T20:38:13.842982+0000 (oldest deadline 2023-06-03T20:38:37.942624+0000) 2023-06-03T20:39:03.086 INFO:journalctl@ceph.osd.7.smithi164.stdout:Jun 03 20:39:02 smithi164 bash[43424]: debug 2023-06-03T20:39:02.976+0000 7f5bbd1c4700 -1 osd.7 324 heartbeat_check: no reply from 172.21.15.89:6814 osd.1 since back 2023-06-03T20:38:23.145022+0000 front 2023-06-03T20:38:23.144779+0000 (oldest deadline 2023-06-03T20:38:47.844517+0000) 2023-06-03T20:39:03.087 INFO:journalctl@ceph.osd.7.smithi164.stdout:Jun 03 20:39:02 smithi164 bash[43424]: debug 2023-06-03T20:39:02.976+0000 7f5bbd1c4700 -1 osd.7 324 heartbeat_check: no reply from 172.21.15.89:6822 osd.2 since back 2023-06-03T20:38:33.046905+0000 front 2023-06-03T20:38:33.047150+0000 (oldest deadline 2023-06-03T20:38:58.847539+0000) 2023-06-03T20:39:03.585 INFO:journalctl@ceph.osd.5.smithi164.stdout:Jun 03 20:39:03 smithi164 bash[34916]: debug 2023-06-03T20:39:03.312+0000 7fa6fa8f6700 -1 osd.5 324 heartbeat_check: no reply from 172.21.15.89:6806 osd.0 since back 2023-06-03T20:38:12.866069+0000 front 2023-06-03T20:38:12.866084+0000 (oldest deadline 2023-06-03T20:38:41.666721+0000) 2023-06-03T20:39:03.586 INFO:journalctl@ceph.osd.5.smithi164.stdout:Jun 03 20:39:03 smithi164 bash[34916]: debug 2023-06-03T20:39:03.312+0000 7fa6fa8f6700 -1 osd.5 324 heartbeat_check: no reply from 172.21.15.89:6814 osd.1 since back 2023-06-03T20:38:25.768372+0000 front 2023-06-03T20:38:25.768328+0000 (oldest deadline 2023-06-03T20:38:49.268137+0000) 2023-06-03T20:39:03.586 INFO:journalctl@ceph.osd.5.smithi164.stdout:Jun 03 20:39:03 smithi164 bash[34916]: debug 2023-06-03T20:39:03.312+0000 7fa6fa8f6700 -1 osd.5 324 heartbeat_check: no reply from 172.21.15.89:6822 osd.2 since back 2023-06-03T20:38:35.670318+0000 front 2023-06-03T20:38:35.670223+0000 (oldest deadline 2023-06-03T20:39:02.070474+0000) 2023-06-03T20:39:04.085 INFO:journalctl@ceph.osd.6.smithi164.stdout:Jun 03 20:39:03 smithi164 bash[39156]: debug 2023-06-03T20:39:03.736+0000 7f06476b1700 -1 osd.6 324 heartbeat_check: no reply from 172.21.15.89:6806 osd.0 since back 2023-06-03T20:38:11.707882+0000 front 2023-06-03T20:38:11.707850+0000 (oldest deadline 2023-06-03T20:38:39.908620+0000) 2023-06-03T20:39:04.086 INFO:journalctl@ceph.osd.6.smithi164.stdout:Jun 03 20:39:03 smithi164 bash[39156]: debug 2023-06-03T20:39:03.736+0000 7f06476b1700 -1 osd.6 324 heartbeat_check: no reply from 172.21.15.89:6814 osd.1 since back 2023-06-03T20:38:23.410387+0000 front 2023-06-03T20:38:23.410208+0000 (oldest deadline 2023-06-03T20:38:47.510018+0000) 2023-06-03T20:39:04.086 INFO:journalctl@ceph.osd.6.smithi164.stdout:Jun 03 20:39:03 smithi164 bash[39156]: debug 2023-06-03T20:39:03.736+0000 7f06476b1700 -1 osd.6 324 heartbeat_check: no reply from 172.21.15.89:6822 osd.2 since back 2023-06-03T20:38:35.012927+0000 front 2023-06-03T20:38:35.012734+0000 (oldest deadline 2023-06-03T20:38:59.712773+0000) 2023-06-03T20:39:04.086 INFO:journalctl@ceph.osd.7.smithi164.stdout:Jun 03 20:39:03 smithi164 bash[43424]: debug 2023-06-03T20:39:03.936+0000 7f5bbd1c4700 -1 osd.7 324 heartbeat_check: no reply from 172.21.15.89:6806 osd.0 since back 2023-06-03T20:38:13.843087+0000 front 2023-06-03T20:38:13.842982+0000 (oldest deadline 2023-06-03T20:38:37.942624+0000) 2023-06-03T20:39:04.087 INFO:journalctl@ceph.osd.7.smithi164.stdout:Jun 03 20:39:03 smithi164 bash[43424]: debug 2023-06-03T20:39:03.936+0000 7f5bbd1c4700 -1 osd.7 324 heartbeat_check: no reply from 172.21.15.89:6814 osd.1 since back 2023-06-03T20:38:23.145022+0000 front 2023-06-03T20:38:23.144779+0000 (oldest deadline 2023-06-03T20:38:47.844517+0000) 2023-06-03T20:39:04.087 INFO:journalctl@ceph.osd.7.smithi164.stdout:Jun 03 20:39:03 smithi164 bash[43424]: debug 2023-06-03T20:39:03.936+0000 7f5bbd1c4700 -1 osd.7 324 heartbeat_check: no reply from 172.21.15.89:6822 osd.2 since back 2023-06-03T20:38:33.046905+0000 front 2023-06-03T20:38:33.047150+0000 (oldest deadline 2023-06-03T20:38:58.847539+0000) 2023-06-03T20:39:04.637 INFO:journalctl@ceph.osd.5.smithi164.stdout:Jun 03 20:39:04 smithi164 bash[34916]: debug 2023-06-03T20:39:04.328+0000 7fa6fa8f6700 -1 osd.5 324 heartbeat_check: no reply from 172.21.15.89:6806 osd.0 since back 2023-06-03T20:38:12.866069+0000 front 2023-06-03T20:38:12.866084+0000 (oldest deadline 2023-06-03T20:38:41.666721+0000) 2023-06-03T20:39:04.637 INFO:journalctl@ceph.osd.5.smithi164.stdout:Jun 03 20:39:04 smithi164 bash[34916]: debug 2023-06-03T20:39:04.328+0000 7fa6fa8f6700 -1 osd.5 324 heartbeat_check: no reply from 172.21.15.89:6814 osd.1 since back 2023-06-03T20:38:25.768372+0000 front 2023-06-03T20:38:25.768328+0000 (oldest deadline 2023-06-03T20:38:49.268137+0000) 2023-06-03T20:39:04.637 INFO:journalctl@ceph.osd.5.smithi164.stdout:Jun 03 20:39:04 smithi164 bash[34916]: debug 2023-06-03T20:39:04.328+0000 7fa6fa8f6700 -1 osd.5 324 heartbeat_check: no reply from 172.21.15.89:6822 osd.2 since back 2023-06-03T20:38:35.670318+0000 front 2023-06-03T20:38:35.670223+0000 (oldest deadline 2023-06-03T20:39:02.070474+0000) 2023-06-03T20:39:04.925 INFO:journalctl@ceph.osd.6.smithi164.stdout:Jun 03 20:39:04 smithi164 bash[39156]: debug 2023-06-03T20:39:04.724+0000 7f06476b1700 -1 osd.6 324 heartbeat_check: no reply from 172.21.15.89:6806 osd.0 since back 2023-06-03T20:38:11.707882+0000 front 2023-06-03T20:38:11.707850+0000 (oldest deadline 2023-06-03T20:38:39.908620+0000) 2023-06-03T20:39:04.926 INFO:journalctl@ceph.osd.6.smithi164.stdout:Jun 03 20:39:04 smithi164 bash[39156]: debug 2023-06-03T20:39:04.724+0000 7f06476b1700 -1 osd.6 324 heartbeat_check: no reply from 172.21.15.89:6814 osd.1 since back 2023-06-03T20:38:23.410387+0000 front 2023-06-03T20:38:23.410208+0000 (oldest deadline 2023-06-03T20:38:47.510018+0000) 2023-06-03T20:39:04.926 INFO:journalctl@ceph.osd.6.smithi164.stdout:Jun 03 20:39:04 smithi164 bash[39156]: debug 2023-06-03T20:39:04.724+0000 7f06476b1700 -1 osd.6 324 heartbeat_check: no reply from 172.21.15.89:6822 osd.2 since back 2023-06-03T20:38:35.012927+0000 front 2023-06-03T20:38:35.012734+0000 (oldest deadline 2023-06-03T20:38:59.712773+0000) 2023-06-03T20:39:05.085 INFO:journalctl@ceph.osd.7.smithi164.stdout:Jun 03 20:39:04 smithi164 bash[43424]: debug 2023-06-03T20:39:04.912+0000 7f5bbd1c4700 -1 osd.7 324 heartbeat_check: no reply from 172.21.15.89:6806 osd.0 since back 2023-06-03T20:38:13.843087+0000 front 2023-06-03T20:38:13.842982+0000 (oldest deadline 2023-06-03T20:38:37.942624+0000) 2023-06-03T20:39:05.086 INFO:journalctl@ceph.osd.7.smithi164.stdout:Jun 03 20:39:04 smithi164 bash[43424]: debug 2023-06-03T20:39:04.912+0000 7f5bbd1c4700 -1 osd.7 324 heartbeat_check: no reply from 172.21.15.89:6814 osd.1 since back 2023-06-03T20:38:23.145022+0000 front 2023-06-03T20:38:23.144779+0000 (oldest deadline 2023-06-03T20:38:47.844517+0000) 2023-06-03T20:39:05.086 INFO:journalctl@ceph.osd.7.smithi164.stdout:Jun 03 20:39:04 smithi164 bash[43424]: debug 2023-06-03T20:39:04.912+0000 7f5bbd1c4700 -1 osd.7 324 heartbeat_check: no reply from 172.21.15.89:6822 osd.2 since back 2023-06-03T20:38:33.046905+0000 front 2023-06-03T20:38:33.047150+0000 (oldest deadline 2023-06-03T20:38:58.847539+0000) 2023-06-03T20:39:05.918 INFO:journalctl@ceph.osd.5.smithi164.stdout:Jun 03 20:39:05 smithi164 bash[164497]: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd-5 2023-06-03T20:39:05.919 INFO:journalctl@ceph.osd.6.smithi164.stdout:Jun 03 20:39:05 smithi164 bash[39156]: debug 2023-06-03T20:39:05.740+0000 7f06476b1700 -1 osd.6 324 heartbeat_check: no reply from 172.21.15.89:6806 osd.0 since back 2023-06-03T20:38:11.707882+0000 front 2023-06-03T20:38:11.707850+0000 (oldest deadline 2023-06-03T20:38:39.908620+0000) 2023-06-03T20:39:05.919 INFO:journalctl@ceph.osd.6.smithi164.stdout:Jun 03 20:39:05 smithi164 bash[39156]: debug 2023-06-03T20:39:05.740+0000 7f06476b1700 -1 osd.6 324 heartbeat_check: no reply from 172.21.15.89:6814 osd.1 since back 2023-06-03T20:38:23.410387+0000 front 2023-06-03T20:38:23.410208+0000 (oldest deadline 2023-06-03T20:38:47.510018+0000) 2023-06-03T20:39:05.919 INFO:journalctl@ceph.osd.6.smithi164.stdout:Jun 03 20:39:05 smithi164 bash[39156]: debug 2023-06-03T20:39:05.740+0000 7f06476b1700 -1 osd.6 324 heartbeat_check: no reply from 172.21.15.89:6822 osd.2 since back 2023-06-03T20:38:35.012927+0000 front 2023-06-03T20:38:35.012734+0000 (oldest deadline 2023-06-03T20:38:59.712773+0000) 2023-06-03T20:39:06.085 INFO:journalctl@ceph.osd.7.smithi164.stdout:Jun 03 20:39:05 smithi164 bash[43424]: debug 2023-06-03T20:39:05.908+0000 7f5bbd1c4700 -1 osd.7 324 heartbeat_check: no reply from 172.21.15.89:6806 osd.0 since back 2023-06-03T20:38:13.843087+0000 front 2023-06-03T20:38:13.842982+0000 (oldest deadline 2023-06-03T20:38:37.942624+0000) 2023-06-03T20:39:06.086 INFO:journalctl@ceph.osd.7.smithi164.stdout:Jun 03 20:39:05 smithi164 bash[43424]: debug 2023-06-03T20:39:05.908+0000 7f5bbd1c4700 -1 osd.7 324 heartbeat_check: no reply from 172.21.15.89:6814 osd.1 since back 2023-06-03T20:38:23.145022+0000 front 2023-06-03T20:38:23.144779+0000 (oldest deadline 2023-06-03T20:38:47.844517+0000) 2023-06-03T20:39:06.086 INFO:journalctl@ceph.osd.7.smithi164.stdout:Jun 03 20:39:05 smithi164 bash[43424]: debug 2023-06-03T20:39:05.908+0000 7f5bbd1c4700 -1 osd.7 324 heartbeat_check: no reply from 172.21.15.89:6822 osd.2 since back 2023-06-03T20:38:33.046905+0000 front 2023-06-03T20:38:33.047150+0000 (oldest deadline 2023-06-03T20:38:58.847539+0000) 2023-06-03T20:39:07.013 INFO:journalctl@ceph.osd.6.smithi164.stdout:Jun 03 20:39:06 smithi164 bash[39156]: debug 2023-06-03T20:39:06.748+0000 7f06476b1700 -1 osd.6 324 heartbeat_check: no reply from 172.21.15.89:6806 osd.0 since back 2023-06-03T20:38:11.707882+0000 front 2023-06-03T20:38:11.707850+0000 (oldest deadline 2023-06-03T20:38:39.908620+0000) 2023-06-03T20:39:07.013 INFO:journalctl@ceph.osd.6.smithi164.stdout:Jun 03 20:39:06 smithi164 bash[39156]: debug 2023-06-03T20:39:06.748+0000 7f06476b1700 -1 osd.6 324 heartbeat_check: no reply from 172.21.15.89:6814 osd.1 since back 2023-06-03T20:38:23.410387+0000 front 2023-06-03T20:38:23.410208+0000 (oldest deadline 2023-06-03T20:38:47.510018+0000) 2023-06-03T20:39:07.013 INFO:journalctl@ceph.osd.6.smithi164.stdout:Jun 03 20:39:06 smithi164 bash[39156]: debug 2023-06-03T20:39:06.748+0000 7f06476b1700 -1 osd.6 324 heartbeat_check: no reply from 172.21.15.89:6822 osd.2 since back 2023-06-03T20:38:35.012927+0000 front 2023-06-03T20:38:35.012734+0000 (oldest deadline 2023-06-03T20:38:59.712773+0000) 2023-06-03T20:39:07.014 INFO:journalctl@ceph.osd.7.smithi164.stdout:Jun 03 20:39:06 smithi164 bash[43424]: debug 2023-06-03T20:39:06.892+0000 7f5bbd1c4700 -1 osd.7 324 heartbeat_check: no reply from 172.21.15.89:6806 osd.0 since back 2023-06-03T20:38:13.843087+0000 front 2023-06-03T20:38:13.842982+0000 (oldest deadline 2023-06-03T20:38:37.942624+0000) 2023-06-03T20:39:07.014 INFO:journalctl@ceph.osd.7.smithi164.stdout:Jun 03 20:39:06 smithi164 bash[43424]: debug 2023-06-03T20:39:06.892+0000 7f5bbd1c4700 -1 osd.7 324 heartbeat_check: no reply from 172.21.15.89:6814 osd.1 since back 2023-06-03T20:38:23.145022+0000 front 2023-06-03T20:38:23.144779+0000 (oldest deadline 2023-06-03T20:38:47.844517+0000) 2023-06-03T20:39:07.015 INFO:journalctl@ceph.osd.7.smithi164.stdout:Jun 03 20:39:06 smithi164 bash[43424]: debug 2023-06-03T20:39:06.892+0000 7f5bbd1c4700 -1 osd.7 324 heartbeat_check: no reply from 172.21.15.89:6822 osd.2 since back 2023-06-03T20:38:33.046905+0000 front 2023-06-03T20:38:33.047150+0000 (oldest deadline 2023-06-03T20:38:58.847539+0000) 2023-06-03T20:39:08.085 INFO:journalctl@ceph.osd.6.smithi164.stdout:Jun 03 20:39:07 smithi164 bash[39156]: debug 2023-06-03T20:39:07.704+0000 7f06476b1700 -1 osd.6 324 heartbeat_check: no reply from 172.21.15.89:6806 osd.0 since back 2023-06-03T20:38:11.707882+0000 front 2023-06-03T20:38:11.707850+0000 (oldest deadline 2023-06-03T20:38:39.908620+0000) 2023-06-03T20:39:08.086 INFO:journalctl@ceph.osd.6.smithi164.stdout:Jun 03 20:39:07 smithi164 bash[39156]: debug 2023-06-03T20:39:07.704+0000 7f06476b1700 -1 osd.6 324 heartbeat_check: no reply from 172.21.15.89:6814 osd.1 since back 2023-06-03T20:38:23.410387+0000 front 2023-06-03T20:38:23.410208+0000 (oldest deadline 2023-06-03T20:38:47.510018+0000) 2023-06-03T20:39:08.086 INFO:journalctl@ceph.osd.6.smithi164.stdout:Jun 03 20:39:07 smithi164 bash[39156]: debug 2023-06-03T20:39:07.704+0000 7f06476b1700 -1 osd.6 324 heartbeat_check: no reply from 172.21.15.89:6822 osd.2 since back 2023-06-03T20:38:35.012927+0000 front 2023-06-03T20:38:35.012734+0000 (oldest deadline 2023-06-03T20:38:59.712773+0000) 2023-06-03T20:39:08.086 INFO:journalctl@ceph.osd.6.smithi164.stdout:Jun 03 20:39:07 smithi164 bash[39156]: debug 2023-06-03T20:39:07.704+0000 7f06476b1700 -1 osd.6 324 heartbeat_check: no reply from 172.21.15.89:6830 osd.3 since back 2023-06-03T20:38:44.215632+0000 front 2023-06-03T20:38:44.215471+0000 (oldest deadline 2023-06-03T20:39:07.115235+0000) 2023-06-03T20:39:08.086 INFO:journalctl@ceph.osd.7.smithi164.stdout:Jun 03 20:39:07 smithi164 bash[43424]: debug 2023-06-03T20:39:07.884+0000 7f5bbd1c4700 -1 osd.7 324 heartbeat_check: no reply from 172.21.15.89:6806 osd.0 since back 2023-06-03T20:38:13.843087+0000 front 2023-06-03T20:38:13.842982+0000 (oldest deadline 2023-06-03T20:38:37.942624+0000) 2023-06-03T20:39:08.087 INFO:journalctl@ceph.osd.7.smithi164.stdout:Jun 03 20:39:07 smithi164 bash[43424]: debug 2023-06-03T20:39:07.884+0000 7f5bbd1c4700 -1 osd.7 324 heartbeat_check: no reply from 172.21.15.89:6814 osd.1 since back 2023-06-03T20:38:23.145022+0000 front 2023-06-03T20:38:23.144779+0000 (oldest deadline 2023-06-03T20:38:47.844517+0000) 2023-06-03T20:39:08.087 INFO:journalctl@ceph.osd.7.smithi164.stdout:Jun 03 20:39:07 smithi164 bash[43424]: debug 2023-06-03T20:39:07.884+0000 7f5bbd1c4700 -1 osd.7 324 heartbeat_check: no reply from 172.21.15.89:6822 osd.2 since back 2023-06-03T20:38:33.046905+0000 front 2023-06-03T20:38:33.047150+0000 (oldest deadline 2023-06-03T20:38:58.847539+0000) 2023-06-03T20:39:08.412 DEBUG:teuthology.orchestra.run.smithi164:> sudo pkill -f 'journalctl -f -n 0 -u ceph-f1db3478-0243-11ee-9b1d-001a4aab830c@osd.5.service' 2023-06-03T20:39:08.444 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-06-03T20:39:08.444 INFO:tasks.cephadm.osd.5:Stopped osd.5 2023-06-03T20:39:08.445 INFO:tasks.cephadm.osd.6:Stopping osd.6... 2023-06-03T20:39:08.445 DEBUG:teuthology.orchestra.run.smithi164:> sudo systemctl stop ceph-f1db3478-0243-11ee-9b1d-001a4aab830c@osd.6 2023-06-03T20:39:08.671 INFO:journalctl@ceph.osd.6.smithi164.stdout:Jun 03 20:39:08 smithi164 systemd[1]: Stopping Ceph osd.6 for f1db3478-0243-11ee-9b1d-001a4aab830c... 2023-06-03T20:39:09.085 INFO:journalctl@ceph.osd.7.smithi164.stdout:Jun 03 20:39:08 smithi164 bash[43424]: debug 2023-06-03T20:39:08.896+0000 7f5bbd1c4700 -1 osd.7 324 heartbeat_check: no reply from 172.21.15.89:6806 osd.0 since back 2023-06-03T20:38:13.843087+0000 front 2023-06-03T20:38:13.842982+0000 (oldest deadline 2023-06-03T20:38:37.942624+0000) 2023-06-03T20:39:09.086 INFO:journalctl@ceph.osd.7.smithi164.stdout:Jun 03 20:39:08 smithi164 bash[43424]: debug 2023-06-03T20:39:08.896+0000 7f5bbd1c4700 -1 osd.7 324 heartbeat_check: no reply from 172.21.15.89:6814 osd.1 since back 2023-06-03T20:38:23.145022+0000 front 2023-06-03T20:38:23.144779+0000 (oldest deadline 2023-06-03T20:38:47.844517+0000) 2023-06-03T20:39:09.086 INFO:journalctl@ceph.osd.7.smithi164.stdout:Jun 03 20:39:08 smithi164 bash[43424]: debug 2023-06-03T20:39:08.896+0000 7f5bbd1c4700 -1 osd.7 324 heartbeat_check: no reply from 172.21.15.89:6822 osd.2 since back 2023-06-03T20:38:33.046905+0000 front 2023-06-03T20:38:33.047150+0000 (oldest deadline 2023-06-03T20:38:58.847539+0000) 2023-06-03T20:39:09.086 INFO:journalctl@ceph.osd.6.smithi164.stdout:Jun 03 20:39:08 smithi164 bash[39156]: debug 2023-06-03T20:39:08.660+0000 7f06476b1700 -1 osd.6 324 heartbeat_check: no reply from 172.21.15.89:6806 osd.0 since back 2023-06-03T20:38:11.707882+0000 front 2023-06-03T20:38:11.707850+0000 (oldest deadline 2023-06-03T20:38:39.908620+0000) 2023-06-03T20:39:09.087 INFO:journalctl@ceph.osd.6.smithi164.stdout:Jun 03 20:39:08 smithi164 bash[39156]: debug 2023-06-03T20:39:08.660+0000 7f06476b1700 -1 osd.6 324 heartbeat_check: no reply from 172.21.15.89:6814 osd.1 since back 2023-06-03T20:38:23.410387+0000 front 2023-06-03T20:38:23.410208+0000 (oldest deadline 2023-06-03T20:38:47.510018+0000) 2023-06-03T20:39:09.087 INFO:journalctl@ceph.osd.6.smithi164.stdout:Jun 03 20:39:08 smithi164 bash[39156]: debug 2023-06-03T20:39:08.660+0000 7f06476b1700 -1 osd.6 324 heartbeat_check: no reply from 172.21.15.89:6822 osd.2 since back 2023-06-03T20:38:35.012927+0000 front 2023-06-03T20:38:35.012734+0000 (oldest deadline 2023-06-03T20:38:59.712773+0000) 2023-06-03T20:39:09.087 INFO:journalctl@ceph.osd.6.smithi164.stdout:Jun 03 20:39:08 smithi164 bash[39156]: debug 2023-06-03T20:39:08.660+0000 7f06476b1700 -1 osd.6 324 heartbeat_check: no reply from 172.21.15.89:6830 osd.3 since back 2023-06-03T20:38:44.215632+0000 front 2023-06-03T20:38:44.215471+0000 (oldest deadline 2023-06-03T20:39:07.115235+0000) 2023-06-03T20:39:09.088 INFO:journalctl@ceph.osd.6.smithi164.stdout:Jun 03 20:39:08 smithi164 bash[39156]: debug 2023-06-03T20:39:08.664+0000 7f064cf38700 -1 received signal: Terminated from /sbin/docker-init -- /usr/bin/ceph-osd -n osd.6 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug (PID: 1) UID: 0 2023-06-03T20:39:09.088 INFO:journalctl@ceph.osd.6.smithi164.stdout:Jun 03 20:39:08 smithi164 bash[39156]: debug 2023-06-03T20:39:08.664+0000 7f064cf38700 -1 osd.6 324 *** Got signal Terminated *** 2023-06-03T20:39:09.088 INFO:journalctl@ceph.osd.6.smithi164.stdout:Jun 03 20:39:08 smithi164 bash[39156]: debug 2023-06-03T20:39:08.664+0000 7f064cf38700 -1 osd.6 324 *** Immediate shutdown (osd_fast_shutdown=true) *** 2023-06-03T20:39:09.952 INFO:journalctl@ceph.osd.6.smithi164.stdout:Jun 03 20:39:09 smithi164 bash[39156]: debug 2023-06-03T20:39:09.616+0000 7f06476b1700 -1 osd.6 324 heartbeat_check: no reply from 172.21.15.89:6806 osd.0 since back 2023-06-03T20:38:11.707882+0000 front 2023-06-03T20:38:11.707850+0000 (oldest deadline 2023-06-03T20:38:39.908620+0000) 2023-06-03T20:39:09.953 INFO:journalctl@ceph.osd.6.smithi164.stdout:Jun 03 20:39:09 smithi164 bash[39156]: debug 2023-06-03T20:39:09.616+0000 7f06476b1700 -1 osd.6 324 heartbeat_check: no reply from 172.21.15.89:6814 osd.1 since back 2023-06-03T20:38:23.410387+0000 front 2023-06-03T20:38:23.410208+0000 (oldest deadline 2023-06-03T20:38:47.510018+0000) 2023-06-03T20:39:09.953 INFO:journalctl@ceph.osd.6.smithi164.stdout:Jun 03 20:39:09 smithi164 bash[39156]: debug 2023-06-03T20:39:09.616+0000 7f06476b1700 -1 osd.6 324 heartbeat_check: no reply from 172.21.15.89:6822 osd.2 since back 2023-06-03T20:38:35.012927+0000 front 2023-06-03T20:38:35.012734+0000 (oldest deadline 2023-06-03T20:38:59.712773+0000) 2023-06-03T20:39:09.953 INFO:journalctl@ceph.osd.6.smithi164.stdout:Jun 03 20:39:09 smithi164 bash[39156]: debug 2023-06-03T20:39:09.616+0000 7f06476b1700 -1 osd.6 324 heartbeat_check: no reply from 172.21.15.89:6830 osd.3 since back 2023-06-03T20:38:44.215632+0000 front 2023-06-03T20:38:44.215471+0000 (oldest deadline 2023-06-03T20:39:07.115235+0000) 2023-06-03T20:39:10.335 INFO:journalctl@ceph.osd.7.smithi164.stdout:Jun 03 20:39:09 smithi164 bash[43424]: debug 2023-06-03T20:39:09.940+0000 7f5bbd1c4700 -1 osd.7 324 heartbeat_check: no reply from 172.21.15.89:6806 osd.0 since back 2023-06-03T20:38:13.843087+0000 front 2023-06-03T20:38:13.842982+0000 (oldest deadline 2023-06-03T20:38:37.942624+0000) 2023-06-03T20:39:10.336 INFO:journalctl@ceph.osd.7.smithi164.stdout:Jun 03 20:39:09 smithi164 bash[43424]: debug 2023-06-03T20:39:09.940+0000 7f5bbd1c4700 -1 osd.7 324 heartbeat_check: no reply from 172.21.15.89:6814 osd.1 since back 2023-06-03T20:38:23.145022+0000 front 2023-06-03T20:38:23.144779+0000 (oldest deadline 2023-06-03T20:38:47.844517+0000) 2023-06-03T20:39:10.336 INFO:journalctl@ceph.osd.7.smithi164.stdout:Jun 03 20:39:09 smithi164 bash[43424]: debug 2023-06-03T20:39:09.940+0000 7f5bbd1c4700 -1 osd.7 324 heartbeat_check: no reply from 172.21.15.89:6822 osd.2 since back 2023-06-03T20:38:33.046905+0000 front 2023-06-03T20:38:33.047150+0000 (oldest deadline 2023-06-03T20:38:58.847539+0000) 2023-06-03T20:39:10.336 INFO:journalctl@ceph.osd.7.smithi164.stdout:Jun 03 20:39:09 smithi164 bash[43424]: debug 2023-06-03T20:39:09.940+0000 7f5bbd1c4700 -1 osd.7 324 heartbeat_check: no reply from 172.21.15.89:6830 osd.3 since back 2023-06-03T20:38:44.649782+0000 front 2023-06-03T20:38:44.649880+0000 (oldest deadline 2023-06-03T20:39:09.850095+0000) 2023-06-03T20:39:10.939 INFO:journalctl@ceph.osd.6.smithi164.stdout:Jun 03 20:39:10 smithi164 bash[39156]: debug 2023-06-03T20:39:10.624+0000 7f06476b1700 -1 osd.6 324 heartbeat_check: no reply from 172.21.15.89:6806 osd.0 since back 2023-06-03T20:38:11.707882+0000 front 2023-06-03T20:38:11.707850+0000 (oldest deadline 2023-06-03T20:38:39.908620+0000) 2023-06-03T20:39:10.940 INFO:journalctl@ceph.osd.6.smithi164.stdout:Jun 03 20:39:10 smithi164 bash[39156]: debug 2023-06-03T20:39:10.624+0000 7f06476b1700 -1 osd.6 324 heartbeat_check: no reply from 172.21.15.89:6814 osd.1 since back 2023-06-03T20:38:23.410387+0000 front 2023-06-03T20:38:23.410208+0000 (oldest deadline 2023-06-03T20:38:47.510018+0000) 2023-06-03T20:39:10.940 INFO:journalctl@ceph.osd.6.smithi164.stdout:Jun 03 20:39:10 smithi164 bash[39156]: debug 2023-06-03T20:39:10.624+0000 7f06476b1700 -1 osd.6 324 heartbeat_check: no reply from 172.21.15.89:6822 osd.2 since back 2023-06-03T20:38:35.012927+0000 front 2023-06-03T20:38:35.012734+0000 (oldest deadline 2023-06-03T20:38:59.712773+0000) 2023-06-03T20:39:10.940 INFO:journalctl@ceph.osd.6.smithi164.stdout:Jun 03 20:39:10 smithi164 bash[39156]: debug 2023-06-03T20:39:10.624+0000 7f06476b1700 -1 osd.6 324 heartbeat_check: no reply from 172.21.15.89:6830 osd.3 since back 2023-06-03T20:38:44.215632+0000 front 2023-06-03T20:38:44.215471+0000 (oldest deadline 2023-06-03T20:39:07.115235+0000) 2023-06-03T20:39:11.335 INFO:journalctl@ceph.osd.7.smithi164.stdout:Jun 03 20:39:10 smithi164 bash[43424]: debug 2023-06-03T20:39:10.928+0000 7f5bbd1c4700 -1 osd.7 324 heartbeat_check: no reply from 172.21.15.89:6806 osd.0 since back 2023-06-03T20:38:13.843087+0000 front 2023-06-03T20:38:13.842982+0000 (oldest deadline 2023-06-03T20:38:37.942624+0000) 2023-06-03T20:39:11.336 INFO:journalctl@ceph.osd.7.smithi164.stdout:Jun 03 20:39:10 smithi164 bash[43424]: debug 2023-06-03T20:39:10.928+0000 7f5bbd1c4700 -1 osd.7 324 heartbeat_check: no reply from 172.21.15.89:6814 osd.1 since back 2023-06-03T20:38:23.145022+0000 front 2023-06-03T20:38:23.144779+0000 (oldest deadline 2023-06-03T20:38:47.844517+0000) 2023-06-03T20:39:11.337 INFO:journalctl@ceph.osd.7.smithi164.stdout:Jun 03 20:39:10 smithi164 bash[43424]: debug 2023-06-03T20:39:10.928+0000 7f5bbd1c4700 -1 osd.7 324 heartbeat_check: no reply from 172.21.15.89:6822 osd.2 since back 2023-06-03T20:38:33.046905+0000 front 2023-06-03T20:38:33.047150+0000 (oldest deadline 2023-06-03T20:38:58.847539+0000) 2023-06-03T20:39:11.337 INFO:journalctl@ceph.osd.7.smithi164.stdout:Jun 03 20:39:10 smithi164 bash[43424]: debug 2023-06-03T20:39:10.928+0000 7f5bbd1c4700 -1 osd.7 324 heartbeat_check: no reply from 172.21.15.89:6830 osd.3 since back 2023-06-03T20:38:44.649782+0000 front 2023-06-03T20:38:44.649880+0000 (oldest deadline 2023-06-03T20:39:09.850095+0000) 2023-06-03T20:39:11.969 INFO:journalctl@ceph.osd.6.smithi164.stdout:Jun 03 20:39:11 smithi164 bash[39156]: debug 2023-06-03T20:39:11.600+0000 7f06476b1700 -1 osd.6 324 heartbeat_check: no reply from 172.21.15.89:6806 osd.0 since back 2023-06-03T20:38:11.707882+0000 front 2023-06-03T20:38:11.707850+0000 (oldest deadline 2023-06-03T20:38:39.908620+0000) 2023-06-03T20:39:11.970 INFO:journalctl@ceph.osd.6.smithi164.stdout:Jun 03 20:39:11 smithi164 bash[39156]: debug 2023-06-03T20:39:11.600+0000 7f06476b1700 -1 osd.6 324 heartbeat_check: no reply from 172.21.15.89:6814 osd.1 since back 2023-06-03T20:38:23.410387+0000 front 2023-06-03T20:38:23.410208+0000 (oldest deadline 2023-06-03T20:38:47.510018+0000) 2023-06-03T20:39:11.970 INFO:journalctl@ceph.osd.6.smithi164.stdout:Jun 03 20:39:11 smithi164 bash[39156]: debug 2023-06-03T20:39:11.600+0000 7f06476b1700 -1 osd.6 324 heartbeat_check: no reply from 172.21.15.89:6822 osd.2 since back 2023-06-03T20:38:35.012927+0000 front 2023-06-03T20:38:35.012734+0000 (oldest deadline 2023-06-03T20:38:59.712773+0000) 2023-06-03T20:39:11.970 INFO:journalctl@ceph.osd.6.smithi164.stdout:Jun 03 20:39:11 smithi164 bash[39156]: debug 2023-06-03T20:39:11.600+0000 7f06476b1700 -1 osd.6 324 heartbeat_check: no reply from 172.21.15.89:6830 osd.3 since back 2023-06-03T20:38:44.215632+0000 front 2023-06-03T20:38:44.215471+0000 (oldest deadline 2023-06-03T20:39:07.115235+0000) 2023-06-03T20:39:12.335 INFO:journalctl@ceph.osd.7.smithi164.stdout:Jun 03 20:39:11 smithi164 bash[43424]: debug 2023-06-03T20:39:11.960+0000 7f5bbd1c4700 -1 osd.7 324 heartbeat_check: no reply from 172.21.15.89:6806 osd.0 since back 2023-06-03T20:38:13.843087+0000 front 2023-06-03T20:38:13.842982+0000 (oldest deadline 2023-06-03T20:38:37.942624+0000) 2023-06-03T20:39:12.336 INFO:journalctl@ceph.osd.7.smithi164.stdout:Jun 03 20:39:11 smithi164 bash[43424]: debug 2023-06-03T20:39:11.960+0000 7f5bbd1c4700 -1 osd.7 324 heartbeat_check: no reply from 172.21.15.89:6814 osd.1 since back 2023-06-03T20:38:23.145022+0000 front 2023-06-03T20:38:23.144779+0000 (oldest deadline 2023-06-03T20:38:47.844517+0000) 2023-06-03T20:39:12.336 INFO:journalctl@ceph.osd.7.smithi164.stdout:Jun 03 20:39:11 smithi164 bash[43424]: debug 2023-06-03T20:39:11.960+0000 7f5bbd1c4700 -1 osd.7 324 heartbeat_check: no reply from 172.21.15.89:6822 osd.2 since back 2023-06-03T20:38:33.046905+0000 front 2023-06-03T20:38:33.047150+0000 (oldest deadline 2023-06-03T20:38:58.847539+0000) 2023-06-03T20:39:12.336 INFO:journalctl@ceph.osd.7.smithi164.stdout:Jun 03 20:39:11 smithi164 bash[43424]: debug 2023-06-03T20:39:11.960+0000 7f5bbd1c4700 -1 osd.7 324 heartbeat_check: no reply from 172.21.15.89:6830 osd.3 since back 2023-06-03T20:38:44.649782+0000 front 2023-06-03T20:38:44.649880+0000 (oldest deadline 2023-06-03T20:39:09.850095+0000) 2023-06-03T20:39:12.835 INFO:journalctl@ceph.osd.6.smithi164.stdout:Jun 03 20:39:12 smithi164 bash[39156]: debug 2023-06-03T20:39:12.552+0000 7f06476b1700 -1 osd.6 324 heartbeat_check: no reply from 172.21.15.89:6806 osd.0 since back 2023-06-03T20:38:11.707882+0000 front 2023-06-03T20:38:11.707850+0000 (oldest deadline 2023-06-03T20:38:39.908620+0000) 2023-06-03T20:39:12.836 INFO:journalctl@ceph.osd.6.smithi164.stdout:Jun 03 20:39:12 smithi164 bash[39156]: debug 2023-06-03T20:39:12.552+0000 7f06476b1700 -1 osd.6 324 heartbeat_check: no reply from 172.21.15.89:6814 osd.1 since back 2023-06-03T20:38:23.410387+0000 front 2023-06-03T20:38:23.410208+0000 (oldest deadline 2023-06-03T20:38:47.510018+0000) 2023-06-03T20:39:12.836 INFO:journalctl@ceph.osd.6.smithi164.stdout:Jun 03 20:39:12 smithi164 bash[39156]: debug 2023-06-03T20:39:12.552+0000 7f06476b1700 -1 osd.6 324 heartbeat_check: no reply from 172.21.15.89:6822 osd.2 since back 2023-06-03T20:38:35.012927+0000 front 2023-06-03T20:38:35.012734+0000 (oldest deadline 2023-06-03T20:38:59.712773+0000) 2023-06-03T20:39:12.836 INFO:journalctl@ceph.osd.6.smithi164.stdout:Jun 03 20:39:12 smithi164 bash[39156]: debug 2023-06-03T20:39:12.552+0000 7f06476b1700 -1 osd.6 324 heartbeat_check: no reply from 172.21.15.89:6830 osd.3 since back 2023-06-03T20:38:44.215632+0000 front 2023-06-03T20:38:44.215471+0000 (oldest deadline 2023-06-03T20:39:07.115235+0000) 2023-06-03T20:39:13.335 INFO:journalctl@ceph.osd.7.smithi164.stdout:Jun 03 20:39:12 smithi164 bash[43424]: debug 2023-06-03T20:39:12.988+0000 7f5bbd1c4700 -1 osd.7 324 heartbeat_check: no reply from 172.21.15.89:6806 osd.0 since back 2023-06-03T20:38:13.843087+0000 front 2023-06-03T20:38:13.842982+0000 (oldest deadline 2023-06-03T20:38:37.942624+0000) 2023-06-03T20:39:13.336 INFO:journalctl@ceph.osd.7.smithi164.stdout:Jun 03 20:39:12 smithi164 bash[43424]: debug 2023-06-03T20:39:12.988+0000 7f5bbd1c4700 -1 osd.7 324 heartbeat_check: no reply from 172.21.15.89:6814 osd.1 since back 2023-06-03T20:38:23.145022+0000 front 2023-06-03T20:38:23.144779+0000 (oldest deadline 2023-06-03T20:38:47.844517+0000) 2023-06-03T20:39:13.336 INFO:journalctl@ceph.osd.7.smithi164.stdout:Jun 03 20:39:12 smithi164 bash[43424]: debug 2023-06-03T20:39:12.988+0000 7f5bbd1c4700 -1 osd.7 324 heartbeat_check: no reply from 172.21.15.89:6822 osd.2 since back 2023-06-03T20:38:33.046905+0000 front 2023-06-03T20:38:33.047150+0000 (oldest deadline 2023-06-03T20:38:58.847539+0000) 2023-06-03T20:39:13.336 INFO:journalctl@ceph.osd.7.smithi164.stdout:Jun 03 20:39:12 smithi164 bash[43424]: debug 2023-06-03T20:39:12.988+0000 7f5bbd1c4700 -1 osd.7 324 heartbeat_check: no reply from 172.21.15.89:6830 osd.3 since back 2023-06-03T20:38:44.649782+0000 front 2023-06-03T20:38:44.649880+0000 (oldest deadline 2023-06-03T20:39:09.850095+0000) 2023-06-03T20:39:13.835 INFO:journalctl@ceph.osd.6.smithi164.stdout:Jun 03 20:39:13 smithi164 bash[39156]: debug 2023-06-03T20:39:13.532+0000 7f06476b1700 -1 osd.6 324 heartbeat_check: no reply from 172.21.15.89:6806 osd.0 since back 2023-06-03T20:38:11.707882+0000 front 2023-06-03T20:38:11.707850+0000 (oldest deadline 2023-06-03T20:38:39.908620+0000) 2023-06-03T20:39:13.835 INFO:journalctl@ceph.osd.6.smithi164.stdout:Jun 03 20:39:13 smithi164 bash[39156]: debug 2023-06-03T20:39:13.532+0000 7f06476b1700 -1 osd.6 324 heartbeat_check: no reply from 172.21.15.89:6814 osd.1 since back 2023-06-03T20:38:23.410387+0000 front 2023-06-03T20:38:23.410208+0000 (oldest deadline 2023-06-03T20:38:47.510018+0000) 2023-06-03T20:39:13.836 INFO:journalctl@ceph.osd.6.smithi164.stdout:Jun 03 20:39:13 smithi164 bash[39156]: debug 2023-06-03T20:39:13.532+0000 7f06476b1700 -1 osd.6 324 heartbeat_check: no reply from 172.21.15.89:6822 osd.2 since back 2023-06-03T20:38:35.012927+0000 front 2023-06-03T20:38:35.012734+0000 (oldest deadline 2023-06-03T20:38:59.712773+0000) 2023-06-03T20:39:13.836 INFO:journalctl@ceph.osd.6.smithi164.stdout:Jun 03 20:39:13 smithi164 bash[39156]: debug 2023-06-03T20:39:13.532+0000 7f06476b1700 -1 osd.6 324 heartbeat_check: no reply from 172.21.15.89:6830 osd.3 since back 2023-06-03T20:38:44.215632+0000 front 2023-06-03T20:38:44.215471+0000 (oldest deadline 2023-06-03T20:39:07.115235+0000) 2023-06-03T20:39:14.335 INFO:journalctl@ceph.osd.7.smithi164.stdout:Jun 03 20:39:14 smithi164 bash[43424]: debug 2023-06-03T20:39:14.028+0000 7f5bbd1c4700 -1 osd.7 324 heartbeat_check: no reply from 172.21.15.89:6806 osd.0 since back 2023-06-03T20:38:13.843087+0000 front 2023-06-03T20:38:13.842982+0000 (oldest deadline 2023-06-03T20:38:37.942624+0000) 2023-06-03T20:39:14.336 INFO:journalctl@ceph.osd.7.smithi164.stdout:Jun 03 20:39:14 smithi164 bash[43424]: debug 2023-06-03T20:39:14.028+0000 7f5bbd1c4700 -1 osd.7 324 heartbeat_check: no reply from 172.21.15.89:6814 osd.1 since back 2023-06-03T20:38:23.145022+0000 front 2023-06-03T20:38:23.144779+0000 (oldest deadline 2023-06-03T20:38:47.844517+0000) 2023-06-03T20:39:14.336 INFO:journalctl@ceph.osd.7.smithi164.stdout:Jun 03 20:39:14 smithi164 bash[43424]: debug 2023-06-03T20:39:14.028+0000 7f5bbd1c4700 -1 osd.7 324 heartbeat_check: no reply from 172.21.15.89:6822 osd.2 since back 2023-06-03T20:38:33.046905+0000 front 2023-06-03T20:38:33.047150+0000 (oldest deadline 2023-06-03T20:38:58.847539+0000) 2023-06-03T20:39:14.336 INFO:journalctl@ceph.osd.7.smithi164.stdout:Jun 03 20:39:14 smithi164 bash[43424]: debug 2023-06-03T20:39:14.028+0000 7f5bbd1c4700 -1 osd.7 324 heartbeat_check: no reply from 172.21.15.89:6830 osd.3 since back 2023-06-03T20:38:44.649782+0000 front 2023-06-03T20:38:44.649880+0000 (oldest deadline 2023-06-03T20:39:09.850095+0000) 2023-06-03T20:39:15.038 INFO:journalctl@ceph.osd.6.smithi164.stdout:Jun 03 20:39:14 smithi164 bash[164717]: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd-6 2023-06-03T20:39:15.335 INFO:journalctl@ceph.osd.7.smithi164.stdout:Jun 03 20:39:15 smithi164 bash[43424]: debug 2023-06-03T20:39:15.028+0000 7f5bbd1c4700 -1 osd.7 324 heartbeat_check: no reply from 172.21.15.89:6806 osd.0 since back 2023-06-03T20:38:13.843087+0000 front 2023-06-03T20:38:13.842982+0000 (oldest deadline 2023-06-03T20:38:37.942624+0000) 2023-06-03T20:39:15.336 INFO:journalctl@ceph.osd.7.smithi164.stdout:Jun 03 20:39:15 smithi164 bash[43424]: debug 2023-06-03T20:39:15.028+0000 7f5bbd1c4700 -1 osd.7 324 heartbeat_check: no reply from 172.21.15.89:6814 osd.1 since back 2023-06-03T20:38:23.145022+0000 front 2023-06-03T20:38:23.144779+0000 (oldest deadline 2023-06-03T20:38:47.844517+0000) 2023-06-03T20:39:15.336 INFO:journalctl@ceph.osd.7.smithi164.stdout:Jun 03 20:39:15 smithi164 bash[43424]: debug 2023-06-03T20:39:15.028+0000 7f5bbd1c4700 -1 osd.7 324 heartbeat_check: no reply from 172.21.15.89:6822 osd.2 since back 2023-06-03T20:38:33.046905+0000 front 2023-06-03T20:38:33.047150+0000 (oldest deadline 2023-06-03T20:38:58.847539+0000) 2023-06-03T20:39:15.336 INFO:journalctl@ceph.osd.7.smithi164.stdout:Jun 03 20:39:15 smithi164 bash[43424]: debug 2023-06-03T20:39:15.028+0000 7f5bbd1c4700 -1 osd.7 324 heartbeat_check: no reply from 172.21.15.89:6830 osd.3 since back 2023-06-03T20:38:44.649782+0000 front 2023-06-03T20:38:44.649880+0000 (oldest deadline 2023-06-03T20:39:09.850095+0000) 2023-06-03T20:39:16.335 INFO:journalctl@ceph.osd.7.smithi164.stdout:Jun 03 20:39:16 smithi164 bash[43424]: debug 2023-06-03T20:39:16.028+0000 7f5bbd1c4700 -1 osd.7 324 heartbeat_check: no reply from 172.21.15.89:6806 osd.0 since back 2023-06-03T20:38:13.843087+0000 front 2023-06-03T20:38:13.842982+0000 (oldest deadline 2023-06-03T20:38:37.942624+0000) 2023-06-03T20:39:16.336 INFO:journalctl@ceph.osd.7.smithi164.stdout:Jun 03 20:39:16 smithi164 bash[43424]: debug 2023-06-03T20:39:16.028+0000 7f5bbd1c4700 -1 osd.7 324 heartbeat_check: no reply from 172.21.15.89:6814 osd.1 since back 2023-06-03T20:38:23.145022+0000 front 2023-06-03T20:38:23.144779+0000 (oldest deadline 2023-06-03T20:38:47.844517+0000) 2023-06-03T20:39:16.336 INFO:journalctl@ceph.osd.7.smithi164.stdout:Jun 03 20:39:16 smithi164 bash[43424]: debug 2023-06-03T20:39:16.028+0000 7f5bbd1c4700 -1 osd.7 324 heartbeat_check: no reply from 172.21.15.89:6822 osd.2 since back 2023-06-03T20:38:33.046905+0000 front 2023-06-03T20:38:33.047150+0000 (oldest deadline 2023-06-03T20:38:58.847539+0000) 2023-06-03T20:39:16.336 INFO:journalctl@ceph.osd.7.smithi164.stdout:Jun 03 20:39:16 smithi164 bash[43424]: debug 2023-06-03T20:39:16.028+0000 7f5bbd1c4700 -1 osd.7 324 heartbeat_check: no reply from 172.21.15.89:6830 osd.3 since back 2023-06-03T20:38:44.649782+0000 front 2023-06-03T20:38:44.649880+0000 (oldest deadline 2023-06-03T20:39:09.850095+0000) 2023-06-03T20:39:16.336 INFO:journalctl@ceph.osd.7.smithi164.stdout:Jun 03 20:39:16 smithi164 bash[43424]: debug 2023-06-03T20:39:16.028+0000 7f5bbd1c4700 -1 osd.7 324 heartbeat_check: no reply from 172.21.15.164:6804 osd.4 since back 2023-06-03T20:38:49.851095+0000 front 2023-06-03T20:38:49.851066+0000 (oldest deadline 2023-06-03T20:39:15.750731+0000) 2023-06-03T20:39:17.180 DEBUG:teuthology.orchestra.run.smithi164:> sudo pkill -f 'journalctl -f -n 0 -u ceph-f1db3478-0243-11ee-9b1d-001a4aab830c@osd.6.service' 2023-06-03T20:39:17.206 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-06-03T20:39:17.207 INFO:tasks.cephadm.osd.6:Stopped osd.6 2023-06-03T20:39:17.208 INFO:tasks.cephadm.osd.7:Stopping osd.7... 2023-06-03T20:39:17.208 DEBUG:teuthology.orchestra.run.smithi164:> sudo systemctl stop ceph-f1db3478-0243-11ee-9b1d-001a4aab830c@osd.7 2023-06-03T20:39:17.335 INFO:journalctl@ceph.osd.7.smithi164.stdout:Jun 03 20:39:17 smithi164 bash[43424]: debug 2023-06-03T20:39:17.044+0000 7f5bbd1c4700 -1 osd.7 324 heartbeat_check: no reply from 172.21.15.89:6806 osd.0 since back 2023-06-03T20:38:13.843087+0000 front 2023-06-03T20:38:13.842982+0000 (oldest deadline 2023-06-03T20:38:37.942624+0000) 2023-06-03T20:39:17.335 INFO:journalctl@ceph.osd.7.smithi164.stdout:Jun 03 20:39:17 smithi164 bash[43424]: debug 2023-06-03T20:39:17.044+0000 7f5bbd1c4700 -1 osd.7 324 heartbeat_check: no reply from 172.21.15.89:6814 osd.1 since back 2023-06-03T20:38:23.145022+0000 front 2023-06-03T20:38:23.144779+0000 (oldest deadline 2023-06-03T20:38:47.844517+0000) 2023-06-03T20:39:17.336 INFO:journalctl@ceph.osd.7.smithi164.stdout:Jun 03 20:39:17 smithi164 bash[43424]: debug 2023-06-03T20:39:17.044+0000 7f5bbd1c4700 -1 osd.7 324 heartbeat_check: no reply from 172.21.15.89:6822 osd.2 since back 2023-06-03T20:38:33.046905+0000 front 2023-06-03T20:38:33.047150+0000 (oldest deadline 2023-06-03T20:38:58.847539+0000) 2023-06-03T20:39:17.336 INFO:journalctl@ceph.osd.7.smithi164.stdout:Jun 03 20:39:17 smithi164 bash[43424]: debug 2023-06-03T20:39:17.044+0000 7f5bbd1c4700 -1 osd.7 324 heartbeat_check: no reply from 172.21.15.89:6830 osd.3 since back 2023-06-03T20:38:44.649782+0000 front 2023-06-03T20:38:44.649880+0000 (oldest deadline 2023-06-03T20:39:09.850095+0000) 2023-06-03T20:39:17.336 INFO:journalctl@ceph.osd.7.smithi164.stdout:Jun 03 20:39:17 smithi164 bash[43424]: debug 2023-06-03T20:39:17.044+0000 7f5bbd1c4700 -1 osd.7 324 heartbeat_check: no reply from 172.21.15.164:6804 osd.4 since back 2023-06-03T20:38:49.851095+0000 front 2023-06-03T20:38:49.851066+0000 (oldest deadline 2023-06-03T20:39:15.750731+0000) 2023-06-03T20:39:17.336 INFO:journalctl@ceph.osd.7.smithi164.stdout:Jun 03 20:39:17 smithi164 systemd[1]: Stopping Ceph osd.7 for f1db3478-0243-11ee-9b1d-001a4aab830c... 2023-06-03T20:39:17.835 INFO:journalctl@ceph.osd.7.smithi164.stdout:Jun 03 20:39:17 smithi164 bash[43424]: debug 2023-06-03T20:39:17.420+0000 7f5bbfa45700 -1 received signal: Terminated from /sbin/docker-init -- /usr/bin/ceph-osd -n osd.7 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug (PID: 1) UID: 0 2023-06-03T20:39:17.835 INFO:journalctl@ceph.osd.7.smithi164.stdout:Jun 03 20:39:17 smithi164 bash[43424]: debug 2023-06-03T20:39:17.420+0000 7f5bbfa45700 -1 osd.7 324 *** Got signal Terminated *** 2023-06-03T20:39:17.836 INFO:journalctl@ceph.osd.7.smithi164.stdout:Jun 03 20:39:17 smithi164 bash[43424]: debug 2023-06-03T20:39:17.420+0000 7f5bbfa45700 -1 osd.7 324 *** Immediate shutdown (osd_fast_shutdown=true) *** 2023-06-03T20:39:18.335 INFO:journalctl@ceph.osd.7.smithi164.stdout:Jun 03 20:39:18 smithi164 bash[43424]: debug 2023-06-03T20:39:18.032+0000 7f5bbd1c4700 -1 osd.7 324 heartbeat_check: no reply from 172.21.15.89:6806 osd.0 since back 2023-06-03T20:38:13.843087+0000 front 2023-06-03T20:38:13.842982+0000 (oldest deadline 2023-06-03T20:38:37.942624+0000) 2023-06-03T20:39:18.336 INFO:journalctl@ceph.osd.7.smithi164.stdout:Jun 03 20:39:18 smithi164 bash[43424]: debug 2023-06-03T20:39:18.032+0000 7f5bbd1c4700 -1 osd.7 324 heartbeat_check: no reply from 172.21.15.89:6814 osd.1 since back 2023-06-03T20:38:23.145022+0000 front 2023-06-03T20:38:23.144779+0000 (oldest deadline 2023-06-03T20:38:47.844517+0000) 2023-06-03T20:39:18.336 INFO:journalctl@ceph.osd.7.smithi164.stdout:Jun 03 20:39:18 smithi164 bash[43424]: debug 2023-06-03T20:39:18.032+0000 7f5bbd1c4700 -1 osd.7 324 heartbeat_check: no reply from 172.21.15.89:6822 osd.2 since back 2023-06-03T20:38:33.046905+0000 front 2023-06-03T20:38:33.047150+0000 (oldest deadline 2023-06-03T20:38:58.847539+0000) 2023-06-03T20:39:18.336 INFO:journalctl@ceph.osd.7.smithi164.stdout:Jun 03 20:39:18 smithi164 bash[43424]: debug 2023-06-03T20:39:18.032+0000 7f5bbd1c4700 -1 osd.7 324 heartbeat_check: no reply from 172.21.15.89:6830 osd.3 since back 2023-06-03T20:38:44.649782+0000 front 2023-06-03T20:38:44.649880+0000 (oldest deadline 2023-06-03T20:39:09.850095+0000) 2023-06-03T20:39:18.337 INFO:journalctl@ceph.osd.7.smithi164.stdout:Jun 03 20:39:18 smithi164 bash[43424]: debug 2023-06-03T20:39:18.032+0000 7f5bbd1c4700 -1 osd.7 324 heartbeat_check: no reply from 172.21.15.164:6804 osd.4 since back 2023-06-03T20:38:49.851095+0000 front 2023-06-03T20:38:49.851066+0000 (oldest deadline 2023-06-03T20:39:15.750731+0000) 2023-06-03T20:39:19.336 INFO:journalctl@ceph.osd.7.smithi164.stdout:Jun 03 20:39:19 smithi164 bash[43424]: debug 2023-06-03T20:39:19.019+0000 7f5bbd1c4700 -1 osd.7 324 heartbeat_check: no reply from 172.21.15.89:6806 osd.0 since back 2023-06-03T20:38:13.843087+0000 front 2023-06-03T20:38:13.842982+0000 (oldest deadline 2023-06-03T20:38:37.942624+0000) 2023-06-03T20:39:19.336 INFO:journalctl@ceph.osd.7.smithi164.stdout:Jun 03 20:39:19 smithi164 bash[43424]: debug 2023-06-03T20:39:19.019+0000 7f5bbd1c4700 -1 osd.7 324 heartbeat_check: no reply from 172.21.15.89:6814 osd.1 since back 2023-06-03T20:38:23.145022+0000 front 2023-06-03T20:38:23.144779+0000 (oldest deadline 2023-06-03T20:38:47.844517+0000) 2023-06-03T20:39:19.337 INFO:journalctl@ceph.osd.7.smithi164.stdout:Jun 03 20:39:19 smithi164 bash[43424]: debug 2023-06-03T20:39:19.019+0000 7f5bbd1c4700 -1 osd.7 324 heartbeat_check: no reply from 172.21.15.89:6822 osd.2 since back 2023-06-03T20:38:33.046905+0000 front 2023-06-03T20:38:33.047150+0000 (oldest deadline 2023-06-03T20:38:58.847539+0000) 2023-06-03T20:39:19.337 INFO:journalctl@ceph.osd.7.smithi164.stdout:Jun 03 20:39:19 smithi164 bash[43424]: debug 2023-06-03T20:39:19.019+0000 7f5bbd1c4700 -1 osd.7 324 heartbeat_check: no reply from 172.21.15.89:6830 osd.3 since back 2023-06-03T20:38:44.649782+0000 front 2023-06-03T20:38:44.649880+0000 (oldest deadline 2023-06-03T20:39:09.850095+0000) 2023-06-03T20:39:19.337 INFO:journalctl@ceph.osd.7.smithi164.stdout:Jun 03 20:39:19 smithi164 bash[43424]: debug 2023-06-03T20:39:19.019+0000 7f5bbd1c4700 -1 osd.7 324 heartbeat_check: no reply from 172.21.15.164:6804 osd.4 since back 2023-06-03T20:38:49.851095+0000 front 2023-06-03T20:38:49.851066+0000 (oldest deadline 2023-06-03T20:39:15.750731+0000) 2023-06-03T20:39:20.335 INFO:journalctl@ceph.osd.7.smithi164.stdout:Jun 03 20:39:20 smithi164 bash[43424]: debug 2023-06-03T20:39:20.003+0000 7f5bbd1c4700 -1 osd.7 324 heartbeat_check: no reply from 172.21.15.89:6806 osd.0 since back 2023-06-03T20:38:13.843087+0000 front 2023-06-03T20:38:13.842982+0000 (oldest deadline 2023-06-03T20:38:37.942624+0000) 2023-06-03T20:39:20.335 INFO:journalctl@ceph.osd.7.smithi164.stdout:Jun 03 20:39:20 smithi164 bash[43424]: debug 2023-06-03T20:39:20.003+0000 7f5bbd1c4700 -1 osd.7 324 heartbeat_check: no reply from 172.21.15.89:6814 osd.1 since back 2023-06-03T20:38:23.145022+0000 front 2023-06-03T20:38:23.144779+0000 (oldest deadline 2023-06-03T20:38:47.844517+0000) 2023-06-03T20:39:20.336 INFO:journalctl@ceph.osd.7.smithi164.stdout:Jun 03 20:39:20 smithi164 bash[43424]: debug 2023-06-03T20:39:20.003+0000 7f5bbd1c4700 -1 osd.7 324 heartbeat_check: no reply from 172.21.15.89:6822 osd.2 since back 2023-06-03T20:38:33.046905+0000 front 2023-06-03T20:38:33.047150+0000 (oldest deadline 2023-06-03T20:38:58.847539+0000) 2023-06-03T20:39:20.336 INFO:journalctl@ceph.osd.7.smithi164.stdout:Jun 03 20:39:20 smithi164 bash[43424]: debug 2023-06-03T20:39:20.003+0000 7f5bbd1c4700 -1 osd.7 324 heartbeat_check: no reply from 172.21.15.89:6830 osd.3 since back 2023-06-03T20:38:44.649782+0000 front 2023-06-03T20:38:44.649880+0000 (oldest deadline 2023-06-03T20:39:09.850095+0000) 2023-06-03T20:39:20.336 INFO:journalctl@ceph.osd.7.smithi164.stdout:Jun 03 20:39:20 smithi164 bash[43424]: debug 2023-06-03T20:39:20.003+0000 7f5bbd1c4700 -1 osd.7 324 heartbeat_check: no reply from 172.21.15.164:6804 osd.4 since back 2023-06-03T20:38:49.851095+0000 front 2023-06-03T20:38:49.851066+0000 (oldest deadline 2023-06-03T20:39:15.750731+0000) 2023-06-03T20:39:21.335 INFO:journalctl@ceph.osd.7.smithi164.stdout:Jun 03 20:39:21 smithi164 bash[43424]: debug 2023-06-03T20:39:21.019+0000 7f5bbd1c4700 -1 osd.7 324 heartbeat_check: no reply from 172.21.15.89:6806 osd.0 since back 2023-06-03T20:38:13.843087+0000 front 2023-06-03T20:38:13.842982+0000 (oldest deadline 2023-06-03T20:38:37.942624+0000) 2023-06-03T20:39:21.336 INFO:journalctl@ceph.osd.7.smithi164.stdout:Jun 03 20:39:21 smithi164 bash[43424]: debug 2023-06-03T20:39:21.019+0000 7f5bbd1c4700 -1 osd.7 324 heartbeat_check: no reply from 172.21.15.89:6814 osd.1 since back 2023-06-03T20:38:23.145022+0000 front 2023-06-03T20:38:23.144779+0000 (oldest deadline 2023-06-03T20:38:47.844517+0000) 2023-06-03T20:39:21.336 INFO:journalctl@ceph.osd.7.smithi164.stdout:Jun 03 20:39:21 smithi164 bash[43424]: debug 2023-06-03T20:39:21.019+0000 7f5bbd1c4700 -1 osd.7 324 heartbeat_check: no reply from 172.21.15.89:6822 osd.2 since back 2023-06-03T20:38:33.046905+0000 front 2023-06-03T20:38:33.047150+0000 (oldest deadline 2023-06-03T20:38:58.847539+0000) 2023-06-03T20:39:21.336 INFO:journalctl@ceph.osd.7.smithi164.stdout:Jun 03 20:39:21 smithi164 bash[43424]: debug 2023-06-03T20:39:21.019+0000 7f5bbd1c4700 -1 osd.7 324 heartbeat_check: no reply from 172.21.15.89:6830 osd.3 since back 2023-06-03T20:38:44.649782+0000 front 2023-06-03T20:38:44.649880+0000 (oldest deadline 2023-06-03T20:39:09.850095+0000) 2023-06-03T20:39:21.336 INFO:journalctl@ceph.osd.7.smithi164.stdout:Jun 03 20:39:21 smithi164 bash[43424]: debug 2023-06-03T20:39:21.019+0000 7f5bbd1c4700 -1 osd.7 324 heartbeat_check: no reply from 172.21.15.164:6804 osd.4 since back 2023-06-03T20:38:49.851095+0000 front 2023-06-03T20:38:49.851066+0000 (oldest deadline 2023-06-03T20:39:15.750731+0000) 2023-06-03T20:39:22.335 INFO:journalctl@ceph.osd.7.smithi164.stdout:Jun 03 20:39:22 smithi164 bash[43424]: debug 2023-06-03T20:39:22.027+0000 7f5bbd1c4700 -1 osd.7 324 heartbeat_check: no reply from 172.21.15.89:6806 osd.0 since back 2023-06-03T20:38:13.843087+0000 front 2023-06-03T20:38:13.842982+0000 (oldest deadline 2023-06-03T20:38:37.942624+0000) 2023-06-03T20:39:22.336 INFO:journalctl@ceph.osd.7.smithi164.stdout:Jun 03 20:39:22 smithi164 bash[43424]: debug 2023-06-03T20:39:22.027+0000 7f5bbd1c4700 -1 osd.7 324 heartbeat_check: no reply from 172.21.15.89:6814 osd.1 since back 2023-06-03T20:38:23.145022+0000 front 2023-06-03T20:38:23.144779+0000 (oldest deadline 2023-06-03T20:38:47.844517+0000) 2023-06-03T20:39:22.336 INFO:journalctl@ceph.osd.7.smithi164.stdout:Jun 03 20:39:22 smithi164 bash[43424]: debug 2023-06-03T20:39:22.027+0000 7f5bbd1c4700 -1 osd.7 324 heartbeat_check: no reply from 172.21.15.89:6822 osd.2 since back 2023-06-03T20:38:33.046905+0000 front 2023-06-03T20:38:33.047150+0000 (oldest deadline 2023-06-03T20:38:58.847539+0000) 2023-06-03T20:39:22.336 INFO:journalctl@ceph.osd.7.smithi164.stdout:Jun 03 20:39:22 smithi164 bash[43424]: debug 2023-06-03T20:39:22.027+0000 7f5bbd1c4700 -1 osd.7 324 heartbeat_check: no reply from 172.21.15.89:6830 osd.3 since back 2023-06-03T20:38:44.649782+0000 front 2023-06-03T20:38:44.649880+0000 (oldest deadline 2023-06-03T20:39:09.850095+0000) 2023-06-03T20:39:22.336 INFO:journalctl@ceph.osd.7.smithi164.stdout:Jun 03 20:39:22 smithi164 bash[43424]: debug 2023-06-03T20:39:22.027+0000 7f5bbd1c4700 -1 osd.7 324 heartbeat_check: no reply from 172.21.15.164:6804 osd.4 since back 2023-06-03T20:38:49.851095+0000 front 2023-06-03T20:38:49.851066+0000 (oldest deadline 2023-06-03T20:39:15.750731+0000) 2023-06-03T20:39:24.085 INFO:journalctl@ceph.osd.7.smithi164.stdout:Jun 03 20:39:23 smithi164 bash[164937]: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-osd-7 2023-06-03T20:39:25.909 DEBUG:teuthology.orchestra.run.smithi164:> sudo pkill -f 'journalctl -f -n 0 -u ceph-f1db3478-0243-11ee-9b1d-001a4aab830c@osd.7.service' 2023-06-03T20:39:25.938 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-06-03T20:39:25.938 INFO:tasks.cephadm.osd.7:Stopped osd.7 2023-06-03T20:39:25.939 INFO:tasks.cephadm.prometheus.a:Stopping prometheus.a... 2023-06-03T20:39:25.939 DEBUG:teuthology.orchestra.run.smithi164:> sudo systemctl stop ceph-f1db3478-0243-11ee-9b1d-001a4aab830c@prometheus.a 2023-06-03T20:39:26.190 INFO:journalctl@ceph.prometheus.a.smithi164.stdout:Jun 03 20:39:25 smithi164 systemd[1]: Stopping Ceph prometheus.a for f1db3478-0243-11ee-9b1d-001a4aab830c... 2023-06-03T20:39:26.585 INFO:journalctl@ceph.prometheus.a.smithi164.stdout:Jun 03 20:39:26 smithi164 bash[55188]: ts=2023-06-03T20:39:26.170Z caller=main.go:775 level=warn msg="Received SIGTERM, exiting gracefully..." 2023-06-03T20:39:26.586 INFO:journalctl@ceph.prometheus.a.smithi164.stdout:Jun 03 20:39:26 smithi164 bash[55188]: ts=2023-06-03T20:39:26.171Z caller=main.go:798 level=info msg="Stopping scrape discovery manager..." 2023-06-03T20:39:26.586 INFO:journalctl@ceph.prometheus.a.smithi164.stdout:Jun 03 20:39:26 smithi164 bash[55188]: ts=2023-06-03T20:39:26.171Z caller=main.go:812 level=info msg="Stopping notify discovery manager..." 2023-06-03T20:39:26.586 INFO:journalctl@ceph.prometheus.a.smithi164.stdout:Jun 03 20:39:26 smithi164 bash[55188]: ts=2023-06-03T20:39:26.171Z caller=main.go:834 level=info msg="Stopping scrape manager..." 2023-06-03T20:39:26.586 INFO:journalctl@ceph.prometheus.a.smithi164.stdout:Jun 03 20:39:26 smithi164 bash[55188]: ts=2023-06-03T20:39:26.171Z caller=main.go:794 level=info msg="Scrape discovery manager stopped" 2023-06-03T20:39:26.586 INFO:journalctl@ceph.prometheus.a.smithi164.stdout:Jun 03 20:39:26 smithi164 bash[55188]: ts=2023-06-03T20:39:26.171Z caller=main.go:808 level=info msg="Notify discovery manager stopped" 2023-06-03T20:39:26.587 INFO:journalctl@ceph.prometheus.a.smithi164.stdout:Jun 03 20:39:26 smithi164 bash[55188]: ts=2023-06-03T20:39:26.171Z caller=manager.go:945 level=info component="rule manager" msg="Stopping rule manager..." 2023-06-03T20:39:26.587 INFO:journalctl@ceph.prometheus.a.smithi164.stdout:Jun 03 20:39:26 smithi164 bash[55188]: ts=2023-06-03T20:39:26.171Z caller=manager.go:955 level=info component="rule manager" msg="Rule manager stopped" 2023-06-03T20:39:26.587 INFO:journalctl@ceph.prometheus.a.smithi164.stdout:Jun 03 20:39:26 smithi164 bash[55188]: ts=2023-06-03T20:39:26.178Z caller=main.go:828 level=info msg="Scrape manager stopped" 2023-06-03T20:39:26.587 INFO:journalctl@ceph.prometheus.a.smithi164.stdout:Jun 03 20:39:26 smithi164 bash[55188]: ts=2023-06-03T20:39:26.183Z caller=notifier.go:600 level=info component=notifier msg="Stopping notification manager..." 2023-06-03T20:39:26.587 INFO:journalctl@ceph.prometheus.a.smithi164.stdout:Jun 03 20:39:26 smithi164 bash[55188]: ts=2023-06-03T20:39:26.183Z caller=main.go:1054 level=info msg="Notifier manager stopped" 2023-06-03T20:39:26.588 INFO:journalctl@ceph.prometheus.a.smithi164.stdout:Jun 03 20:39:26 smithi164 bash[55188]: ts=2023-06-03T20:39:26.183Z caller=main.go:1066 level=info msg="See you next time!" 2023-06-03T20:39:27.133 DEBUG:teuthology.orchestra.run.smithi164:> sudo pkill -f 'journalctl -f -n 0 -u ceph-f1db3478-0243-11ee-9b1d-001a4aab830c@prometheus.a.service' 2023-06-03T20:39:27.162 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-06-03T20:39:27.163 INFO:tasks.cephadm.prometheus.a:Stopped prometheus.a 2023-06-03T20:39:27.163 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid f1db3478-0243-11ee-9b1d-001a4aab830c --force --keep-logs 2023-06-03T20:39:31.947 INFO:journalctl@ceph.alertmanager.a.smithi089.stdout:Jun 03 20:39:31 smithi089 bash[70028]: level=warn ts=2023-06-03T20:39:31.631Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi089.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.89:8443: connect: connection refused" 2023-06-03T20:39:31.947 INFO:journalctl@ceph.alertmanager.a.smithi089.stdout:Jun 03 20:39:31 smithi089 bash[70028]: level=warn ts=2023-06-03T20:39:31.632Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi164.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.164:8443: connect: connection refused" 2023-06-03T20:39:34.029 INFO:journalctl@ceph.node-exporter.a.smithi089.stdout:Jun 03 20:39:33 smithi089 systemd[1]: Stopping Ceph node-exporter.a for f1db3478-0243-11ee-9b1d-001a4aab830c... 2023-06-03T20:39:35.029 INFO:journalctl@ceph.node-exporter.a.smithi089.stdout:Jun 03 20:39:34 smithi089 bash[272134]: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-node-exporter-a 2023-06-03T20:39:35.030 INFO:journalctl@ceph.node-exporter.a.smithi089.stdout:Jun 03 20:39:34 smithi089 systemd[1]: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c@node-exporter.a.service: Main process exited, code=exited, status=143/n/a 2023-06-03T20:39:35.030 INFO:journalctl@ceph.node-exporter.a.smithi089.stdout:Jun 03 20:39:34 smithi089 systemd[1]: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c@node-exporter.a.service: Failed with result 'exit-code'. 2023-06-03T20:39:35.030 INFO:journalctl@ceph.node-exporter.a.smithi089.stdout:Jun 03 20:39:34 smithi089 systemd[1]: Stopped Ceph node-exporter.a for f1db3478-0243-11ee-9b1d-001a4aab830c. 2023-06-03T20:39:35.347 INFO:journalctl@ceph.alertmanager.a.smithi089.stdout:Jun 03 20:39:35 smithi089 systemd[1]: Stopping Ceph alertmanager.a for f1db3478-0243-11ee-9b1d-001a4aab830c... 2023-06-03T20:39:35.347 INFO:journalctl@ceph.alertmanager.a.smithi089.stdout:Jun 03 20:39:35 smithi089 bash[70028]: level=info ts=2023-06-03T20:39:35.257Z caller=main.go:557 msg="Received SIGTERM, exiting gracefully..." 2023-06-03T20:39:36.419 INFO:journalctl@ceph.alertmanager.a.smithi089.stdout:Jun 03 20:39:36 smithi089 bash[272264]: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-alertmanager-a 2023-06-03T20:39:36.420 INFO:journalctl@ceph.alertmanager.a.smithi089.stdout:Jun 03 20:39:36 smithi089 systemd[1]: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c@alertmanager.a.service: Succeeded. 2023-06-03T20:39:36.421 INFO:journalctl@ceph.alertmanager.a.smithi089.stdout:Jun 03 20:39:36 smithi089 systemd[1]: Stopped Ceph alertmanager.a for f1db3478-0243-11ee-9b1d-001a4aab830c. 2023-06-03T20:39:46.894 DEBUG:teuthology.orchestra.run.smithi164:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid f1db3478-0243-11ee-9b1d-001a4aab830c --force --keep-logs 2023-06-03T20:39:53.982 INFO:journalctl@ceph.node-exporter.b.smithi164.stdout:Jun 03 20:39:53 smithi164 systemd[1]: Stopping Ceph node-exporter.b for f1db3478-0243-11ee-9b1d-001a4aab830c... 2023-06-03T20:39:54.835 INFO:journalctl@ceph.node-exporter.b.smithi164.stdout:Jun 03 20:39:54 smithi164 bash[165493]: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-node-exporter-b 2023-06-03T20:39:54.836 INFO:journalctl@ceph.node-exporter.b.smithi164.stdout:Jun 03 20:39:54 smithi164 systemd[1]: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c@node-exporter.b.service: Main process exited, code=exited, status=143/n/a 2023-06-03T20:39:54.836 INFO:journalctl@ceph.node-exporter.b.smithi164.stdout:Jun 03 20:39:54 smithi164 systemd[1]: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c@node-exporter.b.service: Failed with result 'exit-code'. 2023-06-03T20:39:54.836 INFO:journalctl@ceph.node-exporter.b.smithi164.stdout:Jun 03 20:39:54 smithi164 systemd[1]: Stopped Ceph node-exporter.b for f1db3478-0243-11ee-9b1d-001a4aab830c. 2023-06-03T20:39:55.274 INFO:journalctl@ceph.grafana.a.smithi164.stdout:Jun 03 20:39:54 smithi164 systemd[1]: Stopping Ceph grafana.a for f1db3478-0243-11ee-9b1d-001a4aab830c... 2023-06-03T20:39:55.274 INFO:journalctl@ceph.grafana.a.smithi164.stdout:Jun 03 20:39:55 smithi164 bash[53862]: t=2023-06-03T20:39:55+0000 lvl=info msg="Shutdown started" logger=server reason="System signal: terminated" 2023-06-03T20:39:56.085 INFO:journalctl@ceph.grafana.a.smithi164.stdout:Jun 03 20:39:55 smithi164 bash[165623]: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c-grafana-a 2023-06-03T20:39:56.086 INFO:journalctl@ceph.grafana.a.smithi164.stdout:Jun 03 20:39:55 smithi164 systemd[1]: ceph-f1db3478-0243-11ee-9b1d-001a4aab830c@grafana.a.service: Succeeded. 2023-06-03T20:39:56.086 INFO:journalctl@ceph.grafana.a.smithi164.stdout:Jun 03 20:39:55 smithi164 systemd[1]: Stopped Ceph grafana.a for f1db3478-0243-11ee-9b1d-001a4aab830c. 2023-06-03T20:39:59.512 DEBUG:teuthology.orchestra.run.smithi089:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2023-06-03T20:39:59.528 DEBUG:teuthology.orchestra.run.smithi164:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2023-06-03T20:39:59.542 INFO:tasks.cephadm:Archiving crash dumps... 2023-06-03T20:39:59.545 DEBUG:teuthology.misc:Transferring archived files from smithi089:/var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/crash to /home/teuthworker/archive/teuthology-2023-06-03_14:23:09-upgrade:pacific-x-reef-distro-default-smithi/7295631/remote/smithi089/crash 2023-06-03T20:39:59.546 DEBUG:teuthology.orchestra.run.smithi089:> sudo tar c -f - -C /var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/crash -- . 2023-06-03T20:39:59.596 INFO:teuthology.orchestra.run.smithi089.stderr:tar: /var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/crash: Cannot open: No such file or directory 2023-06-03T20:39:59.596 INFO:teuthology.orchestra.run.smithi089.stderr:tar: Error is not recoverable: exiting now 2023-06-03T20:39:59.598 DEBUG:teuthology.misc:Transferring archived files from smithi164:/var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/crash to /home/teuthworker/archive/teuthology-2023-06-03_14:23:09-upgrade:pacific-x-reef-distro-default-smithi/7295631/remote/smithi164/crash 2023-06-03T20:39:59.599 DEBUG:teuthology.orchestra.run.smithi164:> sudo tar c -f - -C /var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/crash -- . 2023-06-03T20:39:59.634 INFO:teuthology.orchestra.run.smithi164.stderr:tar: /var/lib/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/crash: Cannot open: No such file or directory 2023-06-03T20:39:59.635 INFO:teuthology.orchestra.run.smithi164.stderr:tar: Error is not recoverable: exiting now 2023-06-03T20:39:59.636 INFO:tasks.cephadm:Checking cluster log for badness... 2023-06-03T20:39:59.636 DEBUG:teuthology.orchestra.run.smithi089:> sudo egrep '\[ERR\]|\[WRN\]|\[SEC\]' /var/log/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/ceph.log | egrep -v '\(MDS_ALL_DOWN\)' | egrep -v '\(MDS_UP_LESS_THAN_MAX\)' | head -n 1 2023-06-03T20:39:59.655 INFO:teuthology.orchestra.run.smithi089.stderr:grep: /var/log/ceph/f1db3478-0243-11ee-9b1d-001a4aab830c/ceph.log: No such file or directory 2023-06-03T20:39:59.657 INFO:tasks.cephadm:Compressing logs... 2023-06-03T20:39:59.658 DEBUG:teuthology.orchestra.run.smithi089:> sudo find /var/log/ceph /var/log/rbd-target-api -name '*.log' -print0 | sudo xargs -0 --no-run-if-empty -- gzip -- 2023-06-03T20:39:59.702 DEBUG:teuthology.orchestra.run.smithi164:> sudo find /var/log/ceph /var/log/rbd-target-api -name '*.log' -print0 | sudo xargs -0 --no-run-if-empty -- gzip -- 2023-06-03T20:39:59.715 INFO:teuthology.orchestra.run.smithi089.stderr:find: '/var/log/rbd-target-api': No such file or directory 2023-06-03T20:39:59.750 INFO:teuthology.orchestra.run.smithi164.stderr:find: '/var/log/rbd-target-api': No such file or directory 2023-06-03T20:50:53.942 INFO:tasks.cephadm:Archiving logs... 2023-06-03T20:50:53.962 DEBUG:teuthology.misc:Transferring archived files from smithi089:/var/log/ceph to /home/teuthworker/archive/teuthology-2023-06-03_14:23:09-upgrade:pacific-x-reef-distro-default-smithi/7295631/remote/smithi089/log 2023-06-03T20:50:53.965 DEBUG:teuthology.orchestra.run.smithi089:> sudo tar c -f - -C /var/log/ceph -- . 2023-06-03T20:51:56.521 DEBUG:teuthology.misc:Transferring archived files from smithi164:/var/log/ceph to /home/teuthworker/archive/teuthology-2023-06-03_14:23:09-upgrade:pacific-x-reef-distro-default-smithi/7295631/remote/smithi164/log 2023-06-03T20:51:56.523 DEBUG:teuthology.orchestra.run.smithi164:> sudo tar c -f - -C /var/log/ceph -- . 2023-06-03T20:52:48.420 INFO:tasks.cephadm:Removing cluster... 2023-06-03T20:52:48.420 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid f1db3478-0243-11ee-9b1d-001a4aab830c --force 2023-06-03T20:52:50.762 DEBUG:teuthology.orchestra.run.smithi164:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid f1db3478-0243-11ee-9b1d-001a4aab830c --force 2023-06-03T20:52:53.163 INFO:tasks.cephadm:Removing cephadm ... 2023-06-03T20:52:53.163 DEBUG:teuthology.orchestra.run.smithi089:> rm -rf /home/ubuntu/cephtest/cephadm 2023-06-03T20:52:53.186 DEBUG:teuthology.orchestra.run.smithi164:> rm -rf /home/ubuntu/cephtest/cephadm 2023-06-03T20:52:53.193 INFO:tasks.cephadm:Teardown complete 2023-06-03T20:52:53.194 DEBUG:teuthology.run_tasks:Unwinding manager install 2023-06-03T20:52:53.207 INFO:teuthology.task.install.util:Removing shipped files: /home/ubuntu/cephtest/valgrind.supp /usr/bin/daemon-helper /usr/bin/adjust-ulimits... 2023-06-03T20:52:53.207 DEBUG:teuthology.orchestra.run.smithi089:> sudo rm -f -- /home/ubuntu/cephtest/valgrind.supp /usr/bin/daemon-helper /usr/bin/adjust-ulimits 2023-06-03T20:52:53.229 DEBUG:teuthology.orchestra.run.smithi164:> sudo rm -f -- /home/ubuntu/cephtest/valgrind.supp /usr/bin/daemon-helper /usr/bin/adjust-ulimits 2023-06-03T20:52:53.317 DEBUG:teuthology.run_tasks:Unwinding manager clock 2023-06-03T20:52:53.327 INFO:teuthology.task.clock:Checking final clock skew... 2023-06-03T20:52:53.328 DEBUG:teuthology.orchestra.run.smithi089:> PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2023-06-03T20:52:53.331 DEBUG:teuthology.orchestra.run.smithi164:> PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2023-06-03T20:52:53.383 INFO:teuthology.orchestra.run.smithi089.stdout: remote refid st t when poll reach delay offset jitter 2023-06-03T20:52:53.384 INFO:teuthology.orchestra.run.smithi089.stdout:============================================================================== 2023-06-03T20:52:53.384 INFO:teuthology.orchestra.run.smithi089.stdout:*hv01.front.sepi 44.4.53.2 2 u 21 256 377 0.180 3.246 0.419 2023-06-03T20:52:53.384 INFO:teuthology.orchestra.run.smithi089.stdout:+hv02.front.sepi 63.231.80.2 3 u 250 256 377 0.185 1.014 0.160 2023-06-03T20:52:53.384 INFO:teuthology.orchestra.run.smithi089.stdout:+hv03.front.sepi 74.6.168.72 3 u 156 256 377 0.165 -2.316 0.589 2023-06-03T20:52:53.385 INFO:teuthology.orchestra.run.smithi089.stdout: hv04.front.sepi .INIT. 16 u - 1024 0 0.000 0.000 0.000 2023-06-03T20:52:53.393 INFO:teuthology.orchestra.run.smithi164.stdout: remote refid st t when poll reach delay offset jitter 2023-06-03T20:52:53.393 INFO:teuthology.orchestra.run.smithi164.stdout:============================================================================== 2023-06-03T20:52:53.393 INFO:teuthology.orchestra.run.smithi164.stdout:*hv01.front.sepi 44.4.53.2 2 u 230 256 377 0.139 7.097 0.861 2023-06-03T20:52:53.394 INFO:teuthology.orchestra.run.smithi164.stdout:+hv02.front.sepi 63.231.80.2 3 u 94 256 377 0.150 5.950 1.662 2023-06-03T20:52:53.394 INFO:teuthology.orchestra.run.smithi164.stdout:+hv03.front.sepi 74.6.168.72 3 u 156 256 377 0.173 -0.322 2.123 2023-06-03T20:52:53.394 INFO:teuthology.orchestra.run.smithi164.stdout: hv04.front.sepi .INIT. 16 u - 1024 0 0.000 0.000 0.000 2023-06-03T20:52:53.396 DEBUG:teuthology.run_tasks:Unwinding manager ansible.cephlab 2023-06-03T20:52:53.408 INFO:teuthology.task.ansible:Skipping ansible cleanup... 2023-06-03T20:52:53.409 DEBUG:teuthology.run_tasks:Unwinding manager selinux 2023-06-03T20:52:53.424 DEBUG:teuthology.run_tasks:Unwinding manager pcp 2023-06-03T20:52:53.439 DEBUG:teuthology.run_tasks:Unwinding manager internal.timer 2023-06-03T20:52:53.455 INFO:teuthology.task.internal:Duration was 5891.398887 seconds 2023-06-03T20:52:53.456 DEBUG:teuthology.run_tasks:Unwinding manager internal.syslog 2023-06-03T20:52:53.471 INFO:teuthology.task.internal.syslog:Shutting down syslog monitoring... 2023-06-03T20:52:53.471 DEBUG:teuthology.orchestra.run.smithi089:> sudo rm -f -- /etc/rsyslog.d/80-cephtest.conf && sudo service rsyslog restart 2023-06-03T20:52:53.474 DEBUG:teuthology.orchestra.run.smithi164:> sudo rm -f -- /etc/rsyslog.d/80-cephtest.conf && sudo service rsyslog restart 2023-06-03T20:52:53.720 INFO:teuthology.task.internal.syslog:Checking logs for errors... 2023-06-03T20:52:53.721 DEBUG:teuthology.task.internal.syslog:Checking ubuntu@smithi089.front.sepia.ceph.com 2023-06-03T20:52:53.721 DEBUG:teuthology.orchestra.run.smithi089:> egrep --binary-files=text '\bBUG\b|\bINFO\b|\bDEADLOCK\b' /home/ubuntu/cephtest/archive/syslog/kern.log | grep -v 'task .* blocked for more than .* seconds' | grep -v 'lockdep is turned off' | grep -v 'trying to register non-static key' | grep -v 'DEBUG: fsize' | grep -v CRON | grep -v 'BUG: bad unlock balance detected' | grep -v 'inconsistent lock state' | grep -v '*** DEADLOCK ***' | grep -v 'INFO: possible irq lock inversion dependency detected' | grep -v 'INFO: NMI handler (perf_event_nmi_handler) took too long to run' | grep -v 'INFO: recovery required on readonly' | grep -v 'ceph-create-keys: INFO' | grep -v INFO:ceph-create-keys | grep -v 'Loaded datasource DataSourceOpenStack' | grep -v 'container-storage-setup: INFO: Volume group backing root filesystem could not be determined' | egrep -v '\bsalt-master\b|\bsalt-minion\b|\bsalt-api\b' | grep -v ceph-crash | egrep -v '\btcmu-runner\b.*\bINFO\b' | head -n 1 2023-06-03T20:52:53.738 DEBUG:teuthology.task.internal.syslog:Checking ubuntu@smithi164.front.sepia.ceph.com 2023-06-03T20:52:53.739 DEBUG:teuthology.orchestra.run.smithi164:> egrep --binary-files=text '\bBUG\b|\bINFO\b|\bDEADLOCK\b' /home/ubuntu/cephtest/archive/syslog/kern.log | grep -v 'task .* blocked for more than .* seconds' | grep -v 'lockdep is turned off' | grep -v 'trying to register non-static key' | grep -v 'DEBUG: fsize' | grep -v CRON | grep -v 'BUG: bad unlock balance detected' | grep -v 'inconsistent lock state' | grep -v '*** DEADLOCK ***' | grep -v 'INFO: possible irq lock inversion dependency detected' | grep -v 'INFO: NMI handler (perf_event_nmi_handler) took too long to run' | grep -v 'INFO: recovery required on readonly' | grep -v 'ceph-create-keys: INFO' | grep -v INFO:ceph-create-keys | grep -v 'Loaded datasource DataSourceOpenStack' | grep -v 'container-storage-setup: INFO: Volume group backing root filesystem could not be determined' | egrep -v '\bsalt-master\b|\bsalt-minion\b|\bsalt-api\b' | grep -v ceph-crash | egrep -v '\btcmu-runner\b.*\bINFO\b' | head -n 1 2023-06-03T20:52:53.754 INFO:teuthology.task.internal.syslog:Compressing syslogs... 2023-06-03T20:52:53.755 DEBUG:teuthology.orchestra.run.smithi089:> find /home/ubuntu/cephtest/archive/syslog -name '*.log' -print0 | sudo xargs -0 --no-run-if-empty -- gzip -- 2023-06-03T20:52:53.781 DEBUG:teuthology.orchestra.run.smithi164:> find /home/ubuntu/cephtest/archive/syslog -name '*.log' -print0 | sudo xargs -0 --no-run-if-empty -- gzip -- 2023-06-03T20:52:53.967 INFO:teuthology.task.internal.syslog:Gathering journactl -b0... 2023-06-03T20:52:53.968 DEBUG:teuthology.orchestra.run.smithi089:> sudo journalctl -b0 | gzip -9 > /home/ubuntu/cephtest/archive/syslog/journalctl-b0.gz 2023-06-03T20:52:53.970 DEBUG:teuthology.orchestra.run.smithi164:> sudo journalctl -b0 | gzip -9 > /home/ubuntu/cephtest/archive/syslog/journalctl-b0.gz 2023-06-03T20:52:54.536 DEBUG:teuthology.run_tasks:Unwinding manager internal.sudo 2023-06-03T20:52:54.547 INFO:teuthology.task.internal:Restoring /etc/sudoers... 2023-06-03T20:52:54.547 DEBUG:teuthology.orchestra.run.smithi089:> sudo mv -f /etc/sudoers.orig.teuthology /etc/sudoers 2023-06-03T20:52:54.585 DEBUG:teuthology.orchestra.run.smithi164:> sudo mv -f /etc/sudoers.orig.teuthology /etc/sudoers 2023-06-03T20:52:54.636 DEBUG:teuthology.run_tasks:Unwinding manager internal.coredump 2023-06-03T20:52:54.648 DEBUG:teuthology.orchestra.run.smithi089:> sudo sysctl -w kernel.core_pattern=core && sudo bash -c 'for f in `find /home/ubuntu/cephtest/archive/coredump -type f`; do file $f | grep -q systemd-sysusers && rm $f || true ; done' && rmdir --ignore-fail-on-non-empty -- /home/ubuntu/cephtest/archive/coredump 2023-06-03T20:52:54.651 DEBUG:teuthology.orchestra.run.smithi164:> sudo sysctl -w kernel.core_pattern=core && sudo bash -c 'for f in `find /home/ubuntu/cephtest/archive/coredump -type f`; do file $f | grep -q systemd-sysusers && rm $f || true ; done' && rmdir --ignore-fail-on-non-empty -- /home/ubuntu/cephtest/archive/coredump 2023-06-03T20:52:54.675 INFO:teuthology.orchestra.run.smithi089.stdout:kernel.core_pattern = core 2023-06-03T20:52:54.677 INFO:teuthology.orchestra.run.smithi164.stdout:kernel.core_pattern = core 2023-06-03T20:52:54.723 DEBUG:teuthology.orchestra.run.smithi089:> test -e /home/ubuntu/cephtest/archive/coredump 2023-06-03T20:52:54.769 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:52:54.770 DEBUG:teuthology.orchestra.run.smithi164:> test -e /home/ubuntu/cephtest/archive/coredump 2023-06-03T20:52:54.776 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:52:54.777 DEBUG:teuthology.run_tasks:Unwinding manager internal.archive 2023-06-03T20:52:54.791 INFO:teuthology.task.internal:Transferring archived files... 2023-06-03T20:52:54.792 DEBUG:teuthology.misc:Transferring archived files from smithi089:/home/ubuntu/cephtest/archive to /home/teuthworker/archive/teuthology-2023-06-03_14:23:09-upgrade:pacific-x-reef-distro-default-smithi/7295631/remote/smithi089 2023-06-03T20:52:54.792 DEBUG:teuthology.orchestra.run.smithi089:> sudo tar c -f - -C /home/ubuntu/cephtest/archive -- . 2023-06-03T20:52:54.852 DEBUG:teuthology.misc:Transferring archived files from smithi164:/home/ubuntu/cephtest/archive to /home/teuthworker/archive/teuthology-2023-06-03_14:23:09-upgrade:pacific-x-reef-distro-default-smithi/7295631/remote/smithi164 2023-06-03T20:52:54.853 DEBUG:teuthology.orchestra.run.smithi164:> sudo tar c -f - -C /home/ubuntu/cephtest/archive -- . 2023-06-03T20:52:54.884 INFO:teuthology.task.internal:Removing archive directory... 2023-06-03T20:52:54.885 DEBUG:teuthology.orchestra.run.smithi089:> rm -rf -- /home/ubuntu/cephtest/archive 2023-06-03T20:52:54.889 DEBUG:teuthology.orchestra.run.smithi164:> rm -rf -- /home/ubuntu/cephtest/archive 2023-06-03T20:52:54.929 DEBUG:teuthology.run_tasks:Unwinding manager internal.archive_upload 2023-06-03T20:52:54.944 INFO:teuthology.task.internal:Not uploading archives. 2023-06-03T20:52:54.944 DEBUG:teuthology.run_tasks:Unwinding manager internal.base 2023-06-03T20:52:54.960 INFO:teuthology.task.internal:Tidying up after the test... 2023-06-03T20:52:54.961 DEBUG:teuthology.orchestra.run.smithi089:> find /home/ubuntu/cephtest -ls ; rmdir -- /home/ubuntu/cephtest 2023-06-03T20:52:54.963 DEBUG:teuthology.orchestra.run.smithi164:> find /home/ubuntu/cephtest -ls ; rmdir -- /home/ubuntu/cephtest 2023-06-03T20:52:54.969 INFO:teuthology.orchestra.run.smithi089.stdout: 1048601 4 drwxr-xr-x 3 ubuntu ubuntu 4096 Jun 3 20:52 /home/ubuntu/cephtest 2023-06-03T20:52:54.970 INFO:teuthology.orchestra.run.smithi089.stdout: 1194450 4 drwxrwxr-x 3 ubuntu ubuntu 4096 Jun 3 20:32 /home/ubuntu/cephtest/mnt.0 2023-06-03T20:52:54.970 INFO:teuthology.orchestra.run.smithi089.stdout: 1194462 4 drwxrwxr-x 3 ubuntu ubuntu 4096 Jun 3 20:32 /home/ubuntu/cephtest/mnt.0/client.0 2023-06-03T20:52:54.970 INFO:teuthology.orchestra.run.smithi089.stdout: 1966087 4 drwxrwxr-x 2 ubuntu ubuntu 4096 Jun 3 20:32 /home/ubuntu/cephtest/mnt.0/client.0/tmp 2023-06-03T20:52:54.971 INFO:teuthology.orchestra.run.smithi089.stderr:rmdir: failed to remove '/home/ubuntu/cephtest': Directory not empty 2023-06-03T20:52:54.972 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-06-03T20:52:54.972 ERROR:teuthology.run_tasks:Manager failed: internal.base Traceback (most recent call last): File "/home/teuthworker/src/git.ceph.com_teuthology_62814aba234085f597115b9ba9bdc56af117de9e/teuthology/run_tasks.py", line 192, in run_tasks suppress = manager.__exit__(*exc_info) File "/usr/lib/python3.8/contextlib.py", line 120, in __exit__ next(self.gen) File "/home/teuthworker/src/git.ceph.com_teuthology_62814aba234085f597115b9ba9bdc56af117de9e/teuthology/task/internal/__init__.py", line 53, in base run.wait( File "/home/teuthworker/src/git.ceph.com_teuthology_62814aba234085f597115b9ba9bdc56af117de9e/teuthology/orchestra/run.py", line 479, in wait proc.wait() File "/home/teuthworker/src/git.ceph.com_teuthology_62814aba234085f597115b9ba9bdc56af117de9e/teuthology/orchestra/run.py", line 161, in wait self._raise_for_status() File "/home/teuthworker/src/git.ceph.com_teuthology_62814aba234085f597115b9ba9bdc56af117de9e/teuthology/orchestra/run.py", line 181, in _raise_for_status raise CommandFailedError( teuthology.exceptions.CommandFailedError: Command failed on smithi089 with status 1: 'find /home/ubuntu/cephtest -ls ; rmdir -- /home/ubuntu/cephtest' 2023-06-03T20:52:54.978 DEBUG:teuthology.run_tasks:Unwinding manager kernel 2023-06-03T20:52:54.992 DEBUG:teuthology.run_tasks:Unwinding manager console_log 2023-06-03T20:52:55.009 DEBUG:teuthology.run_tasks:Exception was not quenched, exiting: CommandFailedError: Command failed on smithi089 with status 1: 'find /home/ubuntu/cephtest -ls ; rmdir -- /home/ubuntu/cephtest' 2023-06-03T20:52:55.009 INFO:teuthology.nuke:Checking targets against current locks 2023-06-03T20:52:55.016 INFO:teuthology.orchestra.run.smithi164.stdout: 1048602 4 drwxr-xr-x 2 ubuntu ubuntu 4096 Jun 3 20:52 /home/ubuntu/cephtest 2023-06-03T20:52:55.053 DEBUG:teuthology.nuke:shortname: smithi089 2023-06-03T20:52:55.054 INFO:teuthology.task.internal.check_lock:Checking locks... 2023-06-03T20:52:55.075 DEBUG:teuthology.nuke:shortname: smithi164 2023-06-03T20:52:55.076 INFO:teuthology.task.internal.check_lock:Checking locks... 2023-06-03T20:52:55.079 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi089.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/teuthology-2023-06-03_14:23:09-upgrade:pacific-x-reef-distro-default-smithi/7295631', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'ubuntu', 'os_version': '20.04', 'arch': 'x86_64', 'locked': True, 'locked_since': '2023-06-03 19:04:51.482033', 'locked_by': 'scheduled_teuthology@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBFyT/r7xuKfmg50w0OdQebr4A/ptJo1Zk6DyZNZNZM/q5rW+sznh+wvHqeT3e4YtCdZLaXyvMms8o8u8H5nXOYM='} 2023-06-03T20:52:55.098 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi164.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/teuthology-2023-06-03_14:23:09-upgrade:pacific-x-reef-distro-default-smithi/7295631', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'ubuntu', 'os_version': '20.04', 'arch': 'x86_64', 'locked': True, 'locked_since': '2023-06-03 19:04:51.483517', 'locked_by': 'scheduled_teuthology@teuthology', 'mac_address': None, 'ssh_pub_key': 'ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIGETOy3xONfbx7kUFCNiUtRLNfkcrcEsWzPdq6Np/6Bw'} 2023-06-03T20:52:55.133 INFO:teuthology.orchestra.console:Power off smithi089 2023-06-03T20:52:55.135 DEBUG:teuthology.orchestra.console:pexpect command: ipmitool -H smithi089.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power off 2023-06-03T20:52:55.159 DEBUG:teuthology.orchestra.console:pexpect command: ipmitool -H smithi089.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2023-06-03T20:52:55.178 INFO:teuthology.orchestra.console:Power off smithi164 2023-06-03T20:52:55.179 DEBUG:teuthology.orchestra.console:pexpect command: ipmitool -H smithi164.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power off 2023-06-03T20:52:55.204 DEBUG:teuthology.orchestra.console:pexpect command: ipmitool -H smithi164.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2023-06-03T20:52:59.193 DEBUG:teuthology.orchestra.console:pexpect command: ipmitool -H smithi089.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2023-06-03T20:52:59.230 DEBUG:teuthology.orchestra.console:pexpect command: ipmitool -H smithi164.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2023-06-03T20:53:03.308 DEBUG:teuthology.orchestra.console:pexpect command: ipmitool -H smithi089.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2023-06-03T20:53:03.344 DEBUG:teuthology.orchestra.console:pexpect command: ipmitool -H smithi164.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2023-06-03T20:53:03.521 INFO:teuthology.orchestra.console:Power off for smithi089 completed 2023-06-03T20:53:07.459 DEBUG:teuthology.orchestra.console:pexpect command: ipmitool -H smithi164.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2023-06-03T20:53:07.671 INFO:teuthology.orchestra.console:Power off for smithi164 completed 2023-06-03T20:53:07.777 INFO:teuthology.run:Summary data: description: upgrade:pacific-x/stress-split/{0-distro/ubuntu_20.04 0-roles 1-start 2-first-half-tasks/radosbench 3-stress-tasks/{radosbench rbd-cls rbd-import-export rbd_api readwrite snaps-few-objects} 4-second-half-tasks/rbd-import-export mon_election/classic} duration: 5891.398886919022 failure_reason: 'Command failed (workunit test cls/test_cls_rbd.sh) on smithi089 with status 1: ''mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=pacific TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 CEPH_MNT=/home/ubuntu/cephtest/mnt.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/cls/test_cls_rbd.sh''' flavor: default owner: scheduled_teuthology@teuthology sentry_event: https://sentry.ceph.com/organizations/ceph/?query=0b7d7cf0c2b848d082d6a003769122dc status: fail success: false 2023-06-03T20:53:07.778 DEBUG:teuthology.report:Pushing job info to https://paddles.front.sepia.ceph.com/ 2023-06-03T20:53:07.862 INFO:teuthology.run:FAIL